diff --git a/.build/build-rat.xml b/.build/build-rat.xml
index 27e8f63ae43d..ea87fa5a2902 100644
--- a/.build/build-rat.xml
+++ b/.build/build-rat.xml
@@ -49,12 +49,14 @@
                  <exclude name="pylib/cqlshlib/test/config/sslhandling*.config"/>
                  <exclude NAME="src/resources/org/apache/cassandra/cql3/reserved_keywords.txt"/>
                  <exclude NAME="src/resources/org/apache/cassandra/index/sasi/analyzer/filter/*.txt"/>
+                 <exclude name="test/conf/bigtable.yaml"/>
                  <exclude name="test/conf/cdc.yaml"/>
                  <exclude name="test/conf/commitlog*.yaml"/>
                  <exclude name="test/conf/system_keyspaces_directory.yaml"/>
                  <exclude name="test/conf/sstableloader_with_encryption.yaml"/>
                  <exclude name="test/conf/unit-test-conf/test-native-port.yaml"/>
                  <exclude name="test/resources/data/config/YamlConfigurationLoaderTest/*.yaml"/>
+                 <exclude name="test/resources/mockito-extensions/org.mockito.plugins.MockMaker"/>
                  <exclude name="tools/cqlstress-*.yaml"/>
                  <!-- test data -->
                  <exclude name="pylib/cqlshlib/test/test_authproviderhandling_config/*"/>
@@ -69,6 +71,7 @@
                  <exclude name="test/data/**/*.sha1"/>
                  <exclude name="test/data/CASSANDRA-15313/lz4-jvm-crash-failure.txt"/>
                  <exclude name="test/data/jmxdump/cassandra-*-jmx.yaml"/>
+                 <exclude name=".github/**"/>
                  <!-- Documentation files -->
                  <exclude name=".github/pull_request_template.md"/>
                  <exclude NAME="doc/modules/**/*"/>
@@ -77,6 +80,8 @@
                  <exclude NAME="CHANGES.txt"/>
                  <exclude NAME="CASSANDRA-14092.txt"/>
                  <exclude NAME="debian/TODO"/>
+                 <exclude NAME="doc/unified_compaction_level_formula.svg"/>
+                 <exclude NAME="src/java/org/apache/cassandra/db/compaction/unified/*.svg"/>
                  <!-- legal files -->
                  <exclude NAME="NOTICE.txt"/>
                  <exclude NAME="LICENSE.txt"/>
@@ -91,6 +96,9 @@
                  <exclude NAME="debian/control"/>
                  <exclude NAME="debian/dirs"/>
                  <exclude NAME="debian/patches/series"/>
+                 <exclude NAME="update-history/**"/>
+                 <exclude NAME="tools/analytics/plot_adaptive.gnu"/>
+                 <exclude NAME="tools/analytics/plot_static.gnu"/>
             </fileset>
         </rat:report>
         <exec executable="grep" outputproperty="rat.failed.files" failifexecutionfails="false">
diff --git a/.build/build-resolver.xml b/.build/build-resolver.xml
index 55718377e244..a962024a3f69 100644
--- a/.build/build-resolver.xml
+++ b/.build/build-resolver.xml
@@ -53,11 +53,16 @@
     </target>
 
     <target name="resolver-init" depends="init,_resolver_download" unless="resolver-ant-tasks.initialized" description="Initialize Resolver ANT Tasks">
+        <delete>
+            <fileset dir="${build.dir.lib}" includes="**/*"/>
+            <fileset dir="${test.lib}" includes="**/*"/>
+        </delete>
 
         <typedef uri="antlib:org.apache.maven.resolver.ant" resource="org/apache/maven/resolver/ant/antlib.xml" classpathref="resolver-ant-tasks.classpath" />
         <resolver:remoterepos id="all">
             <remoterepo id="resolver-central" url="${artifact.remoteRepository.central}"/>
             <remoterepo id="resolver-apache" url="${artifact.remoteRepository.apache}"/>
+            <remoterepo id="resolver-datastax" url="${artifact.remoteRepository.datastax}"/>
             <!-- Snapshot artifacts must not exist in nor be downloaded by any Cassandra release artifact.
                  Please validate that all artifacts included in parent-pom-template.xml are release
                  artifacts before committing.
@@ -244,9 +249,9 @@
 
         <copy todir="${build.lib}" quiet="true">
             <file file="${local.repository}/org/apache/cassandra/deps/futures-2.1.6-py2.py3-none-any.zip"/>
+            <file file="${local.repository}/org/apache/cassandra/deps/six-1.12.0-py2.py3-none-any.zip"/>
             <file file="${local.repository}/org/apache/cassandra/deps/geomet-0.1.0.zip"/>
-            <file file="${local.repository}/org/apache/cassandra/deps/pure_sasl-0.6.2-py2-none-any.zip"/>
-            <file file="${local.repository}/org/apache/cassandra/deps/wcwidth-0.2.5-py2.py3-none-any.zip"/>
+            <file file="${local.repository}/org/apache/cassandra/deps/wcwidth-0.2.6-py2.py3-none-any.zip"/>
         </copy>
         <copy todir="${build.lib}/sigar-bin/" quiet="true">
             <file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/libsigar-amd64-freebsd-6.so"/>
@@ -269,21 +274,25 @@
             <file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/libsigar-x86-freebsd-6.so"/>
             <file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/libsigar-x86-linux.so"/>
             <file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/libsigar-x86-solaris.so"/>
+            <file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/sigar-amd64-winnt.dll"/>
+            <file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/sigar-x86-winnt.dll"/>
+            <file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/sigar-x86-winnt.lib"/>
         </copy>
 
         <!-- as resolver will copy all dependencies into lib dir, and we are copying jars to lib/{x86_64|aarch64} as well, we would have duplicities -->
         <delete file="${build.lib}/AmazonCorrettoCryptoProvider-2.2.0-linux-x86_64.jar" failonerror="false"/>
         <delete file="${build.lib}/AmazonCorrettoCryptoProvider-2.2.0-linux-aarch_64.jar" failonerror="false"/>
 
-        <delete file="${build.lib}/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar" failonerror="false"/>
-        <delete file="${build.dir.lib}/jars/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar" failonerror="false"/>
+        <delete file="${build.lib}/netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar" failonerror="false"/>
+        <delete file="${build.dir.lib}/jars/netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar" failonerror="false"/>
     </target>
 
     <target name="_resolver-dist-lib_get_files">
         <!-- files.pythonhosted.org/packages -->
         <get src="${artifact.python.pypi}/59/a0/cf4cd997e1750f0c2d91c6ea5abea218251c43c3581bcc2f118b00baf5cf/futures-2.1.6-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/futures-2.1.6-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
         <get src="${artifact.python.pypi}/37/b2/ef1124540ee2c0b417be8d0f74667957e6aa084a3f26621aa67e2e77f3fb/pure_sasl-0.6.2-py2-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/pure_sasl-0.6.2-py2-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
-        <get src="${artifact.python.pypi}/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/wcwidth-0.2.5-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
+        <get src="${artifact.python.pypi}/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/six-1.12.0-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
+        <get src="${artifact.python.pypi}/20/f4/c0584a25144ce20bfcf1aecd041768b8c762c1eb0aa77502a3f0baa83f11/wcwidth-0.2.6-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/wcwidth-0.2.6-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
 
         <!-- apache/cassandra/lib -->
         <get src="${lib.download.base.url}/lib/geomet-0.1.0.zip" dest="${local.repository}/org/apache/cassandra/deps/geomet-0.1.0.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
@@ -308,6 +317,9 @@
             <url url="${lib.download.base.url}/lib/sigar-bin/libsigar-x86-freebsd-6.so"/>
             <url url="${lib.download.base.url}/lib/sigar-bin/libsigar-x86-linux.so"/>
             <url url="${lib.download.base.url}/lib/sigar-bin/libsigar-x86-solaris.so"/>
+            <url url="${lib.download.base.url}/lib/sigar-bin/sigar-amd64-winnt.dll"/>
+            <url url="${lib.download.base.url}/lib/sigar-bin/sigar-x86-winnt.dll"/>
+            <url url="${lib.download.base.url}/lib/sigar-bin/sigar-x86-winnt.lib"/>
         </get>
     </target>
 
diff --git a/.build/cassandra-build-deps-template.xml b/.build/cassandra-build-deps-template.xml
index cc1a25a8c1fc..f12be6b7c050 100644
--- a/.build/cassandra-build-deps-template.xml
+++ b/.build/cassandra-build-deps-template.xml
@@ -17,8 +17,8 @@
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <modelVersion>4.0.0</modelVersion>
   <parent>
-    <artifactId>cassandra-parent</artifactId>
-    <groupId>org.apache.cassandra</groupId>
+    <artifactId>dse-db-parent</artifactId>
+    <groupId>com.datastax.dse</groupId>
     <version>@version@</version>
     <relativePath>@final.name@-parent.pom</relativePath>
   </parent>
@@ -155,5 +155,9 @@
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcutil-jdk18on</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.bpodgursky</groupId>
+      <artifactId>jbool_expressions</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/.build/cassandra-deps-template.xml b/.build/cassandra-deps-template.xml
index ab98e36ab85f..8e5194e83d6b 100644
--- a/.build/cassandra-deps-template.xml
+++ b/.build/cassandra-deps-template.xml
@@ -17,12 +17,12 @@
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <modelVersion>4.0.0</modelVersion>
   <parent>
-    <groupId>org.apache.cassandra</groupId>
-    <artifactId>cassandra-parent</artifactId>
+    <groupId>com.datastax.dse</groupId>
+    <artifactId>dse-db-parent</artifactId>
     <version>@version@</version>
     <relativePath>@final.name@-parent.pom</relativePath>
   </parent>
-  <artifactId>cassandra-all</artifactId>
+  <artifactId>dse-db-all</artifactId>
   <version>@version@</version>
   <name>Apache Cassandra</name>
   <description>The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.</description>
@@ -35,9 +35,9 @@
     </license>
   </licenses>
   <scm>
-    <connection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</connection>
-    <developerConnection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</developerConnection>
-    <url>https://gitbox.apache.org/repos/asf?p=cassandra.git</url>
+    <connection>scm:git:ssh://git@github.com:datastax/cassandra.git</connection>
+    <developerConnection>scm:git:ssh://git@github.com:datastax/cassandra.git</developerConnection>
+    <url>scm:git:ssh://git@github.com:datastax/cassandra.git</url>
   </scm>
   <dependencies>
     <dependency>
@@ -104,6 +104,10 @@
       <groupId>com.fasterxml.jackson.datatype</groupId>
       <artifactId>jackson-datatype-jsr310</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.googlecode.json-simple</groupId>
+      <artifactId>json-simple</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.boundary</groupId>
       <artifactId>high-scale-lib</artifactId>
@@ -136,6 +140,10 @@
       <groupId>com.clearspring.analytics</groupId>
       <artifactId>stream</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.esri.geometry</groupId>
+      <artifactId>esri-geometry-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>ch.qos.logback</groupId>
       <artifactId>logback-core</artifactId>
@@ -201,6 +209,10 @@
       <groupId>net.openhft</groupId>
       <artifactId>chronicle-threads</artifactId>
     </dependency>
+    <dependency>
+      <groupId>net.openhft</groupId>
+      <artifactId>chronicle-map</artifactId>
+    </dependency>
     <dependency>
       <!-- transitive to chronicle-core -->
       <groupId>net.openhft</groupId>
@@ -368,6 +380,10 @@
       <groupId>org.apache.lucene</groupId>
       <artifactId>lucene-analysis-common</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-backward-codecs</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.github.jbellis</groupId>
       <artifactId>jvector</artifactId>
@@ -376,5 +392,17 @@
       <groupId>com.vdurmont</groupId>
       <artifactId>semver4j</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.micrometer</groupId>
+      <artifactId>micrometer-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.latencyutils</groupId>
+      <artifactId>LatencyUtils</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>de.huxhorn.sulky</groupId>
+      <artifactId>de.huxhorn.sulky.ulid</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/.build/parent-pom-template.xml b/.build/parent-pom-template.xml
index a3f507706c02..3b4d3d67781f 100644
--- a/.build/parent-pom-template.xml
+++ b/.build/parent-pom-template.xml
@@ -21,13 +21,13 @@
     <groupId>org.apache</groupId>
     <version>22</version>
   </parent>
-  <groupId>org.apache.cassandra</groupId>
-  <artifactId>cassandra-parent</artifactId>
+  <groupId>com.datastax.dse</groupId>
+  <artifactId>dse-db-parent</artifactId>
   <version>@version@</version>
   <packaging>pom</packaging>
-  <name>Apache Cassandra</name>
-  <description>The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.</description>
-  <url>https://cassandra.apache.org</url>
+  <name>Datastax DB</name>
+  <description>The Apache Cassandra Project develops a highly scalable second-generation distributed database. DataStax, Inc. provides additional improvements on top of Apache Cassandra</description>
+  <url>https://datastax.com</url>
   <inceptionYear>2009</inceptionYear>
   <licenses>
     <license>
@@ -36,8 +36,8 @@
     </license>
   </licenses>
   <properties>
-    <bytebuddy.version>1.12.13</bytebuddy.version>
-    <byteman.version>4.0.20</byteman.version>
+    <bytebuddy.version>1.14.17</bytebuddy.version>
+    <byteman.version>4.0.23</byteman.version>
     <ohc.version>0.5.1</ohc.version>
 
     <!-- These are referenced in build.xml, so need to be propagated from there -->
@@ -239,9 +239,9 @@
     </developer>
   </developers>
   <scm>
-    <connection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</connection>
-    <developerConnection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</developerConnection>
-    <url>https://gitbox.apache.org/repos/asf?p=cassandra.git</url>
+    <connection>scm:git:ssh://git@github.com:datastax/cassandra.git</connection>
+    <developerConnection>scm:git:ssh://git@github.com:datastax/cassandra.git</developerConnection>
+    <url>scm:git:ssh://git@github.com:datastax/cassandra.git</url>
   </scm>
 
   <profiles>
@@ -291,7 +291,7 @@
       <dependency>
         <groupId>org.xerial.snappy</groupId>
         <artifactId>snappy-java</artifactId>
-        <version>1.1.10.4</version>
+        <version>1.1.10.7</version>
       </dependency>
       <dependency>
         <groupId>org.lz4</groupId>
@@ -306,7 +306,7 @@
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>32.0.1-jre</version>
+        <version>33.4.0-jre</version>
         <exclusions>
           <exclusion>
             <artifactId>jsr305</artifactId>
@@ -397,52 +397,57 @@
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-api</artifactId>
-        <version>1.7.36</version>
+        <version>2.0.9</version>
       </dependency>
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>log4j-over-slf4j</artifactId>
-        <version>1.7.36</version>
+        <version>2.0.9</version>
       </dependency>
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>jcl-over-slf4j</artifactId>
-        <version>1.7.36</version>
+        <version>2.0.9</version>
       </dependency>
       <dependency>
         <groupId>ch.qos.logback</groupId>
         <artifactId>logback-core</artifactId>
-        <version>1.2.12</version>
+        <version>1.4.14</version>
       </dependency>
       <dependency>
         <groupId>ch.qos.logback</groupId>
         <artifactId>logback-classic</artifactId>
-        <version>1.2.12</version>
+        <version>1.4.14</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-core</artifactId>
-        <version>2.13.2</version>
+        <version>2.18.3</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-databind</artifactId>
-        <version>2.13.2.2</version>
+        <version>2.18.3</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-annotations</artifactId>
-        <version>2.13.2</version>
+        <version>2.18.3</version>
+      </dependency>
+      <dependency>
+        <groupId>com.googlecode.json-simple</groupId>
+        <artifactId>json-simple</artifactId>
+        <version>1.1</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.datatype</groupId>
         <artifactId>jackson-datatype-jsr310</artifactId>
-        <version>2.13.2</version>
+        <version>2.18.3</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.dataformat</groupId>
         <artifactId>jackson-dataformat-yaml</artifactId>
-        <version>2.13.2</version>
+        <version>2.18.3</version>
         <scope>test</scope>
         <exclusions>
           <exclusion>
@@ -464,12 +469,12 @@
       <dependency>
         <groupId>org.yaml</groupId>
         <artifactId>snakeyaml</artifactId>
-        <version>1.26</version>
+        <version>2.4</version>
       </dependency>
       <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>
-        <version>4.12</version>
+        <version>4.13</version>
         <scope>test</scope>
         <exclusions>
           <exclusion>
@@ -729,7 +734,7 @@
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-all</artifactId>
-        <version>4.1.96.Final</version>
+        <version>4.1.118.Final</version>
         <exclusions>
           <exclusion>
             <groupId>io.netty</groupId>
@@ -800,7 +805,7 @@
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-tcnative-boringssl-static</artifactId>
-        <version>2.0.61.Final</version>
+        <version>2.0.69.Final</version>
       </dependency>
       <dependency>
         <groupId>org.bouncycastle</groupId>
@@ -823,18 +828,18 @@
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-transport-native-epoll</artifactId>
-        <version>4.1.96.Final</version>
+        <version>4.1.118.Final</version>
       </dependency>
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-transport-native-epoll</artifactId>
-        <version>4.1.96.Final</version>
+        <version>4.1.118.Final</version>
         <classifier>linux-x86_64</classifier>
       </dependency>
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-transport-native-epoll</artifactId>
-        <version>4.1.96.Final</version>
+        <version>4.1.118.Final</version>
         <classifier>linux-aarch_64</classifier>
       </dependency>
 
@@ -842,7 +847,7 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-queue</artifactId>
-        <version>5.23.37</version>
+        <version>5.24ea27</version>
         <exclusions>
           <exclusion>
             <artifactId>tools</artifactId>
@@ -858,7 +863,7 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-core</artifactId>
-        <version>2.23.36</version>
+        <version>2.24ea28</version>
         <exclusions>
           <exclusion>
             <artifactId>chronicle-analytics</artifactId>
@@ -873,7 +878,7 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-bytes</artifactId>
-        <version>2.23.33</version>
+        <version>2.24ea20</version>
         <exclusions>
           <exclusion>
             <artifactId>annotations</artifactId>
@@ -884,7 +889,7 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-wire</artifactId>
-        <version>2.23.39</version>
+        <version>2.24ea27</version>
         <exclusions>
           <exclusion>
             <artifactId>compiler</artifactId>
@@ -900,7 +905,19 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-threads</artifactId>
-        <version>2.23.25</version>
+        <version>2.24ea14</version>
+        <exclusions>
+          <exclusion>
+              <!-- pulls in affinity-3.23ea1 which pulls in third-party-bom-3.22.4-SNAPSHOT -->
+            <groupId>net.openhft</groupId>
+            <artifactId>affinity</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>net.openhft</groupId>
+        <artifactId>chronicle-map</artifactId>
+        <version>3.24ea4</version>
         <exclusions>
           <exclusion>
               <!-- pulls in affinity-3.23ea1 which pulls in third-party-bom-3.22.4-SNAPSHOT -->
@@ -926,7 +943,7 @@
       <dependency>
         <groupId>com.google.code.findbugs</groupId>
         <artifactId>jsr305</artifactId>
-        <version>2.0.2</version>
+        <version>3.0.0</version>
       </dependency>
       <dependency>
         <groupId>com.clearspring.analytics</groupId>
@@ -939,6 +956,11 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>com.esri.geometry</groupId>
+        <artifactId>esri-geometry-api</artifactId>
+        <version>2.2.4</version>
+      </dependency>
       <dependency>
         <groupId>com.datastax.cassandra</groupId>
         <artifactId>cassandra-driver-core</artifactId>
@@ -1206,22 +1228,27 @@
       <dependency>
         <groupId>org.agrona</groupId>
         <artifactId>agrona</artifactId>
-        <version>1.17.1</version>
+        <version>1.20.0</version>
       </dependency>
       <dependency>
         <groupId>org.apache.lucene</groupId>
         <artifactId>lucene-core</artifactId>
-        <version>9.7.0</version>
+        <version>9.8.0</version>
       </dependency>
       <dependency>
         <groupId>org.apache.lucene</groupId>
         <artifactId>lucene-analysis-common</artifactId>
-        <version>9.7.0</version>
+        <version>9.8.0</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.lucene</groupId>
+        <artifactId>lucene-backward-codecs</artifactId>
+        <version>9.8.0</version>
       </dependency>
       <dependency>
         <groupId>io.github.jbellis</groupId>
         <artifactId>jvector</artifactId>
-        <version>1.0.2</version>
+        <version>4.0.0-beta.4</version>
       </dependency>
       <dependency>
         <groupId>com.carrotsearch.randomizedtesting</groupId>
@@ -1245,6 +1272,27 @@
         <artifactId>semver4j</artifactId>
         <version>3.1.0</version>
       </dependency>
+      <dependency>
+        <groupId>com.bpodgursky</groupId>
+        <artifactId>jbool_expressions</artifactId>
+        <version>1.24</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>io.micrometer</groupId>
+        <artifactId>micrometer-core</artifactId>
+        <version>1.5.5</version>
+      </dependency>
+      <dependency>
+        <groupId>org.latencyutils</groupId>
+        <artifactId>LatencyUtils</artifactId>
+        <version>2.0.3</version>
+      </dependency>
+      <dependency>
+        <groupId>de.huxhorn.sulky</groupId>
+        <artifactId>de.huxhorn.sulky.ulid</artifactId>
+        <version>8.2.0</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 </project>
diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh
index 5b1307e1546d..13f95c689654 100755
--- a/.build/run-python-dtests.sh
+++ b/.build/run-python-dtests.sh
@@ -68,6 +68,7 @@ ALLOWED_DTEST_VARIANTS="novnode|large|latest|upgrade"
 [[ "${DTEST_TARGET}" =~ ^dtest(-(${ALLOWED_DTEST_VARIANTS}))*$ ]] || { echo >&2 "Unknown dtest target: ${DTEST_TARGET}. Allowed variants are ${ALLOWED_DTEST_VARIANTS}"; exit 1; }
 
 java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
+project_name=$(grep '<project\s*basedir=' build.xml |sed -ne 's/.*name=\"\([^"]*\)\".*/\1/p')
 version=$(grep 'property\s*name=\"base.version\"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
 java_version_default=`grep 'property\s*name="java.default"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'`
 
@@ -81,7 +82,7 @@ python_regx_supported_versions="^(3.8|3.9|3.10|3.11)$"
 [[ $python_version =~ $python_regx_supported_versions ]] || { echo "Python ${python_version} not supported."; exit 1; }
 
 # check project is already built. no cleaning is done, so jenkins unstash works, beware.
-[[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; }
+[[ -f "${DIST_DIR}/${project_name}-${version}.jar" ]] || [[ -f "${DIST_DIR}/${project_name}-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; }
 
 # check if dist artifacts exist, this breaks the dtests
 [[ -d "${DIST_DIR}/dist" ]] && { echo "dtests don't work when build/dist ("${DIST_DIR}/dist") exists (from \`ant artifacts\`)"; exit 1; }
diff --git a/.build/run-tests.sh b/.build/run-tests.sh
index 7529af933a56..02ef2e673108 100755
--- a/.build/run-tests.sh
+++ b/.build/run-tests.sh
@@ -165,6 +165,7 @@ _main() {
 
   # jdk check
   local -r java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
+  local -r project_name=$(grep '<project\s*basedir=' build.xml |sed -ne 's/.*name=\"\([^"]*\)\".*/\1/p')
   local -r version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
   local -r java_version_default=`grep 'property\s*name="java.default"' build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'`
 
@@ -174,7 +175,7 @@ _main() {
   fi
 
   # check project is already built. no cleaning is done, so jenkins unstash works, beware.
-  [[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR} | xargs)"; exit 1; }
+  [[ -f "${DIST_DIR}/${project_name}-${version}.jar" ]] || [[ -f "${DIST_DIR}/${project_name}-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR} | xargs)"; exit 1; }
 
   # check if dist artifacts exist, this breaks the dtests
   [[ -d "${DIST_DIR}/dist" ]] && { echo "tests don't work when build/dist ("${DIST_DIR}/dist") exists (from \`ant artifacts\`)"; exit 1; }
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index abcd2168a0b3..d9f4ed7f0dbe 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,25 +1,5 @@
-Thanks for sending a pull request! Here are some tips if you're new here:
- 
- * Ensure you have added or run the [appropriate tests](https://cassandra.apache.org/_/development/testing.html) for your PR.
- * Be sure to keep the PR description updated to reflect all changes.
- * Write your PR title to summarize what this PR proposes.
- * If possible, provide a concise example to reproduce the issue for a faster review.
- * Read our [contributor guidelines](https://cassandra.apache.org/_/development/index.html)
- * If you're making a documentation change, see our [guide to documentation contribution](https://cassandra.apache.org/_/development/documentation.html)
- 
-Commit messages should follow the following format:
-
-```
-<One sentence description, usually Jira title or CHANGES.txt summary>
-
-<Optional lengthier description (context on patch)>
-
-patch by <Authors>; reviewed by <Reviewers> for CASSANDRA-#####
-
-Co-authored-by: Name1 <email1>
-Co-authored-by: Name2 <email2>
-
-```
-
-The [Cassandra Jira](https://issues.apache.org/jira/projects/CASSANDRA/issues/)
+### What is the issue
+...
 
+### What does this PR fix and why was it fixed
+...
diff --git a/.github/workflows/checklist_comment_on_new_pr.yml b/.github/workflows/checklist_comment_on_new_pr.yml
new file mode 100644
index 000000000000..fda81de21fd2
--- /dev/null
+++ b/.github/workflows/checklist_comment_on_new_pr.yml
@@ -0,0 +1,18 @@
+name: Comment on new Pull Request with checklist
+on:
+  pull_request:
+    types: opened
+
+jobs:
+  checklist-comment:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - name: Add comment
+        run:
+          gh pr comment $PRNUM --body-file .github/workflows/pr_checklist.md
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          PRNUM: ${{ github.event.pull_request.number }}
diff --git a/.github/workflows/pr_checklist.md b/.github/workflows/pr_checklist.md
new file mode 100644
index 000000000000..4f942016c202
--- /dev/null
+++ b/.github/workflows/pr_checklist.md
@@ -0,0 +1,11 @@
+### Checklist before you submit for review
+- [ ] Make sure there is a PR in the CNDB project updating the Converged Cassandra version
+- [ ] Use `NoSpamLogger` for log lines that may appear frequently in the logs
+- [ ] Verify test results on Butler
+- [ ] Test coverage for new/modified code is > 80%
+- [ ] Proper code formatting
+- [ ] Proper title for each commit staring with the project-issue number, like CNDB-1234
+- [ ] Each commit has a meaningful description
+- [ ] Each commit is not very long and contains related changes
+- [ ] Renames, moves and reformatting are in distinct commits
+- [ ] All new files should contain the DataStax copyright header instead of the Apache License one
diff --git a/.gitignore b/.gitignore
index aa9e76c9323d..c5696389ec72 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,6 +56,12 @@ ide/nbproject/private
 nb-configuration.xml
 nbactions.xml
 
+# VScode
+.vscode/
+
+# Aider (aider.chat)
+.aider*
+
 # Maven, etc.
 out/
 target/
@@ -65,6 +71,7 @@ target/
 *.pyc
 *~
 *.bak
+*.log
 *.sw[o,p]
 *.tmp
 .DS_Store
diff --git a/CHANGES.txt b/CHANGES.txt
index a1129aafd527..d8cc44811491 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,12 @@
+Future version (tbd)
+ * Require only MODIFY permission on base when updating table with MV (STAR-564)
+
+
+Merged from 5.1:
+ * Expose current compaction throughput in nodetool (CASSANDRA-13890)
+Merged from 5.0:
+ * Improve error messages when initializing auth classes (CASSANDRA-20368 and CASSANDRA-20450)
+ * Use ParameterizedClass for all auth-related implementations (CASSANDRA-19946 and partially CASSANDRA-18554)
 5.0.2
  * Use SinglePartitionReadCommand for index queries that use strict filtering (CASSANDRA-19968)
  * Always write local expiration time as an int to LivenessInfo digest (CASSANDRA-19989)
@@ -173,13 +182,11 @@ Merged from 3.0:
 
 
 5.0-alpha2
- * Add support for vector search in SAI (CASSANDRA-18715)
  * Remove crc_check_chance from CompressionParams (CASSANDRA-18872)
  * Fix schema loading of UDTs inside vectors inside UDTs (CASSANDRA-18964)
  * Add cqlsh autocompletion for the vector data type (CASSANDRA-18946)
  * Fix nodetool tablehistograms output to avoid printing repeated information and ensure at most two arguments (CASSANDRA-18955)
  * Change the checksum algorithm SAI-related files use from CRC32 to CRC32C (CASSANDRA-18836)
- * Correctly remove Index.Group from IndexRegistry (CASSANDRA-18905)
  * Fix vector type to support DDM's mask_default function (CASSANDRA-18889)
  * Remove unnecessary reporter-config3 dependency (CASSANDRA-18907)
  * Remove support for empty values on the vector data type (CASSANDRA-18876)
diff --git a/NEWS.txt b/NEWS.txt
index 717b5cf37de6..921ce03d4f2d 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -109,7 +109,6 @@ New features
       src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md
     - New `VectorType` (cql `vector<element_type, dimension>`) which adds new fixed-length element arrays. See CASSANDRA-18504
     - Added new vector similarity functions `similarity_cosine`, `similarity_euclidean` and `similarity_dot_product`.
-    - Added ANN vector similarity search via ORDER BY ANN OF syntax on SAI indexes (using jvector library).
     - Removed UDT type migration logic for 3.6+ clusters upgrading to 4.0.  If migration has been disabled, it must be
       enabled before upgrading to 5.0 if the cluster used UDTs. See CASSANDRA-18504
     - Entended max expiration time from 2038-01-19T03:14:06+00:00 to 2106-02-07T06:28:13+00:00
@@ -294,6 +293,11 @@ Deprecation
       Cluster hosts running with dual native ports were not correctly identified in the system.peers tables and server-sent EVENTs,
       causing clients that encrypt traffic to fail to maintain correct connection pools. For more information, see CASSANDRA-19392.
     - Deprecated `use_deterministic_table_id` in cassandra.yaml. Table IDs may still be supplied explicitly on CREATE.
+    - Chronicle Queue has changed the enums used for log rolling (cassandra.yaml -> full_query_logging_options:roll_cycle).
+    Older legacy options will still work for the foreseeable future but you will see warnings in logs and future dependency
+    upgrades may break your log rolling param. The default log rolling param will be changed with the next major release
+    from HOURLY to FAST_HOURLY, primarily different on how frequently indexes are built (256 in FAST_HOURLY vs. 16 in HOURLY).
+    For more info refer to: net.openhft.chronicle.queue.RollCycles
 
 4.1
 ===
diff --git a/NOTICE.txt b/NOTICE.txt
index fd185210450f..5a2c26ae740d 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -8,3 +8,9 @@ Android Code
 Copyright 2005-2008 The Android Open Source Project
 This product includes software developed as part of
 The Android Open Source Project (http://source.android.com).
+
+This project includes software from the Apache Lucene project. Relevant
+portions of its NOTICE are excerpted below:
+=======================================================================
+Apache Lucene
+Copyright 2001-2018 The Apache Software Foundation
diff --git a/README.asc b/README.asc
index be26f9d97b79..0ca628bf817d 100644
--- a/README.asc
+++ b/README.asc
@@ -25,8 +25,8 @@ and running, and demonstrate some simple reads and writes. For a more-complete g
 
 First, we'll unpack our archive:
 
-  $ tar -zxvf apache-cassandra-$VERSION.tar.gz
-  $ cd apache-cassandra-$VERSION
+  $ tar -zxvf dse-db-$VERSION.tar.gz
+  $ cd dse-db-$VERSION
 
 After that we start the server. Running the startup script with the -f argument will cause
 Cassandra to remain in the foreground and log to standard out; it can be stopped with ctrl-C.
diff --git a/bin/cassandra.in.sh b/bin/cassandra.in.sh
index b838c2d4cf9c..5d83b4ed673e 100644
--- a/bin/cassandra.in.sh
+++ b/bin/cassandra.in.sh
@@ -30,7 +30,7 @@ CLASSPATH="$CASSANDRA_CONF"
 # compiled classes. NOTE: This isn't needed by the startup script,
 # it's just used here in constructing the classpath.
 if [ -d $CASSANDRA_HOME/build ] ; then
-    jars_cnt="`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar | grep -v 'javadoc.jar' | grep -v 'sources.jar' | wc -l | xargs echo`"
+    jars_cnt="`ls -1 $CASSANDRA_HOME/build/dse-db*.jar | grep -v 'javadoc.jar' | grep -v 'sources.jar' | wc -l | xargs echo`"
     if [ "$jars_cnt" -gt 1 ]; then
         dir="`cd $CASSANDRA_HOME/build; pwd`"
         echo "There are JAR artifacts for multiple versions in the $dir directory. Please clean the project with 'ant realclean' and build it again." 1>&2
@@ -38,8 +38,8 @@ if [ -d $CASSANDRA_HOME/build ] ; then
     fi
 
     if [ "$jars_cnt" = "1" ]; then
-        cassandra_bin="`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar | grep -v javadoc | grep -v sources`"
-        CLASSPATH="$CLASSPATH:$cassandra_bin"
+        dse_db_bin="`ls -1 $CASSANDRA_HOME/build/dse-db*.jar | grep -v javadoc | grep -v sources`"
+        CLASSPATH="$CLASSPATH:$dse_db_bin"
     fi
 fi
 
@@ -122,11 +122,16 @@ jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR=
 JVM_VERSION=${jvmver%_*}
 short=$(echo "${jvmver}" | cut -c1-2)
 
-JAVA_VERSION=17
+JAVA_VERSION=22
 if [ "$short" = "11" ]  ; then
      JAVA_VERSION=11
 elif [ "$JVM_VERSION" \< "17" ] ; then
-    echo "Cassandra 5.0 requires Java 11 or Java 17."
+    echo "DSE DB 5.0 requires Java 11 or higher."
+    exit 1;
+elif [ "$short" = "17" ]  ; then
+     JAVA_VERSION=17
+elif [ "$JVM_VERSION" \< "22" ] ; then
+    echo "DSE DB 5.0 requires Java 11 or higher."
     exit 1;
 fi
 
@@ -151,7 +156,9 @@ esac
 
 # Read user-defined JVM options from jvm-server.options file
 JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
-if [ $JAVA_VERSION -ge 17 ] ; then
+if [ $JAVA_VERSION -ge 22 ] ; then
+    JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm22${jvmoptions_variant:--clients}.options
+elif [ $JAVA_VERSION -ge 17 ] ; then
     JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm17${jvmoptions_variant:--clients}.options
 elif [ $JAVA_VERSION -ge 11 ] ; then
     JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options
diff --git a/bin/cqlsh.py b/bin/cqlsh.py
index 738f0aeeb716..87bbfa9da3c4 100755
--- a/bin/cqlsh.py
+++ b/bin/cqlsh.py
@@ -56,7 +56,7 @@ def find_zip(libprefix):
     sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver))
 
 # the driver needs dependencies
-third_parties = ('pure_sasl-', 'wcwidth-')
+third_parties = ('pure_sasl-', 'wcwidth-', 'geomet-')
 
 for lib in third_parties:
     lib_zip = find_zip(lib)
diff --git a/build.properties.default b/build.properties.default
index 36676f5712d8..380270479620 100644
--- a/build.properties.default
+++ b/build.properties.default
@@ -21,3 +21,4 @@
 artifact.remoteRepository.central:     https://repo1.maven.org/maven2
 artifact.remoteRepository.apache:      https://repo.maven.apache.org/maven2
 artifact.remoteRepository.apacheSnapshot: https://repository.apache.org/content/repositories/snapshots
+artifact.remoteRepository.datastax:    https://repo.datastax.com/dse
diff --git a/build.xml b/build.xml
index fdebf135acb8..eae801283cf2 100644
--- a/build.xml
+++ b/build.xml
@@ -14,7 +14,7 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
-<project basedir="." default="check" name="apache-cassandra"
+<project basedir="." default="check" name="dse-db"
          xmlns:if="ant:if"
          xmlns:unless="ant:unless">
 
@@ -33,19 +33,18 @@
     <property name="debuglevel" value="source,lines,vars"/>
 
     <!-- default version and SCM information -->
-    <property name="base.version" value="5.0.2"/>
-    <property name="scm.connection" value="scm:https://gitbox.apache.org/repos/asf/cassandra.git"/>
-    <property name="scm.developerConnection" value="scm:https://gitbox.apache.org/repos/asf/cassandra.git"/>
-    <property name="scm.url" value="https://gitbox.apache.org/repos/asf?p=cassandra.git"/>
+    <property name="base.version" value="5.0.2.0"/>
+    <property name="scm.connection" value="scm:git:ssh://git@github.com:datastax/cassandra.git"/>
+    <property name="scm.developerConnection" value="scm:git:ssh://git@github.com:datastax/cassandra.git"/>
+    <property name="scm.url" value="scm:git:ssh://git@github.com:datastax/cassandra.git"/>
 
     <!-- JDKs supported.
         All releases are built with the default JDK.
         Builds with non-default JDKs are considered experimental and for development and testing purposes.
         When building, javac's source and target flags are set to the jdk used, so lower JDKs are not supported at runtime.
-        The use of both CASSANDRA_USE_JDK11 and use-jdk11 is deprecated.
     -->
     <property name="java.default" value="11" />
-    <property name="java.supported" value="11,17" />
+    <property name="java.supported" value="11,17,22" />
 
     <!-- directory details -->
     <property name="basedir" value="."/>
@@ -95,11 +94,12 @@
     <property name="testtag.extra" value="" />
     <property name="dist.dir" value="${build.dir}/dist"/>
 
-    <!-- Use build/tmp for temp files if not otherwise specified. Because Ant properties are immutable, this has no effect if
+    <!-- Use build/test/cassandra for temp files if not otherwise specified. Because Ant properties are immutable, this has no effect if
          the user specifies the tmp.dir property -->
-    <property name="tmp.dir" value="${build.dir}/tmp"/>
-
+    <property name="tmp.dir" value="${build.test.dir}/cassandra"/>
+    <property name="test.logback.configurationFile" value="${test.conf}/logback-test.xml"/>
     <property name="doc.dir" value="${basedir}/doc"/>
+    <property name="repetition" value=""/>
 
     <condition property="version" value="${base.version}">
       <isset property="release"/>
@@ -113,14 +113,14 @@
 
     <!-- details of how and which Maven repository we publish to -->
     <property name="maven.version" value="3.0.3" />
-    <condition property="maven-repository-url" value="https://repository.apache.org/service/local/staging/deploy/maven2">
+    <condition property="maven-repository-url" value="https://repo.aws.dsinternal.org/artifactory/datastax-releases-local">
       <isset property="release"/>
     </condition>
-    <condition property="maven-repository-id" value="apache.releases.https">
+    <condition property="maven-repository-id" value="datastax.releases.https">
       <isset property="release"/>
     </condition>
-    <property name="maven-repository-url" value="https://repository.apache.org/content/repositories/snapshots"/>
-    <property name="maven-repository-id" value="apache.snapshots.https"/>
+    <property name="maven-repository-url" value="https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local"/>
+    <property name="maven-repository-id" value="datastax.snapshots.https"/>
 
     <property name="test.timeout" value="480000" />
     <property name="test.memory.timeout" value="480000" />
@@ -133,16 +133,24 @@
     <property name="cassandra.test.use_prepared" value="true" />
 
     <!-- The number of active processors seen by JVM -->
-    <property name="cassandra.test.processorCount" value="2"/>
+    <property name="cassandra.test.processorCount" value="4"/>
 
     <!-- skip flushing schema tables during tests -->
-    <property name="cassandra.test.flush_local_schema_changes" value="false" />
+    <property name="cassandra.unsafesystem" value="true" />
 
     <!-- fast shutdown of messaging service -->
     <property name="cassandra.test.messagingService.nonGracefulShutdown" value="true"/>
 
+    <property name="cassandra.test.fail_on_forbidden_log_entries" value="false"/>
+
+    <property name="test.noverify" value="false"/>
+
+    <condition property="noverify-arg" value="-noverify" else="-Da=b">
+        <equals arg1="${test.noverify}" arg2="true"/>
+    </condition>
+
     <!-- https://www.eclemma.org/jacoco/ -->
-    <property name="jacoco.version" value="0.8.8"/>
+    <property name="jacoco.version" value="0.8.12"/>
     <property name="jacoco.export.dir" value="${build.dir}/jacoco/" />
     <property name="jacoco.partials.dir" value="${jacoco.export.dir}/partials" />
     <property name="jacoco.partialexecfile" value="${jacoco.partials.dir}/partial.exec" />
@@ -249,8 +257,15 @@
         <string>--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED</string>
         <string>--add-exports java.rmi/sun.rmi.transport.tcp=ALL-UNNAMED</string>
         <string>--add-exports java.sql/java.sql=ALL-UNNAMED</string>
+        <string>--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED</string>
+        <string>--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED</string>
+        <string>--add-exports java.base/jdk.internal.util=ALL-UNNAMED</string>
 
+        <string>--add-opens java.base/java.io=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.util=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.lang=ALL-UNNAMED</string>
         <string>--add-opens java.base/java.lang.module=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.lang.reflect=ALL-UNNAMED</string>
         <string>--add-opens java.base/java.net=ALL-UNNAMED</string>
         <string>--add-opens java.base/jdk.internal.loader=ALL-UNNAMED</string>
         <string>--add-opens java.base/jdk.internal.ref=ALL-UNNAMED</string>
@@ -258,6 +273,7 @@
         <string>--add-opens java.base/jdk.internal.math=ALL-UNNAMED</string>
         <string>--add-opens java.base/jdk.internal.module=ALL-UNNAMED</string>
         <string>--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED</string>
+        <string>--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED</string>
         <string>--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED</string>
     </resources>
     <pathconvert property="_jvm11_args_concat" refid="_jvm11_arg_items" pathsep=" "/>
@@ -298,6 +314,7 @@
         <string>--add-opens java.base/java.lang=ALL-UNNAMED</string>
         <string>--add-opens java.base/java.util=ALL-UNNAMED</string>
         <string>--add-opens java.base/java.nio=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.nio.file.attribute=ALL-UNNAMED</string>
 
         <string>--add-opens java.rmi/sun.rmi.transport.tcp=ALL-UNNAMED</string>
     </resources>
@@ -306,6 +323,78 @@
         <equals arg1="${ant.java.version}" arg2="17"/>
     </condition>
 
+    <resources id="_jvm22_arg_items">
+        <string>-XX:+UnlockDiagnosticVMOptions</string>
+        <string>-Djdk.attach.allowAttachSelf=true</string>
+        <string>-XX:+UseG1GC</string>
+        <string>-XX:+ParallelRefProcEnabled</string>
+        <!-- Below two are used in Cassandra, not in CNDB-->
+        <!--<string>-XX:MaxTenuringThreshold=1</string>
+        <string>-XX:G1HeapRegionSize=16m</string>-->
+        <string>-XX:G1RSetUpdatingPauseTimePercent=5</string>
+        <string>-XX:MaxGCPauseMillis=100</string>
+
+        <!-- Copy-paste for now these options from cndb/docker-entrypoint.sh; we will verify later whether we
+         want all of them for tests-->
+        <string>-XX:-RestrictContended</string>
+        <string>-XX:+UseThreadPriorities</string>
+        <string>-XX:+DebugNonSafepoints</string>
+        <string>-XX:+UseStringDeduplication</string>
+        <string>-XX:StringTableSize=1000003</string>
+        <string>-XX:+PerfDisableSharedMem</string>
+        <string>-XX:+AlwaysPreTouch</string>
+        <string>-XX:+UseTLAB</string>
+        <string>-XX:+ResizeTLAB</string>
+        <string>-XX:+UseNUMA</string>
+
+
+        <string>--add-exports java.base/jdk.internal.misc=ALL-UNNAMED</string>
+        <string>--add-exports java.base/jdk.internal.ref=ALL-UNNAMED</string>
+        <string>--add-exports java.base/jdk.internal.perf=ALL-UNNAMED</string>
+        <string>--add-exports java.base/sun.nio.ch=ALL-UNNAMED</string>
+        <string>--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED</string>
+        <string>--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED</string>
+        <string>--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED</string>
+        <string>--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED</string>
+        <string>--add-exports jdk.naming.dns/com.sun.jndi.dns=java.naming</string>
+        <string>--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED</string>
+
+        <string>--add-opens java.base/java.io=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.lang=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.lang.module=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.lang.ref=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.lang.reflect=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.math=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.net=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.nio=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.nio.charset=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.nio.file.spi=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.util=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED</string>
+        <string>--add-opens java.base/jdk.internal.loader=ALL-UNNAMED</string>
+        <string>--add-opens java.base/jdk.internal.math=ALL-UNNAMED</string>
+        <string>--add-opens java.base/jdk.internal.module=ALL-UNNAMED</string>
+        <string>--add-opens java.base/jdk.internal.ref=ALL-UNNAMED</string>
+        <string>--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED</string>
+        <string>--add-opens java.base/jdk.internal.vm=ALL-UNNAMED</string>
+        <string>--add-opens java.base/sun.nio.ch=ALL-UNNAMED</string>
+        <string>--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED</string>
+        <string>--add-opens jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED</string>
+        <string>--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED</string>
+        <string>--add-opens jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED</string>
+        <!-- required for org.apache.cassandra.Util.getSupportedMTimeGranularity-->
+        <string>--add-opens java.base/java.nio.file.attribute=ALL-UNNAMED</string>
+
+        <!-- below 2 were added to support jamm and fix BatchCommitLogTest -->
+        <string>--add-opens java.base/java.util.concurrent=ALL-UNNAMED</string>
+        <string>--add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED</string>
+        <string>--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED</string>
+    </resources>
+    <pathconvert property="_jvm22_args_concat" refid="_jvm22_arg_items" pathsep=" "/>
+    <condition property="java-jvmargs" value="${_jvm22_args_concat}">
+        <equals arg1="${ant.java.version}" arg2="22"/>
+    </condition>
+
     <!--
       JVM arguments for tests.
 
@@ -333,11 +422,26 @@
     <condition property="_std-test-jvmargs" value="${_jvm17_test_arg_items_concat}">
         <equals arg1="${ant.java.version}" arg2="17"/>
     </condition>
+    <resources id="_jvm22_test_arg_items">
+        <string>-Djava.security.manager=allow</string>
+        <string>-Dio.netty.tryReflectionSetAccessible=true</string>
+        <string>-Dnet.bytebuddy.experimental=true</string>
+
+        <!-- Enable vector incubator feature (simd support)-->
+        <string>--add-modules jdk.incubator.vector</string>
+    </resources>
+    <pathconvert property="_jvm22_test_arg_items_concat" refid="_jvm22_test_arg_items" pathsep=" "/>
+    <condition property="_std-test-jvmargs" value="${_jvm11_test_arg_items_concat}">
+        <equals arg1="${ant.java.version}" arg2="11"/>
+    </condition>
+    <condition property="_std-test-jvmargs" value="${_jvm22_test_arg_items_concat}">
+        <equals arg1="${ant.java.version}" arg2="22"/>
+    </condition>
 
     <!-- needed to compile org.apache.cassandra.utils.JMXServerUtils -->
     <!-- needed to compile org.apache.cassandra.distributed.impl.Instance-->
     <!-- needed to compile org.apache.cassandra.utils.memory.BufferPool -->
-    <property name="jdk11plus-javac-exports" value="--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports java.rmi/sun.rmi.transport.tcp=ALL-UNNAMED --add-exports java.base/jdk.internal.ref=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED" />
+    <property name="jdk11plus-javac-exports" value="--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports java.rmi/sun.rmi.transport.tcp=ALL-UNNAMED --add-exports java.base/jdk.internal.ref=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED"/>
 
     <!--
          Add all the dependencies.
@@ -365,9 +469,9 @@
     <sequential>
       <javadoc destdir="@{destdir}" author="true" version="true" use="true"
         windowtitle="${ant.project.name} API" classpathref="cassandra.classpath"
-        bottom="Copyright &amp;copy; 2009- The Apache Software Foundation"
+        bottom="Copyright &amp;copy; 2009- The Apache Software Foundation; All changes to the original code are Copyright DataStax, Inc."
         useexternalfile="yes" encoding="UTF-8" failonerror="false"
-        maxmemory="256m" additionalparam="${jdk11plus-javac-exports}">
+        maxmemory="1024M" additionalparam="${jdk11plus-javac-exports}">
         <filesets/>
       </javadoc>
       <fail message="javadoc failed">
@@ -412,8 +516,11 @@
     </target>
 
     <target name="clean" description="Remove all locally created artifacts">
-        <delete dir="${build.test.dir}" />
         <delete dir="${build.dir}/tmp" />
+        <delete includeemptydirs="true" failonerror="false">
+            <fileset dir="${build.test.dir}/cassandra" includes="**/*"/>
+            <fileset dir="${build.test.dir}" includes="**/*" excludes="cassandra"/>
+        </delete>
         <delete dir="${build.classes}" />
         <delete dir="${build.src.gen-java}" />
         <delete dir="${version.properties.dir}" />
@@ -426,7 +533,7 @@
         <delete>
           <fileset dir="${build.lib}" excludes="cassandra-driver-internal-only-*,puresasl-internal-only-*"/>
         </delete>
-        <delete dir="${build.dir}" />
+        <delete includeemptydirs="false" dir="${build.dir}" failonerror="false"/>
         <delete dir="${doc.dir}/build" />
         <delete dir="${doc.dir}/source/tools/nodetool" />
     </target>
@@ -445,9 +552,15 @@
 
     <target name="gen-cql3-grammar" depends="check-gen-cql3-grammar" unless="cql3current">
       <echo>Building Grammar ${build.src.antlr}/Cql.g  ...</echo>
+      <!-- TODO - On jdk21/2 POC branches, antlr crashed first time through, then worked 2nd. Forking it fixes this, and it
+      works also with the current branch and JDK11. We do the change in preparation to be able to compile on newer JDK versions.
+      We may want to further investigate why this happens. -->
       <java classname="org.antlr.Tool"
             classpathref="cql3-grammar.classpath"
-            failonerror="true">
+            failonerror="true"
+            fork="true"
+            outputproperty="antlr.output"
+            errorproperty="antlr.error">
          <arg value="-Xconversiontimeout" />
          <arg value="10000" />
          <arg value="${build.src.antlr}/Cql.g" />
@@ -508,6 +621,7 @@
         <jvmarg value="-Dstorage-config=${test.conf}"/>
         <jvmarg value="-Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" /> <!-- enable defensive checks -->
         <jvmarg value="-javaagent:${build.dir.lib}/jars/jamm-${jamm.version}.jar" />
+
         <jvmarg value="-ea"/>
         <jvmarg line="${java-jvmargs}"/>
       </java>
@@ -735,7 +849,7 @@
     -->
     <target name="_main-jar"
             depends="build"
-            description="Assemble Cassandra JAR files">
+            description="Assemble DSE DB JAR files">
       <mkdir dir="${build.classes.main}/META-INF" />
       <copy file="LICENSE.txt"
             tofile="${build.classes.main}/META-INF/LICENSE.txt"/>
@@ -749,9 +863,9 @@
         <manifest>
         <!-- <section name="org/apache/cassandra/infrastructure"> -->
           <attribute name="Multi-Release" value="true"/>
-          <attribute name="Implementation-Title" value="Cassandra"/>
+          <attribute name="Implementation-Title" value="DSE DB"/>
           <attribute name="Implementation-Version" value="${version}"/>
-          <attribute name="Implementation-Vendor" value="Apache"/>
+          <attribute name="Implementation-Vendor" value="DataStax"/>
           <attribute name="Implementation-Git-SHA" value="${git.sha}"/>
         <!-- </section> -->
         </manifest>
@@ -760,7 +874,7 @@
 
     <target name="stress-jar"
             depends="stress-build"
-            description="Assemble Cassandra JAR files">
+            description="Assemble DSE DB JAR files">
         <!-- Stress jar -->
         <manifest file="${stress.manifest}">
             <attribute name="Built-By" value="Pavel Yaskevich"/>
@@ -775,7 +889,7 @@
 
     <target name="fqltool-jar"
             depends="fqltool-build"
-            description="Assemble Cassandra JAR files">
+            description="Assemble DSE DB JAR files">
 
         <!-- fqltool jar -->
         <manifest file="${fqltool.manifest}">
@@ -791,7 +905,7 @@
 
     <target name="simulator-jars"
             depends="simulator-asm-build,simulator-bootstrap-build"
-            description="Assemble Cassandra JAR files">
+            description="Assemble DSE DB JAR files">
 
         <!-- simulator asm jar -->
         <manifest file="${simulator-asm.manifest}">
@@ -826,13 +940,13 @@
 
     <target name="jar"
             depends="_main-jar,build-test,stress-jar,fqltool-jar,simulator-jars,write-poms"
-            description="Assemble Cassandra JAR files">
+            description="Assemble DSE DB JAR files">
     </target>
 
     <!--
         The javadoc-jar target makes cassandra-javadoc.jar output required for publishing to Maven central repository.
     -->
-    <target name="javadoc-jar" depends="javadoc" unless="no-javadoc" description="Assemble Cassandra JavaDoc JAR file">
+    <target name="javadoc-jar" depends="javadoc" unless="no-javadoc" description="Assemble DSE DB JavaDoc JAR file">
       <jar jarfile="${build.dir}/${final.name}-javadoc.jar" basedir="${javadoc.dir}"/>
       <!-- javadoc task always rebuilds so might as well remove the generated docs to prevent
            being pulled into the distribution by accident -->
@@ -842,7 +956,7 @@
     <!--
         The sources-jar target makes cassandra-sources.jar output required for publishing to Maven central repository.
     -->
-    <target name="sources-jar" depends="init" description="Assemble Cassandra Sources JAR file">
+    <target name="sources-jar" depends="init" description="Assemble DSE DB Sources JAR file">
       <jar jarfile="${build.dir}/${final.name}-sources.jar">
         <fileset dir="${build.src.java}" defaultexcludes="yes">
           <include name="org/apache/**/*.java"/>
@@ -909,8 +1023,8 @@
     </target>
 
     <!-- creates release tarballs -->
-    <target name="artifacts" depends="_artifacts-init,check,gen-doc,sources-jar"
-            description="Create Cassandra tarball and maven artifacts">
+    <target name="artifacts" depends="_artifacts-init,eclipse-warnings,gen-doc,sources-jar,javadoc-jar"
+            description="Create DSE DB release artifacts">
       <tar compression="gzip" longfile="gnu"
         destfile="${build.dir}/${final.name}-bin.tar.gz">
 
@@ -1134,6 +1248,9 @@
       <fail message="testing with build.test.dir (${build.test.dir}) not pointing to 'build/test/' will fail, test configurations are hardcoded.">
         <condition><not><equals arg1="${build.test.dir}" arg2="${basedir}/build/test"/></not></condition>
       </fail>
+      <condition property="tag" value="@{testtag}${repetition}" else="@{testtag}">
+          <isset property="repetition"/>
+      </condition>
       <condition property="additionalagent"
                  value="-javaagent:${build.dir.lib}/jars/jacocoagent.jar=destfile=${jacoco.partialexecfile}"
                  else="">
@@ -1146,7 +1263,7 @@
       </taskdef>
       <mkdir dir="${build.test.dir}/cassandra"/>
       <mkdir dir="${build.test.output.dir}"/>
-      <mkdir dir="${build.test.output.dir}/@{testtag}"/>
+      <mkdir dir="${build.test.output.dir}/@{tag}"/>
       <mkdir dir="${tmp.dir}"/>
       <junit-timeout fork="on" forkmode="@{forkmode}" failureproperty="testfailed" maxmemory="1024m" timeout="@{timeout}" showoutput="@{showoutput}">
         <formatter classname="org.apache.cassandra.CassandraXMLJUnitResultFormatter" extension=".xml" usefile="true"/>
@@ -1171,23 +1288,32 @@
         <jvmarg value="-Dcassandra.test.driver.read_timeout_ms=${test.driver.read_timeout_ms}"/>
         <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
         <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
+        <jvmarg value="-Dcassandra.test.messagingService.nonGracefulShutdown=${cassandra.test.messagingService.nonGracefulShutdown}"/>
+        <jvmarg value="-Dcassandra.cluster_version_provider.min_stable_duration_ms=0"/>
+        <jvmarg value="-Dcassandra.test.cluster_version_provider.skip_wait_for_gossip=true"/>
+        <jvmarg value="${noverify-arg}"/>
+
         <jvmarg value="-Dcassandra.test.sstableformatdevelopment=true"/>
         <!-- The first time SecureRandom initializes can be slow if it blocks on /dev/random -->
         <jvmarg value="-Djava.security.egd=file:/dev/urandom" />
-        <jvmarg value="-Dcassandra.testtag=@{testtag}_jdk${ant.java.version}${testtag.extra}"/>
+        <jvmarg value="-Dcassandra.testtag=${tag}_jdk${ant.java.version}${testtag.extra}"/>
         <jvmarg value="-Dcassandra.keepBriefBrief=${cassandra.keepBriefBrief}" />
         <jvmarg value="-Dcassandra.strict.runtime.checks=true" />
         <jvmarg value="-Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" /> <!-- enable defensive checks -->
-        <jvmarg value="-Dcassandra.test.flush_local_schema_changes=${cassandra.test.flush_local_schema_changes}"/>
+        <jvmarg value="-Dcassandra.unsafesystem=${cassandra.unsafesystem}"/>
         <jvmarg value="-Dcassandra.test.messagingService.nonGracefulShutdown=${cassandra.test.messagingService.nonGracefulShutdown}"/>
         <jvmarg value="-Dcassandra.use_nix_recursive_delete=${cassandra.use_nix_recursive_delete}"/>
         <jvmarg value="-Dio.netty.allocator.useCacheForAllThreads=true"/>
         <jvmarg value="-Dio.netty.allocator.maxOrder=11"/>
+
         <jvmarg line="${java-jvmargs}"/>
         <!-- disable shrinks in quicktheories CASSANDRA-15554 -->
         <jvmarg value="-DQT_SHRINKS=0"/>
+        <jvmarg value="-Dcassandra.use_nix_recursive_delete=true"/>
+        <jvmarg value="-Dcassandra.test.fail_on_forbidden_log_entries=${cassandra.test.fail_on_forbidden_log_entries}"/>
         <jvmarg line="${_std-test-jvmargs} " />
         <jvmarg line="${test.jvm.args}" />
+        <jvmarg value="-Djava.io.tmpdir=${build.test.dir}/cassandra"/>
         <optjvmargs/>
         <!-- Uncomment to debug unittest, attach debugger to port 1416 -->
         <!--
@@ -1209,7 +1335,7 @@
               <exclude name="**/ant-*.jar"/>
           </fileset>
         </classpath>
-        <batchtest todir="${build.test.output.dir}/@{testtag}">
+        <batchtest todir="${build.test.output.dir}/@{tag}">
             <fileset dir="@{inputdir}" includes="@{filter}" excludes="@{exclude}"/>
             <filelist dir="@{inputdir}" files="@{filelist}"/>
         </batchtest>
@@ -1232,6 +1358,7 @@
       <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
       <jvmarg value="-Dcassandra.skip_sync=true" />
+
     </testmacro>
     <fileset dir="${test.unit.src}" />
   </target>
@@ -1250,6 +1377,8 @@
         <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
         <jvmarg value="-Dcassandra.skip_sync=true" />
         <jvmarg value="-Xmx${maxMemory}"/>
+
+        <jvmarg value="-Dlogback.configurationFile=file://${test.logback.configurationFile}"/>
       </testmacrohelper>
     </sequential>
   </macrodef>
@@ -1273,6 +1402,8 @@
         <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
         <jvmarg value="-Dcassandra.config=file:///${compressed_yaml}"/>
         <jvmarg value="-Dcassandra.skip_sync=true" />
+
+        <jvmarg value="-Dlogback.configurationFile=file://${test.logback.configurationFile}"/>
       </testmacrohelper>
     </sequential>
   </macrodef>
@@ -1293,6 +1424,8 @@
         <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
         <jvmarg value="-Dcassandra.config=file:///${cdc_yaml}"/>
         <jvmarg value="-Dcassandra.skip_sync=true" />
+
+        <jvmarg value="-Dlogback.configurationFile=file://${test.logback.configurationFile}"/>
       </testmacrohelper>
     </sequential>
   </macrodef>
@@ -1315,6 +1448,7 @@
                 <jvmarg value="-Dcassandra.config=file:///${latest_yaml}"/>
                 <jvmarg value="-Dcassandra.test.storage_compatibility_mode=NONE"/>
                 <jvmarg value="-Dcassandra.skip_sync=true" />
+                <jvmarg value="-Dlogback.configurationFile=file://${test.logback.configurationFile}"/>
             </testmacrohelper>
         </sequential>
     </macrodef>
@@ -1327,7 +1461,7 @@
           <fileset file="${test.conf}/cassandra.yaml"/>
           <fileset file="${test.conf}/storage_compatibility_mode_none.yaml"/>
         </concat>
-      <testmacrohelper inputdir="${test.dir}/${test.classlistprefix}" filelist="@{test.file.list}" 
+      <testmacrohelper inputdir="${test.dir}/${test.classlistprefix}" filelist="@{test.file.list}"
                        exclude="**/*.java" timeout="${test.timeout}" testtag="oa">
         <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
         <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
@@ -1336,10 +1470,32 @@
         <jvmarg value="-Dcassandra.config=file:///${scm_none_yaml}"/>
         <jvmarg value="-Dcassandra.test.storage_compatibility_mode=NONE"/>
         <jvmarg value="-Dcassandra.skip_sync=true" />
+
+        <jvmarg value="-Dlogback.configurationFile=file://${test.logback.configurationFile}"/>
       </testmacrohelper>
     </sequential>
   </macrodef>
 
+    <macrodef name="testlist-nocursor">
+        <attribute name="test.file.list"/>
+        <attribute name="testlist.offset"/>
+        <sequential>
+            <testmacrohelper inputdir="${test.dir}/${test.classlistprefix}" filelist="@{test.file.list}" poffset="@{testlist.offset}"
+                             exclude="**/*.java" timeout="${test.timeout}" testtag="nocursor">
+                <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
+                <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
+                <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
+                <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+                <jvmarg value="-Dcassandra.config.loader=org.apache.cassandra.OffsetAwareConfigurationLoader"/>
+                <jvmarg value="-Dcassandra.skip_sync=true" />
+
+                <jvmarg value="-Dcassandra.allow_cursor_compaction=false" />
+                <jvmarg value="-Dunified_compaction.parallelize_output_shards=false" />
+                <jvmarg value="-Dlogback.configurationFile=file://${test.logback.configurationFile}"/>
+            </testmacrohelper>
+        </sequential>
+    </macrodef>
+
     <macrodef name="testlist-system-keyspace-directory">
     <attribute name="test.file.list" />
     <sequential>
@@ -1356,10 +1512,46 @@
         <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
         <jvmarg value="-Dcassandra.config=file:///${system_keyspaces_directory_yaml}"/>
         <jvmarg value="-Dcassandra.skip_sync=true" />
+
+        <jvmarg value="-Dlogback.configurationFile=file://${test.logback.configurationFile}"/>
       </testmacrohelper>
     </sequential>
   </macrodef>
 
+  <macrodef name="testlist-sai-legacy">
+    <attribute name="test.file.list" />
+    <sequential>
+        <testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}"
+                         exclude="**/*.java" timeout="${test.timeout}" >
+            <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
+            <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
+            <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
+            <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+            <jvmarg value="-Dcassandra.sai.latest.version=aa"/>
+            <jvmarg value="-Dcassandra.trie_index_format_version=bb"/>
+            <jvmarg value="-Dcassandra.skip_sync=true" />
+
+            <jvmarg value="-Dlogback.configurationFile=file://${test.logback.configurationFile}"/>
+        </testmacrohelper>
+    </sequential>
+  </macrodef>
+
+  <macrodef name="testlist-sai">
+      <attribute name="test.file.list" />
+      <sequential>
+          <testmacrohelper inputdir="\${test.unit.src}" filelist="@{test.file.list}"
+                           exclude="**/*.java" timeout="\${test.timeout}" testtag="sai-legacy">
+              <jvmarg value="-Dlegacy-sstable-root=\${test.data}/legacy-sstables"/>
+              <jvmarg value="-Dinvalid-legacy-sstable-root=\${test.data}/invalid-legacy-sstables"/>
+              <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
+              <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+              <jvmarg value="-Dcassandra.skip_sync=true" />
+
+              <jvmarg value="-Dlogback.configurationFile=file://\${test.logback.configurationFile}"/>
+          </testmacrohelper>
+      </sequential>
+  </macrodef>
+
   <!--
     Run named ant task with jacoco, such as "ant jacoco-run -Dtaskname=test"
     the target run must enable the jacoco agent if usejacoco is 'yes' -->
@@ -1385,6 +1577,7 @@
       <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
       <jvmarg value="-Dcassandra.skip_sync=true" />
+
     </testmacro>
   </target>
 
@@ -1398,6 +1591,7 @@
       <test if="withoutMethods" name="${test.name}" todir="${build.test.output.dir}/" outfile="TEST-${test.name}"/>
       <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+
     </testmacro>
   </target>
 
@@ -1410,6 +1604,7 @@
       <test if="withMethods" name="${test.name}" methods="${test.methods}" todir="${build.test.output.dir}/" outfile="TEST-${test.name}-${test.methods}"/>
       <test if="withoutMethods" name="${test.name}" todir="${build.test.output.dir}/" outfile="TEST-${test.name}"/>
       <jvmarg value="-Dlogback.configurationFile=test/conf/logback-burntest.xml"/>
+
     </testmacro>
   </target>
 
@@ -1446,6 +1641,22 @@
     <testhelper testdelegate="testlist-oa"/>
   </target>
 
+  <target name="test-bigtable" depends="maybe-build-test" description="Execute unit tests with using legacy sstable format and sequential sstable identifiers">
+      <path id="all-test-classes-path">
+          <fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
+      </path>
+      <property name="all-test-classes" refid="all-test-classes-path"/>
+      <testhelper testdelegate="testlist-bigtable" />
+  </target>
+
+  <target name="test-nocursor" depends="maybe-build-test" description="Execute unit tests with cursor compaction disabled">
+      <path id="all-test-classes-path">
+          <fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
+      </path>
+      <property name="all-test-classes" refid="all-test-classes-path"/>
+      <testhelper testdelegate="testlist-nocursor" />
+  </target>
+
   <target name="test-system-keyspace-directory" depends="maybe-build-test" description="Execute unit tests with a system keyspaces directory configured">
     <path id="all-test-classes-path">
       <fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
@@ -1454,22 +1665,41 @@
     <testhelper testdelegate="testlist-system-keyspace-directory" />
   </target>
 
+  <target name="test-sai-legacy" depends="build-test" description="Execute unit tests using a legacy SAI index format">
+    <path id="all-test-classes-path">
+        <fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
+    </path>
+    <property name="all-test-classes" refid="all-test-classes-path"/>
+    <testhelper testdelegate="testlist-sai-legacy" />
+  </target>
+
+  <target name="test-sai" depends="build-test" description="Execute SAI unit tests">
+      <path id="all-test-classes-path">
+          <fileset dir="${test.unit.src}" includes="**/sai/**/${test.name}.java"/>
+      </path>
+      <property name="all-test-classes" refid="all-test-classes-path"/>
+      <testhelper testdelegate="testlist-sai-legacy" />
+  </target>
+
   <target name="msg-ser-gen-test" depends="maybe-build-test" description="Generates message serializations">
     <testmacro inputdir="${test.unit.src}"
         timeout="${test.timeout}" filter="**/SerializationsTest.java">
       <jvmarg value="-Dcassandra.test-serialization-writes=True"/>
+
     </testmacro>
   </target>
 
   <target name="msg-ser-test" depends="maybe-build-test" description="Tests message serializations">
       <testmacro inputdir="${test.unit.src}" timeout="${test.timeout}"
                filter="**/SerializationsTest.java"/>
+
   </target>
 
   <target name="msg-ser-test-7" depends="maybe-build-test" description="Generates message serializations">
     <testmacro inputdir="${test.unit.src}"
         timeout="${test.timeout}" filter="**/SerializationsTest.java">
       <jvmarg value="-Dcassandra.version=0.7"/>
+
     </testmacro>
   </target>
 
@@ -1477,6 +1707,7 @@
     <testmacro inputdir="${test.unit.src}"
         timeout="${test.timeout}" filter="**/SerializationsTest.java">
       <jvmarg value="-Dcassandra.version=1.0"/>
+
     </testmacro>
   </target>
 
@@ -1491,6 +1722,7 @@
                timeout="${test.long.timeout}">
       <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+
     </testmacro>
   </target>
 
@@ -1498,6 +1730,7 @@
       <testmacro inputdir="${test.memory.src}"
                  timeout="${test.memory.timeout}">
           <jvmarg value="-javaagent:${build.dir}/test/lib/jars/java-allocation-instrumenter-${allocation-instrumenter.version}.jar"/>
+
       </testmacro>
   </target>
 
@@ -1516,6 +1749,7 @@
         <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
         <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
         <jvmarg value="-Dcassandra.skip_sync=true" />
+
         <classpath>
           <path refid="cassandra.classpath.test" />
           <pathelement location="${test.classes}"/>
@@ -1562,6 +1796,7 @@
         <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
         <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
         <jvmarg value="-Dcassandra.skip_sync=true" />
+
         <classpath>
           <path refid="cassandra.classpath.test" />
           <pathelement location="${test.classes}"/>
@@ -1705,6 +1940,13 @@
       <property name="all-test-classes" refid="all-test-classes-path"/>
       <testhelper testdelegate="testlist-cdc"/>
   </target>
+  <target name="testclasslist-bigtable" depends="maybe-build-test" description="Run tests given in file -Dtest.classlistfile (one-class-per-line, e.g. org/apache/cassandra/db/SomeTest.java)">
+    <path id="all-test-classes-path">
+      <fileset dir="${test.dir}/${test.classlistprefix}" includesfile="${test.classlistfile}"/>
+    </path>
+    <property name="all-test-classes" refid="all-test-classes-path"/>
+    <testhelper testdelegate="testlist-bigtable"/>
+  </target>
   <target name="testclasslist-system-keyspace-directory" depends="maybe-build-test" description="Run tests given in file -Dtest.classlistfile (one-class-per-line, e.g. org/apache/cassandra/db/SomeTest.java)">
       <path id="all-test-classes-path">
           <fileset dir="${test.dir}/${test.classlistprefix}" includesfile="${test.classlistfile}"/>
@@ -1712,6 +1954,13 @@
       <property name="all-test-classes" refid="all-test-classes-path"/>
       <testhelper testdelegate="testlist-system-keyspace-directory"/>
   </target>
+  <target name="testclasslist-sai-legacy" depends="maybe-build-test" description="Run tests given in file -Dtest.classlistfile (one-class-per-line, e.g. org/apache/cassandra/db/SomeTest.java)">
+      <path id="all-test-classes-path">
+          <fileset dir="${test.dir}/${test.classlistprefix}" includesfile="${test.classlistfile}"/>
+      </path>
+      <property name="all-test-classes" refid="all-test-classes-path"/>
+      <testhelper testdelegate="testlist-sai-legacy"/>
+  </target>
 
   <!-- Build a self-contained jar for e.g. remote execution; not currently used for running burn tests with this build script -->
   <target name="burn-test-jar" depends="build-test, build" description="Create dtest-compatible jar, including all dependencies">
@@ -1773,6 +2022,7 @@
       <jvmarg value="-Dcassandra.ring_delay_ms=10000"/>
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
       <jvmarg value="-Dcassandra.skip_sync=true" />
+
     </testmacro>
   </target>
 
@@ -1810,6 +2060,7 @@
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
       <jvmarg value="-Dcassandra.skip_sync=true" />
       <jvmarg value="-Xmx8G"/>
+
     </testmacro>
   </target>
 
@@ -1826,6 +2077,7 @@
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
       <jvmarg value="-Dcassandra.skip_sync=true" />
       <jvmarg value="-Xmx8G"/>
+
     </testmacro>
   </target>
 
@@ -1944,10 +2196,10 @@
 </project>]]></echo>
       <antcall target="_maybe_update_idea_to_java11plus"/>
       <echo>
-      IDE configuration in .idea/ updated for use with JDK${ant.java.version}.
+          IDE configuration in .idea/ updated for use with JDK${ant.java.version}.
 
-      In IntelliJ verify that the SDK is ${ant.java.version}, and its path is valid.
-      This can be verified in 'Project Structure/Project Setting/Project' and 'Project Structure/Platform Setting/SDKs'.
+          In IntelliJ verify that the SDK is ${ant.java.version}, and its path is valid.
+          This can be verified in 'Project Structure/Project Setting/Project' and 'Project Structure/Platform Setting/SDKs'.
       </echo>
   </target>
 
@@ -2047,7 +2299,7 @@
              file="${build.dir}/${final.name}-parent.pom"
              packaging="pom"/>
 
-    <!-- the cassandra-all jar -->
+    <!-- the dse-db-all jar -->
     <install pomFile="${build.dir}/${final.name}.pom"
              file="${build.dir}/${final.name}.jar"/>
     <install pomFile="${build.dir}/${final.name}.pom"
@@ -2068,7 +2320,7 @@
             file="${build.dir}/${final.name}-parent.pom"
             packaging="pom"/>
 
-    <!-- the cassandra-all jar -->
+    <!-- the dse-db-all jar -->
     <deploy pomFile="${build.dir}/${final.name}.pom"
             file="${build.dir}/${final.name}.jar"/>
     <deploy pomFile="${build.dir}/${final.name}.pom"
@@ -2084,6 +2336,36 @@
 
   </target>
 
+  <target name="compactionSim"
+          depends="build-test"
+          description="A test that simulates compactions to see how strategies behave">
+      <sequential>
+          <mkdir dir="${build.test.dir}/logs"/>
+          <delete>
+              <fileset dir="${build.test.dir}/logs">
+                  <include name="*"/>
+              </fileset>
+          </delete>
+          <java classname="org.apache.cassandra.db.compaction.CompactionSimulationTest"
+                fork="true"
+                failonerror="true"
+                dir="${basedir}">
+              <classpath>
+                  <path refid="cassandra.classpath.test" />
+                  <pathelement location="${test.classes}"/>
+                  <pathelement location="${test.conf}"/>
+                  <fileset dir="${test.lib}">
+                      <include name="**/*.jar" />
+                  </fileset>
+              </classpath>
+              <jvmarg value="-Dstorage-config=${test.conf}"/>
+              <jvmarg value="-Dcassandra.logdir=${build.test.dir}/logs"/>
+              <jvmarg value="-Dlogback.configurationFile=file:///${basedir}/conf/logback.xml"/>
+              <jvmarg value="-Dcassandra.config=file:///${test.conf}/cassandra.yaml"/>
+          </java>
+      </sequential>
+  </target>
+
   <import file="${build.helpers.dir}/build-resolver.xml"/>
   <import file="${build.helpers.dir}/build-rat.xml"/>
   <import file="${build.helpers.dir}/build-owasp.xml"/>
diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh
index ba9f9d459641..7d57e6762fcf 100644
--- a/conf/cassandra-env.sh
+++ b/conf/cassandra-env.sh
@@ -98,7 +98,7 @@ echo "$JVM_OPTS" | grep -qe "-[X]log:gc"
 if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
     # only add -Xlog:gc if it's not mentioned in jvm-server.options file
     mkdir -p ${CASSANDRA_LOG_DIR}
-    JVM_OPTS="$JVM_OPTS -Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=${CASSANDRA_LOG_DIR}/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
+    JVM_OPTS="$JVM_OPTS -Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=${CASSANDRA_LOG_DIR}/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
 fi
 
 # Check what parameters were defined on jvm-server.options file to avoid conflicts
@@ -210,9 +210,9 @@ JVM_ON_OUT_OF_MEMORY_ERROR_OPT="-XX:OnOutOfMemoryError=kill -9 %p"
 # for more on configuring JMX through firewalls, etc. (Short version:
 # get it working with no firewall first.)
 #
-# Cassandra ships with JMX accessible *only* from localhost.  
+# Cassandra ships with JMX accessible *only* from localhost.
 # To enable remote JMX connections, uncomment lines below
-# with authentication and/or ssl enabled. See https://wiki.apache.org/cassandra/JmxSecurity 
+# with authentication and/or ssl enabled. See https://wiki.apache.org/cassandra/JmxSecurity
 #
 if [ "x$LOCAL_JMX" = "x" ]; then
     LOCAL_JMX=yes
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 5562a2c1560d..7e0943aefa6c 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -156,6 +156,32 @@ auto_hints_cleanup_enabled: false
 # Min unit: KiB
 batchlog_replay_throttle: 1024KiB
 
+# Strategy to choose the batchlog storage endpoints.
+#
+# Available options:
+#
+# - random_remote
+#   Default, purely random, prevents the local rack, if possible.
+#
+# - prefer_local
+#   Similar to random_remote. Random, except that one of the replications will go to the local rack,
+#   which mean it offers lower availability guarantee than random_remote or dynamic_remote.
+#
+# - dynamic_remote
+#   Using DynamicEndpointSnitch to select batchlog storage endpoints, prevents the
+#   local rack, if possible. This strategy offers the same availability guarantees
+#   as random_remote but selects the fastest endpoints according to the DynamicEndpointSnitch.
+#   (DynamicEndpointSnitch currently only tracks reads and not writes - i.e. write-only
+#   (or mostly-write) workloads might not benefit from this strategy.)
+#   Note: this strategy will fall back to random_remote, if dynamic_snitch is not enabled.
+#
+# - dynamic
+#   Mostly the same as dynamic_remote, except that local rack is not excluded, which mean it offers lower
+#   availability guarantee than random_remote or dynamic_remote.
+#   Note: this strategy will fall back to random_remote, if dynamic_snitch is not enabled.
+#
+# batchlog_endpoint_strategy: random_remote
+
 # Authentication backend, implementing IAuthenticator; used to identify users
 # Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator,
 # PasswordAuthenticator}.
@@ -369,6 +395,11 @@ partitioner: org.apache.cassandra.dht.Murmur3Partitioner
 # data_file_directories:
 #     - /var/lib/cassandra/data
 
+# Metadata directory that holds information about the cluster, local node and its peers.
+# Currently, only a single subdirectory called 'nodes' will be used.
+# If not set, the default directory is $CASSANDRA_HOME/data/metadata.
+# metadata_directory: /var/lib/cassandra/metadata
+
 # Directory were Cassandra should store the data of the local system keyspaces.
 # By default Cassandra will store the data of the local system keyspaces in the first of the data directories specified
 # by data_file_directories.
@@ -661,6 +692,8 @@ commitlog_disk_access_mode: legacy
 # none : Flush without compressing blocks but while still doing checksums.
 # fast : Flush with a fast compressor. If the table is already using a
 #        fast compressor that compressor is used.
+# adaptive : Flush with a fast adaptive compressor. If the table is already using a
+#        fast compressor that compressor is used.
 # table: Always flush with the same compressor that the table uses. This
 #        was the pre 4.0 behavior.
 #
@@ -796,7 +829,7 @@ memtable:
 #
 # offheap_objects
 #    off heap objects
-memtable_allocation_type: heap_buffers
+memtable_allocation_type: offheap_objects
 
 # Limit memory usage for Merkle tree calculations during repairs of a certain
 # table and common token range. Repair commands targetting multiple tables or
@@ -822,7 +855,7 @@ memtable_allocation_type: heap_buffers
 # There isn't a limit by default for backwards compatibility, but this can
 # produce OOM for  commands repairing multiple tables or multiple virtual nodes.
 # A limit of just 1 simultaneous Merkle tree request is generally recommended
-# with no virtual nodes so repair_session_space, and thereof the Merkle tree
+# with no virtual nodes so repair_session_space, and therefore the Merkle tree
 # resolution, can be high. For virtual nodes a value of 1 with the default
 # repair_session_space value will produce higher resolution Merkle trees
 # at the expense of speed. Alternatively, when working with virtual nodes it
@@ -925,7 +958,7 @@ index_summary_resize_interval: 60m
 # buffers. Enable this to avoid sudden dirty buffer flushing from
 # impacting read latencies. Almost always a good idea on SSDs; not
 # necessarily on platters.
-trickle_fsync: false
+trickle_fsync: true
 # Min unit: KiB
 trickle_fsync_interval: 10240KiB
 
@@ -1235,7 +1268,8 @@ sstable_preemptive_open_interval: 50MiB
 # set to true, each newly created sstable will have a UUID based generation identifier and such files are
 # not readable by previous Cassandra versions. At some point, this option will become true by default
 # and eventually get removed from the configuration.
-uuid_sstable_identifiers_enabled: false
+# In Converged Cassandra, we enable this option by default
+uuid_sstable_identifiers_enabled: true
 
 # When enabled, permits Cassandra to zero-copy stream entire eligible
 # SSTables between nodes, including every component.
@@ -1318,6 +1352,16 @@ truncate_request_timeout: 60000ms
 # Lowest acceptable value is 10 ms.
 # Min unit: ms
 request_timeout: 10000ms
+# Upper bound for how long any request received via native transport
+# should be considered live and serviceable by the system. This is
+# currently considered at two points: when the message is dequeued and
+# executed by the NATIVE_TRANSPORT_REQUESTS stage, and when the message
+# is dequeued and executed by an async stage if NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED
+# is set to true. If the request is not completed within this time, an
+# OverloadedException is thrown.
+# Min unit: ms
+native_transport_timeout: 12000ms
+
 
 # Defensive settings for protecting Cassandra from true network partitions.
 # See (CASSANDRA-14358) for details.
@@ -1728,12 +1772,6 @@ transparent_data_encryption_options:
           store_type: JCEKS
           key_password: cassandra
 
-# Storage Attached Indexing options.
-# sai_options:
-  ## Total permitted memory allowed for writing SAI index segments. This memory
-  ## is split between all SAI indexes being built so more indexes will mean smaller
-  ## segment sizes.
-  # segment_write_buffer_size: 1024MiB
 
 #####################
 # SAFETY THRESHOLDS #
@@ -1827,6 +1865,11 @@ unlogged_batch_across_partitions_warn_threshold: 10
 
 # Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs
 # on audit_logging for full details about the various configuration options and production tips.
+# NOTE: Chronicle Queue has changed the enums used for log rolling roll_cycle).
+# Older legacy options will still work for the foreseeable future, but you will see warnings in logs and future dependency
+# upgrades may break your log rolling param. The default log rolling param will be changed with the next major release
+# from HOURLY to FAST_HOURLY, primarily different on how frequently indexes are built. For more info refer to:
+# net.openhft.chronicle.queue.RollCycles
 audit_logging_options:
   enabled: false
   logger:
@@ -1909,6 +1952,10 @@ report_unconfirmed_repaired_data_mismatches: false
 # Defaults to false to disable dynamic data masking.
 # dynamic_data_masking_enabled: false
 
+# This is the page size used internally by aggregation queries. It aims to limit the memory used by aggregation
+# queries when there is a lot of data to aggregate.
+# aggregation_subpage_size_in_kb: 2048
+
 #########################
 # EXPERIMENTAL FEATURES #
 #########################
@@ -1977,7 +2024,7 @@ drop_compact_storage_enabled: false
 # columns_per_table_warn_threshold: -1
 # columns_per_table_fail_threshold: -1
 #
-# Guardrail to warn or fail when creating more secondary indexes per table than threshold.
+# Guardrail to warn or fail when creating more secondary indexes per table than threshold (does not apply to CUSTOM INDEX StorageAttachedIndex).
 # The two thresholds default to -1 to disable.
 # secondary_indexes_per_table_warn_threshold: -1
 # secondary_indexes_per_table_fail_threshold: -1
@@ -1985,6 +2032,16 @@ drop_compact_storage_enabled: false
 # Guardrail to enable or disable the creation of secondary indexes
 # secondary_indexes_enabled: true
 #
+# Failure threshold for number of StorageAttachedIndex per table (only applies to CUSTOM INDEX StorageAttachedIndex)
+# Default is 10 (same when apply_dbaas_defaults is enabled)
+# sai_indexes_per_table_warn_threshold: -1
+# sai_indexes_per_table_fail_threshold: 10
+#
+# Failure threshold for total number of StorageAttachedIndex across all keyspaces (only applies to CUSTOM INDEX StorageAttachedIndex)
+# Default is 10 (same when apply_dbaas_defaults is enabled)
+# sai_indexes_total_warn_threshold: -1
+# sai_indexes_total_fail_threshold: 100
+#
 # Guardrail to warn or fail when creating more materialized views per table than threshold.
 # The two thresholds default to -1 to disable.
 # materialized_views_per_table_warn_threshold: -1
@@ -2113,7 +2170,7 @@ drop_compact_storage_enabled: false
 # Guardrail to warn or fail when creating a vector column with more dimensions than threshold.
 # Default -1 to disable.
 # vector_dimensions_warn_threshold: -1
-# vector_dimensions_fail_threshold: -1
+# vector_dimensions_fail_threshold: 8192
 #
 # Guardrail to indicate whether or not users are allowed to use ALTER TABLE commands to make column changes to tables
 # alter_table_enabled: true
@@ -2184,6 +2241,20 @@ drop_compact_storage_enabled: false
 # sai_vector_term_size_warn_threshold: 16KiB
 # sai_vector_term_size_fail_threshold: 32KiB
 
+# Guardrail to warn or fail when using LIMIT/OFFSET paging skipping more rows than threshold.
+# Default offset_rows_warn_threshold is 10000, may differ if emulate_dbaas_defaults is enabled
+# Default offset_rows_failure_threshold is 20000, may differ if emulate_dbaas_defaults is enabled
+# offset_rows_warn_threshold: 10000
+# offset_rows_failure_threshold: 20000
+
+# Guardrail to warn or fail when a SELECT query has more column value filters than threshold.
+# Note that restrictions on indexed columns can be expanded to multiple column filters if the indexes have an analyzer.
+# In that case, there will be a filter for every token produced by the analyzer for the queried column value. This can
+# prevent that productive analyzers such as n-gram explode the query to a large number of filtering operations.
+# Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+# query_filters_warn_threshold: -1
+# query_filters_fail_threshold: -1
+
 # The default secondary index implementation when CREATE INDEX does not specify one via USING.
 # ex. "legacy_local_table" - (default) legacy secondary index, implemented as a hidden table
 # ex. "sai" - "storage-attched" index, implemented via optimized SSTable/Memtable-attached indexes
@@ -2246,4 +2317,16 @@ drop_compact_storage_enabled: false
 #   and ensures stability. If Cassandra was started at the previous version by accident, a node with disabled
 #   compatibility mode would no longer toggle behaviors as when it was running in the UPGRADING mode.
 #
-storage_compatibility_mode: CASSANDRA_4
+storage_compatibility_mode: NONE
+
+# Emulates DataStax Constellation database-as-a-service defaults.
+#
+# When enabled, some defaults are modified to match those used by DataStax Constellation (DataStax cloud data
+# platform). This includes (but is not limited to) stricter guardrails defaults.
+#
+# This can be used as an convenience to develop and test applications meant to run on DataStax Constellation.
+#
+# Warning: when enabled, the updated defaults reflect those of DataStax Constellation _at the time_ of the currently
+#                 used DSE release. This is a best-effort emulation of said defaults. Further, all nodes must use the same
+#                 config value.
+# emulate_dbaas_defaults: false
diff --git a/conf/cassandra_latest.yaml b/conf/cassandra_latest.yaml
index fd86f149e617..38aa35878700 100644
--- a/conf/cassandra_latest.yaml
+++ b/conf/cassandra_latest.yaml
@@ -1302,6 +1302,15 @@ truncate_request_timeout: 60000ms
 # Lowest acceptable value is 10 ms.
 # Min unit: ms
 request_timeout: 10000ms
+# Upper bound for how long any request received via native transport
+# should be considered live and serviceable by the system. This is
+# currently considered at two points: when the message is dequeued and
+# executed by the NATIVE_TRANSPORT_REQUESTS stage, and when the message
+# is dequeued and executed by an async stage if NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED
+# is set to true. If the request is not completed within this time, an
+# OverloadedException is thrown.
+# Min unit: ms
+native_transport_timeout: 12000ms
 
 # Defensive settings for protecting Cassandra from true network partitions.
 # See (CASSANDRA-14358) for details.
diff --git a/conf/cqlshrc.sample.cloud b/conf/cqlshrc.sample.cloud
new file mode 100644
index 000000000000..62528670c48b
--- /dev/null
+++ b/conf/cqlshrc.sample.cloud
@@ -0,0 +1,17 @@
+; Copyright DataStax, Inc.
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
+; Sample ~/.cqlshrc file with cloud configuration.
+[connection]
+secure_connect_bundle = /path/to/creds.zip
diff --git a/conf/jvm11-clients.options b/conf/jvm11-clients.options
index 3d59816c045f..08ce8f2a30f6 100644
--- a/conf/jvm11-clients.options
+++ b/conf/jvm11-clients.options
@@ -29,18 +29,28 @@
 -Djdk.attach.allowAttachSelf=true
 --add-exports java.base/jdk.internal.misc=ALL-UNNAMED
 --add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/jdk.internal.util=ALL-UNNAMED
 --add-exports java.base/sun.nio.ch=ALL-UNNAMED
 --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
 --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
 --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
 --add-exports java.sql/java.sql=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
 
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.lang=ALL-UNNAMED
 --add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens java.base/java.util=ALL-UNNAMED
 --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
 --add-opens java.base/jdk.internal.ref=ALL-UNNAMED
 --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
 --add-opens java.base/jdk.internal.math=ALL-UNNAMED
 --add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/java.util=ALL-UNNAMED
+--add-opens java.base/jdk.internal.util=ALL-UNNAMED
 --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
 --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
 --add-opens java.base/java.lang.reflect=ALL-UNNAMED
diff --git a/conf/jvm11-server.options b/conf/jvm11-server.options
index f71f6287ffb5..0915b2ca385d 100644
--- a/conf/jvm11-server.options
+++ b/conf/jvm11-server.options
@@ -30,6 +30,8 @@
 # Disable biased locking as it does not benefit Cassandra.
 -XX:-UseBiasedLocking
 
+-XX:ThreadPriorityPolicy=1
+-XX:+UseThreadPriorities
 
 #################
 #  GC SETTINGS  #
@@ -94,14 +96,21 @@
 --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
 --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
 --add-exports java.sql/java.sql=ALL-UNNAMED
+--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
 
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.lang=ALL-UNNAMED
 --add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens=java.base/java.util=ALL-UNNAMED
 --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
 --add-opens java.base/jdk.internal.ref=ALL-UNNAMED
 --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
 --add-opens java.base/jdk.internal.math=ALL-UNNAMED
 --add-opens java.base/jdk.internal.module=ALL-UNNAMED
 --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
 --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
 
 
@@ -110,7 +119,7 @@
 # Java 11 (and newer) GC logging options:
 # See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
 # The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
-#-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+#-Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
 
 # Notes for Java 8 migration:
 #
diff --git a/conf/jvm17-clients.options b/conf/jvm17-clients.options
index 671d91b21f95..36e15c838fce 100644
--- a/conf/jvm17-clients.options
+++ b/conf/jvm17-clients.options
@@ -28,6 +28,8 @@
 
 -Djdk.attach.allowAttachSelf=true
 --add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
 --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
 --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
 --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
diff --git a/conf/jvm17-server.options b/conf/jvm17-server.options
index 1a0f8f9c565f..567a9b76ea57 100644
--- a/conf/jvm17-server.options
+++ b/conf/jvm17-server.options
@@ -22,6 +22,9 @@
 # See jvm-server.options. This file is specific for Java 17 and newer.    #
 ###########################################################################
 
+-XX:ThreadPriorityPolicy=1
+-XX:+UseThreadPriorities
+
 #################
 #  GC SETTINGS  #
 #################
@@ -43,7 +46,7 @@
 ## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
 ## 200ms is the JVM default and lowest viable setting
 ## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
--XX:MaxGCPauseMillis=300
+-XX:MaxGCPauseMillis=500
 
 ## Optional G1 Settings
 # Save CPU time on large (>= 16GB) heaps by delaying region scanning
@@ -63,6 +66,8 @@
 
 -Djdk.attach.allowAttachSelf=true
 --add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
 --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
 --add-exports java.management/com.sun.jmx.remote.security=ALL-UNNAMED
 --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
@@ -70,8 +75,8 @@
 --add-exports java.sql/java.sql=ALL-UNNAMED
 --add-exports java.base/java.lang.ref=ALL-UNNAMED
 --add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
 
---add-opens java.base/java.lang.module=ALL-UNNAMED
 --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
 --add-opens java.base/jdk.internal.ref=ALL-UNNAMED
 --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
@@ -83,15 +88,20 @@
 --add-opens java.base/java.io=ALL-UNNAMED
 --add-opens java.base/java.lang.reflect=ALL-UNNAMED
 --add-opens java.base/java.lang=ALL-UNNAMED
+--add-opens=java.base/java.nio.charset=ALL-UNNAMED
 --add-opens java.base/java.util=ALL-UNNAMED
 --add-opens java.base/java.nio=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
+
+# required for org.apache.cassandra.Util.getSupportedMTimeGranularity
+--add-opens java.base/java.nio.file.attribute=ALL-UNNAMED
 
 ### GC logging options -- uncomment to enable
 
 # Java 11 (and newer) GC logging options:
 # See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
 # The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
-#-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+# -Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
 
 # Notes for Java 8 migration:
 #
@@ -114,5 +124,12 @@
 # Revert changes in defaults introduced in https://netty.io/news/2022/03/10/4-1-75-Final.html
 -Dio.netty.allocator.useCacheForAllThreads=true
 -Dio.netty.allocator.maxOrder=11
+### Enable vector incubator feature (simd support)
+
+--add-modules jdk.incubator.vector
+
+### Compatibility Options
+--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED
+-Djava.security.manager=allow
 
 # The newline in the end of file is intentional
diff --git a/conf/jvm22-clients.options b/conf/jvm22-clients.options
new file mode 100644
index 000000000000..81af895ed216
--- /dev/null
+++ b/conf/jvm22-clients.options
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###########################################################################
+#                         jvm22-clients.options                           #
+#                                                                         #
+# See jvm-clients.options. This file is specific for Java 22 and newer.   #
+###########################################################################
+
+###################
+#  JPMS SETTINGS  #
+###################
+
+-Djdk.attach.allowAttachSelf=true
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
+--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+--add-exports java.sql/java.sql=ALL-UNNAMED
+--add-exports jdk.attach/sun.tools.attach=ALL-UNNAMED
+
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens java.base/jdk.internal.math=ALL-UNNAMED
+--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+--add-opens java.base/sun.nio.ch=ALL-UNNAMED
+--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+
+# The newline in the end of file is intentional
diff --git a/conf/jvm22-server.options b/conf/jvm22-server.options
new file mode 100644
index 000000000000..b836204660bb
--- /dev/null
+++ b/conf/jvm22-server.options
@@ -0,0 +1,128 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###########################################################################
+#                         jvm22-server.options                            #
+#                                                                         #
+# See jvm-server.options. This file is specific for Java 22 and newer.    #
+###########################################################################
+
+#################
+#  GC SETTINGS  #
+#################
+
+### G1 Settings
+## Use the Hotspot garbage-first collector.
+-XX:+UseG1GC
+-XX:+ParallelRefProcEnabled
+
+#
+## Have the JVM do less remembered set work during STW, instead
+## preferring concurrent GC. Reduces p99.9 latency.
+-XX:G1RSetUpdatingPauseTimePercent=5
+#
+## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
+## 200ms is the JVM default and lowest viable setting
+## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
+-XX:MaxGCPauseMillis=500
+
+## Optional G1 Settings
+# Save CPU time on large (>= 16GB) heaps by delaying region scanning
+# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
+#-XX:InitiatingHeapOccupancyPercent=70
+
+# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
+# Otherwise equal to the number of cores when 8 or less.
+# Machines with > 10 cores should try setting these to <= full cores.
+#-XX:ParallelGCThreads=16
+# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
+# Setting both to the same value can reduce STW durations.
+#-XX:ConcGCThreads=16
+
+
+### JPMS
+
+-Djdk.attach.allowAttachSelf=true
+-Djava.security.manager=allow
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/jdk.internal.perf=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
+--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports jdk.naming.dns/com.sun.jndi.dns=java.naming
+--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/java.lang=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens java.base/java.nio.charset=ALL-UNNAMED
+--add-opens java.base/java.nio.file.spi=ALL-UNNAMED
+--add-opens java.base/java.nio=ALL-UNNAMED
+--add-opens java.base/java.net=ALL-UNNAMED
+--add-opens java.base/java.util=ALL-UNNAMED
+--add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED
+--add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens java.base/jdk.internal.math=ALL-UNNAMED
+--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+--add-opens java.base/jdk.internal.vm=ALL-UNNAMED
+--add-opens java.base/sun.nio.ch=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
+--add-opens jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED
+--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+--add-opens jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED
+
+# required for org.apache.cassandra.Util.getSupportedMTimeGranularity
+--add-opens java.base/java.nio.file.attribute=ALL-UNNAMED
+
+### GC logging options -- uncomment to enable
+
+# Java 11 (and newer) GC logging options:
+# See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
+# The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
+# -Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+
+# Notes for Java 8 migration:
+#
+# -XX:+PrintGCDetails                   maps to -Xlog:gc*:... - i.e. add a '*' after "gc"
+# -XX:+PrintGCDateStamps                maps to decorator 'time'
+#
+# -XX:+PrintHeapAtGC                    maps to 'heap' with level 'trace'
+# -XX:+PrintTenuringDistribution        maps to 'age' with level 'debug'
+# -XX:+PrintGCApplicationStoppedTime    maps to 'safepoint' with level 'info'
+# -XX:+PrintPromotionFailure            maps to 'promotion' with level 'trace'
+# -XX:PrintFLSStatistics=1              maps to 'freelist' with level 'trace'
+
+### Netty Options
+
+# On Java >= 9 Netty requires the io.netty.tryReflectionSetAccessible system property to be set to true to enable
+# creation of direct buffers using Unsafe. Without it, this falls back to ByteBuffer.allocateDirect which has
+# inferior performance and risks exceeding MaxDirectMemory
+-Dio.netty.tryReflectionSetAccessible=true
+
+### Enable vector incubator feature (simd support)
+
+--add-modules jdk.incubator.vector
+
+# The newline in the end of file is intentional
diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile
index 959533f77186..314d55acbcf3 100644
--- a/doc/cql3/CQL.textile
+++ b/doc/cql3/CQL.textile
@@ -721,6 +721,8 @@ bc(syntax)..
                             '(' <arg-name> <arg-type> ( ',' <arg-name> <arg-type> )* ')'
                             ( CALLED | RETURNS NULL ) ON NULL INPUT
                             RETURNS <type>
+                            ( DETERMINISTIC )?
+                            ( MONOTONIC ( ON <arg-name> )? )?
                             LANGUAGE <language>
                             AS <body>
 p. 
@@ -766,6 +768,10 @@ If the optional @IF NOT EXISTS@ keywords are used, the function will only be cre
 
 @OR REPLACE@ and @IF NOT EXIST@ cannot be used together.
 
+The optional @DETERMINISTIC@ keyword specifies that the function is deterministic. This means that given a particular input, the function will always produce the same output.
+
+The optional @MONOTONIC@ keyword specifies that the function is monotonic. This means that it is either entirely nonincreasing or nondecreasing. Even if the function is not monotonic on all its arguments, it is possible to specify that it is monotonic @ON@ one of its arguments, meaning that partial applications of the function over that argument will be monotonic. Monotonicity is required to use the function in a @GROUP BY@ clause.
+
 Functions belong to a keyspace. If no keyspace is specified in @<function-name>@, the current keyspace is used (i.e. the keyspace specified using the "@USE@":#useStmt statement). It is not possible to create a user-defined function in one of the system keyspaces.
 
 See the section on "user-defined functions":#udfs for more information.
@@ -806,6 +812,7 @@ bc(syntax)..
                             STYPE <state-type>
                             ( FINALFUNC <final-functionname> )?
                             ( INITCOND <init-cond> )?
+                            ( DETERMINISTIC )?
 p. 
 __Sample:__
 
@@ -826,6 +833,8 @@ See the section on "user-defined aggregates":#udas for a complete example.
 
 @OR REPLACE@ and @IF NOT EXIST@ cannot be used together.
 
+The optional @DETERMINISTIC@ keyword specifies that the aggregate function is deterministic. This means that given a particular input, the function will always produce the same output.
+
 Aggregates belong to a keyspace. If no keyspace is specified in @<aggregate-name>@, the current keyspace is used (i.e. the keyspace specified using the "@USE@":#useStmt statement). It is not possible to create a user-defined aggregate in one of the system keyspaces.
 
 Signatures for user-defined aggregates follow the "same rules":#functionSignature as for user-defined functions.
@@ -1092,8 +1101,9 @@ bc(syntax)..
                   ( GROUP BY <group-by>)?
                   ( ORDER BY <order-by> )?
                   ( PER PARTITION LIMIT <integer> )?
-                  ( LIMIT <integer> )?
+                  ( LIMIT <integer> ( OFFSET <integer> )? )?
                   ( ALLOW FILTERING )?
+                  ( WITH ann_options = <map-literal> )?
 
 <select-clause> ::= DISTINCT? <selection-list>
 
@@ -1228,9 +1238,9 @@ Aggregate functions will produce a separate value for each group. If no @GROUP B
 
 If a column is selected without an aggregate function, in a statement with a @GROUP BY@, the first value encounter in each group will be returned.
 
-h4(#selectLimit). @LIMIT@ and @PER PARTITION LIMIT@
+h4(#selectLimit). @LIMIT@, @OFFSET@ and @PER PARTITION LIMIT@
 
-The @LIMIT@ option to a @SELECT@ statement limits the number of rows returned by a query, while the @PER PARTITION LIMIT@ option limits the number of rows returned for a given partition by the query. Note that both type of limit can used in the same statement.
+The @LIMIT@ option in a @SELECT@ statement limits the number of rows returned by a query. The @LIMIT@ option can include an @OFFSET@ option to skip the first rows of the query result. The @PER PARTITION LIMIT@ option limits the number of rows returned for a given partition by the query. Note that both type of limit can used in the same statement.
 
 h4(#selectAllowFiltering). @ALLOW FILTERING@
 
diff --git a/doc/modules/cassandra/examples/BNF/create_aggregate_statement.bnf b/doc/modules/cassandra/examples/BNF/create_aggregate_statement.bnf
index c0126a23ffd8..1207ec06328c 100644
--- a/doc/modules/cassandra/examples/BNF/create_aggregate_statement.bnf
+++ b/doc/modules/cassandra/examples/BNF/create_aggregate_statement.bnf
@@ -4,3 +4,4 @@ create_aggregate_statement ::= CREATE [ OR REPLACE ] AGGREGATE [ IF NOT EXISTS ]
                                 STYPE cql_type:
                                 [ FINALFUNC function_name]
                                 [ INITCOND term ]
+                                [ DETERMINISTIC ]
diff --git a/doc/modules/cassandra/examples/BNF/create_function_statement.bnf b/doc/modules/cassandra/examples/BNF/create_function_statement.bnf
index 0da769a11fb0..82be39d42911 100644
--- a/doc/modules/cassandra/examples/BNF/create_function_statement.bnf
+++ b/doc/modules/cassandra/examples/BNF/create_function_statement.bnf
@@ -1,6 +1,8 @@
 create_function_statement::= CREATE [ OR REPLACE ] FUNCTION [ IF NOT EXISTS] 
 	function_name '(' arguments_declaration ')' 
 	[ CALLED | RETURNS NULL ] ON NULL INPUT 
-	RETURNS cql_type 
+	RETURNS cql_type
+	[ DETERMINISTIC ]
+	[ MONOTONIC [ ON arg_name ] ]
 	LANGUAGE identifier 
 	AS string arguments_declaration: identifier cql_type ( ',' identifier cql_type )*
diff --git a/doc/modules/cassandra/examples/BNF/select_statement.bnf b/doc/modules/cassandra/examples/BNF/select_statement.bnf
index f53da41da57c..ff630209cbd0 100644
--- a/doc/modules/cassandra/examples/BNF/select_statement.bnf
+++ b/doc/modules/cassandra/examples/BNF/select_statement.bnf
@@ -4,8 +4,9 @@ select_statement::= SELECT [ JSON | DISTINCT ] ( select_clause | '*' )
 	[ GROUP BY `group_by_clause` ]  
 	[ ORDER BY `ordering_clause` ]  
 	[ PER PARTITION LIMIT (`integer` | `bind_marker`) ]  
-	[ LIMIT (`integer` | `bind_marker`) ]  
+	[ LIMIT (`integer` | `bind_marker`) [ OFFSET (`integer` | `bind_marker`) ] ]
 	[ ALLOW FILTERING ]
+	[ WITH ann_options = map-literal ]
 select_clause::= `selector` [ AS `identifier` ] ( ',' `selector` [ AS `identifier` ] ) 
 selector::== `column_name` 
 	| `term`  
diff --git a/doc/modules/cassandra/examples/CQL/query_with_ann_options.cql b/doc/modules/cassandra/examples/CQL/query_with_ann_options.cql
new file mode 100644
index 000000000000..14ddcd47b938
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/query_with_ann_options.cql
@@ -0,0 +1 @@
+SELECT * FROM embeddings ORDER BY vector ANN OF [1.2, 3.4] LIMIT 100 WITH ann_options = { 'rerank_k': 1000 }
diff --git a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc
index 18dd52e13dd7..c21c0aa3e537 100644
--- a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc
@@ -1155,6 +1155,8 @@ CREATE FUNCTION akeyspace.fname IF NOT EXISTS +
 ( someArg int ) +
 CALLED ON NULL INPUT +
 RETURNS text +
+( DETERMINISTIC )? +
+( MONOTONIC ( ON <arg-name> )? )? +
 LANGUAGE java +
 AS $$ +
 // some Java code +
@@ -1194,6 +1196,17 @@ exist.
 
 `OR REPLACE` and `IF NOT EXIST` cannot be used together.
 
+The optional `DETERMINISTIC` keyword specifies that the function is
+deterministic. This means that given a particular input, the function
+will always produce the same output.
+
+The optional `MONOTONIC` keyword specifies that the function is monotonic.
+This means that it is either entirely nonincreasing or nondecreasing.
+Even if the function is not monotonic on all its arguments, it is possible
+to specify that it is monotonic `ON` one of its arguments, meaning that
+partial applications of the function over that argument will be monotonic.
+Monotonicity is required to use the function in a `GROUP BY` clause.
+
 Functions belong to a keyspace. If no keyspace is specified in
 `<function-name>`, the current keyspace is used (i.e. the keyspace
 specified using the link:#useStmt[`USE`] statement). It is not possible
@@ -1243,6 +1256,7 @@ SFUNC  +
 STYPE  +
 ( FINALFUNC )? +
 ( INITCOND )? +
+( DETERMINISTIC )? +
 p. +
 _Sample:_
 
@@ -1268,6 +1282,10 @@ creates an aggregate if it does not already exist.
 
 `OR REPLACE` and `IF NOT EXIST` cannot be used together.
 
+The optional `DETERMINISTIC` keyword specifies that the aggregate
+function is deterministic. This means that given a particular input,
+the function will always produce the same output.
+
 Aggregates belong to a keyspace. If no keyspace is specified in
 `<aggregate-name>`, the current keyspace is used (i.e. the keyspace
 specified using the link:#useStmt[`USE`] statement). It is not possible
@@ -1658,8 +1676,9 @@ FROM  +
 ( GROUP BY )? +
 ( ORDER BY )? +
 ( PER PARTITION LIMIT )? +
-( LIMIT )? +
+( LIMIT ( OFFSET )? )? +
 ( ALLOW FILTERING )?
+( WITH ann_options = )?
 
 ::= DISTINCT?
 
@@ -1878,12 +1897,12 @@ with a `GROUP BY`, the first value encounter in each group will be
 returned.
 
 [[selectLimit]]
-===== `LIMIT` and `PER PARTITION LIMIT`
+===== `LIMIT`, `OFFSET` and `PER PARTITION LIMIT`
 
-The `LIMIT` option to a `SELECT` statement limits the number of rows
-returned by a query, while the `PER PARTITION LIMIT` option limits the
-number of rows returned for a given partition by the query. Note that
-both type of limit can used in the same statement.
+The `LIMIT` option in a `SELECT` statement limits the number of rows returned by a query.
+The `LIMIT` option can include an `OFFSET` option to skip the first rows of the query result.
+The `PER PARTITION LIMIT` option limits the number of rows returned for a given partition by the query.
+Note that both types of limits can used in the same statement.
 
 [[selectAllowFiltering]]
 ===== `ALLOW FILTERING`
diff --git a/doc/modules/cassandra/pages/developing/cql/dml.adoc b/doc/modules/cassandra/pages/developing/cql/dml.adoc
index ef76cdbb38ff..ba272a6415eb 100644
--- a/doc/modules/cassandra/pages/developing/cql/dml.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/dml.adoc
@@ -214,9 +214,10 @@ or the reverse
 [[limit-clause]]
 === Limiting results
 
-The `LIMIT` option to a `SELECT` statement limits the number of rows
-returned by a query. The `PER PARTITION LIMIT` option limits the
-number of rows returned for a given partition by the query. Both types of limits can used in the same statement.
+The `LIMIT` option in a `SELECT` statement limits the number of rows returned by a query.
+The `LIMIT` option can include an `OFFSET` option to skip the first rows of the query result.
+The `PER PARTITION LIMIT` option limits the number of rows returned for a given partition by the query.
+Note that both types of limits can used in the same statement.
 
 [[allow-filtering]]
 === Allowing filtering
@@ -264,6 +265,16 @@ execute:
 include::cassandra:example$CQL/query_nofail_allow_filtering.cql[]
 ----
 
+[[ann-options]]
+=== ANN options
+
+`SELECT` queries using `ANN` ordering can provide a set of options to control the behavior of the ANN search:
+
+[source,cql]
+----
+include::example$CQL/query_with_ann_options.cql[]
+----
+
 [[insert-statement]]
 == INSERT
 
diff --git a/doc/modules/cassandra/pages/developing/cql/functions.adoc b/doc/modules/cassandra/pages/developing/cql/functions.adoc
index 75786de271a3..97c51fae6eb2 100644
--- a/doc/modules/cassandra/pages/developing/cql/functions.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/functions.adoc
@@ -288,6 +288,43 @@ A number of functions allow to obtain the similarity score between vectors of fl
 
 include::cassandra:partial$vector-search/vector_functions.adoc[]
 
+[[index-functions]]
+===== Index functions
+
+====== `sai_analyze`
+
+The `sai_analyze` functions returns the tokens that a SAI index will generate for a certain text value. The arguments
+are that text value and the JSON configuration of the SAI analyzer. This JSON configuration is the same as the one used
+to create the SAI index. For example, this function call:
+
+[source,cql]
+----
+sai_analyze('johnny apples seedlings',
+             '{
+                 "tokenizer": {"name": "whitespace"}
+              }')
+----
+Will return `['johnny', 'apples', 'seedlings']`
+
+This other function call:
+[source,cql]
+----
+sai_analyze('johnny apples seedlings',
+             '{
+                 "tokenizer": {"name": "whitespace"},
+                 "filters": [{"name": "porterstem"}]
+              }')
+----
+Will return `['johnni', 'appl', 'seedl']`
+
+
+[[vector-functions]]
+===== Vector functions
+
+A number of functions to operate with vectors of floats.
+
+include::cassandra:partial$vector-search/vector_functions.adoc[]
+
 [[user-defined-scalar-functions]]
 === User-defined functions
 
@@ -378,6 +415,16 @@ If the optional `IF NOT EXISTS` keywords are used, the function will only be cre
 exist.
 `OR REPLACE` and `IF NOT EXISTS` cannot be used together.
 
+The optional `DETERMINISTIC` keyword specifies that the aggregate function is deterministic.
+This means that given a particular input, the function will always produce the same output.
+
+The optional `MONOTONIC` keyword specifies that the function is monotonic.
+This means that it is either entirely nonincreasing or nondecreasing.
+Even if the function is not monotonic on all its arguments, it is possible
+to specify that it is monotonic `ON` one of its arguments, meaning that
+partial applications of the function over that argument will be monotonic.
+Monotonicity is required to use the function in a `GROUP BY` clause.
+
 Behavior for `null` input values must be defined for each function:
 
 * `RETURNS NULL ON NULL INPUT` declares that the function will always return `null` if any of the input arguments is `null`.
@@ -540,6 +587,9 @@ A `CREATE AGGREGATE` without `OR REPLACE` fails if an aggregate with the same si
 The `CREATE AGGREGATE` command with the optional `IF NOT EXISTS` keywords creates an aggregate if it does not already exist.
 The `OR REPLACE` and `IF NOT EXISTS` phrases cannot be used together.
 
+The optional `DETERMINISTIC` keyword specifies that the aggregate function is deterministic.
+This means that given a particular input, the function will always produce the same output.
+
 The `STYPE` value defines the type of the state value and must be specified.
 The optional `INITCOND` defines the initial state value for the aggregate; the default value is `null`.
 A non-null `INITCOND` must be specified for state functions that are declared with `RETURNS NULL ON NULL INPUT`.
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc
index 3a4fb8d54a2c..a31419fc12e9 100644
--- a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc
@@ -28,7 +28,7 @@ here, however, you'll want to think not only from the customer
 perspective in terms of how the data is written, but also in terms of
 how the data will be queried by downstream use cases.
 
-You natural tendency as might be to focus first on designing the tables
+Your natural tendency might be to focus first on designing the tables
 to store reservation and guest records, and only then start thinking
 about the queries that would access them. You may have felt a similar
 tension already when discussing the shopping queries before, thinking
diff --git a/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc b/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc
index 63f4ba1a1130..f728bfd73001 100644
--- a/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc
+++ b/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc
@@ -150,8 +150,12 @@ auditlogviewer <path1> [<path2>...<pathN>] [options]
     waiting for more records
 `-r,--roll_cycle`::
   How often to roll the log file was rolled. May be;;
-    necessary for Chronicle to correctly parse file names. (MINUTELY,
-    HOURLY, DAILY). Default HOURLY.
+    necessary for Chronicle to correctly parse file names. Some available options are:
+FIVE_MINUTELY, FAST_HOURLY, FAST_DAILY, LargeRollCycles.LARGE_DAILY, LargeRollCycles.XLARGE_DAILY,
+LargeRollCycles.HUGE_DAILY. Deprecated ones still availble but not recommended for new deployments:
+MINUTELY, HOURLY, DAILY
+For more options, refer: net.openhft.chronicle.queue.RollCycles.
+Default is set to FAST_HOURLY
 `-h,--help`::
   display this help message
 
diff --git a/doc/modules/cassandra/partials/vector-search/vector_functions.adoc b/doc/modules/cassandra/partials/vector-search/vector_functions.adoc
index daa4b2b8ce22..e73fc628b466 100644
--- a/doc/modules/cassandra/partials/vector-search/vector_functions.adoc
+++ b/doc/modules/cassandra/partials/vector-search/vector_functions.adoc
@@ -30,12 +30,38 @@ Examples:
 
 Examples:
 
-`similarity_dot_product([0.1, 0.2], null)` -> `null`
+`similarity_dot_product([0.447214, 0.894427], null)` -> `null`
 
-`similarity_dot_product([0.1, 0.2], [0.1, 0.2])` -> `0.525`
+`similarity_dot_product([0.447214, 0.894427], [0.447214, 0.894427])` -> `1`
 
-`similarity_dot_product([0.1, 0.2], [-0.1, -0.2])` -> `0.475`
+`similarity_dot_product([0.447214, 0.894427], [-0.447214, -0.894427])` -> `0`
 
-`similarity_dot_product([0.1, 0.2], [0.9, 0.8])` -> `0.625`
+`similarity_dot_product([0.447214, 0.894427], [-0.447214, 0.894427])` -> `0.8`
+
+`similarity_dot_product([0.447214, 0.894427], [0.447214, -0.894427])` -> `0.2`
+
+| `random_float_vector(int, float, float)` | Returns a new vector of floats with the specified dimension and where
+all components will be in the specified min-max range.
+
+Examples:
+
+`random_float_vector(2, -1.0, 1.0)` -> `[-0.695395, -0.395755]`
+
+`random_float_vector(2, -1.0, 1.0)` -> `[-0.58795, 0.690014]`
+
+`random_float_vector(2, 0.0, 1.0)` -> `[0.423859, 0.630168]`
+
+`random_float_vector(2, 0.0, 1.0)` -> `[0.468159, 0.283808]`
+
+| `normalize_l2(vector)` | Applies L2 normalization to the input vector.
+The result is a vector with the same direction but with a magnitude of 1.
+
+Examples:
+
+`normalize_l2([0.1])` -> `[1]`
+
+`normalize_l2([-0.7])` -> `[1]`
+
+`normalize_l2([3.0, 4.0])` -> `[0.6, 0.8]`
 
 |===
\ No newline at end of file
diff --git a/doc/native_protocol_v4.1.spec b/doc/native_protocol_v4.1.spec
new file mode 100644
index 000000000000..a10fd2404d8f
--- /dev/null
+++ b/doc/native_protocol_v4.1.spec
@@ -0,0 +1,1212 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+                             CQL BINARY PROTOCOL v4.1
+
+
+Table of Contents
+
+  1. Overview
+  2. Frame header
+    2.1. version
+    2.2. flags
+    2.3. stream
+    2.4. opcode
+    2.5. length
+  3. Notations
+  4. Messages
+    4.1. Requests
+      4.1.1. STARTUP
+      4.1.2. AUTH_RESPONSE
+      4.1.3. OPTIONS
+      4.1.4. QUERY
+      4.1.5. PREPARE
+      4.1.6. EXECUTE
+      4.1.7. BATCH
+      4.1.8. REGISTER
+    4.2. Responses
+      4.2.1. ERROR
+      4.2.2. READY
+      4.2.3. AUTHENTICATE
+      4.2.4. SUPPORTED
+      4.2.5. RESULT
+        4.2.5.1. Void
+        4.2.5.2. Rows
+        4.2.5.3. Set_keyspace
+        4.2.5.4. Prepared
+        4.2.5.5. Schema_change
+      4.2.6. EVENT
+      4.2.7. AUTH_CHALLENGE
+      4.2.8. AUTH_SUCCESS
+  5. Compression
+  6. Data Type Serialization Formats
+  7. User Defined Type Serialization
+  8. Result paging
+  9. Error codes
+  10. Changes from v4
+
+
+1. Overview
+
+  The CQL binary protocol is a frame based protocol. Frames are defined as:
+
+      0         8        16        24        32         40
+      +---------+---------+---------+---------+---------+
+      | version |  flags  |      stream       | opcode  |
+      +---------+---------+---------+---------+---------+
+      |                length                 |
+      +---------+---------+---------+---------+
+      |                                       |
+      .            ...  body ...              .
+      .                                       .
+      .                                       .
+      +----------------------------------------
+
+  The protocol is big-endian (network byte order).
+
+  Each frame contains a fixed size header (9 bytes) followed by a variable size
+  body. The header is described in Section 2. The content of the body depends
+  on the header opcode value (the body can in particular be empty for some
+  opcode values). The list of allowed opcodes is defined in Section 2.4 and the
+  details of each corresponding message are described Section 4.
+
+  The protocol distinguishes two types of frames: requests and responses. Requests
+  are those frames sent by the client to the server. Responses are those frames sent
+  by the server to the client. Note, however, that the protocol supports server pushes
+  (events) so a response does not necessarily come right after a client request.
+
+  Note to client implementors: client libraries should always assume that the
+  body of a given frame may contain more data than what is described in this
+  document. It will however always be safe to ignore the remainder of the frame
+  body in such cases. The reason is that this may enable extending the protocol
+  with optional features without needing to change the protocol version.
+
+
+
+2. Frame header
+
+2.1. version
+
+  The version is a single byte that indicates both the direction of the message
+  (request or response) and the version of the protocol in use. The most
+  significant bit of version is used to define the direction of the message:
+  0 indicates a request, 1 indicates a response. This can be useful for protocol
+  analyzers to distinguish the nature of the packet from the direction in which
+  it is moving. The rest of that byte is the protocol version (4 for the protocol
+  defined in this document). In other words, for this version of the protocol,
+  version will be one of:
+    0x04    Request frame for this protocol version
+    0x84    Response frame for this protocol version
+
+  Please note that while every message ships with the version, only one version
+  of messages is accepted on a given connection. In other words, the first message
+  exchanged (STARTUP) sets the version for the connection for the lifetime of this
+  connection.
+
+  This document describes version 4 of the protocol. For the changes made since
+  version 3, see Section 10.
+
+
+2.2. flags
+
+  Flags applying to this frame. The flags have the following meaning (described
+  by the mask that allows selecting them):
+    0x01: Compression flag. If set, the frame body is compressed. The actual
+          compression to use should have been set up beforehand through the
+          Startup message (which thus cannot be compressed; Section 4.1.1).
+    0x02: Tracing flag. For a request frame, this indicates the client requires
+          tracing of the request. Note that only QUERY, PREPARE and EXECUTE queries
+          support tracing. Other requests will simply ignore the tracing flag if 
+          set. If a request supports tracing and the tracing flag is set, the response
+          to this request will have the tracing flag set and contain tracing
+          information.
+          If a response frame has the tracing flag set, its body contains
+          a tracing ID. The tracing ID is a [uuid] and is the first thing in
+          the frame body.
+    0x04: Custom payload flag. For a request or response frame, this indicates
+          that a generic key-value custom payload for a custom QueryHandler
+          implementation is present in the frame. Such a custom payload is simply
+          ignored by the default QueryHandler implementation.
+          Currently, only QUERY, PREPARE, EXECUTE and BATCH requests support
+          payload.
+          Type of custom payload is [bytes map] (see below). If either or both
+          of the tracing and warning flags are set, the custom payload will follow
+          those indicated elements in the frame body. If neither are set, the custom
+          payload will be the first value in the frame body.
+    0x08: Warning flag. The response contains warnings which were generated by the
+          server to go along with this response.
+          If a response frame has the warning flag set, its body will contain the
+          text of the warnings. The warnings are a [string list] and will be the
+          first value in the frame body if the tracing flag is not set, or directly
+          after the tracing ID if it is.
+
+  The rest of flags is currently unused and ignored.
+
+2.3. stream
+
+  A frame has a stream id (a [short] value). When sending request messages, this
+  stream id must be set by the client to a non-negative value (negative stream id
+  are reserved for streams initiated by the server; currently all EVENT messages
+  (section 4.2.6) have a streamId of -1). If a client sends a request message
+  with the stream id X, it is guaranteed that the stream id of the response to
+  that message will be X.
+
+  This helps to enable the asynchronous nature of the protocol. If a client
+  sends multiple messages simultaneously (without waiting for responses), there
+  is no guarantee on the order of the responses. For instance, if the client
+  writes REQ_1, REQ_2, REQ_3 on the wire (in that order), the server might
+  respond to REQ_3 (or REQ_2) first. Assigning different stream ids to these 3
+  requests allows the client to distinguish to which request a received answer
+  responds to. As there can only be 32768 different simultaneous streams, it is up
+  to the client to reuse stream id.
+
+  Note that clients are free to use the protocol synchronously (i.e. wait for
+  the response to REQ_N before sending REQ_N+1). In that case, the stream id
+  can be safely set to 0. Clients should also feel free to use only a subset of
+  the 32768 maximum possible stream ids if it is simpler for its implementation.
+
+2.4. opcode
+
+  An integer byte that distinguishes the actual message:
+    0x00    ERROR
+    0x01    STARTUP
+    0x02    READY
+    0x03    AUTHENTICATE
+    0x05    OPTIONS
+    0x06    SUPPORTED
+    0x07    QUERY
+    0x08    RESULT
+    0x09    PREPARE
+    0x0A    EXECUTE
+    0x0B    REGISTER
+    0x0C    EVENT
+    0x0D    BATCH
+    0x0E    AUTH_CHALLENGE
+    0x0F    AUTH_RESPONSE
+    0x10    AUTH_SUCCESS
+
+  Messages are described in Section 4.
+
+  (Note that there is no 0x04 message in this version of the protocol)
+
+
+2.5. length
+
+  A 4 byte integer representing the length of the body of the frame (note:
+  currently a frame is limited to 256MB in length).
+
+
+3. Notations
+
+  To describe the layout of the frame body for the messages in Section 4, we
+  define the following:
+
+    [int]          A 4 bytes integer
+    [long]         A 8 bytes integer
+    [short]        A 2 bytes unsigned integer
+    [string]       A [short] n, followed by n bytes representing an UTF-8
+                   string.
+    [long string]  An [int] n, followed by n bytes representing an UTF-8 string.
+    [uuid]         A 16 bytes long uuid.
+    [string list]  A [short] n, followed by n [string].
+    [bytes]        A [int] n, followed by n bytes if n >= 0. If n < 0,
+                   no byte should follow and the value represented is `null`.
+    [value]        A [int] n, followed by n bytes if n >= 0.
+                   If n == -1 no byte should follow and the value represented is `null`.
+                   If n == -2 no byte should follow and the value represented is
+                   `not set` not resulting in any change to the existing value.
+                   n < -2 is an invalid value and results in an error.
+    [short bytes]  A [short] n, followed by n bytes if n >= 0.
+
+    [option]       A pair of <id><value> where <id> is a [short] representing
+                   the option id and <value> depends on that option (and can be
+                   of size 0). The supported id (and the corresponding <value>)
+                   will be described when this is used.
+    [option list]  A [short] n, followed by n [option].
+    [inet]         An address (ip and port) to a node. It consists of one
+                   [byte] n, that represents the address size, followed by n
+                   [byte] representing the IP address (in practice n can only be
+                   either 4 (IPv4) or 16 (IPv6)), following by one [int]
+                   representing the port.
+    [consistency]  A consistency level specification. This is a [short]
+                   representing a consistency level with the following
+                   correspondance:
+                     0x0000    ANY
+                     0x0001    ONE
+                     0x0002    TWO
+                     0x0003    THREE
+                     0x0004    QUORUM
+                     0x0005    ALL
+                     0x0006    LOCAL_QUORUM
+                     0x0007    EACH_QUORUM
+                     0x0008    SERIAL
+                     0x0009    LOCAL_SERIAL
+                     0x000A    LOCAL_ONE
+
+    [string map]      A [short] n, followed by n pair <k><v> where <k> and <v>
+                      are [string].
+    [string multimap] A [short] n, followed by n pair <k><v> where <k> is a
+                      [string] and <v> is a [string list].
+    [bytes map]       A [short] n, followed by n pair <k><v> where <k> is a
+                      [string] and <v> is a [bytes].
+
+
+4. Messages
+
+  Dependant on the flags specified in the header, the layout of the message body must be:
+    [<tracing_id>][<warnings>][<custom_payload>]<message>
+  where:
+    - <tracing_id> is a UUID tracing ID, present if this is a request message and the Tracing flag is set.
+    - <warnings> is a string list of warnings (if this is a request message and the Warning flag is set.
+    - <custom_payload> is bytes map for the serialised custom payload present if this is one of the message types
+      which support custom payloads (QUERY, PREPARE, EXECUTE and BATCH) and the Custom payload flag is set.
+    - <message> as defined below through sections 4 and 5.
+
+4.1. Requests
+
+  Note that outside of their normal responses (described below), all requests
+  can get an ERROR message (Section 4.2.1) as response.
+
+4.1.1. STARTUP
+
+  Initialize the connection. The server will respond by either a READY message
+  (in which case the connection is ready for queries) or an AUTHENTICATE message
+  (in which case credentials will need to be provided using AUTH_RESPONSE).
+
+  This must be the first message of the connection, except for OPTIONS that can
+  be sent before to find out the options supported by the server. Once the
+  connection has been initialized, a client should not send any more STARTUP
+  messages.
+
+  The body is a [string map] of options. Possible options are:
+    - "CQL_VERSION": the version of CQL to use. This option is mandatory and
+      currently the only version supported is "3.0.0". Note that this is
+      different from the protocol version.
+    - "COMPRESSION": the compression algorithm to use for frames (See section 5).
+      This is optional; if not specified no compression will be used.
+    - "NO_COMPACT": whether or not connection has to be established in compatibility
+      mode. This mode will make all Thrift and Compact Tables to be exposed as if
+      they were CQL Tables. This is optional; if not specified, the option will
+      not be used.
+    - "THROW_ON_OVERLOAD": In case of server overloaded with too many requests, by default the server puts
+            back pressure on the client connection. Instead, the server can send an OverloadedException error message back to
+            the client if this option is set to true.
+    - "PAGE_UNIT": a list of supported page units.
+
+
+4.1.2. AUTH_RESPONSE
+
+  Answers a server authentication challenge.
+
+  Authentication in the protocol is SASL based. The server sends authentication
+  challenges (a bytes token) to which the client answers with this message. Those
+  exchanges continue until the server accepts the authentication by sending a
+  AUTH_SUCCESS message after a client AUTH_RESPONSE. Note that the exchange
+  begins with the client sending an initial AUTH_RESPONSE in response to a
+  server AUTHENTICATE request.
+
+  The body of this message is a single [bytes] token. The details of what this
+  token contains (and when it can be null/empty, if ever) depends on the actual
+  authenticator used.
+
+  The response to a AUTH_RESPONSE is either a follow-up AUTH_CHALLENGE message,
+  an AUTH_SUCCESS message or an ERROR message.
+
+
+4.1.3. OPTIONS
+
+  Asks the server to return which STARTUP options are supported. The body of an
+  OPTIONS message should be empty and the server will respond with a SUPPORTED
+  message.
+
+
+4.1.4. QUERY
+
+  Performs a CQL query. The body of the message must be:
+    <query><query_parameters>
+  where <query> is a [long string] representing the query and
+  <query_parameters> must be
+    <consistency><flags>[<n>[name_1]<value_1>...[name_n]<value_n>][<result_page_size>][<paging_state>][<serial_consistency>][<timestamp>]
+  where:
+    - <consistency> is the [consistency] level for the operation.
+    - <flags> is a [byte] whose bits define the options for this query and
+      in particular influence what the remainder of the message contains.
+      A flag is set if the bit corresponding to its `mask` is set. Supported
+      flags are, given their mask:
+        0x00000001: Values. If set, a [short] <n> followed by <n> [value]
+                    values are provided. Those values are used for bound variables in
+                    the query. Optionally, if the 0x40 flag is present, each value
+                    will be preceded by a [string] name, representing the name of
+                    the marker the value must be bound to.
+        0x00000002: Skip_metadata. If set, the Result Set returned as a response
+                    to the query (if any) will have the NO_METADATA flag (see
+                    Section 4.2.5.2).
+        0x00000004: Page_size. If set, <result_page_size> is an [int]
+                    controlling the desired page size of the result (in CQL3 rows or bytes).
+                    See the section on paging (Section 8) for more details.
+        0x00000008: With_paging_state. If set, <paging_state> should be present.
+                    <paging_state> is a [bytes] value that should have been returned
+                    in a result set (Section 4.2.5.2). The query will be
+                    executed but starting from a given paging state. This is also to
+                    continue paging on a different node than the one where it
+                    started (See Section 8 for more details).
+        0x00000010: With serial consistency. If set, <serial_consistency> should be
+                    present. <serial_consistency> is the [consistency] level for the
+                    serial phase of conditional updates. That consitency can only be
+                    either SERIAL or LOCAL_SERIAL and if not present, it defaults to
+                    SERIAL. This option will be ignored for anything else other than a
+                    conditional update/insert.
+        0x00000020: With default timestamp. If set, <timestamp> should be present.
+                    <timestamp> is a [long] representing the default timestamp for the query
+                    in microseconds (negative values are forbidden). This will
+                    replace the server side assigned timestamp as default timestamp.
+                    Note that a timestamp in the query itself will still override
+                    this timestamp. This is entirely optional.
+        0x00000040: With names for values. This only makes sense if the 0x01 flag is set and
+                    is ignored otherwise. If present, the values from the 0x01 flag will
+                    be preceded by a name (see above). Note that this is only useful for
+                    QUERY requests where named bind markers are used; for EXECUTE statements,
+                    since the names for the expected values was returned during preparation,
+                    a client can always provide values in the right order without any names
+                    and using this flag, while supported, is almost surely inefficient.
+        0x40000000: When set, the <page_size> is provided in bytes rather than in rows.
+
+
+  Note that the consistency is ignored by some queries (USE, CREATE, ALTER,
+  TRUNCATE, ...).
+
+  The server will respond to a QUERY message with a RESULT message, the content
+  of which depends on the query.
+
+
+4.1.5. PREPARE
+
+  Prepare a query for later execution (through EXECUTE). The body consists of
+  the CQL query to prepare as a [long string].
+
+  The server will respond with a RESULT message with a `prepared` kind (0x0004,
+  see Section 4.2.5).
+
+
+4.1.6. EXECUTE
+
+  Executes a prepared query. The body of the message must be:
+    <id><query_parameters>
+  where <id> is the prepared query ID. It's the [short bytes] returned as a
+  response to a PREPARE message. As for <query_parameters>, it has the exact
+  same definition as in QUERY (see Section 4.1.4).
+
+  The response from the server will be a RESULT message.
+
+
+4.1.7. BATCH
+
+  Allows executing a list of queries (prepared or not) as a batch (note that
+  only DML statements are accepted in a batch). The body of the message must
+  be:
+    <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>]
+  where:
+    - <type> is a [byte] indicating the type of batch to use:
+        - If <type> == 0, the batch will be "logged". This is equivalent to a
+          normal CQL3 batch statement.
+        - If <type> == 1, the batch will be "unlogged".
+        - If <type> == 2, the batch will be a "counter" batch (and non-counter
+          statements will be rejected).
+    - <flags> is a [byte] whose bits define the options for this query and
+      in particular influence what the remainder of the message contains. It is similar
+      to the <flags> from QUERY and EXECUTE methods, except that the 4 rightmost
+      bits must always be 0 as their corresponding options do not make sense for
+      Batch. A flag is set if the bit corresponding to its `mask` is set. Supported
+      flags are, given their mask:
+        0x10: With serial consistency. If set, <serial_consistency> should be
+              present. <serial_consistency> is the [consistency] level for the
+              serial phase of conditional updates. That consistency can only be
+              either SERIAL or LOCAL_SERIAL and if not present, it defaults to
+              SERIAL. This option will be ignored for anything else other than a
+              conditional update/insert.
+        0x20: With default timestamp. If set, <timestamp> should be present.
+              <timestamp> is a [long] representing the default timestamp for the query
+              in microseconds. This will replace the server side assigned
+              timestamp as default timestamp. Note that a timestamp in the query itself
+              will still override this timestamp. This is entirely optional.
+        0x40: With names for values. If set, then all values for all <query_i> must be
+              preceded by a [string] <name_i> that have the same meaning as in QUERY
+              requests [IMPORTANT NOTE: this feature does not work and should not be
+              used. It is specified in a way that makes it impossible for the server
+              to implement. This will be fixed in a future version of the native
+              protocol. See https://issues.apache.org/jira/browse/CASSANDRA-10246 for
+              more details].
+    - <n> is a [short] indicating the number of following queries.
+    - <query_1>...<query_n> are the queries to execute. A <query_i> must be of the
+      form:
+        <kind><string_or_id><n>[<name_1>]<value_1>...[<name_n>]<value_n>
+      where:
+       - <kind> is a [byte] indicating whether the following query is a prepared
+         one or not. <kind> value must be either 0 or 1.
+       - <string_or_id> depends on the value of <kind>. If <kind> == 0, it should be
+         a [long string] query string (as in QUERY, the query string might contain
+         bind markers). Otherwise (that is, if <kind> == 1), it should be a
+         [short bytes] representing a prepared query ID.
+       - <n> is a [short] indicating the number (possibly 0) of following values.
+       - <name_i> is the optional name of the following <value_i>. It must be present
+         if and only if the 0x40 flag is provided for the batch.
+       - <value_i> is the [value] to use for bound variable i (of bound variable <name_i>
+         if the 0x40 flag is used).
+    - <consistency> is the [consistency] level for the operation.
+    - <serial_consistency> is only present if the 0x10 flag is set. In that case,
+      <serial_consistency> is the [consistency] level for the serial phase of
+      conditional updates. That consitency can only be either SERIAL or
+      LOCAL_SERIAL and if not present will defaults to SERIAL. This option will
+      be ignored for anything else other than a conditional update/insert.
+
+  The server will respond with a RESULT message.
+
+
+4.1.8. REGISTER
+
+  Register this connection to receive some types of events. The body of the
+  message is a [string list] representing the event types to register for. See
+  section 4.2.6 for the list of valid event types.
+
+  The response to a REGISTER message will be a READY message.
+
+  Please note that if a client driver maintains multiple connections to a
+  Cassandra node and/or connections to multiple nodes, it is advised to
+  dedicate a handful of connections to receive events, but to *not* register
+  for events on all connections, as this would only result in receiving
+  multiple times the same event messages, wasting bandwidth.
+
+
+4.2. Responses
+
+  This section describes the content of the frame body for the different
+  responses. Please note that to make room for future evolution, clients should
+  support extra informations (that they should simply discard) to the one
+  described in this document at the end of the frame body.
+
+4.2.1. ERROR
+
+  Indicates an error processing a request. The body of the message will be an
+  error code ([int]) followed by a [string] error message. Then, depending on
+  the exception, more content may follow. The error codes are defined in
+  Section 9, along with their additional content if any.
+
+
+4.2.2. READY
+
+  Indicates that the server is ready to process queries. This message will be
+  sent by the server either after a STARTUP message if no authentication is
+  required (if authentication is required, the server indicates readiness by
+  sending a AUTH_RESPONSE message).
+
+  The body of a READY message is empty.
+
+
+4.2.3. AUTHENTICATE
+
+  Indicates that the server requires authentication, and which authentication
+  mechanism to use.
+
+  The authentication is SASL based and thus consists of a number of server
+  challenges (AUTH_CHALLENGE, Section 4.2.7) followed by client responses
+  (AUTH_RESPONSE, Section 4.1.2). The initial exchange is however boostrapped
+  by an initial client response. The details of that exchange (including how
+  many challenge-response pairs are required) are specific to the authenticator
+  in use. The exchange ends when the server sends an AUTH_SUCCESS message or
+  an ERROR message.
+
+  This message will be sent following a STARTUP message if authentication is
+  required and must be answered by a AUTH_RESPONSE message from the client.
+
+  The body consists of a single [string] indicating the full class name of the
+  IAuthenticator in use.
+
+
+4.2.4. SUPPORTED
+
+  Indicates which startup options are supported by the server. This message
+  comes as a response to an OPTIONS message.
+
+  The body of a SUPPORTED message is a [string multimap]. This multimap gives
+  for each of the supported STARTUP options, the list of supported values.
+
+
+4.2.5. RESULT
+
+  The result to a query (QUERY, PREPARE, EXECUTE or BATCH messages).
+
+  The first element of the body of a RESULT message is an [int] representing the
+  `kind` of result. The rest of the body depends on the kind. The kind can be
+  one of:
+    0x0001    Void: for results carrying no information.
+    0x0002    Rows: for results to select queries, returning a set of rows.
+    0x0003    Set_keyspace: the result to a `use` query.
+    0x0004    Prepared: result to a PREPARE message.
+    0x0005    Schema_change: the result to a schema altering query.
+
+  The body for each kind (after the [int] kind) is defined below.
+
+
+4.2.5.1. Void
+
+  The rest of the body for a Void result is empty. It indicates that a query was
+  successful without providing more information.
+
+
+4.2.5.2. Rows
+
+  Indicates a set of rows. The rest of the body of a Rows result is:
+    <metadata><rows_count><rows_content>
+  where:
+    - <metadata> is composed of:
+        <flags><columns_count>[<paging_state>][<global_table_spec>?<col_spec_1>...<col_spec_n>]
+      where:
+        - <flags> is an [int]. The bits of <flags> provides information on the
+          formatting of the remaining information. A flag is set if the bit
+          corresponding to its `mask` is set. Supported flags are, given their
+          mask:
+            0x0001    Global_tables_spec: if set, only one table spec (keyspace
+                      and table name) is provided as <global_table_spec>. If not
+                      set, <global_table_spec> is not present.
+            0x0002    Has_more_pages: indicates whether this is not the last
+                      page of results and more should be retrieved. If set, the
+                      <paging_state> will be present. The <paging_state> is a
+                      [bytes] value that should be used in QUERY/EXECUTE to
+                      continue paging and retrieve the remainder of the result for
+                      this query (See Section 8 for more details).
+            0x0004    No_metadata: if set, the <metadata> is only composed of
+                      these <flags>, the <column_count> and optionally the
+                      <paging_state> (depending on the Has_more_pages flag) but
+                      no other information (so no <global_table_spec> nor <col_spec_i>).
+                      This will only ever be the case if this was requested
+                      during the query (see QUERY and RESULT messages).
+        - <columns_count> is an [int] representing the number of columns selected
+          by the query that produced this result. It defines the number of <col_spec_i>
+          elements in and the number of elements for each row in <rows_content>.
+        - <global_table_spec> is present if the Global_tables_spec is set in
+          <flags>. It is composed of two [string] representing the
+          (unique) keyspace name and table name the columns belong to.
+        - <col_spec_i> specifies the columns returned in the query. There are
+          <column_count> such column specifications that are composed of:
+            (<ksname><tablename>)?<name><type>
+          The initial <ksname> and <tablename> are two [string] and are only present
+          if the Global_tables_spec flag is not set. The <column_name> is a
+          [string] and <type> is an [option] that corresponds to the description
+          (what this description is depends a bit on the context: in results to
+          selects, this will be either the user chosen alias or the selection used
+          (often a colum name, but it can be a function call too). In results to
+          a PREPARE, this will be either the name of the corresponding bind variable
+          or the column name for the variable if it is "anonymous") and type of
+          the corresponding result. The option for <type> is either a native
+          type (see below), in which case the option has no value, or a
+          'custom' type, in which case the value is a [string] representing
+          the fully qualified class name of the type represented. Valid option
+          ids are:
+            0x0000    Custom: the value is a [string], see above.
+            0x0001    Ascii
+            0x0002    Bigint
+            0x0003    Blob
+            0x0004    Boolean
+            0x0005    Counter
+            0x0006    Decimal
+            0x0007    Double
+            0x0008    Float
+            0x0009    Int
+            0x000B    Timestamp
+            0x000C    Uuid
+            0x000D    Varchar
+            0x000E    Varint
+            0x000F    Timeuuid
+            0x0010    Inet
+            0x0011    Date
+            0x0012    Time
+            0x0013    Smallint
+            0x0014    Tinyint
+            0x0020    List: the value is an [option], representing the type
+                            of the elements of the list.
+            0x0021    Map: the value is two [option], representing the types of the
+                           keys and values of the map
+            0x0022    Set: the value is an [option], representing the type
+                            of the elements of the set
+            0x0030    UDT: the value is <ks><udt_name><n><name_1><type_1>...<name_n><type_n>
+                           where:
+                              - <ks> is a [string] representing the keyspace name this
+                                UDT is part of.
+                              - <udt_name> is a [string] representing the UDT name.
+                              - <n> is a [short] representing the number of fields of
+                                the UDT, and thus the number of <name_i><type_i> pairs
+                                following
+                              - <name_i> is a [string] representing the name of the
+                                i_th field of the UDT.
+                              - <type_i> is an [option] representing the type of the
+                                i_th field of the UDT.
+            0x0031    Tuple: the value is <n><type_1>...<type_n> where <n> is a [short]
+                             representing the number of values in the type, and <type_i>
+                             are [option] representing the type of the i_th component
+                             of the tuple
+
+    - <rows_count> is an [int] representing the number of rows present in this
+      result. Those rows are serialized in the <rows_content> part.
+    - <rows_content> is composed of <row_1>...<row_m> where m is <rows_count>.
+      Each <row_i> is composed of <value_1>...<value_n> where n is
+      <columns_count> and where <value_j> is a [bytes] representing the value
+      returned for the jth column of the ith row. In other words, <rows_content>
+      is composed of (<rows_count> * <columns_count>) [bytes].
+
+
+4.2.5.3. Set_keyspace
+
+  The result to a `use` query. The body (after the kind [int]) is a single
+  [string] indicating the name of the keyspace that has been set.
+
+
+4.2.5.4. Prepared
+
+  The result to a PREPARE message. The body of a Prepared result is:
+    <id><metadata><result_metadata>
+  where:
+    - <id> is [short bytes] representing the prepared query ID.
+    - <metadata> is composed of:
+        <flags><columns_count><pk_count>[<pk_index_1>...<pk_index_n>][<global_table_spec>?<col_spec_1>...<col_spec_n>]
+      where:
+        - <flags> is an [int]. The bits of <flags> provides information on the
+          formatting of the remaining information. A flag is set if the bit
+          corresponding to its `mask` is set. Supported masks and their flags
+          are:
+            0x0001    Global_tables_spec: if set, only one table spec (keyspace
+                      and table name) is provided as <global_table_spec>. If not
+                      set, <global_table_spec> is not present.
+        - <columns_count> is an [int] representing the number of bind markers
+          in the prepared statement.  It defines the number of <col_spec_i>
+          elements.
+        - <pk_count> is an [int] representing the number of <pk_index_i>
+          elements to follow. If this value is zero, at least one of the
+          partition key columns in the table that the statement acts on
+          did not have a corresponding bind marker (or the bind marker
+          was wrapped in a function call).
+        - <pk_index_i> is a short that represents the index of the bind marker
+          that corresponds to the partition key column in position i.
+          For example, a <pk_index> sequence of [2, 0, 1] indicates that the
+          table has three partition key columns; the full partition key
+          can be constructed by creating a composite of the values for
+          the bind markers at index 2, at index 0, and at index 1.
+          This allows implementations with token-aware routing to correctly
+          construct the partition key without needing to inspect table
+          metadata.
+        - <global_table_spec> is present if the Global_tables_spec is set in
+          <flags>. If present, it is composed of two [string]s. The first
+          [string] is the name of the keyspace that the statement acts on.
+          The second [string] is the name of the table that the columns
+          represented by the bind markers belong to.
+        - <col_spec_i> specifies the bind markers in the prepared statement.
+          There are <column_count> such column specifications, each with the
+          following format:
+            (<ksname><tablename>)?<name><type>
+          The initial <ksname> and <tablename> are two [string] that are only
+          present if the Global_tables_spec flag is not set. The <name> field
+          is a [string] that holds the name of the bind marker (if named),
+          or the name of the column, field, or expression that the bind marker
+          corresponds to (if the bind marker is "anonymous").  The <type>
+          field is an [option] that represents the expected type of values for
+          the bind marker.  See the Rows documentation (section 4.2.5.2) for
+          full details on the <type> field.
+
+    - <result_metadata> is defined exactly the same as <metadata> in the Rows
+      documentation (section 4.2.5.2).  This describes the metadata for the
+      result set that will be returned when this prepared statement is executed.
+      Note that <result_metadata> may be empty (have the No_metadata flag and
+      0 columns, See section 4.2.5.2) and will be for any query that is not a
+      Select. In fact, there is never a guarantee that this will be non-empty, so
+      implementations should protect themselves accordingly. This result metadata
+      is an optimization that allows implementations to later execute the
+      prepared statement without requesting the metadata (see the Skip_metadata
+      flag in EXECUTE).  Clients can safely discard this metadata if they do not
+      want to take advantage of that optimization.
+
+  Note that the prepared query ID returned is global to the node on which the query
+  has been prepared. It can be used on any connection to that node
+  until the node is restarted (after which the query must be reprepared).
+
+4.2.5.5. Schema_change
+
+  The result to a schema altering query (creation/update/drop of a
+  keyspace/table/index). The body (after the kind [int]) is the same
+  as the body for a "SCHEMA_CHANGE" event, so 3 strings:
+    <change_type><target><options>
+  Please refer to section 4.2.6 below for the meaning of those fields.
+
+  Note that a query to create or drop an index is considered to be a change
+  to the table the index is on.
+
+
+4.2.6. EVENT
+
+  An event pushed by the server. A client will only receive events for the
+  types it has REGISTERed to. The body of an EVENT message will start with a
+  [string] representing the event type. The rest of the message depends on the
+  event type. The valid event types are:
+    - "TOPOLOGY_CHANGE": events related to change in the cluster topology.
+      Currently, events are sent when new nodes are added to the cluster, and
+      when nodes are removed. The body of the message (after the event type)
+      consists of a [string] and an [inet], corresponding respectively to the
+      type of change ("NEW_NODE" or "REMOVED_NODE") followed by the address of
+      the new/removed node.
+    - "STATUS_CHANGE": events related to change of node status. Currently,
+      up/down events are sent. The body of the message (after the event type)
+      consists of a [string] and an [inet], corresponding respectively to the
+      type of status change ("UP" or "DOWN") followed by the address of the
+      concerned node.
+    - "SCHEMA_CHANGE": events related to schema change. After the event type,
+      the rest of the message will be <change_type><target><options> where:
+        - <change_type> is a [string] representing the type of changed involved.
+          It will be one of "CREATED", "UPDATED" or "DROPPED".
+        - <target> is a [string] that can be one of "KEYSPACE", "TABLE", "TYPE",
+          "FUNCTION" or "AGGREGATE" and describes what has been modified
+          ("TYPE" stands for modifications related to user types, "FUNCTION"
+          for modifications related to user defined functions, "AGGREGATE"
+          for modifications related to user defined aggregates).
+        - <options> depends on the preceding <target>:
+          - If <target> is "KEYSPACE", then <options> will be a single [string]
+            representing the keyspace changed.
+          - If <target> is "TABLE" or "TYPE", then
+            <options> will be 2 [string]: the first one will be the keyspace
+            containing the affected object, and the second one will be the name
+            of said affected object (either the table, user type, function, or
+            aggregate name).
+          - If <target> is "FUNCTION" or "AGGREGATE", multiple arguments follow:
+            - [string] keyspace containing the user defined function / aggregate
+            - [string] the function/aggregate name
+            - [string list] one string for each argument type (as CQL type)
+
+  All EVENT messages have a streamId of -1 (Section 2.3).
+
+  Please note that "NEW_NODE" and "UP" events are sent based on internal Gossip
+  communication and as such may be sent a short delay before the binary
+  protocol server on the newly up node is fully started. Clients are thus
+  advised to wait a short time before trying to connect to the node (1 second
+  should be enough), otherwise they may experience a connection refusal at
+  first.
+
+4.2.7. AUTH_CHALLENGE
+
+  A server authentication challenge (see AUTH_RESPONSE (Section 4.1.2) for more
+  details).
+
+  The body of this message is a single [bytes] token. The details of what this
+  token contains (and when it can be null/empty, if ever) depends on the actual
+  authenticator used.
+
+  Clients are expected to answer the server challenge with an AUTH_RESPONSE
+  message.
+
+4.2.8. AUTH_SUCCESS
+
+  Indicates the success of the authentication phase. See Section 4.2.3 for more
+  details.
+
+  The body of this message is a single [bytes] token holding final information
+  from the server that the client may require to finish the authentication
+  process. What that token contains and whether it can be null depends on the
+  actual authenticator used.
+
+
+5. Compression
+
+  Frame compression is supported by the protocol, but then only the frame body
+  is compressed (the frame header should never be compressed).
+
+  Before being used, client and server must agree on a compression algorithm to
+  use, which is done in the STARTUP message. As a consequence, a STARTUP message
+  must never be compressed.  However, once the STARTUP frame has been received
+  by the server, messages can be compressed (including the response to the STARTUP
+  request). Frames do not have to be compressed, however, even if compression has
+  been agreed upon (a server may only compress frames above a certain size at its
+  discretion). A frame body should be compressed if and only if the compressed
+  flag (see Section 2.2) is set.
+
+  As of version 2 of the protocol, the following compressions are available:
+    - lz4 (https://code.google.com/p/lz4/). In that, note that the first four bytes
+      of the body will be the uncompressed length (followed by the compressed
+      bytes).
+    - snappy (https://code.google.com/p/snappy/). This compression might not be
+      available as it depends on a native lib (server-side) that might not be
+      avaivable on some installations.
+
+
+6. Data Type Serialization Formats
+
+  This sections describes the serialization formats for all CQL data types
+  supported by Cassandra through the native protocol.  These serialization
+  formats should be used by client drivers to encode values for EXECUTE
+  messages.  Cassandra will use these formats when returning values in
+  RESULT messages.
+
+  All values are represented as [bytes] in EXECUTE and RESULT messages.
+  The [bytes] format includes an int prefix denoting the length of the value.
+  For that reason, the serialization formats described here will not include
+  a length component.
+
+  For legacy compatibility reasons, note that most non-string types support
+  "empty" values (i.e. a value with zero length).  An empty value is distinct
+  from NULL, which is encoded with a negative length.
+
+  As with the rest of the native protocol, all encodings are big-endian.
+
+6.1. ascii
+
+  A sequence of bytes in the ASCII range [0, 127].  Bytes with values outside of
+  this range will result in a validation error.
+
+6.2 bigint
+
+  An eight-byte two's complement integer.
+
+6.3 blob
+
+  Any sequence of bytes.
+
+6.4 boolean
+
+  A single byte.  A value of 0 denotes "false"; any other value denotes "true".
+  (However, it is recommended that a value of 1 be used to represent "true".)
+
+6.5 date
+
+  An unsigned integer representing days with epoch centered at 2^31.
+  (unix epoch January 1st, 1970).
+  A few examples:
+    0:    -5877641-06-23
+    2^31: 1970-1-1
+    2^32: 5881580-07-11
+
+6.6 decimal
+
+  The decimal format represents an arbitrary-precision number.  It contains an
+  [int] "scale" component followed by a varint encoding (see section 6.17)
+  of the unscaled value.  The encoded value represents "<unscaled>E<-scale>".
+  In other words, "<unscaled> * 10 ^ (-1 * <scale>)".
+
+6.7 double
+
+  An 8 byte floating point number in the IEEE 754 binary64 format.
+
+6.8 float
+
+  A 4 byte floating point number in the IEEE 754 binary32 format.
+
+6.9 inet
+
+  A 4 byte or 16 byte sequence denoting an IPv4 or IPv6 address, respectively.
+
+6.10 int
+
+  A 4 byte two's complement integer.
+
+6.11 list
+
+  A [int] n indicating the number of elements in the list, followed by n
+  elements.  Each element is [bytes] representing the serialized value.
+
+6.12 map
+
+  A [int] n indicating the number of key/value pairs in the map, followed by
+  n entries.  Each entry is composed of two [bytes] representing the key
+  and value.
+
+6.13 set
+
+  A [int] n indicating the number of elements in the set, followed by n
+  elements.  Each element is [bytes] representing the serialized value.
+
+6.14 smallint
+
+  A 2 byte two's complement integer.
+
+6.15 text
+
+  A sequence of bytes conforming to the UTF-8 specifications.
+
+6.16 time
+
+  An 8 byte two's complement long representing nanoseconds since midnight.
+  Valid values are in the range 0 to 86399999999999
+
+6.17 timestamp
+
+  An 8 byte two's complement integer representing a millisecond-precision
+  offset from the unix epoch (00:00:00, January 1st, 1970).  Negative values
+  represent a negative offset from the epoch.
+
+6.18 timeuuid
+
+  A 16 byte sequence representing a version 1 UUID as defined by RFC 4122.
+
+6.19 tinyint
+
+  A 1 byte two's complement integer.
+
+6.20 tuple
+
+  A sequence of [bytes] values representing the items in a tuple.  The encoding
+  of each element depends on the data type for that position in the tuple.
+  Null values may be represented by using length -1 for the [bytes]
+  representation of an element.
+
+6.21 uuid
+
+  A 16 byte sequence representing any valid UUID as defined by RFC 4122.
+
+6.22 varchar
+
+  An alias of the "text" type.
+
+6.23 varint
+
+  A variable-length two's complement encoding of a signed integer.
+
+  The following examples may help implementors of this spec:
+
+  Value | Encoding
+  ------|---------
+      0 |     0x00
+      1 |     0x01
+    127 |     0x7F
+    128 |   0x0080
+    129 |   0x0081
+     -1 |     0xFF
+   -128 |     0x80
+   -129 |   0xFF7F
+
+  Note that positive numbers must use a most-significant byte with a value
+  less than 0x80, because a most-significant bit of 1 indicates a negative
+  value.  Implementors should pad positive values that have a MSB >= 0x80
+  with a leading 0x00 byte.
+
+
+7. User Defined Types
+
+  This section describes the serialization format for User defined types (UDT),
+  as described in section 4.2.5.2.
+
+  A UDT value is composed of successive [bytes] values, one for each field of the UDT
+  value (in the order defined by the type). A UDT value will generally have one value
+  for each field of the type it represents, but it is allowed to have less values than
+  the type has fields.
+
+
+8. Result paging
+
+  The protocol allows for paging the result of queries. For that, the QUERY and
+  EXECUTE messages have a <result_page_size> value that indicate the desired
+  page size in CQL3 rows or bytes.
+
+  If a positive value is provided for <result_page_size>, the result set of the
+  RESULT message returned for the query will contain at most the
+  <result_page_size> first rows or bytes of the query result. If that first page of results
+  contains the full result set for the query, the RESULT message (of kind `Rows`)
+  will have the Has_more_pages flag *not* set. However, if some results are not
+  part of the first response, the Has_more_pages flag will be set and the result
+  will contain a <paging_state> value. In that case, the <paging_state> value
+  should be used in a QUERY or EXECUTE message (that has the *same* query as
+  the original one or the behavior is undefined) to retrieve the next page of
+  results.
+
+  Only CQL3 queries that return a result set (RESULT message with a Rows `kind`)
+  support paging. For other type of queries, the <result_page_size> value is
+  ignored.
+
+  In the previous protocol versions the page size was always provided in rows. Since 4.1
+  the page size can be provided in bytes as well. Whether the page size is specified in
+  rows or bytes is controlled by query flags (see section 4.1.4 for details).
+
+  Note to client implementors:
+  - While <result_page_size> can be as low as 1, it will likely be detrimental
+    to performance to pick a value too low. A value below 100 is probably too
+    low for most use cases.
+  - Clients should not rely on the actual size of the result set returned to
+    decide if there are more results to fetch or not. Instead, they should always
+    check the Has_more_pages flag (unless they did not enable paging for the query
+    obviously). Clients should also not assert that no result will have more than
+    <result_page_size> results. While the current implementation always respects
+    the exact value of <result_page_size>, we reserve the right to return
+    slightly smaller or bigger pages in the future for performance reasons.
+  - The <paging_state> is specific to a protocol version and drivers should not
+    send a <paging_state> returned by a node using the protocol v3 to query a node
+    using the protocol v4 for instance.
+
+
+9. Error codes
+
+  Let us recall that an ERROR message is composed of <code><message>[...]
+  (see 4.2.1 for details). The supported error codes, as well as any additional
+  information the message may contain after the <message> are described below:
+    0x0000    Server error: something unexpected happened. This indicates a
+              server-side bug.
+    0x000A    Protocol error: some client message triggered a protocol
+              violation (for instance a QUERY message is sent before a STARTUP
+              one has been sent)
+    0x0100    Authentication error: authentication was required and failed. The
+              possible reason for failing depends on the authenticator in use,
+              which may or may not include more detail in the accompanying
+              error message.
+    0x1000    Unavailable exception. The rest of the ERROR message body will be
+                <cl><required><alive>
+              where:
+                <cl> is the [consistency] level of the query that triggered
+                     the exception.
+                <required> is an [int] representing the number of nodes that
+                           should be alive to respect <cl>
+                <alive> is an [int] representing the number of replicas that
+                        were known to be alive when the request had been
+                        processed (since an unavailable exception has been
+                        triggered, there will be <alive> < <required>)
+    0x1001    Overloaded: the request cannot be processed because the
+              coordinator node is overloaded
+    0x1002    Is_bootstrapping: the request was a read request but the
+              coordinator node is bootstrapping
+    0x1003    Truncate_error: error during a truncation error.
+    0x1100    Write_timeout: Timeout exception during a write request. The rest
+              of the ERROR message body will be
+                <cl><received><blockfor><writeType>
+              where:
+                <cl> is the [consistency] level of the query having triggered
+                     the exception.
+                <received> is an [int] representing the number of nodes having
+                           acknowledged the request.
+                <blockfor> is an [int] representing the number of replicas whose
+                           acknowledgement is required to achieve <cl>.
+                <writeType> is a [string] that describe the type of the write
+                            that timed out. The value of that string can be one
+                            of:
+                             - "SIMPLE": the write was a non-batched
+                               non-counter write.
+                             - "BATCH": the write was a (logged) batch write.
+                               If this type is received, it means the batch log
+                               has been successfully written (otherwise a
+                               "BATCH_LOG" type would have been sent instead).
+                             - "UNLOGGED_BATCH": the write was an unlogged
+                               batch. No batch log write has been attempted.
+                             - "COUNTER": the write was a counter write
+                               (batched or not).
+                             - "BATCH_LOG": the timeout occurred during the
+                               write to the batch log when a (logged) batch
+                               write was requested.
+                             - "CAS": the timeout occured during the Compare And Set write/update.
+                             - "VIEW": the timeout occured when a write involves
+                                VIEW update and failure to acqiure local view(MV)
+                                lock for key within timeout
+                             - "CDC": the timeout occured when cdc_total_space_in_mb is
+                                exceeded when doing a write to data tracked by cdc.
+    0x1200    Read_timeout: Timeout exception during a read request. The rest
+              of the ERROR message body will be
+                <cl><received><blockfor><data_present>
+              where:
+                <cl> is the [consistency] level of the query having triggered
+                     the exception.
+                <received> is an [int] representing the number of nodes having
+                           answered the request.
+                <blockfor> is an [int] representing the number of replicas whose
+                           response is required to achieve <cl>. Please note that
+                           it is possible to have <received> >= <blockfor> if
+                           <data_present> is false. Also in the (unlikely)
+                           case where <cl> is achieved but the coordinator node
+                           times out while waiting for read-repair acknowledgement.
+                <data_present> is a single byte. If its value is 0, it means
+                               the replica that was asked for data has not
+                               responded. Otherwise, the value is != 0.
+    0x1300    Read_failure: A non-timeout exception during a read request. The rest
+              of the ERROR message body will be
+                <cl><received><blockfor><numfailures><data_present>
+              where:
+                <cl> is the [consistency] level of the query having triggered
+                     the exception.
+                <received> is an [int] representing the number of nodes having
+                           answered the request.
+                <blockfor> is an [int] representing the number of replicas whose
+                           acknowledgement is required to achieve <cl>.
+                <numfailures> is an [int] representing the number of nodes that
+                              experience a failure while executing the request.
+                <data_present> is a single byte. If its value is 0, it means
+                               the replica that was asked for data had not
+                               responded. Otherwise, the value is != 0.
+    0x1400    Function_failure: A (user defined) function failed during execution.
+              The rest of the ERROR message body will be
+                <keyspace><function><arg_types>
+              where:
+                <keyspace> is the keyspace [string] of the failed function
+                <function> is the name [string] of the failed function
+                <arg_types> [string list] one string for each argument type (as CQL type) of the failed function
+    0x1500    Write_failure: A non-timeout exception during a write request. The rest
+              of the ERROR message body will be
+                <cl><received><blockfor><numfailures><write_type>
+              where:
+                <cl> is the [consistency] level of the query having triggered
+                     the exception.
+                <received> is an [int] representing the number of nodes having
+                           answered the request.
+                <blockfor> is an [int] representing the number of replicas whose
+                           acknowledgement is required to achieve <cl>.
+                <numfailures> is an [int] representing the number of nodes that
+                              experience a failure while executing the request.
+                <writeType> is a [string] that describes the type of the write
+                            that failed. The value of that string can be one
+                            of:
+                             - "SIMPLE": the write was a non-batched
+                               non-counter write.
+                             - "BATCH": the write was a (logged) batch write.
+                               If this type is received, it means the batch log
+                               has been successfully written (otherwise a
+                               "BATCH_LOG" type would have been sent instead).
+                             - "UNLOGGED_BATCH": the write was an unlogged
+                               batch. No batch log write has been attempted.
+                             - "COUNTER": the write was a counter write
+                               (batched or not).
+                             - "BATCH_LOG": the failure occured during the
+                               write to the batch log when a (logged) batch
+                               write was requested.
+                             - "CAS": the failure occured during the Compare And Set write/update.
+                             - "VIEW": the failure occured when a write involves
+                                VIEW update and failure to acqiure local view(MV)
+                                lock for key within timeout
+                             - "CDC": the failure occured when cdc_total_space_in_mb is
+                                exceeded when doing a write to data tracked by cdc.
+
+    0x2000    Syntax_error: The submitted query has a syntax error.
+    0x2100    Unauthorized: The logged user doesn't have the right to perform
+              the query.
+    0x2200    Invalid: The query is syntactically correct but invalid.
+    0x2300    Config_error: The query is invalid because of some configuration issue
+    0x2400    Already_exists: The query attempted to create a keyspace or a
+              table that was already existing. The rest of the ERROR message
+              body will be <ks><table> where:
+                <ks> is a [string] representing either the keyspace that
+                     already exists, or the keyspace in which the table that
+                     already exists is.
+                <table> is a [string] representing the name of the table that
+                        already exists. If the query was attempting to create a
+                        keyspace, <table> will be present but will be the empty
+                        string.
+    0x2500    Unprepared: Can be thrown while a prepared statement tries to be
+              executed if the provided prepared statement ID is not known by
+              this host. The rest of the ERROR message body will be [short
+              bytes] representing the unknown ID.
+
+10. Changes from v4
+
+  * Query flags (Section 4.1.4) includes a new flag 0x40000000 which denotes that
+    the page size is specified in bytes rather than in rows.
diff --git a/ide/idea/vcs.xml b/ide/idea/vcs.xml
index 81872fd3f150..8abf2cfaeca9 100644
--- a/ide/idea/vcs.xml
+++ b/ide/idea/vcs.xml
@@ -7,9 +7,17 @@
     <option name="links">
       <list>
         <IssueNavigationLink>
-          <option name="issueRegexp" value="CASSANDRA-(\d+)" />
+          <option name="issueRegexp" value="(?i)CASSANDRA-(\d+)" />
           <option name="linkRegexp" value="https://issues.apache.org/jira/browse/CASSANDRA-$1" />
         </IssueNavigationLink>
+        <IssueNavigationLink>
+          <option name="issueRegexp" value="(?i)CNDB-(\d+)"/>
+          <option name="linkRegexp" value="https://github.com/riptano/cndb/issues/$1"/>
+        </IssueNavigationLink>
+        <IssueNavigationLink>
+          <option name="issueRegexp" value="(?i)(DSP|STAR|HCD|DB)-\d+"/>
+          <option name="linkRegexp" value="https://datastax.jira.com/browse/$0"/>
+        </IssueNavigationLink>
       </list>
     </option>
   </component>
diff --git a/ide/idea/workspace.xml b/ide/idea/workspace.xml
index 89528b240854..de7442fce22b 100644
--- a/ide/idea/workspace.xml
+++ b/ide/idea/workspace.xml
@@ -151,7 +151,9 @@
                                           -Dcassandra.storagedir=$PROJECT_DIR$/data
                                           -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin
                                           -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml
+                                          -Dcassandra.cluster_version_provider.min_stable_duration_ms=0
                                           -XX:HeapDumpPath=build/test
+                                          -Dnet.bytebuddy.experimental=true
                                           -ea" />
       <option name="PROGRAM_PARAMETERS" value="" />
       <option name="WORKING_DIRECTORY" value="" />
@@ -189,21 +191,26 @@
                                           -Dcassandra.ring_delay_ms=1000
                                           -Dcassandra.skip_sync=true
                                           -Dcassandra.strict.runtime.checks=true
-                                          -Dcassandra.test.flush_local_schema_changes=false
+                                          -Dcassandra.unsafesystem=true
                                           -Dcassandra.test.messagingService.nonGracefulShutdown=true
                                           -Dcassandra.tolerate_sstable_size=true
                                           -Dcassandra.use_nix_recursive_delete=true
+                                          -Dcassandra.testtag=ide
+                                          -Dcassandra.cluster_version_provider.min_stable_duration_ms=0
+                                          -Dcassandra.test.cluster_version_provider.skip_wait_for_gossip=true
                                           -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables
                                           -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin
                                           -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables
                                           -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml
                                           -Dmigration-sstable-root=$PROJECT_DIR$/test/data/migration-sstables
-                                          -XX:ActiveProcessorCount=2
+                                          -Djava.io.tmpdir=$PROJECT_DIR$/build/test/cassandra
+                                          -XX:ActiveProcessorCount=4
                                           -XX:HeapDumpPath=build/test
                                           -XX:MaxMetaspaceSize=2G
                                           -XX:SoftRefLRUPolicyMSPerMB=0
                                           -Xmx4G
-                                          -ea" />
+                                          -ea
+                                          -Dnet.bytebuddy.experimental=true" />
       <option name="PARAMETERS" value="" />
       <fork_mode value="class" />
       <option name="WORKING_DIRECTORY" value="" />
@@ -235,6 +242,7 @@
                                           -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml
                                           -XX:HeapDumpPath=build/test
                                           -Xmx1G
+                                          -Dnet.bytebuddy.experimental=true
                                           -ea" />
       <option name="PROGRAM_PARAMETERS" value="" />
       <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
diff --git a/lib/cassandra-driver-internal-only-3.26.0-df103d3a.zip b/lib/cassandra-driver-internal-only-3.26.0-df103d3a.zip
new file mode 100644
index 000000000000..ddb70e77f1c8
Binary files /dev/null and b/lib/cassandra-driver-internal-only-3.26.0-df103d3a.zip differ
diff --git a/mount-tmpfs b/mount-tmpfs
new file mode 100755
index 000000000000..0bdba03f40a8
--- /dev/null
+++ b/mount-tmpfs
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Copyright DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script simply mounts a tmpfs filesystem at build/test/cassandra
+# If there is already a tmpfs mounted at that location, it will be unmounted first
+# The size of the tmpfs is 1GB
+
+# The script is expected to work on both Linux and MacOS
+
+# Run the script from the root of your project. In most cases it should improve performance of running unit tests.
+# Do not run it using sudo, it will ask for sudo password if needed.
+# The script does not expect any arguments.
+
+(grep "apache-cassandra" build.xml || grep "dse-db" build.xml) 1> /dev/null
+
+os="$(uname | tr '[:upper:]' '[:lower:]')"
+
+if [[ "$os" == "darwin" ]]; then
+	existing="$(diskutil list | grep CassandraBuildTmp || true)"
+	if [[ -n "$existing" ]]; then
+		diskname="$(hdiutil | grep CassandraBuildTmp | cut -f 1)"
+		diskutil unmount CassandraBuildTmp
+		hdiutil detach "$diskname"
+	fi
+
+	rm -rf build/test/cassandra
+	mkdir -p build/test/cassandra
+
+	diskutil erasevolume HFS+ CassandraBuildTmp "$(hdiutil attach -nomount ram://2097152)"
+	diskutil mount -mountPoint "$(pwd)/build/test/cassandra" CassandraBuildTmp
+elif [[ "$os" == "linux" ]]; then
+	mount | grep tmpfs | grep /build/test/cassandra | cut -f 3 -d ' ' | while read -r line;
+	do
+		sudo umount "$line"
+	done
+
+	rm -rf build/test/cassandra
+	mkdir -p build/test/cassandra
+
+	sudo mount -t tmpfs -o size=1024M tmpfs "$(pwd)/build/test/cassandra"
+else
+	echo "Unrecognized OS: $os" 1>&2
+	exit 1
+fi
diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh
index c80ba438729d..dca977c1dcdf 100755
--- a/pylib/cassandra-cqlsh-tests.sh
+++ b/pylib/cassandra-cqlsh-tests.sh
@@ -23,35 +23,46 @@
 #
 ################################
 
+set -x
+
 WORKSPACE=$1
 
 [ "x${WORKSPACE}" != "x" ] || WORKSPACE="$(readlink -f $(dirname "$0")/..)"
 [ "x${BUILD_DIR}" != "x" ] || BUILD_DIR="${WORKSPACE}/build"
 [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${WORKSPACE}/build"
+[ "x${TMPDIR}" != "x" ] || TMPDIR="$(mktemp -d ${DIST_DIR}/run-python-dtest.XXXXXX)"
 
-export TMPDIR="$(mktemp -d ${DIST_DIR}/run-python-dtest.XXXXXX)"
+mkdir -p "${TMPDIR}"
+export TMPDIR
 export PYTHONIOENCODING="utf-8"
 export PYTHONUNBUFFERED=true
 export CASS_DRIVER_NO_EXTENSIONS=true
-export CASS_DRIVER_NO_CYTHON=true
 export CCM_MAX_HEAP_SIZE="2048M"
 export CCM_HEAP_NEWSIZE="200M"
 export CCM_CONFIG_DIR="${TMPDIR}/.ccm"
 export NUM_TOKENS="16"
 export CASSANDRA_DIR=${WORKSPACE}
 
-java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
-version=$(grep 'property\s*name=\"base.version\"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
+if [[ -z "${version}" ]]; then
+  version=$(grep 'property\s*name=\"base.version\"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
+fi
+
+if [[ -z "${java_version}" ]]; then
+  java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | sed 's/^1\.//g' | awk -F. '{print $1}')
+fi
 
-python_version="3.8"
-command -v python3 >/dev/null 2>&1 && python_version="$(python3 -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}')"
+if [[ -z "${python_version}" ]]; then
+  python_version="3.8"
+  command -v python3 >/dev/null 2>&1 && python_version="$(python3 -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}')"
+fi
 
 export TESTSUITE_NAME="cqlshlib.python${python_version}.jdk${java_version}"
 
 pushd ${CASSANDRA_DIR} >/dev/null
 
 # check project is already built. no cleaning is done, so jenkins unstash works, beware.
-[[ -f "${BUILD_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${BUILD_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${BUILD_DIR} with: $(ls ${BUILD_DIR})"; exit 1; }
+project_name=$(grep '<project\s*basedir=' build.xml |sed -ne 's/.*name=\"\([^"]*\)\".*/\1/p')
+[[ -f "${BUILD_DIR}/${project_name}-${version}.jar" ]] || [[ -f "${BUILD_DIR}/${project_name}-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${BUILD_DIR} with: $(ls ${BUILD_DIR})"; exit 1; }
 
 # Set up venv with dtest dependencies
 set -e # enable immediate exit if venv setup fails
@@ -59,24 +70,35 @@ set -e # enable immediate exit if venv setup fails
 # fresh virtualenv and test logs results everytime
 rm -fr ${DIST_DIR}/venv ${DIST_DIR}/test/{html,output,logs}
 
+if [ "$cython" = "yes" ]; then
+  cython_suffix="cython"
+else
+  cython_suffix="no-cython"
+fi
+
 # re-use when possible the pre-installed virtualenv found in the cassandra-ubuntu2004_test docker image
-virtualenv-clone ${BUILD_HOME}/env${python_version} ${BUILD_DIR}/venv || virtualenv --python=python3 ${BUILD_DIR}/venv
+virtualenv-clone ${BUILD_HOME}/${cython_suffix}/python${python_version} ${BUILD_DIR}/venv || virtualenv --python=python${python_version} ${BUILD_DIR}/venv
 source ${BUILD_DIR}/venv/bin/activate
 
-pip install --exists-action w -r ${CASSANDRA_DIR}/pylib/requirements.txt
+if [ "$cython" = "yes" ]; then
+  pip install "Cython>=0.29.15,<3.0"
+  unset CASS_DRIVER_NO_CYTHON
+else
+  export CASS_DRIVER_NO_CYTHON=true
+fi
+
+pip install --exists-action s -r ${CASSANDRA_DIR}/pylib/requirements.txt
 pip freeze
 
 if [ "$cython" = "yes" ]; then
-    TESTSUITE_NAME="${TESTSUITE_NAME}.cython"
-    pip install "Cython>=0.29.15,<3.0"
     pushd pylib >/dev/null
     python setup.py build_ext --inplace
     popd >/dev/null
-else
-    TESTSUITE_NAME="${TESTSUITE_NAME}.no_cython"
 fi
 TESTSUITE_NAME="${TESTSUITE_NAME}.$(uname -m)"
 
+TESTSUITE_NAME="${TESTSUITE_NAME}.${cython_suffix}.$(uname -m)"
+
 ################################
 #
 # Main
@@ -85,47 +107,30 @@ TESTSUITE_NAME="${TESTSUITE_NAME}.$(uname -m)"
 
 ccm remove test || true # in case an old ccm cluster is left behind
 ccm create test -n 1 --install-dir=${CASSANDRA_DIR}
-ccm updateconf "user_defined_functions_enabled: true"
-
-version_from_build=$(ccm node1 versionfrombuild)
-export pre_or_post_cdc=$(python -c """from distutils.version import LooseVersion
-print (\"postcdc\" if LooseVersion(\"${version_from_build}\") >= \"3.8\" else \"precdc\")
-""")
-case "${pre_or_post_cdc}" in
-    postcdc)
-        ccm updateconf "cdc_enabled: true"
-        ;;
-    precdc)
-        :
-        ;;
-    *)
-        echo "${pre_or_post_cdc}" is an invalid value.
-        exit 1
-        ;;
-esac
-
-ccm start --wait-for-binary-proto
+ccm updateconf "enable_user_defined_functions: true"
+ccm updateconf "cdc_enabled: true"
+
+set +e # disable immediate exit from this point
+
+ccm start --wait-for-binary-proto --jvm-version=${java_version}
 
 pushd ${CASSANDRA_DIR}/pylib/cqlshlib/ >/dev/null
 
-set +e # disable immediate exit from this point
 pytest --junitxml=${BUILD_DIR}/test/output/cqlshlib.xml
 RETURN="$?"
 
-# remove <testsuites> wrapping elements. `ant generate-unified-test-report` doesn't like it`
-sed -r "s/<[\/]?testsuites>//g" ${BUILD_DIR}/test/output/cqlshlib.xml > /tmp/cqlshlib.xml
-cat /tmp/cqlshlib.xml > ${BUILD_DIR}/test/output/cqlshlib.xml
-
 # don't do inline sed for linux+mac compat
-sed "s/testsuite name=\"pytest\"/testsuite name=\"${TESTSUITE_NAME}\"/g" ${BUILD_DIR}/test/output/cqlshlib.xml > /tmp/cqlshlib.xml
-cat /tmp/cqlshlib.xml > ${BUILD_DIR}/test/output/cqlshlib.xml
-sed "s/testcase classname=\"cqlshlib./testcase classname=\"${TESTSUITE_NAME}./g" ${BUILD_DIR}/test/output/cqlshlib.xml > /tmp/cqlshlib.xml
-cat /tmp/cqlshlib.xml > ${BUILD_DIR}/test/output/cqlshlib.xml
+sed "s/testsuite name=\"pytest\"/testsuite name=\"${TESTSUITE_NAME}\"/g" ${BUILD_DIR}/test/output/cqlshlib.xml > ${TMPDIR}/cqlshlib.xml
+cat ${TMPDIR}/cqlshlib.xml > ${BUILD_DIR}/test/output/cqlshlib.xml
+sed "s/testcase classname=\"cqlshlib./testcase classname=\"${TESTSUITE_NAME}./g" ${BUILD_DIR}/test/output/cqlshlib.xml > ${TMPDIR}/cqlshlib.xml
+cat ${TMPDIR}/cqlshlib.xml > ${BUILD_DIR}/test/output/cqlshlib.xml
 
 # tar up any ccm logs for easy retrieval
-if ls ${TMPDIR}/test/*/logs/* &>/dev/null ; then
+if ls ${TMPDIR}/.ccm/test/*/logs/* &>/dev/null ; then
     mkdir -p ${DIST_DIR}/test/logs
-    tar -C ${TMPDIR} -cJf ${DIST_DIR}/test/logs/ccm_logs.tar.xz */test/*/logs/*
+    pushd ${TMPDIR}/.ccm/test >/dev/null
+    tar -C ${TMPDIR}/.ccm/test -cJf ${DIST_DIR}/test/logs/ccm_logs.tar.xz */logs/*
+    popd >/dev/null
 fi
 
 ccm remove
@@ -136,7 +141,6 @@ ccm remove
 #
 ################################
 
-
 rm -rf ${TMPDIR}
 unset TMPDIR
 deactivate
diff --git a/pylib/cqlshlib/copyutil.py b/pylib/cqlshlib/copyutil.py
index af3573100537..d681d0d49a88 100644
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@ -45,24 +45,27 @@
 from queue import Queue
 
 from cassandra import OperationTimedOut
-from cassandra.cluster import Cluster, DefaultConnection
-from cassandra.cqltypes import ReversedType, UserType, VarcharType, VectorType
+from cassandra.cluster import DefaultConnection
+from cassandra.connection import SniEndPoint
+from cassandra.cqltypes import ReversedType, UserType, BytesType, VarcharType, VectorType
 from cassandra.metadata import protect_name, protect_names, protect_value
-from cassandra.policies import RetryPolicy, WhiteListRoundRobinPolicy, DCAwareRoundRobinPolicy, FallthroughRetryPolicy
+from cassandra.policies import RetryPolicy, DCAwareRoundRobinPolicy, FallthroughRetryPolicy
 from cassandra.query import BatchStatement, BatchType, SimpleStatement, tuple_factory
 from cassandra.util import Date, Time
-from cqlshlib.util import profile_on, profile_off
 
 from cqlshlib.cql3handling import CqlRuleSet
 from cqlshlib.displaying import NO_COLOR_MAP
+from cqlshlib.driver import cluster_factory
 from cqlshlib.formatting import format_value_default, CqlType, DateTimeFormat, EMPTY, get_formatter, BlobType
 from cqlshlib.sslhandling import ssl_settings
+from cqlshlib.util import profile_on, profile_off
 
 PROFILE_ON = False
 STRACE_ON = False
 DEBUG = False  # This may be set to True when initializing the task
 # TODO: review this for MacOS, maybe use in ('Linux', 'Darwin')
 IS_LINUX = platform.system() == 'Linux'
+IS_WINDOWS = platform.system() == 'Windows'
 
 CopyOptions = namedtuple('CopyOptions', 'copy dialect unrecognized')
 
@@ -495,6 +498,7 @@ def make_params(self):
                     port=shell.port,
                     ssl=shell.ssl,
                     auth_provider=shell.auth_provider,
+                    parent_cluster=shell.conn if not IS_WINDOWS else None,
                     cql_version=shell.conn.cql_version,
                     config_file=self.config_file,
                     protocol_version=self.protocol_version,
@@ -717,9 +721,9 @@ def make_range(prev, curr):
         def make_range_data(replicas=None):
             hosts = []
             if replicas:
-                for r in replicas:
-                    if r.is_up is not False and r.datacenter == local_dc:
-                        hosts.append(r.address)
+                # when connected to a cloud cluster r.address is the proxy
+                # so we need to use host_id as correct in both cloud/not cloud
+                hosts = [r.host_id for r in replicas if r.is_up is not False and r.datacenter == local_dc]
             if not hosts:
                 hosts.append(hostname)  # fallback to default host if no replicas in current dc
             return {'hosts': tuple(hosts), 'attempts': 0, 'rows': 0, 'workerno': -1}
@@ -1414,6 +1418,7 @@ def __init__(self, params, target):
         self.connect_timeout = params['connect_timeout']
         self.cql_version = params['cql_version']
         self.auth_provider = params['auth_provider']
+        self.parent_cluster = params['parent_cluster']
         self.ssl = params['ssl']
         self.protocol_version = params['protocol_version']
         self.config_file = params['config_file']
@@ -1681,23 +1686,36 @@ def connect(self, host):
             session.add_request()
             return session
 
-        new_cluster = Cluster(
-            contact_points=(host,),
+        endpoint = self._endpoint_for_host(host)
+
+        new_cluster = cluster_factory(
+            endpoint,
+            whitelist_lbp=endpoint,
+            cloud=self.parent_cluster.cloud,
             port=self.port,
             cql_version=self.cql_version,
             protocol_version=self.protocol_version,
             auth_provider=self.auth_provider,
-            ssl_options=ssl_settings(host, self.config_file) if self.ssl else None,
-            load_balancing_policy=WhiteListRoundRobinPolicy([host]),
+            ssl_options=ssl_settings(endpoint, self.config_file) if self.ssl else None,
             default_retry_policy=ExpBackoffRetryPolicy(self),
             compression=None,
             control_connection_timeout=self.connect_timeout,
             connect_timeout=self.connect_timeout,
             idle_heartbeat_interval=0)
+
         session = ExportSession(new_cluster, self)
         self.hosts_to_sessions[host] = session
         return session
 
+    def _endpoint_for_host(self, host):
+        endpoint = host
+        if isinstance(host, UUID):
+            for h in self.parent_cluster.metadata.all_hosts():
+                if h.host_id == host:
+                    endpoint = h.endpoint if isinstance(h.endpoint, SniEndPoint) else h.endpoint.address
+                    break
+        return endpoint
+
     def attach_callbacks(self, token_range, future, session):
         metadata = session.cluster.metadata
         ks_meta = metadata.keyspaces[self.ks]
@@ -2360,8 +2378,9 @@ def __init__(self, params):
     @property
     def session(self):
         if not self._session:
-            cluster = Cluster(
-                contact_points=(self.hostname,),
+            cluster = cluster_factory(
+                self.hostname,
+                whitelist_lbp=False,
                 port=self.port,
                 cql_version=self.cql_version,
                 protocol_version=self.protocol_version,
@@ -2373,6 +2392,7 @@ def session(self):
                 control_connection_timeout=self.connect_timeout,
                 connect_timeout=self.connect_timeout,
                 idle_heartbeat_interval=0,
+                cloud=self.parent_cluster.cloud,
                 connection_class=ConnectionWrapper)
 
             self._session = cluster.connect(self.ks)
diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py
index 94e4fd0addd1..dc459c06b415 100644
--- a/pylib/cqlshlib/cql3handling.py
+++ b/pylib/cqlshlib/cql3handling.py
@@ -215,6 +215,7 @@ def dequote_value(cqlword):
          | <boolean>
          | <blobLiteral>
          | <collectionLiteral>
+         | <vectorLiteral>
          | <functionLiteral> <functionArguments>
          | "NULL"
          ;
@@ -246,6 +247,9 @@ def dequote_value(cqlword):
 <mapLiteral> ::= "{" <term> ":" <term> ( "," <term> ":" <term> )* "}"
                ;
 
+<vectorLiteral> ::= "[" ( <float> ( "," <float> )* )? "]"
+                ;
+
 <anyFunctionName> ::= ( ksname=<cfOrKsName> dot="." )? udfname=<cfOrKsName> ;
 
 <userFunctionName> ::= ( ksname=<nonSystemKeyspaceName> dot="." )? udfname=<cfOrKsName> ;
@@ -736,8 +740,9 @@ def working_on_keyspace(ctxt):
                           ( "GROUP" "BY" <groupByClause> ( "," <groupByClause> )* )?
                           ( "ORDER" "BY" <orderByClause> ( "," <orderByClause> )* )?
                           ( "PER" "PARTITION" "LIMIT" perPartitionLimit=<wholenumber> )?
-                          ( "LIMIT" limit=<wholenumber> )?
+                          ( "LIMIT" limit=<wholenumber> ( "OFFSET" offset=<wholenumber> )? )?
                           ( "ALLOW" "FILTERING" )?
+                          ( "WITH" "ann_options" "=" <mapLiteral> )?
                     ;
 <whereClause> ::= <relation> ( "AND" <relation> )*
                 ;
@@ -764,7 +769,7 @@ def working_on_keyspace(ctxt):
              ;
 <selectionFunctionArguments> ::= "(" ( <selector> ( "," <selector> )* )? ")"
                           ;
-<orderByClause> ::= [ordercol]=<cident> ( "ASC" | "DESC" )?
+<orderByClause> ::= [ordercol]=<cident> ( "ANN" "OF" <vectorLiteral> )? ( "ASC" | "DESC" )?
                   ;
 <groupByClause> ::= [groupcol]=<cident>
                   | <functionName><groupByFunctionArguments>
@@ -1341,6 +1346,8 @@ def create_cf_composite_primary_key_comma_completer(ctxt, cass):
                             ")" )?
                             ("RETURNS" "NULL" | "CALLED") "ON" "NULL" "INPUT"
                             "RETURNS" <storageType>
+                            ( "DETERMINISTIC" )?
+                            ( "MONOTONIC" ( "ON" <cident> )? )?
                             "LANGUAGE" <cident> "AS" <stringLiteral>
                          ;
 
@@ -1354,6 +1361,7 @@ def create_cf_composite_primary_key_comma_completer(ctxt, cass):
                             "STYPE" <storageType>
                             ( "FINALFUNC" <refUserFunctionName> )?
                             ( "INITCOND" <term> )?
+                            ( "DETERMINISTIC" )?
                          ;
 
 '''
diff --git a/pylib/cqlshlib/cqlhandling.py b/pylib/cqlshlib/cqlhandling.py
index 504371c16dcb..09c376b15a15 100644
--- a/pylib/cqlshlib/cqlhandling.py
+++ b/pylib/cqlshlib/cqlhandling.py
@@ -26,7 +26,7 @@
 
 cql_keywords_reserved = {'add', 'allow', 'alter', 'and', 'apply', 'asc', 'authorize', 'batch', 'begin', 'by',
                          'columnfamily', 'create', 'delete', 'desc', 'describe', 'drop', 'entries', 'execute', 'from',
-                         'full', 'grant', 'if', 'in', 'index', 'infinity', 'insert', 'into', 'is', 'keyspace', 'limit',
+                         'full', 'geo_distance', 'grant', 'if', 'in', 'index', 'infinity', 'insert', 'into', 'is', 'keyspace', 'limit',
                          'materialized', 'modify', 'nan', 'norecursive', 'not', 'null', 'of', 'on', 'or', 'order',
                          'primary', 'rename', 'revoke', 'schema', 'select', 'set', 'table', 'to', 'token', 'truncate',
                          'unlogged', 'update', 'use', 'using', 'view', 'where', 'with'}
diff --git a/pylib/cqlshlib/cqlshhandling.py b/pylib/cqlshlib/cqlshhandling.py
index e6a121fd8067..243ef0576ba9 100644
--- a/pylib/cqlshlib/cqlshhandling.py
+++ b/pylib/cqlshlib/cqlshhandling.py
@@ -180,7 +180,7 @@ def registrator(f):
 '''
 
 cqlsh_tracing_cmd_syntax_rules = r'''
-<tracingCommand> ::= "TRACING" ( switch=( "ON" | "OFF" ) )?
+<tracingCommand> ::= "TRACING" ( switch=( "ON" | "OFF" | "FULL" | "COMPACT" ) )?
                    ;
 '''
 
diff --git a/pylib/cqlshlib/cqlshmain.py b/pylib/cqlshlib/cqlshmain.py
index e3a2c14ae4f2..867c7c10a47b 100755
--- a/pylib/cqlshlib/cqlshmain.py
+++ b/pylib/cqlshlib/cqlshmain.py
@@ -34,16 +34,14 @@
 
 import cassandra
 from cassandra.auth import PlainTextAuthProvider
-from cassandra.cluster import Cluster
 from cassandra.cqltypes import cql_typename
 from cassandra.marshal import int64_unpack
 from cassandra.metadata import (ColumnMetadata, KeyspaceMetadata, TableMetadata)
-from cassandra.policies import WhiteListRoundRobinPolicy
 from cassandra.query import SimpleStatement, ordered_dict_factory, TraceUnavailable
 from cassandra.util import datetime_from_timestamp
 
 from cqlshlib import cql3handling, pylexotron, sslhandling, cqlshhandling, authproviderhandling
-from cqlshlib.copyutil import ExportTask, ImportTask
+from cqlshlib.copyutil import ExportTask, ImportTask, ImportConversion
 from cqlshlib.displaying import (ANSI_RESET, BLUE, COLUMN_NAME_COLORS, CYAN,
                                  RED, WHITE, FormattedValue, colorme)
 from cqlshlib.formatting import (DEFAULT_DATE_FORMAT, DEFAULT_NANOTIME_FORMAT,
@@ -55,6 +53,12 @@
     from cqlshlib.serverversion import version as build_version
 except ImportError:
     build_version = 'UNKNOWN'
+from cqlshlib.geotypes import patch_geotypes_import_conversion  # nopep8
+from cqlshlib.daterangetype import patch_daterange_import_conversion  # nopep
+from cqlshlib.driver import cluster_factory
+
+patch_geotypes_import_conversion(ImportConversion)
+patch_daterange_import_conversion(ImportConversion)
 
 
 UTF8 = 'utf-8'
@@ -92,6 +96,7 @@
 DEFAULT_SSL = False
 DEFAULT_CONNECT_TIMEOUT_SECONDS = 5
 DEFAULT_REQUEST_TIMEOUT_SECONDS = 10
+DEFAULT_NO_FILE_IO = False
 
 DEFAULT_FLOAT_PRECISION = 5
 DEFAULT_DOUBLE_PRECISION = 5
@@ -127,6 +132,8 @@
 parser.add_argument("-u", "--username", help="Authenticate as user.")
 parser.add_argument("-p", "--password", help="Authenticate using password.")
 parser.add_argument('-k', '--keyspace', help='Authenticate to the given keyspace.')
+parser.add_argument('-b', '--secure-connect-bundle',
+                    help="Connect using secure connect bundle. If this option is specified host, port settings are ignored.")
 parser.add_argument("-f", "--file", help="Execute commands from FILE, then exit")
 parser.add_argument('--debug', action='store_true',
                     help='Show additional debugging information')
@@ -147,9 +154,16 @@
 parser.add_argument("--connect-timeout", default=DEFAULT_CONNECT_TIMEOUT_SECONDS, dest='connect_timeout',
                     help='Specify the connection timeout in seconds (default: %(default)s seconds).')
 parser.add_argument("--request-timeout", default=DEFAULT_REQUEST_TIMEOUT_SECONDS, dest='request_timeout',
-                    help='Specify the default request timeout in seconds (default: %(default)s seconds).')
+                    help='Specify the default request timeout in seconds (default: %(default)s seconds)')
+parser.add_argument("--consistency-level", dest='consistency_level',
+                    help='Specify the initial consistency level.')
+parser.add_argument("--serial-consistency-level", dest='serial_consistency_level',
+                    help='Specify the initial serial consistency level.')
 parser.add_argument("-t", "--tty", action='store_true', dest='tty',
                     help='Force tty mode (command prompt).')
+parser.add_argument("--no-file-io", action='store_true', dest='no_file_io',
+                    help='Disable cqlsh commands that perform file I/O.')
+parser.add_argument('--disable-history', action='store_true', help='Disable saving of history', default=False)
 
 # This is a hidden option to suppress the warning when the -p/--password command line option is used.
 # Power users may use this option if they know no other people has access to the system where cqlsh is run or don't care about security.
@@ -206,15 +220,22 @@ def resolve_cql_history_file():
 
 CQL_DIR = os.path.dirname(CONFIG_FILE)
 
-CQL_ERRORS = (
-    cassandra.AlreadyExists, cassandra.AuthenticationFailed, cassandra.CoordinationFailure,
-    cassandra.InvalidRequest, cassandra.Timeout, cassandra.Unauthorized, cassandra.OperationTimedOut,
-    cassandra.cluster.NoHostAvailable,
+CQL_TRACED_ERRORS = (
+    cassandra.CoordinationFailure,  # superclass of ReadFailure
+    cassandra.Timeout,              # superclass of ReadTimeout
+    cassandra.OperationTimedOut,
+    cassandra.protocol.InternalError,
+    cassandra.cluster.NoHostAvailable
+)
+CQL_ERRORS = CQL_TRACED_ERRORS + (
+    cassandra.AlreadyExists, cassandra.AuthenticationFailed,
+    cassandra.InvalidRequest, cassandra.Unauthorized,
     cassandra.connection.ConnectionBusy, cassandra.connection.ProtocolError, cassandra.connection.ConnectionException,
-    cassandra.protocol.ErrorMessage, cassandra.protocol.InternalError, cassandra.query.TraceUnavailable
+    cassandra.protocol.ErrorMessage, cassandra.query.TraceUnavailable
 )
 
 debug_completion = bool(os.environ.get('CQLSH_DEBUG_COMPLETION', '') == 'YES')
+trace_all_errors = bool(os.environ.get('CQLSH_TRACE_ALL_ERRORS', '') == 'YES')
 
 
 class NoKeyspaceError(Exception):
@@ -341,14 +362,19 @@ class Shell(cmd.Cmd):
     last_hist = None
     shunted_query_out = None
     use_paging = True
+    no_file_io = DEFAULT_NO_FILE_IO
 
     default_page_size = 100
+    consistency_level = None
+    serial_consistency_level = None
 
     def __init__(self, hostname, port, color=False,
                  username=None, encoding=None, stdin=None, tty=True,
                  completekey=DEFAULT_COMPLETEKEY, browser=None, use_conn=None,
                  cqlver=None, keyspace=None,
-                 tracing_enabled=False, expand_enabled=False,
+                 secure_connect_bundle=None,
+                 consistency_level=None, serial_consistency_level=None,
+                 tracing_style='off', expand_enabled=False,
                  display_nanotime_format=DEFAULT_NANOTIME_FORMAT,
                  display_timestamp_format=DEFAULT_TIMESTAMP_FORMAT,
                  display_date_format=DEFAULT_DATE_FORMAT,
@@ -361,9 +387,12 @@ def __init__(self, hostname, port, color=False,
                  request_timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,
                  protocol_version=None,
                  connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS,
+                 no_file_io=DEFAULT_NO_FILE_IO,
                  is_subshell=False,
-                 auth_provider=None):
+                 auth_provider=None,
+                 debug=False):
         cmd.Cmd.__init__(self, completekey=completekey)
+        self.debug = debug
         self.hostname = hostname
         self.port = port
         self.auth_provider = auth_provider
@@ -378,22 +407,37 @@ def __init__(self, hostname, port, color=False,
 
         self.keyspace = keyspace
         self.ssl = ssl
-        self.tracing_enabled = tracing_enabled
+        self.tracing_style = tracing_style
         self.page_size = self.default_page_size
         self.expand_enabled = expand_enabled
+
+        if not consistency_level:
+            raise Exception('Argument consistency_level must not be None')
+        if not serial_consistency_level:
+            raise Exception('Argument serial_consistency_level must not be None')
+        self.consistency_level = consistency_level
+        self.serial_consistency_level = serial_consistency_level
+
+        self.secure_connect_bundle = secure_connect_bundle
+
         if use_conn:
             self.conn = use_conn
         else:
             kwargs = {}
             if protocol_version is not None:
                 kwargs['protocol_version'] = protocol_version
-            self.conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=cqlver,
-                                auth_provider=self.auth_provider,
-                                ssl_options=sslhandling.ssl_settings(hostname, CONFIG_FILE) if ssl else None,
-                                load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]),
-                                control_connection_timeout=connect_timeout,
-                                connect_timeout=connect_timeout,
-                                **kwargs)
+            self.conn = cluster_factory(self.hostname,
+                                        port=self.port,
+                                        cql_version=cqlver,
+                                        auth_provider=self.auth_provider,
+                                        ssl_options=sslhandling.ssl_settings(hostname, CONFIG_FILE) if ssl else None,
+                                        control_connection_timeout=connect_timeout,
+                                        connect_timeout=connect_timeout,
+                                        secure_connect_bundle=secure_connect_bundle,
+                                        application_name=description,
+                                        application_version=version,
+                                        **kwargs)
+
         self.owns_connection = not use_conn
 
         if keyspace:
@@ -417,7 +461,9 @@ def __init__(self, hostname, port, color=False,
 
         self.session.default_timeout = request_timeout
         self.session.row_factory = ordered_dict_factory
-        self.session.default_consistency_level = cassandra.ConsistencyLevel.ONE
+        self.session.default_consistency_level = self.consistency_level
+        self.session.default_serial_consistency_level = self.serial_consistency_level
+
         self.get_connection_versions()
         self.set_expanded_cql_version(self.connection_versions['cql'])
 
@@ -448,21 +494,51 @@ def __init__(self, hostname, port, color=False,
             self.show_line_nums = True
         self.stdin = stdin
         self.query_out = sys.stdout
-        self.consistency_level = cassandra.ConsistencyLevel.ONE
-        self.serial_consistency_level = cassandra.ConsistencyLevel.SERIAL
 
         self.empty_lines = 0
         self.statement_error = False
         self.single_statement = single_statement
+        self.no_file_io = no_file_io
         self.is_subshell = is_subshell
 
     def check_build_versions(self):
         baseversion = self.connection_versions['build']
-        extra = baseversion.rfind('-')
-        if extra != -1:
-            baseversion = baseversion[0:extra]
-        if baseversion != build_version:
-            print("WARNING: cqlsh was built against {}, but this server is {}.  All features may not work!".format(build_version, baseversion))
+        buildversion = build_version
+
+        # Remove any pre-release secondary component, e.g. SNAPSHOT, when delimited by either '-' or '.'
+        # ex. 5.0.0-beta2-SNAPSHOT -> 5.0.0-beta2
+        # ex. 5.0.0-beta2.SNAPSHOT -> 5.0.0-beta2
+        # ex. 5.0.2-SNAPSHOT - > 5.0.2
+        if baseversion.count('-') > 0:
+            if baseversion.count('-') > build_version.count('-'):
+                # When the baseversion has more dashes than the build_version
+                # e.g. baseversion=5.0.0-beta2-SNAPSHOT, build_version=5.0.0-beta2
+                extra = baseversion.rfind('-')
+                if extra != -1:
+                    baseversion = baseversion[0:extra]
+            else:
+                # When the pre-release component delimiter is a dot, and beyond the last '-'
+                # e.g. 5.0.0-beta2.SNAPSHOT, build_version=5.0.0-beta2
+                prerelease_start = baseversion.rfind('-')
+                extra = baseversion.rfind('.')
+                if extra > prerelease_start:
+                    baseversion = baseversion[0:extra]
+
+        # CC produces a version string that includes a patch number, e.g. 5.0.0
+        # Remove the patch number if the build version does not include it
+        base_version_parts = baseversion.count('.')
+        build_version_parts = buildversion.count('.')
+        if base_version_parts > 1 and build_version_parts < base_version_parts:
+            base_parts = baseversion.split('-')
+            base_versions = base_parts[0].split('.')
+            if base_versions[-1] == '0':
+                base_versions.pop(-1)
+            baseversion = '.'.join(base_versions)
+            if len(base_parts) > 1:
+                baseversion = baseversion + '-' + base_parts[1]
+
+        if baseversion != buildversion:
+            print("WARNING: cqlsh was built against {}, but this server is {}.  All features may not work!".format(buildversion, baseversion))
 
     @property
     def batch_mode(self):
@@ -524,7 +600,7 @@ def show_version(self):
         print("[cqlsh %(shver)s | Cassandra %(build)s | CQL spec %(cql)s | Native protocol v%(protocol)s]" % vers)
 
     def show_session(self, sessionid, partial_session=False):
-        print_trace_session(self, self.session, sessionid, partial_session)
+        print_trace_session(self, self.tracing_style, self.session, sessionid, partial_session)
 
     def show_replicas(self, token_value, keyspace=None):
         ks = self.current_keyspace if keyspace is None else keyspace
@@ -906,13 +982,13 @@ def do_use(self, parsed):
                 self.current_keyspace = ksname.lower()
 
     def do_select(self, parsed):
-        tracing_was_enabled = self.tracing_enabled
+        tracing_was_enabled = self.tracing_style
         ksname = parsed.get_binding('ksname')
         stop_tracing = ksname == 'system_traces' or (ksname is None and self.current_keyspace == 'system_traces')
-        self.tracing_enabled = self.tracing_enabled and not stop_tracing
+        self.tracing_style = "off" if stop_tracing else self.tracing_style
         statement = parsed.extract_orig()
         self.perform_statement(statement)
-        self.tracing_enabled = tracing_was_enabled
+        self.tracing_style = tracing_was_enabled
 
     def perform_statement(self, statement):
 
@@ -923,10 +999,10 @@ def perform_statement(self, statement):
             if future.warnings:
                 self.print_warnings(future.warnings)
 
-            if self.tracing_enabled:
+            if self.tracing_style:
                 try:
                     for trace in future.get_all_query_traces(max_wait_per=self.max_trace_wait, query_cl=self.consistency_level):
-                        print_trace(self, trace)
+                        print_trace(self, self.tracing_style, trace)
                 except TraceUnavailable:
                     msg = "Statement trace did not complete within %d seconds; trace data may be incomplete." % (self.session.max_trace_wait,)
                     self.writeresult(msg, color=RED)
@@ -965,13 +1041,20 @@ def perform_simple_statement(self, statement):
         if not statement:
             return False, None
 
-        future = self.session.execute_async(statement, trace=self.tracing_enabled)
+        def print_cql_error(err):
+            err_msg = maybe_ensure_text(err.message if hasattr(err, 'message') else str(err))
+            self.printerr(str(err.__class__.__name__) + ": " + err_msg)
+
+        future = self.session.execute_async(statement, trace=self.tracing_style in ["full", "compact"])
         result = None
+        traced_err = None
         try:
             result = future.result()
+        except CQL_TRACED_ERRORS as err:
+            traced_err = err
+            print_cql_error(err)
         except CQL_ERRORS as err:
-            err_msg = err.message if hasattr(err, 'message') else str(err)
-            self.printerr(str(err.__class__.__name__) + ": " + err_msg)
+            print_cql_error(err)
         except Exception:
             import traceback
             self.printerr(traceback.format_exc())
@@ -986,7 +1069,10 @@ def perform_simple_statement(self, statement):
                 self.conn.refresh_schema_metadata(-1)
 
         if result is None:
-            return False, None
+            if traced_err or trace_all_errors:
+                return False, future
+            else:
+                return False, None
 
         if statement.query_string[:6].lower() == 'select':
             self.print_result(result, self.parse_for_select_meta(statement.query_string))
@@ -1062,7 +1148,7 @@ def print_static_result(self, result, table_meta, with_header, tty, row_count_of
         else:
             self.print_formatted_result(formatted_names, formatted_values, with_header, tty)
 
-    def print_formatted_result(self, formatted_names, formatted_values, with_header, tty):
+    def print_formatted_result(self, formatted_names, formatted_values, with_header, tty, justify_last=True):
         # determine column widths
         widths = [n.displaywidth for n in formatted_names]
         if formatted_values is not None:
@@ -1083,7 +1169,11 @@ def print_formatted_result(self, formatted_names, formatted_values, with_header,
 
         # print row data
         for row in formatted_values:
-            line = ' | '.join(col.rjust(w, color=self.color) for (col, w) in zip(row, widths))
+            if justify_last:
+                line = ' | '.join(col.rjust(w, color=self.color) for (col, w) in zip(row, widths))
+            else:
+                line = ' | '.join(col.rjust(w, color=self.color) for (col, w) in zip(row[:-1], widths[:-1])) + ' | ' + \
+                       row[-1].coloredval
             self.writeresult(' ' + line)
 
         if tty:
@@ -1442,6 +1532,10 @@ def do_copy(self, parsed):
         on a line by itself to end the data input.
         """
 
+        if self.no_file_io:
+            self.printerr('No file I/O permitted')
+            return
+
         ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
         if ks is None:
             ks = self.current_keyspace
@@ -1535,6 +1629,11 @@ def do_source(self, parsed):
 
         See also the --file option to cqlsh.
         """
+
+        if self.no_file_io:
+            self.printerr('No file I/O permitted')
+            return
+
         fname = parsed.get_binding('fname')
         fname = os.path.expanduser(self.cql_unprotect_value(fname))
         try:
@@ -1548,7 +1647,9 @@ def do_source(self, parsed):
                          username=self.username,
                          encoding=self.encoding, stdin=f, tty=False, use_conn=self.conn,
                          cqlver=self.cql_version, keyspace=self.current_keyspace,
-                         tracing_enabled=self.tracing_enabled,
+                         consistency_level=self.consistency_level,
+                         serial_consistency_level=self.serial_consistency_level,
+                         tracing_style=self.tracing_style,
                          display_nanotime_format=self.display_nanotime_format,
                          display_timestamp_format=self.display_timestamp_format,
                          display_date_format=self.display_date_format,
@@ -1558,6 +1659,7 @@ def do_source(self, parsed):
                          max_trace_wait=self.max_trace_wait, ssl=self.ssl,
                          request_timeout=self.session.default_timeout,
                          connect_timeout=self.conn.connect_timeout,
+                         no_file_io=self.no_file_io,
                          is_subshell=True,
                          auth_provider=self.auth_provider)
         # duplicate coverage related settings in subshell
@@ -1594,6 +1696,11 @@ def do_capture(self, parsed):
         To inspect the current capture configuration, use CAPTURE with no
         arguments.
         """
+
+        if self.no_file_io:
+            self.printerr('No file I/O permitted')
+            return
+
         fname = parsed.get_binding('fname')
         if fname is None:
             if self.shunted_query_out is not None:
@@ -1636,20 +1743,36 @@ def do_tracing(self, parsed):
 
           Enables or disables request tracing.
 
-        TRACING ON
+        TRACING FULL [synonym: ON]
 
-          Enables tracing for all further requests.
+          Enables full tracing for all further requests.
 
         TRACING OFF
 
           Disables tracing.
 
+        TRACING COMPACT
+
+          Enables compact tracing for all further requests.
+
         TRACING
 
           TRACING with no arguments shows the current tracing status.
         """
-        self.tracing_enabled \
-            = self.on_off_switch("TRACING", self.tracing_enabled, parsed.get_binding('switch'))
+        switch = parsed.get_binding('switch')
+        if switch is None:
+            print(f"Tracing is currently set to {self.tracing_style.upper()}. Use TRACING [OFF|FULL|COMPACT] to change.")
+            return
+
+        switch = switch.upper()
+        if switch in ['ON', 'OFF', 'FULL', 'COMPACT']:
+            self.tracing_style = {'ON': 'full',
+                                  'FULL': 'full',
+                                  'OFF': 'off',
+                                  'COMPACT': 'compact'}[switch]
+            print(f"Tracing set to {self.tracing_style.upper()}.")
+        else:
+            self.printerr(f"Invalid tracing option: {switch}. Use TRACING [OFF|FULL|COMPACT].")
 
     def do_expand(self, parsed):
         """
@@ -1733,8 +1856,9 @@ def do_login(self, parsed):
 
         LOGIN <username> (<password>)
 
-           Login using the specified username. If password is specified, it will be used
-           otherwise, you will be prompted to enter.
+           Login using the specified username.
+           If password is specified it should be wrapped with single quotes.
+           If not specified you will be prompted to enter.
         """
         username = parsed.get_binding('username')
         password = parsed.get_binding('password')
@@ -1745,13 +1869,15 @@ def do_login(self, parsed):
 
         auth_provider = PlainTextAuthProvider(username=username, password=password)
 
-        conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=self.conn.cql_version,
-                       protocol_version=self.conn.protocol_version,
-                       auth_provider=auth_provider,
-                       ssl_options=self.conn.ssl_options,
-                       load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]),
-                       control_connection_timeout=self.conn.connect_timeout,
-                       connect_timeout=self.conn.connect_timeout)
+        conn = cluster_factory(self.hostname,
+                               port=self.port,
+                               cql_version=self.conn.cql_version,
+                               protocol_version=self.conn.protocol_version,
+                               auth_provider=auth_provider,
+                               ssl_options=self.conn.ssl_options,
+                               control_connection_timeout=self.conn.connect_timeout,
+                               connect_timeout=self.conn.connect_timeout,
+                               secure_connect_bundle=self.secure_connect_bundle)
 
         if self.current_keyspace:
             session = conn.connect(self.current_keyspace)
@@ -1762,6 +1888,7 @@ def do_login(self, parsed):
         session.default_timeout = self.session.default_timeout
         session.row_factory = self.session.row_factory
         session.default_consistency_level = self.session.default_consistency_level
+        session.default_serial_consistency_level = self.session.default_serial_consistency_level
         session.max_trace_wait = self.session.max_trace_wait
 
         # Update after we've connected in case we fail to authenticate
@@ -1791,6 +1918,11 @@ def do_clear(self, parsed):
     do_cls = do_clear
 
     def do_debug(self, parsed):
+
+        if self.no_file_io:
+            self.printerr('No file I/O permitted')
+            return
+
         import pdb
         pdb.set_trace()
 
@@ -1946,8 +2078,8 @@ def init_history(self):
             # configure length of history shown
             self.max_history_length_shown = 50
 
-    def save_history(self):
-        if readline is not None:
+    def save_history(self, history_disabled=False):
+        if readline is not None and not history_disabled:
             try:
                 readline.write_history_file(HISTORY)
             except IOError:
@@ -2053,6 +2185,7 @@ def read_options(cmdlineargs, environment=os.environ):
     argvalues.credentials = os.path.expanduser(option_with_default(configs.get, 'authentication', 'credentials',
                                                                    os.path.join(CQL_DIR, 'credentials')))
     argvalues.keyspace = option_with_default(configs.get, 'authentication', 'keyspace')
+    argvalues.secure_connect_bundle = option_with_default(configs.get, 'connection', 'secure_connect_bundle')
     argvalues.browser = option_with_default(configs.get, 'ui', 'browser', None)
     argvalues.completekey = option_with_default(configs.get, 'ui', 'completekey',
                                                 DEFAULT_COMPLETEKEY)
@@ -2082,12 +2215,17 @@ def read_options(cmdlineargs, environment=os.environ):
     argvalues.ssl = option_with_default(configs.getboolean, 'connection', 'ssl', DEFAULT_SSL)
     argvalues.encoding = option_with_default(configs.get, 'ui', 'encoding', UTF8)
 
+    argvalues.consistency_level = option_with_default(configs.get, 'cql', 'consistency_level', None)
+    argvalues.serial_consistency_level = option_with_default(configs.get, 'cql', 'serial_consistency_level', 'SERIAL')
+
     argvalues.tty = option_with_default(configs.getboolean, 'ui', 'tty', sys.stdin.isatty())
     argvalues.protocol_version = option_with_default(configs.getint, 'protocol', 'version', None)
     argvalues.cqlversion = option_with_default(configs.get, 'cql', 'version', None)
     argvalues.connect_timeout = option_with_default(configs.getint, 'connection', 'timeout', DEFAULT_CONNECT_TIMEOUT_SECONDS)
     argvalues.request_timeout = option_with_default(configs.getint, 'connection', 'request_timeout', DEFAULT_REQUEST_TIMEOUT_SECONDS)
     argvalues.execute = None
+    argvalues.no_file_io = option_with_default(configs.getboolean, 'ui', 'no_file_io', DEFAULT_NO_FILE_IO)
+    argvalues.disable_history = option_with_default(configs.getboolean, 'history', 'disabled', False)
     argvalues.insecure_password_without_warning = False
 
     options, arguments = parser.parse_known_args(cmdlineargs, argvalues)
@@ -2138,6 +2276,29 @@ def read_options(cmdlineargs, environment=os.environ):
     options.password = maybe_ensure_text(options.password)
     options.keyspace = maybe_ensure_text(options.keyspace)
 
+    serial_levels = [cassandra.ConsistencyLevel.SERIAL, cassandra.ConsistencyLevel.LOCAL_SERIAL]
+
+    # If unspecified, set the proper defaut CL
+    default_cl = 'LOCAL_QUORUM' if options.secure_connect_bundle else 'ONE'
+    if options.consistency_level is None:
+        options.consistency_level = default_cl
+
+    try:
+        cl = cassandra.ConsistencyLevel.name_to_value[options.consistency_level.upper()]
+        if cl in serial_levels:
+            raise KeyError
+        options.consistency_level = cl
+    except KeyError:
+        parser.error('"{}" is not a valid consistency level'.format(options.consistency_level))
+
+    try:
+        cl = cassandra.ConsistencyLevel.name_to_value[options.serial_consistency_level.upper()]
+        if cl not in serial_levels:
+            raise KeyError
+        options.serial_consistency_level = cl
+    except KeyError:
+        parser.error('"{}" is not a valid serial consistency level'.format(options.serial_consistency_level))
+
     hostname = option_with_default(configs.get, 'connection', 'hostname', DEFAULT_HOST)
     port = option_with_default(configs.get, 'connection', 'port', DEFAULT_PORT)
 
@@ -2265,8 +2426,11 @@ def main(cmdline, pkgpath):
     if options.debug:
         sys.stderr.write("Using CQL driver: %s\n" % (cassandra,))
         sys.stderr.write("Using connect timeout: %s seconds\n" % (options.connect_timeout,))
+        sys.stderr.write("Using consistency level: %s\n" % (cassandra.ConsistencyLevel.value_to_name[options.consistency_level],))
         sys.stderr.write("Using '%s' encoding\n" % (options.encoding,))
         sys.stderr.write("Using ssl: %s\n" % (options.ssl,))
+        if options.secure_connect_bundle:
+            sys.stderr.write("Using secure connect bundle: %s\n" % (options.secure_connect_bundle, ))
 
     # create timezone based on settings, environment or auto-detection
     timezone = None
@@ -2326,9 +2490,13 @@ def main(cmdline, pkgpath):
                       tty=options.tty,
                       completekey=options.completekey,
                       browser=options.browser,
+                      debug=options.debug,
                       protocol_version=options.protocol_version,
                       cqlver=options.cqlversion,
                       keyspace=options.keyspace,
+                      secure_connect_bundle=options.secure_connect_bundle,
+                      consistency_level=options.consistency_level,
+                      serial_consistency_level=options.serial_consistency_level,
                       display_timestamp_format=options.time_format,
                       display_nanotime_format=options.nanotime_format,
                       display_date_format=options.date_format,
@@ -2340,6 +2508,7 @@ def main(cmdline, pkgpath):
                       single_statement=options.execute,
                       request_timeout=options.request_timeout,
                       connect_timeout=options.connect_timeout,
+                      no_file_io=options.no_file_io,
                       encoding=options.encoding,
                       auth_provider=authproviderhandling.load_auth_provider(
                           config_file=CONFIG_FILE,
@@ -2352,8 +2521,6 @@ def main(cmdline, pkgpath):
         sys.exit('Connection error: %s' % (e,))
     except VersionNotSupported as e:
         sys.exit('Unsupported CQL version: %s' % (e,))
-    if options.debug:
-        shell.debug = True
     if options.coverage:
         shell.coverage = True
         import signal
@@ -2366,7 +2533,7 @@ def handle_sighup():
 
     shell.init_history()
     shell.cmdloop()
-    shell.save_history()
+    shell.save_history(options.disable_history)
 
     if shell.batch_mode and shell.statement_error:
         sys.exit(2)
diff --git a/pylib/cqlshlib/daterangetype.py b/pylib/cqlshlib/daterangetype.py
new file mode 100644
index 000000000000..e052001683b6
--- /dev/null
+++ b/pylib/cqlshlib/daterangetype.py
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+from cassandra.util import DateRange, DateRangeBound, DateRangePrecision, OPEN_BOUND, ms_timestamp_from_datetime
+
+
+def _raise_parse_error(msg):
+    """
+    We need to throw ParseError which is defined in copyutil
+    since copyutil imports this file as part of it's initialization though,
+    importing it at the top of this file would create an import loop, so
+    we just import it locally when/if we need it
+    """
+    from cqlshlib import copyutil
+    raise copyutil.ParseError(msg)
+
+
+def _new_date_range_bound(datetime, precision):
+    millis = ms_timestamp_from_datetime(datetime)
+    return DateRangeBound(value=millis, precision=precision)
+
+
+def _parse_date_range_bound(val):
+    if val == '*':
+        return OPEN_BOUND
+    else:
+        year = 1
+        month = 1
+        day = 1
+        hour = 0
+        minute = 0
+        second = 0
+        millisecond = 0
+        precision = DateRangePrecision.MILLISECOND
+
+        date_split = val.replace('Z', '').split('-')
+        if len(date_split) > 0:
+            year = int(date_split[0])
+            precision = DateRangePrecision.YEAR
+        if len(date_split) > 1:
+            month = int(date_split[1])
+            precision = DateRangePrecision.MONTH
+        if len(date_split) > 2:
+            day_split = date_split[2].split('T')
+            day = int(day_split[0])
+            precision = DateRangePrecision.DAY
+            if len(day_split) > 1:
+                time_split = day_split[1].split(':')
+                if len(time_split) > 0:
+                    hour = int(time_split[0])
+                    precision = DateRangePrecision.HOUR
+                if len(time_split) > 1:
+                    minute = int(time_split[1])
+                    precision = DateRangePrecision.MINUTE
+                if len(time_split) > 2:
+                    second_split = time_split[2].split('.')
+                    second = int(second_split[0])
+                    precision = DateRangePrecision.SECOND
+                    if len(second_split) > 1:
+                        millisecond = int(second_split[1])
+                        precision = DateRangePrecision.MILLISECOND
+
+    return _new_date_range_bound(
+        datetime=datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second,
+                          microsecond=millisecond * 1000),
+        precision=precision)
+
+
+def _convert_daterange(val):
+    if val.startswith('['):
+        if val.endswith(']'):
+            bounds = val[1:-1].split(' TO ')
+            if len(bounds) == 2:
+                lower_bound = _parse_date_range_bound(bounds[0].strip()).round_down()
+                upper_bound = _parse_date_range_bound(bounds[1].strip()).round_up()
+                return DateRange(lower_bound=lower_bound, upper_bound=upper_bound)
+            else:
+                _raise_parse_error("If date range starts with [ must contain ' TO '; got {}".format(val))
+        else:
+            _raise_parse_error("If date range starts with [ must end with ]; got {}".format(val))
+    else:
+        bound = _parse_date_range_bound(val).round_down()
+        return DateRange(value=bound)
+
+
+def _patch_get_converters(klass):
+    original_method = klass._get_converter
+
+    def new_method(self, cql_type):
+        if cql_type.typename == 'daterange':
+            return _convert_daterange
+        else:
+            return original_method(self, cql_type)
+
+    klass._get_converter = new_method
+
+
+def patch_daterange_import_conversion(klass):
+    """
+    monkey patches cqlshlib.copyutil.ImportConversion to support DateRangeType
+    """
+    _patch_get_converters(klass)
diff --git a/pylib/cqlshlib/driver.py b/pylib/cqlshlib/driver.py
new file mode 100644
index 000000000000..e0bc7cef951f
--- /dev/null
+++ b/pylib/cqlshlib/driver.py
@@ -0,0 +1,153 @@
+# Copyright DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import random
+import six
+import stat
+from cassandra.cluster import Cluster
+from cassandra.connection import UnixSocketEndPoint
+from cassandra.policies import WhiteListRoundRobinPolicy, RoundRobinPolicy
+from cassandra.pool import HostDistance
+
+
+class EndpointWhiteListRoundRobinPolicy(WhiteListRoundRobinPolicy):
+    """
+    WhileListRoundRobinPolicy dedicated for the cloud connection.
+
+    It is using endpoint instead of host address.
+
+    Note:
+    we want to override _allowed_hosts and _allowed_hosts_resolved which is the reason
+    to not call direct super class init but rather the one from RoundRobinPolicy.
+    """
+
+    def __init__(self, hosts):
+        self._allowed_hosts = self._allowed_hosts_resolved = tuple(hosts)
+        RoundRobinPolicy.__init__(self)
+
+    def populate(self, cluster, hosts):
+        self._live_hosts = frozenset(h for h in hosts if h.endpoint in self._allowed_hosts_resolved)
+
+        if len(hosts) <= 1:
+            self._position = 0
+        else:
+            self._position = random.randint(0, len(hosts) - 1)
+
+    def distance(self, host):
+        if host.endpoint in self._allowed_hosts_resolved:
+            return HostDistance.LOCAL
+        else:
+            return HostDistance.IGNORED
+
+    def on_up(self, host):
+        if host.endpoint in self._allowed_hosts_resolved:
+            RoundRobinPolicy.on_up(self, host)
+
+    def on_add(self, host):
+        if host.endpoint in self._allowed_hosts_resolved:
+            RoundRobinPolicy.on_add(self, host)
+
+
+def cluster_factory(host, whitelist_lbp=True, **kwargs):
+    """
+    Cluster factory to create a cassandra or dse cluster for cqlsh.
+
+    :param host: the host for the connection. This can be the hostname string or an EndPoint instance.
+    :param whitelist_lbp: Specify if a WhiteListRoundRobinPolicy should be applied. This can be set to True,
+                          False or an EndPoint instance, which is used to configure the policy when
+                          connecting to a cloud cluster. On a cloud cluster, the LBP has to be set after the
+                          nodes discovery. Otherwise, the LBP is set as usual to the execution profile.
+                          Default to True.
+    :kwargs: All other keyword arguments are passed to the Cluster constructor.
+    :return: A Cluster instance.
+    """
+
+    is_cloud_cluster = False
+    is_unix_socket_endpoint = False
+    endpoint = host
+    options = kwargs.copy()
+
+    # Configure the cluster contact point type and address.
+    if is_unix_socket(host):
+        # update endpoint and load balancing policy for unix socket
+        endpoint = UnixSocketEndPoint(host)
+        is_unix_socket_endpoint = True
+
+    # Determine if we are trying to connect to a cloud cluster
+    secure_connect_bundle = options.pop('secure_connect_bundle', None)
+    if secure_connect_bundle:
+        is_cloud_cluster = True
+        options['cloud'] = {'secure_connect_bundle': secure_connect_bundle}
+    elif 'cloud' in options:
+        is_cloud_cluster = options['cloud'] is not None
+
+    # Build the Cluster instance
+    if is_cloud_cluster:
+        return _cloud_cluster_factory(whitelist_lbp, **options)
+
+    if is_unix_socket_endpoint and 'port' in options:
+        del options['port']
+
+    contact_points = (endpoint,)
+
+    if whitelist_lbp:
+        lbp_class = WhiteListRoundRobinPolicy
+        if is_unix_socket_endpoint:
+            lbp_class = EndpointWhiteListRoundRobinPolicy
+
+        whitelist = [endpoint] if whitelist_lbp is True else [whitelist_lbp]
+        options['load_balancing_policy'] = lbp_class(whitelist)
+
+    return Cluster(contact_points=contact_points, **options)
+
+
+def is_unix_socket(hostname):
+    if isinstance(hostname, six.string_types) and os.path.exists(hostname):
+        mode = os.stat(hostname).st_mode
+        return stat.S_ISSOCK(mode)
+    return False
+
+
+def _cloud_cluster_factory(whitelist_lbp, **kwargs):
+    """
+    Create cloud cluster from given options.
+    Please notice that:
+    - cloud should be present in option
+    - contact_points, endpoint_factory, ssl_context, and ssl_options cannot be specified with a cloud configuration (will be removed)
+    - whitelist_lbp can be True (random contact point), False (no policy set), or specific endpoint
+    """
+    options = kwargs.copy()
+    assert 'cloud' in options
+
+    cloud_prohibited_options = ['contact_points', 'endpoint_factory', 'ssl_context', 'ssl_options']
+    for opt_name in cloud_prohibited_options:
+        options.pop(opt_name, None)
+
+    cluster = Cluster(**options)
+
+    if whitelist_lbp:
+
+        if whitelist_lbp is True:
+            # applying load balancing policy as we now know the contact points
+            contact_points = [random.choice(cluster.contact_points)]
+        else:
+            # An explicit host was specified
+            contact_points = [whitelist_lbp]
+
+        for execution_profile in cluster.profile_manager.profiles.values():
+            execution_profile.load_balancing_policy = EndpointWhiteListRoundRobinPolicy(contact_points)
+        cluster.endpoints_resolved = contact_points
+
+    return cluster
diff --git a/pylib/cqlshlib/geotypes.py b/pylib/cqlshlib/geotypes.py
new file mode 100644
index 000000000000..bad2f2b85b1b
--- /dev/null
+++ b/pylib/cqlshlib/geotypes.py
@@ -0,0 +1,121 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from numbers import Number
+import re
+
+from cassandra.metadata import protect_value
+from cassandra.util import Point, LineString, Polygon
+from geomet import wkt
+
+
+def _raise_parse_error(msg):
+    """
+    We need to throw ParseError which is defined in copyutil
+    since copyutil imports this file as part of it's initialization though,
+    importing it at the top of this file would create an import loop, so
+    we just import it locally when/if we need it
+    """
+    from cqlshlib import copyutil
+    raise copyutil.ParseError(msg)
+
+
+def _validate_point(val, raw):
+    if not all([isinstance(v, Number) for v in val]):
+        _raise_parse_error("Got non-numeric value in {}".format(raw))
+
+    if len(val) != 2:
+        _raise_parse_error("Got point with {} coordinates in: {}".format(len(val), raw))
+
+
+def _get_coords(val, expected_type):
+    geojson = wkt.loads(val)
+    gjtype = geojson.get('type')
+    if gjtype != expected_type:
+        _raise_parse_error("Expected {} type, but got {} type for {}".format(expected_type, gjtype, val))
+
+    return geojson.get("coordinates", [])
+
+
+def _convert_point(val):
+    coords = _get_coords(val, 'Point')
+    _validate_point(coords, val)
+    point = Point(*coords)
+    return point
+
+
+def _convert_linestring(val):
+    points = _get_coords(val, 'LineString')
+    for xy in points:
+        _validate_point(xy, val)
+    linestring = LineString(points)
+    return linestring
+
+
+def _convert_polygon(val):
+    rings = _get_coords(val, 'Polygon')
+    if len(rings) == 0:
+        return Polygon([])
+
+    for ring in rings:
+        for point in ring:
+            _validate_point(point, val)
+
+    polygon = Polygon(exterior=rings[0], interiors=rings[1:])
+    return polygon
+
+
+def _patch_get_converters(klass):
+    """
+    patches the get converters method to convert WKT to dse.util.{Point, LineString, Polygon}
+    when using prepared statements to batch load
+    """
+    original_method = klass._get_converter
+
+    def new_method(self, cql_type):
+        if cql_type.typename == 'PointType':
+            return _convert_point
+        elif cql_type.typename == 'LineStringType':
+            return _convert_linestring
+        elif cql_type.typename == 'PolygonType':
+            return _convert_polygon
+        else:
+            return original_method(self, cql_type)
+    klass._get_converter = new_method
+
+
+def _patch_init(klass):
+    """
+    patches the constructor method to also protect (quote) geotype values
+    when making queries with string literal values
+    """
+    original_method = klass.__init__
+
+    def new_method(self, *args, **kwargs):
+        original_method(self, *args, **kwargs)
+        ptypes = zip(self.protectors, self.coltypes)
+        clean = lambda t: re.sub("[\\W]", "", t.split('.')[-1])  # discard java package names and ' characters
+        gtypes = {'PointType', 'LineStringType', 'PolygonType'}
+        self.protectors = [protect_value if clean(t) in gtypes else p for p, t in ptypes]
+    klass.__init__ = new_method
+
+
+def patch_geotypes_import_conversion(klass):
+    """
+    monkey patches cqlshlib.copyutil.ImportConversion to support geotypes
+    """
+    _patch_get_converters(klass)
+    _patch_init(klass)
diff --git a/pylib/cqlshlib/test/run_cqlsh.py b/pylib/cqlshlib/test/run_cqlsh.py
index 288d3ed95184..def20fbe5fcb 100644
--- a/pylib/cqlshlib/test/run_cqlsh.py
+++ b/pylib/cqlshlib/test/run_cqlsh.py
@@ -120,6 +120,7 @@ def __init__(self, path, tty=True, env=None, args=()):
             env = {}
         self.env = env
         self.readbuf = ''
+        self._partial_bytes = b''
 
         self.start_proc()
 
@@ -139,9 +140,9 @@ def start_proc(self):
             self.childpty = masterfd
             self.send = self.send_tty
             self.read = self.read_tty
+            self._partial_bytes = b''
         else:
             stdin = stdout = subprocess.PIPE
-            stderr = subprocess.STDOUT
             self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args),
                                          env=self.env, stdin=stdin, stdout=stdout,
                                          stderr=stderr, bufsize=0, close_fds=False)
@@ -149,6 +150,7 @@ def start_proc(self):
             self.read = self.read_pipe
 
     def close(self):
+        self._partial_bytes = b''
         cqlshlog.info("Closing %r subprocess." % (self.exe_path,))
         if self.tty:
             os.close(self.childpty)
@@ -168,7 +170,23 @@ def send_pipe(self, data):
     def read_tty(self, blksize, timeout=None):
         buf = os.read(self.childpty, blksize)
         if isinstance(buf, bytes):
-            buf = buf.decode("utf-8")
+            # Combine with any partial bytes from previous read
+            buf = self._partial_bytes + buf
+            try:
+                result = buf.decode("utf-8")
+                self._partial_bytes = b''
+                return result
+            except UnicodeDecodeError:
+                # Try to decode as much as possible
+                for i in range(len(buf), 0, -1):
+                    try:
+                        result = buf[:i].decode("utf-8")
+                        self._partial_bytes = buf[i:]
+                        return result
+                    except UnicodeDecodeError:
+                        continue
+                self._partial_bytes = buf
+                return ''
         return buf
 
     def read_pipe(self, blksize, timeout=None):
@@ -185,6 +203,7 @@ def read_until(self, until, blksize=4096, timeout=None,
         cqlshlog.debug("Searching for %r" % (until.pattern,))
         got = self.readbuf
         self.readbuf = ''
+        empty_reads = 0
         with timing_out(timeout):
             while True:
                 val = self.read(blksize, ptty_timeout)
@@ -193,7 +212,13 @@ def read_until(self, until, blksize=4096, timeout=None,
                         val = val.replace(replace_target, '')
                 cqlshlog.debug("read %r from subproc" % (val,))
                 if val == '':
-                    raise EOFError("'until' pattern %r not found" % (until.pattern,))
+                    empty_reads += 1
+                    if empty_reads > 1:
+                        raise EOFError("'until' pattern %r not found" % (until.pattern,))
+                    # Read again to allow decoding of UTF-8 characters that span multiple reads
+                    continue
+
+                empty_reads = 0
                 got += val
                 m = until.search(got)
                 if m is not None:
@@ -241,6 +266,8 @@ def __init__(self, path=None, host=None, port=None, keyspace=None, cqlver=None,
         env.setdefault('TERM', 'xterm')
         env.setdefault('CQLSH_NO_BUNDLED', os.environ.get('CQLSH_NO_BUNDLED', ''))
         env.setdefault('PYTHONPATH', os.environ.get('PYTHONPATH', ''))
+        # Initialize the partial bytes buffer
+        self._partial_bytes = b''
         coverage = False
         if ('CQLSH_COVERAGE' in env.keys()):
             coverage = True
diff --git a/pylib/cqlshlib/test/test_copyutil.py b/pylib/cqlshlib/test/test_copyutil.py
index 19fcdd1126f6..c7730552b6e4 100644
--- a/pylib/cqlshlib/test/test_copyutil.py
+++ b/pylib/cqlshlib/test/test_copyutil.py
@@ -23,7 +23,11 @@
 from cassandra.metadata import MIN_LONG, Murmur3Token
 from cassandra.policies import SimpleConvictionPolicy
 from cassandra.pool import Host
-from unittest.mock import Mock
+
+try:
+    from unittest.mock import Mock
+except ImportError:
+    from mock import Mock
 
 from cqlshlib.copyutil import ExportTask
 
@@ -44,7 +48,9 @@ def setUp(self):
             Host('10.0.0.2', SimpleConvictionPolicy, 9000),
             Host('10.0.0.3', SimpleConvictionPolicy, 9000),
             Host('10.0.0.4', SimpleConvictionPolicy, 9000)
-    ]
+        ]
+        for h in self.hosts:
+            h.host_id = h.address
 
     def mock_shell(self):
         """
@@ -78,37 +84,35 @@ def _test_get_ranges_murmur3_base(self, opts, expected_ranges):
         for k, v in opts.items():
             overridden_opts[k] = v
         export_task = ExportTask(shell, self.ks, self.table, self.columns, self.fname, overridden_opts, self.protocol_version, self.config_file)
-        assert export_task.get_ranges() == expected_ranges
+        export_ranges = export_task.get_ranges()
+        assert export_ranges == expected_ranges,\
+            "Expected: {e}\n Actual:{a}".format(e=expected_ranges, a=export_ranges)
         export_task.close()
 
-    def test_get_ranges_murmur3(self):
-        """
-        Test behavior of ExportTask internal get_ranges function
-        """
-
+    def test_murmur3_get_ranges_invalid_input(self):
         # return empty dict and print error if begin_token < min_token
         self._test_get_ranges_murmur3_base({'begintoken': MIN_LONG - 1}, {})
-
-        # return empty dict and print error if begin_token < min_token
         self._test_get_ranges_murmur3_base({'begintoken': 1, 'endtoken': -1}, {})
 
-        # simple case of a single range
+    def test_get_ranges_murmur3_single_range(self):
         expected_ranges = {(1, 2): {'hosts': ('10.0.0.4', '10.0.0.1', '10.0.0.2'), 'attempts': 0, 'rows': 0, 'workerno': -1}}
         self._test_get_ranges_murmur3_base({'begintoken': 1, 'endtoken': 2}, expected_ranges)
 
-        # simple case of two contiguous ranges
+    def test_get_ranges_murmur3_two_continuous_ranges(self):
         expected_ranges = {
             (-4611686018427387903, 0): {'hosts': ('10.0.0.3', '10.0.0.4', '10.0.0.1'), 'attempts': 0, 'rows': 0, 'workerno': -1},
             (0, 1): {'hosts': ('10.0.0.4', '10.0.0.1', '10.0.0.2'), 'attempts': 0, 'rows': 0, 'workerno': -1}
         }
         self._test_get_ranges_murmur3_base({'begintoken': -4611686018427387903, 'endtoken': 1}, expected_ranges)
 
+    def test_get_ranges_murmur3_begin_token_only(self):
         # specify a begintoken only (endtoken defaults to None)
         expected_ranges = {
             (4611686018427387905, None): {'hosts': ('10.0.0.1', '10.0.0.2', '10.0.0.3'), 'attempts': 0, 'rows': 0, 'workerno': -1}
         }
         self._test_get_ranges_murmur3_base({'begintoken': 4611686018427387905}, expected_ranges)
 
+    def test_get_ranges_murmur3_end_token_only(self):
         # specify an endtoken only (begintoken defaults to None)
         expected_ranges = {
             (None, MIN_LONG + 1): {'hosts': ('10.0.0.2', '10.0.0.3', '10.0.0.4'), 'attempts': 0, 'rows': 0, 'workerno': -1}
diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py
index a53df5905dcc..8f5534c8b472 100644
--- a/pylib/cqlshlib/test/test_cqlsh_completion.py
+++ b/pylib/cqlshlib/test/test_cqlsh_completion.py
@@ -1078,4 +1078,19 @@ def test_complete_in_show(self):
         self.trycompletions('SHOW REPLICAS ', choices=['-', '<wholenumber>'])
 
     def test_complete_in_tracing(self):
-        self.trycompletions('TRACING ', choices=[';', '<enter>', 'OFF', 'ON'])
+        self.trycompletions('TRACING ', choices=[';', '<enter>', 'OFF', 'ON', 'FULL', 'COMPACT'])
+
+    def test_complete_in_select_limit_clause(self):
+        self.trycompletions('SELECT * FROM system.peers LI', immediate='MIT ')
+        self.trycompletions('SELECT * FROM system.peers LIMIT ', choices=['<wholenumber>'])
+        self.trycompletions('SELECT * FROM system.peers LIMIT 1 ', choices=[';', 'ALLOW', 'OFFSET', 'WITH'])
+        self.trycompletions('SELECT * FROM system.peers LIMIT 1 OF', immediate='FSET ')
+        self.trycompletions('SELECT * FROM system.peers LIMIT 1 OFFSET ', choices=['<wholenumber>'])
+        self.trycompletions('SELECT * FROM system.peers LIMIT 1 OFFSET 1 ', choices=[';', 'ALLOW', 'WITH'])
+
+    def test_complete_in_select_order_by_ann_of(self):
+        self.trycompletions('SELECT * FROM system.peers ORDER BY vectorcol ', choices=[',', ';', 'ALLOW', 'ANN', 'ASC', 'DESC', 'LIMIT', 'PER', 'WITH'])
+        self.trycompletions('SELECT * FROM system.peers ORDER BY vectorcol ANN ', immediate='OF [ ')
+        self.trycompletions('SELECT * FROM system.peers ORDER BY vectorcol ANN OF [ 1.2, ', choices=['<float>'])
+        self.trycompletions('SELECT * FROM system.peers ORDER BY vectorcol ANN OF [ 1.0, 2.0] LIMIT 100 ', choices=[';', 'ALLOW', 'OFFSET', 'WITH'])
+        self.trycompletions('SELECT * FROM system.peers ORDER BY vectorcol ANN OF [ 1.0, 2.0] LIMIT 100 WITH ', immediate='ann_options = { ')
diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py
index 9ca7988d2e6b..2341bdcca184 100644
--- a/pylib/cqlshlib/test/test_cqlsh_output.py
+++ b/pylib/cqlshlib/test/test_cqlsh_output.py
@@ -686,7 +686,7 @@ def test_describe_columnfamily_output(self):
                 AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
                 AND cdc = false
                 AND comment = ''
-                AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+                AND compaction = {'class': 'org.apache.cassandra.db.compaction.UnifiedCompactionStrategy'}
                 AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
                 AND memtable = 'default'
                 AND crc_check_chance = 1.0
diff --git a/pylib/cqlshlib/tracing.py b/pylib/cqlshlib/tracing.py
index 3db8e3491de0..f5f007817734 100644
--- a/pylib/cqlshlib/tracing.py
+++ b/pylib/cqlshlib/tracing.py
@@ -18,10 +18,10 @@
 import time
 
 from cassandra.query import QueryTrace, TraceUnavailable
-from cqlshlib.displaying import MAGENTA
+from cqlshlib.displaying import colorme, MAGENTA
 
 
-def print_trace_session(shell, session, session_id, partial_session=False):
+def print_trace_session(shell, tracing_mode, session, session_id, partial_session=False):
     """
     Lookup a trace by session and trace session ID, then print it.
     """
@@ -32,31 +32,85 @@ def print_trace_session(shell, session, session_id, partial_session=False):
     except TraceUnavailable:
         shell.printerr("Session %s wasn't found." % session_id)
     else:
-        print_trace(shell, trace)
+        print_trace(shell, tracing_mode, trace)
 
 
-def print_trace(shell, trace):
+def print_trace(shell, tracing_mode, trace):
+    assert tracing_mode in {'full', 'compact'}
+    trace_fn = print_trace_full if tracing_mode == 'full' else print_trace_compact
+    trace_fn(shell, trace)
+
+
+def format_trace(activity):
     """
-    Print an already populated cassandra.query.QueryTrace instance.
+    Format a trace activity, without the control-character mangling we use for other strings.
+    (Trace activity can contain internal newlines that we want to preserve.)
     """
-    rows = make_trace_rows(trace)
+    return colorme(activity.rstrip(), None, 'text')
+
+
+def print_trace_full(shell, trace):
+    """
+    Print a populated cassandra.query.QueryTrace instance.
+    """
+    rows = make_trace_rows(trace, 'full')
     if not rows:
         shell.printerr("No rows for session %s found." % (trace.trace_id,))
         return
     names = ['activity', 'timestamp', 'source', 'source_elapsed', 'client']
 
     formatted_names = list(map(shell.myformat_colname, names))
-    formatted_values = [list(map(shell.myformat_value, row)) for row in rows]
+    formatted_values = [[format_trace(row[0])] + list(map(shell.myformat_value, row[1:])) for row in rows]
+
+    shell.writeresult('')
+    shell.writeresult('Tracing session: ', color=MAGENTA, newline=False)
+    shell.writeresult(trace.trace_id)
+    shell.writeresult('')
+    shell.print_formatted_result(formatted_names, formatted_values, True, shell.tty)
+    shell.writeresult('')
+
+
+def print_trace_compact(shell, trace):
+    """
+    Print a populated cassandra.query.QueryTrace instance.
+    """
+    rows = make_trace_rows(trace, 'compact')
+    if not rows:
+        shell.printerr("No rows for session %s found." % (trace.trace_id,))
+        return
+
+    # Transform the full trace into 3 compact columns: src, elapsed, activity
+    # The original source column is a string version of an ipv4 address, e.g. 192.168.1.121;
+    # We want to generate a "key" mapping those addresses to "A", "B", "C", etc.
+    unique_sources = sorted(set(row[2] for row in rows))
+    if len(unique_sources) > 26:
+        shell.writeresult('Too many sources for compact mode; showing full instead')
+        print_trace_full(shell, trace)
+        return
+    source_mapping = {src: chr(65 + idx) for idx, src in enumerate(unique_sources)}
+
+    names = ['src', 'elapsed', 'activity']
+    formatted_names = list(map(shell.myformat_colname, names))
+    formatted_values = [[shell.myformat_value(source_mapping[row[2]]),
+                         shell.myformat_value(row[3]),
+                         format_trace(row[0])] for row in rows]
 
     shell.writeresult('')
     shell.writeresult('Tracing session: ', color=MAGENTA, newline=False)
     shell.writeresult(trace.trace_id)
     shell.writeresult('')
-    shell.print_formatted_result(formatted_names, formatted_values, with_header=True, tty=shell.tty)
+
+    # Write mapping of sources to keys
+    shell.writeresult('Source mapping:', color=MAGENTA)
+    for src, key in source_mapping.items():
+        shell.writeresult(f"  {key} -> {src}")
+    shell.writeresult('')
+
+    shell.print_formatted_result(formatted_names, formatted_values, True, shell.tty, justify_last=False)
     shell.writeresult('')
 
 
-def make_trace_rows(trace):
+def make_trace_rows(trace, tracing_mode):
     if not trace.events:
         return []
 
@@ -64,7 +118,7 @@ def make_trace_rows(trace):
 
     # append main rows (from events table).
     for event in trace.events:
-        rows.append(["%s [%s]" % (event.description, event.thread_name),
+        rows.append(["%s [%s]" % (event.description, event.thread_name) if tracing_mode == 'full' else event.description,
                      str(datetime_from_utc_to_local(event.datetime)),
                      event.source,
                      total_micro_seconds(event.source_elapsed),
diff --git a/pylib/requirements.txt b/pylib/requirements.txt
index 19e4221b4f27..c208492b42cd 100644
--- a/pylib/requirements.txt
+++ b/pylib/requirements.txt
@@ -19,7 +19,7 @@
 
 -e git+https://github.com/datastax/python-driver.git@3.29.0#egg=cassandra-driver
 # Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases.
--e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm
+-e git+https://github.com/datastax/cassandra-ccm.git@converged-cassandra#egg=ccm
 coverage
 pytest
 wcwidth
diff --git a/relocate-dependencies.pom b/relocate-dependencies.pom
index 96d8d16ee9da..1a71a6d7b313 100644
--- a/relocate-dependencies.pom
+++ b/relocate-dependencies.pom
@@ -36,7 +36,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 
-        <junit.version>4.12</junit.version>
+        <junit.version>4.13</junit.version>
         <maven.compiler.source>${java.version}</maven.compiler.source>
         <maven.compiler.target>${java.version}</maven.compiler.target>
         <dtest-local.version>4.0.0-SNAPSHOT</dtest-local.version>
diff --git a/src/antlr/Lexer.g b/src/antlr/Lexer.g
index 17045b2578d5..6c2f8f4f070d 100644
--- a/src/antlr/Lexer.g
+++ b/src/antlr/Lexer.g
@@ -72,6 +72,7 @@ K_INSERT:      I N S E R T;
 K_UPDATE:      U P D A T E;
 K_WITH:        W I T H;
 K_LIMIT:       L I M I T;
+K_OFFSET:      O F F S E T;
 K_PER:         P E R;
 K_PARTITION:   P A R T I T I O N;
 K_USING:       U S I N G;
@@ -183,6 +184,7 @@ K_WRITETIME:   W R I T E T I M E;
 K_MAXWRITETIME:M A X W R I T E T I M E;
 K_DATE:        D A T E;
 K_TIME:        T I M E;
+K_GEO_DISTANCE:G E O '_' D I S T A N C E;
 
 K_NULL:        N U L L;
 K_NOT:         N O T;
@@ -214,6 +216,8 @@ K_INPUT:       I N P U T;
 K_LANGUAGE:    L A N G U A G E;
 K_OR:          O R;
 K_REPLACE:     R E P L A C E;
+K_DETERMINISTIC: D E T E R M I N I S T I C;
+K_MONOTONIC:   M O N O T O N I C;
 
 K_JSON:        J S O N;
 K_DEFAULT:     D E F A U L T;
@@ -224,8 +228,16 @@ K_MASKED:      M A S K E D;
 K_UNMASK:      U N M A S K;
 K_SELECT_MASKED: S E L E C T '_' M A S K E D;
 
-K_VECTOR:      V E C T O R;
+K_VECTOR:       V E C T O R;
+K_EDGE:        E D G E;
+K_VERTEX:      V E R T E X;
+K_LABEL:       L A B E L;
+
+K_DROPPED:     D R O P P E D;
+K_COLUMN:      C O L U M N;
+K_RECORD:      R E C O R D;
 K_ANN:         A N N;
+K_BM25:        B M '2' '5';
 
 // Case-insensitive alpha characters
 fragment A: ('a'|'A');
diff --git a/src/antlr/Parser.g b/src/antlr/Parser.g
index 434a8691259c..dfe9b98f9841 100644
--- a/src/antlr/Parser.g
+++ b/src/antlr/Parser.g
@@ -27,17 +27,6 @@ options {
     private final List<ErrorListener> listeners = new ArrayList<ErrorListener>();
     protected final List<ColumnIdentifier> bindVariables = new ArrayList<ColumnIdentifier>();
 
-    public static final Set<String> reservedTypeNames = new HashSet<String>()
-    {{
-        add("byte");
-        add("complex");
-        add("enum");
-        add("date");
-        add("interval");
-        add("macaddr");
-        add("bitstring");
-    }};
-
     public AbstractMarker.Raw newBindVariables(ColumnIdentifier name)
     {
         AbstractMarker.Raw marker = new AbstractMarker.Raw(bindVariables.size());
@@ -268,10 +257,12 @@ selectStatement returns [SelectStatement.RawStatement expr]
     @init {
         Term.Raw limit = null;
         Term.Raw perPartitionLimit = null;
+        Term.Raw offset = null;
         List<Ordering.Raw> orderings = new ArrayList<>();
         List<Selectable.Raw> groups = new ArrayList<>();
         boolean allowFiltering = false;
         boolean isJson = false;
+        SelectOptions options = new SelectOptions();
     }
     : K_SELECT
         // json is a valid column name. By consequence, we need to resolve the ambiguity for "json - json"
@@ -281,8 +272,9 @@ selectStatement returns [SelectStatement.RawStatement expr]
       ( K_GROUP K_BY groupByClause[groups] ( ',' groupByClause[groups] )* )?
       ( K_ORDER K_BY orderByClause[orderings] ( ',' orderByClause[orderings] )* )?
       ( K_PER K_PARTITION K_LIMIT rows=intValue { perPartitionLimit = rows; } )?
-      ( K_LIMIT rows=intValue { limit = rows; } )?
+      ( K_LIMIT rows=intValue { limit = rows; } ( K_OFFSET rows=intValue { offset = rows; } )? )?
       ( K_ALLOW K_FILTERING  { allowFiltering = true; } )?
+      ( K_WITH properties[options] )?
       {
           SelectStatement.Parameters params = new SelectStatement.Parameters(orderings,
                                                                              groups,
@@ -290,7 +282,7 @@ selectStatement returns [SelectStatement.RawStatement expr]
                                                                              allowFiltering,
                                                                              isJson);
           WhereClause where = wclause == null ? WhereClause.empty() : wclause.build();
-          $expr = new SelectStatement.RawStatement(cf, params, $sclause.selectors, where, limit, perPartitionLimit);
+          $expr = new SelectStatement.RawStatement(cf, params, $sclause.selectors, where, limit, perPartitionLimit, offset, options);
       }
     ;
 
@@ -446,28 +438,42 @@ sident returns [Selectable.RawIdentifier id]
 
 whereClause returns [WhereClause.Builder clause]
     @init{ $clause = new WhereClause.Builder(); }
-    : relationOrExpression[$clause] (K_AND relationOrExpression[$clause])*
+    : expression[$clause]
+    ;
+
+expression [WhereClause.Builder clause]
+    : primaryExpression[$clause] (booleanOp[$clause] primaryExpression[$clause])*
     ;
 
-relationOrExpression [WhereClause.Builder clause]
-    : relation[$clause]
+primaryExpression [WhereClause.Builder clause]
+    : '(' {clause.startEnclosure(); } expression[$clause] ')' { clause.endEnclosure(); }
+    | relation[$clause]
     | customIndexExpression[$clause]
     ;
 
+booleanOp [WhereClause.Builder clause]
+    : op=K_AND { clause.setCurrentOperator(op.getText()); }
+    | op=K_OR { clause.setCurrentOperator(op.getText()); }
+    ;
+
 customIndexExpression [WhereClause.Builder clause]
     @init{QualifiedName name = new QualifiedName();}
     : 'expr(' idxName[name] ',' t=term ')' { clause.add(new CustomIndexExpression(name, t));}
     ;
 
 orderByClause[List<Ordering.Raw> orderings]
-    @init{
+    @init {
         Ordering.Direction direction = Ordering.Direction.ASC;
+        Ordering.Raw.Expression expr = null;
     }
-    : c=cident (K_ANN K_OF t=term)? (K_ASC | K_DESC { direction = Ordering.Direction.DESC; })?
+    : c=cident
+        ( K_ANN K_OF t=term { expr = new Ordering.Raw.Ann(c, t); }
+        | K_BM25 K_OF t=term { expr = new Ordering.Raw.Bm25(c, t); }
+        )?
+        (K_ASC | K_DESC { direction = Ordering.Direction.DESC; })?
     {
-        Ordering.Raw.Expression expr = (t == null)
-            ? new Ordering.Raw.SingleColumn(c)
-            : new Ordering.Raw.Ann(c, t);
+        if (expr == null)
+            expr = new Ordering.Raw.SingleColumn(c);
         orderings.add(new Ordering.Raw(expr, direction));
     }
     ;
@@ -663,6 +669,7 @@ createAggregateStatement returns [CreateAggregateStatement.Raw stmt]
     @init {
         boolean orReplace = false;
         boolean ifNotExists = false;
+        boolean deterministic = false;
 
         List<CQL3Type.Raw> argTypes = new ArrayList<>();
     }
@@ -684,7 +691,8 @@ createAggregateStatement returns [CreateAggregateStatement.Raw stmt]
       (
         K_INITCOND ival = term
       )?
-      { $stmt = new CreateAggregateStatement.Raw(fn, argTypes, stype, sfunc, ffunc, ival, orReplace, ifNotExists); }
+      ( K_DETERMINISTIC { deterministic = true; } )?
+      { $stmt = new CreateAggregateStatement.Raw(fn, argTypes, stype, sfunc, ffunc, ival, orReplace, ifNotExists, deterministic); }
     ;
 
 dropAggregateStatement returns [DropAggregateStatement.Raw stmt]
@@ -716,6 +724,10 @@ createFunctionStatement returns [CreateFunctionStatement.Raw stmt]
         List<ColumnIdentifier> argNames = new ArrayList<>();
         List<CQL3Type.Raw> argTypes = new ArrayList<>();
         boolean calledOnNullInput = false;
+
+        boolean deterministic = false;
+        boolean monotonic = false;
+        List<ColumnIdentifier> monotonicOn = new ArrayList<>();
     }
     : K_CREATE (K_OR K_REPLACE { orReplace = true; })?
       K_FUNCTION
@@ -729,10 +741,16 @@ createFunctionStatement returns [CreateFunctionStatement.Raw stmt]
       ')'
       ( (K_RETURNS K_NULL) | (K_CALLED { calledOnNullInput=true; })) K_ON K_NULL K_INPUT
       K_RETURNS returnType = comparatorType
+      ( K_DETERMINISTIC { deterministic = true; } )?
+      (
+        K_MONOTONIC { monotonic=true; monotonicOn.addAll(argNames); }
+        | K_MONOTONIC K_ON k=noncol_ident { monotonicOn.add(k); monotonic=monotonicOn.containsAll(argNames); }
+      )?
       K_LANGUAGE language = IDENT
       K_AS body = STRING_LITERAL
       { $stmt = new CreateFunctionStatement.Raw(
-          fn, argNames, argTypes, returnType, calledOnNullInput, $language.text.toLowerCase(), $body.text, orReplace, ifNotExists);
+          fn, argNames, argTypes, returnType, calledOnNullInput, $language.text.toLowerCase(), $body.text, orReplace,
+          ifNotExists, deterministic, monotonic, monotonicOn);
       }
     ;
 
@@ -812,10 +830,30 @@ tablePartitionKey[CreateTableStatement.Raw stmt]
     | '(' k1=ident { l.add(k1); } ( ',' kn=ident { l.add(kn); } )* ')'
     ;
 
+/**
+ * Graph structures are not supported by this version of C*. Skip graph-related keywords.
+ */
+mockVertex
+    : (ident | '(' ident (',' ident)* ')')
+    (',' ident)*
+    ;
+
 tableProperty[CreateTableStatement.Raw stmt]
+    @init { boolean isStatic = false; }
     : property[stmt.attrs]
     | K_COMPACT K_STORAGE { $stmt.setCompactStorage(); }
     | K_CLUSTERING K_ORDER K_BY '(' tableClusteringOrder[stmt] (',' tableClusteringOrder[stmt])* ')'
+    | K_VERTEX K_LABEL ( noncol_ident )? {stmt.attrs.addProperty("dse_vertex_label_property", "vertex");}
+    | K_EDGE K_LABEL ( noncol_ident ) ?
+             K_FROM noncol_ident '(' mockVertex ')'
+             K_TO noncol_ident '(' mockVertex ')'
+             {stmt.attrs.addProperty("dse_edge_label_property", "edge");}
+    | K_DROPPED K_COLUMN K_RECORD
+          k=ident v=comparatorTypeWithMultiCellTuple (K_STATIC {isStatic = true;})?
+          K_USING K_TIMESTAMP t=INTEGER
+      {
+          stmt.attrs.addDroppedColumnRecord(k, v, isStatic, Long.parseLong($t.text));
+      }
     ;
 
 tableClusteringOrder[CreateTableStatement.Raw stmt]
@@ -974,7 +1012,7 @@ alterTableStatement returns [AlterTableStatement.Raw stmt]
                (       id=ident { $stmt.drop(id);  }
                | ('('  id1=ident { $stmt.drop(id1); }
                  ( ',' idn=ident { $stmt.drop(idn); } )* ')') )
-               ( K_USING K_TIMESTAMP t=INTEGER { $stmt.timestamp(Long.parseLong(Constants.Literal.integer($t.text).getText())); } )?
+               ( K_USING K_TIMESTAMP t=INTEGER { $stmt.dropTimestamp(Long.parseLong($t.text)); } )?
 
       | K_RENAME ( K_IF K_EXISTS { $stmt.ifColumnExists(true); } )?
                (        id1=ident K_TO toId1=ident { $stmt.rename(id1, toId1); }
@@ -1069,8 +1107,8 @@ dropMaterializedViewStatement returns [DropViewStatement.Raw stmt]
 /**
   * TRUNCATE <CF>;
   */
-truncateStatement returns [TruncateStatement stmt]
-    : K_TRUNCATE (K_COLUMNFAMILY)? cf=columnFamilyName { $stmt = new TruncateStatement(cf); }
+truncateStatement returns [TruncateStatement.Raw stmt]
+    : K_TRUNCATE (K_COLUMNFAMILY)? cf=columnFamilyName { $stmt = new TruncateStatement.Raw(cf); }
     ;
 
 /**
@@ -1572,7 +1610,7 @@ collectionLiteral returns [Term.Raw value]
 listLiteral returns [Term.Raw value]
     @init {List<Term.Raw> l = new ArrayList<Term.Raw>();}
     @after {$value = new ArrayLiteral(l);}
-    : '[' ( t1=term { l.add(t1); } ( ',' tn=term { l.add(tn); } )* )? ']' { $value = new ArrayLiteral(l); }
+    : '[' ( t1=term { l.add(t1); } ( ',' tn=term { l.add(tn); } )* )? ']'
     ;
 
 usertypeLiteral returns [UserTypes.Literal ut]
@@ -1615,6 +1653,7 @@ allowedFunctionName returns [String s]
     | f=QUOTED_NAME                 { $s = $f.text; }
     | u=unreserved_function_keyword { $s = u; }
     | K_TOKEN                       { $s = "token"; }
+    | K_GEO_DISTANCE                { $s = "GEO_DISTANCE"; }
     | K_COUNT                       { $s = "count"; }
     ;
 
@@ -1768,6 +1807,7 @@ relationType returns [Operator op]
     | '>'  { $op = Operator.GT; }
     | '>=' { $op = Operator.GTE; }
     | '!=' { $op = Operator.NEQ; }
+    | ':'  { $op = Operator.ANALYZER_MATCHES; }
     ;
 
 relation[WhereClause.Builder clauses]
@@ -1776,10 +1816,16 @@ relation[WhereClause.Builder clauses]
     | name=cident K_IS K_NOT K_NULL { $clauses.add(new SingleColumnRelation(name, Operator.IS_NOT, Constants.NULL_LITERAL)); }
     | K_TOKEN l=tupleOfIdentifiers type=relationType t=term
         { $clauses.add(new TokenRelation(l, type, t)); }
+    | K_GEO_DISTANCE '(' name=cident ',' point=term ')' type=relationType distance=term
+        { $clauses.add(new GeoDistanceRelation(name, point, type, distance)); }
     | name=cident K_IN marker=inMarker
         { $clauses.add(new SingleColumnRelation(name, Operator.IN, marker)); }
+    | name=cident K_NOT K_IN marker=inMarker
+        { $clauses.add(new SingleColumnRelation(name, Operator.NOT_IN, marker)); }
     | name=cident K_IN inValues=singleColumnInValues
         { $clauses.add(SingleColumnRelation.createInRelation($name.id, inValues)); }
+    | name=cident K_NOT K_IN inValues=singleColumnInValues
+            { $clauses.add(SingleColumnRelation.createNotInRelation($name.id, inValues)); }
     | name=cident rt=containsOperator t=term { $clauses.add(new SingleColumnRelation(name, rt, t)); }
     | name=cident '[' key=term ']' type=relationType t=term { $clauses.add(new SingleColumnRelation(name, key, type, t)); }
     | ids=tupleOfIdentifiers
@@ -1795,6 +1841,16 @@ relation[WhereClause.Builder clauses]
           | markers=tupleOfMarkersForTuples /* (a, b, c) IN (?, ?, ...) */
               { $clauses.add(MultiColumnRelation.createInRelation(ids, markers)); }
           )
+      | K_NOT K_IN
+          ( '(' ')'
+              { $clauses.add(MultiColumnRelation.createNotInRelation(ids, new ArrayList<Tuples.Literal>())); }
+          | tupleInMarker=inMarkerForTuple /* (a, b, c) NOT IN ? */
+              { $clauses.add(MultiColumnRelation.createSingleMarkerNotInRelation(ids, tupleInMarker)); }
+          | literals=tupleOfTupleLiterals /* (a, b, c) NOT IN ((1, 2, 3), (4, 5, 6), ...) */
+              { $clauses.add(MultiColumnRelation.createNotInRelation(ids, literals)); }
+          | markers=tupleOfMarkersForTuples /* (a, b, c) NOT IN (?, ?, ...) */
+              { $clauses.add(MultiColumnRelation.createNotInRelation(ids, markers)); }
+          )
       | type=relationType literal=tupleLiteral /* (a, b, c) > (1, 2, 3) or (a, b, c) > (?, ?, ?) */
           {
               $clauses.add(MultiColumnRelation.createNonInRelation(ids, type, literal));
@@ -1802,11 +1858,11 @@ relation[WhereClause.Builder clauses]
       | type=relationType tupleMarker=markerForTuple /* (a, b, c) >= ? */
           { $clauses.add(MultiColumnRelation.createNonInRelation(ids, type, tupleMarker)); }
       )
-    | '(' relation[$clauses] ')'
     ;
 
 containsOperator returns [Operator o]
     : K_CONTAINS { o = Operator.CONTAINS; } (K_KEY { o = Operator.CONTAINS_KEY; })?
+    | K_NOT K_CONTAINS { o = Operator.NOT_CONTAINS; } (K_KEY { o = Operator.NOT_CONTAINS_KEY; })?
     ;
 
 inMarker returns [AbstractMarker.INRaw marker]
@@ -1844,16 +1900,17 @@ inMarkerForTuple returns [Tuples.INRaw marker]
     | ':' name=noncol_ident { $marker = newTupleINBindVariables(name); }
     ;
 
-comparatorType returns [CQL3Type.Raw t]
+comparatorTypeWithoutTuples returns [CQL3Type.Raw t]
     : n=native_type     { $t = CQL3Type.Raw.from(n); }
     | c=collection_type { $t = c; }
-    | tt=tuple_type     { $t = tt; }
     | vc=vector_type    { $t = vc; }
     | id=userTypeName   { $t = CQL3Type.Raw.userType(id); }
     | K_FROZEN '<' f=comparatorType '>'
       {
         try {
-            $t = f.freeze();
+            // antlr might try to recover and give a null for f. It will still properly error out in the end, but we want
+            // to avoid NPE in the call to #freeze() so we should bypass this or we'll have a weird user-facing error.
+            $t = f == null ? null : f.freeze();
         } catch (InvalidRequestException e) {
             addRecognitionError(e.getMessage());
         }
@@ -1861,7 +1918,7 @@ comparatorType returns [CQL3Type.Raw t]
     | s=STRING_LITERAL
       {
         try {
-            $t = CQL3Type.Raw.from(new CQL3Type.Custom($s.text));
+            $t = CQL3Type.Raw.custom($s.text);
         } catch (SyntaxException e) {
             addRecognitionError("Cannot parse type " + $s.text + ": " + e.getMessage());
         } catch (ConfigurationException e) {
@@ -1870,6 +1927,16 @@ comparatorType returns [CQL3Type.Raw t]
       }
     ;
 
+// Force frozen tuples.
+comparatorType returns [CQL3Type.Raw t]
+    : ct=comparatorTypeWithoutTuples     { $t = ct; }
+    | tt=tuple_types                     { $t = CQL3Type.Raw.tuple(tt, true); }
+    ;
+comparatorTypeWithMultiCellTuple returns [CQL3Type.Raw t]
+    : ct=comparatorTypeWithoutTuples     { $t = ct; }
+    | tt=tuple_types                     { $t = CQL3Type.Raw.tuple(tt, false); }
+    ;
+
 native_type returns [CQL3Type t]
     : K_ASCII     { $t = CQL3Type.Native.ASCII; }
     | K_BIGINT    { $t = CQL3Type.Native.BIGINT; }
@@ -1907,10 +1974,8 @@ collection_type returns [CQL3Type.Raw pt]
         { if (t != null) $pt = CQL3Type.Raw.set(t); }
     ;
 
-tuple_type returns [CQL3Type.Raw t]
-    @init {List<CQL3Type.Raw> types = new ArrayList<>();}
-    @after {$t = CQL3Type.Raw.tuple(types);}
-    : K_TUPLE '<' t1=comparatorType { types.add(t1); } (',' tn=comparatorType { types.add(tn); })* '>'
+tuple_types returns [List<CQL3Type.Raw> types]
+    : K_TUPLE '<' t1=comparatorType { $types = new ArrayList<>(); $types.add(t1); } (',' tn=comparatorType { $types.add(tn); })* '>'
     ;
 
 vector_type returns [CQL3Type.Raw vt]
@@ -1937,7 +2002,7 @@ mbean
 // Basically the same as cident, but we need to exlude existing CQL3 types
 // (which for some reason are not reserved otherwise)
 non_type_ident returns [ColumnIdentifier id]
-    : t=IDENT                    { if (reservedTypeNames.contains($t.text)) addRecognitionError("Invalid (reserved) user type name " + $t.text); $id = new ColumnIdentifier($t.text, false); }
+    : t=IDENT                    { if (ReservedTypeNames.isReserved($t.text)) addRecognitionError("Invalid (reserved) user type name " + $t.text); $id = new ColumnIdentifier($t.text, false); }
     | t=QUOTED_NAME              { $id = new ColumnIdentifier($t.text, true); }
     | k=basic_unreserved_keyword { $id = new ColumnIdentifier(k, false); }
     | kk=K_KEY                   { $id = new ColumnIdentifier($kk.text, false); }
@@ -2020,6 +2085,16 @@ basic_unreserved_keyword returns [String str]
         | K_UNMASK
         | K_SELECT_MASKED
         | K_VECTOR
+        | K_EDGE
+        | K_VERTEX
+        | K_LABEL
+        | K_DROPPED
+        | K_COLUMN
+        | K_RECORD
         | K_ANN
+        | K_BM25
+        | K_OFFSET
+        | K_DETERMINISTIC
+        | K_MONOTONIC
         ) { $str = $k.text; }
     ;
diff --git a/src/java/org/apache/cassandra/auth/AllowAllAuthorizer.java b/src/java/org/apache/cassandra/auth/AllowAllAuthorizer.java
index a943db06d166..0443f25e6aef 100644
--- a/src/java/org/apache/cassandra/auth/AllowAllAuthorizer.java
+++ b/src/java/org/apache/cassandra/auth/AllowAllAuthorizer.java
@@ -47,8 +47,9 @@ public void revokeAllFrom(RoleResource droppedRole)
     {
     }
 
-    public void revokeAllOn(IResource droppedResource)
+    public Set<RoleResource> revokeAllOn(IResource droppedResource)
     {
+        return Collections.emptySet();
     }
 
     public Set<PermissionDetails> list(AuthenticatedUser performer, Set<Permission> permissions, IResource resource, RoleResource of)
diff --git a/src/java/org/apache/cassandra/auth/AuthCache.java b/src/java/org/apache/cassandra/auth/AuthCache.java
index 6d78d368b4b1..9271eb49ebc2 100644
--- a/src/java/org/apache/cassandra/auth/AuthCache.java
+++ b/src/java/org/apache/cassandra/auth/AuthCache.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ScheduledFuture;
+import java.util.Collection;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.function.BiPredicate;
@@ -32,6 +33,8 @@
 import java.util.function.IntConsumer;
 import java.util.function.IntSupplier;
 import java.util.function.Supplier;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import com.google.common.util.concurrent.Uninterruptibles;
 import org.slf4j.Logger;
@@ -246,6 +249,18 @@ public void invalidate(K k)
             cache.invalidate(k);
     }
 
+    public void maybeInvalidateByFilter(Predicate<? super K> filter)
+    {
+        if (cache != null)
+        {
+            Collection<K> iterable = cache.asMap().keySet()
+                                          .stream()
+                                          .filter(filter)
+                                          .collect(Collectors.toSet());
+            cache.invalidateAll(iterable);
+        }
+    }
+
     /**
      * Time in milliseconds that a value in the cache will expire after.
      * @param validityPeriod in milliseconds
diff --git a/src/java/org/apache/cassandra/auth/AuthConfig.java b/src/java/org/apache/cassandra/auth/AuthConfig.java
index f357825b10b2..67d4490790ee 100644
--- a/src/java/org/apache/cassandra/auth/AuthConfig.java
+++ b/src/java/org/apache/cassandra/auth/AuthConfig.java
@@ -73,7 +73,10 @@ public static void applyAuth()
         IAuthorizer authorizer = authInstantiate(conf.authorizer, AllowAllAuthorizer.class);
 
         if (!authenticator.requireAuthentication() && authorizer.requireAuthorization())
-            throw new ConfigurationException(conf.authenticator.class_name + " can't be used with " + conf.authorizer, false);
+        {
+            throw new ConfigurationException(authorizer.getClass().getName() + " has authorization enabled which requires " +
+                                             authenticator.getClass().getName() + " to enable authentication", false);
+        }
 
         DatabaseDescriptor.setAuthorizer(authorizer);
 
@@ -82,7 +85,7 @@ public static void applyAuth()
         IRoleManager roleManager = authInstantiate(conf.role_manager, CassandraRoleManager.class);
 
         if (authenticator instanceof PasswordAuthenticator && !(roleManager instanceof CassandraRoleManager))
-            throw new ConfigurationException("CassandraRoleManager must be used with PasswordAuthenticator", false);
+            throw new ConfigurationException(authenticator.getClass().getName() + " requires " + CassandraRoleManager.class.getName(), false);
 
         DatabaseDescriptor.setRoleManager(roleManager);
 
@@ -132,6 +135,8 @@ private static <T> T authInstantiate(ParameterizedClass authCls, Class<T> defaul
             return ParameterizedClass.newInstance(authCls, List.of("", authPackage));
         }
 
+        // for now, this has to stay and can not be replaced by ParameterizedClass.newInstance as above
+        // due to that failing for simulator dtests. See CASSANDRA-20450 for more information.
         try
         {
             return defaultCls.newInstance();
diff --git a/src/java/org/apache/cassandra/auth/AuthKeyspace.java b/src/java/org/apache/cassandra/auth/AuthKeyspace.java
index b495c904199e..18c54ffbcd3f 100644
--- a/src/java/org/apache/cassandra/auth/AuthKeyspace.java
+++ b/src/java/org/apache/cassandra/auth/AuthKeyspace.java
@@ -155,7 +155,7 @@ private static TableMetadata parse(String name, String description, String cql)
     public static KeyspaceMetadata metadata()
     {
         return KeyspaceMetadata.create(SchemaConstants.AUTH_KEYSPACE_NAME,
-                                       KeyspaceParams.simple(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())),
+                                       KeyspaceParams.systemDistributed(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())),
                                        Tables.of(Roles, RoleMembers, RolePermissions,
                                                  ResourceRoleIndex, NetworkPermissions,
                                                  CIDRPermissions, CIDRGroups,
diff --git a/src/java/org/apache/cassandra/auth/AuthenticatedUser.java b/src/java/org/apache/cassandra/auth/AuthenticatedUser.java
index 6b31110e9532..af43d87380d0 100644
--- a/src/java/org/apache/cassandra/auth/AuthenticatedUser.java
+++ b/src/java/org/apache/cassandra/auth/AuthenticatedUser.java
@@ -19,6 +19,8 @@
 
 import java.net.InetSocketAddress;
 import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Objects;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -128,6 +130,33 @@ public Set<Permission> getPermissions(IResource resource)
         return permissionsCache.getPermissions(this, resource);
     }
 
+    /**
+     * Clears all entries from the permissions cache.
+     * 
+     * Used by CNDB Authorizer to clear all entries from the cache.
+     */
+    @VisibleForTesting
+    public static void clearCache()
+    {
+        permissionsCache.invalidate();
+    }
+
+    /**
+     * Clears all entries belonging to the given {@link RoleResource}
+     * from the permissions cache.
+     * 
+     * Used by CNDB Authorizer to clear entries after permissions are granted
+     * or revoked.
+     * 
+     * @param roleResource the {@link RoleResource} to clear from the cache 
+     */
+    @VisibleForTesting
+    public static void clearCache(RoleResource roleResource)
+    {
+        permissionsCache.maybeInvalidateByFilter(key -> key.left.equals(new AuthenticatedUser(roleResource.getRoleName())));
+    }
+
+
     /**
      * Check whether this user has login privileges.
      * LOGIN is not inherited from granted roles, so must be directly granted to the primary role for this user
diff --git a/src/java/org/apache/cassandra/auth/CIDRGroupsMappingManager.java b/src/java/org/apache/cassandra/auth/CIDRGroupsMappingManager.java
index e258b50a3a37..6d68d5edee02 100644
--- a/src/java/org/apache/cassandra/auth/CIDRGroupsMappingManager.java
+++ b/src/java/org/apache/cassandra/auth/CIDRGroupsMappingManager.java
@@ -22,7 +22,6 @@
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -31,10 +30,12 @@
 import java.util.stream.Collectors;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CIDR;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
@@ -62,7 +63,7 @@ public class CIDRGroupsMappingManager implements CIDRGroupsMappingManagerMBean
     private SelectStatement getCidrGroupsStatement;
     private SelectStatement getCidrsForCidrGroupStatement;
 
-    private static final int PAGE_SIZE = 128;
+    private static final PageSize PAGE_SIZE = PageSize.inRows(128);
 
     public void setup()
     {
@@ -141,7 +142,7 @@ public Set<Pair<InetAddress, Short>> retrieveCidrsFromRow(UntypedResultSet.Row r
                                        AuthKeyspace.CIDR_GROUPS_TBL_CIDRS_COL_NAME);
 
         Set<Pair<InetAddress, Short>> cidrs = new HashSet<>();
-        TupleType tupleType = new TupleType(Arrays.asList(InetAddressType.instance, ShortType.instance));
+        TupleType tupleType = new TupleType(ImmutableList.of(InetAddressType.instance, ShortType.instance));
 
         Set<ByteBuffer> cidrAsTuples = row.getFrozenSet(AuthKeyspace.CIDR_GROUPS_TBL_CIDRS_COL_NAME, tupleType);
         for (ByteBuffer cidrAsTuple : cidrAsTuples)
diff --git a/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java b/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
index 153ec0e5fecb..bed4b944cec5 100644
--- a/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
+++ b/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
@@ -17,7 +17,14 @@
  */
 package org.apache.cassandra.auth;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -27,23 +34,31 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Table;
 import com.google.common.collect.Sets;
-
+import com.google.common.collect.Table;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.BatchQueryOptions;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.UntypedResultSet.Row;
+import org.apache.cassandra.cql3.VariableSpecifications;
 import org.apache.cassandra.cql3.statements.BatchStatement;
 import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.exceptions.UnauthorizedException;
 import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.Dispatcher;
@@ -172,8 +187,10 @@ public void revokeAllFrom(RoleResource revokee)
     // Called after a resource is removed (DROP KEYSPACE, DROP TABLE, etc.).
     // Execute a logged batch removing all the permissions for the resource
     // as well as the index table entry
-    public void revokeAllOn(IResource droppedResource)
+    @Override
+    public Set<RoleResource> revokeAllOn(IResource droppedResource)
     {
+        Set<RoleResource> affectedRoles = new HashSet<>();
         try
         {
             UntypedResultSet rows = process(String.format("SELECT role FROM %s.%s WHERE resource = '%s'",
@@ -185,10 +202,12 @@ public void revokeAllOn(IResource droppedResource)
             List<CQLStatement> statements = new ArrayList<>();
             for (UntypedResultSet.Row row : rows)
             {
+                String role = row.getString("role");
+                affectedRoles.add(RoleResource.role(role));
                 statements.add(QueryProcessor.getStatement(String.format("DELETE FROM %s.%s WHERE role = '%s' AND resource = '%s'",
                                                                          SchemaConstants.AUTH_KEYSPACE_NAME,
                                                                          AuthKeyspace.ROLE_PERMISSIONS,
-                                                                         escape(row.getString("role")),
+                                                                         escape(role),
                                                                          escape(droppedResource.getName())),
                                                            ClientState.forInternalCalls()));
             }
@@ -205,6 +224,8 @@ public void revokeAllOn(IResource droppedResource)
         {
             logger.warn(String.format("CassandraAuthorizer failed to revoke all permissions on %s", droppedResource), e);
         }
+
+        return affectedRoles;
     }
 
     /**
@@ -244,10 +265,11 @@ private Set<Permission> getExistingPermissions(String roleName,
     private void executeLoggedBatch(List<CQLStatement> statements)
     throws RequestExecutionException, RequestValidationException
     {
-        BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED,
-                                                  VariableSpecifications.empty(),
-                                                  Lists.newArrayList(Iterables.filter(statements, ModificationStatement.class)),
-                                                  Attributes.none());
+        BatchStatement batch = new BatchStatement(null,
+                BatchStatement.Type.LOGGED,
+                VariableSpecifications.empty(),
+                Lists.newArrayList(Iterables.filter(statements, ModificationStatement.class)),
+                Attributes.none());
         processBatch(batch);
     }
 
diff --git a/src/java/org/apache/cassandra/auth/CassandraRoleManager.java b/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
index 0e9c0b56404d..a40ed89d8801 100644
--- a/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
+++ b/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
@@ -26,6 +26,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
@@ -35,6 +36,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.cql3.*;
@@ -94,6 +96,12 @@ public class CassandraRoleManager implements IRoleManager
      */
     static final ConsistencyLevel DEFAULT_SUPERUSER_CONSISTENCY_LEVEL = ConsistencyLevel.QUORUM;
 
+    // In some cases we want to avoid creating default super user accounts (for security purposes). For example CNDB
+    // can create roles without requiring default SU account. So we need a flag to tell C* to skip creating this role.
+    // Even though this says SKIP it's actually safer to create the user as non super user with no login creds
+    // and a timestamp of 1 in case a user forgets the flag on another node.
+    private static final boolean SKIP_DEFAULT_ROLE_SETUP = CassandraRelevantProperties.SKIP_DEFAULT_ROLE_SETUP.getBoolean();
+
     // Transform a row in the AuthKeyspace.ROLES to a Role instance
     private static final Function<UntypedResultSet.Row, Role> ROW_TO_ROLE = row ->
     {
@@ -129,6 +137,7 @@ static int getGensaltLogRounds()
 
     private SelectStatement loadRoleStatement;
     private SelectStatement loadIdentityStatement;
+    private SelectStatement loadRoleMembersStatement;
 
     private final Set<Option> supportedOptions;
     private final Set<Option> alterableOptions;
@@ -148,6 +157,9 @@ public void setup()
     {
         loadRoleStatement();
         loadIdentityStatement();
+        loadRoleMembersStatement = (SelectStatement) prepare("SELECT member FROM %s.%s WHERE role = ?",
+                                                      SchemaConstants.AUTH_KEYSPACE_NAME,
+                                                      AuthKeyspace.ROLE_MEMBERS);
         scheduleSetupTask(() -> {
             setupDefaultRole();
             return null;
@@ -348,6 +360,19 @@ public Set<Role> getRoleDetails(RoleResource grantee)
                .collect(Collectors.toSet());
     }
 
+    public Set<RoleResource> getMembersOf(RoleResource role)
+    {
+        // Get the membership list of the given role
+        QueryOptions options = QueryOptions.forInternalCalls(consistencyForRoleRead(role.getRoleName()),
+                                                             Collections.singletonList(ByteBufferUtil.bytes(role.getRoleName())));
+        ResultMessage.Rows rows = select(loadRoleMembersStatement, options);
+        UntypedResultSet resultSet = UntypedResultSet.create(rows.result);
+
+        return StreamSupport.stream(resultSet.spliterator(), false)
+               .map(row -> RoleResource.role(row.getString("member")))
+               .collect(Collectors.toSet());
+    }
+
     /**
      * We hard-code this query to Quorum regardless of the role or auth credentials of the queryer given the nature of
      * this query: we expect to know *all* roles across the entire cluster when we query this, not just local quorum or
@@ -439,11 +464,15 @@ private static void setupDefaultRole()
     @VisibleForTesting
     public static String createDefaultRoleQuery()
     {
-        return String.format("INSERT INTO %s.%s (role, is_superuser, can_login, salted_hash) VALUES ('%s', true, true, '%s') USING TIMESTAMP 0",
+        return String.format("INSERT INTO %s.%s (role, is_superuser, can_login, salted_hash) " +
+                             "VALUES ('%s', %s, %s, '%s') USING TIMESTAMP %s",
                              SchemaConstants.AUTH_KEYSPACE_NAME,
                              AuthKeyspace.ROLES,
                              DEFAULT_SUPERUSER_NAME,
-                             escape(hashpw(DEFAULT_SUPERUSER_PASSWORD)));
+                             SKIP_DEFAULT_ROLE_SETUP ? "false" : "true",
+                             SKIP_DEFAULT_ROLE_SETUP ? "false" : "true",
+                             SKIP_DEFAULT_ROLE_SETUP ? "" : escape(hashpw(DEFAULT_SUPERUSER_PASSWORD)),
+                             SKIP_DEFAULT_ROLE_SETUP ? "1" : "0");
     }
 
     @VisibleForTesting
diff --git a/src/java/org/apache/cassandra/auth/IAuthorizer.java b/src/java/org/apache/cassandra/auth/IAuthorizer.java
index b39e3153976c..ed246b985b6e 100644
--- a/src/java/org/apache/cassandra/auth/IAuthorizer.java
+++ b/src/java/org/apache/cassandra/auth/IAuthorizer.java
@@ -133,9 +133,10 @@ Set<PermissionDetails> list(AuthenticatedUser performer, Set<Permission> permiss
      * not support it should be sure to throw UnsupportedOperationException.
      *
      * @param droppedResource The resource to revoke all permissions on.
+     * @return the roles that had permissions on {@code droppedResource}
      * @throws java.lang.UnsupportedOperationException
      */
-    void revokeAllOn(IResource droppedResource);
+    Set<RoleResource> revokeAllOn(IResource droppedResource);
 
     /**
      * Set of resources that should be made inaccessible to users and only accessible internally.
diff --git a/src/java/org/apache/cassandra/auth/IRoleManager.java b/src/java/org/apache/cassandra/auth/IRoleManager.java
index 460bf1a16ffc..9570cdeb7198 100644
--- a/src/java/org/apache/cassandra/auth/IRoleManager.java
+++ b/src/java/org/apache/cassandra/auth/IRoleManager.java
@@ -153,6 +153,15 @@ default Set<Role> getRoleDetails(RoleResource grantee)
                                        .collect(Collectors.toSet());
      }
 
+    /**
+     * Return the set of roles that are direct members of the given role (non-recursive).
+     * @return any non-{@code null} value means the found reverse-memberships.
+     *         {@code null} means that the functionality is not supported.
+     * @throws RequestValidationException
+     * @throws RequestExecutionException
+     */
+    Set<RoleResource> getMembersOf(RoleResource role);
+
     /**
      * Called during the execution of an unqualified LIST ROLES query.
      * Returns the total set of distinct roles in the system.
diff --git a/src/java/org/apache/cassandra/auth/Roles.java b/src/java/org/apache/cassandra/auth/Roles.java
index f18851af84e9..0ac484bb20d8 100644
--- a/src/java/org/apache/cassandra/auth/Roles.java
+++ b/src/java/org/apache/cassandra/auth/Roles.java
@@ -22,6 +22,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,6 +44,20 @@ public static void init()
         AuthCacheService.instance.register(cache);
     }
 
+    /**
+     * Clears the given {@link RoleResource} from the roles cache.
+     * 
+     * Used by CNDB RoleManager to clear cache entries after grant, alter
+     * or drop operatons on a role.
+     * 
+     * @param roleResource the {@link RoleResource} to clear from the cache
+     */
+    @VisibleForTesting
+    public static void clearCache(RoleResource roleResource)
+    {
+        cache.invalidate(roleResource);
+    }
+
     /**
      * Identify all roles granted to the supplied Role, including both directly granted
      * and inherited roles.
diff --git a/src/java/org/apache/cassandra/batchlog/Batch.java b/src/java/org/apache/cassandra/batchlog/Batch.java
index 667263b25d88..245b37e3b88e 100644
--- a/src/java/org/apache/cassandra/batchlog/Batch.java
+++ b/src/java/org/apache/cassandra/batchlog/Batch.java
@@ -19,9 +19,9 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.*;
-
-import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.io.IVersionedSerializer;
@@ -81,12 +81,16 @@ public int size()
         return decodedMutations.size() + encodedMutations.size();
     }
     
-    @VisibleForTesting
     public Collection<ByteBuffer> getEncodedMutations()
     {
         return encodedMutations;
     }
 
+    public Collection<Mutation> getDecodedMutations()
+    {
+        return decodedMutations;
+    }
+
     /**
      * Local batches contain only already decoded {@link Mutation} instances. Unlike remote 
      * batches, which contain mutations encoded as {@link ByteBuffer} instances, local batches 
diff --git a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
index fd98a0fec854..6692cf63234d 100644
--- a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
+++ b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
@@ -43,19 +43,21 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.WriteOptions;
 import org.apache.cassandra.db.WriteType;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.WriteFailureException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.hints.Hint;
 import org.apache.cassandra.hints.HintsService;
 import org.apache.cassandra.io.util.DataInputBuffer;
@@ -133,10 +135,14 @@ public static void remove(TimeUUID id)
 
     public static void store(Batch batch)
     {
-        store(batch, true);
+        /**
+         * by default writes are durable, see
+         * {@link org.apache.cassandra.schema.KeyspaceParams#DEFAULT_DURABLE_WRITES}
+         */
+        store(batch, WriteOptions.DEFAULT);
     }
 
-    public static void store(Batch batch, boolean durableWrites)
+    public static void store(Batch batch, WriteOptions writeOptions)
     {
         List<ByteBuffer> mutations = new ArrayList<>(batch.encodedMutations.size() + batch.decodedMutations.size());
         mutations.addAll(batch.encodedMutations);
@@ -161,7 +167,7 @@ public static void store(Batch batch, boolean durableWrites)
                .add("version", MessagingService.current_version)
                .appendAll("mutations", mutations);
 
-        builder.buildAsMutation().apply(durableWrites);
+        builder.buildAsMutation().apply(writeOptions);
     }
 
     @VisibleForTesting
@@ -220,7 +226,7 @@ private void replayFailedBatches()
         String query = String.format("SELECT id, mutations, version FROM %s.%s WHERE token(id) > token(?) AND token(id) <= token(?)",
                                      SchemaConstants.SYSTEM_KEYSPACE_NAME,
                                      SystemKeyspace.BATCHES);
-        UntypedResultSet batches = executeInternalWithPaging(query, pageSize, lastReplayedUuid, limitUuid);
+        UntypedResultSet batches = executeInternalWithPaging(query, PageSize.inRows(pageSize), lastReplayedUuid, limitUuid);
         processBatchlogEntries(batches, pageSize, rateLimiter);
         lastReplayedUuid = limitUuid;
         logger.trace("Finished replayFailedBatches");
@@ -483,10 +489,10 @@ private static ReplayWriteResponseHandler<Mutation> sendSingleReplayMutation(fin
 
             Replica selfReplica = liveAndDown.all().selfIfPresent();
             if (selfReplica != null)
-                mutation.apply();
+                mutation.apply(WriteOptions.FOR_BATCH_REPLAY);
 
             ReplicaLayout.ForTokenWrite liveRemoteOnly = liveAndDown.filter(
-                    r -> FailureDetector.isReplicaAlive.test(r) && r != selfReplica);
+                    r -> IFailureDetector.isReplicaAlive.test(r) && r != selfReplica);
 
             for (Replica replica : liveAndDown.all())
             {
@@ -534,7 +540,7 @@ private static class ReplayWriteResponseHandler<T> extends WriteResponseHandler<
             }
 
             @Override
-            protected int blockFor()
+            public int blockFor()
             {
                 return this.replicaPlan.contacts().size();
             }
diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
index 3826eb412e02..a34f83ce713f 100644
--- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java
+++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
@@ -28,6 +28,8 @@
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.concurrent.NotThreadSafe;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 import org.cliffc.high_scale_lib.NonBlockingHashSet;
 import org.slf4j.Logger;
@@ -37,8 +39,7 @@
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.FSWriteError;
@@ -87,6 +88,8 @@ public interface IStreamFactory
 
     private final CacheSerializer<K, V> cacheLoader;
 
+    private final Supplier<Predicate<K>> keyFilterSupplier;
+
     /*
      * CASSANDRA-10155 required a format change to fix 2i indexes and caching.
      * 2.2 is already at version "c" and 3.0 is at "d".
@@ -128,11 +131,12 @@ public static void setStreamFactory(IStreamFactory streamFactory)
         AutoSavingCache.streamFactory = streamFactory;
     }
 
-    public AutoSavingCache(ICache<K, V> cache, CacheService.CacheType cacheType, CacheSerializer<K, V> cacheloader)
+    public AutoSavingCache(ICache<K, V> cache, CacheService.CacheType cacheType, CacheSerializer<K, V> cacheloader, Supplier<Predicate<K>> keyFilterSupplier)
     {
-        super(cacheType.toString(), cache);
+        super(cacheType, cache);
         this.cacheType = cacheType;
         this.cacheLoader = cacheloader;
+        this.keyFilterSupplier = keyFilterSupplier;
     }
 
     public File getCacheDataPath(String version)
@@ -292,12 +296,13 @@ public Future<?> submitWrite(int keysToSave)
         return CompactionManager.instance.submitCacheWrite(getWriter(keysToSave));
     }
 
-    public class Writer extends CompactionInfo.Holder
+    public class Writer extends AbstractTableOperation
     {
         private final Iterator<K> keyIterator;
-        private final CompactionInfo info;
+        private final OperationProgress info;
         private long keysWritten;
         private final long keysEstimate;
+        private final Predicate<K> keyFilter;
 
         protected Writer(int keysToSave)
         {
@@ -312,6 +317,7 @@ protected Writer(int keysToSave)
                 keyIterator = hotKeyIterator(keysToSave);
                 keysEstimate = keysToSave;
             }
+            this.keyFilter = keyFilterSupplier != null ? keyFilterSupplier.get() : null;
 
             OperationType type;
             if (cacheType == CacheService.CacheType.KEY_CACHE)
@@ -323,13 +329,13 @@ else if (cacheType == CacheService.CacheType.COUNTER_CACHE)
             else
                 type = OperationType.UNKNOWN;
 
-            info = CompactionInfo.withoutSSTables(TableMetadata.minimal(SchemaConstants.SYSTEM_KEYSPACE_NAME, cacheType.toString()),
-                                                  type,
-                                                  0,
-                                                  keysEstimate,
-                                                  Unit.KEYS,
-                                                  nextTimeUUID(),
-                                                  getCacheDataPath(CURRENT_VERSION).toPath().toString());
+            info = OperationProgress.withoutSSTables(TableMetadata.minimal(SchemaConstants.SYSTEM_KEYSPACE_NAME, cacheType.toString()),
+                                                     type,
+                                                     0,
+                                                     keysEstimate,
+                                                     Unit.KEYS, 
+                                                     nextTimeUUID(),
+                                                     getCacheDataPath(CURRENT_VERSION).toPath().toString());
         }
 
         public CacheService.CacheType cacheType()
@@ -337,7 +343,7 @@ public CacheService.CacheType cacheType()
             return cacheType;
         }
 
-        public CompactionInfo getCompactionInfo()
+        public OperationProgress getProgress()
         {
             // keyset can change in size, thus total can too
             // TODO need to check for this one... was: info.forProgress(keysWritten, Math.max(keysWritten, keys.size()));
@@ -364,7 +370,6 @@ public void saveCache()
             try (WrappedDataOutputStreamPlus writer = new WrappedDataOutputStreamPlus(streamFactory.getOutputStream(dataTmpFile, crcTmpFile));
                  FileOutputStreamPlus metadataWriter = metadataTmpFile.newOutputStream(File.WriteMode.OVERWRITE))
             {
-
                 //Need to be able to check schema version because CF names are ambiguous
                 UUID schemaVersion = Schema.instance.getVersion();
                 writer.writeLong(schemaVersion.getMostSignificantBits());
@@ -373,18 +378,23 @@ public void saveCache()
                 while (keyIterator.hasNext())
                 {
                     K key = keyIterator.next();
+                    ColumnFamilyStore cfs;
+                    if ((keyFilter == null || keyFilter.test(key)) && (cfs = Schema.instance.getColumnFamilyStoreInstance(key.tableId)) != null)
+                    {
+                        if (key.indexName != null)
+                            cfs = cfs.indexManager.getIndexByName(key.indexName).getBackingTable().orElse(null);
 
-                    ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(key.tableId);
-                    if (cfs == null)
-                        continue; // the table or 2i has been dropped.
-                    if (key.indexName != null)
-                        cfs = cfs.indexManager.getIndexByName(key.indexName).getBackingTable().orElse(null);
-
-                    cacheLoader.serialize(key, writer, cfs);
+                        cacheLoader.serialize(key, writer, cfs);
 
-                    keysWritten++;
-                    if (keysWritten >= keysEstimate)
-                        break;
+                        keysWritten++;
+                        if (keysWritten >= keysEstimate)
+                            break;
+                    }
+                    else
+                    {
+                        // remove stale entry from cache before saving basing on the existence of a table and provided filter
+                        keyIterator.remove();
+                    }
                 }
 
                 cacheLoader.serializeMetadata(metadataWriter);
@@ -430,7 +440,7 @@ private File getTempCacheFile(File cacheFile)
 
         private void deleteOldCacheFiles()
         {
-            File savedCachesDir = new File(DatabaseDescriptor.getSavedCachesLocation());
+            File savedCachesDir = DatabaseDescriptor.getSavedCachesLocation();
             assert savedCachesDir.exists() && savedCachesDir.isDirectory();
             File[] files = savedCachesDir.tryList();
             if (files != null)
diff --git a/src/java/org/apache/cassandra/cache/CaffeineCache.java b/src/java/org/apache/cassandra/cache/CaffeineCache.java
index 717fb78f6a33..60cd4441a930 100644
--- a/src/java/org/apache/cassandra/cache/CaffeineCache.java
+++ b/src/java/org/apache/cassandra/cache/CaffeineCache.java
@@ -37,12 +37,32 @@ public class CaffeineCache<K extends IMeasurableMemory, V extends IMeasurableMem
     private final Cache<K, V> cache;
     private final Eviction<K, V> policy;
 
+    /**
+     * cache capacity cached for performance reasons
+     * </p>
+     * The need to cache this value arises from the fact that map.capacity() is an expensive operation for
+     * Caffeine caches, which use the cache eviction policy to determine the capacity, which, in turn,
+     * may take eviction lock and do cache maintenance before returning the actual value.
+     * </p>
+     * Cassandra frequently uses cache capacity to determine if a cache is enabled.
+     * See e.g:
+     * {@link org.apache.cassandra.db.ColumnFamilyStore#putCachedCounter},
+     * {@link org.apache.cassandra.db.ColumnFamilyStore#getCachedCounter},
+     * </p>
+     * In more stressful scenarios with many counter cache puts and gets asking the underlying cache to provide the
+     * capacity instead of using the cached value leads to a significant performance degradation.
+     * </p>
+     * No thread-safety guarantees are provided. The value may be a bit stale, and this is good enough.
+     */
+    private long cacheCapacity;
+
     private CaffeineCache(Cache<K, V> cache)
     {
         this.cache = cache;
         this.policy = cache.policy().eviction().orElseThrow(() -> 
             new IllegalArgumentException("Expected a size bounded cache"));
         checkState(policy.isWeighted(), "Expected a weighted cache");
+        this.cacheCapacity = capacitySlow();
     }
 
     /**
@@ -69,14 +89,16 @@ public static <K extends IMeasurableMemory, V extends IMeasurableMemory> Caffein
         });
     }
 
+    @Override
     public long capacity()
     {
-        return policy.getMaximum();
+        return cacheCapacity;
     }
 
     public void setCapacity(long capacity)
     {
         policy.setMaximum(capacity);
+        cacheCapacity = capacitySlow();
     }
 
     public boolean isEmpty()
@@ -138,4 +160,14 @@ public boolean containsKey(K key)
     {
         return cache.asMap().containsKey(key);
     }
+
+    /**
+     * This method is used to get the capacity of the cache. It is a slow method because it may take the eviction lock
+     * and perform cache maintenance before returning the actual value.
+     * @return the capacity of the cache as determined by the eviction policy
+     */
+    private long capacitySlow()
+    {
+        return policy.getMaximum();
+    }
 }
diff --git a/src/java/org/apache/cassandra/cache/ChunkCache.java b/src/java/org/apache/cassandra/cache/ChunkCache.java
index 8f22ee8b7b15..cff45658b63f 100644
--- a/src/java/org/apache/cassandra/cache/ChunkCache.java
+++ b/src/java/org/apache/cassandra/cache/ChunkCache.java
@@ -21,67 +21,307 @@
 package org.apache.cassandra.cache;
 
 import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import java.nio.ByteOrder;
+import java.util.concurrent.CompletableFuture; // checkstyle: permit this import
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.AsyncCache;
+import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
 import com.github.benmanes.caffeine.cache.RemovalCause;
 import com.github.benmanes.caffeine.cache.RemovalListener;
-import org.apache.cassandra.concurrent.ImmediateExecutor;
+import org.apache.cassandra.concurrent.ParkedExecutor;
+import org.apache.cassandra.concurrent.ShutdownableExecutor;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.ChunkReader;
-import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PageAware;
 import org.apache.cassandra.io.util.Rebufferer;
 import org.apache.cassandra.io.util.RebuffererFactory;
 import org.apache.cassandra.metrics.ChunkCacheMetrics;
 import org.apache.cassandra.utils.memory.BufferPool;
 import org.apache.cassandra.utils.memory.BufferPools;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+import org.github.jamm.Unmetered;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CHUNKCACHE_ASYNC_CLEANUP;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CHUNKCACHE_CLEANER_THREADS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CHUNKCACHE_INITIAL_CAPACITY;
 
 public class ChunkCache
-        implements CacheLoader<ChunkCache.Key, ChunkCache.Buffer>, RemovalListener<ChunkCache.Key, ChunkCache.Buffer>, CacheSize
+        implements RemovalListener<ChunkCache.Key, ChunkCache.Chunk>, CacheSize
 {
+    private final static Logger logger = LoggerFactory.getLogger(ChunkCache.class);
+
     public static final int RESERVED_POOL_SPACE_IN_MiB = 32;
-    public static final long cacheSize = 1024L * 1024L * Math.max(0, DatabaseDescriptor.getFileCacheSizeInMiB() - RESERVED_POOL_SPACE_IN_MiB);
+    private static final int INITIAL_CAPACITY = CHUNKCACHE_INITIAL_CAPACITY.getInt();;
+    private static final boolean ASYNC_CLEANUP =CHUNKCACHE_ASYNC_CLEANUP.getBoolean();
+    private static final int CLEANER_THREADS = CHUNKCACHE_CLEANER_THREADS.getInt();
+
+    private static final Class PERFORM_CLEANUP_TASK_CLASS;
+    // cached value in order to not call System.getProperty on a hotpath
+    private static final int CHUNK_CACHE_REBUFFER_WAIT_TIMEOUT_MS = CassandraRelevantProperties.CHUNK_CACHE_REBUFFER_WAIT_TIMEOUT_MS.getInt();
+
+    static
+    {
+        try
+        {
+            logger.info("-Dcassandra.chunkcache.async_cleanup={} dse.chunk.cache.cleaner.threads={}", ASYNC_CLEANUP, CLEANER_THREADS);
+            PERFORM_CLEANUP_TASK_CLASS = Class.forName("com.github.benmanes.caffeine.cache.BoundedLocalCache$PerformCleanupTask");
+        }
+        catch (ClassNotFoundException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
     public static final boolean roundUp = DatabaseDescriptor.getFileCacheRoundUp();
 
-    private static boolean enabled = DatabaseDescriptor.getFileCacheEnabled() && cacheSize > 0;
-    public static final ChunkCache instance = enabled ? new ChunkCache(BufferPools.forChunkCache()) : null;
+    public static final ChunkCache instance = DatabaseDescriptor.getFileCacheEnabled()
+                                              ? new ChunkCache(BufferPools.forChunkCache(), DatabaseDescriptor.getFileCacheSizeInMiB(), ChunkCacheMetrics::create)
+                                              : null;
 
+    @Unmetered
     private final BufferPool bufferPool;
 
-    private final LoadingCache<Key, Buffer> cache;
+    private final AsyncCache<Key, Chunk> cache;
+    private final Cache<Key, Chunk> synchronousCache;
+    private final ConcurrentMap<Key, CompletableFuture<Chunk>> cacheAsMap;
+    private final long cacheSize;
+    @Unmetered
     public final ChunkCacheMetrics metrics;
+    @Unmetered
+    private final ShutdownableExecutor cleanupExecutor;
+
+    private boolean enabled;
+    private Function<ChunkReader, RebuffererFactory> wrapper = this::wrap;
+
+    // File id management
+    private final ConcurrentHashMap<File, Long> fileIdMap = new ConcurrentHashMap<>();
+    private final AtomicLong nextFileId = new AtomicLong(0);
+
+    // number of bits required to store the log2 of the chunk size
+    private final static int CHUNK_SIZE_LOG2_BITS = Integer.numberOfTrailingZeros(Integer.SIZE);
+
+    // number of bits required to store the ready type
+    private final static int READER_TYPE_BITS = Integer.SIZE - Integer.numberOfLeadingZeros(ChunkReader.ReaderType.COUNT - 1);
+
+    public ChunkCache(BufferPool pool, int cacheSizeInMB, Function<ChunkCache, ChunkCacheMetrics> createMetrics)
+    {
+        cacheSize = 1024L * 1024L * Math.max(0, cacheSizeInMB - RESERVED_POOL_SPACE_IN_MiB);
+        cleanupExecutor = ParkedExecutor.createParkedExecutor("ChunkCacheCleanup", CLEANER_THREADS);
+        enabled = cacheSize > 0;
+        bufferPool = pool;
+        metrics = createMetrics.apply(this);
+        cache = Caffeine.newBuilder()
+                        .maximumWeight(cacheSize)
+                        .initialCapacity(INITIAL_CAPACITY)
+                        .executor(r -> {
+                            if (ASYNC_CLEANUP && r.getClass() == PERFORM_CLEANUP_TASK_CLASS)
+                                cleanupExecutor.execute(r);
+                            else
+                                r.run();
+                        })
+                        .weigher((key, buffer) -> ((Chunk) buffer).capacity())
+                        .removalListener(this)
+                        .recordStats(() -> metrics)
+                        .buildAsync();
+        synchronousCache = cache.synchronous();
+        cacheAsMap = cache.asMap();
+    }
+
+
+    private Chunk load(ChunkReader file, long position)
+    {
+        Chunk chunk = null;
+        try
+        {
+            chunk = newChunk(file.chunkSize(), position);  // Note: we need `chunk` to be assigned before we call read to release on error
+            chunk.read(file);
+        }
+        catch (RuntimeException t)
+        {
+            if (chunk != null)
+                chunk.release();
+            throw t;
+        }
+        return chunk;
+    }
+
+    Chunk newChunk(int chunkSize, long position)
+    {
+        if (chunkSize == PageAware.PAGE_SIZE)
+            return new SingleRegionChunk(position, bufferPool.get(PageAware.PAGE_SIZE, BufferType.OFF_HEAP));
+        if (chunkSize < PageAware.PAGE_SIZE)
+            return new SingleRegionChunk(position, bufferPool.get(PageAware.PAGE_SIZE, BufferType.OFF_HEAP).limit(chunkSize).slice());
+
+        ByteBuffer[] buffers = bufferPool.getMultiple(chunkSize, PageAware.PAGE_SIZE, BufferType.OFF_HEAP);
+        if (buffers.length > 1)
+            return new MultiRegionChunk(position, buffers);
+        else
+            return new SingleRegionChunk(position, buffers[0]);
+    }
+
+    @Override
+    public void onRemoval(Key key, Chunk chunk, RemovalCause cause)
+    {
+        chunk.release();
+    }
+
+    /**
+     * Clears the cache, used in the CNDB Writer for testing purposes.
+     */
+    public void clear() {
+        // Clear keysByFile first to prevent unnecessary computation in onRemoval method.
+        synchronousCache.invalidateAll();
+    }
+
+    public void close()
+    {
+        clear();
+        try
+        {
+            cleanupExecutor.shutdown();
+        }
+        catch (InterruptedException e)
+        {
+            logger.debug("Interrupted during shutdown: ", e);
+        }
+    }
+
+    public RebuffererFactory wrap(ChunkReader file)
+    {
+        return new CachingRebufferer(file);
+    }
+
+    public RebuffererFactory maybeWrap(ChunkReader file)
+    {
+        if (!enabled)
+            return file;
+
+        return wrapper.apply(file);
+    }
+
+    @VisibleForTesting
+    public void enable(boolean enabled)
+    {
+        this.enabled = enabled;
+        wrapper = this::wrap;
+        synchronousCache.invalidateAll();
+        metrics.reset();
+    }
+
+    public boolean isEnabled()
+    {
+        return enabled;
+    }
+
+    @VisibleForTesting
+    public void intercept(Function<RebuffererFactory, RebuffererFactory> interceptor)
+    {
+        final Function<ChunkReader, RebuffererFactory> prevWrapper = wrapper;
+        wrapper = rdr -> interceptor.apply(prevWrapper.apply(rdr));
+    }
+
+    /**
+     * Maps a reader to a reader id, used by the cache to find content.
+     *
+     * Uses the file name (through the fileIdMap), reader type and chunk size to define the id.
+     * The lowest {@link #READER_TYPE_BITS} are occupied by reader type, then the next {@link #CHUNK_SIZE_LOG2_BITS}
+     * are occupied by log 2 of chunk size (we assume the chunk size is the power of 2), and the rest of the bits
+     * are occupied by fileId counter which is incremented for each unseen file name.
+     */
+    private long readerIdFor(File file, ChunkReader.ReaderType type, int chunkSize)
+    {
+        return (((fileIdMap.computeIfAbsent(file, this::assignFileId)
+                  << CHUNK_SIZE_LOG2_BITS) | Integer.numberOfTrailingZeros(chunkSize))
+                << READER_TYPE_BITS) | type.ordinal();
+    }
+
+    /**
+     * Maps a reader to a file id, used by the cache to find content.
+     */
+    protected long readerIdFor(ChunkReader source)
+    {
+        return readerIdFor(source.channel().getFile(), source.type(), source.chunkSize());
+    }
+
+    private long assignFileId(File file)
+    {
+        return nextFileId.getAndIncrement();
+    }
+
+    /**
+     * Invalidate all buffers from the given file, i.e. make sure they can not be accessed by any reader using a
+     * FileHandle opened after this call. The buffers themselves will remain in the cache until they get normally
+     * evicted, because it is too costly to remove them.
+     *
+     * Note that this call has no effect of handles that are already opened. The correct usage is to call this when
+     * a file is deleted, or when a file is created for writing. It cannot be used to update and resynchronize the
+     * cached view of an existing file.
+     */
+    public void invalidateFile(File file)
+    {
+        // Removing the name from the id map suffices -- the next time someone wants to read this file, it will get
+        // assigned a fresh id.
+        fileIdMap.remove(file);
+    }
+
+    /**
+     * Invalidate all buffers for the given file, including handles that are already opened. This is a very costly
+     * operation and is only intended to be used by tests.
+     */
+    @VisibleForTesting
+    public void invalidateFileNow(File file)
+    {
+        Long fileIdMaybeNull = fileIdMap.get(file);
+        if (fileIdMaybeNull == null)
+            return;
+        long fileId = fileIdMaybeNull << (CHUNK_SIZE_LOG2_BITS + READER_TYPE_BITS);
+        long mask = - (1 << (CHUNK_SIZE_LOG2_BITS + READER_TYPE_BITS));
+        synchronousCache.invalidateAll(Iterables.filter(cache.asMap().keySet(), x -> (x.readerId & mask) == fileId));
+    }
 
     static class Key
     {
-        final ChunkReader file;
-        final String path;
+        final long readerId;
         final long position;
 
-        public Key(ChunkReader file, long position)
+        private Key(long readerId, long position)
         {
             super();
-            this.file = file;
             this.position = position;
-            this.path = file.channel().filePath();
+            this.readerId = readerId;
         }
 
+        @Override
         public int hashCode()
         {
             final int prime = 31;
             int result = 1;
-            result = prime * result + path.hashCode();
-            result = prime * result + file.getClass().hashCode();
+            result = prime * result + Long.hashCode(readerId);
             result = prime * result + Long.hashCode(position);
             return result;
         }
 
+        @Override
         public boolean equals(Object obj)
         {
             if (this == obj)
@@ -91,119 +331,277 @@ public boolean equals(Object obj)
 
             Key other = (Key) obj;
             return (position == other.position)
-                    && file.getClass() == other.file.getClass()
-                    && path.equals(other.path);
+                   && readerId == other.readerId;
         }
     }
 
-    class Buffer implements Rebufferer.BufferHolder
+    /**
+     * An abstract chunk contains the common implementation for the single and multi-regions chunks, normally
+     * this is related to ref counting and calling the read methods in the chunk reader. The chunk implementations
+     * will then take care of implementing the read target so that they can accommodate the data that was read into
+     * either a single memory region or into multiple memory regions.
+     */
+    abstract static class Chunk
     {
-        private final ByteBuffer buffer;
-        private final long offset;
-        private final AtomicInteger references;
+        /** The offset in the file where the chunk is read */
+        final long offset;
 
-        public Buffer(ByteBuffer buffer, long offset)
+        /** The number of bytes read from disk, this could be less than the memory space allocated */
+        int bytesRead;
+
+        private volatile int references;
+        private static final AtomicIntegerFieldUpdater<Chunk> referencesUpdater = AtomicIntegerFieldUpdater.newUpdater(Chunk.class, "references");
+
+        Chunk(long offset)
         {
-            this.buffer = buffer;
             this.offset = offset;
-            references = new AtomicInteger(1);  // start referenced.
+            this.bytesRead = 0; // To be filled by the read method
+            this.references = 1; // Start referenced
         }
 
-        Buffer reference()
+        /**
+         * Return the correct buffer depending on the position requested by the client, also taking a reference to this
+         * chunk.
+         *
+         * @param position the position requested by the client, this has not been aligned to neither the chunk nor the page
+         * @return the buffer covering this position, or null if no reference can be taken
+         */
+        @Nullable
+        Rebufferer.BufferHolder getReferencedBuffer(long position)
         {
             int refCount;
             do
             {
-                refCount = references.get();
+                refCount = references;
+
                 if (refCount == 0)
-                    // Buffer was released before we managed to reference it.
-                    return null;
-            } while (!references.compareAndSet(refCount, refCount + 1));
+                    return null; // Buffer was released before we managed to reference it.
 
-            return this;
+            } while (!referencesUpdater.compareAndSet(this, refCount, refCount + 1));
+
+            return getBuffer(position);
         }
 
-        @Override
-        public ByteBuffer buffer()
+        /**
+         * Release the chunk when the cache or a reader no longer needs it.
+         */
+        public void release()
         {
-            assert references.get() > 0;
-            return buffer.duplicate();
+            if (referencesUpdater.decrementAndGet(this) == 0)
+                releaseBuffers();
         }
 
-        @Override
         public long offset()
         {
             return offset;
         }
 
-        @Override
-        public void release()
+        /**
+         * Used in assertions, returns false if the chunk is not still referenced.
+         */
+        boolean isReferenced()
         {
-            if (references.decrementAndGet() == 0)
-                bufferPool.put(buffer);
+            return references > 0;
         }
-    }
 
-    private ChunkCache(BufferPool pool)
-    {
-        bufferPool = pool;
-        metrics = new ChunkCacheMetrics(this);
-        cache = Caffeine.newBuilder()
-                        .maximumWeight(cacheSize)
-                        .executor(ImmediateExecutor.INSTANCE)
-                        .weigher((key, buffer) -> ((Buffer) buffer).buffer.capacity())
-                        .removalListener(this)
-                        .recordStats(() -> metrics)
-                        .build(this);
+        /**
+         * Load the data for this chunk.
+         */
+        abstract void read(ChunkReader file);
+
+        /**
+         * Return the correct buffer depending on the position requested by the client. The returned buffer cannot be
+         * null, but may be empty and must contain the given position
+         * (i.e. buf.offset <= position <= buf.offset + buf.buffer.limit where the latter can only be == if at the end
+         * of the file and buffer is empty).
+         *
+         * @param position the position requested by the client, this has not been aligned to neither the chunk nor the page
+         * @return the buffer covering this position, or null if no reference can be taken
+         */
+        abstract Rebufferer.BufferHolder getBuffer(long position);
+
+        /**
+         * @return the space taken by this chunk
+         */
+        abstract int capacity();
+
+        /**
+         * Release the addresses when this chunk is no longer used. Called when all references are released.
+         */
+        abstract void releaseBuffers();
     }
 
-    @Override
-    public Buffer load(Key key)
+    /**
+     * A chunk is a group of buffers of size {@link PageAware#PAGE_SIZE} that will be allocated and released
+     * at the same time. The {@link ChunkReader} will read into the buffers of a chunk.
+     */
+    public class MultiRegionChunk extends Chunk
     {
-        ByteBuffer buffer = bufferPool.get(key.file.chunkSize(), key.file.preferredBufferType());
-        assert buffer != null;
-        key.file.readChunk(key.position, buffer);
-        return new Buffer(buffer, key.position);
-    }
+        /** A list of memory addresses, each page has capacity of PageAware.PAGE_SIZE */
+        private final long[] pages;
+        /** List of attachments, necessary to be able to release pool buffers properly */
+        private final Object[] attachments;
 
-    @Override
-    public void onRemoval(Key key, Buffer buffer, RemovalCause cause)
-    {
-        buffer.release();
-    }
+        public MultiRegionChunk(long offset, ByteBuffer[] buffers)
+        {
+            super(offset);
+            this.pages = new long[buffers.length];
+            this.attachments = new Object[buffers.length];
+            for (int i = 0; i < buffers.length; ++i)
+            {
+                pages[i] = MemoryUtil.getAddress(buffers[i]);
+                attachments[i] = MemoryUtil.getAttachment(buffers[i]);
+            }
+        }
 
-    public void close()
-    {
-        cache.invalidateAll();
-    }
+        void releaseBuffers()
+        {
+            for (int i = 0; i < pages.length; ++i)
+                bufferPool.put(MemoryUtil.allocateByteBuffer(pages[i], PageAware.PAGE_SIZE, PageAware.PAGE_SIZE, ByteOrder.BIG_ENDIAN, attachments[i]));
+        }
 
-    public RebuffererFactory wrap(ChunkReader file)
-    {
-        return new CachingRebufferer(file);
-    }
+        void read(ChunkReader file)
+        {
+            // Note: We cannot use ThreadLocalByteBufferHolder because readChunk uses it for its temporary buffer.
+            // Note: This uses the "Networking" buffer pool, which is meant to serve short-term buffers. Using
+            // the cache's buffer pool can cause problems due to the difference in buffer sizes and lifetime.
+            // Note: As this buffer is not retained in the cache, it can use the chunk reader's preferred buffer type.
+            ByteBuffer scratchBuffer = BufferPools.forNetworking().get(capacity(), file.preferredBufferType());
+            try
+            {
+                file.readChunk(offset, scratchBuffer);
+                int limit = scratchBuffer.limit();
+                int idx = 0;
+                int pageEnd;
+                for (pageEnd = PageAware.PAGE_SIZE; pageEnd <= limit; pageEnd += PageAware.PAGE_SIZE)
+                {
+                    scratchBuffer.limit(pageEnd);
+                    MemoryUtil.setBytes(pages[idx++], scratchBuffer);
+                    scratchBuffer.position(pageEnd);
+                }
+                if (scratchBuffer.position() < limit)   // if the limit is not a multiple of the page size
+                {
+                    scratchBuffer.limit(limit);
+                    MemoryUtil.setBytes(pages[idx], scratchBuffer);
+                }
+                bytesRead = limit;
+            }
+            finally
+            {
+                BufferPools.forNetworking().put(scratchBuffer);
+            }
+        }
 
-    public static RebuffererFactory maybeWrap(ChunkReader file)
-    {
-        if (!enabled)
-            return file;
+        @Nullable
+        Buffer getBuffer(long position)
+        {
+            int index = PageAware.pageNum(position - offset);
+            Preconditions.checkArgument(index >= 0 && index < pages.length, "Invalid position: %s, index: %s", position, index);
 
-        return instance.wrap(file);
-    }
+            long pageAlignedPosition = PageAware.pageStart(position);
+            int bufferSize = Math.min(PageAware.PAGE_SIZE, Math.toIntExact(bytesRead - (index * PageAware.PAGE_SIZE)));
+            assert bufferSize > 0 && bufferSize <= PageAware.PAGE_SIZE : "Wrong buffer size: " + bufferSize + " at position " + position;
 
-    public void invalidatePosition(FileHandle dfile, long position)
-    {
-        if (!(dfile.rebuffererFactory() instanceof CachingRebufferer))
-            return;
+            return new Buffer(pages[index], pageAlignedPosition, bufferSize);
+        }
+
+        public int capacity()
+        {
+            return pages.length * PageAware.PAGE_SIZE;
+        }
+
+        class Buffer implements Rebufferer.BufferHolder
+        {
+            private final long address;
+            private final long offset;
+            private final int limit;
+
+
+            public Buffer(long address, long offset, int limit)
+            {
+                this.address = address;
+                this.offset = offset;
+                this.limit = limit;
+            }
+
+            @Override
+            public ByteBuffer buffer()
+            {
+                assert isReferenced() : "Already unreferenced";
+                return MemoryUtil.allocateByteBuffer(address, limit, PageAware.PAGE_SIZE, ByteOrder.BIG_ENDIAN, null);
+            }
+
+            @Override
+            public long offset()
+            {
+                return offset;
+            }
 
-        ((CachingRebufferer) dfile.rebuffererFactory()).invalidate(position);
+            @Override
+            public void release()
+            {
+                MultiRegionChunk.this.release();
+            }
+        }
     }
 
-    public void invalidateFile(String fileName)
+    /**
+     * A chunk with a single memory region. This is always used for reading chunks of up to PageAware.PAGE_SIZE (note
+     * that the memory allocated will be always at least PageAware.PAGE_SIZE even if the reader requests a smaller
+     * buffer), and may also be used for larger chunks if the buffer pool can produce a contiguous memory buffer.
+     * See {@link this#newChunk}.
+     * <p/>
+     * This class is a chunk but also behaves as a {@link Rebufferer.BufferHolder} to save an allocation when
+     * {@link this#getBuffer(long)} is invoked.
+     */
+    class SingleRegionChunk extends Chunk implements Rebufferer.BufferHolder
     {
-        cache.invalidateAll(Iterables.filter(cache.asMap().keySet(), x -> x.path.equals(fileName)));
-    }
+        private final long address;
+        private final int capacity;
+        private final Object attachment;
 
-    // TODO: Invalidate caches for obsoleted/MOVED_START tables?
+        public SingleRegionChunk(long offset, ByteBuffer buffer)
+        {
+            super(offset);
+            this.address = MemoryUtil.getAddress(buffer);
+            this.capacity = buffer.capacity();
+            this.attachment = MemoryUtil.getAttachment(buffer);
+        }
+
+        public ByteBuffer buffer()
+        {
+            assert isReferenced() : "Already unreferenced";
+            return MemoryUtil.allocateByteBuffer(address, bytesRead, capacity, ByteOrder.BIG_ENDIAN, null);
+        }
+
+        public long offset()
+        {
+            return offset;
+        }
+
+        void releaseBuffers()
+        {
+            bufferPool.put(MemoryUtil.allocateByteBuffer(address, capacity, capacity, ByteOrder.BIG_ENDIAN, attachment));
+        }
+
+        void read(ChunkReader file)
+        {
+            ByteBuffer buffer = buffer();
+            file.readChunk(offset, buffer);
+            bytesRead = buffer.limit();
+        }
+
+        @Nullable
+        Rebufferer.BufferHolder getBuffer(long position)
+        {
+            return this;
+        }
+
+        int capacity()
+        {
+            return capacity;
+        }
+    }
 
     /**
      * Rebufferer providing cached chunks where data is obtained from the specified ChunkReader.
@@ -212,27 +610,79 @@ public void invalidateFile(String fileName)
     class CachingRebufferer implements Rebufferer, RebuffererFactory
     {
         private final ChunkReader source;
+        private final long readerId;
         final long alignmentMask;
 
         public CachingRebufferer(ChunkReader file)
         {
             source = file;
+            readerId = readerIdFor(file);
             int chunkSize = file.chunkSize();
             assert Integer.bitCount(chunkSize) == 1 : String.format("%d must be a power of two", chunkSize);
             alignmentMask = -chunkSize;
         }
 
         @Override
-        public Buffer rebuffer(long position)
+        public BufferHolder rebuffer(long position)
         {
             try
             {
                 long pageAlignedPos = position & alignmentMask;
-                Buffer buf;
-                do
-                    buf = cache.get(new Key(source, pageAlignedPos)).reference();
-                while (buf == null);
-
+                BufferHolder buf = null;
+                Key chunkKey = new Key(readerId, pageAlignedPos);
+
+                int spin = 0;
+                //There is a small window when a released buffer/invalidated chunk
+                //is still in the cache. In this case it will return null
+                //so we spin loop while waiting for the cache to re-populate
+                while (buf == null)
+                {
+                    Chunk chunk;
+                    // Using cache.get(k, compute) results in lots of allocation, rather risk the unlikely race...
+                    CompletableFuture<Chunk> cachedValue = cache.getIfPresent(chunkKey);
+                    if (cachedValue == null)
+                    {
+                        CompletableFuture<Chunk> entry = new CompletableFuture<>();
+                        CompletableFuture<Chunk> existing = cacheAsMap.putIfAbsent(chunkKey, entry);
+                        if (existing == null)
+                        {
+                            try
+                            {
+                                chunk = load(source, pageAlignedPos);
+                            }
+                            catch (Throwable t)
+                            {
+                                // please note that we don't need to remove the entry from the cache here
+                                // because Caffeine automatically removes entries that complete exceptionally
+
+                                // also signal other waiting readers
+                                entry.completeExceptionally(t);
+                                throw t;
+                            }
+                            entry.complete(chunk);
+                        }
+                        else
+                        {
+                            chunk = existing.get(CHUNK_CACHE_REBUFFER_WAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+                        }
+                    }
+                    else
+                    {
+                        chunk = cachedValue.get(CHUNK_CACHE_REBUFFER_WAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+                    }
+
+                    buf = chunk.getReferencedBuffer(position);
+
+                    if (buf == null && ++spin == 1000)
+                    {
+                        String msg = String.format("Could not acquire a reference to for %s after 1000 attempts. " +
+                                                   "This is likely due to the chunk cache being too small for the " +
+                                                   "number of concurrently running requests.", chunkKey);
+                        throw new RuntimeException(msg);
+                        // Note: this might also be caused by reference counting errors, especially double release of
+                        // chunks.
+                    }
+                }
                 return buf;
             }
             catch (Throwable t)
@@ -244,10 +694,11 @@ public Buffer rebuffer(long position)
             }
         }
 
-        public void invalidate(long position)
+        @Override
+        public void invalidateIfCached(long position)
         {
             long pageAlignedPos = position & alignmentMask;
-            cache.invalidate(new Key(source, pageAlignedPos));
+            synchronousCache.invalidate(new Key(readerId, pageAlignedPos));
         }
 
         @Override
@@ -314,8 +765,21 @@ public int size()
     @Override
     public long weightedSize()
     {
-        return cache.policy().eviction()
-                .map(policy -> policy.weightedSize().orElseGet(cache::estimatedSize))
-                .orElseGet(cache::estimatedSize);
+        return synchronousCache.policy().eviction()
+                .map(policy -> policy.weightedSize().orElseGet(synchronousCache::estimatedSize))
+                .orElseGet(synchronousCache::estimatedSize);
+    }
+
+    /**
+     * Returns the number of cached chunks of given file.
+     */
+    @VisibleForTesting
+    public int sizeOfFile(File file) {
+        Long fileIdMaybeNull = fileIdMap.get(file);
+        if (fileIdMaybeNull == null)
+            return 0;
+        long fileId = fileIdMaybeNull << (CHUNK_SIZE_LOG2_BITS + READER_TYPE_BITS);
+        long mask = - (1 << (CHUNK_SIZE_LOG2_BITS + READER_TYPE_BITS));
+        return (int) cacheAsMap.keySet().stream().filter(x -> (x.readerId & mask) == fileId).count();
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/cache/InstrumentingCache.java b/src/java/org/apache/cassandra/cache/InstrumentingCache.java
index 494dbbc5613c..c9be36d97842 100644
--- a/src/java/org/apache/cassandra/cache/InstrumentingCache.java
+++ b/src/java/org/apache/cassandra/cache/InstrumentingCache.java
@@ -20,6 +20,7 @@
 import java.util.Iterator;
 
 import org.apache.cassandra.metrics.CacheMetrics;
+import org.apache.cassandra.service.CacheService;
 
 /**
  * Wraps an ICache in requests + hits tracking.
@@ -27,15 +28,15 @@
 public class InstrumentingCache<K, V>
 {
     private final ICache<K, V> map;
-    private final String type;
+    private final CacheService.CacheType type;
 
     private CacheMetrics metrics;
 
-    public InstrumentingCache(String type, ICache<K, V> map)
+    public InstrumentingCache(CacheService.CacheType type, ICache<K, V> map)
     {
         this.map = map;
         this.type = type;
-        this.metrics = new CacheMetrics(type, map);
+        this.metrics = CacheMetrics.create(type, map);
     }
 
     public void put(K key, V value)
@@ -55,12 +56,11 @@ public boolean replace(K key, V old, V value)
 
     public V get(K key)
     {
-        metrics.requests.mark();
         V v = map.get(key);
         if (v != null)
-            metrics.hits.mark();
+            metrics.recordHits(1);
         else
-            metrics.misses.mark();
+            metrics.recordMisses(1);
         return v;
     }
 
@@ -100,7 +100,7 @@ public void clear()
 
         // this does not clear metered metrics which are defined statically. for testing purposes, these can be
         // cleared by CacheMetrics.reset()
-        metrics = new CacheMetrics(type, map);
+        metrics = CacheMetrics.create(type, map);
     }
 
     public Iterator<K> keyIterator()
diff --git a/src/java/org/apache/cassandra/cache/KeyCacheKey.java b/src/java/org/apache/cassandra/cache/KeyCacheKey.java
index ac6f1f969311..1a722b05e6c4 100644
--- a/src/java/org/apache/cassandra/cache/KeyCacheKey.java
+++ b/src/java/org/apache/cassandra/cache/KeyCacheKey.java
@@ -21,7 +21,6 @@
 import java.util.Arrays;
 import java.util.Objects;
 
-import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -31,9 +30,7 @@ public class KeyCacheKey extends CacheKey
 {
     public final Descriptor desc;
 
-    private static final long EMPTY_SIZE = ObjectSizes.measure(new KeyCacheKey(TableMetadata.builder("ks", "tab")
-                                                                                            .addPartitionKeyColumn("pk", UTF8Type.instance)
-                                                                                            .build(), null, ByteBufferUtil.EMPTY_BYTE_BUFFER));
+    private static final long EMPTY_SIZE = ObjectSizes.measure(new KeyCacheKey());
 
     // keeping an array instead of a ByteBuffer lowers the overhead of the key cache working set,
     // without extra copies on lookup since client-provided key ByteBuffers will be array-backed already
@@ -47,6 +44,13 @@ public KeyCacheKey(TableMetadata tableMetadata, Descriptor desc, ByteBuffer key)
         assert this.key != null;
     }
 
+    private KeyCacheKey() // Only for EMPTY_SIZE
+    {
+        super(null, null);
+        this.desc = null;
+        this.key = null;
+    }
+
     public String toString()
     {
         return String.format("KeyCacheKey(%s, %s)", desc, ByteBufferUtil.bytesToHex(ByteBuffer.wrap(key)));
diff --git a/src/java/org/apache/cassandra/concurrent/ExecutionFailure.java b/src/java/org/apache/cassandra/concurrent/ExecutionFailure.java
index 27ab885e234e..a90d6620fb68 100644
--- a/src/java/org/apache/cassandra/concurrent/ExecutionFailure.java
+++ b/src/java/org/apache/cassandra/concurrent/ExecutionFailure.java
@@ -125,6 +125,8 @@ private static Runnable enforceOptions(WithResources withResources, Runnable wra
     {
         return new Runnable()
         {
+            Throwable callStack = new Throwable();
+
             @Override
             public void run()
             {
diff --git a/src/java/org/apache/cassandra/concurrent/ExecutorLocals.java b/src/java/org/apache/cassandra/concurrent/ExecutorLocals.java
index 29985754aa6c..c60e691d5ec5 100644
--- a/src/java/org/apache/cassandra/concurrent/ExecutorLocals.java
+++ b/src/java/org/apache/cassandra/concurrent/ExecutorLocals.java
@@ -19,20 +19,22 @@
 package org.apache.cassandra.concurrent;
 
 import io.netty.util.concurrent.FastThreadLocal;
+import org.apache.cassandra.sensors.RequestSensors;
 import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.context.OperationContext;
 import org.apache.cassandra.tracing.TraceState;
 import org.apache.cassandra.utils.Closeable;
 import org.apache.cassandra.utils.WithResources;
 
 /*
- * This class only knows about Tracing and ClientWarn, so if any different executor locals are added, it must be
+ * This class only knows about fixed locals, so if any different executor locals are added, it must be
  * updated.
  *
  * We don't enumerate the ExecutorLocal.all array each time because it would be much slower.
  */
 public class ExecutorLocals implements WithResources, Closeable
 {
-    private static final ExecutorLocals none = new ExecutorLocals(null, null);
+    private static final ExecutorLocals none = new ExecutorLocals(null, null, null, null);
     private static final FastThreadLocal<ExecutorLocals> locals = new FastThreadLocal<ExecutorLocals>()
     {
         @Override
@@ -44,20 +46,24 @@ protected ExecutorLocals initialValue()
 
     public static class Impl
     {
-        protected static void set(TraceState traceState, ClientWarn.State clientWarnState)
+        public static void set(TraceState traceState, ClientWarn.State clientWarnState, RequestSensors sensors, OperationContext operationContext)
         {
-            if (traceState == null && clientWarnState == null) locals.set(none);
-            else locals.set(new ExecutorLocals(traceState, clientWarnState));
+            if (traceState == null && clientWarnState == null && sensors == null && operationContext == null) locals.set(none);
+            else locals.set(new ExecutorLocals(traceState, clientWarnState, sensors, operationContext));
         }
     }
 
     public final TraceState traceState;
     public final ClientWarn.State clientWarnState;
+    public final RequestSensors sensors;
+    public final OperationContext operationContext;
 
-    protected ExecutorLocals(TraceState traceState, ClientWarn.State clientWarnState)
+    protected ExecutorLocals(TraceState traceState, ClientWarn.State clientWarnState, RequestSensors sensors, OperationContext operationContext)
     {
         this.traceState = traceState;
         this.clientWarnState = clientWarnState;
+        this.sensors = sensors;
+        this.operationContext = operationContext;
     }
 
     /**
@@ -81,7 +87,13 @@ public static WithResources propagate()
     public static ExecutorLocals create(TraceState traceState)
     {
         ExecutorLocals current = locals.get();
-        return current.traceState == traceState ? current : new ExecutorLocals(traceState, current.clientWarnState);
+        return current.traceState == traceState ? current : new ExecutorLocals(traceState, current.clientWarnState, current.sensors, current.operationContext);
+    }
+
+    public static ExecutorLocals create(RequestSensors sensors)
+    {
+        ExecutorLocals current = locals.get();
+        return current.sensors == sensors ? current : new ExecutorLocals(current.traceState, current.clientWarnState, sensors, current.operationContext);
     }
 
     public static void clear()
diff --git a/src/java/org/apache/cassandra/concurrent/InlinedThreadLocalThread.java b/src/java/org/apache/cassandra/concurrent/InlinedThreadLocalThread.java
new file mode 100644
index 000000000000..0b0822243912
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/InlinedThreadLocalThread.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.concurrent;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.cassandra.io.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.util.concurrent.FastThreadLocalThread;
+import net.nicoulaj.compilecommand.annotations.Inline;
+import org.jctools.util.UnsafeAccess;
+import org.jctools.util.UnsafeRefArrayAccess;
+
+/**
+ * An intermediate class for forcing the position and ordering of reference fields which we intend to use as the backing
+ * storage for {@link InlinedThreadLocalThread} instances.
+ */
+@SuppressWarnings("unused")
+abstract class InlinedThreadLocalThreadLocalRefs extends FastThreadLocalThread
+{
+    // DO NOT ADD FURTHER FIELDS TO THIS CLASS OTHER THAN REFS FOR TL STORAGE
+    // Static fields are fine ;-), See note below after ref fields.
+    static final int MAX_LOCALS = 70;
+    static final long REF_BASE_OFFSET;
+    private static final Logger LOGGER = LoggerFactory.getLogger(InlinedThreadLocalThread.class);
+    private static final long REF_SHIFT = UnsafeRefArrayAccess.REF_ELEMENT_SHIFT;
+    private static final AtomicInteger FIRST_UNUSED_INDEX = new AtomicInteger();
+
+    public static int maxLocalsOverflow;
+
+    static
+    {
+        long minOffset = Long.MAX_VALUE;
+        // fields order is not guaranteed
+        for (int i = 0; i < MAX_LOCALS; i++)
+        {
+            String fieldName = "a" + i;
+            long fieldOffset = UnsafeAccess.fieldOffset(InlinedThreadLocalThreadLocalRefs.class, fieldName);
+            minOffset = Math.min(fieldOffset, minOffset);
+        }
+        REF_BASE_OFFSET = minOffset;
+    }
+
+    // Add further refs as needed and adjust the MAX_LOCALS constant accordingly
+    private Object a0, a1, a2, a3, a4, a5, a6, a7, a8, a9;
+    private Object a10, a11, a12, a13, a14, a15, a16, a17, a18, a19;
+    private Object a20, a21, a22, a23, a24, a25, a26, a27, a28, a29;
+    private Object a30, a31, a32, a33, a34, a35, a36, a37, a38, a39;
+    private Object a40, a41, a42, a43, a44, a45, a46, a47, a48, a49;
+    private Object a50, a51, a52, a53, a54, a55, a56, a57, a58, a59;
+    private Object a60, a61, a62, a63, a64, a65, a66, a67, a68, a69;
+    // DO NOT ADD FURTHER FIELDS TO THIS CLASS OTHER THAN REFS FOR TL REF STORAGE
+    // Adding further fields may cause these fields to mingle with the above allocate fields in term of their relative
+    // position. The get/set methods we use below rely on the reference fields above being laid out as a contiguous
+    // space.
+
+    InlinedThreadLocalThreadLocalRefs()
+    {
+        super();
+    }
+
+    InlinedThreadLocalThreadLocalRefs(Runnable target)
+    {
+        super(target);
+    }
+
+    InlinedThreadLocalThreadLocalRefs(ThreadGroup group, Runnable target)
+    {
+        super(group, target);
+    }
+
+    InlinedThreadLocalThreadLocalRefs(String name)
+    {
+        super(name);
+    }
+
+    InlinedThreadLocalThreadLocalRefs(ThreadGroup group, String name)
+    {
+        super(group, name);
+    }
+
+    InlinedThreadLocalThreadLocalRefs(Runnable target, String name)
+    {
+        super(target, name);
+    }
+
+    InlinedThreadLocalThreadLocalRefs(ThreadGroup group, Runnable target, String name)
+    {
+        super(group, target, name);
+    }
+
+    public InlinedThreadLocalThreadLocalRefs(ThreadGroup group, Runnable target, String name, long stackSize)
+    {
+        super(group, target, name, stackSize);
+    }
+
+    static long getRefOffset(long index)
+    {
+        if (index > MAX_LOCALS)
+        {
+            throw new IllegalStateException("FIRST_UNUSED_INDEX:" + index + " exceeds max: " + MAX_LOCALS);
+        }
+        return REF_BASE_OFFSET + (index << REF_SHIFT);
+    }
+
+    static long nextRefIndex()
+    {
+        int currIndex;
+        do
+        {
+            currIndex = FIRST_UNUSED_INDEX.get();
+            if (currIndex == MAX_LOCALS)
+            {
+                maxLocalsOverflow++;
+                return -1;
+            }
+        }
+        while (!FIRST_UNUSED_INDEX.compareAndSet(currIndex, currIndex + 1));
+        return currIndex;
+    }
+
+    @VisibleForTesting
+    static void resetIndexForTestingOnly()
+    {
+        // ONLY FOR TESTING!!!!
+        FIRST_UNUSED_INDEX.set(0);
+    }
+
+    @Inline
+    Object getThreadLocalRef(long offset)
+    {
+        return UnsafeAccess.UNSAFE.getObject(this, offset);
+    }
+
+    @Inline
+    void setThreadLocalRef(long offset, Object v)
+    {
+        UnsafeAccess.UNSAFE.putObject(this, offset, v);
+    }
+
+    void cleanupOnExit()
+    {
+        for (int i = 0; i < MAX_LOCALS; i++)
+        {
+            long offset = getRefOffset(i);
+            Object threadLocal = getThreadLocalRef(offset);
+            if (threadLocal == null)
+            {
+                continue;
+            }
+            // Maybe a pointless gesture, but might prevent GC nepotism (where dead objects in old gen keep young
+            // objects they refer to alive until an old gen GC removes them). In all likelihood the TL object have
+            // similar lifecycle to the thread and are therefore in the same generation.
+            setThreadLocalRef(offset, null);
+
+            // Rather than wait for the GC to pick these up, clean them now (particularly important for offheap buffers).
+            if (threadLocal instanceof ByteBuffer)
+            {
+                cleanupByteBuffer((ByteBuffer) threadLocal);
+            }
+            else if (threadLocal instanceof Closeable)
+            {
+                cleanupCloseable((Closeable) threadLocal);
+            }
+        }
+    }
+
+    private void cleanupCloseable(Closeable threadLocal)
+    {
+        try
+        {
+            threadLocal.close();
+        }
+        catch (IOException e)
+        {
+            LOGGER.error("Failed to close thread local resource on thread exit", e);
+        }
+    }
+
+    private void cleanupByteBuffer(ByteBuffer threadLocal)
+    {
+        try
+        {
+            FileUtils.clean(threadLocal);
+        }
+        catch (Throwable e)
+        {
+            LOGGER.error("Failed to clean DirectBuffer thread local resource on thread exit", e);
+        }
+    }
+
+}
+
+public class InlinedThreadLocalThread extends InlinedThreadLocalThreadLocalRefs
+{
+
+    public InlinedThreadLocalThread()
+    {
+        super();
+    }
+
+    public InlinedThreadLocalThread(Runnable target)
+    {
+        super(target);
+    }
+
+    public InlinedThreadLocalThread(ThreadGroup group, Runnable target)
+    {
+        super(group, target);
+    }
+
+    public InlinedThreadLocalThread(String name)
+    {
+        super(name);
+    }
+
+    public InlinedThreadLocalThread(ThreadGroup group, String name)
+    {
+        super(group, name);
+    }
+
+    public InlinedThreadLocalThread(Runnable target, String name)
+    {
+        super(target, name);
+    }
+
+    public InlinedThreadLocalThread(ThreadGroup group, Runnable target, String name)
+    {
+        super(group, target, name);
+    }
+
+    public InlinedThreadLocalThread(ThreadGroup group, Runnable target, String name, long stackSize)
+    {
+        super(group, target, name, stackSize);
+    }
+
+    @Override
+    public void run()
+    {
+        try
+        {
+            super.run();
+        }
+        finally
+        {
+            cleanupOnExit();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/concurrent/ParkedExecutor.java b/src/java/org/apache/cassandra/concurrent/ParkedExecutor.java
new file mode 100644
index 000000000000..65b3d072344e
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/ParkedExecutor.java
@@ -0,0 +1,334 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.LockSupport;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.utils.Flags;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.jctools.queues.MpmcArrayQueue;
+import org.jctools.queues.MpscUnboundedArrayQueue;
+
+/**
+ * An executor which relies on the {@link ParkedThreadsMonitor} to unpark it's threads which are waiting on lock-less
+ * queues. How can threads wait on lock-less queues? They just park themselves when they see the queue as empty. Since
+ * this observation is non-atomic to other threads putting work on the queue, and since putting work on the queue of a
+ * parked thread doesn't notify the parked worker, we have the {@link ParkedThreadsMonitor} check when/if new work is
+ * available for parked threads and unpark them.
+ * <p>
+ * This has the benefit of removing the cost of {@link LockSupport#unpark(Thread)} from the threads which are trying to
+ * offload work onto the executor.
+ */
+public abstract class ParkedExecutor implements ShutdownableExecutor
+{
+    private static final Logger logger = LoggerFactory.getLogger(ParkedExecutor.class);
+    private static final Runnable POISON_PILL = () -> {};
+    volatile boolean isShutdown = false;
+
+    protected final ParkedThreadsMonitor monitor = ParkedThreadsMonitor.instance.get().orElseThrow(() -> new IllegalStateException("Cannot use because ParkedThreadsMonitor is disabled"));
+
+    /**
+     * Create a {@code ParkedExecutor} if {@link ParkedThreadsMonitor} is enabled, otherwise a standard executor.
+     *
+     * @param name A base name for the threads in the thread pool (not null)
+     * @param threadCount number of threads for the backing thread pool (>= 1)
+     */
+    public static ShutdownableExecutor createParkedExecutor(String name, int threadCount)
+    {
+        if (threadCount < 1)
+            throw new IllegalArgumentException("Less than 1 thread in thread pool is not allowed, but " + threadCount +" requested for pool " + name);
+        if (threadCount == 1)
+        {
+            return ParkedThreadsMonitor.IS_ENABLED ?
+                   new ParkedSingleThreadedExecutor(name) :
+                   new FallbackExecutor(ExecutorFactory.Global.executorFactory().sequential(name));
+        }
+        return ParkedThreadsMonitor.IS_ENABLED ?
+               new ParkedMultiThreadedExecutor(name, threadCount) :
+               new FallbackExecutor(ExecutorFactory.Global.executorFactory().pooled(name, threadCount));
+    }
+
+    private static void safeRun(Runnable r)
+    {
+        try
+        {
+            r.run();
+        }
+        catch (Throwable throwable)
+        {
+            JVMStabilityInspector.inspectThrowable(throwable);
+            logger.error("Error while runningThreadsBitmap tasks: ", throwable);
+        }
+    }
+
+    static class ParkedMultiThreadedExecutor extends ParkedExecutor implements ParkedThreadsMonitor.MonitorableThread
+    {
+        private final int allThreadsRunning;
+        private final Thread[] workers;
+        // bounded pre-allocated queue to avoid per task allocation overheads
+        private final MpmcArrayQueue<Runnable> tasks = new MpmcArrayQueue<>(1024);
+        // unbounded overflow for when the above is not enough
+        private final Queue<Runnable> overflowTasks = new ConcurrentLinkedQueue<>();
+        // this acts as an isRunning indicator bit map for the threads
+        private final AtomicInteger runningThreadsBitmap = new AtomicInteger();
+        private final int threadCount;
+
+        private ParkedMultiThreadedExecutor(String name, int threadCount)
+        {
+            if (threadCount > 32)
+                throw new IllegalStateException("ParkedExecutors pool size is limited to 32 threads currently");
+            this.threadCount = threadCount;
+            allThreadsRunning = -1 >>> (32 - threadCount);
+            workers = new Thread[threadCount];
+            NamedThreadFactory namedThreadFactory = new NamedThreadFactory(name);
+            for (int i = 0; i < threadCount; i++)
+            {
+                int tIndex = i;
+                workers[i] = namedThreadFactory.newThread(() -> processTasks(tIndex));
+                workers[i].start();
+            }
+
+            monitor.addThreadToMonitor(this);
+        }
+
+
+        @Override
+        public void shutdown() throws InterruptedException
+        {
+            isShutdown = true;
+            for (int i = 0; i < threadCount; i++)
+            {
+                // poison pill always on overflow
+                overflowTasks.offer(POISON_PILL);
+            }
+
+            for (int i = 0; i < threadCount; i++)
+            {
+                workers[i].join();
+            }
+            monitor.removeThreadToMonitor(this);
+        }
+
+        @Override
+        public void execute(Runnable task)
+        {
+            if (isShutdown)
+                throw new RejectedExecutionException("Executor has already been shutdown");
+            // We have a bounded tasks queue, if it overflows we send via the unbounded MPMC queue. This can introduce
+            // task reordering, but this is valid behavior for an Executor.
+            if (!tasks.offer(task))
+                overflowTasks.offer(task);
+        }
+
+        private void processTasks(int threadIndex)
+        {
+            // the bit to set/unset in the runningThreadsBitmap for this thread index
+            int tMask = 1 << threadIndex;
+            setThreadRunning(tMask);
+
+            Runnable r;
+            while (true)
+            {
+                boolean noTask = true;
+                // always check both queues to ensure one cannot starve the other
+                if ((r = overflowTasks.poll()) != null)
+                {
+                    if (r == POISON_PILL)
+                        break;
+                    noTask = false;
+                    safeRun(r);
+                }
+                if ((r = tasks.relaxedPoll()) != null)
+                {
+                    // no poison on tasks queue
+                    noTask = false;
+                    safeRun(r);
+                }
+
+                if (noTask)
+                {
+                    setThreadNotRunning(tMask);
+                    LockSupport.park();
+                    setThreadRunning(tMask);
+
+                    if (Thread.interrupted())
+                        break;
+                }
+            }
+
+            // after poison pill observed make best effort to drain queue
+            while ((r = tasks.poll()) != null)
+            {
+                safeRun(r);
+            }
+        }
+
+        private void setThreadRunning(int tMask)
+        {
+            int current;
+            do
+            {
+                current = runningThreadsBitmap.get();
+            }
+            while (!runningThreadsBitmap.compareAndSet(current, Flags.add(current, tMask)));
+        }
+
+        private void setThreadNotRunning(int tMask)
+        {
+            int current;
+            do
+            {
+                current = runningThreadsBitmap.get();
+            }
+            while (!runningThreadsBitmap.compareAndSet(current, Flags.remove(current, tMask)));
+        }
+
+        @Override
+        public void unpark(long epoch)
+        {
+            int currentlyRunningMask = runningThreadsBitmap.get();
+            for (int i = 0; i < threadCount; i++)
+            {
+                if (!Flags.contains(currentlyRunningMask, 1 << i))
+                {
+                    Thread worker = workers[i];
+                    LockSupport.unpark(worker);
+                    return;
+                }
+            }
+        }
+
+        @Override
+        public boolean shouldUnpark(long epoch)
+        {
+            return (runningThreadsBitmap.get() != allThreadsRunning) &&
+                    !(tasks.isEmpty() && overflowTasks.isEmpty());
+        }
+    }
+
+    /**
+     * A specialized implementation for a single threaded executor allows us to use the unbounded MPSC queue for
+     * improved performance and simpler workflow.
+     */
+    static class ParkedSingleThreadedExecutor extends ParkedExecutor implements ParkedThreadsMonitor.MonitorableThread
+    {
+        private final Thread worker;
+        private final MpscUnboundedArrayQueue<Runnable> tasks = new MpscUnboundedArrayQueue<>(1024);
+        private final AtomicInteger running = new AtomicInteger();
+
+        private ParkedSingleThreadedExecutor(String name)
+        {
+            NamedThreadFactory namedThreadFactory = new NamedThreadFactory(name);
+            worker = namedThreadFactory.newThread(this::processTasks);
+            worker.start();
+            monitor.addThreadToMonitor(this);
+        }
+
+
+        @Override
+        public void shutdown() throws InterruptedException
+        {
+            isShutdown = true;
+            tasks.offer(POISON_PILL);
+            worker.join();
+            monitor.removeThreadToMonitor(this);
+        }
+
+        @Override
+        public void execute(Runnable task)
+        {
+            if (isShutdown)
+                throw new RejectedExecutionException("Executor has already been shutdown");
+            // This is an unbounded queue, so no need to check for failed offers
+            tasks.offer(task);
+        }
+
+        private void processTasks()
+        {
+            running.lazySet(1);
+
+            Runnable r;
+            while ((r = tasks.relaxedPoll()) != POISON_PILL)
+            {
+                if (r == null)
+                {
+                    running.lazySet(0);
+                    LockSupport.park();
+                    running.lazySet(1);
+
+                    if (Thread.interrupted())
+                        break;
+                    continue;
+                }
+                ParkedExecutor.safeRun(r);
+            }
+        }
+
+        @Override
+        public void unpark(long epoch)
+        {
+            int currentlyRunningMask = running.get();
+            if (currentlyRunningMask == 0)
+            {
+                LockSupport.unpark(this.worker);
+            }
+        }
+
+        @Override
+        public boolean shouldUnpark(long epoch)
+        {
+            return running.get() != 1 && !tasks.isEmpty();
+        }
+    }
+
+    static class FallbackExecutor implements ShutdownableExecutor
+    {
+        private final ExecutorService executor;
+
+        private FallbackExecutor(ExecutorService executor)
+        {
+            this.executor = executor;
+        }
+
+        @Override
+        public void shutdown() throws InterruptedException
+        {
+            executor.shutdownNow();
+            executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+        }
+
+        @Override
+        public void execute(Runnable command)
+        {
+            executor.execute(command);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/concurrent/ParkedThreadsMonitor.java b/src/java/org/apache/cassandra/concurrent/ParkedThreadsMonitor.java
new file mode 100644
index 000000000000..e2ef63f0c4bd
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/ParkedThreadsMonitor.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.LockSupport;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.ThreadsFactory;
+import org.jctools.queues.MpscUnboundedArrayQueue;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.THREAD_MONITOR_AUTO_CALIBRATE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.THREAD_MONITOR_ENABLED;
+import static org.apache.cassandra.config.CassandraRelevantProperties.THREAD_MONITOR_SLEEP_INTERVAL_NS;
+
+/**
+ * This class is responsible for keeping tabs on threads that register as a {@link MonitorableThread}.
+ * <p/>
+ * When a {@link MonitorableThread} parks itself this thread wakes it up when ready to execute work, according to the
+ * {@link MonitorableThread#shouldUnpark(long)} method. This allows threads to (for example) use a non-blocking queue 
+ * without constantly spinning.
+ * <p/>
+ * It's up to the {@link MonitorableThread#shouldUnpark(long )} implementation to track its own state.
+ */
+public class ParkedThreadsMonitor
+{
+    public static final boolean IS_ENABLED = THREAD_MONITOR_ENABLED.getBoolean();
+    private static final Logger LOGGER = LoggerFactory.getLogger(ParkedThreadsMonitor.class);
+
+    public static final Supplier<Optional<ParkedThreadsMonitor>> instance = Suppliers.memoize(() ->
+    {
+        if (IS_ENABLED)
+            return Optional.of(new ParkedThreadsMonitor());
+        else
+        {
+            LOGGER.warn("ParkedThreadsMonitor is DISABLED.");
+            return Optional.empty();
+        }
+    });
+
+    /*
+     * NOTE: parkNanos seems to operate as follows (on my machine, expect differences):
+     *  - 1 to 80ns   -> p50% is 400-500ns
+     *  - 80 to 100ns -> some instability, flip between 500ns and 52us
+     *  - > 100ns     -> p50% looks like 52us + parkTime
+     */
+    private static final long SLEEP_INTERVAL_NS = THREAD_MONITOR_SLEEP_INTERVAL_NS.getLong();
+    private static final boolean AUTO_CALIBRATE = THREAD_MONITOR_AUTO_CALIBRATE.getBoolean() && SLEEP_INTERVAL_NS > 0;
+
+    private static final Sleeper SLEEPER = AUTO_CALIBRATE ? new CalibratingSleeper(SLEEP_INTERVAL_NS) : new Sleeper();
+
+    private final MpscUnboundedArrayQueue<Runnable> commands = new MpscUnboundedArrayQueue<>(128);
+    private final ArrayList<MonitorableThread> monitoredThreads =
+        new ArrayList<>(Runtime.getRuntime().availableProcessors() * 2);
+    private final ArrayList<Runnable> loopActions = new ArrayList<>(4);
+    private final Thread watcherThread;
+        
+    private volatile boolean shutdown;
+    
+    /**
+     * The epoch monotonically tracks how many times the monitor has run: each run is a new epoch, and at each run
+     * this will be passed to the {@link MonitorableThread} unpark methods, so they can use it to distinguish between
+     * calls happening for the same (or different) epoch.
+     */
+    private long epoch;
+
+    @VisibleForTesting
+    ParkedThreadsMonitor(long startEpoch)
+    {
+        epoch = startEpoch;
+        shutdown = false;
+        watcherThread = ThreadsFactory.newDaemonThread(this::run, "ParkedThreadsMonitor");
+        watcherThread.setPriority(Thread.MAX_PRIORITY);
+        watcherThread.start();
+    }
+    
+    private ParkedThreadsMonitor()
+    {
+        this(0);
+    }
+
+    private void run()
+    {
+        final ArrayList<Runnable> loopActions = this.loopActions;
+        final ArrayList<MonitorableThread> monitoredThreads = this.monitoredThreads;
+        final MpscUnboundedArrayQueue<Runnable> commands = this.commands;
+        while (!shutdown)
+        {
+            try
+            {
+                runCommands(commands);
+                executeLoopActions(loopActions);
+                monitorThreads(monitoredThreads);
+            }
+            catch (Throwable t)
+            {
+                JVMStabilityInspector.inspectThrowable(t);
+                LOGGER.error("ParkedThreadsMonitor exception: ", t);
+            }
+            finally
+            {
+                epoch++;
+            }
+            SLEEPER.sleep();
+        }
+        unparkOnShutdown(monitoredThreads);
+    }
+
+    private void monitorThreads(ArrayList<MonitorableThread> monitoredThreads)
+    {
+        if (monitoredThreads.isEmpty())
+            return;
+        
+        // To avoid biasing the work threads do based on their order of unparking, we start iterating from an offset
+        // based on the epoch, so it's cheap and always changing.
+        int offset = ((int) epoch & Integer.MAX_VALUE) % monitoredThreads.size();
+        for (int i = 0; i < monitoredThreads.size(); i++)
+        {
+            MonitorableThread thread = monitoredThreads.get((i + offset) % monitoredThreads.size());
+            try
+            {
+                if (thread.shouldUnpark(epoch))
+                {
+                    // TODO: add a counter we can track for unpark rate
+                    thread.unpark(epoch);
+                }
+            }
+            catch (Throwable t)
+            {
+                LOGGER.error("Exception unparking a monitored thread", t);
+            }
+        }
+    }
+
+    private void executeLoopActions(ArrayList<Runnable> loopActions)
+    {
+        for (int i = 0; i < loopActions.size(); i++)
+        {
+            Runnable action = loopActions.get(i);
+            try
+            {
+                action.run();
+            }
+            catch (Throwable t)
+            {
+                LOGGER.error("Exception running an action", t);
+            }
+        }
+    }
+
+    private void runCommands(MpscUnboundedArrayQueue<Runnable> commands)
+    {
+        // queue is almost always empty, so pre check makes sense
+        if (!commands.isEmpty())
+        {
+            commands.drain(Runnable::run);
+        }
+    }
+
+    private void unparkOnShutdown(ArrayList<MonitorableThread> monitoredThreads)
+    {
+        for (MonitorableThread thread : monitoredThreads)
+            thread.unpark(epoch);
+    }
+
+    /**
+     * Adds a collection of threads to monitor
+     */
+    public void addThreadsToMonitor(Collection<MonitorableThread> threads)
+    {
+        threads.forEach(this::addThreadToMonitor);
+    }
+
+    /**
+     * Adds a thread to monitor
+     */
+    public void addThreadToMonitor(MonitorableThread thread)
+    {
+        commands.offer(() -> monitoredThreads.add(thread));
+    }
+
+    /**
+     * Removes a thread from monitoring
+     */
+    public void removeThreadToMonitor(MonitorableThread thread)
+    {
+        commands.offer(() -> monitoredThreads.remove(thread));
+    }
+
+    /**
+     * Removes a collection of threads from monitoring
+     */
+    public void removeThreadsToMonitor(Collection<MonitorableThread> threads)
+    {
+        threads.forEach(this::removeThreadToMonitor);
+    }
+
+    /**
+     * Runs the specified action in each loop. Mainly used to avoid many threads doing the same work
+     * over and over.
+     */
+    public void addAction(Runnable action)
+    {
+        commands.offer(() -> loopActions.add(action));
+    }
+
+    public void shutdown()
+    {
+        shutdown = true;
+    }
+
+    public boolean awaitTermination(long timeout, TimeUnit timeUnit) throws InterruptedException
+    {
+        shutdown();
+        watcherThread.join(timeUnit.toMillis(timeout));
+        return !watcherThread.isAlive();
+    }
+
+    /**
+     * Interface for threads wishing to be watched by the {@link ParkedThreadsMonitor} in order to be unparked.
+     * <p/>
+     * When a Thread has no work to do it should park itself.
+     */
+    public static interface MonitorableThread
+    {
+        /**
+         * What a MonitorableThread should use to track it's current state
+         */
+        enum ThreadState
+        {
+            PARKED, // no work to do
+            UNPARKING, // unparked but not yet working
+            WORKING
+        }
+
+        /**
+         * Will unpark a {@link ThreadState#PARKED} thread.
+         * <br/>
+         * Called by {@link ParkedThreadsMonitor} AFTER {@link #shouldUnpark(long )} returns true.
+         * 
+         * @@param epoch A monotonic counter increased every time the {@link ParkedThreadsMonitor} runs, 
+         * see {@link ParkedThreadsMonitor#epoch}.
+         */
+        void unpark(long epoch);
+
+        /**
+         * Called continuously by the {@link ParkedThreadsMonitor} to decide if unpark() should be called on a
+         * thread.
+         * <br/>
+         * Should return true IFF the thread is parked and there (potentially) is work to be done when the thread is
+         * unparked.
+         * 
+         * @param epoch A monotonic counter increased every time the {@link ParkedThreadsMonitor} runs, 
+         * see {@link ParkedThreadsMonitor#epoch}.
+         */
+        boolean shouldUnpark(long epoch);
+    }
+
+    static class Sleeper
+    {
+        void sleep()
+        {
+            if (SLEEP_INTERVAL_NS > 0)
+                LockSupport.parkNanos(SLEEP_INTERVAL_NS);
+        }
+    }
+
+    @VisibleForTesting
+    public static class CalibratingSleeper extends Sleeper
+    {
+        final long targetNs;
+        long calibratedSleepNs;
+        long expSmoothedSleepTimeNs;
+        int comparisonDelay;
+
+        @VisibleForTesting
+        public CalibratingSleeper(long targetNs)
+        {
+            this.targetNs = targetNs;
+            this.calibratedSleepNs = targetNs;
+            this.expSmoothedSleepTimeNs = 0;
+        }
+
+        @VisibleForTesting
+        public void sleep()
+        {
+            long start = nanoTime();
+            park();
+            long sleptNs = nanoTime() - start;
+            if (expSmoothedSleepTimeNs == 0)
+                expSmoothedSleepTimeNs = sleptNs;
+            else
+                expSmoothedSleepTimeNs = (long) (0.001 * sleptNs + 0.999 * expSmoothedSleepTimeNs);
+
+            // calibrate sleep time if we miss target by more than 10%, wait for at least 100 observations
+            if (comparisonDelay < 100)
+            {
+                comparisonDelay++;
+            }
+            else if (expSmoothedSleepTimeNs > targetNs * 1.1)
+            {
+                calibratedSleepNs = (long) (calibratedSleepNs * 0.9) + 1;
+                expSmoothedSleepTimeNs = 0;
+                comparisonDelay = 0;
+            }
+            else if (expSmoothedSleepTimeNs < targetNs * 0.9)
+            {
+                calibratedSleepNs = (long) (calibratedSleepNs * 1.1);
+                expSmoothedSleepTimeNs = 0;
+                comparisonDelay = 0;
+            }
+        }
+
+        @VisibleForTesting
+        void park()
+        {
+            LockSupport.parkNanos(calibratedSleepNs);
+        }
+
+        @VisibleForTesting
+        long nanoTime()
+        {
+            return MonotonicClock.Global.preciseTime.now();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/concurrent/ShutdownableExecutor.java b/src/java/org/apache/cassandra/concurrent/ShutdownableExecutor.java
new file mode 100644
index 000000000000..e11e5b872295
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/ShutdownableExecutor.java
@@ -0,0 +1,38 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.concurrent;
+
+import java.util.concurrent.Executor;
+
+/**
+ * An executor you can shut down!
+ */
+public interface ShutdownableExecutor extends Executor
+{
+
+    /**
+     * Stop the threads started for this executor(if any), returns when the threads spawned for this executor have all
+     * been stopped. Wait as long as it takes.
+     * <p>
+     * Note that tasks sent via {@link #execute(Runnable)} may be stuck in queue indefinitely.
+     */
+    void shutdown() throws InterruptedException;
+}
diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java
index 05b6c612fa86..6ec700fb9c9d 100644
--- a/src/java/org/apache/cassandra/concurrent/Stage.java
+++ b/src/java/org/apache/cassandra/concurrent/Stage.java
@@ -20,34 +20,52 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
 import java.util.function.IntSupplier;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.metrics.ThreadPoolMetrics;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.ExecutorUtils;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.Future;
 
 import static java.util.stream.Collectors.toMap;
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_STAGE_EXECUTOR_FACTORY_PROPERTY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_TASK_EXECUTION_CALLBACK_CLASS;
 
 public enum Stage
 {
-    READ              (false, "ReadStage",             "request",  DatabaseDescriptor::getConcurrentReaders,        DatabaseDescriptor::setConcurrentReaders,        Stage::multiThreadedLowSignalStage),
-    MUTATION          (true,  "MutationStage",         "request",  DatabaseDescriptor::getConcurrentWriters,        DatabaseDescriptor::setConcurrentWriters,        Stage::multiThreadedLowSignalStage),
-    COUNTER_MUTATION  (true,  "CounterMutationStage",  "request",  DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, Stage::multiThreadedLowSignalStage),
-    VIEW_MUTATION     (true,  "ViewMutationStage",     "request",  DatabaseDescriptor::getConcurrentViewWriters,    DatabaseDescriptor::setConcurrentViewWriters,    Stage::multiThreadedLowSignalStage),
+    READ              (false, "ReadStage",             "request",  DatabaseDescriptor::getConcurrentReaders,        DatabaseDescriptor::setConcurrentReaders,        maybeCustomStageExecutor(Stage::multiThreadedLowSignalStage)),
+    MUTATION          (true,  "MutationStage",         "request",  DatabaseDescriptor::getConcurrentWriters,        DatabaseDescriptor::setConcurrentWriters,        maybeCustomStageExecutor(Stage::multiThreadedLowSignalStage)),
+    COORDINATE_READ   (false, "CoordReadStage",        "request",  DatabaseDescriptor::getConcurrentCoordinatorReaders,        DatabaseDescriptor::setConcurrentCoordinatorReaders,        maybeCustomStageExecutor(Stage::multiThreadedLowSignalStage)),
+    COORDINATE_MUTATION(true, "CoordMutationStage",    "request",  DatabaseDescriptor::getConcurrentCoordinatorWriters,        DatabaseDescriptor::setConcurrentCoordinatorWriters,        maybeCustomStageExecutor(Stage::multiThreadedLowSignalStage)),
+    COUNTER_MUTATION  (true,  "CounterMutationStage",  "request",  DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, maybeCustomStageExecutor(Stage::multiThreadedLowSignalStage)),
+    VIEW_MUTATION     (true,  "ViewMutationStage",     "request",  DatabaseDescriptor::getConcurrentViewWriters,    DatabaseDescriptor::setConcurrentViewWriters,    maybeCustomStageExecutor(Stage::multiThreadedLowSignalStage)),
     GOSSIP            (true,  "GossipStage",           "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
-    REQUEST_RESPONSE  (false, "RequestResponseStage",  "request",  FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedLowSignalStage),
+    REQUEST_RESPONSE  (false, "RequestResponseStage",  "request",  FBUtilities::getAvailableProcessors,             null,                                            maybeCustomStageExecutor(Stage::multiThreadedLowSignalStage)),
     ANTI_ENTROPY      (false, "AntiEntropyStage",      "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
     MIGRATION         (false, "MigrationStage",        "internal", () -> 1,                                         null,                                            Stage::migrationStage),
     MISC              (false, "MiscStage",             "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
@@ -55,7 +73,8 @@ public enum Stage
     INTERNAL_RESPONSE (false, "InternalResponseStage", "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
     IMMEDIATE         (false, "ImmediateStage",        "internal", () -> 0,                                         null,                                            Stage::immediateExecutor),
     PAXOS_REPAIR      (false, "PaxosRepairStage",      "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
-    ;
+    IO                (false, "Internal IO Stage",     "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
+    NATIVE_TRANSPORT_REQUESTS  (false, "Native-Transport-Requests","transport", DatabaseDescriptor::getNativeTransportMaxThreads, DatabaseDescriptor::setNativeTransportMaxThreads, maybeCustomStageExecutor(Stage::multiThreadedLowSignalStage));
 
     public final String jmxName;
     private final Supplier<ExecutorPlus> executorSupplier;
@@ -89,6 +108,15 @@ private static String normalizeName(String stageName)
                                                            .collect(toMap(s -> Stage.normalizeName(s.jmxName),
                                                                           s -> s));
 
+    /**
+     * Set of Stage names that may be used with {@link #fromPoolName(String)}.
+     * Used by CNDB.
+     */
+    public static Set<String> poolNames()
+    {
+        return nameMap.keySet();
+    }
+
     public static Stage fromPoolName(String stageName)
     {
         String upperStageName = normalizeName(stageName);
@@ -121,13 +149,97 @@ public static Stage fromPoolName(String stageName)
         }
     }
 
+    public static Optional<Stage> fromStatement(CQLStatement statement)
+    {
+        if (CassandraRelevantProperties.NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED.getBoolean())
+        {
+            if (statement instanceof SelectStatement)
+            {
+                return Optional.of(Stage.COORDINATE_READ);
+            }
+            else if (statement instanceof ModificationStatement || statement instanceof BatchStatement)
+            {
+                return Optional.of(Stage.COORDINATE_MUTATION);
+            }
+        }
+        return Optional.empty();
+    }
+
     // Convenience functions to execute on this stage
-    public void execute(Runnable task) { executor().execute(task); }
-    public void execute(ExecutorLocals locals, Runnable task) { executor().execute(locals, task); }
-    public void maybeExecuteImmediately(Runnable task) { executor().maybeExecuteImmediately(task); }
-    public <T> Future<T> submit(Callable<T> task) { return executor().submit(task); }
-    public Future<?> submit(Runnable task) { return executor().submit(task); }
-    public <T> Future<T> submit(Runnable task, T result) { return executor().submit(task, result); }
+    public void execute(Runnable command)
+    {
+        if (!CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.isPresent())
+        {
+            executor().execute(command);
+            return;
+        }
+        long enqueueStartTime = Clock.Global.nanoTime();
+        executor().execute(withTimeMeasurement(command, enqueueStartTime));
+    }
+
+    public void execute(Runnable command, ExecutorLocals locals)
+    {
+        if (!CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.isPresent())
+        {
+            executor().execute(locals, command);
+            return;
+        }
+        long enqueueStartTime = Clock.Global.nanoTime();
+        executor().execute(locals, withTimeMeasurement(command, enqueueStartTime));
+    }
+
+    public void maybeExecuteImmediately(Runnable command)
+    {
+        if (!CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.isPresent())
+        {
+            executor().execute(command);
+            return;
+        }
+        long enqueueStartTime = Clock.Global.nanoTime();
+        executor().maybeExecuteImmediately(withTimeMeasurement(command, enqueueStartTime));
+    }
+
+    public <T> Future<T> submit(Callable<T> task)
+    {
+        long enqueueStartTime = Clock.Global.nanoTime();
+        return executor().submit(() -> {
+            try
+            {
+                if (!CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.isPresent())
+                    return task.call();
+                return withTimeMeasurement(task, enqueueStartTime).call();
+            }
+            catch (Exception e)
+            {
+                throw Throwables.unchecked(e);
+            }
+        });
+    }
+
+    public Future<?> submit(Runnable task)
+    {
+        if (!CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.isPresent())
+           return executor().submit(task, executor());
+        long enqueueStartTime = Clock.Global.nanoTime();
+        return executor().submit(withTimeMeasurement(task, enqueueStartTime), executor());
+    }
+
+    public <T> Future<T> submit(Runnable task, T result)
+    {
+
+        if (!CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.isPresent())
+        {
+            return executor().submit(() -> {
+                task.run();
+                return result;
+            });
+        }
+        long enqueueStartTime = Clock.Global.nanoTime();
+        return executor().submit(() -> {
+            withTimeMeasurement(task, enqueueStartTime).run();
+            return result;
+        });
+    }
 
     public ExecutorPlus executor()
     {
@@ -210,6 +322,11 @@ private static ExecutorPlus migrationStage(String jmxName, String jmxType, int n
                .sequential(jmxName);
     }
 
+    private static ExecutorServiceInitialiser maybeCustomStageExecutor(ExecutorServiceInitialiser fallbackInitializer)
+    {
+        return CUSTOM_STAGE_EXECUTOR_FACTORY_PROPERTY.isPresent() ? Stage::customExecutor : fallbackInitializer;
+    }
+
     private static LocalAwareExecutorPlus singleThreadedStage(String jmxName, String jmxType, int numThreads, LocalAwareExecutorPlus.MaximumPoolSizeListener onSetMaximumPoolSize)
     {
         return executorFactory()
@@ -239,6 +356,47 @@ static LocalAwareExecutorPlus immediateExecutor(String jmxName, String jmxType,
         return ImmediateExecutor.INSTANCE;
     }
 
+    static LocalAwareExecutorPlus customExecutor(String jmxName, String jmxType, int numThreads, LocalAwareExecutorPlus.MaximumPoolSizeListener onSetMaximumPoolSize)
+    {
+        StageExecutorFactory customStageExecutorFactory = FBUtilities.construct(CUSTOM_STAGE_EXECUTOR_FACTORY_PROPERTY.getString(), "Custom stage executor factory");
+        return customStageExecutorFactory.init(jmxName, jmxType, numThreads, onSetMaximumPoolSize);
+    }
+
+    public int getPendingTaskCount()
+    {
+        return executor().getPendingTaskCount();
+    }
+
+    public int getActiveTaskCount()
+    {
+        return executor().getActiveTaskCount();
+    }
+
+    public long getCompletedTaskCount()
+    {
+        return executor().getCompletedTaskCount();
+    }
+
+    /**
+     * return additional, executor-related ThreadPoolMetrics for the executor
+     * @param metricsExtractor function to extract metrics from the executor
+     * @return ThreadPoolMetrics or null if the extractor was not able to extract metrics
+     */
+    public @Nullable ThreadPoolMetrics getExecutorMetrics(Function<Executor, ThreadPoolMetrics> metricsExtractor)
+    {
+        return metricsExtractor.apply(executor());
+    }
+
+    public boolean isShutdown()
+    {
+        return executor().isShutdown();
+    }
+
+    public boolean isTerminated()
+    {
+        return executor().isTerminated();
+    }
+
     @FunctionalInterface
     public interface ExecutorServiceInitialiser
     {
@@ -276,4 +434,36 @@ public void setMaximumPoolSize(int newMaximumPoolSize)
     {
         executor().setMaximumPoolSize(newMaximumPoolSize);
     }
+
+    private Runnable withTimeMeasurement(Runnable command, long queueStartTime)
+    {
+        return () -> {
+            long executionStartTime = Clock.Global.nanoTime();
+            try
+            {
+                TaskExecutionCallback.instance.onDequeue(this, executionStartTime - queueStartTime);
+                command.run();
+            }
+            finally
+            {
+                TaskExecutionCallback.instance.onCompleted(this, Clock.Global.nanoTime() - executionStartTime);
+            }
+        };
+    }
+
+    private <T> Callable<T> withTimeMeasurement(Callable<T> command, long queueStartTime)
+    {
+        return () -> {
+            long executionStartTime = Clock.Global.nanoTime();
+            try
+            {
+                TaskExecutionCallback.instance.onDequeue(this, executionStartTime - queueStartTime);
+                return command.call();
+            }
+            finally
+            {
+                TaskExecutionCallback.instance.onCompleted(this, Clock.Global.nanoTime() - executionStartTime);
+            }
+        };
+    }
 }
diff --git a/src/java/org/apache/cassandra/concurrent/StageExecutorFactory.java b/src/java/org/apache/cassandra/concurrent/StageExecutorFactory.java
new file mode 100644
index 000000000000..065c64db2bb1
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/StageExecutorFactory.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+public interface StageExecutorFactory
+{
+    LocalAwareExecutorPlus init(String jmxName, String jmxType, int numThreads, LocalAwareExecutorPlus.MaximumPoolSizeListener onSetMaximumPoolSize);
+}
diff --git a/src/java/org/apache/cassandra/concurrent/TaskExecutionCallback.java b/src/java/org/apache/cassandra/concurrent/TaskExecutionCallback.java
new file mode 100644
index 000000000000..6317987f1615
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/TaskExecutionCallback.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_TASK_EXECUTION_CALLBACK_CLASS;
+
+public interface TaskExecutionCallback
+{
+    TaskExecutionCallback instance = CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.isPresent() ?
+                                        FBUtilities.construct(CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.getString(), "Task execution callback") :
+                                        new NoopTaskExecutionCallback();
+
+    void onCompleted(Stage stage, long executionDurationNanos);
+    void onDequeue(Stage stage, long enqueuedDurationNanos);
+
+    class NoopTaskExecutionCallback implements TaskExecutionCallback
+    {
+        @Override
+        public void onCompleted(Stage stage, long executionDurationNanos)
+        {
+        }
+
+        @Override
+        public void onDequeue(Stage stage, long enqueuedDurationNanos)
+        {
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index 968feae39677..ad1d4ae293f5 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -21,14 +21,22 @@
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-
 import javax.annotation.Nullable;
 
 import com.google.common.primitives.Ints;
 
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.compaction.unified.Reservations;
 import org.apache.cassandra.db.virtual.LogMessagesTable;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.compress.AdaptiveCompressor;
+import org.apache.cassandra.io.compress.LZ4Compressor;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.sensors.SensorsFactory;
 import org.apache.cassandra.service.FileSystemOwnershipCheck;
+import org.apache.cassandra.service.context.OperationContext;
+import org.apache.cassandra.service.reads.range.EndpointGroupingRangeCommandIterator;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.StorageCompatibilityMode;
 
@@ -41,6 +49,7 @@ public enum CassandraRelevantProperties
     ACQUIRE_SLEEP_MS("cassandra.acquire_sleep_ms", "1000"),
     ALLOCATE_TOKENS_FOR_KEYSPACE("cassandra.allocate_tokens_for_keyspace"),
     ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT("cassandra.allow_alter_rf_during_range_movement"),
+    ALLOW_CURSOR_COMPACTION("cassandra.allow_cursor_compaction", "true"),
     /** If we should allow having duplicate keys in the config file, default to true for legacy reasons */
     ALLOW_DUPLICATE_CONFIG_KEYS("cassandra.allow_duplicate_config_keys", "true"),
     /** If we should allow having both new (post CASSANDRA-15234) and old config keys for the same config item in the yaml */
@@ -61,6 +70,42 @@ public enum CassandraRelevantProperties
     AUTO_REPAIR_FREQUENCY_SECONDS("cassandra.auto_repair_frequency_seconds", convertToString(TimeUnit.MINUTES.toSeconds(5))),
     BATCHLOG_REPLAY_TIMEOUT_IN_MS("cassandra.batchlog.replay_timeout_in_ms"),
     BATCH_COMMIT_LOG_SYNC_INTERVAL("cassandra.batch_commitlog_sync_interval_millis", "1000"),
+    /**
+     * A minimal relative change of the fase-positive chance so that it is considered as a reason to recreate the bloom
+     * filter. If the change is smaller than this, it will be ignored.
+     */
+    BF_FP_CHANCE_TOLERANCE("cassandra.bf.fp_chance_tolerance", "0.000001"),
+    /**
+     * The maximum memory to be used by all loaded bloom filters. If the limit is exceeded, pass-through filter will be
+     * used until some filters get unloaded.
+     */
+    BF_MAX_MEMORY_MB("cassandra.bf.max_memory_mb", "0"),
+    /**
+     * If the false-positive chance has changed since the last compaction (for example by alter table statement), and
+     * the node is restarted - the bloom filter can get rebuilt if this property jest set to true.
+     */
+    BF_RECREATE_ON_FP_CHANCE_CHANGE("cassandra.bf.recreate_on_fp_chance_change", "false"),
+
+    // bloom filter lazy loading
+    /**
+     * true if non-local table's bloom filter should be deserialized on read instead of when opening sstable
+     */
+    BLOOM_FILTER_LAZY_LOADING("cassandra.bloom_filter_lazy_loading", "false"),
+
+    /**
+     * sstable primary index hits per second to determine if a sstable is hot. 0 means BF should be loaded immediately on read.
+     *
+     * Note that when WINDOW <= 0, this is used as absolute primary index access count.
+     */
+    BLOOM_FILTER_LAZY_LOADING_THRESHOLD("cassandra.bloom_filter_lazy_loading.threshold", "0"),
+
+    /**
+     * Window of time by minute, available: 1 (default), 5, 15, 120.
+     *
+     * Note that if <= 0 then we use threshold as the absolute count
+     */
+    BLOOM_FILTER_LAZY_LOADING_WINDOW("cassandra.bloom_filter_lazy_loading.window", "1"),
+
     /**
      * When bootstraping how long to wait for schema versions to be seen.
      */
@@ -73,6 +118,7 @@ public enum CassandraRelevantProperties
     BROADCAST_INTERVAL_MS("cassandra.broadcast_interval_ms", "60000"),
     BTREE_BRANCH_SHIFT("cassandra.btree.branchshift", "5"),
     BTREE_FAN_FACTOR("cassandra.btree.fanfactor"),
+    BUFFERPOOL_DISABLE_COMBINED_ALLOCATION("cassandra.bufferpool.disable_combined_allocation"),
     /** Represents the maximum size (in bytes) of a serialized mutation that can be cached **/
     CACHEABLE_MUTATION_SIZE_LIMIT("cassandra.cacheable_mutation_size_limit_bytes", convertToString(1_000_000)),
     CASSANDRA_ALLOW_SIMPLE_STRATEGY("cassandra.allow_simplestrategy"),
@@ -99,17 +145,44 @@ public enum CassandraRelevantProperties
     CASSANDRA_STREAMING_DEBUG_STACKTRACE_LIMIT("cassandra.streaming.debug_stacktrace_limit", "2"),
     CASSANDRA_UNSAFE_TIME_UUID_NODE("cassandra.unsafe.timeuuidnode"),
     CASSANDRA_VERSION("cassandra.version"),
+    CDC_STREAMING_ENABLED("cassandra.cdc.enable_streaming", "true"),
     /** default heartbeating period is 1 minute */
     CHECK_DATA_RESURRECTION_HEARTBEAT_PERIOD("check_data_resurrection_heartbeat_period_milli", "60000"),
     CHRONICLE_ANALYTICS_DISABLE("chronicle.analytics.disable"),
     CHRONICLE_ANNOUNCER_DISABLE("chronicle.announcer.disable"),
+    CHUNKCACHE_ASYNC_CLEANUP("cassandra.chunkcache.async_cleanup", "true"),
+    CHUNKCACHE_CLEANER_THREADS("dse.chunk.cache.cleaner.threads","1"),
+    CHUNKCACHE_INITIAL_CAPACITY("cassandra.chunkcache_initialcapacity", "16"),
+    /**
+     * This property allows configuring the maximum time that CachingRebufferer.rebuffer will wait when waiting for a
+     * CompletableFuture fetched from the cache to complete. This is part of a migitation for DBPE-13261.
+     */
+    CHUNK_CACHE_REBUFFER_WAIT_TIMEOUT_MS("cassandra.chunk_cache_rebuffer_wait_timeout_ms", "30000"),
     CLOCK_GLOBAL("cassandra.clock"),
     CLOCK_MONOTONIC_APPROX("cassandra.monotonic_clock.approx"),
     CLOCK_MONOTONIC_PRECISE("cassandra.monotonic_clock.precise"),
+    /**
+     * The class name of the custom cluster version provider to use.
+     */
+    CLUSTER_VERSION_PROVIDER_CLASS_NAME("cassandra.cluster_version_provider.class_name"),
+
+    /**
+     * Minimum time that needs to pass after the cluster is detected as fully upgraded to report that there is
+     * no upgrade in progress (when using the default cluster version provider).
+     */
+    CLUSTER_VERSION_PROVIDER_MIN_STABLE_DURATION("cassandra.cluster_version_provider.min_stable_duration_ms", "60000"),
+
+    /**
+     * Do not wait for gossip to be enabled before starting stabilisation period. This is required especially for tests
+     * which do not enable gossip at all.
+     */
+    CLUSTER_VERSION_PROVIDER_SKIP_WAIT_FOR_GOSSIP("cassandra.test.cluster_version_provider.skip_wait_for_gossip"),
     COMMITLOG_ALLOW_IGNORE_SYNC_CRC("cassandra.commitlog.allow_ignore_sync_crc"),
     COMMITLOG_IGNORE_REPLAY_ERRORS("cassandra.commitlog.ignorereplayerrors"),
     COMMITLOG_MAX_OUTSTANDING_REPLAY_BYTES("cassandra.commitlog_max_outstanding_replay_bytes", convertToString(1024 * 1024 * 64)),
     COMMITLOG_MAX_OUTSTANDING_REPLAY_COUNT("cassandra.commitlog_max_outstanding_replay_count", "1024"),
+    // Allows skipping advising the OS to free cached pages associated with commitlog flushing
+    COMMITLOG_SKIP_FILE_ADVICE("cassandra.commitlog.skip_file_advice"),
     COMMITLOG_STOP_ON_ERRORS("cassandra.commitlog.stop_on_errors"),
     /**
      * Entities to replay mutations for upon commit log replay, property is meant to contain
@@ -125,6 +198,22 @@ public enum CassandraRelevantProperties
      * If only keyspaces are specified, mutations for all tables in such keyspace will be replayed
      * */
     COMMIT_LOG_REPLAY_LIST("cassandra.replayList"),
+    COMPACTION_HISTORY_ENABLED("cassandra.compaction_history_enabled", "true"),
+    COMPACTION_RATE_LIMIT_GRANULARITY_IN_KB("compaction_rate_limit_granularity_in_kb"),
+
+    /**
+     * If this is true, compaction will not verify that sstables selected for compaction are marked compacted.
+     */
+    COMPACTION_SKIP_COMPACTING_STATE_CHECKING("cassandra.compaction.skip_compacting_state_checking", "false"),
+
+    /**
+     * If this is true, compaction will not verify that sstables selected for compaction are in the same repair
+     * state. This check is done to ensure that incremental repair is not improperly carried out (potentially causing
+     * data loss) if a node has somehow entered an invalid state. The flag should only be used to recover from
+     * situations where sstables are brought in from outside and carry over stale and unapplicable repair state.
+     */
+    COMPACTION_SKIP_REPAIR_STATE_CHECKING("cassandra.compaction.skip_repair_state_checking", "false"),
+
     /**
      * This property indicates the location for the access file. If com.sun.management.jmxremote.authenticate is false,
      * then this property and the password and access files, are ignored. Otherwise, the access file must exist and
@@ -172,19 +261,92 @@ public enum CassandraRelevantProperties
     CONSISTENT_DIRECTORY_LISTINGS("cassandra.consistent_directory_listings"),
     CONSISTENT_RANGE_MOVEMENT("cassandra.consistent.rangemovement", "true"),
     CONSISTENT_SIMULTANEOUS_MOVES_ALLOW("cassandra.consistent.simultaneousmoves.allow"),
+    COUNTER_LOCK_FAIR_LOCK("cassandra.counter_lock.fair_lock", "false"),
+    COUNTER_LOCK_NUM_STRIPES_PER_THREAD("cassandra.counter_lock.num_stripes_per_thread", "1024"),
     CRYPTO_PROVIDER_CLASS_NAME("cassandra.crypto_provider_class_name"),
+    /** Which class to use for coordinator client request metrics */
+    CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY("cassandra.custom_client_request_metrics_provider_class"),
+    /** Which class to use for failure detection */
+    CUSTOM_FAILURE_DETECTOR_PROPERTY("cassandra.custom_failure_detector_class"),
     CUSTOM_GUARDRAILS_CONFIG_PROVIDER_CLASS("cassandra.custom_guardrails_config_provider_class"),
+    CUSTOM_HINTS_ENDPOINT_PROVIDER("cassandra.custom_hints_endpoint_provider"),
+    CUSTOM_HINTS_HANDLER("cassandra.custom_hints_handler"),
+    CUSTOM_HINTS_RATE_LIMITER_FACTORY("cassandra.custom_hints_rate_limiter_factory"),
+    CUSTOM_INDEX_BUILD_DECIDER("cassandra.custom_index_build_decider"),
+    CUSTOM_KEYSPACES_FILTER_PROVIDER("cassandra.custom_keyspaces_filter_provider_class"),
+    /**
+     * Which class to use for messaging metrics for {@link org.apache.cassandra.net.MessagingService}.
+     * The provided class name must point to an implementation of {@link org.apache.cassandra.metrics.MessagingMetrics}.
+     */
+    CUSTOM_MESSAGING_METRICS_PROVIDER_PROPERTY("cassandra.custom_messaging_metrics_provider_class"),
+    /**
+     * Name of a custom implementation of {@link org.apache.cassandra.service.Mutator}.
+     */
+    CUSTOM_MUTATOR_CLASS("cassandra.custom_mutator_class"),
+    /**
+     * custom native library for os access
+     */
+    CUSTOM_NATIVE_LIBRARY("cassandra.custom_native_library"),
     CUSTOM_QUERY_HANDLER_CLASS("cassandra.custom_query_handler_class"),
+    CUSTOM_READ_OBSERVER_FACTORY("cassandra.custom_read_observer_factory_class"),
+    CUSTOM_REPLAY_FILTER_CLASS("cassandra.custom_replay_filter_class"),
+    /**
+     * Allows to set a custom response messages handler for verbs {@link org.apache.cassandra.net.Verb#REQUEST_RSP} and
+     * {@link org.apache.cassandra.net.Verb#FAILURE_RSP}.
+     */
+    CUSTOM_RESPONSE_VERB_HANDLER_PROVIDER("cassandra.custom_response_verb_handler_provider_class"),
+    /** Watcher used when opening sstables to discover extra components, eg. archive component */
+    CUSTOM_SSTABLE_WATCHER("cassandra.custom_sstable_watcher"),
+
+    /**
+     * If provided, this custom factory class will be used to create stage executor for a couple of stages.
+     * @see Stage for details
+     */
+    CUSTOM_STAGE_EXECUTOR_FACTORY_PROPERTY("cassandra.custom_stage_executor_factory_class"),
+    /**
+     * Used to support directory creation for different file system and remote/local conversion
+     */
+    CUSTOM_STORAGE_PROVIDER("cassandra.custom_storage_provider"),
+
+    /**
+     * Which class to use when notifying about stage task execution
+     */
+    CUSTOM_TASK_EXECUTION_CALLBACK_CLASS("cassandra.custom_task_execution_callback_class"),
+
+    /** Which class to use for token metadata provider */
+    CUSTOM_TMD_PROVIDER_PROPERTY("cassandra.custom_token_metadata_provider_class"),
+
     CUSTOM_TRACING_CLASS("cassandra.custom_tracing_class"),
+    /**
+     * If true, while creating or altering schema, NetworkTopologyStrategy won't check if the DC exists.
+     * This is to remain compatible with older workflows that first change the replication before adding the nodes.
+     * Otherwise, it will validate that the names match existing DCs before allowing replication change.
+     */
+    DATACENTER_SKIP_NAME_VALIDATION("cassandra.dc_skip_name_validation", "false"),
     /** Controls the type of bufffer (heap/direct) used for shared scratch buffers */
     DATA_OUTPUT_BUFFER_ALLOCATE_TYPE("cassandra.dob.allocate_type"),
     DATA_OUTPUT_STREAM_PLUS_TEMP_BUFFER_SIZE("cassandra.data_output_stream_plus_temp_buffer_size", "8192"),
     DECAYING_ESTIMATED_HISTOGRAM_RESERVOIR_STRIPE_COUNT("cassandra.dehr_stripe_count", "2"),
+    DEFAULT_COMPACTION_COSTS_READ_MULTIPLIER("default.compaction.costs_read_multiplier"),
+    DEFAULT_COMPACTION_LOGS("default.compaction.logs"),
+    DEFAULT_COMPACTION_LOG_MINUTES("default.compaction.log_minutes"),
+    DEFAULT_INDEX_CLASS("cassandra.default_index_implementation_class"),
     DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES("default.provide.overlapping.tombstones"),
+    /**
+     * Which compression algorithm to use for SSTable compression when not specified explicitly in the sstable options.
+     * Can be "fast", which selects {@link LZ4Compressor}, or "adaptive" which selects {@link AdaptiveCompressor}.
+     */
+    DEFAULT_SSTABLE_COMPRESSION("cassandra.default_sstable_compression", "fast"),
+    // Allow disabling deletions of corrupt index components for troubleshooting
+    DELETE_CORRUPT_SAI_COMPONENTS("cassandra.sai.delete_corrupt_components", "true"),
     /** determinism properties for testing */
     DETERMINISM_SSTABLE_COMPRESSION_DEFAULT("cassandra.sstable_compression_default", "true"),
     DETERMINISM_UNSAFE_UUID_NODE("cassandra.unsafe.deterministicuuidnode"),
     DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS("cassandra.diagnostic_snapshot_interval_nanos", "60000000000"),
+    /**
+     * Whether to disable auto-compaction
+     */
+    DISABLED_AUTO_COMPACTION_PROPERTY("cassandra.disabled_auto_compaction"),
     DISABLE_AUTH_CACHES_REMOTE_CONFIGURATION("cassandra.disable_auth_caches_remote_configuration"),
     /** properties to disable certain behaviours for testing */
     DISABLE_GOSSIP_ENDPOINT_REMOVAL("cassandra.gossip.disable_endpoint_removal"),
@@ -193,6 +355,7 @@ public enum CassandraRelevantProperties
     DISABLE_SSTABLE_ACTIVITY_TRACKING("cassandra.sstable_activity_tracking", "true"),
     DISABLE_STCS_IN_L0("cassandra.disable_stcs_in_l0"),
     DISABLE_TCACTIVE_OPENSSL("cassandra.disable_tcactive_openssl"),
+    DISABLE_UNUSED_CONNECTION_MONITORING("cassandra.messagingService.disableUnusedConnectionMonitoring"),
     /** property for the rate of the scheduled task that monitors disk usage */
     DISK_USAGE_MONITOR_INTERVAL_MS("cassandra.disk_usage.monitor_interval_ms", convertToString(TimeUnit.SECONDS.toMillis(30))),
     /** property for the interval on which the repeated client warnings and diagnostic events about disk usage are ignored */
@@ -204,12 +367,26 @@ public enum CassandraRelevantProperties
      */
     DRAIN_EXECUTOR_TIMEOUT_MS("cassandra.drain_executor_timeout_ms", convertToString(TimeUnit.MINUTES.toMillis(5))),
     DROP_OVERSIZED_READ_REPAIR_MUTATIONS("cassandra.drop_oversized_readrepair_mutations"),
+    /**
+     * The current messaging version. This is used when we add new messaging versions without adopting them immediately,
+     * or to force the node to use a specific version for testing purposes.
+     */
+    DS_CURRENT_MESSAGING_VERSION("ds.current_messaging_version", Integer.toString(MessagingService.VERSION_DS_20)),
     DTEST_API_LOG_TOPOLOGY("cassandra.dtest.api.log.topology"),
     /** This property indicates if the code is running under the in-jvm dtest framework */
     DTEST_IS_IN_JVM_DTEST("org.apache.cassandra.dtest.is_in_jvm_dtest"),
     /** In_JVM dtest property indicating that the test should use "latest" configuration */
     DTEST_JVM_DTESTS_USE_LATEST("jvm_dtests.latest"),
+    // The quantile used by the dynamic endpoint snitch to compute the score for a replica.
+    DYNAMIC_ENDPOINT_SNITCH_QUANTILE("cassandra.dynamic_endpoint_snitch_quantile", "0.5"),
+
+    // whether to quantize the dynamic endpoint snitch score to milliseconds; if set to false the nanosecond measurement
+    // is used
+    DYNAMIC_ENDPOINT_SNITCH_QUANTIZE_TO_MILLIS("cassandra.dynamic_endpoint_snitch_quantize_to_millis", "true"),
+    /** Which class to use for dynamic snitch severity values */
+    DYNAMIC_SNITCH_SEVERITY_PROVIDER("cassandra.dynamic_snitch_severity_provider"),
     ENABLE_DC_LOCAL_COMMIT("cassandra.enable_dc_local_commit", "true"),
+    ENABLE_GUARDRAILS_FOR_ANONYMOUS_USER("cassandra.enable_guardrails_for_anonymous_user", "true"),
     /**
      * Whether {@link org.apache.cassandra.db.ConsistencyLevel#NODE_LOCAL} should be allowed.
      */
@@ -219,9 +396,11 @@ public enum CassandraRelevantProperties
     FAILED_BOOTSTRAP_TIMEOUT("cassandra.failed_bootstrap_timeout_ms"),
     FAILURE_LOGGING_INTERVAL_SECONDS("cassandra.request_failure_log_interval_seconds", "60"),
     FAIL_ON_MISSING_CRYPTO_PROVIDER("cassandra.fail_on_missing_crypto_provider", "false"),
+
     FD_INITIAL_VALUE_MS("cassandra.fd_initial_value_ms"),
     FD_MAX_INTERVAL_MS("cassandra.fd_max_interval_ms"),
     FILE_CACHE_ENABLED("cassandra.file_cache_enabled"),
+    FILE_CACHE_SIZE_IN_MB("cassandra.file_cache_size_in_mb", "2048"),
     /** @deprecated should be removed in favor of enable flag of relevant startup check (FileSystemOwnershipCheck) */
     /** @deprecated See CASSANDRA-17797 */
     @Deprecated(since = "4.1")
@@ -260,7 +439,7 @@ public enum CassandraRelevantProperties
      * Number of polls without gossip state change to consider gossip as settled.
      */
     GOSSIP_SETTLE_POLL_SUCCESSES_REQUIRED("cassandra.gossip_settle_poll_success_required", "3"),
-
+    
     IGNORED_SCHEMA_CHECK_ENDPOINTS("cassandra.skip_schema_check_for_endpoints"),
     IGNORED_SCHEMA_CHECK_VERSIONS("cassandra.skip_schema_check_for_versions"),
     IGNORE_CORRUPTED_SCHEMA_TABLES("cassandra.ignore_corrupted_schema_tables"),
@@ -277,6 +456,11 @@ public enum CassandraRelevantProperties
     /** @deprecated See CASSANDRA-17797 */
     @Deprecated(since = "4.1")
     IGNORE_RACK("cassandra.ignore_rack"),
+    // Allow restoring legacy behavior of deleting sai components before a rebuild (which implies a rebuild cannot be
+    // done without first stopping reads on that index)
+    IMMUTABLE_SAI_COMPONENTS("cassandra.sai.immutable_components", "false"),
+    INDEX_GUARDRAILS_TABLE_FAILURE_THRESHOLD("cassandra.index.guardrails.table_failure_threshold"),
+    INDEX_GUARDRAILS_TOTAL_FAILURE_THRESHOLD("cassandra.index.guardrails.total_failure_threshold"),
     INDEX_SUMMARY_EXPECTED_KEY_SIZE("cassandra.index_summary_expected_key_size", "64"),
     INITIAL_TOKEN("cassandra.initial_token"),
     INTERNODE_EVENT_THREADS("cassandra.internode-event-threads"),
@@ -342,8 +526,14 @@ public enum CassandraRelevantProperties
      * will set it automatically to {@link CassandraRelevantProperties#LOG_DIR} + "/audit".
      */
     LOG_DIR_AUDIT("cassandra.logdir.audit"),
+    /**
+     * Factory to create instances used during log transaction processing
+     */
+    LOG_TRANSACTIONS_FACTORY("cassandra.log_transactions_factory"),
     /** Loosen the definition of "empty" for gossip state, for use during host replacements if things go awry */
     LOOSE_DEF_OF_EMPTY_ENABLED(Config.PROPERTY_PREFIX + "gossiper.loose_empty_enabled"),
+    LWT_MAX_BACKOFF_MS("cassandra.lwt_max_backoff_ms", "50"),
+    LWT_MIN_BACKOFF_MS("cassandra.lwt_min_backoff_ms", "5"),
     MAX_CONCURRENT_RANGE_REQUESTS("cassandra.max_concurrent_range_requests"),
     MAX_HINT_BUFFERS("cassandra.MAX_HINT_BUFFERS", "3"),
     MAX_LOCAL_PAUSE_IN_MS("cassandra.max_local_pause_in_ms", "5000"),
@@ -354,6 +544,7 @@ public enum CassandraRelevantProperties
     MEMTABLE_SHARD_COUNT("cassandra.memtable.shard.count"),
     MEMTABLE_TRIE_SIZE_LIMIT("cassandra.trie_size_limit_mb"),
     MIGRATION_DELAY("cassandra.migration_delay_ms", "60000"),
+    MMAPPED_MAX_SEGMENT_SIZE_IN_MB("cassandra.mmapped_max_segment_size"),
     /** Defines the maximum number of unique timed out queries that will be reported in the logs. Use a negative number to remove any limit. */
     MONITORING_MAX_OPERATIONS("cassandra.monitoring_max_operations", "50"),
     /** Defines the interval for reporting any operations that have timed out. */
@@ -366,15 +557,30 @@ public enum CassandraRelevantProperties
     MX4JPORT("mx4jport"),
     NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL("cassandra.NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL", "10000"),
     NATIVE_EPOLL_ENABLED("cassandra.native.epoll.enabled", "true"),
+    /**
+     * If true, makes read and write CQL statements async, splitting them from the {@link org.apache.cassandra.concurrent.Stage#NATIVE_TRANSPORT_REQUESTS}
+     * stage and into the {@link org.apache.cassandra.concurrent.Stage#COORDINATE_READ} and {@link org.apache.cassandra.concurrent.Stage#COORDINATE_MUTATION}
+     * stages respectively; in other words, the native transport stage will not block, offloading request processing
+     * (and any related blocking behaviour) to the specific read and write stages.
+     */
+    NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED("cassandra.transport.async.read_write", "false"),
     /** This is the port used with RPC address for the native protocol to communicate with clients. Now that thrift RPC is no longer in use there is no RPC port. */
     NATIVE_TRANSPORT_PORT("cassandra.native_transport_port"),
     NEVER_PURGE_TOMBSTONES("cassandra.never_purge_tombstones"),
     NIO_DATA_OUTPUT_STREAM_PLUS_BUFFER_SIZE("cassandra.nio_data_output_stream_plus_buffer_size", convertToString(32 * 1024)),
+    NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE("cassandra.nodes.disablePersitingToSystemKeyspace", "false"),
+    NODES_PERSISTENCE_CLASS("cassandra.nodes.persistence_class"),
     NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS("cassandra.nodetool.jmx_notification_poll_interval_seconds", convertToString(TimeUnit.SECONDS.convert(5, TimeUnit.MINUTES))),
     /** If set, {@link org.apache.cassandra.net.MessagingService} is shutdown abrtuptly without waiting for anything.
      * This is an optimization used in unit tests becuase we never restart a node there. The only node is stopoped
      * when the JVM terminates. Therefore, we can use such optimization and not wait unnecessarily. */
+    NON_GRACEFUL_CLOSE("cassandra.messagingService.nonGracefulClose"),
     NON_GRACEFUL_SHUTDOWN("cassandra.test.messagingService.nonGracefulShutdown"),
+    /**
+     * Allows custom implementation of {@link OperationContext.Factory} to optionally create and configure custom
+     * {@link OperationContext} instances.
+     */
+    OPERATION_CONTEXT_FACTORY("cassandra.operation_context_factory_class"),
     /** for specific tests */
     /** This property indicates whether disable_mbean_registration is true */
     ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION("org.apache.cassandra.disable_mbean_registration"),
@@ -386,6 +592,7 @@ public enum CassandraRelevantProperties
     /** Enabled/disable TCP_NODELAY for intradc connections. Defaults is enabled. */
     OTC_INTRADC_TCP_NODELAY("cassandra.otc_intradc_tcp_nodelay", "true"),
     OVERRIDE_DECOMMISSION("cassandra.override_decommission"),
+    PARALLEL_INDEX_READ_NUM_THREADS("cassandra.index_read.parallel_thread_num"),
     PARENT_REPAIR_STATUS_CACHE_SIZE("cassandra.parent_repair_status_cache_size", "100000"),
     PARENT_REPAIR_STATUS_EXPIRY_SECONDS("cassandra.parent_repair_status_expiry_seconds", convertToString(TimeUnit.SECONDS.convert(1, TimeUnit.DAYS))),
     PARTITIONER("cassandra.partitioner"),
@@ -398,13 +605,34 @@ public enum CassandraRelevantProperties
     PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_RETRY_DELAY_SECONDS("cassandra.paxos_repair_on_topology_change_retry_delay_seconds", "10"),
     PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS("cassandra.paxos_repair_retry_timeout_millis", "60000"),
     PAXOS_USE_SELF_EXECUTION("cassandra.paxos.use_self_execution", "true"),
+    /**
+     * whether to persist prepared statements in the system table
+     */
+    PERSIST_PREPARED_STATEMENTS("cassandra.persist_prepared_statements", "true"),
     PRINT_HEAP_HISTOGRAM_ON_OUT_OF_MEMORY_ERROR("cassandra.printHeapHistogramOnOutOfMemoryError"),
+    /**
+     * Whether to enable the use of {@link EndpointGroupingRangeCommandIterator}
+     */
+    RANGE_READ_ENDPOINT_GROUPING_ENABLED("cassandra.range_read_endpoint_grouping_enabled", "true"),
     READS_THRESHOLDS_COORDINATOR_DEFENSIVE_CHECKS_ENABLED("cassandra.reads.thresholds.coordinator.defensive_checks_enabled"),
     RELEASE_VERSION("cassandra.releaseVersion"),
+    RELOCATED_SHADED_IO_NETTY_TRANSPORT_NONATIVE("relocated.shaded.io.netty.transport.noNative"),
+    /**
+     * The factory for handler of the storage of sstables
+     */
+    REMOTE_STORAGE_HANDLER_FACTORY("cassandra.remote_storage_handler_factory"),
     REPAIR_CLEANUP_INTERVAL_SECONDS("cassandra.repair_cleanup_interval_seconds", convertToString(Ints.checkedCast(TimeUnit.MINUTES.toSeconds(10)))),
     REPAIR_DELETE_TIMEOUT_SECONDS("cassandra.repair_delete_timeout_seconds", convertToString(Ints.checkedCast(TimeUnit.DAYS.toSeconds(1)))),
     REPAIR_FAIL_TIMEOUT_SECONDS("cassandra.repair_fail_timeout_seconds", convertToString(Ints.checkedCast(TimeUnit.DAYS.toSeconds(1)))),
     REPAIR_MUTATION_REPAIR_ROWS_PER_BATCH("cassandra.repair.mutation_repair_rows_per_batch", "100"),
+    /**
+     * Listen to repair parent session lifecycle
+     */
+    REPAIR_PARENT_SESSION_LISTENER("cassandra.custom_parent_repair_session_listener_class"),
+    /**
+     * Repair progress reporter, default using system distributed keyspace
+     */
+    REPAIR_PROGRESS_REPORTER("cassandra.custom_repair_progress_reporter_class"),
     REPAIR_STATUS_CHECK_TIMEOUT_SECONDS("cassandra.repair_status_check_timeout_seconds", convertToString(Ints.checkedCast(TimeUnit.HOURS.toSeconds(1)))),
     /**
      * When doing a host replacement its possible that the gossip state is "empty" meaning that the endpoint is known
@@ -416,6 +644,10 @@ public enum CassandraRelevantProperties
     REPLACE_ADDRESS_FIRST_BOOT("cassandra.replace_address_first_boot"),
     REPLACE_NODE("cassandra.replace_node"),
     REPLACE_TOKEN("cassandra.replace_token"),
+    /**
+     * Number of replicas required to store batchlog for atomicity, only accepts values of 1 or 2.
+     */
+    REQUIRED_BATCHLOG_REPLICA_COUNT("cassandra.batchlog.required_replica_count", "2"),
     /**
      * Whether we reset any found data from previously run bootstraps.
      */
@@ -424,65 +656,89 @@ public enum CassandraRelevantProperties
 
     // SAI specific properties
 
-    /** Controls the maximum number of index query intersections that will take part in a query */
-    SAI_INTERSECTION_CLAUSE_LIMIT("cassandra.sai.intersection_clause_limit", "2"),
-    /** Latest version to be used for SAI index writing */
-    SAI_LATEST_VERSION("cassandra.sai.latest_version", "aa"),
+    /** Class used to discover/load the proper SAI index components file for a given sstable. */
+    SAI_ANN_USE_SYNTHETIC_SCORE("cassandra.sai.ann_use_synthetic_score", "false"),
+    SAI_CUSTOM_COMPONENTS_DISCOVERY_CLASS("cassandra.sai.custom_components_discovery_class"),
+    SAI_ENABLE_EDGES_CACHE("cassandra.sai.enable_edges_cache", "false"),
+    SAI_ENABLE_GENERAL_ORDER_BY("cassandra.sai.general_order_by", "true"),
+    SAI_ENABLE_JVECTOR_DELETES("cassandra.sai.enable_jvector_deletes", "true"),
+    SAI_ENABLE_LTM_CONSTRUCTION("cassandra.sai.ltm_construction", "true"),
+    SAI_ENABLE_RERANK_FLOOR("cassandra.sai.rerank_floor", "true"),
 
-    /** Minimum number of reachable leaves for a given node to be eligible for an auxiliary posting list */
-    SAI_MINIMUM_POSTINGS_LEAVES("cassandra.sai.minimum_postings_leaves", "64"),
+    /** Controls the hnsw vector cache size, in bytes, per index segment. 0 to disable */
+    SAI_HNSW_VECTOR_CACHE_BYTES("cassandra.sai.vector_search.vector_cache_bytes", String.valueOf(4 * 1024 * 1024)),
 
     /**
-     * Skip, or the sampling interval, for selecting a balanced tree level that is eligible for an auxiliary posting list.
-     * Sampling starts from 0, but balanced tree root node is at level 1. For skip = 4, eligible levels are 4, 8, 12, etc. (no
-     * level 0, because there is no node at level 0).
+     * If true, the searcher object created when opening a SAI index will be replaced by a dummy object and index
+     * are never marked queriable (querying one will fail). This is obviously usually undesirable, but can be used if
+     * the node only compact sstables to avoid loading heavy index data structures in memory that are not used.
      */
-    SAI_POSTINGS_SKIP("cassandra.sai.postings_skip", "3"),
+    SAI_INDEX_READS_DISABLED("cassandra.sai.disabled_reads", "false"),
 
-    /**
-     * Used to determine the block size and block mask for the clustering sorted terms.
-     */
-    SAI_SORTED_TERMS_CLUSTERING_BLOCK_SHIFT("cassandra.sai.sorted_terms_clustering_block_shift", "4"),
+    /** Controls the maximum number of index query intersections that will take part in a query */
+    SAI_INTERSECTION_CLAUSE_LIMIT("cassandra.sai.intersection_clause_limit", "2"),
 
-    /**
-     * Used to determine the block size and block mask for the partition sorted terms.
-     */
-    SAI_SORTED_TERMS_PARTITION_BLOCK_SHIFT("cassandra.sai.sorted_terms_partition_block_shift", "4"),
+    SAI_JVECTOR_VERSION("cassandra.sai.jvector_version", "4"),
 
-    SAI_TEST_BALANCED_TREE_DEBUG_ENABLED("cassandra.sai.test.balanced_tree_debug_enabled", "false"),
-    SAI_TEST_DISABLE_TIMEOUT("cassandra.sai.test.timeout_disabled", "false"),
+    /** Latest version to be used for SAI index writing */
+    SAI_LATEST_VERSION("cassandra.sai.latest_version", "dc"),
 
-    /** Whether to allow the user to specify custom options to the hnsw index */
-    SAI_VECTOR_ALLOW_CUSTOM_PARAMETERS("cassandra.sai.vector.allow_custom_parameters", "false"),
+    SAI_MAX_ANALYZED_SIZE("cassandra.sai.max_analyzed_size_kb", "8"),
+    SAI_MAX_FROZEN_TERM_SIZE("cassandra.sai.max_frozen_term_size_kb", "8"),
+    SAI_MAX_STRING_TERM_SIZE("cassandra.sai.max_string_term_size_kb", "8"),
+    SAI_MAX_VECTOR_TERM_SIZE("cassandra.sai.max_vector_term_size_kb", "16"),
 
+    SAI_NUMERIC_VALUES_BLOCK_SIZE("dse.sai.numeric_values.block_size", "128"),
+    SAI_NUMERIC_VALUES_MONOTONIC_BLOCK_SIZE("dse.sai.numeric_values.monotonic_block_size", "16384"),
+    SAI_QUERY_OPT_LEVEL("cassandra.sai.query.optimization.level", "1"),
+    SAI_REDUCE_TOPK_ACROSS_SSTABLES("cassandra.sai.reduce_topk_across_sstables", "true"),
+    SAI_TEST_DISABLE_TIMEOUT("cassandra.sai.test.timeout_disabled", "false"),
+    SAI_TEST_LAST_VALID_SEGMENTS("cassandra.sai.test_last_valid_segments", "-1"),
+    SAI_TEST_SEGMENT_BUILD_MEMORY_LIMIT("cassandra.test.sai.segment_build_memory_limit"),
+    SAI_VALIDATE_MAX_TERM_SIZE_AT_COORDINATOR("cassandra.sai.validate_max_term_size_at_coordinator"),
+    /** Whether to validate terms that will be SAI indexed at the coordinator */
+    SAI_VALIDATE_TERMS_AT_COORDINATOR("cassandra.sai.validate_terms_at_coordinator", "true"),
     /** Controls the maximum top-k limit for vector search */
     SAI_VECTOR_SEARCH_MAX_TOP_K("cassandra.sai.vector_search.max_top_k", "1000"),
-
-    /**
-     * Controls the maximum number of PrimaryKeys that will be read into memory at one time when ordering/limiting
-     * the results of an ANN query constrained by non-ANN predicates.
-     */
-    SAI_VECTOR_SEARCH_ORDER_CHUNK_SIZE("cassandra.sai.vector_search.order_chunk_size", "100000"),
+    SAI_WRITE_JVECTOR3_FORMAT("cassandra.sai.write_jv3_format", "false"),
 
     SCHEMA_PULL_INTERVAL_MS("cassandra.schema_pull_interval_ms", "60000"),
     SCHEMA_UPDATE_HANDLER_FACTORY_CLASS("cassandra.schema.update_handler_factory.class"),
     SEARCH_CONCURRENCY_FACTOR("cassandra.search_concurrency_factor", "1"),
 
-    /**
+     /**
      * The maximum number of seeds returned by a seed provider before emmitting a warning.
      * A large seed list may impact effectiveness of the third gossip round.
      * The default used in SimpleSeedProvider is 20.
      */
     SEED_COUNT_WARN_THRESHOLD("cassandra.seed_count_warn_threshold"),
+    /**
+     * Allows custom implementation of {@link SensorsFactory} to optionally create
+     * and configure {@link org.apache.cassandra.sensors.RequestSensors} instances.
+     */
+    SENSORS_FACTORY("cassandra.sensors_factory_class"),
+    /**
+     * If true, the coordinator will propagate sensors via the native protocol custom payload bytes map.
+     */
+    SENSORS_VIA_NATIVE_PROTOCOL("cassandra.sensors_via_native_protocol", "false"),
     SERIALIZATION_EMPTY_TYPE_NONEMPTY_BEHAVIOR("cassandra.serialization.emptytype.nonempty_behavior"),
     SET_SEP_THREAD_NAME("cassandra.set_sep_thread_name", "true"),
     SHUTDOWN_ANNOUNCE_DELAY_IN_MS("cassandra.shutdown_announce_in_ms", "2000"),
     SIZE_RECORDER_INTERVAL("cassandra.size_recorder_interval", "300"),
+    SKIP_DEFAULT_ROLE_SETUP("cassandra.skip_default_role_setup"),
+    SKIP_MUTATING_STATS_AFTER_ZCS("cassandra.skip_mutating_stats_after_zcs"),
+    /**
+     * Do not try to calculate optimal streaming candidates. This can take a lot of time in some configs specially
+     * with vnodes.
+     */
+    SKIP_OPTIMAL_STREAMING_CANDIDATES_CALCULATION("cassandra.skip_optimal_streaming_candidates_calculation", "false"),
     SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE("cassandra.skip_paxos_repair_on_topology_change"),
     /** If necessary for operational purposes, permit certain keyspaces to be ignored for paxos topology repairs. */
     SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES("cassandra.skip_paxos_repair_on_topology_change_keyspaces"),
     SKIP_PAXOS_REPAIR_VERSION_VALIDATION("cassandra.skip_paxos_repair_version_validation"),
     SKIP_PAXOS_STATE_REBUILD("cassandra.skip_paxos_state_rebuild"),
+    /** Whether to skip rewriting hints when original host id left the cluster */
+    SKIP_REWRITING_HINTS_ON_HOST_LEFT("cassandra.hinted_handoff.skip_rewriting_hints_on_host_left"),
     /** snapshots ttl cleanup initial delay in seconds */
     SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS("cassandra.snapshot.ttl_cleanup_initial_delay_seconds", "5"),
     /** snapshots ttl cleanup period in seconds */
@@ -492,6 +748,11 @@ public enum CassandraRelevantProperties
     SSL_ENABLE("ssl.enable"),
     SSL_STORAGE_PORT("cassandra.ssl_storage_port"),
     SSTABLE_FORMAT_DEFAULT("cassandra.sstable.format.default"),
+
+    // in OSS, when UUID based SSTable generation identifiers are enabled, they use TimeUUID
+    // though, for CNDB we want to use ULID - this property allows for that
+    // valid values for this property are: uuid, ulid
+    SSTABLE_UUID_IMPL("cassandra.sstable.id.uuid_impl", "uuid"),
     START_GOSSIP("cassandra.start_gossip", "true"),
     START_NATIVE_TRANSPORT("cassandra.start_native_transport"),
     STORAGE_DIR("cassandra.storagedir"),
@@ -503,6 +764,7 @@ public enum CassandraRelevantProperties
     /** Platform word size sun.arch.data.model. Examples: "32", "64", "unknown"*/
     SUN_ARCH_DATA_MODEL("sun.arch.data.model"),
     SUN_JAVA_COMMAND("sun.java.command", ""),
+    SUN_NIO_PAGE_ALIGN_DIRECT_MEMORY("sun.nio.PageAlignDirectMemory"),
     /**
      * Controls the JMX server threadpool keap-alive time.
      * Should only be set by in-jvm dtests.
@@ -511,9 +773,22 @@ public enum CassandraRelevantProperties
     SUN_STDERR_ENCODING("sun.stderr.encoding"),
     SUN_STDOUT_ENCODING("sun.stdout.encoding"),
     SUPERUSER_SETUP_DELAY_MS("cassandra.superuser_setup_delay_ms", "10000"),
+    SYNC_LAG_FACTOR("cassandra.commitlog_sync_block_lag_factor", "1.5"),
     SYSTEM_AUTH_DEFAULT_RF("cassandra.system_auth.default_rf", "1"),
     SYSTEM_DISTRIBUTED_DEFAULT_RF("cassandra.system_distributed.default_rf", "3"),
+    SYSTEM_DISTRIBUTED_NTS_DC_OVERRIDE_PROPERTY("cassandra.system_distributed_replication_dc_names"),
+    SYSTEM_DISTRIBUTED_NTS_RF_OVERRIDE_PROPERTY("cassandra.system_distributed_replication_per_dc"),
     SYSTEM_TRACES_DEFAULT_RF("cassandra.system_traces.default_rf", "2"),
+    // Allows admin to include only some system views (see two below)
+    SYSTEM_VIEWS_INCLUDE_ALL("cassandra.system_view.include_all", "true"),
+    //This only applies if include all is false
+    SYSTEM_VIEWS_INCLUDE_INDEXES("cassandra.system_view.include_indexes"),
+    // Default metric aggegration strategy for tables without aggregation explicitly set.
+    TABLE_METRICS_DEFAULT_HISTOGRAMS_AGGREGATION("cassandra.table_metrics_default_histograms_aggregation", TableMetrics.MetricsAggregation.INDIVIDUAL.name()),
+    // Determines if table metrics should be also exported to shared global metric
+    TABLE_METRICS_EXPORT_GLOBALS("cassandra.table_metrics_export_globals", "true"),
+
+    TEST_ALLOW_LOCAL_STRATEGY("test.allow.local_strategy"),
     TEST_BBFAILHELPER_ENABLED("test.bbfailhelper.enabled"),
     TEST_BLOB_SHARED_SEED("cassandra.test.blob.shared.seed"),
     TEST_BYTEMAN_TRANSFORMATIONS_DEBUG("cassandra.test.byteman.transformations.debug"),
@@ -530,12 +805,8 @@ public enum CassandraRelevantProperties
     TEST_DRIVER_READ_TIMEOUT_MS("cassandra.test.driver.read_timeout_ms", "12000"),
     TEST_ENCRYPTION("cassandra.test.encryption", "false"),
     TEST_FAIL_MV_LOCKS_COUNT("cassandra.test.fail_mv_locks_count", "0"),
+    TEST_FAIL_ON_FORBIDDEN_LOG_ENTRIES("cassandra.test.fail_on_forbidden_log_entries", "false"),
     TEST_FAIL_WRITES_KS("cassandra.test.fail_writes_ks", ""),
-    /** Flush changes of {@link org.apache.cassandra.schema.SchemaKeyspace} after each schema modification. In production,
-     * we always do that. However, tests which do not restart nodes may disable this functionality in order to run
-     * faster. Note that this is disabled for unit tests but if an individual test requires schema to be flushed, it
-     * can be also done manually for that particular case: {@code flush(SchemaConstants.SCHEMA_KEYSPACE_NAME);}. */
-    TEST_FLUSH_LOCAL_SCHEMA_CHANGES("cassandra.test.flush_local_schema_changes", "true"),
     TEST_IGNORE_SIGAR("cassandra.test.ignore_sigar"),
     TEST_INVALID_LEGACY_SSTABLE_ROOT("invalid-legacy-sstable-root"),
     TEST_JVM_DTEST_DISABLE_SSL("cassandra.test.disable_ssl"),
@@ -545,6 +816,7 @@ public enum CassandraRelevantProperties
     TEST_READ_ITERATION_DELAY_MS("cassandra.test.read_iteration_delay_ms", "0"),
     TEST_REUSE_PREPARED("cassandra.test.reuse_prepared", "true"),
     TEST_ROW_CACHE_SIZE("cassandra.test.row_cache_size"),
+    TEST_SAI_DISABLE_TIMEOUT("cassandra.sai.test.disable.timeout", "false"),
     TEST_SERIALIZATION_WRITES("cassandra.test-serialization-writes"),
     TEST_SIMULATOR_DEBUG("cassandra.test.simulator.debug"),
     TEST_SIMULATOR_DETERMINISM_CHECK("cassandra.test.simulator.determinismcheck", "none"),
@@ -564,7 +836,7 @@ public enum CassandraRelevantProperties
      *
      * This is a dev/CI only property. Do not use otherwise.
      */
-    TEST_STORAGE_COMPATIBILITY_MODE("cassandra.test.storage_compatibility_mode", StorageCompatibilityMode.CASSANDRA_4.toString()),
+    TEST_STORAGE_COMPATIBILITY_MODE("cassandra.test.storage_compatibility_mode", StorageCompatibilityMode.NONE.toString()),
     TEST_STRICT_LCS_CHECKS("cassandra.test.strict_lcs_checks"),
     /** Turns some warnings into exceptions for testing. */
     TEST_STRICT_RUNTIME_CHECKS("cassandra.strict.runtime.checks"),
@@ -574,25 +846,91 @@ public enum CassandraRelevantProperties
     TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST("org.apache.cassandra.tools.UtilALLOW_TOOL_REINIT_FOR_TEST"),
     /** Activate write survey mode. The node not becoming an active ring member, and you must use JMX StorageService->joinRing() to finalize the ring joining. */
     TEST_WRITE_SURVEY("cassandra.write_survey"),
+    THREAD_MONITOR_AUTO_CALIBRATE("dse.thread_monitor_auto_calibrate", "true"),
+    THREAD_MONITOR_ENABLED("dse.thread_monitor_enabled", "true"),
+    THREAD_MONITOR_SLEEP_INTERVAL_NS("dse.thread_monitor_sleep_nanos", "50000"),
+    // Changes the semantic of the "THREE" consistency level to mean "all but one"
+    // i.e. that all replicas except for at most one in the cluster (across all DCs) must accept the write for it to be successful.
+    THREE_MEANS_ALL_BUT_ONE("dse.consistency_level.three_means_all_but_one", "false"),
     TOLERATE_SSTABLE_SIZE("cassandra.tolerate_sstable_size"),
+    /**
+     * Allows to set custom current trie index format. This node will produce sstables in this format.
+     */
+    TRIE_INDEX_FORMAT_VERSION("cassandra.trie_index_format_version", "cc"),
     TRIGGERS_DIR("cassandra.triggers_dir"),
     TRUNCATE_BALLOT_METADATA("cassandra.truncate_ballot_metadata"),
+    /**
+     * To provide a provider to a different implementation of the truncate statement.
+     */
+    TRUNCATE_STATEMENT_PROVIDER("cassandra.truncate_statement_provider"),
     TYPE_UDT_CONFLICT_BEHAVIOR("cassandra.type.udt.conflict_behavior"),
     // See org.apache.cassandra.db.compaction.unified.Controller for the definition of the UCS parameters
+    UCS_ADAPTIVE_COSTS_READ_MULTIPLIER("unified_compaction.costs_read_multiplier", "0.1"),
+    UCS_ADAPTIVE_COSTS_WRITE_MULTIPLIER("unified_compaction.costs_write_multiplier", "1"),
+    UCS_ADAPTIVE_ENABLED("unified_compaction.adaptive", "false"),
+    UCS_ADAPTIVE_INTERVAL_SEC("unified_compaction.adaptive_interval_sec", "300"),
+    UCS_ADAPTIVE_MAX_SCALING_PARAMETER("unified_compaction.adaptive_max_scaling_parameter", "36"),
+    UCS_ADAPTIVE_MIN_COST("unified_compaction.adaptive_min_cost", "1000"),
+    UCS_ADAPTIVE_MIN_SCALING_PARAMETER("unified_compaction.adaptive_min_scaling_parameter", "-10"),
+    UCS_ADAPTIVE_SAMPLE_TIME_MS("unified_compaction.sample_time_ms", "5000"),
+    UCS_ADAPTIVE_STARTING_SCALING_PARAMETER("unified_compaction.adaptive_starting_scaling_parameter", "0"),
+    UCS_ADAPTIVE_THRESHOLD("unified_compaction.adaptive_threshold", "0.15"),
     UCS_BASE_SHARD_COUNT("unified_compaction.base_shard_count", "4"),
+
+    /**
+     * To provide custom implementation to prioritize compaction tasks in UCS
+     */
+    UCS_COMPACTION_AGGREGATE_PRIORITIZER("unified_compaction.custom_compaction_aggregate_prioritizer"),
+    /**
+     * whether to include non-data files size into compaction space estimaton in UCS
+     */
+    UCS_COMPACTION_INCLUDE_NON_DATA_FILES_SIZE("unified_compaction.include_non_data_files_size", "true"),
+    UCS_DATASET_SIZE("unified_compaction.dataset_size"),
+    UCS_IS_REPLICA_AWARE("unified_compaction.is_replica_aware"),
+    UCS_L0_SHARDS_ENABLED("unified_compaction.l0_shards_enabled", "true"),
+    UCS_MAX_ADAPTIVE_COMPACTIONS("unified_compaction.max_adaptive_compactions", "5"),
+    UCS_MAX_SPACE_OVERHEAD("unified_compaction.max_space_overhead", "0.2"),
     UCS_MIN_SSTABLE_SIZE("unified_compaction.min_sstable_size", "100MiB"),
+    UCS_NUM_SHARDS("unified_compaction.num_shards"),
     UCS_OVERLAP_INCLUSION_METHOD("unified_compaction.overlap_inclusion_method"),
-    UCS_SCALING_PARAMETER("unified_compaction.scaling_parameters", "T4"),
+    UCS_OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES("unified_compaction.override_ucs_config_for_vector_tables", "false"),
+    UCS_PARALLELIZE_OUTPUT_SHARDS("unified_compaction.parallelize_output_shards", "true"),
+    UCS_RESERVATIONS_TYPE_OPTION("unified_compaction.reservations_type_option", Reservations.Type.LEVEL_OR_BELOW.name()),
+    UCS_RESERVED_THREADS("reserved_threads", "max"),
+    UCS_SHARED_STORAGE("unified_compaction.shared_storage", "false"),
     UCS_SSTABLE_GROWTH("unified_compaction.sstable_growth", "0.333"),
+    UCS_STATIC_SCALING_PARAMETERS("unified_compaction.scaling_parameters", "T4"),
     UCS_SURVIVAL_FACTOR("unified_compaction.survival_factor", "1"),
     UCS_TARGET_SSTABLE_SIZE("unified_compaction.target_sstable_size", "1GiB"),
+    UCS_VECTOR_BASE_SHARD_COUNT("unified_compaction.vector_base_shard_count", "1"),
+    UCS_VECTOR_MIN_SSTABLE_SIZE("unified_compaction.vector_min_sstable_size", "1024MiB"),
+    UCS_VECTOR_RESERVED_THREADS("unified_compaction.vector_reserved_threads", "max"),
+    UCS_VECTOR_SCALING_PARAMETERS("unified_compaction.vector_scaling_parameters", "L10"),
+    UCS_VECTOR_SSTABLE_GROWTH("unified_compaction.vector_sstable_growth", "1.0"),
+    UCS_VECTOR_TARGET_SSTABLE_SIZE("unified_compaction.vector_target_sstable_size", "5GiB"),
     UDF_EXECUTOR_THREAD_KEEPALIVE_MS("cassandra.udf_executor_thread_keepalive_ms", "30000"),
-    UNSAFE_SYSTEM("cassandra.unsafesystem"),
+    /** Disables flush changes to local and schema keyspaces. Also, disables recycling all segments of commitlog after
+     * dropping a table. Tests which do not restart nodes may enable this option in order to run faster. Note that this
+     * is enabled for unit tests but if an individual test requires schema to be flushed, it can be also done manually
+     * for that particular case: {@code flush(SchemaConstants.SCHEMA_KEYSPACE_NAME);}. */
+    UNSAFE_SYSTEM("cassandra.unsafesystem", "false"),
     /** User's home directory. */
     USER_HOME("user.home"),
+    /** Set this property to true in order to use DSE-like histogram bucket boundaries and behaviour */
+    USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES("cassandra.use_dse_compatible_histogram_boundaries", "false"),
+    /** Set this property to true in order to switch to micrometer metrics */
+    USE_MICROMETER("cassandra.use_micrometer_metrics", "false"),
     /** When enabled, recursive directory deletion will be executed using a unix command `rm -rf` instead of traversing
      * and removing individual files. This is now used only tests, but eventually we will make it true by default.*/
     USE_NIX_RECURSIVE_DELETE("cassandra.use_nix_recursive_delete"),
+    // Enables parallel index read.
+    USE_PARALLEL_INDEX_READ("cassandra.index_read.parallel", "true"),
+    USE_RANDOM_ALLOCATION_IF_NOT_SUPPORTED("cassandra.token_allocation.use_random_if_not_supported"),
+    USE_STRIPED_COUNTER_LOCK_MANAGER("cassandra.use_striped_counter_lock_manager"),
+    /** Whether vector type only allows float vectors. True by default. **/
+    VECTOR_FLOAT_ONLY("cassandra.float_only_vectors", "true"),
+    /** Enables use of vector type. True by default. **/
+    VECTOR_TYPE_ALLOWED("cassandra.vector_type_allowed", "true"),
     /** Gossiper compute expiration timeout. Default value 3 days. */
     VERY_LONG_TIME_MS("cassandra.very_long_time_ms", "259200000"),
     WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS("cassandra.wait_for_tracing_events_timeout_secs", "0");
@@ -607,7 +945,7 @@ public enum CassandraRelevantProperties
             if (!visited.add(next.getKey()))
                 throw new IllegalStateException("System properties have duplicate key: " + next.getKey());
             if (prev != null && next.name().compareTo(prev.name()) < 0)
-                throw new IllegalStateException("Enum constants are not in alphabetical order: " + prev.name() + " should come before " + next.name());
+                throw new IllegalStateException("Enum constants are not in alphabetical order: " + prev.name() + " should come after " + next.name());
             else
                 prev = next;
         }
@@ -625,6 +963,9 @@ public enum CassandraRelevantProperties
         this.defaultVal = null;
     }
 
+    public static final String CASSANDRA_PREFIX = "cassandra.";
+    public static final String LEGACY_PREFIX = "dse.";
+
     private final String key;
     private final String defaultVal;
 
@@ -633,6 +974,17 @@ public String getKey()
         return key;
     }
 
+    /**
+     * Returns the key with the legacy prefix. If the key starts with "cassandra.", it will be replaced with "dse.".
+     * Otherwise, "dse." will be prepended to the key.
+     */
+    public String getKeyWithLegacyPrefix()
+    {
+        if (key.startsWith(CASSANDRA_PREFIX))
+            return LEGACY_PREFIX + key.substring(CASSANDRA_PREFIX.length());
+        return LEGACY_PREFIX + key;
+    }
+
     /**
      * Gets the value of the indicated system property.
      * @return system property value if it exists, defaultValue otherwise.
@@ -644,6 +996,20 @@ public String getString()
         return value == null ? defaultVal : STRING_CONVERTER.convert(value);
     }
 
+    /**
+     * Gets the value of the indicated system property.
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the default value.
+     * @return system property value if it exists, defaultValue otherwise.
+     */
+    public String getStringWithLegacyFallback()
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        return value == null ? defaultVal : STRING_CONVERTER.convert(value);
+    }
+
     /**
      * Returns default value.
      *
@@ -678,6 +1044,23 @@ public String getString(String overrideDefaultValue)
         return STRING_CONVERTER.convert(value);
     }
 
+    /**
+     * Gets the value of a system property as a String.
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the overrideDefaultValue value.
+     * @return system property String value if it exists, overrideDefaultValue otherwise.
+     */
+    public String getStringWithLegacyFallback(String overrideDefaultValue)
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        if (value == null)
+            return overrideDefaultValue;
+
+        return STRING_CONVERTER.convert(value);
+    }
+
     public <T> T convert(PropertyConverter<T> converter)
     {
         String value = System.getProperty(key);
@@ -726,6 +1109,20 @@ public boolean getBoolean()
         return BOOLEAN_CONVERTER.convert(value == null ? defaultVal : value);
     }
 
+    /**
+     * Gets the value of a system property as a boolean.
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the default value.
+     * @return system property boolean value if it exists, false otherwise().
+     */
+    public boolean getBooleanWithLegacyFallback()
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        return BOOLEAN_CONVERTER.convert(value == null ? defaultVal : value);
+    }
+
     /**
      * Gets the value of a system property as a boolean.
      * @return system property boolean value if it exists, overrideDefaultValue otherwise.
@@ -739,6 +1136,23 @@ public boolean getBoolean(boolean overrideDefaultValue)
         return BOOLEAN_CONVERTER.convert(value);
     }
 
+    /**
+     * Gets the value of a system property as a boolean.
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the overrideDefaultValue value.
+     * @return system property boolean value if it exists, overrideDefaultValue otherwise.
+     */
+    public boolean getBooleanWithLegacyFallback(boolean overrideDefaultValue)
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        if (value == null)
+            return overrideDefaultValue;
+
+        return BOOLEAN_CONVERTER.convert(value);
+    }
+
     /**
      * Sets the value into system properties.
      * @param value to set
@@ -758,6 +1172,14 @@ public void clearValue()
         System.clearProperty(key);
     }
 
+    /**
+     * Clears the value set in the system property using the LEGACY_PREFIX.
+     */
+    public void clearValueWithLegacyPrefix()
+    {
+        System.clearProperty(getKeyWithLegacyPrefix());
+    }
+
     /**
      * Gets the value of a system property as a int.
      * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
@@ -770,6 +1192,22 @@ public int getInt()
         return INTEGER_CONVERTER.convert(value == null ? defaultVal : value);
     }
 
+    /**
+     * Gets the value of a system property as an int.
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the default value.
+     * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
+     */
+    public int getIntWithLegacyFalback()
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        if (value == null && defaultVal == null)
+            throw new ConfigurationException("Missing property value or default value is not set: " + key);
+        return INTEGER_CONVERTER.convert(value == null ? defaultVal : value);
+    }
+
     /**
      * Gets the value of a system property as a long.
      * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
@@ -807,6 +1245,22 @@ public double getDouble()
         return DOUBLE_CONVERTER.convert(value == null ? defaultVal : value);
     }
 
+    /**
+     * Gets the value of a system property as a double.
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the default value.
+     * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
+     */
+    public double getDoubleWithLegacyFallback()
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        if (value == null && defaultVal == null)
+            throw new ConfigurationException("Missing property value or default value is not set: " + key);
+        return DOUBLE_CONVERTER.convert(value == null ? defaultVal : value);
+    }
+
     /**
      * Gets the value of a system property as a double.
      * @return system property value if it exists, defaultValue otherwise.
@@ -820,6 +1274,15 @@ public double getDouble(double overrideDefaultValue)
         return DOUBLE_CONVERTER.convert(value);
     }
 
+    /**
+     * Sets the value into system properties.
+     * @param value to set
+     */
+    public void setDouble(double value)
+    {
+        System.setProperty(key, Double.toString(value));
+    }
+
     /**
      * Gets the value of a system property, given as a human-readable size in bytes (e.g. 100MiB, 10GB, 500B).
      * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
@@ -832,6 +1295,22 @@ public long getSizeInBytes()
         return SIZE_IN_BYTES_CONVERTER.convert(value == null ? defaultVal : value);
     }
 
+    /**
+     * Gets the value of a system property, given as a human-readable size in bytes (e.g. 100MiB, 10GB, 500B).
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the default value.
+     * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
+     */
+    public long getSizeInBytesWithLegacyFallback()
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        if (value == null && defaultVal == null)
+            throw new ConfigurationException("Missing property value or default value is not set: " + key);
+        return SIZE_IN_BYTES_CONVERTER.convert(value == null ? defaultVal : value);
+    }
+
     /**
      * Gets the value of a system property, given as a human-readable size in bytes (e.g. 100MiB, 10GB, 500B).
      * @return System property value if it exists, defaultValue otherwise.
@@ -845,6 +1324,64 @@ public long getSizeInBytes(long overrideDefaultValue)
         return SIZE_IN_BYTES_CONVERTER.convert(value);
     }
 
+    /**
+     * Gets the value of a system property, given as a human-readable size in bytes (e.g. 100MiB, 10GB, 500B).
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the default value.
+     * @return System property value if it exists, overrideDefaultValue otherwise.
+     */
+    public long getSizeInBytesWithLegacyFallback(long overrideDefaultValue)
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        if (value == null)
+            return overrideDefaultValue;
+
+        return SIZE_IN_BYTES_CONVERTER.convert(value);
+    }
+
+    /**
+     * Gets the value of a system property, given as a human-readable size in bytes (e.g. 100MiB, 10GB, 500B).
+     * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
+     */
+    public double getPercentage()
+    {
+        String value = System.getProperty(key);
+        if (value == null && defaultVal == null)
+            throw new ConfigurationException("Missing property value or default value is not set: " + key);
+        return PERCENT_CONVERTER.convert(value == null ? defaultVal : value);
+    }
+
+    /**
+     * Gets the value of a system property, given as a human-readable size in bytes (e.g. 100MiB, 10GB, 500B).
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the default value.
+     * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
+     */
+    public double getPercentageWithLegacyFallback()
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        if (value == null && defaultVal == null)
+            throw new ConfigurationException("Missing property value or default value is not set: " + key);
+        return PERCENT_CONVERTER.convert(value == null ? defaultVal : value);
+    }
+
+    /**
+     * Gets the value of a system property, given as a percentage (e.g. 50%, 0.5).
+     * @return System property value if it exists, defaultValue otherwise.
+     */
+    public double getPercentage(double overrideDefaultValue)
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            return overrideDefaultValue;
+
+        return PERCENT_CONVERTER.convert(value);
+    }
+
     /**
      * Gets the value of a system property as an int.
      * @return system property int value if it exists, overrideDefaultValue otherwise.
@@ -923,6 +1460,29 @@ public <T extends Enum<T>> T getEnum(boolean toUppercase, Class<T> enumClass)
         return Enum.valueOf(enumClass, toUppercase ? value.toUpperCase() : value);
     }
 
+    /**
+     * Gets the value of a system property as an enum, optionally calling {@link String#toUpperCase()} first.
+     * If the property is not set, it will try to get the value from the legacy property. If both are missing,
+     * it returns the default value.
+     * If the value is missing, the default value for this property is used
+     *
+     * @param toUppercase before converting to enum
+     * @param enumClass enumeration class
+     * @param <T> type
+     * @return enum value
+     */
+    public <T extends Enum<T>> T getEnumWithLegacyFallback(boolean toUppercase, Class<T> enumClass)
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            value = System.getProperty(getKeyWithLegacyPrefix());
+        if (value == null && defaultVal == null)
+            throw new ConfigurationException("Missing property value or default value is not set: " + key);
+        if (value == null)
+            value = defaultVal;
+        return Enum.valueOf(enumClass, toUppercase ? value.toUpperCase() : value);
+    }
+
     /**
      * Sets the value into system properties.
      * @param value to set
@@ -973,13 +1533,25 @@ public interface PropertyConverter<T>
         {
             return FBUtilities.parseHumanReadableBytes(value);
         }
-        catch (ConfigurationException e)
+        catch (IllegalArgumentException e)
         {
             throw new ConfigurationException(String.format("Invalid value for system property: " +
                                                            "expected size in bytes with unit but got '%s'\n%s", value, e));
         }
     };
 
+    private static final PropertyConverter<Double> PERCENT_CONVERTER = value -> {
+        try
+        {
+            return FBUtilities.parsePercent(value);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new ConfigurationException(String.format("Invalid value for system property: " +
+                                                           "expected percentage but got '%s'\n%s", value, e));
+        }
+    };
+
     private static final PropertyConverter<Double> DOUBLE_CONVERTER = value ->
     {
         try
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 3a1e9e5e65c9..3b3120704232 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -32,15 +32,17 @@
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
+import org.apache.cassandra.utils.JsonUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.audit.AuditLogOptions;
+import org.apache.cassandra.config.DataStorageSpec.IntBytesBound;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.fql.FullQueryLoggerOptions;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.io.compress.BufferType;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
 import org.apache.cassandra.service.StartupChecks.StartupCheckType;
 import org.apache.cassandra.utils.StorageCompatibilityMode;
 
@@ -148,6 +150,8 @@ public static Set<String> splitCommaDelimited(String src)
 
     @Replaces(oldName = "write_request_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true)
     public volatile DurationSpec.LongMillisecondsBound write_request_timeout = new DurationSpec.LongMillisecondsBound("2000ms");
+    
+    public volatile DurationSpec.LongMillisecondsBound hints_request_timeout = new DurationSpec.LongMillisecondsBound("2000ms");
 
     @Replaces(oldName = "counter_write_request_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true)
     public volatile DurationSpec.LongMillisecondsBound counter_write_request_timeout = new DurationSpec.LongMillisecondsBound("5000ms");
@@ -161,6 +165,8 @@ public static Set<String> splitCommaDelimited(String src)
     @Replaces(oldName = "repair_request_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true)
     public volatile DurationSpec.LongMillisecondsBound repair_request_timeout = new DurationSpec.LongMillisecondsBound("120000ms");
 
+    public volatile DurationSpec.LongMillisecondsBound repair_prepare_message_timeout_in_ms = new DurationSpec.LongMillisecondsBound("10000ms");
+
     public Integer streaming_connections_per_host = 1;
     @Replaces(oldName = "streaming_keep_alive_period_in_secs", converter = Converters.SECONDS_DURATION, deprecated = true)
     public DurationSpec.IntSecondsBound streaming_keep_alive_period = new DurationSpec.IntSecondsBound("300s");
@@ -177,6 +183,8 @@ public static Set<String> splitCommaDelimited(String src)
 
     public int concurrent_reads = 32;
     public int concurrent_writes = 32;
+    public int concurrent_coordinator_reads = 32;
+    public int concurrent_coordinator_writes = 32;
     public int concurrent_counter_writes = 32;
     public int concurrent_materialized_view_writes = 32;
     public int available_processors = -1;
@@ -235,29 +243,29 @@ public MemtableOptions()
     public boolean rpc_keepalive = true;
 
     @Replaces(oldName = "internode_max_message_size_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated=true)
-    public DataStorageSpec.IntBytesBound internode_max_message_size;
+    public IntBytesBound internode_max_message_size;
 
     @Replaces(oldName = "internode_socket_send_buffer_size_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
     @Replaces(oldName = "internode_send_buff_size_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound internode_socket_send_buffer_size = new DataStorageSpec.IntBytesBound("0B");
+    public IntBytesBound internode_socket_send_buffer_size = new IntBytesBound("0B");
     @Replaces(oldName = "internode_socket_receive_buffer_size_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
     @Replaces(oldName = "internode_recv_buff_size_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound internode_socket_receive_buffer_size = new DataStorageSpec.IntBytesBound("0B");
+    public IntBytesBound internode_socket_receive_buffer_size = new IntBytesBound("0B");
 
     // TODO: derive defaults from system memory settings?
     @Replaces(oldName = "internode_application_send_queue_capacity_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound internode_application_send_queue_capacity = new DataStorageSpec.IntBytesBound("4MiB");
+    public IntBytesBound internode_application_send_queue_capacity = new IntBytesBound("4MiB");
     @Replaces(oldName = "internode_application_send_queue_reserve_endpoint_capacity_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound internode_application_send_queue_reserve_endpoint_capacity = new DataStorageSpec.IntBytesBound("128MiB");
+    public IntBytesBound internode_application_send_queue_reserve_endpoint_capacity = new IntBytesBound("128MiB");
     @Replaces(oldName = "internode_application_send_queue_reserve_global_capacity_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound internode_application_send_queue_reserve_global_capacity = new DataStorageSpec.IntBytesBound("512MiB");
+    public IntBytesBound internode_application_send_queue_reserve_global_capacity = new IntBytesBound("512MiB");
 
     @Replaces(oldName = "internode_application_receive_queue_capacity_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound internode_application_receive_queue_capacity = new DataStorageSpec.IntBytesBound("4MiB");
+    public IntBytesBound internode_application_receive_queue_capacity = new IntBytesBound("4MiB");
     @Replaces(oldName = "internode_application_receive_queue_reserve_endpoint_capacity_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound internode_application_receive_queue_reserve_endpoint_capacity = new DataStorageSpec.IntBytesBound("128MiB");
+    public IntBytesBound internode_application_receive_queue_reserve_endpoint_capacity = new IntBytesBound("128MiB");
     @Replaces(oldName = "internode_application_receive_queue_reserve_global_capacity_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound internode_application_receive_queue_reserve_global_capacity = new DataStorageSpec.IntBytesBound("512MiB");
+    public IntBytesBound internode_application_receive_queue_reserve_global_capacity = new IntBytesBound("512MiB");
 
     // Defensive settings for protecting Cassandra from true network partitions. See (CASSANDRA-14358) for details.
     // The amount of time to wait for internode tcp connections to establish.
@@ -297,7 +305,7 @@ public MemtableOptions()
     public volatile boolean native_transport_rate_limiting_enabled = false;
     public volatile int native_transport_max_requests_per_second = 1000000;
     @Replaces(oldName = "native_transport_receive_queue_capacity_in_bytes", converter = Converters.BYTES_DATASTORAGE, deprecated = true)
-    public DataStorageSpec.IntBytesBound native_transport_receive_queue_capacity = new DataStorageSpec.IntBytesBound("1MiB");
+    public IntBytesBound native_transport_receive_queue_capacity = new IntBytesBound("1MiB");
 
     /**
      * Max size of values in SSTables, in MebiBytes.
@@ -364,7 +372,7 @@ public MemtableOptions()
 
     public static class SSTableConfig
     {
-        public String selected_format = BigFormat.NAME;
+        public String selected_format = BtiFormat.NAME;
         public Map<String, Map<String, String>> format = new HashMap<>();
     }
 
@@ -378,6 +386,8 @@ public static class SSTableConfig
 
     public String saved_caches_directory;
 
+    public String metadata_directory;
+
     // Commit Log
     public String commitlog_directory;
     @Replaces(oldName = "commitlog_total_space_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_INT, deprecated = true)
@@ -390,7 +400,7 @@ public static class SSTableConfig
     @Replaces(oldName = "commitlog_segment_size_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_INT, deprecated = true)
     public DataStorageSpec.IntMebibytesBound commitlog_segment_size = new DataStorageSpec.IntMebibytesBound("32MiB");
     public ParameterizedClass commitlog_compression;
-    public FlushCompression flush_compression = FlushCompression.fast;
+    public FlushCompression flush_compression;
     public int commitlog_max_compression_buffers_in_pool = 3;
     public DiskAccessMode commitlog_disk_access_mode = DiskAccessMode.legacy;
     @Replaces(oldName = "periodic_commitlog_sync_lag_block_in_ms", converter = Converters.MILLIS_DURATION_INT, deprecated = true)
@@ -400,6 +410,8 @@ public static class SSTableConfig
     @Replaces(oldName = "max_mutation_size_in_kb", converter = Converters.KIBIBYTES_DATASTORAGE, deprecated = true)
     public DataStorageSpec.IntKibibytesBound max_mutation_size;
 
+    public StorageFlagsConfig storage_flags = new StorageFlagsConfig();
+
     // Change-data-capture logs
     public boolean cdc_enabled = false;
     // When true, new CDC mutations are rejected/blocked when reaching max CDC storage.
@@ -433,6 +445,7 @@ public static class SSTableConfig
     public DataStorageSpec.IntKibibytesBound hinted_handoff_throttle = new DataStorageSpec.IntKibibytesBound("1024KiB");
     @Replaces(oldName = "batchlog_replay_throttle_in_kb", converter = Converters.KIBIBYTES_DATASTORAGE, deprecated = true)
     public DataStorageSpec.IntKibibytesBound batchlog_replay_throttle = new DataStorageSpec.IntKibibytesBound("1024KiB");
+    public BatchlogEndpointStrategy batchlog_endpoint_strategy = BatchlogEndpointStrategy.random_remote;
     public int max_hints_delivery_threads = 2;
     @Replaces(oldName = "hints_flush_period_in_ms", converter = Converters.MILLIS_DURATION_INT, deprecated = true)
     public DurationSpec.IntMillisecondsBound hints_flush_period = new DurationSpec.IntMillisecondsBound("10s");
@@ -445,7 +458,7 @@ public static class SSTableConfig
     public volatile boolean transfer_hints_on_decommission = true;
 
     public volatile boolean incremental_backups = false;
-    public boolean trickle_fsync = false;
+    public boolean trickle_fsync = true;
     @Replaces(oldName = "trickle_fsync_interval_in_kb", converter = Converters.KIBIBYTES_DATASTORAGE, deprecated = true)
     public DataStorageSpec.IntKibibytesBound trickle_fsync_interval = new DataStorageSpec.IntKibibytesBound("10240KiB");
 
@@ -453,6 +466,8 @@ public static class SSTableConfig
     @Replaces(oldName = "sstable_preemptive_open_interval_in_mb", converter = Converters.NEGATIVE_MEBIBYTES_DATA_STORAGE_INT, deprecated = true)
     public volatile DataStorageSpec.IntMebibytesBound sstable_preemptive_open_interval = new DataStorageSpec.IntMebibytesBound("50MiB");
 
+    /** @deprecated CPU-intensive optimization that visibly slows down compaction but does not provide a clear benefit (see STAR-782) */
+    @Deprecated(since = "CC 4.0")
     public volatile boolean key_cache_migrate_during_compaction = true;
     public volatile int key_cache_keys_to_save = Integer.MAX_VALUE;
     @Replaces(oldName = "key_cache_size_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_LONG, deprecated = true)
@@ -512,7 +527,7 @@ public static class SSTableConfig
 
     public boolean inter_dc_tcp_nodelay = true;
 
-    public MemtableAllocationType memtable_allocation_type = MemtableAllocationType.heap_buffers;
+    public MemtableAllocationType memtable_allocation_type = MemtableAllocationType.offheap_objects;
 
     public volatile boolean read_thresholds_enabled = false;
     public volatile DataStorageSpec.LongBytesBound coordinator_read_size_warn_threshold = null;
@@ -684,6 +699,7 @@ public static class SSTableConfig
     public volatile boolean automatic_sstable_upgrade = false;
     public volatile int max_concurrent_automatic_sstable_upgrades = 1;
     public boolean stream_entire_sstables = true;
+    public boolean netty_zerocopy_enabled = true;
 
     public volatile boolean skip_stream_disk_space_check = false;
 
@@ -778,6 +794,7 @@ public static class SSTableConfig
      */
     public volatile boolean log_out_of_token_range_requests = true;
     public volatile boolean reject_out_of_token_range_requests = false;
+    public boolean emulate_dbaas_defaults = false;
 
     /**
      * The intial capacity for creating RangeTombstoneList.
@@ -790,6 +807,8 @@ public static class SSTableConfig
 
     public StorageAttachedIndexOptions sai_options = new StorageAttachedIndexOptions();
 
+    public volatile int aggregation_subpage_size_in_kb = 2048;
+
     /**
      * @deprecated migrate to {@link DatabaseDescriptor#isClientInitialized()} See CASSANDRA-12550
      */
@@ -825,7 +844,7 @@ public static boolean isClientMode()
     public volatile boolean auto_optimise_preview_repair_streams = false;
 
     // see CASSANDRA-17048 and the comment in cassandra.yaml
-    public boolean uuid_sstable_identifiers_enabled = false;
+    public boolean uuid_sstable_identifiers_enabled = true;
 
     /**
      * Client mode means that the process is a pure client, that uses C* code base but does
@@ -852,12 +871,22 @@ public static void setClientMode(boolean clientMode)
     public volatile int tables_fail_threshold = -1;
     public volatile int columns_per_table_warn_threshold = -1;
     public volatile int columns_per_table_fail_threshold = -1;
+    // Legacy 2i guardrail
     public volatile int secondary_indexes_per_table_warn_threshold = -1;
     public volatile int secondary_indexes_per_table_fail_threshold = -1;
+    public volatile int sasi_indexes_per_table_warn_threshold = -1;
+    public volatile int sasi_indexes_per_table_fail_threshold = -1;
+    // SAI indexes guardrail
+    public volatile int sai_indexes_per_table_warn_threshold = -1;
+    public volatile int sai_indexes_per_table_fail_threshold = -1;
+    public volatile int sai_indexes_total_warn_threshold = -1;
+    public volatile int sai_indexes_total_fail_threshold = -1;
     public volatile int materialized_views_per_table_warn_threshold = -1;
     public volatile int materialized_views_per_table_fail_threshold = -1;
     public volatile int page_size_warn_threshold = -1;
     public volatile int page_size_fail_threshold = -1;
+    public volatile DataStorageSpec.IntBytesBound page_weight_warn_threshold = null;
+    public volatile DataStorageSpec.IntBytesBound page_weight_fail_threshold = null;
     public volatile int partition_keys_in_select_warn_threshold = -1;
     public volatile int partition_keys_in_select_fail_threshold = -1;
     public volatile int in_select_cartesian_product_warn_threshold = -1;
@@ -872,6 +901,7 @@ public static void setClientMode(boolean clientMode)
     public volatile boolean user_timestamps_enabled = true;
     public volatile boolean alter_table_enabled = true;
     public volatile boolean group_by_enabled = true;
+    public volatile boolean logged_batch_enabled = true;
     public volatile boolean drop_truncate_table_enabled = true;
     public volatile boolean drop_keyspace_enabled = true;
     public volatile boolean secondary_indexes_enabled = true;
@@ -884,6 +914,7 @@ public static void setClientMode(boolean clientMode)
     public volatile boolean read_before_write_list_operations_enabled = true;
     public volatile boolean allow_filtering_enabled = true;
     public volatile boolean simplestrategy_enabled = true;
+    public volatile boolean counter_enabled = true;
     @Replaces(oldName = "compaction_large_partition_warning_threshold_mb", converter = Converters.LONG_BYTES_DATASTORAGE_MEBIBYTES_INT, deprecated = true)
     @Replaces(oldName = "compaction_large_partition_warning_threshold", converter = Converters.LONG_BYTES_DATASTORAGE_MEBIBYTES_DATASTORAGE, deprecated = true)
     public volatile DataStorageSpec.LongBytesBound partition_size_warn_threshold = null;
@@ -902,6 +933,8 @@ public static void setClientMode(boolean clientMode)
     public volatile boolean vector_type_enabled = true;
     public volatile int vector_dimensions_warn_threshold = -1;
     public volatile int vector_dimensions_fail_threshold = -1;
+    public volatile int sai_ann_rerank_k_warn_threshold = -1;
+    public volatile int sai_ann_rerank_k_fail_threshold = 4 * CassandraRelevantProperties.SAI_VECTOR_SEARCH_MAX_TOP_K.getInt();
     public volatile int data_disk_usage_percentage_warn_threshold = -1;
     public volatile int data_disk_usage_percentage_fail_threshold = -1;
     public volatile DataStorageSpec.LongBytesBound data_disk_usage_max_disk_size = null;
@@ -942,6 +975,14 @@ public static void setClientMode(boolean clientMode)
     public volatile DurationSpec.LongMicrosecondsBound minimum_timestamp_warn_threshold = null;
     public volatile DurationSpec.LongMicrosecondsBound minimum_timestamp_fail_threshold = null;
 
+    // Limit the offset used in SELECT queries
+    public volatile int offset_rows_warn_threshold = -1;
+    public volatile int offset_rows_fail_threshold = -1;
+
+    // Limit the number of column value filters per SELECT query (after applying analyzers, in case they are used)
+    public volatile int query_filters_warn_threshold = -1;
+    public volatile int query_filters_fail_threshold = -1;
+
     /**
      * The variants of paxos implementation and semantics supported by Cassandra.
      */
@@ -1158,6 +1199,7 @@ public enum FlushCompression
     {
         none,
         fast,
+        adaptive,
         table
     }
 
@@ -1217,9 +1259,27 @@ public enum DiskFailurePolicy
 
     public enum CommitFailurePolicy
     {
+        /**
+         * This will stop Gossip and the inter-node transports, and kill the syncing thread.
+         */
         stop,
+        /**
+         * This will kill the syncing thread.
+         */
         stop_commit,
+        /**
+         * This will set a flag that will reject all mutations that require the CL. This flag
+         * will be cleared after a successful sync. This flag is only set for errors when sync-ing
+         * the CL segments, it is identical to ignore for other errors.
+         */
+        fail_writes,
+        /**
+         * This will ignore the error, a new sync will be attempted after a full polling period.
+         */
         ignore,
+        /**
+         * This will kill the process.
+         */
         die,
     }
 
@@ -1249,12 +1309,79 @@ public enum CorruptedTombstoneStrategy
         exception
     }
 
+    public enum BatchlogEndpointStrategy
+    {
+        /**
+         * Old, conventional strategy to select batchlog storage endpoints.
+         * Purely random, prevents the local rack, if possible.
+         */
+        random_remote(false, false),
+
+        /**
+         * Random, except that one of the replications will go to the local rack.
+         * Which means this strategy offers lower availability guarantees than
+         * {@link #random_remote} or {@link #dynamic_remote}.
+         */
+        prefer_local(false, true),
+
+        /**
+         * Strategy using {@link Config#dynamic_snitch} ({@link org.apache.cassandra.locator.DynamicEndpointSnitch})
+         * to select batchlog storage endpoints. Prevents the local rack, if possible.
+         *
+         * This strategy offers the same availability guarantees as {@link #random_remote} but selects the
+         * fastest endpoints according to the {@link org.apache.cassandra.locator.DynamicEndpointSnitch}.
+         *
+         * Hint: {@link org.apache.cassandra.locator.DynamicEndpointSnitch} tracks reads and not writes - i.e.
+         * write-only (or mostly-write) workloads might not benefit from this strategy.
+         *
+         * Note: this strategy will fall back to {@link #random_remote}, if {@link #dynamic_snitch} is not enabled.
+         */
+        dynamic_remote(true, false),
+
+        /**
+         * Strategy using {@link Config#dynamic_snitch} ({@link org.apache.cassandra.locator.DynamicEndpointSnitch})
+         * to select batchlog storage endpoints. Does not prevent the local rack.
+         *
+         * Since the local rack is not excluded, this strategy offers lower availability guarantees than
+         * {@link #random_remote} or {@link #dynamic_remote}.
+         *
+         * Hint: {@link org.apache.cassandra.locator.DynamicEndpointSnitch} tracks reads and not writes - i.e.
+         * write-only (or mostly-write) workloads might not benefit from this strategy.
+         *
+         * Note: this strategy will fall back to {@link #random_remote}, if {@link #dynamic_snitch} is not enabled.
+         */
+        dynamic(true, true);
+
+        /**
+         * If true, dynamic snitch response times will be used to select more responsive nodes to write the batchlog to.
+         * If false, nodes will be randomly selected.
+         */
+        public final boolean useDynamicSnitchScores;
+
+        /**
+         * If true, one of the selected nodes will come from the local rack.
+         * If false, the local rack will not be used except as a last resort with no other racks available.
+         */
+        public final boolean preferLocalRack;
+
+        BatchlogEndpointStrategy(boolean useDynamicSnitchScores, boolean preferLocalRack)
+        {
+            this.useDynamicSnitchScores = useDynamicSnitchScores;
+            this.preferLocalRack = preferLocalRack;
+        }
+    }
+
     private static final Set<String> SENSITIVE_KEYS = new HashSet<String>() {{
         add("client_encryption_options");
         add("server_encryption_options");
     }};
 
     public static void log(Config config)
+    {
+        logger.info("Node configuration:[{}]", config.toDebugString(false));
+    }
+
+    public String toDebugString(boolean json)
     {
         Map<String, String> configMap = new TreeMap<>();
         for (Field field : Config.class.getFields())
@@ -1274,7 +1401,7 @@ public static void log(Config config)
             try
             {
                 // Field.get() can throw NPE if the value of the field is null
-                value = field.get(config).toString();
+                value = field.get(this).toString();
             }
             catch (NullPointerException | IllegalAccessException npe)
             {
@@ -1282,8 +1409,11 @@ public static void log(Config config)
             }
             configMap.put(name, value);
         }
-
-        logger.info("Node configuration:[{}]", Joiner.on("; ").join(configMap.entrySet()));
+        if (json)
+        {
+            return JsonUtils.writeAsJsonString(configMap);
+        }
+        return Joiner.on("; ").join(configMap.entrySet());
     }
 
     public volatile boolean dump_heap_on_uncaught_exception = false;
@@ -1311,6 +1441,7 @@ public enum CQLStartTime
     public DurationSpec.LongMillisecondsBound native_transport_max_backoff_on_queue_overload = new DurationSpec.LongMillisecondsBound("200ms");
 
     // 3.x Cassandra Driver has its "read" timeout set to 12 seconds, default matches this.
-    public DurationSpec.LongMillisecondsBound native_transport_timeout = new DurationSpec.LongMillisecondsBound("12s");
+    @Replaces(oldName = "native_transport_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true)
+    public volatile DurationSpec.LongMillisecondsBound native_transport_timeout = new DurationSpec.LongMillisecondsBound("12s");
     public boolean enforce_native_deadline_for_hints = false;
 }
diff --git a/src/java/org/apache/cassandra/config/DataStorageSpec.java b/src/java/org/apache/cassandra/config/DataStorageSpec.java
index b5d4374088ae..4cf1a3fd6464 100644
--- a/src/java/org/apache/cassandra/config/DataStorageSpec.java
+++ b/src/java/org/apache/cassandra/config/DataStorageSpec.java
@@ -620,7 +620,7 @@ public static DataStorageUnit fromSymbol(String symbol)
         /**
          * The unit symbol
          */
-        private final String symbol;
+        public final String symbol;
 
         DataStorageUnit(String symbol)
         {
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 20e523acf0f3..88b2a6c701d4 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.config;
 
-import java.io.IOError;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
@@ -28,6 +27,7 @@
 import java.net.SocketException;
 import java.net.UnknownHostException;
 import java.nio.file.FileStore;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -51,9 +51,11 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.HashMultiset;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multiset;
 import com.google.common.collect.Sets;
 import com.google.common.primitives.Ints;
 import com.google.common.primitives.Longs;
@@ -78,6 +80,7 @@
 import org.apache.cassandra.config.Config.DiskAccessMode;
 import org.apache.cassandra.config.Config.PaxosOnLinearizabilityViolation;
 import org.apache.cassandra.config.Config.PaxosStatePurging;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager;
 import org.apache.cassandra.db.commitlog.CommitLog;
@@ -89,7 +92,8 @@
 import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.DiskOptimizationStrategy;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
@@ -117,6 +121,7 @@
 import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_LOADER;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CHRONICLE_ANALYTICS_DISABLE;
 import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_STCS_IN_L0;
+import static org.apache.cassandra.config.CassandraRelevantProperties.FILE_CACHE_SIZE_IN_MB;
 import static org.apache.cassandra.config.CassandraRelevantProperties.INITIAL_TOKEN;
 import static org.apache.cassandra.config.CassandraRelevantProperties.IO_NETTY_TRANSPORT_ESTIMATE_SIZE_ON_SUBMIT;
 import static org.apache.cassandra.config.CassandraRelevantProperties.NATIVE_TRANSPORT_PORT;
@@ -135,7 +140,6 @@
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_SKIP_CRYPTO_PROVIDER_INSTALLATION;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_STRICT_RUNTIME_CHECKS;
-import static org.apache.cassandra.config.CassandraRelevantProperties.UNSAFE_SYSTEM;
 import static org.apache.cassandra.config.DataRateSpec.DataRateUnit.BYTES_PER_SECOND;
 import static org.apache.cassandra.config.DataRateSpec.DataRateUnit.MEBIBYTES_PER_SECOND;
 import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.MEBIBYTES;
@@ -161,7 +165,7 @@ public class DatabaseDescriptor
      */
     private static final int MAX_NUM_TOKENS = 1536;
 
-    private static Config conf;
+    private volatile static Config conf;
 
     /**
      * Request timeouts can not be less than below defined value (see CASSANDRA-9375)
@@ -196,6 +200,14 @@ public class DatabaseDescriptor
     private static IRoleManager roleManager;
 
     private static long preparedStatementsCacheSizeInMiB;
+    
+    private static File[] dataDirectories = new File[0];
+    private static File localSystemDataFileDirectory;
+    private static File commitlogDirectory;
+    private static File hintsDirectory;
+    private static File savedCachesDirectory;
+    private static File cdcRawDirectory;
+    private static File metadataDirectory;
 
     private static long keyCacheSizeInMiB;
     private static long paxosCacheSizeInMiB;
@@ -212,12 +224,12 @@ public class DatabaseDescriptor
     private static boolean clientInitialized;
     private static boolean toolInitialized;
     private static boolean daemonInitialized;
+    private static boolean enableMemtableAndCommitLog;
 
     private static final int searchConcurrencyFactor = SEARCH_CONCURRENCY_FACTOR.getInt();
-    private static DurationSpec.IntSecondsBound autoSnapshoTtl;
+    private static DurationSpec.IntSecondsBound autoSnapshoTtl; // TODO rename it
 
     private static volatile boolean disableSTCSInL0 = DISABLE_STCS_IN_L0.getBoolean();
-    private static final boolean unsafeSystem = UNSAFE_SYSTEM.getBoolean();
 
     // turns some warnings into exceptions for testing
     private static final boolean strictRuntimeChecks = TEST_STRICT_RUNTIME_CHECKS.getBoolean();
@@ -238,6 +250,43 @@ public class DatabaseDescriptor
                                                                                                            ? new CommitLogSegmentManagerCDC(c, DatabaseDescriptor.getCommitLogLocation())
                                                                                                            : new CommitLogSegmentManagerStandard(c, DatabaseDescriptor.getCommitLogLocation());
 
+    static void updateInitialized(boolean clientInitialized, boolean toolInitialized, boolean daemonInitialized)
+    {
+        DatabaseDescriptor.clientInitialized = clientInitialized;
+        DatabaseDescriptor.toolInitialized = toolInitialized;
+        DatabaseDescriptor.daemonInitialized = daemonInitialized;
+    }
+
+    /**
+     * This method can be called from tests if we need to re-initialize the DD
+     * again with a new config, partitioner or snitch. Note that this method will
+     * not delete directories or null the config, it simply resets the variables
+     * that prevent changing config or directories.
+     * <p/>
+     * It cannot set the config to null because this would cause NPEs in the
+     * schedulers that are still running.
+     * <p/>
+     * It cannot delete directories because on remote storage this would result
+     * in errors if the test containers for remote storage are being shutdown
+     * concurrently. The caller should delete any directories if required.
+     * TODO If you run into problems with undeleted directories or with the 
+     * caller deleting them, please add additional details here.
+     * <p/>
+     * This method is called by integration tests that run in the same JVM.
+     */
+    @VisibleForTesting
+    public static void resetUnsafe()
+    {
+        updateInitialized(false, false, false);
+
+        setPartitionerUnsafe(null);
+        IEndpointSnitch oldSnitch = getEndpointSnitch();
+        if (oldSnitch instanceof DynamicEndpointSnitch)
+            ((DynamicEndpointSnitch)oldSnitch).close();
+        setEndpointSnitch(null);
+        EndpointSnitchInfo.unregisterMBean();
+    }
+
     public static void daemonInitialization() throws ConfigurationException
     {
         daemonInitialization(DatabaseDescriptor::loadConfig);
@@ -257,6 +306,11 @@ public static void daemonInitialization(Supplier<Config> config) throws Configur
 
         setConfig(config.get());
         applyAll();
+
+        createAllDirectories();
+
+        applyGuardrails(); // GuardrailsOptions.validateDataDiskUsageMaxDiskSize requires created directories
+
         AuthConfig.applyAuth();
     }
 
@@ -307,6 +361,8 @@ public static void toolInitialization(boolean failIfDaemonOrClient)
         applySnitch();
 
         applyEncryptionContext();
+
+        createAllDirectories();
     }
 
     /**
@@ -378,6 +434,20 @@ public static boolean isDaemonInitialized()
         return daemonInitialized;
     }
 
+    /**
+     * Enables lifecycle transactions via {@link org.apache.cassandra.db.lifecycle.Tracker} and
+     * memtable via {@link org.apache.cassandra.db.ColumnFamilyStore}.
+     */
+    public static void setEnableMemtableAndCommitLog()
+    {
+        enableMemtableAndCommitLog = true;
+    }
+
+    public static boolean enableMemtableAndCommitLog()
+    {
+        return daemonInitialized || enableMemtableAndCommitLog;
+    }
+
     public static Config getRawConfig()
     {
         return conf;
@@ -386,14 +456,20 @@ public static Config getRawConfig()
     @VisibleForTesting
     public static Config loadConfig() throws ConfigurationException
     {
+        Config config;
         if (Config.getOverrideLoadConfig() != null)
-            return Config.getOverrideLoadConfig().get();
+        {
+            config = Config.getOverrideLoadConfig().get();
+        }
+        else
+        {
 
         String loaderClass = CONFIG_LOADER.getString();
         ConfigurationLoader loader = loaderClass == null
                                      ? new YamlConfigurationLoader()
                                      : FBUtilities.construct(loaderClass, "configuration loading");
-        Config config = loader.loadConfig();
+            config = loader.loadConfig();
+        }
 
         if (!hasLoggedConfig)
         {
@@ -464,8 +540,6 @@ private static void applyAll() throws ConfigurationException
 
         applySslContext();
 
-        applyGuardrails();
-
         applyStartupChecks();
     }
 
@@ -561,6 +635,16 @@ else if (conf.disk_access_mode == DiskAccessMode.direct)
             throw new ConfigurationException("concurrent_writes must be at least 2, but was " + conf.concurrent_writes, false);
         }
 
+        if (conf.concurrent_coordinator_reads < 2)
+        {
+            throw new ConfigurationException("concurrent_coordinator_reads must be at least 2, but was " + conf.concurrent_coordinator_reads, false);
+        }
+
+        if (conf.concurrent_coordinator_writes < 2)
+        {
+            throw new ConfigurationException("concurrent_coordinator_writes must be at least 2, but was " + conf.concurrent_coordinator_writes, false);
+        }
+
         if (conf.concurrent_counter_writes < 2)
             throw new ConfigurationException("concurrent_counter_writes must be at least 2, but was " + conf.concurrent_counter_writes, false);
 
@@ -568,7 +652,7 @@ else if (conf.disk_access_mode == DiskAccessMode.direct)
             conf.networking_cache_size = new DataStorageSpec.IntMebibytesBound(Math.min(128, (int) (Runtime.getRuntime().maxMemory() / (16 * 1048576))));
 
         if (conf.file_cache_size == null)
-            conf.file_cache_size = new DataStorageSpec.IntMebibytesBound(Math.min(512, (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576))));
+            conf.file_cache_size = new DataStorageSpec.IntMebibytesBound(Math.min(FILE_CACHE_SIZE_IN_MB.getInt(), (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576))));
 
         // round down for SSDs and round up for spinning disks
         if (conf.file_cache_round_up == null)
@@ -640,6 +724,11 @@ else if (conf.repair_session_space.toMebibytes() > (int) (Runtime.getRuntime().m
             conf.hints_directory = storagedirFor("hints");
         }
 
+        if (conf.metadata_directory == null)
+        {
+            conf.metadata_directory = storagedirFor("metadata");
+        }
+
         if (conf.native_transport_max_request_data_in_flight == null)
         {
             conf.native_transport_max_request_data_in_flight = new DataStorageSpec.LongBytesBound(Runtime.getRuntime().maxMemory() / 10);
@@ -688,7 +777,7 @@ else if (conf.repair_session_space.toMebibytes() > (int) (Runtime.getRuntime().m
                 conf.cdc_total_space = new DataStorageSpec.IntMebibytesBound(defaultSpaceInMiB);
             }
 
-            logger.info("cdc_enabled is true. Starting casssandra node with Change-Data-Capture enabled.");
+            logger.info("cdc_enabled is true. Starting cassandra node with Change-Data-Capture enabled.");
         }
 
         if (conf.saved_caches_directory == null)
@@ -714,6 +803,8 @@ else if (conf.repair_session_space.toMebibytes() > (int) (Runtime.getRuntime().m
                 throw new ConfigurationException("hints_directory must not be the same as any data_file_directories", false);
             if (datadir.equals(conf.saved_caches_directory))
                 throw new ConfigurationException("saved_caches_directory must not be the same as any data_file_directories", false);
+            if (datadir.equals(conf.metadata_directory))
+                throw new ConfigurationException("metadata_directory must not be the same as any data_file_directories", false);
 
             dataFreeBytes = saturatedSum(dataFreeBytes, tryGetSpace(datadir, FileStore::getUnallocatedSpace));
         }
@@ -1005,6 +1096,11 @@ else if (conf.max_value_size.toMebibytes() >= 2048)
 
         if (conf.use_deterministic_table_id)
             logger.warn("use_deterministic_table_id is deprecated and will be ignored in a future release.");
+
+        if (conf.aggregation_subpage_size_in_kb < 1)
+            throw new ConfigurationException("aggregation_subpage_size_in_kb must be greater than 0");
+
+        setAggregationSubPageSize(getAggregationSubPageSize());
     }
 
     @VisibleForTesting
@@ -1355,6 +1451,11 @@ static void checkForLowestAcceptedTimeouts(Config conf)
             logInfo("truncate_request_timeout", conf.truncate_request_timeout, LOWEST_ACCEPTED_TIMEOUT);
             conf.truncate_request_timeout = LOWEST_ACCEPTED_TIMEOUT;
         }
+        if(conf.native_transport_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
+        {
+            logInfo("native_transport_timeout", conf.native_transport_timeout, LOWEST_ACCEPTED_TIMEOUT);
+            conf.native_transport_timeout = LOWEST_ACCEPTED_TIMEOUT;
+        }
     }
 
     private static void logInfo(String property, DurationSpec.LongMillisecondsBound actualValue, DurationSpec.LongMillisecondsBound lowestAcceptedValue)
@@ -1410,7 +1511,7 @@ public static void applySnitch()
             throw new ConfigurationException("Missing endpoint_snitch directive", false);
         }
         snitch = createEndpointSnitch(conf.dynamic_snitch, conf.endpoint_snitch);
-        EndpointSnitchInfo.create();
+        EndpointSnitchInfo.registerMBean();
 
         localDC = snitch.getLocalDatacenter();
         localComparator = (replica1, replica2) -> {
@@ -1458,18 +1559,7 @@ private static Pair<DiskAccessMode, Boolean> resolveCommitLogWriteDiskAccessMode
         boolean directIOSupported = false;
         try
         {
-            String commitLogLocation = getCommitLogLocation();
-
-            if (commitLogLocation == null)
-                throw new ConfigurationException("commitlog_directory must be specified", false);
-
-            File commitLogLocationDir = new File(commitLogLocation);
-            PathUtils.createDirectoriesIfNotExists(commitLogLocationDir.toPath());
-            directIOSupported = FileUtils.getBlockSize(commitLogLocationDir) > 0;
-        }
-        catch (IOError | ConfigurationException ex)
-        {
-            throw  ex;
+            directIOSupported = FileUtils.getBlockSize(getCommitLogLocation()) > 0;
         }
         catch (RuntimeException e)
         {
@@ -1554,7 +1644,7 @@ private static void validateSSTableFormatFactories(Iterable<SSTableFormat.Factor
     {
         SSTableFormat<?, ?> selectedFormat;
         if (StringUtils.isBlank(selectedFormatName))
-            selectedFormatName = BigFormat.NAME;
+            selectedFormatName = BtiFormat.NAME;
         selectedFormat = sstableFormats.get(selectedFormatName);
         if (selectedFormat == null)
             throw new ConfigurationException(String.format("Selected sstable format '%s' is not available.", selectedFormatName));
@@ -1578,7 +1668,7 @@ private static void applySSTableFormats()
         ServiceLoader<SSTableFormat.Factory> loader = ServiceLoader.load(SSTableFormat.Factory.class, DatabaseDescriptor.class.getClassLoader());
         List<SSTableFormat.Factory> factories = Iterables.toList(loader);
         if (factories.isEmpty())
-            factories = ImmutableList.of(new BigFormat.BigFormatFactory());
+            factories = ImmutableList.of(new BtiFormat.BtiFormatFactory());
         applySSTableFormats(factories, conf.sstable);
     }
 
@@ -1925,29 +2015,36 @@ public static void createAllDirectories()
             if (conf.data_file_directories.length == 0)
                 throw new ConfigurationException("At least one DataFileDirectory must be specified", false);
 
-            for (String dataFileDirectory : conf.data_file_directories)
-                FileUtils.createDirectory(dataFileDirectory);
+            File[] dataDirs = new File[conf.data_file_directories.length];
+            for (int i = 0; i < conf.data_file_directories.length; i++) 
+                dataDirs[i] = StorageProvider.instance.createDirectory(conf.data_file_directories[i], StorageProvider.DirectoryType.DATA);
+            setDataDirectories(dataDirs);
 
             if (conf.local_system_data_file_directory != null)
-                FileUtils.createDirectory(conf.local_system_data_file_directory);
+                localSystemDataFileDirectory = StorageProvider.instance.createDirectory(conf.local_system_data_file_directory, StorageProvider.DirectoryType.LOCAL_SYSTEM_DATA);
 
             if (conf.commitlog_directory == null)
                 throw new ConfigurationException("commitlog_directory must be specified", false);
-            FileUtils.createDirectory(conf.commitlog_directory);
+            commitlogDirectory = StorageProvider.instance.createDirectory(conf.commitlog_directory, StorageProvider.DirectoryType.COMMITLOG);
 
             if (conf.hints_directory == null)
                 throw new ConfigurationException("hints_directory must be specified", false);
-            FileUtils.createDirectory(conf.hints_directory);
+            hintsDirectory = StorageProvider.instance.createDirectory(conf.hints_directory, StorageProvider.DirectoryType.HINTS);
 
             if (conf.saved_caches_directory == null)
                 throw new ConfigurationException("saved_caches_directory must be specified", false);
-            FileUtils.createDirectory(conf.saved_caches_directory);
+            savedCachesDirectory = StorageProvider.instance.createDirectory(conf.saved_caches_directory, StorageProvider.DirectoryType.SAVED_CACHES);
 
-            if (conf.cdc_enabled)
-            {
-                if (conf.cdc_raw_directory == null)
-                    throw new ConfigurationException("cdc_raw_directory must be specified", false);
-                FileUtils.createDirectory(conf.cdc_raw_directory);
+            if (conf.metadata_directory == null)
+                throw new ConfigurationException("metadata_directory must be specified", false);
+            metadataDirectory = StorageProvider.instance.createDirectory(conf.metadata_directory, StorageProvider.DirectoryType.METADATA);
+
+            if (conf.cdc_raw_directory == null && conf.cdc_enabled) {
+                throw new ConfigurationException("cdc_raw_directory must be specified", false);
+            }
+            
+            if (conf.cdc_raw_directory != null) {
+                cdcRawDirectory = StorageProvider.instance.createDirectory(conf.cdc_raw_directory, StorageProvider.DirectoryType.CDC);
             }
 
             boolean created = maybeCreateHeapDumpPath();
@@ -1982,7 +2079,7 @@ public static IPartitioner setPartitionerUnsafe(IPartitioner newPartitioner)
     {
         IPartitioner old = partitioner;
         partitioner = newPartitioner;
-        partitionerName = partitioner.getClass().getCanonicalName();
+        partitionerName = partitioner != null ? partitioner.getClass().getCanonicalName() : null;
         return old;
     }
 
@@ -2030,7 +2127,7 @@ public static int getColumnIndexCacheSizeInKiB()
         return conf.column_index_cache_size.toKibibytes();
     }
 
-    public static void setColumnIndexCacheSize(int val)
+    public static void setColumnIndexCacheSizeInKiB(int val)
     {
         conf.column_index_cache_size = createIntKibibyteBoundAndEnsureItIsValidForByteConversion(val,"column_index_cache_size");
     }
@@ -2195,6 +2292,11 @@ public static long getWriteRpcTimeout(TimeUnit unit)
         return conf.write_request_timeout.to(unit);
     }
 
+    public static long getHintsRpcTimeout(TimeUnit unit)
+    {
+        return conf.hints_request_timeout.to(unit);
+    }
+
     public static void setWriteRpcTimeout(long timeOutInMillis)
     {
         conf.write_request_timeout = new DurationSpec.LongMillisecondsBound(timeOutInMillis);
@@ -2240,6 +2342,16 @@ public static void setRepairRpcTimeout(Long timeOutInMillis)
         conf.repair_request_timeout = new DurationSpec.LongMillisecondsBound(timeOutInMillis);
     }
 
+    public static long getRepairPrepareMessageTimeout(TimeUnit unit)
+    {
+        return conf.repair_prepare_message_timeout_in_ms.to(unit);
+    }
+
+    public static void setRepairPrepareMessageTimeout(Long timeOutInMillis)
+    {
+        conf.repair_prepare_message_timeout_in_ms = new DurationSpec.LongMillisecondsBound(timeOutInMillis);
+    }
+
     public static boolean hasCrossNodeTimeout()
     {
         return conf.internode_timeout;
@@ -2377,6 +2489,34 @@ public static void setPhiConvictThreshold(double phiConvictThreshold)
         conf.phi_convict_threshold = phiConvictThreshold;
     }
 
+    public static int getConcurrentCoordinatorReaders()
+    {
+        return conf.concurrent_coordinator_reads;
+    }
+
+    public static void setConcurrentCoordinatorReaders(int concurrent_reads)
+    {
+        if (concurrent_reads < 0)
+        {
+            throw new IllegalArgumentException("Concurrent coordinator readers must be non-negative");
+        }
+        conf.concurrent_coordinator_reads = concurrent_reads;
+    }
+
+    public static int getConcurrentCoordinatorWriters()
+    {
+        return conf.concurrent_coordinator_writes;
+    }
+
+    public static void setConcurrentCoordinatorWriters(int concurrent_writers)
+    {
+        if (concurrent_writers < 0)
+        {
+            throw new IllegalArgumentException("Concurrent coordinator writers must be non-negative");
+        }
+        conf.concurrent_coordinator_writes = concurrent_writers;
+    }
+
     public static int getConcurrentReaders()
     {
         return conf.concurrent_reads;
@@ -2386,7 +2526,7 @@ public static void setConcurrentReaders(int concurrent_reads)
     {
         if (concurrent_reads < 0)
         {
-            throw new IllegalArgumentException("Concurrent reads must be non-negative");
+            throw new IllegalArgumentException("Concurrent readers must be non-negative");
         }
         conf.concurrent_reads = concurrent_reads;
     }
@@ -2400,7 +2540,7 @@ public static void setConcurrentWriters(int concurrent_writers)
     {
         if (concurrent_writers < 0)
         {
-            throw new IllegalArgumentException("Concurrent reads must be non-negative");
+            throw new IllegalArgumentException("Concurrent writers must be non-negative");
         }
         conf.concurrent_writes = concurrent_writers;
     }
@@ -2695,7 +2835,13 @@ public static void setEntireSSTableInterDCStreamThroughputOutboundMebibytesPerSe
      */
     public static boolean useSpecificLocationForLocalSystemData()
     {
-        return conf.local_system_data_file_directory != null;
+        return localSystemDataFileDirectory != null;
+    }
+
+    @VisibleForTesting
+    public static void setSpecificLocationForLocalSystemData(File systemDir)
+    {
+        localSystemDataFileDirectory = systemDir;
     }
 
     /**
@@ -2706,13 +2852,12 @@ public static boolean useSpecificLocationForLocalSystemData()
      *
      * @return the locations where should be stored the local system keyspaces data
      */
-    public static String[] getLocalSystemKeyspacesDataFileLocations()
+    public static File[] getLocalSystemKeyspacesDataFileLocations()
     {
         if (useSpecificLocationForLocalSystemData())
-            return new String[] {conf.local_system_data_file_directory};
+            return new File[] {localSystemDataFileDirectory};
 
-        return conf.data_file_directories.length == 0  ? conf.data_file_directories
-                                                       : new String[] {conf.data_file_directories[0]};
+        return dataDirectories.length == 0  ? dataDirectories : new File[] {dataDirectories[0]};
     }
 
     /**
@@ -2720,9 +2865,9 @@ public static String[] getLocalSystemKeyspacesDataFileLocations()
      *
      * @return the locations where the non local system keyspaces data should be stored.
      */
-    public static String[] getNonLocalSystemKeyspacesDataFileLocations()
+    public static File[] getNonLocalSystemKeyspacesDataFileLocations()
     {
-        return conf.data_file_directories;
+        return dataDirectories;
     }
 
     /**
@@ -2730,23 +2875,73 @@ public static String[] getNonLocalSystemKeyspacesDataFileLocations()
      *
      * @return the list of all the directories where the data files can be stored.
      */
-    public static String[] getAllDataFileLocations()
+    public static File[] getAllDataFileLocations()
     {
-        if (conf.local_system_data_file_directory == null)
-            return conf.data_file_directories;
+        if (!useSpecificLocationForLocalSystemData())
+            return dataDirectories;
 
-        return ArrayUtils.addFirst(conf.data_file_directories, conf.local_system_data_file_directory);
+        return ArrayUtils.addFirst(dataDirectories, localSystemDataFileDirectory);
     }
 
-    public static String getCommitLogLocation()
+    @VisibleForTesting
+    public static void setDataDirectories(File[] newDataDirectories)
     {
-        return conf.commitlog_directory;
+        dataDirectories = newDataDirectories;
+    }
+
+    /**
+     * @return Minimum total space for all the data file directories in GB.
+     *         0 if fail to get the total space, or total space is under 1 GB.
+     */
+    public static long getDataFileDirectoriesMinTotalSpaceInGB()
+    {
+        File[] dataDirectories = getAllDataFileLocations();
+        if (dataDirectories.length == 0)
+        {
+            return 0L;
+        }
+
+        Multiset<FileStore> fileStores = HashMultiset.create();
+        for (File dir : dataDirectories)
+        {
+            try
+            {
+                fileStores.add(Files.getFileStore(dir.toPath()));
+            }
+            catch (IOException ioe)
+            {
+                logger.warn("Unable to get FileStore of {}. {}", dir, ioe);
+            }
+        }
+        return getDataFileDirectoriesMinTotalSpaceInGB(fileStores);
     }
 
     @VisibleForTesting
-    public static void setCommitLogLocation(String value)
+    static long getDataFileDirectoriesMinTotalSpaceInGB(Multiset<FileStore> fileStores)
     {
-        conf.commitlog_directory = value;
+        return fileStores.entrySet().stream().mapToLong(entry -> {
+            long totalSpace = 0L;
+            try
+            {
+                totalSpace = FileUtils.handleLargeFileSystem(entry.getElement().getTotalSpace());
+            }
+            catch (IOException ioe)
+            {
+                logger.warn("Unable to get total space of {}. {}", entry.getElement(), ioe);
+            }
+            return (totalSpace >> 30) / entry.getCount();
+        }).min().orElse(0L) * fileStores.size();
+    }
+
+    public static File getCommitLogLocation()
+    {
+        return commitlogDirectory;
+    }
+
+    @VisibleForTesting
+    public static void setCommitLogLocation(File commitLogLocation)
+    {
+        commitlogDirectory = commitLogLocation;
     }
 
     public static ParameterizedClass getCommitLogCompression()
@@ -2762,7 +2957,9 @@ public static void setCommitLogCompression(ParameterizedClass compressor)
 
     public static Config.FlushCompression getFlushCompression()
     {
-        return conf.flush_compression;
+        return Objects.requireNonNullElseGet(conf.flush_compression, () -> shouldUseAdaptiveCompressionByDefault()
+                                                                           ? Config.FlushCompression.adaptive
+                                                                           : Config.FlushCompression.fast);
     }
 
     public static void setFlushCompression(Config.FlushCompression compression)
@@ -2770,7 +2967,12 @@ public static void setFlushCompression(Config.FlushCompression compression)
         conf.flush_compression = compression;
     }
 
-   /**
+    public static boolean shouldUseAdaptiveCompressionByDefault()
+    {
+        return CassandraRelevantProperties.DEFAULT_SSTABLE_COMPRESSION.getString().equals("adaptive");
+    }
+
+    /**
     * Maximum number of buffers in the compression pool. The default value is 3, it should not be set lower than that
     * (one segment in compression, one written to, one in reserve); delays in compression may cause the log to use
     * more, depending on how soon the sync policy stops all writing threads.
@@ -2790,26 +2992,6 @@ public static int getMaxMutationSize()
         return conf.max_mutation_size.toBytes();
     }
 
-    public static int getTombstoneWarnThreshold()
-    {
-        return conf.tombstone_warn_threshold;
-    }
-
-    public static void setTombstoneWarnThreshold(int threshold)
-    {
-        conf.tombstone_warn_threshold = threshold;
-    }
-
-    public static int getTombstoneFailureThreshold()
-    {
-        return conf.tombstone_failure_threshold;
-    }
-
-    public static void setTombstoneFailureThreshold(int threshold)
-    {
-        conf.tombstone_failure_threshold = threshold;
-    }
-
     public static int getCachedReplicaRowsWarnThreshold()
     {
         return conf.replica_filtering_protection.cached_rows_warn_threshold;
@@ -2873,9 +3055,9 @@ public static void initializeCommitLogDiskAccessMode()
         commitLogWriteDiskAccessMode = accessModeDirectIoPair.left;
     }
 
-    public static String getSavedCachesLocation()
+    public static File getSavedCachesLocation()
     {
-        return conf.saved_caches_directory;
+        return savedCachesDirectory;
     }
 
     public static Set<InetAddressAndPort> getSeeds()
@@ -3534,7 +3716,7 @@ public static long getMaxHintsSizePerHost()
 
     public static File getHintsDirectory()
     {
-        return new File(conf.hints_directory);
+        return hintsDirectory;
     }
 
     public static boolean hintWindowPersistentEnabled()
@@ -3542,11 +3724,22 @@ public static boolean hintWindowPersistentEnabled()
         return conf.hint_window_persistent_enabled;
     }
 
+    public static File getMetadataDirectory()
+    {
+        return metadataDirectory;
+    }
+
+   @VisibleForTesting
+    public static void setMetadataDirectory(File metadataDirectory)
+    {
+        DatabaseDescriptor.metadataDirectory = metadataDirectory;
+    }
+
     public static File getSerializedCachePath(CacheType cacheType, String version, String extension)
     {
         String name = cacheType.toString()
                 + (version == null ? "" : '-' + version + '.' + extension);
-        return new File(conf.saved_caches_directory, name);
+        return new File(savedCachesDirectory, name);
     }
 
     public static int getDynamicUpdateInterval()
@@ -3618,6 +3811,22 @@ public static void setBatchlogReplayThrottleInKiB(int throttleInKiB)
         conf.batchlog_replay_throttle = new DataStorageSpec.IntKibibytesBound(throttleInKiB);
     }
 
+    public static boolean isDynamicEndpointSnitch()
+    {
+        // not using config.dynamic_snitch because snitch can be changed via JMX
+        return snitch instanceof DynamicEndpointSnitch;
+    }
+
+    public static Config.BatchlogEndpointStrategy getBatchlogEndpointStrategy()
+    {
+        return conf.batchlog_endpoint_strategy;
+    }
+
+    public static void setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy batchlogEndpointStrategy)
+    {
+        conf.batchlog_endpoint_strategy = batchlogEndpointStrategy;
+    }
+
     public static int getMaxHintsDeliveryThreads()
     {
         return conf.max_hints_delivery_threads;
@@ -3696,6 +3905,17 @@ public static int getFileCacheSizeInMiB()
         return conf.file_cache_size.toMebibytes();
     }
 
+    public static void disableChunkCache()
+    {
+        conf.file_cache_enabled = false;
+    }
+
+    public static void enableChunkCache(int sizeInMB)
+    {
+        conf.file_cache_enabled = true;
+        conf.file_cache_size = new DataStorageSpec.IntMebibytesBound(sizeInMB);
+    }
+
     public static int getNetworkingCacheSizeInMiB()
     {
         if (conf.networking_cache_size == null)
@@ -3734,11 +3954,15 @@ public static long getTotalCommitlogSpaceInMiB()
         return conf.commitlog_total_space.toMebibytes();
     }
 
+    /** @deprecated CPU-intensive optimization that visibly slows down compaction but does not provide a clear benefit (see STAR-782) */
+    @Deprecated(since = "CC 4.0")
     public static boolean shouldMigrateKeycacheOnCompaction()
     {
         return conf.key_cache_migrate_during_compaction;
     }
 
+    /** @deprecated CPU-intensive optimization that visibly slows down compaction but does not provide a clear benefit (see STAR-782) */
+    @Deprecated(since = "CC 4.0")
     public static void setMigrateKeycacheOnCompaction(boolean migrateCacheEntry)
     {
         conf.key_cache_migrate_during_compaction = migrateCacheEntry;
@@ -3888,6 +4112,31 @@ public static int getStreamingConnectionsPerHost()
         return conf.streaming_connections_per_host;
     }
 
+    public static boolean supportsSSTableReadMeter()
+    {
+        return conf.storage_flags.supports_sstable_read_meter;
+    }
+
+    public static boolean supportsBlacklistingDirectory()
+    {
+        return conf.storage_flags.supports_blacklisting_directory;
+    }
+
+    public static boolean supportsHardlinksForEntireSSTableStreaming()
+    {
+        return conf.storage_flags.supports_hardlinks_for_entire_sstable_streaming;
+    }
+
+    public static boolean supportsFlushBeforeStreaming()
+    {
+        return conf.storage_flags.supports_flush_before_streaming;
+    }
+
+    public static boolean nettyZerocopyEnabled()
+    {
+        return conf.netty_zerocopy_enabled;
+    }
+
     public static boolean streamEntireSSTables()
     {
         return conf.stream_entire_sstables;
@@ -4088,6 +4337,11 @@ public static boolean enableUserDefinedFunctionsThreads()
         return conf.user_defined_functions_threads_enabled;
     }
 
+    public static void enableUserDefinedFunctionsThreads(boolean enabled)
+    {
+        conf.user_defined_functions_threads_enabled = enabled;
+    }
+
     public static long getUserDefinedFunctionWarnTimeout()
     {
         return conf.user_defined_functions_warn_timeout.toMilliseconds();
@@ -4245,9 +4499,9 @@ public static void setCDCOnRepairEnabled(boolean val)
         conf.cdc_on_repair_enabled = val;
     }
 
-    public static String getCDCLogLocation()
+    public static File getCDCLogLocation()
     {
-        return conf.cdc_raw_directory;
+        return cdcRawDirectory;
     }
 
     public static long getCDCTotalSpace()
@@ -4277,11 +4531,6 @@ public static int searchConcurrencyFactor()
         return searchConcurrencyFactor;
     }
 
-    public static boolean isUnsafeSystem()
-    {
-        return unsafeSystem;
-    }
-
     public static boolean diagnosticEventsEnabled()
     {
         return conf.diagnostic_events_enabled;
@@ -5139,11 +5388,6 @@ public static ParameterizedClass getDefaultCompaction()
         return conf != null ? conf.default_compaction : null;
     }
 
-    public static DataStorageSpec.IntMebibytesBound getSAISegmentWriteBufferSpace()
-    {
-        return conf.sai_options.segment_write_buffer_size;
-    }
-
     public static RepairRetrySpec getRepairRetrySpec()
     {
         return conf == null ? new RepairRetrySpec() : conf.repair.retries;
@@ -5178,4 +5422,59 @@ public static void setRejectOutOfTokenRangeRequests(boolean enabled)
     {
         conf.reject_out_of_token_range_requests = enabled;
     }
+
+    public static int getSAISegmentWriteBufferSpace()
+    {
+        return conf == null ? StorageAttachedIndexOptions.DEFAULT_SEGMENT_BUFFER_MB : conf.sai_options.segment_write_buffer_space_mb;
+    }
+
+    public static void setSAISegmentWriteBufferSpace(int bufferSpace)
+    {
+        conf.sai_options.segment_write_buffer_space_mb = bufferSpace;
+    }
+
+    public static double getSAIZeroCopyUsedThreshold()
+    {
+        return conf.sai_options.zerocopy_used_threshold;
+    }
+
+    public static void setSAIZeroCopyUsedThreshold(double threshold)
+    {
+        conf.sai_options.zerocopy_used_threshold = threshold;
+    }
+
+    @VisibleForTesting
+    public static boolean setEmulateDbaasDefaults(boolean dbaasDefaults)
+    {
+        return conf.emulate_dbaas_defaults = dbaasDefaults;
+    }
+
+    public static boolean isEmulateDbaasDefaults()
+    {
+        return conf.emulate_dbaas_defaults;
+    }
+
+    public static PageSize getAggregationSubPageSize()
+    {
+        return PageSize.inBytes(conf.aggregation_subpage_size_in_kb * 1024);
+    }
+
+    public static void setAggregationSubPageSize(PageSize pageSize)
+    {
+        Preconditions.checkArgument(!pageSize.isDefined() || pageSize.getUnit() == PageSize.PageUnit.BYTES);
+        Preconditions.checkArgument(pageSize.bytes() >= 1024);
+        conf.aggregation_subpage_size_in_kb = pageSize.bytes() / 1024;
+    }
+
+    public static double getAnnBruteForceExpenseFactor()
+    {
+        return conf.sai_options.ann_brute_force_factor;
+    }
+
+    public static void setAnnBruteForceExpenseFactor(double factor)
+    {
+        Preconditions.checkArgument(factor > 0.0, "ANN brute force expense factor must be greater than zero");
+        Preconditions.checkArgument(factor <= StorageAttachedIndexOptions.MAXIMUM_ANN_BRUTE_FORCE_FACTOR, "ANN brute force expense factor must be at most " + StorageAttachedIndexOptions.MAXIMUM_ANN_BRUTE_FORCE_FACTOR);
+        conf.sai_options.ann_brute_force_factor = factor;
+    }
 }
diff --git a/src/java/org/apache/cassandra/config/EncryptionOptions.java b/src/java/org/apache/cassandra/config/EncryptionOptions.java
index 9db2406a44dd..3da75aea9835 100644
--- a/src/java/org/apache/cassandra/config/EncryptionOptions.java
+++ b/src/java/org/apache/cassandra/config/EncryptionOptions.java
@@ -26,8 +26,10 @@
 import java.util.Set;
 
 import javax.annotation.Nullable;
+import java.util.function.Supplier;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
 
 import org.slf4j.Logger;
@@ -37,6 +39,7 @@
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.security.DisableSslContextFactory;
 import org.apache.cassandra.security.ISslContextFactory;
+import org.apache.cassandra.security.SSLFactory;
 import org.apache.cassandra.utils.FBUtilities;
 
 /**
@@ -401,6 +404,50 @@ public List<String> getAcceptedProtocols()
     {
         return sslContextFactoryInstance == null ? null : sslContextFactoryInstance.getAcceptedProtocols();
     }
+    
+    /* This list is substituted in configurations that have explicitly specified the original "TLS" default,
+     * by extracting it from the default "TLS" SSL Context instance
+     */
+    private static final Supplier<List<String>> TLS_PROTOCOL_SUBSTITUTION = Suppliers.memoize(SSLFactory::tlsInstanceProtocolSubstitution);
+
+    /**
+     * Combine the pre-4.0 protocol field with the accepted_protocols list, substituting a list of
+     * explicit protocols for the previous catchall default of "TLS"
+     * @return array of protocol names suitable for passing to SslContextBuilder.protocols, or null if the default
+     */
+    public List<String> acceptedProtocols()
+    {
+        if (accepted_protocols == null)
+        {
+            if (protocol == null)
+            {
+                return null;
+            }
+            // TLS is accepted by SSLContext.getInstance as a shorthand for give me an engine that
+            // can speak some of the TLS protocols.  It is not supported by SSLEngine.setAcceptedProtocols
+            // so substitute if the user hasn't provided an accepted protocol configuration
+            else if (protocol.equalsIgnoreCase("TLS"))
+            {
+                return TLS_PROTOCOL_SUBSTITUTION.get();
+            }
+            else // the user was trying to limit to a single specific protocol, so try that
+            {
+                return ImmutableList.of(protocol);
+            }
+        }
+
+        if (protocol != null && !protocol.equalsIgnoreCase("TLS") &&
+            accepted_protocols.stream().noneMatch(ap -> ap.equalsIgnoreCase(protocol)))
+        {
+            // If the user provided a non-generic default protocol, append it to accepted_protocols - they wanted
+            // it after all.
+            return ImmutableList.<String>builder().addAll(accepted_protocols).add(protocol).build();
+        }
+        else
+        {
+            return accepted_protocols;
+        }
+    }
 
     public String[] acceptedProtocolsArray()
     {
diff --git a/src/java/org/apache/cassandra/config/GuardrailsOptions.java b/src/java/org/apache/cassandra/config/GuardrailsOptions.java
index 6e82e7f90882..a5e36416440c 100644
--- a/src/java/org/apache/cassandra/config/GuardrailsOptions.java
+++ b/src/java/org/apache/cassandra/config/GuardrailsOptions.java
@@ -55,21 +55,37 @@
 public class GuardrailsOptions implements GuardrailsConfig
 {
     private static final Logger logger = LoggerFactory.getLogger(GuardrailsOptions.class);
+    public static final int UNSET = -2;
 
     private final Config config;
 
     public GuardrailsOptions(Config config)
     {
         this.config = config;
+
+        // SAI Table Failure threshold (may be overridden via system property)
+        int overrideTableFailureThreshold = CassandraRelevantProperties.INDEX_GUARDRAILS_TABLE_FAILURE_THRESHOLD.getInt(UNSET);
+        if (overrideTableFailureThreshold != UNSET)
+            config.sai_indexes_per_table_fail_threshold = overrideTableFailureThreshold;
+
+        // SAI Total Failure threshold (may be overridden via system property)
+        int overrideTotalFailureThreshold = CassandraRelevantProperties.INDEX_GUARDRAILS_TOTAL_FAILURE_THRESHOLD.getInt(UNSET);
+        if (overrideTotalFailureThreshold != UNSET)
+            config.sai_indexes_total_fail_threshold = overrideTotalFailureThreshold;
+
         validateMaxIntThreshold(config.keyspaces_warn_threshold, config.keyspaces_fail_threshold, "keyspaces");
         validateMaxIntThreshold(config.tables_warn_threshold, config.tables_fail_threshold, "tables");
         validateMaxIntThreshold(config.columns_per_table_warn_threshold, config.columns_per_table_fail_threshold, "columns_per_table");
         validateMaxIntThreshold(config.secondary_indexes_per_table_warn_threshold, config.secondary_indexes_per_table_fail_threshold, "secondary_indexes_per_table");
-        validateMaxIntThreshold(config.materialized_views_per_table_warn_threshold, config.materialized_views_per_table_fail_threshold, "materialized_views_per_table");
+        validateMaxIntThreshold(config.sai_indexes_per_table_warn_threshold, config.sai_indexes_per_table_fail_threshold, "sai_indexes_per_table");
+        validateMaxIntThreshold(config.sai_indexes_total_warn_threshold, config.sai_indexes_total_fail_threshold, "sai_indexes_total");
+        validateMaxIntThreshold(config.sasi_indexes_per_table_warn_threshold, config.sasi_indexes_per_table_fail_threshold, "sasi_indexes_per_table");
+        validateMaxIntThreshold(config.materialized_views_per_table_warn_threshold, config.materialized_views_per_table_fail_threshold, "materialized_views_per_table", true);
         config.table_properties_warned = validateTableProperties(config.table_properties_warned, "table_properties_warned");
         config.table_properties_ignored = validateTableProperties(config.table_properties_ignored, "table_properties_ignored");
         config.table_properties_disallowed = validateTableProperties(config.table_properties_disallowed, "table_properties_disallowed");
         validateMaxIntThreshold(config.page_size_warn_threshold, config.page_size_fail_threshold, "page_size");
+        validateSizeThreshold(config.page_weight_warn_threshold, config.page_weight_fail_threshold, false, "page_weight");
         validateMaxIntThreshold(config.partition_keys_in_select_warn_threshold, config.partition_keys_in_select_fail_threshold, "partition_keys_in_select");
         validateMaxIntThreshold(config.in_select_cartesian_product_warn_threshold, config.in_select_cartesian_product_fail_threshold, "in_select_cartesian_product");
         config.read_consistency_levels_warned = validateConsistencyLevels(config.read_consistency_levels_warned, "read_consistency_levels_warned");
@@ -83,6 +99,7 @@ public GuardrailsOptions(Config config)
         validateMaxIntThreshold(config.items_per_collection_warn_threshold, config.items_per_collection_fail_threshold, "items_per_collection");
         validateMaxIntThreshold(config.fields_per_udt_warn_threshold, config.fields_per_udt_fail_threshold, "fields_per_udt");
         validateMaxIntThreshold(config.vector_dimensions_warn_threshold, config.vector_dimensions_fail_threshold, "vector_dimensions");
+        validateMaxIntThreshold(config.sai_ann_rerank_k_warn_threshold, config.sai_ann_rerank_k_fail_threshold, "sai_ann_rerank_k_max_value");
         validatePercentageThreshold(config.data_disk_usage_percentage_warn_threshold, config.data_disk_usage_percentage_fail_threshold, "data_disk_usage_percentage");
         validateDataDiskUsageMaxDiskSize(config.data_disk_usage_max_disk_size);
         validateMinRFThreshold(config.minimum_replication_factor_warn_threshold, config.minimum_replication_factor_fail_threshold);
@@ -93,6 +110,7 @@ public GuardrailsOptions(Config config)
                                  config.sai_sstable_indexes_per_query_fail_threshold,
                                  "sai_sstable_indexes_per_query",
                                  false);
+        validateMaxIntThreshold(config.tombstone_warn_threshold, config.tombstone_failure_threshold, "tombstone_threshold");
     }
 
     @Override
@@ -195,6 +213,81 @@ public void setSecondaryIndexesPerTableThreshold(int warn, int fail)
                                   x -> config.secondary_indexes_per_table_fail_threshold = x);
     }
 
+    @Override
+    public int getSasiIndexesPerTableWarnThreshold()
+    {
+        return config.sasi_indexes_per_table_warn_threshold;
+    }
+
+    @Override
+    public int getSasiIndexesPerTableFailThreshold()
+    {
+        return config.sasi_indexes_per_table_fail_threshold;
+    }
+
+    public void setSasiIndexesPerTableThreshold(int warn, int fail)
+    {
+        validateMaxIntThreshold(warn, fail, "sasi_indexes_per_table");
+        updatePropertyWithLogging("sasi_indexes_per_table_warn_threshold",
+                                  warn,
+                                  () -> config.sasi_indexes_per_table_warn_threshold,
+                                  x -> config.sasi_indexes_per_table_warn_threshold = x);
+        updatePropertyWithLogging("sai_indexes_per_table_fail_threshold",
+                                  fail,
+                                  () -> config.sasi_indexes_per_table_fail_threshold,
+                                  x -> config.sasi_indexes_per_table_fail_threshold = x);
+    }
+
+    @Override
+    public int getStorageAttachedIndexesPerTableWarnThreshold()
+    {
+        return config.sai_indexes_per_table_warn_threshold;
+    }
+
+    @Override
+    public int getStorageAttachedIndexesPerTableFailThreshold()
+    {
+        return config.sai_indexes_per_table_fail_threshold;
+    }
+
+    public void setStorageAttachedIndexesPerTableThreshold(int warn, int fail)
+    {
+        validateMaxIntThreshold(warn, fail, "sai_indexes_per_table");
+        updatePropertyWithLogging("sai_indexes_per_table_warn_threshold",
+                                  warn,
+                                  () -> config.sai_indexes_per_table_warn_threshold,
+                                  x -> config.sai_indexes_per_table_warn_threshold = x);
+        updatePropertyWithLogging("sai_indexes_per_table_fail_threshold",
+                                  fail,
+                                  () -> config.sai_indexes_per_table_fail_threshold,
+                                  x -> config.sai_indexes_per_table_fail_threshold = x);
+    }
+
+    @Override
+    public int getStorageAttachedIndexesTotalWarnThreshold()
+    {
+        return config.sai_indexes_total_warn_threshold;
+    }
+
+    @Override
+    public int getStorageAttachedIndexesTotalFailThreshold()
+    {
+        return config.sai_indexes_total_fail_threshold;
+    }
+
+    public void setStorageAttachedIndexesTotalThreshold(int warn, int fail)
+    {
+        validateMaxIntThreshold(warn, fail, "sai_indexes_total");
+        updatePropertyWithLogging("sai_indexes_total_warn_threshold",
+                                  warn,
+                                  () -> config.sai_indexes_total_warn_threshold,
+                                  x -> config.sai_indexes_total_warn_threshold = x);
+        updatePropertyWithLogging("sai_indexes_total_fail_threshold",
+                                  fail,
+                                  () -> config.sai_indexes_total_fail_threshold,
+                                  x -> config.sai_indexes_total_fail_threshold = x);
+    }
+
     @Override
     public int getMaterializedViewsPerTableWarnThreshold()
     {
@@ -257,6 +350,20 @@ public int getPageSizeFailThreshold()
         return config.page_size_fail_threshold;
     }
 
+    @Override
+    @Nullable
+    public DataStorageSpec.IntBytesBound getPageWeightWarnThreshold()
+    {
+        return config.page_weight_warn_threshold;
+    }
+
+    @Override
+    @Nullable
+    public DataStorageSpec.IntBytesBound getPageWeightFailThreshold()
+    {
+        return config.page_weight_fail_threshold;
+    }
+
     public void setPageSizeThreshold(int warn, int fail)
     {
         validateMaxIntThreshold(warn, fail, "page_size");
@@ -270,6 +377,19 @@ public void setPageSizeThreshold(int warn, int fail)
                                   x -> config.page_size_fail_threshold = x);
     }
 
+    public void setPageWeightThreshold(DataStorageSpec.IntBytesBound warn, DataStorageSpec.IntBytesBound fail)
+    {
+        validateSizeThreshold(warn, fail, false, "page_weight");
+        updatePropertyWithLogging("page_weight_warn_threshold",
+                                  warn,
+                                  () -> config.page_weight_warn_threshold,
+                                  x -> config.page_weight_warn_threshold = x);
+        updatePropertyWithLogging("page_weight_fail_threshold",
+                                  fail,
+                                  () -> config.page_weight_fail_threshold,
+                                  x -> config.page_weight_fail_threshold = x);
+    }
+
     @Override
     public Set<String> getTablePropertiesWarned()
     {
@@ -340,6 +460,20 @@ public void setGroupByEnabled(boolean enabled)
                                   x -> config.group_by_enabled = x);
     }
 
+    @Override
+    public boolean getLoggedBatchEnabled()
+    {
+        return config.logged_batch_enabled;
+    }
+
+    public void setLoggedBatchEnabled(boolean enabled)
+    {
+        updatePropertyWithLogging("logged_batch_enabled",
+                                  enabled,
+                                  () -> config.logged_batch_enabled,
+                                  x -> config.logged_batch_enabled = x);
+    }
+
     @Override
     public boolean getDropTruncateTableEnabled()
     {
@@ -466,6 +600,20 @@ public void setSimpleStrategyEnabled(boolean enabled)
                                   x -> config.simplestrategy_enabled = x);
     }
 
+    @Override
+    public boolean getCounterEnabled()
+    {
+        return config.counter_enabled;
+    }
+
+    public void setCounterEnabled(boolean enabled)
+    {
+        updatePropertyWithLogging("counter_enabled",
+                enabled,
+                () -> config.counter_enabled,
+                x -> config.counter_enabled = x);
+    }
+
     @Override
     public int getInSelectCartesianProductWarnThreshold()
     {
@@ -639,7 +787,7 @@ public DataStorageSpec.LongBytesBound getCollectionSizeFailThreshold()
         return config.collection_size_fail_threshold;
     }
 
-    public void setCollectionSizeThreshold(@Nullable DataStorageSpec.LongBytesBound warn, @Nullable DataStorageSpec.LongBytesBound fail)
+        public void setCollectionSizeThreshold(@Nullable DataStorageSpec.LongBytesBound warn, @Nullable DataStorageSpec.LongBytesBound fail)
     {
         validateSizeThreshold(warn, fail, false, "collection_size");
         updatePropertyWithLogging("collection_size_warn_threshold",
@@ -727,6 +875,31 @@ public void setVectorDimensionsThreshold(int warn, int fail)
                                   x -> config.vector_dimensions_fail_threshold = x);
     }
 
+    @Override
+    public int getSaiAnnRerankKWarnThreshold()
+    {
+        return config.sai_ann_rerank_k_warn_threshold;
+    }
+
+    @Override
+    public int getSaiAnnRerankKFailThreshold()
+    {
+        return config.sai_ann_rerank_k_fail_threshold;
+    }
+
+    public void setSaiAnnRerankKThreshold(int warn, int fail)
+    {
+        validateMaxIntThreshold(warn, fail, "sai_ann_rerank_k_max_value");
+        updatePropertyWithLogging("sai_ann_rerank_k_warn_threshold",
+                                  warn,
+                                  () -> config.sai_ann_rerank_k_warn_threshold,
+                                  x -> config.sai_ann_rerank_k_warn_threshold = x);
+        updatePropertyWithLogging("sai_ann_rerank_k_fail_threshold",
+                                  fail,
+                                  () -> config.sai_ann_rerank_k_fail_threshold,
+                                  x -> config.sai_ann_rerank_k_fail_threshold = x);
+    }
+
     public int getDataDiskUsagePercentageWarnThreshold()
     {
         return config.data_disk_usage_percentage_warn_threshold;
@@ -1058,6 +1231,54 @@ public void setNonPartitionRestrictedQueryEnabled(boolean enabled)
                                   () -> config.non_partition_restricted_index_query_enabled,
                                   x -> config.non_partition_restricted_index_query_enabled = x);
     }
+    
+    public int getTombstoneWarnThreshold()
+    {
+        return config.tombstone_warn_threshold;
+    }
+
+    @Override
+    public int getTombstoneFailThreshold()
+    {
+        return config.tombstone_failure_threshold;
+    }
+
+    public void setTombstonesThreshold(int warn, int fail)
+    {
+        validateMaxLongThreshold(warn, fail, "tombstones", false);
+        updatePropertyWithLogging("tombstones_warn_threshold",
+                                  warn,
+                                  () -> config.tombstone_warn_threshold,
+                                  x -> config.tombstone_warn_threshold = x);
+        updatePropertyWithLogging("tombstones_fail_threshold",
+                                  fail,
+                                  () -> config.tombstone_failure_threshold,
+                                  x -> config.tombstone_failure_threshold = x);
+    }
+
+    @Override
+    public long getBatchSizeWarnThreshold()
+    {
+        return config.batch_size_warn_threshold.toBytesInLong();
+    }
+
+    @Override
+    public long getBatchSizeFailThreshold()
+    {
+        return config.batch_size_fail_threshold.toBytesInLong();
+    }
+
+    @Override
+    public long getUnloggedBatchAcrossPartitionsWarnThreshold()
+    {
+        return config.unlogged_batch_across_partitions_warn_threshold;
+    }
+
+    @Override
+    public long getUnloggedBatchAcrossPartitionsFailThreshold()
+    {
+        return -1;
+    }
 
     @Override
     public boolean getVectorTypeEnabled()
@@ -1074,6 +1295,56 @@ public void setVectorTypeEnabled(boolean enabled)
                                   x -> config.vector_type_enabled = x);
     }
 
+    @Override
+    public int getOffsetRowsWarnThreshold()
+    {
+        return config.offset_rows_warn_threshold;
+    }
+
+    @Override
+    public int getOffsetRowsFailThreshold()
+    {
+        return config.offset_rows_fail_threshold;
+    }
+
+    public void setOffsetRowsThreshold(int warn, int fail)
+    {
+        validateMaxIntThreshold(warn, fail, "offset_rows", false);
+        updatePropertyWithLogging("offset_rows_warn_threshold",
+                                  warn,
+                                  () -> config.offset_rows_warn_threshold,
+                                  x -> config.offset_rows_warn_threshold = x);
+        updatePropertyWithLogging("offset_rows_fail_threshold",
+                                  fail,
+                                  () -> config.offset_rows_fail_threshold,
+                                  x -> config.offset_rows_fail_threshold = x);
+    }
+
+    @Override
+    public int getQueryFiltersWarnThreshold()
+    {
+        return config.query_filters_warn_threshold;
+    }
+
+    @Override
+    public int getQueryFiltersFailThreshold()
+    {
+        return config.query_filters_fail_threshold;
+    }
+
+    public void setQueryFiltersThreshold(int warn, int fail)
+    {
+        validateMaxIntThreshold(warn, fail, "query_filters", false);
+        updatePropertyWithLogging("query_filters_warn_threshold",
+                                  warn,
+                                  () -> config.query_filters_warn_threshold,
+                                  x -> config.query_filters_warn_threshold = x);
+        updatePropertyWithLogging("query_filters_fail_threshold",
+                                  fail,
+                                  () -> config.query_filters_fail_threshold,
+                                  x -> config.query_filters_fail_threshold = x);
+    }
+
     private static <T> void updatePropertyWithLogging(String propertyName, T newValue, Supplier<T> getter, Consumer<T> setter)
     {
         T oldValue = getter.get();
@@ -1127,6 +1398,13 @@ private static void validateMaxIntThreshold(int warn, int fail, String name)
         validateWarnLowerThanFail(warn, fail, name);
     }
 
+    private static void validateMaxIntThreshold(int warn, int fail, String name, boolean allowZero)
+    {
+        validatePositiveNumeric(warn, Integer.MAX_VALUE, name + "_warn_threshold", allowZero);
+        validatePositiveNumeric(fail, Integer.MAX_VALUE, name + "_fail_threshold", allowZero);
+        validateWarnLowerThanFail(warn, fail, name);
+    }
+
     private static void validateMaxLongThreshold(long warn, long fail, String name, boolean allowZero)
     {
         validatePositiveNumeric(warn, Long.MAX_VALUE, name + "_warn_threshold", allowZero);
@@ -1193,30 +1471,30 @@ private static void validateWarnGreaterThanFail(long warn, long fail, String nam
                                                       "than the fail threshold %d", warn, name, fail));
     }
 
-    private static void validateSize(DataStorageSpec.LongBytesBound size, boolean allowZero, String name)
+    private static void validateSize(DataStorageSpec size, boolean allowZero, String name)
     {
         if (size == null)
             return;
 
-        if (!allowZero && size.toBytes() == 0)
+        if (!allowZero && size.quantity() == 0)
             throw new IllegalArgumentException(format("Invalid value for %s: 0 is not allowed; " +
                                                       "if attempting to disable use an empty value",
                                                       name));
     }
 
-    private static void validateSizeThreshold(DataStorageSpec.LongBytesBound warn, DataStorageSpec.LongBytesBound fail, boolean allowZero, String name)
+    private static void validateSizeThreshold(DataStorageSpec warn, DataStorageSpec fail, boolean allowZero, String name)
     {
         validateSize(warn, allowZero, name + "_warn_threshold");
         validateSize(fail, allowZero, name + "_fail_threshold");
         validateWarnLowerThanFail(warn, fail, name);
     }
 
-    private static void validateWarnLowerThanFail(DataStorageSpec.LongBytesBound warn, DataStorageSpec.LongBytesBound fail, String name)
+    private static void validateWarnLowerThanFail(DataStorageSpec warn, DataStorageSpec fail, String name)
     {
         if (warn == null || fail == null)
             return;
 
-        if (fail.toBytes() < warn.toBytes())
+        if (fail.unit().toBytes(fail.quantity()) < warn.unit().toBytes(warn.quantity()))
             throw new IllegalArgumentException(format("The warn threshold %s for %s_warn_threshold should be lower " +
                                                       "than the fail threshold %s", warn, name, fail));
     }
diff --git a/src/java/org/apache/cassandra/config/ParameterizedClass.java b/src/java/org/apache/cassandra/config/ParameterizedClass.java
index 2d0390ef4486..a4866487ab0c 100644
--- a/src/java/org/apache/cassandra/config/ParameterizedClass.java
+++ b/src/java/org/apache/cassandra/config/ParameterizedClass.java
@@ -18,9 +18,12 @@
 package org.apache.cassandra.config;
 
 import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import com.google.common.base.Objects;
 
@@ -45,8 +48,7 @@ public ParameterizedClass()
 
     public ParameterizedClass(String class_name)
     {
-        this.class_name = class_name;
-        this.parameters = Collections.emptyMap();
+        this(class_name, Collections.emptyMap());
     }
 
     public ParameterizedClass(String class_name, Map<String, String> parameters)
@@ -64,7 +66,7 @@ public ParameterizedClass(Map<String, ?> p)
 
     static public <K> K newInstance(ParameterizedClass parameterizedClass, List<String> searchPackages)
     {
-        Exception last = null;
+        Class<?> providerClass = null;
         if (searchPackages == null || searchPackages.isEmpty())
             searchPackages = Collections.singletonList("");
         for (String searchPackage : searchPackages)
@@ -74,32 +76,56 @@ static public <K> K newInstance(ParameterizedClass parameterizedClass, List<Stri
                 if (!searchPackage.isEmpty() && !searchPackage.endsWith("."))
                     searchPackage = searchPackage + '.';
                 String name = searchPackage + parameterizedClass.class_name;
-                Class<?> providerClass = Class.forName(name);
-                try
-                {
-                    Constructor<?> constructor = providerClass.getConstructor(Map.class);
-                    K instance = (K) constructor.newInstance(parameterizedClass.parameters);
-                    return instance;
-                }
-                catch (Exception constructorEx)
-                {
-                    //no-op
-                }
-                // fallback to no arg constructor if no params present
-                if (parameterizedClass.parameters == null || parameterizedClass.parameters.isEmpty())
-                {
-                    Constructor<?> constructor = providerClass.getConstructor();
-                    K instance = (K) constructor.newInstance();
-                    return instance;
-                }
+                providerClass = Class.forName(name);
             }
-            // there are about 5 checked exceptions that could be thrown here.
-            catch (Exception e)
+            catch (ClassNotFoundException e)
             {
-                last = e;
+                //no-op
             }
         }
-        throw new ConfigurationException("Unable to create parameterized class " + parameterizedClass.class_name, last);
+
+        if (providerClass == null)
+        {
+            String pkgList = '[' + searchPackages.stream().map(p -> '"' + p + '"').collect(Collectors.joining(",")) + ']';
+            String error = "Unable to find class " + parameterizedClass.class_name + " in packages " + pkgList;
+            throw new ConfigurationException(error);
+        }
+
+        try
+        {
+            Constructor<?> mapConstructor = filterConstructor(providerClass, c -> c.getParameterTypes().length == 1 && c.getParameterTypes()[0].equals(Map.class));
+            if (mapConstructor != null)
+                return (K) mapConstructor.newInstance(parameterizedClass.parameters == null ? Collections.emptyMap() : parameterizedClass.parameters);
+
+            // Falls-back to no-arg constructor
+            Constructor<?> noArgsConstructor = filterConstructor(providerClass, c -> c.getParameterTypes().length == 0);
+            if (noArgsConstructor != null)
+                return (K) noArgsConstructor.newInstance();
+
+            throw new ConfigurationException("No valid constructor found for class " + parameterizedClass.class_name);
+        }
+        catch (IllegalAccessException | InstantiationException | ExceptionInInitializerError e)
+        {
+            throw new ConfigurationException("Unable to instantiate parameterized class " + parameterizedClass.class_name, e);
+        }
+        catch (InvocationTargetException e)
+        {
+            Throwable cause = e.getCause();
+            String error = "Failed to instantiate class " + parameterizedClass.class_name +
+                           (cause.getMessage() != null ? ": " + cause.getMessage() : "");
+            throw new ConfigurationException(error, cause);
+        }
+    }
+
+    private static Constructor<?> filterConstructor(Class<?> providerClass, Predicate<Constructor<?>> filter)
+    {
+        for (Constructor<?> constructor : providerClass.getDeclaredConstructors())
+        {
+            if (filter.test(constructor))
+                return constructor;
+        }
+
+        return null;
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/config/StorageAttachedIndexOptions.java b/src/java/org/apache/cassandra/config/StorageAttachedIndexOptions.java
index 18bf3506fed6..d9ca8394f9db 100644
--- a/src/java/org/apache/cassandra/config/StorageAttachedIndexOptions.java
+++ b/src/java/org/apache/cassandra/config/StorageAttachedIndexOptions.java
@@ -17,28 +17,57 @@
  */
 package org.apache.cassandra.config;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
 
 import org.apache.cassandra.exceptions.ConfigurationException;
 
 public class StorageAttachedIndexOptions
 {
     public static final int DEFAULT_SEGMENT_BUFFER_MB = 1024;
+    private static final double DEFAULT_ZEROCOPY_USED_THRESHOLD = 0.3;
+    private static final double DEFAULT_ANN_BRUTE_FORCE_FACTOR = 3.0;
 
-    @VisibleForTesting
-    public static final int MAXIMUM_SEGMENT_BUFFER_MB = 32768;
+    private static final int MAXIMUM_SEGMENT_BUFFER_MB = 32768;
+    public static final double MAXIMUM_ANN_BRUTE_FORCE_FACTOR = 100.0;
 
-    @VisibleForTesting
-    public static final String INVALID_BUFFER_SIZE_ERROR = "Invalid value for segment_write_buffer_size. " +
-                                                           "Value must be a positive integer less than " + MAXIMUM_SEGMENT_BUFFER_MB + "MiB";
-
-    public DataStorageSpec.IntMebibytesBound segment_write_buffer_size = new DataStorageSpec.IntMebibytesBound(DEFAULT_SEGMENT_BUFFER_MB);
+    public int segment_write_buffer_space_mb = DEFAULT_SEGMENT_BUFFER_MB;
+    public double zerocopy_used_threshold = DEFAULT_ZEROCOPY_USED_THRESHOLD;
+    public double ann_brute_force_factor = DEFAULT_ANN_BRUTE_FORCE_FACTOR;
 
     public void validate()
     {
-        if (segment_write_buffer_size.toMebibytes() > MAXIMUM_SEGMENT_BUFFER_MB)
+        if ((segment_write_buffer_space_mb < 0) || (segment_write_buffer_space_mb > MAXIMUM_SEGMENT_BUFFER_MB))
+        {
+            throw new ConfigurationException("Invalid value for segment_write_buffer_space_mb. " +
+                                             "Value must be a positive integer less than " + MAXIMUM_SEGMENT_BUFFER_MB);
+        }
+
+        if ((ann_brute_force_factor <= 0.0) || (ann_brute_force_factor > MAXIMUM_ANN_BRUTE_FORCE_FACTOR))
+        {
+            throw new ConfigurationException(String.format("Invalid value for ann_brute_force_factor. Value must be within (0.0,%s]", MAXIMUM_ANN_BRUTE_FORCE_FACTOR));
+        }
+
+        if ((zerocopy_used_threshold < 0.0) || (zerocopy_used_threshold > 1.0))
         {
-            throw new ConfigurationException(INVALID_BUFFER_SIZE_ERROR);
+            throw new ConfigurationException("Invalid value for zero_copy_used_threshold. " +
+                                             "Value must be between 0.0 and 1.0");
         }
     }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        StorageAttachedIndexOptions that = (StorageAttachedIndexOptions) o;
+        return Objects.equal(segment_write_buffer_space_mb, that.segment_write_buffer_space_mb)
+               && Objects.equal(zerocopy_used_threshold, that.zerocopy_used_threshold)
+               && Objects.equal(ann_brute_force_factor, that.ann_brute_force_factor);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(segment_write_buffer_space_mb, zerocopy_used_threshold);
+    }
 }
diff --git a/src/java/org/apache/cassandra/config/StorageFlagsConfig.java b/src/java/org/apache/cassandra/config/StorageFlagsConfig.java
new file mode 100644
index 000000000000..d7a5b244dce5
--- /dev/null
+++ b/src/java/org/apache/cassandra/config/StorageFlagsConfig.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.config;
+
+import java.util.function.Function;
+
+/**
+ * Configuration settings to disable some features in C* that are normally
+ * not required or not supported on custom file system.
+ *
+ * List of configs to be disable for remote FS:
+ * - {@link Config#index_summary_resize_interval_in_minutes} = -1
+ * - {@link Config#netty_zerocopy_enabled} = false
+ * - {@link DatabaseDescriptor#setCommitLogSegmentMgrProvider(Function)} to append host id to commitlog path
+ */
+public class StorageFlagsConfig
+{
+    public boolean supports_blacklisting_directory = true;
+
+    public boolean supports_sstable_read_meter = true;
+
+    public boolean supports_hardlinks_for_entire_sstable_streaming = true;
+
+    public boolean supports_flush_before_streaming = true;
+}
diff --git a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
index edff7be8ece1..8dd076eedefd 100644
--- a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
+++ b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
@@ -52,6 +52,8 @@
 import org.yaml.snakeyaml.introspector.Property;
 import org.yaml.snakeyaml.introspector.PropertyUtils;
 import org.yaml.snakeyaml.nodes.Node;
+import org.yaml.snakeyaml.parser.ParserImpl;
+import org.yaml.snakeyaml.resolver.Resolver;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_DUPLICATE_CONFIG_KEYS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_NEW_OLD_CONFIG_KEYS;
@@ -222,7 +224,7 @@ public static <T> T fromMap(Map<String,Object> map, boolean shouldCheck, Class<T
         constructor.setPropertyUtils(propertiesChecker);
         Yaml yaml = new Yaml(constructor);
         Node node = yaml.represent(map);
-        constructor.setComposer(new Composer(null, null)
+        constructor.setComposer(new Composer(new ParserImpl(null), new Resolver(), new LoaderOptions())
         {
             @Override
             public Node getSingleNode()
@@ -256,7 +258,7 @@ protected Object newInstance(Node node)
         constructor.setPropertyUtils(propertiesChecker);
         Yaml yaml = new Yaml(constructor);
         Node node = yaml.represent(map);
-        constructor.setComposer(new Composer(null, null)
+        constructor.setComposer(new Composer(new ParserImpl(null), new Resolver(), new LoaderOptions())
         {
             @Override
             public Node getSingleNode()
@@ -275,7 +277,7 @@ static class CustomConstructor extends CustomClassLoaderConstructor
     {
         CustomConstructor(Class<?> theRoot, ClassLoader classLoader)
         {
-            super(theRoot, classLoader);
+            super(theRoot, classLoader, new LoaderOptions());
 
             TypeDescription seedDesc = new TypeDescription(ParameterizedClass.class);
             seedDesc.putMapPropertyType("parameters", String.class, String.class);
diff --git a/src/java/org/apache/cassandra/cql3/AbstractMarker.java b/src/java/org/apache/cassandra/cql3/AbstractMarker.java
index 19ce26c6ff5a..3515e95aff62 100644
--- a/src/java/org/apache/cassandra/cql3/AbstractMarker.java
+++ b/src/java/org/apache/cassandra/cql3/AbstractMarker.java
@@ -150,7 +150,7 @@ public INRaw(int bindIndex)
         private static ColumnSpecification makeInReceiver(ColumnSpecification receiver)
         {
             ColumnIdentifier inName = new ColumnIdentifier("in(" + receiver.name + ")", true);
-            return new ColumnSpecification(receiver.ksName, receiver.cfName, inName, ListType.getInstance(receiver.type, false));
+            return new ColumnSpecification(receiver.ksName, receiver.cfName, inName, ListType.getInstance(receiver.type.freeze(), false));
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/cql3/BatchQueryOptions.java b/src/java/org/apache/cassandra/cql3/BatchQueryOptions.java
index 63be061e5ef3..54efcb918f8d 100644
--- a/src/java/org/apache/cassandra/cql3/BatchQueryOptions.java
+++ b/src/java/org/apache/cassandra/cql3/BatchQueryOptions.java
@@ -22,18 +22,18 @@
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.cassandra.utils.MD5Digest;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.service.QueryState;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
+import org.apache.cassandra.utils.MD5Digest;
 
 public abstract class BatchQueryOptions
 {
     public static BatchQueryOptions DEFAULT = withoutPerStatementVariables(QueryOptions.DEFAULT);
 
-    protected final QueryOptions wrapped;
+    public final QueryOptions wrapped;
     private final List<Object> queryOrIdList;
 
     protected BatchQueryOptions(QueryOptions wrapped, List<Object> queryOrIdList)
@@ -59,6 +59,11 @@ public void prepareStatement(int i, List<ColumnSpecification> boundNames)
         forStatement(i).prepare(boundNames);
     }
 
+    public void updateConsistency(ConsistencyLevel updatedLevel)
+    {
+        wrapped.updateConsistency(updatedLevel);
+    }
+
     public ConsistencyLevel getConsistency()
     {
         return wrapped.getConsistency();
@@ -69,9 +74,14 @@ public String getKeyspace()
         return wrapped.getKeyspace();
     }
 
-    public ConsistencyLevel getSerialConsistency()
+    public ConsistencyLevel getSerialConsistency(QueryState queryState)
     {
-        return wrapped.getSerialConsistency();
+        return wrapped.getSerialConsistency(queryState);
+    }
+
+    public List<List<ByteBuffer>> getVariables()
+    {
+        return Collections.emptyList();
     }
 
     public List<Object> getQueryOrIdList()
@@ -105,10 +115,12 @@ public QueryOptions forStatement(int i)
     private static class WithPerStatementVariables extends BatchQueryOptions
     {
         private final List<QueryOptions> perStatementOptions;
+        private final List<List<ByteBuffer>> variables;
 
         private WithPerStatementVariables(QueryOptions wrapped, List<List<ByteBuffer>> variables, List<Object> queryOrIdList)
         {
             super(wrapped, queryOrIdList);
+            this.variables = variables;
             this.perStatementOptions = new ArrayList<>(variables.size());
             for (final List<ByteBuffer> vars : variables)
             {
@@ -122,6 +134,7 @@ public List<ByteBuffer> getValues()
             }
         }
 
+        @Override
         public QueryOptions forStatement(int i)
         {
             return perStatementOptions.get(i);
@@ -147,8 +160,14 @@ private boolean isPreparedStatement(int i)
         {
             return getQueryOrIdList().get(i) instanceof MD5Digest;
         }
+
+        @Override
+        public List<List<ByteBuffer>> getVariables()
+        {
+            return variables;
+        }
     }
-    
+
     @Override
     public String toString()
     {
diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java b/src/java/org/apache/cassandra/cql3/CQL3Type.java
index 43cc9b3f243b..050947c6c2f1 100644
--- a/src/java/org/apache/cassandra/cql3/CQL3Type.java
+++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java
@@ -21,16 +21,47 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Consumer;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.db.marshal.CollectionType.Kind;
-import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.DurationType;
+import org.apache.cassandra.db.marshal.EmptyType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.SimpleDateType;
+import org.apache.cassandra.db.marshal.TimeType;
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.TypeParser;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.schema.CQLTypeParser;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
@@ -44,7 +75,7 @@
 
 public interface CQL3Type
 {
-    static final Logger logger = LoggerFactory.getLogger(CQL3Type.class);
+    Logger logger = LoggerFactory.getLogger(CQL3Type.class);
 
     default boolean isCollection()
     {
@@ -61,7 +92,7 @@ default boolean isVector()
         return false;
     }
 
-    public AbstractType<?> getType();
+    AbstractType<?> getType();
 
     /**
      * Generates CQL literal from a binary value of this type.
@@ -70,6 +101,56 @@ default boolean isVector()
      */
     String toCQLLiteral(ByteBuffer bytes);
 
+    /**
+     * Generates the string representation of this type, with options regarding the addition of frozen.
+     * <p>
+     * This exists only to provide proper {@code toString} and {@link #toSchemaString} implementations and would
+     * be protected if this was an abstract class rather than an interface (but we cannot change it to an abstract
+     * class easily since {@link Native} is an enum).
+     *
+     * @param alreadyFrozen whether the type is within an already frozen type. If {@code true}, and unless
+     * {@code forceFrozen} is also {@code true} and this type is a complex frozen type itself, the returned
+     * representation will omit the 'frozen<>' prefix. Otherwise, the 'frozen<>' prefix will be included if this type
+     * is (complex and) frozen.
+     * @param forceFrozen force the repetition of 'frozen<>' prefixes before complex frozen types, even if the
+     * context is already frozen. See {@link #toSchemaString} as to why we need this.
+     * @return the type string representation (with or without a 'frozen<>' based on {@code alreadyFrozen} and
+     * {@code forceFrozen}).
+     */
+    String toString(boolean alreadyFrozen, boolean forceFrozen);
+
+    /**
+     * Generate the string representation of this type, like {@code toString()}, but more suitable for schema in the
+     * following 2 ways:
+     *   1. tuples that are frozen have an explicit 'frozen<>' prefix. The reason being that while tuples are frozen by
+     *      default in CQL, we internally replace user types by equivalent tuples when recording dropped columns, and
+     *      that forces us to have "some" support for non-frozen tuples. And thus including an explicit 'frozen<>' is
+     *      _necessary_ to ensure we can distinguish before frozen and non-frozen tuple in that specific case. Note
+     *      this does mean the schema uses a special parsing method ({@link CQLTypeParser#parseDroppedType}) for the
+     *      type of dropped columns.
+     *   2. the 'frozen<>' prefix is repeated for every frozen complex types, even if it is already present at a
+     *      higher level (and is such unnecessary). In other words, this will generate
+     *      {@code frozen<list<frozen<set<int>>>>} while {@code toString()} only generates {@code frozen<list<set<int>>>}.
+     *
+     * <p>Note this representation is still a valid representation which can be parsed back by
+     * {@link CQLTypeParser#parse} and so, since it is a less readable representation, should generally be avoided.
+     * But we use it for column types in the schema, primarily due to point 1. above, but also for the following
+     * historical reason: before DB-3084, the default {@code toString()} representation was behaving like this method,
+     * and drivers have indirectly "relied" on it in the sense that while types on the drivers side have a
+     * {@code isFrozen()} method, it only returns {@code true} if the type have an explicit "frozen<>" prefix just
+     * before it. Meaning that reading {@code frozen<list<set<int>>>} in the schema, the driver (at least the java one)
+     * will return that the list is frozen, but not the set. And indirectly, NGDG was also relying on this to some
+     * extent at the time of this writing.
+     *
+     * <p>We may remove the 2nd part of this method when we've coordinated fixing the drivers side to always set their
+     * {@code isFrozen} to {@code true} for sub-types of frozen types, but for now, to avoid risks in breaking external
+     * consumers, we preserve the verbose, if wasteful, representation in the schema.
+     */
+    default String toSchemaString()
+    {
+        return toString(false, true);
+    }
+
     /**
      * Generates a binary value for the CQL literal of this type
      */
@@ -118,6 +199,7 @@ public enum Native implements CQL3Type
             this.type = type;
         }
 
+        @Override
         public AbstractType<?> getType()
         {
             return type;
@@ -130,19 +212,26 @@ public AbstractType<?> getType()
          * {@link org.apache.cassandra.serializers.TypeSerializer#toString(Object)}
          * {@link org.apache.cassandra.serializers.TypeSerializer#deserialize(ByteBuffer)} implementations.
          */
+        @Override
         public String toCQLLiteral(ByteBuffer buffer)
         {
             return type.getSerializer().toCQLLiteral(buffer);
         }
 
         @Override
-        public String toString()
+        public String toString(boolean alreadyFrozen, boolean forceFrozen)
         {
             return super.toString().toLowerCase();
         }
+
+        @Override
+        public String toString()
+        {
+            return toString(false, false);
+        }
     }
 
-    public static class Custom implements CQL3Type
+    class Custom implements CQL3Type
     {
         private final AbstractType<?> type;
 
@@ -156,6 +245,7 @@ public Custom(String className) throws SyntaxException, ConfigurationException
             this(TypeParser.parse(className));
         }
 
+        @Override
         public AbstractType<?> getType()
         {
             return type;
@@ -185,13 +275,19 @@ public final int hashCode()
         }
 
         @Override
-        public String toString()
+        public String toString(boolean alreadyFrozen, boolean forceFrozen)
         {
             return "'" + type + '\'';
         }
+
+        @Override
+        public String toString()
+        {
+            return toString(false, false);
+        }
     }
 
-    public static class Collection implements CQL3Type
+    class Collection implements CQL3Type
     {
         private final CollectionType<?> type;
 
@@ -200,11 +296,13 @@ public Collection(CollectionType<?> type)
             this.type = type;
         }
 
+        @Override
         public CollectionType<?> getType()
         {
             return type;
         }
 
+        @Override
         public boolean isCollection()
         {
             return true;
@@ -293,36 +391,44 @@ public final int hashCode()
         }
 
         @Override
-        public String toString()
+        public String toString(boolean alreadyFrozen, boolean forceFrozen)
         {
-            boolean isFrozen = !this.type.isMultiCell();
-            StringBuilder sb = new StringBuilder(isFrozen ? "frozen<" : "");
+            boolean addFrozen = !this.type.isMultiCell() && (!alreadyFrozen || forceFrozen);
+            alreadyFrozen = !this.type.isMultiCell() || alreadyFrozen;
+            StringBuilder sb = new StringBuilder(addFrozen ? "frozen<" : "");
             switch (type.kind)
             {
                 case LIST:
                     AbstractType<?> listType = ((ListType<?>) type).getElementsType();
-                    sb.append("list<").append(listType.asCQL3Type());
+                    sb.append("list<").append(listType.asCQL3Type().toString(alreadyFrozen, forceFrozen));
                     break;
                 case SET:
                     AbstractType<?> setType = ((SetType<?>) type).getElementsType();
-                    sb.append("set<").append(setType.asCQL3Type());
+                    sb.append("set<").append(setType.asCQL3Type().toString(alreadyFrozen, forceFrozen));
                     break;
                 case MAP:
                     AbstractType<?> keysType = ((MapType<?, ?>) type).getKeysType();
                     AbstractType<?> valuesType = ((MapType<?, ?>) type).getValuesType();
-                    sb.append("map<").append(keysType.asCQL3Type()).append(", ").append(valuesType.asCQL3Type());
+                    sb.append("map<").append(keysType.asCQL3Type().toString(alreadyFrozen, forceFrozen))
+                      .append(", ").append(valuesType.asCQL3Type().toString(alreadyFrozen, forceFrozen));
                     break;
                 default:
                     throw new AssertionError();
             }
             sb.append('>');
-            if (isFrozen)
+            if (addFrozen)
                 sb.append('>');
             return sb.toString();
         }
+
+        @Override
+        public String toString()
+        {
+            return toString(false, false);
+        }
     }
 
-    public static class UserDefined implements CQL3Type
+    class UserDefined implements CQL3Type
     {
         // Keeping this separatly from type just to simplify toString()
         private final String name;
@@ -339,11 +445,13 @@ public static UserDefined create(UserType type)
             return new UserDefined(UTF8Type.instance.compose(type.name), type);
         }
 
+        @Override
         public boolean isUDT()
         {
             return true;
         }
 
+        @Override
         public AbstractType<?> getType()
         {
             return type;
@@ -410,16 +518,22 @@ public final int hashCode()
         }
 
         @Override
-        public String toString()
+        public String toString(boolean alreadyFrozen, boolean forceFrozen)
         {
-            if (type.isMultiCell())
+            if (type.isMultiCell() || (alreadyFrozen && !forceFrozen))
                 return ColumnIdentifier.maybeQuote(name);
             else
                 return "frozen<" + ColumnIdentifier.maybeQuote(name) + '>';
         }
+
+        @Override
+        public String toString()
+        {
+            return toString(false, false);
+        }
     }
 
-    public static class Tuple implements CQL3Type
+    class Tuple implements CQL3Type
     {
         private final TupleType type;
 
@@ -433,11 +547,13 @@ public static Tuple create(TupleType type)
             return new Tuple(type);
         }
 
+        @Override
         public TupleType getType()
         {
             return type;
         }
 
+        @Override
         public String toCQLLiteral(ByteBuffer buffer)
         {
             if (buffer == null)
@@ -497,32 +613,39 @@ public final int hashCode()
         }
 
         @Override
-        public String toString()
-        {
-            return toString(true);
-        }
-
-        public String toString(boolean withFrozen)
-        {
-            StringBuilder sb = new StringBuilder();
-            if (withFrozen)
-                sb.append("frozen<");
+        public String toString(boolean alreadyFrozen, boolean forceFrozen)
+        {
+            // As tuples are frozen by default, we don't output the frozen<> prefix even when frozen, unless forceFrozen
+            // is explicitly provided. Note that in practice, the type should never be non-frozen unless forceFrozen
+            // is called: if that were to happen, either:
+            //  1. we'd have built a non-frozen tuple for something other than a dropped column
+            //  2. or we'd be calling toString() for a dropped column type, rather that toSchemaString()
+            // Both of which being bugs.
+
+            boolean addFrozen = !type.isMultiCell() && forceFrozen;
+            alreadyFrozen = !type.isMultiCell() || alreadyFrozen;
+            StringBuilder sb = new StringBuilder(addFrozen ? "frozen<" : "");
             sb.append("tuple<");
             for (int i = 0; i < type.size(); i++)
             {
                 if (i > 0)
                     sb.append(", ");
-                sb.append(type.type(i).asCQL3Type());
+                sb.append(type.type(i).asCQL3Type().toString(alreadyFrozen, forceFrozen));
             }
             sb.append('>');
-            if (withFrozen)
+            if (addFrozen)
                 sb.append('>');
-
             return sb.toString();
         }
+
+        @Override
+        public String toString()
+        {
+            return toString(false, false);
+        }
     }
 
-    public static class Vector implements CQL3Type
+    class Vector implements CQL3Type
     {
         private final VectorType<?> type;
 
@@ -536,6 +659,7 @@ public Vector(AbstractType<?> elementType, int dimensions)
             this.type = VectorType.getInstance(elementType, dimensions);
         }
 
+        @Override
         public boolean isVector()
         {
             return true;
@@ -583,17 +707,23 @@ public int hashCode()
         }
 
         @Override
-        public String toString()
+        public String toString(boolean alreadyFrozen, boolean forceFrozen)
         {
             StringBuilder sb = new StringBuilder();
             sb.append("vector<").append(type.elementType.asCQL3Type()).append(", ").append(type.dimension).append('>');
             return sb.toString();
         }
+
+        @Override
+        public String toString()
+        {
+            return toString(false, false);
+        }
     }
 
     // For UserTypes, we need to know the current keyspace to resolve the
     // actual type used, so Raw is a "not yet prepared" CQL3Type.
-    public abstract class Raw
+    abstract class Raw
     {
         protected final boolean frozen;
 
@@ -644,6 +774,8 @@ public String keyspace()
             return null;
         }
 
+        public abstract void forEachUserType(Consumer<UTName> userTypeNameConsumer);
+
         public Raw freeze()
         {
             String message = String.format("frozen<> is only allowed on collections, tuples, and user-defined types (got %s)", this);
@@ -652,6 +784,12 @@ public Raw freeze()
 
         public abstract void validate(ClientState state, String name);
 
+        /**
+         * Prepare this raw type given the current keyspace to obtain a proper {@link CQL3Type}.
+         *
+         * @param keyspace the keyspace the type is part of.
+         * @return the prepared type.
+         */
         public CQL3Type prepare(String keyspace)
         {
             KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
@@ -660,13 +798,16 @@ public CQL3Type prepare(String keyspace)
             return prepare(keyspace, ksm.types);
         }
 
+        /**
+         * Prepare this raw type given the current keyspace and its defined user types to obtain a proper
+         * {@link CQL3Type}.
+         *
+         * @param keyspace the keyspace the type is part of.
+         * @param udts user types defined in {@code keyspace}, which this type may rely on.
+         * @return the prepared type.
+         */
         public abstract CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException;
 
-        public CQL3Type prepareInternal(String keyspace, Types udts) throws InvalidRequestException
-        {
-            return prepare(keyspace, udts);
-        }
-
         public boolean referencesUserType(String name)
         {
             return false;
@@ -677,6 +818,11 @@ public static Raw from(CQL3Type type)
             return new RawType(type, false);
         }
 
+        public static Raw custom(String className)
+        {
+            return new RawCustom(className, false);
+        }
+
         public static Raw userType(UTName name)
         {
             return new RawUT(name, false);
@@ -699,7 +845,17 @@ public static Raw set(CQL3Type.Raw t)
 
         public static Raw tuple(List<CQL3Type.Raw> ts)
         {
-            return new RawTuple(ts);
+            return new RawTuple(ts, false, false);
+        }
+
+        public static Raw tuple(List<CQL3Type.Raw> ts, boolean frozenDefault)
+        {
+            return new RawTuple(ts, false, frozenDefault);
+        }
+
+        private static Raw freezeIfSupported(Raw type)
+        {
+            return type != null && type.supportsFreezing() ? type.freeze() : type;
         }
 
         public static Raw vector(CQL3Type.Raw t, int dimension)
@@ -728,26 +884,36 @@ public void validate(ClientState state, String name)
                 }
             }
 
+            @Override
             public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
             {
                 return type;
             }
 
+            @Override
             public boolean supportsFreezing()
             {
                 return false;
             }
 
+            @Override
             public boolean isCounter()
             {
                 return type == Native.COUNTER;
             }
 
+            @Override
             public boolean isDuration()
             {
                 return type == Native.DURATION;
             }
 
+            @Override
+            public void forEachUserType(Consumer<UTName> userTypeNameConsumer)
+            {
+                // no-op
+            }
+
             @Override
             public String toString()
             {
@@ -755,6 +921,58 @@ public String toString()
             }
         }
 
+        private static class RawCustom extends Raw
+        {
+            private final String className;
+
+            private RawCustom(String className, boolean frozen)
+            {
+                super(frozen);
+                this.className = className;
+            }
+
+            @Override
+            public boolean supportsFreezing()
+            {
+                // Technically, we cannot know if the resulting type may be frozen or not, so allow the frozen keyword,
+                // but it might well basically do nothing when used.
+                return true;
+            }
+
+            @Override
+            public RawCustom freeze() throws InvalidRequestException
+            {
+                return new RawCustom(className, true);
+            }
+
+            @Override
+            public void validate(ClientState state, String name)
+            {
+                // no-op
+            }
+
+            @Override
+            public CQL3Type prepare(String keyspace, Types udts)
+            {
+                AbstractType<?> type = TypeParser.parse(className);
+                if (frozen)
+                    type = type.freeze();
+                return new Custom(type);
+            }
+
+            @Override
+            public void forEachUserType(Consumer<UTName> userTypeNameConsumer)
+            {
+                // no-op
+            }
+
+            @Override
+            public String toString()
+            {
+                return '"' + className + '"';
+            }
+        }
+
         private static class RawCollection extends Raw
         {
             private final CollectionType.Kind kind;
@@ -772,29 +990,15 @@ private RawCollection(CollectionType.Kind kind, CQL3Type.Raw keys, CQL3Type.Raw
             @Override
             public RawCollection freeze()
             {
-                CQL3Type.Raw frozenKeys =
-                    null != keys && keys.supportsFreezing()
-                  ? keys.freeze()
-                  : keys;
-
-                CQL3Type.Raw frozenValues =
-                    null != values && values.supportsFreezing()
-                  ? values.freeze()
-                  : values;
-
-                return new RawCollection(kind, frozenKeys, frozenValues, true);
+                return new RawCollection(kind, Raw.freezeIfSupported(keys), Raw.freezeIfSupported(values), true);
             }
 
+            @Override
             public boolean supportsFreezing()
             {
                 return true;
             }
 
-            public boolean isCollection()
-            {
-                return true;
-            }
-
             @Override
             public void validate(ClientState state, String name)
             {
@@ -805,41 +1009,11 @@ public void validate(ClientState state, String name)
                     values.validate(state, name);
             }
 
-            public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
-            {
-                return prepare(keyspace, udts, false);
-            }
-
-            public CQL3Type prepareInternal(String keyspace, Types udts)
-            {
-                return prepare(keyspace, udts, true);
-            }
-
-            public CQL3Type prepare(String keyspace, Types udts, boolean isInternal) throws InvalidRequestException
+            @Override
+            public CQL3Type prepare(String keyspace, Types udts)
             {
                 assert values != null : "Got null values type for a collection";
 
-                if (!frozen && values.supportsFreezing() && !values.frozen)
-                    throwNestedNonFrozenError(values);
-
-                // we represent supercolumns as maps, internally, and we do allow counters in supercolumns. Thus,
-                // for internal type parsing (think schema) we have to make an exception and allow counters as (map) values
-                if (values.isCounter() && !isInternal)
-                    throw new InvalidRequestException("Counters are not allowed inside collections: " + this);
-
-                if (values.isDuration() && kind == Kind.SET)
-                    throw new InvalidRequestException("Durations are not allowed inside sets: " + this);
-
-                if (keys != null)
-                {
-                    if (keys.isCounter())
-                        throw new InvalidRequestException("Counters are not allowed inside collections: " + this);
-                    if (keys.isDuration())
-                        throw new InvalidRequestException("Durations are not allowed as map keys: " + this);
-                    if (!frozen && keys.supportsFreezing() && !keys.frozen)
-                        throwNestedNonFrozenError(keys);
-                }
-
                 AbstractType<?> valueType = values.prepare(keyspace, udts).getType();
                 switch (kind)
                 {
@@ -851,20 +1025,23 @@ public CQL3Type prepare(String keyspace, Types udts, boolean isInternal) throws
                         assert keys != null : "Got null keys type for a collection";
                         return new Collection(MapType.getInstance(keys.prepare(keyspace, udts).getType(), valueType, !frozen));
                 }
+
                 throw new AssertionError();
             }
 
-            private void throwNestedNonFrozenError(Raw innerType)
+            @Override
+            public boolean referencesUserType(String name)
             {
-                if (innerType instanceof RawCollection)
-                    throw new InvalidRequestException("Non-frozen collections are not allowed inside collections: " + this);
-                else if (innerType.isUDT())
-                    throw new InvalidRequestException("Non-frozen UDTs are not allowed inside collections: " + this);
+                return (keys != null && keys.referencesUserType(name)) || values.referencesUserType(name);
             }
 
-            public boolean referencesUserType(String name)
+            @Override
+            public void forEachUserType(Consumer<UTName> userTypeNameConsumer)
             {
-                return (keys != null && keys.referencesUserType(name)) || values.referencesUserType(name);
+                if (keys != null)
+                    keys.forEachUserType(userTypeNameConsumer);
+                if (values != null)
+                    values.forEachUserType(userTypeNameConsumer);
             }
 
             @Override
@@ -932,6 +1109,12 @@ public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestExcept
                 return new Vector(type.getType(), dimension);
             }
 
+            @Override
+            public void forEachUserType(Consumer<UTName> userTypeNameConsumer)
+            {
+                element.forEachUserType(userTypeNameConsumer);
+            }
+
             @Override
             public String toString()
             {
@@ -949,11 +1132,18 @@ private RawUT(UTName name, boolean frozen)
                 this.name = name;
             }
 
+            @Override
             public String keyspace()
             {
                 return name.getKeyspace();
             }
 
+            @Override
+            public void forEachUserType(Consumer<UTName> userTypeNameConsumer)
+            {
+                userTypeNameConsumer.accept(name);
+            }
+
             @Override
             public RawUT freeze()
             {
@@ -966,6 +1156,7 @@ public void validate(ClientState state, String name)
                 // nothing to do here
             }
 
+            @Override
             public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
             {
                 if (name.hasKeyspace())
@@ -991,16 +1182,19 @@ public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestExcept
                 return new UserDefined(name.toString(), type);
             }
 
+            @Override
             public boolean referencesUserType(String name)
             {
                 return this.name.getStringTypeName().equals(name);
             }
 
+            @Override
             public boolean supportsFreezing()
             {
                 return true;
             }
 
+            @Override
             public boolean isUDT()
             {
                 return true;
@@ -1019,24 +1213,26 @@ public String toString()
         private static class RawTuple extends Raw
         {
             private final List<CQL3Type.Raw> types;
+            private final boolean frozenByDefault;
 
-            private RawTuple(List<CQL3Type.Raw> types)
+            private RawTuple(List<CQL3Type.Raw> types, boolean frozen, boolean frozenByDefault)
             {
-                super(true);
-                this.types = types.stream()
-                                  .map(t -> t.supportsFreezing() ? t.freeze() : t)
-                                  .collect(toList());
+                super(frozen);
+                this.types = types;
+                this.frozenByDefault = frozenByDefault;
             }
 
+            @Override
             public boolean supportsFreezing()
             {
                 return true;
             }
 
             @Override
-            public RawTuple freeze()
+            public RawTuple freeze() throws InvalidRequestException
             {
-                return this;
+                List<CQL3Type.Raw> frozenTypes = types.stream().map(Raw::freezeIfSupported).collect(toList());
+                return new RawTuple(frozenTypes, true, frozenByDefault);
             }
 
             @Override
@@ -1046,29 +1242,44 @@ public void validate(ClientState state, String name)
                     t.validate(state, name);
             }
 
+            @Override
             public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
             {
-                List<AbstractType<?>> ts = new ArrayList<>(types.size());
-                for (CQL3Type.Raw t : types)
+                // Externally, tuples are frozen by default, so should always be frozen. But because we store UDTs as
+                // tuples in dropped types (so we don't have to store separately the definition of dropped UDT) and we
+                // now support non-frozen UDTs, we have to at least internally support non-frozen tuples. Note however
+                // that we only support non-frozen UDTs at "top-level", any such UDTs can only have frozen subtypes, so
+                // we can safely freeze any subtype.
+                RawTuple frozenTuple = freeze();
+                List<AbstractType<?>> ts = new ArrayList<>(frozenTuple.types.size());
+                for (CQL3Type.Raw t : frozenTuple.types)
                 {
-                    if (t.isCounter())
-                        throw new InvalidRequestException("Counters are not allowed inside tuples");
-
                     ts.add(t.prepare(keyspace, udts).getType());
                 }
-                return new Tuple(new TupleType(ts));
+
+                // The inner types have been frozen, as mentioned above, the top-level type is also frozen unless it
+                // has !frozenByDefault (and wasn't manually frozen, so !frozen).
+                return new Tuple(new TupleType(ts, !frozen && !frozenByDefault));
             }
 
+            @Override
             public boolean isTuple()
             {
                 return true;
             }
 
+            @Override
             public boolean referencesUserType(String name)
             {
                 return types.stream().anyMatch(t -> t.referencesUserType(name));
             }
 
+            @Override
+            public void forEachUserType(Consumer<UTName> userTypeNameConsumer)
+            {
+                types.forEach(t -> t.forEachUserType(userTypeNameConsumer));
+            }
+
             @Override
             public String toString()
             {
diff --git a/src/java/org/apache/cassandra/cql3/CQLStatement.java b/src/java/org/apache/cassandra/cql3/CQLStatement.java
index 1ac36befcdd5..f8bb5c37a40d 100644
--- a/src/java/org/apache/cassandra/cql3/CQLStatement.java
+++ b/src/java/org/apache/cassandra/cql3/CQLStatement.java
@@ -29,6 +29,14 @@
 
 public interface CQLStatement
 {
+    /**
+     * The query string that produced that statement, if available.
+     *
+     * @return the raw query string that produced that statement, or {@code null} if said string is not available
+     * (typically because the statement has not been built from a string). Note that said string may contain bind markers.
+     */
+    String getRawCQLStatement();
+
     /**
      * Returns all bind variables for the statement
      */
@@ -102,8 +110,19 @@ default public boolean hasConditions()
 
     public static abstract class Raw
     {
+        protected String rawCQLStatement;
         protected VariableSpecifications bindVariables;
 
+        public void setRawCQLStatement(String queryString)
+        {
+            this.rawCQLStatement = queryString;
+        }
+
+        public String getRawCQLStatement()
+        {
+            return rawCQLStatement;
+        }
+
         public void setBindVariables(List<ColumnIdentifier> variables)
         {
             bindVariables = new VariableSpecifications(variables);
@@ -112,8 +131,14 @@ public void setBindVariables(List<ColumnIdentifier> variables)
         public abstract CQLStatement prepare(ClientState state);
     }
 
+    /**
+     * A marker for the statements (prepared) which run against an exact keyspace.
+     */
     public static interface SingleKeyspaceCqlStatement extends CQLStatement
     {
+        /**
+         * Returns a keyspace name associated with this statement.
+         */
         public String keyspace();
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
index 26aeb85795a0..5480636ebfd5 100644
--- a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
+++ b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
@@ -28,6 +28,7 @@
 import org.apache.cassandra.cache.IMeasurableMemory;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.memory.ByteBufferCloner;
@@ -195,6 +196,21 @@ public String toCQLString()
         return maybeQuote(text);
     }
 
+    public static String toCQLString(ByteBuffer name)
+    {
+        try
+        {
+            return maybeQuote(UTF8Type.instance.getString(name));
+        }
+        catch (MarshalException e)
+        {
+            // Note: some (hopefully very, very rare) legacy tables (from thrift) could have column names that
+            // are not text-based, so this could theoretically be triggered. Usually, ColumnIdentifier handles this,
+            // but this method is called in places where it's not worth bothering.
+            return ByteBufferUtil.toDebugHexString(name);
+        }
+    }
+
     public long unsharedHeapSize()
     {
         return EMPTY_SIZE
@@ -228,6 +244,16 @@ public static String maybeQuote(String text)
     {
         if (UNQUOTED_IDENTIFIER.matcher(text).matches() && !ReservedKeywords.isReserved(text))
             return text;
+        return quote(text);
+    }
+
+    /**
+     * Adds double quotes to the given text
+     * @param text the text to double-quote
+     * @return the double-quoted text
+     */
+    public static String quote(String text)
+    {
         return '"' + PATTERN_DOUBLE_QUOTE.matcher(text).replaceAll(ESCAPED_DOUBLE_QUOTE) + '"';
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/ColumnSpecification.java b/src/java/org/apache/cassandra/cql3/ColumnSpecification.java
index 8cf869bd38d7..2c5a7fc126b7 100644
--- a/src/java/org/apache/cassandra/cql3/ColumnSpecification.java
+++ b/src/java/org/apache/cassandra/cql3/ColumnSpecification.java
@@ -17,15 +17,16 @@
  */
 package org.apache.cassandra.cql3;
 
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.function.UnaryOperator;
+
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
 
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ReversedType;
 
-import java.util.Collection;
-import java.util.Iterator;
-
 public class ColumnSpecification
 {
     public final String ksName;
@@ -76,6 +77,19 @@ public static boolean allInSameTable(Collection<ColumnSpecification> names)
         return true;
     }
 
+    public ColumnSpecification withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+    {
+        if (keyspaceMapper == Constants.IDENTITY_STRING_MAPPER)
+            return this;
+
+        String newKeyspaceName = keyspaceMapper.apply(ksName);
+
+        if (ksName.equals(newKeyspaceName))
+            return this;
+
+        return new ColumnSpecification(newKeyspaceName, cfName, name, type.overrideKeyspace(keyspaceMapper));
+    }
+
     @Override
     public boolean equals(Object other)
     {
diff --git a/src/java/org/apache/cassandra/cql3/Constants.java b/src/java/org/apache/cassandra/cql3/Constants.java
index be81bed3b047..777dff23d5ab 100644
--- a/src/java/org/apache/cassandra/cql3/Constants.java
+++ b/src/java/org/apache/cassandra/cql3/Constants.java
@@ -21,10 +21,27 @@
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.function.UnaryOperator;
 
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.DurationType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.NumberType;
+import org.apache.cassandra.db.marshal.StringType;
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -163,6 +180,8 @@ public AbstractType<?> getExactTypeIfKnown(String keyspace)
 
     public static final Value UNSET_VALUE = new Value(ByteBufferUtil.UNSET_BYTE_BUFFER);
 
+    public static final UnaryOperator<String> IDENTITY_STRING_MAPPER = id -> id;
+
     private static class NullLiteral extends Term.Raw
     {
         public Term prepare(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
@@ -211,7 +230,7 @@ public String toString()
 
     public static class Literal extends Term.Raw
     {
-        private final Type type;
+        public final Type type;
         private final String text;
         private final AbstractType<?> preferedType;
 
@@ -268,8 +287,7 @@ public Value prepare(String keyspace, ColumnSpecification receiver) throws Inval
 
         private ByteBuffer parsedValue(AbstractType<?> validator) throws InvalidRequestException
         {
-            if (validator instanceof ReversedType<?>)
-                validator = ((ReversedType<?>) validator).baseType;
+            validator = validator.unwrap();
             try
             {
                 if (type == Type.HEX)
diff --git a/src/java/org/apache/cassandra/cql3/CqlBuilder.java b/src/java/org/apache/cassandra/cql3/CqlBuilder.java
index d3ca1da1db3a..9e88feac8da8 100644
--- a/src/java/org/apache/cassandra/cql3/CqlBuilder.java
+++ b/src/java/org/apache/cassandra/cql3/CqlBuilder.java
@@ -22,6 +22,8 @@
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.db.marshal.AbstractType;
 
@@ -66,6 +68,19 @@ public CqlBuilder append(String str)
         return this;
     }
 
+    public CqlBuilder appendTypeQuotingIfNeeded(String str)
+    {
+        return append(maybeQuoteTypeName(str));
+    }
+
+    @VisibleForTesting
+    public static String maybeQuoteTypeName(String nameAsString)
+    {
+        return ReservedTypeNames.isReserved(nameAsString)
+               ? ColumnIdentifier.quote(nameAsString)
+               : ColumnIdentifier.maybeQuote(nameAsString);
+    }
+
     public CqlBuilder appendQuotingIfNeeded(String str)
     {
         return append(ColumnIdentifier.maybeQuote(str));
diff --git a/src/java/org/apache/cassandra/cql3/GeoDistanceRelation.java b/src/java/org/apache/cassandra/cql3/GeoDistanceRelation.java
new file mode 100644
index 000000000000..cbffc794725a
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/GeoDistanceRelation.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.cassandra.cql3.restrictions.Restriction;
+import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction;
+import org.apache.cassandra.cql3.statements.Bound;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
+/**
+ * A relation using a GEO_DISTANCE function to get vectors of kind lat, lon within a given distance.
+ * Examples:
+ * <ul>
+ * <li>SELECT ... WHERE GEO_DISTANCE(geopoint, :querypoint) &lt; :distance</li>
+ * </ul>
+ */
+public class GeoDistanceRelation extends Relation
+{
+    private final ColumnIdentifier entity;
+    private final Term.Raw point;
+    private final Term.Raw distance;
+
+    public GeoDistanceRelation(ColumnIdentifier entity, Term.Raw point, Operator type, Term.Raw distance)
+    {
+        this.entity = entity;
+        this.point = point;
+        this.relationType = type;
+        this.distance = distance;
+    }
+
+    public Term.Raw getValue()
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    public List<? extends Term.Raw> getInValues()
+    {
+        return null;
+    }
+
+    @Override
+    protected Restriction newSliceRestriction(TableMetadata table,
+                                              VariableSpecifications boundNames,
+                                              Bound bound,
+                                              boolean inclusive)
+    {
+        ColumnMetadata columnDef = table.getExistingColumn(entity);
+        if (bound == Bound.START)
+            throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+        if (!(columnDef.type instanceof VectorType) || ((VectorType<?>) columnDef.type).dimension != 2)
+            throw invalidRequest("GEO_DISTANCE is only supported against vector<float, 2> columns");
+
+        Term pointTerm = toTerm(Collections.singletonList(columnDef), point, table.keyspace, boundNames);
+
+        var vectorRadius = new ColumnSpecification(columnDef.ksName, columnDef.cfName,
+                                                   new ColumnIdentifier("radius", true), FloatType.instance);
+        Term distanceTerm = toTerm(Collections.singletonList(vectorRadius), distance, table.keyspace, boundNames);
+        return new SingleColumnRestriction.BoundedAnnRestriction(columnDef, pointTerm, distanceTerm, inclusive);
+    }
+
+    @Override
+    protected Restriction newEQRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+
+    }
+
+    @Override
+    protected Restriction newNEQRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newNotContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newINRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newNotINRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newIsNotRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newLikeRestriction(TableMetadata table, VariableSpecifications boundNames, Operator operator)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newAnnRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newBm25Restriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Restriction newAnalyzerMatchesRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the GEO_DISTANCE function", operator());
+    }
+
+    @Override
+    protected Term toTerm(List<? extends ColumnSpecification> receivers,
+                          Term.Raw raw,
+                          String keyspace,
+                          VariableSpecifications boundNames) throws InvalidRequestException
+    {
+        Term term = raw.prepare(keyspace, receivers.get(0));
+        term.collectMarkerSpecification(boundNames);
+        return term;
+    }
+
+    @Override
+    public Relation renameIdentifier(ColumnIdentifier from, ColumnIdentifier to)
+    {
+        if (!entity.equals(from))
+            return this;
+
+        return new GeoDistanceRelation(to, point, operator(), distance);
+    }
+
+    @Override
+    public String toCQLString()
+    {
+        return String.format("GEO_DISTANCE(%s, %s) %s %s", entity.toCQLString(), point, relationType, distance);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(relationType, entity, point, distance);
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o)
+            return true;
+
+        if (!(o instanceof GeoDistanceRelation))
+            return false;
+
+        GeoDistanceRelation geoDistanceRelation = (GeoDistanceRelation) o;
+        return relationType == geoDistanceRelation.relationType
+               && entity.equals(geoDistanceRelation.entity)
+               && point.equals(geoDistanceRelation.point)
+               && distance.equals(geoDistanceRelation.distance);
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/Lists.java b/src/java/org/apache/cassandra/cql3/Lists.java
index 605a8bee8e6d..4cda39822139 100644
--- a/src/java/org/apache/cassandra/cql3/Lists.java
+++ b/src/java/org/apache/cassandra/cql3/Lists.java
@@ -21,24 +21,27 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.schema.ColumnMetadata;
 import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.ListType;
-import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.serializers.CollectionSerializer;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -65,14 +68,9 @@ public static ColumnSpecification valueSpecOf(ColumnSpecification column)
         return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), elementsType(column.type));
     }
 
-    private static AbstractType<?> unwrap(AbstractType<?> type)
-    {
-        return type.isReversed() ? unwrap(((ReversedType<?>) type).baseType) : type;
-    }
-
     private static AbstractType<?> elementsType(AbstractType<?> type)
     {
-        return ((ListType<?>) unwrap(type)).getElementsType();
+        return ((ListType<?>) type.unwrap()).getElementsType();
     }
 
     /**
@@ -84,10 +82,10 @@ private static AbstractType<?> elementsType(AbstractType<?> type)
     public static AssignmentTestable.TestResult testListAssignment(ColumnSpecification receiver,
                                                                    List<? extends AssignmentTestable> elements)
     {
-        if (!(receiver.type instanceof ListType))
+        if (!(receiver.type instanceof ListType || receiver.type instanceof VectorType))
             return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
 
-        // If there is no elements, we can't say it's an exact match (an empty list if fundamentally polymorphic).
+        // If there is no elements, we can't say it's an exact match (an empty list is fundamentally polymorphic).
         if (elements.isEmpty())
             return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
 
@@ -131,8 +129,30 @@ public static <T> String listToString(Iterable<T> items, java.util.function.Func
     public static <T> ListType<?> getExactListTypeIfKnown(List<T> items,
                                                           java.util.function.Function<T, AbstractType<?>> mapper)
     {
-        Optional<AbstractType<?>> type = items.stream().map(mapper).filter(Objects::nonNull).findFirst();
-        return type.isPresent() ? ListType.getInstance(type.get(), false) : null;
+        AbstractType<?> type = getElementType(items, mapper);
+        return type != null ? ListType.getInstance(type, false) : null;
+    }
+
+    protected static <T> AbstractType<?> getElementType(List<T> items,
+                                                        java.util.function.Function<T, AbstractType<?>> mapper)
+    {
+        AbstractType<?> type = null;
+        for (T item : items)
+        {
+            AbstractType<?> itemType = mapper.apply(item);
+            if (itemType == null)
+                continue;
+
+            if (type != null && !itemType.isCompatibleWith(type))
+            {
+                if (type.isCompatibleWith(itemType))
+                    continue;
+
+                throw new InvalidRequestException("Invalid collection literal: all selectors must have the same CQL type inside collection literals");
+            }
+            type = itemType;
+        }
+        return type;
     }
 
     public static <T> ListType<?> getPreferredCompatibleType(List<T> items,
@@ -140,7 +160,7 @@ public static <T> ListType<?> getPreferredCompatibleType(List<T> items,
     {
         Set<AbstractType<?>> types = items.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toSet());
         AbstractType<?> type = AssignmentTestable.getCompatibleTypeIfKnown(types);
-        return type == null ? null : ListType.getInstance(type, false);
+        return type == null ? null : ListType.getInstance(type.freeze(), false);
     }
 
     public static class Literal extends Term.Raw
@@ -177,7 +197,7 @@ public Term prepare(String keyspace, ColumnSpecification receiver) throws Invali
 
         private void validateAssignableTo(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
         {
-            AbstractType<?> type = unwrap(receiver.type);
+            AbstractType<?> type = receiver.type.unwrap();
 
             if (!(type instanceof ListType))
                 throw new InvalidRequestException(String.format("Invalid list literal for %s of type %s", receiver.name, receiver.type.asCQL3Type()));
@@ -265,9 +285,9 @@ public List<ByteBuffer> getElements()
     }
 
     /**
-     * Basically similar to a Value, but with some non-pure function (that need
+     * Basically similar to a Value, but with some non-deterministic function (that need
      * to be evaluated at execution time) in it.
-     *
+     * </p>
      * Note: this would also work for a list with bind markers, but we don't support
      * that because 1) it's not excessively useful and 2) we wouldn't have a good
      * column name to return in the ColumnSpecification for those markers (not a
@@ -459,6 +479,7 @@ public void collectMarkerSpecification(VariableSpecifications boundNames)
             idx.collectMarkerSpecification(boundNames);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             // we should not get here for frozen lists
@@ -498,6 +519,7 @@ public Appender(ColumnMetadata column, Term t)
             super(column, t);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to append to a frozen list";
@@ -565,10 +587,16 @@ public void execute(DecoratedKey partitionKey, UpdateParameters params) throws I
             List<ByteBuffer> toAdd = ((Value) value).elements;
             final int totalCount = toAdd.size();
 
+            // Guardrails about collection size are only checked for the added elements without considering
+            // already existent elements. This is done so to avoid read-before-write, having additional checks
+            // during SSTable write.
+            Guardrails.itemsPerCollection.guard(totalCount, column.name.toString(), false, params.clientState);
+
             // we have to obey MAX_NANOS per batch - in the unlikely event a client has decided to prepend a list with
             // an insane number of entries.
             PrecisionTime pt = null;
             int remainingInBatch = 0;
+            int dataSize = 0;
             for (int i = totalCount - 1; i >= 0; i--)
             {
                 if (remainingInBatch == 0)
@@ -580,8 +608,10 @@ public void execute(DecoratedKey partitionKey, UpdateParameters params) throws I
 
                 // TODO: is this safe as part of LWTs?
                 ByteBuffer uuid = ByteBuffer.wrap(atUnixMillisAsBytes(pt.millis, (pt.nanos + remainingInBatch--)));
-                params.addCell(column, CellPath.create(uuid), toAdd.get(i));
+                Cell cell = params.addCell(column, CellPath.create(uuid), toAdd.get(i));
+                dataSize += cell.dataSize();
             }
+            Guardrails.collectionSize.guard(dataSize, column.name.toString(), false, params.clientState);
         }
     }
 
@@ -598,6 +628,7 @@ public boolean requiresRead()
             return true;
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to delete from a frozen list";
@@ -639,6 +670,7 @@ public boolean requiresRead()
             return true;
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to delete an item by index from a frozen list";
diff --git a/src/java/org/apache/cassandra/cql3/Maps.java b/src/java/org/apache/cassandra/cql3/Maps.java
index 9040d1a8d599..1bbbe5c3102f 100644
--- a/src/java/org/apache/cassandra/cql3/Maps.java
+++ b/src/java/org/apache/cassandra/cql3/Maps.java
@@ -30,17 +30,16 @@
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.marshal.ReversedType;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.CellPath;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.serializers.CollectionSerializer;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -66,19 +65,14 @@ public static ColumnSpecification valueSpecOf(ColumnSpecification column)
         return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), valuesType(column.type));
     }
 
-    private static AbstractType<?> unwrap(AbstractType<?> type)
-    {
-        return type.isReversed() ? unwrap(((ReversedType<?>) type).baseType) : type;
-    }
-
     private static AbstractType<?> keysType(AbstractType<?> type)
     {
-        return ((MapType<?, ?>) unwrap(type)).getKeysType();
+        return ((MapType<?, ?>) type.unwrap()).getKeysType();
     }
 
     private static AbstractType<?> valuesType(AbstractType<?> type)
     {
-        return ((MapType<?, ?>) unwrap(type)).getValuesType();
+        return ((MapType<?, ?>) type.unwrap()).getValuesType();
     }
 
     /**
@@ -148,16 +142,30 @@ public static <T> String mapToString(List<Pair<T, T>> items,
         AbstractType<?> valueType = null;
         for (Pair<T, T> entry : entries)
         {
-            if (keyType == null)
-                keyType = mapper.apply(entry.left);
-            if (valueType == null)
-                valueType = mapper.apply(entry.right);
-            if (keyType != null && valueType != null)
-                return MapType.getInstance(keyType, valueType, false);
+            keyType = selectType(keyType, mapper.apply(entry.left));
+            valueType = selectType(valueType, mapper.apply(entry.right));
         }
+
+        if (keyType != null && valueType != null)
+            return MapType.getInstance(keyType.freeze(), valueType.freeze(), false);
         return null;
     }
 
+    private static <T> AbstractType<?> selectType(AbstractType<?> type, AbstractType<?> otherType)
+    {
+        if (otherType == null)
+            return type;
+
+        if (type != null && !otherType.isCompatibleWith(type))
+        {
+            if (type.isCompatibleWith(otherType))
+                return type;
+
+            throw new InvalidRequestException("Invalid collection literal: all selectors must have the same CQL type inside collection literals");
+        }
+        return otherType;
+    }
+
     public static <T> MapType<?, ?> getPreferredCompatibleType(List<Pair<T, T>> entries,
                                                                java.util.function.Function<T, AbstractType<?>> mapper)
     {
@@ -171,7 +179,7 @@ public static <T> String mapToString(List<Pair<T, T>> items,
         if (valueType == null)
             return null;
 
-        return  MapType.getInstance(keyType, valueType, false);
+        return  MapType.getInstance(keyType.freeze(), valueType.freeze(), false);
     }
 
     public static class Literal extends Term.Raw
@@ -210,7 +218,7 @@ public Term prepare(String keyspace, ColumnSpecification receiver) throws Invali
 
         private void validateAssignableTo(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
         {
-            AbstractType<?> type = unwrap(receiver.type);
+            AbstractType<?> type = receiver.type.unwrap();
 
             if (!(type instanceof MapType))
                 throw new InvalidRequestException(String.format("Invalid map literal for %s of type %s", receiver.name, receiver.type.asCQL3Type()));
@@ -388,6 +396,7 @@ public Setter(ColumnMetadata column, Term t)
             super(column, t);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             Term.Terminal value = t.bind(params.options);
@@ -418,6 +427,7 @@ public void collectMarkerSpecification(VariableSpecifications boundNames)
             k.collectMarkerSpecification(boundNames);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to set a value for a single key on a frozen map";
@@ -448,6 +458,7 @@ public Putter(ColumnMetadata column, Term t)
             super(column, t);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to add items to a frozen map";
@@ -503,6 +514,7 @@ public DiscarderByKey(ColumnMetadata column, Term k)
             super(column, k);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to delete a single key in a frozen map";
diff --git a/src/java/org/apache/cassandra/cql3/MarkerOrList.java b/src/java/org/apache/cassandra/cql3/MarkerOrList.java
new file mode 100644
index 000000000000..47ea805e1419
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/MarkerOrList.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.cql3.functions.Function;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkBindValueSet;
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
+
+/**
+ * Represents either a single marker for IN values, or a list of IN values, like:
+ * "SELECT ... WHERE x IN ?"
+ * "SELECT ... WHERE x IN (?, ?, ?)"
+ * "SELECT ... WHERE x IN (a, b, c)"
+ */
+public class MarkerOrList
+{
+    final AbstractMarker marker;
+    final List<Term> values;
+
+    private MarkerOrList(AbstractMarker marker, List<Term> values)
+    {
+        assert ((marker != null) ^ (values != null));
+        this.marker = marker;
+        this.values = values;
+    }
+
+    public static MarkerOrList marker(AbstractMarker marker)
+    {
+        return new MarkerOrList(marker, null);
+    }
+
+    public static MarkerOrList list(List<Term> values)
+    {
+        return new MarkerOrList(null, values);
+    }
+
+    public void addFunctionsTo(List<Function> functions)
+    {
+        if (values != null)
+        {
+            for (Term term : values)
+            {
+                if (term != null)
+                    term.addFunctionsTo(functions);
+            }
+        }
+    }
+
+    public List<ByteBuffer> bindAndGet(QueryOptions options, ColumnIdentifier column)
+    {
+        // (NOT IN | IN) ?,  where ? bound to a list of values
+        if (marker != null)
+        {
+            Term.Terminal term = marker.bind(options);
+            checkNotNull(term, "Invalid null value for column %s", column);
+            checkBindValueSet(term.get(options.getProtocolVersion()), "Invalid unset value for column %s", column);
+            Term.MultiItemTerminal lval = (Term.MultiItemTerminal) term;
+            return lval.getElements();
+        }
+        else
+        // (NOT IN | IN) (value1, value2, ...)
+        {
+            assert values != null;
+            List<ByteBuffer> result = new ArrayList<>(values.size());
+            for (Term term : values)
+            {
+                ByteBuffer value = term.bindAndGet(options);
+                result.add(value);
+            }
+            return result;
+        }
+    }
+
+    public List<List<ByteBuffer>> bindAndGetTuples(QueryOptions options, Collection<ColumnIdentifier> columns)
+    {
+        if (columns.isEmpty())
+            throw new IllegalArgumentException("Columns must not be empty");
+
+        // (NOT IN | IN) ?,  where ? bound to a list of tuples
+        if (marker != null && marker instanceof Tuples.InMarker)
+        {
+            Tuples.InMarker inMarker = (Tuples.InMarker) marker;
+            Tuples.InValue inValue = inMarker.bind(options);
+            checkNotNull(inValue, "Invalid null value for columns %s", columns);
+            return inValue.getSplitValues();
+        }
+        // (NOT IN | IN) ?,  where ? bound to a list of scalar values, we need to convert the elements to singleton lists
+        else if (marker != null)
+        {
+            ColumnIdentifier firstColumn = columns.iterator().next();
+            List<ByteBuffer> values = bindAndGet(options, firstColumn);
+            return values.stream().map(Collections::singletonList).collect(Collectors.toList());
+        }
+        // (NOT IN | IN) (value1, value2, ...)
+        else
+        {
+            assert values != null;
+            return values.stream()
+                         .map(v -> getTuple(v, options))
+                         .collect(Collectors.toList());
+        }
+    }
+
+    private List<ByteBuffer> getTuple(Term value, QueryOptions options)
+    {
+        Term.Terminal terminal = value.bind(options);
+        return (terminal instanceof Term.MultiItemTerminal)
+            ? ((Term.MultiItemTerminal) terminal).getElements()
+            : Collections.singletonList(terminal.get(options.getProtocolVersion()));
+    }
+
+    public String toString()
+    {
+        return (marker != null) ? marker.toString(): values.toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/MarkerOrTerms.java b/src/java/org/apache/cassandra/cql3/MarkerOrTerms.java
new file mode 100644
index 000000000000..d145d9f9ef3d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/MarkerOrTerms.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.cql3.functions.Function;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkBindValueSet;
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
+
+/**
+ * Represents either a single marker for IN values, or a list of IN values, like:
+ * "SELECT ... WHERE x IN ?"
+ * "SELECT ... WHERE x IN (?, ?, ?)"
+ * "SELECT ... WHERE x IN (a, b, c)"
+ */
+public interface MarkerOrTerms
+{
+    void addFunctionsTo(List<Function> functions);
+
+    List<ByteBuffer> bindAndGet(QueryOptions options, ColumnIdentifier column);
+
+    List<List<ByteBuffer>> bindAndGetTuples(QueryOptions options, Collection<ColumnIdentifier> columns);
+
+    /**
+     * {@link MarkerOrTerms} representing a single marker for IN values, as in {@code SELECT ... WHERE x IN ?}
+     */
+    class Marker implements MarkerOrTerms
+    {
+        private final AbstractMarker marker;
+
+        public Marker(AbstractMarker marker)
+        {
+            this.marker = marker;
+        }
+
+        @Override
+        public void addFunctionsTo(List<Function> functions)
+        {
+            // nothing to do here
+        }
+
+        @Override
+        public List<ByteBuffer> bindAndGet(QueryOptions options, ColumnIdentifier column)
+        {
+            // (NOT IN | IN) ?,  where ? bound to a list of values
+            Term.Terminal term = marker.bind(options);
+            checkNotNull(term, "Invalid null value for column %s", column);
+            checkBindValueSet(term.get(options.getProtocolVersion()), "Invalid unset value for column %s", column);
+            Term.MultiItemTerminal lval = (Term.MultiItemTerminal) term;
+            return lval.getElements();
+        }
+
+        public List<List<ByteBuffer>> bindAndGetTuples(QueryOptions options, Collection<ColumnIdentifier> columns)
+        {
+            if (columns.isEmpty())
+                throw new IllegalArgumentException("Columns must not be empty");
+
+            // (NOT IN | IN) ?,  where ? bound to a list of tuples
+            if (marker instanceof Tuples.InMarker)
+            {
+                Tuples.InMarker inMarker = (Tuples.InMarker) marker;
+                Tuples.InValue inValue = inMarker.bind(options);
+                checkNotNull(inValue, "Invalid null value for columns %s", columns);
+                return inValue.getSplitValues();
+            }
+            // (NOT IN | IN) ?,  where ? bound to a list of scalar values, we need to convert the elements to singleton lists
+            ColumnIdentifier firstColumn = columns.iterator().next();
+            List<ByteBuffer> values = bindAndGet(options, firstColumn);
+            return values.stream().map(Collections::singletonList).collect(Collectors.toList());
+        }
+
+        @Override
+        public String toString()
+        {
+            return marker.toString();
+        }
+    }
+
+    /**
+     * {@link MarkerOrTerms} representing a list of IN terms, like:
+     * {@code SELECT ... WHERE x IN (?, ?, ?)}
+     * {@code SELECT ... WHERE x IN (a, b, c)}
+     */
+    class Terms implements MarkerOrTerms
+    {
+        private final List<Term> terms;
+
+        public Terms(List<Term> terms)
+        {
+            this.terms = terms;
+        }
+
+        @Override
+        public void addFunctionsTo(List<Function> functions)
+        {
+            if (terms != null)
+            {
+                for (Term term : terms)
+                {
+                    if (term != null)
+                        term.addFunctionsTo(functions);
+                }
+            }
+        }
+
+        @Override
+        public List<ByteBuffer> bindAndGet(QueryOptions options, ColumnIdentifier column)
+        {
+            // (NOT IN | IN) (value1, value2, ...)
+            List<ByteBuffer> result = new ArrayList<>(terms.size());
+            for (Term term : terms)
+            {
+                ByteBuffer value = term.bindAndGet(options);
+                result.add(value);
+            }
+            return result;
+        }
+
+        @Override
+        public List<List<ByteBuffer>> bindAndGetTuples(QueryOptions options, Collection<ColumnIdentifier> columns)
+        {
+            if (columns.isEmpty())
+                throw new IllegalArgumentException("Columns must not be empty");
+
+            // (NOT IN | IN) (value1, value2, ...)
+            assert terms != null;
+            return terms.stream()
+                        .map(v -> getTuple(v, options))
+                        .collect(Collectors.toList());
+        }
+
+        private static List<ByteBuffer> getTuple(Term value, QueryOptions options)
+        {
+            Term.Terminal terminal = value.bind(options);
+            return (terminal instanceof Term.MultiItemTerminal)
+                   ? ((Term.MultiItemTerminal) terminal).getElements()
+                   : Collections.singletonList(terminal.get(options.getProtocolVersion()));
+        }
+
+        @Override
+        public String toString()
+        {
+            return terms.toString();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java b/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java
index e74b9624d8e2..f56d76e2ced9 100644
--- a/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java
+++ b/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.cql3;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -95,6 +96,18 @@ public static MultiColumnRelation createInRelation(List<ColumnIdentifier> entiti
         return new MultiColumnRelation(entities, Operator.IN, null, inValues, null);
     }
 
+    /**
+     * Creates a multi-column NOT IN relation with a list of NOT IN values or markers.
+     * For example: "SELECT ... WHERE (a, b) NOT IN ((0, 1), (2, 3))"
+     * @param entities the columns on the LHS of the relation
+     * @param inValues a list of Tuples.Literal instances or a Tuples.Raw markers
+     * @return a new <code>MultiColumnRelation</code> instance
+     */
+    public static MultiColumnRelation createNotInRelation(List<ColumnIdentifier> entities, List<? extends Term.MultiColumnRaw> inValues)
+    {
+        return new MultiColumnRelation(entities, Operator.NOT_IN, null, inValues, null);
+    }
+
     /**
      * Creates a multi-column IN relation with a marker for the IN values.
      * For example: "SELECT ... WHERE (a, b) IN ?"
@@ -107,6 +120,18 @@ public static MultiColumnRelation createSingleMarkerInRelation(List<ColumnIdenti
         return new MultiColumnRelation(entities, Operator.IN, null, null, inMarker);
     }
 
+    /**
+     * Creates a multi-column NOT IN relation with a marker for the NOT IN values.
+     * For example: "SELECT ... WHERE (a, b) NOT IN ?"
+     * @param entities the columns on the LHS of the relation
+     * @param inMarker a single IN marker
+     * @return a new <code>MultiColumnRelation</code> instance
+     */
+    public static MultiColumnRelation createSingleMarkerNotInRelation(List<ColumnIdentifier> entities, Tuples.INRaw inMarker)
+    {
+        return new MultiColumnRelation(entities, Operator.NOT_IN, null, null, inMarker);
+    }
+
     public List<ColumnIdentifier> getEntities()
     {
         return entities;
@@ -118,12 +143,12 @@ public List<ColumnIdentifier> getEntities()
      */
     public Term.MultiColumnRaw getValue()
     {
-        return relationType == Operator.IN ? inMarker : valuesOrMarker;
+        return (relationType == Operator.IN || relationType == Operator.NOT_IN) ? inMarker : valuesOrMarker;
     }
 
     public List<? extends Term.Raw> getInValues()
     {
-        assert relationType == Operator.IN;
+        assert relationType == Operator.IN || relationType == Operator.NOT_IN;
         return inValues;
     }
 
@@ -141,6 +166,15 @@ protected Restriction newEQRestriction(TableMetadata table, VariableSpecificatio
         return new MultiColumnRestriction.EQRestriction(receivers, term);
     }
 
+    @Override
+    protected Restriction newNEQRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        List<ColumnMetadata> receivers = receivers(table);
+        Term term = toTerm(receivers, getValue(), table.keyspace, boundNames);
+        MarkerOrTerms skippedValues = new MarkerOrTerms.Terms(Collections.singletonList(term));
+        return MultiColumnRestriction.SliceRestriction.fromSkippedValues(receivers, skippedValues);
+    }
+
     @Override
     protected Restriction newINRestriction(TableMetadata table, VariableSpecifications boundNames)
     {
@@ -149,13 +183,32 @@ protected Restriction newINRestriction(TableMetadata table, VariableSpecificatio
         if (terms == null)
         {
             Term term = toTerm(receivers, getValue(), table.keyspace, boundNames);
-            return new MultiColumnRestriction.InRestrictionWithMarker(receivers, (AbstractMarker) term);
+            return new MultiColumnRestriction.INRestriction(receivers, new MarkerOrTerms.Marker((AbstractMarker) term));
         }
 
         if (terms.size() == 1)
             return new MultiColumnRestriction.EQRestriction(receivers, terms.get(0));
 
-        return new MultiColumnRestriction.InRestrictionWithValues(receivers, terms);
+        return new MultiColumnRestriction.INRestriction(receivers, new MarkerOrTerms.Terms(terms));
+    }
+
+    @Override
+    protected Restriction newNotINRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        List<ColumnMetadata> receivers = receivers(table);
+        List<Term> terms = toTerms(receivers, inValues, table.keyspace, boundNames);
+        MarkerOrTerms values;
+        if (terms == null)
+        {
+            Term term = toTerm(receivers, getValue(), table.keyspace, boundNames);
+            values = new MarkerOrTerms.Marker((AbstractMarker) term);
+        }
+        else
+        {
+            values = new MarkerOrTerms.Terms(terms);
+        }
+
+        return MultiColumnRestriction.SliceRestriction.fromSkippedValues(receivers, values);
     }
 
     @Override
@@ -163,7 +216,7 @@ protected Restriction newSliceRestriction(TableMetadata table, VariableSpecifica
     {
         List<ColumnMetadata> receivers = receivers(table);
         Term term = toTerm(receivers(table), getValue(), table.keyspace, boundNames);
-        return new MultiColumnRestriction.SliceRestriction(receivers, bound, inclusive, term);
+        return MultiColumnRestriction.SliceRestriction.fromBound(receivers, bound, inclusive, term);
     }
 
     @Override
@@ -172,6 +225,12 @@ protected Restriction newContainsRestriction(TableMetadata table, VariableSpecif
         throw invalidRequest("%s cannot be used for multi-column relations", operator());
     }
 
+    @Override
+    protected Restriction newNotContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey)
+    {
+        throw invalidRequest("%s cannot be used for multi-column relations", operator());
+    }
+
     @Override
     protected Restriction newIsNotRestriction(TableMetadata table, VariableSpecifications boundNames)
     {
@@ -185,6 +244,24 @@ protected Restriction newLikeRestriction(TableMetadata table, VariableSpecificat
         throw invalidRequest("%s cannot be used for multi-column relations", operator());
     }
 
+    @Override
+    protected Restriction newAnnRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used for multi-column relations", operator());
+    }
+
+    @Override
+    protected Restriction newBm25Restriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used for multi-column relations", operator());
+    }
+
+    @Override
+    protected Restriction newAnalyzerMatchesRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used for multi-column relations", operator());
+    }
+
     @Override
     protected Term toTerm(List<? extends ColumnSpecification> receivers,
                           Raw raw,
diff --git a/src/java/org/apache/cassandra/cql3/Operation.java b/src/java/org/apache/cassandra/cql3/Operation.java
index 51f5d1304ec3..aeffe138889e 100644
--- a/src/java/org/apache/cassandra/cql3/Operation.java
+++ b/src/java/org/apache/cassandra/cql3/Operation.java
@@ -386,7 +386,7 @@ else if (!(receiver.type.isMultiCell()))
                     ColumnSpecification vr = new ColumnSpecification(receiver.ksName,
                                                                      receiver.cfName,
                                                                      receiver.name,
-                                                                     SetType.getInstance(((MapType)receiver.type).getKeysType(), false));
+                                                                     SetType.getInstance(((MapType)receiver.type).getKeysType().freeze(), false));
                     Term term;
                     try
                     {
diff --git a/src/java/org/apache/cassandra/cql3/Operator.java b/src/java/org/apache/cassandra/cql3/Operator.java
index 55d6779cfe1e..c99941bc1151 100644
--- a/src/java/org/apache/cassandra/cql3/Operator.java
+++ b/src/java/org/apache/cassandra/cql3/Operator.java
@@ -21,12 +21,14 @@
 import java.io.DataOutput;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
 
-import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.serializers.ListSerializer;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 public enum Operator
@@ -40,8 +42,15 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
+            if (indexAnalyzer != null)
+                return ANALYZER_MATCHES.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+
             return type.compareForCQL(leftOperand, rightOperand) == 0;
         }
     },
@@ -54,7 +63,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return type.compareForCQL(leftOperand, rightOperand) < 0;
         }
@@ -68,7 +81,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return type.compareForCQL(leftOperand, rightOperand) <= 0;
         }
@@ -82,7 +99,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return type.compareForCQL(leftOperand, rightOperand) >= 0;
         }
@@ -96,7 +117,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return type.compareForCQL(leftOperand, rightOperand) > 0;
         }
@@ -109,9 +134,13 @@ public String toString()
             return "IN";
         }
 
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
-            ListSerializer<?> serializer = ListType.getInstance(type, false).getSerializer();
+            ListSerializer<?> serializer = ListType.getInstance(type.freeze(), false).getSerializer();
             return serializer.anyMatch(rightOperand, r -> type.compareForCQL(leftOperand, r) == 0);
         }
     },
@@ -124,25 +153,14 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
-            switch(((CollectionType<?>) type).kind)
-            {
-                case LIST :
-                    ListType<?> listType = (ListType<?>) type;
-                    List<?> list = listType.getSerializer().deserialize(leftOperand);
-                    return list.contains(listType.getElementsType().getSerializer().deserialize(rightOperand));
-                case SET:
-                    SetType<?> setType = (SetType<?>) type;
-                    Set<?> set = setType.getSerializer().deserialize(leftOperand);
-                    return set.contains(setType.getElementsType().getSerializer().deserialize(rightOperand));
-                case MAP:
-                    MapType<?, ?> mapType = (MapType<?, ?>) type;
-                    Map<?, ?> map = mapType.getSerializer().deserialize(leftOperand);
-                    return map.containsValue(mapType.getValuesType().getSerializer().deserialize(rightOperand));
-                default:
-                    throw new AssertionError();
-            }
+            CollectionType<?> collectionType = (CollectionType<?>) type;
+            return collectionType.contains(leftOperand, rightOperand);
         }
     },
     CONTAINS_KEY(6)
@@ -154,13 +172,17 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             MapType<?, ?> mapType = (MapType<?, ?>) type;
-            Map<?, ?> map = mapType.getSerializer().deserialize(leftOperand);
-            return map.containsKey(mapType.getKeysType().getSerializer().deserialize(rightOperand));
+            return mapType.containsKey(leftOperand, rightOperand);
         }
     },
+
     NEQ(8)
     {
         @Override
@@ -170,7 +192,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return type.compareForCQL(leftOperand, rightOperand) != 0;
 
@@ -185,7 +211,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             throw new UnsupportedOperationException();
         }
@@ -199,7 +229,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return ByteBufferUtil.startsWith(leftOperand, rightOperand);
         }
@@ -213,7 +247,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return ByteBufferUtil.endsWith(leftOperand, rightOperand);
         }
@@ -227,7 +265,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return ByteBufferUtil.contains(leftOperand, rightOperand);
         }
@@ -240,7 +282,11 @@ public String toString()
             return "LIKE '<term>'";
         }
 
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             return ByteBufferUtil.contains(leftOperand, rightOperand);
         }
@@ -254,7 +300,11 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
             throw new UnsupportedOperationException();
         }
@@ -268,15 +318,292 @@ public String toString()
         }
 
         @Override
-        public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand)
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
         {
-            // The ANN operator is only supported by the vector index so, normally, should never be called directly.
-            // In networked queries (non-local) the coordinator will end up calling the row filter directly. So, this
-            // needs to return true so that the returned values are allowed through to the VectorTopKProcessor
-            return true;
+            throw new UnsupportedOperationException();
+        }
+    },
+    NOT_IN(16)
+    {
+        @Override
+        public String toString()
+        {
+            return "NOT IN";
+        }
+
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            return !IN.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+        }
+    },
+    NOT_CONTAINS(17)
+    {
+        @Override
+        public String toString()
+        {
+            return "NOT CONTAINS";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            return !CONTAINS.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+        }
+
+    },
+    NOT_CONTAINS_KEY(18)
+    {
+        @Override
+        public String toString()
+        {
+            return "NOT CONTAINS KEY";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            return !CONTAINS_KEY.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+        }
+    },
+    NOT_LIKE_PREFIX(19)
+    {
+        @Override
+        public String toString()
+        {
+            return "NOT LIKE '<term>%'";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            return !LIKE_PREFIX.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+        }
+    },
+    NOT_LIKE_SUFFIX(20)
+    {
+        @Override
+        public String toString()
+        {
+            return "NOT LIKE '%<term>'";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            return !LIKE_SUFFIX.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+        }
+    },
+    NOT_LIKE_CONTAINS(21)
+    {
+        @Override
+        public String toString()
+        {
+            return "NOT LIKE '%<term>%'";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            return !LIKE_CONTAINS.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+        }
+    },
+    NOT_LIKE_MATCHES(22)
+    {
+        @Override
+        public String toString()
+        {
+            return "NOT LIKE '<term>'";
+        }
+
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            return !LIKE_MATCHES.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+        }
+    },
+    NOT_LIKE(23)
+    {
+        @Override
+        public String toString()
+        {
+            return "NOT LIKE";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            return !LIKE.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer);
+        }
+    },
+    /**
+     * An operator that only performs matching against analyzed columns.
+     */
+    ANALYZER_MATCHES(100)
+    {
+        @Override
+        public String toString()
+        {
+            return ":";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            assert indexAnalyzer != null && queryAnalyzer != null : ": operation can only be computed by an indexed column with a configured analyzer";
+
+            List<ByteBuffer> rightTokens = queryAnalyzer.analyze(rightOperand);
+            return isSatisfiedBy(type, leftOperand, rightTokens, indexAnalyzer);
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftValue,
+                                     List<ByteBuffer> rightTokens,
+                                     Index.Analyzer indexAnalyzer)
+        {
+            List<ByteBuffer> leftTokens = indexAnalyzer.analyze(leftValue);
+            Iterator<ByteBuffer> it = rightTokens.iterator();
+
+            do
+            {
+                if (!it.hasNext())
+                    return true;
+            }
+            while (hasToken(type, leftTokens, it.next()));
+
+            return false;
+        }
+
+        private boolean hasToken(AbstractType<?> type, List<ByteBuffer> tokens, ByteBuffer token)
+        {
+            for (ByteBuffer t : tokens)
+            {
+                if (type.compareForCQL(t, token) == 0)
+                    return true;
+            }
+            return false;
+        }
+    },
+
+    /**
+     * An operator that performs a distance bounded approximate nearest neighbor search against a vector column such
+     * that all result vectors are within a given distance of the query vector. The notable difference between this
+     * operator and {@link #ANN} is that it does not introduce an arbitrary limit on the number of results returned,
+     * and as a consequence, it can be logically combined with other predicates and even unioned with other
+     * {@code #BOUNDED_ANN} predicates.
+     */
+    BOUNDED_ANN(101)
+    {
+        @Override
+        public String toString()
+        {
+            return "BOUNDED_ANN";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            throw new UnsupportedOperationException();
+        }
+    },
+    // VSTODO find a way to break this out, but for now, we're doing this to get the poc running
+    ORDER_BY_ASC(102)
+    {
+        @Override
+        public String toString()
+        {
+            return "ASC";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            throw new UnsupportedOperationException();
+        }
+    },
+    ORDER_BY_DESC(103)
+    {
+        @Override
+        public String toString()
+        {
+            return "DESC";
         }
-    };
 
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type, 
+                                     ByteBuffer leftOperand, 
+                                     ByteBuffer rightOperand, 
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            throw new UnsupportedOperationException();
+        }
+    },
+    BM25(104)
+    {
+        @Override
+        public String toString()
+        {
+            return "BM25";
+        }
+
+        @Override
+        public boolean isSatisfiedBy(AbstractType<?> type,
+                                     ByteBuffer leftOperand,
+                                     ByteBuffer rightOperand,
+                                     @Nullable Index.Analyzer indexAnalyzer,
+                                     @Nullable Index.Analyzer queryAnalyzer)
+        {
+            throw new UnsupportedOperationException();
+        }
+    };
     /**
      * The binary representation of this <code>Enum</code> value.
      */
@@ -286,7 +613,7 @@ public boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteB
      * Creates a new <code>Operator</code> with the specified binary representation.
      * @param b the binary representation of this <code>Enum</code> value
      */
-    private Operator(int b)
+    Operator(int b)
     {
         this.b = b;
     }
@@ -326,14 +653,51 @@ public static Operator readFrom(DataInput input) throws IOException
 
     /**
      * Whether 2 values satisfy this operator (given the type they should be compared with).
+     *
+     * @param type the type of the values to compare.
+     * @param leftOperand the left operand of the comparison.
+     * @param rightOperand the right operand of the comparison.
+     * @param indexAnalyzer an index-provided function to transform the left-side compared value before comparison,
+     * or {@code null} if the values don't need to be transformed.
+     * @param queryAnalyzer an index-provided function to transform the right-side compared value before comparison,
+     * or {@code null} if the values don't need to be transformed.
      */
-    public abstract boolean isSatisfiedBy(AbstractType<?> type, ByteBuffer leftOperand, ByteBuffer rightOperand);
+    public abstract boolean isSatisfiedBy(AbstractType<?> type,
+                                          ByteBuffer leftOperand,
+                                          ByteBuffer rightOperand,
+                                          @Nullable Index.Analyzer indexAnalyzer,
+                                          @Nullable Index.Analyzer queryAnalyzer);
+    /**
+     * Whether 2 analyzable values satisfy this operator (given the type they should be compared with).
+     *
+     * @param type the type of the values to compare.
+     * @param leftOperand the left operand of the comparison.
+     * @param rightTokens the right operand of the comparison decomposed as analyzed tokens.
+     * @param indexAnalyzer an index-provided function to transform the left-side compared value before comparison,
+     * it shouldn't be {@code null}.
+     */
+    public boolean isSatisfiedBy(AbstractType<?> type,
+                                 ByteBuffer leftOperand,
+                                 List<ByteBuffer> rightTokens,
+                                 Index.Analyzer indexAnalyzer)
+    {
+        throw new UnsupportedOperationException();
+    }
 
     public int serializedSize()
     {
         return 4;
     }
 
+    /**
+     * Checks if this operator is a like operator.
+     * @return {@code true} if this operator is a like operator, {@code false} otherwise.
+     */
+    public boolean isLike()
+    {
+        return this == LIKE_PREFIX || this == LIKE_CONTAINS || this == LIKE_SUFFIX || this == LIKE_MATCHES;
+    }
+
     /**
      * Checks if this operator is a slice operator.
      * @return {@code true} if this operator is a slice operator, {@code false} otherwise.
diff --git a/src/java/org/apache/cassandra/cql3/Ordering.java b/src/java/org/apache/cassandra/cql3/Ordering.java
index 12d87f1f99c9..2dd817818a76 100644
--- a/src/java/org/apache/cassandra/cql3/Ordering.java
+++ b/src/java/org/apache/cassandra/cql3/Ordering.java
@@ -20,6 +20,7 @@
 
 import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction;
 import org.apache.cassandra.cql3.restrictions.SingleRestriction;
+import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 
@@ -41,53 +42,111 @@ public Ordering(Expression expression, Direction direction)
         this.direction = direction;
     }
 
-    public static abstract class Expression
+    public interface Expression
     {
-        protected final ColumnMetadata columnMetadata;
+        boolean hasNonClusteredOrdering();
 
-        public Expression(ColumnMetadata columnMetadata)
+        SingleRestriction toRestriction();
+
+        ColumnMetadata getColumn();
+
+        default boolean isScored()
+        {
+            return false;
+        }
+    }
+
+    /**
+     * Represents a single column in
+     * <code>ORDER BY column</code>
+     */
+    public static class SingleColumn implements Expression
+    {
+        public final ColumnMetadata column;
+        private final Ordering.Direction direction;
+
+        public SingleColumn(ColumnMetadata column, Ordering.Direction direction)
         {
-            this.columnMetadata = columnMetadata;
+            this.column = column;
+            this.direction = direction;
         }
 
+        @Override
         public boolean hasNonClusteredOrdering()
         {
-            return false;
+            return !column.isClusteringColumn();
         }
 
+        @Override
         public SingleRestriction toRestriction()
         {
-            throw new UnsupportedOperationException();
+            return new SingleColumnRestriction.OrderRestriction(column, direction == Direction.DESC ? Operator.ORDER_BY_DESC : Operator.ORDER_BY_ASC);
         }
 
+        @Override
         public ColumnMetadata getColumn()
         {
-            return columnMetadata;
+            return column;
         }
     }
 
     /**
-     * Represents a single column in <code>ORDER BY column</code>
+     * An expression used in Approximate Nearest Neighbor ordering.
+     * <code>ORDER BY column ANN OF value</code>
      */
-    public static class SingleColumn extends Expression
+    public static class Ann implements Expression
     {
-        public SingleColumn(ColumnMetadata columnMetadata)
+        final ColumnMetadata column;
+        final Term vectorValue;
+        final Direction direction;
+
+        public Ann(ColumnMetadata column, Term vectorValue, Direction direction)
+        {
+            this.column = column;
+            this.vectorValue = vectorValue;
+            this.direction = direction;
+        }
+
+        @Override
+        public boolean hasNonClusteredOrdering()
+        {
+            return true;
+        }
+
+        @Override
+        public SingleRestriction toRestriction()
+        {
+            return new SingleColumnRestriction.AnnRestriction(column, vectorValue);
+        }
+
+        @Override
+        public ColumnMetadata getColumn()
         {
-            super(columnMetadata);
+            return column;
+        }
+
+        @Override
+        public boolean isScored()
+        {
+            return SelectStatement.ANN_USE_SYNTHETIC_SCORE;
         }
     }
 
     /**
-     * An expression used in Approximate Nearest Neighbor ordering. <code>ORDER BY column ANN OF value</code>
+     * An expression used in BM25 ordering.
+     * <code>ORDER BY column BM25 OF value</code>
      */
-    public static class Ann extends Expression
+    public static class Bm25 implements Expression
     {
-        final Term vectorValue;
+        final ColumnMetadata column;
+        final Term queryValue;
+        final Direction direction;
 
-        public Ann(ColumnMetadata columnMetadata, Term vectorValue)
+        public Bm25(ColumnMetadata column, Term queryValue, Direction direction)
         {
-            super(columnMetadata);
-            this.vectorValue = vectorValue;
+            this.column = column;
+            this.queryValue = queryValue;
+            this.direction = direction;
         }
 
         @Override
@@ -99,7 +158,19 @@ public boolean hasNonClusteredOrdering()
         @Override
         public SingleRestriction toRestriction()
         {
-            return new SingleColumnRestriction.AnnRestriction(columnMetadata, vectorValue);
+            return new SingleColumnRestriction.Bm25Restriction(column, queryValue);
+        }
+
+        @Override
+        public ColumnMetadata getColumn()
+        {
+            return column;
+        }
+
+        @Override
+        public boolean isScored()
+        {
+            return true;
         }
     }
 
@@ -108,16 +179,16 @@ public enum Direction
 
 
     /**
-     * Represents ANTLR's abstract syntax tree of a single element in the {@code ORDER BY} clause.
+     * Represents the AST of a single element in the ORDER BY clause.
      * This comes directly out of CQL parser.
      */
     public static class Raw
     {
 
         final Expression expression;
-        final Direction direction;
+        final Ordering.Direction direction;
 
-        public Raw(Expression expression, Direction direction)
+        public Raw(Expression expression, Ordering.Direction direction)
         {
             this.expression = expression;
             this.direction = direction;
@@ -129,12 +200,12 @@ public Raw(Expression expression, Direction direction)
          */
         public Ordering bind(TableMetadata table, VariableSpecifications boundNames)
         {
-            return new Ordering(expression.bind(table, boundNames), direction);
+            return new Ordering(expression.bind(table, boundNames, direction), direction);
         }
 
         public interface Expression
         {
-            Ordering.Expression bind(TableMetadata table, VariableSpecifications boundNames);
+            Ordering.Expression bind(TableMetadata table, VariableSpecifications boundNames, Ordering.Direction direction);
         }
 
         public static class SingleColumn implements Expression
@@ -147,9 +218,9 @@ public static class SingleColumn implements Expression
             }
 
             @Override
-            public Ordering.Expression bind(TableMetadata table, VariableSpecifications boundNames)
+            public Ordering.Expression bind(TableMetadata table, VariableSpecifications boundNames, Ordering.Direction direction)
             {
-                return new Ordering.SingleColumn(table.getExistingColumn(column));
+                return new Ordering.SingleColumn(table.getExistingColumn(column), direction);
             }
         }
 
@@ -165,12 +236,33 @@ public static class Ann implements Expression
             }
 
             @Override
-            public Ordering.Expression bind(TableMetadata table, VariableSpecifications boundNames)
+            public Ordering.Expression bind(TableMetadata table, VariableSpecifications boundNames, Direction direction)
             {
                 ColumnMetadata column = table.getExistingColumn(columnId);
                 Term value = vectorValue.prepare(table.keyspace, column);
                 value.collectMarkerSpecification(boundNames);
-                return new Ordering.Ann(column, value);
+                return new Ordering.Ann(column, value, direction);
+            }
+        }
+
+        public static class Bm25 implements Expression
+        {
+            final ColumnIdentifier columnId;
+            final Term.Raw queryValue;
+
+            Bm25(ColumnIdentifier column, Term.Raw queryValue)
+            {
+                this.columnId = column;
+                this.queryValue = queryValue;
+            }
+
+            @Override
+            public Ordering.Expression bind(TableMetadata table, VariableSpecifications boundNames, Direction direction)
+            {
+                ColumnMetadata column = table.getExistingColumn(columnId);
+                Term value = queryValue.prepare(table.keyspace, column);
+                value.collectMarkerSpecification(boundNames);
+                return new Ordering.Bm25(column, value, direction);
             }
         }
     }
diff --git a/src/java/org/apache/cassandra/cql3/PageSize.java b/src/java/org/apache/cassandra/cql3/PageSize.java
new file mode 100644
index 000000000000..518f382e8584
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/PageSize.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import java.util.Objects;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.service.ClientState;
+
+public class PageSize
+{
+    public static final int NO_LIMIT = Integer.MAX_VALUE;
+
+    public static final PageSize NONE = new PageSize(NO_LIMIT, PageUnit.ROWS);
+
+    public enum PageUnit
+    {
+        ROWS, BYTES
+    }
+
+    private final int size;
+    private final PageUnit unit;
+
+    public PageSize(int size, PageUnit unit)
+    {
+        Preconditions.checkArgument(size >= 0);
+        Preconditions.checkNotNull(unit);
+        this.size = size;
+        this.unit = unit;
+    }
+
+    public int getSize()
+    {
+        return size;
+    }
+
+    public PageUnit getUnit()
+    {
+        return unit;
+    }
+
+    public int bytes()
+    {
+        return unit == PageUnit.BYTES ? size : NO_LIMIT;
+    }
+
+    public int rows()
+    {
+        return unit == PageUnit.ROWS ? size : NO_LIMIT;
+    }
+
+    /**
+     * Creates a page size representing {@code count} rows.
+     *
+     * @throws IllegalArgumentException if the size is not strictly positive.
+     */
+    public static PageSize inRows(int rowsCount)
+    {
+        return new PageSize(rowsCount, PageUnit.ROWS);
+    }
+
+    /**
+     * Creates a page size representing {@code size} bytes.
+     *
+     * @throws IllegalArgumentException if the size is not strictly positive.
+     */
+    public static PageSize inBytes(int bytesCount)
+    {
+        return new PageSize(bytesCount, PageUnit.BYTES);
+    }
+
+    /**
+     * Returns the minimum number of rows for the given number and the number of rows represented by this page size.
+     * If this page size is defined in bytes or undefined, it will just return the provided number of rows.
+     */
+    public int minRowsCount(int rowsCount)
+    {
+        return unit == PageUnit.ROWS ? Math.min(rowsCount, size) : rowsCount;
+    }
+
+    /**
+     * Returns the minimum number of bytes for the given number and the number of bytes represented by this page size.
+     * If this page size is defined in rows or undefined, it will just return the provided number of bytes.
+     */
+    public int minBytesCount(int bytesCount)
+    {
+        return unit == PageUnit.BYTES ? Math.min(bytesCount, size) : bytesCount;
+    }
+
+    public boolean isDefined()
+    {
+        return size < NO_LIMIT;
+    }
+
+    public PageSize withDecreasedRows(int rowsCount) {
+        return unit == PageUnit.ROWS && size != NO_LIMIT
+               ? inRows(Math.max(0, size - rowsCount))
+               : this;
+    }
+
+    public PageSize withDecreasedBytes(int bytesCount) {
+        return unit == PageUnit.BYTES && size != NO_LIMIT
+               ? inBytes(Math.max(0, size - bytesCount))
+               : this;
+    }
+
+    /**
+     * Assuming we went through the provided number of rows/bytes, it returns whether the page is completed.
+     * It will always return {@code false} if the page size is undefined (unlimited).
+     */
+    public boolean isCompleted(int count, PageUnit unit)
+    {
+        return this.unit == unit && this.size <= count;
+    }
+
+    /**
+     * Exercise the guardrail for this page size.
+     */
+    public void guard(String what, ClientState state)
+    {
+        if (unit == PageUnit.ROWS)
+            Guardrails.pageSize.guard(size, what, false, state);
+        else if (unit == PageUnit.BYTES)
+            Guardrails.pageWeight.guard(size, what, false, state);
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        PageSize pageSize = (PageSize) o;
+        return size == pageSize.size && unit == pageSize.unit;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(size, unit);
+    }
+
+    @Override
+    public String toString()
+    {
+        if (size == NO_LIMIT)
+            return "unlimited";
+        else
+            return size + " " + unit.name().toLowerCase();
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/QueryEvents.java b/src/java/org/apache/cassandra/cql3/QueryEvents.java
index 52414ebb7413..52cc352f5c27 100644
--- a/src/java/org/apache/cassandra/cql3/QueryEvents.java
+++ b/src/java/org/apache/cassandra/cql3/QueryEvents.java
@@ -25,7 +25,6 @@
 import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
-
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -104,7 +103,6 @@ public void notifyQueryFailure(CQLStatement statement,
     }
 
     public void notifyExecuteSuccess(CQLStatement statement,
-                                     String query,
                                      QueryOptions options,
                                      QueryState state,
                                      long queryTime,
@@ -112,6 +110,7 @@ public void notifyExecuteSuccess(CQLStatement statement,
     {
         try
         {
+            String query = statement.getRawCQLStatement();
             final String maybeObfuscatedQuery = listeners.size() > 0 ? maybeObfuscatePassword(statement, query) : query;
             for (Listener listener : listeners)
                 listener.executeSuccess(statement, maybeObfuscatedQuery, options, state, queryTime, response);
@@ -129,7 +128,7 @@ public void notifyExecuteFailure(QueryHandler.Prepared prepared,
                                      Exception cause)
     {
         CQLStatement statement = prepared != null ? prepared.statement : null;
-        String query = prepared != null ? prepared.rawCQLStatement : null;
+        String query = prepared != null ? prepared.statement.getRawCQLStatement() : null;
         try
         {
             final String maybeObfuscatedQuery = listeners.size() > 0 ? maybeObfuscatePassword(statement, query) : query;
@@ -180,7 +179,7 @@ public void notifyBatchFailure(List<QueryHandler.Prepared> prepared,
             {
                 prepared.forEach(p -> {
                     statements.add(p.statement);
-                    queries.add(p.rawCQLStatement);
+                    queries.add(p.statement.getRawCQLStatement());
                 });
             }
             try
@@ -242,7 +241,7 @@ public void notifyPrepareFailure(@Nullable CQLStatement statement, String query,
         }
     }
 
-    private String maybeObfuscatePassword(CQLStatement statement, String query)
+    private String maybeObfuscatePassword(@Nullable CQLStatement statement, String query)
     {
         // Statement might be null as side-effect of failed parsing, originates from QueryMessage#execute
         if (statement == null)
diff --git a/src/java/org/apache/cassandra/cql3/QueryHandler.java b/src/java/org/apache/cassandra/cql3/QueryHandler.java
index 9637f6e41b0f..2cfd3f1af718 100644
--- a/src/java/org/apache/cassandra/cql3/QueryHandler.java
+++ b/src/java/org/apache/cassandra/cql3/QueryHandler.java
@@ -63,19 +63,12 @@ public static class Prepared
 
         public final MD5Digest resultMetadataId;
 
-        /**
-         * Contains the CQL statement source if the statement has been "regularly" perpared via
-         * {@link QueryHandler#prepare(String, ClientState, Map)}.
-         * Other usages of this class may or may not contain the CQL statement source.
-         */
-        public final String rawCQLStatement;
         public final String keyspace;
         public final boolean fullyQualified;
 
-        public Prepared(CQLStatement statement, String rawCQLStatement, boolean fullyQualified, String keyspace)
+        public Prepared(CQLStatement statement, boolean fullyQualified, String keyspace)
         {
             this.statement = statement;
-            this.rawCQLStatement = rawCQLStatement;
             this.resultMetadataId = ResultSet.ResultMetadata.fromPrepared(statement).getResultMetadataId();
             this.fullyQualified = fullyQualified;
             this.keyspace = keyspace;
diff --git a/src/java/org/apache/cassandra/cql3/QueryInterceptor.java b/src/java/org/apache/cassandra/cql3/QueryInterceptor.java
new file mode 100644
index 000000000000..ec3ff73a3d3d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/QueryInterceptor.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+public interface QueryInterceptor
+{
+    /**
+     * Intercept a statement and process it if necessary. If the interceptor processes the statement, it
+     * returns a {@link ResultMessage}, otherwise returns <code>null</code>.
+     *
+     * @param statement
+     * @param queryState
+     * @param options
+     * @param requestTime
+     * @return a {@link ResultMessage} if the statement is processed by the interceptor,
+     * otherwise returns <code>null</code>
+     */
+    @Nullable
+    default ResultMessage interceptStatement(CQLStatement statement,
+                                             QueryState queryState,
+                                             QueryOptions options,
+                                             Map<String, ByteBuffer> customPayload,
+                                             Dispatcher.RequestTime requestTime)
+    {
+        return null;
+    }
+
+
+    /**
+     * Intercept a batch statement and process it if necessary. If the interceptor processes the batch statement, it
+     * returns a {@link ResultMessage}, otherwise returns <code>null</code>.
+     */
+    @Nullable
+    default ResultMessage interceptBatchStatement(BatchStatement batch,
+                                                  QueryState state,
+                                                  BatchQueryOptions options,
+                                                  Map<String, ByteBuffer> customPayload,
+                                                  Dispatcher.RequestTime requestTime)
+    {
+        return null;
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/QueryOptions.java b/src/java/org/apache/cassandra/cql3/QueryOptions.java
index f69e73c3e801..53904b79e437 100644
--- a/src/java/org/apache/cassandra/cql3/QueryOptions.java
+++ b/src/java/org/apache/cassandra/cql3/QueryOptions.java
@@ -18,18 +18,25 @@
 package org.apache.cassandra.cql3;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
 
 import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 
 import io.netty.buffer.ByteBuf;
-
 import org.apache.cassandra.config.DataStorageSpec;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.pager.PagingState;
 import org.apache.cassandra.transport.CBCodec;
@@ -38,8 +45,6 @@
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.CassandraUInt;
 import org.apache.cassandra.utils.Pair;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
 
 /**
  * Options for a query.
@@ -82,7 +87,7 @@ public static QueryOptions forProtocolVersion(ProtocolVersion protocolVersion)
     public static QueryOptions create(ConsistencyLevel consistency,
                                       List<ByteBuffer> values,
                                       boolean skipMetadata,
-                                      int pageSize,
+                                      PageSize pageSize,
                                       PagingState pagingState,
                                       ConsistencyLevel serialConsistency,
                                       ProtocolVersion version,
@@ -94,7 +99,7 @@ public static QueryOptions create(ConsistencyLevel consistency,
     public static QueryOptions create(ConsistencyLevel consistency,
                                       List<ByteBuffer> values,
                                       boolean skipMetadata,
-                                      int pageSize,
+                                      PageSize pageSize,
                                       PagingState pagingState,
                                       ConsistencyLevel serialConsistency,
                                       ProtocolVersion version,
@@ -114,16 +119,7 @@ public static QueryOptions addColumnSpecifications(QueryOptions options, List<Co
         return new OptionsWithColumnSpecifications(options, columnSpecs);
     }
 
-    public static QueryOptions withConsistencyLevel(QueryOptions options, ConsistencyLevel consistencyLevel)
-    {
-        return new OptionsWithConsistencyLevel(options, consistencyLevel);
-    }
-
-    public static QueryOptions withPageSize(QueryOptions options, int pageSize)
-    {
-        return new OptionsWithPageSize(options, pageSize);
-    }
-
+    public abstract void updateConsistency(ConsistencyLevel updatedLevel);
     public abstract ConsistencyLevel getConsistency();
     public abstract List<ByteBuffer> getValues();
     public abstract boolean skipMetadata();
@@ -195,8 +191,8 @@ public ImmutableList<ColumnSpecification> getColumnSpecifications()
         throw new UnsupportedOperationException();
     }
 
-    /**  The pageSize for this query. Will be {@code <= 0} if not relevant for the query.  */
-    public int getPageSize()
+    /** The pageSize for this query. Will be {@code <= 0} if not relevant for the query.  */
+    public PageSize getPageSize()
     {
         return getSpecificOptions().pageSize;
     }
@@ -208,9 +204,10 @@ public PagingState getPagingState()
     }
 
     /**  Serial consistency for conditional updates. */
-    public ConsistencyLevel getSerialConsistency()
+    public ConsistencyLevel getSerialConsistency(@Nullable QueryState state)
     {
-        return getSpecificOptions().serialConsistency;
+        ConsistencyLevel cl = getSpecificOptions().serialConsistency;
+        return cl != null ? cl : ConsistencyLevel.defaultSerialConsistency(state);
     }
 
     public long getTimestamp(QueryState state)
@@ -336,7 +333,7 @@ public long getCoordinatorReadSizeFailThresholdBytes()
 
     static class DefaultQueryOptions extends QueryOptions
     {
-        private final ConsistencyLevel consistency;
+        private volatile ConsistencyLevel consistency;
         private final List<ByteBuffer> values;
         private final boolean skipMetadata;
 
@@ -354,6 +351,12 @@ static class DefaultQueryOptions extends QueryOptions
             this.protocolVersion = protocolVersion;
         }
 
+        @Override
+        public void updateConsistency(ConsistencyLevel updatedLevel)
+        {
+            consistency = updatedLevel;
+        }
+
         public ConsistencyLevel getConsistency()
         {
             return consistency;
@@ -400,6 +403,11 @@ public List<ByteBuffer> getValues()
             return this.wrapped.getValues();
         }
 
+        public void updateConsistency(ConsistencyLevel updatedLevel)
+        {
+            wrapped.updateConsistency(updatedLevel);
+        }
+
         public ConsistencyLevel getConsistency()
         {
             return wrapped.getConsistency();
@@ -434,40 +442,6 @@ public QueryOptions prepare(List<ColumnSpecification> specs)
         }
     }
 
-    static class OptionsWithConsistencyLevel extends QueryOptionsWrapper
-    {
-        private final ConsistencyLevel consistencyLevel;
-
-        OptionsWithConsistencyLevel(QueryOptions wrapped, ConsistencyLevel consistencyLevel)
-        {
-            super(wrapped);
-            this.consistencyLevel = consistencyLevel;
-        }
-
-        @Override
-        public ConsistencyLevel getConsistency()
-        {
-            return consistencyLevel;
-        }
-    }
-
-    static class OptionsWithPageSize extends QueryOptionsWrapper
-    {
-        private final int pageSize;
-
-        OptionsWithPageSize(QueryOptions wrapped, int pageSize)
-        {
-            super(wrapped);
-            this.pageSize = pageSize;
-        }
-
-        @Override
-        public int getPageSize()
-        {
-            return pageSize;
-        }
-    }
-
     /**
      * <code>QueryOptions</code> decorator that provides access to the column specifications.
      */
@@ -537,16 +511,16 @@ public List<ByteBuffer> getValues()
     // Options that are likely to not be present in most queries
     static class SpecificOptions
     {
-        private static final SpecificOptions DEFAULT = new SpecificOptions(-1, null, null, Long.MIN_VALUE, null, UNSET_NOWINSEC);
+        private static final SpecificOptions DEFAULT = new SpecificOptions(PageSize.NONE, null, null, Long.MIN_VALUE, null, UNSET_NOWINSEC);
 
-        private final int pageSize;
+        private final PageSize pageSize;
         private final PagingState state;
         private final ConsistencyLevel serialConsistency;
         private final long timestamp;
         private final String keyspace;
         private final long nowInSeconds;
 
-        private SpecificOptions(int pageSize,
+        private SpecificOptions(PageSize pageSize,
                                 PagingState state,
                                 ConsistencyLevel serialConsistency,
                                 long timestamp,
@@ -555,7 +529,7 @@ private SpecificOptions(int pageSize,
         {
             this.pageSize = pageSize;
             this.state = state;
-            this.serialConsistency = serialConsistency == null ? ConsistencyLevel.SERIAL : serialConsistency;
+            this.serialConsistency = serialConsistency;
             this.timestamp = timestamp;
             this.keyspace = keyspace;
             this.nowInSeconds = nowInSeconds;
@@ -580,7 +554,30 @@ private enum Flag
             TIMESTAMP,
             NAMES_FOR_VALUES,
             KEYSPACE,
-            NOW_IN_SECONDS;
+            NOW_IN_SECONDS,
+            UNUSED_9,
+            UNUSED_10,
+            UNUSED_11,
+            UNUSED_12,
+            UNUSED_13,
+            UNUSED_14,
+            UNUSED_15,
+            UNUSED_16,
+            UNUSED_17,
+            UNUSED_18,
+            UNUSED_19,
+            UNUSED_20,
+            UNUSED_21,
+            UNUSED_22,
+            UNUSED_23,
+            UNUSED_24,
+            UNUSED_25,
+            UNUSED_26,
+            UNUSED_27,
+            UNUSED_28,
+            UNUSED_29,
+            PAGE_SIZE_IN_BYTES,
+            UNUSED_31;
 
             private static final Flag[] ALL_VALUES = values();
 
@@ -634,9 +631,13 @@ public QueryOptions decode(ByteBuf body, ProtocolVersion version)
             SpecificOptions options = SpecificOptions.DEFAULT;
             if (!flags.isEmpty())
             {
-                int pageSize = flags.contains(Flag.PAGE_SIZE) ? body.readInt() : -1;
+                PageSize pageSize = flags.contains(Flag.PAGE_SIZE)
+                                          ? flags.contains(Flag.PAGE_SIZE_IN_BYTES)
+                                            ? PageSize.inBytes(body.readInt())
+                                            : PageSize.inRows(body.readInt())
+                                          : PageSize.NONE;
                 PagingState pagingState = flags.contains(Flag.PAGING_STATE) ? PagingState.deserialize(CBUtil.readValueNoCopy(body), version) : null;
-                ConsistencyLevel serialConsistency = flags.contains(Flag.SERIAL_CONSISTENCY) ? CBUtil.readConsistencyLevel(body) : ConsistencyLevel.SERIAL;
+                ConsistencyLevel serialConsistency = flags.contains(Flag.SERIAL_CONSISTENCY) ? CBUtil.readConsistencyLevel(body) : null;
                 long timestamp = Long.MIN_VALUE;
                 if (flags.contains(Flag.TIMESTAMP))
                 {
@@ -668,11 +669,11 @@ public void encode(QueryOptions options, ByteBuf dest, ProtocolVersion version)
             if (flags.contains(Flag.VALUES))
                 CBUtil.writeValueList(options.getValues(), dest);
             if (flags.contains(Flag.PAGE_SIZE))
-                dest.writeInt(options.getPageSize());
+                dest.writeInt(options.getPageSize().getSize());
             if (flags.contains(Flag.PAGING_STATE))
                 CBUtil.writeValue(options.getPagingState().serialize(version), dest);
             if (flags.contains(Flag.SERIAL_CONSISTENCY))
-                CBUtil.writeConsistencyLevel(options.getSerialConsistency(), dest);
+                CBUtil.writeConsistencyLevel(options.getSerialConsistency(null), dest);
             if (flags.contains(Flag.TIMESTAMP))
                 dest.writeLong(options.getSpecificOptions().timestamp);
             if (flags.contains(Flag.KEYSPACE))
@@ -701,7 +702,7 @@ public int encodedSize(QueryOptions options, ProtocolVersion version)
             if (flags.contains(Flag.PAGING_STATE))
                 size += CBUtil.sizeOfValue(options.getPagingState().serializedSize(version));
             if (flags.contains(Flag.SERIAL_CONSISTENCY))
-                size += CBUtil.sizeOfConsistencyLevel(options.getSerialConsistency());
+                size += CBUtil.sizeOfConsistencyLevel(options.getSerialConsistency(null));
             if (flags.contains(Flag.TIMESTAMP))
                 size += 8;
             if (flags.contains(Flag.KEYSPACE))
@@ -719,11 +720,11 @@ private EnumSet<Flag> gatherFlags(QueryOptions options, ProtocolVersion version)
                 flags.add(Flag.VALUES);
             if (options.skipMetadata())
                 flags.add(Flag.SKIP_METADATA);
-            if (options.getPageSize() >= 0)
+            if (options.getPageSize().isDefined())
                 flags.add(Flag.PAGE_SIZE);
             if (options.getPagingState() != null)
                 flags.add(Flag.PAGING_STATE);
-            if (options.getSerialConsistency() != ConsistencyLevel.SERIAL)
+            if (options.getSpecificOptions().serialConsistency != null)
                 flags.add(Flag.SERIAL_CONSISTENCY);
             if (options.getSpecificOptions().timestamp != Long.MIN_VALUE)
                 flags.add(Flag.TIMESTAMP);
@@ -734,6 +735,8 @@ private EnumSet<Flag> gatherFlags(QueryOptions options, ProtocolVersion version)
                     flags.add(Flag.KEYSPACE);
                 if (options.getSpecificOptions().nowInSeconds != UNSET_NOWINSEC)
                     flags.add(Flag.NOW_IN_SECONDS);
+                if (options.getSpecificOptions().pageSize.getUnit() == PageSize.PageUnit.BYTES)
+                    flags.add(Flag.PAGE_SIZE_IN_BYTES);
             }
 
             return flags;
diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
index 8c7a01d54406..df438761c36b 100644
--- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java
+++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
@@ -18,15 +18,19 @@
 package org.apache.cassandra.cql3;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
 import com.google.common.collect.*;
@@ -34,14 +38,46 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.antlr.runtime.*;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import org.antlr.runtime.RecognitionException;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.cql3.selection.ResultSetBuilder;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.QualifiedStatement;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.exceptions.CassandraException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.IsBootstrappingException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.CQLMetrics;
 import org.apache.cassandra.metrics.ClientRequestMetrics;
-import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.KeyspaceMetadata;
@@ -49,30 +85,23 @@
 import org.apache.cassandra.schema.SchemaChangeListener;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.cql3.functions.UDAggregate;
-import org.apache.cassandra.cql3.functions.UDFunction;
-import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.cql3.functions.FunctionName;
-import org.apache.cassandra.cql3.selection.ResultSetBuilder;
-import org.apache.cassandra.cql3.statements.*;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.db.partitions.PartitionIterators;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.metrics.CQLMetrics;
-import org.apache.cassandra.service.*;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.pager.QueryPager;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.transport.messages.ResultMessage;
-import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.MD5Digest;
+import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.FutureCombiner;
+import org.apache.cassandra.utils.Pair;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.ENABLE_NODELOCAL_QUERIES;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
@@ -89,7 +118,11 @@ public class QueryProcessor implements QueryHandler
 
     private static final Logger logger = LoggerFactory.getLogger(QueryProcessor.class);
 
-    private static final Cache<MD5Digest, Prepared> preparedStatements;
+    // DSP-24330 heavy contention (almost deadlock) on Caffeine cache on insertions while measuring entry size. We
+    // precompute sizes instead to reduce contention on Caffeine's underlying stipped locking structures. We also avoid
+    // duplication of work as every statement was being measured 2 times: before instertion to check size limits and
+    // upon insertion by Caffeine
+    private static final Cache<MD5Digest, Pair<Prepared, Integer>> preparedStatements;
 
     // A map for prepared statements used internally (which we don't want to mix with user statement, in particular we don't
     // bother with expiration on those.
@@ -101,12 +134,14 @@ public class QueryProcessor implements QueryHandler
 
     private static final AtomicInteger lastMinuteEvictionsCount = new AtomicInteger(0);
 
+    private final List<QueryInterceptor> interceptors = new ArrayList<>();
+
     static
     {
         preparedStatements = Caffeine.newBuilder()
                              .executor(ImmediateExecutor.INSTANCE)
                              .maximumWeight(capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMiB()))
-                             .weigher(QueryProcessor::measure)
+                             .weigher(QueryProcessor::getPrecomputedSize)
                              .removalListener((key, prepared, cause) -> {
                                  MD5Digest md5Digest = (MD5Digest) key;
                                  if (cause.wasEvicted())
@@ -125,7 +160,7 @@ public class QueryProcessor implements QueryHandler
                             DatabaseDescriptor.getPreparedStatementsCacheSizeMiB());
         }, 1, 1, TimeUnit.MINUTES);
 
-        logger.info("Initialized prepared statement caches with {} MiB",
+        logger.debug("Initialized prepared statement caches with {} MiB",
                     DatabaseDescriptor.getPreparedStatementsCacheSizeMiB());
     }
 
@@ -163,11 +198,13 @@ public void preloadPreparedStatements()
                     clientState.setKeyspace(keyspace);
 
                 Prepared prepared = parseAndPrepare(query, clientState, false);
-                preparedStatements.put(id, prepared);
+                int precomputedCacheEntrySize = measurePStatementCacheEntrySize(id, prepared);
+                Pair<Prepared, Integer> cacheValue = Pair.create(prepared, precomputedCacheEntrySize);
+                preparedStatements.put(id, cacheValue);
 
                 // Preload `null` statement for non-fully qualified statements, since it can't be parsed if loaded from cache and will be dropped
                 if (!prepared.fullyQualified)
-                    preparedStatements.get(computeId(query, null), (ignored_) -> prepared);
+                    preparedStatements.get(computeId(query, null), (ignored_) -> cacheValue);
                 return true;
             }
             catch (RequestValidationException e)
@@ -221,12 +258,16 @@ public void evictPrepared(MD5Digest id)
 
     public HashMap<MD5Digest, Prepared> getPreparedStatements()
     {
-        return new HashMap<>(preparedStatements.asMap());
+        HashMap<MD5Digest, Prepared> res = new HashMap<>(preparedStatements.asMap().size());
+        for(Map.Entry<MD5Digest, Pair<Prepared, Integer>> entry: preparedStatements.asMap().entrySet())
+            res.put(entry.getKey(), entry.getValue().left());
+
+        return res;
     }
 
     public Prepared getPrepared(MD5Digest id)
     {
-        return preparedStatements.getIfPresent(id);
+        return extractPreparedFromPair(preparedStatements.getIfPresent(id));
     }
 
     public static void validateKey(ByteBuffer key) throws InvalidRequestException
@@ -248,36 +289,71 @@ public static void validateKey(ByteBuffer key) throws InvalidRequestException
 
     public ResultMessage processStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
+    {
+        return processStatement(statement, queryState, options, Collections.emptyMap(), requestTime);
+    }
+
+    public ResultMessage processStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime)
+    throws RequestExecutionException, RequestValidationException
     {
         logger.trace("Process {} @CL.{}", statement, options.getConsistency());
         ClientState clientState = queryState.getClientState();
+
+        Tracing.trace("Authorizing against client state");
         statement.authorize(clientState);
         statement.validate(clientState);
 
+        Tracing.setupTracedKeyspace(statement);
+
+        for (QueryInterceptor interceptor: interceptors)
+        {
+            ResultMessage result = interceptor.interceptStatement(statement, queryState, options, customPayload, requestTime);
+
+            if (result != null)
+                return result;
+        }
+
+        Tracing.trace("Executing prepared statement");
         ResultMessage result = options.getConsistency() == ConsistencyLevel.NODE_LOCAL
-                             ? processNodeLocalStatement(statement, queryState, options)
-                             : statement.execute(queryState, options, requestTime);
+                               ? processNodeLocalStatement(statement, queryState, options, requestTime)
+                               : statement.execute(queryState, options, requestTime);
 
         return result == null ? new ResultMessage.Void() : result;
     }
 
-    private ResultMessage processNodeLocalStatement(CQLStatement statement, QueryState queryState, QueryOptions options)
+    /**
+     * Register a new {@link QueryInterceptor}
+     * @param interceptor the {@link QueryInterceptor} to register
+     */
+    public void registerInterceptor(QueryInterceptor interceptor)
+    {
+        interceptors.add(Objects.requireNonNull(interceptor));
+    }
+
+    @VisibleForTesting
+    public void clearInterceptors()
+    {
+        interceptors.clear();
+    }
+
+    private ResultMessage processNodeLocalStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
         if (!ENABLE_NODELOCAL_QUERIES.getBoolean())
             throw new InvalidRequestException("NODE_LOCAL consistency level is highly dangerous and should be used only for debugging purposes");
 
         if (statement instanceof BatchStatement || statement instanceof ModificationStatement)
-            return processNodeLocalWrite(statement, queryState, options);
+            return processNodeLocalWrite(statement, queryState, options, requestTime);
         else if (statement instanceof SelectStatement)
-            return processNodeLocalSelect((SelectStatement) statement, queryState, options);
+            return processNodeLocalSelect((SelectStatement) statement, queryState, options, requestTime);
         else
             throw new InvalidRequestException("NODE_LOCAL consistency level can only be used with BATCH, UPDATE, INSERT, DELETE, and SELECT statements");
     }
 
-    private ResultMessage processNodeLocalWrite(CQLStatement statement, QueryState queryState, QueryOptions options)
+    private ResultMessage processNodeLocalWrite(CQLStatement statement, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
-        ClientRequestMetrics  levelMetrics = ClientRequestsMetricsHolder.writeMetricsForLevel(ConsistencyLevel.NODE_LOCAL);
-        ClientRequestMetrics globalMetrics = ClientRequestsMetricsHolder.writeMetrics;
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics((statement instanceof QualifiedStatement) ? ((QualifiedStatement) statement).keyspace() : null);
+        ClientRequestMetrics  levelMetrics = metrics.writeMetricsForLevel(ConsistencyLevel.NODE_LOCAL);
+        ClientRequestMetrics globalMetrics = metrics.writeMetrics;
 
         long startTime = nanoTime();
         try
@@ -286,16 +362,21 @@ private ResultMessage processNodeLocalWrite(CQLStatement statement, QueryState q
         }
         finally
         {
-            long latency = nanoTime() - startTime;
-             levelMetrics.addNano(latency);
-            globalMetrics.addNano(latency);
+            long endTime = nanoTime();
+            long latency = endTime - startTime;
+            long serviceLatency = endTime - requestTime.startedAtNanos();
+            levelMetrics.executionTimeMetrics.addNano(latency);
+            levelMetrics.serviceTimeMetrics.addNano(serviceLatency);
+            globalMetrics.executionTimeMetrics.addNano(latency);
+            globalMetrics.serviceTimeMetrics.addNano(serviceLatency);
         }
     }
 
-    private ResultMessage processNodeLocalSelect(SelectStatement statement, QueryState queryState, QueryOptions options)
+    private ResultMessage processNodeLocalSelect(SelectStatement statement, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
-        ClientRequestMetrics  levelMetrics = ClientRequestsMetricsHolder.readMetricsForLevel(ConsistencyLevel.NODE_LOCAL);
-        ClientRequestMetrics globalMetrics = ClientRequestsMetricsHolder.readMetrics;
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(statement.keyspace());
+        ClientRequestMetrics levelMetrics = metrics.readMetricsForLevel(ConsistencyLevel.NODE_LOCAL);
+        ClientRequestMetrics globalMetrics = metrics.readMetrics;
 
         if (StorageService.instance.isBootstrapMode() && !SchemaConstants.isLocalSystemKeyspace(statement.keyspace()))
         {
@@ -311,9 +392,13 @@ private ResultMessage processNodeLocalSelect(SelectStatement statement, QuerySta
         }
         finally
         {
-            long latency = nanoTime() - startTime;
-             levelMetrics.addNano(latency);
-            globalMetrics.addNano(latency);
+            long endTime = nanoTime();
+            long latency = endTime - startTime;
+            long serviceLatency = endTime - requestTime.startedAtNanos();
+            levelMetrics.executionTimeMetrics.addNano(latency);
+            levelMetrics.serviceTimeMetrics.addNano(serviceLatency);
+            globalMetrics.executionTimeMetrics.addNano(latency);
+            globalMetrics.serviceTimeMetrics.addNano(serviceLatency);
         }
     }
 
@@ -330,17 +415,11 @@ public CQLStatement parse(String queryString, QueryState queryState, QueryOption
         return getStatement(queryString, queryState.getClientState().cloneWithKeyspaceIfSet(options.getKeyspace()));
     }
 
-    public ResultMessage process(CQLStatement statement,
-                                 QueryState state,
+    public ResultMessage process(CQLStatement prepared,
+                                 QueryState queryState,
                                  QueryOptions options,
                                  Map<String, ByteBuffer> customPayload,
                                  Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException
-    {
-        return process(statement, state, options, requestTime);
-    }
-
-    public ResultMessage process(CQLStatement prepared, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
-    throws RequestExecutionException, RequestValidationException
     {
         options.prepare(prepared.getBindVariables());
         if (prepared.getBindVariables().size() != options.getValues().size())
@@ -349,7 +428,13 @@ public ResultMessage process(CQLStatement prepared, QueryState queryState, Query
         if (!queryState.getClientState().isInternal)
             metrics.regularStatementsExecuted.inc();
 
-        return processStatement(prepared, queryState, options, requestTime);
+        return processStatement(prepared, queryState, options, customPayload, requestTime);
+    }
+
+    public ResultMessage process(CQLStatement prepared, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
+    throws RequestExecutionException, RequestValidationException
+    {
+        return process(prepared, queryState, options, Collections.emptyMap(), requestTime);
     }
 
     public static CQLStatement parseStatement(String queryStr, ClientState clientState) throws RequestValidationException
@@ -436,10 +521,7 @@ public static Prepared parseAndPrepare(String query, ClientState clientState, bo
         CQLStatement statement = raw.prepare(clientState);
         statement.validate(clientState);
 
-        if (isInternal)
-            return new Prepared(statement, "", fullyQualified, keyspace);
-        else
-            return new Prepared(statement, query, fullyQualified, keyspace);
+        return new Prepared(statement, fullyQualified, keyspace);
     }
 
     public static UntypedResultSet executeInternal(String query, Object... values)
@@ -526,22 +608,32 @@ public static UntypedResultSet executeInternalWithNowInSec(String query, long no
     public static UntypedResultSet execute(String query, ConsistencyLevel cl, QueryState state, Object... values)
     throws RequestExecutionException
     {
-        try
-        {
-            Prepared prepared = prepareInternal(query);
-            ResultMessage result = prepared.statement.execute(state, makeInternalOptionsWithNowInSec(prepared.statement, state.getNowInSeconds(), values, cl), Dispatcher.RequestTime.forImmediateExecution());
-            if (result instanceof ResultMessage.Rows)
-                return UntypedResultSet.create(((ResultMessage.Rows)result).result);
-            else
-                return null;
-        }
-        catch (RequestValidationException e)
-        {
-            throw new RuntimeException("Error validating " + query, e);
-        }
+        Prepared prepared = prepareInternal(query);
+        ResultMessage<?> result = prepared.statement.execute(state, makeInternalOptionsWithNowInSec(prepared.statement, state.getNowInSeconds(), values, cl), Dispatcher.RequestTime.forImmediateExecution());
+        if (result instanceof ResultMessage.Rows)
+            return UntypedResultSet.create(((ResultMessage.Rows)result).result);
+        else
+            return null;
+    }
+
+    /**
+     * Same than {@link #execute(String, ConsistencyLevel, Object...)}, but to use for queries we know are only executed
+     * once so that the created statement object is not cached.
+     */
+    @VisibleForTesting
+    static UntypedResultSet executeOnce(String query, ConsistencyLevel cl, Object... values)
+    {
+        QueryState queryState = internalQueryState();
+        CQLStatement statement = parseStatement(query, queryState.getClientState());
+        statement.validate(queryState.getClientState());
+        ResultMessage<?> result = statement.execute(queryState, makeInternalOptions(statement, values, cl), Dispatcher.RequestTime.forImmediateExecution());
+        if (result instanceof ResultMessage.Rows)
+            return UntypedResultSet.create(((ResultMessage.Rows)result).result);
+        else
+            return null;
     }
 
-    public static UntypedResultSet executeInternalWithPaging(String query, int pageSize, Object... values)
+    public static UntypedResultSet executeInternalWithPaging(String query, PageSize pageSize, Object... values)
     {
         Prepared prepared = prepareInternal(query);
         if (!(prepared.statement instanceof SelectStatement))
@@ -644,8 +736,8 @@ public boolean useNewPreparedStatementBehaviour()
 
         synchronized (this)
         {
-            CassandraVersion minVersion = Gossiper.instance.getMinVersion(DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
-            if (minVersion != null && minVersion.compareTo(NEW_PREPARED_STATEMENT_BEHAVIOUR_SINCE_40, true) >= 0)
+            CassandraVersion minVersion = Gossiper.instance.getMinVersion();
+            if (minVersion.compareTo(NEW_PREPARED_STATEMENT_BEHAVIOUR_SINCE_40, true) >= 0)
             {
                 logger.info("Fully upgraded to at least {}", minVersion);
                 newPreparedStatementBehaviour = true;
@@ -680,8 +772,8 @@ public ResultMessage.Prepared prepare(String queryString, ClientState clientStat
         boolean useNewPreparedStatementBehaviour = useNewPreparedStatementBehaviour();
         MD5Digest hashWithoutKeyspace = computeId(queryString, null);
         MD5Digest hashWithKeyspace = computeId(queryString, clientState.getRawKeyspace());
-        Prepared cachedWithoutKeyspace = preparedStatements.getIfPresent(hashWithoutKeyspace);
-        Prepared cachedWithKeyspace = preparedStatements.getIfPresent(hashWithKeyspace);
+        Prepared cachedWithoutKeyspace = extractPreparedFromPair(preparedStatements.getIfPresent(hashWithoutKeyspace));
+        Prepared cachedWithKeyspace = extractPreparedFromPair(preparedStatements.getIfPresent(hashWithKeyspace));
         // We assume it is only safe to return cached prepare if we have both instances
         boolean safeToReturnCached = cachedWithoutKeyspace != null && cachedWithKeyspace != null;
 
@@ -751,13 +843,12 @@ public static ResultMessage.Prepared getStoredPreparedStatement(String queryStri
     throws InvalidRequestException
     {
         MD5Digest statementId = computeId(queryString, clientKeyspace);
-        Prepared existing = preparedStatements.getIfPresent(statementId);
+        Prepared existing = extractPreparedFromPair(preparedStatements.getIfPresent(statementId));
         if (existing == null)
             return null;
 
-        checkTrue(queryString.equals(existing.rawCQLStatement),
-                  "MD5 hash collision: query with the same MD5 hash was already prepared. \n Existing: '%s'",
-                  existing.rawCQLStatement);
+        checkTrue(queryString.equals(existing.statement.getRawCQLStatement()),
+                String.format("MD5 hash collision: query with the same MD5 hash was already prepared. \n Existing: '%s'", existing.statement.getRawCQLStatement()));
 
         return createResultMessage(statementId, existing);
     }
@@ -777,16 +868,19 @@ public static ResultMessage.Prepared storePreparedStatement(String queryString,
     {
         // Concatenate the current keyspace so we don't mix prepared statements between keyspace (#5352).
         // (if the keyspace is null, queryString has to have a fully-qualified keyspace so it's fine.
-        long statementSize = ObjectSizes.measureDeep(prepared.statement);
+
+        MD5Digest statementId = computeId(queryString, keyspace);
+        int precomputedCacheEntrySize = measurePStatementCacheEntrySize(statementId, prepared);
         // don't execute the statement if it's bigger than the allowed threshold
-        if (statementSize > capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMiB()))
+        if (precomputedCacheEntrySize > capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMiB()))
             throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d MB: %s...",
-                                                            statementSize,
+                                                            precomputedCacheEntrySize,
                                                             DatabaseDescriptor.getPreparedStatementsCacheSizeMiB(),
                                                             queryString.substring(0, 200)));
-        MD5Digest statementId = computeId(queryString, keyspace);
-        Prepared previous = preparedStatements.get(statementId, (ignored_) -> prepared);
-        if (previous == prepared)
+
+        Pair<Prepared, Integer> cacheValue = Pair.create(prepared, precomputedCacheEntrySize);
+        Pair<Prepared, Integer> previous = preparedStatements.get(statementId, (ignored_) -> cacheValue);
+        if (previous != null && previous.left() == prepared)
             SystemKeyspace.writePreparedStatement(keyspace, statementId, queryString);
 
         ResultSet.PreparedMetadata preparedMetadata = ResultSet.PreparedMetadata.fromPrepared(prepared.statement);
@@ -801,12 +895,6 @@ public ResultMessage processPrepared(CQLStatement statement,
                                          Map<String, ByteBuffer> customPayload,
                                          Dispatcher.RequestTime requestTime)
                                                  throws RequestExecutionException, RequestValidationException
-    {
-        return processPrepared(statement, state, options, requestTime);
-    }
-
-    public ResultMessage processPrepared(CQLStatement statement, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
-    throws RequestExecutionException, RequestValidationException
     {
         List<ByteBuffer> variables = options.getValues();
         // Check to see if there are any bound variables to verify
@@ -824,39 +912,53 @@ public ResultMessage processPrepared(CQLStatement statement, QueryState querySta
         }
 
         metrics.preparedStatementsExecuted.inc();
-        return processStatement(statement, queryState, options, requestTime);
+        return processStatement(statement, state, options, customPayload, requestTime);
+    }
+
+    public ResultMessage processPrepared(CQLStatement statement, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
+    throws RequestExecutionException, RequestValidationException
+    {
+        return processPrepared(statement, queryState, options, Collections.emptyMap(), requestTime);
     }
 
-    public ResultMessage processBatch(BatchStatement statement,
-                                      QueryState state,
+    public ResultMessage processBatch(BatchStatement batch,
+                                      QueryState queryState,
                                       BatchQueryOptions options,
                                       Map<String, ByteBuffer> customPayload,
                                       Dispatcher.RequestTime requestTime)
                                               throws RequestExecutionException, RequestValidationException
-    {
-        return processBatch(statement, state, options, requestTime);
-    }
-
-    public ResultMessage processBatch(BatchStatement batch, QueryState queryState, BatchQueryOptions options, Dispatcher.RequestTime requestTime)
-    throws RequestExecutionException, RequestValidationException
     {
         ClientState clientState = queryState.getClientState().cloneWithKeyspaceIfSet(options.getKeyspace());
+        Tracing.setupTracedKeyspace(batch);
+
+        for (QueryInterceptor interceptor: interceptors)
+        {
+            ResultMessage result = interceptor.interceptBatchStatement(batch, queryState, options, customPayload, requestTime);
+
+            if (result != null)
+                return result;
+        }
+
+        Tracing.trace("Authorizing batch");
         batch.authorize(clientState);
         batch.validate();
         batch.validate(clientState);
+        Tracing.trace("Executing batch");
         return batch.execute(queryState, options, requestTime);
     }
 
+    public ResultMessage processBatch(BatchStatement batch, QueryState queryState, BatchQueryOptions options, Dispatcher.RequestTime requestTime)
+    throws RequestExecutionException, RequestValidationException
+    {
+        return processBatch(batch, queryState, options, Collections.emptyMap(), requestTime);
+    }
+
     public static CQLStatement getStatement(String queryStr, ClientState clientState)
     throws RequestValidationException
     {
         Tracing.trace("Parsing {}", queryStr);
         CQLStatement.Raw statement = parseStatement(queryStr);
 
-        // Set keyspace for statement that require login
-        if (statement instanceof QualifiedStatement)
-            ((QualifiedStatement) statement).setKeyspace(clientState);
-
         Tracing.trace("Preparing statement");
         return statement.prepare(clientState);
     }
@@ -881,7 +983,9 @@ public static CQLStatement.Raw parseStatement(String queryStr) throws SyntaxExce
     {
         try
         {
-            return CQLFragmentParser.parseAnyUnhandled(CqlParser::query, queryStr);
+            CQLStatement.Raw stmt = CQLFragmentParser.parseAnyUnhandled(CqlParser::query, queryStr);
+            stmt.setRawCQLStatement(queryStr);
+            return stmt;
         }
         catch (CassandraException ce)
         {
@@ -901,9 +1005,15 @@ public static CQLStatement.Raw parseStatement(String queryStr) throws SyntaxExce
         }
     }
 
-    private static int measure(Object key, Prepared value)
+    private static int measurePStatementCacheEntrySize(Object key, Prepared value)
+    {
+        Pair<Prepared, Integer> valuePair = Pair.create(value, 0);
+        return Ints.checkedCast(ObjectSizes.measureDeep(key) + ObjectSizes.measureDeep(valuePair));
+    }
+
+    private static int getPrecomputedSize(Object key, Pair<Prepared, Integer> value)
     {
-        return Ints.checkedCast(ObjectSizes.measureDeep(key) + ObjectSizes.measureDeep(value));
+        return value.right;
     }
 
     /**
@@ -921,6 +1031,11 @@ public static void clearPreparedStatementsCache()
         preparedStatements.asMap().clear();
     }
 
+    private static Prepared extractPreparedFromPair(Pair<Prepared, Integer> pair)
+    {
+        return pair != null ? pair.left : null;
+    }
+
     private static class StatementInvalidatingListener implements SchemaChangeListener
     {
         private static void removeInvalidPreparedStatements(String ksName, String cfName)
@@ -933,11 +1048,11 @@ private static void removeInvalidPreparedStatementsForFunction(String ksName, St
         {
             Predicate<Function> matchesFunction = f -> ksName.equals(f.name().keyspace) && functionName.equals(f.name().name);
 
-            for (Iterator<Map.Entry<MD5Digest, Prepared>> iter = preparedStatements.asMap().entrySet().iterator();
+            for (Iterator<Map.Entry<MD5Digest, Pair<Prepared, Integer>>> iter = preparedStatements.asMap().entrySet().iterator();
                  iter.hasNext();)
             {
-                Map.Entry<MD5Digest, Prepared> pstmt = iter.next();
-                if (Iterables.any(pstmt.getValue().statement.getFunctions(), matchesFunction))
+                Map.Entry<MD5Digest, Pair<Prepared, Integer>> pstmt = iter.next();
+                if (Iterables.any(pstmt.getValue().left().statement.getFunctions(), matchesFunction))
                 {
                     SystemKeyspace.removePreparedStatement(pstmt.getKey());
                     iter.remove();
@@ -949,13 +1064,13 @@ private static void removeInvalidPreparedStatementsForFunction(String ksName, St
                                statement -> Iterables.any(statement.statement.getFunctions(), matchesFunction));
         }
 
-        private static void removeInvalidPersistentPreparedStatements(Iterator<Map.Entry<MD5Digest, Prepared>> iterator,
+        private static void removeInvalidPersistentPreparedStatements(Iterator<Map.Entry<MD5Digest, Pair<Prepared, Integer>>> iterator,
                                                                       String ksName, String cfName)
         {
             while (iterator.hasNext())
             {
-                Map.Entry<MD5Digest, Prepared> entry = iterator.next();
-                if (shouldInvalidate(ksName, cfName, entry.getValue().statement))
+                Map.Entry<MD5Digest, Pair<Prepared, Integer>> entry = iterator.next();
+                if (shouldInvalidate(ksName, cfName, entry.getValue().left().statement))
                 {
                     SystemKeyspace.removePreparedStatement(entry.getKey());
                     iterator.remove();
diff --git a/src/java/org/apache/cassandra/cql3/Relation.java b/src/java/org/apache/cassandra/cql3/Relation.java
index e2c3533c7671..42cf3c9c8287 100644
--- a/src/java/org/apache/cassandra/cql3/Relation.java
+++ b/src/java/org/apache/cassandra/cql3/Relation.java
@@ -86,6 +86,27 @@ public final boolean isContainsKey()
         return relationType == Operator.CONTAINS_KEY;
     }
 
+    /**
+     * Checks if the operator of this relation is a <code>NOT_CONTAINS</code>.
+     * @return <code>true</code>  if the operator of this relation is a <code>NOT_CONTAINS</code>, <code>false</code>
+     * otherwise.
+     */
+    public final boolean isNotContains()
+    {
+        return relationType == Operator.NOT_CONTAINS;
+    }
+
+    /**
+     * Checks if the operator of this relation is a <code>NOT_CONTAINS_KEY</code>.
+     * @return <code>true</code>  if the operator of this relation is a <code>NOT_CONTAINS_KEY</code>, <code>false</code>
+     * otherwise.
+     */
+    public final boolean isNotContainsKey()
+    {
+        return relationType == Operator.NOT_CONTAINS_KEY;
+    }
+
+
     /**
      * Checks if the operator of this relation is a <code>IN</code>.
      * @return <code>true</code>  if the operator of this relation is a <code>IN</code>, <code>false</code>
@@ -106,6 +127,16 @@ public final boolean isEQ()
         return relationType == Operator.EQ;
     }
 
+    /**
+     * Checks if the operator of this relation is a <code>NEQ</code>.
+     * @return <code>true</code>  if the operator of this relation is a <code>NEQ</code>, <code>false</code>
+     * otherwise.
+     */
+    public final boolean isNEQ()
+    {
+        return relationType == Operator.NEQ;
+    }
+
     public final boolean isLIKE()
     {
         return relationType == Operator.LIKE_PREFIX
@@ -115,6 +146,16 @@ public final boolean isLIKE()
                 || relationType == Operator.LIKE;
     }
 
+    public final boolean isNotLIKE()
+    {
+        return relationType == Operator.NOT_LIKE_PREFIX
+               || relationType == Operator.NOT_LIKE_SUFFIX
+               || relationType == Operator.NOT_LIKE_CONTAINS
+               || relationType == Operator.NOT_LIKE_MATCHES
+               || relationType == Operator.NOT_LIKE;
+    }
+
+
     /**
      * Checks if the operator of this relation is a <code>Slice</code> (GT, GTE, LTE, LT).
      *
@@ -141,13 +182,17 @@ public final Restriction toRestriction(TableMetadata table, VariableSpecificatio
         switch (relationType)
         {
             case EQ: return newEQRestriction(table, boundNames);
+            case NEQ: return newNEQRestriction(table, boundNames);
             case LT: return newSliceRestriction(table, boundNames, Bound.END, false);
             case LTE: return newSliceRestriction(table, boundNames, Bound.END, true);
             case GTE: return newSliceRestriction(table, boundNames, Bound.START, true);
             case GT: return newSliceRestriction(table, boundNames, Bound.START, false);
             case IN: return newINRestriction(table, boundNames);
+            case NOT_IN: return newNotINRestriction(table, boundNames);
             case CONTAINS: return newContainsRestriction(table, boundNames, false);
             case CONTAINS_KEY: return newContainsRestriction(table, boundNames, true);
+            case NOT_CONTAINS: return newNotContainsRestriction(table, boundNames, false);
+            case NOT_CONTAINS_KEY: return newNotContainsRestriction(table, boundNames, true);
             case IS_NOT: return newIsNotRestriction(table, boundNames);
             case LIKE_PREFIX:
             case LIKE_SUFFIX:
@@ -156,7 +201,11 @@ public final Restriction toRestriction(TableMetadata table, VariableSpecificatio
             case LIKE:
                 return newLikeRestriction(table, boundNames, relationType);
             case ANN:
-                throw invalidRequest("ANN is only supported in ORDER BY");
+                return newAnnRestriction(table, boundNames);
+            case BM25:
+                return newBm25Restriction(table, boundNames);
+            case ANALYZER_MATCHES:
+                return newAnalyzerMatchesRestriction(table, boundNames);
             default: throw invalidRequest("Unsupported \"!=\" relation: %s", this);
         }
     }
@@ -171,6 +220,17 @@ public final Restriction toRestriction(TableMetadata table, VariableSpecificatio
      */
     protected abstract Restriction newEQRestriction(TableMetadata table, VariableSpecifications boundNames);
 
+    /**
+     * Creates a new NEQ restriction instance.
+     *
+     * @param table the table meta data
+     * @param boundNames the variables specification where to collect the bind variables
+     * @return a new NEQ restriction instance.
+     * @throws InvalidRequestException if the relation cannot be converted into an NEQ restriction.
+     */
+    protected abstract Restriction newNEQRestriction(TableMetadata table, VariableSpecifications boundNames);
+
+
     /**
      * Creates a new IN restriction instance.
      *
@@ -181,6 +241,17 @@ public final Restriction toRestriction(TableMetadata table, VariableSpecificatio
      */
     protected abstract Restriction newINRestriction(TableMetadata table, VariableSpecifications boundNames);
 
+
+    /**
+     * Creates a new NOT IN restriction instance.
+     *
+     * @param table the table meta data
+     * @param boundNames the variables specification where to collect the bind variables
+     * @return a new NOT IN restriction instance
+     * @throws InvalidRequestException if the relation cannot be converted into an NOT IN restriction.
+     */
+    protected abstract Restriction newNotINRestriction(TableMetadata table, VariableSpecifications boundNames);
+
     /**
      * Creates a new Slice restriction instance.
      *
@@ -207,10 +278,36 @@ protected abstract Restriction newSliceRestriction(TableMetadata table,
      */
     protected abstract Restriction newContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey);
 
+    /**
+     * Creates a new Contains restriction instance representing a NOT CONTAINS operation.
+     *
+     * @param table the table meta data
+     * @param boundNames the variables specification where to collect the bind variables
+     * @param isKey <code>true</code> if the restriction to create is a NOT CONTAINS KEY
+     * @return a new Contains <code>Restriction</code> instance
+     * @throws InvalidRequestException if the <code>Relation</code> is not valid
+     */
+    protected abstract Restriction newNotContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey);
+
     protected abstract Restriction newIsNotRestriction(TableMetadata table, VariableSpecifications boundNames);
 
     protected abstract Restriction newLikeRestriction(TableMetadata table, VariableSpecifications boundNames, Operator operator);
 
+    /**
+     * Creates a new ANN restriction instance.
+     */
+    protected abstract Restriction newAnnRestriction(TableMetadata table, VariableSpecifications boundNames);
+
+    /**
+     * Creates a new BM25 restriction instance.
+     */
+    protected abstract Restriction newBm25Restriction(TableMetadata table, VariableSpecifications boundNames);
+
+    /**
+     * Creates a new Analyzer Matches restriction instance.
+     */
+    protected abstract Restriction newAnalyzerMatchesRestriction(TableMetadata table, VariableSpecifications boundNames);
+
     /**
      * Converts the specified <code>Raw</code> into a <code>Term</code>.
      * @param receivers the columns to which the values must be associated at
diff --git a/src/java/org/apache/cassandra/cql3/ReservedTypeNames.java b/src/java/org/apache/cassandra/cql3/ReservedTypeNames.java
new file mode 100644
index 000000000000..90c74d50b079
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/ReservedTypeNames.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+
+/**
+ * Set of reserved type names. When creating a user type via CQL, if the type's name is included in this
+ * set, the statement will fail unless the name is double quoted.
+ */
+public class ReservedTypeNames
+{
+    public static final Set<String> reservedTypeNames = Sets.newHashSet(
+    "byte",
+    "complex",
+    "enum",
+    "date",
+    "interval",
+    "macaddr",
+    "bitstring");
+
+    public static boolean isReserved(String typeName)
+    {
+        return reservedTypeNames.contains(typeName);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/cql3/ResultSet.java b/src/java/org/apache/cassandra/cql3/ResultSet.java
index de393f73105c..336164835bd6 100644
--- a/src/java/org/apache/cassandra/cql3/ResultSet.java
+++ b/src/java/org/apache/cassandra/cql3/ResultSet.java
@@ -27,6 +27,7 @@
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.UnaryOperator;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -46,7 +47,7 @@ public class ResultSet
     public static final Codec codec = new Codec();
 
     public final ResultMetadata metadata;
-    public final List<List<ByteBuffer>> rows;
+    public List<List<ByteBuffer>> rows;
 
     public ResultSet(ResultMetadata resultMetadata)
     {
@@ -93,14 +94,11 @@ public void reverse()
         Collections.reverse(rows);
     }
 
-    public void trim(int limit)
+    public ResultSet withMetadata(ResultMetadata newMetadata)
     {
-        int toRemove = rows.size() - limit;
-        if (toRemove > 0)
-        {
-            for (int i = 0; i < toRemove; i++)
-                rows.remove(rows.size() - 1);
-        }
+        if (newMetadata != metadata)
+            return new ResultSet(newMetadata, rows);
+        return this;
     }
 
     @Override
@@ -139,6 +137,15 @@ public String toString()
         }
     }
 
+    public ResultSet withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+    {
+        if (keyspaceMapper == Constants.IDENTITY_STRING_MAPPER)
+            return this;
+
+        ResultMetadata newMetadata = metadata.withOverriddenKeyspace(keyspaceMapper);
+        return metadata == newMetadata ? this : new ResultSet(newMetadata, rows);
+    }
+
     public static class Codec implements CBCodec<ResultSet>
     {
         /*
@@ -228,7 +235,8 @@ public ResultMetadata(List<ColumnSpecification> names, PagingState pagingState)
                 flags.add(Flag.GLOBAL_TABLES_SPEC);
         }
 
-        private ResultMetadata(MD5Digest resultMetadataId, EnumSet<Flag> flags, List<ColumnSpecification> names, int columnCount, PagingState pagingState)
+        @VisibleForTesting
+        ResultMetadata(MD5Digest resultMetadataId, EnumSet<Flag> flags, List<ColumnSpecification> names, int columnCount, PagingState pagingState)
         {
             this.resultMetadataId = resultMetadataId;
             this.flags = flags;
@@ -237,6 +245,13 @@ private ResultMetadata(MD5Digest resultMetadataId, EnumSet<Flag> flags, List<Col
             this.pagingState = pagingState;
         }
 
+        public ResultMetadata withRecomputedResultMetadataIdIfMissing()
+        {
+            if (this.resultMetadataId == null)
+                return new ResultMetadata(computeResultMetadataId(names), EnumSet.copyOf(flags), names, columnCount, pagingState);
+            return this;
+        }
+
         public ResultMetadata copy()
         {
             return new ResultMetadata(resultMetadataId, EnumSet.copyOf(flags), names, columnCount, pagingState);
@@ -366,6 +381,23 @@ public String toString()
             return sb.toString();
         }
 
+        public ResultMetadata withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+        {
+            if ((keyspaceMapper == Constants.IDENTITY_STRING_MAPPER) || (names == null))
+                return this;
+
+            boolean changed = false;
+            List<ColumnSpecification> newNames = new ArrayList<>(names.size());
+            for (ColumnSpecification cs : names)
+            {
+                ColumnSpecification newColumnSpecification = cs.withOverriddenKeyspace(keyspaceMapper);
+                newNames.add(newColumnSpecification);
+                if (newColumnSpecification != cs)
+                    changed = true;
+            }
+            return changed ? new ResultMetadata(computeResultMetadataId(newNames), EnumSet.copyOf(flags), newNames, columnCount, pagingState) : this;
+        }
+
         private static class Codec implements CBCodec<ResultMetadata>
         {
             public ResultMetadata decode(ByteBuf body, ProtocolVersion version)
@@ -430,7 +462,7 @@ public void encode(ResultMetadata m, ByteBuf dest, ProtocolVersion version)
                 if (hasMorePages)
                     CBUtil.writeValue(m.pagingState.serialize(version), dest);
 
-                if (version.isGreaterOrEqualTo(ProtocolVersion.V5)  && metadataChanged)
+                if (version.isGreaterOrEqualTo(ProtocolVersion.V5) && metadataChanged)
                 {
                     assert !noMetadata : "MetadataChanged and NoMetadata are mutually exclusive flags";
                     CBUtil.writeBytes(m.getResultMetadataId().bytes, dest);
@@ -578,6 +610,23 @@ public static PreparedMetadata fromPrepared(CQLStatement statement)
             return new PreparedMetadata(statement.getBindVariables(), statement.getPartitionKeyBindVariableIndexes());
         }
 
+        public PreparedMetadata withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+        {
+            if (keyspaceMapper == Constants.IDENTITY_STRING_MAPPER)
+                return this;
+
+            boolean changed = false;
+            List<ColumnSpecification> newNames = new ArrayList<>(names.size());
+            for (ColumnSpecification cs : names)
+            {
+                ColumnSpecification newColumnSpecification = cs.withOverriddenKeyspace(keyspaceMapper);
+                newNames.add(newColumnSpecification);
+                if (newColumnSpecification != cs)
+                    changed = true;
+            }
+            return changed ? new PreparedMetadata(EnumSet.copyOf(flags), newNames, partitionKeyBindIndexes == null ? null : Arrays.copyOf(partitionKeyBindIndexes, partitionKeyBindIndexes.length)) : this;
+        }
+
         private static class Codec implements CBCodec<PreparedMetadata>
         {
             public PreparedMetadata decode(ByteBuf body, ProtocolVersion version)
diff --git a/src/java/org/apache/cassandra/cql3/Sets.java b/src/java/org/apache/cassandra/cql3/Sets.java
index 00d6870a9206..0e039cfe72b8 100644
--- a/src/java/org/apache/cassandra/cql3/Sets.java
+++ b/src/java/org/apache/cassandra/cql3/Sets.java
@@ -17,8 +17,6 @@
  */
 package org.apache.cassandra.cql3;
 
-import static org.apache.cassandra.cql3.Constants.UNSET_VALUE;
-
 import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.Comparator;
@@ -26,7 +24,6 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -39,7 +36,6 @@
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.marshal.ReversedType;
 import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.CellPath;
@@ -50,6 +46,8 @@
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
+import static org.apache.cassandra.cql3.Constants.UNSET_VALUE;
+
 /**
  * Static helper methods and classes for sets.
  */
@@ -62,14 +60,9 @@ public static ColumnSpecification valueSpecOf(ColumnSpecification column)
         return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), elementsType(column.type));
     }
 
-    private static AbstractType<?> unwrap(AbstractType<?> type)
-    {
-        return type.isReversed() ? unwrap(((ReversedType<?>) type).baseType) : type;
-    }
-
     private static AbstractType<?> elementsType(AbstractType<?> type)
     {
-        return ((SetType<?>) unwrap(type)).getElementsType();
+        return ((SetType<?>) type.unwrap()).getElementsType();
     }
 
     /**
@@ -134,8 +127,8 @@ public static <T> String setToString(Iterable<T> items, java.util.function.Funct
     public static <T> SetType<?> getExactSetTypeIfKnown(List<T> items,
                                                         java.util.function.Function<T, AbstractType<?>> mapper)
     {
-        Optional<AbstractType<?>> type = items.stream().map(mapper).filter(Objects::nonNull).findFirst();
-        return type.isPresent() ? SetType.getInstance(type.get(), false) : null;
+        AbstractType<?> type = Lists.getElementType(items, mapper);
+        return type != null ? SetType.getInstance(type, false) : null;
     }
 
     public static <T> SetType<?> getPreferredCompatibleType(List<T> items,
@@ -143,7 +136,7 @@ public static <T> SetType<?> getPreferredCompatibleType(List<T> items,
     {
         Set<AbstractType<?>> types = items.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toSet());
         AbstractType<?> type = AssignmentTestable.getCompatibleTypeIfKnown(types);
-        return type == null ? null : SetType.getInstance(type, false);
+        return type == null ? null : SetType.getInstance(type.freeze(), false);
     }
 
     public static class Literal extends Term.Raw
@@ -185,7 +178,7 @@ public Term prepare(String keyspace, ColumnSpecification receiver) throws Invali
 
         private void validateAssignableTo(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
         {
-            AbstractType<?> type = unwrap(receiver.type);
+            AbstractType<?> type = receiver.type.unwrap();
 
             if (!(type instanceof SetType))
             {
@@ -367,6 +360,7 @@ public Adder(ColumnMetadata column, Term t)
             super(column, t);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to add items to a frozen set";
@@ -426,6 +420,7 @@ public Discarder(ColumnMetadata column, Term t)
             super(column, t);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to remove items from a frozen set";
@@ -451,6 +446,7 @@ public ElementDiscarder(ColumnMetadata column, Term k)
             super(column, k);
         }
 
+        @Override
         public void execute(DecoratedKey partitionKey, UpdateParameters params) throws InvalidRequestException
         {
             assert column.type.isMultiCell() : "Attempted to delete a single element in a frozen set";
diff --git a/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java b/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java
index cf1cb69066e6..34e846db6e28 100644
--- a/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java
+++ b/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java
@@ -22,6 +22,9 @@
 import java.util.List;
 import java.util.Objects;
 
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.index.sai.analyzer.AnalyzerEqOperatorSupport;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.Term.Raw;
@@ -32,6 +35,7 @@
 import org.apache.cassandra.db.marshal.ListType;
 import org.apache.cassandra.db.marshal.MapType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.service.ClientWarn;
 
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
@@ -101,6 +105,11 @@ public static SingleColumnRelation createInRelation(ColumnIdentifier entity, Lis
         return new SingleColumnRelation(entity, null, Operator.IN, null, inValues);
     }
 
+    public static SingleColumnRelation createNotInRelation(ColumnIdentifier entity, List<Term.Raw> inValues)
+    {
+        return new SingleColumnRelation(entity, null, Operator.NOT_IN, null, inValues);
+    }
+
     public ColumnIdentifier getEntity()
     {
         return entity;
@@ -150,7 +159,7 @@ public String toCQLString()
             entityAsString = String.format("%s[%s]", entityAsString, mapKey);
 
         if (isIN())
-            return String.format("%s IN %s", entityAsString, Tuples.tupleToString(inValues));
+            return String.format("%s IN %s", entityAsString, inValues == null ? value : Tuples.tupleToString(inValues));
 
         return String.format("%s %s %s", entityAsString, relationType, value);
     }
@@ -185,12 +194,49 @@ protected Restriction newEQRestriction(TableMetadata table, VariableSpecificatio
         if (mapKey == null)
         {
             Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
-            return new SingleColumnRestriction.EQRestriction(columnDef, term);
+            // Leave the restriction as EQ if no analyzed index in backwards compatibility mode is present
+            var ebi = IndexRegistry.obtain(table).getEqBehavior(columnDef);
+            if (ebi.behavior == IndexRegistry.EqBehavior.EQ)
+                return new SingleColumnRestriction.EQRestriction(columnDef, term);
+
+            // the index is configured to transform EQ into MATCH for backwards compatibility
+            if (ebi.behavior == IndexRegistry.EqBehavior.MATCH)
+            {
+                ClientWarn.instance.warn(String.format(AnalyzerEqOperatorSupport.EQ_RESTRICTION_ON_ANALYZED_WARNING,
+                                                       columnDef.toString(),
+                                                       ebi.matchIndex.getIndexMetadata().name),
+                                         columnDef);
+                return new SingleColumnRestriction.AnalyzerMatchesRestriction(columnDef, term);
+            }
+
+            // multiple indexes support EQ, this is unsupported
+            assert ebi.behavior == IndexRegistry.EqBehavior.AMBIGUOUS;
+            throw invalidRequest(AnalyzerEqOperatorSupport.EQ_AMBIGUOUS_ERROR,
+                                 columnDef.toString(),
+                                 ebi.matchIndex.getIndexMetadata().name,
+                                 ebi.eqIndex.getIndexMetadata().name);
+        }
+        List<? extends ColumnSpecification> receivers = toReceivers(columnDef);
+        Term entryKey = toTerm(Collections.singletonList(receivers.get(0)), mapKey, table.keyspace, boundNames);
+        Term entryValue = toTerm(Collections.singletonList(receivers.get(1)), value, table.keyspace, boundNames);
+        return new SingleColumnRestriction.ContainsRestriction(columnDef, entryKey, entryValue, false);
+    }
+
+    @Override
+    protected Restriction newNEQRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        ColumnMetadata columnDef = table.getExistingColumn(entity);
+        if (mapKey == null)
+        {
+            Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
+            MarkerOrTerms skippedValues = new MarkerOrTerms.Terms(Collections.singletonList(term));
+            return SingleColumnRestriction.SliceRestriction.fromSkippedValues(columnDef, skippedValues);
         }
+
         List<? extends ColumnSpecification> receivers = toReceivers(columnDef);
         Term entryKey = toTerm(Collections.singletonList(receivers.get(0)), mapKey, table.keyspace, boundNames);
         Term entryValue = toTerm(Collections.singletonList(receivers.get(1)), value, table.keyspace, boundNames);
-        return new SingleColumnRestriction.ContainsRestriction(columnDef, entryKey, entryValue);
+        return new SingleColumnRestriction.ContainsRestriction(columnDef, entryKey, entryValue, true);
     }
 
     @Override
@@ -202,14 +248,33 @@ protected Restriction newINRestriction(TableMetadata table, VariableSpecificatio
         if (terms == null)
         {
             Term term = toTerm(receivers, value, table.keyspace, boundNames);
-            return new SingleColumnRestriction.InRestrictionWithMarker(columnDef, (Lists.Marker) term);
+            return new SingleColumnRestriction.INRestriction(columnDef, new MarkerOrTerms.Marker((Lists.Marker) term));
         }
 
         // An IN restrictions with only one element is the same than an EQ restriction
         if (terms.size() == 1)
             return new SingleColumnRestriction.EQRestriction(columnDef, terms.get(0));
 
-        return new SingleColumnRestriction.InRestrictionWithValues(columnDef, terms);
+        return new SingleColumnRestriction.INRestriction(columnDef, new MarkerOrTerms.Terms(terms));
+    }
+
+    @Override
+    protected Restriction newNotINRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        ColumnMetadata columnDef = table.getExistingColumn(entity);
+        List<? extends ColumnSpecification> receivers = toReceivers(columnDef);
+        List<Term> terms = toTerms(receivers, inValues, table.keyspace, boundNames);
+        MarkerOrTerms values;
+        if (terms == null)
+        {
+            Term term = toTerm(receivers, value, table.keyspace, boundNames);
+            values = new MarkerOrTerms.Marker((Lists.Marker) term);
+        }
+        else
+        {
+            values = new MarkerOrTerms.Terms(terms);
+        }
+        return SingleColumnRestriction.SliceRestriction.fromSkippedValues(columnDef, values);
     }
 
     @Override
@@ -228,8 +293,15 @@ protected Restriction newSliceRestriction(TableMetadata table,
             throw invalidRequest("Slice restrictions are not supported on duration columns");
         }
 
-        Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
-        return new SingleColumnRestriction.SliceRestriction(columnDef, bound, inclusive, term);
+        if (mapKey == null)
+        {
+            Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
+            return SingleColumnRestriction.SliceRestriction.fromBound(columnDef, bound, inclusive, term);
+        }
+        List<? extends ColumnSpecification> receivers = toReceivers(columnDef);
+        Term entryKey = toTerm(Collections.singletonList(receivers.get(0)), mapKey, table.keyspace, boundNames);
+        Term entryValue = toTerm(Collections.singletonList(receivers.get(1)), value, table.keyspace, boundNames);
+        return new SingleColumnRestriction.MapSliceRestriction(columnDef, bound, inclusive, entryKey, entryValue);
     }
 
     @Override
@@ -239,7 +311,17 @@ protected Restriction newContainsRestriction(TableMetadata table,
     {
         ColumnMetadata columnDef = table.getExistingColumn(entity);
         Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
-        return new SingleColumnRestriction.ContainsRestriction(columnDef, term, isKey);
+        return new SingleColumnRestriction.ContainsRestriction(columnDef, term, isKey, false);
+    }
+
+    @Override
+    protected Restriction newNotContainsRestriction(TableMetadata table,
+                                                 VariableSpecifications boundNames,
+                                                 boolean isKey) throws InvalidRequestException
+    {
+        ColumnMetadata columnDef = table.getExistingColumn(entity);
+        Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
+        return new SingleColumnRestriction.ContainsRestriction(columnDef, term, isKey, true);
     }
 
     @Override
@@ -264,6 +346,36 @@ protected Restriction newLikeRestriction(TableMetadata table, VariableSpecificat
         return new SingleColumnRestriction.LikeRestriction(columnDef, operator, term);
     }
 
+    @Override
+    protected Restriction newAnnRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        ColumnMetadata columnDef = table.getExistingColumn(entity);
+        if (!(columnDef.type instanceof VectorType))
+            throw invalidRequest("ANN is only supported against DENSE FLOAT32 columns");
+        Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
+        return new SingleColumnRestriction.AnnRestriction(columnDef, term);
+    }
+
+    @Override
+    protected Restriction newBm25Restriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        ColumnMetadata columnDef = table.getExistingColumn(entity);
+        Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
+        return new SingleColumnRestriction.Bm25Restriction(columnDef, term);
+    }
+
+    @Override
+    protected Restriction newAnalyzerMatchesRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        if (mapKey != null)
+            throw invalidRequest("%s can't be used with collections.", operator());
+
+        ColumnMetadata columnDef = table.getExistingColumn(entity);
+        Term term = toTerm(toReceivers(columnDef), value, table.keyspace, boundNames);
+
+        return new SingleColumnRestriction.AnalyzerMatchesRestriction(columnDef, term);
+    }
+
     /**
      * Returns the receivers for this relation.
      * @param columnDef the column definition
@@ -276,13 +388,15 @@ private List<? extends ColumnSpecification> toReceivers(ColumnMetadata columnDef
 
         checkFalse(isContainsKey() && !(receiver.type instanceof MapType), "Cannot use CONTAINS KEY on non-map column %s", receiver.name);
         checkFalse(isContains() && !(receiver.type.isCollection()), "Cannot use CONTAINS on non-collection column %s", receiver.name);
+        checkFalse(isNotContainsKey() && !(receiver.type instanceof MapType), "Cannot use NOT CONTAINS KEY on non-map column %s", receiver.name);
+        checkFalse(isNotContains() && !(receiver.type.isCollection()), "Cannot use NOT CONTAINS on non-collection column %s", receiver.name);
 
         if (mapKey != null)
         {
             checkFalse(receiver.type instanceof ListType, "Indexes on list entries (%s[index] = value) are not currently supported.", receiver.name);
             checkTrue(receiver.type instanceof MapType, "Column %s cannot be used as a map", receiver.name);
             checkTrue(receiver.type.isMultiCell(), "Map-entry equality predicates on frozen map column %s are not supported", receiver.name);
-            checkTrue(isEQ(), "Only EQ relations are supported on map entries");
+            checkTrue(isEQ() || isNEQ() || isSlice(), "Only EQ, NEQ, and SLICE relations are supported on map entries");
         }
 
         // Non-frozen UDTs don't support any operator
@@ -300,11 +414,11 @@ private List<? extends ColumnSpecification> toReceivers(ColumnMetadata columnDef
                        receiver.type.asCQL3Type(),
                        operator());
 
-            if (isContainsKey() || isContains())
+            if (isContainsKey() || isContains() || isNotContains() || isNotContainsKey())
             {
-                receiver = makeCollectionReceiver(receiver, isContainsKey());
+                receiver = makeCollectionReceiver(receiver, isContainsKey() || isNotContainsKey());
             }
-            else if (receiver.type.isMultiCell() && mapKey != null && isEQ())
+            else if (receiver.type.isMultiCell() && isMapEntryComparison())
             {
                 List<ColumnSpecification> receivers = new ArrayList<>(2);
                 receivers.add(makeCollectionReceiver(receiver, true));
@@ -323,12 +437,12 @@ private static ColumnSpecification makeCollectionReceiver(ColumnSpecification re
 
     private boolean isLegalRelationForNonFrozenCollection()
     {
-        return isContainsKey() || isContains() || isMapEntryEquality();
+        return isContainsKey() || isContains() || isNotContains() || isNotContainsKey() || isMapEntryComparison();
     }
 
-    private boolean isMapEntryEquality()
+    private boolean isMapEntryComparison()
     {
-        return mapKey != null && isEQ();
+        return mapKey != null && (isEQ() || isNEQ() || isSlice());
     }
 
     private boolean canHaveOnlyOneValue()
diff --git a/src/java/org/apache/cassandra/cql3/Term.java b/src/java/org/apache/cassandra/cql3/Term.java
index c94b6141af0a..d19e1c099ce4 100644
--- a/src/java/org/apache/cassandra/cql3/Term.java
+++ b/src/java/org/apache/cassandra/cql3/Term.java
@@ -65,7 +65,7 @@ public interface Term
      * Whether or not that term contains at least one bind marker.
      *
      * Note that this is slightly different from being or not a NonTerminal,
-     * because calls to non pure functions will be NonTerminal (see #5616)
+     * because calls to non-deterministic functions will be NonTerminal (see #5616)
      * even if they don't have bind markers.
      */
     public abstract boolean containsBindMarker();
@@ -151,15 +151,15 @@ public abstract class MultiColumnRaw extends Term.Raw
 
     /**
      * A terminal term, one that can be reduced to a byte buffer directly.
-     *
+     * </p>
      * This includes most terms that don't have a bind marker (an exception
-     * being delayed call for non pure function that are NonTerminal even
+     * being delayed call for non-deterministic function that are NonTerminal even
      * if they don't have bind markers).
-     *
+     * </p>
      * This can be only one of:
      *   - a constant value
      *   - a collection value
-     *
+     * </p>
      * Note that a terminal term will always have been type checked, and thus
      * consumer can (and should) assume so.
      */
@@ -190,10 +190,20 @@ public boolean isTerminal()
          */
         public abstract ByteBuffer get(ProtocolVersion version) throws InvalidRequestException;
 
+        public ByteBuffer getVector(ProtocolVersion protocolVersion) throws InvalidRequestException
+        {
+            throw new InvalidRequestException("Doesn't support getVector");
+        }
+
         public ByteBuffer bindAndGet(QueryOptions options) throws InvalidRequestException
         {
             return get(options.getProtocolVersion());
         }
+
+        public ByteBuffer bindAndGetVector(QueryOptions options) throws InvalidRequestException
+        {
+            return getVector(options.getProtocolVersion());
+        }
     }
 
     public abstract class MultiItemTerminal extends Terminal
@@ -202,14 +212,14 @@ public abstract class MultiItemTerminal extends Terminal
     }
 
     /**
-     * A non terminal term, i.e. a term that can only be reduce to a byte buffer
+     * A non-terminal term, i.e. a term that can only be reduce to a byte buffer
      * at execution time.
-     *
+     * </p>
      * We have the following type of NonTerminal:
      *   - marker for a constant value
      *   - marker for a collection value (list, set, map)
      *   - a function having bind marker
-     *   - a non pure function (even if it doesn't have bind marker - see #5616)
+     *   - a non-deterministic function (even if it doesn't have bind marker - see #5616)
      */
     public abstract class NonTerminal implements Term
     {
diff --git a/src/java/org/apache/cassandra/cql3/TokenRelation.java b/src/java/org/apache/cassandra/cql3/TokenRelation.java
index 139c55d35862..ca849dc82a30 100644
--- a/src/java/org/apache/cassandra/cql3/TokenRelation.java
+++ b/src/java/org/apache/cassandra/cql3/TokenRelation.java
@@ -83,12 +83,23 @@ protected Restriction newEQRestriction(TableMetadata table, VariableSpecificatio
         return new TokenRestriction.EQRestriction(table, columnDefs, term);
     }
 
+    @Override
+    protected Restriction newNEQRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the token function", operator());
+    }
+
     @Override
     protected Restriction newINRestriction(TableMetadata table, VariableSpecifications boundNames)
     {
         throw invalidRequest("%s cannot be used with the token function", operator());
     }
 
+    protected Restriction newNotINRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used with the token function", operator());
+    }
+
     @Override
     protected Restriction newSliceRestriction(TableMetadata table,
                                               VariableSpecifications boundNames,
@@ -106,6 +117,12 @@ protected Restriction newContainsRestriction(TableMetadata table, VariableSpecif
         throw invalidRequest("%s cannot be used with the token function", operator());
     }
 
+    @Override
+    protected Restriction newNotContainsRestriction(TableMetadata table, VariableSpecifications boundNames, boolean isKey)
+    {
+        throw invalidRequest("%s cannot be used with the token function", operator());
+    }
+
     @Override
     protected Restriction newIsNotRestriction(TableMetadata table, VariableSpecifications boundNames)
     {
@@ -118,6 +135,24 @@ protected Restriction newLikeRestriction(TableMetadata table, VariableSpecificat
         throw invalidRequest("%s cannot be used with the token function", operator);
     }
 
+    @Override
+    protected Restriction newAnnRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used for token relations", operator());
+    }
+
+    @Override
+    protected Restriction newBm25Restriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used for token relations", operator());
+    }
+
+    @Override
+    protected Restriction newAnalyzerMatchesRestriction(TableMetadata table, VariableSpecifications boundNames)
+    {
+        throw invalidRequest("%s cannot be used for token relations", operator());
+    }
+
     @Override
     protected Term toTerm(List<? extends ColumnSpecification> receivers,
                           Raw raw,
diff --git a/src/java/org/apache/cassandra/cql3/Tuples.java b/src/java/org/apache/cassandra/cql3/Tuples.java
index 60f963ce4fe2..53ce0a419347 100644
--- a/src/java/org/apache/cassandra/cql3/Tuples.java
+++ b/src/java/org/apache/cassandra/cql3/Tuples.java
@@ -24,11 +24,12 @@
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
+import com.google.common.collect.ImmutableList;
+
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.db.marshal.ListType;
-import org.apache.cassandra.db.marshal.ReversedType;
 import org.apache.cassandra.db.marshal.TupleType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.serializers.MarshalException;
@@ -94,7 +95,7 @@ public Term prepare(String keyspace, List<? extends ColumnSpecification> receive
                 throw new InvalidRequestException(String.format("Expected %d elements in value tuple, but got %d: %s", receivers.size(), elements.size(), this));
 
             List<Term> values = new ArrayList<>(elements.size());
-            List<AbstractType<?>> types = new ArrayList<>(elements.size());
+            ImmutableList.Builder<AbstractType<?>> types = ImmutableList.builderWithExpectedSize(elements.size());
             boolean allTerminal = true;
             for (int i = 0; i < elements.size(); i++)
             {
@@ -105,7 +106,7 @@ public Term prepare(String keyspace, List<? extends ColumnSpecification> receive
                 values.add(t);
                 types.add(receivers.get(i).type);
             }
-            DelayedValue value = new DelayedValue(new TupleType(types), values);
+            DelayedValue value = new DelayedValue(new TupleType(types.build()), values);
             return allTerminal ? value.bind(QueryOptions.DEFAULT) : value;
         }
 
@@ -122,7 +123,7 @@ public AssignmentTestable.TestResult testAssignment(String keyspace, ColumnSpeci
         @Override
         public AbstractType<?> getExactTypeIfKnown(String keyspace)
         {
-            List<AbstractType<?>> types = new ArrayList<>(elements.size());
+            ImmutableList.Builder<AbstractType<?>> types = ImmutableList.builderWithExpectedSize(elements.size());
             for (Term.Raw term : elements)
             {
                 AbstractType<?> type = term.getExactTypeIfKnown(keyspace);
@@ -130,7 +131,7 @@ public AbstractType<?> getExactTypeIfKnown(String keyspace)
                     return null;
                 types.add(type);
             }
-            return new TupleType(types);
+            return new TupleType(types.build());
         }
 
         public String getText()
@@ -175,7 +176,7 @@ public List<ByteBuffer> getElements()
     }
 
     /**
-     * Similar to Value, but contains at least one NonTerminal, such as a non-pure functions or bind marker.
+     * Similar to Value, but contains at least one NonTerminal, such as a non-deterministic functions or bind marker.
      */
     public static class DelayedValue extends Term.NonTerminal
     {
@@ -306,7 +307,7 @@ public Raw(int bindIndex)
 
         private static ColumnSpecification makeReceiver(List<? extends ColumnSpecification> receivers)
         {
-            List<AbstractType<?>> types = new ArrayList<>(receivers.size());
+            ImmutableList.Builder<AbstractType<?>> types = ImmutableList.builderWithExpectedSize(receivers.size());
             StringBuilder inName = new StringBuilder("(");
             for (int i = 0; i < receivers.size(); i++)
             {
@@ -319,7 +320,7 @@ private static ColumnSpecification makeReceiver(List<? extends ColumnSpecificati
             inName.append(')');
 
             ColumnIdentifier identifier = new ColumnIdentifier(inName.toString(), true);
-            TupleType type = new TupleType(types);
+            TupleType type = new TupleType(types.build());
             return new ColumnSpecification(receivers.get(0).ksName, receivers.get(0).cfName, identifier, type);
         }
 
@@ -346,7 +347,7 @@ public INRaw(int bindIndex)
 
         private static ColumnSpecification makeInReceiver(List<? extends ColumnSpecification> receivers) throws InvalidRequestException
         {
-            List<AbstractType<?>> types = new ArrayList<>(receivers.size());
+            ImmutableList.Builder<AbstractType<?>> types = ImmutableList.builderWithExpectedSize(receivers.size());
             StringBuilder inName = new StringBuilder("in(");
             for (int i = 0; i < receivers.size(); i++)
             {
@@ -363,8 +364,8 @@ private static ColumnSpecification makeInReceiver(List<? extends ColumnSpecifica
             inName.append(')');
 
             ColumnIdentifier identifier = new ColumnIdentifier(inName.toString(), true);
-            TupleType type = new TupleType(types);
-            return new ColumnSpecification(receivers.get(0).ksName, receivers.get(0).cfName, identifier, ListType.getInstance(type, false));
+            TupleType type = new TupleType(types.build());
+            return new ColumnSpecification(receivers.get(0).ksName, receivers.get(0).cfName, identifier, ListType.getInstance(type.freeze(), false));
         }
 
         public AbstractType<?> getExactTypeIfKnown(String keyspace)
@@ -455,7 +456,7 @@ public static <T> String tupleToString(Iterable<T> items, java.util.function.Fun
     public static <T> TupleType getExactTupleTypeIfKnown(List<T> items,
                                                          java.util.function.Function<T, AbstractType<?>> mapper)
     {
-        List<AbstractType<?>> types = new ArrayList<>(items.size());
+        ImmutableList.Builder<AbstractType<?>> types = ImmutableList.builderWithExpectedSize(items.size());
         for (T item : items)
         {
             AbstractType<?> type = mapper.apply(item);
@@ -463,7 +464,7 @@ public static <T> TupleType getExactTupleTypeIfKnown(List<T> items,
                 return null;
             types.add(type);
         }
-        return new TupleType(types);
+        return new TupleType(types.build());
     }
 
     /**
@@ -518,13 +519,11 @@ public static AssignmentTestable.TestResult testTupleAssignment(ColumnSpecificat
 
     public static boolean checkIfTupleType(AbstractType<?> tuple)
     {
-        return (tuple instanceof TupleType) ||
-               (tuple instanceof ReversedType && ((ReversedType<?>) tuple).baseType instanceof TupleType);
-
+        return tuple.unwrap() instanceof TupleType;
     }
 
     public static TupleType getTupleType(AbstractType<?> tuple)
     {
-        return (tuple instanceof ReversedType ? ((TupleType) ((ReversedType<?>) tuple).baseType) : (TupleType)tuple);
+        return (TupleType) tuple.unwrap();
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/UTName.java b/src/java/org/apache/cassandra/cql3/UTName.java
index c8567977bdb0..8d4655a80f50 100644
--- a/src/java/org/apache/cassandra/cql3/UTName.java
+++ b/src/java/org/apache/cassandra/cql3/UTName.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.cql3;
 
 import java.nio.ByteBuffer;
+import java.util.function.UnaryOperator;
 
 public class UTName
 {
@@ -40,6 +41,12 @@ public void setKeyspace(String keyspace)
         this.ksName = keyspace;
     }
 
+    public void updateKeyspaceIfDefined(UnaryOperator<String> update)
+    {
+        if (hasKeyspace())
+            setKeyspace(update.apply(getKeyspace()));
+    }
+
     public String getKeyspace()
     {
         return ksName;
diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
index a0201c500a39..a995c8895641 100644
--- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
+++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
@@ -40,7 +40,23 @@
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.ReadExecutionController;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.ComplexColumnData;
@@ -56,6 +72,11 @@
 /** a utility for doing internal cql-based queries */
 public abstract class UntypedResultSet implements Iterable<UntypedResultSet.Row>
 {
+    public Stream<Row> stream()
+    {
+        return StreamSupport.stream(spliterator(), false);
+    }
+
     public static UntypedResultSet create(ResultSet rs)
     {
         return new FromResultSet(rs);
@@ -66,7 +87,7 @@ public static UntypedResultSet create(List<Map<String, ByteBuffer>> results)
         return new FromResultList(results);
     }
 
-    public static UntypedResultSet create(SelectStatement select, QueryPager pager, int pageSize)
+    public static UntypedResultSet create(SelectStatement select, QueryPager pager, PageSize pageSize)
     {
         return new FromPager(select, pager, pageSize);
     }
@@ -80,7 +101,7 @@ public static UntypedResultSet create(SelectStatement select,
                                           ConsistencyLevel cl,
                                           ClientState clientState,
                                           QueryPager pager,
-                                          int pageSize)
+                                          PageSize pageSize)
     {
         return new FromDistributedPager(select, cl, clientState, pager, pageSize);
     }
@@ -90,11 +111,6 @@ public boolean isEmpty()
         return size() == 0;
     }
 
-    public Stream<Row> stream()
-    {
-        return StreamSupport.stream(spliterator(), false);
-    }
-
     public abstract int size();
     public abstract Row one();
 
@@ -189,10 +205,10 @@ private static class FromPager extends UntypedResultSet
     {
         private final SelectStatement select;
         private final QueryPager pager;
-        private final int pageSize;
+        private final PageSize pageSize;
         private final List<ColumnSpecification> metadata;
 
-        private FromPager(SelectStatement select, QueryPager pager, int pageSize)
+        private FromPager(SelectStatement select, QueryPager pager, PageSize pageSize)
         {
             this.select = select;
             this.pager = pager;
@@ -250,13 +266,14 @@ private static class FromDistributedPager extends UntypedResultSet
         private final ConsistencyLevel cl;
         private final ClientState clientState;
         private final QueryPager pager;
-        private final int pageSize;
+        private final PageSize pageSize;
         private final List<ColumnSpecification> metadata;
 
         private FromDistributedPager(SelectStatement select,
                                      ConsistencyLevel cl,
                                      ClientState clientState,
-                                     QueryPager pager, int pageSize)
+                                     QueryPager pager,
+                                     PageSize pageSize)
         {
             this.select = select;
             this.cl = cl;
@@ -401,6 +418,11 @@ public double getDouble(String column)
             return DoubleType.instance.compose(data.get(column));
         }
 
+        public double getFloat(String column)
+        {
+            return FloatType.instance.compose(data.get(column));
+        }
+
         public ByteBuffer getBytes(String column)
         {
             return data.get(column);
diff --git a/src/java/org/apache/cassandra/cql3/Vectors.java b/src/java/org/apache/cassandra/cql3/Vectors.java
index 2c53b9e716d6..7301b396e769 100644
--- a/src/java/org/apache/cassandra/cql3/Vectors.java
+++ b/src/java/org/apache/cassandra/cql3/Vectors.java
@@ -182,7 +182,7 @@ public List<ByteBuffer> getElements()
     }
 
     /**
-     * Basically similar to a Value, but with some non-pure function (that need
+     * Basically similar to a Value, but with some non-deterministic function (that need
      * to be evaluated at execution time) in it.
      */
     public static class DelayedValue<T> extends Term.NonTerminal
diff --git a/src/java/org/apache/cassandra/cql3/WhereClause.java b/src/java/org/apache/cassandra/cql3/WhereClause.java
index dc1a7cfde055..1659e2d6aa87 100644
--- a/src/java/org/apache/cassandra/cql3/WhereClause.java
+++ b/src/java/org/apache/cassandra/cql3/WhereClause.java
@@ -17,30 +17,30 @@
  */
 package org.apache.cassandra.cql3;
 
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 
 import org.antlr.runtime.RecognitionException;
 import org.apache.cassandra.cql3.restrictions.CustomIndexExpression;
 
-import static java.lang.String.join;
-
-import static com.google.common.collect.Iterables.concat;
-import static com.google.common.collect.Iterables.transform;
-
+/**
+ * This is a parsed representation of the expression following the WHERE element
+ * in a CQL statement. It is parsed into an arbitrary sized expression tree consisting
+ * of <code>ExpressionElement</code> elements.
+ */
 public final class WhereClause
 {
-    private static final WhereClause EMPTY = new WhereClause(new Builder());
+    private static final WhereClause EMPTY = new WhereClause(new AndElement(Collections.emptyList()));
 
-    public final List<Relation> relations;
-    public final List<CustomIndexExpression> expressions;
+    private final ExpressionElement rootElement;
 
-    private WhereClause(Builder builder)
+    private WhereClause(ExpressionElement rootElement)
     {
-        relations = builder.relations.build();
-        expressions = builder.expressions.build();
+        this.rootElement = rootElement;
     }
 
     public static WhereClause empty()
@@ -50,26 +50,45 @@ public static WhereClause empty()
 
     public boolean containsCustomExpressions()
     {
-        return !expressions.isEmpty();
+        return rootElement.containsCustomExpressions();
+    }
+
+    public ExpressionElement root()
+    {
+        return rootElement;
     }
 
     /**
      * Renames identifiers in all relations
+     *
      * @param from the old identifier
-     * @param to the new identifier
-     * @return a new WhereClause with with "from" replaced by "to" in all relations
+     * @param to   the new identifier
+     * @return a new WhereClause with "from" replaced by "to" in all relations
      */
     public WhereClause renameIdentifier(ColumnIdentifier from, ColumnIdentifier to)
     {
-        WhereClause.Builder builder = new WhereClause.Builder();
-
-        relations.stream()
-                 .map(r -> r.renameIdentifier(from, to))
-                 .forEach(builder::add);
+        return new WhereClause(rootElement.rename(from, to));
+    }
 
-        expressions.forEach(builder::add);
+    /**
+     * Allows mutation of the relations held within the where clause element
+     * hierarchy
+     *
+     * @param relationMutator the relation mutator
+     * @return a new WhereClause with the relations mutated
+     */
+    public WhereClause mutateRelations(Function<Relation, Relation> relationMutator)
+    {
+        return new WhereClause(rootElement.mutate(relationMutator));
+    }
 
-        return builder.build();
+    /**
+     * @return a new WhereClause with the expression tree transforemd into conjuntive form
+     * @see ExpressionElement#conjunctiveForm()
+     */
+    public WhereClause conjunctiveForm()
+    {
+        return new WhereClause(rootElement.conjunctiveForm());
     }
 
     public static WhereClause parse(String cql) throws RecognitionException
@@ -90,9 +109,7 @@ public String toString()
      */
     public String toCQLString()
     {
-        return join(" AND ",
-                    concat(transform(relations, Relation::toCQLString),
-                           transform(expressions, CustomIndexExpression::toCQLString)));
+        return rootElement.toString();
     }
 
     @Override
@@ -105,13 +122,13 @@ public boolean equals(Object o)
             return false;
 
         WhereClause wc = (WhereClause) o;
-        return relations.equals(wc.relations) && expressions.equals(wc.expressions);
+        return rootElement.toString().equals(wc.rootElement.toString());
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hash(relations, expressions);
+        return Objects.hash(rootElement);
     }
 
     /**
@@ -121,7 +138,7 @@ public int hashCode()
      */
     public boolean containsTokenRelations()
     {
-        for (Relation rel : relations)
+        for (Relation rel : rootElement.relations())
         {
             if (rel.onToken())
                 return true;
@@ -129,26 +146,509 @@ public boolean containsTokenRelations()
         return false;
     }
 
+    /**
+     * This receives fragments from the parse operation and builds them into the final <code>WhereClause</code>.
+     *
+     * The received fragments are:
+     * <ul>
+     *     <li><code>add(Relation)</code> - adds a new relation to the current <code>ParseState</code></li>
+     *     <li><code>add(CustomIndexExpression)</code> - adds a new custom index expression to the current <code>ParseState</code></li>
+     *     <li><code>startEnclosure</code> - responds to a '(' and pushes the current <code>ParseState</code> onto the precedence stack</li>
+     *     <li><code>endEnclosure</code> - responds to a ')' and pulls the <code>ParseState</code> associated with the
+     *     matching <code>startEnclosure</code>. It will pull any intermediate precedence states off the stack until it
+     *     reaches the matching enclosure state</li>
+     *     <li><code>setCurrentOperator</code> - changes the operator in the <code>ParseState</code>. If this new operator is
+     *     of a higher precedence than the current operator, the last expression is popped from the <code>ParseState</code> and
+     *     the state is pushed onto the precedence stack</li>
+     *     <li><code>build</code> - always the last call. This builds the resultant <code>ExpressionTree</code> from the
+     *     precedence stack and the current <code>ParseState</code></li>
+     * </ul>
+     */
     public static final class Builder
     {
-        ImmutableList.Builder<Relation> relations = new ImmutableList.Builder<>();
-        ImmutableList.Builder<CustomIndexExpression> expressions = new ImmutableList.Builder<>();
+        private final Deque<ParseState> precedenceStack = new ArrayDeque<>();
+        private ParseState parseState = new ParseState();
+
+        public void add(Relation relation)
+        {
+            parseState.push(new RelationElement(relation));
+        }
+
+        public void add(CustomIndexExpression customIndexExpression)
+        {
+            parseState.push(new CustomIndexExpressionElement(customIndexExpression));
+        }
+
+        public void startEnclosure()
+        {
+            pushStack(PushState.ENCLOSURE);
+        }
+
+        public void endEnclosure()
+        {
+            do
+            {
+                ExpressionElement expression = generate();
+                parseState = precedenceStack.pop();
+                parseState.push(expression);
+            }
+            while (parseState.enclosure == PushState.PRECEDENCE);
+        }
+
+        public void setCurrentOperator(String value)
+        {
+            Operator operator = Operator.valueOf(value.toUpperCase());
+            if (parseState.isChangeOfOperator(operator))
+            {
+                if (parseState.higherPrecedence(operator))
+                {
+                    // Where we have a = 1 OR b = 1 AND c = 1. When the operator changes to AND
+                    // we need to pop b = 1 from the parseState, push the parseState containing
+                    // a = 1 OR and then add b = 1 to the new parseState
+                    ExpressionElement last = parseState.pop();
+                    pushStack(PushState.PRECEDENCE);
+                    parseState.push(last);
+                }
+                else
+                {
+                    ExpressionElement element = generate();
+                    if (!precedenceStack.isEmpty() && precedenceStack.peek().enclosure == PushState.PRECEDENCE)
+                        parseState = precedenceStack.pop();
+                    else
+                        parseState.clear();
+                    parseState.push(element);
+                }
+            }
+            parseState.operator = operator;
+        }
+
+        public WhereClause build()
+        {
+            while (!precedenceStack.isEmpty())
+            {
+                ExpressionElement expression = generate();
+                parseState = precedenceStack.pop();
+                parseState.push(expression);
+            }
+            return new WhereClause(generate());
+        }
+
+        private void pushStack(PushState enclosure)
+        {
+            parseState.enclosure = enclosure;
+            precedenceStack.push(parseState);
+            parseState = new ParseState();
+        }
+
+        private ExpressionElement generate()
+        {
+            if (parseState.size() == 1)
+                return parseState.pop();
+            return parseState.asContainer();
+        }
+    }
+
+    /**
+     * Represents the state of the parsing operation at a point of enclosure or precedence change.
+     */
+    public static class ParseState
+    {
+        Operator operator = Operator.NONE;
+        PushState enclosure = PushState.NONE;
+        Deque<ExpressionElement> expressionElements = new ArrayDeque<>();
+
+        void push(ExpressionElement element)
+        {
+            expressionElements.add(element);
+        }
+
+        ExpressionElement pop()
+        {
+            return expressionElements.removeLast();
+        }
+
+        int size()
+        {
+            return expressionElements.size();
+        }
+
+        ParseState clear()
+        {
+            expressionElements.clear();
+            return this;
+        }
+
+        boolean isChangeOfOperator(Operator operator)
+        {
+            return this.operator != operator && expressionElements.size() > 1;
+        }
+
+        boolean higherPrecedence(Operator operator)
+        {
+            return operator.compareTo(this.operator) > 0;
+        }
+
+        ContainerElement asContainer()
+        {
+            return operator == Operator.OR
+                    ? new OrElement(expressionElements)
+                    : new AndElement(expressionElements);
+        }
+    }
+
+    enum Operator
+    {
+        NONE, OR, AND;
+
+        public String joinValue()
+        {
+            return " " + name() + " ";
+        }
+    }
+
+    /**
+     * This is the reason why the <code>ParseState</code> was pushed onto the precedence stack.
+     */
+    enum PushState
+    {
+        NONE, PRECEDENCE, ENCLOSURE
+    }
+
+    public static abstract class ExpressionElement
+    {
+        public List<ContainerElement> operations()
+        {
+            return Collections.emptyList();
+        }
+
+        public boolean isDisjunction()
+        {
+            return false;
+        }
+
+        public List<Relation> relations()
+        {
+            return Collections.emptyList();
+        }
 
-        public Builder add(Relation relation)
+        public List<CustomIndexExpression> expressions()
+        {
+            return Collections.emptyList();
+        }
+
+        /**
+         * Returns true if the given function f evaluates to true on any of the expression tree nodes.
+         */
+        public abstract boolean exists(Predicate<ExpressionElement> f);
+
+        /**
+         * Returns true if this expression tree contains more than one relation.
+         */
+        public final boolean isCompound()
+        {
+            return exists(e -> e instanceof ContainerElement && ((ContainerElement) e).children.size() > 1);
+        }
+
+        /**
+         * Returns true if this expression tree contains a CustomIndexExpressionElement node.
+         */
+        public final boolean containsCustomExpressions()
+        {
+            return exists(CustomIndexExpressionElement.class::isInstance);
+        }
+
+        public ExpressionElement rename(ColumnIdentifier from, ColumnIdentifier to)
         {
-            relations.add(relation);
             return this;
         }
 
-        public Builder add(CustomIndexExpression expression)
+        /**
+         * Collapses expression tree levels of the same type to form a semantically equivalent,
+         * but simpler form of this tree.
+         *
+         * Collapsing is possible because OR and AND operations are associative.
+         *
+         * <p>
+         * Examples:
+         * <pre>
+         * AND(a, AND(b, c))      -> AND(a, b, c)
+         * OR(OR(a, b), OR(c, d)) -> OR(a, b, c, d)
+         * AND(a, OR(b, c))       -> AND(a, OR(b, c))
+         * </pre>
+         * </p>
+         *
+         * @return a new tree; this tree is left unmodified
+         */
+        public ExpressionElement flatten()
         {
-            expressions.add(expression);
             return this;
         }
 
-        public WhereClause build()
+        /**
+         * Creates a new tree that is a conjunctive form of this tree, semantically equivalent to this tree.
+         * The root of the conjunctive form is always an AndElement.
+         *
+         * The result tree is flattened so that nested conjunctions are lifted up to become the direct
+         * children of the root element. If the original tree does not have a top-level AndElement,
+         * an AndElement is inserted at the top, and a flattened original tree becomes its only child.
+         *
+         * <p>
+         * Examples:
+         * <pre>
+         * a = 1                                 -> AND(a = 1)
+         * AND()                                 -> AND()
+         * AND(a = 1, b = 2)                     -> AND(a = 1, b = 2)
+         * AND(a = 1, AND(b = 2, c = 3))         -> AND(a = 1, b = 2, c = 3)
+         * OR(a = 1, b = 2)                      -> AND(OR(a = 1, b = 2))
+         * OR(a = 1, OR(b = 2, c = 3))           -> AND(OR(a = 1, b = 2, c = 3))
+         * </pre>
+         * </p>
+         *
+         * @return a new tree; this tree is left unmodified
+         */
+        public final AndElement conjunctiveForm()
         {
-            return new WhereClause(this);
+            ExpressionElement flattened = this.flatten();
+            return flattened instanceof AndElement
+                    ? (AndElement) flattened
+                    : new AndElement(Lists.newArrayList(flattened));
+        }
+
+        protected ExpressionElement mutate(Function<Relation, Relation> relationMutator)
+        {
+            return this;
+        }
+    }
+
+    public static abstract class VariableElement extends ExpressionElement
+    {
+        @Override
+        public boolean exists(Predicate<ExpressionElement> f)
+        {
+            return f.test(this);
+        }
+    }
+
+    public static class RelationElement extends VariableElement
+    {
+        private final Relation relation;
+
+        public RelationElement(Relation relation)
+        {
+            this.relation = relation;
+        }
+
+        @Override
+        public List<Relation> relations()
+        {
+            return Lists.newArrayList(relation);
+        }
+
+        @Override
+        public ExpressionElement rename(ColumnIdentifier from, ColumnIdentifier to)
+        {
+            return new RelationElement(relation.renameIdentifier(from, to));
+        }
+
+        @Override
+        public String toString()
+        {
+            return relation.toString();
+        }
+
+        @Override
+        protected ExpressionElement mutate(Function<Relation, Relation> relationMutator)
+        {
+            return new RelationElement(relationMutator.apply(relation));
+        }
+    }
+
+    public static class CustomIndexExpressionElement extends VariableElement
+    {
+        private final CustomIndexExpression customIndexExpression;
+
+        public CustomIndexExpressionElement(CustomIndexExpression customIndexExpression)
+        {
+            this.customIndexExpression = customIndexExpression;
+        }
+
+        @Override
+        public List<CustomIndexExpression> expressions()
+        {
+            return Lists.newArrayList(customIndexExpression);
+        }
+
+        @Override
+        public String toString()
+        {
+            return customIndexExpression.toString();
+        }
+    }
+
+    public static abstract class ContainerElement extends ExpressionElement
+    {
+        protected final List<ExpressionElement> children;
+
+        protected ContainerElement(Collection<ExpressionElement> children)
+        {
+            this.children = new ArrayList<>(children.size());
+            this.children.addAll(children);
+        }
+
+        /**
+         * Returns a new container of the same type with new children copied from the given collection
+         */
+        protected abstract ContainerElement withChildren(Collection<ExpressionElement> children);
+
+        @Override
+        protected ExpressionElement mutate(Function<Relation, Relation> relationMutator)
+        {
+            List<ExpressionElement> newChildren = children.stream()
+                                                          .map(c -> c.mutate(relationMutator))
+                                                          .collect(Collectors.toList());
+
+            return this.withChildren(newChildren);
+        }
+
+        protected abstract Operator operator();
+
+        protected abstract String emptyValue();
+
+        @Override
+        public List<ContainerElement> operations()
+        {
+            return children.stream()
+                           .filter(c -> (c instanceof ContainerElement))
+                           .map(r -> ((ContainerElement) r))
+                           .collect(Collectors.toList());
+        }
+
+        @Override
+        public List<Relation> relations()
+        {
+            return children.stream()
+                           .filter(c -> (c instanceof RelationElement))
+                           .map(r -> (((RelationElement) r).relation))
+                           .collect(Collectors.toList());
+        }
+
+        @Override
+        public List<CustomIndexExpression> expressions()
+        {
+            return children.stream()
+                           .filter(c -> (c instanceof CustomIndexExpressionElement))
+                           .map(r -> (((CustomIndexExpressionElement) r).customIndexExpression))
+                           .collect(Collectors.toList());
+        }
+
+        @Override
+        public boolean exists(Predicate<ExpressionElement> f)
+        {
+            return f.test(this) || children.stream().anyMatch(f);
+        }
+
+        @Override
+        public ExpressionElement rename(ColumnIdentifier from, ColumnIdentifier to)
+        {
+            List<ExpressionElement> newChildren = children
+                    .stream()
+                    .map(c -> c.rename(from, to))
+                    .collect(Collectors.toList());
+
+            return this.withChildren(newChildren);
+        }
+
+        @Override
+        public ExpressionElement flatten()
+        {
+            List<ExpressionElement> newChildren = new ArrayList<>();
+            for (ExpressionElement child: children)
+            {
+                ExpressionElement flattened = child.flatten();
+                newChildren.add(flattened);
+
+                if (flattened instanceof ContainerElement)
+                {
+                    ContainerElement ce = (ContainerElement) flattened;
+                    if (ce.operator() == this.operator())
+                    {
+                        newChildren.remove(newChildren.size() - 1);
+                        newChildren.addAll(ce.children);
+                    }
+                }
+            }
+
+            return this.withChildren(newChildren);
+        }
+
+        @Override
+        public String toString()
+        {
+            if (children.isEmpty())
+                return emptyValue();
+
+            return children
+                .stream()
+                .map(c -> children.size() > 1 && c.isCompound() ? '(' + c.toString() + ')' : c.toString())
+                .collect(Collectors.joining(operator().joinValue()));
+        }
+    }
+
+    public static class AndElement extends ContainerElement
+    {
+        public AndElement(Collection<ExpressionElement> children)
+        {
+            super(children);
+        }
+
+        @Override
+        protected AndElement withChildren(Collection<ExpressionElement> children)
+        {
+            return new AndElement(children);
+        }
+
+        @Override
+        protected Operator operator()
+        {
+            return Operator.AND;
+        }
+
+        @Override
+        protected String emptyValue()
+        {
+            return "TRUE";
+        }
+    }
+
+    public static class OrElement extends ContainerElement
+    {
+        public OrElement(Collection<ExpressionElement> children)
+        {
+            super(children);
+        }
+
+        @Override
+        protected OrElement withChildren(Collection<ExpressionElement> children)
+        {
+            return new OrElement(children);
+        }
+
+        @Override
+        protected Operator operator()
+        {
+            return Operator.OR;
+        }
+
+        @Override
+        protected String emptyValue()
+        {
+            return "FALSE";
+        }
+
+        @Override
+        public boolean isDisjunction()
+        {
+            return true;
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/conditions/AbstractConditions.java b/src/java/org/apache/cassandra/cql3/conditions/AbstractConditions.java
index 0e2646effd4e..98d9b72fc2ba 100644
--- a/src/java/org/apache/cassandra/cql3/conditions/AbstractConditions.java
+++ b/src/java/org/apache/cassandra/cql3/conditions/AbstractConditions.java
@@ -17,8 +17,11 @@
  */
 package org.apache.cassandra.cql3.conditions;
 
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
+import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.functions.Function;
 
@@ -32,11 +35,18 @@ public void addFunctionsTo(List<Function> functions)
     {
     }
 
+    @Override
     public Iterable<ColumnMetadata> getColumns()
     {
         return null;
     }
 
+    @Override
+    public Set<ColumnMetadata> getAnalyzedColumns(IndexRegistry indexRegistry)
+    {
+        return Collections.emptySet();
+    }
+
     public boolean isEmpty()
     {
         return false;
diff --git a/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java b/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java
index 2f6a9c572c7a..bbcba4ee93ce 100644
--- a/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java
+++ b/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java
@@ -41,6 +41,8 @@
  */
 public abstract class ColumnCondition
 {
+    public static final String ANALYZER_MATCHES_ERROR = "LWT Conditions do not support the : operator";
+
     public final ColumnMetadata column;
     public final Operator operator;
     private final Terms terms;
@@ -255,7 +257,7 @@ else if (otherValue == null)
                 // the condition value is not null, so only NEQ can return true
                 return operator == Operator.NEQ;
             }
-            return operator.isSatisfiedBy(type, otherValue, value);
+            return operator.isSatisfiedBy(type, otherValue, value, null, null); // We don't use any analyzers in LWT, see CNDB-11658
         }
     }
 
@@ -666,7 +668,12 @@ private ByteBuffer rowValue(Row row)
                 return cell == null ? null : cell.buffer();
             }
 
-            Cell<?> cell = getCell(row, column);
+            // getCell returns Cell<?>, which requires a method call to properly convert.
+            return getCellBuffer(getCell(row, column), userType);
+        }
+
+        private <V> ByteBuffer getCellBuffer(Cell<V> cell, UserType userType)
+        {
             return cell == null
                    ? null
                    : userType.split(ByteBufferAccessor.instance, cell.buffer())[userType.fieldPosition(field)];
@@ -829,6 +836,10 @@ public ColumnCondition prepare(String keyspace, ColumnMetadata receiver, TableMe
             if (receiver.type instanceof CounterColumnType)
                 throw invalidRequest("Conditions on counters are not supported");
 
+            // Analyzer matches operator is only supported on SAI indexes for now
+            if (operator == Operator.ANALYZER_MATCHES)
+                throw invalidRequest(ANALYZER_MATCHES_ERROR);
+
             if (collectionElement != null)
             {
                 if (!(receiver.type.isCollection()))
diff --git a/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java b/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java
index 35d4a9570f47..2f8d3c1cb8e0 100644
--- a/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java
+++ b/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java
@@ -20,14 +20,18 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import java.util.Set;
+
+import com.google.common.collect.Iterators;
 
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.statements.CQL3CasRequest;
 import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
@@ -36,7 +40,7 @@
  * A set of <code>ColumnCondition</code>s.
  *
  */
-public final class ColumnConditions extends AbstractConditions
+public final class ColumnConditions extends AbstractConditions implements Iterable<ColumnCondition>
 {
     /**
      * The conditions on regular columns.
@@ -72,9 +76,30 @@ public boolean appliesToRegularColumns()
     @Override
     public Collection<ColumnMetadata> getColumns()
     {
-        return Stream.concat(columnConditions.stream(), staticConditions.stream())
-                     .map(e -> e.column)
-                     .collect(Collectors.toList());
+        List<ColumnMetadata> columns = new ArrayList<>(size());
+
+        for (ColumnCondition condition : this)
+        {
+            columns.add(condition.column);
+        }
+
+        return columns;
+    }
+
+    @Override
+    public Set<ColumnMetadata> getAnalyzedColumns(IndexRegistry indexRegistry)
+    {
+        Set<ColumnMetadata> analyzedColumns = new HashSet<>();
+
+        for (ColumnCondition condition : this)
+        {
+            if (indexRegistry.getIndexAnalyzerFor(condition.column, condition.operator).isPresent())
+            {
+                analyzedColumns.add(condition.column);
+            }
+        }
+
+        return analyzedColumns;
     }
 
     @Override
@@ -83,6 +108,17 @@ public boolean isEmpty()
         return columnConditions.isEmpty() && staticConditions.isEmpty();
     }
 
+    @Override
+    public Iterator<ColumnCondition> iterator()
+    {
+        return Iterators.concat(columnConditions.iterator(), staticConditions.iterator());
+    }
+
+    public int size()
+    {
+        return columnConditions.size() + staticConditions.size();
+    }
+
     /**
      * Adds the conditions to the specified CAS request.
      *
@@ -103,8 +139,7 @@ public void addConditionsTo(CQL3CasRequest request,
     @Override
     public void addFunctionsTo(List<Function> functions)
     {
-        columnConditions.forEach(p -> p.addFunctionsTo(functions));
-        staticConditions.forEach(p -> p.addFunctionsTo(functions));
+        iterator().forEachRemaining(p -> p.addFunctionsTo(functions));
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/cql3/conditions/Conditions.java b/src/java/org/apache/cassandra/cql3/conditions/Conditions.java
index 1a202dff0db2..ebb57f04b881 100644
--- a/src/java/org/apache/cassandra/cql3/conditions/Conditions.java
+++ b/src/java/org/apache/cassandra/cql3/conditions/Conditions.java
@@ -18,11 +18,15 @@
 package org.apache.cassandra.cql3.conditions;
 
 import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
 
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.statements.CQL3CasRequest;
 import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.schema.ColumnMetadata;
 
 /**
@@ -56,8 +60,14 @@ public interface Conditions
      * Returns the column definitions to which apply the conditions.
      * @return the column definitions to which apply the conditions.
      */
+    @Nullable
     Iterable<ColumnMetadata> getColumns();
 
+    /**
+     * @return the column definitions of the conditions supported by a {@link org.apache.cassandra.index.Index.Analyzer}.
+     */
+    Set<ColumnMetadata> getAnalyzedColumns(IndexRegistry indexRegistry);
+
     /**
      * Checks if this <code>Conditions</code> is empty.
      * @return <code>true</code> if this <code>Conditions</code> is empty, <code>false</code> otherwise.
diff --git a/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java b/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
index c3183f64133c..5122b2c5ec64 100644
--- a/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
@@ -21,16 +21,14 @@
 import java.util.List;
 
 import com.google.common.base.Objects;
+import org.apache.commons.lang3.text.StrBuilder;
 
 import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.CQL3Type.Tuple;
 import org.apache.cassandra.cql3.ColumnSpecification;
 import org.apache.cassandra.cql3.CqlBuilder;
 import org.apache.cassandra.db.marshal.AbstractType;
 
-import org.apache.commons.lang3.text.StrBuilder;
-
 import static java.util.stream.Collectors.toList;
 
 /**
@@ -68,7 +66,7 @@ public List<String> argumentsList()
     {
         return argTypes().stream()
                          .map(AbstractType::asCQL3Type)
-                         .map(CQL3Type::toString)
+                         .map(CQL3Type::toSchemaString)
                          .collect(toList());
     }
 
@@ -105,7 +103,7 @@ public final AssignmentTestable.TestResult testAssignment(String keyspace, Colum
         // We should ignore the fact that the receiver type is frozen in our comparison as functions do not support
         // frozen types for return type
         AbstractType<?> returnType = returnType();
-        if (receiver.type.isFreezable() && !receiver.type.isMultiCell())
+        if (!receiver.type.isMultiCell())
             returnType = returnType.freeze();
 
         if (receiver.type.equals(returnType))
@@ -159,8 +157,7 @@ public String elementName()
      */
     protected String toCqlString(AbstractType<?> type)
     {
-        return type.isTuple() ? ((Tuple) type.asCQL3Type()).toString(false)
-                              : type.asCQL3Type().toString();
+        return type.asCQL3Type().toString();
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/cql3/functions/AggregateFcts.java b/src/java/org/apache/cassandra/cql3/functions/AggregateFcts.java
index 9942869a5fb5..5bc810a8b2f6 100644
--- a/src/java/org/apache/cassandra/cql3/functions/AggregateFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/AggregateFcts.java
@@ -23,6 +23,7 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -65,7 +66,9 @@ public static void addFunctionsTo(NativeFunctions functions)
         functions.add(new FunctionFactory("max", FunctionParameter.anyType(true))
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                           List<AbstractType<?>> argTypes,
+                                                           AbstractType<?> receiverType)
             {
                 AbstractType<?> type = argTypes.get(0);
                 return type.isCounter() ? maxFunctionForCounter : makeMaxFunction(type);
@@ -76,7 +79,9 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
         functions.add(new FunctionFactory("min", FunctionParameter.anyType(true))
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                           List<AbstractType<?>> argTypes,
+                                                           AbstractType<?> receiverType)
             {
                 AbstractType<?> type = argTypes.get(0);
                 return type.isCounter() ? minFunctionForCounter : makeMinFunction(type);
diff --git a/src/java/org/apache/cassandra/cql3/functions/CollectionFcts.java b/src/java/org/apache/cassandra/cql3/functions/CollectionFcts.java
index a3bc4726f9cc..06c56c8f82df 100644
--- a/src/java/org/apache/cassandra/cql3/functions/CollectionFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/CollectionFcts.java
@@ -25,6 +25,7 @@
 
 import com.google.common.collect.ImmutableList;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.CollectionType;
@@ -49,7 +50,7 @@ public static void addFunctionsTo(NativeFunctions functions)
         functions.add(new FunctionFactory("map_keys", FunctionParameter.anyMap())
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args, List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return makeMapKeysFunction(name.name, (MapType<?, ?>) argTypes.get(0));
             }
@@ -58,7 +59,7 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
         functions.add(new FunctionFactory("map_values", FunctionParameter.anyMap())
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return makeMapValuesFunction(name.name, (MapType<?, ?>) argTypes.get(0));
             }
@@ -67,7 +68,7 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
         functions.add(new FunctionFactory("collection_count", FunctionParameter.anyCollection())
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return makeCollectionCountFunction(name.name, (CollectionType<?>) argTypes.get(0));
             }
@@ -76,7 +77,7 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
         functions.add(new FunctionFactory("collection_min", FunctionParameter.setOrList())
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return makeCollectionMinFunction(name.name, (CollectionType<?>) argTypes.get(0));
             }
@@ -85,7 +86,7 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
         functions.add(new FunctionFactory("collection_max", FunctionParameter.setOrList())
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return makeCollectionMaxFunction(name.name, (CollectionType<?>) argTypes.get(0));
             }
@@ -94,7 +95,7 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
         functions.add(new FunctionFactory("collection_sum", FunctionParameter.numericSetOrList())
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return makeCollectionSumFunction(name.name, (CollectionType<?>) argTypes.get(0));
             }
@@ -103,7 +104,7 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
         functions.add(new FunctionFactory("collection_avg", FunctionParameter.numericSetOrList())
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return makeCollectionAvgFunction(name.name, (CollectionType<?>) argTypes.get(0));
             }
@@ -121,7 +122,7 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
      */
     private static <K, V> NativeScalarFunction makeMapKeysFunction(String name, MapType<K, V> inputType)
     {
-        SetType<K> outputType = SetType.getInstance(inputType.getKeysType(), false);
+        SetType<K> outputType = SetType.getInstance((AbstractType<K>) inputType.getKeysType().freeze(), false);
 
         return new NativeScalarFunction(name, outputType, inputType)
         {
@@ -149,7 +150,7 @@ public ByteBuffer execute(Arguments arguments)
      */
     private static <K, V> NativeScalarFunction makeMapValuesFunction(String name, MapType<K, V> inputType)
     {
-        ListType<V> outputType = ListType.getInstance(inputType.getValuesType(), false);
+        ListType<V> outputType = ListType.getInstance((AbstractType<V>) inputType.getValuesType().freeze(), false);
 
         return new NativeScalarFunction(name, outputType, inputType)
         {
diff --git a/src/java/org/apache/cassandra/cql3/functions/FromJsonFct.java b/src/java/org/apache/cassandra/cql3/functions/FromJsonFct.java
index 356003e8e82a..7075913e8816 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FromJsonFct.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FromJsonFct.java
@@ -22,6 +22,7 @@
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 
 import org.apache.cassandra.cql3.QueryOptions;
@@ -97,7 +98,9 @@ private Factory(String name)
         }
 
         @Override
-        protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+        protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                       List<AbstractType<?>> argTypes,
+                                                       AbstractType<?> receiverType)
         {
             if (receiverType == null)
                 throw new InvalidRequestException(format("%s() cannot be used in the selection clause of a SELECT statement", name.name));
diff --git a/src/java/org/apache/cassandra/cql3/functions/Function.java b/src/java/org/apache/cassandra/cql3/functions/Function.java
index 5af03b2fd356..94532f6c2f8e 100644
--- a/src/java/org/apache/cassandra/cql3/functions/Function.java
+++ b/src/java/org/apache/cassandra/cql3/functions/Function.java
@@ -34,36 +34,37 @@ public interface Function extends AssignmentTestable
      * A marker buffer used to represent function parameters that cannot be resolved at some stage of CQL processing.
      * This is used for partial function application in particular.
      */
-    public static final ByteBuffer UNRESOLVED = ByteBuffer.allocate(0);
+    ByteBuffer UNRESOLVED = ByteBuffer.allocate(0);
 
-    public FunctionName name();
-    public List<AbstractType<?>> argTypes();
-    public AbstractType<?> returnType();
+    FunctionName name();
+    List<AbstractType<?>> argTypes();
+    AbstractType<?> returnType();
 
     /**
      * Checks whether the function is a native/hard coded one or not.
      *
      * @return {@code true} if the function is a native/hard coded one, {@code false} otherwise.
      */
-    public boolean isNative();
+    boolean isNative();
 
     /**
-     * Checks whether the function is a pure function (as in doesn't depend on, nor produces side effects) or not.
+     * Checks whether the function is a deterministic function (as in given a particular input, will always produce the
+     * same output) or not.
      *
-     * @return {@code true} if the function is a pure function, {@code false} otherwise.
+     * @return {@code true} if the function is a deterministic function, {@code false} otherwise.
      */
-    public boolean isPure();
+    boolean isDeterministic();
 
     /**
      * Checks whether the function is an aggregate function or not.
      *
      * @return {@code true} if the function is an aggregate function, {@code false} otherwise.
      */
-    public boolean isAggregate();
+    boolean isAggregate();
 
-    public void addFunctionsTo(List<Function> functions);
+    void addFunctionsTo(List<Function> functions);
 
-    public boolean referencesUserType(ByteBuffer name);
+    boolean referencesUserType(ByteBuffer name);
 
     /**
      * Returns the name of the function to use within a ResultSet.
@@ -71,7 +72,7 @@ public interface Function extends AssignmentTestable
      * @param columnNames the names of the columns used to call the function
      * @return the name of the function to use within a ResultSet
      */
-    public String columnName(List<String> columnNames);
+    String columnName(List<String> columnNames);
 
     /**
      * Creates some new input arguments for this function.
@@ -81,7 +82,7 @@ public interface Function extends AssignmentTestable
      */
     Arguments newArguments(ProtocolVersion version);
 
-    public default Optional<Difference> compare(Function other)
+    default Optional<Difference> compare(Function other)
     {
         throw new UnsupportedOperationException();
     }
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionFactory.java b/src/java/org/apache/cassandra/cql3/functions/FunctionFactory.java
index 90a2c69d77f9..e1a8fae45e4c 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FunctionFactory.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionFactory.java
@@ -85,10 +85,7 @@ public NativeFunction getOrCreateFunction(List<? extends AssignmentTestable> arg
                                               String receiverKeyspace,
                                               String receiverTable)
     {
-        // validate the number of arguments
-        int numArgs = args.size();
-        if (numArgs < numMandatoryParameters || numArgs > numParameters)
-            throw invalidNumberOfArgumentsException();
+        validateNumberOfArguments(args.size());
 
         // Do a first pass trying to infer the types of the arguments individually, without any context about the types
         // of the other arguments. We don't do any validation during this first pass.
@@ -102,7 +99,9 @@ public NativeFunction getOrCreateFunction(List<? extends AssignmentTestable> arg
 
         // Do a second pass trying to infer the types of the arguments considering the types of other inferred types.
         // We can validate the inferred types during this second pass.
-        for (int i = 0; i < args.size(); i++)
+        // This is done in reverse order to favour a left-to-right reading, so arguments on the right have to match
+        // arguments on the left.
+        for (int i = args.size() - 1; i >= 0; i--)
         {
             AssignmentTestable arg = args.get(i);
             FunctionParameter parameter = parameters.get(i);
@@ -111,29 +110,33 @@ public NativeFunction getOrCreateFunction(List<? extends AssignmentTestable> arg
                 throw new InvalidRequestException(String.format("Cannot infer type of argument %s in call to " +
                                                                 "function %s: use type casts to disambiguate",
                                                                 arg, this));
-            parameter.validateType(name, arg, type);
+            parameter.validateType(this, arg, type);
             type = type.udfType();
             types.set(i, type);
         }
 
-        return doGetOrCreateFunction(types, receiverType);
+        return doGetOrCreateFunction(args, types, receiverType);
     }
 
-    public InvalidRequestException invalidNumberOfArgumentsException()
+    protected void validateNumberOfArguments(int numArgs)
     {
-        return new InvalidRequestException("Invalid number of arguments for function " + this);
+        if (numArgs < numMandatoryParameters || numArgs > numParameters)
+            throw new InvalidRequestException("Invalid number of arguments for function " + this);
     }
 
     /**
      * Returns a function compatible with the specified signature.
      *
+     * @param args the arguments in the function call for which the function is going to be built
      * @param argTypes the types of the function arguments
      * @param receiverType the expected return type of the function
      * @return a function compatible with the specified signature, or {@code null} if this cannot create a function for
      * the supplied arguments but there might be another factory with the same {@link #name()} able to do it.
      */
     @Nullable
-    protected abstract NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType);
+    protected abstract NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                            List<AbstractType<?>> argTypes,
+                                                            AbstractType<?> receiverType);
 
     @Override
     public String toString()
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionParameter.java b/src/java/org/apache/cassandra/cql3/functions/FunctionParameter.java
index 708e4fdd7d33..92f1bd0286c0 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FunctionParameter.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionParameter.java
@@ -20,14 +20,15 @@
 
 import java.util.Arrays;
 import java.util.List;
-import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
 import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.selection.Selectable;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.ListType;
 import org.apache.cassandra.db.marshal.MapType;
 import org.apache.cassandra.db.marshal.NumberType;
@@ -62,7 +63,7 @@ default AbstractType<?> inferType(String keyspace,
         return arg.getCompatibleTypeIfKnown(keyspace);
     }
 
-    void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType);
+    void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType);
 
     /**
      * @return whether this parameter is optional
@@ -92,9 +93,9 @@ public AbstractType<?> inferType(String keyspace,
             }
 
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
-                wrapped.validateType(name, arg, argType);
+                wrapped.validateType(factory, arg, argType);
             }
 
             @Override
@@ -116,14 +117,32 @@ public String toString()
      */
     static FunctionParameter string()
     {
-        return fixed(CQL3Type.Native.TEXT, CQL3Type.Native.VARCHAR, CQL3Type.Native.ASCII);
+        return fixed("string", CQL3Type.Native.TEXT, CQL3Type.Native.VARCHAR, CQL3Type.Native.ASCII);
     }
 
     /**
-     * @param types the accepted data types
+     * @return a function parameter definition that accepts values that can be interpreted as floats
+     */
+    static FunctionParameter float32()
+    {
+        return fixed("float", CQL3Type.Native.FLOAT, CQL3Type.Native.DOUBLE, CQL3Type.Native.INT, CQL3Type.Native.BIGINT);
+    }
+
+    /**
+     * @param type the accepted data type
      * @return a function parameter definition that accepts values of a specific data type
      */
-    static FunctionParameter fixed(CQL3Type... types)
+    static FunctionParameter fixed(CQL3Type type)
+    {
+        return fixed(type.toString(), type);
+    }
+
+    /**
+     * @param name the name of the data type
+     * @param types the accepted data types
+     * @return a function parameter definition that accepts values of the specified data types
+     */
+    static FunctionParameter fixed(String name, CQL3Type... types)
     {
         assert types.length > 0;
 
@@ -140,21 +159,18 @@ public AbstractType<?> inferType(String keyspace,
             }
 
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
                 if (Arrays.stream(types).allMatch(t -> argType.testAssignment(t.getType()) == NOT_ASSIGNABLE))
                     throw new InvalidRequestException(format("Function %s requires an argument of type %s, " +
                                                              "but found argument %s of type %s",
-                                                             name, this, arg, argType.asCQL3Type()));
+                                                             factory, this, arg, argType.asCQL3Type()));
             }
 
             @Override
             public String toString()
             {
-                if (types.length == 1)
-                    return types[0].toString();
-
-                return '[' + Arrays.stream(types).map(Object::toString).collect(Collectors.joining("|")) + ']';
+                return name;
             }
         };
     }
@@ -178,7 +194,7 @@ public AbstractType<?> inferType(String keyspace,
             }
 
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
                 // nothing to do here, all types are accepted
             }
@@ -218,9 +234,9 @@ public AbstractType<?> inferType(String keyspace,
             }
 
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
-                parameter.validateType(name, arg, argType);
+                parameter.validateType(factory, arg, argType);
             }
 
             @Override
@@ -240,12 +256,12 @@ static FunctionParameter anyCollection()
         return new FunctionParameter()
         {
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
                 if (!argType.isCollection())
                     throw new InvalidRequestException(format("Function %s requires a collection argument, " +
                                                              "but found argument %s of type %s",
-                                                             name, arg, argType.asCQL3Type()));
+                                                             factory, arg, argType.asCQL3Type()));
             }
 
             @Override
@@ -264,7 +280,7 @@ static FunctionParameter setOrList()
         return new FunctionParameter()
         {
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
                 if (argType.isCollection())
                 {
@@ -275,7 +291,7 @@ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType
 
                 throw new InvalidRequestException(format("Function %s requires a set or list argument, " +
                                                          "but found argument %s of type %s",
-                                                         name, arg, argType.asCQL3Type()));
+                                                         factory, arg, argType.asCQL3Type()));
             }
 
             @Override
@@ -295,7 +311,7 @@ static FunctionParameter numericSetOrList()
         return new FunctionParameter()
         {
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
                 AbstractType<?> elementType = null;
                 if (argType.isCollection())
@@ -314,7 +330,7 @@ else if (collectionType.kind == CollectionType.Kind.LIST)
                 if (!(elementType instanceof NumberType))
                     throw new InvalidRequestException(format("Function %s requires a numeric set/list argument, " +
                                                              "but found argument %s of type %s",
-                                                             name, arg, argType.asCQL3Type()));
+                                                             factory, arg, argType.asCQL3Type()));
             }
 
             @Override
@@ -334,12 +350,12 @@ static FunctionParameter anyMap()
         return new FunctionParameter()
         {
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
                 if (!argType.isUDT() && !(argType instanceof MapType))
                     throw new InvalidRequestException(format("Function %s requires a map argument, " +
                                                              "but found argument %s of type %s",
-                                                             name, arg, argType.asCQL3Type()));
+                                                             factory, arg, argType.asCQL3Type()));
             }
 
             @Override
@@ -373,7 +389,7 @@ public AbstractType<?> inferType(String keyspace,
             }
 
             @Override
-            public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
             {
                 if (argType.isVector())
                 {
@@ -390,7 +406,7 @@ else if (argType instanceof ListType) // if it's terminal it will be a list
 
                 throw new InvalidRequestException(format("Function %s requires a %s vector argument, " +
                                                          "but found argument %s of type %s",
-                                                         name, type, arg, argType.asCQL3Type()));
+                                                         factory, type, arg, argType.asCQL3Type()));
             }
 
             @Override
@@ -400,4 +416,56 @@ public String toString()
             }
         };
     }
+
+    /**
+     * @param name the name of the function parameter
+     * @param type the accepted type of literal
+     * @param inferredType the inferred type of the literal
+     * @return a function parameter definition that accepts a specific literal type
+     */
+    static FunctionParameter literal(String name, Constants.Type type, AbstractType<?> inferredType)
+    {
+        return new FunctionParameter()
+        {
+            @Override
+            public AbstractType<?> inferType(String keyspace,
+                                             AssignmentTestable arg,
+                                             @Nullable AbstractType<?> receiverType,
+                                             @Nullable List<AbstractType<?>> inferredTypes)
+            {
+                return inferredType;
+            }
+
+            @Override
+            public void validateType(FunctionFactory factory, AssignmentTestable arg, AbstractType<?> argType)
+            {
+                if (arg instanceof Selectable.WithTerm)
+                    arg = ((Selectable.WithTerm) arg).rawTerm;
+
+                if (!(arg instanceof Constants.Literal))
+                    throw invalidArgumentException(factory, arg);
+
+                Constants.Literal literal = (Constants.Literal) arg;
+                if (literal.type != type)
+                    throw invalidArgumentException(factory, arg);
+            }
+
+            private InvalidRequestException invalidArgumentException(FunctionFactory factory, AssignmentTestable arg)
+            {
+                throw new InvalidRequestException(format("Function %s requires a %s argument, but found %s",
+                                                         factory, this, arg));
+            }
+
+            @Override
+            public String toString()
+            {
+                return name;
+            }
+        };
+    }
+
+    static FunctionParameter literalInteger()
+    {
+        return literal("literal_int", Constants.Type.INTEGER, Int32Type.instance);
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/functions/IndexFcts.java b/src/java/org/apache/cassandra/cql3/functions/IndexFcts.java
new file mode 100644
index 000000000000..143eebd2fe83
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/IndexFcts.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import com.google.common.base.Charsets;
+
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.analyzer.JSONAnalyzerParser;
+import org.apache.cassandra.index.sai.analyzer.LuceneAnalyzer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.lucene.analysis.Analyzer;
+
+public abstract class IndexFcts
+{
+    public static void addFunctionsTo(NativeFunctions functions)
+    {
+        functions.add(new SAIAnalyzeFunction());
+    }
+
+    /**
+     * CQL native function to get the tokens produced for given text value and the analyzer defined by the given JSON options.
+     */
+    private static class SAIAnalyzeFunction extends NativeScalarFunction
+    {
+        private static final String NAME = "sai_analyze";
+        private static final ListType<String> returnType = ListType.getInstance(UTF8Type.instance, false);
+
+        private SAIAnalyzeFunction()
+        {
+            super(NAME, returnType, UTF8Type.instance, UTF8Type.instance);
+        }
+
+        @Override
+        public ByteBuffer execute(Arguments arguments) throws InvalidRequestException
+        {
+            if (arguments.get(0) == null)
+                return null;
+            String text = arguments.get(0);
+
+            if (arguments.get(1) == null)
+                throw new InvalidRequestException("Function " + name + " requires a non-null json_analyzer parameter (2nd argument)");
+            String json = arguments.get(1);
+
+            LuceneAnalyzer luceneAnalyzer = null;
+            List<String> tokens = new ArrayList<>();
+            try (Analyzer analyzer = JSONAnalyzerParser.parse(json).left)
+            {
+                luceneAnalyzer = new LuceneAnalyzer(UTF8Type.instance, analyzer, new HashMap<>());
+
+                ByteBuffer toAnalyze = ByteBuffer.wrap(text.getBytes(Charsets.UTF_8));
+                luceneAnalyzer.reset(toAnalyze);
+                ByteBuffer analyzed;
+
+                while (luceneAnalyzer.hasNext())
+                {
+                    analyzed = luceneAnalyzer.next();
+                    tokens.add(ByteBufferUtil.string(analyzed, Charsets.UTF_8));
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidRequestException("Function " + name + " unable to analyze text=" + text + " json_analyzer=" + json, ex);
+            }
+            finally
+            {
+                if (luceneAnalyzer != null)
+                {
+                    luceneAnalyzer.end();
+                }
+            }
+
+            return returnType.decompose(tokens);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/JavaBasedUDFunction.java b/src/java/org/apache/cassandra/cql3/functions/JavaBasedUDFunction.java
index e51b9cbfc6dc..ba4c03581cfb 100644
--- a/src/java/org/apache/cassandra/cql3/functions/JavaBasedUDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/JavaBasedUDFunction.java
@@ -40,6 +40,8 @@
 import com.google.common.io.ByteStreams;
 
 import org.apache.commons.lang3.StringUtils;
+import org.eclipse.jdt.internal.compiler.lookup.LookupEnvironment;
+import org.eclipse.jdt.internal.compiler.lookup.ModuleBinding;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -59,8 +61,6 @@
 import org.eclipse.jdt.internal.compiler.env.INameEnvironment;
 import org.eclipse.jdt.internal.compiler.env.NameEnvironmentAnswer;
 import org.eclipse.jdt.internal.compiler.impl.CompilerOptions;
-import org.eclipse.jdt.internal.compiler.lookup.LookupEnvironment;
-import org.eclipse.jdt.internal.compiler.lookup.ModuleBinding;
 import org.eclipse.jdt.internal.compiler.problem.DefaultProblemFactory;
 
 public final class JavaBasedUDFunction extends UDFunction
@@ -186,10 +186,17 @@ protected URLConnection openConnection(URL u)
 
     private static final Pattern patternJavaDriver = Pattern.compile("com\\.datastax\\.driver\\.core\\.");
 
-    JavaBasedUDFunction(FunctionName name, List<ColumnIdentifier> argNames, List<AbstractType<?>> argTypes,
-                        AbstractType<?> returnType, boolean calledOnNullInput, String body)
+    JavaBasedUDFunction(FunctionName name,
+                        List<ColumnIdentifier> argNames,
+                        List<AbstractType<?>> argTypes,
+                        AbstractType<?> returnType,
+                        boolean calledOnNullInput,
+                        String body,
+                        boolean deterministic,
+                        boolean monotonic,
+                        List<ColumnIdentifier> monotonicOn)
     {
-        super(name, argNames, argTypes, returnType, calledOnNullInput, "java", body);
+        super(name, argNames, argTypes, returnType, calledOnNullInput, "java", body, deterministic, monotonic, monotonicOn);
 
         // put each UDF in a separate package to prevent cross-UDF code access
         String pkgName = BASE_PACKAGE + '.' + generateClassName(name, 'p');
diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java b/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java
index 3437a8d1586f..af45668e6f93 100644
--- a/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java
@@ -41,9 +41,9 @@ public final boolean isNative()
     }
 
     @Override
-    public boolean isPure()
+    public boolean isDeterministic()
     {
-        // Most of our functions are pure, the other ones should override this
+        // Most of our functions are deterministic, the other ones should override this
         return true;
     }
 
diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java b/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java
index 2100fe3f898e..bda9da58fd39 100644
--- a/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java
+++ b/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java
@@ -47,6 +47,7 @@ public class NativeFunctions
             MathFcts.addFunctionsTo(this);
             MaskingFcts.addFunctionsTo(this);
             VectorFcts.addFunctionsTo(this);
+            IndexFcts.addFunctionsTo(this);
         }
     };
 
diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeScalarFunction.java b/src/java/org/apache/cassandra/cql3/functions/NativeScalarFunction.java
index e492f758f10f..3ae0607f7a2e 100644
--- a/src/java/org/apache/cassandra/cql3/functions/NativeScalarFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/NativeScalarFunction.java
@@ -17,9 +17,6 @@
  */
 package org.apache.cassandra.cql3.functions;
 
-import java.nio.ByteBuffer;
-import java.util.List;
-
 import org.apache.cassandra.db.marshal.AbstractType;
 
 /**
@@ -41,17 +38,4 @@ public final boolean isAggregate()
     {
         return false;
     }
-
-    /**
-     * Checks if a partial application of the function is monotonic.
-     *
-     * <p>A function is monotonic if it is either entirely nonincreasing or nondecreasing.</p>
-     *
-     * @param partialParameters the input parameters used to create the partial application of the function
-     * @return {@code true} if the partial application of the function is monotonic {@code false} otherwise.
-     */
-    protected boolean isPartialApplicationMonotonic(List<ByteBuffer> partialParameters)
-    {
-        return isMonotonic();
-    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/functions/PartiallyAppliedScalarFunction.java b/src/java/org/apache/cassandra/cql3/functions/PartiallyAppliedScalarFunction.java
index 7a5e5fb71ff3..3fa2484f446f 100644
--- a/src/java/org/apache/cassandra/cql3/functions/PartiallyAppliedScalarFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/PartiallyAppliedScalarFunction.java
@@ -46,14 +46,13 @@ final class PartiallyAppliedScalarFunction extends NativeScalarFunction implemen
     @Override
     public boolean isMonotonic()
     {
-        return function.isNative() ? ((NativeScalarFunction) function).isPartialApplicationMonotonic(partialParameters)
-                                   : function.isMonotonic();
+        return function.isPartialApplicationMonotonic(partialParameters);
     }
 
     @Override
-    public boolean isPure()
+    public boolean isDeterministic()
     {
-        return function.isPure();
+        return function.isDeterministic();
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/cql3/functions/ScalarFunction.java b/src/java/org/apache/cassandra/cql3/functions/ScalarFunction.java
index 986242a4135c..ce80c22e3be0 100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScalarFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScalarFunction.java
@@ -28,16 +28,16 @@
  */
 public interface ScalarFunction extends Function
 {
-    public boolean isCalledOnNullInput();
+    boolean isCalledOnNullInput();
 
     /**
      * Checks if the function is monotonic.
-     *
-     *<p>A function is monotonic if it is either entirely nonincreasing or nondecreasing given an ordered set of inputs.</p>
+     * </p>
+     * A function is monotonic if it is either entirely nonincreasing or nondecreasing given an ordered set of inputs.
      *
      * @return {@code true} if the function is monotonic {@code false} otherwise.
      */
-    public default boolean isMonotonic()
+    default boolean isMonotonic()
     {
         return false;
     }
@@ -49,7 +49,7 @@ public default boolean isMonotonic()
      * @return the result of applying this function to the arguments
      * @throws InvalidRequestException if this function cannot not be applied to the arguments
      */
-    public ByteBuffer execute(Arguments arguments) throws InvalidRequestException;
+    ByteBuffer execute(Arguments arguments) throws InvalidRequestException;
 
     /**
      * Does a partial application of the function. That is, given only some of the arguments of the function provided,
@@ -69,7 +69,7 @@ public default boolean isMonotonic()
      * @param partialArguments a list of input arguments for the function where some arguments can be {@link #UNRESOLVED}.
      *                         The input <b>must</b> be of size {@code this.argsType().size()}. For convenience, it is
      *                         allowed both to pass a list with all arguments being {@link #UNRESOLVED} (the function is
-     *                         then returned directly) and with none of them unresolved (in which case, if the function is pure,
+     *                         then returned directly) and with none of them unresolved (in which case, if the function is deterministic,
      *                         it is computed and a dummy no-arg function returning the result is returned).
      * @return a function corresponding to the partial application of this function to the arguments of
      * {@code partialArguments} that are not {@link #UNRESOLVED}.
@@ -86,7 +86,7 @@ default ScalarFunction partialApplication(ProtocolVersion protocolVersion, List<
         if (unresolvedCount == argTypes().size())
             return this;
 
-        if (isPure() && unresolvedCount == 0)
+        if (isDeterministic() && unresolvedCount == 0)
         {
             Arguments arguments = newArguments(protocolVersion);
             for (int i = 0, m = partialArguments.size(); i < m; i++)
@@ -103,4 +103,16 @@ default ScalarFunction partialApplication(ProtocolVersion protocolVersion, List<
 
         return new PartiallyAppliedScalarFunction(this, partialArguments, unresolvedCount);
     }
+
+    /**
+     * Checks if a partial application of the function is monotonic.
+     *
+     *<p>A function is monotonic if it is either entirely nonincreasing or nondecreasing.</p>
+     * @param partialParameters the input parameters used to create the partial application of the function
+     * @return {@code true} if the partial application of the function is monotonic {@code false} otherwise.
+     */
+    default boolean isPartialApplicationMonotonic(List<ByteBuffer> partialParameters)
+    {
+        return isMonotonic();
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
index eb547f0f625d..b175d0ce3056 100644
--- a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
@@ -77,7 +77,7 @@ public ByteBuffer execute(Arguments arguments)
         }
 
         @Override
-        public boolean isPure()
+        public boolean isDeterministic()
         {
             return false; // as it returns non-identical results for identical arguments
         }
@@ -294,7 +294,7 @@ protected FloorFunction(AbstractType<?> returnType,
          }
 
          @Override
-         protected boolean isPartialApplicationMonotonic(List<ByteBuffer> partialParameters)
+         public boolean isPartialApplicationMonotonic(List<ByteBuffer> partialParameters)
          {
              return partialParameters.get(0) == UNRESOLVED
                      && partialParameters.get(1) != UNRESOLVED
@@ -461,7 +461,7 @@ protected void validateDuration(Duration duration)
      public static final NativeScalarFunction floorTime = new NativeScalarFunction("floor", TimeType.instance, TimeType.instance, DurationType.instance)
      {
          @Override
-         protected boolean isPartialApplicationMonotonic(List<ByteBuffer> partialParameters)
+         public boolean isPartialApplicationMonotonic(List<ByteBuffer> partialParameters)
          {
              return partialParameters.get(0) == UNRESOLVED && partialParameters.get(1) != UNRESOLVED;
          }
diff --git a/src/java/org/apache/cassandra/cql3/functions/ToJsonFct.java b/src/java/org/apache/cassandra/cql3/functions/ToJsonFct.java
index a1182a9c9e57..37bbda0fb414 100644
--- a/src/java/org/apache/cassandra/cql3/functions/ToJsonFct.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ToJsonFct.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.cql3.functions;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -90,7 +91,9 @@ public Factory(String name)
         }
 
         @Override
-        protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+        protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                       List<AbstractType<?>> argTypes,
+                                                       AbstractType<?> receiverType)
         {
             return ToJsonFct.getInstance(name.name, argTypes);
         }
diff --git a/src/java/org/apache/cassandra/cql3/functions/TokenFct.java b/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
index dd163b6b6da3..839cb6555d53 100644
--- a/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
+++ b/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
@@ -25,7 +25,7 @@
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.CBuilder;
+import org.apache.cassandra.db.ClusteringBuilder;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -59,7 +59,7 @@ private static AbstractType<?>[] getKeyTypes(TableMetadata metadata)
 
     public ByteBuffer execute(Arguments arguments) throws InvalidRequestException
     {
-        CBuilder builder = CBuilder.create(metadata.partitionKeyAsClusteringComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(metadata.partitionKeyAsClusteringComparator());
         for (int i = 0; i < arguments.size(); i++)
         {
             ByteBuffer bb = arguments.get(i);
@@ -96,7 +96,9 @@ public NativeFunction getOrCreateFunction(List<? extends AssignmentTestable> arg
             }
 
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                           List<AbstractType<?>> argTypes,
+                                                           AbstractType<?> receiverType)
             {
                 throw new AssertionError("Should be unreachable");
             }
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
index 2b15c8d9355e..642cbd0be2b3 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
@@ -18,7 +18,11 @@
 package org.apache.cassandra.cql3.functions;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
 
 import com.google.common.base.Objects;
 import com.google.common.collect.Lists;
@@ -31,6 +35,7 @@
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.Difference;
+import org.apache.cassandra.schema.Types;
 import org.apache.cassandra.schema.UserFunctions;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -52,13 +57,15 @@ public class UDAggregate extends UserFunction implements AggregateFunction
     protected final ByteBuffer initcond;
     private final ScalarFunction stateFunction;
     private final ScalarFunction finalFunction;
+    private final boolean deterministic;
 
     public UDAggregate(FunctionName name,
                        List<AbstractType<?>> argTypes,
                        AbstractType<?> returnType,
                        ScalarFunction stateFunc,
                        ScalarFunction finalFunc,
-                       ByteBuffer initcond)
+                       ByteBuffer initcond,
+                       boolean deterministic)
     {
         super(name, argTypes, returnType);
         this.stateFunction = stateFunc;
@@ -67,6 +74,7 @@ public UDAggregate(FunctionName name,
         this.resultType = UDFDataType.wrap(returnType, false);
         this.stateType = stateFunc != null ? UDFDataType.wrap(stateFunc.returnType(), false) : null;
         this.initcond = initcond;
+        this.deterministic = deterministic;
     }
 
     public static UDAggregate create(Collection<UDFunction> functions,
@@ -76,7 +84,8 @@ public static UDAggregate create(Collection<UDFunction> functions,
                                      FunctionName stateFunc,
                                      FunctionName finalFunc,
                                      AbstractType<?> stateType,
-                                     ByteBuffer initcond)
+                                     ByteBuffer initcond,
+                                     boolean deterministic)
     {
         List<AbstractType<?>> stateTypes = new ArrayList<>(argTypes.size() + 1);
         stateTypes.add(stateType);
@@ -87,7 +96,8 @@ public static UDAggregate create(Collection<UDFunction> functions,
                                returnType,
                                findFunction(name, functions, stateFunc, stateTypes),
                                null == finalFunc ? null : findFunction(name, functions, finalFunc, finalTypes),
-                               initcond);
+                               initcond,
+                               deterministic);
     }
 
     private static UDFunction findFunction(FunctionName udaName, Collection<UDFunction> functions, FunctionName name, List<AbstractType<?>> arguments)
@@ -98,10 +108,10 @@ private static UDFunction findFunction(FunctionName udaName, Collection<UDFuncti
                         .orElseThrow(() -> new ConfigurationException(String.format("Unable to find function %s referenced by UDA %s", name, udaName)));
     }
 
-    public boolean isPure()
+    @Override
+    public boolean isDeterministic()
     {
-        // Right now, we have no way to check if an UDA is pure. Due to that we consider them as non pure to avoid any risk.
-        return false;
+        return deterministic;
     }
 
     @Override
@@ -135,7 +145,31 @@ public UDAggregate withUpdatedUserType(Collection<UDFunction> udfs, UserType udt
                                returnType.withUpdatedUserType(udt),
                                findFunction(name, udfs, stateFunction.name(), stateFunction.argTypes()),
                                null == finalFunction ? null : findFunction(name, udfs, finalFunction.name(), finalFunction.argTypes()),
-                               initcond);
+                               initcond,
+                               deterministic);
+    }
+
+    public UDAggregate withNewKeyspace(String newKeyspace, Collection<UDFunction> udfs, Types types)
+    {
+        return new UDAggregate(new FunctionName(newKeyspace, name.name),
+                               withUpdatedUserTypes(argTypes, types),
+                               returnType.withUpdatedUserTypes(types),
+                               findFunction(name,
+                                            udfs,
+                                            new FunctionName(newKeyspace, stateFunction.name().name),
+                                            withUpdatedUserTypes(stateFunction.argTypes(), types)),
+                               null == finalFunction ? null
+                                                     : findFunction(name,
+                                                                    udfs,
+                                                                    new FunctionName(newKeyspace, finalFunction.name().name),
+                                                                    withUpdatedUserTypes(finalFunction.argTypes(), types)),
+                               initcond,
+                               deterministic);
+    }
+
+    private List<AbstractType<?>> withUpdatedUserTypes(List<AbstractType<?>> argTypes, Types types)
+    {
+        return Lists.newArrayList(transform(argTypes, t -> t.withUpdatedUserTypes(types)));
     }
 
     @Override
@@ -372,6 +406,10 @@ public String toCqlString(boolean withInternals, boolean ifNotExists)
                    .append("INITCOND ")
                    .append(stateType().asCQL3Type().toCQLLiteral(initialCondition()));
 
+        if (deterministic)
+            builder.newLine()
+                   .append("DETERMINISTIC");
+
         return builder.append(";")
                       .toString();
     }
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFunction.java b/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
index 538d80e9923f..395cd534c4d1 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
@@ -27,8 +27,8 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.CompletableFuture; // checkstyle: permit this import
 import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture; // checkstyle: permit this import
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
@@ -50,7 +50,9 @@
 import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.exceptions.FunctionExecutionException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Difference;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -75,6 +77,10 @@ public abstract class UDFunction extends UserFunction implements ScalarFunction
     protected final String language;
     protected final String body;
 
+    protected final boolean deterministic;
+    protected final boolean monotonic;
+    protected final List<ColumnIdentifier> monotonicOn;
+
     protected final List<UDFDataType> argumentTypes;
     protected final UDFDataType resultType;
     protected final boolean calledOnNullInput;
@@ -99,6 +105,8 @@ public abstract class UDFunction extends UserFunction implements ScalarFunction
     "com/google/common/reflect/TypeToken",
     "java/io/IOException.class",
     "java/io/Serializable.class",
+    "java/io/ObjectOutputStream.class",
+    "java/io/ObjectInputStream.class",
     "java/lang/",
     "java/math/",
     "java/net/InetAddress.class",
@@ -209,13 +217,19 @@ protected UDFunction(FunctionName name,
                          AbstractType<?> returnType,
                          boolean calledOnNullInput,
                          String language,
-                         String body)
+                         String body,
+                         boolean deterministic,
+                         boolean monotonic,
+                         List<ColumnIdentifier> monotonicOn)
     {
         super(name, argTypes, returnType);
         assert new HashSet<>(argNames).size() == argNames.size() : "duplicate argument names";
         this.argNames = argNames;
         this.language = language;
         this.body = body;
+        this.deterministic = deterministic;
+        this.monotonic = monotonic;
+        this.monotonicOn = monotonicOn;
         this.argumentTypes = UDFDataType.wrap(argTypes, !calledOnNullInput);
         this.resultType = UDFDataType.wrap(returnType, !calledOnNullInput);
         this.calledOnNullInput = calledOnNullInput;
@@ -234,15 +248,18 @@ public static UDFunction tryCreate(FunctionName name,
                                        AbstractType<?> returnType,
                                        boolean calledOnNullInput,
                                        String language,
-                                       String body)
+                                       String body,
+                                       boolean deterministic,
+                                       boolean monotonic,
+                                       List<ColumnIdentifier> monotonicOn)
     {
         try
         {
-            return create(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+            return create(name, argNames, argTypes, returnType, calledOnNullInput, language, body, deterministic, monotonic, monotonicOn);
         }
         catch (InvalidRequestException e)
         {
-            return createBrokenFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body, e);
+            return createBrokenFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body, deterministic, monotonic, monotonicOn, e);
         }
     }
 
@@ -252,11 +269,14 @@ public static UDFunction create(FunctionName name,
                                     AbstractType<?> returnType,
                                     boolean calledOnNullInput,
                                     String language,
-                                    String body)
+                                    String body,
+                                    boolean deterministic,
+                                    boolean monotonic,
+                                    List<ColumnIdentifier> monotonicOn)
     {
         assertUdfsEnabled(language);
 
-        return new JavaBasedUDFunction(name, argNames, argTypes, returnType, calledOnNullInput, body);
+        return new JavaBasedUDFunction(name, argNames, argTypes, returnType, calledOnNullInput, body, deterministic, monotonic, monotonicOn);
     }
 
     /**
@@ -275,9 +295,12 @@ public static UDFunction createBrokenFunction(FunctionName name,
                                                   boolean calledOnNullInput,
                                                   String language,
                                                   String body,
+                                                  boolean deterministic,
+                                                  boolean monotonic,
+                                                  List<ColumnIdentifier> monotonicOn,
                                                   InvalidRequestException reason)
     {
-        return new UDFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body)
+        return new UDFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body, deterministic, monotonic, monotonicOn)
         {
             protected ExecutorService executor()
             {
@@ -341,8 +364,17 @@ public String toCqlString(boolean withInternals, boolean ifNotExists)
                .append(" ON NULL INPUT")
                .newLine()
                .append("RETURNS ")
-               .append(toCqlString(returnType()))
-               .newLine()
+               .append(toCqlString(returnType()));
+
+        if (deterministic)
+            builder.newLine().append("DETERMINISTIC");
+
+        if (monotonic)
+            builder.newLine().append("MONOTONIC");
+        else if (!monotonicOn.isEmpty())
+            builder.newLine().append("MONOTONIC ON ").append(monotonicOn.get(0).toCQLString());
+
+        builder.newLine()
                .append("LANGUAGE ")
                .append(language())
                .newLine()
@@ -354,10 +386,40 @@ public String toCqlString(boolean withInternals, boolean ifNotExists)
     }
 
     @Override
-    public boolean isPure()
+    public boolean isDeterministic()
     {
-        // Right now, we have no way to check if an UDF is pure. Due to that we consider them as non pure to avoid any risk.
-        return false;
+        return deterministic;
+    }
+
+    @Override
+    public boolean isMonotonic()
+    {
+        return monotonic;
+    }
+
+    public List<ColumnIdentifier> monotonicOn()
+    {
+        return monotonicOn;
+    }
+
+    @Override
+    public boolean isPartialApplicationMonotonic(List<ByteBuffer> partialParameters)
+    {
+        assert partialParameters.size() == argNames.size();
+        if (!monotonic)
+        {
+            for (int i = 0; i < partialParameters.size(); i ++)
+            {
+                ByteBuffer partialParameter = partialParameters.get(i);
+                if (partialParameter == Function.UNRESOLVED)
+                {
+                    ColumnIdentifier unresolvedArgumentName = argNames.get(i);
+                    if (!monotonicOn.contains(unresolvedArgumentName))
+                        return false;
+                }
+            }
+        }
+        return true;
     }
 
     @Override
@@ -622,7 +684,24 @@ public UDFunction withUpdatedUserType(UserType udt)
                          returnType.withUpdatedUserType(udt),
                          calledOnNullInput,
                          language,
-                         body);
+                         body,
+                         deterministic,
+                         monotonic,
+                         monotonicOn);
+    }
+
+    public UDFunction withNewKeyspace(String newKeyspace, Types types)
+    {
+        return tryCreate(new FunctionName(newKeyspace, name.name),
+                         argNames,
+                         Lists.newArrayList(transform(argTypes, t -> t.withUpdatedUserTypes(types))),
+                         returnType.withUpdatedUserTypes(types),
+                         calledOnNullInput,
+                         language,
+                         body,
+                         deterministic,
+                         monotonic,
+                         monotonicOn);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/cql3/functions/UuidFcts.java b/src/java/org/apache/cassandra/cql3/functions/UuidFcts.java
index 3e90deb68340..77a71c2fb4b4 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UuidFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UuidFcts.java
@@ -37,5 +37,11 @@ public ByteBuffer execute(Arguments arguments)
         {
             return UUIDSerializer.instance.serialize(UUID.randomUUID());
         }
+
+        @Override
+        public boolean isDeterministic()
+        {
+            return false; // since UUIDs are generated randomly and so function calls are not deterministic
+        }
     };
 }
diff --git a/src/java/org/apache/cassandra/cql3/functions/VectorFcts.java b/src/java/org/apache/cassandra/cql3/functions/VectorFcts.java
index ae219a529bc5..427d428a6a93 100644
--- a/src/java/org/apache/cassandra/cql3/functions/VectorFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/VectorFcts.java
@@ -20,14 +20,24 @@
 
 import java.nio.ByteBuffer;
 import java.util.List;
+import java.util.Random;
 
+import io.github.jbellis.jvector.vector.ArrayVectorFloat;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorUtil;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.NumberType;
 import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.transport.ProtocolVersion;
-import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.index.sai.disk.vector.VectorValidation.isEffectivelyZero;
 
 public class VectorFcts
 {
@@ -36,10 +46,12 @@ public static void addFunctionsTo(NativeFunctions functions)
         functions.add(createSimilarityFunctionFactory("similarity_cosine", VectorSimilarityFunction.COSINE, false));
         functions.add(createSimilarityFunctionFactory("similarity_euclidean", VectorSimilarityFunction.EUCLIDEAN, true));
         functions.add(createSimilarityFunctionFactory("similarity_dot_product", VectorSimilarityFunction.DOT_PRODUCT, true));
+        functions.add(new RandomFloatVectorFunctionFactory());
+        functions.add(new NormalizeL2FunctionFactory());
     }
 
     private static FunctionFactory createSimilarityFunctionFactory(String name,
-                                                                   VectorSimilarityFunction vectorSimilarityFunction,
+                                                                   VectorSimilarityFunction luceneFunction,
                                                                    boolean supportsZeroVectors)
     {
         return new FunctionFactory(name,
@@ -48,14 +60,16 @@ private static FunctionFactory createSimilarityFunctionFactory(String name,
         {
             @Override
             @SuppressWarnings("unchecked")
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                           List<AbstractType<?>> argTypes,
+                                                           AbstractType<?> receiverType)
             {
                 // check that all arguments have the same vector dimensions
                 VectorType<Float> firstArgType = (VectorType<Float>) argTypes.get(0);
                 int dimensions = firstArgType.dimension;
                 if (!argTypes.stream().allMatch(t -> ((VectorType<?>) t).dimension == dimensions))
                     throw new InvalidRequestException("All arguments must have the same vector dimensions");
-                return createSimilarityFunction(name.name, firstArgType, vectorSimilarityFunction, supportsZeroVectors);
+                return createSimilarityFunction(name.name, firstArgType, luceneFunction, supportsZeroVectors);
             }
         };
     }
@@ -65,6 +79,7 @@ private static NativeFunction createSimilarityFunction(String name,
                                                            VectorSimilarityFunction f,
                                                            boolean supportsZeroVectors)
     {
+        var vts = VectorizationProvider.getInstance().getVectorTypeSupport();
         return new NativeScalarFunction(name, FloatType.instance, type, type)
         {
             @Override
@@ -81,25 +96,147 @@ public ByteBuffer execute(Arguments arguments) throws InvalidRequestException
                 if (arguments.containsNulls())
                     return null;
 
-                float[] v1 = arguments.get(0);
-                float[] v2 = arguments.get(1);
+                var v1 = vts.createFloatVector(arguments.get(0));
+                var v2 = vts.createFloatVector(arguments.get(1));
 
-                if (!supportsZeroVectors)
-                {
-                    if (isAllZero(v1) || isAllZero(v2))
-                        throw new InvalidRequestException("Function " + name + " doesn't support all-zero vectors.");
-                }
+                if (!supportsZeroVectors && (isEffectivelyZero(v1) || isEffectivelyZero(v2)))
+                    throw new InvalidRequestException("Function " + name + " doesn't support all-zero vectors.");
 
                 return FloatType.instance.decompose(f.compare(v1, v2));
             }
+        };
+    }
+
+    /**
+     * CQL native function create a random float vector of a certain dimension.
+     * All the components of the vector will be random floats between the specified min and max values.
+     */
+    private static class RandomFloatVectorFunctionFactory extends FunctionFactory
+    {
+        private static final String NAME = "random_float_vector";
+
+        private RandomFloatVectorFunctionFactory()
+        {
+            super(NAME, FunctionParameter.literalInteger(), FunctionParameter.float32(), FunctionParameter.float32());
+        }
 
-            private boolean isAllZero(float[] v)
+        @Override
+        protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                       List<AbstractType<?>> argTypes,
+                                                       AbstractType<?> receiverType)
+        {
+            // Get the vector type from the dimension argument. We need to do this here assuming that the argument is a
+            // literal, so we know the dimension of the return type before actually executing the function.
+            int dimension = Integer.parseInt(args.get(0).toString());
+            VectorType<Float> type = VectorType.getInstance(FloatType.instance, dimension);
+
+            final NumberType<?> minType = (NumberType<?>) argTypes.get(1);
+            final NumberType<?> maxType = (NumberType<?>) argTypes.get(2);
+
+            return new NativeScalarFunction(name.name, type, Int32Type.instance, minType, maxType)
             {
-                for (float f : v)
-                    if (f != 0)
-                        return false;
-                return true;
-            }
-        };
+                private final Random random = new Random();
+
+                @Override
+                public Arguments newArguments(ProtocolVersion version)
+                {
+                    return new FunctionArguments(version,
+                                                 (v, b) -> Int32Type.instance.compose(b),
+                                                 (v, b) -> {
+                                                    if (b == null || !b.hasRemaining())
+                                                        throw new InvalidRequestException(format("Min argument of function %s must not be null", 
+                                                                                                 RandomFloatVectorFunctionFactory.this));
+                                                     return minType.compose(b).floatValue();
+                                                 },
+                                                 (v, b) -> {
+                                                     if (b == null || !b.hasRemaining())
+                                                         throw new InvalidRequestException(format("Max argument of function %s must not be null",
+                                                                                                  RandomFloatVectorFunctionFactory.this));
+                                                     return maxType.compose(b).floatValue();
+                                                 });
+                }
+
+                @Override
+                public ByteBuffer execute(Arguments arguments)
+                {
+                    // get the min argument
+                    float min = arguments.get(1);
+                    if (!Float.isFinite(min))
+                        throw new InvalidRequestException("Min value must be finite");
+
+                    // get the max argument
+                    float max = arguments.get(2);
+                    if (!Float.isFinite(max))
+                        throw new InvalidRequestException("Max value must be finite");
+                    if (max <= min)
+                        throw new InvalidRequestException("Max value must be greater than min value");
+
+                    // generate the random vector within the range defined by min and max
+                    float[] vector = new float[dimension];
+                    for (int i = 0; i < dimension; i++)
+                    {
+                        // promote to double to avoid overflow with large (absolute value) min and/or max
+                        double dmin = min;
+                        double dmax = max;
+                        vector[i] = (float) (dmin + random.nextDouble() * (dmax - dmin));
+                    }
+
+                    return type.getSerializer().serializeFloatArray(vector);
+                }
+            };
+        }
+    }
+
+    /**
+     * CQL native function to normalize a vector using L2 normalization.
+     */
+    private static class NormalizeL2FunctionFactory extends FunctionFactory
+    {
+        private static final String NAME = "normalize_l2";
+
+        public NormalizeL2FunctionFactory()
+        {
+            super(NAME, FunctionParameter.vector(CQL3Type.Native.FLOAT));
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                       List<AbstractType<?>> argTypes,
+                                                       AbstractType<?> receiverType)
+        {
+            var vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+            VectorType<Float> vectorType = (VectorType<Float>) argTypes.get(0);
+
+            return new NativeScalarFunction(name.name, vectorType, vectorType)
+            {
+                @Override
+                public Arguments newArguments(ProtocolVersion version)
+                {
+                    return new FunctionArguments(version,
+                                                 (v, b) -> {
+                                                     if (b == null || !b.hasRemaining())
+                                                         return null;
+                                                     return vectorType.getSerializer().deserializeFloatArray(b);
+                                                 });
+                }
+
+                @Override
+                public ByteBuffer execute(Arguments arguments)
+                {
+                    // get the vector argument
+                    var arg0 = arguments.get(0);
+                    if (arg0 == null)
+                        return null;
+                    var vector = vts.createFloatVector(arg0);
+
+                    // normalize
+                    VectorUtil.l2normalize(vector);
+
+                    // serialize the normalized vector
+                    return vectorType.getSerializer().serializeFloatArray(((ArrayVectorFloat) vector).get());
+                }
+            };
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunction.java
index e5c0ee2e7d6b..75e402368b4e 100644
--- a/src/java/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunction.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.functions.Arguments;
 import org.apache.cassandra.cql3.functions.FunctionArguments;
 import org.apache.cassandra.cql3.functions.FunctionFactory;
@@ -70,7 +71,7 @@ public static FunctionFactory factory()
         return new MaskingFunction.Factory(NAME, FunctionParameter.anyType(false))
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args, List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return new DefaultMaskingFunction(name, argTypes.get(0));
             }
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/HashMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/HashMaskingFunction.java
index 3291b156f0e1..142353ca866c 100644
--- a/src/java/org/apache/cassandra/cql3/functions/masking/HashMaskingFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/HashMaskingFunction.java
@@ -28,6 +28,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
 import org.apache.cassandra.cql3.functions.Arguments;
@@ -136,7 +137,7 @@ public static FunctionFactory factory()
                                            FunctionParameter.optional(FunctionParameter.fixed(CQL3Type.Native.TEXT)))
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args, List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 switch (argTypes.size())
                 {
@@ -145,7 +146,8 @@ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, A
                     case 2:
                         return new HashMaskingFunction(name, argTypes.get(0), true);
                     default:
-                        throw invalidNumberOfArgumentsException();
+                        throw new InvalidRequestException("Invalid number of arguments for function " + this);
+//                        throw invalidNumberOfArgumentsException();
                 }
             }
         };
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/NullMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/NullMaskingFunction.java
index 830a1c3b5484..046fd0e44fbd 100644
--- a/src/java/org/apache/cassandra/cql3/functions/masking/NullMaskingFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/NullMaskingFunction.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.functions.Arguments;
 import org.apache.cassandra.cql3.functions.FunctionArguments;
 import org.apache.cassandra.cql3.functions.FunctionFactory;
@@ -65,7 +66,7 @@ public static FunctionFactory factory()
         return new MaskingFunction.Factory(NAME, FunctionParameter.anyType(false))
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args, List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 return new NullMaskingFunction(name, argTypes.get(0));
             }
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunction.java
index 8f5a794b8e85..a7163d0bed3e 100644
--- a/src/java/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunction.java
@@ -27,6 +27,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang3.StringUtils;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
 import org.apache.cassandra.cql3.functions.Arguments;
@@ -179,7 +180,8 @@ public static Collection<FunctionFactory> factories()
                      .collect(Collectors.toSet());
     }
 
-    private static FunctionFactory factory(Kind kind)
+    @VisibleForTesting
+    public static FunctionFactory factory(Kind kind)
     {
         return new MaskingFunction.Factory(kind.name(),
                                            FunctionParameter.string(),
@@ -189,7 +191,7 @@ private static FunctionFactory factory(Kind kind)
         {
             @Override
             @SuppressWarnings("unchecked")
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args, List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 AbstractType<String> inputType = (AbstractType<String>) argTypes.get(0);
                 return new PartialMaskingFunction(name, kind, inputType, argTypes.size() == 4);
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunction.java
index 373743daf4c5..4f667a7f9a5a 100644
--- a/src/java/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunction.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.functions.Arguments;
 import org.apache.cassandra.cql3.functions.FunctionArguments;
 import org.apache.cassandra.cql3.functions.FunctionFactory;
@@ -67,7 +68,7 @@ public static FunctionFactory factory()
                                            FunctionParameter.sameAs(0, true, FunctionParameter.anyType(true)))
         {
             @Override
-            protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+            protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args, List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
             {
                 AbstractType<?> replacedType = argTypes.get(0);
                 AbstractType<?> replacementType = argTypes.get(1);
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
index ddd9601c834d..e63a0ea0d824 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
@@ -19,17 +19,16 @@
 
 import java.util.*;
 
-import javax.annotation.Nullable;
-
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.statements.Bound;
 import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.btree.BTreeSet;
 
@@ -46,73 +45,26 @@ final class ClusteringColumnRestrictions extends RestrictionSetWrapper
      */
     private final ClusteringComparator comparator;
 
-    /**
-     * <code>true</code> if filtering is allowed for this restriction, <code>false</code> otherwise
-     */
-    private final boolean allowFiltering;
-
-    public ClusteringColumnRestrictions(TableMetadata table, boolean allowFiltering)
-    {
-        this(table.comparator, new RestrictionSet(), allowFiltering);
-    }
-
     private ClusteringColumnRestrictions(ClusteringComparator comparator,
-                                         RestrictionSet restrictionSet,
-                                         boolean allowFiltering)
+                                         RestrictionSet restrictionSet)
     {
         super(restrictionSet);
         this.comparator = comparator;
-        this.allowFiltering = allowFiltering;
-    }
-
-    public ClusteringColumnRestrictions mergeWith(Restriction restriction, @Nullable IndexRegistry indexRegistry) throws InvalidRequestException
-    {
-        SingleRestriction newRestriction = (SingleRestriction) restriction;
-        RestrictionSet newRestrictionSet = restrictions.addRestriction(newRestriction);
-
-        if (!isEmpty() && !allowFiltering && (indexRegistry == null || !newRestriction.hasSupportingIndex(indexRegistry)))
-        {
-            SingleRestriction lastRestriction = restrictions.lastRestriction();
-            assert lastRestriction != null;
-
-            ColumnMetadata lastRestrictionStart = lastRestriction.getFirstColumn();
-            ColumnMetadata newRestrictionStart = restriction.getFirstColumn();
-
-            checkFalse(lastRestriction.isSlice() && newRestrictionStart.position() > lastRestrictionStart.position(),
-                       "Clustering column \"%s\" cannot be restricted (preceding column \"%s\" is restricted by a non-EQ relation)",
-                       newRestrictionStart.name,
-                       lastRestrictionStart.name);
-
-            if (newRestrictionStart.position() < lastRestrictionStart.position() && newRestriction.isSlice())
-                throw invalidRequest("PRIMARY KEY column \"%s\" cannot be restricted (preceding column \"%s\" is restricted by a non-EQ relation)",
-                                     restrictions.nextColumn(newRestrictionStart).name,
-                                     newRestrictionStart.name);
-        }
-
-        return new ClusteringColumnRestrictions(this.comparator, newRestrictionSet, allowFiltering);
-    }
-
-    private boolean hasMultiColumnSlice()
-    {
-        for (SingleRestriction restriction : restrictions)
-        {
-            if (restriction.isMultiColumn() && restriction.isSlice())
-                return true;
-        }
-        return false;
     }
 
     public NavigableSet<Clustering<?>> valuesAsClustering(QueryOptions options, ClientState state) throws InvalidRequestException
     {
-        MultiCBuilder builder = MultiCBuilder.create(comparator, hasIN());
-        for (SingleRestriction r : restrictions)
+        MultiClusteringBuilder builder = MultiClusteringBuilder.create(comparator);
+        List<SingleRestriction> restrictions = restrictions();
+        for (int i = 0; i < restrictions.size(); i++)
         {
+            SingleRestriction r = restrictions.get(i);
             r.appendTo(builder, options);
 
             if (hasIN() && Guardrails.inSelectCartesianProduct.enabled(state))
                 Guardrails.inSelectCartesianProduct.guard(builder.buildSize(), "clustering key", false, state);
 
-            if (builder.hasMissingElements())
+            if (builder.buildIsEmpty())
                 break;
         }
         return builder.build();
@@ -120,65 +72,31 @@ public NavigableSet<Clustering<?>> valuesAsClustering(QueryOptions options, Clie
 
     public NavigableSet<ClusteringBound<?>> boundsAsClustering(Bound bound, QueryOptions options) throws InvalidRequestException
     {
-        MultiCBuilder builder = MultiCBuilder.create(comparator, hasIN() || hasMultiColumnSlice());
+        List<SingleRestriction> restrictionsList = restrictions();
+
+        MultiClusteringBuilder builder = MultiClusteringBuilder.create(comparator);
         int keyPosition = 0;
 
-        for (SingleRestriction r : restrictions)
+        for (int i = 0; i < restrictionsList.size(); i++)
         {
+            SingleRestriction r = restrictionsList.get(i);
             if (handleInFilter(r, keyPosition))
                 break;
 
-            if (r.isSlice())
-            {
-                r.appendBoundTo(builder, bound, options);
-                return builder.buildBoundForSlice(bound.isStart(),
-                                                  r.isInclusive(bound),
-                                                  r.isInclusive(bound.reverse()),
-                                                  r.getColumnDefs());
-            }
-
             r.appendBoundTo(builder, bound, options);
 
-            if (builder.hasMissingElements())
+            if (builder.buildIsEmpty())
                 return BTreeSet.empty(comparator);
 
-            keyPosition = r.getLastColumn().position() + 1;
-        }
-
-        // Everything was an equal (or there was nothing)
-        return builder.buildBound(bound.isStart(), true);
-    }
+            // We allow slice restriction only on the last clustering column restricted by the query.
+            // Any further column restrictions must be handled by indexes or filtering.
+            if (r.isSlice())
+                break;
 
-    /**
-     * Checks if any of the underlying restriction is a CONTAINS or CONTAINS KEY.
-     *
-     * @return <code>true</code> if any of the underlying restriction is a CONTAINS or CONTAINS KEY,
-     * <code>false</code> otherwise
-     */
-    public boolean hasContains()
-    {
-        for (SingleRestriction restriction : restrictions)
-        {
-            if (restriction.isContains())
-                return true;
+            keyPosition = r.getLastColumn().position() + 1;
         }
-        return false;
-    }
 
-    /**
-     * Checks if any of the underlying restriction is a slice restrictions.
-     *
-     * @return <code>true</code> if any of the underlying restriction is a slice restrictions,
-     * <code>false</code> otherwise
-     */
-    public boolean hasSlice()
-    {
-        for (SingleRestriction restriction : restrictions)
-        {
-            if (restriction.isSlice())
-                return true;
-        }
-        return false;
+        return builder.buildBound(bound.isStart());
     }
 
     /**
@@ -191,8 +109,10 @@ public boolean needFiltering()
     {
         int position = 0;
 
-        for (SingleRestriction restriction : restrictions)
+        List<SingleRestriction> restrictions = restrictions();
+        for (int i = 0; i < restrictions.size(); i++)
         {
+            SingleRestriction restriction = restrictions.get(i);
             if (handleInFilter(restriction, position))
                 return true;
 
@@ -203,18 +123,21 @@ public boolean needFiltering()
     }
 
     @Override
-    public void addToRowFilter(RowFilter filter,
+    public void addToRowFilter(RowFilter.Builder filter,
                                IndexRegistry indexRegistry,
-                               QueryOptions options) throws InvalidRequestException
+                               QueryOptions options,
+                               ANNOptions annOptions) throws InvalidRequestException
     {
         int position = 0;
 
-        for (SingleRestriction restriction : restrictions)
+        List<SingleRestriction> restrictions = restrictions();
+        for (int i = 0; i < restrictions.size(); i++)
         {
+            SingleRestriction restriction = restrictions.get(i);
             // We ignore all the clustering columns that can be handled by slices.
             if (handleInFilter(restriction, position) || restriction.hasSupportingIndex(indexRegistry))
             {
-                restriction.addToRowFilter(filter, indexRegistry, options);
+                restriction.addToRowFilter(filter, indexRegistry, options, annOptions);
                 continue;
             }
 
@@ -227,4 +150,70 @@ private boolean handleInFilter(SingleRestriction restriction, int index)
     {
         return restriction.isContains() || restriction.isLIKE() || index != restriction.getFirstColumn().position();
     }
+
+    public static ClusteringColumnRestrictions.Builder builder(TableMetadata table, boolean allowFiltering)
+    {
+        return new Builder(table, allowFiltering, null);
+    }
+
+    public static ClusteringColumnRestrictions.Builder builder(TableMetadata table, boolean allowFiltering, IndexRegistry indexRegistry)
+    {
+        return new Builder(table, allowFiltering, indexRegistry);
+    }
+
+    public static class Builder
+    {
+        private final TableMetadata table;
+        private final boolean allowFiltering;
+        private final IndexRegistry indexRegistry;
+
+        private final RestrictionSet.Builder restrictions = RestrictionSet.builder();
+
+        private Builder(TableMetadata table, boolean allowFiltering, IndexRegistry indexRegistry)
+        {
+            this.table = table;
+            this.allowFiltering = allowFiltering;
+            this.indexRegistry = indexRegistry;
+        }
+
+        public ClusteringColumnRestrictions.Builder addRestriction(Restriction restriction)
+        {
+            return addRestriction(restriction, false);
+        }
+
+        public ClusteringColumnRestrictions.Builder addRestriction(Restriction restriction, boolean isDisjunction)
+        {
+            SingleRestriction newRestriction = (SingleRestriction) restriction;
+            boolean isEmpty = restrictions.isEmpty();
+
+            if (!isEmpty && !allowFiltering && (indexRegistry == null || !newRestriction.hasSupportingIndex(indexRegistry)))
+            {
+                SingleRestriction lastRestriction = restrictions.lastRestriction();
+                ColumnMetadata lastRestrictionStart = lastRestriction.getFirstColumn();
+                ColumnMetadata newRestrictionStart = newRestriction.getFirstColumn();
+                restrictions.addRestriction(newRestriction, isDisjunction);
+
+                checkFalse(lastRestriction.isSlice() && newRestrictionStart.position() > lastRestrictionStart.position(),
+                           "Clustering column \"%s\" cannot be restricted (preceding column \"%s\" is restricted by a non-EQ relation)",
+                           newRestrictionStart.name,
+                           lastRestrictionStart.name);
+
+                if (newRestrictionStart.position() < lastRestrictionStart.position() && newRestriction.isSlice())
+                    throw invalidRequest("PRIMARY KEY column \"%s\" cannot be restricted (preceding column \"%s\" is restricted by a non-EQ relation)",
+                                         restrictions.nextColumn(newRestrictionStart).name,
+                                         newRestrictionStart.name);
+            }
+            else
+            {
+                restrictions.addRestriction(newRestriction, isDisjunction);
+            }
+
+            return this;
+        }
+
+        public ClusteringColumnRestrictions build()
+        {
+            return new ClusteringColumnRestrictions(table.comparator, restrictions.build());
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java b/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
index 569418b3f4b8..b36565586501 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
@@ -25,7 +25,7 @@
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.TableMetadata;
 
-public class CustomIndexExpression
+public class CustomIndexExpression implements ExternalRestriction
 {
     private final ColumnIdentifier valueColId = new ColumnIdentifier("custom index expression", false);
 
@@ -47,7 +47,7 @@ public void prepareValue(TableMetadata table, AbstractType<?> expressionType, Va
         value.collectMarkerSpecification(boundNames);
     }
 
-    public void addToRowFilter(RowFilter filter, TableMetadata table, QueryOptions options)
+    public void addToRowFilter(RowFilter.Builder filter, TableMetadata table, QueryOptions options)
     {
         filter.addCustomIndexExpression(table,
                                         table.indexes
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/ExternalRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/ExternalRestriction.java
new file mode 100644
index 000000000000..c0ace7d1ef15
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/restrictions/ExternalRestriction.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.restrictions;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.schema.TableMetadata;
+
+public interface ExternalRestriction
+{
+    public void addToRowFilter(RowFilter.Builder filter, TableMetadata table, QueryOptions options);
+
+    /**
+     * Returns whether this restriction would need filtering if the specified index group were used.
+     *
+     * @param indexGroup an index group
+     * @return {@code true} if this would need filtering if {@code indexGroup} were used, {@code false} otherwise
+     */
+    public boolean needsFiltering(Index.Group indexGroup);
+}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
index fe01e4188744..a30c7586bd5e 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.cql3.restrictions;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.cassandra.cql3.QualifiedName;
@@ -26,43 +27,128 @@
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
 
-public class IndexRestrictions
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
+public final class IndexRestrictions
 {
+    /**
+     * The empty {@code IndexRestrictions}.
+     */
+    private static final IndexRestrictions EMPTY_RESTRICTIONS = new IndexRestrictions(Collections.EMPTY_LIST, Collections.EMPTY_LIST);
+
     public static final String INDEX_NOT_FOUND = "Invalid index expression, index %s not found for %s";
     public static final String INVALID_INDEX = "Target index %s cannot be used to query %s";
     public static final String CUSTOM_EXPRESSION_NOT_SUPPORTED = "Index %s does not support custom expressions";
     public static final String NON_CUSTOM_INDEX_IN_EXPRESSION = "Only CUSTOM indexes may be used in custom index expressions, %s is not valid";
     public static final String MULTIPLE_EXPRESSIONS = "Multiple custom index expressions in a single query are not supported";
 
-    private final List<Restrictions> regularRestrictions = new ArrayList<>();
-    private final List<CustomIndexExpression> customExpressions = new ArrayList<>();
+    private final List<Restrictions> regularRestrictions;
+    private final List<ExternalRestriction> externalRestrictions;
+
+    private IndexRestrictions(List<Restrictions> regularRestrictions, List<ExternalRestriction> externalExpressions)
+    {
+        this.regularRestrictions = regularRestrictions;
+        this.externalRestrictions = externalExpressions;
+    }
 
-    public void add(Restrictions restrictions)
+    /**
+     * Returns an empty {@code IndexRestrictions}.
+     * @return an empty {@code IndexRestrictions}
+     */
+    public static IndexRestrictions of()
     {
-        regularRestrictions.add(restrictions);
+        return EMPTY_RESTRICTIONS;
     }
 
-    public void add(CustomIndexExpression expression)
+    /**
+     * Creates a new {@code IndexRestrictions.Builder} instance.
+     * @return a new {@code IndexRestrictions.Builder} instance.
+     */
+    public static Builder builder()
     {
-        customExpressions.add(expression);
+        return new IndexRestrictions.Builder();
     }
 
     public boolean isEmpty()
     {
-        return regularRestrictions.isEmpty() && customExpressions.isEmpty();
+        return regularRestrictions.isEmpty() && externalRestrictions.isEmpty();
     }
 
+    /**
+     * Returns the regular restrictions.
+     * @return the regular restrictions
+     */
     public List<Restrictions> getRestrictions()
     {
         return regularRestrictions;
     }
 
-    public List<CustomIndexExpression> getCustomIndexExpressions()
+    /**
+     * Returns the external restrictions.
+     * @return the external restrictions
+     */
+    public List<ExternalRestriction> getExternalExpressions()
+    {
+        return externalRestrictions;
+    }
+
+    /**
+     * Returns the number of restrictions in external expression and regular restrictions.
+     * @return Returns the number of restrictions in external expression and regular restrictions.
+     */
+    private int numOfSupportedRestrictions()
     {
-        return customExpressions;
+        int numberOfRestrictions = getExternalExpressions().size();
+        for (Restrictions restrictions : getRestrictions())
+            numberOfRestrictions += restrictions.size();
+
+        return numberOfRestrictions;
+    }
+
+    /**
+     * Returns whether these restrictions would need filtering if the specified index registry were used.
+     *
+     * @param indexRegistry an index registry
+     * @param hasClusteringColumnRestrictions {@code true} if there are restricted clustering columns
+     * @param hasMultipleContains {@code true} if there are multiple "contains" restrictions
+     * @return {@code true} if this would need filtering if {@code indexRegistry} were used, {@code false} otherwise
+     */
+    public boolean needFiltering(IndexRegistry indexRegistry, boolean hasClusteringColumnRestrictions, boolean hasMultipleContains)
+    {
+        // We need filtering if any clustering columns have restrictions that are not supported
+        // by their indexes.
+        if (numOfSupportedRestrictions() == 0)
+            return hasClusteringColumnRestrictions;
+
+        for (Index.Group group : indexRegistry.listIndexGroups())
+            if (!needFiltering(group, hasMultipleContains))
+                return false;
+
+        return true;
+    }
+
+    /**
+     * Returns whether these restrictions would need filtering if the specified index group were used.
+     *
+     * @param indexGroup an index group
+     * @param hasMultipleContains {@code true} if there are multiple "contains" restrictions
+     * @return {@code true} if this would need filtering if {@code indexGroup} were used, {@code false} otherwise
+     */
+    private boolean needFiltering(Index.Group indexGroup, boolean hasMultipleContains)
+    {
+        if (hasMultipleContains && !indexGroup.supportsMultipleContains())
+            return true;
+
+        for (Restrictions restrictions : regularRestrictions)
+            if (restrictions.needsFiltering(indexGroup))
+                return true;
+
+        for (ExternalRestriction restriction : externalRestrictions)
+            if (restriction.needsFiltering(indexGroup))
+                return true;
+
+        return false;
     }
 
     /**
@@ -99,7 +185,7 @@ private boolean needsFiltering(Index.Group indexGroup)
                 return true;
         }
 
-        for (CustomIndexExpression restriction : customExpressions)
+        for (ExternalRestriction restriction : externalRestrictions)
         {
             if (restriction.needsFiltering(indexGroup))
                 return true;
@@ -108,6 +194,19 @@ private boolean needsFiltering(Index.Group indexGroup)
         return false;
     }
 
+    public boolean indexBeingUsed(Index.Group indexGroup)
+    {
+        for (Restrictions restrictions : regularRestrictions)
+            if (!restrictions.needsFiltering(indexGroup))
+                return true;
+
+        for (ExternalRestriction restriction : externalRestrictions)
+            if (!restriction.needsFiltering(indexGroup))
+                return true;
+
+        return false;
+    }
+
     static InvalidRequestException invalidIndex(QualifiedName indexName, TableMetadata table)
     {
         return new InvalidRequestException(String.format(INVALID_INDEX, indexName.getName(), table));
@@ -120,17 +219,75 @@ static InvalidRequestException indexNotFound(QualifiedName indexName, TableMetad
 
     static InvalidRequestException nonCustomIndexInExpression(QualifiedName indexName)
     {
-        return new InvalidRequestException(String.format(NON_CUSTOM_INDEX_IN_EXPRESSION, indexName.getName()));
+        return invalidRequest(NON_CUSTOM_INDEX_IN_EXPRESSION, indexName.getName());
     }
 
     static InvalidRequestException customExpressionNotSupported(QualifiedName indexName)
     {
-        return new InvalidRequestException(String.format(CUSTOM_EXPRESSION_NOT_SUPPORTED, indexName.getName()));
+        return invalidRequest(CUSTOM_EXPRESSION_NOT_SUPPORTED, indexName.getName());
     }
-    
-    @Override
-    public String toString()
+
+    /**
+     * Builder for IndexRestrictions.
+     */
+    public static final class Builder
     {
-        return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+        /**
+         * Builder for the regular restrictions.
+         */
+        private List<Restrictions> regularRestrictions = new ArrayList<>();
+
+        /**
+         * Builder for the custom expressions.
+         */
+        private List<ExternalRestriction> externalRestrictions = new ArrayList<>();
+
+        private Builder() {}
+
+        /**
+         * Adds the specified restrictions.
+         *
+         * @param restrictions the restrictions to add
+         * @return this {@code Builder}
+         */
+        public Builder add(Restrictions restrictions)
+        {
+            regularRestrictions.add(restrictions);
+            return this;
+        }
+
+        /**
+         * Adds the restrictions and custom expressions from the specified {@code IndexRestrictions}.
+         *
+         * @param restrictions the restrictions and custom expressions to add
+         * @return this {@code Builder}
+         */
+        public Builder add(IndexRestrictions restrictions)
+        {
+            regularRestrictions.addAll(restrictions.regularRestrictions);
+            externalRestrictions.addAll(restrictions.externalRestrictions);
+            return this;
+        }
+
+        /**
+         * Adds the specified external expression.
+         *
+         * @param restriction the external expression to add
+         * @return this {@code Builder}
+         */
+        public Builder add(ExternalRestriction restriction)
+        {
+            externalRestrictions.add(restriction);
+            return this;
+        }
+
+        /**
+         * Builds a new {@code IndexRestrictions} instance
+         * @return a new {@code IndexRestrictions} instance
+         */
+        public IndexRestrictions build()
+        {
+            return new IndexRestrictions(Collections.unmodifiableList(regularRestrictions), Collections.unmodifiableList(externalRestrictions));
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
index 5ad6dabd4c3a..187b1372b09c 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.HashSet;
@@ -28,24 +29,26 @@
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
-import org.apache.cassandra.cql3.AbstractMarker;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.MarkerOrTerms;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.Term;
-import org.apache.cassandra.cql3.Terms;
-import org.apache.cassandra.cql3.Tuples;
 import org.apache.cassandra.cql3.Term.Terminal;
+import org.apache.cassandra.cql3.Tuples;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.statements.Bound;
-import org.apache.cassandra.db.MultiCBuilder;
+import org.apache.cassandra.db.MultiClusteringBuilder;
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.serializers.ListSerializer;
+import org.apache.cassandra.serializers.CollectionSerializer;
 
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
-import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
 import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
 
@@ -126,48 +129,27 @@ protected final String getColumnsInCommons(Restriction otherRestriction)
     public final boolean hasSupportingIndex(IndexRegistry indexRegistry)
     {
         for (Index index : indexRegistry.listIndexes())
-           if (isSupportedBy(index))
-               return true;
-
+            if (isSupportingIndex(index))
+                return true;
         return false;
     }
 
-    @Override
-    public final Index findSupportingIndex(IndexRegistry indexRegistry)
-    {
-        for (Index index : indexRegistry.listIndexes())
-            if (isSupportedBy(index))
-                return index;
-        return null;
-    }
-
-    @Override
-    public Index findSupportingIndexFromQueryPlan(Index.QueryPlan indexQueryPlan)
-    {
-        for (Index index : indexQueryPlan.getIndexes())
-            if (isSupportedBy(index))
-                return index;
-        return null;
-    }
-
     @Override
     public boolean needsFiltering(Index.Group indexGroup)
     {
         for (ColumnMetadata column : columnDefs)
-        {
             if (!isSupportedBy(indexGroup, column))
                 return true;
-        }
+
         return false;
     }
 
     private boolean isSupportedBy(Index.Group indexGroup, ColumnMetadata column)
     {
         for (Index index : indexGroup.getIndexes())
-        {
             if (isSupportedBy(index, column))
                 return true;
-        }
+
         return false;
     }
 
@@ -178,13 +160,12 @@ private boolean isSupportedBy(Index.Group indexGroup, ColumnMetadata column)
      * @return <code>true</code> this type of restriction is supported by the specified index,
      * <code>false</code> otherwise.
      */
-    private boolean isSupportedBy(Index index)
+    private boolean isSupportingIndex(Index index)
     {
         for (ColumnMetadata column : columnDefs)
-        {
             if (isSupportedBy(index, column))
                 return true;
-        }
+
         return false;
     }
 
@@ -192,12 +173,12 @@ private boolean isSupportedBy(Index index)
 
     public static class EQRestriction extends MultiColumnRestriction
     {
-        protected final Term value;
+        protected final Term term;
 
-        public EQRestriction(List<ColumnMetadata> columnDefs, Term value)
+        public EQRestriction(List<ColumnMetadata> columnDefs, Term term)
         {
             super(columnDefs);
-            this.value = value;
+            this.term = term;
         }
 
         @Override
@@ -209,22 +190,34 @@ public boolean isEQ()
         @Override
         public void addFunctionsTo(List<Function> functions)
         {
-            value.addFunctionsTo(functions);
+            term.addFunctionsTo(functions);
         }
 
         @Override
         public String toString()
         {
-            return String.format("EQ(%s)", value);
+            return String.format("EQ(%s)", term);
         }
 
         @Override
         public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
         {
+            if (otherRestriction instanceof SliceRestriction)
+            {
+                SingleRestriction thisAsSlice = this.toSliceRestriction();
+                return thisAsSlice.mergeWith(otherRestriction);
+            }
             throw invalidRequest("%s cannot be restricted by more than one relation if it includes an Equal",
                                  getColumnsInCommons(otherRestriction));
         }
 
+        private SingleRestriction toSliceRestriction()
+        {
+            SliceRestriction start = SliceRestriction.fromBound(columnDefs, Bound.START, true, this.term);
+            SliceRestriction end = SliceRestriction.fromBound(columnDefs, Bound.END, true, this.term);
+            return start.mergeWith(end);
+        }
+
         @Override
         protected boolean isSupportedBy(Index index, ColumnMetadata column)
         {
@@ -232,22 +225,26 @@ protected boolean isSupportedBy(Index index, ColumnMetadata column)
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
-            Tuples.Value t = ((Tuples.Value) value.bind(options));
+            Tuples.Value t = ((Tuples.Value) term.bind(options));
             List<ByteBuffer> values = t.getElements();
             for (int i = 0, m = values.size(); i < m; i++)
             {
-                builder.addElementToAll(values.get(i));
-                checkFalse(builder.containsNull(), "Invalid null value for column %s", columnDefs.get(i).name);
+                ColumnMetadata column = columnDefs.get(i);
+                builder.extend(MultiClusteringBuilder.ClusteringElements.point(values.get(i)), Collections.singletonList(column));
+                checkFalse(builder.containsNull(), "Invalid null value for column %s", column.name);
             }
             return builder;
         }
 
         @Override
-        public final void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+        public final void addToRowFilter(RowFilter.Builder filter,
+                                         IndexRegistry indexRegistry,
+                                         QueryOptions options,
+                                         ANNOptions annOptions)
         {
-            Tuples.Value t = ((Tuples.Value) value.bind(options));
+            Tuples.Value t = ((Tuples.Value) term.bind(options));
             List<ByteBuffer> values = t.getElements();
 
             for (int i = 0, m = columnDefs.size(); i < m; i++)
@@ -258,24 +255,33 @@ public final void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry,
         }
     }
 
-    public abstract static class INRestriction extends MultiColumnRestriction
+    public static class INRestriction extends MultiColumnRestriction
     {
-        public INRestriction(List<ColumnMetadata> columnDefs)
+        private final MarkerOrTerms terms;
+        private final Collection<ColumnIdentifier> columnIdentifiers;
+
+        public INRestriction(List<ColumnMetadata> columnDefs, MarkerOrTerms terms)
         {
             super(columnDefs);
+            this.terms = terms;
+            this.columnIdentifiers = ColumnMetadata.toIdentifiers(columnDefs);
         }
 
         /**
          * {@inheritDoc}
          */
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
-            List<List<ByteBuffer>> splitInValues = splitValues(options);
-            builder.addAllElementsToAll(splitInValues);
+            List<List<ByteBuffer>> values = terms.bindAndGetTuples(options, columnIdentifiers);
+            List<MultiClusteringBuilder.ClusteringElements> elements = new ArrayList<>(values.size());
+            for (List<ByteBuffer> value: values)
+                elements.add(MultiClusteringBuilder.ClusteringElements.point(value));
+
+            builder.extend(elements, columnDefs);
 
             if (builder.containsNull())
-                throw invalidRequest("Invalid null value in condition for columns: %s", ColumnMetadata.toIdentifiers(columnDefs));
+                throw invalidRequest("Invalid null value in condition for columns: %s", columnIdentifiers);
             return builder;
         }
 
@@ -299,20 +305,21 @@ protected boolean isSupportedBy(Index index, ColumnMetadata column)
         }
 
         @Override
-        public final void addToRowFilter(RowFilter filter,
+        public final void addToRowFilter(RowFilter.Builder filter,
                                          IndexRegistry indexRegistry,
-                                         QueryOptions options)
+                                         QueryOptions options,
+                                         ANNOptions annOptions)
         {
             // If the relation is of the type (c) IN ((x),(y),(z)) then it is equivalent to
             // c IN (x, y, z) and we can perform filtering
             if (getColumnDefs().size() == 1)
             {
-                List<List<ByteBuffer>> splitValues = splitValues(options);
+                List<List<ByteBuffer>> splitValues = terms.bindAndGetTuples(options, columnIdentifiers);
                 List<ByteBuffer> values = new ArrayList<>(splitValues.size());
                 for (List<ByteBuffer> splitValue : splitValues)
                     values.add(splitValue.get(0));
 
-                ByteBuffer buffer = ListSerializer.pack(values, values.size());
+                ByteBuffer buffer = CollectionSerializer.pack(values, ByteBufferAccessor.instance, values.size());
                 filter.add(getFirstColumn(), Operator.IN, buffer);
             }
             else
@@ -321,96 +328,43 @@ public final void addToRowFilter(RowFilter filter,
             }
         }
 
-        protected abstract List<List<ByteBuffer>> splitValues(QueryOptions options);
-    }
-
-    /**
-     * An IN restriction that has a set of terms for in values.
-     * For example: "SELECT ... WHERE (a, b, c) IN ((1, 2, 3), (4, 5, 6))" or "WHERE (a, b, c) IN (?, ?)"
-     */
-    public static class InRestrictionWithValues extends INRestriction
-    {
-        protected final List<Term> values;
-
-        public InRestrictionWithValues(List<ColumnMetadata> columnDefs, List<Term> values)
-        {
-            super(columnDefs);
-            this.values = values;
-        }
-
         @Override
         public void addFunctionsTo(List<Function> functions)
         {
-            Terms.addFunctions(values, functions);
+            terms.addFunctionsTo(functions);
         }
 
         @Override
         public String toString()
         {
-            return String.format("IN(%s)", values);
-        }
-
-        @Override
-        protected List<List<ByteBuffer>> splitValues(QueryOptions options)
-        {
-            List<List<ByteBuffer>> buffers = new ArrayList<>(values.size());
-            for (Term value : values)
-            {
-                Term.MultiItemTerminal term = (Term.MultiItemTerminal) value.bind(options);
-                buffers.add(term.getElements());
-            }
-            return buffers;
+            return String.format("IN(%s)", terms);
         }
     }
 
-    /**
-     * An IN restriction that uses a single marker for a set of IN values that are tuples.
-     * For example: "SELECT ... WHERE (a, b, c) IN ?"
-     */
-    public static class InRestrictionWithMarker extends INRestriction
+
+    public static class SliceRestriction extends MultiColumnRestriction
     {
-        protected final AbstractMarker marker;
+        private final TermSlice slice;
+        private final List<MarkerOrTerms> skippedValues; // values passed in NOT IN
 
-        public InRestrictionWithMarker(List<ColumnMetadata> columnDefs, AbstractMarker marker)
+        SliceRestriction(List<ColumnMetadata> columnDefs, TermSlice slice, List<MarkerOrTerms> skippedValues)
         {
             super(columnDefs);
-            this.marker = marker;
-        }
-
-        @Override
-        public void addFunctionsTo(List<Function> functions)
-        {
-        }
-
-        @Override
-        public String toString()
-        {
-            return "IN ?";
-        }
-
-        @Override
-        protected List<List<ByteBuffer>> splitValues(QueryOptions options)
-        {
-            Tuples.InMarker inMarker = (Tuples.InMarker) marker;
-            Tuples.InValue inValue = inMarker.bind(options);
-            checkNotNull(inValue, "Invalid null value for IN restriction");
-            return inValue.getSplitValues();
+            assert slice != null;
+            assert skippedValues != null;
+            this.slice = slice;
+            this.skippedValues = skippedValues;
         }
-    }
 
-    public static class SliceRestriction extends MultiColumnRestriction
-    {
-        private final TermSlice slice;
-
-        public SliceRestriction(List<ColumnMetadata> columnDefs, Bound bound, boolean inclusive, Term term)
+        public static MultiColumnRestriction.SliceRestriction fromBound(List<ColumnMetadata> columnDefs, Bound bound, boolean inclusive, Term term)
         {
-            this(columnDefs, TermSlice.newInstance(bound, inclusive, term));
+            TermSlice slice = TermSlice.newInstance(bound, inclusive, term);
+            return new MultiColumnRestriction.SliceRestriction(columnDefs, slice, Collections.emptyList());
         }
 
-        SliceRestriction(List<ColumnMetadata> columnDefs, TermSlice slice)
+        public static MultiColumnRestriction.SliceRestriction fromSkippedValues(List<ColumnMetadata> columnDefs, MarkerOrTerms skippedValues)
         {
-            super(columnDefs);
-            this.slice = slice;
+            return new MultiColumnRestriction.SliceRestriction(columnDefs, TermSlice.UNBOUNDED, Collections.singletonList(skippedValues));
         }
 
         @Override
@@ -420,22 +374,55 @@ public boolean isSlice()
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
             throw new UnsupportedOperationException();
         }
 
         @Override
-        public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options)
+        public MultiClusteringBuilder appendBoundTo(MultiClusteringBuilder builder, Bound bound, QueryOptions options)
         {
+            List<MultiClusteringBuilder.ClusteringElements> toAdd = new ArrayList<>();
+            addSliceBounds(bound, options, toAdd);
+            addSkippedValues(bound, options, toAdd);
+            return builder.extend(toAdd, columnDefs);
+        }
+
+        /**
+         * Generates a list of clustering bounds based on this slice bounds and adds them to the toAdd list.
+         * Clustering bounds used for the table range scan might not be equal to this slice bounds.
+         * This method has to generate the TOP/BOTTOM bounds if this slice is unbounded on any side.
+         * It is also possible to generate multiple bounds, if clustering columns have mixed order.
+         * Does not guarantee order of results, but does not generate duplicates.
+         *
+         * @param bound the type of bounds to generate (start or end)
+         * @param options needed to get the actual values bound to markers
+         * @param toAdd receiver of the result
+         */
+        private void addSliceBounds(Bound bound, QueryOptions options, List<MultiClusteringBuilder.ClusteringElements> toAdd)
+        {
+            // Stores the direction of sorting of the current processed column.
+            // Used to detect when the next processed column has different direction of sorting than the last one.
+            // If clustering columns are all sorted in the same direction (doesn't matter if ASC or DESC, but must be
+            // the same for all), we can just need to generate only one
             boolean reversed = getFirstColumn().isReversedType();
 
             EnumMap<Bound, List<ByteBuffer>> componentBounds = new EnumMap<>(Bound.class);
             componentBounds.put(Bound.START, componentBounds(Bound.START, options));
             componentBounds.put(Bound.END, componentBounds(Bound.END, options));
 
-            List<List<ByteBuffer>> toAdd = new ArrayList<>();
-            List<ByteBuffer> values = new ArrayList<>();
+            // We will pick a prefix of bounds from `componentBounds` into this array, either start or end bounds
+            // depending on the column clustering direction.
+            List<ByteBuffer> values = Collections.emptyList();
+
+            // Tracks whether the last bound added to `values` is inclusive.
+            // We must start from true, because if there are no bounds at all (unbounded slice),
+            // we must not restrict the clusterings added by other restrictions.
+            boolean inclusive = true;
+
+            // Number of components in the last element added to the toAdd collection.
+            // Used to avoid adding the same composite multiple times.
+            int sizeOfLastElement = -1;
 
             for (int i = 0, m = columnDefs.size(); i < m; i++)
             {
@@ -446,50 +433,90 @@ public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOpti
                 if (reversed != column.isReversedType())
                 {
                     reversed = column.isReversedType();
-                    // As we are switching direction we need to add the current composite
-                    toAdd.add(values);
-
-                    // The new bound side has no value for this component.  just stop
-                    if (!hasComponent(b, i, componentBounds))
-                        continue;
-
-                    // The other side has still some components. We need to end the slice that we have just open.
-                    if (hasComponent(b.reverse(), i, componentBounds))
-                        toAdd.add(values);
-
-                    // We need to rebuild where we are in this bound side
-                    values = new ArrayList<ByteBuffer>();
 
-                    List<ByteBuffer> vals = componentBounds.get(b);
+                    // In the following comments, assume:
+                    // c1 - previous column (== columnDefs.get(i - 1))
+                    // c2 - current column (== columnDefs.get(i))
+                    // x1 - the last bound stored in values (values == [..., x1])
+                    // x2 - the bound of c2
 
-                    int n = Math.min(i, vals.size());
-                    for (int j = 0; j < n; j++)
+                    // Only try to add the current composite if we haven't done it already, to avoid duplicates.
+                    if (values.size() > sizeOfLastElement)
                     {
-                        ByteBuffer v = checkNotNull(vals.get(j),
-                                                    "Invalid null value in condition for column %s",
-                                                    columnDefs.get(j).name);
-                        values.add(v);
+                        sizeOfLastElement = values.size();
+
+                        // note that b.reverse() matches the bound of the last component added to the `values`
+                        if (hasComponent(b.reverse(), i, componentBounds))
+                        {
+                            // (c1, c2) <= (x1, x2) ----> (c1 < x1) || (c1 = x1) && (c2 <= x2)
+                            // (c1, c2) >= (x1, x2) ----> (c1 > x1) || (c1 = x1) && (c2 >= x2)
+                            // (c1, c2) <  (x1, x2) ----> (c1 < x1) || (c1 = x1) && (c2 < x2)
+                            // (c1, c2) >  (x1, x2) ----> (c1 > x1) || (c1 = x1) && (c2 > x2)
+                            //                            ^^^^^^^^^
+                            toAdd.add(MultiClusteringBuilder.ClusteringElements.bound(values, bound, false));
+
+                            // Now add the other side of the union:
+                            // (c1, c2) <= (x1, x2) ----> (c1 < x1) || (c1 = x1) && (c2 < x2)
+                            // (c1, c2) >= (x1, x2) ----> (c1 > x1) || (c1 = x1) && (c2 > x2)
+                            //                                         ^^^^^^^^^
+                            // The other side has still some components. We need to end the slice that we have just open.
+                            // Note that (c2 > x2) will be added by the call to an opposite bound.
+                            toAdd.add(MultiClusteringBuilder.ClusteringElements.point(values));
+                        }
+                        else
+                        {
+                            // The new bound side has no value for this component. Just add current composite as-is.
+                            // No value means min or max, depending on the direction of the comparison.
+                            // (c1, c2) <= (x1, no value) ----> (c1 <= x1)
+                            // (c1, c2) >= (x1, no value) ----> (c1 >= x1)
+                            // (c1, c2) <  (x1, no value) ----> (c1 < x1)
+                            // (c1, c2) >  (x1, no value) ----> (c1 > x1)
+                            toAdd.add(MultiClusteringBuilder.ClusteringElements.bound(values, bound, inclusive));
+                        }
                     }
                 }
 
-                if (!hasComponent(b, i, componentBounds))
-                    continue;
-
-                ByteBuffer v = checkNotNull(componentBounds.get(b).get(i), "Invalid null value in condition for column %s", columnDefs.get(i).name);
-                values.add(v);
+                if (hasComponent(b, i, componentBounds))
+                {
+                    values = componentBounds.get(b).subList(0, i + 1);
+                    inclusive = isInclusive(b);
+                }
             }
-            toAdd.add(values);
 
-            if (bound.isEnd())
-                Collections.reverse(toAdd);
+            if (values.size() > sizeOfLastElement)
+                toAdd.add(MultiClusteringBuilder.ClusteringElements.bound(values, bound, inclusive));
+        }
+
+
 
-            return builder.addAllElementsToAll(toAdd);
+        /**
+         * Generates a list of clustering bounds that exclude the skipped values.
+         * I.e. for skipped elements (s1, s2, ..., sN), generates the slices
+         * (BOTTOM, s1), (s1, s2), ..., (s(N-1), sN), (sN, TOP) and returns the list of
+         * their start or end bounds depending on the selected `bound` param.
+         *
+         * @param bound which bound of the slices we want to generate
+         * @param options needed to get the actual values bound to markers
+         * @param toAdd receiver of the result
+         */
+        private void addSkippedValues(Bound bound, QueryOptions options, List<MultiClusteringBuilder.ClusteringElements> toAdd)
+        {
+            for (MarkerOrTerms markerOrTerms : skippedValues)
+            {
+                for (List<ByteBuffer> tuple: markerOrTerms.bindAndGetTuples(options, ColumnMetadata.toIdentifiers(columnDefs)))
+                {
+                    MultiClusteringBuilder.ClusteringElements element = MultiClusteringBuilder.ClusteringElements.bound(tuple, bound, false);
+                    toAdd.add(element);
+                }
+            }
         }
 
         @Override
         protected boolean isSupportedBy(Index index, ColumnMetadata column)
         {
-            return slice.isSupportedBy(column, index);
+            boolean supportsSlice = slice.isSupportedBy(column, index);
+            boolean supportsNeq = index.supportsExpression(column, Operator.NEQ);
+            return supportsSlice || !skippedValues.isEmpty() && supportsNeq;
         }
 
         @Override
@@ -536,13 +563,19 @@ public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
             SliceRestriction otherSlice = (SliceRestriction) otherRestriction;
             List<ColumnMetadata> newColumnDefs = columnDefs.size() >= otherSlice.columnDefs.size() ? columnDefs : otherSlice.columnDefs;
 
-            return new SliceRestriction(newColumnDefs, slice.merge(otherSlice.slice));
+            int sizeHint = skippedValues.size() + otherSlice.skippedValues.size();
+            List<MarkerOrTerms> newSkippedValues = new ArrayList<>(sizeHint);
+            newSkippedValues.addAll(skippedValues);
+            newSkippedValues.addAll(otherSlice.skippedValues);
+            TermSlice newSlice = slice.merge(otherSlice.slice);
+            return new SliceRestriction(newColumnDefs, newSlice, newSkippedValues);
         }
 
         @Override
-        public final void addToRowFilter(RowFilter filter,
+        public final void addToRowFilter(RowFilter.Builder filter,
                                          IndexRegistry indexRegistry,
-                                         QueryOptions options)
+                                         QueryOptions options,
+                                         ANNOptions annOptions)
         {
             throw invalidRequest("Multi-column slice restrictions cannot be used for filtering.");
         }
@@ -550,7 +583,7 @@ public final void addToRowFilter(RowFilter filter,
         @Override
         public String toString()
         {
-            return "SLICE" + slice;
+            return String.format("SLICE{%s, NOT IN %s}", slice, skippedValues);
         }
 
         /**
@@ -565,14 +598,22 @@ private List<ByteBuffer> componentBounds(Bound b, QueryOptions options)
             if (!slice.hasBound(b))
                 return Collections.emptyList();
 
-            Terminal terminal = slice.bound(b).bind(options);
+            List<ByteBuffer> bounds = bindTuple(slice.bound(b).bind(options), options);
 
-            if (terminal instanceof Tuples.Value)
-            {
-                return ((Tuples.Value) terminal).getElements();
-            }
+            assert bounds.size() <= columnDefs.size();
+            int hasNullAt = bounds.indexOf(null);
+            if (hasNullAt != -1)
+                throw new InvalidRequestException(String.format(
+                    "Invalid null value in condition for column %s", columnDefs.get(hasNullAt).name));
 
-            return Collections.singletonList(terminal.get(options.getProtocolVersion()));
+            return bounds;
+        }
+
+        private static List<ByteBuffer> bindTuple(Terminal terminal, QueryOptions options)
+        {
+            return terminal instanceof Tuples.Value
+                ? ((Tuples.Value) terminal).getElements()
+                : Collections.singletonList(terminal.get(options.getProtocolVersion()));
         }
 
         private boolean hasComponent(Bound b, int index, EnumMap<Bound, List<ByteBuffer>> componentBounds)
@@ -620,13 +661,13 @@ protected boolean isSupportedBy(Index index, ColumnMetadata column)
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
             throw new UnsupportedOperationException("Cannot use IS NOT NULL restriction for slicing");
         }
 
         @Override
-        public final void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+        public final void addToRowFilter(RowFilter.Builder filter, IndexRegistry indexRegistry, QueryOptions options, ANNOptions annOptions)
         {
             throw new UnsupportedOperationException("Secondary indexes do not support IS NOT NULL restrictions");
         }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeyRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeyRestrictions.java
index 822452979ebf..3447ca6460a0 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeyRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeyRestrictions.java
@@ -20,6 +20,7 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.statements.Bound;
@@ -31,7 +32,7 @@
  */
 interface PartitionKeyRestrictions extends Restrictions
 {
-    public PartitionKeyRestrictions mergeWith(Restriction restriction);
+    public PartitionKeyRestrictions mergeWith(Restriction restriction, IndexRegistry indexRegistry);
 
     public List<ByteBuffer> values(QueryOptions options, ClientState state);
 
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
index 914681d691cc..bb9455f60b8c 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
@@ -20,20 +20,20 @@
 import java.nio.ByteBuffer;
 import java.util.*;
 
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.statements.Bound;
 import org.apache.cassandra.db.ClusteringComparator;
-import org.apache.cassandra.db.ClusteringPrefix;
-import org.apache.cassandra.db.MultiCBuilder;
+import org.apache.cassandra.db.MultiClusteringBuilder;
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 
 /**
  * A set of single restrictions on the partition key.
- * <p>This class can only contains <code>SingleRestriction</code> instances. Token restrictions will be handled by
+ * <p>This class can only contain <code>SingleRestriction</code> instances. Token restrictions will be handled by
  * <code>TokenRestriction</code> class or by the <code>TokenFilter</code> class if the query contains a mix of token
  * restrictions and single column restrictions on the partition key.
  */
@@ -44,74 +44,66 @@ final class PartitionKeySingleRestrictionSet extends RestrictionSetWrapper imple
      */
     private final ClusteringComparator comparator;
 
-    public PartitionKeySingleRestrictionSet(ClusteringComparator comparator)
+    private PartitionKeySingleRestrictionSet(RestrictionSet restrictionSet, ClusteringComparator comparator)
     {
-        super(new RestrictionSet());
+        super(restrictionSet);
         this.comparator = comparator;
     }
 
-    private PartitionKeySingleRestrictionSet(PartitionKeySingleRestrictionSet restrictionSet,
-                                       SingleRestriction restriction)
-    {
-        super(restrictionSet.restrictions.addRestriction(restriction));
-        this.comparator = restrictionSet.comparator;
-    }
-
-    private List<ByteBuffer> toByteBuffers(SortedSet<? extends ClusteringPrefix> clusterings)
-    {
-        List<ByteBuffer> l = new ArrayList<>(clusterings.size());
-        for (ClusteringPrefix clustering : clusterings)
-        {
-            // Can not use QueryProcessor.validateKey here to validate each column as that validates that empty are not allowed
-            // but composite partition keys actually allow empty!
-            clustering.validate();
-            l.add(clustering.serializeAsPartitionKey());
-        }
-        return l;
-    }
-
     @Override
-    public PartitionKeyRestrictions mergeWith(Restriction restriction)
+    public PartitionKeyRestrictions mergeWith(Restriction restriction, IndexRegistry indexRegistry)
     {
         if (restriction.isOnToken())
         {
             if (isEmpty())
                 return (PartitionKeyRestrictions) restriction;
 
-            return new TokenFilter(this, (TokenRestriction) restriction);
+            return TokenFilter.create(this, (TokenRestriction) restriction);
         }
 
-        return new PartitionKeySingleRestrictionSet(this, (SingleRestriction) restriction);
+        Builder builder = PartitionKeySingleRestrictionSet.builder(comparator);
+        List<SingleRestriction> restrictions = restrictions();
+        for (int i = 0; i < restrictions.size(); i++)
+        {
+            SingleRestriction r = restrictions.get(i);
+            builder.addRestriction(r);
+        }
+        return builder.addRestriction(restriction)
+                      .build(indexRegistry);
     }
 
     @Override
     public List<ByteBuffer> values(QueryOptions options, ClientState state)
     {
-        MultiCBuilder builder = MultiCBuilder.create(comparator, hasIN());
-        for (SingleRestriction r : restrictions)
+        MultiClusteringBuilder builder = MultiClusteringBuilder.create(comparator);
+        List<SingleRestriction> restrictions = restrictions();
+        for (int i = 0; i < restrictions.size(); i++)
         {
+            SingleRestriction r = restrictions.get(i);
             r.appendTo(builder, options);
 
             if (hasIN() && Guardrails.inSelectCartesianProduct.enabled(state))
                 Guardrails.inSelectCartesianProduct.guard(builder.buildSize(), "partition key", false, state);
 
-            if (builder.hasMissingElements())
+            if (builder.buildIsEmpty())
                 break;
         }
-        return toByteBuffers(builder.build());
+        return builder.buildSerializedPartitionKeys();
     }
 
     @Override
     public List<ByteBuffer> bounds(Bound bound, QueryOptions options)
     {
-        MultiCBuilder builder = MultiCBuilder.create(comparator, hasIN());
-        for (SingleRestriction r : restrictions)
+        MultiClusteringBuilder builder = MultiClusteringBuilder.create(comparator);
+        List<SingleRestriction> restrictions = restrictions();
+        for (int i = 0; i < restrictions.size(); i++)
         {
+            SingleRestriction r = restrictions.get(i);
             r.appendBoundTo(builder, bound, options);
-            if (builder.hasMissingElements())
+            if (builder.buildIsEmpty())
                 return Collections.emptyList();
         }
-        return toByteBuffers(builder.buildBound(bound.isStart(), true));
+        return builder.buildSerializedPartitionKeys();
     }
 
     @Override
@@ -131,13 +123,16 @@ public boolean isInclusive(Bound b)
     }
 
     @Override
-    public void addToRowFilter(RowFilter filter,
+    public void addToRowFilter(RowFilter.Builder filter,
                                IndexRegistry indexRegistry,
-                               QueryOptions options)
+                               QueryOptions options,
+                               ANNOptions annOptions)
     {
-        for (SingleRestriction restriction : restrictions)
+        List<SingleRestriction> restrictions = restrictions();
+        for (int i = 0; i < restrictions.size(); i++)
         {
-             restriction.addToRowFilter(filter, indexRegistry, options);
+            SingleRestriction r = restrictions.get(i);
+            r.addToRowFilter(filter, indexRegistry, options, annOptions);
         }
     }
 
@@ -157,9 +152,68 @@ public boolean hasUnrestrictedPartitionKeyComponents(TableMetadata table)
         return size() < table.partitionKeyColumns().size();
     }
 
-    @Override
-    public boolean hasSlice()
+    public static Builder builder(ClusteringComparator clusteringComparator)
     {
-        return restrictions.hasSlice();
+        return new Builder(clusteringComparator);
+    }
+
+    public static final class Builder
+    {
+        private final ClusteringComparator clusteringComparator;
+
+        private final List<Restriction> restrictions = new ArrayList<>();
+
+        private Builder(ClusteringComparator clusteringComparator)
+        {
+            this.clusteringComparator = clusteringComparator;
+        }
+
+        public Builder addRestriction(Restriction restriction)
+        {
+            restrictions.add(restriction);
+            return this;
+        }
+
+        public PartitionKeyRestrictions build(IndexRegistry indexRegistry)
+        {
+            return build(indexRegistry, false);
+        }
+
+        public PartitionKeyRestrictions build(IndexRegistry indexRegistry, boolean isDisjunction)
+        {
+            RestrictionSet.Builder restrictionSet = RestrictionSet.builder();
+
+            for (int i = 0; i < restrictions.size(); i++)
+            {
+                Restriction restriction = restrictions.get(i);
+
+                // restrictions on tokens are handled in a special way
+                if (restriction.isOnToken())
+                    return buildWithTokens(restrictionSet, i, indexRegistry);
+
+                restrictionSet.addRestriction((SingleRestriction) restriction, isDisjunction);
+            }
+
+            return buildPartitionKeyRestrictions(restrictionSet);
+        }
+
+        private PartitionKeyRestrictions buildWithTokens(RestrictionSet.Builder restrictionSet, int i, IndexRegistry indexRegistry)
+        {
+            PartitionKeyRestrictions merged = buildPartitionKeyRestrictions(restrictionSet);
+
+            for (; i < restrictions.size(); i++)
+            {
+                Restriction restriction = restrictions.get(i);
+
+                merged = merged.mergeWith(restriction, indexRegistry);
+            }
+
+            return merged;
+        }
+
+        private PartitionKeySingleRestrictionSet buildPartitionKeyRestrictions(RestrictionSet.Builder restrictionSet)
+        {
+            return new PartitionKeySingleRestrictionSet(restrictionSet.build(), clusteringComparator);
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java b/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
index 7a774bf0d58e..935cbfe9055e 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
@@ -19,6 +19,7 @@
 
 import java.util.List;
 
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
@@ -31,6 +32,10 @@
  */
 public interface Restriction
 {
+    /**
+     * Check if the restriction is on a partition key
+     * @return <code>true</code> if the restriction is on a partition key, <code>false</code>
+     */
     public default boolean isOnToken()
     {
         return false;
@@ -69,21 +74,6 @@ public default boolean isOnToken()
      */
     boolean hasSupportingIndex(IndexRegistry indexRegistry);
 
-    /**
-     * Find first supporting index for current restriction
-     *
-     * @param indexRegistry the index registry
-     * @return <code>index</code> if the restriction is on indexed columns, <code>null</code>
-     */
-    Index findSupportingIndex(IndexRegistry indexRegistry);
-
-    /**
-     * Find the first supporting index for the current restriction from an {@link Index.QueryPlan}.
-     * @param indexQueryPlan the index query plan
-     * @return <code>index</code> if the restriction is on indexed columns, <code>null</code>
-     */
-    Index findSupportingIndexFromQueryPlan(Index.QueryPlan indexQueryPlan);
-
     /**
      * Returns whether this restriction would need filtering if the specified index group were used.
      *
@@ -98,8 +88,10 @@ public default boolean isOnToken()
      * @param filter the row filter to add expressions to
      * @param indexRegistry the index registry
      * @param options the query options
+     * @param annOptions the query ANN options
      */
-    public void addToRowFilter(RowFilter filter,
+    public void addToRowFilter(RowFilter.Builder filter,
                                IndexRegistry indexRegistry,
-                               QueryOptions options);
+                               QueryOptions options,
+                               ANNOptions annOptions);
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
index 750b728295c8..c720c91b4519 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
@@ -17,393 +17,488 @@
  */
 package org.apache.cassandra.cql3.restrictions;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
 
-import com.google.common.collect.AbstractIterator;
-
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction.ContainsRestriction;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.db.filter.ANNOptions;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
+import org.apache.cassandra.schema.ColumnMetadata;
 
 /**
  * Sets of column restrictions.
  *
  * <p>This class is immutable.</p>
  */
-final class RestrictionSet implements Restrictions, Iterable<SingleRestriction>
+public abstract class RestrictionSet implements Restrictions
 {
     /**
      * The comparator used to sort the <code>Restriction</code>s.
      */
-    private static final Comparator<ColumnMetadata> COLUMN_DEFINITION_COMPARATOR = new Comparator<ColumnMetadata>()
+    private static final Comparator<ColumnMetadata> COLUMN_DEFINITION_COMPARATOR = Comparator.comparingInt(ColumnMetadata::position).thenComparing(column -> column.name.bytes);
+
+    private static final class EmptyRestrictionSet extends RestrictionSet
     {
+        private static final EmptyRestrictionSet INSTANCE = new EmptyRestrictionSet();
+
+        private EmptyRestrictionSet()
+        {
+        }
+
         @Override
-        public int compare(ColumnMetadata column, ColumnMetadata otherColumn)
+        public void addToRowFilter(RowFilter.Builder rowFilter, IndexRegistry indexRegistry, QueryOptions options, ANNOptions annOptions)
         {
-            int value = Integer.compare(column.position(), otherColumn.position());
-            return value != 0 ? value : column.name.bytes.compareTo(otherColumn.name.bytes);
+            // nothing to do here, since there are no restrictions
         }
-    };
 
-    private static final TreeMap<ColumnMetadata, SingleRestriction> EMPTY = new TreeMap<>(COLUMN_DEFINITION_COMPARATOR);
+        @Override
+        public List<ColumnMetadata> getColumnDefs()
+        {
+            return Collections.EMPTY_LIST;
+        }
 
-    /**
-     * The restrictions per column.
-     */
-    private final TreeMap<ColumnMetadata, SingleRestriction> restrictions;
+        @Override
+        public void addFunctionsTo(List<Function> functions)
+        {
+        }
 
-    /**
-     * {@code true} if it contains multi-column restrictions, {@code false} otherwise.
-     */
-    private final boolean hasMultiColumnRestrictions;
+        @Override
+        public boolean isEmpty()
+        {
+            return true;
+        }
 
-    private final boolean hasIn;
-    private final boolean hasContains;
-    private final boolean hasSlice;
-    private final boolean hasAnn;
-    private final boolean hasOnlyEqualityRestrictions;
+        @Override
+        public int size()
+        {
+            return 0;
+        }
 
-    public RestrictionSet()
-    {
-        this(EMPTY, false,
-             false,
-             false,
-             false,
-             false,
-             true);
-    }
+        @Override
+        public boolean hasRestrictionFor(ColumnMetadata.Kind kind)
+        {
+            return false;
+        }
 
-    private RestrictionSet(TreeMap<ColumnMetadata, SingleRestriction> restrictions,
-                           boolean hasMultiColumnRestrictions,
-                           boolean hasIn,
-                           boolean hasContains,
-                           boolean hasSlice,
-                           boolean hasAnn,
-                           boolean hasOnlyEqualityRestrictions)
-    {
-        this.restrictions = restrictions;
-        this.hasMultiColumnRestrictions = hasMultiColumnRestrictions;
-        this.hasIn = hasIn;
-        this.hasContains = hasContains;
-        this.hasSlice = hasSlice;
-        this.hasAnn = hasAnn;
-        this.hasOnlyEqualityRestrictions = hasOnlyEqualityRestrictions;
-    }
+        @Override
+        public Set<Restriction> getRestrictions(ColumnMetadata columnDef)
+        {
+            return Collections.emptySet();
+        }
 
-    @Override
-    public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options) throws InvalidRequestException
-    {
-        for (Restriction restriction : restrictions.values())
-            restriction.addToRowFilter(filter, indexRegistry, options);
-    }
+        @Override
+        public boolean hasSupportingIndex(IndexRegistry indexRegistry)
+        {
+            return false;
+        }
 
-    @Override
-    public boolean needsFiltering(Index.Group indexGroup)
-    {
-        for (SingleRestriction restriction : restrictions.values())
+        @Override
+        public boolean needsFiltering(Index.Group indexGroup)
         {
-            if (restriction.needsFiltering(indexGroup))
-                return true;
+            return false;
         }
-        return false;
-    }
 
-    @Override
-    public List<ColumnMetadata> getColumnDefs()
-    {
-        return new ArrayList<>(restrictions.keySet());
-    }
+        @Override
+        public ColumnMetadata getFirstColumn()
+        {
+            return null;
+        }
 
-    /**
-     * @return a direct reference to the key set from {@link #restrictions} with no defenseive copying
-     */
-    @Override
-    public Collection<ColumnMetadata> getColumnDefinitions()
-    {
-        return restrictions.keySet();
-    }
+        @Override
+        public ColumnMetadata getLastColumn()
+        {
+            return null;
+        }
 
-    @Override
-    public void addFunctionsTo(List<Function> functions)
-    {
-        for (Restriction restriction : this)
-            restriction.addFunctionsTo(functions);
-    }
+        @Override
+        public SingleRestriction lastRestriction()
+        {
+            return null;
+        }
 
-    @Override
-    public boolean isEmpty()
-    {
-        return restrictions.isEmpty();
-    }
+        @Override
+        public boolean hasMultipleContains()
+        {
+            return false;
+        }
 
-    @Override
-    public int size()
-    {
-        return restrictions.size();
-    }
+        @Override
+        public List<SingleRestriction> restrictions()
+        {
+            return Collections.EMPTY_LIST;
+        }
 
-    /**
-     * Checks if one of the restrictions applies to a column of the specific kind.
-     * @param kind the column kind
-     * @return {@code true} if one of the restrictions applies to a column of the specific kind, {@code false} otherwise.
-     */
-    public boolean hasRestrictionFor(ColumnMetadata.Kind kind)
-    {
-        for (ColumnMetadata column : restrictions.keySet())
+        @Override
+        public boolean hasMultiColumnSlice()
         {
-            if (column.kind == kind)
-                return true;
+            return false;
         }
-        return false;
     }
 
-    /**
-     * Adds the specified restriction to this set of restrictions.
-     *
-     * @param restriction the restriction to add
-     * @return the new set of restrictions
-     */
-    public RestrictionSet addRestriction(SingleRestriction restriction)
+    private static final class DefaultRestrictionSet extends RestrictionSet
     {
-        // RestrictionSet is immutable so we need to clone the restrictions map.
-        TreeMap<ColumnMetadata, SingleRestriction> newRestrictions = new TreeMap<>(this.restrictions);
-
-        boolean newHasIn = hasIn || restriction.isIN();
-        boolean newHasContains = hasContains || restriction.isContains();
-        boolean newHasSlice = hasSlice || restriction.isSlice();
-        boolean newHasAnn = hasAnn || restriction.isANN();
-        boolean newHasOnlyEqualityRestrictions = hasOnlyEqualityRestrictions && (restriction.isEQ() || restriction.isIN());
-
-        return new RestrictionSet(mergeRestrictions(newRestrictions, restriction),
-                                  hasMultiColumnRestrictions || restriction.isMultiColumn(),
-                                  newHasIn,
-                                  newHasContains,
-                                  newHasSlice,
-                                  newHasAnn,
-                                  newHasOnlyEqualityRestrictions);
-    }
 
-    private TreeMap<ColumnMetadata, SingleRestriction> mergeRestrictions(TreeMap<ColumnMetadata, SingleRestriction> restrictions,
-                                                                         SingleRestriction restriction)
-    {
-        Collection<ColumnMetadata> columnDefs = restriction.getColumnDefs();
-        Set<SingleRestriction> existingRestrictions = getRestrictions(columnDefs);
+        /**
+         * The keys from the 'restrictions' parameter to the
+         */
+        private final List<ColumnMetadata> restrictionsKeys;
+        /**
+         * The values as returned from {@link #restrictions()}.
+         */
+        private final List<SingleRestriction> restrictionsValues;
+        private final Multimap<ColumnMetadata, SingleRestriction> restrictionsMap;
+        private final int hasBitmap;
+        private final int restrictionForKindBitmap;
+        private static final int maskHasContains = 1;
+        private static final int maskHasSlice = 2;
+        private static final int maskHasIN = 4;
+        private static final int maskHasOnlyEqualityRestrictions = 8;
+        private static final int maskHasMultiColumnSlice = 16;
+        private static final int maskHasMultipleContains = 32;
+
+        private DefaultRestrictionSet(Multimap<ColumnMetadata, SingleRestriction> restrictions,
+                                      boolean hasMultiColumnRestrictions)
+        {
+            this.restrictionsKeys = new ArrayList<>(restrictions.keySet());
+            restrictionsKeys.sort(COLUMN_DEFINITION_COMPARATOR);
+
+            List<SingleRestriction> sortedRestrictions = new ArrayList<>();
+
+            int numberOfContains = 0;
+            int restrictionForBitmap = 0;
+            int bitmap = maskHasOnlyEqualityRestrictions;
+
+            SingleRestriction previous = null;
+            for (int i = 0; i < restrictionsKeys.size(); i++)
+            {
+                ColumnMetadata col = restrictionsKeys.get(i);
+                Collection<SingleRestriction> columnRestrictions = restrictions.get(col);
+
+                for (SingleRestriction singleRestriction : columnRestrictions)
+                {
+                    if (singleRestriction.isContains())
+                    {
+                        bitmap |= maskHasContains;
+                        ContainsRestriction contains = (ContainsRestriction) singleRestriction;
+                        numberOfContains += (contains.numberOfValues() + contains.numberOfKeys() + contains.numberOfEntries());
+                    }
+
+                    if (hasMultiColumnRestrictions)
+                    {
+                        if (singleRestriction.equals(previous))
+                            continue;
+                        previous = singleRestriction;
+                    }
+
+                    restrictionForBitmap |= 1 << col.kind.ordinal();
+
+                    sortedRestrictions.add(singleRestriction);
+
+                    if (singleRestriction.isSlice())
+                    {
+                        bitmap |= maskHasSlice;
+                        if (singleRestriction.isMultiColumn())
+                            bitmap |= maskHasMultiColumnSlice;
+                    }
+
+                    if (singleRestriction.isIN())
+                        bitmap |= maskHasIN;
+                    else if (!singleRestriction.isEQ())
+                        bitmap &= ~maskHasOnlyEqualityRestrictions;
+                }
+            }
+            this.hasBitmap = bitmap | (numberOfContains > 1 ? maskHasMultipleContains : 0);
+            this.restrictionForKindBitmap = restrictionForBitmap;
+
+            this.restrictionsValues = Collections.unmodifiableList(sortedRestrictions);
+            this.restrictionsMap = restrictions;
+        }
 
-        if (existingRestrictions.isEmpty())
+        @Override
+        public void addToRowFilter(RowFilter.Builder rowFilter,
+                                   IndexRegistry indexRegistry,
+                                   QueryOptions options,
+                                   ANNOptions annOptions) throws InvalidRequestException
         {
-            for (ColumnMetadata columnDef : columnDefs)
-                restrictions.put(columnDef, restriction);
+            for (SingleRestriction restriction : restrictionsMap.values())
+                rowFilter.addAllAsConjunction(b -> restriction.addToRowFilter(b, indexRegistry, options, annOptions));
         }
-        else
+
+        @Override
+        public List<ColumnMetadata> getColumnDefs()
         {
-            for (SingleRestriction existing : existingRestrictions)
-            {
-                SingleRestriction newRestriction = mergeRestrictions(existing, restriction);
+            return restrictionsKeys;
+        }
 
-                for (ColumnMetadata columnDef : columnDefs)
-                    restrictions.put(columnDef, newRestriction);
-            }
+        @Override
+        public void addFunctionsTo(List<Function> functions)
+        {
+            for (int i = 0; i < restrictionsValues.size(); i++)
+                restrictionsValues.get(i).addFunctionsTo(functions);
         }
 
-        return restrictions;
-    }
+        @Override
+        public boolean isEmpty()
+        {
+            return false;
+        }
 
-    @Override
-    public Set<Restriction> getRestrictions(ColumnMetadata columnDef)
-    {
-        Restriction existing = restrictions.get(columnDef);
-        return existing == null ? Collections.emptySet() : Collections.singleton(existing);
-    }
+        @Override
+        public int size()
+        {
+            return restrictionsKeys.size();
+        }
 
-    /**
-     * Returns all the restrictions applied to the specified columns.
-     *
-     * @param columnDefs the column definitions
-     * @return all the restrictions applied to the specified columns
-     */
-    private Set<SingleRestriction> getRestrictions(Collection<ColumnMetadata> columnDefs)
-    {
-        Set<SingleRestriction> set = new HashSet<>();
-        for (ColumnMetadata columnDef : columnDefs)
+        @Override
+        public boolean hasRestrictionFor(ColumnMetadata.Kind kind)
         {
-            SingleRestriction existing = restrictions.get(columnDef);
-            if (existing != null)
-                set.add(existing);
+            return 0 != (restrictionForKindBitmap & 1 << kind.ordinal());
         }
-        return set;
-    }
 
-    @Override
-    public boolean hasSupportingIndex(IndexRegistry indexRegistry)
-    {
-        for (Restriction restriction : restrictions.values())
+        @Override
+        public Set<Restriction> getRestrictions(ColumnMetadata columnDef)
         {
-            if (restriction.hasSupportingIndex(indexRegistry))
-                return true;
+            return restrictionsMap.get(columnDef).stream().map(r -> ((Restriction)r)).collect(Collectors.toSet());
         }
-        return false;
-    }
 
-    @Override
-    public Index findSupportingIndex(IndexRegistry indexRegistry)
-    {
-        for (SingleRestriction restriction : restrictions.values())
+        @Override
+        public boolean hasSupportingIndex(IndexRegistry indexRegistry)
         {
-            Index index = restriction.findSupportingIndex(indexRegistry);
-            if (index != null)
-                return index;
+            for (SingleRestriction restriction : restrictionsMap.values())
+                if (restriction.hasSupportingIndex(indexRegistry))
+                    return true;
+            return false;
         }
-        return null;
-    }
 
-    @Override
-    public Index findSupportingIndexFromQueryPlan(Index.QueryPlan indexQueryPlan)
-    {
-        for (SingleRestriction restriction : restrictions.values())
+        @Override
+        public boolean needsFiltering(Index.Group indexGroup)
         {
-            Index index = restriction.findSupportingIndexFromQueryPlan(indexQueryPlan);
-            if (index != null)
-                return index;
+            for (SingleRestriction restriction : restrictionsMap.values())
+                if (restriction.needsFiltering(indexGroup))
+                    return true;
+
+            return false;
         }
-        return null;
-    }
 
-    /**
-     * Returns the column after the specified one.
-     *
-     * @param columnDef the column for which the next one need to be found
-     * @return the column after the specified one.
-     */
-    ColumnMetadata nextColumn(ColumnMetadata columnDef)
-    {
-        return restrictions.tailMap(columnDef, false).firstKey();
-    }
+        @Override
+        public ColumnMetadata getFirstColumn()
+        {
+            return this.restrictionsKeys.get(0);
+        }
 
-    @Override
-    public ColumnMetadata getFirstColumn()
-    {
-        return isEmpty() ? null : this.restrictions.firstKey();
-    }
+        @Override
+        public ColumnMetadata getLastColumn()
+        {
+            return this.restrictionsKeys.get(this.restrictionsKeys.size() - 1);
+        }
 
-    @Override
-    public ColumnMetadata getLastColumn()
-    {
-        return isEmpty() ? null : this.restrictions.lastKey();
+        @Override
+        public SingleRestriction lastRestriction()
+        {
+            return this.restrictionsValues.get(this.restrictionsValues.size() - 1);
+        }
+
+        @Override
+        public boolean hasMultipleContains()
+        {
+            return 0 != (hasBitmap & maskHasMultipleContains);
+        }
+
+        @Override
+        public List<SingleRestriction> restrictions()
+        {
+            return restrictionsValues;
+        }
+
+        @Override
+        public boolean hasIN()
+        {
+            return 0 != (hasBitmap & maskHasIN);
+        }
+
+        @Override
+        public boolean hasContains()
+        {
+            return 0 != (hasBitmap & maskHasContains);
+        }
+
+        @Override
+        public boolean hasSlice()
+        {
+            return 0 != (hasBitmap & maskHasSlice);
+        }
+
+        @Override
+        public boolean hasMultiColumnSlice()
+        {
+            return 0 != (hasBitmap & maskHasMultiColumnSlice);
+        }
+
+        @Override
+        public boolean hasOnlyEqualityRestrictions()
+        {
+            return 0 != (hasBitmap & maskHasOnlyEqualityRestrictions);
+        }
     }
 
+    /**
+     * Checks if one of the restrictions applies to a column of the specific kind.
+     * @param kind the column kind
+     * @return {@code true} if one of the restrictions applies to a column of the specific kind, {@code false} otherwise.
+     */
+    public abstract boolean hasRestrictionFor(ColumnMetadata.Kind kind);
+
     /**
      * Returns the last restriction.
-     *
-     * @return the last restriction.
      */
-    SingleRestriction lastRestriction()
-    {
-        return isEmpty() ? null : this.restrictions.lastEntry().getValue();
-    }
+    public abstract SingleRestriction lastRestriction();
 
     /**
-     * Merges the two specified restrictions.
+     * Checks if the restrictions contains multiple contains, contains key, or map[key] = value.
      *
-     * @param restriction the first restriction
-     * @param otherRestriction the second restriction
-     * @return the merged restriction
-     * @throws InvalidRequestException if the two restrictions cannot be merged
+     * @return <code>true</code> if the restrictions contain multiple contains, contains key, or ,
+     * map[key] = value; <code>false</code> otherwise
      */
-    private static SingleRestriction mergeRestrictions(SingleRestriction restriction,
-                                                       SingleRestriction otherRestriction)
-    {
-        return restriction == null ? otherRestriction
-                                   : restriction.mergeWith(otherRestriction);
-    }
+    public abstract boolean hasMultipleContains();
 
-    @Override
-    public Iterator<SingleRestriction> iterator()
-    {
-        Iterator<SingleRestriction> iterator = restrictions.values().iterator();
-        return hasMultiColumnRestrictions ? new DistinctIterator<>(iterator) : iterator;
-    }
+    public abstract List<SingleRestriction> restrictions();
 
     /**
-     * Checks if any of the underlying restriction is an IN.
-     * @return <code>true</code> if any of the underlying restriction is an IN, <code>false</code> otherwise
+     * Checks if the restrictions contains multiple contains, contains key, or map[key] = value.
+     *
+     * @return <code>true</code> if the restrictions contains multiple contains, contains key, or ,
+     * map[key] = value; <code>false</code> otherwise
      */
-    public final boolean hasIN()
-    {
-        return hasIn;
-    }
+    public abstract boolean hasMultiColumnSlice();
 
-    public boolean hasContains()
+    public static Builder builder()
     {
-        return hasContains;
+        return new Builder();
     }
 
-    public final boolean hasSlice()
+    public static final class Builder
     {
-        return hasSlice;
-    }
+        private final Multimap<ColumnMetadata, SingleRestriction> newRestrictions = ArrayListMultimap.create();
+        private boolean multiColumn = false;
 
-    public boolean hasAnn()
-    {
-        return hasAnn;
-    }
+        private ColumnMetadata lastRestrictionColumn;
+        private SingleRestriction lastRestriction;
 
-    /**
-     * Checks if all of the underlying restrictions are EQ or IN restrictions.
-     *
-     * @return <code>true</code> if all of the underlying restrictions are EQ or IN restrictions,
-     * <code>false</code> otherwise
-     */
-    public final boolean hasOnlyEqualityRestrictions()
-    {
-        return hasOnlyEqualityRestrictions;
-    }
+        private Builder()
+        {
+        }
 
-    /**
-     * {@code Iterator} decorator that removes duplicates in an ordered one.
-     *
-     * @param <E> the iterator element type.
-     */
-    private static final class DistinctIterator<E> extends AbstractIterator<E>
-    {
-        /**
-         * The decorated iterator.
-         */
-        private final Iterator<E> iterator;
+        public void addRestriction(SingleRestriction restriction, boolean isDisjunction)
+        {
+            List<ColumnMetadata> columnDefs = restriction.getColumnDefs();
 
-        /**
-         * The previous element.
-         */
-        private E previous;
+            if (isDisjunction)
+            {
+                // If this restriction is part of a disjunction query then we don't want
+                // to merge the restrictions, we just add the new restriction
+                addRestrictionForColumns(columnDefs, restriction, null);
+            }
+            else
+            {
+                // ANDed together restrictions against the same columns should be merged.
+                Set<SingleRestriction> existingRestrictions = getRestrictions(newRestrictions, columnDefs);
 
-        public DistinctIterator(Iterator<E> iterator)
-        {
-            this.iterator = iterator;
+                // merge the new restriction into an existing one.  note that there is only ever a single
+                // restriction (per column), UNLESS one is ORDER BY BM25 and the other is MATCH.
+                for (var existing : existingRestrictions)
+                {
+                    // shouldMerge exists for the BM25/MATCH case
+                    if (existing.shouldMerge(restriction))
+                    {
+                        var merged = existing.mergeWith(restriction);
+                        addRestrictionForColumns(merged.getColumnDefs(), merged, Set.of(existing));
+                        return;
+                    }
+                }
+
+                // no existing restrictions that we should merge the new one with, add a new one
+                addRestrictionForColumns(columnDefs, restriction, null);
+            }
         }
 
-        protected E computeNext()
+        private void addRestrictionForColumns(List<ColumnMetadata> columnDefs,
+                                              SingleRestriction restriction,
+                                              @Nullable Set<SingleRestriction> replacedRestrictions)
         {
-            while(iterator.hasNext())
+            for (int i = 0; i < columnDefs.size(); i++)
             {
-                E next = iterator.next();
-                if (!next.equals(previous))
+                ColumnMetadata column = columnDefs.get(i);
+                if (lastRestrictionColumn == null || COLUMN_DEFINITION_COMPARATOR.compare(lastRestrictionColumn, column) < 0)
                 {
-                    previous = next;
-                    return next;
+                    lastRestrictionColumn = column;
+                    lastRestriction = restriction;
                 }
+                // If the restriction is a merger of new restriction and existing restrictions then
+                // we need to remove the existing restrictions for the column before adding it
+                if (replacedRestrictions != null)
+                {
+                    for (SingleRestriction r : replacedRestrictions)
+                        newRestrictions.remove(column, r);
+                }
+
+                newRestrictions.put(column, restriction);
             }
-            return endOfData();
+
+            multiColumn |= restriction.isMultiColumn();
+        }
+
+        private static Set<SingleRestriction> getRestrictions(Multimap<ColumnMetadata, SingleRestriction> restrictions,
+                                                              List<ColumnMetadata> columnDefs)
+        {
+            Set<SingleRestriction> set = new HashSet<>();
+            for (int i = 0; i < columnDefs.size(); i++)
+            {
+                Collection<SingleRestriction> existing = restrictions.get(columnDefs.get(i));
+                if (!existing.isEmpty())
+                    set.addAll(existing);
+            }
+            return set;
+        }
+
+        public RestrictionSet build()
+        {
+            return isEmpty() ? EmptyRestrictionSet.INSTANCE : new DefaultRestrictionSet(newRestrictions, multiColumn);
+        }
+
+        public boolean isEmpty()
+        {
+            return newRestrictions.isEmpty();
+        }
+
+        public SingleRestriction lastRestriction()
+        {
+            return lastRestriction;
+        }
+
+        public ColumnMetadata nextColumn(ColumnMetadata columnDef)
+        {
+            // This method is only invoked in the statement-preparation-phase to construct an error message.
+            NavigableSet<ColumnMetadata> columns = new TreeSet<>(COLUMN_DEFINITION_COMPARATOR);
+            columns.addAll(newRestrictions.keySet());
+            return columns.tailSet(columnDef, false).first();
         }
-    }
-    
-    @Override
-    public String toString()
-    {
-        return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
index 049d287cfd5d..b417089efc5a 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
@@ -17,13 +17,10 @@
  */
 package org.apache.cassandra.cql3.restrictions;
 
-import java.util.Collection;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
-
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
@@ -42,34 +39,27 @@ class RestrictionSetWrapper implements Restrictions
      */
     protected final RestrictionSet restrictions;
 
-    public RestrictionSetWrapper(RestrictionSet restrictions)
+    RestrictionSetWrapper(RestrictionSet restrictions)
     {
         this.restrictions = restrictions;
     }
 
-    public void addToRowFilter(RowFilter filter,
+    @Override
+    public void addToRowFilter(RowFilter.Builder rowFilter,
                                IndexRegistry indexRegistry,
-                               QueryOptions options)
+                               QueryOptions options,
+                               ANNOptions annOptions)
     {
-        restrictions.addToRowFilter(filter, indexRegistry, options);
+        restrictions.addToRowFilter(rowFilter, indexRegistry, options, annOptions);
     }
 
+    @Override
     public List<ColumnMetadata> getColumnDefs()
     {
         return restrictions.getColumnDefs();
     }
 
     @Override
-    public Collection<ColumnMetadata> getColumnDefinitions()
-    {
-        return restrictions.getColumnDefinitions();
-    }
-
-    public RestrictionSet getRestrictionSet()
-    {
-        return restrictions;
-    }
-
     public void addFunctionsTo(List<Function> functions)
     {
         restrictions.addFunctionsTo(functions);
@@ -80,26 +70,20 @@ public boolean isEmpty()
         return restrictions.isEmpty();
     }
 
-    public int size()
-    {
-        return restrictions.size();
-    }
-
-    public boolean hasSupportingIndex(IndexRegistry indexRegistry)
+    public List<SingleRestriction> restrictions()
     {
-        return restrictions.hasSupportingIndex(indexRegistry);
+        return restrictions.restrictions();
     }
 
-    @Override
-    public Index findSupportingIndex(IndexRegistry indexRegistry)
+    public int size()
     {
-        return restrictions.findSupportingIndex(indexRegistry);
+        return restrictions.size();
     }
 
     @Override
-    public Index findSupportingIndexFromQueryPlan(Index.QueryPlan indexQueryPlan)
+    public boolean hasSupportingIndex(IndexRegistry indexRegistry)
     {
-        return restrictions.findSupportingIndexFromQueryPlan(indexQueryPlan);
+        return restrictions.hasSupportingIndex(indexRegistry);
     }
 
     @Override
@@ -108,44 +92,45 @@ public boolean needsFiltering(Index.Group indexGroup)
         return restrictions.needsFiltering(indexGroup);
     }
 
+    @Override
     public ColumnMetadata getFirstColumn()
     {
         return restrictions.getFirstColumn();
     }
 
+    @Override
     public ColumnMetadata getLastColumn()
     {
         return restrictions.getLastColumn();
     }
 
+    @Override
     public boolean hasIN()
     {
         return restrictions.hasIN();
     }
 
+    @Override
     public boolean hasContains()
     {
         return restrictions.hasContains();
     }
 
+    @Override
     public boolean hasSlice()
     {
         return restrictions.hasSlice();
     }
 
+    @Override
     public boolean hasOnlyEqualityRestrictions()
     {
         return restrictions.hasOnlyEqualityRestrictions();
     }
 
+    @Override
     public Set<Restriction> getRestrictions(ColumnMetadata columnDef)
     {
         return restrictions.getRestrictions(columnDef);
     }
-    
-    @Override
-    public String toString()
-    {
-        return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
-    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
index 0ad7530e7698..13ca1413e9ff 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
@@ -62,18 +62,28 @@ default Collection<ColumnMetadata> getColumnDefinitions()
      * Checks if any of the underlying restriction is an IN.
      * @return <code>true</code> if any of the underlying restriction is an IN, <code>false</code> otherwise
      */
-    public boolean hasIN();
+    default public boolean hasIN()
+    {
+        return false;
+    }
 
     /**
      * Checks if any of the underlying restrictions is a CONTAINS / CONTAINS KEY restriction.
      * @return <code>true</code> if any of the underlying restrictions is CONTAINS, <code>false</code> otherwise
      */
-    public boolean hasContains();
+    default public boolean hasContains()
+    {
+        return false;
+    }
+
     /**
      * Checks if any of the underlying restrictions is a slice.
      * @return <code>true</code> if any of the underlying restrictions is a slice, <code>false</code> otherwise
      */
-    public boolean hasSlice();
+    default public boolean hasSlice()
+    {
+        return false;
+    }
 
     /**
      * Checks if all of the underlying restrictions are EQ or IN restrictions.
@@ -81,5 +91,8 @@ default Collection<ColumnMetadata> getColumnDefinitions()
      * @return <code>true</code> if all of the underlying restrictions are EQ or IN restrictions,
      * <code>false</code> otherwise
      */
-    public boolean hasOnlyEqualityRestrictions();
+    default public boolean hasOnlyEqualityRestrictions()
+    {
+        return true;
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
index d499bdce3889..8aa05e1757c7 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
@@ -20,24 +20,30 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.serializers.ListSerializer;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.Term.Terminal;
+import org.apache.cassandra.cql3.MarkerOrTerms;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.Terms;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.statements.Bound;
-import org.apache.cassandra.db.MultiCBuilder;
-import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.MultiClusteringBuilder;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Pair;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkBindValueSet;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
-import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
 import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
 
@@ -74,14 +80,9 @@ public ColumnMetadata getLastColumn()
     @Override
     public boolean hasSupportingIndex(IndexRegistry indexRegistry)
     {
-        for (Index index : indexRegistry.listIndexes())
-            if (isSupportedBy(index))
-                return true;
-
-        return false;
+        return findSupportingIndex(indexRegistry) != null;
     }
 
-    @Override
     public Index findSupportingIndex(IndexRegistry indexRegistry)
     {
         for (Index index : indexRegistry.listIndexes())
@@ -91,16 +92,6 @@ public Index findSupportingIndex(IndexRegistry indexRegistry)
         return null;
     }
 
-    @Override
-    public Index findSupportingIndexFromQueryPlan(Index.QueryPlan indexQueryPlan)
-    {
-        for (Index index : indexQueryPlan.getIndexes())
-            if (isSupportedBy(index))
-                return index;
-
-        return null;
-    }
-
     @Override
     public boolean needsFiltering(Index.Group indexGroup)
     {
@@ -154,18 +145,20 @@ boolean canBeConvertedToMultiColumnRestriction()
 
     public static final class EQRestriction extends SingleColumnRestriction
     {
-        private final Term value;
+        public static final String CANNOT_BE_MERGED_ERROR = "%s cannot be restricted by more than one relation if it includes an Equal";
+
+        private final Term term;
 
-        public EQRestriction(ColumnMetadata columnDef, Term value)
+        public EQRestriction(ColumnMetadata columnDef, Term term)
         {
             super(columnDef);
-            this.value = value;
+            this.term = term;
         }
 
         @Override
         public void addFunctionsTo(List<Function> functions)
         {
-            value.addFunctionsTo(functions);
+            term.addFunctionsTo(functions);
         }
 
         @Override
@@ -177,21 +170,23 @@ public boolean isEQ()
         @Override
         MultiColumnRestriction toMultiColumnRestriction()
         {
-            return new MultiColumnRestriction.EQRestriction(Collections.singletonList(columnDef), value);
+            return new MultiColumnRestriction.EQRestriction(Collections.singletonList(columnDef), term);
         }
 
         @Override
-        public void addToRowFilter(RowFilter filter,
+        public void addToRowFilter(RowFilter.Builder filter,
                                    IndexRegistry indexRegistry,
-                                   QueryOptions options)
+                                   QueryOptions options,
+                                   ANNOptions annOptions)
         {
-            filter.add(columnDef, Operator.EQ, value.bindAndGet(options));
+            filter.add(columnDef, Operator.EQ, term.bindAndGet(options));
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
-            builder.addElementToAll(value.bindAndGet(options));
+            List<MultiClusteringBuilder.ClusteringElements> element = Collections.singletonList(MultiClusteringBuilder.ClusteringElements.point(term.bindAndGet(options)));
+            builder.extend(element, getColumnDefs());
             checkFalse(builder.containsNull(), "Invalid null value in condition for column %s", columnDef.name);
             checkFalse(builder.containsUnset(), "Invalid unset value for column %s", columnDef.name);
             return builder;
@@ -200,13 +195,13 @@ public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
         @Override
         public String toString()
         {
-            return String.format("EQ(%s)", value);
+            return String.format("EQ(%s)", term);
         }
 
         @Override
         public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
         {
-            throw invalidRequest("%s cannot be restricted by more than one relation if it includes an Equal", columnDef.name);
+            throw invalidRequest(CANNOT_BE_MERGED_ERROR, columnDef.name);
         }
 
         @Override
@@ -216,11 +211,14 @@ protected boolean isSupportedBy(Index index)
         }
     }
 
-    public static abstract class INRestriction extends SingleColumnRestriction
+    public static class INRestriction extends SingleColumnRestriction
     {
-        public INRestriction(ColumnMetadata columnDef)
+        protected final MarkerOrTerms terms;
+
+        public INRestriction(ColumnMetadata columnDef, MarkerOrTerms terms)
         {
             super(columnDef);
+            this.terms = terms;
         }
 
         @Override
@@ -236,117 +234,82 @@ public final SingleRestriction doMergeWith(SingleRestriction otherRestriction)
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        MultiColumnRestriction toMultiColumnRestriction()
+        {
+            throw new UnsupportedOperationException("Cannot convert to multicolumn restriction");
+        }
+
+        @Override
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
-            builder.addEachElementToAll(getValues(options));
+            List<ByteBuffer> values = this.terms.bindAndGet(options, columnDef.name);
+            List<MultiClusteringBuilder.ClusteringElements> elements = new ArrayList<>(values.size());
+            for (ByteBuffer value: values)
+                elements.add(MultiClusteringBuilder.ClusteringElements.point(value));
+            builder.extend(elements, getColumnDefs());
             checkFalse(builder.containsNull(), "Invalid null value in condition for column %s", columnDef.name);
             checkFalse(builder.containsUnset(), "Invalid unset value for column %s", columnDef.name);
             return builder;
         }
 
         @Override
-        public void addToRowFilter(RowFilter filter,
+        public void addToRowFilter(RowFilter.Builder filter,
                                    IndexRegistry indexRegistry,
-                                   QueryOptions options)
+                                   QueryOptions options,
+                                   ANNOptions annOptions)
         {
-            List<ByteBuffer> values = getValues(options);
+            List<ByteBuffer> values = this.terms.bindAndGet(options, columnDef.name);
+            for (ByteBuffer v : values)
+            {
+                checkNotNull(v, "Invalid null value for column %s", columnDef.name);
+                checkBindValueSet(v, "Invalid unset value for column %s", columnDef.name);
+            }
             ByteBuffer buffer = ListSerializer.pack(values, values.size());
             filter.add(columnDef, Operator.IN, buffer);
         }
 
-        @Override
-        protected final boolean isSupportedBy(Index index)
-        {
-            return index.supportsExpression(columnDef, Operator.IN);
-        }
-
-        protected abstract List<ByteBuffer> getValues(QueryOptions options);
-    }
-
-    public static class InRestrictionWithValues extends INRestriction
-    {
-        protected final List<Term> values;
-
-        public InRestrictionWithValues(ColumnMetadata columnDef, List<Term> values)
-        {
-            super(columnDef);
-            this.values = values;
-        }
-
-        @Override
-        MultiColumnRestriction toMultiColumnRestriction()
-        {
-            return new MultiColumnRestriction.InRestrictionWithValues(Collections.singletonList(columnDef), values);
-        }
-
         @Override
         public void addFunctionsTo(List<Function> functions)
         {
-            Terms.addFunctions(values, functions);
+            terms.addFunctionsTo(functions);
         }
 
         @Override
-        protected List<ByteBuffer> getValues(QueryOptions options)
+        public final boolean isSupportedBy(Index index)
         {
-            List<ByteBuffer> buffers = new ArrayList<>(values.size());
-            for (Term value : values)
-                buffers.add(value.bindAndGet(options));
-            return buffers;
+            return index.supportsExpression(columnDef, Operator.IN);
         }
 
         @Override
         public String toString()
         {
-            return String.format("IN(%s)", values);
+            return String.format("IN(%s)", terms);
         }
     }
 
-    public static class InRestrictionWithMarker extends INRestriction
+    public static class SliceRestriction extends SingleColumnRestriction
     {
-        protected final AbstractMarker marker;
+        private final TermSlice slice;
+        private final List<MarkerOrTerms> skippedValues; // values passed in NOT IN
 
-        public InRestrictionWithMarker(ColumnMetadata columnDef, AbstractMarker marker)
+        private SliceRestriction(ColumnMetadata columnDef, TermSlice slice, List<MarkerOrTerms> skippedValues)
         {
             super(columnDef);
-            this.marker = marker;
-        }
-
-        @Override
-        public void addFunctionsTo(List<Function> functions)
-        {
-        }
-
-        @Override
-        MultiColumnRestriction toMultiColumnRestriction()
-        {
-            return new MultiColumnRestriction.InRestrictionWithMarker(Collections.singletonList(columnDef), marker);
-        }
-
-        @Override
-        protected List<ByteBuffer> getValues(QueryOptions options)
-        {
-            Terminal term = marker.bind(options);
-            checkNotNull(term, "Invalid null value for column %s", columnDef.name);
-            checkFalse(term == Constants.UNSET_VALUE, "Invalid unset value for column %s", columnDef.name);
-            Term.MultiItemTerminal lval = (Term.MultiItemTerminal) term;
-            return lval.getElements();
+            assert slice != null;
+            assert skippedValues != null;
+            this.slice = slice;
+            this.skippedValues = skippedValues;
         }
 
-        @Override
-        public String toString()
+        public static SliceRestriction fromBound(ColumnMetadata columnDef, Bound bound, boolean inclusive, Term term)
         {
-            return "IN ?";
+            TermSlice slice = TermSlice.newInstance(bound, inclusive, term);
+            return new SliceRestriction(columnDef, slice, Collections.emptyList());
         }
-    }
 
-    public static class SliceRestriction extends SingleColumnRestriction
-    {
-        private final TermSlice slice;
-
-        public SliceRestriction(ColumnMetadata columnDef, Bound bound, boolean inclusive, Term term)
+        public static SliceRestriction fromSkippedValues(ColumnMetadata columnDef, MarkerOrTerms skippedValues)
         {
-            super(columnDef);
-            slice = TermSlice.newInstance(bound, inclusive, term);
+            return new SliceRestriction(columnDef, TermSlice.UNBOUNDED, Collections.singletonList(skippedValues));
         }
 
         @Override
@@ -358,7 +321,7 @@ public void addFunctionsTo(List<Function> functions)
         @Override
         MultiColumnRestriction toMultiColumnRestriction()
         {
-            return new MultiColumnRestriction.SliceRestriction(Collections.singletonList(columnDef), slice);
+            return new MultiColumnRestriction.SliceRestriction(Collections.singletonList(columnDef), slice, skippedValues);
         }
 
         @Override
@@ -368,7 +331,7 @@ public boolean isSlice()
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
             throw new UnsupportedOperationException();
         }
@@ -380,17 +343,31 @@ public boolean hasBound(Bound b)
         }
 
         @Override
-        public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options)
+        public MultiClusteringBuilder appendBoundTo(MultiClusteringBuilder builder, Bound bound, QueryOptions options)
         {
             Bound b = bound.reverseIfNeeded(getFirstColumn());
 
-            if (!hasBound(b))
-                return builder;
-
-            ByteBuffer value = slice.bound(b).bindAndGet(options);
-            checkBindValueSet(value, "Invalid unset value for column %s", columnDef.name);
-            return builder.addElementToAll(value);
+            List<MultiClusteringBuilder.ClusteringElements> toAdd = new ArrayList<>(skippedValues.size() + 1);
+            if (hasBound(b))
+            {
+                ByteBuffer value = slice.bound(b).bindAndGet(options);
+                checkBindValueSet(value, "Invalid unset value for column %s", columnDef.name);
+                toAdd.add(MultiClusteringBuilder.ClusteringElements.bound(value, bound, slice.isInclusive(b)));
+            }
+            else
+            {
+                toAdd.add(bound.isStart() ? MultiClusteringBuilder.ClusteringElements.BOTTOM : MultiClusteringBuilder.ClusteringElements.TOP);
+            }
 
+            for (MarkerOrTerms markerOrTerms : skippedValues)
+            {
+                for (ByteBuffer value: markerOrTerms.bindAndGet(options, columnDef.name))
+                {
+                    checkBindValueSet(value, "Invalid unset value for column %s", columnDef.name);
+                    toAdd.add(MultiClusteringBuilder.ClusteringElements.bound(value, bound, false));
+                }
+            }
+            return builder.extend(toAdd, getColumnDefs());
         }
 
         @Override
@@ -414,58 +391,224 @@ public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
             checkFalse(hasBound(Bound.END) && otherSlice.hasBound(Bound.END),
                        "More than one restriction was found for the end bound on %s", columnDef.name);
 
-            return new SliceRestriction(columnDef,  slice.merge(otherSlice.slice));
+            List<MarkerOrTerms> newSkippedValues = new ArrayList<>(skippedValues.size() + otherSlice.skippedValues.size());
+            newSkippedValues.addAll(skippedValues);
+            newSkippedValues.addAll(otherSlice.skippedValues);
+            return new SliceRestriction(columnDef,  slice.merge(otherSlice.slice), newSkippedValues);
         }
 
         @Override
-        public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+        public void addToRowFilter(RowFilter.Builder filter, IndexRegistry indexRegistry, QueryOptions options, ANNOptions annOptions)
         {
             for (Bound b : Bound.values())
                 if (hasBound(b))
                     filter.add(columnDef, slice.getIndexOperator(b), slice.bound(b).bindAndGet(options));
+
+            for (MarkerOrTerms markerOrTerms : skippedValues)
+            {
+                for (ByteBuffer value : markerOrTerms.bindAndGet(options, columnDef.name))
+                   filter.add(columnDef, Operator.NEQ, value);
+            }
         }
 
         @Override
         protected boolean isSupportedBy(Index index)
         {
-            return slice.isSupportedBy(columnDef, index);
+            boolean supportsSlice = slice.isSupportedBy(columnDef, index);
+            boolean supportsNeq = index.supportsExpression(columnDef, Operator.NEQ);
+            return  supportsSlice || !skippedValues.isEmpty() && supportsNeq;
         }
 
         @Override
         public String toString()
         {
-            return String.format("SLICE%s", slice);
+            return String.format("SLICE{%s, NOT IN %s}", slice, skippedValues);
         }
 
-        private SliceRestriction(ColumnMetadata columnDef, TermSlice slice)
+    }
+
+    /**
+     * One or more slice restrictions on a column's map entries.
+     * For a map column of type map&lt;text,int&gt; with name m, here are some examples of valid restrictions:
+     * One restriction:                m['a'] > 1
+     * Restrictions on different keys: m['a'] > 1 AND m['b'] < 2
+     * Restrictions on same key:       m['a'] > 0 AND m['a'] < 2
+     */
+    public static class MapSliceRestriction extends SingleColumnRestriction
+    {
+        // Left is the map's key and right is the slice on the map's value.
+        private final List<Pair<Term, TermSlice>> slices;
+
+        public MapSliceRestriction(ColumnMetadata columnDef, Bound bound, boolean inclusive, Term key, Term value)
         {
             super(columnDef);
-            this.slice = slice;
+            slices = new ArrayList<>();
+            slices.add(Pair.create(key, TermSlice.newInstance(bound, inclusive, value)));
+        }
+
+        private MapSliceRestriction(ColumnMetadata columnDef, List<Pair<Term, TermSlice>> slices)
+        {
+            super(columnDef);
+            this.slices = slices;
+        }
+
+        @Override
+        public void addFunctionsTo(List<Function> functions)
+        {
+            slices.forEach(slice -> {
+                slice.left.addFunctionsTo(functions);
+                slice.right.addFunctionsTo(functions);
+            });
+        }
+
+        @Override
+        MultiColumnRestriction toMultiColumnRestriction()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
+        {
+            // MapSliceRestrictions are not supported on clustering columns.
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean hasBound(Bound b)
+        {
+            // Because a MapSliceRestriction can have multiple slices, we cannot implement this method.
+            throw new UnsupportedOperationException("Bounds not well defined for map slice restrictions");
+        }
+
+        @Override
+        public boolean isInclusive(Bound b)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
+        {
+            checkTrue(otherRestriction instanceof SingleColumnRestriction.MapSliceRestriction,
+                      "Column \"%s\" cannot be restricted by both an inequality relation and \"%s\"",
+                      columnDef.name, otherRestriction);
+
+            var otherMapSlice = ((SingleColumnRestriction.MapSliceRestriction) otherRestriction);
+            // Because the keys are not necessarily bound, we defer on making assertions about boundary violations
+            // until we create the row filter.
+            var newSlices = new ArrayList<>(slices);
+            newSlices.addAll(otherMapSlice.slices);
+            return new MapSliceRestriction(columnDef, newSlices);
+        }
+
+        @Override
+        public void addToRowFilter(RowFilter.Builder filter, IndexRegistry indexRegistry, QueryOptions options, ANNOptions annOptions)
+        {
+            var map = new HashMap<ByteBuffer, TermSlice>();
+            // First, we iterate through to verify that none of the slices create invalid ranges.
+            // We can only do this now because this is the point when we can bind the map's key and
+            // correctly compare them.
+            for (Pair<Term, TermSlice> pair : slices)
+            {
+                final ByteBuffer key = pair.left.bindAndGet(options);
+                final TermSlice otherSlice = pair.right();
+                map.compute(key, (k, slice) -> {
+                    if (slice == null)
+                        return otherSlice;
+
+                    // Validate that the bounds do not conflict
+                    checkFalse(slice.hasBound(Bound.START) && otherSlice.hasBound(Bound.START),
+                               "More than one restriction was found for the start bound on %s", columnDef.name);
+                    checkFalse(slice.hasBound(Bound.END) && otherSlice.hasBound(Bound.END),
+                               "More than one restriction was found for the end bound on %s", columnDef.name);
+                    return slice.merge(otherSlice);
+                });
+            }
+            // Now we can add the filters.
+            for (Map.Entry<ByteBuffer, TermSlice> entry : map.entrySet())
+            {
+                var slice = entry.getValue();
+                var start = slice.bound(Bound.START);
+                if (start != null)
+                    filter.addMapComparison(columnDef,
+                                            entry.getKey(),
+                                            slice.isInclusive(Bound.START) ? Operator.GTE : Operator.GT,
+                                            start.bindAndGet(options));
+                var end = slice.bound(Bound.END);
+                if (end != null)
+                    filter.addMapComparison(columnDef,
+                                            entry.getKey(),
+                                            slice.isInclusive(Bound.END) ? Operator.LTE : Operator.LT,
+                                            end.bindAndGet(options));
+            }
+        }
+
+        @Override
+        protected boolean isSupportedBy(Index index)
+        {
+            for (Pair<Term, TermSlice> slice : slices)
+                if (!slice.right().isSupportedBy(columnDef, index))
+                    return false;
+            return true;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("MAP_SLICE %s", slices);
         }
     }
 
-    // This holds CONTAINS, CONTAINS_KEY, and map[key] = value restrictions because we might want to have any combination of them.
+    // This holds CONTAINS, CONTAINS_KEY, NOT CONTAINS, NOT CONTAINS KEY and map[key] = value restrictions because we might want to have any combination of them.
     public static final class ContainsRestriction extends SingleColumnRestriction
     {
         private final List<Term> values = new ArrayList<>(); // for CONTAINS
+        private final List<Term> negativeValues = new ArrayList<>(); // for NOT_CONTAINS
         private final List<Term> keys = new ArrayList<>(); // for CONTAINS_KEY
+        private final List<Term> negativeKeys = new ArrayList<>(); // for NOT_CONTAINS_KEY
         private final List<Term> entryKeys = new ArrayList<>(); // for map[key] = value
         private final List<Term> entryValues = new ArrayList<>(); // for map[key] = value
+        private final List<Term> negativeEntryKeys = new ArrayList<>(); // for map[key] != value
+        private final List<Term> negativeEntryValues = new ArrayList<>(); // for map[key] != value
 
-        public ContainsRestriction(ColumnMetadata columnDef, Term t, boolean isKey)
+        public ContainsRestriction(ColumnMetadata columnDef, Term t, boolean isKey, boolean isNot)
         {
             super(columnDef);
-            if (isKey)
-                keys.add(t);
+            if (isNot)
+            {
+                if (isKey)
+                    negativeKeys.add(t);
+                else
+                    negativeValues.add(t);
+            }
             else
-                values.add(t);
+            {
+                if (isKey)
+                    keys.add(t);
+                else
+                    values.add(t);
+            }
+        }
+
+        public ContainsRestriction(ColumnMetadata columnDef, Term mapKey, Term mapValue, boolean isNot)
+        {
+            super(columnDef);
+            if (isNot)
+            {
+                negativeEntryKeys.add(mapKey);
+                negativeEntryValues.add(mapValue);
+            }
+            else
+            {
+                entryKeys.add(mapKey);
+                entryValues.add(mapValue);
+            }
         }
 
-        public ContainsRestriction(ColumnMetadata columnDef, Term mapKey, Term mapValue)
+        private ContainsRestriction(ColumnMetadata columnDef)
         {
             super(columnDef);
-            entryKeys.add(mapKey);
-            entryValues.add(mapValue);
         }
 
         @Override
@@ -481,7 +624,7 @@ boolean canBeConvertedToMultiColumnRestriction()
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
             throw new UnsupportedOperationException();
         }
@@ -500,7 +643,6 @@ public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
                       columnDef.name);
 
             SingleColumnRestriction.ContainsRestriction newContains = new ContainsRestriction(columnDef);
-
             copyKeysAndValues(this, newContains);
             copyKeysAndValues((ContainsRestriction) otherRestriction, newContains);
 
@@ -508,18 +650,29 @@ public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
         }
 
         @Override
-        public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+        public void addToRowFilter(RowFilter.Builder filter, IndexRegistry indexRegistry, QueryOptions options, ANNOptions annOptions)
         {
             for (ByteBuffer value : bindAndGet(values, options))
                 filter.add(columnDef, Operator.CONTAINS, value);
             for (ByteBuffer key : bindAndGet(keys, options))
                 filter.add(columnDef, Operator.CONTAINS_KEY, key);
+            for (ByteBuffer value : bindAndGet(negativeValues, options))
+                filter.add(columnDef, Operator.NOT_CONTAINS, value);
+            for (ByteBuffer key : bindAndGet(negativeKeys, options))
+                filter.add(columnDef, Operator.NOT_CONTAINS_KEY, key);
 
             List<ByteBuffer> eks = bindAndGet(entryKeys, options);
             List<ByteBuffer> evs = bindAndGet(entryValues, options);
             assert eks.size() == evs.size();
             for (int i = 0; i < eks.size(); i++)
-                filter.addMapEquality(columnDef, eks.get(i), Operator.EQ, evs.get(i));
+                filter.addMapComparison(columnDef, eks.get(i), Operator.EQ, evs.get(i));
+
+            List<ByteBuffer> neks = bindAndGet(negativeEntryKeys, options);
+            List<ByteBuffer> nevs = bindAndGet(negativeEntryValues, options);
+            assert neks.size() == nevs.size();
+            for (int i = 0; i < neks.size(); i++)
+                filter.addMapComparison(columnDef, neks.get(i), Operator.NEQ, nevs.get(i));
+
         }
 
         @Override
@@ -533,9 +686,18 @@ protected boolean isSupportedBy(Index index)
             if (numberOfKeys() > 0)
                 supported |= index.supportsExpression(columnDef, Operator.CONTAINS_KEY);
 
+            if (numberOfNegativeValues() > 0)
+                supported |= index.supportsExpression(columnDef, Operator.NOT_CONTAINS);
+
+            if (numberOfNegativeKeys() > 0)
+                supported |= index.supportsExpression(columnDef, Operator.NOT_CONTAINS_KEY);
+
             if (numberOfEntries() > 0)
                 supported |= index.supportsExpression(columnDef, Operator.EQ);
 
+            if (numberOfNegativeEntries() > 0)
+                supported |= index.supportsExpression(columnDef, Operator.NEQ);
+
             return supported;
         }
 
@@ -559,16 +721,31 @@ public int numberOfValues()
             return values.size();
         }
 
+        public int numberOfNegativeValues()
+        {
+            return negativeValues.size();
+        }
+
         public int numberOfKeys()
         {
             return keys.size();
         }
 
+        public int numberOfNegativeKeys()
+        {
+            return negativeKeys.size();
+        }
+
         public int numberOfEntries()
         {
             return entryKeys.size();
         }
 
+        public int numberOfNegativeEntries()
+        {
+            return negativeEntryKeys.size();
+        }
+
         @Override
         public void addFunctionsTo(List<Function> functions)
         {
@@ -576,12 +753,18 @@ public void addFunctionsTo(List<Function> functions)
             Terms.addFunctions(keys, functions);
             Terms.addFunctions(entryKeys, functions);
             Terms.addFunctions(entryValues, functions);
+
+            Terms.addFunctions(negativeValues, functions);
+            Terms.addFunctions(negativeKeys, functions);
+            Terms.addFunctions(negativeEntryKeys, functions);
+            Terms.addFunctions(negativeEntryValues, functions);
         }
 
         @Override
         public String toString()
         {
-            return String.format("CONTAINS(values=%s, keys=%s, entryKeys=%s, entryValues=%s)", values, keys, entryKeys, entryValues);
+            return String.format("CONTAINS(values=%s, keys=%s, entryKeys=%s, entryValues=%s)",
+                                 values, keys, entryKeys, entryValues);
         }
 
         @Override
@@ -591,7 +774,7 @@ public boolean hasBound(Bound b)
         }
 
         @Override
-        public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options)
+        public MultiClusteringBuilder appendBoundTo(MultiClusteringBuilder builder, Bound bound, QueryOptions options)
         {
             throw new UnsupportedOperationException();
         }
@@ -626,17 +809,18 @@ private static List<ByteBuffer> bindAndGet(List<Term> terms, QueryOptions option
         private static void copyKeysAndValues(ContainsRestriction from, ContainsRestriction to)
         {
             to.values.addAll(from.values);
+            to.negativeValues.addAll(from.negativeValues);
             to.keys.addAll(from.keys);
+            to.negativeKeys.addAll(from.negativeKeys);
             to.entryKeys.addAll(from.entryKeys);
             to.entryValues.addAll(from.entryValues);
-        }
+            to.negativeEntryKeys.addAll(from.negativeEntryKeys);
+            to.negativeEntryValues.addAll(from.negativeEntryValues);
 
-        private ContainsRestriction(ColumnMetadata columnDef)
-        {
-            super(columnDef);
         }
     }
 
+
     public static final class IsNotNullRestriction extends SingleColumnRestriction
     {
         public IsNotNullRestriction(ColumnMetadata columnDef)
@@ -662,15 +846,16 @@ MultiColumnRestriction toMultiColumnRestriction()
         }
 
         @Override
-        public void addToRowFilter(RowFilter filter,
+        public void addToRowFilter(RowFilter.Builder filter,
                                    IndexRegistry indexRegistry,
-                                   QueryOptions options)
+                                   QueryOptions options,
+                                   ANNOptions annOptions)
         {
             throw new UnsupportedOperationException("Secondary indexes do not support IS NOT NULL restrictions");
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
             throw new UnsupportedOperationException("Cannot use IS NOT NULL restriction for slicing");
         }
@@ -713,12 +898,6 @@ public void addFunctionsTo(List<Function> functions)
             value.addFunctionsTo(functions);
         }
 
-        @Override
-        public boolean isEQ()
-        {
-            return false;
-        }
-
         @Override
         public boolean isLIKE()
         {
@@ -738,9 +917,10 @@ MultiColumnRestriction toMultiColumnRestriction()
         }
 
         @Override
-        public void addToRowFilter(RowFilter filter,
+        public void addToRowFilter(RowFilter.Builder filter,
                                    IndexRegistry indexRegistry,
-                                   QueryOptions options)
+                                   QueryOptions options,
+                                   ANNOptions annOptions)
         {
             Pair<Operator, ByteBuffer> operation = makeSpecific(value.bindAndGet(options));
 
@@ -752,7 +932,7 @@ public void addToRowFilter(RowFilter filter,
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
             // LIKE can be used with clustering columns, but as it doesn't
             // represent an actual clustering value, it can't be used in a
@@ -824,14 +1004,116 @@ else if (ByteBufferUtil.startsWith(value, LIKE_WILDCARD))
             return Pair.create(operator, newValue);
         }
     }
+
+    /**
+     * For now, index based ordering is represented as a restriction.
+     */
+    public static final class OrderRestriction extends SingleColumnRestriction
+    {
+        private final SingleColumnRestriction otherRestriction;
+        private final Operator direction;
+
+        public OrderRestriction(ColumnMetadata columnDef, Operator direction)
+        {
+            this(columnDef, null, direction);
+        }
+
+        private OrderRestriction(ColumnMetadata columnDef, SingleColumnRestriction otherRestriction, Operator direction)
+        {
+            super(columnDef);
+            this.otherRestriction = otherRestriction;
+            this.direction = direction;
+
+            if (direction != Operator.ORDER_BY_ASC && direction != Operator.ORDER_BY_DESC)
+                throw new IllegalArgumentException("Ordering restriction must be ASC or DESC");
+        }
+
+        public Operator getDirection()
+        {
+            return direction;
+        }
+
+        @Override
+        public void addFunctionsTo(List<Function> functions)
+        {
+            if (otherRestriction != null)
+                otherRestriction.addFunctionsTo(functions);
+        }
+
+        @Override
+        MultiColumnRestriction toMultiColumnRestriction()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void addToRowFilter(RowFilter.Builder filter,
+                                   IndexRegistry indexRegistry,
+                                   QueryOptions options,
+                                   ANNOptions annOptions)
+        {
+            filter.add(columnDef, direction, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+            if (otherRestriction != null)
+                otherRestriction.addToRowFilter(filter, indexRegistry, options, annOptions);
+        }
+
+        @Override
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("ORDER BY %s %s", columnDef.name, direction);
+        }
+
+        @Override
+        public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
+        {
+            if (!(otherRestriction instanceof SingleColumnRestriction))
+                throw invalidRequest("%s cannot be restricted by both ORDER BY and %s",
+                                     columnDef.name,
+                                     otherRestriction.toString());
+            var otherSingleColumnRestriction = (SingleColumnRestriction) otherRestriction;
+            if (this.otherRestriction == null)
+                return new OrderRestriction(columnDef, otherSingleColumnRestriction, direction);
+            var mergedOtherRestriction = this.otherRestriction.doMergeWith(otherSingleColumnRestriction);
+            return new OrderRestriction(columnDef, (SingleColumnRestriction) mergedOtherRestriction, direction);
+        }
+
+        @Override
+        protected boolean isSupportedBy(Index index)
+        {
+            return index.supportsExpression(columnDef, direction)
+                   && (otherRestriction == null || otherRestriction.isSupportedBy(index));
+        }
+
+        @Override
+        public boolean isIndexBasedOrdering()
+        {
+            return true;
+        }
+    }
+
     public static final class AnnRestriction extends SingleColumnRestriction
     {
         private final Term value;
+        // This is the only kind of restriction that can be merged into an AnnRestriction because all Ann
+        // are on vector columns, and the only other valid restriction on vector columns is BOUNDED_ANN.
+        private final BoundedAnnRestriction boundedAnnRestriction;
 
         public AnnRestriction(ColumnMetadata columnDef, Term value)
+        {
+            this(columnDef, value, null);
+        }
+
+        private AnnRestriction(ColumnMetadata columnDef, Term value, BoundedAnnRestriction boundedAnnRestriction)
         {
             super(columnDef);
             this.value = value;
+            this.boundedAnnRestriction = boundedAnnRestriction;
         }
 
         public ByteBuffer value(QueryOptions options)
@@ -840,11 +1122,92 @@ public ByteBuffer value(QueryOptions options)
         }
 
         @Override
-        public boolean isANN()
+        public void addFunctionsTo(List<Function> functions)
+        {
+            value.addFunctionsTo(functions);
+            if (boundedAnnRestriction != null)
+                boundedAnnRestriction.addFunctionsTo(functions);
+        }
+
+        @Override
+        MultiColumnRestriction toMultiColumnRestriction()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void addToRowFilter(RowFilter.Builder filter,
+                                   IndexRegistry indexRegistry,
+                                   QueryOptions options,
+                                   ANNOptions annOptions)
+        {
+            filter.addANNExpression(columnDef, value.bindAndGet(options), annOptions);
+            if (boundedAnnRestriction != null)
+                boundedAnnRestriction.addToRowFilter(filter, indexRegistry, options, annOptions);
+        }
+
+        @Override
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("ANN(%s)", value);
+        }
+
+        @Override
+        public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
+        {
+            if (otherRestriction.isIndexBasedOrdering())
+                throw invalidRequest("%s cannot be restricted by multiple ANN restrictions", columnDef.name);
+
+            if (!otherRestriction.isBoundedAnn())
+                throw invalidRequest("%s cannot be restricted by both BOUNDED_ANN and %s", columnDef.name, otherRestriction.toString());
+
+            if (boundedAnnRestriction == null)
+                return new AnnRestriction(columnDef, value, (BoundedAnnRestriction) otherRestriction);
+
+            var mergedAnnRestriction = boundedAnnRestriction.doMergeWith(otherRestriction);
+            return new AnnRestriction(columnDef, value, (BoundedAnnRestriction) mergedAnnRestriction);
+        }
+
+        @Override
+        protected boolean isSupportedBy(Index index)
+        {
+            return index.supportsExpression(columnDef, Operator.ANN) && (boundedAnnRestriction == null || boundedAnnRestriction.isSupportedBy(index));
+        }
+
+        @Override
+        public boolean isIndexBasedOrdering()
         {
             return true;
         }
 
+        @Override
+        public boolean isBoundedAnn()
+        {
+            return boundedAnnRestriction != null;
+        }
+    }
+
+    public static final class Bm25Restriction extends SingleColumnRestriction
+    {
+        private final Term value;
+
+        public Bm25Restriction(ColumnMetadata columnDef, Term value)
+        {
+            super(columnDef);
+            this.value = value;
+        }
+
+        public ByteBuffer value(QueryOptions options)
+        {
+            return value.bindAndGet(options);
+        }
+
         @Override
         public void addFunctionsTo(List<Function> functions)
         {
@@ -858,15 +1221,101 @@ MultiColumnRestriction toMultiColumnRestriction()
         }
 
         @Override
-        public void addToRowFilter(RowFilter filter,
+        public void addToRowFilter(RowFilter.Builder filter, IndexRegistry indexRegistry, QueryOptions options, ANNOptions annOptions)
+        {
+            var index = findSupportingIndex(indexRegistry);
+            var valueBytes = value.bindAndGet(options);
+            var terms = index.getQueryAnalyzer().get().analyze(valueBytes);
+            if (terms.isEmpty())
+                throw invalidRequest("BM25 query must contain at least one term (perhaps your analyzer is discarding tokens you didn't expect)");
+            filter.add(columnDef, Operator.BM25, valueBytes);
+        }
+
+        @Override
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("BM25(%s)", value);
+        }
+
+        @Override
+        public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
+        {
+            throw invalidRequest("%s cannot be restricted by both BM25 and %s", columnDef.name, otherRestriction.toString());
+        }
+
+        @Override
+        protected boolean isSupportedBy(Index index)
+        {
+            return index.supportsExpression(columnDef, Operator.BM25);
+        }
+
+        @Override
+        public boolean isIndexBasedOrdering()
+        {
+            return true;
+        }
+
+        @Override
+        public boolean shouldMerge(SingleRestriction other)
+        {
+            // we don't want to merge MATCH restrictions with ORDER BY BM25
+            // so shouldMerge = false for that scenario, and true for others
+            // (because even though we can't meaningfully merge with others, we want doMergeWith to be called to throw)
+            //
+            // (Note that because ORDER BY is processed before WHERE, we only need this check in the BM25 class)
+            return !other.isAnalyzerMatches();
+        }
+    }
+
+    /**
+     * A Bounded ANN Restriction is one that uses a similarity score as the limiting factor for ANN instead of a number
+     * of results.
+     */
+    public static final class BoundedAnnRestriction extends SingleColumnRestriction
+    {
+        private final Term value;
+        private final Term distance;
+        private final boolean isInclusive;
+
+        public BoundedAnnRestriction(ColumnMetadata columnDef, Term value, Term distance, boolean isInclusive)
+        {
+            super(columnDef);
+            this.value = value;
+            this.distance = distance;
+            this.isInclusive = isInclusive;
+        }
+
+        @Override
+        public void addFunctionsTo(List<Function> functions)
+        {
+            value.addFunctionsTo(functions);
+            distance.addFunctionsTo(functions);
+        }
+
+        @Override
+        MultiColumnRestriction toMultiColumnRestriction()
+        {
+            // only used by partition and clustering restrictions
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void addToRowFilter(RowFilter.Builder filter,
                                    IndexRegistry indexRegistry,
-                                   QueryOptions options)
+                                   QueryOptions options,
+                                   ANNOptions annOptions)
         {
-            filter.add(columnDef, Operator.ANN, value.bindAndGet(options));
+            filter.addGeoDistanceExpression(columnDef, value.bindAndGet(options), isInclusive ? Operator.LTE : Operator.LT, distance.bindAndGet(options));
         }
 
         @Override
-        public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
         {
             throw new UnsupportedOperationException();
         }
@@ -874,19 +1323,119 @@ public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options)
         @Override
         public String toString()
         {
-            return String.format("ANN(%s)", value);
+            return String.format("BOUNDED_ANN(%s)", value);
+        }
+
+        @Override
+        public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
+        {
+            if (!otherRestriction.isBoundedAnn())
+                throw invalidRequest("%s cannot be restricted by both BOUNDED_ANN and %s", columnDef.name, otherRestriction.toString());
+            throw invalidRequest("%s cannot be restricted by multiple BOUNDED_ANN restrictions", columnDef.name, otherRestriction.toString());
+        }
+
+        @Override
+        protected boolean isSupportedBy(Index index)
+        {
+            return index.supportsExpression(columnDef, Operator.BOUNDED_ANN);
+        }
+
+        @Override
+        public boolean isBoundedAnn()
+        {
+            return true;
+        }
+    }
+
+    public static final class AnalyzerMatchesRestriction extends SingleColumnRestriction
+    {
+        public static final String CANNOT_BE_MERGED_ERROR = "%s cannot be restricted by other operators if it includes analyzer match (:)";
+        private final List<Term> values;
+
+        public AnalyzerMatchesRestriction(ColumnMetadata columnDef, Term value)
+        {
+            super(columnDef);
+            this.values = Collections.singletonList(value);
+        }
+
+        public AnalyzerMatchesRestriction(ColumnMetadata columnDef, List<Term> values)
+        {
+            super(columnDef);
+            this.values = values;
+        }
+
+        @Override
+        public boolean isAnalyzerMatches()
+        {
+            return true;
+        }
+
+        List<Term> getValues()
+        {
+            return values;
+        }
+
+        @Override
+        public void addFunctionsTo(List<Function> functions)
+        {
+            for (Term value : values)
+            {
+                value.addFunctionsTo(functions);
+            }
         }
 
+        @Override
+        MultiColumnRestriction toMultiColumnRestriction()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void addToRowFilter(RowFilter.Builder filter,
+                                   IndexRegistry indexRegistry,
+                                   QueryOptions options,
+                                   ANNOptions annOptions)
+        {
+            for (Term value : values)
+            {
+                filter.add(columnDef, Operator.ANALYZER_MATCHES, value.bindAndGet(options));
+            }
+        }
+
+        @Override
+        public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("ANALYZER_MATCHES(%s)", values);
+        }
+
+        /**
+         * Merges this restriction with another restriction. Only called for conjuctive restrictions.
+         */
         @Override
         public SingleRestriction doMergeWith(SingleRestriction otherRestriction)
         {
-            throw invalidRequest("%s cannot be restricted by more than one relation in an ANN ordering", columnDef.name);
+            if (!otherRestriction.isAnalyzerMatches())
+                throw invalidRequest(CANNOT_BE_MERGED_ERROR, columnDef.name);
+
+            List<Term> otherValues = otherRestriction instanceof AnalyzerMatchesRestriction
+                                   ? ((AnalyzerMatchesRestriction) otherRestriction).getValues()
+                                   : List.of(((EQRestriction) otherRestriction).term);
+            List<Term> newValues = new ArrayList<>(values.size() + otherValues.size());
+            newValues.addAll(values);
+            newValues.addAll(otherValues);
+            return new AnalyzerMatchesRestriction(columnDef, newValues);
         }
 
         @Override
         protected boolean isSupportedBy(Index index)
         {
-            return index.supportsExpression(columnDef, Operator.ANN);
+            return index.supportsExpression(columnDef, Operator.ANALYZER_MATCHES);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/SingleRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/SingleRestriction.java
index 8f4e2b5de4bb..0bc9e161f509 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/SingleRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/SingleRestriction.java
@@ -19,7 +19,7 @@
 
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.statements.Bound;
-import org.apache.cassandra.db.MultiCBuilder;
+import org.apache.cassandra.db.MultiClusteringBuilder;
 
 /**
  * A single restriction/clause on one or multiple column.
@@ -36,22 +36,22 @@ public default boolean isEQ()
         return false;
     }
 
-    public default boolean isLIKE()
+    public default boolean isAnalyzerMatches()
     {
         return false;
     }
 
-    public default boolean isIN()
+    public default boolean isLIKE()
     {
         return false;
     }
 
-    public default boolean isContains()
+    public default boolean isIN()
     {
         return false;
     }
 
-    public default boolean isANN()
+    public default boolean isContains()
     {
         return false;
     }
@@ -74,6 +74,16 @@ public default boolean isMultiColumn()
         return false;
     }
 
+    public default boolean isIndexBasedOrdering()
+    {
+        return false;
+    }
+
+    public default boolean isBoundedAnn()
+    {
+        return false;
+    }
+
     /**
      * Checks if the specified bound is set or not.
      * @param b the bound type
@@ -113,7 +123,7 @@ public default boolean isInclusive(Bound b)
      * @param options the query options
      * @return the <code>MultiCBuilder</code>
      */
-    public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options);
+    public MultiClusteringBuilder appendTo(MultiClusteringBuilder builder, QueryOptions options);
 
     /**
      * Appends the values of the <code>SingleRestriction</code> for the specified bound to the specified builder.
@@ -123,8 +133,20 @@ public default boolean isInclusive(Bound b)
      * @param options the query options
      * @return the <code>MultiCBuilder</code>
      */
-    public default MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options)
+    public default MultiClusteringBuilder appendBoundTo(MultiClusteringBuilder builder, Bound bound, QueryOptions options)
     {
         return appendTo(builder, options);
     }
+
+    /**
+     * @return true if the other restriction should be merged with this one.
+     * This is NOT for preventing illegal combinations of restrictions, e.g.
+     * a=1 AND a=2; that is handled by mergeWith.  Instead, this is for the case
+     * where we want two completely different semantics against the same column.
+     * Currently the only such case is BM25 with MATCH.
+     */
+    default boolean shouldMerge(SingleRestriction other)
+    {
+        return true;
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
index d5b6a2a6fd1b..a1eecbfee00c 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
@@ -18,27 +18,53 @@
 package org.apache.cassandra.cql3.restrictions;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Streams;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.Ordering;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.Relation;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.WhereClause;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.statements.Bound;
+import org.apache.cassandra.cql3.statements.SelectOptions;
 import org.apache.cassandra.cql3.statements.StatementType;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.FloatType;
-import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
 import org.apache.cassandra.db.virtual.VirtualTable;
-import org.apache.cassandra.dht.*;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.ExcludingBounds;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.IncludingExcludingBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.IntegerType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
@@ -47,9 +73,7 @@
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.btree.BTreeSet;
 
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
-
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_ENABLE_GENERAL_ORDER_BY;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
@@ -57,8 +81,10 @@
 /**
  * The restrictions corresponding to the relations specified on the where-clause of CQL query.
  */
-public final class StatementRestrictions
+public class StatementRestrictions
 {
+    public static final boolean ENABLE_SAI_GENERAL_ORDER_BY = SAI_ENABLE_GENERAL_ORDER_BY.getBoolean();
+
     private static final String ALLOW_FILTERING_MESSAGE =
             "Cannot execute this query as it might involve data filtering and thus may have unpredictable performance. ";
 
@@ -69,18 +95,37 @@ public final class StatementRestrictions
             "Executing this query despite the performance unpredictability with ALLOW FILTERING has been disabled " +
             "by the allow_filtering_enabled property in cassandra.yaml";
 
-    public static final String ANN_REQUIRES_INDEX_MESSAGE = "ANN ordering by vector requires the column to be indexed";
+    public static final String HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE =
+    "Column '%s' has an index but does not support the operators specified in the query. " +
+    "If you want to execute this query despite the performance unpredictability, use ALLOW FILTERING";
 
-    public static final String VECTOR_INDEXES_ANN_ONLY_MESSAGE = "Vector indexes only support ANN queries";
+    public static final String HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_MULTI =
+    "Columns %s have indexes but do not support the operators specified in the query. " +
+    "If you want to execute this query despite the performance unpredictability, use ALLOW FILTERING";
 
-    public static final String ANN_ONLY_SUPPORTED_ON_VECTOR_MESSAGE = "ANN ordering is only supported on float vector indexes";
+    public static final String INDEX_DOES_NOT_SUPPORT_LIKE_MESSAGE = "Index on column %s does not support LIKE restrictions.";
 
-    public static final String ANN_REQUIRES_INDEXED_FILTERING_MESSAGE = "ANN ordering by vector requires all restricted column(s) to be indexed";
+    public static final String INDEX_DOES_NOT_SUPPORT_ANALYZER_MATCHES_MESSAGE = "Index on column %s does not support ':' restrictions.";
 
-    /**
-     * The type of statement
-     */
-    private final StatementType type;
+    public static final String INDEX_DOES_NOT_SUPPORT_DISJUNCTION =
+    "An index involved in this query does not support disjunctive queries using the OR operator";
+
+    public static final String RESTRICTION_REQUIRES_INDEX_MESSAGE = "%s restriction is only supported on properly indexed columns. %s is not valid.";
+
+    public static final String PARTITION_KEY_RESTRICTION_MUST_BE_TOP_LEVEL =
+    "Restriction on partition key column %s must not be nested under OR operator";
+
+    public static final String GEO_DISTANCE_REQUIRES_INDEX_MESSAGE = "GEO_DISTANCE requires the vector column to be indexed";
+    public static final String BM25_ORDERING_REQUIRES_ANALYZED_INDEX_MESSAGE = "BM25 ordering on column %s requires an analyzed index";
+    public static final String NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE =
+    "Ordering on non-clustering column %s requires the column to be indexed with a non-analyzed index.";
+    public static final String NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE =
+    "Ordering on non-clustering column requires each restricted column to be indexed except for fully-specified partition keys";
+
+    public static final String VECTOR_INDEX_PRESENT_NOT_SUPPORT_GEO_DISTANCE_MESSAGE =
+    "Vector index present, but configuration does not support GEO_DISTANCE queries. GEO_DISTANCE requires similarity_function 'euclidean'";
+    public static final String VECTOR_INDEXES_UNSUPPORTED_OP_MESSAGE = "Vector indexes only support ANN and GEO_DISTANCE queries";
+    public static final String ANN_OPTIONS_WITHOUT_ORDER_BY_ANN = "ANN options specified without ORDER BY ... ANN OF ...";
 
     /**
      * The Column Family meta data
@@ -90,34 +135,39 @@ public final class StatementRestrictions
     /**
      * Restrictions on partitioning columns
      */
-    private PartitionKeyRestrictions partitionKeyRestrictions;
+    protected final PartitionKeyRestrictions partitionKeyRestrictions;
 
     /**
      * Restrictions on clustering columns
      */
-    private ClusteringColumnRestrictions clusteringColumnsRestrictions;
+    private final ClusteringColumnRestrictions clusteringColumnsRestrictions;
 
     /**
      * Restriction on non-primary key columns (i.e. secondary index restrictions)
      */
-    private RestrictionSet nonPrimaryKeyRestrictions;
+    private final RestrictionSet nonPrimaryKeyRestrictions;
 
-    private Set<ColumnMetadata> notNullColumns;
+    private final ImmutableSet<ColumnMetadata> notNullColumns;
 
     /**
      * The restrictions used to build the row filter
      */
-    private final IndexRestrictions filterRestrictions = new IndexRestrictions();
+    private final IndexRestrictions filterRestrictions;
+
+    /**
+     * <code>true</code> if these restrictions form part of an OR query, <code>false</code> otherwise
+     */
+    private boolean isDisjunction;
 
     /**
      * <code>true</code> if the secondary index need to be queried, <code>false</code> otherwise
      */
-    private boolean usesSecondaryIndexing;
+    protected boolean usesSecondaryIndexing;
 
     /**
      * Specify if the query will return a range of partition keys.
      */
-    private boolean isKeyRange;
+    protected boolean isKeyRange;
 
     /**
      * <code>true</code> if nonPrimaryKeyRestrictions contains restriction on a regular column,
@@ -125,238 +175,656 @@ public final class StatementRestrictions
      */
     private boolean hasRegularColumnsRestrictions;
 
+    private final List<StatementRestrictions> children;
+
     /**
      * Creates a new empty <code>StatementRestrictions</code>.
      *
-     * @param type the type of statement
      * @param table the column family meta data
      * @return a new empty <code>StatementRestrictions</code>.
      */
-    public static StatementRestrictions empty(StatementType type, TableMetadata table)
+    public static StatementRestrictions empty(TableMetadata table)
     {
-        return new StatementRestrictions(type, table, false);
+        return new StatementRestrictions(table, false);
     }
 
-    private StatementRestrictions(StatementType type, TableMetadata table, boolean allowFiltering)
+    private StatementRestrictions(TableMetadata table, boolean allowFiltering)
     {
-        this.type = type;
         this.table = table;
-        this.partitionKeyRestrictions = new PartitionKeySingleRestrictionSet(table.partitionKeyAsClusteringComparator());
-        this.clusteringColumnsRestrictions = new ClusteringColumnRestrictions(table, allowFiltering);
-        this.nonPrimaryKeyRestrictions = new RestrictionSet();
-        this.notNullColumns = new HashSet<>();
+        this.partitionKeyRestrictions = PartitionKeySingleRestrictionSet.builder(table.partitionKeyAsClusteringComparator())
+                                                                        .build(IndexRegistry.obtain(table));
+        this.clusteringColumnsRestrictions = ClusteringColumnRestrictions.builder(table, allowFiltering).build();
+        this.nonPrimaryKeyRestrictions = RestrictionSet.builder().build();
+        this.notNullColumns = ImmutableSet.of();
+        this.filterRestrictions = IndexRestrictions.of();
+        this.children = Collections.emptyList();
     }
 
-    public StatementRestrictions(ClientState state,
-                                 StatementType type,
-                                 TableMetadata table,
-                                 WhereClause whereClause,
-                                 VariableSpecifications boundNames,
-                                 List<Ordering> orderings,
-                                 boolean selectsOnlyStaticColumns,
-                                 boolean allowFiltering,
-                                 boolean forView)
+    /**
+     * Adds the following restrictions to the index restrictions.
+     *
+     * @param restrictions the restrictions to add to the index restrictions
+     * @return a new {@code StatementRestrictions} with the new index restrictions
+     */
+    public StatementRestrictions addIndexRestrictions(Restrictions restrictions)
     {
-        this(state, type, table, whereClause, boundNames, orderings, selectsOnlyStaticColumns, type.allowUseOfSecondaryIndices(), allowFiltering, forView);
+        IndexRestrictions newIndexRestrictions = IndexRestrictions.builder()
+                                                                  .add(filterRestrictions)
+                                                                  .add(restrictions)
+                                                                  .build();
+
+        return new StatementRestrictions(table,
+                                         partitionKeyRestrictions,
+                                         clusteringColumnsRestrictions,
+                                         nonPrimaryKeyRestrictions,
+                                         notNullColumns,
+                                         isDisjunction,
+                                         usesSecondaryIndexing,
+                                         isKeyRange,
+                                         hasRegularColumnsRestrictions,
+                                         newIndexRestrictions,
+                                         children);
     }
 
-    /*
-     * We want to override allowUseOfSecondaryIndices flag from the StatementType for MV statements
-     * to avoid initing the Keyspace and SecondaryIndexManager.
+    /**
+     * Adds the following external restrictions (mostly custom and user index expressions) to the index restrictions.
+     *
+     * @param restrictions the restrictions to add to the index restrictions
+     * @return a new {@code StatementRestrictions} with the new index restrictions
      */
-    public StatementRestrictions(ClientState state,
-                                 StatementType type,
-                                 TableMetadata table,
-                                 WhereClause whereClause,
-                                 VariableSpecifications boundNames,
-                                 List<Ordering> orderings,
-                                 boolean selectsOnlyStaticColumns,
-                                 boolean allowUseOfSecondaryIndices,
-                                 boolean allowFiltering,
-                                 boolean forView)
+    public StatementRestrictions addExternalRestrictions(Iterable<ExternalRestriction> restrictions)
     {
-        this(type, table, allowFiltering);
-
-        final IndexRegistry indexRegistry = type.allowUseOfSecondaryIndices() ? IndexRegistry.obtain(table) : null;
-
-        /*
-         * WHERE clause. For a given entity, rules are:
-         *   - EQ relation conflicts with anything else (including a 2nd EQ)
-         *   - Can't have more than one LT(E) relation (resp. GT(E) relation)
-         *   - IN relation are restricted to row keys (for now) and conflicts with anything else (we could
-         *     allow two IN for the same entity but that doesn't seem very useful)
-         *   - The value_alias cannot be restricted in any way (we don't support wide rows with indexed value
-         *     in CQL so far)
-         *   - CONTAINS and CONTAINS_KEY cannot be used with UPDATE or DELETE
-         */
-        for (Relation relation : whereClause.relations)
-        {
-            if ((relation.isContains() || relation.isContainsKey()) && (type.isUpdate() || type.isDelete()))
-            {
-                throw invalidRequest("Cannot use %s with %s", type, relation.operator());
-            }
+        IndexRestrictions.Builder newIndexRestrictions = IndexRestrictions.builder().add(filterRestrictions);
+
+        for (ExternalRestriction restriction : restrictions)
+            newIndexRestrictions.add(restriction);
+
+        return new StatementRestrictions(table,
+                                         partitionKeyRestrictions,
+                                         clusteringColumnsRestrictions,
+                                         nonPrimaryKeyRestrictions,
+                                         notNullColumns,
+                                         isDisjunction,
+                                         usesSecondaryIndexing,
+                                         isKeyRange,
+                                         hasRegularColumnsRestrictions,
+                                         newIndexRestrictions.build(),
+                                         children);
+    }
 
-            if (relation.operator() == Operator.IS_NOT)
-            {
-                if (!forView)
-                    throw new InvalidRequestException("Unsupported restriction: " + relation);
+    private StatementRestrictions(TableMetadata table,
+                                  PartitionKeyRestrictions partitionKeyRestrictions,
+                                  ClusteringColumnRestrictions clusteringColumnsRestrictions,
+                                  RestrictionSet nonPrimaryKeyRestrictions,
+                                  ImmutableSet<ColumnMetadata> notNullColumns,
+                                  boolean isDisjunction,
+                                  boolean usesSecondaryIndexing,
+                                  boolean isKeyRange,
+                                  boolean hasRegularColumnsRestrictions,
+                                  IndexRestrictions filterRestrictions,
+                                  List<StatementRestrictions> children)
+    {
+        this.table = table;
+        this.partitionKeyRestrictions = partitionKeyRestrictions;
+        this.clusteringColumnsRestrictions = clusteringColumnsRestrictions;
+        this.nonPrimaryKeyRestrictions = nonPrimaryKeyRestrictions;
+        this.notNullColumns = notNullColumns;
+        this.filterRestrictions = filterRestrictions;
+        this.isDisjunction = isDisjunction;
+        this.usesSecondaryIndexing = usesSecondaryIndexing;
+        this.isKeyRange = isKeyRange;
+        this.hasRegularColumnsRestrictions = hasRegularColumnsRestrictions;
+        this.children = children;
+    }
 
-                this.notNullColumns.addAll(relation.toRestriction(table, boundNames).getColumnDefs());
-            }
-            else if (relation.isLIKE())
-            {
-                Restriction restriction = relation.toRestriction(table, boundNames);
+    public static StatementRestrictions create(ClientState state,
+                                               StatementType type,
+                                               TableMetadata table,
+                                               WhereClause whereClause,
+                                               VariableSpecifications boundNames,
+                                               List<Ordering> orderings,
+                                               boolean selectsOnlyStaticColumns,
+                                               boolean allowFiltering,
+                                               boolean forView)
+    {
+        return new Builder(state,
+                           type,
+                           table,
+                           whereClause,
+                           boundNames,
+                           orderings,
+                           selectsOnlyStaticColumns,
+                           type.allowUseOfSecondaryIndices(),
+                           allowFiltering,
+                           forView).build();
+    }
 
-                if (!type.allowUseOfSecondaryIndices() || !restriction.hasSupportingIndex(indexRegistry))
-                    throw new InvalidRequestException(String.format("LIKE restriction is only supported on properly " +
-                                                                    "indexed columns. %s is not valid.",
-                                                                    relation));
+    public static StatementRestrictions create(ClientState state,
+                                               StatementType type,
+                                               TableMetadata table,
+                                               WhereClause whereClause,
+                                               VariableSpecifications boundNames,
+                                               List<Ordering> orderings,
+                                               boolean selectsOnlyStaticColumns,
+                                               boolean allowUseOfSecondaryIndices,
+                                               boolean allowFiltering,
+                                               boolean forView)
+    {
+        return new Builder(state,
+                           type,
+                           table,
+                           whereClause,
+                           boundNames,
+                           orderings,
+                           selectsOnlyStaticColumns,
+                           allowUseOfSecondaryIndices,
+                           allowFiltering,
+                           forView).build();
+    }
 
-                addRestriction(restriction, indexRegistry);
-            }
-            else
-            {
-                addRestriction(relation.toRestriction(table, boundNames), indexRegistry);
-            }
+    /**
+     * Build a <code>StatementRestrictions</code> from a <code>WhereClause</code> for a given
+     * <code>StatementType</code>, <code>TableMetadata</code> and <code>VariableSpecifications</code>
+     * </p>
+     * The validation rules for whether the <code>StatementRestrictions</code> are valid depend on a
+     * number of considerations, including whether indexes are being used and whether filtering is being
+     * used.
+     */
+    public static class Builder
+    {
+        private final ClientState state;
+        private final StatementType type;
+        private final TableMetadata table;
+        private final WhereClause whereClause;
+        private final VariableSpecifications boundNames;
+
+        private final List<Ordering> orderings;
+        private final boolean selectsOnlyStaticColumns;
+        private final boolean allowUseOfSecondaryIndices;
+        private final boolean allowFiltering;
+        private final boolean forView;
+
+        public Builder(ClientState state,
+                       StatementType type,
+                       TableMetadata table,
+                       WhereClause whereClause,
+                       VariableSpecifications boundNames,
+                       List<Ordering> orderings,
+                       boolean selectsOnlyStaticColumns,
+                       boolean allowUseOfSecondaryIndices,
+                       boolean allowFiltering,
+                       boolean forView)
+        {
+            this.state = state;
+            this.type = type;
+            this.table = table;
+            this.whereClause = whereClause;
+            this.boundNames = boundNames;
+            this.orderings = orderings;
+            this.selectsOnlyStaticColumns = selectsOnlyStaticColumns;
+            this.allowUseOfSecondaryIndices = allowUseOfSecondaryIndices;
+            this.allowFiltering = allowFiltering;
+            this.forView = forView;
         }
 
-        // ORDER BY clause.
-        // Some indexes can be used for ordering.
-        nonPrimaryKeyRestrictions = addOrderingRestrictions(orderings, nonPrimaryKeyRestrictions);
+        public StatementRestrictions build()
+        {
+            IndexRegistry indexRegistry = null;
 
-        hasRegularColumnsRestrictions = nonPrimaryKeyRestrictions.hasRestrictionFor(ColumnMetadata.Kind.REGULAR);
+            // We want to avoid opening the keyspace during view construction
+            // since we're parsing these for restore and the base table or keyspace might not exist in the current schema.
+            if (allowUseOfSecondaryIndices && type.allowUseOfSecondaryIndices())
+                indexRegistry = IndexRegistry.obtain(table);
 
-        boolean hasQueriableClusteringColumnIndex = false;
-        boolean hasQueriableIndex = false;
+            WhereClause.AndElement root = whereClause.root().conjunctiveForm();
+            return doBuild(root, indexRegistry, 0);
+        }
 
-        if (allowUseOfSecondaryIndices)
+        /**
+         * Processes the WHERE clause expression tree recursively and assigns the restrictions to different sets
+         * based on the columns they are applied to.
+         *
+         * @param element root of the tree
+         * @param nestingLevel recursion depth needed to reject the restrictions that
+         *                     are not allowed to be nested (e.g. partition key restrictions)
+         */
+        StatementRestrictions doBuild(WhereClause.ExpressionElement element, IndexRegistry indexRegistry, int nestingLevel)
         {
-            if (whereClause.containsCustomExpressions())
-                processCustomIndexExpressions(whereClause.expressions, boundNames, indexRegistry);
-
-            hasQueriableClusteringColumnIndex = clusteringColumnsRestrictions.hasSupportingIndex(indexRegistry);
-            hasQueriableIndex = !filterRestrictions.getCustomIndexExpressions().isEmpty()
-                    || hasQueriableClusteringColumnIndex
-                    || partitionKeyRestrictions.hasSupportingIndex(indexRegistry)
-                    || nonPrimaryKeyRestrictions.hasSupportingIndex(indexRegistry);
-        }
+            assert element instanceof WhereClause.AndElement || nestingLevel > 0:
+                    "Root of the WHERE clause expression tree must be a conjunction";
+
+            PartitionKeySingleRestrictionSet.Builder partitionKeyRestrictionSet = PartitionKeySingleRestrictionSet.builder(table.partitionKeyAsClusteringComparator());
+            ClusteringColumnRestrictions.Builder clusteringColumnsRestrictionSet = ClusteringColumnRestrictions.builder(table, allowFiltering, indexRegistry);
+            RestrictionSet.Builder nonPrimaryKeyRestrictionSet = RestrictionSet.builder();
+            ImmutableSet.Builder<ColumnMetadata> notNullColumnsBuilder = ImmutableSet.builder();
+
+
+            // ORDER BY clause. We add it first because orderings are not really restrictions
+            // and by adding first, we ensure that merging restrictions works as expected.
+            // The long term solution will break ordering out into its own abstraction.
+            if (nestingLevel == 0)
+                addOrderingRestrictions(orderings, indexRegistry, nonPrimaryKeyRestrictionSet);
+
+            /*
+             * WHERE clause. For a given entity, rules are:
+             *   - EQ relation conflicts with anything else (including a 2nd EQ)
+             *   - Can't have more than one LT(E) relation (resp. GT(E) relation)
+             *   - IN relation are restricted to row keys (for now) and conflicts with anything else (we could
+             *     allow two IN for the same entity but that doesn't seem very useful)
+             *   - The value_alias cannot be restricted in any way (we don't support wide rows with indexed value
+             *     in CQL so far)
+             *   - CONTAINS and CONTAINS_KEY cannot be used with UPDATE or DELETE
+             */
+            for (Relation relation : element.relations())
+            {
+                if ((relation.isContains() || relation.isContainsKey() || relation.isNotContains() || relation.isNotContainsKey())
+                        && (type.isUpdate() || type.isDelete()))
+                {
+                    throw invalidRequest("Cannot use %s with %s", type, relation.operator());
+                }
 
-        // At this point, the select statement if fully constructed, but we still have a few things to validate
-        processPartitionKeyRestrictions(state, hasQueriableIndex, allowFiltering, forView);
+                if (relation.operator() == Operator.IS_NOT)
+                {
+                    if (!forView)
+                        throw invalidRequest("Unsupported restriction: %s", relation);
 
-        // Some but not all of the partition key columns have been specified;
-        // hence we need turn these restrictions into a row filter.
-        if (usesSecondaryIndexing || partitionKeyRestrictions.needFiltering(table))
-            filterRestrictions.add(partitionKeyRestrictions);
+                    notNullColumnsBuilder.addAll(relation.toRestriction(table, boundNames).getColumnDefs());
+                }
+                else
+                {
+                    Restriction restriction = relation.toRestriction(table, boundNames);
 
-        if (selectsOnlyStaticColumns && hasClusteringColumnsRestrictions())
-        {
-            // If the only updated/deleted columns are static, then we don't need clustering columns.
-            // And in fact, unless it is an INSERT, we reject if clustering colums are provided as that
-            // suggest something unintended. For instance, given:
-            //   CREATE TABLE t (k int, v int, s int static, PRIMARY KEY (k, v))
-            // it can make sense to do:
-            //   INSERT INTO t(k, v, s) VALUES (0, 1, 2)
-            // but both
-            //   UPDATE t SET s = 3 WHERE k = 0 AND v = 1
-            //   DELETE v FROM t WHERE k = 0 AND v = 1
-            // sounds like you don't really understand what your are doing.
-            if (type.isDelete() || type.isUpdate())
-                throw invalidRequest("Invalid restrictions on clustering columns since the %s statement modifies only static columns",
-                                     type);
-            if (type.isSelect())
-                throw invalidRequest("Cannot restrict clustering columns when selecting only static columns");
-        }
+                    if (relation.isLIKE() && (!type.allowUseOfSecondaryIndices() || !restriction.hasSupportingIndex(indexRegistry)))
+                    {
+                        if (getColumnsWithUnsupportedIndexRestrictions(table, ImmutableList.of(restriction)).isEmpty())
+                        {
+                            throw invalidRequest(RESTRICTION_REQUIRES_INDEX_MESSAGE, relation.operator(), relation.toString());
+                        }
+                        else
+                        {
+                            throw invalidRequest(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_LIKE_MESSAGE, restriction.getFirstColumn());
+                        }
+                    }
+                    if (relation.operator() == Operator.ANALYZER_MATCHES)
+                    {
+                        if (!type.allowUseOfSecondaryIndices())
+                        {
+                            throw invalidRequest("Invalid query. %s does not support use of secondary indices, but %s restriction requires a secondary index.", type.name(), relation.toString());
+                        }
+                        if (!restriction.hasSupportingIndex(indexRegistry))
+                        {
+                            if (getColumnsWithUnsupportedIndexRestrictions(table, ImmutableList.of(restriction)).isEmpty())
+                            {
+                                throw invalidRequest(RESTRICTION_REQUIRES_INDEX_MESSAGE, relation.operator(), relation.toString());
+                            }
+                            else
+                            {
+                                throw invalidRequest(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_ANALYZER_MATCHES_MESSAGE, restriction.getFirstColumn());
+                            }
+                        }
+                    }
+    
+                    ColumnMetadata def = restriction.getFirstColumn();
+                    if (def.isPartitionKey())
+                    {
+                        // All partition key restrictions must be a part of the top-level AND operation.
+                        // The read path filtering logic is currently unable to filter rows based on
+                        // partition key restriction that is a part of a complex expression involving disjunctions.
+                        // ALLOW FILTERING does not cut it, as RowFilter can't handle ORed partition
+                        // key restrictions properly.
+                        if (nestingLevel > 0)
+                            throw invalidRequest(StatementRestrictions.PARTITION_KEY_RESTRICTION_MUST_BE_TOP_LEVEL, def);
+
+                        partitionKeyRestrictionSet.addRestriction(restriction);
+                    }
+                    // If a clustering column restriction is nested (under OR operator),
+                    // we can't treat it as a real clustering column,
+                    // but instead we treat it as a regular column and use
+                    // index (if we have one) or use row filtering on it; hence we require nestingLevel == 0 check here
+                    else if (def.isClusteringColumn() && nestingLevel == 0)
+                    {
+                        // If a clustering column restriction is nested (under OR operator),
+                        // we can't treat it as a real clustering column,
+                        // but instead we treat it as a regular column and use
+                        // index (if we have one) or use row filtering on it; hence we require nestingLevel == 0 check here
+                        clusteringColumnsRestrictionSet.addRestriction(restriction);
+                    }
+                    else
+                    {
+                        nonPrimaryKeyRestrictionSet.addRestriction((SingleRestriction) restriction, element.isDisjunction());
+                    }
+                }
+            }
 
-        processClusteringColumnsRestrictions(hasQueriableIndex,
-                                             selectsOnlyStaticColumns,
-                                             forView,
-                                             allowFiltering);
+            PartitionKeyRestrictions partitionKeyRestrictions = partitionKeyRestrictionSet.build(indexRegistry);
+            ClusteringColumnRestrictions clusteringColumnsRestrictions = clusteringColumnsRestrictionSet.build();
+            RestrictionSet nonPrimaryKeyRestrictions = nonPrimaryKeyRestrictionSet.build();
+            ImmutableSet<ColumnMetadata> notNullColumns = notNullColumnsBuilder.build();
+            boolean hasRegularColumnsRestrictions = nonPrimaryKeyRestrictions.hasRestrictionFor(ColumnMetadata.Kind.REGULAR);
+            boolean usesSecondaryIndexing = false;
+            boolean isKeyRange = false;
 
-        // Covers indexes on the first clustering column (among others).
-        if (isKeyRange && hasQueriableClusteringColumnIndex)
-            usesSecondaryIndexing = true;
+            boolean hasQueryableClusteringColumnIndex = false;
+            boolean hasQueryableIndex = false;
 
-        if (usesSecondaryIndexing || clusteringColumnsRestrictions.needFiltering())
-            filterRestrictions.add(clusteringColumnsRestrictions);
+            IndexRestrictions.Builder filterRestrictionsBuilder = IndexRestrictions.builder();
 
-        // Even if usesSecondaryIndexing is false at this point, we'll still have to use one if
-        // there is restrictions not covered by the PK.
-        if (!nonPrimaryKeyRestrictions.isEmpty())
-        {
-            if (!type.allowNonPrimaryKeyInWhereClause())
+            if (allowUseOfSecondaryIndices)
             {
-                Collection<ColumnIdentifier> nonPrimaryKeyColumns =
-                        ColumnMetadata.toIdentifiers(nonPrimaryKeyRestrictions.getColumnDefs());
+                if (element.containsCustomExpressions())
+                {
+                    CustomIndexExpression customExpression = prepareCustomIndexExpression(element.expressions(),
+                                                                                          boundNames,
+                                                                                          indexRegistry);
+                    filterRestrictionsBuilder.add(customExpression);
+                }
 
-                throw invalidRequest("Non PRIMARY KEY columns found in where clause: %s ",
-                                     Joiner.on(", ").join(nonPrimaryKeyColumns));
+                hasQueryableClusteringColumnIndex = clusteringColumnsRestrictions.hasSupportingIndex(indexRegistry);
+                hasQueryableIndex = element.containsCustomExpressions()
+                                    || hasQueryableClusteringColumnIndex
+                                    || partitionKeyRestrictions.hasSupportingIndex(indexRegistry)
+                                    || nonPrimaryKeyRestrictions.hasSupportingIndex(indexRegistry);
             }
 
-            var annRestriction = Streams.stream(nonPrimaryKeyRestrictions).filter(SingleRestriction::isANN).findFirst();
-            if (annRestriction.isPresent())
+            // At this point, the select statement if fully constructed, but we still have a few things to validate
+            if (!type.allowPartitionKeyRanges())
+            {
+                checkFalse(partitionKeyRestrictions.isOnToken(),
+                           "The token function cannot be used in WHERE clauses for %s statements", type);
+    
+                if (partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(table))
+                    throw invalidRequest("Some partition key parts are missing: %s",
+                                         Joiner.on(", ").join(getPartitionKeyUnrestrictedComponents(partitionKeyRestrictions)));
+
+                // slice query
+                checkFalse(partitionKeyRestrictions.hasSlice(),
+                           "Only EQ and IN relation are supported on the partition key (unless you use the token() function)"
+                           + " for %s statements", type);
+            }
+            else
             {
-                // If there is an ANN restriction then it must be for a vector<float, n> column, and it must have an index
-                var annColumn = annRestriction.get().getFirstColumn();
-
-                if (!annColumn.type.isVector() || !(((VectorType<?>)annColumn.type).elementType instanceof FloatType))
-                    throw invalidRequest(StatementRestrictions.ANN_ONLY_SUPPORTED_ON_VECTOR_MESSAGE);
-                if (indexRegistry == null || indexRegistry.listIndexes().stream().noneMatch(i -> i.dependsOn(annColumn)))
-                    throw invalidRequest(StatementRestrictions.ANN_REQUIRES_INDEX_MESSAGE);
-                // We do not allow ANN queries using partition key restrictions that need filtering
-                if (partitionKeyRestrictions.needFiltering(table))
-                    throw invalidRequest(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE);
-                // We do not allow ANN query filtering using non-indexed columns
-                var nonAnnColumns = Streams.stream(nonPrimaryKeyRestrictions)
-                                           .filter(r -> !r.isANN())
-                                           .map(Restriction::getFirstColumn)
-                                           .collect(Collectors.toList());
-                var clusteringColumns = clusteringColumnsRestrictions.getColumnDefinitions();
-                if (!nonAnnColumns.isEmpty() || !clusteringColumns.isEmpty())
+                // If there are no partition restrictions or there's only token restriction, we have to set a key range
+                if (partitionKeyRestrictions.isOnToken())
+                    isKeyRange = true;
+    
+                if (partitionKeyRestrictions.isEmpty() && partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(table))
                 {
-                    var nonIndexedColumns = Stream.concat(nonAnnColumns.stream(), clusteringColumns.stream())
-                                                  .filter(c -> indexRegistry.listIndexes().stream().noneMatch(i -> i.dependsOn(c)))
-                                                  .collect(Collectors.toList());
+                    isKeyRange = true;
+                    usesSecondaryIndexing = hasQueryableIndex;
+                }
 
-                    if (!nonIndexedColumns.isEmpty())
-                    {
-                        // restrictions on non-clustering columns, or clusterings that still need filtering, are invalid
-                        if (!clusteringColumns.containsAll(nonIndexedColumns)
-                                || partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(table)
-                                || clusteringColumnsRestrictions.needFiltering())
-                            throw invalidRequest(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE);
-                    }
+                // If there is a queryable index, no special condition is required on the other restrictions.
+                // But we still need to know 2 things:
+                // - If we don't have a queryable index, is the query ok
+                // - Is it queryable without 2ndary index, which is always more efficient
+                // If a component of the partition key is restricted by a relation, all preceding
+                // components must have a EQ. Only the last partition key component can be in IN relation.
+                // If partition key restrictions exist and this is a disjunction then we may need filtering
+                if (partitionKeyRestrictions.needFiltering(table) || (!partitionKeyRestrictions.isEmpty() && element.isDisjunction()))
+                {
+                    if (!allowFiltering && !forView && !hasQueryableIndex && requiresAllowFilteringIfNotSpecified(table))
+                        throw new InvalidRequestException(allowFilteringMessage(state));
+    
+                    isKeyRange = true;
+                    usesSecondaryIndexing = hasQueryableIndex;
                 }
             }
-            else
+
+            // Some but not all of the partition key columns have been specified or they form part of a disjunction;
+            // hence we need turn these restrictions into a row filter.
+            if (usesSecondaryIndexing || partitionKeyRestrictions.needFiltering(table) || element.isDisjunction())
+                filterRestrictionsBuilder.add(partitionKeyRestrictions);
+
+            if (selectsOnlyStaticColumns && !clusteringColumnsRestrictions.isEmpty())
             {
-                // We do not support indexed vector restrictions that are not part of an ANN ordering
-                var vectorColumn = nonPrimaryKeyRestrictions.getColumnDefs()
-                                                            .stream()
-                                                            .filter(c -> c.type.isVector())
-                                                            .findFirst();
-                if (vectorColumn.isPresent() && indexRegistry.listIndexes().stream().anyMatch(i -> i.dependsOn(vectorColumn.get())))
-                    throw invalidRequest(StatementRestrictions.VECTOR_INDEXES_ANN_ONLY_MESSAGE);
+                // If the only updated/deleted columns are static, then we don't need clustering columns.
+                // And in fact, unless it is an INSERT, we reject if clustering colums are provided as that
+                // suggest something unintended. For instance, given:
+                //   CREATE TABLE t (k int, v int, s int static, PRIMARY KEY (k, v))
+                // it can make sense to do:
+                //   INSERT INTO t(k, v, s) VALUES (0, 1, 2)
+                // but both
+                //   UPDATE t SET s = 3 WHERE k = 0 AND v = 1
+                //   DELETE v FROM t WHERE k = 0 AND v = 1
+                // sounds like you don't really understand what your are doing.
+                if (type.isDelete() || type.isUpdate())
+                    throw invalidRequest("Invalid restrictions on clustering columns since the %s statement modifies only static columns",
+                                         type);
             }
 
-            if (hasQueriableIndex)
+            // Now process and validate the clustering column restrictions
+            checkFalse(!type.allowClusteringColumnSlices() && clusteringColumnsRestrictions.hasSlice(),
+                       "Slice restrictions are not supported on the clustering columns in %s statements", type);
+
+            if (!type.allowClusteringColumnSlices()
+                && (!table.isCompactTable() || (table.isCompactTable() && clusteringColumnsRestrictions.isEmpty())))
             {
-                usesSecondaryIndexing = true;
+                if (!selectsOnlyStaticColumns && (table.clusteringColumns().size() != clusteringColumnsRestrictions.size()))
+                    throw invalidRequest("Some clustering keys are missing: %s",
+                                         Joiner.on(", ").join(getUnrestrictedClusteringColumns(clusteringColumnsRestrictions)));
             }
             else
             {
-                if (!allowFiltering && requiresAllowFilteringIfNotSpecified())
-                    throw invalidRequest(allowFilteringMessage(state));
+                checkFalse(clusteringColumnsRestrictions.hasContains() && !hasQueryableIndex && !allowFiltering,
+                           "Clustering columns can only be restricted with CONTAINS with a secondary index or filtering");
+    
+                if (!clusteringColumnsRestrictions.isEmpty() && clusteringColumnsRestrictions.needFiltering())
+                {
+                    if (hasQueryableIndex || forView)
+                    {
+                        usesSecondaryIndexing = true;
+                    }
+                    else if (!allowFiltering)
+                    {
+                        List<ColumnMetadata> clusteringColumns = table.clusteringColumns();
+                        List<ColumnMetadata> restrictedColumns = clusteringColumnsRestrictions.getColumnDefs();
+    
+                        for (int i = 0, m = restrictedColumns.size(); i < m; i++)
+                        {
+                            ColumnMetadata clusteringColumn = clusteringColumns.get(i);
+                            ColumnMetadata restrictedColumn = restrictedColumns.get(i);
+    
+                            if (!clusteringColumn.equals(restrictedColumn))
+                            {
+                                throw invalidRequest("PRIMARY KEY column \"%s\" cannot be restricted as preceding column \"%s\" is not restricted",
+                                                     restrictedColumn.name,
+                                                     clusteringColumn.name);
+                            }
+                        }
+                    }
+                }
             }
 
-            filterRestrictions.add(nonPrimaryKeyRestrictions);
+            // Covers indexes on the first clustering column (among others).
+            if (isKeyRange && hasQueryableClusteringColumnIndex)
+                usesSecondaryIndexing = true;
+
+            // Because an ANN queries limit the result set based within the SAI, clustering column restrictions
+            // must be added to the filter restrictions.
+            if (orderings.stream().anyMatch(o -> o.expression.hasNonClusteredOrdering()))
+                usesSecondaryIndexing = true;
+
+            if (usesSecondaryIndexing || clusteringColumnsRestrictions.needFiltering())
+                filterRestrictionsBuilder.add(clusteringColumnsRestrictions);
+
+            // Even if usesSecondaryIndexing is false at this point, we'll still have to use one if
+            // there is restrictions not covered by the PK.
+            if (!nonPrimaryKeyRestrictions.isEmpty())
+            {
+                var columnRestrictions = allColumnRestrictions(clusteringColumnsRestrictions, nonPrimaryKeyRestrictions);
+
+                if (!type.allowNonPrimaryKeyInWhereClause())
+                {
+                    Collection<ColumnIdentifier> nonPrimaryKeyColumns =
+                            ColumnMetadata.toIdentifiers(nonPrimaryKeyRestrictions.getColumnDefs());
+    
+                    throw invalidRequest("Non PRIMARY KEY columns found in where clause: %s ",
+                                         Joiner.on(", ").join(nonPrimaryKeyColumns));
+                }
+                if (hasQueryableIndex)
+                    usesSecondaryIndexing = true;
+                else
+                {
+                    var vectorColumn = nonPrimaryKeyRestrictions.getColumnDefs().stream().filter(c -> c.type.isVector()).findFirst();
+                    if (vectorColumn.isPresent())
+                    {
+                        var vc = vectorColumn.get();
+                        var hasIndex = indexRegistry.listIndexes().stream().anyMatch(i -> i.dependsOn(vc));
+                        var isBoundedANN = nonPrimaryKeyRestrictions.restrictions().stream().anyMatch(SingleRestriction::isBoundedAnn);
+                        var isIndexBasedOrdering = nonPrimaryKeyRestrictions.restrictions().stream().anyMatch(SingleRestriction::isIndexBasedOrdering);
+                        if (hasIndex)
+                        {
+                            if (isBoundedANN)
+                                throw invalidRequest(StatementRestrictions.VECTOR_INDEX_PRESENT_NOT_SUPPORT_GEO_DISTANCE_MESSAGE);
+                            else
+                                throw invalidRequest(StatementRestrictions.VECTOR_INDEXES_UNSUPPORTED_OP_MESSAGE, vc);
+                        }
+                        else
+                        {
+                            // We check if ANN vector column has index earlier, so we only need to for bounded ann here
+                            if (isBoundedANN)
+                                throw invalidRequest(StatementRestrictions.GEO_DISTANCE_REQUIRES_INDEX_MESSAGE);
+                            else if (isIndexBasedOrdering)
+                                throw invalidRequest(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE);
+                        }
+                    }
+
+                    if (!allowFiltering  && requiresAllowFilteringIfNotSpecified(table))
+                        throwRequiresAllowFilteringError(table, columnRestrictions, state);
+                }
+                filterRestrictionsBuilder.add(nonPrimaryKeyRestrictions);
+            }
+
+            if (usesSecondaryIndexing)
+                checkFalse(partitionKeyRestrictions.hasIN(),
+                           "Select on indexed columns and with IN clause for the PRIMARY KEY are not supported");
+
+            ImmutableList.Builder<StatementRestrictions> children = ImmutableList.builder();
+
+            for (WhereClause.ContainerElement container : element.operations())
+                children.add(doBuild(container, indexRegistry, nestingLevel + 1));
+
+            return new StatementRestrictions(table,
+                                             partitionKeyRestrictions,
+                                             clusteringColumnsRestrictions,
+                                             nonPrimaryKeyRestrictions,
+                                             notNullColumns,
+                                             element.isDisjunction(),
+                                             usesSecondaryIndexing,
+                                             isKeyRange,
+                                             hasRegularColumnsRestrictions,
+                                             filterRestrictionsBuilder.build(),
+                                             children.build());
         }
 
-        if (usesSecondaryIndexing)
-            validateSecondaryIndexSelections();
+        /**
+         * This is a hack to push ordering down to indexes.
+         * Indexes are selected based on RowFilter only, so we need to turn orderings into restrictions
+         * so they end up in the row filter.
+         *
+         * @param orderings orderings from the select statement
+         * @param indexRegistry used to check if the ordering is supported by an index
+         * @param receiver target restriction builder to receive the additional restrictions
+         */
+        private void addOrderingRestrictions(List<Ordering> orderings, IndexRegistry indexRegistry, RestrictionSet.Builder receiver)
+        {
+            List<Ordering> indexOrderings = orderings.stream().filter(o -> o.expression.hasNonClusteredOrdering()).collect(Collectors.toList());
+
+            if (indexOrderings.size() > 1)
+                throw new InvalidRequestException("Cannot specify more than one ordering column when using SAI indexes");
+            else if (indexOrderings.size() == 1)
+            {
+                if (orderings.size() > 1)
+                    throw new InvalidRequestException("Cannot combine clustering column ordering with non-clustering column ordering");
+                Ordering ordering = indexOrderings.get(0);
+                // TODO remove the instanceof with SelectStatement.ANN_USE_SYNTHETIC_SCORE.
+                if (ordering.direction != Ordering.Direction.ASC && (ordering.expression.isScored() || ordering.expression instanceof Ordering.Ann))
+                    throw new InvalidRequestException("Descending ANN ordering is not supported");
+                if (!ENABLE_SAI_GENERAL_ORDER_BY && ordering.expression instanceof Ordering.SingleColumn)
+                    throw new InvalidRequestException("SAI based ORDER BY on non-vector column is not supported");
+                SingleRestriction restriction = ordering.expression.toRestriction();
+                if (!restriction.hasSupportingIndex(indexRegistry))
+                {
+                    var type = restriction.getFirstColumn().type.asCQL3Type().getType();
+                    // This is a slight hack, but once we support a way to order these types, we can remove it.
+                    if (type instanceof IntegerType || type instanceof DecimalType)
+                        throw new InvalidRequestException(String.format("SAI based ordering on column %s of type %s is not supported",
+                                                          restriction.getFirstColumn(),
+                                                          restriction.getFirstColumn().type.asCQL3Type()));
+                    if (ordering.expression instanceof Ordering.Bm25)
+                        throw new InvalidRequestException(String.format(BM25_ORDERING_REQUIRES_ANALYZED_INDEX_MESSAGE,
+                                                                        restriction.getFirstColumn()));
+                    else
+                        throw new InvalidRequestException(String.format(NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE,
+                                                                        restriction.getFirstColumn()));
+                }
+                receiver.addRestriction(restriction, false);
+            }
+        }
+
+        private CustomIndexExpression prepareCustomIndexExpression(List<CustomIndexExpression> expressions,
+                                                                   VariableSpecifications boundNames,
+                                                                   IndexRegistry indexRegistry)
+        {
+            if (expressions.size() > 1)
+                throw new InvalidRequestException(IndexRestrictions.MULTIPLE_EXPRESSIONS);
+
+            CustomIndexExpression expression = expressions.get(0);
+
+            QualifiedName name = expression.targetIndex;
+
+            if (name.hasKeyspace() && !name.getKeyspace().equals(table.keyspace))
+                throw IndexRestrictions.invalidIndex(expression.targetIndex, table);
+
+            if (!table.indexes.has(expression.targetIndex.getName()))
+                throw IndexRestrictions.indexNotFound(expression.targetIndex, table);
+
+            Index index = indexRegistry.getIndex(table.indexes.get(expression.targetIndex.getName()).orElseThrow());
+            if (!index.getIndexMetadata().isCustom())
+                throw IndexRestrictions.nonCustomIndexInExpression(expression.targetIndex);
+
+            AbstractType<?> expressionType = index.customExpressionValueType();
+            if (expressionType == null)
+                throw IndexRestrictions.customExpressionNotSupported(expression.targetIndex);
+
+            expression.prepareValue(table, expressionType, boundNames);
+            return expression;
+        }
+
+        /**
+         * Returns the partition key components that are not restricted.
+         * @return the partition key components that are not restricted.
+         */
+        private Collection<ColumnIdentifier> getPartitionKeyUnrestrictedComponents(PartitionKeyRestrictions partitionKeyRestrictions)
+        {
+            List<ColumnMetadata> list = new ArrayList<>(table.partitionKeyColumns());
+            list.removeAll(partitionKeyRestrictions.getColumnDefs());
+            return ColumnMetadata.toIdentifiers(list);
+        }
+
+        /**
+         * Returns the clustering columns that are not restricted.
+         * @return the clustering columns that are not restricted.
+         */
+        private Collection<ColumnIdentifier> getUnrestrictedClusteringColumns(ClusteringColumnRestrictions clusteringColumnsRestrictions)
+        {
+            List<ColumnMetadata> missingClusteringColumns = new ArrayList<>(table.clusteringColumns());
+            missingClusteringColumns.removeAll(clusteringColumnsRestrictions.getColumnDefs());
+            return ColumnMetadata.toIdentifiers(missingClusteringColumns);
+        }
     }
 
-    public boolean requiresAllowFilteringIfNotSpecified()
+    public IndexRestrictions filterRestrictions()
+    {
+        return filterRestrictions;
+    }
+
+    public List<StatementRestrictions> children()
+    {
+        return children;
+    }
+
+    public static boolean requiresAllowFilteringIfNotSpecified(TableMetadata table)
     {
         if (!table.isVirtual())
             return true;
@@ -366,15 +834,40 @@ public boolean requiresAllowFilteringIfNotSpecified()
         return !tableNullable.allowFilteringImplicitly();
     }
 
-    private void addRestriction(Restriction restriction, IndexRegistry indexRegistry)
+    public boolean hasIndxBasedOrdering()
+    {
+        return nonPrimaryKeyRestrictions.restrictions().stream().anyMatch(SingleRestriction::isIndexBasedOrdering);
+    }
+
+    public void throwRequiresAllowFilteringError(TableMetadata table, ClientState state)
     {
-        ColumnMetadata def = restriction.getFirstColumn();
-        if (def.isPartitionKey())
-            partitionKeyRestrictions = partitionKeyRestrictions.mergeWith(restriction);
-        else if (def.isClusteringColumn())
-            clusteringColumnsRestrictions = clusteringColumnsRestrictions.mergeWith(restriction, indexRegistry);
+        if (hasIndxBasedOrdering())
+            throw invalidRequest(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
+
+        throwRequiresAllowFilteringError(table, allColumnRestrictions(clusteringColumnsRestrictions, nonPrimaryKeyRestrictions), state);
+    }
+
+    private static void throwRequiresAllowFilteringError(TableMetadata table, Iterable<Restriction> columnRestrictions, ClientState state)
+    {
+        Set<ColumnMetadata> unsupported = getColumnsWithUnsupportedIndexRestrictions(table, columnRestrictions);
+        if (unsupported.isEmpty())
+        {
+            if (requiresAllowFilteringIfNotSpecified(table))
+                throw invalidRequest(allowFilteringMessage(state));
+        }
         else
-            nonPrimaryKeyRestrictions = nonPrimaryKeyRestrictions.addRestriction((SingleRestriction) restriction);
+        {
+            // If there's an index on these columns but the restriction is not supported on this index, throw a more specific error message
+            if (unsupported.size() == 1)
+                throw invalidRequest(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, unsupported.iterator().next()));
+            else
+                throw invalidRequest(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_MULTI, unsupported));
+        }
+    }
+
+    public void throwsRequiresIndexSupportingDisjunctionError()
+    {
+        throw invalidRequest(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_DISJUNCTION);
     }
 
     public void addFunctionsTo(List<Function> functions)
@@ -382,6 +875,9 @@ public void addFunctionsTo(List<Function> functions)
         partitionKeyRestrictions.addFunctionsTo(functions);
         clusteringColumnsRestrictions.addFunctionsTo(functions);
         nonPrimaryKeyRestrictions.addFunctionsTo(functions);
+
+        for (StatementRestrictions child : children)
+            child.addFunctionsTo(functions);
     }
 
     // may be used by QueryHandler implementations
@@ -407,20 +903,23 @@ public Set<ColumnMetadata> nonPKRestrictedColumns(boolean includeNotNullRestrict
 
         if (includeNotNullRestrictions)
         {
-            for (ColumnMetadata def : notNullColumns)
+            for (ColumnMetadata def : notNullColumns())
             {
                 if (!def.isPrimaryKeyColumn())
                     columns.add(def);
             }
         }
 
+        for (StatementRestrictions child : children)
+            columns.addAll(child.nonPKRestrictedColumns(includeNotNullRestrictions));
+
         return columns;
     }
 
     /**
      * @return the set of columns that have an IS NOT NULL restriction on them
      */
-    public Set<ColumnMetadata> notNullColumns()
+    public ImmutableSet<ColumnMetadata> notNullColumns()
     {
         return notNullColumns;
     }
@@ -430,10 +929,17 @@ public Set<ColumnMetadata> notNullColumns()
      */
     public boolean isRestricted(ColumnMetadata column)
     {
-        if (notNullColumns.contains(column))
+        if (notNullColumns().contains(column))
+            return true;
+
+        if (getRestrictions(column.kind).getColumnDefs().contains(column))
             return true;
 
-        return getRestrictions(column.kind).getColumnDefs().contains(column);
+        for (StatementRestrictions child : children)
+            if (child.isRestricted(column))
+                return true;
+
+        return false;
     }
 
     /**
@@ -454,7 +960,7 @@ public boolean keyIsInRelation()
      */
     public boolean isKeyRange()
     {
-        return this.isKeyRange;
+        return isKeyRange;
     }
 
     /**
@@ -494,7 +1000,7 @@ else if (column.kind == ColumnMetadata.Kind.CLUSTERING)
         {
             if (hasClusteringColumnsRestrictions())
             {
-                for (SingleRestriction restriction : clusteringColumnsRestrictions.getRestrictionSet())
+                for (SingleRestriction restriction : clusteringColumnsRestrictions.restrictions())
                 {
                     if (restriction.isEqualityBased())
                     {
@@ -512,7 +1018,7 @@ else if (restriction.getFirstColumn().name.equals(column.name))
         }
         else if (hasNonPrimaryKeyRestrictions())
         {
-            for (SingleRestriction restriction : nonPrimaryKeyRestrictions)
+            for (SingleRestriction restriction : nonPrimaryKeyRestrictions.restrictions())
                 if (restriction.getFirstColumn().name.equals(column.name) && restriction.isEqualityBased())
                     return true;
         }
@@ -520,17 +1026,13 @@ else if (hasNonPrimaryKeyRestrictions())
         return false;
     }
 
-    public boolean isTopK()
-    {
-        return nonPrimaryKeyRestrictions.hasAnn();
-    }
     /**
      * Returns the <code>Restrictions</code> for the specified type of columns.
      *
      * @param kind the column type
      * @return the <code>Restrictions</code> for the specified type of columns
      */
-    private Restrictions getRestrictions(ColumnMetadata.Kind kind)
+    protected Restrictions getRestrictions(ColumnMetadata.Kind kind)
     {
         switch (kind)
         {
@@ -547,84 +1049,14 @@ private Restrictions getRestrictions(ColumnMetadata.Kind kind)
      */
     public boolean usesSecondaryIndexing()
     {
-        return this.usesSecondaryIndexing;
-    }
-
-    /**
-     * This is a hack to push ordering down to indexes.
-     * Indexes are selected based on RowFilter only, so we need to turn orderings into restrictions
-     * so they end up in the row filter.
-     *
-     * @param orderings orderings from the select statement
-     * @return the {@link RestrictionSet} with the added orderings
-     */
-    private RestrictionSet addOrderingRestrictions(List<Ordering> orderings, RestrictionSet restrictionSet)
-    {
-        List<Ordering> annOrderings = orderings.stream().filter(o -> o.expression.hasNonClusteredOrdering()).collect(Collectors.toList());
-
-        if (annOrderings.size() > 1)
-            throw new InvalidRequestException("Cannot specify more than one ANN ordering");
-        else if (annOrderings.size() == 1)
-        {
-            if (orderings.size() > 1)
-                throw new InvalidRequestException("ANN ordering does not support any other ordering");
-            Ordering annOrdering = annOrderings.get(0);
-            if (annOrdering.direction != Ordering.Direction.ASC)
-                throw new InvalidRequestException("Descending ANN ordering is not supported");
-            SingleRestriction restriction = annOrdering.expression.toRestriction();
-            return restrictionSet.addRestriction(restriction);
-        }
-        return restrictionSet;
-    }
-
-    private static Iterable<Restriction> allColumnRestrictions(ClusteringColumnRestrictions clusteringColumnsRestrictions, RestrictionSet nonPrimaryKeyRestrictions)
-    {
-        return Iterables.concat(clusteringColumnsRestrictions.getRestrictionSet(), nonPrimaryKeyRestrictions);
-    }
-
-    private void processPartitionKeyRestrictions(ClientState state, boolean hasQueriableIndex, boolean allowFiltering, boolean forView)
-    {
-        if (!type.allowPartitionKeyRanges())
-        {
-            checkFalse(partitionKeyRestrictions.isOnToken(),
-                       "The token function cannot be used in WHERE clauses for %s statements", type);
-
-            if (partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(table))
-                throw invalidRequest("Some partition key parts are missing: %s",
-                                     Joiner.on(", ").join(getPartitionKeyUnrestrictedComponents()));
-
-            // slice query
-            checkFalse(partitionKeyRestrictions.hasSlice(),
-                    "Only EQ and IN relation are supported on the partition key (unless you use the token() function)"
-                            + " for %s statements", type);
-        }
-        else
-        {
-            // If there are no partition restrictions or there's only token restriction, we have to set a key range
-            if (partitionKeyRestrictions.isOnToken())
-                isKeyRange = true;
-
-            if (partitionKeyRestrictions.isEmpty() && partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(table))
-            {
-                isKeyRange = true;
-                usesSecondaryIndexing = hasQueriableIndex;
-            }
+        if (usesSecondaryIndexing)
+            return true;
 
-            // If there is a queriable index, no special condition is required on the other restrictions.
-            // But we still need to know 2 things:
-            // - If we don't have a queriable index, is the query ok
-            // - Is it queriable without 2ndary index, which is always more efficient
-            // If a component of the partition key is restricted by a relation, all preceding
-            // components must have a EQ. Only the last partition key component can be in IN relation.
-            if (partitionKeyRestrictions.needFiltering(table))
-            {
-                if (!allowFiltering && !forView && !hasQueriableIndex && requiresAllowFilteringIfNotSpecified())
-                    throw new InvalidRequestException(allowFilteringMessage(state));
+        for (StatementRestrictions child: children)
+            if (child.usesSecondaryIndexing)
+                return true;
 
-                isKeyRange = true;
-                usesSecondaryIndexing = hasQueriableIndex;
-            }
-        }
+        return false;
     }
 
     public boolean hasPartitionKeyRestrictions()
@@ -641,17 +1073,6 @@ public boolean hasNonPrimaryKeyRestrictions()
         return !nonPrimaryKeyRestrictions.isEmpty();
     }
 
-    /**
-     * Returns the partition key components that are not restricted.
-     * @return the partition key components that are not restricted.
-     */
-    private Collection<ColumnIdentifier> getPartitionKeyUnrestrictedComponents()
-    {
-        List<ColumnMetadata> list = new ArrayList<>(table.partitionKeyColumns());
-        list.removeAll(partitionKeyRestrictions.getColumnDefs());
-        return ColumnMetadata.toIdentifiers(list);
-    }
-
     /**
      * Checks if the restrictions on the partition key are token restrictions.
      *
@@ -674,74 +1095,6 @@ public boolean clusteringKeyRestrictionsHasIN()
         return clusteringColumnsRestrictions.hasIN();
     }
 
-    /**
-     * Processes the clustering column restrictions.
-     *
-     * @param hasQueriableIndex <code>true</code> if some of the queried data are indexed, <code>false</code> otherwise
-     * @param selectsOnlyStaticColumns <code>true</code> if the selected or modified columns are all statics,
-     * <code>false</code> otherwise.
-     */
-    private void processClusteringColumnsRestrictions(boolean hasQueriableIndex,
-                                                      boolean selectsOnlyStaticColumns,
-                                                      boolean forView,
-                                                      boolean allowFiltering)
-    {
-        checkFalse(!type.allowClusteringColumnSlices() && clusteringColumnsRestrictions.hasSlice(),
-                   "Slice restrictions are not supported on the clustering columns in %s statements", type);
-
-        if (!type.allowClusteringColumnSlices()
-            && (!table.isCompactTable() || (table.isCompactTable() && !hasClusteringColumnsRestrictions())))
-        {
-            if (!selectsOnlyStaticColumns && hasUnrestrictedClusteringColumns())
-                throw invalidRequest("Some clustering keys are missing: %s",
-                                     Joiner.on(", ").join(getUnrestrictedClusteringColumns()));
-        }
-        else
-        {
-            checkFalse(clusteringColumnsRestrictions.hasContains() && !hasQueriableIndex && !allowFiltering,
-                       "Clustering columns can only be restricted with CONTAINS with a secondary index or filtering");
-
-            if (hasClusteringColumnsRestrictions() && clusteringColumnsRestrictions.needFiltering())
-            {
-                if (hasQueriableIndex || forView)
-                {
-                    usesSecondaryIndexing = true;
-                }
-                else if (!allowFiltering)
-                {
-                    List<ColumnMetadata> clusteringColumns = table.clusteringColumns();
-                    List<ColumnMetadata> restrictedColumns = new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs());
-
-                    for (int i = 0, m = restrictedColumns.size(); i < m; i++)
-                    {
-                        ColumnMetadata clusteringColumn = clusteringColumns.get(i);
-                        ColumnMetadata restrictedColumn = restrictedColumns.get(i);
-
-                        if (!clusteringColumn.equals(restrictedColumn))
-                        {
-                            throw invalidRequest("PRIMARY KEY column \"%s\" cannot be restricted as preceding column \"%s\" is not restricted",
-                                                 restrictedColumn.name,
-                                                 clusteringColumn.name);
-                        }
-                    }
-                }
-            }
-
-        }
-
-    }
-
-    /**
-     * Returns the clustering columns that are not restricted.
-     * @return the clustering columns that are not restricted.
-     */
-    private Collection<ColumnIdentifier> getUnrestrictedClusteringColumns()
-    {
-        List<ColumnMetadata> missingClusteringColumns = new ArrayList<>(table.clusteringColumns());
-        missingClusteringColumns.removeAll(new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs()));
-        return ColumnMetadata.toIdentifiers(missingClusteringColumns);
-    }
-
     /**
      * Checks if some clustering columns are not restricted.
      * @return <code>true</code> if some clustering columns are not restricted, <code>false</code> otherwise.
@@ -751,54 +1104,31 @@ private boolean hasUnrestrictedClusteringColumns()
         return table.clusteringColumns().size() != clusteringColumnsRestrictions.size();
     }
 
-    private void processCustomIndexExpressions(List<CustomIndexExpression> expressions,
-                                               VariableSpecifications boundNames,
-                                               IndexRegistry indexRegistry)
+    public RowFilter getRowFilter(IndexRegistry indexManager, QueryOptions options, ClientState state, SelectOptions selectOptions)
     {
-        if (expressions.size() > 1)
-            throw new InvalidRequestException(IndexRestrictions.MULTIPLE_EXPRESSIONS);
-
-        CustomIndexExpression expression = expressions.get(0);
-
-        QualifiedName name = expression.targetIndex;
-
-        if (name.hasKeyspace() && !name.getKeyspace().equals(table.keyspace))
-            throw IndexRestrictions.invalidIndex(expression.targetIndex, table);
-
-        if (!table.indexes.has(expression.targetIndex.getName()))
-            throw IndexRestrictions.indexNotFound(expression.targetIndex, table);
-
-        Index index = indexRegistry.getIndex(table.indexes.get(expression.targetIndex.getName()).get());
-        if (!index.getIndexMetadata().isCustom())
-            throw IndexRestrictions.nonCustomIndexInExpression(expression.targetIndex);
-
-        AbstractType<?> expressionType = index.customExpressionValueType();
-        if (expressionType == null)
-            throw IndexRestrictions.customExpressionNotSupported(expression.targetIndex);
+        boolean hasAnnOptions = selectOptions.hasANNOptions();
 
-        expression.prepareValue(table, expressionType, boundNames);
-
-        filterRestrictions.add(expression);
-    }
+        if (filterRestrictions.isEmpty() && children.isEmpty())
+        {
+            if (hasAnnOptions)
+                throw new InvalidRequestException(ANN_OPTIONS_WITHOUT_ORDER_BY_ANN);
 
-    public RowFilter getRowFilter(IndexRegistry indexRegistry, QueryOptions options)
-    {
-        if (filterRestrictions.isEmpty())
             return RowFilter.none();
+        }
 
         // If there is only one replica, we don't need reconciliation at any consistency level.
         boolean needsReconciliation = !table.isVirtual()
                                       && options.getConsistency().needsReconciliation()
                                       && Keyspace.open(table.keyspace).getReplicationStrategy().getReplicationFactor().allReplicas > 1;
 
-        RowFilter filter = RowFilter.create(needsReconciliation);
-        for (Restrictions restrictions : filterRestrictions.getRestrictions())
-            restrictions.addToRowFilter(filter, indexRegistry, options);
+        ANNOptions annOptions = selectOptions.parseANNOptions();
+        RowFilter rowFilter = RowFilter.builder(needsReconciliation, indexManager)
+                                       .buildFromRestrictions(this, table, options, state, annOptions);
 
-        for (CustomIndexExpression expression : filterRestrictions.getCustomIndexExpressions())
-            expression.addToRowFilter(filter, table, options);
+        if (hasAnnOptions && !rowFilter.hasANN())
+            throw new InvalidRequestException(ANN_OPTIONS_WITHOUT_ORDER_BY_ANN);
 
-        return filter;
+        return rowFilter;
     }
 
     /**
@@ -849,7 +1179,7 @@ private AbstractBounds<PartitionPosition> getPartitionKeyBounds(IPartitioner p,
     {
         // Deal with unrestricted partition key components (special-casing is required to deal with 2i queries on the
         // first component of a composite partition key) queries that filter on the partition key.
-        if (partitionKeyRestrictions.needFiltering(table))
+        if (partitionKeyRestrictions.needFiltering(table) || isDisjunction)
             return new Range<>(p.getMinimumToken().minKeyBound(), p.getMinimumToken().maxKeyBound());
 
         ByteBuffer startKeyBytes = getPartitionKeyBound(Bound.START, options);
@@ -864,13 +1194,13 @@ private AbstractBounds<PartitionPosition> getPartitionKeyBounds(IPartitioner p,
         if (partitionKeyRestrictions.isInclusive(Bound.START))
         {
             return partitionKeyRestrictions.isInclusive(Bound.END)
-                    ? new Bounds<>(startKey, finishKey)
-                    : new IncludingExcludingBounds<>(startKey, finishKey);
+                   ? new Bounds<>(startKey, finishKey)
+                   : new IncludingExcludingBounds<>(startKey, finishKey);
         }
 
         return partitionKeyRestrictions.isInclusive(Bound.END)
-                ? new Range<>(startKey, finishKey)
-                : new ExcludingBounds<>(startKey, finishKey);
+               ? new Range<>(startKey, finishKey)
+               : new ExcludingBounds<>(startKey, finishKey);
     }
 
     private AbstractBounds<PartitionPosition> getPartitionKeyBoundsForTokenRestrictions(IPartitioner p,
@@ -924,6 +1254,21 @@ public boolean hasClusteringColumnsRestrictions()
         return !clusteringColumnsRestrictions.isEmpty();
     }
 
+    /**
+     * Checks if the query has any cluster column restrictions that do not also have a supporting index.
+     * @param table the table metadata
+     * @return <code>true</code> if the query has any cluster column restrictions that do not also have a supporting index,
+     * <code>false</code> otherwise.
+     */
+    public boolean hasClusterColumnRestrictionWithoutSupportingIndex(TableMetadata table)
+    {
+        IndexRegistry registry = IndexRegistry.obtain(table);
+        for (Restriction restriction : clusteringColumnsRestrictions.restrictions())
+            if (!restriction.hasSupportingIndex(registry))
+                return true;
+        return false;
+    }
+
     /**
      * Returns the requested clustering columns.
      *
@@ -954,6 +1299,11 @@ public NavigableSet<ClusteringBound<?>> getClusteringColumnsBounds(Bound b, Quer
         return clusteringColumnsRestrictions.boundsAsClustering(b, options);
     }
 
+    public boolean isDisjunction()
+    {
+        return isDisjunction;
+    }
+
     /**
      * Checks if the query returns a range of columns.
      *
@@ -961,41 +1311,85 @@ public NavigableSet<ClusteringBound<?>> getClusteringColumnsBounds(Bound b, Quer
      */
     public boolean isColumnRange()
     {
-        int numberOfClusteringColumns = table.clusteringColumns().size();
-        if (table.isStaticCompactTable())
-        {
-            // For static compact tables we want to ignore the fake clustering column (note that if we weren't special casing,
-            // this would mean a 'SELECT *' on a static compact table would query whole partitions, even though we'll only return
-            // the static part as far as CQL is concerned. This is thus mostly an optimization to use the query-by-name path).
-            numberOfClusteringColumns = 0;
-        }
-
+        // For static compact tables we want to ignore the fake clustering column (note that if we weren't special casing,
+        // this would mean a 'SELECT *' on a static compact table would query whole partitions, even though we'll only return
+        // the static part as far as CQL is concerned. This is thus mostly an optimization to use the query-by-name path).
+        int numberOfClusteringColumns = table.isStaticCompactTable() ? 0 : table.clusteringColumns().size();
         // it is a range query if it has at least one the column alias for which no relation is defined or is not EQ or IN.
         return clusteringColumnsRestrictions.size() < numberOfClusteringColumns
             || !clusteringColumnsRestrictions.hasOnlyEqualityRestrictions();
     }
 
     /**
-     * Checks if the query need to use filtering.
+     * Checks if the query needs to use filtering.
+     *
      * @return <code>true</code> if the query need to use filtering, <code>false</code> otherwise.
      */
     public boolean needFiltering(TableMetadata table)
     {
         IndexRegistry indexRegistry = IndexRegistry.obtain(table);
-        if (filterRestrictions.needsFiltering(indexRegistry))
+        boolean hasClusteringColumnRestrictions = !clusteringColumnsRestrictions.isEmpty();
+        boolean hasMultipleContains = nonPrimaryKeyRestrictions.hasMultipleContains();
+        if (filterRestrictions.needFiltering(indexRegistry, hasClusteringColumnRestrictions, hasMultipleContains))
             return true;
 
-        int numberOfRestrictions = filterRestrictions.getCustomIndexExpressions().size();
-        for (Restrictions restrictions : filterRestrictions.getRestrictions())
-            numberOfRestrictions += restrictions.size();
+        for (StatementRestrictions child : children)
+            if (child.needFiltering(table))
+                return true;
+
+        return false;
+    }
+
+    public boolean needsDisjunctionSupport(TableMetadata table)
+    {
+        boolean containsDisjunction = isDisjunction || !children.isEmpty();
+
+        if (!containsDisjunction)
+            return false;
+
+        IndexRegistry indexRegistry = IndexRegistry.obtain(table);
+
+        for (Index.Group group : indexRegistry.listIndexGroups())
+            if (filterRestrictions.indexBeingUsed(group) && !group.supportsDisjunction())
+                return true;
+
+        for (StatementRestrictions child : children)
+            if (child.needsDisjunctionSupport(table))
+                return true;
+
+        return false;
+    }
 
-        return numberOfRestrictions == 0 && !clusteringColumnsRestrictions.isEmpty();
+    private static Iterable<Restriction> allColumnRestrictions(ClusteringColumnRestrictions clusteringColumnsRestrictions, RestrictionSet nonPrimaryKeyRestrictions)
+    {
+        return Iterables.concat(clusteringColumnsRestrictions.restrictions(), nonPrimaryKeyRestrictions.restrictions());
     }
 
-    private void validateSecondaryIndexSelections()
+    private static Set<ColumnMetadata> getColumnsWithUnsupportedIndexRestrictions(TableMetadata table, Iterable<Restriction> restrictions)
     {
-        checkFalse(keyIsInRelation(),
-                   "Select on indexed columns and with IN clause for the PRIMARY KEY are not supported");
+        IndexRegistry indexRegistry = IndexRegistry.obtain(table);
+        if (indexRegistry.listIndexes().isEmpty())
+            return Collections.emptySet();
+
+        ImmutableSet.Builder<ColumnMetadata> builder = ImmutableSet.builder();
+
+        for (Restriction restriction : restrictions)
+        {
+            if (!restriction.hasSupportingIndex(indexRegistry))
+            {
+                for (Index index : indexRegistry.listIndexes())
+                {
+                    // If a column restriction has an index which was not picked up by hasSupportingIndex, it means it's an unsupported restriction
+                    for (ColumnMetadata column : restriction.getColumnDefs())
+                    {
+                        if (index.dependsOn(column))
+                            builder.add(column);
+                    }
+                }
+            }
+        }
+
+        return builder.build();
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TermSlice.java b/src/java/org/apache/cassandra/cql3/restrictions/TermSlice.java
index 100fcef64b5d..64606c66ef12 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/TermSlice.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/TermSlice.java
@@ -28,6 +28,12 @@
 
 final class TermSlice
 {
+    /**
+     * Represents a slice with no bounds.
+     * Can be merged with any other slice.
+     */
+    public static final TermSlice UNBOUNDED = new TermSlice(null, false, null, false);
+
     /**
      * The slice boundaries.
      */
@@ -108,21 +114,16 @@ public boolean isInclusive(Bound b)
      */
     public TermSlice merge(TermSlice otherSlice)
     {
-        if (hasBound(Bound.START))
-        {
-            assert !otherSlice.hasBound(Bound.START);
-
-            return new TermSlice(bound(Bound.START),
-                                  isInclusive(Bound.START),
-                                  otherSlice.bound(Bound.END),
-                                  otherSlice.isInclusive(Bound.END));
-        }
-        assert !otherSlice.hasBound(Bound.END);
-
-        return new TermSlice(otherSlice.bound(Bound.START),
-                              otherSlice.isInclusive(Bound.START),
-                              bound(Bound.END),
-                              isInclusive(Bound.END));
+        assert !(hasBound(Bound.START) && otherSlice.hasBound(Bound.START));
+        assert !(hasBound(Bound.END) && otherSlice.hasBound(Bound.END));
+
+        TermSlice sliceForStart = hasBound(Bound.START) ? this : otherSlice;
+        TermSlice sliceForEnd = hasBound(Bound.END) ? this : otherSlice;
+
+        return new TermSlice(sliceForStart.bound(Bound.START),
+                             sliceForStart.isInclusive(Bound.START),
+                             sliceForEnd.bound(Bound.END),
+                             sliceForEnd.isInclusive(Bound.END));
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java b/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
index 74ad7b51a4ce..bdcce9b05704 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
@@ -18,16 +18,16 @@
 package org.apache.cassandra.cql3.restrictions;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 
 import com.google.common.collect.BoundType;
 import com.google.common.collect.ImmutableRangeSet;
 import com.google.common.collect.Range;
 import com.google.common.collect.RangeSet;
 
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.statements.Bound;
@@ -35,8 +35,12 @@
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.db.filter.ANNOptions;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
 
 import static org.apache.cassandra.cql3.statements.Bound.END;
 import static org.apache.cassandra.cql3.statements.Bound.START;
@@ -47,36 +51,108 @@
  * <p>If all partition key columns have non-token restrictions and do not need filtering, they take precedence
  * when calculating bounds, incusiveness etc (see CASSANDRA-12149).</p>
  */
-final class TokenFilter implements PartitionKeyRestrictions
+abstract class TokenFilter implements PartitionKeyRestrictions
 {
     /**
      * The decorated restriction
      */
-    private final PartitionKeyRestrictions restrictions;
+    final PartitionKeyRestrictions restrictions;
 
     /**
      * The restriction on the token
      */
-    private final TokenRestriction tokenRestriction;
+    final TokenRestriction tokenRestriction;
 
     /**
      * Partitioner to manage tokens, extracted from tokenRestriction metadata.
      */
     private final IPartitioner partitioner;
 
-    public boolean hasIN()
+    static TokenFilter create(PartitionKeyRestrictions restrictions, TokenRestriction tokenRestriction)
     {
-        return isOnToken() ? false : restrictions.hasIN();
+        boolean onToken = restrictions.needFiltering(tokenRestriction.metadata) || restrictions.size() < tokenRestriction.size();
+        return onToken ? new TokenFilter.OnToken(restrictions, tokenRestriction)
+                       : new TokenFilter.NotOnToken(restrictions, tokenRestriction);
     }
 
-    public boolean hasContains()
+    private TokenFilter(PartitionKeyRestrictions restrictions, TokenRestriction tokenRestriction)
     {
-        return isOnToken() ? false : restrictions.hasContains();
+        this.restrictions = restrictions;
+        this.tokenRestriction = tokenRestriction;
+        this.partitioner = tokenRestriction.metadata.partitioner;
     }
 
-    public boolean hasOnlyEqualityRestrictions()
+    private static final class OnToken extends TokenFilter
     {
-        return isOnToken() ? false : restrictions.hasOnlyEqualityRestrictions();
+        private OnToken(PartitionKeyRestrictions restrictions, TokenRestriction tokenRestriction)
+        {
+            super(restrictions, tokenRestriction);
+        }
+
+        @Override
+        public boolean isOnToken()
+        {
+            return true;
+        }
+
+        @Override
+        public boolean isInclusive(Bound bound)
+        {
+            return tokenRestriction.isInclusive(bound);
+        }
+
+        @Override
+        public boolean hasBound(Bound bound)
+        {
+            return tokenRestriction.hasBound(bound);
+        }
+
+        @Override
+        public List<ByteBuffer> bounds(Bound bound, QueryOptions options) throws InvalidRequestException
+        {
+            return tokenRestriction.bounds(bound, options);
+        }
+    }
+
+    private static final class NotOnToken extends TokenFilter
+    {
+        private NotOnToken(PartitionKeyRestrictions restrictions, TokenRestriction tokenRestriction)
+        {
+            super(restrictions, tokenRestriction);
+        }
+
+        @Override
+        public boolean isInclusive(Bound bound)
+        {
+            return restrictions.isInclusive(bound);
+        }
+
+        @Override
+        public boolean hasBound(Bound bound)
+        {
+            return restrictions.hasBound(bound);
+        }
+
+        @Override
+        public List<ByteBuffer> bounds(Bound bound, QueryOptions options) throws InvalidRequestException
+        {
+            return restrictions.bounds(bound, options);
+        }
+
+        public boolean hasIN()
+        {
+            return restrictions.hasIN();
+        }
+
+        public boolean hasContains()
+        {
+            return restrictions.hasContains();
+        }
+
+        public boolean hasOnlyEqualityRestrictions()
+        {
+            return restrictions.hasOnlyEqualityRestrictions();
+        }
     }
 
     @Override
@@ -96,12 +172,6 @@ public boolean isOnToken()
         return needFiltering(tokenRestriction.metadata) || restrictions.size() < tokenRestriction.size();
     }
 
-    public TokenFilter(PartitionKeyRestrictions restrictions, TokenRestriction tokenRestriction)
-    {
-        this.restrictions = restrictions;
-        this.tokenRestriction = tokenRestriction;
-        this.partitioner = tokenRestriction.metadata.partitioner;
-    }
 
     @Override
     public List<ByteBuffer> values(QueryOptions options, ClientState state) throws InvalidRequestException
@@ -110,30 +180,12 @@ public List<ByteBuffer> values(QueryOptions options, ClientState state) throws I
     }
 
     @Override
-    public PartitionKeyRestrictions mergeWith(Restriction restriction) throws InvalidRequestException
+    public PartitionKeyRestrictions mergeWith(Restriction restriction, IndexRegistry indexRegistry) throws InvalidRequestException
     {
         if (restriction.isOnToken())
-            return new TokenFilter(restrictions, (TokenRestriction) tokenRestriction.mergeWith(restriction));
+            return TokenFilter.create(restrictions, (TokenRestriction) tokenRestriction.mergeWith(restriction, indexRegistry));
 
-        return new TokenFilter(restrictions.mergeWith(restriction), tokenRestriction);
-    }
-
-    @Override
-    public boolean isInclusive(Bound bound)
-    {
-        return isOnToken() ? tokenRestriction.isInclusive(bound) : restrictions.isInclusive(bound);
-    }
-
-    @Override
-    public boolean hasBound(Bound bound)
-    {
-        return isOnToken() ? tokenRestriction.hasBound(bound) : restrictions.hasBound(bound);
-    }
-
-    @Override
-    public List<ByteBuffer> bounds(Bound bound, QueryOptions options) throws InvalidRequestException
-    {
-        return isOnToken() ? tokenRestriction.bounds(bound, options) : restrictions.bounds(bound, options);
+        return TokenFilter.create(restrictions.mergeWith(restriction, indexRegistry), tokenRestriction);
     }
 
     /**
@@ -281,27 +333,15 @@ public boolean hasSupportingIndex(IndexRegistry indexRegistry)
     }
 
     @Override
-    public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
-    {
-        restrictions.addToRowFilter(filter, indexRegistry, options);
-    }
-
-    @Override
-    public Index findSupportingIndex(IndexRegistry indexRegistry)
-    {
-        return restrictions.findSupportingIndex(indexRegistry);
-    }
-
-    @Override
-    public Index findSupportingIndexFromQueryPlan(Index.QueryPlan indexQueryPlan)
+    public boolean needsFiltering(Index.Group indexGroup)
     {
-        return restrictions.findSupportingIndexFromQueryPlan(indexQueryPlan);
+        return restrictions.needsFiltering(indexGroup);
     }
 
     @Override
-    public boolean needsFiltering(Index.Group indexGroup)
+    public void addToRowFilter(RowFilter.Builder filter, IndexRegistry indexRegistry, QueryOptions options, ANNOptions annOptions)
     {
-        return restrictions.needsFiltering(indexGroup);
+        restrictions.addToRowFilter(filter, indexRegistry, options, annOptions);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
index bf23b336b6c3..dc1bd7d7191b 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
@@ -22,6 +22,7 @@
 
 import com.google.common.base.Joiner;
 
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
@@ -124,23 +125,11 @@ public boolean hasSupportingIndex(IndexRegistry indexRegistry)
     }
 
     @Override
-    public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+    public void addToRowFilter(RowFilter.Builder filter, IndexRegistry indexRegistry, QueryOptions options, ANNOptions annOptions)
     {
         throw new UnsupportedOperationException("Index expression cannot be created for token restriction");
     }
 
-    @Override
-    public Index findSupportingIndex(IndexRegistry indexRegistry)
-    {
-        return null;
-    }
-
-    @Override
-    public Index findSupportingIndexFromQueryPlan(Index.QueryPlan indexQueryPlan)
-    {
-        return null;
-    }
-
     @Override
     public boolean needsFiltering(Index.Group indexGroup)
     {
@@ -170,10 +159,10 @@ protected final String getColumnNamesAsString()
     }
 
     @Override
-    public final PartitionKeyRestrictions mergeWith(Restriction otherRestriction) throws InvalidRequestException
+    public final PartitionKeyRestrictions mergeWith(Restriction otherRestriction, IndexRegistry indexRegistry) throws InvalidRequestException
     {
         if (!otherRestriction.isOnToken())
-            return new TokenFilter(toPartitionKeyRestrictions(otherRestriction), this);
+            return TokenFilter.create(toPartitionKeyRestrictions(otherRestriction, indexRegistry), this);
 
         return doMergeWith((TokenRestriction) otherRestriction);
     }
@@ -191,12 +180,14 @@ public final PartitionKeyRestrictions mergeWith(Restriction otherRestriction) th
      * @return a <code>PartitionKeyRestrictions</code>
      * @throws InvalidRequestException if a problem occurs while converting the restriction
      */
-    private PartitionKeyRestrictions toPartitionKeyRestrictions(Restriction restriction) throws InvalidRequestException
+    private PartitionKeyRestrictions toPartitionKeyRestrictions(Restriction restriction, IndexRegistry indexRegistry) throws InvalidRequestException
     {
         if (restriction instanceof PartitionKeyRestrictions)
             return (PartitionKeyRestrictions) restriction;
 
-        return new PartitionKeySingleRestrictionSet(metadata.partitionKeyAsClusteringComparator()).mergeWith(restriction);
+        return PartitionKeySingleRestrictionSet.builder(metadata.partitionKeyAsClusteringComparator())
+                                               .addRestriction(restriction)
+                                               .build(indexRegistry);
     }
 
     public static final class EQRestriction extends TokenRestriction
diff --git a/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
index f7853aee1fe3..b37fbeb232c5 100644
--- a/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
@@ -208,8 +208,8 @@ private Selector createScalarSelector(QueryOptions options, ScalarFunction funct
                 // We have some terminal arguments, do a partial application
                 ScalarFunction partialFunction = function.partialApplication(version, terminalArgs);
 
-                // If all the arguments are terminal and the function is pure we can reduce to a simple value.
-                if (terminalCount == argSelectors.size() && fun.isPure())
+                // If all the arguments are terminal and the function is deterministic we can reduce to a simple value.
+                if (terminalCount == argSelectors.size() && fun.isDeterministic())
                 {
                     Arguments arguments = partialFunction.newArguments(version);
                     return new TermSelector(partialFunction.execute(arguments), partialFunction.returnType());
diff --git a/src/java/org/apache/cassandra/cql3/selection/ColumnFilterFactory.java b/src/java/org/apache/cassandra/cql3/selection/ColumnFilterFactory.java
index 63fa0520101e..00225cca4108 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ColumnFilterFactory.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ColumnFilterFactory.java
@@ -38,9 +38,21 @@ abstract class ColumnFilterFactory
      */
     abstract ColumnFilter newInstance(List<Selector> selectors);
 
-    public static ColumnFilterFactory wildcard(TableMetadata table)
+    public static ColumnFilterFactory wildcard(TableMetadata table, Set<ColumnMetadata> orderingColumns)
     {
-        return new PrecomputedColumnFilter(ColumnFilter.all(table));
+        ColumnFilter cf;
+        if (orderingColumns.isEmpty())
+        {
+            cf = ColumnFilter.all(table);
+        }
+        else
+        {
+            ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
+            builder.addAll(table.regularAndStaticColumns());
+            builder.addAll(orderingColumns);
+            cf = builder.build();
+        }
+        return new PrecomputedColumnFilter(cf);
     }
 
     public static ColumnFilterFactory fromColumns(TableMetadata table,
diff --git a/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java b/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java
index 4644ba2ec815..a6f8024f837c 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java
@@ -29,7 +29,10 @@
 import org.apache.cassandra.cql3.selection.SimpleSelector.SimpleSelectorFactory;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.rows.CellPath;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -78,9 +81,7 @@ public static AbstractType<?> valueType(CollectionType<?> type)
 
     private static CollectionType<?> getCollectionType(Selector selected)
     {
-        AbstractType<?> type = selected.getType();
-        if (type instanceof ReversedType)
-            type = ((ReversedType<?>) type).baseType;
+        AbstractType<?> type = selected.getType().unwrap();
 
         assert type instanceof MapType || type instanceof SetType : "this shouldn't have passed validation in Selectable";
 
@@ -421,7 +422,7 @@ public void addFetchedColumns(ColumnFilter.Builder builder)
 
         protected ByteBuffer extractSelection(ByteBuffer collection)
         {
-            return type.getSerializer().getSliceFromSerialized(collection, from, to, type.nameComparator(), type.isFrozenCollection());
+            return type.getSerializer().getSliceFromSerialized(collection, from, to, type.nameComparator(), !type.isMultiCell());
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java
index a777bc5feb7f..163496b963f1 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java
@@ -33,6 +33,7 @@
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.serializers.CollectionSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
 
@@ -102,7 +103,9 @@ public ByteBuffer getOutput(ProtocolVersion protocolVersion)
         {
             buffers.add(elements.get(i).getOutput(protocolVersion));
         }
-        return CollectionSerializer.pack(buffers, buffers.size());
+        return type.isVector()
+               ? ((VectorType<?>) type).decomposeRaw(buffers)
+               : CollectionSerializer.pack(buffers, buffers.size());
     }
 
     public void reset()
diff --git a/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java b/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java
index 9ab5ca0370cd..698c38141cf8 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.cql3.selection;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.cassandra.cql3.ResultSet;
@@ -34,7 +33,8 @@
 
 public final class ResultSetBuilder
 {
-    private final ResultSet resultSet;
+    private final ResultMetadata metadata;
+    private final SortedRowsBuilder rows;
 
     /**
      * As multiple thread can access a <code>Selection</code> instance each <code>ResultSetBuilder</code> will use
@@ -57,17 +57,21 @@ public final class ResultSetBuilder
      */
     private Selector.InputRow inputRow;
 
+    private boolean hasResults = false;
+    private int readRowsSize = 0;
+
     private long size = 0;
     private boolean sizeWarningEmitted = false;
 
     public ResultSetBuilder(ResultMetadata metadata, Selectors selectors, boolean unmask)
     {
-        this(metadata, selectors, unmask, null);
+        this(metadata, selectors, unmask, null, SortedRowsBuilder.create());
     }
 
-    public ResultSetBuilder(ResultMetadata metadata, Selectors selectors, boolean unmask, GroupMaker groupMaker)
+    public ResultSetBuilder(ResultMetadata metadata, Selectors selectors, boolean unmask, GroupMaker groupMaker, SortedRowsBuilder rows)
     {
-        this.resultSet = new ResultSet(metadata.copy(), new ArrayList<>());
+        this.metadata = metadata.copy();
+        this.rows = rows;
         this.selectors = selectors;
         this.groupMaker = groupMaker;
         this.unmask = unmask;
@@ -130,15 +134,10 @@ public void newRow(ProtocolVersion protocolVersion, DecoratedKey partitionKey, C
         if (inputRow != null)
         {
             selectors.addInputRow(inputRow);
+            inputRow.reset(!selectors.hasProcessing());
             if (isNewAggregate)
             {
-                resultSet.addRow(getOutputRow());
-                inputRow.reset(!selectors.hasProcessing());
-                selectors.reset();
-            }
-            else
-            {
-                inputRow.reset(!selectors.hasProcessing());
+                addRow();
             }
         }
         else
@@ -159,15 +158,22 @@ public ResultSet build()
         if (inputRow  != null)
         {
             selectors.addInputRow(inputRow);
-            resultSet.addRow(getOutputRow());
             inputRow.reset(!selectors.hasProcessing());
-            selectors.reset();
+            addRow();
         }
 
         // For aggregates we need to return a row even it no records have been found
-        if (resultSet.isEmpty() && groupMaker != null && groupMaker.returnAtLeastOneRow())
-            resultSet.addRow(getOutputRow());
-        return resultSet;
+        if (!hasResults && groupMaker != null && groupMaker.returnAtLeastOneRow())
+        {
+            addRow();
+        }
+
+        return new ResultSet(metadata, rows.build());
+    }
+
+    public int readRowsSize()
+    {
+        return readRowsSize;
     }
 
     private List<ByteBuffer> getOutputRow()
@@ -176,4 +182,19 @@ private List<ByteBuffer> getOutputRow()
         addSize(row);
         return row;
     }
+
+    private void addRow()
+    {
+        List<ByteBuffer> row = getOutputRow();
+        selectors.reset();
+
+        hasResults = true;
+        for (int i = 0, isize = row.size(); i < isize; i++)
+        {
+            ByteBuffer value = row.get(i);
+            readRowsSize += value != null ? value.remaining() : 0;
+        }
+
+        rows.add(row);
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
index 6df2b85b088d..f0171bb9663f 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
@@ -20,6 +20,7 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.functions.ScalarFunction;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -64,6 +65,11 @@ public ByteBuffer getOutput(ProtocolVersion protocolVersion)
     @Override
     public void validateForGroupBy()
     {
+        checkTrue(fun.isNative() || !DatabaseDescriptor.enableUserDefinedFunctionsThreads(),
+                  "User defined functions are not supported in the GROUP BY clause when asynchronous UDF execution " +
+                  "is enabled. Asynchronous UDF execution can be disabled by setting the configuration property " +
+                  "'enable_user_defined_functions_threads' to false in cassandra.yaml, with the security risks " +
+                  "described in the yaml file.");
         checkTrue(fun.isMonotonic(), "Only monotonic functions are supported in the GROUP BY clause. Got: %s ", fun);
         for (int i = 0, m = argSelectors.size(); i < m; i++)
             argSelectors.get(i).validateForGroupBy();
diff --git a/src/java/org/apache/cassandra/cql3/selection/Selectable.java b/src/java/org/apache/cassandra/cql3/selection/Selectable.java
index 56f7f8502560..674648f175e2 100644
--- a/src/java/org/apache/cassandra/cql3/selection/Selectable.java
+++ b/src/java/org/apache/cassandra/cql3/selection/Selectable.java
@@ -18,14 +18,47 @@
  */
 package org.apache.cassandra.cql3.selection;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.cql3.AssignmentTestable;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.Lists;
+import org.apache.cassandra.cql3.Maps;
+import org.apache.cassandra.cql3.Sets;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.Tuples;
+import org.apache.cassandra.cql3.UserTypes;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.Vectors;
+import org.apache.cassandra.cql3.functions.AggregateFcts;
+import org.apache.cassandra.cql3.functions.CastFcts;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionResolver;
+import org.apache.cassandra.cql3.functions.OperationFcts;
 import org.apache.cassandra.cql3.selection.Selector.Factory;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.DurationType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
@@ -36,7 +69,7 @@
 
 public interface Selectable extends AssignmentTestable
 {
-    public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames);
+    Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames);
 
     /**
      * The type of the {@code Selectable} if it can be infered.
@@ -48,14 +81,14 @@ public interface Selectable extends AssignmentTestable
      * literals, the exact type is not inferrable since they are valid for many
      * different types and so this will return {@code null} too).
      */
-    public AbstractType<?> getExactTypeIfKnown(String keyspace);
+    AbstractType<?> getExactTypeIfKnown(String keyspace);
 
     /**
      * Checks if this {@code Selectable} select columns matching the specified predicate.
      * @return {@code true} if this {@code Selectable} select columns matching the specified predicate,
      * {@code false} otherwise.
      */
-    public boolean selectColumns(Predicate<ColumnMetadata> predicate);
+    boolean selectColumns(Predicate<ColumnMetadata> predicate);
 
     /**
      * Checks if the specified Selectables select columns matching the specified predicate.
@@ -63,7 +96,7 @@ public interface Selectable extends AssignmentTestable
      * @return {@code true} if the specified Selectables select columns matching the specified predicate,
       {@code false} otherwise.
      */
-    public static boolean selectColumns(List<Selectable> selectables, Predicate<ColumnMetadata> predicate)
+    static boolean selectColumns(List<Selectable> selectables, Predicate<ColumnMetadata> predicate)
     {
         for (Selectable selectable : selectables)
         {
@@ -77,21 +110,21 @@ public static boolean selectColumns(List<Selectable> selectables, Predicate<Colu
      * Checks if any processing is performed on the selected columns, {@code false} otherwise.
      * @return {@code true} if any processing is performed on the selected columns, {@code false} otherwise.
      */
-    public default boolean processesSelection()
+    default boolean processesSelection()
     {
         // ColumnMetadata is the only case that returns false (if the column is not masked) and overrides this
         return true;
     }
 
     // Term.Raw overrides this since some literals can be WEAKLY_ASSIGNABLE
-    default public TestResult testAssignment(String keyspace, ColumnSpecification receiver)
+    default TestResult testAssignment(String keyspace, ColumnSpecification receiver)
     {
         AbstractType<?> type = getExactTypeIfKnown(keyspace);
         return type == null ? TestResult.NOT_ASSIGNABLE : type.testAssignment(keyspace, receiver);
     }
 
     @Override
-    public default AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+    default AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
     {
         return getExactTypeIfKnown(keyspace);
     }
@@ -118,12 +151,29 @@ default ColumnSpecification specForElementOrSlice(Selectable selected, ColumnSpe
         }
     }
 
-    public interface Raw
+    /**
+     * Checks that this {@code Selectable} is or can be converted into the specified type.
+     * @param table the table schema
+     * @param type the expected type
+     * @throws InvalidRequestException if the {@code Selectable} can not be converted into the specified type
+     */
+    default void validateType(TableMetadata table, AbstractType<?> type)
     {
-        public Selectable prepare(TableMetadata table);
+        ColumnSpecification receiver = new ColumnSpecification(table.keyspace,
+                                                               table.name,
+                                                               new ColumnIdentifier(toString(), true),
+                                                               type);
+
+        if (!testAssignment(table.keyspace, receiver).isAssignable())
+            throw invalidRequest("%s is not of the expected type: %s", this, type.asCQL3Type());
     }
 
-    public static class WithTerm implements Selectable
+    interface Raw
+    {
+        Selectable prepare(TableMetadata table);
+    }
+
+    class WithTerm implements Selectable
     {
         /**
          * The names given to unamed bind markers found in selection. In selection clause, we often don't have a good
@@ -138,7 +188,7 @@ public static class WithTerm implements Selectable
          */
         private static final ColumnIdentifier bindMarkerNameInSelection = new ColumnIdentifier("[selection]", true);
 
-        private final Term.Raw rawTerm;
+        public final Term.Raw rawTerm;
 
         public WithTerm(Term.Raw rawTerm)
         {
@@ -179,6 +229,8 @@ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?>
                 type = expectedType;
                 if (type == null)
                     throw new InvalidRequestException("Cannot infer type for term " + this + " in selection clause (try using a cast to force a type)");
+
+                validateType(table, type);
             }
 
             // The fact we default the name to "[selection]" inconditionally means that any bind marker in a
@@ -230,7 +282,7 @@ public Selectable prepare(TableMetadata table)
         }
     }
 
-    public static class WritetimeOrTTL implements Selectable
+    class WritetimeOrTTL implements Selectable
     {
         // The order of the variants in the Kind enum matters as they are used in ser/deser
         public enum Kind
@@ -297,7 +349,7 @@ public Selector.Factory newSelectorFactory(TableMetadata table,
         public AbstractType<?> getExactTypeIfKnown(String keyspace)
         {
             AbstractType<?> type = kind.returnType;
-            return column.type.isMultiCell() && !kind.aggregatesMultiCell() ? ListType.getInstance(type, false) : type;
+            return column.type.isMultiCell() && !kind.aggregatesMultiCell() ? ListType.getInstance(type.freeze(), false) : type;
         }
 
         @Override
@@ -327,7 +379,7 @@ public WritetimeOrTTL prepare(TableMetadata table)
         }
     }
 
-    public static class WithFunction implements Selectable
+    class WithFunction implements Selectable
     {
         public final Function function;
         public final List<Selectable> args;
@@ -425,7 +477,7 @@ public Selectable prepare(TableMetadata table)
         }
     }
 
-    public static class WithCast implements Selectable
+    class WithCast implements Selectable
     {
         private final CQL3Type type;
         private final Selectable arg;
@@ -497,7 +549,7 @@ public WithCast prepare(TableMetadata table)
     /**
      * Represents the selection of the field of a UDT (eg. t.f).
      */
-    public static class WithFieldSelection implements Selectable
+    class WithFieldSelection implements Selectable
     {
         public final Selectable selected;
         public final FieldIdentifier field;
@@ -549,7 +601,7 @@ public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?>
         public AbstractType<?> getExactTypeIfKnown(String keyspace)
         {
             AbstractType<?> selectedType = selected.getExactTypeIfKnown(keyspace);
-            if (selectedType == null || !(selectedType instanceof UserType))
+            if (!(selectedType instanceof UserType))
                 return null;
 
             UserType ut = (UserType) selectedType;
@@ -589,7 +641,7 @@ public WithFieldSelection prepare(TableMetadata table)
      * <p>The parser cannot differentiate between a single element between parentheses or a single element tuple.
      * By consequence, we are forced to wait until the type is known to be able to differentiate them.</p>
      */
-    public static class BetweenParenthesesOrWithTuple implements Selectable
+    class BetweenParenthesesOrWithTuple implements Selectable
     {
         /**
          * The tuple elements or the element between the parentheses
@@ -623,6 +675,7 @@ public Factory newSelectorFactory(TableMetadata cfm,
                 if (type == null)
                     throw invalidRequest("Cannot infer type for term %s in selection clause (try using a cast to force a type)",
                                          this);
+                validateType(cfm, type);
             }
 
             if (selectables.size() == 1 && !type.isTuple())
@@ -659,7 +712,7 @@ private Factory newTupleSelectorFactory(TableMetadata cfm,
                                                 VariableSpecifications boundNames)
         {
             SelectorFactories factories = createFactoriesAndCollectColumnDefinitions(selectables,
-                                                                                     tupleType.allTypes(),
+                                                                                     tupleType.subTypes,
                                                                                      cfm,
                                                                                      defs,
                                                                                      boundNames);
@@ -862,18 +915,6 @@ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
         {
             return Lists.getPreferredCompatibleType(selectables, p -> p.getCompatibleTypeIfKnown(keyspace));
         }
-
-        @Override
-        public boolean selectColumns(Predicate<ColumnMetadata> predicate)
-        {
-            return Selectable.selectColumns(selectables, predicate);
-        }
-
-        @Override
-        public String toString()
-        {
-            return Lists.listToString(selectables);
-        }
     }
 
     public static class WithVector extends WithArrayLiteral
@@ -931,24 +972,12 @@ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
         {
             return Vectors.getPreferredCompatibleType(selectables, p -> p.getCompatibleTypeIfKnown(keyspace));
         }
-
-        @Override
-        public boolean selectColumns(Predicate<ColumnMetadata> predicate)
-        {
-            return Selectable.selectColumns(selectables, predicate);
-        }
-
-        @Override
-        public String toString()
-        {
-            return Lists.listToString(selectables);
-        }
     }
 
     /**
      * <code>Selectable</code> for literal Sets.
      */
-    public static class WithSet implements Selectable
+    class WithSet implements Selectable
     {
         /**
          * The set elements
@@ -979,6 +1008,7 @@ public Factory newSelectorFactory(TableMetadata cfm,
                 if (type == null)
                     throw invalidRequest("Cannot infer type for term %s in selection clause (try using a cast to force a type)",
                                          this);
+                validateType(cfm, type);
             }
 
             // The parser treats empty Maps as Sets so if the type is a MapType we know that the Map is empty
@@ -1050,7 +1080,7 @@ public Selectable prepare(TableMetadata cfm)
      * {@code ColumnIdentifier} is equivalent to a {@code FieldIdentifier} from a syntax point of view.
      * By consequence, we are forced to wait until the type is known to be able to differentiate them.</p>
      */
-    public static class WithMapOrUdt implements Selectable
+    class WithMapOrUdt implements Selectable
     {
         /**
          * The column family metadata. We need to store them to be able to build the proper data once the type has been
@@ -1089,6 +1119,7 @@ public Factory newSelectorFactory(TableMetadata cfm,
                 if (type == null)
                     throw invalidRequest("Cannot infer type for term %s in selection clause (try using a cast to force a type)",
                                          this);
+                validateType(cfm, type);
             }
 
             if (type.isUDT())
@@ -1234,7 +1265,7 @@ public Selectable prepare(TableMetadata cfm)
     /**
      * <code>Selectable</code> for type hints (e.g. (int) ?).
      */
-    public static class WithTypeHint implements Selectable
+    class WithTypeHint implements Selectable
     {
 
         /**
@@ -1335,9 +1366,7 @@ public Raw( CQL3Type.Raw typeRaw, Selectable.Raw raw)
             public Selectable prepare(TableMetadata cfm)
             {
                 Selectable selectable = raw.prepare(cfm);
-                AbstractType<?> type = this.typeRaw.prepare(cfm.keyspace).getType();
-                if (type.isFreezable())
-                    type = type.freeze();
+                AbstractType<?> type = this.typeRaw.prepare(cfm.keyspace).getType().freeze();
                 return new WithTypeHint(typeRaw.toString(), type, selectable);
             }
         }
@@ -1348,7 +1377,7 @@ public Selectable prepare(TableMetadata cfm)
      * identifier have the same syntax. By consequence, we need to wait until the type is known to create the proper
      * Object: {@code ColumnMetadata} or {@code FieldIdentifier}.
      */
-    public static final class RawIdentifier implements Selectable.Raw
+    final class RawIdentifier implements Selectable.Raw
     {
         private final String text;
 
@@ -1403,7 +1432,7 @@ public String toString()
     /**
      * Represents the selection of an element of a collection (eg. c[x]).
      */
-    public static class WithElementSelection implements Selectable
+    class WithElementSelection implements Selectable
     {
         public final Selectable selected;
         // Note that we can't yet prepare the Term.Raw yet as we need the ColumnSpecificiation corresponding to Selectable, which
@@ -1430,28 +1459,24 @@ public Selector.Factory newSelectorFactory(TableMetadata cfm, AbstractType<?> ex
             Selector.Factory factory = selected.newSelectorFactory(cfm, null, defs, boundNames);
             ColumnSpecification receiver = factory.getColumnSpecification(cfm);
 
-            AbstractType<?> type = receiver.type;
-            if (receiver.isReversedType())
-            {
-                type = ((ReversedType<?>) type).baseType;
-            }
+            AbstractType<?> type = receiver.type.unwrap();
             if (!(type instanceof CollectionType))
                 throw new InvalidRequestException(String.format("Invalid element selection: %s is of type %s is not a collection", selected, type.asCQL3Type()));
 
-            ColumnSpecification boundSpec = specForElementOrSlice(selected, receiver, ((CollectionType) type).kind, "Element");
+            ColumnSpecification boundSpec = specForElementOrSlice(selected, receiver, ((CollectionType<?>) type).kind, "Element");
 
             Term elt = element.prepare(cfm.keyspace, boundSpec);
             elt.collectMarkerSpecification(boundNames);
-            return ElementsSelector.newElementFactory(toString(), factory, (CollectionType)type, elt);
+            return ElementsSelector.newElementFactory(toString(), factory, (CollectionType<?>)type, elt);
         }
 
         public AbstractType<?> getExactTypeIfKnown(String keyspace)
         {
             AbstractType<?> selectedType = selected.getExactTypeIfKnown(keyspace);
-            if (selectedType == null || !(selectedType instanceof CollectionType))
+            if (!(selectedType instanceof CollectionType))
                 return null;
 
-            return ElementsSelector.valueType((CollectionType) selectedType);
+            return ElementsSelector.valueType((CollectionType<?>) selectedType);
         }
 
         @Override
@@ -1487,7 +1512,7 @@ public String toString()
     /**
      * Represents the selection of a slice of a collection (eg. c[x..y]).
      */
-    public static class WithSliceSelection implements Selectable
+    class WithSliceSelection implements Selectable
     {
         public final Selectable selected;
         // Note that we can't yet prepare the Term.Raw yet as we need the ColumnSpecificiation corresponding to Selectable, which
@@ -1517,15 +1542,11 @@ public Selector.Factory newSelectorFactory(TableMetadata cfm, AbstractType<?> ex
             Selector.Factory factory = selected.newSelectorFactory(cfm, expectedType, defs, boundNames);
             ColumnSpecification receiver = factory.getColumnSpecification(cfm);
 
-            AbstractType<?> type = receiver.type;
-            if (receiver.isReversedType())
-            {
-                type = ((ReversedType<?>) type).baseType;
-            }
+            AbstractType<?> type = receiver.type.unwrap();
             if (!(type instanceof CollectionType))
                 throw new InvalidRequestException(String.format("Invalid slice selection: %s of type %s is not a collection", selected, type.asCQL3Type()));
 
-            ColumnSpecification boundSpec = specForElementOrSlice(selected, receiver, ((CollectionType) type).kind, "Slice");
+            ColumnSpecification boundSpec = specForElementOrSlice(selected, receiver, ((CollectionType<?>) type).kind, "Slice");
 
             // If from or to are null, this means the user didn't provide on in the syntax (we had c[x..] or c[..x]).
             // The equivalent of doing this when preparing values would be to use UNSET.
@@ -1533,13 +1554,13 @@ public Selector.Factory newSelectorFactory(TableMetadata cfm, AbstractType<?> ex
             Term t = to == null ? Constants.UNSET_VALUE : to.prepare(cfm.keyspace, boundSpec);
             f.collectMarkerSpecification(boundNames);
             t.collectMarkerSpecification(boundNames);
-            return ElementsSelector.newSliceFactory(toString(), factory, (CollectionType)type, f, t);
+            return ElementsSelector.newSliceFactory(toString(), factory, (CollectionType<?>) type, f, t);
         }
 
         public AbstractType<?> getExactTypeIfKnown(String keyspace)
         {
             AbstractType<?> selectedType = selected.getExactTypeIfKnown(keyspace);
-            if (selectedType == null || !(selectedType instanceof CollectionType))
+            if (!(selectedType instanceof CollectionType))
                 return null;
 
             return selectedType;
diff --git a/src/java/org/apache/cassandra/cql3/selection/Selection.java b/src/java/org/apache/cassandra/cql3/selection/Selection.java
index da87f2619a3c..7d8bb1d267e1 100644
--- a/src/java/org/apache/cassandra/cql3/selection/Selection.java
+++ b/src/java/org/apache/cassandra/cql3/selection/Selection.java
@@ -45,10 +45,24 @@ public abstract class Selection
     private static final Predicate<ColumnMetadata> STATIC_COLUMN_FILTER = (column) -> column.isStatic();
 
     private final TableMetadata table;
+
+    // Full list of columns needed for processing the query, including selected columns, ordering columns,
+    // and columns needed for restrictions.  Wildcard columns are fully materialized here.
+    //
+    // This also includes synthetic columns, because unlike all the other not-physical-columns selectables, they are
+    // computed on the replica instead of the coordinator and so, like physical columns, they need to be sent back
+    // as part of the result.
     private final List<ColumnMetadata> columns;
+
+    // maps ColumnSpecifications (columns, function calls, aliases) to the columns backing them
     private final SelectionColumnMapping columnMapping;
+
+    // metadata matching the ColumnSpcifications
     protected final ResultSet.ResultMetadata metadata;
+
+    // creates a ColumnFilter that breaks columns into `queried` and `fetched`
     protected final ColumnFilterFactory columnFilterFactory;
+
     protected final boolean isJson;
 
     // Columns used to order the result set for JSON queries with post ordering.
@@ -134,9 +148,19 @@ public ResultSet.ResultMetadata getResultMetadata()
 
     public static Selection wildcard(TableMetadata table, boolean isJson, boolean returnStaticContentOnPartitionWithNoRows)
     {
+        return wildcard(table, Collections.emptySet(), isJson, returnStaticContentOnPartitionWithNoRows);
+    }
+
+    public static Selection wildcard(TableMetadata table, Set<ColumnMetadata> orderingColumns, boolean isJson, boolean returnStaticContentOnPartitionWithNoRows)
+    {
+        // Add all table columns, but skip orderingColumns:
         List<ColumnMetadata> all = new ArrayList<>(table.columns().size());
         Iterators.addAll(all, table.allColumnsInSelectOrder());
-        return new SimpleSelection(table, all, Collections.emptySet(), true, isJson, returnStaticContentOnPartitionWithNoRows);
+
+        Set<ColumnMetadata> newOrderingColumns = new HashSet<>(orderingColumns);
+        all.forEach(newOrderingColumns::remove);
+
+        return new SimpleSelection(table, all, newOrderingColumns, true, isJson, returnStaticContentOnPartitionWithNoRows);
     }
 
     public static Selection wildcardWithGroupByOrMaskedColumns(TableMetadata table,
@@ -344,14 +368,14 @@ private static List<ByteBuffer> rowToJson(List<ByteBuffer> row,
         return Arrays.asList(jsonRow);
     }
 
-    public static interface Selectors
+    public interface Selectors
     {
         /**
          * Returns the {@code ColumnFilter} corresponding to those selectors
          *
          * @return the {@code ColumnFilter} corresponding to those selectors
          */
-        public ColumnFilter getColumnFilter();
+        ColumnFilter getColumnFilter();
 
         /**
          * Checks if this Selectors perform some processing
@@ -363,19 +387,19 @@ public static interface Selectors
          * Checks if one of the selectors perform some aggregations.
          * @return {@code true} if one of the selectors perform some aggregations, {@code false} otherwise.
          */
-        public boolean isAggregate();
+        boolean isAggregate();
 
         /**
          * Returns the number of fetched columns
          * @return the number of fetched columns
          */
-        public int numberOfFetchedColumns();
+        int numberOfFetchedColumns();
 
         /**
          * Checks if one of the selectors collect TTLs.
          * @return {@code true} if one of the selectors collect TTLs, {@code false} otherwise.
          */
-        public boolean collectTTLs();
+        boolean collectTTLs();
 
         /**
          * Checks if one of the selectors collects write timestamps.
@@ -390,9 +414,9 @@ public static interface Selectors
          */
         public void addInputRow(InputRow input);
 
-        public List<ByteBuffer> getOutputRow();
+        List<ByteBuffer> getOutputRow();
 
-        public void reset();
+        void reset();
     }
 
     // Special cased selection for when only columns are selected.
@@ -411,7 +435,7 @@ public SimpleSelection(TableMetadata table,
                  selectedColumns,
                  orderingColumns,
                  SelectionColumnMapping.simpleMapping(selectedColumns),
-                 isWildcard ? ColumnFilterFactory.wildcard(table)
+                 isWildcard ? ColumnFilterFactory.wildcard(table, orderingColumns)
                             : ColumnFilterFactory.fromColumns(table, selectedColumns, orderingColumns, Collections.emptySet(), returnStaticContentOnPartitionWithNoRows),
                  isWildcard,
                  isJson);
diff --git a/src/java/org/apache/cassandra/cql3/selection/SortedRowsBuilder.java b/src/java/org/apache/cassandra/cql3/selection/SortedRowsBuilder.java
new file mode 100644
index 000000000000..45de6d77b5e0
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/SortedRowsBuilder.java
@@ -0,0 +1,250 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import com.google.common.math.IntMath;
+
+import org.apache.cassandra.utils.TopKSelector;
+
+import static org.apache.cassandra.db.filter.DataLimits.NO_LIMIT;
+
+/**
+ * Builds a list of query result rows applying the specified order, limit and offset.
+ */
+public abstract class SortedRowsBuilder
+{
+    public final int limit;
+    public final int offset;
+    public final int fetchLimit; // limit + offset, saturated to Integer.MAX_VALUE
+
+    @SuppressWarnings("UnstableApiUsage")
+    private SortedRowsBuilder(int limit, int offset)
+    {
+        assert limit > 0 && offset >= 0;
+        this.limit = limit;
+        this.offset = offset;
+        this.fetchLimit = IntMath.saturatedAdd(limit, offset);
+    }
+
+    /**
+     * Adds the specified row to this builder. The row might be ignored if it's over the specified limit and offset.
+     *
+     * @param row the row to add
+     */
+    public abstract void add(List<ByteBuffer> row);
+
+    /**
+     * @return a list of query result rows based on the specified order, limit and offset.
+     */
+    public abstract List<List<ByteBuffer>> build();
+
+    /**
+     * Returns a new row builder that keeps insertion order.
+     *
+     * @return a rows builder that keeps insertion order.
+     */
+    public static SortedRowsBuilder create()
+    {
+        return new WithInsertionOrder(Integer.MAX_VALUE, 0);
+    }
+
+    /**
+     * Returns a new row builder that keeps insertion order.
+     *
+     * @param limit the query limit
+     * @param offset the query offset
+     * @return a rows builder that keeps insertion order.
+     */
+    public static SortedRowsBuilder create(int limit, int offset)
+    {
+        return new WithInsertionOrder(limit, offset);
+    }
+
+    /**
+     * Returns a new row builder that orders the added rows based on the specified {@link Comparator}.
+     *
+     * @param limit the query limit
+     * @param offset the query offset
+     * @param comparator the comparator to use for ordering
+     * @return a rows builder that orders results based on a comparator.
+     */
+    public static SortedRowsBuilder create(int limit, int offset, Comparator<List<ByteBuffer>> comparator)
+    {
+        return new WithHybridSort(limit, offset, comparator);
+    }
+
+    /**
+     * {@link SortedRowsBuilder} that keeps insertion order.
+     * </p>
+     * It keeps at most {@code limit} rows in memory.
+     */
+    private static class WithInsertionOrder extends SortedRowsBuilder
+    {
+        private final List<List<ByteBuffer>> rows = new ArrayList<>();
+        private int toSkip = offset;
+
+        private WithInsertionOrder(int limit, int offset)
+        {
+            super(limit, offset);
+        }
+
+        @Override
+        public void add(List<ByteBuffer> row)
+        {
+            if (toSkip-- <= 0 && rows.size() < limit)
+                rows.add(row);
+        }
+
+        @Override
+        public List<List<ByteBuffer>> build()
+        {
+            return rows;
+        }
+    }
+
+    /**
+     * {@link SortedRowsBuilder} that orders rows based on the provided comparator.
+     * </p>
+     * It simply stores all the rows in a list, and sorts and trims it when {@link #build()} is called. As such, it can
+     * consume a bunch of resources if the number of rows is high. However, it has good performance for cases where the
+     * number of rows is close to {@code limit + offset}, as it's the case of partition-directed queries.
+     */
+    public static class WithListSort extends SortedRowsBuilder
+    {
+        private final List<List<ByteBuffer>> rows = new ArrayList<>();
+        private final Comparator<List<ByteBuffer>> comparator;
+
+        private WithListSort(int limit,
+                             int offset,
+                             Comparator<List<ByteBuffer>> comparator)
+        {
+            super(limit, offset);
+            this.comparator = comparator;
+        }
+
+        @Override
+        public void add(List<ByteBuffer> row)
+        {
+            rows.add(row);
+        }
+
+        @Override
+        public List<List<ByteBuffer>> build()
+        {
+            rows.sort(comparator);
+            return rows.subList(Math.min(offset, rows.size()),
+                                Math.min(fetchLimit, rows.size()));
+        }
+    }
+
+    /**
+     * {@link SortedRowsBuilder} that orders rows based on the provided comparator.
+     * </p>
+     * It uses a heap to keep at most {@code limit + offset} rows in memory.
+     */
+    public static class WithHeapSort extends SortedRowsBuilder
+    {
+        private final TopKSelector<List<ByteBuffer>> heap;
+
+        private WithHeapSort(int limit, int offset, Comparator<List<ByteBuffer>> comparator)
+        {
+            super(limit, offset);
+            this.heap = new TopKSelector<>(comparator, fetchLimit);
+        }
+
+        @Override
+        public void add(List<ByteBuffer> row)
+        {
+            heap.add(row);
+        }
+
+        public void addAll(Iterable<List<ByteBuffer>> rows)
+        {
+            heap.addAll(rows);
+        }
+
+        @Override
+        public List<List<ByteBuffer>> build()
+        {
+            return heap.getSliced(offset);
+        }
+    }
+
+    /**
+     * {@link SortedRowsBuilder} that tries to combine the benefits of {@link WithListSort} and {@link WithHeapSort}.
+     * </p>
+     * {@link WithListSort} is faster for the first rows, but then it becomes slower than {@link WithHeapSort} as the
+     * number of rows grows. Also, {@link WithHeapSort} has constant {@code limit + offset} memory usage, whereas
+     * {@link WithListSort} memory usage grows linearly with the number of added rows.
+     * </p>
+     * This uses a {@link WithListSort} to sort the first {@code (limit + offset) * }{@link #SWITCH_FACTOR} rows,
+     * and then it switches to a {@link WithHeapSort} if more rows are added.
+     * </p>
+     * It keeps at most {@link #SWITCH_FACTOR} {@code * (limit + offset)} rows in memory.
+     */
+    public static class WithHybridSort extends SortedRowsBuilder
+    {
+        /**
+         * Factor of {@code limit + offset} at which we switch from list to heap.
+         */
+        public static final int SWITCH_FACTOR = 4;
+
+        private final int threshold; // at what number of rows we switch from list to heap, -1 means no switch
+
+        private WithListSort list;
+        private WithHeapSort heap;
+
+        @SuppressWarnings("UnstableApiUsage")
+        private WithHybridSort(int limit, int offset, Comparator<List<ByteBuffer>> comparator)
+        {
+            super(limit, offset);
+            this.list = new WithListSort(limit, offset, comparator);
+
+            // The heap approach is only useful when the limit is smaller than the number of collected rows.
+            // If there is no limit we will return all the collected rows, so we can simply use the list approach.
+            this.threshold = limit == NO_LIMIT ? -1 : IntMath.saturatedMultiply(fetchLimit, SWITCH_FACTOR);
+        }
+
+        @Override
+        public void add(List<ByteBuffer> row)
+        {
+            // start using the heap if the list is full
+            if (list != null && threshold > 0 && list.rows.size() >= threshold)
+            {
+                heap = new WithHeapSort(limit, offset, list.comparator);
+                heap.addAll(list.rows);
+                list = null;
+            }
+
+            if (list != null)
+                list.add(row);
+            else
+                heap.add(row);
+        }
+
+        @Override
+        public List<List<ByteBuffer>> build()
+        {
+            return list != null ? list.build() : heap.build();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java b/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
index 9be0b45d6ff4..637a918eef29 100644
--- a/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
@@ -70,7 +70,7 @@ protected String getColumnName()
             protected AbstractType<?> getReturnType()
             {
                 AbstractType<?> type = kind.returnType;
-                return isMultiCell && !kind.aggregatesMultiCell() ? ListType.getInstance(type, false) : type;
+                return isMultiCell && !kind.aggregatesMultiCell() ? ListType.getInstance(type.freeze(), false) : type;
             }
 
             @Override
@@ -117,6 +117,7 @@ public void addFetchedColumns(ColumnFilter.Builder builder)
         };
     }
 
+    @Override
     public void addFetchedColumns(ColumnFilter.Builder builder)
     {
         selected.addFetchedColumns(builder);
@@ -146,11 +147,13 @@ public void addInput(InputRow input)
         }
     }
 
+    @Override
     public ByteBuffer getOutput(ProtocolVersion protocolVersion)
     {
         return current;
     }
 
+    @Override
     public void reset()
     {
         selected.reset();
@@ -158,10 +161,11 @@ public void reset()
         current = null;
     }
 
+    @Override
     public AbstractType<?> getType()
     {
         AbstractType<?> type = kind.returnType;
-        return isMultiCell ? ListType.getInstance(type, false) : type;
+        return isMultiCell ? ListType.getInstance(type.freeze(), false) : type;
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterRoleStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterRoleStatement.java
index 74ec25f2df36..567d5c32a936 100644
--- a/src/java/org/apache/cassandra/cql3/statements/AlterRoleStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/AlterRoleStatement.java
@@ -54,6 +54,7 @@ public AlterRoleStatement(RoleName name, RoleOptions opts, DCPermissions dcPermi
         this.ifExists = ifExists;
     }
 
+    @Override
     public void validate(ClientState state) throws RequestValidationException
     {
         opts.validate();
diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
index a70a8891a607..3bfcabe5c1ea 100644
--- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
@@ -18,8 +18,20 @@
 package org.apache.cassandra.cql3.statements;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.HashMultiset;
@@ -31,28 +43,47 @@
 
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
+import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.BatchQueryOptions;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.ResultSet;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
 import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.exceptions.UnauthorizedException;
 import org.apache.cassandra.metrics.BatchMetrics;
 import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
-import org.apache.cassandra.service.*;
-import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.Pair;
 
 import static java.util.function.Predicate.isEqual;
-
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
 
 /**
@@ -65,6 +96,7 @@ public enum Type
         LOGGED, UNLOGGED, COUNTER
     }
 
+    private final String rawCQLStatement;
     public final Type type;
     private final VariableSpecifications bindVariables;
     private final List<ModificationStatement> statements;
@@ -82,10 +114,6 @@ public enum Type
 
     private static final Logger logger = LoggerFactory.getLogger(BatchStatement.class);
 
-    private static final String UNLOGGED_BATCH_WARNING = "Unlogged batch covering {} partitions detected " +
-                                                         "against table{} {}. You should use a logged batch for " +
-                                                         "atomicity, or asynchronous writes for performance.";
-
     private static final String LOGGED_BATCH_LOW_GCGS_WARNING = "Executing a LOGGED BATCH on table{} {}, configured with a " +
                                                                 "gc_grace_seconds of 0. The gc_grace_seconds is used to TTL " +
                                                                 "batchlog entries, so setting gc_grace_seconds too low on " +
@@ -101,8 +129,10 @@ public enum Type
      * @param statements the list of statements in the batch
      * @param attrs      additional attributes for statement (CL, timestamp, timeToLive)
      */
-    public BatchStatement(Type type, VariableSpecifications bindVariables, List<ModificationStatement> statements, Attributes attrs)
+    public BatchStatement(String queryString, Type type, VariableSpecifications bindVariables,
+                          List<ModificationStatement> statements, Attributes attrs)
     {
+        this.rawCQLStatement = queryString;
         this.type = type;
         this.bindVariables = bindVariables;
         this.statements = statements;
@@ -136,6 +166,12 @@ public BatchStatement(Type type, VariableSpecifications bindVariables, List<Modi
         this.updatesVirtualTables = updatesVirtualTables;
     }
 
+    @Override
+    public String getRawCQLStatement()
+    {
+        return rawCQLStatement;
+    }
+
     @Override
     public List<ColumnSpecification> getBindVariables()
     {
@@ -255,8 +291,12 @@ private boolean isLogged()
 
     // The batch itself will be validated in either Parsed#prepare() - for regular CQL3 batches,
     //   or in QueryProcessor.processBatch() - for native protocol batches.
+    @Override
     public void validate(ClientState state) throws InvalidRequestException
     {
+        if (isLogged())
+            Guardrails.loggedBatchEnabled.ensureEnabled(state);
+
         for (ModificationStatement statement : statements)
             statement.validate(state);
     }
@@ -328,16 +368,15 @@ public List<? extends IMutation> getMutations(ClientState state,
      *
      * @param mutations - the batch mutations.
      */
-    private static void verifyBatchSize(Collection<? extends IMutation> mutations) throws InvalidRequestException
+    private static void verifyBatchSize(Collection<? extends IMutation> mutations, ClientState clientState) throws InvalidRequestException
     {
         // We only warn for batch spanning multiple mutations (#10876)
         if (mutations.size() <= 1)
             return;
 
-        long warnThreshold = DatabaseDescriptor.getBatchSizeWarnThreshold();
         long size = IMutation.dataSize(mutations);
 
-        if (size > warnThreshold)
+        if (Guardrails.batchSize.triggersOn(size, clientState))
         {
             Set<String> tableNames = new HashSet<>();
             for (IMutation mutation : mutations)
@@ -346,27 +385,11 @@ private static void verifyBatchSize(Collection<? extends IMutation> mutations) t
                     tableNames.add(update.metadata().toString());
             }
 
-            long failThreshold = DatabaseDescriptor.getBatchSizeFailThreshold();
-
-            String format = "Batch for {} is of size {}, exceeding specified threshold of {} by {}.{}";
-            if (size > failThreshold)
-            {
-                Tracing.trace(format, tableNames, FBUtilities.prettyPrintMemory(size), FBUtilities.prettyPrintMemory(failThreshold),
-                              FBUtilities.prettyPrintMemory(size - failThreshold), " (see batch_size_fail_threshold)");
-                logger.error(format, tableNames, FBUtilities.prettyPrintMemory(size), FBUtilities.prettyPrintMemory(failThreshold),
-                             FBUtilities.prettyPrintMemory(size - failThreshold), " (see batch_size_fail_threshold)");
-                throw new InvalidRequestException("Batch too large");
-            }
-            else if (logger.isWarnEnabled())
-            {
-                logger.warn(format, tableNames, FBUtilities.prettyPrintMemory(size), FBUtilities.prettyPrintMemory(warnThreshold),
-                            FBUtilities.prettyPrintMemory(size - warnThreshold), "");
-            }
-            ClientWarn.instance.warn(MessageFormatter.arrayFormat(format, new Object[] {tableNames, size, warnThreshold, size - warnThreshold, ""}).getMessage());
+            Guardrails.batchSize.guard(size, tableNames.toString(), false, clientState);
         }
     }
 
-    private void verifyBatchType(Collection<? extends IMutation> mutations)
+    private void verifyBatchType(Collection<? extends IMutation> mutations, ClientState clientState)
     {
         if (!isLogged() && mutations.size() > 1)
         {
@@ -385,13 +408,9 @@ private void verifyBatchType(Collection<? extends IMutation> mutations)
 
             // CASSANDRA-11529: log only if we have more than a threshold of keys, this was also suggested in the
             // original ticket that introduced this warning, CASSANDRA-9282
-            if (keySet.size() > DatabaseDescriptor.getUnloggedBatchAcrossPartitionsWarnThreshold())
+            if (Guardrails.unloggedBatchAcrossPartitions.triggersOn(keySet.size(), clientState))
             {
-                NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, UNLOGGED_BATCH_WARNING,
-                                 keySet.size(), tableNames.size() == 1 ? "" : "s", tableNames);
-
-                ClientWarn.instance.warn(MessageFormatter.arrayFormat(UNLOGGED_BATCH_WARNING, new Object[]{keySet.size(),
-                                                    tableNames.size() == 1 ? "" : "s", tableNames}).getMessage());
+                Guardrails.unloggedBatchAcrossPartitions.guard(keySet.size(), tableNames.toString(), false, clientState);
             }
         }
     }
@@ -407,17 +426,23 @@ public ResultMessage execute(QueryState queryState, BatchQueryOptions options, D
         long timestamp = options.getTimestamp(queryState);
         long nowInSeconds = options.getNowInSeconds(queryState);
 
-        if (options.getConsistency() == null)
+        ConsistencyLevel cl = options.getConsistency();
+        if (cl == null)
             throw new InvalidRequestException("Invalid empty consistency level");
-        if (options.getSerialConsistency() == null)
+
+        if (options.getSerialConsistency(queryState) == null)
             throw new InvalidRequestException("Invalid empty serial consistency level");
 
         ClientState clientState = queryState.getClientState();
-        Guardrails.writeConsistencyLevels.guard(EnumSet.of(options.getConsistency(), options.getSerialConsistency()),
+        Guardrails.writeConsistencyLevels.guard(EnumSet.of(options.getConsistency(), options.getSerialConsistency(queryState)),
                                                 clientState);
 
         for (int i = 0; i < statements.size(); i++ )
-            statements.get(i).validateDiskUsage(options.forStatement(i), clientState);
+        {
+            ModificationStatement statement = statements.get(i);
+            statement.validateConsistency(cl, clientState);
+            statement.validateDiskUsage(options.forStatement(i), clientState);
+        }
 
         if (hasConditions)
             return executeWithConditions(options, queryState, requestTime);
@@ -426,35 +451,53 @@ public ResultMessage execute(QueryState queryState, BatchQueryOptions options, D
             executeInternalWithoutCondition(queryState, options, requestTime);
         else    
             executeWithoutConditions(getMutations(clientState, options, false, timestamp, nowInSeconds, requestTime),
-                                     options.getConsistency(), requestTime);
+                                     clientState, options.getConsistency(), requestTime);
+
+        ResultMessage<ResultMessage.Void> result = new ResultMessage.Void();
+        RequestSensors sensors = RequestTracker.instance.get();
+        Map<TableId, TableMetadata> tableMetadataById = statements.stream()
+                                                                  .map(ModificationStatement::metadata)
+                                                                  .collect(Collectors.toMap(metadata -> metadata.id, Function.identity(), (existing, replacement) -> existing));
+        for (TableMetadata metadata : tableMetadataById.values())
+        {
+            Context context = Context.from(metadata);
+            SensorsCustomParams.addSensorToCQLResponse(result, options.wrapped.getProtocolVersion(), sensors, context, org.apache.cassandra.sensors.Type.WRITE_BYTES);
+        }
 
-        return new ResultMessage.Void();
+        return result;
     }
 
-    private void executeWithoutConditions(List<? extends IMutation> mutations, ConsistencyLevel cl, Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException
+    private void executeWithoutConditions(List<? extends IMutation> mutations,
+                                          ClientState clientState,
+                                          ConsistencyLevel cl,
+                                          Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException
     {
         if (mutations.isEmpty())
             return;
 
-        verifyBatchSize(mutations);
-        verifyBatchType(mutations);
+        verifyBatchSize(mutations, clientState);
+        verifyBatchType(mutations, clientState);
 
-        updatePartitionsPerBatchMetrics(mutations.size());
+        updatePerBatchMetrics(mutations);
 
         boolean mutateAtomic = (isLogged() && mutations.size() > 1);
-        StorageProxy.mutateWithTriggers(mutations, cl, mutateAtomic, requestTime);
+        StorageProxy.mutateWithTriggers(mutations, cl, mutateAtomic, requestTime, clientState);
         ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(mutations);
     }
 
-    private void updatePartitionsPerBatchMetrics(int updatedPartitions)
+    private void updatePerBatchMetrics(Collection<? extends IMutation> mutations)
     {
-        if (isLogged()) {
-            metrics.partitionsPerLoggedBatch.update(updatedPartitions);
-        } else if (isCounter()) {
-            metrics.partitionsPerCounterBatch.update(updatedPartitions);
-        } else {
-            metrics.partitionsPerUnloggedBatch.update(updatedPartitions);
+        int nrUpdatedPartitions = mutations.size();
+        int nrUpdatedColumns = 0;
+        for (IMutation mutation : mutations)
+        {
+            for (PartitionUpdate update : mutation.getPartitionUpdates())
+            {
+                for (Row row : update.rows())
+                    nrUpdatedColumns += row.columns().size();
+            }
         }
+        metrics.update(type, nrUpdatedPartitions, nrUpdatedColumns);
     }
 
     private ResultMessage executeWithConditions(BatchQueryOptions options, QueryState state, Dispatcher.RequestTime requestTime)
@@ -470,7 +513,7 @@ private ResultMessage executeWithConditions(BatchQueryOptions options, QueryStat
                                                    tableName,
                                                    casRequest.key,
                                                    casRequest,
-                                                   options.getSerialConsistency(),
+                                                   options.getSerialConsistency(state),
                                                    options.getConsistency(),
                                                    state.getClientState(),
                                                    options.getNowInSeconds(state),
@@ -608,7 +651,7 @@ public String toString()
         return String.format("BatchStatement(type=%s, statements=%s)", type, statements);
     }
 
-    public static class Parsed extends QualifiedStatement
+    public static class Parsed extends RawKeyspaceAwareStatement<BatchStatement>
     {
         private final Type type;
         private final Attributes.Raw attrs;
@@ -616,45 +659,35 @@ public static class Parsed extends QualifiedStatement
 
         public Parsed(Type type, Attributes.Raw attrs, List<ModificationStatement.Parsed> parsedStatements)
         {
-            super(null);
             this.type = type;
             this.attrs = attrs;
             this.parsedStatements = parsedStatements;
         }
 
-        // Not doing this in the constructor since we only need this for prepared statements
-        @Override
-        public boolean isFullyQualified()
+        private void setKeyspace(ClientState state) throws InvalidRequestException
         {
             for (ModificationStatement.Parsed statement : parsedStatements)
-                if (!statement.isFullyQualified())
-                    return false;
-
-            return true;
+                statement.setKeyspace(state);
         }
 
-        @Override
-        public void setKeyspace(ClientState state) throws InvalidRequestException
+        public void setKeyspace(Function<ModificationStatement.Parsed, String> convertKeyspace) throws InvalidRequestException
         {
             for (ModificationStatement.Parsed statement : parsedStatements)
-                statement.setKeyspace(state);
+                statement.setKeyspace(convertKeyspace.apply(statement));
         }
 
         @Override
-        public String keyspace()
+        public BatchStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            return null;
-        }
+            setKeyspace(state);
 
-        public BatchStatement prepare(ClientState state)
-        {
             List<ModificationStatement> statements = new ArrayList<>(parsedStatements.size());
-            parsedStatements.forEach(s -> statements.add(s.prepare(state, bindVariables)));
+            parsedStatements.forEach(s -> statements.add(s.prepare(state, bindVariables, keyspaceMapper)));
 
             Attributes prepAttrs = attrs.prepare("[batch]", "[batch]");
             prepAttrs.collectMarkerSpecification(bindVariables);
 
-            BatchStatement batchStatement = new BatchStatement(type, bindVariables, statements, prepAttrs);
+            BatchStatement batchStatement = new BatchStatement(rawCQLStatement, type, bindVariables, statements, prepAttrs);
             batchStatement.validate();
 
             return batchStatement;
diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java
index 521cd2afa6e2..8c97846e9d04 100644
--- a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java
+++ b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java
@@ -98,7 +98,7 @@ public PartitionUpdate.Builder getPartitionUpdateBuilder(TableMetadata metadata,
         {
             RegularAndStaticColumns columns = updatedColumns.get(metadata.id);
             assert columns != null;
-            upd = new PartitionUpdate.Builder(metadata, dk, columns, perPartitionKeyCounts.get(metadata.id).count(dk.getKey()));
+            upd = PartitionUpdate.builder(metadata, dk, columns, perPartitionKeyCounts.get(metadata.id).count(dk.getKey()));
             mut.add(upd);
         }
         return upd;
diff --git a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
index 9671592c16b6..4802602b3262 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
@@ -233,9 +233,10 @@ private RegularAndStaticColumns updatedColumns()
         return builder.build();
     }
 
+    @Override
     public PartitionUpdate makeUpdates(FilteredPartition current, ClientState clientState, Ballot ballot) throws InvalidRequestException
     {
-        PartitionUpdate.Builder updateBuilder = new PartitionUpdate.Builder(metadata, key, updatedColumns(), conditions.size());
+        PartitionUpdate.Builder updateBuilder = PartitionUpdate.builder(metadata, key, updatedColumns(), conditions.size());
         long timeUuidNanos = 0;
         for (RowUpdate upd : updates)
             timeUuidNanos = upd.applyUpdates(current, updateBuilder, clientState, ballot.msb(), timeUuidNanos);
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateRoleStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateRoleStatement.java
index d6e0a1298cde..dd48e6dc2472 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CreateRoleStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CreateRoleStatement.java
@@ -57,6 +57,7 @@ public void authorize(ClientState state) throws UnauthorizedException
         }
     }
 
+    @Override
     public void validate(ClientState state) throws RequestValidationException
     {
         opts.validate();
diff --git a/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java b/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
index 7a50a9b015bd..69a3404c5101 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
@@ -20,9 +20,19 @@
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Operation;
+import org.apache.cassandra.cql3.Operations;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.UpdateParameters;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.WhereClause;
 import org.apache.cassandra.cql3.conditions.ColumnCondition;
 import org.apache.cassandra.cql3.conditions.Conditions;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
@@ -34,8 +44,6 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.Pair;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
 
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
@@ -45,14 +53,15 @@
  */
 public class DeleteStatement extends ModificationStatement
 {
-    private DeleteStatement(VariableSpecifications bindVariables,
+    private DeleteStatement(String queryString,
+                            VariableSpecifications bindVariables,
                             TableMetadata cfm,
                             Operations operations,
                             StatementRestrictions restrictions,
                             Conditions conditions,
                             Attributes attrs)
     {
-        super(StatementType.DELETE, bindVariables, cfm, operations, restrictions, conditions, attrs);
+        super(queryString, StatementType.DELETE, bindVariables, cfm, operations, restrictions, conditions, attrs);
     }
 
     @Override
@@ -171,7 +180,8 @@ protected ModificationStatement prepareInternal(ClientState state,
                                                                  conditions,
                                                                  Collections.emptyList());
 
-            DeleteStatement stmt = new DeleteStatement(bindVariables,
+            DeleteStatement stmt = new DeleteStatement(rawCQLStatement,
+                                                       bindVariables,
                                                        metadata,
                                                        operations,
                                                        restrictions,
diff --git a/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java b/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java
index 9d886455e530..344730e43929 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java
@@ -158,14 +158,17 @@ public ResultMessage executeLocally(QueryState state, QueryOptions options)
         //
 
         long offset = getOffset(pagingState, schema.getVersion());
-        int pageSize = options.getPageSize();
+        PageSize pageSize = options.getPageSize();
+
+        if (pageSize.isDefined() && pageSize.getUnit() != PageSize.PageUnit.ROWS)
+            throw new InvalidRequestException("Paging in bytes is not supported for describe statement. Please specify the page size in rows.");
 
         Stream<? extends T> stream = describe(state.getClientState(), keyspaces);
 
         if (offset > 0L)
             stream = stream.skip(offset);
-        if (pageSize > 0)
-            stream = stream.limit(pageSize);
+        if (pageSize.isDefined())
+            stream = stream.limit(pageSize.getSize());
 
         List<List<ByteBuffer>> rows = stream.map(e -> toRow(e, includeInternalDetails))
                                             .collect(Collectors.toList());
@@ -173,8 +176,10 @@ public ResultMessage executeLocally(QueryState state, QueryOptions options)
         ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(metadata(state.getClientState()));
         ResultSet result = new ResultSet(resultMetadata, rows);
 
-        if (pageSize > 0 && rows.size() == pageSize)
-            result.metadata.setHasMorePages(getPagingState(offset + pageSize, schema.getVersion()));
+        if (pageSize.isDefined() && rows.size() == pageSize.getSize())
+        {
+            result.metadata.setHasMorePages(getPagingState(offset + pageSize.getSize(), schema.getVersion()));
+        }
 
         return new ResultMessage.Rows(result);
     }
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropRoleStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropRoleStatement.java
index 13ba54a52d9d..7480021d2509 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DropRoleStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DropRoleStatement.java
@@ -51,6 +51,7 @@ public void authorize(ClientState state) throws UnauthorizedException
             throw new UnauthorizedException("Only superusers can drop a role with superuser status");
     }
 
+    @Override
     public void validate(ClientState state) throws RequestValidationException
     {
         // validate login here before authorize to avoid leaking user existence to anonymous users.
diff --git a/src/java/org/apache/cassandra/cql3/statements/ListPermissionsStatement.java b/src/java/org/apache/cassandra/cql3/statements/ListPermissionsStatement.java
index 4b5aa601e2ab..b95b85061b0c 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ListPermissionsStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ListPermissionsStatement.java
@@ -64,6 +64,7 @@ public ListPermissionsStatement(Set<Permission> permissions, IResource resource,
         this.grantee = grantee.hasName()? RoleResource.role(grantee.getName()) : null;
     }
 
+    @Override
     public void validate(ClientState state) throws RequestValidationException
     {
         // a check to ensure the existence of the user isn't being leaked by user existence check.
diff --git a/src/java/org/apache/cassandra/cql3/statements/ListRolesStatement.java b/src/java/org/apache/cassandra/cql3/statements/ListRolesStatement.java
index 8a75f8a6c36a..fe4e97986e88 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ListRolesStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ListRolesStatement.java
@@ -67,6 +67,7 @@ public ListRolesStatement(RoleName grantee, boolean recursive)
         this.recursive = recursive;
     }
 
+    @Override
     public void validate(ClientState state) throws UnauthorizedException, InvalidRequestException
     {
         state.ensureNotAnonymous();
diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
index a538348ceab5..fa8b48fde13d 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
@@ -18,7 +18,20 @@
 package org.apache.cassandra.cql3.statements;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.StringJoiner;
+import java.util.function.UnaryOperator;
 
 import com.google.common.collect.HashMultiset;
 import com.google.common.collect.Iterables;
@@ -26,16 +39,26 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.Operation;
+import org.apache.cassandra.cql3.Operations;
+import org.apache.cassandra.cql3.Ordering;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.ResultSet;
+import org.apache.cassandra.cql3.UpdateParameters;
+import org.apache.cassandra.cql3.Validation;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.WhereClause;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.index.sai.analyzer.AnalyzerEqOperatorSupport;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.ViewMetadata;
-import org.apache.cassandra.cql3.*;
 import org.apache.cassandra.cql3.conditions.ColumnCondition;
 import org.apache.cassandra.cql3.conditions.ColumnConditions;
 import org.apache.cassandra.cql3.conditions.Conditions;
@@ -44,15 +67,45 @@
 import org.apache.cassandra.cql3.selection.ResultSetBuilder;
 import org.apache.cassandra.cql3.selection.Selection;
 import org.apache.cassandra.cql3.selection.Selection.Selectors;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBuilder;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.BooleanType;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.exceptions.UnauthorizedException;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaLayout;
 import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster;
@@ -84,6 +137,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
 
     private static final ColumnIdentifier CAS_RESULT_COLUMN = new ColumnIdentifier("[applied]", false);
 
+    private final String rawCQLStatement;
+
     protected final StatementType type;
 
     protected final VariableSpecifications bindVariables;
@@ -103,7 +158,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
 
     private final RegularAndStaticColumns requiresRead;
 
-    public ModificationStatement(StatementType type,
+    public ModificationStatement(String queryString,
+                                 StatementType type,
                                  VariableSpecifications bindVariables,
                                  TableMetadata metadata,
                                  Operations operations,
@@ -111,6 +167,7 @@ public ModificationStatement(StatementType type,
                                  Conditions conditions,
                                  Attributes attrs)
     {
+        this.rawCQLStatement = queryString;
         this.type = type;
         this.bindVariables = bindVariables;
         this.metadata = metadata;
@@ -158,6 +215,12 @@ public ModificationStatement(StatementType type,
         this.requiresRead = requiresReadBuilder.build();
     }
 
+    @Override
+    public String getRawCQLStatement()
+    {
+        return rawCQLStatement;
+    }
+
     @Override
     public List<ColumnSpecification> getBindVariables()
     {
@@ -252,22 +315,13 @@ public void authorize(ClientState state) throws InvalidRequestException, Unautho
         if (hasConditions())
             state.ensureTablePermission(metadata, Permission.SELECT);
 
-        // MV updates need to get the current state from the table, and might update the views
-        // Require Permission.SELECT on the base table, and Permission.MODIFY on the views
-        Iterator<ViewMetadata> views = View.findAll(keyspace(), table()).iterator();
-        if (views.hasNext())
-        {
-            state.ensureTablePermission(metadata, Permission.SELECT);
-            do
-            {
-                state.ensureTablePermission(views.next().metadata, Permission.MODIFY);
-            } while (views.hasNext());
-        }
+        // Modification on base table with MV should skip SELECT access control to base table and WRITE access control to view table.
 
         for (Function function : getFunctions())
             state.ensurePermission(Permission.EXECUTE, function);
     }
 
+    @Override
     public void validate(ClientState state) throws InvalidRequestException
     {
         checkFalse(hasConditions() && attrs.isTimestampSet(), "Cannot provide custom timestamp for conditional updates");
@@ -278,8 +332,19 @@ public void validate(ClientState state) throws InvalidRequestException
         checkFalse(isVirtual() && attrs.isTimeToLiveSet(), "Expiring columns are not supported by virtual tables");
         checkFalse(isVirtual() && hasConditions(), "Conditional updates are not supported by virtual tables");
 
-        if (attrs.isTimestampSet())
+        // there are system queries with USING TIMESTAMP, e.g. SchemaKeyspace#saveSystemKeyspacesSchema
+        if (SchemaConstants.isUserKeyspace(metadata.keyspace) && attrs.isTimestampSet())
             Guardrails.userTimestampsEnabled.ensureEnabled(state);
+
+        // Warn but otherwise accept conditions on analyzed columns. The analyzers won't be used (see CNDB-11658).
+        IndexRegistry indexRegistry = IndexRegistry.obtain(metadata);
+        Set<ColumnMetadata> analyzedColumns = conditions.getAnalyzedColumns(indexRegistry);
+        if (!analyzedColumns.isEmpty())
+        {
+            StringJoiner joiner = new StringJoiner(", ");
+            analyzedColumns.forEach(c -> joiner.add(c.name.toString()));
+            ClientWarn.instance.warn(String.format(AnalyzerEqOperatorSupport.LWT_CONDITION_ON_ANALYZED_WARNING, joiner));
+        }
     }
 
     public void validateDiskUsage(QueryOptions options, ClientState state)
@@ -380,7 +445,7 @@ public NavigableSet<Clustering<?>> createClustering(QueryOptions options, Client
     throws InvalidRequestException
     {
         if (appliesOnlyToStaticColumns() && !restrictions.hasClusteringColumnsRestrictions())
-            return FBUtilities.singleton(CBuilder.STATIC_BUILDER.build(), metadata().comparator);
+            return FBUtilities.singleton(ClusteringBuilder.STATIC_BUILDER.build(), metadata().comparator);
 
         return restrictions.getClusteringColumns(options, state);
     }
@@ -418,6 +483,7 @@ public boolean requiresRead()
 
     private Map<DecoratedKey, Partition> readRequiredLists(Collection<ByteBuffer> partitionKeys,
                                                            ClusteringIndexFilter filter,
+                                                           ClientState state,
                                                            DataLimits limits,
                                                            boolean local,
                                                            ConsistencyLevel cl,
@@ -457,7 +523,7 @@ private Map<DecoratedKey, Partition> readRequiredLists(Collection<ByteBuffer> pa
             }
         }
 
-        try (PartitionIterator iter = group.execute(cl, null, requestTime))
+        try (PartitionIterator iter = group.execute(cl, state, requestTime))
         {
             return asMaterializedMap(iter);
         }
@@ -494,7 +560,7 @@ public ResultMessage execute(QueryState queryState, QueryOptions options, Dispat
         if (options.getConsistency() == null)
             throw new InvalidRequestException("Invalid empty consistency level");
 
-        Guardrails.writeConsistencyLevels.guard(EnumSet.of(options.getConsistency(), options.getSerialConsistency()),
+        Guardrails.writeConsistencyLevels.guard(EnumSet.of(options.getConsistency(), options.getSerialConsistency(queryState)),
                                                 queryState.getClientState());
 
         return hasConditions()
@@ -509,10 +575,7 @@ private ResultMessage executeWithoutCondition(QueryState queryState, QueryOption
             return executeInternalWithoutCondition(queryState, options, requestTime);
 
         ConsistencyLevel cl = options.getConsistency();
-        if (isCounter())
-            cl.validateCounterForWrite(metadata());
-        else
-            cl.validateForWrite();
+        validateConsistency(cl, queryState.getClientState());
 
         validateDiskUsage(options, queryState.getClientState());
         validateTimestamp(queryState, options);
@@ -526,7 +589,7 @@ private ResultMessage executeWithoutCondition(QueryState queryState, QueryOption
                          requestTime);
         if (!mutations.isEmpty())
         {
-            StorageProxy.mutateWithTriggers(mutations, cl, false, requestTime);
+            StorageProxy.mutateWithTriggers(mutations, cl, false, requestTime, queryState.getClientState());
 
             if (!SchemaConstants.isSystemKeyspace(metadata.keyspace))
                 ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(mutations);
@@ -535,6 +598,14 @@ private ResultMessage executeWithoutCondition(QueryState queryState, QueryOption
         return null;
     }
 
+    public void validateConsistency(ConsistencyLevel cl, ClientState clientState)
+    {
+        if (isCounter())
+            cl.validateCounterForWrite(metadata(), clientState);
+        else
+            cl.validateForWrite(metadata().keyspace, clientState);
+    }
+
     private ResultMessage executeWithCondition(QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
         CQL3CasRequest request = makeCasRequest(queryState, options);
@@ -543,7 +614,7 @@ private ResultMessage executeWithCondition(QueryState queryState, QueryOptions o
                                                    table(),
                                                    request.key,
                                                    request,
-                                                   options.getSerialConsistency(),
+                                                   options.getSerialConsistency(queryState),
                                                    options.getConsistency(),
                                                    queryState.getClientState(),
                                                    options.getNowInSeconds(queryState),
@@ -710,9 +781,11 @@ static RowIterator casInternal(ClientState state, CQL3CasRequest request, long t
         SinglePartitionReadQuery readCommand = request.readCommand(nowInSeconds);
         FilteredPartition current;
         try (ReadExecutionController executionController = readCommand.executionController();
-             PartitionIterator iter = readCommand.executeInternal(executionController))
+             PartitionIterator iter = readCommand.executeInternal(executionController);
+             RowIterator row = PartitionIterators.getOnlyElement(iter, readCommand);)
         {
-            current = FilteredPartition.create(PartitionIterators.getOnlyElement(iter, readCommand));
+            // FilteredPartition consumes the row but does not close the iterator
+            current = FilteredPartition.create(row);
         }
 
         if (!request.appliesTo(current))
@@ -873,6 +946,7 @@ private UpdateParameters makeUpdateParameters(Collection<ByteBuffer> keys,
         Map<DecoratedKey, Partition> lists =
             readRequiredLists(keys,
                               filter,
+                              state,
                               limits,
                               local,
                               options.getConsistency(),
@@ -920,7 +994,7 @@ public static Slices toSlices(ClusteringComparator comparator, SortedSet<Cluster
         return builder.build();
     }
 
-    public static abstract class Parsed extends QualifiedStatement
+    public static abstract class Parsed extends QualifiedStatement<ModificationStatement>
     {
         protected final StatementType type;
         private final Attributes.Raw attrs;
@@ -943,16 +1017,19 @@ protected Parsed(QualifiedName name,
             this.ifExists = ifExists;
         }
 
-        public ModificationStatement prepare(ClientState state)
+        @Override
+        public ModificationStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            return prepare(state, bindVariables);
+            setKeyspace(state);
+            return prepare(state, bindVariables, keyspaceMapper);
         }
 
-        public ModificationStatement prepare(ClientState state, VariableSpecifications bindVariables)
+        public ModificationStatement prepare(ClientState state, VariableSpecifications bindVariables, UnaryOperator<String> keyspaceMapper)
         {
-            TableMetadata metadata = Schema.instance.validateTable(keyspace(), name());
+            String ks = keyspaceMapper.apply(keyspace());
+            TableMetadata metadata = Schema.instance.validateTable(ks, name());
 
-            Attributes preparedAttributes = attrs.prepare(keyspace(), name());
+            Attributes preparedAttributes = attrs.prepare(ks, name());
             preparedAttributes.collectMarkerSpecification(bindVariables);
 
             Conditions preparedConditions = prepareConditions(metadata, bindVariables);
@@ -1044,7 +1121,7 @@ protected StatementRestrictions newRestrictions(ClientState state,
                 throw new InvalidRequestException(CUSTOM_EXPRESSIONS_NOT_ALLOWED);
 
             boolean applyOnlyToStaticColumns = appliesOnlyToStaticColumns(operations, conditions);
-            return new StatementRestrictions(state, type, metadata, where, boundNames, orderings, applyOnlyToStaticColumns, false, false);
+            return StatementRestrictions.create(state, type, metadata, where, boundNames, orderings, applyOnlyToStaticColumns, false, false);
         }
 
         public List<Pair<ColumnIdentifier, ColumnCondition.Raw>> getConditions()
diff --git a/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java b/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
index e809a27a45e9..5d8297e28257 100644
--- a/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
@@ -26,6 +26,7 @@
 import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.exceptions.UnauthorizedException;
 import org.apache.cassandra.service.ClientState;
+
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
@@ -42,6 +43,7 @@ protected PermissionsManagementStatement(Set<Permission> permissions, IResource
         this.grantee = RoleResource.role(grantee.getName());
     }
 
+    @Override
     public void validate(ClientState state) throws RequestValidationException
     {
         // validate login here before authorize to avoid leaking user existence to anonymous users.
diff --git a/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java b/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java
index 65ec8fca67c6..d9aaa4f101ae 100644
--- a/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java
+++ b/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java
@@ -92,7 +92,7 @@ protected String getString(String name) throws SyntaxException
         return (String)val;
     }
 
-    protected Map<String, String> getMap(String name) throws SyntaxException
+    public Map<String, String> getMap(String name) throws SyntaxException
     {
         Object val = properties.get(name);
         if (val == null)
@@ -167,4 +167,13 @@ public static double parseDouble(String key, String value) throws SyntaxExceptio
             throw new SyntaxException(format("Invalid double value %s for '%s'", value, key));
         }
     }
+    
+    public Object getProperty(String name)
+    {
+        Object ret = properties.get(name);
+        if (ret == null)
+            throw new SyntaxException(String.format("Invalid value for property '%s'. It should not be null.", name));
+
+        return ret;
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/QualifiedStatement.java b/src/java/org/apache/cassandra/cql3/statements/QualifiedStatement.java
index 4ed41d168888..b7df8e60a161 100644
--- a/src/java/org/apache/cassandra/cql3/statements/QualifiedStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/QualifiedStatement.java
@@ -27,7 +27,7 @@
 /**
  * Abstract class for statements that work on sub-keyspace level (tables, views, indexes, functions, etc.)
  */
-public abstract class QualifiedStatement extends CQLStatement.Raw
+public abstract class QualifiedStatement<R extends CQLStatement.SingleKeyspaceCqlStatement> extends RawKeyspaceAwareStatement<R>
 {
     final QualifiedName qualifiedName;
 
@@ -72,7 +72,7 @@ public String name()
     {
         return qualifiedName.getName();
     }
-    
+
     @Override
     public String toString()
     {
diff --git a/src/java/org/apache/cassandra/cql3/statements/RawKeyspaceAwareStatement.java b/src/java/org/apache/cassandra/cql3/statements/RawKeyspaceAwareStatement.java
new file mode 100644
index 000000000000..9117085b0db2
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/RawKeyspaceAwareStatement.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import java.util.function.UnaryOperator;
+
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.service.ClientState;
+
+/**
+ * A super class for raw (parsed) statements which supports keyspace override during preparation.
+ *
+ * @param <R> a type of the statement produced by preparation of this raw statement
+ */
+public abstract class RawKeyspaceAwareStatement<R extends CQLStatement> extends CQLStatement.Raw
+{
+    /**
+     * Produces a prepared statement of type {@link R} without overriding keyspace.
+     */
+    @Override
+    public final R prepare(ClientState state)
+    {
+        return prepare(state, Constants.IDENTITY_STRING_MAPPER);
+    }
+
+    /**
+     * Produces a prepared statement of type {@link R}, optionally overriding keyspace name in the produced
+     * statement. The keyspace name is overridden using the provided mapping function in the statement and all
+     * contained objects which refer to some keyspace.
+     */
+    public abstract R prepare(ClientState state, UnaryOperator<String> keyspaceMapper);
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/RoleManagementStatement.java b/src/java/org/apache/cassandra/cql3/statements/RoleManagementStatement.java
index a5274dd73834..e383e1ba6531 100644
--- a/src/java/org/apache/cassandra/cql3/statements/RoleManagementStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/RoleManagementStatement.java
@@ -25,6 +25,7 @@
 import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.exceptions.UnauthorizedException;
 import org.apache.cassandra.service.ClientState;
+
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
@@ -44,6 +45,7 @@ public void authorize(ClientState state) throws UnauthorizedException
         super.checkPermission(state, Permission.AUTHORIZE, role);
     }
 
+    @Override
     public void validate(ClientState state) throws RequestValidationException
     {
         state.ensureNotAnonymous();
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectOptions.java b/src/java/org/apache/cassandra/cql3/statements/SelectOptions.java
new file mode 100644
index 000000000000..c7775970d47f
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectOptions.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.statements;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.db.filter.ANNOptions;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.service.ClientState;
+
+/**
+ * {@code WITH option1=... AND option2=...} options for SELECT statements.
+ */
+public class SelectOptions extends PropertyDefinitions
+{
+    public static final SelectOptions EMPTY = new SelectOptions();
+    public static final String ANN_OPTIONS = "ann_options";
+
+    private static final Set<String> keywords = Collections.singleton(ANN_OPTIONS);
+
+    /**
+     * Validates all the {@code SELECT} options.
+     *
+     * @param state the query state
+     * @param limit the {@code SELECT} query user-provided limit
+     * @throws InvalidRequestException if any of the options are invalid
+     */
+    public void validate(ClientState state, String keyspace, int limit) throws RequestValidationException
+    {
+        validate(keywords, Collections.emptySet());
+        parseANNOptions().validate(state, keyspace, limit);
+    }
+
+    /**
+     * Parse the ANN Options. Does not validate values of the options or whether peers will be able to process them.
+     *
+     * @return the ANN options within these options, or {@link ANNOptions#NONE} if no options are present
+     * @throws InvalidRequestException if the ANN options are invalid
+     */
+    public ANNOptions parseANNOptions() throws RequestValidationException
+    {
+        Map<String, String> options = getMap(ANN_OPTIONS);
+
+        return options == null
+               ? ANNOptions.NONE
+               : ANNOptions.fromMap(options);
+    }
+
+    /**
+     * @return {@code true} if these options contain ANN options, {@code false} otherwise
+     */
+    public boolean hasANNOptions()
+    {
+        return properties.containsKey(ANN_OPTIONS);
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 5178b84658b3..0a6ecbcf1ecb 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -18,10 +18,22 @@
 package org.apache.cassandra.cql3.statements;
 
 import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
-
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
 import javax.annotation.concurrent.ThreadSafe;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -29,24 +41,42 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+import com.google.common.math.IntMath;
 
+import org.apache.cassandra.cql3.Ordering;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.restrictions.SingleRestriction;
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.index.Index;
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.ResultSet;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.WhereClause;
+import org.apache.cassandra.cql3.restrictions.ExternalRestriction;
+import org.apache.cassandra.cql3.restrictions.Restrictions;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.db.guardrails.GuardrailsConfigProvider;
+import org.apache.cassandra.cql3.selection.SortedRowsBuilder;
+import org.apache.cassandra.sensors.SensorsCustomParams;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.cql3.*;
 import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.cql3.selection.RawSelector;
 import org.apache.cassandra.cql3.selection.ResultSetBuilder;
 import org.apache.cassandra.cql3.selection.Selectable;
@@ -54,20 +84,51 @@
 import org.apache.cassandra.cql3.selection.Selection;
 import org.apache.cassandra.cql3.selection.Selection.Selectors;
 import org.apache.cassandra.cql3.selection.Selector;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadQuery;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
 import org.apache.cassandra.db.aggregation.AggregationSpecification;
 import org.apache.cassandra.db.aggregation.GroupMaker;
-import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.FloatType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.RowIterator;
 import org.apache.cassandra.db.view.View;
 import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.ReadSizeAbortException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.exceptions.UnauthorizedException;
 import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.Type;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.ClientWarn;
@@ -83,21 +144,20 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.NoSpamLogger;
 
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
-
 import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_ANN_USE_SYNTHETIC_SCORE;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
 import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+import static org.apache.cassandra.db.filter.DataLimits.NO_LIMIT;
 import static org.apache.cassandra.utils.ByteBufferUtil.UNSET_BYTE_BUFFER;
 
 /**
  * Encapsulates a completely parsed SELECT query, including the target
  * column family, expression, result count, and ordering clause.
- * <p>
+ * </p>
  * A number of public methods here are only used internally. However,
  * many of these are made accessible for the benefit of custom
  * QueryHandler implementations, so before reducing their accessibility
@@ -108,26 +168,33 @@
 @ThreadSafe
 public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
 {
+    // TODO remove this when we no longer need to downgrade to replicas that don't know about synthetic columns,
+    // and the related code in
+    //  - StatementRestrictions.addOrderingRestrictions
+    //  - StorageAttachedIndexSearcher.PrimaryKeyIterator constructor
+    public static final boolean ANN_USE_SYNTHETIC_SCORE = SAI_ANN_USE_SYNTHETIC_SCORE.getBoolean();
+
     private static final Logger logger = LoggerFactory.getLogger(SelectStatement.class);
     private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(SelectStatement.logger, 1, TimeUnit.MINUTES);
-
-    public static final int DEFAULT_PAGE_SIZE = 10000;
-    public static final String TOPK_CONSISTENCY_LEVEL_ERROR = "Top-K queries can only be run with consistency level ONE/LOCAL_ONE. Consistency level %s was used.";
-    public static final String TOPK_LIMIT_ERROR = "Top-K queries must have a limit specified and the limit must be less than the query page size";
-    public static final String TOPK_PARTITION_LIMIT_ERROR = "Top-K queries do not support per-partition limits";
+    public static final String USAGE_WARNING_PAGE_WEIGHT = "Applied page weight limit of ";
     public static final String TOPK_AGGREGATION_ERROR = "Top-K queries can not be run with aggregation";
+    public static final String TOPK_CONSISTENCY_LEVEL_ERROR = "Top-K queries can only be run with consistency level ONE/LOCAL_ONE. Consistency level %s was used.";
     public static final String TOPK_CONSISTENCY_LEVEL_WARNING = "Top-K queries can only be run with consistency level ONE " +
                                                                 "/ LOCAL_ONE / NODE_LOCAL. Consistency level %s was requested. " +
                                                                 "Downgrading the consistency level to %s.";
-    public static final String TOPK_PAGE_SIZE_WARNING = "Top-K queries do not support paging and the page size is set to %d, " +
-                                                        "which is less than LIMIT %d. The page size has been set to %d to match the LIMIT.";
+    public static final String TOPK_OFFSET_ERROR = "Top-K queries cannot be run with an offset. Offset was set to %d.";
 
+    private static final int NO_OFFSET = -1; // sentinel value meaning no offset has been explicitly requested
+
+    private final String rawCQLStatement;
     public final VariableSpecifications bindVariables;
     public final TableMetadata table;
     public final Parameters parameters;
     private final Selection selection;
     private final Term limit;
     private final Term perPartitionLimit;
+    private final Term offset;
+    private final SelectOptions selectOptions;
 
     private final StatementRestrictions restrictions;
 
@@ -150,7 +217,8 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                                                                        false,
                                                                        false);
 
-    public SelectStatement(TableMetadata table,
+    public SelectStatement(String queryString,
+                           TableMetadata table,
                            VariableSpecifications bindVariables,
                            Parameters parameters,
                            Selection selection,
@@ -159,8 +227,11 @@ public SelectStatement(TableMetadata table,
                            AggregationSpecification.Factory aggregationSpecFactory,
                            ColumnComparator<List<ByteBuffer>> orderingComparator,
                            Term limit,
-                           Term perPartitionLimit)
+                           Term perPartitionLimit,
+                           Term offset,
+                           SelectOptions selectOptions)
     {
+        this.rawCQLStatement = queryString;
         this.table = table;
         this.bindVariables = bindVariables;
         this.selection = selection;
@@ -171,6 +242,14 @@ public SelectStatement(TableMetadata table,
         this.parameters = parameters;
         this.limit = limit;
         this.perPartitionLimit = perPartitionLimit;
+        this.offset = offset;
+        this.selectOptions = selectOptions;
+    }
+
+    @Override
+    public String getRawCQLStatement()
+    {
+        return rawCQLStatement;
     }
 
     @Override
@@ -222,16 +301,19 @@ public ColumnFilter queriedColumns()
     // queried data through processColumnFamily.
     static SelectStatement forSelection(TableMetadata table, Selection selection)
     {
-        return new SelectStatement(table,
+        return new SelectStatement(null,
+                                   table,
                                    VariableSpecifications.empty(),
                                    defaultParameters,
                                    selection,
-                                   StatementRestrictions.empty(StatementType.SELECT, table),
+                                   StatementRestrictions.empty(table),
                                    false,
                                    null,
                                    null,
                                    null,
-                                   null);
+                                   null,
+                                   null,
+                                   SelectOptions.EMPTY);
     }
 
     public ResultSet.ResultMetadata getResultMetadata()
@@ -271,73 +353,113 @@ public void authorize(ClientState state) throws InvalidRequestException, Unautho
         }
     }
 
+    @Override
     public void validate(ClientState state) throws InvalidRequestException
     {
         if (parameters.allowFiltering && !SchemaConstants.isSystemKeyspace(table.keyspace))
             Guardrails.allowFilteringEnabled.ensureEnabled(state);
     }
 
+    /**
+     * Adds the specified restrictions to the index restrictions.
+     *
+     * @param indexRestrictions the index restrictions to add
+     * @return a new {@code SelectStatement} instance with the added index restrictions
+     */
+    @SuppressWarnings("unused") // this is used by DSE and CNDB to add authorization restrictions
+    public SelectStatement addIndexRestrictions(Restrictions indexRestrictions)
+    {
+        return new SelectStatement(rawCQLStatement,
+                                   table,
+                                   bindVariables,
+                                   parameters,
+                                   selection,
+                                   restrictions.addIndexRestrictions(indexRestrictions),
+                                   isReversed,
+                                   aggregationSpecFactory,
+                                   orderingComparator,
+                                   limit,
+                                   perPartitionLimit,
+                                   offset,
+                                   selectOptions);
+    }
+
+    /**
+     * Adds the specified external restrictions to the index restrictions.
+     *
+     * @param indexRestrictions the index restrictions to add
+     * @return a new {@code SelectStatement} instance with the added index restrictions
+     */
+    public SelectStatement addIndexRestrictions(Iterable<ExternalRestriction> indexRestrictions)
+    {
+        return new SelectStatement(rawCQLStatement,
+                                   table,
+                                   bindVariables,
+                                   parameters,
+                                   selection,
+                                   restrictions.addExternalRestrictions(indexRestrictions),
+                                   isReversed,
+                                   aggregationSpecFactory,
+                                   orderingComparator,
+                                   limit,
+                                   perPartitionLimit,
+                                   offset,
+                                   selectOptions);
+    }
+
+    private void validateQueryOptions(QueryState queryState, QueryOptions options)
+    {
+        if (SchemaConstants.isUserKeyspace(table.keyspace))
+            Guardrails.readConsistencyLevels.guard(EnumSet.of(options.getConsistency()), queryState.getClientState());
+
+        PageSize pageSize = options.getPageSize();
+        pageSize.guard(table(), queryState.getClientState());
+    }
+
+    /**
+     * Returns whether the paging can be skipped based on the user limits and the page size - that is, if the user limit
+     * is provided and is lower than the page size, it means that we will only return at most one page and thus paging
+     * is unnecessary in this case. That applies to the page size defined in rows - if the page size is defined in bytes
+     * we cannot say anything about the relation beteween the user rows limit and the page size.
+     */
+    private boolean canSkipPaging(DataLimits userLimits, PageSize pageSize, boolean topK)
+    {
+        return !pageSize.isDefined() ||
+               pageSize.getUnit() == PageSize.PageUnit.ROWS && !pageSize.isCompleted(userLimits.count(), PageSize.PageUnit.ROWS) ||
+               topK;
+    }
+
+    @Override
     public ResultMessage.Rows execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
         ConsistencyLevel cl = options.getConsistency();
         checkNotNull(cl, "Invalid empty consistency level");
 
         cl.validateForRead();
-        Guardrails.readConsistencyLevels.guard(EnumSet.of(cl), state.getClientState());
+        validateQueryOptions(state, options);
 
         long nowInSec = options.getNowInSeconds(state);
         int userLimit = getLimit(options);
         int userPerPartitionLimit = getPerPartitionLimit(options);
-        int pageSize = options.getPageSize();
+        int userOffset = getOffset(options);
+        PageSize pageSize = options.getPageSize();
         boolean unmask = !table.hasMaskedColumns() || state.getClientState().hasTablePermission(table, Permission.UNMASK);
 
         Selectors selectors = selection.newSelectors(options);
         AggregationSpecification aggregationSpec = getAggregationSpec(options);
-        DataLimits limit = getDataLimits(userLimit, userPerPartitionLimit, pageSize, aggregationSpec);
-
-        // Handle additional validation for topK queries
-        if (restrictions.isTopK())
-        {
-            checkFalse(aggregationSpec != null, TOPK_AGGREGATION_ERROR);
-
-            // We aren't going to allow SERIAL at all, so we can error out on those.
-            checkFalse(options.getConsistency() == ConsistencyLevel.LOCAL_SERIAL ||
-                       options.getConsistency() == ConsistencyLevel.SERIAL,
-                       String.format(TOPK_CONSISTENCY_LEVEL_ERROR, options.getConsistency()));
-
-            if (options.getConsistency().needsReconciliation())
-            {
-                ConsistencyLevel supplied = options.getConsistency();
-                ConsistencyLevel downgrade = supplied.isDatacenterLocal() ? ConsistencyLevel.LOCAL_ONE : ConsistencyLevel.ONE;
-
-                options = QueryOptions.withConsistencyLevel(options, downgrade);
-
-                ClientWarn.instance.warn(String.format(TOPK_CONSISTENCY_LEVEL_WARNING, supplied, downgrade));
-            }
-
-            checkFalse(limit.isUnlimited(), TOPK_LIMIT_ERROR);
-
-            checkFalse(limit.perPartitionCount() != DataLimits.NO_LIMIT, TOPK_PARTITION_LIMIT_ERROR);
-
-            if (pageSize > 0 && pageSize < limit.count())
-            {
-                int oldPageSize = pageSize;
-                pageSize = limit.count();
-                limit = getDataLimits(userLimit, userPerPartitionLimit, pageSize, aggregationSpec);
-                options = QueryOptions.withPageSize(options, pageSize);
-                ClientWarn.instance.warn(String.format(TOPK_PAGE_SIZE_WARNING, oldPageSize, limit.count(), pageSize));
-            }
-        }
-
-        ReadQuery query = getQuery(options, state.getClientState(), selectors.getColumnFilter(), nowInSec, limit);
+        ReadQuery query = getQuery(options, state.getClientState(), selectors.getColumnFilter(),
+                                   nowInSec, userLimit, userPerPartitionLimit, userOffset, aggregationSpec);
 
         if (options.isReadThresholdsEnabled())
             query.trackWarnings();
         ResultMessage.Rows rows;
 
-        if (aggregationSpec == null && (pageSize <= 0 || (query.limits().count() <= pageSize) || query.isTopK()))
+        if (query.limits().isGroupByLimit() && pageSize != null && pageSize.isDefined() && pageSize.getUnit() == PageSize.PageUnit.BYTES)
+            throw new InvalidRequestException("Paging in bytes cannot be specified for aggregation queries");
+
+        if (aggregationSpec == null && canSkipPaging(query.limits(), pageSize, query.isTopK()))
         {
-            rows = execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, null, requestTime, unmask);
+            rows = execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, userOffset, null, requestTime, unmask);
         }
         else
         {
@@ -350,6 +472,7 @@ public ResultMessage.Rows execute(QueryState state, QueryOptions options, Dispat
                            pageSize,
                            nowInSec,
                            userLimit,
+                           userOffset,
                            aggregationSpec,
                            requestTime,
                            unmask);
@@ -374,7 +497,7 @@ public ReadQuery getQuery(QueryOptions options, long nowInSec) throws RequestVal
                         nowInSec,
                         getLimit(options),
                         getPerPartitionLimit(options),
-                        options.getPageSize(),
+                        getOffset(options),
                         getAggregationSpec(options));
     }
 
@@ -384,34 +507,59 @@ public ReadQuery getQuery(QueryOptions options,
                               long nowInSec,
                               int userLimit,
                               int perPartitionLimit,
-                              int pageSize,
+                              int userOffset,
                               AggregationSpecification aggregationSpec)
     {
-        DataLimits limit = getDataLimits(userLimit, perPartitionLimit, pageSize, aggregationSpec);
-
-        return getQuery(options, state, columnFilter, nowInSec, limit);
-    }
+        boolean isPartitionRangeQuery = restrictions.isKeyRange() || restrictions.usesSecondaryIndexing() || restrictions.isDisjunction();
 
-    public ReadQuery getQuery(QueryOptions options,
-                              ClientState state,
-                              ColumnFilter columnFilter,
-                              long nowInSec,
-                              DataLimits limit)
-    {
         RowFilter rowFilter = getRowFilter(options, state);
+        DataLimits limit = getDataLimits(state, userLimit, perPartitionLimit, userOffset, aggregationSpec);
 
-        if (restrictions.isKeyRange())
+        ReadQuery query;
+        if (isPartitionRangeQuery)
         {
-            if (restrictions.usesSecondaryIndexing() && !SchemaConstants.isLocalSystemKeyspace(table.keyspace))
+            if (restrictions.isKeyRange() && restrictions.usesSecondaryIndexing() && !SchemaConstants.isLocalSystemKeyspace(table.keyspace))
                 Guardrails.nonPartitionRestrictedIndexQueryEnabled.ensureEnabled(state);
 
-            return getRangeCommand(options, state, columnFilter, rowFilter, limit, nowInSec);
+            query = getRangeCommand(options, state, columnFilter, rowFilter, limit, nowInSec);
+        }
+        else
+        {
+            query = getSliceCommands(options, state, columnFilter, rowFilter, limit, nowInSec);
         }
 
-        if (restrictions.usesSecondaryIndexing() && !rowFilter.isStrict())
-            return getRangeCommand(options, state, columnFilter, rowFilter, limit, nowInSec);
+        // Handle additional validation for topK queries
+        if (query.isTopK())
+        {
+            // We aren't going to allow SERIAL at all, so we can error out on those.
+            checkFalse(options.getConsistency() == ConsistencyLevel.LOCAL_SERIAL ||
+                       options.getConsistency() == ConsistencyLevel.SERIAL,
+                       String.format(TOPK_CONSISTENCY_LEVEL_ERROR, options.getConsistency()));
+
+            // Consistency levels with more than one replica are downgraded to ONE/LOCAL_ONE.
+            if (options.getConsistency() != ConsistencyLevel.ONE &&
+                options.getConsistency() != ConsistencyLevel.LOCAL_ONE &&
+                options.getConsistency() != ConsistencyLevel.NODE_LOCAL)
+            {
+                ConsistencyLevel supplied = options.getConsistency();
+                ConsistencyLevel downgrade = supplied.isDatacenterLocal() ? ConsistencyLevel.LOCAL_ONE : ConsistencyLevel.ONE;
+                options.updateConsistency(downgrade);
+                ClientWarn.instance.warn(String.format(TOPK_CONSISTENCY_LEVEL_WARNING, supplied, downgrade));
+            }
+
+            // We don't support offset for top-k queries.
+            checkFalse(userOffset != NO_OFFSET, String.format(TOPK_OFFSET_ERROR, userOffset));
+
+            // We don't support aggregation for top-k queries because we don't support paging.
+            checkFalse(aggregationSpec != null, TOPK_AGGREGATION_ERROR);
+        }
+
+        selectOptions.validate(state, table.keyspace, userLimit);
+
+        // If there's a secondary index that the command can use, have it validate the request parameters.
+        query.maybeValidateIndexes();
 
-        return getSliceCommands(options, state, columnFilter, rowFilter, limit, nowInSec);
+        return query;
     }
 
     private ResultMessage.Rows execute(ReadQuery query,
@@ -420,13 +568,14 @@ private ResultMessage.Rows execute(ReadQuery query,
                                        Selectors selectors,
                                        long nowInSec,
                                        int userLimit,
+                                       int userOffset,
                                        AggregationSpecification aggregationSpec,
                                        Dispatcher.RequestTime requestTime,
                                        boolean unmask)
     {
         try (PartitionIterator data = query.execute(options.getConsistency(), state, requestTime))
         {
-            return processResults(data, options, selectors, nowInSec, userLimit, aggregationSpec, unmask, state);
+            return processResults(data, options, selectors, nowInSec, userLimit, userOffset, aggregationSpec, unmask, state);
         }
     }
 
@@ -466,24 +615,33 @@ public PagingState state()
             return pager.state();
         }
 
-        public abstract PartitionIterator fetchPage(int pageSize, Dispatcher.RequestTime requestTime);
+        public abstract PartitionIterator fetchPage(PageSize pageSize, Dispatcher.RequestTime requestTime);
+
+        public abstract PartitionIterator readAll(PageSize pageSize, Dispatcher.RequestTime requestTime);
 
         public static class NormalPager extends Pager
         {
             private final ConsistencyLevel consistency;
             private final ClientState clientState;
 
-            private NormalPager(QueryPager pager, ConsistencyLevel consistency, ClientState clientState)
+            private NormalPager(QueryPager pager, ConsistencyLevel consistency, ClientState queryState)
             {
                 super(pager);
                 this.consistency = consistency;
-                this.clientState = clientState;
+                this.clientState = queryState;
             }
 
-            public PartitionIterator fetchPage(int pageSize, Dispatcher.RequestTime requestTime)
+            @Override
+            public PartitionIterator fetchPage(PageSize pageSize, Dispatcher.RequestTime requestTime)
             {
                 return pager.fetchPage(pageSize, consistency, clientState, requestTime);
             }
+
+            @Override
+            public PartitionIterator readAll(PageSize pageSize, Dispatcher.RequestTime requestTime)
+            {
+                return pager.readAll(pageSize, consistency, clientState, requestTime);
+            }
         }
 
         public static class InternalPager extends Pager
@@ -496,10 +654,17 @@ private InternalPager(QueryPager pager, ReadExecutionController executionControl
                 this.executionController = executionController;
             }
 
-            public PartitionIterator fetchPage(int pageSize, Dispatcher.RequestTime requestTime)
+            @Override
+            public PartitionIterator fetchPage(PageSize pageSize, Dispatcher.RequestTime requestTime)
             {
                 return pager.fetchPageInternal(pageSize, executionController);
             }
+
+            @Override
+            public PartitionIterator readAll(PageSize pageSize, Dispatcher.RequestTime requestTime)
+            {
+                return pager.readAllInternal(pageSize, executionController);
+            }
         }
     }
 
@@ -507,14 +672,15 @@ private ResultMessage.Rows execute(QueryState state,
                                        Pager pager,
                                        QueryOptions options,
                                        Selectors selectors,
-                                       int pageSize,
+                                       PageSize pageSize,
                                        long nowInSec,
                                        int userLimit,
+                                       int userOffset,
                                        AggregationSpecification aggregationSpec,
                                        Dispatcher.RequestTime requestTime,
                                        boolean unmask)
     {
-        Guardrails.pageSize.guard(pageSize, table(), false, state.getClientState());
+        pageSize.guard(table(), state.getClientState());
 
         if (aggregationSpecFactory != null)
         {
@@ -534,16 +700,28 @@ else if (restrictions.keyIsInRelation())
 
         // We can't properly do post-query ordering if we page (see #6722)
         // For GROUP BY or aggregation queries we always page internally even if the user has turned paging off
-        checkFalse(pageSize > 0 && needsPostQueryOrdering(),
+        checkFalse(pageSize.isDefined() && needsPostQueryOrdering(),
                   "Cannot page queries with both ORDER BY and a IN restriction on the partition key;"
                   + " you must either remove the ORDER BY or the IN and sort client side, or disable paging for this query");
 
+        // If the query has an offset we silently ignore user-facing paging and return all the rows specified by the
+        // limit/offset constraints in one go, since regular key-based paging is not supported when using limit/offest
+        // paging. However, we still use the query fetch size to internally page the rows. We do that to avoid loading
+        // in memory all the rows that will be discarded by the offset. Key-based paging is also disabled if the offset
+        // is explicitly set to zero.
         ResultMessage.Rows msg;
-        try (PartitionIterator page = pager.fetchPage(pageSize, requestTime))
+        try (PartitionIterator partitions = userOffset == NO_OFFSET
+                                          ? pager.fetchPage(pageSize, requestTime)
+                                          : pager.readAll(pageSize, requestTime))
         {
-            msg = processResults(page, options, selectors, nowInSec, userLimit, aggregationSpec, unmask, state.getClientState());
+            msg = processResults(partitions, options, selectors, nowInSec, userLimit, userOffset, aggregationSpec, unmask, state.getClientState());
         }
 
+        RequestSensors sensors = RequestTracker.instance.get();
+        Context context = Context.from(this.table);
+        Type sensorType = Type.READ_BYTES;
+        SensorsCustomParams.addSensorToCQLResponse(msg, options.getProtocolVersion(), sensors, context, sensorType);
+
         // Please note that the isExhausted state of the pager only gets updated when we've closed the page, so this
         // shouldn't be moved inside the 'try' above.
         if (!pager.isExhausted() && !pager.pager.isTopK())
@@ -563,14 +741,16 @@ private ResultMessage.Rows processResults(PartitionIterator partitions,
                                               Selectors selectors,
                                               long nowInSec,
                                               int userLimit,
+                                              int userOffset,
                                               AggregationSpecification aggregationSpec,
                                               boolean unmask,
                                               ClientState state) throws RequestValidationException
     {
-        ResultSet rset = process(partitions, options, selectors, nowInSec, userLimit, aggregationSpec, unmask, state);
+        ResultSet rset = process(partitions, options, selectors, nowInSec, userLimit, userOffset, aggregationSpec, unmask, state);
         return new ResultMessage.Rows(rset);
     }
 
+    @Override
     public ResultMessage.Rows executeLocally(QueryState state, QueryOptions options) throws RequestExecutionException, RequestValidationException
     {
         return executeInternal(state, options, options.getNowInSeconds(state), Dispatcher.RequestTime.forImmediateExecution());
@@ -583,7 +763,8 @@ public ResultMessage.Rows executeInternal(QueryState state,
     {
         int userLimit = getLimit(options);
         int userPerPartitionLimit = getPerPartitionLimit(options);
-        int pageSize = options.getPageSize();
+        int userOffset = getOffset(options);
+        PageSize pageSize = options.getPageSize();
         boolean unmask = state.getClientState().hasTablePermission(table, Permission.UNMASK);
 
         Selectors selectors = selection.newSelectors(options);
@@ -594,16 +775,16 @@ public ResultMessage.Rows executeInternal(QueryState state,
                                    nowInSec,
                                    userLimit,
                                    userPerPartitionLimit,
-                                   pageSize,
+                                   userOffset,
                                    aggregationSpec);
 
         try (ReadExecutionController executionController = query.executionController())
         {
-            if (aggregationSpec == null && (pageSize <= 0 || (query.limits().count() <= pageSize) || query.isTopK()))
+            if (aggregationSpec == null && canSkipPaging(query.limits(), pageSize, query.isTopK()))
             {
                 try (PartitionIterator data = query.executeInternal(executionController))
                 {
-                    return processResults(data, options, selectors, nowInSec, userLimit, null, unmask, state.getClientState());
+                    return processResults(data, options, selectors, nowInSec, userLimit, userOffset, null, unmask, state.getClientState());
                 }
             }
 
@@ -616,33 +797,36 @@ public ResultMessage.Rows executeInternal(QueryState state,
                            pageSize,
                            nowInSec,
                            userLimit,
+                           userOffset,
                            aggregationSpec,
                            requestTime,
                            unmask);
         }
     }
 
-    private QueryPager getPager(ReadQuery query, QueryOptions options)
+    @VisibleForTesting
+    public QueryPager getPager(ReadQuery query, QueryOptions options)
     {
         QueryPager pager = query.getPager(options.getPagingState(), options.getProtocolVersion());
 
         if (aggregationSpecFactory == null || query.isEmpty())
             return pager;
 
-        return new AggregationQueryPager(pager, query.limits());
+        return new AggregationQueryPager(pager, DatabaseDescriptor.getAggregationSubPageSize(), query.limits());
     }
 
     public Map<DecoratedKey, List<Row>> executeRawInternal(QueryOptions options, ClientState state, long nowInSec) throws RequestExecutionException, RequestValidationException
     {
         int userLimit = getLimit(options);
         int userPerPartitionLimit = getPerPartitionLimit(options);
-        if (options.getPageSize() > 0)
+        int userOffset = getOffset(options);
+        if (options.getPageSize().isDefined())
             throw new IllegalStateException();
         if (aggregationSpecFactory != null)
             throw new IllegalStateException();
 
         Selectors selectors = selection.newSelectors(options);
-        ReadQuery query = getQuery(options, state, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, Integer.MAX_VALUE, null);
+        ReadQuery query = getQuery(options, state, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, userOffset, null);
 
         Map<DecoratedKey, List<Row>> result = Collections.emptyMap();
         try (ReadExecutionController executionController = query.executionController())
@@ -680,7 +864,7 @@ public ResultSet process(PartitionIterator partitions, long nowInSec, boolean un
     {
         QueryOptions options = QueryOptions.DEFAULT;
         Selectors selectors = selection.newSelectors(options);
-        return process(partitions, options, selectors, nowInSec, getLimit(options), getAggregationSpec(options), unmask, state);
+        return process(partitions, options, selectors, nowInSec, getLimit(options), getOffset(options), getAggregationSpec(options), unmask, state);
     }
 
     @Override
@@ -737,7 +921,7 @@ private ReadQuery getSliceCommands(QueryOptions options, ClientState state, Colu
             SinglePartitionReadQuery.createGroup(table, nowInSec, columnFilter, rowFilter, limit, decoratedKeys, filter);
 
         // If there's a secondary index that the commands can use, have it validate the request parameters.
-        group.maybeValidateIndex();
+        group.maybeValidateIndexes();
 
         return group;
     }
@@ -800,13 +984,7 @@ private ReadQuery getRangeCommand(QueryOptions options, ClientState state, Colum
         if (keyBounds == null)
             return ReadQuery.empty(table);
 
-        ReadQuery command =
-            PartitionRangeReadQuery.create(table, nowInSec, columnFilter, rowFilter, limit, new DataRange(keyBounds, clusteringIndexFilter));
-
-        // If there's a secondary index that the command can use, have it validate the request parameters.
-        command.maybeValidateIndex();
-
-        return command;
+        return PartitionRangeReadQuery.create(table, nowInSec, columnFilter, rowFilter, limit, new DataRange(keyBounds, clusteringIndexFilter));
     }
 
     private ClusteringIndexFilter makeClusteringIndexFilter(QueryOptions options, ClientState state, ColumnFilter columnFilter)
@@ -823,6 +1001,11 @@ private ClusteringIndexFilter makeClusteringIndexFilter(QueryOptions options, Cl
             return new ClusteringIndexSliceFilter(Slices.ALL, false);
         }
 
+        if (restrictions.isDisjunction())
+        {
+            return new ClusteringIndexSliceFilter(Slices.ALL, false);
+        }
+
         if (restrictions.isColumnRange())
         {
             Slices slices = makeSlices(options);
@@ -878,46 +1061,68 @@ public Slices makeSlices(QueryOptions options)
         return builder.build();
     }
 
-    private DataLimits getDataLimits(int userLimit,
+    private DataLimits getDataLimits(ClientState clientState,
+                                     int userLimit,
                                      int perPartitionLimit,
-                                     int pageSize,
+                                     int userOffset,
                                      AggregationSpecification aggregationSpec)
     {
-        int cqlRowLimit = DataLimits.NO_LIMIT;
-        int cqlPerPartitionLimit = DataLimits.NO_LIMIT;
+        assert userOffset == NO_OFFSET || userLimit != NO_LIMIT : "Cannot use OFFSET without LIMIT";
+
+        if (userOffset != NO_OFFSET)
+            Guardrails.offsetRows.guard(userOffset, "Select query", false, clientState);
+
+        int fetchLimit = userLimit == NO_LIMIT || userOffset == NO_OFFSET ? userLimit : IntMath.saturatedAdd(userLimit, userOffset);
+        int cqlRowLimit = NO_LIMIT;
+        int cqlPerPartitionLimit = NO_LIMIT;
 
         // If we do post ordering we need to get all the results sorted before we can trim them.
         if (aggregationSpec != AggregationSpecification.AGGREGATE_EVERYTHING)
         {
-            // If we aren't need post-query ordering but we are doing index ordering (currently ANN only) then
-            // we do need to use the user limit.
-            if (!needsPostQueryOrdering() || needIndexOrdering())
-                cqlRowLimit = userLimit;
+            if (!needsToSkipUserLimit())
+                cqlRowLimit = fetchLimit;
             cqlPerPartitionLimit = perPartitionLimit;
         }
 
-        // Group by and aggregation queries will always be paged internally to avoid OOM.
-        // If the user provided a pageSize we'll use that to page internally (because why not), otherwise we use our default
-        if (pageSize <= 0)
-            pageSize = DEFAULT_PAGE_SIZE;
+        DataLimits limits = null;
 
         // Aggregation queries work fine on top of the group by paging but to maintain
         // backward compatibility we need to use the old way.
         if (aggregationSpec != null && aggregationSpec != AggregationSpecification.AGGREGATE_EVERYTHING)
         {
             if (parameters.isDistinct)
-                return DataLimits.distinctLimits(cqlRowLimit);
-
-            return DataLimits.groupByLimits(cqlRowLimit,
-                                            cqlPerPartitionLimit,
-                                            pageSize,
-                                            aggregationSpec);
+                limits = DataLimits.distinctLimits(cqlRowLimit);
+            else
+                limits = DataLimits.groupByLimits(cqlRowLimit,
+                                                  cqlPerPartitionLimit,
+                                                  NO_LIMIT,
+                                                  NO_LIMIT,
+                                                  aggregationSpec);
+        }
+        else
+        {
+            if (parameters.isDistinct)
+                limits = cqlRowLimit == NO_LIMIT ? DataLimits.DISTINCT_NONE : DataLimits.distinctLimits(cqlRowLimit);
+            else
+                limits = DataLimits.cqlLimits(cqlRowLimit, cqlPerPartitionLimit);
         }
 
-        if (parameters.isDistinct)
-            return cqlRowLimit == DataLimits.NO_LIMIT ? DataLimits.DISTINCT_NONE : DataLimits.distinctLimits(cqlRowLimit);
+        if (!limits.isGroupByLimit())
+        {
+            // if the user does not specify any limit, it means there is no limit - when the guardrail is defined, we
+            // want to limit the number of rows returned to the user
+            if (Guardrails.pageWeight.enabled(clientState))
+            {
+                DataStorageSpec.IntBytesBound pageWeightFailThreshold = GuardrailsConfigProvider.instance.getOrCreate(clientState).getPageWeightFailThreshold();
+                int bytesLimit = pageWeightFailThreshold == null ? NO_LIMIT : pageWeightFailThreshold.toBytes();
+                String limitStr = USAGE_WARNING_PAGE_WEIGHT + FBUtilities.prettyPrintMemory(bytesLimit);
+                ClientWarn.instance.warn(limitStr);
+                logger.trace(limitStr);
+                limits = limits.forPaging(PageSize.inBytes(bytesLimit));
+            }
+        }
 
-        return DataLimits.cqlLimits(cqlRowLimit, cqlPerPartitionLimit);
+        return limits;
     }
 
     /**
@@ -946,7 +1151,7 @@ public int getPerPartitionLimit(QueryOptions options)
 
     private int getLimit(Term limit, QueryOptions options)
     {
-        int userLimit = DataLimits.NO_LIMIT;
+        int userLimit = NO_LIMIT;
 
         if (limit != null)
         {
@@ -969,6 +1174,31 @@ private int getLimit(Term limit, QueryOptions options)
         return userLimit;
     }
 
+    public int getOffset(QueryOptions options)
+    {
+        int userOffset = NO_OFFSET;
+
+        if (offset != null)
+        {
+            ByteBuffer b = checkNotNull(offset.bindAndGet(options), "Invalid null value of offset");
+            // treat UNSET limit value as zero
+            if (b != UNSET_BYTE_BUFFER)
+            {
+                try
+                {
+                    Int32Type.instance.validate(b);
+                    userOffset = Int32Type.instance.compose(b);
+                    checkTrue(userOffset >= 0, "Offset must be positive");
+                }
+                catch (MarshalException e)
+                {
+                    throw new InvalidRequestException("Invalid offset value");
+                }
+            }
+        }
+        return userOffset;
+    }
+
     private NavigableSet<Clustering<?>> getRequestedRows(QueryOptions options, ClientState state) throws InvalidRequestException
     {
         // Note: getRequestedColumns don't handle static columns, but due to CASSANDRA-5762
@@ -983,7 +1213,7 @@ private NavigableSet<Clustering<?>> getRequestedRows(QueryOptions options, Clien
     public RowFilter getRowFilter(QueryOptions options, ClientState state) throws InvalidRequestException
     {
         IndexRegistry indexRegistry = IndexRegistry.obtain(table);
-        RowFilter filter = restrictions.getRowFilter(indexRegistry, options);
+        RowFilter filter = restrictions.getRowFilter(indexRegistry, options, state, selectOptions);
 
         if (filter.needsReconciliation() && filter.isMutableIntersection() && restrictions.needFiltering(table))
             Guardrails.intersectFilteringQueryEnabled.ensureEnabled(state);
@@ -996,12 +1226,14 @@ private ResultSet process(PartitionIterator partitions,
                               Selectors selectors,
                               long nowInSec,
                               int userLimit,
+                              int userOffset,
                               AggregationSpecification aggregationSpec,
                               boolean unmask,
                               ClientState state) throws InvalidRequestException
     {
         GroupMaker groupMaker = aggregationSpec == null ? null : aggregationSpec.newGroupMaker();
-        ResultSetBuilder result = new ResultSetBuilder(getResultMetadata(), selectors, unmask, groupMaker);
+        SortedRowsBuilder rows = sortedRowsBuilder(userLimit, userOffset == NO_OFFSET ? 0 : userOffset);
+        ResultSetBuilder result = new ResultSetBuilder(getResultMetadata(), selectors, unmask, groupMaker, rows);
 
         while (partitions.hasNext())
         {
@@ -1011,13 +1243,10 @@ private ResultSet process(PartitionIterator partitions,
             }
         }
 
+        // maybeWarn requires the result set to be built
         ResultSet cqlRows = result.build();
         maybeWarn(result, options);
 
-        orderResults(cqlRows, options, state);
-
-        cqlRows.trim(userLimit);
-
         return cqlRows;
     }
 
@@ -1075,7 +1304,8 @@ private void maybeFail(ResultSetBuilder result, QueryOptions options)
             // to work around this, treat the coordinator as the only response we care about and mark it failed
             ReadSizeAbortException exception = new ReadSizeAbortException(clientMsg, options.getConsistency(), 0, 1, true,
                                                                           ImmutableMap.of(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.READ_SIZE));
-            StorageProxy.recordReadRegularAbort(options.getConsistency(), exception);
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(options.getKeyspace());
+            StorageProxy.recordReadRegularAbort(options.getConsistency(), exception, metrics);
             throw exception;
         }
     }
@@ -1113,7 +1343,7 @@ public void processPartition(RowIterator partition, QueryOptions options, Result
                             result.add(partition.staticRow().getColumnData(def), nowInSec);
                             break;
                         default:
-                            result.add((ByteBuffer)null);
+                            result.add(null);
                     }
                 }
             }
@@ -1142,17 +1372,27 @@ public void processPartition(RowIterator partition, QueryOptions options, Result
                     case CLUSTERING:
                         result.add(row.clustering().bufferAt(def.position()));
                         break;
+                    case SYNTHETIC:
+                        // treat as REGULAR
                     case REGULAR:
                         result.add(row.getColumnData(def), nowInSec);
                         break;
                     case STATIC:
                         result.add(staticRow.getColumnData(def), nowInSec);
                         break;
+                    default:
+                        throw new AssertionError();
                 }
             }
         }
     }
 
+    private boolean needsToSkipUserLimit()
+    {
+        // if post query ordering is required, and it's not ordered by an index
+        return needsPostQueryOrdering() && !needIndexOrdering();
+    }
+
     private boolean needsPostQueryOrdering()
     {
         // We need post-query ordering only for queries with IN on the partition key and an ORDER BY or index restriction reordering
@@ -1165,35 +1405,42 @@ private boolean needIndexOrdering()
     }
 
     /**
-     * Orders results when multiple keys are selected (using IN).
-     * <p>
-     * In the case of ANN ordering the rows are first ordered in index column order and then by primary key.
+     * Orders results when multiple keys are selected (using IN)
      */
-    private void orderResults(ResultSet cqlRows, QueryOptions options, ClientState state)
+    public SortedRowsBuilder sortedRowsBuilder(int limit, int offset)
     {
-        if (cqlRows.size() == 0 || !needsPostQueryOrdering())
-            return;
+        assert (orderingComparator != null) == needsPostQueryOrdering()
+                : String.format("orderingComparator: %s, needsPostQueryOrdering: %s",
+                                orderingComparator, needsPostQueryOrdering());
 
-        Comparator<List<ByteBuffer>> comparator = orderingComparator.prepareFor(table, getRowFilter(options, state), options);
-        if (comparator != null)
-            cqlRows.rows.sort(comparator);
+        if (orderingComparator == null || orderingComparator.indexOrdering())
+        {
+            return SortedRowsBuilder.create(limit, offset);
+        }
+        else
+        {
+            return SortedRowsBuilder.create(limit, offset, orderingComparator);
+        }
     }
 
-    public static class RawStatement extends QualifiedStatement
+    public static class RawStatement extends QualifiedStatement<SelectStatement>
     {
         public final Parameters parameters;
         public final List<RawSelector> selectClause;
         public final WhereClause whereClause;
         public final Term.Raw limit;
         public final Term.Raw perPartitionLimit;
-        private ClientState state;
+        public final Term.Raw offset;
+        public final SelectOptions options;
 
         public RawStatement(QualifiedName cfName,
                             Parameters parameters,
                             List<RawSelector> selectClause,
                             WhereClause whereClause,
                             Term.Raw limit,
-                            Term.Raw perPartitionLimit)
+                            Term.Raw perPartitionLimit,
+                            Term.Raw offset,
+                            SelectOptions options)
         {
             super(cfName);
             this.parameters = parameters;
@@ -1201,31 +1448,44 @@ public RawStatement(QualifiedName cfName,
             this.whereClause = whereClause;
             this.limit = limit;
             this.perPartitionLimit = perPartitionLimit;
+            this.offset = offset;
+            this.options = options;
         }
 
-        public SelectStatement prepare(ClientState state)
+        @Override
+        public SelectStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            // Cache locally for use by Guardrails
-            this.state = state;
-            return prepare(state, false);
+            setKeyspace(state);
+            return prepare(state, false, keyspaceMapper);
         }
 
-        public SelectStatement prepare(ClientState state, boolean forView) throws InvalidRequestException
+        public SelectStatement prepare(ClientState state, boolean forView, UnaryOperator<String> keyspaceMapper) throws InvalidRequestException
         {
-            TableMetadata table = Schema.instance.validateTable(keyspace(), name());
+            String ks = keyspaceMapper.apply(keyspace());
+            TableMetadata table = Schema.instance.validateTable(ks, name());
 
             List<Selectable> selectables = RawSelector.toSelectables(selectClause, table);
             boolean containsOnlyStaticColumns = selectOnlyStaticColumns(table, selectables);
 
+            // Besides actual restrictions (where clauses), prepareRestrictions will include pseudo-restrictions
+            // on indexed columns to allow pushing ORDER BY into the index; see StatementRestrictions::addOrderingRestrictions.
+            // Therefore, we don't want to convert an ANN Ordering column into a +score column until after that.
             List<Ordering> orderings = getOrderings(table);
-            StatementRestrictions restrictions = prepareRestrictions(state, table, bindVariables, orderings, containsOnlyStaticColumns, forView);
+            StatementRestrictions restrictions = prepareRestrictions(
+                    state, table, bindVariables, orderings, containsOnlyStaticColumns, forView);
 
             // If we order post-query, the sorted column needs to be in the ResultSet for sorting,
             // even if we don't ultimately ship them to the client (CASSANDRA-4911).
             Map<ColumnMetadata, Ordering> orderingColumns = getOrderingColumns(orderings);
+            // +score column for ANN/BM25
+            var scoreOrdering = getScoreOrdering(orderings);
+            assert scoreOrdering == null || orderingColumns.isEmpty() : "can't have both scored ordering and column ordering";
+            if (scoreOrdering != null)
+                orderingColumns = scoreOrdering;
             Set<ColumnMetadata> resultSetOrderingColumns = getResultSetOrdering(restrictions, orderingColumns);
 
-            Selection selection = prepareSelection(table,
+            Selection selection = prepareSelection(state,
+                                                   table,
                                                    selectables,
                                                    bindVariables,
                                                    resultSetOrderingColumns,
@@ -1253,16 +1513,19 @@ public SelectStatement prepare(ClientState state, boolean forView) throws Invali
             if (!orderingColumns.isEmpty())
             {
                 assert !forView;
-                verifyOrderingIsAllowed(restrictions, orderingColumns);
+                verifyOrderingIsAllowed(table, restrictions, orderingColumns);
                 orderingComparator = getOrderingComparator(selection, restrictions, orderingColumns);
-                isReversed = isReversed(table, orderingColumns, restrictions);
+                isReversed = isReversed(table, orderingColumns);
                 if (isReversed && orderingComparator != null)
                     orderingComparator = orderingComparator.reverse();
-           }
+            }
 
-            checkNeedsFiltering(table, restrictions);
+            checkNeedsFiltering(table, restrictions, state);
 
-            return new SelectStatement(table,
+            checkDisjunctionIsSupported(table, restrictions);
+
+            return new SelectStatement(rawCQLStatement,
+                                       table,
                                        bindVariables,
                                        parameters,
                                        selection,
@@ -1270,8 +1533,25 @@ public SelectStatement prepare(ClientState state, boolean forView) throws Invali
                                        isReversed,
                                        aggregationSpecFactory,
                                        orderingComparator,
-                                       prepareLimit(bindVariables, limit, keyspace(), limitReceiver()),
-                                       prepareLimit(bindVariables, perPartitionLimit, keyspace(), perPartitionLimitReceiver()));
+                                       prepareLimit(bindVariables, limit, ks, limitReceiver()),
+                                       prepareLimit(bindVariables, perPartitionLimit, ks, perPartitionLimitReceiver()),
+                                       prepareLimit(bindVariables, offset, ks, offsetReceiver()),
+                                       options);
+        }
+
+        private Map<ColumnMetadata, Ordering> getScoreOrdering(List<Ordering> orderings)
+        {
+            if (orderings.isEmpty())
+                return null;
+
+            var expr = orderings.get(0).expression;
+            if (!expr.isScored())
+                return null;
+
+            // Create synthetic score column
+            ColumnMetadata sourceColumn = expr.getColumn();
+            var cm = ColumnMetadata.syntheticScoreColumn(sourceColumn, FloatType.instance);
+            return Map.of(cm, orderings.get(0));
         }
 
         private Set<ColumnMetadata> getResultSetOrdering(StatementRestrictions restrictions, Map<ColumnMetadata, Ordering> orderingColumns)
@@ -1281,7 +1561,8 @@ private Set<ColumnMetadata> getResultSetOrdering(StatementRestrictions restricti
             return Collections.emptySet();
         }
 
-        private Selection prepareSelection(TableMetadata table,
+        private Selection prepareSelection(ClientState state,
+                                           TableMetadata table,
                                            List<Selectable> selectables,
                                            VariableSpecifications boundNames,
                                            Set<ColumnMetadata> resultSetOrderingColumns,
@@ -1299,7 +1580,7 @@ private Selection prepareSelection(TableMetadata table,
             {
                 return hasGroupBy || table.hasMaskedColumns()
                        ? Selection.wildcardWithGroupByOrMaskedColumns(table, boundNames, resultSetOrderingColumns, isJson, returnStaticContentOnPartitionWithNoRows)
-                       : Selection.wildcard(table, isJson, returnStaticContentOnPartitionWithNoRows);
+                       : Selection.wildcard(table, resultSetOrderingColumns, isJson, returnStaticContentOnPartitionWithNoRows);
             }
 
             return Selection.fromSelectors(table,
@@ -1341,20 +1622,21 @@ private Map<ColumnMetadata, Ordering> getOrderingColumns(List<Ordering> ordering
             if (orderings.isEmpty())
                 return Collections.emptyMap();
 
-            Map<ColumnMetadata, Ordering> orderingColumns = new LinkedHashMap<>();
-            for (Ordering ordering : orderings)
-            {
-                ColumnMetadata column = ordering.expression.getColumn();
-                orderingColumns.put(column, ordering);
-            }
-            return orderingColumns;
+            return orderings.stream()
+                            .filter(ordering -> !ordering.expression.isScored())
+                            .collect(Collectors.toMap(ordering -> ordering.expression.getColumn(),
+                                                      ordering -> ordering,
+                                                      (a, b) -> {
+                                                          throw new IllegalStateException("Duplicate keys");
+                                                      },
+                                                      LinkedHashMap::new));
         }
 
         private List<Ordering> getOrderings(TableMetadata table)
         {
             return parameters.orderings.stream()
-                                       .map(o -> o.bind(table, bindVariables))
-                                       .collect(Collectors.toList());
+                    .map(o -> o.bind(table, bindVariables))
+                    .collect(Collectors.toList());
         }
 
         /**
@@ -1373,15 +1655,15 @@ private StatementRestrictions prepareRestrictions(ClientState state,
                                                           boolean selectsOnlyStaticColumns,
                                                           boolean forView) throws InvalidRequestException
         {
-            return new StatementRestrictions(state,
+            return StatementRestrictions.create(state,
                                              StatementType.SELECT,
-                                             metadata,
-                                             whereClause,
-                                             boundNames,
-                                             orderings,
-                                             selectsOnlyStaticColumns,
-                                             parameters.allowFiltering,
-                                             forView);
+                                                metadata,
+                                                whereClause,
+                                                boundNames,
+                                                orderings,
+                                                selectsOnlyStaticColumns,
+                                                parameters.allowFiltering,
+                                                forView);
         }
 
         /** Returns a Term for the limit or null if no limit is set */
@@ -1396,12 +1678,28 @@ private Term prepareLimit(VariableSpecifications boundNames, Term.Raw limit,
             return prepLimit;
         }
 
-        private static void verifyOrderingIsAllowed(StatementRestrictions restrictions, Map<ColumnMetadata, Ordering> orderingColumns) throws InvalidRequestException
+        private static void verifyOrderingIsAllowed(TableMetadata table, StatementRestrictions restrictions, Map<ColumnMetadata, Ordering> orderingColumns) throws InvalidRequestException
         {
             if (orderingColumns.values().stream().anyMatch(o -> o.expression.hasNonClusteredOrdering()))
                 return;
-            checkFalse(restrictions.usesSecondaryIndexing(), "ORDER BY with 2ndary indexes is not supported, except for ANN queries.");
+
+            checkFalse(restrictions.usesSecondaryIndexing(), "ORDER BY with 2ndary indexes is not supported.");
             checkFalse(restrictions.isKeyRange(), "ORDER BY is only supported when the partition key is restricted by an EQ or an IN.");
+
+            // check that clustering columns are valid
+            int i = 0;
+            for (var entry : orderingColumns.entrySet())
+            {
+                ColumnMetadata def = entry.getKey();
+                checkTrue(def.isClusteringColumn(),
+                          "Order by is currently only supported on indexed columns and the clustered columns of the PRIMARY KEY, got %s", def.name);
+                while (i != def.position())
+                {
+                    checkTrue(restrictions.isColumnRestrictedByEq(table.clusteringColumns().get(i++)),
+                              "Ordering by clustered columns must follow the declared order in the PRIMARY KEY");
+                }
+                i++;
+            }
         }
 
         private static void validateDistinctSelection(TableMetadata metadata,
@@ -1435,7 +1733,7 @@ private static void validateDistinctSelection(TableMetadata metadata,
          * @param metadata the table metadata
          * @param selection the selection
          * @param restrictions the restrictions
-         * @param isDistinct <code>true</code> if the query is a DISTINCT one. 
+         * @param isDistinct <code>true</code> if the query is a DISTINCT one.
          * @return the {@code AggregationSpecification.Factory} used to make the aggregates
          */
         private AggregationSpecification.Factory getAggregationSpecFactory(TableMetadata metadata,
@@ -1529,14 +1827,15 @@ private void validateGroupByFunction(WithFunction withFunction)
 
         private ColumnComparator<List<ByteBuffer>> getOrderingComparator(Selection selection,
                                                                          StatementRestrictions restrictions,
-                                                                         Map<ColumnMetadata, Ordering> orderingColumns) throws InvalidRequestException
+                                                                         Map<ColumnMetadata, Ordering> orderingColumns)
+                                                                   throws InvalidRequestException
         {
             for (Map.Entry<ColumnMetadata, Ordering> e : orderingColumns.entrySet())
             {
                 if (e.getValue().expression.hasNonClusteredOrdering())
                 {
                     Preconditions.checkState(orderingColumns.size() == 1);
-                    return new IndexColumnComparator(e.getValue().expression.toRestriction(), selection.getOrderingIndex(e.getKey()));
+                    return new IndexColumnComparator();
                 }
             }
 
@@ -1551,37 +1850,35 @@ private ColumnComparator<List<ByteBuffer>> getOrderingComparator(Selection selec
                 idToSort.add(selection.getOrderingIndex(orderingColumn));
                 sorters.add(orderingColumn.type);
             }
+
             return idToSort.size() == 1 ? new SingleColumnComparator(idToSort.get(0), sorters.get(0))
                     : new CompositeComparator(sorters, idToSort);
         }
 
-        private boolean isReversed(TableMetadata table, Map<ColumnMetadata, Ordering> orderingColumns, StatementRestrictions restrictions) throws InvalidRequestException
+        private boolean isReversed(TableMetadata table, Map<ColumnMetadata, Ordering> orderingColumns) throws InvalidRequestException
         {
+            // Nonclustered ordering handles descending logic through ScoreOrderedResultRetriever and TKP
             if (orderingColumns.values().stream().anyMatch(o -> o.expression.hasNonClusteredOrdering()))
                 return false;
-            Boolean[] reversedMap = new Boolean[table.clusteringColumns().size()];
-            int i = 0;
+
+            Boolean[] clusteredMap = new Boolean[table.clusteringColumns().size()];
             for (var entry : orderingColumns.entrySet())
             {
                 ColumnMetadata def = entry.getKey();
                 Ordering ordering = entry.getValue();
-                boolean reversed = ordering.direction == Ordering.Direction.DESC;
-
-                checkTrue(def.isClusteringColumn(),
-                          "Order by is currently only supported on the clustered columns of the PRIMARY KEY, got %s", def.name);
-
-                while (i != def.position())
-                {
-                    checkTrue(restrictions.isColumnRestrictedByEq(table.clusteringColumns().get(i++)),
-                              "Order by currently only supports the ordering of columns following their declared order in the PRIMARY KEY");
-                }
-                i++;
-                reversedMap[def.position()] = (reversed != def.isReversedType());
+                // We defined ANN OF to be ASC ordering, as in, "order by near-ness".  But since score goes from
+                // 0 (worst) to 1 (closest), we need to reverse the ordering for the comparator when we're sorting
+                // by synthetic +score column.
+                boolean cqlReversed = ordering.direction == Ordering.Direction.DESC;
+                if (def.position() == ColumnMetadata.NO_POSITION)
+                    return ordering.expression.isScored() || cqlReversed;
+                else
+                    clusteredMap[def.position()] = (cqlReversed != def.isReversedType());
             }
 
-            // Check that all boolean in reversedMap, if set, agrees
+            // Check that all boolean in clusteredMap, if set, agrees
             Boolean isReversed = null;
-            for (Boolean b : reversedMap)
+            for (Boolean b : clusteredMap)
             {
                 // Column on which order is specified can be in any order
                 if (b == null)
@@ -1598,16 +1895,43 @@ private boolean isReversed(TableMetadata table, Map<ColumnMetadata, Ordering> or
             return isReversed;
         }
 
+        /**
+         * This verifies that if the expression contains a disjunction - "value = 1 or value = 2" or "value in (1, 2)"
+         * the indexes involved in the query support disjunction.
+         */
+        private void checkDisjunctionIsSupported(TableMetadata table, StatementRestrictions restrictions) throws InvalidRequestException
+        {
+            if (!parameters.allowFiltering &&
+                restrictions.usesSecondaryIndexing() &&
+                restrictions.needsDisjunctionSupport(table))
+            {
+                restrictions.throwsRequiresIndexSupportingDisjunctionError();
+            }
+        }
+
         /** If ALLOW FILTERING was not specified, this verifies that it is not needed */
-        private void checkNeedsFiltering(TableMetadata table, StatementRestrictions restrictions) throws InvalidRequestException
+        private void checkNeedsFiltering(TableMetadata table, StatementRestrictions restrictions, ClientState state) throws InvalidRequestException
         {
+            if (parameters.allowFiltering && restrictions.hasIndxBasedOrdering())
+            {
+                // ANN queries do not currently work correctly when filtering is required, so
+                // we fail even though ALLOW FILTERING was passed
+                if (restrictions.needFiltering(table))
+                    throw invalidRequest(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
+            }
             // non-key-range non-indexed queries cannot involve filtering underneath
             if (!parameters.allowFiltering && (restrictions.isKeyRange() || restrictions.usesSecondaryIndexing()))
             {
                 // We will potentially filter data if the row filter is not the identity and there isn't any index group
                 // supporting all the expressions in the filter.
-                if (restrictions.requiresAllowFilteringIfNotSpecified())
-                    checkFalse(restrictions.needFiltering(table), StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+                if (restrictions.needFiltering(table))
+                {
+                    restrictions.throwRequiresAllowFilteringError(table, state);
+                }
+                if (restrictions.hasClusteringColumnsRestrictions()
+                    && restrictions.hasIndxBasedOrdering()
+                    && restrictions.hasClusterColumnRestrictionWithoutSupportingIndex(table))
+                        restrictions.throwRequiresAllowFilteringError(table, state);
             }
         }
 
@@ -1621,6 +1945,11 @@ private ColumnSpecification perPartitionLimitReceiver()
             return new ColumnSpecification(keyspace(), name(), new ColumnIdentifier("[per_partition_limit]", true), Int32Type.instance);
         }
 
+        private ColumnSpecification offsetReceiver()
+        {
+            return new ColumnSpecification(keyspace(), name(), new ColumnIdentifier("[offset]", true), Int32Type.instance);
+        }
+
         @Override
         public String toString()
         {
@@ -1678,14 +2007,6 @@ public boolean indexOrdering()
         {
             return false;
         }
-
-        /**
-         * Produces a prepared {@link ColumnComparator} for current table and query-options
-         */
-        public Comparator<T> prepareFor(TableMetadata table, RowFilter rowFilter, QueryOptions options)
-        {
-            return this;
-        }
     }
 
     private static class ReversedColumnComparator<T> extends ColumnComparator<T>
@@ -1702,6 +2023,12 @@ public int compare(T o1, T o2)
         {
             return wrapped.compare(o2, o1);
         }
+
+        @Override
+        public boolean indexOrdering()
+        {
+            return wrapped.indexOrdering();
+        }
     }
 
     /**
@@ -1726,35 +2053,12 @@ public int compare(List<ByteBuffer> a, List<ByteBuffer> b)
 
     private static class IndexColumnComparator extends ColumnComparator<List<ByteBuffer>>
     {
-        private final SingleRestriction restriction;
-        private final int columnIndex;
-
-        public IndexColumnComparator(SingleRestriction restriction, int columnIndex)
-        {
-            this.restriction = restriction;
-            this.columnIndex = columnIndex;
-        }
-
         @Override
         public boolean indexOrdering()
         {
             return true;
         }
 
-        @Override
-        public Comparator<List<ByteBuffer>> prepareFor(TableMetadata table, RowFilter rowFilter, QueryOptions options)
-        {
-            if (table.indexes.isEmpty() || rowFilter.isEmpty())
-                return this;
-
-            Index.QueryPlan indexQueryPlan = Keyspace.openAndGetStore(table).indexManager.getBestIndexQueryPlanFor(rowFilter);
-
-            Index index = restriction.findSupportingIndexFromQueryPlan(indexQueryPlan);
-            assert index != null;
-            Comparator<ByteBuffer> comparator = index.getPostQueryOrdering(restriction, options);
-            return (a, b) -> compare(comparator, a.get(columnIndex), b.get(columnIndex));
-        }
-
         @Override
         public int compare(List<ByteBuffer> o1, List<ByteBuffer> o2)
         {
@@ -1792,7 +2096,7 @@ public int compare(List<ByteBuffer> a, List<ByteBuffer> b)
             return 0;
         }
     }
-    
+
     @Override
     public String toString()
     {
@@ -1920,7 +2224,7 @@ private String asCQL(QueryOptions options, ClientState state)
                 sb.append(" AND ").append(filterString);
         }
 
-        DataLimits limits = getDataLimits(getLimit(options), getPerPartitionLimit(options), options.getPageSize(), getAggregationSpec(options));
+        DataLimits limits = getDataLimits(state, getLimit(options), getPerPartitionLimit(options), getOffset(options), getAggregationSpec(options));
         if (limits != DataLimits.NONE)
             sb.append(' ').append(limits);
         return sb.toString();
diff --git a/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java
index 5ff299eb88d4..f9dd2e36546c 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java
@@ -83,7 +83,7 @@ public PartitionUpdate.Builder getPartitionUpdateBuilder(TableMetadata metadata,
         PartitionUpdate.Builder builder = puBuilders.get(dk.getKey());
         if (builder == null)
         {
-            builder = new PartitionUpdate.Builder(metadata, dk, updatedColumns, perPartitionKeyCounts.count(dk.getKey()));
+            builder = PartitionUpdate.builder(metadata, dk, updatedColumns, perPartitionKeyCounts.count(dk.getKey()));
             puBuilders.put(dk.getKey(), builder);
         }
         return builder;
diff --git a/src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java b/src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java
index 2d1d58c7aa23..982c4c428347 100644
--- a/src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java
@@ -17,17 +17,27 @@
  */
 package org.apache.cassandra.cql3.statements;
 
+import java.lang.reflect.InvocationTargetException;
 import java.util.concurrent.TimeoutException;
+import java.util.function.UnaryOperator;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
-import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.TruncateException;
+import org.apache.cassandra.exceptions.UnauthorizedException;
+import org.apache.cassandra.exceptions.UnavailableException;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
@@ -36,19 +46,36 @@
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.TRUNCATE_STATEMENT_PROVIDER;
 
-public class TruncateStatement extends QualifiedStatement implements CQLStatement
+public class TruncateStatement implements CQLStatement, CQLStatement.SingleKeyspaceCqlStatement
 {
-    public TruncateStatement(QualifiedName name)
+    private final String rawCQLStatement;
+    private final QualifiedName qualifiedName;
+
+    public TruncateStatement(String queryString, QualifiedName name)
+    {
+        this.rawCQLStatement = queryString;
+        this.qualifiedName = name;
+    }
+
+    @Override
+    public String getRawCQLStatement()
     {
-        super(name);
+        return rawCQLStatement;
     }
 
-    public TruncateStatement prepare(ClientState state)
+    @Override
+    public String keyspace()
     {
-        return this;
+        return qualifiedName.getKeyspace();
+    }
+
+    public String name()
+    {
+        return qualifiedName.getName();
     }
 
     public void authorize(ClientState state) throws InvalidRequestException, UnauthorizedException
@@ -56,6 +83,7 @@ public void authorize(ClientState state) throws InvalidRequestException, Unautho
         state.ensureTablePermission(keyspace(), name(), Permission.MODIFY);
     }
 
+    @Override
     public void validate(ClientState state) throws InvalidRequestException
     {
         Schema.instance.validateTable(keyspace(), name());
@@ -68,6 +96,9 @@ public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.
         try
         {
             TableMetadata metaData = Schema.instance.getTableMetadata(keyspace(), name());
+            if (metaData == null)
+                throw new InvalidRequestException(String.format("Unknown keyspace/table %s.%s", keyspace(), name()));
+
             if (metaData.isView())
                 throw new InvalidRequestException("Cannot TRUNCATE materialized view directly; must truncate base table instead");
 
@@ -77,10 +108,10 @@ public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.
             }
             else
             {
-                StorageProxy.truncateBlocking(keyspace(), name());
+                doTruncateBlocking();
             }
         }
-        catch (UnavailableException | TimeoutException e)
+        catch (UnavailableException | TimeoutException | InvalidRequestException e)
         {
             throw new TruncateException(e);
         }
@@ -92,6 +123,9 @@ public ResultMessage executeLocally(QueryState state, QueryOptions options)
         try
         {
             TableMetadata metaData = Schema.instance.getTableMetadata(keyspace(), name());
+            if (metaData == null)
+                throw new InvalidRequestException(String.format("Unknown keyspace/table %s.%s", keyspace(), name()));
+            
             if (metaData.isView())
                 throw new InvalidRequestException("Cannot TRUNCATE materialized view directly; must truncate base table instead");
 
@@ -117,6 +151,11 @@ private void executeForVirtualTable(TableId id)
         VirtualKeyspaceRegistry.instance.getTableNullable(id).truncate();
     }
 
+    protected void doTruncateBlocking() throws TimeoutException
+    {
+        StorageProxy.instance.truncateBlocking(keyspace(), name());
+    }
+
     @Override
     public String toString()
     {
@@ -128,4 +167,57 @@ public AuditLogContext getAuditLogContext()
     {
         return new AuditLogContext(AuditLogEntryType.TRUNCATE, keyspace(), name());
     }
+
+    public static final class Raw extends QualifiedStatement<TruncateStatement>
+    {
+        public Raw(QualifiedName name)
+        {
+            super(name);
+        }
+
+        @Override
+        public TruncateStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
+        {
+            setKeyspace(state);
+            String ks = keyspaceMapper.apply(keyspace());
+            QualifiedName qual = qualifiedName;
+            if (!ks.equals(qual.getKeyspace()))
+                qual = new QualifiedName(ks, qual.getName());
+            return provider.createTruncateStatement(rawCQLStatement, qual);
+        }
+    }
+
+    private static TruncateStatementProvider getProviderFromProperty()
+    {
+        try
+        {
+            return (TruncateStatementProvider)FBUtilities.classForName(TRUNCATE_STATEMENT_PROVIDER.getString(),
+                                                                       "Truncate statement provider")
+                                                         .getConstructor().newInstance();
+        }
+        catch (NoSuchMethodException | IllegalAccessException | InstantiationException |
+               InvocationTargetException e)
+        {
+            throw new RuntimeException("Unable to find a truncate statement provider with name " +
+                                       TRUNCATE_STATEMENT_PROVIDER.getString(), e);
+        }
+    }
+
+    private static final TruncateStatementProvider provider = TRUNCATE_STATEMENT_PROVIDER.isPresent() ?
+                                                              getProviderFromProperty() :
+                                                              new DefaultTruncateStatementProvider();
+
+    public static interface TruncateStatementProvider
+    {
+        public TruncateStatement createTruncateStatement(String rawCQLStatement, QualifiedName qual);
+    }
+
+    public static final class DefaultTruncateStatementProvider implements TruncateStatementProvider
+    {
+        @Override
+        public TruncateStatement createTruncateStatement(String rawCQLStatement, QualifiedName qual)
+        {
+            return new TruncateStatement(rawCQLStatement, qual);
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
index d3f605a00865..dc1a821725c2 100644
--- a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
@@ -21,22 +21,44 @@
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Json;
+import org.apache.cassandra.cql3.Operation;
+import org.apache.cassandra.cql3.Operations;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.SingleColumnRelation;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.UpdateParameters;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.WhereClause;
 import org.apache.cassandra.cql3.conditions.ColumnCondition;
 import org.apache.cassandra.cql3.conditions.Conditions;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.Slice;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.sensors.SensorsCustomParams;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.Type;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Pair;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
 
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkContainsNoDuplicates;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
@@ -49,7 +71,8 @@ public class UpdateStatement extends ModificationStatement
 {
     private static final Constants.Value EMPTY = new Constants.Value(ByteBufferUtil.EMPTY_BYTE_BUFFER);
 
-    private UpdateStatement(StatementType type,
+    private UpdateStatement(String queryString,
+                            StatementType type,
                             VariableSpecifications bindVariables,
                             TableMetadata metadata,
                             Operations operations,
@@ -57,7 +80,7 @@ private UpdateStatement(StatementType type,
                             Conditions conditions,
                             Attributes attrs)
     {
-        super(type, bindVariables, metadata, operations, restrictions, conditions, attrs);
+        super(queryString, type, bindVariables, metadata, operations, restrictions, conditions, attrs);
     }
 
     @Override
@@ -110,6 +133,23 @@ public void addUpdateForKey(PartitionUpdate.Builder update, Slice slice, UpdateP
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime)
+    {
+        ResultMessage result = super.execute(state, options, requestTime);
+
+        if (result == null)
+            result = new ResultMessage.Void();
+
+        RequestSensors sensors = RequestTracker.instance.get();
+        Context context = Context.from(this.metadata());
+        SensorsCustomParams.addSensorToCQLResponse(result, options.getProtocolVersion(), sensors, context, Type.WRITE_BYTES);
+        // CAS updates incorporate read sensors
+        SensorsCustomParams.addSensorToCQLResponse(result, options.getProtocolVersion(), sensors, context, Type.READ_BYTES);
+
+        return result;
+    }
+
     public static class ParsedInsert extends ModificationStatement.Parsed
     {
         private final List<ColumnIdentifier> columnNames;
@@ -178,17 +218,18 @@ protected ModificationStatement prepareInternal(ClientState state,
 
             boolean applyOnlyToStaticColumns = !hasClusteringColumnsSet && appliesOnlyToStaticColumns(operations, conditions);
 
-            StatementRestrictions restrictions = new StatementRestrictions(state,
-                                                                           type,
-                                                                           metadata,
-                                                                           whereClause.build(),
-                                                                           bindVariables,
-                                                                           Collections.emptyList(),
-                                                                           applyOnlyToStaticColumns,
-                                                                           false,
-                                                                           false);
-
-            return new UpdateStatement(type,
+            StatementRestrictions restrictions = StatementRestrictions.create(state,
+                                                                              type,
+                                                                              metadata,
+                                                                              whereClause.build(),
+                                                                              bindVariables,
+                                                                              Collections.emptyList(),
+                                                                              applyOnlyToStaticColumns,
+                                                                              false,
+                                                                              false);
+
+            return new UpdateStatement(rawCQLStatement,
+                                       type,
                                        bindVariables,
                                        metadata,
                                        operations,
@@ -249,17 +290,18 @@ protected ModificationStatement prepareInternal(ClientState state,
 
             boolean applyOnlyToStaticColumns = !hasClusteringColumnsSet && appliesOnlyToStaticColumns(operations, conditions);
 
-            StatementRestrictions restrictions = new StatementRestrictions(state,
-                                                                           type,
-                                                                           metadata,
-                                                                           whereClause.build(),
-                                                                           bindVariables,
-                                                                           Collections.emptyList(),
-                                                                           applyOnlyToStaticColumns,
-                                                                           false,
-                                                                           false);
-
-            return new UpdateStatement(type,
+            StatementRestrictions restrictions = StatementRestrictions.create(state,
+                                                                              type,
+                                                                              metadata,
+                                                                              whereClause.build(),
+                                                                              bindVariables,
+                                                                              Collections.emptyList(),
+                                                                              applyOnlyToStaticColumns,
+                                                                              false,
+                                                                              false);
+
+            return new UpdateStatement(rawCQLStatement,
+                                       type,
                                        bindVariables,
                                        metadata,
                                        operations,
@@ -325,7 +367,8 @@ protected ModificationStatement prepareInternal(ClientState state,
                                                                  conditions,
                                                                  Collections.emptyList());
 
-            return new UpdateStatement(type,
+            return new UpdateStatement(rawCQLStatement,
+                                       type,
                                        bindVariables,
                                        metadata,
                                        operations,
diff --git a/src/java/org/apache/cassandra/cql3/statements/UseStatement.java b/src/java/org/apache/cassandra/cql3/statements/UseStatement.java
index b3819b5cd26b..1dcf904a0231 100644
--- a/src/java/org/apache/cassandra/cql3/statements/UseStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UseStatement.java
@@ -17,6 +17,9 @@
  */
 package org.apache.cassandra.cql3.statements;
 
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -29,12 +32,10 @@
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
 
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
 
-public class UseStatement extends CQLStatement.Raw implements CQLStatement
+public class UseStatement extends CQLStatement.Raw implements CQLStatement.SingleKeyspaceCqlStatement
 {
     private final String keyspace;
 
@@ -73,7 +74,7 @@ public ResultMessage executeLocally(QueryState state, QueryOptions options) thro
         // but for some unit tests we need to set the keyspace (e.g. for tests with DROP INDEX)
         return execute(state, options, Dispatcher.RequestTime.forImmediateExecution());
     }
-    
+
     @Override
     public String toString()
     {
@@ -86,6 +87,7 @@ public AuditLogContext getAuditLogContext()
         return new AuditLogContext(AuditLogEntryType.USE_KEYSPACE, keyspace);
     }
 
+    @Override
     public String keyspace()
     {
         return keyspace;
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
index b8a27af3bab5..98fd6f9d8d9a 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
@@ -19,7 +19,9 @@
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -27,7 +29,7 @@
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
@@ -50,22 +52,38 @@
 import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT;
 import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_TRANSIENT_CHANGES;
 
-public final class AlterKeyspaceStatement extends AlterSchemaStatement
+public final class AlterKeyspaceStatement extends AlterSchemaStatement implements AlterSchemaStatement.WithKeyspaceAttributes
 {
     private static final boolean allow_alter_rf_during_range_movement = ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT.getBoolean();
     private static final boolean allow_unsafe_transient_changes = ALLOW_UNSAFE_TRANSIENT_CHANGES.getBoolean();
-    private final HashSet<String> clientWarnings = new HashSet<>();
 
     private final KeyspaceAttributes attrs;
     private final boolean ifExists;
 
-    public AlterKeyspaceStatement(String keyspaceName, KeyspaceAttributes attrs, boolean ifExists)
+    public AlterKeyspaceStatement(String queryString, String keyspaceName, KeyspaceAttributes attrs, boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.attrs = attrs;
         this.ifExists = ifExists;
     }
 
+    public Object getAttribute(String key)
+    {
+        return attrs.getProperty(key);
+    }
+
+    public void overrideAttribute(String oldKey, String newKey, String newValue)
+    {
+        attrs.removeProperty(oldKey);
+        attrs.addProperty(newKey, newValue);
+    }
+
+    public void overrideAttribute(String oldKey, String newKey, Map<String, String> newValue)
+    {
+        attrs.removeProperty(oldKey);
+        attrs.addProperty(newKey, newValue);
+    }
+
     public Keyspaces apply(Keyspaces schema)
     {
         attrs.validate();
@@ -86,7 +104,7 @@ public Keyspaces apply(Keyspaces schema)
         if (newKeyspace.params.replication.klass.equals(LocalStrategy.class))
             throw ire("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
 
-        newKeyspace.params.validate(keyspaceName, state);
+        newKeyspace.validate(state);
 
         validateNoRangeMovements();
         validateTransientReplication(keyspace.createReplicationStrategy(), newKeyspace.createReplicationStrategy());
@@ -109,6 +127,7 @@ public void authorize(ClientState client)
     @Override
     Set<String> clientWarnings(KeyspacesDiff diff)
     {
+        HashSet<String> clientWarnings = new HashSet<>();
         if (diff.isEmpty())
             return clientWarnings;
 
@@ -204,7 +223,7 @@ public String toString()
         return String.format("%s (%s)", getClass().getSimpleName(), keyspaceName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<AlterKeyspaceStatement>
     {
         private final String keyspaceName;
         private final KeyspaceAttributes attrs;
@@ -217,9 +236,10 @@ public Raw(String keyspaceName, KeyspaceAttributes attrs, boolean ifExists)
             this.ifExists = ifExists;
         }
 
-        public AlterKeyspaceStatement prepare(ClientState state)
+        @Override
+        public AlterKeyspaceStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            return new AlterKeyspaceStatement(keyspaceName, attrs, ifExists);
+            return new AlterKeyspaceStatement(rawCQLStatement, keyspaceMapper.apply(keyspaceName), attrs, ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java
index a75c0ac4f129..5aeb7656e404 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java
@@ -17,7 +17,9 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 
 import com.google.common.collect.ImmutableSet;
 
@@ -38,16 +40,24 @@
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
-abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspaceCqlStatement, SchemaTransformation
+public abstract class AlterSchemaStatement implements CQLStatement.SingleKeyspaceCqlStatement, SchemaTransformation
 {
-    protected final String keyspaceName; // name of the keyspace affected by the statement
+    private final String rawCQLStatement;
+    protected String keyspaceName; // name of the keyspace affected by the statement
     protected ClientState state;
 
-    protected AlterSchemaStatement(String keyspaceName)
+    protected AlterSchemaStatement(String queryString, String keyspaceName)
     {
+        this.rawCQLStatement = queryString;
         this.keyspaceName = keyspaceName;
     }
 
+    @Override
+    public String getRawCQLStatement()
+    {
+        return rawCQLStatement;
+    }
+
     public void validate(ClientState state)
     {
         // validation is performed while executing the statement, in apply()
@@ -68,6 +78,11 @@ public String keyspace()
         return keyspaceName;
     }
 
+    public void overrideKeyspace(Function<String, String> overrideKeyspace)
+    {
+        this.keyspaceName = overrideKeyspace.apply(keyspaceName);
+    }
+
     public ResultMessage executeLocally(QueryState state, QueryOptions options)
     {
         return execute(state, true);
@@ -172,4 +187,13 @@ static InvalidRequestException ire(String format, Object... args)
     {
         return new InvalidRequestException(String.format(format, args));
     }
+
+    public static interface WithKeyspaceAttributes
+    {
+        Object getAttribute(String key);
+
+        void overrideAttribute(String oldKey, String newKey, String newValue);
+
+        void overrideAttribute(String oldKey, String newKey, Map<String, String> newValue);
+    }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
index 5f37acf09b73..76719a47aa6e 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
@@ -26,7 +26,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-
+import java.util.function.UnaryOperator;
 import javax.annotation.Nullable;
 
 import com.google.common.base.Splitter;
@@ -40,10 +40,11 @@
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.QualifiedName;
 import org.apache.cassandra.cql3.functions.masking.ColumnMask;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -55,6 +56,7 @@
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.DroppedColumn;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Keyspaces;
@@ -64,11 +66,13 @@
 import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.schema.Views;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
+import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.NoSpamLogger;
 
 import static java.lang.String.format;
@@ -85,9 +89,9 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
     private final boolean ifExists;
     protected ClientState state;
 
-    public AlterTableStatement(String keyspaceName, String tableName, boolean ifExists)
+    public AlterTableStatement(String queryString, String keyspaceName, String tableName, boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.tableName = tableName;
         this.ifExists = ifExists;
     }
@@ -122,6 +126,11 @@ public Keyspaces apply(Keyspaces schema)
         return schema.withAddedOrUpdated(apply(keyspace, table));
     }
 
+    public ResultMessage execute(QueryState state, boolean locally)
+    {
+        return super.execute(state, locally);
+    }
+
     SchemaChange schemaChangeEvent(KeyspacesDiff diff)
     {
         return new SchemaChange(Change.UPDATED, Target.TABLE, keyspaceName, tableName);
@@ -152,9 +161,9 @@ public String toString()
      */
     public static class AlterColumn extends AlterTableStatement
     {
-        AlterColumn(String keyspaceName, String tableName, boolean ifTableExists)
+        AlterColumn(String queryString, String keyspaceName, String tableName, boolean ifTableExists)
         {
-            super(keyspaceName, tableName, ifTableExists);
+            super(queryString, keyspaceName, tableName, ifTableExists);
         }
 
         public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
@@ -173,14 +182,15 @@ public static class MaskColumn extends AlterTableStatement
         private final ColumnMask.Raw rawMask;
         private final boolean ifColumnExists;
 
-        MaskColumn(String keyspaceName,
+        MaskColumn(String queryString,
+                   String keyspaceName,
                    String tableName,
                    ColumnIdentifier columnName,
                    @Nullable ColumnMask.Raw rawMask,
                    boolean ifTableExists,
                    boolean ifColumnExists)
         {
-            super(keyspaceName, tableName, ifTableExists);
+            super(queryString, keyspaceName, tableName, ifTableExists);
             this.columnName = columnName;
             this.rawMask = rawMask;
             this.ifColumnExists = ifColumnExists;
@@ -256,14 +266,15 @@ private static class Column
                 this.isStatic = isStatic;
                 this.mask = mask;
             }
+
         }
 
         private final Collection<Column> newColumns;
         private final boolean ifColumnNotExists;
 
-        private AddColumns(String keyspaceName, String tableName, Collection<Column> newColumns, boolean ifTableExists, boolean ifColumnNotExists)
+        private AddColumns(String queryString, String keyspaceName, String tableName, Collection<Column> newColumns, boolean ifTableExists, boolean ifColumnNotExists)
         {
-            super(keyspaceName, tableName, ifTableExists);
+            super(queryString, keyspaceName, tableName, ifTableExists);
             this.newColumns = newColumns;
             this.ifColumnNotExists = ifColumnNotExists;
         }
@@ -334,8 +345,8 @@ private void addColumn(KeyspaceMetadata keyspace,
                 {
                     throw ire("Cannot re-add previously dropped column '%s' of type %s, incompatible with previous type %s",
                               name,
-                              type.asCQL3Type(),
-                              droppedColumn.type.asCQL3Type());
+                              type.asCQL3Type().toSchemaString(),
+                              droppedColumn.type.asCQL3Type().toSchemaString());
                 }
 
                 if (droppedColumn.isStatic() != isStatic)
@@ -382,9 +393,9 @@ private static class DropColumns extends AlterTableStatement
         private final boolean ifColumnExists;
         private final Long timestamp;
 
-        private DropColumns(String keyspaceName, String tableName, Set<ColumnIdentifier> removedColumns, boolean ifTableExists, boolean ifColumnExists, Long timestamp)
+        private DropColumns(String queryString, String keyspaceName, String tableName, Set<ColumnIdentifier> removedColumns, boolean ifTableExists, boolean ifColumnExists, Long timestamp)
         {
-            super(keyspaceName, tableName, ifTableExists);
+            super(queryString, keyspaceName, tableName, ifTableExists);
             this.removedColumns = removedColumns;
             this.ifColumnExists = ifColumnExists;
             this.timestamp = timestamp;
@@ -410,14 +421,6 @@ private void dropColumn(KeyspaceMetadata keyspace, TableMetadata table, ColumnId
             if (currentColumn.isPrimaryKeyColumn())
                 throw ire("Cannot drop PRIMARY KEY column %s", column);
 
-            /*
-             * Cannot allow dropping top-level columns of user defined types that aren't frozen because we cannot convert
-             * the type into an equivalent tuple: we only support frozen tuples currently. And as such we cannot persist
-             * the correct type in system_schema.dropped_columns.
-             */
-            if (currentColumn.type.isUDT() && currentColumn.type.isMultiCell())
-                throw ire("Cannot drop non-frozen column %s of user type %s", column, currentColumn.type.asCQL3Type());
-
             // TODO: some day try and find a way to not rely on Keyspace/IndexManager/Index to find dependent indexes
             Set<IndexMetadata> dependentIndexes = Keyspace.openAndGetStore(table).indexManager.getDependentIndexes(currentColumn);
             if (!dependentIndexes.isEmpty())
@@ -451,9 +454,9 @@ private static class RenameColumns extends AlterTableStatement
         private final Map<ColumnIdentifier, ColumnIdentifier> renamedColumns;
         private final boolean ifColumnsExists;
 
-        private RenameColumns(String keyspaceName, String tableName, Map<ColumnIdentifier, ColumnIdentifier> renamedColumns, boolean ifTableExists, boolean ifColumnsExists)
+        private RenameColumns(String queryString, String keyspaceName, String tableName, Map<ColumnIdentifier, ColumnIdentifier> renamedColumns, boolean ifTableExists, boolean ifColumnsExists)
         {
-            super(keyspaceName, tableName, ifTableExists);
+            super(queryString, keyspaceName, tableName, ifTableExists);
             this.renamedColumns = renamedColumns;
             this.ifColumnsExists = ifColumnsExists;
         }
@@ -524,9 +527,9 @@ private static class AlterOptions extends AlterTableStatement
     {
         private final TableAttributes attrs;
 
-        private AlterOptions(String keyspaceName, String tableName, TableAttributes attrs, boolean ifTableExists)
+        private AlterOptions(String queryString, String keyspaceName, String tableName, TableAttributes attrs, boolean ifTableExists)
         {
-            super(keyspaceName, tableName, ifTableExists);
+            super(queryString, keyspaceName, tableName, ifTableExists);
             this.attrs = attrs;
         }
 
@@ -567,7 +570,11 @@ public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
             if (!params.compression.isEnabled())
                 Guardrails.uncompressedTablesEnabled.ensureEnabled(state);
 
-            return keyspace.withSwapped(keyspace.tables.withSwapped(table.withSwapped(params)));
+            TableMetadata.Builder builder = table.unbuild().params(params);
+            for (DroppedColumn.Raw record : attrs.droppedColumnRecords())
+                builder.recordColumnDrop(record.prepare(keyspaceName, tableName, keyspace.types));
+
+            return keyspace.withSwapped(keyspace.tables.withSwapped(builder.build()));
         }
     }
 
@@ -579,9 +586,9 @@ private static class DropCompactStorage extends AlterTableStatement
     {
         private static final Logger logger = LoggerFactory.getLogger(AlterTableStatement.class);
         private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 5L, TimeUnit.MINUTES);
-        private DropCompactStorage(String keyspaceName, String tableName, boolean ifTableExists)
+        private DropCompactStorage(String queryString, String keyspaceName, String tableName, boolean ifTableExists)
         {
-            super(keyspaceName, tableName, ifTableExists);
+            super(queryString, keyspaceName, tableName, ifTableExists);
         }
 
         public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
@@ -619,7 +626,7 @@ private void validateCanDropCompactStorage()
             Set<InetAddressAndPort> preC15897nodes = new HashSet<>();
             Set<InetAddressAndPort> with2xSStables = new HashSet<>();
             Splitter onComma = Splitter.on(',').omitEmptyStrings().trimResults();
-            for (InetAddressAndPort node : StorageService.instance.getTokenMetadata().getAllEndpoints())
+            for (InetAddressAndPort node : StorageService.instance.getTokenMetadataForKeyspace(keyspaceName).getAllEndpoints())
             {
                 if (MessagingService.instance().versions.knows(node) &&
                     MessagingService.instance().versions.getRaw(node) < MessagingService.VERSION_40)
@@ -671,7 +678,7 @@ private void validateCanDropCompactStorage()
         }
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<AlterTableStatement>
     {
         private enum Kind
         {
@@ -700,7 +707,7 @@ private enum Kind
 
         // DROP
         private final Set<ColumnIdentifier> droppedColumns = new HashSet<>();
-        private Long timestamp = null; // will use execution timestamp if not provided by query
+        private Long dropTimestamp = null; // will use execution timestamp if not provided by query
 
         // RENAME
         private final Map<ColumnIdentifier, ColumnIdentifier> renamedColumns = new HashMap<>();
@@ -714,20 +721,24 @@ public Raw(QualifiedName name, boolean ifTableExists)
             this.ifTableExists = ifTableExists;
         }
 
-        public AlterTableStatement prepare(ClientState state)
+        @Override
+        public AlterTableStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
             String tableName = name.getName();
 
             switch (kind)
             {
-                case          ALTER_COLUMN: return new AlterColumn(keyspaceName, tableName, ifTableExists);
-                case           MASK_COLUMN: return new MaskColumn(keyspaceName, tableName, maskedColumn, rawMask, ifTableExists, ifColumnExists);
-                case           ADD_COLUMNS: return new AddColumns(keyspaceName, tableName, addedColumns, ifTableExists, ifColumnNotExists);
-                case          DROP_COLUMNS: return new DropColumns(keyspaceName, tableName, droppedColumns, ifTableExists, ifColumnExists, timestamp);
-                case        RENAME_COLUMNS: return new RenameColumns(keyspaceName, tableName, renamedColumns, ifTableExists, ifColumnExists);
-                case         ALTER_OPTIONS: return new AlterOptions(keyspaceName, tableName, attrs, ifTableExists);
-                case  DROP_COMPACT_STORAGE: return new DropCompactStorage(keyspaceName, tableName, ifTableExists);
+                case          ALTER_COLUMN: return new AlterColumn(rawCQLStatement, keyspaceName, tableName, ifTableExists);
+                case           MASK_COLUMN: return new MaskColumn(rawCQLStatement, keyspaceName, tableName, maskedColumn, rawMask, ifTableExists, ifColumnExists);
+                case           ADD_COLUMNS:
+                    if (keyspaceMapper != Constants.IDENTITY_STRING_MAPPER)
+                        addedColumns.forEach(c -> c.type.forEachUserType(utName -> utName.updateKeyspaceIfDefined(keyspaceMapper)));
+                    return new AddColumns(rawCQLStatement, keyspaceName, tableName, addedColumns, ifTableExists, ifColumnNotExists);
+                case          DROP_COLUMNS: return new DropColumns(rawCQLStatement, keyspaceName, tableName, droppedColumns, ifTableExists, ifColumnExists, dropTimestamp);
+                case        RENAME_COLUMNS: return new RenameColumns(rawCQLStatement, keyspaceName, tableName, renamedColumns, ifTableExists, ifColumnExists);
+                case         ALTER_OPTIONS: return new AlterOptions(rawCQLStatement, keyspaceName, tableName, attrs, ifTableExists);
+                case  DROP_COMPACT_STORAGE: return new DropCompactStorage(rawCQLStatement, keyspaceName, tableName, ifTableExists);
             }
 
             throw new AssertionError();
@@ -772,9 +783,9 @@ public void dropCompactStorage()
             kind = Kind.DROP_COMPACT_STORAGE;
         }
 
-        public void timestamp(long timestamp)
+        public void dropTimestamp(long timestamp)
         {
-            this.timestamp = timestamp;
+            this.dropTimestamp = timestamp;
         }
 
         public void rename(ColumnIdentifier from, ColumnIdentifier to)
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
index 40bca4aac991..075e0fa9aedb 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
@@ -22,11 +22,18 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.UnaryOperator;
+
+import com.google.common.collect.ImmutableList;
 
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.UTName;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UserType;
@@ -37,6 +44,7 @@
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
+import org.apache.cassandra.utils.Collections3;
 
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.filter;
@@ -44,7 +52,6 @@
 import static java.lang.String.join;
 import static java.util.function.Predicate.isEqual;
 import static java.util.stream.Collectors.toList;
-
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
 
 public abstract class AlterTypeStatement extends AlterSchemaStatement
@@ -52,9 +59,9 @@ public abstract class AlterTypeStatement extends AlterSchemaStatement
     protected final String typeName;
     protected final boolean ifExists;
 
-    public AlterTypeStatement(String keyspaceName, String typeName, boolean ifExists)
+    public AlterTypeStatement(String queryString, String keyspaceName, String typeName, boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.ifExists = ifExists;
         this.typeName = typeName;
     }
@@ -84,7 +91,13 @@ public Keyspaces apply(Keyspaces schema)
             return schema;
         }
 
-        return schema.withAddedOrUpdated(keyspace.withUpdatedUserType(apply(keyspace, type)));
+        UserType updated = apply(keyspace, type);
+        CreateTypeStatement.validate(updated);
+
+        KeyspaceMetadata newKeyspace = keyspace.withUpdatedUserType(updated);
+        newKeyspace.validate(state);
+
+        return schema.withAddedOrUpdated(newKeyspace);
     }
 
     abstract UserType apply(KeyspaceMetadata keyspace, UserType type);
@@ -108,9 +121,10 @@ private static final class AddField extends AlterTypeStatement
 
         private ClientState state;
 
-        private AddField(String keyspaceName, String typeName, FieldIdentifier fieldName, CQL3Type.Raw type, boolean ifExists, boolean ifFieldNotExists)
+        private AddField(String queryString, String keyspaceName, String typeName,
+                         FieldIdentifier fieldName, CQL3Type.Raw type, boolean ifExists, boolean ifFieldNotExists)
         {
-            super(keyspaceName, typeName, ifExists);
+            super(queryString, keyspaceName, typeName, ifExists);
             this.fieldName = fieldName;
             this.ifFieldNotExists = ifFieldNotExists;
             this.type = type;
@@ -155,9 +169,8 @@ UserType apply(KeyspaceMetadata keyspace, UserType userType)
             Guardrails.fieldsPerUDT.guard(userType.size() + 1, userType.getNameAsString(), false, state);
             type.validate(state, "Field " + fieldName);
 
-            List<FieldIdentifier> fieldNames = new ArrayList<>(userType.fieldNames()); fieldNames.add(fieldName);
-            List<AbstractType<?>> fieldTypes = new ArrayList<>(userType.fieldTypes()); fieldTypes.add(fieldType);
-
+            ImmutableList<FieldIdentifier> fieldNames = Collections3.withAppended(userType.fieldNames(), fieldName);
+            ImmutableList<AbstractType<?>> fieldTypes = Collections3.withAppended(userType.fieldTypes(), fieldType);
             return new UserType(keyspaceName, userType.name, fieldNames, fieldTypes, true);
         }
 
@@ -176,9 +189,10 @@ private static final class RenameFields extends AlterTypeStatement
         private final Map<FieldIdentifier, FieldIdentifier> renamedFields;
         private final boolean ifFieldExists;
 
-        private RenameFields(String keyspaceName, String typeName, Map<FieldIdentifier, FieldIdentifier> renamedFields, boolean ifExists, boolean ifFieldExists)
+        private RenameFields(String queryString, String keyspaceName, String typeName,
+                             Map<FieldIdentifier, FieldIdentifier> renamedFields, boolean ifExists, boolean ifFieldExists)
         {
-            super(keyspaceName, typeName, ifExists);
+            super(queryString, keyspaceName, typeName, ifExists);
             this.ifFieldExists = ifFieldExists;
             this.renamedFields = renamedFields;
         }
@@ -225,9 +239,9 @@ UserType apply(KeyspaceMetadata keyspace, UserType userType)
 
     private static final class AlterField extends AlterTypeStatement
     {
-        private AlterField(String keyspaceName, String typeName, boolean ifExists)
+        private AlterField(String queryString, String keyspaceName, String typeName, boolean ifExists)
         {
-            super(keyspaceName, typeName, ifExists);
+            super(queryString, keyspaceName, typeName, ifExists);
         }
 
         UserType apply(KeyspaceMetadata keyspace, UserType userType)
@@ -236,7 +250,7 @@ UserType apply(KeyspaceMetadata keyspace, UserType userType)
         }
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<AlterTypeStatement>
     {
         private enum Kind
         {
@@ -263,16 +277,20 @@ public Raw(UTName name, boolean ifExists)
             this.name = name;
         }
 
-        public AlterTypeStatement prepare(ClientState state)
+        @Override
+        public AlterTypeStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
             String typeName = name.getStringTypeName();
 
             switch (kind)
             {
-                case     ADD_FIELD: return new AddField(keyspaceName, typeName, newFieldName, newFieldType, ifExists, ifFieldNotExists);
-                case RENAME_FIELDS: return new RenameFields(keyspaceName, typeName, renamedFields, ifExists, ifFieldExists);
-                case   ALTER_FIELD: return new AlterField(keyspaceName, typeName, ifExists);
+                case     ADD_FIELD:
+                    if (keyspaceMapper != Constants.IDENTITY_STRING_MAPPER)
+                        newFieldType.forEachUserType(utName -> utName.updateKeyspaceIfDefined(keyspaceMapper));
+                    return new AddField(rawCQLStatement, keyspaceName, typeName, newFieldName, newFieldType, ifExists, ifFieldNotExists);
+                case RENAME_FIELDS: return new RenameFields(rawCQLStatement, keyspaceName, typeName, renamedFields, ifExists, ifFieldExists);
+                case   ALTER_FIELD: return new AlterField(rawCQLStatement, keyspaceName, typeName, ifExists);
             }
 
             throw new AssertionError();
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterViewStatement.java
index 7e707f476bed..157bc1d43fd6 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterViewStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterViewStatement.java
@@ -17,14 +17,20 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.function.UnaryOperator;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
@@ -37,9 +43,10 @@ public final class AlterViewStatement extends AlterSchemaStatement
     private ClientState state;
     private final boolean ifExists;
 
-    public AlterViewStatement(String keyspaceName, String viewName, TableAttributes attrs, boolean ifExists)
+    public AlterViewStatement(String queryString, String keyspaceName, String viewName,
+            TableAttributes attrs, boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.viewName = viewName;
         this.attrs = attrs;
         this.ifExists = ifExists;
@@ -85,7 +92,7 @@ public Keyspaces apply(Keyspaces schema)
         if (params.defaultTimeToLive > 0)
         {
             throw ire("Forbidden default_time_to_live detected for a materialized view. " +
-                      "Data in a materialized view always expire at the same time than " +
+                      "Data in a materialized view always expires at the same time as " +
                       "the corresponding data in the parent table. default_time_to_live " +
                       "must be set to zero, see CASSANDRA-12868 for more information");
         }
@@ -117,7 +124,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, viewName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<AlterViewStatement>
     {
         private final QualifiedName name;
         private final TableAttributes attrs;
@@ -130,10 +137,11 @@ public Raw(QualifiedName name, TableAttributes attrs, boolean ifExists)
             this.ifExists = ifExists;
         }
 
-        public AlterViewStatement prepare(ClientState state)
+        @Override
+        public AlterViewStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
-            return new AlterViewStatement(keyspaceName, name.getName(), attrs, ifExists);
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
+            return new AlterViewStatement(rawCQLStatement, keyspaceName, name.getName(), attrs, ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java
index eb9f33a94959..9a63746c202d 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java
@@ -20,6 +20,7 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.Set;
+import java.util.function.UnaryOperator;
 
 import com.google.common.base.Objects;
 import com.google.common.collect.ImmutableSet;
@@ -30,33 +31,36 @@
 import org.apache.cassandra.auth.FunctionResource;
 import org.apache.cassandra.auth.IResource;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.Terms;
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.cql3.functions.ScalarFunction;
 import org.apache.cassandra.cql3.functions.UDAggregate;
 import org.apache.cassandra.cql3.functions.UDFunction;
 import org.apache.cassandra.cql3.functions.UserFunction;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
 
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.transform;
 import static java.lang.String.format;
 import static java.lang.String.join;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
 
-import static com.google.common.collect.Iterables.concat;
-import static com.google.common.collect.Iterables.transform;
-
 public final class CreateAggregateStatement extends AlterSchemaStatement
 {
     private final String aggregateName;
@@ -67,8 +71,10 @@ public final class CreateAggregateStatement extends AlterSchemaStatement
     private final Term.Raw rawInitialValue;
     private final boolean orReplace;
     private final boolean ifNotExists;
+    private final boolean deterministic;
 
-    public CreateAggregateStatement(String keyspaceName,
+    public CreateAggregateStatement(String queryString,
+                                    String keyspaceName,
                                     String aggregateName,
                                     List<CQL3Type.Raw> rawArgumentTypes,
                                     CQL3Type.Raw rawStateType,
@@ -76,9 +82,10 @@ public CreateAggregateStatement(String keyspaceName,
                                     FunctionName finalFunctionName,
                                     Term.Raw rawInitialValue,
                                     boolean orReplace,
-                                    boolean ifNotExists)
+                                    boolean ifNotExists,
+                                    boolean deterministic)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.aggregateName = aggregateName;
         this.rawArgumentTypes = rawArgumentTypes;
         this.rawStateType = rawStateType;
@@ -87,6 +94,7 @@ public CreateAggregateStatement(String keyspaceName,
         this.rawInitialValue = rawInitialValue;
         this.orReplace = orReplace;
         this.ifNotExists = ifNotExists;
+        this.deterministic = deterministic;
     }
 
     public Keyspaces apply(Keyspaces schema)
@@ -201,7 +209,8 @@ public Keyspaces apply(Keyspaces schema)
                             returnType,
                             (ScalarFunction) stateFunction,
                             (ScalarFunction) finalFunction,
-                            initialValue);
+                            initialValue,
+                            deterministic);
 
         UserFunction existingAggregate = keyspace.userFunctions.find(aggregate.name(), argumentTypes).orElse(null);
         if (null != existingAggregate)
@@ -215,7 +224,7 @@ public Keyspaces apply(Keyspaces schema)
             if (!orReplace)
                 throw ire("Aggregate '%s' already exists", aggregateName);
 
-            if (!returnType.isCompatibleWith(existingAggregate.returnType()))
+            if (!returnType.isCompatibleWith(existingAggregate.returnType())) // shouldn't this condition be opposite direction? existingAggregate.returnType().isCompatibleWith(returnType)?
             {
                 throw ire("Cannot replace aggregate '%s', the new return type %s isn't compatible with the return type %s of existing function",
                           aggregateName,
@@ -299,7 +308,7 @@ private String finalFunctionString()
         return format("%s(%s)", finalFunctionName, rawStateType);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<CreateAggregateStatement>
     {
         private final FunctionName aggregateName;
         private final List<CQL3Type.Raw> rawArgumentTypes;
@@ -309,6 +318,7 @@ public static final class Raw extends CQLStatement.Raw
         private final Term.Raw rawInitialValue;
         private final boolean orReplace;
         private final boolean ifNotExists;
+        private final boolean deterministic;
 
         public Raw(FunctionName aggregateName,
                    List<CQL3Type.Raw> rawArgumentTypes,
@@ -317,7 +327,8 @@ public Raw(FunctionName aggregateName,
                    String finalFunctionName,
                    Term.Raw rawInitialValue,
                    boolean orReplace,
-                   boolean ifNotExists)
+                   boolean ifNotExists,
+                   boolean deterministic)
         {
             this.aggregateName = aggregateName;
             this.rawArgumentTypes = rawArgumentTypes;
@@ -327,13 +338,21 @@ public Raw(FunctionName aggregateName,
             this.rawInitialValue = rawInitialValue;
             this.orReplace = orReplace;
             this.ifNotExists = ifNotExists;
+            this.deterministic = deterministic;
         }
 
-        public CreateAggregateStatement prepare(ClientState state)
+        @Override
+        public CreateAggregateStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = aggregateName.hasKeyspace() ? aggregateName.keyspace : state.getKeyspace();
+            String keyspaceName = keyspaceMapper.apply(aggregateName.hasKeyspace() ? aggregateName.keyspace : state.getKeyspace());
+            if (keyspaceMapper != Constants.IDENTITY_STRING_MAPPER)
+            {
+                rawArgumentTypes.forEach(t -> t.forEachUserType(name -> name.updateKeyspaceIfDefined(keyspaceMapper)));
+                rawStateType.forEachUserType(name -> name.updateKeyspaceIfDefined(keyspaceMapper));
+            }
 
-            return new CreateAggregateStatement(keyspaceName,
+            return new CreateAggregateStatement(rawCQLStatement,
+                                                keyspaceName,
                                                 aggregateName.name,
                                                 rawArgumentTypes,
                                                 rawStateType,
@@ -341,7 +360,8 @@ public CreateAggregateStatement prepare(ClientState state)
                                                 null != finalFunctionName ? new FunctionName(keyspaceName, finalFunctionName) : null,
                                                 rawInitialValue,
                                                 orReplace,
-                                                ifNotExists);
+                                                ifNotExists,
+                                                deterministic);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java
index f04ae37cd52b..c729c1f9c5ab 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java
@@ -20,25 +20,29 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.function.UnaryOperator;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
-import org.apache.cassandra.auth.*;
+import org.apache.cassandra.auth.FunctionResource;
+import org.apache.cassandra.auth.IResource;
+import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.cql3.functions.UDFunction;
 import org.apache.cassandra.cql3.functions.UserFunction;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
@@ -57,8 +61,12 @@ public final class CreateFunctionStatement extends AlterSchemaStatement
     private final String body;
     private final boolean orReplace;
     private final boolean ifNotExists;
+    private final boolean deterministic;
+    private final boolean monotonic;
+    private final List<ColumnIdentifier> monotonicOn;
 
-    public CreateFunctionStatement(String keyspaceName,
+    public CreateFunctionStatement(String queryString,
+                                   String keyspaceName,
                                    String functionName,
                                    List<ColumnIdentifier> argumentNames,
                                    List<CQL3Type.Raw> rawArgumentTypes,
@@ -67,9 +75,12 @@ public CreateFunctionStatement(String keyspaceName,
                                    String language,
                                    String body,
                                    boolean orReplace,
-                                   boolean ifNotExists)
+                                   boolean ifNotExists,
+                                   boolean deterministic,
+                                   boolean monotonic,
+                                   List<ColumnIdentifier> monotonicOn)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.functionName = functionName;
         this.argumentNames = argumentNames;
         this.rawArgumentTypes = rawArgumentTypes;
@@ -79,6 +90,10 @@ public CreateFunctionStatement(String keyspaceName,
         this.body = body;
         this.orReplace = orReplace;
         this.ifNotExists = ifNotExists;
+        this.deterministic = deterministic;
+        this.monotonic = monotonic;
+        this.monotonicOn = monotonicOn;
+
     }
 
     // TODO: replace affected aggregates !!
@@ -92,9 +107,14 @@ public Keyspaces apply(Keyspaces schema)
         if (!FunctionName.isNameValid(functionName))
             throw ire("Function name '%s' is invalid", functionName);
 
-        if (new HashSet<>(argumentNames).size() != argumentNames.size())
+        HashSet<ColumnIdentifier> argumentNamesSet = new HashSet<>(argumentNames);
+
+        if (argumentNamesSet.size() != argumentNames.size())
             throw ire("Duplicate argument names for given function %s with argument names %s", functionName, argumentNames);
 
+        if (!argumentNamesSet.containsAll(monotonicOn))
+            throw ire("Monotony should be declared on one of the arguments, '%s' is not an argument", monotonicOn.get(0));
+
         rawArgumentTypes.stream()
                         .filter(raw -> !raw.isImplicitlyFrozen() && raw.isFrozen())
                         .findFirst()
@@ -120,7 +140,10 @@ public Keyspaces apply(Keyspaces schema)
                               returnType,
                               calledOnNullInput,
                               language,
-                              body);
+                              body,
+                              deterministic,
+                              monotonic,
+                              monotonicOn);
 
         UserFunction existingFunction = keyspace.userFunctions.find(function.name(), argumentTypes).orElse(null);
         if (null != existingFunction)
@@ -204,7 +227,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, functionName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<CreateFunctionStatement>
     {
         private final FunctionName name;
         private final List<ColumnIdentifier> argumentNames;
@@ -215,6 +238,9 @@ public static final class Raw extends CQLStatement.Raw
         private final String body;
         private final boolean orReplace;
         private final boolean ifNotExists;
+        private final boolean deterministic;
+        private final boolean monotonic;
+        private final List<ColumnIdentifier> monotonicOn;
 
         public Raw(FunctionName name,
                    List<ColumnIdentifier> argumentNames,
@@ -224,7 +250,10 @@ public Raw(FunctionName name,
                    String language,
                    String body,
                    boolean orReplace,
-                   boolean ifNotExists)
+                   boolean ifNotExists,
+                   boolean deterministic,
+                   boolean monotonic,
+                   List<ColumnIdentifier> monotonicOn)
         {
             this.name = name;
             this.argumentNames = argumentNames;
@@ -235,13 +264,24 @@ public Raw(FunctionName name,
             this.body = body;
             this.orReplace = orReplace;
             this.ifNotExists = ifNotExists;
+            this.deterministic = deterministic;
+            this.monotonic = monotonic;
+            this.monotonicOn = monotonicOn;
         }
 
-        public CreateFunctionStatement prepare(ClientState state)
+        @Override
+        public CreateFunctionStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.keyspace : state.getKeyspace();
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.keyspace : state.getKeyspace());
+
+            if (keyspaceMapper != Constants.IDENTITY_STRING_MAPPER)
+            {
+                rawArgumentTypes.forEach(t -> t.forEachUserType(name -> name.updateKeyspaceIfDefined(keyspaceMapper)));
+                rawReturnType.forEachUserType(name -> name.updateKeyspaceIfDefined(keyspaceMapper));
+            }
 
-            return new CreateFunctionStatement(keyspaceName,
+            return new CreateFunctionStatement(rawCQLStatement,
+                                               keyspaceName,
                                                name.name,
                                                argumentNames,
                                                rawArgumentTypes,
@@ -250,7 +290,10 @@ public CreateFunctionStatement prepare(ClientState state)
                                                language,
                                                body,
                                                orReplace,
-                                               ifNotExists);
+                                               ifNotExists,
+                                               deterministic,
+                                               monotonic,
+                                               monotonicOn);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
index 46a91719342d..a9024b0f686b 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
@@ -17,9 +17,15 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
-import java.util.*;
-
-import com.google.common.base.Strings;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.UnaryOperator;
+import java.util.stream.StreamSupport;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 
@@ -27,19 +33,25 @@
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget.Type;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.guardrails.Threshold;
 import org.apache.cassandra.db.marshal.MapType;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.index.sasi.SASIIndex;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
@@ -88,15 +100,28 @@ public final class CreateIndexStatement extends AlterSchemaStatement
     private final boolean ifNotExists;
 
     private ClientState state;
-
-    public CreateIndexStatement(String keyspaceName,
+    private static final String DSE_INDEX_WARNING = "Index %s was not created. DSE custom index (%s) is not " +
+                                                    "supported. Consult the docs on alternatives (SAI indexes, " +
+                                                    "Secondary Indexes).";
+
+    @VisibleForTesting
+    public static final Set<String> DSE_INDEXES = ImmutableSet.of(
+        "com.datastax.bdp.cassandra.index.solr.SolrSecondaryIndex",
+        "com.datastax.bdp.cassandra.index.solr.ThriftSolrSecondaryIndex",
+        "com.datastax.bdp.cassandra.index.solr.Cql3SolrSecondaryIndex",
+        "com.datastax.bdp.search.solr.ThriftSolrSecondaryIndex",
+        "com.datastax.bdp.search.solr.Cql3SolrSecondaryIndex"
+    );
+
+    public CreateIndexStatement(String queryString,
+                                String keyspaceName,
                                 String tableName,
                                 String indexName,
                                 List<IndexTarget.Raw> rawIndexTargets,
                                 IndexAttributes attrs,
                                 boolean ifNotExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.tableName = tableName;
         this.indexName = indexName;
         this.rawIndexTargets = rawIndexTargets;
@@ -109,12 +134,29 @@ public void validate(ClientState state)
     {
         super.validate(state);
 
+        // Check the length of a valid index name.
+        // Non-valid indexes are validated in IndexMetadata#validate.
+        if (!state.isInternal
+            && SchemaConstants.isValidName(indexName, true)
+            && indexName.length() > SchemaConstants.INDEX_NAME_LENGTH)
+
+            throw ire("Index name shouldn't be more than %s characters long (got %s chars for %s)",
+                      SchemaConstants.INDEX_NAME_LENGTH, indexName.length(), indexName);
+
         // save the query state to use it for guardrails validation in #apply
         this.state = state;
     }
 
     public Keyspaces apply(Keyspaces schema)
     {
+        if (isDseIndexCreateStatement())
+        {
+            // DSE indexes are not supported. The index is not created, the attempt is ignored (doesn't cause error),
+            // a meaningfull warning is returned instead.
+            return schema;
+        }
+
+        attrs.maybeApplyDefaultIndex();
         attrs.validate();
 
         Guardrails.createSecondaryIndexesEnabled.ensureEnabled("Creating secondary indexes", state);
@@ -144,17 +186,9 @@ public Keyspaces apply(Keyspaces schema)
         if (table.isView())
             throw ire(MATERIALIZED_VIEWS_NOT_SUPPORTED);
 
-        if (Keyspace.open(table.keyspace).getReplicationStrategy().hasTransientReplicas())
+        if (keyspace.createReplicationStrategy().hasTransientReplicas())
             throw new InvalidRequestException(TRANSIENTLY_REPLICATED_KEYSPACE_NOT_SUPPORTED);
 
-        // guardrails to limit number of secondary indexes per table.
-        Guardrails.secondaryIndexesPerTable.guard(table.indexes.size() + 1,
-                                                  Strings.isNullOrEmpty(indexName)
-                                                  ? String.format("on table %s", table.name)
-                                                  : String.format("%s on table %s", indexName, table.name),
-                                                  false,
-                                                  state);
-
         List<IndexTarget> indexTargets = Lists.newArrayList(transform(rawIndexTargets, t -> t.prepare(table)));
 
         if (indexTargets.isEmpty() && !attrs.isCustom)
@@ -181,6 +215,27 @@ public Keyspaces apply(Keyspaces schema)
 
         IndexMetadata index = IndexMetadata.fromIndexTargets(indexTargets, name, kind, options);
 
+        String className = index.getIndexClassName();
+        IndexGuardrails guardRails = IndexGuardrails.forClassName(className);
+        String indexDescription = indexName == null ? String.format("on table %s", table.name) : String.format("%s on table %s", indexName, table.name);
+
+        // Guardrail to limit number of secondary indexes (per table)
+        if (guardRails.hasPerTableThreshold())
+        {
+            long indexesOnSameTable = table.indexes.stream().filter(other -> className.equals(other.getIndexClassName())).count();
+            guardRails.perTableThreshold.guard(indexesOnSameTable + 1, indexDescription,false, state);
+        }
+
+        // Guardrail to limit number of secondary indexes (total)
+        if (guardRails.hasTotalThreshold())
+        {
+            long indexesOnAllTables = StreamSupport.stream(Keyspace.all().spliterator(), false).flatMap(ks -> ks.getColumnFamilyStores().stream())
+                                                   .flatMap(ks -> ks.indexManager.listIndexes().stream())
+                                                   .map(i -> i.getIndexMetadata().getIndexClassName())
+                                                   .filter(className::equals).count();
+            guardRails.totalThreshold.guard(indexesOnAllTables + 1, indexDescription, false, state);
+        }
+
         // check to disallow creation of an index which duplicates an existing one in all but name
         IndexMetadata equalIndex = tryFind(table.indexes, i -> i.equalsWithoutName(index)).orNull();
         if (null != equalIndex)
@@ -203,9 +258,17 @@ Set<String> clientWarnings(KeyspacesDiff diff)
         if (attrs.isCustom && attrs.customClass.equals(SASIIndex.class.getName()))
             return ImmutableSet.of(SASIIndex.USAGE_WARNING);
 
+        if (isDseIndexCreateStatement())
+            return ImmutableSet.of(String.format(DSE_INDEX_WARNING, indexName, attrs.customClass));
+
         return ImmutableSet.of();
     }
 
+    private boolean isDseIndexCreateStatement()
+    {
+        return DSE_INDEXES.contains(attrs.customClass);
+    }
+
     private void validateIndexTarget(TableMetadata table, IndexMetadata.Kind kind, IndexTarget target)
     {
         ColumnMetadata column = table.getColumn(target.column);
@@ -215,9 +278,6 @@ private void validateIndexTarget(TableMetadata table, IndexMetadata.Kind kind, I
 
         AbstractType<?> baseType = column.type.unwrap();
 
-        if ((kind == IndexMetadata.Kind.CUSTOM) && !SchemaConstants.isValidName(target.column.toString()))
-            throw ire(INVALID_CUSTOM_INDEX_TARGET, target.column, SchemaConstants.NAME_LENGTH);
-
         if (column.type.referencesDuration())
         {
             if (column.type.isCollection())
@@ -244,10 +304,10 @@ private void validateIndexTarget(TableMetadata table, IndexMetadata.Kind kind, I
         if (column.isPartitionKey() && table.partitionKeyColumns().size() == 1)
             throw ire(ONLY_PARTITION_KEY, column);
 
-        if (baseType.isFrozenCollection() && target.type != Type.FULL)
+        if (baseType.isCollection() && !baseType.isMultiCell() && target.type != Type.FULL)
             throw ire(CREATE_ON_FROZEN_COLUMN, target.type, column, column.name.toCQLString());
 
-        if (!baseType.isFrozenCollection() && target.type == Type.FULL)
+        if ((!baseType.isCollection() || baseType.isMultiCell) && target.type == Type.FULL)
             throw ire(FULL_ON_FROZEN_COLLECTIONS);
 
         if (!baseType.isCollection() && target.type != Type.SIMPLE)
@@ -264,7 +324,7 @@ private String generateIndexName(KeyspaceMetadata keyspace, List<IndexTarget> ta
     {
         String baseName = targets.size() == 1
                         ? IndexMetadata.generateDefaultIndexName(tableName, targets.get(0).column)
-                        : IndexMetadata.generateDefaultIndexName(tableName);
+                        : IndexMetadata.generateDefaultIndexName(tableName, null);
         return keyspace.findAvailableIndexName(baseName);
     }
 
@@ -289,7 +349,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, indexName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<CreateIndexStatement>
     {
         private final QualifiedName tableName;
         private final QualifiedName indexName;
@@ -310,7 +370,8 @@ public Raw(QualifiedName tableName,
             this.ifNotExists = ifNotExists;
         }
 
-        public CreateIndexStatement prepare(ClientState state)
+        @Override
+        public CreateIndexStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
             String keyspaceName = tableName.hasKeyspace()
                                 ? tableName.getKeyspace()
@@ -321,7 +382,7 @@ public CreateIndexStatement prepare(ClientState state)
 
             if (indexName.hasKeyspace() && !keyspaceName.equals(indexName.getKeyspace()))
                 throw ire(KEYSPACE_DOES_NOT_MATCH_INDEX, keyspaceName, tableName);
-            
+
             // Set the configured default 2i implementation if one isn't specified with USING:
             if (attrs.customClass == null)
             {
@@ -331,19 +392,64 @@ public CreateIndexStatement prepare(ClientState state)
                     // However, operators may require an implementation be specified
                     throw ire(MUST_SPECIFY_INDEX_IMPLEMENTATION);
             }
-            
+
             // If we explicitly specify the index type "legacy_local_table", we can just clear the custom class, and the
-            // non-custom 2i creation process will begin. Otherwise, if an index type has been specified with 
+            // non-custom 2i creation process will begin. Otherwise, if an index type has been specified with
             // USING, make sure the appropriate custom index is created.
             if (attrs.customClass != null)
             {
-                if (!attrs.isCustom && attrs.customClass.equalsIgnoreCase(CassandraIndex.NAME))
+                boolean isLegacyLocalTable = attrs.customClass.equalsIgnoreCase(CassandraIndex.NAME);
+                if (isLegacyLocalTable)
                     attrs.customClass = null;
                 else
                     attrs.isCustom = true;
             }
 
-            return new CreateIndexStatement(keyspaceName, tableName.getName(), indexName.getName(), rawIndexTargets, attrs, ifNotExists);
+            return new CreateIndexStatement(rawCQLStatement, keyspaceMapper.apply(keyspaceName), tableName.getName(),
+                                            indexName.getName(), rawIndexTargets, attrs, ifNotExists);
+        }
+    }
+
+    enum IndexGuardrails
+    {
+        LEGACY(Guardrails.secondaryIndexesPerTable, null),
+        SAI(Guardrails.saiIndexesPerTable, Guardrails.saiIndexesTotal),
+        SASI(Guardrails.sasiIndexesPerTable, null),
+        UNKNOWN(null, null);
+
+        final Threshold perTableThreshold;
+        final Threshold totalThreshold;
+
+        IndexGuardrails(Threshold perTableThreshold, Threshold totalThreshold)
+        {
+            this.perTableThreshold = perTableThreshold;
+            this.totalThreshold = totalThreshold;
+        }
+
+        boolean hasPerTableThreshold()
+        {
+            return perTableThreshold != null;
+        }
+
+        boolean hasTotalThreshold()
+        {
+            return totalThreshold != null;
+        }
+
+        static IndexGuardrails forClassName(String className)
+        {
+            switch (className)
+            {
+                case "org.apache.cassandra.index.internal.CassandraIndex":
+                    return IndexGuardrails.LEGACY;
+                case "org.apache.cassandra.index.sasi.SASIIndex":
+                    return IndexGuardrails.SASI;
+                case "org.apache.cassandra.index.sai.StorageAttachedIndex":
+                    return IndexGuardrails.SAI;
+                default:
+                    return IndexGuardrails.UNKNOWN;
+            }
         }
+
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
index 13d52b1e156c..9d7ddaaade0b 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
@@ -18,10 +18,11 @@
 package org.apache.cassandra.cql3.statements.schema;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.function.UnaryOperator;
 
 import com.google.common.collect.ImmutableSet;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,7 +32,7 @@
 import org.apache.cassandra.auth.FunctionResource;
 import org.apache.cassandra.auth.IResource;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.exceptions.AlreadyExistsException;
 import org.apache.cassandra.locator.LocalStrategy;
@@ -45,21 +46,38 @@
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 
-public final class CreateKeyspaceStatement extends AlterSchemaStatement
+public final class CreateKeyspaceStatement extends AlterSchemaStatement implements AlterSchemaStatement.WithKeyspaceAttributes
 {
     private static final Logger logger = LoggerFactory.getLogger(CreateKeyspaceStatement.class);
 
     private final KeyspaceAttributes attrs;
     private final boolean ifNotExists;
-    private final HashSet<String> clientWarnings = new HashSet<>();
 
-    public CreateKeyspaceStatement(String keyspaceName, KeyspaceAttributes attrs, boolean ifNotExists)
+    public CreateKeyspaceStatement(String queryString, String keyspaceName,
+                                   KeyspaceAttributes attrs, boolean ifNotExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.attrs = attrs;
         this.ifNotExists = ifNotExists;
     }
 
+    public Object getAttribute(String key)
+    {
+        return attrs.getProperty(key);
+    }
+
+    public void overrideAttribute(String oldKey, String newKey, String newValue)
+    {
+        attrs.removeProperty(oldKey);
+        attrs.addProperty(newKey, newValue);
+    }
+
+    public void overrideAttribute(String oldKey, String newKey, Map<String, String> newValue)
+    {
+        attrs.removeProperty(oldKey);
+        attrs.addProperty(newKey, newValue);
+    }
+
     public Keyspaces apply(Keyspaces schema)
     {
         attrs.validate();
@@ -79,11 +97,11 @@ public Keyspaces apply(Keyspaces schema)
         }
 
         KeyspaceMetadata keyspace = KeyspaceMetadata.create(keyspaceName, attrs.asNewKeyspaceParams());
+        keyspace.validate(state);
 
         if (keyspace.params.replication.klass.equals(LocalStrategy.class))
             throw ire("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
 
-        keyspace.params.validate(keyspaceName, state);
         return schema.withAddedOrUpdated(keyspace);
     }
 
@@ -123,7 +141,18 @@ public void validate(ClientState state)
         Guardrails.keyspaces.guard(Schema.instance.getUserKeyspaces().size() + 1, keyspaceName, false, state);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    Set<String> clientWarnings(KeyspacesDiff diff)
+    {
+        HashSet<String> clientWarnings = new HashSet<>();
+        if (attrs.hasProperty("graph_engine"))
+        {
+            clientWarnings.add("The unsupported graph property 'graph_engine' was ignored.");
+        }
+
+        return clientWarnings;
+    }
+
+    public static final class Raw extends RawKeyspaceAwareStatement<CreateKeyspaceStatement>
     {
         public final String keyspaceName;
         private final KeyspaceAttributes attrs;
@@ -136,9 +165,10 @@ public Raw(String keyspaceName, KeyspaceAttributes attrs, boolean ifNotExists)
             this.ifNotExists = ifNotExists;
         }
 
-        public CreateKeyspaceStatement prepare(ClientState state)
+        @Override
+        public CreateKeyspaceStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            return new CreateKeyspaceStatement(keyspaceName, attrs, ifNotExists);
+            return new CreateKeyspaceStatement(rawCQLStatement, keyspaceMapper.apply(keyspaceName), attrs, ifNotExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
index 747837f30556..5f6c8cec54d5 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
@@ -17,15 +17,23 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
-
 import javax.annotation.Nullable;
 
 import com.google.common.collect.ImmutableSet;
-
 import org.apache.commons.lang3.StringUtils;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,23 +42,44 @@
 import org.apache.cassandra.auth.DataResource;
 import org.apache.cassandra.auth.IResource;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLFragmentParser;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.CqlParser;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.UTName;
 import org.apache.cassandra.cql3.functions.masking.ColumnMask;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.guardrails.UserKeyspaceFilter;
+import org.apache.cassandra.db.guardrails.UserKeyspaceFilterProvider;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.EmptyType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.AlreadyExistsException;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.DroppedColumn;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.schema.Types;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
 
-import static java.util.Comparator.comparing;
-
 import static com.google.common.collect.Iterables.concat;
+import static java.util.Comparator.comparing;
 
 public final class CreateTableStatement extends AlterSchemaStatement
 {
@@ -68,7 +97,8 @@ public final class CreateTableStatement extends AlterSchemaStatement
     private final boolean ifNotExists;
     private final boolean useCompactStorage;
 
-    public CreateTableStatement(String keyspaceName,
+    public CreateTableStatement(String queryString,
+                                String keyspaceName,
                                 String tableName,
                                 Map<ColumnIdentifier, ColumnProperties.Raw> rawColumns,
                                 Set<ColumnIdentifier> staticColumns,
@@ -79,7 +109,7 @@ public CreateTableStatement(String keyspaceName,
                                 boolean ifNotExists,
                                 boolean useCompactStorage)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.tableName = tableName;
 
         this.rawColumns = rawColumns;
@@ -94,6 +124,11 @@ public CreateTableStatement(String keyspaceName,
         this.useCompactStorage = useCompactStorage;
     }
 
+    public boolean isCompactStorage()
+    {
+        return useCompactStorage;
+    }
+
     public Keyspaces apply(Keyspaces schema)
     {
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
@@ -128,18 +163,28 @@ public void validate(ClientState state)
     {
         super.validate(state);
 
+        if (!state.isInternal && tableName.length() > SchemaConstants.NAME_LENGTH - keyspaceName.length())
+            throw ire("Table name is too long, it needs to fit %s characters (got table name of %s chars for %s.%s)",
+                      SchemaConstants.NAME_LENGTH - keyspaceName.length(), tableName.length(), keyspaceName, tableName);
+
         // Guardrail on table properties
         Guardrails.tableProperties.guard(attrs.updatedProperties(), attrs::removeProperty, state);
 
+        // Guardrail on counter
+        if (rawColumns.values().stream().anyMatch(t -> t.rawType.isCounter()))
+            Guardrails.counterEnabled.ensureEnabled(state);
+
         // Guardrail on columns per table
         Guardrails.columnsPerTable.guard(rawColumns.size(), tableName, false, state);
 
         // Guardrail on number of tables
         if (Guardrails.tables.enabled(state))
         {
+            UserKeyspaceFilter userKeyspaceFilter = UserKeyspaceFilterProvider.instance.get(state);
             int totalUserTables = Schema.instance.getUserKeyspaces()
                                                  .stream()
                                                  .map(Keyspace::open)
+                                                 .filter(userKeyspaceFilter::filter)
                                                  .mapToInt(keyspace -> keyspace.getColumnFamilyStores().size())
                                                  .sum();
             Guardrails.tables.guard(totalUserTables + 1, tableName, false, state);
@@ -190,20 +235,6 @@ public TableMetadata.Builder builder(Types types)
         Map<ColumnIdentifier, ColumnProperties> columns = new TreeMap<>(comparing(o -> o.bytes));
         rawColumns.forEach((column, properties) -> columns.put(column, properties.prepare(keyspaceName, tableName, column, types)));
 
-        // check for nested non-frozen UDTs or collections in a non-frozen UDT
-        columns.forEach((column, properties) ->
-        {
-            AbstractType<?> type = properties.type;
-            if (type.isUDT() && type.isMultiCell())
-            {
-                ((UserType) type).fieldTypes().forEach(field ->
-                {
-                    if (field.isMultiCell())
-                        throw ire("Non-frozen UDTs with nested non-frozen collections are not supported");
-                });
-            }
-        });
-
         /*
          * Deal with PRIMARY KEY columns
          */
@@ -218,22 +249,6 @@ public TableMetadata.Builder builder(Types types)
             if (!primaryKeyColumns.add(column))
                 throw ire("Duplicate column '%s' in PRIMARY KEY clause for table '%s'", column, tableName);
 
-            AbstractType<?> type = properties.type;
-            if (type.isMultiCell())
-            {
-                CQL3Type cqlType = properties.cqlType;
-                if (type.isCollection())
-                    throw ire("Invalid non-frozen collection type %s for PRIMARY KEY column '%s'", cqlType, column);
-                else
-                    throw ire("Invalid non-frozen user-defined type %s for PRIMARY KEY column '%s'", cqlType, column);
-            }
-
-            if (type.isCounter())
-                throw ire("counter type is not supported for PRIMARY KEY column '%s'", column);
-
-            if (type.referencesDuration())
-                throw ire("duration type is not supported for PRIMARY KEY column '%s'", column);
-
             if (staticColumns.contains(column))
                 throw ire("Static column '%s' cannot be part of the PRIMARY KEY", column);
         });
@@ -295,11 +310,6 @@ public TableMetadata.Builder builder(Types types)
         boolean hasCounters = rawColumns.values().stream().anyMatch(c -> c.rawType.isCounter());
         if (hasCounters)
         {
-            // We've handled anything that is not a PRIMARY KEY so columns only contains NON-PK columns. So
-            // if it's a counter table, make sure we don't have non-counter types
-            if (columns.values().stream().anyMatch(t -> !t.type.isCounter()))
-                throw ire("Cannot mix counter and non counter columns in the same table");
-
             if (params.defaultTimeToLive > 0)
                 throw ire("Cannot set %s on a table with counters", TableParams.Option.DEFAULT_TIME_TO_LIVE);
         }
@@ -341,6 +351,8 @@ public TableMetadata.Builder builder(Types types)
                     builder.addRegularColumn(column, properties.type, properties.mask);
             });
         }
+        for (DroppedColumn.Raw record : attrs.droppedColumnRecords())
+            builder.recordColumnDrop(record.prepare(keyspaceName, tableName, types));
         return builder;
     }
 
@@ -419,6 +431,39 @@ else if (!builder.hasRegularColumns())
         }
     }
 
+    @Override
+    public Set<String> clientWarnings(KeyspacesDiff diff)
+    {
+        ImmutableSet.Builder<String> warnings = ImmutableSet.builder();
+
+        if (attrs.hasUnsupportedDseCompaction())
+        {
+            Map<String, String> compactionOptions = attrs.getMap(TableParams.Option.COMPACTION.toString());
+            String strategy = compactionOptions.get(CompactionParams.Option.CLASS.toString());
+            warnings.add(String.format("The given compaction strategy (%s) is not supported. ", strategy) +
+                         "The compaction strategy parameter was overridden with the default " +
+                         String.format("(%s). ", CompactionParams.DEFAULT.klass().getCanonicalName()) +
+                         "Inspect your schema and adjust other table properties if needed.");
+        }
+
+        if (attrs.hasProperty("nodesync"))
+        {
+            warnings.add("The unsupported 'nodesync' table option was ignored.");
+        }
+
+        if (attrs.hasProperty("dse_vertex_label_property"))
+        {
+            warnings.add("The unsupported graph table property was ignored (VERTEX LABEL).");
+        }
+
+        if (attrs.hasProperty("dse_edge_label_property"))
+        {
+            warnings.add("The unsupported graph table property was ignored (EDGE LABEL).");
+        }
+
+        return warnings.build();
+    }
+
     private static class DefaultNames
     {
         private static final String DEFAULT_CLUSTERING_NAME = "column";
@@ -457,14 +502,19 @@ public String defaultCompactValueName()
     }
 
     public static TableMetadata.Builder parse(String cql, String keyspace)
+    {
+        return parse(cql, keyspace, Types.none());
+    }
+
+    public static TableMetadata.Builder parse(String cql, String keyspace, Types types)
     {
         return CQLFragmentParser.parseAny(CqlParser::createTableStatement, cql, "CREATE TABLE")
-                                .keyspace(keyspace)
-                                .prepare(null) // works around a messy ClientState/QueryProcessor class init deadlock
-                                .builder(Types.none());
+                         .keyspace(keyspace)
+                         .prepare(null) // works around a messy ClientState/QueryProcessor class init deadlock
+                         .builder(types);
     }
 
-    public final static class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<CreateTableStatement>
     {
         private final QualifiedName name;
         private final boolean ifNotExists;
@@ -485,14 +535,19 @@ public Raw(QualifiedName name, boolean ifNotExists)
             this.ifNotExists = ifNotExists;
         }
 
-        public CreateTableStatement prepare(ClientState state)
+        @Override
+        public CreateTableStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
+
+            if (keyspaceMapper != Constants.IDENTITY_STRING_MAPPER)
+                rawColumns.values().forEach(t -> t.forEachUserType(utName -> utName.updateKeyspaceIfDefined(keyspaceMapper)));
 
             if (null == partitionKeyColumns)
                 throw ire("No PRIMARY KEY specifed for table '%s' (exactly one required)", name);
 
-            return new CreateTableStatement(keyspaceName,
+            return new CreateTableStatement(rawCQLStatement,
+                                            keyspaceName,
                                             name.getName(),
                                             rawColumns,
                                             staticColumns,
@@ -613,6 +668,11 @@ public ColumnProperties prepare(String keyspace, String table, ColumnIdentifier
                 ColumnMask mask = rawMask == null ? null : rawMask.prepare(keyspace, table, column, type);
                 return new ColumnProperties(type, cqlType, mask);
             }
+
+            public void forEachUserType(Consumer<UTName> keyspaceMapper)
+            {
+                rawType.forEachUserType(keyspaceMapper);
+            }
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTriggerStatement.java
index e85ffd80aecd..39d7242223e1 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTriggerStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTriggerStatement.java
@@ -17,17 +17,22 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.function.UnaryOperator;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.QualifiedName;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TriggerMetadata;
 import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.triggers.TriggerExecutor;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
+import org.apache.cassandra.triggers.TriggerExecutor;
 
 public final class CreateTriggerStatement extends AlterSchemaStatement
 {
@@ -36,9 +41,10 @@ public final class CreateTriggerStatement extends AlterSchemaStatement
     private final String triggerClass;
     private final boolean ifNotExists;
 
-    public CreateTriggerStatement(String keyspaceName, String tableName, String triggerName, String triggerClass, boolean ifNotExists)
+    public CreateTriggerStatement(String queryString, String keyspaceName, String tableName,
+                                  String triggerName, String triggerClass, boolean ifNotExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.tableName = tableName;
         this.triggerName = triggerName;
         this.triggerClass = triggerClass;
@@ -101,7 +107,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, triggerName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<CreateTriggerStatement>
     {
         private final QualifiedName tableName;
         private final String triggerName;
@@ -116,10 +122,12 @@ public Raw(QualifiedName tableName, String triggerName, String triggerClass, boo
             this.ifNotExists = ifNotExists;
         }
 
-        public CreateTriggerStatement prepare(ClientState state)
+        @Override
+        public CreateTriggerStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = tableName.hasKeyspace() ? tableName.getKeyspace() : state.getKeyspace();
-            return new CreateTriggerStatement(keyspaceName, tableName.getName(), triggerName, triggerClass, ifNotExists);
+            String keyspaceName = keyspaceMapper.apply(tableName.hasKeyspace() ? tableName.getKeyspace() : state.getKeyspace());
+            return new CreateTriggerStatement(rawCQLStatement, keyspaceName, tableName.getName(),
+                                              triggerName, triggerClass, ifNotExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java
index d76c8089f60a..9109f594ee3d 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java
@@ -17,15 +17,22 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.UnaryOperator;
 
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.CQLFragmentParser;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.CqlParser;
 import org.apache.cassandra.cql3.FieldIdentifier;
 import org.apache.cassandra.cql3.UTName;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UserType;
@@ -38,9 +45,8 @@
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
 
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-
 import static java.util.stream.Collectors.toList;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
 
 public final class CreateTypeStatement extends AlterSchemaStatement
 {
@@ -49,13 +55,14 @@ public final class CreateTypeStatement extends AlterSchemaStatement
     private final List<CQL3Type.Raw> rawFieldTypes;
     private final boolean ifNotExists;
 
-    public CreateTypeStatement(String keyspaceName,
+    public CreateTypeStatement(String queryString,
+                               String keyspaceName,
                                String typeName,
                                List<FieldIdentifier> fieldNames,
                                List<CQL3Type.Raw> rawFieldTypes,
                                boolean ifNotExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.typeName = typeName;
         this.fieldNames = fieldNames;
         this.rawFieldTypes = rawFieldTypes;
@@ -95,21 +102,14 @@ public Keyspaces apply(Keyspaces schema)
             if (!usedNames.add(name))
                 throw ire("Duplicate field name '%s' in type '%s'", name, typeName);
 
-        for (CQL3Type.Raw type : rawFieldTypes)
-        {
-            if (type.isCounter())
-                throw ire("A user type cannot contain counters");
-
-            if (type.isUDT() && !type.isFrozen())
-                throw ire("A user type cannot contain non-frozen UDTs");
-        }
-
         List<AbstractType<?>> fieldTypes =
             rawFieldTypes.stream()
                          .map(t -> t.prepare(keyspaceName, keyspace.types).getType())
                          .collect(toList());
 
         UserType udt = new UserType(keyspaceName, bytes(typeName), fieldNames, fieldTypes, true);
+        validate(udt);
+
         return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.types.with(udt)));
     }
 
@@ -134,7 +134,64 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, typeName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static UserType parse(String cql, String keyspace)
+    {
+        return parse(cql, keyspace, Types.none());
+    }
+
+    public static UserType parse(String cql, String keyspace, Types userTypes)
+    {
+        return CQLFragmentParser.parseAny(CqlParser::createTypeStatement, cql, "CREATE TYPE")
+                                .keyspace(keyspace)
+                                .prepare(null) // works around a messy ClientState/QueryProcessor class init deadlock
+                                .createType(userTypes);
+    }
+
+    /**
+     * Build the {@link UserType} this statement creates.
+     *
+     * @param existingTypes the user-types existing in the keyspace in which the type is created (and thus on which
+     *                      the created type may depend on).
+     * @return the created type.
+     */
+    private UserType createType(Types existingTypes)
+    {
+        List<AbstractType<?>> fieldTypes = rawFieldTypes.stream()
+                                                        .map(t -> t.prepare(keyspaceName, existingTypes).getType())
+                                                        .collect(toList());
+        UserType type = new UserType(keyspaceName, bytes(typeName), fieldNames, fieldTypes, true);
+        validate(type);
+        return type;
+    }
+
+    /**
+     * Ensures that the created User-Defined Type (UDT) is valid and allowed.
+     * <p/>
+     * Note: Most type validation is performed through {@link AbstractType#validateForColumn} because almost no type
+     * is intrinsically invalid unless used as a column type. For instance, while we don't declare a column with a
+     * {@code set<counter>} type, there is no reason to forbid a UDF in a SELECT clause that takes two separate
+     * counter values and puts them in a set. Thus, {@code set<counter>} is not intrinsically invalid, and this applies
+     * to almost all validation in {@link AbstractType#validateForColumn}.
+     * <p/>
+     * However, since UDTs are created separately from their use, it makes sense for user-friendliness to be a bit
+     * more restrictive: if a UDT cannot ever be used as a column type, it is almost certainly a user error. Waiting
+     * until the type is used to throw an error might be annoying. Therefore, we do not allow creating types that
+     * cannot ever be used as column types, even if this is an arbitrary limitation in some ways (e.g., a user may
+     * "legitimately" want to create a type solely for use as the return type of a UDF, similar to the {@code set<counter>}
+     * example above, but we disallow that).
+     *
+     * @param type the User-Defined Type to validate
+     * @throws IllegalArgumentException if the UDT contains counters, as counters are always disallowed in UDTs
+     */
+    static void validate(UserType type)
+    {
+        // The only thing that is always disallowed is the use of counters within a UDT. Anything else might be acceptable,
+        // though possibly only if the type is used frozen.
+        if (type.referencesCounter())
+            throw ire("A user type cannot contain counters");
+    }
+
+    public static final class Raw extends RawKeyspaceAwareStatement<CreateTypeStatement>
     {
         private final UTName name;
         private final boolean ifNotExists;
@@ -148,10 +205,21 @@ public Raw(UTName name, boolean ifNotExists)
             this.ifNotExists = ifNotExists;
         }
 
-        public CreateTypeStatement prepare(ClientState state)
+        public Raw keyspace(String keyspace)
+        {
+            name.setKeyspace(keyspace);
+            return this;
+        }
+
+        @Override
+        public CreateTypeStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
-            return new CreateTypeStatement(keyspaceName, name.getStringTypeName(), fieldNames, rawFieldTypes, ifNotExists);
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
+            if (keyspaceMapper != Constants.IDENTITY_STRING_MAPPER)
+                rawFieldTypes.forEach(t -> t.forEachUserType(utName -> utName.updateKeyspaceIfDefined(keyspaceMapper)));
+            return new CreateTypeStatement(rawCQLStatement, keyspaceName,
+                                           name.getStringTypeName(), fieldNames,
+                                           rawFieldTypes, ifNotExists);
         }
 
         public void addField(FieldIdentifier name, CQL3Type.Raw type)
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java
index 05629e00fdeb..e9e96598bf49 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java
@@ -17,7 +17,13 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.function.UnaryOperator;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
@@ -27,10 +33,14 @@
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.WhereClause;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.cql3.selection.RawSelector;
 import org.apache.cassandra.cql3.selection.Selectable;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.cql3.statements.StatementType;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -38,18 +48,22 @@
 import org.apache.cassandra.db.view.View;
 import org.apache.cassandra.exceptions.AlreadyExistsException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
 
-import static java.lang.String.join;
-
 import static com.google.common.collect.Iterables.concat;
 import static com.google.common.collect.Iterables.filter;
 import static com.google.common.collect.Iterables.transform;
+import static java.lang.String.join;
 import static org.apache.cassandra.config.CassandraRelevantProperties.MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE;
 
 public final class CreateViewStatement extends AlterSchemaStatement
@@ -70,7 +84,8 @@ public final class CreateViewStatement extends AlterSchemaStatement
 
     private ClientState state;
 
-    public CreateViewStatement(String keyspaceName,
+    public CreateViewStatement(String queryString,
+                               String keyspaceName,
                                String tableName,
                                String viewName,
 
@@ -85,7 +100,7 @@ public CreateViewStatement(String keyspaceName,
 
                                boolean ifNotExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.tableName = tableName;
         this.viewName = viewName;
 
@@ -273,17 +288,16 @@ public Keyspaces apply(Keyspaces schema)
         if (whereClause.containsCustomExpressions())
             throw ire("WHERE clause for materialized view '%s' cannot contain custom index expressions", viewName);
 
-        StatementRestrictions restrictions =
-            new StatementRestrictions(state,
-                                      StatementType.SELECT,
-                                      table,
-                                      whereClause,
-                                      VariableSpecifications.empty(),
-                                      Collections.emptyList(),
-                                      false,
-                                      false,
-                                      true,
-                                      true);
+        StatementRestrictions restrictions = StatementRestrictions.create(state,
+                                                                          StatementType.SELECT,
+                                                                          table,
+                                                                          whereClause,
+                                                                          VariableSpecifications.empty(),
+                                                                          Collections.emptyList(),
+                                                                          false,
+                                                                          false,
+                                                                          true,
+                                                                          true);
 
         List<ColumnIdentifier> nonRestrictedPrimaryKeyColumns =
             Lists.newArrayList(filter(primaryKeyColumns, name -> !restrictions.isRestricted(table.getColumn(name))));
@@ -365,7 +379,7 @@ private AbstractType<?> getType(ColumnMetadata column)
             boolean reverse = !clusteringOrder.get(column.name);
 
             if (type.isReversed() && !reverse)
-                return ((ReversedType<?>) type).baseType;
+                return type.unwrap();
 
             if (!type.isReversed() && reverse)
                 return ReversedType.getInstance(type);
@@ -390,7 +404,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, viewName);
     }
 
-    public final static class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<CreateViewStatement>
     {
         private final QualifiedName tableName;
         private final QualifiedName viewName;
@@ -414,7 +428,8 @@ public Raw(QualifiedName tableName, QualifiedName viewName, List<RawSelector> ra
             this.ifNotExists = ifNotExists;
         }
 
-        public CreateViewStatement prepare(ClientState state)
+        @Override
+        public CreateViewStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
             String keyspaceName = viewName.hasKeyspace() ? viewName.getKeyspace() : state.getKeyspace();
 
@@ -427,7 +442,8 @@ public CreateViewStatement prepare(ClientState state)
             if (null == partitionKeyColumns)
                 throw ire("No PRIMARY KEY specifed for view '%s' (exactly one required)", viewName);
 
-            return new CreateViewStatement(keyspaceName,
+            return new CreateViewStatement(rawCQLStatement,
+                                           keyspaceMapper.apply(keyspaceName),
                                            tableName.getName(),
                                            viewName.getName(),
 
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java
index d83fbbf97f5b..03f2d2f48c7d 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java
@@ -20,6 +20,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.function.Predicate;
+import java.util.function.UnaryOperator;
 import java.util.stream.Stream;
 
 import org.apache.cassandra.audit.AuditLogContext;
@@ -27,23 +28,27 @@
 import org.apache.cassandra.auth.FunctionResource;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.cql3.functions.UDAggregate;
 import org.apache.cassandra.cql3.functions.UserFunction;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 
+import static com.google.common.collect.Iterables.transform;
 import static java.lang.String.format;
 import static java.lang.String.join;
 import static java.util.stream.Collectors.toList;
 
-import static com.google.common.collect.Iterables.transform;
-
 public final class DropAggregateStatement extends AlterSchemaStatement
 {
     private final String aggregateName;
@@ -51,13 +56,14 @@ public final class DropAggregateStatement extends AlterSchemaStatement
     private final boolean argumentsSpeficied;
     private final boolean ifExists;
 
-    public DropAggregateStatement(String keyspaceName,
+    public DropAggregateStatement(String queryString,
+                                  String keyspaceName,
                                   String aggregateName,
                                   List<CQL3Type.Raw> arguments,
                                   boolean argumentsSpeficied,
                                   boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.aggregateName = aggregateName;
         this.arguments = arguments;
         this.argumentsSpeficied = argumentsSpeficied;
@@ -152,7 +158,7 @@ private List<AbstractType<?>> prepareArgumentTypes(Types types)
                         .collect(toList());
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<DropAggregateStatement>
     {
         private final FunctionName name;
         private final List<CQL3Type.Raw> arguments;
@@ -170,10 +176,14 @@ public Raw(FunctionName name,
             this.ifExists = ifExists;
         }
 
-        public DropAggregateStatement prepare(ClientState state)
+        @Override
+        public DropAggregateStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.keyspace : state.getKeyspace();
-            return new DropAggregateStatement(keyspaceName, name.name, arguments, argumentsSpecified, ifExists);
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.keyspace : state.getKeyspace());
+            if (keyspaceMapper != Constants.IDENTITY_STRING_MAPPER)
+                arguments.forEach(t -> t.forEachUserType(name -> name.updateKeyspaceIfDefined(keyspaceMapper)));
+            return new DropAggregateStatement(rawCQLStatement, keyspaceName, name.name,
+                                              arguments, argumentsSpecified, ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java
index af822063226a..a8fa189581de 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java
@@ -20,6 +20,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.function.Predicate;
+import java.util.function.UnaryOperator;
 import java.util.stream.Stream;
 
 import org.apache.cassandra.audit.AuditLogContext;
@@ -27,24 +28,28 @@
 import org.apache.cassandra.auth.FunctionResource;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.cql3.functions.UDFunction;
 import org.apache.cassandra.cql3.functions.UserFunction;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 
+import static com.google.common.collect.Iterables.transform;
 import static java.lang.String.format;
 import static java.lang.String.join;
 import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.toList;
 
-import static com.google.common.collect.Iterables.transform;
-
 public final class DropFunctionStatement extends AlterSchemaStatement
 {
     private final String functionName;
@@ -52,13 +57,14 @@ public final class DropFunctionStatement extends AlterSchemaStatement
     private final boolean argumentsSpeficied;
     private final boolean ifExists;
 
-    public DropFunctionStatement(String keyspaceName,
+    public DropFunctionStatement(String queryString,
+                                 String keyspaceName,
                                  String functionName,
                                  Collection<CQL3Type.Raw> arguments,
                                  boolean argumentsSpeficied,
                                  boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.functionName = functionName;
         this.arguments = arguments;
         this.argumentsSpeficied = argumentsSpeficied;
@@ -169,7 +175,7 @@ private List<AbstractType<?>> prepareArgumentTypes(Types types)
                         .collect(toList());
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<DropFunctionStatement>
     {
         private final FunctionName name;
         private final List<CQL3Type.Raw> arguments;
@@ -187,10 +193,14 @@ public Raw(FunctionName name,
             this.ifExists = ifExists;
         }
 
-        public DropFunctionStatement prepare(ClientState state)
+        @Override
+        public DropFunctionStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.keyspace : state.getKeyspace();
-            return new DropFunctionStatement(keyspaceName, name.name, arguments, argumentsSpecified, ifExists);
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.keyspace : state.getKeyspace());
+            if (keyspaceMapper != Constants.IDENTITY_STRING_MAPPER)
+                arguments.forEach(t -> t.forEachUserType(name -> name.updateKeyspaceIfDefined(keyspaceMapper)));
+            return new DropFunctionStatement(rawCQLStatement, keyspaceName, name.name, arguments,
+                                             argumentsSpecified, ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropIndexStatement.java
index 24b372d8c3c4..445643bb3831 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropIndexStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropIndexStatement.java
@@ -17,14 +17,20 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.function.UnaryOperator;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.QualifiedName;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
+import org.apache.cassandra.schema.Diff;
+import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
@@ -35,9 +41,10 @@ public final class DropIndexStatement extends AlterSchemaStatement
     private final String indexName;
     private final boolean ifExists;
 
-    public DropIndexStatement(String keyspaceName, String indexName, boolean ifExists)
+    public DropIndexStatement(String queryString, String keyspaceName, String indexName,
+                              boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.indexName = indexName;
         this.ifExists = ifExists;
     }
@@ -94,7 +101,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, indexName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<DropIndexStatement>
     {
         private final QualifiedName name;
         private final boolean ifExists;
@@ -105,10 +112,11 @@ public Raw(QualifiedName name, boolean ifExists)
             this.ifExists = ifExists;
         }
 
-        public DropIndexStatement prepare(ClientState state)
+        @Override
+        public DropIndexStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
-            return new DropIndexStatement(keyspaceName, name.getName(), ifExists);
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
+            return new DropIndexStatement(rawCQLStatement, keyspaceName, name.getName(), ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java
index 47e514a527fe..4c2212569776 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java
@@ -17,10 +17,12 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.function.UnaryOperator;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
@@ -32,9 +34,9 @@ public final class DropKeyspaceStatement extends AlterSchemaStatement
 {
     private final boolean ifExists;
 
-    public DropKeyspaceStatement(String keyspaceName, boolean ifExists)
+    public DropKeyspaceStatement(String queryString, String keyspaceName, boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.ifExists = ifExists;
     }
 
@@ -72,7 +74,7 @@ public String toString()
         return String.format("%s (%s)", getClass().getSimpleName(), keyspaceName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<DropKeyspaceStatement>
     {
         private final String keyspaceName;
         private final boolean ifExists;
@@ -83,9 +85,10 @@ public Raw(String keyspaceName, boolean ifExists)
             this.ifExists = ifExists;
         }
 
-        public DropKeyspaceStatement prepare(ClientState state)
+        @Override
+        public DropKeyspaceStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            return new DropKeyspaceStatement(keyspaceName, ifExists);
+            return new DropKeyspaceStatement(rawCQLStatement, keyspaceMapper.apply(keyspaceName), ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropTableStatement.java
index 78c98be3a70c..6266607b0638 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropTableStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropTableStatement.java
@@ -17,32 +17,37 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.function.UnaryOperator;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
 
-import static java.lang.String.join;
-
 import static com.google.common.collect.Iterables.isEmpty;
 import static com.google.common.collect.Iterables.transform;
+import static java.lang.String.join;
 
 public final class DropTableStatement extends AlterSchemaStatement
 {
     private final String tableName;
     private final boolean ifExists;
 
-    public DropTableStatement(String keyspaceName, String tableName, boolean ifExists)
+    public DropTableStatement(String queryString, String keyspaceName, String tableName,
+                              boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.tableName = tableName;
         this.ifExists = ifExists;
     }
@@ -100,7 +105,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, tableName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<DropTableStatement>
     {
         private final QualifiedName name;
         private final boolean ifExists;
@@ -111,10 +116,11 @@ public Raw(QualifiedName name, boolean ifExists)
             this.ifExists = ifExists;
         }
 
-        public DropTableStatement prepare(ClientState state)
+        @Override
+        public DropTableStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
-            return new DropTableStatement(keyspaceName, name.getName(), ifExists);
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
+            return new DropTableStatement(rawCQLStatement, keyspaceName, name.getName(), ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropTriggerStatement.java
index 967e56834f09..99bad41fa83a 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropTriggerStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropTriggerStatement.java
@@ -17,12 +17,17 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.function.UnaryOperator;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.QualifiedName;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TriggerMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
@@ -34,9 +39,10 @@ public final class DropTriggerStatement extends AlterSchemaStatement
     private final String triggerName;
     private final boolean ifExists;
 
-    public DropTriggerStatement(String keyspaceName, String tableName, String triggerName, boolean ifExists)
+    public DropTriggerStatement(String queryString, String keyspaceName, String tableName,
+                                String triggerName, boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.tableName = tableName;
         this.triggerName = triggerName;
         this.ifExists = ifExists;
@@ -87,7 +93,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, triggerName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<DropTriggerStatement>
     {
         private final QualifiedName tableName;
         private final String triggerName;
@@ -100,10 +106,12 @@ public Raw(QualifiedName tableName, String triggerName, boolean ifExists)
             this.ifExists = ifExists;
         }
 
-        public DropTriggerStatement prepare(ClientState state)
+        @Override
+        public DropTriggerStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = tableName.hasKeyspace() ? tableName.getKeyspace() : state.getKeyspace();
-            return new DropTriggerStatement(keyspaceName, tableName.getName(), triggerName, ifExists);
+            String keyspaceName = keyspaceMapper.apply(tableName.hasKeyspace() ? tableName.getKeyspace() : state.getKeyspace());
+            return new DropTriggerStatement(rawCQLStatement, keyspaceName, tableName.getName(),
+                                            triggerName, ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java
index 97830c882aff..8a8eb8d43083 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java
@@ -18,28 +18,27 @@
 package org.apache.cassandra.cql3.statements.schema;
 
 import java.nio.ByteBuffer;
+import java.util.function.UnaryOperator;
 
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.UTName;
 import org.apache.cassandra.cql3.functions.UserFunction;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
 import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
-import org.apache.cassandra.transport.Event.SchemaChange;
-
-import static java.lang.String.join;
 
 import static com.google.common.collect.Iterables.isEmpty;
 import static com.google.common.collect.Iterables.transform;
-
+import static java.lang.String.join;
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
 
 public final class DropTypeStatement extends AlterSchemaStatement
@@ -47,9 +46,10 @@ public final class DropTypeStatement extends AlterSchemaStatement
     private final String typeName;
     private final boolean ifExists;
 
-    public DropTypeStatement(String keyspaceName, String typeName, boolean ifExists)
+    public DropTypeStatement(String queryString, String keyspaceName, String typeName,
+                             boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.typeName = typeName;
         this.ifExists = ifExists;
     }
@@ -134,7 +134,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, typeName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<DropTypeStatement>
     {
         private final UTName name;
         private final boolean ifExists;
@@ -145,10 +145,12 @@ public Raw(UTName name, boolean ifExists)
             this.ifExists = ifExists;
         }
 
-        public DropTypeStatement prepare(ClientState state)
+        @Override
+        public DropTypeStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
-            return new DropTypeStatement(keyspaceName, name.getStringTypeName(), ifExists);
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
+            return new DropTypeStatement(rawCQLStatement, keyspaceName, name.getStringTypeName(),
+                                         ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropViewStatement.java
index 2c73717546c7..fd5e32178401 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropViewStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropViewStatement.java
@@ -17,13 +17,18 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.function.UnaryOperator;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.QualifiedName;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.cql3.statements.RawKeyspaceAwareStatement;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
@@ -34,9 +39,10 @@ public final class DropViewStatement extends AlterSchemaStatement
     private final String viewName;
     private final boolean ifExists;
 
-    public DropViewStatement(String keyspaceName, String viewName, boolean ifExists)
+    public DropViewStatement(String queryString, String keyspaceName, String viewName,
+                             boolean ifExists)
     {
-        super(keyspaceName);
+        super(queryString, keyspaceName);
         this.viewName = viewName;
         this.ifExists = ifExists;
     }
@@ -83,7 +89,7 @@ public String toString()
         return String.format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, viewName);
     }
 
-    public static final class Raw extends CQLStatement.Raw
+    public static final class Raw extends RawKeyspaceAwareStatement<DropViewStatement>
     {
         private final QualifiedName name;
         private final boolean ifExists;
@@ -94,10 +100,11 @@ public Raw(QualifiedName name, boolean ifExists)
             this.ifExists = ifExists;
         }
 
-        public DropViewStatement prepare(ClientState state)
+        @Override
+        public DropViewStatement prepare(ClientState state, UnaryOperator<String> keyspaceMapper)
         {
-            String keyspaceName = name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace();
-            return new DropViewStatement(keyspaceName, name.getName(), ifExists);
+            String keyspaceName = keyspaceMapper.apply(name.hasKeyspace() ? name.getKeyspace() : state.getKeyspace());
+            return new DropViewStatement(rawCQLStatement, keyspaceName, name.getName(), ifExists);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/IndexAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/IndexAttributes.java
index f30c502ca8b7..431529c5408d 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/IndexAttributes.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/IndexAttributes.java
@@ -24,6 +24,8 @@
 import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.exceptions.SyntaxException;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.DEFAULT_INDEX_CLASS;
+
 public class IndexAttributes extends PropertyDefinitions
 {
     private static final String KW_OPTIONS = "options";
@@ -39,6 +41,19 @@ public class IndexAttributes extends PropertyDefinitions
         keywords.add(KW_OPTIONS);
     }
 
+    public void maybeApplyDefaultIndex()
+    {
+        String defaultIndexClass = DEFAULT_INDEX_CLASS.getString();
+        if (defaultIndexClass == null)
+            return;
+
+        if (!isCustom && customClass == null)
+        {
+            isCustom = true;
+            customClass = defaultIndexClass;
+        }
+    }
+
     public void validate() throws RequestValidationException
     {
         validate(keywords, obsoleteKeywords);
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java
index d4d5b984b3c3..189d2aecd571 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java
@@ -38,7 +38,7 @@ public final class KeyspaceAttributes extends PropertyDefinitions
         for (Option option : Option.values())
             validBuilder.add(option.toString());
         validKeywords = validBuilder.build();
-        obsoleteKeywords = ImmutableSet.of();
+        obsoleteKeywords = ImmutableSet.of("graph_engine");
     }
 
     public void validate()
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
index 93d477c8470d..b7e616ecbe25 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
@@ -17,17 +17,25 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.functions.types.utils.Bytes;
 import org.apache.cassandra.cql3.statements.PropertyDefinitions;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.DroppedColumn;
 import org.apache.cassandra.schema.MemtableParams;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableParams;
@@ -41,8 +49,19 @@
 public final class TableAttributes extends PropertyDefinitions
 {
     public static final String ID = "id";
-    private static final Set<String> validKeywords;
-    private static final Set<String> obsoleteKeywords;
+    public static final Set<String> validKeywords;
+    private static final Set<String> obsoleteKeywords = ImmutableSet.of(
+        "nodesync",
+        "dse_vertex_label_property",
+        "dse_edge_label_property"
+    );
+
+    private static final Set<String> UNSUPPORTED_DSE_COMPACTION_STRATEGIES = ImmutableSet.of(
+        "org.apache.cassandra.db.compaction.TieredCompactionStrategy",
+        "TieredCompactionStrategy",
+        "org.apache.cassandra.db.compaction.MemoryOnlyStrategy",
+        "MemoryOnlyStrategy"
+    );
 
     static
     {
@@ -51,15 +70,28 @@ public final class TableAttributes extends PropertyDefinitions
             validBuilder.add(option.toString());
         validBuilder.add(ID);
         validKeywords = validBuilder.build();
-        obsoleteKeywords = ImmutableSet.of();
     }
 
+    private final Map<ColumnIdentifier, DroppedColumn.Raw> droppedColumnRecords = new HashMap<>();
+
     public void validate()
     {
         validate(validKeywords, obsoleteKeywords);
         build(TableParams.builder()).validate();
     }
 
+    public void addDroppedColumnRecord(ColumnIdentifier name, CQL3Type.Raw type, boolean isStatic, long timestamp)
+    {
+        DroppedColumn.Raw newRecord = new DroppedColumn.Raw(name, type, isStatic, timestamp);
+        if (droppedColumnRecords.put(name, newRecord) != null)
+            throw new InvalidRequestException(String.format("Cannot have multiple dropped column record for column %s", name));
+    }
+
+    public Collection<DroppedColumn.Raw> droppedColumnRecords()
+    {
+        return droppedColumnRecords.values();
+    }
+
     TableParams asNewTableParams()
     {
         return build(TableParams.builder());
@@ -95,6 +127,24 @@ public static Set<String> allKeywords()
         return Sets.union(validKeywords, obsoleteKeywords);
     }
 
+    /**
+     * Returs `true` if this attributes instance has a COMPACTION option with a recognized unsupported compaction
+     * strategy class (coming from DSE). `false` otherwise.
+     */
+    boolean hasUnsupportedDseCompaction()
+    {
+        if (hasOption(Option.COMPACTION))
+        {
+            Map<String, String> compactionOptions = getMap(Option.COMPACTION);
+            String strategy = compactionOptions.get(CompactionParams.Option.CLASS.toString());
+            return UNSUPPORTED_DSE_COMPACTION_STRATEGIES.contains(strategy);
+        }
+        else
+        {
+            return false;
+        }
+    }
+
     private TableParams build(TableParams.Builder builder)
     {
         if (hasOption(ALLOW_AUTO_SNAPSHOT))
@@ -108,22 +158,57 @@ private TableParams build(TableParams.Builder builder)
 
         if (hasOption(COMMENT))
             builder.comment(getString(COMMENT));
-
-        if (hasOption(COMPACTION))
-            builder.compaction(CompactionParams.fromMap(getMap(COMPACTION)));
+        
+        if (hasOption(Option.COMPACTION))
+        {
+            if (hasUnsupportedDseCompaction())
+                builder.compaction(CompactionParams.DEFAULT);
+            else
+                builder.compaction(CompactionParams.fromMap(getMap(Option.COMPACTION)));
+        }
 
         if (hasOption(COMPRESSION))
             builder.compression(CompressionParams.fromMap(getMap(COMPRESSION)));
 
         if (hasOption(MEMTABLE))
-            builder.memtable(MemtableParams.get(getString(MEMTABLE)));
+        {
+            // Handle deserialzation of Astra/CC 4.0 schema with memtable option as a map
+            if (properties.get(MEMTABLE.toString()) instanceof Map)
+            {
+                String memtableClass = getMap(MEMTABLE)
+                             .entrySet()
+                             .stream()
+                             .filter(e -> e.getKey().equals("class"))
+                             .map(Map.Entry::getValue)
+                             .findFirst()
+                             .orElse(null);
+                // Not exhaustive, but avoids raising an error upgrading from a CC 4.0 schema
+                if (memtableClass == null)
+                    builder.memtable(MemtableParams.get(null));
+                else if ("SkipListMemtable".equalsIgnoreCase(memtableClass))
+                    builder.memtable(MemtableParams.get("skiplist"));
+                else if ("PersistentMemoryMemtable".equalsIgnoreCase(memtableClass))
+                    builder.memtable(MemtableParams.get("persistent_memory"));
+                else
+                    builder.memtable(MemtableParams.get("trie"));
+            }
+            else
+                builder.memtable(MemtableParams.get(getString(MEMTABLE)));
+        }
 
         if (hasOption(DEFAULT_TIME_TO_LIVE))
             builder.defaultTimeToLive(getInt(DEFAULT_TIME_TO_LIVE));
 
+        // extensions in CQL are strings, but are stored as a frozen map<string,bytes>
+        if (hasOption(EXTENSIONS))
+            builder.extensions(getMap(EXTENSIONS)
+                               .entrySet()
+                               .stream()
+                               .collect(Collectors.toMap(Map.Entry::getKey, entry -> Bytes.fromHexString(entry.getValue()))));
+
         if (hasOption(GC_GRACE_SECONDS))
             builder.gcGraceSeconds(getInt(GC_GRACE_SECONDS));
-        
+
         if (hasOption(INCREMENTAL_BACKUPS))
             builder.incrementalBackups(getBoolean(INCREMENTAL_BACKUPS.toString(), true));
 
diff --git a/src/java/org/apache/cassandra/db/AbstractCompactionController.java b/src/java/org/apache/cassandra/db/AbstractCompactionController.java
index db533ee870f3..25f773536d7e 100644
--- a/src/java/org/apache/cassandra/db/AbstractCompactionController.java
+++ b/src/java/org/apache/cassandra/db/AbstractCompactionController.java
@@ -20,6 +20,7 @@
 
 import java.util.function.LongPredicate;
 
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.schema.CompactionParams;
 
@@ -28,30 +29,20 @@
  */
 public abstract class AbstractCompactionController implements AutoCloseable
 {
-    public final ColumnFamilyStore cfs;
+    public final CompactionRealm realm;
     public final long gcBefore;
     public final CompactionParams.TombstoneOption tombstoneOption;
 
-    public AbstractCompactionController(final ColumnFamilyStore cfs, final long gcBefore, CompactionParams.TombstoneOption tombstoneOption)
+    protected AbstractCompactionController(final CompactionRealm realm, final long gcBefore, CompactionParams.TombstoneOption tombstoneOption)
     {
-        assert cfs != null;
-        this.cfs = cfs;
+        assert realm != null;
+        this.realm = realm;
         this.gcBefore = gcBefore;
         this.tombstoneOption = tombstoneOption;
     }
 
     public abstract boolean compactingRepaired();
 
-    public String getKeyspace()
-    {
-        return cfs.getKeyspaceName();
-    }
-
-    public String getColumnFamily()
-    {
-        return cfs.name;
-    }
-
     public Iterable<UnfilteredRowIterator> shadowSources(DecoratedKey key, boolean tombstoneOnly)
     {
         return null;
diff --git a/src/java/org/apache/cassandra/db/AbstractReadQuery.java b/src/java/org/apache/cassandra/db/AbstractReadQuery.java
index 448069cfca10..20ee9b070807 100644
--- a/src/java/org/apache/cassandra/db/AbstractReadQuery.java
+++ b/src/java/org/apache/cassandra/db/AbstractReadQuery.java
@@ -18,6 +18,8 @@
 package org.apache.cassandra.db;
 
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.statements.SelectOptions;
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
@@ -111,6 +113,10 @@ public String toCQLString()
         if (limits() != DataLimits.NONE)
             sb.append(' ').append(limits());
 
+        ANNOptions annOptions = rowFilter().annOptions();
+        if (annOptions != ANNOptions.NONE)
+            sb.append(" WITH ").append(SelectOptions.ANN_OPTIONS).append(" = ").append(annOptions.toCQLString());
+
         // ALLOW FILTERING might not be strictly necessary
         sb.append(" ALLOW FILTERING");
 
diff --git a/src/java/org/apache/cassandra/db/ArrayClustering.java b/src/java/org/apache/cassandra/db/ArrayClustering.java
index b04910c434cb..9b98b7563fe3 100644
--- a/src/java/org/apache/cassandra/db/ArrayClustering.java
+++ b/src/java/org/apache/cassandra/db/ArrayClustering.java
@@ -32,7 +32,7 @@ public ArrayClustering(byte[]... values)
 
     public long unsharedHeapSize()
     {
-        if (this == ByteArrayAccessor.factory.clustering() || this == ByteArrayAccessor.factory.staticClustering())
+        if (this == ByteArrayAccessor.factory.clustering())
             return 0;
         long arrayRefSize = ObjectSizes.sizeOfArray(values);
         long elementsSize = 0;
@@ -43,7 +43,7 @@ public long unsharedHeapSize()
 
     public long unsharedHeapSizeExcludingData()
     {
-        if (this == ByteArrayAccessor.factory.clustering() || this == ByteArrayAccessor.factory.staticClustering())
+        if (this == ByteArrayAccessor.factory.clustering())
             return 0;
         return EMPTY_SIZE + ObjectSizes.sizeOfArray(values);
     }
diff --git a/src/java/org/apache/cassandra/db/CBuilder.java b/src/java/org/apache/cassandra/db/CBuilder.java
deleted file mode 100644
index 7b28684344b1..000000000000
--- a/src/java/org/apache/cassandra/db/CBuilder.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.db.marshal.ValueAccessor;
-
-/**
- * Allows to build ClusteringPrefixes, either Clustering or ClusteringBound.
- */
-public abstract class CBuilder
-{
-    public static CBuilder STATIC_BUILDER = new CBuilder()
-    {
-        public int count()
-        {
-            return 0;
-        }
-
-        public int remainingCount()
-        {
-            return 0;
-        }
-
-        public ClusteringComparator comparator()
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public <T> CBuilder add(T value, ValueAccessor<T> accessor)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public CBuilder add(Object value)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public Clustering<?> build()
-        {
-            return Clustering.STATIC_CLUSTERING;
-        }
-
-        public ClusteringBound<?> buildBound(boolean isStart, boolean isInclusive)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public Clustering<?> buildWith(List<ByteBuffer> newValues)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public ClusteringBound<?> buildBoundWith(List<ByteBuffer> newValues, boolean isStart, boolean isInclusive)
-        {
-            throw new UnsupportedOperationException();
-        }
-    };
-
-    public static CBuilder create(ClusteringComparator comparator)
-    {
-        return new ArrayBackedBuilder(comparator);
-    }
-
-    public abstract int count();
-    public abstract int remainingCount();
-    public abstract ClusteringComparator comparator();
-    public final CBuilder add(ByteBuffer value)
-    {
-        return add(value, ByteBufferAccessor.instance);
-    }
-    public final <V> CBuilder add(ClusteringPrefix<V> prefix, int i)
-    {
-        return add(prefix.get(i), prefix.accessor());
-    }
-    public abstract <V> CBuilder add(V value, ValueAccessor<V> accessor);
-    public abstract CBuilder add(Object value);
-    public abstract Clustering<?> build();
-    public abstract ClusteringBound<?> buildBound(boolean isStart, boolean isInclusive);
-    public abstract Clustering<?> buildWith(List<ByteBuffer> newValues);
-    public abstract ClusteringBound<?> buildBoundWith(List<ByteBuffer> newValues, boolean isStart, boolean isInclusive);
-
-    private static class ArrayBackedBuilder extends CBuilder
-    {
-        private final ClusteringComparator type;
-        private final ByteBuffer[] values;
-        private int size;
-        private boolean built;
-
-        public ArrayBackedBuilder(ClusteringComparator type)
-        {
-            this.type = type;
-            this.values = new ByteBuffer[type.size()];
-        }
-
-        public int count()
-        {
-            return size;
-        }
-
-        public int remainingCount()
-        {
-            return values.length - size;
-        }
-
-        public ClusteringComparator comparator()
-        {
-            return type;
-        }
-
-        public <V> CBuilder add(V value, ValueAccessor<V> accessor)
-        {
-            if (isDone())
-                throw new IllegalStateException();
-            values[size++] = accessor.toBuffer(value);
-            return this;
-        }
-
-        public CBuilder add(Object value)
-        {
-            return add(((AbstractType)type.subtype(size)).decompose(value));
-        }
-
-        private boolean isDone()
-        {
-            return remainingCount() == 0 || built;
-        }
-
-        public Clustering<?> build()
-        {
-            // We don't allow to add more element to a builder that has been built so
-            // that we don't have to copy values.
-            built = true;
-
-            // Currently, only dense table can leave some clustering column out (see #7990)
-            return size == 0 ? Clustering.EMPTY : Clustering.make(values);
-        }
-
-        public ClusteringBound<?> buildBound(boolean isStart, boolean isInclusive)
-        {
-            // We don't allow to add more element to a builder that has been built so
-            // that we don't have to copy values (even though we have to do it in most cases).
-            built = true;
-
-            if (size == 0)
-                return isStart ? BufferClusteringBound.BOTTOM : BufferClusteringBound.TOP;
-
-            return BufferClusteringBound.create(ClusteringBound.boundKind(isStart, isInclusive),
-                                size == values.length ? values : Arrays.copyOfRange(values, 0, size));
-        }
-
-        public Clustering<?> buildWith(List<ByteBuffer> newValues)
-        {
-            assert size + newValues.size() <= type.size();
-            ByteBuffer[] buffers = Arrays.copyOf(values, type.size());
-            int newSize = size;
-            for (ByteBuffer value : newValues)
-                buffers[newSize++] = value;
-
-            return Clustering.make(buffers);
-        }
-
-        public ClusteringBound<?> buildBoundWith(List<ByteBuffer> newValues, boolean isStart, boolean isInclusive)
-        {
-            ByteBuffer[] buffers = Arrays.copyOf(values, size + newValues.size());
-            int newSize = size;
-            for (ByteBuffer value : newValues)
-                buffers[newSize++] = value;
-
-            return BufferClusteringBound.create(ClusteringBound.boundKind(isStart, isInclusive), buffers);
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java b/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java
index ef9d0d137778..b1bb09048a3c 100644
--- a/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java
+++ b/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java
@@ -39,7 +39,7 @@ public CassandraKeyspaceWriteHandler(Keyspace keyspace)
     }
 
     @Override
-    public WriteContext beginWrite(Mutation mutation, boolean makeDurable) throws RequestExecutionException
+    public WriteContext beginWrite(Mutation mutation, WriteOptions writeOptions) throws RequestExecutionException
     {
         OpOrder.Group group = null;
         try
@@ -48,7 +48,7 @@ public WriteContext beginWrite(Mutation mutation, boolean makeDurable) throws Re
 
             // write the mutation to the commitlog and memtables
             CommitLogPosition position = null;
-            if (makeDurable)
+            if (writeOptions.shouldWriteCommitLog(mutation.getKeyspaceName()))
             {
                 position = addToCommitLog(mutation);
             }
diff --git a/src/java/org/apache/cassandra/db/Clusterable.java b/src/java/org/apache/cassandra/db/Clusterable.java
index 118b2724f503..8963f32f619d 100644
--- a/src/java/org/apache/cassandra/db/Clusterable.java
+++ b/src/java/org/apache/cassandra/db/Clusterable.java
@@ -20,8 +20,11 @@
 /**
  * Common class for objects that are identified by a clustering prefix, and can be thus sorted by a
  * {@link ClusteringComparator}.
+ *
+ * Note that clusterings can have mixed accessors (necessary because the static clustering is always of ByteBuffer
+ * accessor) and thus the accessor type cannot be set here.
  */
-public interface Clusterable<T>
+public interface Clusterable
 {
-    public ClusteringPrefix<T> clustering();
+    public ClusteringPrefix<?> clustering();
 }
diff --git a/src/java/org/apache/cassandra/db/ClusteringBound.java b/src/java/org/apache/cassandra/db/ClusteringBound.java
index 4afdfe628504..2214b424b2ab 100644
--- a/src/java/org/apache/cassandra/db/ClusteringBound.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBound.java
@@ -111,7 +111,7 @@ static <V> ClusteringBound<V> exclusiveEndOf(ClusteringPrefix<V> from)
 
     static ClusteringBound<?> create(ClusteringComparator comparator, boolean isStart, boolean isInclusive, Object... values)
     {
-        CBuilder builder = CBuilder.create(comparator);
+        ClusteringBuilder builder = ClusteringBuilder.create(comparator);
         for (Object val : values)
         {
             if (val instanceof ByteBuffer)
@@ -135,4 +135,4 @@ default ClusteringBound<V> asEndBound()
         assert isEnd();
         return this;
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/db/ClusteringBoundary.java b/src/java/org/apache/cassandra/db/ClusteringBoundary.java
index 9e0a87c2efab..df35a5d1bb14 100644
--- a/src/java/org/apache/cassandra/db/ClusteringBoundary.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBoundary.java
@@ -49,4 +49,4 @@ default ClusteringBound<V> asEndBound()
     {
         return closeBound(false);
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/db/ClusteringBuilder.java b/src/java/org/apache/cassandra/db/ClusteringBuilder.java
new file mode 100644
index 000000000000..6c448dd9d25b
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/ClusteringBuilder.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.ValueAccessor;
+
+/**
+ * Allows to build ClusteringPrefixes, either Clustering or ClusteringBound.
+ */
+public abstract class ClusteringBuilder
+{
+    public static ClusteringBuilder STATIC_BUILDER = new ClusteringBuilder()
+    {
+        public int count()
+        {
+            return 0;
+        }
+
+        public int remainingCount()
+        {
+            return 0;
+        }
+
+        public ClusteringComparator comparator()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public <T> ClusteringBuilder add(T value, ValueAccessor<T> accessor)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public ClusteringBuilder add(Object value)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public Clustering<?> build()
+        {
+            return Clustering.STATIC_CLUSTERING;
+        }
+
+        public ClusteringBound<?> buildBound(boolean isStart, boolean isInclusive)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public Clustering<?> buildWith(List<ByteBuffer> newValues)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public ClusteringBound<?> buildBoundWith(List<ByteBuffer> newValues, boolean isStart, boolean isInclusive)
+        {
+            throw new UnsupportedOperationException();
+        }
+    };
+
+    public static ClusteringBuilder create(ClusteringComparator comparator)
+    {
+        return new ArrayBackedBuilder(comparator);
+    }
+
+    public abstract int count();
+    public abstract int remainingCount();
+    public abstract ClusteringComparator comparator();
+    public final ClusteringBuilder add(ByteBuffer value)
+    {
+        return add(value, ByteBufferAccessor.instance);
+    }
+    public final <V> ClusteringBuilder add(ClusteringPrefix<V> prefix, int i)
+    {
+        return add(prefix.get(i), prefix.accessor());
+    }
+    public abstract <V> ClusteringBuilder add(V value, ValueAccessor<V> accessor);
+    public abstract ClusteringBuilder add(Object value);
+    public abstract Clustering<?> build();
+    public abstract ClusteringBound<?> buildBound(boolean isStart, boolean isInclusive);
+    public abstract Clustering<?> buildWith(List<ByteBuffer> newValues);
+    public abstract ClusteringBound<?> buildBoundWith(List<ByteBuffer> newValues, boolean isStart, boolean isInclusive);
+
+    private static class ArrayBackedBuilder extends ClusteringBuilder
+    {
+        private final ClusteringComparator type;
+        private final ByteBuffer[] values;
+        private int size;
+        private boolean built;
+
+        public ArrayBackedBuilder(ClusteringComparator type)
+        {
+            this.type = type;
+            this.values = new ByteBuffer[type.size()];
+        }
+
+        public int count()
+        {
+            return size;
+        }
+
+        public int remainingCount()
+        {
+            return values.length - size;
+        }
+
+        public ClusteringComparator comparator()
+        {
+            return type;
+        }
+
+        public <V> ClusteringBuilder add(V value, ValueAccessor<V> accessor)
+        {
+            if (isDone())
+                throw new IllegalStateException();
+            values[size++] = accessor.toBuffer(value);
+            return this;
+        }
+
+        public ClusteringBuilder add(Object value)
+        {
+            return add(((AbstractType)type.subtype(size)).decompose(value));
+        }
+
+        private boolean isDone()
+        {
+            return remainingCount() == 0 || built;
+        }
+
+        public Clustering<?> build()
+        {
+            // We don't allow to add more element to a builder that has been built so
+            // that we don't have to copy values.
+            built = true;
+
+            // Currently, only dense table can leave some clustering column out (see #7990)
+            return size == 0 ? Clustering.EMPTY : Clustering.make(values);
+        }
+
+        public ClusteringBound<?> buildBound(boolean isStart, boolean isInclusive)
+        {
+            // We don't allow to add more element to a builder that has been built so
+            // that we don't have to copy values (even though we have to do it in most cases).
+            built = true;
+
+            if (size == 0)
+                return isStart ? BufferClusteringBound.BOTTOM : BufferClusteringBound.TOP;
+
+            return BufferClusteringBound.create(ClusteringBound.boundKind(isStart, isInclusive),
+                                size == values.length ? values : Arrays.copyOfRange(values, 0, size));
+        }
+
+        public Clustering<?> buildWith(List<ByteBuffer> newValues)
+        {
+            assert size + newValues.size() <= type.size();
+            ByteBuffer[] buffers = Arrays.copyOf(values, type.size());
+            int newSize = size;
+            for (ByteBuffer value : newValues)
+                buffers[newSize++] = value;
+
+            return Clustering.make(buffers);
+        }
+
+        public ClusteringBound<?> buildBoundWith(List<ByteBuffer> newValues, boolean isStart, boolean isInclusive)
+        {
+            ByteBuffer[] buffers = Arrays.copyOf(values, size + newValues.size());
+            int newSize = size;
+            for (ByteBuffer value : newValues)
+                buffers[newSize++] = value;
+
+            return BufferClusteringBound.create(ClusteringBound.boundKind(isStart, isInclusive), buffers);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/ClusteringComparator.java b/src/java/org/apache/cassandra/db/ClusteringComparator.java
index 2949130707fe..00f48b1506c2 100644
--- a/src/java/org/apache/cassandra/db/ClusteringComparator.java
+++ b/src/java/org/apache/cassandra/db/ClusteringComparator.java
@@ -26,20 +26,19 @@
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ValueAccessor;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.io.sstable.format.big.IndexInfo;
 import org.apache.cassandra.serializers.MarshalException;
-
-import org.apache.cassandra.io.sstable.IndexInfo;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
 import static org.apache.cassandra.utils.bytecomparable.ByteSource.EXCLUDED;
 import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT;
-import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_EMPTY;
-import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_EMPTY_REVERSED;
 import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_NULL;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_NULL_REVERSED;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_CLUSTERING_NULL;
 import static org.apache.cassandra.utils.bytecomparable.ByteSource.TERMINATOR;
 
 /**
@@ -57,8 +56,15 @@ public class ClusteringComparator implements Comparator<Clusterable>
     private final Comparator<IndexInfo> indexReverseComparator;
     private final Comparator<Clusterable> reverseComparator;
 
-    private final Comparator<Row> rowComparator = (r1, r2) -> compare((ClusteringPrefix<?>) r1.clustering(),
-                                                                      (ClusteringPrefix<?>) r2.clustering());
+    private final Comparator<Row> rowComparator = new Comparator<>()
+    {
+        @Override
+        public int compare(Row r1, Row r2)
+        {
+            return ClusteringComparator.this.compare((ClusteringPrefix<?>) r1.clustering(),
+                           (ClusteringPrefix<?>) r2.clustering());
+        }
+    };
 
     public ClusteringComparator(AbstractType<?>... clusteringTypes)
     {
@@ -70,11 +76,34 @@ public ClusteringComparator(Iterable<AbstractType<?>> clusteringTypes)
         // copy the list to ensure despatch is monomorphic
         this.clusteringTypes = ImmutableList.copyOf(clusteringTypes);
 
-        this.indexComparator = (o1, o2) -> ClusteringComparator.this.compare((ClusteringPrefix<?>) o1.lastName,
-                                                                             (ClusteringPrefix<?>) o2.lastName);
-        this.indexReverseComparator = (o1, o2) -> ClusteringComparator.this.compare((ClusteringPrefix<?>) o1.firstName,
-                                                                                    (ClusteringPrefix<?>) o2.firstName);
-        this.reverseComparator = (c1, c2) -> ClusteringComparator.this.compare(c2, c1);
+        this.indexComparator = new Comparator<>()
+        {
+            @Override
+            public int compare(IndexInfo o1, IndexInfo o2)
+            {
+                return ClusteringComparator.this.compare((ClusteringPrefix<?>) o1.lastName,
+                               (ClusteringPrefix<?>) o2.lastName);
+            }
+        };
+
+        this.indexReverseComparator = new Comparator<>()
+        {
+            @Override
+            public int compare(IndexInfo o1, IndexInfo o2)
+            {
+                return ClusteringComparator.this.compare((ClusteringPrefix<?>) o1.firstName,
+                               (ClusteringPrefix<?>) o2.firstName);
+            }
+        };
+
+        this.reverseComparator = new Comparator<>()
+        {
+            @Override
+            public int compare(Clusterable o1, Clusterable o2)
+            {
+                return ClusteringComparator.this.compare(o2, o1);
+            }
+        };
         for (AbstractType<?> type : clusteringTypes)
             type.checkComparable(); // this should already be enforced by TableMetadata.Builder.addColumn, but we check again for other constructors
     }
@@ -121,7 +150,7 @@ public Clustering<?> make(Object... values)
         if (values.length != size())
             throw new IllegalArgumentException(String.format("Invalid number of components, expecting %d but got %d", size(), values.length));
 
-        CBuilder builder = CBuilder.create(this);
+        ClusteringBuilder builder = ClusteringBuilder.create(this);
         for (Object val : values)
         {
             if (val instanceof ByteBuffer)
@@ -196,34 +225,6 @@ public <V1, V2> int compareComponent(int i, ClusteringPrefix<V1> v1, ClusteringP
         return compareComponent(i, v1.get(i), v1.accessor(), v2.get(i), v2.accessor());
     }
 
-    /**
-     * Returns whether this clustering comparator is compatible with the provided one,
-     * that is if the provided one can be safely replaced by this new one.
-     *
-     * @param previous the previous comparator that we want to replace and test
-     * compatibility with.
-     *
-     * @return whether {@code previous} can be safely replaced by this comparator.
-     */
-    public boolean isCompatibleWith(ClusteringComparator previous)
-    {
-        if (this == previous)
-            return true;
-
-        // Extending with new components is fine, shrinking is not
-        if (size() < previous.size())
-            return false;
-
-        for (int i = 0; i < previous.size(); i++)
-        {
-            AbstractType<?> tprev = previous.subtype(i);
-            AbstractType<?> tnew = subtype(i);
-            if (!tnew.isCompatibleWith(tprev))
-                return false;
-        }
-        return true;
-    }
-
     /**
      * Validates the provided prefix for corrupted data.
      *
@@ -296,6 +297,7 @@ public int next()
                 {
                     if (current != null)
                     {
+                        // Process bytes of the current component.
                         int b = current.next();
                         if (b > END_OF_STREAM)
                             return b;
@@ -303,24 +305,26 @@ public int next()
                     }
 
                     int sz = src.size();
-                    if (srcnum == sz)
+                    if (srcnum == sz) // already produced the Kind byte, we are done
                         return END_OF_STREAM;
 
                     ++srcnum;
                     if (srcnum == sz)
                         return src.kind().asByteComparableValue(version);
+                    else
+                        return advanceToComponent(src.get(srcnum));
+                }
 
-                    final V nextComponent = src.get(srcnum);
-                    // We can have a null as the clustering component (this is a relic of COMPACT STORAGE, but also
-                    // can appear in indexed partitions with no rows but static content),
+                private int advanceToComponent(V nextComponent)
+                {
                     if (nextComponent == null)
                     {
-                        if (version != Version.LEGACY)
-                            return NEXT_COMPONENT_NULL; // always sorts before non-nulls, including for reversed types
+                        if (version == Version.OSS50)
+                            return NEXT_CLUSTERING_NULL; // always sorts before non-nulls, including for reversed types
                         else
                         {
                             // legacy version did not permit nulls in clustering keys and treated these as null values
-                            return subtype(srcnum).isReversed() ? NEXT_COMPONENT_EMPTY_REVERSED : NEXT_COMPONENT_EMPTY;
+                            return nextComponentNull(subtype(srcnum).isReversed());
                         }
                     }
 
@@ -328,13 +332,18 @@ public int next()
                     // and also null values for some types (e.g. int, varint but not text) that are encoded as empty
                     // buffers.
                     if (current == null)
-                        return subtype(srcnum).isReversed() ? NEXT_COMPONENT_EMPTY_REVERSED : NEXT_COMPONENT_EMPTY;
+                        return nextComponentNull(subtype(srcnum).isReversed());
 
                     return NEXT_COMPONENT;
                 }
             };
         }
 
+        private int nextComponentNull(boolean isReversed)
+        {
+            return isReversed ? NEXT_COMPONENT_NULL_REVERSED : NEXT_COMPONENT_NULL;
+        }
+
         public String toString()
         {
             return src.clusteringString(subtypes());
@@ -344,14 +353,31 @@ public String toString()
     /**
      * Produces a clustering from the given byte-comparable value. The method will throw an exception if the value
      * does not correctly encode a clustering of this type, including if it encodes a position before or after a
-     * clustering (i.e. a bound/boundary).
+     * clustering (i.e. a bound/boundary). Uses the OSS50 version of the byte-comparable encoding.
      *
      * @param accessor Accessor to use to construct components.
      * @param comparable The clustering encoded as a byte-comparable sequence.
      */
-    public <V> Clustering<V> clusteringFromByteComparable(ValueAccessor<V> accessor, ByteComparable comparable)
+    public <V> Clustering<?> clusteringFromByteComparable(ValueAccessor<V> accessor, ByteComparable comparable)
+    {
+        return clusteringFromByteComparable(accessor, comparable, ByteComparable.Version.OSS50);
+    }
+
+    /**
+     * Produces a clustering from the given byte-comparable value. The method will throw an exception if the value
+     * does not correctly encode a clustering of this type, including if it encodes a position before or after a
+     * clustering (i.e. a bound/boundary). Uses the OSS50 version of the byte-comparable encoding.
+     *
+     * @param accessor Accessor to use to construct components. Because this will be used to construct individual
+     *                 arrays/buffers for each component, it may be sensible to use an accessor that allocates larger
+     *                 buffers in advance.
+     * @param comparable The clustering encoded as a byte-comparable sequence.
+     * @param version The version of the byte-comparable encoding.
+     */
+    public <V> Clustering<?> clusteringFromByteComparable(ValueAccessor<V> accessor,
+                                                          ByteComparable comparable,
+                                                          ByteComparable.Version version)
     {
-        ByteComparable.Version version = ByteComparable.Version.OSS50;
         ByteSource.Peekable orderedBytes = ByteSource.peekable(comparable.asComparableBytes(version));
         if (orderedBytes == null)
             return null;
@@ -364,7 +390,7 @@ public <V> Clustering<V> clusteringFromByteComparable(ValueAccessor<V> accessor,
             assert size() == 0 : "Terminator should be after " + size() + " components, got 0";
             return accessor.factory().clustering();
         case EXCLUDED:
-            return accessor.factory().staticClustering();
+            return Clustering.STATIC_CLUSTERING;
         default:
             // continue with processing
         }
@@ -376,11 +402,11 @@ public <V> Clustering<V> clusteringFromByteComparable(ValueAccessor<V> accessor,
         {
             switch (sep)
             {
-            case NEXT_COMPONENT_NULL:
+            case NEXT_CLUSTERING_NULL:
                 components[cc] = null;
                 break;
-            case NEXT_COMPONENT_EMPTY:
-            case NEXT_COMPONENT_EMPTY_REVERSED:
+            case NEXT_COMPONENT_NULL:
+            case NEXT_COMPONENT_NULL_REVERSED:
                 components[cc] = subtype(cc).fromComparableBytes(accessor, null, version);
                 break;
             case NEXT_COMPONENT:
@@ -428,11 +454,11 @@ public <V> ClusteringBound<V> boundFromByteComparable(ValueAccessor<V> accessor,
         {
             switch (sep)
             {
-            case NEXT_COMPONENT_NULL:
+            case NEXT_CLUSTERING_NULL:
                 components[cc] = null;
                 break;
-            case NEXT_COMPONENT_EMPTY:
-            case NEXT_COMPONENT_EMPTY_REVERSED:
+            case NEXT_COMPONENT_NULL:
+            case NEXT_COMPONENT_NULL_REVERSED:
                 components[cc] = subtype(cc).fromComparableBytes(accessor, null, version);
                 break;
             case NEXT_COMPONENT:
@@ -485,11 +511,11 @@ public <V> ClusteringBoundary<V> boundaryFromByteComparable(ValueAccessor<V> acc
         {
             switch (sep)
             {
-            case NEXT_COMPONENT_NULL:
+            case NEXT_CLUSTERING_NULL:
                 components[cc] = null;
                 break;
-            case NEXT_COMPONENT_EMPTY:
-            case NEXT_COMPONENT_EMPTY_REVERSED:
+            case NEXT_COMPONENT_NULL:
+            case NEXT_COMPONENT_NULL_REVERSED:
                 components[cc] = subtype(cc).fromComparableBytes(accessor, null, version);
                 break;
             case NEXT_COMPONENT:
diff --git a/src/java/org/apache/cassandra/db/ClusteringPrefix.java b/src/java/org/apache/cassandra/db/ClusteringPrefix.java
index 02f9330b430b..fa1784ade142 100644
--- a/src/java/org/apache/cassandra/db/ClusteringPrefix.java
+++ b/src/java/org/apache/cassandra/db/ClusteringPrefix.java
@@ -19,17 +19,19 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.List;
+import java.util.Objects;
 import java.util.function.ToIntFunction;
 
 import org.apache.cassandra.cache.IMeasurableMemory;
-import org.apache.cassandra.config.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ByteArrayAccessor;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.db.marshal.CompositeType;
 import org.apache.cassandra.db.marshal.ValueAccessor;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -52,7 +54,7 @@
  *   3) {@code ClusteringBoundary} represents the threshold between two adjacent range tombstones.
  * See those classes for more details.
  */
-public interface ClusteringPrefix<V> extends IMeasurableMemory, Clusterable<V>
+public interface ClusteringPrefix<V> extends IMeasurableMemory, Clusterable
 {
     public static final Serializer serializer = new Serializer();
 
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 99ee013024d6..c75ea903a2d8 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -19,15 +19,12 @@
 
 import java.io.IOException;
 import java.io.PrintStream;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
 import java.nio.ByteBuffer;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -36,6 +33,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
@@ -44,10 +42,13 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import javax.management.MalformedObjectNameException;
 import javax.management.ObjectName;
 import javax.management.openmbean.CompositeData;
@@ -68,6 +69,7 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import com.google.common.primitives.Longs;
 import com.google.common.util.concurrent.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -83,15 +85,24 @@
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.commitlog.IntervalSet;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.db.compaction.CompactionStrategy;
+import org.apache.cassandra.db.compaction.CompactionStrategyContainer;
+import org.apache.cassandra.db.compaction.CompactionStrategyFactory;
 import org.apache.cassandra.db.compaction.CompactionStrategyManager;
+import org.apache.cassandra.db.compaction.CompactionStrategyOptions;
 import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
+import org.apache.cassandra.db.compaction.unified.Environment;
+import org.apache.cassandra.db.compaction.unified.RealEnvironment;
 import org.apache.cassandra.db.filter.ClusteringIndexFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.Tracker;
 import org.apache.cassandra.db.lifecycle.View;
@@ -101,14 +112,12 @@
 import org.apache.cassandra.db.partitions.CachedPartition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.repair.CassandraTableRepairManager;
-import org.apache.cassandra.db.rows.CellPath;
 import org.apache.cassandra.db.streaming.CassandraStreamManager;
 import org.apache.cassandra.db.view.TableViews;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -126,21 +135,23 @@
 import org.apache.cassandra.io.sstable.SSTableId;
 import org.apache.cassandra.io.sstable.SSTableIdFactory;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.StorageHandler;
+import org.apache.cassandra.io.sstable.filter.BloomFilterTracker;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.metrics.KeyspaceMetrics;
 import org.apache.cassandra.metrics.Sampler;
 import org.apache.cassandra.metrics.Sampler.Sample;
 import org.apache.cassandra.metrics.Sampler.SamplerType;
 import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.metrics.TopPartitionTracker;
 import org.apache.cassandra.repair.TableRepairManager;
-import org.apache.cassandra.repair.consistent.admin.CleanupSummary;
 import org.apache.cassandra.repair.consistent.admin.PendingStat;
-import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
 import org.apache.cassandra.schema.CompressionParams;
@@ -151,6 +162,10 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.Type;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.service.StorageService;
@@ -162,6 +177,7 @@
 import org.apache.cassandra.service.snapshot.TableSnapshot;
 import org.apache.cassandra.streaming.TableStreamManager;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.DefaultValue;
 import org.apache.cassandra.utils.ExecutorUtils;
 import org.apache.cassandra.utils.FBUtilities;
@@ -169,16 +185,20 @@
 import org.apache.cassandra.utils.JsonUtils;
 import org.apache.cassandra.utils.MBeanWrapper;
 import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.OverlapIterator;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.WrappedRunnable;
 import org.apache.cassandra.utils.concurrent.CountDownLatch;
 import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.utils.concurrent.Refs;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLED_AUTO_COMPACTION_PROPERTY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UNSAFE_SYSTEM;
 import static org.apache.cassandra.config.DatabaseDescriptor.getFlushWriters;
 import static org.apache.cassandra.db.commitlog.CommitLogPosition.NONE;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@@ -186,12 +206,12 @@
 import static org.apache.cassandra.utils.FBUtilities.now;
 import static org.apache.cassandra.utils.Throwables.maybeFail;
 import static org.apache.cassandra.utils.Throwables.merge;
+import static org.apache.cassandra.utils.Throwables.perform;
 import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
 
-public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner, SSTable.Owner
+public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner, SSTable.Owner, CompactionRealm
 {
     private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyStore.class);
-
     /*
     We keep a pool of threads for each data directory, size of each pool is memtable_flush_writers.
     When flushing we start a Flush runnable in the flushExecutor. Flush calculates how to split the
@@ -200,20 +220,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     are finished. By having flushExecutor size the same size as each of the perDiskflushExecutors we make sure we can
     have that many flushes going at the same time.
     */
-    private static final ExecutorPlus flushExecutor = DatabaseDescriptor.isDaemonInitialized() 
+    private static final ExecutorPlus flushExecutor = DatabaseDescriptor.enableMemtableAndCommitLog()
                                                       ? executorFactory().withJmxInternal().pooled("MemtableFlushWriter", getFlushWriters())
                                                       : null;
 
     // post-flush executor is single threaded to provide guarantee that any flush Future on a CF will never return until prior flushes have completed
-    private static final ExecutorPlus postFlushExecutor = DatabaseDescriptor.isDaemonInitialized()
+    private static final ExecutorPlus postFlushExecutor = DatabaseDescriptor.enableMemtableAndCommitLog()
                                                           ? executorFactory().withJmxInternal().sequential("MemtablePostFlush")
                                                           : null;
 
-    private static final ExecutorPlus reclaimExecutor = DatabaseDescriptor.isDaemonInitialized()
+    private static final ExecutorPlus reclaimExecutor = DatabaseDescriptor.enableMemtableAndCommitLog()
                                                         ? executorFactory().withJmxInternal().sequential("MemtableReclaimMemory")
                                                         : null;
 
-    private static final PerDiskFlushExecutors perDiskflushExecutors = DatabaseDescriptor.isDaemonInitialized()
+    private static final PerDiskFlushExecutors perDiskflushExecutors = DatabaseDescriptor.enableMemtableAndCommitLog()
                                                                        ? new PerDiskFlushExecutors(DatabaseDescriptor.getFlushWriters(),
                                                                                                   DatabaseDescriptor.getNonLocalSystemKeyspacesDataFileLocations(),
                                                                                                   DatabaseDescriptor.useSpecificLocationForLocalSystemData())
@@ -245,7 +265,10 @@ public enum FlushReason
         ANTICOMPACTION,
         SCHEMA_CHANGE,
         OWNED_RANGES_CHANGE,
-        UNIT_TESTS // explicitly requested flush needed for a test
+        UNIT_TESTS, // explicitly requested flush needed for a test
+        /** Flush performed to a remote storage. Used by remote commit log replay */
+        REMOTE_REPLAY,
+        BATCHLOG_REPLAY
     }
 
     private static final String[] COUNTER_NAMES = new String[]{"table", "count", "error", "value"};
@@ -284,7 +307,39 @@ public enum FlushReason
     /** @deprecated See CASSANDRA-9448 */
     @Deprecated(since = "3.0")
     private final String oldMBeanName;
-    private volatile boolean valid = true;
+
+    public enum STATUS
+    {
+        /**
+         * Initial status when CFS is created
+         */
+        VALID,
+        /**
+         * When table is invalidated with unloading data
+         */
+        INVALID_UNLOADED,
+        /**
+         * When table is invalidated with dropping data
+         */
+        INVALID_DROPPED;
+
+        /**
+         * @return true if CFS is not invalidated
+         */
+        public boolean isValid()
+        {
+            return this == VALID;
+        }
+
+        /**
+         * @return true if CFS is invalidated and sstables should be dropped locally and remotely
+         */
+        public boolean isInvalidAndShouldDropData()
+        {
+            return this == INVALID_DROPPED;
+        }
+    }
+    private volatile STATUS status = STATUS.VALID;
 
     private volatile Memtable.Factory memtableFactory;
 
@@ -293,7 +348,7 @@ public enum FlushReason
      *
      * We synchronize on the Tracker to ensure isolation when we want to make sure
      * that the memtable we're acting on doesn't change out from under us.  I.e., flush
-     * syncronizes on it to make sure it can submit on both executors atomically,
+     * synchronizes on it to make sure it can submit on both executors atomically,
      * so anyone else who wants to make sure flush doesn't interfere should as well.
      */
     private final Tracker data;
@@ -304,6 +359,7 @@ public enum FlushReason
     /* This is used to generate the next index for a SSTable */
     private final Supplier<? extends SSTableId> sstableIdGenerator;
 
+    private final StorageHandler storageHandler;
     public final SecondaryIndexManager indexManager;
     public final TableViews viewManager;
 
@@ -312,11 +368,12 @@ public enum FlushReason
     private volatile DefaultValue<Integer> maxCompactionThreshold;
     private volatile DefaultValue<Double> crcCheckChance;
 
-    private final CompactionStrategyManager compactionStrategyManager;
+    private final CompactionStrategyFactory strategyFactory;
+    private volatile CompactionStrategyContainer strategyContainer;
 
     private final Directories directories;
 
-    public final TableMetrics metric;
+    public volatile TableMetrics metric;
     public volatile long sampleReadLatencyMicros;
     public volatile long additionalWriteLatencyMicros;
 
@@ -333,6 +390,12 @@ public enum FlushReason
     // Tombtone partitions that ignore the gc_grace_seconds during compaction
     private final Set<DecoratedKey> partitionKeySetIgnoreGcGrace = ConcurrentHashMap.newKeySet();
 
+    /** The local ranges are used by the {@link DiskBoundaryManager} to create the disk boundaries but can also be
+     * used independently. They are created lazily and invalidated whenever {@link this#invalidateLocalRangesAndDiskBoundaries()}
+     * is called.
+     */
+    private volatile SortedLocalRanges localRanges;
+
     @VisibleForTesting
     final DiskBoundaryManager diskBoundaryManager = new DiskBoundaryManager();
     private volatile ShardBoundaries cachedShardBoundaries = null;
@@ -362,6 +425,11 @@ TablePaxosRepairHistory get()
 
     private final PaxosRepairHistoryLoader paxosRepairHistory = new PaxosRepairHistoryLoader();
 
+    // BloomFilterTracker is updated from corresponding {@link SSTableReader}s. Metrics are queried via CFS instance.
+    private final BloomFilterTracker bloomFilterTracker = BloomFilterTracker.createMeterTracker();
+
+    private final RequestTracker requestTracker = RequestTracker.instance;
+
     public static void shutdownPostFlushExecutor() throws InterruptedException
     {
         postFlushExecutor.shutdown();
@@ -376,6 +444,11 @@ public static void shutdownExecutorsAndWait(long timeout, TimeUnit unit) throws
         ExecutorUtils.shutdownAndWait(timeout, unit, executors);
     }
 
+    public boolean isReadyToServeData()
+    {
+        return storageHandler.isReady();
+    }
+
     public void reload()
     {
         // metadata object has been mutated directly. make all the members jibe with new settings.
@@ -383,7 +456,7 @@ public void reload()
         // only update these runtime-modifiable settings if they have not been modified.
         if (!minCompactionThreshold.isModified())
             for (ColumnFamilyStore cfs : concatWithIndexes())
-                cfs.minCompactionThreshold = new DefaultValue(metadata().params.compaction.minCompactionThreshold());
+                cfs.minCompactionThreshold = new DefaultValue<>(metadata().params.compaction.minCompactionThreshold());
         if (!maxCompactionThreshold.isModified())
             for (ColumnFamilyStore cfs : concatWithIndexes())
                 cfs.maxCompactionThreshold = new DefaultValue(metadata().params.compaction.maxCompactionThreshold());
@@ -391,42 +464,72 @@ public void reload()
             for (ColumnFamilyStore cfs : concatWithIndexes())
                 cfs.crcCheckChance = new DefaultValue(metadata().params.crcCheckChance);
 
-        compactionStrategyManager.maybeReloadParamsFromSchema(metadata().params.compaction);
+        reloadCompactionStrategy(metadata().params.compaction, CompactionStrategyContainer.ReloadReason.METADATA_CHANGE);
 
         indexManager.reload();
 
         memtableFactory = metadata().params.memtable.factory();
 
-        if (DatabaseDescriptor.isDaemonInitialized())
+        if (DatabaseDescriptor.enableMemtableAndCommitLog())
             switchMemtableOrNotify(FlushReason.SCHEMA_CHANGE, Memtable::metadataUpdated);
+
+        if (metric.metricsAggregation != TableMetrics.MetricsAggregation.fromMetadata(metadata()))
+        { // Reload the metrics if histogram aggregation has changed
+            metric.release(); // release first because of those static tables containing metric names
+            metric = new TableMetrics(this, memtableFactory.createMemtableMetrics(metadata));
+        }
+    }
+
+    /**
+     * Reload the compaction strategy using the given compaction parameters and reason.
+     */
+    private void reloadCompactionStrategy(CompactionParams compactionParams, CompactionStrategyContainer.ReloadReason reason)
+    {
+        CompactionStrategyContainer previous = strategyContainer;
+        strategyContainer = strategyFactory.reload(strategyContainer, compactionParams, reason, storageHandler.enableAutoCompaction());
+        if (strategyContainer != previous)
+        {
+            getTracker().subscribe(strategyContainer);
+            if (previous != null)
+                getTracker().unsubscribe(previous);
+        }
     }
 
     public static Runnable getBackgroundCompactionTaskSubmitter()
     {
-        return () -> {
-            for (Keyspace keyspace : Keyspace.all())
-                for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
-                    CompactionManager.instance.submitBackground(cfs);
-        };
+        return () -> CompactionManager.instance.submitBackground(ImmutableSet.copyOf(all()));
+    }
+
+    @VisibleForTesting
+    public CompactionStrategyFactory getCompactionFactory()
+    {
+        return strategyFactory;
+    }
+
+    @Override
+    public CompactionParams getCompactionParams()
+    {
+        return strategyContainer.getCompactionParams();
     }
 
+    @Override
     public Map<String, String> getCompactionParameters()
     {
-        return compactionStrategyManager.getCompactionParams().asMap();
+        return getCompactionParams().asMap();
     }
 
+    @Override
     public String getCompactionParametersJson()
     {
         return JsonUtils.writeAsJsonString(getCompactionParameters());
     }
 
+    @Override
     public void setCompactionParameters(Map<String, String> options)
     {
         try
         {
-            CompactionParams compactionParams = CompactionParams.fromMap(options);
-            compactionParams.validate();
-            compactionStrategyManager.overrideLocalParams(compactionParams);
+            reloadCompactionStrategy(CompactionParams.fromMap(options), CompactionStrategyContainer.ReloadReason.JMX_REQUEST);
         }
         catch (Throwable t)
         {
@@ -436,21 +539,25 @@ public void setCompactionParameters(Map<String, String> options)
         }
     }
 
+    @Override
     public void setCompactionParametersJson(String options)
     {
         setCompactionParameters(JsonUtils.fromJsonMap(options));
     }
 
+    @Override
     public Map<String,String> getCompressionParameters()
     {
         return metadata.getLocal().params.compression.asMap();
     }
 
+    @Override
     public String getCompressionParametersJson()
     {
         return JsonUtils.writeAsJsonString(getCompressionParameters());
     }
 
+    @Override
     public void setCompressionParameters(Map<String,String> opts)
     {
         try
@@ -465,6 +572,7 @@ public void setCompressionParameters(Map<String,String> opts)
         }
     }
 
+    @Override
     public void setCompressionParametersJson(String options)
     {
         setCompressionParameters(JsonUtils.fromJsonMap(options));
@@ -485,7 +593,6 @@ public ColumnFamilyStore(Keyspace keyspace,
 
         this.keyspace = keyspace;
         this.metadata = metadata;
-        this.directories = directories;
         name = columnFamilyName;
         minCompactionThreshold = new DefaultValue<>(metadata.get().params.compaction.minCompactionThreshold());
         maxCompactionThreshold = new DefaultValue<>(metadata.get().params.compaction.maxCompactionThreshold());
@@ -496,38 +603,57 @@ public ColumnFamilyStore(Keyspace keyspace,
         additionalWriteLatencyMicros = DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.MICROSECONDS) / 2;
         memtableFactory = metadata.get().params.memtable.factory();
 
-        logger.info("Initializing {}.{}", getKeyspaceName(), name);
+        logger.debug("Initializing {}.{}", getKeyspaceName(), name);
 
         // Create Memtable and its metrics object only on online
         Memtable initialMemtable = null;
         TableMetrics.ReleasableMetric memtableMetrics = null;
-        if (DatabaseDescriptor.isDaemonInitialized())
+        if (DatabaseDescriptor.enableMemtableAndCommitLog())
         {
             initialMemtable = createMemtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()));
             memtableMetrics = memtableFactory.createMemtableMetrics(metadata);
+            data = new Tracker(this, initialMemtable, loadSSTables);
+        }
+        else
+        {
+            data = new Tracker(this, null, false);
         }
-        data = new Tracker(this, initialMemtable, loadSSTables);
 
         // Note that this needs to happen before we load the first sstables, or the global sstable tracker will not
         // be notified on the initial loading.
         data.subscribe(StorageService.instance.sstablesTracker);
 
+        /**
+         * When creating a CFS offline we change the default logic needed by CASSANDRA-8671
+         * and link the passed directories to be picked up by the compaction strategy
+         */
+        if (offline)
+            this.directories = directories;
+        else
+            this.directories = new Directories(metadata.get());
+
+        storageHandler = StorageHandler.create(this, metadata, directories, data);
+        logger.debug("Initialized storage handler with {} for {}.{}", storageHandler.getClass().getSimpleName(), keyspace.getName(), name);
+
         Collection<SSTableReader> sstables = null;
         // scan for sstables corresponding to this cf and load them
-        if (data.loadsstables)
-        {
-            Directories.SSTableLister sstableFiles = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
-            sstables = SSTableReader.openAll(this, sstableFiles.list().entrySet(), metadata);
-            data.addInitialSSTablesWithoutUpdatingSize(sstables);
-        }
+        if (loadSSTables)
+            sstables = storageHandler.loadInitialSSTables();
 
         // compaction strategy should be created after the CFS has been prepared
-        compactionStrategyManager = new CompactionStrategyManager(this);
-
-        if (maxCompactionThreshold.value() <= 0 || minCompactionThreshold.value() <=0)
+        this.strategyFactory = new CompactionStrategyFactory(this);
+        this.strategyContainer = strategyFactory.reload(null,
+                                                        metadata.get().params.compaction,
+                                                        CompactionStrategyContainer.ReloadReason.FULL,
+                                                        storageHandler.enableAutoCompaction());
+        getTracker().subscribe(strategyContainer);
+
+        if (!strategyContainer.isEnabled() || DISABLED_AUTO_COMPACTION_PROPERTY.getBoolean())
         {
-            logger.warn("Disabling compaction strategy by setting compaction thresholds to 0 is deprecated, set the compaction option 'enabled' to 'false' instead.");
-            this.compactionStrategyManager.disable();
+            logger.info("Strategy driven background compactions for {} are disabled: strategy container={}, {}={}",
+                        metadata, strategyContainer.isEnabled(), DISABLED_AUTO_COMPACTION_PROPERTY.getKey(),
+                        DISABLED_AUTO_COMPACTION_PROPERTY.getBoolean());
+            this.strategyContainer.disable();
         }
 
         // create the private ColumnFamilyStores for the secondary column indexes
@@ -539,7 +665,7 @@ public ColumnFamilyStore(Keyspace keyspace,
 
         metric = new TableMetrics(this, memtableMetrics);
 
-        if (data.loadsstables)
+        if (data.loadsstables && sstables != null)
         {
             data.updateInitialSSTableSize(sstables);
         }
@@ -588,8 +714,8 @@ public void updateSpeculationThreshold()
     {
         try
         {
-            sampleReadLatencyMicros = metadata().params.speculativeRetry.calculateThreshold(metric.coordinatorReadLatency, sampleReadLatencyMicros);
-            additionalWriteLatencyMicros = metadata().params.additionalWritePolicy.calculateThreshold(metric.coordinatorWriteLatency, additionalWriteLatencyMicros);
+            sampleReadLatencyMicros = metadata().params.speculativeRetry.calculateThreshold(metric.coordinatorReadLatency.tableOrKeyspaceTimer(), sampleReadLatencyMicros);
+            additionalWriteLatencyMicros = metadata().params.additionalWritePolicy.calculateThreshold(metric.coordinatorWriteLatency.tableOrKeyspaceTimer(), additionalWriteLatencyMicros);
         }
         catch (Throwable e)
         {
@@ -612,16 +738,46 @@ public TableRepairManager getRepairManager()
         return repairManager;
     }
 
+    @Override
+    public Environment makeUCSEnvironment()
+    {
+        return new RealEnvironment(this);
+    }
+
     public TableMetadata metadata()
     {
         return metadata.get();
     }
 
+    @Override
+    public TableMetadataRef metadataRef()
+    {
+        return metadata;
+    }
+
+    @Override
+    public TableMetrics metrics()
+    {
+        return metric;
+    }
+
+    @Override
+    public AbstractReplicationStrategy getKeyspaceReplicationStrategy()
+    {
+        return keyspace.getReplicationStrategy();
+    }
+
+    @Override
     public Directories getDirectories()
     {
         return directories;
     }
 
+    public StorageHandler getStorageHandler()
+    {
+        return storageHandler;
+    }
+
     @Override
     public List<String> getDataPaths() throws IOException
     {
@@ -656,7 +812,7 @@ public boolean streamFromMemtable()
 
     public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
     {
-        return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, null, 0, header, lifecycleNewTracker);
+        return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, IntervalSet.empty(), 0, header, lifecycleNewTracker);
     }
 
     public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, IntervalSet<CommitLogPosition> commitLogPositions, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
@@ -666,12 +822,13 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long k
 
     public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, IntervalSet<CommitLogPosition> commitLogPositions, int sstableLevel, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
     {
-        return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, commitLogPositions, sstableLevel, header, indexManager.listIndexGroups(), lifecycleNewTracker);
+        return getCompactionStrategy().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, commitLogPositions, sstableLevel, header, indexManager.listIndexGroups(), lifecycleNewTracker);
     }
 
+    @Override
     public boolean supportsEarlyOpen()
     {
-        return compactionStrategyManager.supportsEarlyOpen();
+        return strategyContainer.supportsEarlyOpen();
     }
 
     /** call when dropping or renaming a CF. Performs mbean housekeeping and invalidates CFS to other operations */
@@ -687,8 +844,14 @@ public void invalidate(boolean expectMBean)
 
     public void invalidate(boolean expectMBean, boolean dropData)
     {
+        if (logger.isTraceEnabled())
+        {
+            logger.trace("Invalidating CFS {}, status: {}, expectMBean: {}, dropData: {}",
+                         metadata.name, status, expectMBean, dropData);
+        }
+
         // disable and cancel in-progress compactions before invalidating
-        valid = false;
+        status = dropData ? STATUS.INVALID_DROPPED : STATUS.INVALID_UNLOADED;
 
         try
         {
@@ -704,20 +867,41 @@ public void invalidate(boolean expectMBean, boolean dropData)
             }
         }
 
-        compactionStrategyManager.shutdown();
+        strategyContainer.shutdown();
 
         // Do not remove truncation records for index CFs, given they have the same ID as their backing/base tables.
         if (!metadata.get().isIndex())
             SystemKeyspace.removeTruncationRecord(metadata.id);
 
-        if (dropData)
-        {
-            data.dropSSTables();
-            LifecycleTransaction.waitForDeletions();
-        }
-        indexManager.dropAllIndexes(dropData);
+        storageHandler.runWithReloadingDisabled(() -> {
+            if (status.isInvalidAndShouldDropData())
+            {
+                data.dropSSTables();
+
+                indexManager.dropAllIndexes(dropData);
+            }
+            else
+            {
+                // In CNDB, because of multi-tenancy, we might just unload a CFS without deleting the data as
+                // a tenant can be moved to a different set of nodes, which will then need to read data from remote storage
+                data.unloadSSTables();
+
+                indexManager.unloadAllIndexes();
+            }
+
+            storageHandler.unload();
+
+            // wait for sstable GlobalTidy to complete
+            if (!status.isValid())
+            {
+                LifecycleTransaction.waitForDeletions(); // just in case an index had a reference on the sstable
+            }
+        });
 
         invalidateCaches();
+        if (logger.isTraceEnabled())
+            logger.trace("CFS {} invalidated", metadata.name);
+
         if (topPartitions != null)
             topPartitions.close();
     }
@@ -817,7 +1001,7 @@ public static void  scrubDataDirectories(TableMetadata metadata) throws StartupE
 
         // cleanup incomplete saved caches
         Pattern tmpCacheFilePattern = Pattern.compile(metadata.keyspace + '-' + metadata.name + "-(Key|Row)Cache.*\\.tmp$");
-        File dir = new File(DatabaseDescriptor.getSavedCachesLocation());
+        File dir = DatabaseDescriptor.getSavedCachesLocation();
 
         if (dir.exists())
         {
@@ -852,6 +1036,7 @@ public static void loadNewSSTables(String ksName, String cfName)
 
     /** @deprecated See CASSANDRA-6719 */
     @Deprecated(since = "4.0")
+    @Override
     public void loadNewSSTables()
     {
 
@@ -859,10 +1044,11 @@ public void loadNewSSTables()
         sstableImporter.importNewSSTables(options);
     }
 
+    /**
+     * #{@inheritDoc}
+     */
     @Override
-    public List<String> importNewSSTables(Set<String> srcPaths, boolean resetLevel, boolean clearRepaired,
-                                          boolean verifySSTables, boolean verifyTokens, boolean invalidateCaches,
-                                          boolean extendedVerify, boolean copyData)
+    public synchronized List<String> importNewSSTables(Set<String> srcPaths, boolean resetLevel, boolean clearRepaired, boolean verifySSTables, boolean verifyTokens, boolean invalidateCaches, boolean extendedVerify, boolean copyData)
     {
         return sstableImporter.importNewSSTables(SSTableImporter.Options.options(srcPaths)
                                                                         .resetLevel(resetLevel)
@@ -875,9 +1061,7 @@ public List<String> importNewSSTables(Set<String> srcPaths, boolean resetLevel,
     }
 
     @Override
-    public List<String> importNewSSTables(Set<String> srcPaths, boolean resetLevel, boolean clearRepaired,
-                                          boolean verifySSTables, boolean verifyTokens, boolean invalidateCaches,
-                                          boolean extendedVerify)
+    public List<String> importNewSSTables(Set<String> srcPaths, boolean resetLevel, boolean clearRepaired, boolean verifySSTables, boolean verifyTokens, boolean invalidateCaches, boolean extendedVerify)
     {
         return sstableImporter.importNewSSTables(SSTableImporter.Options.options(srcPaths)
                                                                         .resetLevel(resetLevel)
@@ -938,37 +1122,32 @@ public static void rebuildSecondaryIndex(String ksName, String cfName, String...
         cfs.indexManager.rebuildIndexesBlocking(Sets.newHashSet(Arrays.asList(idxNames)));
     }
 
-    public AbstractCompactionStrategy createCompactionStrategyInstance(CompactionParams compactionParams)
-    {
-        try
-        {
-            Constructor<? extends AbstractCompactionStrategy> constructor =
-                compactionParams.klass().getConstructor(ColumnFamilyStore.class, Map.class);
-            return constructor.newInstance(this, compactionParams.options());
-        }
-        catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e)
-        {
-            throw new RuntimeException(e);
-        }
-    }
-
     /** @deprecated See CASSANDRA-9448 */
     @Deprecated(since = "3.0")
+    @Override
     public String getColumnFamilyName()
     {
         return getTableName();
     }
 
+    @Override
     public String getTableName()
     {
         return name;
     }
 
+    @Override
     public String getKeyspaceName()
     {
         return keyspace.getName();
     }
 
+    public KeyspaceMetrics getKeyspaceMetrics()
+    {
+        return keyspace.metric;
+    }
+
+    @Override
     public Descriptor newSSTableDescriptor(File directory)
     {
         return newSSTableDescriptor(directory, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion());
@@ -996,11 +1175,14 @@ public Descriptor newSSTableDescriptor(File directory, Version version)
      */
     private void switchMemtableOrNotify(FlushReason reason, Consumer<Memtable> elseNotify)
     {
-        Memtable currentMemtable = data.getView().getCurrentMemtable();
-        if (currentMemtable.shouldSwitch(reason))
-            switchMemtableIfCurrent(currentMemtable, reason);
-        else
-            elseNotify.accept(currentMemtable);
+        if (!data.getView().liveMemtables.isEmpty())
+        {
+            Memtable currentMemtable = data.getView().getCurrentMemtable();
+            if (currentMemtable.shouldSwitch(reason))
+                switchMemtableIfCurrent(currentMemtable, reason);
+            else
+                elseNotify.accept(currentMemtable);
+        }
     }
 
     /**
@@ -1049,7 +1231,7 @@ private void logFlush(FlushReason reason)
         for (ColumnFamilyStore indexCfs : indexManager.getAllIndexColumnFamilyStores())
             indexCfs.getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage);
 
-        logger.info("Enqueuing flush of {}.{}, Reason: {}, Usage: {}", getKeyspaceName(), name, reason, usage);
+        logger.debug("Enqueuing flush of {}.{}, Reason: {}, Usage: {}", getKeyspaceName(), name, reason, usage);
     }
 
 
@@ -1063,11 +1245,18 @@ public Future<CommitLogPosition> forceFlush(FlushReason reason)
     {
         synchronized (data)
         {
-            Memtable current = data.getView().getCurrentMemtable();
-            for (ColumnFamilyStore cfs : concatWithIndexes())
-                if (!cfs.data.getView().getCurrentMemtable().isClean())
-                    return flushMemtable(current, reason);
-            return waitForFlushes();
+            if (!data.getView().liveMemtables.isEmpty())
+            {
+                Memtable current = data.getView().getCurrentMemtable();
+                for (ColumnFamilyStore cfs : concatWithIndexes())
+                    if (!cfs.data.getView().getCurrentMemtable().isClean())
+                        return flushMemtable(current, reason);
+                return waitForFlushes();
+            }
+            else
+            {
+                return ImmediateFuture.success(CommitLogPosition.NONE);
+            }
         }
     }
 
@@ -1128,6 +1317,7 @@ private PostFlush(Memtable mainMemtable)
             this.mainMemtable = mainMemtable;
         }
 
+        @Override
         public CommitLogPosition call()
         {
             try
@@ -1223,6 +1413,7 @@ private Flush(boolean truncate)
             postFlushTask = new FutureTask<>(postFlush);
         }
 
+        @Override
         public void run()
         {
             if (logger.isTraceEnabled())
@@ -1277,17 +1468,24 @@ public Collection<SSTableReader> flushMemtable(ColumnFamilyStore cfs, Memtable m
 
             if (memtable.isClean() || truncate)
             {
-                cfs.replaceFlushed(memtable, Collections.emptyList());
-                reclaim(memtable);
-                return Collections.emptyList();
+                try
+                {
+                    cfs.replaceFlushed(memtable, Collections.emptyList(), Optional.empty());
+                    return Collections.emptyList();
+                }
+                finally
+                {
+                    if (!cfs.getTracker().getView().flushingMemtables.contains(memtable))
+                        reclaim(memtable);
+                }
             }
-
+            long start = Clock.Global.nanoTime();
             List<Future<SSTableMultiWriter>> futures = new ArrayList<>();
             long totalBytesOnDisk = 0;
             long maxBytesOnDisk = 0;
             long minBytesOnDisk = Long.MAX_VALUE;
             List<SSTableReader> sstables = new ArrayList<>();
-            try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH))
+            try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH, metadata))
             {
                 List<Flushing.FlushRunnable> flushRunnables = null;
                 List<SSTableMultiWriter> flushResults = null;
@@ -1310,12 +1508,25 @@ public Collection<SSTableReader> flushMemtable(ColumnFamilyStore cfs, Memtable m
                     if (flushNonCf2i)
                         indexManager.flushAllNonCFSBackedIndexesBlocking(memtable);
 
+                    // It may be worthwhile to add an early abort mechanism here if one of the futures throws.
+                    // In such a case this code will run the other threads to completion and only then abort the operation.
                     flushResults = Lists.newArrayList(FBUtilities.waitOnFutures(futures));
                 }
                 catch (Throwable t)
                 {
+                    logger.error("Flushing {} failed with error", memtable.toString(), t);
                     t = Flushing.abortRunnables(flushRunnables, t);
+
+                    // wait for any flush runnables that were submitted (after aborting they should complete immediately)
+                    // this ensures that the writers are aborted by FlushRunnable.writeSortedContents(), in the worst
+                    // case we'll repeat the same exception twice if the initial exception was thrown whilst waiting
+                    // on a future
+                    t = perform(t, () -> FBUtilities.waitOnFutures(futures));
+
+                    //finally abort the transaction
                     t = txn.abort(t);
+
+                    // and re-throw
                     Throwables.throwIfUnchecked(t);
                     throw new RuntimeException(t);
                 }
@@ -1328,7 +1539,7 @@ public Collection<SSTableReader> flushMemtable(ColumnFamilyStore cfs, Memtable m
                         SSTableMultiWriter writer = writerIterator.next();
                         if (writer.getBytesWritten() > 0)
                         {
-                            writer.setOpenResult(true).prepareToCommit();
+                            writer.prepareToCommit();
                         }
                         else
                         {
@@ -1336,6 +1547,15 @@ public Collection<SSTableReader> flushMemtable(ColumnFamilyStore cfs, Memtable m
                             writerIterator.remove();
                         }
                     }
+
+                    // This can throw on remote storage, e.g. if a file cannot be uploaded
+                    txn.prepareToCommit();
+
+                    // Open the underlying readers, the one that will be returne below by `finished()`.
+                    // Currently needs to be called before commit, because committing will close a certain number
+                    // of resources used by the writers which are accessed to open the readers.
+                    for (SSTableMultiWriter writer : flushResults)
+                        writer.openResult();
                 }
                 catch (Throwable t)
                 {
@@ -1346,13 +1566,12 @@ public Collection<SSTableReader> flushMemtable(ColumnFamilyStore cfs, Memtable m
                     throw new RuntimeException(t);
                 }
 
-                txn.prepareToCommit();
-
                 Throwable accumulate = null;
+
                 for (SSTableMultiWriter writer : flushResults)
                 {
                     accumulate = writer.commit(accumulate);
-                    metric.flushSizeOnDisk.update(writer.getOnDiskBytesWritten());
+                    metric.flushSizeOnDisk().update(writer.getOnDiskBytesWritten());
                 }
 
                 maybeFail(txn.commit(accumulate));
@@ -1372,16 +1591,25 @@ public Collection<SSTableReader> flushMemtable(ColumnFamilyStore cfs, Memtable m
                         }
                     }
                 }
+                metric.memTableFlushCompleted(Clock.Global.nanoTime() - start);
+
+                cfs.replaceFlushed(memtable, sstables, Optional.of(txn.opId()));
+            }
+            finally
+            {
+                if (!cfs.getTracker().getView().flushingMemtables.contains(memtable))
+                    reclaim(memtable);
+            }
+            cfs.strategyFactory.getCompactionLogger().flush(sstables);
+            if (logger.isTraceEnabled())
+            {
+                logger.trace("Flushed to {} ({} sstables, {}), biggest {}, smallest {}",
+                             sstables,
+                             sstables.size(),
+                             FBUtilities.prettyPrintMemory(totalBytesOnDisk),
+                             FBUtilities.prettyPrintMemory(maxBytesOnDisk),
+                             FBUtilities.prettyPrintMemory(minBytesOnDisk));
             }
-            cfs.replaceFlushed(memtable, sstables);
-            reclaim(memtable);
-            cfs.compactionStrategyManager.compactionLogger.flush(sstables);
-            logger.debug("Flushed to {} ({} sstables, {}), biggest {}, smallest {}",
-                         sstables,
-                         sstables.size(),
-                         FBUtilities.prettyPrintMemory(totalBytesOnDisk),
-                         FBUtilities.prettyPrintMemory(maxBytesOnDisk),
-                         FBUtilities.prettyPrintMemory(minBytesOnDisk));
             return sstables;
         }
 
@@ -1392,6 +1620,7 @@ private void reclaim(final Memtable memtable)
             readBarrier.issue();
             postFlushTask.addListener(new WrappedRunnable()
             {
+                @Override
                 public void runMayThrow()
                 {
                     readBarrier.await();
@@ -1454,6 +1683,12 @@ public Iterable<Memtable> getIndexMemtables()
                                    cfs -> cfs.getTracker().getView().getCurrentMemtable());
     }
 
+    @Override
+    public SecondaryIndexManager getIndexManager()
+    {
+        return indexManager;
+    }
+
     /**
      * Insert/Update the column family for this key.
      * Caller is responsible for acquiring Keyspace.switchLock
@@ -1475,8 +1710,10 @@ public void apply(PartitionUpdate update, CassandraWriteContext context, boolean
             DecoratedKey key = update.partitionKey();
             invalidateCachedPartition(key);
             metric.topWritePartitionFrequency.addSample(key.getKey(), 1);
+            int dataSize = update.dataSize();
             if (metric.topWritePartitionSize.isEnabled()) // dont compute datasize if not needed
-                metric.topWritePartitionSize.addSample(key.getKey(), update.dataSize());
+                metric.topWritePartitionSize.addSample(key.getKey(), dataSize);
+            metric.bytesInserted.inc(dataSize);
             StorageHook.instance.reportWrite(metadata.id, update);
             metric.writeLatency.addNano(nanoTime() - start);
             // CASSANDRA-11117 - certain resolution paths on memtable put can result in very
@@ -1486,18 +1723,32 @@ public void apply(PartitionUpdate update, CassandraWriteContext context, boolean
             // to update.
             if(timeDelta < Long.MAX_VALUE)
                 metric.colUpdateTimeDeltaHistogram.update(Math.min(18165375903306L, timeDelta));
+
+            if (!isIndex())
+            {
+                RequestSensors sensors = requestTracker.get();
+                if (sensors != null)
+                {
+                    Context puContext = Context.from(this.metadata.get());
+                    sensors.registerSensor(puContext, Type.WRITE_BYTES);
+                    sensors.incrementSensor(puContext, Type.WRITE_BYTES, dataSize);
+                }
+            }
         }
         catch (RuntimeException e)
         {
-            String message = e.getMessage() + " for ks: " + keyspace.getName() + ", table: " + name;
-
             if (e instanceof InvalidRequestException)
-                throw new InvalidRequestException(message, e);
-
-            throw new RuntimeException(message, e);
+            {
+                throw new InvalidRequestException(e.getMessage()
+                                                  + " for ks: "
+                                                  + keyspace.getName() + ", table: " + name, e);
+            }
+            throw new RuntimeException(e.getMessage()
+                                       + " for ks: "
+                                       + getKeyspaceName() + ", table: " + name, e);
         }
     }
-    
+
     private UpdateTransaction newUpdateTransaction(PartitionUpdate update, CassandraWriteContext context, boolean updateIndexes, Memtable memtable)
     {
         return updateIndexes
@@ -1505,52 +1756,6 @@ private UpdateTransaction newUpdateTransaction(PartitionUpdate update, Cassandra
                : UpdateTransaction.NO_OP;
     }
 
-    public static class VersionedLocalRanges extends ArrayList<Splitter.WeightedRange>
-    {
-        public final long ringVersion;
-
-        public VersionedLocalRanges(long ringVersion, int initialSize)
-        {
-            super(initialSize);
-            this.ringVersion = ringVersion;
-        }
-    }
-
-    public VersionedLocalRanges localRangesWeighted()
-    {
-        if (!SchemaConstants.isLocalSystemKeyspace(getKeyspaceName())
-            && getPartitioner() == StorageService.instance.getTokenMetadata().partitioner)
-        {
-            DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
-            Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
-            long ringVersion = versionedLocalRanges.ringVersion;
-
-            if (!localRanges.isEmpty())
-            {
-                VersionedLocalRanges weightedRanges = new VersionedLocalRanges(ringVersion, localRanges.size());
-                for (Range<Token> r : localRanges)
-                {
-                    // WeightedRange supports only unwrapped ranges as it relies
-                    // on right - left == num tokens equality
-                    for (Range<Token> u: r.unwrap())
-                        weightedRanges.add(new Splitter.WeightedRange(1.0, u));
-                }
-                weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
-                return weightedRanges;
-            }
-            else
-            {
-                return fullWeightedRange(ringVersion, getPartitioner());
-            }
-        }
-        else
-        {
-            // Local tables need to cover the full token range and don't care about ring changes.
-            // We also end up here if the table's partitioner is not the database's, which can happen in tests.
-            return fullWeightedRange(RING_VERSION_IRRELEVANT, getPartitioner());
-        }
-    }
-
     @Override
     public ShardBoundaries localRangeSplits(int shardCount)
     {
@@ -1562,35 +1767,28 @@ public ShardBoundaries localRangeSplits(int shardCount)
         if (shardBoundaries == null ||
             shardBoundaries.shardCount() != shardCount ||
             (shardBoundaries.ringVersion != RING_VERSION_IRRELEVANT &&
-             shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion()))
+             shardBoundaries.ringVersion != keyspace.getReplicationStrategy().getTokenMetadata().getRingVersion()))
         {
-            VersionedLocalRanges weightedRanges = localRangesWeighted();
-
-            List<Token> boundaries = getPartitioner().splitter().get().splitOwnedRanges(shardCount, weightedRanges, false);
-            shardBoundaries = new ShardBoundaries(boundaries.subList(0, boundaries.size() - 1),
-                                                  weightedRanges.ringVersion);
+            SortedLocalRanges localRanges = getLocalRanges();
+            List<Token> positions = localRanges.split(shardCount);
+            shardBoundaries = new ShardBoundaries(positions.subList(0, positions.size() - 1),
+                                                  localRanges.getRingVersion());
             cachedShardBoundaries = shardBoundaries;
-            logger.debug("Memtable shard boundaries for {}.{}: {}", getKeyspaceName(), getTableName(), boundaries);
+            logger.debug("Memtable shard boundaries for {}.{}: {}", keyspace.getName(), getTableName(), positions);
         }
         return shardBoundaries;
     }
 
-    @VisibleForTesting
-    public static VersionedLocalRanges fullWeightedRange(long ringVersion, IPartitioner partitioner)
-    {
-        VersionedLocalRanges ranges = new VersionedLocalRanges(ringVersion, 1);
-        ranges.add(new Splitter.WeightedRange(1.0, new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken())));
-        return ranges;
-    }
-
     /**
      * @param sstables
      * @return sstables whose key range overlaps with that of the given sstables, not including itself.
      * (The given sstables may or may not overlap with each other.)
      */
-    public Collection<SSTableReader> getOverlappingLiveSSTables(Iterable<SSTableReader> sstables)
+    @Override
+    public Set<SSTableReader> getOverlappingLiveSSTables(Iterable<? extends CompactionSSTable> sstables)
     {
-        logger.trace("Checking for sstables overlapping {}", sstables);
+        if (logger.isTraceEnabled())
+            logger.trace("Checking for sstables overlapping {}", sstables);
 
         // a normal compaction won't ever have an empty sstables list, but we create a skeleton
         // compaction controller for streaming, and that passes an empty list.
@@ -1599,11 +1797,11 @@ public Collection<SSTableReader> getOverlappingLiveSSTables(Iterable<SSTableRead
 
         View view = data.getView();
 
-        List<SSTableReader> sortedByFirst = Lists.newArrayList(sstables);
-        sortedByFirst.sort(SSTableReader.firstKeyComparator);
+        List<CompactionSSTable> sortedByFirst = Lists.newArrayList(sstables);
+        sortedByFirst.sort(CompactionSSTable.firstKeyComparator);
 
         List<AbstractBounds<PartitionPosition>> bounds = new ArrayList<>();
-        DecoratedKey first = null, last = null;
+        PartitionPosition first = null, last = null;
         /*
         normalize the intervals covered by the sstables
         assume we have sstables like this (brackets representing first/last key in the sstable);
@@ -1613,7 +1811,7 @@ public Collection<SSTableReader> getOverlappingLiveSSTables(Iterable<SSTableRead
         only query the tree 2 times, for these intervals;
         [         ]    [          ]
          */
-        for (SSTableReader sstable : sortedByFirst)
+        for (CompactionSSTable sstable : sortedByFirst)
         {
             if (first == null)
             {
@@ -1636,12 +1834,14 @@ public Collection<SSTableReader> getOverlappingLiveSSTables(Iterable<SSTableRead
             }
         }
         bounds.add(AbstractBounds.bounds(first, true, last, true));
-        Set<SSTableReader> results = new HashSet<>();
+        Set<SSTableReader> overlaps = new HashSet<>();
 
         for (AbstractBounds<PartitionPosition> bound : bounds)
-            Iterables.addAll(results, view.liveSSTablesInBounds(bound.left, bound.right));
+            Iterables.addAll(overlaps, view.liveSSTablesInBounds(bound.left, bound.right));
 
-        return Sets.difference(results, ImmutableSet.copyOf(sstables));
+        for (CompactionSSTable sstable : sstables)
+            overlaps.remove(sstable);
+        return overlaps;
     }
 
     /**
@@ -1651,7 +1851,7 @@ public Refs<SSTableReader> getAndReferenceOverlappingLiveSSTables(Iterable<SSTab
     {
         while (true)
         {
-            Iterable<SSTableReader> overlapped = getOverlappingLiveSSTables(sstables);
+            Set<SSTableReader> overlapped = getOverlappingLiveSSTables(sstables);
             Refs<SSTableReader> refs = Refs.tryRef(overlapped);
             if (refs != null)
                 return refs;
@@ -1667,15 +1867,22 @@ public Refs<SSTableReader> getAndReferenceOverlappingLiveSSTables(Iterable<SSTab
      *
      * param @ filename - filename just flushed to disk
      */
+    @VisibleForTesting
     public void addSSTable(SSTableReader sstable)
     {
         assert sstable.getColumnFamilyName().equals(name);
-        addSSTables(Collections.singletonList(sstable));
+        addSSTables(Collections.singletonList(sstable), OperationType.UNKNOWN);
     }
 
+    @VisibleForTesting
     public void addSSTables(Collection<SSTableReader> sstables)
     {
-        data.addSSTables(sstables);
+        addSSTables(sstables, OperationType.UNKNOWN);
+    }
+
+    public void addSSTables(Collection<SSTableReader> sstables, OperationType operationType)
+    {
+        data.addSSTables(sstables, operationType);
         CompactionManager.instance.submitBackground(this);
     }
 
@@ -1692,11 +1899,12 @@ public void addSSTables(Collection<SSTableReader> sstables)
      * @param operation Operation type
      * @return Expected file size of SSTable after compaction
      */
+    @Override
     public long getExpectedCompactedFileSize(Iterable<SSTableReader> sstables, OperationType operation)
     {
         if (operation != OperationType.CLEANUP || isIndex())
         {
-            return SSTableReader.getTotalBytes(sstables);
+            return CompactionSSTable.getTotalDataBytes(sstables);
         }
 
         // cleanup size estimation only counts bytes for keys local to this node
@@ -1716,24 +1924,6 @@ public long getExpectedCompactedFileSize(Iterable<SSTableReader> sstables, Opera
         return expectedFileSize;
     }
 
-    /*
-     *  Find the maximum size file in the list .
-     */
-    public SSTableReader getMaxSizeFile(Iterable<SSTableReader> sstables)
-    {
-        long maxSize = 0L;
-        SSTableReader maxFile = null;
-        for (SSTableReader sstable : sstables)
-        {
-            if (sstable.onDiskLength() > maxSize)
-            {
-                maxSize = sstable.onDiskLength();
-                maxFile = sstable;
-            }
-        }
-        return maxFile;
-    }
-
     public CompactionManager.AllSSTableOpStatus forceCleanup(int jobs) throws ExecutionException, InterruptedException
     {
         return CompactionManager.instance.performCleanup(ColumnFamilyStore.this, jobs);
@@ -1829,18 +2019,29 @@ public void markObsolete(Collection<SSTableReader> sstables, OperationType compa
         maybeFail(data.dropSSTables(Predicates.in(sstables), compactionType, null));
     }
 
-    void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
+    /**
+     * Beware, this code doesn't have noexcept guarantees
+     */
+    void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables, Optional<TimeUUID> operationId)
     {
-        data.replaceFlushed(memtable, sstables);
+        data.replaceFlushed(memtable, sstables, operationId);
         if (sstables != null && !sstables.isEmpty())
             CompactionManager.instance.submitBackground(this);
     }
 
     public boolean isValid()
     {
-        return valid;
+        return status.isValid();
     }
 
+    /**
+     * @return status of the current column family store
+     */
+    public STATUS status()
+    {
+        return status;
+    }
+    
     /**
      * Package protected for access from the CompactionManager.
      */
@@ -1849,19 +2050,51 @@ public Tracker getTracker()
         return data;
     }
 
+
+    /**
+     * Convenience method for getting the set of live sstables associated with this ColumnFamilyStore. Note that this
+     * will also contain any early-opened sstables.
+     * @return the tracker's current view's {@link SSTableSet#LIVE} sstables
+     */
+    @Override
     public Set<SSTableReader> getLiveSSTables()
     {
-        return data.getView().liveSSTables();
+        return data.getLiveSSTables();
     }
 
+    @Override
     public Iterable<SSTableReader> getSSTables(SSTableSet sstableSet)
     {
         return data.getView().select(sstableSet);
     }
 
-    public Iterable<SSTableReader> getUncompactingSSTables()
+    public Iterable<SSTableReader> getNoncompactingSSTables()
+    {
+        return data.getNoncompacting();
+    }
+
+    @Override
+    public <S extends CompactionSSTable> Iterable<S> getNoncompactingSSTables(Iterable<S> candidates)
     {
-        return data.getUncompacting();
+        return data.getNoncompacting(candidates);
+    }
+
+    @Override
+    public Set<SSTableReader> getCompactingSSTables()
+    {
+        return data.getCompacting();
+    }
+
+    @Override
+    public Iterable<Memtable> getAllMemtables()
+    {
+        return data.getView().getAllMemtables();
+    }
+
+    @Override
+    public OpOrder readOrdering()
+    {
+        return readOrdering;
     }
 
     public Map<TimeUUID, PendingStat> getPendingRepairStats()
@@ -1887,29 +2120,6 @@ public Map<TimeUUID, PendingStat> getPendingRepairStats()
         return stats;
     }
 
-    /**
-     * promotes (or demotes) data attached to an incremental repair session that has either completed successfully,
-     * or failed
-     *
-     * @return session ids whose data could not be released
-     */
-    public CleanupSummary releaseRepairData(Collection<TimeUUID> sessions, boolean force)
-    {
-        if (force)
-        {
-            Predicate<SSTableReader> predicate = sst -> {
-                TimeUUID session = sst.getPendingRepair();
-                return session != null && sessions.contains(session);
-            };
-            return runWithCompactionsDisabled(() -> compactionStrategyManager.releaseRepairData(sessions),
-                                              predicate, OperationType.STREAM, false, true, true);
-        }
-        else
-        {
-            return compactionStrategyManager.releaseRepairData(sessions);
-        }
-    }
-
     public boolean isFilterFullyCoveredBy(ClusteringIndexFilter filter,
                                           DataLimits limits,
                                           CachedPartition cached,
@@ -2003,16 +2213,19 @@ public ViewFragment select(Function<View, Iterable<SSTableReader>> filter)
     }
 
     // WARNING: this returns the set of LIVE sstables only, which may be only partially written
+    @Override
     public List<String> getSSTablesForKey(String key)
     {
         return getSSTablesForKey(key, false);
     }
 
+    @Override
     public List<String> getSSTablesForKey(String key, boolean hexFormat)
     {
         return withSSTablesForKey(key, hexFormat, SSTableReader::getFilename);
     }
 
+    @Override
     public Map<Integer, Set<String>> getSSTablesForKeyWithLevel(String key, boolean hexFormat)
     {
         List<Pair<Integer, String>> ssts = withSSTablesForKey(key, hexFormat, sstr -> Pair.create(sstr.getSSTableLevel(), sstr.getFilename()));
@@ -2118,11 +2331,13 @@ public ClusteringComparator getComparator()
         return metadata().comparator;
     }
 
+    @Override
     public TableSnapshot snapshotWithoutMemtable(String snapshotName)
     {
         return snapshotWithoutMemtable(snapshotName, now());
     }
 
+    @Override
     public TableSnapshot snapshotWithoutMemtable(String snapshotName, Instant creationTime)
     {
         return snapshotWithoutMemtable(snapshotName, null, false, null, null, creationTime);
@@ -2445,6 +2660,7 @@ public void invalidateCachedPartition(RowCacheKey key)
         CacheService.instance.rowCache.remove(key);
     }
 
+    @Override
     public void invalidateCachedPartition(DecoratedKey key)
     {
         if (!isRowCacheEnabled())
@@ -2453,18 +2669,18 @@ public void invalidateCachedPartition(DecoratedKey key)
         invalidateCachedPartition(new RowCacheKey(metadata(), key));
     }
 
-    public ClockAndCount getCachedCounter(ByteBuffer partitionKey, Clustering<?> clustering, ColumnMetadata column, CellPath path)
+    public ClockAndCount getCachedCounter(CounterCacheKey key)
     {
         if (CacheService.instance.counterCache.getCapacity() == 0L) // counter cache disabled.
             return null;
-        return CacheService.instance.counterCache.get(CounterCacheKey.create(metadata(), partitionKey, clustering, column, path));
+        return CacheService.instance.counterCache.get(key);
     }
 
-    public void putCachedCounter(ByteBuffer partitionKey, Clustering<?> clustering, ColumnMetadata column, CellPath path, ClockAndCount clockAndCount)
+    public void putCachedCounter(CounterCacheKey key, ClockAndCount clockAndCount)
     {
         if (CacheService.instance.counterCache.getCapacity() == 0L) // counter cache disabled.
             return;
-        CacheService.instance.counterCache.put(CounterCacheKey.create(metadata(), partitionKey, clustering, column, path), clockAndCount);
+        CacheService.instance.counterCache.put(key, clockAndCount);
     }
 
     public void forceMajorCompaction()
@@ -2472,11 +2688,23 @@ public void forceMajorCompaction()
         forceMajorCompaction(false);
     }
 
+    @Override
     public void forceMajorCompaction(boolean splitOutput)
     {
         CompactionManager.instance.performMaximal(this, splitOutput);
     }
 
+    @Override
+    public void forceMajorCompaction(int parallelism)
+    {
+        CompactionManager.instance.performMaximal(this, false, parallelism);
+    }
+
+    public void forceMajorCompaction(boolean splitOutput, int parallelism)
+    {
+        CompactionManager.instance.performMaximal(this, splitOutput, parallelism);
+    }
+
     @Override
     public void forceCompactionForTokenRange(Collection<Range<Token>> tokenRanges) throws ExecutionException, InterruptedException
     {
@@ -2533,6 +2761,7 @@ public void forceCompactionKeysIgnoringGcGrace(String... partitionKeysIgnoreGcGr
         }
     }
 
+    @Override
     public boolean shouldIgnoreGcGraceForKey(DecoratedKey dk)
     {
         return partitionKeySetIgnoreGcGrace.contains(dk);
@@ -2593,10 +2822,10 @@ public void writeAndAddMemtableRanges(TimeUUID repairSessionID,
                 for (SSTableReader rdr : sstables)
                 {
                     rdr.selfRef().release();
-                    logger.info("Memtable ranges (keys {} size {}) written in {}",
-                                rdr.estimatedKeys(),
-                                rdr.getDataChannel().size(),
-                                rdr);
+                    logger.debug("Memtable ranges (keys {} size {}) written in {}",
+                                 rdr.estimatedKeys(),
+                                 rdr.getDataChannel().size(),
+                                 rdr);
                 }
             }
             catch (Throwable t)
@@ -2661,26 +2890,33 @@ private SSTableMultiWriter writeMemtableRanges(Supplier<Collection<Range<Partiti
 
     private static final LifecycleNewTracker DO_NOT_TRACK = new LifecycleNewTracker()
     {
+        @Override
         public void trackNew(SSTable table)
         {
             // not tracking
         }
 
+        @Override
         public void untrackNew(SSTable table)
         {
             // not tracking
         }
 
+        @Override
         public OperationType opType()
         {
             return OperationType.FLUSH;
         }
     };
 
-    /**
+    /*
      * For testing.  No effort is made to clear historical or even the current memtables, nor for
      * thread safety.  All we do is wipe the sstable containers clean, while leaving the actual
      * data files present on disk.  (This allows tests to easily call loadNewSSTables on them.)
+     *
+     * This will release references to current SSTableReader instances. Test that wishes to keep
+     * references to these sstables must reference then prior to calling this method via
+     * {@link SSTableReader#selfRef().ref()}.
      */
     @VisibleForTesting
     public void clearUnsafe()
@@ -2689,8 +2925,9 @@ public void clearUnsafe()
         {
             cfs.runWithCompactionsDisabled((Callable<Void>) () -> {
                 cfs.data.reset(memtableFactory.create(new AtomicReference<>(CommitLogPosition.NONE), cfs.metadata, cfs));
+                cfs.reloadCompactionStrategy(metadata().params.compaction, CompactionStrategyContainer.ReloadReason.FULL);
                 return null;
-            }, OperationType.P0, true, false);
+            }, OperationType.P0, true, false, TableOperation.StopTrigger.UNIT_TESTS);
         }
     }
 
@@ -2704,6 +2941,12 @@ public void truncateBlockingWithoutSnapshot()
         truncateBlocking(true);
     }
 
+    @FunctionalInterface
+    interface AdaptiveLogger
+    {
+        void log(String template, Object... args);
+    }
+
     /**
      * Truncate deletes the entire column family's data with no expensive tombstone creation
      * @param noSnapshot if {@code true} no snapshot will be taken
@@ -2722,13 +2965,21 @@ private void truncateBlocking(boolean noSnapshot)
         // beginning if we restart before they [the CL segments] are discarded for
         // normal reasons post-truncate.  To prevent this, we store truncation
         // position in the System keyspace.
-        logger.info("Truncating {}.{}", getKeyspaceName(), name);
+        AdaptiveLogger log = truncateLogger();
+
+        log.log("Truncating {}.{}", getKeyspaceName(), name);
 
         viewManager.stopBuild();
 
         final long truncatedAt;
         final CommitLogPosition replayAfter;
 
+        // This is a no-op on local storage, but on remote storage where compaction runs offline, this
+        // ensures that any live sstables created by the compaction process before it was interrupted,
+        // will actually be obsoleted by one of the writers - since all writers must be running for
+        // a truncate to run, then at least one writer per token range will load sstables created by compaction
+        storageHandler.reloadSSTables(StorageHandler.ReloadReason.TRUNCATION);
+
         if (!noSnapshot &&
                ((keyspace.getMetadata().params.durableWrites && !memtableWritesAreDurable())  // need to clear dirty regions
                || isAutoSnapshotEnabled()))
@@ -2754,36 +3005,46 @@ private void truncateBlocking(boolean noSnapshot)
 
         Runnable truncateRunnable = new Runnable()
         {
+            @Override
             public void run()
             {
-                logger.info("Truncating {}.{} with truncatedAt={}", getKeyspaceName(), getTableName(), truncatedAt);
+                log.log("Truncating {}.{} with truncatedAt={}", getKeyspaceName(), getTableName(), truncatedAt);
                 // since truncation can happen at different times on different nodes, we need to make sure
                 // that any repairs are aborted, otherwise we might clear the data on one node and then
                 // stream in data that is actually supposed to have been deleted
                 ActiveRepairService.instance().abort((prs) -> prs.getTableIds().contains(metadata.id),
                                                    "Stopping parent sessions {} due to truncation of tableId="+metadata.id);
-                data.notifyTruncated(truncatedAt);
+                data.notifyTruncated(replayAfter, truncatedAt);
 
-            if (!noSnapshot && isAutoSnapshotEnabled())
-                snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, SNAPSHOT_TRUNCATE_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl());
+                if (!noSnapshot && isAutoSnapshotEnabled())
+                    snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, SNAPSHOT_TRUNCATE_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl());
 
-            discardSSTables(truncatedAt);
+                discardSSTables(truncatedAt);
 
-            indexManager.truncateAllIndexesBlocking(truncatedAt);
-            viewManager.truncateBlocking(replayAfter, truncatedAt);
+                indexManager.truncateAllIndexesBlocking(truncatedAt);
+                viewManager.truncateBlocking(replayAfter, truncatedAt);
 
-                SystemKeyspace.saveTruncationRecord(ColumnFamilyStore.this, truncatedAt, replayAfter);
+                SystemKeyspace.saveTruncationRecord(metadata.id, truncatedAt, replayAfter);
                 logger.trace("cleaning out row cache");
                 invalidateCaches();
 
             }
         };
 
-        runWithCompactionsDisabled(FutureTask.callable(truncateRunnable), OperationType.P0, true, true);
+        storageHandler.runWithReloadingDisabled(() -> {
+            runWithCompactionsDisabled(FutureTask.callable(truncateRunnable), OperationType.P0, true, true, AbstractTableOperation.StopTrigger.TRUNCATE);
+        });
 
         viewManager.build();
+        log.log("Truncate of {}.{} is complete", getKeyspaceName(), name);
+    }
 
-        logger.info("Truncate of {}.{} is complete", getKeyspaceName(), name);
+    private AdaptiveLogger truncateLogger()
+    {
+        if (keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
+            return logger::debug;
+        else
+            return logger::info;
     }
 
     /**
@@ -2810,9 +3071,10 @@ public void unloadCf()
             FBUtilities.waitOnFuture(dumpMemtable());
     }
 
-    public <V> V runWithCompactionsDisabled(Callable<V> callable, OperationType operationType, boolean interruptValidation, boolean interruptViews)
+    @Override
+    public <V> V runWithCompactionsDisabled(Callable<V> callable, OperationType operationType, boolean interruptValidation, boolean interruptViews, TableOperation.StopTrigger trigger)
     {
-        return runWithCompactionsDisabled(callable, (sstable) -> true, operationType, interruptValidation, interruptViews, true);
+        return runWithCompactionsDisabled(callable, (sstable) -> true, operationType, interruptValidation, interruptViews, true, trigger);
     }
 
     /**
@@ -2824,42 +3086,43 @@ public <V> V runWithCompactionsDisabled(Callable<V> callable, OperationType oper
      * @param interruptViews if we should interrupt view compactions
      * @param interruptIndexes if we should interrupt compactions on indexes. NOTE: if you set this to true your sstablePredicate
      *                         must be able to handle LocalPartitioner sstables!
+     * @param trigger the cause for interrupting compactions
      */
-    public <V> V runWithCompactionsDisabled(Callable<V> callable, Predicate<SSTableReader> sstablesPredicate, OperationType operationType, boolean interruptValidation, boolean interruptViews, boolean interruptIndexes)
+    public <V> V runWithCompactionsDisabled(Callable<V> callable,
+                                            Predicate<SSTableReader> sstablesPredicate,
+                                            OperationType operationType, 
+                                            boolean interruptValidation,
+                                            boolean interruptViews,
+                                            boolean interruptIndexes,
+                                            TableOperation.StopTrigger trigger)
     {
         // synchronize so that concurrent invocations don't re-enable compactions partway through unexpectedly,
         // and so we only run one major compaction at a time
         synchronized (this)
         {
             logger.debug("Cancelling in-progress compactions for {}", metadata.name);
-            Iterable<ColumnFamilyStore> toInterruptFor = interruptIndexes
-                                                         ? concatWithIndexes()
-                                                         : Collections.singleton(this);
-
-            toInterruptFor = interruptViews
-                             ? Iterables.concat(toInterruptFor, viewManager.allViewsCfs())
-                             : toInterruptFor;
+            Iterable<ColumnFamilyStore> toInterruptFor = concatWith(interruptIndexes, interruptViews);
 
             Iterable<TableMetadata> toInterruptForMetadata = Iterables.transform(toInterruptFor, ColumnFamilyStore::metadata);
 
             try (CompactionManager.CompactionPauser pause = CompactionManager.instance.pauseGlobalCompaction();
                  CompactionManager.CompactionPauser pausedStrategies = pauseCompactionStrategies(toInterruptFor))
             {
-                List<CompactionInfo.Holder> uninterruptibleTasks = CompactionManager.instance.getCompactionsMatching(toInterruptForMetadata,
-                                                                                                                     (info) -> info.getTaskType().priority <= operationType.priority);
+                List<TableOperation> uninterruptibleTasks = CompactionManager.instance.getCompactionsMatching(toInterruptForMetadata,
+                                                                                                              (progress) -> progress.operationType().priority <= operationType.priority);
                 if (!uninterruptibleTasks.isEmpty())
                 {
                     logger.info("Unable to cancel in-progress compactions, since they're running with higher or same priority: {}. You can abort these operations using `nodetool stop`.",
                                 uninterruptibleTasks.stream().map((compaction) -> String.format("%s@%s (%s)",
-                                                                                                compaction.getCompactionInfo().getTaskType(),
-                                                                                                compaction.getCompactionInfo().getTable(),
-                                                                                                compaction.getCompactionInfo().getTaskId()))
+                                                                                                compaction.getProgress().operationType(),
+                                                                                                compaction.getProgress().metadata().name,
+                                                                                                compaction.getProgress().operationId()))
                                                     .collect(Collectors.joining(",")));
                     return null;
                 }
 
                 // interrupt in-progress compactions
-                CompactionManager.instance.interruptCompactionForCFs(toInterruptFor, sstablesPredicate, interruptValidation);
+                CompactionManager.instance.interruptCompactionForCFs(toInterruptFor, sstablesPredicate, interruptValidation, trigger);
                 CompactionManager.instance.waitForCessation(toInterruptFor, sstablesPredicate);
 
                 // doublecheck that we finished, instead of timing out
@@ -2896,7 +3159,7 @@ private static CompactionManager.CompactionPauser pauseCompactionStrategies(Iter
             for (ColumnFamilyStore cfs : toPause)
             {
                 successfullyPaused.ensureCapacity(successfullyPaused.size() + 1); // to avoid OOM:ing after pausing the strategies
-                cfs.getCompactionStrategyManager().pause();
+                cfs.getCompactionStrategy().pause();
                 successfullyPaused.add(cfs);
             }
             return () -> maybeFail(resumeAll(null, toPause));
@@ -2914,7 +3177,7 @@ private static Throwable resumeAll(Throwable accumulate, Iterable<ColumnFamilySt
         {
             try
             {
-                cfs.getCompactionStrategyManager().resume();
+                cfs.getCompactionStrategy().resume();
             }
             catch (Throwable t)
             {
@@ -2924,18 +3187,17 @@ private static Throwable resumeAll(Throwable accumulate, Iterable<ColumnFamilySt
         return accumulate;
     }
 
-    public <T> T withAllSSTables(final OperationType operationType, Function<LifecycleTransaction, T> op)
+    public <T> T withAllSSTables(final OperationType operationType, TableOperation.StopTrigger trigger, Function<LifecycleTransaction, T> op)
     {
         Callable<LifecycleTransaction> callable = () -> {
             assert data.getCompacting().isEmpty() : data.getCompacting();
-            Iterable<SSTableReader> sstables = getLiveSSTables();
-            sstables = AbstractCompactionStrategy.filterSuspectSSTables(sstables);
+            Iterable<SSTableReader> sstables = Iterables.filter(getLiveSSTables(), sstable -> !sstable.isMarkedSuspect());
             LifecycleTransaction modifier = data.tryModify(sstables, operationType);
             assert modifier != null: "something marked things compacting while compactions are disabled";
             return modifier;
         };
 
-        try (LifecycleTransaction compacting = runWithCompactionsDisabled(callable, operationType, false, false))
+        try (LifecycleTransaction compacting = runWithCompactionsDisabled(callable, operationType, false, false, trigger))
         {
             return op.apply(compacting);
         }
@@ -2944,17 +3206,32 @@ public <T> T withAllSSTables(final OperationType operationType, Function<Lifecyc
     @Override
     public String toString()
     {
-        return "CFS(" +
-               "Keyspace='" + getKeyspaceName() + '\'' +
-               ", ColumnFamily='" + name + '\'' +
-               ')';
+        return String.format("%s.%s", getKeyspaceName(), getTableName());
+    }
+
+    /**
+     * Do not use outside of tests / benchmarks.
+     * Ensures no flushing will run while the given runnable is executing.
+     */
+    @VisibleForTesting
+    public void unsafeRunWithoutFlushing(Runnable r)
+    {
+        synchronized (data)
+        {
+            r.run();
+        }
     }
 
     public void disableAutoCompaction()
     {
         // we don't use CompactionStrategy.pause since we don't want users flipping that on and off
         // during runWithCompactionsDisabled
-        compactionStrategyManager.disable();
+        strategyContainer.disable();
+    }
+
+    public boolean compactionShouldBeEnabled()
+    {
+        return strategyContainer.getCompactionParams().isEnabled();
     }
 
     public void enableAutoCompaction()
@@ -2964,36 +3241,97 @@ public void enableAutoCompaction()
 
     /**
      * used for tests - to be able to check things after a minor compaction
-     * @param waitForFutures if we should block until autocompaction is done
+     * @param waitForFuture if we should block until autocompaction is done
      */
     @VisibleForTesting
-    public void enableAutoCompaction(boolean waitForFutures)
+    public void enableAutoCompaction(boolean waitForFuture)
     {
-        compactionStrategyManager.enable();
-        List<Future<?>> futures = CompactionManager.instance.submitBackground(this);
-        if (waitForFutures)
-            FBUtilities.waitOnFutures(futures);
+        strategyContainer.enable();
+        Future<?> future = CompactionManager.instance.submitBackground(this);
+        if (waitForFuture)
+            FBUtilities.waitOnFuture(future);
     }
 
+    @Override
     public boolean isAutoCompactionDisabled()
     {
-        return !this.compactionStrategyManager.isEnabled();
+        return !this.strategyContainer.isEnabled();
     }
 
-    /*
-     JMX getters and setters for the Default<T>s.
-       - get/set minCompactionThreshold
-       - get/set maxCompactionThreshold
-       - get     memsize
-       - get     memops
-       - get/set memtime
+    public List<SSTableReader> getCandidatesForUpgrade()
+    {
+        Set<SSTableReader> compacting = getTracker().getCompacting();
+        return getLiveSSTables().stream()
+                                .filter(s -> !compacting.contains(s) && !s.descriptor.version.isLatestVersion())
+                                .sorted((o1, o2) -> {
+                                    File f1 = o1.descriptor.fileFor(Components.DATA);
+                                    File f2 = o2.descriptor.fileFor(Components.DATA);
+                                    return Longs.compare(f1.lastModified(), f2.lastModified());
+                                }).collect(Collectors.toList());
+    }
+
+    public SortedLocalRanges getLocalRanges()
+    {
+        synchronized (this)
+        {
+            if (localRanges != null && !localRanges.isOutOfDate())
+                return localRanges;
+
+            localRanges = SortedLocalRanges.create(this);
+            return localRanges;
+        }
+    }
+
+    /**
+     * Return the compaction strategy for this CFS. Even though internally the strategy container
+     * implements the strategy, we would like to just expose {@link CompactionStrategy} externally.
+     * This is not currently possible for the reasons explained in {@link this#getCompactionStrategyContainer()},
+     * so we expose the container as well, but using a separate method, marked as deprecated.
+     *
+     * @return the compaction strategy for this CFS
+     */
+    public CompactionStrategy getCompactionStrategy()
+    {
+        return strategyContainer;
+    }
+
+    /**
+     * The reasons for exposing the compaction strategy container are the following:
+     *
+     * - Unit tests
+     * - Repair
+     *
+     * Eventually we would like to only expose the {@link CompactionStrategy}, so for new code call
+     * {@link this#getCompactionStrategy()} instead.
+     *
+     * @return the compaction strategy container
      */
+    /** @deprecated See STAR-13 */
+    @Deprecated(since = "unknown")
+    @VisibleForTesting
+    public CompactionStrategyContainer getCompactionStrategyContainer()
+    {
+        return strategyContainer;
+    }
 
-    public CompactionStrategyManager getCompactionStrategyManager()
+    /**
+     * This option determines if tombstones should only be removed when the sstable has been repaired.
+     * Because this option was introduced in patch releases (I'm guessing), the compaction parameters were
+     * abused. Eventually this option should be moved out of the compaction parameters. TODO: move it
+     * to the new compaction strategy interface.
+     *
+     * @return true if tombstones can only be removed if the sstable has been repaired
+     */
+    @Override
+    public boolean onlyPurgeRepairedTombstones()
     {
-        return compactionStrategyManager;
+        // Here we need to ask the CSM for the parameters in case they were changed over JMX without changing the schema,
+        // for now the CSM has the up-to-date copy of the params
+        CompactionParams params = strategyContainer.getCompactionParams();
+        return Boolean.parseBoolean(params.options().get(CompactionStrategyOptions.ONLY_PURGE_REPAIRED_TOMBSTONES));
     }
 
+    @Override
     public void setCrcCheckChance(double crcCheckChance)
     {
         try
@@ -3020,6 +3358,7 @@ public Double getCrcCheckChance()
         return crcCheckChance.value();
     }
 
+    @Override
     public void setCompactionThresholds(int minThreshold, int maxThreshold)
     {
         validateCompactionThresholds(minThreshold, maxThreshold);
@@ -3029,22 +3368,26 @@ public void setCompactionThresholds(int minThreshold, int maxThreshold)
         CompactionManager.instance.submitBackground(this);
     }
 
+    @Override
     public int getMinimumCompactionThreshold()
     {
         return minCompactionThreshold.value();
     }
 
+    @Override
     public void setMinimumCompactionThreshold(int minCompactionThreshold)
     {
         validateCompactionThresholds(minCompactionThreshold, maxCompactionThreshold.value());
         this.minCompactionThreshold.set(minCompactionThreshold);
     }
 
+    @Override
     public int getMaximumCompactionThreshold()
     {
         return maxCompactionThreshold.value();
     }
 
+    @Override
     public void setMaximumCompactionThreshold(int maxCompactionThreshold)
     {
         validateCompactionThresholds(minCompactionThreshold.value(), maxCompactionThreshold);
@@ -3064,6 +3407,17 @@ private void validateCompactionThresholds(int minThreshold, int maxThreshold)
 
     // End JMX get/set.
 
+    @Override
+    public boolean isCompactionActive()
+    {
+        return getCompactionStrategyContainer().isActive();
+    }
+
+    public long getMaxSSTableBytes()
+    {
+        return getCompactionStrategy().getMaxSSTableBytes();
+    }
+
     public int getMeanEstimatedCellPerPartitionCount()
     {
         long sum = 0;
@@ -3090,7 +3444,7 @@ public double getMeanPartitionSize()
         return count > 0 ? sum * 1.0 / count : 0;
     }
 
-    public int getMeanRowCount()
+    public int getMeanRowsPerPartition()
     {
         long totalRows = 0;
         long totalPartitions = 0;
@@ -3103,6 +3457,7 @@ public int getMeanRowCount()
         return totalPartitions > 0 ? (int) (totalRows / totalPartitions) : 0;
     }
 
+    @Override
     public long estimateKeys()
     {
         long n = 0;
@@ -3111,6 +3466,7 @@ public long estimateKeys()
         return n;
     }
 
+    @Override
     public IPartitioner getPartitioner()
     {
         return metadata().partitioner;
@@ -3121,6 +3477,52 @@ public DecoratedKey decorateKey(ByteBuffer key)
         return getPartitioner().decorateKey(key);
     }
 
+    @Override
+    public BloomFilterTracker getBloomFilterTracker()
+    {
+        return bloomFilterTracker;
+    }
+
+    public long getBloomFilterFalsePositiveCount()
+    {
+        return bloomFilterTracker.getFalsePositiveCount();
+    }
+
+    public long getBloomFilterTruePositiveCount()
+    {
+        return bloomFilterTracker.getTruePositiveCount();
+    }
+
+    public long getBloomFilterTrueNegativeCount()
+    {
+        return bloomFilterTracker.getTrueNegativeCount();
+    }
+
+    public double getRecentBloomFilterFalsePositiveRate()
+    {
+        return bloomFilterTracker.getRecentFalsePositiveRate();
+    }
+
+    public double getRecentBloomFilterTruePositiveRate()
+    {
+        return bloomFilterTracker.getRecentTruePositiveRate();
+    }
+
+    public double getRecentBloomFilterTrueNegativeRate()
+    {
+        return bloomFilterTracker.getRecentTrueNegativeRate();
+    }
+
+    public long getReadRequests()
+    {
+        return metric == null ? 0 : metric.readRequests.getCount();
+    }
+
+    public long getBytesInserted()
+    {
+        return metric == null ? 0 : metric.bytesInserted.getCount();
+    }
+
     /** true if this CFS contains secondary index data */
     public boolean isIndex()
     {
@@ -3128,12 +3530,27 @@ public boolean isIndex()
     }
 
     public Iterable<ColumnFamilyStore> concatWithIndexes()
+    {
+        return concatWith(true, false);
+    }
+
+    public Iterable<ColumnFamilyStore> concatWith(boolean includeIndexes, boolean includeViews)
     {
         // we return the main CFS first, which we rely on for simplicity in switchMemtable(), for getting the
         // latest commit log segment position
-        return Iterables.concat(Collections.singleton(this), indexManager.getAllIndexColumnFamilyStores());
+        Set<ColumnFamilyStore> mainCFS = Collections.singleton(this);
+        if (includeIndexes && includeViews)
+            return Iterables.concat(mainCFS,
+                                    indexManager.getAllIndexColumnFamilyStores(),
+                                    viewManager.allViewsCfs());
+        if (includeIndexes)
+            return Iterables.concat(mainCFS, indexManager.getAllIndexColumnFamilyStores());
+        if (includeViews)
+            return Iterables.concat(mainCFS, viewManager.allViewsCfs());
+        return mainCFS;
     }
 
+    @Override
     public List<String> getBuiltIndexes()
     {
         return indexManager.getBuiltIndexNames();
@@ -3142,37 +3559,40 @@ public List<String> getBuiltIndexes()
     @Override
     public int getUnleveledSSTables()
     {
-        return compactionStrategyManager.getUnleveledSSTables();
+        if (strategyContainer instanceof CompactionStrategyManager)
+            return ((CompactionStrategyManager) strategyContainer).getUnleveledSSTables();
+        else
+            return 0;
     }
 
     @Override
     public int[] getSSTableCountPerLevel()
     {
-        return compactionStrategyManager.getSSTableCountPerLevel();
+        return strategyContainer.getSSTableCountPerLevel();
     }
 
     @Override
     public long[] getPerLevelSizeBytes()
     {
-        return compactionStrategyManager.getPerLevelSizeBytes();
+        return strategyContainer.getPerLevelSizeBytes();
     }
 
     @Override
     public boolean isLeveledCompaction()
     {
-        return compactionStrategyManager.isLeveledCompaction();
+        return strategyContainer.isLeveledCompaction();
     }
 
     @Override
     public int[] getSSTableCountPerTWCSBucket()
     {
-        return compactionStrategyManager.getSSTableCountPerTWCSBucket();
+        return strategyContainer.getSSTableCountPerTWCSBucket();
     }
 
     @Override
     public int getLevelFanoutSize()
     {
-        return compactionStrategyManager.getLevelFanoutSize();
+        return strategyContainer.getLevelFanoutSize();
     }
 
     public static class ViewFragment
@@ -3202,6 +3622,7 @@ public void release()
             refs.release();
         }
 
+        @Override
         public void close()
         {
             refs.release();
@@ -3252,6 +3673,7 @@ public boolean isTableIncrementalBackupsEnabled()
     public void discardSSTables(long truncatedAt)
     {
         assert data.getCompacting().isEmpty() : data.getCompacting();
+        AdaptiveLogger log = truncateLogger();
 
         List<SSTableReader> truncatedSSTables = new ArrayList<>();
         int keptSSTables = 0;
@@ -3264,15 +3686,31 @@ public void discardSSTables(long truncatedAt)
             else
             {
                 keptSSTables++;
-                logger.info("Truncation is keeping {} maxDataAge={} truncatedAt={}", sstable, sstable.maxDataAge, truncatedAt);
+                log.log("Truncation is keeping {} maxDataAge={} truncatedAt={}", sstable, sstable.maxDataAge, truncatedAt);
             }
         }
 
         if (!truncatedSSTables.isEmpty())
         {
-            logger.info("Truncation is dropping {} sstables and keeping {} due to sstable.maxDataAge > truncatedAt", truncatedSSTables.size(), keptSSTables);
-            markObsolete(truncatedSSTables, OperationType.UNKNOWN);
+            log.log("Truncation is dropping {} sstables and keeping {} due to sstable.maxDataAge > truncatedAt", truncatedSSTables.size(), keptSSTables);
+            markObsolete(truncatedSSTables, OperationType.TRUNCATE_TABLE);
+        }
+    }
+
+    /**
+     * Discard sstables that matches given filter with provided operation type
+     */
+    public void discardSSTables(Iterable<SSTableReader> sstables, Predicate<SSTableReader> filter, OperationType operationType)
+    {
+        List<SSTableReader> discarded = new ArrayList<>();
+        for (SSTableReader sstable : sstables)
+        {
+            if (filter.apply(sstable))
+                discarded.add(sstable);
         }
+
+        if (!discarded.isEmpty())
+            markObsolete(discarded, operationType);
     }
 
     @Override
@@ -3343,11 +3781,16 @@ public static TableMetrics metricsFor(TableId tableId)
         return Objects.requireNonNull(getIfExists(tableId)).metric;
     }
 
-    /**
-     * Grabs the global first/last tokens among sstables and returns the range of data directories that start/end with those tokens.
-     *
-     * This is done to avoid grabbing the disk boundaries for every sstable in case of huge compactions.
-     */
+    // Used by CNDB
+    public long getMemtablesLiveSize()
+    {
+        long liveSize = 0L;
+        for (Memtable memtable : data.getView().getAllMemtables())
+            liveSize += memtable.getLiveDataSize();
+        return liveSize;
+    }
+
+    @Override
     public List<File> getDirectoriesForFiles(Set<SSTableReader> sstables)
     {
         Directories.DataDirectory[] writeableLocations = directories.getWriteableLocations();
@@ -3373,13 +3816,20 @@ public List<File> getDirectoriesForFiles(Set<SSTableReader> sstables)
         return diskBoundaries.getDisksInBounds(first, last).stream().map(directories::getLocationForDisk).collect(Collectors.toList());
     }
 
+    @Override
     public DiskBoundaries getDiskBoundaries()
     {
         return diskBoundaryManager.getDiskBoundaries(this);
     }
 
-    public void invalidateLocalRanges()
+    public void invalidateLocalRangesAndDiskBoundaries()
     {
+        synchronized (this)
+        {
+            if (localRanges != null)
+                localRanges.invalidate();
+        }
+
         diskBoundaryManager.invalidate();
 
         switchMemtableOrNotify(FlushReason.OWNED_RANGES_CHANGE, Memtable::localRangesUpdated);
@@ -3405,17 +3855,39 @@ public boolean getNeverPurgeTombstones()
     void onTableDropped()
     {
         indexManager.markAllIndexesRemoved();
+        if (logger.isTraceEnabled())
+            logger.trace("CFS {} is being dropped: indexes removed", name);
 
-        CompactionManager.instance.interruptCompactionForCFs(concatWithIndexes(), (sstable) -> true, true);
+        CompactionManager.instance.interruptCompactionForCFs(concatWithIndexes(), (sstable) -> true, true, TableOperation.StopTrigger.DROP_TABLE);
+        if (logger.isTraceEnabled())
+            logger.trace("CFS {} is being dropped: compactions stopped", name);
 
         if (isAutoSnapshotEnabled())
             snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, ColumnFamilyStore.SNAPSHOT_DROP_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl());
 
-        CommitLog.instance.forceRecycleAllSegments(Collections.singleton(metadata.id));
+        if (getTracker().isDummy())
+        {
+            // offline services (e.g. standalone compactor) don't have Memtables or CommitLog. An attempt to flush would
+            // throw an exception
+            logger.debug("Memtables and CommitLog are disabled; not recycling or flushing {}", metadata);
+        }
+        else
+        {
+            if (!UNSAFE_SYSTEM.getBoolean())
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("Recycling CL segments for dropping {}", metadata);
+                CommitLog.instance.forceRecycleAllSegments(Collections.singleton(metadata.id));
+            }
+        }
 
-        compactionStrategyManager.shutdown();
+        if (logger.isTraceEnabled())
+            logger.trace("Dropping CFS {}: shutting down compaction strategy", name);
+        strategyContainer.shutdown();
 
         // wait for any outstanding reads/writes that might affect the CFS
+        if (logger.isTraceEnabled())
+            logger.trace("Dropping CFS {}: waiting for read and write barriers", name);
         Keyspace.writeOrder.awaitNewBarrier();
         readOrdering.awaitNewBarrier();
     }
@@ -3447,7 +3919,7 @@ private static final class PerDiskFlushExecutors
         private final boolean useSpecificExecutorForSystemKeyspaces;
 
         public PerDiskFlushExecutors(int flushWriters,
-                                     String[] locationsForNonSystemKeyspaces,
+                                     File[] locationsForNonSystemKeyspaces,
                                      boolean useSpecificLocationForSystemKeyspaces)
         {
             ExecutorPlus[] flushExecutors = createPerDiskFlushWriters(locationsForNonSystemKeyspaces.length, flushWriters);
@@ -3574,4 +4046,186 @@ public TableMetrics getMetrics()
     {
         return metric;
     }
+
+    private static void verifyMetadata(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair, boolean isTransient)
+    {
+        if (!Objects.equals(pendingRepair, sstable.getPendingRepair()))
+            throw new IllegalStateException(String.format("Failed setting pending repair to %s on %s (pending repair is %s)", pendingRepair, sstable, sstable.getPendingRepair()));
+        if (repairedAt != sstable.getRepairedAt())
+            throw new IllegalStateException(String.format("Failed setting repairedAt to %d on %s (repairedAt is %d)", repairedAt, sstable, sstable.getRepairedAt()));
+        if (isTransient != sstable.isTransient())
+            throw new IllegalStateException(String.format("Failed setting isTransient to %b on %s (isTransient is %b)", isTransient, sstable, sstable.isTransient()));
+    }
+
+    /**
+     * This method is exposed for testing only
+     * NotThreadSafe
+     */
+    @VisibleForTesting
+    public int mutateRepaired(Collection<SSTableReader> sstables, long repairedAt, TimeUUID pendingRepair, boolean isTransient) throws IOException
+    {
+        Set<SSTableReader> changed = new HashSet<>();
+        try
+        {
+            for (SSTableReader sstable: sstables)
+            {
+                sstable.mutateRepairedAndReload(repairedAt, pendingRepair, isTransient);
+                verifyMetadata(sstable, repairedAt, pendingRepair, isTransient);
+                changed.add(sstable);
+            }
+        }
+        finally
+        {
+            // if there was an exception mutating repairedAt, we should still notify for the
+            // sstables that we were able to modify successfully before releasing the lock
+            getTracker().notifySSTableRepairedStatusChanged(changed);
+        }
+        return changed.size();
+    }
+
+    /**
+     * Mutates sstable repairedAt times and notifies listeners of the change with the writeLock held. Prevents races
+     * with other processes between when the metadata is changed and when sstables are moved between strategies.
+     */
+    public int mutateRepaired(@Nullable final ReentrantReadWriteLock.WriteLock writeLock,
+                              Collection<SSTableReader> sstables,
+                              long repairedAt,
+                              TimeUUID pendingRepair,
+                              boolean isTransient) throws IOException
+    {
+        if (writeLock == null)
+            return mutateRepaired(sstables, repairedAt, pendingRepair, isTransient);
+
+        writeLock.lock();
+        try
+        {
+            return mutateRepaired(sstables, repairedAt, pendingRepair, isTransient);
+        }
+        finally
+        {
+            writeLock.unlock();
+        }
+    }
+
+    @Override
+    public int mutateRepairedWithLock(Collection<SSTableReader> sstables, long repairedAt, TimeUUID pendingRepair, boolean isTransient) throws IOException
+    {
+        return mutateRepaired(getCompactionStrategyContainer().getWriteLock(), sstables, repairedAt, pendingRepair, isTransient);
+    }
+
+    @Override
+    public void repairSessionCompleted(TimeUUID sessionID)
+    {
+        getCompactionStrategyContainer().repairSessionCompleted(sessionID);
+    }
+
+    public boolean hasPendingRepairSSTables(TimeUUID sessionID)
+    {
+        return Iterables.any(data.getLiveSSTables(), pendingRepairPredicate(sessionID));
+    }
+
+    public Set<SSTableReader> getPendingRepairSSTables(TimeUUID sessionID)
+    {
+        return Sets.filter(data.getLiveSSTables(), pendingRepairPredicate(sessionID));
+    }
+
+    public static Predicate<SSTableReader> pendingRepairPredicate(@Nonnull TimeUUID sessionID)
+    {
+        return sstable -> sstable.getPendingRepair() != null && sessionID.equals(sstable.getPendingRepair());
+    }
+
+    @Override
+    public LifecycleTransaction tryModify(Iterable<? extends CompactionSSTable> ssTableReaders,
+                                          OperationType operationType,
+                                          TimeUUID id)
+    {
+        return data.tryModify(Iterables.transform(ssTableReaders, SSTableReader.class::cast), operationType, id);
+    }
+
+    @Override
+    public CompactionRealm.OverlapTracker getOverlapTracker(Iterable<SSTableReader> sources)
+    {
+        return new OverlapTracker(sources);
+    }
+
+    class OverlapTracker implements CompactionRealm.OverlapTracker
+    {
+        final Iterable<SSTableReader> compacting;
+        private Refs<SSTableReader> overlappingSSTables;
+        private OverlapIterator<PartitionPosition, SSTableReader> overlapIterator;
+
+        OverlapTracker(Iterable<SSTableReader> compacting)
+        {
+            this.compacting = compacting;
+            collectOverlaps();
+        }
+
+        @Override
+        public Collection<? extends CompactionSSTable> overlaps()
+        {
+            return overlappingSSTables;
+        }
+
+        @Override
+        public Collection<? extends CompactionSSTable> overlaps(DecoratedKey key)
+        {
+            overlapIterator.update(key);
+            return overlapIterator.overlaps();
+        }
+
+        @Override
+        public <V> Iterable<V> openSelectedOverlappingSSTables(DecoratedKey key,
+                                                               Predicate<CompactionSSTable> filter,
+                                                               Function<SSTableReader, V> transformation)
+        {
+            overlapIterator.update(key);
+
+            Iterable<SSTableReader> overlaps = overlapIterator.overlaps();
+            Iterable<V> transformed = Iterables.transform(overlaps, sstable -> filter.apply(sstable)
+                                                                               ? transformation.apply(sstable)
+                                                                               : null);
+            return Iterables.filter(transformed, Predicates.notNull());
+        }
+
+        @Override
+        public void close()
+        {
+            overlapIterator = null;
+            overlappingSSTables.release();
+        }
+
+        @Override
+        public boolean maybeRefresh()
+        {
+            for (CompactionSSTable reader : overlappingSSTables)
+            {
+                if (reader.isMarkedCompacted())
+                {
+                    close();
+                    collectOverlaps();
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        public void refreshOverlaps()
+        {
+            if (this.overlappingSSTables != null)
+                close();
+            collectOverlaps();
+        }
+        
+        private void collectOverlaps()
+        {
+            if (compacting == null)
+                overlappingSSTables = Refs.tryRef(Collections.<SSTableReader>emptyList());
+            else
+                overlappingSSTables = getAndReferenceOverlappingLiveSSTables(compacting);
+            this.overlapIterator = new OverlapIterator<>(SSTableIntervalTree.buildIntervals(overlappingSSTables));
+
+            if (logger.isTraceEnabled())
+                logger.trace("Refreshed overlaps: {}", overlappingSSTables);
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index 7d7b9e58eebc..83b69712aeea 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -52,6 +52,15 @@ public interface ColumnFamilyStoreMBean
      */
     public void forceMajorCompaction(boolean splitOutput) throws ExecutionException, InterruptedException;
 
+    /**
+     * force a major compaction of this column family
+     *
+     * @param permittedParallelism The maximum number of compaction threads that can be used by the operation.
+     *                             If 0, the operation can use all available threads.
+     *                             If <0, the default parallelism will be used.
+     */
+    public void forceMajorCompaction(int permittedParallelism) throws ExecutionException, InterruptedException;
+
     /**
      * Forces a major compaction of specified token ranges in this column family.
      * <p>
@@ -251,13 +260,14 @@ public List<String> importNewSSTables(Set<String> srcPaths,
     /** @deprecated See CASSANDRA-6719 */
     @Deprecated(since = "4.0")
     public void loadNewSSTables();
+
     /**
      * @return the number of SSTables in L0.  Always return 0 if Leveled compaction is not enabled.
      */
     public int getUnleveledSSTables();
 
     /**
-     * @return sstable count for each level. null unless leveled compaction is used.
+     * @return sstable count for each level. empty unless leveled or unified compaction is used.
      *         array index corresponds to level(int[0] is for level 0, ...).
      */
     public int[] getSSTableCountPerLevel();
@@ -280,7 +290,7 @@ public List<String> importNewSSTables(Set<String> srcPaths,
     public int[] getSSTableCountPerTWCSBucket();
 
     /**
-     * @return sstable fanout size for level compaction strategy.
+     * @return sstable fanout size for level or unified compaction strategies. Default LCS fanout size otherwise.
      */
     public int getLevelFanoutSize();
 
diff --git a/src/java/org/apache/cassandra/db/Columns.java b/src/java/org/apache/cassandra/db/Columns.java
index 275d000dd369..bdf7ac0cf5b9 100644
--- a/src/java/org/apache/cassandra/db/Columns.java
+++ b/src/java/org/apache/cassandra/db/Columns.java
@@ -19,7 +19,11 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.AbstractCollection;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Objects;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
@@ -28,6 +32,7 @@
 
 import net.nicoulaj.compilecommand.annotations.DontInline;
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.rows.ColumnData;
@@ -36,6 +41,7 @@
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.serializers.AbstractTypeSerializer;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.SearchIterator;
@@ -76,6 +82,11 @@ public class Columns extends AbstractCollection<ColumnMetadata> implements Colle
     private final Object[] columns;
     private final int complexIdx; // Index of the first complex column
 
+    /**
+     * The columns passed to this constructor MUST BE SORTED with natural order - this is not checked in the constructor!
+     * The constructor remains private to ensure that this invariant is maintained - all the methods that call it
+     * ensure that the columns are properly sorted.
+     */
     private Columns(Object[] columns, int complexIdx)
     {
         assert complexIdx <= BTree.size(columns);
@@ -456,39 +467,121 @@ public String toString()
 
     public static class Serializer
     {
+        AbstractTypeSerializer typeSerializer = new AbstractTypeSerializer();
+
         public void serialize(Columns columns, DataOutputPlus out) throws IOException
         {
-            out.writeUnsignedVInt32(columns.size());
+            int regularCount = 0;
+            int syntheticCount = 0;
+
+            // Count regular and synthetic columns
             for (ColumnMetadata column : columns)
-                ByteBufferUtil.writeWithVIntLength(column.name.bytes, out);
+            {
+                if (column.isSynthetic())
+                    syntheticCount++;
+                else
+                    regularCount++;
+            }
+
+            // Jam the two counts into a single value to avoid massive backwards compatibility issues
+            long packedCount = getPackedCount(syntheticCount, regularCount);
+            out.writeUnsignedVInt(packedCount);
+
+            // First pass - write synthetic columns with their full metadata
+            for (ColumnMetadata column : columns)
+            {
+                if (column.isSynthetic())
+                {
+                    ByteBufferUtil.writeWithVIntLength(column.name.bytes, out);
+                    ByteBufferUtil.writeWithVIntLength(column.sythenticSourceColumn.bytes, out);
+                    typeSerializer.serialize(column.type, out);
+                }
+            }
+
+            // Second pass - write regular columns
+            for (ColumnMetadata column : columns)
+            {
+                if (!column.isSynthetic())
+                    ByteBufferUtil.writeWithVIntLength(column.name.bytes, out);
+            }
+        }
+
+        private static long getPackedCount(int syntheticCount, int regularCount)
+        {
+            // Left shift of 20 gives us over 1M regular columns, and up to 4 synthetic columns
+            // before overflowing to a 4th byte.
+            return ((long) syntheticCount << 20) | regularCount;
         }
 
         public long serializedSize(Columns columns)
         {
-            long size = TypeSizes.sizeofUnsignedVInt(columns.size());
+            int regularCount = 0;
+            int syntheticCount = 0;
+            long size = 0;
+
+            // Count and calculate sizes
             for (ColumnMetadata column : columns)
-                size += ByteBufferUtil.serializedSizeWithVIntLength(column.name.bytes);
-            return size;
+            {
+                if (column.isSynthetic())
+                {
+                    syntheticCount++;
+                    size += ByteBufferUtil.serializedSizeWithVIntLength(column.name.bytes);
+                    size += ByteBufferUtil.serializedSizeWithVIntLength(column.sythenticSourceColumn.bytes);
+                    size += typeSerializer.serializedSize(column.type);
+                }
+                else
+                {
+                    regularCount++;
+                    size += ByteBufferUtil.serializedSizeWithVIntLength(column.name.bytes);
+                }
+            }
+
+            return TypeSizes.sizeofUnsignedVInt(getPackedCount(syntheticCount, regularCount)) +
+                   size;
         }
 
         public Columns deserialize(DataInputPlus in, TableMetadata metadata) throws IOException
         {
-            int length = in.readUnsignedVInt32();
             try (BTree.FastBuilder<ColumnMetadata> builder = BTree.fastBuilder())
             {
-                for (int i = 0; i < length; i++)
+                long packedCount = in.readUnsignedVInt() ;
+                int regularCount = (int) (packedCount & 0xFFFFF);
+                int syntheticCount = (int) (packedCount >> 20);
+
+                // First pass - synthetic columns
+                for (int i = 0; i < syntheticCount; i++)
+                {
+                    ByteBuffer name = ByteBufferUtil.readWithVIntLength(in);
+                    ByteBuffer sourceColumnName = ByteBufferUtil.readWithVIntLength(in);
+                    AbstractType<?> type = typeSerializer.deserialize(in);
+
+                    if (!name.equals(ColumnMetadata.SYNTHETIC_SCORE_ID.bytes))
+                        throw new IllegalStateException("Unknown synthetic column " + UTF8Type.instance.getString(name));
+
+                    ColumnMetadata sourceColumn = metadata.getColumn(sourceColumnName);
+                    if (sourceColumn == null)
+                    {
+                        // If we don't find the definition, it could be we have data for a dropped column
+                        sourceColumn = metadata.getDroppedColumn(name);
+                        if (sourceColumn == null)
+                            throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization of " + metadata.keyspace + '.' + metadata.name);
+                    }
+
+                    ColumnMetadata column = ColumnMetadata.syntheticScoreColumn(sourceColumn, type);
+                    builder.add(column);
+                }
+
+                // Second pass - regular columns
+                for (int i = 0; i < regularCount; i++)
                 {
                     ByteBuffer name = ByteBufferUtil.readWithVIntLength(in);
                     ColumnMetadata column = metadata.getColumn(name);
                     if (column == null)
                     {
-                        // If we don't find the definition, it could be we have data for a dropped column, and we shouldn't
-                        // fail deserialization because of that. So we grab a "fake" ColumnMetadata that ensure proper
-                        // deserialization. The column will be ignore later on anyway.
+                        // If we don't find the definition, it could be we have data for a dropped column
                         column = metadata.getDroppedColumn(name);
-
                         if (column == null)
-                            throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization");
+                            throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization of " + metadata.keyspace + '.' + metadata.name);
                     }
                     builder.add(column);
                 }
@@ -581,6 +674,50 @@ else if (superset.size() >= 64)
             }
         }
 
+        /**
+         * Deserialize a columns subset, placing the selected columns in the given array and returning the number of
+         * columns.
+         *
+         * @param superset the full list of columns
+         * @param in file from which the subset should be read
+         * @param placeInto An array where the selected columns will be placed, in the same order as superset. Must
+         *                  be at least superset.length long.
+         * @return the number of items placed in the target array, <= superset.length.
+         * @throws IOException
+         */
+        public int deserializeSubset(ColumnMetadata[] superset,
+                                     DataInputPlus in,
+                                     ColumnMetadata[] placeInto)
+        throws IOException
+        {
+            long encoded = in.readUnsignedVInt();
+            if (encoded == 0L)
+            {
+                // this is wasteful, but we don't expect to be called in this case (rows will have a flag set that
+                // bypasses this path).
+                System.arraycopy(superset, 0, placeInto, 0, superset.length);
+                return superset.length;
+            }
+            else if (superset.length >= 64)
+            {
+                return deserializeLargeSubset(in, superset, (int) encoded, placeInto);
+            }
+            else
+            {
+                int count = 0;
+                for (ColumnMetadata column : superset)
+                {
+                    if ((encoded & 1) == 0)
+                        placeInto[count++] = column;
+
+                    encoded >>>= 1;
+                }
+                if (encoded != 0)
+                    throw new IOException("Invalid Columns subset bytes; too many bits set:" + Long.toBinaryString(encoded));
+                return count;
+            }
+        }
+
         // encodes a 1 bit for every *missing* column, on the assumption presence is more common,
         // and because this is consistent with encoding 0 to represent all present
         private static long encodeBitmap(Collection<ColumnMetadata> columns, Columns superset, int supersetCount)
@@ -663,7 +800,18 @@ private Columns deserializeLargeSubset(DataInputPlus in, Columns superset, int d
                     int skipped = 0;
                     while (true)
                     {
-                        int nextMissingIndex = skipped < delta ? in.readUnsignedVInt32() : supersetCount;
+                        int nextMissingIndex;
+                        if (skipped < delta)
+                        {
+                            nextMissingIndex = (int) in.readUnsignedVInt32();
+                            if (nextMissingIndex >= supersetCount)
+                                throw new IOException("Invalid Columns subset bytes; encoded not existing column: " + nextMissingIndex);
+                        }
+                        else
+                        {
+                            nextMissingIndex = supersetCount;
+                        }
+
                         while (idx < nextMissingIndex)
                         {
                             ColumnMetadata def = iter.next();
@@ -681,6 +829,44 @@ private Columns deserializeLargeSubset(DataInputPlus in, Columns superset, int d
             }
         }
 
+        @DontInline
+        private int deserializeLargeSubset(DataInputPlus in,
+                                           ColumnMetadata[] superset,
+                                           int delta,
+                                           ColumnMetadata[] placeInto)
+        throws IOException
+        {
+            int supersetCount = superset.length;
+            int columnCount = supersetCount - delta;
+
+            int count = 0;
+            if (columnCount < supersetCount / 2)
+            {
+                for (int i = 0 ; i < columnCount ; i++)
+                {
+                    int idx = (int) in.readUnsignedVInt();
+                    placeInto[count++] = superset[idx];
+                }
+            }
+            else
+            {
+                int idx = 0;
+                int skipped = 0;
+                while (true)
+                {
+                    int nextMissingIndex = skipped < delta ? (int)in.readUnsignedVInt() : supersetCount;
+                    while (idx < nextMissingIndex)
+                        placeInto[count++] = superset[idx++];
+
+                    if (idx == supersetCount)
+                        break;
+                    idx++;
+                    skipped++;
+                }
+            }
+            return count;
+        }
+
         @DontInline
         private int serializeLargeSubsetSize(Collection<ColumnMetadata> columns, int columnCount, Columns superset, int supersetCount)
         {
@@ -714,6 +900,5 @@ private int serializeLargeSubsetSize(Collection<ColumnMetadata> columns, int col
             }
             return size;
         }
-
     }
 }
diff --git a/src/java/org/apache/cassandra/db/ConsistencyLevel.java b/src/java/org/apache/cassandra/db/ConsistencyLevel.java
index 7c21c1287a7a..7f1a6bdeba49 100644
--- a/src/java/org/apache/cassandra/db/ConsistencyLevel.java
+++ b/src/java/org/apache/cassandra/db/ConsistencyLevel.java
@@ -17,10 +17,12 @@
  */
 package org.apache.cassandra.db;
 
-
 import java.util.Locale;
 
+import javax.annotation.Nullable;
+
 import com.carrotsearch.hppc.ObjectIntHashMap;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.InOurDc;
 import org.apache.cassandra.schema.TableMetadata;
@@ -28,8 +30,11 @@
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.ProtocolException;
 
+import static org.apache.cassandra.db.guardrails.Guardrails.CONFIG_PROVIDER;
 import static org.apache.cassandra.locator.Replicas.addToCountPerDc;
 
 public enum ConsistencyLevel
@@ -47,6 +52,8 @@ public enum ConsistencyLevel
     LOCAL_ONE   (10, true),
     NODE_LOCAL  (11, true);
 
+    public static final boolean THREE_MEANS_ALL_BUT_ONE = CassandraRelevantProperties.THREE_MEANS_ALL_BUT_ONE.getBoolean();
+
     // Used by the binary protocol
     public final int code;
     private final boolean isDCLocal;
@@ -83,6 +90,16 @@ public static ConsistencyLevel fromCode(int code)
         return codeIdx[code];
     }
 
+    @Override
+    public String toString()
+    {
+        if (this == THREE && THREE_MEANS_ALL_BUT_ONE)
+        {
+            return "THREE (ALL_BUT_ONE)";
+        }
+        return super.toString();
+    }
+
     public static ConsistencyLevel fromString(String str)
     {
         return valueOf(str.toUpperCase(Locale.US));
@@ -93,6 +110,12 @@ public static int quorumFor(AbstractReplicationStrategy replicationStrategy)
         return (replicationStrategy.getReplicationFactor().allReplicas / 2) + 1;
     }
 
+    static int allButOneFor(AbstractReplicationStrategy replicationStrategy)
+    {
+        int rf = replicationStrategy.getReplicationFactor().fullReplicas;
+        return rf <= 1 ? rf : rf - 1;
+    }
+
     public static int localQuorumFor(AbstractReplicationStrategy replicationStrategy, String dc)
     {
         return (replicationStrategy instanceof NetworkTopologyStrategy)
@@ -142,6 +165,10 @@ public int blockFor(AbstractReplicationStrategy replicationStrategy)
             case TWO:
                 return 2;
             case THREE:
+                if (THREE_MEANS_ALL_BUT_ONE)
+                {
+                    return allButOneFor(replicationStrategy);
+                }
                 return 3;
             case QUORUM:
             case SERIAL:
@@ -214,7 +241,7 @@ public void validateForRead() throws InvalidRequestException
         }
     }
 
-    public void validateForWrite() throws InvalidRequestException
+    public void validateForWrite(String keyspaceName, ClientState clientState) throws InvalidRequestException
     {
         switch (this)
         {
@@ -225,7 +252,7 @@ public void validateForWrite() throws InvalidRequestException
     }
 
     // This is the same than validateForWrite really, but we include a slightly different error message for SERIAL/LOCAL_SERIAL
-    public void validateForCasCommit(AbstractReplicationStrategy replicationStrategy) throws InvalidRequestException
+    public void validateForCasCommit(AbstractReplicationStrategy replicationStrategy, String keyspaceName, ClientState clientState) throws InvalidRequestException
     {
         switch (this)
         {
@@ -238,7 +265,7 @@ public void validateForCasCommit(AbstractReplicationStrategy replicationStrategy
         }
     }
 
-    public void validateForCas() throws InvalidRequestException
+    public void validateForCas(String keyspaceName, ClientState clientState) throws InvalidRequestException
     {
         if (!isSerialConsistency())
             throw new InvalidRequestException("Invalid consistency for conditional update. Must be one of SERIAL or LOCAL_SERIAL");
@@ -249,7 +276,7 @@ public boolean isSerialConsistency()
         return this == SERIAL || this == LOCAL_SERIAL;
     }
 
-    public void validateCounterForWrite(TableMetadata metadata) throws InvalidRequestException
+    public void validateCounterForWrite(TableMetadata metadata, ClientState clientState) throws InvalidRequestException
     {
         if (this == ConsistencyLevel.ANY)
             throw new InvalidRequestException("Consistency level ANY is not yet supported for counter table " + metadata.name);
@@ -259,7 +286,7 @@ public void validateCounterForWrite(TableMetadata metadata) throws InvalidReques
     }
 
     /**
-     * With a replication factor greater than one, reads that contact more than one replica will require 
+     * With a replication factor greater than one, reads that contact more than one replica will require
      * reconciliation of the individual replica results at the coordinator.
      *
      * @return true if reads at this consistency level require merging at the coordinator
@@ -275,4 +302,22 @@ private void requireNetworkTopologyStrategy(AbstractReplicationStrategy replicat
             throw new InvalidRequestException(String.format("consistency level %s not compatible with replication strategy (%s)",
                                                             this, replicationStrategy.getClass().getName()));
     }
+
+    /**
+     * Returns the strictest consistency level allowed by Guardrails.
+     *
+     * @param state the query state, used to skip the guardrails check if the query is internal or is done by a superuser.
+     * @return the strictest allowed serial consistency level
+     * @throws InvalidRequestException if all serial consistency level are disallowed
+     */
+    public static ConsistencyLevel defaultSerialConsistency(@Nullable QueryState state) throws InvalidRequestException
+    {
+        ClientState clientState = state == null ? null : state.getClientState();
+        if (DatabaseDescriptor.getRawConfig() == null || !CONFIG_PROVIDER.getOrCreate(clientState).getWriteConsistencyLevelsDisallowed().contains(ConsistencyLevel.SERIAL))
+            return ConsistencyLevel.SERIAL;
+        else if (!CONFIG_PROVIDER.getOrCreate(clientState).getWriteConsistencyLevelsDisallowed().contains(ConsistencyLevel.LOCAL_SERIAL))
+            return ConsistencyLevel.LOCAL_SERIAL;
+
+        throw new InvalidRequestException("Serial consistency levels are disallowed by disallowedWriteConsistencies Guardrail");
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/CounterMutation.java b/src/java/org/apache/cassandra/db/CounterMutation.java
index ed64e0aad7d1..faf9710fc9e8 100644
--- a/src/java/org/apache/cassandra/db/CounterMutation.java
+++ b/src/java/org/apache/cassandra/db/CounterMutation.java
@@ -18,49 +18,108 @@
 package org.apache.cassandra.db;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.Objects;
+import com.google.common.base.Supplier;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
-import com.google.common.util.concurrent.Striped;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import org.apache.cassandra.cache.CounterCacheKey;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.counters.CounterLockManager;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.partitions.*;
 import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+import org.apache.cassandra.metrics.LatencyMetrics;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.CounterId;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.btree.BTreeSet;
-
-import static java.util.concurrent.TimeUnit.*;
+import org.apache.cassandra.utils.concurrent.Future;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_10;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_11;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_20;
 import static org.apache.cassandra.net.MessagingService.VERSION_40;
 import static org.apache.cassandra.net.MessagingService.VERSION_50;
+import static org.apache.cassandra.net.MessagingService.VERSION_DSE_68;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
+
 public class CounterMutation implements IMutation
 {
+    private static final Logger logger = LoggerFactory.getLogger(CounterMutation.class);
+    private static final NoSpamLogger nospamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.SECONDS);
+
     public static final CounterMutationSerializer serializer = new CounterMutationSerializer();
 
-    private static final Striped<Lock> LOCKS = Striped.lazyWeakLock(DatabaseDescriptor.getConcurrentCounterWriters() * 1024);
+    /**
+     * This metric tracks the number of timeouts that occurred because the locks could not be
+     * acquired within DatabaseDescriptor.getCounterWriteRpcTimeout().
+     */
+    public static final Counter lockTimeout = Metrics.counter(DefaultNameFactory.createMetricName("Counter", "lock_timeout", null));
+
+    /**
+     * This metric tracks how long it took to acquire all the locks
+     * that must be acquired before applying the counter mutation.
+     */
+    public static final LatencyMetrics lockAcquireTime = new LatencyMetrics("Counter", "lock_acquire_time");
+
+    /**
+     * This metric tracks the number of locks that must be acquired before applying the counter
+     * mutation. A mutation normally has one partition only, unless it comes from a batch,
+     * where the same partition key is used across different tables.
+     * For each partition, we need to acquire one lock for each column on each row.
+     * The locks are striped, see {@link CounterMutation#LOCKS} for details.
+     */
+    public static final Histogram locksPerUpdate = Metrics.histogram(DefaultNameFactory
+                                                                     .createMetricName("Counter",
+                                                                                       "locks_per_update",
+                                                                                       null),
+                                                                     false);
+
+    private static final String LOCK_TIMEOUT_MESSAGE = "Failed to acquire locks for counter mutation on keyspace {} for longer than {} millis, giving up";
+    private static final String LOCK_TIMEOUT_TRACE = "Failed to acquire locks for counter mutation for longer than {} millis, giving up";
 
     private final Mutation mutation;
     private final ConsistencyLevel consistency;
 
+
     public CounterMutation(Mutation mutation, ConsistencyLevel consistency)
     {
         this.mutation = mutation;
@@ -72,6 +131,11 @@ public String getKeyspaceName()
         return mutation.getKeyspaceName();
     }
 
+    public Keyspace getKeyspace()
+    {
+        return mutation.getKeyspace();
+    }
+
     public Collection<TableId> getTableIds()
     {
         return mutation.getTableIds();
@@ -131,13 +195,17 @@ public Mutation applyCounterMutation() throws WriteTimeoutException
         Mutation.PartitionUpdateCollector resultBuilder = new Mutation.PartitionUpdateCollector(getKeyspaceName(), key());
         Keyspace keyspace = Keyspace.open(getKeyspaceName());
 
-        List<Lock> locks = new ArrayList<>();
+        List<CounterLockManager.LockHandle> lockHandles = new ArrayList<>();
         Tracing.trace("Acquiring counter locks");
+
+        long clock = FBUtilities.timestampMicros();
+        CounterId counterId = CounterId.getLocalId();
+
         try
         {
-            grabCounterLocks(keyspace, locks);
+            grabCounterLocks(keyspace, lockHandles);
             for (PartitionUpdate upd : getPartitionUpdates())
-                resultBuilder.add(processModifications(upd));
+                resultBuilder.add(processModifications(upd, clock, counterId));
 
             Mutation result = resultBuilder.build();
             result.apply();
@@ -145,35 +213,94 @@ public Mutation applyCounterMutation() throws WriteTimeoutException
         }
         finally
         {
-            for (Lock lock : locks)
-                lock.unlock();
+            // iterate over all locks in reverse order and unlock them
+            for (int i = lockHandles.size() - 1; i >= 0; i--)
+                lockHandles.get(i).release();
         }
     }
 
+    /**
+     * Applies the counter mutation with the provided time and {@link CounterId}. As opposed to
+     * {@link #applyCounterMutation()} this method doesn't acquire cell-level locks.
+     * <p/>
+     * This method is used in CDC counter write path (CNDB).
+     * <p/>
+     * The time and counter values are evaluated and propagated to all replicas by CDC Service. The replicas
+     * use this method to apply the mutation locally without locks. The locks are not needed in the CDC
+     * path as all the writes to the same partition are serialized by CDC Service.
+     */
+    public Future<Mutation> applyCounterMutationWithoutLocks(long systemClockMicros, CounterId counterId)
+    {
+        Mutation.PartitionUpdateCollector resultBuilder = new Mutation.PartitionUpdateCollector(getKeyspaceName(), key());
+        for (PartitionUpdate upd : getPartitionUpdates())
+            resultBuilder.add(processModifications(upd, systemClockMicros, counterId));
+
+        Mutation mutatation = resultBuilder.build();
+        return mutatation.applyFuture(WriteOptions.DEFAULT).map(o -> mutatation);
+    }
+
     public void apply()
     {
         applyCounterMutation();
     }
 
-    private void grabCounterLocks(Keyspace keyspace, List<Lock> locks) throws WriteTimeoutException
+    private int countDistinctLocks(Iterable<CounterLockManager.LockHandle> sortedLocks)
     {
+        CounterLockManager.LockHandle prev = null;
+        int counter = 0;
+        for(CounterLockManager.LockHandle l: sortedLocks)
+        {
+            if (prev != l)
+                counter++;
+            prev = l;
+        }
+        return counter;
+    }
+
+    @VisibleForTesting
+    public void grabCounterLocks(Keyspace keyspace, List<CounterLockManager.LockHandle> lockHandles) throws WriteTimeoutException
+    {
+        assert lockHandles.isEmpty();
         long startTime = nanoTime();
 
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
-        for (Lock lock : LOCKS.bulkGet(getCounterLockKeys()))
+        List<CounterLockManager.LockHandle> sortedLockHandles = CounterLockManager.instance.grabLocks(getCounterLockKeys());
+        // always return all the locks to the caller, this way they can be released even in case of errors
+        lockHandles.addAll(sortedLockHandles);
+        locksPerUpdate.update(countDistinctLocks(sortedLockHandles));
+        try
         {
-            long timeout = getTimeout(NANOSECONDS) - (nanoTime() - startTime);
-            try
-            {
-                if (!lock.tryLock(timeout, NANOSECONDS))
-                    throw new WriteTimeoutException(WriteType.COUNTER, consistency(), 0, consistency().blockFor(replicationStrategy));
-                locks.add(lock);
-            }
-            catch (InterruptedException e)
+            for (CounterLockManager.LockHandle lockHandle : sortedLockHandles)
             {
-                throw new WriteTimeoutException(WriteType.COUNTER, consistency(), 0, consistency().blockFor(replicationStrategy));
+            long timeout = getTimeout(NANOSECONDS) - (nanoTime() - startTime);
+                try
+                {
+                    if (!lockHandle.tryLock(timeout, NANOSECONDS))
+                        handleLockTimeoutAndThrow(replicationStrategy);
+
+                }
+                catch (InterruptedException e)
+                {
+                    handleLockTimeoutAndThrow(replicationStrategy);
+                }
             }
         }
+        finally
+        {
+            lockAcquireTime.addNano(Clock.Global.nanoTime() - startTime);
+        }
+    }
+
+    private void handleLockTimeoutAndThrow(AbstractReplicationStrategy replicationStrategy)
+    {
+        lockTimeout.inc();
+
+        nospamLogger.error(LOCK_TIMEOUT_MESSAGE,
+                           getKeyspaceName(),
+                           DatabaseDescriptor.getCounterWriteRpcTimeout(MILLISECONDS));
+        Tracing.trace(LOCK_TIMEOUT_TRACE, DatabaseDescriptor.getCounterWriteRpcTimeout(MILLISECONDS));
+
+        throw new WriteTimeoutException(WriteType.COUNTER, consistency(), 0, consistency().blockFor(replicationStrategy));
     }
 
     /**
@@ -181,19 +308,19 @@ private void grabCounterLocks(Keyspace keyspace, List<Lock> locks) throws WriteT
      * Striped#bulkGet() depends on Object#hashCode(), so here we make sure that the cf id and the partition key
      * all get to be part of the hashCode() calculation.
      */
-    private Iterable<Object> getCounterLockKeys()
+    private Iterable<Integer> getCounterLockKeys()
     {
-        return Iterables.concat(Iterables.transform(getPartitionUpdates(), new Function<PartitionUpdate, Iterable<Object>>()
+        return Iterables.concat(Iterables.transform(getPartitionUpdates(), new Function<PartitionUpdate, Iterable<Integer>>()
         {
-            public Iterable<Object> apply(final PartitionUpdate update)
+            public Iterable<Integer> apply(final PartitionUpdate update)
             {
-                return Iterables.concat(Iterables.transform(update, new Function<Row, Iterable<Object>>()
+                return Iterables.concat(Iterables.transform(update.rows(), new Function<Row, Iterable<Integer>>()
                 {
-                    public Iterable<Object> apply(final Row row)
+                    public Iterable<Integer> apply(final Row row)
                     {
-                        return Iterables.concat(Iterables.transform(row, new Function<ColumnData, Object>()
+                        return Iterables.concat(Iterables.transform(row, new Function<ColumnData, Integer>()
                         {
-                            public Object apply(final ColumnData data)
+                            public Integer apply(final ColumnData data)
                             {
                                 return Objects.hashCode(update.metadata().id, key(), row.clustering(), data.column());
                             }
@@ -204,64 +331,84 @@ public Object apply(final ColumnData data)
         }));
     }
 
-    private PartitionUpdate processModifications(PartitionUpdate changes)
+    private PartitionUpdate processModifications(PartitionUpdate changes,
+                                                 long systemClockMicros,
+                                                 CounterId counterId)
     {
         ColumnFamilyStore cfs = Keyspace.open(getKeyspaceName()).getColumnFamilyStore(changes.metadata().id);
 
-        List<PartitionUpdate.CounterMark> marks = changes.collectCounterMarks();
+        List<Pair<PartitionUpdate.CounterMark, CounterCacheKey>> marks = changes.collectCounterMarks().stream()
+                                                                                .map(mark -> Pair.create(mark, cacheKeyForMark(cfs, mark)))
+                                                                                .collect(Collectors.toList());
 
         if (CacheService.instance.counterCache.getCapacity() != 0)
         {
             Tracing.trace("Fetching {} counter values from cache", marks.size());
-            updateWithCurrentValuesFromCache(marks, cfs);
+            updateWithCurrentValuesFromCache(marks, cfs, systemClockMicros, counterId);
             if (marks.isEmpty())
                 return changes;
         }
 
         Tracing.trace("Reading {} counter values from the CF", marks.size());
-        updateWithCurrentValuesFromCFS(marks, cfs);
+        updateWithCurrentValuesFromCFS(marks, cfs, systemClockMicros, counterId);
 
         // What's remain is new counters
-        for (PartitionUpdate.CounterMark mark : marks)
-            updateWithCurrentValue(mark, ClockAndCount.BLANK, cfs);
+        for (Pair<PartitionUpdate.CounterMark, CounterCacheKey> mark : marks)
+            updateWithCurrentValue(mark, ClockAndCount.BLANK, cfs, systemClockMicros, counterId);
 
         return changes;
     }
 
-    private void updateWithCurrentValue(PartitionUpdate.CounterMark mark, ClockAndCount currentValue, ColumnFamilyStore cfs)
+    private CounterCacheKey cacheKeyForMark(ColumnFamilyStore cfs, PartitionUpdate.CounterMark mark)
+    {
+        return CounterCacheKey.create(cfs.metadata(), key().getKey(), mark.clustering(), mark.column(), mark.path());
+    }
+
+    private void updateWithCurrentValue(Pair<PartitionUpdate.CounterMark, CounterCacheKey> mark,
+                                        ClockAndCount currentValue,
+                                        ColumnFamilyStore cfs,
+                                        long systemClockMicros,
+                                        CounterId counterId)
     {
-        long clock = Math.max(FBUtilities.timestampMicros(), currentValue.clock + 1L);
-        long count = currentValue.count + CounterContext.instance().total(mark.value(), ByteBufferAccessor.instance);
+        long clock = Math.max(systemClockMicros, currentValue.clock + 1L);
+        long count = currentValue.count + CounterContext.instance().total(mark.left.value(), ByteBufferAccessor.instance);
 
-        mark.setValue(CounterContext.instance().createGlobal(CounterId.getLocalId(), clock, count));
+        mark.left.setValue(CounterContext.instance().createGlobal(counterId, clock, count));
 
         // Cache the newly updated value
-        cfs.putCachedCounter(key().getKey(), mark.clustering(), mark.column(), mark.path(), ClockAndCount.create(clock, count));
+        cfs.putCachedCounter(mark.right, ClockAndCount.create(clock, count));
     }
 
     // Returns the count of cache misses.
-    private void updateWithCurrentValuesFromCache(List<PartitionUpdate.CounterMark> marks, ColumnFamilyStore cfs)
+    private void updateWithCurrentValuesFromCache(List<Pair<PartitionUpdate.CounterMark, CounterCacheKey>> marks,
+                                                  ColumnFamilyStore cfs,
+                                                  long systemClockMicros,
+                                                  CounterId counterId)
     {
-        Iterator<PartitionUpdate.CounterMark> iter = marks.iterator();
+        Iterator<Pair<PartitionUpdate.CounterMark, CounterCacheKey>> iter = marks.iterator();
         while (iter.hasNext())
         {
-            PartitionUpdate.CounterMark mark = iter.next();
-            ClockAndCount cached = cfs.getCachedCounter(key().getKey(), mark.clustering(), mark.column(), mark.path());
+            Pair<PartitionUpdate.CounterMark, CounterCacheKey> mark = iter.next();
+            ClockAndCount cached = cfs.getCachedCounter(mark.right);
             if (cached != null)
             {
-                updateWithCurrentValue(mark, cached, cfs);
+                updateWithCurrentValue(mark, cached, cfs, systemClockMicros, counterId);
                 iter.remove();
             }
         }
     }
 
     // Reads the missing current values from the CFS.
-    private void updateWithCurrentValuesFromCFS(List<PartitionUpdate.CounterMark> marks, ColumnFamilyStore cfs)
+    private void updateWithCurrentValuesFromCFS(List<Pair<PartitionUpdate.CounterMark, CounterCacheKey>> marks,
+                                                ColumnFamilyStore cfs,
+                                                long systemClockMicros,
+                                                CounterId counterId)
     {
         ColumnFilter.Builder builder = ColumnFilter.selectionBuilder();
         BTreeSet.Builder<Clustering<?>> names = BTreeSet.builder(cfs.metadata().comparator);
-        for (PartitionUpdate.CounterMark mark : marks)
+        for (Pair<PartitionUpdate.CounterMark, CounterCacheKey> markAndKey : marks)
         {
+            PartitionUpdate.CounterMark mark = markAndKey.left;
             if (mark.clustering() != Clustering.STATIC_CLUSTERING)
                 names.add(mark.clustering());
             if (mark.path() == null)
@@ -273,18 +420,18 @@ private void updateWithCurrentValuesFromCFS(List<PartitionUpdate.CounterMark> ma
         long nowInSec = FBUtilities.nowInSeconds();
         ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(names.build(), false);
         SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(cfs.metadata(), nowInSec, key(), builder.build(), filter);
-        PeekingIterator<PartitionUpdate.CounterMark> markIter = Iterators.peekingIterator(marks.iterator());
+        PeekingIterator<Pair<PartitionUpdate.CounterMark, CounterCacheKey>> markIter = Iterators.peekingIterator(marks.iterator());
         try (ReadExecutionController controller = cmd.executionController();
              RowIterator partition = UnfilteredRowIterators.filter(cmd.queryMemtableAndDisk(cfs, controller), nowInSec))
         {
-            updateForRow(markIter, partition.staticRow(), cfs);
+            updateForRow(markIter, partition.staticRow(), cfs, systemClockMicros, counterId);
 
             while (partition.hasNext())
             {
                 if (!markIter.hasNext())
                     return;
 
-                updateForRow(markIter, partition.next(), cfs);
+                updateForRow(markIter, partition.next(), cfs, systemClockMicros, counterId);
             }
         }
     }
@@ -299,11 +446,15 @@ private int compare(Clustering<?> c1, Clustering<?> c2, ColumnFamilyStore cfs)
         return cfs.getComparator().compare(c1, c2);
     }
 
-    private void updateForRow(PeekingIterator<PartitionUpdate.CounterMark> markIter, Row row, ColumnFamilyStore cfs)
+    private void updateForRow(PeekingIterator<Pair<PartitionUpdate.CounterMark, CounterCacheKey>> markIter,
+                              Row row,
+                              ColumnFamilyStore cfs,
+                              long systemClockMicros,
+                              CounterId counterId)
     {
         int cmp = 0;
         // If the mark is before the row, we have no value for this mark, just consume it
-        while (markIter.hasNext() && (cmp = compare(markIter.peek().clustering(), row.clustering(), cfs)) < 0)
+        while (markIter.hasNext() && (cmp = compare(markIter.peek().left().clustering(), row.clustering(), cfs)) < 0)
             markIter.next();
 
         if (!markIter.hasNext())
@@ -311,17 +462,19 @@ private void updateForRow(PeekingIterator<PartitionUpdate.CounterMark> markIter,
 
         while (cmp == 0)
         {
-            PartitionUpdate.CounterMark mark = markIter.next();
+            Pair<PartitionUpdate.CounterMark, CounterCacheKey> markAndKey = markIter.next();
+            PartitionUpdate.CounterMark mark = markAndKey.left;
             Cell<?> cell = mark.path() == null ? row.getCell(mark.column()) : row.getCell(mark.column(), mark.path());
             if (cell != null)
             {
-                updateWithCurrentValue(mark, CounterContext.instance().getLocalClockAndCount(cell.buffer()), cfs);
+                ClockAndCount localClockAndCount = CounterContext.instance().getLocalClockAndCount(cell.buffer());
+                updateWithCurrentValue(markAndKey, localClockAndCount, cfs, systemClockMicros, counterId);
                 markIter.remove();
             }
             if (!markIter.hasNext())
                 return;
 
-            cmp = compare(markIter.peek().clustering(), row.clustering(), cfs);
+            cmp = compare(markIter.peek().left().clustering(), row.clustering(), cfs);
         }
     }
 
@@ -332,6 +485,10 @@ public long getTimeout(TimeUnit unit)
 
     private int serializedSize40;
     private int serializedSize50;
+    private int serializedSizeDS10;
+    private int serializedSizeDS11;
+    private int serializedSizeDS20;
+    private int serializedSizeDSE68;
 
     public int serializedSize(int version)
     {
@@ -345,6 +502,22 @@ public int serializedSize(int version)
                 if (serializedSize50 == 0)
                     serializedSize50 = (int) serializer.serializedSize(this, VERSION_50);
                 return serializedSize50;
+            case VERSION_DS_10:
+                if (serializedSizeDS10 == 0)
+                    serializedSizeDS10 = (int) serializer.serializedSize(this, VERSION_DS_10);
+                return serializedSizeDS10;
+            case VERSION_DS_11:
+                if (serializedSizeDS11 == 0)
+                    serializedSizeDS11 = (int) serializer.serializedSize(this, VERSION_DS_11);
+                return serializedSizeDS11;
+            case VERSION_DS_20:
+                if (serializedSizeDS20 == 0)
+                    serializedSizeDS20 = (int) serializer.serializedSize(this, VERSION_DS_20);
+                return serializedSizeDS20;
+            case VERSION_DSE_68:
+                if (serializedSizeDSE68 == 0)
+                    serializedSizeDSE68 = (int) serializer.serializedSize(this, VERSION_DSE_68);
+                return serializedSizeDSE68;
             default:
                 throw new IllegalStateException("Unknown serialization version: " + version);
         }
diff --git a/src/java/org/apache/cassandra/db/CounterMutationCallback.java b/src/java/org/apache/cassandra/db/CounterMutationCallback.java
new file mode 100644
index 000000000000..648997637b6f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/CounterMutationCallback.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+
+/**
+ * A counter mutation callback that encapsulates {@link RequestSensors} and replica count
+ */
+public class CounterMutationCallback implements Runnable
+{
+    private final Message<CounterMutation> requestMessage;
+    private final InetAddressAndPort respondToAddress;
+    private final RequestSensors sensors;
+    private int replicaCount = 0;
+
+    public CounterMutationCallback(Message<CounterMutation> requestMessage, InetAddressAndPort respondToAddress, RequestSensors sensors)
+    {
+        this.requestMessage = requestMessage;
+        this.respondToAddress = respondToAddress;
+        this.sensors = sensors;
+    }
+
+    /**
+     * Sets replica count including the local one.
+     */
+    public void setReplicaCount(Integer replicaCount)
+    {
+        this.replicaCount = replicaCount;
+    }
+
+    @Override
+    public void run()
+    {
+        Message.Builder<NoPayload> responseBuilder = requestMessage.emptyResponseBuilder();
+        int replicaMultiplier = replicaCount == 0 ?
+                                1 : // replica count was not explicitly set (default). At the bare minimum, we should send the response accomodating for the local replica (aka. mutation leader) sensor values
+                                replicaCount;
+        SensorsCustomParams.addSensorsToInternodeResponse(sensors, s -> s.getValue() * replicaMultiplier, responseBuilder);
+        MessagingService.instance().send(responseBuilder.build(), respondToAddress);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java b/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java
index 3c38497150e5..fcfa31351fee 100644
--- a/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java
@@ -17,13 +17,23 @@
  */
 package org.apache.cassandra.db;
 
+import java.util.Collection;
+import java.util.stream.Collectors;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.SensorsFactory;
+import org.apache.cassandra.sensors.Type;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.transport.Dispatcher;
 
@@ -39,6 +49,19 @@ protected void applyMutation(final Message<CounterMutation> message, InetAddress
         final CounterMutation cm = message.payload;
         logger.trace("Applying forwarded {}", cm);
 
+        // Initialize the sensor and set ExecutorLocals
+        RequestSensors requestSensors = SensorsFactory.instance.createRequestSensors(message.payload.getKeyspaceName());
+        Collection<TableMetadata> tables = message.payload.getPartitionUpdates().stream().map(PartitionUpdate::metadata).collect(Collectors.toSet());
+        RequestTracker.instance.set(requestSensors);
+
+        // Initialize internode bytes with the inbound message size:
+        for (TableMetadata tm : tables)
+        {
+            Context context = Context.from(tm);
+            requestSensors.registerSensor(context, Type.INTERNODE_BYTES);
+            requestSensors.incrementSensor(context, Type.INTERNODE_BYTES, message.payloadSize(MessagingService.current_version) / tables.size());
+        }
+
         String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
         // We should not wait for the result of the write in this thread,
         // otherwise we could have a distributed deadlock between replicas
@@ -49,7 +72,7 @@ protected void applyMutation(final Message<CounterMutation> message, InetAddress
         // it's own in that case.
         StorageProxy.applyCounterMutationOnLeader(cm,
                                                   localDataCenter,
-                                                  () -> MessagingService.instance().send(message.emptyResponse(), respondToAddress),
+                                                  new CounterMutationCallback(message, message.from(), requestSensors),
                                                   Dispatcher.RequestTime.forImmediateExecution());
     }
 }
diff --git a/src/java/org/apache/cassandra/db/DataRange.java b/src/java/org/apache/cassandra/db/DataRange.java
index 9912ac56e919..98a9b2423028 100644
--- a/src/java/org/apache/cassandra/db/DataRange.java
+++ b/src/java/org/apache/cassandra/db/DataRange.java
@@ -347,8 +347,8 @@ public static void appendKeyString(StringBuilder sb, AbstractType<?> type, ByteB
         {
             CompositeType ct = (CompositeType)type;
             ByteBuffer[] values = ct.split(key);
-            for (int i = 0; i < ct.types.size(); i++)
-                sb.append(i == 0 ? "" : ", ").append(ct.types.get(i).toCQLString(values[i]));
+            for (int i = 0; i < ct.subTypes.size(); i++)
+                sb.append(i == 0 ? "" : ", ").append(ct.subTypes.get(i).toCQLString(values[i]));
         }
         else
         {
diff --git a/src/java/org/apache/cassandra/db/DecoratedKey.java b/src/java/org/apache/cassandra/db/DecoratedKey.java
index 03d6374112a4..3271558b9d7e 100644
--- a/src/java/org/apache/cassandra/db/DecoratedKey.java
+++ b/src/java/org/apache/cassandra/db/DecoratedKey.java
@@ -76,6 +76,7 @@ public boolean equals(Object obj)
         return ByteBufferUtil.compareUnsigned(getKey(), other.getKey()) == 0; // we compare faster than BB.equals for array backed BB
     }
 
+    @Override
     public int compareTo(PartitionPosition pos)
     {
         if (this == pos)
@@ -106,7 +107,7 @@ public ByteSource asComparableBytes(Version version)
     {
         // Note: In the legacy version one encoding could be a prefix of another as the escaping is only weakly
         // prefix-free (see ByteSourceTest.testDecoratedKeyPrefixes()).
-        // The OSS50 version avoids this by adding a terminator.
+        // The OSS41 and 50 versions avoids this by adding a terminator.
         return ByteSource.withTerminatorMaybeLegacy(version,
                                                     ByteSource.END_OF_STREAM,
                                                     token.asComparableBytes(version),
@@ -232,6 +233,14 @@ static <T extends DecoratedKey> T fromByteComparable(ByteComparable byteComparab
         return decoratedKeyFactory.apply(token, keyBytes);
     }
 
+    public static byte[] keyFromByteComparable(ByteComparable byteComparable,
+                                               Version version,
+                                               IPartitioner partitioner)
+    {
+        return keyFromByteSource(ByteSource.peekable(byteComparable.asComparableBytes(version)), version, partitioner);
+    }
+
+
     public static byte[] keyFromByteSource(ByteSource.Peekable peekableByteSource,
                                            Version version,
                                            IPartitioner partitioner)
diff --git a/src/java/org/apache/cassandra/db/DeletionInfo.java b/src/java/org/apache/cassandra/db/DeletionInfo.java
index bbc4eee95056..ecdea45cd4d5 100644
--- a/src/java/org/apache/cassandra/db/DeletionInfo.java
+++ b/src/java/org/apache/cassandra/db/DeletionInfo.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.db;
 
 import java.util.Iterator;
+import java.util.SortedSet;
 
 import org.apache.cassandra.cache.IMeasurableMemory;
 import org.apache.cassandra.db.rows.EncodingStats;
@@ -52,6 +53,8 @@ public interface DeletionInfo extends IMeasurableMemory
 
     public Iterator<RangeTombstone> rangeIterator(Slice slice, boolean reversed);
 
+    public Iterator<RangeTombstone> rangeIterator(SortedSet<Clustering<?>> names, boolean isRevered);
+
     public RangeTombstone rangeCovering(Clustering<?> name);
 
     public void collectStats(EncodingStats.Collector collector);
@@ -72,4 +75,6 @@ public interface DeletionInfo extends IMeasurableMemory
     public MutableDeletionInfo mutableCopy();
 
     public DeletionInfo clone(ByteBufferCloner cloner);
+
+    public RangeTombstoneList copyRanges(ByteBufferCloner cloner);
 }
diff --git a/src/java/org/apache/cassandra/db/DeletionTime.java b/src/java/org/apache/cassandra/db/DeletionTime.java
index 5970fbb042a4..5c8153886604 100644
--- a/src/java/org/apache/cassandra/db/DeletionTime.java
+++ b/src/java/org/apache/cassandra/db/DeletionTime.java
@@ -45,7 +45,7 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
      */
     public static final DeletionTime LIVE = new DeletionTime(Long.MIN_VALUE, Long.MAX_VALUE);
 
-    private static final Serializer serializer = new Serializer();
+    public static final Serializer serializer = new Serializer();
     private static final Serializer legacySerializer = new LegacySerializer();
 
     private final long markedForDeleteAt;
@@ -143,7 +143,7 @@ public final int hashCode()
     @Override
     public String toString()
     {
-        return String.format("deletedAt=%d, localDeletion=%d", markedForDeleteAt(), localDeletionTime());
+        return this == LIVE ? "LIVE" : String.format("deletedAt=%d, localDeletion=%d", markedForDeleteAt(), localDeletionTime());
     }
 
     public int compareTo(DeletionTime dt)
@@ -160,6 +160,11 @@ public boolean supersedes(DeletionTime dt)
         return markedForDeleteAt() > dt.markedForDeleteAt() || (markedForDeleteAt() == dt.markedForDeleteAt() && localDeletionTime() > dt.localDeletionTime());
     }
 
+    public static DeletionTime merge(DeletionTime d1, DeletionTime d2)
+    {
+        return d2.supersedes(d1) ? d2 : d1;
+    }
+
     public boolean deletes(LivenessInfo info)
     {
         return deletes(info.timestamp());
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 5ef59adc37fa..5a8f75a75a4a 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -45,8 +45,10 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
@@ -67,10 +69,12 @@
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableId;
 import org.apache.cassandra.io.sstable.SSTableIdFactory;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileStoreUtils;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.snapshot.SnapshotManifest;
@@ -132,29 +136,28 @@ public class Directories
      * the details if it does not.
      *
      * @param dir File object of the directory.
-     * @param dataDir String representation of the directory's location
      * @return status representing Cassandra's RWX permissions to the supplied folder location.
      */
-    public static boolean verifyFullPermissions(File dir, String dataDir)
+    public static boolean verifyFullPermissions(File dir)
     {
         if (!dir.isDirectory())
         {
-            logger.error("Not a directory {}", dataDir);
+            logger.error("Not a directory {}", dir);
             return false;
         }
         else if (!FileAction.hasPrivilege(dir, FileAction.X))
         {
-            logger.error("Doesn't have execute permissions for {} directory", dataDir);
+            logger.error("Doesn't have execute permissions for {} directory", dir);
             return false;
         }
         else if (!FileAction.hasPrivilege(dir, FileAction.R))
         {
-            logger.error("Doesn't have read permissions for {} directory", dataDir);
+            logger.error("Doesn't have read permissions for {} directory", dir);
             return false;
         }
         else if (dir.exists() && !FileAction.hasPrivilege(dir, FileAction.W))
         {
-            logger.error("Doesn't have write permissions for {} directory", dataDir);
+            logger.error("Doesn't have write permissions for {} directory", dir);
             return false;
         }
 
@@ -211,9 +214,19 @@ public Directories(final TableMetadata metadata)
         this(metadata, dataDirectories.getDataDirectoriesFor(metadata));
     }
 
+    public Directories(final KeyspaceMetadata ksMetadata, final TableMetadata metadata)
+    {
+        this(ksMetadata, metadata, dataDirectories.getDataDirectoriesFor(metadata));
+    }
+
     public Directories(final TableMetadata metadata, Collection<DataDirectory> paths)
     {
-        this(metadata, paths.toArray(new DataDirectory[paths.size()]));
+        this(null, metadata, paths.toArray(new DataDirectory[paths.size()]));
+    }
+
+    public Directories(final TableMetadata metadata, DataDirectory[] paths)
+    {
+        this(null, metadata, paths);
     }
 
     /**
@@ -222,10 +235,11 @@ public Directories(final TableMetadata metadata, Collection<DataDirectory> paths
      *
      * @param metadata metadata of ColumnFamily
      */
-    public Directories(final TableMetadata metadata, DataDirectory[] paths)
+    public Directories(@Nullable KeyspaceMetadata ksMetadata, final TableMetadata metadata, DataDirectory[] dirs)
     {
         this.metadata = metadata;
-        this.paths = paths;
+        this.paths = StorageProvider.instance.createDataDirectories(ksMetadata, metadata, dirs);
+
         ImmutableMap.Builder<Path, DataDirectory> canonicalPathsBuilder = ImmutableMap.builder();
         String tableId = metadata.id.toHexString();
         int idx = metadata.name.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
@@ -297,13 +311,33 @@ public Directories(final TableMetadata metadata, DataDirectory[] paths)
                 {
                     File destFile = new File(dataPath, indexFile.name());
                     logger.trace("Moving index file {} to {}", indexFile, destFile);
-                    FileUtils.renameWithConfirm(indexFile, destFile);
+                    indexFile.move(destFile);
                 }
             }
         }
         canonicalPathToDD = canonicalPathsBuilder.build();
     }
 
+    /**
+     * A special constructor used to mock SSTables for CNDB tests.
+     *
+     * This constructor fixes the data path and path to whichever directory is passed in. No other manipulations
+     * to the data paths are performed, unlike in the other constructors. The directory should therefore already
+     * contain information related to the keyspace and table, whether it is local or remote.
+     */
+    @VisibleForTesting
+    public Directories(final TableMetadata metadata, Path directory)
+    {
+        ImmutableMap.Builder<Path, DataDirectory> canonicalPathsBuilder = ImmutableMap.builder();
+
+        this.metadata = metadata;
+        this.paths = new DataDirectory[] { new DataDirectory(directory) };
+        this.dataPaths = new File[] { paths[0].location };
+        
+        canonicalPathsBuilder.put(dataPaths[0].toCanonical().toPath(), paths[0]);
+        this.canonicalPathToDD = canonicalPathsBuilder.build();
+    }
+
     /**
      * Returns SSTable location which is inside given data directory.
      *
@@ -331,6 +365,12 @@ public DataDirectory getDataDirectoryForFile(Descriptor descriptor)
         return null;
     }
 
+    /**
+     * This method looks for the file name passed in and resolves it into a descriptor
+     * if the file exists.
+     *
+     * @return a descriptor for the file name passed in
+     */
     public Descriptor find(String filename)
     {
         for (File dir : dataPaths)
@@ -342,6 +382,20 @@ public Descriptor find(String filename)
         return null;
     }
 
+    /**
+     * This method resolves the filename against the specified directory number, whether
+     * the file exists or not.
+     *
+     * @return a descriptor for the passed in filename
+     */
+    public Descriptor resolve(String filename, int dirNumber)
+    {
+        Preconditions.checkArgument(dirNumber < dataPaths.length, "Invalid dir number: " + dirNumber);
+        File dir = dataPaths[dirNumber];
+        File file = dir.resolve(filename);
+        return Descriptor.fromFile(file);
+    }
+
     /**
      * Basically the same as calling {@link #getWriteableLocationAsFile(long)} with an unknown size ({@code -1L}),
      * which may return any allowed directory - even a data directory that has no usable space.
@@ -791,6 +845,20 @@ public long getRawSize()
             return FileUtils.folderSize(location);
         }
 
+        // Used by CNDB
+        @VisibleForTesting
+        public long getTotalSpace()
+        {
+            return PathUtils.tryGetSpace(location.toPath(), FileStore::getTotalSpace);
+        }
+
+        // Used by CNDB
+        @VisibleForTesting
+        public long getSpaceUsed()
+        {
+            return getTotalSpace() - getAvailableSpace();
+        }
+
         @Override
         public boolean equals(Object o)
         {
@@ -832,17 +900,17 @@ public static final class DataDirectories implements Iterable<DataDirectory>
         private final DataDirectory[] nonLocalSystemKeyspacesDirectories;
 
 
-        public DataDirectories(String[] locationsForNonSystemKeyspaces, String[] locationsForSystemKeyspace)
+        public DataDirectories(File[] locationsForNonSystemKeyspaces, File[] locationsForSystemKeyspace)
         {
             nonLocalSystemKeyspacesDirectories = toDataDirectories(locationsForNonSystemKeyspaces);
             localSystemKeyspaceDataDirectories = toDataDirectories(locationsForSystemKeyspace);
         }
 
-        private static DataDirectory[] toDataDirectories(String... locations)
+        private static DataDirectory[] toDataDirectories(File... locations)
         {
             DataDirectory[] directories = new DataDirectory[locations.length];
             for (int i = 0; i < locations.length; ++i)
-                directories[i] = new DataDirectory(new File(locations[i]));
+                directories[i] = new DataDirectory(locations[i]);
             return directories;
         }
 
@@ -953,7 +1021,10 @@ public enum FileType
         TEMPORARY,
 
         /** A transaction log file (contains information on final and temporary files). */
-        TXN_LOG;
+        TXN_LOG,
+
+        /** An sstable file that was marked for deletion */
+        OBSOLETE;
     }
 
     /**
@@ -1121,6 +1192,7 @@ private BiPredicate<File, FileType> getFilter(boolean includeForeignTables)
                 switch (type)
                 {
                     case TXN_LOG:
+                    case OBSOLETE:
                         return false;
                     case TEMPORARY:
                         if (skipTemporary)
@@ -1162,7 +1234,7 @@ private BiPredicate<File, FileType> getFilter(boolean includeForeignTables)
                         return false;
 
                     default:
-                        throw new AssertionError();
+                        throw new AssertionError("unexpected file type: " + type + " for file " + file);
                 }
             };
         }
diff --git a/src/java/org/apache/cassandra/db/DisallowedDirectories.java b/src/java/org/apache/cassandra/db/DisallowedDirectories.java
index e666bad78599..2f0e354a0c70 100644
--- a/src/java/org/apache/cassandra/db/DisallowedDirectories.java
+++ b/src/java/org/apache/cassandra/db/DisallowedDirectories.java
@@ -27,6 +27,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.utils.MBeanWrapper;
 
@@ -84,6 +85,9 @@ public void markUnwritable(String path)
      */
     public static File maybeMarkUnreadable(File path)
     {
+        if (!DatabaseDescriptor.supportsBlacklistingDirectory())
+            return null;
+
         File directory = getDirectory(path);
         if (instance.unreadableDirectories.add(directory))
         {
@@ -102,6 +106,9 @@ public static File maybeMarkUnreadable(File path)
      */
     public static File maybeMarkUnwritable(File path)
     {
+        if (!DatabaseDescriptor.supportsBlacklistingDirectory())
+            return null;
+
         File directory = getDirectory(path);
         if (instance.unwritableDirectories.add(directory))
         {
diff --git a/src/java/org/apache/cassandra/db/DiskBoundaries.java b/src/java/org/apache/cassandra/db/DiskBoundaries.java
index 7fe10f4c1336..c19b031366a0 100644
--- a/src/java/org/apache/cassandra/db/DiskBoundaries.java
+++ b/src/java/org/apache/cassandra/db/DiskBoundaries.java
@@ -21,34 +21,48 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import javax.annotation.Nullable;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.service.StorageService;
 
 public class DiskBoundaries
 {
-    public final List<Directories.DataDirectory> directories;
-    public final ImmutableList<PartitionPosition> positions;
-    final long ringVersion;
+    @Nullable public final List<Directories.DataDirectory> directories;
+    /**
+     * End-inclusive list of boundaries between directories.
+     * I.e. directories[0] covers [min, positions[0]]
+     *      directories[1] covers (positions[0], positions[1]]
+     *      ...
+     *      directories[last] covers (positions[last-1], positions[last]==max]
+     */
+    @Nullable private final ImmutableList<Token> positions;
+    public final SortedLocalRanges localRanges;
     final int directoriesVersion;
     private final ColumnFamilyStore cfs;
     private volatile boolean isInvalid = false;
 
-    public DiskBoundaries(ColumnFamilyStore cfs, Directories.DataDirectory[] directories, int diskVersion)
+    public DiskBoundaries(ColumnFamilyStore cfs,
+                          @Nullable Directories.DataDirectory[] directories,
+                          SortedLocalRanges localRanges,
+                          int diskVersion)
     {
-        this(cfs, directories, null, -1, diskVersion);
+        this(cfs, directories, null, localRanges, diskVersion);
     }
 
-    @VisibleForTesting
-    public DiskBoundaries(ColumnFamilyStore cfs, Directories.DataDirectory[] directories, List<PartitionPosition> positions, long ringVersion, int diskVersion)
+    public DiskBoundaries(ColumnFamilyStore cfs,
+                          @Nullable Directories.DataDirectory[] directories,
+                          @Nullable List<Token> positions,
+                          SortedLocalRanges localRanges,
+                          int diskVersion)
     {
         this.directories = directories == null ? null : ImmutableList.copyOf(directories);
         this.positions = positions == null ? null : ImmutableList.copyOf(positions);
-        this.ringVersion = ringVersion;
+        this.localRanges = localRanges;
         this.directoriesVersion = diskVersion;
         this.cfs = cfs;
     }
@@ -60,17 +74,17 @@ public boolean equals(Object o)
 
         DiskBoundaries that = (DiskBoundaries) o;
 
-        if (ringVersion != that.ringVersion) return false;
-        if (directoriesVersion != that.directoriesVersion) return false;
-        if (!directories.equals(that.directories)) return false;
-        return positions != null ? positions.equals(that.positions) : that.positions == null;
+        return Objects.equals(localRanges, that.localRanges) &&
+               directoriesVersion == that.directoriesVersion &&
+               Objects.equals(directories, that.directories) &&
+               Objects.equals(positions, that.positions);
     }
 
     public int hashCode()
     {
         int result = directories != null ? directories.hashCode() : 0;
         result = 31 * result + (positions != null ? positions.hashCode() : 0);
-        result = 31 * result + (int) (ringVersion ^ (ringVersion >>> 32));
+        result = 31 * result + localRanges.hashCode();
         result = 31 * result + directoriesVersion;
         return result;
     }
@@ -80,7 +94,7 @@ public String toString()
         return "DiskBoundaries{" +
                "directories=" + directories +
                ", positions=" + positions +
-               ", ringVersion=" + ringVersion +
+               ", localRanges=" + localRanges.toString() +
                ", directoriesVersion=" + directoriesVersion +
                '}';
     }
@@ -92,9 +106,9 @@ public boolean isOutOfDate()
     {
         if (isInvalid)
             return true;
+
         int currentDiskVersion = DisallowedDirectories.getDirectoriesVersion();
-        long currentRingVersion = StorageService.instance.getTokenMetadata().getRingVersion();
-        return currentDiskVersion != directoriesVersion || (ringVersion != -1 && currentRingVersion != ringVersion);
+        return currentDiskVersion != directoriesVersion || localRanges.isOutOfDate();
     }
 
     public void invalidate()
@@ -102,16 +116,15 @@ public void invalidate()
         this.isInvalid = true;
     }
 
-    public int getDiskIndex(SSTableReader sstable)
+    public int getDiskIndexFromKey(CompactionSSTable sstable)
     {
         if (positions == null)
         {
-            return getBoundariesFromSSTableDirectory(sstable.descriptor);
+            return getBoundariesFromSSTableDirectory(sstable.getDescriptor());
         }
 
-        int pos = Collections.binarySearch(positions, sstable.getFirst());
-        assert pos < 0; // boundaries are .minkeybound and .maxkeybound so they should never be equal
-        return -pos - 1;
+        int pos = Collections.binarySearch(positions, sstable.getFirst().getToken());
+        return pos >= 0 ? pos : -pos - 1;   // disk boundaries are end-inclusive
     }
 
     /**
@@ -131,7 +144,7 @@ public int getBoundariesFromSSTableDirectory(Descriptor descriptor)
 
     public Directories.DataDirectory getCorrectDiskForSSTable(SSTableReader sstable)
     {
-        return directories.get(getDiskIndex(sstable));
+        return directories.get(getDiskIndexFromKey(sstable));
     }
 
     public Directories.DataDirectory getCorrectDiskForKey(DecoratedKey key)
@@ -139,29 +152,40 @@ public Directories.DataDirectory getCorrectDiskForKey(DecoratedKey key)
         if (positions == null)
             return null;
 
-        return directories.get(getDiskIndex(key));
+        return directories.get(getDiskIndexFromKey(key));
     }
 
     public boolean isInCorrectLocation(SSTableReader sstable, Directories.DataDirectory currentLocation)
     {
-        int diskIndex = getDiskIndex(sstable);
-        PartitionPosition diskLast = positions.get(diskIndex);
-        return directories.get(diskIndex).equals(currentLocation) && sstable.getLast().compareTo(diskLast) <= 0;
+        int diskIndex = getDiskIndexFromKey(sstable);
+        Token diskLast = positions.get(diskIndex);
+        return directories.get(diskIndex).equals(currentLocation) && sstable.last.getToken().compareTo(diskLast) <= 0;
     }
 
-    private int getDiskIndex(DecoratedKey key)
+    /**
+     * Return the number of boundaries. If this instance was created with token boundaries (positions) then this
+     * is the number of boundaries. If this instance was created without boundaries but only with directories, then
+     * this is the number of directories.
+     *
+     * @return the number of boundaries.
+     */
+    public int getNumBoundaries()
     {
-        int pos = Collections.binarySearch(positions, key);
-        assert pos < 0;
-        return -pos - 1;
+        return positions == null ? directories.size() : positions.size();
+    }
+
+    private int getDiskIndexFromKey(DecoratedKey key)
+    {
+        int pos = Collections.binarySearch(positions, key.getToken());
+        return pos >= 0 ? pos : -pos - 1;   // disk boundaries are end-inclusive
     }
 
     public List<Directories.DataDirectory> getDisksInBounds(DecoratedKey first, DecoratedKey last)
     {
         if (positions == null || first == null || last == null)
             return directories;
-        int firstIndex = getDiskIndex(first);
-        int lastIndex = getDiskIndex(last);
+        int firstIndex = getDiskIndexFromKey(first);
+        int lastIndex = getDiskIndexFromKey(last);
         return directories.subList(firstIndex, lastIndex + 1);
     }
 
@@ -171,4 +195,27 @@ public boolean isEquivalentTo(DiskBoundaries oldBoundaries)
                Objects.equals(positions, oldBoundaries.positions) &&
                Objects.equals(directories, oldBoundaries.directories);
     }
+    
+    /**
+     * Return the local sorted ranges, which contain the local ranges for this node, sorted.
+     * See {@link SortedLocalRanges}.
+     *
+     * @return the local ranges, see {@link SortedLocalRanges}.
+     */
+    public SortedLocalRanges getLocalRanges()
+    {
+        return localRanges;
+    }
+
+    /**
+     * Returns a non-modifiable list of the disk boundary positions. This will be null if the token space is not split
+     * for the disks, this is not normally the case).
+     *
+     * Extracted as a method (instead of direct access to the final field) to permit mocking in tests.
+     */
+    @Nullable
+    public List<Token> getPositions()
+    {
+        return positions;
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/DiskBoundaryManager.java b/src/java/org/apache/cassandra/db/DiskBoundaryManager.java
index 7857d0cff888..2ce5299078e3 100644
--- a/src/java/org/apache/cassandra/db/DiskBoundaryManager.java
+++ b/src/java/org/apache/cassandra/db/DiskBoundaryManager.java
@@ -18,8 +18,6 @@
 
 package org.apache.cassandra.db;
 
-import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
 
 import org.slf4j.Logger;
@@ -27,14 +25,8 @@
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.locator.RangesAtEndpoint;
-import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.service.PendingRangeCalculatorService;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
 
 public class DiskBoundaryManager
 {
@@ -43,18 +35,21 @@ public class DiskBoundaryManager
 
     public DiskBoundaries getDiskBoundaries(ColumnFamilyStore cfs)
     {
-        if (!cfs.getPartitioner().splitter().isPresent())
-            return new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(), DisallowedDirectories.getDirectoriesVersion());
         if (diskBoundaries == null || diskBoundaries.isOutOfDate())
         {
             synchronized (this)
             {
                 if (diskBoundaries == null || diskBoundaries.isOutOfDate())
                 {
-                    logger.debug("Refreshing disk boundary cache for {}.{}", cfs.getKeyspaceName(), cfs.getTableName());
+                    logger.debug("Refreshing disk boundary cache for {}.{}", cfs.keyspace.getName(), cfs.getTableName());
+                    SortedLocalRanges localRanges = cfs.getLocalRanges();
+
                     DiskBoundaries oldBoundaries = diskBoundaries;
-                    diskBoundaries = getDiskBoundaryValue(cfs);
-                    logger.debug("Updating boundaries from {} to {} for {}.{}", oldBoundaries, diskBoundaries, cfs.getKeyspaceName(), cfs.getTableName());
+                    diskBoundaries = !cfs.getPartitioner().splitter().isPresent()
+                                     ? new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(), localRanges, DisallowedDirectories.getDirectoriesVersion())
+                                     : getDiskBoundaryValue(cfs, localRanges);
+
+                    logger.debug("Updating boundaries from {} to {} for {}.{}", oldBoundaries, diskBoundaries, cfs.keyspace.getName(), cfs.getTableName());
                 }
             }
         }
@@ -67,43 +62,9 @@ public void invalidate()
            diskBoundaries.invalidate();
     }
 
-    static class VersionedRangesAtEndpoint
-    {
-        public final RangesAtEndpoint rangesAtEndpoint;
-        public final long ringVersion;
-
-        VersionedRangesAtEndpoint(RangesAtEndpoint rangesAtEndpoint, long ringVersion)
-        {
-            this.rangesAtEndpoint = rangesAtEndpoint;
-            this.ringVersion = ringVersion;
-        }
-    }
-
-    public static VersionedRangesAtEndpoint getVersionedLocalRanges(ColumnFamilyStore cfs)
-    {
-        RangesAtEndpoint localRanges;
-
-        long ringVersion;
-        TokenMetadata tmd;
-        do
-        {
-            tmd = StorageService.instance.getTokenMetadata();
-            ringVersion = tmd.getRingVersion();
-            localRanges = getLocalRanges(cfs, tmd);
-            logger.debug("Got local ranges {} (ringVersion = {})", localRanges, ringVersion);
-        }
-        while (ringVersion != tmd.getRingVersion()); // if ringVersion is different here it means that
-        // it might have changed before we calculated localRanges - recalculate
-
-        return new VersionedRangesAtEndpoint(localRanges, ringVersion);
-    }
 
-    private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs)
+    private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs, SortedLocalRanges localRanges)
     {
-        VersionedRangesAtEndpoint rangesAtEndpoint = getVersionedLocalRanges(cfs);
-        RangesAtEndpoint localRanges = rangesAtEndpoint.rangesAtEndpoint;
-        long ringVersion = rangesAtEndpoint.ringVersion;
-
         int directoriesVersion;
         Directories.DataDirectory[] dirs;
         do
@@ -113,31 +74,11 @@ private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs)
         }
         while (directoriesVersion != DisallowedDirectories.getDirectoriesVersion()); // if directoriesVersion has changed we need to recalculate
 
-        if (localRanges == null || localRanges.isEmpty())
-            return new DiskBoundaries(cfs, dirs, null, ringVersion, directoriesVersion);
-
-        List<PartitionPosition> positions = getDiskBoundaries(localRanges, cfs.getPartitioner(), dirs);
-
-        return new DiskBoundaries(cfs, dirs, positions, ringVersion, directoriesVersion);
-    }
+        if (localRanges == null || localRanges.getRanges().isEmpty())
+            return new DiskBoundaries(cfs, dirs, null, localRanges, directoriesVersion);
 
-    private static RangesAtEndpoint getLocalRanges(ColumnFamilyStore cfs, TokenMetadata tmd)
-    {
-        RangesAtEndpoint localRanges;
-        if (StorageService.instance.isBootstrapMode()
-        && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
-        {
-            PendingRangeCalculatorService.instance.blockUntilFinished();
-            localRanges = tmd.getPendingRanges(cfs.getKeyspaceName(), FBUtilities.getBroadcastAddressAndPort());
-        }
-        else
-        {
-            // Reason we use use the future settled TMD is that if we decommission a node, we want to stream
-            // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places.
-            // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled
-            localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort());
-        }
-        return localRanges;
+        List<Token> positions = getDiskBoundaries(localRanges.getRanges(), cfs.getPartitioner(), dirs);
+        return new DiskBoundaries(cfs, dirs, positions, localRanges, directoriesVersion);
     }
 
     /**
@@ -149,32 +90,15 @@ private static RangesAtEndpoint getLocalRanges(ColumnFamilyStore cfs, TokenMetad
      *
      * The final entry in the returned list will always be the partitioner maximum tokens upper key bound
      */
-    private static List<PartitionPosition> getDiskBoundaries(RangesAtEndpoint replicas, IPartitioner partitioner, Directories.DataDirectory[] dataDirectories)
+    private static List<Token> getDiskBoundaries(List<Splitter.WeightedRange> weightedRanges, IPartitioner partitioner, Directories.DataDirectory[] dataDirectories)
     {
         assert partitioner.splitter().isPresent();
 
         Splitter splitter = partitioner.splitter().get();
-        boolean dontSplitRanges = DatabaseDescriptor.getNumTokens() > 1;
-
-        List<Splitter.WeightedRange> weightedRanges = new ArrayList<>(replicas.size());
-        // note that Range.sort unwraps any wraparound ranges, so we need to sort them here
-        for (Range<Token> r : Range.sort(replicas.onlyFull().ranges()))
-            weightedRanges.add(new Splitter.WeightedRange(1.0, r));
-
-        for (Range<Token> r : Range.sort(replicas.onlyTransient().ranges()))
-            weightedRanges.add(new Splitter.WeightedRange(0.1, r));
+        Splitter.SplitType splitType = DatabaseDescriptor.getNumTokens() > 1 ? Splitter.SplitType.PREFER_WHOLE : Splitter.SplitType.ALWAYS_SPLIT;
 
-        weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
-
-        List<Token> boundaries = splitter.splitOwnedRanges(dataDirectories.length, weightedRanges, dontSplitRanges);
-        // If we can't split by ranges, split evenly to ensure utilisation of all disks
-        if (dontSplitRanges && boundaries.size() < dataDirectories.length)
-            boundaries = splitter.splitOwnedRanges(dataDirectories.length, weightedRanges, false);
-
-        List<PartitionPosition> diskBoundaries = new ArrayList<>();
-        for (int i = 0; i < boundaries.size() - 1; i++)
-            diskBoundaries.add(boundaries.get(i).maxKeyBound());
-        diskBoundaries.add(partitioner.getMaximumToken().maxKeyBound());
-        return diskBoundaries;
+        List<Token> boundaries = splitter.splitOwnedRanges(dataDirectories.length, weightedRanges, splitType).boundaries;
+        assert boundaries.size() == dataDirectories.length : "Wrong number of boundaries for directories: " + boundaries.size();
+        return boundaries;
     }
 }
diff --git a/src/java/org/apache/cassandra/db/IMutation.java b/src/java/org/apache/cassandra/db/IMutation.java
index 1998e2c0353c..c392d9b50f25 100644
--- a/src/java/org/apache/cassandra/db/IMutation.java
+++ b/src/java/org/apache/cassandra/db/IMutation.java
@@ -38,6 +38,7 @@ public interface IMutation
     String toString(boolean shallow);
     Collection<PartitionUpdate> getPartitionUpdates();
     Supplier<Mutation> hintOnFailure();
+    Keyspace getKeyspace();
 
     default void validateIndexedColumns(ClientState state)
     {
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 05b354a74bad..98fdd28e25bd 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -31,6 +31,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.Lock;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -47,11 +48,14 @@
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.repair.CassandraKeyspaceRepairManager;
 import org.apache.cassandra.db.view.ViewManager;
+import org.apache.cassandra.exceptions.InternalRequestExecutionException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.UnknownKeyspaceException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.SecondaryIndexManager;
-import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.metrics.KeyspaceMetrics;
 import org.apache.cassandra.repair.KeyspaceRepairManager;
@@ -67,11 +71,12 @@
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.OpOrder;
-import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 import org.apache.cassandra.utils.concurrent.Promise;
+import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -106,6 +111,9 @@ public class Keyspace
     //Keyspaces in the case of Views (batchlog of view mutations)
     public static final OpOrder writeOrder = new OpOrder();
 
+    // Set during draining to indicate that no more mutations should be accepted
+    private volatile OpOrder.Barrier writeBarrier = null;
+
     /* ColumnFamilyStore per column family */
     private final ConcurrentMap<TableId, ColumnFamilyStore> columnFamilyStores = new ConcurrentHashMap<>();
 
@@ -152,24 +160,27 @@ public static Keyspace open(String keyspaceName)
     }
 
     // to only be used by org.apache.cassandra.tools.Standalone* classes
-    public static Keyspace openWithoutSSTables(String keyspaceName)
+    public static Keyspace openWithoutSSTables(String keyspaceName) throws UnknownKeyspaceException
     {
         return open(keyspaceName, Schema.instance, false);
     }
 
-    public static Keyspace open(String keyspaceName, SchemaProvider schema, boolean loadSSTables)
+    public static Keyspace open(String keyspaceName, SchemaProvider schema, boolean loadSSTables) throws UnknownKeyspaceException
     {
-        return schema.maybeAddKeyspaceInstance(keyspaceName, () -> new Keyspace(keyspaceName, schema, loadSSTables));
+        return schema.maybeAddKeyspaceInstance(keyspaceName, () -> {
+            logger.debug("New instance created for keyspace {}", keyspaceName);
+            return new Keyspace(keyspaceName, schema, loadSSTables);
+        });
     }
 
     public static ColumnFamilyStore openAndGetStore(TableMetadataRef tableRef)
     {
-        return open(tableRef.keyspace).getColumnFamilyStore(tableRef.id);
+        return open(tableRef.keyspace).getColumnFamilyStore(tableRef.get());
     }
 
     public static ColumnFamilyStore openAndGetStore(TableMetadata table)
     {
-        return open(table.keyspace).getColumnFamilyStore(table.id);
+        return open(table.keyspace).getColumnFamilyStore(table);
     }
 
     public static ColumnFamilyStore openAndGetStoreIfExists(TableMetadata table)
@@ -217,14 +228,33 @@ public ColumnFamilyStore getColumnFamilyStore(String cfName)
         TableMetadata table = schema.getTableMetadata(getName(), cfName);
         if (table == null)
             throw new IllegalArgumentException(String.format("Unknown keyspace/cf pair (%s.%s)", getName(), cfName));
-        return getColumnFamilyStore(table.id);
+        return getColumnFamilyStore(table);
+    }
+
+    public ColumnFamilyStore getColumnFamilyStore(TableMetadata table)
+    {
+        return getColumnFamilyStore(table.id,
+                                    () -> String.format("Cannot find table %s.%s with id %s, it may have been dropped",
+                                                        getName(), table.name, table.id));
     }
 
     public ColumnFamilyStore getColumnFamilyStore(TableId id)
+    {
+        return getColumnFamilyStore(id,
+                                    () -> String.format("Cannot find table with id %s in keyspace %s, it may have been dropped",
+                                                        id, getName()));
+    }
+
+    private ColumnFamilyStore getColumnFamilyStore(TableId id, Supplier<String> errorMsg)
     {
         ColumnFamilyStore cfs = columnFamilyStores.get(id);
         if (cfs == null)
-            throw new IllegalArgumentException("Unknown CF " + id);
+        {
+            // We log a more detailed error message here rather than complicating the client facing exception message
+            logger.error(errorMsg.get());
+            throw new IllegalArgumentException("Cannot find table, it may have been dropped. Table id" + id);
+        }
+
         return cfs;
     }
 
@@ -330,12 +360,13 @@ public Stream<TableSnapshot> getAllSnapshots()
         return getColumnFamilyStores().stream().flatMap(cfs -> cfs.listSnapshots().values().stream());
     }
 
-    private Keyspace(String keyspaceName, SchemaProvider schema, boolean loadSSTables)
+    private Keyspace(String keyspaceName, SchemaProvider schema, boolean loadSSTables) throws UnknownKeyspaceException
     {
         this.schema = schema;
         metadata = schema.getKeyspaceMetadata(keyspaceName);
-        assert metadata != null : "Unknown keyspace " + keyspaceName;
-        
+        if (metadata == null)
+            throw new UnknownKeyspaceException(keyspaceName);
+
         if (metadata.isVirtual())
             throw new IllegalStateException("Cannot initialize Keyspace with virtual metadata " + keyspaceName);
         createReplicationStrategy(metadata);
@@ -376,12 +407,12 @@ public static Keyspace mockKS(KeyspaceMetadata metadata)
 
     private void createReplicationStrategy(KeyspaceMetadata ksm)
     {
-        logger.info("Creating replication strategy " + ksm.name + " params " + ksm.params);
+        logger.debug("Creating replication strategy " + ksm.name + " params " + ksm.params);
         replicationStrategy = ksm.createReplicationStrategy();
         if (!ksm.params.replication.equals(replicationParams))
         {
             logger.debug("New replication settings for keyspace {} - invalidating disk boundary caches", ksm.name);
-            columnFamilyStores.values().forEach(ColumnFamilyStore::invalidateLocalRanges);
+            columnFamilyStores.values().forEach(ColumnFamilyStore::invalidateLocalRangesAndDiskBoundaries);
         }
         replicationParams = ksm.params.replication;
     }
@@ -391,10 +422,18 @@ public void dropCf(TableId tableId, boolean dropData)
     {
         ColumnFamilyStore cfs = columnFamilyStores.remove(tableId);
         if (cfs == null)
+        {
+            logger.debug("No CFS found when trying to drop table {}, {}", tableId, schema.getTableMetadata(tableId).name);
             return;
+        }
 
         cfs.onTableDropped();
+
+        if (logger.isTraceEnabled())
+            logger.trace("Dropping CFS {}: unloading CFS", cfs.name);
         unloadCf(cfs, dropData);
+        if (logger.isTraceEnabled())
+            logger.trace("Dropping CFS {}: completed", cfs.name);
     }
 
     /**
@@ -407,11 +446,35 @@ public void unload(boolean dropData)
         metric.release();
     }
 
-    // disassociate a cfs from this keyspace instance.
+    /**
+     * Unload the column family. For online services, it will also flush beforehand.
+     *
+     * Because this method is called by schema operations, it will not throw in case
+     * of failures, but just log an error.
+     *
+     * @param cfs the table to unload
+     * @param dropData true when data should also be dropped
+     */
     private void unloadCf(ColumnFamilyStore cfs, boolean dropData)
     {
-        cfs.unloadCf();
-        cfs.invalidate(true, dropData);
+        logger.debug("Unloading column family store for table {} with dropData={}", cfs.metadata, dropData);
+
+        Throwable err = null;
+
+        // offline services (e.g. standalone compactor) don't have Memtables or CommitLog. An attempt to flush would
+        // throw an exception
+        if (!cfs.getTracker().isDummy())
+            err = Throwables.perform(err, () -> cfs.unloadCf());
+
+        err = Throwables.perform(err, () -> cfs.invalidate(true, dropData));
+
+        if (err != null)
+        {
+            logger.error("Failed to unload {}:", cfs.metadata(), err);
+            JVMStabilityInspector.inspectThrowable(err);
+        }
+
+        logger.debug("Column family store has been unloaded for table {} with dropData={}", cfs.metadata, dropData);
     }
 
     /**
@@ -448,6 +511,9 @@ public KeyspaceWriteHandler getWriteHandler()
      */
     public void initCf(TableMetadataRef metadata, boolean loadSSTables)
     {
+        logger.debug("Initializing column family store for table {} with loadSSTables={}",
+                     metadata, loadSSTables);
+
         ColumnFamilyStore cfs = columnFamilyStores.get(metadata.id);
 
         if (cfs == null)
@@ -467,28 +533,19 @@ public void initCf(TableMetadataRef metadata, boolean loadSSTables)
             assert cfs.name.equals(metadata.name);
             cfs.reload();
         }
-    }
-
-    public Future<?> applyFuture(Mutation mutation, boolean writeCommitLog, boolean updateIndexes)
-    {
-        return applyInternal(mutation, writeCommitLog, updateIndexes, true, true, new AsyncPromise<>());
-    }
 
-    public Future<?> applyFuture(Mutation mutation, boolean writeCommitLog, boolean updateIndexes, boolean isDroppable,
-                                            boolean isDeferrable)
-    {
-        return applyInternal(mutation, writeCommitLog, updateIndexes, isDroppable, isDeferrable, new AsyncPromise<>());
+        logger.debug("Column family store initialized for table {} with loadSSTables={}",
+                     metadata, loadSSTables);
     }
 
-    public void apply(Mutation mutation, boolean writeCommitLog, boolean updateIndexes)
+    public Future<?> applyFuture(Mutation mutation, WriteOptions writeOptions)
     {
-        apply(mutation, writeCommitLog, updateIndexes, true);
+        return applyInternal(mutation, writeOptions, true, new AsyncPromise<>());
     }
 
-    public void apply(final Mutation mutation,
-                      final boolean writeCommitLog)
+    public Future<?> applyFuture(Mutation mutation, WriteOptions writeOptions, boolean isDeferrable)
     {
-        apply(mutation, writeCommitLog, true, true);
+        return applyInternal(mutation, writeOptions, isDeferrable, new AsyncPromise<>());
     }
 
     /**
@@ -496,43 +553,51 @@ public void apply(final Mutation mutation,
      * Otherwise there is a race condition where ALL mutation workers are beeing blocked ending
      * in a complete deadlock of the mutation stage. See CASSANDRA-12689.
      *
-     * @param mutation       the row to write.  Must not be modified after calling apply, since commitlog append
-     *                       may happen concurrently, depending on the CL Executor type.
-     * @param makeDurable    if true, don't return unless write has been made durable
-     * @param updateIndexes  false to disable index updates (used by CollationController "defragmenting")
-     * @param isDroppable    true if this should throw WriteTimeoutException if it does not acquire lock within write_request_timeout
+     * @param mutation     the row to write.  Must not be modified after calling apply, since commitlog append
+     *                     may happen concurrently, depending on the CL Executor type.
+     * @param writeOptions describes desired write properties
+     */
+    public void apply(Mutation mutation, WriteOptions writeOptions)
+    {
+        applyInternal(mutation, writeOptions, false, null);
+    }
+
+    /**
+     * Close this keyspace to further mutations, called when draining or shutting down.
+     *
+     * A final write barrier is issued and returned. After this barrier is set, new mutations
+     * will be rejected, see {@link Keyspace#applyInternal(Mutation, WriteOptions, boolean, Promise)}.
      */
-    public void apply(final Mutation mutation,
-                      final boolean makeDurable,
-                      boolean updateIndexes,
-                      boolean isDroppable)
+    public OpOrder.Barrier stopMutations()
     {
-        applyInternal(mutation, makeDurable, updateIndexes, isDroppable, false, null);
+        assert writeBarrier == null : "Keyspace has already been closed to mutations";
+        writeBarrier = writeOrder.newBarrier();
+        writeBarrier.issue();
+        return writeBarrier;
     }
 
     /**
      * This method appends a row to the global CommitLog, then updates memtables and indexes.
      *
-     * @param mutation       the row to write.  Must not be modified after calling apply, since commitlog append
-     *                       may happen concurrently, depending on the CL Executor type.
-     * @param makeDurable    if true, don't return unless write has been made durable
-     * @param updateIndexes  false to disable index updates (used by CollationController "defragmenting")
-     * @param isDroppable    true if this should throw WriteTimeoutException if it does not acquire lock within write_request_timeout
-     * @param isDeferrable   true if caller is not waiting for future to complete, so that future may be deferred
+     * @param mutation     the row to write.  Must not be modified after calling apply, since commitlog append
+     *                     may happen concurrently, depending on the CL Executor type.
+     * @param writeOptions describes desired write properties
+     * @param isDeferrable true if caller is not waiting for future to complete, so that future may be deferred
      */
-    private Future<?> applyInternal(final Mutation mutation,
-                                               final boolean makeDurable,
-                                               boolean updateIndexes,
-                                               boolean isDroppable,
-                                               boolean isDeferrable,
-                                               Promise<?> future)
+    private Future<?> applyInternal(Mutation mutation,
+                                    WriteOptions writeOptions,
+                                    boolean isDeferrable,
+                                    Promise<?> future)
     {
         if (TEST_FAIL_WRITES && metadata.name.equals(TEST_FAIL_WRITES_KS))
             throw new RuntimeException("Testing write failures");
 
+        if (writeBarrier != null)
+            return failDueToWriteBarrier(mutation, future);
+
         Lock[] locks = null;
 
-        boolean requiresViewUpdate = updateIndexes && viewManager.updatesAffectView(Collections.singleton(mutation), false);
+        boolean requiresViewUpdate = writeOptions.requiresViewUpdate(viewManager, mutation);
 
         if (requiresViewUpdate)
         {
@@ -559,7 +624,7 @@ private Future<?> applyInternal(final Mutation mutation,
                     if (lock == null)
                     {
                         //throw WTE only if request is droppable
-                        if (isDroppable && (approxTime.isAfter(mutation.approxCreatedAtNanos + DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS))))
+                        if (writeOptions.isDroppable && (approxTime.isAfter(mutation.approxCreatedAtNanos + DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS))))
                         {
                             for (int j = 0; j < i; j++)
                                 locks[j].unlock();
@@ -581,9 +646,9 @@ else if (isDeferrable)
                                 locks[j].unlock();
 
                             // This view update can't happen right now. so rather than keep this thread busy
-                            // we will re-apply ourself to the queue and try again later
+                            // we will re-apply ourselve to the queue and try again later
                             Stage.MUTATION.execute(() ->
-                                                   applyInternal(mutation, makeDurable, true, isDroppable, true, future)
+                                                   applyInternal(mutation, writeOptions, true, future)
                             );
                             return future;
                         }
@@ -616,13 +681,13 @@ else if (isDeferrable)
             long acquireTime = currentTimeMillis() - mutation.viewLockAcquireStart.get();
             // Metrics are only collected for droppable write operations
             // Bulk non-droppable operations (e.g. commitlog replay, hint delivery) are not measured
-            if (isDroppable)
+            if (writeOptions.isDroppable)
             {
                 for(TableId tableId : tableIds)
                     columnFamilyStores.get(tableId).metric.viewLockAcquireTime.update(acquireTime, MILLISECONDS);
             }
         }
-        try (WriteContext ctx = getWriteHandler().beginWrite(mutation, makeDurable))
+        try (WriteContext ctx = getWriteHandler().beginWrite(mutation, writeOptions))
         {
             for (PartitionUpdate upd : mutation.getPartitionUpdates())
             {
@@ -639,7 +704,7 @@ else if (isDeferrable)
                     try
                     {
                         Tracing.trace("Creating materialized view mutations from base table replica");
-                        viewManager.forTable(upd.metadata().id).pushViewReplicaUpdates(upd, makeDurable, baseComplete);
+                        viewManager.forTable(upd.metadata().id).pushViewReplicaUpdates(upd, writeOptions, baseComplete);
                     }
                     catch (Throwable t)
                     {
@@ -650,7 +715,7 @@ else if (isDeferrable)
                     }
                 }
 
-                cfs.getWriteHandler().write(upd, ctx, updateIndexes);
+                cfs.getWriteHandler().write(upd, ctx, writeOptions.updateIndexes);
 
                 if (requiresViewUpdate)
                     baseComplete.set(currentTimeMillis());
@@ -672,6 +737,17 @@ else if (isDeferrable)
         }
     }
 
+    private Promise<?> failDueToWriteBarrier(Mutation mutation, Promise<?> future)
+    {
+        assert writeBarrier != null : "Expected non-null write barrier";
+
+        logger.error("Attempted to apply mutation "+ mutation+" after final write barrier", new Throwable());
+        BarrierRejectionException exception = new BarrierRejectionException("Keyspace closed to new mutations");
+        if (future != null)
+            future.setFailure(exception);
+        throw exception;
+    }
+
     public AbstractReplicationStrategy getReplicationStrategy()
     {
         return replicationStrategy;
@@ -782,4 +858,18 @@ public String getName()
     {
         return metadata.name;
     }
+
+    public static class BarrierRejectionException extends RejectException implements InternalRequestExecutionException
+    {
+        public BarrierRejectionException(String msg)
+        {
+            super(msg);
+        }
+
+        @Override
+        public RequestFailureReason getReason()
+        {
+            return RequestFailureReason.UNKNOWN;
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/KeyspaceWriteHandler.java b/src/java/org/apache/cassandra/db/KeyspaceWriteHandler.java
index 19cca7243210..81205b1a866f 100644
--- a/src/java/org/apache/cassandra/db/KeyspaceWriteHandler.java
+++ b/src/java/org/apache/cassandra/db/KeyspaceWriteHandler.java
@@ -22,8 +22,9 @@
 
 public interface KeyspaceWriteHandler
 {
-    // mutation can be null if makeDurable is false
-    WriteContext beginWrite(Mutation mutation, boolean makeDurable) throws RequestExecutionException;
+    // mutation can be null if writeOptions.writeCommitLog is false
+    WriteContext beginWrite(Mutation mutation, WriteOptions writeOptions) throws RequestExecutionException;
+
     WriteContext createContextForIndexing();
     WriteContext createContextForRead();
 }
diff --git a/src/java/org/apache/cassandra/db/LivenessInfo.java b/src/java/org/apache/cassandra/db/LivenessInfo.java
index 987c5fdf90fd..1cbcf8e77158 100644
--- a/src/java/org/apache/cassandra/db/LivenessInfo.java
+++ b/src/java/org/apache/cassandra/db/LivenessInfo.java
@@ -224,6 +224,11 @@ public boolean supersedes(LivenessInfo other)
         return isExpiring();
     }
 
+    public static LivenessInfo merge(LivenessInfo a, LivenessInfo b)
+    {
+        return b.supersedes(a) ? b : a;
+    }
+
     protected boolean isExpired()
     {
         return false;
@@ -256,7 +261,7 @@ public LivenessInfo withUpdatedTimestampAndLocalDeletionTime(long newTimestamp,
     @Override
     public String toString()
     {
-        return String.format("[ts=%d]", timestamp);
+        return timestamp == NO_TIMESTAMP ? "[ts=EMPTY]" : String.format("[ts=%d]", timestamp);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/MultiCBuilder.java b/src/java/org/apache/cassandra/db/MultiCBuilder.java
deleted file mode 100644
index 435e418eb3a7..000000000000
--- a/src/java/org/apache/cassandra/db/MultiCBuilder.java
+++ /dev/null
@@ -1,514 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.NavigableSet;
-
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.btree.BTreeSet;
-
-/**
- * Builder that allow to build multiple Clustering/ClusteringBound at the same time.
- */
-public abstract class MultiCBuilder
-{
-    /**
-     * The table comparator.
-     */
-    protected final ClusteringComparator comparator;
-
-    /**
-     * The number of clustering elements that have been added.
-     */
-    protected int size;
-
-    /**
-     * <code>true</code> if the clusterings have been build, <code>false</code> otherwise.
-     */
-    protected boolean built;
-
-    /**
-     * <code>true</code> if the clusterings contains some <code>null</code> elements.
-     */
-    protected boolean containsNull;
-
-    /**
-     * <code>true</code> if the composites contains some <code>unset</code> elements.
-     */
-    protected boolean containsUnset;
-
-    /**
-     * <code>true</code> if some empty collection have been added.
-     */
-    protected boolean hasMissingElements;
-
-    protected MultiCBuilder(ClusteringComparator comparator)
-    {
-        this.comparator = comparator;
-    }
-
-    /**
-     * Creates a new empty {@code MultiCBuilder}.
-     */
-    public static MultiCBuilder create(ClusteringComparator comparator, boolean forMultipleValues)
-    {
-        return forMultipleValues
-             ? new MultiClusteringBuilder(comparator)
-             : new OneClusteringBuilder(comparator);
-    }
-
-    /**
-     * Adds the specified element to all the clusterings.
-     * <p>
-     * If this builder contains 2 clustering: A-B and A-C a call to this method to add D will result in the clusterings:
-     * A-B-D and A-C-D.
-     * </p>
-     *
-     * @param value the value of the next element
-     * @return this <code>MulitCBuilder</code>
-     */
-    public abstract MultiCBuilder addElementToAll(ByteBuffer value);
-
-    /**
-     * Adds individually each of the specified elements to the end of all of the existing clusterings.
-     * <p>
-     * If this builder contains 2 clusterings: A-B and A-C a call to this method to add D and E will result in the 4
-     * clusterings: A-B-D, A-B-E, A-C-D and A-C-E.
-     * </p>
-     *
-     * @param values the elements to add
-     * @return this <code>CompositeBuilder</code>
-     */
-    public abstract MultiCBuilder addEachElementToAll(List<ByteBuffer> values);
-
-    /**
-     * Adds individually each of the specified list of elements to the end of all of the existing composites.
-     * <p>
-     * If this builder contains 2 composites: A-B and A-C a call to this method to add [[D, E], [F, G]] will result in the 4
-     * composites: A-B-D-E, A-B-F-G, A-C-D-E and A-C-F-G.
-     * </p>
-     *
-     * @param values the elements to add
-     * @return this <code>CompositeBuilder</code>
-     */
-    public abstract MultiCBuilder addAllElementsToAll(List<List<ByteBuffer>> values);
-
-    protected void checkUpdateable()
-    {
-        if (!hasRemaining() || built)
-            throw new IllegalStateException("this builder cannot be updated anymore");
-    }
-
-    /**
-     * Returns the number of elements that can be added to the clusterings.
-     *
-     * @return the number of elements that can be added to the clusterings.
-     */
-    public int remainingCount()
-    {
-        return comparator.size() - size;
-    }
-
-    /**
-     * Returns the current number of results when {@link #build()} is called
-     *
-     * @return the current number of build results
-     */
-    public abstract int buildSize();
-
-    /**
-     * Checks if the clusterings contains null elements.
-     *
-     * @return <code>true</code> if the clusterings contains <code>null</code> elements, <code>false</code> otherwise.
-     */
-    public boolean containsNull()
-    {
-        return containsNull;
-    }
-
-    /**
-     * Checks if the clusterings contains unset elements.
-     *
-     * @return <code>true</code> if the clusterings contains <code>unset</code> elements, <code>false</code> otherwise.
-     */
-    public boolean containsUnset()
-    {
-        return containsUnset;
-    }
-
-    /**
-     * Checks if some empty list of values have been added
-     * @return <code>true</code> if the clusterings have some missing elements, <code>false</code> otherwise.
-     */
-    public boolean hasMissingElements()
-    {
-        return hasMissingElements;
-    }
-
-    /**
-     * Builds the <code>clusterings</code>.
-     *
-     * @return the clusterings
-     */
-    public abstract NavigableSet<Clustering<?>> build();
-
-    /**
-     * Builds the <code>ClusteringBound</code>s for slice restrictions.
-     *
-     * @param isStart specify if the bound is a start one
-     * @param isInclusive specify if the bound is inclusive or not
-     * @param isOtherBoundInclusive specify if the other bound is inclusive or not
-     * @param columnDefs the columns of the slice restriction
-     * @return the <code>ClusteringBound</code>s
-     */
-    public abstract NavigableSet<ClusteringBound<?>> buildBoundForSlice(boolean isStart,
-                                                                        boolean isInclusive,
-                                                                        boolean isOtherBoundInclusive,
-                                                                        List<ColumnMetadata> columnDefs);
-
-    /**
-     * Builds the <code>ClusteringBound</code>s
-     *
-     * @param isStart specify if the bound is a start one
-     * @param isInclusive specify if the bound is inclusive or not
-     * @return the <code>ClusteringBound</code>s
-     */
-    public abstract NavigableSet<ClusteringBound<?>> buildBound(boolean isStart, boolean isInclusive);
-
-    /**
-     * Checks if some elements can still be added to the clusterings.
-     *
-     * @return <code>true</code> if it is possible to add more elements to the clusterings, <code>false</code> otherwise.
-     */
-    public boolean hasRemaining()
-    {
-        return remainingCount() > 0;
-    }
-
-    /**
-     * Specialization of MultiCBuilder when we know only one clustering/bound is created.
-     */
-    private static class OneClusteringBuilder extends MultiCBuilder
-    {
-        /**
-         * The elements of the clusterings
-         */
-        private final ByteBuffer[] elements;
-
-        public OneClusteringBuilder(ClusteringComparator comparator)
-        {
-            super(comparator);
-            this.elements = new ByteBuffer[comparator.size()];
-        }
-
-        public MultiCBuilder addElementToAll(ByteBuffer value)
-        {
-            checkUpdateable();
-
-            if (value == null)
-                containsNull = true;
-            if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
-                containsUnset = true;
-
-            elements[size++] = value;
-            return this;
-        }
-
-        public MultiCBuilder addEachElementToAll(List<ByteBuffer> values)
-        {
-            if (values.isEmpty())
-            {
-                hasMissingElements = true;
-                return this;
-            }
-
-            assert values.size() == 1;
-
-            return addElementToAll(values.get(0));
-        }
-
-        public MultiCBuilder addAllElementsToAll(List<List<ByteBuffer>> values)
-        {
-            if (values.isEmpty())
-            {
-                hasMissingElements = true;
-                return this;
-            }
-
-            assert values.size() == 1;
-            return addEachElementToAll(values.get(0));
-        }
-
-        @Override
-        public int buildSize()
-        {
-            return hasMissingElements ? 0 : 1;
-        }
-
-        public NavigableSet<Clustering<?>> build()
-        {
-            built = true;
-
-            if (hasMissingElements)
-                return BTreeSet.empty(comparator);
-
-            return BTreeSet.of(comparator, size == 0 ? Clustering.EMPTY : Clustering.make(elements));
-        }
-
-        @Override
-        public NavigableSet<ClusteringBound<?>> buildBoundForSlice(boolean isStart,
-                                                                   boolean isInclusive,
-                                                                   boolean isOtherBoundInclusive,
-                                                                   List<ColumnMetadata> columnDefs)
-        {
-            return buildBound(isStart, columnDefs.get(0).isReversedType() ? isOtherBoundInclusive : isInclusive);
-        }
-
-        public NavigableSet<ClusteringBound<?>> buildBound(boolean isStart, boolean isInclusive)
-        {
-            built = true;
-
-            if (hasMissingElements)
-                return BTreeSet.empty(comparator);
-
-            if (size == 0)
-                return BTreeSet.of(comparator, isStart ? BufferClusteringBound.BOTTOM : BufferClusteringBound.TOP);
-
-            ByteBuffer[] newValues = size == elements.length
-                                   ? elements
-                                   : Arrays.copyOf(elements, size);
-
-            return BTreeSet.of(comparator, BufferClusteringBound.create(ClusteringBound.boundKind(isStart, isInclusive), newValues));
-        }
-    }
-
-    /**
-     * MultiCBuilder implementation actually supporting the creation of multiple clustering/bound.
-     */
-    private static class MultiClusteringBuilder extends MultiCBuilder
-    {
-        /**
-         * The elements of the clusterings
-         */
-        private final List<List<ByteBuffer>> elementsList = new ArrayList<>();
-
-        public MultiClusteringBuilder(ClusteringComparator comparator)
-        {
-            super(comparator);
-        }
-
-        public MultiCBuilder addElementToAll(ByteBuffer value)
-        {
-            checkUpdateable();
-
-            if (elementsList.isEmpty())
-                elementsList.add(new ArrayList<>());
-
-            if (value == null)
-                containsNull = true;
-            else if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
-                containsUnset = true;
-
-            for (int i = 0, m = elementsList.size(); i < m; i++)
-                elementsList.get(i).add(value);
-
-            size++;
-            return this;
-        }
-
-        public MultiCBuilder addEachElementToAll(List<ByteBuffer> values)
-        {
-            checkUpdateable();
-
-            if (elementsList.isEmpty())
-                elementsList.add(new ArrayList<>());
-
-            if (values.isEmpty())
-            {
-                hasMissingElements = true;
-            }
-            else
-            {
-                for (int i = 0, m = elementsList.size(); i < m; i++)
-                {
-                    List<ByteBuffer> oldComposite = elementsList.remove(0);
-
-                    for (int j = 0, n = values.size(); j < n; j++)
-                    {
-                        List<ByteBuffer> newComposite = new ArrayList<>(oldComposite);
-                        elementsList.add(newComposite);
-
-                        ByteBuffer value = values.get(j);
-
-                        if (value == null)
-                            containsNull = true;
-                        if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
-                            containsUnset = true;
-
-                        newComposite.add(values.get(j));
-                    }
-                }
-            }
-            size++;
-            return this;
-        }
-
-        public MultiCBuilder addAllElementsToAll(List<List<ByteBuffer>> values)
-        {
-            checkUpdateable();
-
-            if (elementsList.isEmpty())
-                elementsList.add(new ArrayList<>());
-
-            if (values.isEmpty())
-            {
-                hasMissingElements = true;
-            }
-            else
-            {
-                for (int i = 0, m = elementsList.size(); i < m; i++)
-                {
-                    List<ByteBuffer> oldComposite = elementsList.remove(0);
-
-                    for (int j = 0, n = values.size(); j < n; j++)
-                    {
-                        List<ByteBuffer> newComposite = new ArrayList<>(oldComposite);
-                        elementsList.add(newComposite);
-
-                        List<ByteBuffer> value = values.get(j);
-
-                        if (value.contains(null))
-                            containsNull = true;
-                        if (value.contains(ByteBufferUtil.UNSET_BYTE_BUFFER))
-                            containsUnset = true;
-
-                        newComposite.addAll(value);
-                    }
-                }
-                size += values.get(0).size();
-            }
-            return this;
-        }
-
-        @Override
-        public int buildSize()
-        {
-            return hasMissingElements ? 0 : elementsList.size();
-        }
-
-        public NavigableSet<Clustering<?>> build()
-        {
-            built = true;
-
-            if (hasMissingElements)
-                return BTreeSet.empty(comparator);
-
-            CBuilder builder = CBuilder.create(comparator);
-
-            if (elementsList.isEmpty())
-                return BTreeSet.of(builder.comparator(), builder.build());
-
-            BTreeSet.Builder<Clustering<?>> set = BTreeSet.builder(builder.comparator());
-            for (int i = 0, m = elementsList.size(); i < m; i++)
-            {
-                List<ByteBuffer> elements = elementsList.get(i);
-                set.add(builder.buildWith(elements));
-            }
-            return set.build();
-        }
-
-        public NavigableSet<ClusteringBound<?>> buildBoundForSlice(boolean isStart,
-                                                                   boolean isInclusive,
-                                                                   boolean isOtherBoundInclusive,
-                                                                   List<ColumnMetadata> columnDefs)
-        {
-            built = true;
-
-            if (hasMissingElements)
-                return BTreeSet.empty(comparator);
-
-            CBuilder builder = CBuilder.create(comparator);
-
-            if (elementsList.isEmpty())
-                return BTreeSet.of(comparator, builder.buildBound(isStart, isInclusive));
-
-            // Use a TreeSet to sort and eliminate duplicates
-            BTreeSet.Builder<ClusteringBound<?>> set = BTreeSet.builder(comparator);
-
-            // The first column of the slice might not be the first clustering column (e.g. clustering_0 = ? AND (clustering_1, clustering_2) >= (?, ?)
-            int offset = columnDefs.get(0).position();
-
-            for (int i = 0, m = elementsList.size(); i < m; i++)
-            {
-                List<ByteBuffer> elements = elementsList.get(i);
-
-                // Handle the no bound case
-                if (elements.size() == offset)
-                {
-                    set.add(builder.buildBoundWith(elements, isStart, true));
-                    continue;
-                }
-
-                // In the case of mixed order columns, we will have some extra slices where the columns change directions.
-                // For example: if we have clustering_0 DESC and clustering_1 ASC a slice like (clustering_0, clustering_1) > (1, 2)
-                // will produce 2 slices: [BOTTOM, 1) and (1.2, 1]
-                // So, the END bound will return 2 bounds with the same values 1
-                ColumnMetadata lastColumn = columnDefs.get(columnDefs.size() - 1);
-                if (elements.size() <= lastColumn.position() && i < m - 1 && elements.equals(elementsList.get(i + 1)))
-                {
-                    set.add(builder.buildBoundWith(elements, isStart, false));
-                    set.add(builder.buildBoundWith(elementsList.get(i++), isStart, true));
-                    continue;
-                }
-
-                // Handle the normal bounds
-                ColumnMetadata column = columnDefs.get(elements.size() - 1 - offset);
-                set.add(builder.buildBoundWith(elements, isStart, column.isReversedType() ? isOtherBoundInclusive : isInclusive));
-            }
-            return set.build();
-        }
-
-        public NavigableSet<ClusteringBound<?>> buildBound(boolean isStart, boolean isInclusive)
-        {
-            built = true;
-
-            if (hasMissingElements)
-                return BTreeSet.empty(comparator);
-
-            CBuilder builder = CBuilder.create(comparator);
-
-            if (elementsList.isEmpty())
-                return BTreeSet.of(comparator, builder.buildBound(isStart, isInclusive));
-
-            // Use a TreeSet to sort and eliminate duplicates
-            BTreeSet.Builder<ClusteringBound<?>> set = BTreeSet.builder(comparator);
-
-            for (int i = 0, m = elementsList.size(); i < m; i++)
-            {
-                List<ByteBuffer> elements = elementsList.get(i);
-                set.add(builder.buildBoundWith(elements, isStart, isInclusive));
-            }
-            return set.build();
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/db/MultiClusteringBuilder.java b/src/java/org/apache/cassandra/db/MultiClusteringBuilder.java
new file mode 100644
index 000000000000..49e880f46439
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/MultiClusteringBuilder.java
@@ -0,0 +1,501 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.cql3.statements.Bound;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UniqueComparator;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.btree.BTreeSet;
+
+/**
+ * Builder that allows to build multiple {@link Clustering}/{@link ClusteringBound} at the same time.
+ * Builds a set of clusterings incrementally, by computing cartesian products of
+ * sets of values present in each statement restriction. The typical use of this builder is as follows:
+ * <ol>
+ * <li>Call {@link MultiClusteringBuilder#extend(ClusteringElements, List)} or {@link MultiClusteringBuilder#extend(List, List)}
+ * method once per each restriction. Slice restrictons, if they exist, must be added last.
+ * <li>Finally, call {@link MultiClusteringBuilder#build()} or {@link MultiClusteringBuilder#buildBound(boolean)} method
+ * to obtain the set of clusterings / clustering bounds.</li>
+ * </ol>
+ * <p>
+ * Important: When dealing with slices, you likely want the number of start and end bounds to match.
+ * If some columns are restricted from one side only, you can use the special {@link ClusteringElements#BOTTOM} or
+ * {@link ClusteringElements#TOP} values to generate a proper clustering bound for the "unbounded"
+ * side of the restriction.
+ * </p>
+ * <h1>Example</h1>
+ * <p>
+ *
+ * Imagine we have a CQL query with multiple restrictions joined by AND:
+ * <pre>
+ * SELECT * FROM tab
+ * WHERE a IN (a1, a2)
+ *   AND b IN (b1, b2, b3)
+ *   AND c > c1
+ * </pre>
+ * <p>
+ * We need to generate a list of clustering bounds that will be used to fetch proper contiguous chunks of the partition.
+ *
+ * <p>
+ * The builder initial state is a single empty clustering, denoted by the {@code ROOT} constant,
+ * which is a natural zero element of cartesian set multiplication. This significantly simplifies the logic.
+ * <pre>
+ * point: ()
+ * </pre>
+ *
+ * After adding the IN restriction on column {@code a} we get 2 points:
+ * <pre>
+ * point: (a1)
+ * point: (a2)
+ * </pre>
+ *
+ * Next when we add the IN restrction on column {@code b}, we get a cartesian product of all values
+ * of {@code a} with all values of {@code b}:
+ * <pre>
+ * point: (a1, b1)
+ * point: (a1, b2)
+ * point: (a1, b3)
+ * point: (a2, b1)
+ * point: (a2, b2)
+ * point: (a2, b3)
+ * </pre>
+ *
+ * Finally, we add the slice of column {@code c} by specifying the lower and upper bound
+ * (we use {@code TOP} for the upper bound), and we get the final set of clustering bounds:
+ * <pre>
+ * excl start: (a1, b1, c1)
+ * incl end:   (a1, b1)
+ * excl start: (a1, b2, c1)
+ * incl end:   (a1, b2)
+ * excl start: (a1, b3, c1)
+ * incl end:   (a1, b3)
+ * excl start: (a2, b1, c1)
+ * incl end:   (a2, b1)
+ * excl start: (a2, b2, c1)
+ * incl end:   (a2, b2)
+ * excl start: (a2, b3, c1)
+ * incl end:   (a2, b3)
+ * </pre>
+ */
+public class MultiClusteringBuilder
+{
+    /**
+     * Represents a building block of a clustering.
+     * Either a point or a bound.
+     * Can consist of multiple column values.
+     *
+     * <p>
+     * For bounds, it additionally stores the inclusiveness of a bound and whether it is start or end, so that
+     * it is possible to mix bounds of different inclusiveness.
+     */
+    public static class ClusteringElements
+    {
+        public enum Kind
+        {
+            POINT, INCL_START, EXCL_START, INCL_END, EXCL_END
+        }
+
+        public static final ClusteringElements BOTTOM = new ClusteringElements(Collections.emptyList(), Kind.INCL_START);
+        public static final ClusteringElements TOP = new ClusteringElements(Collections.emptyList(), Kind.INCL_END);
+        public static final ClusteringElements ROOT = new ClusteringElements(Collections.emptyList(), Kind.POINT);
+
+        final List<ByteBuffer> values;
+        final Kind kind;
+
+
+        private ClusteringElements(List<ByteBuffer> values, Kind kind)
+        {
+            this.values = values;
+            this.kind = kind;
+        }
+
+        public static ClusteringElements point(ByteBuffer value)
+        {
+            return point(Collections.singletonList(value));
+        }
+
+        public static ClusteringElements point(List<ByteBuffer> values)
+        {
+            return new ClusteringElements(values, Kind.POINT);
+        }
+
+        public static ClusteringElements bound(ByteBuffer value, Bound bound, boolean inclusive)
+        {
+            return bound(Collections.singletonList(value), bound, inclusive);
+        }
+
+        public static ClusteringElements bound(List<ByteBuffer> values, Bound bound, boolean inclusive)
+        {
+            Kind kind;
+            if (bound.isStart())
+                kind = (inclusive ? Kind.INCL_START : Kind.EXCL_START);
+            else
+                kind = (inclusive ? Kind.INCL_END : Kind.EXCL_END);
+            return new ClusteringElements(values, kind);
+        }
+
+        public  boolean isBound()
+        {
+            return kind != Kind.POINT;
+        }
+
+        public boolean isStart()
+        {
+            return kind == ClusteringElements.Kind.EXCL_START ||
+                   kind == ClusteringElements.Kind.INCL_START;
+        }
+
+        public boolean isInclusive()
+        {
+            return kind == Kind.INCL_START ||
+                   kind == Kind.INCL_END ||
+                   kind == Kind.POINT;
+        }
+
+        public String toString()
+        {
+            return "Element{" +
+                   "kind=" + kind +
+                   ", value=" + values +
+                   '}';
+        }
+    }
+
+    /**
+     * The table comparator.
+     */
+    private final ClusteringComparator comparator;
+
+    /**
+     * Columns corresponding to the already added elements.
+     */
+    private final List<ColumnMetadata> columns = new ArrayList<>();
+
+    /**
+     * The elements of the clusterings.
+     */
+    private List<ClusteringElements> clusterings = Collections.singletonList(ClusteringElements.ROOT);
+
+
+    /**
+     * <code>true</code> if the clusterings have been build, <code>false</code> otherwise.
+     */
+    private boolean built;
+
+    /**
+     * <code>true</code> if the clusterings contains some <code>null</code> elements.
+     */
+    private boolean containsNull;
+
+    /**
+     * <code>true</code> if the composites contains some <code>unset</code> elements.
+     */
+    private boolean containsUnset;
+
+    /**
+     * <code>true</code> if the composites contains some slice bound elements.
+     */
+    private boolean containsSliceBound;
+
+
+    private MultiClusteringBuilder(ClusteringComparator comparator)
+    {
+        this.comparator = comparator;
+    }
+
+    /**
+     * Creates a new empty {@code MultiCBuilder}.
+     */
+    public static MultiClusteringBuilder create(ClusteringComparator comparator)
+    {
+        return new MultiClusteringBuilder(comparator);
+    }
+
+    protected void checkUpdateable()
+    {
+        if (!hasRemaining() || built)
+            throw new IllegalStateException("This builder cannot be updated anymore");
+        if (containsSliceBound)
+            throw new IllegalStateException("Cannot extend clustering that contains a slice bound");
+    }
+
+    /**
+     * Returns the number of elements that can be added to the clusterings.
+     *
+     * @return the number of elements that can be added to the clusterings.
+     */
+    public int remainingCount()
+    {
+        return comparator.size() - columns.size();
+    }
+
+    /**
+     * Checks if the clusterings contains null elements.
+     *
+     * @return <code>true</code> if the clusterings contains <code>null</code> elements, <code>false</code> otherwise.
+     */
+    public boolean containsNull()
+    {
+        return containsNull;
+    }
+
+    /**
+     * Checks if the clusterings contains unset elements.
+     *
+     * @return <code>true</code> if the clusterings contains <code>unset</code> elements, <code>false</code> otherwise.
+     */
+    public boolean containsUnset()
+    {
+        return containsUnset;
+    }
+
+    /**
+     * Returns the current number of results when {@link #build()} is called
+     *
+     * @return the current number of build results
+     */
+    public int buildSize()
+    {
+        return clusterings.size();
+    }
+
+    /**
+     * Returns <code>true</code> if the current number of build results is zero.
+     */
+    public boolean buildIsEmpty()
+    {
+        return clusterings.isEmpty();
+    }
+
+    /**
+     * Checks if some elements can still be added to the clusterings.
+     *
+     * @return <code>true</code> if it is possible to add more elements to the clusterings, <code>false</code> otherwise.
+     */
+    public boolean hasRemaining()
+    {
+        return remainingCount() > 0;
+    }
+
+    /**
+     * Extends each clustering with the given element(s).
+     *
+     * <p>
+     * If this builder contains 2 composites: A-B and A-C a call to this method to add D will result in the
+     * clusterings A-B-D and A-C-D.
+     * </p>
+     *
+     * @param suffix the element to add
+     * @param suffixColumns column definitions in the element; must match the subsequent comparator subtypes
+     * @return this <code>CompositeBuilder</code>
+     */
+    public final MultiClusteringBuilder extend(ClusteringElements suffix, List<ColumnMetadata> suffixColumns)
+    {
+        return extend(Collections.singletonList(suffix), suffixColumns);
+    }
+
+    /**
+     * Adds individually each of the specified elements to the end of all the existing clusterings.
+     * The number of result clusterings is the product of the number of current clusterings and the number
+     * of elements added.
+     *
+     * <p>
+     * If this builder contains 2 composites: A-B and A-C a call to this method to add D and E will result in the 4
+     * clusterings: A-B-D, A-B-E, A-C-D and A-C-E.
+     * </p>
+     *
+     * <p>
+     * Added elements can be composites as well.
+     * If this builder contains 2 composites: A-B and A-C a call to this method to add [[D, E], [F, G]] will result in
+     * 4 composites: A-B-D-E, A-B-F-G, A-C-D-E and A-C-F-G.
+     * </p>
+     *
+     * @param suffixes the elements to add
+     * @param suffixColumns column definitions in each element; must match the subsequent comparator subtypes
+     * @return this <code>CompositeBuilder</code>
+     */
+    public MultiClusteringBuilder extend(List<ClusteringElements> suffixes, List<ColumnMetadata> suffixColumns)
+    {
+        checkUpdateable();
+
+        for (int i = 0; i < suffixColumns.size(); i++)
+        {
+            AbstractType<?> expectedType = comparator.subtype(columns.size() + i);
+            AbstractType<?> actualType = suffixColumns.get(i).type;
+            if (!actualType.equals(expectedType))
+            {
+                throw new IllegalStateException(
+                        String.format("Unexpected column type %s != %s.", actualType, expectedType));
+            }
+        }
+
+        for (ClusteringElements suffix: suffixes)
+        {
+            if (suffix.kind != ClusteringElements.Kind.POINT)
+                containsSliceBound = true;
+            if (suffix.values.contains(null))
+                containsNull = true;
+            // Cannot use `value.contains(UNSET_BYTE_BUFFER)`
+            // because UNSET_BYTE_BUFFER.equals(EMPTY_BYTE_BUFFER) but UNSET_BYTE_BUFFER != EMPTY_BYTE_BUFFER
+            if (suffix.values.stream().anyMatch(b -> b == ByteBufferUtil.UNSET_BYTE_BUFFER))
+                containsUnset = true;
+        }
+
+        this.clusterings = columns.isEmpty() ? suffixes : cartesianProduct(clusterings, suffixes);
+        this.columns.addAll(suffixColumns);
+
+        assert columns.size() <= comparator.size();
+        return this;
+    }
+
+    private static ArrayList<ClusteringElements> cartesianProduct(List<ClusteringElements> prefixes, List<ClusteringElements> suffixes)
+    {
+        ArrayList<ClusteringElements> newElements = new ArrayList<>(prefixes.size() * suffixes.size());
+        for (ClusteringElements prefix: prefixes)
+        {
+            for (ClusteringElements suffix: suffixes)
+            {
+                List<ByteBuffer> newValue = new ArrayList<>(prefix.values.size() + suffix.values.size());
+                newValue.addAll(prefix.values);
+                newValue.addAll(suffix.values);
+                newElements.add(new ClusteringElements(newValue, suffix.kind));
+            }
+        }
+        assert newElements.size() == prefixes.size() * suffixes.size();
+        return newElements;
+    }
+
+    /**
+     * Builds the <code>clusterings</code>.
+     * This cannot be used if slice restrictions were added.
+     */
+    public NavigableSet<Clustering<?>> build()
+    {
+        built = true;
+
+        ClusteringBuilder builder = ClusteringBuilder.create(comparator);
+        BTreeSet.Builder<Clustering<?>> set = BTreeSet.builder(builder.comparator());
+        for (ClusteringElements element: clusterings)
+        {
+            assert element.kind == ClusteringElements.Kind.POINT : String.format("Not a point: %s", element);
+            if (!element.values.isEmpty())
+                set.add(builder.buildWith(element.values));
+            else
+                set.add(Clustering.EMPTY);
+        }
+        return set.build();
+    }
+
+    /**
+     * Builds the <code>ClusteringBound</code>s for slice restrictions.
+     * The number of start bounds equals the number of end bounds.
+     *
+     * @param isStart if true, start bounds are returned, otherwise end bounds are returned
+     */
+    public NavigableSet<ClusteringBound<?>> buildBound(boolean isStart)
+    {
+        built = true;
+        ClusteringBuilder builder = ClusteringBuilder.create(comparator);
+
+        // Use UniqueComparator to allow duplicates.
+        // We deal with start bounds and end bounds separately, so it is a bad idea to lose duplicates,
+        // as this would cause the number of start bounds differ from the number of end bounds, if accidentally
+        // two bounds on one end collide but their corresponding bounds on the other end do not.
+        BTreeSet.Builder<org.apache.cassandra.db.ClusteringBound<?>> set = BTreeSet.builder(new UniqueComparator<>(comparator));
+        for (ClusteringElements element: clusterings)
+        {
+            if (element.isBound() && element.isStart() != isStart)
+                continue;
+
+            org.apache.cassandra.db.ClusteringBound<?> bound = element.values.isEmpty()
+                ? builder.buildBound(isStart, element.isInclusive())
+                : builder.buildBoundWith(element.values, isStart, element.isInclusive());
+
+            set.add(bound);
+        }
+        return set.build();
+    }
+
+    /**
+     * Builds the serialized partition keys.
+     *
+     * @return the serialized partition keys
+     */
+    public List<ByteBuffer> buildSerializedPartitionKeys()
+    {
+        built = true;
+
+        if (clusterings.isEmpty())
+            return Collections.emptyList();
+
+        if (clusterings.get(0) == ClusteringElements.ROOT)
+            return ImmutableList.of(ByteBufferUtil.EMPTY_BYTE_BUFFER);
+
+        // Use a TreeSet here to return the values in comparator sorted order
+        TreeSet<ByteBuffer> set = comparator.size() == 1
+                ? new TreeSet<>(comparator.subtype(0))
+                : new TreeSet<>(CompositeType.getInstance(comparator.subtypes()));
+
+        for (ClusteringElements c: clusterings)
+            set.add(c.values.size() == 1 ? c.values.get(0) : toComposite(c.values));
+
+        return new ArrayList<>(set);
+    }
+
+    protected static ByteBuffer toComposite(ByteBuffer[] components)
+    {
+        int sum = 0;
+        for (ByteBuffer v : components)
+        {
+            sum += v == null ? 0 : v.remaining();
+        }
+        if (sum > FBUtilities.MAX_UNSIGNED_SHORT)
+            throw new InvalidRequestException(String.format("Key length of %d is longer than maximum of %d",
+                                                            sum,
+                                                            FBUtilities.MAX_UNSIGNED_SHORT));
+
+        return CompositeType.build(ByteBufferAccessor.instance, components);
+    }
+
+    private ByteBuffer toComposite(List<ByteBuffer> elements)
+    {
+        ByteBuffer[] tmp = new ByteBuffer[elements.size()];
+        for (int i = 0, m = elements.size(); i < m; i++)
+        {
+            tmp[i] = elements.get(i);
+        }
+        return toComposite(tmp);
+    }
+
+}
+
diff --git a/src/java/org/apache/cassandra/db/MultiRangeReadCommand.java b/src/java/org/apache/cassandra/db/MultiRangeReadCommand.java
new file mode 100644
index 000000000000..d5da5d9ad8d5
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/MultiRangeReadCommand.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.service.reads.ReadCallback;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * Used by {@code EndpointGroupingCoordinator} to query all involved ranges on a given replica at once.
+ *
+ * Note: digest is not supported because each replica is responsible for different token ranges, there is no point on
+ * sending digest.
+ */
+public class MultiRangeReadCommand extends ReadCommand
+{
+    protected static final SelectionDeserializer selectionDeserializer = new Deserializer();
+
+    private final List<DataRange> dataRanges;
+
+    private MultiRangeReadCommand(boolean isDigest,
+                                  int digestVersion,
+                                  boolean acceptsTransient,
+                                  TableMetadata metadata,
+                                  long nowInSec,
+                                  ColumnFilter columnFilter,
+                                  RowFilter rowFilter,
+                                  DataLimits limits,
+                                  List<DataRange> dataRanges,
+                                  Index.QueryPlan indexQueryPlan,
+                                  boolean trackWarnings)
+    {
+        super(Kind.MULTI_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, null);
+
+        assert dataRanges.size() > 0;
+        this.dataRanges = dataRanges;
+    }
+
+    /**
+     *
+     * @param command current partition range command
+     * @param ranges token ranges to be queried on specific endpoint
+     * @param isRangeContinuation whether it's querying the first range in the batch
+     * @return multi-range read command for specific endpoint
+     */
+    @VisibleForTesting
+    public static MultiRangeReadCommand create(PartitionRangeReadCommand command, List<AbstractBounds<PartitionPosition>> ranges, boolean isRangeContinuation)
+    {
+        List<DataRange> dataRanges = new ArrayList<>(ranges.size());
+        for (AbstractBounds<PartitionPosition> range : ranges)
+            dataRanges.add(command.dataRange().forSubRange(range));
+
+        return new MultiRangeReadCommand(command.isDigestQuery(),
+                                         command.digestVersion(),
+                                         command.acceptsTransient(),
+                                         command.metadata(),
+                                         command.nowInSec(),
+                                         command.columnFilter(),
+                                         command.rowFilter(),
+                                         isRangeContinuation ? command.limits() : command.limits().withoutState(),
+                                         dataRanges,
+                                         command.indexQueryPlan(),
+                                         false);
+    }
+
+    /**
+     * @param subrangeHandlers handlers for all vnode ranges replicated in current endpoint.
+     * @return multi-range read command for specific endpoint
+     */
+    public static MultiRangeReadCommand create(List<ReadCallback<?, ?>> subrangeHandlers)
+    {
+        assert !subrangeHandlers.isEmpty();
+
+        PartitionRangeReadCommand command = (PartitionRangeReadCommand) subrangeHandlers.get(0).command();
+        List<DataRange> dataRanges = new ArrayList<>(subrangeHandlers.size());
+        boolean trackWarnings = false;
+        for (ReadCallback<?, ?> handler : subrangeHandlers)
+        {
+            dataRanges.add(((PartitionRangeReadCommand) handler.command()).dataRange());
+            trackWarnings |= handler.command().isTrackingWarnings();
+        }
+
+        return new MultiRangeReadCommand(command.isDigestQuery(),
+                                         command.digestVersion(),
+                                         command.acceptsTransient(),
+                                         command.metadata(),
+                                         command.nowInSec(),
+                                         command.columnFilter(),
+                                         command.rowFilter(),
+                                         command.limits(),
+                                         dataRanges,
+                                         command.indexQueryPlan(),
+                                         trackWarnings);
+    }
+
+    // we need to override this method to return instances of MultiRangeReadResponse that don't mess with the serializer
+    @Override
+    public ReadResponse createEmptyResponse()
+    {
+        UnfilteredPartitionIterator iterator = EmptyIterators.unfilteredPartition(metadata());
+
+        return isDigestQuery()
+               ? ReadResponse.createDigestResponse(iterator, this)
+               : MultiRangeReadResponse.createDataResponse(iterator, this);
+    }
+
+    /**
+     * @return all token ranges to be queried
+     */
+    public List<DataRange> ranges()
+    {
+        return dataRanges;
+    }
+
+    @Override
+    public String loggableTokens()
+    {
+        StringBuilder loggableTokens = new StringBuilder();
+        boolean first = true;
+        for (DataRange dataRange : dataRanges)
+        {
+            if (first)
+                first = false;
+            else
+                loggableTokens.append(", ");
+            loggableTokens.append(loggableTokens(dataRange));
+        }
+        return loggableTokens.toString();
+    }
+
+    private StringBuilder loggableTokens(DataRange dataRange)
+    {
+        return new StringBuilder()
+                .append("token range: ")
+                .append(dataRange.keyRange.inclusiveLeft() ? '[' : '(')
+                .append(dataRange.keyRange.left.getToken().toString())
+                .append(", ")
+                .append(dataRange.keyRange.right.getToken().toString())
+                .append(dataRange.keyRange.inclusiveRight() ? ']' : ')');
+    }
+
+    @Override
+    protected void serializeSelection(DataOutputPlus out, int version) throws IOException
+    {
+        int rangeCount = dataRanges.size();
+        out.writeInt(rangeCount);
+
+        for (DataRange range : dataRanges)
+            DataRange.serializer.serialize(range, out, version, metadata());
+    }
+
+    @Override
+    protected long selectionSerializedSize(int version)
+    {
+        int rangeCount = dataRanges.size();
+        long size = TypeSizes.sizeof(rangeCount);
+
+        for (DataRange range : dataRanges)
+            size += DataRange.serializer.serializedSize(range, version, metadata());
+
+        return size;
+    }
+
+    @Override
+    public boolean isLimitedToOnePartition()
+    {
+        if (dataRanges.size() != 1)
+            return false;
+
+        DataRange dataRange = dataRanges.get(0);
+        return dataRange.keyRange() instanceof Bounds
+               && dataRange.startKey().kind() == PartitionPosition.Kind.ROW_KEY
+               && dataRange.startKey().equals(dataRange.stopKey());
+    }
+
+    @Override
+    public boolean isRangeRequest()
+    {
+        return false;
+    }
+
+    @Override
+    public ReadCommand withUpdatedLimit(DataLimits newLimits)
+    {
+        return new MultiRangeReadCommand(isDigestQuery(),
+                                         digestVersion(),
+                                         acceptsTransient(),
+                                         metadata(),
+                                         nowInSec(),
+                                         columnFilter(),
+                                         rowFilter(),
+                                         newLimits,
+                                         dataRanges,
+                                         indexQueryPlan(),
+                                         isTrackingWarnings());
+    }
+
+    @Override
+    public long getTimeout(TimeUnit unit)
+    {
+        return DatabaseDescriptor.getRangeRpcTimeout(unit);
+    }
+
+    @Override
+    public ReadResponse createResponse(UnfilteredPartitionIterator iterator, RepairedDataInfo rdi)
+    {
+        assert !isDigestQuery();
+        return MultiRangeReadResponse.createDataResponse(iterator, this);
+    }
+
+    @Override
+    public ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key)
+    {
+        for (DataRange dataRange : ranges())
+        {
+            if (dataRange.keyRange().contains(key))
+                return dataRange.clusteringIndexFilter(key);
+        }
+
+        throw new IllegalArgumentException(key + " is not in data ranges " + dataRanges.stream().map(r -> r.toString(metadata())).collect(Collectors.toList()));
+    }
+
+    @Override
+    public ReadCommand copy()
+    {
+        return new MultiRangeReadCommand(isDigestQuery(),
+                                         digestVersion(),
+                                         acceptsTransient(),
+                                         metadata(),
+                                         nowInSec(),
+                                         columnFilter(),
+                                         rowFilter(),
+                                         limits(),
+                                         dataRanges,
+                                         indexQueryPlan(),
+                                         isTrackingWarnings());
+    }
+
+    @Override
+    protected ReadCommand copyAsTransientQuery()
+    {
+        return new MultiRangeReadCommand(false,
+                                          0,
+                                          true,
+                                          metadata(),
+                                          nowInSec(),
+                                          columnFilter(),
+                                          rowFilter(),
+                                          limits(),
+                                          dataRanges,
+                                          indexQueryPlan(),
+                                          isTrackingWarnings());
+    }
+
+    @Override
+    protected ReadCommand copyAsDigestQuery()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public UnfilteredPartitionIterator queryStorage(ColumnFamilyStore cfs, ReadExecutionController executionController)
+    {
+        return UnfilteredPartitionIterators.concat(dataRanges.stream()
+                                                             .map(this::toPartitionRangeReadCommand)
+                                                             .map(command -> command.queryStorage(cfs, executionController))
+                                                             .collect(Collectors.toList()));
+    }
+
+    @Override
+    protected boolean intersects(SSTableReader sstable)
+    {
+        return dataRanges.stream().anyMatch(dataRange -> dataRange.clusteringIndexFilter.intersects(sstable.metadata().comparator, sstable.getSSTableMetadata().coveredClustering));
+    }
+
+    @Override
+    public UnfilteredPartitionIterator searchStorage(Index.Searcher searcher, ReadExecutionController controller)
+    {
+        if (indexQueryPlan.supportsMultiRangeReadCommand())
+        {
+            // SAI supports fetching multiple ranges at once
+            return super.searchStorage(searcher, controller);
+        }
+        else
+        {
+            // search each subrange separately as they don't support MultiRangeReadCommand
+            return UnfilteredPartitionIterators.concat(dataRanges.stream()
+                                                                 .map(this::toPartitionRangeReadCommand)
+                                                                 .map(command -> command.searchStorage(searcher, controller))
+                                                                 .collect(Collectors.toList()));
+        }
+    }
+
+    private PartitionRangeReadCommand toPartitionRangeReadCommand(DataRange dataRange)
+    {
+        return PartitionRangeReadCommand.create(metadata(), nowInSec(), columnFilter(), rowFilter(), limits(), dataRange, indexQueryPlan(), isTrackingWarnings());
+    }
+
+    @Override
+    public boolean isReversed()
+    {
+        return ranges().get(0).isReversed();
+    }
+
+    @Override
+    protected void recordReadLatency(TableMetrics metric, long latencyNanos)
+    {
+        metric.rangeLatency.addNano(latencyNanos);
+    }
+
+    @Override
+    protected void recordReadRequest(TableMetrics metric)
+    {
+        metric.rangeRequests.inc();
+    }
+
+    @Override
+    public Verb verb()
+    {
+        return Verb.MULTI_RANGE_REQ;
+    }
+
+    @Override
+    protected void appendCQLWhereClause(StringBuilder sb)
+    {
+        if (ranges().size() == 1 && ranges().get(0).isUnrestricted(metadata()) && rowFilter().isEmpty())
+            return;
+
+        sb.append(" WHERE ");
+        // We put the row filter first because the data range can end by "ORDER BY"
+        if (!rowFilter().isEmpty())
+        {
+            sb.append(rowFilter());
+            sb.append(" AND ");
+        }
+
+        boolean isFirst = true;
+        for (int i = 0; i < ranges().size(); i++)
+        {
+            DataRange dataRange = ranges().get(i);
+            if (!dataRange.isUnrestricted(metadata()))
+            {
+                if (!isFirst)
+                    sb.append(" AND ");
+                isFirst = false;
+                sb.append(dataRange.toCQLString(metadata(), rowFilter()));
+            }
+        }
+    }
+
+    @Override
+    public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, final Dispatcher.RequestTime requestTime) throws RequestExecutionException
+    {
+        // MultiRangeReadCommand should only be executed on the replica side
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public DataRange dataRange()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public QueryPager getPager(PagingState pagingState, ProtocolVersion protocolVersion)
+    {
+        // MultiRangeReadCommand should only be executed at replica side"
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean selectsKey(DecoratedKey key)
+    {
+        for (DataRange dataRange : ranges())
+        {
+            if (!dataRange.contains(key))
+                continue;
+
+            return rowFilter().partitionKeyRestrictionsAreSatisfiedBy(key, metadata().partitionKeyType);
+        }
+
+        return false;
+    }
+
+    @Override
+    public boolean selectsClustering(DecoratedKey key, Clustering clustering)
+    {
+        if (clustering == Clustering.STATIC_CLUSTERING)
+            return !columnFilter().fetchedColumns().statics.isEmpty();
+
+        for (DataRange dataRange : ranges())
+        {
+            if (!dataRange.keyRange().contains(key) || !dataRange.clusteringIndexFilter(key).selects(clustering))
+                continue;
+
+            if (rowFilter().clusteringKeyRestrictionsAreSatisfiedBy(clustering))
+                return true;
+        }
+
+        return false;
+    }
+
+    @Override
+    public boolean selectsFullPartition()
+    {
+        return metadata().isStaticCompactTable() ||
+               (ranges().stream().allMatch(DataRange::selectsAllPartition) && !rowFilter().hasExpressionOnClusteringOrRegularColumns());
+    }
+
+    private static class Deserializer extends SelectionDeserializer
+    {
+        @Override
+        public ReadCommand deserialize(DataInputPlus in,
+                                       int version,
+                                       boolean isDigest,
+                                       int digestVersion,
+                                       boolean acceptsTransient,
+                                       TableMetadata metadata,
+                                       long nowInSec,
+                                       ColumnFilter columnFilter,
+                                       RowFilter rowFilter,
+                                       DataLimits limits,
+                                       Index.QueryPlan indexQueryPlan)
+        throws IOException
+        {
+            int rangeCount = in.readInt();
+
+            List<DataRange> ranges = new ArrayList<>(rangeCount);
+            for (int i = 0; i < rangeCount; i++)
+                ranges.add(DataRange.serializer.deserialize(in, version, metadata));
+
+            return new MultiRangeReadCommand(isDigest,
+                                             digestVersion,
+                                             acceptsTransient,
+                                             metadata,
+                                             nowInSec,
+                                             columnFilter,
+                                             rowFilter,
+                                             limits,
+                                             ranges,
+                                             indexQueryPlan,
+                                             false);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/MultiRangeReadResponse.java b/src/java/org/apache/cassandra/db/MultiRangeReadResponse.java
new file mode 100644
index 000000000000..29a7d1ef10b5
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/MultiRangeReadResponse.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.NoSuchElementException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * It's used to store response of multi-range read request from a given endpoint,
+ * {@link ReadResponse} of subrange can be extracted via {@link #subrangeResponse(MultiRangeReadCommand, AbstractBounds)};
+ */
+public abstract class MultiRangeReadResponse extends ReadResponse
+{
+    protected static final Logger logger = LoggerFactory.getLogger(MultiRangeReadResponse.class);
+
+    public static final IVersionedSerializer<ReadResponse> serializer = new Serializer();
+
+    private MultiRangeReadResponse()
+    {
+    }
+
+    /**
+     * @param data results of multiple ranges
+     * @param command current multi-range read command
+     * @return multi-range read response
+     */
+    static ReadResponse createDataResponse(UnfilteredPartitionIterator data, MultiRangeReadCommand command)
+    {
+        return new LocalDataResponse(data, command);
+    }
+
+    /**
+     * @param command current multi-range read command
+     * @param range target subrange
+     * @return response corresponding to the given range
+     */
+    public abstract ReadResponse subrangeResponse(MultiRangeReadCommand command, AbstractBounds<PartitionPosition> range);
+
+    @Override
+    public ByteBuffer digest(ReadCommand command)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean isDigestResponse()
+    {
+        return false;
+    }
+
+    @Override
+    public ByteBuffer repairedDataDigest()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean isRepairedDigestConclusive()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean mayIncludeRepairedDigest()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public String toDebugString(ReadCommand command, DecoratedKey key)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * A local response that is not meant to be serialized or used for caching remote endpoint's multi-range response.
+     */
+    private static class LocalResponse extends MultiRangeReadResponse
+    {
+        private final RangeBoundPartitionIterator iterator;
+
+        LocalResponse(UnfilteredPartitionIterator response)
+        {
+            this.iterator = new RangeBoundPartitionIterator(response);
+        }
+
+        @Override
+        public UnfilteredPartitionIterator makeIterator(ReadCommand command)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ReadResponse subrangeResponse(MultiRangeReadCommand command, AbstractBounds<PartitionPosition> range)
+        {
+            // deliver already cached content without deserialization.
+            return new LocalSubrangeResponse(iterator, range);
+        }
+
+        class RangeBoundPartitionIterator
+        {
+            private final UnfilteredPartitionIterator iterator;
+            private UnfilteredRowIterator next = null;
+
+            RangeBoundPartitionIterator(UnfilteredPartitionIterator iterator)
+            {
+                this.iterator = iterator;
+            }
+
+            public boolean hasNext(AbstractBounds<PartitionPosition> range)
+            {
+                if (next != null)
+                    return range.contains(next.partitionKey());
+
+                if (iterator.hasNext())
+                {
+                    next = iterator.next();
+                    if (range.contains(next.partitionKey()))
+                        return true;
+                }
+                return false;
+            }
+
+            public UnfilteredRowIterator next()
+            {
+                if (next != null)
+                {
+                    UnfilteredRowIterator result = next;
+                    next = null;
+                    return result;
+                }
+                throw new NoSuchElementException();
+            }
+        }
+    }
+
+    private static class LocalSubrangeResponse extends ReadResponse
+    {
+        private final LocalResponse.RangeBoundPartitionIterator iterator;
+        private final AbstractBounds<PartitionPosition> range;
+
+        LocalSubrangeResponse(LocalResponse.RangeBoundPartitionIterator iterator, AbstractBounds<PartitionPosition> range)
+        {
+            this.iterator = iterator;
+            this.range = range;
+        }
+
+        @Override
+        public UnfilteredPartitionIterator makeIterator(ReadCommand command)
+        {
+            return new AbstractUnfilteredPartitionIterator()
+            {
+                @Override
+                public TableMetadata metadata()
+                {
+                    return command.metadata();
+                }
+
+                @Override
+                public boolean hasNext()
+                {
+                    return iterator.hasNext(range);
+                }
+
+                @Override
+                public UnfilteredRowIterator next()
+                {
+                    return iterator.next();
+                }
+            };
+        }
+
+        @Override
+        public ByteBuffer digest(ReadCommand command)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ByteBuffer repairedDataDigest()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean isRepairedDigestConclusive()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean mayIncludeRepairedDigest()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean isDigestResponse()
+        {
+            return false;
+        }
+    }
+
+    /**
+     * A local response that needs to be serialized, i.e. sent to another node. The iterator
+     * is serialized by the build method and can be closed as soon as this response has been created.
+     */
+    private static class LocalDataResponse extends DataResponse
+    {
+        private LocalDataResponse(UnfilteredPartitionIterator iterator, MultiRangeReadCommand command)
+        {
+            super(build(iterator, command.columnFilter()), MessagingService.current_version, DeserializationHelper.Flag.FROM_REMOTE);
+        }
+
+        private static ByteBuffer build(UnfilteredPartitionIterator iterator, ColumnFilter selection)
+        {
+            try (DataOutputBuffer buffer = new DataOutputBuffer())
+            {
+                UnfilteredPartitionIterators.serializerForIntraNode().serialize(iterator, selection, buffer, MessagingService.current_version);
+                return buffer.buffer();
+            }
+            catch (IOException e)
+            {
+                // We're serializing in memory so this shouldn't happen
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    /**
+     * A response received from a remove node. We keep the response serialized in the byte buffer.
+     */
+    private static class RemoteDataResponse extends DataResponse
+    {
+        RemoteDataResponse(ByteBuffer data,
+                           int dataSerializationVersion)
+        {
+            super(data, dataSerializationVersion, DeserializationHelper.Flag.FROM_REMOTE);
+        }
+    }
+
+    /**
+     * The command base class for local or remote responses that stay serialized in a byte buffer,
+     * the data.
+     */
+    static abstract class DataResponse extends MultiRangeReadResponse
+    {
+        // The response, serialized in the current messaging version
+        private final ByteBuffer data;
+        private final int dataSerializationVersion;
+        private final DeserializationHelper.Flag flag;
+
+        private MultiRangeReadResponse.LocalResponse cached;
+
+        DataResponse(ByteBuffer data,
+                     int dataSerializationVersion,
+                     DeserializationHelper.Flag flag)
+        {
+            this.data = data;
+            this.dataSerializationVersion = dataSerializationVersion;
+            this.flag = flag;
+        }
+
+        public UnfilteredPartitionIterator makeIterator(ReadCommand command)
+        {
+            try (DataInputBuffer in = new DataInputBuffer(data, true))
+            {
+                // Note that the command parameter shadows the 'command' field and this is intended because
+                // the later can be null (for RemoteDataResponse as those are created in the serializers and
+                // those don't have easy access to the command). This is also why we need the command as parameter here.
+                return UnfilteredPartitionIterators.serializerForIntraNode().deserialize(in,
+                                                                                         dataSerializationVersion,
+                                                                                         command.metadata(),
+                                                                                         command.columnFilter(),
+                                                                                         flag);
+            }
+            catch (IOException e)
+            {
+                // We're deserializing in memory so this shouldn't happen
+                throw new RuntimeException(e);
+            }
+        }
+
+        public ByteBuffer repairedDataDigest()
+        {
+            return ByteBufferUtil.EMPTY_BYTE_BUFFER;
+        }
+
+        @Override
+        public boolean isRepairedDigestConclusive()
+        {
+            return true;
+        }
+
+        @Override
+        public boolean mayIncludeRepairedDigest()
+        {
+            return dataSerializationVersion >= MessagingService.VERSION_40;
+        }
+
+        @Override
+        public ReadResponse subrangeResponse(MultiRangeReadCommand command, AbstractBounds<PartitionPosition> range)
+        {
+            if (cached == null)
+            {
+                try (DataInputBuffer in = new DataInputBuffer(data, true))
+                {
+                    @SuppressWarnings("resource") // The close operation is a noop for a deserialized UPI
+                    UnfilteredPartitionIterator iterator = UnfilteredPartitionIterators.serializerForIntraNode()
+                                                                                       .deserialize(in,
+                                                                                                    dataSerializationVersion,
+                                                                                                    command.metadata(),
+                                                                                                    command.columnFilter(),
+                                                                                                    flag);
+                    cached = new LocalResponse(iterator);
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            }
+
+            return cached.subrangeResponse(command, range);
+        }
+    }
+
+    /**
+     * A copy of {@code ReadResponse.Serializer} that doesn't support a digest response
+     */
+    private static class Serializer implements IVersionedSerializer<ReadResponse>
+    {
+        public void serialize(ReadResponse response, DataOutputPlus out, int version) throws IOException
+        {
+            ByteBuffer digest = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+            ByteBufferUtil.writeWithVIntLength(digest, out);
+            if (version >= MessagingService.VERSION_40)
+            {
+                ByteBufferUtil.writeWithVIntLength(response.repairedDataDigest(), out);
+                out.writeBoolean(response.isRepairedDigestConclusive());
+            }
+            ByteBuffer data = ((DataResponse)response).data;
+            ByteBufferUtil.writeWithVIntLength(data, out);
+        }
+
+        public ReadResponse deserialize(DataInputPlus in, int version) throws IOException
+        {
+            ByteBuffer digest = ByteBufferUtil.readWithVIntLength(in);
+            assert !digest.hasRemaining();
+
+            if (version >= MessagingService.VERSION_40)
+            {
+                ByteBufferUtil.readWithVIntLength(in);
+                in.readBoolean();
+            }
+            ByteBuffer data = ByteBufferUtil.readWithVIntLength(in);
+            return new RemoteDataResponse(data, version);
+        }
+
+        public long serializedSize(ReadResponse response, int version)
+        {
+            ByteBuffer digest = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+            long size = ByteBufferUtil.serializedSizeWithVIntLength(digest);
+
+            if (version >= MessagingService.VERSION_40)
+            {
+                size += ByteBufferUtil.serializedSizeWithVIntLength(response.repairedDataDigest());
+                size += 1;
+            }
+            assert version >= MessagingService.VERSION_30;
+            ByteBuffer data = ((DataResponse)response).data;
+            size += ByteBufferUtil.serializedSizeWithVIntLength(data);
+            return size;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/MutableDeletionInfo.java b/src/java/org/apache/cassandra/db/MutableDeletionInfo.java
index c8d9fd18116d..09abf33b48e3 100644
--- a/src/java/org/apache/cassandra/db/MutableDeletionInfo.java
+++ b/src/java/org/apache/cassandra/db/MutableDeletionInfo.java
@@ -19,6 +19,7 @@
 
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.SortedSet;
 
 import com.google.common.base.Objects;
 
@@ -32,7 +33,7 @@
  */
 public class MutableDeletionInfo implements DeletionInfo
 {
-    private static final long EMPTY_SIZE = ObjectSizes.measure(new MutableDeletionInfo(0, 0));
+    protected static final long EMPTY_SIZE = ObjectSizes.measure(new MutableDeletionInfo(0, 0));
 
     /**
      * This represents a deletion of the entire partition. We can't represent this within the RangeTombstoneList, so it's
@@ -85,12 +86,17 @@ public MutableDeletionInfo mutableCopy()
 
     @Override
     public MutableDeletionInfo clone(ByteBufferCloner cloner)
+    {
+        return new MutableDeletionInfo(partitionDeletion, copyRanges(cloner));
+    }
+
+    @Override
+    public RangeTombstoneList copyRanges(ByteBufferCloner cloner)
     {
         RangeTombstoneList rangesCopy = null;
         if (ranges != null)
-             rangesCopy = ranges.clone(cloner);
-
-        return new MutableDeletionInfo(partitionDeletion, rangesCopy);
+            rangesCopy = ranges.clone(cloner);
+        return rangesCopy;
     }
 
     /**
@@ -160,6 +166,11 @@ public Iterator<RangeTombstone> rangeIterator(Slice slice, boolean reversed)
         return ranges == null ? Collections.emptyIterator() : ranges.iterator(slice, reversed);
     }
 
+    public Iterator<RangeTombstone> rangeIterator(SortedSet<Clustering<?>> names, boolean reversed)
+    {
+        return ranges == null ? Collections.emptyIterator() : ranges.iterator(names, reversed);
+    }
+
     public RangeTombstone rangeCovering(Clustering<?> name)
     {
         return ranges == null ? null : ranges.search(name);
@@ -167,7 +178,7 @@ public RangeTombstone rangeCovering(Clustering<?> name)
 
     public int dataSize()
     {
-        int size = TypeSizes.sizeof(partitionDeletion.markedForDeleteAt());
+        int size = (int) DeletionTime.serializer.serializedSize(partitionDeletion); // small enough so cast is okay
         return size + (ranges == null ? 0 : ranges.dataSize());
     }
 
diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java
index cceb8ea51016..0b09cffa82ac 100644
--- a/src/java/org/apache/cassandra/db/Mutation.java
+++ b/src/java/org/apache/cassandra/db/Mutation.java
@@ -18,7 +18,13 @@
 package org.apache.cassandra.db;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
@@ -44,17 +50,23 @@
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
 import org.apache.cassandra.service.AbstractWriteResponseHandler;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.concurrent.Future;
 
 import static org.apache.cassandra.net.MessagingService.VERSION_40;
 import static org.apache.cassandra.net.MessagingService.VERSION_50;
+import static org.apache.cassandra.net.MessagingService.VERSION_DSE_68;
 import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_10;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_11;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_20;
 
 public class Mutation implements IMutation, Supplier<Mutation>
 {
-    public static final MutationSerializer serializer = new MutationSerializer();
+    public static final MutationSerializer serializer = new MutationSerializer(PartitionUpdate.serializer);
 
     // todo this is redundant
     // when we remove it, also restore SerializationsTest.testMutationRead to not regenerate new Mutations each test
@@ -70,6 +82,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
     final AtomicLong viewLockAcquireStart = new AtomicLong(0);
 
     private final boolean cdcEnabled;
+    private final RequestTracker requestTracker;
 
     private static final int SERIALIZATION_VERSION_COUNT = MessagingService.Version.values().length;
     // Contains serialized representations of this mutation.
@@ -97,6 +110,7 @@ public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, Par
         this.modifications = modifications;
         this.cdcEnabled = cdcEnabled;
         this.approxCreatedAtNanos = approxCreatedAtNanos;
+        this.requestTracker = RequestTracker.instance;
     }
 
     private static boolean cdcEnabled(Iterable<PartitionUpdate> modifications)
@@ -134,6 +148,11 @@ public String getKeyspaceName()
         return keyspaceName;
     }
 
+    public Keyspace getKeyspace()
+    {
+        return Keyspace.open(keyspaceName);
+    }
+
     public Collection<TableId> getTableIds()
     {
         return modifications.keySet();
@@ -227,31 +246,29 @@ public static Mutation merge(List<Mutation> mutations)
             if (updates.isEmpty())
                 continue;
 
-            modifications.put(table, updates.size() == 1 ? updates.get(0) : PartitionUpdate.merge(updates));
+            modifications.put(table, PartitionUpdate.merge(updates));
             updates.clear();
         }
         return new Mutation(ks, key, modifications.build(), approxTime.now());
     }
 
-    public Future<?> applyFuture()
+    public Future<?> applyFuture(WriteOptions writeOptions)
     {
         Keyspace ks = Keyspace.open(keyspaceName);
-        return ks.applyFuture(this, Keyspace.open(keyspaceName).getMetadata().params.durableWrites, true);
+        return ks.applyFuture(this, writeOptions, true).addListener(f -> {
+            RequestSensors sensors = requestTracker.get();
+            if (sensors != null)
+                sensors.syncAllSensors();
+        });
     }
 
-    private void apply(Keyspace keyspace, boolean durableWrites, boolean isDroppable)
+    public void apply(WriteOptions writeOptions)
     {
-        keyspace.apply(this, durableWrites, true, isDroppable);
-    }
+        Keyspace.open(keyspaceName).apply(this, writeOptions);
 
-    public void apply(boolean durableWrites, boolean isDroppable)
-    {
-        apply(Keyspace.open(keyspaceName), durableWrites, isDroppable);
-    }
-
-    public void apply(boolean durableWrites)
-    {
-        apply(durableWrites, true);
+        RequestSensors sensors = requestTracker.get();
+        if (sensors != null)
+            sensors.syncAllSensors();
     }
 
     /*
@@ -260,13 +277,12 @@ public void apply(boolean durableWrites)
      */
     public void apply()
     {
-        Keyspace keyspace = Keyspace.open(keyspaceName);
-        apply(keyspace, keyspace.getMetadata().params.durableWrites, true);
+        apply(WriteOptions.DEFAULT);
     }
 
     public void applyUnsafe()
     {
-        apply(false);
+        apply(WriteOptions.DEFAULT_WITHOUT_COMMITLOG);
     }
 
     public long getTimeout(TimeUnit unit)
@@ -317,6 +333,10 @@ public String toString(boolean shallow)
 
     private int serializedSize40;
     private int serializedSize50;
+    private int serializedSizeDS10;
+    private int serializedSizeDS11;
+    private int serializedSizeDS20;
+    private int serializedSizeDSE68;
 
     public int serializedSize(int version)
     {
@@ -330,7 +350,21 @@ public int serializedSize(int version)
                 if (serializedSize50 == 0)
                     serializedSize50 = (int) serializer.serializedSize(this, VERSION_50);
                 return serializedSize50;
-
+            case VERSION_DS_10:
+                if (serializedSizeDS10 == 0)
+                    serializedSizeDS10 = (int) serializer.serializedSize(this, VERSION_DS_10);
+                return serializedSizeDS10;
+            case VERSION_DS_11:
+                if (serializedSizeDS11 == 0)
+                    serializedSizeDS11 = (int) serializer.serializedSize(this, VERSION_DS_11);
+                return serializedSizeDS11;
+            case VERSION_DS_20:
+                if (serializedSizeDS20 == 0)
+                    serializedSizeDS20 = (int) serializer.serializedSize(this, VERSION_DS_20);
+                return serializedSizeDS20;
+            case VERSION_DSE_68:
+                if (serializedSizeDSE68 == 0)
+                    serializedSizeDSE68 = (int) serializer.serializedSize(this, VERSION_DSE_68);
             default:
                 throw new IllegalStateException("Unknown serialization version: " + version);
         }
@@ -404,9 +438,16 @@ public interface SimpleBuilder
 
     public static class MutationSerializer implements IVersionedSerializer<Mutation>
     {
+        private final PartitionUpdate.PartitionUpdateSerializer partitionUpdateSerializer;
+
+        public MutationSerializer(PartitionUpdate.PartitionUpdateSerializer partitionUpdateSerializer)
+        {
+            this.partitionUpdateSerializer = partitionUpdateSerializer;
+        }
+
         public void serialize(Mutation mutation, DataOutputPlus out, int version) throws IOException
         {
-            serialization(mutation, version).serialize(PartitionUpdate.serializer, mutation, out, version);
+            serialization(mutation, version).serialize(partitionUpdateSerializer, mutation, out, version);
         }
 
         /**
@@ -439,7 +480,7 @@ private Serialization serialization(Mutation mutation, int version)
             if (serialization == null)
             {
                 serialization = new SizeOnlyCacheableSerialization();
-                long serializedSize = serialization.serializedSize(PartitionUpdate.serializer, mutation, version);
+                long serializedSize = serialization.serializedSize(partitionUpdateSerializer, mutation, version);
 
                 // Excessively large mutation objects cause GC pressure and huge allocations when serialized.
                 // so we only cache serialized mutations when they are below the defined limit.
@@ -447,7 +488,7 @@ private Serialization serialization(Mutation mutation, int version)
                 {
                     try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
                     {
-                        serializeInternal(PartitionUpdate.serializer, mutation, dob, version);
+                        serializeInternal(partitionUpdateSerializer, mutation, dob, version);
                         serialization = new CachedSerialization(dob.toByteArray());
                     }
                     catch (IOException e)
@@ -490,7 +531,7 @@ public Mutation deserialize(DataInputPlus in, int version, DeserializationHelper
                 int size = teeIn.readUnsignedVInt32();
                 assert size > 0;
 
-                PartitionUpdate update = PartitionUpdate.serializer.deserialize(teeIn, version, flag);
+                PartitionUpdate update = partitionUpdateSerializer.deserialize(teeIn, version, flag);
                 if (size == 1)
                 {
                     m = new Mutation(update);
@@ -503,7 +544,7 @@ public Mutation deserialize(DataInputPlus in, int version, DeserializationHelper
                     modifications.put(update.metadata().id, update);
                     for (int i = 1; i < size; ++i)
                     {
-                        update = PartitionUpdate.serializer.deserialize(teeIn, version, flag);
+                        update = partitionUpdateSerializer.deserialize(teeIn, version, flag);
                         modifications.put(update.metadata().id, update);
                     }
                     m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now());
@@ -524,7 +565,7 @@ public Mutation deserialize(DataInputPlus in, int version) throws IOException
 
         public long serializedSize(Mutation mutation, int version)
         {
-            return serialization(mutation, version).serializedSize(PartitionUpdate.serializer, mutation, version);
+            return serialization(mutation, version).serializedSize(partitionUpdateSerializer, mutation, version);
         }
     }
 
diff --git a/src/java/org/apache/cassandra/db/MutationVerbHandler.java b/src/java/org/apache/cassandra/db/MutationVerbHandler.java
index 6704febf07cb..c533c98d3932 100644
--- a/src/java/org/apache/cassandra/db/MutationVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/MutationVerbHandler.java
@@ -17,10 +17,26 @@
  */
 package org.apache.cassandra.db;
 
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.*;
+import org.apache.cassandra.net.ForwardingInfo;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.ParamType;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.SensorsFactory;
+import org.apache.cassandra.sensors.Type;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.MonotonicClock;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
@@ -30,10 +46,14 @@ public class MutationVerbHandler extends AbstractMutationVerbHandler<Mutation>
 {
     public static final MutationVerbHandler instance = new MutationVerbHandler();
 
-    private void respond(Message<?> respondTo, InetAddressAndPort respondToAddress)
+    private void respond(RequestSensors requestSensors, Message<Mutation> respondToMessage, InetAddressAndPort respondToAddress)
     {
         Tracing.trace("Enqueuing response to {}", respondToAddress);
-        MessagingService.instance().send(respondTo.emptyResponse(), respondToAddress);
+
+        Message.Builder<NoPayload> response = respondToMessage.emptyResponseBuilder();
+        // no need to calculate outbound internode bytes because the response is NoPayload
+        SensorsCustomParams.addSensorsToInternodeResponse(requestSensors, response);
+        MessagingService.instance().send(response.build(), respondToAddress);
     }
 
     private void failed()
@@ -52,6 +72,12 @@ public void doVerb(Message<Mutation> message)
         }
 
         message.payload.validateSize(MessagingService.current_version, ENTRY_OVERHEAD_SIZE);
+        if (MonotonicClock.Global.approxTime.now() > message.expiresAtNanos())
+        {
+            Tracing.trace("Discarding mutation from {} (timed out)", message.from());
+            MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+            return;
+        }
 
         // Check if there were any forwarding headers in this message
         ForwardingInfo forwardTo = message.forwardTo();
@@ -72,23 +98,36 @@ public void doVerb(Message<Mutation> message)
     @Override
     protected void applyMutation(Message<Mutation> message, InetAddressAndPort respondToAddress)
     {
-        message.payload.applyFuture().addCallback(o -> respond(message, respondToAddress), wto -> failed());
+        // Initialize the sensor and set ExecutorLocals
+        RequestSensors requestSensors = SensorsFactory.instance.createRequestSensors(message.payload.getKeyspaceName());
+        RequestTracker.instance.set(requestSensors);
+
+        // Initialize internode bytes with the inbound message size:
+        Collection<TableMetadata> tables = message.payload.getPartitionUpdates().stream().map(PartitionUpdate::metadata).collect(Collectors.toList());
+        for (TableMetadata tm : tables)
+        {
+            Context context = Context.from(tm);
+            requestSensors.registerSensor(context, Type.INTERNODE_BYTES);
+            requestSensors.incrementSensor(context, Type.INTERNODE_BYTES, message.payloadSize(MessagingService.current_version) / tables.size());
+        }
+
+        message.payload.applyFuture(WriteOptions.DEFAULT).addCallback(o -> respond(requestSensors, message, respondToAddress), wto -> failed());
     }
 
     private static void forwardToLocalNodes(Message<Mutation> originalMessage, ForwardingInfo forwardTo)
     {
         Message.Builder<Mutation> builder =
-            Message.builder(originalMessage)
-                   .withParam(ParamType.RESPOND_TO, originalMessage.from())
-                   .withoutParam(ParamType.FORWARD_TO);
+        Message.builder(originalMessage)
+               .withParam(ParamType.RESPOND_TO, originalMessage.from())
+               .withoutParam(ParamType.FORWARD_TO);
 
         // reuse the same Message if all ids are identical (as they will be for 4.0+ node originated messages)
         Message<Mutation> message = builder.build();
 
         forwardTo.forEach((id, target) ->
-        {
-            Tracing.trace("Enqueuing forwarded write to {}", target);
-            MessagingService.instance().send(message, target);
-        });
+                          {
+                              Tracing.trace("Enqueuing forwarded write to {}", target);
+                              MessagingService.instance().send(message, target);
+                          });
     }
 }
diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
index 095e2507f4c9..54990635f466 100644
--- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
+++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
@@ -122,6 +122,28 @@ private static PartitionRangeReadCommand create(boolean isDigest,
                                              trackWarnings);
     }
 
+    public static PartitionRangeReadCommand create(TableMetadata metadata,
+                                                   long nowInSec,
+                                                   ColumnFilter columnFilter,
+                                                   RowFilter rowFilter,
+                                                   DataLimits limits,
+                                                   DataRange dataRange,
+                                                   Index.QueryPlan indexQueryPlan,
+                                                   boolean trackWarnings)
+    {
+        return new PartitionRangeReadCommand(false,
+                                             0,
+                                             false,
+                                             metadata,
+                                             nowInSec,
+                                             columnFilter,
+                                             rowFilter,
+                                             limits,
+                                             dataRange,
+                                             indexQueryPlan,
+                                             trackWarnings);
+    }
+
     public static PartitionRangeReadCommand create(TableMetadata metadata,
                                                    long nowInSec,
                                                    ColumnFilter columnFilter,
@@ -299,16 +321,22 @@ public boolean isReversed()
         return dataRange.isReversed();
     }
 
+    @Override
     public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
     {
-        return StorageProxy.getRangeSlice(this, consistency, requestTime);
+        return StorageProxy.getRangeSlice(this, consistency, requestTime, state);
     }
 
-    protected void recordLatency(TableMetrics metric, long latencyNanos)
+    protected void recordReadLatency(TableMetrics metric, long latencyNanos)
     {
         metric.rangeLatency.addNano(latencyNanos);
     }
 
+    protected void recordReadRequest(TableMetrics metric)
+    {
+        metric.rangeRequests.inc();
+    }
+
     @VisibleForTesting
     public UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadExecutionController controller)
     {
@@ -319,11 +347,15 @@ public UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, Rea
         InputCollector<UnfilteredPartitionIterator> inputCollector = iteratorsForRange(view, controller);
         try
         {
+            // avoid iterating over the memtable if we purge all tombstones
+            boolean useMinLocalDeletionTime = cfs.onlyPurgeRepairedTombstones();
+
             SSTableReadsListener readCountUpdater = newReadCountUpdater();
             for (Memtable memtable : view.memtables)
             {
                 UnfilteredPartitionIterator iter = memtable.partitionIterator(columnFilter(), dataRange(), readCountUpdater);
-                controller.updateMinOldestUnrepairedTombstone(memtable.getMinLocalDeletionTime());
+                if (useMinLocalDeletionTime)
+                    controller.updateMinOldestUnrepairedTombstone(memtable.getMinLocalDeletionTime());
                 inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false));
             }
 
@@ -455,18 +487,6 @@ public String loggableTokens()
                (dataRange.keyRange.inclusiveRight() ? ']' : ')');
     }
 
-    /**
-     * Allow to post-process the result of the query after it has been reconciled on the coordinator
-     * but before it is passed to the CQL layer to return the ResultSet.
-     *
-     * See CASSANDRA-8717 for why this exists.
-     */
-    public PartitionIterator postReconciliationProcessing(PartitionIterator result)
-    {
-        Index.QueryPlan queryPlan = indexQueryPlan();
-        return queryPlan == null ? result : queryPlan.postProcessor(this).apply(result);
-    }
-
     @Override
     public String toString()
     {
diff --git a/src/java/org/apache/cassandra/db/RangeTombstoneList.java b/src/java/org/apache/cassandra/db/RangeTombstoneList.java
index 8b8cee2d39bd..b50802115f44 100644
--- a/src/java/org/apache/cassandra/db/RangeTombstoneList.java
+++ b/src/java/org/apache/cassandra/db/RangeTombstoneList.java
@@ -21,6 +21,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.SortedSet;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.utils.AbstractIterator;
@@ -333,6 +334,14 @@ private RangeTombstone rangeTombstone(int idx)
         return new RangeTombstone(Slice.make(starts[idx], ends[idx]), DeletionTime.buildUnsafeWithUnsignedInteger(markedAts[idx], delTimesUnsignedIntegers[idx]));
     }
 
+    /**
+     * Return range tombstone with give clustering and recorded deletion time.
+     */
+    private RangeTombstone rangeTombstone(int idx, Clustering clustering)
+    {
+        return new RangeTombstone(Slice.make(clustering), DeletionTime.buildUnsafeWithUnsignedInteger(markedAts[idx], delTimesUnsignedIntegers[idx]));
+    }
+
     private RangeTombstone rangeTombstoneWithNewStart(int idx, ClusteringBound<?> newStart)
     {
         return new RangeTombstone(Slice.make(newStart, ends[idx]), DeletionTime.buildUnsafeWithUnsignedInteger(markedAts[idx], delTimesUnsignedIntegers[idx]));
@@ -382,6 +391,36 @@ protected RangeTombstone computeNext()
              };
     }
 
+    public Iterator<RangeTombstone> iterator(SortedSet<Clustering<?>> names, boolean isReversed)
+    {
+        return new AbstractIterator<RangeTombstone>() {
+
+            int startIdx = 0;
+            int endIdx = size;
+            Iterator<Clustering<?>> iterator = names.iterator();
+
+            @Override
+            protected RangeTombstone computeNext()
+            {
+                int idx = -1;
+                Clustering clustering = null;
+
+                while (idx < 0 && iterator.hasNext())
+                {
+                    clustering = iterator.next();
+                    idx = searchInternal(clustering, startIdx, endIdx);
+
+                    if (isReversed)
+                        endIdx = (idx < 0 ? -idx - 2 : idx) + 1; // exclusive
+                    else
+                        startIdx = idx < 0 ? -idx - 1 : idx;
+                }
+
+                return idx < 0 ? endOfData() : rangeTombstone(idx, clustering);
+            }
+        };
+    }
+
     public Iterator<RangeTombstone> iterator(final Slice slice, boolean reversed)
     {
         return reversed ? reverseIterator(slice) : forwardIterator(slice);
diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java
index a61d0a037ef7..d6d24f47140a 100644
--- a/src/java/org/apache/cassandra/db/ReadCommand.java
+++ b/src/java/org/apache/cassandra/db/ReadCommand.java
@@ -38,13 +38,29 @@
 import io.netty.util.concurrent.FastThreadLocal;
 import org.apache.cassandra.config.*;
 import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.guardrails.Threshold;
 import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.net.MessageFlag;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.ParamType;
 import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.db.partitions.*;
 import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PurgeFunction;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.db.transform.RTBoundCloser;
 import org.apache.cassandra.db.transform.RTBoundValidator;
 import org.apache.cassandra.db.transform.RTBoundValidator.Stage;
@@ -65,8 +81,9 @@
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.SchemaProvider;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.read.TrackingRowIterator;
 import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.CassandraUInt;
 import org.apache.cassandra.transport.Dispatcher;
@@ -109,7 +126,7 @@ public abstract class ReadCommand extends AbstractReadQuery
     protected final DataRange dataRange;
 
     @Nullable
-    private final Index.QueryPlan indexQueryPlan;
+    protected final Index.QueryPlan indexQueryPlan;
 
     protected static abstract class SelectionDeserializer
     {
@@ -129,7 +146,8 @@ public abstract ReadCommand deserialize(DataInputPlus in,
     protected enum Kind
     {
         SINGLE_PARTITION (SinglePartitionReadCommand.selectionDeserializer),
-        PARTITION_RANGE  (PartitionRangeReadCommand.selectionDeserializer);
+        PARTITION_RANGE  (PartitionRangeReadCommand.selectionDeserializer),
+        MULTI_RANGE      (MultiRangeReadCommand.selectionDeserializer);
 
         private final SelectionDeserializer selectionDeserializer;
 
@@ -375,10 +393,20 @@ public ReadResponse createResponse(UnfilteredPartitionIterator iterator, Repaire
     public ReadResponse createEmptyResponse()
     {
         UnfilteredPartitionIterator iterator = EmptyIterators.unfilteredPartition(metadata());
-        
+
         return isDigestQuery()
-               ? ReadResponse.createDigestResponse(iterator, this)
-               : ReadResponse.createDataResponse(iterator, this, RepairedDataInfo.NO_OP_REPAIRED_DATA_INFO);
+                ? ReadResponse.createDigestResponse(iterator, this)
+                : ReadResponse.createDataResponse(iterator, this, RepairedDataInfo.NO_OP_REPAIRED_DATA_INFO);
+    }
+    
+    public DataLimits.Counter createLimitedCounter(boolean assumeLiveData)
+    {
+        return limits().newCounter(nowInSec(), assumeLiveData, selectsFullPartition(), metadata().enforceStrictLiveness()).onlyCount();
+    }
+
+    public DataLimits.Counter createUnlimitedCounter(boolean assumeLiveData)
+    {
+        return DataLimits.NONE.newCounter(nowInSec(), assumeLiveData, selectsFullPartition(), metadata().enforceStrictLiveness());
     }
 
     long indexSerializedSize(int version)
@@ -388,6 +416,13 @@ long indexSerializedSize(int version)
              : 0;
     }
 
+    public Index getIndex(ColumnFamilyStore cfs)
+    {
+        return null != indexQueryPlan
+             ? indexQueryPlan.getFirst()
+             : null;
+    }
+
     static Index.QueryPlan findIndexQueryPlan(TableMetadata table, RowFilter rowFilter)
     {
         if (table.indexes.isEmpty() || rowFilter.isEmpty())
@@ -405,7 +440,7 @@ static Index.QueryPlan findIndexQueryPlan(TableMetadata table, RowFilter rowFilt
      * violates the implementation specific validation rules.
      */
     @Override
-    public void maybeValidateIndex()
+    public void maybeValidateIndexes()
     {
         if (null != indexQueryPlan)
         {
@@ -447,7 +482,10 @@ public UnfilteredPartitionIterator executeLocally(ReadExecutionController execut
                                             .collect(Collectors.joining(",")));
             }
 
-            UnfilteredPartitionIterator iterator = (null == searcher) ? queryStorage(cfs, executionController) : searcher.search(executionController);
+            Context context = Context.from(this);
+            var storageTarget = (null == searcher) ? queryStorage(cfs, executionController)
+                                                   : searchStorage(searcher, executionController);
+            UnfilteredPartitionIterator iterator = Transformation.apply(storageTarget, new TrackingRowIterator(context));
             iterator = RTBoundValidator.validate(iterator, Stage.MERGED, false);
 
             try
@@ -455,6 +493,7 @@ public UnfilteredPartitionIterator executeLocally(ReadExecutionController execut
                 iterator = withQuerySizeTracking(iterator);
                 iterator = maybeSlowDownForTesting(iterator);
                 iterator = withQueryCancellation(iterator);
+                iterator = withReadObserver(iterator);
                 iterator = RTBoundValidator.validate(withoutPurgeableTombstones(iterator, cfs, executionController), Stage.PURGED, false);
                 iterator = withMetricsRecording(iterator, cfs.metric, startTimeNanos);
 
@@ -504,18 +543,101 @@ public UnfilteredPartitionIterator executeLocally(ReadExecutionController execut
         }
     }
 
-    protected abstract void recordLatency(TableMetrics metric, long latencyNanos);
+    public UnfilteredPartitionIterator withReadObserver(UnfilteredPartitionIterator partitions)
+    {
+        ReadObserver observer = ReadObserverFactory.instance.create(this.metadata());
+
+        // skip if observer is disabled
+        if (observer == ReadObserver.NO_OP)
+            return partitions;
+
+        class ReadObserverTransformation extends Transformation<UnfilteredRowIterator>
+        {
+            @Override
+            protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+            {
+                observer.onPartition(partition.partitionKey(), partition.partitionLevelDeletion());
+                return Transformation.apply(partition, this);
+            }
+
+            @Override
+            protected Row applyToStatic(Row row)
+            {
+                if (!row.isEmpty())
+                    observer.onStaticRow(row);
+                return row;
+            }
+
+            @Override
+            protected Row applyToRow(Row row)
+            {
+                observer.onUnfiltered(row);
+                return row;
+            }
+
+            @Override
+            protected RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker)
+            {
+                observer.onUnfiltered(marker);
+                return marker;
+            }
+
+            @Override
+            protected void onClose()
+            {
+                observer.onComplete();
+            }
+        }
+
+        return Transformation.apply(partitions, new ReadObserverTransformation());
+    }
+
+    public UnfilteredPartitionIterator searchStorage(Index.Searcher searcher, ReadExecutionController executionController)
+    {
+        return searcher.search(executionController);
+    }
+
+    protected abstract void recordReadRequest(TableMetrics metric);
+    protected abstract void recordReadLatency(TableMetrics metric, long latencyNanos);
 
     public ReadExecutionController executionController(boolean trackRepairedStatus)
     {
         return ReadExecutionController.forCommand(this, trackRepairedStatus);
     }
 
+    /**
+     * Allow to post-process the result of the query after it has been reconciled on the coordinator
+     * but before it is passed to the CQL layer to return the ResultSet.
+     *
+     * See CASSANDRA-8717 for why this exists.
+     */
+    public PartitionIterator postReconciliationProcessing(PartitionIterator result)
+    {
+        return indexQueryPlan == null ? result : indexQueryPlan.postProcessor(this).apply(result);
+    }
+
+    @Override
+    public PartitionIterator executeInternal(ReadExecutionController controller)
+    {
+        return postReconciliationProcessing(UnfilteredPartitionIterators.filter(executeLocally(controller), nowInSec()));
+    }
+
     public ReadExecutionController executionController()
     {
         return ReadExecutionController.forCommand(this, false);
     }
 
+    /**
+     * Whether tombstone guardrail ({@link Guardrails#scannedTombstones} should be respected for this query.
+     *
+     * @return {@code true} if the tombstone thresholds should be respected for the query. If {@code false}, no
+     * tombstone warning will ever be logged, and the query will never fail due to tombstones.
+     */
+    protected boolean shouldRespectTombstoneThresholds()
+    {
+        return !SchemaConstants.isLocalSystemKeyspace(ReadCommand.this.metadata().keyspace);
+    }
+
     /**
      * Wraps the provided iterator so that metrics on what is scanned by the command are recorded.
      * This also log warning/trow TombstoneOverwhelmingException if appropriate.
@@ -524,19 +646,28 @@ private UnfilteredPartitionIterator withMetricsRecording(UnfilteredPartitionIter
     {
         class MetricRecording extends Transformation<UnfilteredRowIterator>
         {
-            private final int failureThreshold = DatabaseDescriptor.getTombstoneFailureThreshold();
-            private final int warningThreshold = DatabaseDescriptor.getTombstoneWarnThreshold();
-
-            private final boolean respectTombstoneThresholds = !SchemaConstants.isLocalSystemKeyspace(ReadCommand.this.metadata().keyspace);
             private final boolean enforceStrictLiveness = metadata().enforceStrictLiveness();
 
             private int liveRows = 0;
             private int lastReportedLiveRows = 0;
-            private int tombstones = 0;
-            private int lastReportedTombstones = 0;
+            private final Threshold.GuardedCounter tombstones = createTombstoneCounter();
+            private long lastReportedTombstones = 0;
 
             private DecoratedKey currentKey;
 
+            private Threshold.GuardedCounter createTombstoneCounter()
+            {
+                Threshold guardrail = shouldRespectTombstoneThresholds()
+                                                ? Guardrails.scannedTombstones
+                                                : Threshold.NEVER_TRIGGERED;
+                return guardrail.newCounter(ReadCommand.this::toCQLString, true, null);
+            }
+
+            private MetricRecording()
+            {
+                recordReadRequest(metric);
+            }
+
             @Override
             public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator iter)
             {
@@ -585,18 +716,18 @@ public RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker)
 
             private void countTombstone(ClusteringPrefix<?> clustering)
             {
-                ++tombstones;
-                if (tombstones > failureThreshold && respectTombstoneThresholds)
+                try
+                {
+                    tombstones.add(1);
+                }
+                catch (InvalidRequestException e)
                 {
-                    String query = ReadCommand.this.toCQLString();
-                    Tracing.trace("Scanned over {} tombstones for query {}; query aborted (see tombstone_failure_threshold)", failureThreshold, query);
                     metric.tombstoneFailures.inc();
-                    if (trackWarnings)
-                    {
-                        MessageParams.remove(ParamType.TOMBSTONE_WARNING);
-                        MessageParams.add(ParamType.TOMBSTONE_FAIL, tombstones);
-                    }
-                    throw new TombstoneOverwhelmingException(tombstones, query, ReadCommand.this.metadata(), currentKey, clustering);
+                    throw new TombstoneOverwhelmingException(tombstones.get(),
+                                                             ReadCommand.this.toCQLString(),
+                                                             ReadCommand.this.metadata(),
+                                                             currentKey,
+                                                             clustering);
                 }
             }
 
@@ -604,7 +735,7 @@ private void countTombstone(ClusteringPrefix<?> clustering)
             protected void onPartitionClose()
             {
                 int lr = liveRows - lastReportedLiveRows;
-                int ts = tombstones - lastReportedTombstones;
+                long ts = tombstones.get() - lastReportedTombstones;
 
                 if (lr > 0)
                     metric.topReadPartitionRowCount.addSample(currentKey.getKey(), lr);
@@ -613,38 +744,18 @@ protected void onPartitionClose()
                     metric.topReadPartitionTombstoneCount.addSample(currentKey.getKey(), ts);
 
                 lastReportedLiveRows = liveRows;
-                lastReportedTombstones = tombstones;
+                lastReportedTombstones = tombstones.get();
             }
 
             @Override
             public void onClose()
             {
-                recordLatency(metric, nanoTime() - startTimeNanos);
-
-                metric.tombstoneScannedHistogram.update(tombstones);
-                metric.liveScannedHistogram.update(liveRows);
-
-                boolean warnTombstones = tombstones > warningThreshold && respectTombstoneThresholds;
-                if (warnTombstones)
-                {
-                    String msg = String.format(
-                            "Read %d live rows and %d tombstone cells for query %1.512s; token %s (see tombstone_warn_threshold)",
-                            liveRows, tombstones, ReadCommand.this.toCQLString(), currentKey.getToken());
-                    if (trackWarnings)
-                        MessageParams.add(ParamType.TOMBSTONE_WARNING, tombstones);
-                    else
-                        ClientWarn.instance.warn(msg);
-                    if (tombstones < failureThreshold)
-                    {
-                        metric.tombstoneWarnings.inc();
-                    }
+                recordReadLatency(metric, nanoTime() - startTimeNanos);
 
-                    logger.warn(msg);
-                }
+                metric.incLiveRows(liveRows);
+                metric.incTombstones(tombstones.get(), tombstones.checkAndTriggerWarning());
 
-                Tracing.trace("Read {} live rows and {} tombstone cells{}",
-                        liveRows, tombstones,
-                        (warnTombstones ? " (see tombstone_warn_threshold)" : ""));
+                Tracing.trace("Read {} live rows and {} tombstone ones", liveRows, tombstones.get());
             }
         }
 
@@ -846,7 +957,7 @@ protected boolean hasPartitionLevelDeletions(SSTableReader sstable)
     // Skip purgeable tombstones. We do this because it's safe to do (post-merge of the memtable and sstable at least), it
     // can save us some bandwith, and avoid making us throw a TombstoneOverwhelmingException for purgeable tombstones (which
     // are to some extend an artefact of compaction lagging behind and hence counting them is somewhat unintuitive).
-    protected UnfilteredPartitionIterator withoutPurgeableTombstones(UnfilteredPartitionIterator iterator, 
+    protected UnfilteredPartitionIterator withoutPurgeableTombstones(UnfilteredPartitionIterator iterator,
                                                                      ColumnFamilyStore cfs,
                                                                      ReadExecutionController controller)
     {
@@ -855,7 +966,7 @@ class WithoutPurgeableTombstones extends PurgeFunction
             public WithoutPurgeableTombstones()
             {
                 super(nowInSec(), cfs.gcBefore(nowInSec()), controller.oldestUnrepairedTombstone(),
-                      cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones(),
+                      cfs.onlyPurgeRepairedTombstones(),
                       iterator.metadata().enforceStrictLiveness());
             }
 
@@ -935,7 +1046,7 @@ static class InputCollector<T extends AutoCloseable>
         {
             this.repairedDataInfo = controller.getRepairedDataInfo();
             this.isTrackingRepairedStatus = controller.isTrackingRepairedStatus();
-            
+
             if (isTrackingRepairedStatus)
             {
                 for (SSTableReader sstable : view.sstables)
@@ -1122,7 +1233,7 @@ public void serialize(ReadCommand command, DataOutputPlus out, int version) thro
             if (command.isDigestQuery())
                 out.writeUnsignedVInt32(command.digestVersion());
             command.metadata().id.serialize(out);
-            out.writeInt(version >= MessagingService.VERSION_50 ? CassandraUInt.fromLong(command.nowInSec()) : (int) command.nowInSec());
+            out.writeInt(MessagingService.Version.supportsExtendedDeletionTime(version) ? CassandraUInt.fromLong(command.nowInSec()) : (int) command.nowInSec());
             ColumnFilter.serializer.serialize(command.columnFilter(), out, version);
             RowFilter.serializer.serialize(command.rowFilter(), out, version);
             DataLimits.serializer.serialize(command.limits(), out, version, command.metadata().comparator);
@@ -1145,17 +1256,31 @@ public ReadCommand deserialize(DataInputPlus in, int version) throws IOException
             // better complain loudly than doing the wrong thing.
             if (isForThrift(flags))
                 throw new IllegalStateException("Received a command with the thrift flag set. "
-                                              + "This means thrift is in use in a mixed 3.0/3.X and 4.0+ cluster, "
-                                              + "which is unsupported. Make sure to stop using thrift before "
-                                              + "upgrading to 4.0");
+                                               + "This means thrift is in use in a mixed 3.0/3.X and 4.0+ cluster, "
+                                               + "which is unsupported. Make sure to stop using thrift before "
+                                               + "upgrading to 4.0");
 
             boolean hasIndex = hasIndex(flags);
             int digestVersion = isDigest ? in.readUnsignedVInt32() : 0;
             boolean needsReconciliation = needsReconciliation(flags);
 
             TableMetadata metadata = schema.getExistingTableMetadata(TableId.deserialize(in));
-            long nowInSec = version >= MessagingService.VERSION_50 ? CassandraUInt.toLong(in.readInt()) : in.readInt();
+            long nowInSec = MessagingService.Version.supportsExtendedDeletionTime(version) ? CassandraUInt.toLong(in.readInt()) : in.readInt();
             ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata);
+
+            // add synthetic columns to the tablemetadata so we can serialize them in our response
+            var tmb = metadata.unbuild();
+            for (var it = columnFilter.fetchedColumns().regulars.simpleColumns(); it.hasNext(); )
+            {
+                var c = it.next();
+                // synthetic columns sort first, so when we hit the first non-synthetic, we're done
+                if (!c.isSynthetic())
+                    break;
+                assert c.sythenticSourceColumn != null;
+                tmb.addColumn(c);
+            }
+            metadata = tmb.build();
+
             RowFilter rowFilter = RowFilter.serializer.deserialize(in, version, metadata, needsReconciliation);
             DataLimits limits = DataLimits.serializer.deserialize(in, version,  metadata);
 
@@ -1163,16 +1288,19 @@ public ReadCommand deserialize(DataInputPlus in, int version) throws IOException
             if (hasIndex)
             {
                 IndexMetadata index = deserializeIndexMetadata(in, version, metadata);
-                Index.Group indexGroup =  Keyspace.openAndGetStore(metadata).indexManager.getIndexGroup(index);
-                if (indexGroup != null)
-                    indexQueryPlan = indexGroup.queryPlanFor(rowFilter);
+                if (index != null)
+                {
+                    Index.Group indexGroup = Keyspace.openAndGetStore(metadata).indexManager.getIndexGroup(index);
+                    if (indexGroup != null)
+                        indexQueryPlan = indexGroup.queryPlanFor(rowFilter);
+                }
             }
 
             return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, acceptsTransient, 
                                                           metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan);
         }
 
-        private IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, TableMetadata metadata) throws IOException
+        private @Nullable IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, TableMetadata metadata) throws IOException
         {
             try
             {
diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
index 58d45c998f3f..1db851ec98e1 100644
--- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
@@ -31,7 +31,14 @@
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.RequestTracker;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.SensorsFactory;
+import org.apache.cassandra.sensors.Type;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.NoSpamLogger;
 
@@ -47,9 +54,21 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
 
     public void doVerb(Message<ReadCommand> message)
     {
-        if (StorageService.instance.isBootstrapMode())
+        TableMetadata metadata = message.payload.metadata();
+        if (metadata.isVirtual())
         {
-            throw new RuntimeException("Cannot service reads while bootstrapping!");
+            if (StorageService.instance.isBootstrapMode())
+            {
+                throw new RuntimeException("Cannot service reads while bootstrapping!");
+            }
+        }
+        else
+        {
+            ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata);
+            if (!cfs.isReadyToServeData())
+            {
+                throw new RuntimeException("Cannot service reads while bootstrapping!");
+            }
         }
 
         ReadCommand command = message.payload;
@@ -80,6 +99,16 @@ public void doVerb(Message<ReadCommand> message)
         validateTransientStatus(message);
         MessageParams.reset();
 
+        // Initialize the sensor and set ExecutorLocals
+        RequestSensors requestSensors = SensorsFactory.instance.createRequestSensors(command.metadata().keyspace);
+        Context context = Context.from(command);
+        requestSensors.registerSensor(context, Type.READ_BYTES);
+        RequestTracker.instance.set(requestSensors);
+
+        // Initialize internode bytes with the inbound message size:
+        requestSensors.registerSensor(context, Type.INTERNODE_BYTES);
+        requestSensors.incrementSensor(context, Type.INTERNODE_BYTES, message.payloadSize(MessagingService.current_version));
+
         long timeout = message.expiresAtNanos() - message.createdAtNanos();
         command.setMonitoringTime(message.createdAtNanos(), message.isCrossNode(), timeout, DatabaseDescriptor.getSlowQueryTimeout(NANOSECONDS));
 
@@ -120,8 +149,14 @@ public void doVerb(Message<ReadCommand> message)
 
         if (command.complete())
         {
+            Message.Builder<ReadResponse> replyBuilder = message.responseWithBuilder(response);
+            int size = replyBuilder.currentPayloadSize(MessagingService.current_version);
+            requestSensors.incrementSensor(context, Type.INTERNODE_BYTES, size);
+            requestSensors.syncAllSensors();
+            SensorsCustomParams.addSensorsToInternodeResponse(requestSensors, replyBuilder);
+
             Tracing.trace("Enqueuing response to {}", message.from());
-            Message<ReadResponse> reply = message.responseWith(response);
+            Message<ReadResponse> reply = replyBuilder.build();
             reply = MessageParams.addToMessage(reply);
             MessagingService.instance().send(reply, message.from());
         }
@@ -141,8 +176,10 @@ private void validateTransientStatus(Message<ReadCommand> message)
 
         if (command instanceof SinglePartitionReadCommand)
             token = ((SinglePartitionReadCommand) command).partitionKey().getToken();
-        else
+        else if (command instanceof PartitionRangeReadCommand)
             token = ((PartitionRangeReadCommand) command).dataRange().keyRange().right.getToken();
+        else
+            return;
 
         Replica replica = Keyspace.open(command.metadata().keyspace)
                                   .getReplicationStrategy()
@@ -150,6 +187,7 @@ private void validateTransientStatus(Message<ReadCommand> message)
 
         if (replica == null)
         {
+            // it's fine for serverless which unloads stale sstables, SEE VECTOR-30
             if (command.isTopK())
                 return;
 
diff --git a/src/java/org/apache/cassandra/db/ReadExecutionController.java b/src/java/org/apache/cassandra/db/ReadExecutionController.java
index 8a62ea390d3e..d4625705b390 100644
--- a/src/java/org/apache/cassandra/db/ReadExecutionController.java
+++ b/src/java/org/apache/cassandra/db/ReadExecutionController.java
@@ -22,9 +22,14 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import com.codahale.metrics.Histogram;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.metrics.DecayingEstimatedHistogramReservoir;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.context.OperationContext;
+import org.apache.cassandra.service.context.OperationContextTracker;
+import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.MonotonicClock;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 
@@ -49,6 +54,8 @@ public class ReadExecutionController implements AutoCloseable
     private final RepairedDataInfo repairedDataInfo;
     private long oldestUnrepairedTombstone = Long.MAX_VALUE;
 
+    public final Histogram sstablesScannedPerRowRead;
+
     ReadExecutionController(ReadCommand command,
                             OpOrder.Group baseOp,
                             TableMetadata baseMetadata,
@@ -67,6 +74,8 @@ public class ReadExecutionController implements AutoCloseable
         this.command = command;
         this.createdAtNanos = createdAtNanos;
 
+        this.sstablesScannedPerRowRead = new Histogram(new DecayingEstimatedHistogramReservoir(true));
+
         if (trackRepairedStatus)
         {
             DataLimits.Counter repairedReadCount = command.limits().newCounter(command.nowInSec(),
@@ -79,6 +88,9 @@ public class ReadExecutionController implements AutoCloseable
         {
             repairedDataInfo = RepairedDataInfo.NO_OP_REPAIRED_DATA_INFO;
         }
+
+        if (Tracing.isTracing())
+            Tracing.instance.setRangeQuery(isRangeCommand());
     }
 
     public boolean isRangeCommand()
@@ -133,6 +145,8 @@ static ReadExecutionController forCommand(ReadCommand command, boolean trackRepa
 
         long createdAtNanos = baseCfs.metric.topLocalReadQueryTime.isEnabled() ? clock.now() : NO_SAMPLING;
 
+        OperationContextTracker.start(OperationContext.FACTORY.forRead(command, baseCfs));
+
         if (indexCfs == null)
             return new ReadExecutionController(command, baseCfs.readOrdering.start(), baseCfs.metadata(), null, null, createdAtNanos, trackRepairedStatus);
 
@@ -166,6 +180,7 @@ static ReadExecutionController forCommand(ReadCommand command, boolean trackRepa
                 if (indexController != null)
                     indexController.close();
             }
+            OperationContextTracker.endCurrent();
             throw e;
         }
     }
@@ -207,9 +222,21 @@ public void close()
             }
         }
 
+        OperationContextTracker.endCurrent();
+
         if (createdAtNanos != NO_SAMPLING)
             addSample();
-    }
+
+        if (Tracing.traceSinglePartitions())
+        {
+            var sstablesHistogram = sstablesScannedPerRowRead.getSnapshot();
+            Tracing.trace("Scanned {} rows; average {} sstables scanned per row with stdev {} and max {}",
+                          sstablesScannedPerRowRead.getCount(),
+                          sstablesHistogram.getMean(),
+                          sstablesHistogram.getStdDev(),
+                          sstablesHistogram.getMax());
+        }
+}
 
     public boolean isTrackingRepairedStatus()
     {
@@ -241,4 +268,9 @@ private void addSample()
         if (cfs != null)
             cfs.metric.topLocalReadQueryTime.addSample(cql, timeMicros);
     }
+
+    public void updateSstablesIteratedPerRow(int mergedSSTablesIterated)
+    {
+        sstablesScannedPerRowRead.update(mergedSSTablesIterated);
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/ReadObserver.java b/src/java/org/apache/cassandra/db/ReadObserver.java
new file mode 100644
index 000000000000..65767af435fe
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/ReadObserver.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+
+/**
+ * An interface that allows to capture what local data has been read
+ * <p>
+ * This is used by CNDB remote file cache warmup strategy to track access pattern
+ */
+public interface ReadObserver
+{
+    ReadObserver NO_OP = new ReadObserver() {};
+
+    /**
+     * Called on every partition read
+     *
+     * @param partitionKey the partition key
+     * @param deletionTime partition deletion time
+     */
+    default void onPartition(DecoratedKey partitionKey, DeletionTime deletionTime) {}
+
+    /**
+     * Called on every static row read.
+     *
+     * @param staticRow static row of the partition
+     */
+    default void onStaticRow(Row staticRow) {}
+
+    /**
+     * Called on every unfiltered read.
+     *
+     * @param unfiltered either row or range tombstone.
+     */
+    default void onUnfiltered(Unfiltered unfiltered) {}
+
+    /**
+     * Called on read request completion
+     */
+    default void onComplete() {}
+}
diff --git a/src/java/org/apache/cassandra/db/ReadObserverFactory.java b/src/java/org/apache/cassandra/db/ReadObserverFactory.java
new file mode 100644
index 000000000000..a0539eb70eeb
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/ReadObserverFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_READ_OBSERVER_FACTORY;
+
+
+/**
+ * Provides custom factory that creates a {@link ReadObserver} instance per read request
+ */
+public interface ReadObserverFactory
+{
+    ReadObserverFactory instance = CUSTOM_READ_OBSERVER_FACTORY.getString() == null ?
+                                   new ReadObserverFactory() {} :
+                                   FBUtilities.construct(CassandraRelevantProperties.CUSTOM_READ_OBSERVER_FACTORY.getString(), "custom read observer factory");
+
+    default ReadObserver create(TableMetadata table)
+    {
+        return ReadObserver.NO_OP;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/ReadQuery.java b/src/java/org/apache/cassandra/db/ReadQuery.java
index ee383b963194..3d6db1fb1be8 100644
--- a/src/java/org/apache/cassandra/db/ReadQuery.java
+++ b/src/java/org/apache/cassandra/db/ReadQuery.java
@@ -256,7 +256,7 @@ public default boolean isEmpty()
      * validation method to check that nothing in this query's parameters
      * violates the implementation specific validation rules.
      */
-    default void maybeValidateIndex()
+    default void maybeValidateIndexes()
     {
     }
 
@@ -265,10 +265,7 @@ default void trackWarnings()
     }
 
     /**
-     * The query is a top-k query if the query has an {@link org.apache.cassandra.index.Index.QueryPlan} that
-     * supports top-k ordering.
-     *
-     * @return {@code true} if this is a top-k query
+     * @return true given read query is a top-k request
      */
     default boolean isTopK()
     {
diff --git a/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java b/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java
index a34be9d9c1ac..20956448bf28 100644
--- a/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java
@@ -38,7 +38,7 @@ public void doVerb(Message<Mutation> message) throws IOException
     @Override
     void applyMutation(Message<Mutation> message, InetAddressAndPort respondToAddress)
     {
-        message.payload.apply();
+        message.payload.apply(WriteOptions.FOR_READ_REPAIR);
         MessagingService.instance().send(message.emptyResponse(), respondToAddress);
     }
 }
diff --git a/src/java/org/apache/cassandra/db/RegularAndStaticColumns.java b/src/java/org/apache/cassandra/db/RegularAndStaticColumns.java
index b6da183d013f..55533eda0e97 100644
--- a/src/java/org/apache/cassandra/db/RegularAndStaticColumns.java
+++ b/src/java/org/apache/cassandra/db/RegularAndStaticColumns.java
@@ -163,7 +163,7 @@ public Builder add(ColumnMetadata c)
             }
             else
             {
-                assert c.isRegular();
+                assert c.isRegular() || c.isSynthetic();
                 if (regularColumns == null)
                     regularColumns = BTree.builder(naturalOrder());
                 regularColumns.add(c);
@@ -197,7 +197,7 @@ public Builder addAll(RegularAndStaticColumns columns)
 
         public RegularAndStaticColumns build()
         {
-            return new RegularAndStaticColumns(staticColumns  == null ? Columns.NONE : Columns.from(staticColumns),
+            return new RegularAndStaticColumns(staticColumns == null ? Columns.NONE : Columns.from(staticColumns),
                                                regularColumns == null ? Columns.NONE : Columns.from(regularColumns));
         }
     }
diff --git a/src/java/org/apache/cassandra/db/RepairedDataInfo.java b/src/java/org/apache/cassandra/db/RepairedDataInfo.java
index 1f03654d25c7..347e2faf7a3d 100644
--- a/src/java/org/apache/cassandra/db/RepairedDataInfo.java
+++ b/src/java/org/apache/cassandra/db/RepairedDataInfo.java
@@ -39,7 +39,7 @@
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 @NotThreadSafe
-class RepairedDataInfo
+public class RepairedDataInfo
 {
     public static final RepairedDataInfo NO_OP_REPAIRED_DATA_INFO = new RepairedDataInfo(null)
     {
@@ -332,7 +332,7 @@ private static class RepairedDataPurger extends PurgeFunction
             super(nowInSec,
                   cfs.gcBefore(nowInSec),
                   oldestUnrepairedTombstone,
-                  cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones(),
+                  cfs.onlyPurgeRepairedTombstones(),
                   cfs.metadata.get().enforceStrictLiveness());
         }
 
diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java
index 8ad79003d37b..42e10ebab1a3 100644
--- a/src/java/org/apache/cassandra/db/SSTableImporter.java
+++ b/src/java/org/apache/cassandra/db/SSTableImporter.java
@@ -28,14 +28,18 @@
 import java.util.UUID;
 
 import com.google.common.annotations.VisibleForTesting;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.TargetParser;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.IVerifier;
@@ -44,6 +48,9 @@
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.OutputHandler;
@@ -107,23 +114,39 @@ synchronized List<String> importNewSSTables(Options options)
                                 Index.Group saiIndexGroup = cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.GROUP_KEY);
                                 if (saiIndexGroup != null)
                                 {
-                                    IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor,
-                                                                                             cfs.getPartitioner(),
-                                                                                             cfs.metadata().comparator);
-
                                     String keyspace = cfs.getKeyspaceName();
                                     String table = cfs.getTableName();
 
-                                    if (!indexDescriptor.isPerSSTableIndexBuildComplete())
+                                    SSTableReader reader = SSTableReader.open(cfs, descriptor);
+                                    StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+                                    if (group == null)
+                                        throw new IllegalStateException(String.format("Missing SAI index group to import for SSTable %s on %s.%s",
+                                                                                      descriptor.toString(),
+                                                                                      keyspace,
+                                                                                      table));
+
+                                    IndexDescriptor indexDescriptor = group.descriptorFor(reader);
+                                    if (!indexDescriptor.perSSTableComponents().isComplete())
                                         throw new IllegalStateException(String.format("Missing SAI index to import for SSTable %s on %s.%s",
-                                                                                      indexDescriptor.sstableDescriptor.toString(),
+                                                                                      descriptor.toString(),
                                                                                       keyspace,
                                                                                       table));
 
                                     for (Index index : saiIndexGroup.getIndexes())
                                     {
-                                        IndexIdentifier indexIdentifier = new IndexIdentifier(keyspace, table, index.getIndexMetadata().name);
-                                        if (!indexDescriptor.isPerColumnIndexBuildComplete(indexIdentifier))
+                                        TableMetadata tableMetadata = cfs.metadata();
+                                        IndexMetadata indexMetadata = index.getIndexMetadata();
+                                        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(tableMetadata, indexMetadata);
+                                        IndexContext indexContext = new IndexContext(tableMetadata.keyspace,
+                                                                                     tableMetadata.name,
+                                                                                     tableMetadata.id,
+                                                                                     tableMetadata.partitionKeyType,
+                                                                                     tableMetadata.comparator,
+                                                                                     target.left,
+                                                                                     target.right,
+                                                                                     indexMetadata,
+                                                                                     cfs);
+                                        if (!indexDescriptor.perIndexComponents(indexContext).isComplete())
                                             throw new IllegalStateException(String.format("Missing SAI index to import for index %s on %s.%s",
                                                                                           index.getIndexMetadata().name,
                                                                                           keyspace,
@@ -227,7 +250,7 @@ synchronized List<String> importNewSSTables(Options options)
             if (!cfs.indexManager.validateSSTableAttachedIndexes(newSSTables, false, options.validateIndexChecksum))
                 cfs.indexManager.buildSSTableAttachedIndexesBlocking(newSSTables);
 
-            cfs.getTracker().addSSTables(newSSTables);
+            cfs.getTracker().addSSTables(newSSTables, OperationType.UNKNOWN);
             for (SSTableReader reader : newSSTables)
             {
                 if (options.invalidateCaches && cfs.isRowCacheEnabled())
@@ -302,7 +325,7 @@ private File getTargetDirectory(String srcPath, Descriptor descriptor, Set<Compo
             if (sstable != null)
                 sstable.selfRef().release();
         }
-        return targetDirectory == null ? cfs.getDirectories().getWriteableLocationToLoadFile(descriptor.baseFile()) : targetDirectory;
+        return targetDirectory == null ? cfs.getDirectories().getWriteableLocationToLoadFile(new File(descriptor.baseFileUri())) : targetDirectory;
     }
 
     /**
@@ -323,7 +346,7 @@ private List<Pair<Directories.SSTableLister, String>> getSSTableListers(Set<Stri
                 {
                     throw new RuntimeException(String.format("Directory %s does not exist", path));
                 }
-                if (!Directories.verifyFullPermissions(dir, path))
+                if (!Directories.verifyFullPermissions(dir))
                 {
                     throw new RuntimeException("Insufficient permissions on directory " + path);
                 }
diff --git a/src/java/org/apache/cassandra/db/SerializationHeader.java b/src/java/org/apache/cassandra/db/SerializationHeader.java
index 841f7b305198..89353ad52623 100644
--- a/src/java/org/apache/cassandra/db/SerializationHeader.java
+++ b/src/java/org/apache/cassandra/db/SerializationHeader.java
@@ -19,15 +19,29 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CompositeType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.exceptions.InvalidColumnTypeException;
 import org.apache.cassandra.exceptions.UnknownColumnException;
+import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.metadata.IMetadataComponentSerializer;
@@ -42,6 +56,8 @@
 
 public class SerializationHeader
 {
+    private final static Logger logger = LoggerFactory.getLogger(SerializationHeader.class);
+
     public static final Serializer serializer = new Serializer();
 
     private final boolean isForSSTable;
@@ -82,7 +98,7 @@ public static SerializationHeader make(TableMetadata metadata, Collection<SSTabl
         // kind of ok because those stats are only used for optimizing the underlying storage format and so we
         // just have to strive for as good as possible. Currently, we stick to a relatively naive merge of existing
         // global stats because it's simple and probably good enough in most situation but we could probably
-        // improve our marging of inaccuracy through the use of more fine-grained stats in the future.
+        // improve our margin of inaccuracy through the use of more fine-grained stats in the future.
         // Note however that to avoid seeing our accuracy degrade through successive compactions, we don't base
         // our stats merging on the compacted files headers, which as we just said can be somewhat inaccurate,
         // but rather on their stats stored in StatsMetadata that are fully accurate.
@@ -223,7 +239,7 @@ public long ttlSerializedSize(int ttl)
     public long deletionTimeSerializedSize(DeletionTime dt)
     {
         return timestampSerializedSize(dt.markedForDeleteAt())
-             + localDeletionTimeSerializedSize(dt.localDeletionTime());
+               + localDeletionTimeSerializedSize(dt.localDeletionTime());
     }
 
     public void skipTimestamp(DataInputPlus in) throws IOException
@@ -249,8 +265,8 @@ public void skipDeletionTime(DataInputPlus in) throws IOException
 
     public Component toComponent()
     {
-        Map<ByteBuffer, AbstractType<?>> staticColumns = new LinkedHashMap<>();
-        Map<ByteBuffer, AbstractType<?>> regularColumns = new LinkedHashMap<>();
+        LinkedHashMap<ByteBuffer, AbstractType<?>> staticColumns = new LinkedHashMap<>();
+        LinkedHashMap<ByteBuffer, AbstractType<?>> regularColumns = new LinkedHashMap<>();
         for (ColumnMetadata column : columns.statics)
             staticColumns.put(column.name.bytes, column.type);
         for (ColumnMetadata column : columns.regulars)
@@ -272,14 +288,14 @@ public static class Component extends MetadataComponent
     {
         private final AbstractType<?> keyType;
         private final List<AbstractType<?>> clusteringTypes;
-        private final Map<ByteBuffer, AbstractType<?>> staticColumns;
-        private final Map<ByteBuffer, AbstractType<?>> regularColumns;
+        private final LinkedHashMap<ByteBuffer, AbstractType<?>> staticColumns;
+        private final LinkedHashMap<ByteBuffer, AbstractType<?>> regularColumns;
         private final EncodingStats stats;
 
         private Component(AbstractType<?> keyType,
                           List<AbstractType<?>> clusteringTypes,
-                          Map<ByteBuffer, AbstractType<?>> staticColumns,
-                          Map<ByteBuffer, AbstractType<?>> regularColumns,
+                          LinkedHashMap<ByteBuffer, AbstractType<?>> staticColumns,
+                          LinkedHashMap<ByteBuffer, AbstractType<?>> regularColumns,
                           EncodingStats stats)
         {
             this.keyType = keyType;
@@ -289,12 +305,159 @@ private Component(AbstractType<?> keyType,
             this.stats = stats;
         }
 
+        /**
+         * <em>Only</em> exposed for {@link org.apache.cassandra.io.sstable.SSTableHeaderFix}.
+         */
+        public static Component buildComponentForTools(AbstractType<?> keyType,
+                                                       List<AbstractType<?>> clusteringTypes,
+                                                       LinkedHashMap<ByteBuffer, AbstractType<?>> staticColumns,
+                                                       LinkedHashMap<ByteBuffer, AbstractType<?>> regularColumns,
+                                                       EncodingStats stats)
+        {
+            return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats);
+        }
+
         public MetadataType getType()
         {
             return MetadataType.HEADER;
         }
 
-        public SerializationHeader toHeader(TableMetadata metadata) throws UnknownColumnException
+        private static AbstractType<?> validateAndMaybeFixColumnType(String description,
+                                                                     TableMetadata metadata,
+                                                                     ByteBuffer columnName,
+                                                                     AbstractType<?> type,
+                                                                     boolean allowImplicitlyFrozenTuples,
+                                                                     boolean isForOfflineTool)
+        {
+            boolean dropped = metadata.getDroppedColumn(columnName) != null;
+            boolean isPrimaryKeyColumn = Iterables.any(metadata.primaryKeyColumns(), cd -> cd.name.bytes.equals(columnName));
+
+            try
+            {
+                type.validateForColumn(columnName, isPrimaryKeyColumn, metadata.isCounter(), dropped, isForOfflineTool);
+                return type;
+            }
+            catch (InvalidColumnTypeException e)
+            {
+                AbstractType<?> fixed = allowImplicitlyFrozenTuples ? tryFix(type, columnName, isPrimaryKeyColumn, metadata.isCounter(), dropped, isForOfflineTool) : null;
+                if (fixed == null)
+                {
+                    // We don't know how to fix. We throw an error here because reading such table may result in corruption
+                    String msg = String.format("Error reading SSTable header %s, the type for column %s in %s is %s, which is invalid (%s); " +
+                                               "The type could not be automatically fixed.",
+                                               description, ColumnIdentifier.toCQLString(columnName), metadata, type.asCQL3Type().toSchemaString(),
+                                               e.getMessage());
+                    throw new IllegalArgumentException(msg, e);
+                }
+                else
+                {
+                    logger.debug("Error reading SSTable header {}, the type for column {} in {} is {}, which is " +
+                                 "invalid ({}); The type has been automatically fixed to {}, but please contact " +
+                                 "support if this is incorrect",
+                                 description, ColumnIdentifier.toCQLString(columnName), metadata, type.asCQL3Type().toSchemaString(),
+                                 e.getMessage(), fixed.asCQL3Type().toSchemaString());
+                    return fixed;
+                }
+            }
+        }
+
+        /**
+         * Attempts to return a "fixed" (and thus valid) version of the type. Doing is so is only possible in restrained
+         * case where we know why the type is invalid and are confident we know what it should be.
+         *
+         * @return if we know how to auto-magically fix the invalid type that triggered this exception, the hopefully
+         * fixed version of said type. Otherwise, {@code null}.
+         */
+        public static AbstractType<?> tryFix(AbstractType<?> invalidType, ByteBuffer name, boolean isPrimaryKeyColumn, boolean isCounterTable, boolean isDroppedColumn, boolean isForOfflineTool)
+        {
+            AbstractType<?> fixed = tryFixInternal(invalidType, isPrimaryKeyColumn, isDroppedColumn);
+            if (fixed != null)
+            {
+                try
+                {
+                    // Make doubly sure the fixed type is valid before returning it.
+                    fixed.validateForColumn(name, isPrimaryKeyColumn, isCounterTable, isDroppedColumn, isForOfflineTool);
+                    return fixed;
+                }
+                catch (InvalidColumnTypeException e2)
+                {
+                    // Continue as if we hadn't been able to fix, since we haven't
+                }
+            }
+            return null;
+        }
+
+        private static AbstractType<?> tryFixInternal(AbstractType<?> invalidType, boolean isPrimaryKeyColumn, boolean isDroppedColumn)
+        {
+            if (isPrimaryKeyColumn)
+            {
+                // The only issue we have a fix to in that case if the type is not frozen; we can then just freeze it.
+                if (invalidType.isMultiCell())
+                    return invalidType.freeze();
+            }
+            else
+            {
+                // Here again, it's mainly issues of frozen-ness that are fixable, namely multi-cell types that either:
+                // - are tuples, yet not for a dropped column (and so _should_ be frozen). In which case we freeze it.
+                // - has non-frozen subtypes. In which case, we just freeze all subtypes.
+                if (invalidType.isMultiCell())
+                {
+                    boolean isMultiCell = !invalidType.isTuple() || isDroppedColumn;
+                    return invalidType.with(AbstractType.freeze(invalidType.subTypes()), isMultiCell);
+                }
+
+            }
+            // In other case, we don't know how to fix (at least somewhat auto-magically) and will have to fail.
+            return null;
+        }
+
+        private static AbstractType<?> validateAndMaybeFixPartitionKeyType(String descriptor,
+                                                                           TableMetadata metadata,
+                                                                           AbstractType<?> fullType,
+                                                                           boolean allowImplicitlyFrozenTuples,
+                                                                           boolean isForOfflineTool)
+        {
+            List<ColumnMetadata> pkColumns = metadata.partitionKeyColumns();
+            int pkCount = pkColumns.size();
+
+            if (pkCount == 1)
+                return validateAndMaybeFixColumnType(descriptor, metadata, pkColumns.get(0).name.bytes, fullType, allowImplicitlyFrozenTuples, isForOfflineTool);
+
+            List<AbstractType<?>> subTypes = fullType.subTypes();
+            assert fullType instanceof CompositeType && subTypes.size() == pkCount
+            : String.format("In %s, got %s as table %s partition key type but partition key is %s",
+                            descriptor, fullType, metadata, pkColumns);
+
+            return CompositeType.getInstance(validateAndMaybeFixPKTypes(descriptor, metadata, pkColumns, subTypes, allowImplicitlyFrozenTuples, isForOfflineTool));
+        }
+
+        private static List<AbstractType<?>> validateAndMaybeFixPKTypes(String descriptor,
+                                                                        TableMetadata table,
+                                                                        List<ColumnMetadata> pkColumns,
+                                                                        List<AbstractType<?>> pkTypes,
+                                                                        boolean allowImplicitlyFrozenTuples,
+                                                                        boolean isForOfflineTool)
+        {
+            int count = pkTypes.size();
+            List<AbstractType<?>> updated = new ArrayList<>(count);
+            for (int i = 0; i < count; i++)
+            {
+                updated.add(validateAndMaybeFixColumnType(descriptor,
+                                                          table,
+                                                          pkColumns.get(i).name.bytes,
+                                                          pkTypes.get(i),
+                                                          allowImplicitlyFrozenTuples,
+                                                          isForOfflineTool));
+            }
+            return updated;
+        }
+
+        public SerializationHeader toHeader(Descriptor descriptor, TableMetadata metadata) throws UnknownColumnException
+        {
+            return toHeader(descriptor.toString(), metadata, descriptor.version, false);
+        }
+
+        public SerializationHeader toHeader(String descriptor, TableMetadata metadata, Version sstableVersion, boolean isForOfflineTool) throws UnknownColumnException
         {
             Map<ByteBuffer, AbstractType<?>> typeMap = new HashMap<>(staticColumns.size() + regularColumns.size());
 
@@ -305,8 +468,9 @@ public SerializationHeader toHeader(TableMetadata metadata) throws UnknownColumn
                 for (Map.Entry<ByteBuffer, AbstractType<?>> e : map.entrySet())
                 {
                     ByteBuffer name = e.getKey();
-                    AbstractType<?> other = typeMap.put(name, e.getValue());
-                    if (other != null && !other.equals(e.getValue()))
+                    AbstractType<?> type = validateAndMaybeFixColumnType(descriptor, metadata, name, e.getValue(), sstableVersion.hasImplicitlyFrozenTuples(), isForOfflineTool);
+                    AbstractType<?> other = typeMap.put(name, type);
+                    if (other != null && !other.equals(type))
                         throw new IllegalStateException("Column " + name + " occurs as both regular and static with types " + other + "and " + e.getValue());
 
                     ColumnMetadata column = metadata.getColumn(name);
@@ -319,7 +483,7 @@ public SerializationHeader toHeader(TableMetadata metadata) throws UnknownColumn
 
                         // If we don't find the definition, it could be we have data for a dropped column, and we shouldn't
                         // fail deserialization because of that. So we grab a "fake" ColumnDefinition that ensure proper
-                        // deserialization. The column will be ignore later on anyway.
+                        // deserialization. The column will be ignored later on anyway.
                         column = metadata.getDroppedColumn(name, isStatic);
                         if (column == null)
                             throw new UnknownColumnException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization");
@@ -328,6 +492,14 @@ public SerializationHeader toHeader(TableMetadata metadata) throws UnknownColumn
                 }
             }
 
+            AbstractType<?> keyType = validateAndMaybeFixPartitionKeyType(descriptor, metadata, this.keyType, sstableVersion.hasImplicitlyFrozenTuples(), isForOfflineTool);
+            List<AbstractType<?>> clusteringTypes = validateAndMaybeFixPKTypes(descriptor,
+                                                                               metadata,
+                                                                               metadata.clusteringColumns(),
+                                                                               this.clusteringTypes,
+                                                                               sstableVersion.hasImplicitlyFrozenTuples(),
+                                                                               isForOfflineTool);
+
             return new SerializationHeader(true, keyType, clusteringTypes, builder.build(), stats, typeMap);
         }
 
@@ -382,6 +554,28 @@ public EncodingStats getEncodingStats()
         {
             return stats;
         }
+
+        @SuppressWarnings("unused")
+        public Component withMigratedKeyspaces(Map<String, String> keyspaceMapping)
+        {
+            if (keyspaceMapping.isEmpty())
+                return this;
+
+            AbstractType<?> newKeyType = keyType.overrideKeyspace(ks -> keyspaceMapping.getOrDefault(ks, ks));
+            List<AbstractType<?>> clusteringTypes = this.clusteringTypes.stream().map(t -> t.overrideKeyspace(ks -> keyspaceMapping.getOrDefault(ks, ks))).collect(Collectors.toList());
+            LinkedHashMap<ByteBuffer, AbstractType<?>> staticColumns = this.staticColumns.entrySet().stream().collect(Collectors.toMap(
+                    Map.Entry::getKey,
+                    e -> e.getValue().overrideKeyspace(ks -> keyspaceMapping.getOrDefault(ks, ks)),
+                    (a, b) -> { throw new IllegalArgumentException("Duplicate key"); },
+                    LinkedHashMap::new));
+            LinkedHashMap<ByteBuffer, AbstractType<?>> regularColumns = this.regularColumns.entrySet().stream().collect(Collectors.toMap(
+                    Map.Entry::getKey,
+                    e -> e.getValue().overrideKeyspace(ks -> keyspaceMapping.getOrDefault(ks, ks)),
+                    (a, b) -> { throw new IllegalArgumentException("Duplicate key"); },
+                    LinkedHashMap::new));
+            return new Component(newKeyType, clusteringTypes, staticColumns, regularColumns, stats);
+        }
+
     }
 
     public static class Serializer implements IMetadataComponentSerializer<Component>
@@ -467,8 +661,8 @@ public Component deserialize(Version version, DataInputPlus in) throws IOExcepti
             AbstractType<?> keyType = typeSerializer.deserialize(in);
             List<AbstractType<?>> clusteringTypes = typeSerializer.deserializeList(in);
 
-            Map<ByteBuffer, AbstractType<?>> staticColumns = readColumnsWithType(in);
-            Map<ByteBuffer, AbstractType<?>> regularColumns = readColumnsWithType(in);
+            LinkedHashMap<ByteBuffer, AbstractType<?>> staticColumns = readColumnsWithType(in);
+            LinkedHashMap<ByteBuffer, AbstractType<?>> regularColumns = readColumnsWithType(in);
 
             return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats);
         }
@@ -507,10 +701,10 @@ private long sizeofColumnsWithTypes(Map<ByteBuffer, AbstractType<?>> columns)
             return size;
         }
 
-        private Map<ByteBuffer, AbstractType<?>> readColumnsWithType(DataInputPlus in) throws IOException
+        private LinkedHashMap<ByteBuffer, AbstractType<?>> readColumnsWithType(DataInputPlus in) throws IOException
         {
             int length =  in.readUnsignedVInt32();
-            Map<ByteBuffer, AbstractType<?>> typeMap = new LinkedHashMap<>(length);
+            LinkedHashMap<ByteBuffer, AbstractType<?>> typeMap = new LinkedHashMap<>(length);
             for (int i = 0; i < length; i++)
             {
                 ByteBuffer name = ByteBufferUtil.readWithVIntLength(in);
diff --git a/src/java/org/apache/cassandra/db/SimpleBuilders.java b/src/java/org/apache/cassandra/db/SimpleBuilders.java
index 3564eb1f100a..5466c7b2856d 100644
--- a/src/java/org/apache/cassandra/db/SimpleBuilders.java
+++ b/src/java/org/apache/cassandra/db/SimpleBuilders.java
@@ -223,7 +223,7 @@ public PartitionUpdate build()
 
             // Note that rowBuilders.size() could include the static column so could be 1 off the really need capacity
             // of the final PartitionUpdate, but as that's just a sizing hint, we'll live.
-            PartitionUpdate.Builder update = new PartitionUpdate.Builder(metadata, key, columns.build(), rowBuilders.size());
+            PartitionUpdate.Builder update = PartitionUpdate.builder(metadata, key, columns.build(), rowBuilders.size());
 
             update.addPartitionDeletion(partitionDeletion);
             if (rangeBuilders != null)
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
index 3abae3bd485b..2775bd7b8f8f 100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
@@ -26,6 +26,7 @@
 import java.util.NavigableSet;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -34,6 +35,7 @@
 import org.apache.cassandra.cache.IRowCacheEntry;
 import org.apache.cassandra.cache.RowCacheKey;
 import org.apache.cassandra.cache.RowCacheSentinel;
+import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.filter.ClusteringIndexFilter;
 import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
@@ -51,6 +53,7 @@
 import org.apache.cassandra.db.partitions.PartitionIterators;
 import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.BaseRowIterator;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Rows;
@@ -79,6 +82,7 @@
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.btree.BTreeSet;
 
@@ -474,14 +478,19 @@ public PartitionIterator execute(ConsistencyLevel consistency, ClientState state
         if (clusteringIndexFilter.isEmpty(metadata().comparator))
             return EmptyIterators.partition();
 
-        return StorageProxy.read(Group.one(this), consistency, requestTime);
+        return StorageProxy.read(Group.one(this), consistency, state, requestTime);
     }
 
-    protected void recordLatency(TableMetrics metric, long latencyNanos)
+    protected void recordReadLatency(TableMetrics metric, long latencyNanos)
     {
         metric.readLatency.addNano(latencyNanos);
     }
 
+    protected void recordReadRequest(TableMetrics metric)
+    {
+        metric.readRequests.inc();
+    }
+
     protected UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadExecutionController executionController)
     {
         // skip the row cache and go directly to sstables/memtable if repaired status of
@@ -528,7 +537,7 @@ private UnfilteredRowIterator getThroughCache(ColumnFamilyStore cfs, ReadExecuti
                 cfs.metric.rowCacheHit.inc();
                 Tracing.trace("Row cache hit");
                 UnfilteredRowIterator unfilteredRowIterator = clusteringIndexFilter().getUnfilteredRowIterator(columnFilter(), cachedPartition);
-                cfs.metric.updateSSTableIterated(0);
+                cfs.metric.updateSSTableIterated(0, 0, 0);
                 return unfilteredRowIterator;
             }
 
@@ -656,12 +665,35 @@ public Unfiltered next()
     public UnfilteredRowIterator queryMemtableAndDisk(ColumnFamilyStore cfs, ReadExecutionController executionController)
     {
         assert executionController != null && executionController.validForReadOn(cfs);
-        Tracing.trace("Executing single-partition query on {}", cfs.name);
+        if (Tracing.traceSinglePartitions())
+            Tracing.trace("Executing single-partition query on {}; stage READ pending: {}, active: {}", cfs.name, Stage.READ.getPendingTaskCount(), Stage.READ.getActiveTaskCount());
+
+        return queryMemtableAndDiskInternal(cfs, executionController, Clock.Global.nanoTime());
+    }
+
+    public UnfilteredRowIterator queryMemtableAndDisk(ColumnFamilyStore cfs,
+                                                      ColumnFamilyStore.ViewFragment view,
+                                                      Function<Object, Transformation<BaseRowIterator<?>>> rowTransformer,
+                                                      ReadExecutionController executionController)
+    {
+        assert executionController != null && executionController.validForReadOn(cfs);
+        if (Tracing.traceSinglePartitions())
+            Tracing.trace("Executing single-partition query on {}; stage READ pending: {}, active: {}", cfs.name, Stage.READ.getPendingTaskCount(), Stage.READ.getActiveTaskCount());
 
-        return queryMemtableAndDiskInternal(cfs, executionController);
+        return queryMemtableAndDiskInternal(cfs, view, rowTransformer, executionController, Clock.Global.nanoTime());
     }
 
-    private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs, ReadExecutionController controller)
+
+    private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs, ReadExecutionController controller, long startTimeNanos)
+    {
+        var view = cfs.select(View.select(SSTableSet.LIVE, partitionKey()));
+        return queryMemtableAndDiskInternal(cfs, view, null, controller, startTimeNanos);
+    }
+    private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs,
+                                                               ColumnFamilyStore.ViewFragment view,
+                                                               Function<Object, Transformation<BaseRowIterator<?>>> rowTransformer,
+                                                               ReadExecutionController controller,
+                                                               long startTimeNanos)
     {
         /*
          * We have 2 main strategies:
@@ -685,11 +717,12 @@ private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs
             && !queriesMulticellType()
             && !controller.isTrackingRepairedStatus())
         {
-            return queryMemtableAndSSTablesInTimestampOrder(cfs, (ClusteringIndexNamesFilter)clusteringIndexFilter(), controller);
+            return queryMemtableAndSSTablesInTimestampOrder(cfs, view, rowTransformer, (ClusteringIndexNamesFilter)clusteringIndexFilter(), controller, startTimeNanos);
         }
 
-        Tracing.trace("Acquiring sstable references");
-        ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, partitionKey()));
+        if (Tracing.traceSinglePartitions())
+            Tracing.trace("Acquiring sstable references");
+
         view.sstables.sort(SSTableReader.maxTimestampDescending);
         ClusteringIndexFilter filter = clusteringIndexFilter();
         long minTimestamp = Long.MAX_VALUE;
@@ -708,9 +741,11 @@ private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs
                 if (memtable.getMinTimestamp() != Memtable.NO_MIN_TIMESTAMP)
                     minTimestamp = Math.min(minTimestamp, memtable.getMinTimestamp());
 
+                var wrapped = rowTransformer != null ? Transformation.apply(iter, rowTransformer.apply(memtable)) : iter;
+
                 // Memtable data is always considered unrepaired
                 controller.updateMinOldestUnrepairedTombstone(memtable.getMinLocalDeletionTime());
-                inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false));
+                inputCollector.addMemtableIterator(RTBoundValidator.validate(wrapped, RTBoundValidator.Stage.MEMTABLE, false));
 
                 mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
                                                         iter.partitionLevelDeletion().markedForDeleteAt());
@@ -789,7 +824,11 @@ private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs
                     {
                         if (!sstable.isRepaired())
                             controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime());
-                        inputCollector.addSSTableIterator(sstable, iter);
+
+                        var wrapped = rowTransformer != null ? Transformation.apply(iter, rowTransformer.apply(sstable.getId())) : iter;
+
+                        inputCollector.addSSTableIterator(sstable, wrapped);
+
                         includedDueToTombstones++;
                         mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
                                                                 iter.partitionLevelDeletion().markedForDeleteAt());
@@ -801,7 +840,7 @@ private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs
                 }
             }
 
-            if (Tracing.isTracing())
+            if (Tracing.traceSinglePartitions())
                 Tracing.trace("Skipped {}/{} non-slice-intersecting sstables, included {} due to tombstones",
                                nonIntersectingSSTables, view.sstables.size(), includedDueToTombstones);
 
@@ -811,7 +850,7 @@ private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs
             StorageHook.instance.reportRead(cfs.metadata().id, partitionKey());
 
             List<UnfilteredRowIterator> iterators = inputCollector.finalizeIterators(cfs, nowInSec(), controller.oldestUnrepairedTombstone());
-            return withSSTablesIterated(iterators, cfs.metric, metricsCollector);
+            return withSSTablesIterated(iterators, controller, view.sstables.size(), cfs.metric, metricsCollector, startTimeNanos);
         }
         catch (RuntimeException | Error e)
         {
@@ -879,28 +918,41 @@ private UnfilteredRowIterator makeRowIteratorWithSkippedNonStaticContent(ColumnF
      * would cause all iterators to be initialized and hence all sstables to be accessed.
      */
     private UnfilteredRowIterator withSSTablesIterated(List<UnfilteredRowIterator> iterators,
+                                                       ReadExecutionController controller,
+                                                       int totalIntersectingSSTables,
                                                        TableMetrics metrics,
-                                                       SSTableReadMetricsCollector metricsCollector)
+                                                       SSTableReadMetricsCollector metricsCollector,
+                                                       long startTimeNanos)
     {
         UnfilteredRowIterator merged = UnfilteredRowIterators.merge(iterators);
 
-        if (!merged.isEmpty())
+        return withSSTablesIterated(merged, controller, totalIntersectingSSTables, metrics, metricsCollector, startTimeNanos);
+    }
+
+    private UnfilteredRowIterator withSSTablesIterated(UnfilteredRowIterator iterator,
+                                                       ReadExecutionController controller,
+                                                       int totalIntersectingSSTables,
+                                                       TableMetrics metrics,
+                                                       SSTableReadMetricsCollector metricsCollector,
+                                                       long startTimeNanos)
+    {
+        if (!iterator.isEmpty())
         {
-            DecoratedKey key = merged.partitionKey();
+            DecoratedKey key = iterator.partitionKey();
             metrics.topReadPartitionFrequency.addSample(key.getKey(), 1);
             metrics.topReadPartitionSSTableCount.addSample(key.getKey(), metricsCollector.getMergedSSTables());
         }
 
         class UpdateSstablesIterated extends Transformation<UnfilteredRowIterator>
         {
-           public void onPartitionClose()
-           {
-               int mergedSSTablesIterated = metricsCollector.getMergedSSTables();
-               metrics.updateSSTableIterated(mergedSSTablesIterated);
-               Tracing.trace("Merged data from memtables and {} sstables", mergedSSTablesIterated);
-           }
+            public void onPartitionClose()
+            {
+                int mergedSSTablesIterated = metricsCollector.getMergedSSTables();
+                metrics.updateSSTableIterated(mergedSSTablesIterated, totalIntersectingSSTables, Clock.Global.nanoTime() - startTimeNanos);
+                controller.updateSstablesIteratedPerRow(mergedSSTablesIterated);
+            }
         }
-        return Transformation.apply(merged, new UpdateSstablesIterated());
+        return Transformation.apply(iterator, new UpdateSstablesIterated());
     }
 
     private boolean queriesMulticellType()
@@ -922,15 +974,22 @@ private boolean queriesMulticellType()
      * no collection or counters are included).
      * This method assumes the filter is a {@code ClusteringIndexNamesFilter}.
      */
-    private UnfilteredRowIterator queryMemtableAndSSTablesInTimestampOrder(ColumnFamilyStore cfs, ClusteringIndexNamesFilter filter, ReadExecutionController controller)
+    private UnfilteredRowIterator queryMemtableAndSSTablesInTimestampOrder(ColumnFamilyStore cfs,
+                                                                           ColumnFamilyStore.ViewFragment view,
+                                                                           Function<Object, Transformation<BaseRowIterator<?>>> rowTransformer,
+                                                                           ClusteringIndexNamesFilter filter,
+                                                                           ReadExecutionController controller,
+                                                                           long startTimeNanos)
     {
-        Tracing.trace("Acquiring sstable references");
-        ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, partitionKey()));
+        if (Tracing.traceSinglePartitions())
+            Tracing.trace("Acquiring sstable references");
 
         ImmutableBTreePartition result = null;
         SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector();
 
-        Tracing.trace("Merging memtable contents");
+        if (Tracing.traceSinglePartitions())
+            Tracing.trace("Merging memtable contents");
+
         for (Memtable memtable : view.memtables)
         {
             try (UnfilteredRowIterator iter = memtable.rowIterator(partitionKey, filter.getSlices(metadata()), columnFilter(), isReversed(), metricsCollector))
@@ -938,7 +997,8 @@ private UnfilteredRowIterator queryMemtableAndSSTablesInTimestampOrder(ColumnFam
                 if (iter == null)
                     continue;
 
-                result = add(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false),
+                var wrapped = rowTransformer != null ? Transformation.apply(iter, rowTransformer.apply(memtable)) : iter;
+                result = add(RTBoundValidator.validate(wrapped, RTBoundValidator.Stage.MEMTABLE, false),
                              result,
                              filter,
                              false,
@@ -1009,7 +1069,8 @@ private UnfilteredRowIterator queryMemtableAndSSTablesInTimestampOrder(ColumnFam
                 if (iter.isEmpty())
                     continue;
 
-                result = add(RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false),
+                var wrapped = rowTransformer != null ? Transformation.apply(iter, rowTransformer.apply(sstable.getId())) : iter;
+                result = add(RTBoundValidator.validate(wrapped, RTBoundValidator.Stage.SSTABLE, false),
                              result,
                              filter,
                              sstable.isRepaired(),
@@ -1017,7 +1078,7 @@ private UnfilteredRowIterator queryMemtableAndSSTablesInTimestampOrder(ColumnFam
             }
         }
 
-        cfs.metric.updateSSTableIterated(metricsCollector.getMergedSSTables());
+        cfs.metric.updateSSTableIterated(metricsCollector.getMergedSSTables(), view.sstables.size(), Clock.Global.nanoTime() - startTimeNanos);
 
         if (result == null || result.isEmpty())
             return EmptyIterators.unfilteredRow(metadata(), partitionKey(), false);
@@ -1027,7 +1088,9 @@ private UnfilteredRowIterator queryMemtableAndSSTablesInTimestampOrder(ColumnFam
         cfs.metric.topReadPartitionSSTableCount.addSample(key.getKey(), metricsCollector.getMergedSSTables());
         StorageHook.instance.reportRead(cfs.metadata.id, partitionKey());
 
-        return result.unfilteredIterator(columnFilter(), Slices.ALL, clusteringIndexFilter().isReversed());
+        var iterator = result.unfilteredIterator(columnFilter(), Slices.ALL, clusteringIndexFilter().isReversed());
+        return withSSTablesIterated(iterator, controller, view.sstables.size(), cfs.metric, metricsCollector, startTimeNanos);
+
     }
 
     private ImmutableBTreePartition add(UnfilteredRowIterator iter, ImmutableBTreePartition result, ClusteringIndexNamesFilter filter, boolean isRepaired, ReadExecutionController controller)
@@ -1282,7 +1345,7 @@ public static Group create(List<SinglePartitionReadCommand> commands, DataLimits
 
         public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
         {
-            return StorageProxy.read(this, consistency, requestTime);
+            return StorageProxy.read(this, consistency, state, requestTime);
         }
     }
 
@@ -1337,6 +1400,12 @@ private static final class SSTableReadMetricsCollector implements SSTableReadsLi
          */
         private int mergedSSTables;
 
+        @Override
+        public void onSSTablePartitionIndexAccessed(SSTableReader sstable)
+        {
+            sstable.incrementIndexReadCount();
+        }
+
         @Override
         public void onSSTableSelected(SSTableReader sstable, SelectionReason reason)
         {
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java b/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java
index 5409cde8c493..dbea1d924053 100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java
@@ -177,10 +177,10 @@ public Group(List<T> queries, DataLimits limits)
         }
 
         @Override
-        public void maybeValidateIndex()
+        public void maybeValidateIndexes()
         {
             for (ReadQuery query : queries)
-                query.maybeValidateIndex();
+                query.maybeValidateIndexes();
         }
 
         public long nowInSec()
diff --git a/src/java/org/apache/cassandra/db/Slice.java b/src/java/org/apache/cassandra/db/Slice.java
index 1fc60ba1f727..c5216f13cef8 100644
--- a/src/java/org/apache/cassandra/db/Slice.java
+++ b/src/java/org/apache/cassandra/db/Slice.java
@@ -84,7 +84,7 @@ public static Slice make(ClusteringBound<?> start, ClusteringBound<?> end)
 
     public static Slice make(ClusteringComparator comparator, Object... values)
     {
-        CBuilder builder = CBuilder.create(comparator);
+        ClusteringBuilder builder = ClusteringBuilder.create(comparator);
         for (Object val : values)
         {
             if (val instanceof ByteBuffer)
diff --git a/src/java/org/apache/cassandra/db/SortedLocalRanges.java b/src/java/org/apache/cassandra/db/SortedLocalRanges.java
new file mode 100644
index 000000000000..0f97dc8f4494
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/SortedLocalRanges.java
@@ -0,0 +1,262 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Splitter;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.service.PendingRangeCalculatorService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * This class contains the local ranges for a given table, sorted.  At least one range is always present.
+ */
+public class SortedLocalRanges
+{
+    private static final Logger logger = LoggerFactory.getLogger(SortedLocalRanges.class);
+
+    private final CompactionRealm realm;
+    private final long ringVersion;
+    private final List<Splitter.WeightedRange> ranges;
+    private final Map<Integer, List<Token>> splits;
+
+    private volatile boolean valid;
+
+    public SortedLocalRanges(CompactionRealm realm, long ringVersion, List<Splitter.WeightedRange> ranges)
+    {
+        this.realm = realm;
+        this.ringVersion = ringVersion;
+
+        if (ranges == null)
+        {
+            IPartitioner partitioner = realm.getPartitioner();
+            var range = new Splitter.WeightedRange(1.0,
+                                                   new Range<>(partitioner.getMinimumToken(),
+                                                               partitioner.getMinimumToken()));
+            this.ranges = List.of(range);
+        }
+        else if (ranges.isEmpty())
+        {
+            this.ranges = ranges;
+        }
+        else
+        {
+            List<Splitter.WeightedRange> sortedRanges = new ArrayList<>(ranges.size());
+            for (Splitter.WeightedRange range : ranges)
+            {
+                for (Range<Token> unwrapped : range.range().unwrap())
+                {
+                    sortedRanges.add(new Splitter.WeightedRange(range.weight(), unwrapped));
+                }
+            }
+            assert !sortedRanges.isEmpty() : "Got empty ranges unwrapping " + ranges;
+            sortedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
+
+            this.ranges = sortedRanges;
+        }
+        this.splits = new ConcurrentHashMap<>();
+        this.valid = true;
+    }
+
+    /**
+     * Create a set of sorted local ranges based on the current token metadata and ring version.
+     *
+     * This method should preferably only be called by {@link ColumnFamilyStore} because later on,
+     * ranges may need invalidating, see {@link this#invalidate()} and so a reference must be
+     * kept to ranges that are passed around, and current cfs does this.
+     */
+    static SortedLocalRanges create(ColumnFamilyStore cfs)
+    {
+        RangesAtEndpoint localRanges;
+        List<Splitter.WeightedRange> weightedRanges;
+        long ringVersion;
+        TokenMetadata tmd;
+
+        do
+        {
+            tmd = cfs.keyspace.getReplicationStrategy().getTokenMetadata();
+            ringVersion = tmd.getRingVersion();
+            localRanges = getLocalRanges(cfs, tmd);
+
+            weightedRanges = new ArrayList<>(localRanges.size());
+            for (Range<Token> r : localRanges.onlyFull().ranges())
+                weightedRanges.add(new Splitter.WeightedRange(1.0, r));
+
+            for (Range<Token> r : localRanges.onlyTransient().ranges())
+                weightedRanges.add(new Splitter.WeightedRange(0.1, r));
+
+            if (logger.isTraceEnabled())
+                logger.trace("Got local ranges {} (ringVersion = {})", localRanges, ringVersion);
+        }
+        while (ringVersion != tmd.getRingVersion()); // if ringVersion is different here it means that
+        // it might have changed before we calculated localRanges - recalculate
+
+        return new SortedLocalRanges(cfs, ringVersion, weightedRanges);
+    }
+
+    private static RangesAtEndpoint getLocalRanges(ColumnFamilyStore cfs, TokenMetadata tmd)
+    {
+        RangesAtEndpoint localRanges;
+        if (StorageService.instance.isBootstrapMode()
+            && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
+        {
+            PendingRangeCalculatorService.instance.blockUntilFinished();
+            localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddressAndPort());
+        }
+        else
+        {
+            // Reason we use use the future settled TMD is that if we decommission a node, we want to stream
+            // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places.
+            // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled
+            localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort());
+        }
+        return localRanges;
+    }
+
+    @VisibleForTesting
+    public static SortedLocalRanges forTesting(CompactionRealm realm, List<Splitter.WeightedRange> ranges)
+    {
+        return new SortedLocalRanges(realm, 0, ranges);
+    }
+
+    public static SortedLocalRanges forTestingFull(CompactionRealm realm)
+    {
+        return forTesting(realm, null);
+    }
+
+    /**
+     * check if the given disk boundaries are out of date due not being set or to having too old diskVersion/ringVersion
+     */
+    public boolean isOutOfDate()
+    {
+        return !valid || ringVersion != realm.getKeyspaceReplicationStrategy().getTokenMetadata().getRingVersion();
+    }
+
+    public void invalidate()
+    {
+        this.valid = false;
+    }
+
+    public List<Splitter.WeightedRange> getRanges()
+    {
+        return ranges;
+    }
+
+    public long getRingVersion()
+    {
+        return ringVersion;
+    }
+
+    /**
+     * Split the local ranges into the given number of parts.
+     *
+     * @param numParts the number of parts to split into
+     *
+     * @return a list of positions into which the local ranges were split
+     */
+    public List<Token> split(int numParts)
+    {
+        return splits.computeIfAbsent(numParts, this::doSplit);
+    }
+
+    private List<Token> doSplit(int numParts)
+    {
+        Splitter splitter = realm.getPartitioner().splitter().orElse(null);
+
+        List<Token> boundaries;
+        if (splitter == null)
+        {
+            logger.debug("Could not split local ranges into {} parts for {}.{} (no splitter)", numParts, realm.getKeyspaceName(), realm.getTableName());
+            boundaries = ranges.stream().map(Splitter.WeightedRange::right).collect(Collectors.toList());
+        }
+        else
+        {
+            logger.debug("Splitting local ranges into {} parts for {}.{}", numParts, realm.getKeyspaceName(), realm.getTableName());
+            boundaries = splitter.splitOwnedRanges(numParts, ranges, Splitter.SplitType.ALWAYS_SPLIT).boundaries;
+        }
+
+        logger.debug("Boundaries for {}.{}: {} ({} splits)", realm.getKeyspaceName(), realm.getTableName(), boundaries, boundaries.size());
+        return boundaries;
+    }
+
+    /**
+     * Returns the intersection of this list with the given range.
+     */
+    public List<Splitter.WeightedRange> subrange(Range<Token> range)
+    {
+        return ranges.stream()
+                     .map(r -> {
+                         Range<Token> subRange = r.range().intersectionNonWrapping(range);
+                         return subRange == null ? null : new Splitter.WeightedRange(r.weight(), subRange);
+                     })
+                     .filter(Objects::nonNull)
+                     .collect(Collectors.toList());
+    }
+
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        SortedLocalRanges that = (SortedLocalRanges) o;
+        if (ringVersion != that.ringVersion)
+            return false;
+
+        if (!realm.equals(that.realm))
+            return false;
+
+        return ranges.equals(that.ranges);
+    }
+
+    public int hashCode()
+    {
+        int result = realm.hashCode();
+        result = 31 * result + Long.hashCode(ringVersion);
+        result = 31 * result + ranges.hashCode();
+        return result;
+    }
+
+    public String toString()
+    {
+        return "LocalRanges{" +
+               "table=" + realm.getKeyspaceName() + "." + realm.getTableName() +
+               ", ring version=" + ringVersion +
+               ", num ranges=" + ranges.size() + '}';
+    }
+
+    public CompactionRealm getRealm()
+    {
+        return realm;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index 7bfbda4705eb..75cb96d695ea 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -20,7 +20,6 @@
 import java.io.IOError;
 import java.io.IOException;
 import java.net.InetAddress;
-import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -43,15 +42,15 @@
 import javax.management.openmbean.TabularData;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
 import com.google.common.collect.SetMultimap;
 import com.google.common.collect.Sets;
 import com.google.common.io.ByteStreams;
+import org.apache.commons.lang3.ObjectUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -68,7 +67,6 @@
 import org.apache.cassandra.db.marshal.TimeUUIDType;
 import org.apache.cassandra.db.marshal.TupleType;
 import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Rows;
@@ -79,15 +77,16 @@
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.SSTableId;
 import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
-import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.RebufferingInputStream;
-import org.apache.cassandra.locator.IEndpointSnitch;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.RestorableMeter;
 import org.apache.cassandra.metrics.TopPartitionTracker;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.nodes.INodeInfo;
+import org.apache.cassandra.nodes.IPeerInfo;
+import org.apache.cassandra.nodes.Nodes;
+import org.apache.cassandra.nodes.TruncationRecord;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
@@ -99,8 +98,11 @@
 import org.apache.cassandra.schema.Types;
 import org.apache.cassandra.schema.UserFunctions;
 import org.apache.cassandra.schema.Views;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.paxos.Ballot;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.Type;
 import org.apache.cassandra.service.paxos.Commit;
 import org.apache.cassandra.service.paxos.Commit.Accepted;
 import org.apache.cassandra.service.paxos.Commit.AcceptedWithTTL;
@@ -123,6 +125,8 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PERSIST_PREPARED_STATEMENTS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UNSAFE_SYSTEM;
 import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy;
 import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
@@ -153,7 +157,7 @@ private SystemKeyspace()
     public static final String PEERS_V2 = "peers_v2";
     public static final String PEER_EVENTS_V2 = "peer_events_v2";
     public static final String COMPACTION_HISTORY = "compaction_history";
-    public static final String SSTABLE_ACTIVITY_V2 = "sstable_activity_v2"; // v2 has modified generation column type (v1 - int, v2 - blob), see CASSANDRA-17048
+    public static final String SSTABLE_ACTIVITY_V2 = "sstable_activity_v2"; // v2 has modified generation column type (v1 - int, v2 - text), see CASSANDRA-17048
     public static final String TABLE_ESTIMATES = "table_estimates";
     public static final String TABLE_ESTIMATES_TYPE_PRIMARY = "primary";
     public static final String TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY = "local_primary";
@@ -234,6 +238,7 @@ private SystemKeyspace()
                 .compaction(CompactionParams.lcs(emptyMap()))
                 .indexes(PaxosUncommittedIndex.indexes())
                 .build();
+    private static final Context PaxosContext = Context.from(Paxos);
 
     private static final TableMetadata BuiltIndexes =
         parse(BUILT_INDEXES,
@@ -256,7 +261,8 @@ private SystemKeyspace()
                 + "WITH COMMENT='Last successful paxos repairs by range'")
         .build();
 
-    private static final TableMetadata Local =
+    // Used by CNDB
+    public static final TableMetadata Local =
         parse(LOCAL,
                 "information about the local node",
                 "CREATE TABLE %s ("
@@ -284,7 +290,8 @@ private SystemKeyspace()
                 ).recordDeprecatedSystemColumn("thrift_version", UTF8Type.instance)
                 .build();
 
-    private static final TableMetadata PeersV2 =
+    // Used by CNDB
+    public static final TableMetadata PeersV2 =
         parse(PEERS_V2,
                 "information about known peers in the cluster",
                 "CREATE TABLE %s ("
@@ -323,7 +330,7 @@ private SystemKeyspace()
                 + "columnfamily_name text,"
                 + "compacted_at timestamp,"
                 + "keyspace_name text,"
-                + "rows_merged map<int, bigint>,"
+                + "rows_merged map<int, bigint>," // Note that we currently store partitions, not rows!
                 + "compaction_properties frozen<map<text, text>>,"
                 + "PRIMARY KEY ((id)))")
                 .defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(7))
@@ -469,7 +476,8 @@ private SystemKeyspace()
 
     /** @deprecated See CASSANDRA-7544 */
     @Deprecated(since = "4.0")
-    private static final TableMetadata LegacyPeers =
+    // Used by CNDB
+    public static final TableMetadata LegacyPeers =
         parse(LEGACY_PEERS,
             "information about known peers in the cluster",
             "CREATE TABLE %s ("
@@ -561,8 +569,6 @@ private static Tables tables()
                          TopPartitions);
     }
 
-    private static volatile Map<TableId, Pair<CommitLogPosition, Long>> truncationRecords;
-
     public enum BootstrapState
     {
         NEEDS_BOOTSTRAP,
@@ -579,38 +585,20 @@ public static void persistLocalMetadata()
     @VisibleForTesting
     public static void persistLocalMetadata(Supplier<UUID> nodeIdSupplier)
     {
-        String req = "INSERT INTO system.%s (" +
-                     "key," +
-                     "cluster_name," +
-                     "release_version," +
-                     "cql_version," +
-                     "native_protocol_version," +
-                     "data_center," +
-                     "rack," +
-                     "partitioner," +
-                     "rpc_address," +
-                     "rpc_port," +
-                     "broadcast_address," +
-                     "broadcast_port," +
-                     "listen_address," +
-                     "listen_port" +
-                     ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
-        IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
-        executeOnceInternal(format(req, LOCAL),
-                            LOCAL,
-                            DatabaseDescriptor.getClusterName(),
-                            FBUtilities.getReleaseVersionString(),
-                            QueryProcessor.CQL_VERSION.toString(),
-                            String.valueOf(ProtocolVersion.CURRENT.asInt()),
-                            snitch.getLocalDatacenter(),
-                            snitch.getLocalRack(),
-                            DatabaseDescriptor.getPartitioner().getClass().getName(),
-                            FBUtilities.getJustBroadcastNativeAddress(),
-                            DatabaseDescriptor.getNativeTransportPort(),
-                            FBUtilities.getJustBroadcastAddress(),
-                            DatabaseDescriptor.getStoragePort(),
-                            FBUtilities.getJustLocalAddress(),
-                            DatabaseDescriptor.getStoragePort());
+        Nodes.local().update(info -> {
+            info.setClusterName(DatabaseDescriptor.getClusterName());
+            info.setReleaseVersion(SystemKeyspace.CURRENT_VERSION);
+            info.setCqlVersion(QueryProcessor.CQL_VERSION);
+            info.setNativeProtocolVersion(ProtocolVersion.CURRENT);
+            info.setBroadcastAddressAndPort(FBUtilities.getBroadcastAddressAndPort());
+            info.setDataCenter(DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter());
+            info.setRack(DatabaseDescriptor.getEndpointSnitch().getLocalRack());
+            info.setPartitionerClass(DatabaseDescriptor.getPartitioner().getClass());
+            info.setNativeTransportAddressAndPort(InetAddressAndPort.getByAddressOverrideDefaults(DatabaseDescriptor.getRpcAddress(), DatabaseDescriptor.getNativeTransportPort()));
+            info.setBroadcastAddressAndPort(FBUtilities.getBroadcastAddressAndPort());
+            info.setListenAddressAndPort(FBUtilities.getLocalAddressAndPort());
+            return info;
+        }, true, true);
 
         // We should store host ID as soon as possible in the system.local table and flush that table to disk so that
         // we can be sure that those changes are stored in sstable and not in the commit log (see CASSANDRA-18153).
@@ -626,12 +614,13 @@ public static void updateCompactionHistory(TimeUUID taskId,
                                                long compactedAt,
                                                long bytesIn,
                                                long bytesOut,
-                                               Map<Integer, Long> rowsMerged,
+                                               Map<Integer, Long> partitionsMerged,
                                                Map<String, String> compactionProperties)
     {
         // don't write anything when the history table itself is compacted, since that would in turn cause new compactions
         if (ksname.equals("system") && cfname.equals(COMPACTION_HISTORY))
             return;
+        // For historical reasons (pre 3.0 refactor) we call the final field rows_merged but we actually store partitions!
         String req = "INSERT INTO system.%s (id, keyspace_name, columnfamily_name, compacted_at, bytes_in, bytes_out, rows_merged, compaction_properties) VALUES (?, ?, ?, ?, ?, ?, ?, ?)";
         executeInternal(format(req, COMPACTION_HISTORY),
                         taskId,
@@ -640,7 +629,7 @@ public static void updateCompactionHistory(TimeUUID taskId,
                         ByteBufferUtil.bytes(compactedAt),
                         bytesIn,
                         bytesOut,
-                        rowsMerged,
+                        partitionsMerged,
                         compactionProperties);
     }
 
@@ -742,142 +731,57 @@ public static Map<Range<Token>, Pair<Token, Long>> getViewBuildStatus(String ksn
         return status;
     }
 
-    public static synchronized void saveTruncationRecord(ColumnFamilyStore cfs, long truncatedAt, CommitLogPosition position)
+    public static void saveTruncationRecord(TableId tableId, long truncatedAt, CommitLogPosition position)
     {
-        String req = "UPDATE system.%s SET truncated_at = truncated_at + ? WHERE key = '%s'";
-        executeInternal(format(req, LOCAL, LOCAL), truncationAsMapEntry(cfs, truncatedAt, position));
-        truncationRecords = null;
-        forceBlockingFlush(LOCAL);
+        Nodes.local().update(info -> info.addTruncationRecord(tableId.asUUID(), new TruncationRecord(position, truncatedAt)), true);
     }
 
     /**
      * This method is used to remove information about truncation time for specified column family
      */
-    public static synchronized void removeTruncationRecord(TableId id)
-    {
-        Pair<CommitLogPosition, Long> truncationRecord = getTruncationRecord(id);
-        if (truncationRecord == null)
-            return;
-
-        String req = "DELETE truncated_at[?] from system.%s WHERE key = '%s'";
-        executeInternal(format(req, LOCAL, LOCAL), id.asUUID());
-        truncationRecords = null;
-        forceBlockingFlush(LOCAL);
-    }
-
-    private static Map<UUID, ByteBuffer> truncationAsMapEntry(ColumnFamilyStore cfs, long truncatedAt, CommitLogPosition position)
+    public static void removeTruncationRecord(TableId id)
     {
-        try (DataOutputBuffer out = DataOutputBuffer.scratchBuffer.get())
-        {
-            CommitLogPosition.serializer.serialize(position, out);
-            out.writeLong(truncatedAt);
-            return singletonMap(cfs.metadata.id.asUUID(), out.asNewBuffer());
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException(e);
-        }
+        Nodes.local().update(info -> info.removeTruncationRecord(id.asUUID()), true);
     }
 
     public static CommitLogPosition getTruncatedPosition(TableId id)
     {
-        Pair<CommitLogPosition, Long> record = getTruncationRecord(id);
-        return record == null ? null : record.left;
+        TruncationRecord record = Nodes.local().get().getTruncationRecords().get(id.asUUID());
+        return record != null ? record.position : null;
     }
 
     public static long getTruncatedAt(TableId id)
     {
-        Pair<CommitLogPosition, Long> record = getTruncationRecord(id);
-        return record == null ? Long.MIN_VALUE : record.right;
-    }
-
-    private static synchronized Pair<CommitLogPosition, Long> getTruncationRecord(TableId id)
-    {
-        if (truncationRecords == null)
-            truncationRecords = readTruncationRecords();
-        return truncationRecords.get(id);
-    }
-
-    private static Map<TableId, Pair<CommitLogPosition, Long>> readTruncationRecords()
-    {
-        UntypedResultSet rows = executeInternal(format("SELECT truncated_at FROM system.%s WHERE key = '%s'", LOCAL, LOCAL));
-
-        Map<TableId, Pair<CommitLogPosition, Long>> records = new HashMap<>();
-
-        if (!rows.isEmpty() && rows.one().has("truncated_at"))
-        {
-            Map<UUID, ByteBuffer> map = rows.one().getMap("truncated_at", UUIDType.instance, BytesType.instance);
-            for (Map.Entry<UUID, ByteBuffer> entry : map.entrySet())
-                records.put(TableId.fromUUID(entry.getKey()), truncationRecordFromBlob(entry.getValue()));
-        }
-
-        return records;
-    }
-
-    private static Pair<CommitLogPosition, Long> truncationRecordFromBlob(ByteBuffer bytes)
-    {
-        try (RebufferingInputStream in = new DataInputBuffer(bytes, true))
-        {
-            return Pair.create(CommitLogPosition.serializer.deserialize(in), in.available() > 0 ? in.readLong() : Long.MIN_VALUE);
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException(e);
-        }
+        TruncationRecord record = Nodes.local().get().getTruncationRecords().get(id.asUUID());
+        return record != null ? record.truncatedAt : Long.MIN_VALUE;
     }
 
     /**
      * Record tokens being used by another node
      */
-    public static synchronized void updateTokens(InetAddressAndPort ep, Collection<Token> tokens)
+    public static void updateTokens(InetAddressAndPort ep, Collection<Token> tokens)
     {
         if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
             return;
 
-        String req = "INSERT INTO system.%s (peer, tokens) VALUES (?, ?)";
-        executeInternal(String.format(req, LEGACY_PEERS), ep.getAddress(), tokensAsSet(tokens));
-        req = "INSERT INTO system.%s (peer, peer_port, tokens) VALUES (?, ?, ?)";
-        executeInternal(String.format(req, PEERS_V2), ep.getAddress(), ep.getPort(), tokensAsSet(tokens));
+        Nodes.peers().update(ep, peer -> peer.setTokens(tokens), false);
     }
 
-    public static synchronized boolean updatePreferredIP(InetAddressAndPort ep, InetAddressAndPort preferred_ip)
+    public static boolean updatePreferredIP(InetAddressAndPort ep, InetAddressAndPort preferredIP)
     {
-        if (preferred_ip.equals(getPreferredIP(ep)))
+        if (preferredIP.equals(getPreferredIP(ep)))
             return false;
 
-        String req = "INSERT INTO system.%s (peer, preferred_ip) VALUES (?, ?)";
-        executeInternal(String.format(req, LEGACY_PEERS), ep.getAddress(), preferred_ip.getAddress());
-        req = "INSERT INTO system.%s (peer, peer_port, preferred_ip, preferred_port) VALUES (?, ?, ?, ?)";
-        executeInternal(String.format(req, PEERS_V2), ep.getAddress(), ep.getPort(), preferred_ip.getAddress(), preferred_ip.getPort());
-        forceBlockingFlush(LEGACY_PEERS, PEERS_V2);
+        Nodes.peers().update(ep, info -> info.setPreferredAddressAndPort(preferredIP), true);
         return true;
     }
 
-    public static synchronized void updatePeerInfo(InetAddressAndPort ep, String columnName, Object value)
-    {
-        if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
-            return;
-
-        String req = "INSERT INTO system.%s (peer, %s) VALUES (?, ?)";
-        executeInternal(String.format(req, LEGACY_PEERS, columnName), ep.getAddress(), value);
-        //This column doesn't match across the two tables
-        if (columnName.equals("rpc_address"))
-        {
-            columnName = "native_address";
-        }
-        req = "INSERT INTO system.%s (peer, peer_port, %s) VALUES (?, ?, ?)";
-        executeInternal(String.format(req, PEERS_V2, columnName), ep.getAddress(), ep.getPort(), value);
-    }
-
-    public static synchronized void updatePeerNativeAddress(InetAddressAndPort ep, InetAddressAndPort address)
+    public static void updatePeerNativeAddress(InetAddressAndPort ep, InetAddressAndPort address)
     {
         if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
             return;
 
-        String req = "INSERT INTO system.%s (peer, rpc_address) VALUES (?, ?)";
-        executeInternal(String.format(req, LEGACY_PEERS), ep.getAddress(), address.getAddress());
-        req = "INSERT INTO system.%s (peer, peer_port, native_address, native_port) VALUES (?, ?, ?, ?)";
-        executeInternal(String.format(req, PEERS_V2), ep.getAddress(), ep.getPort(), address.getAddress(), address.getPort());
+        Nodes.peers().update(ep, info -> info.setNativeTransportAddressAndPort(address), false);
     }
 
 
@@ -890,63 +794,31 @@ public static synchronized void updateHintsDropped(InetAddressAndPort ep, TimeUU
         executeInternal(String.format(req, PEER_EVENTS_V2), timePeriod, value, ep.getAddress(), ep.getPort());
     }
 
-    public static synchronized void updateSchemaVersion(UUID version)
-    {
-        String req = "INSERT INTO system.%s (key, schema_version) VALUES ('%s', ?)";
-        executeInternal(format(req, LOCAL, LOCAL), version);
-    }
-
-    private static Set<String> tokensAsSet(Collection<Token> tokens)
+    public static void updateSchemaVersion(UUID version)
     {
-        if (tokens.isEmpty())
-            return Collections.emptySet();
-        Token.TokenFactory factory = StorageService.instance.getTokenFactory();
-        Set<String> s = new HashSet<>(tokens.size());
-        for (Token tk : tokens)
-            s.add(factory.toString(tk));
-        return s;
-    }
-
-    private static Collection<Token> deserializeTokens(Collection<String> tokensStrings)
-    {
-        Token.TokenFactory factory = StorageService.instance.getTokenFactory();
-        List<Token> tokens = new ArrayList<>(tokensStrings.size());
-        for (String tk : tokensStrings)
-            tokens.add(factory.fromString(tk));
-        return tokens;
+        Nodes.local().update(info -> info.setSchemaVersion(version), false);
     }
 
     /**
      * Remove stored tokens being used by another node
      */
-    public static synchronized void removeEndpoint(InetSocketAddress ep)
+    public static void removeEndpoint(InetAddressAndPort ep)
     {
-        String req = "DELETE FROM system.%s WHERE peer = ?";
-        executeInternal(String.format(req, LEGACY_PEERS), ep.getAddress());
-        req = String.format("DELETE FROM system.%s WHERE peer = ? AND peer_port = ?", PEERS_V2);
-        executeInternal(req, ep.getAddress(), ep.getPort());
-        forceBlockingFlush(LEGACY_PEERS, PEERS_V2);
+        Nodes.peers().remove(ep, true, false);
     }
 
     /**
      * This method is used to update the System Keyspace with the new tokens for this node
      */
-    public static synchronized void updateTokens(Collection<Token> tokens)
+    public static void updateTokens(Collection<Token> tokens)
     {
         assert !tokens.isEmpty() : "removeEndpoint should be used instead";
-
-        Collection<Token> savedTokens = getSavedTokens();
-        if (tokens.containsAll(savedTokens) && tokens.size() == savedTokens.size())
-            return;
-
-        String req = "INSERT INTO system.%s (key, tokens) VALUES ('%s', ?)";
-        executeInternal(format(req, LOCAL, LOCAL), tokensAsSet(tokens));
-        forceBlockingFlush(LOCAL);
+        Nodes.getInstance().getLocal().update(info -> info.setTokens(tokens), true);
     }
 
     public static void forceBlockingFlush(String ...cfnames)
     {
-        if (!DatabaseDescriptor.isUnsafeSystem())
+        if (!UNSAFE_SYSTEM.getBoolean())
         {
             List<Future<CommitLogPosition>> futures = new ArrayList<>();
 
@@ -967,15 +839,7 @@ public static void forceBlockingFlush(String ...cfnames)
     public static SetMultimap<InetAddressAndPort, Token> loadTokens()
     {
         SetMultimap<InetAddressAndPort, Token> tokenMap = HashMultimap.create();
-        for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, tokens FROM system." + PEERS_V2))
-        {
-            InetAddress address = row.getInetAddress("peer");
-            Integer port = row.getInt("peer_port");
-            InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
-            if (row.has("tokens"))
-                tokenMap.putAll(peer, deserializeTokens(row.getSet("tokens", UTF8Type.instance)));
-        }
-
+        Nodes.peers().get().filter(IPeerInfo::isExisting).forEach(info -> tokenMap.putAll(info.getPeerAddressAndPort(), info.getTokens()));
         return tokenMap;
     }
 
@@ -985,18 +849,7 @@ public static SetMultimap<InetAddressAndPort, Token> loadTokens()
      */
     public static Map<InetAddressAndPort, UUID> loadHostIds()
     {
-        Map<InetAddressAndPort, UUID> hostIdMap = new HashMap<>();
-        for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, host_id FROM system." + PEERS_V2))
-        {
-            InetAddress address = row.getInetAddress("peer");
-            Integer port = row.getInt("peer_port");
-            InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
-            if (row.has("host_id"))
-            {
-                hostIdMap.put(peer, row.getUUID("host_id"));
-            }
-        }
-        return hostIdMap;
+        return Nodes.peers().get().filter(IPeerInfo::isExisting).collect(Collectors.toMap(IPeerInfo::getPeerAddressAndPort, INodeInfo::getHostId));
     }
 
     /**
@@ -1007,38 +860,27 @@ public static Map<InetAddressAndPort, UUID> loadHostIds()
      */
     public static InetAddressAndPort getPreferredIP(InetAddressAndPort ep)
     {
-        Preconditions.checkState(DatabaseDescriptor.isDaemonInitialized()); // Make sure being used as a daemon, not a tool
-
-        String req = "SELECT preferred_ip, preferred_port FROM system.%s WHERE peer=? AND peer_port = ?";
-        UntypedResultSet result = executeInternal(String.format(req, PEERS_V2), ep.getAddress(), ep.getPort());
-        if (!result.isEmpty() && result.one().has("preferred_ip"))
-        {
-            UntypedResultSet.Row row = result.one();
-            return InetAddressAndPort.getByAddressOverrideDefaults(row.getInetAddress("preferred_ip"), row.getInt("preferred_port"));
-        }
-        return ep;
+        IPeerInfo info = Nodes.peers().get(ep);
+        if (info != null && info.getPreferredAddressAndPort() != null && info.isExisting())
+            return info.getPreferredAddressAndPort();
+        else
+            return ep;
     }
 
     /**
      * Return a map of IP addresses containing a map of dc and rack info
      */
-    public static Map<InetAddressAndPort, Map<String,String>> loadDcRackInfo()
+    public static Map<InetAddressAndPort, Map<String, String>> loadDcRackInfo()
     {
-        Map<InetAddressAndPort, Map<String, String>> result = new HashMap<>();
-        for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, data_center, rack from system." + PEERS_V2))
-        {
-            InetAddress address = row.getInetAddress("peer");
-            Integer port = row.getInt("peer_port");
-            InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
-            if (row.has("data_center") && row.has("rack"))
-            {
-                Map<String, String> dcRack = new HashMap<>();
-                dcRack.put("data_center", row.getString("data_center"));
-                dcRack.put("rack", row.getString("rack"));
-                result.put(peer, dcRack);
-            }
-        }
-        return result;
+        return Nodes.peers()
+                    .get()
+                    .filter(p -> p.getDataCenter() != null && p.getRack() != null && p.isExisting())
+                    .collect(Collectors.toMap(IPeerInfo::getPeerAddressAndPort, p -> {
+                        Map<String, String> dcRack = new HashMap<>();
+                        dcRack.put("data_center", p.getDataCenter());
+                        dcRack.put("rack", p.getRack());
+                        return dcRack;
+                    }));
     }
 
     /**
@@ -1050,25 +892,12 @@ public static Map<InetAddressAndPort, Map<String,String>> loadDcRackInfo()
      */
     public static CassandraVersion getReleaseVersion(InetAddressAndPort ep)
     {
-        try
-        {
-            if (FBUtilities.getBroadcastAddressAndPort().equals(ep))
-            {
-                return CURRENT_VERSION;
-            }
-            String req = "SELECT release_version FROM system.%s WHERE peer=? AND peer_port=?";
-            UntypedResultSet result = executeInternal(String.format(req, PEERS_V2), ep.getAddress(), ep.getPort());
-            if (result != null && result.one().has("release_version"))
-            {
-                return new CassandraVersion(result.one().getString("release_version"));
-            }
-            // version is unknown
-            return null;
-        }
-        catch (IllegalArgumentException e)
+        if (FBUtilities.getBroadcastAddressAndPort().equals(ep))
+            return CURRENT_VERSION;
+        else
         {
-            // version string cannot be parsed
-            return null;
+            IPeerInfo peer = Nodes.peers().get(ep);
+            return peer != null && peer.isExisting() ? peer.getReleaseVersion() : null;
         }
     }
 
@@ -1095,11 +924,8 @@ public static void checkHealth() throws ConfigurationException
         }
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(LOCAL);
 
-        String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
-        UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
-
-        if (result.isEmpty() || !result.one().has("cluster_name"))
-        {
+        String savedClusterName = Nodes.local().get().getClusterName();
+        if (savedClusterName == null) {
             // this is a brand new node
             if (!cfs.getLiveSSTables().isEmpty())
                 throw new ConfigurationException("Found system keyspace files, but they couldn't be loaded!");
@@ -1107,23 +933,18 @@ public static void checkHealth() throws ConfigurationException
             // no system files.  this is a new node.
             return;
         }
-
-        String savedClusterName = result.one().getString("cluster_name");
         if (!DatabaseDescriptor.getClusterName().equals(savedClusterName))
             throw new ConfigurationException("Saved cluster name " + savedClusterName + " != configured name " + DatabaseDescriptor.getClusterName());
     }
 
     public static Collection<Token> getSavedTokens()
     {
-        String req = "SELECT tokens FROM system.%s WHERE key='%s'";
-        UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
-        return result.isEmpty() || !result.one().has("tokens")
-             ? Collections.<Token>emptyList()
-             : deserializeTokens(result.one().getSet("tokens", UTF8Type.instance));
+        return Nodes.local().get().getTokens();
     }
 
     public static int incrementAndGetGeneration()
     {
+        // gossip generation is specific to Gossip thus it is not handled by Nodes.Local
         String req = "SELECT gossip_generation FROM system.%s WHERE key='%s'";
         UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
 
@@ -1161,13 +982,7 @@ public static int incrementAndGetGeneration()
 
     public static BootstrapState getBootstrapState()
     {
-        String req = "SELECT bootstrapped FROM system.%s WHERE key='%s'";
-        UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
-
-        if (result.isEmpty() || !result.one().has("bootstrapped"))
-            return BootstrapState.NEEDS_BOOTSTRAP;
-
-        return BootstrapState.valueOf(result.one().getString("bootstrapped"));
+        return ObjectUtils.firstNonNull(Nodes.local().get().getBootstrapState(), BootstrapState.NEEDS_BOOTSTRAP);
     }
 
     public static boolean bootstrapComplete()
@@ -1187,12 +1002,7 @@ public static boolean wasDecommissioned()
 
     public static void setBootstrapState(BootstrapState state)
     {
-        if (getBootstrapState() == state)
-            return;
-
-        String req = "INSERT INTO system.%s (key, bootstrapped) VALUES ('%s', ?)";
-        executeInternal(format(req, LOCAL, LOCAL), state.name());
-        forceBlockingFlush(LOCAL);
+        Nodes.local().update(info -> info.setBootstrapState(state), true);
     }
 
     public static boolean isIndexBuilt(String keyspaceName, String indexName)
@@ -1231,14 +1041,7 @@ public static List<String> getBuiltIndexes(String keyspaceName, Set<String> inde
      */
     public static UUID getLocalHostId()
     {
-        String req = "SELECT host_id FROM system.%s WHERE key='%s'";
-        UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
-
-        // Look up the Host UUID (return it if found)
-        if (result != null && !result.isEmpty() && result.one().has("host_id"))
-            return result.one().getUUID("host_id");
-
-        return null;
+        return Nodes.local().get().getHostId();
     }
 
     /**
@@ -1265,12 +1068,9 @@ private static synchronized UUID getOrInitializeLocalHostId(Supplier<UUID> nodeI
     /**
      * Sets the local host ID explicitly.  Should only be called outside of SystemTable when replacing a node.
      */
-    public static synchronized UUID setLocalHostId(UUID hostId)
+    public static UUID setLocalHostId(UUID hostId)
     {
-        String req = "INSERT INTO system.%s (key, host_id) VALUES ('%s', ?)";
-        executeInternal(format(req, LOCAL, LOCAL), hostId);
-        forceBlockingFlush(LOCAL);
-        return hostId;
+        return Nodes.local().update(info -> info.setHostId(hostId), false).getHostId();
     }
 
     /**
@@ -1278,13 +1078,7 @@ public static synchronized UUID setLocalHostId(UUID hostId)
      */
     public static UUID getSchemaVersion()
     {
-        String req = "SELECT schema_version FROM system.%s WHERE key='%s'";
-        UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
-
-        if (!result.isEmpty() && result.one().has("schema_version"))
-            return result.one().getUUID("schema_version");
-
-        return null;
+        return Nodes.local().get().getSchemaVersion();
     }
 
     /**
@@ -1292,14 +1086,7 @@ public static UUID getSchemaVersion()
      */
     public static String getRack()
     {
-        String req = "SELECT rack FROM system.%s WHERE key='%s'";
-        UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
-
-        // Look up the Rack (return it if found)
-        if (!result.isEmpty() && result.one().has("rack"))
-            return result.one().getString("rack");
-
-        return null;
+        return Nodes.local().get().getRack();
     }
 
     /**
@@ -1307,14 +1094,7 @@ public static String getRack()
      */
     public static String getDatacenter()
     {
-        String req = "SELECT data_center FROM system.%s WHERE key='%s'";
-        UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
-
-        // Look up the Data center (return it if found)
-        if (!result.isEmpty() && result.one().has("data_center"))
-            return result.one().getString("data_center");
-
-        return null;
+        return Nodes.local().get().getDataCenter();
     }
 
     /**
@@ -1324,8 +1104,15 @@ public static String getDatacenter()
      */
     public static PaxosState.Snapshot loadPaxosState(DecoratedKey partitionKey, TableMetadata metadata, long nowInSec)
     {
+        // Track bytes read from the Paxos system table for the commit that initiated Paxos
+        registerPaxosSensor(Type.READ_BYTES);
+
         String cql = "SELECT * FROM system." + PAXOS + " WHERE row_key = ? AND cf_id = ?";
         List<Row> results = QueryProcessor.executeInternalRawWithNow(nowInSec, cql, partitionKey.getKey(), metadata.id.asUUID()).get(partitionKey);
+
+        // transfer bytes read off of Paxos system table to the user table for the commit that initiated Paxos
+        transferPaxosSensorBytes(metadata, Type.READ_BYTES);
+
         if (results == null || results.isEmpty())
         {
             Committed noneCommitted = Committed.none(partitionKey, metadata);
@@ -1382,21 +1169,21 @@ public static void savePaxosWritePromise(DecoratedKey key, TableMetadata metadat
         if (paxosStatePurging() == legacy)
         {
             String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? AND TTL ? SET in_progress_ballot = ? WHERE row_key = ? AND cf_id = ?";
-            executeInternal(cql,
+            trackPaxosBytes(metadata, () -> executeInternal(cql,
                             ballot.unixMicros(),
                             legacyPaxosTtlSec(metadata),
                             ballot,
                             key.getKey(),
-                            metadata.id.asUUID());
+                            metadata.id.asUUID()));
         }
         else
         {
             String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? SET in_progress_ballot = ? WHERE row_key = ? AND cf_id = ?";
-            executeInternal(cql,
+            trackPaxosBytes(metadata, () -> executeInternal(cql,
                             ballot.unixMicros(),
                             ballot,
                             key.getKey(),
-                            metadata.id.asUUID());
+                            metadata.id.asUUID()));
         }
     }
 
@@ -1405,21 +1192,21 @@ public static void savePaxosReadPromise(DecoratedKey key, TableMetadata metadata
         if (paxosStatePurging() == legacy)
         {
             String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? AND TTL ? SET in_progress_read_ballot = ? WHERE row_key = ? AND cf_id = ?";
-            executeInternal(cql,
+            trackPaxosBytes(metadata, () -> executeInternal(cql,
                             ballot.unixMicros(),
                             legacyPaxosTtlSec(metadata),
                             ballot,
                             key.getKey(),
-                            metadata.id.asUUID());
+                            metadata.id.asUUID()));
         }
         else
         {
             String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? SET in_progress_read_ballot = ? WHERE row_key = ? AND cf_id = ?";
-            executeInternal(cql,
+            trackPaxosBytes(metadata, () -> executeInternal(cql,
                             ballot.unixMicros(),
                             ballot,
                             key.getKey(),
-                            metadata.id.asUUID());
+                            metadata.id.asUUID()));
         }
     }
 
@@ -1431,7 +1218,7 @@ public static void savePaxosProposal(Commit proposal)
             int ttlInSec = legacyPaxosTtlSec(proposal.update.metadata());
             long nowInSec = localDeletionTime - ttlInSec;
             String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? AND TTL ? SET proposal_ballot = ?, proposal = ?, proposal_version = ? WHERE row_key = ? AND cf_id = ?";
-            executeInternalWithNowInSec(cql,
+            trackPaxosBytes(proposal, () -> executeInternalWithNowInSec(cql,
                                         nowInSec,
                                         proposal.ballot.unixMicros(),
                                         ttlInSec,
@@ -1439,18 +1226,18 @@ public static void savePaxosProposal(Commit proposal)
                                         PartitionUpdate.toBytes(proposal.update, MessagingService.current_version),
                                         MessagingService.current_version,
                                         proposal.update.partitionKey().getKey(),
-                                        proposal.update.metadata().id.asUUID());
+                                        proposal.update.metadata().id.asUUID()));
         }
         else
         {
             String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? SET proposal_ballot = ?, proposal = ?, proposal_version = ? WHERE row_key = ? AND cf_id = ?";
-            executeInternal(cql,
+            trackPaxosBytes(proposal, () -> executeInternal(cql,
                             proposal.ballot.unixMicros(),
                             proposal.ballot,
                             PartitionUpdate.toBytes(proposal.update, MessagingService.current_version),
                             MessagingService.current_version,
                             proposal.update.partitionKey().getKey(),
-                            proposal.update.metadata().id.asUUID());
+                            proposal.update.metadata().id.asUUID()));
         }
     }
 
@@ -1464,7 +1251,7 @@ public static void savePaxosCommit(Commit commit)
             int ttlInSec = legacyPaxosTtlSec(commit.update.metadata());
             long nowInSec = localDeletionTime - ttlInSec;
             String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? AND TTL ? SET proposal_ballot = null, proposal = null, proposal_version = null, most_recent_commit_at = ?, most_recent_commit = ?, most_recent_commit_version = ? WHERE row_key = ? AND cf_id = ?";
-            executeInternalWithNowInSec(cql,
+            trackPaxosBytes(commit, () -> executeInternalWithNowInSec(cql,
                             nowInSec,
                             commit.ballot.unixMicros(),
                             ttlInSec,
@@ -1472,18 +1259,18 @@ public static void savePaxosCommit(Commit commit)
                             PartitionUpdate.toBytes(commit.update, MessagingService.current_version),
                             MessagingService.current_version,
                             commit.update.partitionKey().getKey(),
-                            commit.update.metadata().id.asUUID());
+                            commit.update.metadata().id.asUUID()));
         }
         else
         {
             String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? SET proposal_ballot = null, proposal = null, proposal_version = null, most_recent_commit_at = ?, most_recent_commit = ?, most_recent_commit_version = ? WHERE row_key = ? AND cf_id = ?";
-            executeInternal(cql,
+            trackPaxosBytes(commit, () -> executeInternal(cql,
                             commit.ballot.unixMicros(),
                             commit.ballot,
                             PartitionUpdate.toBytes(commit.update, MessagingService.current_version),
                             MessagingService.current_version,
                             commit.update.partitionKey().getKey(),
-                            commit.update.metadata().id.asUUID());
+                            commit.update.metadata().id.asUUID()));
         }
     }
 
@@ -1517,6 +1304,52 @@ public static PaxosRepairHistory loadPaxosRepairHistory(String keyspace, String
         return PaxosRepairHistory.fromTupleBufferList(points);
     }
 
+    /**
+     * Decorates a paxos comit consumer with methods to track bytes written to the Paxos system table under the context of the user table that initiated Paxos.
+     */
+    private static void trackPaxosBytes(TableMetadata metadata, Runnable paxosCommitConsumer)
+    {
+        // Track bytes written to the Paxos system table for the commit that initiated Paxos
+        registerPaxosSensor(Type.WRITE_BYTES);
+        paxosCommitConsumer.run();
+        // transfer bytes written to the Paxos system table to the user table for the commit that initiated Paxos
+        transferPaxosSensorBytes(metadata, Type.WRITE_BYTES);
+    }
+
+    /**
+     * Decorates a paxos comit consumer with methods to track bytes written to the Paxos system table under the context of the user table that initiated Paxos.
+     */
+    private static void trackPaxosBytes(Commit commit, Runnable paxosCommitConsumer)
+    {
+        // Track bytes written to the Paxos system table for the commit that initiated Paxos
+        registerPaxosSensor(Type.WRITE_BYTES);
+        paxosCommitConsumer.run();
+        // transfer bytes written to the Paxos system table to the user table for the commit that initiated Paxos
+        transferPaxosSensorBytes(commit.update.metadata(), Type.WRITE_BYTES);
+    }
+
+    private static void registerPaxosSensor(Type type)
+    {
+        RequestSensors sensors = RequestTracker.instance.get();
+        if (sensors != null)
+        {
+            sensors.registerSensor(PaxosContext, type);
+        }
+    }
+
+    /**
+     * Populates sensor values of a given {@link Type} associated with the user commit that initiated Paxos.
+     */
+    private static void transferPaxosSensorBytes(TableMetadata targetSensorMetadata, Type type)
+    {
+        RequestSensors sensors = RequestTracker.instance.get();
+        if (sensors != null)
+            sensors.getSensor(PaxosContext, type).ifPresent(paxosSensor -> {
+                sensors.incrementSensor(Context.from(targetSensorMetadata), type, paxosSensor.getValue());
+                sensors.syncAllSensors();
+            });
+    }
+
     /**
      * Returns a RestorableMeter tracking the average read rate of a particular SSTable, restoring the last-seen rate
      * from values in system.sstable_activity if present.
@@ -1529,12 +1362,12 @@ public static RestorableMeter getSSTableReadMeter(String keyspace, String table,
         UntypedResultSet results = readSSTableActivity(keyspace, table, id);
 
         if (results.isEmpty())
-            return new RestorableMeter();
+            return RestorableMeter.createWithDefaultRates();
 
         UntypedResultSet.Row row = results.one();
         double m15rate = row.getDouble("rate_15m");
         double m120rate = row.getDouble("rate_120m");
-        return new RestorableMeter(m15rate, m120rate);
+        return RestorableMeter.builder().withM15Rate(m15rate).withM120Rate(m120rate).build();
     }
 
     @VisibleForTesting
@@ -1595,7 +1428,7 @@ public static void updateSizeEstimates(String keyspace, String table, Map<Range<
     {
         long timestamp = FBUtilities.timestampMicros();
         long nowInSec = FBUtilities.nowInSeconds();
-        PartitionUpdate.Builder update = new PartitionUpdate.Builder(LegacySizeEstimates, UTF8Type.instance.decompose(keyspace), LegacySizeEstimates.regularAndStaticColumns(), estimates.size());
+        PartitionUpdate.Builder update = PartitionUpdate.builder(LegacySizeEstimates, UTF8Type.instance.decompose(keyspace), LegacySizeEstimates.regularAndStaticColumns(), estimates.size());
         // delete all previous values with a single range tombstone.
         update.add(new RangeTombstone(Slice.make(LegacySizeEstimates.comparator, table), DeletionTime.build(timestamp - 1, nowInSec)));
 
@@ -1620,7 +1453,7 @@ public static void updateTableEstimates(String keyspace, String table, String ty
     {
         long timestamp = FBUtilities.timestampMicros();
         long nowInSec = FBUtilities.nowInSeconds();
-        PartitionUpdate.Builder update = new PartitionUpdate.Builder(TableEstimates, UTF8Type.instance.decompose(keyspace), TableEstimates.regularAndStaticColumns(), estimates.size());
+        PartitionUpdate.Builder update = PartitionUpdate.builder(TableEstimates, UTF8Type.instance.decompose(keyspace), TableEstimates.regularAndStaticColumns(), estimates.size());
 
         // delete all previous values with a single range tombstone.
         update.add(new RangeTombstone(Slice.make(TableEstimates.comparator, table, type), DeletionTime.build(timestamp - 1, nowInSec)));
@@ -1793,7 +1626,7 @@ public static void snapshotOnVersionChange() throws IOException
      * Primary source of truth is the release version in system.local. If the previous
      * version cannot be determined by looking there then either:
      * * the node never had a C* install before
-     * * the was a very old version (pre 1.2) installed, which did not include system.local
+     * * the node was at very old version (pre 1.2), which did not include system.local
      *
      * @return either a version read from the system.local table or one of two special values
      * indicating either no previous version (SystemUpgrade.NULL_VERSION) or an unreadable,
@@ -1801,9 +1634,8 @@ public static void snapshotOnVersionChange() throws IOException
      */
     private static String getPreviousVersionString()
     {
-        String req = "SELECT release_version FROM system.%s WHERE key='%s'";
-        UntypedResultSet result = executeInternal(format(req, SystemKeyspace.LOCAL, SystemKeyspace.LOCAL));
-        if (result.isEmpty() || !result.one().has("release_version"))
+        CassandraVersion version = Nodes.local().get().getReleaseVersion();
+        if (version == null)
         {
             // it isn't inconceivable that one might try to upgrade a node straight from <= 1.1 to whatever
             // the current version is. If we couldn't read a previous version from system.local we check for
@@ -1822,7 +1654,7 @@ private static String getPreviousVersionString()
             return NULL_VERSION.toString();
         }
         // report back whatever we found in the system table
-        return result.one().getString("release_version");
+        return version.toString();
     }
 
     @VisibleForTesting
@@ -1869,10 +1701,15 @@ private static Range<Token> byteBufferToRange(ByteBuffer rawRange, IPartitioner
 
     public static void writePreparedStatement(String loggedKeyspace, MD5Digest key, String cql)
     {
-        executeInternal(format("INSERT INTO %s (logged_keyspace, prepared_id, query_string) VALUES (?, ?, ?)",
-                               PreparedStatements.toString()),
-                        loggedKeyspace, key.byteBuffer(), cql);
-        logger.debug("stored prepared statement for logged keyspace '{}': '{}'", loggedKeyspace, cql);
+        if (PERSIST_PREPARED_STATEMENTS.getBoolean())
+        {
+            executeInternal(format("INSERT INTO %s (logged_keyspace, prepared_id, query_string) VALUES (?, ?, ?)",
+                                   PreparedStatements.toString()),
+                            loggedKeyspace, key.byteBuffer(), cql);
+            logger.debug("stored prepared statement for logged keyspace '{}': '{}'", loggedKeyspace, cql);
+        }
+        else
+            logger.debug("not persisting prepared statement for logged keyspace '{}': '{}'", loggedKeyspace, cql);
     }
 
     public static void removePreparedStatement(MD5Digest key)
@@ -1948,7 +1785,7 @@ public static TopPartitionTracker.StoredTopPartitions getTopPartitions(TableMeta
                 return TopPartitionTracker.StoredTopPartitions.EMPTY;
 
             List<TopPartitionTracker.TopPartition> topPartitions = new ArrayList<>(top.size());
-            TupleType tupleType = new TupleType(Lists.newArrayList(UTF8Type.instance, LongType.instance));
+            TupleType tupleType = new TupleType(ImmutableList.of(UTF8Type.instance, LongType.instance));
             for (ByteBuffer bb : top)
             {
                 ByteBuffer[] components = tupleType.split(ByteBufferAccessor.instance, bb);
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java b/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java
index ab9f01f94500..1d8366edaf47 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java
@@ -30,6 +30,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.marshal.BytesType;
@@ -52,6 +53,7 @@
 public class SystemKeyspaceMigrator41
 {
     private static final Logger logger = LoggerFactory.getLogger(SystemKeyspaceMigrator41.class);
+    private static final PageSize DEFAULT_PAGE_SIZE = PageSize.inRows(1000);
 
     private SystemKeyspaceMigrator41()
     {
@@ -162,7 +164,7 @@ static void migrateSSTableActivity()
                      })
         );
     }
-    
+
     @VisibleForTesting
     static void migrateCompactionHistory()
     {
@@ -191,7 +193,7 @@ static void migrateCompactionHistory()
     /**
      * Perform table migration by reading data from the old table, converting it, and adding to the new table.
      * If oldName and newName are same, it means data in the table will be refreshed.
-     * 
+     *
      * @param truncateIfExists truncate the existing table if it exists before migration; if it is disabled
      *                         and the new table is not empty and oldName is not equal to newName, no migration is performed
      * @param oldName          old table name
@@ -217,10 +219,10 @@ static void migrateTable(boolean truncateIfExists, String oldName, String newNam
         String insert = String.format("INSERT INTO %s.%s (%s) VALUES (%s)", SchemaConstants.SYSTEM_KEYSPACE_NAME, newName,
                                       StringUtils.join(columns, ", "), StringUtils.repeat("?", ", ", columns.length));
 
-        UntypedResultSet rows = QueryProcessor.executeInternal(query);
+        UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, DEFAULT_PAGE_SIZE);
 
         assert rows != null : String.format("Migrating rows from legacy %s to %s was not done as returned rows from %s are null!", oldName, newName, oldName);
-        
+
         int transferred = 0;
         logger.info("Migrating rows from legacy {} to {}", oldName, newName);
         for (UntypedResultSet.Row row : rows)
diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
index 856b27c0a3a0..665d2beea3e5 100644
--- a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
+++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
@@ -169,4 +169,5 @@ public void skipNext() throws IOException
             UnfilteredSerializer.serializer.skipRowBody(in);
         }
     }
+
 }
diff --git a/src/java/org/apache/cassandra/db/WriteOptions.java b/src/java/org/apache/cassandra/db/WriteOptions.java
new file mode 100644
index 000000000000..f84cdd696c47
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/WriteOptions.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.Collections;
+
+import org.apache.cassandra.db.view.ViewManager;
+import org.apache.cassandra.streaming.StreamOperation;
+
+public enum WriteOptions
+{
+    /**
+     * Default write options for client initiated requests.
+     */
+    DEFAULT(null, true, true, null, true),
+
+    /**
+     * Does not persist commit log but updates indexes and is droppable. Used for tests and
+     * when commit log persistence is not required.
+     */
+    DEFAULT_WITHOUT_COMMITLOG(false, true, true, null, true),
+
+    /**
+     * Same as default, except it does not perform paired view replication since it's delayed write
+     */
+    FOR_READ_REPAIR(null, true, true, null, false),
+
+    /**
+     * Streaming with CDC always needs to write to commit log. It's also not droppable since it's not a client-initiated request.
+     *
+     * The difference from this to {@link this#FOR_STREAMING} is that we can safely skip updating views when updating
+     * sstables through the commit log, since we can ensure view sstables will be streamed from other replicas.
+     */
+    FOR_BOOTSTRAP_STREAMING(true, true, false, false, false),
+
+    /**
+     * Streaming with CDC always needs to write to commit log. It's also not droppable since it's not a client-initiated request.
+     */
+    FOR_STREAMING(true, true, false, null, false),
+
+    /**
+     * Streaming with MVs does not need to write to commit log, since it can be recovered on crash. It's also not
+     * droppable since it's not a client-initiated request.
+     */
+    FOR_STREAMING_WITH_MV(false, true, false, null, false),
+
+    /**
+     * Commit log replay obviously does not need to write to the commit log.
+     * It's also not droppable since it's not a client-initiated request.
+     */
+    FOR_COMMITLOG_REPLAY(false, true, false, null, false),
+
+    /**
+     * Paxos commit must write to commit log, independent of keyspace settings.
+     *
+     * It can also used paired view replication, since we can ensure it's the first time the
+     * mutation is written.
+     */
+    FOR_PAXOS_COMMIT(true, true, true, null, true),
+
+    /**
+     * View rebuild uses paired view replication since all nodes will build the view simultaneously
+     */
+    FOR_VIEW_BUILD(true, true, false, null, true),
+
+    /**
+     * For use on SecondaryIndexTest
+     */
+    SKIP_INDEXES_AND_COMMITLOG(false, false, false, null, false),
+
+    /**
+     * Batchlog replay uses default settings but does not perform paired view replications for view writes.
+     */
+    FOR_BATCH_REPLAY(null, true, true, null, false),
+
+    /**
+     * Batchlog replay but done by CNDB, so without using the commit log.
+     */
+    FOR_BATCH_CNDB_REPLAY(false, true, true, null, false),
+    /**
+     * Hint replay uses default settings but does not perform paired view replications for view writes.
+     */
+    FOR_HINT_REPLAY(null, true, true, null, false);
+
+
+    /**
+     * Disable index updates (used by CollationController "defragmenting")
+     */
+    public final boolean updateIndexes;
+    /**
+     * Should this update Materialized Views? Used by {@link this#FOR_BOOTSTRAP_STREAMING} to skip building
+     * views when receiving from streaming.
+     *
+     * When unset, it will only perform view updates when {@link this#updateIndexes} is true and there are views
+     * in the table being written to.
+     */
+    public final Boolean updateViews;
+    /**
+     * Throws WriteTimeoutException if write does not acquire lock within write_request_timeout_in_ms
+     */
+    public final boolean isDroppable;
+    /**
+     * Whether paired view replication should be used for view writes.
+     *
+     * This is only the case for {@link this#DEFAULT}, {@link this#DEFAULT_WITHOUT_COMMITLOG}
+     * and {@link this#FOR_VIEW_BUILD}.
+     */
+    public final boolean usePairedViewReplication;
+    /**
+     * Whether the write should be appened to the commit log. A null value means default keyspace settings are used.
+     */
+    private final Boolean writeCommitLog;
+
+    WriteOptions(Boolean writeCommitLog, boolean updateIndexes, boolean isDroppable, Boolean updateViews,
+            boolean usePairedViewReplication)
+    {
+        this.writeCommitLog = writeCommitLog;
+        this.updateIndexes = updateIndexes;
+        this.isDroppable = isDroppable;
+        this.usePairedViewReplication = usePairedViewReplication;
+        this.updateViews = updateViews;
+    }
+
+    public static WriteOptions forStreaming(StreamOperation streamOperation, boolean cdcEnabled)
+    {
+        if (cdcEnabled)
+            return streamOperation == StreamOperation.BOOTSTRAP
+                   ? FOR_BOOTSTRAP_STREAMING
+                   : FOR_STREAMING;
+
+        return FOR_STREAMING_WITH_MV;
+    }
+
+    public boolean shouldWriteCommitLog(String keyspaceName)
+    {
+        return writeCommitLog != null
+               ? writeCommitLog
+               : Keyspace.open(keyspaceName).getMetadata().params.durableWrites;
+    }
+
+    public boolean requiresViewUpdate(ViewManager viewManager, Mutation mutation)
+    {
+        return updateViews != null ? updateViews :
+               updateIndexes && viewManager.updatesAffectView(Collections.singleton(mutation), false);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
index 549955dd180a..9dc38151366e 100644
--- a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BooleanSupplier;
 
@@ -60,6 +61,7 @@
 import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.SYNCHRONIZED;
 import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.Allocation;
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 import static org.apache.cassandra.utils.concurrent.WaitQueue.newWaitQueue;
 
 /**
@@ -70,9 +72,15 @@ public abstract class AbstractCommitLogSegmentManager
 {
     static final Logger logger = LoggerFactory.getLogger(AbstractCommitLogSegmentManager.class);
 
+    /**
+     * The latest id to replay, which is also the base for the next id: kept separate for clarity.
+     */
+    private volatile long replayLimitId = 0;
+    private volatile long idBase = 0;
+
     /**
      * Segment that is ready to be used. The management thread fills this and blocks until consumed.
-     *
+     * <p>
      * A single management thread produces this, and consumers are already synchronizing to make sure other work is
      * performed atomically with consuming this. Volatile to make sure writes by the management thread become
      * visible (ordered/lazySet would suffice). Consumers (advanceAllocatingFrom and discardAvailableSegment) must
@@ -82,17 +90,19 @@ public abstract class AbstractCommitLogSegmentManager
 
     private final WaitQueue segmentPrepared = newWaitQueue();
 
-    /** Active segments, containing unflushed data. The tail of this queue is the one we allocate writes to */
+    /**
+     * Active segments, containing unflushed data. The tail of this queue is the one we allocate writes to
+     */
     private final ConcurrentLinkedQueue<CommitLogSegment> activeSegments = new ConcurrentLinkedQueue<>();
 
     /**
      * The segment we are currently allocating commit log records to.
-     *
+     * <p>
      * Written by advanceAllocatingFrom which synchronizes on 'this'. Volatile to ensure reads get current value.
      */
     private volatile CommitLogSegment allocatingFrom = null;
 
-    final String storageDirectory;
+    final File storageDirectory;
 
     /**
      * Tracks commitlog size, in multiples of the segment size.  We need to do this so we can "promise" size
@@ -112,10 +122,13 @@ public abstract class AbstractCommitLogSegmentManager
 
     private volatile SimpleCachedBufferPool bufferPool;
 
-    AbstractCommitLogSegmentManager(final CommitLog commitLog, String storageDirectory)
+    private final static AtomicInteger nextId = new AtomicInteger(1);
+
+    AbstractCommitLogSegmentManager(final CommitLog commitLog, File storageDirectory)
     {
         this.commitLog = commitLog;
         this.storageDirectory = storageDirectory;
+        init();
     }
 
     private CommitLogSegment.Builder createSegmentBuilder(CommitLog.Configuration config)
@@ -138,6 +151,10 @@ else if (config.diskAccessMode == DiskAccessMode.mmap)
         {
             return new MemoryMappedSegment.MemoryMappedSegmentBuilder(this);
         }
+        else if (config.diskAccessMode == DiskAccessMode.standard)
+        {
+            return new UncompressedSegment.UncompressedSegmentBuilder(this);
+        }
 
         throw new AssertionError("Unsupported disk access mode: " + config.diskAccessMode);
     }
@@ -147,6 +164,38 @@ CommitLog.Configuration getConfiguration()
         return commitLog.configuration;
     }
 
+    private void init()
+    {
+        AtomicLong id = new AtomicLong();
+        FileUtils.listPaths(storageDirectory.toPath()).forEach(file -> {
+            long maxId = Long.MIN_VALUE;
+            String fileName = file.getFileName().toString();
+            if (CommitLogDescriptor.isValid(fileName))
+                maxId = Math.max(CommitLogDescriptor.fromFileName(fileName).id, maxId);
+
+            id.set(maxId);
+        });
+        replayLimitId = idBase = Math.max(currentTimeMillis(), id.get() + 1);
+    }
+
+    long getNextId()
+    {
+        return idBase + nextId.getAndIncrement();
+    }
+
+    boolean shouldReplay(String name)
+    {
+        return CommitLogDescriptor.fromFileName(name).id < replayLimitId;
+    }
+
+    /**
+     * FOR TESTING PURPOSES.
+     */
+    void resetReplayLimit()
+    {
+        replayLimitId = getNextId();
+    }
+    
     void start()
     {
         assert this.segmentBuilder == null;
@@ -357,6 +406,13 @@ void awaitAvailableSegment(CommitLogSegment currentAllocatingFrom)
     void forceRecycleAll(Collection<TableId> droppedTables)
     {
         List<CommitLogSegment> segmentsToRecycle = new ArrayList<>(activeSegments);
+
+        if (segmentsToRecycle.isEmpty())
+        {
+            logger.debug("No segments to recycle");
+            return;
+        }
+        
         CommitLogSegment last = segmentsToRecycle.get(segmentsToRecycle.size() - 1);
         advanceAllocatingFrom(last);
 
@@ -419,9 +475,21 @@ void archiveAndDiscard(final CommitLogSegment segment)
      */
     void handleReplayedSegment(final File file)
     {
-        // (don't decrease managed size, since this was never a "live" segment)
-        logger.trace("(Unopened) segment {} is no longer needed and will be deleted now", file);
-        FileUtils.deleteWithConfirm(file);
+        handleReplayedSegment(file, false);
+    }
+
+    void handleReplayedSegment(final File file, boolean hasInvalidOrFailedMutations)
+    {
+        if (!hasInvalidOrFailedMutations)
+        {
+            // (don't decrease managed size, since this was never a "live" segment)
+            logger.trace("(Unopened) segment {} is no longer needed and will be deleted now", file);
+            FileUtils.deleteWithConfirm(file);
+        }
+        else
+        {
+            logger.debug("File {} should not be deleted as it contains invalid or failed mutations", file.name());
+        }
     }
 
     /**
@@ -613,7 +681,7 @@ public Collection<CommitLogSegment> getActiveSegments()
      */
     CommitLogPosition getCurrentPosition()
     {
-        return allocatingFrom.getCurrentCommitLogPosition();
+        return allocatingFrom != null ? allocatingFrom.getCurrentCommitLogPosition() : CommitLogPosition.NONE;
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogService.java b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogService.java
index 6b5378fcddda..717d0eb57738 100644
--- a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogService.java
+++ b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogService.java
@@ -43,9 +43,6 @@
 import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
 import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
 import static org.apache.cassandra.concurrent.Interruptible.State.SHUTTING_DOWN;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.apache.cassandra.utils.MonotonicClock.Global.preciseTime;
 import static org.apache.cassandra.utils.concurrent.Semaphore.newSemaphore;
 import static org.apache.cassandra.utils.concurrent.WaitQueue.newWaitQueue;
 
@@ -60,7 +57,10 @@ public abstract class AbstractCommitLogService
     private volatile Interruptible executor;
 
     // all Allocations written before this time will be synced
-    protected volatile long lastSyncedAt = currentTimeMillis();
+    protected volatile long lastSyncedAt;
+
+    // set to true when there is any error sync-ing and set to false upon a successful sync
+    private volatile boolean syncError = false;
 
     // counts of total written, and pending, log messages
     private final AtomicLong written = new AtomicLong(0);
@@ -84,6 +84,11 @@ public abstract class AbstractCommitLogService
      */
     final long markerIntervalNanos;
 
+    /**
+     * Provides time related functions for commit log syncing scheduling.
+     */
+    protected final MonotonicClock clock;
+
     /**
      * A flag that callers outside of the sync thread can use to signal they want the commitlog segments
      * to be flushed to disk. Note: this flag is primarily to support commit log's batch mode, which requires
@@ -99,9 +104,9 @@ public abstract class AbstractCommitLogService
      *
      * Subclasses may be notified when a sync finishes by using the syncComplete WaitQueue.
      */
-    AbstractCommitLogService(final CommitLog commitLog, final String name, long syncIntervalMillis)
+    AbstractCommitLogService(final CommitLog commitLog, final String name, long syncIntervalMillis, MonotonicClock clock)
     {
-        this (commitLog, name, syncIntervalMillis, false);
+        this (commitLog, name, syncIntervalMillis, clock, false);
     }
 
     /**
@@ -112,10 +117,12 @@ public abstract class AbstractCommitLogService
      *
      * @param markHeadersFaster true if the chained markers should be updated more frequently than on the disk sync bounds.
      */
-    AbstractCommitLogService(final CommitLog commitLog, final String name, long syncIntervalMillis, boolean markHeadersFaster)
+    AbstractCommitLogService(final CommitLog commitLog, final String name, long syncIntervalMillis, MonotonicClock clock, boolean markHeadersFaster)
     {
         this.commitLog = commitLog;
         this.name = name;
+        this.clock = clock;
+        this.lastSyncedAt = clock.now();
 
         final long markerIntervalMillis;
         if (syncIntervalMillis < 0)
@@ -152,7 +159,7 @@ void start()
             throw new IllegalArgumentException(String.format("Commit log flush interval must be positive: %fms",
                                                              syncIntervalNanos * 1e-6));
 
-        SyncRunnable sync = new SyncRunnable(preciseTime);
+        SyncRunnable sync = new SyncRunnable(clock);
         executor = executorFactory().infiniteLoop(name, sync, SAFE, NON_DAEMON, SYNCHRONIZED);
     }
 
@@ -176,7 +183,7 @@ public void run(Interruptible.State state) throws InterruptedException
             {
                 // sync and signal
                 long pollStarted = clock.now();
-                boolean flushToDisk = lastSyncedAt + syncIntervalNanos <= pollStarted || state != NORMAL || syncRequested;
+                boolean flushToDisk = lastSyncedAt + syncIntervalNanos - pollStarted <= 0 || state != NORMAL || syncRequested;
                 // synchronized to prevent thread interrupts while performing IO operations and also
                 // clear interrupted status to prevent ClosedByInterruptException in CommitLog::sync
                 synchronized (this)
@@ -207,17 +214,20 @@ public void run(Interruptible.State state) throws InterruptedException
                 }
                 else
                 {
+                    syncError = false;
                     long now = clock.now();
                     if (flushToDisk)
                         maybeLogFlushLag(pollStarted, now);
 
                     long wakeUpAt = pollStarted + markerIntervalNanos;
-                    if (wakeUpAt > now)
+                    if (wakeUpAt - now > 0)
                         haveWork.tryAcquireUntil(1, wakeUpAt);
                 }
             }
             catch (Throwable t)
             {
+                syncError = true;
+
                 if (!CommitLog.handleCommitError("Failed to persist commits to disk", t))
                     throw new TerminateException();
                 else // sleep for full poll-interval after an error, so we don't spam the log file
@@ -236,7 +246,7 @@ boolean maybeLogFlushLag(long pollStarted, long now)
 
             // this is the timestamp by which we should have completed the flush
             long maxFlushTimestamp = pollStarted + syncIntervalNanos;
-            if (maxFlushTimestamp > now)
+            if (maxFlushTimestamp - now > 0)
                 return false;
 
             // if we have lagged noticeably, update our lag counter
@@ -247,7 +257,7 @@ boolean maybeLogFlushLag(long pollStarted, long now)
                 syncCount = 1;
                 totalSyncDuration = flushDuration;
             }
-            syncExceededIntervalBy += now - maxFlushTimestamp;
+            syncExceededIntervalBy += Math.abs(now - maxFlushTimestamp);
             lagCount++;
 
             if (firstLagAt > 0)
@@ -259,7 +269,7 @@ boolean maybeLogFlushLag(long pollStarted, long now)
                                                   MINUTES,
                                                   "Out of {} commit log syncs over the past {}s with average duration of {}ms, {} have exceeded the configured commit interval by an average of {}ms",
                                                   syncCount,
-                                                  String.format("%.2f", (now - firstLagAt) * 1e-9d),
+                                                  String.format("%.2f", Math.abs(now - firstLagAt) * 1e-9d),
                                                   String.format("%.2f", totalSyncDuration * 1e-6d / syncCount),
                                                   lagCount,
                                                   String.format("%.2f", syncExceededIntervalBy * 1e-6d / lagCount));
@@ -310,7 +320,7 @@ public void shutdown()
      */
     public void syncBlocking()
     {
-        long requestTime = nanoTime();
+        long requestTime = clock.now();
         requestExtraSync();
         awaitSyncAt(requestTime, null);
     }
@@ -320,12 +330,12 @@ void awaitSyncAt(long syncTime, Context context)
         do
         {
             WaitQueue.Signal signal = context != null ? syncComplete.register(context, Context::stop) : syncComplete.register();
-            if (lastSyncedAt < syncTime)
+            if (lastSyncedAt - syncTime < 0)
                 signal.awaitUninterruptibly();
             else
                 signal.cancel();
         }
-        while (lastSyncedAt < syncTime);
+        while (lastSyncedAt - syncTime < 0);
     }
 
     public void awaitTermination() throws InterruptedException
@@ -342,4 +352,6 @@ public long getPendingTasks()
     {
         return pending.get();
     }
+
+    public boolean getSyncError() { return  syncError; }
 }
diff --git a/src/java/org/apache/cassandra/db/commitlog/BatchCommitLogService.java b/src/java/org/apache/cassandra/db/commitlog/BatchCommitLogService.java
index e913e678d0a5..0f818dafda6a 100644
--- a/src/java/org/apache/cassandra/db/commitlog/BatchCommitLogService.java
+++ b/src/java/org/apache/cassandra/db/commitlog/BatchCommitLogService.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.db.commitlog;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.BATCH_COMMIT_LOG_SYNC_INTERVAL;
+import org.apache.cassandra.utils.MonotonicClock;
 
 class BatchCommitLogService extends AbstractCommitLogService
 {
@@ -28,9 +29,9 @@ class BatchCommitLogService extends AbstractCommitLogService
      */
     private static final int POLL_TIME_MILLIS = BATCH_COMMIT_LOG_SYNC_INTERVAL.getInt();
 
-    public BatchCommitLogService(CommitLog commitLog)
+    public BatchCommitLogService(CommitLog commitLog, MonotonicClock clock)
     {
-        super(commitLog, "COMMIT-LOG-WRITER", POLL_TIME_MILLIS);
+        super(commitLog, "COMMIT-LOG-WRITER", POLL_TIME_MILLIS, clock);
     }
 
     protected void maybeWaitForSync(CommitLogSegment.Allocation alloc)
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
index 9b38336a04b3..2264ecf19ddf 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
@@ -27,6 +27,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
@@ -43,6 +44,8 @@
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.exceptions.CDCWriteException;
 import org.apache.cassandra.io.FSWriteError;
@@ -60,8 +63,11 @@
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MBeanWrapper;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.STARTUP;
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.Allocation;
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@@ -75,20 +81,25 @@
 public class CommitLog implements CommitLogMBean
 {
     private static final Logger logger = LoggerFactory.getLogger(CommitLog.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 10, TimeUnit.SECONDS);
 
     public static final CommitLog instance = CommitLog.construct();
 
-    private static final BiPredicate<File, String> unmanagedFilesFilter = (dir, name) -> CommitLogDescriptor.isValid(name) && CommitLogSegment.shouldReplay(name);
+    private volatile AbstractCommitLogSegmentManager segmentManager;
 
-    final public AbstractCommitLogSegmentManager segmentManager;
+    private final BiPredicate<File, String> unmanagedFilesFilter = (dir, name) -> CommitLogDescriptor.isValid(name) && segmentManager.shouldReplay(name);
 
     public final CommitLogArchiver archiver;
     public final CommitLogMetrics metrics;
     final AbstractCommitLogService executor;
+    private Set<String> segmentsWithInvalidOrFailedMutations;
 
     volatile Configuration configuration;
     private boolean started = false;
 
+    @VisibleForTesting
+    final MonotonicClock clock;
+
     private static CommitLog construct()
     {
         CommitLog log = new CommitLog(CommitLogArchiver.construct(), DatabaseDescriptor.getCommitLogSegmentMgrProvider());
@@ -113,16 +124,18 @@ private static CommitLog construct()
         this.archiver = archiver;
         metrics = new CommitLogMetrics();
 
+        this.clock = MonotonicClock.Global.preciseTime;
+
         switch (DatabaseDescriptor.getCommitLogSync())
         {
             case periodic:
-                executor = new PeriodicCommitLogService(this);
+                executor = new PeriodicCommitLogService(this, clock);
                 break;
             case batch:
-                executor = new BatchCommitLogService(this);
+                executor = new BatchCommitLogService(this, clock);
                 break;
             case group:
-                executor = new GroupCommitLogService(this);
+                executor = new GroupCommitLogService(this, clock);
                 break;
             default:
                 throw new IllegalArgumentException("Unknown commitlog service type: " + DatabaseDescriptor.getCommitLogSync());
@@ -167,19 +180,38 @@ public boolean hasFilesToReplay()
 
     public File[] getUnmanagedFiles()
     {
-        File[] files = new File(segmentManager.storageDirectory).tryList(unmanagedFilesFilter);
+        File[] files = segmentManager.storageDirectory.tryList(unmanagedFilesFilter);
         if (files == null)
             return new File[0];
         return files;
     }
 
+    /**
+     * Updates the commit log storage directory and re-initializes the segment manager accordingly.
+     * <p/>
+     * Used by CNDB.
+     *
+     * @param commitLogLocation storage directory to update to
+     * @return this commit log with updated storage directory
+     */
+    public CommitLog forPath(File commitLogLocation)
+    {
+        segmentManager = new CommitLogSegmentManagerStandard(this, commitLogLocation);
+        return this;
+    }
+
     /**
      * Perform recovery on commit logs located in the directory specified by the config file.
+     * The recovery is executed as a commit log read followed by a flush.
      *
-     * @return the number of mutations replayed
+     * @param flushReason the reason for flushing that fallows commit log reading, use
+     *                    {@link org.apache.cassandra.db.ColumnFamilyStore.FlushReason#STARTUP} when recovering on a
+     *                    node start. Use {@link org.apache.cassandra.db.ColumnFamilyStore.FlushReason#REMOTE_REPLAY}
+     *                    when replying commit logs to a remote storage.
+     * @return keyspaces and the corresponding number of partition updates
      * @throws IOException
      */
-    public int recoverSegmentsOnDisk() throws IOException
+    public Map<Keyspace, Integer> recoverSegmentsOnDisk(ColumnFamilyStore.FlushReason flushReason) throws IOException
     {
         // submit all files for this segment manager for archiving prior to recovery - CASSANDRA-6904
         // The files may have already been archived by normal CommitLog operation. This may cause errors in this
@@ -195,7 +227,7 @@ public int recoverSegmentsOnDisk() throws IOException
 
         // List the files again as archiver may have added segments.
         File[] files = getUnmanagedFiles();
-        int replayed = 0;
+        Map<Keyspace, Integer> replayedKeyspaces = Collections.emptyMap();
         if (files.length == 0)
         {
             logger.info("No commitlog files found; skipping replay");
@@ -205,36 +237,46 @@ public int recoverSegmentsOnDisk() throws IOException
             Arrays.sort(files, new CommitLogSegment.CommitLogSegmentFileComparator());
             logger.info("Replaying {}", StringUtils.join(files, ", "));
             long startTime = nanoTime();
-            replayed = recoverFiles(files);
+            replayedKeyspaces = recoverFiles(flushReason, files);
             long endTime = nanoTime();
-            logger.info("Log replay complete, {} replayed mutations in {} ms", replayed,
+            logger.info("Log replay complete, {} replayed mutations in {} ms",
+                        replayedKeyspaces.values().stream().reduce(Integer::sum).orElse(0),
                         TimeUnit.NANOSECONDS.toMillis(endTime - startTime));
 
             for (File f : files)
-                segmentManager.handleReplayedSegment(f);
+            {
+                boolean hasInvalidOrFailedMutations = segmentsWithInvalidOrFailedMutations.contains(f.name());
+                segmentManager.handleReplayedSegment(f, hasInvalidOrFailedMutations);
+            }
         }
 
-        return replayed;
+        return replayedKeyspaces;
     }
 
     /**
-     * Perform recovery on a list of commit log files.
+     * Perform recovery on a list of commit log files. The recovery is executed as a commit log read followed by a
+     * flush.
      *
+     * @param flushReason the reason for flushing that follows commit log reading
      * @param clogs   the list of commit log files to replay
-     * @return the number of mutations replayed
+     * @return keyspaces and the corresponding number of partition updates
      */
-    public int recoverFiles(File... clogs) throws IOException
+    @VisibleForTesting
+    public Map<Keyspace, Integer> recoverFiles(ColumnFamilyStore.FlushReason flushReason, File... clogs) throws IOException
     {
         CommitLogReplayer replayer = CommitLogReplayer.construct(this, getLocalHostId());
         replayer.replayFiles(clogs);
-        return replayer.blockForWrites();
+
+        Map<Keyspace, Integer> res = replayer.blockForWrites(flushReason);
+        segmentsWithInvalidOrFailedMutations = replayer.getSegmentWithInvalidOrFailedMutations();
+        return res;
     }
 
-    public void recoverPath(String path) throws IOException
+    public void recoverPath(String path, boolean tolerateTruncation) throws IOException
     {
         CommitLogReplayer replayer = CommitLogReplayer.construct(this, getLocalHostId());
-        replayer.replayPath(new File(path), false);
-        replayer.blockForWrites();
+        replayer.replayPath(new File(PathUtils.getPath(path)), tolerateTruncation);
+        replayer.blockForWrites(STARTUP);
     }
 
     private static UUID getLocalHostId()
@@ -247,7 +289,7 @@ private static UUID getLocalHostId()
      */
     public void recover(String path) throws IOException
     {
-        recoverPath(path);
+        recoverPath(path, false);
     }
 
     /**
@@ -291,11 +333,23 @@ public void requestExtraSync()
         executor.requestExtraSync();
     }
 
+    /**
+     * If there was an exception when sync-ing, and if the commit log failure policy is
+     * {@link Config.CommitFailurePolicy#fail_writes} then mutations will be rejected until
+     * the sync error is cleared, which happens after a successful sync.
+     * @return
+     */
+    @VisibleForTesting
+    public boolean shouldRejectMutations()
+    {
+        return executor.getSyncError() &&
+                DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.fail_writes;
+    }
+
     /**
      * Add a Mutation to the commit log. If CDC is enabled, this can fail.
      *
      * @param mutation the Mutation to add to the log
-     * @throws CDCWriteException
      */
     public CommitLogPosition add(Mutation mutation) throws CDCWriteException
     {
@@ -303,6 +357,13 @@ public CommitLogPosition add(Mutation mutation) throws CDCWriteException
 
         mutation.validateSize(MessagingService.current_version, ENTRY_OVERHEAD_SIZE);
 
+        if (shouldRejectMutations())
+        {
+            String errorMsg = "Rejecting mutation due to a failure sync-ing commit log segments";
+            noSpamLogger.error(errorMsg);
+            throw new FSWriteError(new IllegalStateException(errorMsg), segmentManager.allocatingFrom().getPath());
+        }
+
         try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
         {
             Mutation.serializer.serialize(mutation, dob, MessagingService.current_version);
@@ -510,10 +571,10 @@ synchronized public void shutdownBlocking() throws InterruptedException
 
     /**
      * FOR TESTING PURPOSES
-     * @return the number of files recovered
+     * @return keyspaces and the corresponding number of partition updates
      */
     @VisibleForTesting
-    synchronized public int resetUnsafe(boolean deleteSegments) throws IOException
+    synchronized public Map<Keyspace, Integer> resetUnsafe(boolean deleteSegments) throws IOException
     {
         stopUnsafe(deleteSegments);
         resetConfiguration();
@@ -551,9 +612,9 @@ synchronized public void stopUnsafe(boolean deleteSegments)
             throw new UncheckedInterruptedException(e);
         }
         segmentManager.stopUnsafe(deleteSegments);
-        CommitLogSegment.resetReplayLimit();
+        segmentManager.resetReplayLimit();
         if (DatabaseDescriptor.isCDCEnabled() && deleteSegments)
-            for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).tryList())
+            for (File f : DatabaseDescriptor.getCDCLogLocation().tryList())
                 f.delete();
     }
 
@@ -561,21 +622,21 @@ synchronized public void stopUnsafe(boolean deleteSegments)
      * FOR TESTING PURPOSES
      */
     @VisibleForTesting
-    synchronized public int restartUnsafe() throws IOException
+    synchronized public Map<Keyspace, Integer> restartUnsafe() throws IOException
     {
         started = false;
-        return start().recoverSegmentsOnDisk();
+        return start().recoverSegmentsOnDisk(ColumnFamilyStore.FlushReason.STARTUP);
     }
 
     public static long freeDiskSpace()
     {
-        return PathUtils.tryGetSpace(new File(DatabaseDescriptor.getCommitLogLocation()).toPath(), FileStore::getTotalSpace);
+        return PathUtils.tryGetSpace(DatabaseDescriptor.getCommitLogLocation().toPath(), FileStore::getTotalSpace);
     }
 
     @VisibleForTesting
     public static boolean handleCommitError(String message, Throwable t)
     {
-        JVMStabilityInspector.inspectCommitLogThrowable(t);
+        JVMStabilityInspector.inspectCommitLogThrowable(message, t);
         switch (DatabaseDescriptor.getCommitFailurePolicy())
         {
             // Needed here for unit tests to not fail on default assertion
@@ -587,6 +648,7 @@ public static boolean handleCommitError(String message, Throwable t)
                 String errorMsg = String.format("%s. Commit disk failure policy is %s; terminating thread.", message, DatabaseDescriptor.getCommitFailurePolicy());
                 logger.error(addAdditionalInformationIfPossible(errorMsg), t);
                 return false;
+            case fail_writes:
             case ignore:
                 logger.error(addAdditionalInformationIfPossible(message), t);
                 return true;
@@ -614,6 +676,11 @@ private static String addAdditionalInformationIfPossible(String msg)
         return msg;
     }
 
+    public AbstractCommitLogSegmentManager getSegmentManager()
+    {
+        return segmentManager;
+    }
+
     public static final class Configuration
     {
         /**
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java
index c0bba5b1dabf..00ba0cf38b22 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java
@@ -294,7 +294,7 @@ else if (fromHeader != null)
                     descriptor = fromHeader;
                 else descriptor = fromName;
 
-                if (descriptor.version > CommitLogDescriptor.current_version)
+                if (descriptor.version > CommitLogDescriptor.CURRENT_VERSION)
                     throw new IllegalStateException("Unsupported commit log version: " + descriptor.version);
 
                 if (descriptor.compression != null)
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
index 08bb189c908f..4d72c4374f03 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
@@ -59,17 +59,26 @@ public class CommitLogDescriptor
     static final String COMPRESSION_PARAMETERS_KEY = "compressionParameters";
     static final String COMPRESSION_CLASS_KEY = "compressionClass";
 
+    /**
+     * the versions below ARE NOT the same thing as MessagingService versions
+     * see {@link #getMessagingVersion()}
+     */
     // We don't support anything pre-3.0
     public static final int VERSION_30 = 6;
     public static final int VERSION_40 = 7;
     public static final int VERSION_50 = 8;
+    // Stargazer 1.0 messaging
+    static final int VERSION_DS_10 = MessagingService.VERSION_DS_10;
+    static final int VERSION_DS_11 = MessagingService.VERSION_DS_11;
+    static final int VERSION_DS_20 = MessagingService.VERSION_DS_20;
+    // For compatibility with CNDB
+    public static final int VERSION_DSE_68 = 680;
 
     /**
      * Increment this number if there is a changes in the commit log disc layout or MessagingVersion changes.
      * Note: make sure to handle {@link #getMessagingVersion()}
      */
-    @VisibleForTesting
-    public static final int current_version = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50;
+    public static final int CURRENT_VERSION = MessagingService.current_version;
 
     final int version;
     public final long id;
@@ -86,7 +95,7 @@ public CommitLogDescriptor(int version, long id, ParameterizedClass compression,
 
     public CommitLogDescriptor(long id, ParameterizedClass compression, EncryptionContext encryptionContext)
     {
-        this(current_version, id, compression, encryptionContext);
+        this(CURRENT_VERSION, id, compression, encryptionContext);
     }
 
     public static void writeHeader(ByteBuffer out, CommitLogDescriptor descriptor)
@@ -220,11 +229,19 @@ public int getMessagingVersion()
         switch (version)
         {
             case VERSION_30:
-                return MessagingService.VERSION_30;
+                return MessagingService.Version.VERSION_30.value;
             case VERSION_40:
-                return MessagingService.VERSION_40;
+                return MessagingService.Version.VERSION_40.value;
             case VERSION_50:
-                return MessagingService.VERSION_50;
+                return MessagingService.Version.VERSION_50.value;
+            case VERSION_DS_10:
+                return MessagingService.Version.VERSION_DS_10.value;
+            case VERSION_DS_11:
+                return MessagingService.Version.VERSION_DS_11.value;
+            case VERSION_DS_20:
+                return MessagingService.Version.VERSION_DS_20.value;
+            case VERSION_DSE_68:
+                return MessagingService.Version.VERSION_DSE_68.value;
             default:
                 throw new IllegalStateException("Unknown commitlog version " + version);
         }
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java
index 3b3a21af3c56..628719881c1f 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java
@@ -98,12 +98,6 @@ public String toString()
                ')';
     }
 
-    public CommitLogPosition clone()
-    {
-        return new CommitLogPosition(segmentId, position);
-    }
-
-
     public static class CommitLogPositionSerializer implements ISerializer<CommitLogPosition>
     {
         public void serialize(CommitLogPosition clsp, DataOutputPlus out) throws IOException
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReadHandler.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReadHandler.java
index ee052354db81..503fa56bd6fc 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReadHandler.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReadHandler.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.schema.TableId;
 
 public interface CommitLogReadHandler
 {
@@ -73,4 +74,11 @@ class CommitLogReadException extends IOException
      * @param desc CommitLogDescriptor for mutation being processed
      */
     void handleMutation(Mutation m, int size, int entryLocation, CommitLogDescriptor desc);
+
+    /**
+     * Process an invalid mutation
+     *
+     * @param id table id corresponding to the invalid mutation
+     */
+    void handleInvalidMutation(TableId id);
 }
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java
index 451ee37595d0..8c7a43d58ee4 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java
@@ -57,6 +57,7 @@ public class CommitLogReader
     public static final int ALL_MUTATIONS = -1;
     private final CRC32 checksum;
     private final Map<TableId, AtomicInteger> invalidMutations;
+    private final Set<String> segmentsWithInvalidMutations;
 
     private byte[] buffer;
 
@@ -64,9 +65,15 @@ public CommitLogReader()
     {
         checksum = new CRC32();
         invalidMutations = new HashMap<>();
+        segmentsWithInvalidMutations = new HashSet<>();
         buffer = new byte[4096];
     }
 
+    public Set<String> getSegmentsWithInvalidMutations()
+    {
+        return segmentsWithInvalidMutations;
+    }
+
     public Set<Map.Entry<TableId, AtomicInteger>> getInvalidMutations()
     {
         return invalidMutations.entrySet();
@@ -444,9 +451,12 @@ protected void readMutation(CommitLogReadHandler handler,
             {
                 i = new AtomicInteger(1);
                 invalidMutations.put(ex.id, i);
+                segmentsWithInvalidMutations.add(desc.fileName());
             }
             else
                 i.incrementAndGet();
+
+            handler.handleInvalidMutation(ex.id);
             return;
         }
         catch (Throwable t)
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 8e26425ed544..902969a23938 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@ -24,25 +24,27 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Ordering;
-
-import org.apache.cassandra.io.util.File;
+import com.google.common.util.concurrent.FutureCallback;
+import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.commons.lang3.StringUtils;
-
-import org.apache.cassandra.utils.concurrent.Future;
-import org.cliffc.high_scale_lib.NonBlockingHashSet;
+import org.apache.commons.lang3.tuple.Triple;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -52,8 +54,10 @@
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.WriteOptions;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.schema.Schema;
@@ -61,14 +65,25 @@
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.WrappedRunnable;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.Promise;
+import org.jctools.maps.NonBlockingHashMap;
+import org.jctools.maps.NonBlockingHashSet;
 
 import static java.lang.String.format;
-import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_IGNORE_REPLAY_ERRORS;
-import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_MAX_OUTSTANDING_REPLAY_BYTES;
-import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_MAX_OUTSTANDING_REPLAY_COUNT;
-import static org.apache.cassandra.config.CassandraRelevantProperties.COMMIT_LOG_REPLAY_LIST;
+import static org.apache.cassandra.config.CassandraRelevantProperties.*;
 
+/**
+ * Replays commit logs (reads commit logs and flushes new sstables).
+ *
+ * Note that instances of this class are meant to be used for a single replay only. Do not reuse the same
+ * instance for another replay as internal accumulated state (keyspacesReplayed) is not
+ * reset before the replay.
+ */
 public class CommitLogReplayer implements CommitLogReadHandler
 {
     @VisibleForTesting
@@ -76,12 +91,14 @@ public class CommitLogReplayer implements CommitLogReadHandler
     @VisibleForTesting
     public static MutationInitiator mutationInitiator = new MutationInitiator();
     private static final Logger logger = LoggerFactory.getLogger(CommitLogReplayer.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 10, TimeUnit.SECONDS);
     private static final int MAX_OUTSTANDING_REPLAY_COUNT = COMMITLOG_MAX_OUTSTANDING_REPLAY_COUNT.getInt();
 
-    private final Set<Keyspace> keyspacesReplayed;
+    private final Map<Keyspace, AtomicInteger> keyspacesReplayed;
     private final Queue<Future<Integer>> futures;
 
-    private final AtomicInteger replayedCount;
+    private final Set<String> segmentsWithFailedMutations;
+
     private final Map<TableId, IntervalSet<CommitLogPosition>> cfPersisted;
     private final CommitLogPosition globalPosition;
 
@@ -97,15 +114,16 @@ public class CommitLogReplayer implements CommitLogReadHandler
     @VisibleForTesting
     protected CommitLogReader commitLogReader;
 
+    private volatile boolean replayed = false;
+
     CommitLogReplayer(CommitLog commitLog,
                       CommitLogPosition globalPosition,
                       Map<TableId, IntervalSet<CommitLogPosition>> cfPersisted,
                       ReplayFilter replayFilter)
     {
-        this.keyspacesReplayed = new NonBlockingHashSet<>();
+        this.keyspacesReplayed = new NonBlockingHashMap<>();
         this.futures = new ArrayDeque<>();
-        // count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
-        this.replayedCount = new AtomicInteger();
+        this.segmentsWithFailedMutations = new NonBlockingHashSet<>();
         this.cfPersisted = cfPersisted;
         this.globalPosition = globalPosition;
         this.replayFilter = replayFilter;
@@ -188,14 +206,20 @@ public static CommitLogReplayer construct(CommitLog commitLog, UUID localHostId)
 
     public void replayPath(File file, boolean tolerateTruncation) throws IOException
     {
+        Preconditions.checkArgument(!replayed, "CommitlogReplayer can only replay once");
+
         sawCDCMutation = false;
         commitLogReader.readCommitLogSegment(this, file, globalPosition, CommitLogReader.ALL_MUTATIONS, tolerateTruncation);
         if (sawCDCMutation)
             handleCDCReplayCompletion(file);
+
+        replayed = true;
     }
 
     public void replayFiles(File[] clogs) throws IOException
     {
+        Preconditions.checkArgument(!replayed, "CommitlogReplayer can only replay once");
+
         List<File> filteredLogs = CommitLogReader.filterCommitLogFiles(clogs);
         int i = 0;
         for (File file: filteredLogs)
@@ -206,6 +230,8 @@ public void replayFiles(File[] clogs) throws IOException
             if (sawCDCMutation)
                 handleCDCReplayCompletion(file);
         }
+
+        replayed = true;
     }
 
 
@@ -216,7 +242,7 @@ public void replayFiles(File[] clogs) throws IOException
     private void handleCDCReplayCompletion(File f) throws IOException
     {
         // Can only reach this point if CDC is enabled, thus we have a CDCSegmentManager
-        ((CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager).addCDCSize(f.length());
+        ((CommitLogSegmentManagerCDC)CommitLog.instance.getSegmentManager()).addCDCSize(f.length());
 
         File dest = new File(DatabaseDescriptor.getCDCLogLocation(), f.name());
 
@@ -239,9 +265,10 @@ private void handleCDCReplayCompletion(File f) throws IOException
 
     /**
      * Flushes all keyspaces associated with this replayer in parallel, blocking until their flushes are complete.
-     * @return the number of mutations replayed
+     * @param flushReason the reason for flushing
+     * @return keyspaces and the corresponding number of partition updates
      */
-    public int blockForWrites()
+    public Map<Keyspace, Integer> blockForWrites(ColumnFamilyStore.FlushReason flushReason)
     {
         for (Map.Entry<TableId, AtomicInteger> entry : commitLogReader.getInvalidMutations())
             logger.warn("Skipped {} mutations from unknown (probably removed) CF with id {}", entry.getValue(), entry.getKey());
@@ -255,12 +282,28 @@ public int blockForWrites()
         boolean flushingSystem = false;
 
         List<Future<?>> futures = new ArrayList<Future<?>>();
-        for (Keyspace keyspace : keyspacesReplayed)
+        for (Keyspace keyspace : keyspacesReplayed.keySet())
         {
             if (keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
                 flushingSystem = true;
 
-            futures.addAll(keyspace.flush(ColumnFamilyStore.FlushReason.STARTUP));
+            for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+            {
+                Future<CommitLogPosition> f = cfs.forceFlush(flushReason);
+                futures.add(f);
+                f.addCallback(new FutureCallback<CommitLogPosition>()
+                {
+                    public void onSuccess(CommitLogPosition result)
+                    {
+                        mutationInitiator.onFlushed(cfs.metadata.id);
+                    }
+
+                    public void onFailure(Throwable t)
+                    {
+                        // no-op
+                    }
+                }, ImmediateExecutor.INSTANCE);
+            }
         }
 
         // also flush batchlog incase of any MV updates
@@ -271,7 +314,7 @@ public int blockForWrites()
 
         FBUtilities.waitOnFutures(futures);
 
-        return replayedCount.get();
+        return Collections.unmodifiableMap(Maps.transformValues(keyspacesReplayed, AtomicInteger::get));
     }
 
     /*
@@ -281,8 +324,15 @@ public int blockForWrites()
     @VisibleForTesting
     public static class MutationInitiator
     {
+        protected void onInvalidMutation(TableId id)
+        {
+            logger.debug("Invalid mutation detected for table id {}", id);
+        }
+
+        protected void onFailedMutation(String keyspace, Collection<TableId> tableIds) {}
+
         protected Future<Integer> initiateMutation(final Mutation mutation,
-                                                   final long segmentId,
+                                                   final CommitLogDescriptor desc,
                                                    final int serializedSize,
                                                    final int entryLocation,
                                                    final CommitLogReplayer commitLogReplayer)
@@ -304,6 +354,8 @@ public void runMayThrow()
                     // or c) are part of a cf that was dropped.
                     // Keep in mind that the cf.name() is suspect. do every thing based on the cfid instead.
                     Mutation.PartitionUpdateCollector newPUCollector = null;
+                    List<Triple<PartitionUpdate, Long, Integer>> updatesAndPositions = new ArrayList<>();
+                    int replayedCount = 0;
                     for (PartitionUpdate update : commitLogReplayer.replayFilter.filter(mutation))
                     {
                         if (Schema.instance.getTableMetadata(update.metadata().id) == null)
@@ -311,24 +363,78 @@ public void runMayThrow()
 
                         // replay if current segment is newer than last flushed one or,
                         // if it is the last known segment, if we are after the commit log segment position
-                        if (commitLogReplayer.shouldReplay(update.metadata().id, new CommitLogPosition(segmentId, entryLocation)))
+                        if (shouldReplay(update.metadata().id, commitLogReplayer, desc.id, entryLocation))
                         {
                             if (newPUCollector == null)
                                 newPUCollector = new Mutation.PartitionUpdateCollector(mutation.getKeyspaceName(), mutation.key());
                             newPUCollector.add(update);
-                            commitLogReplayer.replayedCount.incrementAndGet();
+                            replayedCount++;
+                            updatesAndPositions.add(Triple.of(update, desc.id, entryLocation));
+                        }
+                        else
+                        {
+                            onSkipped(update);
                         }
                     }
                     if (newPUCollector != null)
                     {
                         assert !newPUCollector.isEmpty();
 
-                        Keyspace.open(newPUCollector.getKeyspaceName()).apply(newPUCollector.build(), false, true, false);
-                        commitLogReplayer.keyspacesReplayed.add(keyspace);
+                        Keyspace.open(newPUCollector.getKeyspaceName()).applyFuture(newPUCollector.build(), WriteOptions.FOR_COMMITLOG_REPLAY, false)
+                                .addListener(() -> {
+                                    for (Triple<PartitionUpdate, Long, Integer> updateAndPosition : updatesAndPositions)
+                                        onReplayed(updateAndPosition.getLeft(), updateAndPosition.getMiddle(), updateAndPosition.getRight());
+                                });
+
+                        commitLogReplayer.keyspacesReplayed.computeIfAbsent(keyspace, k -> new AtomicInteger(0))
+                                .addAndGet(replayedCount);
                     }
                 }
             };
-            return Stage.MUTATION.submit(runnable, serializedSize);
+            Promise<Integer> returnFuture = new AsyncPromise<>();
+            Future<Integer> mutationFuture = Stage.MUTATION.submit(runnable, serializedSize);//.addCallback((integer, ex) -> {
+            mutationFuture.addListener(() ->
+            {
+                try
+                {
+                    Integer result = mutationFuture.get();
+                    returnFuture.trySuccess(result);
+                }
+                catch (Throwable t)
+                {
+                    noSpamLogger.warn("Failed applying mutation for keyspace {}", mutation.getKeyspaceName(), t);
+                    onFailedMutation(mutation.getKeyspaceName(), mutation.getTableIds());
+                    commitLogReplayer.segmentsWithFailedMutations.add(desc.fileName());
+                    returnFuture.tryFailure(Throwables.unchecked(t.getCause()));
+                }
+            }, ImmediateExecutor.INSTANCE);
+            return returnFuture;
+        }
+
+        /**
+         * Return true if mutation at given commitlog position should be replayed into memtable
+         */
+        protected boolean shouldReplay(TableId tableId, CommitLogReplayer commitLogReplayer, long segmentId, int entryLocation)
+        {
+            return commitLogReplayer.shouldReplay(tableId, new CommitLogPosition(segmentId, entryLocation));
+        }
+
+        /**
+         * Called when a table is flushed successfully, including table without replayed mutation
+         */
+        protected void onFlushed(TableId tableId)
+        {
+            // CNDB will override it to monitor flush status
+        }
+
+        protected void onReplayed(PartitionUpdate update, long segmentId, int entryLocation)
+        {
+            // Override for test purposes
+        }
+
+        protected void onSkipped(PartitionUpdate update)
+        {
+            // Override for test purposes
         }
     }
 
@@ -381,7 +487,7 @@ public static CommitLogPosition firstNotCovered(Collection<IntervalSet<CommitLog
                 .get(); // iteration is per known-CF, there must be at least one.
     }
 
-    abstract static class ReplayFilter
+    public abstract static class ReplayFilter
     {
         public abstract Iterable<PartitionUpdate> filter(Mutation mutation);
 
@@ -396,8 +502,18 @@ public static ReplayFilter create()
         {
             String replayList = COMMIT_LOG_REPLAY_LIST.getString();
 
+            // If no replaylist is supplied an empty array of strings is used to replay everything.
             if (replayList == null)
+            {
+                String customReplayFilter = CUSTOM_REPLAY_FILTER_CLASS.getString();
+                if (customReplayFilter != null)
+                    return FBUtilities.construct(customReplayFilter, "custom_replay_filter");
                 return new AlwaysReplayFilter();
+            }
+            else
+            {
+                logger.info("Commit log replay list set by cassandra.replayList property to: {}", replayList);
+            }
 
             Multimap<String, String> toReplay = HashMultimap.create();
             for (String rawPair : replayList.split(","))
@@ -496,7 +612,8 @@ public boolean includes(TableMetadataRef metadata)
      */
     private boolean shouldReplay(TableId tableId, CommitLogPosition position)
     {
-        return !cfPersisted.get(tableId).contains(position);
+        IntervalSet<CommitLogPosition> intervalSet = cfPersisted.get(tableId);
+        return intervalSet == null || !intervalSet.contains(position);
     }
 
     protected boolean pointInTimeExceeded(Mutation fm)
@@ -509,6 +626,18 @@ protected boolean pointInTimeExceeded(Mutation fm)
         return false;
     }
 
+    public Set<String> getSegmentWithInvalidOrFailedMutations()
+    {
+        Set<String> union = new HashSet<>(segmentsWithFailedMutations);
+        union.addAll(commitLogReader.getSegmentsWithInvalidMutations());
+        return union;
+    }
+
+    public void handleInvalidMutation(TableId id)
+    {
+        mutationInitiator.onInvalidMutation(id);
+    }
+
     public void handleMutation(Mutation m, int size, int entryLocation, CommitLogDescriptor desc)
     {
         if (DatabaseDescriptor.isCDCEnabled() && m.trackedByCDC())
@@ -516,7 +645,7 @@ public void handleMutation(Mutation m, int size, int entryLocation, CommitLogDes
 
         pendingMutationBytes += size;
         futures.offer(mutationInitiator.initiateMutation(m,
-                                                         desc.id,
+                                                         desc,
                                                          size,
                                                          entryLocation,
                                                          this));
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
index b64f05b00fe9..41422518cfd1 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
@@ -54,7 +54,6 @@
 import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.utils.concurrent.WaitQueue;
 
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
 import static org.apache.cassandra.utils.concurrent.WaitQueue.newWaitQueue;
 
@@ -65,8 +64,6 @@
  */
 public abstract class CommitLogSegment
 {
-    private final static long idBase;
-
     private CDCState cdcState = CDCState.PERMITTED;
     public enum CDCState
     {
@@ -77,16 +74,14 @@ public enum CDCState
     final Object cdcStateLock = new Object();
 
     private final static AtomicInteger nextId = new AtomicInteger(1);
-    private static long replayLimitId;
     static
     {
         long maxId = Long.MIN_VALUE;
-        for (File file : new File(DatabaseDescriptor.getCommitLogLocation()).tryList())
+        for (File file : DatabaseDescriptor.getCommitLogLocation().tryList())
         {
             if (CommitLogDescriptor.isValid(file.name()))
                 maxId = Math.max(CommitLogDescriptor.fromFileName(file.name()).id, maxId);
         }
-        replayLimitId = idBase = Math.max(currentTimeMillis(), maxId + 1);
     }
 
     // The commit log entry overhead in bytes (int: length + int: head checksum + int: tail checksum)
@@ -138,11 +133,6 @@ public enum CDCState
 
     public final CommitLogDescriptor descriptor;
 
-    static long getNextId()
-    {
-        return idBase + nextId.getAndIncrement();
-    }
-
     /**
      * Constructs a new segment file.
      */
@@ -150,7 +140,7 @@ static long getNextId()
     {
         this.manager = manager;
 
-        id = getNextId();
+        id = manager.getNextId();
         descriptor = new CommitLogDescriptor(id,
                                              manager.getConfiguration().getCompressorClass(),
                                              manager.getConfiguration().getEncryptionContext());
@@ -222,20 +212,6 @@ Allocation allocate(Mutation mutation, int size)
         }
     }
 
-    static boolean shouldReplay(String name)
-    {
-        return CommitLogDescriptor.fromFileName(name).id < replayLimitId;
-    }
-
-    /**
-     * FOR TESTING PURPOSES.
-     */
-    @VisibleForTesting
-    public static void resetReplayLimit()
-    {
-        replayLimitId = getNextId();
-    }
-
     // allocate bytes in the segment, or return -1 if not enough space
     private int allocate(int size)
     {
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
index 7dfe7add8fc6..1fbaec7d414e 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
@@ -50,10 +50,10 @@ public class CommitLogSegmentManagerCDC extends AbstractCommitLogSegmentManager
     static final Logger logger = LoggerFactory.getLogger(CommitLogSegmentManagerCDC.class);
     private final CDCSizeTracker cdcSizeTracker;
 
-    public CommitLogSegmentManagerCDC(final CommitLog commitLog, String storageDirectory)
+    public CommitLogSegmentManagerCDC(final CommitLog commitLog, File storageDirectory)
     {
         super(commitLog, storageDirectory);
-        cdcSizeTracker = new CDCSizeTracker(this, new File(DatabaseDescriptor.getCDCLogLocation()));
+        cdcSizeTracker = new CDCSizeTracker(this, DatabaseDescriptor.getCDCLogLocation());
     }
 
     @Override
@@ -93,7 +93,7 @@ public long deleteOldLinkedCDCCommitLogSegment(long bytesToFree)
         if (bytesToFree <= 0)
             return 0;
 
-        File cdcDir = new File(DatabaseDescriptor.getCDCLogLocation());
+        File cdcDir = DatabaseDescriptor.getCDCLogLocation();
         Preconditions.checkState(cdcDir.isDirectory(), "The CDC directory does not exist.");
         File[] files = cdcDir.tryList(f -> CommitLogDescriptor.isValid(f.name()));
         if (files == null || files.length == 0)
@@ -254,9 +254,9 @@ public CommitLogSegment createSegment()
      * @param file segment file that is no longer in use.
      */
     @Override
-    void handleReplayedSegment(final File file)
+    void handleReplayedSegment(final File file, boolean hasInvalidMutations)
     {
-        super.handleReplayedSegment(file);
+        super.handleReplayedSegment(file, hasInvalidMutations);
 
         // delete untracked cdc segment hard link files if their index files do not exist
         File cdcFile = new File(DatabaseDescriptor.getCDCLogLocation(), file.name());
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerStandard.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerStandard.java
index 6ca662a3db9b..ab1aad5e3c87 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerStandard.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerStandard.java
@@ -19,11 +19,12 @@
 package org.apache.cassandra.db.commitlog;
 
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 
 public class CommitLogSegmentManagerStandard extends AbstractCommitLogSegmentManager
 {
-    public CommitLogSegmentManagerStandard(final CommitLog commitLog, String storageDirectory)
+    public CommitLogSegmentManagerStandard(final CommitLog commitLog, File storageDirectory)
     {
         super(commitLog, storageDirectory);
     }
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentReader.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentReader.java
index b2dc9a51e27c..8bf0f9929119 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentReader.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentReader.java
@@ -100,9 +100,9 @@ protected SyncSegment computeNext()
         {
             while (true)
             {
+                final int currentStart = end;
                 try
                 {
-                    final int currentStart = end;
                     end = readSyncMarker(descriptor, currentStart, reader);
                     if (end == -1)
                     {
@@ -118,6 +118,7 @@ protected SyncSegment computeNext()
                 }
                 catch(CommitLogSegmentReader.SegmentReadException e)
                 {
+                    logger.debug("Error reading commit log", e);
                     try
                     {
                         handler.handleUnrecoverableError(new CommitLogReadException(
@@ -132,6 +133,7 @@ protected SyncSegment computeNext()
                 }
                 catch (IOException e)
                 {
+                    logger.debug("Error reading commit log", e);
                     try
                     {
                         boolean tolerateErrorsInSection = tolerateTruncation & segmenter.tolerateSegmentErrors(end, reader.length());
@@ -146,6 +148,13 @@ protected SyncSegment computeNext()
                         throw new RuntimeException(ioe);
                     }
                 }
+
+                // if we've not been able to read the sync marker, or the file is truncated,
+                // then return end of data, otherwise continue the loop
+                if (currentStart == end)
+                {
+                    return endOfData();
+                }
             }
         }
     }
@@ -177,21 +186,21 @@ private int readSyncMarker(CommitLogDescriptor descriptor, int offset, RandomAcc
             {
                 logger.warn("Skipping sync marker CRC check at position {} (end={}, calculated crc={}) of commit log {}." +
                             "Using per-mutation CRC checks to ensure correctness...",
-                            offset, end, crc.getValue(), reader.getPath());
+                            offset, end, crc.getValue(), reader.getFile());
                 return end;
             }
 
             if (end != 0 || filecrc != 0)
             {
                 String msg = String.format("Encountered bad header at position %d of commit log %s, with invalid CRC. " +
-                             "The end of segment marker should be zero.", offset, reader.getPath());
+                             "The end of segment marker should be zero.", offset, reader.getFile());
                 throw new SegmentReadException(msg, true);
             }
             return -1;
         }
         else if (end < offset || end > reader.length())
         {
-            String msg = String.format("Encountered bad header at position %d of commit log %s, with bad position but valid CRC", offset, reader.getPath());
+            String msg = String.format("Encountered bad header at position %d of commit log %s, with bad position but valid CRC", offset, reader.getFile());
             throw new SegmentReadException(msg, false);
         }
         return end;
@@ -315,7 +324,7 @@ public SyncSegment nextSegment(final int startPosition, final int nextSectionSta
                uncompressedBuffer = new byte[(int) (1.2 * uncompressedLength)];
             int count = compressor.uncompress(compressedBuffer, 0, compressedLength, uncompressedBuffer, 0);
             nextLogicalStart += SYNC_MARKER_SIZE;
-            FileDataInput input = new FileSegmentInputStream(ByteBuffer.wrap(uncompressedBuffer, 0, count), reader.getPath(), nextLogicalStart);
+            FileDataInput input = new FileSegmentInputStream(ByteBuffer.wrap(uncompressedBuffer, 0, count), reader.getFile(), nextLogicalStart);
             nextLogicalStart += uncompressedLength;
             return new SyncSegment(input, startPosition, nextSectionStartPosition, (int)nextLogicalStart, tolerateSegmentErrors(nextSectionStartPosition, reader.length()));
         }
@@ -361,7 +370,7 @@ public EncryptedSegmenter(CommitLogDescriptor descriptor, RandomAccessReader rea
             }
             catch (IOException ioe)
             {
-                throw new FSReadError(ioe, reader.getPath());
+                throw new FSReadError(ioe, reader.getFile());
             }
 
             chunkProvider = () -> {
@@ -375,7 +384,7 @@ public EncryptedSegmenter(CommitLogDescriptor descriptor, RandomAccessReader rea
                 }
                 catch (IOException e)
                 {
-                    throw new FSReadError(e, reader.getPath());
+                    throw new FSReadError(e, reader.getFile());
                 }
             };
         }
@@ -386,7 +395,7 @@ public SyncSegment nextSegment(int startPosition, int nextSectionStartPosition)
             currentSegmentEndPosition = nextSectionStartPosition - 1;
 
             nextLogicalStart += SYNC_MARKER_SIZE;
-            FileDataInput input = new EncryptedFileSegmentInputStream(reader.getPath(), nextLogicalStart, 0, totalPlainTextLength, chunkProvider);
+            FileDataInput input = new EncryptedFileSegmentInputStream(reader.getFile(), nextLogicalStart, 0, totalPlainTextLength, chunkProvider);
             nextLogicalStart += totalPlainTextLength;
             return new SyncSegment(input, startPosition, nextSectionStartPosition, (int)nextLogicalStart, tolerateSegmentErrors(nextSectionStartPosition, reader.length()));
         }
diff --git a/src/java/org/apache/cassandra/db/commitlog/DirectIOSegment.java b/src/java/org/apache/cassandra/db/commitlog/DirectIOSegment.java
index b4819aa20121..02e8dea8f178 100644
--- a/src/java/org/apache/cassandra/db/commitlog/DirectIOSegment.java
+++ b/src/java/org/apache/cassandra/db/commitlog/DirectIOSegment.java
@@ -30,7 +30,6 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.compress.BufferType;
-import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.SimpleCachedBufferPool;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -163,7 +162,7 @@ protected static class DirectIOSegmentBuilder extends CommitLogSegment.Builder
 
         public DirectIOSegmentBuilder(AbstractCommitLogSegmentManager segmentManager)
         {
-            this(segmentManager, FileUtils.getBlockSize(new File(segmentManager.storageDirectory)));
+            this(segmentManager, FileUtils.getBlockSize(segmentManager.storageDirectory));
         }
 
         @VisibleForTesting
diff --git a/src/java/org/apache/cassandra/db/commitlog/EncryptedFileSegmentInputStream.java b/src/java/org/apache/cassandra/db/commitlog/EncryptedFileSegmentInputStream.java
index 9da3d5041d43..171c138dce35 100644
--- a/src/java/org/apache/cassandra/db/commitlog/EncryptedFileSegmentInputStream.java
+++ b/src/java/org/apache/cassandra/db/commitlog/EncryptedFileSegmentInputStream.java
@@ -24,6 +24,7 @@
 import java.nio.ByteBuffer;
 
 import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.FileSegmentInputStream;
 
@@ -42,7 +43,7 @@ public class EncryptedFileSegmentInputStream extends FileSegmentInputStream impl
      */
     private int totalChunkOffset;
 
-    public EncryptedFileSegmentInputStream(String filePath, long segmentOffset, int position, int expectedLength, ChunkProvider chunkProvider)
+    public EncryptedFileSegmentInputStream(File filePath, long segmentOffset, int position, int expectedLength, ChunkProvider chunkProvider)
     {
         super(chunkProvider.nextChunk(), filePath, position);
         this.segmentOffset = segmentOffset;
@@ -89,8 +90,8 @@ public void seek(long position)
         if (buffer == null || bufferPos < 0 || bufferPos > buffer.capacity())
             throw new IllegalArgumentException(
                     String.format("Unable to seek to position %d in %s (%d bytes) in partial mode",
-                            position,
-                            getPath(),
+                                  position,
+                                  getFile(),
                             segmentOffset + expectedLength));
         buffer.position((int) bufferPos);
     }
diff --git a/src/java/org/apache/cassandra/db/commitlog/GroupCommitLogService.java b/src/java/org/apache/cassandra/db/commitlog/GroupCommitLogService.java
index ad4448a3ded5..7364ff6fb171 100644
--- a/src/java/org/apache/cassandra/db/commitlog/GroupCommitLogService.java
+++ b/src/java/org/apache/cassandra/db/commitlog/GroupCommitLogService.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.db.commitlog;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.MonotonicClock;
 
 /**
  * A commitlog service that will block returning an ACK back to the a coordinator/client
@@ -26,9 +27,9 @@
  */
 public class GroupCommitLogService extends AbstractCommitLogService
 {
-    public GroupCommitLogService(CommitLog commitLog)
+    public GroupCommitLogService(CommitLog commitLog, MonotonicClock clock)
     {
-        super(commitLog, "GROUP-COMMIT-LOG-WRITER", (int) DatabaseDescriptor.getCommitLogSyncGroupWindow());
+        super(commitLog, "GROUP-COMMIT-LOG-WRITER", (int) DatabaseDescriptor.getCommitLogSyncGroupWindow(), clock);
     }
 
     protected void maybeWaitForSync(CommitLogSegment.Allocation alloc)
diff --git a/src/java/org/apache/cassandra/db/commitlog/MemoryMappedSegment.java b/src/java/org/apache/cassandra/db/commitlog/MemoryMappedSegment.java
index fb671130a176..1bdd7c13f8c7 100644
--- a/src/java/org/apache/cassandra/db/commitlog/MemoryMappedSegment.java
+++ b/src/java/org/apache/cassandra/db/commitlog/MemoryMappedSegment.java
@@ -25,13 +25,19 @@
 import java.nio.file.StandardOpenOption;
 
 import net.openhft.chronicle.core.util.ThrowingFunction;
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.SimpleCachedBufferPool;
 import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.INativeLibrary;
 import org.apache.cassandra.utils.SyncUtil;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_SKIP_FILE_ADVICE;
+
 /*
  * Memory-mapped segment. Maps the destination channel into an appropriately-sized memory-mapped buffer in which the
  * mutation threads write. On sync forces the buffer to disk.
@@ -39,7 +45,11 @@
  */
 public class MemoryMappedSegment extends CommitLogSegment
 {
-    private final int fd;
+    @VisibleForTesting
+    final int fd;
+
+    @VisibleForTesting
+    static boolean skipFileAdviseToFreePageCache = COMMITLOG_SKIP_FILE_ADVICE.getBoolean();
 
     /**
      * Constructs a new segment file.
@@ -51,7 +61,7 @@ public class MemoryMappedSegment extends CommitLogSegment
         int firstSync = buffer.position();
         buffer.putInt(firstSync + 0, 0);
         buffer.putInt(firstSync + 4, 0);
-        fd = NativeLibrary.getfd(channel);
+        fd = NativeLibrary.instance.getfd(channel);
     }
 
     @Override
@@ -96,7 +106,16 @@ protected void flush(int startMarker, int nextMarker)
         {
             throw new FSWriteError(e, getPath());
         }
-        NativeLibrary.trySkipCache(fd, startMarker, nextMarker, logFile.absolutePath());
+
+        if (!skipFileAdviseToFreePageCache)
+        {
+            adviceOnFileToFreePageCache(fd, startMarker, nextMarker, logFile);
+        }
+    }
+
+    void adviceOnFileToFreePageCache(int fd, int startMarker, int nextMarker, File logFile)
+    {
+        INativeLibrary.instance.trySkipCache(fd, startMarker, nextMarker, logFile.absolutePath());
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/commitlog/PeriodicCommitLogService.java b/src/java/org/apache/cassandra/db/commitlog/PeriodicCommitLogService.java
index ae170a87d51e..5d5e14422e9b 100644
--- a/src/java/org/apache/cassandra/db/commitlog/PeriodicCommitLogService.java
+++ b/src/java/org/apache/cassandra/db/commitlog/PeriodicCommitLogService.java
@@ -20,23 +20,24 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.MonotonicClock;
 
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SYNC_LAG_FACTOR;
 
 class PeriodicCommitLogService extends AbstractCommitLogService
 {
-    private static final long blockWhenSyncLagsNanos = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getPeriodicCommitLogSyncBlock());
+    private static final long blockWhenSyncLagsNanos = (long) (TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getCommitLogSyncPeriod()) * SYNC_LAG_FACTOR.getDouble());
 
-    public PeriodicCommitLogService(final CommitLog commitLog)
+    public PeriodicCommitLogService(final CommitLog commitLog, MonotonicClock clock)
     {
-        super(commitLog, "PERIODIC-COMMIT-LOG-SYNCER", DatabaseDescriptor.getCommitLogSyncPeriod(),
+        super(commitLog, "PERIODIC-COMMIT-LOG-SYNCER", DatabaseDescriptor.getCommitLogSyncPeriod(), clock,
               !(commitLog.configuration.useCompression() || commitLog.configuration.useEncryption()));
     }
 
     protected void maybeWaitForSync(CommitLogSegment.Allocation alloc)
     {
-        long expectedSyncTime = nanoTime() - blockWhenSyncLagsNanos;
-        if (lastSyncedAt < expectedSyncTime)
+        long expectedSyncTime = clock.now() - blockWhenSyncLagsNanos;
+        if (lastSyncedAt - expectedSyncTime < 0)
         {
             pending.incrementAndGet();
             awaitSyncAt(expectedSyncTime, commitLog.metrics.waitingOnCommit.time());
diff --git a/src/java/org/apache/cassandra/db/commitlog/UncompressedSegment.java b/src/java/org/apache/cassandra/db/commitlog/UncompressedSegment.java
new file mode 100644
index 000000000000..7f820df1fae7
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/commitlog/UncompressedSegment.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.commitlog;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import net.openhft.chronicle.core.util.ThrowingFunction;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.SimpleCachedBufferPool;
+import org.apache.cassandra.utils.SyncUtil;
+
+/**
+ * Uncompressed commit log segment. Provides an in-memory buffer for the mutation threads. On sync writes anything
+ * unwritten to disk and waits for the writes to materialize.
+ *
+ * The format of the uncompressed commit log is as follows:
+ * - standard commit log header (as written by {@link CommitLogDescriptor#writeHeader(ByteBuffer, CommitLogDescriptor)})
+ * - a series of 'sync segments' that are written every time the commit log is sync()'ed
+ * -- a sync section header, see {@link CommitLogSegment#writeSyncMarker(long, ByteBuffer, int, int, int)}
+ * -- a block of uncompressed data
+ */
+public class UncompressedSegment extends FileDirectSegment
+{
+    /**
+     * Constructs a new segment file.
+     */
+    UncompressedSegment(AbstractCommitLogSegmentManager manager, ThrowingFunction<Path, FileChannel, IOException> channelFactory)
+    {
+        super(manager, channelFactory);
+    }
+
+    @Override
+    synchronized void write(int startMarker, int nextMarker)
+    {
+        int contentStart = startMarker + SYNC_MARKER_SIZE;
+        int length = nextMarker - contentStart;
+        // The length may be 0 when the segment is being closed.
+        assert length > 0 || length == 0 && !isStillAllocating();
+
+        try
+        {
+            writeSyncMarker(id, buffer, startMarker, startMarker, nextMarker);
+
+            ByteBuffer inputBuffer = buffer.duplicate();
+            inputBuffer.limit(nextMarker).position(startMarker);
+
+            // Only one thread can be here at a given time.
+            // Protected by synchronization on CommitLogSegment.sync().
+            manager.addSize(inputBuffer.remaining());
+            channel.write(inputBuffer);
+            lastWrittenPos = nextMarker;
+            assert channel.position() == nextMarker;
+            SyncUtil.force(channel, true);
+        }
+        catch (Exception e)
+        {
+            throw new FSWriteError(e, getPath());
+        }
+    }
+
+    @Override
+    public long onDiskSize()
+    {
+        return lastWrittenPos;
+    }
+
+    protected static class UncompressedSegmentBuilder extends CommitLogSegment.Builder
+    {
+        public UncompressedSegmentBuilder(AbstractCommitLogSegmentManager segmentManager)
+        {
+            super(segmentManager);
+        }
+
+        @Override
+        public UncompressedSegment build()
+        {
+            return new UncompressedSegment(segmentManager,
+                                           path ->  FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE));
+        }
+
+        @Override
+        public SimpleCachedBufferPool createBufferPool()
+        {
+            return new SimpleCachedBufferPool(DatabaseDescriptor.getCommitLogMaxCompressionBuffersInPool(),
+                                              DatabaseDescriptor.getCommitLogSegmentSize(),
+                                              BufferType.OFF_HEAP);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index fd210d6d028f..29e1f248ae3e 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -20,18 +20,18 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
@@ -43,53 +43,33 @@
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.Overlaps;
 
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-
-/**
- * Pluggable compaction strategy determines how SSTables get merged.
- *
- * There are two main goals:
- *  - perform background compaction constantly as needed; this typically makes a tradeoff between
- *    i/o done by compaction, and merging done at read time.
- *  - perform a full (maximum possible) compaction if requested by the user
- */
-public abstract class AbstractCompactionStrategy
+abstract class AbstractCompactionStrategy implements CompactionStrategy
 {
-    private static final Logger logger = LoggerFactory.getLogger(AbstractCompactionStrategy.class);
+    public static final Class<? extends CompactionStrategyContainer> CONTAINER_CLASS = CompactionStrategyManager.class;
 
-    protected static final float DEFAULT_TOMBSTONE_THRESHOLD = 0.2f;
-    // minimum interval needed to perform tombstone removal compaction in seconds, default 86400 or 1 day.
-    protected static final long DEFAULT_TOMBSTONE_COMPACTION_INTERVAL = 86400;
-    protected static final boolean DEFAULT_UNCHECKED_TOMBSTONE_COMPACTION_OPTION = false;
-    protected static final boolean DEFAULT_LOG_ALL_OPTION = false;
+    protected static final Logger logger = LoggerFactory.getLogger(AbstractCompactionStrategy.class);
 
-    protected static final String TOMBSTONE_THRESHOLD_OPTION = "tombstone_threshold";
-    protected static final String TOMBSTONE_COMPACTION_INTERVAL_OPTION = "tombstone_compaction_interval";
-    // disable range overlap check when deciding if an SSTable is candidate for tombstone compaction (CASSANDRA-6563)
-    protected static final String UNCHECKED_TOMBSTONE_COMPACTION_OPTION = "unchecked_tombstone_compaction";
-    protected static final String LOG_ALL_OPTION = "log_all";
-    protected static final String COMPACTION_ENABLED = "enabled";
-    public static final String ONLY_PURGE_REPAIRED_TOMBSTONES = "only_purge_repaired_tombstones";
+    private static int logCount = 0;
 
-    protected Map<String, String> options;
-
-    protected final ColumnFamilyStore cfs;
-    protected float tombstoneThreshold;
-    protected long tombstoneCompactionInterval;
-    protected boolean uncheckedTombstoneCompaction;
-    protected boolean disableTombstoneCompactions = false;
-    protected boolean logAll = true;
+    protected final CompactionStrategyOptions options;
+    /** The column family store should only be used when creating writers. However it is currently also used
+     * by legacy strategies and compaction tasks.
+     */
+    protected final CompactionRealm realm;
 
-    private final Directories directories;
+    protected final CompactionLogger compactionLogger;
+    protected final Directories directories;
+    /**
+     * This class groups all the compaction tasks that are pending, submitted, in progress and completed.
+     */
+    protected final BackgroundCompactions backgroundCompactions;
 
     /**
      * pause/resume/getNextBackgroundTask must synchronize.  This guarantees that after pause completes,
@@ -101,48 +81,58 @@ public abstract class AbstractCompactionStrategy
      *
      * See CASSANDRA-3430
      */
-    protected boolean isActive = false;
+    protected volatile boolean isActive = false;
 
-    protected AbstractCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
+    protected AbstractCompactionStrategy(CompactionStrategyFactory factory, BackgroundCompactions backgroundCompactions, Map<String, String> options)
     {
-        assert cfs != null;
-        this.cfs = cfs;
-        this.options = ImmutableMap.copyOf(options);
+        Preconditions.checkNotNull(factory);
+        Preconditions.checkNotNull(backgroundCompactions);
+
+        this.realm = Objects.requireNonNull(factory.getRealm());
+        this.compactionLogger = Objects.requireNonNull(factory.getCompactionLogger());
+        this.options = new CompactionStrategyOptions(getClass(), options, false);
+        this.directories = Objects.requireNonNull(realm.getDirectories());
+        this.backgroundCompactions = backgroundCompactions;
+    }
 
-        /* checks must be repeated here, as user supplied strategies might not call validateOptions directly */
+    public CompactionStrategyOptions getOptions()
+    {
+        return options;
+    }
 
-        try
-        {
-            validateOptions(options);
-            String optionValue = options.get(TOMBSTONE_THRESHOLD_OPTION);
-            tombstoneThreshold = optionValue == null ? DEFAULT_TOMBSTONE_THRESHOLD : Float.parseFloat(optionValue);
-            optionValue = options.get(TOMBSTONE_COMPACTION_INTERVAL_OPTION);
-            tombstoneCompactionInterval = optionValue == null ? DEFAULT_TOMBSTONE_COMPACTION_INTERVAL : Long.parseLong(optionValue);
-            optionValue = options.get(UNCHECKED_TOMBSTONE_COMPACTION_OPTION);
-            uncheckedTombstoneCompaction = optionValue == null ? DEFAULT_UNCHECKED_TOMBSTONE_COMPACTION_OPTION : Boolean.parseBoolean(optionValue);
-            optionValue = options.get(LOG_ALL_OPTION);
-            logAll = optionValue == null ? DEFAULT_LOG_ALL_OPTION : Boolean.parseBoolean(optionValue);
-        }
-        catch (ConfigurationException e)
-        {
-            logger.warn("Error setting compaction strategy options ({}), defaults will be used", e.getMessage());
-            tombstoneThreshold = DEFAULT_TOMBSTONE_THRESHOLD;
-            tombstoneCompactionInterval = DEFAULT_TOMBSTONE_COMPACTION_INTERVAL;
-            uncheckedTombstoneCompaction = DEFAULT_UNCHECKED_TOMBSTONE_COMPACTION_OPTION;
-        }
+    @Override
+    public CompactionLogger getCompactionLogger()
+    {
+        return compactionLogger;
+    }
+
+    public CompactionRealm getRealm() { return realm; }
 
-        directories = cfs.getDirectories();
+    //
+    // Compaction Observer
+    //
+
+    @Override
+    public void onInProgress(CompactionProgress progress)
+    {
+        backgroundCompactions.onInProgress(progress);
     }
 
-    public Directories getDirectories()
+    @Override
+    public void onCompleted(TimeUUID id, Throwable err)
     {
-        return directories;
+        backgroundCompactions.onCompleted(this, id);
     }
 
+    //
+    // CompactionStrategy
+    //
+
     /**
      * For internal, temporary suspension of background compactions so that we can do exceptional
      * things like truncate or major compaction
      */
+    @Override
     public synchronized void pause()
     {
         isActive = false;
@@ -152,6 +142,7 @@ public synchronized void pause()
      * For internal, temporary suspension of background compactions so that we can do exceptional
      * things like truncate or major compaction
      */
+    @Override
     public synchronized void resume()
     {
         isActive = true;
@@ -160,6 +151,7 @@ public synchronized void resume()
     /**
      * Performs any extra initialization required
      */
+    @Override
     public void startup()
     {
         isActive = true;
@@ -168,29 +160,43 @@ public void startup()
     /**
      * Releases any resources if this strategy is shutdown (when the CFS is reloaded after a schema change).
      */
+    @Override
     public void shutdown()
     {
         isActive = false;
     }
 
     /**
-     * @param gcBefore throw away tombstones older than this
-     *
-     * @return the next background/minor compaction task to run; null if nothing to do.
-     *
-     * Is responsible for marking its sstables as compaction-pending.
-     */
-    public abstract AbstractCompactionTask getNextBackgroundTask(final long gcBefore);
-
-    /**
-     * @param gcBefore throw away tombstones older than this
-     *
+     * @param gcBefore             throw away tombstones older than this
+     * @param permittedParallelism the maximum permitted parallelism for the operation
      * @return a compaction task that should be run to compact this columnfamilystore
      * as much as possible.  Null if nothing to do.
-     *
+     * <p>
      * Is responsible for marking its sstables as compaction-pending.
      */
-    public abstract Collection<AbstractCompactionTask> getMaximalTask(final long gcBefore, boolean splitOutput);
+    @Override
+    @SuppressWarnings("resource")
+    public synchronized CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism)
+    {
+        Iterable<? extends CompactionSSTable> filteredSSTables = Iterables.filter(getSSTables(), sstable -> !sstable.isMarkedSuspect());
+        if (Iterables.isEmpty(filteredSSTables))
+            return CompactionTasks.empty();
+        LifecycleTransaction txn = realm.tryModify(filteredSSTables, OperationType.COMPACTION);
+        if (txn == null)
+            return CompactionTasks.empty();
+        return CompactionTasks.create(Collections.singleton(createCompactionTask(gcBefore, txn, true, splitOutput)));
+    }
+
+    @Override
+    public synchronized CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism, OperationType operationType)
+    {
+        CompactionTasks maximalTasks = getMaximalTasks(gcBefore, splitOutput, permittedParallelism);
+        for (AbstractCompactionTask task: maximalTasks)
+        {
+            task.setCompactionType(operationType);
+        }
+        return maximalTasks;
+    }
 
     /**
      * @param sstables SSTables to compact. Must be marked as compacting.
@@ -201,320 +207,160 @@ public void shutdown()
      *
      * Is responsible for marking its sstables as compaction-pending.
      */
-    public abstract AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, final long gcBefore);
-
-    public AbstractCompactionTask getCompactionTask(LifecycleTransaction txn, final long gcBefore, long maxSSTableBytes)
+    @Override
+    @SuppressWarnings("resource")
+    public synchronized CompactionTasks getUserDefinedTasks(Collection<? extends CompactionSSTable> sstables, long gcBefore)
     {
-        return new CompactionTask(cfs, txn, gcBefore);
-    }
+        assert !sstables.isEmpty(); // checked for by CM.submitUserDefined
 
-    /**
-     * @return the number of background tasks estimated to still be needed for this columnfamilystore
-     */
-    public abstract int getEstimatedRemainingTasks();
+        LifecycleTransaction modifier = realm.tryModify(sstables, OperationType.COMPACTION);
+        if (modifier == null)
+        {
+            logger.trace("Unable to mark {} for compaction; probably a background compaction got to it first.  You can disable background compactions temporarily if this is a problem", sstables);
+            return CompactionTasks.empty();
+        }
 
-    /**
-     * @return the estimated number of background tasks needed, assuming an additional number of SSTables
-     */
-    int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes)
-    {
-        return getEstimatedRemainingTasks() + (int)Math.ceil((double)additionalSSTables / cfs.getMaximumCompactionThreshold());
+        return CompactionTasks.create(ImmutableList.of(createCompactionTask(gcBefore, modifier, false, false).setUserDefined(true)));
     }
 
     /**
-     * @return size in bytes of the largest sstables for this strategy
+     * Create a compaction task for a maximal, user defined or background compaction without aggregates (legacy strategies).
+     * Background compactions for strategies that extend {@link LegacyAbstractCompactionStrategy.WithAggregates} will use
+     * {@link LegacyAbstractCompactionStrategy.WithAggregates#createCompactionTask(long, LifecycleTransaction, boolean, boolean)} instead.
+     *
+     * @param gcBefore tombstone threshold, older tombstones can be discarded
+     * @param txn the transaction containing the files to be compacted
+     * @param isMaximal set to true only when it's a maximal compaction
+     * @param splitOutput false except for maximal compactions and passed in by the user to indicate to SizeTieredCompactionStrategy to split the out,
+     *                    ignored otherwise
+     *
+     * @return a compaction task, see {@link AbstractCompactionTask} and sub-classes
      */
-    public abstract long getMaxSSTableBytes();
+    protected AbstractCompactionTask createCompactionTask(final long gcBefore, LifecycleTransaction txn, boolean isMaximal, boolean splitOutput)
+    {
+        return new CompactionTask(realm, txn, gcBefore, false, this);
+    }
 
     /**
-     * Filters SSTables that are to be excluded from the given collection
+     * Create a compaction task for operations that are not driven by the strategies.
      *
-     * @param originalCandidates The collection to check for excluded SSTables
-     * @return list of the SSTables with excluded ones filtered out
+     * @param txn the transaction containing the files to be compacted
+     * @param gcBefore tombstone threshold, older tombstones can be discarded
+     * @param maxSSTableBytes the maximum size in bytes for an output sstables
+     *
+     * @return a compaction task, see {@link AbstractCompactionTask} and sub-classes
      */
-    public static List<SSTableReader> filterSuspectSSTables(Iterable<SSTableReader> originalCandidates)
+    @Override
+    public AbstractCompactionTask createCompactionTask(LifecycleTransaction txn, final long gcBefore, long maxSSTableBytes)
     {
-        List<SSTableReader> filtered = new ArrayList<>();
-        for (SSTableReader sstable : originalCandidates)
-        {
-            if (!sstable.isMarkedSuspect())
-                filtered.add(sstable);
-        }
-        return filtered;
+        return new CompactionTask(realm, txn, gcBefore, false, this);
     }
 
-
-    public ScannerList getScanners(Collection<SSTableReader> sstables, Range<Token> range)
-    {
-        return range == null ? getScanners(sstables, (Collection<Range<Token>>)null) : getScanners(sstables, Collections.singleton(range));
-    }
     /**
-     * Returns a list of KeyScanners given sstables and a range on which to scan.
-     * The default implementation simply grab one SSTableScanner per-sstable, but overriding this method
-     * allow for a more memory efficient solution if we know the sstable don't overlap (see
-     * LeveledCompactionStrategy for instance).
+     * @return a list of the compaction aggregates, e.g. the levels or buckets. Note that legacy strategies that derive from
+     * {@link LeveledCompactionStrategy.WithSSTableList} will return an empty list.
      */
-    public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+    public Collection<CompactionAggregate> getAggregates()
     {
-        ArrayList<ISSTableScanner> scanners = new ArrayList<>();
-        try
-        {
-            for (SSTableReader sstable : sstables)
-                scanners.add(sstable.getScanner(ranges));
-        }
-        catch (Throwable t)
-        {
-            ISSTableScanner.closeAllAndPropagate(scanners, t);
-        }
-        return new ScannerList(scanners);
+        return backgroundCompactions.getAggregates();
     }
 
-    public String getName()
+    /**
+     * @return the estimated number of background tasks needed, assuming an additional number of SSTables
+     */
+    int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes)
     {
-        return getClass().getSimpleName();
+        return getEstimatedRemainingTasks() + (int)Math.ceil((double)additionalSSTables / realm.getMaximumCompactionThreshold());
     }
 
-    /**
-     * Replaces sstables in the compaction strategy
-     *
-     * Note that implementations must be able to handle duplicate notifications here (that removed are already gone and
-     * added have already been added)
-     * */
-    public synchronized void replaceSSTables(Collection<SSTableReader> removed, Collection<SSTableReader> added)
+    @Override
+    public int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes, boolean isIncremental) throws IllegalArgumentException
     {
-        for (SSTableReader remove : removed)
-            removeSSTable(remove);
-        addSSTables(added);
+        return getEstimatedRemainingTasks(additionalSSTables, additionalBytes);
     }
 
     /**
-     * Adds sstable, note that implementations must handle duplicate notifications here (added already being in the compaction strategy)
+     * @return the total number of background compactions, pending or in progress
      */
-    public abstract void addSSTable(SSTableReader added);
-
-    /**
-     * Adds sstables, note that implementations must handle duplicate notifications here (added already being in the compaction strategy)
-     */
-    public synchronized void addSSTables(Iterable<SSTableReader> added)
+    @Override
+    public int getTotalCompactions()
     {
-        for (SSTableReader sstable : added)
-            addSSTable(sstable);
+        return getEstimatedRemainingTasks() + backgroundCompactions.getCompactionsInProgress().size();
     }
 
     /**
-     * Removes sstable from the strategy, implementations must be able to handle the sstable having already been removed.
-     */
-    public abstract void removeSSTable(SSTableReader sstable);
-
-    /**
-     * Removes sstables from the strategy, implementations must be able to handle the sstables having already been removed.
+     * Return the statistics. Only strategies that implement {@link LegacyAbstractCompactionStrategy.WithAggregates} will provide non-empty statistics,
+     * the legacy strategies will always have empty statistics.
+     * <p/>
+     * @return statistics about this compaction picks.
      */
-    public void removeSSTables(Iterable<SSTableReader> removed)
+    @Override
+    public List<CompactionStrategyStatistics> getStatistics()
     {
-        for (SSTableReader sstable : removed)
-            removeSSTable(sstable);
+        return ImmutableList.of(backgroundCompactions.getStatistics(this));
     }
 
-    /**
-     * Returns the sstables managed by this strategy instance
-     */
-    @VisibleForTesting
-    protected abstract Set<SSTableReader> getSSTables();
-
-    /**
-     * Called when the metadata has changed for an sstable - for example if the level changed
-     *
-     * Not called when repair status changes (which is also metadata), because this results in the
-     * sstable getting removed from the compaction strategy instance.
-     */
-    public void metadataChanged(StatsMetadata oldMetadata, SSTableReader sstable)
+    public static Iterable<CompactionSSTable> nonSuspectAndNotIn(Iterable<CompactionSSTable> sstables, Set<? extends CompactionSSTable> compacting)
     {
+        return Iterables.filter(sstables, x -> !x.isMarkedSuspect() && !compacting.contains(x));
     }
 
-    public static class ScannerList implements AutoCloseable
+    @Override
+    public int[] getSSTableCountPerLevel()
     {
-        public final List<ISSTableScanner> scanners;
-        public ScannerList(List<ISSTableScanner> scanners)
-        {
-            this.scanners = scanners;
-        }
-
-        public long getTotalBytesScanned()
-        {
-            long bytesScanned = 0L;
-            for (int i=0, isize=scanners.size(); i<isize; i++)
-                bytesScanned += scanners.get(i).getBytesScanned();
-
-            return bytesScanned;
-        }
-
-        public long getTotalCompressedSize()
-        {
-            long compressedSize = 0;
-            for (int i=0, isize=scanners.size(); i<isize; i++)
-                compressedSize += scanners.get(i).getCompressedLengthInBytes();
-
-            return compressedSize;
-        }
-
-        public double getCompressionRatio()
-        {
-            double compressed = 0.0;
-            double uncompressed = 0.0;
-
-            for (int i=0, isize=scanners.size(); i<isize; i++)
-            {
-                ISSTableScanner scanner = scanners.get(i);
-                compressed += scanner.getCompressedLengthInBytes();
-                uncompressed += scanner.getLengthInBytes();
-            }
+        return new int[0];
+    }
 
-            if (compressed == uncompressed || uncompressed == 0)
-                return MetadataCollector.NO_COMPRESSION_RATIO;
+    @Override
+    public long[] getPerLevelSizeBytes()
+    {
+        return new long[0];
+    }
 
-            return compressed / uncompressed;
-        }
+    @Override
+    public boolean isLeveledCompaction()
+    {
+        return false;
+    }
 
-        public void close()
-        {
-            ISSTableScanner.closeAllAndPropagate(scanners, null);
-        }
+    @Override
+    public int[] getSSTableCountPerTWCSBucket()
+    {
+        return new int[0];
     }
 
-    public ScannerList getScanners(Collection<SSTableReader> toCompact)
+    @Override
+    public int getLevelFanoutSize()
     {
-        return getScanners(toCompact, (Collection<Range<Token>>)null);
+        return LeveledCompactionStrategy.DEFAULT_LEVEL_FANOUT_SIZE; // this makes no sense but it's the existing behaviour
     }
 
     /**
-     * Check if given sstable is worth dropping tombstones at gcBefore.
-     * Check is skipped if tombstone_compaction_interval time does not elapse since sstable creation and returns false.
-     *
-     * @param sstable SSTable to check
-     * @param gcBefore time to drop tombstones
-     * @return true if given sstable's tombstones are expected to be removed
+     * Returns a list of KeyScanners given sstables and a range on which to scan.
+     * The default implementation simply grab one SSTableScanner per-sstable, but overriding this method
+     * allow for a more memory efficient solution if we know the sstable don't overlap (see
+     * LeveledCompactionStrategy for instance).
      */
-    protected boolean worthDroppingTombstones(SSTableReader sstable, long gcBefore)
-    {
-        if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || cfs.getNeverPurgeTombstones())
-            return false;
-        // since we use estimations to calculate, there is a chance that compaction will not drop tombstones actually.
-        // if that happens we will end up in infinite compaction loop, so first we check enough if enough time has
-        // elapsed since SSTable created.
-        if (currentTimeMillis() < sstable.getDataCreationTime() + tombstoneCompactionInterval * 1000)
-           return false;
-
-        double droppableRatio = sstable.getEstimatedDroppableTombstoneRatio(gcBefore);
-        if (droppableRatio <= tombstoneThreshold)
-            return false;
-
-        //sstable range overlap check is disabled. See CASSANDRA-6563.
-        if (uncheckedTombstoneCompaction)
-            return true;
-
-        Collection<SSTableReader> overlaps = cfs.getOverlappingLiveSSTables(Collections.singleton(sstable));
-        if (overlaps.isEmpty())
-        {
-            // there is no overlap, tombstones are safely droppable
-            return true;
-        }
-        else if (CompactionController.getFullyExpiredSSTables(cfs, Collections.singleton(sstable), overlaps, gcBefore).size() > 0)
-        {
-            return true;
-        }
-        else
-        {
-            // what percentage of columns do we expect to compact outside of overlap?
-            if (!sstable.isEstimationInformative())
-            {
-                // we have too few samples to estimate correct percentage
-                return false;
-            }
-            // first, calculate estimated keys that do not overlap
-            long keys = sstable.estimatedKeys();
-            Set<Range<Token>> ranges = new HashSet<>(overlaps.size());
-            for (SSTableReader overlap : overlaps)
-                ranges.add(new Range<>(overlap.getFirst().getToken(), overlap.getLast().getToken()));
-            long remainingKeys = keys - sstable.estimatedKeysForRanges(ranges);
-            // next, calculate what percentage of columns we have within those keys
-            long columns = sstable.getEstimatedCellPerPartitionCount().mean() * remainingKeys;
-            double remainingColumnsRatio = ((double) columns) / (sstable.getEstimatedCellPerPartitionCount().count() * sstable.getEstimatedCellPerPartitionCount().mean());
-
-            // return if we still expect to have droppable tombstones in rest of columns
-            return remainingColumnsRatio * droppableRatio > tombstoneThreshold;
-        }
+    @Override
+    public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+    {
+        return ScannerList.of(sstables, ranges);
     }
 
-    public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
+    @Override
+    public String getName()
     {
-        String threshold = options.get(TOMBSTONE_THRESHOLD_OPTION);
-        if (threshold != null)
-        {
-            try
-            {
-                float thresholdValue = Float.parseFloat(threshold);
-                if (thresholdValue < 0)
-                {
-                    throw new ConfigurationException(String.format("%s must be greater than 0, but was %f", TOMBSTONE_THRESHOLD_OPTION, thresholdValue));
-                }
-            }
-            catch (NumberFormatException e)
-            {
-                throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", threshold, TOMBSTONE_THRESHOLD_OPTION), e);
-            }
-        }
-
-        String interval = options.get(TOMBSTONE_COMPACTION_INTERVAL_OPTION);
-        if (interval != null)
-        {
-            try
-            {
-                long tombstoneCompactionInterval = Long.parseLong(interval);
-                if (tombstoneCompactionInterval < 0)
-                {
-                    throw new ConfigurationException(String.format("%s must be greater than 0, but was %d", TOMBSTONE_COMPACTION_INTERVAL_OPTION, tombstoneCompactionInterval));
-                }
-            }
-            catch (NumberFormatException e)
-            {
-                throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", interval, TOMBSTONE_COMPACTION_INTERVAL_OPTION), e);
-            }
-        }
-
-        String unchecked = options.get(UNCHECKED_TOMBSTONE_COMPACTION_OPTION);
-        if (unchecked != null)
-        {
-            if (!unchecked.equalsIgnoreCase("true") && !unchecked.equalsIgnoreCase("false"))
-                throw new ConfigurationException(String.format("'%s' should be either 'true' or 'false', not '%s'", UNCHECKED_TOMBSTONE_COMPACTION_OPTION, unchecked));
-        }
-
-        String logAll = options.get(LOG_ALL_OPTION);
-        if (logAll != null)
-        {
-            if (!logAll.equalsIgnoreCase("true") && !logAll.equalsIgnoreCase("false"))
-            {
-                throw new ConfigurationException(String.format("'%s' should either be 'true' or 'false', not %s", LOG_ALL_OPTION, logAll));
-            }
-        }
+        return getClass().getSimpleName();
+    }
 
-        String compactionEnabled = options.get(COMPACTION_ENABLED);
-        if (compactionEnabled != null)
-        {
-            if (!compactionEnabled.equalsIgnoreCase("true") && !compactionEnabled.equalsIgnoreCase("false"))
-            {
-                throw new ConfigurationException(String.format("enabled should either be 'true' or 'false', not %s", compactionEnabled));
-            }
-        }
+    protected BackgroundCompactions getBackgroundCompactions()
+    {
+        return backgroundCompactions;
+    }
 
-        Map<String, String> uncheckedOptions = new HashMap<String, String>(options);
-        uncheckedOptions.remove(TOMBSTONE_THRESHOLD_OPTION);
-        uncheckedOptions.remove(TOMBSTONE_COMPACTION_INTERVAL_OPTION);
-        uncheckedOptions.remove(UNCHECKED_TOMBSTONE_COMPACTION_OPTION);
-        uncheckedOptions.remove(LOG_ALL_OPTION);
-        uncheckedOptions.remove(COMPACTION_ENABLED);
-        uncheckedOptions.remove(ONLY_PURGE_REPAIRED_TOMBSTONES);
-        uncheckedOptions.remove(CompactionParams.Option.PROVIDE_OVERLAPPING_TOMBSTONES.toString());
-        return uncheckedOptions;
+    public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
+    {
+        return CompactionStrategyOptions.validateOptions(options);
     }
 
     /**
@@ -522,17 +368,20 @@ public static Map<String, String> validateOptions(Map<String, String> options) t
      * anti-compaction to determine which SSTables should be anitcompacted
      * as a group. If a given compaction strategy creates sstables which
      * cannot be merged due to some constraint it must override this method.
+     * @param sstablesToGroup
+     * @return
      */
-    public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> sstablesToGroup)
+    @Override
+    public Collection<Collection<CompactionSSTable>> groupSSTablesForAntiCompaction(Collection<? extends CompactionSSTable> sstablesToGroup)
     {
         int groupSize = 2;
-        List<SSTableReader> sortedSSTablesToGroup = new ArrayList<>(sstablesToGroup);
-        Collections.sort(sortedSSTablesToGroup, SSTableReader.firstKeyComparator);
+        List<CompactionSSTable> sortedSSTablesToGroup = new ArrayList<>(sstablesToGroup);
+        Collections.sort(sortedSSTablesToGroup, CompactionSSTable.firstKeyComparator);
 
-        Collection<Collection<SSTableReader>> groupedSSTables = new ArrayList<>();
-        Collection<SSTableReader> currGroup = new ArrayList<>(groupSize);
+        Collection<Collection<CompactionSSTable>> groupedSSTables = new ArrayList<>();
+        Collection<CompactionSSTable> currGroup = new ArrayList<>(groupSize);
 
-        for (SSTableReader sstable : sortedSSTablesToGroup)
+        for (CompactionSSTable sstable : sortedSSTablesToGroup)
         {
             currGroup.add(sstable);
             if (currGroup.size() == groupSize)
@@ -547,11 +396,6 @@ public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Coll
         return groupedSSTables;
     }
 
-    public CompactionLogger.Strategy strategyLogger()
-    {
-        return CompactionLogger.Strategy.none;
-    }
-
     public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
                                                        long keyCount,
                                                        long repairedAt,
@@ -568,16 +412,43 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
                                                repairedAt,
                                                pendingRepair,
                                                isTransient,
-                                               cfs.metadata,
+                                               realm.metadataRef(),
                                                commitLogPositions,
                                                sstableLevel,
                                                header,
                                                indexGroups,
-                                               lifecycleNewTracker, cfs);
+                                               lifecycleNewTracker,
+                                               realm);
     }
 
+    @Override
     public boolean supportsEarlyOpen()
     {
         return true;
     }
+
+    public void periodicReport()
+    {
+        logCount++;
+        CompactionLogger logger = this.getCompactionLogger();
+        CompactionStrategyOptions options = this.getOptions();
+        BackgroundCompactions backgroundCompactions = this.getBackgroundCompactions();
+        int interval = options.getLogPeriodMinutes();
+        boolean logAll = options.isLogAll();
+        if (logger != null && logger.enabled() && logAll && logCount % interval == 0)
+        {
+            logCount = 0;
+            logger.statistics(this, "periodic", backgroundCompactions.getStatistics(this));
+        }
+    }
+
+    @Override
+    public Map<String, String> getMaxOverlapsMap()
+    {
+        final Set<? extends CompactionSSTable> liveSSTables = getSSTables();
+        return ImmutableMap.of("all", Integer.toString(Overlaps.maxOverlap(liveSSTables,
+                                                                           CompactionSSTable.startsAfter,
+                                                                           CompactionSSTable.firstKeyComparator,
+                                                                           CompactionSSTable.lastKeyComparator)));
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
index 40c4cb49e123..514c80dad652 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
@@ -17,43 +17,68 @@
  */
 package org.apache.cassandra.db.compaction;
 
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Set;
 
 import com.google.common.base.Preconditions;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
 import org.apache.cassandra.io.FSDiskFullWriteError;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.WrappedRunnable;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+
+import static com.google.common.base.Throwables.propagate;
+
 
 public abstract class AbstractCompactionTask extends WrappedRunnable
 {
-    protected final ColumnFamilyStore cfs;
-    protected LifecycleTransaction transaction;
+    // See CNDB-10549
+    static final boolean SKIP_REPAIR_STATE_CHECKING =
+        CassandraRelevantProperties.COMPACTION_SKIP_REPAIR_STATE_CHECKING.getBoolean();
+    static final boolean SKIP_COMPACTING_STATE_CHECKING =
+    CassandraRelevantProperties.COMPACTION_SKIP_COMPACTING_STATE_CHECKING.getBoolean();
+
+    protected final CompactionRealm realm;
+    protected ILifecycleTransaction transaction;
     protected boolean isUserDefined;
     protected OperationType compactionType;
+    protected TableOperationObserver opObserver;
+    protected final List<CompactionObserver> compObservers;
 
     /**
-     * @param cfs
+     * @param realm
      * @param transaction the modifying managing the status of the sstables we're replacing
      */
-    public AbstractCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction transaction)
+    protected AbstractCompactionTask(CompactionRealm realm, ILifecycleTransaction transaction)
     {
-        this.cfs = cfs;
+        this.realm = realm;
         this.transaction = transaction;
         this.isUserDefined = false;
         this.compactionType = OperationType.COMPACTION;
-        // enforce contract that caller should mark sstables compacting
-        Set<SSTableReader> compacting = transaction.tracker.getCompacting();
-        for (SSTableReader sstable : transaction.originals())
-            assert compacting.contains(sstable) : sstable.getFilename() + " is not correctly marked compacting";
+        this.opObserver = TableOperationObserver.NOOP;
+        this.compObservers = new ArrayList<>();
+
+        try
+        {
+            if (!SKIP_COMPACTING_STATE_CHECKING && !transaction.isOffline())
+            {
+                // enforce contract that caller should mark sstables compacting
+                var compacting = realm.getCompactingSSTables();
+                for (SSTableReader sstable : transaction.originals())
+                    assert compacting.contains(sstable) : sstable.getFilename() + " is not correctly marked compacting";
+            }
 
-        validateSSTables(transaction.originals());
+            validateSSTables(transaction.originals());
+        }
+        catch (Throwable err)
+        {
+            propagate(cleanup(err));
+        }
     }
 
     /**
@@ -61,7 +86,10 @@ public AbstractCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction transa
      */
     private void validateSSTables(Set<SSTableReader> sstables)
     {
-        // do not allow  to be compacted together
+        if (SKIP_REPAIR_STATE_CHECKING)
+            return;
+
+        // do not allow sstables in different repair states to be compacted together
         if (!sstables.isEmpty())
         {
             Iterator<SSTableReader> iter = sstables.iterator();
@@ -91,28 +119,60 @@ private void validateSSTables(Set<SSTableReader> sstables)
     }
 
     /**
-     * executes the task and unmarks sstables compacting
+     * Executes the task after setting a new observer, normally the observer is the
+     * compaction manager metrics.
      */
-    public int execute(ActiveCompactionsTracker activeCompactions)
+    public void execute(TableOperationObserver observer)
     {
+        setOpObserver(observer).execute();
+    }
+
+    /** Executes the task */
+    public void execute()
+    {
+        Throwable t = null;
         try
         {
-            return executeInternal(activeCompactions);
+            executeInternal();
         }
-        catch(FSDiskFullWriteError e)
+        catch (FSDiskFullWriteError e)
         {
             RuntimeException cause = new RuntimeException("Converted from FSDiskFullWriteError: " + e.getMessage());
             cause.setStackTrace(e.getStackTrace());
+            t = cause;
             throw new RuntimeException("Throwing new Runtime to bypass exception handler when disk is full", cause);
         }
+        catch (Throwable t1)
+        {
+            t = t1;
+            throw t1;
+        }
         finally
         {
-            transaction.close();
+            Throwables.maybeFail(cleanup(t));
         }
     }
-    public abstract CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables);
 
-    protected abstract int executeInternal(ActiveCompactionsTracker activeCompactions);
+    public Throwable rejected(Throwable t)
+    {
+        return cleanup(t);
+    }
+
+    protected Throwable cleanup(Throwable err)
+    {
+        final Throwable originalError = err;
+        for (CompactionObserver compObserver : compObservers)
+            err = Throwables.perform(err, () -> compObserver.onCompleted(transaction.opId(), originalError));
+
+        return Throwables.perform(err, () -> transaction.close());
+    }
+
+    protected void executeInternal()
+    {
+        run();
+    }
+
+    // TODO Eventually these three setters should be passed in to the constructor.
 
     public AbstractCompactionTask setUserDefined(boolean isUserDefined)
     {
@@ -120,12 +180,57 @@ public AbstractCompactionTask setUserDefined(boolean isUserDefined)
         return this;
     }
 
+    /**
+     * @return The type of compaction this task is performing. Used by CNDB.
+     */
+    public OperationType getCompactionType()
+    {
+        return compactionType;
+    }
+
     public AbstractCompactionTask setCompactionType(OperationType compactionType)
     {
         this.compactionType = compactionType;
         return this;
     }
 
+    /**
+     * Override the NO OP observer, this is normally overridden by the compaction metrics.
+     */
+    public AbstractCompactionTask setOpObserver(TableOperationObserver opObserver)
+    {
+        this.opObserver = opObserver;
+        return this;
+    }
+
+    public void addObserver(CompactionObserver compObserver)
+    {
+        compObservers.add(compObserver);
+    }
+
+    /**
+     * Returns the space overhead of this compaction. This can be used to limit running compactions to they fit under
+     * a given space budget. Only implemented for the types of tasks used by the unified compaction strategy and used
+     * by CNDB.
+     */
+    public abstract long getSpaceOverhead();
+
+    /**
+     * @return The compaction observers for this task. Used by CNDB.
+     */
+    public List<CompactionObserver> getCompObservers()
+    {
+        return compObservers;
+    }
+
+    /**
+     * Return the transaction that this task is working on. Used by CNDB as well as tests.
+     */
+    public ILifecycleTransaction getTransaction()
+    {
+        return transaction;
+    }
+
     public String toString()
     {
         return "CompactionTask(" + transaction + ")";
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
index a1471c77468c..805661a026f7 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
@@ -27,7 +27,6 @@
 
 import com.google.common.base.Preconditions;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.commitlog.IntervalSet;
@@ -50,23 +49,23 @@
  */
 public abstract class AbstractStrategyHolder
 {
-    public static class TaskSupplier implements Comparable<TaskSupplier>
+    public static class TasksSupplier implements Comparable<TasksSupplier>
     {
         private final int numRemaining;
-        private final Supplier<AbstractCompactionTask> supplier;
+        private final Supplier<Collection<AbstractCompactionTask>> supplier;
 
-        TaskSupplier(int numRemaining, Supplier<AbstractCompactionTask> supplier)
+        TasksSupplier(int numRemaining, Supplier<Collection<AbstractCompactionTask>> supplier)
         {
             this.numRemaining = numRemaining;
             this.supplier = supplier;
         }
 
-        public AbstractCompactionTask getTask()
+        public Collection<AbstractCompactionTask> getTasks()
         {
             return supplier.get();
         }
 
-        public int compareTo(TaskSupplier o)
+        public int compareTo(TasksSupplier o)
         {
             return o.numRemaining - numRemaining;
         }
@@ -74,17 +73,17 @@ public int compareTo(TaskSupplier o)
 
     public static interface DestinationRouter
     {
-        int getIndexForSSTable(SSTableReader sstable);
+        int getIndexForSSTable(CompactionSSTable sstable);
         int getIndexForSSTableDirectory(Descriptor descriptor);
     }
 
     /**
      * Maps sstables to their token partition bucket
      */
-    public static class GroupedSSTableContainer
+    public static class GroupedSSTableContainer<S extends CompactionSSTable>
     {
         private final AbstractStrategyHolder holder;
-        private final Set<SSTableReader>[] groups;
+        private final Set<S>[] groups;
 
         private GroupedSSTableContainer(AbstractStrategyHolder holder)
         {
@@ -93,7 +92,7 @@ private GroupedSSTableContainer(AbstractStrategyHolder holder)
             groups = new Set[holder.numTokenPartitions];
         }
 
-        void add(SSTableReader sstable)
+        void add(S sstable)
         {
             Preconditions.checkArgument(holder.managesSSTable(sstable), "this strategy holder doesn't manage %s", sstable);
             int idx = holder.router.getIndexForSSTable(sstable);
@@ -108,10 +107,10 @@ public int numGroups()
             return groups.length;
         }
 
-        public Set<SSTableReader> getGroup(int i)
+        public Set<S> getGroup(int i)
         {
             Preconditions.checkArgument(i >= 0 && i < groups.length);
-            Set<SSTableReader> group = groups[i];
+            Set<S> group = groups[i];
             return group != null ? group : Collections.emptySet();
         }
 
@@ -129,13 +128,15 @@ boolean isEmpty()
         }
     }
 
-    protected final ColumnFamilyStore cfs;
+    protected final CompactionRealm realm;
+    protected final CompactionStrategyFactory strategyFactory;
     final DestinationRouter router;
     private int numTokenPartitions = -1;
 
-    AbstractStrategyHolder(ColumnFamilyStore cfs, DestinationRouter router)
+    AbstractStrategyHolder(CompactionRealm realm, CompactionStrategyFactory strategyFactory, DestinationRouter router)
     {
-        this.cfs = cfs;
+        this.realm = realm;
+        this.strategyFactory = strategyFactory;
         this.router = router;
     }
 
@@ -161,34 +162,33 @@ final void setStrategy(CompactionParams params, int numTokenPartitions)
      */
     public abstract boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient);
 
-    public boolean managesSSTable(SSTableReader sstable)
+    public boolean managesSSTable(CompactionSSTable sstable)
     {
         return managesRepairedGroup(sstable.isRepaired(), sstable.isPendingRepair(), sstable.isTransient());
     }
 
-    public abstract AbstractCompactionStrategy getStrategyFor(SSTableReader sstable);
+    public abstract LegacyAbstractCompactionStrategy getStrategyFor(CompactionSSTable sstable);
 
-    public abstract Iterable<AbstractCompactionStrategy> allStrategies();
+    public abstract Iterable<LegacyAbstractCompactionStrategy> allStrategies();
 
-    public abstract Collection<TaskSupplier> getBackgroundTaskSuppliers(long gcBefore);
+    public abstract Collection<TasksSupplier> getBackgroundTaskSuppliers(long gcBefore);
 
-    public abstract Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, boolean splitOutput);
+    public abstract Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism);
 
-    public abstract Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableContainer sstables, long gcBefore);
+    public abstract Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableContainer<CompactionSSTable> sstables, long gcBefore);
 
-    public GroupedSSTableContainer createGroupedSSTableContainer()
+    public <S extends CompactionSSTable> GroupedSSTableContainer<S> createGroupedSSTableContainer()
     {
-        return new GroupedSSTableContainer(this);
+        return new GroupedSSTableContainer<>(this);
     }
 
-    public abstract void addSSTable(SSTableReader sstable);
-    public abstract void addSSTables(GroupedSSTableContainer sstables);
+    public abstract void addSSTables(GroupedSSTableContainer<CompactionSSTable> sstables);
 
-    public abstract void removeSSTables(GroupedSSTableContainer sstables);
+    public abstract void removeSSTables(GroupedSSTableContainer<CompactionSSTable> sstables);
 
-    public abstract void replaceSSTables(GroupedSSTableContainer removed, GroupedSSTableContainer added);
+    public abstract void replaceSSTables(GroupedSSTableContainer<CompactionSSTable> removed, GroupedSSTableContainer<CompactionSSTable> added);
 
-    public abstract List<ISSTableScanner> getScanners(GroupedSSTableContainer sstables, Collection<Range<Token>> ranges);
+    public abstract List<ISSTableScanner> getScanners(GroupedSSTableContainer<SSTableReader> sstables, Collection<Range<Token>> ranges);
 
 
     public abstract SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
@@ -202,13 +202,7 @@ public abstract SSTableMultiWriter createSSTableMultiWriter(Descriptor descripto
                                                                 Collection<Index.Group> indexGroups,
                                                                 LifecycleNewTracker lifecycleNewTracker);
 
-    /**
-     * Return the directory index the given compaction strategy belongs to, or -1
-     * if it's not held by this holder
-     */
-    public abstract int getStrategyIndex(AbstractCompactionStrategy strategy);
-
-    public abstract boolean containsSSTable(SSTableReader sstable);
+    public abstract boolean containsSSTable(CompactionSSTable sstable);
 
     public abstract int getEstimatedRemainingTasks();
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractTableOperation.java b/src/java/org/apache/cassandra/db/compaction/AbstractTableOperation.java
new file mode 100644
index 000000000000..85602202d374
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractTableOperation.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * This is a base abstract implementing some default methods of {@link TableOperation}.
+ * <p/>
+ * In previous versions it used to be called CompactionInfo and CompactionInfo.Holder.
+ * <p/>
+ * This class implements serializable to allow structured info to be returned via JMX.
+ **/
+public abstract class AbstractTableOperation implements TableOperation
+{
+    private volatile boolean stopRequested = false;
+    private volatile StopTrigger trigger = StopTrigger.NONE;
+
+    /**
+     * Interrupt the current operation if possible and if the predicate is true.
+     *
+     * @param trigger cause of compaction interruption
+     */
+    @Override
+    public void stop(StopTrigger trigger)
+    {
+        this.stopRequested = true;
+        if (!this.trigger.isFinal())
+            this.trigger = trigger;
+    }
+
+    /**
+     * @return true if the operation has received a request to be interrupted.
+     */
+    @Override
+    public boolean isStopRequested()
+    {
+        return stopRequested || (isGlobal() && CompactionManager.instance.isGlobalCompactionPaused());
+    }
+
+    /**
+     * Return true if the predicate for the given sstables holds, or if the operation
+     * does not consider any sstables, in which case it will always return true (the
+     * default behaviour).
+     */
+    @Override
+    public boolean shouldStop(Predicate<SSTableReader> predicate)
+    {
+        Progress progress = getProgress();
+        final Set<SSTableReader> sstables = progress.sstables();
+        if (sstables.isEmpty())
+            return true;
+
+        return sstables.stream().anyMatch(predicate);
+    }
+
+    /**
+     * @return cause of compaction interruption.
+     */
+    @Override
+    public StopTrigger trigger()
+    {
+        return trigger;
+    }
+
+    /**
+     * The progress information for an operation, refer to the description of the class properties.
+     */
+    public static class OperationProgress implements Serializable, Progress
+    {
+        private static final long serialVersionUID = 3695381572726744816L;
+
+        /**
+         * The table metadata
+         */
+        private final TableMetadata metadata;
+        /**
+         * The type of operation
+         */
+        private final OperationType operationType;
+        /**
+         * Normally the bytes processed so far by this operation, but depending on the unit it could mean something else, e.g. ranges or keys.
+         */
+        private final long completed;
+        /**
+         * The total bytes that need to be processed, for example the size of the input files. Depending on the unit it could mean something else, e.g. ranges or keys.
+         */
+        private final long total;
+        /**
+         * The unit for {@link this#completed} and for {@link this#total}.
+         */
+        private final Unit unit;
+        /**
+         * A unique ID for this operation
+         */
+        private final TimeUUID operationId;
+        /**
+         * A set of SSTables participating in this operation
+         */
+        private final ImmutableSet<SSTableReader> sstables;
+        private final String targetDirectory;
+
+        public OperationProgress(TableMetadata metadata, OperationType operationType, long bytesComplete, long totalBytes, TimeUUID operationId, Collection<SSTableReader> sstables, String targetDirectory)
+        {
+            this(metadata, operationType, bytesComplete, totalBytes, Unit.BYTES, operationId, sstables, targetDirectory);
+        }
+
+        public OperationProgress(TableMetadata metadata, OperationType operationType, long bytesComplete, long totalBytes, TimeUUID operationId, Collection<? extends SSTableReader> sstables)
+        {
+            this(metadata, operationType, bytesComplete, totalBytes, Unit.BYTES, operationId, sstables, null);
+        }
+
+        public OperationProgress(TableMetadata metadata, OperationType operationType, long bytesComplete, long totalBytes, long totalBytesScanned, TimeUUID operationId, Collection<? extends SSTableReader> sstables)
+        {
+            this(metadata, operationType, bytesComplete, totalBytes, Unit.BYTES, operationId, sstables, null);
+        }
+
+        public OperationProgress(TableMetadata metadata, OperationType operationType, long completed, long total, Unit unit, TimeUUID operationId, Collection<? extends SSTableReader> sstables, String targetDirectory)
+        {
+            this.operationType = operationType;
+            this.completed = completed;
+            this.total = total;
+            this.metadata = metadata;
+            this.unit = unit;
+            this.operationId = operationId;
+            this.sstables = ImmutableSet.copyOf(sstables);
+            this.targetDirectory = targetDirectory;
+        }
+
+        /**
+         * @return A copy of this OperationProgress with updated progress.
+         */
+        public OperationProgress forProgress(long complete, long total)
+        {
+            return new OperationProgress(metadata, operationType, complete, total, unit, operationId, sstables, targetDirectory);
+        }
+
+        /**
+         * Special operation progress where we always need to cancel the compaction - for example ViewBuilderTask where we don't know
+         * the sstables at construction
+         */
+        public static OperationProgress withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, AbstractTableOperation.Unit unit, TimeUUID compactionId)
+        {
+            return withoutSSTables(metadata, tasktype, completed, total, unit, compactionId, null);
+        }
+
+        /**
+         * Special operation progress where we always need to cancel the compaction - for example AutoSavingCache where we don't know
+         * the sstables at construction
+         */
+        public static OperationProgress withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, AbstractTableOperation.Unit unit, TimeUUID compactionId, String targetDirectory)
+        {
+            return new OperationProgress(metadata, tasktype, completed, total, unit, compactionId, ImmutableSet.of(), targetDirectory);
+        }
+
+        @Override
+        public Optional<String> keyspace()
+        {
+            return metadata != null ? Optional.of(metadata.keyspace) : Optional.empty();
+        }
+
+        @Override
+        public Optional<String> table()
+        {
+            return metadata != null ? Optional.of(metadata.name) : Optional.empty();
+        }
+
+        @Override
+        public TableMetadata metadata()
+        {
+            return metadata;
+        }
+
+        @Override
+        public long completed()
+        {
+            return completed;
+        }
+
+        @Override
+        public long total()
+        {
+            return total;
+        }
+
+        @Override
+        public OperationType operationType()
+        {
+            return operationType;
+        }
+
+        @Override
+        public TimeUUID operationId()
+        {
+            return operationId;
+        }
+
+        @Override
+        public Unit unit()
+        {
+            return unit;
+        }
+
+        @Override
+        public Set<SSTableReader> sstables()
+        {
+            return sstables;
+        }
+
+        @Override
+        public String targetDirectory()
+        {
+            if (targetDirectory == null)
+                return "";
+
+            try
+            {
+                return new File(targetDirectory).canonicalPath();
+            }
+            catch (Throwable t)
+            {
+                throw new RuntimeException("Unable to resolve canonical path for " + targetDirectory);
+            }
+        }
+
+        public String toString()
+        {
+            return progressToString();
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java b/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java
deleted file mode 100644
index 4e238ad95d46..000000000000
--- a/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.compaction;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.File;
-
-public class ActiveCompactions implements ActiveCompactionsTracker
-{
-    // a synchronized identity set of running tasks to their compaction info
-    private final Set<CompactionInfo.Holder> compactions = Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<>()));
-
-    public List<CompactionInfo.Holder> getCompactions()
-    {
-        return new ArrayList<>(compactions);
-    }
-
-    public void beginCompaction(CompactionInfo.Holder ci)
-    {
-        compactions.add(ci);
-    }
-
-    public void finishCompaction(CompactionInfo.Holder ci)
-    {
-        compactions.remove(ci);
-        CompactionManager.instance.getMetrics().bytesCompacted.inc(ci.getCompactionInfo().getTotal());
-        CompactionManager.instance.getMetrics().totalCompactionsCompleted.mark();
-    }
-
-    /**
-     * Get the estimated number of bytes remaining to write per sstable directory
-     */
-    public Map<File, Long> estimatedRemainingWriteBytes()
-    {
-        synchronized (compactions)
-        {
-            Map<File, Long> writeBytesPerSSTableDir = new HashMap<>();
-            for (CompactionInfo.Holder holder : compactions)
-            {
-                CompactionInfo compactionInfo = holder.getCompactionInfo();
-                List<File> directories = compactionInfo.getTargetDirectories();
-                if (directories == null || directories.isEmpty())
-                    continue;
-                long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteBytes() / directories.size();
-                for (File directory : directories)
-                    writeBytesPerSSTableDir.merge(directory, remainingWriteBytesPerDataDir, Long::sum);
-            }
-            return writeBytesPerSSTableDir;
-        }
-    }
-
-    /**
-     * Iterates over the active compactions and tries to find CompactionInfos with the given compactionType for the given sstable
-     *
-     * Number of entries in compactions should be small (< 10) but avoid calling in any time-sensitive context
-     */
-    public Collection<CompactionInfo> getCompactionsForSSTable(SSTableReader sstable, OperationType compactionType)
-    {
-        List<CompactionInfo> toReturn = null;
-        synchronized (compactions)
-        {
-            for (CompactionInfo.Holder holder : compactions)
-            {
-                CompactionInfo compactionInfo = holder.getCompactionInfo();
-                if (compactionInfo.getSSTables().contains(sstable) && compactionInfo.getTaskType() == compactionType)
-                {
-                    if (toReturn == null)
-                        toReturn = new ArrayList<>();
-                    toReturn.add(compactionInfo);
-                }
-            }
-        }
-        return toReturn;
-    }
-}
diff --git a/src/java/org/apache/cassandra/db/compaction/ActiveCompactionsTracker.java b/src/java/org/apache/cassandra/db/compaction/ActiveCompactionsTracker.java
deleted file mode 100644
index c1bbbd8e67bf..000000000000
--- a/src/java/org/apache/cassandra/db/compaction/ActiveCompactionsTracker.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.compaction;
-
-public interface ActiveCompactionsTracker
-{
-    public void beginCompaction(CompactionInfo.Holder ci);
-    public void finishCompaction(CompactionInfo.Holder ci);
-
-    public static final ActiveCompactionsTracker NOOP = new ActiveCompactionsTracker()
-    {
-        public void beginCompaction(CompactionInfo.Holder ci)
-        {}
-
-        public void finishCompaction(CompactionInfo.Holder ci)
-        {}
-    };
-}
diff --git a/src/java/org/apache/cassandra/db/compaction/ActiveOperations.java b/src/java/org/apache/cassandra/db/compaction/ActiveOperations.java
new file mode 100644
index 000000000000..b49a70a90398
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/ActiveOperations.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import com.google.common.collect.ImmutableList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.NonThrowingCloseable;
+
+@ThreadSafe
+public class ActiveOperations implements TableOperationObserver
+{
+    private static final Logger logger = LoggerFactory.getLogger(ActiveOperations.class);
+
+    // The operations ordered by keyspace.table for all the operations that are currently in progress.
+    private static final Set<TableOperation> operations = Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<>()));
+
+    // Keep registered listeners to be called onStart and close
+    private final List<CompactionProgressListener> listeners = new CopyOnWriteArrayList<>();
+
+    public interface CompactionProgressListener
+    {
+        /**
+         * Called when compaction started
+         */
+        default void onStarted(TableOperation.Progress progress) {}
+
+        /**
+         * Called when compaction completed
+         */
+        default void onCompleted(TableOperation.Progress progressOnCompleted) {}
+    }
+
+    public void registerListener(CompactionProgressListener listener)
+    {
+        listeners.add(listener);
+    }
+
+    public void unregisterListener(CompactionProgressListener listener)
+    {
+        listeners.remove(listener);
+    }
+
+    /**
+     * @return all the table operations currently in progress. This is mostly compactions but it can include other
+     *         operations too, basically any operation that calls {@link this#onOperationStart(TableOperation)}.
+     */
+    public List<TableOperation> getTableOperations()
+    {
+        ImmutableList.Builder<TableOperation> builder = ImmutableList.builder();
+        synchronized (operations)
+        {
+            builder.addAll(operations);
+        }
+        return builder.build();
+    }
+
+    @Override
+    public NonThrowingCloseable onOperationStart(TableOperation op)
+    {
+        TableOperation.Progress progress = op.getProgress();
+        for (CompactionProgressListener listener : listeners)
+        {
+            try
+            {
+                listener.onStarted(progress);
+            }
+            catch (Throwable t)
+            {
+                String listenerName = listener.getClass().getName();
+                logger.error("Unable to notify listener {} while trying to start compaction {} on table {}",
+                             listenerName, progress.operationType(), progress.metadata(), t);
+            }
+        }
+        operations.add(op);
+        return () -> completeOperation(op);
+    }
+
+    private void completeOperation(TableOperation op)
+    {
+        operations.remove(op);
+        TableOperation.Progress progressOnCompleted = op.getProgress();
+        CompactionManager.instance.getMetrics().bytesCompacted.inc(progressOnCompleted.total());
+        CompactionManager.instance.getMetrics().totalCompactionsCompleted.mark();
+
+        for (CompactionProgressListener listener : listeners)
+        {
+            try
+            {
+                listener.onCompleted(progressOnCompleted);
+            }
+            catch (Throwable t)
+            {
+                String listenerName = listener.getClass().getName();
+                logger.error("Unable to notify listener {} while trying to complete compaction {} on table {}",
+                             listenerName, progressOnCompleted.operationType(), progressOnCompleted.metadata(), t);
+            }
+        }
+    }
+
+    /**
+     * Get the estimated number of bytes remaining to write per sstable directory
+     */
+    public Map<File, Long> estimatedRemainingWriteBytes()
+    {
+        synchronized (operations)
+        {
+            Map<File, Long> writeBytesPerSSTableDir = new HashMap<>();
+            for (TableOperation holder : operations)
+            {
+                TableOperation.Progress compactionInfo = holder.getProgress();
+                List<File> directories = compactionInfo.getTargetDirectories();
+                if (directories == null || directories.isEmpty())
+                    continue;
+                long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteBytes() / directories.size();
+                for (File directory : directories)
+                    writeBytesPerSSTableDir.merge(directory, remainingWriteBytesPerDataDir, Long::sum);
+            }
+            return writeBytesPerSSTableDir;
+        }
+    }
+
+    /**
+     * Iterates over the active operations and tries to find OperationProgresses with the given operation type for the given sstable
+     *
+     * Number of entries in operations should be small (< 10) but avoid calling in any time-sensitive context
+     */
+    public Collection<TableOperation.Progress> getOperationsForSSTable(SSTableReader sstable, OperationType operationType)
+    {
+        List<TableOperation.Progress> toReturn = null;
+
+        synchronized (operations)
+        {
+            for (TableOperation op : operations)
+            {
+                TableOperation.Progress progress = op.getProgress();
+                if (progress.sstables().contains(sstable) && progress.operationType() == operationType)
+                {
+                    if (toReturn == null)
+                        toReturn = new ArrayList<>();
+                    toReturn.add(progress);
+                }
+            }
+        }
+        return toReturn;
+    }
+
+    /**
+     * @return true if given table operation is still active
+     */
+    public boolean isActive(TableOperation op)
+    {
+        return getTableOperations().contains(op);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/ArenaSelector.java b/src/java/org/apache/cassandra/db/compaction/ArenaSelector.java
new file mode 100644
index 000000000000..f3a38f30620a
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/ArenaSelector.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.db.DiskBoundaries;
+import org.apache.cassandra.db.compaction.unified.Controller;
+
+/**
+ * Arena selector, used by UnifiedCompactionStrategy to distribute SSTables to separate compaction arenas.
+ *
+ * This is used to:
+ * - ensure that sstables that should not be compacted together (e.g. repaired with unrepaired) are separated
+ * - ensure that each disk's sstables are compacted separately
+ */
+public class ArenaSelector implements Comparator<CompactionSSTable>
+{
+    private final EquivClassSplitter[] classSplitters;
+    final Controller controller;
+    final DiskBoundaries diskBoundaries;
+
+    public ArenaSelector(Controller controller, DiskBoundaries diskBoundaries)
+    {
+        this.controller = controller;
+        this.diskBoundaries = diskBoundaries;
+
+        ArrayList<EquivClassSplitter> ret = new ArrayList<>(2);
+
+        ret.add(RepairEquivClassSplitter.INSTANCE);
+
+        if (diskBoundaries.getNumBoundaries() > 1)
+        {
+            ret.add(new DiskIndexEquivClassSplitter());
+        }
+
+        classSplitters = ret.toArray(new EquivClassSplitter[0]);
+    }
+
+    @Override
+    public int compare(CompactionSSTable o1, CompactionSSTable o2)
+    {
+        int res = 0;
+        for (int i = 0; res == 0 && i < classSplitters.length; i++)
+            res = classSplitters[i].compare(o1, o2);
+        return res;
+    }
+
+    public String name(CompactionSSTable t)
+    {
+        return Arrays.stream(classSplitters)
+                     .map(e -> e.name(t))
+                     .collect(Collectors.joining("-"));
+    }
+
+    /**
+     * An equivalence class is a function that compares two sstables and returns 0 when they fall in the same class.
+     * For example, the repair status or disk index may define equivalence classes. See the concrete equivalence classes below.
+     */
+    private interface EquivClassSplitter extends Comparator<CompactionSSTable> {
+
+        @Override
+        int compare(CompactionSSTable a, CompactionSSTable b);
+
+        /** Return a name that describes the equivalence class */
+        String name(CompactionSSTable ssTableReader);
+    }
+
+    /**
+     * Split sstables by their repair state: repaired, unrepaired, pending repair with a specific UUID (one group per pending repair).
+     */
+    private static final class RepairEquivClassSplitter implements EquivClassSplitter
+    {
+        public static final EquivClassSplitter INSTANCE = new RepairEquivClassSplitter();
+
+        @Override
+        public int compare(CompactionSSTable a, CompactionSSTable b)
+        {
+            // This is the same as name(a).compareTo(name(b))
+            int af = repairClassValue(a);
+            int bf = repairClassValue(b);
+            if (af != 0 || bf != 0)
+                return Integer.compare(af, bf);
+            return a.getPendingRepair().compareTo(b.getPendingRepair());
+        }
+
+        private static int repairClassValue(CompactionSSTable a)
+        {
+            if (a.isRepaired())
+                return 1;
+            if (!a.isPendingRepair())
+                return 2;
+            else
+                return 0;
+        }
+
+        @Override
+        public String name(CompactionSSTable ssTableReader)
+        {
+            if (ssTableReader.isRepaired())
+                return "repaired";
+            else if (!ssTableReader.isPendingRepair())
+                return "unrepaired";
+            else
+                return "pending_repair_" + ssTableReader.getPendingRepair();
+        }
+    }
+
+    /**
+     * Group sstables by their disk index.
+     */
+    private final class DiskIndexEquivClassSplitter implements EquivClassSplitter
+    {
+        @Override
+        public int compare(CompactionSSTable a, CompactionSSTable b)
+        {
+            return Integer.compare(diskBoundaries.getDiskIndexFromKey(a), diskBoundaries.getDiskIndexFromKey(b));
+        }
+
+        @Override
+        public String name(CompactionSSTable ssTableReader)
+        {
+            return "disk_" + diskBoundaries.getDiskIndexFromKey(ssTableReader);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/BackgroundCompactionRunner.java b/src/java/org/apache/cassandra/db/compaction/BackgroundCompactionRunner.java
new file mode 100644
index 000000000000..8e33d166de76
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/BackgroundCompactionRunner.java
@@ -0,0 +1,500 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture; // checkstyle: permit this import
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.concurrent.WrappedExecutorPlus;
+import org.apache.cassandra.utils.concurrent.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.FSDiskFullWriteError;
+import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+
+public class BackgroundCompactionRunner implements Runnable
+{
+    private static final Logger logger = LoggerFactory.getLogger(BackgroundCompactionRunner.class);
+
+    public static final String NO_SPACE_LEFT_MESSAGE = "No space left on device";
+
+    public enum RequestResult
+    {
+        /**
+         * when the compaction check was done and there were no compaction tasks for the CFS
+         */
+        NOT_NEEDED,
+
+        /**
+         * when the compaction was aborted for the CFS because the CF got dropped in the meantime
+         */
+        ABORTED,
+
+        /**
+         * compaction tasks were completed for the CFS
+         */
+        COMPLETED
+    }
+
+    private final ScheduledExecutorPlus checkExecutor;
+
+    /**
+     * CFSs for which a compaction was requested mapped to the promise returned to the requesting code
+     */
+    private final ConcurrentMap<ColumnFamilyStore, FutureRequestResult> compactionRequests = new ConcurrentHashMap<>();
+
+    private final AtomicInteger ongoingUpgrades = new AtomicInteger(0);
+
+    /**
+     * Tracks the number of currently requested compactions. Used to delay checking for new compactions until there's
+     * room in the executing threads.
+     */
+    private final AtomicInteger ongoingCompactions = new AtomicInteger(0);
+
+    private final Random random = new Random();
+
+    private final WrappedExecutorPlus compactionExecutor;
+
+    private final ActiveOperations activeOperations;
+
+
+    BackgroundCompactionRunner(WrappedExecutorPlus compactionExecutor, ActiveOperations activeOperations)
+    {
+        this(compactionExecutor, executorFactory().scheduled("BackgroundTaskExecutor"), activeOperations);
+    }
+
+    @VisibleForTesting
+    BackgroundCompactionRunner(WrappedExecutorPlus compactionExecutor, ScheduledExecutorPlus checkExecutor, ActiveOperations activeOperations)
+    {
+        this.compactionExecutor = compactionExecutor;
+        this.checkExecutor = checkExecutor;
+        this.activeOperations = activeOperations;
+    }
+
+    /**
+     * This extends and behave like a {@link CompletableFuture}, with the exception that one cannot call
+     * {@link #cancel}, {@link #setSuccess} and {@link #setFailure} (they throw {@link UnsupportedOperationException}).
+     */
+    public static class FutureRequestResult extends AsyncPromise<RequestResult>
+    {
+        @Override
+        public Promise<RequestResult> setSuccess(RequestResult t)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean cancel(boolean interruptIfRunning)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Promise<RequestResult> setFailure(Throwable throwable)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        private void completeInternal(RequestResult t)
+        {
+            super.trySuccess(t);
+        }
+
+        private void completeExceptionallyInternal(Throwable throwable)
+        {
+            super.tryFailure(throwable);
+        }
+    }
+
+    /**
+     * Marks each CFS in a set for compaction. See {@link #markForCompactionCheck(ColumnFamilyStore)} for details.
+     */
+    void markForCompactionCheck(Set<ColumnFamilyStore> cfss)
+    {
+        List<FutureRequestResult> results = cfss.stream()
+                                                .map(this::requestCompactionInternal)
+                                                .filter(Objects::nonNull)
+                                                .collect(Collectors.toList());
+
+        if (!results.isEmpty() && !maybeScheduleNextCheck())
+        {
+            logger.info("Executor has been shut down, background compactions check will not be scheduled");
+            results.forEach(r -> r.completeInternal(RequestResult.ABORTED));
+        }
+    }
+
+    /**
+     * Marks a CFS for compaction. Since marked, it will become a possible candidate for compaction. The mark will be
+     * cleared when we actually run the compaction for CFS.
+     *
+     * @return a promise which will be completed when the mark is cleared. The returned future should not be cancelled or
+     * completed by the caller.
+     */
+    Promise<RequestResult> markForCompactionCheck(ColumnFamilyStore cfs)
+    {
+        FutureRequestResult p = requestCompactionInternal(cfs);
+        if (p == null)
+            return new AsyncPromise<RequestResult>().setSuccess(RequestResult.ABORTED);
+
+        if (!maybeScheduleNextCheck())
+        {
+            logger.info("Executor has been shut down, background compactions check will not be scheduled");
+            p.completeInternal(RequestResult.ABORTED);
+        }
+        return p;
+    }
+
+    private FutureRequestResult requestCompactionInternal(ColumnFamilyStore cfs)
+    {
+        logger.trace("Requested background compaction for {}", cfs);
+        if (!cfs.isValid())
+        {
+            logger.trace("Aborting compaction for dropped CF {}", cfs);
+            return null;
+        }
+        if (cfs.isAutoCompactionDisabled())
+        {
+            logger.trace("Autocompaction is disabled");
+            return null;
+        }
+
+        return compactionRequests.computeIfAbsent(cfs, ignored -> new FutureRequestResult());
+    }
+
+    void shutdown()
+    {
+        checkExecutor.shutdown();
+        compactionRequests.values().forEach(promise -> promise.completeInternal(RequestResult.ABORTED));
+        // it's okay to complete a CompletableFuture more than one on race between request, run and shutdown
+    }
+
+    @VisibleForTesting
+    int getOngoingCompactionsCount()
+    {
+        return ongoingCompactions.get();
+    }
+
+    @VisibleForTesting
+    int getOngoingUpgradesCount()
+    {
+        return ongoingUpgrades.get();
+    }
+
+    @VisibleForTesting
+    Set<ColumnFamilyStore> getMarkedCFSs()
+    {
+        return ImmutableSet.copyOf(compactionRequests.keySet());
+    }
+
+    @Override
+    public void run()
+    {
+        logger.trace("Running background compactions check");
+
+        // When the executor is fully occupied, we delay acting on this request until a thread is available. This
+        // helps make a better decision what exactly to compact (e.g. if we issue the request now we may select n
+        // sstables, while by the time this request actually has a thread to execute on more may have accumulated,
+        // and it may be better to compact all).
+        // Note that we make a request whenever a task completes and thus this method is guaranteed to run again
+        // when threads free up.
+        if (ongoingCompactions.get() >= compactionExecutor.getMaximumPoolSize())
+        {
+            logger.trace("Background compaction threads are busy; delaying new compactions check until there are free threads");
+            return;
+        }
+
+        // We shuffle the CFSs for which the compaction was requested so that with each run we traverse those CFSs
+        // in different order and make each CFS have equal chance to be selected
+        ArrayList<ColumnFamilyStore> compactionRequestsList = new ArrayList<>(compactionRequests.keySet());
+        Collections.shuffle(compactionRequestsList, random);
+
+        for (ColumnFamilyStore cfs : compactionRequestsList)
+        {
+            if (ongoingCompactions.get() >= compactionExecutor.getMaximumPoolSize())
+            {
+                logger.trace("Background compaction threads are busy; delaying new compactions check until there are free threads");
+                return;
+            }
+
+            FutureRequestResult promise = compactionRequests.remove(cfs);
+            assert promise != null : "Background compaction checker must be single-threaded";
+
+            if (promise.isDone())
+            {
+                // A shutdown request may abort processing while we are still processing
+                try
+                {
+                    assert promise.get() == RequestResult.ABORTED : "Background compaction checker must be single-threaded";
+                }
+                catch (InterruptedException e)
+                {
+                    throw new UncheckedInterruptedException(e);
+                }
+                catch (ExecutionException e)
+                {
+                    throw new RuntimeException(e);
+                }
+
+                logger.trace("The request for {} was aborted due to shutdown", cfs);
+                continue;
+            }
+
+            if (!cfs.isValid())
+            {
+                logger.trace("Aborting compaction for dropped CF {}", cfs);
+                promise.completeInternal(RequestResult.ABORTED);
+                continue;
+            }
+
+            logger.trace("Running a background task check for {} with {}", cfs, cfs.getCompactionStrategy().getName());
+
+            CompletableFuture<Void> compactionTasks = startCompactionTasks(cfs);
+            if (compactionTasks == null)
+                compactionTasks = startUpgradeTasks(cfs);
+
+            if (compactionTasks != null)
+            {
+                compactionTasks.handle((ignored, throwable) -> {
+                    if (throwable != null)
+                    {
+                        handleCompactionError(throwable, cfs);
+                        promise.completeExceptionallyInternal(throwable);
+                    }
+                    else
+                    {
+                        logger.trace("Finished compaction for {}", cfs);
+                        promise.completeInternal(RequestResult.COMPLETED);
+                    }
+                    return null;
+                });
+
+                // The compaction strategy may return more subsequent tasks if we ask for them. Therefore, we request
+                // the compaction again on that CFS early (without waiting for the currently scheduled/started
+                // compaction tasks to finish). We can start them in the next check round if we have free slots
+                // in the compaction executor.
+                markForCompactionCheck(cfs);
+            }
+            else
+            {
+                promise.completeInternal(RequestResult.NOT_NEEDED);
+            }
+        }
+    }
+
+    private boolean maybeScheduleNextCheck()
+    {
+        if (checkExecutor.getPendingTaskCount() == 0)
+        {
+            try
+            {
+                checkExecutor.execute(this);
+            }
+            catch (RejectedExecutionException ex)
+            {
+                if (checkExecutor.isShutdown())
+                    logger.info("Executor has been shut down, background compactions check will not be scheduled");
+                else
+                    logger.error("Failed to submit background compactions check", ex);
+
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private CompletableFuture<Void> startCompactionTasks(ColumnFamilyStore cfs)
+    {
+        Collection<AbstractCompactionTask> compactionTasks = cfs.getCompactionStrategy()
+                                                                .getNextBackgroundTasks(CompactionManager.getDefaultGcBefore(cfs, FBUtilities.nowInSeconds()));
+        CompletableFuture<?>[] compactionTaskFutures = startCompactionTasks(cfs, compactionTasks);
+        return compactionTaskFutures != null ? CompletableFuture.allOf(compactionTaskFutures) : null;
+    }
+    
+    CompletableFuture<?>[] startCompactionTasks(ColumnFamilyStore cfs, Collection<AbstractCompactionTask> compactionTasks)
+    {
+        if (!compactionTasks.isEmpty())
+        {
+            logger.debug("Running compaction tasks: {}", compactionTasks);
+            CompletableFuture<Void>[] arr = new CompletableFuture[compactionTasks.size()];
+            int index = 0;
+            for (AbstractCompactionTask task : compactionTasks)
+                arr[index++] = startTask(cfs, task);
+
+            return arr;
+        }
+        else
+        {
+            logger.trace("No compaction tasks for {}", cfs);
+            return null;
+        }
+    }
+
+    private CompletableFuture<Void> startTask(ColumnFamilyStore cfs, AbstractCompactionTask task)
+    {
+        ongoingCompactions.incrementAndGet();
+        try
+        {
+            return CompletableFuture.runAsync(
+            () -> {
+                try
+                {
+                    task.execute(activeOperations);
+                }
+                finally
+                {
+                    ongoingCompactions.decrementAndGet();
+
+                    // Request a new round of checking for compactions. We do this for two reasons:
+                    //  - a task has completed and there may now be new compaction possibilities in this CFS,
+                    //  - a thread has freed up, and a new compaction task (from any CFS) can be scheduled on it
+                    markForCompactionCheck(cfs);
+                }
+            }, compactionExecutor);
+        }
+        catch (RejectedExecutionException ex)
+        {
+            ongoingCompactions.decrementAndGet();
+            logger.debug("Background compaction task for {} was rejected", cfs);
+            return CompletableFuture.completedFuture(null);
+        }
+    }
+
+    private CompletableFuture<Void> startUpgradeTasks(ColumnFamilyStore cfs)
+    {
+        AbstractCompactionTask upgradeTask = getUpgradeSSTableTask(cfs);
+
+        if (upgradeTask != null)
+        {
+            logger.debug("Running upgrade task: {}", upgradeTask);
+            return startTask(cfs, upgradeTask).handle((ignored1, ignored2) -> {
+                ongoingUpgrades.decrementAndGet();
+                return null;
+            });
+        }
+        else
+        {
+            logger.trace("No upgrade tasks for {}", cfs);
+            return null;
+        }
+    }
+
+    /**
+     * Finds the oldest (by modification date) non-latest-version sstable on disk and creates an upgrade task for it
+     */
+    @VisibleForTesting
+    public AbstractCompactionTask getUpgradeSSTableTask(ColumnFamilyStore cfs)
+    {
+        logger.trace("Checking for upgrade tasks {}", cfs);
+
+        if (!DatabaseDescriptor.automaticSSTableUpgrade())
+        {
+            logger.trace("Automatic sstable upgrade is disabled - will not try to upgrade sstables of {}", cfs);
+            return null;
+        }
+
+        if (ongoingUpgrades.incrementAndGet() <= DatabaseDescriptor.maxConcurrentAutoUpgradeTasks())
+        {
+            List<SSTableReader> potentialUpgrade = cfs.getCandidatesForUpgrade();
+            for (SSTableReader sstable : potentialUpgrade)
+            {
+                LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UPGRADE_SSTABLES);
+                if (txn != null)
+                {
+                    logger.debug("Found tasks for automatic sstable upgrade of {}", sstable);
+                    return cfs.getCompactionStrategy().createCompactionTask(txn, Integer.MIN_VALUE, Long.MAX_VALUE);
+                }
+            }
+        }
+        else
+        {
+            logger.trace("Skipped upgrade task for {} because the limit {} of concurrent upgrade tasks has been reached",
+                         cfs, DatabaseDescriptor.maxConcurrentAutoUpgradeTasks());
+        }
+
+        ongoingUpgrades.decrementAndGet();
+        return null;
+    }
+
+    public static void handleCompactionError(Throwable t, ColumnFamilyStore cfs)
+    {
+        t = Throwables.unwrapped(t);
+
+        // FSDiskFullWriteErrors is thrown when checking disk space before starting flush or compaction task. They
+        // are expected to be recoverable because we haven't actually hit No-Space-Left error yet, so we don't explicitly
+        // trigger the disk failure policy because of them (see CASSANDRA-12385).
+        if (t instanceof IOError && !(t instanceof FSDiskFullWriteError))
+        {
+            logger.error("Potentially unrecoverable error during background compaction of table {}", cfs, t);
+            // Strictly speaking it's also possible to hit a read-related IOError during compaction, although the
+            // chances for that are much lower than the chances for write-related IOError. If we want to handle that,
+            // we might have to rely on error message parsing...
+            t = t instanceof FSError ? t : new FSWriteError(t);
+            JVMStabilityInspector.inspectThrowable(t);
+        }
+        // No-Space-Left IO exception is thrown by JDK when disk has reached its capacity. The key difference between this
+        // and the earlier case with `FSDiskFullWriteError` is that here we have definitively run out of disk space, and
+        // no further writes can be performed until disk space is freed, potentially leading to data corruption or
+        // system instability if not handled properly. We must trigger the disk failure policy.
+        else if (Throwables.isCausedBy(t, IOException.class) && t.toString().contains(NO_SPACE_LEFT_MESSAGE))
+        {
+            logger.error("Encountered no space left error on {}", cfs, t);
+            // wrap it with FSWriteError so that JVMStabilityInspector can properly stop or die
+            t = t instanceof FSError ? t : new FSWriteError(t);
+            JVMStabilityInspector.inspectThrowable(t);
+        }
+        else if (t instanceof CompactionInterruptedException)
+        {
+            logger.warn(String.format("Aborting background compaction of %s due to interruption", cfs), Throwables.unwrapped(t));
+        }
+        else
+        {
+            logger.error("Exception during background compaction of table {}", cfs, t);
+            CompactionManager.instance.incrementFailed();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/BackgroundCompactions.java b/src/java/org/apache/cassandra/db/compaction/BackgroundCompactions.java
new file mode 100644
index 000000000000..0ff616ee90d0
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/BackgroundCompactions.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.google.common.collect.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ExpMovingAverage;
+import org.apache.cassandra.utils.MovingAverage;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * A class for grouping the background compactions picked by a strategy, either pending or in progress.
+ *
+ * A compaction strategy has a {@link BackgroundCompactions} object as part of its state. Each
+ * {@link LegacyAbstractCompactionStrategy} instance has its {@link BackgroundCompactions}, and their lifespans are the
+ * same. In the case of {@link UnifiedCompactionStrategy} the new strategy instance inherits
+ * {@link BackgroundCompactions} from its predecessor.
+ */
+public class BackgroundCompactions
+{
+    private static final Logger logger = LoggerFactory.getLogger(BackgroundCompactions.class);
+
+    /** The table metadata */
+    private final TableMetadata metadata;
+
+    /** The compaction aggregates with either pending or ongoing compactions, or both. This is a private map
+     * whose access needs to be synchronized. */
+    private final TreeMap<CompactionAggregate.Key, CompactionAggregate> aggregatesMap;
+
+    /**
+     * The current list of compaction aggregates, this list must be recreated every time the aggregates
+     * map is changed.
+     *
+     * We publish aggregates to a separate variable instead of calling {@code aggregatesMap.values()} so that reads
+     * that race with updates always observe a consistent snapshot.
+     */
+    private volatile List<CompactionAggregate> aggregates;
+
+    /**  The ongoing compactions grouped by unique operation ID. */
+    private final ConcurrentHashMap<TimeUUID, CompactionPick> compactions = new ConcurrentHashMap<>();
+
+    /**
+     * Rate of progress (per thread) of recent compactions for the CFS. Used by the UnifiedCompactionStrategy to
+     * limit the number of running compactions to no more than what is sufficient to saturate the throughput limit.
+     * This needs to be a longer-running average to ensure that the rate limiter stalling a new thread can't cause
+     * the compaction rate to temporarily drop to levels that permit an extra thread.
+     */
+    MovingAverage compactionRate = ExpMovingAverage.decayBy1000();
+
+    BackgroundCompactions(CompactionRealm realm)
+    {
+        this.metadata = realm.metadata();
+        this.aggregatesMap = new TreeMap<>();
+        this.aggregates = ImmutableList.of();
+    }
+
+    /**
+     * Updates the list of pending compactions, while preserving the set of running ones. This is done
+     * by creating new aggregates with the pending aggregates but adding any existing aggregates with
+     * compactions in progress. If there is a matching pending aggregate then the existing compactions
+     * are transferred to it, otherwise the old aggregate is stripped of its pending compactios and then
+     * it is kept with the compactions in progress only.
+     *
+     * @param pending compaction aggregates with pending compactions
+     */
+    synchronized void setPending(CompactionStrategy strategy, Collection<? extends CompactionAggregate> pending)
+    {
+        if (pending == null)
+            throw new IllegalArgumentException("argument cannot be null");
+
+        if (logger.isTraceEnabled())
+            logger.trace("Resetting pending aggregates for strategy {}/{}, received {} new aggregates",
+                         strategy.getName(), strategy.hashCode(), pending.size());
+
+        // First remove the existing aggregates
+        aggregatesMap.clear();
+
+        // Then add all the pending aggregates
+        for (CompactionAggregate aggregate : pending)
+        {
+            CompactionAggregate prev = aggregatesMap.get(aggregate.getKey());
+            if (logger.isTraceEnabled())
+                logger.trace("Adding new pending aggregate: prev={}, current={}", prev, aggregate);
+
+            if (prev == null)
+                aggregatesMap.put(aggregate.getKey(), aggregate);
+            else
+                aggregatesMap.put(aggregate.getKey(), prev.mergeWith(aggregate));
+        }
+
+        // Then add the old aggregates but only if they have ongoing compactions
+        for (CompactionAggregate oldAggregate : this.aggregates)
+        {
+            Collection<CompactionPick> compacting = oldAggregate.getInProgress();
+            if (compacting.isEmpty())
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("Existing aggregate {} has no in progress compactions, removing it", oldAggregate);
+
+                continue;
+            }
+
+            // See if we have a matching aggregate in the pending aggregates, if so add all the existing compactions to it
+            // otherwise strip the pending and selected compactions from the old one and keep it only with the compactions in progress
+            CompactionAggregate newAggregate;
+            CompactionAggregate matchingAggregate = oldAggregate.getMatching(aggregatesMap);
+            if (matchingAggregate != null)
+            {
+                // add the old compactions to the new aggregate
+                // the key will change slightly for STCS so remove it before adding it again
+                aggregatesMap.remove(matchingAggregate.getKey());
+                newAggregate = matchingAggregate.withAdditionalCompactions(compacting);
+
+                if (logger.isTraceEnabled())
+                    logger.trace("Removed matching aggregate {}", matchingAggregate);
+            }
+            else
+            {
+                // keep the old aggregate but only with the compactions already in progress and not yet completed
+                newAggregate = oldAggregate.withOnlyTheseCompactions(compacting);
+
+                if (logger.isTraceEnabled())
+                    logger.trace("Keeping old aggregate but only with compactions {}", oldAggregate);
+            }
+
+            if (logger.isTraceEnabled())
+                logger.trace("Adding new aggregate with previous compactions {}", newAggregate);
+
+            aggregatesMap.put(newAggregate.getKey(), newAggregate);
+        }
+
+        // Publish the new aggregates
+        this.aggregates = ImmutableList.copyOf(aggregatesMap.values());
+
+        CompactionLogger compactionLogger = strategy.getCompactionLogger();
+        if (compactionLogger != null && compactionLogger.enabled())
+        {
+            // compactionLogger.statistics(strategy, "pending", getStatistics()); // too much noise
+            compactionLogger.pending(strategy, getEstimatedRemainingTasks());
+        }
+    }
+
+    void setSubmitted(CompactionStrategy strategy, TimeUUID id, CompactionAggregate aggregate)
+    {
+        if (id == null || aggregate == null)
+            throw new IllegalArgumentException("arguments cannot be null");
+
+        logger.debug("Submitting background compaction {}", id);
+        CompactionPick compaction = aggregate.getSelected();
+
+        CompactionPick prev = compactions.put(id, compaction);
+        if (prev != null)
+            throw new IllegalArgumentException("Found existing compaction with same id: " + id);
+
+        compaction.setSubmitted(id);
+
+        synchronized (this)
+        {
+            CompactionAggregate existingAggregate = aggregate.getMatching(aggregatesMap);
+            boolean aggregatesMapChanged = false;
+
+            if (existingAggregate == null)
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("Could not find aggregate for compaction using the one passed in: {}", aggregate);
+
+                aggregatesMapChanged = true;
+                aggregatesMap.put(aggregate.getKey(), aggregate);
+            }
+            else
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("Found aggregate for compaction: {}", existingAggregate);
+
+                Pair<Boolean, CompactionPick> contains = existingAggregate.containsSameInstance(compaction);
+                if (!contains.left)
+                {
+                    // add the compaction just submitted to the aggregate that was found if it doesn't already contain it
+                    // (the same exact instance that is because when we set the progress in compactions we ideally would like
+                    // the instance in the aggregates map to also be updated)
+                    // because for STCS the key may change slightly, first remove the existing aggregate, before re-inserting it
+                    aggregatesMapChanged = true;
+                    aggregatesMap.remove(existingAggregate.getKey());
+                    CompactionAggregate newAggregate = existingAggregate.withReplacedCompaction(compaction, contains.right);
+                    aggregatesMap.put(newAggregate.getKey(), newAggregate);
+
+                    if (logger.isTraceEnabled())
+                        logger.trace("Added compaction to existing aggregate: {} -> {}", existingAggregate, newAggregate);
+                }
+                else
+                {
+                    if (logger.isTraceEnabled())
+                        logger.trace("Existing aggregate {} already had compaction", existingAggregate);
+                }
+            }
+
+            // Publish the new aggregates if needed
+            if (aggregatesMapChanged)
+                this.aggregates = ImmutableList.copyOf(aggregatesMap.values());
+        }
+
+        CompactionLogger compactionLogger = strategy.getCompactionLogger();
+        if (compactionLogger != null && compactionLogger.enabled())
+            compactionLogger.statistics(strategy, "submitted", getStatistics(strategy));
+    }
+
+    public void onInProgress(CompactionProgress progress)
+    {
+        if (progress == null)
+            throw new IllegalArgumentException("argument cannot be null");
+
+        updateCompactionRate(progress);
+
+        TimeUUID id = progress.operationId();
+        CompactionPick compaction = compactions.computeIfAbsent(id,
+                                                                uuid ->
+                                                                CompactionPick.createWithUnknownParent(id,
+                                                                                                       progress.inSSTables()));
+
+        logger.debug("Setting background compaction {} as in progress", id);
+        compaction.setProgress(progress);
+    }
+
+    public void onCompleted(CompactionStrategy strategy, TimeUUID id)
+    {
+        if (id == null)
+            throw new IllegalArgumentException("argument cannot be null");
+
+        logger.debug("Removing compaction {}", id);
+
+        // log the statistics before completing the compaction so that we see the stats for the
+        // compaction that just completed
+        CompactionLogger compactionLogger = strategy.getCompactionLogger();
+        if (compactionLogger != null && compactionLogger.enabled())
+            compactionLogger.statistics(strategy, "completed", getStatistics(strategy));
+
+        CompactionPick completed = compactions.remove(id);
+        if (completed != null)
+        {
+            CompactionProgress progress = completed.progress();
+            updateCompactionRate(progress);
+            completed.setCompleted();
+        }
+
+        // We rely on setPending() to refresh the aggregates again even though in some cases it may not be
+        // called immediately (e.g. compactions disabled)
+    }
+
+    private void updateCompactionRate(CompactionProgress progress)
+    {
+        if (progress != null)
+        {
+            final long durationInMillis = progress.durationInMillis();
+            final long outputDiskSize = progress.outputDiskSize();
+            if (durationInMillis > 0 && outputDiskSize > 0)
+                compactionRate.update(outputDiskSize * 1.e3 / durationInMillis);
+        }
+    }
+
+    public Collection<CompactionAggregate> getAggregates()
+    {
+        return aggregates;
+    }
+
+    /**
+     * @return the number of background compactions estimated to still be needed
+     */
+    public int getEstimatedRemainingTasks()
+    {
+        return CompactionAggregate.numEstimatedCompactions(aggregates);
+    }
+
+    /**
+     * @return the compactions currently in progress
+     */
+    public Collection<CompactionPick> getCompactionsInProgress()
+    {
+        return Collections.unmodifiableCollection(compactions.values());
+    }
+
+    /**
+     * @return the total number of background compactions, pending or in progress
+     */
+    public int getTotalCompactions()
+    {
+        return compactions.size() + getEstimatedRemainingTasks();
+    }
+
+    /**
+     * Return the compaction statistics for this strategy.
+     *
+     * @return statistics about this compaction strategy.
+     */
+    public CompactionStrategyStatistics getStatistics(CompactionStrategy strategy)
+    {
+        return CompactionAggregate.getStatistics(metadata, strategy, aggregates);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CleanupTask.java b/src/java/org/apache/cassandra/db/compaction/CleanupTask.java
new file mode 100644
index 000000000000..2076d628bbed
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CleanupTask.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.repair.consistent.admin.CleanupSummary;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TimeUUID;
+
+public class CleanupTask
+{
+    private static final Logger logger = LoggerFactory.getLogger(CleanupTask.class);
+
+    private final CompactionRealm realm;
+    private final List<Pair<TimeUUID, RepairFinishedCompactionTask>> tasks;
+
+    public CleanupTask(CompactionRealm realm, List<Pair<TimeUUID, RepairFinishedCompactionTask>> tasks)
+    {
+        this.realm = realm;
+        this.tasks = tasks;
+    }
+
+    public CleanupSummary cleanup()
+    {
+        Set<TimeUUID> successful = new HashSet<>();
+        Set<TimeUUID> unsuccessful = new HashSet<>();
+        for (Pair<TimeUUID, RepairFinishedCompactionTask> pair : tasks)
+        {
+            TimeUUID session = pair.left;
+            RepairFinishedCompactionTask task = pair.right;
+
+            if (task != null)
+            {
+                try
+                {
+                    task.run();
+                    successful.add(session);
+                }
+                catch (Throwable t)
+                {
+                    t = task.transaction.abort(t);
+                    logger.error("Failed cleaning up " + session, t);
+                    unsuccessful.add(session);
+                }
+            }
+            else
+            {
+                unsuccessful.add(session);
+            }
+        }
+        return new CleanupSummary(realm, successful, unsuccessful);
+    }
+
+    public Throwable abort(Throwable accumulate)
+    {
+        for (Pair<TimeUUID, RepairFinishedCompactionTask> pair : tasks)
+            accumulate = pair.right.transaction.abort(accumulate);
+        return accumulate;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionAggregate.java b/src/java/org/apache/cassandra/db/compaction/CompactionAggregate.java
new file mode 100644
index 000000000000..82bca520f81e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionAggregate.java
@@ -0,0 +1,1099 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * A compaction aggregate is either a level in {@link LeveledCompactionStrategy} or a tier (bucket) in other
+ * compaction strategies.
+ * <p/>
+ * It contains a list of {@link CompactionPick}, which are the compactions either in progress or pending.
+ * It also contains a selected {@link CompactionPick}, which is a compaction about to be submitted. The submitted
+ * compaction is also part of the compactions. Lastly, it contains a set of all the sstables in this aggregate,
+ * regardless of whether they need compaction.
+ */
+public abstract class CompactionAggregate
+{
+    private static final Logger logger = LoggerFactory.getLogger(CompactionAggregate.class);
+
+    /** The unique key that identifies this aggregate. */
+    final Key key;
+
+    /** The sstables in this aggregate, whether they are compaction candidates or not */
+    final Set<CompactionSSTable> sstables;
+
+    /** The compaction that was selected for this aggregate when it was created. It is also part of {@link this#compactions}. */
+    final CompactionPick selected;
+
+    /** The compactions that are part of this aggregate, they could be pending or in progress. */
+    final LinkedHashSet<CompactionPick> compactions;
+
+    CompactionAggregate(Key key, Iterable<? extends CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> pending)
+    {
+        if (sstables == null || selected == null || pending == null)
+            throw new IllegalArgumentException("Arguments cannot be null");
+
+        this.key = key;
+        this.sstables = new HashSet<>(); sstables.forEach(this.sstables::add);
+        this.selected = selected;
+
+        // Here we want to keep the iteration order since normally pending compactions are ordered by a strategy
+        // and the selected compaction should be the first one
+        this.compactions = new LinkedHashSet<>();
+        if (!selected.isEmpty())
+            compactions.add(selected);
+
+        for (CompactionPick p : pending)
+        {
+            if (p == null || p.isEmpty())
+                throw new IllegalArgumentException("Pending compactions should be valid compactions");
+
+            compactions.add(p);
+        }
+    }
+
+    public CompactionPick getSelected()
+    {
+        return selected;
+    }
+
+    /**
+     * @return the total sstable size for all the compaction picks that are either pending or still in progress
+     */
+    public long getPendingBytes()
+    {
+        long ret = 0;
+        for (CompactionPick comp : compactions)
+        {
+            if (!comp.submitted())
+                ret += comp.totSizeInBytes();
+        }
+        return ret;
+    }
+
+    /**
+     * @return compactions that have not yet been submitted (no compaction id).
+     */
+    public List<CompactionPick> getPending()
+    {
+        List<CompactionPick> ret = new ArrayList<>(compactions.size());
+        for (CompactionPick comp : compactions)
+        {
+            if (!comp.submitted())
+                ret.add(comp);
+        }
+
+        return ret;
+    }
+
+    /**
+     * @return compactions that have already been submitted (compaction id is available) and haven't completed yet
+     */
+    public List<CompactionPick> getInProgress()
+    {
+        List<CompactionPick> ret = new ArrayList<>(compactions.size());
+        for (CompactionPick comp : compactions)
+        {
+            if (comp.submitted() && !comp.completed())
+                ret.add(comp);
+        }
+
+        return ret;
+    }
+
+    /**
+     * @return all the compactions we have
+     */
+    public List<CompactionPick> getActive()
+    {
+        return new ArrayList<>(compactions);
+    }
+
+    /**
+     * @return true if this aggregate has no compactions
+     */
+    public boolean isEmpty()
+    {
+        return compactions.isEmpty();
+    }
+
+    /**
+     * Merge the pending compactions and the compactions in progress to create some aggregated statistics.
+     *
+     * @return the statistics for this compaction aggregate, see {@link CompactionAggregateStatistics}.
+     */
+    public abstract CompactionAggregateStatistics getStatistics();
+
+    /**
+     * Calculates basic compaction statistics, common for all types of {@link CompactionAggregate}s.
+     *
+     * @param trackHotness Indicates whether aggregate (tier/bucket) hotness is relevant and should be calculated.
+     *                     If this is {@code false}, a default value of {@link Double#NaN} will be used to indicate
+     *                     that hotness hasn't been calculated.
+     *
+     * @return a new {@link CompactionAggregateStatistics} instance, containing all the common statistics for the
+     *         different types of {@link CompactionAggregate}s (see above for the caveat about hotness).
+     */
+    CompactionAggregateStatistics getCommonStatistics(boolean trackHotness)
+    {
+        int numCompactions = 0;
+        int numCompactionsInProgress = 0;
+        int numCandidateSSTables = 0;
+        int numCompactingSSTables = 0;
+        int numExpiredSSTables = 0;
+        long tot = 0;
+        long expiredTot = 0;
+        double hotness = trackHotness ? 0.0 : Double.NaN;
+        long read = 0;
+        long written = 0;
+        double readThroughput = 0;
+        double writeThroughput = 0;
+
+        for (CompactionPick compaction : compactions)
+        {
+            if (compaction.completed())
+                continue;
+
+            numCompactions++;
+            numCandidateSSTables += compaction.sstables().size();
+            numExpiredSSTables += compaction.expired().size();
+            tot += compaction.sstables().stream().mapToLong(CompactionSSTable::uncompressedLength).reduce(0L, Long::sum);
+            expiredTot += compaction.expired().stream().mapToLong(CompactionSSTable::uncompressedLength).reduce(0L, Long::sum);
+            if (trackHotness)
+                hotness += compaction.hotness();
+
+            if (compaction.submitted())
+            {
+                numCompactionsInProgress++;
+                numCompactingSSTables += compaction.sstables().size();
+            }
+
+            if (compaction.inProgress())
+            {
+                final CompactionProgress progress = compaction.progress();
+                read += progress.uncompressedBytesRead();
+                written += progress.uncompressedBytesWritten();
+                readThroughput += progress.readThroughput();
+                writeThroughput += progress.writeThroughput();
+            }
+        }
+
+        return new CompactionAggregateStatistics(numCompactions,
+                                                 numCompactionsInProgress,
+                                                 sstables.size(),
+                                                 numExpiredSSTables,
+                                                 numCandidateSSTables,
+                                                 numCompactingSSTables,
+                                                 getTotSizeBytes(sstables),
+                                                 tot,
+                                                 expiredTot,
+                                                 read,
+                                                 written,
+                                                 readThroughput,
+                                                 writeThroughput,
+                                                 hotness);
+    }
+
+    /**
+     * @return the number of estimated compactions that are still pending.
+     */
+    public int numEstimatedCompactions()
+    {
+        return getPending().size();
+    }
+
+    /**
+     * @return a key that ensures the uniqueness of an aggregate but also that allows identify future identical aggregates,
+     *         e.g. when an aggregate is merged with an older aggregate that has still ongoing compactions like a level
+     *         in LCS or a bucket in the unified strategy or STCS or a time window in TWCS
+     */
+    public Key getKey()
+    {
+        return key;
+    }
+
+    /**
+     * Return a matching aggregate from the map passed in or null. Normally this is just a matter of finding
+     * the key in the map but for STCS we need to look at the possible min and maximum average sizes and so
+     * {@link SizeTiered} overrides this method.
+     *
+     * @param others a map of other aggregates
+     *
+     * @return an aggregate with the same key or null
+     */
+    @Nullable CompactionAggregate getMatching(NavigableMap<Key, CompactionAggregate> others)
+    {
+        return others.get(getKey());
+    }
+
+    /**
+     * Create a copy of this aggregate with the new parameters
+     *
+     * @return a deep copy of this aggregate
+     */
+    protected abstract CompactionAggregate clone(Iterable<CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> compactions);
+
+    /**
+     * Add expired sstables to the selected compaction pick and return a new compaction aggregate.
+     */
+    CompactionAggregate withExpired(Collection<CompactionSSTable> expired)
+    {
+       return clone(Iterables.concat(sstables, expired), selected.withExpiredSSTables(expired), compactions);
+    }
+
+    /**
+     * Check if this aggregate compactions contain the compaction passed in. Here we're looking for
+     * the exact same instance, not just a compaction that is equal to it.
+     *
+     * @param compaction the compaction to check if it can be found
+     *
+     * @return a pair containing the result on the left (true if the compaction is found, false otherwise), and
+     * a matching compaction on the right (any compaction that is equal, including the same instance).
+     */
+    public Pair<Boolean, CompactionPick> containsSameInstance(CompactionPick compaction)
+    {
+        List<CompactionPick> activeCompactions = getActive();
+        int existingCompactionIdx = activeCompactions.indexOf(compaction);
+        CompactionPick existingCompaction = existingCompactionIdx == -1 ? null : activeCompactions.get(existingCompactionIdx);
+        boolean containsSameInstance = existingCompaction != null && existingCompaction == compaction;
+        return Pair.create(containsSameInstance, existingCompaction);
+    }
+    
+    /**
+     * Replace an existing compaction pick with a new one, this is used by CNDB because it creates new
+     * compactions from etcd. If the existing compaction is null, simply add the replacement.
+     */
+    public CompactionAggregate withReplacedCompaction(CompactionPick replacement, @Nullable CompactionPick existing)
+    {
+        Preconditions.checkArgument(existing == null || this.compactions.contains(existing), "Expected existing to be part of compactions");
+        if (existing == null)
+            return withAdditionalCompactions(ImmutableList.of(replacement));
+
+        List<CompactionSSTable> sstables = new ArrayList<>(this.sstables.size());
+        LinkedHashSet<CompactionPick> compactions = new LinkedHashSet<>(this.compactions.size());
+        for (CompactionPick comp : this.compactions)
+        {
+            if (comp == existing)
+            {
+                compactions.add(replacement);
+                sstables.addAll(replacement.sstables());
+            }
+            else
+            {
+                compactions.add(comp);
+                sstables.addAll(comp.sstables());
+            }
+        }
+
+        return clone(sstables, existing == selected ? replacement : selected, compactions);
+    }
+
+    /**
+     * Add existing compactions to our own compactions and return a new compaction aggregate
+     */
+    public CompactionAggregate withAdditionalCompactions(Collection<CompactionPick> comps)
+    {
+        List<CompactionSSTable> added = comps.stream().flatMap(comp -> comp.sstables().stream()).collect(Collectors.toList());
+        return clone(Iterables.concat(sstables, added), selected, Iterables.concat(compactions, comps));
+    }
+
+    /**
+     * Only keep the compactions passed in, strip everything else.
+     */
+    public CompactionAggregate withOnlyTheseCompactions(Collection<CompactionPick> comps)
+    {
+        List<CompactionSSTable> retained = comps.stream().flatMap(comp -> comp.sstables().stream()).collect(Collectors.toList());
+        return clone(retained, CompactionPick.EMPTY, comps);
+    }
+
+    /**
+     * Merge an aggregate with another one with the same key.
+     */
+    protected CompactionAggregate mergeWith(CompactionAggregate other)
+    {
+        return withAdditionalCompactions(other.compactions);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(sstables, selected, compactions);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj == this)
+            return true;
+
+        if (!(obj instanceof CompactionAggregate))
+            return false;
+
+        CompactionAggregate that = (CompactionAggregate) obj;
+        return sstables.equals(that.sstables) &&
+               selected.equals(that.selected) &&
+               compactions.equals(that.compactions);
+    }
+
+    /**
+     * Contains information about a levelled compaction aggregate, this is equivalent to a level in {@link LeveledCompactionStrategy}.
+     */
+    public static final class Leveled extends CompactionAggregate
+    {
+        /** The current level number */
+        final int level;
+
+        /** The next level number */
+        final int nextLevel;
+
+        /** The score of this level as defined in {@link LeveledCompactionStrategy}. */
+        final double score;
+
+        /** The maximum size of each output sstable that will be produced by compaction, Long.MAX_VALUE if no maximum exists */
+        final long maxSSTableBytes;
+
+        /**
+         * How many more compactions this level is expected to perform. This is required because for LCS we cannot
+         * easily identify candidate sstables to put into the pending picks.
+         */
+        final int pendingCompactions;
+
+        /** The fanout size */
+        final int fanout;
+
+        Leveled(Iterable<? extends CompactionSSTable> sstables,
+                CompactionPick selected,
+                Iterable<CompactionPick> compactions,
+                int level,
+                int nextLevel,
+                double score,
+                long maxSSTableBytes,
+                int pendingCompactions,
+                int fanout)
+        {
+            super(new Key(level), sstables, selected, compactions);
+
+            this.level = level;
+            this.nextLevel = nextLevel;
+            this.score = score;
+            this.maxSSTableBytes = maxSSTableBytes;
+            this.pendingCompactions = pendingCompactions;
+            this.fanout = fanout;
+        }
+
+        @Override
+        protected CompactionAggregate clone(Iterable<CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> compactions)
+        {
+            return new Leveled(sstables, selected, compactions, level, nextLevel, score, maxSSTableBytes, pendingCompactions, fanout);
+        }
+
+        @Override
+        public CompactionAggregateStatistics getStatistics()
+        {
+            CompactionAggregateStatistics stats = getCommonStatistics(false);
+
+            long readLevel = 0L;
+
+            for (CompactionPick compaction : compactions)
+                if (!compaction.completed() && compaction.inProgress())
+                    readLevel += compaction.progress().uncompressedBytesRead(level);
+
+            return new LeveledCompactionStatistics(stats, level, score, pendingCompactions, readLevel);
+        }
+
+        @Override
+        public int numEstimatedCompactions()
+        {
+            return pendingCompactions;
+        }
+
+        @Override
+        public boolean isEmpty()
+        {
+            return super.isEmpty() && pendingCompactions == 0;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("Level %d with %d sstables, %d compactions and %d pending", level, sstables.size(), compactions.size(), pendingCompactions);
+        }
+    }
+
+    /**
+     * Create a level where we have a compaction candidate.
+     */
+    static CompactionAggregate.Leveled createLeveled(Collection<? extends CompactionSSTable> all,
+                                                     Collection<? extends CompactionSSTable> candidates,
+                                                     int pendingCompactions,
+                                                     long maxSSTableBytes,
+                                                     int level,
+                                                     int nextLevel,
+                                                     double score,
+                                                     int fanout)
+    {
+        return new Leveled(all,
+                           CompactionPick.create(level, candidates),
+                           ImmutableList.of(),
+                           level,
+                           nextLevel,
+                           score,
+                           maxSSTableBytes,
+                           pendingCompactions,
+                           fanout);
+    }
+
+    /**
+     * Create a level when we only have estimated tasks.
+     */
+    static CompactionAggregate.Leveled createLeveled(Collection<? extends CompactionSSTable> all,
+                                                     int pendingCompactions,
+                                                     long maxSSTableBytes,
+                                                     int level,
+                                                     double score,
+                                                     int fanout)
+    {
+        return new Leveled(all,
+                           CompactionPick.EMPTY,
+                           ImmutableList.of(),
+                           level,
+                           level + 1,
+                           score,
+                           maxSSTableBytes,
+                           pendingCompactions,
+                           fanout);
+    }
+
+    /**
+     * Create a leveled aggregate when LCS is doing STCS on level 0
+     */
+    static CompactionAggregate.Leveled createLeveledForSTCS(Collection<? extends CompactionSSTable> all,
+                                                            CompactionPick pick,
+                                                            int pendingCompactions,
+                                                            double score,
+                                                            int fanout)
+    {
+        return new Leveled(all,
+                           pick,
+                           ImmutableList.of(),
+                           0,
+                           0,
+                           score,
+                           Long.MAX_VALUE,
+                           pendingCompactions,
+                           fanout);
+    }
+
+    /**
+     * Contains information about a size-tiered compaction aggregate, this is equivalent to a bucket in {@link SizeTieredCompactionStrategy}.
+     */
+    public static final class SizeTiered extends CompactionAggregate
+    {
+        /** The total read hotness of the sstables in this tier, as defined by {@link CompactionSSTable#hotness()} */
+        final double hotness;
+
+        /** The average on disk size in bytes of the sstables in this tier */
+        final long avgSizeBytes;
+
+        /** The minimum on disk size in bytes for this tier, this is normally the avg size times the STCS bucket low and it is
+         * used to find compacting aggregates that are on the same tier. */
+        final long minSizeBytes;
+
+        /** The maximum on disk size in bytes for this tier, this is normally the avg size times the STCS bucket high and it is
+         * used to find compacting aggregates that are on the same tier. */
+        final long maxSizeBytes;
+
+        SizeTiered(Iterable<? extends CompactionSSTable> sstables,
+                   CompactionPick selected,
+                   Iterable<CompactionPick> pending,
+                   double hotness,
+                   long avgSizeBytes,
+                   long minSizeBytes,
+                   long maxSizeBytes)
+        {
+            super(new Key(avgSizeBytes), sstables, selected, pending);
+
+            this.hotness = hotness;
+            this.avgSizeBytes = avgSizeBytes;
+            this.minSizeBytes = minSizeBytes;
+            this.maxSizeBytes = maxSizeBytes;
+        }
+
+        @Override
+        protected CompactionAggregate clone(Iterable<CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> compactions)
+        {
+            return new SizeTiered(sstables, selected, compactions, getTotHotness(sstables), getAvgSizeBytes(sstables), minSizeBytes, maxSizeBytes);
+        }
+
+        @Override
+        public CompactionAggregateStatistics getStatistics()
+        {
+            CompactionAggregateStatistics stats = getCommonStatistics(true);
+
+            return new SizeTieredCompactionStatistics(stats, avgSizeBytes);
+        }
+
+        @Override
+        @Nullable CompactionAggregate getMatching(NavigableMap<Key, CompactionAggregate> others)
+        {
+            SortedMap<Key, CompactionAggregate> subMap = others.subMap(new Key(minSizeBytes), new Key(maxSizeBytes));
+            if (subMap.isEmpty())
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("Found no matching aggregate for {}",
+                                 FBUtilities.prettyPrintMemory(avgSizeBytes));
+
+                return null;
+            }
+
+            if (logger.isTraceEnabled())
+                logger.trace("Found {} matching aggregates for {}",
+                             subMap.size(),
+                             FBUtilities.prettyPrintMemory(avgSizeBytes));
+
+            Key closest = null;
+            long minDiff = 0;
+            for (Key m : subMap.keySet())
+            {
+                long diff = Math.abs(m.index - avgSizeBytes);
+                if (closest == null || diff < minDiff)
+                {
+                    closest = m;
+                    minDiff = diff;
+                }
+            }
+
+            if (logger.isTraceEnabled())
+                logger.trace("Using closest matching aggregate for {}: {}",
+                             FBUtilities.prettyPrintMemory(avgSizeBytes),
+                             FBUtilities.prettyPrintMemory(closest != null ? closest.index : -1));
+
+            return others.get(closest);
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("Size tiered %s/%s/%s with %d sstables, %d compactions",
+                                 FBUtilities.prettyPrintMemory(minSizeBytes),
+                                 FBUtilities.prettyPrintMemory(avgSizeBytes),
+                                 FBUtilities.prettyPrintMemory(maxSizeBytes),
+                                 sstables.size(),
+                                 compactions.size());
+        }
+    }
+
+    static CompactionAggregate createSizeTiered(Collection<? extends CompactionSSTable> all,
+                                                CompactionPick selected,
+                                                List<CompactionPick> pending,
+                                                double hotness,
+                                                long avgSizeBytes,
+                                                long minSizeBytes,
+                                                long maxSizeBytes)
+    {
+        return new SizeTiered(all, selected, pending, hotness, avgSizeBytes, minSizeBytes, maxSizeBytes);
+    }
+
+    /**
+     * Contains information about a size-tiered compaction aggregate, this is equivalent to a bucket in {@link SizeTieredCompactionStrategy}.
+     */
+    public static final class TimeTiered extends CompactionAggregate
+    {
+        /** The timestamp of this aggregate */
+        final long timestamp;
+
+        TimeTiered(Iterable<CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> pending, long timestamp)
+        {
+            super(new Key(timestamp), sstables, selected, pending);
+            this.timestamp = timestamp;
+        }
+
+        @Override
+        protected CompactionAggregate clone(Iterable<CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> compactions)
+        {
+            return new TimeTiered(sstables, selected, compactions, timestamp);
+        }
+
+        @Override
+        public CompactionAggregateStatistics getStatistics()
+        {
+            CompactionAggregateStatistics stats = getCommonStatistics(true);
+            return new TimeTieredCompactionStatistics(stats, timestamp);
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("Time tiered %d with %d sstables, %d compactions", timestamp, sstables.size(), compactions.size());
+        }
+    }
+
+    static CompactionAggregate createTimeTiered(Collection<CompactionSSTable> sstables, long timestamp)
+    {
+        return new TimeTiered(sstables, CompactionPick.create(timestamp, sstables), ImmutableList.of(), timestamp);
+    }
+
+    static CompactionAggregate createTimeTiered(Collection<CompactionSSTable> sstables, CompactionPick selected, List<CompactionPick> pending, long timestamp)
+    {
+        return new TimeTiered(sstables, selected, pending, timestamp);
+    }
+
+    public static class UnifiedAggregate extends CompactionAggregate
+    {
+        /** The arena to which this level belongs */
+        private final UnifiedCompactionStrategy.Arena arena;
+
+        /** The level generated by the compaction strategy */
+        private final UnifiedCompactionStrategy.Level level;
+
+        private UnifiedCompactionStrategy.ShardingStats shardingStats;
+
+        /** The maximum number of overlapping sstables in the level. */
+        private final int maxOverlap;
+
+        private int permittedParallelism;
+
+        UnifiedAggregate(Iterable<? extends CompactionSSTable> sstables,
+                         int maxOverlap,
+                         CompactionPick selected,
+                         Iterable<CompactionPick> pending,
+                         UnifiedCompactionStrategy.Arena arena,
+                         UnifiedCompactionStrategy.Level level)
+        {
+            super(new ArenaedKey(arena, level.index), sstables, selected, pending);
+            this.maxOverlap = maxOverlap;
+            this.arena = arena;
+            this.level = level;
+        }
+
+        public UnifiedCompactionStrategy.Arena getArena()
+        {
+            return arena;
+        }
+
+        public void setShardingStats(UnifiedCompactionStrategy.ShardingStats shardingStats)
+        {
+            assert this.shardingStats == null;
+            this.shardingStats = shardingStats;
+        }
+
+        public UnifiedCompactionStrategy.ShardingStats getShardingStats()
+        {
+            return shardingStats;
+        }
+
+        @Override
+        public CompactionAggregateStatistics getStatistics()
+        {
+            CompactionAggregateStatistics stats = getCommonStatistics(false);
+
+            return new UnifiedCompactionStatistics(stats,
+                                                   level.index,
+                                                   level.survivalFactor,
+                                                   level.scalingParameter,
+                                                   level.min,
+                                                   level.max,
+                                                   maxOverlap,
+                                                   arena.name());
+        }
+
+        @Override
+        protected CompactionAggregate clone(Iterable<CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> compactions)
+        {
+            return new UnifiedAggregate(sstables, maxOverlap, selected, compactions, arena, level);
+        }
+
+        @Override
+        protected CompactionAggregate mergeWith(CompactionAggregate other)
+        {
+            return new UnifiedAggregate(Iterables.concat(sstables, other.sstables),
+                                        Math.max(maxOverlap, ((UnifiedAggregate) other).maxOverlap),
+                                        selected,
+                                        Iterables.concat(compactions, other.compactions),
+                                        arena,
+                                        level);
+        }
+
+        public int bucketIndex()
+        {
+            return level.index;
+        }
+
+        // used by CNDB, "bucket" name is historical
+        public double bucketMin()
+        {
+            return level.min;
+        }
+
+        // used by CNDB, "bucket" name is historical
+        public double bucketMax()
+        {
+            return level.max;
+        }
+
+        public int maxOverlap()
+        {
+            return maxOverlap;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("Unified arena %s level %d with %d sstables (max overlap %d) and %d compactions",
+                                 arena.name(),
+                                 level.index,
+                                 sstables.size(),
+                                 maxOverlap,
+                                 compactions.size());
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (obj == this)
+                return true;
+
+            if (!(obj instanceof UnifiedAggregate))
+                return false;
+
+            UnifiedAggregate that = (UnifiedAggregate) obj;
+            return sstables.equals(that.sstables) &&
+                   selected.equals(that.selected) &&
+                   compactions.equals(that.compactions) &&
+                   level.equals(that.level) &&
+                   arena.equals(that.arena);
+            // no need to compare maxOverlap, that's a feature of sstables
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(sstables, selected, compactions, level, arena);
+        }
+
+        public Range<Token> operationRange()
+        {
+            return null;
+        }
+
+        public boolean keepOriginals()
+        {
+            return false;
+        }
+
+        public void setPermittedParallelism(int parallelism)
+        {
+            this.permittedParallelism = parallelism;
+        }
+
+        public int getPermittedParallelism()
+        {
+            return permittedParallelism;
+        }
+    }
+
+    /**
+     * A unified compaction aggregate for compaction over a specified subrange of the given sources. This would be a
+     * part of a larger composite transaction over the same inputs sstables, thus a ranged aggregate's tasks cannot
+     * delete any of the input sstables, which needs to be done in addition to the execution of this aggregate.
+     * The intended use of this is to parallelize compactions over multiple nodes in CNDB.
+     * See RangedAggregatesTest for an example of how this would be used.
+     */
+    public static class UnifiedWithRange extends UnifiedAggregate
+    {
+        private final Range<Token> operationRange;
+
+        UnifiedWithRange(Iterable<? extends CompactionSSTable> sstables,
+                         int maxOverlap,
+                         CompactionPick selected,
+                         Iterable<CompactionPick> pending,
+                         UnifiedCompactionStrategy.Arena arena,
+                         UnifiedCompactionStrategy.Level level,
+                         int permittedParallelism,
+                         Range<Token> operationRange)
+        {
+            super(sstables, maxOverlap, selected, pending, arena, level);
+            this.operationRange = operationRange;
+            setPermittedParallelism(permittedParallelism);
+        }
+
+        @Override
+        public Range<Token> operationRange()
+        {
+            return operationRange;
+        }
+
+        @Override
+        public boolean keepOriginals()
+        {
+            return true; // if an aggregate is partial, the sources cannot be deleted as they are needed for the other parts
+        }
+
+        @Override
+        public String toString()
+        {
+            return super.toString() + " range " + operationRange;
+        }
+    }
+
+    public static UnifiedAggregate createUnified(Collection<? extends CompactionSSTable> sstables,
+                                                 int maxOverlap,
+                                                 CompactionPick selected,
+                                                 Iterable<CompactionPick> pending,
+                                                 UnifiedCompactionStrategy.Arena arena,
+                                                 UnifiedCompactionStrategy.Level level)
+    {
+        return new UnifiedAggregate(sstables, maxOverlap, selected, pending, arena, level);
+    }
+
+    /**
+     * Create a ranged portion of the specified aggregate. To be used by CNDB to split compaction over nodes.
+     */
+    public static UnifiedAggregate createUnifiedWithRange(UnifiedAggregate base,
+                                                          Collection<? extends CompactionSSTable> rangeSSTables,
+                                                          Range<Token> range,
+                                                          int permittedParallelism)
+    {
+        return new UnifiedWithRange(rangeSSTables,
+                                    base.maxOverlap,
+                                    CompactionPick.create(base.bucketIndex(), rangeSSTables),
+                                    Collections.emptySet(),
+                                    base.arena,
+                                    base.level,
+                                    permittedParallelism,
+                                    range);
+    }
+
+
+
+    /** An aggregate that is created for a compaction issued only to drop tombstones */
+    public static final class TombstoneAggregate extends CompactionAggregate
+    {
+        TombstoneAggregate(Iterable<CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> pending)
+        {
+            super(new Key(-1), sstables, selected, pending);
+        }
+
+        @Override
+        protected CompactionAggregate clone(Iterable<CompactionSSTable> sstables, CompactionPick selected, Iterable<CompactionPick> compactions)
+        {
+            return new TombstoneAggregate(sstables, selected, compactions);
+        }
+
+        @Override
+        public CompactionAggregateStatistics getStatistics()
+        {
+            return getCommonStatistics(false);
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("Tombstones with %d sstables, %d compactions", sstables.size(), compactions.size());
+        }
+    }
+
+    static CompactionAggregate createForTombstones(CompactionSSTable sstable)
+    {
+        List<CompactionSSTable> sstables = ImmutableList.of(sstable);
+        CompactionPick comp = CompactionPick.create(-1, sstables);
+        return new TombstoneAggregate(sstables, comp, ImmutableList.of());
+    }
+
+    /**
+     * A key suitable for a strategy that has no arenas, that is a legacy strategy that is
+     * managed by CompactionStrategyManager.
+     */
+    public static class Key implements Comparable<Key>
+    {
+        protected final long index;
+
+        Key(long index)
+        {
+            this.index = index;
+        }
+
+        @Override
+        public int compareTo(Key key)
+        {
+            return Long.compare(index, key.index);
+        }
+
+        @Override
+        public String toString()
+        {
+            return Long.toString(index);
+        }
+    }
+
+    /**
+     * A key suitable for a strategy using arenas, first it compares by arena, and then by level index.
+     */
+    private static final class ArenaedKey extends Key
+    {
+        private final UnifiedCompactionStrategy.Arena arena;
+
+        ArenaedKey(UnifiedCompactionStrategy.Arena arena, long index)
+        {
+            super(index);
+            this.arena = arena;
+        }
+
+        @Override
+        public int compareTo(Key key)
+        {
+            if (key instanceof ArenaedKey)
+            {
+                ArenaedKey arenaedKey = (ArenaedKey) key;
+
+                int ret = arena.compareTo(arenaedKey.arena);
+                if (ret != 0)
+                    return ret;
+            }
+
+            // either not arenaed or same arena
+            return Long.compare(index, key.index);
+        }
+
+        @Override
+        public String toString()
+        {
+            return index + "-" + arena;
+        }
+    }
+
+    /**
+     * Return the compaction statistics for this strategy and list of compactions that are either pending or in progress.
+     *
+     * @param aggregates the compaction aggregates
+     *
+     * @return the statistics about this compactions
+     */
+    static CompactionStrategyStatistics getStatistics(TableMetadata metadata,
+                                                      CompactionStrategy strategy,
+                                                      Collection<CompactionAggregate> aggregates)
+    {
+        List<CompactionAggregateStatistics> statistics = new ArrayList<>(aggregates.size());
+
+        for (CompactionAggregate aggregate : aggregates)
+            statistics.add(aggregate.getStatistics());
+
+        return new CompactionStrategyStatistics(metadata, strategy.getClass().getSimpleName(), statistics);
+    }
+
+    /**
+     * Return the number of compactions that are still pending;
+     * @param aggregates the compaction aggregates
+     *
+     * @return the number of compactions that are still pending (net yet submitted)
+     */
+    static int numEstimatedCompactions(Collection<CompactionAggregate> aggregates)
+    {
+        int ret = 0;
+        for (CompactionAggregate aggregate : aggregates)
+            ret += aggregate.numEstimatedCompactions();
+
+        return ret;
+    }
+
+    /**
+     * Given a sorted list of compactions, return the first selected pick.
+     *
+     * @param aggregates a sorted list of compaction aggregates from most interesting to least interesting, some may be empty
+     *
+     * @return the compaction pick of the first aggregate
+     */
+    static CompactionPick getSelected(List<CompactionAggregate> aggregates)
+    {
+        return aggregates.isEmpty() ? CompactionPick.EMPTY : aggregates.get(0).getSelected();
+    }
+
+    /**
+     * Given a list of sstables, return their average size on disk.
+     *
+     * @param sstables the sstables
+     * @return average sstable size on disk or zero.
+     */
+    static long getAvgSizeBytes(Iterable<? extends CompactionSSTable> sstables)
+    {
+        long ret = 0;
+        long num = 0;
+        for (CompactionSSTable sstable : sstables)
+        {
+            ret += sstable.onDiskLength();
+            num++;
+        }
+
+        return num > 0 ? ret / num : 0;
+    }
+
+    /**
+     * Given a list of sstables, return their total size on disk.
+     *
+     * @param sstables the sstables
+     * @return total sstable size on disk or zero.
+     */
+    static long getTotSizeBytes(Iterable<? extends CompactionSSTable> sstables)
+    {
+        long ret = 0;
+        for (CompactionSSTable sstable : sstables)
+            ret += sstable.onDiskLength();
+
+        return ret;
+    }
+
+    /**
+     * Given a list of sstables, return their total read hotness.
+     *
+     * @param sstables the sstables
+     * @return total read hotness or zero.
+     */
+    static double getTotHotness(Iterable<? extends CompactionSSTable> sstables)
+    {
+        double ret = 0;
+        for (CompactionSSTable sstable : sstables)
+            ret += sstable.hotness();
+
+        return ret;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionAggregateStatistics.java b/src/java/org/apache/cassandra/db/compaction/CompactionAggregateStatistics.java
new file mode 100644
index 000000000000..35ca5d89873f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionAggregateStatistics.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+import com.google.common.collect.ImmutableList;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemoryPerSecond;
+
+/**
+ * The statistics for a {@link CompactionAggregate}.
+ * <p/>
+ * It must be serializable for JMX and convertible to JSON for insights. The JSON
+ * properties are published to insights so changing them has a downstream impact.
+ */
+public class CompactionAggregateStatistics implements Serializable
+{
+    public static final String NO_SHARD = "";
+
+    protected static final Collection<String> HEADER = ImmutableList.of("Tot. SSTables",
+                                                                        "Tot. size (bytes)",
+                                                                        "Compactions",
+                                                                        "Comp. SSTables",
+                                                                        "Read (bytes/sec)",
+                                                                        "Write (bytes/sec)",
+                                                                        "Tot. comp. size/Read/Written (bytes)");
+    /** The number of compactions that are either pending or in progress */
+    protected final int numCompactions;
+
+    /** The number of compactions that are in progress */
+    protected final int numCompactionsInProgress;
+
+    /** The total number of sstables, whether they need compacting or not */
+    protected final int numSSTables;
+
+    /** The total number of expired sstables */
+    protected final int numExpiredSSTables;
+
+    /** The number of sstables that are compaction candidates */
+    protected final int numCandidateSSTables;
+
+    /** The number of sstables that are currently compacting */
+    protected final int numCompactingSSTables;
+
+    /** The size in bytes (on disk) of the total sstables */
+    protected final long sizeInBytes;
+
+    /** The total uncompressed size of the sstables selected for compaction */
+    protected final long totBytesToCompact;
+
+    /** The total uncompressed size of the expired sstables that are going to be dropped during compaction */
+    protected final long totalBytesToDrop;
+
+    /** The number of bytes read so far for the compactions here - read throughput is calculated based on this */
+    protected final long readBytes;
+
+    /** The number of bytes written so far for the compaction here - write throughput is calculated based on this */
+    protected final long writtenBytes;
+
+    /** The read throughput in bytes per second */
+    protected final double readThroughput;
+
+    /** The write throughput in bytes per second */
+    protected final double writeThroughput;
+
+    /** The hotness of this aggregate (where applicable) */
+    protected final double hotness;
+
+    CompactionAggregateStatistics(int numCompactions,
+                                  int numCompactionsInProgress,
+                                  int numSSTables,
+                                  int numExpiredSSTables,
+                                  int numCandidateSSTables,
+                                  int numCompactingSSTables,
+                                  long sizeInBytes,
+                                  long totBytesToCompact,
+                                  long totBytesToDrop,
+                                  long readBytes,
+                                  long writtenBytes,
+                                  double readThroughput,
+                                  double writeThroughput,
+                                  double hotness)
+    {
+        this.numCompactions = numCompactions;
+        this.numCompactionsInProgress = numCompactionsInProgress;
+        this.numCandidateSSTables = numCandidateSSTables;
+        this.numCompactingSSTables = numCompactingSSTables;
+        this.numSSTables = numSSTables;
+        this.numExpiredSSTables = numExpiredSSTables;
+        this.sizeInBytes = sizeInBytes;
+        this.totBytesToCompact = totBytesToCompact;
+        this.totalBytesToDrop = totBytesToDrop;
+        this.readBytes = readBytes;
+        this.writtenBytes = writtenBytes;
+        this.readThroughput = readThroughput;
+        this.writeThroughput = writeThroughput;
+        this.hotness = hotness;
+    }
+
+    CompactionAggregateStatistics(CompactionAggregateStatistics base)
+    {
+        this.numCompactions = base.numCompactions;
+        this.numCompactionsInProgress = base.numCompactionsInProgress;
+        this.numCandidateSSTables = base.numCandidateSSTables;
+        this.numCompactingSSTables = base.numCompactingSSTables;
+        this.numExpiredSSTables = base.numExpiredSSTables;
+        this.numSSTables = base.numSSTables;
+        this.sizeInBytes = base.sizeInBytes;
+        this.totBytesToCompact = base.totBytesToCompact;
+        this.totalBytesToDrop = base.totalBytesToDrop;
+        this.readBytes = base.readBytes;
+        this.writtenBytes = base.writtenBytes;
+        this.readThroughput = base.readThroughput;
+        this.writeThroughput = base.writeThroughput;
+        this.hotness = base.hotness;
+    }
+
+    /** The number of compactions that are either pending or in progress */
+    @JsonProperty
+    public int numCompactions()
+    {
+        return numCompactions;
+    }
+
+    /** The number of compactions that are in progress */
+    @JsonProperty
+    public int numCompactionsInProgress()
+    {
+        return numCompactionsInProgress;
+    }
+
+    /** The total number of sstables, whether they need compacting or not */
+    @JsonProperty
+    public int numSSTables()
+    {
+        return numSSTables;
+    }
+
+    /** The number of sstables that are part of this level */
+    @JsonProperty
+    public int numCandidateSSTables()
+    {
+        return numCandidateSSTables;
+    }
+
+    /** The number of sstables that are currently part of a compaction operation */
+    @JsonProperty
+    public int numCompactingSSTables()
+    {
+        return numCompactingSSTables;
+    }
+
+    /** The size in bytes (on disk) of the total sstables */
+    public long sizeInBytes()
+    {
+        return sizeInBytes;
+    }
+
+    /** The read throughput in bytes per second */
+    @JsonProperty
+    public double readThroughput()
+    {
+        return readThroughput;
+    }
+
+    /** The write throughput in bytes per second */
+    @JsonProperty
+    public double writeThroughput()
+    {
+        return writeThroughput;
+    }
+
+    /** The total uncompressed size of the sstables selected for compaction */
+    @JsonProperty
+    public long tot()
+    {
+        return totBytesToCompact;
+    }
+
+    /** The number of bytes read so far for the compactions here - read throughput is calculated based on this */
+    @JsonProperty
+    public long read()
+    {
+        return readBytes;
+    }
+
+    /** The number of bytes written so far for the compaction here - write throughput is calculated based on this */
+    @JsonProperty
+    public long written()
+    {
+        return writtenBytes;
+    }
+
+    /** The hotness of this aggregate (where applicable) */
+    @JsonProperty
+    public double hotness()
+    {
+        return hotness;
+    }
+
+    /** The name of the shard, empty if the compaction is not sharded (the default). */
+    @JsonProperty
+    public String shard()
+    {
+        return NO_SHARD;
+    }
+
+    @Override
+    public String toString()
+    {
+        return data().toString();
+    }
+
+    protected Collection<String> header()
+    {
+        return HEADER;
+    }
+
+    protected Collection<String> data()
+    {
+        return ImmutableList.of(Integer.toString(numSSTables),
+                                prettyPrintMemory(sizeInBytes),
+                                        Integer.toString(numCompactions()) + '/' + numCompactionsInProgress(),
+                                        Integer.toString(numCandidateSSTables()) + '/' + numCompactingSSTables(),
+                                prettyPrintMemoryPerSecond((long) readThroughput()),
+                                prettyPrintMemoryPerSecond((long) writeThroughput()),
+                                prettyPrintMemory(totBytesToCompact) + '/' + prettyPrintMemory(readBytes) + '/' + prettyPrintMemory(writtenBytes));
+    }
+
+    protected String toString(long value)
+    {
+        return FBUtilities.prettyPrintMemory(value);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
index a9fcad73c971..55abbab9b1f1 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@ -17,37 +17,34 @@
  */
 package org.apache.cassandra.db.compaction;
 
-import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.LongPredicate;
+import java.util.function.UnaryOperator;
+
+import javax.annotation.Nullable;
 
-import com.google.common.base.Predicates;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.AbstractCompactionController;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
-import org.apache.cassandra.utils.OverlapIterator;
-import org.apache.cassandra.utils.concurrent.Refs;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.NEVER_PURGE_TOMBSTONES;
-import static org.apache.cassandra.db.lifecycle.SSTableIntervalTree.buildIntervals;
 
 /**
  * Manage compaction options.
@@ -58,79 +55,81 @@ public class CompactionController extends AbstractCompactionController
     static final boolean NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE = NEVER_PURGE_TOMBSTONES.getBoolean();
 
     private final boolean compactingRepaired;
-    // note that overlapIterator and overlappingSSTables will be null if NEVER_PURGE_TOMBSTONES is set - this is a
+    // note that overlapTracker will be null if NEVER_PURGE_TOMBSTONES is set - this is a
     // good thing so that noone starts using them and thinks that if overlappingSSTables is empty, there
     // is no overlap.
-    private Refs<SSTableReader> overlappingSSTables;
-    private OverlapIterator<PartitionPosition, SSTableReader> overlapIterator;
+    @Nullable
+    private final CompactionRealm.OverlapTracker overlapTracker;
+    @Nullable
     private final Iterable<SSTableReader> compacting;
+    @Nullable
     private final RateLimiter limiter;
     private final long minTimestamp;
-    final Map<SSTableReader, FileDataInput> openDataFiles = new HashMap<>();
+    private final Map<SSTableReader, FileDataInput> openDataFiles = new HashMap<>();
 
-    protected CompactionController(ColumnFamilyStore cfs, long maxValue)
+    protected CompactionController(CompactionRealm realm, long maxValue)
     {
-        this(cfs, null, maxValue);
+        this(realm, null, maxValue);
     }
 
-    public CompactionController(ColumnFamilyStore cfs, Set<SSTableReader> compacting, long gcBefore)
+    public CompactionController(CompactionRealm realm, Set<SSTableReader> compacting, long gcBefore)
     {
-        this(cfs, compacting, gcBefore, null,
-             cfs.getCompactionStrategyManager().getCompactionParams().tombstoneOption());
+        this(realm, compacting, gcBefore, null, realm.getCompactionParams().tombstoneOption());
     }
 
-    public CompactionController(ColumnFamilyStore cfs, Set<SSTableReader> compacting, long gcBefore, RateLimiter limiter, TombstoneOption tombstoneOption)
+    public CompactionController(CompactionRealm realm, Set<SSTableReader> compacting, long gcBefore, RateLimiter limiter, TombstoneOption tombstoneOption)
     {
         //When making changes to the method, be aware that some of the state of the controller may still be uninitialized
         //(e.g. TWCS sets up the value of ignoreOverlaps() after this completes)
-        super(cfs, gcBefore, tombstoneOption);
+        super(realm, gcBefore, tombstoneOption);
         this.compacting = compacting;
         this.limiter = limiter;
         compactingRepaired = compacting != null && compacting.stream().allMatch(SSTableReader::isRepaired);
         this.minTimestamp = compacting != null && !compacting.isEmpty()       // check needed for test
                           ? compacting.stream().mapToLong(SSTableReader::getMinTimestamp).min().getAsLong()
                           : 0;
-        refreshOverlaps();
-        if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE)
-            logger.warn("You are running with -D{}=true, this is dangerous!", NEVER_PURGE_TOMBSTONES.getKey());
+
+        if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || realm.getNeverPurgeTombstones())
+        {
+            overlapTracker = null;
+            if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE)
+                logger.warn("You are running with -D{}=true, this is dangerous!", NEVER_PURGE_TOMBSTONES.getKey());
+            else
+                logger.debug("Not using overlaps for {}.{} - neverPurgeTombstones is enabled", realm.getKeyspaceName(), realm.getTableName());
+        }
+        else
+            overlapTracker = realm.getOverlapTracker(compacting);
+
+        logger.debug("Compaction controller created for {} with {} compacting sstables, {} overlapping sstables, tsOption={}, compactingRepaired={}",
+                     realm.metadata(), compacting == null ? 0 : compacting.size(), overlapTracker == null ? 0 : overlapTracker.overlaps().size(), tombstoneOption, compactingRepaired());
     }
 
     public void maybeRefreshOverlaps()
     {
-        if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE)
-        {
-            logger.debug("not refreshing overlaps - running with -D{}=true", NEVER_PURGE_TOMBSTONES.getKey());
-            return;
-        }
+        if (overlapTracker != null && overlapTracker.maybeRefresh())
+            closeDataFiles();
+    }
 
-        if (cfs.getNeverPurgeTombstones())
+    public void refreshOverlaps()
+    {
+        if (overlapTracker != null)
         {
-            logger.debug("not refreshing overlaps for {}.{} - neverPurgeTombstones is enabled", cfs.getKeyspaceName(), cfs.getTableName());
-            return;
+            overlapTracker.refreshOverlaps();
+            closeDataFiles();
         }
-
-        if (overlappingSSTables == null || overlappingSSTables.stream().anyMatch(SSTableReader::isMarkedCompacted))
-            refreshOverlaps();
     }
 
-    void refreshOverlaps()
+    void closeDataFiles()
     {
-        if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || cfs.getNeverPurgeTombstones())
-            return;
-
-        if (this.overlappingSSTables != null)
-            close();
-
-        if (compacting == null)
-            overlappingSSTables = Refs.tryRef(Collections.<SSTableReader>emptyList());
-        else
-            overlappingSSTables = cfs.getAndReferenceOverlappingLiveSSTables(compacting);
-        this.overlapIterator = new OverlapIterator<>(buildIntervals(overlappingSSTables));
+        FileUtils.closeQuietly(openDataFiles.values());
+        openDataFiles.clear();
     }
 
-    public Set<SSTableReader> getFullyExpiredSSTables()
+    public Set<CompactionSSTable> getFullyExpiredSSTables()
     {
-        return getFullyExpiredSSTables(cfs, compacting, overlappingSSTables, gcBefore, ignoreOverlaps());
+        if (overlapTracker == null)
+            return Collections.emptySet();
+        return getFullyExpiredSSTables(realm, compacting, c -> overlapTracker.overlaps(), gcBefore, ignoreOverlaps());
     }
 
     /**
@@ -143,95 +142,93 @@ public Set<SSTableReader> getFullyExpiredSSTables()
      *    - if not droppable, remove from candidates
      * 4. return candidates.
      *
-     * @param cfStore
+     * @param realm
      * @param compacting we take the drop-candidates from this set, it is usually the sstables included in the compaction
-     * @param overlapping the sstables that overlap the ones in compacting.
+     * @param overlappingSupplier called on the compacting sstables to compute the set of sstables that overlap with them if needed
      * @param gcBefore
      * @param ignoreOverlaps don't check if data shadows/overlaps any data in other sstables
      * @return
      */
-    public static Set<SSTableReader> getFullyExpiredSSTables(ColumnFamilyStore cfStore,
-                                                             Iterable<SSTableReader> compacting,
-                                                             Iterable<SSTableReader> overlapping,
-                                                             long gcBefore,
-                                                             boolean ignoreOverlaps)
+    public static
+    Set<CompactionSSTable> getFullyExpiredSSTables(CompactionRealm realm,
+                                                   Iterable<? extends CompactionSSTable> compacting,
+                                                   UnaryOperator<Iterable<? extends CompactionSSTable>> overlappingSupplier,
+                                                   long gcBefore,
+                                                   boolean ignoreOverlaps)
     {
-        logger.trace("Checking droppable sstables in {}", cfStore);
+        logger.trace("Checking droppable sstables in {}", realm);
 
-        if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || compacting == null || cfStore.getNeverPurgeTombstones() || overlapping == null)
+        if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || compacting == null || realm.getNeverPurgeTombstones())
             return Collections.emptySet();
 
-        if (cfStore.getCompactionStrategyManager().onlyPurgeRepairedTombstones() && !Iterables.all(compacting, SSTableReader::isRepaired))
+        if (realm.onlyPurgeRepairedTombstones() && !Iterables.all(compacting, CompactionSSTable::isRepaired))
             return Collections.emptySet();
 
-        if (ignoreOverlaps)
+        long minTimestamp;
+        if (!ignoreOverlaps)
         {
-            Set<SSTableReader> fullyExpired = new HashSet<>();
-            for (SSTableReader candidate : compacting)
-            {
-                if (candidate.getMaxLocalDeletionTime() < gcBefore)
-                {
-                    fullyExpired.add(candidate);
-                    logger.trace("Dropping overlap ignored expired SSTable {} (maxLocalDeletionTime={}, gcBefore={})",
-                                 candidate, candidate.getMaxLocalDeletionTime(), gcBefore);
-                }
-            }
-            return fullyExpired;
+            var overlapping = overlappingSupplier.apply(compacting);
+            minTimestamp = Math.min(Math.min(minSurvivingTimestamp(overlapping, gcBefore),
+                                             minSurvivingTimestamp(compacting, gcBefore)),
+                                    minTimestamp(realm.getAllMemtables()));
         }
-
-        List<SSTableReader> candidates = new ArrayList<>();
-        long minTimestamp = Long.MAX_VALUE;
-
-        for (SSTableReader sstable : overlapping)
+        else
         {
-            // Overlapping might include fully expired sstables. What we care about here is
-            // the min timestamp of the overlapping sstables that actually contain live data.
-            if (sstable.getMaxLocalDeletionTime() >= gcBefore)
-                minTimestamp = Math.min(minTimestamp, sstable.getMinTimestamp());
+            minTimestamp = Long.MAX_VALUE;
         }
 
-        for (SSTableReader candidate : compacting)
+        // At this point, minTimestamp denotes the lowest timestamp of any relevant
+        // SSTable or Memtable that contains a constructive value. Any compacting sstable with only expired content that
+        // also has (getMaxTimestamp() < minTimestamp) serves no purpose anymore.
+
+        Set<CompactionSSTable> expired = new HashSet<>();
+        for (CompactionSSTable candidate : compacting)
         {
-            if (candidate.getMaxLocalDeletionTime() < gcBefore)
-                candidates.add(candidate);
-            else
-                minTimestamp = Math.min(minTimestamp, candidate.getMinTimestamp());
+            if (candidate.getMaxLocalDeletionTime() < gcBefore &&
+                candidate.getMaxTimestamp() < minTimestamp)
+            {
+                logger.trace("Dropping {}expired SSTable {} (maxLocalDeletionTime={}, gcBefore={})",
+                             ignoreOverlaps ? "overlap ignored " : "",
+                             candidate, candidate.getMaxLocalDeletionTime(), gcBefore);
+                expired.add(candidate);
+            }
         }
+        return expired;
+    }
 
-        for (Memtable memtable : cfStore.getTracker().getView().getAllMemtables())
+    private static long minTimestamp(Iterable<Memtable> memtables)
+    {
+        long minTimestamp = Long.MAX_VALUE;
+        for (Memtable memtable : memtables)
         {
             if (memtable.getMinTimestamp() != Memtable.NO_MIN_TIMESTAMP)
                 minTimestamp = Math.min(minTimestamp, memtable.getMinTimestamp());
         }
+        return minTimestamp;
+    }
 
-        // At this point, minTimestamp denotes the lowest timestamp of any relevant
-        // SSTable or Memtable that contains a constructive value. candidates contains all the
-        // candidates with no constructive values. The ones out of these that have
-        // (getMaxTimestamp() < minTimestamp) serve no purpose anymore.
-
-        Iterator<SSTableReader> iterator = candidates.iterator();
-        while (iterator.hasNext())
+    private static long minSurvivingTimestamp(Iterable<? extends CompactionSSTable> ssTables,
+                                              long gcBefore)
+    {
+        long minTimestamp = Long.MAX_VALUE;
+        for (CompactionSSTable sstable : ssTables)
         {
-            SSTableReader candidate = iterator.next();
-            if (candidate.getMaxTimestamp() >= minTimestamp)
-            {
-                iterator.remove();
-            }
-            else
-            {
-               logger.trace("Dropping expired SSTable {} (maxLocalDeletionTime={}, gcBefore={})",
-                        candidate, candidate.getMaxLocalDeletionTime(), gcBefore);
-            }
+            // Overlapping might include fully expired sstables. What we care about here is
+            // the min timestamp of the overlapping sstables that actually contain live data.
+            if (sstable.getMaxLocalDeletionTime() >= gcBefore)
+                minTimestamp = Math.min(minTimestamp, sstable.getMinTimestamp());
         }
-        return new HashSet<>(candidates);
+
+        return minTimestamp;
     }
 
-    public static Set<SSTableReader> getFullyExpiredSSTables(ColumnFamilyStore cfStore,
-                                                             Iterable<SSTableReader> compacting,
-                                                             Iterable<SSTableReader> overlapping,
-                                                             long gcBefore)
+    public static
+    Set<CompactionSSTable> getFullyExpiredSSTables(CompactionRealm realm,
+                                                   Iterable<? extends CompactionSSTable> compacting,
+                                                   UnaryOperator<Iterable<? extends CompactionSSTable>> overlappingSupplier,
+                                                   long gcBefore)
     {
-        return getFullyExpiredSSTables(cfStore, compacting, overlapping, gcBefore, false);
+        return getFullyExpiredSSTables(realm, compacting, overlappingSupplier, gcBefore, false);
     }
 
     /**
@@ -244,35 +241,52 @@ public static Set<SSTableReader> getFullyExpiredSSTables(ColumnFamilyStore cfSto
     @Override
     public LongPredicate getPurgeEvaluator(DecoratedKey key)
     {
-        if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || !compactingRepaired() || cfs.getNeverPurgeTombstones() || overlapIterator == null)
+        if (overlapTracker == null || !compactingRepaired())
             return time -> false;
 
-        overlapIterator.update(key);
-        Set<SSTableReader> filteredSSTables = overlapIterator.overlaps();
-        Iterable<Memtable> memtables = cfs.getTracker().getView().getAllMemtables();
+        Collection<? extends CompactionSSTable> filteredSSTables = overlapTracker.overlaps(key);
+        Iterable<Memtable> memtables = realm.getAllMemtables();
         long minTimestampSeen = Long.MAX_VALUE;
         boolean hasTimestamp = false;
 
-        for (SSTableReader sstable: filteredSSTables)
+        // TODO: Evaluate if doing this in sort order to minimize mayContainAssumingKeyIsRange calls is a performance improvement.
+        for (CompactionSSTable sstable: filteredSSTables)
         {
-            if (sstable.mayContainAssumingKeyIsInRange(key))
+            long sstableMinTimestamp = sstable.getMinTimestamp();
+            // if we don't have bloom filter(bf_fp_chance=1.0 or filter file is missing),
+            // we check index file instead.
+            if (sstableMinTimestamp < minTimestampSeen && sstable.mayContainAssumingKeyIsInRange(key))
             {
-                minTimestampSeen = Math.min(minTimestampSeen, sstable.getMinTimestamp());
+                minTimestampSeen = sstableMinTimestamp;
                 hasTimestamp = true;
             }
         }
 
-        for (Memtable memtable : memtables)
+        OpOrder.Group readGroup = null;
+        try
         {
-            if (memtable.getMinTimestamp() != Memtable.NO_MIN_TIMESTAMP)
+            for (Memtable memtable : memtables)
             {
-                if (memtable.rowIterator(key) != null)
+                long memtableMinTimestamp = memtable.getMinTimestamp();
+                if (memtableMinTimestamp >= minTimestampSeen || memtableMinTimestamp == Memtable.NO_MIN_TIMESTAMP)
+                    continue;
+
+                if (readGroup == null)
+                    readGroup = memtable.readOrdering().start(); // the read order is the same for all memtables of a CFS
+
+                Partition partition = memtable.getPartition(key);
+                if (partition != null)
                 {
-                    minTimestampSeen = Math.min(minTimestampSeen, memtable.getMinTimestamp());
+                    minTimestampSeen = Math.min(minTimestampSeen, partition.stats().minTimestamp);
                     hasTimestamp = true;
                 }
             }
         }
+        finally
+        {
+            if (readGroup != null)
+                readGroup.close();
+        }
 
         if (!hasTimestamp)
             return time -> true;
@@ -285,39 +299,58 @@ public LongPredicate getPurgeEvaluator(DecoratedKey key)
 
     public void close()
     {
-        if (overlappingSSTables != null)
-            overlappingSSTables.release();
-
-        FileUtils.closeQuietly(openDataFiles.values());
-        openDataFiles.clear();
+        closeDataFiles();
+        FileUtils.closeQuietly(overlapTracker);
     }
 
     public boolean compactingRepaired()
     {
-        return !cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones() || compactingRepaired;
+        return !realm.onlyPurgeRepairedTombstones() || compactingRepaired;
     }
 
-    boolean provideTombstoneSources()
+    boolean shouldProvideTombstoneSources()
     {
-        return tombstoneOption != TombstoneOption.NONE;
+        return tombstoneOption != TombstoneOption.NONE && compactingRepaired() && overlapTracker != null;
     }
 
     // caller must close iterators
     public Iterable<UnfilteredRowIterator> shadowSources(DecoratedKey key, boolean tombstoneOnly)
     {
-        if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || cfs.getNeverPurgeTombstones())
+        if (!shouldProvideTombstoneSources())
             return null;
-        overlapIterator.update(key);
-        return Iterables.filter(Iterables.transform(overlapIterator.overlaps(),
-                                                    reader -> getShadowIterator(reader, key, tombstoneOnly)),
-                                Predicates.notNull());
+
+        return overlapTracker.openSelectedOverlappingSSTables(key,
+                                                              tombstoneOnly ? this::isTombstoneShadowSource
+                                                                            : this::isCellDataShadowSource,
+                                                              sstable -> {
+                                                                  long position = sstable.getPosition(key, SSTableReader.Operator.EQ, false);
+                                                                  if (position < 0)
+                                                                      return null;
+
+                                                                  return sstable.simpleIterator(openDataFiles.computeIfAbsent(sstable,
+                                                                                                                                  this::openDataFile),
+                                                                                                    key,
+                                                                                                    position,
+                                                                                                    tombstoneOnly);
+                                                              });
+    }
+
+    // TODO verify this stuff
+    private boolean isTombstoneShadowSource(CompactionSSTable ssTable)
+    {
+        return isCellDataShadowSource(ssTable) && ssTable.mayHaveTombstones();
+    }
+
+    private boolean isCellDataShadowSource(CompactionSSTable ssTable)
+    {
+        return !ssTable.isMarkedSuspect() && ssTable.getMaxTimestamp() > minTimestamp;
     }
 
     private UnfilteredRowIterator getShadowIterator(SSTableReader reader, DecoratedKey key, boolean tombstoneOnly)
     {
         if (reader.isMarkedSuspect() ||
-            reader.getMaxTimestamp() <= minTimestamp ||
-            tombstoneOnly && !reader.mayHaveTombstones())
+                reader.getMaxTimestamp() <= minTimestamp ||
+                tombstoneOnly && !reader.mayHaveTombstones())
             return null;
         long position = reader.getPosition(key, SSTableReader.Operator.EQ);
         if (position < 0)
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionCursor.java b/src/java/org/apache/cassandra/db/compaction/CompactionCursor.java
new file mode 100644
index 000000000000..89c9761bc45b
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionCursor.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.cassandra.db.compaction.writers.SSTableDataSink;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.compaction.SortedStringTableCursor;
+import org.apache.cassandra.io.sstable.compaction.IteratorFromCursor;
+import org.apache.cassandra.io.sstable.compaction.PurgeCursor;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursor;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursorMerger;
+import org.apache.cassandra.io.sstable.compaction.SkipEmptyDataCursor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Clock;
+
+/**
+ * Counterpart to CompactionIterator. Maintains sstable cursors, applies limiter and produces metrics. In the future it
+ * should also pass information to observers and deal with expired tombstones and garbage-collection compactions.
+ */
+public class CompactionCursor implements SSTableCursorMerger.MergeListener, AutoCloseable
+{
+    private static final long MILLISECONDS_TO_UPDATE_PROGRESS = 1000;
+
+    private final OperationType type;
+    private final CompactionController controller;
+    private final SSTableCursor cursor;
+    private final Row.Builder rowBuilder;
+
+    private final long totalBytes;
+    private volatile long currentBytes;
+    private long currentProgressMillisSinceStartup;
+
+    /**
+     * Merged frequency counters for partitions and rows (AKA histograms).
+     * The array index represents the number of sstables containing the row or partition minus one. So index 0 contains
+     * the number of rows or partitions coming from a single sstable (therefore copied rather than merged), index 1 contains
+     * the number of rows or partitions coming from two sstables and so forth.
+     */
+    private final long[] mergedPartitionsHistogram;
+    private final long[] mergedRowsHistogram;
+
+    public CompactionCursor(OperationType type, Collection<SSTableReader> readers, Range<Token> tokenRange, CompactionController controller, RateLimiter limiter, long nowInSec)
+    {
+        this.controller = controller;
+        this.type = type;
+        this.mergedPartitionsHistogram = new long[readers.size()];
+        this.mergedRowsHistogram = new long[readers.size()];
+        this.rowBuilder = BTreeRow.sortedBuilder();
+        this.cursor = makeMergedAndPurgedCursor(readers, tokenRange, controller, limiter, nowInSec);
+        this.totalBytes = cursor.bytesTotal();
+        this.currentBytes = 0;
+        this.currentProgressMillisSinceStartup = Clock.Global.currentTimeMillis();
+    }
+
+    private SSTableCursor makeMergedAndPurgedCursor(Collection<SSTableReader> readers,
+                                                    Range<Token> tokenRange,
+                                                    CompactionController controller,
+                                                    RateLimiter limiter,
+                                                    long nowInSec)
+    {
+        if (readers.isEmpty())
+            return SSTableCursor.empty();
+
+        SSTableCursor merged = new SSTableCursorMerger(readers.stream()
+                                                              .map(r -> new SortedStringTableCursor(r, tokenRange, limiter))
+                                                              .collect(Collectors.toList()),
+                                                       metadata(),
+                                                       this);
+
+        if (Iterables.any(readers, SSTableReader::mayHaveTombstones))
+        {
+            merged = new PurgeCursor(merged, controller, nowInSec);
+            merged = new SkipEmptyDataCursor(merged);
+        }
+        return merged;
+    }
+
+    public SSTableCursor.Type copyOne(SSTableDataSink writer) throws IOException
+    {
+        boolean wasInitialized = true;
+        if (cursor.type() == SSTableCursor.Type.UNINITIALIZED)
+        {
+            cursor.advance();
+            wasInitialized = false;
+        }
+
+        switch (cursor.type())
+        {
+            case ROW:
+                Row row = collectRow();
+                if (!row.isEmpty())
+                    writer.addUnfiltered(row);
+                return SSTableCursor.Type.ROW;
+            case RANGE_TOMBSTONE:
+                writer.addUnfiltered(collectRangeTombstoneMarker());
+                return SSTableCursor.Type.RANGE_TOMBSTONE;
+            case PARTITION:
+                if (wasInitialized)
+                    writer.endPartition();
+                maybeUpdateProgress();
+                // The writer can reject a partition (e.g. due to long key). Loop until it accepts one.
+                while (!writer.startPartition(cursor.partitionKey(), cursor.partitionLevelDeletion()))
+                {
+                    if (!skipToNextPartition())
+                        return SSTableCursor.Type.EXHAUSTED;
+                }
+                cursor.advance();
+                return SSTableCursor.Type.PARTITION;
+            case EXHAUSTED:
+                if (wasInitialized)
+                    writer.endPartition();
+                updateProgress(Long.MAX_VALUE);
+                return SSTableCursor.Type.EXHAUSTED;
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    private void maybeUpdateProgress()
+    {
+        long now = Clock.Global.currentTimeMillis();
+        if (now - currentProgressMillisSinceStartup > MILLISECONDS_TO_UPDATE_PROGRESS)
+            updateProgress(now);
+    }
+
+    private void updateProgress(long now)
+    {
+        currentBytes = cursor.bytesProcessed();
+        currentProgressMillisSinceStartup = now;
+    }
+
+    private Row collectRow()
+    {
+        return IteratorFromCursor.collectRow(cursor, rowBuilder);
+    }
+
+    private Unfiltered collectRangeTombstoneMarker()
+    {
+        return IteratorFromCursor.collectRangeTombstoneMarker(cursor);
+    }
+
+    private boolean skipToNextPartition()
+    {
+        while (true)
+        {
+            switch (cursor.advance())
+            {
+                case EXHAUSTED:
+                    return false;
+                case PARTITION:
+                    return true;
+                default:
+                    break;  // continue loop
+            }
+        }
+    }
+
+    /**
+     * @return A {@link TableOperation} backed by this iterator. This operation can be observed for progress
+     * and for interrupting provided that it is registered with a {@link TableOperationObserver}, normally the
+     * metrics in the compaction manager. The caller is responsible for registering the operation and checking
+     * {@link TableOperation#isStopRequested()}.
+     */
+    public TableOperation createOperation(TableOperation.Progress progress)
+    {
+        return new AbstractTableOperation() {
+
+            @Override
+            public Progress getProgress()
+            {
+                return progress;
+            }
+
+            @Override
+            public boolean isGlobal()
+            {
+                return false;
+            }
+        };
+    }
+
+    public TableMetadata metadata()
+    {
+        return controller.realm.metadata();
+    }
+
+    long bytesRead()
+    {
+        // Note: This may be called from other threads. Reading the current positions in the sources is not safe as
+        // random access readers aren't thread-safe. To avoid problems we track the progress in the processing thread
+        // and store it in a volatile field.
+        return currentBytes;
+    }
+
+    long totalBytes()
+    {
+        return totalBytes;
+    }
+
+    long totalSourcePartitions()
+    {
+        return Arrays.stream(mergedPartitionsHistogram).reduce(0L, Long::sum);
+    }
+
+    long totalSourceRows()
+    {
+        return Arrays.stream(mergedRowsHistogram).reduce(0L, Long::sum);
+    }
+
+    long[] mergedPartitionsHistogram()
+    {
+        return mergedPartitionsHistogram;
+    }
+
+    long[] mergedRowsHistogram()
+    {
+        return mergedRowsHistogram;
+    }
+
+    public void onItem(SSTableCursor cursor, int numVersions)
+    {
+        switch (cursor.type())
+        {
+            case PARTITION:
+                mergedPartitionsHistogram[numVersions - 1] += 1;
+                break;
+            case ROW:
+                mergedRowsHistogram[numVersions - 1] += 1;
+                break;
+            default:
+                break;
+        }
+    }
+
+    public void remove()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public void close()
+    {
+        cursor.close();
+    }
+
+    public String toString()
+    {
+        return String.format("%s: %s, (%d/%d)", type, metadata(), bytesRead(), totalBytes());
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
deleted file mode 100644
index 0bfc925a7d0d..000000000000
--- a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.compaction;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.Predicate;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.utils.TimeUUID;
-
-public final class CompactionInfo
-{
-    public static final String ID = "id";
-    public static final String KEYSPACE = "keyspace";
-    public static final String COLUMNFAMILY = "columnfamily";
-    public static final String COMPLETED = "completed";
-    public static final String TOTAL = "total";
-    public static final String TASK_TYPE = "taskType";
-    public static final String UNIT = "unit";
-    public static final String COMPACTION_ID = "compactionId";
-    public static final String SSTABLES = "sstables";
-    public static final String TARGET_DIRECTORY = "targetDirectory";
-
-    private final TableMetadata metadata;
-    private final OperationType tasktype;
-    private final long completed;
-    private final long total;
-    private final Unit unit;
-    private final TimeUUID compactionId;
-    private final ImmutableSet<SSTableReader> sstables;
-    private final String targetDirectory;
-
-    public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, Collection<? extends SSTableReader> sstables, String targetDirectory)
-    {
-        this.tasktype = tasktype;
-        this.completed = completed;
-        this.total = total;
-        this.metadata = metadata;
-        this.unit = unit;
-        this.compactionId = compactionId;
-        this.sstables = ImmutableSet.copyOf(sstables);
-        this.targetDirectory = targetDirectory;
-    }
-
-    public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection<SSTableReader> sstables, String targetDirectory)
-    {
-        this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, targetDirectory);
-    }
-
-    public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection<? extends SSTableReader> sstables)
-    {
-        this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, null);
-    }
-
-    /**
-     * Special compaction info where we always need to cancel the compaction - for example ViewBuilderTask where we don't know
-     * the sstables at construction
-     */
-    public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId)
-    {
-        return withoutSSTables(metadata, tasktype, completed, total, unit, compactionId, null);
-    }
-
-    /**
-     * Special compaction info where we always need to cancel the compaction - for example AutoSavingCache where we don't know
-     * the sstables at construction
-     */
-    public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, String targetDirectory)
-    {
-        return new CompactionInfo(metadata, tasktype, completed, total, unit, compactionId, ImmutableSet.of(), targetDirectory);
-    }
-
-    /** @return A copy of this CompactionInfo with updated progress. */
-    public CompactionInfo forProgress(long complete, long total)
-    {
-        return new CompactionInfo(metadata, tasktype, complete, total, unit, compactionId, sstables, targetDirectory);
-    }
-
-    public Optional<String> getKeyspace()
-    {
-        return Optional.ofNullable(metadata != null ? metadata.keyspace : null);
-    }
-
-    public Optional<String> getTable()
-    {
-        return Optional.ofNullable(metadata != null ? metadata.name : null);
-    }
-
-    public TableMetadata getTableMetadata()
-    {
-        return metadata;
-    }
-
-    public long getCompleted()
-    {
-        return completed;
-    }
-
-    public long getTotal()
-    {
-        return total;
-    }
-
-    public OperationType getTaskType()
-    {
-        return tasktype;
-    }
-
-    public TimeUUID getTaskId()
-    {
-        return compactionId;
-    }
-
-    public Unit getUnit()
-    {
-        return unit;
-    }
-
-    public Set<SSTableReader> getSSTables()
-    {
-        return sstables;
-    }
-
-    /**
-     * Get the directories this compaction could possibly write to.
-     *
-     * @return the directories that we might write to, or empty list if we don't know the metadata
-     * (like for index summary redistribution), or null if we don't have any disk boundaries
-     */
-    public List<File> getTargetDirectories()
-    {
-        if (metadata != null && !metadata.isIndex())
-        {
-            ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(metadata.id);
-            if (cfs != null)
-                return cfs.getDirectoriesForFiles(sstables);
-        }
-        return Collections.emptyList();
-    }
-
-    public String targetDirectory()
-    {
-        if (targetDirectory == null)
-            return "";
-
-        try
-        {
-            return new File(targetDirectory).canonicalPath();
-        }
-        catch (Throwable t)
-        {
-            throw new RuntimeException("Unable to resolve canonical path for " + targetDirectory);
-        }
-    }
-
-    /**
-     * Note that this estimate is based on the amount of data we have left to read - it assumes input
-     * size == output size for a compaction, which is not really true, but should most often provide a worst case
-     * remaining write size.
-     */
-    public long estimatedRemainingWriteBytes()
-    {
-        if (unit == Unit.BYTES && tasktype.writesData)
-            return getTotal() - getCompleted();
-        return 0;
-    }
-
-    @Override
-    public String toString()
-    {
-        if (metadata != null)
-        {
-            return String.format("%s(%s, %s / %s %s)@%s(%s, %s)",
-                                 tasktype, compactionId, completed, total, unit,
-                                 metadata.id, metadata.keyspace, metadata.name);
-        }
-        else
-        {
-            return String.format("%s(%s, %s / %s %s)",
-                                 tasktype, compactionId, completed, total, unit);
-        }
-    }
-
-    public Map<String, String> asMap()
-    {
-        Map<String, String> ret = new HashMap<String, String>();
-        ret.put(ID, metadata != null ? metadata.id.toString() : "");
-        ret.put(KEYSPACE, getKeyspace().orElse(null));
-        ret.put(COLUMNFAMILY, getTable().orElse(null));
-        ret.put(COMPLETED, Long.toString(completed));
-        ret.put(TOTAL, Long.toString(total));
-        ret.put(TASK_TYPE, tasktype.toString());
-        ret.put(UNIT, unit.toString());
-        ret.put(COMPACTION_ID, compactionId == null ? "" : compactionId.toString());
-        ret.put(SSTABLES, Joiner.on(',').join(sstables));
-        ret.put(TARGET_DIRECTORY, targetDirectory());
-        return ret;
-    }
-
-    boolean shouldStop(Predicate<SSTableReader> sstablePredicate)
-    {
-        if (sstables.isEmpty())
-        {
-            return true;
-        }
-        return sstables.stream().anyMatch(sstablePredicate);
-    }
-
-    public static abstract class Holder
-    {
-        private volatile boolean stopRequested = false;
-        public abstract CompactionInfo getCompactionInfo();
-
-        public void stop()
-        {
-            stopRequested = true;
-        }
-
-        /**
-         * if this compaction involves several/all tables we can safely check globalCompactionsPaused
-         * in isStopRequested() below
-         */
-        public abstract boolean isGlobal();
-
-        public boolean isStopRequested()
-        {
-            return stopRequested || (isGlobal() && CompactionManager.instance.isGlobalCompactionPaused());
-        }
-    }
-
-    public enum Unit
-    {
-        BYTES("bytes"), RANGES("token range parts"), KEYS("keys");
-
-        private final String name;
-
-        Unit(String name)
-        {
-            this.name = name;
-        }
-
-        @Override
-        public String toString()
-        {
-            return this.name;
-        }
-
-        public static boolean isFileSize(String unit)
-        {
-            return BYTES.toString().equals(unit);
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionInterruptedException.java b/src/java/org/apache/cassandra/db/compaction/CompactionInterruptedException.java
index b9174ec262f8..d59d7a156c3d 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionInterruptedException.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionInterruptedException.java
@@ -24,8 +24,10 @@ public class CompactionInterruptedException extends RuntimeException
 {
     private static final long serialVersionUID = -8651427062512310398L;
 
-    public CompactionInterruptedException(Object info)
+    public CompactionInterruptedException(Object info, TableOperation.StopTrigger trigger)
     {
-        super("Compaction interrupted: " + info);
+        super(String.format("Compaction interrupted due to %s: %s",
+                            (trigger == null ? TableOperation.StopTrigger.NONE : trigger).toString().toLowerCase(),
+                            info));
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
index 589cf39c77a4..a5b69b87bfe9 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.db.compaction;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -30,7 +31,6 @@
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.AbstractCompactionController;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Columns;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionTime;
@@ -38,7 +38,6 @@
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.transform.DuplicateRowChecker;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.partitions.PurgeFunction;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -51,12 +50,14 @@
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.db.rows.WrappingUnfilteredRowIterator;
+import org.apache.cassandra.db.transform.DuplicateRowChecker;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.transactions.CompactionTransaction;
 import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.metrics.TopPartitionTracker;
 import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
 import org.apache.cassandra.schema.Schema;
@@ -65,6 +66,7 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.paxos.PaxosRepairHistory;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
@@ -87,7 +89,7 @@
  *   <li>keep tracks of the compaction progress.</li>
  * </ul>
  */
-public class CompactionIterator extends CompactionInfo.Holder implements UnfilteredPartitionIterator
+public class CompactionIterator implements UnfilteredPartitionIterator
 {
     private static final long UNFILTERED_TO_UPDATE_PROGRESS = 100;
 
@@ -98,107 +100,172 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
     private final long nowInSec;
     private final TimeUUID compactionId;
     private final long totalBytes;
-    private long bytesRead;
-    private long totalSourceCQLRows;
+    private volatile long[] bytesReadByLevel;
 
     // Keep targetDirectory for compactions, needed for `nodetool compactionstats`
     private volatile String targetDirectory;
 
-    /*
-     * counters for merged rows.
-     * array index represents (number of merged rows - 1), so index 0 is counter for no merge (1 row),
-     * index 1 is counter for 2 rows merged, and so on.
+    /**
+     * Merged frequency counters for partitions and rows (AKA histograms).
+     * The array index represents the number of sstables containing the row or partition minus one. So index 0 contains
+     * the number of rows or partitions coming from a single sstable (therefore copied rather than merged), index 1 contains
+     * the number of rows or partitions coming from two sstables and so forth.
      */
-    private final long[] mergeCounters;
+    private final long[] mergedPartitionsHistogram;
+    private final long[] mergedRowsHistogram;
 
     private final UnfilteredPartitionIterator compacted;
-    private final ActiveCompactionsTracker activeCompactions;
+    private final TableOperation op;
 
     public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, AbstractCompactionController controller, long nowInSec, TimeUUID compactionId)
     {
-        this(type, scanners, controller, nowInSec, compactionId, ActiveCompactionsTracker.NOOP, null);
+        this(type, scanners, controller, nowInSec, compactionId, null, null);
     }
 
-    public CompactionIterator(OperationType type,
-                              List<ISSTableScanner> scanners,
-                              AbstractCompactionController controller,
-                              long nowInSec,
-                              TimeUUID compactionId,
-                              ActiveCompactionsTracker activeCompactions,
-                              TopPartitionTracker.Collector topPartitionCollector)
+    @SuppressWarnings("resource") // We make sure to close mergedIterator in close() and CompactionIterator is itself an AutoCloseable
+    public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, AbstractCompactionController controller, long nowInSec, TimeUUID compactionId, TopPartitionTracker.Collector topPartitionCollector, CompactionProgress progress)
     {
         this.controller = controller;
         this.type = type;
         this.scanners = scanners;
         this.nowInSec = nowInSec;
         this.compactionId = compactionId;
-        this.bytesRead = 0;
+        this.bytesReadByLevel = new long[LeveledGenerations.MAX_LEVEL_COUNT];
 
         long bytes = 0;
         for (ISSTableScanner scanner : scanners)
             bytes += scanner.getLengthInBytes();
         this.totalBytes = bytes;
-        this.mergeCounters = new long[scanners.size()];
+        this.mergedPartitionsHistogram = new long[scanners.size()];
+        this.mergedRowsHistogram = new long[scanners.size()];
         // note that we leak `this` from the constructor when calling beginCompaction below, this means we have to get the sstables before
         // calling that to avoid a NPE.
         sstables = scanners.stream().map(ISSTableScanner::getBackingSSTables).flatMap(Collection::stream).collect(ImmutableSet.toImmutableSet());
-        this.activeCompactions = activeCompactions == null ? ActiveCompactionsTracker.NOOP : activeCompactions;
-        this.activeCompactions.beginCompaction(this); // note that CompactionTask also calls this, but CT only creates CompactionIterator with a NOOP ActiveCompactions
+        op = createOperation(progress);
 
         UnfilteredPartitionIterator merged = scanners.isEmpty()
-                                           ? EmptyIterators.unfilteredPartition(controller.cfs.metadata())
+                                           ? EmptyIterators.unfilteredPartition(controller.realm.metadata())
                                            : UnfilteredPartitionIterators.merge(scanners, listener());
         if (topPartitionCollector != null) // need to count tombstones before they are purged
             merged = Transformation.apply(merged, new TopPartitionTracker.TombstoneCounter(topPartitionCollector, nowInSec));
         merged = Transformation.apply(merged, new GarbageSkipper(controller));
-        Transformation<UnfilteredRowIterator> purger = isPaxos(controller.cfs) && paxosStatePurging() != legacy
+        Transformation<UnfilteredRowIterator> purger = isPaxos(controller.realm) && paxosStatePurging() != legacy
                                                        ? new PaxosPurger(nowInSec)
                                                        : new Purger(controller, nowInSec);
         merged = Transformation.apply(merged, purger);
         merged = DuplicateRowChecker.duringCompaction(merged, type);
-        compacted = Transformation.apply(merged, new AbortableUnfilteredPartitionTransformation(this));
+        compacted = Transformation.apply(merged, new AbortableUnfilteredPartitionTransformation(op));
+    }
+
+    protected TableOperation createOperation(CompactionProgress progress)
+    {
+        return new AbstractTableOperation() {
+
+            @Override
+            public Progress getProgress()
+            {
+                return progress != null
+                       ? progress
+                       : new AbstractTableOperation.OperationProgress(controller.realm.metadata(), type, bytesRead(), totalBytes, compactionId, sstables);
+            }
+
+            @Override
+            public boolean isGlobal()
+            {
+                return false;
+            }
+        };
+    }
+
+    /**
+     * @return A {@link TableOperation} backed by this iterator. This operation can be observed for progress
+     * and for interrupting provided that it is registered with a {@link TableOperationObserver}, normally the
+     * metrics in the compaction manager. The caller is responsible for registering the operation and checking
+     * {@link TableOperation#isStopRequested()}.
+     */
+    public TableOperation getOperation()
+    {
+        return op;
     }
 
     public TableMetadata metadata()
     {
-        return controller.cfs.metadata();
+        return controller.realm.metadata();
     }
 
-    public CompactionInfo getCompactionInfo()
+    public long bytesRead()
     {
-        return new CompactionInfo(controller.cfs.metadata(),
-                                  type,
-                                  bytesRead,
-                                  totalBytes,
-                                  compactionId,
-                                  sstables,
-                                  targetDirectory);
+        long bytesScanned = 0L;
+        for (ISSTableScanner scanner : scanners)
+            bytesScanned += scanner.getBytesScanned();
+
+        return bytesScanned;
     }
 
-    public boolean isGlobal()
+    long bytesRead(int level)
     {
-        return false;
+        return level >= 0 && level < bytesReadByLevel.length ? bytesReadByLevel[level] : 0;
     }
 
-    public void setTargetDirectory(final String targetDirectory)
+    long totalBytes()
     {
-        this.targetDirectory = targetDirectory;
+        return totalBytes;
+    }
+
+    long totalSourcePartitions()
+    {
+        return Arrays.stream(mergedPartitionsHistogram).reduce(0L, Long::sum);
+    }
+
+    long totalSourceRows()
+    {
+        return Arrays.stream(mergedRowsHistogram).reduce(0L, Long::sum);
+    }
+
+    public long getTotalCompressedSize()
+    {
+        long compressedSize = 0;
+        for (ISSTableScanner scanner : scanners)
+            compressedSize += scanner.getCompressedLengthInBytes();
+
+        return compressedSize;
+    }
+
+    public double getCompressionRatio()
+    {
+        double compressed = 0.0;
+        double uncompressed = 0.0;
+
+        for (ISSTableScanner scanner : scanners)
+        {
+            compressed += scanner.getCompressedLengthInBytes();
+            uncompressed += scanner.getLengthInBytes();
+        }
+
+        if (compressed == uncompressed || uncompressed == 0)
+            return MetadataCollector.NO_COMPRESSION_RATIO;
+
+        return compressed / uncompressed;
     }
 
-    private void updateCounterFor(int rows)
+    long[] mergedPartitionsHistogram()
     {
-        assert rows > 0 && rows - 1 < mergeCounters.length;
-        mergeCounters[rows - 1] += 1;
+        return mergedPartitionsHistogram;
     }
 
-    public long[] getMergedRowCounts()
+    long[] mergedRowsHistogram()
+    {
+        return mergedRowsHistogram;
+    }
+
+    public boolean isGlobal()
     {
-        return mergeCounters;
+        return false;
     }
 
-    public long getTotalSourceCQLRows()
+    public void setTargetDirectory(final String targetDirectory)
     {
-        return totalSourceCQLRows;
+        this.targetDirectory = targetDirectory;
     }
 
     private UnfilteredPartitionIterators.MergeListener listener()
@@ -208,7 +275,7 @@ private UnfilteredPartitionIterators.MergeListener listener()
             private boolean rowProcessingNeeded()
             {
                 return (type == OperationType.COMPACTION || type == OperationType.MAJOR_COMPACTION)
-                       && controller.cfs.indexManager.handles(IndexTransaction.Type.COMPACTION);
+                       && controller.realm.getIndexManager().handles(IndexTransaction.Type.COMPACTION);
             }
 
             @Override
@@ -219,49 +286,18 @@ public boolean preserveOrder()
 
             public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
             {
-                int merged = 0;
+                int numVersions = 0;
                 for (int i=0, isize=versions.size(); i<isize; i++)
                 {
                     UnfilteredRowIterator iter = versions.get(i);
                     if (iter != null)
-                        merged++;
+                        numVersions++;
                 }
 
-                assert merged > 0;
-
-                CompactionIterator.this.updateCounterFor(merged);
-
-                if (!rowProcessingNeeded())
-                    return null;
+                mergedPartitionsHistogram[numVersions - 1] += 1;
 
-                Columns statics = Columns.NONE;
-                Columns regulars = Columns.NONE;
-                for (int i=0, isize=versions.size(); i<isize; i++)
-                {
-                    UnfilteredRowIterator iter = versions.get(i);
-                    if (iter != null)
-                    {
-                        statics = statics.mergeTo(iter.columns().statics);
-                        regulars = regulars.mergeTo(iter.columns().regulars);
-                    }
-                }
-                final RegularAndStaticColumns regularAndStaticColumns = new RegularAndStaticColumns(statics, regulars);
-
-                // If we have a 2ndary index, we must update it with deleted/shadowed cells.
-                // we can reuse a single CleanupTransaction for the duration of a partition.
-                // Currently, it doesn't do any batching of row updates, so every merge event
-                // for a single partition results in a fresh cycle of:
-                // * Get new Indexer instances
-                // * Indexer::start
-                // * Indexer::onRowMerge (for every row being merged by the compaction)
-                // * Indexer::commit
-                // A new OpOrder.Group is opened in an ARM block wrapping the commits
-                // TODO: this should probably be done asynchronously and batched.
-                final CompactionTransaction indexTransaction =
-                    controller.cfs.indexManager.newCompactionTransaction(partitionKey,
-                                                                         regularAndStaticColumns,
-                                                                         versions.size(),
-                                                                         nowInSec);
+                final CompactionTransaction indexTransaction = rowProcessingNeeded() ? getIndexTransaction(partitionKey,versions)
+                                                                                     : null;
 
                 return new UnfilteredRowIterators.MergeListener()
                 {
@@ -271,9 +307,23 @@ public void onMergedPartitionLevelDeletion(DeletionTime mergedDeletion, Deletion
                     @Override
                     public void onMergedRows(Row merged, Row[] versions)
                     {
-                        indexTransaction.start();
-                        indexTransaction.onRowMerge(merged, versions);
-                        indexTransaction.commit();
+                        int numVersions = 0;
+                        for (Row v : versions)
+                        {
+                            if (v != null)
+                                numVersions++;
+                        }
+
+                        assert numVersions > 0 && numVersions - 1 < mergedRowsHistogram.length;
+                        mergedRowsHistogram[numVersions - 1] += 1;
+
+                        if (indexTransaction != null)
+                        {
+                            indexTransaction.start();
+                            indexTransaction.onRowMerge(merged, versions);
+                            indexTransaction.commit();
+                        }
+
                     }
 
                     @Override
@@ -286,17 +336,46 @@ public void close() {}
         };
     }
 
-    private void updateBytesRead()
+    private CompactionTransaction getIndexTransaction(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
     {
-        long n = 0;
-        for (ISSTableScanner scanner : scanners)
-            n += scanner.getCurrentPosition();
-        bytesRead = n;
+        Columns statics = Columns.NONE;
+        Columns regulars = Columns.NONE;
+        for (int i=0, isize=versions.size(); i<isize; i++)
+        {
+            @SuppressWarnings("resource")
+            UnfilteredRowIterator iter = versions.get(i);
+            if (iter != null)
+            {
+                statics = statics.mergeTo(iter.columns().statics);
+                regulars = regulars.mergeTo(iter.columns().regulars);
+            }
+        }
+        final RegularAndStaticColumns regularAndStaticColumns = new RegularAndStaticColumns(statics, regulars);
+        // If we have a 2ndary index, we must update it with deleted/shadowed cells.
+        // we can reuse a single CleanupTransaction for the duration of a partition.
+        // Currently, it doesn't do any batching of row updates, so every merge event
+        // for a single partition results in a fresh cycle of:
+        // * Get new Indexer instances
+        // * Indexer::start
+        // * Indexer::onRowMerge (for every row being merged by the compaction)
+        // * Indexer::commit
+        // A new OpOrder.Group is opened in an ARM block wrapping the commits
+        // TODO: this should probably be done asynchronously and batched.
+        return controller.realm.getIndexManager().newCompactionTransaction(partitionKey, regularAndStaticColumns, versions.size(), nowInSec);
     }
 
-    public long getBytesRead()
+    private void updateBytesRead()
     {
-        return bytesRead;
+        long[] bytesReadByLevel = new long[this.bytesReadByLevel.length];
+        for (ISSTableScanner scanner : scanners)
+        {
+            int level = scanner.level();
+            long n = scanner.getBytesScanned();
+
+            if (level >= 0 && level < bytesReadByLevel.length)
+                bytesReadByLevel[level] += n;
+        }
+        this.bytesReadByLevel = bytesReadByLevel;
     }
 
     public boolean hasNext()
@@ -316,22 +395,17 @@ public void remove()
 
     public void close()
     {
-        try
-        {
-            compacted.close();
-        }
-        finally
-        {
-            activeCompactions.finishCompaction(this);
-        }
+        updateBytesRead();
+
+        Throwables.maybeFail(Throwables.close(null, compacted));
     }
 
     public String toString()
     {
-        return this.getCompactionInfo().toString();
+        return String.format("%s: %s, (%d/%d)", type, metadata(), bytesRead(), totalBytes());
     }
 
-    private class Purger extends PurgeFunction
+    class Purger extends PurgeFunction
     {
         private final AbstractCompactionController controller;
 
@@ -343,8 +417,8 @@ private class Purger extends PurgeFunction
         private Purger(AbstractCompactionController controller, long nowInSec)
         {
             super(nowInSec, controller.gcBefore, controller.compactingRepaired() ? Long.MAX_VALUE : Integer.MIN_VALUE,
-                  controller.cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones(),
-                  controller.cfs.metadata.get().enforceStrictLiveness());
+                  controller.realm.onlyPurgeRepairedTombstones(),
+                  controller.realm.metadata().enforceStrictLiveness());
             this.controller = controller;
         }
 
@@ -352,7 +426,7 @@ private Purger(AbstractCompactionController controller, long nowInSec)
         protected void onEmptyPartitionPostPurge(DecoratedKey key)
         {
             if (type == OperationType.COMPACTION)
-                controller.cfs.invalidateCachedPartition(key);
+                controller.realm.invalidateCachedPartition(key);
         }
 
         @Override
@@ -365,7 +439,6 @@ protected void onNewPartition(DecoratedKey key)
         @Override
         protected void updateProgress()
         {
-            totalSourceCQLRows++;
             if ((++compactedUnfiltered) % UNFILTERED_TO_UPDATE_PROGRESS == 0)
                 updateBytesRead();
         }
@@ -379,7 +452,7 @@ protected void updateProgress()
         @Override
         protected boolean shouldIgnoreGcGrace()
         {
-            return controller.cfs.shouldIgnoreGcGraceForKey(currentKey);
+            return controller.realm.shouldIgnoreGcGraceForKey(currentKey);
         }
 
         /*
@@ -651,7 +724,7 @@ private PaxosPurger(long nowInSec)
         protected void onEmptyPartitionPostPurge(DecoratedKey key)
         {
             if (type == OperationType.COMPACTION)
-                controller.cfs.invalidateCachedPartition(key);
+                controller.realm.invalidateCachedPartition(key);
         }
 
         protected void updateProgress()
@@ -709,40 +782,40 @@ protected Row applyToRow(Row row)
     private static class AbortableUnfilteredPartitionTransformation extends Transformation<UnfilteredRowIterator>
     {
         private final AbortableUnfilteredRowTransformation abortableIter;
+        private final TableOperation op;
 
-        private AbortableUnfilteredPartitionTransformation(CompactionIterator iter)
+        private AbortableUnfilteredPartitionTransformation(TableOperation op)
         {
-            this.abortableIter = new AbortableUnfilteredRowTransformation(iter);
+            this.op = op;
+            this.abortableIter = new AbortableUnfilteredRowTransformation(op);
         }
 
         @Override
         protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
         {
-            if (abortableIter.iter.isStopRequested())
-                throw new CompactionInterruptedException(abortableIter.iter.getCompactionInfo());
+            op.throwIfStopRequested();
             return Transformation.apply(partition, abortableIter);
         }
     }
 
     private static class AbortableUnfilteredRowTransformation extends Transformation<UnfilteredRowIterator>
     {
-        private final CompactionIterator iter;
+        private final TableOperation op;
 
-        private AbortableUnfilteredRowTransformation(CompactionIterator iter)
+        private AbortableUnfilteredRowTransformation(TableOperation op)
         {
-            this.iter = iter;
+            this.op = op;
         }
 
         public Row applyToRow(Row row)
         {
-            if (iter.isStopRequested())
-                throw new CompactionInterruptedException(iter.getCompactionInfo());
+            op.throwIfStopRequested();
             return row;
         }
     }
 
-    private static boolean isPaxos(ColumnFamilyStore cfs)
+    private static boolean isPaxos(CompactionRealm realm)
     {
-        return cfs.name.equals(SystemKeyspace.PAXOS) && cfs.getKeyspaceName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME);
+        return realm.getTableName().equals(SystemKeyspace.PAXOS) && realm.getKeyspaceName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME);
     }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java b/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
index dd4983ddda6b..1089db5bcc99 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
@@ -18,15 +18,17 @@
 
 package org.apache.cassandra.db.compaction;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
-import java.lang.ref.WeakReference;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -36,6 +38,8 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.MapMaker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -45,37 +49,29 @@
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.Throwables;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.config.CassandraRelevantProperties.LOG_DIR;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 
+/**
+ * This is a Compaction logger that logs compaction events in a file called compactions.log.
+ * It was added by CASSANDRA-10805.
+ */
 public class CompactionLogger
 {
-    public interface Strategy
-    {
-        JsonNode sstable(SSTableReader sstable);
-
-        JsonNode options();
-
-        static Strategy none = new Strategy()
-        {
-            public JsonNode sstable(SSTableReader sstable)
-            {
-                return null;
-            }
-
-            public JsonNode options()
-            {
-                return null;
-            }
-        };
-    }
+    private static final DateTimeFormatter dateFormatter = DateTimeFormatter
+                                                           .ofPattern("yyyy-MM-dd' 'HH:mm:ss.SSS")
+                                                           .withZone(ZoneId.systemDefault() );
 
     /**
      * This will produce the compaction strategy's starting information.
@@ -88,8 +84,13 @@ public interface StrategySummary
     /**
      * This is an interface to allow writing to a different interface.
      */
-    public interface Writer
+    public interface Writer extends Closeable
     {
+        /**
+         * @param toWrite This should be written out to the medium capturing the logs
+         */
+        void write(String toWrite);
+
         /**
          * This is used when we are already trying to write out the start of a
          * @param statement This should be written out to the medium capturing the logs
@@ -104,141 +105,141 @@ public interface Writer
          * @param tag       This is an identifier for a strategy; each strategy should have a distinct Object
          */
         void write(JsonNode statement, StrategySummary summary, Object tag);
-    }
 
-    private interface CompactionStrategyAndTableFunction
-    {
-        JsonNode apply(AbstractCompactionStrategy strategy, SSTableReader sstable);
+        /**
+         * Closes the writer
+         */
+        @Override
+        void close();
     }
 
     private static final JsonNodeFactory json = JsonNodeFactory.instance;
     private static final Logger logger = LoggerFactory.getLogger(CompactionLogger.class);
-    private static final CompactionLogSerializer serializer = new CompactionLogSerializer();
-    private final WeakReference<ColumnFamilyStore> cfsRef;
-    private final WeakReference<CompactionStrategyManager> csmRef;
+
+    private static final ExecutorPlus loggerService = executorFactory().sequential("CompactionLogger");
+    private static final CompactionLogSerializer jsonWriter = new CompactionLogSerializer("compaction", "log", loggerService);
+
+    private final String keyspace;
+    private final String table;
     private final AtomicInteger identifier = new AtomicInteger(0);
-    private final Map<AbstractCompactionStrategy, String> compactionStrategyMapping = new MapMaker().weakKeys().makeMap();
+    private final Map<CompactionStrategy, String> compactionStrategyMapping = new MapMaker().weakKeys().makeMap();
+    private final Map<CompactionStrategy, Map<String, Writer>> csvWriters = new MapMaker().makeMap();
     private final AtomicBoolean enabled = new AtomicBoolean(false);
 
-    public CompactionLogger(ColumnFamilyStore cfs, CompactionStrategyManager csm)
+    CompactionLogger(TableMetadata metadata)
     {
-        csmRef = new WeakReference<>(csm);
-        cfsRef = new WeakReference<>(cfs);
+        this.keyspace = metadata.keyspace;
+        this.table = metadata.name;
     }
 
-    private void forEach(Consumer<AbstractCompactionStrategy> consumer)
+    void strategyCreated(CompactionStrategy strategy)
     {
-        CompactionStrategyManager csm = csmRef.get();
-        if (csm == null)
-            return;
-        csm.getStrategies()
-           .forEach(l -> l.forEach(consumer));
+        compactionStrategyMapping.computeIfAbsent(strategy, s -> String.valueOf(identifier.getAndIncrement()));
     }
 
-    private ArrayNode compactionStrategyMap(Function<AbstractCompactionStrategy, JsonNode> select)
+    /**
+     * Visit all the strategies.
+     *
+     * @param consumer a consumer function that receives all the strategies one by one
+     */
+    private void visitStrategies(Consumer<CompactionStrategy> consumer)
+    {
+        compactionStrategyMapping.keySet().forEach(consumer);
+    }
+
+    /**
+     * Rely on {@link this#visitStrategies(Consumer)} to visit all the strategies
+     * and add the properties extracted by the function passed in to a json node that is returned.
+     *
+     * @param select a function that given a strategy returns a json node
+     *
+     * @return a json node containing information on all the strategies returned by the strategy manager and the function passed in.
+     */
+    private ArrayNode getStrategiesJsonNode(Function<CompactionStrategy, JsonNode> select)
     {
         ArrayNode node = json.arrayNode();
-        forEach(acs -> node.add(select.apply(acs)));
+        visitStrategies(acs -> node.add(select.apply(acs)));
         return node;
     }
 
-    private ArrayNode sstableMap(Collection<SSTableReader> sstables, CompactionStrategyAndTableFunction csatf)
+    private ArrayNode sstableMap(Collection<SSTableReader> sstables)
     {
-        CompactionStrategyManager csm = csmRef.get();
         ArrayNode node = json.arrayNode();
-        if (csm == null)
-            return node;
-        sstables.forEach(t -> node.add(csatf.apply(csm.getCompactionStrategyFor(t), t)));
+        sstables.forEach(t -> node.add(describeSSTable(t)));
         return node;
     }
 
-    private String getId(AbstractCompactionStrategy strategy)
+    private String getId(CompactionStrategy strategy)
     {
-        return compactionStrategyMapping.computeIfAbsent(strategy, s -> String.valueOf(identifier.getAndIncrement()));
+        return compactionStrategyMapping.getOrDefault(strategy, "-1"); // there should always be a strategy because of strategyCreated()
     }
 
-    private JsonNode formatSSTables(AbstractCompactionStrategy strategy)
+    private JsonNode formatSSTables(CompactionStrategy strategy)
     {
         ArrayNode node = json.arrayNode();
-        CompactionStrategyManager csm = csmRef.get();
-        ColumnFamilyStore cfs = cfsRef.get();
-        if (csm == null || cfs == null)
-            return node;
-        for (SSTableReader sstable : cfs.getLiveSSTables())
-        {
-            if (csm.getCompactionStrategyFor(sstable) == strategy)
-                node.add(formatSSTable(strategy, sstable));
-        }
+        for (CompactionSSTable sstable : strategy.getSSTables())
+            node.add(formatSSTable(sstable));
+
         return node;
     }
 
-    private JsonNode formatSSTable(AbstractCompactionStrategy strategy, SSTableReader sstable)
+    private JsonNode formatSSTable(CompactionSSTable sstable)
     {
         ObjectNode node = json.objectNode();
-        node.put("generation", sstable.descriptor.id.toString());
-        node.put("version", sstable.descriptor.version.version);
+        node.put("generation", sstable.getDescriptor().id.toString());
+        node.put("version", sstable.getDescriptor().version.version);
         node.put("size", sstable.onDiskLength());
-        JsonNode logResult = strategy.strategyLogger().sstable(sstable);
-        if (logResult != null)
-            node.set("details", logResult);
+
+        // The details are only relevant or available for some strategies, e.g. LCS or Date tiered but
+        // it doesn't hurt to log them all the time in order to simplify things
+        ObjectNode details = json.objectNode();
+        details.put("level", sstable.getSSTableLevel());
+        details.put("min_token", sstable.getFirst().getToken().toString());
+        details.put("max_token", sstable.getLast().getToken().toString());
+        details.put("min_timestamp", sstable.getMinTimestamp());
+        details.put("max_timestamp", sstable.getMaxTimestamp());
+
+        node.put("details", details);
+
         return node;
     }
 
-    private JsonNode startStrategy(AbstractCompactionStrategy strategy)
+    private JsonNode getStrategyDetails(CompactionStrategy strategy)
     {
         ObjectNode node = json.objectNode();
-        CompactionStrategyManager csm = csmRef.get();
-        if (csm == null)
-            return node;
         node.put("strategyId", getId(strategy));
         node.put("type", strategy.getName());
         node.set("tables", formatSSTables(strategy));
-        node.put("repaired", csm.isRepaired(strategy));
-        List<String> folders = csm.getStrategyFolders(strategy);
-        ArrayNode folderNode = json.arrayNode();
-        for (String folder : folders)
-        {
-            folderNode.add(folder);
-        }
-        node.set("folders", folderNode);
-
-        JsonNode logResult = strategy.strategyLogger().options();
-        if (logResult != null)
-            node.set("options", logResult);
         return node;
     }
 
-    private JsonNode shutdownStrategy(AbstractCompactionStrategy strategy)
+    private JsonNode getStrategyId(CompactionStrategy strategy)
     {
         ObjectNode node = json.objectNode();
         node.put("strategyId", getId(strategy));
         return node;
     }
 
-    private JsonNode describeSSTable(AbstractCompactionStrategy strategy, SSTableReader sstable)
+    private JsonNode describeSSTable(SSTableReader sstable)
     {
         ObjectNode node = json.objectNode();
-        node.put("strategyId", getId(strategy));
-        node.set("table", formatSSTable(strategy, sstable));
+        node.put("table", formatSSTable(sstable));
         return node;
     }
 
-    private void describeStrategy(ObjectNode node)
+    private void maybeAddSchemaAndTimeInfo(ObjectNode node)
     {
-        ColumnFamilyStore cfs = cfsRef.get();
-        if (cfs == null)
-            return;
-        node.put("keyspace", cfs.getKeyspaceName());
-        node.put("table", cfs.getTableName());
+        node.put("keyspace", keyspace);
+        node.put("table", table);
         node.put("time", currentTimeMillis());
     }
 
-    private JsonNode startStrategies()
+    private JsonNode getEventJsonNode()
     {
         ObjectNode node = json.objectNode();
         node.put("type", "enable");
-        describeStrategy(node);
-        node.set("strategies", compactionStrategyMap(this::startStrategy));
+        maybeAddSchemaAndTimeInfo(node);
+        node.set("strategies", getStrategiesJsonNode(this::getStrategyDetails));
         return node;
     }
 
@@ -246,7 +247,7 @@ public void enable()
     {
         if (enabled.compareAndSet(false, true))
         {
-            serializer.writeStart(startStrategies(), this);
+            jsonWriter.writeStart(getEventJsonNode(), this);
         }
     }
 
@@ -256,70 +257,151 @@ public void disable()
         {
             ObjectNode node = json.objectNode();
             node.put("type", "disable");
-            describeStrategy(node);
-            node.set("strategies", compactionStrategyMap(this::shutdownStrategy));
-            serializer.write(node, this::startStrategies, this);
+            maybeAddSchemaAndTimeInfo(node);
+            node.set("strategies", getStrategiesJsonNode(this::getStrategyId));
+            jsonWriter.write(node, this::getEventJsonNode, this);
+
+            visitStrategies(strategy -> csvWriters.computeIfPresent(strategy, (s, writers) -> { writers.values().forEach(Writer::close); return null; }));
         }
     }
 
+    public boolean enabled()
+    {
+        return enabled.get();
+    }
+
     public void flush(Collection<SSTableReader> sstables)
     {
         if (enabled.get())
         {
             ObjectNode node = json.objectNode();
             node.put("type", "flush");
-            describeStrategy(node);
-            node.set("tables", sstableMap(sstables, this::describeSSTable));
-            serializer.write(node, this::startStrategies, this);
+            maybeAddSchemaAndTimeInfo(node);
+            node.set("tables", sstableMap(sstables));
+            jsonWriter.write(node, this::getEventJsonNode, this);
         }
     }
 
-    public void compaction(long startTime, Collection<SSTableReader> input, long endTime, Collection<SSTableReader> output)
+    public void compaction(long startTime, Collection<SSTableReader> input, Range<Token> tokenRange, long endTime, Collection<SSTableReader> output)
     {
         if (enabled.get())
         {
             ObjectNode node = json.objectNode();
             node.put("type", "compaction");
-            describeStrategy(node);
+            maybeAddSchemaAndTimeInfo(node);
             node.put("start", String.valueOf(startTime));
             node.put("end", String.valueOf(endTime));
-            node.set("input", sstableMap(input, this::describeSSTable));
-            node.set("output", sstableMap(output, this::describeSSTable));
-            serializer.write(node, this::startStrategies, this);
+            node.set("input", sstableMap(input));
+            node.set("output", sstableMap(output));
+            if (tokenRange != null)
+                node.put("range", tokenRange.toString());
+            jsonWriter.write(node, this::getEventJsonNode, this);
         }
     }
 
-    public void pending(AbstractCompactionStrategy strategy, int remaining)
+    public void pending(CompactionStrategy strategy, int remaining)
     {
         if (remaining != 0 && enabled.get())
         {
             ObjectNode node = json.objectNode();
             node.put("type", "pending");
-            describeStrategy(node);
+            maybeAddSchemaAndTimeInfo(node);
             node.put("strategyId", getId(strategy));
             node.put("pending", remaining);
-            serializer.write(node, this::startStrategies, this);
+            jsonWriter.write(node, this::getEventJsonNode, this);
+        }
+    }
+
+    /**
+     * Write the strategy statistics formatted as CSV.
+     **/
+    public void statistics(CompactionStrategy strategy, String event, CompactionStrategyStatistics statistics)
+    {
+        if (logger.isTraceEnabled())
+            logger.trace("Compaction statistics for strategy {} and event {}: {}", strategy, event, statistics);
+
+        if (!enabled.get())
+            return;
+
+        for (CompactionAggregateStatistics aggregateStatistics : statistics.aggregates())
+        {
+            Writer writer = getCsvWriter(strategy, statistics.getHeader(), aggregateStatistics);
+            writer.write(String.join(",", Iterables.concat(ImmutableList.of(currentTime(), event), aggregateStatistics.data())) + System.lineSeparator());
         }
     }
 
+    private Writer getCsvWriter(CompactionStrategy strategy, Collection<String> header, CompactionAggregateStatistics statistics)
+    {
+        Map<String, Writer> writers = csvWriters.get(strategy);
+        if (writers == null)
+        {
+            writers = new MapMaker().makeMap();
+            if (csvWriters.putIfAbsent(strategy, writers) != null)
+            {
+                writers = csvWriters.get(strategy);
+            }
+        }
+
+        String shard = statistics.shard();
+        Writer writer = writers.get(shard);
+        if (writer != null)
+            return writer;
+
+        String fileName = String.format("compaction-%s-%s-%s-%s",
+                                        strategy.getName(),
+                                        keyspace,
+                                        table,
+                                        getId(strategy));
+
+        if (!shard.isEmpty())
+            fileName += '-' + shard;
+
+        writer = new CompactionLogSerializer(fileName, "csv", loggerService);
+        if (writers.putIfAbsent(shard, writer) == null)
+        {
+            writer.write(String.join(",", Iterables.concat(ImmutableList.of("Timestamp", "Event"), header)) + System.lineSeparator());
+            return writer;
+        }
+        else
+        {
+            writer.close();
+            return writers.get(shard);
+        }
+    }
+
+    private String currentTime()
+    {
+        return dateFormatter.format(Instant.ofEpochMilli(currentTimeMillis()));
+    }
+
     private static class CompactionLogSerializer implements Writer
     {
         private static final String logDirectory = LOG_DIR.getString();
-        private final ExecutorPlus loggerService = executorFactory().sequential("CompactionLogger");
         // This is only accessed on the logger service thread, so it does not need to be thread safe
-        private final Set<Object> rolled = new HashSet<>();
+        private final String fileName;
+        private final String fileExt;
+        private final ExecutorPlus loggerService;
+        private final Set<Object> rolled;
         private OutputStreamWriter stream;
 
-        private static OutputStreamWriter createStream() throws IOException
+        CompactionLogSerializer(String fileName, String fileExt, ExecutorPlus loggerService)
+        {
+            this.fileName = fileName;
+            this.fileExt = fileExt;
+            this.loggerService = loggerService;
+            this.rolled = new HashSet<>();
+        }
+
+        private OutputStreamWriter createStream() throws IOException
         {
             int count = 0;
-            Path compactionLog = new File(logDirectory, "compaction.log").toPath();
+            Path compactionLog = new File(logDirectory,  String.format("%s.%s", fileName, fileExt)).toPath();
             if (Files.exists(compactionLog))
             {
                 Path tryPath = compactionLog;
                 while (Files.exists(tryPath))
                 {
-                    tryPath = new File(logDirectory, String.format("compaction-%d.log", count++)).toPath();
+                    tryPath = new File(logDirectory, String.format("%s-%d.%s", fileName, count++, fileExt)).toPath();
                 }
                 Files.move(compactionLog, tryPath);
             }
@@ -327,50 +409,77 @@ private static OutputStreamWriter createStream() throws IOException
             return new OutputStreamWriter(Files.newOutputStream(compactionLog, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE));
         }
 
-        private void writeLocal(String toWrite)
+        private interface ThrowingConsumer<T>
         {
-            try
-            {
-                if (stream == null)
-                    stream = createStream();
-                stream.write(toWrite);
-                stream.flush();
-            }
-            catch (IOException ioe)
+            void accept(T stream) throws IOException;
+        }
+
+        private void performWrite(ThrowingConsumer<OutputStreamWriter> writeTask)
+        {
+            loggerService.execute(() ->
             {
-                // We'll drop the change and log the error to the logger.
-                NoSpamLogger.log(logger, NoSpamLogger.Level.ERROR, 1, TimeUnit.MINUTES,
-                                 "Could not write to the log file: {}", ioe);
-            }
+              try
+              {
+                  if (stream == null)
+                      stream = createStream();
+
+                  writeTask.accept(stream);
+                  stream.flush();
+              }
+              catch (IOException ioe)
+              {
+                  // We'll drop the change and log the error to the logger.
+                  NoSpamLogger.log(logger, NoSpamLogger.Level.ERROR, 1, TimeUnit.MINUTES,
+                                   "Could not write to the log file: {}", ioe);
+              }
+            });
+        }
 
+        public void write(String toWrite)
+        {
+            performWrite(s -> s.write(toWrite));
         }
 
         public void writeStart(JsonNode statement, Object tag)
         {
             final String toWrite = statement.toString() + System.lineSeparator();
-            loggerService.execute(() -> {
+            performWrite(s -> {
                 rolled.add(tag);
-                writeLocal(toWrite);
+                s.write(toWrite);
             });
         }
 
         public void write(JsonNode statement, StrategySummary summary, Object tag)
         {
             final String toWrite = statement.toString() + System.lineSeparator();
-            loggerService.execute(() -> {
+            performWrite(s -> {
                 if (!rolled.contains(tag))
                 {
-                    writeLocal(summary.getSummary().toString() + System.lineSeparator());
+                    s.write(toWrite);
                     rolled.add(tag);
                 }
-                writeLocal(toWrite);
             });
         }
+
+        public void close()
+        {
+            if (stream != null)
+            {
+                Throwable err = Throwables.close(null, stream);
+                if (err != null)
+                {
+                    JVMStabilityInspector.inspectThrowable(err);
+                    logger.error("Failed to close {}: {}", String.format("%s.%s", fileName, fileExt), err);
+                }
+
+                stream = null;
+            }
+        }
     }
 
     public static void shutdownNowAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
     {
-        ExecutorUtils.shutdownNowAndWait(timeout, unit, serializer.loggerService);
+        ExecutorUtils.shutdownNowAndWait(timeout, unit, jsonWriter.loggerService);
     }
 
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 791a8e3c67ca..4c14be437c15 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.db.compaction;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -29,6 +30,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture; // checkstyle: permit this import
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.RejectedExecutionException;
@@ -37,6 +39,7 @@
 import java.util.function.BooleanSupplier;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import javax.management.openmbean.OpenDataException;
 import javax.management.openmbean.TabularData;
@@ -46,21 +49,20 @@
 import com.google.common.base.Predicates;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Collections2;
-import com.google.common.collect.ConcurrentHashMultiset;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
-import com.google.common.collect.Multiset;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.RateLimiter;
 import com.google.common.util.concurrent.Uninterruptibles;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
-import net.openhft.chronicle.core.util.ThrowingSupplier;
+import com.codahale.metrics.Meter;
+import io.netty.util.concurrent.FastThreadLocal;
 import org.apache.cassandra.cache.AutoSavingCache;
 import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.concurrent.WrappedExecutorPlus;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -70,7 +72,7 @@
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.compaction.CompactionInfo.Holder;
+import org.apache.cassandra.db.compaction.BackgroundCompactionRunner.RequestResult;
 import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
@@ -89,9 +91,12 @@
 import org.apache.cassandra.io.sstable.IScrubber;
 import org.apache.cassandra.io.sstable.IVerifier;
 import org.apache.cassandra.io.sstable.SSTableRewriter;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.util.File;
@@ -102,6 +107,7 @@
 import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.StorageService;
@@ -109,6 +115,7 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MBeanWrapper;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
@@ -120,6 +127,7 @@
 import static java.util.Collections.singleton;
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.concurrent.FutureTask.callable;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMPACTION_RATE_LIMIT_GRANULARITY_IN_KB;
 import static org.apache.cassandra.config.DatabaseDescriptor.getConcurrentCompactors;
 import static org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutor.compactionThreadGroup;
 import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
@@ -141,19 +149,33 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
     private static final Logger logger = LoggerFactory.getLogger(CompactionManager.class);
     public static final CompactionManager instance;
 
-    @VisibleForTesting
-    public final AtomicInteger currentlyBackgroundUpgrading = new AtomicInteger(0);
-
     public static final int NO_GC = Integer.MIN_VALUE;
     public static final int GC_ALL = Integer.MAX_VALUE;
 
+    // A thread local that tells us if the current thread is owned by the compaction manager. Used
+    // by CounterContext to figure out if it should log a warning for invalid counter shards.
+    public static final FastThreadLocal<Boolean> isCompactionManager = new FastThreadLocal<Boolean>()
+    {
+        @Override
+        protected Boolean initialValue()
+        {
+            return false;
+        }
+    };
+    private static final int ACQUIRE_GRANULARITY = COMPACTION_RATE_LIMIT_GRANULARITY_IN_KB.getInt(128) * 1024;
+
     static
     {
         instance = new CompactionManager();
 
         MBeanWrapper.instance.registerMBean(instance, MBEAN_OBJECT_NAME);
-    }
 
+        /*Schedule periodic reports to run every minute*/
+        ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(CompactionManager::periodicReports, 1, 1, TimeUnit.MINUTES);
+
+        /*Store Controller Config for UCS every hour*/
+        ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(CompactionManager::storeControllerConfig, 10, 60, TimeUnit.MINUTES);
+    }
     private final CompactionExecutor executor = new CompactionExecutor();
     private final ValidationExecutor validationExecutor = new ValidationExecutor();
     private final CompactionExecutor cacheCleanupExecutor = new CacheCleanupExecutor();
@@ -166,16 +188,89 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
 
     private final CompactionMetrics metrics = new CompactionMetrics(executor, validationExecutor, viewBuildExecutor, secondaryIndexExecutor);
 
-    @VisibleForTesting
-    final Multiset<ColumnFamilyStore> compactingCF = ConcurrentHashMultiset.create();
+    public final ActiveOperations active = new ActiveOperations();
 
-    public final ActiveCompactions active = new ActiveCompactions();
+    private final BackgroundCompactionRunner backgroundCompactionRunner = new BackgroundCompactionRunner(executor, active);
 
     // used to temporarily pause non-strategy managed compactions (like index summary redistribution)
     private final AtomicInteger globalCompactionPauseCount = new AtomicInteger(0);
 
     private final RateLimiter compactionRateLimiter = RateLimiter.create(Double.MAX_VALUE);
 
+    protected static void periodicReports()
+    {
+        if (!Keyspace.isInitialized())
+            return;
+
+        for (String keyspace : Schema.instance.getKeyspaces())
+        {
+            if (Schema.instance.getKeyspaceInstance(keyspace) != null)
+            {
+                for (ColumnFamilyStore cfs : Schema.instance.getKeyspaceInstance(keyspace).getColumnFamilyStores())
+                {
+                    CompactionStrategy strat = cfs.getCompactionStrategy();
+                    strat.periodicReport();
+                }
+            }
+        }
+    }
+
+    @VisibleForTesting
+    public static void storeControllerConfig()
+    {
+        /*Delete any controller-config.JSON files that correspond to a table that no longer exists*/
+        if (!Keyspace.isInitialized())
+            return;
+
+        cleanupControllerConfig();
+
+        for (String keyspace : Schema.instance.getKeyspaces())
+        {
+            //don't store config files for system tables
+            if (Schema.instance.getKeyspaceInstance(keyspace) != null && !SchemaConstants.isSystemKeyspace(keyspace))
+            {
+                for (ColumnFamilyStore cfs : Schema.instance.getKeyspaceInstance(keyspace).getColumnFamilyStores())
+                {
+                    CompactionStrategy strat = cfs.getCompactionStrategy();
+                    if (strat instanceof UnifiedCompactionContainer)
+                    {
+                        UnifiedCompactionStrategy ucs = (UnifiedCompactionStrategy) ((UnifiedCompactionContainer) strat).getStrategies().get(0);
+                        ucs.storeControllerConfig();
+                    }
+                }
+            }
+        }
+    }
+
+    @VisibleForTesting
+    public static void cleanupControllerConfig()
+    {
+        Pattern fileNamePattern = Pattern.compile("-controller-config.JSON", Pattern.LITERAL);
+        Pattern keyspaceNameSeparator = Pattern.compile("\\.");
+        File dir = DatabaseDescriptor.getMetadataDirectory();
+        if (dir != null)
+        {
+            for (File file : dir.tryList())
+            {
+                if (file.name().contains("-controller-config.JSON"))
+                {
+                    String[] names = keyspaceNameSeparator.split(fileNamePattern.matcher(file.name()).replaceAll(""));
+                    try
+                    {
+                        //table exists so keep the file
+                        Schema.instance.getKeyspaceInstance(names[0]).getColumnFamilyStore(names[1]);
+                    }
+                    catch(NullPointerException e)
+                    {
+                        //table does not exist so delete the file
+                        logger.debug("Removing " + file + " because it does not correspond to an existing table");
+                        file.delete();
+                    }
+                }
+            }
+        }
+    }
+
     public CompactionMetrics getMetrics()
     {
         return metrics;
@@ -220,45 +315,39 @@ public void setRateInBytes(final double throughputBytesPerSec)
             compactionRateLimiter.setRate(throughput);
     }
 
+    public Meter getCompactionThroughput()
+    {
+        return metrics.bytesCompactedThroughput;
+    }
+
     /**
-     * Call this whenever a compaction might be needed on the given columnfamily.
+     * Call this whenever a compaction might be needed on the given column family store.
      * It's okay to over-call (within reason) if a call is unnecessary, it will
      * turn into a no-op in the bucketing/candidate-scan phase.
      */
-    public List<Future<?>> submitBackground(final ColumnFamilyStore cfs)
+    public Future<RequestResult> submitBackground(final ColumnFamilyStore cfs)
     {
-        if (cfs.isAutoCompactionDisabled())
-        {
-            logger.trace("Autocompaction is disabled");
-            return Collections.emptyList();
-        }
+        return backgroundCompactionRunner.markForCompactionCheck(cfs);
+    }
 
-        /**
-         * If a CF is currently being compacted, and there are no idle threads, submitBackground should be a no-op;
-         * we can wait for the current compaction to finish and re-submit when more information is available.
-         * Otherwise, we should submit at least one task to prevent starvation by busier CFs, and more if there
-         * are idle threads stil. (CASSANDRA-4310)
-         */
-        int count = compactingCF.count(cfs);
-        if (count > 0 && executor.getActiveTaskCount() >= executor.getMaximumPoolSize())
-        {
-            logger.trace("Background compaction is still running for {}.{} ({} remaining). Skipping",
-                         cfs.getKeyspaceName(), cfs.name, count);
-            return Collections.emptyList();
-        }
+    public void submitBackground(Set<ColumnFamilyStore> cfss)
+    {
+        backgroundCompactionRunner.markForCompactionCheck(cfss);
+    }
 
-        logger.trace("Scheduling a background task check for {}.{} with {}",
-                     cfs.getKeyspaceName(),
-                     cfs.name,
-                     cfs.getCompactionStrategyManager().getName());
+    public int getOngoingBackgroundCompactionsCount()
+    {
+        return backgroundCompactionRunner.getOngoingCompactionsCount();
+    }
 
-        List<Future<?>> futures = new ArrayList<>(1);
-        Future<?> fut = executor.submitIfRunning(new BackgroundCompactionCandidate(cfs), "background task");
-        if (!fut.isCancelled())
-            futures.add(fut);
-        else
-            compactingCF.remove(cfs);
-        return futures;
+    public CompletableFuture<?>[] startCompactionTasks(ColumnFamilyStore cfs, Collection<AbstractCompactionTask> tasks)
+    {
+        return backgroundCompactionRunner.startCompactionTasks(cfs, tasks);
+    }
+
+    public int getOngoingBackgroundUpgradesCount()
+    {
+        return backgroundCompactionRunner.getOngoingUpgradesCount();
     }
 
     public boolean isCompacting(Iterable<ColumnFamilyStore> cfses, Predicate<SSTableReader> sstablePredicate)
@@ -272,7 +361,7 @@ public boolean isCompacting(Iterable<ColumnFamilyStore> cfses, Predicate<SSTable
     @VisibleForTesting
     public boolean hasOngoingOrPendingTasks()
     {
-        if (!active.getCompactions().isEmpty() || !compactingCF.isEmpty())
+        if (!active.getTableOperations().isEmpty())
             return true;
 
         int pendingTasks = executor.getPendingTaskCount() +
@@ -299,6 +388,7 @@ public boolean hasOngoingOrPendingTasks()
     public void forceShutdown()
     {
         // shutdown executors to prevent further submission
+        backgroundCompactionRunner.shutdown();
         executor.shutdown();
         validationExecutor.shutdown();
         viewBuildExecutor.shutdown();
@@ -306,9 +396,9 @@ public void forceShutdown()
         secondaryIndexExecutor.shutdown();
 
         // interrupt compactions and validations
-        for (Holder compactionHolder : active.getCompactions())
+        for (TableOperation operationSource : active.getTableOperations())
         {
-            compactionHolder.stop();
+            operationSource.stop(TableOperation.StopTrigger.SHUTDOWN);
         }
 
         // wait for tasks to terminate
@@ -331,86 +421,11 @@ public void forceShutdown()
 
     public void finishCompactionsAndShutdown(long timeout, TimeUnit unit) throws InterruptedException
     {
+        backgroundCompactionRunner.shutdown();
         executor.shutdown();
         executor.awaitTermination(timeout, unit);
     }
 
-    // the actual sstables to compact are not determined until we run the BCT; that way, if new sstables
-    // are created between task submission and execution, we execute against the most up-to-date information
-    @VisibleForTesting
-    class BackgroundCompactionCandidate implements Runnable
-    {
-        private final ColumnFamilyStore cfs;
-
-        BackgroundCompactionCandidate(ColumnFamilyStore cfs)
-        {
-            compactingCF.add(cfs);
-            this.cfs = cfs;
-        }
-
-        public void run()
-        {
-            boolean ranCompaction = false;
-            try
-            {
-                logger.trace("Checking {}.{}", cfs.getKeyspaceName(), cfs.name);
-                if (!cfs.isValid())
-                {
-                    logger.trace("Aborting compaction for dropped CF");
-                    return;
-                }
-
-                CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
-                AbstractCompactionTask task = strategy.getNextBackgroundTask(getDefaultGcBefore(cfs, FBUtilities.nowInSeconds()));
-                if (task == null)
-                {
-                    if (DatabaseDescriptor.automaticSSTableUpgrade())
-                        ranCompaction = maybeRunUpgradeTask(strategy);
-                }
-                else
-                {
-                    task.execute(active);
-                    ranCompaction = true;
-                }
-            }
-            finally
-            {
-                compactingCF.remove(cfs);
-            }
-            if (ranCompaction) // only submit background if we actually ran a compaction - otherwise we end up in an infinite loop submitting noop background tasks
-                submitBackground(cfs);
-        }
-
-        boolean maybeRunUpgradeTask(CompactionStrategyManager strategy)
-        {
-            logger.debug("Checking for upgrade tasks {}.{}", cfs.getKeyspaceName(), cfs.getTableName());
-            try
-            {
-                if (currentlyBackgroundUpgrading.incrementAndGet() <= DatabaseDescriptor.maxConcurrentAutoUpgradeTasks())
-                {
-                    AbstractCompactionTask upgradeTask = strategy.findUpgradeSSTableTask();
-                    if (upgradeTask != null)
-                    {
-                        upgradeTask.execute(active);
-                        return true;
-                    }
-                }
-            }
-            finally
-            {
-                currentlyBackgroundUpgrading.decrementAndGet();
-            }
-            logger.trace("No tasks available");
-            return false;
-        }
-    }
-
-    @VisibleForTesting
-    public BackgroundCompactionCandidate getBackgroundCompactionCandidate(ColumnFamilyStore cfs)
-    {
-        return new BackgroundCompactionCandidate(cfs);
-    }
-
     /**
      * Run an operation over all sstables using jobs threads
      *
@@ -422,12 +437,13 @@ public BackgroundCompactionCandidate getBackgroundCompactionCandidate(ColumnFami
     private AllSSTableOpStatus parallelAllSSTableOperation(final ColumnFamilyStore cfs,
                                                            final OneSSTableOperation operation,
                                                            int jobs,
-                                                           OperationType operationType)
+                                                           OperationType operationType,
+                                                           TableOperation.StopTrigger trigger)
     {
         String operationName = operationType.name();
         String keyspace = cfs.getKeyspaceName();
         String table = cfs.getTableName();
-        return cfs.withAllSSTables(operationType, (compacting) -> {
+        return cfs.withAllSSTables(operationType, trigger, (compacting) -> {
             logger.info("Starting {} for {}.{}", operationType, cfs.getKeyspaceName(), cfs.getTableName());
             List<LifecycleTransaction> transactions = new ArrayList<>();
             List<Future<?>> futures = new ArrayList<>();
@@ -526,7 +542,7 @@ public void execute(LifecycleTransaction input)
             {
                 scrubOne(cfs, input, options, active);
             }
-        }, jobs, OperationType.SCRUB);
+        }, jobs, OperationType.SCRUB, TableOperation.StopTrigger.SCRUB);
     }
 
     public AllSSTableOpStatus performVerify(ColumnFamilyStore cfs, IVerifier.Options options) throws InterruptedException, ExecutionException
@@ -545,7 +561,7 @@ public void execute(LifecycleTransaction input)
             {
                 verifyOne(cfs, input.onlyOne(), options, active);
             }
-        }, 0, OperationType.VERIFY);
+        }, 0, OperationType.VERIFY, TableOperation.StopTrigger.VERIFY);
     }
 
     public AllSSTableOpStatus performSSTableRewrite(final ColumnFamilyStore cfs,
@@ -587,7 +603,7 @@ public AllSSTableOpStatus performSSTableRewrite(final ColumnFamilyStore cfs, Pre
             public Iterable<SSTableReader> filterSSTables(LifecycleTransaction transaction)
             {
                 List<SSTableReader> sortedSSTables = Lists.newArrayList(transaction.originals());
-                Collections.sort(sortedSSTables, SSTableReader.sizeComparator.reversed());
+                Collections.sort(sortedSSTables, CompactionSSTable.sizeComparator.reversed());
                 Iterator<SSTableReader> iter = sortedSSTables.iterator();
                 while (iter.hasNext())
                 {
@@ -604,12 +620,12 @@ public Iterable<SSTableReader> filterSSTables(LifecycleTransaction transaction)
             @Override
             public void execute(LifecycleTransaction txn)
             {
-                AbstractCompactionTask task = cfs.getCompactionStrategyManager().getCompactionTask(txn, NO_GC, Long.MAX_VALUE);
+                AbstractCompactionTask task = cfs.getCompactionStrategy().createCompactionTask(txn, NO_GC, Long.MAX_VALUE);
                 task.setUserDefined(true);
                 task.setCompactionType(OperationType.UPGRADE_SSTABLES);
                 task.execute(active);
             }
-        }, jobs, OperationType.UPGRADE_SSTABLES);
+        }, jobs, OperationType.UPGRADE_SSTABLES, TableOperation.StopTrigger.UPGRADE_SSTABLES);
     }
 
     public AllSSTableOpStatus performCleanup(final ColumnFamilyStore cfStore, int jobs) throws InterruptedException, ExecutionException
@@ -661,7 +677,7 @@ public Iterable<SSTableReader> filterSSTables(LifecycleTransaction transaction)
                 }
                 logger.info("Skipping cleanup for {}/{} sstables for {}.{} since they are fully contained in owned ranges (full ranges: {}, transient ranges: {})",
                             skippedSStables, totalSSTables, cfStore.getKeyspaceName(), cfStore.getTableName(), fullRanges, transientRanges);
-                sortedSSTables.sort(SSTableReader.sizeComparator);
+                sortedSSTables.sort(CompactionSSTable.sizeComparator);
                 return sortedSSTables;
             }
 
@@ -671,7 +687,7 @@ public void execute(LifecycleTransaction txn) throws IOException
                 CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfStore, allRanges, transientRanges, txn.onlyOne().isRepaired(), FBUtilities.nowInSeconds());
                 doCleanupOne(cfStore, txn, cleanupStrategy, replicas.ranges(), hasIndexes);
             }
-        }, jobs, OperationType.CLEANUP);
+        }, jobs, OperationType.CLEANUP, TableOperation.StopTrigger.CLEANUP);
     }
 
     public AllSSTableOpStatus performGarbageCollection(final ColumnFamilyStore cfStore, TombstoneOption tombstoneOption, int jobs) throws InterruptedException, ExecutionException
@@ -683,60 +699,25 @@ public AllSSTableOpStatus performGarbageCollection(final ColumnFamilyStore cfSto
             @Override
             public Iterable<SSTableReader> filterSSTables(LifecycleTransaction transaction)
             {
-                List<SSTableReader> filteredSSTables = new ArrayList<>();
-                if (cfStore.getCompactionStrategyManager().onlyPurgeRepairedTombstones())
-                {
-                    for (SSTableReader sstable : transaction.originals())
-                    {
-                        if (!sstable.isRepaired())
-                        {
-                            try
-                            {
-                                transaction.cancel(sstable);
-                            }
-                            catch (Throwable t)
-                            {
-                                logger.warn(String.format("Unable to cancel %s from transaction %s", sstable, transaction.opId()), t);
-                            }
-                        }
-                        else
-                        {
-                            filteredSSTables.add(sstable);
-                        }
-                    }
-                }
-                else
-                {
-                    filteredSSTables.addAll(transaction.originals());
-                }
-
-                filteredSSTables.sort(SSTableReader.maxTimestampAscending);
-                return filteredSSTables;
+                Iterable<SSTableReader> originals = transaction.originals();
+                if (cfStore.onlyPurgeRepairedTombstones())
+                    originals = Iterables.filter(originals, SSTableReader::isRepaired);
+                List<SSTableReader> sortedSSTables = Lists.newArrayList(originals);
+                Collections.sort(sortedSSTables, SSTableReader.maxTimestampAscending);
+                return sortedSSTables;
             }
 
             @Override
-            public void execute(LifecycleTransaction txn) throws IOException
+            public void execute(LifecycleTransaction txn)
             {
                 logger.debug("Garbage collecting {}", txn.originals());
-                CompactionTask task = new CompactionTask(cfStore, txn, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()))
-                {
-                    @Override
-                    protected CompactionController getCompactionController(Set<SSTableReader> toCompact)
-                    {
-                        return new CompactionController(cfStore, toCompact, gcBefore, null, tombstoneOption);
-                    }
-
-                    @Override
-                    protected int getLevel()
-                    {
-                        return txn.onlyOne().getSSTableLevel();
-                    }
-                };
-                task.setUserDefined(true);
-                task.setCompactionType(OperationType.GARBAGE_COLLECT);
+                AbstractCompactionTask task = CompactionTask.forGarbageCollection(cfStore,
+                                                                                  txn,
+                                                                                  getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()),
+                                                                                  tombstoneOption);
                 task.execute(active);
             }
-        }, jobs, OperationType.GARBAGE_COLLECT);
+        }, jobs, OperationType.GARBAGE_COLLECT, TableOperation.StopTrigger.GARBAGE_COLLECT);
     }
 
     public AllSSTableOpStatus relocateSSTables(final ColumnFamilyStore cfs, int jobs) throws ExecutionException, InterruptedException
@@ -782,7 +763,7 @@ public Iterable<SSTableReader> filterSSTables(LifecycleTransaction transaction)
 
             public Map<Integer, List<SSTableReader>> groupByDiskIndex(Set<SSTableReader> needsRelocation)
             {
-                return needsRelocation.stream().collect(Collectors.groupingBy((s) -> diskBoundaries.getDiskIndex(s)));
+                return needsRelocation.stream().collect(Collectors.groupingBy((s) -> diskBoundaries.getDiskIndexFromKey(s)));
             }
 
             private boolean inCorrectLocation(SSTableReader sstable)
@@ -799,12 +780,12 @@ private boolean inCorrectLocation(SSTableReader sstable)
             public void execute(LifecycleTransaction txn)
             {
                 logger.debug("Relocating {}", txn.originals());
-                AbstractCompactionTask task = cfs.getCompactionStrategyManager().getCompactionTask(txn, NO_GC, Long.MAX_VALUE);
+                AbstractCompactionTask task = cfs.getCompactionStrategy().createCompactionTask(txn, NO_GC, Long.MAX_VALUE);
                 task.setUserDefined(true);
                 task.setCompactionType(OperationType.RELOCATE);
                 task.execute(active);
             }
-        }, jobs, OperationType.RELOCATE);
+        }, jobs, OperationType.RELOCATE, TableOperation.StopTrigger.RELOCATE);
     }
 
     /**
@@ -863,8 +844,8 @@ private static void mutateFullyContainedSSTables(ColumnFamilyStore cfs,
 
         Set<SSTableReader> fullyContainedSSTables = findSSTablesToAnticompact(sstableIterator, normalizedRanges, sessionID);
 
-        cfs.metric.bytesMutatedAnticompaction.inc(SSTableReader.getTotalBytes(fullyContainedSSTables));
-        cfs.getCompactionStrategyManager().mutateRepaired(fullyContainedSSTables, UNREPAIRED_SSTABLE, sessionID, isTransient);
+        cfs.metric.bytesMutatedAnticompaction.inc(CompactionSSTable.getTotalDataBytes(fullyContainedSSTables));
+        cfs.mutateRepaired(fullyContainedSSTables, UNREPAIRED_SSTABLE, sessionID, isTransient);
         // since we're just re-writing the sstable metdata for the fully contained sstables, we don't want
         // them obsoleted when the anti-compaction is complete. So they're removed from the transaction here
         txn.cancel(fullyContainedSSTables);
@@ -900,7 +881,7 @@ public void performAnticompaction(ColumnFamilyStore cfs,
             }
             catch (NoSuchRepairSessionException e)
             {
-                throw new CompactionInterruptedException(e.getMessage());
+                throw new CompactionInterruptedException(e.getMessage(), TableOperation.StopTrigger.ANTICOMPACTION);
             }
             Preconditions.checkArgument(!prs.isPreview(), "Cannot anticompact for previews");
             Preconditions.checkArgument(!replicas.isEmpty(), "No ranges to anti-compact");
@@ -965,7 +946,7 @@ static Set<SSTableReader> findSSTablesToAnticompact(Iterator<SSTableReader> ssta
                 // ranges are normalized - no wrap around - if first and last are contained we know that all tokens are contained in the range
                 if (r.contains(sstable.getFirst().getToken()) && r.contains(sstable.getLast().getToken()))
                 {
-                    logger.info("{} SSTable {} fully contained in range {}, mutating repairedAt instead of anticompacting", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, r);
+                    logger.info("{} SSTable {} fully contained in range {}, mutating repairedAt to unrepaired instead of anticompacting", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, r);
                     fullyContainedSSTables.add(sstable);
                     sstableIterator.remove();
                     break;
@@ -984,17 +965,45 @@ public void performMaximal(final ColumnFamilyStore cfStore, boolean splitOutput)
         FBUtilities.waitOnFutures(submitMaximal(cfStore, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()), splitOutput));
     }
 
+    public void performMaximal(final ColumnFamilyStore cfStore, boolean splitOutput, int parallelism)
+    {
+        FBUtilities.waitOnFutures(submitMaximal(cfStore, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()), splitOutput, parallelism, active, OperationType.MAJOR_COMPACTION));
+    }
+
     public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore, final long gcBefore, boolean splitOutput)
     {
-            return submitMaximal(cfStore, gcBefore, splitOutput, OperationType.MAJOR_COMPACTION);
+            return submitMaximal(cfStore, gcBefore, splitOutput, active, OperationType.MAJOR_COMPACTION);
     }
 
-    public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore, final long gcBefore, boolean splitOutput, OperationType operationType)
+    public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore,
+                                         final long gcBefore,
+                                         boolean splitOutput,
+                                         TableOperationObserver obs,
+                                         OperationType operationType)
     {
+        return submitMaximal(cfStore, gcBefore, splitOutput, -1, obs, operationType);
+    }
+
+    @VisibleForTesting
+    @SuppressWarnings("resource") // the tasks are executed in parallel on the executor, making sure that they get closed
+    public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore,
+                                         final long gcBefore,
+                                         boolean splitOutput,
+                                         int permittedParallelism,
+                                         TableOperationObserver obs,
+                                         OperationType operationType)
+    {
+        // The default parallelism is half the number of compaction threads to leave enough room for other compactions.
+        if (permittedParallelism < 0)
+            permittedParallelism = getCoreCompactorThreads() / 2;
+        else if (permittedParallelism == 0)
+            permittedParallelism = Integer.MAX_VALUE;
+
         // here we compute the task off the compaction executor, so having that present doesn't
         // confuse runWithCompactionsDisabled -- i.e., we don't want to deadlock ourselves, waiting
         // for ourselves to finish/acknowledge cancellation before continuing.
-        CompactionTasks tasks = cfStore.getCompactionStrategyManager().getMaximalTasks(gcBefore, splitOutput, operationType);
+
+        CompactionTasks tasks = cfStore.getCompactionStrategy().getMaximalTasks(gcBefore, splitOutput, permittedParallelism, operationType);
 
         if (tasks.isEmpty())
             return Collections.emptyList();
@@ -1011,16 +1020,25 @@ public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore, final long
             {
                 protected void runMayThrow()
                 {
-                    task.execute(active);
+                    task.execute(obs);
                 }
             };
 
             Future<?> fut = executor.submitIfRunning(runnable, "maximal task");
             if (!fut.isCancelled())
                 futures.add(fut);
+            else
+            {
+                Throwable error = task.rejected(new RejectedExecutionException("rejected by executor"));
+                if (error != null)
+                    futures.add(ImmediateFuture.failure(error));
+            }
         }
         if (nonEmptyTasks > 1)
-            logger.info("Major compaction will not result in a single sstable - repaired and unrepaired data is kept separate and compaction runs per data_file_directory.");
+            logger.info("Major compaction of {}.{} will not result in a single sstable - " +
+                        "repaired and unrepaired data is kept separate, compaction runs per data_file_directory, " +
+                        "and some compaction strategies will construct multiple non-overlapping sstables.",
+                        cfStore.getKeyspaceName(), cfStore.getTableName());
 
         return futures;
     }
@@ -1034,7 +1052,7 @@ public void forceCompaction(ColumnFamilyStore cfStore, Supplier<Collection<SSTab
                 logger.debug("No sstables found for the provided token range");
                 return CompactionTasks.empty();
             }
-            return cfStore.getCompactionStrategyManager().getUserDefinedTasks(sstables, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()));
+            return cfStore.getCompactionStrategy().getUserDefinedTasks(sstables, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()));
         };
 
         try (CompactionTasks tasks = cfStore.runWithCompactionsDisabled(taskCreator,
@@ -1042,7 +1060,8 @@ public void forceCompaction(ColumnFamilyStore cfStore, Supplier<Collection<SSTab
                                                                         OperationType.MAJOR_COMPACTION,
                                                                         false,
                                                                         false,
-                                                                        false))
+                                                                        false,
+                                                                        TableOperation.StopTrigger.COMPACTION))
         {
             if (tasks.isEmpty())
                 return;
@@ -1262,7 +1281,7 @@ protected void runMayThrow() throws Exception
                 }
                 else
                 {
-                    try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstables, gcBefore))
+                    try (CompactionTasks tasks = cfs.getCompactionStrategy().getUserDefinedTasks(sstables, gcBefore))
                     {
                         for (AbstractCompactionTask task : tasks)
                         {
@@ -1305,38 +1324,24 @@ public void disableAutoCompaction()
     }
 
     @VisibleForTesting
-    void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, IScrubber.Options options, ActiveCompactionsTracker activeCompactions)
+    void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, IScrubber.Options options, TableOperationObserver activeCompactions)
     {
-        CompactionInfo.Holder scrubInfo = null;
         SSTableFormat format = modifier.onlyOne().descriptor.getFormat();
-        try (IScrubber scrubber = format.getScrubber(cfs, modifier, new OutputHandler.LogOutput(), options))
+        try (IScrubber scrubber = format.getScrubber(cfs, modifier, new OutputHandler.LogOutput(), options);
+             NonThrowingCloseable c = activeCompactions.onOperationStart(scrubber.getScrubInfo()))
         {
-            scrubInfo = scrubber.getScrubInfo();
-            activeCompactions.beginCompaction(scrubInfo);
             scrubber.scrub();
         }
-        finally
-        {
-            if (scrubInfo != null)
-                activeCompactions.finishCompaction(scrubInfo);
-        }
     }
 
     @VisibleForTesting
-    void verifyOne(ColumnFamilyStore cfs, SSTableReader sstable, IVerifier.Options options, ActiveCompactionsTracker activeCompactions)
+    void verifyOne(ColumnFamilyStore cfs, SSTableReader sstable, IVerifier.Options options, TableOperationObserver activeCompactions)
     {
-        CompactionInfo.Holder verifyInfo = null;
-        try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, options))
+        try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, options);
+             NonThrowingCloseable c = activeCompactions.onOperationStart(verifier.getVerifyInfo()))
         {
-            verifyInfo = verifier.getVerifyInfo();
-            activeCompactions.beginCompaction(verifyInfo);
             verifier.verify();
         }
-        finally
-        {
-            if (verifyInfo != null)
-                activeCompactions.finishCompaction(verifyInfo);
-        }
     }
 
     /**
@@ -1444,8 +1449,8 @@ private void doCleanupOne(final ColumnFamilyStore cfs,
         try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge);
              ISSTableScanner scanner = cleanupStrategy.getScanner(sstable);
              CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
-             Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable));
-             CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, nextTimeUUID(), active, null))
+             Refs<SSTableReader> refs = Refs.ref(singleton(sstable));
+             CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, nextTimeUUID()))
         {
             StatsMetadata metadata = sstable.getSSTableMetadata();
             writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, txn));
@@ -1465,9 +1470,8 @@ private void doCleanupOne(final ColumnFamilyStore cfs,
 
                     long bytesScanned = scanner.getBytesScanned();
 
-                    compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
-
-                    lastBytesScanned = bytesScanned;
+                    if (compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio))
+                        lastBytesScanned = bytesScanned;
                 }
             }
 
@@ -1492,9 +1496,22 @@ private void doCleanupOne(final ColumnFamilyStore cfs,
 
     }
 
-    static void compactionRateLimiterAcquire(RateLimiter limiter, long bytesScanned, long lastBytesScanned, double compressionRatio)
+    protected boolean compactionRateLimiterAcquire(RateLimiter limiter, long bytesScanned, long lastBytesScanned, double compressionRatio)
     {
+        if (DatabaseDescriptor.getCompactionThroughputMebibytesPerSecAsInt() == 0)
+            return false;
+
         long lengthRead = (long) ((bytesScanned - lastBytesScanned) * compressionRatio) + 1;
+        // Acquire at 128k granularity. At worst we'll exceed the limit a bit, but acquire is quite expensive.
+        if (lengthRead < ACQUIRE_GRANULARITY)
+            return false;
+
+        return actuallyAcquire(limiter, lengthRead);
+    }
+
+    private boolean actuallyAcquire(RateLimiter limiter, long lengthRead)
+    {
+        metrics.bytesCompactedThroughput.mark(lengthRead);
         while (lengthRead >= Integer.MAX_VALUE)
         {
             limiter.acquire(Integer.MAX_VALUE);
@@ -1504,6 +1521,7 @@ static void compactionRateLimiterAcquire(RateLimiter limiter, long bytesScanned,
         {
             limiter.acquire((int) lengthRead);
         }
+        return true;
     }
 
     private static abstract class CleanupStrategy
@@ -1606,7 +1624,7 @@ public UnfilteredRowIterator cleanup(UnfilteredRowIterator partition)
         }
     }
 
-    public static SSTableWriter createWriter(ColumnFamilyStore cfs,
+    public static SSTableWriter createWriter(CompactionRealm cfs,
                                              File compactionFileLocation,
                                              long expectedBloomFilterSize,
                                              long repairedAt,
@@ -1623,11 +1641,11 @@ public static SSTableWriter createWriter(ColumnFamilyStore cfs,
                          .setRepairedAt(repairedAt)
                          .setPendingRepair(pendingRepair)
                          .setTransientSSTable(isTransient)
-                         .setTableMetadataRef(cfs.metadata)
+                         .setTableMetadataRef(cfs.metadataRef())
                          .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator).sstableLevel(sstable.getSSTableLevel()))
                          .setSerializationHeader(sstable.header)
-                         .addDefaultComponents(cfs.indexManager.listIndexGroups())
-                         .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups())
+                         .addDefaultComponents(cfs.getIndexManager().listIndexGroups())
+                         .setSecondaryIndexGroups(cfs.getIndexManager().listIndexGroups())
                          .build(txn, cfs);
     }
 
@@ -1678,7 +1696,7 @@ public static SSTableWriter createWriterForAntiCompaction(ColumnFamilyStore cfs,
      * @param cfs
      * @param txn a transaction over the repaired sstables to anticompact
      * @param ranges full and transient ranges to be placed into one of the new sstables. The repaired table will be tracked via
-     *   the {@link org.apache.cassandra.io.sstable.metadata.StatsMetadata#pendingRepair} field.
+     *   the {@link StatsMetadata#pendingRepair} field.
      * @param pendingRepair the repair session we're anti-compacting for
      * @param isCancelled function that indicates if active anti-compaction should be canceled
      */
@@ -1699,14 +1717,14 @@ private void doAntiCompaction(ColumnFamilyStore cfs,
         // repairedAt values for these, we still avoid anti-compacting already repaired sstables, as we currently don't
         // make use of any actual repairedAt value and splitting up sstables just for that is not worth it at this point.
         Set<SSTableReader> unrepairedSSTables = sstables.stream().filter((s) -> !s.isRepaired()).collect(Collectors.toSet());
-        cfs.metric.bytesAnticompacted.inc(SSTableReader.getTotalBytes(unrepairedSSTables));
-        Collection<Collection<SSTableReader>> groupedSSTables = cfs.getCompactionStrategyManager().groupSSTablesForAntiCompaction(unrepairedSSTables);
+        cfs.metric.bytesAnticompacted.inc(CompactionSSTable.getTotalDataBytes(unrepairedSSTables));
+        Collection<Collection<CompactionSSTable>> groupedSSTables = cfs.getCompactionStrategy().groupSSTablesForAntiCompaction(unrepairedSSTables);
 
         // iterate over sstables to check if the full / transient / unrepaired ranges intersect them.
         int antiCompactedSSTableCount = 0;
-        for (Collection<SSTableReader> sstableGroup : groupedSSTables)
+        for (Collection<CompactionSSTable> sstableGroup : groupedSSTables)
         {
-            try (LifecycleTransaction groupTxn = txn.split(sstableGroup))
+            try (LifecycleTransaction groupTxn = txn.split(Collections2.transform(sstableGroup, SSTableReader.class::cast)))
             {
                 int antiCompacted = antiCompactGroup(cfs, ranges, groupTxn, pendingRepair, isCancelled);
                 antiCompactedSSTableCount += antiCompacted;
@@ -1780,55 +1798,55 @@ public void obsoleteOriginals() {}
             public void close() {}
         }
 
-        CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
+        CompactionStrategy strategy = cfs.getCompactionStrategy();
         try (SharedTxn sharedTxn = new SharedTxn(txn);
              SSTableRewriter fullWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
              SSTableRewriter transWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
              SSTableRewriter unrepairedWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
 
-             AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(txn.originals());
+             ScannerList scanners = strategy.getScanners(txn.originals());
              CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs, nowInSec));
-             CompactionIterator ci = getAntiCompactionIterator(scanners.scanners, controller, nowInSec, nextTimeUUID(), active, isCancelled))
+             CompactionIterator ci = getAntiCompactionIterator(scanners.scanners, controller, nowInSec, nextTimeUUID(), isCancelled))
         {
-            int expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, (int)(SSTableReader.getApproximateKeyCount(sstableAsSet)));
-
-            fullWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, false, sstableAsSet, txn));
-            transWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, true, sstableAsSet, txn));
-            unrepairedWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, sstableAsSet, txn));
+            TableOperation op = ci.getOperation();
+            try (NonThrowingCloseable cls = active.onOperationStart(op))
+            {
+                int expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, (int)(SSTableReader.getApproximateKeyCount(sstableAsSet)));
 
-            Predicate<Token> fullChecker = !ranges.onlyFull().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.onlyFull().ranges()) : t -> false;
-            Predicate<Token> transChecker = !ranges.onlyTransient().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.onlyTransient().ranges()) : t -> false;
-            double compressionRatio = scanners.getCompressionRatio();
-            if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
-                compressionRatio = 1.0;
+                fullWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, false, sstableAsSet, txn));
+                transWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, true, sstableAsSet, txn));
+                unrepairedWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, sstableAsSet, txn));
 
-            long lastBytesScanned = 0;
+                Predicate<Token> fullChecker = !ranges.onlyFull().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.onlyFull().ranges()) : t -> false;
+                Predicate<Token> transChecker = !ranges.onlyTransient().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.onlyTransient().ranges()) : t -> false;
+                double compressionRatio = scanners.getCompressionRatio();
+                if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
+                    compressionRatio = 1.0;
 
-            while (ci.hasNext())
-            {
-                try (UnfilteredRowIterator partition = ci.next())
+                long lastBytesScanned = 0;
+                while (ci.hasNext())
                 {
-                    Token token = partition.partitionKey().getToken();
-                    // if this row is contained in the full or transient ranges, append it to the appropriate sstable
-                    if (fullChecker.test(token))
-                    {
-                        fullWriter.append(partition);
-                        ci.setTargetDirectory(fullWriter.currentWriter().getFilename());
-                    }
-                    else if (transChecker.test(token))
+                    try (UnfilteredRowIterator partition = ci.next())
                     {
-                        transWriter.append(partition);
-                        ci.setTargetDirectory(transWriter.currentWriter().getFilename());
-                    }
-                    else
-                    {
-                        // otherwise, append it to the unrepaired sstable
-                        unrepairedWriter.append(partition);
-                        ci.setTargetDirectory(unrepairedWriter.currentWriter().getFilename());
+                        Token token = partition.partitionKey().getToken();
+                        // if this row is contained in the full or transient ranges, append it to the appropriate sstable
+                        if (fullChecker.test(token))
+                        {
+                            fullWriter.append(partition);
+                        }
+                        else if (transChecker.test(token))
+                        {
+                            transWriter.append(partition);
+                        }
+                        else
+                        {
+                            // otherwise, append it to the unrepaired sstable
+                            unrepairedWriter.append(partition);
+                        }
+                        long bytesScanned = scanners.getTotalBytesScanned();
+                        if (compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio))
+                            lastBytesScanned = bytesScanned;
                     }
-                    long bytesScanned = scanners.getTotalBytesScanned();
-                    compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
-                    lastBytesScanned = bytesScanned;
                 }
             }
 
@@ -1857,57 +1875,87 @@ else if (transChecker.test(token))
                         pendingRepair);
             return fullSSTables.size() + transSSTables.size() + unrepairedSSTables.size();
         }
-        catch (Throwable e)
+        catch (CompactionInterruptedException e)
         {
-            if (e instanceof CompactionInterruptedException)
+            if (isCancelled.getAsBoolean())
             {
-                if (isCancelled.getAsBoolean())
-                {
-                    logger.info("Anticompaction has been canceled for session {}", pendingRepair);
-                    logger.trace(e.getMessage(), e);
-                }
-                else
-                {
-                    logger.info("Anticompaction for session {} has been stopped by request.", pendingRepair);
-                }
+                logger.info("Anticompaction has been canceled for session {}", pendingRepair);
+                logger.trace(e.getMessage(), e);
             }
             else
             {
-                JVMStabilityInspector.inspectThrowable(e);
-                logger.error("Error anticompacting " + txn + " for " + pendingRepair, e);
+                logger.info("Anticompaction for session {} has been stopped by request.", pendingRepair);
             }
             throw e;
         }
+        catch (Throwable e)
+        {
+            JVMStabilityInspector.inspectThrowable(e);
+            logger.error("Error anticompacting " + txn + " for " + pendingRepair, e);
+            throw e;
+        }
     }
 
     @VisibleForTesting
-    public static CompactionIterator getAntiCompactionIterator(List<ISSTableScanner> scanners, CompactionController controller, long nowInSec, TimeUUID timeUUID, ActiveCompactionsTracker activeCompactions, BooleanSupplier isCancelled)
+    public static CompactionIterator getAntiCompactionIterator(List<ISSTableScanner> scanners, CompactionController controller, long nowInSec, TimeUUID timeUUID, BooleanSupplier isCancelled)
     {
-        return new CompactionIterator(OperationType.ANTICOMPACTION, scanners, controller, nowInSec, timeUUID, activeCompactions, null)
+        return new CompactionIterator(OperationType.ANTICOMPACTION, scanners, controller, nowInSec, timeUUID) {
+            @Override
+            public TableOperation createOperation(CompactionProgress progress)
+            {
+                return getAntiCompactionOperation(super.createOperation(progress), isCancelled);
+            }
+        };
+    }
+
+    @VisibleForTesting
+    public static TableOperation getAntiCompactionOperation(TableOperation compaction, BooleanSupplier isCancelled)
+    {
+        return new AbstractTableOperation()
         {
+            @Override
+            public boolean isGlobal()
+            {
+                return false;
+            }
+
+            @Override
+            public Progress getProgress()
+            {
+                return compaction.getProgress();
+            }
+
+            @Override
+            public void stop(StopTrigger trigger)
+            {
+                compaction.stop(trigger);
+            }
+
+            @Override
             public boolean isStopRequested()
             {
-                return super.isStopRequested() || isCancelled.getAsBoolean();
+                return compaction.isStopRequested() || isCancelled.getAsBoolean();
+            }
+
+            @Override
+            public StopTrigger trigger()
+            {
+                return compaction.trigger();
             }
         };
     }
 
     @VisibleForTesting
-    Future<?> submitIndexBuild(final SecondaryIndexBuilder builder, ActiveCompactionsTracker activeCompactions)
+    Future<?> submitIndexBuild(final SecondaryIndexBuilder builder, TableOperationObserver activeCompactions)
     {
         Runnable runnable = new Runnable()
         {
             public void run()
             {
-                activeCompactions.beginCompaction(builder);
-                try
+                try (NonThrowingCloseable c = activeCompactions.onOperationStart(builder))
                 {
                     builder.build();
                 }
-                finally
-                {
-                    activeCompactions.finishCompaction(builder);
-                }
             }
         };
 
@@ -1927,7 +1975,7 @@ public Future<?> submitCacheWrite(final AutoSavingCache.Writer writer)
         return submitCacheWrite(writer, active);
     }
 
-    Future<?> submitCacheWrite(final AutoSavingCache.Writer writer, ActiveCompactionsTracker activeCompactions)
+    Future<?> submitCacheWrite(final AutoSavingCache.Writer writer, TableOperationObserver activeCompactions)
     {
         Runnable runnable = new Runnable()
         {
@@ -1935,20 +1983,15 @@ public void run()
             {
                 if (!AutoSavingCache.flushInProgress.add(writer.cacheType()))
                 {
-                    logger.trace("Cache flushing was already in progress: skipping {}", writer.getCompactionInfo());
+                    logger.trace("Cache flushing was already in progress: skipping {}", writer.getProgress());
                     return;
                 }
                 try
                 {
-                    activeCompactions.beginCompaction(writer);
-                    try
+                    try (NonThrowingCloseable c = activeCompactions.onOperationStart(writer))
                     {
                         writer.saveCache();
                     }
-                    finally
-                    {
-                        activeCompactions.finishCompaction(writer);
-                    }
                 }
                 finally
                 {
@@ -1960,16 +2003,17 @@ public void run()
         return executor.submitIfRunning(runnable, "cache write");
     }
 
-    public <T, E extends Throwable> T runAsActiveCompaction(Holder activeCompactionInfo, ThrowingSupplier<T, E> callable) throws E
+    public <T extends SSTableReader & IndexSummarySupport<T>> List<T> runIndexSummaryRedistribution(IndexSummaryRedistribution redistribution) throws IOException
     {
-        active.beginCompaction(activeCompactionInfo);
-        try
-        {
-            return callable.get();
-        }
-        finally
+        return runIndexSummaryRedistribution(redistribution, active);
+    }
+
+    @VisibleForTesting
+    <T extends SSTableReader & IndexSummarySupport<T>> List<T> runIndexSummaryRedistribution(IndexSummaryRedistribution redistribution, TableOperationObserver activeCompactions) throws IOException
+    {
+        try(Closeable c = activeCompactions.onOperationStart(redistribution))
         {
-            active.finishCompaction(activeCompactionInfo);
+            return redistribution.redistributeSummaries();
         }
     }
 
@@ -1986,24 +2030,19 @@ public Future<Long> submitViewBuilder(final ViewBuilderTask task)
     }
 
     @VisibleForTesting
-    Future<Long> submitViewBuilder(final ViewBuilderTask task, ActiveCompactionsTracker activeCompactions)
+    Future<Long> submitViewBuilder(final ViewBuilderTask task, TableOperationObserver activeCompactions)
     {
         return viewBuildExecutor.submitIfRunning(() -> {
-            activeCompactions.beginCompaction(task);
-            try
+            try(Closeable c = activeCompactions.onOperationStart(task))
             {
                 return task.call();
             }
-            finally
-            {
-                activeCompactions.finishCompaction(task);
-            }
         }, "view build");
     }
 
     public int getActiveCompactions()
     {
-        return active.getCompactions().size();
+        return active.getTableOperations().size();
     }
 
     public static boolean isCompactor(Thread thread)
@@ -2138,6 +2177,11 @@ public void incrementAborted()
         metrics.compactionsAborted.inc();
     }
 
+    public void incrementFailed()
+    {
+        metrics.totalCompactionsFailed.inc();
+    }
+
     public void incrementCompactionsReduced()
     {
         metrics.compactionsReduced.inc();
@@ -2156,23 +2200,34 @@ public SecondaryIndexExecutor()
         }
     }
 
+    public void incrementRemovedExpiredSSTables(long num)
+    {
+        metrics.removedExpiredSSTables.mark(num);
+    }
+
+    public void incrementDeleteOnlyCompactions()
+    {
+        metrics.deleteOnlyCompactions.mark();
+    }
+
+
     @Override
     public List<Map<String, String>> getCompactions()
     {
-        List<Holder> compactionHolders = active.getCompactions();
-        List<Map<String, String>> out = new ArrayList<Map<String, String>>(compactionHolders.size());
-        for (CompactionInfo.Holder ci : compactionHolders)
-            out.add(ci.getCompactionInfo().asMap());
+        List<TableOperation> operationSources = active.getTableOperations();
+        List<Map<String, String>> out = new ArrayList<Map<String, String>>(operationSources.size());
+        for (TableOperation op : operationSources)
+            out.add(op.getProgress().asMap());
         return out;
     }
 
     @Override
     public List<String> getCompactionSummary()
     {
-        List<Holder> compactionHolders = active.getCompactions();
-        List<String> out = new ArrayList<String>(compactionHolders.size());
-        for (CompactionInfo.Holder ci : compactionHolders)
-            out.add(ci.getCompactionInfo().toString());
+        List<TableOperation> operationSources = active.getTableOperations();
+        List<String> out = new ArrayList<String>(operationSources.size());
+        for (TableOperation ci : operationSources)
+            out.add(ci.getProgress().toString());
         return out;
     }
 
@@ -2214,21 +2269,21 @@ public long getCompletedTasks()
     public void stopCompaction(String type)
     {
         OperationType operation = OperationType.valueOf(type);
-        for (Holder holder : active.getCompactions())
+        for (TableOperation operationSource : active.getTableOperations())
         {
-            if (holder.getCompactionInfo().getTaskType() == operation)
-                holder.stop();
+            if (operationSource.getProgress().operationType() == operation)
+                operationSource.stop(TableOperation.StopTrigger.USER_STOP);
         }
     }
 
     @Override
     public void stopCompactionById(String compactionId)
     {
-        for (Holder holder : active.getCompactions())
+        for (TableOperation operationSource : active.getTableOperations())
         {
-            TimeUUID holderId = holder.getCompactionInfo().getTaskId();
+            TimeUUID holderId = operationSource.getProgress().operationId();
             if (holderId != null && holderId.equals(TimeUUID.fromString(compactionId)))
-                holder.stop();
+                operationSource.stop(TableOperation.StopTrigger.USER_STOP);
         }
     }
 
@@ -2409,24 +2464,74 @@ public void setMaxConcurrentAutoUpgradeTasks(int value)
         }
     }
 
-    public List<Holder> getCompactionsMatching(Iterable<TableMetadata> columnFamilies, Predicate<CompactionInfo> predicate)
+    public List<TableOperation> getCompactionsMatching(Iterable<TableMetadata> columnFamilies, Predicate<TableOperation.Progress> predicate)
     {
         Preconditions.checkArgument(columnFamilies != null, "Attempted to getCompactionsMatching in CompactionManager with no columnFamilies specified.");
 
-        List<Holder> matched = new ArrayList<>();
+        List<TableOperation> matched = new ArrayList<>();
         // consider all in-progress compactions
-        for (Holder holder : active.getCompactions())
+        for (TableOperation holder : active.getTableOperations())
         {
-            CompactionInfo info = holder.getCompactionInfo();
-            if (info.getTableMetadata() == null || Iterables.contains(columnFamilies, info.getTableMetadata()))
+            TableOperation.Progress progress = holder.getProgress();
+            if (progress.metadata() == null || Iterables.contains(columnFamilies, progress.metadata()))
             {
-                if (predicate.test(info))
+                if (predicate.test(progress))
                     matched.add(holder);
             }
         }
         return matched;
     }
 
+    /**
+     * Try to stop all of the compactions for given tables.
+     *
+     * Note that this method does not wait for all compactions to finish; you'll need to loop against
+     * isCompacting if you want that behavior.
+     *
+     * @param tables The tables to try to stop compaction upon.
+     * @param opPredicate Predicate to define which compaction operation to stop, based on its type.
+     * @param readerPredicate Predicate to define which compaction to stop based on candidate sstables.
+     * @param waitForInterruption whether to wait until interrupted compaction has fully stopped
+     *
+     * @return True if any compaction has been interrupted false otherwise.
+     */
+    public boolean interruptCompactionFor(Iterable<TableMetadata> tables, Predicate<OperationType> opPredicate, Predicate<SSTableReader> readerPredicate,
+                                          boolean waitForInterruption, TableOperation.StopTrigger trigger)
+    {
+        assert tables != null;
+
+        // interrupt in-progress compactions
+        Set<TableOperation> interrupted = new HashSet<>();
+        for (TableOperation operationSource : active.getTableOperations())
+        {
+            TableOperation.Progress info = operationSource.getProgress();
+
+            if (Iterables.contains(tables, info.metadata()) && opPredicate.test(info.operationType()))
+            {
+                operationSource.stop(trigger);
+                interrupted.add(operationSource);
+            }
+        }
+
+        if (waitForInterruption)
+        {
+            // wait at most 2 minutes
+            long start = nanoTime();
+            long wait = TimeUnit.MINUTES.toNanos(2);
+
+            for (TableOperation operation : interrupted)
+            {
+                while (active.isActive(operation) && nanoTime() - start < wait)
+                    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+
+                if (active.isActive(operation))
+                    throw new RuntimeException(String.format("Compaction task (%s) didn't finish within 2 minutes", operation.getProgress()));
+            }
+        }
+
+        return !interrupted.isEmpty();
+    }
+
     /**
      * Try to stop all of the compactions for given ColumnFamilies.
      *
@@ -2436,33 +2541,50 @@ public List<Holder> getCompactionsMatching(Iterable<TableMetadata> columnFamilie
      * @param columnFamilies The ColumnFamilies to try to stop compaction upon.
      * @param sstablePredicate the sstable predicate to match on
      * @param interruptValidation true if validation operations for repair should also be interrupted
+     * @return True if any compaction has been interrupted false otherwise.
      */
-    public void interruptCompactionFor(Iterable<TableMetadata> columnFamilies, Predicate<SSTableReader> sstablePredicate, boolean interruptValidation)
+    public boolean interruptCompactionFor(Iterable<TableMetadata> columnFamilies,
+                                          Predicate<SSTableReader> sstablePredicate,
+                                          boolean interruptValidation,
+                                          TableOperation.StopTrigger trigger)
     {
         assert columnFamilies != null;
 
         // interrupt in-progress compactions
-        for (Holder compactionHolder : active.getCompactions())
+        boolean interrupted = false;
+        for (TableOperation operationSource : active.getTableOperations())
         {
-            CompactionInfo info = compactionHolder.getCompactionInfo();
-            if ((info.getTaskType() == OperationType.VALIDATION) && !interruptValidation)
+            TableOperation.Progress info = operationSource.getProgress();
+            if ((info.operationType() == OperationType.VALIDATION) && !interruptValidation)
                 continue;
 
-            if (info.getTableMetadata() == null || Iterables.contains(columnFamilies, info.getTableMetadata()))
+            if (info.metadata() == null || Iterables.contains(columnFamilies, info.metadata()))
             {
-                if (info.shouldStop(sstablePredicate))
-                    compactionHolder.stop();
+                if (operationSource.shouldStop(sstablePredicate))
+                {
+                    operationSource.stop(trigger);
+                    interrupted = true;
+                }
             }
         }
+        return interrupted;
+    }
+
+    public boolean interruptCompactionFor(Iterable<TableMetadata> tables, TableOperation.StopTrigger trigger)
+    {
+        return interruptCompactionFor(tables, Predicates.alwaysTrue(), true, trigger);
     }
 
-    public void interruptCompactionForCFs(Iterable<ColumnFamilyStore> cfss, Predicate<SSTableReader> sstablePredicate, boolean interruptValidation)
+    public void interruptCompactionForCFs(Iterable<ColumnFamilyStore> cfss,
+                                          Predicate<SSTableReader> sstablePredicate,
+                                          boolean interruptValidation,
+                                          TableOperation.StopTrigger trigger)
     {
         List<TableMetadata> metadata = new ArrayList<>();
         for (ColumnFamilyStore cfs : cfss)
             metadata.add(cfs.metadata());
 
-        interruptCompactionFor(metadata, sstablePredicate, interruptValidation);
+        interruptCompactionFor(metadata, sstablePredicate, interruptValidation, trigger);
     }
 
     public void waitForCessation(Iterable<ColumnFamilyStore> cfss, Predicate<SSTableReader> sstablePredicate)
@@ -2480,14 +2602,14 @@ public void waitForCessation(Iterable<ColumnFamilyStore> cfss, Predicate<SSTable
     }
 
 
-    public List<CompactionInfo> getSSTableTasks()
+    public List<TableOperation.Progress> getSSTableTasks()
     {
-        return active.getCompactions()
+        return active.getTableOperations()
                      .stream()
-                     .map(CompactionInfo.Holder::getCompactionInfo)
-                     .filter(task -> task.getTaskType() != OperationType.COUNTER_CACHE_SAVE
-                                     && task.getTaskType() != OperationType.KEY_CACHE_SAVE
-                                     && task.getTaskType() != OperationType.ROW_CACHE_SAVE)
+                     .map(TableOperation::getProgress)
+                     .filter(progress -> progress.operationType() != OperationType.COUNTER_CACHE_SAVE
+                                     && progress.operationType() != OperationType.KEY_CACHE_SAVE
+                                     && progress.operationType() != OperationType.ROW_CACHE_SAVE)
                      .collect(Collectors.toList());
     }
 
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionObserver.java b/src/java/org/apache/cassandra/db/compaction/CompactionObserver.java
new file mode 100644
index 000000000000..8de942a2d90d
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionObserver.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.apache.cassandra.utils.TimeUUID;
+
+import javax.annotation.Nullable;
+
+/**
+ * An observer of a compaction operation. It is notified when a compaction operation is started.
+ * <p/>
+ * It returns a closeable that is invoked when the compaction is finished.
+ * <p/>
+ * The progress can be queried at any time to obtain real-time updates of the compaction operation.
+ */
+public interface CompactionObserver
+{
+    CompactionObserver NO_OP = new CompactionObserver()
+    {
+        @Override
+        public void onInProgress(CompactionProgress progress) { }
+
+        @Override
+        public void onCompleted(TimeUUID id, @Nullable Throwable error) { }
+    };
+
+    /**
+     * Indicates that a compaction has started.
+     * <p/>
+     * @param progress the compaction progress, it contains the unique id and real-time progress information
+     */
+    void onInProgress(CompactionProgress progress);
+
+    /**
+     * Indicates that a compaction with the given id has completed.
+     * <p/>
+     * @param id  the id of the compaction
+     * @param error error if compaction failed with any exceptions; or null if completed successfully
+     */
+    void onCompleted(TimeUUID id, @Nullable Throwable error);
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionPick.java b/src/java/org/apache/cassandra/db/compaction/CompactionPick.java
new file mode 100644
index 000000000000..17a6c8cff38a
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionPick.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * A set of sstables that were picked for compaction along with some other relevant properties.
+ * <p/>
+ * This is a list of sstables that should be compacted together after having been picked by a compaction strategy,
+ * for example from a bucket in {@link SizeTieredCompactionStrategy} or from a level in {@link LeveledCompactionStrategy}.
+ * Also, it contains other useful parameters such as a score that was assigned to this candidate (the read hotness or level
+ * score depending on the strategy) and the level, if applicable.
+ **/
+@NotThreadSafe
+public class CompactionPick
+{
+    final static CompactionPick EMPTY = create(-1, Collections.emptyList(), 0);
+
+    /** The key to the parent compaction aggregate, e.g. a level number or tier avg size, -1 if no parent */
+    private final long parent;
+
+    /** The sstables to be compacted */
+    private final ImmutableSet<CompactionSSTable> sstables;
+
+    /** Only expired sstables */
+    private final ImmutableSet<CompactionSSTable> expired;
+
+    /** The sum of all the sstable hotness scores */
+    private final double hotness;
+
+    /** The average size in bytes for the sstables in this compaction */
+    private final long avgSizeInBytes;
+
+    /** The total size on disk for the sstables in this compaction */
+    private final long totSizeInBytes;
+
+    /** The unique compaction id, this is available from the beginning and *MUST BE* used to create the transaction,
+     * when it is submitted */
+    @Nonnull
+    private final TimeUUID id;
+
+    /** The total space overhead for this compaction, including primary and secondary indexes. */
+    private final long totalOverheadInBytes;
+
+    /** This is set to true when the compaction is submitted */
+    private volatile boolean submitted;
+
+    /** The compaction progress, this is only available when compaction actually starts and will be null as long as
+     * the candidate is still pending execution, also some tasks cannot report a progress at all, e.g. {@link SingleSSTableLCSTask}.
+     * */
+    @Nullable 
+    private volatile CompactionProgress progress;
+
+    /** Set to true when the compaction has completed */
+    private volatile boolean completed;
+
+    private CompactionPick(TimeUUID id,
+                           long parent,
+                           Collection<? extends CompactionSSTable> compacting,
+                           Collection<? extends CompactionSSTable> expired,
+                           double hotness,
+                           long avgSizeInBytes,
+                           long totSizeInBytes,
+                           long totalOverheadInBytes)
+    {
+        this.id = Objects.requireNonNull(id);
+        this.parent = parent;
+        this.sstables = ImmutableSet.copyOf(compacting);
+        this.expired = ImmutableSet.copyOf(expired);
+        this.hotness = hotness;
+        this.avgSizeInBytes = avgSizeInBytes;
+        this.totSizeInBytes = totSizeInBytes;
+        this.totalOverheadInBytes = totalOverheadInBytes;
+    }
+
+    /**
+     * Create a pending compaction candidate with the given id, and average hotness and size.
+     * This method will use the data file size as the space overhead and should not be used by the unified compaction
+     * strategy where the overhead can be configurable.
+     */
+    public static CompactionPick create(TimeUUID id,
+                                        long parent,
+                                        Collection<? extends CompactionSSTable> sstables,
+                                        Collection<? extends CompactionSSTable> expired)
+    {
+        Collection<CompactionSSTable> nonExpiring = sstables.stream().filter(sstable -> !expired.contains(sstable)).collect(Collectors.toList());
+        final long totSizeBytes = CompactionAggregate.getTotSizeBytes(nonExpiring);
+        return create(id,
+                      parent,
+                      sstables,
+                      expired,
+                      CompactionAggregate.getTotHotness(nonExpiring),
+                      totSizeBytes / Math.max(nonExpiring.size(), 1),
+                      totSizeBytes,
+                      totSizeBytes);
+    }
+
+    /**
+     * Create a pending compaction candidate calculating hotness and avg and total size.
+     */
+    public static CompactionPick create(long parent, Collection<? extends CompactionSSTable> sstables, Collection<? extends CompactionSSTable> expired)
+    {
+        Collection<CompactionSSTable> nonExpiring = sstables.stream().filter(sstable -> !expired.contains(sstable)).collect(Collectors.toList());
+        final long totSizeBytes = CompactionAggregate.getTotSizeBytes(nonExpiring);
+        return create(LifecycleTransaction.newId(),
+                      parent,
+                      sstables,
+                      expired,
+                      CompactionAggregate.getTotHotness(nonExpiring),
+                      totSizeBytes / Math.max(nonExpiring.size(), 1),
+                      totSizeBytes,
+                      totSizeBytes);
+    }
+
+    static CompactionPick create(long parent, Collection<? extends CompactionSSTable> sstables)
+    {
+        return create(parent, sstables, Collections.emptyList());
+    }
+
+    static CompactionPick createWithUnknownParent(TimeUUID id, Collection<? extends CompactionSSTable> sstables)
+    {
+        return create(id, -1, sstables, Collections.emptyList());
+    }
+
+    /**
+     * Create a pending compaction candidate calculating avg and total size.
+     * This method will use the data file size as the space overhead and should not be used by the unified compaction
+     * strategy where the overhead can be configurable.
+     */
+    static CompactionPick create(long parent, Collection<? extends CompactionSSTable> sstables, double hotness)
+    {
+        final long totSizeBytes = CompactionAggregate.getTotSizeBytes(sstables);
+        return create(LifecycleTransaction.newId(),
+                      parent,
+                      sstables,
+                      Collections.emptyList(),
+                      hotness,
+                      totSizeBytes / Math.max(sstables.size(), 1),
+                      totSizeBytes,
+                      totSizeBytes);
+    }
+
+    /**
+     * Create a pending compaction candidate with the given parameters.
+     */
+    static CompactionPick create(TimeUUID id,
+                                 long parent,
+                                 Collection<? extends CompactionSSTable> sstables,
+                                 Collection<? extends CompactionSSTable> expired,
+                                 double hotness,
+                                 long avgSizeInBytes,
+                                 long totSizeInBytes,
+                                 long totalOverheadInBytes)
+    {
+        return new CompactionPick(id, parent, sstables, expired, hotness, avgSizeInBytes, totSizeInBytes, totalOverheadInBytes);
+    }
+
+    public double hotness()
+    {
+        return hotness;
+    }
+
+    public long avgSizeInBytes()
+    {
+        return avgSizeInBytes;
+    }
+
+    public long totSizeInBytes()
+    {
+        return totSizeInBytes;
+    }
+
+    public long totalOverheadInBytes()
+    {
+        return totalOverheadInBytes;
+    }
+
+    public double overheadToDataRatio()
+    {
+        return totalOverheadInBytes / Math.max(totSizeInBytes, 1.0);
+    }
+
+    public long parent()
+    {
+        return parent;
+    }
+
+    public ImmutableSet<CompactionSSTable> sstables()
+    {
+        return sstables;
+    }
+
+    public ImmutableSet<CompactionSSTable> expired()
+    {
+        return expired;
+    }
+
+    public TimeUUID id()
+    {
+        return id;
+    }
+    
+    public CompactionProgress progress()
+    {
+        return progress;
+    }
+    
+    public boolean inProgress()
+    {
+        return progress != null;
+    }
+    
+    public boolean completed()
+    {
+        return completed;
+    }
+
+    public boolean submitted() { return submitted; }
+
+    public void setSubmitted(TimeUUID id)
+    {
+        if (id == null || !this.id.equals(id))
+            throw new IllegalArgumentException("Id should have been " + this.id);
+
+        this.submitted = true;
+    }
+
+    /**
+     * Set the compaction progress, this means the compaction pick has started executing.
+     */
+    public void setProgress(CompactionProgress progress)
+    {
+        if (progress == null)
+            throw new IllegalArgumentException("Progress cannot be null");
+
+        if (this.progress != null)
+        {
+            if (this.progress.operationId() == progress.operationId())
+                return;
+            else
+                throw new IllegalStateException("Already compacting with different id");
+        }
+
+        if (!this.submitted())
+            setSubmitted(progress.operationId());
+        else if (this.id != progress.operationId())
+            throw new IllegalStateException("Submitted with a different id");
+
+        this.progress = progress;
+    }
+
+    public void setCompleted()
+    {
+        this.completed = true;
+    }
+
+    /**
+     * Create new compaction pick similar to the one provided but with a new parent.
+     */
+    CompactionPick withParent(long parent)
+    {
+        return new CompactionPick(id,
+                                  parent,
+                                  sstables,
+                                  expired,
+                                  hotness,
+                                  avgSizeInBytes,
+                                  totSizeInBytes,
+                                  totalOverheadInBytes);
+    }
+
+    /**
+     * Add more sstables to the collection of sstables initially picked.
+     * <p/>
+     * This is currently used by {@link TimeWindowCompactionStrategy} to add expired sstables.
+     *
+     * @param expired the sstables to add
+     */
+    CompactionPick withExpiredSSTables(Collection<CompactionSSTable> expired)
+    {
+        ImmutableSet<CompactionSSTable> newSSTables = ImmutableSet.<CompactionSSTable>builder()
+                                                              .addAll(this.sstables)
+                                                              .addAll(expired)
+                                                              .build();
+        ImmutableSet<CompactionSSTable> newExpired = ImmutableSet.<CompactionSSTable>builder()
+                                                             .addAll(this.expired)
+                                                             .addAll(expired)
+                                                             .build();
+        return new CompactionPick(id,
+                                  parent,
+                                  newSSTables,
+                                  newExpired,
+                                  hotness,
+                                  avgSizeInBytes,
+                                  totSizeInBytes,
+                                  totalOverheadInBytes);
+    }
+
+    /**
+     * @return true if this compaction candidate is empty, that is it has no sstables to compact.
+     */
+    boolean isEmpty()
+    {
+        return sstables.isEmpty();
+    }
+
+    boolean hasExpiredOnly()
+    {
+        return sstables.size() == expired.size();
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(id, parent, sstables, expired);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj == this)
+            return true;
+
+        if (!(obj instanceof CompactionPick))
+            return false;
+
+        CompactionPick that = (CompactionPick) obj;
+
+        // a pick is the same if the sstables are the same given that
+        // the other properties are derived from sstables and two
+        // picks are the same whether compaction has started or not so
+        // the progress and completed properties should not determine equality
+        return id.equals(that.id)
+               && parent == that.parent
+               && sstables.equals(that.sstables)
+               && expired.equals(that.expired);
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Id: %s, Parent: %d, Hotness: %f, Avg size in bytes: %d, sstables: %s, expired: %s",
+                             id,
+                             parent,
+                             hotness,
+                             avgSizeInBytes,
+                             sstables,
+                             expired);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionProgress.java b/src/java/org/apache/cassandra/db/compaction/CompactionProgress.java
new file mode 100644
index 000000000000..9425278df83b
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionProgress.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Clock;
+
+/**
+ * The progress information for a compaction operation. This adds compaction
+ * specific information to {@link TableOperation.Progress}.
+ */
+public interface CompactionProgress extends TableOperation.Progress
+{
+    /**
+     * The compaction strategy if available, otherwise null.
+     * <p/>
+     * The compaction strategy may not be available for some operations that use compaction task such
+     * as GC or sstable splitting.
+     *
+     * @return the compaction strategy when available or null.
+     */
+    @Nullable
+    CompactionStrategy strategy();
+
+    /**
+     * @return input sstables
+     */
+    Collection<SSTableReader> inSSTables();
+
+    /**
+     * @return output sstables
+     */
+    Collection<SSTableReader> outSSTables();
+
+    /**
+     * @return Size on disk (compressed) of the input sstables.
+     */
+    long inputDiskSize();
+
+    /**
+     * @return The uncompressed size of the input sstables.
+     */
+    long inputUncompressedSize();
+
+    /** Same as {@link this#inputDiskSize()} except for LCS where it estimates
+     * the compressed size for number of keys that will be read from the input sstables,
+     * see {@link org.apache.cassandra.db.compaction.LeveledCompactionStrategy}. */
+    long adjustedInputDiskSize();
+
+    /**
+     * @return Size on disk (compressed) of the output sstables.
+     */
+    long outputDiskSize();
+
+    /**
+     * @return the number of bytes processed by the compaction iterator. For compressed or encrypted sstables,
+     *         this is the number of bytes processed by the iterator after decompression, so this is the current
+     *         position in the uncompressed sstable files.
+     */
+    long uncompressedBytesRead();
+
+    /**
+     * @return the number of bytes processed by the compaction iterator for sstables on the specified level.
+     *         For compressed or encrypted sstables, this is the number of bytes processed by the iterator after decompression,
+     *         so this is the current position in the uncompressed sstable files.
+     */
+    long uncompressedBytesRead(int level);
+
+    /**
+     * @return the number of bytes that were written before compression is applied (uncompressed size).
+     */
+    long uncompressedBytesWritten();
+
+    /**
+     * @return the start time of this operation in millis since the epoch, i.e. as {@link System#currentTimeMillis}
+     * would report it.
+     */
+    long startTimeMillis();
+
+    /**
+     * @return the duration so far in milliseconds.
+     */
+    default long durationInMillis()
+    {
+        return Clock.Global.currentTimeMillis() - startTimeMillis();
+    }
+
+    /**
+     * @return total number of partitions read
+     */
+    long partitionsRead();
+
+    /**
+     * @return otal number of rows read
+     */
+    long rowsRead();
+
+    /**
+     * The partitions histogram maps the number of sstables to the number of partitions that were merged with that number of input sstables.
+     *
+     * @return the partitions histogram
+     */
+    long[] partitionsHistogram();
+
+    /**
+     * The rows histogram maps the number of sstables to the number of rows that were merged with that number of input sstables.
+     *
+     * @return the rows histogram
+     */
+    long[] rowsHistogram();
+
+    /**
+     * @return the ratio of bytes before and after compaction, using the adjusted input and output disk sizes (uncompressed values).
+     */
+    default double sizeRatio()
+    {
+        long estInputSizeBytes = adjustedInputDiskSize();
+        if (estInputSizeBytes > 0)
+            return outputDiskSize() / (double) estInputSizeBytes;
+
+        // this is a valid case, when there are no sstables to actually compact
+        // the previous code would return a NaN that would be logged as zero
+        return 0;
+    }
+
+    default double readThroughput()
+    {
+        long durationMillis = durationInMillis();
+        return durationMillis == 0 ? 0 : ((double) uncompressedBytesRead() / durationMillis) * TimeUnit.SECONDS.toMillis(1);
+    }
+
+    default double writeThroughput()
+    {
+        long durationMillis = durationInMillis();
+        return durationMillis == 0 ? 0 : ((double) uncompressedBytesWritten() / durationMillis) * TimeUnit.SECONDS.toMillis(1);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionRealm.java b/src/java/org/apache/cassandra/db/compaction/CompactionRealm.java
new file mode 100644
index 000000000000..41f033ac23aa
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionRealm.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.DiskBoundaries;
+import org.apache.cassandra.db.compaction.unified.Environment;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.service.snapshot.TableSnapshot;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * An interface for supplying the CFS data relevant to compaction. This is implemented by {@link org.apache.cassandra.db.ColumnFamilyStore} and
+ * works together with the {@link CompactionSSTable} interface as an abstraction of the space where compaction
+ * strategies operate.
+ *
+ * ColumnFamilyStore uses its SSTableReaders (which are already open to serve reads) as the CompactionSSTable instances,
+ * but alternate implementations can choose to maintain lighter representations (e.g. metadata-only local versions of
+ * remote sstables) which need only be switched to readers when the compaction is selected for execution and locks the
+ * sstable copies using {@link #tryModify}.
+ */
+public interface CompactionRealm extends SSTableReader.Owner
+{
+    /**
+     * @return the UCS environment of this table.
+     */
+    Environment makeUCSEnvironment();
+
+    /**
+     * @return a {@link ShardManager} for this specific compaction realm. If null is returned, UCS will build its own
+     * shard manager.
+     */
+    default ShardManager buildShardManager()
+    {
+        return null;
+    }
+
+    /**
+     * @return the schema metadata of this table.
+     */
+    default TableMetadata metadata()
+    {
+        return metadataRef().get();
+    }
+
+    /**
+     * @return the schema metadata of this table as a reference, used for long-living objects to keep up-to-date with
+     *         changes.
+     */
+    TableMetadataRef metadataRef();
+
+    default String getTableName()
+    {
+        return metadata().name;
+    }
+
+    default String getKeyspaceName()
+    {
+        return metadata().keyspace;
+    }
+
+    AbstractReplicationStrategy getKeyspaceReplicationStrategy();
+
+    /**
+     * @return the partitioner used by this table.
+     */
+    default IPartitioner getPartitioner()
+    {
+        return metadata().partitioner;
+    }
+
+    /**
+     * @return the {@link Directories} backing this table.
+     */
+    Directories getDirectories();
+
+    /**
+     * Grabs the global first/last tokens among sstables and returns the range of data directories that start/end with those tokens.
+     *
+     * This is done to avoid grabbing the disk boundaries for every sstable in case of huge compactions.
+     */
+    List<File> getDirectoriesForFiles(Set<SSTableReader> sstables);
+
+    /**
+     * @return the {@link DiskBoundaries} that are currently applied to the directories backing table.
+     */
+    DiskBoundaries getDiskBoundaries();
+
+    /**
+     * @return metrics object for the realm. This can be null during the initial construction of a compaction strategy,
+     * but should be set when the strategy is asked to select or run compactions.
+     */
+    TableMetrics metrics();
+
+    /**
+     * Return the estimated partition count, used when the number of partitions in an sstable is not sufficient to give
+     * a sensible range estimation.
+     */
+    default long estimatedPartitionCountInSSTables()
+    {
+        final long INITIAL_ESTIMATED_PARTITION_COUNT = 1 << 16; // If we don't yet have a count, use a sensible default.
+        if (metrics() == null)
+            return INITIAL_ESTIMATED_PARTITION_COUNT;
+        final Long estimation = metrics().estimatedPartitionCountInSSTablesCached.getValue();
+        if (estimation == null || estimation == 0)
+            return INITIAL_ESTIMATED_PARTITION_COUNT;
+        return estimation;
+    }
+
+    /**
+     * @return the secondary index manager, which is responsible for all secondary indexes.
+     */
+    SecondaryIndexManager getIndexManager();
+
+    /**
+     * @return true if tombstones should be purged only from repaired sstables.
+     */
+    boolean onlyPurgeRepairedTombstones();
+
+    /**
+     * @param sstables
+     * @return sstables whose key range overlaps with that of the given sstables, not including itself.
+     * (The given sstables may or may not overlap with each other.)
+     */
+    Set<? extends CompactionSSTable> getOverlappingLiveSSTables(Iterable<? extends CompactionSSTable> sstables);
+
+    /**
+     * @return true if compaction is operating and false if it has been stopped.
+     */
+    boolean isCompactionActive();
+
+    /**
+     * @return the compaction parameters associated with this table.
+     */
+    CompactionParams getCompactionParams();
+
+    /**
+     * @return true if the table is operating in a mode where no tombstones are allowed to be deleted.
+     */
+    boolean getNeverPurgeTombstones();
+
+    /**
+     * @return the minimum compaction threshold for size-tiered compaction (also when used as helper in leveled and
+     * time-window compaction strategies).
+     */
+    int getMinimumCompactionThreshold();
+    /**
+     * @return the maximum compaction threshold for size-tiered compaction (also when used as helper in leveled and
+     * time-window compaction strategies).
+     */
+    int getMaximumCompactionThreshold();
+
+    /**
+     * @return the write amplification (bytes flushed + bytes compacted / bytes flushed).
+     */
+    default double getWA()
+    {
+        TableMetrics metric = metrics();
+        if (metric == null)
+            return 0;
+
+        double bytesCompacted = metric.compactionBytesWritten.getCount();
+        double bytesFlushed = metric.bytesFlushed.getCount();
+        return bytesFlushed <= 0 ? 0 : (bytesFlushed + bytesCompacted) / bytesFlushed;
+    }
+
+    /**
+     * @return the level fanout factor for leveled compaction.
+     */
+    int getLevelFanoutSize();
+    /**
+     * @return true if the table and its compaction strategy support opening of incomplete compaction results early.
+     */
+    boolean supportsEarlyOpen();
+    /**
+     * @return the expected total size of the result of compacting the given sstables, taking into account ranges in
+     * the sstables that would be thrown away because they are no longer processed by this node.
+     */
+    long getExpectedCompactedFileSize(Iterable<SSTableReader> sstables, OperationType operationType);
+    /**
+     * @return true if compaction should check if the result of an operation fits in the disk space and reduce its scope
+     * when it does not.
+     */
+    boolean isCompactionDiskSpaceCheckEnabled();
+
+    /**
+     * @return all live memtables, or empty if no memtables are available.
+     */
+    Iterable<Memtable> getAllMemtables();
+
+    /**
+     * @return the set of all live sstables.
+     */
+    Set<? extends CompactionSSTable> getLiveSSTables();
+    /**
+     * @return the set of sstables which are currently compacting.
+     */
+    Set<? extends CompactionSSTable> getCompactingSSTables();
+
+    /**
+     * Return the subset of the given sstable set which is not currently compacting.
+     */
+    <S extends CompactionSSTable> Iterable<S> getNoncompactingSSTables(Iterable<S> sstables);
+
+    /**
+     * Return the given subset of sstables, i.e. LIVE, NONCOMPACTING or CANONICAL.
+     */
+    Iterable<? extends CompactionSSTable> getSSTables(SSTableSet set);
+
+    /**
+     * Invalidate the given key from local caches.
+     */
+    void invalidateCachedPartition(DecoratedKey key);
+
+    /**
+     * Construct a descriptor for a new sstable in the given location.
+     */
+    Descriptor newSSTableDescriptor(File locationForDisk);
+
+    /**
+     * Initiate a transaction to modify the given sstables and operation type, most often a compaction.
+     * The transaction will convert the given CompactionSSTable handles into open SSTableReaders.
+     */
+    LifecycleTransaction tryModify(Iterable<? extends CompactionSSTable> sstables,
+                                   OperationType operationType,
+                                   TimeUUID id);
+
+    /**
+     * Initiate a transaction to modify the given sstables and operation type, most often a compaction.
+     * The transaction will convert the given CompactionSSTable handles into open SSTableReaders.
+     */
+    default LifecycleTransaction tryModify(Iterable<? extends CompactionSSTable> sstables,
+                                           OperationType operationType)
+    {
+        return tryModify(sstables, operationType, LifecycleTransaction.newId());
+    }
+
+    /**
+     * Create an overlap tracker for the given set of source sstables. The tracker is used to identify all sstables
+     * that overlap with the given sources, which is used to decide if tombstones or other data can be purged.
+     */
+    OverlapTracker getOverlapTracker(Iterable<SSTableReader> sources);
+
+    interface OverlapTracker extends AutoCloseable
+    {
+        /**
+         * @return all sstables that overlap with the given source set.
+         */
+        Collection<? extends CompactionSSTable> overlaps();
+
+        /**
+         * @return the sstables whose span covers the given key.
+         */
+        Collection<? extends CompactionSSTable> overlaps(DecoratedKey key);
+
+        /**
+         * Get all the sstables whose span covers the given key, open (i.e. convert to SSTableReader) the ones selected
+         * by the given filter, and collect the non-null results of applying the given transformation to the resulting
+         * SSTableReaders.
+         * Used to select shadow sources (i.e. sources of tombstones or data) for garbage-collecting compactions.
+         */
+        <V> Iterable<V> openSelectedOverlappingSSTables(DecoratedKey key,
+                                                        Predicate<CompactionSSTable> filter,
+                                                        Function<SSTableReader, V> transformation);
+
+        /**
+         * Refresh the overlapping sstables to reflect compactions applied to any of them.
+         * Done to avoid holding on to references of obsolete sstables, which will prevent them from being deleted.
+         */
+        boolean maybeRefresh();
+        
+        void refreshOverlaps();
+    }
+
+    /**
+     * Create a CFS snapshot with the given name.
+     */
+    TableSnapshot snapshotWithoutMemtable(String snapshotId);
+
+    /**
+     * Create a CFS snapshot with the given name and timestamp.
+     */
+    TableSnapshot snapshotWithoutMemtable(String snapshotName, Instant creationTime);
+
+    /**
+     * Change the repaired status of a set of sstables, usually to reflect a completed repair operation.
+     */
+    int mutateRepairedWithLock(Collection<SSTableReader> originals, long repairedAt, TimeUUID pendingRepair, boolean isTransient) throws IOException;
+
+    /**
+     * Signal that a repair session has completed.
+     */
+    void repairSessionCompleted(TimeUUID sessionID);
+
+    boolean shouldIgnoreGcGraceForKey(DecoratedKey dk);
+
+    /**
+     * Run an operation with concurrent compactions being stopped.
+     */
+    <V> V runWithCompactionsDisabled(Callable<V> callable, OperationType operationType, boolean interruptValidation, boolean interruptViews, TableOperation.StopTrigger trigger);
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionSSTable.java b/src/java/org/apache/cassandra/db/compaction/CompactionSSTable.java
new file mode 100644
index 000000000000..0dfe9177ad20
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionSSTable.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.function.BiPredicate;
+
+import javax.annotation.Nullable;
+
+import com.google.common.collect.Ordering;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.sstable.SSTableIdFactory;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * An SSTable abstraction used by compaction. Implemented by {@link SSTableReader} and provided by
+ * {@link CompactionRealm} instances.
+ *
+ * This abstraction is used to select the sstables to compact. When a compaction is initiated using
+ * {@link CompactionRealm#tryModify}, the compaction operation receives the SSTableReaders corresponding to the passed
+ * CompactionSSTables.
+ */
+public interface CompactionSSTable
+{
+    // Note: please do not replace with Comparator.comparing, this code can be on a hot path.
+    Comparator<CompactionSSTable> maxTimestampDescending = (o1, o2) -> Long.compare(o2.getMaxTimestamp(), o1.getMaxTimestamp());
+    Comparator<CompactionSSTable> maxTimestampAscending = (o1, o2) -> Long.compare(o1.getMaxTimestamp(), o2.getMaxTimestamp());
+    Comparator<CompactionSSTable> firstKeyComparator = (o1, o2) -> o1.getFirst().compareTo(o2.getFirst());
+    Comparator<CompactionSSTable> lastKeyComparator = (o1, o2) -> o1.getLast().compareTo(o2.getLast());
+    Ordering<CompactionSSTable> firstKeyOrdering = Ordering.from(firstKeyComparator);
+    Comparator<CompactionSSTable> sizeComparator = (o1, o2) -> Long.compare(o1.onDiskLength(), o2.onDiskLength());
+    Comparator<CompactionSSTable> idComparator = (o1, o2) -> SSTableIdFactory.COMPARATOR.compare(o1.getId(), o2.getId());
+    Comparator<CompactionSSTable> idReverseComparator = idComparator.reversed();
+    BiPredicate<CompactionSSTable, CompactionSSTable> startsAfter = (a, b) -> a.getFirst().compareTo(b.getLast()) > 0;
+
+    /**
+     * @return the position of the first partition in the sstable
+     */
+    PartitionPosition getFirst();
+
+    /**
+     * @return the position of the last partition in the sstable
+     */
+    PartitionPosition getLast();
+
+    /**
+     * @return the bounds spanned by this sstable, from first to last keys.
+     */
+    AbstractBounds<Token> getBounds();
+
+    /**
+     * @return the length in bytes of the all on-disk components' file size for this SSTable.
+     */
+    long onDiskComponentsSize();
+
+    /**
+     * @return the length in bytes of the on disk data file size for this SSTable. For compressed files, this is not the same
+     * thing as the data length (see {@link #uncompressedLength})
+     */
+    long onDiskLength();
+
+    /**
+     * @return the length in bytes of the data for this SSTable. For compressed files, this is not the same thing as the
+     * on disk size (see {@link #onDiskLength})
+     */
+    long uncompressedLength();
+
+    /**
+     * @return the fraction of the token space for which this sstable has content. In the simplest case this is just the
+     * size of the interval returned by {@link #getBounds()}, but the sstable may contain "holes" when the locally-owned
+     * range is not contiguous (e.g. with vnodes).
+     * As this is affected by the local ranges which can change, the token space fraction is calculated at the time of
+     * writing the sstable and stored with its metadata.
+     * For older sstables that do not contain this metadata field, this method returns NaN.
+     */
+    double tokenSpaceCoverage();
+
+    /**
+     * @return the sum of the on-disk size of the given sstables.
+     */
+    static long getTotalDataBytes(Iterable<? extends CompactionSSTable> sstables)
+    {
+        long sum = 0;
+        for (CompactionSSTable sstable : sstables)
+            sum += sstable.onDiskLength();
+        return sum;
+    }
+
+    /*
+     * @return the total number of bytes in all on-disk components of the given sstables.
+     */
+    static long getTotalOnDiskComponentsBytes(Iterable<? extends CompactionSSTable> sstables)
+    {
+        long total = 0;
+        for (CompactionSSTable sstable : sstables)
+            total += sstable.onDiskComponentsSize();
+
+        // We estimate the compaction overhead to be the same as the all components size of the input sstables including SAI files
+        // This is because even though we have a cache, the output sstable data files will be on disk
+        // first, and only added to the cache at the end. We could improve flushed sstables, since we know that
+        // the output will be 1 / RF of the input size, but we don't have this information handy, and normally
+        // L0 sstables have a small overhead, the overhead is mostly significant for the sstables at the higher levels.
+        return total;
+    }
+
+    /**
+     * @return the sum of the uncompressed size of the given sstables.
+     */
+    static long getTotalUncompressedBytes(Iterable<? extends CompactionSSTable> sstables)
+    {
+        long sum = 0;
+        for (CompactionSSTable sstable : sstables)
+            sum += sstable.uncompressedLength();
+
+        return sum;
+    }
+
+    /**
+      * @return the smallest timestamp of all cells contained in this sstable.
+      */
+    long getMinTimestamp();
+
+    /**
+      * @return the largest timestamp of all cells contained in this sstable.
+      */
+    long getMaxTimestamp();
+
+    /**
+     * @return the smallest deletion time of all deletions contained in this sstable.
+     */
+    long getMinLocalDeletionTime();
+
+    /**
+     * @return the larget deletion time of all deletions contained in this sstable.
+     */
+    long getMaxLocalDeletionTime();
+
+    /**
+     * Called by {@link org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy} and other compaction strategies
+     * to determine the read hotness of this sstables, this method returna a "read hotness" which is calculated by
+     * looking at the last two hours read rate and dividing this number by the estimated number of keys.
+     * <p/>
+     * Note that some system tables do not have read meters, in which case this method will return zero.
+     *
+     * @return the last two hours read rate per estimated key
+     */
+    double hotness();
+
+    /**
+      * @return true if this sstable was repaired by a repair service, false otherwise.
+      */
+    boolean isRepaired();
+
+    /**
+     * @return the time of repair when isRepaired is true, otherwise UNREPAIRED_SSTABLE.
+     */
+    long getRepairedAt();
+
+    /**
+      * @return true if this sstable is pending repair, false otherwise.
+      */
+    boolean isPendingRepair();
+
+    /**
+     * @return the id of the repair session when isPendingRepair is true, otherwise null.
+     */
+    @Nullable
+    TimeUUID getPendingRepair();
+
+    /**
+     * @return true if this sstable belongs to a transient range.
+     */
+    boolean isTransient();
+
+    /**
+     * @return an estimate of the number of keys in this SSTable based on the index summary.
+     */
+    long estimatedKeys();
+
+    /**
+      * @return the level of this sstable according to {@link LeveledCompactionStrategy}, zero for other strategies.
+      */
+    int getSSTableLevel();
+
+    /**
+      * @return true if this sstable can take part into a compaction.
+      */
+    boolean isSuitableForCompaction();
+
+    /**
+      * @return true if this sstable was marked for obsoletion by a compaction.
+      */
+    boolean isMarkedCompacted();
+
+    /**
+      * @return true if this sstable is suspect, that is it was involved in an operation that failed, such
+      *         as a write or read that resulted in {@link CorruptSSTableException}.
+      */
+    boolean isMarkedSuspect();
+
+    /**
+     * Whether the sstable may contain tombstones or if it is guaranteed to not contain any.
+     * <p>
+     * Note that having that method return {@code false} guarantees the sstable has no tombstones whatsoever (so no cell
+     * tombstone, no range tombstone maker and no expiring columns), but having it return {@code true} doesn't guarantee
+     * it contains any as it may simply have non-expired cells.
+     */
+    boolean mayHaveTombstones();
+
+    /**
+     * The method verifies whether the sstable may contain the provided key. The method does approximation using
+     * Bloom filter if it is present and if it is not, performs accurate check in the index.
+     */
+    boolean mayContainAssumingKeyIsInRange(DecoratedKey key);
+
+    Descriptor getDescriptor();
+    Path getFile();
+    default String getColumnFamilyName()
+    {
+        return getDescriptor().cfname;
+    }
+    default String getKeyspaceName()
+    {
+        return getDescriptor().ksname;
+    }
+    default SSTableId getId()
+    {
+        return getDescriptor().id;
+    }
+
+    /**
+     * @param component component to get timestamp.
+     * @return last modified time for given component. 0 if given component does not exist or IO error occurs.
+     */
+    default long getCreationTimeFor(Component component)
+    {
+        return getDescriptor().fileFor(component).lastModified();
+    }
+
+    /**
+     * @return an estimate of the ratio of the tombstones present in the sstable that could be dropped for the given
+     * garbage collection threshold.
+     */
+    double getEstimatedDroppableTombstoneRatio(long gcBefore);
+
+    /**
+     * Changes the SSTable level as used by {@link LeveledCompactionStrategy}.
+     * @throws IOException
+     */
+    void mutateLevelAndReload(int newLevel) throws IOException;
+
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategy.java
new file mode 100644
index 000000000000..57a305ab9d21
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategy.java
@@ -0,0 +1,230 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.ScannerList;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * The common interface between legacy compaction strategies (those that extend {@link LegacyAbstractCompactionStrategy}
+ * and the new compaction strategy, {@link UnifiedCompactionStrategy}.
+ */
+public interface CompactionStrategy extends CompactionObserver
+{
+    /**
+     * @return the compaction logger optionally logs events in a csv file.
+     */
+    CompactionLogger getCompactionLogger();
+
+    /**
+     * For internal, temporary suspension of background compactions so that we can do exceptional
+     * things like truncate or major compaction
+     */
+    void pause();
+
+    /**
+     * For internal, temporary suspension of background compactions so that we can do exceptional
+     * things like truncate or major compaction
+     */
+    void resume();
+
+    /**
+     * Performs any extra initialization required
+     */
+    void startup();
+
+    /**
+     * Releases any resources if this strategy is shutdown (when the CFS is reloaded after a schema change).
+     */
+    void shutdown();
+
+    /**
+     * @param gcBefore throw away tombstones older than this
+     *
+     * @return the next background/minor compaction tasks to run; empty if nothing to do.
+     *
+     * Is responsible for marking its sstables as compaction-pending.
+     */
+    Collection<AbstractCompactionTask> getNextBackgroundTasks(long gcBefore);
+
+    /**
+     * @param gcBefore             throw away tombstones older than this
+     * @param splitOutput          whether the output of the compaction should be split (only applicable to STCS)
+     * @param permittedParallelism the maximum number of tasks that can be run in parallel, if the operation can be
+     *                             parallelized (UCS with parallelize_output_shards enabled)
+     * @return compaction tasks that should be run to compact this table as much as possible.
+     * <p>
+     * Is responsible for marking its sstables as compaction-pending.
+     */
+    @SuppressWarnings("resource")
+    CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism);
+
+    CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism, OperationType operationType);
+
+    /**
+     * @param sstables SSTables to compact. Must be marked as compacting.
+     * @param gcBefore throw away tombstones older than this
+     *
+     * @return a compaction task corresponding to the requested sstables.
+     * Will not be null. (Will throw if user requests an invalid compaction.)
+     *
+     * Is responsible for marking its sstables as compaction-pending.
+     */
+    @SuppressWarnings("resource")
+    CompactionTasks getUserDefinedTasks(Collection<? extends CompactionSSTable> sstables, long gcBefore);
+
+    /**
+     * Get the estimated remaining compactions.
+     *
+     * @return the number of background tasks estimated to still be needed for this strategy
+     */
+    int getEstimatedRemainingTasks();
+
+    int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes, boolean isIncremental);
+
+    /**
+     * Create a compaction task for the sstables in the transaction.
+     *
+     * @return a valid compaction task that can be executed.
+     */
+    AbstractCompactionTask createCompactionTask(LifecycleTransaction txn, long gcBefore, long maxSSTableBytes);
+
+    /**
+     * @return the total number of background compactions, pending or in progress
+     */
+    int getTotalCompactions();
+
+    /**
+     * Return the statistics. Not all strategies will provide non-empty statistics,
+     * the legacy strategies that do not support aggregates will return empty statistics.
+     * <p/>
+     * @return statistics about this compaction picks.
+     */
+    List<CompactionStrategyStatistics> getStatistics();
+
+    /**
+     * @return size in bytes of the largest sstables for this strategy
+     */
+    long getMaxSSTableBytes();
+
+    /**
+     * @return the number of sstables for each level, if this strategy supports levels. Otherwise return an empty array.
+     */
+    int[] getSSTableCountPerLevel();
+
+    /**
+     * @return total size on disk for each level. null unless leveled compaction is used.
+     */
+    long[] getPerLevelSizeBytes();
+
+    /**
+     * @return true if the table is using LeveledCompactionStrategy. false otherwise.
+     */
+    boolean isLeveledCompaction();
+
+    /**
+     * @return sstable count for each bucket in TWCS. null unless time window compaction is used.
+     */
+    int[] getSSTableCountPerTWCSBucket();
+
+    /**
+     * @return the level fanout size if applicable to this strategy. Otherwise return the default LCS fanout size.
+     */
+    int getLevelFanoutSize();
+
+    /**
+     * Returns a list of KeyScanners given sstables and a range on which to scan.
+     * The default implementation simply grab one SSTableScanner per-sstable, but overriding this method
+     * allow for a more memory efficient solution if we know the sstable don't overlap (see
+     * LeveledCompactionStrategy for instance).
+     */
+    ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges);
+
+    default ScannerList getScanners(Collection<SSTableReader> toCompact)
+    {
+        return getScanners(toCompact, null);
+    }
+
+    /**
+     * @return the name of the strategy
+     */
+    String getName();
+
+    /**
+     * Returns the sstables managed by the strategy
+     */
+    Set<? extends CompactionSSTable> getSSTables();
+
+    /**
+     * Group sstables that can be anti-compacted togetehr.
+     * @param sstablesToGroup
+     * @return
+     */
+    Collection<Collection<CompactionSSTable>> groupSSTablesForAntiCompaction(Collection<? extends CompactionSSTable> sstablesToGroup);
+
+    /**
+     * Create an sstable writer that is suitable for the strategy.
+     */
+    SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
+                                                long keyCount,
+                                                long repairedAt,
+                                                TimeUUID pendingRepair,
+                                                boolean isTransient,
+                                                IntervalSet<CommitLogPosition> commitLogPositions,
+                                                int sstableLevel,
+                                                SerializationHeader header,
+                                                Collection<Index.Group> indexGroups,
+                                                LifecycleNewTracker lifecycleNewTracker);
+
+    /**
+     * @return true if the strategy supports early open
+     */
+    boolean supportsEarlyOpen();
+
+    /**
+     * Return whether this strategy can be used with cursor compaction.
+     * Currently we report true for all strategies.
+     */
+    default boolean supportsCursorCompaction()
+    {
+        return true;
+    }
+
+    void periodicReport();
+
+    /**
+     * Returns a map of sstable regions (e.g. repaired, unrepaired, possibly combined with level information) to the
+     * maximum overlap between the sstables in the region.
+     */
+    Map<String, String> getMaxOverlapsMap();
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyContainer.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyContainer.java
new file mode 100644
index 000000000000..46535665c989
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyContainer.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.List;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * A strategy container manages compaction strategies for a {@link ColumnFamilyStore}.
+ *
+ * This class is responsible for:
+ * - providing a single interface for possibly multiple active strategy instances - e.g. due to having
+ * multiple arenas for repaired, unrepaired, pending, transient SSTables.
+ * - updating or recreating the strategies when configuration change - e.g. compaction parameters
+ * or disk boundaries
+ */
+public interface CompactionStrategyContainer extends CompactionStrategy, INotificationConsumer
+{
+    /**
+     * Enable compaction.
+     */
+    void enable();
+
+    /**
+     * Disable compaction.
+     */
+    void disable();
+
+    /**
+     * @return {@code true} if compaction is enabled and running; e.g. if autocompaction has been disabled via nodetool
+     *         or JMX, this should return {@code false}, even if the underlying compaction strategy hasn't been paused.
+     */
+    boolean isEnabled();
+
+    /**
+     * @return {@code true} if compaction is running, i.e. if the underlying compaction strategy is not currently
+     *         paused or being shut down.
+     */
+    boolean isActive();
+
+    /**
+     * The reason for reloading
+     */
+    enum ReloadReason
+    {
+        /** A new strategy container has been created.  */
+        FULL,
+
+        /** A new strategy container has been reloaded due to table metadata changes, e.g. a schema change. */
+        METADATA_CHANGE,
+
+        /** A request over JMX to update the compaction parameters only locally, without changing the schema permanently. */
+        JMX_REQUEST,
+
+        /** The disk boundaries were updated, in this case the strategies may need to be recreated even if the params haven't changed */
+        DISK_BOUNDARIES_UPDATED
+    }
+
+    /**
+     * Reload the strategy container taking into account the state of the previous strategy container instance
+     * ({@code this}, in case we're not reloading after switching between containers), the new compaction parameters,
+     * and the reason for reloading.
+     * <p/>
+     * Depending on the reason, different actions are taken, for example the schema parameters are not updated over
+     * JMX and the decision on whether to enable or disable compaction depends only on the parameters over JMX, but
+     * also on the previous JMX directive in case of a full reload. Also, the disk boundaries are not updated over JMX.
+     * <p/>
+     * See the implementations of this method for more details.
+     *
+     * @param previous the strategy container instance which state needs to be inherited/taken into account, in many
+     *                 cases the same as {@code this}, but never {@code null}.
+     * @param compactionParams the new compaction parameters
+     * @param reason the reason for reloading
+     *
+     * @return existing or new container with updated parameters
+     */
+    CompactionStrategyContainer reload(@Nonnull CompactionStrategyContainer previous,
+                                       CompactionParams compactionParams,
+                                       ReloadReason reason);
+
+    /**
+     * @param params new compaction parameters
+     * @param reason the reason for reloading
+     * @return {@code true} if the compaction parameters should be updated on reload
+     */
+    default boolean shouldReload(CompactionParams params, ReloadReason reason)
+    {
+        return reason != CompactionStrategyContainer.ReloadReason.METADATA_CHANGE || !params.equals(getMetadataCompactionParams());
+    }
+
+    /**
+     * Creates new {@link CompactionStrategyContainer} and loads its parameters
+     *
+     * This method is used by {@link CompactionStrategyFactory} to create a
+     * {@link CompactionStrategyContainer}s via reflection.
+     *
+     * @param previous the strategy container instance which state needs to be inherited/taken into account
+     *                 or {@code null} if there was no container to inherit from.
+     * @param strategyFactory the factory instance responsible for creating the CSM
+     * @param compactionParams the new compaction parameters
+     * @param reason the reason for creating a new container
+     * @param enableAutoCompaction true if auto compaction should be enabled
+     *
+     * @return a new {@link CompactionStrategyContainer} with newly loaded parameters
+     */
+    static CompactionStrategyContainer create(@Nullable CompactionStrategyContainer previous,
+                                              CompactionStrategyFactory strategyFactory,
+                                              CompactionParams compactionParams,
+                                              CompactionStrategyContainer.ReloadReason reason,
+                                              boolean enableAutoCompaction)
+    {
+        throw new UnsupportedOperationException("Implementations of CompactionStrategyContainer must implement static create method");
+    }
+
+    /**
+     * Return the compaction parameters. These are not necessarily the same as the ones specified in the schema, they
+     * may have been overwritten over JMX.
+     *
+     * @return the compaction params currently active
+     */
+    CompactionParams getCompactionParams();
+
+    /**
+     * Returns the compaction parameters set via metadata.
+     *
+     * This method is useful to decide if we should update the compaction strategy due to a
+     * metadata change such as a schema changed caused by an ALTER TABLE.
+     *
+     * If a user changes the local compaction strategy via JMX and then later ALTERs a compaction parameter,
+     * we will use the new compaction parameters but we will not override the JMX parameters if compaction
+     * was not changed by the ALTER.
+     *
+     * @return the compaction parameters set via metadata changes
+     */
+    CompactionParams getMetadataCompactionParams();
+
+    /**
+     * This method is to keep compatibility with strategies baked by {@link CompactionStrategyManager} where
+     * there are multiple inner strategies handling sstables by repair status.
+     *
+     * @return all inner compaction strategies
+     */
+    List<CompactionStrategy> getStrategies();
+
+    /**
+     * This method is to keep compatibility with strategies baked by {@link CompactionStrategyManager} where
+     * there are multiple inner strategies handling sstables by repair status.
+     *
+     * Note that if {@code isRepaired} is true, {@code pendingRepair} must be null.
+     *
+     * @param isRepaired will return strategies for repaired SSTables; must be {@code false} if
+     *                   {@code pendingRepair} is specified
+     * @param pendingRepair will return strategies for the given pending repair; must be {@code null}
+     *                      if {@code isRepaired} is true
+     *
+     * @return a list of inner strategies that match given parameters
+     */
+    List<CompactionStrategy> getStrategies(boolean isRepaired, @Nullable TimeUUID pendingRepair);
+
+    /**
+     * Called to clean up state when a repair session completes.
+     *
+     * @param sessionID repair session id.
+     */
+    void repairSessionCompleted(TimeUUID sessionID);
+
+    /**
+     * The method is for CompactionStrategyManager to use with {@link CompactionRealm#mutateRepairedWithLock}.
+     * UnifiedCompactionContainer does not need it.
+     */
+    ReentrantReadWriteLock.WriteLock getWriteLock();
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyFactory.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyFactory.java
new file mode 100644
index 000000000000..83d4aefe9ed5
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyFactory.java
@@ -0,0 +1,188 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.schema.CompactionParams;
+
+/**
+ * The factory for compaction strategies and their containers.
+ */
+public class CompactionStrategyFactory
+{
+    private final CompactionRealm realm;
+    private final CompactionLogger compactionLogger;
+
+    public CompactionStrategyFactory(CompactionRealm realm)
+    {
+        this.realm = realm;
+        this.compactionLogger = new CompactionLogger(realm.metadata());
+    }
+
+    /**
+     * Reload the existing strategy container, possibly creating a new one if required.
+     *
+     * @param current the current strategy container, or {@code null} if this is the first time we're loading a
+     *                compaction strategy
+     * @param compactionParams the new compaction parameters
+     * @param reason the reason for reloading
+     * @param enableAutoCompaction true if auto compaction should be enabled
+     *
+     * @return Either a new strategy container or the current one, but reloaded with the given compaction parameters.
+     */
+    public CompactionStrategyContainer reload(@Nullable CompactionStrategyContainer current,
+                                              CompactionParams compactionParams,
+                                              CompactionStrategyContainer.ReloadReason reason,
+                                              boolean enableAutoCompaction)
+    {
+        // If we were called due to a metadata change but the compaction parameters are the same then
+        // don't reload since we risk overriding parameters set via JMX
+        if (current != null && !current.shouldReload(compactionParams, reason))
+            return current;
+
+        Class<? extends CompactionStrategyContainer> containerClass = containerForStrategy(compactionParams.klass());
+        CompactionStrategyContainer ret;
+
+        // if the strategy belongs to the same container, we can just reload
+        if (current != null && current.getClass().equals(containerClass))
+            ret = current.reload(current, compactionParams, reason);
+        else
+        {
+            // otherwise we need to re-create the container
+            ret = createStrategyContainer(containerClass, current, compactionParams, reason, enableAutoCompaction);
+        }
+        
+        return ret;
+    }
+
+    static boolean enableCompactionOnReload(@Nullable CompactionStrategyContainer previous,
+                                            CompactionParams compactionParams,
+                                            CompactionStrategyContainer.ReloadReason reason)
+    {
+        // If this is a JMX request, we only consider the params passed by it
+        if (reason == CompactionStrategyContainer.ReloadReason.JMX_REQUEST)
+            return compactionParams.isEnabled();
+        // If the enabled state flag and the params of the previous container differ, compaction was forcefully
+        // enabled/disabled by JMX/nodetool, and we should inherit that setting through the enabled state flag
+        if (previous != null && previous.isEnabled() != previous.getCompactionParams().isEnabled())
+            return previous.isEnabled();
+
+        return compactionParams.isEnabled();
+    }
+
+    /**
+     * Returns a {@link CompactionStrategyContainer} class for the given strategy class.
+     *
+     * We need this method to create correct container for the strategy, but also to distinguish
+     * between situations when a container should reloaded or recreated.
+     */
+    private Class<? extends CompactionStrategyContainer> containerForStrategy(Class<? extends CompactionStrategy> strategyClass)
+    {
+        Class<? extends CompactionStrategyContainer> containerClass;
+        try
+        {
+            Field containerClassField = strategyClass.getField("CONTAINER_CLASS");
+            containerClass = (Class<? extends CompactionStrategyContainer>) containerClassField.get(null);
+        }
+        catch (IllegalAccessException | NoSuchFieldException e)
+        {
+            containerClass = CompactionStrategyManager.class;
+        }
+
+        return containerClass;
+    }
+
+    private CompactionStrategyContainer createStrategyContainer(Class<? extends CompactionStrategyContainer> containerClass,
+                                                                CompactionStrategyContainer previous,
+                                                                CompactionParams compactionParams,
+                                                                CompactionStrategyContainer.ReloadReason reason,
+                                                                boolean enableAutoCompaction)
+    {
+        CompactionStrategyContainer ret;
+        try
+        {
+            Method createMethod = containerClass.getMethod("create",
+                                                           CompactionStrategyContainer.class,
+                                                           CompactionStrategyFactory.class,
+                                                           CompactionParams.class,
+                                                           CompactionStrategyContainer.ReloadReason.class,
+                                                           boolean.class);
+            ret = (CompactionStrategyContainer) createMethod.invoke(null,
+                                                                    previous,
+                                                                    this,
+                                                                    compactionParams,
+                                                                    reason,
+                                                                    enableAutoCompaction);
+        }
+        catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e)
+        {
+            ret = new CompactionStrategyManager(this, enableAutoCompaction);
+            ret.reload(previous, compactionParams, reason);
+        }
+        return ret;
+    }
+
+    public CompactionLogger getCompactionLogger()
+    {
+        return compactionLogger;
+    }
+
+    public CompactionRealm getRealm()
+    {
+        return realm;
+    }
+
+    /**
+     * Creates a compaction strategy that is managed by {@link CompactionStrategyManager} and its strategy holders.
+     * These strategies must extend {@link LegacyAbstractCompactionStrategy}.
+     *
+     * @return an instance of the compaction strategy specified in the parameters so long as it extends {@link LegacyAbstractCompactionStrategy}
+     * @throws IllegalArgumentException if the params do not contain a strategy that extends  {@link LegacyAbstractCompactionStrategy}
+     */
+    LegacyAbstractCompactionStrategy createLegacyStrategy(CompactionParams compactionParams)
+    {
+        try
+        {
+            if (!LegacyAbstractCompactionStrategy.class.isAssignableFrom(compactionParams.klass()))
+                throw new IllegalArgumentException("Expected compaction params for legacy strategy: " + compactionParams);
+
+            Constructor<? extends CompactionStrategy> constructor =
+            compactionParams.klass().getConstructor(CompactionStrategyFactory.class, Map.class);
+            LegacyAbstractCompactionStrategy ret = (LegacyAbstractCompactionStrategy) constructor.newInstance(this, compactionParams.options());
+            compactionLogger.strategyCreated(ret);
+            return ret;
+        }
+        catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e)
+        {
+            throw org.apache.cassandra.utils.Throwables.cleaned(e);
+        }
+    }
+
+    /**
+     * Create a compaction strategy. This is only called by tiered storage so we forward to the legacy strategy.
+     */
+    public CompactionStrategy createStrategy(CompactionParams compactionParams)
+    {
+        return createLegacyStrategy(compactionParams);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
index becd3b954af1..1a12bcc77751 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
@@ -25,7 +25,6 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.commitlog.IntervalSet;
@@ -43,25 +42,25 @@
 
 public class CompactionStrategyHolder extends AbstractStrategyHolder
 {
-    private final List<AbstractCompactionStrategy> strategies = new ArrayList<>();
+    private final List<LegacyAbstractCompactionStrategy> strategies = new ArrayList<>();
     private final boolean isRepaired;
 
-    public CompactionStrategyHolder(ColumnFamilyStore cfs, DestinationRouter router, boolean isRepaired)
+    public CompactionStrategyHolder(CompactionRealm realm, CompactionStrategyFactory strategyFactory, DestinationRouter router, boolean isRepaired)
     {
-        super(cfs, router);
+        super(realm, strategyFactory, router);
         this.isRepaired = isRepaired;
     }
 
     @Override
     public void startup()
     {
-        strategies.forEach(AbstractCompactionStrategy::startup);
+        strategies.forEach(CompactionStrategy::startup);
     }
 
     @Override
     public void shutdown()
     {
-        strategies.forEach(AbstractCompactionStrategy::shutdown);
+        strategies.forEach(CompactionStrategy::shutdown);
     }
 
     @Override
@@ -69,7 +68,7 @@ public void setStrategyInternal(CompactionParams params, int numTokenPartitions)
     {
         strategies.clear();
         for (int i = 0; i < numTokenPartitions; i++)
-            strategies.add(cfs.createCompactionStrategyInstance(params));
+            strategies.add(strategyFactory.createLegacyStrategy(params));
     }
 
     @Override
@@ -89,43 +88,41 @@ public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair,
     }
 
     @Override
-    public AbstractCompactionStrategy getStrategyFor(SSTableReader sstable)
+    public LegacyAbstractCompactionStrategy getStrategyFor(CompactionSSTable sstable)
     {
         Preconditions.checkArgument(managesSSTable(sstable), "Attempting to get compaction strategy from wrong holder");
         return strategies.get(router.getIndexForSSTable(sstable));
     }
 
     @Override
-    public Iterable<AbstractCompactionStrategy> allStrategies()
+    public Iterable<LegacyAbstractCompactionStrategy> allStrategies()
     {
         return strategies;
     }
 
     @Override
-    public Collection<TaskSupplier> getBackgroundTaskSuppliers(long gcBefore)
+    public Collection<TasksSupplier> getBackgroundTaskSuppliers(long gcBefore)
     {
-        List<TaskSupplier> suppliers = new ArrayList<>(strategies.size());
-        for (AbstractCompactionStrategy strategy : strategies)
-            suppliers.add(new TaskSupplier(strategy.getEstimatedRemainingTasks(), () -> strategy.getNextBackgroundTask(gcBefore)));
+        List<TasksSupplier> suppliers = new ArrayList<>(strategies.size());
+        for (CompactionStrategy strategy : strategies)
+            suppliers.add(new TasksSupplier(strategy.getEstimatedRemainingTasks(), () -> strategy.getNextBackgroundTasks(gcBefore)));
 
         return suppliers;
     }
 
     @Override
-    public Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, boolean splitOutput)
+    public Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism)
     {
         List<AbstractCompactionTask> tasks = new ArrayList<>(strategies.size());
-        for (AbstractCompactionStrategy strategy : strategies)
+        for (CompactionStrategy strategy : strategies)
         {
-            Collection<AbstractCompactionTask> task = strategy.getMaximalTask(gcBefore, splitOutput);
-            if (task != null)
-                tasks.addAll(task);
+           tasks.addAll(strategy.getMaximalTasks(gcBefore, splitOutput, permittedParallelism));
         }
         return tasks;
     }
 
     @Override
-    public Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableContainer sstables, long gcBefore)
+    public Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableContainer<CompactionSSTable> sstables, long gcBefore)
     {
         List<AbstractCompactionTask> tasks = new ArrayList<>(strategies.size());
         for (int i = 0; i < strategies.size(); i++)
@@ -133,19 +130,13 @@ public Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableCont
             if (sstables.isGroupEmpty(i))
                 continue;
 
-            tasks.add(strategies.get(i).getUserDefinedTask(sstables.getGroup(i), gcBefore));
+            tasks.addAll(strategies.get(i).getUserDefinedTasks(sstables.getGroup(i), gcBefore));
         }
         return tasks;
     }
 
     @Override
-    public void addSSTable(SSTableReader sstable)
-    {
-        getStrategyFor(sstable).addSSTable(sstable);
-    }
-
-    @Override
-    public void addSSTables(GroupedSSTableContainer sstables)
+    public void addSSTables(GroupedSSTableContainer<CompactionSSTable> sstables)
     {
         Preconditions.checkArgument(sstables.numGroups() == strategies.size());
         for (int i = 0; i < strategies.size(); i++)
@@ -156,7 +147,7 @@ public void addSSTables(GroupedSSTableContainer sstables)
     }
 
     @Override
-    public void removeSSTables(GroupedSSTableContainer sstables)
+    public void removeSSTables(GroupedSSTableContainer<CompactionSSTable> sstables)
     {
         Preconditions.checkArgument(sstables.numGroups() == strategies.size());
         for (int i = 0; i < strategies.size(); i++)
@@ -167,7 +158,7 @@ public void removeSSTables(GroupedSSTableContainer sstables)
     }
 
     @Override
-    public void replaceSSTables(GroupedSSTableContainer removed, GroupedSSTableContainer added)
+    public void replaceSSTables(GroupedSSTableContainer<CompactionSSTable> removed, GroupedSSTableContainer<CompactionSSTable> added)
     {
         Preconditions.checkArgument(removed.numGroups() == strategies.size());
         Preconditions.checkArgument(added.numGroups() == strategies.size());
@@ -189,7 +180,7 @@ public AbstractCompactionStrategy first()
     }
 
     @Override
-    public List<ISSTableScanner> getScanners(GroupedSSTableContainer sstables, Collection<Range<Token>> ranges)
+    public List<ISSTableScanner> getScanners(GroupedSSTableContainer<SSTableReader> sstables, Collection<Range<Token>> ranges)
     {
         List<ISSTableScanner> scanners = new ArrayList<>(strategies.size());
         for (int i = 0; i < strategies.size(); i++)
@@ -202,13 +193,13 @@ public List<ISSTableScanner> getScanners(GroupedSSTableContainer sstables, Colle
         return scanners;
     }
 
-    Collection<Collection<SSTableReader>> groupForAnticompaction(Iterable<SSTableReader> sstables)
+    Collection<Collection<CompactionSSTable>> groupForAnticompaction(Iterable<? extends CompactionSSTable> sstables)
     {
         Preconditions.checkState(!isRepaired);
-        GroupedSSTableContainer group = createGroupedSSTableContainer();
+        GroupedSSTableContainer<CompactionSSTable> group = this.createGroupedSSTableContainer();
         sstables.forEach(group::add);
 
-        Collection<Collection<SSTableReader>> anticompactionGroups = new ArrayList<>();
+        Collection<Collection<CompactionSSTable>> anticompactionGroups = new ArrayList<>();
         for (int i = 0; i < strategies.size(); i++)
         {
             if (group.isGroupEmpty(i))
@@ -245,7 +236,7 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
         Preconditions.checkArgument(pendingRepair == null,
                                     "CompactionStrategyHolder can't create sstable writer with pendingRepair id");
         // to avoid creating a compaction strategy for the wrong pending repair manager, we get the index based on where the sstable is to be written
-        AbstractCompactionStrategy strategy = strategies.get(router.getIndexForSSTableDirectory(descriptor));
+        CompactionStrategy strategy = strategies.get(router.getIndexForSSTableDirectory(descriptor));
         return strategy.createSSTableMultiWriter(descriptor,
                                                  keyCount,
                                                  repairedAt,
@@ -259,13 +250,7 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
     }
 
     @Override
-    public int getStrategyIndex(AbstractCompactionStrategy strategy)
-    {
-        return strategies.indexOf(strategy);
-    }
-
-    @Override
-    public boolean containsSSTable(SSTableReader sstable)
+    public boolean containsSSTable(CompactionSSTable sstable)
     {
         return Iterables.any(strategies, acs -> acs.getSSTables().contains(sstable));
     }
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
index 010d4d77d253..36ae60677b93 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.db.compaction;
 
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -26,9 +25,9 @@
 import java.util.Comparator;
 import java.util.ConcurrentModificationException;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -36,24 +35,21 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
-import com.google.common.primitives.Longs;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.DiskBoundaries;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.commitlog.IntervalSet;
-import org.apache.cassandra.db.compaction.AbstractStrategyHolder.TaskSupplier;
-import org.apache.cassandra.db.compaction.PendingRepairManager.CleanupTask;
+import org.apache.cassandra.db.compaction.AbstractStrategyHolder.TasksSupplier;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
@@ -64,18 +60,13 @@
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.notifications.INotification;
-import org.apache.cassandra.notifications.INotificationConsumer;
 import org.apache.cassandra.notifications.SSTableAddedNotification;
 import org.apache.cassandra.notifications.SSTableDeletingNotification;
 import org.apache.cassandra.notifications.SSTableListChangedNotification;
-import org.apache.cassandra.notifications.SSTableMetadataChanged;
 import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
-import org.apache.cassandra.repair.consistent.admin.CleanupSummary;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.TimeUUID;
@@ -99,17 +90,18 @@
  *
  * Whenever the {@link DiskBoundaries} change, the compaction strategies must be reloaded, so in order to ensure
  * the compaction strategy placement reflect most up-to-date disk boundaries, call {@link this#maybeReloadDiskBoundaries()}
- * before acquiring the read lock to acess the strategies.
+ * before acquiring the read lock to access the strategies.
  *
  */
 
-public class CompactionStrategyManager implements INotificationConsumer
+public class CompactionStrategyManager implements CompactionStrategyContainer
 {
     private static final Logger logger = LoggerFactory.getLogger(CompactionStrategyManager.class);
     public final CompactionLogger compactionLogger;
-    private final ColumnFamilyStore cfs;
+    private final CompactionRealm realm;
     private final boolean partitionSSTablesByTokenRange;
     private final Supplier<DiskBoundaries> boundariesSupplier;
+    private final boolean enableAutoCompaction;
 
     /**
      * Performs mutual exclusion on the variables below
@@ -134,13 +126,16 @@ public class CompactionStrategyManager implements INotificationConsumer
     private volatile boolean isActive = true;
 
     /*
-        We keep a copy of the schema compaction parameters here to be able to decide if we
-        should update the compaction strategy in maybeReload() due to an ALTER.
+        We keep a copy of the table metadata compaction parameters here to be able to decide if we
+        should update the compaction strategy due to a metadata change such as a schema changed
+        caused by an ALTER TABLE.
 
-        If a user changes the local compaction strategy and then later ALTERs a compaction parameter,
-        we will use the new compaction parameters.
+        If a user changes the local compaction strategy via JMX and then later ALTERs a compaction parameter,
+        we will use the new compaction parameters but we will not override the JMX parameters if compaction
+        was not changed by the ALTER.
      */
-    private volatile CompactionParams schemaCompactionParams;
+    @SuppressWarnings("thread-safe")
+    private volatile CompactionParams metadataParams;
     private volatile boolean supportsEarlyOpen;
     private volatile int fanout;
     private volatile long maxSSTableSizeBytes;
@@ -148,18 +143,24 @@ should update the compaction strategy in maybeReload() due to an ALTER.
 
     public static int TWCS_BUCKET_COUNT_MAX = 128;
 
-    public CompactionStrategyManager(ColumnFamilyStore cfs)
+
+    public CompactionStrategyManager(CompactionStrategyFactory strategyFactory, boolean enableAutoCompaction)
     {
-        this(cfs, cfs::getDiskBoundaries, cfs.getPartitioner().splitter().isPresent());
+        this(strategyFactory,
+             () -> strategyFactory.getRealm().getDiskBoundaries(),
+             strategyFactory.getRealm().getPartitioner().splitter().isPresent(),
+             enableAutoCompaction);
     }
 
     @VisibleForTesting
-    public CompactionStrategyManager(ColumnFamilyStore cfs, Supplier<DiskBoundaries> boundariesSupplier,
-                                     boolean partitionSSTablesByTokenRange)
+    public CompactionStrategyManager(CompactionStrategyFactory strategyFactory,
+                                     Supplier<DiskBoundaries> boundariesSupplier,
+                                     boolean partitionSSTablesByTokenRange,
+                                     boolean enableAutoCompaction)
     {
         AbstractStrategyHolder.DestinationRouter router = new AbstractStrategyHolder.DestinationRouter()
         {
-            public int getIndexForSSTable(SSTableReader sstable)
+            public int getIndexForSSTable(CompactionSSTable sstable)
             {
                 return compactionStrategyIndexFor(sstable);
             }
@@ -169,68 +170,82 @@ public int getIndexForSSTableDirectory(Descriptor descriptor)
                 return compactionStrategyIndexForDirectory(descriptor);
             }
         };
-        transientRepairs = new PendingRepairHolder(cfs, router, true);
-        pendingRepairs = new PendingRepairHolder(cfs, router, false);
-        repaired = new CompactionStrategyHolder(cfs, router, true);
-        unrepaired = new CompactionStrategyHolder(cfs, router, false);
+
+        this.enableAutoCompaction = enableAutoCompaction;
+        realm = strategyFactory.getRealm();
+
+        transientRepairs = new PendingRepairHolder(realm, strategyFactory, router, true);
+        pendingRepairs = new PendingRepairHolder(realm, strategyFactory, router, false);
+        repaired = new CompactionStrategyHolder(realm, strategyFactory, router, true);
+        unrepaired = new CompactionStrategyHolder(realm, strategyFactory, router, false);
         holders = ImmutableList.of(transientRepairs, pendingRepairs, repaired, unrepaired);
 
-        cfs.getTracker().subscribe(this);
-        logger.trace("{} subscribed to the data tracker.", this);
-        this.cfs = cfs;
-        this.compactionLogger = new CompactionLogger(cfs, this);
+        compactionLogger = strategyFactory.getCompactionLogger();
         this.boundariesSupplier = boundariesSupplier;
         this.partitionSSTablesByTokenRange = partitionSSTablesByTokenRange;
 
         currentBoundaries = boundariesSupplier.get();
-        params = schemaCompactionParams = cfs.metadata().params.compaction;
+        params = realm.metadata().params.compaction;
         enabled = params.isEnabled();
-        setStrategy(schemaCompactionParams);
-        startup();
+    }
+
+    public static CompactionStrategyContainer create(@Nullable CompactionStrategyContainer previous,
+                                                     CompactionStrategyFactory strategyFactory,
+                                                     CompactionParams compactionParams,
+                                                     CompactionStrategyContainer.ReloadReason reason,
+                                                     boolean enableAutoCompaction)
+    {
+        CompactionStrategyManager csm = new CompactionStrategyManager(strategyFactory, enableAutoCompaction);
+        csm.reload(previous != null ? previous : csm, compactionParams, reason);
+        return csm;
     }
 
     /**
      * Return the next background task
      *
-     * Returns a task for the compaction strategy that needs it the most (most estimated remaining tasks)
-     */
-    public AbstractCompactionTask getNextBackgroundTask(long gcBefore)
+     * Legacy strategies will always return one task but we wrap this in a collection because new strategies
+     * might return multiple tasks.
+     *
+     * @return the task for the compaction strategy that needs it the most (most estimated remaining tasks)     */
+    @Override
+    public Collection<AbstractCompactionTask> getNextBackgroundTasks(long gcBefore)
     {
         maybeReloadDiskBoundaries();
         readLock.lock();
         try
         {
             if (!isEnabled())
-                return null;
+                return ImmutableList.of();
 
             int numPartitions = getNumTokenPartitions();
 
             // first try to promote/demote sstables from completed repairs
-            AbstractCompactionTask repairFinishedTask;
-            repairFinishedTask = pendingRepairs.getNextRepairFinishedTask();
-            if (repairFinishedTask != null)
-                return repairFinishedTask;
+            Collection<AbstractCompactionTask> repairFinishedTasks;
+            repairFinishedTasks = pendingRepairs.getNextRepairFinishedTasks();
+            if (!repairFinishedTasks.isEmpty())
+                return repairFinishedTasks;
 
-            repairFinishedTask = transientRepairs.getNextRepairFinishedTask();
-            if (repairFinishedTask != null)
-                return repairFinishedTask;
+            repairFinishedTasks = transientRepairs.getNextRepairFinishedTasks();
+            if (!repairFinishedTasks.isEmpty())
+                return repairFinishedTasks;
 
             // sort compaction task suppliers by remaining tasks descending
-            List<TaskSupplier> suppliers = new ArrayList<>(numPartitions * holders.size());
+            List<TasksSupplier> suppliers = new ArrayList<>(numPartitions * holders.size());
             for (AbstractStrategyHolder holder : holders)
                 suppliers.addAll(holder.getBackgroundTaskSuppliers(gcBefore));
 
             Collections.sort(suppliers);
 
-            // return the first non-null task
-            for (TaskSupplier supplier : suppliers)
+            // return the first non-empty list, we could enhance it to return all tasks of all
+            // suppliers but this would change existing behavior
+            for (TasksSupplier supplier : suppliers)
             {
-                AbstractCompactionTask task = supplier.getTask();
-                if (task != null)
-                    return task;
+                Collection<AbstractCompactionTask> tasks = supplier.getTasks();
+                if (!tasks.isEmpty())
+                    return tasks;
             }
 
-            return null;
+            return ImmutableList.of();
         }
         finally
         {
@@ -238,46 +253,25 @@ public AbstractCompactionTask getNextBackgroundTask(long gcBefore)
         }
     }
 
-    /**
-     * finds the oldest (by modification date) non-latest-version sstable on disk and creates an upgrade task for it
-     * @return
-     */
-    @VisibleForTesting
-    AbstractCompactionTask findUpgradeSSTableTask()
+    @Override
+    public CompactionLogger getCompactionLogger()
     {
-        if (!isEnabled() || !DatabaseDescriptor.automaticSSTableUpgrade())
-            return null;
-        Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
-        List<SSTableReader> potentialUpgrade = cfs.getLiveSSTables()
-                                                  .stream()
-                                                  .filter(s -> !compacting.contains(s) && !s.descriptor.version.isLatestVersion())
-                                                  .sorted((o1, o2) -> {
-                                                      File f1 = o1.descriptor.fileFor(Components.DATA);
-                                                      File f2 = o2.descriptor.fileFor(Components.DATA);
-                                                      return Longs.compare(f1.lastModified(), f2.lastModified());
-                                                  }).collect(Collectors.toList());
-        for (SSTableReader sstable : potentialUpgrade)
-        {
-            LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UPGRADE_SSTABLES);
-            if (txn != null)
-            {
-                logger.debug("Running automatic sstable upgrade for {}", sstable);
-                return getCompactionStrategyFor(sstable).getCompactionTask(txn, Integer.MIN_VALUE, Long.MAX_VALUE);
-            }
-        }
-        return null;
+        return compactionLogger;
     }
 
+    @Override
     public boolean isEnabled()
     {
-        return enabled && isActive;
+        return enableAutoCompaction && enabled && isActive;
     }
 
+    @Override
     public boolean isActive()
     {
         return isActive;
     }
 
+    @Override
     public void resume()
     {
         writeLock.lock();
@@ -296,6 +290,7 @@ public void resume()
      *
      * Separate call from enable/disable to not have to save the enabled-state externally
       */
+    @Override
     public void pause()
     {
         writeLock.lock();
@@ -310,19 +305,19 @@ public void pause()
 
     }
 
-    private void startup()
+    @Override
+    public void startup()
     {
         writeLock.lock();
         try
         {
-            for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
+            for (CompactionSSTable sstable : realm.getSSTables(SSTableSet.CANONICAL))
             {
-                if (sstable.openReason != SSTableReader.OpenReason.EARLY)
+                if (sstable.isSuitableForCompaction())
                     compactionStrategyFor(sstable).addSSTable(sstable);
             }
             holders.forEach(AbstractStrategyHolder::startup);
             supportsEarlyOpen = repaired.first().supportsEarlyOpen();
-            fanout = (repaired.first() instanceof LeveledCompactionStrategy) ? ((LeveledCompactionStrategy) repaired.first()).getLevelFanoutSize() : LeveledCompactionStrategy.DEFAULT_LEVEL_FANOUT_SIZE;
             maxSSTableSizeBytes = repaired.first().getMaxSSTableBytes();
             name = repaired.first().getName();
         }
@@ -331,25 +326,23 @@ private void startup()
             writeLock.unlock();
         }
 
-        if (repaired.first().logAll)
+        if (repaired.first().getOptions().isLogEnabled())
             compactionLogger.enable();
     }
 
     /**
-     * return the compaction strategy for the given sstable
-     *
      * returns differently based on the repaired status and which vnode the compaction strategy belongs to
      * @param sstable
-     * @return
+     * @return the compaction strategy for the given sstable
      */
-    public AbstractCompactionStrategy getCompactionStrategyFor(SSTableReader sstable)
+    LegacyAbstractCompactionStrategy getCompactionStrategyFor(CompactionSSTable sstable)
     {
         maybeReloadDiskBoundaries();
         return compactionStrategyFor(sstable);
     }
 
     @VisibleForTesting
-    AbstractCompactionStrategy compactionStrategyFor(SSTableReader sstable)
+    LegacyAbstractCompactionStrategy compactionStrategyFor(CompactionSSTable sstable)
     {
         // should not call maybeReloadDiskBoundaries because it may be called from within lock
         readLock.lock();
@@ -374,7 +367,7 @@ AbstractCompactionStrategy compactionStrategyFor(SSTableReader sstable)
      * @param sstable
      * @return
      */
-    int compactionStrategyIndexFor(SSTableReader sstable)
+    int compactionStrategyIndexFor(CompactionSSTable sstable)
     {
         // should not call maybeReloadDiskBoundaries because it may be called from within lock
         readLock.lock();
@@ -385,7 +378,7 @@ int compactionStrategyIndexFor(SSTableReader sstable)
             if (!partitionSSTablesByTokenRange)
                 return 0;
 
-            return currentBoundaries.getDiskIndex(sstable);
+            return currentBoundaries.getDiskIndexFromKey(sstable);
         }
         finally
         {
@@ -430,33 +423,7 @@ PendingRepairHolder getTransientRepairsUnsafe()
         return transientRepairs;
     }
 
-    public boolean hasDataForPendingRepair(TimeUUID sessionID)
-    {
-        readLock.lock();
-        try
-        {
-            return pendingRepairs.hasDataForSession(sessionID) || transientRepairs.hasDataForSession(sessionID);
-        }
-        finally
-        {
-            readLock.unlock();
-        }
-    }
-
-    @VisibleForTesting
-    public boolean hasPendingRepairSSTable(TimeUUID sessionID, SSTableReader sstable)
-    {
-        readLock.lock();
-        try
-        {
-            return pendingRepairs.hasPendingRepairSSTable(sessionID, sstable) || transientRepairs.hasPendingRepairSSTable(sessionID, sstable);
-        }
-        finally
-        {
-            readLock.unlock();
-        }
-    }
-
+    @Override
     public void shutdown()
     {
         writeLock.lock();
@@ -473,19 +440,30 @@ public void shutdown()
     }
 
     /**
-     * Maybe reload the compaction strategies. Called after changing configuration.
+     * Checks if the disk boundaries changed and reloads the compaction strategies
+     * to reflect the most up-to-date disk boundaries.
+     * <p>
+     * This is typically called before acquiring the {@link this#readLock} to ensure the most up-to-date
+     * disk locations and boundaries are used.
+     * <p>
+     * This should *never* be called inside by a thread holding the {@link this#readLock}, since it
+     * will potentially acquire the {@link this#writeLock} to update the compaction strategies
+     * what can cause a deadlock.
+     * <p>
+     * TODO: improve this to reload after receiving a notification rather than trying to reload on every operation
      */
-    public void maybeReloadParamsFromSchema(CompactionParams params)
+    @VisibleForTesting
+    protected void maybeReloadDiskBoundaries()
     {
-        // compare the old schema configuration to the new one, ignore any locally set changes.
-        if (params.equals(schemaCompactionParams))
+        if (!currentBoundaries.isOutOfDate())
             return;
 
         writeLock.lock();
         try
         {
-            if (!params.equals(schemaCompactionParams))
-                reloadParamsFromSchema(params);
+            if (!currentBoundaries.isOutOfDate())
+                return;
+            doReload(this, params, ReloadReason.DISK_BOUNDARIES_UPDATED);
         }
         finally
         {
@@ -493,125 +471,67 @@ public void maybeReloadParamsFromSchema(CompactionParams params)
         }
     }
 
-    /**
-     * @param newParams new CompactionParams set in via CQL
-     */
-    private void reloadParamsFromSchema(CompactionParams newParams)
-    {
-        logger.debug("Recreating compaction strategy for {}.{} - compaction parameters changed via CQL",
-                     cfs.getKeyspaceName(), cfs.getTableName());
-
-        /*
-         * It's possible for compaction to be explicitly enabled/disabled
-         * via JMX when already enabled/disabled via params. In that case,
-         * if we now toggle enabled/disabled via params, we'll technically
-         * be overriding JMX-set value with params-set value.
-         */
-        boolean enabledWithJMX = enabled && !shouldBeEnabled();
-        boolean disabledWithJMX = !enabled && shouldBeEnabled();
-
-        schemaCompactionParams = newParams;
-        setStrategy(newParams);
-
-        // enable/disable via JMX overrides CQL params, but please see the comment above
-        if (enabled && !shouldBeEnabled() && !enabledWithJMX)
-            disable();
-        else if (!enabled && shouldBeEnabled() && !disabledWithJMX)
-            enable();
-
-        startup();
-    }
-
-    private void maybeReloadParamsFromJMX(CompactionParams params)
+    @Override
+    public CompactionStrategyContainer reload(@Nonnull CompactionStrategyContainer previous, CompactionParams newCompactionParams, ReloadReason reason)
     {
-        // compare the old local configuration to the new one, ignoring schema
-        if (params.equals(this.params))
-            return;
-
         writeLock.lock();
         try
         {
-            if (!params.equals(this.params))
-                reloadParamsFromJMX(params);
+            doReload(previous, newCompactionParams, reason);
         }
         finally
         {
             writeLock.unlock();
         }
+        if (previous != this)
+            previous.shutdown();
+
+        return this;
     }
 
-    /**
-     * @param newParams new CompactionParams set via JMX
-     */
-    private void reloadParamsFromJMX(CompactionParams newParams)
+    private void doReload(CompactionStrategyContainer previous, CompactionParams compactionParams, ReloadReason reason)
     {
-        logger.debug("Recreating compaction strategy for {}.{} - compaction parameters changed via JMX",
-                     cfs.getKeyspaceName(), cfs.getTableName());
+        boolean updateDiskBoundaries = currentBoundaries == null || currentBoundaries.isOutOfDate();
+        boolean enabledOnReload = CompactionStrategyFactory.enableCompactionOnReload(previous, compactionParams, reason) && enableAutoCompaction;
 
-        setStrategy(newParams);
+        logger.debug("Recreating compaction strategy for {}.{}, reason: {}, params updated: {}, disk boundaries updated: {}, enabled: {}, params: {} -> {}, metadataParams: {}",
+                     realm.getKeyspaceName(), realm.getTableName(), reason, !compactionParams.equals(params), updateDiskBoundaries, enabledOnReload, params, compactionParams, metadataParams);
 
-        // compaction params set via JMX override enable/disable via JMX
-        if (enabled && !shouldBeEnabled())
-            disable();
-        else if (!enabled && shouldBeEnabled())
-            enable();
+        if (updateDiskBoundaries)
+            currentBoundaries = boundariesSupplier.get();
 
-        startup();
-    }
+        int numPartitions = getNumTokenPartitions();
+        for (AbstractStrategyHolder holder : holders)
+            holder.setStrategy(compactionParams, numPartitions);
 
-    /**
-     * Checks if the disk boundaries changed and reloads the compaction strategies
-     * to reflect the most up-to-date disk boundaries.
-     * <p>
-     * This is typically called before acquiring the {@link this#readLock} to ensure the most up-to-date
-     * disk locations and boundaries are used.
-     * <p>
-     * This should *never* be called inside by a thread holding the {@link this#readLock}, since it
-     * will potentially acquire the {@link this#writeLock} to update the compaction strategies
-     * what can cause a deadlock.
-     * <p>
-     * TODO: improve this to reload after receiving a notification rather than trying to reload on every operation
-     */
-    @VisibleForTesting
-    protected void maybeReloadDiskBoundaries()
-    {
-        if (!currentBoundaries.isOutOfDate())
-            return;
+        params = compactionParams;
 
-        writeLock.lock();
-        try
-        {
-            if (currentBoundaries.isOutOfDate())
-                reloadDiskBoundaries(boundariesSupplier.get());
-        }
-        finally
-        {
-            writeLock.unlock();
-        }
-    }
+        // full reload or switch from a strategy not managed by CompactionStrategyManager
+        if (metadataParams == null || reason == ReloadReason.FULL)
+            metadataParams = realm.metadata().params.compaction;
+        else if (reason == ReloadReason.METADATA_CHANGE)
+            // metadataParams are aligned with compactionParams. We do not access TableParams.COMPACTION to avoid racing with
+            // concurrent ALTER TABLE metadata change.
+            metadataParams = compactionParams;
+
+        // no-op for DISK_BOUNDARIES_UPDATED and JMX_REQUEST. DISK_BOUNDARIES_UPDATED does not change compaction params
+        // and JMX changes do not affect table metadata
 
-    /**
-     * @param newBoundaries new DiskBoundaries - potentially functionally equivalent to current ones
-     */
-    private void reloadDiskBoundaries(DiskBoundaries newBoundaries)
-    {
-        DiskBoundaries oldBoundaries = currentBoundaries;
-        currentBoundaries = newBoundaries;
 
-        if (newBoundaries.isEquivalentTo(oldBoundaries))
+        if (params.maxCompactionThreshold() <= 0 || params.minCompactionThreshold() <= 0)
         {
-            logger.debug("Not recreating compaction strategy for {}.{} - disk boundaries are equivalent",
-                         cfs.getKeyspaceName(), cfs.getTableName());
-            return;
+            logger.warn("Disabling compaction strategy by setting compaction thresholds to 0 is deprecated, set the compaction option 'enabled' to 'false' instead.");
+            disable();
         }
+        else if (!enabledOnReload)
+            disable();
+        else
+            enable();
 
-        logger.debug("Recreating compaction strategy for {}.{} - disk boundaries are out of date",
-                     cfs.getKeyspaceName(), cfs.getTableName());
-        setStrategy(params);
         startup();
     }
 
-    private Iterable<AbstractCompactionStrategy> getAllStrategies()
+    private Iterable<CompactionStrategy> getAllStrategies()
     {
         return Iterables.concat(Iterables.transform(holders, AbstractStrategyHolder::allStrategies));
     }
@@ -625,7 +545,7 @@ public int getUnleveledSSTables()
             if (repaired.first() instanceof LeveledCompactionStrategy)
             {
                 int count = 0;
-                for (AbstractCompactionStrategy strategy : getAllStrategies())
+                for (CompactionStrategy strategy : getAllStrategies())
                     count += ((LeveledCompactionStrategy) strategy).getLevelSize(0);
                 return count;
             }
@@ -637,11 +557,13 @@ public int getUnleveledSSTables()
         return 0;
     }
 
+    @Override
     public int getLevelFanoutSize()
     {
-        return fanout;
+        return repaired.first().getLevelFanoutSize();
     }
 
+    @Override
     public int[] getSSTableCountPerLevel()
     {
         maybeReloadDiskBoundaries();
@@ -651,19 +573,22 @@ public int[] getSSTableCountPerLevel()
             if (repaired.first() instanceof LeveledCompactionStrategy)
             {
                 int[] res = new int[LeveledGenerations.MAX_LEVEL_COUNT];
-                for (AbstractCompactionStrategy strategy : getAllStrategies())
+                for (CompactionStrategy strategy : getAllStrategies())
                 {
                     int[] repairedCountPerLevel = ((LeveledCompactionStrategy) strategy).getAllLevelSize();
                     res = sumArrays(res, repairedCountPerLevel);
                 }
                 return res;
             }
+            else
+            {
+                return new int[0];
+            }
         }
         finally
         {
             readLock.unlock();
         }
-        return null;
     }
 
     public long[] getPerLevelSizeBytes()
@@ -674,7 +599,7 @@ public long[] getPerLevelSizeBytes()
             if (repaired.first() instanceof LeveledCompactionStrategy)
             {
                 long [] res = new long[LeveledGenerations.MAX_LEVEL_COUNT];
-                for (AbstractCompactionStrategy strategy : getAllStrategies())
+                for (CompactionStrategy strategy : getAllStrategies())
                 {
                     long[] repairedCountPerLevel = ((LeveledCompactionStrategy) strategy).getAllLevelSizeBytes();
                     res = sumArrays(res, repairedCountPerLevel);
@@ -760,13 +685,13 @@ else if (i < a.length)
     /**
      * Should only be called holding the readLock
      */
-    private void handleFlushNotification(Iterable<SSTableReader> added)
+    private void handleFlushNotification(Iterable<? extends CompactionSSTable> added)
     {
-        for (SSTableReader sstable : added)
-            getHolder(sstable).addSSTable(sstable);
+        for (CompactionSSTable sstable : added)
+            compactionStrategyFor(sstable).addSSTable(sstable);
     }
 
-    private int getHolderIndex(SSTableReader sstable)
+    private int getHolderIndex(CompactionSSTable sstable)
     {
         for (int i = 0; i < holders.size(); i++)
         {
@@ -777,7 +702,7 @@ private int getHolderIndex(SSTableReader sstable)
         throw new IllegalStateException("No holder claimed " + sstable);
     }
 
-    private AbstractStrategyHolder getHolder(SSTableReader sstable)
+    private AbstractStrategyHolder getHolder(CompactionSSTable sstable)
     {
         for (AbstractStrategyHolder holder : holders)
         {
@@ -819,15 +744,16 @@ ImmutableList<AbstractStrategyHolder> getHolders()
      *
      * lives in matches the list index of the holder that's responsible for it
      */
-    public List<GroupedSSTableContainer> groupSSTables(Iterable<SSTableReader> sstables)
+    public <S extends CompactionSSTable>
+    List<GroupedSSTableContainer<S>> groupSSTables(Iterable<? extends S> sstables)
     {
-        List<GroupedSSTableContainer> classified = new ArrayList<>(holders.size());
+        List<GroupedSSTableContainer<S>> classified = new ArrayList<>(holders.size());
         for (AbstractStrategyHolder holder : holders)
         {
             classified.add(holder.createGroupedSSTableContainer());
         }
 
-        for (SSTableReader sstable : sstables)
+        for (S sstable : sstables)
         {
             classified.get(getHolderIndex(sstable)).add(sstable);
         }
@@ -838,10 +764,10 @@ public List<GroupedSSTableContainer> groupSSTables(Iterable<SSTableReader> sstab
     /**
      * Should only be called holding the readLock
      */
-    private void handleListChangedNotification(Iterable<SSTableReader> added, Iterable<SSTableReader> removed)
+    private void handleListChangedNotification(Iterable<? extends CompactionSSTable> added, Iterable<? extends CompactionSSTable> removed)
     {
-        List<GroupedSSTableContainer> addedGroups = groupSSTables(added);
-        List<GroupedSSTableContainer> removedGroups = groupSSTables(removed);
+        List<GroupedSSTableContainer<CompactionSSTable>> addedGroups = groupSSTables(added);
+        List<GroupedSSTableContainer<CompactionSSTable>> removedGroups = groupSSTables(removed);
         for (int i=0; i<holders.size(); i++)
         {
             holders.get(i).replaceSSTables(removedGroups.get(i), addedGroups.get(i));
@@ -851,12 +777,12 @@ private void handleListChangedNotification(Iterable<SSTableReader> added, Iterab
     /**
      * Should only be called holding the readLock
      */
-    private void handleRepairStatusChangedNotification(Iterable<SSTableReader> sstables)
+    private void handleRepairStatusChangedNotification(Iterable<? extends CompactionSSTable> sstables)
     {
-        List<GroupedSSTableContainer> groups = groupSSTables(sstables);
+        List<GroupedSSTableContainer<CompactionSSTable>> groups = groupSSTables(sstables);
         for (int i = 0; i < holders.size(); i++)
         {
-            GroupedSSTableContainer group = groups.get(i);
+            GroupedSSTableContainer<CompactionSSTable> group = groups.get(i);
 
             if (group.isEmpty())
                 continue;
@@ -877,15 +803,7 @@ private void handleRepairStatusChangedNotification(Iterable<SSTableReader> sstab
     /**
      * Should only be called holding the readLock
      */
-    private void handleMetadataChangedNotification(SSTableReader sstable, StatsMetadata oldMetadata)
-    {
-        compactionStrategyFor(sstable).metadataChanged(oldMetadata, sstable);
-    }
-
-    /**
-     * Should only be called holding the readLock
-     */
-    private void handleDeletingNotification(SSTableReader deleted)
+    private void handleDeletingNotification(CompactionSSTable deleted)
     {
         compactionStrategyFor(deleted).removeSSTable(deleted);
     }
@@ -917,11 +835,6 @@ else if (notification instanceof SSTableDeletingNotification)
             {
                 handleDeletingNotification(((SSTableDeletingNotification) notification).deleting);
             }
-            else if (notification instanceof SSTableMetadataChanged)
-            {
-                SSTableMetadataChanged lcNotification = (SSTableMetadataChanged) notification;
-                handleMetadataChangedNotification(lcNotification.sstable, lcNotification.oldMetadata);
-            }
         }
         finally
         {
@@ -929,6 +842,7 @@ else if (notification instanceof SSTableMetadataChanged)
         }
     }
 
+    @Override
     public void enable()
     {
         writeLock.lock();
@@ -943,6 +857,7 @@ public void enable()
         }
     }
 
+    @Override
     public void disable()
     {
         writeLock.lock();
@@ -964,19 +879,19 @@ public void disable()
      * @param ranges
      * @return
      */
-    public AbstractCompactionStrategy.ScannerList maybeGetScanners(Collection<SSTableReader> sstables,  Collection<Range<Token>> ranges)
+    private ScannerList maybeGetScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
     {
         maybeReloadDiskBoundaries();
         List<ISSTableScanner> scanners = new ArrayList<>(sstables.size());
         readLock.lock();
         try
         {
-            List<GroupedSSTableContainer> sstableGroups = groupSSTables(sstables);
+            List<GroupedSSTableContainer<SSTableReader>> sstableGroups = groupSSTables(sstables);
 
             for (int i = 0; i < holders.size(); i++)
             {
                 AbstractStrategyHolder holder = holders.get(i);
-                GroupedSSTableContainer group = sstableGroups.get(i);
+                GroupedSSTableContainer<SSTableReader> group = sstableGroups.get(i);
                 scanners.addAll(holder.getScanners(group, ranges));
             }
         }
@@ -988,10 +903,11 @@ public AbstractCompactionStrategy.ScannerList maybeGetScanners(Collection<SSTabl
         {
             readLock.unlock();
         }
-        return new AbstractCompactionStrategy.ScannerList(scanners);
+        return new ScannerList(scanners);
     }
 
-    public AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables,  Collection<Range<Token>> ranges)
+    @Override
+    public ScannerList getScanners(Collection<SSTableReader> sstables,  Collection<Range<Token>> ranges)
     {
         while (true)
         {
@@ -1006,12 +922,22 @@ public AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableRead
         }
     }
 
-    public AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables)
+    @Override
+    public ScannerList getScanners(Collection<SSTableReader> sstables)
     {
         return getScanners(sstables, null);
     }
 
-    public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> sstablesToGroup)
+    @Override
+    public Set<CompactionSSTable> getSSTables()
+    {
+        return getStrategies().stream()
+                              .flatMap(strategy -> strategy.getSSTables().stream())
+                              .collect(Collectors.toSet());
+    }
+
+    @Override
+    public Collection<Collection<CompactionSSTable>> groupSSTablesForAntiCompaction(Collection<? extends CompactionSSTable> sstablesToGroup)
     {
         maybeReloadDiskBoundaries();
         readLock.lock();
@@ -1025,45 +951,46 @@ public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Coll
         }
     }
 
+    @Override
     public long getMaxSSTableBytes()
     {
         return maxSSTableSizeBytes;
     }
 
-    public AbstractCompactionTask getCompactionTask(LifecycleTransaction txn, long gcBefore, long maxSSTableBytes)
+    @Override
+    public AbstractCompactionTask createCompactionTask(LifecycleTransaction txn, long gcBefore, long maxSSTableBytes)
     {
         maybeReloadDiskBoundaries();
         readLock.lock();
         try
         {
             validateForCompaction(txn.originals());
-            return compactionStrategyFor(txn.originals().iterator().next()).getCompactionTask(txn, gcBefore, maxSSTableBytes);
+            return compactionStrategyFor(txn.originals().iterator().next()).createCompactionTask(txn, gcBefore, maxSSTableBytes);
         }
         finally
         {
             readLock.unlock();
         }
-
     }
 
-    private void validateForCompaction(Iterable<SSTableReader> input)
+    private void validateForCompaction(Iterable<? extends CompactionSSTable> input)
     {
         readLock.lock();
         try
         {
-            SSTableReader firstSSTable = Iterables.getFirst(input, null);
+            CompactionSSTable firstSSTable = Iterables.getFirst(input, null);
             assert firstSSTable != null;
             boolean repaired = firstSSTable.isRepaired();
             int firstIndex = compactionStrategyIndexFor(firstSSTable);
             boolean isPending = firstSSTable.isPendingRepair();
-            TimeUUID pendingRepair = firstSSTable.getSSTableMetadata().pendingRepair;
-            for (SSTableReader sstable : input)
+            TimeUUID pendingRepair = firstSSTable.getPendingRepair();
+            for (CompactionSSTable sstable : input)
             {
                 if (sstable.isRepaired() != repaired)
                     throw new UnsupportedOperationException("You can't mix repaired and unrepaired data in a compaction");
                 if (firstIndex != compactionStrategyIndexFor(sstable))
                     throw new UnsupportedOperationException("You can't mix sstables from different directories in a compaction");
-                if (isPending && !pendingRepair.equals(sstable.getSSTableMetadata().pendingRepair))
+                if (isPending && !pendingRepair.equals(sstable.getPendingRepair()))
                     throw new UnsupportedOperationException("You can't compact sstables from different pending repair sessions");
             }
         }
@@ -1073,20 +1000,27 @@ private void validateForCompaction(Iterable<SSTableReader> input)
         }
     }
 
-    public CompactionTasks getMaximalTasks(final long gcBefore, final boolean splitOutput, OperationType operationType)
+    @Override
+    public CompactionTasks getMaximalTasks(final long gcBefore, final boolean splitOutput, int permittedParallelism)
+    {
+        return this.getMaximalTasks(gcBefore, splitOutput, permittedParallelism, OperationType.MAJOR_COMPACTION);
+    }
+
+    @Override
+    public synchronized CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism, OperationType operationType)
     {
         maybeReloadDiskBoundaries();
         // runWithCompactionsDisabled cancels active compactions and disables them, then we are able
         // to make the repaired/unrepaired strategies mark their own sstables as compacting. Once the
         // sstables are marked the compactions are re-enabled
-        return cfs.runWithCompactionsDisabled(() -> {
+        return realm.runWithCompactionsDisabled(() -> {
             List<AbstractCompactionTask> tasks = new ArrayList<>();
             readLock.lock();
             try
             {
                 for (AbstractStrategyHolder holder : holders)
                 {
-                    for (AbstractCompactionTask task: holder.getMaximalTasks(gcBefore, splitOutput))
+                    for (AbstractCompactionTask task: holder.getMaximalTasks(gcBefore, splitOutput, permittedParallelism))
                     {
                         tasks.add(task.setCompactionType(operationType));
                     }
@@ -1096,8 +1030,8 @@ public CompactionTasks getMaximalTasks(final long gcBefore, final boolean splitO
             {
                 readLock.unlock();
             }
-            return CompactionTasks.create(tasks);
-        }, operationType, false, false);
+            return CompactionTasks.create(CompositeCompactionTask.applyParallelismLimit(tasks, permittedParallelism));
+        }, operationType, false, false, TableOperation.StopTrigger.COMPACTION);
     }
 
     /**
@@ -1109,14 +1043,15 @@ public CompactionTasks getMaximalTasks(final long gcBefore, final boolean splitO
      * @param gcBefore gc grace period, throw away tombstones older than this
      * @return a list of compaction tasks corresponding to the sstables requested
      */
-    public CompactionTasks getUserDefinedTasks(Collection<SSTableReader> sstables, long gcBefore)
+    @Override
+    public CompactionTasks getUserDefinedTasks(Collection<? extends CompactionSSTable> sstables, long gcBefore)
     {
         maybeReloadDiskBoundaries();
         List<AbstractCompactionTask> ret = new ArrayList<>();
         readLock.lock();
         try
         {
-            List<GroupedSSTableContainer> groupedSSTables = groupSSTables(sstables);
+            List<GroupedSSTableContainer<CompactionSSTable>> groupedSSTables = groupSSTables(sstables);
             for (int i = 0; i < holders.size(); i++)
             {
                 ret.addAll(holders.get(i).getUserDefinedTasks(groupedSSTables.get(i), gcBefore));
@@ -1129,21 +1064,13 @@ public CompactionTasks getUserDefinedTasks(Collection<SSTableReader> sstables, l
         }
     }
 
+    @Override
     public int getEstimatedRemainingTasks()
     {
-        maybeReloadDiskBoundaries();
-        int tasks = 0;
-        readLock.lock();
-        try
-        {
-            for (AbstractCompactionStrategy strategy : getAllStrategies())
-                tasks += strategy.getEstimatedRemainingTasks();
-        }
-        finally
-        {
-            readLock.unlock();
-        }
-        return tasks;
+        return getStrategies(false).stream()
+                                   .flatMap(list -> list.stream())
+                                   .mapToInt(CompactionStrategy::getEstimatedRemainingTasks)
+                                   .sum();
     }
 
     public int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes, boolean isIncremental)
@@ -1157,7 +1084,7 @@ public int getEstimatedRemainingTasks(int additionalSSTables, long additionalByt
         {
             int tasks = pendingRepairs.getEstimatedRemainingTasks();
 
-            Iterable<AbstractCompactionStrategy> strategies;
+            Iterable<LegacyAbstractCompactionStrategy> strategies;
             if (isIncremental)
             {
                 // Note that it is unlikely that we are behind in the pending strategies (as they only have a small fraction
@@ -1186,19 +1113,32 @@ public int getEstimatedRemainingTasks(int additionalSSTables, long additionalByt
         }
     }
 
-    public boolean shouldBeEnabled()
+    @Override
+    public int getTotalCompactions()
     {
-        return params.isEnabled();
+        return getStrategies(false).stream()
+                                   .flatMap(list -> list.stream())
+                                   .mapToInt(CompactionStrategy::getTotalCompactions)
+                                   .sum();
     }
 
+    @Override
     public String getName()
     {
         return name;
     }
 
-    public List<List<AbstractCompactionStrategy>> getStrategies()
+    @Override
+    public List<CompactionStrategy> getStrategies()
     {
-        maybeReloadDiskBoundaries();
+        return getStrategies(true).stream().flatMap(List::stream).collect(Collectors.toList());
+    }
+
+    private List<List<CompactionStrategy>> getStrategies(boolean checkBoundaries)
+    {
+        if (checkBoundaries)
+            maybeReloadDiskBoundaries();
+
         readLock.lock();
         try
         {
@@ -1212,35 +1152,57 @@ public List<List<AbstractCompactionStrategy>> getStrategies()
         }
     }
 
-    public void overrideLocalParams(CompactionParams params)
+    @Override
+    public List<CompactionStrategy> getStrategies(boolean isRepaired, @Nullable TimeUUID pendingRepair)
     {
-        logger.info("Switching local compaction strategy from {} to {}", this.params, params);
-        maybeReloadParamsFromJMX(params);
+        readLock.lock();
+        try
+        {
+            if (isRepaired)
+                return Lists.newArrayList(repaired.allStrategies());
+            else if (pendingRepair != null)
+                return Lists.newArrayList(pendingRepairs.getStrategiesFor(pendingRepair));
+            else
+                return Lists.newArrayList(unrepaired.allStrategies());
+        }
+        finally
+        {
+            readLock.unlock();
+        }
     }
 
-    private int getNumTokenPartitions()
+    /**
+     * @return the statistics for the compaction strategies that have compactions in progress or pending
+     */
+    @Override
+    public List<CompactionStrategyStatistics> getStatistics()
     {
-        return partitionSSTablesByTokenRange ? currentBoundaries.directories.size() : 1;
+        return getStrategies(false).stream()
+                                   .flatMap(list -> list.stream())
+                                   .filter(strategy -> strategy.getTotalCompactions() > 0)
+                                   .map(CompactionStrategy::getStatistics)
+                                   .flatMap(List::stream)
+                                   .collect(Collectors.toList());
     }
 
-    private void setStrategy(CompactionParams params)
+    private int getNumTokenPartitions()
     {
-        int numPartitions = getNumTokenPartitions();
-        for (AbstractStrategyHolder holder : holders)
-            holder.setStrategy(params, numPartitions);
-        this.params = params;
+        return partitionSSTablesByTokenRange && currentBoundaries != null ? currentBoundaries.directories.size() : 1;
     }
 
+    @Override
     public CompactionParams getCompactionParams()
     {
         return params;
     }
 
-    public boolean onlyPurgeRepairedTombstones()
+    @Override
+    public CompactionParams getMetadataCompactionParams()
     {
-        return Boolean.parseBoolean(params.options().get(AbstractCompactionStrategy.ONLY_PURGE_REPAIRED_TOMBSTONES));
+        return metadataParams;
     }
 
+    @Override
     public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
                                                        long keyCount,
                                                        long repairedAt,
@@ -1274,123 +1236,70 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
         }
     }
 
-    public boolean isRepaired(AbstractCompactionStrategy strategy)
+    @Override
+    public boolean supportsEarlyOpen()
     {
-        return repaired.getStrategyIndex(strategy) >= 0;
+        return supportsEarlyOpen;
     }
 
-    public List<String> getStrategyFolders(AbstractCompactionStrategy strategy)
+    @Override
+    public void periodicReport()
     {
-        readLock.lock();
-        try
+        for (CompactionStrategy strat : getAllStrategies())
         {
-            Directories.DataDirectory[] locations = cfs.getDirectories().getWriteableLocations();
-            if (partitionSSTablesByTokenRange)
-            {
-                for (AbstractStrategyHolder holder : holders)
-                {
-                    int idx = holder.getStrategyIndex(strategy);
-                    if (idx >= 0)
-                        return Collections.singletonList(locations[idx].location.absolutePath());
-                }
-            }
-            List<String> folders = new ArrayList<>(locations.length);
-            for (Directories.DataDirectory location : locations)
-            {
-                folders.add(location.location.absolutePath());
-            }
-            return folders;
-        }
-        finally
-        {
-            readLock.unlock();
+            strat.periodicReport();
         }
     }
 
-    public boolean supportsEarlyOpen()
+    public ReentrantReadWriteLock.WriteLock getWriteLock()
     {
-        return supportsEarlyOpen;
+        return this.writeLock;
     }
 
+    /**
+     * This method is exposed for testing only
+     * @return the LocalSession sessionIDs of any pending repairs
+     */
     @VisibleForTesting
-    List<PendingRepairManager> getPendingRepairManagers()
+    public Set<TimeUUID> pendingRepairs()
     {
-        maybeReloadDiskBoundaries();
-        readLock.lock();
-        try
-        {
-            return Lists.newArrayList(pendingRepairs.getManagers());
-        }
-        finally
-        {
-            readLock.unlock();
-        }
+        Set<TimeUUID> ids = new HashSet<>();
+        pendingRepairs.getManagers().forEach(p -> ids.addAll(p.getSessions()));
+        return ids;
     }
 
-    /**
-     * Mutates sstable repairedAt times and notifies listeners of the change with the writeLock held. Prevents races
-     * with other processes between when the metadata is changed and when sstables are moved between strategies.
-      */
-    public void mutateRepaired(Collection<SSTableReader> sstables, long repairedAt, TimeUUID pendingRepair, boolean isTransient) throws IOException
+    @Override
+    public void repairSessionCompleted(TimeUUID sessionID)
     {
-        if (sstables.isEmpty())
-            return;
-        Set<SSTableReader> changed = new HashSet<>();
-
-        writeLock.lock();
-        try
-        {
-            for (SSTableReader sstable: sstables)
-            {
-                sstable.mutateRepairedAndReload(repairedAt, pendingRepair, isTransient);
-                verifyMetadata(sstable, repairedAt, pendingRepair, isTransient);
-                changed.add(sstable);
-            }
-        }
-        finally
-        {
-            try
-            {
-                // if there was an exception mutating repairedAt, we should still notify for the
-                // sstables that we were able to modify successfully before releasing the lock
-                cfs.getTracker().notifySSTableRepairedStatusChanged(changed);
-            }
-            finally
-            {
-                writeLock.unlock();
-            }
-        }
+        for (PendingRepairManager manager : pendingRepairs.getManagers())
+            manager.removeSessionIfEmpty(sessionID);
     }
 
-    private static void verifyMetadata(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair, boolean isTransient)
+    //
+    // CompactionObserver - because the strategies observe compactions, for CSM this is currently a no-op
+    //
+
+    @Override
+    public void onInProgress(CompactionProgress progress)
     {
-        if (!Objects.equals(pendingRepair, sstable.getPendingRepair()))
-            throw new IllegalStateException(String.format("Failed setting pending repair to %s on %s (pending repair is %s)", pendingRepair, sstable, sstable.getPendingRepair()));
-        if (repairedAt != sstable.getRepairedAt())
-            throw new IllegalStateException(String.format("Failed setting repairedAt to %d on %s (repairedAt is %d)", repairedAt, sstable, sstable.getRepairedAt()));
-        if (isTransient != sstable.isTransient())
-            throw new IllegalStateException(String.format("Failed setting isTransient to %b on %s (isTransient is %b)", isTransient, sstable, sstable.isTransient()));
+
     }
 
-    public CleanupSummary releaseRepairData(Collection<TimeUUID> sessions)
+    @Override
+    public void onCompleted(TimeUUID id, Throwable err)
     {
-        List<CleanupTask> cleanupTasks = new ArrayList<>();
-        readLock.lock();
-        try
-        {
-            for (PendingRepairManager prm : Iterables.concat(pendingRepairs.getManagers(), transientRepairs.getManagers()))
-                cleanupTasks.add(prm.releaseSessionData(sessions));
-        }
-        finally
-        {
-            readLock.unlock();
-        }
 
-        CleanupSummary summary = new CleanupSummary(cfs, Collections.emptySet(), Collections.emptySet());
+    }
+
+    @Override
+    public Map<String, String> getMaxOverlapsMap()
+    {
+        Map<String, String> result = new LinkedHashMap<>();
 
-        for (CleanupTask task : cleanupTasks)
-            summary = CleanupSummary.add(summary, task.cleanup());
+        for (AbstractStrategyHolder holder : holders)
+            for (LegacyAbstractCompactionStrategy strategy : holder.allStrategies())
+                result.putAll(strategy.getMaxOverlapsMap());
 
-        return summary;
+        return result;
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyOptions.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyOptions.java
new file mode 100644
index 000000000000..dc097e8a3e85
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyOptions.java
@@ -0,0 +1,502 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.utils.Throwables;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DEFAULT_COMPACTION_COSTS_READ_MULTIPLIER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DEFAULT_COMPACTION_LOGS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DEFAULT_COMPACTION_LOG_MINUTES;
+
+/**
+ * This class contains all compaction options that are shared by all strategies.
+ */
+public class CompactionStrategyOptions
+{
+    public static final int DEFAULT_MIN_THRESHOLD = 4;
+    public static final int DEFAULT_MAX_THRESHOLD = 32;
+    private static final Logger logger = LoggerFactory.getLogger(CompactionStrategyOptions.class);
+
+    public static final Map<String, String> DEFAULT_THRESHOLDS =
+    ImmutableMap.of(CompactionParams.Option.MIN_THRESHOLD.toString(), Integer.toString(DEFAULT_MIN_THRESHOLD),
+                    CompactionParams.Option.MAX_THRESHOLD.toString(), Integer.toString(DEFAULT_MAX_THRESHOLD));
+
+    public static final String ONLY_PURGE_REPAIRED_TOMBSTONES = "only_purge_repaired_tombstones";
+
+    public static final String DEFAULT_TOMBSTONE_THRESHOLD = "0.2";
+    // minimum interval needed to perform tombstone removal compaction in seconds, default 86400 or 1 day.
+    public static final String DEFAULT_TOMBSTONE_COMPACTION_INTERVAL = "86400";
+    public static final String DEFAULT_UNCHECKED_TOMBSTONE_COMPACTION_OPTION = "false";
+    public static final String DEFAULT_LOG_TYPE_OPTION = DEFAULT_COMPACTION_LOGS.getString("none");
+    public static final String DEFAULT_LOG_PERIOD_MINUTES_OPTION = DEFAULT_COMPACTION_LOG_MINUTES.getString("1");
+    public static final String DEFAULT_READ_MULTIPLIER_OPTION = DEFAULT_COMPACTION_COSTS_READ_MULTIPLIER.getString("1.0");
+    public static final String DEFAULT_WRITE_MULTIPLIER_OPTION = DEFAULT_COMPACTION_COSTS_READ_MULTIPLIER.getString("1.0");
+
+    public static final String TOMBSTONE_THRESHOLD_OPTION = "tombstone_threshold";
+    public static final String TOMBSTONE_COMPACTION_INTERVAL_OPTION = "tombstone_compaction_interval";
+    // disable range overlap check when deciding if an SSTable is candidate for tombstone compaction (CASSANDRA-6563)
+    public static final String UNCHECKED_TOMBSTONE_COMPACTION_OPTION = "unchecked_tombstone_compaction";
+    public static final String LOG_ALL_OPTION = "log_all";
+    public static final String LOG_TYPE_OPTION = "log";
+    public static final String LOG_PERIOD_MINUTES_OPTION = "log_period_minutes";
+
+    /** The multipliers can be used by users if they wish to adjust the costs. We reduce the read costs because writes are batch processes (flush and compaction)
+     * and therefore the costs tend to be lower that for reads, so by reducing read costs we make the costs more comparable.
+     */
+    public static final String READ_MULTIPLIER_OPTION = "costs_read_multiplier";
+    public static final String WRITE_MULTIPLIER_OPTION = "costs_write_multiplier";
+    public static final String COMPACTION_ENABLED = "enabled";
+
+    private final Class<? extends CompactionStrategy> klass;
+    private final Map<String, String> options;
+    private final float tombstoneThreshold;
+    private final long tombstoneCompactionInterval;
+    private final boolean uncheckedTombstoneCompaction;
+    private boolean disableTombstoneCompactions = false;
+    public enum LogType
+    {
+        NONE, EVENTS_ONLY, ALL;
+    }
+    private final LogType logType;
+    private final int logPeriodMinutes;
+    private final double readMultiplier;
+    private final double writeMultiplier;
+
+    public CompactionStrategyOptions(Class<? extends CompactionStrategy> klass, Map<String, String> options, boolean throwOnInvalidOption)
+    {
+        this.klass = klass;
+        this.options = copyOptions(klass, options);
+
+        boolean useDefault = false;
+        try
+        {
+            validate(); // will throw ConfigurationException if the options are invalid
+        }
+        catch (ConfigurationException e)
+        {
+            // when called from CompactionParams we throw but when called from AbstractCompactionStrategy we use defaults
+            // could probably not bother with the latter (?)
+            if (throwOnInvalidOption)
+            {
+                throw e;
+            }
+            else
+            {
+                logger.warn("Error setting compaction strategy options ({}), defaults will be used", e.getMessage());
+                useDefault = true;
+            }
+        }
+
+        tombstoneThreshold = Float.parseFloat(getOption(TOMBSTONE_THRESHOLD_OPTION, useDefault, DEFAULT_TOMBSTONE_THRESHOLD));
+        tombstoneCompactionInterval = Long.parseLong(getOption(TOMBSTONE_COMPACTION_INTERVAL_OPTION, useDefault, DEFAULT_TOMBSTONE_COMPACTION_INTERVAL));
+        uncheckedTombstoneCompaction = Boolean.parseBoolean(getOption(UNCHECKED_TOMBSTONE_COMPACTION_OPTION, useDefault, DEFAULT_UNCHECKED_TOMBSTONE_COMPACTION_OPTION));
+        if (options.containsKey(LOG_ALL_OPTION))
+        {
+            if (options.get(LOG_ALL_OPTION).equalsIgnoreCase("true"))
+                logType = LogType.ALL;
+            else
+                logType = LogType.NONE;
+        }
+        else
+            logType = LogType.valueOf(getOption(LOG_TYPE_OPTION, useDefault, DEFAULT_LOG_TYPE_OPTION).toUpperCase());
+        logPeriodMinutes = Integer.parseInt(getOption(LOG_PERIOD_MINUTES_OPTION, useDefault, DEFAULT_LOG_PERIOD_MINUTES_OPTION));
+        readMultiplier = Double.parseDouble(getOption(READ_MULTIPLIER_OPTION, useDefault, DEFAULT_READ_MULTIPLIER_OPTION));
+        writeMultiplier = Double.parseDouble(getOption(WRITE_MULTIPLIER_OPTION, useDefault, DEFAULT_WRITE_MULTIPLIER_OPTION));
+    }
+
+    private Map<String, String> copyOptions(Class<? extends CompactionStrategy> klass, Map<String, String> options)
+    {
+        Map<String, String> newOptions = new HashMap<>(options);
+
+        // For legacy compatibility reasons, for some compaction strategies we want to see the default min and max threshold
+        // in the compaction parameters that can be seen in CQL when retrieving the table from the schema tables so for
+        // these strategies we need to add these options when they have not been specified by the user
+        if (supportsThresholdParams(klass))
+        {
+            newOptions.putIfAbsent(CompactionParams.Option.MIN_THRESHOLD.toString(), Integer.toString(DEFAULT_MIN_THRESHOLD));
+            newOptions.putIfAbsent(CompactionParams.Option.MAX_THRESHOLD.toString(), Integer.toString(DEFAULT_MAX_THRESHOLD));
+        }
+
+        return newOptions;
+    }
+
+    /**
+     * All strategies except {@link UnifiedCompactionStrategy} support the minimum and maximum thresholds
+     */
+    @SuppressWarnings("unchecked")
+    public static boolean supportsThresholdParams(Class<? extends CompactionStrategy> klass)
+    {
+        try
+        {
+            Map<String, String> unrecognizedOptions =
+            (Map<String, String>) klass.getMethod("validateOptions", Map.class)
+                                       .invoke(null, DEFAULT_THRESHOLDS);
+
+            return unrecognizedOptions.isEmpty();
+        }
+        catch (Exception e)
+        {
+            throw Throwables.cleaned(e);
+        }
+    }
+
+    private String getOption(String optionName, boolean useDefault, String defaultValue)
+    {
+        if (useDefault)
+            return defaultValue;
+
+        String optionValue = options.get(optionName);
+        if (optionValue == null)
+            return defaultValue;
+
+        return optionValue;
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("class", klass.getName())
+                          .add("options", options)
+                          .toString();
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o)
+            return true;
+
+        if (!(o instanceof CompactionStrategyOptions))
+            return false;
+
+        CompactionStrategyOptions that = (CompactionStrategyOptions) o;
+
+        return klass.equals(that.klass) && options.equals(that.options);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(klass, options);
+    }
+
+    private Map<String, String> validate()
+    {
+        try
+        {
+            // Each strategy currently implements a static validateOptions() method for custom validation, the default behavior
+            // is to simply call validateOptions() below, through AbstractCompactionStrategy.validateOptions(), we could simplify
+            // all this assuming we don't need to support any user-defined compaction strategy
+            Map<String, String> unknownOptions = (Map<String, String>) klass.getMethod("validateOptions", Map.class).invoke(null, options);
+            if (!unknownOptions.isEmpty())
+            {
+                throw new ConfigurationException(format("Properties specified %s are not understood by %s",
+                                                        unknownOptions.keySet(),
+                                                        klass.getSimpleName()));
+            }
+
+            return unknownOptions;
+        }
+        catch (NoSuchMethodException e)
+        {
+            logger.warn("Compaction strategy {} does not have a static validateOptions method. Validation ignored", klass.getName());
+        }
+        catch (InvocationTargetException e)
+        {
+            if (e.getTargetException() instanceof ConfigurationException)
+                throw (ConfigurationException) e.getTargetException();
+
+            Throwable cause = e.getCause() == null
+                              ? e
+                              : e.getCause();
+
+            throw new ConfigurationException(format("%s.validateOptions() threw an error: %s %s",
+                                                    klass.getName(),
+                                                    cause.getClass().getName(),
+                                                    cause.getMessage()),
+                                             e);
+        }
+        catch (IllegalAccessException e)
+        {
+            throw new ConfigurationException("Cannot access method validateOptions in " + klass.getName(), e);
+        }
+
+        if (minCompactionThreshold() <= 0 || maxCompactionThreshold() <= 0)
+        {
+            throw new ConfigurationException("Disabling compaction by setting compaction thresholds to 0 has been removed,"
+                                             + " set the compaction option 'enabled' to false instead.");
+        }
+
+        if (minCompactionThreshold() <= 1)
+        {
+            throw new ConfigurationException(format("Min compaction threshold cannot be less than 2 (got %d)",
+                                                    minCompactionThreshold()));
+        }
+
+        if (minCompactionThreshold() > maxCompactionThreshold())
+        {
+            throw new ConfigurationException(format("Min compaction threshold (got %d) cannot be greater than max compaction threshold (got %d)",
+                                                    minCompactionThreshold(),
+                                                    maxCompactionThreshold()));
+        }
+
+        return options;
+    }
+
+    public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
+    {
+        String minThreshold = options.get(CompactionParams.Option.MIN_THRESHOLD.toString());
+        if (minThreshold != null && !StringUtils.isNumeric(minThreshold))
+        {
+            throw new ConfigurationException(format("Invalid value %s for '%s' compaction sub-option - must be an integer",
+                                                    minThreshold,
+                                                    CompactionParams.Option.MIN_THRESHOLD));
+        }
+
+        String maxThreshold = options.get(CompactionParams.Option.MAX_THRESHOLD.toString());
+        if (maxThreshold != null && !StringUtils.isNumeric(maxThreshold))
+        {
+            throw new ConfigurationException(format("Invalid value %s for '%s' compaction sub-option - must be an integer",
+                                                    maxThreshold,
+                                                    CompactionParams.Option.MAX_THRESHOLD));
+        }
+
+        String threshold = options.get(TOMBSTONE_THRESHOLD_OPTION);
+        if (threshold != null)
+        {
+            try
+            {
+                float thresholdValue = Float.parseFloat(threshold);
+                if (thresholdValue < 0)
+                {
+                    throw new ConfigurationException(String.format("%s must be greater than 0, but was %f", TOMBSTONE_THRESHOLD_OPTION, thresholdValue));
+                }
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", threshold, TOMBSTONE_THRESHOLD_OPTION), e);
+            }
+        }
+
+        String interval = options.get(TOMBSTONE_COMPACTION_INTERVAL_OPTION);
+        if (interval != null)
+        {
+            try
+            {
+                long tombstoneCompactionInterval = Long.parseLong(interval);
+                if (tombstoneCompactionInterval < 0)
+                {
+                    throw new ConfigurationException(String.format("%s must be greater than 0, but was %d", TOMBSTONE_COMPACTION_INTERVAL_OPTION, tombstoneCompactionInterval));
+                }
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", interval, TOMBSTONE_COMPACTION_INTERVAL_OPTION), e);
+            }
+        }
+
+        String unchecked = options.get(UNCHECKED_TOMBSTONE_COMPACTION_OPTION);
+        if (unchecked != null && !unchecked.equalsIgnoreCase("true") && !unchecked.equalsIgnoreCase("false"))
+        {
+            throw new ConfigurationException(String.format("'%s' should be either 'true' or 'false', not '%s'", UNCHECKED_TOMBSTONE_COMPACTION_OPTION, unchecked));
+        }
+
+        String logAll = options.get(LOG_ALL_OPTION);
+        if (logAll != null && !logAll.equalsIgnoreCase("true") && !logAll.equalsIgnoreCase("false"))
+        {
+            throw new ConfigurationException(String.format("'%s' should either be 'true' or 'false', not %s", LOG_ALL_OPTION, logAll));
+        }
+
+        String logType = options.get(LOG_TYPE_OPTION);
+        if (logType != null && !logType.equalsIgnoreCase("all") && !logType.equalsIgnoreCase("events_only") && !logType.equalsIgnoreCase("none"))
+        {
+            throw new ConfigurationException(String.format("'%s' should either be 'all' or 'events_only' or 'none', not %s", LOG_TYPE_OPTION, logType));
+        }
+
+        if (logAll != null && logType != null)
+        {
+            throw new ConfigurationException(String.format("Either '%s' or '%s' should be used, not both", LOG_ALL_OPTION, LOG_TYPE_OPTION));
+        }
+
+        String logPeriodMinutes = options.get(LOG_PERIOD_MINUTES_OPTION);
+        if (logPeriodMinutes != null)
+        {
+            try
+            {
+                long minutes = Integer.parseInt(logPeriodMinutes);
+                if (minutes < 1)
+                {
+                    throw new ConfigurationException(String.format("%s must be greater than or equal to 1, but was %d", LOG_PERIOD_MINUTES_OPTION, minutes));
+                }
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", logPeriodMinutes, LOG_PERIOD_MINUTES_OPTION), e);
+            }
+        }
+
+        String readMultiplier = options.get(READ_MULTIPLIER_OPTION);
+        if (readMultiplier != null)
+        {
+            try
+            {
+                double multiplier = Double.parseDouble(readMultiplier);
+                if (!(multiplier > 0 && multiplier <= 1))
+                {
+                    throw new ConfigurationException(String.format("%s must be between 0 and 1, but was %d", READ_MULTIPLIER_OPTION, multiplier));
+                }
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s is not a parsable double (base10) for %s", readMultiplier, READ_MULTIPLIER_OPTION), e);
+            }
+        }
+
+        String writeMultiplier = options.get(WRITE_MULTIPLIER_OPTION);
+        if (writeMultiplier != null)
+        {
+            try
+            {
+                double multiplier = Double.parseDouble(writeMultiplier);
+                if (!(multiplier > 0 && multiplier <= 1))
+                {
+                    throw new ConfigurationException(String.format("%s must be between 0 and 1, but was %d", WRITE_MULTIPLIER_OPTION, multiplier));
+                }
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s is not a parsable double (base10) for %s", writeMultiplier, WRITE_MULTIPLIER_OPTION), e);
+            }
+        }
+
+        String compactionEnabled = options.get(COMPACTION_ENABLED);
+        if (compactionEnabled != null && !compactionEnabled.equalsIgnoreCase("true") && !compactionEnabled.equalsIgnoreCase("false"))
+        {
+            throw new ConfigurationException(String.format("enabled should either be 'true' or 'false', not %s", compactionEnabled));
+        }
+
+        Map<String, String> uncheckedOptions = new HashMap<>(options);
+        uncheckedOptions.remove(TOMBSTONE_THRESHOLD_OPTION);
+        uncheckedOptions.remove(TOMBSTONE_COMPACTION_INTERVAL_OPTION);
+        uncheckedOptions.remove(UNCHECKED_TOMBSTONE_COMPACTION_OPTION);
+        uncheckedOptions.remove(LOG_ALL_OPTION);
+        uncheckedOptions.remove(LOG_TYPE_OPTION);
+        uncheckedOptions.remove(LOG_PERIOD_MINUTES_OPTION);
+        uncheckedOptions.remove(READ_MULTIPLIER_OPTION);
+        uncheckedOptions.remove(WRITE_MULTIPLIER_OPTION);
+        uncheckedOptions.remove(COMPACTION_ENABLED);
+        uncheckedOptions.remove(ONLY_PURGE_REPAIRED_TOMBSTONES);
+        uncheckedOptions.remove(CompactionParams.Option.PROVIDE_OVERLAPPING_TOMBSTONES.toString());
+        return uncheckedOptions;
+    }
+
+    public int minCompactionThreshold()
+    {
+        String threshold = options.get(CompactionParams.Option.MIN_THRESHOLD.toString());
+        return threshold == null
+               ? DEFAULT_MIN_THRESHOLD
+               : Integer.parseInt(threshold);
+    }
+
+    public int maxCompactionThreshold()
+    {
+        String threshold = options.get(CompactionParams.Option.MAX_THRESHOLD.toString());
+        return threshold == null
+               ? DEFAULT_MAX_THRESHOLD
+               : Integer.parseInt(threshold);
+    }
+
+    public Class<? extends CompactionStrategy> klass()
+    {
+        return klass;
+    }
+
+    public Map<String, String> getOptions()
+    {
+        return options;
+    }
+
+    public float getTombstoneThreshold()
+    {
+        return tombstoneThreshold;
+    }
+
+    public long getTombstoneCompactionInterval()
+    {
+        return tombstoneCompactionInterval;
+    }
+
+    public boolean isUncheckedTombstoneCompaction()
+    {
+        return uncheckedTombstoneCompaction;
+    }
+
+    public boolean isDisableTombstoneCompactions()
+    {
+        return disableTombstoneCompactions;
+    }
+
+    /**
+     * {@link TimeWindowCompactionStrategy} disable this parameter if other parameters aren't available.
+     */
+    public void setDisableTombstoneCompactions(boolean disableTombstoneCompactions)
+    {
+        this.disableTombstoneCompactions = disableTombstoneCompactions;
+    }
+
+    public boolean isLogEnabled()
+    {
+        return (logType == LogType.ALL || logType == LogType.EVENTS_ONLY);
+    }
+
+    public boolean isLogAll()
+    {
+        return (logType == LogType.ALL);
+    }
+
+    public int getLogPeriodMinutes()
+    {
+        return logPeriodMinutes;
+    }
+
+    public double getReadMultiplier()
+    {
+        return readMultiplier;
+    }
+
+    public double getWriteMultiplier()
+    {
+        return writeMultiplier;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyStatistics.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyStatistics.java
new file mode 100644
index 000000000000..307810b912ee
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyStatistics.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * The statistics for a compaction strategy, to be published over JMX and insights.
+ * <p/>
+ * Implements serializable to allow structured info to be returned via JMX.  The JSON
+ * properties are published to insights so changing them has a downstream impact.
+ */
+public class CompactionStrategyStatistics implements Serializable
+{
+    private static final long serialVersionUID = 3695927592357744816L;
+
+    private final String keyspace;
+    private final String table;
+    private final String strategy;
+    private final List<CompactionAggregateStatistics> aggregates;
+
+    CompactionStrategyStatistics(TableMetadata metadata,
+                                 String strategy,
+                                 List<CompactionAggregateStatistics> aggregates)
+    {
+        this.keyspace = metadata.keyspace;
+        this.table = metadata.name;
+        this.strategy = strategy;
+        this.aggregates = new ArrayList<>(aggregates);
+    }
+
+    public String keyspace()
+    {
+        return keyspace;
+    }
+
+    public String table()
+    {
+        return table;
+    }
+
+    @JsonProperty
+    public String strategy()
+    {
+        return strategy;
+    }
+
+    @JsonProperty
+    public List<CompactionAggregateStatistics> aggregates()
+    {
+        return aggregates;
+    }
+
+    @Override
+    public String toString()
+    {
+        StringBuilder ret = new StringBuilder(1024);
+        ret.append(keyspace)
+           .append('.')
+           .append(table)
+           .append('/')
+           .append(strategy)
+           .append('\n');
+
+        if (!aggregates.isEmpty())
+        {
+            Collection<String> header = aggregates.get(0).header(); // all headers are identical
+            int[] lengths = new int[header.size()]; // the max lengths of each column
+            Iterator<String> it = header.iterator();
+
+            for (int i = 0; i < lengths.length; i++)
+                lengths[i] = it.next().length();
+
+            Map<String, List<String[]>> rowsByShard = new LinkedHashMap<>();
+            for (CompactionAggregateStatistics aggregate : aggregates)
+            {
+                String shard = aggregate.shard();
+                List<String[]> rows = rowsByShard.computeIfAbsent(shard, key -> new ArrayList<>(aggregates.size()));
+                String[] data = new String[header.size()];
+
+                it = aggregate.data().iterator();
+                for (int i = 0; i < lengths.length; i++)
+                {
+                    data[i] = it.next();
+                    if (data[i].length() > lengths[i])
+                        lengths[i] = data[i].length();
+                }
+
+                rows.add(data);
+            }
+
+            for (Map.Entry<String, List<String[]>> entry : rowsByShard.entrySet())
+            {
+                // optional shard
+                if (!entry.getKey().isEmpty())
+                    ret.append("Shard/").append(entry.getKey()).append('\n');
+
+                // header
+                it = header.iterator();
+                for (int i = 0; i < header.size(); i++)
+                    ret.append(String.format("%-" + lengths[i] + "s\t", it.next()));
+
+                ret.append('\n');
+
+                // rows
+                for (String[] row : entry.getValue())
+                {
+                    for (int i = 0; i < row.length; i++)
+                        ret.append(String.format("%-" + lengths[i] + "s\t", row[i]));
+
+                    ret.append('\n');
+                }
+
+                ret.append('\n');
+            }
+        }
+
+        return ret.toString();
+    }
+
+    Collection<String> getHeader()
+    {
+        return aggregates.isEmpty() ? ImmutableList.of() : aggregates.get(0).header();
+    }
+
+    Collection<Collection<String>> getData()
+    {
+        return aggregates.stream().map(CompactionAggregateStatistics::data).collect(Collectors.toList());
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 4ca0e0f53ff5..f6fe778b9bd1 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -17,90 +17,180 @@
  */
 package org.apache.cassandra.db.compaction;
 
+import java.io.Closeable;
+import java.io.IOException;
 import java.time.Instant;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.annotation.Nullable;
 
-import com.google.common.base.Predicate;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.RateLimiter;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.unified.UnifiedCompactionTask;
 import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
 import org.apache.cassandra.db.compaction.writers.DefaultCompactionWriter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Refs;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMPACTION_HISTORY_ENABLED;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_CURSOR_COMPACTION;
 import static org.apache.cassandra.db.compaction.CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.FBUtilities.now;
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemoryPerSecond;
 
 public class CompactionTask extends AbstractCompactionTask
 {
     protected static final Logger logger = LoggerFactory.getLogger(CompactionTask.class);
+
     protected final long gcBefore;
     protected final boolean keepOriginals;
-    protected static long totalBytesCompacted = 0;
-    private ActiveCompactionsTracker activeCompactions;
-
-    public CompactionTask(ColumnFamilyStore cfs, LifecycleTransaction txn, long gcBefore)
+    /** for trace logging purposes only */
+    private static final AtomicLong totalBytesCompacted = new AtomicLong();
+
+    // The compaction strategy is not necessarily available for all compaction tasks (e.g. GC or sstable splitting)
+    @Nullable
+    private final CompactionStrategy strategy;
+    protected OperationTotals totals;
+
+    public CompactionTask(CompactionRealm realm,
+                          ILifecycleTransaction txn,
+                          long gcBefore,
+                          boolean keepOriginals,
+                          @Nullable CompactionStrategy strategy)
     {
-        this(cfs, txn, gcBefore, false);
+        this(realm, txn, null, gcBefore, keepOriginals, strategy, strategy);
     }
 
-    public CompactionTask(ColumnFamilyStore cfs, LifecycleTransaction txn, long gcBefore, boolean keepOriginals)
+    public CompactionTask(CompactionRealm realm,
+                          ILifecycleTransaction txn,
+                          OperationTotals totals,
+                          long gcBefore,
+                          boolean keepOriginals,
+                          @Nullable CompactionStrategy strategy,
+                          CompactionObserver observer)
     {
-        super(cfs, txn);
+        super(realm, txn);
         this.gcBefore = gcBefore;
         this.keepOriginals = keepOriginals;
+        this.strategy = strategy;
+        this.totals = totals;
+
+        if (observer != null)
+            addObserver(observer);
+
+        logger.debug("Created compaction task with id {} and strategy {}", txn.opIdString(), strategy);
+    }
+
+
+    /**
+     * Create a compaction task for deleted data collection.
+     */
+    public static AbstractCompactionTask forGarbageCollection(CompactionRealm realm,
+                                                              ILifecycleTransaction txn,
+                                                              long gcBefore,
+                                                              CompactionParams.TombstoneOption tombstoneOption)
+    {
+        return new GarbageCollectionTask(realm, txn, gcBefore, tombstoneOption);
     }
 
-    public static synchronized long addToTotalBytesCompacted(long bytesCompacted)
+    public static class GarbageCollectionTask extends CompactionTask
     {
-        return totalBytesCompacted += bytesCompacted;
+        private final CompactionParams.TombstoneOption tombstoneOption;
+
+        public GarbageCollectionTask(CompactionRealm realm, ILifecycleTransaction txn, long gcBefore, CompactionParams.TombstoneOption tombstoneOption)
+        {
+            super(realm, txn, gcBefore, false, null);
+            this.tombstoneOption = tombstoneOption;
+            setCompactionType(OperationType.GARBAGE_COLLECT);
+            setUserDefined(true);
+        }
+
+        @Override
+        protected CompactionController getCompactionController(Set<SSTableReader> toCompact)
+        {
+            return new CompactionController(realm, toCompact, gcBefore, null, tombstoneOption);
+        }
+
+        @Override
+        protected int getLevel()
+        {
+            return transaction.onlyOne().getSSTableLevel();
+        }
     }
 
-    protected int executeInternal(ActiveCompactionsTracker activeCompactions)
+    private static long addToTotalBytesCompacted(long bytesCompacted)
     {
-        this.activeCompactions = activeCompactions == null ? ActiveCompactionsTracker.NOOP : activeCompactions;
-        run();
-        return transaction.originals().size();
+        return totalBytesCompacted.addAndGet(bytesCompacted);
     }
 
+    /*
+     *  Find the maximum size file in the list .
+     */
+    private SSTableReader getMaxSizeFile(Iterable<SSTableReader> sstables)
+    {
+        long maxSize = 0L;
+        SSTableReader maxFile = null;
+        for (SSTableReader sstable : sstables)
+        {
+            if (sstable.onDiskLength() > maxSize)
+            {
+                maxSize = sstable.onDiskLength();
+                maxFile = sstable;
+            }
+        }
+        return maxFile;
+    }
+
+    @VisibleForTesting
     public boolean reduceScopeForLimitedSpace(Set<SSTableReader> nonExpiredSSTables, long expectedSize)
     {
-        if (partialCompactionsAcceptable() && transaction.originals().size() > 1)
+        if (partialCompactionsAcceptable() && nonExpiredSSTables.size() > 1)
         {
             // Try again w/o the largest one.
-            SSTableReader removedSSTable = cfs.getMaxSizeFile(nonExpiredSSTables);
+            SSTableReader removedSSTable = getMaxSizeFile(nonExpiredSSTables);
             logger.warn("insufficient space to compact all requested files. {}MiB required, {} for compaction {} - removing largest SSTable: {}",
                         (float) expectedSize / 1024 / 1024,
-                        StringUtils.join(transaction.originals(), ", "),
-                        transaction.opId(),
+                        StringUtils.join(nonExpiredSSTables, ", "),
+                        transaction.opIdString(),
                         removedSSTable);
             // Note that we have removed files that are still marked as compacting.
             // This suboptimal but ok since the caller will unmark all the sstables at the end.
             transaction.cancel(removedSSTable);
+            nonExpiredSSTables.remove(removedSSTable);
             return true;
         }
         return false;
@@ -111,208 +201,741 @@ public boolean reduceScopeForLimitedSpace(Set<SSTableReader> nonExpiredSSTables,
      * which are properly serialized.
      * Caller is in charge of marking/unmarking the sstables as compacting.
      */
+    @Override
     protected void runMayThrow() throws Exception
     {
         // The collection of sstables passed may be empty (but not null); even if
         // it is not empty, it may compact down to nothing if all rows are deleted.
         assert transaction != null;
 
-        if (transaction.originals().isEmpty())
+        if (inputSSTables().isEmpty())
             return;
 
-        // Note that the current compaction strategy, is not necessarily the one this task was created under.
-        // This should be harmless; see comments to CFS.maybeReloadCompactionStrategy.
-        CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
-
         if (DatabaseDescriptor.isSnapshotBeforeCompaction())
         {
             Instant creationTime = now();
-            cfs.snapshotWithoutMemtable(creationTime.toEpochMilli() + "-compact-" + cfs.name, creationTime);
+            realm.snapshotWithoutMemtable(creationTime.toEpochMilli() + "-compact-" + realm.getTableName(), creationTime);
         }
 
-        try (CompactionController controller = getCompactionController(transaction.originals()))
+        // The set of sstables given here may be later modified by buildCompactionCandidatesForAvailableDiskSpace() and
+        // the compaction iterators in CompactionController and OverlapTracker will reflect the updated set of sstables.
+        try (CompactionController controller = getCompactionController(inputSSTables());
+             CompactionOperation operation = createCompactionOperation(controller, strategy))
+        {
+            operation.execute();
+        }
+    }
+
+    /**
+     * @return The token range that the operation should compact. This is usually null, but if we have a parallelizable
+     * multi-task operation (see {@link UnifiedCompactionStrategy#createAndAddTasks}), it will specify a subrange.
+     */
+    protected Range<Token> tokenRange()
+    {
+        return null;
+    }
+
+    /**
+     * If this is a partial compaction, its progress reports are shared between tasks. This method returns the shared
+     * progress object.
+     */
+    protected SharedCompactionProgress sharedProgress()
+    {
+        return null;
+    }
+
+    /**
+     * @return The set of input sstables for this compaction. This must be a subset of the transaction originals and
+     * must reflect any removal of sstables from the originals set for correct overlap tracking.
+     * See {@link UnifiedCompactionTask} for an example.
+     */
+    protected Set<SSTableReader> inputSSTables()
+    {
+        return transaction.originals();
+    }
+
+    /**
+     * @return True if the task should try to limit the operation size to the available space by removing sstables from
+     * the compacting set. This cannot be done if this is part of a multi-task operation with a shared transaction.
+     */
+    protected boolean shouldReduceScopeForSpace()
+    {
+        return true;
+    }
+
+    private CompactionOperation createCompactionOperation(CompactionController controller, CompactionStrategy strategy)
+    {
+        Set<CompactionSSTable> fullyExpiredSSTables = controller.getFullyExpiredSSTables();
+        Set<SSTableReader> actuallyCompact = new HashSet<>(inputSSTables());
+        actuallyCompact.removeAll(fullyExpiredSSTables);
+        // select SSTables to compact based on available disk space.
+        if (shouldReduceScopeForSpace() && !buildCompactionCandidatesForAvailableDiskSpace(actuallyCompact, transaction.opId(), !fullyExpiredSSTables.isEmpty()))
         {
+            // The set of sstables has changed (one or more were excluded due to limited available disk space).
+            // We need to recompute the overlaps between sstables. The iterators used in the compaction controller
+            // and tracker will reflect the changed set of sstables made by LifecycleTransaction.cancel(),
+            // so refreshing the overlaps will be based on the updated set of sstables.
+            controller.refreshOverlaps();
+        }
 
-            final Set<SSTableReader> fullyExpiredSSTables = controller.getFullyExpiredSSTables();
+        // Calculate the operation total sizes if not already set
+        if (totals == null)
+            totals = getOperationTotals(actuallyCompact, tokenRange());
+
+        // sanity check: sstables to compact is a subset of the transaction originals
+        assert transaction.originals().containsAll(actuallyCompact);
+        // sanity check: all sstables must belong to the same table
+        assert !Iterables.any(transaction.originals(), sstable -> !sstable.descriptor.cfname.equals(realm.getTableName()));
+
+
+        // Cursors currently don't support:
+        boolean compactByIterators = !ALLOW_CURSOR_COMPACTION.getBoolean()
+                                     || strategy != null && !strategy.supportsCursorCompaction()  // strategy does not support it
+                                     || controller.shouldProvideTombstoneSources()  // garbagecollect
+                                     || realm.getIndexManager().hasIndexes()  // indexes
+                                     || realm.metadata().enforceStrictLiveness();   // strict liveness
+
+        logger.debug("Compacting in {} by {}: {} {} {} {} {}",
+                     realm.toString(),
+                     compactByIterators ? "iterators" : "cursors",
+                     ALLOW_CURSOR_COMPACTION.getBoolean() ? "" : "cursors disabled",
+                     strategy == null ? "no table compaction strategy"
+                                      : !strategy.supportsCursorCompaction() ? "no cursor support"
+                                                                             : "",
+                     controller.shouldProvideTombstoneSources() ? "tombstone sources" : "",
+                     realm.getIndexManager().hasIndexes() ? "has indexes" : "",
+                     realm.metadata().enforceStrictLiveness() ? "strict liveness" : "");
+
+        if (compactByIterators)
+            return new CompactionOperationIterator(controller, actuallyCompact, fullyExpiredSSTables.size());
+        else
+            return new CompactionOperationCursor(controller, actuallyCompact, fullyExpiredSSTables.size());
+    }
 
-            TimeUUID taskId = transaction.opId();
-            // select SSTables to compact based on available disk space.
-            if (!buildCompactionCandidatesForAvailableDiskSpace(fullyExpiredSSTables, taskId))
+    public static class OperationTotals
+    {
+        public final long inputDiskSize;
+        public final long inputUncompressedSize;
+
+        OperationTotals(long inputDiskSize, long inputUncompressedSize)
+        {
+            this.inputDiskSize = inputDiskSize;
+            this.inputUncompressedSize = inputUncompressedSize;
+        }
+    }
+
+    public static OperationTotals getOperationTotals(Collection<SSTableReader> sstables, Range<Token> tokenRange)
+    {
+        long inputDiskSize = 0;
+        long inputUncompressedSize = 0;
+        if (tokenRange == null)
+        {
+            for (SSTableReader rdr : sstables)
             {
-                // The set of sstables has changed (one or more were excluded due to limited available disk space).
-                // We need to recompute the overlaps between sstables.
-                controller.refreshOverlaps();
+                inputUncompressedSize += rdr.uncompressedLength();
+                inputDiskSize += rdr.onDiskLength();
             }
+        }
+        else
+        {
+            var rangeList = ImmutableList.of(tokenRange);
+            for (SSTableReader rdr : sstables)
+            {
+                final List<SSTableReader.PartitionPositionBounds> positionsForRanges = rdr.getPositionsForRanges(rangeList);
+                for (SSTableReader.PartitionPositionBounds pp : positionsForRanges)
+                    inputUncompressedSize += pp.upperPosition - pp.lowerPosition;
+                inputDiskSize += rdr.onDiskSizeForPartitionPositions(positionsForRanges);
+            }
+        }
+        return new OperationTotals(inputDiskSize, inputUncompressedSize);
+    }
 
-            // sanity check: all sstables must belong to the same cfs
-            assert !Iterables.any(transaction.originals(), new Predicate<SSTableReader>()
+    @Override
+    public long getSpaceOverhead()
+    {
+        // This value should be quick to return and never change.
+        // We can calculate the total number of bytes in the inputSSTables, but that's something that can change if
+        // we remove sstable because expired sstables or fitting under the available disk space.
+        // So we throw instead and let UnifiedCompactionStrategy override this method.
+        throw new UnsupportedOperationException("Unimplemented in base class.");
+    }
+
+    /**
+     *  The compaction operation is a special case of an {@link AbstractTableOperation} and takes care of executing the
+     *  actual compaction and releasing any resources when the compaction is finished.
+     *  <p/>
+     *  This class also extends {@link AbstractTableOperation} for reporting compaction-specific progress information.
+     */
+    public abstract class CompactionOperation implements AutoCloseable, CompactionProgress
+    {
+        final CompactionController controller;
+        final TimeUUID taskId;
+        final String taskIdString;
+        final RateLimiter limiter;
+        private final long startTimeMillis;
+        final Set<SSTableReader> actuallyCompact;
+        private final int fullyExpiredSSTablesCount;
+        private final long inputDiskSize;
+        private final long inputUncompressedSize;
+
+        // resources that are updated and may be read by another thread
+        volatile Collection<SSTableReader> newSStables;
+        volatile long totalKeysWritten;
+        volatile long estimatedKeys;
+
+        // resources that are updated but only read by this thread
+        boolean completed;
+        long lastCheckObsoletion;
+
+        // resources that need closing
+        Refs<SSTableReader> sstableRefs;
+        TableOperation op;
+        Closeable obsCloseable;
+        CompactionAwareWriter writer;
+
+        /**
+         * Create a new compaction operation.
+         * <p/>
+         *
+         * @param controller the compaction controller is needed by the scanners and compaction iterator to manage options
+         * @param actuallyCompact the set of sstables to compact (excludes any fully expired ones)
+         * @param fullyExpiredSSTablesCount the number of fully expired sstables (used in metrics)
+         */
+        private CompactionOperation(CompactionController controller, Set<SSTableReader> actuallyCompact, int fullyExpiredSSTablesCount)
+        {
+            this.controller = controller;
+            this.actuallyCompact = actuallyCompact;
+            this.taskId = transaction.opId();
+            this.taskIdString = transaction.opIdString();
+
+            this.limiter = CompactionManager.instance.getRateLimiter();
+            this.startTimeMillis = Clock.Global.currentTimeMillis();
+            this.newSStables = Collections.emptyList();
+            this.fullyExpiredSSTablesCount = fullyExpiredSSTablesCount;
+            this.totalKeysWritten = 0;
+            this.estimatedKeys = 0;
+            this.completed = false;
+            this.inputDiskSize = totals.inputDiskSize;
+            this.inputUncompressedSize = totals.inputUncompressedSize;
+
+            Directories dirs = getDirectories();
+
+            try
             {
-                @Override
-                public boolean apply(SSTableReader sstable)
+                // resources that need closing, must be created last in case of exceptions and released if there is an exception in the c.tor
+                this.sstableRefs = Refs.ref(actuallyCompact);
+                this.op = initializeSource(tokenRange());
+                this.writer = getCompactionAwareWriter(realm, dirs, actuallyCompact);
+                CompactionProgress progress = this;
+                var sharedProgress = sharedProgress();
+                if (sharedProgress != null)
                 {
-                    return !sstable.descriptor.cfname.equals(cfs.name);
+                    sharedProgress.addSubtask(this);
+                    progress = sharedProgress;
                 }
-            });
 
-            // new sstables from flush can be added during a compaction, but only the compaction can remove them,
-            // so in our single-threaded compaction world this is a valid way of determining if we're compacting
-            // all the sstables (that existed when we started)
-            StringBuilder ssTableLoggerMsg = new StringBuilder("[");
-            for (SSTableReader sstr : transaction.originals())
+                this.obsCloseable = opObserver.onOperationStart(op);
+                for (var obs : getCompObservers())
+                    obs.onInProgress(progress);
+            }
+            catch (Throwable t)
             {
-                ssTableLoggerMsg.append(String.format("%s:level=%d, ", sstr.getFilename(), sstr.getSSTableLevel()));
+                close(t);
+                throw new AssertionError(t); // unreachable (close will throw when t is not null). Added for static analysis.
             }
-            ssTableLoggerMsg.append("]");
+        }
+
+        abstract TableOperation initializeSource(Range<Token> tokenRange) throws Throwable;
+
+        private void execute()
+        {
+            try
+            {
+                // new sstables from flush can be added during a compaction, but only the compaction can remove them,
+                // so in our single-threaded compaction world this is a valid way of determining if we're compacting
+                // all the sstables (that existed when we started)
+                if (logger.isDebugEnabled())
+                {
+                    debugLogCompactingMessage(taskIdString);
+                }
 
-            logger.info("Compacting ({}) {}", taskId, ssTableLoggerMsg);
+                if (!controller.realm.isCompactionActive())
+                    throw new CompactionInterruptedException(op.getProgress(), op.trigger());
 
-            RateLimiter limiter = CompactionManager.instance.getRateLimiter();
-            long start = nanoTime();
-            long startTime = currentTimeMillis();
-            long totalKeysWritten = 0;
-            long estimatedKeys = 0;
-            long inputSizeBytes;
-            long timeSpentWritingKeys;
+                estimatedKeys = writer.estimatedKeys();
 
-            Set<SSTableReader> actuallyCompact = Sets.difference(transaction.originals(), fullyExpiredSSTables);
-            Collection<SSTableReader> newSStables;
+                execute0();
 
-            long[] mergedRowCounts;
-            long totalSourceCQLRows;
+                // point of no return
+                newSStables = writer.finish();
 
-            long nowInSec = FBUtilities.nowInSeconds();
-            try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
-                 AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact);
-                 CompactionIterator ci = new CompactionIterator(compactionType, scanners.scanners, controller, nowInSec, taskId))
+                completed = true;
+            }
+            catch (Throwable t)
             {
-                long lastCheckObsoletion = start;
-                inputSizeBytes = scanners.getTotalCompressedSize();
-                double compressionRatio = scanners.getCompressionRatio();
-                if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
-                    compressionRatio = 1.0;
+                Throwables.maybeFail(onError(t));
+            }
+        }
 
-                long lastBytesScanned = 0;
+        private Throwable onError(Throwable e)
+        {
+            if (e instanceof AssertionError)
+            {
+                // Add additional information to help operators.
+                AssertionError error = new AssertionError(
+                String.format("Illegal input has been generated, most probably due to corruption in the input sstables\n" +
+                              "\t%s\n" +
+                              "Try scrubbing the sstables by running\n" +
+                              "\tnodetool scrub %s %s\n",
+                              transaction.originals(),
+                              realm.getKeyspaceName(),
+                              realm.getTableName()));
+                error.addSuppressed(e);
+                return error;
+            }
 
-                activeCompactions.beginCompaction(ci);
-                try (CompactionAwareWriter writer = getCompactionAwareWriter(cfs, getDirectories(), transaction, actuallyCompact))
-                {
-                    // Note that we need to re-check this flag after calling beginCompaction above to avoid a window
-                    // where the compaction does not exist in activeCompactions but the CSM gets paused.
-                    // We already have the sstables marked compacting here so CompactionManager#waitForCessation will
-                    // block until the below exception is thrown and the transaction is cancelled.
-                    if (!controller.cfs.getCompactionStrategyManager().isActive())
-                        throw new CompactionInterruptedException(ci.getCompactionInfo());
-                    estimatedKeys = writer.estimatedKeys();
-                    while (ci.hasNext())
-                    {
-                        if (writer.append(ci.next()))
-                            totalKeysWritten++;
+            return e;
+        }
 
-                        ci.setTargetDirectory(writer.getSStableDirectory().path());
-                        long bytesScanned = scanners.getTotalBytesScanned();
+        void maybeStopOrUpdateState()
+        {
+            op.throwIfStopRequested();
 
-                        // Rate limit the scanners, and account for compression
-                        CompactionManager.compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
+            long now = Clock.Global.nanoTime();
+            if (now - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L))
+            {
+                controller.maybeRefreshOverlaps();
+                lastCheckObsoletion = now;
+            }
+        }
 
-                        lastBytesScanned = bytesScanned;
+        abstract void execute0();
 
-                        if (nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L))
-                        {
-                            controller.maybeRefreshOverlaps();
-                            lastCheckObsoletion = nanoTime();
-                        }
-                    }
-                    timeSpentWritingKeys = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start);
+        //
+        // Closeable
+        //
 
-                    // point of no return
-                    newSStables = writer.finish();
-                }
-                finally
-                {
-                    activeCompactions.finishCompaction(ci);
-                    mergedRowCounts = ci.getMergedRowCounts();
-                    totalSourceCQLRows = ci.getTotalSourceCQLRows();
-                }
-            }
+        @Override
+        public void close()
+        {
+            close(null);
+        }
+
+        public void close(Throwable errorsSoFar)
+        {
+            Throwable err = Throwables.close(errorsSoFar, obsCloseable, writer, sstableRefs);
+            final long elapsedTimeMillis = Clock.Global.currentTimeMillis() - startTimeMillis;
 
             if (transaction.isOffline())
+            {
+                if (completed)
+                {
+                    // update basic metrics
+                    realm.metrics().incBytesCompacted(adjustedInputDiskSize(),
+                                                      outputDiskSize(),
+                                                      elapsedTimeMillis);
+                }
+                Throwables.maybeFail(err);
                 return;
+            }
 
-            // log a bunch of statistics about the result and save to system table compaction_history
-            long durationInNano = nanoTime() - start;
-            long dTime = TimeUnit.NANOSECONDS.toMillis(durationInNano);
-            long startsize = inputSizeBytes;
-            long endsize = SSTableReader.getTotalBytes(newSStables);
-            double ratio = (double) endsize / (double) startsize;
-
-            StringBuilder newSSTableNames = new StringBuilder();
-            for (SSTableReader reader : newSStables)
-                newSSTableNames.append(reader.descriptor.baseFile()).append(",");
-            long totalSourceRows = 0;
-            for (int i = 0; i < mergedRowCounts.length; i++)
-                totalSourceRows += mergedRowCounts[i] * (i + 1);
-
-            String mergeSummary = updateCompactionHistory(taskId, cfs.getKeyspaceName(), cfs.getTableName(), mergedRowCounts, startsize, endsize,
-                                                          ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType.type));
-
-            logger.info(String.format("Compacted (%s) %d sstables to [%s] to level=%d.  %s to %s (~%d%% of original) in %,dms.  Read Throughput = %s, Write Throughput = %s, Row Throughput = ~%,d/s.  %,d total partitions merged to %,d.  Partition merge counts were {%s}. Time spent writing keys = %,dms",
-                                       taskId,
-                                       transaction.originals().size(),
-                                       newSSTableNames.toString(),
-                                       getLevel(),
-                                       FBUtilities.prettyPrintMemory(startsize),
-                                       FBUtilities.prettyPrintMemory(endsize),
-                                       (int) (ratio * 100),
-                                       dTime,
-                                       FBUtilities.prettyPrintMemoryPerSecond(startsize, durationInNano),
-                                       FBUtilities.prettyPrintMemoryPerSecond(endsize, durationInNano),
-                                       (int) totalSourceCQLRows / (TimeUnit.NANOSECONDS.toSeconds(durationInNano) + 1),
-                                       totalSourceRows,
-                                       totalKeysWritten,
-                                       mergeSummary,
-                                       timeSpentWritingKeys));
-            if (logger.isTraceEnabled())
+            if (completed)
             {
-                logger.trace("CF Total Bytes Compacted: {}", FBUtilities.prettyPrintMemory(CompactionTask.addToTotalBytesCompacted(endsize)));
-                logger.trace("Actual #keys: {}, Estimated #keys:{}, Err%: {}", totalKeysWritten, estimatedKeys, ((double)(totalKeysWritten - estimatedKeys)/totalKeysWritten));
+                boolean shouldSignalCompletion = true;
+                var sharedProgress = sharedProgress();
+                if (sharedProgress != null)
+                    shouldSignalCompletion = sharedProgress.completeSubtask(this);
+
+                if (shouldSignalCompletion)
+                {
+                    if (COMPACTION_HISTORY_ENABLED.getBoolean())
+                    {
+                        updateCompactionHistory(taskId, realm.getKeyspaceName(), realm.getTableName(), this, ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType.type));
+                    }
+                    CompactionManager.instance.incrementRemovedExpiredSSTables(fullyExpiredSSTablesCount);
+                    if (!transaction.originals().isEmpty() && actuallyCompact.isEmpty())
+                        // this CompactionOperation only deleted fully expired SSTables without compacting anything
+                        CompactionManager.instance.incrementDeleteOnlyCompactions();
+                }
+
+                if (logger.isDebugEnabled())
+                    debugLogCompactionSummaryInfo(taskIdString, elapsedTimeMillis, totalKeysWritten, newSStables, this);
+                if (logger.isTraceEnabled())
+                    traceLogCompactionSummaryInfo(totalKeysWritten, estimatedKeys, this);
+                if (strategy != null)
+                    strategy.getCompactionLogger().compaction(startTimeMillis,
+                                                              transaction.originals(),
+                                                              tokenRange(),
+                                                              Clock.Global.currentTimeMillis(),
+                                                              newSStables);
+
+                // update the metrics
+                realm.metrics().incBytesCompacted(adjustedInputDiskSize(),
+                                                  outputDiskSize(),
+                                                  elapsedTimeMillis);
             }
-            cfs.getCompactionStrategyManager().compactionLogger.compaction(startTime, transaction.originals(), currentTimeMillis(), newSStables);
 
-            // update the metrics
-            cfs.metric.compactionBytesWritten.inc(endsize);
+            Throwables.maybeFail(err);
+        }
+
+
+        //
+        // TableOperation.Progress methods
+        //
+
+        @Override
+        public Optional<String> keyspace()
+        {
+            return Optional.of(metadata().keyspace);
+        }
+
+        @Override
+        public Optional<String> table()
+        {
+            return Optional.of(metadata().name);
+        }
+
+        @Override
+        public TableMetadata metadata()
+        {
+            return realm.metadata();
+        }
+
+        @Override
+        public OperationType operationType()
+        {
+            return compactionType;
+        }
+
+        @Override
+        public TimeUUID operationId()
+        {
+            return taskId;
+        }
+
+        @Override
+        public TableOperation.Unit unit()
+        {
+            return TableOperation.Unit.BYTES;
+        }
+
+        @Override
+        public Set<SSTableReader> sstables()
+        {
+            return transaction.originals();
+        }
+
+        @Override
+        public String toString()
+        {
+            return progressToString();
+        }
+
+        //
+        // CompactionProgress
+        //
+
+        @Override
+        @Nullable
+        public CompactionStrategy strategy()
+        {
+            return CompactionTask.this.strategy;
+        }
+
+        @Override
+        public Collection<SSTableReader> inSSTables()
+        {
+            // TODO should we use transaction.originals() and include the expired sstables?
+            // This would be more correct but all the metrics we get from CompactionIterator will not be compatible
+            return actuallyCompact;
+        }
+        @Override
+        public Collection<SSTableReader> outSSTables()
+        {
+            return newSStables;
+        }
+
+        @Override
+        public long inputDiskSize()
+        {
+            return inputDiskSize;
+        }
+
+        /**
+         * @return the initial number of bytes for input sstables. For compressed or encrypted sstables,
+         *         this is the number of bytes after decompression, so this is the uncompressed length of sstable files.
+         */
+        public long total()
+        {
+            return inputUncompressedSize;
+        }
+
+        @Override
+        public long inputUncompressedSize()
+        {
+            return inputUncompressedSize;
+        }
+
+        @Override
+        public long outputDiskSize()
+        {
+            return CompactionSSTable.getTotalDataBytes(newSStables);
+        }
+
+        @Override
+        public long uncompressedBytesWritten()
+        {
+            return writer.bytesWritten();
+        }
+
+        @Override
+        public long startTimeMillis()
+        {
+            return startTimeMillis;
         }
     }
 
-    @Override
-    public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs,
-                                                          Directories directories,
-                                                          LifecycleTransaction transaction,
-                                                          Set<SSTableReader> nonExpiredSSTables)
+    /**
+     *  The compaction operation is a special case of an {@link AbstractTableOperation} and takes care of executing the
+     *  actual compaction and releasing any resources when the compaction is finished.
+     *  <p/>
+     *  This class also extends {@link AbstractTableOperation} for reporting compaction-specific progress information.
+     */
+    public final class CompactionOperationIterator extends CompactionOperation
     {
-        return new DefaultCompactionWriter(cfs, directories, transaction, nonExpiredSSTables, keepOriginals, getLevel());
+        // resources that need closing
+        private ScannerList scanners;
+        private CompactionIterator compactionIterator;
+
+        /**
+         * Create a new compaction operation.
+         * <p/>
+         * @param controller the compaction controller is needed by the scanners and compaction iterator to manage options
+         */
+        CompactionOperationIterator(CompactionController controller, Set<SSTableReader> actuallyCompact, int fullyExpiredSSTablesCount)
+        {
+            super(controller, actuallyCompact, fullyExpiredSSTablesCount);
+        }
+
+        @Override
+        TableOperation initializeSource(Range<Token> tokenRange)
+        {
+            var rangeList = tokenRange != null ? ImmutableList.of(tokenRange) : null;
+            this.scanners = strategy != null ? strategy.getScanners(actuallyCompact, rangeList)
+                                             : ScannerList.of(actuallyCompact, rangeList);
+            // We use `this` rather than `sharedProgress()` because the `TableOperation` tracks individual compactions.
+            this.compactionIterator = new CompactionIterator(compactionType, scanners.scanners, controller, FBUtilities.nowInSeconds(), taskId, null, this);
+            return compactionIterator.getOperation();
+        }
+
+        void execute0()
+        {
+            double compressionRatio = compactionIterator.getCompressionRatio();
+            if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
+                compressionRatio = 1.0;
+
+            long lastBytesScanned = 0;
+
+            while (compactionIterator.hasNext())
+            {
+                UnfilteredRowIterator partition = compactionIterator.next();
+                if (writer.append(partition) != null)
+                    totalKeysWritten++;
+
+                long bytesScanned = compactionIterator.bytesRead();
+
+                // Rate limit the scanners, and account for compression
+                if (CompactionManager.instance.compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio))
+                    lastBytesScanned = bytesScanned;
+
+                maybeStopOrUpdateState();
+            }
+        }
+
+        @Override
+        public void close(Throwable errorsSoFar)
+        {
+            super.close(Throwables.close(errorsSoFar, compactionIterator, scanners));
+        }
+
+        /**
+         * @return the number of bytes read by the compaction iterator. For compressed or encrypted sstables,
+         *         this is the number of bytes processed by the iterator after decompression, so this is the current
+         *         position in the uncompressed sstable files.
+         */
+        @Override
+        public long completed()
+        {
+            return compactionIterator.bytesRead();
+        }
+
+        @Override
+        public long adjustedInputDiskSize()
+        {
+            return compactionIterator.getTotalCompressedSize();
+        }
+
+        @Override
+        public long uncompressedBytesRead()
+        {
+            return compactionIterator.bytesRead();
+        }
+
+        @Override
+        public long uncompressedBytesRead(int level)
+        {
+            return compactionIterator.bytesRead(level);
+        }
+
+        @Override
+        public long partitionsRead()
+        {
+            return compactionIterator.totalSourcePartitions();
+        }
+
+        @Override
+        public long rowsRead()
+        {
+            return compactionIterator.totalSourceRows();
+        }
+
+        @Override
+        public long[] partitionsHistogram()
+        {
+            return compactionIterator.mergedPartitionsHistogram();
+        }
+
+        @Override
+        public long[] rowsHistogram()
+        {
+            return compactionIterator.mergedRowsHistogram();
+        }
+
     }
 
-    public static String updateCompactionHistory(TimeUUID taskId, String keyspaceName, String columnFamilyName, long[] mergedRowCounts, long startSize, long endSize, Map<String, String> compactionProperties)
+    /**
+     *  Cursor version of the above.
+     */
+    public final class CompactionOperationCursor extends CompactionOperation
     {
-        StringBuilder mergeSummary = new StringBuilder(mergedRowCounts.length * 10);
-        Map<Integer, Long> mergedRows = new HashMap<>();
-        for (int i = 0; i < mergedRowCounts.length; i++)
+        // resources that need closing
+        private CompactionCursor compactionCursor;
+
+        /**
+         * Create a new compaction operation.
+         * <p/>
+         * @param controller the compaction controller is needed by the scanners and compaction iterator to manage options
+         */
+        CompactionOperationCursor(CompactionController controller, Set<SSTableReader> actuallyCompact, int fullyExpiredSSTablesCount)
         {
-            long count = mergedRowCounts[i];
-            if (count == 0)
-                continue;
+            super(controller, actuallyCompact, fullyExpiredSSTablesCount);
+        }
 
-            int rows = i + 1;
-            mergeSummary.append(String.format("%d:%d, ", rows, count));
-            mergedRows.put(rows, count);
+        @Override
+        TableOperation initializeSource(Range<Token> tokenRange)
+        {
+            this.compactionCursor = new CompactionCursor(compactionType, actuallyCompact, tokenRange, controller, limiter, FBUtilities.nowInSeconds());
+            // We use `this` rather than `sharedProgress()` because the `TableOperation` tracks individual compactions.
+            return compactionCursor.createOperation(this);
         }
-        SystemKeyspace.updateCompactionHistory(taskId, keyspaceName, columnFamilyName, currentTimeMillis(), startSize, endSize, mergedRows, compactionProperties);
-        return mergeSummary.toString();
+
+        void execute0()
+        {
+            try
+            {
+                writeLoop:
+                while (true)
+                {
+                    op.throwIfStopRequested();
+
+                    switch (compactionCursor.copyOne(writer))
+                    {
+                        case EXHAUSTED:
+                            break writeLoop;
+                        case PARTITION:
+                            ++totalKeysWritten;
+                            maybeStopOrUpdateState();
+                            break;
+                    }
+                }
+            }
+            catch (IOException e)
+            {
+                throw new FSWriteError(e, writer.getCurrentFileName());
+            }
+        }
+
+        @Override
+        public void close(Throwable errorsSoFar)
+        {
+            super.close(Throwables.close(errorsSoFar, compactionCursor));
+        }
+
+        /**
+         * @return the number of bytes read by the compaction iterator. For compressed or encrypted sstables,
+         *         this is the number of bytes processed by the iterator after decompression, so this is the current
+         *         position in the uncompressed sstable files.
+         */
+        @Override
+        public long completed()
+        {
+            return compactionCursor.bytesRead();
+        }
+
+        @Override
+        public long adjustedInputDiskSize()
+        {
+            return inputDiskSize();
+        }
+
+        @Override
+        public long uncompressedBytesRead()
+        {
+            return compactionCursor.bytesRead();
+        }
+
+        @Override
+        public long uncompressedBytesRead(int level)
+        {
+            // Cursors don't implement LCS per-level progress tracking.
+            return 0L;
+        }
+
+        @Override
+        public long partitionsRead()
+        {
+            return compactionCursor.totalSourcePartitions();
+        }
+
+        @Override
+        public long rowsRead()
+        {
+            return compactionCursor.totalSourceRows();
+        }
+
+        @Override
+        public long[] partitionsHistogram()
+        {
+            return compactionCursor.mergedPartitionsHistogram();
+        }
+
+        @Override
+        public long[] rowsHistogram()
+        {
+            return compactionCursor.mergedRowsHistogram();
+        }
+    }
+
+    public CompactionAwareWriter getCompactionAwareWriter(CompactionRealm realm,
+                                                          Directories directories,
+                                                          Set<SSTableReader> nonExpiredSSTables)
+    {
+        return new DefaultCompactionWriter(realm, directories, transaction, nonExpiredSSTables, keepOriginals, getLevel());
     }
 
     protected Directories getDirectories()
     {
-        return cfs.getDirectories();
+        return realm.getDirectories();
     }
 
     public static long getMinRepairedAt(Set<SSTableReader> actuallyCompact)
@@ -336,7 +959,12 @@ public static TimeUUID getPendingRepair(Set<SSTableReader> sstables)
             ids.add(sstable.getSSTableMetadata().pendingRepair);
 
         if (ids.size() != 1)
-            throw new RuntimeException(String.format("Attempting to compact pending repair sstables with sstables from other repair, or sstables not pending repair: %s", ids));
+        {
+            if (!SKIP_REPAIR_STATE_CHECKING)
+                throw new RuntimeException(String.format("Attempting to compact pending repair sstables with sstables from other repair, or sstables not pending repair: %s", ids));
+            // otherwise we should continue but mark the result as unrepaired
+            return ActiveRepairService.NO_PENDING_REPAIR;
+        }
 
         return ids.iterator().next();
     }
@@ -358,23 +986,21 @@ public static boolean getIsTransient(Set<SSTableReader> sstables)
         return isTransient;
     }
 
-
-    /*
+    /**
      * Checks if we have enough disk space to execute the compaction.  Drops the largest sstable out of the Task until
      * there's enough space (in theory) to handle the compaction.
      *
      * @return true if there is enough disk space to execute the complete compaction, false if some sstables are excluded.
+     *         If SSTables are excluded, they are removed from the transaction as well as the nonExpiredSSTables set.
      */
-    protected boolean buildCompactionCandidatesForAvailableDiskSpace(final Set<SSTableReader> fullyExpiredSSTables, TimeUUID taskId)
+    protected boolean buildCompactionCandidatesForAvailableDiskSpace(Set<SSTableReader> nonExpiredSSTables, TimeUUID taskId, boolean containsExpired)
     {
-        if(!cfs.isCompactionDiskSpaceCheckEnabled() && compactionType == OperationType.COMPACTION)
+        if(!realm.isCompactionDiskSpaceCheckEnabled() && compactionType == OperationType.COMPACTION)
         {
             logger.info("Compaction space check is disabled - trying to compact all sstables");
             return true;
         }
 
-        final Set<SSTableReader> nonExpiredSSTables = Sets.difference(transaction.originals(), fullyExpiredSSTables);
-        CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
         int sstablesRemoved = 0;
 
         while(!nonExpiredSSTables.isEmpty())
@@ -383,9 +1009,10 @@ protected boolean buildCompactionCandidatesForAvailableDiskSpace(final Set<SSTab
             long writeSize;
             try
             {
-                writeSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
+                // TODO verify this, I'm not sure if this is ok
+                writeSize = realm.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
                 Map<File, Long> expectedNewWriteSize = new HashMap<>();
-                List<File> newCompactionDatadirs = cfs.getDirectoriesForFiles(nonExpiredSSTables);
+                List<File> newCompactionDatadirs = realm.getDirectoriesForFiles(nonExpiredSSTables);
                 long writeSizePerOutputDatadir = writeSize / Math.max(newCompactionDatadirs.size(), 1);
                 for (File directory : newCompactionDatadirs)
                     expectedNewWriteSize.put(directory, writeSizePerOutputDatadir);
@@ -393,7 +1020,7 @@ protected boolean buildCompactionCandidatesForAvailableDiskSpace(final Set<SSTab
                 Map<File, Long> expectedWriteSize = CompactionManager.instance.active.estimatedRemainingWriteBytes();
 
                 // todo: abort streams if they block compactions
-                if (cfs.getDirectories().hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSize, expectedWriteSize))
+                if (realm.getDirectories().hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSize, expectedWriteSize))
                     break;
             }
             catch (Exception e)
@@ -407,18 +1034,20 @@ protected boolean buildCompactionCandidatesForAvailableDiskSpace(final Set<SSTab
                 // we end up here if we can't take any more sstables out of the compaction.
                 // usually means we've run out of disk space
 
-                // but we can still compact expired SSTables
-                if(partialCompactionsAcceptable() && fullyExpiredSSTables.size() > 0 )
+                // but we can still remove expired SSTables
+                if (partialCompactionsAcceptable() && containsExpired)
                 {
-                    // sanity check to make sure we compact only fully expired SSTables.
-                    assert transaction.originals().equals(fullyExpiredSSTables);
+                    for (SSTableReader rdr : nonExpiredSSTables)
+                        transaction.cancel(rdr);
+                    nonExpiredSSTables.clear();
+                    assert transaction.originals().size() > 0;
                     break;
                 }
 
                 String msg = String.format("Not enough space for compaction (%s) of %s.%s, estimated sstables = %d, expected write size = %d",
                                            taskId,
-                                           cfs.getKeyspaceName(),
-                                           cfs.name,
+                                           realm.getKeyspaceName(),
+                                           realm.getTableName(),
                                            Math.max(1, writeSize / strategy.getMaxSSTableBytes()),
                                            writeSize);
                 logger.warn(msg);
@@ -447,7 +1076,7 @@ protected int getLevel()
 
     protected CompactionController getCompactionController(Set<SSTableReader> toCompact)
     {
-        return new CompactionController(cfs, toCompact, gcBefore);
+        return new CompactionController(realm, toCompact, gcBefore);
     }
 
     protected boolean partialCompactionsAcceptable()
@@ -465,4 +1094,111 @@ public static long getMaxDataAge(Collection<SSTableReader> sstables)
         }
         return max;
     }
+
+    private void debugLogCompactionSummaryInfo(String taskId,
+                                               long durationInMillis,
+                                               long totalKeysWritten,
+                                               Collection<SSTableReader> newSStables,
+                                               CompactionProgress progress)
+    {
+        // log a bunch of statistics about the result and save to system table compaction_history
+        long totalMergedPartitions = 0;
+        long[] mergedPartitionCounts = progress.partitionsHistogram();
+        StringBuilder mergeSummary = new StringBuilder(mergedPartitionCounts.length * 10);
+        mergeSummary.append('{');
+        for (int i = 0; i < mergedPartitionCounts.length; i++)
+        {
+            long mergedPartitionCount = mergedPartitionCounts[i];
+            if (mergedPartitionCount != 0)
+            {
+                totalMergedPartitions += mergedPartitionCount * (i + 1);
+                mergeSummary.append(i).append(':').append(mergedPartitionCount).append(", ");
+            }
+        }
+        mergeSummary.append('}');
+
+        StringBuilder newSSTableNames = new StringBuilder(newSStables.size() * 100);
+        for (SSTableReader reader : newSStables)
+            newSSTableNames.append(reader.descriptor.baseFileUri()).append(',');
+        long durationInNano = TimeUnit.MILLISECONDS.toNanos(durationInMillis);
+        logger.debug("Compacted ({}{}) {} sstables to [{}] to level={}. {} to {} (~{}% of original) in {}ms. " +
+                     "Read Throughput = {}, Write Throughput = {}, Row Throughput = ~{}/s, Partition Throughput = ~{}/s." +
+                     " {} total partitions merged to {}. Partition merge counts were {}.",
+                     taskId,
+                     tokenRange() != null ? " range " + tokenRange() : "",
+                     transaction.originals().size(),
+                     newSSTableNames,
+                     getLevel(),
+                     prettyPrintMemory(progress.adjustedInputDiskSize()),
+                     prettyPrintMemory(progress.outputDiskSize()),
+                     (int) (progress.sizeRatio() * 100),
+                     durationInMillis,
+                     prettyPrintMemoryPerSecond(progress.adjustedInputDiskSize(), durationInNano),
+                     prettyPrintMemoryPerSecond(progress.outputDiskSize(), durationInNano),
+                     (long) (progress.rowsRead() * 1.0e-3 / durationInMillis),
+                     (long) (progress.partitionsRead() * 1.0e-3 / durationInMillis),
+                     totalMergedPartitions,
+                     totalKeysWritten,
+                     mergeSummary);
+    }
+
+    private void debugLogCompactingMessage(String taskId)
+    {
+        Set<SSTableReader> originals = transaction.originals();
+        StringBuilder ssTableLoggerMsg = new StringBuilder(originals.size() * 100);
+        ssTableLoggerMsg.append("Compacting (").append(taskId);
+        if (tokenRange() != null)
+            ssTableLoggerMsg.append(" range ").append(tokenRange());
+        ssTableLoggerMsg.append(") [");
+        for (SSTableReader sstr : originals)
+        {
+            ssTableLoggerMsg.append(sstr.getFilename());
+            if (sstr.getSSTableLevel() != 0)
+                ssTableLoggerMsg.append(":level=")
+                                .append(sstr.getSSTableLevel());
+            ssTableLoggerMsg.append(", ");
+        }
+        ssTableLoggerMsg.append(']');
+
+        logger.debug(ssTableLoggerMsg.toString());
+    }
+
+
+    private static void updateCompactionHistory(TimeUUID id,
+                                                String keyspaceName,
+                                                String columnFamilyName,
+                                                CompactionProgress progress,
+                                                Map<String, String> compactionProperties)
+    {
+        long[] mergedPartitionsHistogram = progress.partitionsHistogram();
+        Map<Integer, Long> mergedPartitions = new HashMap<>(mergedPartitionsHistogram.length);
+        for (int i = 0; i < mergedPartitionsHistogram.length; i++)
+        {
+            long count = mergedPartitionsHistogram[i];
+            if (count == 0)
+                continue;
+
+            int rows = i + 1;
+            mergedPartitions.put(rows, count);
+        }
+        SystemKeyspace.updateCompactionHistory(id,
+                                               keyspaceName,
+                                               columnFamilyName,
+                                               Clock.Global.currentTimeMillis(),
+                                               progress.adjustedInputDiskSize(),
+                                               progress.outputDiskSize(),
+                                               mergedPartitions,
+                                               compactionProperties);
+    }
+
+    private void traceLogCompactionSummaryInfo(long totalKeysWritten,
+                                               long estimatedKeys,
+                                               CompactionProgress progress)
+    {
+        logger.trace("CF Total Bytes Compacted: {}", prettyPrintMemory(addToTotalBytesCompacted(progress.outputDiskSize())));
+        logger.trace("Actual #keys: {}, Estimated #keys:{}, Err%: {}",
+                     totalKeysWritten,
+                     estimatedKeys,
+                     ((double) (totalKeysWritten - estimatedKeys) / totalKeysWritten));
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/CompositeCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompositeCompactionTask.java
new file mode 100644
index 000000000000..b33268795703
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/CompositeCompactionTask.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.utils.Throwables;
+
+/// A composition of several compaction tasks into one. This object executes the given tasks sequentially and
+/// is used to limit the parallelism of some compaction tasks that split into a large number of parallelizable ones
+/// but should not be allowed to take all compaction executor threads.
+public class CompositeCompactionTask extends AbstractCompactionTask
+{
+    @VisibleForTesting
+    final ArrayList<AbstractCompactionTask> tasks;
+
+    public CompositeCompactionTask(AbstractCompactionTask first)
+    {
+        super(first.realm, first.realm.tryModify(Collections.emptyList(), OperationType.COMPACTION, first.transaction.opId()));
+        tasks = new ArrayList<>();
+        addTask(first);
+    }
+
+    /// Add a task to the composition.
+    public CompositeCompactionTask addTask(AbstractCompactionTask task)
+    {
+        tasks.add(task);
+        return this;
+    }
+
+    @Override
+    protected void runMayThrow() throws Exception
+    {
+        // Run all tasks in sequence, regardless if any of them fail.
+        Throwable accumulate = null;
+        for (AbstractCompactionTask task : tasks)
+        {
+            accumulate = Throwables.perform(accumulate, () -> task.execute(opObserver));
+            // The previous operation may have completed due to a requested stop. We do not stop other tasks in our
+            // list if that is the case, because if the tasks are related, the [SharedTableOperation] will have already
+            // requested a stop from the other components as well. If we stopped the other tasks here, we may
+            // overrespond to a user's request to stop an individual operation.
+            // On the other hand, [CompactionManager] sometimes requests a stop of all ongoing operations e.g. to
+            // initiate a table drop. Such requests, however, do not affect tasks in the executor queue; as this class
+            // is acting similarly to an executor queue, we do not apply such stop requests to the remaining tasks
+            // either.
+        }
+        Throwables.maybeFail(accumulate);
+    }
+
+    @Override
+    public Throwable rejected(Throwable t)
+    {
+        for (AbstractCompactionTask task : tasks)
+            t = task.rejected(t);
+        return super.rejected(t);
+    }
+
+    @Override
+    public AbstractCompactionTask setUserDefined(boolean isUserDefined)
+    {
+        for (AbstractCompactionTask task : tasks)
+            task.setUserDefined(isUserDefined);
+        return super.setUserDefined(isUserDefined);
+    }
+
+    @Override
+    public AbstractCompactionTask setCompactionType(OperationType compactionType)
+    {
+        for (AbstractCompactionTask task : tasks)
+            task.setCompactionType(compactionType);
+        return super.setCompactionType(compactionType);
+    }
+
+    @Override
+    public void addObserver(CompactionObserver compObserver)
+    {
+        for (AbstractCompactionTask task : tasks)
+            task.addObserver(compObserver);
+        super.addObserver(compObserver);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Composite " + tasks;
+    }
+
+    @Override
+    public long getSpaceOverhead()
+    {
+        throw new UnsupportedOperationException("Cannot calculate space overhead for composite tasks");
+    }
+
+    /// Limit the parallelism of a list of compaction tasks by combining them into a smaller number of composite tasks.
+    /// This method assumes that the caller has preference for the tasks to be executed in order close to the order of
+    /// the input list. See [UnifiedCompactionStrategy#getMaximalTasks] for an example of how to use this method.
+    public static List<AbstractCompactionTask> applyParallelismLimit(List<AbstractCompactionTask> tasks, int parallelismLimit)
+    {
+        if (tasks.size() <= parallelismLimit || parallelismLimit <= 0)
+            return tasks;
+
+        List<AbstractCompactionTask> result = new ArrayList<>(parallelismLimit);
+        int taskIndex = 0;
+        for (AbstractCompactionTask task : tasks)
+        {
+            if (result.size() < parallelismLimit)
+                result.add(task);
+            else
+            {
+                result.set(taskIndex, combineTasks(result.get(taskIndex), task));
+                if (++taskIndex == parallelismLimit)
+                    taskIndex = 0;
+            }
+        }
+        return result;
+    }
+
+    /// Make a composite tasks that combines two tasks. If the former is already a composite task, the latter is added
+    /// to it. Otherwise, a new composite task is created.
+    public static CompositeCompactionTask combineTasks(AbstractCompactionTask task1, AbstractCompactionTask task2)
+    {
+        CompositeCompactionTask composite;
+        if (task1 instanceof CompositeCompactionTask)
+            composite = (CompositeCompactionTask) task1;
+        else
+            composite = new CompositeCompactionTask(task1);
+        return composite.addTask(task2);
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/DelegatingShardManager.java b/src/java/org/apache/cassandra/db/compaction/DelegatingShardManager.java
new file mode 100644
index 000000000000..a9702e056cc7
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/DelegatingShardManager.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.function.IntFunction;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+/**
+ * A shard manager that delegates to a token generator for determining shard boundaries.
+ */
+public class DelegatingShardManager implements ShardManager
+{
+    private final IntFunction<Token[]> tokenGenerator;
+    private CompactionRealm realm;
+
+    public DelegatingShardManager(IntFunction<Token[]> tokenGenerator, CompactionRealm realm)
+    {
+        this.tokenGenerator = tokenGenerator;
+        this.realm = realm;
+    }
+
+    @Override
+    public double rangeSpanned(Range<Token> tableRange)
+    {
+        return tableRange.left.size(tableRange.right);
+    }
+
+    @Override
+    public double localSpaceCoverage()
+    {
+        // This manager is global, so it owns the whole range.
+        return 1;
+    }
+
+    @Override
+    public double shardSetCoverage()
+    {
+        // For now there are no disks defined, so this is the same as localSpaceCoverage
+        return 1;
+    }
+
+    @Override
+    public double minimumPerPartitionSpan()
+    {
+        return localSpaceCoverage() / Math.max(1, realm.estimatedPartitionCountInSSTables());
+    }
+
+    @Override
+    public ShardTracker boundaries(int shardCount)
+    {
+        var tokens = tokenGenerator.apply(shardCount);
+        return new SimpleShardTracker(tokens);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/ExpirationTask.java b/src/java/org/apache/cassandra/db/compaction/ExpirationTask.java
new file mode 100644
index 000000000000..08899ae9508d
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/ExpirationTask.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+
+/// SSTable expiration task.
+///
+/// This is used when compaction identifies fully-expired SSTables that can be safely deleted. Executing the task
+/// simply commits the associated transaction which has the effect of deleting the source SSTables.
+public class ExpirationTask extends AbstractCompactionTask
+{
+    protected ExpirationTask(CompactionRealm realm, ILifecycleTransaction transaction)
+    {
+        super(realm, transaction);
+    }
+
+    @Override
+    protected void runMayThrow() throws Exception
+    {
+        transaction.obsoleteOriginals();
+        transaction.prepareToCommit();
+        transaction.commit();
+        CompactionManager.instance.incrementDeleteOnlyCompactions();
+    }
+
+    @Override
+    public long getSpaceOverhead()
+    {
+        return 0;   // This is just deleting files, no overhead.
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/LegacyAbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LegacyAbstractCompactionStrategy.java
new file mode 100644
index 000000000000..fee7aa9825b0
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/LegacyAbstractCompactionStrategy.java
@@ -0,0 +1,354 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Clock;
+
+/**
+ * Pluggable compaction strategy determines how SSTables get merged.
+ *
+ * There are two main goals:
+ *  - perform background compaction constantly as needed; this typically makes a tradeoff between
+ *    i/o done by compaction, and merging done at read time.
+ *  - perform a full (maximum possible) compaction if requested by the user
+ */
+abstract class LegacyAbstractCompactionStrategy extends AbstractCompactionStrategy
+{
+    protected LegacyAbstractCompactionStrategy(CompactionStrategyFactory factory, Map<String, String> options)
+    {
+        super(factory, new BackgroundCompactions(factory.getRealm()), options);
+        assert factory != null;
+    }
+
+    /**
+     * Helper base class for strategies that provide CompactionAggregates, implementing the typical
+     * getNextBackgroundTasks logic based on a getNextBackgroundAggregate method.
+     */
+    protected static abstract class WithAggregates extends LegacyAbstractCompactionStrategy
+    {
+        protected WithAggregates(CompactionStrategyFactory factory, Map<String, String> options)
+        {
+            super(factory, options);
+        }
+
+        @Override
+        @SuppressWarnings("resource")
+        public Collection<AbstractCompactionTask> getNextBackgroundTasks(long gcBefore)
+        {
+            CompactionPick previous = null;
+            while (true)
+            {
+                CompactionAggregate compaction = getNextBackgroundAggregate(gcBefore);
+                if (compaction == null || compaction.isEmpty())
+                    return ImmutableList.of();
+
+                // Already tried acquiring references without success. It means there is a race with
+                // the tracker but candidate SSTables were not yet replaced in the compaction strategy manager
+                if (compaction.getSelected().equals(previous))
+                {
+                    logger.warn("Could not acquire references for compacting SSTables {} which is not a problem per se," +
+                                "unless it happens frequently, in which case it must be reported. Will retry later.",
+                                compaction.getSelected());
+                    return ImmutableList.of();
+                }
+
+                CompactionPick selected = compaction.getSelected();
+                Preconditions.checkNotNull(selected);
+
+                LifecycleTransaction transaction = realm.tryModify(selected.sstables(),
+                                                                   OperationType.COMPACTION,
+                                                                   selected.id());
+                if (transaction != null)
+                {
+                    backgroundCompactions.setSubmitted(this, transaction.opId(), compaction);
+                    return ImmutableList.of(createCompactionTask(gcBefore, transaction, compaction));
+                }
+
+                // Getting references to the sstables failed. This may be because we tried to compact sstables that are
+                // no longer present (due to races in getting the notification), or because we still haven't
+                // received any replace notifications. Remove any non-live sstables we track and try again.
+                removeDeadSSTables();
+
+                previous = selected;
+            }
+        }
+
+        /**
+         * Select the next compaction to perform. This method is typically synchronized.
+         */
+        protected abstract CompactionAggregate getNextBackgroundAggregate(long gcBefore);
+
+        protected AbstractCompactionTask createCompactionTask(final long gcBefore, LifecycleTransaction txn, CompactionAggregate compaction)
+        {
+            return new CompactionTask(realm, txn, gcBefore, false, this);
+        }
+
+        /**
+         * Get the estimated remaining compactions. Strategies that implement {@link WithAggregates} can delegate this
+         * to {@link BackgroundCompactions} because they set the pending aggregates as background compactions but legacy
+         * strategies that do not support aggregates must implement this method.
+         * <p/>
+         * @return the number of background tasks estimated to still be needed for this strategy
+         */
+        @Override
+        public int getEstimatedRemainingTasks()
+        {
+            return backgroundCompactions.getEstimatedRemainingTasks();
+        }
+    }
+
+    /**
+     * Helper base class for (older, deprecated) strategies that provide a list of tables to compact, implementing the
+     * typical getNextBackgroundTask logic based on a getNextBackgroundSSTables method.
+     */
+    protected static abstract class WithSSTableList extends LegacyAbstractCompactionStrategy
+    {
+        protected WithSSTableList(CompactionStrategyFactory factory, Map<String, String> options)
+        {
+            super(factory, options);
+        }
+
+        @Override
+        @SuppressWarnings("resource")
+        public Collection<AbstractCompactionTask> getNextBackgroundTasks(long gcBefore)
+        {
+            List<CompactionSSTable> previousCandidate = null;
+            while (true)
+            {
+                List<CompactionSSTable> latestBucket = getNextBackgroundSSTables(gcBefore);
+
+                if (latestBucket.isEmpty())
+                    return ImmutableList.of();
+
+                // Already tried acquiring references without success. It means there is a race with
+                // the tracker but candidate SSTables were not yet replaced in the compaction strategy manager
+                if (latestBucket.equals(previousCandidate))
+                {
+                    logger.warn("Could not acquire references for compacting SSTables {} which is not a problem per se," +
+                                "unless it happens frequently, in which case it must be reported. Will retry later.",
+                                latestBucket);
+                    return ImmutableList.of();
+                }
+
+                LifecycleTransaction modifier = realm.tryModify(latestBucket, OperationType.COMPACTION);
+                if (modifier != null)
+                    return ImmutableList.of(createCompactionTask(gcBefore, modifier, false, false));
+
+                // Getting references to the sstables failed. This may be because we tried to compact sstables that are
+                // no longer present (due to races in getting the notification), or because we still haven't
+                // received any replace notifications. Remove any non-live sstables we track and try again.
+                removeDeadSSTables();
+
+                previousCandidate = latestBucket;
+            }
+        }
+
+        /**
+         * Select the next tables to compact. This method is typically synchronized.
+         * @return
+         */
+        protected abstract List<CompactionSSTable> getNextBackgroundSSTables(final long gcBefore);
+    }
+
+    /**
+     * Replaces sstables in the compaction strategy
+     *
+     * Note that implementations must be able to handle duplicate notifications here (that removed are already gone and
+     * added have already been added)
+     */
+    public abstract void replaceSSTables(Collection<CompactionSSTable> removed, Collection<CompactionSSTable> added);
+
+    /**
+     * Adds sstable, note that implementations must handle duplicate notifications here (added already being in the compaction strategy)
+     */
+    abstract void addSSTable(CompactionSSTable added);
+
+    /**
+     * Adds sstables, note that implementations must handle duplicate notifications here (added already being in the compaction strategy)
+     */
+    public synchronized void addSSTables(Iterable<CompactionSSTable> added)
+    {
+        for (CompactionSSTable sstable : added)
+            addSSTable(sstable);
+    }
+
+    /**
+     * Removes sstable from the strategy, implementations must be able to handle the sstable having already been removed.
+     */
+    abstract void removeSSTable(CompactionSSTable sstable);
+
+    /**
+     * Removes sstables from the strategy, implementations must be able to handle the sstables having already been removed.
+     */
+    public void removeSSTables(Iterable<CompactionSSTable> removed)
+    {
+        for (CompactionSSTable sstable : removed)
+            removeSSTable(sstable);
+    }
+
+    /**
+     * Remove any tracked sstable that is no longer in the live set. Note that because we get notifications after the
+     * tracker is modified, anything we know of must be already in the live set. If it is not, it has been removed
+     * from there, and we either haven't received the removal notification yet, or we did and we messed it up (i.e.
+     * we got it before the addition). The former is transient, but the latter can cause persistent problems, including
+     * fully stopping compaction. In any case, we should remove any such sstables.
+     * There is a special-case implementation of this in LeveledManifest.
+     */
+    abstract void removeDeadSSTables();
+
+    void removeDeadSSTables(Iterable<CompactionSSTable> sstables)
+    {
+        synchronized (sstables)
+        {
+            int removed = 0;
+            Set<? extends CompactionSSTable> liveSet = realm.getLiveSSTables();
+            for (Iterator<CompactionSSTable> it = sstables.iterator(); it.hasNext(); )
+            {
+                CompactionSSTable sstable = it.next();
+                if (!liveSet.contains(sstable))
+                {
+                    it.remove();
+                    ++removed;
+                }
+            }
+
+            if (removed > 0)
+                logger.debug("Removed {} dead sstables from the compactions tracked list.", removed);
+        }
+    }
+
+    @Override
+    public synchronized CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism)
+    {
+        removeDeadSSTables();
+        return super.getMaximalTasks(gcBefore, splitOutput, permittedParallelism);
+    }
+
+    @Override
+    public synchronized CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism, OperationType operationType)
+    {
+        removeDeadSSTables();
+        return super.getMaximalTasks(gcBefore, splitOutput, permittedParallelism, operationType);
+    }
+
+
+    /**
+     * Select a table for tombstone-removing compaction from the given set. Returns null if no table is suitable.
+     */
+    @Nullable
+    CompactionAggregate makeTombstoneCompaction(long gcBefore,
+                                                Iterable<CompactionSSTable> candidates,
+                                                Function<Collection<CompactionSSTable>, CompactionSSTable> selector)
+    {
+        List<CompactionSSTable> sstablesWithTombstones = new ArrayList<>();
+        for (CompactionSSTable sstable : candidates)
+        {
+            if (worthDroppingTombstones(sstable, gcBefore))
+                sstablesWithTombstones.add(sstable);
+        }
+        if (sstablesWithTombstones.isEmpty())
+            return null;
+
+        final CompactionSSTable sstable = selector.apply(sstablesWithTombstones);
+        return CompactionAggregate.createForTombstones(sstable);
+    }
+
+    /**
+     * Check if given sstable is worth dropping tombstones at gcBefore.
+     * Check is skipped if tombstone_compaction_interval time does not elapse since sstable creation and returns false.
+     *
+     * @param sstable SSTable to check
+     * @param gcBefore time to drop tombstones
+     * @return true if given sstable's tombstones are expected to be removed
+     */
+    protected boolean worthDroppingTombstones(CompactionSSTable sstable, long gcBefore)
+    {
+        if (options.isDisableTombstoneCompactions()
+            || CompactionController.NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE
+            || realm.getNeverPurgeTombstones())
+            return false;
+        // since we use estimations to calculate, there is a chance that compaction will not drop tombstones actually.
+        // if that happens we will end up in infinite compaction loop, so first we check enough if enough time has
+        // elapsed since SSTable created.
+        if (Clock.Global.currentTimeMillis() < sstable.getCreationTimeFor(SSTableFormat.Components.DATA)+ options.getTombstoneCompactionInterval() * 1000)
+            return false;
+
+        double droppableRatio = sstable.getEstimatedDroppableTombstoneRatio(gcBefore);
+        if (droppableRatio <= options.getTombstoneThreshold())
+            return false;
+
+        //sstable range overlap check is disabled. See CASSANDRA-6563.
+        if (options.isUncheckedTombstoneCompaction())
+            return true;
+
+        Set<? extends CompactionSSTable> overlaps = realm.getOverlappingLiveSSTables(Collections.singleton(sstable));
+        if (overlaps.isEmpty())
+        {
+            // there is no overlap, tombstones are safely droppable
+            return true;
+        }
+        else if (CompactionController.getFullyExpiredSSTables(realm, Collections.singleton(sstable), c -> overlaps, gcBefore).size() > 0)
+        {
+            return true;
+        }
+        else
+        {
+            if (!(sstable instanceof SSTableReader))
+                return false; // Correctly estimating percentage requires data that CompactionSSTable does not provide.
+
+            SSTableReader reader = (SSTableReader) sstable;
+            // what percentage of columns do we expect to compact outside of overlap?
+            if (reader.isEstimationInformative())
+            {
+                // we have too few samples to estimate correct percentage
+                return false;
+            }
+            // first, calculate estimated keys that do not overlap
+            long keys = reader.estimatedKeys();
+            Set<Range<Token>> ranges = new HashSet<Range<Token>>(overlaps.size());
+            for (CompactionSSTable overlap : overlaps)
+                ranges.add(new Range<>(overlap.getFirst().getToken(), overlap.getLast().getToken()));
+            long remainingKeys = keys - reader.estimatedKeysForRanges(ranges);
+            // next, calculate what percentage of columns we have within those keys
+            long columns = reader.getEstimatedCellPerPartitionCount().mean() * remainingKeys;
+            double remainingColumnsRatio = ((double) columns) / (reader.getEstimatedCellPerPartitionCount().count() *
+                                                                 reader.getEstimatedCellPerPartitionCount().mean());
+
+            // return if we still expect to have droppable tombstones in rest of columns
+            return remainingColumnsRatio * droppableRatio > options.getTombstoneThreshold();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStatistics.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStatistics.java
new file mode 100644
index 000000000000..15fd1b725470
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStatistics.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+/**
+ * The statistics for leveled compaction.
+ * <p/>
+ * Implements serializable to allow structured info to be returned via JMX.
+ */
+public class LeveledCompactionStatistics extends CompactionAggregateStatistics
+{
+    private static final Collection<String> HEADER = ImmutableList.copyOf(Iterables.concat(ImmutableList.of("Level", "Score"),
+                                                                                           CompactionAggregateStatistics.HEADER,
+                                                                                           ImmutableList.of("Read: Tot/Prev/Next",
+                                                                                                            "Written: Tot/New",
+                                                                                                            "WA (tot_written/read_prev)")));
+
+    private static final long serialVersionUID = 3695927592357744816L;
+
+    /** The current level */
+    private final int level;
+
+    /** The score of this level */
+    private final double score;
+
+    /**
+     * How many more compactions this level is expected to perform. This is required because for LCS we cannot
+     * easily identify candidate sstables to put into the pending picks.
+     */
+    private final int pendingCompactions;
+
+    /**
+     * Bytes read from the current level (N) during compaction between levels N and N+1. Note that {@link #readBytes}
+     * includes bytes read from both the current level (N) and the target level (N+1).
+     */
+    private final long readLevel;
+
+    /**
+     * Additional RocksDB metrics we may want to consider:
+     * Moved(GB): Bytes moved to level N+1 during compaction. In this case there is no IO other than updating the manifest to indicate that a file which used to be in level X is now in level Y
+     * Rd(MB/s): The rate at which data is read during compaction between levels N and N+1. This is (Read(GB) * 1024) / duration where duration is the time for which compactions are in progress from level N to N+1.
+     * Wr(MB/s): The rate at which data is written during compaction. See Rd(MB/s).
+     * Rn(cnt): Total files read from level N during compaction between levels N and N+1
+     * Rnp1(cnt): Total files read from level N+1 during compaction between levels N and N+1
+     * Wnp1(cnt): Total files written to level N+1 during compaction between levels N and N+1
+     * Wnew(cnt): (Wnp1(cnt) - Rnp1(cnt)) -- Increase in file count as result of compaction between levels N and N+1
+     * Comp(sec): Total time spent doing compactions between levels N and N+1
+     * Comp(cnt): Total number of compactions between levels N and N+1
+     * Avg(sec): Average time per compaction between levels N and N+1
+     * Stall(sec): Total time writes were stalled because level N+1 was uncompacted (compaction score was high)
+     * Stall(cnt): Total number of writes stalled because level N+1 was uncompacted
+     * Avg(ms): Average time in milliseconds a write was stalled because level N+1 was uncompacted
+     * KeyIn: number of records compared during compaction
+     * KeyDrop: number of records dropped (not written out) during compaction
+     */
+
+    public LeveledCompactionStatistics(CompactionAggregateStatistics base,
+                                       int level,
+                                       double score,
+                                       int pendingCompactions,
+                                       long readLevel)
+    {
+        super(base);
+        this.level = level;
+        this.score = score;
+        this.pendingCompactions = pendingCompactions;
+        this.readLevel = readLevel;
+    }
+
+    /** The number of compactions that are either pending or in progress */
+    @Override
+    @JsonProperty
+    public int numCompactions()
+    {
+        return numCompactions + pendingCompactions;
+    }
+
+    /** The current level */
+    @JsonProperty
+    public int level()
+    {
+        return level;
+    }
+
+    /** The score of a level is the level size in bytes of all its files dived by the ideal
+     * level size if applicable, or zero for tiered strategies */
+    @JsonProperty
+    public double score()
+    {
+        return score;
+    }
+
+    /**
+     * Bytes read from the current level (N) during compaction between levels N and N+1. Note that
+     * {@link #read()} includes bytes read from both the current level (N) and the target level (N+1).
+     */    @JsonProperty
+    public long readLevel()
+    {
+        return readLevel;
+    }
+
+    /** Uncompressed bytes read from the next level (N+1) during compaction between levels N and N+1 */
+    @JsonProperty
+    public long readNext()
+    {
+        return readBytes - readLevel;
+    }
+
+    /** Uncompressed  bytes written to level N+1, calculated as total bytes written - bytes read from N+1 */
+    @JsonProperty
+    public long writtenNew()
+    {
+        return writtenBytes - readNext();
+    }
+
+    /** W-Amp: total bytes written divided by the bytes read from level N. */
+    @JsonProperty
+    public double writeAmpl()
+    {
+        return readLevel() > 0 ? (double) writtenBytes / readLevel() : Double.NaN;
+    }
+
+    @Override
+    protected Collection<String> header()
+    {
+        return HEADER;
+    }
+
+    @Override
+    protected Collection<String> data()
+    {
+        List<String> data = new ArrayList<>(HEADER.size());
+        data.add(Integer.toString(level()));
+        data.add(String.format("%.3f", score()));
+
+        data.addAll(super.data());
+
+        data.add(toString(read()) + '/' + toString(readLevel()) + '/' + toString(readNext()));
+        data.add(toString(written()) + '/' + toString(writtenNew()));
+        data.add(String.format("%.3f", writeAmpl()));
+
+        return data;
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index 58166630bc26..62daea8a2346 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -17,39 +17,49 @@
  */
 package org.apache.cassandra.db.compaction;
 
-import java.util.*;
-
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.*;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
 import com.google.common.primitives.Doubles;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.schema.CompactionParams;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.TableMetadata;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.TOLERATE_SSTABLE_SIZE;
 
-public class LeveledCompactionStrategy extends AbstractCompactionStrategy
+public class LeveledCompactionStrategy extends LegacyAbstractCompactionStrategy.WithAggregates
 {
     private static final Logger logger = LoggerFactory.getLogger(LeveledCompactionStrategy.class);
-    private static final String SSTABLE_SIZE_OPTION = "sstable_size_in_mb";
+    static final String SSTABLE_SIZE_OPTION = "sstable_size_in_mb";
     private static final boolean tolerateSstableSize = TOLERATE_SSTABLE_SIZE.getBoolean();
-    private static final String LEVEL_FANOUT_SIZE_OPTION = "fanout_size";
+    static final String LEVEL_FANOUT_SIZE_OPTION = "fanout_size";
     private static final String SINGLE_SSTABLE_UPLEVEL_OPTION = "single_sstable_uplevel";
     public static final int DEFAULT_LEVEL_FANOUT_SIZE = 10;
 
@@ -59,9 +69,9 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
     private final int levelFanoutSize;
     private final boolean singleSSTableUplevel;
 
-    public LeveledCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
+    public LeveledCompactionStrategy(CompactionStrategyFactory factory, Map<String, String> options)
     {
-        super(cfs, options);
+        super(factory, options);
         int configuredMaxSSTableSize = 160;
         int configuredLevelFanoutSize = DEFAULT_LEVEL_FANOUT_SIZE;
         boolean configuredSingleSSTableUplevel = false;
@@ -75,10 +85,10 @@ public LeveledCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> opti
                 {
                     if (configuredMaxSSTableSize >= 1000)
                         logger.warn("Max sstable size of {}MB is configured for {}.{}; having a unit of compaction this large is probably a bad idea",
-                                configuredMaxSSTableSize, cfs.name, cfs.getTableName());
+                                    configuredMaxSSTableSize, realm.getKeyspaceName(), realm.getTableName());
                     if (configuredMaxSSTableSize < 50)
                         logger.warn("Max sstable size of {}MB is configured for {}.{}.  Testing done for CASSANDRA-5727 indicates that performance improves up to 160MB",
-                                configuredMaxSSTableSize, cfs.name, cfs.getTableName());
+                                    configuredMaxSSTableSize, realm.getKeyspaceName(), realm.getTableName());
                 }
             }
 
@@ -96,11 +106,11 @@ public LeveledCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> opti
         levelFanoutSize = configuredLevelFanoutSize;
         singleSSTableUplevel = configuredSingleSSTableUplevel;
 
-        manifest = new LeveledManifest(cfs, this.maxSSTableSizeInMiB, this.levelFanoutSize, localOptions);
+        manifest = new LeveledManifest(realm, this.maxSSTableSizeInMiB, this.levelFanoutSize, localOptions);
         logger.trace("Created {}", manifest);
     }
 
-    public int getLevelSize(int i)
+    int getLevelSize(int i)
     {
         return manifest.getLevelSize(i);
     }
@@ -115,6 +125,12 @@ public long[] getAllLevelSizeBytes()
         return manifest.getAllLevelSizeBytes();
     }
 
+    @Override
+    public int[] getSSTableCountPerLevel()
+    {
+        return manifest.getSSTableCountPerLevel();
+    }
+
     @Override
     public void startup()
     {
@@ -122,95 +138,62 @@ public void startup()
         super.startup();
     }
 
-    /**
-     * the only difference between background and maximal in LCS is that maximal is still allowed
-     * (by explicit user request) even when compaction is disabled.
-     */
-    public AbstractCompactionTask getNextBackgroundTask(long gcBefore)
+    @Override
+    protected CompactionAggregate getNextBackgroundAggregate(long gcBefore)
     {
-        Collection<SSTableReader> previousCandidate = null;
-        while (true)
-        {
-            OperationType op;
-            LeveledManifest.CompactionCandidate candidate = manifest.getCompactionCandidates();
-            if (candidate == null)
-            {
-                // if there is no sstable to compact in standard way, try compacting based on droppable tombstone ratio
-                SSTableReader sstable = findDroppableSSTable(gcBefore);
-                if (sstable == null)
-                {
-                    logger.trace("No compaction necessary for {}", this);
-                    return null;
-                }
-                candidate = new LeveledManifest.CompactionCandidate(Collections.singleton(sstable),
-                                                                    sstable.getSSTableLevel(),
-                                                                    getMaxSSTableBytes());
-                op = OperationType.TOMBSTONE_COMPACTION;
-            }
-            else
-            {
-                op = OperationType.COMPACTION;
-            }
+        CompactionAggregate.Leveled candidate = manifest.getCompactionCandidate();
+        backgroundCompactions.setPending(this, manifest.getEstimatedTasks(candidate));
 
-            // Already tried acquiring references without success. It means there is a race with
-            // the tracker but candidate SSTables were not yet replaced in the compaction strategy manager
-            if (candidate.sstables.equals(previousCandidate))
-            {
-                logger.warn("Could not acquire references for compacting SSTables {} which is not a problem per se," +
-                            "unless it happens frequently, in which case it must be reported. Will retry later.",
-                            candidate.sstables);
-                return null;
-            }
-
-            LifecycleTransaction txn = cfs.getTracker().tryModify(candidate.sstables, OperationType.COMPACTION);
-            if (txn != null)
-            {
-                AbstractCompactionTask newTask;
-                if (!singleSSTableUplevel || op == OperationType.TOMBSTONE_COMPACTION || txn.originals().size() > 1)
-                    newTask = new LeveledCompactionTask(cfs, txn, candidate.level, gcBefore, candidate.maxSSTableBytes, false);
-                else
-                    newTask = new SingleSSTableLCSTask(cfs, txn, candidate.level);
+        if (candidate != null)
+            return candidate;
 
-                newTask.setCompactionType(op);
-                return newTask;
-            }
-            previousCandidate = candidate.sstables;
-        }
+        return findDroppableSSTable(gcBefore);
     }
 
-    public synchronized Collection<AbstractCompactionTask> getMaximalTask(long gcBefore, boolean splitOutput)
+    @Override
+    protected AbstractCompactionTask createCompactionTask(final long gcBefore, LifecycleTransaction txn, CompactionAggregate compaction)
     {
-        Iterable<SSTableReader> sstables = manifest.getSSTables();
+        long maxxSSTableBytes;
+        int nextLevel;
+        OperationType op;
+
+        if (compaction instanceof CompactionAggregate.TombstoneAggregate)
+        {
+            op = OperationType.TOMBSTONE_COMPACTION;
+            nextLevel = Iterables.getOnlyElement(compaction.selected.sstables()).getSSTableLevel();
+            maxxSSTableBytes = getMaxSSTableBytes();    // TODO: verify this is expected as it can split L0 tables
+        }
+        else
+        {
+            CompactionAggregate.Leveled candidate = (CompactionAggregate.Leveled) compaction;
+            op = OperationType.COMPACTION;
+            nextLevel = candidate.nextLevel;
+            maxxSSTableBytes = candidate.maxSSTableBytes;
+        }
 
-        Iterable<SSTableReader> filteredSSTables = filterSuspectSSTables(sstables);
-        if (Iterables.isEmpty(sstables))
-            return null;
-        LifecycleTransaction txn = cfs.getTracker().tryModify(filteredSSTables, OperationType.COMPACTION);
-        if (txn == null)
-            return null;
-        return Arrays.<AbstractCompactionTask>asList(new LeveledCompactionTask(cfs, txn, 0, gcBefore, getMaxSSTableBytes(), true));
 
+        AbstractCompactionTask newTask;
+        if (!singleSSTableUplevel || op == OperationType.TOMBSTONE_COMPACTION || txn.originals().size() > 1)
+            newTask = new LeveledCompactionTask(this, txn, nextLevel, gcBefore, maxxSSTableBytes, false);
+        else
+            newTask = new SingleSSTableLCSTask(this, txn, nextLevel);
+
+        newTask.setCompactionType(op);
+        return newTask;
     }
 
+
     @Override
-    public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, long gcBefore)
+    protected AbstractCompactionTask createCompactionTask(final long gcBefore, LifecycleTransaction txn, boolean isMaximal, boolean splitOutput)
     {
-
-        if (sstables.isEmpty())
-            return null;
-
-        LifecycleTransaction transaction = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
-        if (transaction == null)
-        {
-            logger.trace("Unable to mark {} for compaction; probably a background compaction got to it first.  You can disable background compactions temporarily if this is a problem", sstables);
-            return null;
-        }
+        Collection<SSTableReader> sstables = txn.originals();
         int level = sstables.size() > 1 ? 0 : sstables.iterator().next().getSSTableLevel();
-        return new LeveledCompactionTask(cfs, transaction, level, gcBefore, level == 0 ? Long.MAX_VALUE : getMaxSSTableBytes(), false);
+        long maxSSTableBytes = (level == 0 && !isMaximal) ? Long.MAX_VALUE : getMaxSSTableBytes();
+        return new LeveledCompactionTask(this, txn, level, gcBefore, maxSSTableBytes, isMaximal);
     }
 
     @Override
-    public AbstractCompactionTask getCompactionTask(LifecycleTransaction txn, long gcBefore, long maxSSTableBytes)
+    public AbstractCompactionTask createCompactionTask(LifecycleTransaction txn, long gcBefore, long maxSSTableBytes)
     {
         assert txn.originals().size() > 0;
         int level = -1;
@@ -222,7 +205,7 @@ public AbstractCompactionTask getCompactionTask(LifecycleTransaction txn, long g
             if (level != sstable.getSSTableLevel())
                 level = 0;
         }
-        return new LeveledCompactionTask(cfs, txn, level, gcBefore, maxSSTableBytes, false);
+        return new LeveledCompactionTask(this, txn, level, gcBefore, maxSSTableBytes, false);
     }
 
     /**
@@ -233,28 +216,28 @@ public AbstractCompactionTask getCompactionTask(LifecycleTransaction txn, long g
      * @return Groups of sstables from the same level
      */
     @Override
-    public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> ssTablesToGroup)
+    public Collection<Collection<CompactionSSTable>> groupSSTablesForAntiCompaction(Collection<? extends CompactionSSTable> ssTablesToGroup)
     {
         int groupSize = 2;
-        Map<Integer, Collection<SSTableReader>> sstablesByLevel = new HashMap<>();
-        for (SSTableReader sstable : ssTablesToGroup)
+        Map<Integer, Collection<CompactionSSTable>> sstablesByLevel = new HashMap<>();
+        for (CompactionSSTable sstable : ssTablesToGroup)
         {
             Integer level = sstable.getSSTableLevel();
-            Collection<SSTableReader> sstablesForLevel = sstablesByLevel.get(level);
+            Collection<CompactionSSTable> sstablesForLevel = sstablesByLevel.get(level);
             if (sstablesForLevel == null)
             {
-                sstablesForLevel = new ArrayList<SSTableReader>();
+                sstablesForLevel = new ArrayList<>();
                 sstablesByLevel.put(level, sstablesForLevel);
             }
             sstablesForLevel.add(sstable);
         }
 
-        Collection<Collection<SSTableReader>> groupedSSTables = new ArrayList<>();
+        Collection<Collection<CompactionSSTable>> groupedSSTables = new ArrayList<>();
 
-        for (Collection<SSTableReader> levelOfSSTables : sstablesByLevel.values())
+        for (Collection<CompactionSSTable> levelOfSSTables : sstablesByLevel.values())
         {
-            Collection<SSTableReader> currGroup = new ArrayList<>(groupSize);
-            for (SSTableReader sstable : levelOfSSTables)
+            Collection<CompactionSSTable> currGroup = new ArrayList<>(groupSize);
+            for (CompactionSSTable sstable : levelOfSSTables)
             {
                 currGroup.add(sstable);
                 if (currGroup.size() == groupSize)
@@ -271,19 +254,6 @@ public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Coll
 
     }
 
-    public int getEstimatedRemainingTasks()
-    {
-        int n = manifest.getEstimatedTasks();
-        cfs.getCompactionStrategyManager().compactionLogger.pending(this, n);
-        return n;
-    }
-
-    @Override
-    int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes)
-    {
-        return manifest.getEstimatedTasks(additionalBytes);
-    }
-
     public long getMaxSSTableBytes()
     {
         return maxSSTableSizeInMiB * 1024L * 1024L;
@@ -296,7 +266,7 @@ public int getLevelFanoutSize()
 
     public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
     {
-        Set<SSTableReader>[] sstablesPerLevel = manifest.getSStablesPerLevelSnapshot();
+        Set<CompactionSSTable>[] sstablesPerLevel = manifest.getSStablesPerLevelSnapshot();
 
         Multimap<Integer, SSTableReader> byLevel = ArrayListMultimap.create();
         for (SSTableReader sstable : sstables)
@@ -334,7 +304,7 @@ public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Ra
                     Collection<SSTableReader> intersecting = LeveledScanner.intersecting(byLevel.get(level), ranges);
                     if (!intersecting.isEmpty())
                     {
-                        ISSTableScanner scanner = new LeveledScanner(cfs.metadata(), intersecting, ranges);
+                        ISSTableScanner scanner = new LeveledScanner(realm.metadata(), intersecting, ranges, level);
                         scanners.add(scanner);
                     }
                 }
@@ -349,38 +319,43 @@ public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Ra
     }
 
     @Override
-    public void replaceSSTables(Collection<SSTableReader> removed, Collection<SSTableReader> added)
+    public void replaceSSTables(Collection<CompactionSSTable> removed, Collection<CompactionSSTable> added)
     {
         manifest.replace(removed, added);
     }
 
-    @Override
-    public void metadataChanged(StatsMetadata oldMetadata, SSTableReader sstable)
+    public void metadataChanged(StatsMetadata oldMetadata, CompactionSSTable sstable)
     {
         if (sstable.getSSTableLevel() != oldMetadata.sstableLevel)
             manifest.newLevel(sstable, oldMetadata.sstableLevel);
     }
 
     @Override
-    public void addSSTables(Iterable<SSTableReader> sstables)
+    public void addSSTables(Iterable<CompactionSSTable> sstables)
     {
         manifest.addSSTables(sstables);
     }
 
     @Override
-    public void addSSTable(SSTableReader added)
+    void removeDeadSSTables()
+    {
+        manifest.removeDeadSSTables();
+    }
+
+    @Override
+    public void addSSTable(CompactionSSTable added)
     {
         manifest.addSSTables(Collections.singleton(added));
     }
 
     @Override
-    public void removeSSTable(SSTableReader sstable)
+    public void removeSSTable(CompactionSSTable sstable)
     {
         manifest.remove(sstable);
     }
 
     @Override
-    protected Set<SSTableReader> getSSTables()
+    public Set<CompactionSSTable> getSSTables()
     {
         return manifest.getSSTables();
     }
@@ -392,6 +367,7 @@ private static class LeveledScanner extends AbstractIterator<UnfilteredRowIterat
         private final TableMetadata metadata;
         private final Collection<Range<Token>> ranges;
         private final List<SSTableReader> sstables;
+        private final int level;
         private final Iterator<SSTableReader> sstableIterator;
         private final long totalLength;
         private final long compressedLength;
@@ -400,13 +376,14 @@ private static class LeveledScanner extends AbstractIterator<UnfilteredRowIterat
         private long positionOffset;
         private long totalBytesScanned = 0;
 
-        public LeveledScanner(TableMetadata metadata, Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+        public LeveledScanner(TableMetadata metadata, Collection<SSTableReader> sstables, Collection<Range<Token>> ranges, int level)
         {
             this.metadata = metadata;
             this.ranges = ranges;
 
             // add only sstables that intersect our range, and estimate how much data that involves
             this.sstables = new ArrayList<>(sstables.size());
+            this.level = level;
             long length = 0;
             long cLength = 0;
             for (SSTableReader sstable : sstables)
@@ -424,7 +401,7 @@ public LeveledScanner(TableMetadata metadata, Collection<SSTableReader> sstables
 
             totalLength = length;
             compressedLength = cLength;
-            Collections.sort(this.sstables, SSTableReader.firstKeyComparator);
+            Collections.sort(this.sstables, CompactionSSTable.firstKeyComparator);
             sstableIterator = this.sstables.iterator();
             assert sstableIterator.hasNext(); // caller should check intersecting first
             SSTableReader currentSSTable = sstableIterator.next();
@@ -510,63 +487,43 @@ public Set<SSTableReader> getBackingSSTables()
         {
             return ImmutableSet.copyOf(sstables);
         }
+
+        public int level()
+        {
+            return level;
+        }
     }
 
     @Override
     public String toString()
     {
-        return String.format("LCS@%d(%s)", hashCode(), cfs.name);
+        return String.format("LCS@%d(%s)", hashCode(), realm.getTableName());
     }
 
-    private SSTableReader findDroppableSSTable(final long gcBefore)
+    private CompactionAggregate findDroppableSSTable(final long gcBefore)
     {
-        level:
+        Comparator<CompactionSSTable> comparator = (o1, o2) -> {
+            double r1 = o1.getEstimatedDroppableTombstoneRatio(gcBefore);
+            double r2 = o2.getEstimatedDroppableTombstoneRatio(gcBefore);
+            return -1 * Doubles.compare(r1, r2);
+        };
+        Function<Collection<CompactionSSTable>, CompactionSSTable> selector = list -> Collections.max(list, comparator);
+        Set<? extends CompactionSSTable> compacting = realm.getCompactingSSTables();
+
         for (int i = manifest.getLevelCount(); i >= 0; i--)
         {
-            if (manifest.getLevelSize(i) == 0)
-                continue;
-            // sort sstables by droppable ratio in descending order
-            List<SSTableReader> tombstoneSortedSSTables = manifest.getLevelSorted(i, (o1, o2) -> {
-                double r1 = o1.getEstimatedDroppableTombstoneRatio(gcBefore);
-                double r2 = o2.getEstimatedDroppableTombstoneRatio(gcBefore);
-                return -1 * Doubles.compare(r1, r2);
-            });
-
-            Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
-            for (SSTableReader sstable : tombstoneSortedSSTables)
-            {
-                if (sstable.getEstimatedDroppableTombstoneRatio(gcBefore) <= tombstoneThreshold)
-                    continue level;
-                else if (!compacting.contains(sstable) && !sstable.isMarkedSuspect() && worthDroppingTombstones(sstable, gcBefore))
-                    return sstable;
-            }
+            CompactionAggregate tombstoneAggregate = makeTombstoneCompaction(gcBefore,
+                                                                             nonSuspectAndNotIn(manifest.getLevel(i), compacting),
+                                                                             selector);
+            if (tombstoneAggregate != null)
+                return tombstoneAggregate;
         }
         return null;
     }
 
-    public CompactionLogger.Strategy strategyLogger()
-    {
-        return new CompactionLogger.Strategy()
-        {
-            public JsonNode sstable(SSTableReader sstable)
-            {
-                ObjectNode node = JsonNodeFactory.instance.objectNode();
-                node.put("level", sstable.getSSTableLevel());
-                node.put("min_token", sstable.getFirst().getToken().toString());
-                node.put("max_token", sstable.getLast().getToken().toString());
-                return node;
-            }
-
-            public JsonNode options()
-            {
-                return null;
-            }
-        };
-    }
-
     public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
     {
-        Map<String, String> uncheckedOptions = AbstractCompactionStrategy.validateOptions(options);
+        Map<String, String> uncheckedOptions = CompactionStrategyOptions.validateOptions(options);
 
         String size = options.containsKey(SSTABLE_SIZE_OPTION) ? options.get(SSTABLE_SIZE_OPTION) : "1";
         try
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
index 8f5a70a84d16..4f6fbbad0a6f 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
@@ -19,6 +19,7 @@
 
 import java.util.Set;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
@@ -26,7 +27,7 @@
 import org.apache.cassandra.db.compaction.writers.MajorLeveledCompactionWriter;
 import org.apache.cassandra.db.compaction.writers.MaxSSTableSizeWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
 
 public class LeveledCompactionTask extends CompactionTask
 {
@@ -34,29 +35,36 @@ public class LeveledCompactionTask extends CompactionTask
     private final long maxSSTableBytes;
     private final boolean majorCompaction;
 
-    public LeveledCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction txn, int level, long gcBefore, long maxSSTableBytes, boolean majorCompaction)
+    public LeveledCompactionTask(LeveledCompactionStrategy strategy, ILifecycleTransaction txn, int level, long gcBefore, long maxSSTableBytes, boolean majorCompaction)
     {
-        super(cfs, txn, gcBefore);
+        super(strategy.realm, txn, gcBefore, false, strategy);
+        this.level = level;
+        this.maxSSTableBytes = maxSSTableBytes;
+        this.majorCompaction = majorCompaction;
+    }
+
+    public LeveledCompactionTask(ColumnFamilyStore cfs, ILifecycleTransaction txn, int level, long gcBefore, long maxSSTableBytes, boolean majorCompaction, @Nullable CompactionStrategy strategy) {
+        super(cfs, txn, gcBefore, false, strategy);
         this.level = level;
         this.maxSSTableBytes = maxSSTableBytes;
         this.majorCompaction = majorCompaction;
     }
 
     @Override
-    public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs,
+    public CompactionAwareWriter getCompactionAwareWriter(CompactionRealm realm,
                                                           Directories directories,
-                                                          LifecycleTransaction txn,
                                                           Set<SSTableReader> nonExpiredSSTables)
     {
         if (majorCompaction)
-            return new MajorLeveledCompactionWriter(cfs, directories, txn, nonExpiredSSTables, maxSSTableBytes, false);
-        return new MaxSSTableSizeWriter(cfs, directories, txn, nonExpiredSSTables, maxSSTableBytes, getLevel(), false);
+            return new MajorLeveledCompactionWriter(realm, directories, transaction, nonExpiredSSTables, maxSSTableBytes, false);
+        return new MaxSSTableSizeWriter(realm, directories, transaction, nonExpiredSSTables, maxSSTableBytes, getLevel(), false);
     }
 
     @Override
     protected boolean partialCompactionsAcceptable()
     {
-        return level == 0;
+        // LCS allows removing L0 sstable from L0/L1 compaction task for limited disk space. It's handled in #reduceScopeForLimitedSpace
+        return level <= 1;
     }
 
     protected int getLevel()
@@ -67,7 +75,7 @@ protected int getLevel()
     @Override
     public boolean reduceScopeForLimitedSpace(Set<SSTableReader> nonExpiredSSTables, long expectedSize)
     {
-        if (transaction.originals().size() > 1 && level <= 1)
+        if (nonExpiredSSTables.size() > 1 && level <= 1)
         {
             // Try again w/o the largest one.
             logger.warn("insufficient space to do L0 -> L{} compaction. {}MiB required, {} for compaction {}",
@@ -77,7 +85,7 @@ public boolean reduceScopeForLimitedSpace(Set<SSTableReader> nonExpiredSSTables,
                                    .stream()
                                    .map(sstable -> String.format("%s (level=%s, size=%s)", sstable, sstable.getSSTableLevel(), sstable.onDiskLength()))
                                    .collect(Collectors.joining(",")),
-                        transaction.opId());
+                        transaction.opIdString());
             // Note that we have removed files that are still marked as compacting.
             // This suboptimal but ok since the caller will unmark all the sstables at the end.
             int l0SSTableCount = 0;
@@ -98,8 +106,9 @@ public boolean reduceScopeForLimitedSpace(Set<SSTableReader> nonExpiredSSTables,
                             largestL0SSTable,
                             largestL0SSTable.getSSTableLevel(),
                             largestL0SSTable.onDiskLength(),
-                            transaction.opId());
+                            transaction.opIdString());
                 transaction.cancel(largestL0SSTable);
+                nonExpiredSSTables.remove(largestL0SSTable);
                 return true;
             }
         }
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java b/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
index 513e02aad99e..9f221df1fdcb 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
@@ -33,12 +33,9 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.io.sstable.SSTableIdFactory;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_STRICT_LCS_CHECKS;
@@ -68,16 +65,16 @@ class LeveledGenerations
      * do allSSTables.get(instance_with_moved_starts) we will get the NORMAL sstable back, which we can then remove
      * from the TreeSet.
      */
-    private final Map<SSTableReader, SSTableReader> allSSTables = new HashMap<>();
-    private final Set<SSTableReader> l0 = new HashSet<>();
+    private final Map<CompactionSSTable, CompactionSSTable> allSSTables = new HashMap<>();
+    private final Set<CompactionSSTable> l0 = new HashSet<>();
     private static long lastOverlapCheck = nanoTime();
     // note that since l0 is broken out, levels[0] represents L1:
-    private final TreeSet<SSTableReader> [] levels = new TreeSet[MAX_LEVEL_COUNT - 1];
+    private final TreeSet<CompactionSSTable> [] levels = new TreeSet[MAX_LEVEL_COUNT - 1];
 
-    private static final Comparator<SSTableReader> nonL0Comparator = (o1, o2) -> {
-        int cmp = SSTableReader.firstKeyComparator.compare(o1, o2);
+    private static final Comparator<CompactionSSTable> nonL0Comparator = (o1, o2) -> {
+        int cmp = CompactionSSTable.firstKeyComparator.compare(o1, o2);
         if (cmp == 0)
-            cmp = SSTableIdFactory.COMPARATOR.compare(o1.descriptor.id, o2.descriptor.id);
+            cmp = CompactionSSTable.idComparator.compare(o1, o2);
         return cmp;
     };
 
@@ -87,7 +84,7 @@ class LeveledGenerations
             levels[i] = new TreeSet<>(nonL0Comparator);
     }
 
-    Set<SSTableReader> get(int level)
+    Set<CompactionSSTable> get(int level)
     {
         if (level > levelCount() - 1 || level < 0)
             throw new ArrayIndexOutOfBoundsException("Invalid generation " + level + " - maximum is " + (levelCount() - 1));
@@ -113,28 +110,13 @@ int levelCount()
      *
      * todo: group sstables per level, add all if level is currently empty, improve startup speed
      */
-    void addAll(Iterable<SSTableReader> readers)
+    void addAll(Iterable<? extends CompactionSSTable> readers)
     {
         logDistribution();
-        for (SSTableReader sstable : readers)
+        for (CompactionSSTable sstable : readers)
         {
             assert sstable.getSSTableLevel() < levelCount() : "Invalid level " + sstable.getSSTableLevel() + " out of " + (levelCount() - 1);
-            int existingLevel = getLevelIfExists(sstable);
-            if (existingLevel != -1)
-            {
-                if (sstable.getSSTableLevel() != existingLevel)
-                {
-                    logger.error("SSTable {} on the wrong level in the manifest - {} instead of {} as recorded in the sstable metadata, removing from level {}", sstable, existingLevel, sstable.getSSTableLevel(), existingLevel);
-                    if (strictLCSChecksTest)
-                        throw new AssertionError("SSTable not in matching level in manifest: "+sstable + ": "+existingLevel+" != " + sstable.getSSTableLevel());
-                }
-                else
-                {
-                    logger.info("Manifest already contains {} in level {} - replacing instance", sstable, existingLevel);
-                }
-                get(existingLevel).remove(sstable);
-                allSSTables.remove(sstable);
-            }
+            removeIfExists(sstable);
 
             allSSTables.put(sstable, sstable);
             if (sstable.getSSTableLevel() == 0)
@@ -143,7 +125,7 @@ void addAll(Iterable<SSTableReader> readers)
                 continue;
             }
 
-            TreeSet<SSTableReader> level = levels[sstable.getSSTableLevel() - 1];
+            TreeSet<CompactionSSTable> level = levels[sstable.getSSTableLevel() - 1];
             /*
             current level: |-----||----||----|        |---||---|
               new sstable:                      |--|
@@ -151,8 +133,8 @@ void addAll(Iterable<SSTableReader> readers)
                                                         ^ after
                 overlap if before.last >= newsstable.first or after.first <= newsstable.last
              */
-            SSTableReader after = level.ceiling(sstable);
-            SSTableReader before = level.floor(sstable);
+            CompactionSSTable after = level.ceiling(sstable);
+            CompactionSSTable before = level.floor(sstable);
 
             if (before != null && before.getLast().compareTo(sstable.getFirst()) >= 0 ||
                 after != null && after.getFirst().compareTo(sstable.getLast()) <= 0)
@@ -172,7 +154,7 @@ void addAll(Iterable<SSTableReader> readers)
      *
      * SSTable should not exist in the manifest
      */
-    private void sendToL0(SSTableReader sstable)
+    private void sendToL0(CompactionSSTable sstable)
     {
         try
         {
@@ -183,34 +165,53 @@ private void sendToL0(SSTableReader sstable)
             // Adding it to L0 and marking suspect is probably the best we can do here - it won't create overlap
             // and we won't pick it for later compactions.
             logger.error("Failed mutating sstable metadata for {} - adding it to L0 to avoid overlap. Marking suspect", sstable, e);
-            sstable.markSuspect();
         }
         l0.add(sstable);
     }
 
     /**
-     * Tries to find the sstable in the levels without using the sstable-recorded level
+     * Tries to find the sstable in the levels without using the sstable-recorded level, and removes it if it does find
+     * it.
      *
      * Used to make sure we don't try to re-add an existing sstable
      */
-    private int getLevelIfExists(SSTableReader sstable)
+    private void removeIfExists(CompactionSSTable sstable)
     {
-        for (int i = 0; i < levelCount(); i++)
+        for (int level = 0; level < levelCount(); level++)
         {
-            if (get(i).contains(sstable))
-                return i;
+            if (get(level).contains(sstable))
+            {
+                if (sstable.getSSTableLevel() != level)
+                {
+                    logger.error("SSTable {} on the wrong level in the manifest - {} instead of {} as recorded in the sstable metadata, removing from level {}",
+                                 sstable,
+                                 level,
+                                 sstable.getSSTableLevel(),
+                                 level);
+                    if (strictLCSChecksTest)
+                        throw new AssertionError("SSTable not in matching level in manifest: " + sstable + ": " + level + " != " +
+                                                 sstable.getSSTableLevel());
+                }
+                else
+                {
+                    logger.info("Manifest already contains {} in level {} - replacing instance",
+                                sstable,
+                                level);
+                }
+                get(level).remove(sstable);
+                allSSTables.remove(sstable);
+            }
         }
-        return -1;
     }
 
-    int remove(Collection<SSTableReader> readers)
+    int remove(Collection<? extends CompactionSSTable> readers)
     {
         int minLevel = Integer.MAX_VALUE;
-        for (SSTableReader sstable : readers)
+        for (CompactionSSTable sstable : readers)
         {
             int level = sstable.getSSTableLevel();
             minLevel = Math.min(minLevel, level);
-            SSTableReader versionInManifest = allSSTables.get(sstable);
+            CompactionSSTable versionInManifest = allSSTables.get(sstable);
             if (versionInManifest != null)
             {
                 get(level).remove(versionInManifest);
@@ -232,15 +233,15 @@ long[] getAllLevelSizeBytes()
     {
         long[] sums = new long[levelCount()];
         for (int i = 0; i < sums.length; i++)
-            sums[i] = get(i).stream().map(SSTableReader::onDiskLength).reduce(0L, Long::sum);
+            sums[i] = get(i).stream().map(CompactionSSTable::onDiskLength).reduce(0L, Long::sum);
         return sums;
     }
 
-    Set<SSTableReader> allSSTables()
+    Set<CompactionSSTable> allSSTables()
     {
-        ImmutableSet.Builder<SSTableReader> builder = ImmutableSet.builder();
+        ImmutableSet.Builder<CompactionSSTable> builder = ImmutableSet.builder();
         builder.addAll(l0);
-        for (Set<SSTableReader> sstables : levels)
+        for (Set<CompactionSSTable> sstables : levels)
             builder.addAll(sstables);
         return builder.build();
     }
@@ -249,21 +250,21 @@ Set<SSTableReader> allSSTables()
      * given a level with sstables with first tokens [0, 10, 20, 30] and a lastCompactedSSTable with last = 15, we will
      * return an Iterator over [20, 30, 0, 10].
      */
-    Iterator<SSTableReader> wrappingIterator(int lvl, SSTableReader lastCompactedSSTable)
+    Iterator<CompactionSSTable> wrappingIterator(int lvl, CompactionSSTable lastCompactedSSTable)
     {
         assert lvl > 0; // only makes sense in L1+
-        TreeSet<SSTableReader> level = levels[lvl - 1];
+        TreeSet<CompactionSSTable> level = levels[lvl - 1];
         if (level.isEmpty())
             return Collections.emptyIterator();
         if (lastCompactedSSTable == null)
             return level.iterator();
 
-        PeekingIterator<SSTableReader> tail = Iterators.peekingIterator(level.tailSet(lastCompactedSSTable).iterator());
-        SSTableReader pivot = null;
+        PeekingIterator<CompactionSSTable> tail = Iterators.peekingIterator(level.tailSet(lastCompactedSSTable).iterator());
+        CompactionSSTable pivot = null;
         // then we need to make sure that the first token of the pivot is greater than the last token of the lastCompactedSSTable
         while (tail.hasNext())
         {
-            SSTableReader potentialPivot = tail.peek();
+            CompactionSSTable potentialPivot = tail.peek();
             if (potentialPivot.getFirst().compareTo(lastCompactedSSTable.getLast()) > 0)
             {
                 pivot = potentialPivot;
@@ -284,22 +285,22 @@ void logDistribution()
         {
             for (int i = 0; i < levelCount(); i++)
             {
-                Set<SSTableReader> level = get(i);
+                Set<CompactionSSTable> level = get(i);
                 if (!level.isEmpty())
                 {
                     logger.trace("L{} contains {} SSTables ({}) in {}",
                                  i,
                                  level.size(),
-                                 FBUtilities.prettyPrintMemory(SSTableReader.getTotalBytes(level)),
+                                 FBUtilities.prettyPrintMemory(CompactionSSTable.getTotalDataBytes(level)),
                                  this);
                 }
             }
         }
     }
 
-    Set<SSTableReader>[] snapshot()
+    Set<CompactionSSTable>[] snapshot()
     {
-        Set<SSTableReader> [] levelsCopy = new Set[levelCount()];
+        Set<CompactionSSTable> [] levelsCopy = new Set[levelCount()];
         for (int i = 0; i < levelCount(); i++)
             levelsCopy[i] = ImmutableSet.copyOf(get(i));
         return levelsCopy;
@@ -318,8 +319,8 @@ private void maybeVerifyLevels()
         lastOverlapCheck = nanoTime();
         for (int i = 1; i < levelCount(); i++)
         {
-            SSTableReader prev = null;
-            for (SSTableReader sstable : get(i))
+            CompactionSSTable prev = null;
+            for (CompactionSSTable sstable : get(i))
             {
                 // no overlap:
                 assert prev == null || prev.getLast().compareTo(sstable.getFirst()) < 0;
@@ -335,9 +336,9 @@ private void maybeVerifyLevels()
         }
     }
 
-    void newLevel(SSTableReader sstable, int oldLevel)
+    void newLevel(CompactionSSTable sstable, int oldLevel)
     {
-        SSTableReader versionInManifest = allSSTables.remove(sstable);
+        CompactionSSTable versionInManifest = allSSTables.remove(sstable);
         boolean removed = false;
         if (versionInManifest != null)
             removed = get(oldLevel).remove(versionInManifest);
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index a8cafeba2218..573dddf4a824 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.db.compaction;
 
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -27,7 +27,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.Function;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicates;
@@ -35,19 +34,16 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
-import com.google.common.primitives.Ints;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.Pair;
 
 import static org.apache.cassandra.db.compaction.LeveledGenerations.MAX_LEVEL_COUNT;
 
@@ -59,7 +55,13 @@ public class LeveledManifest
      * if we have more than MAX_COMPACTING_L0 sstables in L0, we will run a round of STCS with at most
      * cfs.getMaxCompactionThreshold() sstables.
      */
-    private static final int MAX_COMPACTING_L0 = 32;
+    @VisibleForTesting
+    static final int MAX_COMPACTING_L0 = 32;
+
+    /**
+     * The maximum number of sstables in L0 for calculating the maximum number of bytes in L0.
+     */
+    static final int MAX_SSTABLES_L0 = 4;
 
     /**
      * If we go this many rounds without compacting
@@ -68,36 +70,36 @@ public class LeveledManifest
      */
     private static final int NO_COMPACTION_LIMIT = 25;
 
-    private final ColumnFamilyStore cfs;
+    private final CompactionRealm realm;
 
     private final LeveledGenerations generations;
 
-    private final SSTableReader[] lastCompactedSSTables;
+    private final CompactionSSTable[] lastCompactedSSTables;
     private final long maxSSTableSizeInBytes;
     private final SizeTieredCompactionStrategyOptions options;
     private final int [] compactionCounter;
     private final int levelFanoutSize;
 
-    LeveledManifest(ColumnFamilyStore cfs, int maxSSTableSizeInMB, int fanoutSize, SizeTieredCompactionStrategyOptions options)
+    LeveledManifest(CompactionRealm realm, int maxSSTableSizeInMB, int fanoutSize, SizeTieredCompactionStrategyOptions options)
     {
-        this.cfs = cfs;
+        this.realm = realm;
         this.maxSSTableSizeInBytes = maxSSTableSizeInMB * 1024L * 1024L;
         this.options = options;
         this.levelFanoutSize = fanoutSize;
 
-        lastCompactedSSTables = new SSTableReader[MAX_LEVEL_COUNT];
+        lastCompactedSSTables = new CompactionSSTable[MAX_LEVEL_COUNT];
         generations = new LeveledGenerations();
         compactionCounter = new int[MAX_LEVEL_COUNT];
     }
 
-    public static LeveledManifest create(ColumnFamilyStore cfs, int maxSSTableSize, int fanoutSize, List<SSTableReader> sstables)
+    public static LeveledManifest create(CompactionRealm realm, int maxSSTableSize, int fanoutSize, List<CompactionSSTable> sstables)
     {
-        return create(cfs, maxSSTableSize, fanoutSize, sstables, new SizeTieredCompactionStrategyOptions());
+        return create(realm, maxSSTableSize, fanoutSize, sstables, new SizeTieredCompactionStrategyOptions());
     }
 
-    public static LeveledManifest create(ColumnFamilyStore cfs, int maxSSTableSize, int fanoutSize, Iterable<SSTableReader> sstables, SizeTieredCompactionStrategyOptions options)
+    public static LeveledManifest create(CompactionRealm realm, int maxSSTableSize, int fanoutSize, Iterable<CompactionSSTable> sstables, SizeTieredCompactionStrategyOptions options)
     {
-        LeveledManifest manifest = new LeveledManifest(cfs, maxSSTableSize, fanoutSize, options);
+        LeveledManifest manifest = new LeveledManifest(realm, maxSSTableSize, fanoutSize, options);
 
         // ensure all SSTables are in the manifest
         manifest.addSSTables(sstables);
@@ -113,16 +115,16 @@ void calculateLastCompactedKeys()
     {
         for (int i = 0; i < generations.levelCount() - 1; i++)
         {
-            Set<SSTableReader> level = generations.get(i + 1);
+            Set<CompactionSSTable> level = generations.get(i + 1);
             // this level is empty
             if (level.isEmpty())
                 continue;
 
-            SSTableReader sstableWithMaxModificationTime = null;
+            CompactionSSTable sstableWithMaxModificationTime = null;
             long maxModificationTime = Long.MIN_VALUE;
-            for (SSTableReader ssTableReader : level)
+            for (CompactionSSTable ssTableReader : level)
             {
-                long modificationTime = ssTableReader.getDataCreationTime();
+                long modificationTime = ssTableReader.getCreationTimeFor(SSTableFormat.Components.DATA);
                 if (modificationTime >= maxModificationTime)
                 {
                     sstableWithMaxModificationTime = ssTableReader;
@@ -134,12 +136,12 @@ void calculateLastCompactedKeys()
         }
     }
 
-    public synchronized void addSSTables(Iterable<SSTableReader> readers)
+    public synchronized void addSSTables(Iterable<? extends CompactionSSTable> readers)
     {
         generations.addAll(readers);
     }
 
-    public synchronized void replace(Collection<SSTableReader> removed, Collection<SSTableReader> added)
+    public synchronized void replace(Collection<CompactionSSTable> removed, Collection<CompactionSSTable> added)
     {
         assert !removed.isEmpty(); // use add() instead of promote when adding new sstables
         if (logger.isTraceEnabled())
@@ -159,17 +161,43 @@ public synchronized void replace(Collection<SSTableReader> removed, Collection<S
         if (logger.isTraceEnabled())
             logger.trace("Adding [{}]", toString(added));
         generations.addAll(added);
-        lastCompactedSSTables[minLevel] = SSTableReader.firstKeyOrdering.max(added);
+        lastCompactedSSTables[minLevel] = CompactionSSTable.firstKeyOrdering.max(added);
     }
 
-    private String toString(Collection<SSTableReader> sstables)
+    /**
+     * See {@link AbstractCompactionStrategy#removeDeadSSTables}
+     */
+    public synchronized void removeDeadSSTables()
+    {
+        int removed = 0;
+        Set<? extends CompactionSSTable> liveSet = realm.getLiveSSTables();
+
+        for (int i = 0; i < generations.levelCount(); i++)
+        {
+            Iterator<CompactionSSTable> it = generations.get(i).iterator();
+            while (it.hasNext())
+            {
+                CompactionSSTable sstable = it.next();
+                if (!liveSet.contains(sstable))
+                {
+                    it.remove();
+                    ++removed;
+                }
+            }
+        }
+
+        if (removed > 0)
+            logger.debug("Removed {} dead sstables from the compactions tracked list.", removed);
+    }
+
+    private String toString(Collection<? extends CompactionSSTable> sstables)
     {
         StringBuilder builder = new StringBuilder();
-        for (SSTableReader sstable : sstables)
+        for (CompactionSSTable sstable : sstables)
         {
-            builder.append(sstable.descriptor.cfname)
+            builder.append(sstable.getColumnFamilyName())
                    .append('-')
-                   .append(sstable.descriptor.id)
+                   .append(sstable.getId())
                    .append("(L")
                    .append(sstable.getSSTableLevel())
                    .append("), ");
@@ -185,7 +213,7 @@ public long maxBytesForLevel(int level, long maxSSTableSizeInBytes)
     public static long maxBytesForLevel(int level, int levelFanoutSize, long maxSSTableSizeInBytes)
     {
         if (level == 0)
-            return 4L * maxSSTableSizeInBytes;
+            return MAX_SSTABLES_L0 * maxSSTableSizeInBytes;
         double bytes = Math.pow(levelFanoutSize, level) * maxSSTableSizeInBytes;
         if (bytes > Long.MAX_VALUE)
             throw new RuntimeException("At most " + Long.MAX_VALUE + " bytes may be in a compaction level; your maxSSTableSize must be absurdly high to compute " + bytes);
@@ -196,17 +224,17 @@ public static long maxBytesForLevel(int level, int levelFanoutSize, long maxSSTa
      * @return highest-priority sstables to compact, and level to compact them to
      * If no compactions are necessary, will return null
      */
-    public synchronized CompactionCandidate getCompactionCandidates()
+    synchronized CompactionAggregate.Leveled getCompactionCandidate()
     {
         // during bootstrap we only do size tiering in L0 to make sure
         // the streamed files can be placed in their original levels
         if (StorageService.instance.isBootstrapMode())
         {
-            List<SSTableReader> mostInteresting = getSSTablesForSTCS(generations.get(0));
+            CompactionPick mostInteresting = getSSTablesForSTCS(generations.get(0));
             if (!mostInteresting.isEmpty())
             {
                 logger.info("Bootstrapping - doing STCS in L0");
-                return new CompactionCandidate(mostInteresting, 0, Long.MAX_VALUE);
+                return getSTCSAggregate(mostInteresting);
             }
             return null;
         }
@@ -240,17 +268,17 @@ public synchronized CompactionCandidate getCompactionCandidates()
 
         // Let's check that L0 is far enough behind to warrant STCS.
         // If it is, it will be used before proceeding any of higher level
-        CompactionCandidate l0Compaction = getSTCSInL0CompactionCandidate();
+        CompactionAggregate.Leveled l0Compactions = getSTCSInL0CompactionCandidate();
 
         for (int i = generations.levelCount() - 1; i > 0; i--)
         {
-            Set<SSTableReader> sstables = generations.get(i);
+            Set<CompactionSSTable> sstables = generations.get(i);
             if (sstables.isEmpty())
                 continue; // mostly this just avoids polluting the debug log with zero scores
             // we want to calculate score excluding compacting ones
-            Set<SSTableReader> sstablesInLevel = Sets.newHashSet(sstables);
-            Set<SSTableReader> remaining = Sets.difference(sstablesInLevel, cfs.getTracker().getCompacting());
-            long remainingBytesForLevel = SSTableReader.getTotalBytes(remaining);
+            Set<CompactionSSTable> sstablesInLevel = Sets.newHashSet(sstables);
+            Set<CompactionSSTable> remaining = Sets.difference(sstablesInLevel, realm.getCompactingSSTables());
+            long remainingBytesForLevel = CompactionSSTable.getTotalDataBytes(remaining);
             long maxBytesForLevel = maxBytesForLevel(i, maxSSTableSizeInBytes);
             double score = (double) remainingBytesForLevel / (double) maxBytesForLevel;
             logger.trace("Compaction score for level {} is {}", i, score);
@@ -267,18 +295,20 @@ public synchronized CompactionCandidate getCompactionCandidates()
                 }
 
                 // before proceeding with a higher level, let's see if L0 is far enough behind to warrant STCS
-                if (l0Compaction != null)
-                    return l0Compaction;
+                if (l0Compactions != null)
+                    return l0Compactions;
 
                 // L0 is fine, proceed with this level
-                Collection<SSTableReader> candidates = getCandidatesFor(i);
+                Collection<CompactionSSTable> candidates = getCandidatesFor(i);
+                int pendingCompactions = Math.max(0, getEstimatedPendingTasks(i) - 1);
+
                 if (!candidates.isEmpty())
                 {
                     int nextLevel = getNextLevel(candidates);
                     candidates = getOverlappingStarvedSSTables(nextLevel, candidates);
                     if (logger.isTraceEnabled())
                         logger.trace("Compaction candidates for L{} are {}", i, toString(candidates));
-                    return new CompactionCandidate(candidates, nextLevel, maxSSTableSizeInBytes);
+                    return CompactionAggregate.createLeveled(sstablesInLevel, candidates, pendingCompactions, maxSSTableSizeInBytes, i, nextLevel, score, levelFanoutSize);
                 }
                 else
                 {
@@ -288,44 +318,61 @@ public synchronized CompactionCandidate getCompactionCandidates()
         }
 
         // Higher levels are happy, time for a standard, non-STCS L0 compaction
-        if (generations.get(0).isEmpty())
+        Set<CompactionSSTable> sstables = getLevel(0);
+
+        if (sstables.isEmpty())
             return null;
-        Collection<SSTableReader> candidates = getCandidatesFor(0);
+        Collection<CompactionSSTable> candidates = getCandidatesFor(0);
         if (candidates.isEmpty())
         {
             // Since we don't have any other compactions to do, see if there is a STCS compaction to perform in L0; if
             // there is a long running compaction, we want to make sure that we continue to keep the number of SSTables
             // small in L0.
-            return l0Compaction;
+            return l0Compactions;
         }
-        return new CompactionCandidate(candidates, getNextLevel(candidates), maxSSTableSizeInBytes);
+        double l0Score = (double) CompactionSSTable.getTotalDataBytes(sstables) / (double) maxBytesForLevel(0, maxSSTableSizeInBytes);
+        int l0PendingCompactions = Math.max(0, getEstimatedPendingTasks(0) - 1);
+        return CompactionAggregate.createLeveled(sstables, candidates, l0PendingCompactions, maxSSTableSizeInBytes, 0, getNextLevel(candidates), l0Score, levelFanoutSize);
     }
 
-    private CompactionCandidate getSTCSInL0CompactionCandidate()
+    private CompactionAggregate.Leveled getSTCSInL0CompactionCandidate()
     {
         if (!DatabaseDescriptor.getDisableSTCSInL0() && generations.get(0).size() > MAX_COMPACTING_L0)
         {
-            List<SSTableReader> mostInteresting = getSSTablesForSTCS(generations.get(0));
+            CompactionPick mostInteresting = getSSTablesForSTCS(getLevel(0));
             if (!mostInteresting.isEmpty())
             {
                 logger.debug("L0 is too far behind, performing size-tiering there first");
-                return new CompactionCandidate(mostInteresting, 0, Long.MAX_VALUE);
+                return getSTCSAggregate(mostInteresting);
             }
         }
 
         return null;
     }
 
-    private List<SSTableReader> getSSTablesForSTCS(Collection<SSTableReader> sstables)
+    private CompactionAggregate.Leveled getSTCSAggregate(CompactionPick compaction)
+    {
+        Set<CompactionSSTable> sstables = getLevel(0);
+        double score = (double) CompactionSSTable.getTotalDataBytes(sstables) / (double) maxBytesForLevel(0, maxSSTableSizeInBytes);
+        int remainingSSTables = sstables.size() - compaction.sstables().size();
+        int pendingTasks = remainingSSTables > realm.getMinimumCompactionThreshold()
+                           ? (int) Math.ceil(remainingSSTables / realm.getMaximumCompactionThreshold())
+                           : 0;
+        return CompactionAggregate.createLeveledForSTCS(sstables, compaction, pendingTasks, score, levelFanoutSize);
+    }
+
+    private CompactionPick getSSTablesForSTCS(Collection<CompactionSSTable> sstables)
     {
-        Iterable<SSTableReader> candidates = cfs.getTracker().getUncompacting(sstables);
-        List<Pair<SSTableReader,Long>> pairs = SizeTieredCompactionStrategy.createSSTableAndLengthPairs(AbstractCompactionStrategy.filterSuspectSSTables(candidates));
-        List<List<SSTableReader>> buckets = SizeTieredCompactionStrategy.getBuckets(pairs,
-                                                                                    options.bucketHigh,
-                                                                                    options.bucketLow,
-                                                                                    options.minSSTableSize);
-        return SizeTieredCompactionStrategy.mostInterestingBucket(buckets,
-                cfs.getMinimumCompactionThreshold(), cfs.getMaximumCompactionThreshold());
+        Iterable<? extends CompactionSSTable> candidates = realm.getNoncompactingSSTables(sstables);
+
+        SizeTieredCompactionStrategy.SizeTieredBuckets sizeTieredBuckets;
+        sizeTieredBuckets = new SizeTieredCompactionStrategy.SizeTieredBuckets(candidates,
+                                                                               options,
+                                                                               realm.getMinimumCompactionThreshold(),
+                                                                               realm.getMaximumCompactionThreshold());
+        sizeTieredBuckets.aggregate();
+
+        return CompactionAggregate.getSelected(sizeTieredBuckets.getAggregates());
     }
 
     /**
@@ -339,9 +386,9 @@ private List<SSTableReader> getSSTablesForSTCS(Collection<SSTableReader> sstable
      * @param candidates the original sstables to compact
      * @return
      */
-    private Collection<SSTableReader> getOverlappingStarvedSSTables(int targetLevel, Collection<SSTableReader> candidates)
+    private Collection<CompactionSSTable> getOverlappingStarvedSSTables(int targetLevel, Collection<CompactionSSTable> candidates)
     {
-        Set<SSTableReader> withStarvedCandidate = new HashSet<>(candidates);
+        Set<CompactionSSTable> withStarvedCandidate = new HashSet<>(candidates);
 
         for (int i = generations.levelCount() - 1; i > 0; i--)
             compactionCounter[i]++;
@@ -364,7 +411,7 @@ private Collection<SSTableReader> getOverlappingStarvedSSTables(int targetLevel,
                     // contained within 0 -> 33 to the compaction
                     PartitionPosition max = null;
                     PartitionPosition min = null;
-                    for (SSTableReader candidate : candidates)
+                    for (CompactionSSTable candidate : candidates)
                     {
                         if (min == null || candidate.getFirst().compareTo(min) < 0)
                             min = candidate.getFirst();
@@ -373,9 +420,9 @@ private Collection<SSTableReader> getOverlappingStarvedSSTables(int targetLevel,
                     }
                     if (min == null || max == null || min.equals(max)) // single partition sstables - we cannot include a high level sstable.
                         return candidates;
-                    Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
+                    Set<? extends CompactionSSTable> compacting = realm.getCompactingSSTables();
                     Range<PartitionPosition> boundaries = new Range<>(min, max);
-                    for (SSTableReader sstable : generations.get(i))
+                    for (CompactionSSTable sstable : generations.get(i))
                     {
                         Range<PartitionPosition> r = new Range<>(sstable.getFirst(), sstable.getLast());
                         if (boundaries.contains(r) && !compacting.contains(sstable))
@@ -398,6 +445,14 @@ public synchronized int getLevelSize(int i)
         return generations.get(i).size();
     }
 
+    public synchronized int[] getSSTableCountPerLevel()
+    {
+        int[] counts = new int[getLevelCount()];
+        for (int i = 0; i < counts.length; i++)
+            counts[i] = getLevel(i).size();
+        return counts;
+    }
+
     public synchronized int[] getAllLevelSize()
     {
         return generations.getAllLevelSize();
@@ -409,7 +464,7 @@ public synchronized long[] getAllLevelSizeBytes()
     }
 
     @VisibleForTesting
-    public synchronized int remove(SSTableReader reader)
+    public synchronized int remove(CompactionSSTable reader)
     {
         int level = reader.getSSTableLevel();
         assert level >= 0 : reader + " not present in manifest: "+level;
@@ -417,12 +472,12 @@ public synchronized int remove(SSTableReader reader)
         return level;
     }
 
-    public synchronized Set<SSTableReader> getSSTables()
+    public synchronized Set<CompactionSSTable> getSSTables()
     {
         return generations.allSSTables();
     }
 
-    private static Set<SSTableReader> overlapping(Collection<SSTableReader> candidates, Iterable<SSTableReader> others)
+    private static Set<CompactionSSTable> overlapping(Collection<CompactionSSTable> candidates, Iterable<CompactionSSTable> others)
     {
         assert !candidates.isEmpty();
         /*
@@ -436,8 +491,8 @@ private static Set<SSTableReader> overlapping(Collection<SSTableReader> candidat
          * Thus, the correct approach is to pick sstables overlapping anything between the first key in all
          * the candidate sstables, and the last.
          */
-        Iterator<SSTableReader> iter = candidates.iterator();
-        SSTableReader sstable = iter.next();
+        Iterator<CompactionSSTable> iter = candidates.iterator();
+        CompactionSSTable sstable = iter.next();
         Token first = sstable.getFirst().getToken();
         Token last = sstable.getLast().getToken();
         while (iter.hasNext())
@@ -449,7 +504,7 @@ private static Set<SSTableReader> overlapping(Collection<SSTableReader> candidat
         return overlapping(first, last, others);
     }
 
-    private static Set<SSTableReader> overlappingWithBounds(SSTableReader sstable, Map<SSTableReader, Bounds<Token>> others)
+    static Set<CompactionSSTable> overlappingWithBounds(CompactionSSTable sstable, Map<CompactionSSTable, Bounds<Token>> others)
     {
         return overlappingWithBounds(sstable.getFirst().getToken(), sstable.getLast().getToken(), others);
     }
@@ -458,18 +513,18 @@ private static Set<SSTableReader> overlappingWithBounds(SSTableReader sstable, M
      * @return sstables from @param sstables that contain keys between @param start and @param end, inclusive.
      */
     @VisibleForTesting
-    static Set<SSTableReader> overlapping(Token start, Token end, Iterable<SSTableReader> sstables)
+    static Set<CompactionSSTable> overlapping(Token start, Token end, Iterable<CompactionSSTable> sstables)
     {
         return overlappingWithBounds(start, end, genBounds(sstables));
     }
 
-    private static Set<SSTableReader> overlappingWithBounds(Token start, Token end, Map<SSTableReader, Bounds<Token>> sstables)
+    private static Set<CompactionSSTable> overlappingWithBounds(Token start, Token end, Map<CompactionSSTable, Bounds<Token>> sstables)
     {
         assert start.compareTo(end) <= 0;
-        Set<SSTableReader> overlapped = new HashSet<>();
+        Set<CompactionSSTable> overlapped = new HashSet<>();
         Bounds<Token> promotedBounds = new Bounds<>(start, end);
 
-        for (Map.Entry<SSTableReader, Bounds<Token>> pair : sstables.entrySet())
+        for (Map.Entry<CompactionSSTable, Bounds<Token>> pair : sstables.entrySet())
         {
             if (pair.getValue().intersects(promotedBounds))
                 overlapped.add(pair.getKey());
@@ -477,10 +532,11 @@ private static Set<SSTableReader> overlappingWithBounds(Token start, Token end,
         return overlapped;
     }
 
-    private static Map<SSTableReader, Bounds<Token>> genBounds(Iterable<SSTableReader> ssTableReaders)
+    @VisibleForTesting
+    static Map<CompactionSSTable, Bounds<Token>> genBounds(Iterable<? extends CompactionSSTable> ssTableReaders)
     {
-        Map<SSTableReader, Bounds<Token>> boundsMap = new HashMap<>();
-        for (SSTableReader sstable : ssTableReaders)
+        Map<CompactionSSTable, Bounds<Token>> boundsMap = new HashMap<>();
+        for (CompactionSSTable sstable : ssTableReaders)
         {
             boundsMap.put(sstable, new Bounds<>(sstable.getFirst().getToken(), sstable.getLast().getToken()));
         }
@@ -488,24 +544,30 @@ private static Map<SSTableReader, Bounds<Token>> genBounds(Iterable<SSTableReade
     }
 
     /**
+     * Determine the highest-priority sstables to compact for the given level and add any overlapping sstables
+     * from the next level.
+     * <p/>
      * @return highest-priority sstables to compact for the given level.
      * If no compactions are possible (because of concurrent compactions or because some sstables are excluded
      * for prior failure), will return an empty list.  Never returns null.
+     *
+     * @param level the level number
+     * @return highest-priority sstables to compact for the given level.
      */
-    private Collection<SSTableReader> getCandidatesFor(int level)
+    private Collection<CompactionSSTable> getCandidatesFor(int level)
     {
         assert !generations.get(level).isEmpty();
         logger.trace("Choosing candidates for L{}", level);
 
-        final Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
+        final Set<? extends CompactionSSTable> compacting = realm.getCompactingSSTables();
 
         if (level == 0)
         {
-            Set<SSTableReader> compactingL0 = getCompactingL0();
+            Set<CompactionSSTable> compactingL0 = getCompactingL0();
 
             PartitionPosition lastCompactingKey = null;
             PartitionPosition firstCompactingKey = null;
-            for (SSTableReader candidate : compactingL0)
+            for (CompactionSSTable candidate : compactingL0)
             {
                 if (firstCompactingKey == null || candidate.getFirst().compareTo(firstCompactingKey) < 0)
                     firstCompactingKey = candidate.getFirst();
@@ -526,40 +588,40 @@ private Collection<SSTableReader> getCandidatesFor(int level)
             // Note that we ignore suspect-ness of L1 sstables here, since if an L1 sstable is suspect we're
             // basically screwed, since we expect all or most L0 sstables to overlap with each L1 sstable.
             // So if an L1 sstable is suspect we can't do much besides try anyway and hope for the best.
-            Set<SSTableReader> candidates = new HashSet<>();
-            Map<SSTableReader, Bounds<Token>> remaining = genBounds(Iterables.filter(generations.get(0), Predicates.not(SSTableReader::isMarkedSuspect)));
+            Set<CompactionSSTable> candidates = new HashSet<>();
+            Map<CompactionSSTable, Bounds<Token>> remaining = genBounds(Iterables.filter(generations.get(0), Predicates.not(CompactionSSTable::isMarkedSuspect)));
 
-            for (SSTableReader sstable : ageSortedSSTables(remaining.keySet()))
+            for (CompactionSSTable sstable : ageSortedSSTables(remaining.keySet()))
             {
                 if (candidates.contains(sstable))
                     continue;
 
-                Sets.SetView<SSTableReader> overlappedL0 = Sets.union(Collections.singleton(sstable), overlappingWithBounds(sstable, remaining));
+                Sets.SetView<CompactionSSTable> overlappedL0 = Sets.union(Collections.singleton(sstable), overlappingWithBounds(sstable, remaining));
                 if (!Sets.intersection(overlappedL0, compactingL0).isEmpty())
                     continue;
 
-                for (SSTableReader newCandidate : overlappedL0)
+                for (CompactionSSTable newCandidate : overlappedL0)
                 {
                     if (firstCompactingKey == null || lastCompactingKey == null || overlapping(firstCompactingKey.getToken(), lastCompactingKey.getToken(), Collections.singleton(newCandidate)).size() == 0)
                         candidates.add(newCandidate);
                     remaining.remove(newCandidate);
                 }
 
-                if (candidates.size() > cfs.getMaximumCompactionThreshold())
+                if (candidates.size() > realm.getMaximumCompactionThreshold())
                 {
                     // limit to only the cfs.getMaximumCompactionThreshold() oldest candidates
-                    candidates = new HashSet<>(ageSortedSSTables(candidates).subList(0, cfs.getMaximumCompactionThreshold()));
+                    candidates = new HashSet<>(ageSortedSSTables(candidates).subList(0, realm.getMaximumCompactionThreshold()));
                     break;
                 }
             }
 
             // leave everything in L0 if we didn't end up with a full sstable's worth of data
-            if (SSTableReader.getTotalBytes(candidates) > maxSSTableSizeInBytes)
+            if (CompactionSSTable.getTotalDataBytes(candidates) > maxSSTableSizeInBytes)
             {
                 // add sstables from L1 that overlap candidates
                 // if the overlapping ones are already busy in a compaction, leave it out.
                 // TODO try to find a set of L0 sstables that only overlaps with non-busy L1 sstables
-                Set<SSTableReader> l1overlapping = overlapping(candidates, generations.get(1));
+                Set<CompactionSSTable> l1overlapping = overlapping(candidates, generations.get(1));
                 if (Sets.intersection(l1overlapping, compacting).size() > 0)
                     return Collections.emptyList();
                 if (!overlapping(candidates, compactingL0).isEmpty())
@@ -574,14 +636,14 @@ private Collection<SSTableReader> getCandidatesFor(int level)
 
         // look for a non-suspect keyspace to compact with, starting with where we left off last time,
         // and wrapping back to the beginning of the generation if necessary
-        Map<SSTableReader, Bounds<Token>> sstablesNextLevel = genBounds(generations.get(level + 1));
-        Iterator<SSTableReader> levelIterator = generations.wrappingIterator(level, lastCompactedSSTables[level]);
+        Map<CompactionSSTable, Bounds<Token>> sstablesNextLevel = genBounds(generations.get(level + 1));
+        Iterator<CompactionSSTable> levelIterator = generations.wrappingIterator(level, lastCompactedSSTables[level]);
         while (levelIterator.hasNext())
         {
-            SSTableReader sstable = levelIterator.next();
-            Set<SSTableReader> candidates = Sets.union(Collections.singleton(sstable), overlappingWithBounds(sstable, sstablesNextLevel));
+            CompactionSSTable sstable = levelIterator.next();
+            Set<CompactionSSTable> candidates = Sets.union(Collections.singleton(sstable), overlappingWithBounds(sstable, sstablesNextLevel));
 
-            if (Iterables.any(candidates, SSTableReader::isMarkedSuspect))
+            if (Iterables.any(candidates, CompactionSSTable::isMarkedSuspect))
                 continue;
             if (Sets.intersection(candidates, compacting).isEmpty())
                 return candidates;
@@ -591,11 +653,11 @@ private Collection<SSTableReader> getCandidatesFor(int level)
         return Collections.emptyList();
     }
 
-    private Set<SSTableReader> getCompactingL0()
+    private Set<CompactionSSTable> getCompactingL0()
     {
-        Set<SSTableReader> sstables = new HashSet<>();
-        Set<SSTableReader> levelSSTables = new HashSet<>(generations.get(0));
-        for (SSTableReader sstable : cfs.getTracker().getCompacting())
+        Set<CompactionSSTable> sstables = new HashSet<>();
+        Set<CompactionSSTable> levelSSTables = new HashSet<>(generations.get(0));
+        for (CompactionSSTable sstable : realm.getCompactingSSTables())
         {
             if (levelSSTables.contains(sstable))
                 sstables.add(sstable);
@@ -604,12 +666,12 @@ private Set<SSTableReader> getCompactingL0()
     }
 
     @VisibleForTesting
-    List<SSTableReader> ageSortedSSTables(Collection<SSTableReader> candidates)
+    List<CompactionSSTable> ageSortedSSTables(Collection<CompactionSSTable> candidates)
     {
-        return ImmutableList.sortedCopyOf(SSTableReader.maxTimestampAscending, candidates);
+        return ImmutableList.sortedCopyOf(CompactionSSTable.maxTimestampAscending, candidates);
     }
 
-    public synchronized Set<SSTableReader>[] getSStablesPerLevelSnapshot()
+    public synchronized Set<CompactionSSTable>[] getSStablesPerLevelSnapshot()
     {
         return generations.snapshot();
     }
@@ -630,52 +692,79 @@ public synchronized int getLevelCount()
         return 0;
     }
 
-    public int getEstimatedTasks()
+    public synchronized List<CompactionAggregate> getEstimatedTasks(CompactionAggregate.Leveled selected)
     {
-        return getEstimatedTasks(0);
-    }
+        List<CompactionAggregate> ret = new ArrayList<>(generations.levelCount());
 
-    int getEstimatedTasks(long additionalLevel0Bytes)
-    {
-        return getEstimatedTasks((level) -> SSTableReader.getTotalBytes(getLevel(level)) + (level == 0 ? additionalLevel0Bytes : 0));
+        for (int i = generations.levelCount() - 1; i >= 0; i--)
+        {
+            Set<CompactionSSTable> sstables = generations.get(i);
+
+            // do not log high levels that are empty, only log after we've found a non-empty level
+            if (sstables.isEmpty() && ret.isEmpty())
+                continue;
+
+            if (selected != null && selected.level == i)
+            {
+                ret.add(selected);
+                continue; // pending tasks already calculated by getCompactionCandidate()
+            }
+
+            if (i == 0)
+            { // for L0 if it is too far behind then pick the STCS choice
+                CompactionAggregate l0Compactions = getSTCSInL0CompactionCandidate();
+                if (l0Compactions != null)
+                {
+                    ret.add(l0Compactions);
+                    continue;
+                }
+            }
+
+            int pendingTasks = getEstimatedPendingTasks(i);
+            double score = (double) CompactionSSTable.getTotalDataBytes(sstables) / (double) maxBytesForLevel(i, maxSSTableSizeInBytes);
+            ret.add(CompactionAggregate.createLeveled(sstables, pendingTasks, maxSSTableSizeInBytes, i, score, levelFanoutSize));
+        }
+
+        logger.trace("Estimating {} compactions to do for {}", ret.size(), realm.metadata());
+        return ret;
     }
 
-    private synchronized int getEstimatedTasks(Function<Integer,Long> fnTotalSizeBytesByLevel)
+    /**
+     * @return the estimated number of LCS compactions for a given level with the given sstables. Because it compacts one sstable at
+     *         a time, this number is determined as the number of bytes above the maximum divided the maximum sstable size in bytes.
+     *
+     *         This is however incorrect for L0. If the STCS threshold has been exceeded, we simply divide by the max threshold,
+     *         otherwise we currently use a very pessimistic estimate (no overlapping sstables).
+     */
+    private int getEstimatedPendingTasks(int level)
     {
-        long tasks = 0;
-        long[] estimated = new long[generations.levelCount()];
+        final Set<CompactionSSTable> sstables = getLevel(level);
+        if (sstables.isEmpty())
+            return 0;
 
-        for (int i = generations.levelCount() - 1; i >= 0; i--)
-        {
-            // If there is 1 byte over TBL - (MBL * 1.001), there is still a task left, so we need to round up.
-            estimated[i] = (long)Math.ceil((double)Math.max(0L, fnTotalSizeBytesByLevel.apply(i) - (long)(maxBytesForLevel(i, maxSSTableSizeInBytes) * 1.001)) / (double)maxSSTableSizeInBytes);
-            tasks += estimated[i];
-        }
+        final Set<? extends CompactionSSTable> compacting = realm.getCompactingSSTables();
+        final Set<CompactionSSTable> remaining = Sets.difference(Sets.newHashSet(sstables), compacting);
 
-        if (!DatabaseDescriptor.getDisableSTCSInL0() && generations.get(0).size() > cfs.getMaximumCompactionThreshold())
-        {
-            int l0compactions = generations.get(0).size() / cfs.getMaximumCompactionThreshold();
-            tasks += l0compactions;
-            estimated[0] += l0compactions;
-        }
+        if (level == 0 && !DatabaseDescriptor.getDisableSTCSInL0() && remaining.size() > MAX_COMPACTING_L0)
+            return remaining.size() / realm.getMaximumCompactionThreshold();
 
-        logger.trace("Estimating {} compactions to do for {}.{}",
-                     Arrays.toString(estimated), cfs.getKeyspaceName(), cfs.name);
-        return Ints.checkedCast(tasks);
+        // If there is 1 byte over TBL - (MBL * 1.001), there is still a task left, so we need to round up.
+        return Math.toIntExact((long) Math.ceil((Math.max(0L, CompactionSSTable.getTotalDataBytes(remaining) -
+                                                              (maxBytesForLevel(level, maxSSTableSizeInBytes) * 1.001)) / (double) maxSSTableSizeInBytes)));
     }
 
-    public int getNextLevel(Collection<SSTableReader> sstables)
+    int getNextLevel(Collection<CompactionSSTable> sstables)
     {
         int maximumLevel = Integer.MIN_VALUE;
         int minimumLevel = Integer.MAX_VALUE;
-        for (SSTableReader sstable : sstables)
+        for (CompactionSSTable sstable : sstables)
         {
             maximumLevel = Math.max(sstable.getSSTableLevel(), maximumLevel);
             minimumLevel = Math.min(sstable.getSSTableLevel(), minimumLevel);
         }
 
         int newLevel;
-        if (minimumLevel == 0 && minimumLevel == maximumLevel && SSTableReader.getTotalBytes(sstables) < maxSSTableSizeInBytes)
+        if (minimumLevel == 0 && minimumLevel == maximumLevel && CompactionSSTable.getTotalDataBytes(sstables) < maxSSTableSizeInBytes)
         {
             newLevel = 0;
         }
@@ -687,33 +776,19 @@ public int getNextLevel(Collection<SSTableReader> sstables)
         return newLevel;
     }
 
-    synchronized Set<SSTableReader> getLevel(int level)
+    synchronized Set<CompactionSSTable> getLevel(int level)
     {
         return ImmutableSet.copyOf(generations.get(level));
     }
 
-    synchronized List<SSTableReader> getLevelSorted(int level, Comparator<SSTableReader> comparator)
+    synchronized List<CompactionSSTable> getLevelSorted(int level, Comparator<CompactionSSTable> comparator)
     {
         return ImmutableList.sortedCopyOf(comparator, generations.get(level));
     }
 
-    synchronized void newLevel(SSTableReader sstable, int oldLevel)
+    synchronized void newLevel(CompactionSSTable sstable, int oldLevel)
     {
         generations.newLevel(sstable, oldLevel);
         lastCompactedSSTables[oldLevel] = sstable;
     }
-
-    public static class CompactionCandidate
-    {
-        public final Collection<SSTableReader> sstables;
-        public final int level;
-        public final long maxSSTableBytes;
-
-        public CompactionCandidate(Collection<SSTableReader> sstables, int level, long maxSSTableBytes)
-        {
-            this.sstables = sstables;
-            this.level = level;
-            this.maxSSTableBytes = maxSSTableBytes;
-        }
-    }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/OperationType.java b/src/java/org/apache/cassandra/db/compaction/OperationType.java
index 2a5ffc61e678..2a0ba1f704be 100644
--- a/src/java/org/apache/cassandra/db/compaction/OperationType.java
+++ b/src/java/org/apache/cassandra/db/compaction/OperationType.java
@@ -17,6 +17,16 @@
  */
 package org.apache.cassandra.db.compaction;
 
+import com.google.common.base.Predicate;
+
+/**
+ * The types of operations that can be observed with {@link AbstractTableOperation} and tracked by
+ * {@link org.apache.cassandra.db.lifecycle.LifecycleTransaction}.
+ * <p/>
+ * Historically these operations have been broadly described as "compactions", even though they have
+ * nothing to do with actual compactions. Any operation that can report progress and that normally
+ * involves files, either for reading or writing, is a valid operation.
+ */
 public enum OperationType
 {
     /** Each modification here should be also applied to {@link org.apache.cassandra.tools.nodetool.Stop#compactionType} */
@@ -49,10 +59,28 @@ public enum OperationType
     KEY_CACHE_SAVE("Key cache save", false, 6),
     ROW_CACHE_SAVE("Row cache save", false, 6),
     COUNTER_CACHE_SAVE("Counter cache save", false, 6),
-    INDEX_SUMMARY("Index summary redistribution", false, 6);
+    INDEX_SUMMARY("Index summary redistribution", false, 6),
+    // FIXME CNDB-11008: Review port of STAR-979 to review values of `writesData` and `priority` for the added operations below
+    RESTORE("Restore", false, 6),
+    // operations used for sstables on remote storage
+    REMOTE_RELOAD("Remote reload", false, 6, true), // reload locally sstables that already exist remotely
+    REMOTE_COMPACTION("Remote compaction", false, 6, true), // no longer used, kept for backward compatibility
+    REMOTE_RELOAD_FOR_REPAIR("Remote reload for repair", false, 6, true, false), // reload locally sstables that already exist remotely for repair
+    TRUNCATE_TABLE("Table truncated", false, 6),
+    DROP_TABLE("Table dropped", false, 6),
+    REMOVE_UNREADEABLE("Remove unreadable sstables", false, 6),
+    REGION_BOOTSTRAP("Region Bootstrap", false, 6),
+    REGION_DECOMMISSION("Region Decommission", false, 6),
+    REGION_REPAIR("Region Repair", false, 6),
+    SSTABLE_DISCARD("Local-only sstable discard", false, 6, true),
+    INITIAL_LOAD("Local-only sstable loading during node initialization", false, 6, true);
 
     public final String type;
     public final String fileName;
+    /** true if the transaction of this type should NOT be uploaded remotely */
+    public final boolean localOnly;
+    /** true if the transaction should remove unfinished leftovers for CNDB */
+    public final boolean removeTransactionLeftovers;
 
     /**
      * For purposes of calculating space for interim compactions in flight, whether or not this OperationType is expected
@@ -67,11 +95,23 @@ public enum OperationType
     public final int priority;
 
     OperationType(String type, boolean writesData, int priority)
+    {
+        this(type, writesData, priority, false);
+    }
+
+    OperationType(String type, boolean writesData, int priority, boolean localOnly)
+    {
+        this(type, writesData, priority, localOnly, true);
+    }
+
+    OperationType(String type, boolean writesData, int priority, boolean localOnly, boolean removeTransactionLeftovers)
     {
         this.type = type;
         this.fileName = type.toLowerCase().replace(" ", "");
         this.writesData = writesData;
         this.priority = priority;
+        this.localOnly = localOnly;
+        this.removeTransactionLeftovers = removeTransactionLeftovers;
     }
 
     public static OperationType fromFileName(String fileName)
@@ -83,8 +123,20 @@ public static OperationType fromFileName(String fileName)
         throw new IllegalArgumentException("Invalid fileName for operation type: " + fileName);
     }
 
+    public boolean isCacheSave()
+    {
+        return this == COUNTER_CACHE_SAVE || this == KEY_CACHE_SAVE || this == ROW_CACHE_SAVE;
+    }
+
     public String toString()
     {
         return type;
     }
+
+    public static final Predicate<OperationType> EXCEPT_VALIDATIONS = o -> o != VALIDATION;
+    public static final Predicate<OperationType> COMPACTIONS_ONLY = o -> o == COMPACTION || o == TOMBSTONE_COMPACTION;
+    public static final Predicate<OperationType> REWRITES_SSTABLES = o -> o == COMPACTION || o == CLEANUP || o == SCRUB ||
+                                                                          o == TOMBSTONE_COMPACTION || o == ANTICOMPACTION ||
+                                                                          o == UPGRADE_SSTABLES || o == RELOCATE ||
+                                                                          o == GARBAGE_COLLECT;
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
index 0c5d53c1d8a3..9d8672d28589 100644
--- a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
@@ -24,9 +24,9 @@
 import java.util.List;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.commitlog.IntervalSet;
@@ -47,9 +47,9 @@ public class PendingRepairHolder extends AbstractStrategyHolder
     private final List<PendingRepairManager> managers = new ArrayList<>();
     private final boolean isTransient;
 
-    public PendingRepairHolder(ColumnFamilyStore cfs, DestinationRouter router, boolean isTransient)
+    public PendingRepairHolder(CompactionRealm realm, CompactionStrategyFactory strategyFactory, DestinationRouter router, boolean isTransient)
     {
-        super(cfs, router);
+        super(realm, strategyFactory, router);
         this.isTransient = isTransient;
     }
 
@@ -70,7 +70,7 @@ public void setStrategyInternal(CompactionParams params, int numTokenPartitions)
     {
         managers.clear();
         for (int i = 0; i < numTokenPartitions; i++)
-            managers.add(new PendingRepairManager(cfs, params, isTransient));
+            managers.add(new PendingRepairManager(realm, strategyFactory, params, isTransient));
     }
 
     @Override
@@ -82,24 +82,24 @@ public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair,
     }
 
     @Override
-    public AbstractCompactionStrategy getStrategyFor(SSTableReader sstable)
+    public LegacyAbstractCompactionStrategy getStrategyFor(CompactionSSTable sstable)
     {
         Preconditions.checkArgument(managesSSTable(sstable), "Attempting to get compaction strategy from wrong holder");
         return managers.get(router.getIndexForSSTable(sstable)).getOrCreate(sstable);
     }
 
     @Override
-    public Iterable<AbstractCompactionStrategy> allStrategies()
+    public Iterable<LegacyAbstractCompactionStrategy> allStrategies()
     {
         return Iterables.concat(Iterables.transform(managers, PendingRepairManager::getStrategies));
     }
 
-    Iterable<AbstractCompactionStrategy> getStrategiesFor(TimeUUID session)
+    Iterable<LegacyAbstractCompactionStrategy> getStrategiesFor(TimeUUID session)
     {
-        List<AbstractCompactionStrategy> strategies = new ArrayList<>(managers.size());
+        List<LegacyAbstractCompactionStrategy> strategies = new ArrayList<>(managers.size());
         for (PendingRepairManager manager : managers)
         {
-            AbstractCompactionStrategy strategy = manager.get(session);
+            LegacyAbstractCompactionStrategy strategy = manager.get(session);
             if (strategy != null)
                 strategies.add(strategy);
         }
@@ -112,24 +112,22 @@ public Iterable<PendingRepairManager> getManagers()
     }
 
     @Override
-    public Collection<TaskSupplier> getBackgroundTaskSuppliers(long gcBefore)
+    public Collection<TasksSupplier> getBackgroundTaskSuppliers(long gcBefore)
     {
-        List<TaskSupplier> suppliers = new ArrayList<>(managers.size());
+        List<TasksSupplier> suppliers = new ArrayList<>(managers.size());
         for (PendingRepairManager manager : managers)
-            suppliers.add(new TaskSupplier(manager.getMaxEstimatedRemainingTasks(), () -> manager.getNextBackgroundTask(gcBefore)));
+            suppliers.add(new TasksSupplier(manager.getMaxEstimatedRemainingTasks(), () -> manager.getNextBackgroundTasks(gcBefore)));
 
         return suppliers;
     }
 
     @Override
-    public Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, boolean splitOutput)
+    public Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism)
     {
         List<AbstractCompactionTask> tasks = new ArrayList<>(managers.size());
         for (PendingRepairManager manager : managers)
         {
-            Collection<AbstractCompactionTask> task = manager.getMaximalTasks(gcBefore, splitOutput);
-            if (task != null)
-                tasks.addAll(task);
+            tasks.addAll(manager.getMaximalTasks(gcBefore, splitOutput, permittedParallelism));
         }
         return tasks;
     }
@@ -149,38 +147,31 @@ public Collection<AbstractCompactionTask> getUserDefinedTasks(GroupedSSTableCont
         return tasks;
     }
 
-    @Override
-    public void addSSTable(SSTableReader sstable)
-    {
-        Preconditions.checkArgument(managesSSTable(sstable), "Attempting to add sstable from wrong holder");
-        managers.get(router.getIndexForSSTable(sstable)).addSSTable(sstable);
-    }
-
-    AbstractCompactionTask getNextRepairFinishedTask()
+    Collection<AbstractCompactionTask> getNextRepairFinishedTasks()
     {
-        List<TaskSupplier> repairFinishedSuppliers = getRepairFinishedTaskSuppliers();
+        List<TasksSupplier> repairFinishedSuppliers = getRepairFinishedTaskSuppliers();
         if (!repairFinishedSuppliers.isEmpty())
         {
             Collections.sort(repairFinishedSuppliers);
-            for (TaskSupplier supplier : repairFinishedSuppliers)
+            for (TasksSupplier supplier : repairFinishedSuppliers)
             {
-                AbstractCompactionTask task = supplier.getTask();
-                if (task != null)
-                    return task;
+                Collection<AbstractCompactionTask> tasks = supplier.getTasks();
+                if (!tasks.isEmpty())
+                    return tasks;
             }
         }
-        return null;
+        return ImmutableList.of();
     }
 
-    private ArrayList<TaskSupplier> getRepairFinishedTaskSuppliers()
+    private ArrayList<TasksSupplier> getRepairFinishedTaskSuppliers()
     {
-        ArrayList<TaskSupplier> suppliers = new ArrayList<>(managers.size());
+        ArrayList<TasksSupplier> suppliers = new ArrayList<>(managers.size());
         for (PendingRepairManager manager : managers)
         {
             int numPending = manager.getNumPendingRepairFinishedTasks();
             if (numPending > 0)
             {
-                suppliers.add(new TaskSupplier(numPending, manager::getNextRepairFinishedTask));
+                suppliers.add(new TasksSupplier(numPending, manager::getNextRepairFinishedTasks));
             }
         }
 
@@ -227,7 +218,7 @@ public void replaceSSTables(GroupedSSTableContainer removed, GroupedSSTableConta
     }
 
     @Override
-    public List<ISSTableScanner> getScanners(GroupedSSTableContainer sstables, Collection<Range<Token>> ranges)
+    public List<ISSTableScanner> getScanners(GroupedSSTableContainer<SSTableReader> sstables, Collection<Range<Token>> ranges)
     {
         List<ISSTableScanner> scanners = new ArrayList<>(managers.size());
         for (int i = 0; i < managers.size(); i++)
@@ -257,7 +248,7 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
         Preconditions.checkArgument(pendingRepair != null,
                                     "PendingRepairHolder can't create sstable writer without pendingRepair id");
         // to avoid creating a compaction strategy for the wrong pending repair manager, we get the index based on where the sstable is to be written
-        AbstractCompactionStrategy strategy = managers.get(router.getIndexForSSTableDirectory(descriptor)).getOrCreate(pendingRepair);
+        CompactionStrategy strategy = managers.get(router.getIndexForSSTableDirectory(descriptor)).getOrCreate(pendingRepair);
         return strategy.createSSTableMultiWriter(descriptor,
                                                  keyCount,
                                                  repairedAt,
@@ -270,24 +261,13 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
                                                  lifecycleNewTracker);
     }
 
-    @Override
-    public int getStrategyIndex(AbstractCompactionStrategy strategy)
-    {
-        for (int i = 0; i < managers.size(); i++)
-        {
-            if (managers.get(i).hasStrategy(strategy))
-                return i;
-        }
-        return -1;
-    }
-
     public boolean hasDataForSession(TimeUUID sessionID)
     {
         return Iterables.any(managers, prm -> prm.hasDataForSession(sessionID));
     }
 
     @Override
-    public boolean containsSSTable(SSTableReader sstable)
+    public boolean containsSSTable(CompactionSSTable sstable)
     {
         return Iterables.any(managers, prm -> prm.containsSSTable(sstable));
     }
@@ -301,8 +281,8 @@ public int getEstimatedRemainingTasks()
         return tasks;
     }
 
-    public boolean hasPendingRepairSSTable(TimeUUID sessionID, SSTableReader sstable)
+    public int size()
     {
-        return Iterables.any(managers, prm -> prm.hasPendingRepairSSTable(sessionID, sstable));
+        return managers.size();
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
index 7251c04dcf02..78d9d31f84cf 100644
--- a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
@@ -28,31 +28,26 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.repair.consistent.admin.CleanupSummary;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.TimeUUID;
 
 /**
- * Companion to CompactionStrategyManager which manages the sstables marked pending repair.
+ * This class manages the sstables marked pending repair so that they can be assigned to legacy compaction
+ * strategies via the legacy strategy container or manager.
  *
  * SSTables are classified as pending repair by the anti-compaction performed at the beginning
  * of an incremental repair, or when they're streamed in with a pending repair id. This prevents
@@ -63,10 +58,11 @@ class PendingRepairManager
 {
     private static final Logger logger = LoggerFactory.getLogger(PendingRepairManager.class);
 
-    private final ColumnFamilyStore cfs;
+    private final CompactionRealm realm;
+    private final CompactionStrategyFactory strategyFactory;
     private final CompactionParams params;
     private final boolean isTransient;
-    private volatile ImmutableMap<TimeUUID, AbstractCompactionStrategy> strategies = ImmutableMap.of();
+    private volatile ImmutableMap<TimeUUID, LegacyAbstractCompactionStrategy> strategies = ImmutableMap.of();
 
     /**
      * Indicates we're being asked to do something with an sstable that isn't marked pending repair
@@ -79,34 +75,35 @@ public IllegalSSTableArgumentException(String s)
         }
     }
 
-    PendingRepairManager(ColumnFamilyStore cfs, CompactionParams params, boolean isTransient)
+    PendingRepairManager(CompactionRealm realm, CompactionStrategyFactory strategyFactory, CompactionParams params, boolean isTransient)
     {
-        this.cfs = cfs;
+        this.realm = realm;
+        this.strategyFactory = strategyFactory;
         this.params = params;
         this.isTransient = isTransient;
     }
 
-    private ImmutableMap.Builder<TimeUUID, AbstractCompactionStrategy> mapBuilder()
+    private ImmutableMap.Builder<TimeUUID, LegacyAbstractCompactionStrategy> mapBuilder()
     {
         return ImmutableMap.builder();
     }
 
-    AbstractCompactionStrategy get(TimeUUID id)
+    LegacyAbstractCompactionStrategy get(TimeUUID id)
     {
         return strategies.get(id);
     }
 
-    AbstractCompactionStrategy get(SSTableReader sstable)
+    LegacyAbstractCompactionStrategy get(CompactionSSTable sstable)
     {
         assert sstable.isPendingRepair();
-        return get(sstable.getSSTableMetadata().pendingRepair);
+        return get(sstable.getPendingRepair());
     }
 
-    AbstractCompactionStrategy getOrCreate(TimeUUID id)
+    LegacyAbstractCompactionStrategy getOrCreate(TimeUUID id)
     {
         checkPendingID(id);
         assert id != null;
-        AbstractCompactionStrategy strategy = get(id);
+        LegacyAbstractCompactionStrategy strategy = get(id);
         if (strategy == null)
         {
             synchronized (this)
@@ -115,8 +112,8 @@ AbstractCompactionStrategy getOrCreate(TimeUUID id)
 
                 if (strategy == null)
                 {
-                    logger.debug("Creating {}.{} compaction strategy for pending repair: {}", cfs.metadata.keyspace, cfs.metadata.name, id);
-                    strategy = cfs.createCompactionStrategyInstance(params);
+                    logger.debug("Creating {}.{} compaction strategy for pending repair: {}", realm.getKeyspaceName(), realm.getTableName(), id);
+                    strategy = strategyFactory.createLegacyStrategy(params);
                     strategies = mapBuilder().putAll(strategies).put(id, strategy).build();
                 }
             }
@@ -132,58 +129,57 @@ private static void checkPendingID(TimeUUID pendingID)
         }
     }
 
-    AbstractCompactionStrategy getOrCreate(SSTableReader sstable)
+    LegacyAbstractCompactionStrategy getOrCreate(CompactionSSTable sstable)
     {
-        return getOrCreate(sstable.getSSTableMetadata().pendingRepair);
+        return getOrCreate(sstable.getPendingRepair());
     }
 
-    private synchronized void removeSessionIfEmpty(TimeUUID sessionID)
+    synchronized void removeSessionIfEmpty(TimeUUID sessionID)
     {
         if (!strategies.containsKey(sessionID) || !strategies.get(sessionID).getSSTables().isEmpty())
             return;
 
-        logger.debug("Removing compaction strategy for pending repair {} on  {}.{}", sessionID, cfs.metadata.keyspace, cfs.metadata.name);
+        logger.debug("Removing compaction strategy for pending repair {} on  {}.{}", sessionID, realm.getKeyspaceName(), realm.getTableName());
         strategies = ImmutableMap.copyOf(Maps.filterKeys(strategies, k -> !k.equals(sessionID)));
     }
 
-    synchronized void removeSSTable(SSTableReader sstable)
+    synchronized void removeSSTable(CompactionSSTable sstable)
     {
-        for (Map.Entry<TimeUUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+        for (Map.Entry<TimeUUID, LegacyAbstractCompactionStrategy> entry : strategies.entrySet())
         {
             entry.getValue().removeSSTable(sstable);
             removeSessionIfEmpty(entry.getKey());
         }
     }
 
-
-    void removeSSTables(Iterable<SSTableReader> removed)
+    void removeSSTables(Iterable<CompactionSSTable> removed)
     {
-        for (SSTableReader sstable : removed)
+        for (CompactionSSTable sstable : removed)
             removeSSTable(sstable);
     }
 
-    synchronized void addSSTable(SSTableReader sstable)
+    synchronized void addSSTable(CompactionSSTable sstable)
     {
         Preconditions.checkArgument(sstable.isTransient() == isTransient);
         getOrCreate(sstable).addSSTable(sstable);
     }
 
-    void addSSTables(Iterable<SSTableReader> added)
+    void addSSTables(Iterable<? extends CompactionSSTable> added)
     {
-        for (SSTableReader sstable : added)
+        for (CompactionSSTable sstable : added)
             addSSTable(sstable);
     }
 
-    synchronized void replaceSSTables(Set<SSTableReader> removed, Set<SSTableReader> added)
+    synchronized void replaceSSTables(Set<CompactionSSTable> removed, Set<CompactionSSTable> added)
     {
         if (removed.isEmpty() && added.isEmpty())
             return;
 
         // left=removed, right=added
-        Map<TimeUUID, Pair<Set<SSTableReader>, Set<SSTableReader>>> groups = new HashMap<>();
-        for (SSTableReader sstable : removed)
+        Map<TimeUUID, Pair<Set<CompactionSSTable>, Set<CompactionSSTable>>> groups = new HashMap<>();
+        for (CompactionSSTable sstable : removed)
         {
-            TimeUUID sessionID = sstable.getSSTableMetadata().pendingRepair;
+            TimeUUID sessionID = sstable.getPendingRepair();
             if (!groups.containsKey(sessionID))
             {
                 groups.put(sessionID, Pair.create(new HashSet<>(), new HashSet<>()));
@@ -191,9 +187,9 @@ synchronized void replaceSSTables(Set<SSTableReader> removed, Set<SSTableReader>
             groups.get(sessionID).left.add(sstable);
         }
 
-        for (SSTableReader sstable : added)
+        for (CompactionSSTable sstable : added)
         {
-            TimeUUID sessionID = sstable.getSSTableMetadata().pendingRepair;
+            TimeUUID sessionID = sstable.getPendingRepair();
             if (!groups.containsKey(sessionID))
             {
                 groups.put(sessionID, Pair.create(new HashSet<>(), new HashSet<>()));
@@ -201,11 +197,11 @@ synchronized void replaceSSTables(Set<SSTableReader> removed, Set<SSTableReader>
             groups.get(sessionID).right.add(sstable);
         }
 
-        for (Map.Entry<TimeUUID, Pair<Set<SSTableReader>, Set<SSTableReader>>> entry : groups.entrySet())
+        for (Map.Entry<TimeUUID, Pair<Set<CompactionSSTable>, Set<CompactionSSTable>>> entry : groups.entrySet())
         {
-            AbstractCompactionStrategy strategy = getOrCreate(entry.getKey());
-            Set<SSTableReader> groupRemoved = entry.getValue().left;
-            Set<SSTableReader> groupAdded = entry.getValue().right;
+            LegacyAbstractCompactionStrategy strategy = getOrCreate(entry.getKey());
+            Set<CompactionSSTable> groupRemoved = entry.getValue().left;
+            Set<CompactionSSTable> groupAdded = entry.getValue().right;
 
             if (!groupRemoved.isEmpty())
                 strategy.replaceSSTables(groupRemoved, groupAdded);
@@ -218,12 +214,12 @@ synchronized void replaceSSTables(Set<SSTableReader> removed, Set<SSTableReader>
 
     synchronized void startup()
     {
-        strategies.values().forEach(AbstractCompactionStrategy::startup);
+        strategies.values().forEach(CompactionStrategy::startup);
     }
 
     synchronized void shutdown()
     {
-        strategies.values().forEach(AbstractCompactionStrategy::shutdown);
+        strategies.values().forEach(CompactionStrategy::shutdown);
     }
 
     private int getEstimatedRemainingTasks(TimeUUID sessionID, AbstractCompactionStrategy strategy)
@@ -244,7 +240,7 @@ int getEstimatedRemainingTasks()
     int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes)
     {
         int tasks = 0;
-        for (Map.Entry<TimeUUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+        for (Map.Entry<TimeUUID, LegacyAbstractCompactionStrategy> entry : strategies.entrySet())
         {
             tasks += getEstimatedRemainingTasks(entry.getKey(), entry.getValue(), additionalSSTables, additionalBytes);
         }
@@ -257,7 +253,7 @@ int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes)
     int getMaxEstimatedRemainingTasks()
     {
         int tasks = 0;
-        for (Map.Entry<TimeUUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+        for (Map.Entry<TimeUUID, LegacyAbstractCompactionStrategy> entry : strategies.entrySet())
         {
             tasks = Math.max(tasks, getEstimatedRemainingTasks(entry.getKey(), entry.getValue()));
         }
@@ -267,63 +263,13 @@ int getMaxEstimatedRemainingTasks()
     private RepairFinishedCompactionTask getRepairFinishedCompactionTask(TimeUUID sessionID)
     {
         Preconditions.checkState(canCleanup(sessionID));
-        AbstractCompactionStrategy compactionStrategy = get(sessionID);
+        LegacyAbstractCompactionStrategy compactionStrategy = get(sessionID);
         if (compactionStrategy == null)
             return null;
-        Set<SSTableReader> sstables = compactionStrategy.getSSTables();
+        Set<? extends CompactionSSTable> sstables = compactionStrategy.getSSTables();
         long repairedAt = ActiveRepairService.instance().consistent.local.getFinalSessionRepairedAt(sessionID);
-        LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
-        return txn == null ? null : new RepairFinishedCompactionTask(cfs, txn, sessionID, repairedAt);
-    }
-
-    public static class CleanupTask
-    {
-        private final ColumnFamilyStore cfs;
-        private final List<Pair<TimeUUID, RepairFinishedCompactionTask>> tasks;
-
-        public CleanupTask(ColumnFamilyStore cfs, List<Pair<TimeUUID, RepairFinishedCompactionTask>> tasks)
-        {
-            this.cfs = cfs;
-            this.tasks = tasks;
-        }
-
-        public CleanupSummary cleanup()
-        {
-            Set<TimeUUID> successful = new HashSet<>();
-            Set<TimeUUID> unsuccessful = new HashSet<>();
-            for (Pair<TimeUUID, RepairFinishedCompactionTask> pair : tasks)
-            {
-                TimeUUID session = pair.left;
-                RepairFinishedCompactionTask task = pair.right;
-
-                if (task != null)
-                {
-                    try
-                    {
-                        task.run();
-                        successful.add(session);
-                    }
-                    catch (Throwable t)
-                    {
-                        t = task.transaction.abort(t);
-                        logger.error("Failed cleaning up " + session, t);
-                        unsuccessful.add(session);
-                    }
-                }
-                else
-                {
-                    unsuccessful.add(session);
-                }
-            }
-            return new CleanupSummary(cfs, successful, unsuccessful);
-        }
-
-        public Throwable abort(Throwable accumulate)
-        {
-            for (Pair<TimeUUID, RepairFinishedCompactionTask> pair : tasks)
-                accumulate = pair.right.transaction.abort(accumulate);
-            return accumulate;
-        }
+        LifecycleTransaction txn = realm.tryModify(sstables, OperationType.COMPACTION);
+        return txn == null ? null : new RepairFinishedCompactionTask(realm, txn, sessionID, repairedAt, isTransient);
     }
 
     public CleanupTask releaseSessionData(Collection<TimeUUID> sessionIDs)
@@ -336,7 +282,7 @@ public CleanupTask releaseSessionData(Collection<TimeUUID> sessionIDs)
                 tasks.add(Pair.create(session, getRepairFinishedCompactionTask(session)));
             }
         }
-        return new CleanupTask(cfs, tasks);
+        return new CleanupTask(realm, tasks);
     }
 
     synchronized int getNumPendingRepairFinishedTasks()
@@ -352,26 +298,29 @@ synchronized int getNumPendingRepairFinishedTasks()
         return count;
     }
 
-    synchronized AbstractCompactionTask getNextRepairFinishedTask()
+    synchronized Collection<AbstractCompactionTask> getNextRepairFinishedTasks()
     {
         for (TimeUUID sessionID : strategies.keySet())
         {
             if (canCleanup(sessionID))
             {
-                return getRepairFinishedCompactionTask(sessionID);
+                RepairFinishedCompactionTask task = getRepairFinishedCompactionTask(sessionID);
+                if (task != null)
+                    return ImmutableList.of(task);
+                else
+                    return ImmutableList.of();
             }
         }
-        return null;
+        return ImmutableList.of();
     }
 
-    synchronized AbstractCompactionTask getNextBackgroundTask(long gcBefore)
+    synchronized Collection<AbstractCompactionTask> getNextBackgroundTasks(long gcBefore)
     {
         if (strategies.isEmpty())
-            return null;
-
+            return ImmutableList.of();
         Map<TimeUUID, Integer> numTasks = new HashMap<>(strategies.size());
         ArrayList<TimeUUID> sessions = new ArrayList<>(strategies.size());
-        for (Map.Entry<TimeUUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+        for (Map.Entry<TimeUUID, LegacyAbstractCompactionStrategy> entry : strategies.entrySet())
         {
             if (canCleanup(entry.getKey()))
             {
@@ -382,22 +331,22 @@ synchronized AbstractCompactionTask getNextBackgroundTask(long gcBefore)
         }
 
         if (sessions.isEmpty())
-            return null;
+            return ImmutableList.of();
 
         // we want the session with the most compactions at the head of the list
         sessions.sort((o1, o2) -> numTasks.get(o2) - numTasks.get(o1));
 
         TimeUUID sessionID = sessions.get(0);
-        return get(sessionID).getNextBackgroundTask(gcBefore);
+        return get(sessionID).getNextBackgroundTasks(gcBefore);
     }
 
-    synchronized Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, boolean splitOutput)
+    synchronized Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism)
     {
         if (strategies.isEmpty())
-            return null;
+            return ImmutableList.of();
 
         List<AbstractCompactionTask> maximalTasks = new ArrayList<>(strategies.size());
-        for (Map.Entry<TimeUUID, AbstractCompactionStrategy> entry : strategies.entrySet())
+        for (Map.Entry<TimeUUID, LegacyAbstractCompactionStrategy> entry : strategies.entrySet())
         {
             if (canCleanup(entry.getKey()))
             {
@@ -405,15 +354,13 @@ synchronized Collection<AbstractCompactionTask> getMaximalTasks(long gcBefore, b
             }
             else
             {
-                Collection<AbstractCompactionTask> tasks = entry.getValue().getMaximalTask(gcBefore, splitOutput);
-                if (tasks != null)
-                    maximalTasks.addAll(tasks);
+                maximalTasks.addAll(entry.getValue().getMaximalTasks(gcBefore, splitOutput, permittedParallelism));
             }
         }
-        return !maximalTasks.isEmpty() ? maximalTasks : null;
+        return maximalTasks;
     }
 
-    Collection<AbstractCompactionStrategy> getStrategies()
+    Collection<LegacyAbstractCompactionStrategy> getStrategies()
     {
         return strategies.values();
     }
@@ -438,7 +385,7 @@ synchronized Set<ISSTableScanner> getScanners(Collection<SSTableReader> sstables
         Map<TimeUUID, Set<SSTableReader>> sessionSSTables = new HashMap<>();
         for (SSTableReader sstable : sstables)
         {
-            TimeUUID sessionID = sstable.getSSTableMetadata().pendingRepair;
+            TimeUUID sessionID = sstable.getPendingRepair();
             checkPendingID(sessionID);
             sessionSSTables.computeIfAbsent(sessionID, k -> new HashSet<>()).add(sstable);
         }
@@ -458,7 +405,7 @@ synchronized Set<ISSTableScanner> getScanners(Collection<SSTableReader> sstables
         return scanners;
     }
 
-    public boolean hasStrategy(AbstractCompactionStrategy strategy)
+    public boolean hasStrategy(CompactionStrategy strategy)
     {
         return strategies.values().contains(strategy);
     }
@@ -468,7 +415,7 @@ public synchronized boolean hasDataForSession(TimeUUID sessionID)
         return strategies.containsKey(sessionID);
     }
 
-    boolean containsSSTable(SSTableReader sstable)
+    boolean containsSSTable(CompactionSSTable sstable)
     {
         if (!sstable.isPendingRepair())
             return false;
@@ -477,91 +424,9 @@ boolean containsSSTable(SSTableReader sstable)
         return strategy != null && strategy.getSSTables().contains(sstable);
     }
 
-    public Collection<AbstractCompactionTask> createUserDefinedTasks(Collection<SSTableReader> sstables, long gcBefore)
-    {
-        Map<TimeUUID, List<SSTableReader>> group = sstables.stream().collect(Collectors.groupingBy(s -> s.getSSTableMetadata().pendingRepair));
-        return group.entrySet().stream().map(g -> strategies.get(g.getKey()).getUserDefinedTask(g.getValue(), gcBefore)).collect(Collectors.toList());
-    }
-
-    @VisibleForTesting
-    public synchronized boolean hasPendingRepairSSTable(TimeUUID sessionID, SSTableReader sstable)
-    {
-        AbstractCompactionStrategy strat = strategies.get(sessionID);
-        if (strat == null)
-            return false;
-        return strat.getSSTables().contains(sstable);
-    }
-
-    /**
-     * promotes/demotes sstables involved in a consistent repair that has been finalized, or failed
-     */
-    class RepairFinishedCompactionTask extends AbstractCompactionTask
+    public Collection<AbstractCompactionTask> createUserDefinedTasks(Collection<CompactionSSTable> sstables, long gcBefore)
     {
-        private final TimeUUID sessionID;
-        private final long repairedAt;
-
-        RepairFinishedCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction transaction, TimeUUID sessionID, long repairedAt)
-        {
-            super(cfs, transaction);
-            this.sessionID = sessionID;
-            this.repairedAt = repairedAt;
-        }
-
-        @VisibleForTesting
-        TimeUUID getSessionID()
-        {
-            return sessionID;
-        }
-
-        protected void runMayThrow() throws Exception
-        {
-            boolean completed = false;
-            boolean obsoleteSSTables = isTransient && repairedAt > 0;
-            try
-            {
-                if (obsoleteSSTables)
-                {
-                    logger.info("Obsoleting transient repaired sstables for {}", sessionID);
-                    Preconditions.checkState(Iterables.all(transaction.originals(), SSTableReader::isTransient));
-                    transaction.obsoleteOriginals();
-                }
-                else
-                {
-                    logger.info("Moving {} from pending to repaired with repaired at = {} and session id = {}", transaction.originals(), repairedAt, sessionID);
-                    cfs.getCompactionStrategyManager().mutateRepaired(transaction.originals(), repairedAt, ActiveRepairService.NO_PENDING_REPAIR, false);
-                }
-                completed = true;
-            }
-            finally
-            {
-                if (obsoleteSSTables)
-                {
-                    transaction.finish();
-                }
-                else
-                {
-                    // we abort here because mutating metadata isn't guarded by LifecycleTransaction, so this won't roll
-                    // anything back. Also, we don't want to obsolete the originals. We're only using it to prevent other
-                    // compactions from marking these sstables compacting, and unmarking them when we're done
-                    transaction.abort();
-                }
-                if (completed)
-                {
-                    removeSessionIfEmpty(sessionID);
-                }
-            }
-        }
-
-        public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        protected int executeInternal(ActiveCompactionsTracker activeCompactions)
-        {
-            run();
-            return transaction.originals().size();
-        }
+        Map<TimeUUID, List<CompactionSSTable>> group = sstables.stream().collect(Collectors.groupingBy(s -> s.getPendingRepair()));
+        return group.entrySet().stream().map(g -> strategies.get(g.getKey()).getUserDefinedTasks(g.getValue(), gcBefore)).flatMap(Collection::stream).collect(Collectors.toList());
     }
-
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/RepairFinishedCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/RepairFinishedCompactionTask.java
new file mode 100644
index 000000000000..9e11df81780b
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/RepairFinishedCompactionTask.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * promotes/demotes sstables involved in a consistent repair that has been finalized, or failed
+ */
+public class RepairFinishedCompactionTask extends AbstractCompactionTask
+{
+    private static final Logger logger = LoggerFactory.getLogger(RepairFinishedCompactionTask.class);
+
+    private final TimeUUID sessionID;
+    private final long repairedAt;
+    private final boolean isTransient;
+
+    public RepairFinishedCompactionTask(CompactionRealm realm,
+                                        ILifecycleTransaction transaction,
+                                        TimeUUID sessionID,
+                                        long repairedAt,
+                                        boolean isTransient)
+    {
+        super(realm, transaction);
+        this.sessionID = sessionID;
+        this.repairedAt = repairedAt;
+        this.isTransient = isTransient;
+    }
+
+    @VisibleForTesting
+    TimeUUID getSessionID()
+    {
+        return sessionID;
+    }
+
+    protected void runMayThrow() throws Exception
+    {
+        boolean completed = false;
+        boolean obsoleteSSTables = isTransient && repairedAt > 0;
+        try
+        {
+            if (obsoleteSSTables)
+            {
+                logger.info("Obsoleting transient repaired sstables for {}", sessionID);
+                Preconditions.checkState(Iterables.all(transaction.originals(), SSTableReader::isTransient));
+                transaction.obsoleteOriginals();
+            }
+            else
+            {
+                logger.info("Moving {} from pending to repaired with repaired at = {} for session id = {}", transaction.originals(), repairedAt, sessionID);
+                realm.mutateRepairedWithLock(transaction.originals(),
+                                             repairedAt,
+                                             ActiveRepairService.NO_PENDING_REPAIR,
+                                             false);
+                realm.repairSessionCompleted(sessionID);
+            }
+            completed = true;
+        }
+        finally
+        {
+            if (obsoleteSSTables)
+            {
+                transaction.prepareToCommit();
+                transaction.commit();
+            }
+            else
+            {
+                // we abort here because mutating metadata isn't guarded by LifecycleTransaction, so this won't roll
+                // anything back. Also, we don't want to obsolete the originals. We're only using it to prevent other
+                // compactions from marking these sstables compacting, and unmarking them when we're done
+                transaction.abort();
+            }
+            if (completed)
+            {
+                realm.repairSessionCompleted(sessionID);
+            }
+        }
+    }
+
+    @Override
+    public long getSpaceOverhead()
+    {
+        return 0;   // This is just metadata modification, no overhead.
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java b/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
index 6f68c340f0e5..9cad8f7ddbb2 100644
--- a/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
+++ b/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
@@ -17,36 +17,37 @@
  */
 package org.apache.cassandra.db.compaction;
 
-import java.util.*;
+import java.util.Set;
 import java.util.function.LongPredicate;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
 import org.apache.cassandra.db.compaction.writers.MaxSSTableSizeWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 
 public class SSTableSplitter 
 {
-    private final SplittingCompactionTask task;
+    private final AbstractCompactionTask task;
 
-    public SSTableSplitter(ColumnFamilyStore cfs, LifecycleTransaction transaction, int sstableSizeInMB)
+    public SSTableSplitter(CompactionRealm realm, LifecycleTransaction transaction, int sstableSizeInMB)
     {
-        this.task = new SplittingCompactionTask(cfs, transaction, sstableSizeInMB);
+        this.task = new SplittingCompactionTask(realm, transaction, sstableSizeInMB);
     }
 
     public void split()
     {
-        task.execute(ActiveCompactionsTracker.NOOP);
+        task.execute();
     }
 
-    public static class SplittingCompactionTask extends CompactionTask
+    private static class SplittingCompactionTask extends CompactionTask
     {
         private final int sstableSizeInMiB;
 
-        public SplittingCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction transaction, int sstableSizeInMB)
+        public SplittingCompactionTask(CompactionRealm realm, LifecycleTransaction transaction, int sstableSizeInMB)
         {
-            super(cfs, transaction, CompactionManager.NO_GC, false);
+            super(realm, transaction, CompactionManager.NO_GC, false, null);
             this.sstableSizeInMiB = sstableSizeInMB;
 
             if (sstableSizeInMB <= 0)
@@ -56,16 +57,15 @@ public SplittingCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction trans
         @Override
         protected CompactionController getCompactionController(Set<SSTableReader> toCompact)
         {
-            return new SplitController(cfs);
+            return new SplitController(realm);
         }
 
         @Override
-        public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs,
+        public CompactionAwareWriter getCompactionAwareWriter(CompactionRealm realm,
                                                               Directories directories,
-                                                              LifecycleTransaction txn,
                                                               Set<SSTableReader> nonExpiredSSTables)
         {
-            return new MaxSSTableSizeWriter(cfs, directories, txn, nonExpiredSSTables, sstableSizeInMiB * 1024L * 1024L, 0, false);
+            return new MaxSSTableSizeWriter(realm, directories, transaction, nonExpiredSSTables, sstableSizeInMiB * 1024L * 1024L, 0, false);
         }
 
         @Override
@@ -77,7 +77,7 @@ protected boolean partialCompactionsAcceptable()
 
     public static class SplitController extends CompactionController
     {
-        public SplitController(ColumnFamilyStore cfs)
+        public SplitController(CompactionRealm cfs)
         {
             super(cfs, CompactionManager.NO_GC);
         }
diff --git a/src/java/org/apache/cassandra/db/compaction/ShardManager.java b/src/java/org/apache/cassandra/db/compaction/ShardManager.java
index 6ea2cd72a84c..6754e0df75f5 100644
--- a/src/java/org/apache/cassandra/db/compaction/ShardManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/ShardManager.java
@@ -18,74 +18,94 @@
 
 package org.apache.cassandra.db.compaction;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
 import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.ObjIntConsumer;
 
-import com.google.common.collect.ImmutableList;
-
-import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DiskBoundaries;
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.SortedLocalRanges;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.SortingIterator;
 
 public interface ShardManager
 {
-    /**
-     * Single-partition, and generally sstables with very few partitions, can cover very small sections of the token
-     * space, resulting in very high densities.
-     * Additionally, sstables that have completely fallen outside of the local token ranges will end up with a zero
-     * coverage.
-     * To avoid problems with both we check if coverage is below the minimum, and replace it with 1.
-     */
-    static final double MINIMUM_TOKEN_COVERAGE = Math.scalb(1.0, -48);
+    /// Single-partition, and generally sstables with very few partitions, can cover very small sections of the token
+    /// space, resulting in very high densities.
+    ///
+    /// When the number of partitions in an sstable is smaller than this threshold, we will use a per-partition minimum
+    /// span, calculated from the total number of partitions in this table.
+    long PER_PARTITION_SPAN_THRESHOLD = 100;
+
+    /// Additionally, sstables that have completely fallen outside the local token ranges will end up with a zero
+    /// coverage.
+    ///
+    /// To avoid problems with this we check if coverage is below the minimum, and replace it using the per-partition
+    /// calculation.
+    double MINIMUM_TOKEN_COVERAGE = Math.scalb(1.0, -48);
 
-    static ShardManager create(ColumnFamilyStore cfs)
+    static ShardManager create(DiskBoundaries diskBoundaries, AbstractReplicationStrategy rs, boolean isReplicaAware)
     {
-        final ImmutableList<PartitionPosition> diskPositions = cfs.getDiskBoundaries().positions;
-        ColumnFamilyStore.VersionedLocalRanges localRanges = cfs.localRangesWeighted();
-        IPartitioner partitioner = cfs.getPartitioner();
+        List<Token> diskPositions = diskBoundaries.getPositions();
+
+        SortedLocalRanges localRanges = diskBoundaries.getLocalRanges();
+        IPartitioner partitioner = localRanges.getRealm().getPartitioner();
+        // this should only happen in tests that change partitioners, but we don't want UCS to throw
+        // where other strategies work even if the situations are unrealistic.
+        if (localRanges.getRanges().isEmpty() || !localRanges.getRanges()
+                                                             .get(0)
+                                                             .range()
+                                                             .left
+                                                             .getPartitioner()
+                                                             .equals(localRanges.getRealm().getPartitioner()))
+            localRanges = new SortedLocalRanges(localRanges.getRealm(),
+                                                localRanges.getRingVersion(),
+                                                null);
+
 
         if (diskPositions != null && diskPositions.size() > 1)
-            return new ShardManagerDiskAware(localRanges, diskPositions.stream()
-                                                                       .map(PartitionPosition::getToken)
-                                                                       .collect(Collectors.toList()));
+            return new ShardManagerDiskAware(localRanges, diskPositions);
         else if (partitioner.splitter().isPresent())
-            return new ShardManagerNoDisks(localRanges);
+            if (isReplicaAware)
+                return new ShardManagerReplicaAware(rs, localRanges.getRealm());
+            else
+                return new ShardManagerNoDisks(localRanges);
         else
             return new ShardManagerTrivial(partitioner);
     }
 
-    boolean isOutOfDate(long ringVersion);
-
-    /**
-     * The token range fraction spanned by the given range, adjusted for the local range ownership.
-     */
+    /// The token range fraction spanned by the given range, adjusted for the local range ownership.
     double rangeSpanned(Range<Token> tableRange);
 
-    /**
-     * The total fraction of the token space covered by the local ranges.
-     */
+    /// The total fraction of the token space covered by the local ranges.
     double localSpaceCoverage();
 
-    /**
-     * The fraction of the token space covered by a shard set, i.e. the space that is split in the requested number of
-     * shards.
-     * If no disks are defined, this is the same as localSpaceCoverage(). Otherwise, it is the token coverage of a disk.
-     */
+    /// The fraction of the token space covered by a shard set, i.e. the space that is split in the requested number of
+    /// shards.
+    ///
+    /// If no disks are defined, this is the same as localSpaceCoverage(). Otherwise, it is the token coverage of a disk.
     double shardSetCoverage();
 
-    /**
-     * Construct a boundary/shard iterator for the given number of shards.
-     *
-     * Note: This does not offer a method of listing the shard boundaries it generates, just to advance to the
-     * corresponding one for a given token.  The only usage for listing is currently in tests. Should a need for this
-     * arise, see {@link CompactionSimulationTest} for a possible implementation.
-     */
+    /// The minimum token space share per partition that should be assigned to sstables with small numbers of partitions
+    /// or which have fallen outside the local token ranges.
+    double minimumPerPartitionSpan();
+
+    /// Construct a boundary/shard iterator for the given number of shards.
+    ///
+    /// If a list of the ranges for each shard is required instead, use [#getShardRanges].
     ShardTracker boundaries(int shardCount);
 
-    static Range<Token> coveringRange(SSTableReader sstable)
+    static Range<Token> coveringRange(CompactionSSTable sstable)
     {
         return coveringRange(sstable.getFirst(), sstable.getLast());
     }
@@ -97,26 +117,32 @@ static Range<Token> coveringRange(PartitionPosition first, PartitionPosition las
     }
 
 
-    /**
-     * Return the token space share that the given SSTable spans, excluding any non-locally owned space.
-     * Returns a positive floating-point number between 0 and 1.
-     */
-    default double rangeSpanned(SSTableReader rdr)
+    /// Return the token space share that the given SSTable spans, excluding any non-locally owned space.
+    /// Returns a positive floating-point number between 0 and 1.
+    default double rangeSpanned(CompactionSSTable rdr)
     {
         double reported = rdr.tokenSpaceCoverage();
+
         double span;
         if (reported > 0)   // also false for NaN
             span = reported;
         else
             span = rangeSpanned(rdr.getFirst(), rdr.getLast());
 
-        if (span >= MINIMUM_TOKEN_COVERAGE)
+        long partitionCount = rdr.estimatedKeys();
+        return adjustSmallSpans(span, partitionCount);
+    }
+
+    private double adjustSmallSpans(double span, long partitionCount)
+    {
+        if (partitionCount >= PER_PARTITION_SPAN_THRESHOLD && span >= MINIMUM_TOKEN_COVERAGE)
             return span;
 
-        // Too small ranges are expected to be the result of either a single-partition sstable or falling outside
-        // of the local token ranges. In these cases we substitute it with 1 because for them sharding and density
-        // tiering does not make sense.
-        return 1.0;  // This will be chosen if span is NaN too.
+        // Too small ranges are expected to be the result of either an sstable with a very small number of partitions,
+        // or falling outside the local token ranges. In these cases we apply a per-partition minimum calculated from
+        // the number of partitions in the table.
+        double perPartitionMinimum = Math.min(partitionCount * minimumPerPartitionSpan(), 1.0);
+        return span > perPartitionMinimum ? span : perPartitionMinimum;
     }
 
     default double rangeSpanned(PartitionPosition first, PartitionPosition last)
@@ -124,43 +150,211 @@ default double rangeSpanned(PartitionPosition first, PartitionPosition last)
         return rangeSpanned(ShardManager.coveringRange(first, last));
     }
 
-    /**
-     * Return the density of an SSTable, i.e. its size divided by the covered token space share.
-     * This is an improved measure of the compaction age of an SSTable that grows both with STCS-like full-SSTable
-     * compactions (where size grows, share is constant), LCS-like size-threshold splitting (where size is constant
-     * but share shrinks), UCS-like compactions (where size may grow and covered shards i.e. share may decrease)
-     * and can reproduce levelling structure that corresponds to all, including their mixtures.
-     */
-    default double density(SSTableReader rdr)
+    /// Return the density of an SSTable, i.e. its size divided by the covered token space share.
+    /// This is an improved measure of the compaction age of an SSTable that grows both with STCS-like full-SSTable
+    /// compactions (where size grows, share is constant), LCS-like size-threshold splitting (where size is constant
+    /// but share shrinks), UCS-like compactions (where size may grow and covered shards i.e. share may decrease)
+    /// and can reproduce levelling structure that corresponds to all, including their mixtures.
+    default double density(CompactionSSTable rdr)
     {
         return rdr.onDiskLength() / rangeSpanned(rdr);
     }
 
-    default int compareByDensity(SSTableReader a, SSTableReader b)
+    default double density(long onDiskLength, PartitionPosition min, PartitionPosition max, long approximatePartitionCount)
     {
-        return Double.compare(density(a), density(b));
+        double span = rangeSpanned(min, max);
+        return onDiskLength / adjustSmallSpans(span, approximatePartitionCount);
     }
 
-    /**
-     * Estimate the density of the sstable that will be the result of compacting the given sources.
-     */
-    default double calculateCombinedDensity(Set<? extends SSTableReader> sstables)
+
+    /// Seggregate the given sstables into the shard ranges that intersect sstables from the collection, and call
+    /// the given function on the intersecting sstable set, with access to the shard tracker from which information
+    /// about the shard can be recovered.
+    ///
+    /// If an operationRange is given, this method restricts the collection to the given range and assumes all sstables
+    /// cover at least some portion of that range.
+    private <R extends CompactionSSTable> void assignSSTablesInShards(Collection<R> sstables,
+                                                                      Range<Token> operationRange,
+                                                                      int numShardsForDensity,
+                                                                      BiConsumer<Collection<R>, ShardTracker> consumer)
     {
-        if (sstables.isEmpty())
-            return 0;
-        long onDiskLength = 0;
-        PartitionPosition min = null;
-        PartitionPosition max = null;
-        for (SSTableReader sstable : sstables)
+        var boundaries = boundaries(numShardsForDensity);
+        SortingIterator<R> items = SortingIterator.create(CompactionSSTable.firstKeyComparator, sstables);
+        PriorityQueue<R> active = new PriorityQueue<>(CompactionSSTable.lastKeyComparator);
+        // Advance inside the range. This will add all sstables that start before the end of the covering shard.
+        if (operationRange != null)
+            boundaries.advanceTo(operationRange.left.nextValidToken());
+        while (items.hasNext() || !active.isEmpty())
         {
-            onDiskLength += sstable.onDiskLength();
-            min = min == null || min.compareTo(sstable.getFirst()) > 0 ? sstable.getFirst() : min;
-            max = max == null || max.compareTo(sstable.getLast()) < 0 ? sstable.getLast() : max;
+            if (active.isEmpty())
+            {
+                boundaries.advanceTo(items.peek().getFirst().getToken());
+                active.add(items.next());
+            }
+            Token shardEnd = boundaries.shardEnd();
+            if (operationRange != null &&
+                !operationRange.right.isMinimum() &&
+                shardEnd != null &&
+                shardEnd.compareTo(operationRange.right) >= 0)
+                shardEnd = null;    // Take all remaining sstables.
+
+            while (items.hasNext() && (shardEnd == null || items.peek().getFirst().getToken().compareTo(shardEnd) <= 0))
+                active.add(items.next());
+
+            consumer.accept(active, boundaries);
+
+            while (!active.isEmpty() && (shardEnd == null || active.peek().getLast().getToken().compareTo(shardEnd) <= 0))
+                active.poll();
+
+            if (!active.isEmpty()) // shardEnd must be non-null (otherwise the line above exhausts all)
+                boundaries.advanceTo(shardEnd.nextValidToken());
         }
-        double span = rangeSpanned(min, max);
-        if (span >= MINIMUM_TOKEN_COVERAGE)
-            return onDiskLength / span;
-        else
-            return onDiskLength;
+    }
+
+    /// Seggregate the given sstables into the shard ranges that intersect sstables from the collection, and call
+    /// the given function on the combination of each shard index and the intersecting sstable set.
+    ///
+    /// If an operationRange is given, this method restricts the collection to the given range and assumes all sstables
+    /// cover at least some portion of that range.
+    default <R extends CompactionSSTable> void assignSSTablesToShardIndexes(Collection<R> sstables,
+                                                                            Range<Token> operationRange,
+                                                                            int numShardsForDensity,
+                                                                            ObjIntConsumer<Collection<R>> consumer)
+    {
+        assignSSTablesInShards(sstables, operationRange, numShardsForDensity,
+                               (rangeSSTables, boundaries) -> consumer.accept(rangeSSTables, boundaries.shardIndex()));
+    }
+
+    /// Seggregate the given sstables into the shard ranges that intersect sstables from the collection, and call
+    /// the given function on the combination of each shard range and the intersecting sstable set.
+    default <T, R extends CompactionSSTable> List<T> splitSSTablesInShards(Collection<R> sstables,
+                                                                           int numShardsForDensity,
+                                                                           BiFunction<Collection<R>, Range<Token>, T> maker)
+    {
+        return splitSSTablesInShards(sstables, null, numShardsForDensity, maker);
+    }
+
+    /// Seggregate the given sstables into the shard ranges that intersect sstables from the collection, and call
+    /// the given function on the combination of each shard range and the intersecting sstable set.
+    ///
+    /// This version restricts the operation to the given token range, and assumes all sstables cover at least some
+    /// portion of that range.
+    default <T, R extends CompactionSSTable> List<T> splitSSTablesInShards(Collection<R> sstables,
+                                                                           Range<Token> operationRange,
+                                                                           int numShardsForDensity,
+                                                                           BiFunction<Collection<R>, Range<Token>, T> maker)
+    {
+        List<T> tasks = new ArrayList<>();
+        assignSSTablesInShards(sstables, operationRange, numShardsForDensity, (rangeSSTables, boundaries) -> {
+            final T result = maker.apply(rangeSSTables, boundaries.shardSpan());
+            if (result != null)
+                tasks.add(result);
+        });
+        return tasks;
+    }
+
+    /// Seggregate the given sstables into the shard ranges that intersect sstables from the collection, and call
+    /// the given function on the combination of each shard range and the intersecting sstable set.
+    ///
+    /// This version restricts the operation to the given token range (which may be null) and accepts a parallelism
+    /// limit and will group shards together to fit within that limit.
+    default <T, R extends CompactionSSTable> List<T> splitSSTablesInShardsLimited(Collection<R> sstables,
+                                                                                  Range<Token> operationRange,
+                                                                                  int numShardsForDensity,
+                                                                                  int coveredShards,
+                                                                                  int maxParallelism,
+                                                                                  BiFunction<Collection<R>, Range<Token>, T> maker)
+    {
+        if (coveredShards <= maxParallelism)
+            return splitSSTablesInShards(sstables, operationRange, numShardsForDensity, maker);
+        // We may be in a simple case where we can reduce the number of shards by some power of 2.
+        int multiple = Integer.highestOneBit(coveredShards / maxParallelism);
+        if (maxParallelism * multiple == coveredShards)
+            return splitSSTablesInShards(sstables, operationRange, numShardsForDensity / multiple, maker);
+
+        var shards = splitSSTablesInShards(sstables,
+                                           operationRange,
+                                           numShardsForDensity,
+                                           (rangeSSTables, range) -> Pair.create(Set.copyOf(rangeSSTables), range));
+        return applyMaxParallelism(maxParallelism, maker, shards);
+    }
+
+    private static <T, R extends CompactionSSTable> List<T> applyMaxParallelism(int maxParallelism, BiFunction<Collection<R>, Range<Token>, T> maker, List<Pair<Set<R>, Range<Token>>> shards)
+    {
+        int actualParallelism = shards.size();
+        if (maxParallelism >= actualParallelism)
+        {
+            // We can fit within the parallelism limit without grouping, because some ranges are empty.
+            // This is not expected to happen often, but if it does, take advantage.
+            List<T> tasks = new ArrayList<>();
+            for (Pair<Set<R>, Range<Token>> pair : shards)
+                tasks.add(maker.apply(pair.left, pair.right));
+            return tasks;
+        }
+
+        // Otherwise we have to group shards together. Define a target token span per task and greedily group
+        // to be as close to it as possible.
+        double spanPerTask = shards.stream().map(Pair::right).mapToDouble(t -> t.left.size(t.right)).sum() / maxParallelism;
+        double currentSpan = 0;
+        Set<R> currentSSTables = new HashSet<>();
+        Token rangeStart = null;
+        Token prevEnd = null;
+        List<T> tasks = new ArrayList<>(maxParallelism);
+        for (var pair : shards)
+        {
+            final Token currentEnd = pair.right.right;
+            final Token currentStart = pair.right.left;
+            double span = currentStart.size(currentEnd);
+            if (rangeStart == null)
+                rangeStart = currentStart;
+            if (currentSpan + span >= spanPerTask - 0.001) // rounding error safety
+            {
+                boolean includeCurrent = currentSpan + span - spanPerTask <= spanPerTask - currentSpan;
+                if (includeCurrent)
+                    currentSSTables.addAll(pair.left);
+                tasks.add(maker.apply(currentSSTables, new Range<>(rangeStart, includeCurrent ? currentEnd : prevEnd)));
+                currentSpan -= spanPerTask;
+                rangeStart = null;
+                currentSSTables.clear();
+                if (!includeCurrent)
+                {
+                    currentSSTables.addAll(pair.left);
+                    rangeStart = currentStart;
+                }
+            }
+            else
+                currentSSTables.addAll(pair.left);
+
+            currentSpan += span;
+            prevEnd = currentEnd;
+        }
+        assert currentSSTables.isEmpty();
+        return tasks;
+    }
+
+    /// Return the number of shards that the given range of positions (start- and end-inclusive) spans.
+    default int coveredShardCount(PartitionPosition first, PartitionPosition last, int numShardsForDensity)
+    {
+        var boundaries = boundaries(numShardsForDensity);
+        boundaries.advanceTo(first.getToken());
+        int firstShard = boundaries.shardIndex();
+        boundaries.advanceTo(last.getToken());
+        int lastShard = boundaries.shardIndex();
+        return lastShard - firstShard + 1;
+    }
+
+    /// Get the list of shard ranges for the given shard count. Useful for diagnostics and debugging.
+    default List<Range<Token>> getShardRanges(int shardCount)
+    {
+        var boundaries = boundaries(shardCount);
+        var result = new ArrayList<Range<Token>>(shardCount);
+        while (true)
+        {
+            result.add(boundaries.shardSpan());
+            if (boundaries.shardEnd() == null)
+                break;
+            boundaries.advanceTo(boundaries.shardEnd().nextValidToken());
+        }
+        return result;
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/ShardManagerDiskAware.java b/src/java/org/apache/cassandra/db/compaction/ShardManagerDiskAware.java
index 4f8aba283aba..afbbcc03dcbe 100644
--- a/src/java/org/apache/cassandra/db/compaction/ShardManagerDiskAware.java
+++ b/src/java/org/apache/cassandra/db/compaction/ShardManagerDiskAware.java
@@ -23,8 +23,8 @@
 
 import javax.annotation.Nullable;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.SortedLocalRanges;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
@@ -39,14 +39,14 @@ public class ShardManagerDiskAware extends ShardManagerNoDisks
     private final int[] diskStartRangeIndex;
     private final List<Token> diskBoundaries;
 
-    public ShardManagerDiskAware(ColumnFamilyStore.VersionedLocalRanges localRanges, List<Token> diskBoundaries)
+    public ShardManagerDiskAware(SortedLocalRanges localRanges, List<Token> diskBoundaries)
     {
         super(localRanges);
         assert diskBoundaries != null && !diskBoundaries.isEmpty();
         this.diskBoundaries = diskBoundaries;
 
         double position = 0;
-        final List<Splitter.WeightedRange> ranges = localRanges;
+        final List<Splitter.WeightedRange> ranges = localRanges.getRanges();
         int diskIndex = 0;
         diskBoundaryPositions = new double[diskBoundaries.size()];
         diskStartRangeIndex = new int[diskBoundaryPositions.length];
@@ -110,7 +110,7 @@ public class BoundaryTrackerDiskAware implements ShardTracker
         public BoundaryTrackerDiskAware(int countPerDisk)
         {
             this.countPerDisk = countPerDisk;
-            currentStart = localRanges.get(0).left();
+            currentStart = localRanges.getRanges().get(0).left();
             diskIndex = -1;
         }
 
@@ -133,25 +133,37 @@ private Token getEndToken(double toPos)
                 right = localRangePositions[++currentRange];
             }
 
-            final Range<Token> range = localRanges.get(currentRange).range();
+            final Range<Token> range = localRanges.getRanges().get(currentRange).range();
             return currentStart.getPartitioner().split(range.left, range.right, (toPos - left) / (right - left));
         }
 
         public Token shardStart()
         {
+            ensureInitialized();
             return currentStart;
         }
 
         public Token shardEnd()
         {
+            ensureInitialized();
             return currentEnd;
         }
 
         public Range<Token> shardSpan()
         {
+            ensureInitialized();
             return new Range<>(currentStart, currentEnd != null ? currentEnd : currentStart.minValue());
         }
 
+        private void ensureInitialized()
+        {
+            if (diskIndex < 0)
+            {
+                enterDisk(0);
+                setEndToken();
+            }
+        }
+
         public double shardSpanSize()
         {
             return shardStep;
@@ -204,7 +216,7 @@ private void setEndToken()
 
         public int count()
         {
-            return countPerDisk;
+            return countPerDisk * diskBoundaryPositions.length;
         }
 
         /**
@@ -231,7 +243,7 @@ public double rangeSpanned(PartitionPosition first, PartitionPosition last)
 
         public int shardIndex()
         {
-            return nextShardIndex - 1;
+            return diskIndex * countPerDisk + nextShardIndex - 1;
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/ShardManagerNoDisks.java b/src/java/org/apache/cassandra/db/compaction/ShardManagerNoDisks.java
index 6174612a94aa..0d8d76d8e4c3 100644
--- a/src/java/org/apache/cassandra/db/compaction/ShardManagerNoDisks.java
+++ b/src/java/org/apache/cassandra/db/compaction/ShardManagerNoDisks.java
@@ -22,15 +22,15 @@
 
 import javax.annotation.Nullable;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.SortedLocalRanges;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
 
 public class ShardManagerNoDisks implements ShardManager
 {
-    final ColumnFamilyStore.VersionedLocalRanges localRanges;
+    final SortedLocalRanges localRanges;
 
     /**
      * Ending positions for the local token ranges, in covered token range; in other words, the accumulated share of
@@ -39,11 +39,11 @@ public class ShardManagerNoDisks implements ShardManager
      */
     final double[] localRangePositions;
 
-    public ShardManagerNoDisks(ColumnFamilyStore.VersionedLocalRanges localRanges)
+    public ShardManagerNoDisks(SortedLocalRanges localRanges)
     {
         this.localRanges = localRanges;
         double position = 0;
-        final List<Splitter.WeightedRange> ranges = localRanges;
+        final List<Splitter.WeightedRange> ranges = localRanges.getRanges();
         localRangePositions = new double[ranges.size()];
         for (int i = 0; i < localRangePositions.length; ++i)
         {
@@ -53,12 +53,6 @@ public ShardManagerNoDisks(ColumnFamilyStore.VersionedLocalRanges localRanges)
         }
     }
 
-    public boolean isOutOfDate(long ringVersion)
-    {
-        return ringVersion != localRanges.ringVersion &&
-               localRanges.ringVersion != ColumnFamilyStore.RING_VERSION_IRRELEVANT;
-    }
-
     @Override
     public double rangeSpanned(Range<Token> tableRange)
     {
@@ -69,7 +63,7 @@ public double rangeSpanned(Range<Token> tableRange)
     private double rangeSizeNonWrapping(Range<Token> tableRange)
     {
         double size = 0;
-        for (Splitter.WeightedRange range : localRanges)
+        for (Splitter.WeightedRange range : localRanges.getRanges())
         {
             Range<Token> ix = range.range().intersectionNonWrapping(tableRange); // local and table ranges are non-wrapping
             if (ix == null)
@@ -91,6 +85,11 @@ public double shardSetCoverage()
         return localSpaceCoverage();
     }
 
+    public double minimumPerPartitionSpan()
+    {
+        return localSpaceCoverage() / Math.max(1, localRanges.getRealm().estimatedPartitionCountInSSTables());
+    }
+
     @Override
     public ShardTracker boundaries(int shardCount)
     {
@@ -111,7 +110,7 @@ public BoundaryTracker(int count)
         {
             this.count = count;
             rangeStep = localSpaceCoverage() / count;
-            currentStart = localRanges.get(0).left();
+            currentStart = localRanges.getRanges().get(0).left();
             currentRange = 0;
             nextShardIndex = 1;
             if (nextShardIndex == count)
@@ -130,7 +129,7 @@ private Token getEndToken(double toPos)
                 right = localRangePositions[++currentRange];
             }
 
-            final Range<Token> range = localRanges.get(currentRange).range();
+            final Range<Token> range = localRanges.getRanges().get(currentRange).range();
             return currentStart.getPartitioner().split(range.left, range.right, (toPos - left) / (right - left));
         }
 
diff --git a/src/java/org/apache/cassandra/db/compaction/ShardManagerReplicaAware.java b/src/java/org/apache/cassandra/db/compaction/ShardManagerReplicaAware.java
new file mode 100644
index 000000000000..0bfafb0e28ef
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/ShardManagerReplicaAware.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.dht.tokenallocator.IsolatedTokenAllocator;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.TokenMetadata;
+
+/**
+ * A {@link ShardManager} implementation that aligns UCS and replica shards to limit the amount of sstables that are
+ * partially owned by replicas. It takes an {@link AbstractReplicationStrategy} as input and uses it to determine
+ * current and future replica token boundaries to use as sharding split points to ensure that for current and
+ * future states of the cluster, the generated sstable shard ranges will not span multiple nodes for sufficiently high
+ * levels of compaction.
+ * <p>
+ * If more compaction requires more shards than the already allocated tokens can satisfy, use the
+ * {@link org.apache.cassandra.dht.tokenallocator.TokenAllocator} to allocate more tokens and then use those tokens
+ * as split points. This implementation relies on the fact that token allocation is deterministic after the first
+ * token has been selected.
+ */
+public class ShardManagerReplicaAware implements ShardManager
+{
+    private static final Logger logger = LoggerFactory.getLogger(ShardManagerReplicaAware.class);
+    public static final Token[] EMPTY_TOKENS = new Token[0];
+    private final AbstractReplicationStrategy rs;
+    private final TokenMetadata tokenMetadata;
+    private final IPartitioner partitioner;
+    private final ConcurrentHashMap<Integer, Token[]> splitPointCache;
+    private final CompactionRealm realm;
+
+    public ShardManagerReplicaAware(AbstractReplicationStrategy rs, CompactionRealm realm)
+    {
+        this.rs = rs;
+        // Clone the map to ensure it has a consistent view of the tokenMetadata. UCS creates a new instance of the
+        // ShardManagerTokenAware class when the token metadata changes.
+        this.tokenMetadata = rs.getTokenMetadata().cloneOnlyTokenMap();
+        this.splitPointCache = new ConcurrentHashMap<>();
+        this.partitioner = tokenMetadata.partitioner;
+        this.realm = realm;
+    }
+
+    @Override
+    public double rangeSpanned(Range<Token> tableRange)
+    {
+        return tableRange.left.size(tableRange.right);
+    }
+
+    @Override
+    public double localSpaceCoverage()
+    {
+        // This manager is global, so it owns the whole range.
+        return 1;
+    }
+
+    @Override
+    public double shardSetCoverage()
+    {
+        // For now there are no disks defined, so this is the same as localSpaceCoverage
+        return 1;
+    }
+
+    @Override
+    public double minimumPerPartitionSpan()
+    {
+        return localSpaceCoverage() / Math.max(1, realm.estimatedPartitionCountInSSTables());
+    }
+
+    @Override
+    public ShardTracker boundaries(int shardCount)
+    {
+        try
+        {
+            var splitPoints = splitPointCache.computeIfAbsent(shardCount, this::computeBoundaries);
+            return new SimpleShardTracker(splitPoints);
+        }
+        catch (Throwable t)
+        {
+            logger.error("Error creating shard boundaries", t);
+            throw t;
+        }
+    }
+
+    private Token[] computeBoundaries(int shardCount)
+    {
+        logger.debug("Creating shard boundaries for {} shards", shardCount);
+        // Because sstables do not wrap around, we need shardCount - 1 splits.
+        var splitPointCount = shardCount - 1;
+        if (splitPointCount == 0)
+            return new Token[]{partitioner.getMinimumToken()};
+
+        // Copy array list. The current token allocation logic doesn't consider our copy of tokenMetadata, so
+        // modifying the sorted tokens here won't give us much benefit.
+        var sortedTokensList = new ArrayList<>(tokenMetadata.sortedTokens());
+        if (splitPointCount > sortedTokensList.size())
+        {
+            // Not enough tokens, allocate them.
+            int additionalSplits = splitPointCount - sortedTokensList.size();
+            var newTokens = IsolatedTokenAllocator.allocateTokens(additionalSplits, rs);
+            sortedTokensList.addAll(newTokens);
+            sortedTokensList.sort(Token::compareTo);
+        }
+
+        // Short circuit on equal.
+        if (sortedTokensList.size() == splitPointCount)
+        {
+            var sortedTokens = new Token[shardCount];
+            sortedTokens[0] = partitioner.getMinimumToken();
+            for (int i = 0; i < splitPointCount; i++)
+                sortedTokens[i + 1] = sortedTokensList.get(i);
+            return sortedTokens;
+        }
+
+        var sortedTokens = sortedTokensList.toArray(EMPTY_TOKENS);
+
+        // Get the ideal split points and then map them to their nearest neighbor.
+        var evenSplitPoints = computeUniformSplitPoints(splitPointCount);
+        var nodeAlignedSplitPoints = new Token[shardCount];
+        nodeAlignedSplitPoints[0] = partitioner.getMinimumToken();
+
+        // UCS requires that the splitting points for a given density are also splitting points for
+        // all higher densities, so we pick from among the existing tokens.
+        int pos = 0;
+        for (int i = 0; i < evenSplitPoints.length; i++)
+        {
+            int min = pos;
+            int max = sortedTokens.length - evenSplitPoints.length + i;
+            Token value = evenSplitPoints[i];
+            pos = Arrays.binarySearch(sortedTokens, min, max, value);
+            if (pos < 0)
+                pos = -pos - 1;
+
+            if (pos == min)
+            {
+                // No left neighbor, so choose the right neighbor
+                nodeAlignedSplitPoints[i + 1] = sortedTokens[pos];
+                pos++;
+            }
+            else if (pos == max)
+            {
+                // No right neighbor, so choose the left neighbor
+                // This also means that for all greater indexes we don't have a choice.
+                for (; i < evenSplitPoints.length; ++i)
+                    nodeAlignedSplitPoints[i + 1] = sortedTokens[pos++ - 1];
+            }
+            else
+            {
+                // Check the neighbors
+                Token leftNeighbor = sortedTokens[pos - 1];
+                Token rightNeighbor = sortedTokens[pos];
+
+                // Choose the nearest neighbor. By convention, prefer left if value is midpoint, but don't
+                // choose the same token twice.
+                if (leftNeighbor.size(value) <= value.size(rightNeighbor))
+                {
+                    nodeAlignedSplitPoints[i + 1] = leftNeighbor;
+                    // No need to bump pos because we decremented it to find the right split token.
+                }
+                else
+                {
+                    nodeAlignedSplitPoints[i + 1] = rightNeighbor;
+                    pos++;
+                }
+            }
+        }
+
+        return nodeAlignedSplitPoints;
+    }
+
+
+    private Token[] computeUniformSplitPoints(int splitPointCount)
+    {
+        // Want the shard count here to get the right ratio.
+        var rangeStep = 1.0 / (splitPointCount + 1);
+        var tokens = new Token[splitPointCount];
+        for (int i = 0; i < splitPointCount; i++)
+        {
+            // Multiply the step by the index + 1 to get the ratio to the left of the minimum token.
+            var ratioToLeft = rangeStep * (i + 1);
+            tokens[i] = partitioner.split(partitioner.getMinimumToken(), partitioner.getMaximumToken(), ratioToLeft);
+        }
+        return tokens;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/ShardManagerTrivial.java b/src/java/org/apache/cassandra/db/compaction/ShardManagerTrivial.java
index 407bff4f0d67..0aa37499c2d3 100644
--- a/src/java/org/apache/cassandra/db/compaction/ShardManagerTrivial.java
+++ b/src/java/org/apache/cassandra/db/compaction/ShardManagerTrivial.java
@@ -18,13 +18,15 @@
 
 package org.apache.cassandra.db.compaction;
 
+import java.util.Collection;
+import java.util.List;
 import java.util.Set;
+import java.util.function.BiFunction;
 
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
 
 public class ShardManagerTrivial implements ShardManager
 {
@@ -35,31 +37,30 @@ public ShardManagerTrivial(IPartitioner partitioner)
         this.partitioner = partitioner;
     }
 
-    public boolean isOutOfDate(long ringVersion)
+    @Override
+    public double rangeSpanned(Range<Token> tableRange)
     {
-        // We don't do any routing, always up to date
-        return false;
+        return 1;
     }
 
     @Override
-    public double rangeSpanned(Range<Token> tableRange)
+    public double rangeSpanned(CompactionSSTable rdr)
     {
         return 1;
     }
 
     @Override
-    public double rangeSpanned(SSTableReader rdr)
+    public double density(long onDiskLength, PartitionPosition min, PartitionPosition max, long approximatePartitionCount)
     {
-        return 1;
+        return onDiskLength;
     }
 
     @Override
-    public double calculateCombinedDensity(Set<? extends SSTableReader> sstables)
+    public <T, R extends CompactionSSTable> List<T> splitSSTablesInShards(Collection<R> sstables,
+                                                                          int numShardsForDensity,
+                                                                          BiFunction<Collection<R>, Range<Token>, T> maker)
     {
-        double totalSize = 0;
-        for (SSTableReader sstable : sstables)
-            totalSize += sstable.onDiskLength();
-        return totalSize;
+        return List.of(maker.apply(sstables, new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken())));
     }
 
     @Override
@@ -74,6 +75,11 @@ public double shardSetCoverage()
         return 1;
     }
 
+    public double minimumPerPartitionSpan()
+    {
+        throw new AssertionError(); // rangeSpanned is overridden and does not call this method
+    }
+
     ShardTracker iterator = new ShardTracker()
     {
         @Override
@@ -85,7 +91,7 @@ public Token shardStart()
         @Override
         public Token shardEnd()
         {
-            return partitioner.getMinimumToken();
+            return null;
         }
 
         @Override
@@ -131,10 +137,10 @@ public int shardIndex()
         }
 
         @Override
-        public long shardAdjustedKeyCount(Set<SSTableReader> sstables)
+        public long shardAdjustedKeyCount(Set<? extends CompactionSSTable> sstables)
         {
             long shardAdjustedKeyCount = 0;
-            for (SSTableReader sstable : sstables)
+            for (CompactionSSTable sstable : sstables)
                 shardAdjustedKeyCount += sstable.estimatedKeys();
             return shardAdjustedKeyCount;
         }
diff --git a/src/java/org/apache/cassandra/db/compaction/ShardTracker.java b/src/java/org/apache/cassandra/db/compaction/ShardTracker.java
index 46b20638dbd4..0010bf8a7705 100644
--- a/src/java/org/apache/cassandra/db/compaction/ShardTracker.java
+++ b/src/java/org/apache/cassandra/db/compaction/ShardTracker.java
@@ -24,7 +24,6 @@
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
 
 public interface ShardTracker
@@ -38,12 +37,12 @@ public interface ShardTracker
 
     double shardSpanSize();
 
-    /**
-     * Advance to the given token (e.g. before writing a key). Returns true if this resulted in advancing to a new
-     * shard, and false otherwise.
-     */
+    /// Advance to the given token (e.g. before writing a key). Returns true if this resulted in advancing to a new
+    /// shard, and false otherwise.
     boolean advanceTo(Token nextToken);
 
+    /// Returns the number of shards tracked by this tracker. This is not necessarily the number of shards requested
+    /// when [ShardManager#boundaries] was called, because this requests the per-disk number.
     int count();
 
     /**
@@ -54,13 +53,14 @@ public interface ShardTracker
 
     double rangeSpanned(PartitionPosition first, PartitionPosition last);
 
+    /// The index of the shard this tracker is currently on, between `0` and `count() - 1`.
     int shardIndex();
 
-    default long shardAdjustedKeyCount(Set<SSTableReader> sstables)
+    default long shardAdjustedKeyCount(Set<? extends CompactionSSTable> sstables)
     {
         // Note: computationally non-trivial; can be optimized if we save start/stop shards and size per table.
         long shardAdjustedKeyCount = 0;
-        for (SSTableReader sstable : sstables)
+        for (CompactionSSTable sstable : sstables)
             shardAdjustedKeyCount += sstable.estimatedKeys() * fractionInShard(ShardManager.coveringRange(sstable));
         return shardAdjustedKeyCount;
     }
diff --git a/src/java/org/apache/cassandra/db/compaction/SharedCompactionObserver.java b/src/java/org/apache/cassandra/db/compaction/SharedCompactionObserver.java
new file mode 100644
index 000000000000..01e0e282d059
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/SharedCompactionObserver.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.cassandra.utils.TimeUUID;
+
+/// Utility class to share a compaction observer among multiple compaction tasks and only report start and completion
+/// once when the first task starts and completion when all tasks complete (successfully or not, where the passed
+/// `isSuccess` state is a logical and of the subtasks').
+///
+/// Because subtasks may start in any order, we need to know the number of tasks in advance. This is done by calling
+/// [#registerExpectedSubtask] once per subtask before starting any of them.
+///
+/// This class assumes that all subtasks use the same progress object and the same transaction id, and will verify that
+/// if assertions are enabled.
+public class SharedCompactionObserver implements CompactionObserver
+{
+    private final AtomicInteger toReportOnComplete = new AtomicInteger(0);
+    private final AtomicReference<Throwable> onCompleteException = new AtomicReference(null);
+    private final AtomicReference<CompactionProgress> inProgressReported = new AtomicReference<>(null);
+    private final CompactionObserver observer;
+
+    public SharedCompactionObserver(CompactionObserver observer)
+    {
+        this.observer = observer;
+    }
+
+    public void registerExpectedSubtask()
+    {
+        toReportOnComplete.incrementAndGet();
+        assert inProgressReported.get() == null
+            : "Task started before all subtasks registered for operation " + inProgressReported.get().operationId();
+    }
+
+    @Override
+    public void onInProgress(CompactionProgress progress)
+    {
+        if (inProgressReported.compareAndSet(null, progress))
+            observer.onInProgress(progress);
+        else
+            assert inProgressReported.get() == progress; // progress object must also be shared
+    }
+
+    @Override
+    public void onCompleted(TimeUUID id, Throwable err)
+    {
+        onCompleteException.compareAndSet(null, err); // acts like AND
+        final int remainingToComplete = toReportOnComplete.decrementAndGet();
+        assert inProgressReported.get() != null : "onCompleted called before onInProgress";
+        assert remainingToComplete >= 0 : "onCompleted called without corresponding registerExpectedSubtask";
+        // The individual operation ID given here may be different from the shared ID. Pass on the shared one.
+        if (remainingToComplete == 0)
+            observer.onCompleted(inProgressReported.get().operationId(), onCompleteException.get());
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/SharedCompactionProgress.java b/src/java/org/apache/cassandra/db/compaction/SharedCompactionProgress.java
new file mode 100644
index 000000000000..9316b29e0e43
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/SharedCompactionProgress.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.TimeUUID;
+
+/// Shared compaction progress tracker. This combines the progress tracking of multiple compaction tasks into a single
+/// progress tracker, and of reporting completion of all tasks when all of them complete.
+///
+/// Subtasks may start and add themselves in any order. There may also be periods of time when all started tasks have
+/// completed but there are new ones to still initiate. Because of this all parameters returned by this progress may
+/// increase over time, including the total sizes and sstable lists.
+///
+/// To know how many subtasks to expect, this class's [#registerExpectedSubtask] method must be called once per subtask
+/// before any of them start.
+public class SharedCompactionProgress implements CompactionProgress
+{
+    private final List<CompactionProgress> sources = new CopyOnWriteArrayList<>();
+    private final AtomicInteger toComplete = new AtomicInteger(0);
+    private final AtomicLong totalSize = new AtomicLong(0);
+    private final AtomicLong totalCompressedSize = new AtomicLong(0);
+    private final AtomicLong totalUncompressedSize = new AtomicLong(0);
+    private final TimeUUID operationId;
+    private final TableOperation.Unit unit;
+
+    public SharedCompactionProgress(TimeUUID operationId, OperationType operationType, TableOperation.Unit unit)
+    {
+        this.operationId = operationId;
+        // ignore operationType; TODO: remove the argument
+        this.unit = unit;
+    }
+
+    /// Register a subtask to be expected to run. This must be called once per subtask before any of them start.
+    ///
+    /// @param taskSize The size of the task that its [CompactionProgress#total] will report.
+    public void registerExpectedSubtask(long taskSize, long taskCompressedSize, long taskUncompressedSize)
+    {
+        toComplete.incrementAndGet();
+        totalSize.addAndGet(taskSize);
+        totalCompressedSize.addAndGet(taskCompressedSize);
+        totalUncompressedSize.addAndGet(taskUncompressedSize);
+    }
+
+    public void addSubtask(CompactionProgress progress)
+    {
+        sources.add(progress);
+        assert sources.isEmpty() || progress.operationType() == sources.get(0).operationType();
+        assert progress.unit() == unit;
+    }
+
+    /// Mark a subtask as complete. Returns true if the caller is the last subtask to complete.
+    /// This must be called once per subtask.
+    /// Note that completion is determined by the number of tasks expected to run, not by the set that is currently
+    /// registered/running.
+    /// @param progress The progress of the subtask that is complete (currently unused)
+    public boolean completeSubtask(CompactionProgress progress)
+    {
+        return toComplete.decrementAndGet() == 0;
+    }
+
+    @Nullable
+    @Override
+    public CompactionStrategy strategy()
+    {
+        if (sources.isEmpty())
+            return null;
+        return sources.get(0).strategy();
+    }
+
+
+    @Override
+    public Optional<String> keyspace()
+    {
+        if (sources.isEmpty())
+            return Optional.empty();
+        return sources.get(0).keyspace();
+    }
+
+    @Override
+    public Optional<String> table()
+    {
+        if (sources.isEmpty())
+            return Optional.empty();
+        return sources.get(0).table();
+    }
+
+    @Nullable
+    @Override
+    public TableMetadata metadata()
+    {
+        if (sources.isEmpty())
+            return null;
+        return sources.get(0).metadata();
+    }
+
+    @Override
+    public OperationType operationType()
+    {
+        return sources.isEmpty() ? OperationType.COMPACTION : sources.get(0).operationType();
+    }
+
+    @Override
+    public TimeUUID operationId()
+    {
+        return operationId;
+    }
+
+    @Override
+    public TableOperation.Unit unit()
+    {
+        return unit;
+    }
+
+    @Override
+    public Set<SSTableReader> inSSTables()
+    {
+        Set<SSTableReader> set = new HashSet<>();
+        for (CompactionProgress source : sources)
+            set.addAll(source.inSSTables());
+
+        return set;
+    }
+
+    @Override
+    public Set<SSTableReader> outSSTables()
+    {
+        Set<SSTableReader> set = new HashSet<>();
+        for (CompactionProgress source : sources)
+            set.addAll(source.outSSTables());
+
+        return set;
+    }
+
+    @Override
+    public Set<SSTableReader> sstables()
+    {
+        Set<SSTableReader> set = new HashSet<>();
+        for (CompactionProgress p : sources)
+            set.addAll(p.sstables());
+
+        return set;
+    }
+
+    @Override
+    public long inputDiskSize()
+    {
+        return totalCompressedSize.get();
+    }
+
+    @Override
+    public long inputUncompressedSize()
+    {
+        return totalUncompressedSize.get();
+    }
+
+    @Override
+    public long adjustedInputDiskSize()
+    {
+        long sum = 0L;
+        for (CompactionProgress source : sources)
+            sum += source.adjustedInputDiskSize();
+
+        return sum;
+    }
+
+    @Override
+    public long outputDiskSize()
+    {
+        long sum = 0L;
+        for (CompactionProgress source : sources)
+            sum += source.outputDiskSize();
+
+        return sum;
+    }
+
+    @Override
+    public long uncompressedBytesRead()
+    {
+        long sum = 0L;
+        for (CompactionProgress source : sources)
+            sum += source.uncompressedBytesRead();
+
+        return sum;
+    }
+
+    @Override
+    public long uncompressedBytesRead(int level)
+    {
+        long sum = 0L;
+        for (CompactionProgress source : sources)
+            sum += source.uncompressedBytesRead(level);
+
+        return sum;
+    }
+
+    @Override
+    public long uncompressedBytesWritten()
+    {
+        long sum = 0L;
+        for (CompactionProgress source : sources)
+            sum += source.uncompressedBytesWritten();
+
+        return sum;
+    }
+
+    @Override
+    public long partitionsRead()
+    {
+        long sum = 0L;
+        for (CompactionProgress source : sources)
+            sum += source.partitionsRead();
+
+        return sum;
+    }
+
+    @Override
+    public long rowsRead()
+    {
+        long sum = 0L;
+        for (CompactionProgress source : sources)
+            sum += source.rowsRead();
+
+        return sum;
+    }
+
+    @Override
+    public long completed()
+    {
+        long sum = 0L;
+        for (CompactionProgress source : sources)
+            sum += source.completed();
+
+        return sum;
+    }
+
+    @Override
+    public long total()
+    {
+        return totalSize.get();
+    }
+
+    @Override
+    public long startTimeMillis()
+    {
+        long min = Long.MAX_VALUE;
+        for (CompactionProgress source : sources)
+            min = Math.min(min, source.startTimeMillis());
+
+        return min;
+    }
+
+    @Override
+    public long[] partitionsHistogram()
+    {
+        return mergeHistograms(CompactionProgress::partitionsHistogram);
+    }
+
+    @Override
+    public long[] rowsHistogram()
+    {
+        return mergeHistograms(CompactionProgress::rowsHistogram);
+    }
+
+    private long[] mergeHistograms(Function<CompactionProgress, long[]> retriever)
+    {
+        long[] merged = new long[0];
+        for (CompactionProgress source : sources)
+        {
+            long[] histogram = retriever.apply(source);
+            if (histogram.length > merged.length)
+                merged = Arrays.copyOf(merged, histogram.length);
+            for (int i = 0; i < histogram.length; i++)
+                merged[i] += histogram[i];
+        }
+        return merged;
+    }
+
+    @Override
+    public String toString()
+    {
+        return progressToString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/SharedTableOperation.java b/src/java/org/apache/cassandra/db/compaction/SharedTableOperation.java
new file mode 100644
index 000000000000..a8336b3403ea
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/SharedTableOperation.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.cassandra.utils.NonThrowingCloseable;
+
+/// A [TableOperation] tracking the progress and offering stop control of a composite operation.
+/// This class is used for [UnifiedCompactionStrategy]'s parallelized compactions together with
+/// [SharedCompactionProgress] and [SharedCompactionObserver]. It uses a shared progress to present an integrated view
+/// of the composite operation for a [TableOperationObserver] (e.g [org.apache.cassandra.db.compaction.ActiveOperations]).
+public class SharedTableOperation extends AbstractTableOperation implements TableOperation, TableOperationObserver
+{
+    private final Progress sharedProgress;
+    private NonThrowingCloseable obsCloseable;
+    private final List<TableOperation> components = new CopyOnWriteArrayList<>();
+    private final AtomicBoolean started = new AtomicBoolean(false);
+    private final AtomicInteger toClose = new AtomicInteger(0);
+    private final AtomicReference<TableOperationObserver> observer = new AtomicReference<>(null);
+    private volatile boolean isGlobal;
+
+    public SharedTableOperation(Progress sharedProgress)
+    {
+        this.sharedProgress = sharedProgress;
+    }
+
+    public void registerExpectedSubtask()
+    {
+        toClose.incrementAndGet();
+    }
+
+    @Override
+    public Progress getProgress()
+    {
+        return sharedProgress;
+    }
+
+    @Override
+    public void stop(StopTrigger trigger)
+    {
+        super.stop(trigger);
+        // Stop all ongoing subtasks
+        for (TableOperation component : components)
+            component.stop(trigger);
+        // We will also issue a stop immediately after the start of any operation that is still to initiate in
+        // [onOperationStart].
+    }
+
+    @Override
+    public boolean isGlobal()
+    {
+        return isGlobal;
+    }
+
+    public TableOperationObserver wrapObserver(TableOperationObserver observer)
+    {
+        if (!this.observer.compareAndSet(null, observer))
+            assert this.observer.get() == observer : "All components must use the same observer";
+        // We will register with the observer when one of the components starts.
+
+        // Note: if the observer is Noop, we still want to wrap to complete the shared operation when all subtasks complete.
+        return this;
+    }
+
+    @Override
+    public NonThrowingCloseable onOperationStart(TableOperation operation)
+    {
+        if (started.compareAndSet(false, true))
+        {
+            obsCloseable = observer.get().onOperationStart(this);
+            isGlobal = operation.isGlobal();
+        }
+        // Save the component reference to be able to stop it if needed.
+        components.add(operation);
+
+        if (isStopRequested())
+            operation.stop(trigger());
+        return this::closeOne;
+    }
+
+    private void closeOne()
+    {
+        final int stillToClose = toClose.decrementAndGet();
+        if (stillToClose == 0 && obsCloseable != null)
+            obsCloseable.close();
+        assert stillToClose >= 0 : "Closed more than expected";
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/SimpleShardTracker.java b/src/java/org/apache/cassandra/db/compaction/SimpleShardTracker.java
new file mode 100644
index 000000000000..4833d98b9ac7
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/SimpleShardTracker.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Set;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+
+/**
+ * A shard tracker that uses the provided tokens as a complete list of split points. The first token is typically
+ * the minimum token.
+ */
+class SimpleShardTracker implements ShardTracker
+{
+    private final Token[] sortedTokens;
+    private int index;
+    private Token currentEnd;
+
+    SimpleShardTracker(Token[] sortedTokens)
+    {
+        assert sortedTokens.length > 0;
+        assert sortedTokens[0].isMinimum();
+        this.sortedTokens = sortedTokens;
+        this.index = 0;
+        this.currentEnd = calculateCurrentEnd();
+    }
+
+    @Override
+    public Token shardStart()
+    {
+        return sortedTokens[index];
+    }
+
+    @Nullable
+    @Override
+    public Token shardEnd()
+    {
+        return currentEnd;
+    }
+
+    @Override
+    public Range<Token> shardSpan()
+    {
+        return new Range<>(shardStart(), end());
+    }
+
+    @Override
+    public double shardSpanSize()
+    {
+        // No weight applied because weighting is a local range property.
+        return shardStart().size(end());
+    }
+
+    /**
+     * Non-nullable implementation of {@link ShardTracker#shardEnd()}. Returns the first token if the current shard
+     * is the last shard.
+     * @return the end token of the current shard
+     */
+    private Token end()
+    {
+        Token end = shardEnd();
+        return end != null ? end : sortedTokens[0];
+    }
+
+    private Token calculateCurrentEnd()
+    {
+        return index + 1 < sortedTokens.length ? sortedTokens[index + 1] : null;
+    }
+
+    @Override
+    public boolean advanceTo(Token nextToken)
+    {
+        if (currentEnd == null || nextToken.compareTo(currentEnd) <= 0)
+            return false;
+        do
+        {
+            index++;
+            currentEnd = calculateCurrentEnd();
+            if (currentEnd == null)
+                break;
+        }
+        while (nextToken.compareTo(currentEnd) > 0);
+        return true;
+    }
+
+    @Override
+    public int count()
+    {
+        return sortedTokens.length;
+    }
+
+    @Override
+    public double fractionInShard(Range<Token> targetSpan)
+    {
+        Range<Token> shardSpan = shardSpan();
+        Range<Token> covered = targetSpan.intersectionNonWrapping(shardSpan);
+        if (covered == null)
+            return 0;
+        if (covered == targetSpan)
+            return 1;
+        double inShardSize = covered.left.size(covered.right);
+        double totalSize = targetSpan.left.size(targetSpan.right);
+        return inShardSize / totalSize;
+    }
+
+    @Override
+    public double rangeSpanned(PartitionPosition first, PartitionPosition last)
+    {
+        // Ignore local range owndership for initial implementation.
+        return first.getToken().size(last.getToken());
+    }
+
+    @Override
+    public int shardIndex()
+    {
+        return index;
+    }
+
+    @Override
+    public long shardAdjustedKeyCount(Set<? extends CompactionSSTable> sstables)
+    {
+        // Not sure if this needs a custom implementation yet
+        return ShardTracker.super.shardAdjustedKeyCount(sstables);
+    }
+
+    @Override
+    public void applyTokenSpaceCoverage(SSTableWriter writer)
+    {
+        // Not sure if this needs a custom implementation yet
+        ShardTracker.super.applyTokenSpaceCoverage(writer);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java b/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java
index 1f73c4cd30de..34cd41488397 100644
--- a/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java
@@ -18,15 +18,10 @@
 
 package org.apache.cassandra.db.compaction;
 
-import java.util.Set;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -42,25 +37,20 @@ public class SingleSSTableLCSTask extends AbstractCompactionTask
     private static final Logger logger = LoggerFactory.getLogger(SingleSSTableLCSTask.class);
 
     private final int level;
+    private final LeveledCompactionStrategy strategy;
 
-    public SingleSSTableLCSTask(ColumnFamilyStore cfs, LifecycleTransaction txn, int level)
+    public SingleSSTableLCSTask(LeveledCompactionStrategy strategy, ILifecycleTransaction txn, int level)
     {
-        super(cfs, txn);
+        super(strategy.realm, txn);
+        this.strategy = strategy;
         assert txn.originals().size() == 1;
         this.level = level;
+        addObserver(strategy);
     }
 
-    @Override
-    public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
-    {
-        throw new UnsupportedOperationException("This method should never be called on SingleSSTableLCSTask");
-    }
-
-    @Override
-    protected int executeInternal(ActiveCompactionsTracker activeCompactions)
+    int getLevel()
     {
-        run();
-        return 1;
+        return level;
     }
 
     @Override
@@ -84,7 +74,7 @@ protected void runMayThrow()
                 transaction.abort();
                 throw new CorruptSSTableException(t, sstable.descriptor.fileFor(Components.DATA));
             }
-            cfs.getTracker().notifySSTableMetadataChanged(sstable, metadataBefore);
+            strategy.metadataChanged(metadataBefore, sstable);
         }
         finishTransaction(sstable);
     }
@@ -97,4 +87,10 @@ private void finishTransaction(SSTableReader sstable)
         transaction.prepareToCommit();
         transaction.commit();
     }
+
+    @Override
+    public long getSpaceOverhead()
+    {
+        return 0;   // This is just metadata modification, no overhead.
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStatistics.java b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStatistics.java
new file mode 100644
index 000000000000..e5592ed48c9b
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStatistics.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * The statistics for size tiered compaction.
+ * <p/>
+ * Implements serializable to allow structured info to be returned via JMX.
+ */
+public class SizeTieredCompactionStatistics extends TieredCompactionStatistics
+{
+    /** The average sstable size in this tier */
+    private final long avgSSTableSize;
+
+    SizeTieredCompactionStatistics(CompactionAggregateStatistics base, long avgSSTableSize)
+    {
+        super(base);
+        this.avgSSTableSize = avgSSTableSize;
+    }
+
+    /** The average sstable size in this tier */
+    public long avgSSTableSize()
+    {
+        return avgSSTableSize;
+    }
+
+    @Override
+    @JsonProperty("Bucket")
+    protected String tierValue()
+    {
+        return toString(avgSSTableSize);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
index 74a96ca211ab..1d7dba6b6ac6 100644
--- a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
@@ -17,16 +17,27 @@
  */
 package org.apache.cassandra.db.compaction;
 
-import java.util.*;
-import java.util.Map.Entry;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.NotThreadSafe;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
 import org.apache.cassandra.db.compaction.writers.SplittingSizeTieredCompactionWriter;
@@ -36,272 +47,301 @@
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.utils.Pair;
 
-import static com.google.common.collect.Iterables.filter;
-
-public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy
+public class SizeTieredCompactionStrategy extends LegacyAbstractCompactionStrategy.WithAggregates
 {
     private static final Logger logger = LoggerFactory.getLogger(SizeTieredCompactionStrategy.class);
 
-    private static final Comparator<Pair<List<SSTableReader>,Double>> bucketsByHotnessComparator = new Comparator<Pair<List<SSTableReader>, Double>>()
-    {
-        public int compare(Pair<List<SSTableReader>, Double> o1, Pair<List<SSTableReader>, Double> o2)
-        {
-            int comparison = Double.compare(o1.right, o2.right);
-            if (comparison != 0)
-                return comparison;
-
-            // break ties by compacting the smallest sstables first (this will probably only happen for
-            // system tables and new/unread sstables)
-            return Long.compare(avgSize(o1.left), avgSize(o2.left));
-        }
-
-        private long avgSize(List<SSTableReader> sstables)
-        {
-            long n = 0;
-            for (SSTableReader sstable : sstables)
-                n += sstable.bytesOnDisk();
-            return n / sstables.size();
-        }
-    };
+    /**
+     * Compare {@link CompactionPick} instances by hotness first and in case of a tie by sstable size by
+     * selecting the largest first (a tie would happen for system tables and new/unread sstables).
+     * <p/>
+     * Note that in previous version there is a comment saying "break ties by compacting the smallest sstables first"
+     * but the code was doing the opposite. I preserved the behavior and fixed the comment.
+     */
+    private static final Comparator<CompactionPick> comparePicksByHotness = Comparator.comparing(CompactionPick::hotness)
+                                                                                      .thenComparing(CompactionPick::avgSizeInBytes);
 
     protected SizeTieredCompactionStrategyOptions sizeTieredOptions;
-    protected volatile int estimatedRemainingTasks;
     @VisibleForTesting
-    protected final Set<SSTableReader> sstables = new HashSet<>();
+    protected final Set<CompactionSSTable> sstables = new HashSet<>();
 
-    public SizeTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
+    public SizeTieredCompactionStrategy(CompactionStrategyFactory factory, Map<String, String> options)
     {
-        super(cfs, options);
-        this.estimatedRemainingTasks = 0;
+        super(factory, options);
         this.sizeTieredOptions = new SizeTieredCompactionStrategyOptions(options);
     }
 
-    private synchronized List<SSTableReader> getNextBackgroundSSTables(final long gcBefore)
+    @Override
+    protected synchronized CompactionAggregate getNextBackgroundAggregate(final long gcBefore)
     {
         // make local copies so they can't be changed out from under us mid-method
-        int minThreshold = cfs.getMinimumCompactionThreshold();
-        int maxThreshold = cfs.getMaximumCompactionThreshold();
+        int minThreshold = realm.getMinimumCompactionThreshold();
+        int maxThreshold = realm.getMaximumCompactionThreshold();
+
+        List<CompactionSSTable> candidates = new ArrayList<>();
+        synchronized (sstables)
+        {
+            Iterables.addAll(candidates, nonSuspectAndNotIn(sstables, realm.getCompactingSSTables()));
+        }
 
-        Iterable<SSTableReader> candidates = filterSuspectSSTables(filter(cfs.getUncompactingSSTables(), sstables::contains));
+        SizeTieredBuckets sizeTieredBuckets = new SizeTieredBuckets(candidates, sizeTieredOptions, minThreshold, maxThreshold);
+        sizeTieredBuckets.aggregate();
 
-        List<List<SSTableReader>> buckets = getBuckets(createSSTableAndLengthPairs(candidates), sizeTieredOptions.bucketHigh, sizeTieredOptions.bucketLow, sizeTieredOptions.minSSTableSize);
-        logger.trace("Compaction buckets are {}", buckets);
-        estimatedRemainingTasks = getEstimatedCompactionsByTasks(cfs, buckets);
-        cfs.getCompactionStrategyManager().compactionLogger.pending(this, estimatedRemainingTasks);
-        List<SSTableReader> mostInteresting = mostInterestingBucket(buckets, minThreshold, maxThreshold);
-        if (!mostInteresting.isEmpty())
-            return mostInteresting;
+        backgroundCompactions.setPending(this, sizeTieredBuckets.getAggregates());
+
+        CompactionAggregate ret = sizeTieredBuckets.getAggregates().isEmpty() ? null : sizeTieredBuckets.getAggregates().get(0);
 
         // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
         // ratio is greater than threshold.
-        List<SSTableReader> sstablesWithTombstones = new ArrayList<>();
-        for (SSTableReader sstable : candidates)
-        {
-            if (worthDroppingTombstones(sstable, gcBefore))
-                sstablesWithTombstones.add(sstable);
-        }
-        if (sstablesWithTombstones.isEmpty())
-            return Collections.emptyList();
+        if (ret == null || ret.isEmpty())
+            ret = makeTombstoneCompaction(gcBefore, candidates, list -> Collections.max(list, CompactionSSTable.sizeComparator));
 
-        return Collections.singletonList(Collections.max(sstablesWithTombstones, SSTableReader.sizeComparator));
+        return ret;
     }
 
-
     /**
-     * @param buckets list of buckets from which to return the most interesting, where "interesting" is the total hotness for reads
-     * @param minThreshold minimum number of sstables in a bucket to qualify as interesting
-     * @param maxThreshold maximum number of sstables to compact at once (the returned bucket will be trimmed down to this)
-     * @return a bucket (list) of sstables to compact
+     * This class contains the logic for {@link SizeTieredCompactionStrategy}:
+     *
+     * - sorts the sstables by length on disk
+     * - it sorts the candidates into buckets
+     * - takes a snapshot of the sstable hotness
+     * - it organizes the buckets into a list of {@link CompactionAggregate}, an aggregate per bucket.
+     *   An aggregate will have a list of compaction picks, each pick is a list of sstables below the max threshold,
+     *   sorted by hotness.
+     * - the aggregates are sorted by comparing the total hotness of the first pick of each aggregate
+     * - the aggregate with the hottest first pick will have its first pick submitted for compaction.
      */
-    public static List<SSTableReader> mostInterestingBucket(List<List<SSTableReader>> buckets, int minThreshold, int maxThreshold)
+    @NotThreadSafe
+    final static class SizeTieredBuckets
     {
-        // skip buckets containing less than minThreshold sstables, and limit other buckets to maxThreshold sstables
-        final List<Pair<List<SSTableReader>, Double>> prunedBucketsAndHotness = new ArrayList<>(buckets.size());
-        for (List<SSTableReader> bucket : buckets)
+        private final SizeTieredCompactionStrategyOptions options;
+        private final List<CompactionSSTable> tablesBySize;
+        private final Map<Long, List<CompactionSSTable>> buckets;
+        private final Map<CompactionSSTable, Double> hotnessSnapshot;
+        private final int minThreshold;
+        private final int maxThreshold;
+
+        /**
+         * This is the list of compactions order by most interesting first
+         */
+        private List<CompactionAggregate> aggregates;
+
+        /**
+         * @param candidates   list sstables that are not yet compacting
+         * @param options      the options for size tiered compaction strategy
+         * @param minThreshold minimum number of sstables in a bucket to qualify as interesting
+         * @param maxThreshold maximum number of sstables to compact at once (the returned bucket will be trimmed down to this)
+         */
+        SizeTieredBuckets(Iterable<? extends CompactionSSTable> candidates,
+                          SizeTieredCompactionStrategyOptions options,
+                          int minThreshold,
+                          int maxThreshold)
         {
-            Pair<List<SSTableReader>, Double> bucketAndHotness = trimToThresholdWithHotness(bucket, maxThreshold);
-            if (bucketAndHotness != null && bucketAndHotness.left.size() >= minThreshold)
-                prunedBucketsAndHotness.add(bucketAndHotness);
+            this.options = options;
+            this.tablesBySize = new ArrayList<>();
+            Iterables.addAll(this.tablesBySize, candidates);
+            this.tablesBySize.sort(CompactionSSTable.sizeComparator);
+            this.buckets = getBuckets(tablesBySize, options);
+            this.hotnessSnapshot = getHotnessSnapshot(buckets.values());
+            this.minThreshold = minThreshold;
+            this.maxThreshold = maxThreshold;
+
+            this.aggregates = new ArrayList<>(buckets.size());
+
+            if (logger.isTraceEnabled())
+                logger.trace("Compaction buckets are {}", buckets);
         }
-        if (prunedBucketsAndHotness.isEmpty())
-            return Collections.emptyList();
 
-        Pair<List<SSTableReader>, Double> hottest = Collections.max(prunedBucketsAndHotness, bucketsByHotnessComparator);
-        return hottest.left;
-    }
-
-    /**
-     * Returns a (bucket, hotness) pair or null if there were not enough sstables in the bucket to meet minThreshold.
-     * If there are more than maxThreshold sstables, the coldest sstables will be trimmed to meet the threshold.
-     **/
-    @VisibleForTesting
-    static Pair<List<SSTableReader>, Double> trimToThresholdWithHotness(List<SSTableReader> bucket, int maxThreshold)
-    {
-        // Sort by sstable hotness (descending). We first build a map because the hotness may change during the sort.
-        final Map<SSTableReader, Double> hotnessSnapshot = getHotnessMap(bucket);
-        Collections.sort(bucket, new Comparator<SSTableReader>()
+        /**
+         * Group sstables of similar on disk size into buckets.
+         * The given set must be sorted using CompactionSSTable.sizeComparator
+         */
+        private static Map<Long, List<CompactionSSTable>> getBuckets(List<CompactionSSTable> sstables, SizeTieredCompactionStrategyOptions options)
         {
-            public int compare(SSTableReader o1, SSTableReader o2)
+            if (sstables.isEmpty())
+                return Collections.EMPTY_MAP;
+
+            Map<Long, List<CompactionSSTable>> buckets = new HashMap<>();
+
+            long currentAverageSize = 0;
+            List<CompactionSSTable> currentBucket = new ArrayList<>();
+
+            for (CompactionSSTable sstable: sstables)
             {
-                return -1 * Double.compare(hotnessSnapshot.get(o1), hotnessSnapshot.get(o2));
+                long size = sstable.onDiskLength();
+                assert size >= currentAverageSize;
+
+                if (size >= currentAverageSize * options.bucketHigh
+                    && size >= options.minSSTableSize
+                    && currentAverageSize > 0)   // false for first table only
+                {
+                    // Switch to new bucket
+                    buckets.put(currentAverageSize, currentBucket);
+                    currentBucket = new ArrayList<>();
+                }
+                // TODO: Is it okay that the bucket max can grow unboundedly?
+
+                currentAverageSize = (currentAverageSize * currentBucket.size() + size) / (currentBucket.size() + 1);
+                currentBucket.add(sstable);
             }
-        });
 
-        // and then trim the coldest sstables off the end to meet the maxThreshold
-        List<SSTableReader> prunedBucket = bucket.subList(0, Math.min(bucket.size(), maxThreshold));
+            buckets.put(currentAverageSize, currentBucket);
+            return buckets;
+        }
 
-        // bucket hotness is the sum of the hotness of all sstable members
-        double bucketHotness = 0.0;
-        for (SSTableReader sstr : prunedBucket)
-            bucketHotness += hotness(sstr);
+        /**
+         * For each bucket with at least minThreshold sstables:
+         * <p>
+         * - sort the sstables by hotness
+         * - divide the bucket into max threshold sstables and add it to a temporary list of candidates along with the total hotness of the bucket section
+         * <p>
+         * Then select the candidate with the max hotness and the most interesting bucket and put the remaining candidates in the pending list.
+         *
+         * @return the parent object {@link SizeTieredBuckets}
+         */
+        SizeTieredBuckets aggregate()
+        {
+            if (!aggregates.isEmpty())
+                return this; // already called
 
-        return Pair.create(prunedBucket, bucketHotness);
-    }
+            List<CompactionAggregate> aggregatesWithoutCompactions = new ArrayList<>(buckets.size());
+            List<CompactionAggregate> aggregatesWithCompactions = new ArrayList<>(buckets.size());
 
-    private static Map<SSTableReader, Double> getHotnessMap(Collection<SSTableReader> sstables)
-    {
-        Map<SSTableReader, Double> hotness = new HashMap<>(sstables.size());
-        for (SSTableReader sstable : sstables)
-            hotness.put(sstable, hotness(sstable));
-        return hotness;
-    }
+            for (Map.Entry<Long, List<CompactionSSTable>> entry : buckets.entrySet())
+            {
+                long avgSizeBytes = entry.getKey();
+                long minSizeBytes = (long) (avgSizeBytes * options.bucketLow);
+                long maxSizeBytes = (long) (avgSizeBytes * options.bucketHigh);
 
-    /**
-     * Returns the reads per second per key for this sstable, or 0.0 if the sstable has no read meter
-     */
-    private static double hotness(SSTableReader sstr)
-    {
-        // system tables don't have read meters, just use 0.0 for the hotness
-        return sstr.getReadMeter() == null ? 0.0 : sstr.getReadMeter().twoHourRate() / sstr.estimatedKeys();
-    }
+                List<CompactionSSTable> bucket = entry.getValue();
+                double hotness = totHotness(bucket, hotnessSnapshot);
+
+                if (bucket.size() < minThreshold)
+                {
+                    if (logger.isTraceEnabled())
+                        logger.trace("Aggregate with {} avg bytes for {} files not considered for compaction: {}", avgSizeBytes, bucket.size(), bucket);
+
+                    aggregatesWithoutCompactions.add(CompactionAggregate.createSizeTiered(bucket,
+                                                                                          CompactionPick.EMPTY,
+                                                                                          ImmutableList.of(),
+                                                                                          hotness,
+                                                                                          avgSizeBytes,
+                                                                                          minSizeBytes,
+                                                                                          maxSizeBytes));
+
+                    continue;
+                }
+
+                // sort the bucket by hotness
+                Collections.sort(bucket, (o1, o2) -> -1 * Double.compare(hotnessSnapshot.get(o1), hotnessSnapshot.get(o2)));
+
+                // now divide the candidates into a list of picks, each pick with at most max threshold sstables
+                int i = 0;
+                CompactionPick selected = null;
+                List<CompactionPick> pending = new ArrayList<>();
 
-    public AbstractCompactionTask getNextBackgroundTask(long gcBefore)
-    {
-        List<SSTableReader> previousCandidate = null;
-        while (true)
-        {
-            List<SSTableReader> hottestBucket = getNextBackgroundSSTables(gcBefore);
 
-            if (hottestBucket.isEmpty())
-                return null;
+                while ((bucket.size() - i) >= minThreshold)
+                {
+                    List<CompactionSSTable> sstables = bucket.subList(i, i + Math.min(bucket.size() - i, maxThreshold));
+                    if (selected == null)
+                        selected = CompactionPick.create(avgSizeBytes, sstables, totHotness(sstables, hotnessSnapshot));
+                    else
+                        pending.add(CompactionPick.create(avgSizeBytes, sstables, totHotness(sstables, hotnessSnapshot)));
+
+                    i += sstables.size();
+                }
+
+                if (logger.isTraceEnabled())
+                    logger.trace("Aggregate with {} avg bytes for {} files considered for compaction: {}", avgSizeBytes, bucket.size(), bucket);
+
+                // Finally create the new aggregate with the new pending compactions and those already compacting and not yet completed
+                aggregatesWithCompactions.add(CompactionAggregate.createSizeTiered(bucket, selected, pending, hotness, avgSizeBytes, minSizeBytes, maxSizeBytes));
+            }
 
-            // Already tried acquiring references without success. It means there is a race with
-            // the tracker but candidate SSTables were not yet replaced in the compaction strategy manager
-            if (hottestBucket.equals(previousCandidate))
+            // This sorts the aggregates based on the hotness of their selected pick so that the aggregate with the hottest selected pick
+            // be first in the list and get submitted
+            if (!aggregatesWithCompactions.isEmpty())
             {
-                logger.warn("Could not acquire references for compacting SSTables {} which is not a problem per se," +
-                            "unless it happens frequently, in which case it must be reported. Will retry later.",
-                            hottestBucket);
-                return null;
+                Collections.sort(aggregatesWithCompactions, (a1, a2) -> comparePicksByHotness.compare(a2.getSelected(), a1.getSelected()));
+
+                if (logger.isTraceEnabled())
+                    logger.trace("Found compaction for aggregate {}", aggregatesWithCompactions.get(0));
+            }
+            else
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("No compactions found");
             }
 
-            LifecycleTransaction transaction = cfs.getTracker().tryModify(hottestBucket, OperationType.COMPACTION);
-            if (transaction != null)
-                return new CompactionTask(cfs, transaction, gcBefore);
-            previousCandidate = hottestBucket;
+            // publish the results
+            this.aggregates.addAll(aggregatesWithCompactions); // those with compactions first, because the first one will be the one submitted
+            this.aggregates.addAll(aggregatesWithoutCompactions); // then add those empty
+            return this;
         }
-    }
 
-    public synchronized Collection<AbstractCompactionTask> getMaximalTask(final long gcBefore, boolean splitOutput)
-    {
-        Iterable<SSTableReader> filteredSSTables = filterSuspectSSTables(sstables);
-        if (Iterables.isEmpty(filteredSSTables))
-            return null;
-        LifecycleTransaction txn = cfs.getTracker().tryModify(filteredSSTables, OperationType.COMPACTION);
-        if (txn == null)
-            return null;
-        if (splitOutput)
-            return Arrays.<AbstractCompactionTask>asList(new SplittingCompactionTask(cfs, txn, gcBefore));
-        return Arrays.<AbstractCompactionTask>asList(new CompactionTask(cfs, txn, gcBefore));
-    }
+        /**
+         * For diagnostics only. Returns the sorted tables paired with their on-disk length.
+         */
+        public Collection<Pair<CompactionSSTable, Long>> pairs()
+        {
+            return Collections2.transform(tablesBySize, (CompactionSSTable table) -> Pair.create(table, table.onDiskLength()));
+        }
 
-    public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, final long gcBefore)
-    {
-        assert !sstables.isEmpty(); // checked for by CM.submitUserDefined
+        public List<List<CompactionSSTable>> buckets()
+        {
+            return new ArrayList<>(buckets.values());
+        }
 
-        LifecycleTransaction transaction = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
-        if (transaction == null)
+        public List<CompactionAggregate> getAggregates()
         {
-            logger.trace("Unable to mark {} for compaction; probably a background compaction got to it first.  You can disable background compactions temporarily if this is a problem", sstables);
-            return null;
+            return aggregates;
         }
 
-        return new CompactionTask(cfs, transaction, gcBefore).setUserDefined(true);
+        public List<CompactionPick> getCompactions()
+        {
+            return aggregates.stream().flatMap(aggr -> aggr.getActive().stream()).collect(Collectors.toList());
+        }
     }
 
-    public int getEstimatedRemainingTasks()
+    /**
+     * @return a snapshot mapping sstables to their current read hotness.
+     */
+    @VisibleForTesting
+    static Map<CompactionSSTable, Double> getHotnessSnapshot(Collection<List<CompactionSSTable>> buckets)
     {
-        return estimatedRemainingTasks;
-    }
+        Map<CompactionSSTable, Double> ret = new HashMap<>();
 
-    public static List<Pair<SSTableReader, Long>> createSSTableAndLengthPairs(Iterable<SSTableReader> sstables)
-    {
-        List<Pair<SSTableReader, Long>> sstableLengthPairs = new ArrayList<>(Iterables.size(sstables));
-        for(SSTableReader sstable : sstables)
-            sstableLengthPairs.add(Pair.create(sstable, sstable.onDiskLength()));
-        return sstableLengthPairs;
+        for (List<CompactionSSTable> sstables: buckets)
+        {
+            for (CompactionSSTable sstable : sstables)
+                ret.put(sstable, sstable.hotness());
+        }
+
+        return ret;
     }
 
-    /*
-     * Group files of similar size into buckets.
+    /**
+     * @return the sum of the hotness of all the sstables
      */
-    public static <T> List<List<T>> getBuckets(Collection<Pair<T, Long>> files, double bucketHigh, double bucketLow, long minSSTableSize)
+    private static double totHotness(Iterable<CompactionSSTable> sstables, @Nullable final Map<CompactionSSTable, Double> hotnessSnapshot)
     {
-        // Sort the list in order to get deterministic results during the grouping below
-        List<Pair<T, Long>> sortedFiles = new ArrayList<Pair<T, Long>>(files);
-        Collections.sort(sortedFiles, new Comparator<Pair<T, Long>>()
-        {
-            public int compare(Pair<T, Long> p1, Pair<T, Long> p2)
-            {
-                return p1.right.compareTo(p2.right);
-            }
-        });
-
-        Map<Long, List<T>> buckets = new HashMap<Long, List<T>>();
-
-        outer:
-        for (Pair<T, Long> pair: sortedFiles)
+        double hotness = 0.0;
+        for (CompactionSSTable sstable : sstables)
         {
-            long size = pair.right;
-
-            // look for a bucket containing similar-sized files:
-            // group in the same bucket if it's w/in 50% of the average for this bucket,
-            // or this file and the bucket are all considered "small" (less than `minSSTableSize`)
-            for (Entry<Long, List<T>> entry : buckets.entrySet())
-            {
-                List<T> bucket = entry.getValue();
-                long oldAverageSize = entry.getKey();
-                if ((size > (oldAverageSize * bucketLow) && size < (oldAverageSize * bucketHigh))
-                    || (size < minSSTableSize && oldAverageSize < minSSTableSize))
-                {
-                    // remove and re-add under new new average size
-                    buckets.remove(oldAverageSize);
-                    long totalSize = bucket.size() * oldAverageSize;
-                    long newAverageSize = (totalSize + size) / (bucket.size() + 1);
-                    bucket.add(pair.left);
-                    buckets.put(newAverageSize, bucket);
-                    continue outer;
-                }
-            }
-
-            // no similar bucket found; put it in a new one
-            ArrayList<T> bucket = new ArrayList<T>();
-            bucket.add(pair.left);
-            buckets.put(size, bucket);
+            double h = hotnessSnapshot == null ? 0.0 : hotnessSnapshot.getOrDefault(sstable, 0.0);
+            hotness += h == 0.0  ? sstable.hotness() : h;
         }
 
-        return new ArrayList<List<T>>(buckets.values());
+        return hotness;
     }
 
-    public static int getEstimatedCompactionsByTasks(ColumnFamilyStore cfs, List<List<SSTableReader>> tasks)
+    @Override
+    protected AbstractCompactionTask createCompactionTask(final long gcBefore, LifecycleTransaction txn, boolean isMaximal, boolean splitOutput)
     {
-        int n = 0;
-        for (List<SSTableReader> bucket : tasks)
-        {
-            if (bucket.size() >= cfs.getMinimumCompactionThreshold())
-                n += Math.ceil((double)bucket.size() / cfs.getMaximumCompactionThreshold());
-        }
-        return n;
+        return isMaximal && splitOutput
+               ? new SplittingCompactionTask(realm, txn, gcBefore, this)
+               : new CompactionTask(realm, txn, gcBefore, false, this);
     }
 
     public long getMaxSSTableBytes()
@@ -311,7 +351,7 @@ public long getMaxSSTableBytes()
 
     public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
     {
-        Map<String, String> uncheckedOptions = AbstractCompactionStrategy.validateOptions(options);
+        Map<String, String> uncheckedOptions = CompactionStrategyOptions.validateOptions(options);
         uncheckedOptions = SizeTieredCompactionStrategyOptions.validateOptions(options, uncheckedOptions);
 
         uncheckedOptions.remove(CompactionParams.Option.MIN_THRESHOLD.toString());
@@ -321,44 +361,69 @@ public static Map<String, String> validateOptions(Map<String, String> options) t
     }
 
     @Override
-    public synchronized void addSSTable(SSTableReader added)
+    public void replaceSSTables(Collection<CompactionSSTable> removed, Collection<CompactionSSTable> added)
+    {
+        synchronized (sstables)
+        {
+            for (CompactionSSTable remove : removed)
+                sstables.remove(remove);
+            sstables.addAll(added);
+        }
+    }
+
+    @Override
+    public void addSSTable(CompactionSSTable added)
     {
-        sstables.add(added);
+        synchronized (sstables)
+        {
+            sstables.add(added);
+        }
     }
 
     @Override
-    public synchronized void removeSSTable(SSTableReader sstable)
+    void removeDeadSSTables()
     {
-        sstables.remove(sstable);
+        removeDeadSSTables(sstables);
     }
 
     @Override
-    protected synchronized Set<SSTableReader> getSSTables()
+    public void removeSSTable(CompactionSSTable sstable)
     {
-        return ImmutableSet.copyOf(sstables);
+        synchronized (sstables)
+        {
+            sstables.remove(sstable);
+        }
+    }
+
+    @Override
+    public Set<CompactionSSTable> getSSTables()
+    {
+        synchronized (sstables)
+        {
+            return ImmutableSet.copyOf(sstables);
+        }
     }
 
     public String toString()
     {
         return String.format("SizeTieredCompactionStrategy[%s/%s]",
-            cfs.getMinimumCompactionThreshold(),
-            cfs.getMaximumCompactionThreshold());
+                             realm.getMinimumCompactionThreshold(),
+                             realm.getMaximumCompactionThreshold());
     }
 
     private static class SplittingCompactionTask extends CompactionTask
     {
-        public SplittingCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction txn, long gcBefore)
+        public SplittingCompactionTask(CompactionRealm realm, LifecycleTransaction txn, long gcBefore, CompactionStrategy strategy)
         {
-            super(cfs, txn, gcBefore);
+            super(realm, txn, gcBefore, false, strategy);
         }
 
         @Override
-        public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs,
+        public CompactionAwareWriter getCompactionAwareWriter(CompactionRealm realm,
                                                               Directories directories,
-                                                              LifecycleTransaction txn,
                                                               Set<SSTableReader> nonExpiredSSTables)
         {
-            return new SplittingSizeTieredCompactionWriter(cfs, directories, txn, nonExpiredSSTables);
+            return new SplittingSizeTieredCompactionWriter(realm, directories, transaction, nonExpiredSSTables);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyOptions.java b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyOptions.java
index eb1d8f97afe2..84179d4570d5 100644
--- a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyOptions.java
+++ b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyOptions.java
@@ -23,12 +23,12 @@
 
 public final class SizeTieredCompactionStrategyOptions
 {
-    protected static final long DEFAULT_MIN_SSTABLE_SIZE = 50L * 1024L * 1024L;
-    protected static final double DEFAULT_BUCKET_LOW = 0.5;
-    protected static final double DEFAULT_BUCKET_HIGH = 1.5;
-    protected static final String MIN_SSTABLE_SIZE_KEY = "min_sstable_size";
-    protected static final String BUCKET_LOW_KEY = "bucket_low";
-    protected static final String BUCKET_HIGH_KEY = "bucket_high";
+    static final long DEFAULT_MIN_SSTABLE_SIZE = 50L * 1024L * 1024L;
+    static final double DEFAULT_BUCKET_LOW = 0.5;
+    static final double DEFAULT_BUCKET_HIGH = 1.5;
+    static final String MIN_SSTABLE_SIZE_KEY = "min_sstable_size";
+    static final String BUCKET_LOW_KEY = "bucket_low";
+    static final String BUCKET_HIGH_KEY = "bucket_high";
 
     protected long minSSTableSize;
     protected double bucketLow;
@@ -46,9 +46,14 @@ public SizeTieredCompactionStrategyOptions(Map<String, String> options)
 
     public SizeTieredCompactionStrategyOptions()
     {
-        minSSTableSize = DEFAULT_MIN_SSTABLE_SIZE;
-        bucketLow = DEFAULT_BUCKET_LOW;
-        bucketHigh = DEFAULT_BUCKET_HIGH;
+        this(DEFAULT_MIN_SSTABLE_SIZE, DEFAULT_BUCKET_LOW, DEFAULT_BUCKET_HIGH);
+    }
+
+    SizeTieredCompactionStrategyOptions(long minSSTableSize, double bucketLow, double bucketHigh)
+    {
+        this.minSSTableSize = minSSTableSize;
+        this.bucketLow = bucketLow;
+        this.bucketHigh = bucketHigh;
     }
 
     private static double parseDouble(Map<String, String> options, String key, double defaultValue) throws ConfigurationException
diff --git a/src/java/org/apache/cassandra/db/compaction/TableOperation.java b/src/java/org/apache/cassandra/db/compaction/TableOperation.java
new file mode 100644
index 000000000000..3b7f03bd853c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/TableOperation.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Joiner;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Shared;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * This is a table operation that must be able to report the operation progress and to
+ * interrupt the operation when requested.
+ * <p/>
+ * Any operation defined by {@link OperationType} is normally implementing this interface,
+ * for example index building, view building, cache saving, anti-compaction, compaction,
+ * scrubbing, verifying, tombstone collection and others.
+ * <p/>
+ * These operations have in common that they run on the compaction executor and used to be
+ * known as "compaction".
+ * */
+public interface TableOperation
+{
+    /**
+     * @return the progress of the operation, see {@link Progress}.
+     */
+    TableOperation.Progress getProgress();
+
+    /**
+     * Interrupt the current operation if possible.
+     *
+     * @param trigger cause of compaction interruption
+     */
+    void stop(StopTrigger trigger);
+
+    /**
+     * @return true if the operation has been requested to be interrupted.
+     */
+    boolean isStopRequested();
+
+    default void throwIfStopRequested()
+    {
+        if (isStopRequested())
+            throw new CompactionInterruptedException(getProgress(), trigger());
+    }
+
+    /**
+     * Return true if the predicate for the given sstables holds, or if the operation
+     * does not consider any sstables, in which case it will always return true (the
+     * default behaviour).
+     * <p/>
+     *
+     * @param predicate the predicate to be applied to the operation sstables
+     *
+     * @return true by default, see overrides for different behaviors
+     */
+    boolean shouldStop(Predicate<SSTableReader> predicate);
+
+    /**
+     * @return cause of compaction interruption.
+     */
+    StopTrigger trigger();
+
+    /**
+     * if this compaction involves several/all tables we can safely check globalCompactionsPaused
+     * in isStopRequested() below
+     */
+    boolean isGlobal();
+
+    /**
+     * The unit for the {@link Progress} report.
+     */
+    enum Unit
+    {
+        BYTES("bytes"), RANGES("token range parts"), KEYS("keys");
+
+        private final String name;
+
+        Unit(String name)
+        {
+            this.name = name;
+        }
+
+        @Override
+        public String toString()
+        {
+            return this.name;
+        }
+
+        public static boolean isFileSize(String unit)
+        {
+            return BYTES.toString().equals(unit);
+        }
+    }
+
+    @Shared
+    enum StopTrigger
+    {
+        NONE("Unknown reason", false),
+        TRUNCATE("Truncated table", true),
+        DROP_TABLE("Dropped table", true),
+        INVALIDATE_INDEX("Index invalidation", true),
+        SHUTDOWN("Shutdown", true),
+        USER_STOP("User request", true),
+        COMPACTION("Compaction", true),
+        CLEANUP("Cleanup", true),
+        ANTICOMPACTION("Anticompaction after repair", true),
+        INDEX_BUILD("Secondary index build", true),
+        SCRUB("Scrub", true),
+        VERIFY("Verify", true),
+        RELOCATE("Relocation", true),
+        GARBAGE_COLLECT("Garbage collection", true),
+        UPGRADE_SSTABLES("SStable upgrade", true),
+        UNIT_TESTS("Unit tests", true);
+
+        private final String name;
+        private final boolean isFinal;
+
+        StopTrigger(String name, boolean isFinal)
+        {
+            this.name = name;
+            this.isFinal = isFinal;
+        }
+
+        // A stop trigger marked as final should not be overwritten. So a table operation that is
+        // marked with a final stop trigger cannot have its stop trigger changed to another value.
+        public boolean isFinal()
+        {
+            return isFinal;
+        }
+
+        @Override
+        public String toString()
+        {
+            return name;
+        }
+    }
+
+    /**
+     * The progress of a table operation.
+     */
+    interface Progress
+    {
+        String ID = "id";
+        String KEYSPACE = "keyspace";
+        String COLUMNFAMILY = "columnfamily";
+        String COMPLETED = "completed";
+        String TOTAL = "total";
+        String OPERATION_TYPE = "operationType";
+        String UNIT = "unit";
+        String OPERATION_ID = "operationId";
+        String SSTABLES = "sstables";
+        String TARGET_DIRECTORY = "targetDirectory";
+
+        /**
+         * @return the keyspace name, if the metadata is not null.
+         */
+        Optional<String> keyspace();
+
+        /**
+         * @return the table name, if the metadata is not null.
+         */
+        Optional<String> table();
+
+        /**
+         * @return the table metadata, this may be null if the operation has no metadata.
+         */
+        @Nullable TableMetadata metadata();
+
+        /**
+         * @return the number of units completed, see {@link this#unit()}.
+         */
+        long completed();
+
+        /**
+         * @return the total number of units that must be processed by the operation, see {@link this#unit()}.
+         */
+        long total();
+
+        /**
+         * @return the type of operation, see {@link OperationType}.
+         */
+        OperationType operationType();
+
+        /**
+         * @return a unique identifier for this operation.
+         */
+        TimeUUID operationId();
+
+        /**
+         * @return the unit to be used for {@link this#completed()} and {@link this#total()}, see {@link Unit}.
+         */
+        Unit unit();
+
+        /**
+         * @return a set of SSTables participating in this operation
+         */
+        Set<SSTableReader> sstables();
+
+        default String targetDirectory()
+        {
+            return "";
+        }
+
+        /**
+         * Note that this estimate is based on the amount of data we have left to read - it assumes input
+         * size == output size for a compaction, which is not really true, but should most often provide a worst case
+         * remaining write size.
+         */
+        default long estimatedRemainingWriteBytes()
+        {
+            if (unit() == Unit.BYTES && operationType().writesData)
+                return total() - completed();
+            return 0;
+        }
+
+        /**
+         * Get the directories this compaction could possibly write to.
+         *
+         * @return the directories that we might write to, or empty list if we don't know the metadata
+         * (like for index summary redistribution), or null if we don't have any disk boundaries
+         */
+        default List<File> getTargetDirectories()
+        {
+            if (metadata() != null && !metadata().isIndex())
+            {
+                ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(metadata().id);
+                if (cfs != null)
+                    return cfs.getDirectoriesForFiles(sstables());
+            }
+            return Collections.emptyList();
+        }
+
+        default String progressToString()
+        {
+            StringBuilder buff = new StringBuilder();
+            buff.append(String.format("%s(%s, %s / %s %s)", operationType(), operationId(), completed(), total(), unit()));
+            TableMetadata metadata = metadata();
+            if (metadata != null)
+            {
+                buff.append(String.format("@%s(%s, %s)", metadata.id, metadata.keyspace, metadata.name));
+            }
+            return buff.toString();
+        }
+
+        default Map<String, String> asMap()
+        {
+            Map<String, String> ret = new HashMap<>(8);
+            TableMetadata metadata = metadata();
+            ret.put(ID, metadata != null ? metadata.id.toString() : "");
+            ret.put(KEYSPACE, keyspace().orElse(null));
+            ret.put(COLUMNFAMILY, table().orElse(null));
+            ret.put(COMPLETED, Long.toString(completed()));
+            ret.put(TOTAL, Long.toString(total()));
+            ret.put(OPERATION_TYPE, operationType().toString());
+            ret.put(UNIT, unit().toString());
+            ret.put(OPERATION_ID, operationId() == null ? "" : operationId().toString());
+            ret.put(SSTABLES, Joiner.on(',').join(sstables()));
+            ret.put(TARGET_DIRECTORY, targetDirectory());
+            return ret;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/TableOperationObserver.java b/src/java/org/apache/cassandra/db/compaction/TableOperationObserver.java
new file mode 100644
index 000000000000..93dc643a5685
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/TableOperationObserver.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.apache.cassandra.utils.NonThrowingCloseable;
+
+/**
+ * An observer of {@link AbstractTableOperation}.
+ * <p/>
+ * The observer is notified when an operation is started. It returns a closeable that will be closed
+ * when the operation is finished. The operation can be queried at any time to get the progress information.
+ */
+public interface TableOperationObserver
+{
+    TableOperationObserver NOOP = operation -> () -> {};
+
+    /**
+     * Signal to the observer that an operation is starting.
+     *
+     * @param operation the operation starting
+     *
+     * @return a closeable that the caller should close when the operation completes
+     */
+    NonThrowingCloseable onOperationStart(TableOperation operation);
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/TieredCompactionStatistics.java b/src/java/org/apache/cassandra/db/compaction/TieredCompactionStatistics.java
new file mode 100644
index 000000000000..f785a180efdf
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/TieredCompactionStatistics.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+abstract class TieredCompactionStatistics extends CompactionAggregateStatistics
+{
+    private static final Collection<String> HEADER = ImmutableList.copyOf(Iterables.concat(ImmutableList.of("Bucket", "Hotness"),
+                                                                                           CompactionAggregateStatistics.HEADER));
+
+    private static final long serialVersionUID = 3695927592357987916L;
+
+    public TieredCompactionStatistics(CompactionAggregateStatistics base)
+    {
+        super(base);
+    }
+
+    /** The total read hotness of the sstables */
+    @JsonProperty
+    public double hotness()
+    {
+        return hotness;
+    }
+
+    @Override
+    protected Collection<String> header()
+    {
+        return HEADER;
+    }
+
+    @Override
+    protected Collection<String> data()
+    {
+        List<String> data = new ArrayList<>(HEADER.size());
+        data.add(tierValue());
+        data.add(String.format("%.4f", hotness));
+
+        data.addAll(super.data());
+
+        return data;
+    }
+
+    protected abstract String tierValue();
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/TimeTieredCompactionStatistics.java b/src/java/org/apache/cassandra/db/compaction/TimeTieredCompactionStatistics.java
new file mode 100644
index 000000000000..ce935e058932
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/TimeTieredCompactionStatistics.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.text.DateFormat;
+import java.util.Date;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * The statistics for time tiered compaction.
+ * <p/>
+ * Implements serializable to allow structured info to be returned via JMX.
+ */
+public class TimeTieredCompactionStatistics extends TieredCompactionStatistics
+{
+    protected static final DateFormat bucketFormatter = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
+
+    /** The timestamp in this tier */
+    private final long timestamp;
+
+    TimeTieredCompactionStatistics(CompactionAggregateStatistics base, long timestamp)
+    {
+        super(base);
+
+        this.timestamp = timestamp;
+    }
+
+    /** The timestamp in this tier */
+    public long timestamp()
+    {
+        return timestamp;
+    }
+
+    @Override
+    @JsonProperty("Bucket")
+    protected String tierValue()
+    {
+        return bucketFormatter.format(new Date(timestamp));
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionController.java b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionController.java
index e896a6c8e825..9e88baf3e666 100644
--- a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionController.java
@@ -24,7 +24,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 
 public class TimeWindowCompactionController extends CompactionController
@@ -33,9 +32,9 @@ public class TimeWindowCompactionController extends CompactionController
 
     private final boolean ignoreOverlaps;
 
-    public TimeWindowCompactionController(ColumnFamilyStore cfs, Set<SSTableReader> compacting, long gcBefore, boolean ignoreOverlaps)
+    public TimeWindowCompactionController(CompactionRealm realm, Set<SSTableReader> compacting, long gcBefore, boolean ignoreOverlaps)
     {
-        super(cfs, compacting, gcBefore);
+        super(realm, compacting, gcBefore);
         this.ignoreOverlaps = ignoreOverlaps;
         if (ignoreOverlaps)
             logger.warn("You are running with sstables overlapping checks disabled, it can result in loss of data");
diff --git a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
index 2709d43ae56d..0a3936d0550c 100644
--- a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
@@ -22,89 +22,68 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Iterator;
-import java.util.Objects;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Objects;
 import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.*;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.CompactionParams;
-import org.apache.cassandra.utils.Pair;
 
 import static com.google.common.collect.Iterables.filter;
+import static org.apache.cassandra.db.compaction.CompactionStrategyOptions.TOMBSTONE_COMPACTION_INTERVAL_OPTION;
+import static org.apache.cassandra.db.compaction.CompactionStrategyOptions.TOMBSTONE_THRESHOLD_OPTION;
+import static org.apache.cassandra.db.compaction.CompactionStrategyOptions.UNCHECKED_TOMBSTONE_COMPACTION_OPTION;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 
-public class TimeWindowCompactionStrategy extends AbstractCompactionStrategy
+public class TimeWindowCompactionStrategy extends LegacyAbstractCompactionStrategy.WithAggregates
 {
     private static final Logger logger = LoggerFactory.getLogger(TimeWindowCompactionStrategy.class);
 
-    private final TimeWindowCompactionStrategyOptions options;
-    protected volatile int estimatedRemainingTasks;
-    private final Set<SSTableReader> sstables = new HashSet<>();
+    private final TimeWindowCompactionStrategyOptions twcsOptions;
+    private final Set<CompactionSSTable> sstables = new HashSet<>();
     private long lastExpiredCheck;
     private long highestWindowSeen;
 
     // This is accessed in both the threading context of compaction / repair and also JMX
     private volatile Map<Long, Integer> sstableCountByBuckets = Collections.emptyMap();
 
-    public TimeWindowCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
+    public TimeWindowCompactionStrategy(CompactionStrategyFactory factory, Map<String, String> options)
     {
-        super(cfs, options);
-        this.estimatedRemainingTasks = 0;
-        this.options = new TimeWindowCompactionStrategyOptions(options);
+        super(factory, options);
+        this.twcsOptions = new TimeWindowCompactionStrategyOptions(options);
         String[] tsOpts = { UNCHECKED_TOMBSTONE_COMPACTION_OPTION, TOMBSTONE_COMPACTION_INTERVAL_OPTION, TOMBSTONE_THRESHOLD_OPTION };
-        if (Arrays.stream(tsOpts).map(o -> options.get(o)).filter(Objects::nonNull).anyMatch(v -> !v.equals("false")))
+        if (Arrays.stream(tsOpts).map(options::get).filter(Objects::nonNull).anyMatch(v -> !v.equals("false")))
         {
             logger.debug("Enabling tombstone compactions for TWCS");
         }
         else
         {
             logger.debug("Disabling tombstone compactions for TWCS");
-            disableTombstoneCompactions = true;
+            super.options.setDisableTombstoneCompactions(true);
         }
     }
 
     @Override
-    public AbstractCompactionTask getNextBackgroundTask(long gcBefore)
+    public AbstractCompactionTask createCompactionTask(final long gcBefore,
+                                                       LifecycleTransaction txn,
+                                                       boolean isMaximal,
+                                                       boolean splitOutput)
     {
-        List<SSTableReader> previousCandidate = null;
-        while (true)
-        {
-            List<SSTableReader> latestBucket = getNextBackgroundSSTables(gcBefore);
-
-            if (latestBucket.isEmpty())
-                return null;
-
-            // Already tried acquiring references without success. It means there is a race with
-            // the tracker but candidate SSTables were not yet replaced in the compaction strategy manager
-            if (latestBucket.equals(previousCandidate))
-            {
-                logger.warn("Could not acquire references for compacting SSTables {} which is not a problem per se," +
-                            "unless it happens frequently, in which case it must be reported. Will retry later.",
-                            latestBucket);
-                return null;
-            }
-
-            LifecycleTransaction modifier = cfs.getTracker().tryModify(latestBucket, OperationType.COMPACTION);
-            if (modifier != null)
-                return new TimeWindowCompactionTask(cfs, modifier, gcBefore, options.ignoreOverlaps);
-            previousCandidate = latestBucket;
-        }
+        return new TimeWindowCompactionTask(realm, txn, gcBefore, ignoreOverlaps(), this);
     }
 
     /**
@@ -112,21 +91,30 @@ public AbstractCompactionTask getNextBackgroundTask(long gcBefore)
      * @param gcBefore
      * @return
      */
-    private synchronized List<SSTableReader> getNextBackgroundSSTables(final long gcBefore)
+    @Override
+    protected synchronized CompactionAggregate getNextBackgroundAggregate(final long gcBefore)
     {
-        if (Iterables.isEmpty(cfs.getSSTables(SSTableSet.LIVE)))
-            return Collections.emptyList();
+        if (realm.getLiveSSTables().isEmpty())
+            return null;
 
-        Set<SSTableReader> uncompacting = ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains));
+        Set<? extends CompactionSSTable> compacting = realm.getCompactingSSTables();
+        Set<CompactionSSTable> noncompacting;
+        synchronized (sstables)
+        {
+            noncompacting = ImmutableSet.copyOf(filter(sstables, sstable -> !compacting.contains(sstable)));
+        }
 
         // Find fully expired SSTables. Those will be included no matter what.
-        Set<SSTableReader> expired = Collections.emptySet();
+        Set<CompactionSSTable> expired = Collections.emptySet();
 
-        if (currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency)
+        if (currentTimeMillis() - lastExpiredCheck > twcsOptions.expiredSSTableCheckFrequency)
         {
             logger.debug("TWCS expired check sufficiently far in the past, checking for fully expired SSTables");
-            expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, options.ignoreOverlaps ? Collections.emptySet() : cfs.getOverlappingLiveSSTables(uncompacting),
-                                                                   gcBefore, options.ignoreOverlaps);
+            expired = CompactionController.getFullyExpiredSSTables(realm,
+                                                                   noncompacting,
+                                                                   realm::getOverlappingLiveSSTables,
+                                                                   gcBefore,
+                                                                   twcsOptions.ignoreOverlaps);
             lastExpiredCheck = currentTimeMillis();
         }
         else
@@ -134,294 +122,286 @@ private synchronized List<SSTableReader> getNextBackgroundSSTables(final long gc
             logger.debug("TWCS skipping check for fully expired SSTables");
         }
 
-        Set<SSTableReader> candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting));
+        Set<CompactionSSTable> candidates = Sets.newHashSet(Iterables.filter(noncompacting, sstable -> !sstable.isMarkedSuspect()));
 
-        List<SSTableReader> compactionCandidates = new ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates, expired), gcBefore));
-        if (!expired.isEmpty())
+        CompactionAggregate compactionCandidate = getNextNonExpiredSSTables(Sets.difference(candidates, expired), gcBefore);
+        if (expired.isEmpty())
+            return compactionCandidate;
+
+        logger.debug("Including expired sstables: {}", expired);
+        if (compactionCandidate == null)
         {
-            logger.debug("Including expired sstables: {}", expired);
-            compactionCandidates.addAll(expired);
+            long timestamp = getWindowBoundsInMillis(twcsOptions.sstableWindowUnit, twcsOptions.sstableWindowSize,
+                                                     Collections.max(expired, Comparator.comparing(CompactionSSTable::getMaxTimestamp)).getMaxTimestamp());
+            return CompactionAggregate.createTimeTiered(expired, timestamp);
         }
 
-        return compactionCandidates;
+        return compactionCandidate.withExpired(expired);
     }
 
-    private List<SSTableReader> getNextNonExpiredSSTables(Iterable<SSTableReader> nonExpiringSSTables, final long gcBefore)
+    private CompactionAggregate getNextNonExpiredSSTables(Iterable<CompactionSSTable> nonExpiringSSTables, final long gcBefore)
     {
-        List<SSTableReader> mostInteresting = getCompactionCandidates(nonExpiringSSTables);
+        List<CompactionAggregate> candidates = getCompactionCandidates(nonExpiringSSTables);
+        backgroundCompactions.setPending(this, candidates);
 
-        if (mostInteresting != null)
-        {
-            return mostInteresting;
-        }
+        CompactionAggregate ret = candidates.isEmpty() ? null : candidates.get(0);
 
         // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
         // ratio is greater than threshold.
-        List<SSTableReader> sstablesWithTombstones = new ArrayList<>();
-        for (SSTableReader sstable : nonExpiringSSTables)
-        {
-            if (worthDroppingTombstones(sstable, gcBefore))
-                sstablesWithTombstones.add(sstable);
-        }
-        if (sstablesWithTombstones.isEmpty())
-            return Collections.emptyList();
+        if (ret == null || ret.isEmpty())
+            ret = makeTombstoneCompaction(gcBefore, nonExpiringSSTables, list -> Collections.min(list, CompactionSSTable.sizeComparator));
 
-        return Collections.singletonList(Collections.min(sstablesWithTombstones, SSTableReader.sizeComparator));
+        return ret;
     }
 
-    private List<SSTableReader> getCompactionCandidates(Iterable<SSTableReader> candidateSSTables)
+    private List<CompactionAggregate> getCompactionCandidates(Iterable<CompactionSSTable> candidateSSTables)
     {
-        Pair<HashMultimap<Long, SSTableReader>, Long> buckets = getBuckets(candidateSSTables, options.sstableWindowUnit, options.sstableWindowSize, options.timestampResolution);
+        NavigableMap<Long, List<CompactionSSTable>> buckets = getBuckets(candidateSSTables, twcsOptions.sstableWindowUnit, twcsOptions.sstableWindowSize, twcsOptions.timestampResolution);
         // Update the highest window seen, if necessary
-        if(buckets.right > this.highestWindowSeen)
-            this.highestWindowSeen = buckets.right;
-
-        NewestBucket mostInteresting = newestBucket(buckets.left,
-                cfs.getMinimumCompactionThreshold(),
-                cfs.getMaximumCompactionThreshold(),
-                options.stcsOptions,
-                this.highestWindowSeen);
-
-        this.estimatedRemainingTasks = mostInteresting.estimatedRemainingTasks;
-        this.sstableCountByBuckets = buckets.left.keySet().stream().collect(Collectors.toMap(Function.identity(), k -> buckets.left.get(k).size()));
-        if (!mostInteresting.sstables.isEmpty())
-            return mostInteresting.sstables;
-        return null;
+        if (!buckets.isEmpty())
+        {
+            long maxKey = buckets.lastKey();
+            if (maxKey > this.highestWindowSeen)
+                this.highestWindowSeen = maxKey;
+        }
+
+        return getBucketAggregates(buckets,
+                                   realm.getMinimumCompactionThreshold(),
+                                   realm.getMaximumCompactionThreshold(),
+                                   twcsOptions.stcsOptions,
+                                   this.highestWindowSeen);
     }
 
+
     @Override
-    public synchronized void addSSTable(SSTableReader sstable)
+    public void replaceSSTables(Collection<CompactionSSTable> removed, Collection<CompactionSSTable> added)
     {
-        sstables.add(sstable);
+        synchronized (sstables)
+        {
+            for (CompactionSSTable remove : removed)
+                sstables.remove(remove);
+            sstables.addAll(added);
+        }
     }
 
     @Override
-    public synchronized void removeSSTable(SSTableReader sstable)
+    public void addSSTable(CompactionSSTable sstable)
     {
-        sstables.remove(sstable);
+        synchronized (sstables)
+        {
+            sstables.add(sstable);
+        }
     }
 
     @Override
-    protected synchronized Set<SSTableReader> getSSTables()
+    void removeDeadSSTables()
     {
-        return ImmutableSet.copyOf(sstables);
+        removeDeadSSTables(sstables);
     }
 
-    /**
-     * Find the lowest and highest timestamps in a given timestamp/unit pair
-     * Returns milliseconds, caller should adjust accordingly
-     */
-    public static Pair<Long,Long> getWindowBoundsInMillis(TimeUnit windowTimeUnit, int windowTimeSize, long timestampInMillis)
+    @Override
+    public void removeSSTable(CompactionSSTable sstable)
     {
-        long lowerTimestamp;
-        long upperTimestamp;
-        long timestampInSeconds = TimeUnit.SECONDS.convert(timestampInMillis, TimeUnit.MILLISECONDS);
+        synchronized (sstables)
+        {
+            sstables.remove(sstable);
+        }
+    }
 
-        switch(windowTimeUnit)
+    @Override
+    public Set<CompactionSSTable> getSSTables()
+    {
+        synchronized (sstables)
         {
-            case MINUTES:
-                lowerTimestamp = timestampInSeconds - ((timestampInSeconds) % (60L * windowTimeSize));
-                upperTimestamp = (lowerTimestamp + (60L * (windowTimeSize - 1L))) + 59L;
-                break;
-            case HOURS:
-                lowerTimestamp = timestampInSeconds - ((timestampInSeconds) % (3600L * windowTimeSize));
-                upperTimestamp = (lowerTimestamp + (3600L * (windowTimeSize - 1L))) + 3599L;
-                break;
-            case DAYS:
-            default:
-                lowerTimestamp = timestampInSeconds - ((timestampInSeconds) % (86400L * windowTimeSize));
-                upperTimestamp = (lowerTimestamp + (86400L * (windowTimeSize - 1L))) + 86399L;
-                break;
+            return ImmutableSet.copyOf(sstables);
         }
+    }
 
-        return Pair.create(TimeUnit.MILLISECONDS.convert(lowerTimestamp, TimeUnit.SECONDS),
-                           TimeUnit.MILLISECONDS.convert(upperTimestamp, TimeUnit.SECONDS));
+    /**
+     * Find the lowest timestamp in a given window/unit pair and
+     * return it expressed as milliseconds, the caller should adjust accordingly
+     */
+    static long getWindowBoundsInMillis(TimeUnit windowTimeUnit, int windowTimeSize, long timestampInMillis)
+    {
 
+        long sizeInMillis = TimeUnit.MILLISECONDS.convert(windowTimeSize, windowTimeUnit);
+        return (timestampInMillis / sizeInMillis) * sizeInMillis;
     }
 
     /**
      * Group files with similar max timestamp into buckets.
+     * <p/>
+     * The max timestamp of each sstable is converted into the timestamp resolution and then the window bounds are
+     * calculated by calling {@link #getWindowBoundsInMillis(TimeUnit, int, long)}. The sstable is added to the bucket
+     * with the same lower timestamp bound. If the lower timestamp bound is higher than any other seen, then it is recorded
+     * as the max timestamp seen that will be returned.
      *
-     * @param files pairs consisting of a file and its min timestamp
-     * @param sstableWindowUnit
-     * @param sstableWindowSize
-     * @param timestampResolution
-     * @return A pair, where the left element is the bucket representation (map of timestamp to sstablereader), and the right is the highest timestamp seen
+     * @param files the candidate sstables
+     * @param sstableWindowUnit the time unit for {@code sstableWindowSize}
+     * @param sstableWindowSize the size of the time window by which sstables are grouped
+     * @param timestampResolution the time unit for converting the sstable timestamp
+     * @return A pair, where the left element is the bucket representation (multi-map of lower bound timestamp to sstables),
+     *         and the right is the highest lower bound timestamp seen
      */
     @VisibleForTesting
-    static Pair<HashMultimap<Long, SSTableReader>, Long> getBuckets(Iterable<SSTableReader> files, TimeUnit sstableWindowUnit, int sstableWindowSize, TimeUnit timestampResolution)
+    static NavigableMap<Long, List<CompactionSSTable>> getBuckets(Iterable<CompactionSSTable> files, TimeUnit sstableWindowUnit, int sstableWindowSize, TimeUnit timestampResolution)
     {
-        HashMultimap<Long, SSTableReader> buckets = HashMultimap.create();
+        NavigableMap<Long, List<CompactionSSTable>> buckets = new TreeMap<>(Long::compare);
 
-        long maxTimestamp = 0;
-        // Create hash map to represent buckets
         // For each sstable, add sstable to the time bucket
         // Where the bucket is the file's max timestamp rounded to the nearest window bucket
-        for (SSTableReader f : files)
+        for (CompactionSSTable f : files)
         {
             assert TimeWindowCompactionStrategyOptions.validTimestampTimeUnits.contains(timestampResolution);
             long tStamp = TimeUnit.MILLISECONDS.convert(f.getMaxTimestamp(), timestampResolution);
-            Pair<Long,Long> bounds = getWindowBoundsInMillis(sstableWindowUnit, sstableWindowSize, tStamp);
-            buckets.put(bounds.left, f);
-            if (bounds.left > maxTimestamp)
-                maxTimestamp = bounds.left;
+            addToBuckets(buckets, f, tStamp, sstableWindowUnit, sstableWindowSize);
         }
 
-        logger.trace("buckets {}, max timestamp {}", buckets, maxTimestamp);
-        return Pair.create(buckets, maxTimestamp);
+        logger.trace("buckets {}, max timestamp {}", buckets, buckets.isEmpty() ? "none" : buckets.lastKey().toString());
+        return buckets;
     }
 
-    static final class NewestBucket
+    @VisibleForTesting
+    static void addToBuckets(NavigableMap<Long, List<CompactionSSTable>> buckets, CompactionSSTable f, long tStamp, TimeUnit sstableWindowUnit, int sstableWindowSize)
     {
-        /** The sstables that should be compacted next */
-        final List<SSTableReader> sstables;
-
-        /** The number of tasks estimated */
-        final int estimatedRemainingTasks;
-
-        NewestBucket(List<SSTableReader> sstables, int estimatedRemainingTasks)
-        {
-            this.sstables = sstables;
-            this.estimatedRemainingTasks = estimatedRemainingTasks;
-        }
-
-        @Override
-        public String toString()
-        {
-            return String.format("sstables: %s, estimated remaining tasks: %d", sstables, estimatedRemainingTasks);
-        }
+        long bound = getWindowBoundsInMillis(sstableWindowUnit, sstableWindowSize, tStamp);
+        buckets.computeIfAbsent(bound,
+                                key -> new ArrayList<>())
+               .add(f);
     }
 
-
     /**
-     * @param buckets list of buckets, sorted from newest to oldest, from which to return the newest bucket within thresholds.
+     * If the current bucket has at least minThreshold SSTables, choose that one. For any other bucket, at least 2 SSTables is enough.
+     * In any case, limit to maxThreshold SSTables.
+     *
+     * @param buckets A map from a bucket id to a set of tables, sorted by id and then by table size
      * @param minThreshold minimum number of sstables in a bucket to qualify.
      * @param maxThreshold maximum number of sstables to compact at once (the returned bucket will be trimmed down to this).
-     * @return a bucket (list) of sstables to compact.
+     * @param stcsOptions the options for {@link SizeTieredCompactionStrategy} to be used in the newest bucket
+     * @param now the latest timestamp in milliseconds
+     *
+     * @return a list of compaction aggregates, one per time bucket
      */
     @VisibleForTesting
-    static NewestBucket newestBucket(HashMultimap<Long, SSTableReader> buckets, int minThreshold, int maxThreshold, SizeTieredCompactionStrategyOptions stcsOptions, long now)
+    static List<CompactionAggregate> getBucketAggregates(NavigableMap<Long, List<CompactionSSTable>> buckets,
+                                                         int minThreshold,
+                                                         int maxThreshold,
+                                                         SizeTieredCompactionStrategyOptions stcsOptions,
+                                                         long now)
     {
-        // If the current bucket has at least minThreshold SSTables, choose that one.
-        // For any other bucket, at least 2 SSTables is enough.
-        // In any case, limit to maxThreshold SSTables.
-
-        List<SSTableReader> sstables = Collections.emptyList();
-        int estimatedRemainingTasks = 0;
-
-        TreeSet<Long> allKeys = new TreeSet<>(buckets.keySet());
+        List<CompactionAggregate> ret = new ArrayList<>(buckets.size());
+        boolean nextCompactionFound = false; // set to true once the first bucket with a compaction is found
 
-        Iterator<Long> it = allKeys.descendingIterator();
-        while(it.hasNext())
+        for (Map.Entry<Long, List<CompactionSSTable>> entry : buckets.descendingMap().entrySet())
         {
-            Long key = it.next();
-            Set<SSTableReader> bucket = buckets.get(key);
+            Long key = entry.getKey();
+            List<CompactionSSTable> bucket = entry.getValue();
             logger.trace("Key {}, now {}", key, now);
+
+            CompactionPick selected = CompactionPick.EMPTY;
+            List<CompactionPick> pending = new ArrayList<>(1);
+
             if (bucket.size() >= minThreshold && key >= now)
             {
                 // If we're in the newest bucket, we'll use STCS to prioritize sstables
-                List<Pair<SSTableReader,Long>> pairs = SizeTieredCompactionStrategy.createSSTableAndLengthPairs(bucket);
-                List<List<SSTableReader>> stcsBuckets = SizeTieredCompactionStrategy.getBuckets(pairs, stcsOptions.bucketHigh, stcsOptions.bucketLow, stcsOptions.minSSTableSize);
-                List<SSTableReader> stcsInterestingBucket = SizeTieredCompactionStrategy.mostInterestingBucket(stcsBuckets, minThreshold, maxThreshold);
+                SizeTieredCompactionStrategy.SizeTieredBuckets stcsBuckets = new SizeTieredCompactionStrategy.SizeTieredBuckets(bucket,
+                                                                                                                                stcsOptions,
+                                                                                                                                minThreshold,
+                                                                                                                                maxThreshold);
+                stcsBuckets.aggregate();
 
-                // If the tables in the current bucket aren't eligible in the STCS strategy, we'll skip it and look for other buckets
-                if (!stcsInterestingBucket.isEmpty())
+                for (CompactionAggregate stcsAggregate : stcsBuckets.getAggregates())
                 {
-                    double remaining = bucket.size() - maxThreshold;
-                    estimatedRemainingTasks +=  1 + (remaining > minThreshold ? Math.ceil(remaining / maxThreshold) : 0);
-                    if (sstables.isEmpty())
+                    if (selected.isEmpty())
                     {
-                        logger.debug("Using STCS compaction for first window of bucket: data files {} , options {}", pairs, stcsOptions);
-                        sstables = stcsInterestingBucket;
+                        selected = stcsAggregate.getSelected().withParent(key);
+                        for (CompactionPick comp : stcsAggregate.getActive())
+                        {
+                            if (comp != stcsAggregate.getSelected())
+                                pending.add(comp);
+                        }
                     }
                     else
                     {
-                        logger.trace("First window of bucket is eligible but not selected: data files {} , options {}", pairs, stcsOptions);
+                        pending.addAll(stcsAggregate.getActive());
                     }
                 }
+
+                if (!selected.isEmpty())
+                    logger.debug("Newest window has STCS compaction candidates, {}, data files {} , options {}",
+                                 nextCompactionFound ? "eligible but not selected due to prior candidate" : "will be selected for compaction",
+                                 stcsBuckets.pairs(),
+                                 stcsOptions);
+                else
+                    logger.debug("No STCS compactions found for first window, data files {}, options {}", stcsBuckets.pairs(), stcsOptions);
+
+                if (!nextCompactionFound && !selected.isEmpty())
+                {
+                    nextCompactionFound = true;
+                    ret.add(0, CompactionAggregate.createTimeTiered(bucket, selected, pending, key)); // the first one will be submitted for compaction
+                }
+                else
+                {
+                    ret.add(CompactionAggregate.createTimeTiered(bucket, selected, pending, key));
+                }
             }
             else if (bucket.size() >= 2 && key < now)
             {
-                double remaining = bucket.size() - maxThreshold;
-                estimatedRemainingTasks +=  1 + (remaining > minThreshold ? Math.ceil(remaining / maxThreshold) : 0);
-                if (sstables.isEmpty())
+                List<CompactionSSTable> sstables = bucket;
+
+                // Sort the largest sstables off the end before splitting by maxThreshold
+                Collections.sort(sstables, CompactionSSTable.sizeComparator);
+
+                int i = 0;
+                while ((bucket.size() - i) >= 2)
+                {
+                    List<CompactionSSTable> pick = sstables.subList(i, i + Math.min(bucket.size() - i, maxThreshold));
+                    if (selected.isEmpty())
+                        selected = CompactionPick.create(key, pick);
+                    else
+                        pending.add(CompactionPick.create(key, pick));
+
+                    i += pick.size();
+                }
+
+                if (!nextCompactionFound)
                 {
                     logger.debug("bucket size {} >= 2 and not in current bucket, compacting what's here: {}", bucket.size(), bucket);
-                    sstables = trimToThreshold(bucket, maxThreshold);
+                    nextCompactionFound = true;
+                    ret.add(0, CompactionAggregate.createTimeTiered(bucket, selected, pending, key)); // the first one will be submitted for compaction
                 }
                 else
                 {
                     logger.trace("bucket size {} >= 2 and not in current bucket, eligible but not selected: {}", bucket.size(), bucket);
+                    ret.add(CompactionAggregate.createTimeTiered(bucket, selected, pending, key));
                 }
             }
             else
             {
                 logger.trace("No compaction necessary for bucket size {} , key {}, now {}", bucket.size(), key, now);
+                ret.add(CompactionAggregate.createTimeTiered(bucket, selected, pending, key)); // add an empty aggregate anyway so we get a full view
             }
         }
-        return new NewestBucket(sstables, estimatedRemainingTasks);
-    }
-
-    /**
-     * @param bucket set of sstables
-     * @param maxThreshold maximum number of sstables in a single compaction task.
-     * @return A bucket trimmed to the maxThreshold newest sstables.
-     */
-    @VisibleForTesting
-    static List<SSTableReader> trimToThreshold(Set<SSTableReader> bucket, int maxThreshold)
-    {
-        List<SSTableReader> ssTableReaders = new ArrayList<>(bucket);
-
-        // Trim the largest sstables off the end to meet the maxThreshold
-        Collections.sort(ssTableReaders, SSTableReader.sizeComparator);
-
-        return ImmutableList.copyOf(Iterables.limit(ssTableReaders, maxThreshold));
-    }
-
-    @Override
-    public synchronized Collection<AbstractCompactionTask> getMaximalTask(long gcBefore, boolean splitOutput)
-    {
-        Iterable<SSTableReader> filteredSSTables = filterSuspectSSTables(sstables);
-        if (Iterables.isEmpty(filteredSSTables))
-            return null;
-        LifecycleTransaction txn = cfs.getTracker().tryModify(filteredSSTables, OperationType.COMPACTION);
-        if (txn == null)
-            return null;
-        return Collections.singleton(new TimeWindowCompactionTask(cfs, txn, gcBefore, options.ignoreOverlaps));
+        return ret;
     }
 
     /**
      * TWCS should not group sstables for anticompaction - this can mix new and old data
      */
     @Override
-    public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> sstablesToGroup)
+    public Collection<Collection<CompactionSSTable>> groupSSTablesForAntiCompaction(Collection<? extends CompactionSSTable> sstablesToGroup)
     {
-        Collection<Collection<SSTableReader>> groups = new ArrayList<>(sstablesToGroup.size());
-        for (SSTableReader sstable : sstablesToGroup)
+        Collection<Collection<CompactionSSTable>> groups = new ArrayList<>(sstablesToGroup.size());
+        for (CompactionSSTable sstable : sstablesToGroup)
         {
             groups.add(Collections.singleton(sstable));
         }
         return groups;
     }
 
-    @Override
-    public synchronized AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, long gcBefore)
-    {
-        assert !sstables.isEmpty(); // checked for by CM.submitUserDefined
-
-        LifecycleTransaction modifier = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
-        if (modifier == null)
-        {
-            logger.debug("Unable to mark {} for compaction; probably a background compaction got to it first.  You can disable background compactions temporarily if this is a problem", sstables);
-            return null;
-        }
-
-        return new TimeWindowCompactionTask(cfs, modifier, gcBefore, options.ignoreOverlaps).setUserDefined(true);
-    }
-
-    public int getEstimatedRemainingTasks()
+    boolean ignoreOverlaps()
     {
-        return this.estimatedRemainingTasks;
+        return twcsOptions.ignoreOverlaps;
     }
 
     public long getMaxSSTableBytes()
@@ -436,7 +416,7 @@ public Map<Long, Integer> getSSTableCountByBuckets()
 
     public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
     {
-        Map<String, String> uncheckedOptions = AbstractCompactionStrategy.validateOptions(options);
+        Map<String, String> uncheckedOptions = CompactionStrategyOptions.validateOptions(options);
         uncheckedOptions = TimeWindowCompactionStrategyOptions.validateOptions(options, uncheckedOptions);
 
         uncheckedOptions.remove(CompactionParams.Option.MIN_THRESHOLD.toString());
@@ -448,7 +428,7 @@ public static Map<String, String> validateOptions(Map<String, String> options) t
     public String toString()
     {
         return String.format("TimeWindowCompactionStrategy[%s/%s]",
-                cfs.getMinimumCompactionThreshold(),
-                cfs.getMaximumCompactionThreshold());
+                             realm.getMinimumCompactionThreshold(),
+                             realm.getMaximumCompactionThreshold());
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionTask.java
index 33604e52e636..e9ea0331ef7c 100644
--- a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionTask.java
@@ -20,7 +20,6 @@
 
 import java.util.Set;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 
@@ -28,15 +27,15 @@ public class TimeWindowCompactionTask extends CompactionTask
 {
     private final boolean ignoreOverlaps;
 
-    public TimeWindowCompactionTask(ColumnFamilyStore cfs, LifecycleTransaction txn, long gcBefore, boolean ignoreOverlaps)
+    public TimeWindowCompactionTask(CompactionRealm realm, LifecycleTransaction txn, long gcBefore, boolean ignoreOverlaps, TimeWindowCompactionStrategy strategy)
     {
-        super(cfs, txn, gcBefore);
+        super(realm, txn, gcBefore, false, strategy);
         this.ignoreOverlaps = ignoreOverlaps;
     }
 
     @Override
     public CompactionController getCompactionController(Set<SSTableReader> toCompact)
     {
-        return new TimeWindowCompactionController(cfs, toCompact, gcBefore, ignoreOverlaps);
+        return new TimeWindowCompactionController(realm, toCompact, gcBefore, ignoreOverlaps);
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionContainer.java b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionContainer.java
new file mode 100644
index 000000000000..fd389bbf8468
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionContainer.java
@@ -0,0 +1,421 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.ScannerList;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.utils.TimeUUID;
+
+public class UnifiedCompactionContainer implements CompactionStrategyContainer
+{
+    private final CompactionStrategyFactory factory;
+    private final CompactionParams params;
+    private final CompactionParams metadataParams;
+    private final UnifiedCompactionStrategy strategy;
+    private final boolean enableAutoCompaction;
+    private final boolean hasVector;
+
+    AtomicBoolean enabled;
+
+    UnifiedCompactionContainer(CompactionStrategyFactory factory,
+                               BackgroundCompactions backgroundCompactions,
+                               CompactionParams params,
+                               CompactionParams metadataParams,
+                               boolean enabled,
+                               boolean enableAutoCompaction)
+    {
+        this.factory = factory;
+        this.params = params;
+        this.metadataParams = metadataParams;
+        this.strategy = new UnifiedCompactionStrategy(factory, backgroundCompactions, params.options());
+        this.enabled = new AtomicBoolean(enabled);
+        this.enableAutoCompaction = enableAutoCompaction;
+        this.hasVector = strategy.getController().hasVectorType();
+
+        factory.getCompactionLogger().strategyCreated(this.strategy);
+
+        if (this.strategy.getOptions().isLogEnabled())
+            factory.getCompactionLogger().enable();
+        else
+            factory.getCompactionLogger().disable();
+
+        startup();
+    }
+
+    @Override
+    public void enable()
+    {
+        this.enabled.set(true);
+    }
+
+    @Override
+    public void disable()
+    {
+        this.enabled.set(false);
+    }
+
+    @Override
+    public boolean isEnabled()
+    {
+        return enableAutoCompaction && enabled.get() && strategy.isActive;
+    }
+
+    @Override
+    public boolean isActive()
+    {
+        return strategy.isActive;
+    }
+
+    public static CompactionStrategyContainer create(@Nullable CompactionStrategyContainer previous,
+                                                     CompactionStrategyFactory strategyFactory,
+                                                     CompactionParams compactionParams,
+                                                     CompactionStrategyContainer.ReloadReason reason,
+                                                     boolean enableAutoCompaction)
+    {
+        boolean enabled = CompactionStrategyFactory.enableCompactionOnReload(previous, compactionParams, reason);
+        BackgroundCompactions backgroundCompactions;
+        // inherit compactions history from previous UCS container
+        if (previous instanceof UnifiedCompactionContainer)
+            backgroundCompactions = ((UnifiedCompactionContainer) previous).getBackgroundCompactions();
+
+        // for other cases start from scratch
+        // We don't inherit from legacy compactions right now because there are multiple strategies and we'd need
+        // to merge their BackgroundCompactions to support that. Merging per se is not tricky, but the bigger problem
+        // is aggregate cleanup. We'd need to unsubscribe from compaction tasks by legacy strategies and subscribe
+        // by the new UCS to remove inherited ongoing compactions when they complete.
+        // We might want to revisit this issue later to improve UX.
+        else
+            backgroundCompactions = new BackgroundCompactions(strategyFactory.getRealm());
+        CompactionParams metadataParams = createMetadataParams(previous, compactionParams, reason);
+
+        if (previous != null)
+            previous.shutdown();
+
+        return new UnifiedCompactionContainer(strategyFactory,
+                                              backgroundCompactions,
+                                              compactionParams,
+                                              metadataParams,
+                                              enabled,
+                                              enableAutoCompaction);
+    }
+
+    @Override
+    public CompactionStrategyContainer reload(@Nonnull CompactionStrategyContainer previous,
+                                              CompactionParams compactionParams,
+                                              ReloadReason reason)
+    {
+        return create(previous, factory, compactionParams, reason, enableAutoCompaction);
+    }
+
+    @Override
+    public boolean shouldReload(CompactionParams params, ReloadReason reason)
+    {
+        return reason != CompactionStrategyContainer.ReloadReason.METADATA_CHANGE
+               || !params.equals(getMetadataCompactionParams())
+               || hasVector != factory.getRealm().metadata().hasVectorType();
+    }
+
+    private static CompactionParams createMetadataParams(@Nullable CompactionStrategyContainer previous,
+                                                         CompactionParams compactionParams,
+                                                         ReloadReason reason)
+    {
+        CompactionParams metadataParams;
+        if (reason == CompactionStrategyContainer.ReloadReason.METADATA_CHANGE)
+            // metadataParams are aligned with compactionParams. We do not access TableParams.compaction to avoid racing with
+            // concurrent ALTER TABLE metadata change.
+            metadataParams = compactionParams;
+        else if (previous != null)
+            metadataParams = previous.getMetadataCompactionParams();
+        else
+            metadataParams = null;
+
+        return metadataParams;
+    }
+
+    @Override
+    public CompactionParams getCompactionParams()
+    {
+        return params;
+    }
+
+    @Override
+    public CompactionParams getMetadataCompactionParams()
+    {
+        return metadataParams;
+    }
+
+    @Override
+    public List<CompactionStrategy> getStrategies()
+    {
+        return ImmutableList.of(strategy);
+    }
+
+    @Override
+    public List<CompactionStrategy> getStrategies(boolean isRepaired, @Nullable TimeUUID pendingRepair)
+    {
+        return getStrategies();
+    }
+
+    @Override
+    public void repairSessionCompleted(TimeUUID sessionID)
+    {
+        // We are not tracking SSTables, so nothing to do here.
+    }
+
+    /**
+     * UCC does not need to use this method with {@link CompactionRealm#mutateRepairedWithLock}
+     * @return null
+     */
+    @Override
+    public ReentrantReadWriteLock.WriteLock getWriteLock()
+    {
+        return null;
+    }
+
+    @Override
+    public CompactionLogger getCompactionLogger()
+    {
+        return strategy.compactionLogger;
+    }
+
+    @Override
+    public void pause()
+    {
+        strategy.pause();
+    }
+
+    @Override
+    public void resume()
+    {
+        strategy.resume();
+    }
+
+    @Override
+    public void startup()
+    {
+        strategy.startup();
+    }
+
+    @Override
+    public void shutdown()
+    {
+        strategy.shutdown();
+    }
+
+    @Override
+    public Collection<AbstractCompactionTask> getNextBackgroundTasks(long gcBefore)
+    {
+        return strategy.getNextBackgroundTasks(gcBefore);
+    }
+
+    @Override
+    public CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism)
+    {
+        return strategy.getMaximalTasks(gcBefore, splitOutput, permittedParallelism);
+    }
+
+    @Override
+    public synchronized CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism, OperationType operationType)
+    {
+        return strategy.getMaximalTasks(gcBefore, splitOutput, permittedParallelism, operationType);
+    }
+
+    @Override
+    public CompactionTasks getUserDefinedTasks(Collection<? extends CompactionSSTable> sstables, long gcBefore)
+    {
+        return strategy.getUserDefinedTasks(sstables, gcBefore);
+    }
+
+    @Override
+    public int getEstimatedRemainingTasks()
+    {
+        return strategy.getEstimatedRemainingTasks();
+    }
+
+    @Override
+    public int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes, boolean isIncremental)
+    {
+        return strategy.getEstimatedRemainingTasks(additionalSSTables, additionalBytes, isIncremental);
+    }
+
+    @Override
+    public AbstractCompactionTask createCompactionTask(LifecycleTransaction txn, long gcBefore, long maxSSTableBytes)
+    {
+        return strategy.createCompactionTask(txn, gcBefore, maxSSTableBytes);
+    }
+
+    @Override
+    public int getTotalCompactions()
+    {
+        return strategy.getTotalCompactions();
+    }
+
+    @Override
+    public List<CompactionStrategyStatistics> getStatistics()
+    {
+        return strategy.getStatistics();
+    }
+
+    @Override
+    public long getMaxSSTableBytes()
+    {
+        return strategy.getMaxSSTableBytes();
+    }
+
+    @Override
+    public int[] getSSTableCountPerLevel()
+    {
+        return strategy.getSSTableCountPerLevel();
+    }
+
+    @Override
+    public long[] getPerLevelSizeBytes()
+    {
+        return strategy.getPerLevelSizeBytes();
+    }
+
+    @Override
+    public boolean isLeveledCompaction()
+    {
+        return strategy.isLeveledCompaction();
+    }
+
+    @Override
+    public int[] getSSTableCountPerTWCSBucket()
+    {
+        return strategy.getSSTableCountPerTWCSBucket();
+    }
+
+    @Override
+    public int getLevelFanoutSize()
+    {
+        return strategy.getLevelFanoutSize();
+    }
+
+    @Override
+    public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+    {
+        return strategy.getScanners(sstables, ranges);
+    }
+
+    @Override
+    public String getName()
+    {
+        return strategy.getName();
+    }
+
+    @Override
+    public Set<? extends CompactionSSTable> getSSTables()
+    {
+        return strategy.getSSTables();
+    }
+
+    @Override
+    public Collection<Collection<CompactionSSTable>> groupSSTablesForAntiCompaction(Collection<? extends CompactionSSTable> sstablesToGroup)
+    {
+        return strategy.groupSSTablesForAntiCompaction(sstablesToGroup);
+    }
+
+    @Override
+    public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
+                                                       long keyCount,
+                                                       long repairedAt,
+                                                       TimeUUID pendingRepair,
+                                                       boolean isTransient,
+                                                       IntervalSet<CommitLogPosition> commitLogPositions,
+                                                       int sstableLevel,
+                                                       SerializationHeader header,
+                                                       Collection<Index.Group> indexGroups,
+                                                       LifecycleNewTracker lifecycleNewTracker)
+    {
+        return strategy.createSSTableMultiWriter(descriptor,
+                                                 keyCount,
+                                                 repairedAt,
+                                                 pendingRepair,
+                                                 isTransient,
+                                                 commitLogPositions,
+                                                 sstableLevel,
+                                                 header,
+                                                 indexGroups,
+                                                 lifecycleNewTracker);
+    }
+
+    @Override
+    public boolean supportsEarlyOpen()
+    {
+        return strategy.supportsEarlyOpen();
+    }
+
+    @Override
+    public void periodicReport()
+    {
+        strategy.periodicReport();
+    }
+
+    @Override
+    public Map<String, String> getMaxOverlapsMap()
+    {
+        return strategy.getMaxOverlapsMap();
+    }
+
+    BackgroundCompactions getBackgroundCompactions()
+    {
+        return strategy.backgroundCompactions;
+    }
+
+    @Override
+    public void onInProgress(CompactionProgress progress)
+    {
+        strategy.onInProgress(progress);
+    }
+
+    @Override
+    public void onCompleted(TimeUUID id, Throwable err)
+    {
+        strategy.onCompleted(id, err);
+    }
+
+    @Override
+    public void handleNotification(INotification notification, Object sender)
+    {
+        // TODO - this is a no-op because the strategy is stateless but we could detect here
+        // sstables that are added either because of streaming or because of nodetool refresh
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStatistics.java b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStatistics.java
new file mode 100644
index 000000000000..3d6453349bf9
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStatistics.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * The statistics for size tiered compaction.
+ * <p/>
+ * Implements serializable to allow structured info to be returned via JMX.
+ */
+public class UnifiedCompactionStatistics extends CompactionAggregateStatistics
+{
+    private static final Collection<String> HEADER = ImmutableList.copyOf(Iterables.concat(ImmutableList.of("Level", "W", "Min Density", "Max Density", "Overlap"),
+                                                                                           CompactionAggregateStatistics.HEADER));
+
+    private static final long serialVersionUID = 3695927592357345266L;
+
+    /** The bucket number */
+    private final int bucket;
+
+    /** The survival factor o */
+    private final double survivalFactor;
+
+    /** The scaling parameter W */
+    private final int scalingParameter;
+
+    /** The minimum density for an SSTable that belongs to this bucket */
+    private final double minDensityBytes;
+
+    /** The maximum density for an SSTable run that belongs to this bucket */
+    private final double maxDensityBytes;
+
+    /** The maximum number of overlapping sstables in the shard */
+    private final int maxOverlap;
+
+    /** The name of the shard */
+    private final String shard;
+
+    UnifiedCompactionStatistics(CompactionAggregateStatistics base,
+                                int bucketIndex,
+                                double survivalFactor,
+                                int scalingParameter,
+                                double minDensityBytes,
+                                double maxDensityBytes,
+                                int maxOverlap,
+                                String shard)
+    {
+        super(base);
+
+        this.bucket = bucketIndex;
+        this.survivalFactor = survivalFactor;
+        this.scalingParameter = scalingParameter;
+        this.minDensityBytes = minDensityBytes;
+        this.maxDensityBytes = maxDensityBytes;
+        this.maxOverlap = maxOverlap;
+        this.shard = shard;
+    }
+
+    /** The bucket number */
+    @JsonProperty
+    public int bucket()
+    {
+        return bucket;
+    }
+
+    /** The survival factor o, currently always one */
+    @JsonProperty
+    public double survivalFactor()
+    {
+        return survivalFactor;
+    }
+
+    /** The scaling parameter W */
+    @JsonProperty
+    public int scalingParameter()
+    {
+        return scalingParameter;
+    }
+
+    /** The minimum size for an SSTable that belongs to this bucket */
+    @JsonProperty
+    public double minDensityBytes()
+    {
+        return minDensityBytes;
+    }
+
+    /** The maximum size for an SSTable that belongs to this bucket */
+    @JsonProperty
+    public double maxDensityBytes()
+    {
+        return maxDensityBytes;
+    }
+
+    /** The maximum number of overlapping sstables in this bucket */
+    @JsonProperty
+    public int maxOverlap()
+    {
+        return maxOverlap;
+    }
+
+    /** The name of the shard, empty if the compaction is not sharded (the default). */
+    @JsonProperty
+    @Override
+    public String shard()
+    {
+        return shard;
+    }
+
+    @Override
+    protected Collection<String> header()
+    {
+        return HEADER;
+    }
+
+    @Override
+    protected Collection<String> data()
+    {
+        List<String> data = new ArrayList<>(HEADER.size());
+        data.add(Integer.toString(bucket()));
+        data.add(UnifiedCompactionStrategy.printScalingParameter(scalingParameter));
+        data.add(FBUtilities.prettyPrintBinary(minDensityBytes, "B", " "));
+        data.add(FBUtilities.prettyPrintBinary(maxDensityBytes, "B", " "));
+
+        data.add(Integer.toString(maxOverlap));
+
+        data.addAll(super.data());
+
+        return data;
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java
index 8ff0565064ef..866807e2e2fa 100644
--- a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java
@@ -1,13 +1,11 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Copyright DataStax, Inc.
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,55 +17,71 @@
 package org.apache.cassandra.db.compaction;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.BiPredicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.DiskBoundaries;
+import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.compaction.unified.Controller;
+import org.apache.cassandra.db.compaction.unified.Reservations;
 import org.apache.cassandra.db.compaction.unified.ShardedMultiWriter;
 import org.apache.cassandra.db.compaction.unified.UnifiedCompactionTask;
+import org.apache.cassandra.db.lifecycle.CompositeLifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.PartialLifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Overlaps;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.Throwables;
 
-/**
- * The design of the unified compaction strategy is described in the accompanying UnifiedCompactionStrategy.md.
- *
- * See CEP-26: https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-26%3A+Unified+Compaction+Strategy
- */
+import static org.apache.cassandra.utils.Throwables.perform;
+
+/// The design of the unified compaction strategy is described in [UnifiedCompactionStrategy.md](./UnifiedCompactionStrategy.md).
+///
+/// See also [CEP-26](https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-26%3A+Unified+Compaction+Strategy).
 public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
 {
+    @SuppressWarnings("unused") // accessed via reflection
+    public static final Class<? extends CompactionStrategyContainer> CONTAINER_CLASS = UnifiedCompactionContainer.class;
+
     private static final Logger logger = LoggerFactory.getLogger(UnifiedCompactionStrategy.class);
 
-    static final int MAX_LEVELS = 32;   // This is enough for a few petabytes of data (with the worst case fan factor
+    public static final int MAX_LEVELS = 32;   // This is enough for a few petabytes of data (with the worst case fan factor
     // at W=0 this leaves room for 2^32 sstables, presumably of at least 1MB each).
 
     private static final Pattern SCALING_PARAMETER_PATTERN = Pattern.compile("(N)|L(\\d+)|T(\\d+)|([+-]?\\d+)");
@@ -75,32 +89,46 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
                                                                                                 .replaceAll("[()]", "")
                                                                                                 .replace("\\d", "[0-9]");
 
+    /// Special level definition for major compactions.
+    static final Level LEVEL_MAXIMAL = new Level(-1, 0, 0, 0, 1, 0, Double.POSITIVE_INFINITY);
+
     private final Controller controller;
 
-    private volatile ShardManager shardManager;
+    private volatile ArenaSelector currentArenaSelector;
+    private volatile ShardManager currentShardManager;
 
     private long lastExpiredCheck;
 
-    protected volatile int estimatedRemainingTasks;
-    @VisibleForTesting
-    protected final Set<SSTableReader> sstables = new HashSet<>();
+    public UnifiedCompactionStrategy(CompactionStrategyFactory factory, BackgroundCompactions backgroundCompactions, Map<String, String> options)
+    {
+        this(factory, backgroundCompactions, options, Controller.fromOptions(factory.getRealm(), options));
+    }
 
-    public UnifiedCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
+    public UnifiedCompactionStrategy(CompactionStrategyFactory factory, BackgroundCompactions backgroundCompactions, Controller controller)
     {
-        this(cfs, options, Controller.fromOptions(cfs, options));
+        this(factory, backgroundCompactions, new HashMap<>(), controller);
     }
 
-    public UnifiedCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options, Controller controller)
+    public UnifiedCompactionStrategy(CompactionStrategyFactory factory, BackgroundCompactions backgroundCompactions, Map<String, String> options, Controller controller)
     {
-        super(cfs, options);
+        super(factory, backgroundCompactions, options);
         this.controller = controller;
-        estimatedRemainingTasks = 0;
-        lastExpiredCheck = Clock.Global.currentTimeMillis();
+    }
+
+    @VisibleForTesting
+    public UnifiedCompactionStrategy(CompactionStrategyFactory factory, Controller controller)
+    {
+        this(factory, new BackgroundCompactions(factory.getRealm()), new HashMap<>(), controller);
     }
 
     public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
     {
-        return Controller.validateOptions(AbstractCompactionStrategy.validateOptions(options));
+        return Controller.validateOptions(CompactionStrategyOptions.validateOptions(options));
+    }
+
+    public void storeControllerConfig()
+    {
+        getController().storeControllerConfig();
     }
 
     public static int fanoutFromScalingParameter(int w)
@@ -139,123 +167,472 @@ private static int atLeast2(int value, String str)
     public static String printScalingParameter(int w)
     {
         if (w < 0)
-            return "L" + Integer.toString(2 - w);
+            return 'L' + Integer.toString(2 - w);
         else if (w > 0)
-            return "T" + Integer.toString(w + 2);
+            return 'T' + Integer.toString(w + 2);
         else
             return "N";
     }
 
+    /// Make a time-based UUID for unified compaction tasks with sequence 0. The reason to do this is to accommodate
+    /// parallelized compactions:
+    /// - Sequence 0 (visible as `-8000-` in the UUID string) denotes single-task (i.e. non-parallelized) compactions.
+    /// - Sequence >0 (`-800n-`) denotes the individual task's index of a parallelized compaction.
+    /// - Parallelized compactions use sequence 0 as the transaction id, and sequences from 1 to the number of tasks
+    ///   for the ids of individual tasks.
+    public static TimeUUID nextTimeUUID()
+    {
+        return TimeUUID.Generator.withSequence(TimeUUID.Generator.nextTimeUUID(), 0);
+    }
+
     @Override
-    public synchronized Collection<AbstractCompactionTask> getMaximalTask(long gcBefore, boolean splitOutput)
+    public Collection<Collection<CompactionSSTable>> groupSSTablesForAntiCompaction(Collection<? extends CompactionSSTable> sstablesToGroup)
     {
-        maybeUpdateShardManager();
-        // The tasks are split by repair status and disk, as well as in non-overlapping sections to enable some
-        // parallelism (to the amount that L0 sstables are split, i.e. at least base_shard_count). The result will be
-        // split across shards according to its density. Depending on the parallelism, the operation may require up to
-        // 100% extra space to complete.
-        List<AbstractCompactionTask> tasks = new ArrayList<>();
-        List<Set<SSTableReader>> nonOverlapping = splitInNonOverlappingSets(filterSuspectSSTables(getSSTables()));
-        for (Set<SSTableReader> set : nonOverlapping)
+        Collection<Collection<CompactionSSTable>> groups = new ArrayList<>();
+        for (Arena arena : getCompactionArenas(sstablesToGroup, (i1, i2) -> true)) // take all sstables
         {
-            LifecycleTransaction txn = cfs.getTracker().tryModify(set, OperationType.COMPACTION);
-            if (txn != null)
-                tasks.add(createCompactionTask(txn, gcBefore));
+            groups.addAll(super.groupSSTablesForAntiCompaction(arena.sstables));
         }
-        return tasks;
+
+        return groups;
+    }
+
+    @Override
+    public synchronized CompactionTasks getUserDefinedTasks(Collection<? extends CompactionSSTable> sstables, long gcBefore)
+    {
+        // The tasks need to be split by repair status and disk, but otherwise we must assume the user knows what they
+        // are doing.
+        List<AbstractCompactionTask> tasks = new ArrayList<>();
+        for (Arena arena : getCompactionArenas(sstables, UnifiedCompactionStrategy::isSuitableForCompaction))
+            tasks.addAll(super.getUserDefinedTasks(arena.sstables, gcBefore));
+        return CompactionTasks.create(tasks);
+    }
+
+    /// Get a list of maximal aggregates that can be compacted independently in parallel to achieve a major compaction.
+    ///
+    /// These aggregates split the sstables in each arena into non-overlapping groups where the boundaries between these
+    /// groups are also boundaries of the current sharding configuration. Compacting the groups independently has the
+    /// same effect as compacting all of the sstables in the arena together in one operation.
+    public synchronized List<CompactionAggregate.UnifiedAggregate> getMaximalAggregates()
+    {
+        return getMaximalAggregates(realm.getLiveSSTables());
+    }
+
+    public synchronized List<CompactionAggregate.UnifiedAggregate> getMaximalAggregates(Collection<? extends CompactionSSTable> sstables)
+    {
+        maybeUpdateSelector(); // must be called before computing compaction arenas
+        return getMaximalAggregatesWithArenas(getCompactionArenas(sstables, UnifiedCompactionStrategy::isSuitableForCompaction));
     }
 
-    private static List<Set<SSTableReader>> splitInNonOverlappingSets(Collection<SSTableReader> sstables)
+    private synchronized List<CompactionAggregate.UnifiedAggregate> getMaximalAggregatesWithArenas(Collection<Arena> compactionArenas)
     {
-        List<Set<SSTableReader>> overlapSets = Overlaps.constructOverlapSets(new ArrayList<>(sstables),
-                                                                             UnifiedCompactionStrategy::startsAfter,
-                                                                             SSTableReader.firstKeyComparator,
-                                                                             SSTableReader.lastKeyComparator);
-        if (overlapSets.isEmpty())
-            return overlapSets;
+        // The aggregates are split into arenas by repair status and disk, as well as in non-overlapping sections to
+        // enable some parallelism and efficient use of extra space. The result will be split across shards according to
+        // its density.
+        // Depending on the parallelism, the operation may require up to 100% extra space to complete.
+        List<CompactionAggregate.UnifiedAggregate> aggregates = new ArrayList<>();
 
-        Set<SSTableReader> group = overlapSets.get(0);
-        List<Set<SSTableReader>> groups = new ArrayList<>();
-        for (int i = 1; i < overlapSets.size(); ++i)
+        for (Arena arena : compactionArenas)
         {
-            Set<SSTableReader> current = overlapSets.get(i);
-            if (Sets.intersection(current, group).isEmpty())
+            // If possible, we want to issue separate compactions for non-overlapping sets of sstables, to allow
+            // for smaller extra space requirements. However, if the sharding configuration has changed, a major
+            // compaction should combine non-overlapping sets if they are split on a boundary that is no longer
+            // in effect.
+            List<Set<CompactionSSTable>> groups =
+            getShardManager().splitSSTablesInShards(arena.sstables,
+                                                    makeShardingStats(arena.sstables).shardCountForDensity,
+                                                    (sstableShard, shardRange) -> Sets.newHashSet(sstableShard));
+
+            // Now combine all of these groups that share an sstable so that we have valid independent transactions.
+            groups = Overlaps.combineSetsWithCommonElement(groups);
+
+            for (Set<CompactionSSTable> group : groups)
             {
-                groups.add(group);
-                group = current;
-            }
-            else
-            {
-                group.addAll(current);
+                aggregates.add(CompactionAggregate.createUnified(group,
+                                                                 Overlaps.maxOverlap(group,
+                                                                                     CompactionSSTable.startsAfter,
+                                                                                     CompactionSSTable.firstKeyComparator,
+                                                                                     CompactionSSTable.lastKeyComparator),
+                                                                 createPick(nextTimeUUID(), LEVEL_MAXIMAL.index, group),
+                                                                 Collections.emptyList(),
+                                                                 arena,
+                                                                 LEVEL_MAXIMAL));
             }
         }
-        groups.add(group);
-        return groups;
+        return aggregates;
     }
 
     @Override
-    public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, final long gcBefore)
+    public synchronized CompactionTasks getMaximalTasks(long gcBefore, boolean splitOutput, int permittedParallelism)
     {
-        assert !sstables.isEmpty(); // checked for by CM.submitUserDefined
+        if (permittedParallelism <= 0)
+            permittedParallelism = Integer.MAX_VALUE;
 
-        LifecycleTransaction transaction = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
-        if (transaction == null)
+        List<AbstractCompactionTask> tasks = new ArrayList<>();
+        try
         {
-            logger.trace("Unable to mark {} for compaction; probably a background compaction got to it first.  You can disable background compactions temporarily if this is a problem", sstables);
-            return null;
+            // Split the space into independently compactable groups.
+            for (var aggregate : getMaximalAggregates())
+            {
+                LifecycleTransaction txn = realm.tryModify(aggregate.getSelected().sstables(),
+                                                           OperationType.COMPACTION,
+                                                           aggregate.getSelected().id());
+
+                // Create (potentially parallelized) tasks for each group.
+                if (txn != null)
+                    createAndAddTasks(gcBefore, txn, getShardingStats(aggregate), permittedParallelism, tasks);
+                // we ignore splitOutput (always split according to the strategy's sharding) and do not need isMaximal
+
+                // Note: major compactions should not end up in the background compactions tracker to avoid wreaking
+                // havok in the thread assignment logic.
+            }
+
+            // If we have more arenas/non-overlapping sets than the permitted parallelism, we will try to run all the
+            // individual tasks in parallel (including as parallelized compactions) so that they finish quickest and release
+            // any space they hold, and then reuse the compaction thread to run the next set of tasks.
+            return CompactionTasks.create(CompositeCompactionTask.applyParallelismLimit(tasks, permittedParallelism));
+        }
+        catch (Throwable t)
+        {
+            throw rejectTasks(tasks, t);
         }
+    }
 
-        return createCompactionTask(transaction, gcBefore).setUserDefined(true);
+    @Override
+    public void startup()
+    {
+        perform(super::startup,
+                () -> controller.startup(this, ScheduledExecutors.scheduledTasks));
     }
 
-    /**
-     * Returns a compaction task to run next.
-     *
-     * This method is synchronized because task creation is significantly more expensive in UCS; the strategy is
-     * stateless, therefore it has to compute the shard/bucket structure on each call.
-     *
-     * @param gcBefore throw away tombstones older than this
-     */
     @Override
-    public synchronized UnifiedCompactionTask getNextBackgroundTask(long gcBefore)
+    public void shutdown()
+    {
+        perform(super::shutdown,
+                controller::shutdown);
+    }
+
+    /// Returns a collections of compaction tasks.
+    ///
+    /// This method is synchornized because task creation is significantly more expensive in UCS; the strategy is
+    /// stateless, therefore it has to compute the shard/bucket structure on each call.
+    ///
+    /// @param gcBefore throw away tombstones older than this
+    /// @return collection of AbstractCompactionTask, which could be either a CompactionTask or an UnifiedCompactionTask
+    @Override
+    public synchronized Collection<AbstractCompactionTask> getNextBackgroundTasks(long gcBefore)
+    {
+        // TODO - we should perhaps consider executing this code less frequently than legacy strategies
+        // since it's more expensive, and we should therefore prevent a second concurrent thread from executing at all
+
+        // Repairs can leave behind sstables in pending repair state if they race with a compaction on those sstables.
+        // Both the repair and the compact process can't modify the same sstables set at the same time. So compaction
+        // is left to eventually move those sstables from FINALIZED repair sessions away from repair states.
+        Collection<AbstractCompactionTask> repairFinalizationTasks = ActiveRepairService
+                                                                     .instance()
+                                                                     .consistent
+                                                                     .local
+                                                                     .getZombieRepairFinalizationTasks(realm, realm.getLiveSSTables());
+        if (!repairFinalizationTasks.isEmpty())
+            return repairFinalizationTasks;
+
+        // Expirations have to run before compaction (if run in parallel they may cause overlap tracker to leave
+        // unnecessary tombstones in place), so return only them if found.
+        Collection<AbstractCompactionTask> expirationTasks = getExpirationTasks(gcBefore);
+        if (expirationTasks != null)
+            return expirationTasks;
+
+        return getNextBackgroundTasks(getNextCompactionAggregates(), gcBefore);
+    }
+
+    /// Check for fully expired sstables and return a collection of expiration tasks if found.
+    public Collection<AbstractCompactionTask> getExpirationTasks(long gcBefore)
+    {
+        long ts = Clock.Global.currentTimeMillis();
+        boolean expiredCheck = ts - lastExpiredCheck > controller.getExpiredSSTableCheckFrequency();
+        if (!expiredCheck)
+            return null;
+        lastExpiredCheck = ts;
+
+        var expired = getFullyExpiredSSTables(gcBefore);
+        if (expired.isEmpty())
+            return null;
+
+        if (logger.isDebugEnabled())
+            logger.debug("Expiration check found {} fully expired SSTables", expired.size());
+
+        return createExpirationTasks(expired);
+    }
+
+    /// Create expiration tasks for the given set of expired sstables.
+    /// Used by CNDB
+    public List<AbstractCompactionTask> createExpirationTasks(Set<CompactionSSTable> expired)
     {
-        while (true)
+        // if we found sstables to expire, split them to arenas to correctly isolate their repair status.
+        var tasks = new ArrayList<AbstractCompactionTask>();
+        try
         {
-            CompactionPick pick = getNextCompactionPick(gcBefore);
-            if (pick == null)
-                return null;
-            UnifiedCompactionTask task = createCompactionTask(pick, gcBefore);
-            if (task != null)
-                return task;
+            for (var arena : getCompactionArenas(expired, (i1, i2) -> true))
+            {
+                LifecycleTransaction txn = realm.tryModify(arena.sstables, OperationType.COMPACTION);
+                if (txn != null)
+                    tasks.add(createExpirationTask(txn));
+                else
+                    logger.warn("Failed to submit expiration task because a transaction could not be created. If this happens frequently, it should be reported");
+            }
+            return tasks;
         }
+        catch (Throwable t)
+        {
+            throw rejectTasks(tasks, t);
+        }
+    }
+
+    /// Get all expired sstables, regardless of expiration status.
+    /// This is simpler and faster than per-arena collection, and will find nothing in most calls.
+    /// Used by CNDB
+    public Set<CompactionSSTable> getFullyExpiredSSTables(long gcBefore)
+    {
+        return CompactionController.getFullyExpiredSSTables(realm,
+                                                                   getSuitableSSTables(),
+                                                                   realm::getOverlappingLiveSSTables,
+                                                                   gcBefore,
+                                                                   controller.getIgnoreOverlapsInExpirationCheck());
     }
 
-    private UnifiedCompactionTask createCompactionTask(CompactionPick pick, long gcBefore)
+    /// Used by CNDB where compaction aggregates come from etcd rather than the strategy
+    /// @return collection of `AbstractCompactionTask`, which could be either a `CompactionTask` or a `UnifiedCompactionTask`
+    public synchronized Collection<AbstractCompactionTask> getNextBackgroundTasks(Collection<CompactionAggregate> aggregates, long gcBefore)
     {
-        Preconditions.checkNotNull(pick);
-        Preconditions.checkArgument(!pick.isEmpty());
+        controller.onStrategyBackgroundTaskRequest();
+        return createCompactionTasks(aggregates, gcBefore);
+    }
 
-        LifecycleTransaction transaction = cfs.getTracker().tryModify(pick,
-                                                                      OperationType.COMPACTION);
+    private Collection<AbstractCompactionTask> createCompactionTasks(Collection<CompactionAggregate> aggregates, long gcBefore)
+    {
+        Collection<AbstractCompactionTask> tasks = new ArrayList<>(aggregates.size());
+        try
+        {
+            for (CompactionAggregate aggregate : aggregates)
+                createAndAddTasks(gcBefore, (CompactionAggregate.UnifiedAggregate) aggregate, tasks);
+
+            return tasks;
+        }
+        catch (Throwable t)
+        {
+            throw rejectTasks(tasks, t);
+        }
+    }
+
+    /// Create compaction tasks for the given aggregate and add them to the given tasks list.
+    public void createAndAddTasks(long gcBefore, CompactionAggregate.UnifiedAggregate aggregate, Collection<? super UnifiedCompactionTask> tasks)
+    {
+        CompactionPick selected = aggregate.getSelected();
+        int parallelism = aggregate.getPermittedParallelism();
+        Preconditions.checkNotNull(selected);
+        Preconditions.checkArgument(!selected.isEmpty());
+
+        LifecycleTransaction transaction = realm.tryModify(selected.sstables(),
+                                                           OperationType.COMPACTION,
+                                                           selected.id());
         if (transaction != null)
         {
-            return createCompactionTask(transaction, gcBefore);
+            // This will ignore the range of the operation, which is fine.
+            backgroundCompactions.setSubmitted(this, transaction.opId(), aggregate);
+            createAndAddTasks(gcBefore, transaction, aggregate.operationRange(), aggregate.keepOriginals(), getShardingStats(aggregate), parallelism, tasks);
         }
         else
         {
-            // This can happen e.g. due to a race with upgrade tasks.
-            logger.warn("Failed to submit compaction {} because a transaction could not be created. If this happens frequently, it should be reported", pick);
-            // This may be an indication of an SSTableReader reference leak. See CASSANDRA-18342.
-            return null;
+            // This can happen e.g. due to a race with upgrade tasks
+            logger.error("Failed to submit compaction {} because a transaction could not be created. If this happens frequently, it should be reported", aggregate);
         }
     }
 
-    /**
-     * Create the sstable writer used for flushing.
-     *
-     * @return an sstable writer that will split sstables into a number of shards as calculated by the controller for
-     *         the expected flush density.
-     */
+    private static RuntimeException rejectTasks(Iterable<? extends AbstractCompactionTask> tasks, Throwable error)
+    {
+        for (var task : tasks)
+            error = task.rejected(error);
+        throw Throwables.throwAsUncheckedException(error);
+    }
+
+    public static class ShardingStats
+    {
+        public final PartitionPosition min;
+        public final PartitionPosition max;
+        public final long totalOnDiskSize;
+        public final double overheadToDataRatio;
+        public final double uniqueKeyRatio;
+        public final double density;
+        public final int shardCountForDensity;
+        public final int coveredShardCount;
+
+        public ShardingStats(Collection<? extends CompactionSSTable> sstables, ShardManager shardManager, Controller controller)
+        {
+            this(sstables, shardManager, getOverheadToDataRatio(sstables, controller), controller);
+        }
+
+        /// Construct sharding statistics for the given collection of sstables that are to be compacted in full.
+        public ShardingStats(Collection<? extends CompactionSSTable> sstables, ShardManager shardManager, double overheadToDataRatio, Controller controller)
+        {
+            assert !sstables.isEmpty();
+            // the partition count aggregation is costly, so we only perform this once when the aggregate is selected for execution.
+            long onDiskLength = 0;
+            long partitionCountSum = 0;
+            PartitionPosition min = null;
+            PartitionPosition max = null;
+            boolean hasOnlySSTableReaders = true;
+            for (CompactionSSTable sstable : sstables)
+            {
+                onDiskLength += sstable.onDiskLength();
+                partitionCountSum += sstable.estimatedKeys();
+                min = min == null || min.compareTo(sstable.getFirst()) > 0 ? sstable.getFirst() : min;
+                max = max == null || max.compareTo(sstable.getLast()) < 0 ? sstable.getLast() : max;
+                if (!(sstable instanceof SSTableReader)
+                    || ((SSTableReader) sstable).descriptor == null)    // for tests
+                    hasOnlySSTableReaders = false;
+            }
+            long estimatedPartitionCount;
+            if (hasOnlySSTableReaders)
+                estimatedPartitionCount = SSTableReader.getApproximateKeyCount(Iterables.filter(sstables, SSTableReader.class));
+            else
+                estimatedPartitionCount = partitionCountSum;
+
+            this.totalOnDiskSize = onDiskLength;
+            this.overheadToDataRatio = overheadToDataRatio;
+            this.uniqueKeyRatio = 1.0 * estimatedPartitionCount / partitionCountSum;
+            this.min = min;
+            this.max = max;
+            this.density = shardManager.density(onDiskLength, min, max, estimatedPartitionCount);
+            this.shardCountForDensity = controller.getNumShards(this.density * shardManager.shardSetCoverage());
+            this.coveredShardCount = shardManager.coveredShardCount(min, max, shardCountForDensity);
+        }
+
+        /// Construct sharding statistics for the given collection of sstables that are to be partially compacted
+        /// in the given operation range. Done by adjusting numbers by the fraction of the sstable that is in range.
+        public ShardingStats(Collection<? extends CompactionSSTable> sstables, Range<Token> operationRange, ShardManager shardManager, double overheadToDataRatio, Controller controller)
+        {
+            assert !sstables.isEmpty();
+            assert operationRange != null;
+            long onDiskLengthInRange = 0;
+            long partitionCountSum = 0;
+            long partitionCountSumInRange = 0;
+            PartitionPosition min = null;
+            PartitionPosition max = null;
+            boolean hasOnlySSTableReaders = true;
+            for (CompactionSSTable sstable : sstables)
+            {
+                PartitionPosition left = sstable.getFirst();
+                PartitionPosition right = sstable.getLast();
+                boolean extendsBefore = left.getToken().compareTo(operationRange.left) <= 0;
+                boolean extendsAfter = !operationRange.right.isMinimum() && right.getToken().compareTo(operationRange.right) > 0;
+                if (extendsBefore)
+                    left = operationRange.left.nextValidToken().minKeyBound();
+                if (extendsAfter)
+                    right = operationRange.right.maxKeyBound();
+                double fractionInRange = extendsBefore || extendsAfter
+                                         ? shardManager.rangeSpanned(left, right) / shardManager.rangeSpanned(sstable.getFirst(), sstable.getLast())
+                                         : 1;
+
+                onDiskLengthInRange += (long) (sstable.onDiskLength() * fractionInRange);
+                partitionCountSumInRange += (long) (sstable.estimatedKeys() * fractionInRange);
+                partitionCountSum += sstable.estimatedKeys();
+                min = min == null || min.compareTo(left) > 0 ? left : min;
+                max = max == null || max.compareTo(right) < 0 ? right : max;
+                if (!(sstable instanceof SSTableReader)
+                    || ((SSTableReader) sstable).descriptor == null)    // for tests
+                    hasOnlySSTableReaders = false;
+            }
+            long estimatedPartitionCount;
+            if (hasOnlySSTableReaders)
+                estimatedPartitionCount = SSTableReader.getApproximateKeyCount(Iterables.filter(sstables, SSTableReader.class));
+            else
+                estimatedPartitionCount = partitionCountSum;
+
+            this.min = min;
+            this.max = max;
+            this.totalOnDiskSize = onDiskLengthInRange;
+            this.overheadToDataRatio = overheadToDataRatio;
+            this.uniqueKeyRatio = 1.0 * estimatedPartitionCount / partitionCountSum;
+            this.density = shardManager.density(onDiskLengthInRange, min, max, (long) (partitionCountSumInRange * uniqueKeyRatio));
+            this.shardCountForDensity = controller.getNumShards(this.density * shardManager.shardSetCoverage());
+            this.coveredShardCount = shardManager.coveredShardCount(min, max, shardCountForDensity);
+        }
+
+        /// Testing only, use specified values.
+        @VisibleForTesting
+        ShardingStats(PartitionPosition min, PartitionPosition max, long totalOnDiskSize, double overheadToDataRatio, double uniqueKeyRatio, double density, int shardCountForDensity, int coveredShardCount)
+        {
+
+            this.min = min;
+            this.max = max;
+            this.totalOnDiskSize = totalOnDiskSize;
+            this.overheadToDataRatio = overheadToDataRatio;
+            this.uniqueKeyRatio = uniqueKeyRatio;
+            this.density = density;
+            this.shardCountForDensity = shardCountForDensity;
+            this.coveredShardCount = coveredShardCount;
+        }
+    }
+
+    /// Get and store the sharding stats for a given aggregate
+    public ShardingStats getShardingStats(CompactionAggregate.UnifiedAggregate aggregate)
+    {
+        var shardingStats = aggregate.getShardingStats();
+        if (shardingStats == null)
+        {
+            final Range<Token> operationRange = aggregate.operationRange();
+            shardingStats = operationRange != null
+                            ? new ShardingStats(aggregate.getSelected().sstables(), operationRange, getShardManager(), aggregate.getSelected().overheadToDataRatio(), controller)
+                            : new ShardingStats(aggregate.getSelected().sstables(), getShardManager(), aggregate.getSelected().overheadToDataRatio(), controller);
+            aggregate.setShardingStats(shardingStats);
+        }
+        return shardingStats;
+    }
+
+    ShardingStats makeShardingStats(ILifecycleTransaction txn)
+    {
+        return makeShardingStats(txn.originals());
+    }
+
+    ShardingStats makeShardingStats(Collection<? extends CompactionSSTable> sstables)
+    {
+        return new ShardingStats(sstables, getShardManager(), controller);
+    }
+
+    static double getOverheadToDataRatio(Collection<? extends CompactionSSTable> sstables, Controller controller)
+    {
+        final long totSizeBytes = CompactionAggregate.getTotSizeBytes(sstables);
+        return controller.getOverheadSizeInBytes(sstables, totSizeBytes) / Math.max(1.0, totSizeBytes);
+    }
+
+    void createAndAddTasks(long gcBefore,
+                           LifecycleTransaction transaction,
+                           ShardingStats shardingStats,
+                           int parallelism,
+                           Collection<? super CompactionTask> tasks)
+    {
+        createAndAddTasks(gcBefore, transaction, null, false, shardingStats, parallelism, tasks);
+    }
+
+    @VisibleForTesting
+    void createAndAddTasks(long gcBefore,
+                           LifecycleTransaction transaction,
+                           Range<Token> operationRange,
+                           boolean keepOriginals,
+                           ShardingStats shardingStats,
+                           int parallelism,
+                           Collection<? super UnifiedCompactionTask> tasks)
+    {
+        if (controller.parallelizeOutputShards() && parallelism > 1)
+            tasks.addAll(createParallelCompactionTasks(transaction, operationRange, keepOriginals, shardingStats, gcBefore, parallelism));
+        else
+            tasks.add(createCompactionTask(transaction, operationRange, keepOriginals, shardingStats, gcBefore));
+    }
+
+    /// Create the sstable writer used for flushing.
+    ///
+    /// @return an sstable writer that will split sstables into a number of shards as calculated by the controller for
+    ///         the expected flush density.
     @Override
     public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
                                                        long keyCount,
@@ -269,9 +646,9 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
                                                        LifecycleNewTracker lifecycleNewTracker)
     {
         ShardManager shardManager = getShardManager();
-        double flushDensity = cfs.metric.flushSizeOnDisk.get() * shardManager.shardSetCoverage() / shardManager.localSpaceCoverage();
-        ShardTracker boundaries = shardManager.boundaries(controller.getNumShards(flushDensity));
-        return new ShardedMultiWriter(cfs,
+        double flushDensity = realm.metrics().flushSizeOnDisk().get() * shardManager.shardSetCoverage() / shardManager.localSpaceCoverage();
+        ShardTracker boundaries = shardManager.boundaries(controller.getFlushShards(flushDensity));
+        return new ShardedMultiWriter(realm,
                                       descriptor,
                                       keyCount,
                                       repairedAt,
@@ -284,113 +661,454 @@ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
                                       boundaries);
     }
 
-    /**
-     * Create the task that in turns creates the sstable writer used for compaction.
-     *
-     * @return a sharded compaction task that in turn will create a sharded compaction writer.
-     */
-    private UnifiedCompactionTask createCompactionTask(LifecycleTransaction transaction, long gcBefore)
+    /// Create the task that in turns creates the sstable writer used for compaction.
+    ///
+    /// @return a sharded compaction task that in turn will create a sharded compaction writer.
+    private UnifiedCompactionTask createCompactionTask(LifecycleTransaction transaction, ShardingStats shardingStats, long gcBefore)
+    {
+        return new UnifiedCompactionTask(realm, this, transaction, gcBefore, getShardManager(), shardingStats);
+    }
+
+    /// Create the task that in turns creates the sstable writer used for compaction. This version is for a ranged task,
+    /// where we produce outputs but cannot delete the input sstables until all components of the operation are complete.
+    ///
+    /// @return a sharded compaction task that in turn will create a sharded compaction writer.
+    private UnifiedCompactionTask createCompactionTask(LifecycleTransaction transaction, Range<Token> operationRange, boolean keepOriginals, ShardingStats shardingStats, long gcBefore)
+    {
+        return new UnifiedCompactionTask(realm, this, transaction, gcBefore, keepOriginals, getShardManager(), shardingStats, operationRange, transaction.originals(), null, null, null);
+    }
+
+    @Override
+    protected UnifiedCompactionTask createCompactionTask(final long gcBefore, LifecycleTransaction txn, boolean isMaximal, boolean splitOutput)
+    {
+        return createCompactionTask(txn, makeShardingStats(txn), gcBefore);
+    }
+
+    @Override
+    public UnifiedCompactionTask createCompactionTask(LifecycleTransaction txn, final long gcBefore, long maxSSTableBytes)
+    {
+        return createCompactionTask(txn, makeShardingStats(txn), gcBefore);
+    }
+
+    /// Create a collection of parallelized compaction tasks that perform the compaction in parallel.
+    private Collection<UnifiedCompactionTask> createParallelCompactionTasks(LifecycleTransaction transaction,
+                                                                            Range<Token> operationRange,
+                                                                            boolean keepOriginals,
+                                                                            ShardingStats shardingStats,
+                                                                            long gcBefore,
+                                                                            int parallelism)
+    {
+        final int coveredShardCount = shardingStats.coveredShardCount;
+        assert parallelism > 1;
+
+        Collection<SSTableReader> sstables = transaction.originals();
+        ShardManager shardManager = getShardManager();
+        CompositeLifecycleTransaction compositeTransaction = new CompositeLifecycleTransaction(transaction);
+        SharedCompactionProgress sharedProgress = new SharedCompactionProgress(transaction.opId(), transaction.opType(), TableOperation.Unit.BYTES);
+        SharedCompactionObserver sharedObserver = new SharedCompactionObserver(this);
+        SharedTableOperation sharedOperation = new SharedTableOperation(sharedProgress);
+        List<UnifiedCompactionTask> tasks = shardManager.splitSSTablesInShardsLimited(
+            sstables,
+            operationRange,
+            shardingStats.shardCountForDensity,
+            shardingStats.coveredShardCount,
+            parallelism,
+            (rangeSSTables, range) -> new UnifiedCompactionTask(realm,
+                                                                this,
+                                                                new PartialLifecycleTransaction(compositeTransaction),
+                                                                gcBefore,
+                                                                keepOriginals,
+                                                                shardManager,
+                                                                shardingStats,
+                                                                range,
+                                                                rangeSSTables,
+                                                                sharedProgress,
+                                                                sharedObserver,
+                                                                sharedOperation)
+        );
+        compositeTransaction.completeInitialization();
+        assert tasks.size() <= parallelism;
+        assert tasks.size() <= coveredShardCount;
+
+        if (tasks.isEmpty())
+            transaction.close(); // this should not be reachable normally, close the transaction for safety
+
+        if (tasks.size() == 1) // if there's just one range, make it a non-ranged task (to apply early open etc.)
+        {
+            assert tasks.get(0).inputSSTables().equals(sstables);
+            return Collections.singletonList(createCompactionTask(transaction, operationRange, keepOriginals, shardingStats, gcBefore));
+        }
+        else
+            return tasks;
+    }
+
+    private ExpirationTask createExpirationTask(LifecycleTransaction transaction)
     {
-        return new UnifiedCompactionTask(cfs, this, transaction, gcBefore, getShardManager());
+        return new ExpirationTask(realm, transaction);
     }
 
-    private void maybeUpdateShardManager()
+    private void maybeUpdateSelector()
     {
-        if (shardManager != null && !shardManager.isOutOfDate(StorageService.instance.getTokenMetadata().getRingVersion()))
+        if (currentArenaSelector != null && !currentArenaSelector.diskBoundaries.isOutOfDate())
             return; // the disk boundaries (and thus the local ranges too) have not changed since the last time we calculated
 
         synchronized (this)
         {
-            // Recheck after entering critical section, another thread may have beaten us to it.
-            while (shardManager == null || shardManager.isOutOfDate(StorageService.instance.getTokenMetadata().getRingVersion()))
-                shardManager = ShardManager.create(cfs);
+            if (currentArenaSelector != null && !currentArenaSelector.diskBoundaries.isOutOfDate())
+                return; // another thread beat us to the update
+
+            DiskBoundaries currentBoundaries = realm.getDiskBoundaries();
+            var maybeShardManager = realm.buildShardManager();
+            currentShardManager = maybeShardManager != null
+                           ? maybeShardManager
+                           : ShardManager.create(currentBoundaries, realm.getKeyspaceReplicationStrategy(), controller.isReplicaAware());
+            currentArenaSelector = new ArenaSelector(controller, currentBoundaries);
             // Note: this can just as well be done without the synchronization (races would be benign, just doing some
             // redundant work). For the current usages of this blocking is fine and expected to perform no worse.
         }
     }
 
-    @VisibleForTesting
-    ShardManager getShardManager()
+    /// Get the current shard manager. Used internally, in tests and by CNDB.
+    public ShardManager getShardManager()
     {
-        maybeUpdateShardManager();
-        return shardManager;
+        maybeUpdateSelector();
+        return currentShardManager;
     }
 
-    /**
-     * Selects a compaction to run next.
-     */
-    @VisibleForTesting
-    CompactionPick getNextCompactionPick(long gcBefore)
+    ArenaSelector getArenaSelector()
     {
-        SelectionContext context = new SelectionContext(controller);
-        List<SSTableReader> suitable = getCompactableSSTables(getSSTables(), UnifiedCompactionStrategy::isSuitableForCompaction);
-        Set<SSTableReader> expired = maybeGetExpiredSSTables(gcBefore, suitable);
-        suitable.removeAll(expired);
+        maybeUpdateSelector();
+        return currentArenaSelector;
+    }
 
-        CompactionPick selected = chooseCompactionPick(suitable, context);
-        estimatedRemainingTasks = context.estimatedRemainingTasks;
-        if (selected == null)
+    private CompactionLimits getCurrentLimits(int maxConcurrentCompactions)
+    {
+        // Calculate the running compaction limits, i.e. the overall number of compactions permitted, which is either
+        // the compaction thread count, or the compaction throughput divided by the compaction rate (to prevent slowing
+        // down individual compaction progress).
+        String rateLimitLog = "";
+
+        // identify space limit
+        long spaceOverheadLimit = controller.maxCompactionSpaceBytes();
+
+        // identify throughput limit
+        double throughputLimit = controller.maxThroughput();
+        int maxCompactions;
+        if (throughputLimit < Double.MAX_VALUE)
         {
-            if (expired.isEmpty())
-                return null;
+            int maxCompactionsForThroughput;
+
+            double compactionRate = backgroundCompactions.compactionRate.get();
+            if (compactionRate > 0)
+            {
+                // Start as many as can saturate the limit, making sure to also account for compactions that have
+                // already been started but don't have progress yet.
+
+                // Note: the throughput limit is adjusted here because the limiter won't let compaction proceed at more
+                // than the given rate, and small hiccups or rounding errors could cause this to go above the current
+                // running count when we are already at capacity.
+                // Allow up to 5% variability, or if we are permitted more than 20 concurrent compactions, one/maxcount
+                // so that we don't issue less tasks than we should.
+                double adjustment = Math.min(0.05, 1.0 / maxConcurrentCompactions);
+                maxCompactionsForThroughput = (int) Math.ceil(throughputLimit * (1 - adjustment) / compactionRate);
+            }
             else
-                return new CompactionPick(-1, -1, expired);
+            {
+                // If we don't have running compactions we don't know the effective rate.
+                // Allow only one compaction; this will be called again soon enough to recheck.
+                maxCompactionsForThroughput = 1;
+            }
+
+            rateLimitLog = String.format(" rate-based limit %d (rate %s/%s)",
+                                         maxCompactionsForThroughput,
+                                         FBUtilities.prettyPrintMemoryPerSecond((long) compactionRate),
+                                         FBUtilities.prettyPrintMemoryPerSecond((long) throughputLimit));
+            maxCompactions = Math.min(maxConcurrentCompactions, maxCompactionsForThroughput);
+        }
+        else
+            maxCompactions = maxConcurrentCompactions;
+
+        // Now that we have a count, make sure it is spread close to equally among levels. In other words, reserve
+        // floor(permitted / levels) compactions for each level and don't permit more than ceil(permitted / levels) on
+        // any, to make sure that no level hogs all threads and thus lowest-level ops (which need to run more often but
+        // complete quickest) have a chance to run frequently. Also, running compactions can't go above the specified
+        // space overhead limit.
+        // To do this we count the number and size of already running compactions on each level and make sure any new
+        // ones we select satisfy these constraints.
+        int[] perLevel = new int[MAX_LEVELS];
+        int levelCount = 1; // Start at 1 to avoid division by zero if the aggregates list is empty.
+        int runningCompactions = 0;
+        long spaceAvailable = spaceOverheadLimit;
+        int remainingAdaptiveCompactions = controller.getMaxRecentAdaptiveCompactions(); //limit for number of compactions triggered by new W value
+        if (remainingAdaptiveCompactions == -1)
+            remainingAdaptiveCompactions = Integer.MAX_VALUE;
+        for (CompactionPick compaction : backgroundCompactions.getCompactionsInProgress())
+        {
+            final int level = levelOf(compaction);
+            if (level < 0)  // expire-only compactions are allowed to run outside of the limits
+                continue;
+            ++perLevel[level];
+            ++runningCompactions;
+            levelCount = Math.max(levelCount, level + 1);
+            spaceAvailable -= compaction.totalOverheadInBytes();
+            if (controller.isRecentAdaptive(compaction))
+                --remainingAdaptiveCompactions;
         }
 
-        selected.addAll(expired);
-        return selected;
+        CompactionLimits limits = new CompactionLimits(runningCompactions,
+                                                       maxCompactions,
+                                                       maxConcurrentCompactions,
+                                                       perLevel,
+                                                       levelCount,
+                                                       spaceAvailable,
+                                                       remainingAdaptiveCompactions);
+        logger.trace("Selecting up to {} new compactions of up to {}, concurrency limit {}{}",
+                     Math.max(0, limits.maxCompactions - limits.runningCompactions),
+                     FBUtilities.prettyPrintMemory(limits.spaceAvailable),
+                     limits.maxConcurrentCompactions,
+                     rateLimitLog);
+        return limits;
     }
 
-    private Set<SSTableReader> maybeGetExpiredSSTables(long gcBefore, List<SSTableReader> suitable)
+    private Collection<CompactionAggregate> updateLevelCountWithParentAndGetSelection(final CompactionLimits limits,
+                                                                                      List<CompactionAggregate.UnifiedAggregate> pending)
     {
-        Set<SSTableReader> expired;
-        long ts = Clock.Global.currentTimeMillis();
-        if (ts - lastExpiredCheck > controller.getExpiredSSTableCheckFrequency())
+        long totalCompactionLimit = controller.maxCompactionSpaceBytes();
+        int levelCount = limits.levelCount;
+        for (CompactionAggregate.UnifiedAggregate aggregate : pending)
         {
-            lastExpiredCheck = ts;
-            expired = CompactionController.getFullyExpiredSSTables(cfs,
-                                                                   suitable,
-                                                                   cfs.getOverlappingLiveSSTables(suitable),
-                                                                   gcBefore,
-                                                                   controller.getIgnoreOverlapsInExpirationCheck());
-            if (logger.isTraceEnabled() && !expired.isEmpty())
-                logger.trace("Expiration check for {}.{} found {} fully expired SSTables",
-                             cfs.getKeyspaceName(),
-                             cfs.getTableName(),
-                             expired.size());
+            warnIfSizeAbove(aggregate, totalCompactionLimit);
+
+            // Make sure the level count includes all levels for which we have sstables (to be ready to compact
+            // as soon as the threshold is crossed)...
+            levelCount = Math.max(levelCount, aggregate.bucketIndex() + 1);
+            CompactionPick selected = aggregate.getSelected();
+            if (selected != null)
+            {
+                // ... and also the levels that a layout-preserving selection would create.
+                levelCount = Math.max(levelCount, levelOf(selected) + 1);
+            }
         }
-        else
-            expired = Collections.emptySet();
-        return expired;
+        int[] perLevel = limits.perLevel;
+        if (levelCount != perLevel.length)
+            perLevel = Arrays.copyOf(perLevel, levelCount);
+
+        return getSelection(pending,
+                            limits.maxCompactions,
+                            perLevel,
+                            limits.spaceAvailable,
+                            limits.remainingAdaptiveCompactions);
+    }
+
+    /// Selects compactions to run next.
+    ///
+    /// @return a subset of compaction aggregates to run next
+    private Collection<CompactionAggregate> getNextCompactionAggregates()
+    {
+        final CompactionLimits limits = getCurrentLimits(controller.maxConcurrentCompactions());
+
+        List<CompactionAggregate.UnifiedAggregate> pending = getPendingCompactionAggregates(limits.spaceAvailable);
+        setPendingCompactionAggregates(pending);
+
+        return updateLevelCountWithParentAndGetSelection(limits, pending);
+    }
+
+    /// Selects compactions to run next from the passed aggregates.
+    ///
+    /// The intention here is to use this method directly from outside processes, to run compactions from a set
+    /// of pre-existing aggregates, that have been generated out of process.
+    ///
+    /// @param aggregates a collection of aggregates from which to select the next compactions
+    /// @param maxConcurrentCompactions the maximum number of concurrent compactions
+    /// @return a subset of compaction aggregates to run next
+    public Collection<CompactionAggregate> getNextCompactionAggregates(Collection<CompactionAggregate.UnifiedAggregate> aggregates,
+                                                                       int maxConcurrentCompactions)
+    {
+        final CompactionLimits limits = getCurrentLimits(maxConcurrentCompactions);
+        maybeUpdateSelector();
+        return updateLevelCountWithParentAndGetSelection(limits, new ArrayList<>(aggregates));
+    }
+
+    /// Returns all pending compaction aggregates.
+    ///
+    /// This method is used by CNDB to find all pending compactions and put them to etcd.
+    ///
+    /// @return all pending compaction aggregates
+    public Collection<CompactionAggregate.UnifiedAggregate> getPendingCompactionAggregates()
+    {
+        return getPendingCompactionAggregates(controller.maxCompactionSpaceBytes());
+    }
+
+    /// Set the compaction aggregates passed in as pending in [BackgroundCompactions]. This ensures
+    /// that the compaction statistics will be accurate.
+    ///
+    /// This is called by [#getNextCompactionAggregates()]
+    /// and externally after calling [#getPendingCompactionAggregates()]
+    /// or before submitting tasks.
+    ///
+    /// Also, note that skipping the call to [#setPending(CompactionStrategy,Collection)]
+    /// would result in memory leaks: the aggregates added in [#setSubmitted(CompactionStrategy,TimeUUID,CompactionAggregate)]
+    /// would never be removed, and the aggregates hold references to the compaction tasks, so they retain a significant
+    /// size of heap memory.
+    ///
+    /// @param pending the aggregates that should be set as pending compactions
+    public void setPendingCompactionAggregates(Collection<? extends CompactionAggregate> pending)
+    {
+        backgroundCompactions.setPending(this, pending);
+    }
+
+    private List<CompactionAggregate.UnifiedAggregate> getPendingCompactionAggregates(long spaceAvailable)
+    {
+        maybeUpdateSelector();
+
+        List<CompactionAggregate.UnifiedAggregate> pending = new ArrayList<>();
+
+        for (Map.Entry<Arena, List<Level>> entry : getLevels().entrySet())
+        {
+            Arena arena = entry.getKey();
+
+            for (Level level : entry.getValue())
+            {
+                Collection<CompactionAggregate.UnifiedAggregate> aggregates = level.getCompactionAggregates(arena, controller, spaceAvailable);
+                // Note: We allow empty aggregates into the list of pending compactions. The pending compactions list
+                // is for progress tracking only, and it is helpful to see empty levels there.
+                pending.addAll(aggregates);
+            }
+        }
+
+        return pending;
+    }
+
+    /// This method logs a warning related to the fact that the space overhead limit also applies when a
+    /// single compaction is above that limit. This should prevent running out of space at the expense of ending up
+    /// with several extra sstables at the highest-level (compared to the number of sstables that we should have
+    /// as per config of the strategy), i.e. slightly higher read amplification. This is a sensible tradeoff but
+    /// the operators must be warned if this happens, and that's the purpose of this warning.
+    private void warnIfSizeAbove(CompactionAggregate.UnifiedAggregate aggregate, long spaceOverheadLimit)
+    {
+        if (aggregate.getSelected().totalOverheadInBytes() > spaceOverheadLimit)
+            logger.warn("Compaction needs to perform an operation that is bigger than the current space overhead " +
+                        "limit - size {} (compacting {} sstables in arena {}/bucket {}); limit {} = {}% of dataset size {}. " +
+                        "To honor the limit, this operation will not be performed, which may result in degraded performance.\n" +
+                        "Please verify the compaction parameters, specifically {} and {}.",
+                        FBUtilities.prettyPrintMemory(aggregate.getSelected().totalOverheadInBytes()),
+                        aggregate.getSelected().sstables().size(),
+                        aggregate.getArena().name(),
+                        aggregate.bucketIndex(),
+                        FBUtilities.prettyPrintMemory(spaceOverheadLimit),
+                        controller.getMaxSpaceOverhead() * 100,
+                        FBUtilities.prettyPrintMemory(controller.getDataSetSizeBytes()),
+                        Controller.DATASET_SIZE_OPTION,
+                        Controller.MAX_SPACE_OVERHEAD_OPTION);
     }
 
-    private CompactionPick chooseCompactionPick(List<SSTableReader> suitable, SelectionContext context)
+    /// Returns a selection of the compactions to be submitted. The selection will be chosen so that the total
+    /// number of compactions is at most totalCount, where each level gets a share that is the whole part of the ratio
+    /// between the total permitted number of compactions, and the remainder gets distributed among the levels
+    /// according to the preferences of the [#prioritize] method. Usually this means preferring
+    /// compaction picks with a higher max overlap, with a random selection when multiple picks have the same maximum.
+    /// Note that if a level does not have tasks to fill its share, its quota will remain unused in this
+    /// allocation.
+    ///
+    /// The selection also limits the size of the newly scheduled compactions to be below spaceAvailable by not
+    /// scheduling compactions if they would push the combined size above that limit.
+    ///
+    /// @param pending list of all current aggregates with possible selection for each bucket
+    /// @param totalCount maximum number of compactions permitted to run
+    /// @param perLevel int array with the number of in-progress compactions per level
+    /// @param spaceAvailable amount of space in bytes available for the new compactions
+    /// @param remainingAdaptiveCompactions number of adaptive compactions (i.e. ones triggered by scaling parameter
+    ///                                     change by the adaptive controller) that can still be scheduled
+    List<CompactionAggregate> getSelection(List<CompactionAggregate.UnifiedAggregate> pending,
+                                           int totalCount,
+                                           int[] perLevel,
+                                           long spaceAvailable,
+                                           int remainingAdaptiveCompactions)
     {
-        // Select the level with the highest overlap; when multiple levels have the same overlap, prefer the lower one
-        // (i.e. reduction of RA for bigger token coverage).
-        int maxOverlap = -1;
-        CompactionPick selected = null;
-        for (Level level : formLevels(suitable))
+        Controller controller = getController();
+        Reservations reservations = Reservations.create(totalCount,
+                                                        perLevel,
+                                                        controller.getReservedThreads(),
+                                                        controller.getReservationsType());
+        // If the inclusion method is not transitive, we may have multiple buckets/selections for the same sstable.
+        boolean shouldCheckSSTableSelected = controller.overlapInclusionMethod() != Overlaps.InclusionMethod.TRANSITIVE;
+        // If so, make sure we only select one such compaction.
+        Set<CompactionSSTable> selectedSSTables = shouldCheckSSTableSelected ? new HashSet<>() : null;
+
+        int remaining = totalCount;
+        for (int countInLevel : perLevel)
+            remaining -= countInLevel;
+
+        // Note: if we are in the middle of changes in the parameters or level count, remainder might become negative.
+        // This is okay, some buckets will temporarily not get their rightful share until these tasks complete.
+
+        // Let the controller prioritize the compactions.
+        pending = controller.prioritize(pending);
+        int proposed = 0;
+
+        // Select the first ones, permitting only the specified number per level.
+        List<CompactionAggregate> selected = new ArrayList<>(pending.size());
+        for (CompactionAggregate.UnifiedAggregate aggregate : pending)
         {
-            CompactionPick pick = level.getCompactionPick(context);
-            int levelOverlap = level.maxOverlap;
-            if (levelOverlap > maxOverlap)
+            if (remaining == 0)
+                break; // no threads to allocate from
+
+            final CompactionPick pick = aggregate.getSelected();
+            if (pick.isEmpty())
+                continue;
+
+            ++proposed;
+            long overheadSizeInBytes = pick.totalOverheadInBytes();
+            if (overheadSizeInBytes > spaceAvailable)
+                continue; // compaction is too large for current cycle
+
+            int currentLevel = levelOf(pick);
+            boolean isAdaptive = controller.isRecentAdaptive(pick);
+            // avoid computing sharding stats if are not going to schedule the compaction at all
+            if (!reservations.hasRoom(currentLevel))
+                continue;  // honor the reserved thread counts
+            if (isAdaptive && remainingAdaptiveCompactions <= 0)
+                continue; // do not allow more than remainingAdaptiveCompactions to limit latency spikes upon changing W
+            if (shouldCheckSSTableSelected && !Collections.disjoint(selectedSSTables, pick.sstables()))
+                continue; // do not allow multiple selections of the same sstable
+
+            int parallelism = controller.parallelizeOutputShards() ? getShardingStats(aggregate).coveredShardCount : 1;
+            if (parallelism > remaining)
+                parallelism = remaining;
+            assert currentLevel >= 0 : "Invalid level in " + pick;
+
+            if (isAdaptive)
             {
-                maxOverlap = levelOverlap;
-                selected = pick;
+                if (parallelism > remainingAdaptiveCompactions)
+                {
+                    parallelism = remainingAdaptiveCompactions;
+                    assert parallelism > 0; // we checked the remainingAdaptiveCompactions in advance
+                }
             }
+
+            parallelism = reservations.accept(currentLevel, parallelism);
+            assert parallelism > 0; // we checked hasRoom in advance, there must always be at least one thread to use
+
+            // Note: the reservations tracker assumes it is the last check and a pick is accepted if it returns true.
+
+            if (isAdaptive)
+                remainingAdaptiveCompactions -= parallelism;
+            remaining -= parallelism;
+            spaceAvailable -= overheadSizeInBytes;
+            aggregate.setPermittedParallelism(parallelism);
+            selected.add(aggregate);
+            if (shouldCheckSSTableSelected)
+                selectedSSTables.addAll(pick.sstables());
         }
-        if (logger.isDebugEnabled() && selected != null)
-            logger.debug("Selected compaction on level {} overlap {} sstables {}",
-                         selected.level, selected.overlap, selected.size());
 
+        reservations.debugOutput(selected.size(), proposed, remaining);
         return selected;
     }
 
     @Override
     public int getEstimatedRemainingTasks()
     {
-        return estimatedRemainingTasks;
+        return backgroundCompactions.getEstimatedRemainingTasks();
     }
 
     @Override
@@ -399,133 +1117,297 @@ public long getMaxSSTableBytes()
         return Long.MAX_VALUE;
     }
 
+    @Override
+    public Set<? extends CompactionSSTable> getSSTables()
+    {
+        return realm.getLiveSSTables();
+    }
+
     @VisibleForTesting
+    public int getW(int index)
+    {
+        return controller.getScalingParameter(index);
+    }
+
     public Controller getController()
     {
         return controller;
     }
 
-    public static boolean isSuitableForCompaction(SSTableReader rdr)
+    /// Group candidate sstables into compaction arenas.
+    /// Each compaction arena is obtained by comparing using a compound comparator for the equivalence classes
+    /// configured in the arena selector of this strategy.
+    ///
+    /// @param sstables a collection of the sstables to be assigned to arenas
+    /// @param compactionFilter a bifilter (sstable, isCompacting) to include CompactionSSTables suitable for compaction
+    /// @return a list of arenas, where each arena contains sstables that belong to that arena
+    public Collection<Arena> getCompactionArenas(Collection<? extends CompactionSSTable> sstables,
+                                                 BiPredicate<CompactionSSTable, Boolean> compactionFilter)
     {
-        return !rdr.isMarkedSuspect() && rdr.openReason != SSTableReader.OpenReason.EARLY;
+        return getCompactionArenas(sstables, compactionFilter, getArenaSelector());
     }
 
-    @Override
-    public synchronized void addSSTable(SSTableReader added)
+    Collection<Arena> getCompactionArenas(Collection<? extends CompactionSSTable> sstables,
+                                          BiPredicate<CompactionSSTable, Boolean> compactionFilter,
+                                          ArenaSelector arenaSelector)
     {
-        sstables.add(added);
+        Map<CompactionSSTable, Arena> arenasBySSTables = new TreeMap<>(arenaSelector);
+        Set<? extends CompactionSSTable> compacting = realm.getCompactingSSTables();
+        for (CompactionSSTable sstable : sstables)
+            if (compactionFilter.test(sstable, compacting.contains(sstable)))
+                arenasBySSTables.computeIfAbsent(sstable, t -> new Arena(arenaSelector))
+                      .add(sstable);
+
+        return arenasBySSTables.values();
     }
 
-    @Override
-    public synchronized void removeSSTable(SSTableReader sstable)
+    @SuppressWarnings("unused") // used by CNDB to deserialize aggregates
+    public Arena getCompactionArena(Collection<? extends CompactionSSTable> sstables)
     {
-        sstables.remove(sstable);
+        Arena arena = new Arena(getArenaSelector());
+        for (CompactionSSTable table : sstables)
+            arena.add(table);
+        return arena;
     }
 
-    @Override
-    protected synchronized Set<SSTableReader> getSSTables()
+    @SuppressWarnings("unused") // used by CNDB to deserialize aggregates
+    public Level getLevel(int index, double min, double max)
     {
-        // Filter the set of sstables through the live set. This is to ensure no zombie sstables are picked for
-        // compaction (see CASSANDRA-18342).
-        return ImmutableSet.copyOf(Iterables.filter(cfs.getLiveSSTables(), sstables::contains));
+        return new Level(controller, index, min, max);
     }
 
-    /**
-     * @return a list of the levels in the compaction hierarchy
-     */
+    /// @return a LinkedHashMap of arenas with buckets where order of arenas are preserved
     @VisibleForTesting
-    List<Level> getLevels()
+    Map<Arena, List<Level>> getLevels()
     {
-        return getLevels(getSSTables(), UnifiedCompactionStrategy::isSuitableForCompaction);
+        return getLevels(realm.getLiveSSTables(), UnifiedCompactionStrategy::isSuitableForCompaction);
     }
 
-    /**
-     * Groups the sstables passed in into levels. This is used by the strategy to determine
-     * new compactions, and by external tools to analyze the strategy decisions.
-     *
-     * @param sstables a collection of the sstables to be assigned to levels
-     * @param compactionFilter a filter to exclude CompactionSSTables,
-     *                         e.g., {@link #isSuitableForCompaction}
-     *
-     * @return a list of the levels in the compaction hierarchy
-     */
-    public List<Level> getLevels(Collection<SSTableReader> sstables,
-                                 Predicate<SSTableReader> compactionFilter)
+    private static boolean isSuitableForCompaction(CompactionSSTable sstable, boolean isCompacting)
     {
-        List<SSTableReader> suitable = getCompactableSSTables(sstables, compactionFilter);
-        return formLevels(suitable);
+        return sstable.isSuitableForCompaction() && !isCompacting;
     }
 
-    private List<Level> formLevels(List<SSTableReader> suitable)
+    Iterable<? extends CompactionSSTable> getSuitableSSTables()
     {
-        maybeUpdateShardManager();
-        List<Level> levels = new ArrayList<>(MAX_LEVELS);
-        suitable.sort(shardManager::compareByDensity);
+        return getFilteredSSTables(UnifiedCompactionStrategy::isSuitableForCompaction);
+    }
 
-        double maxDensity = controller.getMaxLevelDensity(0, controller.getBaseSstableSize(controller.getFanout(0)) / shardManager.localSpaceCoverage());
-        int index = 0;
-        Level level = new Level(controller, index, 0, maxDensity);
-        for (SSTableReader candidate : suitable)
-        {
-            final double density = shardManager.density(candidate);
-            if (density < level.max)
-            {
-                level.add(candidate);
-                continue;
-            }
+    Iterable<? extends CompactionSSTable> getFilteredSSTables(BiPredicate<CompactionSSTable, Boolean> predicate)
+    {
+        Set<? extends CompactionSSTable> compacting = realm.getCompactingSSTables();
+        return Iterables.filter(realm.getLiveSSTables(), s -> predicate.test(s, compacting.contains(s)));
+    }
 
-            level.complete();
-            levels.add(level); // add even if empty
+    /// Groups the sstables passed in into arenas and buckets. This is used by the strategy to determine
+    /// new compactions, and by external tools in CNDB to analyze the strategy decisions.
+    ///
+    /// @param sstables a collection of the sstables to be assigned to arenas
+    /// @param compactionFilter a bifilter(sstable, isCompacting) to include CompactionSSTables,
+    ///                         e.g., [#isSuitableForCompaction()]
+    ///
+    /// @return a map of arenas to their buckets
+    public Map<Arena, List<Level>> getLevels(Collection<? extends CompactionSSTable> sstables,
+                                             BiPredicate<CompactionSSTable, Boolean> compactionFilter)
+    {
+        // Copy to avoid race condition
+        var currentShardManager = getShardManager();
+        Collection<Arena> arenas = getCompactionArenas(sstables, compactionFilter);
+        Map<Arena, List<Level>> ret = new LinkedHashMap<>(); // should preserve the order of arenas
 
-            while (true)
+        for (Arena arena : arenas)
+        {
+            List<Level> levels = new ArrayList<>(MAX_LEVELS);
+
+            // Precompute the density, then sort.
+            List<SSTableWithDensity> ssTableWithDensityList = new ArrayList<>(arena.sstables.size());
+            for (CompactionSSTable sstable : arena.sstables)
+                ssTableWithDensityList.add(new SSTableWithDensity(sstable, currentShardManager.density(sstable)));
+            Collections.sort(ssTableWithDensityList);
+
+            double maxSize = controller.getMaxLevelDensity(0, controller.getBaseSstableSize(controller.getFanout(0)) / currentShardManager.localSpaceCoverage());
+            int index = 0;
+            Level level = new Level(controller, index, 0, maxSize);
+            for (SSTableWithDensity candidateWithDensity : ssTableWithDensityList)
             {
-                ++index;
-                double minDensity = maxDensity;
-                maxDensity = controller.getMaxLevelDensity(index, minDensity);
-                level = new Level(controller, index, minDensity, maxDensity);
-                if (density < level.max)
+                final CompactionSSTable candidate = candidateWithDensity.sstable;
+                final double size = candidateWithDensity.density;
+                if (size < level.max)
                 {
                     level.add(candidate);
-                    break;
+                    continue;
                 }
-                else
+
+                level.complete();
+                levels.add(level); // add even if empty
+
+                while (true)
                 {
-                    levels.add(level); // add the empty level
+                    ++index;
+                    double minSize = maxSize;
+                    maxSize = controller.getMaxLevelDensity(index, minSize);
+                    level = new Level(controller, index, minSize, maxSize);
+                    if (size < level.max)
+                    {
+                        level.add(candidate);
+                        break;
+                    }
+                    else
+                    {
+                        levels.add(level); // add the empty level
+                    }
                 }
             }
-        }
 
-        if (!level.sstables.isEmpty())
-        {
-            level.complete();
-            levels.add(level);
+            if (!level.sstables.isEmpty())
+            {
+                level.complete();
+                levels.add(level);
+            }
+
+            if (!levels.isEmpty())
+                ret.put(arena, levels);
+
+            if (logger.isTraceEnabled())
+                logger.trace("Arena {} has {} levels", arena, levels.size());
         }
 
-        return levels;
+        logger.trace("Found {} arenas with buckets for {}.{}", ret.size(), realm.getKeyspaceName(), realm.getTableName());
+        return ret;
     }
 
-    private List<SSTableReader> getCompactableSSTables(Collection<SSTableReader> sstables,
-                                                       Predicate<SSTableReader> compactionFilter)
+    /**
+     * Creates a map of maximum overlap, organized as a map from arena:level to the maximum number of sstables that
+     * overlap in that level, as well as a list showing the per-shard maximum overlap.
+     *
+     * The number of shards to list is calculated based on the maximum density of the sstables in the realm.
+     */
+    @Override
+    public Map<String, String> getMaxOverlapsMap()
     {
-        Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
-        List<SSTableReader> suitable = new ArrayList<>(sstables.size());
-        for (SSTableReader rdr : sstables)
+        final Set<? extends CompactionSSTable> liveSSTables = realm.getLiveSSTables();
+        Map<UnifiedCompactionStrategy.Arena, List<UnifiedCompactionStrategy.Level>> arenas =
+                getLevels(liveSSTables, (i1, i2) -> true); // take all sstables
+
+        ShardManager shardManager = getShardManager();
+        Map<String, String> map = new LinkedHashMap<>();
+
+        // max general overlap (max # of sstables per query)
+        map.put("all", getMaxOverlapsPerShardString(liveSSTables, shardManager));
+
+        for (var arena : arenas.entrySet())
         {
-            if (compactionFilter.test(rdr) && !compacting.contains(rdr))
-                suitable.add(rdr);
+            final String arenaName = arena.getKey().name();
+            for (var level : arena.getValue())
+                map.put(arenaName + "-L" + level.getIndex(), getMaxOverlapsPerShardString(level.getSSTables(), shardManager));
         }
-        return suitable;
+        return map;
+    }
+
+    private String getMaxOverlapsPerShardString(Collection<? extends CompactionSSTable> sstables, ShardManager shardManager)
+    {
+        // Find the sstable with the biggest density to define the shard count.
+        // This is better than using a level's max bound as that will show more shards than there actually are.
+        double maxDensity = 0;
+        for (CompactionSSTable liveSSTable : sstables)
+            maxDensity = Math.max(maxDensity, shardManager.density(liveSSTable));
+        int shardCount = controller.getNumShards(maxDensity);
+
+        int[] overlapsMap = getMaxOverlapsPerShard(sstables, shardManager, shardCount);
+        int max = 0;
+        for (int i : overlapsMap)
+            max = Math.max(max, i);
+        return max + " (per shard: " + Arrays.toString(overlapsMap) + ")";
+    }
+
+    public static int[] getMaxOverlapsPerShard(Collection<? extends CompactionSSTable> sstables, ShardManager shardManager, int shardCount)
+    {
+        int[] overlapsMap = new int[shardCount];
+        shardManager.assignSSTablesToShardIndexes(sstables, null, shardCount,
+                                                  (shardSSTables, shard) ->
+                                                  // Note: the shard index we are given is the global index, which includes
+                                                  // other arenas. The modulo below converts it to an index for the arena.
+                                                  // If an sstable extends outside a disk's region (because e.g. local
+                                                  // ownership changed and disk boundaries moved), it will be incorrectly
+                                                  // counted. This is not trivial to recognize here and is not corrected.
+                                                  overlapsMap[shard % shardCount] = Overlaps.maxOverlap(shardSSTables,
+                                                                                                        CompactionSSTable.startsAfter,
+                                                                                                        CompactionSSTable.firstKeyComparator,
+                                                                                                        CompactionSSTable.lastKeyComparator));
+        // Indexes that do not have sstables are left with 0 overlaps.
+        return overlapsMap;
+    }
+
+    private static int levelOf(CompactionPick pick)
+    {
+        return (int) pick.parent();
     }
 
     public TableMetadata getMetadata()
     {
-        return cfs.metadata();
+        return realm.metadata();
+    }
+
+    CompactionPick createPick(TimeUUID id, long parent, Collection<? extends CompactionSSTable> sstables)
+    {
+        return createPick(controller, id, parent, sstables);
+    }
+
+    static CompactionPick createPick(Controller controller, TimeUUID id, long parent, Collection<? extends CompactionSSTable> sstables)
+    {
+        long totalDataSize = CompactionAggregate.getTotSizeBytes(sstables);
+        long totalSpaceOverhead = controller.getOverheadSizeInBytes(sstables, totalDataSize);
+        return CompactionPick.create(id,
+                                     parent,
+                                     sstables,
+                                     Collections.emptyList(),
+                                     0,
+                                     totalDataSize / Math.max(sstables.size(), 1),
+                                     totalDataSize,
+                                     totalSpaceOverhead);
     }
 
-    private static boolean startsAfter(SSTableReader a, SSTableReader b)
+    /// A compaction arena contains the list of sstables that belong to this arena as well as the arena
+    /// selector used for comparison.
+    public static class Arena implements Comparable<Arena>
     {
-        // Strict comparison because the span is end-inclusive.
-        return a.getFirst().compareTo(b.getLast()) > 0;
+        final List<CompactionSSTable> sstables;
+        final ArenaSelector selector;
+
+        Arena(ArenaSelector selector)
+        {
+            this.sstables = new ArrayList<>();
+            this.selector = selector;
+        }
+
+        void add(CompactionSSTable ssTableReader)
+        {
+            sstables.add(ssTableReader);
+        }
+
+        public String name()
+        {
+            CompactionSSTable t = sstables.get(0);
+            return selector.name(t);
+        }
+
+        @Override
+        public int compareTo(Arena o)
+        {
+            return selector.compare(this.sstables.get(0), o.sstables.get(0));
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s, %d sstables", name(), sstables.size());
+        }
+
+        @VisibleForTesting
+        public List<CompactionSSTable> getSSTables()
+        {
+            return sstables;
+        }
     }
 
     @Override
@@ -534,12 +1416,10 @@ public String toString()
         return String.format("Unified strategy %s", getMetadata());
     }
 
-    /**
-     * A level: index, sstables and some properties.
-     */
+    /// A level: index, sstables and some properties.
     public static class Level
     {
-        final List<SSTableReader> sstables;
+        final List<CompactionSSTable> sstables;
         final int index;
         final double survivalFactor;
         final int scalingParameter; // scaling parameter used to calculate fanout and threshold
@@ -547,22 +1427,33 @@ public static class Level
         final int threshold; // number of SSTables that trigger a compaction
         final double min; // min density of sstables for this level
         final double max; // max density of sstables for this level
-        int maxOverlap = -1; // maximum number of overlapping sstables, i.e. maximum number of sstables that need
-                             // to be queried on this level for any given key
+        double avg = 0; // avg size of sstables in this level
+        int maxOverlap = -1; // maximum number of overlapping sstables
 
-        Level(Controller controller, int index, double minSize, double maxSize)
+        Level(int index, int scalingParameter, int fanout, int threshold, double survivalFactor, double min, double max)
         {
             this.index = index;
-            this.survivalFactor = controller.getSurvivalFactor(index);
-            this.scalingParameter = controller.getScalingParameter(index);
-            this.fanout = controller.getFanout(index);
-            this.threshold = controller.getThreshold(index);
+            this.scalingParameter = scalingParameter;
+            this.fanout = fanout;
+            this.threshold = threshold;
+            this.survivalFactor = survivalFactor;
+            this.min = min;
+            this.max = max;
             this.sstables = new ArrayList<>(threshold);
-            this.min = minSize;
-            this.max = maxSize;
         }
 
-        public Collection<SSTableReader> getSSTables()
+        Level(Controller controller, int index, double min, double max)
+        {
+            this(index,
+                 controller.getScalingParameter(index),
+                 controller.getFanout(index),
+                 controller.getThreshold(index),
+                 controller.getSurvivalFactor(index),
+                 min,
+                 max);
+        }
+
+        public Collection<CompactionSSTable> getSSTables()
         {
             return sstables;
         }
@@ -572,9 +1463,13 @@ public int getIndex()
             return index;
         }
 
-        void add(SSTableReader sstable)
+        void add(CompactionSSTable sstable)
         {
             this.sstables.add(sstable);
+            // consider size of all components to reduce chance of out-of-disk
+            long size = CassandraRelevantProperties.UCS_COMPACTION_INCLUDE_NON_DATA_FILES_SIZE.getBoolean()
+                        ? sstable.onDiskComponentsSize() : sstable.onDiskLength();
+            this.avg += (size - avg) / sstables.size();
         }
 
         void complete()
@@ -583,100 +1478,51 @@ void complete()
                 logger.trace("Level: {}", this);
         }
 
-        /**
-         * Return the compaction pick for this level.
-         * <p>
-         * This is done by splitting the level into buckets that we can treat as independent regions for compaction.
-         * We then use the maxOverlap value (i.e. the maximum number of sstables that can contain data for any covered
-         * key) of each bucket to determine if compactions are needed, and to prioritize the buckets that contribute
-         * most to the complexity of queries: if maxOverlap is below the level's threshold, no compaction is needed;
-         * otherwise, we choose one from the buckets that have the highest maxOverlap.
-         */
-        CompactionPick getCompactionPick(SelectionContext context)
+        /// Return the compaction aggregate
+        Collection<CompactionAggregate.UnifiedAggregate> getCompactionAggregates(Arena arena,
+                                                                                 Controller controller,
+                                                                                 long spaceAvailable)
         {
-            List<Bucket> buckets = getBuckets(context);
-            if (buckets == null)
-            {
-                if (logger.isDebugEnabled())
-                    logger.debug("Level {} sstables {} max overlap {} buckets with compactions {} tasks {}",
-                                 index, sstables.size(), maxOverlap, 0, 0);
-                return null;    // nothing crosses the threshold in this level, nothing to do
-            }
-
-            int estimatedRemainingTasks = 0;
-            int overlapMatchingCount = 0;
-            Bucket selectedBucket = null;
-            Controller controller = context.controller;
-            for (Bucket bucket : buckets)
-            {
-                // We can have just one pick in each level. Pick one bucket randomly out of the ones with
-                // the highest overlap.
-                // The random() part below implements reservoir sampling with size 1, giving us a uniformly random selection.
-                if (bucket.maxOverlap == maxOverlap && controller.random().nextInt(++overlapMatchingCount) == 0)
-                    selectedBucket = bucket;
-                // The estimated remaining tasks is a measure of the remaining amount of work, thus we prefer to
-                // calculate the number of tasks we would do in normal operation, even though we may compact in bigger
-                // chunks when we are late.
-                estimatedRemainingTasks += bucket.maxOverlap / threshold;
-            }
-            context.estimatedRemainingTasks += estimatedRemainingTasks;
-            assert selectedBucket != null;
-
-            if (logger.isDebugEnabled())
-                logger.debug("Level {} sstables {} max overlap {} buckets with compactions {} tasks {}",
-                             index, sstables.size(), maxOverlap, buckets.size(), estimatedRemainingTasks);
-
-            CompactionPick selected = selectedBucket.constructPick(controller);
-
             if (logger.isTraceEnabled())
-                logger.trace("Returning compaction pick with selected compaction {}",
-                             selected);
-            return selected;
-        }
-
-        /**
-         * Group the sstables in this level into buckets.
-         * <p>
-         * The buckets are formed by grouping sstables that overlap at some key together, and then expanded to cover
-         * any overlapping sstable according to the overlap inclusion method. With the usual TRANSITIVE method this
-         * results into non-overlapping buckets that can't affect one another and can be compacted in parallel without
-         * any loss of efficiency.
-         * <p>
-         * Other overlap inclusion methods are provided to cover situations where we may be okay with compacting
-         * sstables partially and doing more than the strictly necessary amount of compaction to solve a problem: e.g.
-         * after an upgrade from LCS where transitive overlap may cause a complete level to be compacted together
-         * (creating an operation that will take a very long time to complete) and we want to make some progress as
-         * quickly as possible at the cost of redoing some work.
-         * <p>
-         * The number of sstables that overlap at some key defines the "overlap" of a set of sstables. The maximum such
-         * value in the bucket is its "maxOverlap", i.e. the highest number of sstables we need to read to find the
-         * data associated with a given key.
-         */
-        @VisibleForTesting
-        List<Bucket> getBuckets(SelectionContext context)
-        {
-            List<SSTableReader> liveSet = sstables;
+                logger.trace("Creating compaction aggregate with sstable set {}", sstables);
 
-            if (logger.isTraceEnabled())
-                logger.trace("Creating compaction pick with live set {}", liveSet);
 
-            List<Set<SSTableReader>> overlaps = Overlaps.constructOverlapSets(liveSet,
-                                                                              UnifiedCompactionStrategy::startsAfter,
-                                                                              SSTableReader.firstKeyComparator,
-                                                                              SSTableReader.lastKeyComparator);
-            for (Set<SSTableReader> overlap : overlaps)
+            // Note that adjacent overlap sets may include deduplicated sstable
+            List<Set<CompactionSSTable>> overlaps = Overlaps.constructOverlapSets(sstables,
+                                                                                  CompactionSSTable.startsAfter,
+                                                                                  CompactionSSTable.firstKeyComparator,
+                                                                                  CompactionSSTable.lastKeyComparator);
+            for (Set<CompactionSSTable> overlap : overlaps)
                 maxOverlap = Math.max(maxOverlap, overlap.size());
-            if (maxOverlap < threshold)
-                return null;
+            List<CompactionSSTable> unbucketed = new ArrayList<>();
 
             List<Bucket> buckets = Overlaps.assignOverlapsIntoBuckets(threshold,
-                                                                      context.controller.overlapInclusionMethod(),
+                                                                      controller.overlapInclusionMethod(),
                                                                       overlaps,
-                                                                      this::makeBucket);
-            return buckets;
+                                                                      this::makeBucket,
+                                                                      unbucketed::addAll);
+
+            List<CompactionAggregate.UnifiedAggregate> aggregates = new ArrayList<>();
+            for (Bucket bucket : buckets)
+                aggregates.add(bucket.constructAggregate(controller, spaceAvailable, arena));
+
+            // Add all unbucketed sstables separately. Note that this will list the level (with its set of sstables)
+            // even if it does not need compaction.
+            if (!unbucketed.isEmpty())
+                aggregates.add(CompactionAggregate.createUnified(unbucketed,
+                                                                 maxOverlap,
+                                                                 CompactionPick.EMPTY,
+                                                                 Collections.emptySet(),
+                                                                 arena,
+                                                                 this));
+
+            if (logger.isTraceEnabled())
+                logger.trace("Returning compaction aggregates {} for level {} of arena {}",
+                             aggregates, this, arena);
+            return aggregates;
         }
 
-        private Bucket makeBucket(List<Set<SSTableReader>> overlaps, int startIndex, int endIndex)
+        private Bucket makeBucket(List<Set<CompactionSSTable>> overlaps, int startIndex, int endIndex)
         {
             return endIndex == startIndex + 1
                    ? new SimpleBucket(this, overlaps.get(startIndex))
@@ -703,187 +1549,317 @@ private String densityAsString(double density)
         }
     }
 
-
-    /**
-     * A compaction bucket, i.e. a selection of overlapping sstables from which a compaction should be selected.
-     */
+    /// A compaction bucket, i.e. a selection of overlapping sstables from which a compaction should be selected.
     static abstract class Bucket
     {
         final Level level;
-        final List<SSTableReader> allSSTablesSorted;
+        final List<CompactionSSTable> allSSTablesSorted;
         final int maxOverlap;
 
-        Bucket(Level level, Collection<SSTableReader> allSSTablesSorted, int maxOverlap)
+        Bucket(Level level, Collection<CompactionSSTable> allSSTablesSorted, int maxOverlap)
         {
             // single section
             this.level = level;
             this.allSSTablesSorted = new ArrayList<>(allSSTablesSorted);
-            this.allSSTablesSorted.sort(SSTableReader.maxTimestampDescending);  // we remove entries from the back
+            this.allSSTablesSorted.sort(CompactionSSTable.maxTimestampDescending);  // we remove entries from the back
             this.maxOverlap = maxOverlap;
         }
 
-        Bucket(Level level, List<Set<SSTableReader>> overlapSections)
+        Bucket(Level level, List<Set<CompactionSSTable>> overlapSections)
         {
             // multiple sections
             this.level = level;
             int maxOverlap = 0;
-            Set<SSTableReader> all = new HashSet<>();
-            for (Set<SSTableReader> section : overlapSections)
+            Set<CompactionSSTable> all = new HashSet<>();
+            for (Set<CompactionSSTable> section : overlapSections)
             {
                 maxOverlap = Math.max(maxOverlap, section.size());
                 all.addAll(section);
             }
             this.allSSTablesSorted = new ArrayList<>(all);
-            this.allSSTablesSorted.sort(SSTableReader.maxTimestampDescending);  // we remove entries from the back
+            this.allSSTablesSorted.sort(CompactionSSTable.maxTimestampDescending);  // we remove entries from the back
             this.maxOverlap = maxOverlap;
         }
 
-        /**
-         * Select compactions from this bucket. Normally this would form a compaction out of all sstables in the
-         * bucket, but if compaction is very late we may prefer to act more carefully:
-         * - we should not use more inputs than the permitted maximum
-         * - we should select SSTables in a way that preserves the structure of the compaction hierarchy
-         * These impose a limit on the size of a compaction; to make sure we always reduce the read amplification by
-         * this much, we treat this number as a limit on overlapping sstables, i.e. if A and B don't overlap with each
-         * other but both overlap with C and D, all four will be selected to form a limit-three compaction. A limit-two
-         * one may choose CD, ABC or ABD.
-         * Also, the subset is selected by max timestamp order, oldest first, to avoid violating sstable time order. In
-         * the example above, if B is oldest and C is older than D, the limit-two choice would be ABC (if A is older
-         * than D) or BC (if A is younger, avoiding combining C with A skipping D).
-         *
-         * @param controller The compaction controller.
-         * @return A compaction pick to execute next.
-         */
-        CompactionPick constructPick(Controller controller)
+        /// Select compactions from this bucket. Normally this would form a compaction out of all sstables in the
+        /// bucket, but if compaction is very late we may prefer to act more carefully:
+        /// - we should not use more inputs than the permitted maximum
+        /// - we should not select a compaction whose execution will use more temporary space than is available
+        /// - we should select SSTables in a way that preserves the structure of the compaction hierarchy
+        /// These impose a limit on the size of a compaction; to make sure we always reduce the read amplification by
+        /// this much, we treat this number as a limit on overlapping sstables, i.e. if A and B don't overlap with each
+        /// other but both overlap with C and D, all four will be selected to form a limit-three compaction. A limit-two
+        /// one may choose CD, ABC or ABD.
+        /// Also, the subset is selected by max timestamp order, oldest first, to avoid violating sstable time order. In
+        /// the example above, if B is oldest and C is older than D, the limit-two choice would be ABC (if A is older
+        /// than D) or BC (if A is younger, avoiding combining C with A skipping D).
+        ///
+        /// @param controller The compaction controller.
+        /// @param spaceAvailable The amount of space available for compaction, limits the maximum number of sstables
+        ///                       that can be selected. This only applies after the first fanout-many overlapping
+        ///                       sstables have been selected, to ensure that the compaction strategy can honor its
+        ///                       write amplification expectations.
+        /// @return A compaction pick to execute next.
+        CompactionAggregate.UnifiedAggregate constructAggregate(Controller controller, long spaceAvailable, Arena arena)
         {
             int count = maxOverlap;
             int threshold = level.threshold;
             int fanout = level.fanout;
             int index = level.index;
-            int maxSSTablesToCompact = Math.max(fanout, controller.maxSSTablesToCompact());
+            int maxSSTablesToCompact = Math.max(fanout, (int) Math.min(spaceAvailable / level.avg, controller.maxSSTablesToCompact()));
 
             assert count >= threshold;
             if (count <= fanout)
             {
-                /**
-                 * Happy path. We are not late or (for levelled) we are only so late that a compaction now will
-                 * have the same effect as doing levelled compactions one by one. Compact all. We do not cap
-                 * this pick at maxSSTablesToCompact due to an assumption that maxSSTablesToCompact is much
-                 * greater than F. See {@link Controller#MAX_SSTABLES_TO_COMPACT_OPTION} for more details.
-                 */
-                return new CompactionPick(index, count, allSSTablesSorted);
+                // Happy path. We are not late or (for levelled) we are only so late that a compaction now will
+                // have the same effect as doing levelled compactions one by one. Compact all. We do not cap
+                // this pick at maxSSTablesToCompact, or reduce the size of the compaction to the available disk
+                // space because that would violate the strategy's write amplification promises.
+                // If a compaction is too big to fit the available space, protections in [getSelection] will
+                // prevent if from being selected; space may be available on a later compaction round.
+                return CompactionAggregate.createUnified(allSSTablesSorted,
+                                                         maxOverlap,
+                                                         createPick(controller, nextTimeUUID(), index, allSSTablesSorted),
+                                                         Collections.emptySet(),
+                                                         arena,
+                                                         level);
             }
+            // The choices below assume that pulling the oldest sstables will reduce maxOverlap by the selected
+            // number of sstables. This is not always true (we may, e.g. select alternately from different overlap
+            // sections if the structure is complex enough), but is good enough heuristic that results in usable
+            // compaction sets.
             else if (count <= fanout * controller.getFanout(index + 1) || maxSSTablesToCompact == fanout)
             {
                 // Compaction is a bit late, but not enough to jump levels via layout compactions. We need a special
                 // case to cap compaction pick at maxSSTablesToCompact.
                 if (count <= maxSSTablesToCompact)
-                    return new CompactionPick(index, count, allSSTablesSorted);
+                    return CompactionAggregate.createUnified(allSSTablesSorted,
+                                                             maxOverlap,
+                                                             createPick(controller, nextTimeUUID(), index, allSSTablesSorted),
+                                                             Collections.emptySet(),
+                                                             arena,
+                                                             level);
+
+                CompactionPick pick = createPick(controller, nextTimeUUID(), index, pullOldestSSTables(maxSSTablesToCompact));
+                count -= maxSSTablesToCompact;
+                List<CompactionPick> pending = new ArrayList<>();
+                while (count >= threshold)
+                {
+                    pending.add(createPick(controller, nextTimeUUID(), index, pullOldestSSTables(maxSSTablesToCompact)));
+                    count -= maxSSTablesToCompact;
+                }
 
-                return new CompactionPick(index, maxSSTablesToCompact, pullOldestSSTables(maxSSTablesToCompact));
+                return CompactionAggregate.createUnified(allSSTablesSorted, maxOverlap, pick, pending, arena, level);
             }
+            // We may, however, have accumulated a lot more than T if compaction is very late, or a set of small
+            // tables was dumped on us (e.g. when converting from legacy LCS or for tests).
             else
             {
-                // We may, however, have accumulated a lot more than T if compaction is very late.
-                // In this case we pick a compaction in such a way that the result of doing it spreads the data in
+                // We need to pick the compactions in such a way that the result of doing them all spreads the data in
                 // a similar way to how compaction would lay them if it was able to keep up. This means:
                 // - for tiered compaction (w >= 0), compact in sets of as many as required to get to a level.
-                //   for example, for w=2 and 55 sstables, pick a compaction of 16 sstables (on the next calls, given no
-                //   new files, 2 more of 16, 1 of 4, and leaving the other 3 sstables alone).
+                //   for example, for w=2 and 55 sstables, do 3 compactions of 16 sstables, 1 of 4, and leave the other 3 alone
                 // - for levelled compaction (w < 0), compact all that would reach a level.
-                //   for w=-2 and 55, this means pick a compaction of 48 (on the next calls, given no new files, one of
-                //   4, and one of 3 sstables).
-                int pickSize = selectPickSize(controller, maxSSTablesToCompact);
-                return new CompactionPick(index, pickSize, pullOldestSSTables(pickSize));
+                //   for w=-2 and 55, this means one compaction of 48, one of 4, and one of 3 sstables.
+                List<CompactionPick> picks = layoutCompactions(controller, maxSSTablesToCompact);
+                // Out of the set of necessary compactions, choose the one to run randomly. This gives a better
+                // distribution among levels and should result in more compactions running in parallel in a big data
+                // dump.
+                assert !picks.isEmpty();  // we only enter this if count > F: layoutCompactions must have selected something to run
+                CompactionPick selected = picks.remove(controller.random().nextInt(picks.size()));
+                return CompactionAggregate.createUnified(allSSTablesSorted, maxOverlap, selected, picks, arena, level);
             }
         }
 
-        private int selectPickSize(Controller controller, int maxSSTablesToCompact)
+        private List<CompactionPick> layoutCompactions(Controller controller, int maxSSTablesToCompact)
         {
-            int pickSize;
-            int fanout = level.fanout;
-            int nextStep = fanout;
-            int index = level.index;
-            int limit = Math.min(maxSSTablesToCompact, maxOverlap);
-            do
+            List<CompactionPick> pending = new ArrayList<>();
+            int pos = layoutCompactions(controller, level.index + 1, level.fanout, maxSSTablesToCompact, pending);
+            int size = maxOverlap;
+            if (size - pos >= level.threshold) // can only happen in the levelled case.
             {
-                pickSize = nextStep;
-                fanout = controller.getFanout(++index);
-                nextStep *= fanout;
+                assert size - pos < maxSSTablesToCompact; // otherwise it should have already been picked
+                pending.add(createPick(controller, nextTimeUUID(), level.index, allSSTablesSorted));
             }
-            while (nextStep <= limit);
+            return pending;
+        }
 
-            if (level.scalingParameter < 0)
+        /// Collects in {@param list} compactions of {@param sstables} such that they land in {@param level} and higher.
+        ///
+        /// Recursively combines SSTables into [CompactionPick]s in way that up to {@param maxSSTablesToCompact}
+        /// SSTables are combined to reach the highest possible level, then the rest is combined for the level before,
+        /// etc up to {@param level}.
+        ///
+        /// To agree with what compaction normally does, the first sstables from the list are placed in the picks that
+        /// combine to reach the highest levels.
+        ///
+        /// @param level minimum target level for compactions to land
+        /// @param step - number of source SSTables required to reach level
+        /// @param maxSSTablesToCompact limit on the number of sstables per compaction
+        /// @param list - result list of layout-preserving compaction picks
+        /// @return index of the last used SSTable from {@param sstables}; the number of remaining sstables will be lower
+        ///         than step
+        private int layoutCompactions(Controller controller,
+                                      int level,
+                                      int step,
+                                      int maxSSTablesToCompact,
+                                      List<CompactionPick> list)
+        {
+            if (step > maxOverlap || step > maxSSTablesToCompact)
+                return 0;
+
+            int w = controller.getScalingParameter(level);
+            int f = controller.getFanout(level);
+            int pos = layoutCompactions(controller,
+                                        level + 1,
+                                        step * f,
+                                        maxSSTablesToCompact,
+                                        list);
+
+            int total = maxOverlap;
+            // step defines the number of source sstables that are needed to reach this level (ignoring overwrites
+            // and deletions).
+            // For tiered compaction we will select batches of this many.
+            int pickSize = step;
+            if (w < 0)
             {
                 // For levelled compaction all the sstables that would reach this level need to be compacted to one,
-                // so select the highest multiple of step that fits.
-                pickSize *= limit / pickSize;
-                assert pickSize > 0;
+                // so select the highest multiple of step that is available, but make sure we don't do a compaction
+                // bigger than the limit.
+                pickSize *= Math.min(total - pos, maxSSTablesToCompact) / pickSize;
+
+                if (pickSize == 0)  // Not enough sstables to reach this level, we can skip the processing below.
+                    return pos;     // Note: this cannot happen on the top level, but can on lower ones.
             }
-            return pickSize;
+
+            while (pos + pickSize <= total)
+            {
+                // Note that we assign these compactions to the level that would normally produce them, which means that
+                // they won't be taking up threads dedicated to the busy level.
+                // Normally sstables end up on a level when a compaction on the previous brings their size to the
+                // threshold (which corresponds to pickSize == step, always the case for tiered); in the case of
+                // levelled compaction, when we compact more than 1 but less than F sstables on a level (which
+                // corresponds to pickSize > step), it is an operation that is triggered on the same level.
+                list.add(createPick(controller,
+                                    nextTimeUUID(),
+                                    pickSize > step ? level : level - 1,
+                                    pullOldestSSTables(pickSize)));
+                pos += pickSize;
+            }
+
+            // In the levelled case, if we had to adjust pickSize due to maxSSTablesToCompact, there may
+            // still be enough sstables to reach this level (e.g. if max was enough for 2*step, but we had 3*step).
+            if (pos + step <= total)
+            {
+                pickSize = ((total - pos) / step) * step;
+                list.add(createPick(controller,
+                                    nextTimeUUID(),
+                                    pickSize > step ? level : level - 1,
+                                    pullOldestSSTables(pickSize)));
+                pos += pickSize;
+            }
+            return pos;
+        }
+
+        static <T> List<T> pullLast(List<T> source, int limit)
+        {
+            List<T> result = new ArrayList<>(limit);
+            while (--limit >= 0)
+                result.add(source.remove(source.size() - 1));
+            return result;
         }
 
         /**
          * Pull the oldest sstables to get at most limit-many overlapping sstables to compact in each overlap section.
          */
-        abstract Collection<SSTableReader> pullOldestSSTables(int overlapLimit);
+        abstract Collection<CompactionSSTable> pullOldestSSTables(int overlapLimit);
     }
 
     public static class SimpleBucket extends Bucket
     {
-        public SimpleBucket(Level level, Collection<SSTableReader> sstables)
+        public SimpleBucket(Level level, Collection<CompactionSSTable> sstables)
         {
             super(level, sstables, sstables.size());
         }
 
-        Collection<SSTableReader> pullOldestSSTables(int overlapLimit)
+        Collection<CompactionSSTable> pullOldestSSTables(int overlapLimit)
         {
             if (allSSTablesSorted.size() <= overlapLimit)
                 return allSSTablesSorted;
-            return Overlaps.pullLast(allSSTablesSorted, overlapLimit);
+            return pullLast(allSSTablesSorted, overlapLimit);
         }
     }
 
     public static class MultiSetBucket extends Bucket
     {
-        final List<Set<SSTableReader>> overlapSets;
+        final List<Set<CompactionSSTable>> overlapSets;
 
-        public MultiSetBucket(Level level, List<Set<SSTableReader>> overlapSets)
+        public MultiSetBucket(Level level, List<Set<CompactionSSTable>> overlapSets)
         {
             super(level, overlapSets);
             this.overlapSets = overlapSets;
         }
 
-        Collection<SSTableReader> pullOldestSSTables(int overlapLimit)
+        Collection<CompactionSSTable> pullOldestSSTables(int overlapLimit)
         {
             return Overlaps.pullLastWithOverlapLimit(allSSTablesSorted, overlapSets, overlapLimit);
         }
     }
 
-    /**
-     * Utility class holding a collection of sstables for compaction.
-     */
-    static class CompactionPick extends ArrayList<SSTableReader>
+    static class CompactionLimits
     {
-        final int level;
-        final int overlap;
+        final int runningCompactions;
+        final int maxConcurrentCompactions;
+        final int maxCompactions;
+        final int[] perLevel;
+        int levelCount;
+        final long spaceAvailable;
+        final int remainingAdaptiveCompactions;
+
+        public CompactionLimits(int runningCompactions,
+                                int maxCompactions,
+                                int maxConcurrentCompactions,
+                                int[] perLevel,
+                                int levelCount,
+                                long spaceAvailable,
+                                int remainingAdaptiveCompactions)
+        {
+            this.runningCompactions = runningCompactions;
+            this.maxCompactions = maxCompactions;
+            this.maxConcurrentCompactions = maxConcurrentCompactions;
+            this.perLevel = perLevel;
+            this.levelCount = levelCount;
+            this.spaceAvailable = spaceAvailable;
+            this.remainingAdaptiveCompactions = remainingAdaptiveCompactions;
+        }
 
-        CompactionPick(int level, int overlap, Collection<SSTableReader> sstables)
+        @Override
+        public String toString()
         {
-            super(sstables);
-            this.level = level;
-            this.overlap = overlap;
+            return String.format("Current limits: running=%d, max=%d, maxConcurrent=%d, perLevel=%s, levelCount=%d, spaceAvailable=%s, remainingAdaptiveCompactions=%d",
+                                 runningCompactions, maxCompactions, maxConcurrentCompactions, Arrays.toString(perLevel), levelCount,
+                                 FBUtilities.prettyPrintMemory(spaceAvailable), remainingAdaptiveCompactions);
         }
     }
 
-    static class SelectionContext
+    /**
+     * Utility wrapper to efficiently store the density of an SSTable with the SSTable itself.
+     */
+    private static class SSTableWithDensity implements Comparable<SSTableWithDensity>
     {
-        final Controller controller;
-        int estimatedRemainingTasks = 0;
+        final CompactionSSTable sstable;
+        final double density;
 
-        SelectionContext(Controller controller)
+        SSTableWithDensity(CompactionSSTable sstable, double density)
+        {
+            this.sstable = sstable;
+            this.density = density;
+        }
+
+        @Override
+        public int compareTo(SSTableWithDensity o)
         {
-            this.controller = controller;
+            return Double.compare(density, o.density);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md
index 5f8d548af97c..71c45664e83b 100644
--- a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md
+++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md
@@ -1,19 +1,19 @@
 <!--
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# Copyright DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
 -->
 
 # Unified compaction strategy (UCS)
@@ -218,12 +218,12 @@ This sharding mechanism is independent of the compaction specification.
 
 This sharding scheme easily admits extensions. In particular, when the size of the data set is expected to grow very
 large, to avoid having to pre-specify a high enough target size to avoid problems with per-sstable overhead, we can
-apply an "SSTtable growth" parameter, which determines what part of the density growth should be assigned to increased
+apply an "sstable growth" parameter, which determines what part of the density growth should be assigned to increased
 SSTable size, reducing the growth of the number of shards (and hence non-overlapping sstables).
 
 Additionally, to allow for a mode of operation with a fixed number of shards, and splitting conditional on reaching
-a minimum size, we provide for a "minimum SSTable size" that reduces the base shard count whenever that would result
-in SSTables smaller than the provided minimum.
+a minimum size, we provide for a "minimum sstable size" that reduces the base shard count whenever that would result
+in sstables smaller than the provided minimum.
 
 Generally, the user can specify four sharding parameters:
 
@@ -239,8 +239,8 @@ S =
 \begin{cases}
 1
     & \text{if } d < m \\
-min(2^{\left\lfloor \log_2 \frac d m \right\rfloor}, x)
-    & \text{if } d < mb \text{, where } x \text{ is the largest power of 2 divisor of } b \\
+2^{\left\lfloor \log_2 \frac d m \right\rfloor}
+    & \text{if } d < mb \\
 b
     & \text{if } d < tb \\
 2^{\left\lfloor (1-\lambda) \cdot \log_2 \left( {\frac d t \cdot \frac 1 b}\right)\right\rceil} \cdot b
@@ -262,14 +262,14 @@ Some useful combinations of these parameters:
 ![Graph with lambda 0.5](unified/shards_graph_lambda_0_5.svg)
 
 - Similarly, $\lambda = 1/3$ makes the sstable growth the cubic root of the density growth, i.e. the sstable size
-  grows with the square root of the growth of the shard count. The graph below uses $b=1$ and $t = 1\mathrm{GB}$
+  grows with the square root of the growth of the shard count. The graph below uses $b=1$ and $t = 1\mathrm{GB}$ 
   (note: when $b=1$ the minimal size has no effect):
 
 ![Graph with lambda 0.33](unified/shards_graph_lambda_0_33.svg)
 
 - A growth component of 1 constructs a hierarchy with exactly $b$ shards at every level. Combined with a minumum
-  sstable size, this defines a mode of operation where we use a pre-specified
-  number of shards, but split only after reaching a minimum size. Illustrated below for $b=10$ and $m=100\mathrm{MB}$
+  sstable size, this defines a mode of operation similar to UCS V1 (as used in DSE 6.8), where we use a pre-specified
+  number of shards, but split only after reaching a minimum size. Illustrated below for $b=10$ and $m=100\mathrm{MB}$ 
   (note: the target sstable size is irrelevant when $\lambda=1$):
 
 ![Graph with lambda 1](unified/shards_graph_lambda_1.svg)
@@ -306,8 +306,8 @@ than this set alone.
 
 It is possible for our sharding scheme to end up constructing sstables spanning differently-sized shards for the same
 level. One clear example is the case of levelled compaction, where, for example, sstables enter at some density, and
-after the first compaction the result &mdash; being 2x bigger than that density &mdash; is split in the middle because 
-it has double the density. As another sstable enters the same level, we will have separate overlap sets for the first 
+after the first compaction the result &mdash; being 2x bigger than that density &mdash; is split in the middle because
+it has double the density. As another sstable enters the same level, we will have separate overlap sets for the first
 and second half of that older sstable; to be efficient, the compaction that is triggered next needs to select both.
 
 To deal with this and any other cases of partial overlap, the compaction strategy will transitively extend
@@ -319,27 +319,92 @@ on the number of overlapping sources we compact; in that case we use the collect
 select at most limit-many in any included overlap set, making sure that if an sstable is included in this compaction,
 all older ones are also included to maintain time order.
 
-## Selecting the compaction to run
+## Prioritization of compactions
 
 Compaction strategies aim to minimize the read amplification of queries, which is defined by the number of sstables
-that overlap on any given key. In order to do this most efficiently in situations where compaction is late, we select
-a compaction bucket whose overlap is the highest among the possible choices. If there are multiple such choices, we 
-choose one uniformly randomly within each level, and between the levels we prefer the lowest level (as this is expected 
-to cover a larger fraction of the token space for the same amount of work).
-
-Under sustained load, this mechanism prevents the accumulation of sstables on some level that could sometimes happen 
-with legacy strategies (e.g. all resources consumed by L0 and sstables accumulating on L1) and can lead to a 
-steady state where compactions always use more sstables than the assigned threshold and fan factor and maintain a tiered
-hierarchy based on the lowest overlap they are able to maintain for the load.
+that overlap on any given key. In order to do this most efficiently in situations where compaction is late, we
+prioritize compaction buckets whose overlap is higher. If there are multiple such choices, we choose one uniformly
+randomly within each level, and between the levels we prefer the lowest level (as this is expected to cover a larger
+fraction of the token space for the same amount of work).
+
+Under sustained load, this mechanism in combination with the above prevents the accumulation of sstables on some level
+that could sometimes happen with legacy strategies (e.g. all resources consumed by L0 and sstables accumulating on L1)
+and can lead to a steady state where compactions always use more sstables than the assigned threshold and fan factor and
+maintain a tiered hierarchy based on the lowest overlap they are able to maintain for the load.
+
+## Compaction thread allocation
+
+Because of sharding, UCS can do more compactions in parallel. This is especially true for higher levels of the
+hierarchy, where we would often end up with 10s or 100s of pending compactions in a very short time as new data pushes 
+all shards over the threshold at almost the same time.
+
+The above can cause all compaction threads to start work on such levels, starving other levels of computing resources
+to run. The starvation is especially apparent for level 0, where new sstables quickly accummulate and increase overlap. 
+This is a real but manageable problem when sstables are small, where the short compaction time combined with the 
+prioritization mechanism above can improve the situation relatively quickly.
+
+However, with higher lambdas (especially in fixed shards mode), higher-level compactions can take a very long time and 
+will often hog all available threads, causing very large accummulations of sstables on the lowest levels that can remain
+present for a long time and cause significant problems. To prevent this, UCS will by default limit the number of threads
+that can perform higher-level compactions to only a fair share of the total number of threads. This is fully
+configurable through parameters for a number of thread reservations, as well as a reservation mode (`per_level` or 
+`level_or_below`). 
+
+When the number of reservations is 0, the mode does not matter and all compaction threads are assigned according to the 
+prioritization explained in the previous paragraph. This provides the best utilization of compaction threads in the 
+system and works well with small sstable sizes. Even in these cases, it will result in small spikes of sstable overlap 
+on lower levels of the hierarchy when compactions on top levels are initiated.
+
+In `per-level` mode, when the number of reservations is set to an integer, UCS will reserve that many threads for each
+level of the hierarchy and assign work to the rest of the threads according to the prioritization above. Some threads 
+will be idle if no work is needed on the associated level. UCS will also reserve the given number of threads for the top
+level before it needs any compaction, to be able to respond to a new need quickly.
+
+When the number of reservations is set to `max`, or exceeds the number of available threads divided by the number of 
+levels, UCS will reserve the integer part of that ratio for each level, and will assign the remainder according
+to the prioritization, but only up to one additional compaction per level. This setting provides better smoothness, 
+reducing or fully eliminating the overlap spikes, and is imperative when sstables can grow large (i.e. with higher
+lambda). The downside of this setting is that this means fewer compaction threads will be actively used. This is thus 
+best combined with higher compaction thread counts.
+
+Using the `level_and_below` mode splits the threads as above, but makes threads for higher levels available for 
+lower-level work. In other words, it only limits the resources that higher levels may use: up to the given number plus
+any remainder for the top level, up to two times that number plus the remainder for the top two levels and so on. This 
+still solves the original problem (higher-level compactions starving low levels of resources) while making better use of
+the compaction threads. This is the mode (with `max` reservations) used by default.
+
+## Output shard parallelization
+
+Because the sharding of the output of a compaction operation is known in advance, we can parallelize the compaction
+process by starting a separate task for each shard. This can dramatically speed the throughput of compaction and is
+especially helpful for the lower levels of the compaction heirarchy, where the number of input shards is very low
+(often just one). To make sure that we correctly change the state of input and output sstables, such operations will
+share a transaction and will complete only when all individual tasks complete (and, conversely, abort if any of the
+individual tasks abort). Early opening of sstables is not supported in this mode, because we currently do not support
+arbitraty filtering of the requests to an sstable; it is expected that the smaller size and quicker completion time of
+compactions should make up for this.
+
+This is controlled by the `parallelize_output_shards` parameter, which is `true` by default.
 
 ## Major compaction
 
-Under the working principles of UCS, a major compaction is an operation which compacts together all sstables that have
-(transitive) overlap, and where the output is split on shard boundaries appropriate for the expected result density.
-
-In other words, it is expected that a major compaction will result in $b$ concurrent compactions, each containing all
-sstables covered in each of the base shards, and that the result will be split on shard boundaries whose number 
-depends on the total size of data contained in the shard.
+Major compaction in UCS always splits the output into a shard number suitable for the expected result density.
+If the input sstables can be split into non-overlapping sets that correspond to current shard boundaries, the compaction
+will construct independent operations that work over these sets, to improve the space overhead of the operation as well
+as the time needed to persistently complete individual steps. Because all levels will usually be split in $b$ shards,
+it will very often be the case that major compactions split into $b$ individual jobs, reducing the space overhead by a
+factor close to $b$. Note that this does not always apply; for example, if a topology change causes the sharding
+boundaries to move, the mismatch between old and new sharding boundaries will cause the compaction to produce a single
+operation and require 100% space overhead.
+
+Output shard parallelization also applies to major compactions: if the `parallelize_output_shards` option is enabled,
+shards of individual compactions will be compacted concurrently, which can significantly reduce the time needed to
+perform the compaction; if the option is not enabled, major compaction will only be parallelized up to the number of
+individual non-overlapping sets the sstables can be split into. In either case, the number of parallel operations is
+limited to a number specified as a parameter of the operation (e.g. `nodetool compact -j n`), which is set to half the
+compaction thread count by default. Using a jobs of 0 will let the compaction use all available threads and run
+as quickly as possible, but this will prevent other compaction operations from running until it completes and thus
+should be used with caution, only while the database is known to not receive any writes.
 
 ## Differences with STCS and LCS
 
@@ -398,7 +463,7 @@ the span of the lower-density ones.
 
 UCS accepts these compaction strategy parameters:
 
-* **scaling_parameters**. A list of per-level scaling parameters, specified as L*f*, T*f*, N, or an integer value
+* `scaling_parameters` A list of per-level scaling parameters, specified as L*f*, T*f*, N, or an integer value
   specifying $w$ directly. If more levels are present than the length of this list, the last value is used for all
   higher levels. Often this will be a single parameter, specifying the behaviour for all levels of the
   hierarchy.  
@@ -410,24 +475,26 @@ UCS accepts these compaction strategy parameters:
   expense of making reads more difficult.  
   N is the middle ground that has the features of levelled (one sstable run per level) as well as tiered (one
   compaction to be promoted to the next level) and a fan factor of 2. This can also be specified as T2 or L2.  
-  The default value is T4, matching the default STCS behaviour with threshold 4. To select an equivalent of LCS
-  with its default fan factor 10, use L10.
-* **target_sstable_size**. The target sstable size $t$, specified as a human-friendly size in bytes (e.g. 100 MiB =
+  The default value is T4, matching the default STCS behaviour with threshold 4. The default value in vector mode (see
+  paragraph below) is L10, equivalent to LCS with its default fan factor 10.
+* `target_sstable_size` The target sstable size $t$, specified as a human-friendly size in bytes (e.g. 100 MiB =
   $100\cdot 2^{20}$ B or (10 MB = 10,000,000 B)). The strategy will split data in shards that aim to produce sstables
   of size between $t / \sqrt 2$ and $t \cdot \sqrt 2$.  
   Smaller sstables improve streaming and repair, and make compactions shorter. On the other hand, each sstable
   on disk has a non-trivial in-memory footprint that also affects garbage collection times.  
-  Increase this if the memory pressure from the number of sstables in the system becomes too high.  
-  The default value is 1 GiB.
-* **base_shard_count**. The minimum number of shards $b$, used for levels with the smallest density. This gives the
+  Increase this if the memory pressure from the number of sstables in the system becomes too high. Also see
+  `sstable_growth` below.  
+  The default value is 1 GiB. The default value in vector mode is 5GiB.
+* `base_shard_count` The minimum number of shards $b$, used for levels with the smallest density. This gives the
   minimum compaction concurrency for the lowest levels. A low number would result in larger L0 sstables but may limit
-  the overall maximum write throughput (as every piece of data has to go through L0). The base shard count only applies after `min_sstable_size` is reached. 
-  The default value is 4 for all tables
-* **sstable_growth** The sstable growth component $\lambda$, applied as a factor in the shard exponent calculation.
+  the overall maximum write throughput (as every piece of data has to go through L0). The base shard count only applies 
+  after `min_sstable_size` is reached.  
+  The default value is 4. The default value in vector mode is 1. 
+* `sstable_growth` The sstable growth component $\lambda$, applied as a factor in the shard exponent calculation.
   This is a number between 0 and 1 that controls what part of the density growth should apply to individual sstable
   size and what part should increase the number of shards. Using a value of 1 has the effect of fixing the shard
   count to the base value. Using 0.5 makes the shard count and sstable size grow with the square root of the density
-  growth.
+  growth.  
   This is useful to decrease the sheer number of sstables that will be created for very large data sets. For
   example, without growth correction a data set of 10TiB with 1GiB target size would result in over 10k sstables,
   which may present as too much overhead both as on-heap memory used by per-sstable structures as well as time to look
@@ -435,19 +502,54 @@ UCS accepts these compaction strategy parameters:
   in this scenario (with base count 4) will reduce the potential number of sstables to ~160 of ~64GiB, which is still
   manageable both as memory overhead and individual compaction duration and space overhead. The balance between the
   two can be further tweaked by increasing $\lambda$ to get fewer but bigger sstables on the top level, and decreasing
-  it to favour a higher count of smaller sstables. The default value is 0.333 meaning the sstable size
-  grows with the square root of the growth of the shard count.
-* **min_sstable_size** The minimum sstable size $m$, applicable when the base shard count will result is sstables
+  it to favour a higher count of smaller sstables.  
+  The default value is 0.333 meaning the sstable size grows with the square root of the growth of the shard count.
+  The default value in vector mode is 1 which means the shard count will be fixed to the base value.
+* `min_sstable_size` The minimum sstable size $m$, applicable when the base shard count will result is sstables
   that are considered too small. If set, the strategy will split the space into fewer than the base count shards, to
-  make the estimated sstables size at least as large as this value. A value of 0 disables this feature.
-  The default value is 100MiB.
-* **expired_sstable_check_frequency_seconds**. Determines how often to check for expired SSTables.  
+  make the estimated sstables size at least as large as this value. A value of 0 disables this feature. 
+  A value of `auto` sets the minimum sstable size to the size of sstables resulting from flushes. 
+  The default value is 100MiB. The default value in vector mode is 1GiB.
+* `reserved_threads` Specifies the number of threads to reserve per level. Any remaining threads will take
+  work according to the prioritization mechanism (i.e. higher overlap first). Higher reservations mean better
+  responsiveness of the compaction strategy to new work, or smoother performance, at the expense of reducing the
+  overall utilization of compaction threads. Higher values work best with high `concurrent_compactors` values.  
+  The default value is `max`, which spreads all threads as close to evenly between levels as possible. It is recommended
+  to keep this option and the next at their defaults, which should offer a good balance between responsiveness and
+  thread utilization.
+* `reservations_type` Specifies whether reservations can be used by lower levels. If set to `per_level`, the
+  reservations are only used by the specific level. If set to `level_or_below`, the reservations can be used by this
+  level as well as any one below it.  
+  The default value is `level_or_below`.
+* `parallelize_output_shards` Enables or disables parallelization of compaction tasks for the output shards of a
+  compaction. This can dramatically improve compaction throughput especially on the lowest levels of the hierarchy,
+  but disables early open and thus may be less efficient when compaction is configured to produce very large
+  sstables.   
+  The default value is `true`.
+* `expired_sstable_check_frequency_seconds` Determines how often to check for expired SSTables.  
   The default value is 10 minutes.
-
-In **cassandra.yaml**:
-
-* **concurrent_compactors**. The number of compaction threads available. Higher values increase compaction performance
-  but may increase read and write latencies.
+* `num_shards` Specifying this switches the strategy to UCS V1 mode, where the number of shards is fixed, but a
+  minimum sstable size applies for the lowest levels. Provided for compatibility with DSE 6.8's UCS implementation.
+  Sets $b$ to the specified value, $\lambda$ to 1, and the default minimum sstable size to 'auto'.  
+  Disabled by default and cannot be used in combination with `base_shard_count`, `target_sstable_size` or
+  `sstable_growth`.
+
+All UCS options can also be supplied as system properties, using the prefix `unified_compaction.`, e.g. 
+`-Dunified_compaction.sstable_growth=0.5` sets the default `sstable_growth` to 0.5.
+
+In addition to this, the strategy permits different defaults to be applied to tables that have a vector column when the 
+system property `unified_compaction.override_ucs_config_for_vector_tables` is set to `true`. If this is enabled and the
+table has a column of type `vector`, the "vector mode" defaults in the list above apply. These vector defaults can be 
+altered using the prefix `unified_compaction.vector_`, e.g. 
+`-Dunified_compaction.vector_sstable_growth=1` in combination with 
+`-Dunified_compaction.override_ucs_config_for_vector_tables=true` sets the growth to 1 only for tables with a vector
+column.
+
+In `cassandra.yaml`:
+
+* `concurrent_compactors` The number of compaction threads available. Higher values increase compaction performance
+  but may increase read and write latencies. Combine a high compactor count with thread reservations for more consistent
+  performance with sustained loads.
 
 [^1]: Note: in addition to TRANSITIVE, "overlap inclusion methods" of NONE and SINGLE are also implemented for
     experimentation, but they are not recommended for the UCS sharding scheme.
diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
index 9e4c4dd7502b..4d16de759c7a 100644
--- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
+++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
@@ -17,18 +17,17 @@
  */
 package org.apache.cassandra.db.compaction;
 
-import java.util.Collections;
 import java.util.function.LongPredicate;
 
 import com.google.common.base.Throwables;
 import com.google.common.collect.Sets;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableRewriter;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -41,59 +40,50 @@
 
 public class Upgrader
 {
-    private final ColumnFamilyStore cfs;
+    private final CompactionRealm realm;
     private final SSTableReader sstable;
     private final LifecycleTransaction transaction;
     private final File directory;
 
     private final CompactionController controller;
-    private final CompactionStrategyManager strategyManager;
-    private final long estimatedRows;
 
     private final OutputHandler outputHandler;
 
-    public Upgrader(ColumnFamilyStore cfs, LifecycleTransaction txn, OutputHandler outputHandler)
+    public Upgrader(CompactionRealm realm, LifecycleTransaction txn, OutputHandler outputHandler)
     {
-        this.cfs = cfs;
+        this.realm = realm;
         this.transaction = txn;
         this.sstable = txn.onlyOne();
         this.outputHandler = outputHandler;
-
         this.directory = new File(sstable.getFilename()).parent();
-
-        this.controller = new UpgradeController(cfs);
-
-        this.strategyManager = cfs.getCompactionStrategyManager();
-        long estimatedTotalKeys = Math.max(cfs.metadata().params.minIndexInterval, SSTableReader.getApproximateKeyCount(Collections.singletonList(this.sstable)));
-        long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(Collections.singletonList(this.sstable)) / strategyManager.getMaxSSTableBytes());
-        this.estimatedRows = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables);
+        this.controller = new UpgradeController(realm);
     }
 
     private SSTableWriter createCompactionWriter(StatsMetadata metadata)
     {
-        MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator());
+        MetadataCollector sstableMetadataCollector = new MetadataCollector(realm.metadata().comparator);
         sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel());
 
-        Descriptor descriptor = cfs.newSSTableDescriptor(directory);
+        Descriptor descriptor = realm.newSSTableDescriptor(directory);
         return descriptor.getFormat().getWriterFactory().builder(descriptor)
-                         .setKeyCount(estimatedRows)
+                         .setKeyCount(metadata.totalRows) // TODO is it correct? I don't know why did we estimate that value instead of just copying it from metadata
                          .setRepairedAt(metadata.repairedAt)
                          .setPendingRepair(metadata.pendingRepair)
                          .setTransientSSTable(metadata.isTransient)
-                         .setTableMetadataRef(cfs.metadata)
+                         .setTableMetadataRef(realm.metadataRef())
                          .setMetadataCollector(sstableMetadataCollector)
-                         .setSerializationHeader(SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable)))
-                         .addDefaultComponents(cfs.indexManager.listIndexGroups())
-                         .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups())
-                         .build(transaction, cfs);
+                         .setSerializationHeader(SerializationHeader.make(realm.metadata(), Sets.newHashSet(sstable)))
+                         .addDefaultComponents(realm.getIndexManager().listIndexGroups())
+                         .setSecondaryIndexGroups(realm.getIndexManager().listIndexGroups())
+                         .build(transaction, realm);
     }
 
     public void upgrade(boolean keepOriginals)
     {
         outputHandler.output("Upgrading " + sstable);
         long nowInSec = FBUtilities.nowInSeconds();
-        try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, keepOriginals, CompactionTask.getMaxDataAge(transaction.originals()));
-             AbstractCompactionStrategy.ScannerList scanners = strategyManager.getScanners(transaction.originals());
+        try (SSTableRewriter writer = SSTableRewriter.construct(realm, transaction, keepOriginals, CompactionTask.getMaxDataAge(transaction.originals()));
+             ScannerList scanners = ScannerList.of(transaction.originals(), null);
              CompactionIterator iter = new CompactionIterator(transaction.opType(), scanners.scanners, controller, nowInSec, nextTimeUUID()))
         {
             writer.switchWriter(createCompactionWriter(sstable.getSSTableMetadata()));
@@ -117,7 +107,7 @@ public void upgrade(boolean keepOriginals)
 
     private static class UpgradeController extends CompactionController
     {
-        public UpgradeController(ColumnFamilyStore cfs)
+        public UpgradeController(CompactionRealm cfs)
         {
             super(cfs, Integer.MAX_VALUE);
         }
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/AdaptiveController.java b/src/java/org/apache/cassandra/db/compaction/unified/AdaptiveController.java
new file mode 100644
index 000000000000..b7735eb25045
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/unified/AdaptiveController.java
@@ -0,0 +1,613 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.compaction.CompactionPick;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileReader;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.Overlaps;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_ADAPTIVE_INTERVAL_SEC;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_ADAPTIVE_MAX_SCALING_PARAMETER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_ADAPTIVE_MIN_COST;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_ADAPTIVE_MIN_SCALING_PARAMETER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_ADAPTIVE_THRESHOLD;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_MAX_ADAPTIVE_COMPACTIONS;
+
+/**
+ * The adaptive compaction controller dynamically calculates the optimal scaling parameter W.
+ * <p/>
+ * Generally it tries to find a local minimum for the total IO cost that is projected
+ * by the strategy. The projected IO cost is composed by two parts: the read amplification,
+ * which is weighted by the number of partitions read by the user, and the write amplification, which
+ * is weighted by the number of bytes inserted into memtables. Other parameters are also considered, such
+ * as the cache miss rate and the time it takes to read and write from disk. See also the comments in
+ * {@link CostsCalculator}.
+ *
+ * Design doc: TODO: link to design doc or SEP
+ */
+public class AdaptiveController extends Controller
+{
+    private static final Logger logger = LoggerFactory.getLogger(AdaptiveController.class);
+
+    /** The starting value for the scaling parameter */
+    private static final int DEFAULT_STARTING_SCALING_PARAMETER = 0;
+
+    /** The minimum valid value for the scaling parameter */
+    static final String MIN_SCALING_PARAMETER = "adaptive_min_scaling_parameter";
+    static private final int DEFAULT_MIN_SCALING_PARAMETER = UCS_ADAPTIVE_MIN_SCALING_PARAMETER.getIntWithLegacyFalback();
+
+    /** The maximum valid value for the scaling parameter */
+    static final String MAX_SCALING_PARAMETER = "adaptive_max_scaling_parameter";
+    static private final int DEFAULT_MAX_SCALING_PARAMETER = UCS_ADAPTIVE_MAX_SCALING_PARAMETER.getIntWithLegacyFalback();
+
+    /** The interval for periodically checking the optimal value for the scaling parameter */
+    static final String INTERVAL_SEC = "adaptive_interval_sec";
+    static private final int DEFAULT_INTERVAL_SEC = UCS_ADAPTIVE_INTERVAL_SEC.getIntWithLegacyFalback();
+
+    /** The gain is a number between 0 and 1 used to determine if a new choice of the scaling parameter is better than the current one */
+    static final String THRESHOLD = "adaptive_threshold";
+    private static final double DEFAULT_THRESHOLD = UCS_ADAPTIVE_THRESHOLD.getDoubleWithLegacyFallback();
+
+    /** Below the minimum cost we don't try to optimize the scaling parameter, we consider the current scaling parameter good enough. This is necessary because the cost
+     * can vanish to zero when there are neither reads nor writes and right now we don't know how to handle this case.  */
+    static final String MIN_COST = "adaptive_min_cost";
+    static private final int DEFAULT_MIN_COST = UCS_ADAPTIVE_MIN_COST.getIntWithLegacyFalback();
+
+    /** The maximum number of concurrent Adaptive Compactions */
+    static final String MAX_ADAPTIVE_COMPACTIONS = "max_adaptive_compactions";
+    private static final int DEFAULT_MAX_ADAPTIVE_COMPACTIONS = UCS_MAX_ADAPTIVE_COMPACTIONS.getIntWithLegacyFalback();
+    private final int intervalSec;
+    private final int minScalingParameter;
+    private final int maxScalingParameter;
+    private final double threshold;
+    private final int minCost;
+    /** Protected by the synchronized block in UnifiedCompactionStrategy#getNextBackgroundTasks */
+    private int[] scalingParameters;
+    private int[] previousScalingParameters;
+    private volatile long lastChecked;
+    private final int maxAdaptiveCompactions;
+
+    @VisibleForTesting
+    public AdaptiveController(MonotonicClock clock,
+                              Environment env,
+                              int[] scalingParameters,
+                              int[] previousScalingParameters,
+                              double[] survivalFactors,
+                              long dataSetSize,
+                              long minSSTableSize,
+                              long flushSizeOverride,
+                              long currentFlushSize,
+                              double maxSpaceOverhead,
+                              int maxSSTablesToCompact,
+                              long expiredSSTableCheckFrequency,
+                              boolean ignoreOverlapsInExpirationCheck,
+                              int baseShardCount,
+                              boolean isReplicaAware,
+                              long targetSStableSize,
+                              double sstableGrowthModifier,
+                              int reservedThreadsPerLevel,
+                              Reservations.Type reservationsType,
+                              Overlaps.InclusionMethod overlapInclusionMethod,
+                              boolean parallelizeOutputShards,
+                              boolean hasVectorType,
+                              int intervalSec,
+                              int minScalingParameter,
+                              int maxScalingParameter,
+                              double threshold,
+                              int minCost,
+                              int maxAdaptiveCompactions,
+                              String keyspaceName,
+                              String tableName)
+    {
+        super(clock,
+              env,
+              survivalFactors,
+              dataSetSize,
+              minSSTableSize,
+              flushSizeOverride,
+              currentFlushSize,
+              maxSpaceOverhead,
+              maxSSTablesToCompact,
+              expiredSSTableCheckFrequency,
+              ignoreOverlapsInExpirationCheck,
+              baseShardCount,
+              isReplicaAware,
+              targetSStableSize,
+              sstableGrowthModifier,
+              reservedThreadsPerLevel,
+              reservationsType,
+              overlapInclusionMethod,
+              parallelizeOutputShards,
+              hasVectorType);
+
+        this.scalingParameters = scalingParameters;
+        this.previousScalingParameters = previousScalingParameters;
+        this.intervalSec = intervalSec;
+        this.minScalingParameter = minScalingParameter;
+        this.maxScalingParameter = maxScalingParameter;
+        this.threshold = threshold;
+        this.minCost = minCost;
+        this.maxAdaptiveCompactions = maxAdaptiveCompactions;
+        this.keyspaceName = keyspaceName;
+        this.tableName = tableName;
+    }
+
+    static Controller fromOptions(Environment env,
+                                  double[] survivalFactors,
+                                  long dataSetSize,
+                                  long minSSTableSize,
+                                  long flushSizeOverride,
+                                  double maxSpaceOverhead,
+                                  int maxSSTablesToCompact,
+                                  long expiredSSTableCheckFrequency,
+                                  boolean ignoreOverlapsInExpirationCheck,
+                                  int baseShardCount,
+                                  boolean isReplicaAware,
+                                  long targetSSTableSize,
+                                  double sstableGrowthModifier,
+                                  int reservedThreadsPerLevel,
+                                  Reservations.Type reservationsType,
+                                  Overlaps.InclusionMethod overlapInclusionMethod,
+                                  boolean parallelizeOutputShards,
+                                  boolean hasVectorType,
+                                  String keyspaceName,
+                                  String tableName,
+                                  Map<String, String> options)
+    {
+        int[] scalingParameters = null;
+        long currentFlushSize = flushSizeOverride;
+
+        File f = getControllerConfigPath(keyspaceName, tableName);
+        try
+        {
+            JSONParser jsonParser = new JSONParser();
+            JSONObject jsonObject = (JSONObject) jsonParser.parse(new FileReader(f));
+            scalingParameters = readStoredScalingParameters((JSONArray) jsonObject.get("scaling_parameters"));
+            if (jsonObject.get("current_flush_size") != null && flushSizeOverride == 0)
+            {
+                currentFlushSize = (long) jsonObject.get("current_flush_size");
+                logger.debug("Successfully read stored current_flush_size from disk");
+            }
+        }
+        catch (IOException e)
+        {
+            logger.debug("No controller config file found. Using starting value instead.");
+        }
+        catch (ParseException e)
+        {
+            logger.warn("Unable to parse saved options. Using starting value instead:", e);
+        }
+        catch (FSError e)
+        {
+            logger.warn("Unable to read controller config file. Using starting value instead:", e);
+        }
+        catch (Throwable e)
+        {
+            logger.warn("Unable to read controller config file. Using starting value instead:", e);
+            JVMStabilityInspector.inspectThrowable(e);
+        }
+
+        if (scalingParameters == null)
+        {
+            logger.info("Unable to read scaling_parameters. Using starting value instead.");
+            scalingParameters = new int[UnifiedCompactionStrategy.MAX_LEVELS];
+            String staticScalingParameters = options.remove(SCALING_PARAMETERS_OPTION);
+            String staticScalingFactors = options.remove(STATIC_SCALING_FACTORS_OPTION);
+
+            if (staticScalingParameters != null)
+            {
+                int[] parameters = parseScalingParameters(staticScalingParameters);
+                for (int i = 0; i < scalingParameters.length; i++)
+                {
+                    if (i < parameters.length)
+                        scalingParameters[i] = parameters[i];
+                    else
+                        scalingParameters[i] = scalingParameters[i-1];
+                }
+            }
+            else if (staticScalingFactors != null)
+            {
+                int[] factors = parseScalingParameters(staticScalingFactors);
+                for (int i = 0; i < scalingParameters.length; i++)
+                {
+                    if (i < factors.length)
+                        scalingParameters[i] = factors[i];
+                    else
+                        scalingParameters[i] = scalingParameters[i-1];
+                }
+                logger.info("Option: '{}' used to initialize scaling parameters for Adaptive Controller", STATIC_SCALING_FACTORS_OPTION);
+            }
+            else
+                Arrays.fill(scalingParameters, DEFAULT_STARTING_SCALING_PARAMETER);
+        }
+        else
+        {
+            logger.debug("Successfully read stored scaling parameters from disk.");
+            if (options.containsKey(SCALING_PARAMETERS_OPTION))
+                logger.warn("Option: '{}' is defined but not used.  Stored configuration was used instead", SCALING_PARAMETERS_OPTION);
+            if (options.containsKey(STATIC_SCALING_FACTORS_OPTION))
+                logger.warn("Option: '{}' is defined but not used.  Stored configuration was used instead", STATIC_SCALING_FACTORS_OPTION);
+        }
+        int[] previousScalingParameters = scalingParameters.clone();
+
+        int minScalingParameter = options.containsKey(MIN_SCALING_PARAMETER) ? Integer.parseInt(options.get(MIN_SCALING_PARAMETER)) : DEFAULT_MIN_SCALING_PARAMETER;
+        int maxScalingParameter = options.containsKey(MAX_SCALING_PARAMETER) ? Integer.parseInt(options.get(MAX_SCALING_PARAMETER)) : DEFAULT_MAX_SCALING_PARAMETER;
+        int intervalSec = options.containsKey(INTERVAL_SEC) ? Integer.parseInt(options.get(INTERVAL_SEC)) : DEFAULT_INTERVAL_SEC;
+        double threshold = options.containsKey(THRESHOLD) ? Double.parseDouble(options.get(THRESHOLD)) : DEFAULT_THRESHOLD;
+        int minCost = options.containsKey(MIN_COST) ? Integer.parseInt(options.get(MIN_COST)) : DEFAULT_MIN_COST;
+        int maxAdaptiveCompactions = options.containsKey(MAX_ADAPTIVE_COMPACTIONS) ? Integer.parseInt(options.get(MAX_ADAPTIVE_COMPACTIONS)) : DEFAULT_MAX_ADAPTIVE_COMPACTIONS;
+
+        return new AdaptiveController(MonotonicClock.Global.preciseTime,
+                                      env,
+                                      scalingParameters,
+                                      previousScalingParameters,
+                                      survivalFactors,
+                                      dataSetSize,
+                                      minSSTableSize,
+                                      flushSizeOverride,
+                                      currentFlushSize,
+                                      maxSpaceOverhead,
+                                      maxSSTablesToCompact,
+                                      expiredSSTableCheckFrequency,
+                                      ignoreOverlapsInExpirationCheck,
+                                      baseShardCount,
+                                      isReplicaAware,
+                                      targetSSTableSize,
+                                      sstableGrowthModifier,
+                                      reservedThreadsPerLevel,
+                                      reservationsType,
+                                      overlapInclusionMethod,
+                                      parallelizeOutputShards,
+                                      hasVectorType,
+                                      intervalSec,
+                                      minScalingParameter,
+                                      maxScalingParameter,
+                                      threshold,
+                                      minCost,
+                                      maxAdaptiveCompactions,
+                                      keyspaceName,
+                                      tableName);
+    }
+
+    private static int[] readStoredScalingParameters(JSONArray storedScalingParameters)
+    {
+        if (storedScalingParameters.size() > 0)
+        {
+            int[] scalingParameters = new int[UnifiedCompactionStrategy.MAX_LEVELS];
+            for (int i = 0; i < scalingParameters.length; i++)
+            {
+                //if the file does not have enough entries, use the last entry for the rest of the levels
+                if (i < storedScalingParameters.size())
+                    scalingParameters[i] = ((Long) storedScalingParameters.get(i)).intValue();
+                else
+                    scalingParameters[i] = scalingParameters[i-1];
+            }
+            //successfuly read scaling_parameters
+            return scalingParameters;
+        }
+        else
+        {
+            return null;
+        }
+    }
+
+    public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
+    {
+        int scalingParameter = DEFAULT_STARTING_SCALING_PARAMETER;
+        int minScalingParameter = DEFAULT_MIN_SCALING_PARAMETER;
+        int maxScalingParameter = DEFAULT_MAX_SCALING_PARAMETER;
+
+        String s;
+        String staticScalingFactors = options.remove(STATIC_SCALING_FACTORS_OPTION);
+        String staticScalingParameters = options.remove(SCALING_PARAMETERS_OPTION);
+        if (staticScalingFactors != null && staticScalingParameters != null)
+            throw new ConfigurationException(String.format("Either '%s' or '%s' should be used, not both", SCALING_PARAMETERS_OPTION, STATIC_SCALING_FACTORS_OPTION));
+        else if (staticScalingFactors != null)
+            parseScalingParameters(staticScalingFactors);
+        else if (staticScalingParameters != null)
+            parseScalingParameters(staticScalingParameters);
+        s = options.remove(MIN_SCALING_PARAMETER);
+        if (s != null)
+            minScalingParameter = Integer.parseInt(s);
+        s = options.remove(MAX_SCALING_PARAMETER);
+        if (s != null)
+            maxScalingParameter = Integer.parseInt(s);
+
+        if (minScalingParameter >= maxScalingParameter || scalingParameter < minScalingParameter || scalingParameter > maxScalingParameter)
+            throw new ConfigurationException(String.format("Invalid configuration for the scaling parameter: %d, min: %d, max: %d", scalingParameter, minScalingParameter, maxScalingParameter));
+
+        s = options.remove(INTERVAL_SEC);
+        if (s != null)
+        {
+            int intervalSec = Integer.parseInt(s);
+            if (intervalSec <= 0)
+                throw new ConfigurationException(String.format("Invalid configuration for interval, it should be positive: %d", intervalSec));
+        }
+        s = options.remove(THRESHOLD);
+        if (s != null)
+        {
+            double threshold = Double.parseDouble(s);
+            if (threshold <= 0 || threshold > 1)
+            {
+                throw new ConfigurationException(String.format("Invalid configuration for threshold, it should be within (0,1]: %f", threshold));
+            }
+        }
+        s = options.remove(MIN_COST);
+        if (s != null)
+        {
+            int minCost = Integer.parseInt(s);
+            if (minCost <= 0)
+                throw new ConfigurationException(String.format("Invalid configuration for minCost, it should be positive: %d", minCost));
+        }
+        s = options.remove(MAX_ADAPTIVE_COMPACTIONS);
+        if (s != null)
+        {
+            int maxAdaptiveCompactions = Integer.parseInt(s);
+            if (maxAdaptiveCompactions < -1)
+                throw new ConfigurationException(String.format("Invalid configuration for maxAdaptiveCompactions, it should be >= -1 (-1 for no limit): %d", maxAdaptiveCompactions));
+        }
+        return options;
+    }
+
+    @Override
+    void startup(UnifiedCompactionStrategy strategy, CostsCalculator calculator)
+    {
+        super.startup(strategy, calculator);
+        this.lastChecked = clock.now();
+    }
+
+    @Override
+    public int getScalingParameter(int index)
+    {
+        if (index < 0)
+            throw new IllegalArgumentException("Index should be >= 0: " + index);
+
+        return index < scalingParameters.length ? scalingParameters[index] : scalingParameters[scalingParameters.length - 1];
+    }
+
+    @Override
+    public int getPreviousScalingParameter(int index)
+    {
+        if (index < 0)
+            throw new IllegalArgumentException("Index should be >= 0: " + index);
+
+        return index < previousScalingParameters.length ? previousScalingParameters[index] : previousScalingParameters[previousScalingParameters.length - 1];
+    }
+
+    @Override
+    @Nullable
+    public CostsCalculator getCalculator()
+    {
+        return calculator;
+    }
+
+    public int getInterval()
+    {
+        return intervalSec;
+    }
+
+    public int getMinScalingParameter()
+    {
+        return minScalingParameter;
+    }
+
+    public int getMaxScalingParameter()
+    {
+        return maxScalingParameter;
+    }
+
+    public double getThreshold()
+    {
+        return threshold;
+    }
+
+    public int getMinCost()
+    {
+        return minCost;
+    }
+
+    /**
+     * Checks to see if the chosen compaction is a result of recent adaptive parameter change.
+     * An adaptive compaction is a compaction triggered by changing the scaling parameter W
+     */
+    @Override
+    public boolean isRecentAdaptive(CompactionPick pick)
+    {
+        int numTables = pick.sstables().size();
+        int level = (int) pick.parent();
+        return (numTables >= getThreshold(level) && numTables < getPreviousThreshold(level));
+    }
+
+    @Override
+    public int getMaxRecentAdaptiveCompactions()
+    {
+        return maxAdaptiveCompactions;
+    }
+
+    /** Protected by the synchronized block in UnifiedCompactionStrategy#getNextBackgroundTasks */
+    @Override
+    public void onStrategyBackgroundTaskRequest()
+    {
+        if (!isRunning())
+            return;
+
+        long now = clock.now();
+        if (now - lastChecked < TimeUnit.SECONDS.toNanos(intervalSec))
+            return;
+
+        try
+        {
+            maybeUpdate(now);
+        }
+        finally
+        {
+            lastChecked = now;
+        }
+    }
+
+    /**
+     * Maybe updates the scaling parameter according to the data size, read, and write costs.
+     *
+     * The scaling parameter calculation is based on current read and write query costs for the entire data size.
+     * We use the entire data size instead of shard size here because query cost calculations do not take
+     * sharding into account. Also, the same scaling parameter is going to be used across all shards.
+     *
+     * Protected by the synchronized block in UnifiedCompactionStrategy#getNextBackgroundTasks
+     *
+     * @param now current timestamp only used for debug logging
+     */
+    private void maybeUpdate(long now)
+    {
+        final long targetSize = Math.max(getDataSetSizeBytes(), (long) Math.ceil(calculator.spaceUsed()));
+
+        final int RA = readAmplification(targetSize, scalingParameters[0]);
+        final int WA = writeAmplification(targetSize, scalingParameters[0]);
+
+        final double readCost = calculator.getReadCostForQueries(RA);
+        final double writeCost = calculator.getWriteCostForQueries(WA);
+        final double cost =  readCost + writeCost;
+
+        if (cost <= minCost)
+        {
+            logger.debug("Adaptive compaction controller not updated, cost for current scaling parameter {} is below minimum cost {}: read cost: {}, write cost: {}\nAverages: {}", scalingParameters[0], minCost, readCost, writeCost, calculator);
+            return;
+        }
+
+        final double[] totCosts = new double[maxScalingParameter - minScalingParameter + 1];
+        final double[] readCosts = new double[maxScalingParameter - minScalingParameter + 1];
+        final double[] writeCosts = new double[maxScalingParameter - minScalingParameter + 1];
+        int candScalingParameter = scalingParameters[0];
+        double candCost = cost;
+
+        for (int i = minScalingParameter; i <= maxScalingParameter; i++)
+        {
+            final int idx = i - minScalingParameter;
+            if (i == scalingParameters[0])
+            {
+                readCosts[idx] = readCost;
+                writeCosts[idx] = writeCost;
+            }
+            else
+            {
+                final int ra = readAmplification(targetSize, i);
+                final int wa = writeAmplification(targetSize, i);
+
+                readCosts[idx] = calculator.getReadCostForQueries(ra);
+                writeCosts[idx] = calculator.getWriteCostForQueries(wa);
+            }
+            totCosts[idx] = readCosts[idx] + writeCosts[idx];
+            // in case of a tie, for neg.ve scalingParameters we prefer higher scalingParameters (smaller WA), but not for pos.ve scalingParameters we prefer lower scalingParameters (more parallelism)
+            if (totCosts[idx] < candCost || (i < 0 && totCosts[idx] == candCost))
+            {
+                candScalingParameter = i;
+                candCost = totCosts[idx];
+            }
+        }
+
+        logger.debug("Min cost: {}, min scaling parameter: {}, target sstable size: {}\nread costs: {}\nwrite costs: {}\ntot costs: {}\nAverages: {}",
+                     candCost,
+                     candScalingParameter,
+                     FBUtilities.prettyPrintMemory(getTargetSSTableSize()),
+                     Arrays.toString(readCosts),
+                     Arrays.toString(writeCosts),
+                     Arrays.toString(totCosts),
+                     calculator);
+
+        StringBuilder str = new StringBuilder(100);
+        str.append("Adaptive compaction controller ");
+
+        if (scalingParameters[0] != candScalingParameter && (cost - candCost) >= threshold * cost)
+        {
+            //scaling parameter is updated
+            str.append("updated ").append(scalingParameters[0]).append(" -> ").append(candScalingParameter);
+            this.previousScalingParameters[0] = scalingParameters[0]; //need to keep track of the previous scaling parameter for isAdaptive check
+            this.scalingParameters[0] = candScalingParameter;
+
+            //store updated scaling parameters in case a node fails and needs to restart
+            storeControllerConfig();
+        }
+        else if (scalingParameters[0] == candScalingParameter)
+        {
+            // only update the lowest level that is not equal to candScalingParameter
+            // example: candScalingParameter = 4, scalingParameters = {4, 4, 12, 16} --> scalingParameters = {4, 4, 4, 16}
+            // as a result, higher levels will be less prone to changes
+            for (int i = 1; i < scalingParameters.length; i++)
+            {
+                if (scalingParameters[i] != candScalingParameter)
+                {
+                    str.append("updated for level ").append(i).append(": ").append(scalingParameters[i]).append(" -> ").append(candScalingParameter);
+                    this.previousScalingParameters[i] = scalingParameters[i];
+                    this.scalingParameters[i] = candScalingParameter;
+
+                    //store updated scaling parameters in case a node fails and needs to restart
+                    storeControllerConfig();
+                    break;
+                }
+                else if (i == scalingParameters.length-1)
+                {
+                    str.append("unchanged because all levels have the same scaling parameter");
+                }
+            }
+        }
+        else
+        {
+            //scaling parameter is not updated
+            str.append("unchanged");
+        }
+
+        str.append(", data size: ").append(FBUtilities.prettyPrintMemory(targetSize));
+        str.append(", query cost: ").append(cost);
+        str.append(", new query cost: ").append(candCost);
+        str.append(", took ").append(TimeUnit.NANOSECONDS.toMicros(clock.now() - now)).append(" us");
+
+        logger.debug(str.toString());
+    }
+
+    @Override
+    public void storeControllerConfig()
+    {
+        storeOptions(keyspaceName, tableName, scalingParameters, getFlushSizeBytes());
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("t: %s, o: %s, scalingParameters: %s - %s", FBUtilities.prettyPrintMemory(targetSSTableSize), Arrays.toString(survivalFactors), Arrays.toString(scalingParameters), calculator);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/Controller.java b/src/java/org/apache/cassandra/db/compaction/unified/Controller.java
index cd7a35d44d92..5b370ff5217b 100644
--- a/src/java/org/apache/cassandra/db/compaction/unified/Controller.java
+++ b/src/java/org/apache/cassandra/db/compaction/unified/Controller.java
@@ -1,13 +1,11 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Copyright DataStax, Inc.
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,76 +16,173 @@
 
 package org.apache.cassandra.db.compaction.unified;
 
+import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
 
-import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.CompactionAggregate;
+import org.apache.cassandra.db.compaction.CompactionPick;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.db.compaction.CompactionStrategy;
 import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.utils.Overlaps;
+import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileWriter;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+import org.apache.cassandra.metrics.MetricNameFactory;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.Overlaps;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.*;
 
 /**
 * The controller provides compaction parameters to the unified compaction strategy
 */
-public class Controller
+// TODO there is a lot to be done with configuration - conversions, parsing, defaulting of configuration options should
+//   moved to some configuration utilities or use the existing ones. Also, maybe we should consider moving the config
+//   part out from this class into a dedicated compaction configuration class.
+public abstract class Controller
 {
     protected static final Logger logger = LoggerFactory.getLogger(Controller.class);
+    private static final ConcurrentMap<TableMetadata, Controller.Metrics> allMetrics = new ConcurrentHashMap<>();
 
     /**
-     * The scaling parameters W, one per bucket index and separated by a comma.
-     * Higher indexes will use the value of the last index with a W specified.
+     * The data size in GB, it will be assumed that the node will have on disk roughly this size of data when it
+     * reaches equilibrium. The default is calculated by looking at the free space on all data directories, adjusting
+     * for ones belonging to the same drive.
+     */
+    public static final String DATASET_SIZE_OPTION = "dataset_size";
+    /** @deprecated See STAR-1878 */
+    @Deprecated(since = "CC 4.0")
+    public static final String DATASET_SIZE_OPTION_GB = "dataset_size_in_gb";
+    static final long DEFAULT_DATASET_SIZE = UCS_DATASET_SIZE.getSizeInBytesWithLegacyFallback(DatabaseDescriptor.getDataFileDirectoriesMinTotalSpaceInGB() << 30);
+
+    /**
+     * The number of shards. This is the main configuration option for UCS V1 (i.e. before the density/overlap
+     * improvements). If the value is set, the strategy will switch to V1 mode which entails:
+     * <ul>
+     * <li>base_shard_count = num_shards
+     * <li>sstable_growth = 1 (i.e. always use the same number of shards)
+     * <li>min_sstable_size = auto (i.e. set from the size of first flush)
+     * <li>reserved_threads_per_level = max
+     * </ul>
+     * The option is undefined by default to engage the density version of UCS.
+     */
+    /** @deprecated See STAR-1878 */
+    @Deprecated(since = "CC 4.0")
+    static final String NUM_SHARDS_OPTION = "num_shards";
+
+    /**
+     * The default number of shards defined via system property, see {@link #NUM_SHARDS_OPTION}.
+     * The property exists for backward compatibility, and is deprecated. It allows for configuring compactors, writers
+     * and replayers in CNDB without having to change the schema for each tenant.
      */
-    final static String SCALING_PARAMETERS_OPTION = "scaling_parameters";
-    private final static String DEFAULT_SCALING_PARAMETERS =
-        CassandraRelevantProperties.UCS_SCALING_PARAMETER.getString();
+    /** @deprecated See STAR-1898 */
+    @Deprecated(since = "CC 4.0")
+    static final Optional<Integer> DEFAULT_NUM_SHARDS = Optional.ofNullable(UCS_NUM_SHARDS.getStringWithLegacyFallback()).map(Integer::valueOf);
 
     /**
      * The minimum sstable size. Sharded writers split sstables over shard only if they are at least as large as the
      * minimum size.
+     * <p>
+     * This is mainly present to support UCS V1 mode, which relies heavily on minimal SSTable
+     * size, and defaults to 0 which provides minimal parallelism on all levels of the hierarchy.
+     * In UCS V1 mode (engaged by using "num_shards" above) the default 'auto'.
      */
     static final String MIN_SSTABLE_SIZE_OPTION = "min_sstable_size";
+    /** @deprecated See STAR-1878 */
+    @Deprecated(since = "CC 4.0")
+    static final String MIN_SSTABLE_SIZE_OPTION_MB = "min_sstable_size_in_mb";
+    static final String MIN_SSTABLE_SIZE_OPTION_AUTO = "auto";
 
-    private static final String DEFAULT_MIN_SSTABLE_SIZE = CassandraRelevantProperties.UCS_MIN_SSTABLE_SIZE.getString();
+    static final long DEFAULT_MIN_SSTABLE_SIZE = UCS_MIN_SSTABLE_SIZE.getSizeInBytesWithLegacyFallback();
+    static final long DEFAULT_VECTOR_MIN_SSTABLE_SIZE = UCS_VECTOR_MIN_SSTABLE_SIZE.getSizeInBytesWithLegacyFallback();
+    /**
+     * Value to use to set the min sstable size from the flush size.
+     */
+    static final long MIN_SSTABLE_SIZE_AUTO = -1;
 
     /**
      * Override for the flush size in MB. The database should be able to calculate this from executing flushes, this
      * should only be necessary in rare cases.
      */
     static final String FLUSH_SIZE_OVERRIDE_OPTION = "flush_size_override";
+    /** @deprecated See STAR-1878 */
+    @Deprecated(since = "CC 4.0")
+    static final String FLUSH_SIZE_OVERRIDE_OPTION_MB = "flush_size_override_mb";
+
+    /**
+     * The maximum tolerable compaction-induced space amplification, as fraction of the dataset size. The idea behind
+     * this property is to be able to tune how much to limit concurrent "oversized" compactions in different shards.
+     * On one hand allowing such compactions concurrently running in all shards allows for STCS-like space
+     * amplification, where at some point you might need free space double the size of your working set to do a (top
+     * tier) compaction, while on the other hand limiting such compactions too much might lead to compaction lagging
+     * behind, higher read amplification, and other problems of that nature.
+     */
+    public static final String MAX_SPACE_OVERHEAD_OPTION = "max_space_overhead";
+    static final double DEFAULT_MAX_SPACE_OVERHEAD = UCS_MAX_SPACE_OVERHEAD.getDoubleWithLegacyFallback();
+    static final double MAX_SPACE_OVERHEAD_LOWER_BOUND = 0.01;
+    static final double MAX_SPACE_OVERHEAD_UPPER_BOUND = 1.0;
 
     static final String BASE_SHARD_COUNT_OPTION = "base_shard_count";
     /**
-     * Default base shard count, used when a base count is not explicitly supplied. This value applies as long as the
-     * table is not a system one, and directories are not defined.
+     * Default base shard count, used when a base count is not explicitly supplied. This value applies to all tables as
+     * long as they are larger than the minimum sstable size.
      *
      * For others a base count of 1 is used as system tables are usually small and do not need as much compaction
      * parallelism, while having directories defined provides for parallelism in a different way.
      */
-    public static final int DEFAULT_BASE_SHARD_COUNT =
-        CassandraRelevantProperties.UCS_BASE_SHARD_COUNT.getInt();
+    public static final int DEFAULT_BASE_SHARD_COUNT = UCS_BASE_SHARD_COUNT.getIntWithLegacyFalback();
+    public static final int DEFAULT_VECTOR_BASE_SHARD_COUNT = UCS_VECTOR_BASE_SHARD_COUNT.getIntWithLegacyFalback();
 
+    /**
+     * The target SSTable size. This is the size of the SSTables that the controller will try to create.
+     */
     static final String TARGET_SSTABLE_SIZE_OPTION = "target_sstable_size";
-    public static final long DEFAULT_TARGET_SSTABLE_SIZE =
-        CassandraRelevantProperties.UCS_TARGET_SSTABLE_SIZE.getSizeInBytes();
+    public static final long DEFAULT_TARGET_SSTABLE_SIZE = UCS_TARGET_SSTABLE_SIZE.getSizeInBytesWithLegacyFallback();
+    public static final long DEFAULT_VECTOR_TARGET_SSTABLE_SIZE = UCS_VECTOR_TARGET_SSTABLE_SIZE.getSizeInBytesWithLegacyFallback();
     static final long MIN_TARGET_SSTABLE_SIZE = 1L << 20;
 
+    static final String IS_REPLICA_AWARE_OPTION = "is_replica_aware";
+    public static final boolean DEFAULT_IS_REPLICA_AWARE = UCS_IS_REPLICA_AWARE.getBoolean();
+
     /**
-     * Provision for growth of the constructed SSTables as the size of the data grows. By default, the target SSTable
-     * size is fixed for all levels. In some scenarios it may be better to reduce the overall number of SSTables when
+     * Provision for growth of the constructed SSTables as the size of the data grows. By default the target SSTable
+     * size is fixed for all levels. In some scenarios is may be better to reduce the overall number of SSTables when
      * the data size becomes larger to avoid using too much memory and processing for the corresponding structures.
      * The setting enables such control and determines how much we reduce the growth of the number of split points as
-     * the data size grows. The number specifies the SSTable growth part, and the difference from 1 is the shard count
+     * the data size grows. The number specifies the sstable growth part, and the difference from 1 is the shard count
      * growth component, which is a multiplier applied to the logarithm of the data size, before it is rounded and
      * applied as an exponent in the number of split points. In other words, the given value applies as a negative
      * exponent in the calculation of the number of split points.
@@ -96,7 +191,7 @@ public class Controller
      * target size. Setting this number to 1 will make UCS never split beyong the base shard count. Using 0.5 will
      * make the number of split points a square root of the required number for the target SSTable size, making
      * the number of split points and the size of SSTables grow in lockstep as the density grows. Using
-     * 0.333 (the default) makes the sstable growth the cubic root of the density growth, i.e. the SSTable size
+     * 0.333 (the default) makes the sstable growth the cubic root of the density growth, i.e. the sstable size
      * grows with the square root of the growth of the shard count.
      * <p>
      * For example, given a data size of 1TiB on the top density level and 1GiB target size with base shard count of 1,
@@ -108,38 +203,96 @@ public class Controller
      * a growth value of 0.333, and 64 (~16GiB each) for a growth value of 0.5.
      */
     static final String SSTABLE_GROWTH_OPTION = "sstable_growth";
-    private static final double DEFAULT_SSTABLE_GROWTH = CassandraRelevantProperties.UCS_SSTABLE_GROWTH.getDouble();
+    static final double DEFAULT_SSTABLE_GROWTH = UCS_SSTABLE_GROWTH.getPercentageWithLegacyFallback();
+    static final double DEFAULT_VECTOR_SSTABLE_GROWTH = UCS_VECTOR_SSTABLE_GROWTH.getPercentageWithLegacyFallback();
+
+    /**
+     * Number of reserved threads to keep for each compaction level. This is used to ensure that there are always
+     * threads ready to start processing a level when new data arrives. This is most valuable to prevent large
+     * compactions from keeping all threads busy for a long time; with smaller target sizes the overlap-driven
+     * preference mechanism should achieve better results.
+     * <p>
+     * If the number is greater than the number of compaction threads divided by the number of levels rounded down, the
+     * latter will apply. Specifying "max" reserves as many threads as possible for each level.
+     * <p>
+     * The default value is max, all compaction threads are distributed among the levels.
+     */
+    static final String RESERVED_THREADS_OPTION = "reserved_threads";
+    public static final int DEFAULT_RESERVED_THREADS = FBUtilities.parseIntAllowingMax(UCS_RESERVED_THREADS.getStringWithLegacyFallback("max"));
+    public static final int DEFAULT_VECTOR_RESERVED_THREADS = FBUtilities.parseIntAllowingMax(UCS_VECTOR_RESERVED_THREADS.getStringWithLegacyFallback("max"));
+
+    /**
+     * Reservation type, defining whether reservations can be used by lower levels. If set to `per_level`, the
+     * reservations are only used by the specific level. If set to `level_or_below`, the reservations can be used by
+     * the specific level as well as any one below it.
+     * <p>
+     * The default value is `level_or_below`.
+     */
+    static final String RESERVATIONS_TYPE_OPTION = "reservations_type";
+    public static final Reservations.Type DEFAULT_RESERVED_THREADS_TYPE = UCS_RESERVATIONS_TYPE_OPTION.getEnumWithLegacyFallback(true, Reservations.Type.class);
 
     /**
      * This parameter is intended to modify the shape of the LSM by taking into account the survival ratio of data, for now it is fixed to one.
      */
-    static final double DEFAULT_SURVIVAL_FACTOR =
-        CassandraRelevantProperties.UCS_SURVIVAL_FACTOR.getDouble();
-    static final double[] DEFAULT_SURVIVAL_FACTORS = new double[] { DEFAULT_SURVIVAL_FACTOR };
+    static final double DEFAULT_SURVIVAL_FACTOR = UCS_SURVIVAL_FACTOR.getDoubleWithLegacyFallback();
+    final static double[] DEFAULT_SURVIVAL_FACTORS = new double[] { DEFAULT_SURVIVAL_FACTOR };
+
+    /**
+     * Either true or false. This parameter determines which controller will be used.
+     */
+    static final String ADAPTIVE_OPTION = "adaptive";
+    static final boolean DEFAULT_ADAPTIVE = UCS_ADAPTIVE_ENABLED.getBooleanWithLegacyFallback();
 
     /**
      * The maximum number of sstables to compact in one operation.
      *
-     * The default is 32, which aims to keep the length of operations under control and prevent accummulation of
-     * sstables while compactions are taking place.
+     * This is expected to be large and never be reached, but compaction going very very late may cause the accumulation
+     * of thousands and even tens of thousands of sstables which may cause problems if compacted in one long operation.
+     * The default is chosen to be half of the maximum permitted space overhead when the source sstables are of the
+     * minimum sstable size.
      *
      * If the fanout factor is larger than the maximum number of sstables, the strategy will ignore the latter.
      */
     static final String MAX_SSTABLES_TO_COMPACT_OPTION = "max_sstables_to_compact";
 
     static final String ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION = "unsafe_aggressive_sstable_expiration";
-    static final boolean ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION =
-        CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.getBoolean();
+    static final String ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_PROPERTY = Config.PROPERTY_PREFIX + "allow_unsafe_aggressive_sstable_expiration";
+    static final boolean ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION = CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.getBoolean();
     static final boolean DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION = false;
 
+    /**
+     * This property allows seperate defaults for vector and non-vector tables.  If this property is set to true
+     * and the table has a {@link VectorType}, the "vector" defaults are used over the regular defaults.  For instance,
+     * "-Dunified_compaction.vector_sstable_growth" will be used over "-Dunified_compaction.sstable_growth".
+     */
+    static final boolean OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES = UCS_OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES.getBoolean(false);
+
     static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS = 60 * 10;
     static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION = "expired_sstable_check_frequency_seconds";
 
+    /**
+     * Either true or false. This parameter determines whether L0 will use
+     * shards or not. If L0 does not use shards then:
+     * - all flushed sstables use an ordinary writer, not a sharded writer
+     * - the arena selector disregards the first token of L0 sstables, placing
+     *   them all in a unique shard.
+     */
+    static final String L0_SHARDS_ENABLED_OPTION = "l0_shards_enabled";
+    final static boolean DEFAULT_L0_SHARDS_ENABLED = UCS_L0_SHARDS_ENABLED.getBoolean();
+
+    /**
+     * True if L0 data may be coming from different replicas.
+     */
+    public static final String SHARED_STORAGE = "shared_storage";
+
     /** The maximum exponent for shard splitting. The maximum number of shards is this number the base count shifted this many times left. */
     static final int MAX_SHARD_SHIFT = 20;
     /** The maximum splitting factor for shards. The maximum number of shards is this number multiplied by the base count. */
     static final double MAX_SHARD_SPLIT = Math.scalb(1, MAX_SHARD_SHIFT);
 
+    private static final double INVERSE_LOG_2 = 1.0 / Math.log(2);
+    private static final double INVERSE_SQRT_2 = Math.sqrt(0.5);
+
     /**
      * Overlap inclusion method. NONE for participating sstables only (not recommended), SINGLE to only include sstables
      * that overlap with participating (LCS-like, higher concurrency during upgrades but some double compaction),
@@ -147,115 +300,198 @@ public class Controller
      */
     static final String OVERLAP_INCLUSION_METHOD_OPTION = "overlap_inclusion_method";
     static final Overlaps.InclusionMethod DEFAULT_OVERLAP_INCLUSION_METHOD =
-        CassandraRelevantProperties.UCS_OVERLAP_INCLUSION_METHOD.getEnum(Overlaps.InclusionMethod.TRANSITIVE);
+        Overlaps.InclusionMethod.valueOf(UCS_OVERLAP_INCLUSION_METHOD.getStringWithLegacyFallback(Overlaps.InclusionMethod.TRANSITIVE.toString()).toUpperCase());
+
+    /**
+     * Whether to create subtask for the output shards of individual compactions and execute them in parallel.
+     * Defaults to true for improved parallelization and efficiency.
+     */
+    static final String PARALLELIZE_OUTPUT_SHARDS_OPTION = "parallelize_output_shards";
+    static final boolean DEFAULT_PARALLELIZE_OUTPUT_SHARDS = UCS_PARALLELIZE_OUTPUT_SHARDS.getBooleanWithLegacyFallback();
+
+    /**
+     * The scaling parameters W, one per bucket index and separated by a comma.
+     * Higher indexes will use the value of the last index with a W specified.
+     */
+    static final String SCALING_PARAMETERS_OPTION = "scaling_parameters";
+    /** @deprecated See STAR-1898 */
+    @Deprecated(since = "CC 4.0")
+    static final String STATIC_SCALING_FACTORS_OPTION = "static_scaling_factors";
 
-    protected final ColumnFamilyStore cfs;
     protected final MonotonicClock clock;
-    private final int[] scalingParameters;
+    protected final Environment env;
     protected final double[] survivalFactors;
+    protected final long dataSetSize;
     protected volatile long minSSTableSize;
+    protected final double maxSpaceOverhead;
     protected final long flushSizeOverride;
     protected volatile long currentFlushSize;
     protected final int maxSSTablesToCompact;
     protected final long expiredSSTableCheckFrequency;
     protected final boolean ignoreOverlapsInExpirationCheck;
+    protected final boolean parallelizeOutputShards;
+    protected String keyspaceName;
+    protected String tableName;
 
     protected final int baseShardCount;
+    private final boolean isReplicaAware;
 
-    protected final double targetSSTableSize;
-
+    protected final long targetSSTableSize;
     protected final double sstableGrowthModifier;
 
-    static final double INVERSE_SQRT_2 = Math.sqrt(0.5);
+    protected final int reservedThreads;
+    protected final Reservations.Type reservationsType;
 
-    private static final double INVERSE_LOG_2 = 1.0 / Math.log(2);
+    @Nullable protected volatile CostsCalculator calculator;
+    @Nullable private volatile Metrics metrics;
 
     protected final Overlaps.InclusionMethod overlapInclusionMethod;
 
-    Controller(ColumnFamilyStore cfs,
-               MonotonicClock clock,
-               int[] scalingParameters,
+    final boolean l0ShardsEnabled;
+    final boolean hasVectorType;
+
+    Controller(MonotonicClock clock,
+               Environment env,
                double[] survivalFactors,
+               long dataSetSize,
                long minSSTableSize,
                long flushSizeOverride,
+               long currentFlushSize,
+               double maxSpaceOverhead,
                int maxSSTablesToCompact,
                long expiredSSTableCheckFrequency,
                boolean ignoreOverlapsInExpirationCheck,
                int baseShardCount,
-               double targetSStableSize,
+               boolean isReplicaAware,
+               long targetSStableSize,
                double sstableGrowthModifier,
-               Overlaps.InclusionMethod overlapInclusionMethod)
+               int reservedThreads,
+               Reservations.Type reservationsType,
+               Overlaps.InclusionMethod overlapInclusionMethod,
+               boolean parallelizeOutputShards,
+               boolean hasVectorType)
     {
-        this.cfs = cfs;
         this.clock = clock;
-        this.scalingParameters = scalingParameters;
+        this.env = env;
         this.survivalFactors = survivalFactors;
+        this.dataSetSize = dataSetSize;
         this.minSSTableSize = minSSTableSize;
         this.flushSizeOverride = flushSizeOverride;
-        this.currentFlushSize = flushSizeOverride;
+        this.currentFlushSize = currentFlushSize;
         this.expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(expiredSSTableCheckFrequency, TimeUnit.SECONDS);
         this.baseShardCount = baseShardCount;
+        this.isReplicaAware = isReplicaAware;
         this.targetSSTableSize = targetSStableSize;
         this.overlapInclusionMethod = overlapInclusionMethod;
         this.sstableGrowthModifier = sstableGrowthModifier;
+        this.reservedThreads = reservedThreads;
+        this.reservationsType = reservationsType;
+        this.maxSpaceOverhead = maxSpaceOverhead;
+        this.l0ShardsEnabled = UCS_L0_SHARDS_ENABLED.getBooleanWithLegacyFallback(false); // FIXME VECTOR-23
+        this.parallelizeOutputShards = parallelizeOutputShards;
+        this.hasVectorType = hasVectorType;
 
-        if (maxSSTablesToCompact <= 0)
-            maxSSTablesToCompact = Integer.MAX_VALUE;
+        if (maxSSTablesToCompact <= 0)  // use half the maximum permitted compaction size as upper bound by default
+            maxSSTablesToCompact = (int) (dataSetSize * this.maxSpaceOverhead * 0.5 / getMinSstableSizeBytes());
 
         this.maxSSTablesToCompact = maxSSTablesToCompact;
 
         if (ignoreOverlapsInExpirationCheck && !ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION)
         {
-            logger.warn("Not enabling aggressive SSTable expiration, as the system property '" +
-                        CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.name() +
-                        "' is set to 'false'. " +
-                        "Set it to 'true' to enable aggressive SSTable expiration.");
+            logger.warn("Not enabling aggressive SSTable expiration, as the system property '" + ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_PROPERTY + "' is set to 'false'. " +
+                    "Set it to 'true' to enable aggressive SSTable expiration.");
         }
         this.ignoreOverlapsInExpirationCheck = ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION && ignoreOverlapsInExpirationCheck;
     }
 
-    /**
-     * @return the scaling parameter W
-     * @param index
-     */
-    public int getScalingParameter(int index)
+    public static File getControllerConfigPath(String keyspaceName, String tableName)
     {
-        if (index < 0)
-            throw new IllegalArgumentException("Index should be >= 0: " + index);
+        String fileName = keyspaceName + '.' + tableName + '-' + "controller-config.JSON";
+        return new File(DatabaseDescriptor.getMetadataDirectory(), fileName);
+    }
+
+    public static void storeOptions(String keyspaceName, String tableName, int[] scalingParameters, long flushSizeBytes)
+    {
+        if (SchemaConstants.isSystemKeyspace(keyspaceName))
+            return;
+        File f = getControllerConfigPath(keyspaceName, tableName);
+        try(FileWriter fileWriter = new FileWriter(f, File.WriteMode.OVERWRITE);)
+        {
+            JSONArray jsonArray = new JSONArray();
+            JSONObject jsonObject = new JSONObject();
+            for (int i = 0; i < scalingParameters.length; i++)
+            {
+                jsonArray.add(scalingParameters[i]);
+            }
+            jsonObject.put("scaling_parameters", jsonArray);
+            jsonObject.put("current_flush_size", flushSizeBytes);
+            fileWriter.write(jsonObject.toString());
+            fileWriter.flush();
 
-        return index < scalingParameters.length ? scalingParameters[index] : scalingParameters[scalingParameters.length - 1];
+            logger.debug(String.format("Writing current scaling parameters and flush size to file %s: %s", f.toPath().toString(), jsonObject));
+        }
+        catch (IOException | FSError e)
+        {
+            logger.warn("Unable to save current scaling parameters and flush size. Current controller configuration will be lost if a node restarts: ", e);
+        }
+        catch (Throwable e)
+        {
+            logger.warn("Unable to save current scaling parameters and flush size. Current controller configuration will be lost if a node restarts: ", e);
+            JVMStabilityInspector.inspectThrowable(e);
+        }
     }
 
-    @Override
-    public String toString()
+    public abstract void storeControllerConfig();
+
+    @VisibleForTesting
+    public Environment getEnv()
     {
-        return String.format("Controller, m: %s, o: %s, Ws: %s",
-                             FBUtilities.prettyPrintBinary(targetSSTableSize, "B", ""),
-                             Arrays.toString(survivalFactors),
-                             printScalingParameters(scalingParameters));
+        return env;
     }
 
+    /**
+     * @return the scaling parameter W
+     * @param index
+     */
+    public abstract int getScalingParameter(int index);
+
+    public abstract int getPreviousScalingParameter(int index);
+
+    public abstract int getMaxRecentAdaptiveCompactions();
+    public abstract boolean isRecentAdaptive(CompactionPick pick);
+
     public int getFanout(int index) {
-        int W = getScalingParameter(index);
-        return UnifiedCompactionStrategy.fanoutFromScalingParameter(W);
+        return UnifiedCompactionStrategy.fanoutFromScalingParameter(getScalingParameter(index));
     }
 
     public int getThreshold(int index) {
-        int W = getScalingParameter(index);
-        return UnifiedCompactionStrategy.thresholdFromScalingParameter(W);
+        return UnifiedCompactionStrategy.thresholdFromScalingParameter(getScalingParameter(index));
+    }
+
+    public int getPreviousFanout(int index) {
+        return UnifiedCompactionStrategy.fanoutFromScalingParameter(getPreviousScalingParameter(index));
+    }
+
+    public int getPreviousThreshold(int index) {
+        return UnifiedCompactionStrategy.thresholdFromScalingParameter(getPreviousScalingParameter(index));
+    }
+
+    public int getFlushShards(double density)
+    {
+        return areL0ShardsEnabled() ? getNumShards(density) : 1;
     }
 
     /**
-     * Calculate the number of shards to split the local token space in for the given SSTable density.
-     * This is calculated as a power-of-two multiple of baseShardCount, so that the expected size of resulting SSTables
+     * Calculate the number of shards to split the local token space in for the given sstable density.
+     * This is calculated as a power-of-two multiple of baseShardCount, so that the expected size of resulting sstables
      * is between sqrt(0.5) and sqrt(2) times the target size, which is calculated from targetSSTableSize to grow
      * at the given sstableGrowthModifier of the exponential growth of the density.
      * <p>
-     * Additionally, if a minimum SSTable size is set, we can go below the baseShardCount when that would result in
-     * SSTables smaller than that minimum. Note that in the case of a non-power-of-two base count, we will only
-     * split to divisors of baseShardCount.
+     * Additionally, if a minimum sstable size is set, we can go below the baseShardCount when that would result in
+     * sstables smaller than that minimum. Note that in the case of a non-power-of-two base count this will cause
+     * smaller sstables to not be aligned with the ones whose size is enough for the base count.
      * <p>
-     * Note that to get the SSTables resulting from this splitting within the bounds, the density argument must be
+     * Note that to get the sstables resulting from this splitting within the bounds, the density argument must be
      * normalized to the span that is being split. In other words, if no disks are defined, the density should be
      * scaled by the token coverage of the locally-owned ranges. If multiple data directories are defined, the density
      * should be scaled by the token coverage of the respective data directory. That is, localDensity = size / span,
@@ -265,9 +501,10 @@ public int getNumShards(double localDensity)
     {
         int shards;
         // Check the minimum size first.
-        if (minSSTableSize > 0)
+        long minSize = getMinSstableSizeBytes();
+        if (minSize > 0)
         {
-            double count = localDensity / minSSTableSize;
+            double count = localDensity / minSize;
             // Minimum size only applies if the base count would result in smaller sstables.
             // We also want to use the min size if we don't yet know the flush size (density is NaN).
             // Note: the minimum size cannot be larger than the target size's minimum.
@@ -281,8 +518,8 @@ public int getNumShards(double localDensity)
                     logger.debug("Shard count {} for density {}, {} times min size {}",
                                  shards,
                                  FBUtilities.prettyPrintBinary(localDensity, "B", " "),
-                                 localDensity / minSSTableSize,
-                                 FBUtilities.prettyPrintBinary(minSSTableSize, "B", " "));
+                                 localDensity / minSize,
+                                 FBUtilities.prettyPrintBinary(minSize, "B", " "));
 
                 return shards;
             }
@@ -291,9 +528,10 @@ public int getNumShards(double localDensity)
         if (sstableGrowthModifier == 1)
         {
             shards = baseShardCount;
-            logger.debug("Shard count {} for density {} in fixed shards mode",
+            logger.debug("Shard count {} for density {} in fixed shards mode. SStableGrowthModifier {}",
                          shards,
-                         FBUtilities.prettyPrintBinary(localDensity, "B", " "));
+                         FBUtilities.prettyPrintBinary(localDensity, "B", " "),
+                         sstableGrowthModifier);
             return shards;
         }
         else if (sstableGrowthModifier == 0)
@@ -313,11 +551,12 @@ else if (sstableGrowthModifier == 0)
             shards = baseShardCount * Integer.highestOneBit((int) count | 1);
 
             if (logger.isDebugEnabled())
-                logger.debug("Shard count {} for density {}, {} times target {}",
+                logger.debug("Shard count {} for density {}, {} times target {}. SStableGrowthModifier {}",
                              shards,
                              FBUtilities.prettyPrintBinary(localDensity, "B", " "),
                              localDensity / targetSSTableSize,
-                             FBUtilities.prettyPrintBinary(targetSSTableSize, "B", " "));
+                             FBUtilities.prettyPrintBinary(targetSSTableSize, "B", " "),
+                             sstableGrowthModifier);
             return shards;
         }
         else
@@ -330,7 +569,7 @@ else if (sstableGrowthModifier == 0)
             // targetSSTableSize * sqrt(2). Finally, make sure the exponent is at least 0 and not greater than the
             // fixed maximum.
             // Note: This code also works correctly for the special cases of sstableGrowthModifier == 0 and 1,
-            //       but the above code avoids the imprecise floating point arithmetic for these common cases.
+            //       but the above code avoids the floating point arithmetic for these common cases.
             // Note: We use log instead of getExponent because we also need the non-integer part of the logarithm
             //       in order to apply the growth modifier correctly.
             final double countLog = Math.log(count);
@@ -345,16 +584,35 @@ else if (pow >= 0)
             if (logger.isDebugEnabled())
             {
                 long targetSize = (long) (targetSSTableSize * Math.exp(countLog * sstableGrowthModifier));
-                logger.debug("Shard count {} for density {}, {} times target {}",
+                logger.debug("Shard count {} for density {}, {} times target {}. SStableGrowthModifier {}",
                              shards,
                              FBUtilities.prettyPrintBinary(localDensity, "B", " "),
                              localDensity / targetSize,
-                             FBUtilities.prettyPrintBinary(targetSize, "B", " "));
+                             FBUtilities.prettyPrintBinary(targetSize, "B", " "),
+                             sstableGrowthModifier);
             }
             return shards;
         }
     }
 
+    public boolean parallelizeOutputShards()
+    {
+        return parallelizeOutputShards;
+    }
+
+    public boolean isReplicaAware()
+    {
+        return isReplicaAware;
+    }
+
+    /**
+     * @return whether L0 should use shards
+     */
+    public boolean areL0ShardsEnabled()
+    {
+        return l0ShardsEnabled;
+    }
+
     /**
      * @return the survival factor o
      * @param index
@@ -367,6 +625,57 @@ public double getSurvivalFactor(int index)
         return index < survivalFactors.length ? survivalFactors[index] : survivalFactors[survivalFactors.length - 1];
     }
 
+    /**
+     * The user specified dataset size.
+     *
+     * @return the target size of the entire data set, in bytes.
+     */
+    public long getDataSetSizeBytes()
+    {
+        return dataSetSize;
+    }
+
+    public long getTargetSSTableSize()
+    {
+        return targetSSTableSize;
+    }
+
+    /**
+     * Return the sstable size in bytes.
+     *
+     * This is either set by the user in the options or calculated by rounding up the first flush size to 50 MB.
+     *
+     * @return the minimum sstable size in bytes.
+     */
+    public long getMinSstableSizeBytes()
+    {
+        if (minSSTableSize >= 0)
+            return minSSTableSize;
+
+        synchronized (this)
+        {
+            if (minSSTableSize >= 0)
+                return minSSTableSize;
+
+            // round the avg flush size to the nearest byte
+            long envFlushSize = Math.round(env.flushSize());
+            long fiftyMB = 50 << 20;
+
+            // round up to 50 MB
+            long flushSize = ((Math.max(1, envFlushSize) + fiftyMB - 1) / fiftyMB) * fiftyMB;
+
+            // If the env flush size is positive, then we've flushed at least once and we use this value permanently
+            if (envFlushSize > 0)
+            {
+                // When a target size is specified, the minimum cannot be higher than the lower bound for that target size.
+                flushSize = Math.min(flushSize, (long) (targetSSTableSize * INVERSE_SQRT_2));
+                minSSTableSize = flushSize;
+            }
+
+            return flushSize;
+        }
+    }
+
     /**
      * Return the flush sstable size in bytes.
      *
@@ -381,7 +690,7 @@ public long getFlushSizeBytes()
         if (flushSizeOverride > 0)
             return flushSizeOverride;
 
-        double envFlushSize = cfs.metric.flushSizeOnDisk.get();
+        double envFlushSize = env.flushSize();
         if (currentFlushSize == 0 || Math.abs(1 - (currentFlushSize / envFlushSize)) > 0.5)
         {
             // The current size is not initialized, or it differs by over 50% from the observed.
@@ -391,6 +700,36 @@ public long getFlushSizeBytes()
         return currentFlushSize;
     }
 
+    /**
+     * Returns the maximum tolerable compaction-induced space amplification, as a fraction of the dataset size.
+     * Currently this is not a strict limit for which compaction gives an ironclad guarantee never to exceed it, but
+     * the main input in a simple heuristic that is designed to limit UCS' space amplification in exchange of some
+     * delay in top bucket compactions.
+     *
+     * @return a {@code double} value between 0.01 and 1.0, representing the fraction of the expected uncompacted
+     * dataset size that should be additionally available for compaction's space amplification overhead.
+     */
+    public double getMaxSpaceOverhead()
+    {
+        return maxSpaceOverhead;
+    }
+
+    /**
+     * Returns the number of reserved threads per level. If the size of SSTables is small, this can be 0 as operations
+     * finish quickly and the prioritization will do a good job of assigning threads to the levels. If the size of
+     * SSTables can grow large, threads must be reserved to ensure that compactions, esp. on level 0, do not have to
+     * wait for long operations to complete.
+     */
+    public int getReservedThreads()
+    {
+        return reservedThreads;
+    }
+
+    public Reservations.Type getReservationsType()
+    {
+        return reservationsType;
+    }
+
     /**
      * @return whether is allowed to drop expired SSTables without checking if partition keys appear in other SSTables.
      * Same behavior as in TWCS.
@@ -405,13 +744,195 @@ public long getExpiredSSTableCheckFrequency()
         return expiredSSTableCheckFrequency;
     }
 
-    public static Controller fromOptions(ColumnFamilyStore cfs, Map<String, String> options)
+    /**
+     * Perform any initialization that requires the strategy.
+     */
+    public void startup(UnifiedCompactionStrategy strategy, ScheduledExecutorService executorService)
+    {
+        if (calculator != null)
+            throw new IllegalStateException("Already started");
+
+        startup(strategy, new CostsCalculator(env, strategy, executorService));
+    }
+
+    @VisibleForTesting
+    void startup(UnifiedCompactionStrategy strategy, CostsCalculator calculator)
+    {
+        this.calculator = calculator;
+        metrics = allMetrics.computeIfAbsent(strategy.getMetadata(), Controller.Metrics::new);
+        metrics.setController(this);
+        logger.debug("Started compaction {}", this);
+    }
+
+    /**
+     * Signals that the strategy is about to be deleted or stopped.
+     */
+    public void shutdown()
+    {
+        if (calculator == null)
+            return;
+
+        calculator.close();
+        calculator = null;
+
+        if (metrics != null)
+        {
+            metrics.release();
+            metrics.removeController();
+            metrics = null;
+        }
+
+        logger.debug("Stopped compaction controller {}", this);
+    }
+
+    public boolean hasVectorType()
+    {
+        return hasVectorType;
+    }
+
+    /**
+     * @return true if the controller is running
+     */
+    public boolean isRunning()
+    {
+        return calculator != null;
+    }
+
+    /**
+     * @return the cost calculator, will be null until {@link this#startup(UnifiedCompactionStrategy, ScheduledExecutorService)} is called.
+     */
+    @Nullable
+    @VisibleForTesting
+    public CostsCalculator getCalculator()
+    {
+        return calculator;
+    }
+
+    /**
+     * The strategy will call this method each time {@link CompactionStrategy#getNextBackgroundTasks(long)} is called.
+     */
+    public void onStrategyBackgroundTaskRequest()
+    {
+    }
+
+    /**
+     * Calculate the read amplification assuming a single scaling parameter W and a given total
+     * length of data on disk.
+     *
+     * @param length the total length on disk
+     * @param scalingParameter the scaling parameter to use for the calculation
+     *
+     * @return the read amplification of all the buckets needed to cover the total length
+     */
+    public int readAmplification(long length, int scalingParameter)
     {
-        int[] Ws = parseScalingParameters(options.getOrDefault(SCALING_PARAMETERS_OPTION, DEFAULT_SCALING_PARAMETERS));
+        double o = getSurvivalFactor(0);
+        long m = getFlushSizeBytes();
+
+        int F = UnifiedCompactionStrategy.fanoutFromScalingParameter(scalingParameter);
+        int T = UnifiedCompactionStrategy.thresholdFromScalingParameter(scalingParameter);
+        int maxIndex = maxBucketIndex(length, F);
+
+        int ret = 0;
+        for (int i = 0; i < maxIndex; i++)
+            ret += T - 1;
+
+        if (scalingParameter >= 0)
+            ret += Math.max(0, Math.ceil(length / (m * Math.pow(o * F, maxIndex))) - 1);
+        else
+            ret += 1;
+
+        return ret;
+    }
 
-        long flushSizeOverride = FBUtilities.parseHumanReadableBytes(options.getOrDefault(FLUSH_SIZE_OVERRIDE_OPTION,
-                                                                                          "0MiB"));
-        int maxSSTablesToCompact = Integer.parseInt(options.getOrDefault(MAX_SSTABLES_TO_COMPACT_OPTION, "0"));
+    /**
+     * Calculate the write amplification assuming a single scaling parameter W and a given total
+     * length of data on disk.
+     *
+     * @param length the total length on disk
+     * @param scalingParameter the scaling parameter to use for the calculation
+     *
+     * @return the write amplification of all the buckets needed to cover the total length
+     */
+    public int writeAmplification(long length, int scalingParameter)
+    {
+        double o = getSurvivalFactor(0);
+        long m = getFlushSizeBytes();
+
+        int F = UnifiedCompactionStrategy.fanoutFromScalingParameter(scalingParameter);
+        int maxIndex = maxBucketIndex(length, F);
+
+        int ret = 0;
+
+        if (scalingParameter >= 0)
+        {   // for tiered, at each level the WA is 1. We start at level 0 and end up at level maxIndex so that's a WA of maxIndex.
+            ret += maxIndex + 1;
+        }
+        else
+        {   // for leveled, at each level the WA is F - 1 except for the last one, where it's (size / size of previous level) - 1
+            // or (size / (m*(o*F)^maxIndex)) - 1
+            for (int i = 0; i < maxIndex; i++)
+                ret += F - 1;
+
+            ret += Math.max(0, Math.ceil(length / (m * Math.pow(o * F, maxIndex))));
+        }
+
+        return ret;
+    }
+
+    /**
+     * Returns a maximum bucket index for the given data size and fanout.
+     */
+    private int maxBucketIndex(long totalLength, int fanout)
+    {
+        double o = getSurvivalFactor(0);
+        long m = getFlushSizeBytes();
+        return Math.max(0, (int) Math.floor((Math.log(totalLength) - Math.log(m)) / (Math.log(fanout) - Math.log(o))));
+    }
+
+    private double getReadIOCost()
+    {
+        if (calculator == null)
+            return 0;
+
+        int scalingParameter = getScalingParameter(0);
+        long length = (long) Math.ceil(calculator.spaceUsed());
+        return calculator.getReadCostForQueries(readAmplification(length, scalingParameter));
+    }
+
+    private double getWriteIOCost()
+    {
+        if (calculator == null)
+            return 0;
+
+        int scalingParameter = getScalingParameter(0);
+        long length = (long) Math.ceil(calculator.spaceUsed());
+        return calculator.getWriteCostForQueries(writeAmplification(length, scalingParameter));
+    }
+
+    public static Controller fromOptions(CompactionRealm realm, Map<String, String> options)
+    {
+        // Note: These options have been validated, but the defaults are configured with -D options that may be
+        // different. We thus may end up with configurations combinations that do not make sense.
+        // We will attempt to correct such combinations and issue warnings where possible.
+
+        boolean hasVectorType = realm.metadata().hasVectorType();
+        boolean vectorOverride = OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES;
+        boolean useVectorOptions = hasVectorType && vectorOverride;
+        if (logger.isTraceEnabled())
+        {
+            if (useVectorOptions)
+                logger.trace("Using UCS configuration optimized for vector for {}.{}", realm.getKeyspaceName(), realm.getTableName());
+            else
+                logger.trace("Using non-vector UCS configuration for {}.{}", realm.getKeyspaceName(), realm.getTableName());
+        }
+        boolean adaptive = options.containsKey(ADAPTIVE_OPTION) ? Boolean.parseBoolean(options.get(ADAPTIVE_OPTION)) : DEFAULT_ADAPTIVE;
+        long dataSetSize = getSizeWithAlt(options, DATASET_SIZE_OPTION, DATASET_SIZE_OPTION_GB, 30, DEFAULT_DATASET_SIZE);
+        long flushSizeOverride = getSizeWithAlt(options, FLUSH_SIZE_OVERRIDE_OPTION, FLUSH_SIZE_OVERRIDE_OPTION_MB, 20, 0);
+        double maxSpaceOverhead = options.containsKey(MAX_SPACE_OVERHEAD_OPTION)
+                ? FBUtilities.parsePercent(options.get(MAX_SPACE_OVERHEAD_OPTION))
+                : DEFAULT_MAX_SPACE_OVERHEAD;
+        int maxSSTablesToCompact = Integer.parseInt(options.getOrDefault(MAX_SSTABLES_TO_COMPACT_OPTION, "32"));
         long expiredSSTableCheckFrequency = options.containsKey(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION)
                 ? Long.parseLong(options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION))
                 : DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS;
@@ -426,115 +947,193 @@ public static Controller fromOptions(ColumnFamilyStore cfs, Map<String, String>
         }
         else
         {
-            baseShardCount = DEFAULT_BASE_SHARD_COUNT;
+            baseShardCount = useVectorOptions ? DEFAULT_VECTOR_BASE_SHARD_COUNT : DEFAULT_BASE_SHARD_COUNT;
         }
 
+        boolean isReplicaAware = options.containsKey(IS_REPLICA_AWARE_OPTION)
+                                 ? Boolean.parseBoolean(options.get(IS_REPLICA_AWARE_OPTION))
+                                 : DEFAULT_IS_REPLICA_AWARE;
+
         long targetSStableSize = options.containsKey(TARGET_SSTABLE_SIZE_OPTION)
-                ? FBUtilities.parseHumanReadableBytes(options.get(TARGET_SSTABLE_SIZE_OPTION))
-                : DEFAULT_TARGET_SSTABLE_SIZE;
+                                 ? FBUtilities.parseHumanReadableBytes(options.get(TARGET_SSTABLE_SIZE_OPTION))
+                                 : useVectorOptions ? DEFAULT_VECTOR_TARGET_SSTABLE_SIZE : DEFAULT_TARGET_SSTABLE_SIZE;
 
-        long minSSTableSize = options.containsKey(MIN_SSTABLE_SIZE_OPTION)
-                ? FBUtilities.parseHumanReadableBytes(options.get(MIN_SSTABLE_SIZE_OPTION))
-                : FBUtilities.parseHumanReadableBytes(DEFAULT_MIN_SSTABLE_SIZE);
+        long minSSTableSize;
+        if (MIN_SSTABLE_SIZE_OPTION_AUTO.equalsIgnoreCase(options.get(MIN_SSTABLE_SIZE_OPTION)))
+            minSSTableSize = MIN_SSTABLE_SIZE_AUTO;
+        else
+            minSSTableSize = getSizeWithAlt(options,
+                                            MIN_SSTABLE_SIZE_OPTION,
+                                            MIN_SSTABLE_SIZE_OPTION_MB,
+                                            20,
+                                            useVectorOptions ? DEFAULT_VECTOR_MIN_SSTABLE_SIZE : DEFAULT_MIN_SSTABLE_SIZE);
 
-        double sstableGrowthModifier = DEFAULT_SSTABLE_GROWTH;
+        double sstableGrowthModifier = useVectorOptions ? DEFAULT_VECTOR_SSTABLE_GROWTH : DEFAULT_SSTABLE_GROWTH;
         if (options.containsKey(SSTABLE_GROWTH_OPTION))
             sstableGrowthModifier = FBUtilities.parsePercent(options.get(SSTABLE_GROWTH_OPTION));
 
-        Overlaps.InclusionMethod inclusionMethod = options.containsKey(OVERLAP_INCLUSION_METHOD_OPTION)
-                ? Overlaps.InclusionMethod.valueOf(options.get(OVERLAP_INCLUSION_METHOD_OPTION).toUpperCase())
-                : DEFAULT_OVERLAP_INCLUSION_METHOD;
-
-        return new Controller(cfs,
-                              MonotonicClock.Global.preciseTime,
-                              Ws,
-                              DEFAULT_SURVIVAL_FACTORS,
-                              minSSTableSize,
-                              flushSizeOverride,
-                              maxSSTablesToCompact,
-                              expiredSSTableCheckFrequency,
-                              ignoreOverlapsInExpirationCheck,
-                              baseShardCount,
-                              targetSStableSize,
-                              sstableGrowthModifier,
-                              inclusionMethod);
+        int reservedThreadsPerLevel = options.containsKey(RESERVED_THREADS_OPTION)
+                                      ? FBUtilities.parseIntAllowingMax(options.get(RESERVED_THREADS_OPTION))
+                                      : useVectorOptions ? DEFAULT_VECTOR_RESERVED_THREADS : DEFAULT_RESERVED_THREADS;
+        Reservations.Type reservationsType = options.containsKey(RESERVATIONS_TYPE_OPTION)
+                                                  ? Reservations.Type.valueOf(options.get(RESERVATIONS_TYPE_OPTION).toUpperCase())
+                                                  : DEFAULT_RESERVED_THREADS_TYPE;
+
+        if (options.containsKey(NUM_SHARDS_OPTION) || DEFAULT_NUM_SHARDS.isPresent())
+        {
+            // Legacy V1 mode is enabled when the number of shards is defined and has a positive value.
+            // Table property takes precendence over system property.
+            int numShards = options.containsKey(NUM_SHARDS_OPTION)
+                            ? Integer.parseInt(options.get(NUM_SHARDS_OPTION))
+                            : DEFAULT_NUM_SHARDS.get();
+
+            if (numShards > 0)
+            {
+                if (!options.containsKey(MIN_SSTABLE_SIZE_OPTION))
+                    minSSTableSize = MIN_SSTABLE_SIZE_AUTO;
+                baseShardCount = numShards;
+                sstableGrowthModifier = 1.0;
+                targetSStableSize = Long.MAX_VALUE; // this no longer plays a part, the result of getNumShards before
+                // accounting for minimum size is always baseShardCount
+
+                double maxSpaceOverheadLowerBound = 1.0d / numShards;
+                if (maxSpaceOverhead < maxSpaceOverheadLowerBound)
+                {
+                    logger.warn("{} shards are not enough to maintain the required maximum space overhead of {}!\n" +
+                                "Falling back to {}={} instead. If this limit needs to be satisfied, please increase the number" +
+                                " of shards.",
+                                numShards,
+                                maxSpaceOverhead,
+                                MAX_SPACE_OVERHEAD_OPTION,
+                                String.format("%.3f", maxSpaceOverheadLowerBound));
+                    maxSpaceOverhead = maxSpaceOverheadLowerBound;
+                }
+            }
+        }
+
+        if (baseShardCount > 1 && sstableGrowthModifier != 1.0 && minSSTableSize != MIN_SSTABLE_SIZE_AUTO && minSSTableSize > targetSStableSize * INVERSE_SQRT_2)
+        {
+            // Note: not checked for baseShardCount == 1 as min size is irrelevant when the base count is 1.
+            // Note: not checked for sstableGrowthModifier = 1.0 as target size is irrelevant when the growth is 1.
+            long newTargetSize = (long) (minSSTableSize / INVERSE_SQRT_2);
+            logger.warn("Minimum sstable size {} is larger than target sstable size's minimum bound {}. Adjusting target size to {}.",
+                        FBUtilities.prettyPrintMemory(minSSTableSize),
+                        FBUtilities.prettyPrintMemory((long) (targetSStableSize * INVERSE_SQRT_2)),
+                        FBUtilities.prettyPrintMemory(newTargetSize));
+            targetSStableSize = newTargetSize;
+        }
+
+        Environment env = realm.makeUCSEnvironment();
+
+        // For remote storage, the sstables on L0 are created by the different replicas, and therefore it is likely
+        // that there are RF identical copies, so here we adjust the survival factor for L0
+        double[] survivalFactors = !UCS_SHARED_STORAGE.getBooleanWithLegacyFallback()
+                                   ? DEFAULT_SURVIVAL_FACTORS
+                                   : new double[] { DEFAULT_SURVIVAL_FACTOR / realm.getKeyspaceReplicationStrategy().getReplicationFactor().allReplicas, DEFAULT_SURVIVAL_FACTOR };
+
+        Overlaps.InclusionMethod overlapInclusionMethod = options.containsKey(OVERLAP_INCLUSION_METHOD_OPTION)
+                                                          ? Overlaps.InclusionMethod.valueOf(options.get(OVERLAP_INCLUSION_METHOD_OPTION).toUpperCase())
+                                                          : DEFAULT_OVERLAP_INCLUSION_METHOD;
+
+        boolean parallelizeOutputShards = options.containsKey(PARALLELIZE_OUTPUT_SHARDS_OPTION)
+                                          ? Boolean.parseBoolean(options.get(PARALLELIZE_OUTPUT_SHARDS_OPTION))
+                                          : DEFAULT_PARALLELIZE_OUTPUT_SHARDS;
+
+        return adaptive
+               ? AdaptiveController.fromOptions(env,
+                                                survivalFactors,
+                                                dataSetSize,
+                                                minSSTableSize,
+                                                flushSizeOverride,
+                                                maxSpaceOverhead,
+                                                maxSSTablesToCompact,
+                                                expiredSSTableCheckFrequency,
+                                                ignoreOverlapsInExpirationCheck,
+                                                baseShardCount,
+                                                isReplicaAware,
+                                                targetSStableSize,
+                                                sstableGrowthModifier,
+                                                reservedThreadsPerLevel,
+                                                reservationsType,
+                                                overlapInclusionMethod,
+                                                parallelizeOutputShards,
+                                                hasVectorType,
+                                                realm.getKeyspaceName(),
+                                                realm.getTableName(),
+                                                options)
+               : StaticController.fromOptions(env,
+                                              survivalFactors,
+                                              dataSetSize,
+                                              minSSTableSize,
+                                              flushSizeOverride,
+                                              maxSpaceOverhead,
+                                              maxSSTablesToCompact,
+                                              expiredSSTableCheckFrequency,
+                                              ignoreOverlapsInExpirationCheck,
+                                              baseShardCount,
+                                              isReplicaAware,
+                                              targetSStableSize,
+                                              sstableGrowthModifier,
+                                              reservedThreadsPerLevel,
+                                              reservationsType,
+                                              overlapInclusionMethod,
+                                              parallelizeOutputShards,
+                                              hasVectorType,
+                                              realm.getKeyspaceName(),
+                                              realm.getTableName(),
+                                              options,
+                                              useVectorOptions);
     }
 
     public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
     {
+        // Note: Validation must ignore the defaults set with -D options, because this node may be getting a configuration
+        // applied via a different coordinator which had different -D settings. If we abort because of such differences,
+        // we may cause schema mismatches between nodes which can quickly become a serious problem.
+
+        String nonPositiveErr = "Invalid configuration, %s should be positive: %d";
+        String intParseErr = "%s is not a parsable int (base10) for %s";
+        String longParseErr = "%s is not a parsable long (base10) for %s";
+        String floatParseErr = "%s is not a parsable float for %s";
         options = new HashMap<>(options);
         String s;
+        long minSSTableSize = -1;
+        long targetSSTableSize = -1;
 
-        s = options.remove(SCALING_PARAMETERS_OPTION);
-        if (s != null)
-            parseScalingParameters(s);
-
-        s = options.remove(BASE_SHARD_COUNT_OPTION);
+        s = options.remove(NUM_SHARDS_OPTION);
         if (s != null)
         {
             try
             {
                 int numShards = Integer.parseInt(s);
-                if (numShards <= 0)
-                    throw new ConfigurationException(String.format("Invalid configuration, %s should be positive: %d",
-                                                                   BASE_SHARD_COUNT_OPTION,
+                if (numShards <= 0 && numShards != -1)
+                    throw new ConfigurationException(String.format("Invalid configuration, %s=%d should be positive, or -1 " +
+                                                                   "to explicitly disable static sharding for this table.",
+                                                                   NUM_SHARDS_OPTION,
                                                                    numShards));
-            }
-            catch (NumberFormatException e)
-            {
-                throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s",
-                                                               s,
-                                                               BASE_SHARD_COUNT_OPTION), e);
-            }
-        }
-
-        // preserve the configuration for later use during min_sstable_size.
-        long targetSSTableSize = DEFAULT_TARGET_SSTABLE_SIZE;
-        s = options.remove(TARGET_SSTABLE_SIZE_OPTION);
-        if (s != null)
-        {
-            try
-            {
-                targetSSTableSize = FBUtilities.parseHumanReadableBytes(s);
-                if (targetSSTableSize < MIN_TARGET_SSTABLE_SIZE)
+                if (numShards != -1)
                 {
-                    throw new ConfigurationException(String.format("%s %s is not acceptable, size must be at least %s",
-                                                                   TARGET_SSTABLE_SIZE_OPTION,
-                                                                   s,
-                                                                   FBUtilities.prettyPrintMemory(MIN_TARGET_SSTABLE_SIZE)));
+                    List<String> incompatibleOptions = List.of(TARGET_SSTABLE_SIZE_OPTION, SSTABLE_GROWTH_OPTION, BASE_SHARD_COUNT_OPTION);
+                    if (incompatibleOptions.stream().anyMatch(options::containsKey))
+                    {
+                        throw new ConfigurationException(String.format("Option %s cannot be used in combination with %s",
+                                                                       NUM_SHARDS_OPTION,
+                                                                       incompatibleOptions.stream().filter(options::containsKey).collect(Collectors.joining(", "))));
+                    }
                 }
             }
             catch (NumberFormatException e)
             {
-                throw new ConfigurationException(String.format("%s %s is not a valid size in bytes: %s",
-                                                               TARGET_SSTABLE_SIZE_OPTION,
-                                                               s,
-                                                               e.getMessage()),
-                                                 e);
+                throw new ConfigurationException(String.format(intParseErr, s, NUM_SHARDS_OPTION), e);
             }
         }
 
-        s = options.remove(FLUSH_SIZE_OVERRIDE_OPTION);
-        if (s != null)
-        {
-            try
-            {
-                long flushSize = FBUtilities.parseHumanReadableBytes(s);
-                if (flushSize < MIN_TARGET_SSTABLE_SIZE)
-                    throw new ConfigurationException(String.format("%s %s is not acceptable, size must be at least %s",
-                                                                   FLUSH_SIZE_OVERRIDE_OPTION,
-                                                                   s,
-                                                                   FBUtilities.prettyPrintMemory(MIN_TARGET_SSTABLE_SIZE)));
-            }
-            catch (NumberFormatException e)
-            {
-                throw new ConfigurationException(String.format("%s %s is not a valid size in bytes: %s",
-                                                               FLUSH_SIZE_OVERRIDE_OPTION,
-                                                               s,
-                                                               e.getMessage()),
-                                                 e);
-            }
-        }
+        boolean adaptive = validateBoolean(options, ADAPTIVE_OPTION, DEFAULT_ADAPTIVE);
+        validateBoolean(options, IS_REPLICA_AWARE_OPTION, DEFAULT_IS_REPLICA_AWARE);
+        validateBoolean(options, PARALLELIZE_OUTPUT_SHARDS_OPTION, DEFAULT_PARALLELIZE_OUTPUT_SHARDS);
+
+        validateSizeWithAlt(options, FLUSH_SIZE_OVERRIDE_OPTION, FLUSH_SIZE_OVERRIDE_OPTION_MB, 20);
+        validateSizeWithAlt(options, DATASET_SIZE_OPTION, DATASET_SIZE_OPTION_GB, 30);
 
         s = options.remove(MAX_SSTABLES_TO_COMPACT_OPTION);
         if (s != null)
@@ -545,7 +1144,7 @@ public static Map<String, String> validateOptions(Map<String, String> options) t
              }
              catch (NumberFormatException e)
              {
-                 throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s",
+                 throw new ConfigurationException(String.format(intParseErr,
                                                                 s,
                                                                 MAX_SSTABLES_TO_COMPACT_OPTION),
                                                   e);
@@ -558,90 +1157,244 @@ public static Map<String, String> validateOptions(Map<String, String> options) t
             {
                 long expiredSSTableCheckFrequency = Long.parseLong(s);
                 if (expiredSSTableCheckFrequency <= 0)
-                    throw new ConfigurationException(String.format("Invalid configuration, %s should be positive: %d",
+                    throw new ConfigurationException(String.format(nonPositiveErr,
                                                                    EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION,
                                                                    expiredSSTableCheckFrequency));
             }
             catch (NumberFormatException e)
             {
-                throw new ConfigurationException(String.format("%s is not a parsable long (base10) for %s",
+                throw new ConfigurationException(String.format(longParseErr,
                                                                s,
                                                                EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION),
                                                  e);
             }
         }
 
-        s = options.remove(ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION);
-        if (s != null && !s.equalsIgnoreCase("true") && !s.equalsIgnoreCase("false"))
+        s = options.remove(MAX_SPACE_OVERHEAD_OPTION);
+        if (s != null)
         {
-            throw new ConfigurationException(String.format("%s should either be 'true' or 'false', not %s",
-                                                           ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION, s));
+            try
+            {
+                double maxSpaceOverhead = FBUtilities.parsePercent(s);
+                if (maxSpaceOverhead < MAX_SPACE_OVERHEAD_LOWER_BOUND || maxSpaceOverhead > MAX_SPACE_OVERHEAD_UPPER_BOUND)
+                    throw new ConfigurationException(String.format("Invalid configuration, %s must be between %f and %f: %s",
+                                                                   MAX_SPACE_OVERHEAD_OPTION,
+                                                                   MAX_SPACE_OVERHEAD_LOWER_BOUND,
+                                                                   MAX_SPACE_OVERHEAD_UPPER_BOUND,
+                                                                   s));
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format(floatParseErr,
+                                                               s,
+                                                               MAX_SPACE_OVERHEAD_OPTION),
+                                                 e);
+            }
         }
 
-        s = options.remove(OVERLAP_INCLUSION_METHOD_OPTION);
+        validateBoolean(options, ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION, false);
+
+        s = options.remove(SSTABLE_GROWTH_OPTION);
         if (s != null)
         {
             try
             {
-                Overlaps.InclusionMethod.valueOf(s.toUpperCase());
+                double ssTableGrowthModifier = FBUtilities.parsePercent(s);
+                if (ssTableGrowthModifier < 0 || ssTableGrowthModifier > 1)
+                    throw new ConfigurationException(String.format("%s %s must be between 0 and 1",
+                                                                   SSTABLE_GROWTH_OPTION,
+                                                                   s));
             }
-            catch (IllegalArgumentException e)
+            catch (NumberFormatException e)
             {
-                throw new ConfigurationException(String.format("Invalid overlap inclusion method %s. The valid options are %s.",
-                                                               s,
-                                                               Arrays.toString(Overlaps.InclusionMethod.values())));
+                throw new ConfigurationException(String.format("%s is not a valid number between 0 and 1: %s",
+                                                               SSTABLE_GROWTH_OPTION,
+                                                               e.getMessage()),
+                                                 e);
             }
         }
 
-        s = options.remove(MIN_SSTABLE_SIZE_OPTION);
+        s = options.remove(BASE_SHARD_COUNT_OPTION);
         if (s != null)
         {
             try
             {
-                long sizeInBytes = FBUtilities.parseHumanReadableBytes(s);
-                // zero is a valid option to disable feature
-                if (sizeInBytes < 0)
-                    throw new ConfigurationException(String.format("Invalid configuration, %s should be greater than or equal to 0 (zero)",
-                                                                   MIN_SSTABLE_SIZE_OPTION));
-                int limit = (int) Math.ceil(targetSSTableSize * INVERSE_SQRT_2);
-                if (sizeInBytes >= limit)
-                    throw new ConfigurationException(String.format("Invalid configuration, %s (%s) should be less than the target size minimum: %s",
-                                                                   MIN_SSTABLE_SIZE_OPTION,
-                                                                   FBUtilities.prettyPrintMemory(sizeInBytes),
-                                                                   FBUtilities.prettyPrintMemory(limit)));
+                int baseShardCount = Integer.parseInt(s);
+                if (baseShardCount <= 0)
+                    throw new ConfigurationException(String.format(nonPositiveErr,
+                                                                   BASE_SHARD_COUNT_OPTION,
+                                                                   baseShardCount));
             }
             catch (NumberFormatException e)
             {
-                throw new ConfigurationException(String.format("%s is not a valid size in bytes for %s",
-                                                               s,
-                                                               MIN_SSTABLE_SIZE_OPTION),
+                throw new ConfigurationException(String.format(intParseErr, s, BASE_SHARD_COUNT_OPTION), e);
+            }
+        }
+
+        s = options.remove(TARGET_SSTABLE_SIZE_OPTION);
+        if (s != null)
+        {
+            try
+            {
+                targetSSTableSize = FBUtilities.parseHumanReadableBytes(s);
+                if (targetSSTableSize < MIN_TARGET_SSTABLE_SIZE)
+                    throw new ConfigurationException(String.format("%s %s is not acceptable, size must be at least %s",
+                                                                   TARGET_SSTABLE_SIZE_OPTION,
+                                                                   s,
+                                                                   FBUtilities.prettyPrintMemory(MIN_TARGET_SSTABLE_SIZE)));
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s is not a valid size in bytes: %s",
+                                                               TARGET_SSTABLE_SIZE_OPTION,
+                                                               e.getMessage()),
                                                  e);
             }
         }
 
-        s = options.remove(SSTABLE_GROWTH_OPTION);
+        minSSTableSize = validateSizeWithAlt(options, MIN_SSTABLE_SIZE_OPTION, MIN_SSTABLE_SIZE_OPTION_MB, 20, MIN_SSTABLE_SIZE_OPTION_AUTO, -1, -1);
+        // If both target and min sstable size are defined, check that they are compatible.
+        if (minSSTableSize > 0 && targetSSTableSize > 0 && minSSTableSize > targetSSTableSize * INVERSE_SQRT_2)
+            throw new ConfigurationException(String.format("The minimum sstable size %s cannot be larger than the target size's lower bound %s.",
+                                                           FBUtilities.prettyPrintMemory(minSSTableSize),
+                                                           FBUtilities.prettyPrintMemory((long) (targetSSTableSize * INVERSE_SQRT_2))));
+
+        s = options.remove(RESERVED_THREADS_OPTION);
         if (s != null)
         {
             try
             {
-                double targetSSTableGrowth  = FBUtilities.parsePercent(s);
-                if (targetSSTableGrowth < 0 || targetSSTableGrowth > 1)
-                {
-                    throw new ConfigurationException(String.format("%s %s must be between 0 and 1",
-                                                                   SSTABLE_GROWTH_OPTION,
+                int reservedThreads = FBUtilities.parseIntAllowingMax(s);
+                if (reservedThreads < 0)
+                    throw new ConfigurationException(String.format("%s %s must be an integer >= 0 or \"max\"",
+                                                                   RESERVED_THREADS_OPTION,
                                                                    s));
-                }
             }
             catch (NumberFormatException e)
             {
-                throw new ConfigurationException(String.format("%s is not a valid number between 0 and 1: %s",
-                                                               SSTABLE_GROWTH_OPTION,
+                throw new ConfigurationException(String.format("%s is not a valid integer >= 0 or \"max\": %s",
+                                                               RESERVED_THREADS_OPTION,
                                                                e.getMessage()),
                                                  e);
             }
         }
 
-        return options;
+        s = options.remove(RESERVATIONS_TYPE_OPTION);
+        if (s != null)
+        {
+            try
+            {
+                Reservations.Type.valueOf(s.toUpperCase());
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new ConfigurationException(String.format("Invalid reserved threads type %s. The valid options are %s.",
+                                                               s,
+                                                               Arrays.toString(Reservations.Type.values())));
+            }
+        }
+
+        s = options.remove(OVERLAP_INCLUSION_METHOD_OPTION);
+        if (s != null)
+        {
+            try
+            {
+                Overlaps.InclusionMethod.valueOf(s.toUpperCase());
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new ConfigurationException(String.format("Invalid overlap inclusion method %s. The valid options are %s.",
+                                                               s,
+                                                               Arrays.toString(Overlaps.InclusionMethod.values())));
+            }
+        }
+
+        return adaptive ? AdaptiveController.validateOptions(options) : StaticController.validateOptions(options);
+    }
+
+    private static long getSizeWithAlt(Map<String, String> options, String optionHumanReadable, String optionAlt, int altShift, long defaultValue)
+    {
+        if (options.containsKey(optionHumanReadable))
+            return FBUtilities.parseHumanReadableBytes(options.get(optionHumanReadable));
+        else if (options.containsKey(optionAlt))
+            return Long.parseLong(options.get(optionAlt)) << altShift;
+        else
+            return defaultValue;
+    }
+
+    private static boolean validateBoolean(Map<String, String> options, String option, boolean defaultValue) throws ConfigurationException
+    {
+        var s = options.remove(option);
+        if (s != null)
+        {
+            if (!s.equalsIgnoreCase("true") && !s.equalsIgnoreCase("false"))
+                throw new ConfigurationException(String.format("%s should either be 'true' or 'false', not %s", option, s));
+            return Boolean.parseBoolean(s);
+        }
+        return defaultValue;
+    }
+
+    private static void validateSizeWithAlt(Map<String, String> options, String optionHumanReadable, String optionAlt, int altShift)
+    {
+        validateSizeWithAlt(options, optionHumanReadable, optionAlt, altShift, null, 0, 0);
+    }
+
+    private static long validateSizeWithAlt(Map<String, String> options, String optionHumanReadable, String optionAlt, int altShift, String specialText, long specialValue, long defaultValue)
+    {
+        validateOneOf(options, optionHumanReadable, optionAlt);
+        long sizeInBytes;
+        String s = null;
+        String opt = optionHumanReadable;
+        try
+        {
+            s = options.remove(opt);
+            if (s != null)
+            {
+                if (s.equalsIgnoreCase(specialText))
+                    return specialValue; // all good
+                sizeInBytes = FBUtilities.parseHumanReadableBytes(s);
+            }
+            else
+            {
+                opt = optionAlt;
+                s = options.remove(opt);
+                if (s != null)
+                    sizeInBytes = Long.parseLong(s) << altShift;
+                else
+                    return defaultValue;
+            }
+
+        }
+        catch (NumberFormatException e)
+        {
+            if (specialText != null)
+                throw new ConfigurationException(String.format("%s must be a valid size in bytes or %s for %s",
+                                                               s,
+                                                               specialText,
+                                                               opt),
+                                                 e);
+            else
+                throw new ConfigurationException(String.format("%s is not a valid size in bytes for %s",
+                                                               s,
+                                                               opt),
+                                                 e);
+        }
+
+        if (sizeInBytes < 0)
+            throw new ConfigurationException(String.format("Invalid configuration, %s should be positive: %s",
+                                                           opt,
+                                                           s));
+        return sizeInBytes;
+    }
+
+    private static void validateOneOf(Map<String, String> options, String option1, String option2)
+    {
+        if (options.containsKey(option1) && options.containsKey(option2))
+        {
+            throw new ConfigurationException(String.format("Cannot specify both %s and %s",
+                                                           option1,
+                                                           option2));
+        }
     }
 
     // The methods below are implemented here (rather than directly in UCS) to aid testability.
@@ -666,15 +1419,25 @@ public double getMaxLevelDensity(int index, double minSize)
 
     public double maxThroughput()
     {
-        double compactionThroughputMbPerSec = DatabaseDescriptor.getCompactionThroughputMebibytesPerSec();
-        if (compactionThroughputMbPerSec <= 0)
-            return Double.MAX_VALUE;
-        return Math.scalb(compactionThroughputMbPerSec, 20);
+        return env.maxThroughput();
+    }
+
+    public long getOverheadSizeInBytes(Iterable<? extends CompactionSSTable> sstables, long totalDataSize)
+    {
+        return env.getOverheadSizeInBytes(sstables, totalDataSize);
     }
 
     public int maxConcurrentCompactions()
     {
-        return DatabaseDescriptor.getConcurrentCompactors();
+        return env.maxConcurrentCompactions();
+    }
+
+    public long maxCompactionSpaceBytes()
+    {
+        // Note: Compaction will not proceed with operations larger than this size (i.e. it will compact on the lower
+        // levels but will accumulate sstables on the top until the space on the drive fills up). This sounds risky but
+        // is less of a problem than running out of space during compaction.
+        return (long) (getDataSetSizeBytes() * getMaxSpaceOverhead());
     }
 
     public int maxSSTablesToCompact()
@@ -737,4 +1500,100 @@ public static String printScalingParameters(int[] parameters)
         builder.append(UnifiedCompactionStrategy.printScalingParameter(parameters[i]));
         return builder.toString();
     }
+
+    /**
+     * Prioritize the given aggregates. Because overlap is the primary measure we aim to control, reducing the max
+     * overlap of the aggregates is the primary goal. We do this by sorting the aggregates by max overlap, so that
+     * the ones with the highest overlap are chosen first.
+     * Among choices with matching overlap, we order randomly to give each level and bucket a good chance to run.
+     */
+    public List<CompactionAggregate.UnifiedAggregate> prioritize(List<CompactionAggregate.UnifiedAggregate> aggregates)
+    {
+        // Randomize the list.
+        Collections.shuffle(aggregates, random());
+        // Sort the array so that aggregates with the highest overlap come first. On ties, prefer lower levels.
+        // Because this is a stable sort, entries with the same overlap and level will remain randomly ordered.
+        aggregates.sort((a1, a2) -> {
+            int cmp = Long.compare(a2.maxOverlap(), a1.maxOverlap());
+            if (cmp != 0)
+                return cmp;
+            else
+                return Integer.compare(a1.bucketIndex(), a2.bucketIndex());
+        });
+        return aggregates;
+    }
+
+    static final class Metrics
+    {
+        private final MetricNameFactory factory;
+        private final AtomicReference<Controller> controllerRef;
+        private final Gauge<Double> totWAGauge;
+        private final Gauge<Double> readIOCostGauge;
+        private final Gauge<Double> writeIOCostGauge;
+        private final Gauge<Double> totIOCostGauge;
+
+        Metrics(TableMetadata metadata)
+        {
+            this.factory = new DefaultNameFactory("CompactionCosts",
+                                                  String.format("%s.%s", metadata.keyspace, metadata.name));
+            this.controllerRef = new AtomicReference<>();
+            this.totWAGauge = org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics.register(factory.createMetricName("WA"), this::getMeasuredWA);
+            this.readIOCostGauge = org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics.register(factory.createMetricName("ReadIOCost"), this::getReadIOCost);
+            this.writeIOCostGauge = org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics.register(factory.createMetricName("WriteIOCost"), this::getWriteIOCost);
+            this.totIOCostGauge = org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics.register(factory.createMetricName("TotIOCost"), this::getTotalIOCost);
+        }
+
+        void setController(Controller controller)
+        {
+            this.controllerRef.set(controller);
+        }
+
+        void removeController()
+        {
+           this.controllerRef.set(null);
+        }
+
+        void release()
+        {
+            org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics.remove(factory.createMetricName("WA"));
+            org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics.remove(factory.createMetricName("ReadIOCost"));
+            org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics.remove(factory.createMetricName("WriteIOCost"));
+            org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics.remove(factory.createMetricName("TotIOCost"));
+        }
+
+        double getMeasuredWA()
+        {
+            double ret = 0;
+            Controller controller = controllerRef.get();
+            if (controller != null)
+                ret = controller.env.WA();
+
+            return ret;
+        }
+
+        double getReadIOCost()
+        {
+            double ret = 0;
+            Controller controller = controllerRef.get();
+            if (controller != null)
+                ret = controller.getReadIOCost();
+
+            return ret;
+        }
+
+        double getWriteIOCost()
+        {
+            double ret = 0;
+            Controller controller = controllerRef.get();
+            if (controller != null)
+                ret = controller.getWriteIOCost();
+
+            return ret;
+        }
+
+        double getTotalIOCost()
+        {
+            return getReadIOCost() + getWriteIOCost();
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/CostsCalculator.java b/src/java/org/apache/cassandra/db/compaction/unified/CostsCalculator.java
new file mode 100644
index 000000000000..f21b11c25058
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/unified/CostsCalculator.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.metrics.CompactionMetrics;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.MovingAverage;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_ADAPTIVE_SAMPLE_TIME_MS;
+
+/**
+ * This class periodically retrieves delta values from the environment and stores them into exponentially weighted averages.
+ * It then uses these values to calculate IO costs that are exported to {@link CompactionMetrics} and used by {@link AdaptiveController}
+ * to choose the optimal configuration for compaction.
+ */
+public class CostsCalculator
+{
+    private final static Logger logger = LoggerFactory.getLogger(CostsCalculator.class);
+
+    /** How often values are sampled. Sampling for periods that are too short (<= 1 second) may not give good results since
+     * we many not collect sufficient data. */
+    final static int samplingPeriodMs = UCS_ADAPTIVE_SAMPLE_TIME_MS.getIntWithLegacyFalback();
+
+    private final Environment env;
+    private final MovingAverageOfDelta partitionsReadPerPeriod;
+    private final MovingAverageOfDelta bytesInsertedPerPeriod;
+    private final MovingAverage numSSTables;
+    private final MovingAverage spaceUsed;
+    private final UnifiedCompactionStrategy strategy;
+
+    private final ReentrantReadWriteLock lock;
+    private final ReentrantReadWriteLock.ReadLock readLock;
+    private final ReentrantReadWriteLock.WriteLock writeLock;
+    private final ScheduledFuture<?> future;
+
+    CostsCalculator(Environment env,
+                    UnifiedCompactionStrategy strategy,
+                    ScheduledExecutorService executorService)
+    {
+        this.env = env;
+        this.partitionsReadPerPeriod = new MovingAverageOfDelta(env.makeExpMovAverage());
+        this.bytesInsertedPerPeriod = new MovingAverageOfDelta(env.makeExpMovAverage());
+        this.numSSTables = env.makeExpMovAverage();
+        this.spaceUsed = env.makeExpMovAverage();
+        this.strategy = strategy;
+        this.lock = new ReentrantReadWriteLock();
+        this.readLock = lock.readLock();
+        this.writeLock = lock.writeLock();
+        this.future = executorService.scheduleAtFixedRate(this::sampleValues, samplingPeriodMs, samplingPeriodMs, TimeUnit.MILLISECONDS);
+    }
+
+    public void close()
+    {
+        writeLock.lock();
+
+        try
+        {
+            logger.debug("Stopping cost calculations for {}", strategy.getMetadata());
+            future.cancel(false);
+            logger.debug("Stopped cost calculations for {}", strategy.getMetadata());
+        }
+        finally
+        {
+            writeLock.unlock();
+        }
+    }
+
+    @VisibleForTesting
+    void sampleValues()
+    {
+        writeLock.lock();
+
+        try
+        {
+            partitionsReadPerPeriod.update(env.partitionsRead());
+            bytesInsertedPerPeriod.update(env.bytesInserted());
+
+            numSSTables.update(strategy.getSSTables().size());
+            spaceUsed.update(strategy.getSSTables().stream().map(CompactionSSTable::onDiskLength).reduce(0L, Long::sum));
+        }
+        catch (Throwable err)
+        {
+            JVMStabilityInspector.inspectThrowable(err);
+            logger.error("Failed to update values: {}/{}", err.getClass().getName(), err.getMessage(), err);
+        }
+        finally
+        {
+            writeLock.unlock();
+        }
+    }
+
+    /**
+     * @return the estimated read cost for the given number of partitions, in milliseconds
+     */
+    private double getReadCost(double partitionsRead)
+    {
+        return (env.sstablePartitionReadLatencyNanos() * partitionsRead) / TimeUnit.MILLISECONDS.toNanos(1);
+    }
+
+    /**
+     * Calculate the projected read cost for user queries.
+     *
+     * The projected read cost is given by the number of partitions read, times the mean partition latency and is calculated
+     * by {@link this#getReadCost(double)}. This value is then multiplied by the number of sstables we're likely to hit
+     * per partition read and the read multiplier.
+     * <p/>
+     * The number of sstables is calculated as Math.min(1 + env.bloomFilterFpRatio() * RA / survivalFactor, RA). Here we
+     * assume there is going to be at least one sstable accessed, possibly more in case of :
+     *
+     * - bloom filter's false positives;
+     * - partitions not surviving a compaction (1/survivalFactor is the limit of the sum of (1-survivalFactor)^n), that
+     *   is partitions that would not exist if compaction was done; Note that the survival factor is currently fixed to 1.
+     *
+     * The RA is then a cap since we cannot read more than RA sstables, which are the sstables that exist because
+     * compactions allows them to exist.
+     * </p>
+     * The read multiplier is a factor that operators can use to tweak the algorithm.
+     * </p>
+     * @param RA the expected read amplification due to the current choice of compaction strategy
+     *
+     * @return the projected read cost for user queries
+     */
+    public double getReadCostForQueries(int RA)
+    {
+        readLock.lock();
+
+        try
+        {
+            return getReadCost(partitionsReadPerPeriod.avg.get()) * RA * strategy.getOptions().getReadMultiplier();
+        }
+        finally
+        {
+            readLock.unlock();
+        }
+    }
+
+    private double getFlushCost(double bytesWritten)
+    {
+        return ((bytesWritten / (1 << 10)) * env.flushTimePerKbInNanos()) / (double) TimeUnit.MILLISECONDS.toNanos(1);
+    }
+
+    private double getCompactionCost(double bytesWritten)
+    {
+        // So, the compaction latency will depend on the size of the sstables, so in the correct solution each level
+        // should pass its output size and we should measure latency in MB or something like that
+        return ((bytesWritten / (1 << 10)) * env.compactionTimePerKbInNanos()) / (double)  TimeUnit.MILLISECONDS.toNanos(1);
+    }
+
+    /**
+     * Calculate the projected write cost for user insertions.
+     *
+     * The projected write cost is given by the number of bytes that were inserted times the flush cost
+     * plus the same number of bytes times the compaction cost and the compaction WA. We also multiply by
+     * a write multiplier to let users change the weights if needed.
+     *
+     * @param WA the expected write amplification due to compaction
+     *
+     * @return the projected flush and write cost.
+     */
+    public double getWriteCostForQueries(int WA)
+    {
+        readLock.lock();
+
+        try
+        {
+            double bytesInserted = this.bytesInsertedPerPeriod.avg.get();
+            // using bytesInserted for the compaction cost doesn't take into account overwrites but for now it's good enough
+            return (getFlushCost(bytesInserted) + getCompactionCost(bytesInserted) * WA) * strategy.getOptions().getWriteMultiplier();
+        }
+        finally
+        {
+            readLock.unlock();
+        }
+    }
+
+    public double partitionsRead()
+    {
+        return partitionsReadPerPeriod.avg.get();
+    }
+
+    public double numSSTables()
+    {
+        return numSSTables.get();
+    }
+
+    public double spaceUsed()
+    {
+        return spaceUsed.get();
+    }
+
+    public Environment getEnv()
+    {
+        return env;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("num partitions read %s, bytes inserted: %s, num sstables %s; Environment: %s",
+                             partitionsReadPerPeriod, bytesInsertedPerPeriod, numSSTables, env);
+    }
+
+    @NotThreadSafe
+    private static final class MovingAverageOfDelta
+    {
+        private final MovingAverage avg;
+        private volatile double prev;
+
+        MovingAverageOfDelta(MovingAverage avg)
+        {
+            this.avg = avg;
+            this.prev = Double.MIN_VALUE;
+        }
+
+        void update(double val)
+        {
+            if (prev != Double.MIN_VALUE)
+                avg.update(val - prev);
+
+            prev = val;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s/%d sec", FBUtilities.prettyPrintMemory((long) (avg != null ? avg.get() : 0)), TimeUnit.MILLISECONDS.toSeconds(samplingPeriodMs));
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/Environment.java b/src/java/org/apache/cassandra/db/compaction/unified/Environment.java
new file mode 100644
index 000000000000..8306ce5c4549
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/unified/Environment.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.utils.MovingAverage;
+
+/**
+ * This class supplies to the cost calculator the required parameters for the calculations.
+ * There are two implementations, one used in real life and one for the simulation.
+ */
+public interface Environment
+{
+    /**
+     * @return an exponential moving average. New values have greater representation in the average, and older samples'
+     * effect exponentially decays with new data.
+     */
+    MovingAverage makeExpMovAverage();
+
+    /**
+     * @return the cache miss ratio in the last 5 minutes
+     */
+    double cacheMissRatio();
+
+    /**
+     * @return the bloom filter false positive ratio for all sstables
+     */
+    double bloomFilterFpRatio();
+
+    /**
+     * @return the size of the chunk that read from disk.
+     */
+    int chunkSize();
+
+    /**
+     * @return the total bytes inserted into the memtables so far
+     */
+    long bytesInserted();
+
+    /**
+     * @return the total number of partitions read so far
+     */
+    long partitionsRead();
+
+    /**
+     * @return the mean read latency in nano seconds to read a partition from an sstable
+     */
+    double sstablePartitionReadLatencyNanos();
+
+    /**
+     * @return the mean compaction time per 1 Kb of input, in nano seconds
+     */
+    double compactionTimePerKbInNanos();
+
+    /**
+     * @return the mean flush latency per 1 Kb of input, in nano seconds
+     */
+    double flushTimePerKbInNanos();
+
+    /**
+     * @return the write amplification (bytes flushed + bytes compacted / bytes flushed).
+     */
+    double WA();
+
+    /**
+     * @return the average size of sstables when they are flushed, averaged over the last 5 minutes.
+     */
+    double flushSize();
+
+    /**
+     * @return the maximum number of concurrent compactions that can be running at any one time
+     */
+    int maxConcurrentCompactions();
+
+    /**
+     * @return the maximum compaction throughput
+     */
+    double maxThroughput();
+
+    /**
+     * This method returns the expected temporary space overhead of performing
+     * a compaction. This overhead is due to the fact that whilst compactions
+     * are in progress, both input and output sstables need to be present, since
+     * the input sstables can only be deleted after compaction has completed.
+     * <p/>
+     * The default implementation looks at the size of the input data files of the
+     * compaction, assuming that the output compaction will be just as large.
+     * This does not take into account indexes, and thus may underestimate the
+     * total required space. This method is used to evaluate the actual space
+     * that may be required.
+     *
+     * @param sstables set of sstables to be compacted
+     * @param totalDataSize precalculated data size, to use when total space
+     *                      adjustment is not required
+     * @return the expecte overhead size in bytes for compacting the given sstables
+     */
+    default long getOverheadSizeInBytes(Iterable<? extends CompactionSSTable> sstables, long totalDataSize)
+    {
+        return totalDataSize;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/RealEnvironment.java b/src/java/org/apache/cassandra/db/compaction/unified/RealEnvironment.java
new file mode 100644
index 000000000000..614786bdaea1
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/unified/RealEnvironment.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.ExpMovingAverage;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MovingAverage;
+
+/**
+ * An implementation of {@link Environment} that returns
+ * real values.
+ */
+public class RealEnvironment implements Environment
+{
+    private final CompactionRealm realm;
+
+    public RealEnvironment(CompactionRealm realm)
+    {
+        assert realm != null;
+        this.realm = realm;
+    }
+
+    private TableMetrics metrics()
+    {
+        return realm.metrics();
+    }
+
+    @Override
+    public MovingAverage makeExpMovAverage()
+    {
+        return ExpMovingAverage.decayBy100();
+    }
+
+    @Override
+    public double cacheMissRatio()
+    {
+        double hitRate = ChunkCache.instance != null ? ChunkCache.instance.metrics.hitRate() : Double.NaN;
+        if (Double.isNaN(hitRate))
+            return 1; // if the cache is not yet initialized then assume all requests are a cache miss
+
+        return 1 - Math.min(1, hitRate); // hit rate should never be > 1 but just in case put a check
+    }
+
+    @Override
+    public double bloomFilterFpRatio()
+    {
+        return metrics() == null ? 0.0 : metrics().bloomFilterFalseRatio.getValue();
+    }
+
+    @Override
+    public int chunkSize()
+    {
+        CompressionParams compressionParams = realm.metadata().params.compression;
+        if (compressionParams.isEnabled())
+            return compressionParams.chunkLength();
+
+        return PageAware.PAGE_SIZE;
+    }
+
+    @Override
+    public long partitionsRead()
+    {
+        return metrics() == null ? 0 : metrics().readRequests.getCount();
+    }
+
+    @Override
+    public double sstablePartitionReadLatencyNanos()
+    {
+        return metrics() == null ? 0.0 : metrics().sstablePartitionReadLatency.get();
+    }
+
+    @Override
+    public double compactionTimePerKbInNanos()
+    {
+        return metrics() == null ? 0.0 : metrics().compactionTimePerKb.get();
+    }
+
+    @Override
+    public double flushTimePerKbInNanos()
+    {
+        return metrics() == null ? 0.0 : metrics().flushTimePerKb.get();
+    }
+
+    @Override
+    public long bytesInserted()
+    {
+        return metrics() == null ? 0 : metrics().bytesInserted.getCount();
+    }
+
+    @Override
+    public double WA()
+    {
+        return realm.getWA();
+    }
+
+    @Override
+    public double flushSize()
+    {
+        return metrics() == null ? 0.0 : metrics().flushSizeOnDisk().get();
+    }
+
+    @Override
+    public int maxConcurrentCompactions()
+    {
+        return CompactionManager.instance.getMaximumCompactorThreads();
+    }
+
+    @Override
+    public double maxThroughput()
+    {
+        final int compactionThroughputMbPerSec = DatabaseDescriptor.getCompactionThroughputMebibytesPerSecAsInt();
+        if (compactionThroughputMbPerSec <= 0)
+            return Double.MAX_VALUE;
+        return compactionThroughputMbPerSec * 1024.0 * 1024.0;
+    }
+
+    /**
+     * @return the compaction overhead size in bytes of the given sstables, i.e. the value used to determine how many
+     * compactions we can run without exceeding the available space.
+     * This is configurable via {@link CassandraRelevantProperties#UCS_COMPACTION_INCLUDE_NON_DATA_FILES_SIZE} to
+     * either report only the data file size, or the total size of all sstable components on disk.
+     */
+    public static long getCompactionOverheadSizeInBytes(Iterable<? extends CompactionSSTable> sstables)
+    {
+        if (CassandraRelevantProperties.UCS_COMPACTION_INCLUDE_NON_DATA_FILES_SIZE.getBoolean())
+            return CompactionSSTable.getTotalOnDiskComponentsBytes(sstables);
+        else
+            return CompactionSSTable.getTotalDataBytes(sstables); // only includes data file size
+    }
+
+    /**
+     * @return the compaction overhead size in bytes of the given sstables, i.e. the value used to determine how many
+     * compactions we can run without exceeding the available space.
+     * This is configurable via {@link CassandraRelevantProperties#UCS_COMPACTION_INCLUDE_NON_DATA_FILES_SIZE} to
+     * either report only the data file size, or the total size of all sstable components on disk.
+     * This variation of the method uses a pre-calculated total data size.
+     */
+    public static long getCompactionOverheadSizeInBytes(Iterable<? extends CompactionSSTable> sstables, long totalDataSize)
+    {
+        if (CassandraRelevantProperties.UCS_COMPACTION_INCLUDE_NON_DATA_FILES_SIZE.getBoolean())
+            return CompactionSSTable.getTotalOnDiskComponentsBytes(sstables);
+        else
+            return totalDataSize; // only includes data file size
+    }
+
+    @Override
+    public long getOverheadSizeInBytes(Iterable<? extends CompactionSSTable> sstables, long totalDataSize)
+    {
+        return getCompactionOverheadSizeInBytes(sstables, totalDataSize);
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Default Environment for %s - Read latency: %d us / partition, flush latency: %d us / KiB, " +
+                             "compaction latency: %d us / KiB, bfpr: %f, measured WA: %.2f, flush size %s",
+                             realm.metadata(),
+                             TimeUnit.NANOSECONDS.toMicros((long) sstablePartitionReadLatencyNanos()),
+                             TimeUnit.NANOSECONDS.toMicros((long) flushTimePerKbInNanos()),
+                             TimeUnit.NANOSECONDS.toMicros((long) compactionTimePerKbInNanos()),
+                             bloomFilterFpRatio(),
+                             WA(),
+                             FBUtilities.prettyPrintMemory((long)flushSize()));
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/Reservations.java b/src/java/org/apache/cassandra/db/compaction/unified/Reservations.java
new file mode 100644
index 000000000000..6354f7d15dd5
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/unified/Reservations.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/// Reservations management for compaction. Defines the two types of reservations, and implements the code for accepting
+/// or rejecting compactions to satisfy the reservation requirements.
+public abstract class Reservations
+{
+    public enum Type
+    {
+        /// The given number of reservations can be used only for the level.
+        PER_LEVEL,
+        /// The reservations can be used for the level, or any one below it.
+        LEVEL_OR_BELOW
+    }
+
+    private static final Logger logger = LoggerFactory.getLogger(Reservations.class);
+
+    /// Number of compactions to reserve for each level.
+    final int perLevelCount;
+    /// Remainder of compactions to be distributed among the levels.
+    final int remainder;
+    /// Whether only one compaction over the reservation count is allowed per level.
+    final boolean oneRemainderPerLevel;
+    /// Number of compactions already running or selected in each level.
+    final int[] perLevel;
+
+    private Reservations(int totalCount, int[] perLevel, int reservedThreadsTarget)
+    {
+        this.perLevel = perLevel;
+
+        int levelCount = perLevel.length;
+        // Each level has this number of tasks reserved for it.
+        perLevelCount = Math.min(totalCount / levelCount, reservedThreadsTarget);
+        // The remainder is distributed according to the prioritization.
+        remainder = totalCount - perLevelCount * levelCount;
+        // If the user requested more than we can give, do not allow more than one extra per level.
+        oneRemainderPerLevel = perLevelCount < reservedThreadsTarget;
+    }
+
+    /// Accept a compaction in the given level if possible.
+    /// @param parallelismRequested The number of threads requested for the compaction.
+    /// @returns The number of threads given to the compaction, or 0 if the compaction cannot be accepted.
+    public abstract int accept(int inLevel, int parallelismRequested);
+
+    public abstract boolean hasRoom(int inLevel);
+
+    public abstract void debugOutput(int selectedCount, int proposedCount, int remaining);
+
+    public static Reservations create(int totalCount, int[] perLevel, int reservedThreadsTarget, Type reservationsType)
+    {
+        if (reservedThreadsTarget == 0)
+            return new Trivial(totalCount, perLevel);
+        return reservationsType == Type.PER_LEVEL
+               ? new PerLevel(totalCount, perLevel, reservedThreadsTarget)
+               : new LevelOrBelow(totalCount, perLevel, reservedThreadsTarget);
+    }
+
+    /// Trivial tracker used when there are no reservations. All compactions are accepted.
+    private static class Trivial extends Reservations
+    {
+        private Trivial(int totalCount, int[] perLevel)
+        {
+            super(totalCount, perLevel, 0);
+        }
+
+        @Override
+        public int accept(int inLevel, int requestedParallelism)
+        {
+            perLevel[inLevel] += requestedParallelism;
+            return requestedParallelism;
+        }
+
+        @Override
+        public boolean hasRoom(int inLevel)
+        {
+            return true;
+        }
+
+        @Override
+        public void debugOutput(int selectedCount, int proposedCount, int remaining)
+        {
+            if (proposedCount > 0)
+                logger.debug("Selected {} compactions (out of {} pending). Compactions per level {} (no reservations) remaining {}.",
+                             selectedCount, proposedCount, perLevel, remaining);
+            else
+                logger.trace("Selected {} compactions (out of {} pending). Compactions per level {} (no reservations) remaining {}.",
+                             selectedCount, proposedCount, perLevel, remaining);
+        }
+    }
+
+    /// Per-level tracker.
+    ///
+    /// Reservations are applied by tracking how much of the remainder threads are being used, and only allowing
+    /// compactions in a level if their number is below the per-level count, or if there is a remainder slot to be given.
+    private static class PerLevel extends Reservations
+    {
+        int remainderDistributed;
+
+        PerLevel(int totalCount, int[] perLevel, int reservedThreadsTarget)
+        {
+            super(totalCount, perLevel, reservedThreadsTarget);
+
+            remainderDistributed = 0;
+            for (int countInLevel : perLevel)
+                if (countInLevel > perLevelCount)
+                    remainderDistributed += countInLevel - perLevelCount;
+        }
+
+        @Override
+        public int accept(int inLevel, int requestedParallelism)
+        {
+            int assigned = perLevelCount - perLevel[inLevel];
+            assigned = Math.min(assigned, requestedParallelism);
+            assigned = Math.max(assigned, 0);
+
+            if (assigned < requestedParallelism && remainderDistributed < remainder)
+            {
+                // we have a remainder to distribute
+                if (oneRemainderPerLevel)
+                {
+                    if (perLevel[inLevel] <= perLevelCount) // we can only give one above, and only if that one is not yet used
+                    {
+                        ++assigned;
+                        ++remainderDistributed;
+                    }
+                }
+                else
+                {
+                    int requestedFromRemainder = requestedParallelism - assigned;
+                    int assignedFromRemainder = Math.min(requestedFromRemainder, remainder - remainderDistributed);
+                    assigned += assignedFromRemainder;
+                    remainderDistributed += assignedFromRemainder;
+                }
+            }
+
+            perLevel[inLevel] += assigned;
+            return assigned;
+        }
+
+        @Override
+        public boolean hasRoom(int inLevel)
+        {
+            // If we have room in the level, we can accommodate.
+            return (perLevel[inLevel] < perLevelCount) ||
+                   // Otherwise, we need to have remainder to distribute, and not used the one extra if we are in oneRemainderPerLevel mode.
+                   (remainderDistributed < remainder) && (!oneRemainderPerLevel || perLevel[inLevel] == perLevelCount);
+        }
+
+        @Override
+        public void debugOutput(int selectedCount, int proposedCount, int remaining)
+        {
+            int remainingNonReserved = remainder - remainderDistributed;
+            logger.debug("Selected {} compactions (out of {} pending). Compactions per level {} (reservations {}{}) remaining reserved {} non-reserved {}.",
+                         selectedCount, proposedCount, perLevel, perLevelCount, oneRemainderPerLevel ? "+1" : "", remaining - remainingNonReserved, remainingNonReserved);
+        }
+    }
+
+    /// Tracker for the level or below case.
+    ///
+    /// For any given level, the reservations are satisfied if the total sum of compactions for the level and all levels
+    /// above it is at most the product of the number of levels and the per-level count, plus any remainder (up to the
+    /// number of levels when oneRemainderPerLevel is true).
+    ///
+    /// To permit a compaction, we gather this sum for all levels above, and make sure this property will not be violated
+    /// by adding the new compaction for the current, as well as all levels below it. The latter is necessary because
+    /// a lower level may have already used up all allocations for this one.
+    private static class LevelOrBelow extends Reservations
+    {
+        LevelOrBelow(int totalCount, int[] perLevel, int reservedThreadsTarget)
+        {
+            super(totalCount, perLevel, reservedThreadsTarget);
+        }
+
+        @Override
+        public int accept(int inLevel, int requestedParallelism)
+        {
+            return checkRoom(inLevel, requestedParallelism, true);
+        }
+
+        @Override
+        public boolean hasRoom(int inLevel)
+        {
+            return checkRoom(inLevel, 1, false) > 0;
+        }
+
+        public int checkRoom(int inLevel, int requestedParallelism, boolean markUse)
+        {
+            // Limit the sum of the number of threads of any level and all higher to their number
+            // times perLevelCount, plus any remainder (up to the number when oneRemainderPerLevel is true).
+            int sum = 0;
+            int permittedQuota = 0;
+            int permittedRemainder = oneRemainderPerLevel ? 0 : remainder;
+            int level = perLevel.length - 1;
+            int tentativelyAssigned = requestedParallelism;
+            // For all higher levels, calculate the total number of threads used and permitted.
+            for (; level > inLevel; --level)
+            {
+                sum += perLevel[level];
+                permittedQuota += perLevelCount;
+                if (oneRemainderPerLevel && permittedRemainder < remainder)
+                    ++permittedRemainder;
+            }
+
+            // Also adjust for the limit as it applies for this level and all below.
+            for (; level >= 0; --level)
+            {
+                sum += perLevel[level];
+                permittedQuota += perLevelCount;
+                if (oneRemainderPerLevel && permittedRemainder < remainder)
+                    ++permittedRemainder;
+                if (tentativelyAssigned > permittedQuota + permittedRemainder - sum)
+                {
+                    tentativelyAssigned = permittedQuota + permittedRemainder - sum;
+                    if (tentativelyAssigned <= 0)
+                        return 0; // some lower level used up our share
+                }
+            }
+            if (markUse)
+                perLevel[inLevel] += tentativelyAssigned;
+            return tentativelyAssigned;
+        }
+
+        @Override
+        public void debugOutput(int selectedCount, int proposedCount, int remaining)
+        {
+            if (proposedCount > 0)
+                logger.debug("Selected {} compactions (out of {} pending). Compactions per level {} (reservations level or below {}{}) remaining {}.",
+                             selectedCount, proposedCount, perLevel, perLevelCount, oneRemainderPerLevel ? "+1" : "", remaining);
+            else
+                logger.trace("Selected {} compactions (out of {} pending). Compactions per level {} (reservations level or below {}{}) remaining {}.",
+                             selectedCount, proposedCount, perLevel, perLevelCount, oneRemainderPerLevel ? "+1" : "", remaining);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriter.java
index ca5e99749cca..d9e23bcff3f9 100644
--- a/src/java/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriter.java
@@ -1,13 +1,11 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Copyright DataStax, Inc.
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -23,14 +21,18 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.db.compaction.ShardTracker;
 import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.utils.FBUtilities;
 
 /**
@@ -45,20 +47,20 @@ public class ShardedCompactionWriter extends CompactionAwareWriter
 
     private final ShardTracker boundaries;
 
-    public ShardedCompactionWriter(ColumnFamilyStore cfs,
+    /// @param uniqueKeyRatio the expected ratio between the expected number of unique keys in the output sstable and
+    /// the number of keys in the individual inputs.
+    public ShardedCompactionWriter(CompactionRealm realm,
                                    Directories directories,
-                                   LifecycleTransaction txn,
+                                   ILifecycleTransaction txn,
                                    Set<SSTableReader> nonExpiredSSTables,
+                                   double uniqueKeyRatio,
                                    boolean keepOriginals,
+                                   boolean earlyOpenAllowed,
                                    ShardTracker boundaries)
     {
-        super(cfs, directories, txn, nonExpiredSSTables, keepOriginals);
-
+        super(realm, directories, txn, nonExpiredSSTables, keepOriginals, earlyOpenAllowed);
         this.boundaries = boundaries;
-        long totalKeyCount = nonExpiredSSTables.stream()
-                                               .mapToLong(SSTableReader::estimatedKeys)
-                                               .sum();
-        this.uniqueKeyRatio = 1.0 * SSTableReader.getApproximateKeyCount(nonExpiredSSTables) / totalKeyCount;
+        this.uniqueKeyRatio = uniqueKeyRatio;
     }
 
     @Override
@@ -74,7 +76,7 @@ protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
                          key.getToken(), boundaries.shardStart(),
                          boundaries.shardIndex(),
                          FBUtilities.prettyPrintMemory(uncompressedBytesWritten),
-                         cfs.getKeyspaceName(), cfs.getTableName());
+                         realm.getKeyspaceName(), realm.getTableName());
             return true;
         }
 
@@ -82,16 +84,23 @@ protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
     }
 
     @Override
-    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
+    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, Token nextKey)
     {
         if (nextKey != null)
-            boundaries.advanceTo(nextKey.getToken());
-        return super.sstableWriter(directory, nextKey);
-    }
+            boundaries.advanceTo(nextKey);
 
-    protected long sstableKeyCount()
-    {
-        return shardAdjustedKeyCount(boundaries, nonExpiredSSTables, uniqueKeyRatio);
+        Descriptor descriptor = realm.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
+        return descriptor.getFormat().getWriterFactory().builder(descriptor)
+                         .setKeyCount(shardAdjustedKeyCount(boundaries, nonExpiredSSTables, uniqueKeyRatio))
+                         .setRepairedAt(minRepairedAt)
+                         .setPendingRepair(pendingRepair)
+                         .setTransientSSTable(isTransient)
+                         .setTableMetadataRef(realm.metadataRef())
+                         .setMetadataCollector(new MetadataCollector(txn.originals(), realm.metadata().comparator))
+                         .setSerializationHeader(SerializationHeader.make(realm.metadata(), nonExpiredSSTables))
+                         .addDefaultComponents(realm.getIndexManager().listIndexGroups())
+                         .setSecondaryIndexGroups(realm.getIndexManager().listIndexGroups())
+                         .build(txn, realm);
     }
 
     private static long shardAdjustedKeyCount(ShardTracker boundaries,
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java b/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java
index a5b5df9e4967..10a04bbc7820 100644
--- a/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java
@@ -1,13 +1,11 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Copyright DataStax, Inc.
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -25,11 +23,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.db.compaction.ShardTracker;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -45,17 +43,19 @@
 
 /**
  * A {@link SSTableMultiWriter} that splits the output sstable at the partition boundaries of the compaction
- * shards used by {@link org.apache.cassandra.db.compaction.UnifiedCompactionStrategy}.
+ * shards used by {@link org.apache.cassandra.db.compaction.UnifiedCompactionStrategy} as long as the size of
+ * the sstable so far is sufficiently large.
  * <p/>
- * This is class is similar to {@link ShardedCompactionWriter} but for flushing. Unfortunately
+ * This is class is similar to {@link ShardedMultiWriter} but for flushing. Unfortunately
  * we currently have 2 separate writers hierarchy that are not compatible and so we must
- * duplicate the functionality.
+ * duplicate the functionality of splitting sstables over compaction shards if they have
+ * reached a minimum size.
  */
 public class ShardedMultiWriter implements SSTableMultiWriter
 {
     protected final static Logger logger = LoggerFactory.getLogger(ShardedMultiWriter.class);
 
-    private final ColumnFamilyStore cfs;
+    private final CompactionRealm realm;
     private final Descriptor descriptor;
     private final long keyCount;
     private final long repairedAt;
@@ -69,19 +69,19 @@ public class ShardedMultiWriter implements SSTableMultiWriter
     private final SSTableWriter[] writers;
     private int currentWriter;
 
-    public ShardedMultiWriter(ColumnFamilyStore cfs,
-                              Descriptor descriptor,
-                              long keyCount,
-                              long repairedAt,
-                              TimeUUID pendingRepair,
-                              boolean isTransient,
-                              IntervalSet<CommitLogPosition> commitLogPositions,
-                              SerializationHeader header,
-                              Collection<Index.Group> indexGroups,
-                              LifecycleNewTracker lifecycleNewTracker,
-                              ShardTracker boundaries)
-    {
-        this.cfs = cfs;
+    public ShardedMultiWriter(CompactionRealm realm,
+                                Descriptor descriptor,
+                                long keyCount,
+                                long repairedAt,
+                                TimeUUID pendingRepair,
+                                boolean isTransient,
+                                IntervalSet<CommitLogPosition> commitLogPositions,
+                                SerializationHeader header,
+                                Collection<Index.Group> indexGroups,
+                                LifecycleNewTracker lifecycleNewTracker,
+                                ShardTracker boundaries)
+    {
+        this.realm = realm;
         this.descriptor = descriptor;
         this.keyCount = keyCount;
         this.repairedAt = repairedAt;
@@ -100,25 +100,24 @@ public ShardedMultiWriter(ColumnFamilyStore cfs,
 
     private SSTableWriter createWriter()
     {
-        Descriptor newDesc = cfs.newSSTableDescriptor(descriptor.directory);
+        Descriptor newDesc = realm.newSSTableDescriptor(descriptor.directory);
         return createWriter(newDesc);
     }
 
-    private SSTableWriter createWriter(Descriptor descriptor)
+    private SSTableWriter createWriter(Descriptor desc)
     {
-        MetadataCollector metadataCollector = new MetadataCollector(cfs.metadata().comparator)
-                                              .commitLogIntervals(commitLogPositions != null ? commitLogPositions : IntervalSet.empty());
-        return descriptor.getFormat().getWriterFactory().builder(descriptor)
-                         .setKeyCount(forSplittingKeysBy(boundaries.count()))
-                         .setRepairedAt(repairedAt)
-                         .setPendingRepair(pendingRepair)
-                         .setTransientSSTable(isTransient)
-                         .setTableMetadataRef(cfs.metadata)
-                         .setMetadataCollector(metadataCollector)
-                         .setSerializationHeader(header)
-                         .addDefaultComponents(indexGroups)
-                         .setSecondaryIndexGroups(indexGroups)
-                         .build(lifecycleNewTracker, cfs);
+        SSTableWriter.Builder<?, ?> builder = desc.getFormat().getWriterFactory().builder(desc);
+        return builder
+               .setKeyCount(forSplittingKeysBy(boundaries.count()))
+               .setRepairedAt(repairedAt)
+               .setPendingRepair(pendingRepair)
+               .setTransientSSTable(isTransient)
+               .setTableMetadataRef(realm.metadataRef())
+               .setMetadataCollector(new MetadataCollector(realm.metadata().comparator).commitLogIntervals(commitLogPositions))
+               .setSerializationHeader(header)
+               .addDefaultComponents(indexGroups)
+               .setSecondaryIndexGroups(indexGroups)
+               .build(lifecycleNewTracker, realm);
     }
 
     private long forSplittingKeysBy(long splits) {
@@ -137,7 +136,7 @@ public void append(UnfilteredRowIterator partition)
             logger.debug("Switching writer at boundary {}/{} index {}, with uncompressed size {} for {}.{}",
                          key.getToken(), boundaries.shardStart(), currentWriter,
                          FBUtilities.prettyPrintMemory(currentUncompressedSize),
-                         cfs.getKeyspaceName(), cfs.getTableName());
+                         realm.getKeyspaceName(), realm.getTableName());
 
             writers[++currentWriter] = createWriter();
         }
@@ -169,12 +168,11 @@ public Collection<SSTableReader> finished()
     }
 
     @Override
-    public SSTableMultiWriter setOpenResult(boolean openResult)
+    public void openResult()
     {
         for (SSTableWriter writer : writers)
             if (writer != null)
-                writer.setOpenResult(openResult);
-        return this;
+                writer.openResult();
     }
 
     @Override
@@ -190,8 +188,11 @@ public String getFilename()
     public long getBytesWritten()
     {
         long bytesWritten = 0;
-        for (int i = 0; i <= currentWriter; ++i)
-            bytesWritten += writers[i].getFilePointer();
+        for (int i = 0; i <= currentWriter; ++i) 
+        {
+            if (writers[i] != null)
+                bytesWritten += writers[i].getFilePointer();
+        }
         return bytesWritten;
     }
 
@@ -200,14 +201,22 @@ public long getOnDiskBytesWritten()
     {
         long bytesWritten = 0;
         for (int i = 0; i <= currentWriter; ++i)
-            bytesWritten += writers[i].getEstimatedOnDiskBytesWritten();
+        {
+            if (writers[i] != null)
+                bytesWritten += writers[i].getEstimatedOnDiskBytesWritten();
+        }
         return bytesWritten;
     }
 
+    public int getSegmentCount()
+    {
+        return currentWriter + 1;
+    }
+
     @Override
     public TableId getTableId()
     {
-        return cfs.metadata().id;
+        return realm.metadata().id;
     }
 
     @Override
@@ -241,7 +250,7 @@ public void prepareToCommit()
             {
                 boundaries.applyTokenSpaceCoverage(writer);
                 writer.prepareToCommit();
-    }
+            }
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/StaticController.java b/src/java/org/apache/cassandra/db/compaction/unified/StaticController.java
new file mode 100644
index 000000000000..66586db835e5
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/unified/StaticController.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.compaction.CompactionPick;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileReader;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.Overlaps;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_STATIC_SCALING_PARAMETERS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_VECTOR_SCALING_PARAMETERS;
+
+/**
+ * The static compaction controller periodically checks the IO costs
+ * that result from the current configuration of the {@link UnifiedCompactionStrategy}.
+ */
+public class StaticController extends Controller
+{
+    /**
+     * The scaling parameters W, one per bucket index and separated by a comma.
+     * Higher indexes will use the value of the last index with a W specified.
+     */
+    static final String STATIC_SCALING_FACTORS_OPTION = "static_scaling_factors";
+    private final static String DEFAULT_STATIC_SCALING_PARAMETERS = UCS_STATIC_SCALING_PARAMETERS.getStringWithLegacyFallback();
+    final static String DEFAULT_VECTOR_STATIC_SCALING_PARAMETERS = UCS_VECTOR_SCALING_PARAMETERS.getStringWithLegacyFallback();
+
+    private final int[] scalingParameters;
+
+    @VisibleForTesting // comp. simulation
+    public StaticController(Environment env,
+                            int[] scalingParameters,
+                            double[] survivalFactors,
+                            long dataSetSize,
+                            long minSSTableSize,
+                            long flushSizeOverride,
+                            long currentFlushSize,
+                            double maxSpaceOverhead,
+                            int maxSSTablesToCompact,
+                            long expiredSSTableCheckFrequency,
+                            boolean ignoreOverlapsInExpirationCheck,
+                            int baseShardCount,
+                            boolean isReplicaAware,
+                            long targetSStableSize,
+                            double sstableGrowthModifier,
+                            int reservedThreadsPerLevel,
+                            Reservations.Type reservationsType,
+                            Overlaps.InclusionMethod overlapInclusionMethod,
+                            boolean parallelizeOutputShards,
+                            boolean hasVectorType,
+                            String keyspaceName,
+                            String tableName)
+    {
+        super(MonotonicClock.Global.preciseTime,
+              env,
+              survivalFactors,
+              dataSetSize,
+              minSSTableSize,
+              flushSizeOverride,
+              currentFlushSize,
+              maxSpaceOverhead,
+              maxSSTablesToCompact,
+              expiredSSTableCheckFrequency,
+              ignoreOverlapsInExpirationCheck,
+              baseShardCount,
+              isReplicaAware,
+              targetSStableSize,
+              sstableGrowthModifier,
+              reservedThreadsPerLevel,
+              reservationsType,
+              overlapInclusionMethod,
+              parallelizeOutputShards,
+              hasVectorType);
+        this.scalingParameters = scalingParameters;
+        this.keyspaceName = keyspaceName;
+        this.tableName = tableName;
+    }
+
+    static Controller fromOptions(Environment env,
+                                  double[] survivalFactors,
+                                  long dataSetSize,
+                                  long minSSTableSize,
+                                  long flushSizeOverride,
+                                  double maxSpaceOverhead,
+                                  int maxSSTablesToCompact,
+                                  long expiredSSTableCheckFrequency,
+                                  boolean ignoreOverlapsInExpirationCheck,
+                                  int baseShardCount,
+                                  boolean isReplicaAware,
+                                  long targetSStableSize,
+                                  double sstableGrowthModifier,
+                                  int reservedThreadsPerLevel,
+                                  Reservations.Type reservationsType,
+                                  Overlaps.InclusionMethod overlapInclusionMethod,
+                                  boolean parallelizeOutputShards,
+                                  boolean hasVectorType,
+                                  String keyspaceName,
+                                  String tableName,
+                                  Map<String, String> options,
+                                  boolean useVectorOptions)
+    {
+        int[] scalingParameters;
+        if (options.containsKey(STATIC_SCALING_FACTORS_OPTION))
+            scalingParameters = parseScalingParameters(options.get(STATIC_SCALING_FACTORS_OPTION));
+        else
+            scalingParameters = parseScalingParameters(options.getOrDefault(SCALING_PARAMETERS_OPTION,
+                                                                            useVectorOptions ? DEFAULT_VECTOR_STATIC_SCALING_PARAMETERS
+                                                                                             : DEFAULT_STATIC_SCALING_PARAMETERS));
+
+        long currentFlushSize = flushSizeOverride;
+
+        File f = getControllerConfigPath(keyspaceName, tableName);
+        try
+        {
+            JSONParser jsonParser = new JSONParser();
+            JSONObject jsonObject = (JSONObject) jsonParser.parse(new FileReader(f));
+            if (jsonObject.get("current_flush_size") != null && flushSizeOverride == 0)
+            {
+                currentFlushSize = (long) jsonObject.get("current_flush_size");
+                logger.debug("Successfully read stored current_flush_size from disk");
+            }
+        }
+        catch (IOException e)
+        {
+            logger.debug("No controller config file found. Using starting value instead.");
+        }
+        catch (ParseException e)
+        {
+            logger.warn("Unable to parse saved flush size. Using starting value instead:", e);
+        }
+        catch (FSError e)
+        {
+            logger.warn("Unable to read controller config file. Using starting value instead:", e);
+        }
+        catch (Throwable e)
+        {
+            logger.warn("Unable to read controller config file. Using starting value instead:", e);
+            JVMStabilityInspector.inspectThrowable(e);
+        }
+        return new StaticController(env,
+                                    scalingParameters,
+                                    survivalFactors,
+                                    dataSetSize,
+                                    minSSTableSize,
+                                    flushSizeOverride,
+                                    currentFlushSize,
+                                    maxSpaceOverhead,
+                                    maxSSTablesToCompact,
+                                    expiredSSTableCheckFrequency,
+                                    ignoreOverlapsInExpirationCheck,
+                                    baseShardCount,
+                                    isReplicaAware,
+                                    targetSStableSize,
+                                    sstableGrowthModifier,
+                                    reservedThreadsPerLevel,
+                                    reservationsType,
+                                    overlapInclusionMethod,
+                                    parallelizeOutputShards,
+                                    hasVectorType,
+                                    keyspaceName,
+                                    tableName);
+    }
+
+    public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
+    {
+        String parameters = options.remove(SCALING_PARAMETERS_OPTION);
+        if (parameters != null)
+            parseScalingParameters(parameters);
+        String factors = options.remove(STATIC_SCALING_FACTORS_OPTION);
+        if (factors != null)
+            parseScalingParameters(factors);
+        if (parameters != null && factors != null)
+            throw new ConfigurationException(String.format("Either '%s' or '%s' should be used, not both", SCALING_PARAMETERS_OPTION, STATIC_SCALING_FACTORS_OPTION));
+        return options;
+    }
+
+    @Override
+    public int getScalingParameter(int index)
+    {
+        if (index < 0)
+            throw new IllegalArgumentException("Index should be >= 0: " + index);
+
+        return index < scalingParameters.length ? scalingParameters[index] : scalingParameters[scalingParameters.length - 1];
+    }
+
+    @Override
+    public int getPreviousScalingParameter(int index)
+    {
+        //scalingParameters is not updated in StaticController so previous scalingParameters = scalingParameters
+        return getScalingParameter(index);
+    }
+
+    @Override
+    public boolean isRecentAdaptive(CompactionPick pick)
+    {
+        return false;
+    }
+
+    @Override
+    public int getMaxRecentAdaptiveCompactions()
+    {
+        return Integer.MAX_VALUE;
+    }
+
+    @Override
+    public void storeControllerConfig()
+    {
+        storeOptions(keyspaceName, tableName, scalingParameters, getFlushSizeBytes());
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Static controller, m: %d, o: %s, scalingParameters: %s, cost: %s", minSSTableSize,
+                             Arrays.toString(survivalFactors),
+                             printScalingParameters(scalingParameters),
+                             calculator);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/UnifiedCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/unified/UnifiedCompactionTask.java
index 5a729f643299..430cb39e2b9f 100644
--- a/src/java/org/apache/cassandra/db/compaction/unified/UnifiedCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/unified/UnifiedCompactionTask.java
@@ -1,13 +1,11 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Copyright DataStax, Inc.
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,44 +16,159 @@
 
 package org.apache.cassandra.db.compaction.unified;
 
+import java.util.Collection;
 import java.util.Set;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.compaction.AbstractCompactionTask;
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.db.compaction.CompactionTask;
 import org.apache.cassandra.db.compaction.ShardManager;
+import org.apache.cassandra.db.compaction.SharedCompactionObserver;
+import org.apache.cassandra.db.compaction.SharedCompactionProgress;
+import org.apache.cassandra.db.compaction.SharedTableOperation;
+import org.apache.cassandra.db.compaction.TableOperationObserver;
 import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
 import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 
-/**
- * The sole purpose of this class is to currently create a {@link ShardedCompactionWriter}.
- */
 public class UnifiedCompactionTask extends CompactionTask
 {
     private final ShardManager shardManager;
-    private final Controller controller;
+    private final Range<Token> operationRange;
+    private final Set<SSTableReader> actuallyCompact;
+    private final SharedCompactionProgress sharedProgress;
+    private final SharedTableOperation sharedOperation;
+    private final UnifiedCompactionStrategy.ShardingStats shardingStats;
+
+    public UnifiedCompactionTask(CompactionRealm cfs,
+                                 UnifiedCompactionStrategy strategy,
+                                 ILifecycleTransaction txn,
+                                 long gcBefore,
+                                 ShardManager shardManager,
+                                 UnifiedCompactionStrategy.ShardingStats shardingStats)
+    {
+        this(cfs, strategy, txn, gcBefore, false, shardManager, shardingStats, null, null, null, null, null);
+    }
+
 
-    public UnifiedCompactionTask(ColumnFamilyStore cfs,
+    public UnifiedCompactionTask(CompactionRealm cfs,
                                  UnifiedCompactionStrategy strategy,
-                                 LifecycleTransaction txn,
+                                 ILifecycleTransaction txn,
                                  long gcBefore,
-                                 ShardManager shardManager)
+                                 boolean keepOriginals,
+                                 ShardManager shardManager,
+                                 UnifiedCompactionStrategy.ShardingStats shardingStats,
+                                 Range<Token> operationRange,
+                                 Collection<SSTableReader> actuallyCompact,
+                                 SharedCompactionProgress sharedProgress,
+                                 SharedCompactionObserver sharedObserver,
+                                 SharedTableOperation sharedOperation)
     {
-        super(cfs, txn, gcBefore);
-        this.controller = strategy.getController();
+        super(cfs,
+              txn,
+              // Set the total operation sizes early to use in shared progress tracking. This assumes that:
+              // - there are no expired sstables in the compaction (UCS processes them separately)
+              // - sstable exclusion for lack of space does not apply (shared progress is only use when an operation
+              //   range applies, which disables this)
+              sharedProgress != null ? getOperationTotals(actuallyCompact, operationRange) : null,
+              gcBefore,
+              keepOriginals,
+              strategy,
+              sharedObserver != null ? sharedObserver : strategy);
         this.shardManager = shardManager;
+        this.shardingStats = shardingStats;
+
+        if (operationRange != null)
+            assert actuallyCompact != null : "Ranged tasks should use a set of sstables to compact";
+
+        this.operationRange = operationRange;
+        this.sharedProgress = sharedProgress;
+        this.sharedOperation = sharedOperation;
+        if (sharedProgress != null)
+            sharedProgress.registerExpectedSubtask(totals.inputUncompressedSize, totals.inputDiskSize, totals.inputUncompressedSize);
+        if (sharedObserver != null)
+            sharedObserver.registerExpectedSubtask();
+        if (sharedOperation != null)
+            sharedOperation.registerExpectedSubtask();
+        // To make sure actuallyCompact tracks any removals from txn.originals(), we intersect the given set with it.
+        // This should not be entirely necessary (as shouldReduceScopeForSpace() is false for ranged tasks), but it
+        // is cleaner to enforce inputSSTables()'s requirements.
+        this.actuallyCompact = actuallyCompact != null ? Sets.intersection(ImmutableSet.copyOf(actuallyCompact),
+                                                                           txn.originals())
+                                                       : txn.originals();
     }
 
     @Override
-    public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs,
+    public CompactionAwareWriter getCompactionAwareWriter(CompactionRealm realm,
                                                           Directories directories,
-                                                          LifecycleTransaction txn,
                                                           Set<SSTableReader> nonExpiredSSTables)
     {
-        double density = shardManager.calculateCombinedDensity(nonExpiredSSTables);
-        int numShards = controller.getNumShards(density * shardManager.shardSetCoverage());
-        return new ShardedCompactionWriter(cfs, directories, txn, nonExpiredSSTables, keepOriginals, shardManager.boundaries(numShards));
+        // In multi-task operations we need to expire many ranges in a source sstable for early open. Not doable yet.
+        final boolean earlyOpenAllowed = operationRange == null;
+        return new ShardedCompactionWriter(realm,
+                                           directories,
+                                           transaction,
+                                           nonExpiredSSTables,
+                                           shardingStats.uniqueKeyRatio,
+                                           keepOriginals,
+                                           earlyOpenAllowed,
+                                           shardManager.boundaries(shardingStats.shardCountForDensity));
+    }
+
+    @Override
+    protected Range<Token> tokenRange()
+    {
+        return operationRange;
+    }
+
+    @Override
+    protected SharedCompactionProgress sharedProgress()
+    {
+        return sharedProgress;
+    }
+
+    @Override
+    protected boolean shouldReduceScopeForSpace()
+    {
+        // Because parallelized tasks share input sstables, we can't reduce the scope of individual tasks
+        // (as doing that will leave some part of an sstable out of the compaction but still drop the whole sstable
+        // when the task set completes).
+        return tokenRange() == null;
+    }
+
+    @Override
+    public Set<SSTableReader> inputSSTables()
+    {
+        return actuallyCompact;
+    }
+
+    @Override
+    public AbstractCompactionTask setOpObserver(TableOperationObserver opObserver)
+    {
+        if (sharedOperation != null)
+            opObserver = sharedOperation.wrapObserver(opObserver);
+        return super.setOpObserver(opObserver);
+    }
+
+    @Override
+    public long getSpaceOverhead()
+    {
+        if (operationRange != null)
+        {
+            // totals must be precalculated for ranged tasks
+            return (long) (totals.inputDiskSize * shardingStats.overheadToDataRatio);
+        }
+        else
+        {
+            // if we don't have a range, the sharding stats have precise total disk space
+            return (long) (shardingStats.totalOnDiskSize * shardingStats.overheadToDataRatio);
+        }
     }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0.svg b/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0.svg
index e3d36653492f..afb50514676e 100644
--- a/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0.svg
+++ b/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0.svg
@@ -1,20 +1,4 @@
 <?xml version="1.0" encoding="utf-8" standalone="no"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
   "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 <svg xmlns:xlink="http://www.w3.org/1999/xlink" width="460.8pt" height="345.6pt" viewBox="0 0 460.8 345.6" xmlns="http://www.w3.org/2000/svg" version="1.1">
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0_33.svg b/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0_33.svg
index 25b101e6abb0..43fd38677453 100644
--- a/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0_33.svg
+++ b/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0_33.svg
@@ -1,20 +1,4 @@
 <?xml version="1.0" encoding="utf-8" standalone="no"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
   "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 <svg xmlns:xlink="http://www.w3.org/1999/xlink" width="460.8pt" height="345.6pt" viewBox="0 0 460.8 345.6" xmlns="http://www.w3.org/2000/svg" version="1.1">
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0_5.svg b/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0_5.svg
index f0e583f0ad2c..55ea68ac46b3 100644
--- a/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0_5.svg
+++ b/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_0_5.svg
@@ -1,20 +1,4 @@
 <?xml version="1.0" encoding="utf-8" standalone="no"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
   "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 <svg xmlns:xlink="http://www.w3.org/1999/xlink" width="460.8pt" height="345.6pt" viewBox="0 0 460.8 345.6" xmlns="http://www.w3.org/2000/svg" version="1.1">
diff --git a/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_1.svg b/src/java/org/apache/cassandra/db/compaction/unified/shards_graph_lambda_1.svg
old mode 100755
new mode 100644
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
index 145163a39cf6..b35a039dab02 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
@@ -26,15 +26,18 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.DiskBoundaries;
-import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.db.compaction.CompactionTask;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableRewriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -45,16 +48,15 @@
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Transactional;
 
-
 /**
  * Class that abstracts away the actual writing of files to make it possible to use CompactionTask for more
  * use cases.
  */
-public abstract class CompactionAwareWriter extends Transactional.AbstractTransactional implements Transactional
+public abstract class CompactionAwareWriter extends Transactional.AbstractTransactional implements Transactional, SSTableDataSink
 {
     protected static final Logger logger = LoggerFactory.getLogger(CompactionAwareWriter.class);
 
-    protected final ColumnFamilyStore cfs;
+    protected final CompactionRealm realm;
     protected final Directories directories;
     protected final Set<SSTableReader> nonExpiredSSTables;
     protected final long estimatedTotalKeys;
@@ -64,31 +66,41 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
     protected final boolean isTransient;
 
     protected final SSTableRewriter sstableWriter;
-    protected final LifecycleTransaction txn;
+    protected final ILifecycleTransaction txn;
     private final List<Directories.DataDirectory> locations;
-    private final List<PartitionPosition> diskBoundaries;
+    private final List<Token> diskBoundaries;
     private int locationIndex;
     protected Directories.DataDirectory currentDirectory;
 
-    public CompactionAwareWriter(ColumnFamilyStore cfs,
-                                 Directories directories,
-                                 LifecycleTransaction txn,
-                                 Set<SSTableReader> nonExpiredSSTables,
-                                 boolean keepOriginals)
+    protected CompactionAwareWriter(CompactionRealm realm,
+                                    Directories directories,
+                                    ILifecycleTransaction txn,
+                                    Set<SSTableReader> nonExpiredSSTables,
+                                    boolean keepOriginals)
+    {
+        this(realm, directories, txn, nonExpiredSSTables, keepOriginals, true);
+    }
+
+    protected CompactionAwareWriter(CompactionRealm realm,
+                                    Directories directories,
+                                    ILifecycleTransaction txn,
+                                    Set<SSTableReader> nonExpiredSSTables,
+                                    boolean keepOriginals,
+                                    boolean earlyOpenAllowed)
     {
-        this.cfs = cfs;
+        this.realm = realm;
         this.directories = directories;
         this.nonExpiredSSTables = nonExpiredSSTables;
         this.txn = txn;
 
         estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
         maxAge = CompactionTask.getMaxDataAge(nonExpiredSSTables);
-        sstableWriter = SSTableRewriter.construct(cfs, txn, keepOriginals, maxAge);
+        sstableWriter = SSTableRewriter.construct(realm, txn, keepOriginals, maxAge, earlyOpenAllowed);
         minRepairedAt = CompactionTask.getMinRepairedAt(nonExpiredSSTables);
         pendingRepair = CompactionTask.getPendingRepair(nonExpiredSSTables);
         isTransient = CompactionTask.getIsTransient(nonExpiredSSTables);
-        DiskBoundaries db = cfs.getDiskBoundaries();
-        diskBoundaries = db.positions;
+        DiskBoundaries db = realm.getDiskBoundaries();
+        diskBoundaries = db.getPositions();
         locations = db.directories;
         locationIndex = -1;
     }
@@ -132,13 +144,42 @@ public long estimatedKeys()
 
     /**
      * Writes a partition in an implementation specific way
+     *
      * @param partition the partition to append
      * @return true if the partition was written, false otherwise
      */
-    public final boolean append(UnfilteredRowIterator partition)
+    public AbstractRowIndexEntry append(UnfilteredRowIterator partition)
     {
         maybeSwitchWriter(partition.partitionKey());
-        return realAppend(partition);
+        return appendWithoutSwitchingWriters(partition);
+    }
+
+    @Override
+    public boolean startPartition(DecoratedKey partitionKey, DeletionTime deletionTime) throws IOException
+    {
+        maybeSwitchWriter(partitionKey);
+        return sstableWriter.startPartition(partitionKey, deletionTime);
+    }
+
+    @Override
+    public AbstractRowIndexEntry endPartition() throws IOException
+    {
+        return sstableWriter.endPartition();
+    }
+
+    @Override
+    public void addUnfiltered(Unfiltered unfiltered) throws IOException
+    {
+        sstableWriter.addUnfiltered(unfiltered);
+    }
+
+    /**
+     * Write a partition without considering location change.
+     * Exposed for TieredCompactionStrategy which needs to control the location itself.
+     */
+    AbstractRowIndexEntry appendWithoutSwitchingWriters(UnfilteredRowIterator partition)
+    {
+        return sstableWriter.append(partition);
     }
 
     public final File getSStableDirectory() throws IOException
@@ -153,11 +194,6 @@ protected Throwable doPostCleanup(Throwable accumulate)
         return super.doPostCleanup(accumulate);
     }
 
-    protected boolean realAppend(UnfilteredRowIterator partition)
-    {
-        return sstableWriter.append(partition) != null;
-    }
-
     /**
      * Switches the writer if necessary, i.e. if the new key should be placed in a different data directory, or if the
      * specific strategy has decided a new sstable is needed.
@@ -178,7 +214,7 @@ protected void maybeSwitchWriter(DecoratedKey key)
      */
     protected boolean maybeSwitchLocation(DecoratedKey key)
     {
-        if (diskBoundaries == null)
+        if (key == null || diskBoundaries == null)
         {
             if (locationIndex < 0)
             {
@@ -190,11 +226,11 @@ protected boolean maybeSwitchLocation(DecoratedKey key)
             return false;
         }
 
-        if (locationIndex > -1 && key.compareTo(diskBoundaries.get(locationIndex)) < 0)
+        if (locationIndex > -1 && key.getToken().compareTo(diskBoundaries.get(locationIndex)) < 0)
             return false;
 
         int prevIdx = locationIndex;
-        while (locationIndex == -1 || key.compareTo(diskBoundaries.get(locationIndex)) > 0)
+        while (locationIndex == -1 || key.getToken().compareTo(diskBoundaries.get(locationIndex)) > 0)
             locationIndex++;
         Directories.DataDirectory newLocation = locations.get(locationIndex);
         if (prevIdx >= 0)
@@ -220,35 +256,25 @@ protected boolean maybeSwitchLocation(DecoratedKey key)
     protected void switchCompactionWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
     {
         currentDirectory = directory;
-        sstableWriter.switchWriter(sstableWriter(directory, nextKey));
+        sstableWriter.switchWriter(sstableWriter(directory, nextKey != null ? nextKey.getToken() : null));
     }
 
-    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
+    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, Token diskBoundary)
     {
-        Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
-        MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator)
-                                      .sstableLevel(sstableLevel());
-        SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
-
-        return newWriterBuilder(descriptor).setMetadataCollector(collector)
-                                           .setSerializationHeader(header)
-                                           .setKeyCount(sstableKeyCount())
-                                           .build(txn, cfs);
-    }
-
-    /**
-     * Returns the level that should be used when creating sstables.
-     */
-    protected int sstableLevel()
-    {
-        return 0;
+        Descriptor descriptor = realm.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
+        return descriptor.getFormat().getWriterFactory().builder(descriptor)
+                         .setKeyCount(estimatedTotalKeys)
+                         .setRepairedAt(minRepairedAt)
+                         .setPendingRepair(pendingRepair)
+                         .setTransientSSTable(isTransient)
+                         .setTableMetadataRef(realm.metadataRef())
+                         .setMetadataCollector(new MetadataCollector(txn.originals(), realm.metadata().comparator))
+                         .setSerializationHeader(SerializationHeader.make(realm.metadata(), nonExpiredSSTables))
+                         .addDefaultComponents(realm.getIndexManager().listIndexGroups())
+                         .setSecondaryIndexGroups(realm.getIndexManager().listIndexGroups())
+                         .build(txn, realm);
     }
 
-    /**
-     * Returns the key count with which created sstables should be set up.
-     */
-    abstract protected long sstableKeyCount();
-
     /**
      * The directories we can write to
      */
@@ -302,7 +328,7 @@ public CompactionAwareWriter setRepairedAt(long repairedAt)
 
     protected long getExpectedWriteSize()
     {
-        return cfs.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
+        return realm.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
     }
 
     /**
@@ -314,11 +340,21 @@ protected long getExpectedWriteSize()
     protected SSTableWriter.Builder<?, ?> newWriterBuilder(Descriptor descriptor)
     {
         return descriptor.getFormat().getWriterFactory().builder(descriptor)
-                         .setTableMetadataRef(cfs.metadata)
+                         .setTableMetadataRef(realm.metadataRef())
                          .setTransientSSTable(isTransient)
                          .setRepairedAt(minRepairedAt)
                          .setPendingRepair(pendingRepair)
-                         .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups())
-                         .addDefaultComponents(cfs.indexManager.listIndexGroups());
+                         .setSecondaryIndexGroups(realm.getIndexManager().listIndexGroups())
+                         .addDefaultComponents(realm.getIndexManager().listIndexGroups());
+    }
+
+    public long bytesWritten()
+    {
+        return sstableWriter.bytesWritten();
+    }
+
+    public String getCurrentFileName()
+    {
+        return sstableWriter.currentWriter().getFilename();
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
index fbb0e27a99b0..d0722edaffef 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
@@ -23,11 +23,16 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 
 /**
  * The default compaction writer - creates one output file in L0
@@ -37,14 +42,15 @@ public class DefaultCompactionWriter extends CompactionAwareWriter
     protected static final Logger logger = LoggerFactory.getLogger(DefaultCompactionWriter.class);
     private final int sstableLevel;
 
-    public DefaultCompactionWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
+    public DefaultCompactionWriter(CompactionRealm realm, Directories directories, ILifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
     {
-        this(cfs, directories, txn, nonExpiredSSTables, false, 0);
+        this(realm, directories, txn, nonExpiredSSTables, false, 0);
     }
 
-    public DefaultCompactionWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables, boolean keepOriginals, int sstableLevel)
+    @SuppressWarnings("resource")
+    public DefaultCompactionWriter(CompactionRealm realm, Directories directories, ILifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables, boolean keepOriginals, int sstableLevel)
     {
-        super(cfs, directories, txn, nonExpiredSSTables, keepOriginals);
+        super(realm, directories, txn, nonExpiredSSTables, keepOriginals);
         this.sstableLevel = sstableLevel;
     }
 
@@ -54,9 +60,22 @@ protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
         return false;
     }
 
-    protected int sstableLevel()
+    @SuppressWarnings("resource")
+    @Override
+    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, Token diskBoundary)
     {
-        return sstableLevel;
+        Descriptor descriptor = realm.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
+        return descriptor.getFormat().getWriterFactory().builder(descriptor)
+                         .setKeyCount(estimatedTotalKeys)
+                         .setRepairedAt(minRepairedAt)
+                         .setPendingRepair(pendingRepair)
+                         .setTransientSSTable(isTransient)
+                         .setTableMetadataRef(realm.metadataRef())
+                         .setMetadataCollector(new MetadataCollector(txn.originals(), realm.metadata().comparator, sstableLevel))
+                         .setSerializationHeader(SerializationHeader.make(realm.metadata(), nonExpiredSSTables))
+                         .addDefaultComponents(realm.getIndexManager().listIndexGroups())
+                         .setSecondaryIndexGroups(realm.getIndexManager().listIndexGroups())
+                         .build(txn, realm);
     }
 
     protected long sstableKeyCount()
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index 09263df8530b..77952fecf797 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -19,13 +19,20 @@
 
 import java.util.Set;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.compaction.LeveledManifest;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.compaction.LeveledManifest;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 
 public class MajorLeveledCompactionWriter extends CompactionAwareWriter
 {
@@ -38,34 +45,34 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
     private final long keysPerSSTable;
     private final int levelFanoutSize;
 
-    public MajorLeveledCompactionWriter(ColumnFamilyStore cfs,
+    public MajorLeveledCompactionWriter(CompactionRealm realm,
                                         Directories directories,
-                                        LifecycleTransaction txn,
+                                        ILifecycleTransaction txn,
                                         Set<SSTableReader> nonExpiredSSTables,
                                         long maxSSTableSize)
     {
-        this(cfs, directories, txn, nonExpiredSSTables, maxSSTableSize, false);
+        this(realm, directories, txn, nonExpiredSSTables, maxSSTableSize, false);
     }
 
-    public MajorLeveledCompactionWriter(ColumnFamilyStore cfs,
+    public MajorLeveledCompactionWriter(CompactionRealm realm,
                                         Directories directories,
-                                        LifecycleTransaction txn,
+                                        ILifecycleTransaction txn,
                                         Set<SSTableReader> nonExpiredSSTables,
                                         long maxSSTableSize,
                                         boolean keepOriginals)
     {
-        super(cfs, directories, txn, nonExpiredSSTables, keepOriginals);
+        super(realm, directories, txn, nonExpiredSSTables, keepOriginals);
         this.maxSSTableSize = maxSSTableSize;
-        this.levelFanoutSize = cfs.getLevelFanoutSize();
-        long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(nonExpiredSSTables) / maxSSTableSize);
+        this.levelFanoutSize = realm.getLevelFanoutSize();
+        long estimatedSSTables = Math.max(1, CompactionSSTable.getTotalDataBytes(nonExpiredSSTables) / maxSSTableSize);
         keysPerSSTable = estimatedTotalKeys / estimatedSSTables;
     }
 
     @Override
-    public boolean realAppend(UnfilteredRowIterator partition)
+    public AbstractRowIndexEntry append(UnfilteredRowIterator partition)
     {
         partitionsWritten++;
-        return super.realAppend(partition);
+        return super.append(partition);
     }
 
     @Override
@@ -95,14 +102,22 @@ public void switchCompactionWriter(Directories.DataDirectory location, Decorated
         super.switchCompactionWriter(location, nextKey);
     }
 
-    protected int sstableLevel()
-    {
-        return currentLevel;
-    }
-
-    protected long sstableKeyCount()
+    @Override
+    @SuppressWarnings("resource")
+    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, Token diskBoundary)
     {
-        return keysPerSSTable;
+        Descriptor descriptor = realm.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
+        return descriptor.getFormat().getWriterFactory().builder(descriptor)
+                         .setKeyCount(keysPerSSTable)
+                         .setRepairedAt(minRepairedAt)
+                         .setPendingRepair(pendingRepair)
+                         .setTransientSSTable(isTransient)
+                         .setTableMetadataRef(realm.metadataRef())
+                         .setMetadataCollector(new MetadataCollector(txn.originals(), realm.metadata().comparator, currentLevel))
+                         .setSerializationHeader(SerializationHeader.make(realm.metadata(), txn.originals()))
+                         .addDefaultComponents(realm.getIndexManager().listIndexGroups())
+                         .setSecondaryIndexGroups(realm.getIndexManager().listIndexGroups())
+                         .build(txn, realm);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 1ded2128e77d..49a1f06da2a7 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -19,12 +19,17 @@
 
 import java.util.Set;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 
 public class MaxSSTableSizeWriter extends CompactionAwareWriter
 {
@@ -32,36 +37,36 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
     private final int level;
     private final long estimatedSSTables;
 
-    public MaxSSTableSizeWriter(ColumnFamilyStore cfs,
+    public MaxSSTableSizeWriter(CompactionRealm realm,
                                 Directories directories,
-                                LifecycleTransaction txn,
+                                ILifecycleTransaction txn,
                                 Set<SSTableReader> nonExpiredSSTables,
                                 long maxSSTableSize,
                                 int level)
     {
-        this(cfs, directories, txn, nonExpiredSSTables, maxSSTableSize, level, false);
+        this(realm, directories, txn, nonExpiredSSTables, maxSSTableSize, level, false);
     }
 
-    public MaxSSTableSizeWriter(ColumnFamilyStore cfs,
+    public MaxSSTableSizeWriter(CompactionRealm realm,
                                 Directories directories,
-                                LifecycleTransaction txn,
+                                ILifecycleTransaction txn,
                                 Set<SSTableReader> nonExpiredSSTables,
                                 long maxSSTableSize,
                                 int level,
                                 boolean keepOriginals)
     {
-        super(cfs, directories, txn, nonExpiredSSTables, keepOriginals);
+        super(realm, directories, txn, nonExpiredSSTables, keepOriginals);
         this.level = level;
         this.maxSSTableSize = maxSSTableSize;
 
-        long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, txn.opType());
+        long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, realm, txn.opType());
         estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
     }
 
     /**
      * Gets the estimated total amount of data to write during compaction
      */
-    private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+    private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, CompactionRealm realm, OperationType compactionType)
     {
         long estimatedKeysBeforeCompaction = 0;
         for (SSTableReader sstable : nonExpiredSSTables)
@@ -69,7 +74,7 @@ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables
         estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
         double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
 
-        return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
+        return Math.round(estimatedCompactionRatio * realm.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
     }
 
     @Override
@@ -78,14 +83,21 @@ protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
         return sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > maxSSTableSize;
     }
 
-    protected int sstableLevel()
-    {
-        return level;
-    }
-
-    protected long sstableKeyCount()
+    @Override
+    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, Token diskBoundary)
     {
-        return estimatedTotalKeys / estimatedSSTables;
+        Descriptor descriptor = realm.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
+        return descriptor.getFormat().getWriterFactory().builder(descriptor)
+                         .setKeyCount(estimatedTotalKeys / estimatedSSTables)
+                         .setRepairedAt(minRepairedAt)
+                         .setPendingRepair(pendingRepair)
+                         .setTransientSSTable(isTransient)
+                         .setTableMetadataRef(realm.metadataRef())
+                         .setMetadataCollector(new MetadataCollector(txn.originals(), realm.metadata().comparator, level))
+                         .setSerializationHeader(SerializationHeader.make(realm.metadata(), nonExpiredSSTables))
+                         .addDefaultComponents(realm.getIndexManager().listIndexGroups())
+                         .setSecondaryIndexGroups(realm.getIndexManager().listIndexGroups())
+                         .build(txn, realm);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SSTableDataSink.java b/src/java/org/apache/cassandra/db/compaction/writers/SSTableDataSink.java
new file mode 100644
index 000000000000..982bd54e001c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/writers/SSTableDataSink.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.writers;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+
+/**
+ * Abstraction of compaction result writer, implemented by CompactionAwareWriter and tests.
+ */
+public interface SSTableDataSink
+{
+    /**
+     * Append the given partition.
+     * This is equivalent to a sequence of startPartition, addUnfiltered for each item in the partition, and endPartition.
+     */
+    AbstractRowIndexEntry append(UnfilteredRowIterator partition);
+
+    /**
+     * Start a partition with the given key and deletion time.
+     * Returns false if the partition could not be added (e.g. if the key is too long).
+     */
+    boolean startPartition(DecoratedKey partitionKey, DeletionTime deletionTime) throws IOException;
+
+    /**
+     * Complete a partition. Must be called once for every startPartition.
+     *
+     * @return
+     */
+    AbstractRowIndexEntry endPartition() throws IOException;
+
+    /**
+     * Add a new row or marker in the current partition. Must be preceded by startPartition.
+     */
+    void addUnfiltered(Unfiltered unfiltered) throws IOException;
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
index 4cd0858e18a4..ea0ac680d40e 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
@@ -23,11 +23,16 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 
 /**
  * CompactionAwareWriter that splits input in differently sized sstables
@@ -46,16 +51,16 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter
     private long currentBytesToWrite;
     private int currentRatioIndex = 0;
 
-    public SplittingSizeTieredCompactionWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
+    public SplittingSizeTieredCompactionWriter(CompactionRealm realm, Directories directories, ILifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
     {
-        this(cfs, directories, txn, nonExpiredSSTables, DEFAULT_SMALLEST_SSTABLE_BYTES);
+        this(realm, directories, txn, nonExpiredSSTables, DEFAULT_SMALLEST_SSTABLE_BYTES);
     }
 
-    public SplittingSizeTieredCompactionWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables, long smallestSSTable)
+    public SplittingSizeTieredCompactionWriter(CompactionRealm realm, Directories directories, ILifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables, long smallestSSTable)
     {
-        super(cfs, directories, txn, nonExpiredSSTables, false);
+        super(realm, directories, txn, nonExpiredSSTables, false);
         this.allSSTables = txn.originals();
-        totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
+        totalSize = realm.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
         double[] potentialRatios = new double[20];
         double currentRatio = 1;
         for (int i = 0; i < potentialRatios.length; i++)
@@ -91,16 +96,24 @@ protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
         return false;
     }
 
-    protected int sstableLevel()
-    {
-        return 0;
-    }
-
-    protected long sstableKeyCount()
+    @Override
+    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, Token diskBoundary)
     {
         long currentPartitionsToWrite = Math.round(ratios[currentRatioIndex] * estimatedTotalKeys);
         logger.trace("Switching writer, currentPartitionsToWrite = {}", currentPartitionsToWrite);
-        return currentPartitionsToWrite;
+
+        Descriptor descriptor = realm.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
+        return descriptor.getFormat().getWriterFactory().builder(descriptor)
+                         .setKeyCount(currentPartitionsToWrite)
+                         .setRepairedAt(minRepairedAt)
+                         .setPendingRepair(pendingRepair)
+                         .setTransientSSTable(isTransient)
+                         .setTableMetadataRef(realm.metadataRef())
+                         .setMetadataCollector(new MetadataCollector(allSSTables, realm.metadata().comparator))
+                         .setSerializationHeader(SerializationHeader.make(realm.metadata(), nonExpiredSSTables))
+                         .addDefaultComponents(realm.getIndexManager().listIndexGroups())
+                         .setSecondaryIndexGroups(realm.getIndexManager().listIndexGroups())
+                         .build(txn, realm);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/counters/CachedCounterLockManager.java b/src/java/org/apache/cassandra/db/counters/CachedCounterLockManager.java
new file mode 100644
index 000000000000..63fa7adb36ae
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/counters/CachedCounterLockManager.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.counters;
+
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+/**
+ * Implemetation of {@link CounterLockManager} that uses a cache of locks.
+ * Note: this implemetation tries to reduce the chance of having two counters lock each other, but as the counters
+ * are identified by the hash of the primary key of the row, it is still possible to
+ * have some cross-counter contention for counters with different primary keys but the same hash.
+ * <p>
+ * This code is copied from
+ * <a href="https://github.com/diennea/herddb/blob/master/herddb-utils/src/main/java/herddb/utils/LocalLockManager.java">LocalLockManager</a> from the HerdDB project (Apache 2 licensed).
+ */
+public class CachedCounterLockManager implements CounterLockManager
+{
+    private final static int EXPECTED_CONCURRENCY = DatabaseDescriptor.getConcurrentCounterWriters() * 16;
+    /**
+     * The mapping function in {@link #makeLockForKey(Integer)} relies on the ConcurrentHashMap guarantee that the remapping function is run only once per compute, and that it is run atomically
+     */
+    private final ConcurrentHashMap<Integer, RefCountedLock> locks = new ConcurrentHashMap<>(EXPECTED_CONCURRENCY);
+
+    @Override
+    public List<LockHandle> grabLocks(Iterable<Integer> keys)
+    {
+        // we must return the locks in order to avoid deadlocks
+        // please note that the list may contain duplicates
+        return StreamSupport.stream(keys.spliterator(), false)
+                            .sorted()
+                            .map(this::makeLockForKey)
+                            .collect(Collectors.toList());
+    }
+
+    private ReentrantLock makeLock()
+    {
+        return new ReentrantLock();
+    }
+
+    private LockHandleImpl makeLockForKey(Integer key)
+    {
+        RefCountedLock instance = locks.compute(key, (k, existing) -> {
+            if (existing != null)
+            {
+                existing.count++;
+                return existing;
+            }
+            else
+            {
+                return new RefCountedLock(makeLock(), 1);
+            }
+        });
+        return new LockHandleImpl(key, instance);
+    }
+
+    private void releaseLockForKey(RefCountedLock instance, Integer key) throws IllegalStateException
+    {
+        locks.compute(key, (Integer t, RefCountedLock u) -> {
+            if (instance != u)
+            {
+                throw new IllegalStateException("trying to release un-owned lock");
+            }
+            if (--u.count == 0)
+            {
+                return null;
+            }
+            else
+            {
+                return u;
+            }
+        });
+    }
+
+    @Override
+    public boolean hasNumKeys()
+    {
+        return true;
+    }
+
+    @Override
+    public int getNumKeys()
+    {
+        return locks.size();
+    }
+
+    /**
+     * This class is not thread safe, it is expected to be used by a single thread.
+     */
+    private class LockHandleImpl implements LockHandle
+    {
+        private boolean acquired;
+        private final Integer key;
+        private final RefCountedLock handle;
+
+        private LockHandleImpl(Integer key, RefCountedLock handle)
+        {
+            this.key = key;
+            this.handle = handle;
+        }
+
+        @Override
+        public void release()
+        {
+            if (acquired)
+                handle.lock.unlock();
+            releaseLockForKey(handle, key);
+        }
+
+        @Override
+        public boolean tryLock(long timeout, TimeUnit timeUnit) throws InterruptedException
+        {
+            return acquired = handle.lock.tryLock(timeout, timeUnit);
+        }
+
+        @Override
+        public String toString()
+        {
+            return "{key=" + key + '}';
+        }
+    }
+
+    private static class RefCountedLock
+    {
+
+        private final ReentrantLock lock;
+        private int count;
+
+        private RefCountedLock(ReentrantLock lock, int count)
+        {
+            this.lock = lock;
+            this.count = count;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "RefCountedStampedLock{" + "lock=" + lock + ", count=" + count + '}';
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/counters/CounterLockManager.java b/src/java/org/apache/cassandra/db/counters/CounterLockManager.java
new file mode 100644
index 000000000000..f0a9efb21302
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/counters/CounterLockManager.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.counters;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+
+/**
+ * Interface for managing locks for CounterMutation.
+ * CounterMutation needs to ensure that each local counter is accessed by only one thread at a time.
+ * Please note that the id of the counter is an integer hash of the primary key of the row.
+ */
+public interface CounterLockManager
+{
+    boolean USE_STRIPED_COUNTER_LOCK_MANAGER = CassandraRelevantProperties.USE_STRIPED_COUNTER_LOCK_MANAGER.getBoolean();
+    CounterLockManager instance = USE_STRIPED_COUNTER_LOCK_MANAGER ? new StripedCounterLockManager() : new CachedCounterLockManager();
+
+    /**
+     * Handle to a lock for a particular key.
+     * Some expectations:
+     * - instances of this class are not thread-safe
+     * - it is not required that the underlying lock is reentrant
+     */
+    interface LockHandle
+    {
+        /**
+         * Try to get the lock. This method can be called at most once.
+         *
+         * @param timeout  timeout.
+         * @param timeUnit time unit.
+         * @return false in case the lock could not be acquired within the timeout.
+         * @throws InterruptedException in case the thread is interrupted while waiting for the lock.
+         */
+        boolean tryLock(long timeout, TimeUnit timeUnit) throws InterruptedException;
+
+        /**
+         * Unlock the lock if it was acquired and release the handle. This method is to be called even if the acquire method failed or even if tryLock has never been called.
+         * This method is to be called only once.
+         */
+        void release();
+    }
+
+    /**
+     * Grab locks for the given keys. The returned handles must be released by calling {@link LockHandle#release()}.
+     * The returned list is re-ordered in order to prevent deadlocks.
+     * It is expected that the caller will release the locks in the inverse order they were acquired.
+     * The initial set may contain duplicates, it is expected that this method will return a list with the same number of elements.
+     *
+     * @param keys list of keys, the Iterable is scanned only once in order to prevent side effects.
+     * @return a list of lock handles. The List can be iterated multiple times without side effects.
+     */
+    List<LockHandle> grabLocks(Iterable<Integer> keys);
+
+    /**
+     * Check if the implementation can return the number of keys that are handled by the lock manager.
+     * This method is useful only for testing.
+     *
+     * @return true if the implementation can return the number of keys.
+     */
+    boolean hasNumKeys();
+
+    /**
+     * Get the number of keys that are handled by the lock manager.
+     * This method is useful only for testing.
+     *
+     * @return the number of keys.
+     */
+    default int getNumKeys()
+    {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/counters/StripedCounterLockManager.java b/src/java/org/apache/cassandra/db/counters/StripedCounterLockManager.java
new file mode 100644
index 000000000000..88e2a1d97550
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/counters/StripedCounterLockManager.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.counters;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.google.common.base.Supplier;
+import com.google.common.util.concurrent.Striped;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.COUNTER_LOCK_FAIR_LOCK;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COUNTER_LOCK_NUM_STRIPES_PER_THREAD;
+/**
+ * Legacy implementation of {@link CounterLockManager} that uses a fixed set of locks.
+ * On a workload with many different counters it is likely to see two counters sharing the same lock.
+ */
+public class StripedCounterLockManager implements CounterLockManager
+{
+    private final Striped<java.util.concurrent.locks.Lock> locks;
+
+    StripedCounterLockManager()
+    {
+        int numStripes = COUNTER_LOCK_NUM_STRIPES_PER_THREAD.getInt() * DatabaseDescriptor.getConcurrentCounterWriters();
+        if (COUNTER_LOCK_FAIR_LOCK.getBoolean())
+        {
+            try
+            {
+                Class<?> stripedClass = Striped.class;
+
+                // Get the custom method Striped.custom
+                Method customMethod = stripedClass.getDeclaredMethod("custom", int.class, Supplier.class);
+                customMethod.setAccessible(true);
+
+                Supplier<java.util.concurrent.locks.Lock> lockSupplier = () -> new ReentrantLock(true);
+                locks = (Striped<java.util.concurrent.locks.Lock>) customMethod.invoke(null, numStripes, lockSupplier);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+        else
+        {
+            locks = Striped.lock(numStripes);
+        }
+    }
+
+    @Override
+    public List<LockHandle> grabLocks(Iterable<Integer> keys)
+    {
+        List<LockHandle> result = new ArrayList<>();
+        Iterable<java.util.concurrent.locks.Lock> locks = this.locks.bulkGet(keys);
+        locks.forEach(l -> result.add(new LockImpl(l)));
+        return result;
+    }
+
+    @Override
+    public boolean hasNumKeys()
+    {
+        return false;
+    }
+
+    private static class LockImpl implements LockHandle
+    {
+        private final java.util.concurrent.locks.Lock lock;
+        private boolean acquired;
+
+        public LockImpl(java.util.concurrent.locks.Lock lock)
+        {
+            this.lock = lock;
+        }
+
+        @Override
+        public void release()
+        {
+            if (acquired)
+                lock.unlock();
+        }
+
+        @Override
+        public boolean tryLock(long timeout, TimeUnit timeUnit) throws InterruptedException
+        {
+            acquired = lock.tryLock(timeout, timeUnit);
+            return acquired;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/filter/ANNOptions.java b/src/java/org/apache/cassandra/db/filter/ANNOptions.java
new file mode 100644
index 000000000000..f73e1f13e056
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/filter/ANNOptions.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.filter;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * {@code SELECT} query options for ANN search.
+ */
+public class ANNOptions
+{
+    public static final String RERANK_K_OPTION_NAME = "rerank_k";
+
+    public static final ANNOptions NONE = new ANNOptions(null);
+
+    public static final Serializer serializer = new Serializer();
+
+    /**
+     * The amplified limit for the ANN query to get more accurate results.
+     * A value lesser or equals to zero means no reranking.
+     * A {@code null} value means the option is not present.
+     */
+    @Nullable
+    public final Integer rerankK;
+
+    private ANNOptions(@Nullable Integer rerankK)
+    {
+        this.rerankK = rerankK;
+    }
+
+    public static ANNOptions create(@Nullable Integer rerankK)
+    {
+        // if all the options are null, return the NONE instance
+        return rerankK == null ? NONE : new ANNOptions(rerankK);
+    }
+
+    /**
+     * Validates the ANN options by checking that they are within the guardrails and that peers support the options.
+     */
+    public void validate(ClientState state, String keyspace, int limit)
+    {
+        if (rerankK == null)
+            return;
+
+        if (rerankK < limit)
+            throw new InvalidRequestException(String.format("Invalid rerank_k value %d lesser than limit %d", rerankK, limit));
+
+        Guardrails.annRerankKMaxValue.guard(rerankK, "ANN options", false, state);
+
+        // Ensure that all nodes in the cluster are in a version that supports ANN options, including this one
+        assert keyspace != null;
+        Set<InetAddressAndPort> badNodes = MessagingService.instance().endpointsWithConnectionsOnVersionBelow(keyspace, MessagingService.VERSION_DS_11);
+        if (MessagingService.current_version < MessagingService.VERSION_DS_11)
+            badNodes.add(FBUtilities.getBroadcastAddressAndPort());
+        if (!badNodes.isEmpty())
+            throw new InvalidRequestException("ANN options are not supported in clusters below DS 11.");
+    }
+
+    /**
+     * Returns the ANN options stored the given map of options.
+     *
+     * @param map the map of options in the {@code WITH ANN_OPTION} of a {@code SELECT} query
+     * @return the ANN options in the specified {@code SELECT} options, or {@link #NONE} if no options are present
+     */
+    public static ANNOptions fromMap(Map<String, String> map)
+    {
+        Integer rerankK = null;
+
+        for (Map.Entry<String, String> entry : map.entrySet())
+        {
+            String name = entry.getKey();
+            String value = entry.getValue();
+
+            if (name.equals(RERANK_K_OPTION_NAME))
+            {
+                rerankK = parseRerankK(value);
+            }
+            else
+            {
+                throw new InvalidRequestException("Unknown ANN option: " + name);
+            }
+        }
+
+        return ANNOptions.create(rerankK);
+    }
+
+    private static int parseRerankK(String value)
+    {
+        int rerankK;
+
+        try
+        {
+            rerankK = Integer.parseInt(value);
+        }
+        catch (NumberFormatException e)
+        {
+            throw new InvalidRequestException(String.format("Invalid '%s' ANN option. Expected a positive int but found: %s",
+                                                            RERANK_K_OPTION_NAME, value));
+        }
+
+        return rerankK;
+    }
+
+    public String toCQLString()
+    {
+        return String.format("{'%s': %d}", RERANK_K_OPTION_NAME, rerankK);
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ANNOptions that = (ANNOptions) o;
+        return Objects.equals(rerankK, that.rerankK);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(rerankK);
+    }
+
+    /**
+     * Serializer for {@link ANNOptions}.
+     * </p>
+     * This serializer writes an int containing bit flags that indicate which options are present, allowing the future
+     * addition of new options without increasing the messaging version. We should be able to create compatible messages
+     * in the future if we add new options and those are not explicitly set in the user query. If we receive a message
+     * with unknown newer options from a newer node, we will reject it.
+     * </p>
+     * This approach should be more space-efficient than simply using a map, as we do with the index creation options.
+     * Space is more important in this case because the {@link ANNOptions} are sent with every {@code SELECT} query. The
+     * downside is that we only allow for up to 32 options, which seems reasonable. If we ever need more options, we can
+     * use the last bit flag to indicate that we need to read more flags from the input.
+     */
+    public static class Serializer
+    {
+        /** Bit flags mask to check if the rerank K option is present. */
+        private static final int RERANK_K_MASK = 1;
+
+        /** Bit flags mask to check if there are any unknown options. It's the negation of all the known flags. */
+        private static final int UNKNOWN_OPTIONS_MASK = ~RERANK_K_MASK;
+
+        /*
+         * If you add a new option, then update ANNOptionsTest.FutureANNOptions and possibly add a new test verifying
+         * that the serialization of the updated and original versions of the options are compatible.
+         */
+
+        public void serialize(ANNOptions options, DataOutputPlus out, int version) throws IOException
+        {
+            // ANN options are only supported in DS 11 and above, so don't serialize anything if the messaging version is lower
+            if (version < MessagingService.VERSION_DS_11)
+            {
+                if (options != NONE)
+                    throw new IllegalStateException("Unable to serialize ANN options with messaging version: " + version);
+                return;
+            }
+
+            int flags = flags(options);
+            out.writeInt(flags);
+
+            if (options.rerankK != null)
+                out.writeUnsignedVInt32(options.rerankK);
+        }
+
+        public ANNOptions deserialize(DataInputPlus in, int version) throws IOException
+        {
+            // ANN options are only supported in DS 11 and above, so don't read anything if the messaging version is lower
+            if (version < MessagingService.VERSION_DS_11)
+                return ANNOptions.NONE;
+
+            int flags = in.readInt();
+
+            // Reject any flags for unknown options that may have been written by a node running newer code.
+            if ((flags & UNKNOWN_OPTIONS_MASK) != 0)
+                throw new IOException("Found unsupported ANN options, likely due to the ANN options containing " +
+                                      "new options that are not supported by this node.");
+
+            Integer rerankK = hasRerankK(flags) ? (int) in.readUnsignedVInt() : null;
+
+            return ANNOptions.create(rerankK);
+        }
+
+        public long serializedSize(ANNOptions options, int version)
+        {
+            // ANN options are only supported in DS 11 and above, so no size if the messaging version is lower
+            if (version < MessagingService.VERSION_DS_11)
+                return 0;
+
+            int flags = flags(options);
+            long size = TypeSizes.sizeof(flags);
+
+            if (options.rerankK != null)
+                size += TypeSizes.sizeofUnsignedVInt(options.rerankK);
+
+            return size;
+        }
+
+        private static int flags(ANNOptions options)
+        {
+            int flags = 0;
+
+            if (options == NONE)
+                return flags;
+
+            if (options.rerankK != null)
+                flags |= RERANK_K_MASK;
+
+            return flags;
+        }
+
+        private static boolean hasRerankK(int flags)
+        {
+            return (flags & RERANK_K_MASK) == RERANK_K_MASK;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
index 90fc9f3a1126..3dcb79f196fa 100644
--- a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
@@ -69,6 +69,9 @@ public abstract class ColumnFilter
 
     public static final Serializer serializer = new Serializer();
 
+    // TODO remove this with ANN_USE_SYNTHETIC_SCORE
+    public abstract boolean fetchesExplicitly(ColumnMetadata column);
+
     /**
      * The fetching strategy for the different queries.
      */
@@ -93,7 +96,8 @@ boolean fetchesAllColumns(boolean isStatic)
             @Override
             RegularAndStaticColumns getFetchedColumns(TableMetadata metadata, RegularAndStaticColumns queried)
             {
-                return metadata.regularAndStaticColumns();
+                var merged = queried.regulars.mergeTo(metadata.regularColumns());
+                return new RegularAndStaticColumns(metadata.staticColumns(), merged);
             }
         },
 
@@ -114,7 +118,8 @@ boolean fetchesAllColumns(boolean isStatic)
             @Override
             RegularAndStaticColumns getFetchedColumns(TableMetadata metadata, RegularAndStaticColumns queried)
             {
-                return new RegularAndStaticColumns(queried.statics, metadata.regularColumns());
+                var merged = queried.regulars.mergeTo(metadata.regularColumns());
+                return new RegularAndStaticColumns(queried.statics, merged);
             }
         },
 
@@ -208,14 +213,16 @@ public static ColumnFilter selection(TableMetadata metadata,
     }
 
     /**
-     * The columns that needs to be fetched internally for this filter.
+     * The columns that needs to be fetched internally.  See FetchingStrategy for why this is
+     * always a superset of the queried columns.
      *
      * @return the columns to fetch for this filter.
      */
     public abstract RegularAndStaticColumns fetchedColumns();
 
     /**
-     * The columns actually queried by the user.
+     * The columns needed to process the query, including selected columns, ordering columns,
+     * restriction (predicate) columns, and synthetic columns.
      * <p>
      * Note that this is in general not all the columns that are fetched internally (see {@link #fetchedColumns}).
      */
@@ -510,9 +517,7 @@ private SortedSetMultimap<ColumnIdentifier, ColumnSubselection> buildSubSelectio
      */
     public static class WildCardColumnFilter extends ColumnFilter
     {
-        /**
-         * The queried and fetched columns.
-         */
+        // for wildcards, there is no distinction between fetched and queried because queried is already "everything"
         private final RegularAndStaticColumns fetchedAndQueried;
 
         /**
@@ -558,6 +563,12 @@ public boolean fetches(ColumnMetadata column)
             return true;
         }
 
+        @Override
+        public boolean fetchesExplicitly(ColumnMetadata column)
+        {
+            return false;
+        }
+
         @Override
         public boolean fetchedColumnIsQueried(ColumnMetadata column)
         {
@@ -630,14 +641,9 @@ public static class SelectionColumnFilter extends ColumnFilter
     {
         public final FetchingStrategy fetchingStrategy;
 
-        /**
-         * The selected columns
-         */
+        // Materializes the columns required to implement queriedColumns() and fetchedColumns(),
+        // see the comments to superclass's methods
         private final RegularAndStaticColumns queried;
-
-        /**
-         * The columns that need to be fetched to be able
-         */
         private final RegularAndStaticColumns fetched;
 
         private final SortedSetMultimap<ColumnIdentifier, ColumnSubselection> subSelections; // can be null
@@ -711,6 +717,12 @@ public boolean fetches(ColumnMetadata column)
             return fetchingStrategy.fetchesAllColumns(column.isStatic()) || fetched.contains(column);
         }
 
+        @Override
+        public boolean fetchesExplicitly(ColumnMetadata column)
+        {
+            return fetched.contains(column);
+        }
+
         /**
          * Whether the provided complex cell (identified by its column and path), which is assumed to be _fetched_ by
          * this filter, is also _queried_ by the user.
diff --git a/src/java/org/apache/cassandra/db/filter/DataLimits.java b/src/java/org/apache/cassandra/db/filter/DataLimits.java
index 60203c3047ff..69ad2d37bf69 100644
--- a/src/java/org/apache/cassandra/db/filter/DataLimits.java
+++ b/src/java/org/apache/cassandra/db/filter/DataLimits.java
@@ -19,35 +19,65 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-
-import org.apache.cassandra.db.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.aggregation.AggregationSpecification;
 import org.apache.cassandra.db.aggregation.GroupMaker;
 import org.apache.cassandra.db.aggregation.GroupingState;
-import org.apache.cassandra.db.aggregation.AggregationSpecification;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.partitions.CachedPartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.BaseRowIterator;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.transform.BasePartitions;
 import org.apache.cassandra.db.transform.BaseRows;
 import org.apache.cassandra.db.transform.StoppingTransformation;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 /**
- * Object in charge of tracking if we have fetch enough data for a given query.
- *
- * This is more complicated than a single count because we support PER PARTITION
- * limits, but also due to GROUP BY and paging.
+ * Object in charge of tracking if we have fetched enough data for a given query.
+ * <p>
+ * This is more complicated than a single count because we support {@code PER PARTITION}
+ * limits, but also due to {@code GROUP BY} and paging.
+ * </p>
+ * <p>
+ * Tracking happens by row count ({@see count()}) and bytes ({@see bytes()}), with the first exhausted limit
+ * taking precedence.
+ * </p>
+ * <p>
+ * When paging is used (see {@code forPaging} methods), the minimum number between the page size and the rows/bytes
+ * limit is enforced, meaning that we'll never return more rows than requested.
+ * </p>
  */
 public abstract class DataLimits
 {
+    private static final Logger logger = LoggerFactory.getLogger(DataLimits.class);
     public static final Serializer serializer = new Serializer();
 
     public static final int NO_LIMIT = Integer.MAX_VALUE;
 
-    public static final DataLimits NONE = new CQLLimits(NO_LIMIT)
+    public static final DataLimits NONE = new CQLLimits(NO_LIMIT, NO_LIMIT, NO_LIMIT, false)
     {
         @Override
         public boolean hasEnoughLiveData(CachedPartition cached, long nowInSec, boolean countPartitionsWithOnlyStaticData, boolean enforceStrictLiveness)
@@ -80,45 +110,92 @@ public PartitionIterator filter(PartitionIterator iter, long nowInSec, boolean c
 
     // We currently deal with distinct queries by querying full partitions but limiting the result at 1 row per
     // partition (see SelectStatement.makeFilter). So an "unbounded" distinct is still actually doing some filtering.
-    public static final DataLimits DISTINCT_NONE = new CQLLimits(NO_LIMIT, 1, true);
+    public static final DataLimits DISTINCT_NONE = new CQLLimits(NO_LIMIT, NO_LIMIT, 1, true);
 
     public enum Kind
     {
-        CQL_LIMIT,
-        CQL_PAGING_LIMIT,
+        CQL_LIMIT(0),
+        CQL_PAGING_LIMIT(1),
         /** @deprecated See CASSANDRA-16582 */
-        @Deprecated(since = "4.0") THRIFT_LIMIT, //Deprecated and unused in 4.0, stop publishing in 5.0, reclaim in 6.0
+        @Deprecated(since = "4.0") THRIFT_LIMIT(), //Deprecated and unused in 4.0, stop publishing in 5.0, reclaim in 6.0
         /** @deprecated See CASSANDRA-16582 */
-        @Deprecated(since = "4.0") SUPER_COLUMN_COUNTING_LIMIT, //Deprecated and unused in 4.0, stop publishing in 5.0, reclaim in 6.0
-        CQL_GROUP_BY_LIMIT,
-        CQL_GROUP_BY_PAGING_LIMIT,
+        @Deprecated(since = "4.0") SUPER_COLUMN_COUNTING_LIMIT(), //Deprecated and unused in 4.0, stop publishing in 5.0, reclaim in 6.0
+        CQL_GROUP_BY_LIMIT(2),
+        CQL_GROUP_BY_PAGING_LIMIT(3);
+
+        /**
+         * DSE compatibility ordinal for Kind values unknown to DSE.
+         */
+        private static final int UNDEFINED = -1;
+        /**
+         * DSE compatibility values for Kind values. Some of the ordinals may be undefined, in which case the value is null.
+         */
+        private static final Kind[] DSE_COMPATIBILITY_VALUES;
+
+        static
+        {
+            Kind[] values = values();
+            DSE_COMPATIBILITY_VALUES = new Kind[values.length];
+            for (Kind kind : values)
+            {
+                if (kind.dseCompatibilityOrdinal != UNDEFINED)
+                {
+                    assert DSE_COMPATIBILITY_VALUES[kind.dseCompatibilityOrdinal] == null : "Duplicate DSE compatibility ordinal " + kind.dseCompatibilityOrdinal;
+                    DSE_COMPATIBILITY_VALUES[kind.dseCompatibilityOrdinal] = kind;
+                }
+            }
+        }
+
+        /**
+         * Used with DSE compatibility protocol {@link org.apache.cassandra.net.MessagingService.Version#VERSION_30}.
+         * DSE doesn't know {@link #THRIFT_LIMIT} and {@link #SUPER_COLUMN_COUNTING_LIMIT}, so the compatibility
+         * ordinals are shifted by 2.
+         */
+        private final int dseCompatibilityOrdinal;
+
+        Kind(int dseCompatibilityOrdinal)
+        {
+            this.dseCompatibilityOrdinal = dseCompatibilityOrdinal;
+        }
+
+        Kind()
+        {
+            this(UNDEFINED);
+        }
+
+        public int dseCompatibilityOrdinal()
+        {
+            assert dseCompatibilityOrdinal != UNDEFINED : "DSE compatibility ordinal not defined for kind " + this;
+            return dseCompatibilityOrdinal;
+        }
     }
 
     public static DataLimits cqlLimits(int cqlRowLimit)
     {
-        return cqlRowLimit == NO_LIMIT ? NONE : new CQLLimits(cqlRowLimit);
+        return cqlRowLimit == NO_LIMIT ? NONE : new CQLLimits(NO_LIMIT, cqlRowLimit, NO_LIMIT, false);
     }
 
     public static DataLimits cqlLimits(int cqlRowLimit, int perPartitionLimit)
     {
         return cqlRowLimit == NO_LIMIT && perPartitionLimit == NO_LIMIT
              ? NONE
-             : new CQLLimits(cqlRowLimit, perPartitionLimit);
+             : new CQLLimits(NO_LIMIT, cqlRowLimit, perPartitionLimit, false);
     }
 
-    private static DataLimits cqlLimits(int cqlRowLimit, int perPartitionLimit, boolean isDistinct)
+    private static DataLimits cqlLimits(int bytesLimit, int cqlRowLimit, int perPartitionLimit, boolean isDistinct)
     {
-        return cqlRowLimit == NO_LIMIT && perPartitionLimit == NO_LIMIT && !isDistinct
+        return bytesLimit == NO_LIMIT && cqlRowLimit == NO_LIMIT && perPartitionLimit == NO_LIMIT && !isDistinct
              ? NONE
-             : new CQLLimits(cqlRowLimit, perPartitionLimit, isDistinct);
+             : new CQLLimits(bytesLimit, cqlRowLimit, perPartitionLimit, isDistinct);
     }
 
     public static DataLimits groupByLimits(int groupLimit,
                                            int groupPerPartitionLimit,
+                                           int bytesLimit,
                                            int rowLimit,
                                            AggregationSpecification groupBySpec)
     {
-        return new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, rowLimit, groupBySpec);
+        return new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, bytesLimit, rowLimit, groupBySpec);
     }
 
     public static DataLimits distinctLimits(int cqlRowLimit)
@@ -136,13 +213,20 @@ public boolean isGroupByLimit()
         return false;
     }
 
-    public boolean isExhausted(Counter counter)
+    /**
+     * Returns true if the count limit is not reached.
+     *
+     * Note: currently this method's only usage is for paging, where it is checked after processing a page as a quick
+     * signal that the data for the query is complete - if the count limit is not reached at the end of the page, this
+     * must be because there is no more data to return.
+     */
+    public boolean isCounterBelowLimits(Counter counter)
     {
-        return counter.counted() < count();
+        return counter.counted() < count() && counter.bytesCounted() < bytes();
     }
 
-    public abstract DataLimits forPaging(int pageSize);
-    public abstract DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining);
+    public abstract DataLimits forPaging(PageSize pageSize);
+    public abstract DataLimits forPaging(PageSize pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining);
 
     public abstract DataLimits forShortReadRetry(int toFetch);
 
@@ -180,6 +264,23 @@ public abstract Counter newCounter(long nowInSec,
                                        boolean countPartitionsWithOnlyStaticData,
                                        boolean enforceStrictLiveness);
 
+    /**
+     * The max number of bytes this limits enforces.
+     * <p>
+     * Note that if this value is set, less rows might be returned if the size of the current rows exceeds the bytes limit.
+     *
+     * @return the maximum number of bytes this limits enforces.
+     */
+    public abstract int bytes();
+
+    /**
+     * The max number of rows this limits enforces. Note that this means traversed rows, regardless we use grouping or not.
+     * <p>
+     * @return the maximum number of rows this limits enforces.
+     */
+    @VisibleForTesting
+    public abstract int rows();
+
     /**
      * The max number of results this limits enforces.
      * <p>
@@ -198,6 +299,17 @@ public abstract Counter newCounter(long nowInSec,
      */
     public abstract DataLimits withoutState();
 
+    /**
+     * Returns a copy of this DataLimits with updated counted limit whatever it is (either the rows limit
+     * or groups limit depending on the actual implementation)
+     */
+    public abstract DataLimits withCountedLimit(int newCountedLimit);
+
+    /**
+     * Returns a copy of this DataLimits with updated bytes limit.
+     */
+    public abstract DataLimits withBytesLimit(int bytesLimit);
+
     public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter,
                                               long nowInSec,
                                               boolean countPartitionsWithOnlyStaticData)
@@ -283,6 +395,12 @@ public RowIterator applyTo(RowIterator partition)
 
         public abstract int countedInCurrentPartition();
 
+        /**
+         * The number of bytes for the counted rows.
+         *
+         * @return the number of bytes counted.
+         */
+        public abstract int bytesCounted();
         /**
          * The number of rows counted.
          *
@@ -342,36 +460,32 @@ public void onClose()
     }
 
     /**
-     * Limits used by CQL; this counts rows.
+     * Limits used by CQL; this counts rows or bytes read. Please note:
+     * <ul>
+     * <li>When paging on rows, the minimum number of rows between the current limit and the page size is used as actual limit.</li>
+     * <li>When paging on bytes, the number of bytes takes precedence over the rows limit.</li>
+     * </ul>
      */
     private static class CQLLimits extends DataLimits
     {
+        protected final int bytesLimit;
         protected final int rowLimit;
         protected final int perPartitionLimit;
 
         // Whether the query is a distinct query or not.
         protected final boolean isDistinct;
 
-        private CQLLimits(int rowLimit)
+        private CQLLimits(int bytesLimit, int rowsLimit, int perPartitionLimit, boolean isDistinct)
         {
-            this(rowLimit, NO_LIMIT);
-        }
-
-        private CQLLimits(int rowLimit, int perPartitionLimit)
-        {
-            this(rowLimit, perPartitionLimit, false);
-        }
-
-        private CQLLimits(int rowLimit, int perPartitionLimit, boolean isDistinct)
-        {
-            this.rowLimit = rowLimit;
+            this.bytesLimit = bytesLimit;
+            this.rowLimit = rowsLimit;
             this.perPartitionLimit = perPartitionLimit;
             this.isDistinct = isDistinct;
         }
 
         private static CQLLimits distinct(int rowLimit)
         {
-            return new CQLLimits(rowLimit, 1, true);
+            return new CQLLimits(NO_LIMIT, rowLimit, 1, true);
         }
 
         public Kind kind()
@@ -381,7 +495,7 @@ public Kind kind()
 
         public boolean isUnlimited()
         {
-            return rowLimit == NO_LIMIT && perPartitionLimit == NO_LIMIT;
+            return bytesLimit == NO_LIMIT && rowLimit == NO_LIMIT && perPartitionLimit == NO_LIMIT;
         }
 
         public boolean isDistinct()
@@ -389,19 +503,27 @@ public boolean isDistinct()
             return isDistinct;
         }
 
-        public DataLimits forPaging(int pageSize)
+        public DataLimits forPaging(PageSize pageSize)
         {
-            return new CQLLimits(pageSize, perPartitionLimit, isDistinct);
+            return new CQLLimits(pageSize.minBytesCount(bytesLimit),
+                                 pageSize.minRowsCount(rowLimit),
+                                 perPartitionLimit,
+                                 isDistinct);
         }
 
-        public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+        public DataLimits forPaging(PageSize pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
         {
-            return new CQLPagingLimits(pageSize, perPartitionLimit, isDistinct, lastReturnedKey, lastReturnedKeyRemaining);
+            return new CQLPagingLimits(pageSize.minBytesCount(bytesLimit),
+                                       pageSize.minRowsCount(rowLimit),
+                                       perPartitionLimit,
+                                       isDistinct,
+                                       lastReturnedKey,
+                                       lastReturnedKeyRemaining);
         }
 
         public DataLimits forShortReadRetry(int toFetch)
         {
-            return new CQLLimits(toFetch, perPartitionLimit, isDistinct);
+            return new CQLLimits(bytesLimit, toFetch, perPartitionLimit, isDistinct);
         }
 
         public boolean hasEnoughLiveData(CachedPartition cached, long nowInSec, boolean countPartitionsWithOnlyStaticData, boolean enforceStrictLiveness)
@@ -438,6 +560,16 @@ public Counter newCounter(long nowInSec,
             return new CQLCounter(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData, enforceStrictLiveness);
         }
 
+        public int bytes()
+        {
+            return bytesLimit;
+        }
+
+        public int rows()
+        {
+            return rowLimit;
+        }
+
         public int count()
         {
             return rowLimit;
@@ -453,6 +585,18 @@ public DataLimits withoutState()
             return this;
         }
 
+        @Override
+        public DataLimits withCountedLimit(int newCountedLimit)
+        {
+            return new CQLLimits(bytesLimit, newCountedLimit, perPartitionLimit, isDistinct);
+        }
+
+        @Override
+        public DataLimits withBytesLimit(int bytesLimit)
+        {
+            return new CQLLimits(bytesLimit, rowLimit, perPartitionLimit, isDistinct);
+        }
+
         public float estimateTotalResults(ColumnFamilyStore cfs)
         {
             // TODO: we should start storing stats on the number of rows (instead of the number of cells, which
@@ -463,10 +607,16 @@ public float estimateTotalResults(ColumnFamilyStore cfs)
 
         protected class CQLCounter extends Counter
         {
+            /**
+             * Bytes and rows counted by this counter.
+             */
+            protected int bytesCounted;
             protected int rowsCounted;
             protected int rowsInCurrentPartition;
             protected final boolean countPartitionsWithOnlyStaticData;
 
+            protected int staticRowBytes;
+
             protected boolean hasLiveStaticRow;
 
             public CQLCounter(long nowInSec,
@@ -483,13 +633,14 @@ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
             {
                 rowsInCurrentPartition = 0;
                 hasLiveStaticRow = !staticRow.isEmpty() && isLive(staticRow);
+                staticRowBytes = hasLiveStaticRow && bytesLimit != NO_LIMIT ? staticRow.dataSize() : 0;
             }
 
             @Override
             public Row applyToRow(Row row)
             {
                 if (isLive(row))
-                    incrementRowCount();
+                    incrementRowCount(bytesLimit != NO_LIMIT ? row.dataSize() : 0);
                 return row;
             }
 
@@ -500,15 +651,18 @@ public void onPartitionClose()
                 // rows in the partition. However, if we only have the static row, it will be returned as one row
                 // so count it.
                 if (countPartitionsWithOnlyStaticData && hasLiveStaticRow && rowsInCurrentPartition == 0)
-                    incrementRowCount();
+                    incrementRowCount(staticRowBytes);
                 super.onPartitionClose();
             }
 
-            protected void incrementRowCount()
+            protected void incrementRowCount(int rowSize)
             {
-                if (++rowsCounted >= rowLimit)
+                bytesCounted += rowSize;
+                rowsCounted++;
+                rowsInCurrentPartition++;
+                if (bytesCounted >= bytesLimit || rowsCounted >= rowLimit)
                     stop();
-                if (++rowsInCurrentPartition >= perPartitionLimit)
+                if (rowsInCurrentPartition >= perPartitionLimit)
                     stopInPartition();
             }
 
@@ -522,6 +676,11 @@ public int countedInCurrentPartition()
                 return rowsInCurrentPartition;
             }
 
+            public int bytesCounted()
+            {
+                return bytesCounted;
+            }
+
             public int rowsCounted()
             {
                 return rowsCounted;
@@ -534,31 +693,35 @@ public int rowsCountedInCurrentPartition()
 
             public boolean isDone()
             {
-                return rowsCounted >= rowLimit;
+                return rowsCounted >= rowLimit || bytesCounted >= bytesLimit || counted() >= count();
             }
 
             public boolean isDoneForPartition()
             {
                 return isDone() || rowsInCurrentPartition >= perPartitionLimit;
             }
+
+            @Override
+            public String toString()
+            {
+                return String.format("%s(bytes=%s/%s, rows=%s/%s, partition-rows=%s/%s)", this.getClass().getName(),
+                                     bytesCounted(), bytesLimit, rowsCounted(), rowLimit, rowsCountedInCurrentPartition(), perPartitionLimit);
+            }
         }
 
         @Override
         public String toString()
         {
-            StringBuilder sb = new StringBuilder();
+            List<String> limits = new ArrayList<>(3);
 
+            if (bytesLimit != NO_LIMIT)
+                limits.add("BYTES LIMIT " + bytesLimit);
             if (rowLimit != NO_LIMIT)
-            {
-                sb.append("LIMIT ").append(rowLimit);
-                if (perPartitionLimit != NO_LIMIT)
-                    sb.append(' ');
-            }
-
+                limits.add("ROWS LIMIT " + rowLimit);
             if (perPartitionLimit != NO_LIMIT)
-                sb.append("PER PARTITION LIMIT ").append(perPartitionLimit);
+                limits.add("PER PARTITION LIMIT " + perPartitionLimit);
 
-            return sb.toString();
+            return String.join(" ", limits);
         }
     }
 
@@ -567,9 +730,9 @@ private static class CQLPagingLimits extends CQLLimits
         private final ByteBuffer lastReturnedKey;
         private final int lastReturnedKeyRemaining;
 
-        public CQLPagingLimits(int rowLimit, int perPartitionLimit, boolean isDistinct, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+        public CQLPagingLimits(int bytesLimit, int rowLimit, int perPartitionLimit, boolean isDistinct, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
         {
-            super(rowLimit, perPartitionLimit, isDistinct);
+            super(bytesLimit, rowLimit, perPartitionLimit, isDistinct);
             this.lastReturnedKey = lastReturnedKey;
             this.lastReturnedKeyRemaining = lastReturnedKeyRemaining;
         }
@@ -581,13 +744,13 @@ public Kind kind()
         }
 
         @Override
-        public DataLimits forPaging(int pageSize)
+        public DataLimits forPaging(PageSize pageSize)
         {
             throw new UnsupportedOperationException();
         }
 
         @Override
-        public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+        public DataLimits forPaging(PageSize pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
         {
             throw new UnsupportedOperationException();
         }
@@ -595,7 +758,19 @@ public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastRe
         @Override
         public DataLimits withoutState()
         {
-            return new CQLLimits(rowLimit, perPartitionLimit, isDistinct);
+            return new CQLLimits(bytesLimit, rowLimit, perPartitionLimit, isDistinct);
+        }
+
+        @Override
+        public DataLimits withCountedLimit(int newCountedLimit)
+        {
+            return new CQLPagingLimits(bytesLimit, newCountedLimit, perPartitionLimit, isDistinct, lastReturnedKey, lastReturnedKeyRemaining);
+        }
+
+        @Override
+        public DataLimits withBytesLimit(int bytesLimit)
+        {
+            return new CQLPagingLimits(bytesLimit, rowLimit, perPartitionLimit, isDistinct, lastReturnedKey, lastReturnedKeyRemaining);
         }
 
         @Override
@@ -625,6 +800,7 @@ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
                     // if any already, so force hasLiveStaticRow to false so we make sure to not count it
                     // once more.
                     hasLiveStaticRow = false;
+                    staticRowBytes = 0;
                 }
                 else
                 {
@@ -632,6 +808,16 @@ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
                 }
             }
         }
+
+        @Override
+        public String toString()
+        {
+            return new StringJoiner(", ", CQLPagingLimits.class.getSimpleName() + "[", "]")
+                   .add("super=" + super.toString())
+                   .add("lastReturnedKey=" + (lastReturnedKey != null ? ByteBufferUtil.bytesToHex(lastReturnedKey) : null))
+                   .add("lastReturnedKeyRemaining=" + lastReturnedKeyRemaining)
+                   .toString();
+        }
     }
 
     /**
@@ -664,19 +850,21 @@ private static class CQLGroupByLimits extends CQLLimits
 
         public CQLGroupByLimits(int groupLimit,
                                 int groupPerPartitionLimit,
+                                int bytesLimit,
                                 int rowLimit,
                                 AggregationSpecification groupBySpec)
         {
-            this(groupLimit, groupPerPartitionLimit, rowLimit, groupBySpec, GroupingState.EMPTY_STATE);
+            this(groupLimit, groupPerPartitionLimit, bytesLimit, rowLimit, groupBySpec, GroupingState.EMPTY_STATE);
         }
 
         private CQLGroupByLimits(int groupLimit,
                                  int groupPerPartitionLimit,
+                                 int bytesLimit,
                                  int rowLimit,
                                  AggregationSpecification groupBySpec,
                                  GroupingState state)
         {
-            super(rowLimit, NO_LIMIT, false);
+            super(bytesLimit, rowLimit, NO_LIMIT, false);
             this.groupLimit = groupLimit;
             this.groupPerPartitionLimit = groupPerPartitionLimit;
             this.groupBySpec = groupBySpec;
@@ -697,12 +885,12 @@ public boolean isGroupByLimit()
 
         public boolean isUnlimited()
         {
-            return groupLimit == NO_LIMIT && groupPerPartitionLimit == NO_LIMIT && rowLimit == NO_LIMIT;
+            return groupLimit == NO_LIMIT && groupPerPartitionLimit == NO_LIMIT && super.isUnlimited();
         }
 
         public DataLimits forShortReadRetry(int toFetch)
         {
-            return new CQLLimits(toFetch);
+            return new CQLLimits(NO_LIMIT, toFetch, NO_LIMIT, false);
         }
 
         @Override
@@ -715,21 +903,35 @@ public float estimateTotalResults(ColumnFamilyStore cfs)
         }
 
         @Override
-        public DataLimits forPaging(int pageSize)
+        public DataLimits forPaging(PageSize pageSize)
         {
-            return new CQLGroupByLimits(pageSize,
+            if (logger.isTraceEnabled())
+                logger.trace("{} forPaging({})", hashCode(), pageSize);
+
+            return new CQLGroupByLimits(groupLimit,
                                         groupPerPartitionLimit,
-                                        rowLimit,
+                                        pageSize.minBytesCount(bytesLimit),
+                                        pageSize.minRowsCount(rowLimit),
                                         groupBySpec,
                                         state);
         }
 
         @Override
-        public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
-        {
-            return new CQLGroupByPagingLimits(pageSize,
+        public DataLimits forPaging(PageSize pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+        {
+            if (logger.isTraceEnabled())
+                logger.trace("{} forPaging({}, {}, {}) vs state {}/{}",
+                             hashCode(),
+                             pageSize,
+                             lastReturnedKey == null ? "null" : ByteBufferUtil.bytesToHex(lastReturnedKey),
+                             lastReturnedKeyRemaining,
+                             state.partitionKey() == null ? "null" : ByteBufferUtil.bytesToHex(state.partitionKey()),
+                             state.clustering() == null ? "null" : state.clustering().toString());
+
+            return new CQLGroupByPagingLimits(groupLimit,
                                               groupPerPartitionLimit,
-                                              rowLimit,
+                                              pageSize.minBytesCount(bytesLimit),
+                                              pageSize.minRowsCount(rowLimit),
                                               groupBySpec,
                                               state,
                                               lastReturnedKey,
@@ -739,8 +941,9 @@ public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastRe
         @Override
         public DataLimits forGroupByInternalPaging(GroupingState state)
         {
-            return new CQLGroupByLimits(rowLimit,
+            return new CQLGroupByLimits(groupLimit,
                                         groupPerPartitionLimit,
+                                        bytesLimit,
                                         rowLimit,
                                         groupBySpec,
                                         state);
@@ -772,41 +975,44 @@ public DataLimits withoutState()
         {
             return state == GroupingState.EMPTY_STATE
                  ? this
-                 : new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, rowLimit, groupBySpec);
+                 : new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, bytesLimit, rowLimit, groupBySpec);
+        }
+
+        @Override
+        public DataLimits withCountedLimit(int newCountedLimit)
+        {
+            return new CQLGroupByLimits(newCountedLimit, groupPerPartitionLimit, bytesLimit, rowLimit, groupBySpec, state);
         }
 
+        @Override
+        public DataLimits withBytesLimit(int bytesLimit)
+        {
+            return new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, bytesLimit, rowLimit, groupBySpec, state);
+        }
+
+
+
         @Override
         public String toString()
         {
-            StringBuilder sb = new StringBuilder();
+            List<String> limits = new ArrayList<>(4);
 
             if (groupLimit != NO_LIMIT)
-            {
-                sb.append("GROUP LIMIT ").append(groupLimit);
-                if (groupPerPartitionLimit != NO_LIMIT || rowLimit != NO_LIMIT)
-                    sb.append(' ');
-            }
-
+                limits.add("GROUP LIMIT " + groupLimit);
             if (groupPerPartitionLimit != NO_LIMIT)
-            {
-                sb.append("GROUP PER PARTITION LIMIT ").append(groupPerPartitionLimit);
-                if (rowLimit != NO_LIMIT)
-                    sb.append(' ');
-            }
-
+                limits.add("GROUP PER PARTITION LIMIT " + groupPerPartitionLimit);
+            if (bytesLimit != NO_LIMIT)
+                limits.add("BYTES LIMIT " + bytesLimit);
             if (rowLimit != NO_LIMIT)
-            {
-                sb.append("LIMIT ").append(rowLimit);
-            }
+                limits.add("ROWS LIMIT " + rowLimit);
 
-            return sb.toString();
+            return String.join(" ", limits);
         }
 
         @Override
-        public boolean isExhausted(Counter counter)
+        public boolean isCounterBelowLimits(Counter counter)
         {
-            return ((GroupByAwareCounter) counter).rowsCounted < rowLimit
-                    && counter.counted() < groupLimit;
+            return counter.rowsCounted() < rowLimit && counter.bytesCounted() < bytesLimit && counter.counted() < groupLimit;
         }
 
         protected class GroupByAwareCounter extends Counter
@@ -820,6 +1026,11 @@ protected class GroupByAwareCounter extends Counter
              */
             protected DecoratedKey currentPartitionKey;
 
+            /**
+             * The number of bytes counted so far.
+             */
+            protected int bytesCounted;
+
             /**
              * The number of rows counted so far.
              */
@@ -845,6 +1056,8 @@ protected class GroupByAwareCounter extends Counter
 
             protected boolean hasLiveStaticRow;
 
+            protected int staticRowBytes;
+
             protected boolean hasReturnedRowsFromCurrentPartition;
 
             private GroupByAwareCounter(long nowInSec,
@@ -865,6 +1078,10 @@ private GroupByAwareCounter(long nowInSec,
             @Override
             public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
             {
+                if (logger.isTraceEnabled())
+                    logger.trace("{} - GroupByAwareCounter.newPartition {} with state {}", hashCode(),
+                                 ByteBufferUtil.bytesToHex(partitionKey.getKey()), state.partitionKey() != null ? ByteBufferUtil.bytesToHex(state.partitionKey()) : "null");
+
                 if (partitionKey.getKey().equals(state.partitionKey()))
                 {
                     // The only case were we could have state.partitionKey() equals to the partition key
@@ -874,6 +1091,7 @@ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
                     // the static row if any already, so force hasLiveStaticRow to false so we make sure to not count it
                     // once more.
                     hasLiveStaticRow = false;
+                    staticRowBytes = 0;
                     hasReturnedRowsFromCurrentPartition = true;
                     hasUnfinishedGroup = true;
                 }
@@ -897,6 +1115,7 @@ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
                     }
                     hasReturnedRowsFromCurrentPartition = false;
                     hasLiveStaticRow = !staticRow.isEmpty() && isLive(staticRow);
+                    staticRowBytes = hasLiveStaticRow ? staticRow.dataSize() : 0;
                 }
                 currentPartitionKey = partitionKey;
                 // If we are done we need to preserve the groupInCurrentPartition and rowsCountedInCurrentPartition
@@ -911,12 +1130,19 @@ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
             @Override
             protected Row applyToStatic(Row row)
             {
+                if (logger.isTraceEnabled())
+                    logger.trace("{} - GroupByAwareCounter.applyToStatic {}/{}",
+                                 hashCode(),
+                                 currentPartitionKey != null ? ByteBufferUtil.bytesToHex(currentPartitionKey.getKey()) : "null",
+                                 row == null ? "null" : row.clustering().toString());
+
                 // It's possible that we're "done" if the partition we just started bumped the number of groups (in
                 // applyToPartition() above), in which case Transformation will still call this method. In that case, we
                 // want to ignore the static row, it should (and will) be returned with the next page/group if needs be.
                 if (enforceLimits && isDone())
                 {
                     hasLiveStaticRow = false; // The row has not been returned
+                    staticRowBytes = 0;
                     return Rows.EMPTY_STATIC_ROW;
                 }
                 return row;
@@ -925,6 +1151,12 @@ protected Row applyToStatic(Row row)
             @Override
             public Row applyToRow(Row row)
             {
+                if (logger.isTraceEnabled())
+                    logger.trace("{} - GroupByAwareCounter.applyToRow {}/{}",
+                                 hashCode(),
+                                 ByteBufferUtil.bytesToHex(currentPartitionKey.getKey()),
+                                 row.clustering().toString());
+
                 // We want to check if the row belongs to a new group even if it has been deleted. The goal being
                 // to minimize the chances of having to go through the same data twice if we detect on the next
                 // non deleted row that we have reached the limit.
@@ -949,7 +1181,7 @@ public Row applyToRow(Row row)
                 if (isLive(row))
                 {
                     hasUnfinishedGroup = true;
-                    incrementRowCount();
+                    incrementRowCount(bytesLimit != NO_LIMIT ? row.dataSize() : 0);
                     hasReturnedRowsFromCurrentPartition = true;
                 }
 
@@ -968,6 +1200,12 @@ public int countedInCurrentPartition()
                 return groupInCurrentPartition;
             }
 
+            @Override
+            public int bytesCounted()
+            {
+                return bytesCounted;
+            }
+
             @Override
             public int rowsCounted()
             {
@@ -980,10 +1218,12 @@ public int rowsCountedInCurrentPartition()
                 return rowsCountedInCurrentPartition;
             }
 
-            protected void incrementRowCount()
+            protected void incrementRowCount(int rowSize)
             {
                 rowsCountedInCurrentPartition++;
-                if (++rowsCounted >= rowLimit)
+                rowsCounted++;
+                bytesCounted += rowSize;
+                if (rowsCounted >= rowLimit || bytesCounted >= bytesLimit)
                     stop();
             }
 
@@ -1021,7 +1261,7 @@ public void onPartitionClose()
                 // so count it.
                 if (countPartitionsWithOnlyStaticData && hasLiveStaticRow && !hasReturnedRowsFromCurrentPartition)
                 {
-                    incrementRowCount();
+                    incrementRowCount(staticRowBytes);
                     incrementGroupCount();
                     incrementGroupInCurrentPartitionCount();
                     hasUnfinishedGroup = false;
@@ -1038,7 +1278,7 @@ public void onClose()
                 // 2) the end of the data is reached
                 // We know that the end of the data is reached if the group limit has not been reached
                 // and the number of rows counted is smaller than the internal page size.
-                if (hasUnfinishedGroup && groupCounted < groupLimit && rowsCounted < rowLimit)
+                if (hasUnfinishedGroup && groupCounted < groupLimit && bytesCounted < bytesLimit && rowsCounted < rowLimit)
                 {
                     incrementGroupCount();
                     incrementGroupInCurrentPartitionCount();
@@ -1046,6 +1286,13 @@ public void onClose()
 
                 super.onClose();
             }
+
+            @Override
+            public String toString()
+            {
+                return String.format("%s(bytes=%s/%s, rows=%s/%s, partition-rows=%s/%s, groups=%s/%s, partition-groups=%s/%s)", this.getClass().getName(),
+                                     bytesCounted(), bytesLimit, rowsCounted(), rowLimit, rowsCountedInCurrentPartition(), perPartitionLimit, groupCounted, groupLimit, groupInCurrentPartition, groupPerPartitionLimit);
+            }
         }
     }
 
@@ -1057,6 +1304,7 @@ private static class CQLGroupByPagingLimits extends CQLGroupByLimits
 
         public CQLGroupByPagingLimits(int groupLimit,
                                       int groupPerPartitionLimit,
+                                      int bytesLimit,
                                       int rowLimit,
                                       AggregationSpecification groupBySpec,
                                       GroupingState state,
@@ -1065,6 +1313,7 @@ public CQLGroupByPagingLimits(int groupLimit,
         {
             super(groupLimit,
                   groupPerPartitionLimit,
+                  bytesLimit,
                   rowLimit,
                   groupBySpec,
                   state);
@@ -1080,13 +1329,13 @@ public Kind kind()
         }
 
         @Override
-        public DataLimits forPaging(int pageSize)
+        public DataLimits forPaging(PageSize pageSize)
         {
             throw new UnsupportedOperationException();
         }
 
         @Override
-        public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+        public DataLimits forPaging(PageSize pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
         {
             throw new UnsupportedOperationException();
         }
@@ -1107,9 +1356,23 @@ public Counter newCounter(long nowInSec, boolean assumeLiveData, boolean countPa
         @Override
         public DataLimits withoutState()
         {
-            return new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, rowLimit, groupBySpec);
+            return new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, bytesLimit, rowLimit, groupBySpec);
         }
 
+        @Override
+        public DataLimits withCountedLimit(int newCountedLimit)
+        {
+            return new CQLGroupByPagingLimits(newCountedLimit, groupPerPartitionLimit, bytesLimit, rowLimit, groupBySpec, state, lastReturnedKey, lastReturnedKeyRemaining);
+        }
+
+        @Override
+        public DataLimits withBytesLimit(int bytesLimit)
+        {
+            return new CQLGroupByPagingLimits(groupLimit, groupPerPartitionLimit, bytesLimit, rowLimit, groupBySpec, state, lastReturnedKey, lastReturnedKeyRemaining);
+        }
+
+
+
         private class PagingGroupByAwareCounter extends GroupByAwareCounter
         {
             private PagingGroupByAwareCounter(long nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData, boolean enforceStrictLiveness)
@@ -1120,12 +1383,17 @@ private PagingGroupByAwareCounter(long nowInSec, boolean assumeLiveData, boolean
             @Override
             public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
             {
+                if (logger.isTraceEnabled())
+                    logger.trace("{} - CQLGroupByPagingLimits.applyToPartition {}",
+                                 hashCode(), ByteBufferUtil.bytesToHex(partitionKey.getKey()));
+
                 if (partitionKey.getKey().equals(lastReturnedKey))
                 {
                     currentPartitionKey = partitionKey;
                     groupInCurrentPartition = groupPerPartitionLimit - lastReturnedKeyRemaining;
                     hasReturnedRowsFromCurrentPartition = true;
                     hasLiveStaticRow = false;
+                    staticRowBytes = 0;
                     hasUnfinishedGroup = state.hasClustering();
                 }
                 else
@@ -1134,13 +1402,29 @@ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
                 }
             }
         }
+
+        @Override
+        public String toString()
+        {
+            return new StringJoiner(", ", CQLGroupByPagingLimits.class.getSimpleName() + "[", "]")
+                   .add("super=" + super.toString())
+                   .add("lastReturnedKey=" + (lastReturnedKey != null ? ByteBufferUtil.bytesToHex(lastReturnedKey) : null))
+                   .add("lastReturnedKeyRemaining=" + lastReturnedKeyRemaining)
+                   .toString();
+        }
     }
 
     public static class Serializer
     {
         public void serialize(DataLimits limits, DataOutputPlus out, int version, ClusteringComparator comparator) throws IOException
         {
-            out.writeByte(limits.kind().ordinal());
+            // VERSION_30 is used for migration only (DSE <-> CC compatibility is required).
+            // DSE doesn't know THRIFT_LIMIT(2) and SUPER_COLUMN_COUNTING_LIMIT(3), so DSE ordinals must be used here.
+            if (version == MessagingService.VERSION_30)
+                out.writeByte(limits.kind().dseCompatibilityOrdinal());
+            else
+                out.writeByte(limits.kind().ordinal());
+
             switch (limits.kind())
             {
                 case CQL_LIMIT:
@@ -1148,6 +1432,8 @@ public void serialize(DataLimits limits, DataOutputPlus out, int version, Cluste
                     CQLLimits cqlLimits = (CQLLimits)limits;
                     out.writeUnsignedVInt32(cqlLimits.rowLimit);
                     out.writeUnsignedVInt32(cqlLimits.perPartitionLimit);
+                    if (version >= MessagingService.VERSION_DS_10)
+                        out.writeUnsignedVInt32(cqlLimits.bytesLimit);
                     out.writeBoolean(cqlLimits.isDistinct);
                     if (limits.kind() == Kind.CQL_PAGING_LIMIT)
                     {
@@ -1162,6 +1448,8 @@ public void serialize(DataLimits limits, DataOutputPlus out, int version, Cluste
                     out.writeUnsignedVInt32(groupByLimits.groupLimit);
                     out.writeUnsignedVInt32(groupByLimits.groupPerPartitionLimit);
                     out.writeUnsignedVInt32(groupByLimits.rowLimit);
+                    if (version >= MessagingService.VERSION_DS_10)
+                        out.writeUnsignedVInt32(groupByLimits.bytesLimit);
 
                     AggregationSpecification groupBySpec = groupByLimits.groupBySpec;
                     AggregationSpecification.serializer.serialize(groupBySpec, out, version);
@@ -1180,7 +1468,13 @@ public void serialize(DataLimits limits, DataOutputPlus out, int version, Cluste
 
         public DataLimits deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
         {
-            Kind kind = Kind.values()[in.readUnsignedByte()];
+            int ordinal = in.readUnsignedByte();
+            Kind kind = version == MessagingService.VERSION_30 ?
+                        Kind.DSE_COMPATIBILITY_VALUES[ordinal] :
+                        Kind.values()[ordinal];
+
+            assert kind != null : "Unknown DataLimits.Kind with ordinal " + ordinal + " and version " + version;
+
             switch (kind)
             {
                 case CQL_LIMIT:
@@ -1188,12 +1482,13 @@ public DataLimits deserialize(DataInputPlus in, int version, TableMetadata metad
                 {
                     int rowLimit = in.readUnsignedVInt32();
                     int perPartitionLimit = in.readUnsignedVInt32();
+                    int bytesLimit = version >= MessagingService.VERSION_DS_10 ? (int) in.readUnsignedVInt() : NO_LIMIT;
                     boolean isDistinct = in.readBoolean();
                     if (kind == Kind.CQL_LIMIT)
-                        return cqlLimits(rowLimit, perPartitionLimit, isDistinct);
+                        return cqlLimits(bytesLimit, rowLimit, perPartitionLimit, isDistinct);
                     ByteBuffer lastKey = ByteBufferUtil.readWithVIntLength(in);
                     int lastRemaining = in.readUnsignedVInt32();
-                    return new CQLPagingLimits(rowLimit, perPartitionLimit, isDistinct, lastKey, lastRemaining);
+                    return new CQLPagingLimits(bytesLimit, rowLimit, perPartitionLimit, isDistinct, lastKey, lastRemaining);
                 }
                 case CQL_GROUP_BY_LIMIT:
                 case CQL_GROUP_BY_PAGING_LIMIT:
@@ -1201,6 +1496,7 @@ public DataLimits deserialize(DataInputPlus in, int version, TableMetadata metad
                     int groupLimit = in.readUnsignedVInt32();
                     int groupPerPartitionLimit = in.readUnsignedVInt32();
                     int rowLimit = in.readUnsignedVInt32();
+                    int bytesLimit = version >= MessagingService.VERSION_DS_10 ? (int) in.readUnsignedVInt() : NO_LIMIT;
 
                     AggregationSpecification groupBySpec = AggregationSpecification.serializer.deserialize(in, version, metadata);
 
@@ -1209,6 +1505,7 @@ public DataLimits deserialize(DataInputPlus in, int version, TableMetadata metad
                     if (kind == Kind.CQL_GROUP_BY_LIMIT)
                         return new CQLGroupByLimits(groupLimit,
                                                     groupPerPartitionLimit,
+                                                    bytesLimit,
                                                     rowLimit,
                                                     groupBySpec,
                                                     state);
@@ -1217,6 +1514,7 @@ public DataLimits deserialize(DataInputPlus in, int version, TableMetadata metad
                     int lastRemaining = in.readUnsignedVInt32();
                     return new CQLGroupByPagingLimits(groupLimit,
                                                       groupPerPartitionLimit,
+                                                      bytesLimit,
                                                       rowLimit,
                                                       groupBySpec,
                                                       state,
@@ -1237,6 +1535,8 @@ public long serializedSize(DataLimits limits, int version, ClusteringComparator
                     CQLLimits cqlLimits = (CQLLimits) limits;
                     size += TypeSizes.sizeofUnsignedVInt(cqlLimits.rowLimit);
                     size += TypeSizes.sizeofUnsignedVInt(cqlLimits.perPartitionLimit);
+                    if (version >= MessagingService.VERSION_DS_10)
+                        size += TypeSizes.sizeofUnsignedVInt(cqlLimits.bytesLimit);
                     size += TypeSizes.sizeof(cqlLimits.isDistinct);
                     if (limits.kind() == Kind.CQL_PAGING_LIMIT)
                     {
@@ -1251,6 +1551,8 @@ public long serializedSize(DataLimits limits, int version, ClusteringComparator
                     size += TypeSizes.sizeofUnsignedVInt(groupByLimits.groupLimit);
                     size += TypeSizes.sizeofUnsignedVInt(groupByLimits.groupPerPartitionLimit);
                     size += TypeSizes.sizeofUnsignedVInt(groupByLimits.rowLimit);
+                    if (version >= MessagingService.VERSION_DS_10)
+                        size += TypeSizes.sizeofUnsignedVInt(groupByLimits.bytesLimit);
 
                     AggregationSpecification groupBySpec = groupByLimits.groupBySpec;
                     size += AggregationSpecification.serializer.serializedSize(groupBySpec, version);
diff --git a/src/java/org/apache/cassandra/db/filter/RowFilter.java b/src/java/org/apache/cassandra/db/filter/RowFilter.java
index beb3edfcd792..38329f5e2723 100644
--- a/src/java/org/apache/cassandra/db/filter/RowFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java
@@ -21,11 +21,15 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
 
 import com.google.common.base.Objects;
 import org.slf4j.Logger;
@@ -34,23 +38,28 @@
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.restrictions.ExternalRestriction;
+import org.apache.cassandra.cql3.restrictions.Restrictions;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.cql3.statements.SelectOptions;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionPurger;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.context.CounterContext;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.db.marshal.CollectionType;
 import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.FloatType;
 import org.apache.cassandra.db.marshal.ListType;
 import org.apache.cassandra.db.marshal.LongType;
 import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.BaseRowIterator;
@@ -62,14 +71,20 @@
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.index.sai.utils.GeoUtil;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.lucene.util.SloppyMath;
 
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkBindValueSet;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
@@ -83,70 +98,79 @@
  * be handled by a 2ndary index, and the rest is simply filtered out from the
  * result set (the later can only happen if the query was using ALLOW FILTERING).
  */
-public class RowFilter implements Iterable<RowFilter.Expression>
+public class RowFilter
 {
     private static final Logger logger = LoggerFactory.getLogger(RowFilter.class);
 
     public static final Serializer serializer = new Serializer();
-    private static final RowFilter NONE = new RowFilter(Collections.emptyList(), false);
+    public static final RowFilter NONE = new RowFilter(FilterElement.NONE, false);
 
-    protected final List<Expression> expressions;
+    private final FilterElement root;
 
     private final boolean needsReconciliation;
 
-    protected RowFilter(List<Expression> expressions, boolean needsReconciliation)
+    protected RowFilter(FilterElement root, boolean needsReconciliation)
     {
-        this.expressions = expressions;
+        this.root = root;
         this.needsReconciliation = needsReconciliation;
     }
 
-    /**
-     * 
-     * @param needsReconciliation whether or not this filter belongs to a read that requires coordinator reconciliation 
-     * 
-     * @return a new {@link RowFilter} with an empty {@link Expression} list
-     */
-    public static RowFilter create(boolean needsReconciliation)
-    {
-        return new RowFilter(new ArrayList<>(), needsReconciliation);
-    }
-
     public static RowFilter none()
     {
         return NONE;
     }
 
-    public SimpleExpression add(ColumnMetadata def, Operator op, ByteBuffer value)
+    public FilterElement root()
     {
-        SimpleExpression expression = new SimpleExpression(def, op, value);
-        add(expression);
-        return expression;
+        return root;
     }
 
-    public void addMapEquality(ColumnMetadata def, ByteBuffer key, Operator op, ByteBuffer value)
+    /**
+     * @return all the expressions in this filter expression tree by traversing it in pre-order
+     */
+    public List<Expression> expressions()
     {
-        add(new MapEqualityExpression(def, key, op, value));
+        return root.traversedExpressions();
     }
 
-    public void addCustomIndexExpression(TableMetadata metadata, IndexMetadata targetIndex, ByteBuffer value)
+    /**
+     * @return {@code true} if this filter contains any expression with an ANN operator, {@code false} otherwise.
+     */
+    public boolean hasANN()
     {
-        add(new CustomExpression(metadata, targetIndex, value));
+        for (Expression expression : root.expressions()) // ANN expressions are always on the first tree level
+        {
+            if (expression.operator == Operator.ANN)
+                return true;
+        }
+        return false;
     }
 
-    private void add(Expression expression)
+    /**
+     * @return the {@link ANNOptions} of the ANN expression in this filter, or {@link ANNOptions#NONE} if there is
+     * no ANN expression.
+     */
+    public ANNOptions annOptions()
     {
-        expression.validate();
-        expressions.add(expression);
+        for (Expression expression : root.expressions()) // ANN expressions are always on the first tree level
+        {
+            if (expression.operator == Operator.ANN)
+                return expression.annOptions();
+        }
+        return ANNOptions.NONE;
     }
 
-    public List<Expression> getExpressions()
+    /**
+     * @return {@code true} if this filter contains any disjunction, {@code false} otherwise.
+     */
+    public boolean containsDisjunctions()
     {
-        return expressions;
+        return root.containsDisjunctions();
     }
 
     /**
      * @return true if this filter belongs to a read that requires reconciliation at the coordinator
-     * @see StatementRestrictions#getRowFilter(IndexRegistry, QueryOptions)
+     * @see StatementRestrictions#getRowFilter(IndexRegistry, QueryOptions, ClientState, SelectOptions)
      */
     public boolean needsReconciliation()
     {
@@ -173,7 +197,7 @@ public boolean isStrict()
      */
     public boolean isMutableIntersection()
     {
-        return expressions.stream().filter(e -> !e.column.isPrimaryKeyColumn()).count() > 1;
+        return expressions().stream().filter(e -> !e.column.isPrimaryKeyColumn()).count() > 1;
     }
 
     /**
@@ -182,7 +206,7 @@ public boolean isMutableIntersection()
      */
     public boolean hasExpressionOnClusteringOrRegularColumns()
     {
-        for (Expression expression : expressions)
+        for (Expression expression : expressions())
         {
             ColumnMetadata column = expression.column();
             if (column.isClusteringColumn() || column.isRegular())
@@ -191,28 +215,12 @@ public boolean hasExpressionOnClusteringOrRegularColumns()
         return false;
     }
 
-    /**
-     * Note that the application of this transformation does not yet take {@link #isStrict()} into account. This means
-     * that even when strict filtering is not safe, expressions will be applied as intersections rather than unions.
-     * The filter will always be evaluated strictly in conjunction with replica filtering protection at the 
-     * coordinator, however, even after CASSANDRA-19007 is addressed.
-     * 
-     * @see <a href="https://issues.apache.org/jira/browse/CASSANDRA-190007">CASSANDRA-19007</a>
-     */
     protected Transformation<BaseRowIterator<?>> filter(TableMetadata metadata, long nowInSec)
     {
-        List<Expression> partitionLevelExpressions = new ArrayList<>();
-        List<Expression> rowLevelExpressions = new ArrayList<>();
-        for (Expression e: expressions)
-        {
-            if (e.column.isStatic() || e.column.isPartitionKey())
-                partitionLevelExpressions.add(e);
-            else
-                rowLevelExpressions.add(e);
-        }
+        FilterElement partitionLevelOperation = root.partitionLevelTree();
+        FilterElement rowLevelOperation = root.rowLevelTree();
 
-        long numberOfRegularColumnExpressions = rowLevelExpressions.size();
-        final boolean filterNonStaticColumns = numberOfRegularColumnExpressions > 0;
+        final boolean filterNonStaticColumns = rowLevelOperation.size() > 0;
 
         return new Transformation<>()
         {
@@ -224,12 +232,11 @@ protected BaseRowIterator<?> applyToPartition(BaseRowIterator<?> partition)
                 pk = partition.partitionKey();
 
                 // Short-circuit all partitions that won't match based on static and partition keys
-                for (Expression e : partitionLevelExpressions)
-                    if (!e.isSatisfiedBy(metadata, partition.partitionKey(), partition.staticRow()))
-                    {
-                        partition.close();
-                        return null;
-                    }
+                if (!partitionLevelOperation.isSatisfiedBy(metadata, partition.partitionKey(), partition.staticRow()))
+                {
+                    partition.close();
+                    return null;
+                }
 
                 BaseRowIterator<?> iterator = partition instanceof UnfilteredRowIterator
                                               ? Transformation.apply((UnfilteredRowIterator) partition, this)
@@ -251,9 +258,8 @@ public Row applyToRow(Row row)
                 if (purged == null)
                     return null;
 
-                for (Expression e : rowLevelExpressions)
-                    if (!e.isSatisfiedBy(metadata, pk, purged))
-                        return null;
+                if (!rowLevelOperation.isSatisfiedBy(metadata, pk, purged))
+                    return null;
 
                 return row;
             }
@@ -270,7 +276,7 @@ public Row applyToRow(Row row)
      */
     public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, long nowInSec)
     {
-        return expressions.isEmpty() ? iter : Transformation.apply(iter, filter(iter.metadata(), nowInSec));
+        return root.isEmpty() ? iter : Transformation.apply(iter, filter(iter.metadata(), nowInSec));
     }
 
     /**
@@ -283,7 +289,7 @@ public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, long
      */
     public PartitionIterator filter(PartitionIterator iter, TableMetadata metadata, long nowInSec)
     {
-        return expressions.isEmpty() ? iter : Transformation.apply(iter, filter(metadata, nowInSec));
+        return root.isEmpty() ? iter : Transformation.apply(iter, filter(metadata, nowInSec));
     }
 
     /**
@@ -300,23 +306,18 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey,
         // We purge all tombstones as the expressions isSatisfiedBy methods expects it
         Row purged = row.purge(DeletionPurger.PURGE_ALL, nowInSec, metadata.enforceStrictLiveness());
         if (purged == null)
-            return expressions.isEmpty();
+            return root.isEmpty();
 
-        for (Expression e : expressions)
-        {
-            if (!e.isSatisfiedBy(metadata, partitionKey, purged))
-                return false;
-        }
-        return true;
+        return root.isSatisfiedBy(metadata, partitionKey, purged);
     }
 
     /**
-     * Returns true if all of the expressions within this filter that apply to the partition key are satisfied by
+     * Returns true if all the expressions within this filter that apply to the partition key are satisfied by
      * the given key, false otherwise.
      */
     public boolean partitionKeyRestrictionsAreSatisfiedBy(DecoratedKey key, AbstractType<?> keyValidator)
     {
-        for (Expression e : expressions)
+        for (Expression e : expressions())
         {
             if (!e.column.isPartitionKey())
                 continue;
@@ -324,27 +325,25 @@ public boolean partitionKeyRestrictionsAreSatisfiedBy(DecoratedKey key, Abstract
             ByteBuffer value = keyValidator instanceof CompositeType
                              ? ((CompositeType) keyValidator).split(key.getKey())[e.column.position()]
                              : key.getKey();
-            if (!e.operator().isSatisfiedBy(e.column.type, value, e.value))
+            if (!e.operator().isSatisfiedBy(e.column.type, value, e.value, e.indexAnalyzer(), e.queryAnalyzer()))
                 return false;
         }
         return true;
     }
 
     /**
-     * Returns true if all of the expressions within this filter that apply to the clustering key are satisfied by
+     * Returns true if all the expressions within this filter that apply to the clustering key are satisfied by
      * the given Clustering, false otherwise.
      */
     public boolean clusteringKeyRestrictionsAreSatisfiedBy(Clustering<?> clustering)
     {
-        for (Expression e : expressions)
+        for (Expression e : expressions())
         {
             if (!e.column.isClusteringColumn())
                 continue;
 
-            if (!e.operator().isSatisfiedBy(e.column.type, clustering.bufferAt(e.column.position()), e.value))
-            {
+            if (!e.operator().isSatisfiedBy(e.column.type, clustering.bufferAt(e.column.position()), e.value, e.indexAnalyzer(), e.queryAnalyzer()))
                 return false;
-            }
         }
         return true;
     }
@@ -355,16 +354,11 @@ public boolean clusteringKeyRestrictionsAreSatisfiedBy(Clustering<?> clustering)
      */
     public RowFilter without(Expression expression)
     {
-        assert expressions.contains(expression);
-        if (expressions.size() == 1)
+        assert root.contains(expression);
+        if (root.size() == 1)
             return RowFilter.none();
 
-        List<Expression> newExpressions = new ArrayList<>(expressions.size() - 1);
-        for (Expression e : expressions)
-            if (!e.equals(expression))
-                newExpressions.add(e);
-
-        return withNewExpressions(newExpressions);
+        return new RowFilter(root.filter(e -> !e.equals(expression)), needsReconciliation);
     }
 
     /**
@@ -376,32 +370,30 @@ public RowFilter without(ColumnMetadata column, Operator op, ByteBuffer value)
         if (isEmpty())
             return this;
 
-        List<Expression> newExpressions = new ArrayList<>(expressions.size() - 1);
-        for (Expression e : expressions)
-            if (!e.column().equals(column) || e.operator() != op || !e.value.equals(value))
-                newExpressions.add(e);
-
-        return withNewExpressions(newExpressions);
+        return new RowFilter(root.filter(e -> !e.column().equals(column) || e.operator() != op || !e.value.equals(value)), needsReconciliation);
     }
 
     public RowFilter withoutExpressions()
     {
-        return withNewExpressions(Collections.emptyList());
+        return new RowFilter(root.filter(e -> false), needsReconciliation);
     }
 
-    protected RowFilter withNewExpressions(List<Expression> expressions)
+    /**
+     * @return this filter pruning all its disjunction branches
+     */
+    public RowFilter withoutDisjunctions()
     {
-        return new RowFilter(expressions, needsReconciliation);
+        return new RowFilter(root.withoutDisjunctions(), needsReconciliation);
     }
 
-    public boolean isEmpty()
+    public RowFilter restrict(Predicate<Expression> filter)
     {
-        return expressions.isEmpty();
+        return new RowFilter(root.filter(filter), needsReconciliation);
     }
 
-    public Iterator<Expression> iterator()
+    public boolean isEmpty()
     {
-        return expressions.iterator();
+        return root.isEmpty();
     }
 
     @Override
@@ -422,27 +414,528 @@ public String toCQLString()
 
     private String toString(boolean cql)
     {
-        StringBuilder sb = new StringBuilder();
-        for (int i = 0; i < expressions.size(); i++)
+        return root.toString(cql);
+    }
+
+    public static Builder builder(boolean needsReconciliation)
+    {
+        return new Builder(needsReconciliation, null);
+    }
+
+    public static Builder builder(boolean needsReconciliation, IndexRegistry indexRegistry)
+    {
+        return new Builder(needsReconciliation, indexRegistry);
+    }
+
+    public static class Builder
+    {
+        private FilterElement.Builder current = new FilterElement.Builder(false);
+        boolean needsReconciliation = false;
+
+        public Builder(boolean needsReconciliation, IndexRegistry indexRegistry)
+        {
+            this.needsReconciliation = needsReconciliation;
+            this.indexRegistry = indexRegistry;
+        }
+
+        private final IndexRegistry indexRegistry;
+
+        public RowFilter build()
+        {
+            return new RowFilter(current.build(), needsReconciliation);
+        }
+
+        public RowFilter buildFromRestrictions(StatementRestrictions restrictions,
+                                               TableMetadata table,
+                                               QueryOptions options,
+                                               ClientState state,
+                                               ANNOptions annOptions)
+        {
+            FilterElement root = doBuild(restrictions, table, options, annOptions);
+
+            if (Guardrails.queryFilters.enabled(state))
+                Guardrails.queryFilters.guard(root.numFilteredValues(), "Select query", false, state);
+
+            return new RowFilter(root, needsReconciliation);
+        }
+
+        private FilterElement doBuild(StatementRestrictions restrictions,
+                                      TableMetadata table,
+                                      QueryOptions options,
+                                      ANNOptions annOptions)
+        {
+            FilterElement.Builder element = new FilterElement.Builder(restrictions.isDisjunction());
+            this.current = element;
+
+            for (Restrictions restrictionSet : restrictions.filterRestrictions().getRestrictions())
+                restrictionSet.addToRowFilter(this, indexRegistry, options, annOptions);
+
+            for (ExternalRestriction expression : restrictions.filterRestrictions().getExternalExpressions())
+                addAllAsConjunction(b -> expression.addToRowFilter(b, table, options));
+
+            for (StatementRestrictions child : restrictions.children())
+                element.children.add(doBuild(child, table, options, annOptions));
+
+            // Optimize out any conjunctions / disjunctions with TRUE.
+            // This is not needed for correctness.
+            if (restrictions.isDisjunction())
+            {
+                // `OR TRUE` swallows all other restrictions in disjunctions.
+                // Therefore, replace this node with an always true element.
+                if (element.children.stream().anyMatch(FilterElement::isAlwaysTrue))
+                    element = new FilterElement.Builder(false);
+            }
+            else
+            {
+                // `AND TRUE` does nothing in conjunctions, so remove it.
+                element.children.removeIf(FilterElement::isAlwaysTrue);
+            }
+
+            return element.build();
+        }
+
+        /**
+         * Adds multiple filter expressions to this {@link RowFilter.Builder} and joins them with AND (conjunction),
+         * regardless of the current mode (conjunction / disjunction) of the {@link RowFilter.Builder}.
+         * <p>
+         *
+         * This wrapper method makes sure we pass a {@code RowFilter.Builder} that is always in conjunction mode to the
+         * respective {@code addToRowFilterDelegate} method. If multiple expressions are added to the row filter, this 
+         * method makes sure they are joined with AND in their own {@link FilterElement}.
+         *
+         * @param addToRowFilterDelegate a function that adds expressions / child filter elements
+         *                               to a provided {@link RowFilter.Builder}, and expects all
+         *                               added expressions to be joined with AND operator
+         */
+        public void addAllAsConjunction(Consumer<Builder> addToRowFilterDelegate)
+        {
+            if (current.isDisjunction)
+            {
+                // If we're in disjunction mode, we must not pass the current builder to addToRowFilter.
+                // We create a new conjunction sub-builder instead and add all expressions there.
+                var builder = new Builder(needsReconciliation, indexRegistry);
+                addToRowFilterDelegate.accept(builder);
+
+                if (builder.current.expressions.size() == 1 && builder.current.children.isEmpty())
+                {
+                    // Optimization:
+                    // if there is one expression, we can just add it directly to the current FilterElement
+                    // making the result tree flatter
+                    current.expressions.add(builder.current.expressions.get(0));
+                }
+                else if (builder.current.children.size() == 1 && builder.current.expressions.isEmpty())
+                {
+                    // Optimization:
+                    // if there is one child, we can just add it directly to the current FilterElement,
+                    // making the result tree flatter
+                    current.children.add(builder.current.children.get(0));
+                }
+                else
+                {
+                    // More expressions means we have to create a new child node (AND) for them.
+                    // Also note that we use this for adding zero expressions/children as well.
+                    // A conjunction with no restrictions means selecting everything, so if we didn't add an empty
+                    // AND node in such case, we could end up with a filter that misses to match some rows.
+                    current.children.add(builder.current.build());
+                }
+            }
+            else
+            {
+                // Just an optimisation. If we're already in the conjunction mode, we don't need to create
+                // a sub-builder; we can just use this one to collect the expressions.
+                addToRowFilterDelegate.accept(this);
+            }
+        }
+
+        /**
+         * Adds the specified simple filter expression to this builder.
+         *
+         * @param def the filtered column
+         * @param op the filtering operator, shouldn't be {@link Operator#ANN}.
+         * @param value the filtered value
+         * @return the added expression
+         */
+        public SimpleExpression add(ColumnMetadata def, Operator op, ByteBuffer value)
+        {
+            assert op != Operator.ANN : "ANN expressions should be added with the addANNExpression method";
+            SimpleExpression expression = new SimpleExpression(def, op, value, indexAnalyzer(def, op), queryAnalyzer(def, op), null);
+            add(expression);
+            return expression;
+        }
+
+        /**
+         * Adds the specified ANN expression to this builder.
+         *
+         * @param def the column for ANN ordering
+         * @param value the value for ANN ordering
+         * @param annOptions the ANN options
+         */
+        public void addANNExpression(ColumnMetadata def, ByteBuffer value, ANNOptions annOptions)
+        {
+            add(new SimpleExpression(def, Operator.ANN, value, null, null, annOptions));
+        }
+
+        public void addMapComparison(ColumnMetadata def, ByteBuffer key, Operator op, ByteBuffer value)
+        {
+            add(new MapComparisonExpression(def, key, op, value, indexAnalyzer(def, op), queryAnalyzer(def, op)));
+        }
+
+        @Nullable
+        private Index.Analyzer indexAnalyzer(ColumnMetadata def, Operator op)
+        {
+            return indexRegistry == null ? null : indexRegistry.getIndexAnalyzerFor(def, op).orElse(null);
+        }
+
+        @Nullable
+        private Index.Analyzer queryAnalyzer(ColumnMetadata def, Operator op)
+        {
+            return indexRegistry == null ? null : indexRegistry.getQueryAnalyzerFor(def, op).orElse(null);
+        }
+
+        public void addGeoDistanceExpression(ColumnMetadata def, ByteBuffer point, Operator op, ByteBuffer distance)
+        {
+            var primaryGeoDistanceExpression = new GeoDistanceExpression(def, point, op, distance);
+            // The following logic optionally adds a second search expression in the event that the query area
+            // crosses then antimeridian.
+            if (primaryGeoDistanceExpression.crossesAntimeridian())
+            {
+                // The primry GeoDistanceExpression includes points on/over the antimeridian. Since we search
+                // using the lat/lon coordinates, we must create a shifted expression that will collect
+                // results on the other side of the antimeridian.
+                var shiftedGeoDistanceExpression = primaryGeoDistanceExpression.buildShiftedExpression();
+                if (current.isDisjunction)
+                {
+                    // We can add both expressions to this level of the tree because it is a disjunction.
+                    add(primaryGeoDistanceExpression);
+                    add(shiftedGeoDistanceExpression);
+                }
+                else
+                {
+                    // We need to add a new level to the tree so that we can get all results that match the primary
+                    // or the shifted expressions.
+                    var builder = new FilterElement.Builder(true);
+                    primaryGeoDistanceExpression.validate();
+                    shiftedGeoDistanceExpression.validate();
+                    builder.expressions.add(primaryGeoDistanceExpression);
+                    builder.expressions.add(shiftedGeoDistanceExpression);
+                    current.children.add(builder.build());
+                }
+            }
+            else
+            {
+                add(primaryGeoDistanceExpression);
+            }
+        }
+
+        public void addCustomIndexExpression(TableMetadata metadata, IndexMetadata targetIndex, ByteBuffer value)
+        {
+            add(CustomExpression.build(metadata, targetIndex, value));
+        }
+
+        public Builder add(Expression expression)
+        {
+            expression.validate();
+            current.expressions.add(expression);
+            return this;
+        }
+
+        public void addUserExpression(UserExpression e)
+        {
+            current.expressions.add(e);
+        }
+    }
+
+    public static class FilterElement
+    {
+        public static final Serializer serializer = new Serializer();
+
+        public static final FilterElement NONE = new FilterElement(false, Collections.emptyList(), Collections.emptyList());
+
+        private final boolean isDisjunction;
+
+        private final List<Expression> expressions;
+
+        private final List<FilterElement> children;
+
+        public FilterElement(boolean isDisjunction, List<Expression> expressions, List<FilterElement> children)
+        {
+            this.isDisjunction = isDisjunction;
+            this.expressions = expressions;
+            this.children = children;
+        }
+
+        public boolean isDisjunction()
+        {
+            return isDisjunction;
+        }
+
+        private boolean containsDisjunctions()
+        {
+            if (isDisjunction)
+                return true;
+
+            for (FilterElement child : children)
+                if (child.containsDisjunctions())
+                    return true;
+
+            return false;
+        }
+
+        public List<Expression> expressions()
+        {
+            return expressions;
+        }
+
+        private List<Expression> traversedExpressions()
+        {
+            List<Expression> allExpressions = new ArrayList<>(expressions);
+            for (FilterElement child : children)
+                allExpressions.addAll(child.traversedExpressions());
+            return allExpressions;
+        }
+
+        private FilterElement withoutDisjunctions()
+        {
+            if (isDisjunction)
+                return NONE;
+
+            FilterElement.Builder builder = new Builder(false);
+            builder.expressions.addAll(expressions);
+
+            for (FilterElement child : children)
+            {
+                if (!child.isDisjunction)
+                    builder.children.add(child);
+            }
+
+            return builder.build();
+        }
+
+        public FilterElement filter(Predicate<Expression> filter)
+        {
+            FilterElement.Builder builder = new Builder(isDisjunction);
+
+            expressions.stream().filter(filter).forEach(builder.expressions::add);
+
+            children.stream().map(c -> c.filter(filter)).forEach(builder.children::add);
+
+            return builder.build();
+        }
+
+        public List<FilterElement> children()
+        {
+            return children;
+        }
+
+        public boolean isEmpty()
+        {
+            return expressions.isEmpty() && children.isEmpty();
+        }
+
+        public boolean isAlwaysTrue()
+        {
+            return !isDisjunction && isEmpty();
+        }
+
+        public boolean contains(Expression expression)
+        {
+            return expressions.contains(expression) || children.stream().anyMatch(c -> contains(expression));
+        }
+
+        public FilterElement partitionLevelTree()
+        {
+            return new FilterElement(isDisjunction,
+                                     expressions.stream()
+                                                .filter(e -> e.column.isStatic() || e.column.isPartitionKey())
+                                                .collect(Collectors.toList()),
+                                     children.stream()
+                                             .map(FilterElement::partitionLevelTree)
+                                             .collect(Collectors.toList()));
+        }
+
+        public FilterElement rowLevelTree()
+        {
+            return new FilterElement(isDisjunction,
+                                     expressions.stream()
+                                                .filter(e -> !e.column.isStatic() && !e.column.isPartitionKey())
+                                                .collect(Collectors.toList()),
+                                     children.stream()
+                                             .map(FilterElement::rowLevelTree)
+                                             .collect(Collectors.toList()));
+        }
+
+        public int size()
+        {
+            return expressions.size() + children.stream().mapToInt(FilterElement::size).sum();
+        }
+
+        public boolean isSatisfiedBy(TableMetadata table, DecoratedKey key, Row row)
+        {
+            if (isEmpty())
+                return true;
+            if (isDisjunction)
+            {
+                for (Expression e : expressions)
+                    if (e.isSatisfiedBy(table, key, row))
+                        return true;
+                for (FilterElement child : children)
+                    if (child.isSatisfiedBy(table, key, row))
+                        return true;
+                return false;
+            }
+            else
+            {
+                for (Expression e : expressions)
+                    if (!e.isSatisfiedBy(table, key, row))
+                        return false;
+                for (FilterElement child : children)
+                    if (!child.isSatisfiedBy(table, key, row))
+                        return false;
+                return true;
+            }
+        }
+
+        /**
+         * Returns the number of values that this filter will filter out after applying any index analyzers.
+         */
+        private int numFilteredValues()
+        {
+            int result = 0;
+
+            for (Expression expression : expressions)
+                result += expression.numFilteredValues();
+
+            for (FilterElement child : children)
+                result += child.numFilteredValues();
+
+            return result;
+        }
+
+        public String toString(boolean cql)
+        {
+            StringBuilder sb = new StringBuilder();
+            for (int i = 0; i < expressions.size(); i++)
+            {
+                if (sb.length() > 0)
+                    sb.append(isDisjunction ? " OR " : " AND ");
+                sb.append(expressions.get(i).toString(cql));
+            }
+            for (int i = 0; i < children.size(); i++)
+            {
+                if (sb.length() > 0)
+                    sb.append(isDisjunction ? " OR " : " AND ");
+                sb.append('(');
+                sb.append(children.get(i));
+                sb.append(')');
+            }
+            return sb.toString();
+        }
+
+        public static class Builder
+        {
+            private final boolean isDisjunction;
+            private final List<Expression> expressions = new ArrayList<>();
+            private final List<FilterElement> children = new ArrayList<>();
+
+            public Builder(boolean isDisjunction)
+            {
+                this.isDisjunction = isDisjunction;
+            }
+
+            public FilterElement build()
+            {
+                return new FilterElement(isDisjunction, expressions, children);
+            }
+        }
+
+        public static class Serializer
         {
-            if (i > 0)
-                sb.append(" AND ");
-            sb.append(expressions.get(i).toString(cql));
+            public void serialize(FilterElement operation, DataOutputPlus out, int version) throws IOException
+            {
+                assert (!operation.isDisjunction && operation.children().isEmpty()) || version >= MessagingService.VERSION_DS_10 :
+                "Attempting to serialize a disjunct row filter to a node that doesn't support disjunction";
+
+                out.writeUnsignedVInt32(operation.expressions.size());
+                for (Expression expr : operation.expressions)
+                    Expression.serializer.serialize(expr, out, version);
+
+                if (version < MessagingService.VERSION_DS_10)
+                    return;
+
+                out.writeBoolean(operation.isDisjunction);
+                out.writeUnsignedVInt32(operation.children.size());
+                for (FilterElement child : operation.children)
+                    serialize(child, out, version);
+            }
+
+            public FilterElement deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
+            {
+                int size = in.readUnsignedVInt32();
+                List<Expression> expressions = new ArrayList<>(size);
+                for (int i = 0; i < size; i++)
+                    expressions.add(Expression.serializer.deserialize(in, version, metadata));
+
+                if (version < MessagingService.VERSION_DS_10)
+                    return new FilterElement(false, expressions, Collections.emptyList());
+
+                boolean isDisjunction = in.readBoolean();
+                size = in.readUnsignedVInt32();
+                List<FilterElement> children = new ArrayList<>(size);
+                for (int i  = 0; i < size; i++)
+                    children.add(deserialize(in, version, metadata));
+                return new FilterElement(isDisjunction, expressions, children);
+            }
+
+            public long serializedSize(FilterElement operation, int version)
+            {
+                long size = TypeSizes.sizeofUnsignedVInt(operation.expressions.size());
+                for (Expression expr : operation.expressions)
+                    size += Expression.serializer.serializedSize(expr, version);
+
+                if (version < MessagingService.VERSION_DS_10)
+                    return size;
+
+                size++; // isDisjunction boolean
+                size += TypeSizes.sizeofUnsignedVInt(operation.children.size());
+                for (FilterElement child : operation.children)
+                    size += serializedSize(child, version);
+                return size;
+            }
         }
-        return sb.toString();
     }
 
     public static abstract class Expression
     {
-        private static final Serializer serializer = new Serializer();
+        public static final Serializer serializer = new Serializer();
 
-        // Note: the order of this enum matter, it's used for serialization,
+        // Note: the val of this enum is used for serialization,
         // and this is why we have some UNUSEDX for values we don't use anymore
         // (we could clean those on a major protocol update, but it's not worth
         // the trouble for now)
-        protected enum Kind { SIMPLE, MAP_EQUALITY, UNUSED1, CUSTOM, USER }
+        // VECTOR
+        protected enum Kind
+        {
+            SIMPLE(0), MAP_COMPARISON(1), UNUSED1(2), CUSTOM(3), USER(4), VECTOR_RADIUS(100);
+            private final int val;
+            Kind(int v) { val = v; }
+            public int getVal() { return val; }
+            public static Kind fromVal(int val)
+            {
+                switch (val)
+                {
+                    case 0: return SIMPLE;
+                    case 1: return MAP_COMPARISON;
+                    case 2: return UNUSED1;
+                    case 3: return CUSTOM;
+                    case 4: return USER;
+                    case 100: return VECTOR_RADIUS;
+                    default: throw new IllegalArgumentException("Unknown index expression kind: " + val);
+                }
+            }
+        }
 
         protected abstract Kind kind();
+
         protected final ColumnMetadata column;
         protected final Operator operator;
         protected final ByteBuffer value;
@@ -474,6 +967,24 @@ public Operator operator()
             return operator;
         }
 
+        @Nullable
+        public Index.Analyzer indexAnalyzer()
+        {
+            return null;
+        }
+
+        @Nullable
+        public Index.Analyzer queryAnalyzer()
+        {
+            return null;
+        }
+
+        @Nullable
+        public ANNOptions annOptions()
+        {
+            return null;
+        }
+
         /**
          * Checks if the operator of this <code>IndexExpression</code> is a <code>CONTAINS</code> operator.
          *
@@ -522,8 +1033,7 @@ public void validateForIndexing()
         /**
          * Returns whether the provided row satisfied this expression or not.
          *
-         *
-         * @param metadata
+         * @param metadata the metadata of the queried table
          * @param partitionKey the partition key for row to check.
          * @param row the row to check. It should *not* contain deleted cells
          * (i.e. it should come from a RowIterator).
@@ -531,6 +1041,14 @@ public void validateForIndexing()
          */
         public abstract boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row);
 
+        /**
+         * Returns the number of values that this expression will check after applying any index analyzers.
+         */
+        protected int numFilteredValues()
+        {
+            return 1;
+        }
+
         protected ByteBuffer getValue(TableMetadata metadata, DecoratedKey partitionKey, Row row)
         {
             switch (column.kind)
@@ -558,9 +1076,9 @@ public boolean equals(Object o)
 
             Expression that = (Expression)o;
 
-            return Objects.equal(this.kind(), that.kind())
+            return this.kind() == that.kind()
+                && this.operator == that.operator
                 && Objects.equal(this.column.name, that.column.name)
-                && Objects.equal(this.operator, that.operator)
                 && Objects.equal(this.value, that.value);
         }
 
@@ -586,13 +1104,13 @@ public String toCQLString()
             return toString(true);
         }
 
-        protected abstract String toString(boolean cql);
+        public abstract String toString(boolean cql);
 
-        private static class Serializer
+        public static class Serializer
         {
             public void serialize(Expression expression, DataOutputPlus out, int version) throws IOException
             {
-                out.writeByte(expression.kind().ordinal());
+                out.writeByte(expression.kind().getVal());
 
                 // Custom expressions include neither a column or operator, but all
                 // other expressions do.
@@ -616,18 +1134,26 @@ public void serialize(Expression expression, DataOutputPlus out, int version) th
                 {
                     case SIMPLE:
                         ByteBufferUtil.writeWithShortLength(expression.value, out);
+                        if (expression.operator == Operator.ANN)
+                            ANNOptions.serializer.serialize(expression.annOptions(), out, version);
                         break;
-                    case MAP_EQUALITY:
-                        MapEqualityExpression mexpr = (MapEqualityExpression)expression;
+                    case MAP_COMPARISON:
+                        MapComparisonExpression mexpr = (MapComparisonExpression)expression;
                         ByteBufferUtil.writeWithShortLength(mexpr.key, out);
                         ByteBufferUtil.writeWithShortLength(mexpr.value, out);
                         break;
+                    case VECTOR_RADIUS:
+                        GeoDistanceExpression gexpr = (GeoDistanceExpression) expression;
+                        gexpr.distanceOperator.writeTo(out);
+                        ByteBufferUtil.writeWithShortLength(gexpr.distance, out);
+                        ByteBufferUtil.writeWithShortLength(gexpr.value, out);
+                        break;
                 }
             }
 
             public Expression deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
             {
-                Kind kind = Kind.values()[in.readByte()];
+                Kind kind = Kind.fromVal(in.readByte());
 
                 // custom expressions (3.0+ only) do not contain a column or operator, only a value
                 if (kind == Kind.CUSTOM)
@@ -643,6 +1169,9 @@ public Expression deserialize(DataInputPlus in, int version, TableMetadata metad
                 ByteBuffer name = ByteBufferUtil.readWithShortLength(in);
                 Operator operator = Operator.readFrom(in);
                 ColumnMetadata column = metadata.getColumn(name);
+                IndexRegistry indexRegistry = IndexRegistry.obtain(metadata);
+                Index.Analyzer indexAnalyzer = indexRegistry.getIndexAnalyzerFor(column, operator).orElse(null);
+                Index.Analyzer queryAnalyzer = indexRegistry.getQueryAnalyzerFor(column, operator).orElse(null);
 
                 // Compact storage tables, when used with thrift, used to allow falling through this withouot throwing an
                 // exception. However, since thrift was removed in 4.0, this behaviour was not restored in CASSANDRA-16217
@@ -652,11 +1181,18 @@ public Expression deserialize(DataInputPlus in, int version, TableMetadata metad
                 switch (kind)
                 {
                     case SIMPLE:
-                        return new SimpleExpression(column, operator, ByteBufferUtil.readWithShortLength(in));
-                    case MAP_EQUALITY:
-                        ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
                         ByteBuffer value = ByteBufferUtil.readWithShortLength(in);
-                        return new MapEqualityExpression(column, key, operator, value);
+                        ANNOptions annOptions = operator == Operator.ANN ? ANNOptions.serializer.deserialize(in, version) : null;
+                        return new SimpleExpression(column, operator, value, indexAnalyzer, queryAnalyzer, annOptions);
+                    case MAP_COMPARISON:
+                        ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
+                        ByteBuffer val = ByteBufferUtil.readWithShortLength(in);
+                        return new MapComparisonExpression(column, key, operator, val, indexAnalyzer, queryAnalyzer);
+                    case VECTOR_RADIUS:
+                        Operator boundaryOperator = Operator.readFrom(in);
+                        ByteBuffer distance = ByteBufferUtil.readWithShortLength(in);
+                        ByteBuffer searchVector = ByteBufferUtil.readWithShortLength(in);
+                        return new GeoDistanceExpression(column, searchVector, boundaryOperator, distance);
                 }
                 throw new AssertionError();
             }
@@ -674,10 +1210,12 @@ public long serializedSize(Expression expression, int version)
                 switch (expression.kind())
                 {
                     case SIMPLE:
-                        size += ByteBufferUtil.serializedSizeWithShortLength(((SimpleExpression)expression).value);
+                        size += ByteBufferUtil.serializedSizeWithShortLength((expression).value);
+                        if (expression.operator == Operator.ANN)
+                            size += ANNOptions.serializer.serializedSize(expression.annOptions(), version);
                         break;
-                    case MAP_EQUALITY:
-                        MapEqualityExpression mexpr = (MapEqualityExpression)expression;
+                    case MAP_COMPARISON:
+                        MapComparisonExpression mexpr = (MapComparisonExpression)expression;
                         size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.key)
                               + ByteBufferUtil.serializedSizeWithShortLength(mexpr.value);
                         break;
@@ -688,6 +1226,12 @@ public long serializedSize(Expression expression, int version)
                     case USER:
                         size += UserExpression.serializedSize((UserExpression)expression, version);
                         break;
+                    case VECTOR_RADIUS:
+                        GeoDistanceExpression geoDistanceRelation = (GeoDistanceExpression) expression;
+                        size += ByteBufferUtil.serializedSizeWithShortLength(geoDistanceRelation.distance)
+                                + ByteBufferUtil.serializedSizeWithShortLength(geoDistanceRelation.value)
+                                + geoDistanceRelation.distanceOperator.serializedSize();
+                        break;
                 }
                 return size;
             }
@@ -695,15 +1239,74 @@ public long serializedSize(Expression expression, int version)
     }
 
     /**
-     * An expression of the form 'column' 'op' 'value'.
+     * An expression that can be associated with an {@link Index.Analyzer}.
      */
-    public static class SimpleExpression extends Expression
+    public abstract static class AnalyzableExpression extends Expression
     {
-        SimpleExpression(ColumnMetadata column, Operator operator, ByteBuffer value)
+        @Nullable
+        protected final Index.Analyzer indexAnalyzer;
+
+        @Nullable
+        protected final Index.Analyzer queryAnalyzer;
+
+        public AnalyzableExpression(ColumnMetadata column,
+                                    Operator operator,
+                                    ByteBuffer value,
+                                    @Nullable Index.Analyzer indexAnalyzer,
+                                    @Nullable Index.Analyzer queryAnalyzer)
         {
             super(column, operator, value);
+            this.indexAnalyzer = indexAnalyzer;
+            this.queryAnalyzer = queryAnalyzer;
+        }
+
+        @Nullable
+        public final Index.Analyzer indexAnalyzer()
+        {
+            return indexAnalyzer;
+        }
+
+        @Nullable
+        public final Index.Analyzer queryAnalyzer()
+        {
+            return queryAnalyzer;
+        }
+
+        @Override
+        public int numFilteredValues()
+        {
+            return queryAnalyzer == null
+                   ? super.numFilteredValues()
+                   : queryAnalyzer().analyze(value).size();
+        }
+    }
+
+    /**
+     * An expression of the form 'column' 'op' 'value'.
+     */
+    public static class SimpleExpression extends AnalyzableExpression
+    {
+        @Nullable
+        private final ANNOptions annOptions;
+
+        public SimpleExpression(ColumnMetadata column,
+                                Operator operator,
+                                ByteBuffer value,
+                                @Nullable Index.Analyzer indexAnalyzer,
+                                @Nullable Index.Analyzer queryAnalyzer,
+                                @Nullable ANNOptions annOptions)
+        {
+            super(column, operator, value, indexAnalyzer, queryAnalyzer);
+            this.annOptions = annOptions;
         }
 
+        @Nullable
+        public ANNOptions annOptions()
+        {
+            return annOptions;
+        }
+
+        @Override
         public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
         {
             // We support null conditions for LWT (in ColumnCondition) but not for RowFilter.
@@ -714,6 +1317,7 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey,
             {
                 case EQ:
                 case IN:
+                case NOT_IN:
                 case LT:
                 case LTE:
                 case GTE:
@@ -730,13 +1334,13 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey,
                                 return false;
 
                             ByteBuffer counterValue = LongType.instance.decompose(CounterContext.instance().total(foundValue, ByteBufferAccessor.instance));
-                            return operator.isSatisfiedBy(LongType.instance, counterValue, value);
+                            return operator.isSatisfiedBy(LongType.instance, counterValue, value, indexAnalyzer, queryAnalyzer);
                         }
                         else
                         {
                             // Note that CQL expression are always of the form 'x < 4', i.e. the tested value is on the left.
                             ByteBuffer foundValue = getValue(metadata, partitionKey, row);
-                            return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value);
+                            return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value, indexAnalyzer, queryAnalyzer);
                         }
                     }
                 case NEQ:
@@ -744,97 +1348,125 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey,
                 case LIKE_SUFFIX:
                 case LIKE_CONTAINS:
                 case LIKE_MATCHES:
+                case ANALYZER_MATCHES:
                 case ANN:
+                case BM25:
                     {
                         assert !column.isComplex() : "Only CONTAINS and CONTAINS_KEY are supported for collection types";
                         ByteBuffer foundValue = getValue(metadata, partitionKey, row);
                         // Note that CQL expression are always of the form 'x < 4', i.e. the tested value is on the left.
-                        return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value);
+                        return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value, indexAnalyzer, queryAnalyzer);
                     }
                 case CONTAINS:
-                    assert column.type.isCollection();
-                    CollectionType<?> type = (CollectionType<?>)column.type;
-                    if (column.isComplex())
+                    return contains(metadata, partitionKey, row);
+                case CONTAINS_KEY:
+                    return containsKey(metadata, partitionKey, row);
+                case NOT_CONTAINS:
+                    return !contains(metadata, partitionKey, row);
+                case NOT_CONTAINS_KEY:
+                    return !containsKey(metadata, partitionKey, row);
+            }
+            throw new AssertionError("Unsupported operator: " + operator);
+        }
+
+        private boolean contains(TableMetadata metadata, DecoratedKey partitionKey, Row row)
+        {
+            assert column.type.isCollection();
+            assert (indexAnalyzer == null) == (queryAnalyzer == null);
+
+            CollectionType<?> type = (CollectionType<?>) column.type;
+            List<ByteBuffer> analyzedValues = queryAnalyzer == null ? null : queryAnalyzer.analyze(value);
+
+            if (column.isComplex())
+            {
+                ComplexColumnData complexData = row.getComplexColumnData(column);
+                if (complexData != null)
+                {
+                    AbstractType<?> elementType = type.kind == CollectionType.Kind.SET ? type.nameComparator() : type.valueComparator();
+                    for (Cell<?> cell : complexData)
                     {
-                        ComplexColumnData complexData = row.getComplexColumnData(column);
-                        if (complexData != null)
+                        ByteBuffer elementValue = type.kind == CollectionType.Kind.SET ? cell.path().get(0) : cell.buffer();
+                        if (analyzedValues == null)
                         {
-                            for (Cell<?> cell : complexData)
-                            {
-                                if (type.kind == CollectionType.Kind.SET)
-                                {
-                                    if (type.nameComparator().compare(cell.path().get(0), value) == 0)
-                                        return true;
-                                }
-                                else
-                                {
-                                    if (type.valueComparator().compare(cell.buffer(), value) == 0)
-                                        return true;
-                                }
-                            }
+                            if (elementType.compare(elementValue, value) == 0)
+                                return true;
                         }
-                        return false;
-                    }
-                    else
-                    {
-                        ByteBuffer foundValue = getValue(metadata, partitionKey, row);
-                        if (foundValue == null)
-                            return false;
-
-                        switch (type.kind)
+                        else
                         {
-                            case LIST:
-                                ListType<?> listType = (ListType<?>)type;
-                                return listType.compose(foundValue).contains(listType.getElementsType().compose(value));
-                            case SET:
-                                SetType<?> setType = (SetType<?>)type;
-                                return setType.compose(foundValue).contains(setType.getElementsType().compose(value));
-                            case MAP:
-                                MapType<?,?> mapType = (MapType<?, ?>)type;
-                                return mapType.compose(foundValue).containsValue(mapType.getValuesType().compose(value));
+                            if (Operator.ANALYZER_MATCHES.isSatisfiedBy(elementType, elementValue, analyzedValues, indexAnalyzer))
+                                return true;
                         }
-                        throw new AssertionError();
                     }
-                case CONTAINS_KEY:
-                    assert column.type.isCollection() && column.type instanceof MapType;
-                    MapType<?, ?> mapType = (MapType<?, ?>)column.type;
-                    if (column.isComplex())
-                    {
-                         return row.getCell(column, CellPath.create(value)) != null;
-                    }
-                    else
+                }
+                return false;
+            }
+            else
+            {
+                ByteBuffer foundValue = getValue(metadata, partitionKey, row);
+                return foundValue != null && Operator.CONTAINS.isSatisfiedBy(type, foundValue, value, indexAnalyzer, queryAnalyzer);
+            }
+        }
+
+        private boolean containsKey(TableMetadata metadata, DecoratedKey partitionKey, Row row)
+        {
+            assert column.type.isCollection() && column.type instanceof MapType;
+            MapType<?, ?> mapType = (MapType<?, ?>) column.type;
+            if (column.isComplex())
+            {
+                if (queryAnalyzer != null)
+                {
+                    assert indexAnalyzer != null;
+                    List<ByteBuffer> values = queryAnalyzer.analyze(value);
+                    for (Cell<?> cell : row.getComplexColumnData(column))
                     {
-                        ByteBuffer foundValue = getValue(metadata, partitionKey, row);
-                        return foundValue != null && mapType.getSerializer().getSerializedValue(foundValue, value, mapType.getKeysType()) != null;
+                        AbstractType<?> elementType = mapType.nameComparator();
+                        ByteBuffer elementValue = cell.path().get(0);
+                        if (Operator.ANALYZER_MATCHES.isSatisfiedBy(elementType, elementValue, values, indexAnalyzer))
+                            return true;
                     }
+                    return false;
+                }
+                return row.getCell(column, CellPath.create(value)) != null;
+            }
+            else
+            {
+                ByteBuffer foundValue = getValue(metadata, partitionKey, row);
+                return foundValue != null && Operator.CONTAINS_KEY.isSatisfiedBy(mapType, foundValue, value, indexAnalyzer, queryAnalyzer);
             }
-            throw new AssertionError();
         }
 
         @Override
-        protected String toString(boolean cql)
+        public String toString(boolean cql)
         {
             AbstractType<?> type = column.type;
             switch (operator)
             {
                 case CONTAINS:
+                case NOT_CONTAINS:
                     assert type instanceof CollectionType;
                     CollectionType<?> ct = (CollectionType<?>)type;
                     type = ct.kind == CollectionType.Kind.SET ? ct.nameComparator() : ct.valueComparator();
                     break;
                 case CONTAINS_KEY:
+                case NOT_CONTAINS_KEY:
                     assert type instanceof MapType;
                     type = ((MapType<?, ?>)type).nameComparator();
                     break;
                 case IN:
-                    type = ListType.getInstance(type, false);
+                case NOT_IN:
+                    type = ListType.getInstance(type.freeze(), false);
                     break;
+                case ORDER_BY_ASC:
+                case ORDER_BY_DESC:
+                    // These don't have a value, so we return here to prevent an error calling type.getString(value)
+                    return String.format("%s %s", column.name, operator);
                 default:
                     break;
             }
+
             return cql
-                 ? String.format("%s %s %s", column.name.toCQLString(), operator, type.toCQLString(value) )
-                 : String.format("%s %s %s", column.name.toString(), operator, type.getString(value));
+                   ? String.format("%s %s %s", column.name.toCQLString(), operator, type.toCQLString(value, true))
+                   : String.format("%s %s %s", column.name.toString(), operator, type.getString(value, true));
         }
 
         @Override
@@ -845,17 +1477,24 @@ protected Kind kind()
     }
 
     /**
-     * An expression of the form 'column' ['key'] = 'value' (which is only
-     * supported when 'column' is a map).
+     * An expression of the form 'column' ['key'] OPERATOR 'value' (which is only
+     * supported when 'column' is a map) and where the operator can be {@link Operator#EQ}, {@link Operator#NEQ},
+     * {@link Operator#LT}, {@link Operator#LTE}, {@link Operator#GT}, or {@link Operator#GTE}.
      */
-    private static class MapEqualityExpression extends Expression
+    public static class MapComparisonExpression extends AnalyzableExpression
     {
         private final ByteBuffer key;
-
-        public MapEqualityExpression(ColumnMetadata column, ByteBuffer key, Operator operator, ByteBuffer value)
+        private ByteBuffer indexValue = null;
+
+        public MapComparisonExpression(ColumnMetadata column,
+                                       ByteBuffer key,
+                                       Operator operator,
+                                       ByteBuffer value,
+                                       @Nullable Index.Analyzer indexAnalyzer,
+                                       @Nullable Index.Analyzer queryAnalyzer)
         {
-            super(column, operator, value);
-            assert column.type instanceof MapType && operator == Operator.EQ;
+            super(column, operator, value, indexAnalyzer, queryAnalyzer);
+            assert column.type instanceof MapType && (operator == Operator.EQ || operator == Operator.NEQ || operator.isSlice());
             this.key = key;
         }
 
@@ -871,10 +1510,29 @@ public void validate() throws InvalidRequestException
         @Override
         public ByteBuffer getIndexValue()
         {
-            return CompositeType.build(ByteBufferAccessor.instance, key, value);
+            if (indexValue == null)
+                indexValue = CompositeType.build(ByteBufferAccessor.instance, key, value);
+            return indexValue;
         }
 
+        /**
+         * Returns whether the provided row satisfies this expression. For equality, it validates that the row contains
+         * the exact key/value pair. For inequalities, it validates that the row contains the key, then that the value
+         * satisfies the inequality.
+         *
+         * @param metadata the metadata of the queried table
+         * @param partitionKey the partition key for row to check.
+         * @param row the row to check. It should *not* contain deleted cells
+         * (i.e. it should come from a RowIterator).
+         * @return whether the row is satisfied by this expression.
+         */
+        @Override
         public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
+        {
+            return isSatisfiedByEq(metadata, partitionKey, row) ^ (operator == Operator.NEQ);
+        }
+
+        private boolean isSatisfiedByEq(TableMetadata metadata, DecoratedKey partitionKey, Row row)
         {
             assert key != null;
             // We support null conditions for LWT (in ColumnCondition) but not for RowFilter.
@@ -884,11 +1542,14 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey,
             if (row.isStatic() != column.isStatic())
                 return true;
 
+            int comp;
             MapType<?, ?> mt = (MapType<?, ?>)column.type;
             if (column.isComplex())
             {
                 Cell<?> cell = row.getCell(column, CellPath.create(key));
-                return cell != null && mt.valueComparator().compare(cell.buffer(), value) == 0;
+                if (cell == null)
+                    return false;
+                comp = mt.valueComparator().compare(cell.buffer(), value);
             }
             else
             {
@@ -897,19 +1558,36 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey,
                     return false;
 
                 ByteBuffer foundValue = mt.getSerializer().getSerializedValue(serializedMap, key, mt.getKeysType());
-                return foundValue != null && mt.valueComparator().compare(foundValue, value) == 0;
+                if (foundValue == null)
+                    return false;
+                comp = mt.valueComparator().compare(foundValue, value);
+            }
+            switch (operator) {
+                case EQ:
+                case NEQ: // NEQ is inverted in calling method. We do this to simplify handling of null cells.
+                    return comp == 0;
+                case LT:
+                    return comp < 0;
+                case LTE:
+                    return comp <= 0;
+                case GT:
+                    return comp > 0;
+                case GTE:
+                    return comp >= 0;
+                default:
+                    throw new AssertionError("Unsupported operator: " + operator);
             }
         }
 
         @Override
-        protected String toString(boolean cql)
+        public String toString(boolean cql)
         {
             MapType<?, ?> mt = (MapType<?, ?>) column.type;
             AbstractType<?> nt = mt.nameComparator();
             AbstractType<?> vt = mt.valueComparator();
             return cql
-                 ? String.format("%s[%s] = %s", column.name.toCQLString(), nt.toCQLString(key), vt.toCQLString(value))
-                 : String.format("%s[%s] = %s", column.name.toString(), nt.getString(key), vt.getString(value));
+                 ? String.format("%s[%s] %s %s", column.name.toCQLString(), nt.toCQLString(key), operator, vt.toCQLString(value))
+                 : String.format("%s[%s] %s %s", column.name.toString(), nt.getString(key), operator, vt.getString(value));
         }
 
         @Override
@@ -918,13 +1596,13 @@ public boolean equals(Object o)
             if (this == o)
                 return true;
 
-            if (!(o instanceof MapEqualityExpression))
+            if (!(o instanceof MapComparisonExpression))
                 return false;
 
-            MapEqualityExpression that = (MapEqualityExpression)o;
+            MapComparisonExpression that = (MapComparisonExpression)o;
 
             return Objects.equal(this.column.name, that.column.name)
-                && Objects.equal(this.operator, that.operator)
+                && this.operator == that.operator
                 && Objects.equal(this.key, that.key)
                 && Objects.equal(this.value, that.value);
         }
@@ -938,7 +1616,184 @@ public int hashCode()
         @Override
         protected Kind kind()
         {
-            return Kind.MAP_EQUALITY;
+            return Kind.MAP_COMPARISON;
+        }
+
+        /**
+         * Get the lower bound for this expression. When the expression is EQ, GT, or GTE, the lower bound is the
+         * expression itself. When the expression is LT or LTE, the lower bound is the map's key becuase
+         * {@link ByteBuffer} comparisons will work correctly.
+         * @return the lower bound for this expression.
+         */
+        public ByteBuffer getLowerBound()
+        {
+            switch (operator) {
+                case EQ:
+                case GT:
+                case GTE:
+                    return this.getIndexValue();
+                case LT:
+                case LTE:
+                    return CompositeType.extractFirstComponentAsTrieSearchPrefix(getIndexValue(), true);
+                default:
+                    throw new AssertionError("Unsupported operator: " + operator);
+            }
+        }
+
+        /**
+         * Get the upper bound for this expression. When the expression is EQ, LT, or LTE, the upper bound is the
+         * expression itself. When the expression is GT or GTE, the upper bound is the map's key with the last byte
+         * set to 1 so that {@link ByteBuffer} comparisons will work correctly.
+         * @return the upper bound for this express
+         */
+        public ByteBuffer getUpperBound()
+        {
+            switch (operator) {
+                case GT:
+                case GTE:
+                    return CompositeType.extractFirstComponentAsTrieSearchPrefix(getIndexValue(), false);
+                case EQ:
+                case LT:
+                case LTE:
+                    return this.getIndexValue();
+                default:
+                    throw new AssertionError("Unsupported operator: " + operator);
+            }
+        }
+    }
+
+    public static class GeoDistanceExpression extends Expression
+    {
+        private final ByteBuffer distance;
+        private final Operator distanceOperator;
+        private final float searchRadiusMeters;
+        private final float searchLat;
+        private final float searchLon;
+        // Whether this is a shifted expression, which is used to handle crossing the antimeridian
+        private final boolean isShifted;
+
+        public GeoDistanceExpression(ColumnMetadata column, ByteBuffer point, Operator operator, ByteBuffer distance)
+        {
+            this(column, point, operator, distance, false);
+        }
+
+        private GeoDistanceExpression(ColumnMetadata column, ByteBuffer point, Operator operator, ByteBuffer distance, boolean isShifted)
+        {
+            super(column, Operator.BOUNDED_ANN, point);
+            assert column.type instanceof VectorType && (operator == Operator.LTE || operator == Operator.LT);
+            this.isShifted = isShifted;
+            this.distanceOperator = operator;
+            this.distance = distance;
+            searchRadiusMeters = FloatType.instance.compose(distance);
+            var pointVector = TypeUtil.decomposeVector(column.type, point);
+            // This is validated earlier in the parser because the column requires size 2, so only assert on it
+            assert pointVector.length == 2 : "GEO_DISTANCE requires search vector to have 2 dimensions.";
+            searchLat = pointVector[0];
+            searchLon = pointVector[1];
+        }
+
+        public boolean crossesAntimeridian()
+        {
+            return GeoUtil.crossesAntimeridian(searchLat, searchLon, searchRadiusMeters);
+        }
+
+        /**
+         * @return a new {@link GeoDistanceExpression} that is shifted by 360 degrees and can correctly search
+         * on the opposite side of the antimeridian.
+         */
+        public GeoDistanceExpression buildShiftedExpression()
+        {
+            float shiftedLon = searchLon > 0 ? searchLon - 360 : searchLon + 360;
+            var newPoint = VectorType.getInstance(FloatType.instance, 2)
+                                     .decompose(List.of(searchLat, shiftedLon));
+            return new GeoDistanceExpression(column, newPoint, distanceOperator, distance, true);
+        }
+
+        public Operator getDistanceOperator()
+        {
+            return distanceOperator;
+        }
+
+        public ByteBuffer getDistance()
+        {
+            return distance;
+        }
+
+        @Override
+        public void validate() throws InvalidRequestException
+        {
+            checkBindValueSet(distance, "Unsupported unset distance for column %s", column.name);
+            checkBindValueSet(value, "Unsupported unset vector value for column %s", column.name);
+
+            if (searchRadiusMeters <= 0)
+                throw new InvalidRequestException("GEO_DISTANCE radius must be positive, got " + searchRadiusMeters);
+
+            if (searchLat < -90 || searchLat > 90)
+                throw new InvalidRequestException("GEO_DISTANCE latitude must be between -90 and 90 degrees, got " + searchLat);
+            if (!isShifted && (searchLon < -180 || searchLon > 180))
+                throw new InvalidRequestException("GEO_DISTANCE longitude must be between -180 and 180 degrees, got " + searchLon);
+        }
+
+        @Override
+        public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
+        {
+            ByteBuffer foundValue = getValue(metadata, partitionKey, row);
+            if (foundValue == null)
+                return false;
+            var foundVector = TypeUtil.decomposeVector(column.type, foundValue);
+            double haversineDistance = SloppyMath.haversinMeters(foundVector[0], foundVector[1], searchLat, searchLon);
+            switch (distanceOperator)
+            {
+                case LTE:
+                    return haversineDistance <= searchRadiusMeters;
+                case LT:
+                    return haversineDistance < searchRadiusMeters;
+                default:
+                    throw new AssertionError("Unsupported operator: " + operator);
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            return toString(false);
+        }
+
+        @Override
+        public String toString(boolean cql)
+        {
+            return String.format("GEO_DISTANCE(%s, %s) %s %s", cql ? column.name.toCQLString() : column.name.toString(),
+                                 column.type.getString(value),
+                                 distanceOperator, FloatType.instance.getString(distance));
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o)
+                return true;
+
+            if (!(o instanceof GeoDistanceExpression))
+                return false;
+
+            GeoDistanceExpression that = (GeoDistanceExpression)o;
+
+            return Objects.equal(this.column.name, that.column.name)
+                   && this.distanceOperator == that.distanceOperator
+                   && Objects.equal(this.distance, that.distance)
+                   && Objects.equal(this.value, that.value);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hashCode(column.name, distanceOperator, value, distance);
+        }
+
+        @Override
+        protected Kind kind()
+        {
+            return Kind.VECTOR_RADIUS;
         }
     }
 
@@ -946,7 +1801,7 @@ protected Kind kind()
      * A custom index expression for use with 2i implementations which support custom syntax and which are not
      * necessarily linked to a single column in the base table.
      */
-    public static final class CustomExpression extends Expression
+    public static class CustomExpression extends Expression
     {
         private final IndexMetadata targetIndex;
         private final TableMetadata table;
@@ -959,6 +1814,12 @@ public CustomExpression(TableMetadata table, IndexMetadata targetIndex, ByteBuff
             this.table = table;
         }
 
+        public static CustomExpression build(TableMetadata metadata, IndexMetadata targetIndex, ByteBuffer value)
+        {
+            // delegate the expression creation to the target custom index
+            return Keyspace.openAndGetStore(metadata).indexManager.getIndex(targetIndex).customExpressionFor(metadata, value);
+        }
+
         private static ColumnMetadata makeDefinition(TableMetadata table, IndexMetadata index)
         {
             // Similarly to how we handle non-defined columns in thift, we create a fake column definition to
@@ -977,7 +1838,7 @@ public ByteBuffer getValue()
         }
 
         @Override
-        protected String toString(boolean cql)
+        public String toString(boolean cql)
         {
             return String.format("expr(%s, %s)",
                                  cql ? ColumnIdentifier.maybeQuote(targetIndex.name) : targetIndex.name,
@@ -987,12 +1848,14 @@ protected String toString(boolean cql)
                                          .customExpressionValueType());
         }
 
+        @Override
         protected Kind kind()
         {
             return Kind.CUSTOM;
         }
 
         // Filtering by custom expressions isn't supported yet, so just accept any row
+        @Override
         public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
         {
             return true;
@@ -1011,7 +1874,7 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey,
      * is important, new types should registered last and obsoleted types should still be registered (
      * or dummy implementations registered in their place) to preserve consistent identifiers across
      * the cluster).
-     *
+     * </p>
      * During serialization, the identifier for the Deserializer implementation is prepended to the
      * implementation specific payload. To deserialize, the identifier is read first to obtain the
      * Deserializer, which then provides the concrete expression instance.
@@ -1084,6 +1947,7 @@ protected UserExpression(ColumnMetadata column, Operator operator, ByteBuffer va
             super(column, operator, value);
         }
 
+        @Override
         protected Kind kind()
         {
             return Kind.USER;
@@ -1098,30 +1962,21 @@ public static class Serializer
         public void serialize(RowFilter filter, DataOutputPlus out, int version) throws IOException
         {
             out.writeBoolean(false); // Old "is for thrift" boolean
-            out.writeUnsignedVInt32(filter.expressions.size());
-            for (Expression expr : filter.expressions)
-                Expression.serializer.serialize(expr, out, version);
+            FilterElement.serializer.serialize(filter.root, out, version);
 
         }
 
         public RowFilter deserialize(DataInputPlus in, int version, TableMetadata metadata, boolean needsReconciliation) throws IOException
         {
             in.readBoolean(); // Unused
-            int size = in.readUnsignedVInt32();
-            List<Expression> expressions = new ArrayList<>(size);
-            for (int i = 0; i < size; i++)
-                expressions.add(Expression.serializer.deserialize(in, version, metadata));
-
-            return new RowFilter(expressions, needsReconciliation);
+            FilterElement operation = FilterElement.serializer.deserialize(in, version, metadata);
+            return new RowFilter(operation, needsReconciliation);
         }
 
         public long serializedSize(RowFilter filter, int version)
         {
-            long size = 1 // unused boolean
-                      + TypeSizes.sizeofUnsignedVInt(filter.expressions.size());
-            for (Expression expr : filter.expressions)
-                size += Expression.serializer.serializedSize(expr, version);
-            return size;
+            return 1 // unused boolean
+                   + FilterElement.serializer.serializedSize(filter.root, version);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java b/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java
index efca3ac4db44..c8a88f19bafe 100644
--- a/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java
+++ b/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java
@@ -20,18 +20,29 @@
 
 import java.nio.ByteBuffer;
 
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RejectException;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.exceptions.InternalRequestExecutionException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.*;
 
-public class TombstoneOverwhelmingException extends RejectException
+public class TombstoneOverwhelmingException extends RejectException implements InternalRequestExecutionException
 {
-    public TombstoneOverwhelmingException(int numTombstones, String query, TableMetadata metadata, DecoratedKey lastPartitionKey, ClusteringPrefix<?> lastClustering)
+    public TombstoneOverwhelmingException(long numTombstones, String query, TableMetadata metadata, DecoratedKey lastPartitionKey, ClusteringPrefix<?> lastClustering)
     {
         super(String.format("Scanned over %d tombstones during query '%s' (last scanned row token was %s and partion key was (%s)); query aborted",
                             numTombstones, query, lastPartitionKey.getToken(), makePKString(metadata, lastPartitionKey.getKey(), lastClustering)));
     }
 
+    @Override
+    public RequestFailureReason getReason()
+    {
+        return RequestFailureReason.READ_TOO_MANY_TOMBSTONES;
+    }
+
     private static String makePKString(TableMetadata metadata, ByteBuffer partitionKey, ClusteringPrefix<?> clustering)
     {
         StringBuilder sb = new StringBuilder();
@@ -49,7 +60,7 @@ private static String makePKString(TableMetadata metadata, ByteBuffer partitionK
             {
                 if (i > 0)
                     sb.append(", ");
-                sb.append(ct.types.get(i).getString(values[i]));
+                sb.append(ct.subTypes.get(i).getString(values[i]));
             }
         }
         else
diff --git a/src/java/org/apache/cassandra/db/guardrails/CustomUserKeyspaceFilterProvider.java b/src/java/org/apache/cassandra/db/guardrails/CustomUserKeyspaceFilterProvider.java
new file mode 100644
index 000000000000..c76e7e2b774b
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/guardrails/CustomUserKeyspaceFilterProvider.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.guardrails;
+
+public class CustomUserKeyspaceFilterProvider
+{
+    public static UserKeyspaceFilterProvider make(String customImpl)
+    {
+        try
+        {
+            return (UserKeyspaceFilterProvider) Class.forName(customImpl).getDeclaredConstructor().newInstance();
+        }
+        catch (Throwable ex)
+        {
+            throw new IllegalStateException("Unknown user keyspace filter provider: " + customImpl, ex);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/guardrails/DefaultUserKeyspaceFilter.java b/src/java/org/apache/cassandra/db/guardrails/DefaultUserKeyspaceFilter.java
new file mode 100644
index 000000000000..0a0c453ff826
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/guardrails/DefaultUserKeyspaceFilter.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.guardrails;
+
+import org.apache.cassandra.db.Keyspace;
+
+/**
+ * A default implementation of the UserKeyspaceFilter, which just includes all keyspaces automatically.
+ * This is done so that "max table count" guardrails will still work in C* databases that don't have
+ * keyspaces prefixed by tenant values.
+ */
+public class DefaultUserKeyspaceFilter implements UserKeyspaceFilter
+{
+    public boolean filter(Keyspace keyspace)
+    {
+        return true;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/guardrails/DefaultUserKeyspaceFilterProvider.java b/src/java/org/apache/cassandra/db/guardrails/DefaultUserKeyspaceFilterProvider.java
new file mode 100644
index 000000000000..9642728a1796
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/guardrails/DefaultUserKeyspaceFilterProvider.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.guardrails;
+
+import org.apache.cassandra.service.ClientState;
+
+public class DefaultUserKeyspaceFilterProvider implements UserKeyspaceFilterProvider
+{
+    @Override
+    public UserKeyspaceFilter get(ClientState clientState)
+    {
+        return new DefaultUserKeyspaceFilter();
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/guardrails/Guardrail.java b/src/java/org/apache/cassandra/db/guardrails/Guardrail.java
index fbd1a5b8803e..a603c5f554a4 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Guardrail.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Guardrail.java
@@ -162,19 +162,24 @@ String decorateMessage(String message)
      *                              default 0 means always log and trigger listeners.
      * @return current guardrail
      */
-    Guardrail minNotifyIntervalInMs(long minNotifyIntervalInMs)
+    public Guardrail minNotifyIntervalInMs(long minNotifyIntervalInMs)
     {
         assert minNotifyIntervalInMs >= 0;
         this.minNotifyIntervalInMs = minNotifyIntervalInMs;
         return this;
     }
+    
+    public long minNotifyIntervalInMs()
+    {
+        return minNotifyIntervalInMs;
+    }
 
     /**
      * reset last notify time to make sure it will notify downstream when {@link this#warn(String, String)}
      * or {@link this#fail(String, ClientState)} is called next time.
      */
     @VisibleForTesting
-    void resetLastNotifyTime()
+    public void resetLastNotifyTime()
     {
         lastFailInMs = 0;
         lastWarnInMs = 0;
diff --git a/src/java/org/apache/cassandra/db/guardrails/Guardrails.java b/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
index 829dab056237..0b2984b62666 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
@@ -108,6 +108,39 @@ public final class Guardrails implements GuardrailsMBean
                                : format("Tables cannot have more than %s secondary indexes, aborting the creation of secondary index %s",
                                         threshold, what));
 
+    public static final MaxThreshold sasiIndexesPerTable =
+    new MaxThreshold("sasi_indexes_per_table",
+                     null,
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getSasiIndexesPerTableWarnThreshold(),
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getSasiIndexesPerTableFailThreshold(),
+                     (isWarning, what, value, threshold) ->
+                     isWarning ? format("Creating SASI index %s, current number of indexes %s exceeds warning threshold of %s.",
+                                        what, value, threshold)
+                               : format("Tables cannot have more than %s SASI indexes, aborting the creation of secondary index %s",
+                                        threshold, what));
+
+    public static final MaxThreshold saiIndexesPerTable =
+    new MaxThreshold("sai_indexes_per_table",
+                     null,
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getStorageAttachedIndexesPerTableWarnThreshold(),
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getStorageAttachedIndexesPerTableFailThreshold(),
+                     (isWarning, what, value, threshold) ->
+                     isWarning ? format("Creating StorageAttachedIndex secondary index %s, current number of StorageAttachedIndex secondary indexes %s exceeds warning threshold of %s.",
+                                        what, value, threshold)
+                               : format("Tables cannot have more than %s StorageAttachedIndex secondary indexes, aborting the creation of secondary index %s",
+                                        threshold, what));
+
+    public static final MaxThreshold saiIndexesTotal =
+    new MaxThreshold("sai_indexes_total",
+                     null,
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getStorageAttachedIndexesTotalWarnThreshold(),
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getStorageAttachedIndexesTotalFailThreshold(),
+                     (isWarning, what, value, threshold) ->
+                     isWarning ? format("Creating StorageAttachedIndex secondary index %s, current number of StorageAttachedIndex secondary indexes across all keyspaces %s exceeds warning threshold of %s.",
+                                        what, value, threshold)
+                               : format("Cannot have more than %s StorageAttachedIndex secondary indexes across all keyspaces, aborting the creation of secondary index %s",
+                                        threshold, what));
+
     /**
      * Guardrail disabling user's ability to create secondary indexes
      */
@@ -151,6 +184,12 @@ public final class Guardrails implements GuardrailsMBean
                    state -> CONFIG_PROVIDER.getOrCreate(state).getUserTimestampsEnabled(),
                    "User provided timestamps (USING TIMESTAMP)");
 
+    public static final EnableFlag loggedBatchEnabled =
+    new EnableFlag("logged_batch",
+                   null,
+                   state -> CONFIG_PROVIDER.getOrCreate(state).getLoggedBatchEnabled(),
+                   "LOGGED batch");
+
     public static final EnableFlag groupByEnabled =
     new EnableFlag("group_by",
                    null,
@@ -245,6 +284,20 @@ public final class Guardrails implements GuardrailsMBean
                                : format("Aborting query for table %s, page size %s exceeds fail threshold of %s.",
                                         what, value, threshold));
 
+    /**
+     * Guardrail on the weight (bytes) of elements returned within page.
+     */
+    public static final MaxThreshold pageWeight =
+    new MaxThreshold("page_weight",
+                     null,
+                     state -> sizeToBytes(CONFIG_PROVIDER.getOrCreate(state).getPageWeightWarnThreshold()),
+                     state -> sizeToBytes(CONFIG_PROVIDER.getOrCreate(state).getPageWeightFailThreshold()),
+                     (isWarning, what, value, threshold) ->
+                     isWarning ? format("Query for table %s with page weight %s bytes exceeds warning threshold of %s bytes.",
+                                        what, value, threshold)
+                               : format("Aborting query for table %s, page weight %s bytes exceeds fail threshold of %s bytes.",
+                                        what, value, threshold));
+
     /**
      * Guardrail on the number of partition keys in the IN clause.
      */
@@ -288,6 +341,15 @@ public final class Guardrails implements GuardrailsMBean
                    state -> CONFIG_PROVIDER.getOrCreate(state).getSimpleStrategyEnabled(),
                    "SimpleStrategy");
 
+    /**
+     * Guardrail disabling use of Counters
+     */
+    public static final EnableFlag counterEnabled =
+    new EnableFlag("counter",
+                   null,
+                   state -> CONFIG_PROVIDER.getOrCreate(state).getCounterEnabled(),
+                   "Counter");
+
     /**
      * Guardrail on the number of restrictions created by a cartesian product of a CQL's {@code IN} query.
      */
@@ -425,6 +487,18 @@ public final class Guardrails implements GuardrailsMBean
                      format("%s has a vector of %s dimensions, this exceeds the %s threshold of %s.",
                             what, value, isWarning ? "warning" : "failure", threshold));
 
+    /**
+     * Guardrail on the maximum value for the rerank_k parameter, an ANN query option.
+     */
+    public static final MaxThreshold annRerankKMaxValue =
+    new MaxThreshold("sai_ann_rerank_k_max_value",
+                     null,
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getSaiAnnRerankKWarnThreshold(),
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getSaiAnnRerankKFailThreshold(),
+                      (isWarning, what, value, threshold) ->
+                      format("%s specifies rerank_k=%s, this exceeds the %s threshold of %s.",
+                             what, value, isWarning ? "warning" : "failure", threshold));
+
     /**
      * Guardrail on the data disk usage on the local node, used by a periodic task to calculate and propagate that status.
      * See {@link org.apache.cassandra.service.disk.usage.DiskUsageMonitor} and {@link DiskUsageBroadcaster}.
@@ -461,6 +535,8 @@ public final class Guardrails implements GuardrailsMBean
         long minNotifyInterval = CassandraRelevantProperties.DISK_USAGE_NOTIFY_INTERVAL_MS.getLong();
         localDataDiskUsage.minNotifyIntervalInMs(minNotifyInterval);
         replicaDiskUsage.minNotifyIntervalInMs(minNotifyInterval);
+        collectionSize.minNotifyIntervalInMs(minNotifyInterval);
+        itemsPerCollection.minNotifyIntervalInMs(minNotifyInterval);
     }
 
     /**
@@ -555,6 +631,58 @@ public final class Guardrails implements GuardrailsMBean
                    "Executing a query on secondary indexes without partition key restriction might degrade performance",
                    state -> CONFIG_PROVIDER.getOrCreate(state).getNonPartitionRestrictedQueryEnabled(),
                    "Non-partition key restricted query");
+    public static final Threshold scannedTombstones =
+    new MaxThreshold("scanned_tombstones",
+                      null,
+                      state -> CONFIG_PROVIDER.getOrCreate(state).getTombstoneWarnThreshold(),
+                      state -> CONFIG_PROVIDER.getOrCreate(state).getTombstoneFailThreshold(),
+                      (isWarning, what, v, t) -> isWarning ?
+                                                 format("Scanned over %s tombstone rows for query %1.512s - more than the warning threshold %s", v, what, t) :
+                                                 format("Scanned over %s tombstone rows during query %1.512s - more than the maximum allowed %s; query aborted", v, what, t));
+
+
+    public static final Threshold batchSize =
+    new MaxThreshold("batch_size",
+                      null,
+                      state -> CONFIG_PROVIDER.getOrCreate(state).getBatchSizeWarnThreshold(),
+                      state -> CONFIG_PROVIDER.getOrCreate(state).getBatchSizeFailThreshold(),
+                      (isWarning, what, v, t) -> isWarning
+                                                 ? format("Batch for %s is of size %s, exceeding specified warning threshold %s", what, v, t)
+                                                 : format("Batch for %s is of size %s, exceeding specified failure threshold %s", what, v, t));
+
+    public static final Threshold unloggedBatchAcrossPartitions =
+    new MaxThreshold("unlogged_batch_across_partitions",
+                      null,
+                      state -> CONFIG_PROVIDER.getOrCreate(state).getUnloggedBatchAcrossPartitionsWarnThreshold(),
+                      state -> CONFIG_PROVIDER.getOrCreate(state).getUnloggedBatchAcrossPartitionsFailThreshold(),
+                      (x, what, v, t) -> format("Unlogged batch covering %s partitions detected " +
+                                                "against table%s %s. You should use a logged batch for " +
+                                                "atomicity, or asynchronous writes for performance.",
+                                                v, what.contains(", ") ? "s" : "", what));
+
+    /**
+     * Guardrail on the number of rows that a SELECT query with LIMIT/OFFSET can skip.
+     */
+    public static final Threshold offsetRows =
+    new MaxThreshold("offset_rows",
+                     null,
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getOffsetRowsWarnThreshold(),
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getOffsetRowsFailThreshold(),
+                     (isWarning, what, v, t) -> isWarning
+                                                ? format("%s requested to skip %s rows, this exceeds the warning threshold of %s.", what, v, t)
+                                                : format("%s requested to skip %s rows, this exceeds the failure threshold of %s.", what, v, t));
+
+    /**
+     * Guardrail on the number of query filtering operations per SELECT query (after analysis).
+     */
+    public static final Threshold queryFilters =
+    new MaxThreshold("query_filters",
+                      null,
+                      state -> CONFIG_PROVIDER.getOrCreate(state).getQueryFiltersWarnThreshold(),
+                      state -> CONFIG_PROVIDER.getOrCreate(state).getQueryFiltersFailThreshold(),
+                      (isWarning, what, v, t) -> isWarning
+                                                 ? format("%s has %s column value filters after analysis, this exceeds the warning threshold of %s.", what, v, t)
+                                                 : format("%s has %s column value filters after analysis, this exceeds the failure threshold of %s.", what, v, t));
 
     private Guardrails()
     {
@@ -639,6 +767,42 @@ public boolean getSecondaryIndexesEnabled()
         return DEFAULT_CONFIG.getSecondaryIndexesEnabled();
     }
 
+    @Override
+    public int getStorageAttachedIndexesPerTableWarnThreshold()
+    {
+        return DEFAULT_CONFIG.getStorageAttachedIndexesPerTableWarnThreshold();
+    }
+
+    @Override
+    public int getStorageAttachedIndexesPerTableFailThreshold()
+    {
+        return DEFAULT_CONFIG.getStorageAttachedIndexesPerTableFailThreshold();
+    }
+
+    @Override
+    public void setStorageAttachedIndexesPerTableThreshold(int warn, int fail)
+    {
+        DEFAULT_CONFIG.setStorageAttachedIndexesPerTableThreshold(warn, fail);
+    }
+
+    @Override
+    public int getStorageAttachedIndexesTotalWarnThreshold()
+    {
+        return DEFAULT_CONFIG.getStorageAttachedIndexesPerTableWarnThreshold();
+    }
+
+    @Override
+    public int getStorageAttachedIndexesTotalFailThreshold()
+    {
+        return DEFAULT_CONFIG.getStorageAttachedIndexesPerTableFailThreshold();
+    }
+
+    @Override
+    public void setStorageAttachedIndexesTotalThreshold(int warn, int fail)
+    {
+        DEFAULT_CONFIG.setStorageAttachedIndexesTotalThreshold(warn, fail);
+    }
+
     @Override
     public void setSecondaryIndexesEnabled(boolean enabled)
     {
@@ -834,6 +998,18 @@ public void setGroupByEnabled(boolean enabled)
         DEFAULT_CONFIG.setGroupByEnabled(enabled);
     }
 
+    @Override
+    public boolean getLoggedBatchEnabled()
+    {
+        return DEFAULT_CONFIG.getLoggedBatchEnabled();
+    }
+
+    @Override
+    public void setLoggedBatchEnabled(boolean enabled)
+    {
+        DEFAULT_CONFIG.setLoggedBatchEnabled(enabled);
+    }
+
     @Override
     public boolean getDropTruncateTableEnabled()
     {
@@ -876,6 +1052,26 @@ public void setPageSizeThreshold(int warn, int fail)
         DEFAULT_CONFIG.setPageSizeThreshold(warn, fail);
     }
 
+    @Override
+    @Nullable
+    public String getPageWeightWarnThreshold()
+    {
+        return sizeToString(DEFAULT_CONFIG.getPageWeightWarnThreshold());
+    }
+
+    @Override
+    @Nullable
+    public String getPageWeightFailThreshold()
+    {
+        return sizeToString(DEFAULT_CONFIG.getPageWeightFailThreshold());
+    }
+
+    @Override
+    public void setPageWeightThreshold(@Nullable String warnSize, @Nullable String failSize)
+    {
+        DEFAULT_CONFIG.setPageWeightThreshold(intSizeFromString(warnSize), intSizeFromString(failSize));
+    }
+
     @Override
     public boolean getReadBeforeWriteListOperationsEnabled()
     {
@@ -1152,6 +1348,24 @@ public void setVectorDimensionsThreshold(int warn, int fail)
         DEFAULT_CONFIG.setVectorDimensionsThreshold(warn, fail);
     }
 
+    @Override
+    public int getSaiAnnRerankKWarnThreshold()
+    {
+        return DEFAULT_CONFIG.getSaiAnnRerankKWarnThreshold();
+    }
+
+    @Override
+    public int getSaiAnnRerankKFailThreshold()
+    {
+        return DEFAULT_CONFIG.getSaiAnnRerankKFailThreshold();
+    }
+
+    @Override
+    public void setSaiAnnRerankKThreshold(int warn, int fail)
+    {
+        DEFAULT_CONFIG.setSaiAnnRerankKThreshold(warn, fail);
+    }
+
     @Override
     public void setVectorTypeEnabled(boolean enabled)
     {
@@ -1405,6 +1619,42 @@ public void setIntersectFilteringQueryEnabled(boolean value)
         DEFAULT_CONFIG.setIntersectFilteringQueryEnabled(value);
     }
 
+    @Override
+    public int getOffsetRowsWarnThreshold()
+    {
+        return DEFAULT_CONFIG.getOffsetRowsWarnThreshold();
+    }
+
+    @Override
+    public int getOffsetRowsFailThreshold()
+    {
+        return DEFAULT_CONFIG.getOffsetRowsFailThreshold();
+    }
+
+    @Override
+    public void setOffsetRowsThreshold(int warn, int fail)
+    {
+        DEFAULT_CONFIG.setOffsetRowsThreshold(warn, fail);
+    }
+
+    @Override
+    public int getQueryFiltersWarnThreshold()
+    {
+        return DEFAULT_CONFIG.getQueryFiltersWarnThreshold();
+    }
+
+    @Override
+    public int getQueryFiltersFailThreshold()
+    {
+        return DEFAULT_CONFIG.getQueryFiltersFailThreshold();
+    }
+
+    @Override
+    public void setQueryFiltersThreshold(int warn, int fail)
+    {
+        DEFAULT_CONFIG.setQueryFiltersThreshold(warn, fail);
+    }
+
     private static String toCSV(Set<String> values)
     {
         return values == null || values.isEmpty() ? "" : String.join(",", values);
@@ -1444,6 +1694,11 @@ private static Long sizeToBytes(@Nullable DataStorageSpec.LongBytesBound size)
         return size == null ? -1 : size.toBytes();
     }
 
+    private static Integer sizeToBytes(@Nullable DataStorageSpec.IntBytesBound size)
+    {
+        return size == null ? -1 : size.toBytes();
+    }
+
     private static String sizeToString(@Nullable DataStorageSpec size)
     {
         return size == null ? null : size.toString();
@@ -1454,6 +1709,11 @@ private static DataStorageSpec.LongBytesBound sizeFromString(@Nullable String si
         return StringUtils.isEmpty(size) ? null : new DataStorageSpec.LongBytesBound(size);
     }
 
+    private static DataStorageSpec.IntBytesBound intSizeFromString(@Nullable String size)
+    {
+        return StringUtils.isEmpty(size) ? null : new DataStorageSpec.IntBytesBound(size);
+    }
+
     private static String durationToString(@Nullable DurationSpec duration)
     {
         return duration == null ? null : duration.toString();
diff --git a/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java b/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
index ece387461d78..ba5933b65b09 100644
--- a/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
+++ b/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
@@ -23,6 +23,7 @@
 import javax.annotation.Nullable;
 
 import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.config.DataStorageSpec.IntBytesBound;
 import org.apache.cassandra.config.DurationSpec;
 import org.apache.cassandra.db.ConsistencyLevel;
 
@@ -91,6 +92,36 @@ public interface GuardrailsConfig
      */
     boolean getSecondaryIndexesEnabled();
 
+    /**
+     * @return The threshold to warn when creating more SASI indexes per table than threshold.
+     */
+    int getSasiIndexesPerTableWarnThreshold();
+
+    /**
+     * @return The threshold to fail when creating more SASI indexes per table than threshold.
+     */
+    int getSasiIndexesPerTableFailThreshold();
+
+    /**
+     * @return The threshold to warn when creating more SAI indexes per table than threshold.
+     */
+    int getStorageAttachedIndexesPerTableWarnThreshold();
+
+    /**
+     * @return The threshold to fail when creating more SAI indexes per table than threshold.
+     */
+    int getStorageAttachedIndexesPerTableFailThreshold();
+
+    /**
+     * @return The threshold to warn when creating more SAI indexes in total than threshold.
+     */
+    int getStorageAttachedIndexesTotalWarnThreshold();
+
+    /**
+     * @return The threshold to fail when creating more SAI indexes in total than threshold.
+     */
+    int getStorageAttachedIndexesTotalFailThreshold();
+
     /**
      * @return The threshold to warn when creating more materialized views per table than threshold.
      */
@@ -161,6 +192,13 @@ public interface GuardrailsConfig
      */
     boolean getGroupByEnabled();
 
+    /**
+     * Returns whether logged batches are allowed
+     *
+     * @return {@code true} if allowed, {@code false} otherwise.
+     */
+    boolean getLoggedBatchEnabled();
+
     /**
      * Returns whether TRUNCATE or DROP table are allowed
      *
@@ -176,15 +214,25 @@ public interface GuardrailsConfig
     boolean getDropKeyspaceEnabled();
 
     /**
-     * @return The threshold to warn when page size exceeds given size.
+     * @return The threshold to warn when page size exceeds given size in rows.
      */
     int getPageSizeWarnThreshold();
 
     /**
-     * @return The threshold to fail when page size exceeds given size.
+     * @return The threshold to fail when page size exceeds given size in rows.
      */
     int getPageSizeFailThreshold();
 
+    /**
+     * @return The threshold to warn when page size exceeds given size in bytes.
+     */
+    IntBytesBound getPageWeightWarnThreshold();
+
+    /**
+     * @return The threshold to fail when page size exceeds given size in bytes.
+     */
+    IntBytesBound getPageWeightFailThreshold();
+
     /**
      * Returns whether list operations that require read before write are allowed.
      *
@@ -206,6 +254,13 @@ public interface GuardrailsConfig
      */
     boolean getSimpleStrategyEnabled();
 
+    /**
+     * Returns whether use of Counters is enabled
+     *
+     * @return {@code true} if Counters are allowed, {@code false} otherwise.
+     */
+    boolean getCounterEnabled();
+
     /**
      * @return The threshold to warn when an IN query creates a cartesian product with a size exceeding threshold.
      * -1 means disabled.
@@ -326,6 +381,16 @@ public interface GuardrailsConfig
      */
     int getVectorDimensionsFailThreshold();
 
+    /**
+     * @return The threshold to warn when creating a vector with more dimensions than threshold.
+     */
+    int getSaiAnnRerankKWarnThreshold();
+
+    /**
+     * @return The threshold to fail when creating a vector with more dimensions than threshold.
+     */
+    int getSaiAnnRerankKFailThreshold();
+
     /**
      * @return The threshold to warn when local disk usage percentage exceeds that threshold.
      * Allowed values are in the range {@code [1, 100]}, and -1 means disabled.
@@ -545,4 +610,62 @@ void setMinimumTimestampThreshold(@Nullable DurationSpec.LongMicrosecondsBound w
      * @param enabled {@code true} if a query without partition key is enabled or not
      */
     void setNonPartitionRestrictedQueryEnabled(boolean enabled);
+
+    /*
+     * @return The threshold to warn when a read scans more tombstones than threshold.
+     */
+    int getTombstoneWarnThreshold();
+
+    /**
+     * @return The threshold to fail when a read scans more tombstones than threshold.
+     */
+    int getTombstoneFailThreshold();
+
+    /**
+     * Sets warning and failure thresholds for the number of tombstones read by a query
+     *
+     * @param warn value to set for warn threshold
+     * @param fail value to set for fail threshold
+     */
+    void setTombstonesThreshold(int warn, int fail);
+
+    /**
+     * @return The threshold to warn when the number of batch mutations is more than threshold.
+     */
+    long getBatchSizeWarnThreshold();
+
+    /**
+     * @return The threshold to fail when the number of batch mutations is more than threshold.
+     */
+    long getBatchSizeFailThreshold();
+
+    /**
+     * @return The threshold to warn when the number of unlogged batch partitions is more than threshold.
+     */
+    long getUnloggedBatchAcrossPartitionsWarnThreshold();
+
+    /**
+     * @return The threshold to fail when the numner of unlogged batch partitions is more than threshold.
+     */
+    long getUnloggedBatchAcrossPartitionsFailThreshold();
+
+    /**
+     * @return the warning threshold for the offset rows used in SELECT queries
+     */
+    int getOffsetRowsWarnThreshold();
+
+    /**
+     * @return the failure threshold for the offset rows used in SELECT queries
+     */
+    int getOffsetRowsFailThreshold();
+
+    /**
+     * @return the warning threshold for the number of query filtering operations per SELECT query (after analysis)
+     */
+    int getQueryFiltersWarnThreshold();
+
+    /**
+     * @return the failure threshold for the number of query filtering operations per SELECT query (after analysis)
+     */
+    int getQueryFiltersFailThreshold();
 }
diff --git a/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java b/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
index d40092314a3e..3427b2159707 100644
--- a/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
+++ b/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
@@ -116,6 +116,38 @@ public interface GuardrailsMBean
      */
     void setSecondaryIndexesEnabled(boolean enabled);
 
+    /**
+     * @return The threshold to warn when creating more storage attached indexes per table than threshold. -1 means disabled.
+     */
+    int getStorageAttachedIndexesPerTableWarnThreshold();
+
+    /**
+     * @return The threshold to prevent creating more storage attached indexes per table than threshold. -1 means disabled.
+     */
+    int getStorageAttachedIndexesPerTableFailThreshold();
+
+    /**
+     * @param warn The threshold to warn when creating more storage attached indexes per table than threshold. -1 means disabled.
+     * @param fail The threshold to prevent creating more storage attached indexes per table than threshold. -1 means disabled.
+     */
+    void setStorageAttachedIndexesPerTableThreshold(int warn, int fail);
+
+    /**
+     * @return The threshold to warn when creating more storage attached indexes total than threshold. -1 means disabled.
+     */
+    int getStorageAttachedIndexesTotalWarnThreshold();
+
+    /**
+     * @return The threshold to prevent creating more storage attached indexes total than threshold. -1 means disabled.
+     */
+    int getStorageAttachedIndexesTotalFailThreshold();
+
+    /**
+     * @param warn The threshold to warn when creating more storage attached indexes total than threshold. -1 means disabled.
+     * @param fail The threshold to prevent creating more storage attahced indexes total than threshold. -1 means disabled.
+     */
+    void setStorageAttachedIndexesTotalThreshold(int warn, int fail);
+
     /**
      * @return The threshold to warn when creating more materialized views per table than threshold.
      * -1 means disabled.
@@ -292,6 +324,18 @@ public interface GuardrailsMBean
      */
     void setGroupByEnabled(boolean enabled);
 
+    /**
+     * Returns whether logged batches are allowed.
+     *
+     * @return {@code true} if allowed, {@code false} otherwise.
+     */
+    boolean getLoggedBatchEnabled();
+
+    /**
+     * Sets whether logged batches are allowed.
+     */
+    void setLoggedBatchEnabled(boolean enabled);
+
     /**
      * Returns whether users can TRUNCATE or DROP TABLE
      *
@@ -334,6 +378,31 @@ public interface GuardrailsMBean
      */
     void setPageSizeThreshold(int warn, int fail);
 
+    /**
+     * @return The threshold to warn when requesting page with more data (bytes) than threshold, as a string formatted as in,
+     * for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}. A {@code null} value means disabled.
+     */
+    @Nullable
+    String getPageWeightWarnThreshold();
+
+    /**
+     * @return The threshold to fail when requesting page with more data (bytes) than threshold, as a string formatted as in,
+     * for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}. A {@code null} value means disabled.
+     */
+    @Nullable
+    String getPageWeightFailThreshold();
+
+    /**
+     * @param warnSize The threshold to warn when encountering page weights larger than threshold, as a string formatted
+     *                 as in, for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}.
+     *                 A {@code null} value means disabled.
+     * @param failSize The threshold to fail when encountering page weights larger than threshold, as a string formatted
+     *                 as in, for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}.
+     *                 A {@code null} value means disabled. Triggering a failure emits a log message and a diagnostic
+     *                 event, but it desn't throw an exception interrupting the offending sstable write.
+     */
+    void setPageWeightThreshold(@Nullable String warnSize, @Nullable String failSize);
+
     /**
      * Returns whether list operations that require read before write are allowed.
      *
@@ -613,6 +682,22 @@ public interface GuardrailsMBean
      */
     void setVectorDimensionsThreshold(int warn, int fail);
 
+    /**
+     * @return The threshold to warn for the rerank_k parameter, an ANN query option.
+     */
+    int getSaiAnnRerankKWarnThreshold();
+
+    /**
+     * @return The threshold to fail for the rerank_k parameter, an ANN query option.
+     */
+    int getSaiAnnRerankKFailThreshold();
+
+    /**
+     * @param warn The threshold to warn for the rerank_k parameter, an ANN query option.
+     * @param fail The threshold to prevent setting the rerank_k parameter, an ANN query option.
+     */
+    void setSaiAnnRerankKThreshold(int warn, int fail);
+
     /**
      * @param enabled {@code true} if vector type usage is enabled.
      */
@@ -888,7 +973,7 @@ public interface GuardrailsMBean
     void setNonPartitionRestrictedQueryEnabled(boolean enabled);
 
     /**
-     * @return true if a client warning is emitted for a filtering query with an intersection on mutable columns at a 
+     * @return true if a client warning is emitted for a filtering query with an intersection on mutable columns at a
      *         consistency level requiring coordinator reconciliation
      */
     boolean getIntersectFilteringQueryWarned();
@@ -902,4 +987,40 @@ public interface GuardrailsMBean
     boolean getIntersectFilteringQueryEnabled();
     
     void setIntersectFilteringQueryEnabled(boolean value);
+
+    /**
+     * @return the warning threshold for the offset rows used in SELECT queries
+     * -1 means disabled.
+     */
+    int getOffsetRowsWarnThreshold();
+
+    /**
+     * @return the failure threshold for the offset rows used in SELECT queries
+     * -1 means disabled.
+     */
+    int getOffsetRowsFailThreshold();
+
+    /**
+     * @param warn the warning threshold for the offset rows used in SELECT queries. -1 means disabled.
+     * @param fail the failure threshold for the offset rows used in SELECT queries. -1 means disabled.
+     */
+    void setOffsetRowsThreshold(int warn, int fail);
+
+    /**
+     * @return the warning threshold for the offset rows used in SELECT queries
+     * -1 means disabled.
+     */
+    int getQueryFiltersWarnThreshold();
+
+    /**
+     * @return the failure threshold for the offset rows used in SELECT queries
+     * -1 means disabled.
+     */
+    int getQueryFiltersFailThreshold();
+
+    /**
+     * @param warn the warning threshold for the offset rows used in SELECT queries. -1 means disabled.
+     * @param fail the failure threshold for the offset rows used in SELECT queries. -1 means disabled.
+     */
+    void setQueryFiltersThreshold(int warn, int fail);
 }
diff --git a/src/java/org/apache/cassandra/db/guardrails/Threshold.java b/src/java/org/apache/cassandra/db/guardrails/Threshold.java
index 257ab013b760..91421ce551c3 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Threshold.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Threshold.java
@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.db.guardrails;
 
+import java.util.function.Supplier;
 import java.util.function.ToLongFunction;
 import javax.annotation.Nullable;
 
@@ -33,6 +34,30 @@
  */
 public abstract class Threshold extends Guardrail
 {
+    /**
+     * A {@link Threshold} with both failure and warning thresholds disabled, so that cannot ever be triggered.
+     */
+    public static final Threshold NEVER_TRIGGERED = new Threshold("never_triggered", null, state -> -1L, state -> -1L, null)
+    {
+        @Override
+        protected boolean compare(long value, long threshold)
+        {
+            return false;
+        }
+
+        @Override
+        protected long failValue(ClientState state)
+        {
+            return Long.MAX_VALUE;
+        }
+
+        @Override
+        protected long warnValue(ClientState state)
+        {
+            return Long.MAX_VALUE;
+        }
+    };
+
     protected ToLongFunction<ClientState> warnThreshold;
     protected ToLongFunction<ClientState> failThreshold;
     protected final ErrorMessageProvider messageProvider;
@@ -170,4 +195,94 @@ interface ErrorMessageProvider
          */
         String createMessage(boolean isWarning, String what, String value, String threshold);
     }
+
+    /**
+     * Creates a new {@link GuardedCounter} guarded by this threshold guardrail.
+     *
+     * @param whatFct          a function called when either a warning or failure is triggered by the created counter to
+     *                         describe the value. This is equivalent to the {@code what} argument of {@link #guard} but is a function to
+     *                         allow the output string to be compute lazily (only if a failure/warn ends up being triggered).
+     * @param containsUserData if a warning or failure is triggered by the created counter and the {@code whatFct}
+     *                         is called, indicates whether the create string contains user data. This is the exact equivalent to the
+     *                         similarly named argument of {@link #guard}.
+     * @param clientState      the client state, used to skip the check if the query is internal or is done by a superuser.
+     *                         A {@code null} value means that the check should be done regardless of the query.
+     * @return the newly created guarded counter.
+     */
+    public GuardedCounter newCounter(Supplier<String> whatFct, boolean containsUserData, @Nullable ClientState clientState)
+    {
+        Threshold threshold = enabled(clientState) ? this : NEVER_TRIGGERED;
+        return threshold.new GuardedCounter(whatFct, containsUserData, clientState);
+    }
+
+    /**
+     * A facility for when the value to guard is built incrementally, but we want to trigger failures as soon
+     * as the failure threshold is reached, but only trigger the warning on the final value (and so only if the
+     * failure threshold hasn't also been reached).
+     * <p>
+     * Note that instances are neither thread safe nor reusable.
+     */
+    public class GuardedCounter
+    {
+        private final long warnValue;
+        private final long failValue;
+        private final Supplier<String> what;
+        private final boolean containsUserData;
+
+        private long accumulated;
+
+        private GuardedCounter(Supplier<String> what, boolean containsUserData, ClientState clientState)
+        {
+            // We capture the warn and fail value at the time of the counter construction to ensure we use
+            // stable value during the counter lifetime (and reading a final field is possibly at tad faster).
+            this.warnValue = warnValue(clientState);
+            this.failValue = failValue(clientState);
+            this.what = what;
+            this.containsUserData = containsUserData;
+        }
+
+        /**
+         * The currently accumulated value of the counter.
+         */
+        public long get()
+        {
+            return accumulated;
+        }
+
+        /**
+         * Add the provided increment to the counter, triggering a failure if the counter after this addition
+         * crosses the failure threshold.
+         *
+         * @param increment the increment to add.
+         */
+        public void add(long increment)
+        {
+            accumulated += increment;
+            if (accumulated > failValue)
+            {
+                // Pass any ClientState so GuardrailViolatedException will be thrown by Guardrail#fail
+                ClientState dummyClientState = ClientState.forInternalCalls();
+                triggerFail(accumulated, failValue, what.get(), containsUserData, dummyClientState);
+            }
+        }
+
+        /**
+         * Trigger the warn if the currently accumulated counter value crosses warning threshold and the failure
+         * has not been triggered yet.
+         * <p>
+         * This is generally meant to be called when the guarded value is complete.
+         *
+         * @return {@code true} and trigger a warning if the current counter value is greater than the warning
+         * threshold and less than or equal to the failure threshold, {@code false} otherwise.
+         */
+        public boolean checkAndTriggerWarning()
+        {
+            if (accumulated > warnValue && accumulated <= failValue)
+            {
+                triggerWarn(accumulated, warnValue, what.get(), containsUserData);
+                return true;
+            }
+            return false;
+        }
+    }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/guardrails/UserKeyspaceFilter.java b/src/java/org/apache/cassandra/db/guardrails/UserKeyspaceFilter.java
new file mode 100644
index 000000000000..9f0126cc8e52
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/guardrails/UserKeyspaceFilter.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.guardrails;
+
+import org.apache.cassandra.db.Keyspace;
+
+public interface UserKeyspaceFilter
+{
+    /**
+     * Returns true if the keyspace should be included.
+     */
+    boolean filter(Keyspace keyspaceName);
+}
diff --git a/src/java/org/apache/cassandra/db/guardrails/UserKeyspaceFilterProvider.java b/src/java/org/apache/cassandra/db/guardrails/UserKeyspaceFilterProvider.java
new file mode 100644
index 000000000000..7ab3cffb5b06
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/guardrails/UserKeyspaceFilterProvider.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.guardrails;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.service.ClientState;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_KEYSPACES_FILTER_PROVIDER;
+
+public interface UserKeyspaceFilterProvider
+{
+    UserKeyspaceFilterProvider instance = getCustomProviderClass() == null ?
+                                          new DefaultUserKeyspaceFilterProvider() :
+                                          CustomUserKeyspaceFilterProvider.make(getCustomProviderClass());
+
+    UserKeyspaceFilter get(ClientState clientState);
+
+    @Nullable
+    static String getCustomProviderClass()
+    {
+        return CUSTOM_KEYSPACES_FILTER_PROVIDER.getString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/guardrails/Values.java b/src/java/org/apache/cassandra/db/guardrails/Values.java
index 9504a3d63bbe..d97a3168802c 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Values.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Values.java
@@ -18,12 +18,14 @@
 
 package org.apache.cassandra.db.guardrails;
 
+import java.util.HashSet;
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Sets;
 
 import org.apache.cassandra.service.ClientState;
@@ -117,6 +119,7 @@ public void guard(Set<T> values, Consumer<T> ignoreAction, @Nullable ClientState
         {
             warn(format("Ignoring provided values %s as they are not supported for %s (ignored values are: %s)",
                         toIgnore.stream().sorted().collect(Collectors.toList()), what, ignored));
+            toIgnore = new HashSet<>(toIgnore); // defensive copy as the action may modify the underlying set
             toIgnore.forEach(ignoreAction);
         }
 
@@ -126,4 +129,28 @@ public void guard(Set<T> values, Consumer<T> ignoreAction, @Nullable ClientState
             warn(format("Provided values %s are not recommended for %s (warned values are: %s)",
                         toWarn.stream().sorted().collect(Collectors.toList()), what, warned));
     }
+    
+    // Used by CNDB
+    @VisibleForTesting
+    public Set<T> disallowedValues(Set<T> values, ClientState state)
+    {
+        Set<T> disallowed = disallowedValues.apply(state);
+        return Sets.intersection(values, disallowed);
+    }
+
+    // Used by CNDB
+    @VisibleForTesting
+    public Set<T> ignoredValues(Set<T> values, ClientState state)
+    {
+        Set<T> ignored = ignoredValues.apply(state);
+        return Sets.intersection(values, ignored);
+    }
+
+    // Used by CNDB
+    @VisibleForTesting
+    public Set<T> warnedValues(Set<T> values, ClientState state)
+    {
+        Set<T> warned = warnedValues.apply(state);
+        return Sets.intersection(values, warned);
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/AbstractLogTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/AbstractLogTransaction.java
new file mode 100644
index 000000000000..c2fc7e627883
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/AbstractLogTransaction.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.List;
+
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+/**
+ * A class that tracks sstable files involved in a transaction across sstables:
+ * if the transaction succeeds the old files should be deleted and the new ones kept;
+ * vice-versa if it fails.
+ */
+public abstract class AbstractLogTransaction extends Transactional.AbstractTransactional implements Transactional, LifecycleNewTracker
+{
+    public abstract OperationType type();
+
+    public abstract TimeUUID id();
+
+    public abstract Throwable prepareForObsoletion(Iterable<SSTableReader> readers,
+                                          List<Obsoletion> obsoletions,
+                                          Tracker tracker,
+                                          Throwable accumulate);
+
+    public static class Obsoletion
+    {
+        final SSTableReader reader;
+        final ReaderTidier tidier;
+
+        public Obsoletion(SSTableReader reader, ReaderTidier tidier)
+        {
+            this.reader = reader;
+            this.tidier = tidier;
+        }
+    }
+
+    /**
+     * An interface received by sstable readers ({@link SSTableReader}) when the sstable is marked for obsoletion.
+     * They must call either {@link this#commit()} or {@link this#abort(Throwable)}. If neither method is called then
+     * the parent transaction won't be able to run its own cleanup.
+     * <p/>
+     * Obsoletion may be aborted due to an exception, in which case {@link this#abort(Throwable)} should be called.
+     * Otherwise the sstable reader must call {@link this#commit()} when all the references to the reader have been
+     * released, i.e. when it is OK to delete the sstable files.
+     */
+    public interface ReaderTidier
+    {
+        /**
+         * To be called when all references to the sstable reader have been released and the sstable files can be
+         * deleted.
+         */
+        void commit();
+
+        /**
+         * To be called if the obsoletion is aborted, i.e. if the sstable must be kept after all because the parent
+         * transaction has been aborted.
+         */
+        Throwable abort(Throwable accumulate);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/CompositeLifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/CompositeLifecycleTransaction.java
new file mode 100644
index 000000000000..07f3ad8d1dbc
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/CompositeLifecycleTransaction.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.utils.TimeUUID;
+
+/// Composite lifecycle transaction. This is a wrapper around a lifecycle transaction that allows for multiple partial
+/// operations that comprise the whole transaction. This is used to parallelize compaction operations over individual
+/// output shards where the compaction sources are shared among the operations; in this case we can only release the
+/// shared sources once all operations are complete.
+///
+/// A composite transaction is initialized with a main transaction that will be used to commit the transaction. Each
+/// part of the composite transaction must be registered with the transaction before it is used. The transaction must
+/// be initialized by calling [#completeInitialization()] before any of the processing is allowed to proceed.
+///
+/// The transaction is considered complete when all parts have been committed or aborted. If any part is aborted, the
+/// whole transaction is also aborted ([PartialLifecycleTransaction] will also throw an exception on other parts when
+/// they access it if the composite is already aborted).
+///
+/// When all parts are committed, the full transaction is applied by performing a checkpoint, obsoletion of the
+/// originals if any of the parts requested it, preparation and commit. This may somewhat violate the rules of
+/// transactions as a part that has been committed may actually have no effect if another part is aborted later.
+/// There are also restrictions on the operations that this model can accept, e.g. replacement of sources and partial
+/// checkpointing are not supported (as they are parts of early open which we don't aim to support at this time),
+/// and we consider that all parts will have the same opinion about the obsoletion of the originals.
+public class CompositeLifecycleTransaction
+{
+    protected static final Logger logger = LoggerFactory.getLogger(CompositeLifecycleTransaction.class);
+
+    final LifecycleTransaction mainTransaction;
+    private final AtomicInteger partsToCommitOrAbort;
+    private volatile boolean obsoleteOriginalsRequested;
+    private volatile boolean wasAborted;
+    private volatile boolean initializationComplete;
+    private volatile int partsCount = 0;
+
+    /// Create a composite transaction wrapper over the given transaction. After construction, the individual parts of
+    /// the operation must be registered using [#register] and the composite sealed by calling [#completeInitialization].
+    /// The composite will then track the state of the parts and commit after all of them have committed (respectively
+    /// abort if one aborts but only after waiting for all the other tasks to complete, successfully or not).
+    ///
+    /// To make it easy to recognize the parts of a composite transaction, the given transaction should have an id with
+    /// sequence number 0, and partial transactions should use the id that [#register] returns.
+    public CompositeLifecycleTransaction(LifecycleTransaction mainTransaction)
+    {
+        this.mainTransaction = mainTransaction;
+        this.partsToCommitOrAbort = new AtomicInteger(0);
+        this.wasAborted = false;
+        this.obsoleteOriginalsRequested = false;
+    }
+
+    /// Register one part of the composite transaction. Every part must register itself before the composite transaction
+    /// is initialized and the parts are allowed to proceed.
+    /// @param part the part to register
+    public TimeUUID register(PartialLifecycleTransaction part)
+    {
+        int index = partsToCommitOrAbort.incrementAndGet();
+        return TimeUUID.Generator.withSequence(mainTransaction.opId(), index);
+    }
+
+    /// Complete the initialization of the composite transaction. This must be called before any of the parts are
+    /// executed.
+    public void completeInitialization()
+    {
+        partsCount = partsToCommitOrAbort.get();
+        initializationComplete = true;
+        if (logger.isTraceEnabled())
+            logger.trace("Composite transaction {} initialized with {} parts.", mainTransaction.opIdString(), partsCount);
+    }
+
+    /// Get the number of parts in the composite transaction. 0 if the transaction is not yet initialized.
+    public int partsCount()
+    {
+        return partsCount;
+    }
+
+    /// Request that the original sstables are obsoleted when the transaction is committed. Note that this class has
+    /// an expectation that all parts will have the same opinion about this, and one request will be sufficient to
+    /// trigger obsoletion.
+    public void requestObsoleteOriginals()
+    {
+        obsoleteOriginalsRequested = true;
+    }
+
+    /// Commit a part of the composite transaction. This will trigger the final commit of the whole transaction if it is
+    /// the last part to complete. A part has to commit or abort exactly once.
+    public void commitPart()
+    {
+        partCommittedOrAborted();
+    }
+
+    /// Signal an abort of one part of the transaction. If this is the last part to signal, the whole transaction will
+    /// now abort. Otherwise the composite transaction will wait for the other parts to complete and will abort the
+    /// composite when they all give their commit or abort signal. A part has to commit or abort exactly once.
+    ///
+    /// [PartialLifecycleTransaction] will attempt to abort other parts sooner by throwing an exception when any of its
+    /// methods are called when the composite transaction is already aborted.
+    public void abortPart()
+    {
+        wasAborted = true;
+        partCommittedOrAborted();
+    }
+
+    boolean wasAborted()
+    {
+        return wasAborted;
+    }
+
+    private void partCommittedOrAborted()
+    {
+        if (!initializationComplete)
+            throw new IllegalStateException("Composite transaction used before initialization is complete.");
+        if (partsToCommitOrAbort.decrementAndGet() == 0)
+        {
+            if (wasAborted)
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("Composite transaction {} with {} parts aborted.",
+                                 mainTransaction.opIdString(),
+                                 partsCount);
+
+                mainTransaction.abort();
+            }
+            else
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("Composite transaction {} with {} parts completed{}.",
+                                 mainTransaction.opIdString(),
+                                 partsCount,
+                                 obsoleteOriginalsRequested ? " with obsoletion" : "");
+
+                mainTransaction.checkpoint();
+                if (obsoleteOriginalsRequested)
+                    mainTransaction.obsoleteOriginals();
+                mainTransaction.prepareToCommit();
+                mainTransaction.commit();
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/FailedTransactionDeletionHandler.java b/src/java/org/apache/cassandra/db/lifecycle/FailedTransactionDeletionHandler.java
new file mode 100644
index 000000000000..cc021b64be48
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/FailedTransactionDeletionHandler.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+/**
+ * An interface for retrying failed log transaction deletion
+ */
+public interface FailedTransactionDeletionHandler
+{
+    /**
+     * Rescheduled failed log transaction deletion due to mmap not being finalized or Windows constraint.
+     */
+    void rescheduleFailedDeletions();
+}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
index 134beec11643..80f06780c442 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
@@ -29,9 +29,7 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 
-import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.utils.Throwables;
 
 import static com.google.common.base.Predicates.and;
 import static com.google.common.base.Predicates.equalTo;
@@ -51,7 +49,7 @@ class Helpers
      * really present, and that the items to add are not (unless we're also removing them)
      * @return a new set with the contents of the provided one modified
      */
-    static <T> Set<T> replace(Set<T> original, Set<T> remove, Iterable<T> add)
+    static <T> Set<T> replace(Set<T> original, Set<? extends T> remove, Iterable<? extends T> add)
     {
         return ImmutableSet.copyOf(replace(identityMap(original), remove, add).keySet());
     }
@@ -65,7 +63,7 @@ static <T> Map<T, T> replace(Map<T, T> original, Set<? extends T> remove, Iterab
     {
         // ensure the ones being removed are the exact same ones present
         for (T reader : remove)
-            assert original.get(reader) == reader;
+            assert original.get(reader) == reader : String.format("%s not found in original set: %s", reader, original);
 
         // ensure we don't already contain any we're adding, that we aren't also removing
         assert !any(add, and(not(in(remove)), in(original.keySet()))) : String.format("original:%s remove:%s add:%s", original.keySet(), remove, add);
@@ -118,12 +116,12 @@ static void checkNotReplaced(Iterable<SSTableReader> readers)
             assert !reader.isReplaced();
     }
 
-    static Throwable markObsolete(List<LogTransaction.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable markObsolete(List<AbstractLogTransaction.Obsoletion> obsoletions, Throwable accumulate)
     {
         if (obsoletions == null || obsoletions.isEmpty())
             return accumulate;
 
-        for (LogTransaction.Obsoletion obsoletion : obsoletions)
+        for (AbstractLogTransaction.Obsoletion obsoletion : obsoletions)
         {
             try
             {
@@ -137,33 +135,26 @@ static Throwable markObsolete(List<LogTransaction.Obsoletion> obsoletions, Throw
         return accumulate;
     }
 
-    static Throwable prepareForObsoletion(Iterable<SSTableReader> readers, LogTransaction txnLogs, List<LogTransaction.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable prepareForObsoletion(Iterable<SSTableReader> readers,
+                                          AbstractLogTransaction txnLogs,
+                                          List<AbstractLogTransaction.Obsoletion> obsoletions,
+                                          Tracker tracker,
+                                          Throwable accumulate)
     {
-        Map<SSTable, LogRecord> logRecords = txnLogs.makeRemoveRecords(readers);
-        for (SSTableReader reader : readers)
-        {
-            try
-            {
-                obsoletions.add(new LogTransaction.Obsoletion(reader, txnLogs.obsoleted(reader, logRecords.get(reader))));
-            }
-            catch (Throwable t)
-            {
-                accumulate = Throwables.merge(accumulate, t);
-            }
-        }
-        return accumulate;
+
+        return txnLogs.prepareForObsoletion(readers, obsoletions, tracker, accumulate);
     }
 
-    static Throwable abortObsoletion(List<LogTransaction.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable abortObsoletion(List<AbstractLogTransaction.Obsoletion> obsoletions, Throwable accumulate)
     {
         if (obsoletions == null || obsoletions.isEmpty())
             return accumulate;
 
-        for (LogTransaction.Obsoletion obsoletion : obsoletions)
+        for (AbstractLogTransaction.Obsoletion obsoletion : obsoletions)
         {
             try
             {
-                obsoletion.tidier.abort();
+                obsoletion.tidier.abort(accumulate);
             }
             catch (Throwable t)
             {
diff --git a/src/java/org/apache/cassandra/db/lifecycle/ILifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/ILifecycleTransaction.java
index c014e3865f93..f3e9c9b7c5f1 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/ILifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/ILifecycleTransaction.java
@@ -21,7 +21,11 @@
 import java.util.Collection;
 import java.util.Set;
 
+import com.google.common.collect.Iterables;
+
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Transactional;
 
 public interface ILifecycleTransaction extends Transactional, LifecycleNewTracker
@@ -29,10 +33,37 @@ public interface ILifecycleTransaction extends Transactional, LifecycleNewTracke
     void checkpoint();
     void update(SSTableReader reader, boolean original);
     void update(Collection<SSTableReader> readers, boolean original);
-    public SSTableReader current(SSTableReader reader);
+    SSTableReader current(SSTableReader reader);
     void obsolete(SSTableReader reader);
     void obsoleteOriginals();
     Set<SSTableReader> originals();
     boolean isObsolete(SSTableReader reader);
     boolean isOffline();
+    TimeUUID opId();
+
+    /// Op identifier as a string to use in debug prints. Usually just the opId, with added part information for partial
+    /// transactions.
+    default String opIdString()
+    {
+        return opId().toString();
+    }
+
+    void cancel(SSTableReader removedSSTable);
+
+    default void abort()
+    {
+        Throwables.maybeFail(abort(null));
+    }
+
+    default void commit()
+    {
+        Throwables.maybeFail(commit(null));
+    }
+
+    default SSTableReader onlyOne()
+    {
+        final Set<SSTableReader> originals = originals();
+        assert originals.size() == 1;
+        return Iterables.getFirst(originals, null);
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/ILogAwareFileLister.java b/src/java/org/apache/cassandra/db/lifecycle/ILogAwareFileLister.java
new file mode 100644
index 000000000000..c12a4f18222c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/ILogAwareFileLister.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.BiPredicate;
+
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.util.File;
+
+/**
+ * An interface for listing files in a folder
+ */
+public interface ILogAwareFileLister
+{
+    /**
+     * Listing files that are not removed by log transactions in a folder.
+     *
+     * @param folder The folder to scan
+     * @param filter The filter determines which files the client wants returned
+     * @param onTxnErr The behavior when we fail to list files
+     * @return all files that are not removed by log transactions
+     */
+    List<File> list(Path folder, BiPredicate<File, Directories.FileType> filter, Directories.OnTxnErr onTxnErr);
+}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/ILogFileCleaner.java b/src/java/org/apache/cassandra/db/lifecycle/ILogFileCleaner.java
new file mode 100644
index 000000000000..414012cced57
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/ILogFileCleaner.java
@@ -0,0 +1,42 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import org.apache.cassandra.io.util.File;
+
+/**
+ * Removes any leftovers from unfinished log transactions as indicated by any transaction log files
+ */
+public interface ILogFileCleaner
+{
+    /**
+     * list all log files under given directory
+     *
+     * @param directory directory to scan
+     */
+    void list(File directory);
+
+    /**
+     * Removes any leftovers from unfinished transactions as indicated by any transaction log files that
+     * are found via {@link #list(File)}
+     */
+    boolean removeUnfinishedLeftovers();
+}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/ILogTransactionsFactory.java b/src/java/org/apache/cassandra/db/lifecycle/ILogTransactionsFactory.java
new file mode 100644
index 000000000000..0a0a1b20ee59
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/ILogTransactionsFactory.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOG_TRANSACTIONS_FACTORY;
+
+/**
+ * Factory to create instances used during log transaction processing:
+ * - {@link AbstractLogTransaction}: tracks sstable files invovled in a transastion cross sstable.
+ * - {@link ILogAwareFileLister}: list files which are not removed by log transactions
+ * - {@link ILogFileCleaner}: removes any leftovers from unfinished log transactions
+ * - {@link FailedTransactionDeletionHandler}: retries failed log transaction deletions
+ */
+public interface ILogTransactionsFactory
+{
+    Logger logger = LoggerFactory.getLogger(ILogTransactionsFactory.class);
+
+    ILogTransactionsFactory instance = !LOG_TRANSACTIONS_FACTORY.isPresent()
+                                      ? new LogTransactionsFactory()
+                                      : FBUtilities.construct(LOG_TRANSACTIONS_FACTORY.getString(), "log transactions factory");
+
+    /**
+     * Create {@link AbstractLogTransaction} that tracks sstable files involved in a transaction across sstables:
+     */
+    AbstractLogTransaction createLogTransaction(OperationType operationType,
+                                                TimeUUID uuid,
+                                                TableMetadataRef metadata);
+
+    /**
+     * Create {@link ILogAwareFileLister} that lists files which are not removed by log transactions in a folder.
+     */
+    ILogAwareFileLister createLogAwareFileLister();
+
+    /**
+     * Create {@link ILogFileCleaner} that removes any leftovers from unfinished log transactions as indicated by any transaction log files
+     */
+    ILogFileCleaner createLogFileCleaner();
+
+    /**
+     * Create {@link FailedTransactionDeletionHandler} used to retry failed log transaction deletions
+     */
+    FailedTransactionDeletionHandler createFailedTransactionDeletionHandler();
+}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleNewTracker.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleNewTracker.java
index 9a0785c43f80..f7d616b15360 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleNewTracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleNewTracker.java
@@ -28,11 +28,27 @@
 public interface LifecycleNewTracker
 {
     /**
-     * Called when a new table is about to be created, so that this table can be tracked by a transaction.
+     * Called when a new sstable is about to be created, so that this table can be tracked by a transaction.
      * @param table - the new table to be tracked
      */
     void trackNew(SSTable table);
 
+    /**
+     * Called when a new sstable and its indexes bave been fully written
+     * @param table - the newly written sstable to be tracked
+     */
+    default void trackNewWritten(SSTable table)
+    {
+    }
+
+    /**
+     * Track new index files attached to the given sstable. Used by CNDB to upload new archive file
+     *
+     * @param table on which index files should be tracked
+     */
+    default void trackNewAttachedIndexFiles(SSTable table)
+    {
+    }
 
     /**
      * Called when a new table is no longer required, so that this table can be untracked by a transaction.
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index 20e1d92eafd4..e4c931bf450c 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -24,7 +24,9 @@
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.function.BiPredicate;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -32,9 +34,11 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.Runnables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.compaction.OperationType;
@@ -43,6 +47,8 @@
 import org.apache.cassandra.io.sstable.format.SSTableReader.UniqueIdentifier;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Transactional;
@@ -69,6 +75,7 @@
 import static org.apache.cassandra.db.lifecycle.View.updateCompacting;
 import static org.apache.cassandra.db.lifecycle.View.updateLiveSet;
 import static org.apache.cassandra.utils.Throwables.maybeFail;
+import static org.apache.cassandra.utils.Throwables.merge;
 import static org.apache.cassandra.utils.concurrent.Refs.release;
 import static org.apache.cassandra.utils.concurrent.Refs.selfRefs;
 
@@ -125,9 +132,9 @@ public String toString()
         }
     }
 
-    public final Tracker tracker;
+    private final Tracker tracker;
     // The transaction logs keep track of new and old sstable files
-    private final LogTransaction log;
+    private final AbstractLogTransaction log;
     // the original readers this transaction was opened over, and that it guards
     // (no other transactions may operate over these readers concurrently)
     private final Set<SSTableReader> originals = new HashSet<>();
@@ -144,50 +151,57 @@ public String toString()
     private final State staged = new State();
 
     // the tidier and their readers, to be used for marking readers obsoleted during a commit
-    private List<LogTransaction.Obsoletion> obsoletions;
+    private List<AbstractLogTransaction.Obsoletion> obsoletions;
 
     // commit/rollback hooks
     private List<Runnable> commitHooks = new ArrayList<>();
     private List<Runnable> abortHooks = new ArrayList<>();
 
+    /**
+     * Creates a new unique id that is suitable for a transaction.
+     */
+    public static TimeUUID newId()
+    {
+        return TimeUUID.Generator.nextTimeUUID();
+    }
+
     /**
      * construct a Transaction for use in an offline operation
      */
     public static LifecycleTransaction offline(OperationType operationType, SSTableReader reader)
     {
-        return offline(operationType, singleton(reader));
+        return offline(operationType, reader.metadataRef(), singleton(reader));
     }
 
     /**
      * construct a Transaction for use in an offline operation
      */
-    public static LifecycleTransaction offline(OperationType operationType, Iterable<SSTableReader> readers)
+    public static LifecycleTransaction offline(OperationType operationType, TableMetadataRef metadata, Iterable<SSTableReader> readers)
     {
         // if offline, for simplicity we just use a dummy tracker
-        Tracker dummy = Tracker.newDummyTracker();
+        Tracker dummy = Tracker.newDummyTracker(metadata);
         dummy.addInitialSSTables(readers);
         dummy.apply(updateCompacting(emptySet(), readers));
-        return new LifecycleTransaction(dummy, operationType, readers);
+        return new LifecycleTransaction(dummy, operationType, readers, newId());
     }
 
     /**
      * construct an empty Transaction with no existing readers
      */
-    public static LifecycleTransaction offline(OperationType operationType)
+    public static LifecycleTransaction offline(OperationType operationType, TableMetadataRef metadata)
     {
-        Tracker dummy = Tracker.newDummyTracker();
-        return new LifecycleTransaction(dummy, new LogTransaction(operationType, dummy), Collections.emptyList());
+        Tracker dummy = Tracker.newDummyTracker(metadata);
+        return new LifecycleTransaction(dummy, operationType, Collections.emptyList(), newId());
     }
 
-    LifecycleTransaction(Tracker tracker, OperationType operationType, Iterable<? extends SSTableReader> readers)
-    {
-        this(tracker, new LogTransaction(operationType, tracker), readers);
-    }
-
-    LifecycleTransaction(Tracker tracker, LogTransaction log, Iterable<? extends SSTableReader> readers)
+    @VisibleForTesting
+    public LifecycleTransaction(Tracker tracker,
+                                OperationType operationType,
+                                Iterable<? extends SSTableReader> readers,
+                                TimeUUID uuid)
     {
         this.tracker = tracker;
-        this.log = log;
+        this.log = ILogTransactionsFactory.instance.createLogTransaction(operationType, uuid, tracker.metadata);
         for (SSTableReader reader : readers)
         {
             originals.add(reader);
@@ -196,7 +210,7 @@ public static LifecycleTransaction offline(OperationType operationType)
         }
     }
 
-    public LogTransaction log()
+    public AbstractLogTransaction log()
     {
         return log;
     }
@@ -204,7 +218,7 @@ public LogTransaction log()
     @Override //LifecycleNewTracker
     public OperationType opType()
     {
-        return log.type();
+        return log.opType();
     }
 
     public TimeUUID opId()
@@ -222,7 +236,10 @@ public void doPrepare()
 
         // prepare for compaction obsolete readers as long as they were part of the original set
         // since those that are not original are early readers that share the same desc with the finals
-        maybeFail(prepareForObsoletion(filterIn(logged.obsolete, originals), log, obsoletions = new ArrayList<>(), null));
+        maybeFail(prepareForObsoletion(filterIn(logged.obsolete, originals), log, obsoletions = new ArrayList<>(), tracker, null));
+
+        // This needs to be called after checkpoint and having prepared the obsoletions because it will upload the deletion
+        // marks in CNDB
         log.prepareToCommit();
     }
 
@@ -250,7 +267,7 @@ public Throwable doCommit(Throwable accumulate)
         accumulate = tracker.updateSizeTracking(logged.obsolete, logged.update, accumulate);
         accumulate = runOnCommitHooks(accumulate);
         accumulate = release(selfRefs(logged.obsolete), accumulate);
-        accumulate = tracker.notifySSTablesChanged(originals, logged.update, log.type(), accumulate);
+        accumulate = tracker.notifySSTablesChanged(originals, logged.update, log.opType(), Optional.of(log.id()), accumulate);
 
         return accumulate;
     }
@@ -273,7 +290,7 @@ public Throwable doAbort(Throwable accumulate)
         Iterable<SSTableReader> obsolete = filterOut(concatUniq(staged.update, logged.update), originals);
         logger.trace("Obsoleting {}", obsolete);
 
-        accumulate = prepareForObsoletion(obsolete, log, obsoletions = new ArrayList<>(), accumulate);
+        accumulate = prepareForObsoletion(obsolete, log, obsoletions = new ArrayList<>(), tracker, accumulate);
         // it's safe to abort even if committed, see maybeFail in doCommit() above, in this case it will just report
         // a failure to abort, which is useful information to have for debug
         accumulate = log.abort(accumulate);
@@ -283,7 +300,7 @@ public Throwable doAbort(Throwable accumulate)
         List<SSTableReader> restored = restoreUpdatedOriginals();
         List<SSTableReader> invalid = Lists.newArrayList(Iterables.concat(logged.update, logged.obsolete));
         accumulate = tracker.apply(updateLiveSet(logged.update, restored), accumulate);
-        accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, accumulate);
+        accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, Optional.of(log.id()), accumulate);
         // setReplaced immediately preceding versions that have not been obsoleted
         accumulate = setReplaced(logged.update, accumulate);
         accumulate = runOnAbortooks(accumulate);
@@ -336,6 +353,12 @@ public boolean isOffline()
         return tracker.isDummy();
     }
 
+    @VisibleForTesting
+    public void unsafeClose()
+    {
+        log.close();
+    }
+
     /**
      * call when a consistent batch of changes is ready to be made atomically visible
      * these will be exposed in the Tracker atomically, or an exception will be thrown; in this case
@@ -359,6 +382,10 @@ private Throwable checkpoint(Throwable accumulate)
         // check the current versions of the readers we're replacing haven't somehow been replaced by someone else
         checkNotReplaced(filterIn(toUpdate, staged.update));
 
+        // notify the tracker of the new readers are about to be added and visible
+        if (!fresh.isEmpty())
+            accumulate = merge(accumulate, tracker.notifyAdding(fresh, null, null, opType(), Optional.of(opId())));
+
         // ensure any new readers are in the compacting set, since we aren't done with them yet
         // and don't want anyone else messing with them
         // apply atomically along with updating the live set of readers
@@ -552,7 +579,7 @@ public LifecycleTransaction split(Collection<SSTableReader> readers)
             originals.remove(reader);
             marked.remove(reader);
         }
-        return new LifecycleTransaction(tracker, log.type(), readers);
+        return new LifecycleTransaction(tracker, log.opType(), readers, newId());
     }
 
     /**
@@ -572,11 +599,12 @@ private Throwable unmarkCompacting(Set<SSTableReader> unmark, Throwable accumula
         // when the CFS is invalidated, it will call unreferenceSSTables().  However, unreferenceSSTables only deals
         // with sstables that aren't currently being compacted.  If there are ongoing compactions that finish or are
         // interrupted after the CFS is invalidated, those sstables need to be unreferenced as well, so we do that here.
-        accumulate = tracker.dropSSTablesIfInvalid(accumulate);
+        accumulate = tracker.dropOrUnloadSSTablesIfInvalid("for transaction " + log.id(), accumulate);
         return accumulate;
     }
 
     // convenience method for callers that know only one sstable is involved in the transaction
+    // overridden to avoid defensive copying
     public SSTableReader onlyOne()
     {
         assert originals.size() == 1;
@@ -591,6 +619,18 @@ public void trackNew(SSTable table)
         log.trackNew(table);
     }
 
+    @Override
+    public void trackNewWritten(SSTable table)
+    {
+        log.trackNewWritten(table);
+    }
+
+    @Override
+    public void trackNewAttachedIndexFiles(SSTable table)
+    {
+        log.trackNewAttachedIndexFiles(table);
+    }
+
     @Override
     public void untrackNew(SSTable table)
     {
@@ -599,12 +639,36 @@ public void untrackNew(SSTable table)
 
     public static boolean removeUnfinishedLeftovers(ColumnFamilyStore cfs)
     {
-        return LogTransaction.removeUnfinishedLeftovers(cfs.getDirectories().getCFDirectories());
+        return removeUnfinishedLeftovers(cfs.getDirectories().getCFDirectories());
     }
 
+    /**
+     * Removes any leftovers from unifinished transactions as indicated by any transaction log files that
+     * are found in the table directories. This means that any old sstable files for transactions that were committed,
+     * or any new sstable files for transactions that were aborted or still in progress, should be removed *if
+     * it is safe to do so*. Refer to the checks in LogFile.verify for further details on the safety checks
+     * before removing transaction leftovers and refer to the comments at the beginning of this file or in NEWS.txt
+     * for further details on transaction logs.
+     *
+     * This method is called on startup and by the standalone sstableutil tool when the cleanup option is specified,
+     * @see org.apache.cassandra.tools.StandaloneSSTableUtil
+     *
+     * @return true if the leftovers of all transaction logs found were removed, false otherwise.
+     *
+     */
     public static boolean removeUnfinishedLeftovers(TableMetadata metadata)
     {
-        return LogTransaction.removeUnfinishedLeftovers(metadata);
+        return removeUnfinishedLeftovers(new Directories(metadata).getCFDirectories());
+    }
+
+    public static boolean removeUnfinishedLeftovers(List<File> directories)
+    {
+        // List<Path> directories
+        ILogFileCleaner cleaner = ILogTransactionsFactory.instance.createLogFileCleaner();
+        for (File dir : directories)
+            cleaner.list(dir);
+
+        return cleaner.removeUnfinishedLeftovers();
     }
 
     /**
@@ -621,7 +685,7 @@ public static boolean removeUnfinishedLeftovers(TableMetadata metadata)
      */
     public static List<File> getFiles(Path folder, BiPredicate<File, Directories.FileType> filter, Directories.OnTxnErr onTxnErr)
     {
-        return new LogAwareFileLister(folder, filter, onTxnErr).list();
+        return ILogTransactionsFactory.instance.createLogAwareFileLister().list(folder, filter, onTxnErr);
     }
 
     /**
@@ -630,7 +694,7 @@ public static List<File> getFiles(Path folder, BiPredicate<File, Directories.Fil
      */
     public static void rescheduleFailedDeletions()
     {
-        LogTransaction.rescheduleFailedDeletions();
+        ILogTransactionsFactory.instance.createFailedTransactionDeletionHandler().rescheduleFailedDeletions();
     }
 
     /**
@@ -639,7 +703,7 @@ public static void rescheduleFailedDeletions()
      */
     public static void waitForDeletions()
     {
-        LogTransaction.waitForDeletions();
+        FBUtilities.waitOnFuture(ScheduledExecutors.nonPeriodicTasks.schedule(Runnables.doNothing(), 0, TimeUnit.MILLISECONDS));
     }
 
     // a class representing the current state of the reader within this transaction, encoding the actions both logged
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogAwareFileLister.java b/src/java/org/apache/cassandra/db/lifecycle/LogAwareFileLister.java
index 120f9bc42a7e..654d9fdbf158 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogAwareFileLister.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogAwareFileLister.java
@@ -1,5 +1,4 @@
 /*
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -8,15 +7,13 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package org.apache.cassandra.db.lifecycle;
 
@@ -24,53 +21,38 @@
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
 import java.util.function.BiPredicate;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.io.util.File;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.util.File;
 
-import static org.apache.cassandra.db.Directories.*;
+import static org.apache.cassandra.db.Directories.FileType;
+import static org.apache.cassandra.db.Directories.OnTxnErr;
 
 /**
  * A class for listing files in a folder.
  */
-final class LogAwareFileLister
+final class LogAwareFileLister implements ILogAwareFileLister
 {
     private static final Logger logger = LoggerFactory.getLogger(LogAwareFileLister.class);
 
-    // The folder to scan
-    private final Path folder;
-
-    // The filter determines which files the client wants returned
-    private final BiPredicate<File, FileType> filter; //file, file type
-
-    // The behavior when we fail to list files
-    private final OnTxnErr onTxnErr;
-
-    // The unfiltered result
-    NavigableMap<File, Directories.FileType> files = new TreeMap<>();
-
-    @VisibleForTesting
-    LogAwareFileLister(Path folder, BiPredicate<File, FileType> filter, OnTxnErr onTxnErr)
-    {
-        this.folder = folder;
-        this.filter = filter;
-        this.onTxnErr = onTxnErr;
-    }
-
-    public List<File> list()
+    @Override
+    public List<File> list(Path folder, BiPredicate<File, FileType> filter, OnTxnErr onTxnErr)
     {
         try
         {
-            return innerList();
+            return innerList(folder, filter, onTxnErr);
         }
         catch (Throwable t)
         {
@@ -78,8 +60,11 @@ public List<File> list()
         }
     }
 
-    List<File> innerList() throws Throwable
+    protected List<File> innerList(Path folder, BiPredicate<File, FileType> filter, OnTxnErr onTxnErr) throws Throwable
     {
+        // The unfiltered result
+        NavigableMap<File, Directories.FileType> files = new TreeMap<>();
+
         list(Files.newDirectoryStream(folder))
         .stream()
         .filter((f) -> !LogFile.isLogFile(f))
@@ -92,7 +77,7 @@ List<File> innerList() throws Throwable
         list(Files.newDirectoryStream(folder, '*' + LogFile.EXT))
         .stream()
         .filter(LogFile::isLogFile)
-        .forEach(this::classifyFiles);
+        .forEach(txnFile -> classifyFiles(folder, txnFile, onTxnErr, files));
 
         // Finally we apply the user filter before returning our result
         return files.entrySet().stream()
@@ -120,36 +105,36 @@ static List<File> list(DirectoryStream<Path> stream) throws IOException
      * We read txn log files, if we fail we throw only if the user has specified
      * OnTxnErr.THROW, else we log an error and apply the txn log anyway
      */
-    void classifyFiles(File txnFile)
+    void classifyFiles(Path folder, File txnFile, OnTxnErr onTxnErr, NavigableMap<File, Directories.FileType> files)
     {
         try (LogFile txn = LogFile.make(txnFile))
         {
-            readTxnLog(txn);
-            classifyFiles(txn);
+            readTxnLog(txn, onTxnErr);
+            classifyFiles(folder, txn, onTxnErr, files);
             files.put(txnFile, FileType.TXN_LOG);
         }
     }
 
-    void readTxnLog(LogFile txn)
+    void readTxnLog(LogFile txn, OnTxnErr onTxnErr)
     {
         if (!txn.verify() && onTxnErr == OnTxnErr.THROW)
             throw new LogTransaction.CorruptTransactionLogException("Some records failed verification. See earlier in log for details.", txn);
     }
 
-    void classifyFiles(LogFile txnFile)
+    void classifyFiles(Path folder, LogFile txnFile, OnTxnErr onTxnErr, NavigableMap<File, Directories.FileType> files)
     {
         Map<LogRecord, Set<File>> oldFiles = txnFile.getFilesOfType(folder, files.navigableKeySet(), LogRecord.Type.REMOVE);
         Map<LogRecord, Set<File>> newFiles = txnFile.getFilesOfType(folder, files.navigableKeySet(), LogRecord.Type.ADD);
 
         if (txnFile.completed())
         { // last record present, filter regardless of disk status
-            setTemporary(txnFile, oldFiles.values(), newFiles.values());
+            setTemporary(txnFile, oldFiles.values(), newFiles.values(), files);
             return;
         }
 
         if (allFilesPresent(oldFiles))
         {  // all old files present, transaction is in progress, this will filter as aborted
-            setTemporary(txnFile, oldFiles.values(), newFiles.values());
+            setTemporary(txnFile, oldFiles.values(), newFiles.values(), files);
             return;
         }
 
@@ -161,11 +146,11 @@ void classifyFiles(LogFile txnFile)
             return;
 
         // otherwise read the file again to see if it is completed now
-        readTxnLog(txnFile);
+        readTxnLog(txnFile, onTxnErr);
 
         if (txnFile.completed())
         { // if after re-reading the txn is completed then filter accordingly
-            setTemporary(txnFile, oldFiles.values(), newFiles.values());
+            setTemporary(txnFile, oldFiles.values(), newFiles.values(), files);
             return;
         }
 
@@ -194,11 +179,11 @@ private static boolean allFilesPresent(Map<LogRecord, Set<File>> oldFiles)
                         .findFirst().isPresent();
     }
 
-    private void setTemporary(LogFile txnFile, Collection<Set<File>> oldFiles, Collection<Set<File>> newFiles)
+    private void setTemporary(LogFile txnFile, Collection<Set<File>> oldFiles, Collection<Set<File>> newFiles, NavigableMap<File, Directories.FileType> files)
     {
         Collection<Set<File>> temporary = txnFile.committed() ? oldFiles : newFiles;
         temporary.stream()
                  .flatMap(Set::stream)
-                 .forEach((f) -> this.files.put(f, FileType.TEMPORARY));
+                 .forEach((f) -> files.put(f, FileType.TEMPORARY));
     }
 }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
index 9decc248b92c..8584c6cf3107 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
@@ -65,7 +65,9 @@
  * of unfinished leftovers when a transaction is completed, or aborted, or when
  * we clean up on start-up.
  *
- * @see LogTransaction
+ * Note: this is used by {@link LogTransaction}
+ *
+ * @see AbstractLogTransaction
  */
 @NotThreadSafe
 final class LogFile implements AutoCloseable
@@ -374,8 +376,7 @@ private LogRecord makeRecord(Type type, SSTable table, LogRecord record)
     private void maybeCreateReplica(SSTable sstable)
     {
         File directory = sstable.descriptor.directory;
-        String fileName = StringUtils.join(directory, File.pathSeparator(), getFileName());
-        replicas.maybeCreateReplica(directory, fileName, onDiskRecords);
+        replicas.maybeCreateReplica(directory, getFileName(), onDiskRecords);
     }
 
     void addRecord(LogRecord record)
@@ -515,7 +516,7 @@ List<File> getFiles()
     }
 
     @VisibleForTesting
-    List<String> getFilePaths()
+    List<File> getFilePaths()
     {
         return replicas.getFilePaths();
     }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogFileCleaner.java b/src/java/org/apache/cassandra/db/lifecycle/LogFileCleaner.java
new file mode 100644
index 000000000000..27ad59492810
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogFileCleaner.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.util.File;
+
+final class LogFileCleaner implements ILogFileCleaner
+{
+    private static final Logger logger = LoggerFactory.getLogger(LogFileCleaner.class);
+
+    // This maps a transaction log file name to a list of physical files. Each sstable
+    // can have multiple directories and a transaction is trakced by identical transaction log
+    // files, one per directory. So for each transaction file name we can have multiple
+    // physical files.
+    Map<String, List<File>> files = new HashMap<>();
+
+    @Override
+    public void list(File directory)
+    {
+        Arrays.stream(directory.tryList(LogFile::isLogFile)).forEach(this::add);
+    }
+
+    void add(File file)
+    {
+        List<File> filesByName = files.get(file.name());
+        if (filesByName == null)
+        {
+            filesByName = new ArrayList<>();
+            files.put(file.name(), filesByName);
+        }
+
+        filesByName.add(file);
+    }
+
+    @Override
+    public boolean removeUnfinishedLeftovers()
+    {
+        return files.entrySet()
+                    .stream()
+                    .map(LogFileCleaner::removeUnfinishedLeftovers)
+                    .allMatch(Predicate.isEqual(true));
+    }
+
+    static boolean removeUnfinishedLeftovers(Map.Entry<String, List<File>> entry)
+    {
+        try(LogFile txn = LogFile.make(entry.getKey(), entry.getValue()))
+        {
+            logger.info("Verifying logfile transaction {}", txn);
+            if (txn.verify())
+            {
+                Throwable failure = txn.removeUnfinishedLeftovers(null);
+                if (failure != null)
+                {
+                    logger.error("Failed to remove unfinished transaction leftovers for transaction log {}",
+                                 txn.toString(true), failure);
+                    return false;
+                }
+
+                return true;
+            }
+            else
+            {
+                logger.error("Unexpected disk state: failed to read transaction log {}, " +
+                             "check logs before last shutdown for any errors, and ensure txn log files were not edited manually.",
+                             txn.toString(true));
+                return false;
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java b/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
index 34fd0daf91bf..81bc5b6203a4 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
@@ -49,6 +49,8 @@
 /**
  * A decoded line in a transaction log file replica.
  *
+ * Note: this is used by {@link LogTransaction}
+ *
  * @see LogReplica and LogFile.
  */
 final class LogRecord
@@ -161,8 +163,8 @@ public static LogRecord makeAbort(long updateTime)
 
     public static LogRecord make(Type type, SSTable table)
     {
-        String absoluteTablePath = absolutePath(table.descriptor.baseFile());
-        return make(type, getExistingFiles(absoluteTablePath), table.getAllFilePaths().size(), absoluteTablePath);
+        String absoluteTablePath = table.descriptor.baseFileUri() + Component.SEPARATOR;
+        return make(type, getExistingFiles(absoluteTablePath), table.getComponentSize(), absoluteTablePath);
     }
 
     public static Map<SSTable, LogRecord> make(Type type, Iterable<SSTableReader> tables)
@@ -170,7 +172,7 @@ public static Map<SSTable, LogRecord> make(Type type, Iterable<SSTableReader> ta
         // contains a mapping from sstable absolute path (everything up until the 'Data'/'Index'/etc part of the filename) to the sstable
         Map<String, SSTable> absolutePaths = new HashMap<>();
         for (SSTableReader table : tables)
-            absolutePaths.put(absolutePath(table.descriptor.baseFile()), table);
+            absolutePaths.put(table.descriptor.baseFileUri() + Component.SEPARATOR, table);
 
         // maps sstable base file name to the actual files on disk
         Map<String, List<File>> existingFiles = getExistingFiles(absolutePaths.keySet());
@@ -180,16 +182,11 @@ public static Map<SSTable, LogRecord> make(Type type, Iterable<SSTableReader> ta
             List<File> filesOnDisk = entry.getValue();
             String baseFileName = entry.getKey();
             SSTable sstable = absolutePaths.get(baseFileName);
-            records.put(sstable, make(type, filesOnDisk, sstable.getAllFilePaths().size(), baseFileName));
+            records.put(sstable, make(type, filesOnDisk, sstable.getComponentSize(), baseFileName));
         }
         return records;
     }
 
-    private static String absolutePath(File baseFile)
-    {
-        return baseFile.withSuffix(String.valueOf(Component.separator)).canonicalPath();
-    }
-
     public LogRecord withExistingFiles(List<File> existingFiles)
     {
         return make(type, existingFiles, 0, absolutePath.get());
@@ -293,7 +290,7 @@ private String format()
 
     public static List<File> getExistingFiles(String absoluteFilePath)
     {
-        File file = new File(absoluteFilePath);
+        File file = new File(PathUtils.getPath(absoluteFilePath));
         File[] files = file.parent().tryList((dir, name) -> name.startsWith(file.name()));
         // files may be null if the directory does not exist yet, e.g. when tracking new files
         return files == null ? Collections.emptyList() : Arrays.asList(files);
@@ -312,10 +309,10 @@ public static Map<String, List<File>> getExistingFiles(Set<String> absoluteFileP
         Map<File, TreeSet<String>> dirToFileNamePrefix = new HashMap<>();
         for (String absolutePath : absoluteFilePaths)
         {
-            Path fullPath = new File(absolutePath).toPath();
-            Path path = fullPath.getParent();
-            if (path != null)
-                dirToFileNamePrefix.computeIfAbsent(new File(path), (k) -> new TreeSet<>()).add(fullPath.getFileName().toString());
+            File file = new File(PathUtils.getPath(absolutePath));
+            File parent = file.parent();
+            if (parent != null)
+                dirToFileNamePrefix.computeIfAbsent(parent, (k) -> new TreeSet<>()).add(file.name());
         }
 
         BiPredicate<File, String> ff = (dir, name) -> {
@@ -327,8 +324,8 @@ public static Map<String, List<File>> getExistingFiles(Set<String> absoluteFileP
             String baseName = dirSet.floor(name);
             if (baseName != null && name.startsWith(baseName))
             {
-                String absolutePath = new File(dir, baseName).path();
-                fileMap.computeIfAbsent(absolutePath, k -> new ArrayList<>()).add(new File(dir, name));
+                String absolutePath = dir.resolve(baseName).toUri().toString();
+                fileMap.computeIfAbsent(absolutePath, k -> new ArrayList<>()).add(dir.resolve(name));
             }
             return false;
         };
@@ -353,7 +350,7 @@ String fileName()
 
     boolean isInFolder(Path folder)
     {
-        return absolutePath.isPresent() && PathUtils.isContained(folder, new File(absolutePath.get()).toPath());
+        return absolutePath.isPresent() && PathUtils.isContained(folder, new File(PathUtils.getPath(absolutePath.get())).toPath());
     }
 
     String absolutePath()
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogReplica.java b/src/java/org/apache/cassandra/db/lifecycle/LogReplica.java
index 073ac7c61c16..f84b98e17c00 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogReplica.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogReplica.java
@@ -31,7 +31,7 @@
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.INativeLibrary;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORE_MISSING_NATIVE_FILE_HINTS;
 
@@ -44,6 +44,8 @@
  * partial records in case we crashed after writing to one replica but
  * before compliting the write to another replica.
  *
+ * Note: this is used by {@link LogTransaction}
+ *
  * @see LogFile
  */
 final class LogReplica implements AutoCloseable
@@ -57,7 +59,7 @@ final class LogReplica implements AutoCloseable
 
     static LogReplica create(File directory, String fileName)
     {
-        int folderFD = NativeLibrary.tryOpenDirectory(directory.path());
+        int folderFD = INativeLibrary.instance.tryOpenDirectory(directory);
         if (folderFD == -1  && REQUIRE_FD)
         {
             if (DatabaseDescriptor.isClientInitialized())
@@ -70,12 +72,12 @@ static LogReplica create(File directory, String fileName)
             }
         }
 
-        return new LogReplica(new File(fileName), folderFD);
+        return new LogReplica(directory.resolve(fileName), folderFD);
     }
 
     static LogReplica open(File file)
     {
-        int folderFD = NativeLibrary.tryOpenDirectory(file.parent().path());
+        int folderFD = INativeLibrary.instance.tryOpenDirectory(file.parent());
         if (folderFD == -1)
         {
             if (DatabaseDescriptor.isClientInitialized())
@@ -141,7 +143,7 @@ void syncDirectory()
         try
         {
             if (directoryDescriptor >= 0)
-                NativeLibrary.trySync(directoryDescriptor);
+                INativeLibrary.instance.trySync(directoryDescriptor);
         }
         catch (FSError e)
         {
@@ -165,7 +167,7 @@ public void close()
     {
         if (directoryDescriptor >= 0)
         {
-            NativeLibrary.tryCloseFD(directoryDescriptor);
+            INativeLibrary.instance.tryCloseFD(directoryDescriptor);
             directoryDescriptor = -1;
         }
     }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogReplicaSet.java b/src/java/org/apache/cassandra/db/lifecycle/LogReplicaSet.java
index 5076a960d3c4..38db1aecd96b 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogReplicaSet.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogReplicaSet.java
@@ -29,6 +29,7 @@
 import javax.annotation.concurrent.NotThreadSafe;
 
 import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.io.util.File;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,11 +42,13 @@
  * A set of log replicas. This class mostly iterates over replicas when writing or reading,
  * ensuring consistency among them and hiding replication details from LogFile.
  *
+ * Note: this is used by {@link LogTransaction}
+ *
  * @see LogReplica
  * @see LogFile
  */
 @NotThreadSafe
-public class LogReplicaSet implements AutoCloseable
+final class LogReplicaSet implements AutoCloseable
 {
     private static final Logger logger = LoggerFactory.getLogger(LogReplicaSet.class);
 
@@ -223,13 +226,28 @@ void printContentsWithAnyErrors(StringBuilder str)
      */
     void append(LogRecord record)
     {
-        Throwable err = Throwables.perform(null, replicas().stream().map(r -> () -> r.append(record)));
+        Throwable err = null;
+        int failed = 0;
+        for (LogReplica replica : replicas())
+        {
+            try
+            {
+                replica.append(record);
+            }
+            catch (Throwable t)
+            {
+                logger.warn("Failed to add record to a replica: {}", t.getMessage());
+                err = Throwables.merge(err, t);
+                failed++;
+            }
+        }
+
         if (err != null)
         {
-            if (!record.isFinal() || err.getSuppressed().length == replicas().size() -1)
+            if (!record.isFinal() || failed == replicas().size())
                 Throwables.maybeFail(err);
 
-            logger.error("Failed to add record '{}' to some replicas '{}'", record, this);
+            logger.error("Failed to add record '{}' to some replicas '{}'", record, this, err);
         }
     }
 
@@ -267,8 +285,8 @@ List<File> getFiles()
     }
 
     @VisibleForTesting
-    List<String> getFilePaths()
+    List<File> getFilePaths()
     {
-        return replicas().stream().map(LogReplica::file).map(File::path).collect(Collectors.toList());
+        return replicas().stream().map(LogReplica::file).collect(Collectors.toList());
     }
 }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
index 6d248742ce79..eff77687f794 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
@@ -22,26 +22,20 @@
 import java.io.PrintStream;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Predicate;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.Runnables;
-
-import com.codahale.metrics.Counter;
+import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.Counter;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LogRecord.Type;
@@ -52,16 +46,11 @@
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.concurrent.RefCounted;
-import org.apache.cassandra.utils.concurrent.Transactional;
-
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
 /**
  * IMPORTANT: When this object is involved in a transactional graph, and is not encapsulated in a LifecycleTransaction,
@@ -98,7 +87,7 @@
  *
  * See CASSANDRA-7066 for full details.
  */
-class LogTransaction extends Transactional.AbstractTransactional implements Transactional
+final class LogTransaction extends AbstractLogTransaction
 {
     private static final Logger logger = LoggerFactory.getLogger(LogTransaction.class);
 
@@ -117,7 +106,6 @@ public CorruptTransactionLogException(String message, LogFile txnFile)
         }
     }
 
-    private final Tracker tracker;
     private final LogFile txnFile;
     // We need an explicit lock because the transaction tidier cannot store a reference to the transaction
     private final Object lock;
@@ -125,17 +113,14 @@ public CorruptTransactionLogException(String message, LogFile txnFile)
     // Deleting sstables is tricky because the mmapping might not have been finalized yet.
     // Additionally, we need to make sure to delete the data file first, so on restart the others
     // will be recognized as GCable.
-    private static final Queue<Runnable> failedDeletions = new ConcurrentLinkedQueue<>();
+    protected static final Queue<Runnable> failedDeletions = new ConcurrentLinkedQueue<>();
 
-    LogTransaction(OperationType opType)
+    LogTransaction(OperationType opType, TimeUUID uuid)
     {
-        this(opType, null);
-    }
+        Preconditions.checkNotNull(opType);
+        Preconditions.checkNotNull(uuid);
 
-    LogTransaction(OperationType opType, Tracker tracker)
-    {
-        this.tracker = tracker;
-        this.txnFile = new LogFile(opType, nextTimeUUID());
+        this.txnFile = new LogFile(opType, uuid);
         this.lock = new Object();
         this.selfRef = new Ref<>(this, new TransactionTidier(txnFile, lock));
 
@@ -146,7 +131,8 @@ public CorruptTransactionLogException(String message, LogFile txnFile)
     /**
      * Track a reader as new.
      **/
-    void trackNew(SSTable table)
+    @Override
+    public void trackNew(SSTable table)
     {
         synchronized (lock)
         {
@@ -160,7 +146,8 @@ void trackNew(SSTable table)
     /**
      * Stop tracking a reader as new.
      */
-    void untrackNew(SSTable table)
+    @Override
+    public void untrackNew(SSTable table)
     {
         synchronized (lock)
         {
@@ -168,19 +155,25 @@ void untrackNew(SSTable table)
         }
     }
 
+    @Override
+    public OperationType opType()
+    {
+        return txnFile.type();
+    }
+
     /**
      * helper method for tests, creates the remove records per sstable
      */
     @VisibleForTesting
-    SSTableTidier obsoleted(SSTableReader sstable)
+    ReaderTidier obsoleted(SSTableReader sstable)
     {
-        return obsoleted(sstable, LogRecord.make(Type.REMOVE, sstable));
+        return obsoleted(sstable, LogRecord.make(Type.REMOVE, sstable), null);
     }
 
     /**
      * Schedule a reader for deletion as soon as it is fully unreferenced.
      */
-    SSTableTidier obsoleted(SSTableReader reader, LogRecord logRecord)
+    ReaderTidier obsoleted(SSTableReader reader, LogRecord logRecord, @Nullable Tracker tracker)
     {
         synchronized (lock)
         {
@@ -192,7 +185,7 @@ SSTableTidier obsoleted(SSTableReader reader, LogRecord logRecord)
                 if (txnFile.contains(Type.REMOVE, reader, logRecord))
                     throw new IllegalArgumentException();
 
-                return new SSTableTidier(reader, true, this);
+                return new SSTableTidier(reader, true, this, tracker);
             }
 
             txnFile.addRecord(logRecord);
@@ -200,7 +193,7 @@ SSTableTidier obsoleted(SSTableReader reader, LogRecord logRecord)
             if (tracker != null)
                 tracker.notifyDeleting(reader);
 
-            return new SSTableTidier(reader, false, this);
+            return new SSTableTidier(reader, false, this, tracker);
         }
     }
 
@@ -212,17 +205,40 @@ Map<SSTable, LogRecord> makeRemoveRecords(Iterable<SSTableReader> sstables)
         }
     }
 
-
-    OperationType type()
+    @Override
+    public OperationType type()
     {
         return txnFile.type();
     }
 
-    TimeUUID id()
+    @Override
+    public TimeUUID id()
     {
         return txnFile.id();
     }
 
+    @Override
+    public Throwable prepareForObsoletion(Iterable<SSTableReader> readers,
+                                          List<AbstractLogTransaction.Obsoletion> obsoletions,
+                                          Tracker tracker,
+                                          Throwable accumulate)
+    {
+
+        Map<SSTable, LogRecord> logRecords = makeRemoveRecords(readers);
+        for (SSTableReader reader : readers)
+        {
+            try
+            {
+                obsoletions.add(new AbstractLogTransaction.Obsoletion(reader, obsoleted(reader, logRecords.get(reader), tracker)));
+            }
+            catch (Throwable t)
+            {
+                accumulate = Throwables.merge(accumulate, t);
+            }
+        }
+        return accumulate;
+    }
+
     @VisibleForTesting
     LogFile txnFile()
     {
@@ -236,7 +252,7 @@ List<File> logFiles()
     }
 
     @VisibleForTesting
-    List<String> logFilePaths()
+    List<File> logFilePaths()
     {
         return txnFile.getFilePaths();
     }
@@ -246,7 +262,7 @@ static void delete(File file)
         try
         {
             if (!StorageService.instance.isDaemonSetupCompleted())
-                logger.info("Unfinished transaction log, deleting {} ", file);
+                logger.debug("Unfinished transaction log, deleting {} ", file);
             else if (logger.isTraceEnabled())
                 logger.trace("Deleting {}", file);
 
@@ -335,25 +351,13 @@ public void run()
         }
     }
 
-    static class Obsoletion
-    {
-        final SSTableReader reader;
-        final SSTableTidier tidier;
-
-        Obsoletion(SSTableReader reader, SSTableTidier tidier)
-        {
-            this.reader = reader;
-            this.tidier = tidier;
-        }
-    }
-
     /**
      * The SSTableReader tidier. When a reader is fully released and no longer referenced
      * by any one, we run this. It keeps a reference to the parent transaction and releases
      * it when done, so that the final transaction cleanup can run when all obsolete readers
      * are released.
      */
-    public static class SSTableTidier implements Runnable
+    private static class SSTableTidier implements ReaderTidier
     {
         // must not retain a reference to the SSTableReader, else leak detection cannot kick in
         private final Descriptor desc;
@@ -361,15 +365,17 @@ public static class SSTableTidier implements Runnable
         private final boolean wasNew;
         private final Object lock;
         private final Ref<LogTransaction> parentRef;
+        private final boolean onlineTxn;
         private final Counter totalDiskSpaceUsed;
 
-        public SSTableTidier(SSTableReader referent, boolean wasNew, LogTransaction parent)
+        public SSTableTidier(SSTableReader referent, boolean wasNew, LogTransaction parent, Tracker tracker)
         {
             this.desc = referent.descriptor;
             this.sizeOnDisk = referent.bytesOnDisk();
             this.wasNew = wasNew;
             this.lock = parent.lock;
             this.parentRef = parent.selfRef.tryRef();
+            this.onlineTxn = tracker != null && !tracker.isDummy();
 
             if (this.parentRef == null)
                 throw new IllegalStateException("Transaction already completed");
@@ -377,16 +383,15 @@ public SSTableTidier(SSTableReader referent, boolean wasNew, LogTransaction pare
             // While the parent cfs may be dropped in the interim of us taking a reference to this and using it, at worst
             // we'll be updating a metric for a now dropped ColumnFamilyStore. We do not hold a reference to the tracker or
             // cfs as that would create a strong ref loop and violate our ability to do leak detection.
-            totalDiskSpaceUsed = parent.tracker != null && parent.tracker.cfstore != null ?
-                                 parent.tracker.cfstore.metric.totalDiskSpaceUsed :
+            totalDiskSpaceUsed = tracker != null && tracker.cfstore != null ?
+                                 tracker.cfstore.metric.totalDiskSpaceUsed :
                                  null;
         }
 
-        public void run()
+        @Override
+        public void commit()
         {
-            // While this may be a dummy tracker w/out information in the metrics table, we attempt to delete regardless
-            // and allow the delete to silently fail if this is an invalid ks + cf combination at time of tidy run.
-            if (DatabaseDescriptor.isDaemonInitialized())
+            if (onlineTxn && DatabaseDescriptor.supportsSSTableReadMeter())
                 SystemKeyspace.clearSSTableReadMeter(desc.ksname, desc.cfname, desc.id);
 
             synchronized (lock)
@@ -394,9 +399,8 @@ public void run()
                 try
                 {
                     // If we can't successfully delete the DATA component, set the task to be retried later: see TransactionTidier
-
                     if (logger.isTraceEnabled())
-                        logger.trace("Tidier running for old sstable {}", desc);
+                        logger.trace("Tidier running for old sstable {}", desc.baseFileUri());
 
                     if (!desc.fileFor(Components.DATA).exists() && !wasNew)
                         logger.error("SSTableTidier ran with no existing data file for an sstable that was not new");
@@ -406,7 +410,7 @@ public void run()
                 catch (Throwable t)
                 {
                     logger.error("Failed deletion for {}, we'll retry after GC and on server restart", desc);
-                    failedDeletions.add(this);
+                    failedDeletions.add(this::commit);
                     return;
                 }
 
@@ -421,11 +425,12 @@ public void run()
             }
         }
 
-        public void abort()
+        @Override
+        public Throwable abort(Throwable accumulate)
         {
             synchronized (lock)
             {
-                parentRef.release();
+                return Throwables.perform(accumulate, parentRef::release);
             }
         }
     }
@@ -438,11 +443,6 @@ static void rescheduleFailedDeletions()
             ScheduledExecutors.nonPeriodicTasks.submit(task);
     }
 
-    static void waitForDeletions()
-    {
-        FBUtilities.waitOnFuture(ScheduledExecutors.nonPeriodicTasks.schedule(Runnables.doNothing(), 0, TimeUnit.MILLISECONDS));
-    }
-
     @VisibleForTesting
     Throwable complete(Throwable accumulate)
     {
@@ -479,90 +479,4 @@ protected Throwable doAbort(Throwable accumulate)
     }
 
     protected void doPrepare() { }
-
-    /**
-     * Removes any leftovers from unifinished transactions as indicated by any transaction log files that
-     * are found in the table directories. This means that any old sstable files for transactions that were committed,
-     * or any new sstable files for transactions that were aborted or still in progress, should be removed *if
-     * it is safe to do so*. Refer to the checks in LogFile.verify for further details on the safety checks
-     * before removing transaction leftovers and refer to the comments at the beginning of this file or in NEWS.txt
-     * for further details on transaction logs.
-     *
-     * This method is called on startup and by the standalone sstableutil tool when the cleanup option is specified,
-     * @see org.apache.cassandra.tools.StandaloneSSTableUtil
-     *
-     * @return true if the leftovers of all transaction logs found were removed, false otherwise.
-     *
-     */
-    static boolean removeUnfinishedLeftovers(TableMetadata metadata)
-    {
-        return removeUnfinishedLeftovers(new Directories(metadata).getCFDirectories());
-    }
-
-    @VisibleForTesting
-    static boolean removeUnfinishedLeftovers(List<File> directories)
-    {
-        LogFilesByName logFiles = new LogFilesByName();
-        directories.forEach(logFiles::list);
-        return logFiles.removeUnfinishedLeftovers();
-    }
-
-    private static final class LogFilesByName
-    {
-        // This maps a transaction log file name to a list of physical files. Each sstable
-        // can have multiple directories and a transaction is trakced by identical transaction log
-        // files, one per directory. So for each transaction file name we can have multiple
-        // physical files.
-        Map<String, List<File>> files = new HashMap<>();
-
-        void list(File directory)
-        {
-            Arrays.stream(directory.tryList(LogFile::isLogFile)).forEach(this::add);
-        }
-
-        void add(File file)
-        {
-            List<File> filesByName = files.get(file.name());
-            if (filesByName == null)
-            {
-                filesByName = new ArrayList<>();
-                files.put(file.name(), filesByName);
-            }
-
-            filesByName.add(file);
-        }
-
-        boolean removeUnfinishedLeftovers()
-        {
-            return files.entrySet()
-                        .stream()
-                        .map(LogFilesByName::removeUnfinishedLeftovers)
-                        .allMatch(Predicate.isEqual(true));
-        }
-
-        static boolean removeUnfinishedLeftovers(Map.Entry<String, List<File>> entry)
-        {
-            try(LogFile txn = LogFile.make(entry.getKey(), entry.getValue()))
-            {
-                logger.info("Verifying logfile transaction {}", txn);
-                if (txn.verify())
-                {
-                    Throwable failure = txn.removeUnfinishedLeftovers(null);
-                    if (failure != null)
-                    {
-                        logger.error("Failed to remove unfinished transaction leftovers for transaction log {}",
-                                     txn.toString(true), failure);
-                        return false;
-                    }
-
-                    return true;
-                }
-                else
-                {
-                    logger.error("Unexpected disk state: failed to read transaction log {}", txn.toString(true));
-                    return false;
-                }
-            }
-        }
-    }
 }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogTransactionsFactory.java b/src/java/org/apache/cassandra/db/lifecycle/LogTransactionsFactory.java
new file mode 100644
index 000000000000..2a078c136fc6
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogTransactionsFactory.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.TimeUUID;
+
+final class LogTransactionsFactory implements ILogTransactionsFactory
+{
+    @Override
+    public AbstractLogTransaction createLogTransaction(OperationType operationType, TimeUUID uuid, TableMetadataRef metadata)
+    {
+        logger.debug("Creating a transaction for {} on {}", operationType, metadata);
+        return new LogTransaction(operationType, uuid);
+    }
+
+    @Override
+    public ILogAwareFileLister createLogAwareFileLister()
+    {
+        return new LogAwareFileLister();
+    }
+
+    @Override
+    public ILogFileCleaner createLogFileCleaner()
+    {
+        return new LogFileCleaner();
+    }
+
+    @Override
+    public FailedTransactionDeletionHandler createFailedTransactionDeletionHandler()
+    {
+        return LogTransaction::rescheduleFailedDeletions;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/PartialLifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/PartialLifecycleTransaction.java
new file mode 100644
index 000000000000..77702d243efe
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/PartialLifecycleTransaction.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
+
+/// Partial lifecycle transaction. This works together with a CompositeLifecycleTransaction to allow for multiple
+/// tasks using a shared transaction to be committed or aborted together. This is used to parallelize compaction
+/// operations over the same sources. See [CompositeLifecycleTransaction] for more details.
+///
+/// This class takes care of synchronizing various operations on the shared transaction, making sure that an abort
+/// or commit signal is given exactly once (provided that this partial transaction is closed), and throwing an exception
+/// when progress is made when the transaction was already aborted by another part.
+public class PartialLifecycleTransaction implements ILifecycleTransaction
+{
+    final CompositeLifecycleTransaction composite;
+    final ILifecycleTransaction mainTransaction;
+    final AtomicBoolean committedOrAborted = new AtomicBoolean(false);
+    final TimeUUID id;
+
+    public PartialLifecycleTransaction(CompositeLifecycleTransaction composite)
+    {
+        this.composite = composite;
+        this.mainTransaction = composite.mainTransaction;
+        this.id = composite.register(this);
+    }
+
+    public void checkpoint()
+    {
+        // don't do anything, composite will checkpoint at end
+    }
+
+    private RuntimeException earlyOpenUnsupported()
+    {
+        throw new UnsupportedOperationException("PartialLifecycleTransaction does not support early opening of SSTables");
+    }
+
+    public void update(SSTableReader reader, boolean original)
+    {
+        throwIfCompositeAborted();
+        if (original)
+            throw earlyOpenUnsupported();
+
+        synchronized (mainTransaction)
+        {
+            mainTransaction.update(reader, original);
+        }
+    }
+
+    public void update(Collection<SSTableReader> readers, boolean original)
+    {
+        throwIfCompositeAborted();
+        if (original)
+            throw earlyOpenUnsupported();
+
+        synchronized (mainTransaction)
+        {
+            mainTransaction.update(readers, original);
+        }
+    }
+
+    public SSTableReader current(SSTableReader reader)
+    {
+        synchronized (mainTransaction)
+        {
+            return mainTransaction.current(reader);
+        }
+    }
+
+    public void obsolete(SSTableReader reader)
+    {
+        earlyOpenUnsupported();
+    }
+
+    public void obsoleteOriginals()
+    {
+        composite.requestObsoleteOriginals();
+    }
+
+    public Set<SSTableReader> originals()
+    {
+        return mainTransaction.originals();
+    }
+
+    public boolean isObsolete(SSTableReader reader)
+    {
+        throw earlyOpenUnsupported();
+    }
+
+    private boolean markCommittedOrAborted()
+    {
+        return committedOrAborted.compareAndSet(false, true);
+    }
+
+    /// Commit the transaction part. Because this is a part of a composite transaction, the actual commit will be
+    /// carried out only after all parts have committed.
+    public Throwable commit(Throwable accumulate)
+    {
+        Throwables.maybeFail(accumulate); // we must be called with a null accumulate
+        if (markCommittedOrAborted())
+            composite.commitPart();
+        else
+            throw new IllegalStateException("Partial transaction already committed or aborted.");
+        return null;
+    }
+
+    public Throwable abort(Throwable accumulate)
+    {
+        Throwables.maybeFail(accumulate); // we must be called with a null accumulate
+        if (markCommittedOrAborted())
+            composite.abortPart();
+        else
+            throw new IllegalStateException("Partial transaction already committed or aborted.");
+        return null;
+    }
+
+    private void throwIfCompositeAborted()
+    {
+        if (composite.wasAborted())
+            throw new AbortedException("Transaction aborted, likely by another partial operation.");
+    }
+
+    public void prepareToCommit()
+    {
+        if (committedOrAborted.get())
+            throw new IllegalStateException("Partial transaction already committed or aborted.");
+
+        throwIfCompositeAborted();
+        // nothing else to do, the composite transaction will perform the preparation when all parts are done
+    }
+
+    public void close()
+    {
+        if (markCommittedOrAborted())   // close should abort if not committed
+            composite.abortPart();
+    }
+
+    public void trackNew(SSTable table)
+    {
+        throwIfCompositeAborted();
+        synchronized (mainTransaction)
+        {
+            mainTransaction.trackNew(table);
+        }
+    }
+
+    @Override
+    public void trackNewWritten(SSTable table)
+    {
+        throwIfCompositeAborted();
+        synchronized (mainTransaction)
+        {
+            mainTransaction.trackNewWritten(table);
+        }
+    }
+
+    @Override
+    public void trackNewAttachedIndexFiles(SSTable table)
+    {
+        throwIfCompositeAborted();
+        synchronized (mainTransaction)
+        {
+            mainTransaction.trackNewAttachedIndexFiles(table);
+        }
+    }
+
+    public void untrackNew(SSTable table)
+    {
+        synchronized (mainTransaction)
+        {
+            mainTransaction.untrackNew(table);
+        }
+    }
+
+    public OperationType opType()
+    {
+        return mainTransaction.opType();
+    }
+
+    public boolean isOffline()
+    {
+        return mainTransaction.isOffline();
+    }
+
+    @Override
+    public TimeUUID opId()
+    {
+        return id;
+    }
+
+    @Override
+    public String opIdString()
+    {
+        return String.format("%s (%d/%d)", id, TimeUUID.Generator.sequence(id), composite.partsCount());
+    }
+
+    @Override
+    public void cancel(SSTableReader removedSSTable)
+    {
+        synchronized (mainTransaction)
+        {
+            mainTransaction.cancel(removedSSTable);
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return opIdString();
+    }
+
+    public static class AbortedException extends RuntimeException
+    {
+        public AbortedException(String message)
+        {
+            super(message);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java b/src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java
index 91005d39dd91..f08d4c36a214 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java
@@ -27,6 +27,7 @@
 import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.utils.Interval;
 import org.apache.cassandra.utils.IntervalTree;
@@ -50,11 +51,11 @@ public static SSTableIntervalTree build(Iterable<SSTableReader> sstables)
         return new SSTableIntervalTree(buildIntervals(sstables));
     }
 
-    public static List<Interval<PartitionPosition, SSTableReader>> buildIntervals(Iterable<SSTableReader> sstables)
+    public static <S extends CompactionSSTable> List<Interval<PartitionPosition, S>> buildIntervals(Iterable<? extends S> sstables)
     {
-        List<Interval<PartitionPosition, SSTableReader>> intervals = new ArrayList<>(Iterables.size(sstables));
-        for (SSTableReader sstable : sstables)
-            intervals.add(Interval.<PartitionPosition, SSTableReader>create(sstable.getFirst(), sstable.getLast(), sstable));
+        List<Interval<PartitionPosition, S>> intervals = new ArrayList<>(Iterables.size(sstables));
+        for (S sstable : sstables)
+            intervals.add(Interval.create(sstable.getFirst(), sstable.getLast(), sstable));
         return intervals;
     }
 }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/SSTableSet.java b/src/java/org/apache/cassandra/db/lifecycle/SSTableSet.java
index 07a3b2b4999c..c1ccc3446a38 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/SSTableSet.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/SSTableSet.java
@@ -28,5 +28,6 @@ public enum SSTableSet
     CANONICAL,
     // returns the live versions of all sstables, i.e. including partially written sstables
     LIVE,
+    // returns the non-compacting sstables, i.e. the difference between live and compacting ones
     NONCOMPACTING
 }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index f5cedf38e877..a131b27c716a 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -21,9 +21,12 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
@@ -38,10 +41,10 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.metrics.StorageMetrics;
@@ -52,13 +55,15 @@
 import org.apache.cassandra.notifications.MemtableRenewedNotification;
 import org.apache.cassandra.notifications.MemtableSwitchedNotification;
 import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableAddingNotification;
 import org.apache.cassandra.notifications.SSTableDeletingNotification;
 import org.apache.cassandra.notifications.SSTableListChangedNotification;
-import org.apache.cassandra.notifications.SSTableMetadataChanged;
 import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
 import org.apache.cassandra.notifications.TruncationNotification;
+import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 
 import static com.google.common.base.Predicates.and;
@@ -87,50 +92,76 @@ public class Tracker
     private static final Logger logger = LoggerFactory.getLogger(Tracker.class);
 
     private final List<INotificationConsumer> subscribers = new CopyOnWriteArrayList<>();
+    private final List<INotificationConsumer> lateSubscribers = new CopyOnWriteArrayList<>();
 
     public final ColumnFamilyStore cfstore;
+    public final TableMetadataRef metadata;
     final AtomicReference<View> view;
     public final boolean loadsstables;
 
     /**
-     * @param columnFamilyStore
+     * @param columnFamilyStore column family store for the table
      * @param memtable Initial Memtable. Can be null.
      * @param loadsstables true to indicate to load SSTables (TODO: remove as this is only accessed from 2i)
      */
     public Tracker(ColumnFamilyStore columnFamilyStore, Memtable memtable, boolean loadsstables)
     {
-        this.cfstore = columnFamilyStore;
+        this.cfstore = Objects.requireNonNull(columnFamilyStore);
+        this.metadata = columnFamilyStore.metadata;
         this.view = new AtomicReference<>();
         this.loadsstables = loadsstables;
         this.reset(memtable);
     }
 
-    public static Tracker newDummyTracker()
+    /**
+     * @param metadata metadata reference for the table
+     * @param memtable Initial Memtable. Can be null.
+     * @param loadsstables true to indicate to load SSTables (TODO: remove as this is only accessed from 2i)
+     */
+    public Tracker(TableMetadataRef metadata, Memtable memtable, boolean loadsstables)
+    {
+        this.cfstore = null;
+        this.metadata = Objects.requireNonNull(metadata);
+        this.view = new AtomicReference<>();
+        this.loadsstables = loadsstables;
+        this.reset(memtable);
+    }
+
+    public static Tracker newDummyTracker(TableMetadataRef metadata)
     {
-        return new Tracker(null, null, false);
+        return new Tracker(metadata, null, false);
     }
 
     public LifecycleTransaction tryModify(SSTableReader sstable, OperationType operationType)
     {
-        return tryModify(singleton(sstable), operationType);
+        return tryModify(singleton(sstable), operationType, LifecycleTransaction.newId());
+    }
+
+    public LifecycleTransaction tryModify(Iterable<? extends SSTableReader> sstables,
+                                          OperationType operationType)
+    {
+        return tryModify(sstables, operationType, LifecycleTransaction.newId());
     }
 
     /**
      * @return a Transaction over the provided sstables if we are able to mark the given @param sstables as compacted, before anyone else
      */
-    public LifecycleTransaction tryModify(Iterable<? extends SSTableReader> sstables, OperationType operationType)
+    public LifecycleTransaction tryModify(Iterable<? extends SSTableReader> sstables,
+                                          OperationType operationType,
+                                          TimeUUID uuid)
     {
         if (Iterables.isEmpty(sstables))
-            return new LifecycleTransaction(this, operationType, sstables);
+            return new LifecycleTransaction(this, operationType, sstables, uuid);
         if (null == apply(permitCompacting(sstables), updateCompacting(emptySet(), sstables)))
             return null;
-        return new LifecycleTransaction(this, operationType, sstables);
+        return new LifecycleTransaction(this, operationType, sstables, uuid);
     }
 
 
     // METHODS FOR ATOMICALLY MODIFYING THE VIEW
 
-    Pair<View, View> apply(Function<View, View> function)
+    @VisibleForTesting
+    public Pair<View, View> apply(Function<View, View> function)
     {
         return apply(Predicates.alwaysTrue(), function);
     }
@@ -217,7 +248,7 @@ Throwable updateSizeTracking(Iterable<SSTableReader> oldSSTables, Iterable<SSTab
         return accumulate;
     }
 
-    public void updateLiveDiskSpaceUsed(long adjustment)
+    public void updateSizeTracking(long adjustment)
     {
         cfstore.metric.liveDiskSpaceUsed.inc(adjustment);
         cfstore.metric.totalDiskSpaceUsed.inc(adjustment);
@@ -227,12 +258,12 @@ public void updateLiveDiskSpaceUsed(long adjustment)
 
     public void addInitialSSTables(Iterable<SSTableReader> sstables)
     {
-        addSSTablesInternal(sstables, true, false, true);
+        addSSTablesInternal(sstables, OperationType.INITIAL_LOAD, true, false, true);
     }
 
     public void addInitialSSTablesWithoutUpdatingSize(Iterable<SSTableReader> sstables)
     {
-        addSSTablesInternal(sstables, true, false, false);
+        addSSTablesInternal(sstables, OperationType.INITIAL_LOAD, true, false, false);
     }
 
     public void updateInitialSSTableSize(Iterable<SSTableReader> sstables)
@@ -240,16 +271,18 @@ public void updateInitialSSTableSize(Iterable<SSTableReader> sstables)
         maybeFail(updateSizeTracking(emptySet(), sstables, null));
     }
 
-    public void addSSTables(Iterable<SSTableReader> sstables)
+    public void addSSTables(Iterable<SSTableReader> sstables, OperationType operationType)
     {
-        addSSTablesInternal(sstables, false, true, true);
+        addSSTablesInternal(sstables, operationType, false, true, true);
     }
 
     private void addSSTablesInternal(Iterable<SSTableReader> sstables,
+                                     OperationType operationType,
                                      boolean isInitialSSTables,
                                      boolean maybeIncrementallyBackup,
                                      boolean updateSize)
     {
+        notifyAdding(sstables, operationType);
         if (!isDummy())
             setupOnline(sstables);
         apply(updateLiveSet(emptySet(), sstables));
@@ -257,7 +290,7 @@ private void addSSTablesInternal(Iterable<SSTableReader> sstables,
             maybeFail(updateSizeTracking(emptySet(), sstables, null));
         if (maybeIncrementallyBackup)
             maybeIncrementallyBackup(sstables);
-        notifyAdded(sstables, isInitialSSTables);
+        notifyAdded(sstables, operationType, isInitialSSTables);
     }
 
     /** (Re)initializes the tracker, purging all references. */
@@ -271,10 +304,22 @@ public void reset(Memtable memtable)
                           SSTableIntervalTree.empty()));
     }
 
-    public Throwable dropSSTablesIfInvalid(Throwable accumulate)
+    public Throwable dropOrUnloadSSTablesIfInvalid(String message, @Nullable Throwable accumulate)
     {
         if (!isDummy() && !cfstore.isValid())
-            accumulate = dropSSTables(accumulate);
+        {
+            ColumnFamilyStore.STATUS status = cfstore.status();
+            if (status.isInvalidAndShouldDropData())
+            {
+                logger.info("Dropping sstables for invalidated table {} with status {} {}", metadata.toString(), status, message);
+                return dropSSTables(accumulate);
+            }
+            else
+            {
+                logger.info("Unloading sstables for invalidated table {} with status {} {}", metadata.toString(), status, message);
+                return unloadSSTables(accumulate);
+            }
+        }
         return accumulate;
     }
 
@@ -285,7 +330,7 @@ public void dropSSTables()
 
     public Throwable dropSSTables(Throwable accumulate)
     {
-        return dropSSTables(Predicates.alwaysTrue(), OperationType.UNKNOWN, accumulate);
+        return dropSSTables(Predicates.alwaysTrue(), OperationType.DROP_TABLE, accumulate);
     }
 
     /**
@@ -293,7 +338,12 @@ public Throwable dropSSTables(Throwable accumulate)
      */
     public Throwable dropSSTables(final Predicate<SSTableReader> remove, OperationType operationType, Throwable accumulate)
     {
-        try (LogTransaction txnLogs = new LogTransaction(operationType, this))
+        logger.debug("Dropping sstables for {} with operation {}: {}",
+                     metadata.name, operationType, accumulate == null ? "null" : accumulate.getMessage());
+
+        try (AbstractLogTransaction txnLogs = ILogTransactionsFactory.instance.createLogTransaction(operationType,
+                                                                                                    LifecycleTransaction.newId(),
+                                                                                                    metadata))
         {
             Pair<View, View> result = apply(view -> {
                 Set<SSTableReader> toremove = copyOf(filter(view.sstables, and(remove, notIn(view.compacting))));
@@ -305,8 +355,8 @@ public Throwable dropSSTables(final Predicate<SSTableReader> remove, OperationTy
 
             // It is important that any method accepting/returning a Throwable never throws an exception, and does its best
             // to complete the instructions given to it
-            List<LogTransaction.Obsoletion> obsoletions = new ArrayList<>();
-            accumulate = prepareForObsoletion(removed, txnLogs, obsoletions, accumulate);
+            List<AbstractLogTransaction.Obsoletion> obsoletions = new ArrayList<>();
+            accumulate = prepareForObsoletion(removed, txnLogs, obsoletions, this, accumulate);
             try
             {
                 txnLogs.finish();
@@ -316,12 +366,32 @@ public Throwable dropSSTables(final Predicate<SSTableReader> remove, OperationTy
                     accumulate = updateSizeTracking(removed, emptySet(), accumulate);
                     accumulate = release(selfRefs(removed), accumulate);
                     // notifySSTablesChanged -> LeveledManifest.promote doesn't like a no-op "promotion"
-                    accumulate = notifySSTablesChanged(removed, Collections.emptySet(), txnLogs.type(), accumulate);
+                    accumulate = notifySSTablesChanged(removed, Collections.emptySet(), txnLogs.opType(), Optional.of(txnLogs.id()), accumulate);
                 }
             }
             catch (Throwable t)
             {
-                accumulate = abortObsoletion(obsoletions, accumulate);
+                logger.error("Failed to commit transaction for obsoleting sstables of {}", metadata.name, t);
+                Throwable err = abortObsoletion(obsoletions, null);
+                if (err == null && cfstore != null && cfstore.isValid())
+                {
+                    // if the obsoletions were cancelled and the table is still valid, i.e. not dropped, restore the sstables since they are valid, and for CNDB they are in etcd as well
+                    err = apply(updateLiveSet(emptySet(), removed), accumulate);
+                }
+                else if (cfstore != null && !cfstore.isValid())
+                {
+                    // if the table is invalid, i.e. dropped, send in the notifications anyway because otherwise CNDB etcd does not get updated
+                    err = notifySSTablesChanged(removed, Collections.emptySet(), txnLogs.opType(), Optional.of(txnLogs.id()), err);
+                }
+                else
+                {
+                    // cfstore should always be != null and either valid or not, so we get here only in case err != null
+                    logger.error("Failed to abort obsoletions for {}, some sstables will be missing from liveset", metadata.name, err);
+                }
+
+                if (err != null)
+                    accumulate = Throwables.merge(accumulate, err);
+
                 accumulate = Throwables.merge(accumulate, t);
             }
         }
@@ -330,9 +400,30 @@ public Throwable dropSSTables(final Predicate<SSTableReader> remove, OperationTy
             accumulate = Throwables.merge(accumulate, t);
         }
 
+        logger.debug("Sstables for {} dropped with operation {}: {}",
+                     metadata.name, operationType, accumulate == null ? "null" : accumulate.getMessage());
         return accumulate;
     }
 
+    /**
+     * Unload all sstables from current tracker without deleting files
+     */
+    public void unloadSSTables()
+    {
+        maybeFail(unloadSSTables(null));
+    }
+
+    public Throwable unloadSSTables(@Nullable Throwable accumulate)
+    {
+        Pair<View, View> result = apply(view -> {
+            Set<SSTableReader> toUnload = copyOf(filter(view.sstables, notIn(view.compacting)));
+            return updateLiveSet(toUnload, emptySet()).apply(view);
+        });
+
+        // compacting sstables will be cleaned up by their transaction in {@link LifecycleTransaction#unmarkCompacting}
+        Set<SSTableReader> toRelease = Sets.difference(result.left.sstables, result.right.sstables);
+        return release(selfRefs(toRelease), accumulate);
+    }
 
     /**
      * Removes every SSTable in the directory from the Tracker's view.
@@ -340,7 +431,7 @@ public Throwable dropSSTables(final Predicate<SSTableReader> remove, OperationTy
      */
     public void removeUnreadableSSTables(final File directory)
     {
-        maybeFail(dropSSTables(reader -> reader.descriptor.directory.equals(directory), OperationType.UNKNOWN, null));
+        maybeFail(dropSSTables(reader -> reader.descriptor.directory.equals(directory), OperationType.REMOVE_UNREADEABLE, null));
     }
 
 
@@ -391,7 +482,7 @@ public void markFlushing(Memtable memtable)
         apply(View.markFlushing(memtable));
     }
 
-    public void replaceFlushed(Memtable memtable, Iterable<SSTableReader> sstables)
+    public void replaceFlushed(Memtable memtable, Iterable<SSTableReader> sstables, Optional<TimeUUID> operationId)
     {
         assert !isDummy();
         if (Iterables.isEmpty(sstables))
@@ -406,19 +497,20 @@ public void replaceFlushed(Memtable memtable, Iterable<SSTableReader> sstables)
         // back up before creating a new Snapshot (which makes the new one eligible for compaction)
         maybeIncrementallyBackup(sstables);
 
+        Throwable fail;
+        fail = notifyAdding(sstables, memtable, null, OperationType.FLUSH, operationId);
+
         apply(View.replaceFlushed(memtable, sstables));
 
-        Throwable fail;
-        fail = updateSizeTracking(emptySet(), sstables, null);
+        fail = updateSizeTracking(emptySet(), sstables, fail);
 
         // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
-        fail = notifyAdded(sstables, false, memtable, fail);
+        fail = notifyAdded(sstables, OperationType.FLUSH, operationId, false, memtable, fail);
 
-        // make sure index sees flushed index files before dicarding memtable index
+        // make sure SAI sees newly flushed index files before discarding memtable index
         notifyDiscarded(memtable);
 
-        if (!isDummy() && !cfstore.isValid())
-            dropSSTables();
+        fail = dropOrUnloadSSTablesIfInvalid("during flush", fail);
 
         maybeFail(fail);
     }
@@ -432,14 +524,26 @@ public Set<SSTableReader> getCompacting()
         return view.get().compacting;
     }
 
-    public Iterable<SSTableReader> getUncompacting()
+    public Iterable<SSTableReader> getNoncompacting()
     {
         return view.get().select(SSTableSet.NONCOMPACTING);
     }
 
-    public Iterable<SSTableReader> getUncompacting(Iterable<SSTableReader> candidates)
+    public <S extends CompactionSSTable> Iterable<S> getNoncompacting(Iterable<S> candidates)
     {
-        return view.get().getUncompacting(candidates);
+        return view.get().getNoncompacting(candidates);
+    }
+
+    public Set<SSTableReader> getLiveSSTables()
+    {
+        return view.get().liveSSTables();
+    }
+
+    // used by CNDB
+    @Nullable
+    public SSTableReader getLiveSSTable(String filename)
+    {
+        return view.get().getLiveSSTable(filename);
     }
 
     public void maybeIncrementallyBackup(final Iterable<SSTableReader> sstables)
@@ -456,79 +560,53 @@ public void maybeIncrementallyBackup(final Iterable<SSTableReader> sstables)
 
     // NOTIFICATION
 
-    Throwable notifySSTablesChanged(Collection<SSTableReader> removed, Collection<SSTableReader> added, OperationType compactionType, Throwable accumulate)
+    public Throwable notifySSTablesChanged(Collection<SSTableReader> removed, Collection<SSTableReader> added, OperationType operationType, Optional<TimeUUID> operationId, Throwable accumulate)
     {
-        INotification notification = new SSTableListChangedNotification(added, removed, compactionType);
-        for (INotificationConsumer subscriber : subscribers)
-        {
-            try
-            {
-                subscriber.handleNotification(notification, this);
-            }
-            catch (Throwable t)
-            {
-                accumulate = merge(accumulate, t);
-            }
-        }
-        return accumulate;
+        return notify(new SSTableListChangedNotification(added, removed, operationType, operationId), accumulate);
     }
 
-    Throwable notifyAdded(Iterable<SSTableReader> added, boolean isInitialSSTables, Memtable memtable, Throwable accumulate)
+    Throwable notifyAdded(Iterable<SSTableReader> added, OperationType operationType, Optional<TimeUUID> operationId, boolean isInitialSSTables, Memtable memtable, Throwable accumulate)
     {
         INotification notification;
         if (!isInitialSSTables)
-            notification = new SSTableAddedNotification(added, memtable);
+            notification = new SSTableAddedNotification(added, memtable, operationType, operationId);
         else
             notification = new InitialSSTableAddedNotification(added);
 
-        for (INotificationConsumer subscriber : subscribers)
-        {
-            try
-            {
-                subscriber.handleNotification(notification, this);
-            }
-            catch (Throwable t)
-            {
-                accumulate = merge(accumulate, t);
-            }
-        }
-        return accumulate;
+        return notify(notification, accumulate);
     }
 
-    void notifyAdded(Iterable<SSTableReader> added, boolean isInitialSSTables)
+    Throwable notifyAdding(Iterable<SSTableReader> added, @Nullable Memtable memtable, Throwable accumulate, OperationType type, Optional<TimeUUID> operationId)
     {
-        maybeFail(notifyAdded(added, isInitialSSTables, null, null));
+        return notify(new SSTableAddingNotification(added, memtable, type, operationId), accumulate);
     }
 
-    public void notifySSTableRepairedStatusChanged(Collection<SSTableReader> repairStatusesChanged)
+    public void notifyAdding(Iterable<SSTableReader> added, OperationType operationType)
     {
-        if (repairStatusesChanged.isEmpty())
-            return;
-        INotification notification = new SSTableRepairStatusChanged(repairStatusesChanged);
-        for (INotificationConsumer subscriber : subscribers)
-            subscriber.handleNotification(notification, this);
+        maybeFail(notifyAdding(added, null, null, operationType, Optional.empty()));
     }
 
-    public void notifySSTableMetadataChanged(SSTableReader levelChanged, StatsMetadata oldMetadata)
+    @VisibleForTesting
+    public void notifyAdded(Iterable<SSTableReader> added, OperationType operationType, boolean isInitialSSTables)
     {
-        INotification notification = new SSTableMetadataChanged(levelChanged, oldMetadata);
-        for (INotificationConsumer subscriber : subscribers)
-            subscriber.handleNotification(notification, this);
+        maybeFail(notifyAdded(added, operationType, Optional.empty(), isInitialSSTables, null, null));
+    }
 
+    public void notifySSTableRepairedStatusChanged(Collection<SSTableReader> repairStatusesChanged)
+    {
+        if (repairStatusesChanged.isEmpty())
+            return;
+        notify(new SSTableRepairStatusChanged(repairStatusesChanged));
     }
 
     public void notifyDeleting(SSTableReader deleting)
     {
-        INotification notification = new SSTableDeletingNotification(deleting);
-        for (INotificationConsumer subscriber : subscribers)
-            subscriber.handleNotification(notification, this);
+        notify(new SSTableDeletingNotification(deleting));
     }
 
-    public void notifyTruncated(long truncatedAt)
+    public void notifyTruncated(CommitLogPosition replayAfter, long truncatedAt)
     {
-        INotification notification = new TruncationNotification(truncatedAt);
-        for (INotificationConsumer subscriber : subscribers)
-            subscriber.handleNotification(notification, this);
+        notify(new TruncationNotification(replayAfter, truncatedAt));
     }
 
     public void notifyRenewed(Memtable renewed)
@@ -547,30 +625,72 @@ public void notifyDiscarded(Memtable discarded)
     }
 
     private void notify(INotification notification)
+    {
+        maybeFail(notify(notification, null));
+    }
+
+    private Throwable notify(INotification notification, @Nullable Throwable accumulate)
     {
         for (INotificationConsumer subscriber : subscribers)
+            accumulate = notifyOne(subscriber, notification, accumulate);
+        for (INotificationConsumer subscriber : lateSubscribers)
+            accumulate = notifyOne(subscriber, notification, accumulate);
+        return accumulate;
+    }
+
+    private Throwable notifyOne(INotificationConsumer subscriber, INotification notification, @Nullable Throwable accumulate)
+    {
+        try
+        {
             subscriber.handleNotification(notification, this);
+            return accumulate;
+        }
+        catch (Throwable t)
+        {
+            return merge(accumulate, t);
+        }
     }
 
     public boolean isDummy()
     {
-        return cfstore == null || !DatabaseDescriptor.isDaemonInitialized();
+        return cfstore == null || !DatabaseDescriptor.enableMemtableAndCommitLog();
     }
 
     public void subscribe(INotificationConsumer consumer)
     {
         subscribers.add(consumer);
+        if (logger.isTraceEnabled())
+            logger.trace("{} subscribed to the data tracker.", consumer);
+    }
+
+    /**
+     * Subscribes the provided consumer for data tracker notifications, similarly to {@link #subscribe}, but the
+     * consumer subscribed by this method are guaranteed to be notificed _after_ all the consumers subscribed with
+     * {@link #subscribe}.
+     * <p>
+     * The consumers registered by this method are notified in order of subscription (with not particular guarantee
+     * in case of concurrent calls), but again, they all execute after those of {@link #subscribe}.
+     * <p>
+     * This method is mainly targeted for non-Cassandra internal subscribers that want to register for notifications
+     * but need to make sure they are notified only after all the Cassandra internal subscribers have executed.
+     */
+    public void subscribeLateConsumer(INotificationConsumer consumer)
+    {
+        lateSubscribers.add(consumer);
+        if (logger.isTraceEnabled())
+            logger.trace("{} subscribed to the data tracker (as a 'late' consumer).", consumer);
     }
 
     @VisibleForTesting
     public boolean contains(INotificationConsumer consumer)
     {
-        return subscribers.contains(consumer);
+        return subscribers.contains(consumer) || lateSubscribers.contains(consumer);
     }
 
     public void unsubscribe(INotificationConsumer consumer)
     {
         subscribers.remove(consumer);
+        lateSubscribers.remove(consumer);
     }
 
     private static Set<SSTableReader> emptySet()
@@ -586,6 +706,12 @@ public View getView()
     @VisibleForTesting
     public void removeUnsafe(Set<SSTableReader> toRemove)
     {
-        Pair<View, View> result = apply(view -> updateLiveSet(toRemove, emptySet()).apply(view));
+        apply(view -> updateLiveSet(toRemove, emptySet()).apply(view));
+    }
+
+    @VisibleForTesting
+    public void removeCompactingUnsafe(Set<SSTableReader> toRemove)
+    {
+        apply(view -> updateCompacting(toRemove, emptySet()).apply(view));
     }
 }
diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java
index b238d24d582d..bb2759892b22 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@ -23,6 +23,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.Functions;
@@ -30,8 +32,13 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -60,6 +67,8 @@
  */
 public class View
 {
+    private static final Logger logger = LoggerFactory.getLogger(View.class);
+
     /**
      * ordinarily a list of size 1, but when preparing to flush will contain both the memtable we will flush
      * and the new replacement memtable, until all outstanding write operations on the old table complete.
@@ -73,6 +82,7 @@ public class View
     public final List<Memtable> flushingMemtables;
     final Set<SSTableReader> compacting;
     final Set<SSTableReader> sstables;
+    final Map<String, SSTableReader> sstablesByFilename;
     // we use a Map here so that we can easily perform identity checks as well as equality checks.
     // When marking compacting, we now  indicate if we expect the sstables to be present (by default we do),
     // and we then check that not only are they all present in the live set, but that the exact instance present is
@@ -98,6 +108,9 @@ public class View
         this.compactingMap = compacting;
         this.compacting = compactingMap.keySet();
         this.intervalTree = intervalTree;
+        this.sstablesByFilename = Maps.newHashMapWithExpectedSize(sstables.size());
+        for (SSTableReader sstable : this.sstables)
+            this.sstablesByFilename.put(sstable.getDataFile().name(), sstable);
     }
 
     public Memtable getCurrentMemtable()
@@ -119,6 +132,15 @@ public Set<SSTableReader> liveSSTables()
         return sstables;
     }
 
+    @Nullable
+    /**
+     * @return the sstable with the provided file name (not a full path), or null if it is not present in this view
+     */
+    public SSTableReader getLiveSSTable(String filename)
+    {
+        return sstablesByFilename.get(filename);
+    }
+
     public Iterable<SSTableReader> sstables(SSTableSet sstableSet, Predicate<SSTableReader> filter)
     {
         return filter(select(sstableSet), filter);
@@ -175,15 +197,11 @@ public Iterable<SSTableReader> select(SSTableSet sstableSet)
         }
     }
 
-    public Iterable<SSTableReader> getUncompacting(Iterable<SSTableReader> candidates)
+
+    public <S extends CompactionSSTable>
+    Iterable<S> getNoncompacting(Iterable<S> candidates)
     {
-        return filter(candidates, new Predicate<SSTableReader>()
-        {
-            public boolean apply(SSTableReader sstable)
-            {
-                return !compacting.contains(sstable);
-            }
-        });
+        return filter(candidates, sstable -> !compacting.contains(sstable));
     }
 
     public boolean isEmpty()
@@ -263,7 +281,8 @@ public static Function<View, Iterable<SSTableReader>> selectLive(AbstractBounds<
     // METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW:
 
     // return a function to un/mark the provided readers compacting in a view
-    static Function<View, View> updateCompacting(final Set<? extends SSTableReader> unmark, final Iterable<? extends SSTableReader> mark)
+    @VisibleForTesting
+    public static Function<View, View> updateCompacting(final Set<? extends SSTableReader> unmark, final Iterable<? extends SSTableReader> mark)
     {
         if (unmark.isEmpty() && Iterables.isEmpty(mark))
             return Functions.identity();
@@ -289,7 +308,11 @@ public boolean apply(View view)
             {
                 for (SSTableReader reader : readers)
                     if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted())
+                    {
+                        logger.debug("Refusing to compact {}, already compacting={}, suspect={}, compacted={}", reader,
+                                     view.compacting.contains(reader), reader.isMarkedSuspect(), reader.isMarkedCompacted());
                         return false;
+                    }
                 return true;
             }
         };
diff --git a/src/java/org/apache/cassandra/db/lifecycle/WrappedLifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/WrappedLifecycleTransaction.java
index 12c46a9573ca..d62cbcade7c9 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/WrappedLifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/WrappedLifecycleTransaction.java
@@ -24,6 +24,7 @@
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
 
 public class WrappedLifecycleTransaction implements ILifecycleTransaction
 {
@@ -99,6 +100,12 @@ public void trackNew(SSTable table)
         delegate.trackNew(table);
     }
 
+    @Override
+    public void trackNewWritten(SSTable table)
+    {
+        delegate.trackNewWritten(table);
+    }
+
     public void untrackNew(SSTable table)
     {
         delegate.untrackNew(table);
@@ -113,4 +120,16 @@ public boolean isOffline()
     {
         return delegate.isOffline();
     }
+
+    @Override
+    public TimeUUID opId()
+    {
+        return delegate.opId();
+    }
+
+    @Override
+    public void cancel(SSTableReader removedSSTable)
+    {
+        delegate.cancel(removedSSTable);
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java
index 27a67cdfbb66..1b5778e1f925 100644
--- a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java
@@ -23,6 +23,8 @@
 import java.util.List;
 import java.util.regex.Pattern;
 
+import com.google.common.collect.ImmutableList;
+
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -36,9 +38,9 @@
  */
 public abstract class AbstractCompositeType extends AbstractType<ByteBuffer>
 {
-    protected AbstractCompositeType()
+    protected AbstractCompositeType(ImmutableList<AbstractType<?>> subTypes)
     {
-        super(ComparisonType.CUSTOM);
+        super(ComparisonType.CUSTOM, false, subTypes);
     }
 
     @Override
@@ -66,8 +68,8 @@ public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right
         while (!accessorL.isEmptyFromOffset(left, offsetL) && !accessorR.isEmptyFromOffset(right, offsetR))
         {
             AbstractType<?> comparator = getComparator(i, left, accessorL, right, accessorR, offsetL, offsetR);
-            offsetL += getComparatorSize(i, left, accessorL, offsetL);
-            offsetR += getComparatorSize(i, right, accessorR, offsetR);
+            offsetL += getComparatorSize(left, accessorL, offsetL);
+            offsetR += getComparatorSize(right, accessorR, offsetR);
 
             VL value1 = accessorL.sliceWithShortLength(left, offsetL);
             offsetL += accessorL.sizeWithShortLength(value1);
@@ -110,10 +112,9 @@ public ByteBuffer[] split(ByteBuffer bb)
         boolean isStatic = readIsStatic(bb, ByteBufferAccessor.instance);
         int offset = startingOffset(isStatic);
 
-        int i = 0;
         while (!ByteBufferAccessor.instance.isEmptyFromOffset(bb, offset))
         {
-            offset += getComparatorSize(i++, bb, ByteBufferAccessor.instance, offset);
+            offset += getComparatorSize(bb, ByteBufferAccessor.instance, offset);
             ByteBuffer value = ByteBufferAccessor.instance.sliceWithShortLength(bb, offset);
             offset += ByteBufferAccessor.instance.sizeWithShortLength(value);
             l.add(value);
@@ -192,7 +193,7 @@ public <V> String getString(V input, ValueAccessor<V> accessor)
                 sb.append(":");
 
             AbstractType<?> comparator = getAndAppendComparator(i, input, accessor, sb, offset);
-            offset += getComparatorSize(i, input, accessor, offset);
+            offset += getComparatorSize(input, accessor, offset);
             V value = accessor.sliceWithShortLength(input, offset);
             offset += accessor.sizeWithShortLength(value);
 
@@ -291,7 +292,7 @@ public  <V> void validate(V input, ValueAccessor<V> accessor)
         while (!accessor.isEmptyFromOffset(input, offset))
         {
             AbstractType<?> comparator = validateComparator(i, input, accessor, offset);
-            offset += getComparatorSize(i, input, accessor, offset);
+            offset += getComparatorSize(input, accessor, offset);
 
             if (accessor.sizeFromOffset(input, offset) < 2)
                 throw new MarshalException("Not enough bytes to read value size of component " + i);
@@ -318,7 +319,7 @@ public  <V> void validate(V input, ValueAccessor<V> accessor)
 
     public abstract ByteBuffer decompose(Object... objects);
 
-    abstract protected <V> int getComparatorSize(int i, V value, ValueAccessor<V> accessor, int offset);
+    abstract protected <V> int getComparatorSize(V value, ValueAccessor<V> accessor, int offset);
     /**
      * @return the comparator for the given component. static CompositeType will consult
      * @param i DynamicCompositeType will read the type information from @param bb
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractGeometricType.java b/src/java/org/apache/cassandra/db/marshal/AbstractGeometricType.java
new file mode 100644
index 000000000000..6c46ddaae06e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractGeometricType.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.db.marshal.geometry.GeometricType;
+import org.apache.cassandra.db.marshal.geometry.OgcGeometry;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TypeSerializer;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.JsonUtils;
+
+public abstract class AbstractGeometricType<T extends OgcGeometry> extends AbstractType<T>
+{
+    private final TypeSerializer<T> serializer = new TypeSerializer<T>()
+    {
+        @Override
+        public ByteBuffer serialize(T geometry)
+        {
+            return geoSerializer.toWellKnownBinary(geometry);
+        }
+
+        @Override
+        public <V> T deserialize(V value, ValueAccessor<V> accessor)
+        {
+            // OGCGeometry does not respect the current position of the buffer, so you need to use slice()
+            try
+            {
+                ByteBuffer byteBuffer = accessor.toBuffer(value);
+                return geoSerializer.fromWellKnownBinary(byteBuffer.slice());
+            }
+            catch (IndexOutOfBoundsException ex)
+            {
+                throw new MarshalException("Not enough bytes to deserialize value", ex);
+            }
+        }
+
+        @Override
+        public <V> void validate(V value, ValueAccessor<V> accessor) throws MarshalException
+        {
+            try
+            {
+                ByteBuffer byteBuffer = accessor.toBuffer(value);
+                int pos = byteBuffer.position();
+                // OGCGeometry does not respect the current position of the buffer, so you need to use slice()
+                geoSerializer.fromWellKnownBinary(byteBuffer.slice()).validate();
+                byteBuffer.position(pos);
+            }
+            catch (IndexOutOfBoundsException ex)
+            {
+                throw new MarshalException("Not enough bytes to deserialize value", ex);
+            }
+        }
+
+        @Override
+        public String toString(T geometry)
+        {
+            return geoSerializer.toWellKnownText(geometry);
+        }
+
+        @Override
+        public Class<T> getType()
+        {
+            return klass;
+        }
+    };
+
+    private final GeometricType type;
+    private final Class<T> klass;
+    private final OgcGeometry.Serializer<T> geoSerializer;
+
+    public AbstractGeometricType(GeometricType type)
+    {
+        super(ComparisonType.BYTE_ORDER);
+        this.type = type;
+        this.klass = (Class<T>) type.getGeoClass();
+        this.geoSerializer = type.getSerializer();
+    }
+
+    public GeometricType getGeoType()
+    {
+        return type;
+    }
+
+    @Override
+    public ByteBuffer fromString(String s) throws MarshalException
+    {
+        try
+        {
+            T geometry = geoSerializer.fromWellKnownText(s);
+            geometry.validate();
+            return geoSerializer.toWellKnownBinary(geometry);
+        }
+        catch (Exception e)
+        {
+            String parentMsg = e.getMessage() != null ? " " + e.getMessage() : "";
+            String msg = String.format("Unable to make %s from '%s'", getClass().getSimpleName(), s) + parentMsg;
+            throw new MarshalException(msg, e);
+        }
+    }
+
+    @Override
+    public Term fromJSONObject(Object parsed) throws MarshalException
+    {
+        if (!(parsed instanceof String))
+        {
+            try
+            {
+                parsed = JsonUtils.JSON_OBJECT_MAPPER.writeValueAsString(parsed);
+            }
+            catch (IOException e)
+            {
+                throw new MarshalException(e.getMessage());
+            }
+        }
+
+        T geometry;
+        try
+        {
+            geometry = geoSerializer.fromGeoJson((String) parsed);
+        }
+        catch (MarshalException e)
+        {
+            try
+            {
+                geometry = geoSerializer.fromWellKnownText((String) parsed);
+            }
+            catch (MarshalException ignored)
+            {
+                throw new MarshalException(e.getMessage());
+            }
+        }
+        geometry.validate();
+        return new Constants.Value(geoSerializer.toWellKnownBinary(geometry));
+    }
+
+    @Override
+    public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
+    {
+        // OGCGeometry does not respect the current position of the buffer, so you need to use slice()
+        return geoSerializer.toGeoJson(geoSerializer.fromWellKnownBinary(buffer.slice()));
+    }
+
+    @Override
+    public TypeSerializer<T> getSerializer()
+    {
+        return serializer;
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractTimeUUIDType.java b/src/java/org/apache/cassandra/db/marshal/AbstractTimeUUIDType.java
index 35778aff24ba..0cd59279834d 100644
--- a/src/java/org/apache/cassandra/db/marshal/AbstractTimeUUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractTimeUUIDType.java
@@ -98,7 +98,7 @@ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteC
         swizzled.putLong(0, TimeUUIDType.reorderTimestampBytes(hiBits));
         swizzled.putLong(8, accessor.getLong(data, 8) ^ 0x8080808080808080L);
 
-        return ByteSource.fixedLength(swizzled);
+        return ByteSource.preencoded(swizzled);
     }
 
     @Override
@@ -224,8 +224,14 @@ public ByteBuffer now()
     }
 
     @Override
-    public boolean equals(Object obj)
+    public final boolean equals(Object obj)
     {
         return obj instanceof AbstractTimeUUIDType<?>;
     }
+
+    @Override
+    public final int hashCode()
+    {
+        return AbstractTimeUUIDType.class.hashCode();
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java
index fe8b5498372b..b4249d6080e0 100644
--- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java
@@ -21,22 +21,35 @@
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.BiPredicate;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Streams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.ColumnSpecification;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
+import org.apache.cassandra.cql3.statements.schema.AlterTableStatement;
 import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.exceptions.InvalidColumnTypeException;
 import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -46,11 +59,12 @@
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 import org.github.jamm.Unmetered;
 
+import static com.google.common.collect.Iterables.transform;
 import static org.apache.cassandra.db.marshal.AbstractType.ComparisonType.CUSTOM;
 
 /**
  * Specifies a Comparator for a specific type of ByteBuffer.
- *
+ * <p>
  * Note that empty ByteBuffer are used to represent "start at the beginning"
  * or "stop at the end" arguments to get_slice, so the Comparator
  * should always handle those values even if they normally do not
@@ -59,9 +73,9 @@
 @Unmetered
 public abstract class AbstractType<T> implements Comparator<ByteBuffer>, AssignmentTestable
 {
-    private final static int VARIABLE_LENGTH = -1;
+    private final static Logger logger = LoggerFactory.getLogger(AbstractType.class);
 
-    public final Comparator<ByteBuffer> reverseComparator;
+    private final static int VARIABLE_LENGTH = -1;
 
     public enum ComparisonType
     {
@@ -84,12 +98,41 @@ public enum ComparisonType
     public final ComparisonType comparisonType;
     public final boolean isByteOrderComparable;
     public final ValueComparators comparatorSet;
+    public final boolean isMultiCell;
+    public final ImmutableList<AbstractType<?>> subTypes;
+
+    private final int hashCode;
 
     protected AbstractType(ComparisonType comparisonType)
     {
+        this(comparisonType, false, ImmutableList.of());
+    }
+
+    protected AbstractType(ComparisonType comparisonType, boolean isMultiCell, ImmutableList<AbstractType<?>> subTypes)
+    {
+        this.isMultiCell = isMultiCell;
         this.comparisonType = comparisonType;
         this.isByteOrderComparable = comparisonType == ComparisonType.BYTE_ORDER;
-        reverseComparator = (o1, o2) -> AbstractType.this.compare(o2, o1);
+
+        // A frozen type can only have frozen subtypes, basically by definition. So make sure we don't mess it up
+        // when constructing types by forgetting to set some multi-cell flag.
+        if (!isMultiCell)
+        {
+            if (Iterables.any(subTypes, AbstractType::isMultiCell))
+                this.subTypes = ImmutableList.copyOf(Iterables.transform(subTypes, AbstractType::freeze));
+            else
+                this.subTypes = subTypes;
+        }
+        else
+        {
+            this.subTypes = subTypes;
+        }
+        if (subTypes != this.subTypes)
+            logger.warn("Detected corrupted type: creating a frozen {} but with some non-frozen subtypes {}. " +
+                        "This is likely a bug and should be reported.",
+                        getClass(),
+                        subTypes.stream().filter(AbstractType::isMultiCell).map(AbstractType::toString).collect(Collectors.joining(", ")));
+
         try
         {
             Method custom = getClass().getMethod("compareCustom", Object.class, ValueAccessor.class, Object.class, ValueAccessor.class);
@@ -103,8 +146,23 @@ protected AbstractType(ComparisonType comparisonType)
             throw new IllegalStateException();
         }
 
-        comparatorSet = new ValueComparators((l, r) -> compare(l, ByteArrayAccessor.instance, r, ByteArrayAccessor.instance),
-                                             (l, r) -> compare(l, ByteBufferAccessor.instance, r, ByteBufferAccessor.instance));
+        comparatorSet = new ValueComparators(new Comparator<>()
+        {
+            @Override
+            public int compare(byte[] l, byte[] r)
+            {
+                return AbstractType.this.compare(l, ByteArrayAccessor.instance, r, ByteArrayAccessor.instance);
+            }
+        }, new Comparator<>()
+        {
+            @Override
+            public int compare(ByteBuffer l, ByteBuffer r)
+            {
+                return AbstractType.this.compare(l, ByteBufferAccessor.instance, r, ByteBufferAccessor.instance);
+            }
+        });
+
+        hashCode = Objects.hash(getClass(), this.isMultiCell, this.subTypes);
     }
 
     static <VL, VR, T extends Comparable<T>> int compareComposed(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR, AbstractType<T> type)
@@ -160,11 +218,28 @@ public final String getString(ByteBuffer bytes)
         return getString(bytes, ByteBufferAccessor.instance);
     }
 
+    public final String getString(ByteBuffer bytes, boolean truncate)
+    {
+        String s = getString(bytes);
+        return truncate ? truncateString(s) : s;
+    }
+
     public String toCQLString(ByteBuffer bytes)
     {
         return asCQL3Type().toCQLLiteral(bytes);
     }
 
+    public String toCQLString(ByteBuffer bytes, boolean truncate)
+    {
+        String s = toCQLString(bytes);
+        return truncate ? truncateString(s) : s;
+    }
+
+    private static String truncateString(String valueString)
+    {
+        return valueString.length() <= 9 ? valueString : valueString.substring(0, 6) + "...";
+    }
+
     /** get a byte representation of the given string. */
     public abstract ByteBuffer fromString(String source) throws MarshalException;
 
@@ -267,6 +342,12 @@ public int compareForCQL(ByteBuffer v1, ByteBuffer v2)
         return compare(v1, v2);
     }
 
+    /**
+     * Returns the serializer for this type.
+     * Note that the method must return a different instance of serializer for different types even if the types
+     * use the same serializer - in this case, the method should return separate instances for which equals() returns
+     * false.
+     */
     public abstract TypeSerializer<T> getSerializer();
 
     /**
@@ -277,27 +358,11 @@ public ArgumentDeserializer getArgumentDeserializer()
         return new DefaultArgumentDeserializer(this);
     }
 
-    /* convenience method */
-    public String getString(Collection<ByteBuffer> names)
-    {
-        StringBuilder builder = new StringBuilder();
-        for (ByteBuffer name : names)
-        {
-            builder.append(getString(name)).append(",");
-        }
-        return builder.toString();
-    }
-
     public boolean isCounter()
     {
         return false;
     }
 
-    public boolean isFrozenCollection()
-    {
-        return isCollection() && !isMultiCell();
-    }
-
     public boolean isReversed()
     {
         return false;
@@ -308,6 +373,11 @@ public AbstractType<T> unwrap()
         return isReversed() ? ((ReversedType<T>) this).baseType.unwrap() : this;
     }
 
+    public boolean isList()
+    {
+        return false;
+    }
+
     public static AbstractType<?> parseDefaultParameters(AbstractType<?> baseType, TypeParser parser) throws SyntaxException
     {
         Map<String, String> parameters = parser.getKeyValueParameters();
@@ -323,14 +393,20 @@ public static AbstractType<?> parseDefaultParameters(AbstractType<?> baseType, T
     }
 
     /**
-     * Returns true if this comparator is compatible with the provided
-     * previous comparator, that is if previous can safely be replaced by this.
+     * Returns true if this comparator is compatible with the provided previous comparator, that is if previous can
+     * safely be replaced by this.
      * A comparator cn should be compatible with a previous one cp if forall columns c1 and c2,
      * if   cn.validate(c1) and cn.validate(c2) and cn.compare(c1, c2) == v,
      * then cp.validate(c1) and cp.validate(c2) and cp.compare(c1, c2) == v.
-     *
-     * Note that a type should be compatible with at least itself and when in
-     * doubt, keep the default behavior of not being compatible with any other comparator!
+     * <p/>
+     * Note that a type should be compatible with at least itself and when in doubt, keep the default behavior
+     * of not being compatible with any other comparator!
+     * <p/>
+     * Used for user functions and aggregates to validate the returning type when the function is replaced.
+     * Used for validation of table metadata when replacing metadata in ref (alterting a table) and when scrubbing
+     * an sstable to validate whether metadata stored in the sstable is compatible with the current metadata.
+     * <p/>
+     * Note that this will never return true when one type is multicell and the other is not.
      */
     public boolean isCompatibleWith(AbstractType<?> previous)
     {
@@ -338,36 +414,58 @@ public boolean isCompatibleWith(AbstractType<?> previous)
     }
 
     /**
-     * Returns true if values of the other AbstractType can be read and "reasonably" interpreted by the this
+     * Returns true if values of the other AbstractType can be read and "reasonably" interpreted by this
      * AbstractType. Note that this is a weaker version of isCompatibleWith, as it does not require that both type
      * compare values the same way.
-     *
+     * <p/>
      * The restriction on the other type being "reasonably" interpreted is to prevent, for example, IntegerType from
      * being compatible with all other types.  Even though any byte string is a valid IntegerType value, it doesn't
      * necessarily make sense to interpret a UUID or a UTF8 string as an integer.
-     *
+     * <p/>
      * Note that a type should be compatible with at least itself.
+     * <p/>
+     * Also note that to ensure consistent handling of the {@link ReversedType} (which should be ignored as far as this
+     * method goes since it only impacts sorting), this method is final and subclasses should override the
+     * {@link #isValueCompatibleWithInternal} method instead.
+     * <p/>
+     * Used for type casting and values assignment. It valid if we can compose L values which were decomposed using R
+     * serializer. Therefore, it does not care about whether the type is reversed or not. It should not whether the
+     * type is fixed or variable length as for compose/decompose we always deal with all remaining data in the buffer
+     * (so for example, a variable length type may be compatible with fixed length type given the interpretation is
+     * consistent, like between BigInt and Long).
      */
-    public boolean isValueCompatibleWith(AbstractType<?> previous)
+    public final boolean isValueCompatibleWith(AbstractType<?> previous)
     {
-        AbstractType<?> thisType =          isReversed() ? ((ReversedType<?>)     this).baseType : this;
-        AbstractType<?> thatType = previous.isReversed() ? ((ReversedType<?>) previous).baseType : previous;
-        return thisType.isValueCompatibleWithInternal(thatType);
+        if (previous == null)
+            return false;
+
+        AbstractType<T> unwrapped = this.unwrap();
+        AbstractType<?> previousUnwrapped = previous.unwrap();
+        if (unwrapped.equals(previousUnwrapped))
+            return true;
+
+        return unwrapped.isValueCompatibleWithInternal(previousUnwrapped);
     }
 
     /**
-     * Needed to handle ReversedType in value-compatibility checks.  Subclasses should implement this instead of
-     * isValueCompatibleWith().
+     * Needed to handle {@link ReversedType} in value-compatibility checks. Subclasses should override this instead of
+     * {@link #isValueCompatibleWith}. However, if said override has subtypes on which they need to check value
+     * compatibility recursively, they should call {@link #isValueCompatibleWith} instead of this method
+     * so that reversed types are ignored even if nested.
      */
-    protected boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        return isCompatibleWith(otherType);
+        return isCompatibleWith(previous);
     }
 
     /**
      * Similar to {@link #isValueCompatibleWith(AbstractType)}, but takes into account {@link Cell} encoding.
      * In particular, this method doesn't consider two types serialization compatible if one of them has fixed
      * length (overrides {@link #valueLengthIfFixed()}, and the other one doesn't.
+     * </p>
+     * Used in {@link AlterTableStatement} when adding a column with the same name as the previously dropped column.
+     * The new column type must be serialization compatible with the old one. We must be able to read cells of the new
+     * type which were serialized as cells of the old type.
      */
     public boolean isSerializationCompatibleWith(AbstractType<?> previous)
     {
@@ -414,42 +512,65 @@ public boolean isVector()
         return false;
     }
 
-    public boolean isMultiCell()
+    public final boolean isMultiCell()
     {
-        return false;
-    }
-
-    public boolean isFreezable()
-    {
-        return false;
+        return isMultiCell;
     }
 
+    /**
+     * If the type is a multi-cell one ({@link #isMultiCell()} is true), returns a frozen copy of this type (one
+     * for which {@link #isMultiCell()} returns false).
+     * <p>
+     * Note that as mentioned on {@link #isMultiCell()}, a frozen type necessarily has all its subtypes frozen, so
+     * this method also ensures that no subtypes (recursively) are marked as multi-cell.
+     *
+     * @return a frozen version of this type. If this type is not multi-cell (whether because it is not a "complex"
+     * type, or because it is already a frozen one), this should return {@code this}.
+     */
     public AbstractType<?> freeze()
     {
-        return this;
-    }
+        if (!isMultiCell())
+            return this;
 
-    public AbstractType<?> unfreeze()
-    {
-        return this;
+        return with(freeze(subTypes()), false);
     }
 
-    public List<AbstractType<?>> subTypes()
+    /**
+     * Creates an instance of this type (the concrete type extending this class) with the provided updated multi-cell
+     * flag and subtypes.
+     * <p>
+     * Any other information (other than multi-cellness and subtypes) the type may have is expected to be left unchanged
+     * in the created type.
+     *
+     * @param isMultiCell whether the returned type must be a multi-cell one or not.
+     * @param subTypes the subtypes to use for the returned type as a list. The list will have subtypes in the exact
+     * same order as returned by {@link #subTypes()}, and exactly as many as the concrete class expects.
+     * @return the created type, which can be {@code this} if the provided subTypes and multi-cell flag are the same
+     * as that of this type.
+     */
+    public AbstractType<T> with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
     {
-        return Collections.emptyList();
+        // Default implementation for types that can neither be multi-cell, nor have subtypes (and thus where this
+        // is basically a no-op). Any other type must override this.
+
+        assert this.subTypes.isEmpty() && subTypes.isEmpty() :
+        String.format("Invalid call to 'with' on %s with subTypes %s (provided subTypes: %s)",
+                      this, this.subTypes, subTypes);
+
+        assert !this.isMultiCell() && !isMultiCell:
+        String.format("Invalid call to 'with' on %s with isMultiCell %b (provided isMultiCell: %b)",
+                      this, this.isMultiCell(), isMultiCell);
+
+        return this;
     }
 
     /**
-     * Returns an AbstractType instance that is equivalent to this one, but with all nested UDTs and collections
-     * explicitly frozen.
-     *
-     * This is only necessary for {@code 2.x -> 3.x} schema migrations, and can be removed in Cassandra 4.0.
-     *
-     * See CASSANDRA-11609 and CASSANDRA-11613.
+     * If the type has "complex" values that depend on subtypes, return those (direct) subtypes (in undefined order),
+     * and an empty list otherwise.
      */
-    public AbstractType<?> freezeNestedMulticellTypes()
+    public final ImmutableList<AbstractType<?>> subTypes()
     {
-        return this;
+        return subTypes;
     }
 
     /**
@@ -465,21 +586,23 @@ public boolean isEmptyValueMeaningless()
      */
     public String toString(boolean ignoreFreezing)
     {
-        return this.toString();
+        return getClass().getName();
     }
 
     /**
-     * Return a list of the "subcomponents" this type has.
-     * This always return a singleton list with the type itself except for CompositeType.
+     * To override keyspace name in {@link UserType}
      */
-    public List<AbstractType<?>> getComponents()
+    public AbstractType<T> overrideKeyspace(Function<String, String> overrideKeyspace)
     {
-        return Collections.<AbstractType<?>>singletonList(this);
+        if (subTypes.isEmpty())
+            return this;
+        else
+            return with(subTypes.stream().map(t -> t.overrideKeyspace(overrideKeyspace)).collect(ImmutableList.toImmutableList()), isMultiCell);
     }
 
     /**
-     * The length of values for this type if all values are of fixed length, -1 otherwise. This has an impact on
-     * serialization.
+     * The length of values for this type, in bytes, if all values are of fixed length, -1 otherwise.
+     * This has an impact on serialization.
      * <lu>
      *  <li> see {@link #writeValue} </li>
      *  <li> see {@link #read} </li>
@@ -614,9 +737,22 @@ public final boolean referencesUserType(ByteBuffer name)
         return referencesUserType(name, ByteBufferAccessor.instance);
     }
 
+    /**
+     * Returns true if this type is or references a user type with provided name.
+     */
     public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
     {
-        return false;
+        // Note that non-complex types have no subtypes, so will return false, and UserType overrides this to return
+        // true if the provided name matches.
+        return subTypes().stream().anyMatch(t -> t.referencesUserType(name, accessor));
+    }
+
+    /**
+     * Whether this type is or contains any UDT.
+     */
+    public final boolean referencesUserTypes()
+    {
+        return isUDT() || subTypes().stream().anyMatch(AbstractType::referencesUserTypes);
     }
 
     /**
@@ -625,23 +761,55 @@ public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
      */
     public AbstractType<?> withUpdatedUserType(UserType udt)
     {
-        return this;
+        if (!referencesUserType(udt.name))
+            return this;
+
+        ImmutableList.Builder<AbstractType<?>> builder = ImmutableList.builder();
+        for (AbstractType<?> subType : subTypes)
+            builder.add(subType.withUpdatedUserType(udt));
+
+        return with(builder.build(), isMultiCell());
+    }
+
+    /**
+     * Returns an instance of this type with all references to the provided user types recursively replaced with their new
+     * definition.
+     */
+    public final AbstractType<?> withUpdatedUserTypes(Iterable<UserType> udts)
+    {
+        if (!referencesUserTypes())
+            return this;
+
+        AbstractType<?> type = this;
+        for (UserType udt : udts)
+            type = type.withUpdatedUserType(udt);
+
+        return type;
     }
 
     /**
      * Replace any instances of UserType with equivalent TupleType-s.
-     *
+     * <p>
      * We need it for dropped_columns, to allow safely dropping unused user types later without retaining any references
      * to them in system_schema.dropped_columns.
      */
     public AbstractType<?> expandUserTypes()
     {
-        return this;
+        return referencesUserTypes()
+               ? with(ImmutableList.copyOf(transform(subTypes, AbstractType::expandUserTypes)), isMultiCell())
+               : this;
     }
 
     public boolean referencesDuration()
     {
-        return false;
+        // Note that non-complex types have no subtypes, so will return false, and DurationType overrides this to return
+        // true.
+        return subTypes().stream().anyMatch(AbstractType::referencesDuration);
+    }
+
+    public final boolean referencesCounter()
+    {
+        return isCounter() || subTypes().stream().anyMatch(AbstractType::referencesCounter);
     }
 
     /**
@@ -652,7 +820,7 @@ public AssignmentTestable.TestResult testAssignment(AbstractType<?> receiverType
         // testAssignement is for CQL literals and native protocol values, none of which make a meaningful
         // difference between frozen or not and reversed or not.
 
-        if (isFreezable() && !isMultiCell())
+        if (!isMultiCell())
             receiverType = receiverType.freeze();
 
         if (isReversed() && !receiverType.isReversed())
@@ -667,6 +835,133 @@ public AssignmentTestable.TestResult testAssignment(AbstractType<?> receiverType
         return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
     }
 
+    /**
+     * Validates whether this type is valid as a column type for a column of the provided kind.
+     * <p>
+     * A number of limits must be respected by column types (possibly depending on the type of columns). For
+     * instance, primary key columns must always be frozen, cannot use counters, etc. And for regular columns, amongst
+     * other things, we currently only support non-frozen types at top-level, so any type with a non-frozen subtype
+     * is invalid (note that it's valid to <b>create</b> a type with non-frozen subtypes, with a {@code CREATE TYPE}
+     * for instance, but they cannot be used as column types without being frozen).
+     *
+     * @param columnName         the name of the column whose type is checked.
+     * @param isPrimaryKeyColumn whether {@code columnName} is a primary key column or not.
+     * @param isCounterTable     whether the table the {@code columnName} is part of is a counter table.
+     * @throws InvalidColumnTypeException if this type is not a valid column type for {@code columnName}.
+     */
+    public void validateForColumn(ByteBuffer columnName,
+                                  boolean isPrimaryKeyColumn,
+                                  boolean isCounterTable,
+                                  boolean isDroppedColumn,
+                                  boolean isForOfflineTool)
+    {
+        if (isPrimaryKeyColumn)
+        {
+            if (isMultiCell())
+                throw columnException(columnName,
+                                      "non-frozen %s are not supported for PRIMARY KEY columns", category());
+            if (referencesCounter())
+                throw columnException(columnName,
+                                      "counters are not supported within PRIMARY KEY columns");
+
+            // We don't allow durations in anything sorted (primary key here, or in the "name-comparator" part of
+            // collections below). This isn't really a technical limitation, but duration sorts in a somewhat random
+            // way, so CASSANDRA-11873 decided to reject them when sorting was involved.
+            if (referencesDuration())
+                throw columnException(columnName,
+                                      "duration types are not supported within PRIMARY KEY columns");
+
+            if (comparisonType == ComparisonType.NOT_COMPARABLE)
+                throw columnException(columnName,
+                                      "type %s is not comparable and cannot be used for PRIMARY KEY columns", asCQL3Type().toSchemaString());
+        }
+        else
+        {
+            if (isMultiCell())
+            {
+                if (isTuple() && !isDroppedColumn && !isForOfflineTool)
+                    throw columnException(columnName,
+                                          "tuple type %s is not frozen, which should not have happened",
+                                          asCQL3Type().toSchemaString());
+
+                for (AbstractType<?> subType : subTypes())
+                {
+                    if (subType.isMultiCell())
+                    {
+                        throw columnException(columnName,
+                                              "non-frozen %s are only supported at top-level: subtype %s of %s must be frozen",
+                                              subType.category(), subType.asCQL3Type().toSchemaString(), asCQL3Type().toSchemaString());
+                    }
+                }
+
+                if (this instanceof MultiCellCapableType)
+                {
+                    AbstractType<?> nameComparator = ((MultiCellCapableType<?>) this).nameComparator();
+                    // As mentioned above, CASSANDRA-11873 decided to reject durations when sorting was involved.
+                    if (nameComparator.referencesDuration())
+                    {
+                        // Trying to profile a more precise error message
+                        String what = this instanceof MapType
+                                      ? "map keys"
+                                      : (this instanceof SetType ? "sets" : category());
+                        throw columnException(columnName, "duration types are not supported within non-frozen %s", what);
+                    }
+                }
+            }
+
+            // Mixing counter with non counter columns is not supported (#2614)
+            if (isCounterTable)
+            {
+                // Everything within a counter table must be a counter, and we don't allow nesting (collections of
+                // counters), except for legacy backward-compatibility, in the super-column map used to support old
+                // super columns.
+                if (!isCounter() && !TableMetadata.isSuperColumnMapColumnName(columnName))
+                {
+                    // We don't allow counter inside collections, but to be fair, at least for map, it's a bit of an
+                    // arbitrary limitation (it works internally, we don't expose it mostly because counters have
+                    // their limitations, and we want to restrict how user can use them to hopefully make user think
+                    // twice about their usage). In any case, a slightly more user-friendly message is probably nice.
+                    if (referencesCounter())
+                        throw columnException(columnName, "counters are not allowed within %s", category());
+
+                    throw columnException(columnName, "Cannot mix counter and non counter columns in the same table");
+                }
+            }
+            else
+            {
+                if (isCounter())
+                    throw columnException(columnName, "Cannot mix counter and non counter columns in the same table");
+
+                // For nested counters, we prefer complaining about the nested-ness rather than this not being a counter
+                // table, because the table won't be marked as a counter one even if it has only nested counters, and so
+                // that's overall a more intuitive message.
+                if (referencesCounter())
+                    throw columnException(columnName, "counters are not allowed within %s", category());
+            }
+        }
+
+    }
+
+    private InvalidColumnTypeException columnException(ByteBuffer columnName,
+                                                       String reason,
+                                                       Object... args)
+    {
+        String msg = args.length == 0 ? reason : String.format(reason, args);
+        return new InvalidColumnTypeException(columnName, this, msg);
+    }
+
+    private String category()
+    {
+        if (isCollection())
+            return "collections";
+        else if (isTuple())
+            return "tuples";
+        else if (isUDT())
+            return "user types";
+        else
+            return "types";
+    }
+
     /**
      * Produce a byte-comparable representation of the given value, i.e. a sequence of bytes that compares the same way
      * using lexicographical unsigned byte comparison as the original value using the type's comparator.
@@ -738,9 +1033,9 @@ public final ByteBuffer fromComparableBytes(ByteSource.Peekable comparableBytes,
      * For CQL purposes the short name is fine.
      */
     @Override
-    public String toString()
+    public final String toString()
     {
-        return getClass().getName();
+        return toString(false);
     }
 
     public void checkComparable()
@@ -771,6 +1066,36 @@ public ByteBuffer getMaskedValue()
         throw new UnsupportedOperationException("There isn't a defined masked value for type " + asCQL3Type());
     }
 
+    protected static <K, V extends AbstractType<?>> V getInstance(ConcurrentMap<K, V> instances, K key, Supplier<V> value)
+    {
+        V cached = instances.get(key);
+        if (cached != null)
+            return cached;
+
+        // We avoid constructor calls in Map#computeIfAbsent to avoid recursive update exceptions because the automatic
+        // fixing of subtypes done by the top-level constructor might attempt a recursive update to the instances map.
+        V instance = value.get();
+        return instances.computeIfAbsent(key, k -> instance);
+    }
+
+    /**
+     * Utility method that freezes a list of types.
+     *
+     * @param types the list of types to freeze.
+     * @return a new (unmodifiable) list containing the result of applying {@link #freeze()} on every type of
+     * {@code types}.
+     */
+    public static ImmutableList<AbstractType<?>> freeze(Iterable<AbstractType<?>> types)
+    {
+        if (Iterables.isEmpty(types))
+            return ImmutableList.of();
+
+        ImmutableList.Builder<AbstractType<?>> builder = ImmutableList.builder();
+        for (AbstractType<?> type : types)
+            builder.add(type.freeze());
+        return builder.build();
+    }
+
     /**
      * {@link ArgumentDeserializer} that uses the type deserialization.
      */
@@ -792,4 +1117,44 @@ public Object deserialize(ProtocolVersion protocolVersion, ByteBuffer buffer)
             return type.compose(buffer);
         }
     }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        if (this.hashCode() != o.hashCode())
+            return false;
+        AbstractType<?> that = (AbstractType<?>) o;
+        return isMultiCell == that.isMultiCell && Objects.equals(subTypes, that.subTypes);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return hashCode;
+    }
+
+    /**
+     * Checks whether this type's subtypes are compatible with the provided type's subtypes using a provided predicate.
+     * Regardless of the predicate, this method returns false if this type has fewer subtypes than the provided type
+     * because in that case it could not safely replace the provided type in any situation.
+     *
+     * @param previous  the type against which the verification is done - in other words, the type which was originally
+     *                  used to serialize the values
+     * @param predicate one of the methodsd isXXXCompatibleWith
+     * @return {@code true} if this type has at least the same number of subtypes as the previous type and the predicate
+     * is satisfied for the corresponding subtypes
+     */
+    protected boolean isSubTypesCompatibleWith(AbstractType<?> previous, BiPredicate<AbstractType<?>, AbstractType<?>> predicate)
+    {
+        if (subTypes.size() < previous.subTypes.size())
+            return false;
+
+        return Streams.zip(subTypes.stream().limit(previous.subTypes.size()), previous.subTypes.stream(), predicate::test)
+                      .allMatch(Predicate.isEqual(true));
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/db/marshal/AsciiType.java b/src/java/org/apache/cassandra/db/marshal/AsciiType.java
index 119965abeb95..2ecb6b4f9a9f 100644
--- a/src/java/org/apache/cassandra/db/marshal/AsciiType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AsciiType.java
@@ -44,7 +44,7 @@ public class AsciiType extends StringType
 
     AsciiType() {super(ComparisonType.BYTE_ORDER);} // singleton
 
-    private final FastThreadLocal<CharsetEncoder> encoder = new FastThreadLocal<CharsetEncoder>()
+    private final FastThreadLocal<CharsetEncoder> encoder = new FastThreadLocal<>()
     {
         @Override
         protected CharsetEncoder initialValue()
diff --git a/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java b/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java
index 8877acbf5a7e..0a6bd9b500ca 100644
--- a/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java
+++ b/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java
@@ -18,7 +18,14 @@
 
 package org.apache.cassandra.db.marshal;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.AbstractArrayClusteringPrefix;
+import org.apache.cassandra.db.ArrayClustering;
+import org.apache.cassandra.db.ArrayClusteringBound;
+import org.apache.cassandra.db.ArrayClusteringBoundary;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.ClusteringPrefix;
 import org.apache.cassandra.db.rows.ArrayCell;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.CellPath;
@@ -35,27 +42,6 @@ public String toString(TableMetadata metadata)
         }
     };
 
-    public static final Clustering<byte[]> STATIC_CLUSTERING = new ArrayClustering(AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY)
-    {
-        @Override
-        public Kind kind()
-        {
-            return Kind.STATIC_CLUSTERING;
-        }
-
-        @Override
-        public String toString()
-        {
-            return "STATIC";
-        }
-
-        @Override
-        public String toString(TableMetadata metadata)
-        {
-            return toString();
-        }
-    };
-
     static final ValueAccessor.ObjectFactory<byte[]> instance = new ByteArrayObjectFactory();
 
     private ByteArrayObjectFactory() {}
@@ -89,11 +75,6 @@ public Clustering<byte[]> clustering()
         return EMPTY_CLUSTERING;
     }
 
-    public Clustering<byte[]> staticClustering()
-    {
-        return STATIC_CLUSTERING;
-    }
-
     public ClusteringBound<byte[]> bound(ClusteringPrefix.Kind kind, byte[]... values)
     {
         return new ArrayClusteringBound(kind, values);
diff --git a/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java b/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java
index 76e49b72713f..549cc07c64c0 100644
--- a/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java
+++ b/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java
@@ -20,7 +20,14 @@
 
 import java.nio.ByteBuffer;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.AbstractBufferClusteringPrefix;
+import org.apache.cassandra.db.BufferClustering;
+import org.apache.cassandra.db.BufferClusteringBound;
+import org.apache.cassandra.db.BufferClusteringBoundary;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.ClusteringPrefix;
 import org.apache.cassandra.db.rows.BufferCell;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.CellPath;
@@ -61,11 +68,6 @@ public Clustering<ByteBuffer> clustering()
         return Clustering.EMPTY;
     }
 
-    public Clustering<ByteBuffer> staticClustering()
-    {
-        return Clustering.STATIC_CLUSTERING;
-    }
-
     public ClusteringBound<ByteBuffer> bound(ClusteringPrefix.Kind kind, ByteBuffer... values)
     {
         return new BufferClusteringBound(kind, values);
diff --git a/src/java/org/apache/cassandra/db/marshal/BytesType.java b/src/java/org/apache/cassandra/db/marshal/BytesType.java
index 7ad0280e550c..289a697bf93d 100644
--- a/src/java/org/apache/cassandra/db/marshal/BytesType.java
+++ b/src/java/org/apache/cassandra/db/marshal/BytesType.java
@@ -23,9 +23,9 @@
 import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
-import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.serializers.BytesSerializer;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Hex;
@@ -82,13 +82,14 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
     @Override
     public boolean isCompatibleWith(AbstractType<?> previous)
     {
+        // TODO BytesType is actually compatible with all types which use BYTE_ORDER comparison type
         // Both asciiType and utf8Type really use bytes comparison and
         // bytesType validate everything, so it is compatible with the former.
         return this == previous || previous == AsciiType.instance || previous == UTF8Type.instance;
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
         // BytesType can read anything
         return true;
diff --git a/src/java/org/apache/cassandra/db/marshal/CollectionType.java b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
index 0dcf25b0b75d..8e1209976bd9 100644
--- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
@@ -17,15 +17,16 @@
  */
 package org.apache.cassandra.db.marshal;
 
-import java.nio.ByteBuffer;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.List;
 import java.util.Iterator;
-import java.util.Objects;
+import java.util.List;
 import java.util.function.Consumer;
 import java.util.Locale;
 
+import com.google.common.collect.ImmutableList;
+
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.ColumnSpecification;
 import org.apache.cassandra.cql3.Lists;
@@ -50,7 +51,7 @@
  * Please note that this comparator shouldn't be used "manually" (as a custom
  * type for instance).
  */
-public abstract class CollectionType<T> extends AbstractType<T>
+public abstract class CollectionType<T> extends MultiCellCapableType<T>
 {
     public static CellPath.Serializer cellPathSerializer = new CollectionPathSerializer();
 
@@ -89,13 +90,12 @@ public String toString()
 
     public final Kind kind;
 
-    protected CollectionType(ComparisonType comparisonType, Kind kind)
+    protected CollectionType(ComparisonType comparisonType, Kind kind, boolean isMultiCell, ImmutableList<AbstractType<?>> subTypes)
     {
-        super(comparisonType);
+        super(comparisonType, isMultiCell, subTypes);
         this.kind = kind;
     }
 
-    public abstract AbstractType<?> nameComparator();
     public abstract AbstractType<?> valueComparator();
 
     protected abstract List<ByteBuffer> serializedValues(Iterator<Cell<?>> cells);
@@ -156,12 +156,6 @@ public boolean isMap()
         return kind == Kind.MAP;
     }
 
-    @Override
-    public boolean isFreezable()
-    {
-        return true;
-    }
-
     // Overrided by maps
     protected int collectionSize(List<ByteBuffer> values)
     {
@@ -177,102 +171,43 @@ public ByteBuffer serializeForNativeProtocol(Iterator<Cell<?>> cells)
     }
 
     @Override
-    public boolean isCompatibleWith(AbstractType<?> previous)
+    protected boolean isCompatibleWithFrozen(MultiCellCapableType<?> previous)
     {
-        if (this == previous)
-            return true;
-
-        if (!getClass().equals(previous.getClass()))
-            return false;
-
-        CollectionType<?> tprev = (CollectionType<?>) previous;
-        if (this.isMultiCell() != tprev.isMultiCell())
+        if (getClass() != previous.getClass())
             return false;
 
-        // subclasses should handle compatibility checks for frozen collections
-        if (!this.isMultiCell())
-            return isCompatibleWithFrozen(tprev);
-
-        if (!this.nameComparator().isCompatibleWith(tprev.nameComparator()))
-            return false;
-
-        // the value comparator is only used for Cell values, so sorting doesn't matter
-        return this.valueComparator().isSerializationCompatibleWith(tprev.valueComparator());
+        // When frozen, the full collection is a blob, so everything must be sorted-compatible for the whole blob to
+        // be sorted-compatible.
+        return isSubTypesCompatibleWith(previous, AbstractType::isCompatibleWith);
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> previous)
+    protected boolean isCompatibleWithMultiCell(MultiCellCapableType<?> previous)
     {
-        // for multi-cell collections, compatibility and value-compatibility are the same
-        if (this.isMultiCell())
-            return isCompatibleWith(previous);
-
-        if (this == previous)
-            return true;
-
-        if (!getClass().equals(previous.getClass()))
-            return false;
-
-        CollectionType<?> tprev = (CollectionType<?>) previous;
-        if (this.isMultiCell() != tprev.isMultiCell())
+        if (getClass() != previous.getClass())
             return false;
 
-        // subclasses should handle compatibility checks for frozen collections
-        return isValueCompatibleWithFrozen(tprev);
+        // When multi-cell, the name comparator is the one used to compare cell-path so must be sorted-compatible
+        // but the value comparator is never used for sorting so serialization-compatibility is enough.
+        return this.nameComparator().isCompatibleWith(previous.nameComparator()) &&
+               this.valueComparator().isSerializationCompatibleWith(((CollectionType<?>) previous).valueComparator());
     }
 
     @Override
-    public boolean isSerializationCompatibleWith(AbstractType<?> previous)
+    protected boolean isValueCompatibleWithFrozen(MultiCellCapableType<?> previous)
     {
-        if (!isValueCompatibleWith(previous))
+        if (getClass() != previous.getClass())
             return false;
 
-        return valueComparator().isSerializationCompatibleWith(((CollectionType<?>)previous).valueComparator());
+        return nameComparator().isCompatibleWith(previous.nameComparator()) &&
+               valueComparator().isValueCompatibleWith(((CollectionType<?>) previous).valueComparator());
     }
 
-    /** A version of isCompatibleWith() to deal with non-multicell (frozen) collections */
-    protected abstract boolean isCompatibleWithFrozen(CollectionType<?> previous);
-
-    /** A version of isValueCompatibleWith() to deal with non-multicell (frozen) collections */
-    protected abstract boolean isValueCompatibleWithFrozen(CollectionType<?> previous);
-
     public CQL3Type asCQL3Type()
     {
         return new CQL3Type.Collection(this);
     }
 
-    @Override
-    public boolean equals(Object o)
-    {
-        if (this == o)
-            return true;
-
-        if (!(o instanceof CollectionType))
-            return false;
-
-        CollectionType<?> other = (CollectionType<?>) o;
-
-        if (kind != other.kind)
-            return false;
-
-        if (isMultiCell() != other.isMultiCell())
-            return false;
-
-        return nameComparator().equals(other.nameComparator()) && valueComparator().equals(other.valueComparator());
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(kind, isMultiCell(), nameComparator(), valueComparator());
-    }
-
-    @Override
-    public String toString()
-    {
-        return this.toString(false);
-    }
-
     static <VL, VR> int compareListOrSet(AbstractType<?> elementsComparator, VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
     {
         // Note that this is only used if the collection is frozen
@@ -358,6 +293,14 @@ public static String setOrListToJsonString(ByteBuffer buffer, AbstractType<?> el
         return sb.append("]").toString();
     }
 
+    /**
+     * Checks if the specified serialized collection contains the specified serialized collection element.
+     *
+     * @param element a serialized collection element
+     * @return {@code true} if the collection contains the value, {@code false} otherwise
+     */
+    public abstract boolean contains(ByteBuffer collection, ByteBuffer element);
+
     private static class CollectionPathSerializer implements CellPath.Serializer
     {
         public void serialize(CellPath path, DataOutputPlus out) throws IOException
@@ -387,4 +330,20 @@ public int size(ByteBuffer buffer)
     }
 
     public abstract void forEach(ByteBuffer input, Consumer<ByteBuffer> action);
+
+    @Override
+    public String toString(boolean ignoreFreezing)
+    {
+        boolean includeFrozenType = !ignoreFreezing && !isMultiCell();
+
+        StringBuilder sb = new StringBuilder();
+        if (includeFrozenType)
+            sb.append(FrozenType.class.getName()).append('(');
+        sb.append(getClass().getName());
+        sb.append(TypeParser.stringifyTypeParameters(subTypes, ignoreFreezing || !isMultiCell));
+        if (includeFrozenType)
+            sb.append(')');
+        return sb.toString();
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/db/marshal/CompositeType.java b/src/java/org/apache/cassandra/db/marshal/CompositeType.java
index df7ee99070de..b22f5618a2a8 100644
--- a/src/java/org/apache/cassandra/db/marshal/CompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CompositeType.java
@@ -27,7 +27,6 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
 
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.SyntaxException;
@@ -39,9 +38,6 @@
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
-import static com.google.common.collect.Iterables.any;
-import static com.google.common.collect.Iterables.transform;
-
 /*
  * The encoding of a CompositeType column name should be:
  *   <component><component><component> ...
@@ -73,9 +69,9 @@ public static class Serializer extends BytesSerializer
     {
         // types are held to make sure the serializer is unique for each collection of types, this is to make sure it's
         // safe to cache in all cases
-        public final List<AbstractType<?>> types;
+        public final ImmutableList<AbstractType<?>> types;
 
-        public Serializer(List<AbstractType<?>> types)
+        public Serializer(ImmutableList<AbstractType<?>> types)
         {
             this.types = types;
         }
@@ -98,11 +94,10 @@ public int hashCode()
 
     private static final int STATIC_MARKER = 0xFFFF;
 
-    public final List<AbstractType<?>> types;
     private final Serializer serializer;
 
     // interning instances
-    private static final ConcurrentMap<List<AbstractType<?>>, CompositeType> instances = new ConcurrentHashMap<>();
+    private static final ConcurrentMap<ImmutableList<AbstractType<?>>, CompositeType> instances = new ConcurrentHashMap<>();
 
     public static CompositeType getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
     {
@@ -111,12 +106,12 @@ public static CompositeType getInstance(TypeParser parser) throws ConfigurationE
 
     public static CompositeType getInstance(Iterable<AbstractType<?>> types)
     {
-        return getInstance(Lists.newArrayList(types));
+        return getInstance(ImmutableList.copyOf(types));
     }
 
-    public static CompositeType getInstance(AbstractType... types)
+    public static CompositeType getInstance(AbstractType<?>... types)
     {
-        return getInstance(Arrays.asList(types));
+        return getInstance(ImmutableList.copyOf(types));
     }
 
     protected static int startingOffsetInternal(boolean isStatic)
@@ -159,25 +154,31 @@ private static boolean readStatic(ByteBuffer bb)
         return true;
     }
 
-    public static CompositeType getInstance(List<AbstractType<?>> types)
+    public static CompositeType getInstance(ImmutableList<AbstractType<?>> types)
     {
         assert types != null && !types.isEmpty();
-        CompositeType t = instances.get(types);
-        return null == t
-             ? instances.computeIfAbsent(types, CompositeType::new)
-             : t;
+        ImmutableList<AbstractType<?>> typesCopy = freeze(types);
+        return getInstance(instances, typesCopy, () -> new CompositeType(typesCopy));
+    }
+
+    protected CompositeType(Iterable<AbstractType<?>> subTypes)
+    {
+        this(ImmutableList.copyOf(subTypes));
     }
 
-    protected CompositeType(List<AbstractType<?>> types)
+    protected CompositeType(ImmutableList<AbstractType<?>> types)
     {
-        this.types = ImmutableList.copyOf(types);
-        this.serializer = new Serializer(this.types);
+        super(types);
+        this.serializer = new Serializer(this.subTypes);
     }
 
     @Override
-    public List<AbstractType<?>> subTypes()
+    public CompositeType with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
     {
-        return types;
+        if (isMultiCell)
+            throw new IllegalArgumentException("Cannot create a multi-cell CompositeType");
+
+        return getInstance(subTypes);
     }
 
     @Override
@@ -190,7 +191,7 @@ protected <V> AbstractType<?> getComparator(int i, V value, ValueAccessor<V> acc
     {
         try
         {
-            return types.get(i);
+            return subTypes.get(i);
         }
         catch (IndexOutOfBoundsException e)
         {
@@ -210,16 +211,21 @@ protected <VL, VR> AbstractType<?> getComparator(int i, VL left, ValueAccessor<V
 
     protected <V> AbstractType<?> getAndAppendComparator(int i, V value, ValueAccessor<V> accessor, StringBuilder sb, int offset)
     {
-        return types.get(i);
+        return subTypes.get(i);
     }
 
     @Override
     public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+    {
+        return asComparableBytes(accessor, data, version, ByteSource.TERMINATOR);
+    }
+
+    public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version, int terminator)
     {
         if (data == null || accessor.isEmpty(data))
             return null;
 
-        ByteSource[] srcs = new ByteSource[types.size() * 2 + 1];
+        ByteSource[] srcs = new ByteSource[subTypes.size() * 2 + 1];
         int length = accessor.size(data);
 
         // statics go first
@@ -237,7 +243,7 @@ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Versi
 
             int componentLength = accessor.getUnsignedShort(data, offset);
             offset += 2;
-            srcs[i * 2 + 1] = types.get(i).asComparableBytes(accessor, accessor.slice(data, offset, componentLength), version);
+            srcs[i * 2 + 1] = subTypes.get(i).asComparableBytes(accessor, accessor.slice(data, offset, componentLength), version);
             offset += componentLength;
             lastEoc = accessor.getByte(data, offset);
             offset += 1;
@@ -272,17 +278,17 @@ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable
         int separator = comparableBytes.next();
         boolean isStatic = ByteSourceInverse.nextComponentNull(separator);
         int i = 0;
-        V[] buffers = accessor.createArray(types.size());
+        V[] buffers = accessor.createArray(subTypes.size());
         byte lastEoc = 0;
 
-        while ((separator = comparableBytes.next()) != ByteSource.TERMINATOR && i < types.size())
+        while ((separator = comparableBytes.next()) != ByteSource.TERMINATOR && i < subTypes.size())
         {
             // Only the end-of-component byte of the last component of this composite can be non-zero, so the
             // component before can't have a non-zero end-of-component byte.
             assert lastEoc == 0 : lastEoc;
 
             // Get the next type and decode its payload.
-            AbstractType<?> type = types.get(i);
+            AbstractType<?> type = subTypes.get(i);
             V decoded = type.fromComparableBytes(accessor,
                                                  ByteSourceInverse.nextComponentSource(comparableBytes, separator),
                                                  version);
@@ -295,29 +301,29 @@ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable
 
     protected ParsedComparator parseComparator(int i, String part)
     {
-        return new StaticParsedComparator(types.get(i), part);
+        return new StaticParsedComparator(subTypes.get(i), part);
     }
 
     protected <V> AbstractType<?> validateComparator(int i, V value, ValueAccessor<V> accessor, int offset) throws MarshalException
     {
-        if (i >= types.size())
+        if (i >= subTypes.size())
             throw new MarshalException("Too many bytes for comparator");
-        return types.get(i);
+        return subTypes.get(i);
     }
 
-    protected <V> int getComparatorSize(int i, V value, ValueAccessor<V> accessor, int offset)
+    protected <V> int getComparatorSize(V value, ValueAccessor<V> accessor, int offset)
     {
         return 0;
     }
 
     public ByteBuffer decompose(Object... objects)
     {
-        assert objects.length == types.size() : String.format("Expected length %d but given %d", types.size(), objects.length);
+        assert objects.length == subTypes.size() : String.format("Expected length %d but given %d", subTypes.size(), objects.length);
 
         ByteBuffer[] serialized = new ByteBuffer[objects.length];
         for (int i = 0; i < objects.length; i++)
         {
-            ByteBuffer buffer = ((AbstractType) types.get(i)).decompose(objects[i]);
+            ByteBuffer buffer = ((AbstractType) subTypes.get(i)).decompose(objects[i]);
             serialized[i] = buffer;
         }
         return build(ByteBufferAccessor.instance, serialized);
@@ -328,7 +334,7 @@ public ByteBuffer[] split(ByteBuffer name)
     {
         // Assume all components, we'll trunk the array afterwards if need be, but
         // most names will be complete.
-        ByteBuffer[] l = new ByteBuffer[types.size()];
+        ByteBuffer[] l = new ByteBuffer[subTypes.size()];
         ByteBuffer bb = name.duplicate();
         readStatic(bb);
         int i = 0;
@@ -373,86 +379,55 @@ public static ByteBuffer extractComponent(ByteBuffer bb, int idx)
         return null;
     }
 
-    public static <V> boolean isStaticName(V value, ValueAccessor<V> accessor)
+    public static ByteBuffer extractFirstComponentAsTrieSearchPrefix(ByteBuffer bb, boolean isLowerBound)
     {
-        return accessor.size(value) >= 2 && (accessor.getUnsignedShort(value, 0) & 0xFFFF) == STATIC_MARKER;
+        bb = bb.duplicate();
+        readStatic(bb);
+        if (bb.remaining() == 0)
+            return null;
+
+        // We want to return the first two bytes, the component itself, and the end-of-component byte
+        int componentLength = bb.getShort(bb.position()) + 3;
+        int endOfComponentPosition = componentLength - 1;
+        // If this buffer is the lower bound or if the end-of-component byte is 1, we just need to set the limit
+        if (isLowerBound || bb.get(bb.position() + endOfComponentPosition) == (byte) 1)
+            return bb.limit(componentLength);
+
+        // We need to copy the first component and set the end-of-component byte to 1.
+        // See class's javadoc for explanation.
+        var dest = ByteBuffer.allocate(componentLength);
+        ByteBufferUtil.copyBytes(bb, bb.position(), dest, 0, endOfComponentPosition);
+        dest.put(endOfComponentPosition, (byte) 1);
+        return dest;
     }
 
-    @Override
-    public List<AbstractType<?>> getComponents()
+    public static <V> boolean isStaticName(V value, ValueAccessor<V> accessor)
     {
-        return types;
+        return accessor.size(value) >= 2 && (accessor.getUnsignedShort(value, 0) & 0xFFFF) == STATIC_MARKER;
     }
 
     @Override
     public boolean isCompatibleWith(AbstractType<?> previous)
     {
-        if (this == previous)
+        if (Objects.equals(this, previous))
             return true;
 
         if (!(previous instanceof CompositeType))
             return false;
 
-        // Extending with new components is fine
-        CompositeType cp = (CompositeType)previous;
-        if (types.size() < cp.types.size())
-            return false;
-
-        for (int i = 0; i < cp.types.size(); i++)
-        {
-            AbstractType tprev = cp.types.get(i);
-            AbstractType tnew = types.get(i);
-            if (!tnew.isCompatibleWith(tprev))
-                return false;
-        }
-        return true;
+        return isSubTypesCompatibleWith(previous, AbstractType::isCompatibleWith);
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        if (this == otherType)
+        if (Objects.equals(this, previous))
             return true;
 
-        if (!(otherType instanceof CompositeType))
-            return false;
-
-        // Extending with new components is fine
-        CompositeType cp = (CompositeType) otherType;
-        if (types.size() < cp.types.size())
+        if (!(previous instanceof CompositeType))
             return false;
 
-        for (int i = 0; i < cp.types.size(); i++)
-        {
-            AbstractType tprev = cp.types.get(i);
-            AbstractType tnew = types.get(i);
-            if (!tnew.isValueCompatibleWith(tprev))
-                return false;
-        }
-        return true;
-    }
-
-    @Override
-    public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
-    {
-        return any(types, t -> t.referencesUserType(name, accessor));
-    }
-
-    @Override
-    public CompositeType withUpdatedUserType(UserType udt)
-    {
-        if (!referencesUserType(udt.name))
-            return this;
-
-        instances.remove(types);
-
-        return getInstance(transform(types, t -> t.withUpdatedUserType(udt)));
-    }
-
-    @Override
-    public AbstractType<?> expandUserTypes()
-    {
-        return getInstance(transform(types, AbstractType::expandUserTypes));
+        return isSubTypesCompatibleWith(previous, AbstractType::isValueCompatibleWith);
     }
 
     private static class StaticParsedComparator implements ParsedComparator
@@ -485,24 +460,11 @@ public void serializeComparator(ByteBuffer bb) {}
     }
 
     @Override
-    public boolean equals(Object o)
-    {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        CompositeType that = (CompositeType) o;
-        return types.equals(that.types);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(types);
-    }
-
-    @Override
-    public String toString()
+    public String toString(boolean ignoreFreezing)
     {
-        return getClass().getName() + TypeParser.stringifyTypeParameters(types);
+        // Subtypes will always be frozen (since CompositeType always is), but we don't include it in the string
+        // representation (so that we ignore our parameter).
+        return getClass().getName() + TypeParser.stringifyTypeParameters(subTypes, true);
     }
 
     @SafeVarargs
diff --git a/src/java/org/apache/cassandra/db/marshal/DateRangeType.java b/src/java/org/apache/cassandra/db/marshal/DateRangeType.java
new file mode 100644
index 000000000000..b3db74f3321e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/DateRangeType.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.db.marshal.datetime.DateRange;
+import org.apache.cassandra.db.marshal.datetime.DateRangeUtil;
+import org.apache.cassandra.serializers.DateRangeSerializer;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TypeSerializer;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+
+/**
+ * Date range C* type with lower and upper bounds represented as timestamps with a millisecond precision.
+ */
+public class DateRangeType extends AbstractType<DateRange>
+{
+    public static final DateRangeType instance = new DateRangeType();
+
+    private static final ByteBuffer MASKED_VALUE = DateRangeSerializer.instance.serialize(new DateRange(DateRange.DateRangeBound.UNBOUNDED, DateRange.DateRangeBound.UNBOUNDED));
+
+    private DateRangeType()
+    {
+        super(ComparisonType.BYTE_ORDER);
+    }
+
+    @Override
+    public ByteBuffer fromString(String source) throws MarshalException
+    {
+        if (source.isEmpty())
+        {
+            return ByteBufferUtil.EMPTY_BYTE_BUFFER;
+        }
+        try
+        {
+            DateRange dateRange = DateRangeUtil.parseDateRange(source);
+            return decompose(dateRange);
+        }
+        catch (Exception e)
+        {
+            throw new MarshalException(String.format("Could not parse date range: %s %s", source, e.getMessage()), e);
+        }
+    }
+
+    @Override
+    public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
+    {
+        DateRange dateRange = this.getSerializer().deserialize(buffer);
+        return '"' + dateRange.formatToSolrString() + '"';
+    }
+
+    @Override
+    public Term fromJSONObject(Object parsed) throws MarshalException
+    {
+        if (parsed instanceof String)
+        {
+            return new Constants.Value(fromString((String) parsed));
+        }
+        throw new MarshalException(String.format(
+                "Expected a string representation of a date range value, but got a %s: %s",
+                parsed.getClass().getSimpleName(), parsed));
+    }
+
+    @Override
+    public boolean isEmptyValueMeaningless()
+    {
+        return true;
+    }
+
+    @Override
+    public TypeSerializer<DateRange> getSerializer()
+    {
+        return DateRangeSerializer.instance;
+    }
+
+    @Override
+    public ByteBuffer getMaskedValue()
+    {
+        return MASKED_VALUE;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/DateType.java b/src/java/org/apache/cassandra/db/marshal/DateType.java
index 05a6c4cf3667..63c2fa3f56e3 100644
--- a/src/java/org/apache/cassandra/db/marshal/DateType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DateType.java
@@ -20,16 +20,16 @@
 import java.nio.ByteBuffer;
 import java.util.Date;
 
-import org.apache.cassandra.cql3.Constants;
-import org.apache.cassandra.cql3.Term;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
-import org.apache.cassandra.serializers.TypeSerializer;
-import org.apache.cassandra.serializers.TimestampSerializer;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TimestampSerializer;
+import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@@ -46,6 +46,8 @@ public class DateType extends AbstractType<Date>
     private static final Logger logger = LoggerFactory.getLogger(DateType.class);
 
     public static final DateType instance = new DateType();
+
+    private static final TypeSerializer<Date> serializer = new TimestampSerializer();
     private static final ArgumentDeserializer ARGUMENT_DESERIALIZER = new DefaultArgumentDeserializer(instance);
     private static final ByteBuffer MASKED_VALUE = instance.decompose(new Date(0));
 
@@ -121,9 +123,9 @@ public boolean isCompatibleWith(AbstractType<?> previous)
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        return this == otherType || otherType == TimestampType.instance || otherType == LongType.instance;
+        return this == previous || previous == TimestampType.instance || previous == LongType.instance;
     }
 
     @Override
@@ -134,7 +136,7 @@ public CQL3Type asCQL3Type()
 
     public TypeSerializer<Date> getSerializer()
     {
-        return TimestampSerializer.instance;
+        return serializer;
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/marshal/DecimalType.java b/src/java/org/apache/cassandra/db/marshal/DecimalType.java
index 00da39ac35d0..eb3492f7d4ee 100644
--- a/src/java/org/apache/cassandra/db/marshal/DecimalType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DecimalType.java
@@ -234,7 +234,7 @@ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable
         // but when decoding we don't need that property on the transient mantissa value.
         BigInteger mantissa = BigInteger.ZERO;
         int curr = comparableBytes.next();
-        while (curr != DECIMAL_LAST_BYTE)
+        while (curr > DECIMAL_LAST_BYTE)
         {
             // The mantissa value is constructed by a standard positional notation value calculation.
             // The value of the next digit is the next most-significant mantissa byte as an unsigned integer,
@@ -338,7 +338,7 @@ public ArgumentDeserializer getArgumentDeserializer()
      * @param number the value to convert
      * @return the converted value
      */
-    protected BigDecimal toBigDecimal(Number number)
+    public BigDecimal toBigDecimal(Number number)
     {
         if (number instanceof BigDecimal)
             return (BigDecimal) number;
diff --git a/src/java/org/apache/cassandra/db/marshal/DurationType.java b/src/java/org/apache/cassandra/db/marshal/DurationType.java
index 0c466175543f..d53a83994795 100644
--- a/src/java/org/apache/cassandra/db/marshal/DurationType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DurationType.java
@@ -55,12 +55,6 @@ public ByteBuffer fromString(String source) throws MarshalException
         return decompose(Duration.from(source));
     }
 
-    @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
-    {
-        return this == otherType;
-    }
-
     public Term fromJSONObject(Object parsed) throws MarshalException
     {
         try
diff --git a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
index 658a147bc785..9dc9f4f5bfa1 100644
--- a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
@@ -22,17 +22,21 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,12 +49,11 @@
 import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
-import static com.google.common.collect.Iterables.any;
-
 /*
  * The encoding of a DynamicCompositeType column name should be:
  *   <component><component><component> ...
@@ -76,9 +79,9 @@ public static class Serializer extends BytesSerializer
     {
         // aliases are held to make sure the serializer is unique for each collection of types, this is to make sure it's
         // safe to cache in all cases
-        private final Map<Byte, AbstractType<?>> aliases;
+        private final ImmutableMap<Byte, AbstractType<?>> aliases;
 
-        public Serializer(Map<Byte, AbstractType<?>> aliases)
+        public Serializer(ImmutableMap<Byte, AbstractType<?>> aliases)
         {
             this.aliases = aliases;
         }
@@ -104,12 +107,12 @@ public int hashCode()
     private static final String REVERSED_TYPE = ReversedType.class.getSimpleName();
 
     @VisibleForTesting
-    public final Map<Byte, AbstractType<?>> aliases;
-    private final Map<AbstractType<?>, Byte> inverseMapping;
+    public final ImmutableMap<Byte, AbstractType<?>> aliases;
+    private final ImmutableMap<AbstractType<?>, Byte> inverseMapping;
     private final Serializer serializer;
 
     // interning instances
-    private static final ConcurrentHashMap<Map<Byte, AbstractType<?>>, DynamicCompositeType> instances = new ConcurrentHashMap<>();
+    private static final ConcurrentHashMap<ImmutableMap<Byte, AbstractType<?>>, DynamicCompositeType> instances = new ConcurrentHashMap<>();
 
     public static DynamicCompositeType getInstance(TypeParser parser)
     {
@@ -118,30 +121,36 @@ public static DynamicCompositeType getInstance(TypeParser parser)
 
     public static DynamicCompositeType getInstance(Map<Byte, AbstractType<?>> aliases)
     {
-        DynamicCompositeType dct = instances.get(aliases);
-        return null == dct
-             ? instances.computeIfAbsent(aliases, DynamicCompositeType::new)
-             : dct;
+        ImmutableMap<Byte, AbstractType<?>> aliasesCopy = ImmutableMap.copyOf(new TreeMap<>(Maps.transformValues(aliases, AbstractType::freeze)));
+        return getInstance(instances, aliasesCopy, () -> new DynamicCompositeType(aliasesCopy));
     }
 
-    private DynamicCompositeType(Map<Byte, AbstractType<?>> aliases)
+    private DynamicCompositeType(ImmutableMap<Byte, AbstractType<?>> aliases)
     {
-        this.aliases = ImmutableMap.copyOf(aliases);
+        super(ImmutableList.copyOf(aliases.values()));
+        this.aliases = aliases;
         this.serializer = new Serializer(this.aliases);
-        this.inverseMapping = new HashMap<>();
+        LinkedHashMap<AbstractType<?>, Byte> inverseMappingBuilder = new LinkedHashMap<>();
         for (Map.Entry<Byte, AbstractType<?>> en : aliases.entrySet())
-            this.inverseMapping.put(en.getValue(), en.getKey());
-    }
-
-    public int size()
-    {
-        return aliases.size();
+            inverseMappingBuilder.put(en.getValue(), en.getKey());
+        this.inverseMapping = ImmutableMap.copyOf(inverseMappingBuilder);
     }
 
     @Override
-    public List<AbstractType<?>> subTypes()
+    public AbstractType<ByteBuffer> with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
     {
-        return new ArrayList<>(aliases.values());
+        Preconditions.checkArgument(!isMultiCell, "Cannot create a multi-cell DynamicCompositeType");
+        Preconditions.checkArgument(subTypes.size() == aliases.size(),
+                                    "Invalid number of subTypes for DynamicCompositeType (got %s, expected %s)", subTypes.size(), aliases.size());
+
+        if (subTypes.equals(this.subTypes()) && isMultiCell == isMultiCell())
+            return this;
+
+        ImmutableMap.Builder<Byte, AbstractType<?>> copiedAliases = ImmutableMap.builderWithExpectedSize(subTypes.size());
+        Streams.zip(aliases.keySet().stream(), subTypes.stream(), Pair::create)
+               .forEachOrdered(p -> copiedAliases.put(p.left, p.right));
+
+        return new DynamicCompositeType(copiedAliases.build());
     }
 
     @Override
@@ -161,7 +170,7 @@ protected int startingOffset(boolean isStatic)
         return 0;
     }
 
-    protected <V> int getComparatorSize(int i, V value, ValueAccessor<V> accessor, int offset)
+    protected <V> int getComparatorSize(V value, ValueAccessor<V> accessor, int offset)
     {
         int header = accessor.getShort(value, offset);
         if ((header & 0x8000) == 0)
@@ -181,7 +190,6 @@ private <V> AbstractType<?> getComparator(V value, ValueAccessor<V> accessor, in
             int header = accessor.getShort(value, offset);
             if ((header & 0x8000) == 0)
             {
-
                 String name = accessor.toString(accessor.slice(value, offset + 2, header));
                 return TypeParser.parse(name);
             }
@@ -211,10 +219,10 @@ protected <VL, VR> AbstractType<?> getComparator(int i, VL left, ValueAccessor<V
          * If both types are ReversedType(Type), we need to compare on the wrapped type (which may differ between the two types) to avoid
          * incompatible comparisons being made.
          */
-        if ((comp1 instanceof ReversedType) && (comp2 instanceof ReversedType))
+        if (comp1.isReversed() && comp2.isReversed())
         {
-            comp1 = ((ReversedType<?>) comp1).baseType;
-            comp2 = ((ReversedType<?>) comp2).baseType;
+            comp1 = comp1.unwrap();
+            comp2 = comp2.unwrap();
         }
 
         // Fast test if the comparator uses singleton instances
@@ -284,11 +292,11 @@ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Versi
             assert lastEoc == 0 : lastEoc;
 
             AbstractType<?> comp = getComparator(data, accessor, offset);
-            offset += getComparatorSize(i, data, accessor, offset);
+            offset += getComparatorSize(data, accessor, offset);
             // The comparable bytes for the component need to ensure comparisons consistent with
             // AbstractCompositeType.compareCustom(ByteBuffer, ByteBuffer) and
             // DynamicCompositeType.getComparator(int, ByteBuffer, ByteBuffer):
-            if (version == Version.LEGACY || !(comp instanceof ReversedType))
+            if (version == Version.LEGACY || !comp.isReversed())
             {
                 // ...most often that means just adding the short name of the type, followed by the full name of the type.
                 srcs.add(ByteSource.of(comp.getClass().getSimpleName(), version));
@@ -296,14 +304,14 @@ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Versi
             }
             else
             {
-                // ...however some times the component uses a complex type (currently the only supported complex type
+                // ...however sometimes the component uses a complex type (currently the only supported complex type
                 // is ReversedType - we can't have elements that are of MapType, CompositeType, TupleType, etc.)...
-                ReversedType<?> reversedComp = (ReversedType<?>) comp;
+                AbstractType<?> baseType = comp.unwrap();
                 // ...in this case, we need to add the short name of ReversedType before the short name of the base
                 // type, to ensure consistency with DynamicCompositeType.getComparator(int, ByteBuffer, ByteBuffer).
                 srcs.add(ByteSource.of(REVERSED_TYPE, version));
-                srcs.add(ByteSource.of(reversedComp.baseType.getClass().getSimpleName(), version));
-                srcs.add(ByteSource.of(reversedComp.baseType.getClass().getName(), version));
+                srcs.add(ByteSource.of(baseType.getClass().getSimpleName(), version));
+                srcs.add(ByteSource.of(baseType.getClass().getName(), version));
             }
             // Only then the payload of the component gets encoded.
             int componentLength = accessor.getUnsignedShort(data, offset);
@@ -529,7 +537,7 @@ public ByteBuffer decompose(Object... objects)
     @Override
     public boolean isCompatibleWith(AbstractType<?> previous)
     {
-        if (this == previous)
+        if (Objects.equals(this, previous))
             return true;
 
         if (!(previous instanceof DynamicCompositeType))
@@ -539,41 +547,8 @@ public boolean isCompatibleWith(AbstractType<?> previous)
         // Note that modifying the type for an alias to a compatible type is
         // *not* fine since this would deal correctly with mixed aliased/not
         // aliased component.
-        DynamicCompositeType cp = (DynamicCompositeType)previous;
-        if (aliases.size() < cp.aliases.size())
-            return false;
-
-        for (Map.Entry<Byte, AbstractType<?>> entry : cp.aliases.entrySet())
-        {
-            AbstractType<?> tprev = entry.getValue();
-            AbstractType<?> tnew = aliases.get(entry.getKey());
-            if (tnew == null || tnew != tprev)
-                return false;
-        }
-        return true;
-    }
-
-    @Override
-    public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
-    {
-        return any(aliases.values(), t -> t.referencesUserType(name, accessor));
-    }
-
-    @Override
-    public DynamicCompositeType withUpdatedUserType(UserType udt)
-    {
-        if (!referencesUserType(udt.name))
-            return this;
-
-        instances.remove(aliases);
-
-        return getInstance(Maps.transformValues(aliases, v -> v.withUpdatedUserType(udt)));
-    }
-
-    @Override
-    public AbstractType<?> expandUserTypes()
-    {
-        return getInstance(Maps.transformValues(aliases, v -> v.expandUserTypes()));
+        DynamicCompositeType tprev = (DynamicCompositeType)previous;
+        return aliases.entrySet().containsAll(tprev.aliases.entrySet());
     }
 
     private class DynamicParsedComparator implements ParsedComparator
@@ -660,8 +635,11 @@ public void serializeComparator(ByteBuffer bb)
     @Override
     public boolean equals(Object o)
     {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
+        if (o == this)
+            return true;
+        if (!super.equals(o))
+            return false;
+
         DynamicCompositeType that = (DynamicCompositeType) o;
         return aliases.equals(that.aliases);
     }
@@ -673,8 +651,9 @@ public int hashCode()
     }
 
     @Override
-    public String toString()
+    public String toString(boolean ignoreFreezing)
     {
+        // DCT is always frozen, but implicitly so (FrozenType is never used), so we ignore our parameter
         return getClass().getName() + TypeParser.stringifyAliasesParameters(aliases);
     }
 
diff --git a/src/java/org/apache/cassandra/db/marshal/EmptyType.java b/src/java/org/apache/cassandra/db/marshal/EmptyType.java
index d9c7c22815ac..d6783a32c58f 100644
--- a/src/java/org/apache/cassandra/db/marshal/EmptyType.java
+++ b/src/java/org/apache/cassandra/db/marshal/EmptyType.java
@@ -75,7 +75,16 @@ private static NonEmptyWriteBehavior parseNonEmptyWriteBehavior()
     @Override
     public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
     {
-        return null;
+        switch (version)
+        {
+            case LEGACY:
+            case OSS41:
+                return null;
+            case OSS50:
+            default:
+                // EmptyType is being used in tuples where a null ByteSource is not acceptable. Use an empty source.
+                return ByteSource.EMPTY;
+        }
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/marshal/GeometryCodec.java b/src/java/org/apache/cassandra/db/marshal/GeometryCodec.java
new file mode 100644
index 000000000000..62c8804235e9
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/GeometryCodec.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.ProtocolVersion;
+import com.datastax.driver.core.TypeCodec;
+import com.datastax.driver.core.exceptions.InvalidTypeException;
+import org.apache.cassandra.db.marshal.geometry.LineString;
+import org.apache.cassandra.db.marshal.geometry.OgcGeometry;
+import org.apache.cassandra.db.marshal.geometry.Point;
+import org.apache.cassandra.db.marshal.geometry.Polygon;
+
+public class GeometryCodec<T extends OgcGeometry> extends TypeCodec<T>
+{
+    public static final TypeCodec<Point> pointCodec = new GeometryCodec<>(PointType.instance);
+    public static final TypeCodec<LineString> lineStringCodec = new GeometryCodec<>(LineStringType.instance);
+    public static final TypeCodec<Polygon> polygonCodec = new GeometryCodec<>(PolygonType.instance);
+
+    private final OgcGeometry.Serializer<T> serializer;
+
+    public GeometryCodec(AbstractGeometricType type)
+    {
+        super(DataType.custom(type.getClass().getName()), (Class<T>) type.getGeoType().getGeoClass());
+        this.serializer = (OgcGeometry.Serializer<T>) type.getGeoType().getSerializer();
+    }
+
+    @Override
+    public T deserialize(ByteBuffer bb, ProtocolVersion protocolVersion) throws InvalidTypeException
+    {
+        return bb == null || bb.remaining() == 0 ? null : serializer.fromWellKnownBinary(bb);
+    }
+
+    @Override
+    public ByteBuffer serialize(T geometry, ProtocolVersion protocolVersion) throws InvalidTypeException
+    {
+        return geometry == null ? null : geometry.asWellKnownBinary();
+    }
+
+    @Override
+    public T parse(String s) throws InvalidTypeException
+    {
+        if (s == null || s.isEmpty() || s.equalsIgnoreCase("NULL"))
+            return null;
+        return serializer.fromWellKnownText(s);
+    }
+
+    @Override
+    public String format(T geometry) throws InvalidTypeException
+    {
+        return geometry == null ? "NULL" : geometry.asWellKnownText();
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/IntegerType.java b/src/java/org/apache/cassandra/db/marshal/IntegerType.java
index 2dc0ae223861..aaf036f048fa 100644
--- a/src/java/org/apache/cassandra/db/marshal/IntegerType.java
+++ b/src/java/org/apache/cassandra/db/marshal/IntegerType.java
@@ -26,9 +26,9 @@
 import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
-import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.serializers.IntegerSerializer;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@@ -183,7 +183,7 @@ public static <VL, VR> int compareIntegers(VL lhs, ValueAccessor<VL> accessorL,
      *    2^56-1        as FEFFFFFFFFFFFFFF
      *    2^56          as FF000100000000000000
      *
-     * See {@link #asComparableBytesLegacy} for description of the legacy format.
+     * See {@link #asComparableBytes41} for description of the legacy format.
      */
     @Override
     public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
@@ -204,12 +204,12 @@ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteC
             }
         }
 
-        if (version != ByteComparable.Version.LEGACY)
+        if (version == ByteComparable.Version.OSS50)
             return (limit - p < FULL_FORM_THRESHOLD)
                    ? encodeAsVarInt(accessor, data, limit)
-                   : asComparableBytesCurrent(accessor, data, p, limit, (signbyte >> 7) & 0xFF);
+                   : asComparableBytes50(accessor, data, p, limit, (signbyte >> 7) & 0xFF);
         else
-            return asComparableBytesLegacy(accessor, data, p, limit, signbyte);
+            return asComparableBytes41(accessor, data, p, limit, signbyte);
     }
 
     /**
@@ -266,13 +266,14 @@ private <V> ByteSource encodeAsVarInt(ValueAccessor<V> accessor, V data, int lim
      * The representations are prefix-free, because representations of different length always have length bytes that
      * differ.
      */
-    private <V> ByteSource asComparableBytesCurrent(ValueAccessor<V> accessor, V data, int startpos, int limit, int signbyte)
+    private <V> ByteSource asComparableBytes50(ValueAccessor<V> accessor, V data, int startpos, int limit, int signbyte)
     {
+        assert startpos >= 0 && startpos + FULL_FORM_THRESHOLD <= limit;
         // start with sign as a byte, then variable-length-encoded length, then bytes (stripped leading sign)
         return new ByteSource()
         {
             int pos = -2;
-            ByteSource lengthEncoding = new VariableLengthUnsignedInteger(limit - startpos - FULL_FORM_THRESHOLD);
+            ByteSource lengthEncoding = new VariableLengthUnsignedInteger(limit - (startpos + FULL_FORM_THRESHOLD));
 
             @Override
             public int next()
@@ -323,7 +324,7 @@ else if (pos == -1)
      *    2^31          as 8380000000
      *    2^32          as 840100000000
      */
-    private <V> ByteSource asComparableBytesLegacy(ValueAccessor<V> accessor, V data, int startpos, int limit, int signbyte)
+    private <V> ByteSource asComparableBytes41(ValueAccessor<V> accessor, V data, int startpos, int limit, int signbyte)
     {
         return new ByteSource()
         {
@@ -366,6 +367,51 @@ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable
         if (comparableBytes == null)
             return accessor.empty();
 
+        switch (version)
+        {
+            case OSS41:
+                return fromComparableBytes41(accessor, comparableBytes);
+            case OSS50:
+                return fromComparableBytes50(accessor, comparableBytes);
+            case LEGACY:
+                throw new AssertionError("Legacy byte-comparable format is not revertible.");
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    private <V> V fromComparableBytes41(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes)
+    {
+        int valueBytes;
+        byte signedZero;
+        // Consume the first byte to determine whether the encoded number is positive and
+        // start iterating through the length header bytes and collecting the number of value bytes.
+        int curr = comparableBytes.next();
+        if (curr >= POSITIVE_VARINT_HEADER) // positive number
+        {
+            valueBytes = curr - POSITIVE_VARINT_HEADER + 1;
+            while (curr == POSITIVE_VARINT_LENGTH_HEADER)
+            {
+                curr = comparableBytes.next();
+                valueBytes += curr - POSITIVE_VARINT_HEADER + 1;
+            }
+            signedZero = 0;
+        }
+        else // negative number
+        {
+            valueBytes = POSITIVE_VARINT_HEADER - curr;
+            while (curr == NEGATIVE_VARINT_LENGTH_HEADER)
+            {
+                curr = comparableBytes.next();
+                valueBytes += POSITIVE_VARINT_HEADER - curr;
+            }
+            signedZero = -1;
+        }
+        return extractBytes(accessor, comparableBytes, signedZero, valueBytes);
+    }
+
+    public <V> V fromComparableBytes50(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes)
+    {
         // Consume the first byte to determine whether the encoded number is positive and
         // start iterating through the length header bytes and collecting the number of value bytes.
         int sign = comparableBytes.peek() ^ 0xFF;   // FF if negative, 00 if positive
@@ -491,9 +537,9 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        return this == otherType || Int32Type.instance.isValueCompatibleWith(otherType) || LongType.instance.isValueCompatibleWith(otherType);
+        return this == previous || Int32Type.instance.isValueCompatibleWith(previous) || LongType.instance.isValueCompatibleWith(previous);
     }
 
     public CQL3Type asCQL3Type()
diff --git a/src/java/org/apache/cassandra/db/marshal/LineStringType.java b/src/java/org/apache/cassandra/db/marshal/LineStringType.java
new file mode 100644
index 000000000000..21fe7e931593
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/LineStringType.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+
+import com.esri.core.geometry.ogc.OGCLineString;
+import org.apache.cassandra.db.marshal.geometry.GeometricType;
+import org.apache.cassandra.db.marshal.geometry.LineString;
+
+public class LineStringType extends AbstractGeometricType<LineString>
+{
+    public static final LineStringType instance = new LineStringType();
+
+    private static final ByteBuffer MASKED_VALUE = new LineString((OGCLineString) OGCLineString.fromText("LINESTRING EMPTY")).asWellKnownBinary();
+
+    public LineStringType()
+    {
+        super(GeometricType.LINESTRING);
+    }
+
+    @Override
+    public ByteBuffer getMaskedValue()
+    {
+        return MASKED_VALUE;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/ListType.java b/src/java/org/apache/cassandra/db/marshal/ListType.java
index 71f400dbb1b4..ed71cc0d6ab8 100644
--- a/src/java/org/apache/cassandra/db/marshal/ListType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ListType.java
@@ -25,18 +25,23 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Consumer;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
 import org.apache.cassandra.cql3.Lists;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.serializers.CollectionSerializer;
 import org.apache.cassandra.serializers.ListSerializer;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.JsonUtils;
 import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
 public class ListType<T> extends CollectionType<List<T>>
 {
@@ -46,7 +51,6 @@ public class ListType<T> extends CollectionType<List<T>>
 
     private final AbstractType<T> elements;
     public final ListSerializer<T> serializer;
-    private final boolean isMultiCell;
 
     public static ListType<?> getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
     {
@@ -57,50 +61,31 @@ public static ListType<?> getInstance(TypeParser parser) throws ConfigurationExc
         return getInstance(l.get(0).freeze(), true);
     }
 
+    @SuppressWarnings("unchecked")
     public static <T> ListType<T> getInstance(AbstractType<T> elements, boolean isMultiCell)
     {
-        ConcurrentHashMap<AbstractType<?>, ListType> internMap = isMultiCell ? instances : frozenInstances;
-        ListType<T> t = internMap.get(elements);
-        return null == t
-             ? internMap.computeIfAbsent(elements, k -> new ListType<>(k, isMultiCell))
-             : t;
+        return getInstance(isMultiCell ? instances : frozenInstances,
+                           elements,
+                           () -> new ListType<>(elements, isMultiCell));
     }
 
     private ListType(AbstractType<T> elements, boolean isMultiCell)
     {
-        super(ComparisonType.CUSTOM, Kind.LIST);
+        super(ComparisonType.CUSTOM, Kind.LIST, isMultiCell, ImmutableList.of(elements));
         this.elements = elements;
         this.serializer = ListSerializer.getInstance(elements.getSerializer());
-        this.isMultiCell = isMultiCell;
     }
 
     @Override
-    public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
+    @SuppressWarnings("unchecked")
+    public ListType<T> with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
     {
-        return elements.referencesUserType(name, accessor);
-    }
+        Preconditions.checkArgument(subTypes.size() == 1, "Invalid number of subTypes for ListType (got %s)", subTypes.size());
 
-    @Override
-    public ListType<?> withUpdatedUserType(UserType udt)
-    {
-        if (!referencesUserType(udt.name))
+        if (subTypes.equals(this.subTypes()) && isMultiCell == this.isMultiCell())
             return this;
 
-        (isMultiCell ? instances : frozenInstances).remove(elements);
-
-        return getInstance(elements.withUpdatedUserType(udt), isMultiCell);
-    }
-
-    @Override
-    public AbstractType<?> expandUserTypes()
-    {
-        return getInstance(elements.expandUserTypes(), isMultiCell);
-    }
-
-    @Override
-    public boolean referencesDuration()
-    {
-        return getElementsType().referencesDuration();
+        return getInstance((AbstractType<T>) subTypes.get(0), isMultiCell);
     }
 
     public AbstractType<T> getElementsType()
@@ -108,6 +93,7 @@ public AbstractType<T> getElementsType()
         return elements;
     }
 
+    @Override
     public AbstractType<TimeUUID> nameComparator()
     {
         return TimeUUIDType.instance;
@@ -123,72 +109,66 @@ public ListSerializer<T> getSerializer()
         return serializer;
     }
 
-    @Override
-    public AbstractType<?> freeze()
-    {
-        // freeze elements to match org.apache.cassandra.cql3.CQL3Type.Raw.RawCollection.freeze
-        return isMultiCell ? getInstance(this.elements.freeze(), false) : this;
-    }
-
-    @Override
-    public AbstractType<?> unfreeze()
-    {
-        return isMultiCell ? this : getInstance(this.elements, true);
-    }
-
-    @Override
-    public AbstractType<?> freezeNestedMulticellTypes()
-    {
-        if (!isMultiCell())
-            return this;
-
-        if (elements.isFreezable() && elements.isMultiCell())
-            return getInstance(elements.freeze(), isMultiCell);
-
-        return getInstance(elements.freezeNestedMulticellTypes(), isMultiCell);
-    }
-
-    @Override
-    public List<AbstractType<?>> subTypes()
+    public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
     {
-        return Collections.singletonList(elements);
+        return compareListOrSet(elements, left, accessorL, right, accessorR);
     }
 
     @Override
-    public boolean isMultiCell()
+    public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
     {
-        return isMultiCell;
+        return asComparableBytesListOrSet(getElementsType(), accessor, data, version);
     }
 
     @Override
-    public boolean isCompatibleWithFrozen(CollectionType<?> previous)
+    public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, Version version)
     {
-        assert !isMultiCell;
-        return this.elements.isCompatibleWith(((ListType<?>) previous).elements);
+        return fromComparableBytesListOrSet(accessor, comparableBytes, version, getElementsType());
     }
 
-    @Override
-    public boolean isValueCompatibleWithFrozen(CollectionType<?> previous)
+    static <V> ByteSource asComparableBytesListOrSet(AbstractType<?> elementsComparator,
+                                                     ValueAccessor<V> accessor,
+                                                     V data,
+                                                     Version version)
     {
-        assert !isMultiCell;
-        return this.elements.isValueCompatibleWithInternal(((ListType<?>) previous).elements);
-    }
+        if (accessor.isEmpty(data))
+            return null;
 
-    public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
-    {
-        return compareListOrSet(elements, left, accessorL, right, accessorR);
+        int offset = 0;
+        int size = CollectionSerializer.readCollectionSize(data, accessor);
+        offset += CollectionSerializer.sizeOfCollectionSize();
+        ByteSource[] srcs = new ByteSource[size];
+        for (int i = 0; i < size; ++i)
+        {
+            V v = CollectionSerializer.readValue(data, accessor, offset);
+            offset += CollectionSerializer.sizeOfValue(v, accessor);
+            srcs[i] = elementsComparator.asComparableBytes(accessor, v, version);
+        }
+        return ByteSource.withTerminatorMaybeLegacy(version, 0x00, srcs);
     }
 
-    @Override
-    public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+    static <V> V fromComparableBytesListOrSet(ValueAccessor<V> accessor,
+                                              ByteSource.Peekable comparableBytes,
+                                              Version version,
+                                              AbstractType<?> elementType)
     {
-        return asComparableBytesListOrSet(getElementsType(), accessor, data, version);
-    }
+        if (comparableBytes == null)
+            return accessor.empty();
 
-    @Override
-    public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, Version version)
-    {
-        return fromComparableBytesListOrSet(accessor, comparableBytes, version, getElementsType());
+        List<V> buffers = new ArrayList<>();
+        int terminator = version == Version.LEGACY
+                         ? 0x00
+                         : ByteSource.TERMINATOR;
+        int separator = comparableBytes.next();
+        while (separator != terminator)
+        {
+            if (!ByteSourceInverse.nextComponentNull(separator))
+                buffers.add(elementType.fromComparableBytes(accessor, comparableBytes, version));
+            else
+                buffers.add(null);
+            separator = comparableBytes.next();
+        }
+        return CollectionSerializer.pack(buffers, accessor, buffers.size());
     }
 
     @Override
@@ -198,11 +178,11 @@ public String toString(boolean ignoreFreezing)
 
         StringBuilder sb = new StringBuilder();
         if (includeFrozenType)
-            sb.append(FrozenType.class.getName()).append("(");
+            sb.append(FrozenType.class.getName()).append('(');
         sb.append(getClass().getName());
-        sb.append(TypeParser.stringifyTypeParameters(Collections.<AbstractType<?>>singletonList(elements), ignoreFreezing || !isMultiCell));
+        sb.append(TypeParser.stringifyTypeParameters(subTypes, ignoreFreezing || !isMultiCell()));
         if (includeFrozenType)
-            sb.append(")");
+            sb.append(')');
         return sb.toString();
     }
 
@@ -237,12 +217,6 @@ public Term fromJSONObject(Object parsed) throws MarshalException
         return new Lists.DelayedValue(terms);
     }
 
-    public ByteBuffer getSliceFromSerialized(ByteBuffer collection, ByteBuffer from, ByteBuffer to)
-    {
-        // We don't support slicing on lists so we don't need that function
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
     {
@@ -260,4 +234,15 @@ public ByteBuffer getMaskedValue()
     {
         return decompose(Collections.emptyList());
     }
+
+    public boolean isList()
+    {
+        return true;
+    }
+
+    @Override
+    public boolean contains(ByteBuffer list, ByteBuffer element)
+    {
+        return CollectionSerializer.contains(getElementsType(), list, element, false, false);
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/marshal/LongType.java b/src/java/org/apache/cassandra/db/marshal/LongType.java
index 97b9f7546879..1dd431e19dc1 100644
--- a/src/java/org/apache/cassandra/db/marshal/LongType.java
+++ b/src/java/org/apache/cassandra/db/marshal/LongType.java
@@ -26,9 +26,9 @@
 import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
-import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.serializers.LongSerializer;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@@ -77,7 +77,7 @@ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteC
     {
         if (accessor.isEmpty(data))
             return null;
-        if (version == ByteComparable.Version.LEGACY)
+        if (version != ByteComparable.Version.OSS50)
             return ByteSource.signedFixedLengthNumber(accessor, data);
         else
             return ByteSource.variableLengthInteger(accessor.getLong(data, 0));
@@ -88,7 +88,7 @@ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable
     {
         if (comparableBytes == null)
             return accessor.empty();
-        if (version == ByteComparable.Version.LEGACY)
+        if (version != ByteComparable.Version.OSS50)
             return ByteSourceInverse.getSignedFixedLength(accessor, comparableBytes, 8);
         else
             return accessor.valueOf(ByteSourceInverse.getVariableLengthInteger(comparableBytes));
@@ -142,9 +142,9 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        return this == otherType || otherType == DateType.instance || otherType == TimestampType.instance;
+        return this == previous || previous == DateType.instance || previous == TimestampType.instance;
     }
 
     public CQL3Type asCQL3Type()
diff --git a/src/java/org/apache/cassandra/db/marshal/MapType.java b/src/java/org/apache/cassandra/db/marshal/MapType.java
index 16d533ec5c9e..ce3e2a74650c 100644
--- a/src/java/org/apache/cassandra/db/marshal/MapType.java
+++ b/src/java/org/apache/cassandra/db/marshal/MapType.java
@@ -19,7 +19,6 @@
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -28,6 +27,9 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Consumer;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
 import org.apache.cassandra.cql3.Maps;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.db.rows.Cell;
@@ -38,11 +40,11 @@
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.JsonUtils;
+import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
-import org.apache.cassandra.utils.Pair;
 
 public class MapType<K, V> extends CollectionType<Map<K, V>>
 {
@@ -53,7 +55,6 @@ public class MapType<K, V> extends CollectionType<Map<K, V>>
     private final AbstractType<K> keys;
     private final AbstractType<V> values;
     private final MapSerializer<K, V> serializer;
-    private final boolean isMultiCell;
 
     public static MapType<?, ?> getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
     {
@@ -64,55 +65,32 @@ public class MapType<K, V> extends CollectionType<Map<K, V>>
         return getInstance(l.get(0).freeze(), l.get(1).freeze(), true);
     }
 
+    @SuppressWarnings("unchecked")
     public static <K, V> MapType<K, V> getInstance(AbstractType<K> keys, AbstractType<V> values, boolean isMultiCell)
     {
-        ConcurrentHashMap<Pair<AbstractType<?>, AbstractType<?>>, MapType> internMap = isMultiCell ? instances : frozenInstances;
-        Pair<AbstractType<?>, AbstractType<?>> p = Pair.create(keys, values);
-        MapType<K, V> t = internMap.get(p);
-        return null == t
-             ? internMap.computeIfAbsent(p, k -> new MapType<>(k.left, k.right, isMultiCell))
-             : t;
+        return getInstance(isMultiCell ? instances : frozenInstances, Pair.create(keys, values), () -> new MapType<>(keys, values, isMultiCell));
     }
 
     private MapType(AbstractType<K> keys, AbstractType<V> values, boolean isMultiCell)
     {
-        super(ComparisonType.CUSTOM, Kind.MAP);
+        super(ComparisonType.CUSTOM, Kind.MAP, isMultiCell, ImmutableList.of(keys, values));
         this.keys = keys;
         this.values = values;
         this.serializer = MapSerializer.getInstance(keys.getSerializer(),
                                                     values.getSerializer(),
                                                     keys.comparatorSet);
-        this.isMultiCell = isMultiCell;
     }
 
     @Override
-    public <T> boolean referencesUserType(T name, ValueAccessor<T> accessor)
+    @SuppressWarnings("unchecked")
+    public MapType<K, V> with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
     {
-        return keys.referencesUserType(name, accessor) || values.referencesUserType(name, accessor);
-    }
+        Preconditions.checkArgument(subTypes.size() == 2, "Invalid number of subTypes for MapType (got %s)", subTypes.size());
 
-    @Override
-    public MapType<?,?> withUpdatedUserType(UserType udt)
-    {
-        if (!referencesUserType(udt.name))
+        if (subTypes.equals(this.subTypes()) && isMultiCell == this.isMultiCell())
             return this;
 
-        (isMultiCell ? instances : frozenInstances).remove(Pair.create(keys, values));
-
-        return getInstance(keys.withUpdatedUserType(udt), values.withUpdatedUserType(udt), isMultiCell);
-    }
-
-    @Override
-    public AbstractType<?> expandUserTypes()
-    {
-        return getInstance(keys.expandUserTypes(), values.expandUserTypes(), isMultiCell);
-    }
-
-    @Override
-    public boolean referencesDuration()
-    {
-        // Maps cannot be created with duration as keys
-        return getValuesType().referencesDuration();
+        return getInstance((AbstractType<K>) subTypes.get(0), (AbstractType<V>) subTypes.get(1), isMultiCell);
     }
 
     public AbstractType<K> getKeysType()
@@ -125,6 +103,7 @@ public AbstractType<V> getValuesType()
         return values;
     }
 
+    @Override
     public AbstractType<K> nameComparator()
     {
         return keys;
@@ -135,64 +114,6 @@ public AbstractType<V> valueComparator()
         return values;
     }
 
-    @Override
-    public boolean isMultiCell()
-    {
-        return isMultiCell;
-    }
-
-    @Override
-    public List<AbstractType<?>> subTypes()
-    {
-        return Arrays.asList(keys, values);
-    }
-
-    @Override
-    public AbstractType<?> freeze()
-    {
-        // freeze key/value to match org.apache.cassandra.cql3.CQL3Type.Raw.RawCollection.freeze
-        return isMultiCell ? getInstance(this.keys.freeze(), this.values.freeze(), false) : this;
-    }
-
-    @Override
-    public AbstractType<?> unfreeze()
-    {
-        return isMultiCell ? this : getInstance(this.keys, this.values, true);
-    }
-
-    @Override
-    public AbstractType<?> freezeNestedMulticellTypes()
-    {
-        if (!isMultiCell())
-            return this;
-
-        AbstractType<?> keyType = (keys.isFreezable() && keys.isMultiCell())
-                                ? keys.freeze()
-                                : keys.freezeNestedMulticellTypes();
-
-        AbstractType<?> valueType = (values.isFreezable() && values.isMultiCell())
-                                  ? values.freeze()
-                                  : values.freezeNestedMulticellTypes();
-
-        return getInstance(keyType, valueType, isMultiCell);
-    }
-
-    @Override
-    public boolean isCompatibleWithFrozen(CollectionType<?> previous)
-    {
-        assert !isMultiCell;
-        MapType<?, ?> tprev = (MapType<?, ?>) previous;
-        return keys.isCompatibleWith(tprev.keys) && values.isCompatibleWith(tprev.values);
-    }
-
-    @Override
-    public boolean isValueCompatibleWithFrozen(CollectionType<?> previous)
-    {
-        assert !isMultiCell;
-        MapType<?, ?> tprev = (MapType<?, ?>) previous;
-        return keys.isCompatibleWith(tprev.keys) && values.isValueCompatibleWith(tprev.values);
-    }
-
     public <RL, TR> int compareCustom(RL left, ValueAccessor<RL> accessorL, TR right, ValueAccessor<TR> accessorR)
     {
         return compareMaps(keys, values, left, accessorL, right, accessorR);
@@ -307,19 +228,6 @@ protected int collectionSize(List<ByteBuffer> values)
         return values.size() / 2;
     }
 
-    public String toString(boolean ignoreFreezing)
-    {
-        boolean includeFrozenType = !ignoreFreezing && !isMultiCell();
-
-        StringBuilder sb = new StringBuilder();
-        if (includeFrozenType)
-            sb.append(FrozenType.class.getName()).append("(");
-        sb.append(getClass().getName()).append(TypeParser.stringifyTypeParameters(Arrays.asList(keys, values), ignoreFreezing || !isMultiCell));
-        if (includeFrozenType)
-            sb.append(")");
-        return sb.toString();
-    }
-
     public List<ByteBuffer> serializedValues(Iterator<Cell<?>> cells)
     {
         assert isMultiCell;
@@ -398,4 +306,29 @@ public ByteBuffer getMaskedValue()
     {
         return decompose(Collections.emptyMap());
     }
+
+    /**
+     * Checks if the specified serialized map contains the specified serialized map value.
+     *
+     * @param map a serialized map
+     * @param value a serialized map value
+     * @return {@code true} if the map contains the value, {@code false} otherwise
+     */
+    @Override
+    public boolean contains(ByteBuffer map, ByteBuffer value)
+    {
+        return CollectionSerializer.contains(getValuesType(), map, value, true, false);
+    }
+
+    /**
+     * Checks if the specified serialized map contains the specified serialized map key.
+     *
+     * @param map a serialized map
+     * @param key a serialized map key
+     * @return {@code true} if the map contains the key, {@code false} otherwise
+     */
+    public boolean containsKey(ByteBuffer map, ByteBuffer key)
+    {
+        return CollectionSerializer.contains(getKeysType(), map, key, true, true);
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/marshal/MultiCellCapableType.java b/src/java/org/apache/cassandra/db/marshal/MultiCellCapableType.java
new file mode 100644
index 000000000000..c57311fbcea3
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/MultiCellCapableType.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+
+/**
+ * Base class for all types that can be multi-cell (when not frozen).
+ * <p/>
+ * A multi-cell type is one whose value is composed of multiple sub-values that are laid out on multiple {@link Cell}
+ * instances (one for each sub-value), typically collections. This layout allows partial updates (of only some of
+ * the sub-values) without requiring a read-before-write operation.
+ * <p/>
+ * All multi-cell capable types can either be used as truly multi-cell types or can be used in a frozen state.
+ * In the latter case, the values are not laid out in multiple cells; instead, the entire value (with all its sub-values)
+ * is packed within a single cell value. This implies that partial updates without read-before-write are not possible.
+ * The {@link AbstractType#isMultiCell()} method indicates whether a given type is a multi-cell variant or a frozen one.
+ * Both variants are technically different types but represent the same values from a user perspective, with different
+ * capabilities.
+ *
+ * @param <T> the type of the values of this type.
+ */
+public abstract class MultiCellCapableType<T> extends AbstractType<T>
+{
+    protected MultiCellCapableType(ComparisonType comparisonType, boolean isMultiCell, ImmutableList<AbstractType<?>> subTypes)
+    {
+        super(comparisonType, isMultiCell, subTypes);
+    }
+
+    /**
+     * Returns the subtype/comparator to use for the {@link CellPath} part of cells forming values for this type when
+     * used in its multi-cell variant.
+     * <p/>
+     * Note: In theory, this method should not be accessed on frozen instances (where {@code isMultiCell() == false}).
+     * However, for convenience, it is expected that this method always returns a proper value "as if" the type was a
+     * multi-cell variant, even if it is not.
+     *
+     * @return the comparator for the {@link CellPath} component of cells of this type, regardless of whether the type
+     * is frozen or not.
+     */
+    public abstract AbstractType<?> nameComparator();
+
+    @Override
+    public final boolean isCompatibleWith(AbstractType<?> previous)
+    {
+        if (equals(previous))
+            return true;
+
+        if (!(previous instanceof MultiCellCapableType))
+            return false;
+
+        if (this.isMultiCell() != previous.isMultiCell())
+            return false;
+
+        MultiCellCapableType<?> prevType = (MultiCellCapableType<?>) previous;
+        return this.isMultiCell() ? isCompatibleWithMultiCell(prevType)
+                                  : isCompatibleWithFrozen(prevType);
+    }
+
+    /**
+     * Whether {@code this} type is compatible (including for sorting) with {@code previous}, assuming both are
+     * of the same class (so {@code previous} can be safely cast to whichever class implements this) and both are
+     * frozen.
+     */
+    protected abstract boolean isCompatibleWithFrozen(@Nonnull MultiCellCapableType<?> previous);
+
+    /**
+     * Whether {@code this} type is compatible (including for sorting) with {@code previous}, assuming both are
+     * of the same class (so {@code previous} can be safely cast to whichever class implements this) but neither
+     * are frozen.
+     */
+    protected abstract boolean isCompatibleWithMultiCell(@Nonnull MultiCellCapableType<?> previous);
+
+    @Override
+    public final boolean isSerializationCompatibleWith(AbstractType<?> previous)
+    {
+        if (equals(previous))
+            return true;
+
+        if (!(previous instanceof MultiCellCapableType))
+            return false;
+
+        if (this.isMultiCell() != previous.isMultiCell())
+            return false;
+
+        MultiCellCapableType<?> prevType = (MultiCellCapableType<?>) previous;
+        return isMultiCell() ? isSerializationCompatibleWithMultiCell(prevType)
+                             : isSerializationCompatibleWithFrozen(prevType);
+    }
+
+    /**
+     * Determines if the current type is serialization compatible with the given previous type.
+     * <p>
+     * Serialization compatibility is primarily concerned with the ability to read the serialized value from a buffer
+     * that contains other data after the value. This means the value must either have a fixed length or its length must
+     * be explicitly stored. In frozen collections or tuples, all serialized values are prefixed with their length,
+     * regardless of whether the value has a fixed or variable length. Therefore, to ensure serialization compatibility,
+     * it is sufficient to verify whether the types are value-compatible when frozen, in addition to checking the
+     * isMultiCell and exact type conditions.
+     * </p>
+     *
+     * @param previous the previous type to check compatibility against
+     * @return {@code true} if the current type is serialization compatible with the previous type, false otherwise
+     */
+    protected boolean isSerializationCompatibleWithFrozen(MultiCellCapableType<?> previous)
+    {
+        return isValueCompatibleWithFrozen(previous);
+    }
+
+    protected boolean isSerializationCompatibleWithMultiCell(MultiCellCapableType<?> previous)
+    {
+        return isCompatibleWithMultiCell(previous);
+    }
+
+    @Override
+    protected final boolean isValueCompatibleWithInternal(AbstractType<?> previous)
+    {
+        if (equals(previous))
+            return true;
+
+        if (!(previous instanceof MultiCellCapableType))
+            return false;
+
+        if (this.isMultiCell() != previous.isMultiCell())
+            return false;
+
+        MultiCellCapableType<?> prevType = (MultiCellCapableType<?>) previous;
+        return isMultiCell() ? isValueCompatibleWithMultiCell(prevType)
+                             : isValueCompatibleWithFrozen(prevType);
+    }
+
+    /**
+     * Whether {@code this} type is value-compatible with {@code previous}, assuming both are of the same class (so
+     * {@code previous} can be safely cast to whichever class implements this) and both are are frozen.
+     */
+    protected abstract boolean isValueCompatibleWithFrozen(MultiCellCapableType<?> previous);
+
+    protected boolean isValueCompatibleWithMultiCell(MultiCellCapableType<?> previous)
+    {
+        return isCompatibleWithMultiCell(previous);
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java b/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java
index 3186e6e08b82..bac9e5699ce1 100644
--- a/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java
+++ b/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 import java.util.Objects;
+import javax.annotation.Nullable;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Term;
@@ -33,20 +34,17 @@
 import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
-import javax.annotation.Nullable;
-
 /** for sorting columns representing row keys in the row ordering as determined by a partitioner.
  * Not intended for user-defined CFs, and will in fact error out if used with such. */
 public class PartitionerDefinedOrder extends AbstractType<ByteBuffer>
 {
     private final IPartitioner partitioner;
     private final AbstractType<?> partitionKeyType;
+    private final int hashCode;
     
     public PartitionerDefinedOrder(IPartitioner partitioner)
     {
-        super(ComparisonType.CUSTOM);
-        this.partitioner = partitioner;
-        this.partitionKeyType = null;
+        this(partitioner, null);
     }
 
     public PartitionerDefinedOrder(IPartitioner partitioner, AbstractType<?> partitionKeyType)
@@ -54,6 +52,7 @@ public PartitionerDefinedOrder(IPartitioner partitioner, AbstractType<?> partiti
         super(ComparisonType.CUSTOM);
         this.partitioner = partitioner;
         this.partitionKeyType = partitionKeyType;
+        this.hashCode = Objects.hash(partitioner, partitionKeyType);
     }
 
     public static AbstractType<?> getInstance(TypeParser parser)
@@ -159,11 +158,12 @@ public ArgumentDeserializer getArgumentDeserializer()
     }
 
     @Override
-    public String toString()
+    public String toString(boolean ignoreFreezing)
     {
         if (partitionKeyType != null && !DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5))
         {
-            return String.format("%s(%s:%s)", getClass().getName(), partitioner.getClass().getName(), partitionKeyType);
+            // TODO Partition key is always frozen, though - should we pass ignoreFreezing to partitionKeyType.toString()? The default toString assumed ignoreFreezing=false so leaving that way for now
+            return String.format("%s(%s:%s)", getClass().getName(), partitioner.getClass().getName(), partitionKeyType.toString(false));
         }
         // if Cassandra's major version is before 5, use the old behaviour
         return String.format("%s(%s)", getClass().getName(), partitioner.getClass().getName());
@@ -176,17 +176,20 @@ public AbstractType<?>  getPartitionKeyType()
     }
 
     @Override
-    public boolean equals(Object obj)
+    public final boolean equals(Object obj)
     {
         if (this == obj)
-        {
             return true;
-        }
-        if (obj instanceof PartitionerDefinedOrder)
-        {
-            PartitionerDefinedOrder other = (PartitionerDefinedOrder) obj;
-            return partitioner.equals(other.partitioner) && Objects.equals(partitionKeyType, other.partitionKeyType);
-        }
-        return false;
+        if (!super.equals(obj))
+            return false;
+
+        PartitionerDefinedOrder other = (PartitionerDefinedOrder) obj;
+        return partitioner.equals(other.partitioner) && Objects.equals(partitionKeyType, other.partitionKeyType);
+    }
+
+    @Override
+    public final int hashCode()
+    {
+        return hashCode;
     }
 }
diff --git a/src/java/org/apache/cassandra/db/marshal/PointType.java b/src/java/org/apache/cassandra/db/marshal/PointType.java
new file mode 100644
index 000000000000..efd095730d9f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/PointType.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.marshal.geometry.GeometricType;
+import org.apache.cassandra.db.marshal.geometry.Point;
+
+public class PointType extends AbstractGeometricType<Point>
+{
+    public static final PointType instance = new PointType();
+
+    private static final ByteBuffer MASKED_VALUE = new Point(0, 0).asWellKnownBinary();
+
+    public PointType()
+    {
+        super(GeometricType.POINT);
+    }
+
+    @Override
+    public ByteBuffer getMaskedValue()
+    {
+        return MASKED_VALUE;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/PolygonType.java b/src/java/org/apache/cassandra/db/marshal/PolygonType.java
new file mode 100644
index 000000000000..b9a7ca8af7c4
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/PolygonType.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+
+import com.esri.core.geometry.ogc.OGCPolygon;
+import org.apache.cassandra.db.marshal.geometry.GeometricType;
+import org.apache.cassandra.db.marshal.geometry.Polygon;
+
+public class PolygonType extends AbstractGeometricType<Polygon>
+{
+    public static final PolygonType instance = new PolygonType();
+
+    private static final ByteBuffer MASKED_VALUE = new Polygon((OGCPolygon) OGCPolygon.fromText("POLYGON EMPTY")).asWellKnownBinary();
+
+    public PolygonType()
+    {
+        super(GeometricType.POLYGON);
+    }
+
+    @Override
+    public ByteBuffer getMaskedValue()
+    {
+        return MASKED_VALUE;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/ReversedType.java b/src/java/org/apache/cassandra/db/marshal/ReversedType.java
index 89d1adb0399e..006c33c260cf 100644
--- a/src/java/org/apache/cassandra/db/marshal/ReversedType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ReversedType.java
@@ -18,10 +18,13 @@
 package org.apache.cassandra.db.marshal;
 
 import java.nio.ByteBuffer;
-import java.util.Map;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
@@ -35,7 +38,7 @@
 public class ReversedType<T> extends AbstractType<T>
 {
     // interning instances
-    private static final Map<AbstractType<?>, ReversedType> instances = new ConcurrentHashMap<>();
+    private static final Map<AbstractType<?>, ReversedType<?>> instances = new ConcurrentHashMap<>();
 
     public final AbstractType<T> baseType;
 
@@ -49,18 +52,41 @@ public static <T> ReversedType<T> getInstance(TypeParser parser)
 
     public static <T> ReversedType<T> getInstance(AbstractType<T> baseType)
     {
-        ReversedType<T> t = instances.get(baseType);
-        return null == t
-             ? instances.computeIfAbsent(baseType, ReversedType::new)
-             : t;
+        ReversedType<?> type = instances.get(baseType);
+        if (type != null)
+            return (ReversedType<T>) type;
+
+        // Stacking {@code ReversedType} is not only unnecessary but can also break some of the code.
+        // For instance, {@code AbstractType#isValueCompatibleWith} would end up triggering the exception thrown by
+        // {@code ReversedType#isValueCompatibleWithInternal}. Therefore, an exception should be thrown if such stacking
+        // is detected.
+        Preconditions.checkArgument(!(baseType instanceof ReversedType),
+                                    "Detected a type with 2 ReversedType() back-to-back, which is not allowed.");
+
+        // We avoid constructor calls in Map#computeIfAbsent to avoid recursive update exceptions because the automatic
+        // fixing of subtypes done by the top-level constructor might attempt a recursive update to the instances map.
+        ReversedType<T> instance = new ReversedType<>(baseType);
+        return (ReversedType<T>) instances.computeIfAbsent(baseType, k -> instance);
     }
 
     private ReversedType(AbstractType<T> baseType)
     {
-        super(ComparisonType.CUSTOM);
+        super(ComparisonType.CUSTOM, baseType.isMultiCell(), ImmutableList.of(baseType));
         this.baseType = baseType;
     }
 
+    @Override
+    public AbstractType<T> with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
+    {
+        Preconditions.checkArgument(subTypes.size() == 1,
+                                    "Invalid number of subTypes for ReversedType (got %s)", subTypes.size());
+
+        if (subTypes.equals(subTypes()) && isMultiCell == isMultiCell())
+            return this;
+
+        return (AbstractType<T>) getInstance(subTypes.get(0));
+    }
+
     public boolean isEmptyValueMeaningless()
     {
         return baseType.isEmptyValueMeaningless();
@@ -128,10 +154,16 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
     @Override
     public boolean isCompatibleWith(AbstractType<?> otherType)
     {
-        if (!(otherType instanceof ReversedType))
+        if (!otherType.isReversed())
             return false;
 
-        return this.baseType.isCompatibleWith(((ReversedType) otherType).baseType);
+        return this.baseType.isCompatibleWith(otherType.unwrap());
+    }
+
+    @Override
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    {
+        throw new AssertionError("This should have never been called on the ReversedType");
     }
 
     @Override
@@ -152,29 +184,6 @@ public ArgumentDeserializer getArgumentDeserializer()
         return baseType.getArgumentDeserializer();
     }
 
-    @Override
-    public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
-    {
-        return baseType.referencesUserType(name, accessor);
-    }
-
-    @Override
-    public AbstractType<?> expandUserTypes()
-    {
-        return getInstance(baseType.expandUserTypes());
-    }
-
-    @Override
-    public ReversedType<?> withUpdatedUserType(UserType udt)
-    {
-        if (!referencesUserType(udt.name))
-            return this;
-
-        instances.remove(baseType);
-
-        return getInstance(baseType.withUpdatedUserType(udt));
-    }
-
     @Override
     public int valueLengthIfFixed()
     {
@@ -188,12 +197,12 @@ public boolean isReversed()
     }
 
     @Override
-    public String toString()
+    public String toString(boolean ignoreFreezing)
     {
-        return getClass().getName() + "(" + baseType + ")";
+        return getClass().getName() + '(' + baseType + ')';
     }
 
-    private static final class ReversedPeekableByteSource extends ByteSource.Peekable
+    private static final class ReversedPeekableByteSource extends ByteSource.PeekableImpl
     {
         private final ByteSource.Peekable original;
 
diff --git a/src/java/org/apache/cassandra/db/marshal/SetType.java b/src/java/org/apache/cassandra/db/marshal/SetType.java
index 0a12bfa2fc3b..06fa4476b989 100644
--- a/src/java/org/apache/cassandra/db/marshal/SetType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SetType.java
@@ -18,15 +18,24 @@
 package org.apache.cassandra.db.marshal;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Consumer;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
 import org.apache.cassandra.cql3.Sets;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.serializers.CollectionSerializer;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.serializers.SetSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -42,7 +51,6 @@ public class SetType<T> extends CollectionType<Set<T>>
 
     private final AbstractType<T> elements;
     private final SetSerializer<T> serializer;
-    private final boolean isMultiCell;
 
     public static SetType<?> getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
     {
@@ -53,44 +61,32 @@ public static SetType<?> getInstance(TypeParser parser) throws ConfigurationExce
         return getInstance(l.get(0).freeze(), true);
     }
 
+    @SuppressWarnings("unchecked")
     public static <T> SetType<T> getInstance(AbstractType<T> elements, boolean isMultiCell)
     {
-        ConcurrentHashMap<AbstractType<?>, SetType> internMap = isMultiCell ? instances : frozenInstances;
-        SetType<T> t = internMap.get(elements);
-        return null == t
-             ? internMap.computeIfAbsent(elements, k -> new SetType<>(k, isMultiCell))
-             : t;
+        return getInstance(isMultiCell ? instances : frozenInstances,
+                           elements,
+                           () -> new SetType<>(elements, isMultiCell));
     }
 
     public SetType(AbstractType<T> elements, boolean isMultiCell)
     {
-        super(ComparisonType.CUSTOM, Kind.SET);
+        super(ComparisonType.CUSTOM, Kind.SET, isMultiCell, ImmutableList.of(elements));
         this.elements = elements;
         this.serializer = SetSerializer.getInstance(elements.getSerializer(), elements.comparatorSet);
-        this.isMultiCell = isMultiCell;
     }
 
     @Override
-    public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
+    @SuppressWarnings("unchecked")
+    public SetType<T> with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
     {
-        return elements.referencesUserType(name, accessor);
-    }
+        Preconditions.checkArgument(subTypes.size() == 1,
+                                    "Invalid number of subTypes for SetType (got %s)", subTypes.size());
 
-    @Override
-    public SetType<?> withUpdatedUserType(UserType udt)
-    {
-        if (!referencesUserType(udt.name))
+        if (subTypes.equals(this.subTypes()) && isMultiCell == this.isMultiCell())
             return this;
 
-        (isMultiCell ? instances : frozenInstances).remove(elements);
-
-        return getInstance(elements.withUpdatedUserType(udt), isMultiCell);
-    }
-
-    @Override
-    public AbstractType<?> expandUserTypes()
-    {
-        return getInstance(elements.expandUserTypes(), isMultiCell);
+        return getInstance((AbstractType<T>) subTypes.get(0), isMultiCell);
     }
 
     public AbstractType<T> getElementsType()
@@ -98,6 +94,7 @@ public AbstractType<T> getElementsType()
         return elements;
     }
 
+    @Override
     public AbstractType<T> nameComparator()
     {
         return elements;
@@ -108,57 +105,6 @@ public AbstractType<?> valueComparator()
         return EmptyType.instance;
     }
 
-    @Override
-    public boolean isMultiCell()
-    {
-        return isMultiCell;
-    }
-
-    @Override
-    public AbstractType<?> freeze()
-    {
-        // freeze elements to match org.apache.cassandra.cql3.CQL3Type.Raw.RawCollection.freeze
-        return isMultiCell ? getInstance(this.elements.freeze(), false) : this;
-    }
-
-    @Override
-    public AbstractType<?> unfreeze()
-    {
-        return isMultiCell ? this : getInstance(this.elements, true);
-    }
-
-    @Override
-    public List<AbstractType<?>> subTypes()
-    {
-        return Collections.singletonList(elements);
-    }
-
-    @Override
-    public AbstractType<?> freezeNestedMulticellTypes()
-    {
-        if (!isMultiCell())
-            return this;
-
-        if (elements.isFreezable() && elements.isMultiCell())
-            return getInstance(elements.freeze(), isMultiCell);
-
-        return getInstance(elements.freezeNestedMulticellTypes(), isMultiCell);
-    }
-
-    @Override
-    public boolean isCompatibleWithFrozen(CollectionType<?> previous)
-    {
-        assert !isMultiCell;
-        return this.elements.isCompatibleWith(((SetType<?>) previous).elements);
-    }
-
-    @Override
-    public boolean isValueCompatibleWithFrozen(CollectionType<?> previous)
-    {
-        // because sets are ordered, any changes to the type must maintain the ordering
-        return isCompatibleWithFrozen(previous);
-    }
-
     public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
     {
         return compareListOrSet(elements, left, accessorL, right, accessorR);
@@ -181,21 +127,6 @@ public SetSerializer<T> getSerializer()
         return serializer;
     }
 
-    @Override
-    public String toString(boolean ignoreFreezing)
-    {
-        boolean includeFrozenType = !ignoreFreezing && !isMultiCell();
-
-        StringBuilder sb = new StringBuilder();
-        if (includeFrozenType)
-            sb.append(FrozenType.class.getName()).append("(");
-        sb.append(getClass().getName());
-        sb.append(TypeParser.stringifyTypeParameters(Collections.<AbstractType<?>>singletonList(elements), ignoreFreezing || !isMultiCell));
-        if (includeFrozenType)
-            sb.append(")");
-        return sb.toString();
-    }
-
     public List<ByteBuffer> serializedValues(Iterator<Cell<?>> cells)
     {
         List<ByteBuffer> bbs = new ArrayList<>();
@@ -243,4 +174,10 @@ public ByteBuffer getMaskedValue()
     {
         return decompose(Collections.emptySet());
     }
+
+    @Override
+    public boolean contains(ByteBuffer set, ByteBuffer element)
+    {
+        return CollectionSerializer.contains(getElementsType(), set, element, false, false);
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java b/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
index a474d39a81c3..23cd91b45495 100644
--- a/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
@@ -75,9 +75,9 @@ public long toTimeInMillis(ByteBuffer buffer) throws MarshalException
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        return this == otherType || otherType == Int32Type.instance;
+        return this == previous || previous == Int32Type.instance;
     }
 
     public Term fromJSONObject(Object parsed) throws MarshalException
diff --git a/src/java/org/apache/cassandra/db/marshal/TimeType.java b/src/java/org/apache/cassandra/db/marshal/TimeType.java
index 67cf7dbceb39..4f2062e0b2fc 100644
--- a/src/java/org/apache/cassandra/db/marshal/TimeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TimeType.java
@@ -21,13 +21,13 @@
 import java.time.LocalTime;
 import java.time.ZoneOffset;
 
+import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
+import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.serializers.TimeSerializer;
-import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.serializers.TypeSerializer;
-import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
@@ -67,9 +67,9 @@ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        return this == otherType || otherType == LongType.instance;
+        return this == previous || previous == LongType.instance;
     }
 
     public Term fromJSONObject(Object parsed) throws MarshalException
diff --git a/src/java/org/apache/cassandra/db/marshal/TimestampType.java b/src/java/org/apache/cassandra/db/marshal/TimestampType.java
index 124060de995f..b0466a0f6af5 100644
--- a/src/java/org/apache/cassandra/db/marshal/TimestampType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TimestampType.java
@@ -20,16 +20,16 @@
 import java.nio.ByteBuffer;
 import java.util.Date;
 
-import org.apache.cassandra.cql3.Constants;
-import org.apache.cassandra.cql3.Duration;
-import org.apache.cassandra.cql3.Term;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.serializers.TypeSerializer;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Duration;
+import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.serializers.TimestampSerializer;
+import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@@ -151,9 +151,9 @@ public boolean isCompatibleWith(AbstractType<?> previous)
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        return this == otherType || otherType == DateType.instance || otherType == LongType.instance;
+        return this == previous || previous == DateType.instance || previous == LongType.instance;
     }
 
     public CQL3Type asCQL3Type()
diff --git a/src/java/org/apache/cassandra/db/marshal/TupleType.java b/src/java/org/apache/cassandra/db/marshal/TupleType.java
index 24d948425d37..73c3d492ac14 100644
--- a/src/java/org/apache/cassandra/db/marshal/TupleType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TupleType.java
@@ -24,29 +24,30 @@
 import java.util.List;
 import java.util.regex.Pattern;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
 
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.Tuples;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.serializers.*;
+import org.apache.cassandra.serializers.CollectionSerializer;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TupleSerializer;
+import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.JsonUtils;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
-import static com.google.common.collect.Iterables.any;
-import static com.google.common.collect.Iterables.transform;
-
 /**
  * This is essentially like a CompositeType, but it's not primarily meant for comparison, just
  * to pack multiple values together so has a more friendly encoding.
  */
-public class TupleType extends AbstractType<ByteBuffer>
+public class TupleType extends MultiCellCapableType<ByteBuffer>
 {
     private static final String COLON = ":";
     private static final Pattern COLON_PAT = Pattern.compile(COLON);
@@ -56,26 +57,22 @@ public class TupleType extends AbstractType<ByteBuffer>
     private static final Pattern AT_PAT = Pattern.compile(AT);
     private static final String ESCAPED_AT = "\\\\@";
     private static final Pattern ESCAPED_AT_PAT = Pattern.compile(ESCAPED_AT);
-    
-    protected final List<AbstractType<?>> types;
-
     private final TupleSerializer serializer;
 
-    public TupleType(List<AbstractType<?>> types)
+    public TupleType(Iterable<AbstractType<?>> subTypes)
     {
-        this(types, true);
+        this(freeze(subTypes), false);
     }
 
-    @VisibleForTesting
-    public TupleType(List<AbstractType<?>> types, boolean freezeInner)
+    public TupleType(Iterable<AbstractType<?>> subTypes, boolean isMultiCell)
     {
-        super(ComparisonType.CUSTOM);
+        this(ImmutableList.copyOf(subTypes), isMultiCell);
+    }
 
-        if (freezeInner)
-            this.types = Lists.newArrayList(transform(types, AbstractType::freeze));
-        else
-            this.types = types;
-        this.serializer = new TupleSerializer(fieldSerializers(types));
+    public TupleType(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
+    {
+        super(ComparisonType.CUSTOM, isMultiCell, subTypes);
+        this.serializer = new TupleSerializer(fieldSerializers(subTypes));
     }
 
     @Override
@@ -96,56 +93,29 @@ private static List<TypeSerializer<?>> fieldSerializers(List<AbstractType<?>> ty
     public static TupleType getInstance(TypeParser parser) throws ConfigurationException, SyntaxException
     {
         List<AbstractType<?>> types = parser.getTypeParameters();
-        for (int i = 0; i < types.size(); i++)
-            types.set(i, types.get(i).freeze());
-        return new TupleType(types);
-    }
-
-    @Override
-    public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
-    {
-        return any(types, t -> t.referencesUserType(name, accessor));
-    }
-
-    @Override
-    public TupleType withUpdatedUserType(UserType udt)
-    {
-        return referencesUserType(udt.name)
-             ? new TupleType(Lists.newArrayList(transform(types, t -> t.withUpdatedUserType(udt))))
-             : this;
+        return new TupleType(types, true);
     }
 
     @Override
-    public AbstractType<?> expandUserTypes()
+    public TupleType with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
     {
-        return new TupleType(Lists.newArrayList(transform(types, AbstractType::expandUserTypes)));
+        return new TupleType(subTypes, isMultiCell);
     }
 
     @Override
-    public boolean referencesDuration()
+    public ShortType nameComparator()
     {
-        return allTypes().stream().anyMatch(f -> f.referencesDuration());
+        return ShortType.instance;
     }
 
     public AbstractType<?> type(int i)
     {
-        return types.get(i);
+        return subTypes.get(i);
     }
 
     public int size()
     {
-        return types.size();
-    }
-
-    @Override
-    public List<AbstractType<?>> subTypes()
-    {
-        return types;
-    }
-
-    public List<AbstractType<?>> allTypes()
-    {
-        return types;
+        return subTypes.size();
     }
 
     public boolean isTuple()
@@ -161,9 +131,9 @@ public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right
         int offsetL = 0;
         int offsetR = 0;
 
-        for (int i = 0; !accessorL.isEmptyFromOffset(left, offsetL) && !accessorR.isEmptyFromOffset(right, offsetR) && i < types.size(); i++)
+        for (int i = 0; !accessorL.isEmptyFromOffset(left, offsetL) && !accessorR.isEmptyFromOffset(right, offsetR) && i < subTypes.size(); i++)
         {
-            AbstractType<?> comparator = types.get(i);
+            AbstractType<?> comparator = subTypes.get(i);
 
             int sizeL = accessorL.getInt(left, offsetL);
             offsetL += TypeSizes.INT_SIZE;
@@ -216,23 +186,24 @@ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteC
         switch (version)
         {
             case LEGACY:
-                return asComparableBytesLegacy(accessor, data);
+            case OSS41:
+                return asComparableBytesLegacy(accessor, data, version);
             case OSS50:
-                return asComparableBytesNew(accessor, data, version);
+                return asComparableBytes50(accessor, data, version);
             default:
                 throw new AssertionError();
         }
     }
 
-    private <V> ByteSource asComparableBytesLegacy(ValueAccessor<V> accessor, V data)
+    private <V> ByteSource asComparableBytesLegacy(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
     {
         if (accessor.isEmpty(data))
             return null;
 
         V[] bufs = split(accessor, data);  // this may be shorter than types.size -- other srcs remain null in that case
-        ByteSource[] srcs = new ByteSource[types.size()];
+        ByteSource[] srcs = new ByteSource[subTypes.size()];
         for (int i = 0; i < bufs.length; ++i)
-            srcs[i] = bufs[i] != null ? types.get(i).asComparableBytes(accessor, bufs[i], ByteComparable.Version.LEGACY) : null;
+            srcs[i] = bufs[i] != null ? subTypes.get(i).asComparableBytes(accessor, bufs[i], version) : null;
 
         // We always have a fixed number of sources, with the trailing ones possibly being nulls.
         // This can only result in a prefix if the last type in the tuple allows prefixes. Since that type is required
@@ -240,7 +211,7 @@ private <V> ByteSource asComparableBytesLegacy(ValueAccessor<V> accessor, V data
         return ByteSource.withTerminatorLegacy(ByteSource.END_OF_STREAM, srcs);
     }
 
-    private <V> ByteSource asComparableBytesNew(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+    private <V> ByteSource asComparableBytes50(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
     {
         if (accessor.isEmpty(data))
             return null;
@@ -253,7 +224,7 @@ private <V> ByteSource asComparableBytesNew(ValueAccessor<V> accessor, V data, B
 
         ByteSource[] srcs = new ByteSource[lengthWithoutTrailingNulls];
         for (int i = 0; i < lengthWithoutTrailingNulls; ++i)
-            srcs[i] = bufs[i] != null ? types.get(i).asComparableBytes(accessor, bufs[i], version) : null;
+            srcs[i] = bufs[i] != null ? subTypes.get(i).asComparableBytes(accessor, bufs[i], version) : null;
 
         // Because we stop early when there are trailing nulls, there needs to be an explicit terminator to make the
         // type prefix-free.
@@ -263,27 +234,30 @@ private <V> ByteSource asComparableBytesNew(ValueAccessor<V> accessor, V data, B
     @Override
     public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
     {
-        assert version == ByteComparable.Version.OSS50; // Reverse translation is not supported for the legacy version.
+        assert version != ByteComparable.Version.LEGACY; // Reverse translation is not supported for the legacy version.
         if (comparableBytes == null)
             return accessor.empty();
 
-        V[] componentBuffers = accessor.createArray(types.size());
-        for (int i = 0; i < types.size(); ++i)
+        V[] componentBuffers = accessor.createArray(subTypes.size());
+        for (int i = 0; i < subTypes.size(); ++i)
         {
             if (comparableBytes.peek() == ByteSource.TERMINATOR)
                 break;  // the rest of the fields remain null
-            AbstractType<?> componentType = types.get(i);
+            AbstractType<?> componentType = subTypes.get(i);
             ByteSource.Peekable component = ByteSourceInverse.nextComponentSource(comparableBytes);
             if (component != null)
                 componentBuffers[i] = componentType.fromComparableBytes(accessor, component, version);
             else
                 componentBuffers[i] = null;
         }
-        // consume terminator
-        int terminator = comparableBytes.next();
-        assert terminator == ByteSource.TERMINATOR : String.format("Expected TERMINATOR (0x%2x) after %d components",
-                                                                   ByteSource.TERMINATOR,
-                                                                   types.size());
+        if (version == ByteComparable.Version.OSS50)
+        {
+            // consume terminator
+            int terminator = comparableBytes.next();
+            assert terminator == ByteSource.TERMINATOR : String.format("Expected TERMINATOR (0x%2x) after %d components",
+                                                                       ByteSource.TERMINATOR,
+                                                                       subTypes.size());
+        }
         return buildValue(accessor, componentBuffers);
     }
 
@@ -440,13 +414,13 @@ public Term fromJSONObject(Object parsed) throws MarshalException
 
         List<?> list = (List<?>) parsed;
 
-        if (list.size() > types.size())
-            throw new MarshalException(String.format("Tuple contains extra items (expected %s): %s", types.size(), parsed));
-        else if (types.size() > list.size())
-            throw new MarshalException(String.format("Tuple is missing items (expected %s): %s", types.size(), parsed));
+        if (list.size() > subTypes.size())
+            throw new MarshalException(String.format("Tuple contains extra items (expected %s): %s", subTypes.size(), parsed));
+        else if (subTypes.size() > list.size())
+            throw new MarshalException(String.format("Tuple is missing items (expected %s): %s", subTypes.size(), parsed));
 
         List<Term> terms = new ArrayList<>(list.size());
-        Iterator<AbstractType<?>> typeIterator = types.iterator();
+        Iterator<AbstractType<?>> typeIterator = subTypes.iterator();
         for (Object element : list)
         {
             if (element == null)
@@ -469,7 +443,7 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
         ByteBuffer duplicated = buffer.duplicate();
         int offset = 0;
         StringBuilder sb = new StringBuilder("[");
-        for (int i = 0; i < types.size(); i++)
+        for (int i = 0; i < subTypes.size(); i++)
         {
             if (i > 0)
                 sb.append(", ");
@@ -479,7 +453,7 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
             if (value == null)
                 sb.append("null");
             else
-                sb.append(types.get(i).toJSONString(value, protocolVersion));
+                sb.append(subTypes.get(i).toJSONString(value, protocolVersion));
         }
         return sb.append("]").toString();
     }
@@ -490,61 +464,30 @@ public TypeSerializer<ByteBuffer> getSerializer()
     }
 
     @Override
-    public boolean isCompatibleWith(AbstractType<?> previous)
+    protected boolean isCompatibleWithFrozen(MultiCellCapableType<?> previous)
     {
         if (!(previous instanceof TupleType))
             return false;
 
-        // Extending with new components is fine, removing is not
-        TupleType tt = (TupleType)previous;
-        if (size() < tt.size())
-            return false;
-
-        for (int i = 0; i < tt.size(); i++)
-        {
-            AbstractType<?> tprev = tt.type(i);
-            AbstractType<?> tnew = type(i);
-            if (!tnew.isCompatibleWith(tprev))
-                return false;
-        }
-        return true;
+        return isSubTypesCompatibleWith(previous, AbstractType::isCompatibleWith);
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isCompatibleWithMultiCell(MultiCellCapableType<?> previous)
     {
-        if (!(otherType instanceof TupleType))
-            return false;
-
-        // Extending with new components is fine, removing is not
-        TupleType tt = (TupleType) otherType;
-        if (size() < tt.size())
+        if (!(previous instanceof TupleType))
             return false;
 
-        for (int i = 0; i < tt.size(); i++)
-        {
-            AbstractType<?> tprev = tt.type(i);
-            AbstractType<?> tnew = type(i);
-            if (!tnew.isValueCompatibleWith(tprev))
-                return false;
-        }
-        return true;
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hashCode(types);
+        return isSubTypesCompatibleWith(previous, AbstractType::isSerializationCompatibleWith);
     }
 
     @Override
-    public boolean equals(Object o)
+    protected boolean isValueCompatibleWithFrozen(MultiCellCapableType<?> previous)
     {
-        if (o.getClass() != TupleType.class)
+        if (!(previous instanceof TupleType))
             return false;
 
-        TupleType that = (TupleType)o;
-        return types.equals(that.types);
+        return isSubTypesCompatibleWith(previous, AbstractType::isValueCompatibleWith);
     }
 
     @Override
@@ -554,18 +497,35 @@ public CQL3Type asCQL3Type()
     }
 
     @Override
-    public String toString()
+    public String toString(boolean ignoreFreezing)
+    {
+        boolean includeFrozenType = !ignoreFreezing && !isMultiCell();
+
+        StringBuilder sb = new StringBuilder();
+        if (includeFrozenType)
+            sb.append(FrozenType.class.getName()).append('(');
+        sb.append(getClass().getName());
+        // FrozenType applies to anything nested (it wouldn't make sense otherwise) and so we only put once at the
+        // highest level. So we can ignore freezing in the subtypes if either we're already within a frozen type
+        // (we're a sub-type ourselves and frozenType has been included at the outer level), or we're frozen.
+        sb.append(stringifyTypeParameters(ignoreFreezing || !isMultiCell()));
+        if (includeFrozenType)
+            sb.append(')');
+        return sb.toString();
+    }
+
+    protected String stringifyTypeParameters(boolean ignoreFreezing)
     {
-        return getClass().getName() + TypeParser.stringifyTypeParameters(types, true);
+        return TypeParser.stringifyTypeParameters(subTypes, ignoreFreezing);
     }
 
     @Override
     public ByteBuffer getMaskedValue()
     {
-        ByteBuffer[] buffers = new ByteBuffer[types.size()];
-        for (int i = 0; i < types.size(); i++)
+        ByteBuffer[] buffers = new ByteBuffer[subTypes.size()];
+        for (int i = 0; i < subTypes.size(); i++)
         {
-            AbstractType<?> type = types.get(i);
+            AbstractType<?> type = subTypes.get(i);
             buffers[i] = type.getMaskedValue();
         }
 
diff --git a/src/java/org/apache/cassandra/db/marshal/TypeParser.java b/src/java/org/apache/cassandra/db/marshal/TypeParser.java
index 87df2c380506..4b2fce0e849f 100644
--- a/src/java/org/apache/cassandra/db/marshal/TypeParser.java
+++ b/src/java/org/apache/cassandra/db/marshal/TypeParser.java
@@ -30,6 +30,7 @@
 
 import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableMap;
+
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.FieldIdentifier;
 import org.apache.cassandra.dht.IPartitioner;
@@ -50,7 +51,7 @@ public class TypeParser
     // A cache of parsed string, specially useful for DynamicCompositeType
     private static volatile ImmutableMap<String, AbstractType<?>> cache = ImmutableMap.of();
 
-    public static final TypeParser EMPTY_PARSER = new TypeParser("", 0);
+    private static final TypeParser EMPTY_PARSER = new TypeParser("", 0);
 
     private TypeParser(String str, int idx)
     {
@@ -64,7 +65,9 @@ public TypeParser(String str)
     }
 
     /**
-     * Parse a string containing an type definition.
+     * Creates a new TypeParser and uses it to parse the given type definition string.
+     *
+     * @param str the string to parse.
      */
     public static AbstractType<?> parse(String str) throws SyntaxException, ConfigurationException
     {
@@ -118,15 +121,10 @@ public static AbstractType<?> parse(String str) throws SyntaxException, Configur
         }
     }
 
-    public static AbstractType<?> parse(CharSequence compareWith) throws SyntaxException, ConfigurationException
-    {
-        return parse(compareWith == null ? null : compareWith.toString());
-    }
-
     /**
      * Parse an AbstractType from current position of this parser.
      */
-    public AbstractType<?> parse() throws SyntaxException, ConfigurationException
+    private AbstractType<?> parse() throws SyntaxException, ConfigurationException
     {
         skipBlank();
         String name = readNextIdentifier();
@@ -229,18 +227,13 @@ public Map<String, String> getKeyValueParameters() throws SyntaxException
             }
             else if (str.charAt(idx) != ',' && str.charAt(idx) != ')')
             {
-                throwSyntaxError("unexpected character '" + str.charAt(idx) + "'");
+                throwSyntaxError("unexpected character '" + str.charAt(idx) + '\'');
             }
             map.put(k, v);
         }
         throw new SyntaxException(String.format("Syntax error parsing '%s' at char %d: unexpected end of string", str, idx));
     }
 
-    public static String stringifyVectorParameters(AbstractType<?> type, boolean ignoreFreezing, int dimension)
-    {
-        return "(" + type.toString(ignoreFreezing) + " , " + dimension + ")";
-    }
-
     public Vector getVectorParameters()
     {
         if (isEOS())
@@ -344,51 +337,6 @@ public Map<Byte, AbstractType<?>> getAliasParameters() throws SyntaxException, C
         throw new SyntaxException(String.format("Syntax error parsing '%s' at char %d: unexpected end of string", str, idx));
     }
 
-    public Map<ByteBuffer, CollectionType> getCollectionsParameters() throws SyntaxException, ConfigurationException
-    {
-        Map<ByteBuffer, CollectionType> map = new HashMap<>();
-
-        if (isEOS())
-            return map;
-
-        if (str.charAt(idx) != '(')
-            throw new IllegalStateException();
-
-        ++idx; // skipping '('
-
-        while (skipBlankAndComma())
-        {
-            if (str.charAt(idx) == ')')
-            {
-                ++idx;
-                return map;
-            }
-
-            ByteBuffer bb = fromHex(readNextIdentifier());
-
-            skipBlank();
-            if (str.charAt(idx) != ':')
-                throwSyntaxError("expecting ':' token");
-
-            ++idx;
-            skipBlank();
-            try
-            {
-                AbstractType<?> type = parse();
-                if (!(type instanceof CollectionType))
-                    throw new SyntaxException(type + " is not a collection type");
-                map.put(bb, (CollectionType)type);
-            }
-            catch (SyntaxException e)
-            {
-                SyntaxException ex = new SyntaxException(String.format("Exception while parsing '%s' around char %d", str, idx));
-                ex.initCause(e);
-                throw ex;
-            }
-        }
-        throw new SyntaxException(String.format("Syntax error parsing '%s' at char %d: unexpected end of string", str, idx));
-    }
-
     private ByteBuffer fromHex(String hex) throws SyntaxException
     {
         try
@@ -402,7 +350,7 @@ private ByteBuffer fromHex(String hex) throws SyntaxException
         }
     }
 
-    public Pair<Pair<String, ByteBuffer>, List<Pair<ByteBuffer, AbstractType>>> getUserTypeParameters() throws SyntaxException, ConfigurationException
+    public Pair<Pair<String, ByteBuffer>, List<Pair<ByteBuffer, AbstractType<?>>>> getUserTypeParameters() throws SyntaxException, ConfigurationException
     {
 
         if (isEOS() || str.charAt(idx) != '(')
@@ -414,7 +362,7 @@ public Pair<Pair<String, ByteBuffer>, List<Pair<ByteBuffer, AbstractType>>> getU
         String keyspace = readNextIdentifier();
         skipBlankAndComma();
         ByteBuffer typeName = fromHex(readNextIdentifier());
-        List<Pair<ByteBuffer, AbstractType>> defs = new ArrayList<>();
+        List<Pair<ByteBuffer, AbstractType<?>>> defs = new ArrayList<>();
 
         while (skipBlankAndComma())
         {
@@ -432,7 +380,7 @@ public Pair<Pair<String, ByteBuffer>, List<Pair<ByteBuffer, AbstractType>>> getU
             skipBlank();
             try
             {
-                AbstractType type = parse();
+                AbstractType<?> type = parse();
                 defs.add(Pair.create(name, type));
             }
             catch (SyntaxException e)
@@ -445,10 +393,14 @@ public Pair<Pair<String, ByteBuffer>, List<Pair<ByteBuffer, AbstractType>>> getU
         throw new SyntaxException(String.format("Syntax error parsing '%s' at char %d: unexpected end of string", str, idx));
     }
 
-    private static AbstractType<?> getAbstractType(String compareWith) throws ConfigurationException
+    /**
+     * Parse a type string and return the corresponding {@link AbstractType}. It is used for the type definition which
+     * is not followed by an opening parenthesis, e.g. "org.apache.cassandra.db.marshal.UTF8Type".
+     */
+    private static AbstractType<?> getAbstractType(String typeName) throws ConfigurationException
     {
-        String className = compareWith.contains(".") ? compareWith : "org.apache.cassandra.db.marshal." + compareWith;
-        Class<? extends AbstractType<?>> typeClass = FBUtilities.<AbstractType<?>>classForName(className, "abstract-type");
+        String className = typeName.contains(".") ? typeName : "org.apache.cassandra.db.marshal." + typeName;
+        Class<? extends AbstractType<?>> typeClass = FBUtilities.classForName(className, "abstract-type");
         try
         {
             Field field = typeClass.getDeclaredField("instance");
@@ -461,10 +413,16 @@ private static AbstractType<?> getAbstractType(String compareWith) throws Config
         }
     }
 
+    /**
+     * Parse a type string and return the corresponding {@link AbstractType}. It is used for the type definition
+     * which is followed by an opening parenthesis, e.g.
+     * "org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.UTF8Type)".
+     */
     private static AbstractType<?> getAbstractType(String compareWith, TypeParser parser) throws SyntaxException, ConfigurationException
     {
         String className = compareWith.contains(".") ? compareWith : "org.apache.cassandra.db.marshal." + compareWith;
-        Class<? extends AbstractType<?>> typeClass = FBUtilities.<AbstractType<?>>classForName(className, "abstract-type");
+        Class<? extends AbstractType<?>> typeClass = FBUtilities.classForName(className, "abstract-type");
+
         try
         {
             Method method = typeClass.getDeclaredMethod("getInstance", TypeParser.class);
@@ -478,12 +436,15 @@ private static AbstractType<?> getAbstractType(String compareWith, TypeParser pa
         }
         catch (InvocationTargetException e)
         {
-            ConfigurationException ex = new ConfigurationException("Invalid definition for comparator " + typeClass.getName() + ".");
-            ex.initCause(e.getTargetException());
-            throw ex;
+            throw new ConfigurationException("Invalid definition for comparator " + typeClass.getName(), e.getTargetException());
         }
     }
 
+    /**
+     * Parse a type string and return the corresponding AbstractType. It is used for the type definition which is not
+     * followed by an opening parenthesis, e.g. "org.apache.cassandra.db.marshal.UTF8Type", but does not have static
+     * {@code instance} field.
+     */
     private static AbstractType<?> getRawAbstractType(Class<? extends AbstractType<?>> typeClass) throws ConfigurationException
     {
         try
@@ -510,9 +471,7 @@ private static AbstractType<?> getRawAbstractType(Class<? extends AbstractType<?
         }
         catch (InvocationTargetException e)
         {
-            ConfigurationException ex = new ConfigurationException("Invalid definition for comparator " + typeClass.getName() + ".");
-            ex.initCause(e.getTargetException());
-            throw ex;
+            throw new ConfigurationException("Invalid definition for comparator " + typeClass.getName(), e.getTargetException());
         }
     }
 
@@ -609,25 +568,19 @@ public static String stringifyAliasesParameters(Map<Byte, AbstractType<?>> alias
         if (iter.hasNext())
         {
             Map.Entry<Byte, AbstractType<?>> entry = iter.next();
-            sb.append((char)(byte)entry.getKey()).append("=>").append(entry.getValue());
+            // Aliases are only used by DynamicCompositeType that is always frozen but without requiring a 'Frozen()'
+            // in its subtypes' representation.
+            sb.append((char)(byte)entry.getKey()).append("=>").append(entry.getValue().toString(true));
         }
         while (iter.hasNext())
         {
             Map.Entry<Byte, AbstractType<?>> entry = iter.next();
-            sb.append(',').append((char)(byte)entry.getKey()).append("=>").append(entry.getValue());
+            sb.append(',').append((char)(byte)entry.getKey()).append("=>").append(entry.getValue().toString(true));
         }
         sb.append(')');
         return sb.toString();
     }
 
-    /**
-     * Helper function to ease the writing of AbstractType.toString() methods.
-     */
-    public static String stringifyTypeParameters(List<AbstractType<?>> types)
-    {
-        return stringifyTypeParameters(types, false);
-    }
-
     /**
      * Helper function to ease the writing of AbstractType.toString() methods.
      */
@@ -637,40 +590,22 @@ public static String stringifyTypeParameters(List<AbstractType<?>> types, boolea
         for (int i = 0; i < types.size(); i++)
         {
             if (i > 0)
-                sb.append(",");
+                sb.append(',');
             sb.append(types.get(i).toString(ignoreFreezing));
         }
         return sb.append(')').toString();
     }
 
-    public static String stringifyCollectionsParameters(Map<ByteBuffer, ? extends CollectionType> collections)
-    {
-        StringBuilder sb = new StringBuilder();
-        sb.append('(');
-        boolean first = true;
-        for (Map.Entry<ByteBuffer, ? extends CollectionType> entry : collections.entrySet())
-        {
-            if (!first)
-                sb.append(',');
-
-            first = false;
-            sb.append(ByteBufferUtil.bytesToHex(entry.getKey())).append(":");
-            sb.append(entry.getValue());
-        }
-        sb.append(')');
-        return sb.toString();
-    }
-
     public static String stringifyUserTypeParameters(String keysace, ByteBuffer typeName, List<FieldIdentifier> fields,
                                                      List<AbstractType<?>> columnTypes, boolean ignoreFreezing)
     {
         StringBuilder sb = new StringBuilder();
-        sb.append('(').append(keysace).append(",").append(ByteBufferUtil.bytesToHex(typeName));
+        sb.append('(').append(keysace).append(',').append(ByteBufferUtil.bytesToHex(typeName));
 
         for (int i = 0; i < fields.size(); i++)
         {
             sb.append(',');
-            sb.append(ByteBufferUtil.bytesToHex(fields.get(i).bytes)).append(":");
+            sb.append(ByteBufferUtil.bytesToHex(fields.get(i).bytes)).append(':');
             sb.append(columnTypes.get(i).toString(ignoreFreezing));
         }
         sb.append(')');
diff --git a/src/java/org/apache/cassandra/db/marshal/UUIDType.java b/src/java/org/apache/cassandra/db/marshal/UUIDType.java
index a5abd922342d..5228c21cb555 100644
--- a/src/java/org/apache/cassandra/db/marshal/UUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/UUIDType.java
@@ -27,14 +27,14 @@
 import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.functions.ArgumentDeserializer;
-import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.serializers.UUIDSerializer;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UUIDGen;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
-import org.apache.cassandra.utils.UUIDGen;
 
 /**
  * Compares UUIDs using the following criteria:<br>
@@ -136,7 +136,7 @@ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteC
         swizzled.putLong(8, accessor.getLong(data, 8));
 
         // fixed-length thus prefix-free
-        return ByteSource.fixedLength(swizzled);
+        return ByteSource.preencoded(swizzled);
     }
 
     @Override
@@ -177,9 +177,9 @@ static <V> V makeUuidBytes(ValueAccessor<V> accessor, long high, long low)
     }
 
     @Override
-    public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
+    protected boolean isValueCompatibleWithInternal(AbstractType<?> previous)
     {
-        return otherType instanceof UUIDType || otherType instanceof TimeUUIDType;
+        return previous instanceof UUIDType || previous instanceof TimeUUIDType;
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/marshal/UserType.java b/src/java/org/apache/cassandra/db/marshal/UserType.java
index a5e79aca8409..089183574e2b 100644
--- a/src/java/org/apache/cassandra/db/marshal/UserType.java
+++ b/src/java/org/apache/cassandra/db/marshal/UserType.java
@@ -18,16 +18,28 @@
 package org.apache.cassandra.db.marshal;
 
 import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
 
 import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-
+import com.google.common.collect.ImmutableList;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.SchemaElement;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.UserTypes;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.CellPath;
 import org.apache.cassandra.schema.Difference;
@@ -57,70 +69,80 @@ public class UserType extends TupleType implements SchemaElement
 
     public final String keyspace;
     public final ByteBuffer name;
-    private final List<FieldIdentifier> fieldNames;
-    private final List<String> stringFieldNames;
-    private final boolean isMultiCell;
+    private final ImmutableList<FieldIdentifier> fieldNames;
+    private final ImmutableList<String> stringFieldNames;
     private final UserTypeSerializer serializer;
+    private final int hashCode;
+
+    public UserType(String keyspace, ByteBuffer name, Iterable<FieldIdentifier> fieldNames, Iterable<AbstractType<?>> fieldTypes, boolean isMultiCell)
+    {
+        this(keyspace, name, ImmutableList.copyOf(fieldNames), ImmutableList.copyOf(fieldTypes), isMultiCell);
+    }
 
-    public UserType(String keyspace, ByteBuffer name, List<FieldIdentifier> fieldNames, List<AbstractType<?>> fieldTypes, boolean isMultiCell)
+    public UserType(String keyspace, ByteBuffer name, ImmutableList<FieldIdentifier> fieldNames, ImmutableList<AbstractType<?>> fieldTypes, boolean isMultiCell)
     {
-        super(fieldTypes, false);
+        super(isMultiCell ? fieldTypes : freeze(fieldTypes), isMultiCell);
         assert fieldNames.size() == fieldTypes.size();
+        this.hashCode = Objects.hashCode(fieldNames, keyspace, name, super.hashCode());
         this.keyspace = keyspace;
         this.name = name;
         this.fieldNames = fieldNames;
-        this.stringFieldNames = new ArrayList<>(fieldNames.size());
-        this.isMultiCell = isMultiCell;
+        ImmutableList.Builder<String> stringFieldNamesBuilder = ImmutableList.builderWithExpectedSize(this.fieldNames.size());
 
-        LinkedHashMap<String , TypeSerializer<?>> fieldSerializers = new LinkedHashMap<>(fieldTypes.size());
-        for (int i = 0, m = fieldNames.size(); i < m; i++)
+        LinkedHashMap<String , TypeSerializer<?>> fieldSerializers = new LinkedHashMap<>(subTypes().size());
+        for (int i = 0; i < this.fieldNames.size(); i++)
         {
-            String stringFieldName = fieldNames.get(i).toString();
-            stringFieldNames.add(stringFieldName);
-            TypeSerializer<?> existing = fieldSerializers.put(stringFieldName, fieldTypes.get(i).getSerializer());
+            String stringFieldName = this.fieldNames.get(i).toString();
+            stringFieldNamesBuilder.add(stringFieldName);
+            TypeSerializer<?> existing = fieldSerializers.put(stringFieldName, subTypes().get(i).getSerializer());
             if (existing != null)
                 CONFLICT_BEHAVIOR.onConflict(keyspace, getNameAsString(), stringFieldName);
         }
+        this.stringFieldNames = stringFieldNamesBuilder.build();
         this.serializer = new UserTypeSerializer(fieldSerializers);
     }
 
+    @Override
+    public UserType with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
+    {
+        return new UserType(keyspace, name, fieldNames, subTypes, isMultiCell);
+    }
+
     public static UserType getInstance(TypeParser parser)
     {
-        Pair<Pair<String, ByteBuffer>, List<Pair<ByteBuffer, AbstractType>>> params = parser.getUserTypeParameters();
+        Pair<Pair<String, ByteBuffer>, List<Pair<ByteBuffer, AbstractType<?>>>> params = parser.getUserTypeParameters();
         String keyspace = params.left.left;
         ByteBuffer name = params.left.right;
-        List<FieldIdentifier> columnNames = new ArrayList<>(params.right.size());
-        List<AbstractType<?>> columnTypes = new ArrayList<>(params.right.size());
-        for (Pair<ByteBuffer, AbstractType> p : params.right)
+        ImmutableList.Builder<FieldIdentifier> columnNames = ImmutableList.builderWithExpectedSize(params.right.size());
+        ImmutableList.Builder<AbstractType<?>> columnTypes = ImmutableList.builderWithExpectedSize(params.right.size());
+        for (Pair<ByteBuffer, AbstractType<?>> p : params.right)
         {
             columnNames.add(new FieldIdentifier(p.left));
             columnTypes.add(p.right);
         }
 
-        return new UserType(keyspace, name, columnNames, columnTypes, true);
+        return new UserType(keyspace, name, columnNames.build(), columnTypes.build(), true);
     }
 
     @Override
-    public boolean isUDT()
+    public UserType overrideKeyspace(Function<String, String> overrideKeyspace)
     {
-        return true;
-    }
+        String newKeyspace = overrideKeyspace.apply(keyspace);
+        if (newKeyspace.equals(keyspace))
+            return this;
 
-    public boolean isTuple()
-    {
-        return false;
+        return new UserType(newKeyspace, name, fieldNames, subTypes().stream().map(t -> t.overrideKeyspace(overrideKeyspace)).collect(ImmutableList.toImmutableList()), isMultiCell());
     }
 
     @Override
-    public boolean isMultiCell()
+    public boolean isUDT()
     {
-        return isMultiCell;
+        return true;
     }
 
-    @Override
-    public boolean isFreezable()
+    public boolean isTuple()
     {
-        return true;
+        return false;
     }
 
     public AbstractType<?> fieldType(int i)
@@ -128,9 +150,9 @@ public AbstractType<?> fieldType(int i)
         return type(i);
     }
 
-    public List<AbstractType<?>> fieldTypes()
+    public ImmutableList<AbstractType<?>> fieldTypes()
     {
-        return types;
+        return subTypes;
     }
 
     public FieldIdentifier fieldName(int i)
@@ -143,7 +165,7 @@ public String fieldNameAsString(int i)
         return stringFieldNames.get(i);
     }
 
-    public List<FieldIdentifier> fieldNames()
+    public ImmutableList<FieldIdentifier> fieldNames()
     {
         return fieldNames;
     }
@@ -164,11 +186,6 @@ public CellPath cellPathForField(FieldIdentifier fieldName)
         return CellPath.create(ByteBufferUtil.bytes((short)fieldPosition(fieldName)));
     }
 
-    public ShortType nameComparator()
-    {
-        return ShortType.instance;
-    }
-
     public ByteBuffer serializeForNativeProtocol(Iterator<Cell<?>> cells, ProtocolVersion protocolVersion)
     {
         assert isMultiCell;
@@ -223,13 +240,13 @@ public Term fromJSONObject(Object parsed) throws MarshalException
 
         JsonUtils.handleCaseSensitivity(map);
 
-        List<Term> terms = new ArrayList<>(types.size());
+        List<Term> terms = new ArrayList<>(subTypes.size());
 
         Set keys = map.keySet();
         assert keys.isEmpty() || keys.iterator().next() instanceof String;
 
         int foundValues = 0;
-        for (int i = 0; i < types.size(); i++)
+        for (int i = 0; i < subTypes.size(); i++)
         {
             Object value = map.get(stringFieldNames.get(i));
             if (value == null)
@@ -238,7 +255,7 @@ public Term fromJSONObject(Object parsed) throws MarshalException
             }
             else
             {
-                terms.add(types.get(i).fromJSONObject(value));
+                terms.add(subTypes.get(i).fromJSONObject(value));
                 foundValues += 1;
             }
         }
@@ -262,7 +279,7 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
     {
         ByteBuffer[] buffers = split(ByteBufferAccessor.instance, buffer);
         StringBuilder sb = new StringBuilder("{");
-        for (int i = 0; i < types.size(); i++)
+        for (int i = 0; i < subTypes.size(); i++)
         {
             if (i > 0)
                 sb.append(", ");
@@ -279,7 +296,7 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
             if (valueBuffer == null)
                 sb.append("null");
             else
-                sb.append(types.get(i).toJSONString(valueBuffer, protocolVersion));
+                sb.append(subTypes.get(i).toJSONString(valueBuffer, protocolVersion));
         }
         return sb.append("}").toString();
     }
@@ -287,72 +304,25 @@ public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
     @Override
     public UserType freeze()
     {
-        return isMultiCell ? new UserType(keyspace, name, fieldNames, fieldTypes(), false) : this;
-    }
-
-    @Override
-    public UserType unfreeze()
-    {
-        return isMultiCell ? this : new UserType(keyspace, name, fieldNames, fieldTypes(), true);
-    }
-
-    @Override
-    public AbstractType<?> freezeNestedMulticellTypes()
-    {
-        if (!isMultiCell())
-            return this;
-
-        // the behavior here doesn't exactly match the method name: we want to freeze everything inside of UDTs
-        List<AbstractType<?>> newTypes = fieldTypes().stream()
-                .map(subtype -> (subtype.isFreezable() && subtype.isMultiCell() ? subtype.freeze() : subtype))
-                .collect(Collectors.toList());
-
-        return new UserType(keyspace, name, fieldNames, newTypes, isMultiCell);
+        return (UserType) super.freeze();
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hashCode(keyspace, name, fieldNames, types, isMultiCell);
-    }
-
-    @Override
-    public boolean isValueCompatibleWith(AbstractType<?> previous)
-    {
-        if (this == previous)
-            return true;
-
-        if (!(previous instanceof UserType))
-            return false;
-
-        UserType other = (UserType) previous;
-        if (isMultiCell != other.isMultiCell())
-            return false;
-
-        if (!keyspace.equals(other.keyspace))
-            return false;
-
-        Iterator<AbstractType<?>> thisTypeIter = types.iterator();
-        Iterator<AbstractType<?>> previousTypeIter = other.types.iterator();
-        while (thisTypeIter.hasNext() && previousTypeIter.hasNext())
-        {
-            if (!thisTypeIter.next().isCompatibleWith(previousTypeIter.next()))
-                return false;
-        }
-
-        // it's okay for the new type to have additional fields, but not for the old type to have additional fields
-        return !previousTypeIter.hasNext();
+        return hashCode;
     }
 
     @Override
     public boolean equals(Object o)
     {
-        if (o.getClass() != UserType.class)
+        if (o == this)
+            return true;
+        if (!super.equals(o))
             return false;
 
-        UserType that = (UserType)o;
-
-        return equalsWithoutTypes(that) && types.equals(that.types);
+        UserType that = (UserType) o;
+        return equalsWithoutTypes(that);
     }
 
     private boolean equalsWithoutTypes(UserType other)
@@ -402,49 +372,27 @@ public <V> boolean referencesUserType(V name, ValueAccessor<V> accessor)
     @Override
     public UserType withUpdatedUserType(UserType udt)
     {
-        if (!referencesUserType(udt.name))
-            return this;
+        // If we're not the UDT to update, we can rely on the default implementation
+        if (!name.equals(udt.name))
+            return (UserType) super.withUpdatedUserType(udt);
 
-        // preserve frozen/non-frozen status of the updated UDT
-        if (name.equals(udt.name))
-        {
-            return isMultiCell == udt.isMultiCell
-                 ? udt
-                 : new UserType(keyspace, name, udt.fieldNames(), udt.fieldTypes(), isMultiCell);
-        }
+        assert udt.isMultiCell();
 
-        return new UserType(keyspace,
-                            name,
-                            fieldNames,
-                            Lists.newArrayList(transform(fieldTypes(), t -> t.withUpdatedUserType(udt))),
-                            isMultiCell());
+        // The type we're updating may be frozen, while the updated user type will never be (a UDT is never frozen in
+        // its definition, only in its use). So if we are frozen, we should freeze the UDT we switch to.
+        return isMultiCell() ? udt : udt.freeze();
     }
 
     @Override
-    public boolean referencesDuration()
+    public AbstractType<?> expandUserTypes()
     {
-        return fieldTypes().stream().anyMatch(f -> f.referencesDuration());
+        return new TupleType(ImmutableList.copyOf(transform(subTypes, AbstractType::expandUserTypes)), isMultiCell());
     }
 
     @Override
-    public String toString()
+    protected String stringifyTypeParameters(boolean ignoreFreezing)
     {
-        return this.toString(false);
-    }
-
-    @Override
-    public String toString(boolean ignoreFreezing)
-    {
-        boolean includeFrozenType = !ignoreFreezing && !isMultiCell();
-
-        StringBuilder sb = new StringBuilder();
-        if (includeFrozenType)
-            sb.append(FrozenType.class.getName()).append("(");
-        sb.append(getClass().getName());
-        sb.append(TypeParser.stringifyUserTypeParameters(keyspace, name, fieldNames, types, ignoreFreezing || !isMultiCell));
-        if (includeFrozenType)
-            sb.append(")");
-        return sb.toString();
+        return TypeParser.stringifyUserTypeParameters(keyspace, name, fieldNames, subTypes, ignoreFreezing || !isMultiCell());
     }
 
     public String getCqlTypeName()
@@ -489,7 +437,7 @@ public String toCqlString(boolean withInternals, boolean ifNotExists)
 
         builder.appendQuotingIfNeeded(keyspace)
                .append('.')
-               .appendQuotingIfNeeded(getNameAsString())
+               .appendTypeQuotingIfNeeded(getNameAsString())
                .append(" (")
                .newLine()
                .increaseIndent();
diff --git a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java
index b4f775de9522..935714901fb1 100644
--- a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java
+++ b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java
@@ -41,7 +41,12 @@
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.vint.VIntCoding;
 
-import static org.apache.cassandra.db.ClusteringPrefix.Kind.*;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.EXCL_END_BOUND;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.EXCL_START_BOUND;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.INCL_END_BOUND;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.INCL_START_BOUND;
 
 /**
  * ValueAccessor allows serializers and other code dealing with raw bytes to operate on different backing types
@@ -69,7 +74,7 @@ public interface ObjectFactory<V>
         Cell<V> cell(ColumnMetadata column, long timestamp, int ttl, long localDeletionTime, V value, CellPath path);
         Clustering<V> clustering(V... values);
         Clustering<V> clustering();
-        Clustering<V> staticClustering();
+        // Note: the static clustering is always Clustering.STATIC_CLUSTERING (of ByteBuffer accessor).
         ClusteringBound<V> bound(ClusteringPrefix.Kind kind, V... values);
         ClusteringBound<V> bound(ClusteringPrefix.Kind kind);
         ClusteringBoundary<V> boundary(ClusteringPrefix.Kind kind, V... values);
diff --git a/src/java/org/apache/cassandra/db/marshal/VectorType.java b/src/java/org/apache/cassandra/db/marshal/VectorType.java
index d922d37b4262..ed33dcc04f2c 100644
--- a/src/java/org/apache/cassandra/db/marshal/VectorType.java
+++ b/src/java/org/apache/cassandra/db/marshal/VectorType.java
@@ -19,15 +19,19 @@
 package org.apache.cassandra.db.marshal;
 
 import java.nio.ByteBuffer;
+import java.nio.FloatBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
-
 import javax.annotation.Nullable;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.Vectors;
@@ -74,18 +78,27 @@ public int hashCode()
             return Objects.hash(type, dimension);
         }
     }
+
     @SuppressWarnings("rawtypes")
-    private static final ConcurrentHashMap<Key, VectorType> instances = new ConcurrentHashMap<>();
+    private static final ConcurrentHashMap<Key, VectorType<?>> instances = new ConcurrentHashMap<>();
 
     public final AbstractType<T> elementType;
     public final int dimension;
     private final TypeSerializer<T> elementSerializer;
     private final int valueLengthIfFixed;
     private final VectorSerializer serializer;
+    private final int hashCode;
+
+    private static final boolean isVectorTypeAllowed = CassandraRelevantProperties.VECTOR_TYPE_ALLOWED.getBoolean();
+    private static final boolean isVectorTypeFloatOnly = CassandraRelevantProperties.VECTOR_FLOAT_ONLY.getBoolean();
 
     private VectorType(AbstractType<T> elementType, int dimension)
     {
-        super(ComparisonType.CUSTOM);
+        super(ComparisonType.CUSTOM, false, ImmutableList.of(elementType));
+        if (!isVectorTypeAllowed)
+            throw new InvalidRequestException("vector type is not allowed");
+        if (isVectorTypeFloatOnly && !(elementType instanceof FloatType))
+            throw new InvalidRequestException(String.format("vectors may only use float. given %s", elementType.asCQL3Type()));
         if (dimension <= 0)
             throw new InvalidRequestException(String.format("vectors may only have positive dimensions; given %d", dimension));
         this.elementType = elementType;
@@ -97,13 +110,14 @@ private VectorType(AbstractType<T> elementType, int dimension)
         this.serializer = elementType.isValueLengthFixed() ?
                           new FixedLengthSerializer() :
                           new VariableLengthSerializer();
+        this.hashCode = Objects.hash(elementType, dimension);
     }
 
     @SuppressWarnings("unchecked")
     public static <T> VectorType<T> getInstance(AbstractType<T> elements, int dimension)
     {
         Key key = new Key(elements, dimension);
-        return instances.computeIfAbsent(key, Key::create);
+        return (VectorType<T>) getInstance(instances, key, key::create);
     }
 
     public static VectorType<?> getInstance(TypeParser parser)
@@ -112,6 +126,19 @@ public static VectorType<?> getInstance(TypeParser parser)
         return getInstance(v.type.freeze(), v.dimension);
     }
 
+    @Override
+    @SuppressWarnings("unchecked")
+    public VectorType<T> with(ImmutableList<AbstractType<?>> subTypes, boolean isMultiCell)
+    {
+        Preconditions.checkArgument(subTypes.size() == 1, "Invalid number of subTypes for VectorType (got %s)", subTypes.size());
+        Preconditions.checkArgument(!isMultiCell, "Cannot create a multi-cell VectorType");
+
+        if (subTypes.equals(this.subTypes()))
+            return this;
+
+        return getInstance((AbstractType<T>) subTypes.get(0), dimension);
+    }
+
     @Override
     public boolean isVector()
     {
@@ -262,12 +289,6 @@ public ByteBuffer fromString(String source) throws MarshalException
         }
     }
 
-    @Override
-    public List<AbstractType<?>> subTypes()
-    {
-        return Collections.singletonList(elementType);
-    }
-
     @Override
     public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
     {
@@ -317,28 +338,24 @@ public Term fromJSONObject(Object parsed) throws MarshalException
     @Override
     public boolean equals(Object o)
     {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
+        if (this == o)
+            return true;
+        if (!super.equals(o))
+            return false;
         VectorType<?> that = (VectorType<?>) o;
-        return dimension == that.dimension && Objects.equals(elementType, that.elementType);
+        return dimension == that.dimension;
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hash(elementType, dimension);
-    }
-
-    @Override
-    public String toString()
-    {
-        return toString(false);
+        return hashCode;
     }
 
     @Override
     public String toString(boolean ignoreFreezing)
     {
-        return getClass().getName() + TypeParser.stringifyVectorParameters(elementType, ignoreFreezing, dimension);
+        return String.format("%s(%s,%d)", getClass().getName(), elementType, dimension);
     }
 
     private void check(List<?> values)
@@ -380,6 +397,8 @@ public abstract class VectorSerializer extends TypeSerializer<List<T>>
 
         public abstract <V> List<V> split(V buffer, ValueAccessor<V> accessor);
         public abstract <V> V serializeRaw(List<V> elements, ValueAccessor<V> accessor);
+        public abstract float[] deserializeFloatArray(ByteBuffer input);
+        public abstract ByteBuffer serializeFloatArray(float[] value);
 
         @Override
         public String toString(List<T> value)
@@ -426,6 +445,8 @@ public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL,
         {
             if (elementType.isByteOrderComparable)
                 return ValueAccessor.compare(left, accessorL, right, accessorR);
+            if (accessorL.isEmpty(left) || accessorR.isEmpty(right))
+                return Boolean.compare(accessorR.isEmpty(right), accessorL.isEmpty(left));
             int offset = 0;
             int elementLength = elementType.valueLengthIfFixed();
             for (int i = 0; i < dimension; i++)
@@ -510,6 +531,34 @@ public <V> List<T> deserialize(V input, ValueAccessor<V> accessor)
             return result;
         }
 
+        @Override
+        public float[] deserializeFloatArray(ByteBuffer input)
+        {
+            if (input == null || input.remaining() == 0)
+                return null;
+
+            FloatBuffer floatBuffer = input.asFloatBuffer();
+            float[] floatArray = new float[floatBuffer.remaining()];
+            floatBuffer.get(floatArray);
+
+            return floatArray;
+        }
+
+        @Override
+        public ByteBuffer serializeFloatArray(float[] value)
+        {
+            if (elementType != FloatType.instance)
+                throw new UnsupportedOperationException();
+
+            if (value.length != dimension)
+                throw new MarshalException(String.format("Required %d elements, but saw %d", dimension, value.length));
+
+            var fb = FloatBuffer.wrap(value);
+            var bb = ByteBuffer.allocate(fb.capacity() * Float.BYTES);
+            bb.asFloatBuffer().put(fb);
+            return bb;
+        }
+
         @Override
         public <V> void validate(V input, ValueAccessor<V> accessor) throws MarshalException
         {
@@ -543,6 +592,9 @@ private VariableLengthSerializer()
         public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL,
                                           VR right, ValueAccessor<VR> accessorR)
         {
+            if (accessorL.isEmpty(left) || accessorR.isEmpty(right))
+                return Boolean.compare(accessorR.isEmpty(right), accessorL.isEmpty(left));
+
             int leftOffset = 0;
             int rightOffset = 0;
             for (int i = 0; i < dimension; i++)
@@ -648,6 +700,17 @@ public <V> List<T> deserialize(V input, ValueAccessor<V> accessor)
             return result;
         }
 
+        public float[] deserializeFloatArray(ByteBuffer input)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ByteBuffer serializeFloatArray(float[] value)
+        {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public <V> void validate(V input, ValueAccessor<V> accessor) throws MarshalException
         {
diff --git a/src/java/org/apache/cassandra/db/marshal/datetime/DateRange.java b/src/java/org/apache/cassandra/db/marshal/datetime/DateRange.java
new file mode 100644
index 000000000000..867528a6470a
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/datetime/DateRange.java
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.datetime;
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.util.Locale;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+
+import org.apache.cassandra.db.marshal.DateRangeType;
+
+import static java.time.temporal.ChronoField.DAY_OF_MONTH;
+import static java.time.temporal.ChronoField.HOUR_OF_DAY;
+import static java.time.temporal.ChronoField.MILLI_OF_SECOND;
+import static java.time.temporal.ChronoField.MINUTE_OF_HOUR;
+import static java.time.temporal.ChronoField.MONTH_OF_YEAR;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+
+/**
+ * Domain object of type {@link DateRangeType}. Lower and upper bounds are inclusive. Value type.
+ */
+public class DateRange
+{
+    private final DateRangeBound lowerBound;
+    private final DateRangeBound upperBound;
+
+    public DateRange(DateRangeBound lowerBound)
+    {
+        Preconditions.checkArgument(lowerBound != null);
+        this.lowerBound = lowerBound;
+        this.upperBound = null;
+    }
+
+    public DateRange(DateRangeBound lowerBound, DateRangeBound upperBound)
+    {
+        Preconditions.checkArgument(lowerBound != null);
+        Preconditions.checkArgument(upperBound != null);
+        Preconditions.checkArgument(upperBound.isAfter(lowerBound), "Wrong order: " + lowerBound + " TO " + upperBound);
+        this.lowerBound = lowerBound;
+        this.upperBound = upperBound;
+    }
+
+    private DateRange(DateRangeBuilder builder)
+    {
+        this.lowerBound = builder.lowerBound;
+        this.upperBound = builder.upperBound;
+    }
+
+    public DateRangeBound getLowerBound()
+    {
+        return lowerBound;
+    }
+
+    public DateRangeBound getUpperBound()
+    {
+        return upperBound;
+    }
+
+    public boolean isUpperBoundDefined()
+    {
+        return upperBound != null;
+    }
+
+    public String formatToSolrString()
+    {
+        if (isUpperBoundDefined())
+        {
+            return String.format("[%s TO %s]", lowerBound, upperBound);
+        }
+        else
+        {
+            return lowerBound.toString();
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                .add("lowerBound", lowerBound)
+                .add("precision", lowerBound.getPrecision())
+                .add("upperBound", upperBound)
+                .add("precision", upperBound != null ? upperBound.getPrecision() : "null")
+                .toString();
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj == null || obj.getClass() != getClass())
+        {
+            return false;
+        }
+        if (obj == this)
+        {
+            return true;
+        }
+
+        DateRange rhs = (DateRange) obj;
+        return new EqualsBuilder()
+                .append(lowerBound, rhs.lowerBound)
+                .append(upperBound, rhs.upperBound)
+                .isEquals();
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(lowerBound, upperBound);
+    }
+
+    public static class DateRangeBound
+    {
+        public static final DateRangeBound UNBOUNDED = new DateRangeBound();
+
+        private final ZonedDateTime timestamp;
+        private final Precision precision;
+
+        private DateRangeBound(ZonedDateTime timestamp, Precision precision)
+        {
+            Preconditions.checkArgument(timestamp != null);
+            Preconditions.checkArgument(precision != null);
+            this.timestamp = timestamp;
+            this.precision = precision;
+        }
+
+        private DateRangeBound()
+        {
+            this.timestamp = null;
+            this.precision = null;
+        }
+
+        public static DateRangeBound lowerBound(Instant timestamp, Precision precision)
+        {
+            return lowerBound(ZonedDateTime.ofInstant(timestamp, ZoneOffset.UTC), precision);
+        }
+
+        public static DateRangeBound lowerBound(ZonedDateTime timestamp, Precision precision)
+        {
+            ZonedDateTime roundedLowerBound = DateRangeUtil.roundLowerBoundTimestampToPrecision(timestamp, precision);
+            return new DateRangeBound(roundedLowerBound, precision);
+        }
+
+        public static DateRangeBound upperBound(Instant timestamp, Precision precision)
+        {
+            return upperBound(ZonedDateTime.ofInstant(timestamp, ZoneOffset.UTC), precision);
+        }
+
+        public static DateRangeBound upperBound(ZonedDateTime timestamp, Precision precision)
+        {
+            ZonedDateTime roundedUpperBound = DateRangeUtil.roundUpperBoundTimestampToPrecision(timestamp, precision);
+            return new DateRangeBound(roundedUpperBound, precision);
+        }
+
+        public boolean isUnbounded()
+        {
+            return timestamp == null;
+        }
+
+        public boolean isAfter(DateRangeBound other)
+        {
+            return isUnbounded() || other.isUnbounded() || timestamp.isAfter(other.timestamp);
+        }
+
+        public Instant getTimestamp()
+        {
+            return timestamp.toInstant();
+        }
+
+        public Precision getPrecision()
+        {
+            return precision;
+        }
+
+        @Override
+        public String toString()
+        {
+            if (isUnbounded())
+            {
+                return "*";
+            }
+
+            return precision.formatter.format(timestamp);
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (obj == null || obj.getClass() != getClass())
+            {
+                return false;
+            }
+            if (obj == this)
+            {
+                return true;
+            }
+
+            DateRangeBound rhs = (DateRangeBound) obj;
+            return new EqualsBuilder()
+                    .append(isUnbounded(), rhs.isUnbounded())
+                    .append(timestamp, rhs.timestamp)
+                    .append(precision, rhs.precision)
+                    .isEquals();
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hashCode(timestamp, precision);
+        }
+
+        public enum Precision
+        {
+            YEAR(0x00,
+                    new DateTimeFormatterBuilder()
+                            .parseCaseSensitive()
+                            .parseStrict()
+                            .appendPattern("uuuu")
+                            .parseDefaulting(MONTH_OF_YEAR, 1)
+                            .parseDefaulting(DAY_OF_MONTH, 1)
+                            .parseDefaulting(HOUR_OF_DAY, 0)
+                            .parseDefaulting(MINUTE_OF_HOUR, 0)
+                            .parseDefaulting(SECOND_OF_MINUTE, 0)
+                            .parseDefaulting(MILLI_OF_SECOND, 0)
+                            .toFormatter()
+                            .withZone(ZoneOffset.UTC)
+                            .withLocale(Locale.ROOT)),
+
+            MONTH(0x01,
+                    new DateTimeFormatterBuilder()
+                            .parseCaseSensitive()
+                            .parseStrict()
+                            .appendPattern("uuuu-MM")
+                            .parseDefaulting(DAY_OF_MONTH, 1)
+                            .parseDefaulting(HOUR_OF_DAY, 0)
+                            .parseDefaulting(MINUTE_OF_HOUR, 0)
+                            .parseDefaulting(SECOND_OF_MINUTE, 0)
+                            .parseDefaulting(MILLI_OF_SECOND, 0)
+                            .toFormatter()
+                            .withZone(ZoneOffset.UTC)
+                            .withLocale(Locale.ROOT)),
+
+            DAY(0x02,
+                    new DateTimeFormatterBuilder()
+                            .parseCaseSensitive()
+                            .parseStrict()
+                            .appendPattern("uuuu-MM-dd")
+                            .parseDefaulting(HOUR_OF_DAY, 0)
+                            .parseDefaulting(MINUTE_OF_HOUR, 0)
+                            .parseDefaulting(SECOND_OF_MINUTE, 0)
+                            .parseDefaulting(MILLI_OF_SECOND, 0)
+                            .toFormatter()
+                            .withZone(ZoneOffset.UTC)
+                            .withLocale(Locale.ROOT)),
+
+            HOUR(0x03,
+                    new DateTimeFormatterBuilder()
+                            .parseCaseSensitive()
+                            .parseStrict()
+                            .appendPattern("uuuu-MM-dd'T'HH")
+                            .parseDefaulting(MINUTE_OF_HOUR, 0)
+                            .parseDefaulting(SECOND_OF_MINUTE, 0)
+                            .parseDefaulting(MILLI_OF_SECOND, 0)
+                            .toFormatter()
+                            .withZone(ZoneOffset.UTC)
+                            .withLocale(Locale.ROOT)),
+
+            MINUTE(0x04,
+                    new DateTimeFormatterBuilder()
+                            .parseCaseSensitive()
+                            .parseStrict()
+                            .appendPattern("uuuu-MM-dd'T'HH:mm")
+                            .parseDefaulting(SECOND_OF_MINUTE, 0)
+                            .parseDefaulting(MILLI_OF_SECOND, 0)
+                            .toFormatter()
+                            .withZone(ZoneOffset.UTC)
+                            .withLocale(Locale.ROOT)),
+
+            SECOND(0x05,
+                    new DateTimeFormatterBuilder()
+                            .parseCaseSensitive()
+                            .parseStrict()
+                            .appendPattern("uuuu-MM-dd'T'HH:mm:ss")
+                            .parseDefaulting(MILLI_OF_SECOND, 0)
+                            .toFormatter()
+                            .withZone(ZoneOffset.UTC)
+                            .withLocale(Locale.ROOT)),
+
+            MILLISECOND(0x06,
+                    new DateTimeFormatterBuilder()
+                            .parseCaseSensitive()
+                            .parseStrict()
+                            .appendPattern("uuuu-MM-dd'T'HH:mm:ss.SSS")
+                            .optionalStart()
+                            .appendZoneId()
+                            .optionalEnd()
+                            .toFormatter()
+                            .withZone(ZoneOffset.UTC)
+                            .withLocale(Locale.ROOT));
+
+            private final int encoded;
+            private final DateTimeFormatter formatter;
+
+            Precision(int encoded, DateTimeFormatter formatter)
+            {
+                this.encoded = encoded;
+                this.formatter = formatter;
+            }
+
+            public int toEncoded()
+            {
+                return encoded;
+            }
+
+            public static Precision fromEncoded(byte encoded)
+            {
+                for (Precision precision : values())
+                {
+                    if (precision.encoded == encoded)
+                    {
+                        return precision;
+                    }
+                }
+                throw new IllegalArgumentException("Invalid precision encoding: " + encoded);
+            }
+        }
+    }
+
+    public static class DateRangeBuilder
+    {
+        private DateRangeBound lowerBound = null;
+        private DateRangeBound upperBound = null;
+
+        private DateRangeBuilder() {}
+
+        public static DateRangeBuilder dateRange()
+        {
+            return new DateRangeBuilder();
+        }
+
+        public DateRangeBuilder withLowerBound(String lowerBound, DateRangeBound.Precision precision)
+        {
+            return withLowerBound(Instant.parse(lowerBound), precision);
+        }
+
+        public DateRangeBuilder withUnboundedLowerBound()
+        {
+            this.lowerBound = DateRangeBound.UNBOUNDED;
+            return this;
+        }
+
+        public DateRangeBuilder withUnboundedUpperBound()
+        {
+            this.upperBound = DateRangeBound.UNBOUNDED;
+            return this;
+        }
+
+        public DateRangeBuilder withUpperBound(String upperBound, DateRangeBound.Precision precision)
+        {
+            return withUpperBound(Instant.parse(upperBound), precision);
+        }
+
+        public DateRangeBuilder withLowerBound(Instant lowerBound, DateRangeBound.Precision precision)
+        {
+            this.lowerBound = DateRangeBound.lowerBound(lowerBound, precision);
+            return this;
+        }
+
+        public DateRangeBuilder withUpperBound(Instant upperBound, DateRangeBound.Precision precision)
+        {
+            this.upperBound = DateRangeBound.upperBound(upperBound, precision);
+            return this;
+        }
+
+        public DateRange build()
+        {
+            return new DateRange(this);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/datetime/DateRangeUtil.java b/src/java/org/apache/cassandra/db/marshal/datetime/DateRangeUtil.java
new file mode 100644
index 000000000000..8b73cfaa0899
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/datetime/DateRangeUtil.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.datetime;
+
+import java.text.ParseException;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoField;
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.TimeZone;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBound;
+
+import static java.time.temporal.TemporalAdjusters.firstDayOfMonth;
+import static java.time.temporal.TemporalAdjusters.firstDayOfYear;
+import static java.time.temporal.TemporalAdjusters.lastDayOfMonth;
+import static java.time.temporal.TemporalAdjusters.lastDayOfYear;
+
+public class DateRangeUtil
+{
+    private static final int YEAR_LEVEL = 3;
+    private static final int[] FIELD_BY_LEVEL =
+    {
+        -1/*unused*/, -1, -1, Calendar.YEAR, Calendar.MONTH, Calendar.DAY_OF_MONTH,
+        Calendar.HOUR_OF_DAY, Calendar.MINUTE, Calendar.SECOND, Calendar.MILLISECOND
+    };
+    private static final TimeZone UTC_TIME_ZONE = TimeZone.getTimeZone("UTC");
+
+    public static DateRange parseDateRange(String source) throws ParseException
+    {
+        if (StringUtils.isBlank(source))
+        {
+            throw new IllegalArgumentException("Date range is null or blank");
+        }
+        if (source.charAt(0) == '[')
+        {
+            if (source.charAt(source.length() - 1) != ']')
+            {
+                throw new IllegalArgumentException("If date range starts with [ must end with ]; got " + source);
+            }
+            int middle = source.indexOf(" TO ");
+            if (middle < 0)
+            {
+                throw new IllegalArgumentException("If date range starts with [ must contain ' TO '; got " + source);
+            }
+            String lowerBoundString = source.substring(1, middle);
+            String upperBoundString = source.substring(middle + " TO ".length(), source.length() - 1);
+            return new DateRange(parseLowerBound(lowerBoundString), parseUpperBound(upperBoundString));
+        }
+        else
+        {
+            return new DateRange(parseLowerBound(source));
+        }
+    }
+
+    public static ZonedDateTime roundUpperBoundTimestampToPrecision(ZonedDateTime timestamp, DateRangeBound.Precision precision)
+    {
+        switch (precision)
+        {
+            case YEAR:
+                timestamp = timestamp.with(lastDayOfYear());
+            case MONTH:
+                timestamp = timestamp.with(lastDayOfMonth());
+            case DAY:
+                timestamp = timestamp.with(ChronoField.HOUR_OF_DAY, 23);
+            case HOUR:
+                timestamp = timestamp.with(ChronoField.MINUTE_OF_HOUR, 59);
+            case MINUTE:
+                timestamp = timestamp.with(ChronoField.SECOND_OF_MINUTE, 59);
+            case SECOND:
+                timestamp = timestamp.with(ChronoField.MILLI_OF_SECOND, 999);
+            case MILLISECOND:
+                // DateRangeField ignores any precision beyond milliseconds
+                return timestamp;
+            default:
+                throw new IllegalStateException("Unsupported date time precision for the upper bound: " + precision);
+        }
+    }
+
+    public static ZonedDateTime roundLowerBoundTimestampToPrecision(ZonedDateTime timestamp, DateRangeBound.Precision precision)
+    {
+        switch (precision)
+        {
+            case YEAR:
+                timestamp = timestamp.with(firstDayOfYear());
+            case MONTH:
+                timestamp = timestamp.with(firstDayOfMonth());
+            case DAY:
+                timestamp = timestamp.with(ChronoField.HOUR_OF_DAY, 0);
+            case HOUR:
+                timestamp = timestamp.with(ChronoField.MINUTE_OF_HOUR, 0);
+            case MINUTE:
+                timestamp = timestamp.with(ChronoField.SECOND_OF_MINUTE, 0);
+            case SECOND:
+                timestamp = timestamp.with(ChronoField.MILLI_OF_SECOND, 0);
+            case MILLISECOND:
+                // DateRangeField ignores any precision beyond milliseconds
+                return timestamp;
+            default:
+                throw new IllegalStateException("Unsupported date time precision for the upper bound: " + precision);
+        }
+    }
+
+    private static DateRangeBound parseLowerBound(String source) throws ParseException
+    {
+        Calendar lowerBoundCalendar = parseCalendar(source);
+        int calPrecisionField = getCalPrecisionField(lowerBoundCalendar);
+        if (calPrecisionField < 0)
+        {
+            return DateRangeBound.UNBOUNDED;
+        }
+        return DateRangeBound.lowerBound(toZonedDateTime(lowerBoundCalendar), getCalendarPrecision(calPrecisionField));
+    }
+
+    private static DateRangeBound parseUpperBound(String source) throws ParseException
+    {
+        Calendar upperBoundCalendar = parseCalendar(source);
+        int calPrecisionField = getCalPrecisionField(upperBoundCalendar);
+        if (calPrecisionField < 0)
+        {
+            return DateRangeBound.UNBOUNDED;
+        }
+        ZonedDateTime upperBoundDateTime = toZonedDateTime(upperBoundCalendar);
+        DateRangeBound.Precision precision = getCalendarPrecision(calPrecisionField);
+        return DateRangeBound.upperBound(upperBoundDateTime, precision);
+    }
+
+    /**
+     * This method was extracted from org.apache.lucene.spatial.prefix.tree.DateRangePrefixTree
+     * (Apache Lucene™) for compatibility with DSE.
+     * The class is distributed under Apache-2.0 License attached to this release.
+     *
+     * Calendar utility method:
+     * Gets the Calendar field code of the last field that is set prior to an unset field. It only
+     * examines fields relevant to the prefix tree. If no fields are set, it returns -1. */
+    private static int getCalPrecisionField(Calendar cal) {
+        int lastField = -1;
+        for (int level = YEAR_LEVEL; level < FIELD_BY_LEVEL.length; level++) {
+            int field = FIELD_BY_LEVEL[level];
+            if (!cal.isSet(field))
+                break;
+            lastField = field;
+        }
+        return lastField;
+    }
+
+    /**
+     * This method was extracted from org.apache.lucene.spatial.prefix.tree.DateRangePrefixTree
+     * (Apache Lucene™) for compatibility with DSE.
+     * The class is distributed under Apache-2.0 License attached to this release.
+     *
+     * Calendar utility method:
+     * It will only set the fields found, leaving
+     * the remainder in an un-set state. A leading '-' or '+' is optional (positive assumed), and a
+     * trailing 'Z' is also optional.
+     * @param str not null and not empty
+     * @return not null
+     */
+    private static Calendar parseCalendar(String str) throws ParseException {
+        // example: +2014-10-23T21:22:33.159Z
+        if (str == null || str.isEmpty())
+            throw new IllegalArgumentException("str is null or blank");
+        Calendar cal = Calendar.getInstance(UTC_TIME_ZONE, Locale.ROOT);
+        cal.clear();
+        if (str.equals("*"))
+            return cal;
+        int offset = 0;//a pointer
+        try {
+            //year & era:
+            int lastOffset = str.charAt(str.length()-1) == 'Z' ? str.length() - 1 : str.length();
+            int hyphenIdx = str.indexOf('-', 1);//look past possible leading hyphen
+            if (hyphenIdx < 0)
+                hyphenIdx = lastOffset;
+            int year = Integer.parseInt(str.substring(offset, hyphenIdx));
+            cal.set(Calendar.ERA, year <= 0 ? 0 : 1);
+            cal.set(Calendar.YEAR, year <= 0 ? -1*year + 1 : year);
+            offset = hyphenIdx + 1;
+            if (lastOffset < offset)
+                return cal;
+
+            //NOTE: We aren't validating separator chars, and we unintentionally accept leading +/-.
+            // The str.substring()'s hopefully get optimized to be stack-allocated.
+
+            //month:
+            cal.set(Calendar.MONTH, Integer.parseInt(str.substring(offset, offset+2)) - 1);//starts at 0
+            offset += 3;
+            if (lastOffset < offset)
+                return cal;
+            //day:
+            cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(str.substring(offset, offset+2)));
+            offset += 3;
+            if (lastOffset < offset)
+                return cal;
+            //hour:
+            cal.set(Calendar.HOUR_OF_DAY, Integer.parseInt(str.substring(offset, offset+2)));
+            offset += 3;
+            if (lastOffset < offset)
+                return cal;
+            //minute:
+            cal.set(Calendar.MINUTE, Integer.parseInt(str.substring(offset, offset+2)));
+            offset += 3;
+            if (lastOffset < offset)
+                return cal;
+            //second:
+            cal.set(Calendar.SECOND, Integer.parseInt(str.substring(offset, offset+2)));
+            offset += 3;
+            if (lastOffset < offset)
+                return cal;
+            //ms:
+            cal.set(Calendar.MILLISECOND, Integer.parseInt(str.substring(offset, offset+3)));
+            offset += 3;//last one, move to next char
+            if (lastOffset == offset)
+                return cal;
+        } catch (Exception e) {
+            ParseException pe = new ParseException("Improperly formatted date: "+str, offset);
+            pe.initCause(e);
+            throw pe;
+        }
+        throw new ParseException("Improperly formatted date: "+str, offset);
+    }
+
+    private static DateRangeBound.Precision getCalendarPrecision(int calendarPrecision)
+    {
+        switch (calendarPrecision)
+        {
+            case Calendar.YEAR:
+                return DateRangeBound.Precision.YEAR;
+            case Calendar.MONTH:
+                return DateRangeBound.Precision.MONTH;
+            case Calendar.DAY_OF_MONTH:
+                return DateRangeBound.Precision.DAY;
+            case Calendar.HOUR_OF_DAY:
+                return DateRangeBound.Precision.HOUR;
+            case Calendar.MINUTE:
+                return DateRangeBound.Precision.MINUTE;
+            case Calendar.SECOND:
+                return DateRangeBound.Precision.SECOND;
+            case Calendar.MILLISECOND:
+                return DateRangeBound.Precision.MILLISECOND;
+            default:
+                throw new IllegalStateException("Unsupported date time precision: " + calendarPrecision);
+        }
+    }
+
+    private static ZonedDateTime toZonedDateTime(Calendar calendar)
+    {
+        int year = calendar.get(Calendar.YEAR);
+        if (calendar.get(Calendar.ERA) == 0)
+        {
+            // BC era; 1 BC == 0 AD, 0 BD == -1 AD, etc
+            year -= 1;
+            if (year > 0)
+            {
+                year = -year;
+            }
+        }
+        LocalDateTime localDateTime = LocalDateTime.of(year,
+                calendar.get(Calendar.MONTH) + 1,
+                calendar.get(Calendar.DAY_OF_MONTH),
+                calendar.get(Calendar.HOUR_OF_DAY),
+                calendar.get(Calendar.MINUTE),
+                calendar.get(Calendar.SECOND));
+        localDateTime = localDateTime.with(ChronoField.MILLI_OF_SECOND, calendar.get(Calendar.MILLISECOND));
+        return ZonedDateTime.of(localDateTime, ZoneOffset.UTC);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/geometry/GeometricType.java b/src/java/org/apache/cassandra/db/marshal/geometry/GeometricType.java
new file mode 100644
index 000000000000..2516d7d23bda
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/geometry/GeometricType.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.geometry;
+
+public enum GeometricType
+{
+    POINT(Point.class, Point.serializer),
+    LINESTRING(LineString.class, LineString.serializer),
+    POLYGON(Polygon.class, Polygon.serializer);
+
+    private final Class<? extends OgcGeometry> geoClass;
+    private final OgcGeometry.Serializer serializer;
+
+    GeometricType(Class<? extends OgcGeometry> geoClass, OgcGeometry.Serializer serializer)
+    {
+        this.geoClass = geoClass;
+        this.serializer = serializer;
+    }
+
+    public Class<? extends OgcGeometry> getGeoClass()
+    {
+        return geoClass;
+    }
+
+    public OgcGeometry.Serializer getSerializer()
+    {
+        return serializer;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/geometry/LineString.java b/src/java/org/apache/cassandra/db/marshal/geometry/LineString.java
new file mode 100644
index 000000000000..a31854fa9100
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/geometry/LineString.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.geometry;
+
+import java.nio.ByteBuffer;
+
+import com.esri.core.geometry.GeoJsonExportFlags;
+import com.esri.core.geometry.Operator;
+import com.esri.core.geometry.OperatorExportToGeoJson;
+import com.esri.core.geometry.OperatorFactoryLocal;
+import com.esri.core.geometry.ogc.OGCGeometry;
+import com.esri.core.geometry.ogc.OGCLineString;
+import org.apache.cassandra.serializers.MarshalException;
+
+public class LineString extends OgcGeometry
+{
+    public static final Serializer<LineString> serializer = new Serializer<LineString>()
+    {
+        @Override
+        public String toWellKnownText(LineString geometry)
+        {
+            return geometry.lineString.asText();
+        }
+
+        @Override
+        public ByteBuffer toWellKnownBinaryNativeOrder(LineString geometry)
+        {
+            return geometry.lineString.asBinary();
+        }
+
+        @Override
+        public String toGeoJson(LineString geometry)
+        {
+            OperatorExportToGeoJson op = (OperatorExportToGeoJson) OperatorFactoryLocal.getInstance().getOperator(Operator.Type.ExportToGeoJson);
+            return op.execute(GeoJsonExportFlags.geoJsonExportSkipCRS, geometry.lineString.esriSR, geometry.lineString.getEsriGeometry());
+        }
+
+        @Override
+        public LineString fromWellKnownText(String source)
+        {
+            return new LineString(fromOgcWellKnownText(source, OGCLineString.class));
+        }
+
+        @Override
+        public LineString fromWellKnownBinary(ByteBuffer source)
+        {
+            return new LineString(fromOgcWellKnownBinary(source, OGCLineString.class));
+        }
+
+        @Override
+        public LineString fromGeoJson(String source)
+        {
+            return new LineString(fromOgcGeoJson(source, OGCLineString.class));
+        }
+    };
+
+    private final OGCLineString lineString;
+
+    public LineString(OGCLineString lineString)
+    {
+        this.lineString = lineString;
+        validate();
+    }
+
+    @Override
+    public GeometricType getType()
+    {
+        return GeometricType.LINESTRING;
+    }
+
+    @Override
+    public void validate() throws MarshalException
+    {
+        validateOgcGeometry(lineString);
+    }
+
+    @Override
+    public Serializer getSerializer()
+    {
+        return serializer;
+    }
+
+    @Override
+    protected OGCGeometry getOgcGeometry()
+    {
+        return lineString;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        LineString that = (LineString) o;
+
+        return !(lineString != null ? !lineString.equals(that.lineString) : that.lineString != null);
+
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return lineString != null ? lineString.hashCode() : 0;
+    }
+
+    @Override
+    public String toString()
+    {
+        return asWellKnownText();
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/geometry/OgcGeometry.java b/src/java/org/apache/cassandra/db/marshal/geometry/OgcGeometry.java
new file mode 100644
index 000000000000..6f3be38e7415
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/geometry/OgcGeometry.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.geometry;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import com.esri.core.geometry.GeometryException;
+import com.esri.core.geometry.JsonGeometryException;
+import com.esri.core.geometry.SpatialReference;
+import com.esri.core.geometry.ogc.OGCGeometry;
+import org.apache.cassandra.serializers.MarshalException;
+
+public abstract class OgcGeometry
+{
+
+    // default spatial reference for wkt/wkb
+    public static final SpatialReference SPATIAL_REFERENCE_4326 = SpatialReference.create(4326);
+
+    public interface Serializer<T extends OgcGeometry>
+    {
+        String toWellKnownText(T geometry);
+
+        // We need to return a Big Endian ByteBuffer as that's required by org.apache.cassandra.db.NativeDecoratedKey
+        // when the memtable allocation type is "offheap_objects". See https://datastax.jira.com/browse/DSP-16302
+        // Note that the order set here may not match the actual endianess. OGC serialization encodes actual endianess
+        // and discards BB order set here.
+        default ByteBuffer toWellKnownBinary(T geometry)
+        {
+            return toWellKnownBinaryNativeOrder(geometry).order(ByteOrder.BIG_ENDIAN);
+        }
+
+        ByteBuffer toWellKnownBinaryNativeOrder(T geometry);
+
+        String toGeoJson(T geometry);
+
+        T fromWellKnownText(String source);
+
+        T fromWellKnownBinary(ByteBuffer source);
+
+        T fromGeoJson(String source);
+    }
+
+    public abstract GeometricType getType();
+
+    public abstract void validate() throws MarshalException;
+
+    public abstract Serializer getSerializer();
+
+    static void validateType(OGCGeometry geometry, Class<? extends OGCGeometry> klass)
+    {
+        if (!geometry.getClass().equals(klass))
+        {
+            throw new MarshalException(String.format("%s is not of type %s",
+                    geometry.getClass().getSimpleName(),
+                    klass.getSimpleName()));
+        }
+    }
+
+    static ByteBuffer getWkb(OGCGeometry geometry)
+    {
+        try
+        {
+            return geometry.asBinary();
+        }
+        catch (GeometryException | IllegalArgumentException e)
+        {
+            throw new MarshalException("Invalid Geometry", e);
+        }
+    }
+
+    static String getWkt(OGCGeometry geometry)
+    {
+        try
+        {
+            return geometry.asText();
+        }
+        catch (GeometryException | IllegalArgumentException e)
+        {
+            throw new MarshalException("Invalid Geometry", e);
+        }
+    }
+
+    static void validateNormalization(OGCGeometry geometry, ByteBuffer source)
+    {
+        ByteBuffer normalized = getWkb(geometry);
+        ByteBuffer inputCopy = source.slice();
+
+        // since the data we get is sometimes part of a longer string of bytes, we set the limit to the normalized
+        // buffer length. Normalization only ever adds and rearranges points though, so this should be ok
+        if (inputCopy.remaining() > normalized.remaining())
+        {
+            inputCopy.limit(normalized.remaining());
+        }
+
+        if (!normalized.equals(inputCopy))
+        {
+            String klass = geometry.getClass().getSimpleName();
+            String msg = String.format("%s is not normalized. %s should be defined/serialized as: %s", klass, klass, getWkt(geometry));
+            throw new MarshalException(msg);
+        }
+    }
+
+    static <T extends OGCGeometry> T fromOgcWellKnownText(String source, Class<T> klass)
+    {
+        OGCGeometry geometry;
+        try
+        {
+            geometry = OGCGeometry.fromText(source);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new MarshalException(e.getMessage());
+        }
+        validateType(geometry, klass);
+        return (T) geometry;
+    }
+
+    static <T extends OGCGeometry> T fromOgcWellKnownBinary(ByteBuffer source, Class<T> klass)
+    {
+        OGCGeometry geometry;
+        try
+        {
+            geometry = OGCGeometry.fromBinary(source);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new MarshalException(e.getMessage());
+        }
+        validateType(geometry, klass);
+        validateNormalization(geometry, source);
+        return (T) geometry;
+    }
+
+    static <T extends OGCGeometry> T fromOgcGeoJson(String source, Class<T> klass)
+    {
+        OGCGeometry geometry;
+        try
+        {
+            geometry = OGCGeometry.fromGeoJson(source);
+        }
+        catch (IllegalArgumentException | JsonGeometryException e)
+        {
+            throw new MarshalException(e.getMessage());
+        }
+        validateType(geometry, klass);
+        return (T) geometry;
+    }
+
+    public boolean contains(OgcGeometry geometry)
+    {
+        if (!(geometry instanceof OgcGeometry))
+        {
+            throw new UnsupportedOperationException(String.format("%s is not compatible with %s.contains",
+                    geometry.getClass().getSimpleName(), getClass().getSimpleName()));
+        }
+
+        OGCGeometry thisGeometry = getOgcGeometry();
+        OGCGeometry thatGeometry = ((OgcGeometry) geometry).getOgcGeometry();
+        if (thisGeometry != null && thatGeometry != null)
+        {
+            return thisGeometry.contains(thatGeometry);
+        }
+        else
+        {
+            return false;
+        }
+    }
+
+    protected abstract OGCGeometry getOgcGeometry();
+
+    static void validateOgcGeometry(OGCGeometry geometry)
+    {
+        try
+        {
+            if (geometry.is3D())
+            {
+                throw new MarshalException(String.format("'%s' is not 2D", getWkt(geometry)));
+            }
+
+            if (!geometry.isSimple())
+            {
+                throw new MarshalException(String.format("'%s' is not simple. Points and edges cannot self-intersect.", getWkt(geometry)));
+            }
+        }
+        catch (GeometryException e)
+        {
+            throw new MarshalException("Invalid geometry", e);
+        }
+    }
+
+    public String asWellKnownText()
+    {
+        return getSerializer().toWellKnownText(this);
+    }
+
+    public ByteBuffer asWellKnownBinary()
+    {
+        return getSerializer().toWellKnownBinary(this);
+    }
+
+    public String asGeoJson()
+    {
+        return getSerializer().toGeoJson(this);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/geometry/Point.java b/src/java/org/apache/cassandra/db/marshal/geometry/Point.java
new file mode 100644
index 000000000000..0992bd768725
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/geometry/Point.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.geometry;
+
+import java.nio.ByteBuffer;
+
+import com.esri.core.geometry.GeoJsonExportFlags;
+import com.esri.core.geometry.Operator;
+import com.esri.core.geometry.OperatorExportToGeoJson;
+import com.esri.core.geometry.OperatorFactoryLocal;
+import com.esri.core.geometry.ogc.OGCGeometry;
+import com.esri.core.geometry.ogc.OGCPoint;
+import org.apache.cassandra.serializers.MarshalException;
+
+public class Point extends OgcGeometry
+{
+    public static final Serializer<Point> serializer = new Serializer<Point>()
+    {
+        @Override
+        public String toWellKnownText(Point geometry)
+        {
+            return geometry.point.asText();
+        }
+
+        @Override
+        public ByteBuffer toWellKnownBinaryNativeOrder(Point geometry)
+        {
+            return geometry.point.asBinary();
+        }
+
+        @Override
+        public String toGeoJson(Point geometry)
+        {
+            OperatorExportToGeoJson op = (OperatorExportToGeoJson) OperatorFactoryLocal.getInstance().getOperator(Operator.Type.ExportToGeoJson);
+            return op.execute(GeoJsonExportFlags.geoJsonExportSkipCRS, geometry.point.esriSR, geometry.point.getEsriGeometry());
+        }
+
+        @Override
+        public Point fromWellKnownText(String source)
+        {
+            return new Point(fromOgcWellKnownText(source, OGCPoint.class));
+        }
+
+        @Override
+        public Point fromWellKnownBinary(ByteBuffer source)
+        {
+            return new Point(fromOgcWellKnownBinary(source, OGCPoint.class));
+        }
+
+        @Override
+        public Point fromGeoJson(String source)
+        {
+            return new Point(fromOgcGeoJson(source, OGCPoint.class));
+        }
+    };
+
+    final OGCPoint point;
+
+    public Point(double x, double y)
+    {
+        this(new OGCPoint(new com.esri.core.geometry.Point(x, y), OgcGeometry.SPATIAL_REFERENCE_4326));
+    }
+
+    private Point(OGCPoint point)
+    {
+        this.point = point;
+        validate();
+    }
+
+    @Override
+    public boolean contains(OgcGeometry geometry)
+    {
+        return false;
+    }
+
+    @Override
+    public GeometricType getType()
+    {
+        return GeometricType.POINT;
+    }
+
+    @Override
+    public void validate() throws MarshalException
+    {
+        validateOgcGeometry(point);
+        if (point.isEmpty() || point.is3D())
+            throw new MarshalException(getClass().getSimpleName() + " requires exactly 2 coordinate values");
+    }
+
+    @Override
+    protected OGCGeometry getOgcGeometry()
+    {
+        return point;
+    }
+
+    @Override
+    public Serializer getSerializer()
+    {
+        return serializer;
+    }
+
+    public OGCPoint getOgcPoint()
+    {
+        return point;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Point point1 = (Point) o;
+
+        return !(point != null ? !point.equals(point1.point) : point1.point != null);
+
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return point != null ? point.hashCode() : 0;
+    }
+
+    @Override
+    public String toString()
+    {
+        return asWellKnownText();
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/marshal/geometry/Polygon.java b/src/java/org/apache/cassandra/db/marshal/geometry/Polygon.java
new file mode 100644
index 000000000000..d51181566d04
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/marshal/geometry/Polygon.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.geometry;
+
+import java.nio.ByteBuffer;
+
+import com.esri.core.geometry.GeoJsonExportFlags;
+import com.esri.core.geometry.Operator;
+import com.esri.core.geometry.OperatorExportToGeoJson;
+import com.esri.core.geometry.OperatorFactoryLocal;
+import com.esri.core.geometry.ogc.OGCGeometry;
+import com.esri.core.geometry.ogc.OGCPolygon;
+import org.apache.cassandra.serializers.MarshalException;
+
+public class Polygon extends OgcGeometry
+{
+    public static final Serializer<Polygon> serializer = new Serializer<Polygon>()
+    {
+        @Override
+        public String toWellKnownText(Polygon geometry)
+        {
+            return geometry.polygon.asText();
+        }
+
+        @Override
+        public ByteBuffer toWellKnownBinaryNativeOrder(Polygon geometry)
+        {
+            return geometry.polygon.asBinary();
+        }
+
+        @Override
+        public String toGeoJson(Polygon geometry)
+        {
+            OperatorExportToGeoJson op = (OperatorExportToGeoJson) OperatorFactoryLocal.getInstance().getOperator(Operator.Type.ExportToGeoJson);
+            return op.execute(GeoJsonExportFlags.geoJsonExportSkipCRS, geometry.polygon.esriSR, geometry.polygon.getEsriGeometry());
+        }
+
+        @Override
+        public Polygon fromWellKnownText(String source)
+        {
+            return new Polygon(fromOgcWellKnownText(source, OGCPolygon.class));
+        }
+
+        @Override
+        public Polygon fromWellKnownBinary(ByteBuffer source)
+        {
+            return new Polygon(fromOgcWellKnownBinary(source, OGCPolygon.class));
+        }
+
+        @Override
+        public Polygon fromGeoJson(String source)
+        {
+            return new Polygon(fromOgcGeoJson(source, OGCPolygon.class));
+        }
+    };
+
+    OGCPolygon polygon;
+
+    public Polygon(OGCPolygon polygon)
+    {
+        this.polygon = polygon;
+        validate();
+    }
+
+    @Override
+    protected OGCGeometry getOgcGeometry()
+    {
+        return polygon;
+    }
+
+    @Override
+    public GeometricType getType()
+    {
+        return GeometricType.POLYGON;
+    }
+
+    @Override
+    public void validate() throws MarshalException
+    {
+        validateOgcGeometry(polygon);
+    }
+
+    @Override
+    public Serializer getSerializer()
+    {
+        return serializer;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Polygon polygon1 = (Polygon) o;
+
+        return !(polygon != null ? !polygon.equals(polygon1.polygon) : polygon1.polygon != null);
+
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return polygon != null ? polygon.hashCode() : 0;
+    }
+
+    @Override
+    public String toString()
+    {
+        return asWellKnownText();
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java
index 8526dace3925..f66020f827af 100644
--- a/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java
@@ -31,7 +31,11 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
@@ -48,6 +52,8 @@
 import org.apache.cassandra.utils.memory.SlabPool;
 import org.github.jamm.Unmetered;
 
+import static org.apache.cassandra.io.sstable.SSTableReadsListener.NOOP_LISTENER;
+
 /**
  * A memtable that uses memory tracked and maybe allocated via a MemtableAllocator from a MemtablePool.
  * Provides methods of memory tracking and triggering flushes when the relevant limits are reached.
@@ -74,6 +80,14 @@ public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithComm
 
     private final long creationNano = Clock.Global.nanoTime();
 
+    /**
+     * Keeps an estimate of the average row size in this memtable, computed from a small sample of rows.
+     * Because computing this estimate is potentially costly, as it requires iterating the rows,
+     * the estimate is updated only whenever the number of operations on the memtable increases significantly from the
+     * last update. This estimate is not very accurate but should be ok for planning or diagnostic purposes.
+     */
+    private volatile MemtableAverageRowSize estimatedAverageRowSize;
+
     @VisibleForTesting
     static MemtablePool createMemtableAllocatorPool()
     {
@@ -111,7 +125,6 @@ public static MemtablePool createMemtableAllocatorPoolInternal(Config.MemtableAl
         }
     }
 
-    // only to be used by init(), to setup the very first memtable for the cfs
     public AbstractAllocatorMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner)
     {
         super(metadataRef, commitLogLowerBound);
@@ -127,6 +140,30 @@ public MemtableAllocator getAllocator()
         return allocator;
     }
 
+    public long rowCount(final ColumnFilter columnFilter, final DataRange dataRange)
+    {
+        int total = 0;
+        for (var iter = partitionIterator(columnFilter, dataRange, NOOP_LISTENER); iter.hasNext(); )
+        {
+            for (UnfilteredRowIterator it = iter.next(); it.hasNext(); )
+            {
+                Unfiltered uRow = it.next();
+                if (uRow.isRow())
+                    total++;
+            }
+        }
+
+        return total;
+    }
+
+    @Override
+    public long getEstimatedAverageRowSize()
+    {
+        if (estimatedAverageRowSize == null || currentOperations.get() > estimatedAverageRowSize.operations * 1.5)
+            estimatedAverageRowSize = new MemtableAverageRowSize(this);
+        return estimatedAverageRowSize.rowSize;
+    }
+    
     @Override
     public boolean shouldSwitch(ColumnFamilyStore.FlushReason reason)
     {
@@ -142,6 +179,12 @@ public boolean shouldSwitch(ColumnFamilyStore.FlushReason reason)
         }
     }
 
+    @Override
+    public OpOrder readOrdering()
+    {
+        return owner.readOrdering();
+    }
+
     public void metadataUpdated()
     {
         // We decided not to swap out this memtable, but if the flush period has changed we must schedule it for the
@@ -182,7 +225,16 @@ public String toString()
                              usage);
     }
 
-    @Override
+    /**
+     * For testing only. Give this memtable too big a size to make it always fail flushing.
+     */
+    @VisibleForTesting
+    public void makeUnflushable()
+    {
+        liveDataSize.addAndGet(1024L * 1024 * 1024 * 1024 * 1024);
+    }
+
+@Override
     public void addMemoryUsageTo(MemoryUsage stats)
     {
         stats.ownershipRatioOnHeap += getAllocator().onHeap().ownershipRatio();
@@ -193,14 +245,21 @@ public void addMemoryUsageTo(MemoryUsage stats)
 
     public void markExtraOnHeapUsed(long additionalSpace, OpOrder.Group opGroup)
     {
-        getAllocator().onHeap().allocate(additionalSpace, opGroup);
+        getAllocator().onHeap().adjust(additionalSpace, opGroup);
     }
 
     public void markExtraOffHeapUsed(long additionalSpace, OpOrder.Group opGroup)
     {
-        getAllocator().offHeap().allocate(additionalSpace, opGroup);
+        getAllocator().offHeap().adjust(additionalSpace, opGroup);
     }
 
+    @Override
+    public long unusedReservedOnHeapMemory()
+    {
+        return allocator.unusedReservedOnHeapMemory();
+    }
+
+
     void scheduleFlush()
     {
         int period = metadata().params.memtableFlushPeriodInMs;
diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
index 2f2c2a25516c..f01fc49a0cd8 100644
--- a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
@@ -41,6 +41,7 @@
 public abstract class AbstractMemtable implements Memtable
 {
     private final AtomicReference<LifecycleTransaction> flushTransaction = new AtomicReference<>(null);
+    protected final AtomicLong liveDataSize = new AtomicLong(0);
     protected final AtomicLong currentOperations = new AtomicLong(0);
     protected final ColumnsCollector columnsCollector;
     protected final StatsCollector statsCollector = new StatsCollector();
@@ -74,6 +75,11 @@ public TableMetadata metadata()
         return metadata.get();
     }
 
+    @Override
+    public long getLiveDataSize()
+    {
+        return liveDataSize.get();
+    }
     @Override
     public long operationCount()
     {
@@ -81,6 +87,13 @@ public long operationCount()
     }
 
     @Override
+    /**
+     * Returns the minTS if one available, otherwise NO_MIN_TIMESTAMP.
+     * 
+     * EncodingStats uses a synthetic epoch TS at 2015. We don't want to leak that (CASSANDRA-18118) so we return NO_MIN_TIMESTAMP instead.
+     * 
+     * @return The minTS or NO_MIN_TIMESTAMP if none available
+     */
     public long getMinTimestamp()
     {
         return minTimestamp.get() != EncodingStats.NO_STATS.minTimestamp ? minTimestamp.get() : NO_MIN_TIMESTAMP;
diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractShardedMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractShardedMemtable.java
index 081570188b3f..e4a33528daa7 100644
--- a/src/java/org/apache/cassandra/db/memtable/AbstractShardedMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/AbstractShardedMemtable.java
@@ -43,7 +43,12 @@ public abstract class AbstractShardedMemtable extends AbstractAllocatorMemtable
     }
 
     // default shard count, used when a specific number of shards is not specified in the options
-    private static volatile int defaultShardCount = MEMTABLE_SHARD_COUNT.getInt(FBUtilities.getAvailableProcessors());
+    private static volatile int defaultShardCount = MEMTABLE_SHARD_COUNT.getInt(autoShardCount());
+
+    private static int autoShardCount()
+    {
+        return 4 * FBUtilities.getAvailableProcessors();
+    }
 
     // The boundaries for the keyspace as they were calculated when the memtable is created.
     // The boundaries will be NONE for system keyspaces or if StorageService is not yet initialized.
@@ -69,7 +74,7 @@ public void setDefaultShardCount(String shardCount)
         {
             if ("auto".equalsIgnoreCase(shardCount))
             {
-                defaultShardCount = FBUtilities.getAvailableProcessors();
+                defaultShardCount = autoShardCount();
             }
             else
             {
diff --git a/src/java/org/apache/cassandra/db/memtable/Flushing.java b/src/java/org/apache/cassandra/db/memtable/Flushing.java
index 3fc856858294..278543c37dac 100644
--- a/src/java/org/apache/cassandra/db/memtable/Flushing.java
+++ b/src/java/org/apache/cassandra/db/memtable/Flushing.java
@@ -22,7 +22,9 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicReference;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import org.slf4j.Logger;
@@ -39,13 +41,18 @@
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.FSDiskFullWriteError;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
 
+import static org.apache.cassandra.utils.Throwables.maybeFail;
+
 public class Flushing
 {
     private static final Logger logger = LoggerFactory.getLogger(Flushing.class);
@@ -65,8 +72,18 @@ public static List<FlushRunnable> flushRunnables(ColumnFamilyStore cfs,
                                  cfs.name);
 
         DiskBoundaries diskBoundaries = cfs.getDiskBoundaries();
-        List<PartitionPosition> boundaries = diskBoundaries.positions;
+        List<Token> boundaries = diskBoundaries.getPositions();
         List<Directories.DataDirectory> locations = diskBoundaries.directories;
+        return flushRunnables(cfs, memtable, boundaries, locations, txn);
+    }
+
+    @VisibleForTesting
+    static List<FlushRunnable> flushRunnables(ColumnFamilyStore cfs,
+                                              Memtable memtable,
+                                              List<Token> boundaries,
+                                              List<Directories.DataDirectory> locations,
+                                              LifecycleTransaction txn)
+    {
         if (boundaries == null)
         {
             FlushRunnable runnable = flushRunnable(cfs, memtable, null, null, txn, null);
@@ -79,7 +96,7 @@ public static List<FlushRunnable> flushRunnables(ColumnFamilyStore cfs,
         {
             for (int i = 0; i < boundaries.size(); i++)
             {
-                PartitionPosition t = boundaries.get(i);
+                PartitionPosition t = boundaries.get(i).maxKeyBound();
                 FlushRunnable runnable = flushRunnable(cfs, memtable, rangeStart, t, txn, locations.get(i));
 
                 runnables.add(runnable);
@@ -89,9 +106,7 @@ public static List<FlushRunnable> flushRunnables(ColumnFamilyStore cfs,
         }
         catch (Throwable e)
         {
-            Throwable t = abortRunnables(runnables, e);
-            Throwables.throwIfUnchecked(t);
-            throw new RuntimeException(t);
+            throw Throwables.propagate(abortRunnables(runnables, e));
         }
     }
 
@@ -106,9 +121,19 @@ static FlushRunnable flushRunnable(ColumnFamilyStore cfs,
         SSTableFormat<?, ?> format = DatabaseDescriptor.getSelectedSSTableFormat();
         long estimatedSize = format.getWriterFactory().estimateSize(flushSet);
 
-        Descriptor descriptor = flushLocation == null
-                                ? cfs.newSSTableDescriptor(cfs.getDirectories().getWriteableLocationAsFile(estimatedSize), format)
-                                : cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(flushLocation), format);
+        Descriptor descriptor;
+        if (flushLocation == null)
+        {
+            descriptor = cfs.newSSTableDescriptor(cfs.getDirectories().getWriteableLocationAsFile(estimatedSize), format);
+        }
+        else
+        {
+            // exclude directory if its total writeSize does not fit to data directory
+            if (flushLocation.getAvailableSpace() < estimatedSize)
+                throw new FSDiskFullWriteError(cfs.metadata.keyspace, estimatedSize);
+
+            descriptor = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(flushLocation), format);
+        }
 
         SSTableMultiWriter writer = createFlushWriter(cfs,
                                                       flushSet,
@@ -123,10 +148,26 @@ public static Throwable abortRunnables(List<FlushRunnable> runnables, Throwable
     {
         if (runnables != null)
             for (FlushRunnable runnable : runnables)
-                t = runnable.writer.abort(t);
+                t = runnable.abort(t);
         return t;
     }
 
+    /**
+     * The valid states for {@link FlushRunnable} writers. The thread writing the contents
+     * will transition from IDLE -> RUNNING and back to IDLE when finished using the writer
+     * or from ABORTING -> ABORTED if another thread has transitioned from RUNNING -> ABORTING.
+     * We can also transition directly from IDLE -> ABORTED. Whichever threads transitions
+     * to ABORTED is responsible to abort the writer.
+     */
+    @VisibleForTesting
+    enum FlushRunnableWriterState
+    {
+        IDLE, // the runnable is idle, either not yet started or completed but with the writer waiting to be committed
+        RUNNING, // the runnable is executing, therefore the writer cannot be aborted or else a SEGV may ensue
+        ABORTING, // an abort request has been issued, this only happens if abort() is called whilst RUNNING
+        ABORTED  // the writer has been aborted, no resources will be leaked
+    }
+
     public static class FlushRunnable implements Callable<SSTableMultiWriter>
     {
         private final Memtable.FlushablePartitionSet<?> toFlush;
@@ -135,6 +176,7 @@ public static class FlushRunnable implements Callable<SSTableMultiWriter>
         private final TableMetrics metrics;
         private final boolean isBatchLogTable;
         private final boolean logCompletion;
+        private final AtomicReference<FlushRunnableWriterState> state;
 
         public FlushRunnable(Memtable.FlushablePartitionSet<?> flushSet,
                              SSTableMultiWriter writer,
@@ -146,42 +188,77 @@ public FlushRunnable(Memtable.FlushablePartitionSet<?> flushSet,
             this.metrics = metrics;
             this.isBatchLogTable = toFlush.metadata() == SystemKeyspace.Batches;
             this.logCompletion = logCompletion;
+            this.state = new AtomicReference<>(FlushRunnableWriterState.IDLE);
         }
 
         private void writeSortedContents()
         {
-            logger.info("Writing {}, flushed range = [{}, {})", toFlush.memtable(), toFlush.from(), toFlush.to());
+            if (!state.compareAndSet(FlushRunnableWriterState.IDLE, FlushRunnableWriterState.RUNNING))
+            {
+                logger.debug("Failed to write {}, flushed range = ({}, {}], state: {}",
+                             toFlush.memtable().toString(), toFlush.from(), toFlush.to(), state);
+                return;
+            }
+
+            long before = Clock.Global.nanoTime();
+            logger.debug("Writing {}, flushed range = ({}, {}], state: {}",
+                         toFlush.memtable().toString(), toFlush.from(), toFlush.to(), state);
 
-            // (we can't clear out the map as-we-go to free up memory,
-            //  since the memtable is being used for queries in the "pending flush" category)
-            for (Partition partition : toFlush)
+            try
             {
-                // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
-                // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
-                // we don't need to preserve tombstones for repair. So if both operation are in this
-                // memtable (which will almost always be the case if there is no ongoing failure), we can
-                // just skip the entry (CASSANDRA-4667).
-                if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
-                    continue;
-
-                if (!partition.isEmpty())
+                // (we can't clear out the map as-we-go to free up memory,
+                //  since the memtable is being used for queries in the "pending flush" category)
+                for (Partition partition : toFlush)
                 {
-                    try (UnfilteredRowIterator iter = partition.unfilteredIterator())
+                    if (state.get() == FlushRunnableWriterState.ABORTING)
+                        break;
+
+                    // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
+                    // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
+                    // we don't need to preserve tombstones for repair. So if both operation are in this
+                    // memtable (which will almost always be the case if there is no ongoing failure), we can
+                    // just skip the entry (CASSANDRA-4667).
+                    if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
+                        continue;
+
+                    if (!partition.isEmpty())
                     {
-                        writer.append(iter);
+                        try (UnfilteredRowIterator iter = partition.unfilteredIterator())
+                        {
+                            writer.append(iter);
+                        }
                     }
                 }
             }
-
-            if (logCompletion)
+            finally
             {
-                long bytesFlushed = writer.getBytesWritten();
-                logger.info("Completed flushing {} ({}) for commitlog position {}",
-                            writer.getFilename(),
-                            FBUtilities.prettyPrintMemory(bytesFlushed),
-                            toFlush.memtable().getFinalCommitLogUpperBound());
-                // Update the metrics
-                metrics.bytesFlushed.inc(bytesFlushed);
+                while (true)
+                {
+                    if (state.compareAndSet(FlushRunnableWriterState.RUNNING, FlushRunnableWriterState.IDLE))
+                    {
+                        if (logCompletion)
+                        {
+                            long bytesFlushed = writer.getBytesWritten();
+                            long segmentCount = writer.getSegmentCount();
+                            logger.debug("Completed flushing {} ({}/{} files) for commitlog position {}",
+                                         writer.getFilename(),
+                                         FBUtilities.prettyPrintMemory(bytesFlushed),
+                                         segmentCount,
+                                         toFlush.memtable().getFinalCommitLogUpperBound());
+                            // Update the metrics
+                            metrics.incBytesFlushed(toFlush.memtable().getLiveDataSize(), bytesFlushed, Clock.Global.nanoTime() - before);
+                            metrics.flushSegmentCount.update(segmentCount);
+                        }
+
+                        break;
+                    }
+                    else if (state.compareAndSet(FlushRunnableWriterState.ABORTING, FlushRunnableWriterState.ABORTED))
+                    {
+                        logger.debug("Flushing of {} aborted", writer.getFilename());
+                        maybeFail(writer.abort(null));
+                        break;
+                    }
+                }
             }
         }
 
@@ -198,6 +275,29 @@ public String toString()
         {
             return "Flush " + toFlush.metadata().keyspace + '.' + toFlush.metadata().name;
         }
+
+        public Throwable abort(Throwable throwable)
+        {
+            while (true)
+            {
+                if (state.compareAndSet(FlushRunnableWriterState.IDLE, FlushRunnableWriterState.ABORTED))
+                {
+                    logger.debug("Flushing of {} aborted", writer.getFilename());
+                    return writer.abort(throwable);
+                }
+                else if (state.compareAndSet(FlushRunnableWriterState.RUNNING, FlushRunnableWriterState.ABORTING))
+                {
+                    // thread currently executing writeSortedContents() will take care of aborting and throw any exceptions
+                    return throwable;
+                }
+            }
+        }
+
+        @VisibleForTesting
+        FlushRunnableWriterState state()
+        {
+            return state.get();
+        }
     }
 
     public static SSTableMultiWriter createFlushWriter(ColumnFamilyStore cfs,
diff --git a/src/java/org/apache/cassandra/db/memtable/Memtable.java b/src/java/org/apache/cassandra/db/memtable/Memtable.java
index 5ce59f6191ab..3e7f2a08efa4 100644
--- a/src/java/org/apache/cassandra/db/memtable/Memtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/Memtable.java
@@ -21,12 +21,14 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import javax.annotation.concurrent.NotThreadSafe;
+import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdate;
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.EncodingStats;
@@ -38,6 +40,8 @@
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 
 /**
@@ -148,6 +152,14 @@ default TableMetrics.ReleasableMetric createMemtableMetrics(TableMetadataRef met
         {
             return null;
         }
+
+        /**
+         * Override this method to provide a custom partition update factory for more efficient merging of updates.
+         */
+        default PartitionUpdate.Factory partitionUpdateFactory()
+        {
+            return BTreePartitionUpdate.FACTORY;
+        }
     }
 
     /**
@@ -177,6 +189,11 @@ interface Owner
          * {@link #localRangesUpdated()} call.
          */
         ShardBoundaries localRangeSplits(int shardCount);
+
+        /**
+         * Get the op-order primitive that protects data for the duration of reads.
+         */
+        public OpOrder readOrdering();
     }
 
     // Main write and read operations
@@ -195,6 +212,20 @@ interface Owner
      */
     long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup);
 
+    /**
+     * Get the partition for the specified key. Returns null if no such partition is present.
+     */
+    Partition getPartition(DecoratedKey key);
+
+    interface MemtableUnfilteredPartitionIterator extends UnfilteredPartitionIterator
+    {
+        /**
+         * Returns the minimum local deletion time for all partitions in the range.
+         * Required for the efficiency of partition range read commands.
+         */
+        long getMinLocalDeletionTime();
+    }
+
     // Read operations are provided by the UnfilteredSource interface.
 
     // Statistics
@@ -205,12 +236,24 @@ interface Owner
     /** Size of the data not accounting for any metadata / mapping overheads */
     long getLiveDataSize();
 
+    /** Average size of the data of each row */
+    long getEstimatedAverageRowSize();
+
     /**
      * Number of "operations" (in the sense defined in {@link PartitionUpdate#operationCount()}) the memtable has
      * executed.
      */
     long operationCount();
 
+    /** Minimum timestamp of all stored data */
+    long getMinTimestamp();
+
+    /** Min partition key inserted so far. */
+    DecoratedKey minPartitionKey();
+
+    /** Max partition key inserted so far. */
+    DecoratedKey maxPartitionKey();
+
     /**
      * The table's definition metadata.
      *
@@ -219,6 +262,16 @@ interface Owner
      */
     TableMetadata metadata();
 
+    /**
+     * The {@link OpOrder} that guards reads from this memtable. This is used to ensure that the memtable does not corrupt any
+     * active reads because of other operations on it. Returns null if the memtable is not protected by an OpOrder
+     * (overridden by {@link AbstractAllocatorMemtable}).
+     */
+    default OpOrder readOrdering()
+    {
+        return null;
+    }
+
 
     // Memory usage tracking
 
@@ -250,6 +303,28 @@ static MemoryUsage getMemoryUsage(Memtable memtable)
         return usage;
     }
 
+    /**
+     * Estimates the total number of rows stored in the memtable.
+     * It is optimized for speed, not for accuracy.
+     */
+    static long estimateRowCount(Memtable memtable)
+    {
+        long rowSize = memtable.getEstimatedAverageRowSize();
+        return rowSize > 0 ? memtable.getLiveDataSize() / rowSize : 0;
+    }
+
+    /**
+     * Returns the amount of on-heap memory that has been allocated for this memtable but is not yet used.
+     * This is not counted in the memory usage to have a better flushing decision behaviour -- we do not want to flush
+     * immediately after allocating a new buffer but when we have actually used the space provided.
+     * The method is provided for testing the memory usage tracking of memtables.
+     */
+    @VisibleForTesting
+    default long unusedReservedOnHeapMemory()
+    {
+        return 0;
+    }
+
     @NotThreadSafe
     class MemoryUsage
     {
@@ -330,6 +405,7 @@ default TableMetadata metadata()
             return memtable().metadata();
         }
 
+        long partitionCount();
         default boolean isEmpty()
         {
             return partitionCount() > 0;
diff --git a/src/java/org/apache/cassandra/db/memtable/MemtableAverageRowSize.java b/src/java/org/apache/cassandra/db/memtable/MemtableAverageRowSize.java
new file mode 100644
index 000000000000..24afc3df6b03
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/memtable/MemtableAverageRowSize.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+
+class MemtableAverageRowSize
+{
+    private final static long MAX_ROWS = 100;
+
+    public final long rowSize;
+    public final long operations;
+
+
+    public MemtableAverageRowSize(Memtable memtable)
+    {
+        DataRange range = DataRange.allData(memtable.metadata().partitioner);
+        ColumnFilter columnFilter = ColumnFilter.allRegularColumnsBuilder(memtable.metadata(), true).build();
+
+        long rowCount = 0;
+        long totalSize = 0;
+
+        try (var partitionsIter = memtable.partitionIterator(columnFilter, range, SSTableReadsListener.NOOP_LISTENER))
+        {
+            while (partitionsIter.hasNext() && rowCount < MAX_ROWS)
+            {
+                UnfilteredRowIterator rowsIter = partitionsIter.next();
+                while (rowsIter.hasNext() && rowCount < MAX_ROWS)
+                {
+                    Unfiltered uRow = rowsIter.next();
+                    if (uRow.isRow())
+                    {
+                        rowCount++;
+                        totalSize += ((Row) uRow).dataSize();
+                    }
+                }
+            }
+        }
+        this.operations = memtable.operationCount();
+        this.rowSize = (rowCount > 0)
+                       ? totalSize / rowCount
+                       : 0;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/memtable/PersistentMemoryMemtable.java b/src/java/org/apache/cassandra/db/memtable/PersistentMemoryMemtable.java
new file mode 100644
index 000000000000..3becbaa6f38d
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/memtable/PersistentMemoryMemtable.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+/**
+ * Skeleton for persistent memory memtable.
+ */
+public class PersistentMemoryMemtable
+//extends AbstractMemtable
+extends SkipListMemtable        // to test framework
+{
+    public PersistentMemoryMemtable(TableMetadataRef metadaRef, Owner owner)
+    {
+        super(null, metadaRef, owner);
+        // We should possibly link the persistent data of this memtable
+    }
+
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    {
+        // TODO: implement
+        return super.put(update, indexer, opGroup);
+    }
+
+    public MemtableUnfilteredPartitionIterator partitionIterator(ColumnFilter columnFilter, DataRange dataRange)
+    {
+        // TODO: implement
+        return super.partitionIterator(columnFilter, dataRange, SSTableReadsListener.NOOP_LISTENER);
+    }
+
+    public Partition getPartition(DecoratedKey key)
+    {
+        // TODO: implement
+        return super.getPartition(key);
+    }
+
+    public long partitionCount()
+    {
+        // TODO: implement
+        return super.partitionCount();
+    }
+
+    public FlushablePartitionSet<?> getFlushSet(PartitionPosition from, PartitionPosition to)
+    {
+        // TODO: implement
+        // FIXME: If the memtable can still be written to, this uses a view of the metadata that may not be up-to-date
+        // with the content. This may cause streaming to fail e.g. if a new column appears and is added to some row in
+        // the memtable between the time that this is constructed and the relevant row is written. Such failures should
+        // be recoverable by redoing the stream.
+        // If an implementation can produce a view/snapshot of the data at a point before the features were collected,
+        // this problem will not occur.
+        return super.getFlushSet(from, to);
+    }
+
+    public boolean shouldSwitch(ColumnFamilyStore.FlushReason reason)
+    {
+        // We want to avoid all flushing.
+        switch (reason)
+        {
+        case STARTUP: // Called after reading and replaying the commit log.
+        case DRAIN: // Called to flush data before shutdown.
+        case INTERNALLY_FORCED: // Called to ensure ordering and persistence of system table events.
+        case MEMTABLE_PERIOD_EXPIRED: // The specified memtable expiration time elapsed.
+        case INDEX_TABLE_FLUSH: // Flush requested on index table because main table is flushing.
+        case STREAMS_RECEIVED: // Flush to save streamed data that was written to memtable.
+            return false;   // do not do anything
+
+        case INDEX_BUILD_COMPLETED:
+        case INDEX_REMOVED:
+            // Both of these are needed as safepoints for index management. Nothing to do.
+            return false;
+
+        case VIEW_BUILD_STARTED:
+        case INDEX_BUILD_STARTED:
+            // TODO: Figure out secondary indexes and views.
+            return false;
+
+        case SCHEMA_CHANGE:
+            if (!(metadata().params.memtable.factory() instanceof Factory))
+                return true;    // User has switched to a different memtable class. Flush and release all held data.
+            // Otherwise, assuming we can handle the change, don't switch.
+            // TODO: Handle
+            return false;
+
+        case STREAMING: // Called to flush data so it can be streamed. TODO: How dow we stream?
+        case VALIDATION: // Called to flush data for repair. TODO: How do we repair?
+            // ColumnFamilyStore will create sstables of the affected ranges which will not be consulted on reads and
+            // will be deleted after streaming.
+            return false;
+
+        case SNAPSHOT:
+            // We don't flush for this. Returning false will trigger a performSnapshot call.
+            return false;
+
+        case DROP: // Called when a table is dropped. This memtable is no longer necessary.
+        case TRUNCATE: // The data is being deleted, but the table remains.
+            // Returning true asks the ColumnFamilyStore to replace this memtable object without flushing.
+            // This will call discard() below to delete all held data.
+            return true;
+
+        case MEMTABLE_LIMIT: // The memtable size limit is reached, and this table was selected for flushing.
+                             // Also passed if we call owner.signalLimitReached()
+        case COMMITLOG_DIRTY: // Commitlog thinks it needs to keep data from this table.
+            // Neither of the above should happen as we specify writesAreDurable and don't use an allocator/cleaner.
+            throw new AssertionError();
+
+        case USER_FORCED:
+        case UNIT_TESTS:
+            return false;
+        default:
+            throw new AssertionError();
+        }
+    }
+
+    public void metadataUpdated()
+    {
+        // TODO: handle
+    }
+
+    public void performSnapshot(String snapshotName)
+    {
+        // TODO: implement. Figure out how to restore snapshot (with external tools).
+    }
+
+    public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
+    {
+        super.switchOut(writeBarrier, commitLogUpperBound);
+        // This can prepare the memtable data for deletion; it will still be used while the flush is proceeding.
+        // A discard call will follow.
+    }
+
+    public void discard()
+    {
+        // This will be called to release/delete all held data because the memtable is switched, due to having
+        // its data flushed, due to a truncate/drop, or due to a schema change to a different memtable class.
+
+        // TODO: Implement. This should delete all memtable data from pmem.
+        super.discard();
+    }
+
+    public CommitLogPosition getApproximateCommitLogLowerBound()
+    {
+        // We don't maintain commit log positions
+        return CommitLogPosition.NONE;
+    }
+
+    public CommitLogPosition getCommitLogLowerBound()
+    {
+        // We don't maintain commit log positions
+        return CommitLogPosition.NONE;
+    }
+
+    public LastCommitLogPosition getFinalCommitLogUpperBound()
+    {
+        // We don't maintain commit log positions
+        return new LastCommitLogPosition(CommitLogPosition.NONE);
+    }
+
+    public boolean isClean()
+    {
+        return partitionCount() == 0;
+    }
+
+    public boolean mayContainDataBefore(CommitLogPosition position)
+    {
+        // We don't track commit log positions, so if we are dirty, we may.
+        return !isClean();
+    }
+
+    public void addMemoryUsageTo(MemoryUsage stats)
+    {
+        // our memory usage is not counted
+    }
+
+    public void markExtraOnHeapUsed(long additionalSpace, OpOrder.Group opGroup)
+    {
+        // we don't track this
+    }
+
+    public void markExtraOffHeapUsed(long additionalSpace, OpOrder.Group opGroup)
+    {
+        // we don't track this
+    }
+
+    public static Factory factory(Map<String, String> furtherOptions)
+    {
+        Boolean skipOption = Boolean.parseBoolean(furtherOptions.remove("skipCommitLog"));
+        return skipOption ? commitLogSkippingFactory : commitLogWritingFactory;
+    }
+
+    private static final Factory commitLogSkippingFactory = new Factory(true);
+    private static final Factory commitLogWritingFactory = new Factory(false);
+
+    static class Factory implements Memtable.Factory
+    {
+        private final boolean skipCommitLog;
+
+        public Factory(boolean skipCommitLog)
+        {
+            this.skipCommitLog = skipCommitLog;
+        }
+
+        public Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound,
+                               TableMetadataRef metadaRef,
+                               Owner owner)
+        {
+            return new PersistentMemoryMemtable(metadaRef, owner);
+        }
+
+        public boolean writesShouldSkipCommitLog()
+        {
+            return skipCommitLog;
+        }
+
+        public boolean writesAreDurable()
+        {
+            return true;
+        }
+
+        public boolean streamToMemtable()
+        {
+            return true;
+        }
+
+        public boolean streamFromMemtable()
+        {
+            return true;
+        }
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java b/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java
index 864899f6a40b..4dc20bf7869a 100644
--- a/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java
+++ b/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java
@@ -22,8 +22,10 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 
 /**
@@ -31,8 +33,8 @@
  * In practice, each keyspace has its associated boundaries, see {@link Keyspace}.
  * <p>
  * Technically, if we use {@code n} shards, this is a list of {@code n-1} tokens and each token {@code tk} gets assigned
- * to the shard ID corresponding to the slot of the smallest token in the list that is greater to {@code tk}, or {@code n}
- * if {@code tk} is bigger than any token in the list.
+ * to the shard ID corresponding to the slot of the smallest token in the list that is equal or greater than {@code tk},
+ * or {@code n} if {@code tk} is bigger than any token in the list.
  */
 public class ShardBoundaries
 {
@@ -67,7 +69,7 @@ public int getShardForToken(Token tk)
     {
         for (int i = 0; i < boundaries.length; i++)
         {
-            if (tk.compareTo(boundaries[i]) < 0)
+            if (tk.compareTo(boundaries[i]) <= 0)   // boundaries are end-inclusive
                 return i;
         }
         return boundaries.length;
@@ -86,6 +88,33 @@ public int getShardForKey(PartitionPosition key)
         return getShardForToken(key.getToken());
     }
 
+    public AbstractBounds<PartitionPosition> getBounds(int shard)
+    {
+        checkShardIndex(shard);
+        return AbstractBounds.bounds(getMinBound(shard), false, getMaxBound(shard), true);
+    }
+
+    private void checkShardIndex(int shard)
+    {
+        if (shard < 0 || shard > boundaries.length)
+            throw new IllegalArgumentException(String.format("Shard %d out of bounds [0, %d]", shard, boundaries.length));
+    }
+
+
+    private PartitionPosition getMinBound(int shard)
+    {
+        return (shard == 0)
+               ? DatabaseDescriptor.getPartitioner().getMinimumToken().maxKeyBound()
+               : boundaries[shard - 1].maxKeyBound();
+    }
+
+    private PartitionPosition getMaxBound(int shard)
+    {
+        return (shard == boundaries.length)
+               ? DatabaseDescriptor.getPartitioner().getMaximumToken().maxKeyBound()
+               : boundaries[shard].maxKeyBound();
+    }
+
     /**
      * The number of shards that this boundaries support, that is how many different shard ids {@link #getShardForToken} might
      * possibly return.
diff --git a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
index 92cdbbad9fe0..71687bd1f288 100644
--- a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
@@ -40,6 +40,7 @@
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
 import org.apache.cassandra.db.partitions.AtomicBTreePartition;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdate;
 import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -181,6 +182,30 @@ public long getMinLocalDeletionTime()
         return min;
     }
 
+    @Override
+    public DecoratedKey minPartitionKey()
+    {
+        for (int i = 0; i < shards.length; i++)
+        {
+            MemtableShard shard = shards[i];
+            if (!shard.isClean())
+                return shard.minPartitionKey();
+        }
+        return null;
+    }
+
+    @Override
+    public DecoratedKey maxPartitionKey()
+    {
+        for (int i = shards.length - 1; i >= 0; i--)
+        {
+            MemtableShard shard = shards[i];
+            if (!shard.isClean())
+                return shard.maxPartitionKey();
+        }
+        return null;
+    }
+
     @Override
     RegularAndStaticColumns columns()
     {
@@ -237,7 +262,7 @@ private Iterator<AtomicBTreePartition> getPartitionIterator(PartitionPosition le
         return iterator;
     }
 
-    private Partition getPartition(DecoratedKey key)
+    public Partition getPartition(DecoratedKey key)
     {
         int shardIndex = boundaries.getShardForKey(key);
         return shards[shardIndex].partitions.get(key);
@@ -369,7 +394,7 @@ public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction inde
                 }
             }
 
-            BTreePartitionUpdater updater = previous.addAll(update, cloner, opGroup, indexer);
+            BTreePartitionUpdater updater = previous.addAll(BTreePartitionUpdate.asBTreeUpdate(update), cloner, opGroup, indexer);
             updateMin(minTimestamp, update.stats().minTimestamp);
             updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime);
             liveDataSize.addAndGet(initialSize + updater.dataSize);
@@ -434,6 +459,22 @@ public long minLocalDeletionTime()
         {
             return minLocalDeletionTime.get();
         }
+
+        public DecoratedKey minPartitionKey()
+        {
+            Map.Entry<PartitionPosition, AtomicBTreePartition> entry = partitions.firstEntry();
+            return (entry != null)
+                   ? entry.getValue().partitionKey()
+                   : null;
+        }
+
+        public DecoratedKey maxPartitionKey()
+        {
+            Map.Entry<PartitionPosition, AtomicBTreePartition> entry = partitions.lastEntry();
+            return (entry != null)
+                   ? entry.getValue().partitionKey()
+                   : null;
+        }
     }
 
     public static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator
diff --git a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
index 379b1fe0a595..a045aface8f8 100644
--- a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
@@ -41,10 +41,10 @@
 import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
 import org.apache.cassandra.db.partitions.AtomicBTreePartition;
 import org.apache.cassandra.db.partitions.BTreePartitionData;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdate;
 import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Bounds;
@@ -86,11 +86,25 @@ public class SkipListMemtable extends AbstractAllocatorMemtable
 
     private final AtomicLong liveDataSize = new AtomicLong(0);
 
-    protected SkipListMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner)
+    /**
+     * Keeps an estimate of the average row size in this memtable, computed from a small sample of rows.
+     * Because computing this estimate is potentially costly, as it requires iterating the rows,
+     * the estimate is updated only whenever the number of operations on the memtable increases significantly from the
+     * last update. This estimate is not very accurate but should be ok for planning or diagnostic purposes.
+     */
+    private volatile MemtableAverageRowSize estimatedAverageRowSize;
+
+    SkipListMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner)
     {
         super(commitLogLowerBound, metadataRef, owner);
     }
 
+    @Override
+    public void addMemoryUsageTo(MemoryUsage stats)
+    {
+        super.addMemoryUsageTo(stats);
+    }
+
     @Override
     public boolean isClean()
     {
@@ -127,7 +141,7 @@ public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group
             }
         }
 
-        BTreePartitionUpdater updater = previous.addAll(update, cloner, opGroup, indexer);
+        BTreePartitionUpdater updater = previous.addAll(BTreePartitionUpdate.asBTreeUpdate(update), cloner, opGroup, indexer);
         updateMin(minTimestamp, update.stats().minTimestamp);
         updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime);
         liveDataSize.addAndGet(initialSize + updater.dataSize);
@@ -191,7 +205,7 @@ private Map<PartitionPosition, AtomicBTreePartition> getPartitionsSubMap(Partiti
         }
     }
 
-    Partition getPartition(DecoratedKey key)
+    public Partition getPartition(DecoratedKey key)
     {
         return partitions.get(key);
     }
@@ -213,6 +227,24 @@ public UnfilteredRowIterator rowIterator(DecoratedKey key)
         return p != null ? p.unfilteredIterator() : null;
     }
 
+    @Override
+    public DecoratedKey minPartitionKey()
+    {
+        Map.Entry<PartitionPosition, AtomicBTreePartition> entry = partitions.firstEntry();
+        return (entry != null)
+               ? entry.getValue().partitionKey()
+               : null;
+    }
+
+    @Override
+    public DecoratedKey maxPartitionKey()
+    {
+        Map.Entry<PartitionPosition, AtomicBTreePartition> entry = partitions.lastEntry();
+        return (entry != null)
+            ? entry.getValue().partitionKey()
+            : null;
+    }
+
     private static int estimateRowOverhead(final int count)
     {
         // calculate row overhead
@@ -319,21 +351,33 @@ public long partitionKeysSize()
     }
 
 
-    private static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator
+    public static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements Memtable.MemtableUnfilteredPartitionIterator
     {
         private final TableMetadata metadata;
         private final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter;
+        private final Map<PartitionPosition, AtomicBTreePartition> source;
         private final ColumnFilter columnFilter;
         private final DataRange dataRange;
 
-        MemtableUnfilteredPartitionIterator(TableMetadata metadata, Map<PartitionPosition, AtomicBTreePartition> map, ColumnFilter columnFilter, DataRange dataRange)
+        public MemtableUnfilteredPartitionIterator(TableMetadata metadata, Map<PartitionPosition, AtomicBTreePartition> map, ColumnFilter columnFilter, DataRange dataRange)
         {
             this.metadata = metadata;
+            this.source = map;
             this.iter = map.entrySet().iterator();
             this.columnFilter = columnFilter;
             this.dataRange = dataRange;
         }
 
+        @Override
+        public long getMinLocalDeletionTime()
+        {
+            long minLocalDeletionTime = Long.MAX_VALUE;
+            for (AtomicBTreePartition partition : source.values())
+                minLocalDeletionTime = Math.min(minLocalDeletionTime, partition.stats().minLocalDeletionTime);
+
+            return minLocalDeletionTime;
+        }
+        
         @Override
         public TableMetadata metadata()
         {
diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java
index 83b02db06a0c..4d0db3a0be65 100644
--- a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java
@@ -21,24 +21,22 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Predicate;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterators;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.BufferDecoratedKey;
-import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.MutableDeletionInfo;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.Slices;
@@ -46,17 +44,19 @@
 import org.apache.cassandra.db.filter.ClusteringIndexFilter;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
-import org.apache.cassandra.db.partitions.BTreePartitionData;
-import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.TrieBackedPartition;
+import org.apache.cassandra.db.partitions.TriePartitionUpdate;
+import org.apache.cassandra.db.partitions.TriePartitionUpdater;
 import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.tries.Direction;
 import org.apache.cassandra.db.tries.InMemoryTrie;
 import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.db.tries.TrieEntriesWalker;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
+import org.apache.cassandra.db.tries.TrieTailsIterator;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.IncludingExcludingBounds;
@@ -64,15 +64,16 @@
 import org.apache.cassandra.index.transactions.UpdateTransaction;
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
-import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.metrics.TrieMemtableMetricsView;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.utils.memory.EnsureOnHeap;
+import org.apache.cassandra.utils.memory.HeapCloner;
 import org.apache.cassandra.utils.memory.MemtableAllocator;
 import org.github.jamm.Unmetered;
 
@@ -93,12 +94,13 @@ public class TrieMemtable extends AbstractShardedMemtable
     /** Buffer type to use for memtable tries (on- vs off-heap) */
     public static final BufferType BUFFER_TYPE = DatabaseDescriptor.getMemtableAllocationType().toBufferType();
 
-    /** If keys is below this length, we will use a recursive procedure for inserting data in the memtable trie. */
-    @VisibleForTesting
-    public static final int MAX_RECURSIVE_KEY_LENGTH = 128;
+    /**
+     * Force copy checker (see InMemoryTrie.ApplyState) ensuring all modifications apply atomically and consistently to
+     * the whole partition.
+     */
+    public static final Predicate<InMemoryTrie.NodeFeatures<Object>> FORCE_COPY_PARTITION_BOUNDARY = features -> isPartitionBoundary(features.content());
 
-    /** The byte-ordering conversion version to use for memtables. */
-    public static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS50;
+    public static final Predicate<Object> IS_PARTITION_BOUNDARY = TrieMemtable::isPartitionBoundary;
 
     // Set to true when the memtable requests a switch (e.g. for trie size limit being reached) to ensure only one
     // thread calls cfs.switchMemtableIfCurrent.
@@ -115,34 +117,46 @@ public class TrieMemtable extends AbstractShardedMemtable
      * A merged view of the memtable map. Used for partition range queries and flush.
      * For efficiency we serve single partition requests off the shard which offers more direct InMemoryTrie methods.
      */
-    private final Trie<BTreePartitionData> mergedTrie;
+    private final Trie<Object> mergedTrie;
 
     @Unmetered
     private final TrieMemtableMetricsView metrics;
 
+    /**
+     * Keeps an estimate of the average row size in this memtable, computed from a small sample of rows.
+     * Because computing this estimate is potentially costly, as it requires iterating the rows,
+     * the estimate is updated only whenever the number of operations on the memtable increases significantly from the
+     * last update. This estimate is not very accurate but should be ok for planning or diagnostic purposes.
+     */
+    private volatile MemtableAverageRowSize estimatedAverageRowSize;
+
     TrieMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption)
     {
         super(commitLogLowerBound, metadataRef, owner, shardCountOption);
-        this.metrics = new TrieMemtableMetricsView(metadataRef.keyspace, metadataRef.name);
-        this.shards = generatePartitionShards(boundaries.shardCount(), allocator, metadataRef, metrics);
+        this.metrics = TrieMemtableMetricsView.getOrCreate(metadataRef.keyspace, metadataRef.name);
+        this.shards = generatePartitionShards(boundaries.shardCount(), metadataRef, metrics, owner.readOrdering());
         this.mergedTrie = makeMergedTrie(shards);
+        logger.trace("Created memtable with {} shards", this.shards.length);
     }
 
     private static MemtableShard[] generatePartitionShards(int splits,
-                                                           MemtableAllocator allocator,
                                                            TableMetadataRef metadata,
-                                                           TrieMemtableMetricsView metrics)
+                                                           TrieMemtableMetricsView metrics,
+                                                           OpOrder opOrder)
     {
+        if (splits == 1)
+            return new MemtableShard[] { new MemtableShard(metadata, metrics, opOrder) };
+
         MemtableShard[] partitionMapContainer = new MemtableShard[splits];
         for (int i = 0; i < splits; i++)
-            partitionMapContainer[i] = new MemtableShard(metadata, allocator, metrics);
+            partitionMapContainer[i] = new MemtableShard(metadata, metrics, opOrder);
 
         return partitionMapContainer;
     }
 
-    private static Trie<BTreePartitionData> makeMergedTrie(MemtableShard[] shards)
+    private static Trie<Object> makeMergedTrie(MemtableShard[] shards)
     {
-        List<Trie<BTreePartitionData>> tries = new ArrayList<>(shards.length);
+        List<Trie<Object>> tries = new ArrayList<>(shards.length);
         for (MemtableShard shard : shards)
             tries.add(shard.data);
         return Trie.mergeDistinct(tries);
@@ -157,6 +171,16 @@ public boolean isClean()
         return true;
     }
 
+    @VisibleForTesting
+    @Override
+    public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
+    {
+        super.switchOut(writeBarrier, commitLogUpperBound);
+
+        for (MemtableShard shard : shards)
+            shard.allocator.setDiscarding();
+    }
+
     @Override
     public void discard()
     {
@@ -170,6 +194,7 @@ public void discard()
         // the buffer release is a longer-running process, do it in a separate loop to not make the metrics update wait
         for (MemtableShard shard : shards)
         {
+            shard.allocator.setDiscarded();
             shard.data.discardBuffers();
         }
     }
@@ -183,32 +208,32 @@ public void discard()
     @Override
     public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
     {
-        try
+        DecoratedKey key = update.partitionKey();
+        MemtableShard shard = shards[boundaries.getShardForKey(key)];
+        long colUpdateTimeDelta = shard.put(update, indexer, opGroup);
+
+        if (shard.data.reachedAllocatedSizeThreshold() && !switchRequested.getAndSet(true))
         {
-            DecoratedKey key = update.partitionKey();
-            MemtableShard shard = shards[boundaries.getShardForKey(key)];
-            long colUpdateTimeDelta = shard.put(key, update, indexer, opGroup);
+            logger.info("Scheduling flush due to trie size limit reached.");
+            owner.signalFlushRequired(this, ColumnFamilyStore.FlushReason.MEMTABLE_LIMIT);
+        }
 
-            if (shard.data.reachedAllocatedSizeThreshold() && !switchRequested.getAndSet(true))
-            {
-                logger.info("Scheduling flush due to trie size limit reached.");
-                owner.signalFlushRequired(this, ColumnFamilyStore.FlushReason.MEMTABLE_LIMIT);
-            }
+        return colUpdateTimeDelta;
+    }
 
-            return colUpdateTimeDelta;
-        }
-        catch (InMemoryTrie.SpaceExhaustedException e)
+    @Override
+    public void addMemoryUsageTo(MemoryUsage stats)
+    {
+        super.addMemoryUsageTo(stats);
+        for (MemtableShard shard : shards)
         {
-            // This should never happen as {@link InMemoryTrie#reachedAllocatedSizeThreshold} should become
-            // true and trigger a memtable switch long before this limit is reached.
-            throw new IllegalStateException(e);
+            stats.ownsOnHeap += shard.allocator.onHeap().owns();
+            stats.ownsOffHeap += shard.allocator.offHeap().owns();
+            stats.ownershipRatioOnHeap += shard.allocator.onHeap().ownershipRatio();
+            stats.ownershipRatioOffHeap += shard.allocator.offHeap().ownershipRatio();
         }
     }
 
-    /**
-     * Technically we should scatter gather on all the core threads because the size in following calls are not
-     * using volatile variables, but for metrics purpose this should be good enough.
-     */
     @Override
     public long getLiveDataSize()
     {
@@ -232,10 +257,15 @@ public long partitionCount()
     {
         int total = 0;
         for (MemtableShard shard : shards)
-            total += shard.size();
+            total += shard.partitionCount();
         return total;
     }
 
+    public int getShardCount()
+    {
+        return shards.length;
+    }
+
     /**
      * Returns the minTS if one available, otherwise NO_MIN_TIMESTAMP.
      *
@@ -248,7 +278,7 @@ public long getMinTimestamp()
     {
         long min = Long.MAX_VALUE;
         for (MemtableShard shard : shards)
-            min =  Long.min(min, shard.minTimestamp());
+            min =  EncodingStats.mergeMinTimestamp(min, shard.stats);
         return min != EncodingStats.NO_STATS.minTimestamp ? min : NO_MIN_TIMESTAMP;
     }
 
@@ -257,15 +287,39 @@ public long getMinLocalDeletionTime()
     {
         long min = Long.MAX_VALUE;
         for (MemtableShard shard : shards)
-            min =  Long.min(min, shard.minLocalDeletionTime());
+            min =  EncodingStats.mergeMinLocalDeletionTime(min, shard.stats);
         return min;
     }
 
+    @Override
+    public DecoratedKey minPartitionKey()
+    {
+        for (int i = 0; i < shards.length; i++)
+        {
+            MemtableShard shard = shards[i];
+            if (!shard.isClean())
+                return shard.minPartitionKey();
+        }
+        return null;
+    }
+
+    @Override
+    public DecoratedKey maxPartitionKey()
+    {
+        for (int i = shards.length - 1; i >= 0; i--)
+        {
+            MemtableShard shard = shards[i];
+            if (!shard.isClean())
+                return shard.maxPartitionKey();
+        }
+        return null;
+    }
+
     @Override
     RegularAndStaticColumns columns()
     {
         for (MemtableShard shard : shards)
-            columnsCollector.update(shard.columnsCollector);
+            columnsCollector.update(shard.columns);
         return columnsCollector.get();
     }
 
@@ -273,10 +327,17 @@ RegularAndStaticColumns columns()
     EncodingStats encodingStats()
     {
         for (MemtableShard shard : shards)
-            statsCollector.update(shard.statsCollector.get());
+            statsCollector.update(shard.stats);
         return statsCollector.get();
     }
 
+    static boolean isPartitionBoundary(Object content)
+    {
+        // In the trie we use PartitionData for the root of a partition, but PartitionUpdates come with DeletionInfo.
+        // Both are descendants of DeletionInfo.
+        return content instanceof DeletionInfo;
+    }
+
     @Override
     public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter columnFilter,
                                                                  final DataRange dataRange,
@@ -284,35 +345,53 @@ public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter
     {
         AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
 
-        PartitionPosition left = keyRange.left;
-        PartitionPosition right = keyRange.right;
-        if (left.isMinimum())
-            left = null;
-        if (right.isMinimum())
-            right = null;
-
         boolean isBound = keyRange instanceof Bounds;
         boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
         boolean includeStop = isBound || keyRange instanceof Range;
 
-        Trie<BTreePartitionData> subMap = mergedTrie.subtrie(left, includeStart, right, includeStop);
+        Trie<Object> subMap = mergedTrie.subtrie(toComparableBound(keyRange.left, includeStart),
+                                                 toComparableBound(keyRange.right, !includeStop));
 
         return new MemtableUnfilteredPartitionIterator(metadata(),
                                                        allocator.ensureOnHeap(),
                                                        subMap,
                                                        columnFilter,
-                                                       dataRange);
-        // readsListener is ignored as it only accepts sstable signals
+                                                       dataRange,
+                                                       getMinLocalDeletionTime());
+        // Note: the minLocalDeletionTime reported by the iterator is the memtable's minLocalDeletionTime. This is okay
+        // because we only need to report a lower bound that will eventually advance, and calculating a more precise
+        // bound would be an unnecessary expense.
     }
 
-    private Partition getPartition(DecoratedKey key)
+    private static ByteComparable toComparableBound(PartitionPosition position, boolean before)
+    {
+        return position.isMinimum() ? null : position.asComparableBound(before);
+    }
+
+    public Partition getPartition(DecoratedKey key)
     {
         int shardIndex = boundaries.getShardForKey(key);
-        BTreePartitionData data = shards[shardIndex].data.get(key);
-        if (data != null)
-            return createPartition(metadata(), allocator.ensureOnHeap(), key, data);
-        else
+        Trie<Object> trie = shards[shardIndex].data.tailTrie(key);
+        return createPartition(metadata(), allocator.ensureOnHeap(), key, trie);
+    }
+
+    private static TrieBackedPartition createPartition(TableMetadata metadata, EnsureOnHeap ensureOnHeap, DecoratedKey key, Trie<Object> trie)
+    {
+        if (trie == null)
             return null;
+        PartitionData holder = (PartitionData) trie.get(ByteComparable.EMPTY);
+        // If we found a matching path in the trie, it must be the root of this partition (because partition keys are
+        // prefix-free, it can't be a prefix for a different path, or have another partition key as prefix) and contain
+        // PartitionData (because the attachment of a new or modified partition to the trie is atomic).
+        assert holder != null : "Entry for " + key + " without associated PartitionData";
+
+        return TrieBackedPartition.create(key,
+                                          holder.columns(),
+                                          holder.stats(),
+                                          holder.rowCountIncludingStatic(),
+                                          trie,
+                                          metadata,
+                                          ensureOnHeap);
     }
 
     @Override
@@ -332,40 +411,113 @@ public UnfilteredRowIterator rowIterator(DecoratedKey key)
         return p != null ? p.unfilteredIterator() : null;
     }
 
-    private static MemtablePartition createPartition(TableMetadata metadata, EnsureOnHeap ensureOnHeap, DecoratedKey key, BTreePartitionData data)
+    private static DecoratedKey getPartitionKeyFromPath(TableMetadata metadata, ByteComparable path)
     {
-        return new MemtablePartition(metadata, ensureOnHeap, key, data);
+        return BufferDecoratedKey.fromByteComparable(path,
+                                                     TrieBackedPartition.BYTE_COMPARABLE_VERSION,
+                                                     metadata.partitioner);
     }
 
-    private static MemtablePartition getPartitionFromTrieEntry(TableMetadata metadata, EnsureOnHeap ensureOnHeap, Map.Entry<ByteComparable, BTreePartitionData> en)
+    /**
+     * Metadata object signifying the root node of a partition. Holds the deletion information as well as a link
+     * to the owning subrange, which is used for compiling statistics and column sets.
+     *
+     * Descends from MutableDeletionInfo to permit tail tries to be passed directly to TrieBackedPartition.
+     */
+    public static class PartitionData extends MutableDeletionInfo
     {
-        DecoratedKey key = BufferDecoratedKey.fromByteComparable(en.getKey(),
-                                                                 BYTE_COMPARABLE_VERSION,
-                                                                 metadata.partitioner);
-        return createPartition(metadata, ensureOnHeap, key, en.getValue());
+        @Unmetered
+        public final MemtableShard owner;
+
+        private int rowCountIncludingStatic;
+
+        public static final long HEAP_SIZE = ObjectSizes.measure(new PartitionData(DeletionInfo.LIVE, null));
+
+        public PartitionData(DeletionInfo deletion,
+                             MemtableShard owner)
+        {
+            super(deletion.getPartitionDeletion(), deletion.copyRanges(HeapCloner.instance));
+            this.owner = owner;
+            this.rowCountIncludingStatic = 0;
+        }
+
+        public PartitionData(PartitionData existing,
+                             DeletionInfo update)
+        {
+            // Start with the update content, to properly copy it
+            this(update, existing.owner);
+            rowCountIncludingStatic = existing.rowCountIncludingStatic;
+            add(existing);
+        }
+
+        public RegularAndStaticColumns columns()
+        {
+            return owner.columns;
+        }
+
+        public EncodingStats stats()
+        {
+            return owner.stats;
+        }
+
+        public int rowCountIncludingStatic()
+        {
+            return rowCountIncludingStatic;
+        }
+
+        public void markInsertedRows(int howMany)
+        {
+            rowCountIncludingStatic += howMany;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "partition " + super.toString();
+        }
+
+        @Override
+        public long unsharedHeapSize()
+        {
+            return super.unsharedHeapSize() + HEAP_SIZE - MutableDeletionInfo.EMPTY_SIZE;
+        }
     }
 
 
-    @Override
-    public FlushablePartitionSet<MemtablePartition> getFlushSet(PartitionPosition from, PartitionPosition to)
+    class KeySizeAndCountCollector extends TrieEntriesWalker<Object, Void>
     {
-        Trie<BTreePartitionData> toFlush = mergedTrie.subtrie(from, true, to, false);
         long keySize = 0;
         int keyCount = 0;
 
-        for (Iterator<Map.Entry<ByteComparable, BTreePartitionData>> it = toFlush.entryIterator(); it.hasNext(); )
+        @Override
+        public Void complete()
         {
-            Map.Entry<ByteComparable, BTreePartitionData> en = it.next();
-            byte[] keyBytes = DecoratedKey.keyFromByteSource(ByteSource.peekable(en.getKey().asComparableBytes(BYTE_COMPARABLE_VERSION)),
-                                                             BYTE_COMPARABLE_VERSION,
+            return null;
+        }
+
+        @Override
+        protected void content(Object content, byte[] bytes, int byteLength)
+        {
+            // This is used with processSkippingBranches which should ensure that we only see the partition roots.
+            assert content instanceof PartitionData;
+            ++keyCount;
+            byte[] keyBytes = DecoratedKey.keyFromByteSource(ByteSource.preencoded(bytes, 0, byteLength),
+                                                             TrieBackedPartition.BYTE_COMPARABLE_VERSION,
                                                              metadata().partitioner);
             keySize += keyBytes.length;
-            keyCount++;
         }
-        long partitionKeySize = keySize;
-        int partitionCount = keyCount;
+    }
+
+    public FlushablePartitionSet<TrieBackedPartition> getFlushSet(PartitionPosition from, PartitionPosition to)
+    {
+        Trie<Object> toFlush = mergedTrie.subtrie(from, true, to, false);
+
+        var counter = new KeySizeAndCountCollector(); // need to jump over tails keys
+        toFlush.processSkippingBranches(counter, Direction.FORWARD);
+        int partitionCount = counter.keyCount;
+        long partitionKeySize = counter.keySize;
 
-        return new AbstractFlushablePartitionSet<MemtablePartition>()
+        return new AbstractFlushablePartitionSet<TrieBackedPartition>()
         {
             public Memtable memtable()
             {
@@ -387,12 +539,9 @@ public long partitionCount()
                 return partitionCount;
             }
 
-            public Iterator<MemtablePartition> iterator()
+            public Iterator<TrieBackedPartition> iterator()
             {
-                return Iterators.transform(toFlush.entryIterator(),
-                                           // During flushing we are certain the memtable will remain at least until
-                                           // the flush completes. No copying to heap is necessary.
-                                           entry -> getPartitionFromTrieEntry(metadata(), EnsureOnHeap.NOOP, entry));
+                return new PartitionIterator(toFlush, metadata(), EnsureOnHeap.NOOP);
             }
 
             public long partitionKeysSize()
@@ -402,7 +551,7 @@ public long partitionKeysSize()
         };
     }
 
-    static class MemtableShard
+    public static class MemtableShard
     {
         // The following fields are volatile as we have to make sure that when we
         // collect results from all sub-ranges, the thread accessing the value
@@ -417,6 +566,8 @@ static class MemtableShard
 
         private volatile long currentOperations = 0;
 
+        private volatile int partitionCount = 0;
+
         @Unmetered
         private final ReentrantLock writeLock = new ReentrantLock();
 
@@ -433,11 +584,11 @@ static class MemtableShard
         // unsafely, meaning that the memtable will not be discarded as long as the data is used, or whether the data
         // should be copied on heap for off-heap allocators.
         @VisibleForTesting
-        final InMemoryTrie<BTreePartitionData> data;
+        final InMemoryTrie<Object> data;
 
-        private final ColumnsCollector columnsCollector;
+        RegularAndStaticColumns columns;
 
-        private final StatsCollector statsCollector;
+        EncodingStats stats;
 
         @Unmetered  // total pool size should not be included in memtable's deep size
         private final MemtableAllocator allocator;
@@ -445,19 +596,27 @@ static class MemtableShard
         @Unmetered
         private final TrieMemtableMetricsView metrics;
 
+        private final TableMetadataRef metadata;
+
+        MemtableShard(TableMetadataRef metadata, TrieMemtableMetricsView metrics, OpOrder opOrder)
+        {
+            this(metadata, AbstractAllocatorMemtable.MEMORY_POOL.newAllocator(metadata.toString()), metrics, opOrder);
+        }
+
         @VisibleForTesting
-        MemtableShard(TableMetadataRef metadata, MemtableAllocator allocator, TrieMemtableMetricsView metrics)
+        MemtableShard(TableMetadataRef metadata, MemtableAllocator allocator, TrieMemtableMetricsView metrics, OpOrder opOrder)
         {
-            this.data = new InMemoryTrie<>(BUFFER_TYPE);
-            this.columnsCollector = new AbstractMemtable.ColumnsCollector(metadata.get().regularAndStaticColumns());
-            this.statsCollector = new AbstractMemtable.StatsCollector();
+            this.metadata = metadata;
+            this.data = InMemoryTrie.longLived(TrieBackedPartition.BYTE_COMPARABLE_VERSION, BUFFER_TYPE, opOrder);
+            this.columns = RegularAndStaticColumns.NONE;
+            this.stats = EncodingStats.NO_STATS;
             this.allocator = allocator;
             this.metrics = metrics;
         }
 
-        public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) throws InMemoryTrie.SpaceExhaustedException
+        public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
         {
-            BTreePartitionUpdater updater = new BTreePartitionUpdater(allocator, allocator.cloner(opGroup), opGroup, indexer);
+            TriePartitionUpdater updater = new TriePartitionUpdater(allocator.cloner(opGroup), indexer, metadata.get(), this);
             boolean locked = writeLock.tryLock();
             if (locked)
             {
@@ -474,25 +633,37 @@ public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction inde
             {
                 try
                 {
-                    long onHeap = data.sizeOnHeap();
-                    long offHeap = data.sizeOffHeap();
+                    indexer.start();
+                    // Add the initial trie size on the first operation. This technically isn't correct (other shards
+                    // do take their memory share even if they are empty) but doing it during construction may cause
+                    // the allocator to block while we are trying to flush a memtable and become a deadlock.
+                    long onHeap = data.isEmpty() ? 0 : data.usedSizeOnHeap();
+                    long offHeap = data.isEmpty() ? 0 : data.usedSizeOffHeap();
                     // Use the fast recursive put if we know the key is small enough to not cause a stack overflow.
-                    data.putSingleton(key,
-                                      update,
-                                      updater::mergePartitions,
-                                      key.getKeyLength() < MAX_RECURSIVE_KEY_LENGTH);
-                    allocator.offHeap().adjust(data.sizeOffHeap() - offHeap, opGroup);
-                    allocator.onHeap().adjust(data.sizeOnHeap() - onHeap, opGroup);
+                    try
+                    {
+                        data.apply(TriePartitionUpdate.asMergableTrie(update),
+                                   updater,
+                                   FORCE_COPY_PARTITION_BOUNDARY);
+                    }
+                    catch (TrieSpaceExhaustedException e)
+                    {
+                        // This should never really happen as a flush would be triggered long before this limit is reached.
+                        throw new AssertionError(e);
+                    }
+                    allocator.offHeap().adjust(data.usedSizeOffHeap() - offHeap, opGroup);
+                    allocator.onHeap().adjust((data.usedSizeOnHeap() - onHeap) + updater.heapSize, opGroup);
+                    partitionCount += updater.partitionsAdded;
                 }
                 finally
                 {
-                    minTimestamp = Math.min(minTimestamp, update.stats().minTimestamp);
-                    minLocalDeletionTime = Math.min(minLocalDeletionTime, update.stats().minLocalDeletionTime);
-                    liveDataSize += updater.dataSize;
-                    currentOperations += update.operationCount();
+                    indexer.commit();
+                    updateMinTimestamp(update.stats().minTimestamp);
+                    updateLiveDataSize(updater.dataSize);
+                    updateCurrentOperations(update.operationCount());
 
-                    columnsCollector.update(update.columns());
-                    statsCollector.update(update.stats());
+                    columns = columns.mergeTo(update.columns());
+                    stats = stats.mergeWith(update.stats());
                 }
             }
             finally
@@ -507,151 +678,130 @@ public boolean isClean()
             return data.isEmpty();
         }
 
-        public int size()
+        private void updateMinTimestamp(long timestamp)
         {
-            return data.valuesCount();
+            if (timestamp < minTimestamp)
+                minTimestamp = timestamp;
         }
 
-        long minTimestamp()
+        void updateLiveDataSize(long size)
         {
-            return minTimestamp;
+            liveDataSize = liveDataSize + size;
         }
 
-        long liveDataSize()
+        private void updateCurrentOperations(long op)
         {
-            return liveDataSize;
+            currentOperations = currentOperations + op;
         }
 
-        long currentOperations()
+        public int partitionCount()
         {
-            return currentOperations;
+            return partitionCount;
         }
 
-        long minLocalDeletionTime()
+        long liveDataSize()
         {
-            return minLocalDeletionTime;
+            return liveDataSize;
         }
-    }
 
-    static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator
-    {
-        private final TableMetadata metadata;
-        private final EnsureOnHeap ensureOnHeap;
-        private final Iterator<Map.Entry<ByteComparable, BTreePartitionData>> iter;
-        private final ColumnFilter columnFilter;
-        private final DataRange dataRange;
-
-        public MemtableUnfilteredPartitionIterator(TableMetadata metadata,
-                                                   EnsureOnHeap ensureOnHeap,
-                                                   Trie<BTreePartitionData> source,
-                                                   ColumnFilter columnFilter,
-                                                   DataRange dataRange)
+        long currentOperations()
         {
-            this.metadata = metadata;
-            this.ensureOnHeap = ensureOnHeap;
-            this.iter = source.entryIterator();
-            this.columnFilter = columnFilter;
-            this.dataRange = dataRange;
+            return currentOperations;
         }
 
-        public TableMetadata metadata()
+        private DecoratedKey firstPartitionKey(Direction direction)
         {
-            return metadata;
+            Iterator<Map.Entry<ByteComparable.Preencoded, PartitionData>> iter = data.filteredEntryIterator(direction, PartitionData.class);
+            if (!iter.hasNext())
+                return null;
+
+            Map.Entry<ByteComparable.Preencoded, PartitionData> entry = iter.next();
+            return getPartitionKeyFromPath(metadata.get(), entry.getKey());
         }
 
-        public boolean hasNext()
+        public DecoratedKey minPartitionKey()
         {
-            return iter.hasNext();
+            return firstPartitionKey(Direction.FORWARD);
         }
 
-        public UnfilteredRowIterator next()
+        public DecoratedKey maxPartitionKey()
         {
-            Partition partition = getPartitionFromTrieEntry(metadata(), ensureOnHeap, iter.next());
-            DecoratedKey key = partition.partitionKey();
-            ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
-
-            return filter.getUnfilteredRowIterator(columnFilter, partition);
+            return firstPartitionKey(Direction.REVERSE);
         }
     }
 
-    static class MemtablePartition extends ImmutableBTreePartition
+    static class PartitionIterator extends TrieTailsIterator<Object, TrieBackedPartition>
     {
-
-        private final EnsureOnHeap ensureOnHeap;
-
-        private MemtablePartition(TableMetadata table, EnsureOnHeap ensureOnHeap, DecoratedKey key, BTreePartitionData data)
+        final TableMetadata metadata;
+        final EnsureOnHeap ensureOnHeap;
+        PartitionIterator(Trie<Object> source, TableMetadata metadata, EnsureOnHeap ensureOnHeap)
         {
-            super(table, key, data);
+            super(source, Direction.FORWARD, PartitionData.class::isInstance);
+            this.metadata = metadata;
             this.ensureOnHeap = ensureOnHeap;
         }
 
         @Override
-        protected boolean canHaveShadowedData()
-        {
-            // The BtreePartitionData we store in the memtable are build iteratively by BTreePartitionData.add(), which
-            // doesn't make sure there isn't shadowed data, so we'll need to eliminate any.
-            return true;
-        }
-
-
-        @Override
-        public DeletionInfo deletionInfo()
-        {
-            return ensureOnHeap.applyToDeletionInfo(super.deletionInfo());
-        }
-
-        @Override
-        public Row staticRow()
-        {
-            return ensureOnHeap.applyToStatic(super.staticRow());
-        }
-
-        @Override
-        public DecoratedKey partitionKey()
-        {
-            return ensureOnHeap.applyToPartitionKey(super.partitionKey());
+        protected TrieBackedPartition mapContent(Object content, Trie<Object> tailTrie, byte[] bytes, int byteLength)
+        {
+            PartitionData pd = (PartitionData) content;
+            DecoratedKey key = getPartitionKeyFromPath(metadata,
+                                                       ByteComparable.preencoded(TrieBackedPartition.BYTE_COMPARABLE_VERSION,
+                                                                                 bytes, 0, byteLength));
+            return TrieBackedPartition.create(key,
+                                              pd.columns(),
+                                              pd.stats(),
+                                              pd.rowCountIncludingStatic(),
+                                              tailTrie,
+                                              metadata,
+                                              ensureOnHeap);
         }
+    }
 
-        @Override
-        public Row getRow(Clustering<?> clustering)
-        {
-            return ensureOnHeap.applyToRow(super.getRow(clustering));
-        }
+    static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements Memtable.MemtableUnfilteredPartitionIterator
+    {
+        private final TableMetadata metadata;
+        private final Iterator<TrieBackedPartition> iter;
+        private final ColumnFilter columnFilter;
+        private final DataRange dataRange;
+        private final long minLocalDeletionTime;
 
-        @Override
-        public Row lastRow()
+        public MemtableUnfilteredPartitionIterator(TableMetadata metadata,
+                                                   EnsureOnHeap ensureOnHeap,
+                                                   Trie<Object> source,
+                                                   ColumnFilter columnFilter,
+                                                   DataRange dataRange,
+                                                   long minLocalDeletionTime)
         {
-            return ensureOnHeap.applyToRow(super.lastRow());
+            this.iter = new PartitionIterator(source, metadata, ensureOnHeap);
+            this.metadata = metadata;
+            this.columnFilter = columnFilter;
+            this.dataRange = dataRange;
+            this.minLocalDeletionTime = minLocalDeletionTime;
         }
 
-        @Override
-        public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, Slices slices, boolean reversed)
+        public long getMinLocalDeletionTime()
         {
-            return unfilteredIterator(holder(), selection, slices, reversed);
+            return minLocalDeletionTime;
         }
 
-        @Override
-        public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, NavigableSet<Clustering<?>> clusteringsInQueryOrder, boolean reversed)
+        public TableMetadata metadata()
         {
-            return ensureOnHeap.applyToPartition(super.unfilteredIterator(selection, clusteringsInQueryOrder, reversed));
+            return metadata;
         }
 
-        @Override
-        public UnfilteredRowIterator unfilteredIterator()
+        public boolean hasNext()
         {
-            return unfilteredIterator(ColumnFilter.selection(super.columns()), Slices.ALL, false);
+            return iter.hasNext();
         }
 
-        @Override
-        public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
+        public UnfilteredRowIterator next()
         {
-            return ensureOnHeap.applyToPartition(super.unfilteredIterator(current, selection, slices, reversed));
-        }
+            Partition partition = iter.next();
+            DecoratedKey key = partition.partitionKey();
+            ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
 
-        @Override
-        public Iterator<Row> iterator()
-        {
-            return ensureOnHeap.applyToPartition(super.iterator());
+            return filter.getUnfilteredRowIterator(columnFilter, partition);
         }
     }
 
@@ -659,54 +809,30 @@ public static Factory factory(Map<String, String> optionsCopy)
     {
         String shardsString = optionsCopy.remove(SHARDS_OPTION);
         Integer shardCount = shardsString != null ? Integer.parseInt(shardsString) : null;
-        return new Factory(shardCount);
+        return new TrieMemtableFactory(shardCount);
     }
 
-    static class Factory implements Memtable.Factory
+    @Override
+    public long unusedReservedOnHeapMemory()
     {
-        final Integer shardCount;
-
-        Factory(Integer shardCount)
-        {
-            this.shardCount = shardCount;
-        }
-
-        public Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound,
-                               TableMetadataRef metadaRef,
-                               Owner owner)
-        {
-            return new TrieMemtable(commitLogLowerBound, metadaRef, owner, shardCount);
-        }
-
-        @Override
-        public TableMetrics.ReleasableMetric createMemtableMetrics(TableMetadataRef metadataRef)
-        {
-            TrieMemtableMetricsView metrics = new TrieMemtableMetricsView(metadataRef.keyspace, metadataRef.name);
-            return metrics::release;
-        }
-
-        public boolean equals(Object o)
-        {
-            if (this == o)
-                return true;
-            if (o == null || getClass() != o.getClass())
-                return false;
-            Factory factory = (Factory) o;
-            return Objects.equals(shardCount, factory.shardCount);
-        }
-
-        public int hashCode()
+        long size = 0;
+        for (MemtableShard shard : shards)
         {
-            return Objects.hash(shardCount);
+            size += shard.data.unusedReservedOnHeapMemory();
+            size += shard.allocator.unusedReservedOnHeapMemory();
         }
+        size += this.allocator.unusedReservedOnHeapMemory();
+        return size;
     }
 
+    /**
+     * Release all recycled content references, including the ones waiting in still incomplete recycling lists.
+     * This is a test method and can cause null pointer exceptions if used on a live trie.
+     */
     @VisibleForTesting
-    public long unusedReservedMemory()
+    void releaseReferencesUnsafe()
     {
-        long size = 0;
         for (MemtableShard shard : shards)
-            size += shard.data.unusedReservedMemory();
-        return size;
+            shard.data.releaseReferencesUnsafe();
     }
 }
diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtableFactory.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtableFactory.java
new file mode 100644
index 000000000000..7af6c71d29d0
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtableFactory.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.config.InheritingClass;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.partitions.TriePartitionUpdate;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.metrics.TrieMemtableMetricsView;
+import org.apache.cassandra.schema.TableMetadataRef;
+
+import static org.apache.cassandra.db.partitions.PartitionUpdate.*;
+
+/**
+ * This class makes better sense as an inner class to TrieMemtable (which could be as simple as
+ * FACTORY = TrieMemtable::new), but having it there causes the TrieMemtable class to be initialized the first
+ * time it is referenced (e.g. during default memtable factory construction).
+ *
+ * Some tests want to setup table parameters before initializing DatabaseDescriptor -- this allows them to do so, and
+ * also makes sure the memtable memory pools are not created for offline tools.
+ */
+public class TrieMemtableFactory implements Memtable.Factory
+{
+    final Integer shardCount;
+
+    TrieMemtableFactory(Integer shardCount)
+    {
+        this.shardCount = shardCount;
+    }
+
+    @Override
+    public Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadaRef, Memtable.Owner owner)
+    {
+        return new TrieMemtable(commitLogLowerBound, metadaRef, owner, shardCount);
+    }
+
+    public static final TrieMemtableFactory INSTANCE = new TrieMemtableFactory(null);
+    public static InheritingClass CONFIGURATION = new InheritingClass(null, TrieMemtable.class.getName(), ImmutableMap.of());
+
+    @Override
+    public Factory partitionUpdateFactory()
+    {
+        return TriePartitionUpdate.FACTORY;
+    }
+
+    @Override
+    public TableMetrics.ReleasableMetric createMemtableMetrics(TableMetadataRef metadataRef)
+    {
+        TrieMemtableMetricsView metrics = TrieMemtableMetricsView.getOrCreate(metadataRef.keyspace, metadataRef.name);
+        return metrics::release;
+    }
+
+    public boolean equals(Object o)
+    {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        TrieMemtableFactory factory = (TrieMemtableFactory) o;
+        return Objects.equals(shardCount, factory.shardCount);
+    }
+
+    public int hashCode()
+    {
+        return Objects.hash(shardCount);
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtableStage1.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtableStage1.java
new file mode 100644
index 000000000000..80549cb57039
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtableStage1.java
@@ -0,0 +1,833 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.memtable;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Iterators;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.BTreePartitionData;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdate;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.tries.Direction;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IncludingExcludingBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.metrics.TrieMemtableMetricsView;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.Cloner;
+import org.apache.cassandra.utils.memory.EnsureOnHeap;
+import org.apache.cassandra.utils.memory.MemtableAllocator;
+import org.github.jamm.Unmetered;
+
+import static org.apache.cassandra.io.sstable.SSTableReadsListener.NOOP_LISTENER;
+
+/**
+ * Previous TrieMemtable implementation, provided for two reasons:
+ * <ul>
+ * <li> to easily compare current and earlier implementations of the trie memtable
+ * <li> to have an option to change a database back to the older implementation if we find a bug or a performance problem
+ *   with the new code.
+ *   </ul>
+ * <p>
+ * To switch a table to this version, use
+ * <code><pre>
+ *   ALTER TABLE ... WITH memtable = {'class': 'TrieMemtableStage1'}
+ * </pre></code>
+ * or add
+ * <code><pre>
+ *   memtable:
+ *     class: TrieMemtableStage1
+ * </pre></code>
+ * in <code>cassandra.yaml</code> to switch a node to it as default.
+ *
+ */
+public class TrieMemtableStage1 extends AbstractAllocatorMemtable
+{
+    private static final Logger logger = LoggerFactory.getLogger(TrieMemtableStage1.class);
+
+    public static final Factory FACTORY = new TrieMemtableStage1.Factory();
+
+    static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS41;
+
+    /** If keys is below this length, we will use a recursive procedure for inserting data in the memtable trie. */
+    @VisibleForTesting
+    public static final int MAX_RECURSIVE_KEY_LENGTH = 128;
+
+    // Set to true when the memtable requests a switch (e.g. for trie size limit being reached) to ensure only one
+    // thread calls cfs.switchMemtableIfCurrent.
+    private AtomicBoolean switchRequested = new AtomicBoolean(false);
+
+
+    // The boundaries for the keyspace as they were calculated when the memtable is created.
+    // The boundaries will be NONE for system keyspaces or if StorageService is not yet initialized.
+    // The fact this is fixed for the duration of the memtable lifetime, guarantees we'll always pick the same core
+    // for the a given key, even if we race with the StorageService initialization or with topology changes.
+    @Unmetered
+    private final ShardBoundaries boundaries;
+
+    /**
+     * Core-specific memtable regions. All writes must go through the specific core. The data structures used
+     * are concurrent-read safe, thus reads can be carried out from any thread.
+     */
+    private final MemtableShard[] shards;
+
+    /**
+     * A merged view of the memtable map. Used for partition range queries and flush.
+     * For efficiency we serve single partition requests off the shard which offers more direct InMemoryTrie methods.
+     */
+    private final Trie<BTreePartitionData> mergedTrie;
+
+    @Unmetered
+    private final TrieMemtableMetricsView metrics;
+
+    /**
+     * Keeps an estimate of the average row size in this memtable, computed from a small sample of rows.
+     * Because computing this estimate is potentially costly, as it requires iterating the rows,
+     * the estimate is updated only whenever the number of operations on the memtable increases significantly from the
+     * last update. This estimate is not very accurate but should be ok for planning or diagnostic purposes.
+     */
+    private volatile MemtableAverageRowSize estimatedAverageRowSize;
+
+    // only to be used by init(), to setup the very first memtable for the cfs
+    TrieMemtableStage1(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner)
+    {
+        super(commitLogLowerBound, metadataRef, owner);
+        this.boundaries = owner.localRangeSplits(AbstractShardedMemtable.getDefaultShardCount());
+        this.metrics = TrieMemtableMetricsView.getOrCreate(metadataRef.keyspace, metadataRef.name);
+        this.shards = generatePartitionShards(boundaries.shardCount(), metadataRef, metrics);
+        this.mergedTrie = makeMergedTrie(shards);
+        logger.trace("Created memtable with {} shards", this.shards.length);
+    }
+
+    private static MemtableShard[] generatePartitionShards(int splits,
+                                                           TableMetadataRef metadata,
+                                                           TrieMemtableMetricsView metrics)
+    {
+        if (splits == 1)
+            return new MemtableShard[] { new MemtableShard(metadata, metrics) };
+
+        MemtableShard[] partitionMapContainer = new MemtableShard[splits];
+        for (int i = 0; i < splits; i++)
+            partitionMapContainer[i] = new MemtableShard(metadata, metrics);
+
+        return partitionMapContainer;
+    }
+
+    private static Trie<BTreePartitionData> makeMergedTrie(MemtableShard[] shards)
+    {
+        List<Trie<BTreePartitionData>> tries = new ArrayList<>(shards.length);
+        for (MemtableShard shard : shards)
+            tries.add(shard.data);
+        return Trie.mergeDistinct(tries);
+    }
+
+    public boolean isClean()
+    {
+        for (MemtableShard shard : shards)
+            if (!shard.isEmpty())
+                return false;
+        return true;
+    }
+
+    @VisibleForTesting
+    @Override
+    public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
+    {
+        super.switchOut(writeBarrier, commitLogUpperBound);
+
+        for (MemtableShard shard : shards)
+            shard.allocator.setDiscarding();
+    }
+
+    @Override
+    public void discard()
+    {
+        super.discard();
+        // metrics here are not thread safe, but I think we can live with that
+        metrics.lastFlushShardDataSizes.reset();
+        for (MemtableShard shard : shards)
+        {
+            metrics.lastFlushShardDataSizes.update(shard.liveDataSize());
+        }
+        for (MemtableShard shard : shards)
+        {
+            shard.allocator.setDiscarded();
+            shard.data.discardBuffers();
+        }
+    }
+
+    /**
+     * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
+     * OpOrdering.
+     *
+     * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
+     */
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    {
+        DecoratedKey key = update.partitionKey();
+        MemtableShard shard = shards[boundaries.getShardForKey(key)];
+        long colUpdateTimeDelta = shard.put(key, update, indexer, opGroup);
+
+        if (shard.data.reachedAllocatedSizeThreshold() && !switchRequested.getAndSet(true))
+        {
+            logger.info("Scheduling flush due to trie size limit reached.");
+            owner.signalFlushRequired(this, ColumnFamilyStore.FlushReason.MEMTABLE_LIMIT);
+        }
+
+        return colUpdateTimeDelta;
+    }
+
+    @Override
+    public void addMemoryUsageTo(MemoryUsage stats)
+    {
+        super.addMemoryUsageTo(stats);
+        for (MemtableShard shard : shards)
+        {
+            stats.ownsOnHeap += shard.allocator.onHeap().owns();
+            stats.ownsOffHeap += shard.allocator.offHeap().owns();
+            stats.ownershipRatioOnHeap += shard.allocator.onHeap().ownershipRatio();
+            stats.ownershipRatioOffHeap += shard.allocator.offHeap().ownershipRatio();
+        }
+    }
+
+    /**
+     * Technically we should scatter gather on all the core threads because the size in following calls are not
+     * using volatile variables, but for metrics purpose this should be good enough.
+     */
+    @Override
+    public long getLiveDataSize()
+    {
+        long total = 0L;
+        for (MemtableShard shard : shards)
+            total += shard.liveDataSize();
+        return total;
+    }
+
+    @Override
+    public long operationCount()
+    {
+        long total = 0L;
+        for (MemtableShard shard : shards)
+            total += shard.currentOperations();
+        return total;
+    }
+
+    @Override
+    public long partitionCount()
+    {
+        int total = 0;
+        for (MemtableShard shard : shards)
+            total += shard.partitionCount();
+        return total;
+    }
+
+    public int getShardCount()
+    {
+        return shards.length;
+    }
+
+    public long rowCount(final ColumnFilter columnFilter, final DataRange dataRange)
+    {
+        int total = 0;
+        for (MemtableUnfilteredPartitionIterator iter = partitionIterator(columnFilter, dataRange, NOOP_LISTENER); iter.hasNext(); )
+        {
+            for (UnfilteredRowIterator it = iter.next(); it.hasNext(); )
+            {
+                Unfiltered uRow = it.next();
+                if (uRow.isRow())
+                    total++;
+            }
+        }
+
+        return total;
+    }
+
+    @Override
+    public long getEstimatedAverageRowSize()
+    {
+        if (estimatedAverageRowSize == null || currentOperations.get() > estimatedAverageRowSize.operations * 1.5)
+            estimatedAverageRowSize = new MemtableAverageRowSize(this);
+        return estimatedAverageRowSize.rowSize;
+    }
+
+    @Override
+    public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter columnFilter, boolean reversed, SSTableReadsListener listener)
+    {
+        Partition p = getPartition(key);
+        if (p == null)
+            return null;
+        else
+            return p.unfilteredIterator(columnFilter, slices, reversed);
+    }
+
+    @Override
+    public UnfilteredRowIterator rowIterator(DecoratedKey key)
+    {
+        Partition p = getPartition(key);
+        return p != null ? p.unfilteredIterator() : null;
+    }
+
+    /**
+     * Returns the minTS if one available, otherwise NO_MIN_TIMESTAMP.
+     *
+     * EncodingStats uses a synthetic epoch TS at 2015. We don't want to leak that (CASSANDRA-18118) so we return NO_MIN_TIMESTAMP instead.
+     *
+     * @return The minTS or NO_MIN_TIMESTAMP if none available
+     */
+    @Override
+    public long getMinTimestamp()
+    {
+        long min = Long.MAX_VALUE;
+        for (MemtableShard shard : shards)
+            min =  EncodingStats.mergeMinTimestamp(min, shard.stats);
+        return min != EncodingStats.NO_STATS.minTimestamp ? min : NO_MIN_TIMESTAMP;
+    }
+
+    @Override
+    public DecoratedKey minPartitionKey()
+    {
+        for (int i = 0; i < shards.length; i++)
+        {
+            MemtableShard shard = shards[i];
+            if (!shard.isEmpty())
+                return shard.minPartitionKey();
+        }
+        return null;
+    }
+
+    @Override
+    public DecoratedKey maxPartitionKey()
+    {
+        for (int i = shards.length - 1; i >= 0; i--)
+        {
+            MemtableShard shard = shards[i];
+            if (!shard.isEmpty())
+                return shard.maxPartitionKey();
+        }
+        return null;
+    }
+
+    @Override
+    RegularAndStaticColumns columns()
+    {
+        for (MemtableShard shard : shards)
+            columnsCollector.update(shard.columns);
+        return columnsCollector.get();
+    }
+
+    @Override
+    EncodingStats encodingStats()
+    {
+        for (MemtableShard shard : shards)
+            statsCollector.update(shard.stats);
+        return statsCollector.get();
+    }
+
+    @Override
+    public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter columnFilter,
+                                                                 final DataRange dataRange,
+                                                                 SSTableReadsListener readsListener)
+    {
+        AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
+
+        PartitionPosition left = keyRange.left;
+        PartitionPosition right = keyRange.right;
+        if (left.isMinimum())
+            left = null;
+        if (right.isMinimum())
+            right = null;
+
+        boolean isBound = keyRange instanceof Bounds;
+        boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
+        boolean includeStop = isBound || keyRange instanceof Range;
+
+        Trie<BTreePartitionData> subMap = mergedTrie.subtrie(left, includeStart, right, includeStop);
+
+        return new MemtableUnfilteredPartitionIterator(metadata(),
+                                                       allocator.ensureOnHeap(),
+                                                       subMap,
+                                                       columnFilter,
+                                                       dataRange);
+    }
+
+    public Partition getPartition(DecoratedKey key)
+    {
+        int shardIndex = boundaries.getShardForKey(key);
+        BTreePartitionData data = shards[shardIndex].data.get(key);
+        if (data != null)
+            return createPartition(metadata(), allocator.ensureOnHeap(), key, data);
+        else
+            return null;
+    }
+
+    private static MemtablePartition createPartition(TableMetadata metadata, EnsureOnHeap ensureOnHeap, DecoratedKey key, BTreePartitionData data)
+    {
+        return new MemtablePartition(metadata, ensureOnHeap, key, data);
+    }
+
+    private static MemtablePartition getPartitionFromTrieEntry(TableMetadata metadata, EnsureOnHeap ensureOnHeap, Map.Entry<? extends ByteComparable, BTreePartitionData> en)
+    {
+        DecoratedKey key = BufferDecoratedKey.fromByteComparable(en.getKey(),
+                                                                 BYTE_COMPARABLE_VERSION,
+                                                                 metadata.partitioner);
+        return createPartition(metadata, ensureOnHeap, key, en.getValue());
+    }
+
+    private static DecoratedKey getPartitionKeyFromPath(TableMetadata metadata, ByteComparable path)
+    {
+        return BufferDecoratedKey.fromByteComparable(path, BYTE_COMPARABLE_VERSION, metadata.partitioner);
+    }
+
+    public FlushablePartitionSet<MemtablePartition> getFlushSet(PartitionPosition from, PartitionPosition to)
+    {
+        Trie<BTreePartitionData> toFlush = mergedTrie.subtrie(from, true, to, false);
+        long keySize = 0;
+        int keyCount = 0;
+
+        for (Iterator<Map.Entry<ByteComparable.Preencoded, BTreePartitionData>> it = toFlush.entryIterator(); it.hasNext(); )
+        {
+            Map.Entry<ByteComparable.Preencoded, BTreePartitionData> en = it.next();
+            byte[] keyBytes = DecoratedKey.keyFromByteComparable(en.getKey(), BYTE_COMPARABLE_VERSION, metadata().partitioner);
+            keySize += keyBytes.length;
+            keyCount++;
+        }
+        long partitionKeySize = keySize;
+        int partitionCount = keyCount;
+
+        return new AbstractFlushablePartitionSet<MemtablePartition>()
+        {
+            public Memtable memtable()
+            {
+                return TrieMemtableStage1.this;
+            }
+
+            public PartitionPosition from()
+            {
+                return from;
+            }
+
+            public PartitionPosition to()
+            {
+                return to;
+            }
+
+            public long partitionCount()
+            {
+                return partitionCount;
+            }
+
+            public Iterator<MemtablePartition> iterator()
+            {
+                return Iterators.transform(toFlush.entryIterator(),
+                                           // During flushing we are certain the memtable will remain at least until
+                                           // the flush completes. No copying to heap is necessary.
+                                           entry -> getPartitionFromTrieEntry(metadata(), EnsureOnHeap.NOOP, entry));
+            }
+
+            public long partitionKeysSize()
+            {
+                return partitionKeySize;
+            }
+        };
+    }
+
+    static class MemtableShard
+    {
+        // The following fields are volatile as we have to make sure that when we
+        // collect results from all sub-ranges, the thread accessing the value
+        // is guaranteed to see the changes to the values.
+
+        // The smallest timestamp for all partitions stored in this shard
+        private volatile long minTimestamp = Long.MAX_VALUE;
+
+        private volatile long liveDataSize = 0;
+
+        private volatile long currentOperations = 0;
+
+        private volatile int partitionCount = 0;
+
+        @Unmetered
+        private ReentrantLock writeLock = new ReentrantLock();
+
+        // Content map for the given shard. This is implemented as a memtable trie which uses the prefix-free
+        // byte-comparable ByteSource representations of the keys to address the partitions.
+        //
+        // This map is used in a single-producer, multi-consumer fashion: only one thread will insert items but
+        // several threads may read from it and iterate over it. Iterators are created when a the first item of
+        // a flow is requested for example, and then used asynchronously when sub-sequent items are requested.
+        //
+        // Therefore, iterators should not throw ConcurrentModificationExceptions if the underlying map is modified
+        // during iteration, they should provide a weakly consistent view of the map instead.
+        //
+        // Also, this data is backed by memtable memory, when accessing it callers must specify if it can be accessed
+        // unsafely, meaning that the memtable will not be discarded as long as the data is used, or whether the data
+        // should be copied on heap for off-heap allocators.
+        @VisibleForTesting
+        final InMemoryTrie<BTreePartitionData> data;
+
+        RegularAndStaticColumns columns;
+
+        EncodingStats stats;
+
+        private final MemtableAllocator allocator;
+
+        @Unmetered
+        private final TrieMemtableMetricsView metrics;
+
+        private TableMetadataRef metadata;
+
+        MemtableShard(TableMetadataRef metadata, TrieMemtableMetricsView metrics)
+        {
+            this(metadata, AbstractAllocatorMemtable.MEMORY_POOL.newAllocator(metadata.toString()), metrics);
+        }
+
+        @VisibleForTesting
+        MemtableShard(TableMetadataRef metadata, MemtableAllocator allocator, TrieMemtableMetricsView metrics)
+        {
+            this.data = InMemoryTrie.shortLived(BYTE_COMPARABLE_VERSION, TrieMemtable.BUFFER_TYPE);
+            this.columns = RegularAndStaticColumns.NONE;
+            this.stats = EncodingStats.NO_STATS;
+            this.allocator = allocator;
+            this.metrics = metrics;
+            this.metadata = metadata;
+        }
+
+        public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+        {
+            Cloner cloner = allocator.cloner(opGroup);
+            BTreePartitionUpdater updater = new BTreePartitionUpdater(allocator, cloner, opGroup, indexer);
+            boolean locked = writeLock.tryLock();
+            if (locked)
+            {
+                metrics.uncontendedPuts.inc();
+            }
+            else
+            {
+                metrics.contendedPuts.inc();
+                long lockStartTime = Clock.Global.nanoTime();
+                writeLock.lock();
+                metrics.contentionTime.addNano(Clock.Global.nanoTime() - lockStartTime);
+            }
+            try
+            {
+                try
+                {
+                    // Add the initial trie size on the first operation. This technically isn't correct (other shards
+                    // do take their memory share even if they are empty) but doing it during construction may cause
+                    // the allocator to block while we are trying to flush a memtable and become a deadlock.
+                    long onHeap = data.isEmpty() ? 0 : data.usedSizeOnHeap();
+                    long offHeap = data.isEmpty() ? 0 : data.usedSizeOffHeap();
+                    // Use the fast recursive put if we know the key is small enough to not cause a stack overflow.
+                    try
+                    {
+                        data.putSingleton(key,
+                                          BTreePartitionUpdate.asBTreeUpdate(update),
+                                          updater::mergePartitions,
+                                          key.getKeyLength() < MAX_RECURSIVE_KEY_LENGTH);
+                    }
+                    catch (TrieSpaceExhaustedException e)
+                    {
+                        // This should never really happen as a flush would be triggered long before this limit is reached.
+                        throw Throwables.propagate(e);
+                    }
+                    allocator.offHeap().adjust(data.usedSizeOffHeap() - offHeap, opGroup);
+                    allocator.onHeap().adjust(data.usedSizeOnHeap() - onHeap, opGroup);
+                    partitionCount += updater.partitionsAdded;
+                }
+                finally
+                {
+                    updateMinTimestamp(update.stats().minTimestamp);
+                    updateLiveDataSize(updater.dataSize);
+                    updateCurrentOperations(update.operationCount());
+
+                    columns = columns.mergeTo(update.columns());
+                    stats = stats.mergeWith(update.stats());
+                }
+            }
+            finally
+            {
+                writeLock.unlock();
+            }
+            return updater.colUpdateTimeDelta;
+        }
+
+        public boolean isEmpty()
+        {
+            return data.isEmpty();
+        }
+
+        private void updateMinTimestamp(long timestamp)
+        {
+            if (timestamp < minTimestamp)
+                minTimestamp = timestamp;
+        }
+
+        void updateLiveDataSize(long size)
+        {
+            liveDataSize = liveDataSize + size;
+        }
+
+        private void updateCurrentOperations(long op)
+        {
+            currentOperations = currentOperations + op;
+        }
+
+        public int partitionCount()
+        {
+            return partitionCount;
+        }
+
+        long liveDataSize()
+        {
+            return liveDataSize;
+        }
+
+        long currentOperations()
+        {
+            return currentOperations;
+        }
+
+        private DecoratedKey firstPartitionKey(Direction direction)
+        {
+            Iterator<Map.Entry<ByteComparable.Preencoded, BTreePartitionData>> iter = data.entryIterator(direction);
+            if (!iter.hasNext())
+                return null;
+
+            Map.Entry<ByteComparable.Preencoded, BTreePartitionData> entry = iter.next();
+            return getPartitionKeyFromPath(metadata.get(), entry.getKey());
+        }
+
+        public DecoratedKey minPartitionKey()
+        {
+            return firstPartitionKey(Direction.FORWARD);
+        }
+
+        public DecoratedKey maxPartitionKey()
+        {
+            return firstPartitionKey(Direction.REVERSE);
+        }
+    }
+
+    static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements Memtable.MemtableUnfilteredPartitionIterator
+    {
+        private final TableMetadata metadata;
+        private final EnsureOnHeap ensureOnHeap;
+        private final Trie<BTreePartitionData> source;
+        private final Iterator<Map.Entry<ByteComparable.Preencoded, BTreePartitionData>> iter;
+        private final ColumnFilter columnFilter;
+        private final DataRange dataRange;
+
+        public MemtableUnfilteredPartitionIterator(TableMetadata metadata,
+                                                   EnsureOnHeap ensureOnHeap,
+                                                   Trie<BTreePartitionData> source,
+                                                   ColumnFilter columnFilter,
+                                                   DataRange dataRange)
+        {
+            this.metadata = metadata;
+            this.ensureOnHeap = ensureOnHeap;
+            this.iter = source.entryIterator();
+            this.source = source;
+            this.columnFilter = columnFilter;
+            this.dataRange = dataRange;
+        }
+
+        public long getMinLocalDeletionTime()
+        {
+            long minLocalDeletionTime = Long.MAX_VALUE;
+            for (BTreePartitionData partition : source.values())
+                minLocalDeletionTime = EncodingStats.mergeMinLocalDeletionTime(minLocalDeletionTime, partition.stats);
+
+            return minLocalDeletionTime;
+        }
+
+        public TableMetadata metadata()
+        {
+            return metadata;
+        }
+
+        public boolean hasNext()
+        {
+            return iter.hasNext();
+        }
+
+        public UnfilteredRowIterator next()
+        {
+            Partition partition = getPartitionFromTrieEntry(metadata(), ensureOnHeap, iter.next());
+            DecoratedKey key = partition.partitionKey();
+            ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
+
+            return filter.getUnfilteredRowIterator(columnFilter, partition);
+        }
+    }
+
+    static class MemtablePartition extends ImmutableBTreePartition
+    {
+
+        private final EnsureOnHeap ensureOnHeap;
+
+        private MemtablePartition(TableMetadata table, EnsureOnHeap ensureOnHeap, DecoratedKey key, BTreePartitionData data)
+        {
+            super(table, key, data);
+            this.ensureOnHeap = ensureOnHeap;
+        }
+
+        @Override
+        protected boolean canHaveShadowedData()
+        {
+            // The BtreePartitionData we store in the memtable are build iteratively by BTreePartitionData.add(), which
+            // doesn't make sure there isn't shadowed data, so we'll need to eliminate any.
+            return true;
+        }
+
+
+        @Override
+        public DeletionInfo deletionInfo()
+        {
+            return ensureOnHeap.applyToDeletionInfo(super.deletionInfo());
+        }
+
+        @Override
+        public Row staticRow()
+        {
+            return ensureOnHeap.applyToStatic(super.staticRow());
+        }
+
+        @Override
+        public DecoratedKey partitionKey()
+        {
+            return ensureOnHeap.applyToPartitionKey(super.partitionKey());
+        }
+
+        @Override
+        public Row getRow(Clustering<?> clustering)
+        {
+            return ensureOnHeap.applyToRow(super.getRow(clustering));
+        }
+
+        @Override
+        public Row lastRow()
+        {
+            return ensureOnHeap.applyToRow(super.lastRow());
+        }
+
+        @Override
+        public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, Slices slices, boolean reversed)
+        {
+            return unfilteredIterator(holder(), selection, slices, reversed);
+        }
+
+        @Override
+        public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, NavigableSet<Clustering<?>> clusteringsInQueryOrder, boolean reversed)
+        {
+            return ensureOnHeap
+                            .applyToPartition(super.unfilteredIterator(selection, clusteringsInQueryOrder, reversed));
+        }
+
+        @Override
+        public UnfilteredRowIterator unfilteredIterator()
+        {
+            return unfilteredIterator(ColumnFilter.selection(super.columns()), Slices.ALL, false);
+        }
+
+        @Override
+        public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
+        {
+            return ensureOnHeap
+                            .applyToPartition(super.unfilteredIterator(current, selection, slices, reversed));
+        }
+
+        @Override
+        public Iterator<Row> rowIterator()
+        {
+            return ensureOnHeap.applyToPartition(super.rowIterator());
+        }
+    }
+
+    static class Factory implements Memtable.Factory
+    {
+        public Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound,
+                               TableMetadataRef metadaRef,
+                               Owner owner)
+        {
+            return new TrieMemtableStage1(commitLogLowerBound, metadaRef, owner);
+        }
+
+        @Override
+        public TableMetrics.ReleasableMetric createMemtableMetrics(TableMetadataRef metadataRef)
+        {
+            TrieMemtableMetricsView metrics = TrieMemtableMetricsView.getOrCreate(metadataRef.keyspace, metadataRef.name);
+            return metrics::release;
+        }
+    }
+
+    @Override
+    @VisibleForTesting
+    public long unusedReservedOnHeapMemory()
+    {
+        long size = 0;
+        for (MemtableShard shard : shards)
+        {
+            size += shard.data.unusedReservedOnHeapMemory();
+            size += shard.allocator.unusedReservedOnHeapMemory();
+        }
+        return size;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/monitoring/MonitorableImpl.java b/src/java/org/apache/cassandra/db/monitoring/MonitorableImpl.java
index 31b54043c834..28438acd7548 100644
--- a/src/java/org/apache/cassandra/db/monitoring/MonitorableImpl.java
+++ b/src/java/org/apache/cassandra/db/monitoring/MonitorableImpl.java
@@ -18,6 +18,8 @@
 
 package org.apache.cassandra.db.monitoring;
 
+import org.apache.cassandra.index.sai.QueryContext;
+
 import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 
 public abstract class MonitorableImpl implements Monitorable
@@ -123,6 +125,9 @@ public boolean complete()
 
     private void check()
     {
+        if (QueryContext.DISABLE_TIMEOUT)
+            return;
+
         if (approxCreationTimeNanos < 0 || state != MonitoringState.IN_PROGRESS)
             return;
 
diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
index 33272375733f..4b390bd9c158 100644
--- a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
@@ -33,7 +33,7 @@
 
 import static org.apache.cassandra.utils.btree.BTree.Dir.desc;
 
-public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
+public abstract class AbstractBTreePartition implements Partition
 {
     protected final DecoratedKey partitionKey;
 
@@ -352,50 +352,22 @@ protected static BTreePartitionData build(RowIterator rows, DeletionInfo deletio
     @Override
     public String toString()
     {
-        return toString(true);
-    }
-
-    public String toString(boolean includeFullDetails)
-    {
-        StringBuilder sb = new StringBuilder();
-        if (includeFullDetails)
-        {
-            sb.append(String.format("[%s.%s] key=%s partition_deletion=%s columns=%s",
-                                    metadata().keyspace,
-                                    metadata().name,
-                                    metadata().partitionKeyType.getString(partitionKey().getKey()),
-                                    partitionLevelDeletion(),
-                                    columns()));
-        }
-        else
-        {
-            sb.append("key=").append(metadata().partitionKeyType.getString(partitionKey().getKey()));
-        }
-
-        if (staticRow() != Rows.EMPTY_STATIC_ROW)
-            sb.append("\n    ").append(staticRow().toString(metadata(), includeFullDetails));
-
-        try (UnfilteredRowIterator iter = unfilteredIterator())
-        {
-            while (iter.hasNext())
-                sb.append("\n    ").append(iter.next().toString(metadata(), includeFullDetails));
-        }
-        return sb.toString();
+        return Partition.toString(this);
     }
 
     @Override
     public boolean equals(Object obj)
     {
-        if (!(obj instanceof PartitionUpdate))
+        if (!(obj instanceof BTreePartitionUpdate))
             return false;
 
-        PartitionUpdate that = (PartitionUpdate) obj;
+        BTreePartitionUpdate that = (BTreePartitionUpdate) obj;
         BTreePartitionData a = this.holder(), b = that.holder();
         return partitionKey.equals(that.partitionKey)
                && metadata().id.equals(that.metadata().id)
                && a.deletionInfo.equals(b.deletionInfo)
                && a.staticRow.equals(b.staticRow)
-               && Iterators.elementsEqual(iterator(), that.iterator());
+               && Iterators.elementsEqual(rowIterator(), that.rowIterator());
     }
 
     public int rowCount()
@@ -403,7 +375,7 @@ public int rowCount()
         return BTree.size(holder().tree);
     }
 
-    public Iterator<Row> iterator()
+    public Iterator<Row> rowIterator()
     {
         return BTree.<Row>iterator(holder().tree);
     }
diff --git a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
index c9035befbde5..5ba04184abc5 100644
--- a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 import java.util.Iterator;
+import java.util.NavigableSet;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 
@@ -117,7 +118,10 @@ protected boolean canHaveShadowedData()
      * @return an array containing first the difference in size seen after merging the updates, and second the minimum
      * time delta between updates.
      */
-    public BTreePartitionUpdater addAll(final PartitionUpdate update, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
+    public BTreePartitionUpdater addAll(final BTreePartitionUpdate update,
+                                        Cloner cloner,
+                                        OpOrder.Group writeOp,
+                                        UpdateTransaction indexer)
     {
         return new Updater(allocator, cloner, writeOp, indexer).addAll(update);
     }
@@ -143,7 +147,7 @@ public Updater(MemtableAllocator allocator, Cloner cloner, OpOrder.Group writeOp
             super(allocator, cloner, writeOp, indexer);
         }
 
-        Updater addAll(final PartitionUpdate update)
+        Updater addAll(final BTreePartitionUpdate update)
         {
             try
             {
@@ -176,7 +180,7 @@ Updater addAll(final PartitionUpdate update)
             }
         }
 
-        private boolean tryUpdateData(PartitionUpdate update)
+        private boolean tryUpdateData(BTreePartitionUpdate update)
         {
             current = ref;
             this.dataSize = 0;
@@ -216,6 +220,24 @@ public Row lastRow()
         return allocator.ensureOnHeap().applyToRow(super.lastRow());
     }
 
+    @Override
+    public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, Slices slices, boolean reversed)
+    {
+        return allocator.ensureOnHeap().applyToPartition(super.unfilteredIterator(selection, slices, reversed));
+    }
+
+    @Override
+    public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, NavigableSet<Clustering<?>> clusteringsInQueryOrder, boolean reversed)
+    {
+        return allocator.ensureOnHeap().applyToPartition(super.unfilteredIterator(selection, clusteringsInQueryOrder, reversed));
+    }
+
+    @Override
+    public UnfilteredRowIterator unfilteredIterator()
+    {
+        return allocator.ensureOnHeap().applyToPartition(super.unfilteredIterator());
+    }
+
     @Override
     public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
     {
@@ -223,9 +245,9 @@ public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, Colu
     }
 
     @Override
-    public Iterator<Row> iterator()
+    public Iterator<Row> rowIterator()
     {
-        return allocator.ensureOnHeap().applyToPartition(super.iterator());
+        return allocator.ensureOnHeap().applyToPartition(super.rowIterator());
     }
 
     private boolean shouldLock(OpOrder.Group writeOp)
diff --git a/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdate.java
new file mode 100644
index 000000000000..7dedd2d66544
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdate.java
@@ -0,0 +1,630 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.partitions;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.MutableDeletionInfo;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.utils.btree.BTree;
+import org.apache.cassandra.utils.btree.UpdateFunction;
+
+/**
+ * Implementation of PartitionUpdate using a BTree of rows.
+ */
+public class BTreePartitionUpdate extends AbstractBTreePartition implements PartitionUpdate
+{
+    protected static final Logger logger = LoggerFactory.getLogger(BTreePartitionUpdate.class);
+
+    public static final BTreeFactory FACTORY = new BTreeFactory();
+
+    private final BTreePartitionData holder;
+    private final DeletionInfo deletionInfo;
+    private final TableMetadata metadata;
+
+    private final boolean canHaveShadowedData;
+
+    private BTreePartitionUpdate(TableMetadata metadata,
+                                 DecoratedKey key,
+                                 BTreePartitionData holder,
+                                 MutableDeletionInfo deletionInfo,
+                                 boolean canHaveShadowedData)
+    {
+        super(key);
+        this.metadata = metadata;
+        this.holder = holder;
+        this.deletionInfo = deletionInfo;
+        this.canHaveShadowedData = canHaveShadowedData;
+    }
+
+    /**
+     * Creates a empty immutable partition update.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the created update.
+     *
+     * @return the newly created empty (and immutable) update.
+     */
+    public static BTreePartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey key)
+    {
+        MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
+        BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
+        return new BTreePartitionUpdate(metadata, key, holder, deletionInfo, false);
+    }
+
+    /**
+     * Creates an immutable partition update that entirely deletes a given partition.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the partition that the created update should delete.
+     * @param timestamp the timestamp for the deletion.
+     * @param nowInSec the current time in seconds to use as local deletion time for the partition deletion.
+     *
+     * @return the newly created partition deletion update.
+     */
+    public static BTreePartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, long nowInSec)
+    {
+        MutableDeletionInfo deletionInfo = new MutableDeletionInfo(timestamp, nowInSec);
+        BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
+        return new BTreePartitionUpdate(metadata, key, holder, deletionInfo, false);
+    }
+
+    /**
+     * Creates an immutable partition update that contains a single row update.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the partition to update.
+     * @param row the row for the update (may be null).
+     * @param row the static row for the update (may be null).
+     *
+     * @return the newly created partition update containing only {@code row}.
+     */
+    public static BTreePartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row, Row staticRow)
+    {
+        MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
+        BTreePartitionData holder = new BTreePartitionData(
+            new RegularAndStaticColumns(
+                staticRow == null ? Columns.NONE : Columns.from(staticRow),
+                row == null ? Columns.NONE : Columns.from(row)
+            ),
+            row == null ? BTree.empty() : BTree.singleton(row),
+            deletionInfo,
+            staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow,
+            EncodingStats.NO_STATS
+        );
+        return new BTreePartitionUpdate(metadata, key, holder, deletionInfo, false);
+    }
+
+    /**
+     * Creates an immutable partition update that contains a single row update.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the partition to update.
+     * @param row the row for the update (may be static).
+     *
+     * @return the newly created partition update containing only {@code row}.
+     */
+    public static BTreePartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row)
+    {
+        return singleRowUpdate(metadata, key, row.isStatic() ? null : row, row.isStatic() ? row : null);
+    }
+
+    /**
+     * Creates an immutable partition update that contains a single row update.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the partition to update.
+     * @param row the row for the update.
+     *
+     * @return the newly created partition update containing only {@code row}.
+     */
+    public static BTreePartitionUpdate singleRowUpdate(TableMetadata metadata, ByteBuffer key, Row row)
+    {
+        return singleRowUpdate(metadata, metadata.partitioner.decorateKey(key), row);
+    }
+
+    @Override
+    public PartitionUpdate withOnlyPresentColumns()
+    {
+        Set<ColumnMetadata> columnSet = new HashSet<>();
+
+        for (Row row : rows())
+            for (ColumnData column : row)
+                columnSet.add(column.column());
+
+        RegularAndStaticColumns columns = RegularAndStaticColumns.builder().addAll(columnSet).build();
+        return new BTreePartitionUpdate(metadata, partitionKey, holder.withColumns(columns), deletionInfo.mutableCopy(), false);
+    }
+
+    /**
+     * Turns the given iterator into an update.
+     *
+     * @param iterator the iterator to turn into updates.
+     *
+     * Warning: this method does not close the provided iterator, it is up to
+     * the caller to close it.
+     */
+    @SuppressWarnings("resource")
+    public static BTreePartitionUpdate fromIterator(UnfilteredRowIterator iterator)
+    {
+        BTreePartitionData holder = build(iterator, 16);
+        MutableDeletionInfo deletionInfo = (MutableDeletionInfo) holder.deletionInfo;
+        return new BTreePartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
+    }
+
+    /**
+     * Turns the given iterator into an update.
+     *
+     * @param iterator the iterator to turn into updates.
+     * @param filter the column filter used when querying {@code iterator}. This is used to make
+     * sure we don't include data for which the value has been skipped while reading (as we would
+     * then be writing something incorrect).
+     *
+     * Warning: this method does not close the provided iterator, it is up to
+     * the caller to close it.
+     */
+    @SuppressWarnings("resource")
+    public static BTreePartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter)
+    {
+        return fromIterator(UnfilteredRowIterators.withOnlyQueriedData(iterator, filter));
+    }
+
+    protected boolean canHaveShadowedData()
+    {
+        return canHaveShadowedData;
+    }
+
+    /**
+     * Creates a partition update that entirely deletes a given partition.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the partition that the created update should delete.
+     * @param timestamp the timestamp for the deletion.
+     * @param nowInSec the current time in seconds to use as local deletion time for the partition deletion.
+     *
+     * @return the newly created partition deletion update.
+     */
+    public static BTreePartitionUpdate fullPartitionDelete(TableMetadata metadata, ByteBuffer key, long timestamp, int nowInSec)
+    {
+        return fullPartitionDelete(metadata, metadata.partitioner.decorateKey(key), timestamp, nowInSec);
+    }
+
+    public static BTreePartitionUpdate asBTreeUpdate(PartitionUpdate update)
+    {
+        if (update instanceof BTreePartitionUpdate)
+            return (BTreePartitionUpdate) update;
+
+        try (UnfilteredRowIterator iterator = update.unfilteredIterator())
+        {
+            return fromIterator(iterator);
+        }
+    }
+
+    // We override this, because the version in the super-class calls holder(), which build the update preventing
+    // further updates, but that's not necessary here and being able to check at least the partition deletion without
+    // "locking" the update is nice (and used in DataResolver.RepairMergeListener.MergeListener).
+    @Override
+    public DeletionInfo deletionInfo()
+    {
+        return deletionInfo;
+    }
+
+    /**
+     * The number of "operations" contained in the update.
+     * <p>
+     * This is used by {@code Memtable} to approximate how much work this update does. In practice, this
+     * count how many rows are updated and how many ranges are deleted by the partition update.
+     *
+     * @return the number of "operations" performed by the update.
+     */
+    @Override
+    public int operationCount()
+    {
+        return rowCount()
+             + (staticRow().isEmpty() ? 0 : 1)
+             + deletionInfo.rangeCount()
+             + (deletionInfo.getPartitionDeletion().isLive() ? 0 : 1);
+    }
+
+    /**
+     * The size of the data contained in this update.
+     *
+     * @return the size of the data contained in this update.
+     */
+    @Override
+    public int dataSize()
+    {
+        return Ints.saturatedCast(BTree.<Row>accumulate(holder.tree, (row, value) -> row.dataSize() + value, 0L)
+                + holder.staticRow.dataSize());
+    }
+
+    /**
+     * The size of the data contained in this update.
+     *
+     * @return the size of the data contained in this update.
+     */
+    @Override
+    public long unsharedHeapSize()
+    {
+        return BTree.<Row>accumulate(holder.tree, (row, value) -> row.unsharedHeapSize() + value, 0L)
+               + holder.staticRow.unsharedHeapSize();
+    }
+
+    @Override
+    public TableMetadata metadata()
+    {
+        return metadata;
+    }
+
+    @Override
+    public RegularAndStaticColumns columns()
+    {
+        // The superclass implementation calls holder(), but that triggers a build of the PartitionUpdate. But since
+        // the columns are passed to the ctor, we know the holder always has the proper columns even if it doesn't have
+        // the built rows yet, so just bypass the holder() method.
+        return holder.columns;
+    }
+
+    protected BTreePartitionData holder()
+    {
+        return holder;
+    }
+
+    @Override
+    public EncodingStats stats()
+    {
+        return holder().stats;
+    }
+
+    /**
+     * The maximum timestamp used in this update.
+     *
+     * @return the maximum timestamp used in this update.
+     */
+    @Override
+    public long maxTimestamp()
+    {
+        long maxTimestamp = deletionInfo.maxTimestamp();
+        for (Row row : rows())
+            maxTimestamp = Math.max(maxTimestamp, Rows.collectMaxTimestamp(row));
+
+        if (this.holder.staticRow != null)
+            maxTimestamp = Math.max(maxTimestamp, Rows.collectMaxTimestamp(this.holder.staticRow));
+
+        return maxTimestamp;
+    }
+
+    /**
+     * For an update on a counter table, returns a list containing a {@code CounterMark} for
+     * every counter contained in the update.
+     *
+     * @return a list with counter marks for every counter in this update.
+     */
+    @Override
+    public List<CounterMark> collectCounterMarks()
+    {
+        assert metadata().isCounter();
+        // We will take aliases on the rows of this update, and update them in-place. So we should be sure the
+        // update is now immutable for all intent and purposes.
+        List<CounterMark> marks = new ArrayList<>();
+        addMarksForRow(staticRow(), marks);
+        for (Row row : rows())
+            addMarksForRow(row, marks);
+        return marks;
+    }
+
+    private static void addMarksForRow(Row row, List<CounterMark> marks)
+    {
+        for (Cell<?> cell : row.cells())
+        {
+            if (cell.isCounterCell())
+                marks.add(new CounterMark(row, cell.column(), cell.path()));
+        }
+    }
+
+    @Override
+    public void validateIndexedColumns(ClientState state)
+    {
+        IndexRegistry.obtain(metadata()).validate(this, state);
+    }
+
+    @VisibleForTesting
+    public static BTreePartitionUpdate unsafeConstruct(TableMetadata metadata,
+                                                       DecoratedKey key,
+                                                       BTreePartitionData holder,
+                                                       MutableDeletionInfo deletionInfo,
+                                                       boolean canHaveShadowedData)
+    {
+        return new BTreePartitionUpdate(metadata, key, holder, deletionInfo, canHaveShadowedData);
+    }
+
+    @Override
+    public BTreePartitionUpdate withUpdatedTimestamps(long timestamp)
+    {
+        return new Builder(this, rowCount()).updateAllTimestamp(timestamp).build();
+    }
+
+
+    /**
+     * Builder for PartitionUpdates
+     *
+     * This class is not thread safe, but the PartitionUpdate it produces is (since it is immutable).
+     */
+    public static class Builder implements PartitionUpdate.Builder
+    {
+        private final TableMetadata metadata;
+        private final DecoratedKey key;
+        private final MutableDeletionInfo deletionInfo;
+        private final boolean canHaveShadowedData;
+        private Object[] tree = BTree.empty();
+        private final BTree.Builder<Row> rowBuilder;
+        private Row staticRow = Rows.EMPTY_STATIC_ROW;
+        private final RegularAndStaticColumns columns;
+        private boolean isBuilt = false;
+
+        public Builder(TableMetadata metadata,
+                       DecoratedKey key,
+                       RegularAndStaticColumns columns,
+                       int initialRowCapacity,
+                       boolean canHaveShadowedData)
+        {
+            this(metadata, key, columns, initialRowCapacity, canHaveShadowedData, Rows.EMPTY_STATIC_ROW, MutableDeletionInfo.live(), BTree.empty());
+        }
+
+        private Builder(TableMetadata metadata,
+                       DecoratedKey key,
+                       RegularAndStaticColumns columns,
+                       int initialRowCapacity,
+                       boolean canHaveShadowedData,
+                       BTreePartitionData holder)
+        {
+            this(metadata, key, columns, initialRowCapacity, canHaveShadowedData, holder.staticRow, holder.deletionInfo, holder.tree);
+        }
+
+        private Builder(TableMetadata metadata,
+                        DecoratedKey key,
+                        RegularAndStaticColumns columns,
+                        int initialRowCapacity,
+                        boolean canHaveShadowedData,
+                        Row staticRow,
+                        DeletionInfo deletionInfo,
+                        Object[] tree)
+        {
+            this.metadata = metadata;
+            this.key = key;
+            this.columns = columns;
+            this.rowBuilder = rowBuilder(initialRowCapacity);
+            this.canHaveShadowedData = canHaveShadowedData;
+            this.deletionInfo = deletionInfo.mutableCopy();
+            this.staticRow = staticRow;
+            this.tree = tree;
+        }
+
+        public Builder(TableMetadata metadata, DecoratedKey key, RegularAndStaticColumns columnDefinitions, int size)
+        {
+            this(metadata, key, columnDefinitions, size, true);
+        }
+
+        public Builder(BTreePartitionUpdate base, int initialRowCapacity)
+        {
+            this(base.metadata, base.partitionKey, base.columns(), initialRowCapacity, base.canHaveShadowedData, base.holder);
+        }
+
+        public Builder(TableMetadata metadata,
+                        ByteBuffer key,
+                        RegularAndStaticColumns columns,
+                        int initialRowCapacity)
+        {
+            this(metadata, metadata.partitioner.decorateKey(key), columns, initialRowCapacity, true);
+        }
+
+        /**
+         * Adds a row to this update.
+         *
+         * There is no particular assumption made on the order of row added to a partition update. It is further
+         * allowed to add the same row (more precisely, multiple row objects for the same clustering).
+         *
+         * Note however that the columns contained in the added row must be a subset of the columns used when
+         * creating this update.
+         *
+         * @param row the row to add.
+         */
+        public void add(Row row)
+        {
+            if (row.isEmpty())
+                return;
+
+            if (row.isStatic())
+            {
+                // this assert is expensive, and possibly of limited value; we should consider removing it
+                // or introducing a new class of assertions for test purposes
+                assert columns().statics.containsAll(row.columns()) : columns().statics + " is not superset of " + row.columns();
+                staticRow = staticRow.isEmpty()
+                            ? row
+                            : Rows.merge(staticRow, row);
+            }
+            else
+            {
+                // this assert is expensive, and possibly of limited value; we should consider removing it
+                // or introducing a new class of assertions for test purposes
+                assert columns().regulars.containsAll(row.columns()) : columns().regulars + " is not superset of " + row.columns();
+                rowBuilder.add(row);
+            }
+        }
+
+        public void addPartitionDeletion(DeletionTime deletionTime)
+        {
+            deletionInfo.add(deletionTime);
+        }
+
+        public void add(RangeTombstone range)
+        {
+            deletionInfo.add(range, metadata.comparator);
+        }
+
+        public DecoratedKey partitionKey()
+        {
+            return key;
+        }
+
+        public TableMetadata metadata()
+        {
+            return metadata;
+        }
+
+        public BTreePartitionUpdate build()
+        {
+            // assert that we are not calling build() several times
+            assert !isBuilt : "A PartitionUpdate.Builder should only get built once";
+            Object[] add = rowBuilder.build();
+            Object[] merged = BTree.<Row, Row, Row>update(tree, add, metadata.comparator,
+                                                          UpdateFunction.Simple.of(Rows::merge));
+
+            EncodingStats newStats = EncodingStats.Collector.collect(staticRow, BTree.iterator(merged), deletionInfo);
+
+            isBuilt = true;
+            return new BTreePartitionUpdate(metadata,
+                                            partitionKey(),
+                                            new BTreePartitionData(columns,
+                                                              merged,
+                                                              deletionInfo,
+                                                              staticRow,
+                                                              newStats),
+                                            deletionInfo,
+                                            canHaveShadowedData);
+        }
+
+        public RegularAndStaticColumns columns()
+        {
+            return columns;
+        }
+
+        public DeletionTime partitionLevelDeletion()
+        {
+            return deletionInfo.getPartitionDeletion();
+        }
+
+        private BTree.Builder<Row> rowBuilder(int initialCapacity)
+        {
+            return BTree.<Row>builder(metadata.comparator, initialCapacity)
+                   .setQuickResolver(Rows::merge);
+        }
+        /**
+         * Modify this update to set every timestamp for live data to {@code newTimestamp} and
+         * every deletion timestamp to {@code newTimestamp - 1}.
+         *
+         * There is no reason to use that expect on the Paxos code path, where we need ensure that
+         * anything inserted use the ballot timestamp (to respect the order of update decided by
+         * the Paxos algorithm). We use {@code newTimestamp - 1} for deletions because tombstones
+         * always win on timestamp equality and we don't want to delete our own insertions
+         * (typically, when we overwrite a collection, we first set a complex deletion to delete the
+         * previous collection before adding new elements. If we were to set that complex deletion
+         * to the same timestamp that the new elements, it would delete those elements). And since
+         * tombstones always wins on timestamp equality, using -1 guarantees our deletion will still
+         * delete anything from a previous update.
+         */
+        public Builder updateAllTimestamp(long newTimestamp)
+        {
+            deletionInfo.updateAllTimestamp(newTimestamp - 1);
+            tree = BTree.<Row, Row>transformAndFilter(tree, (x) -> x.updateAllTimestamp(newTimestamp));
+            staticRow = this.staticRow.updateAllTimestamp(newTimestamp);
+            return this;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Builder{" +
+                   "metadata=" + metadata +
+                   ", key=" + key +
+                   ", deletionInfo=" + deletionInfo +
+                   ", canHaveShadowedData=" + canHaveShadowedData +
+                   ", staticRow=" + staticRow +
+                   ", columns=" + columns +
+                   ", isBuilt=" + isBuilt +
+                   '}';
+        }
+
+    }
+
+    public static class BTreeFactory implements PartitionUpdate.Factory
+    {
+
+        @Override
+        public PartitionUpdate.Builder builder(TableMetadata metadata, DecoratedKey partitionKey, RegularAndStaticColumns columns, int initialRowCapacity)
+        {
+            return new Builder(metadata, partitionKey, columns, initialRowCapacity);
+        }
+
+        @Override
+        public PartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey partitionKey)
+        {
+            return BTreePartitionUpdate.emptyUpdate(metadata, partitionKey);
+        }
+
+        @Override
+        public PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey valueKey, Row row)
+        {
+            return BTreePartitionUpdate.singleRowUpdate(metadata, valueKey, row);
+        }
+
+        @Override
+        public PartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, long nowInSec)
+        {
+            return BTreePartitionUpdate.fullPartitionDelete(metadata, key, timestamp, nowInSec);
+        }
+
+        @Override
+        public PartitionUpdate fromIterator(UnfilteredRowIterator iterator)
+        {
+            return BTreePartitionUpdate.fromIterator(iterator);
+        }
+
+        @Override
+        public PartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter)
+        {
+            return BTreePartitionUpdate.fromIterator(iterator, filter);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdater.java b/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdater.java
index 023019242d2c..aa708ab23819 100644
--- a/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdater.java
+++ b/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdater.java
@@ -20,8 +20,6 @@
 
 import org.apache.cassandra.db.DeletionInfo;
 import org.apache.cassandra.db.RegularAndStaticColumns;
-import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.db.rows.ColumnData;
 import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Rows;
@@ -36,32 +34,48 @@
 /**
  *  the function we provide to the trie and btree utilities to perform any row and column replacements
  */
-public class BTreePartitionUpdater implements UpdateFunction<Row, Row>, ColumnData.PostReconciliationFunction
+public class BTreePartitionUpdater extends BasePartitionUpdater implements UpdateFunction<Row, Row>
 {
     final MemtableAllocator allocator;
     final OpOrder.Group writeOp;
-    final Cloner cloner;
     final UpdateTransaction indexer;
-    public long dataSize;
-    long heapSize;
-    public long colUpdateTimeDelta = Long.MAX_VALUE;
+    public int partitionsAdded = 0;
 
     public BTreePartitionUpdater(MemtableAllocator allocator, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
     {
+        super(cloner);
         this.allocator = allocator;
-        this.cloner = cloner;
         this.writeOp = writeOp;
         this.indexer = indexer;
-        this.heapSize = 0;
-        this.dataSize = 0;
     }
 
-    public BTreePartitionData mergePartitions(BTreePartitionData current, final PartitionUpdate update)
+    @Override
+    public Row insert(Row insert)
+    {
+        Row data = insert.clone(cloner);
+        indexer.onInserted(insert);
+
+        this.dataSize += data.dataSize();
+        this.heapSize += data.unsharedHeapSizeExcludingData();
+        return data;
+    }
+
+    @Override
+    public Row merge(Row existing, Row update)
+    {
+        Row reconciled = Rows.merge(existing, update, this);
+        indexer.onUpdated(existing, reconciled);
+
+        return reconciled;
+    }
+
+    public BTreePartitionData mergePartitions(BTreePartitionData current, final BTreePartitionUpdate update)
     {
         if (current == null)
         {
             current = BTreePartitionData.EMPTY;
-            onAllocatedOnHeap(BTreePartitionData.UNSHARED_HEAP_SIZE);
+            this.onAllocatedOnHeap(BTreePartitionData.UNSHARED_HEAP_SIZE);
+            ++partitionsAdded;
         }
 
         try
@@ -77,7 +91,7 @@ public BTreePartitionData mergePartitions(BTreePartitionData current, final Part
         }
     }
 
-    protected BTreePartitionData makeMergedPartition(BTreePartitionData current, PartitionUpdate update)
+    protected BTreePartitionData makeMergedPartition(BTreePartitionData current, BTreePartitionUpdate update)
     {
         DeletionInfo newDeletionInfo = merge(current.deletionInfo, update.deletionInfo());
 
@@ -122,60 +136,6 @@ private DeletionInfo merge(DeletionInfo existing, DeletionInfo update)
         return newInfo;
     }
 
-    @Override
-    public Row insert(Row insert)
-    {
-        Row data = insert.clone(cloner);
-        indexer.onInserted(insert);
-
-        dataSize += data.dataSize();
-        heapSize += data.unsharedHeapSizeExcludingData();
-        return data;
-    }
-
-    public Row merge(Row existing, Row update)
-    {
-        Row reconciled = Rows.merge(existing, update, this);
-        indexer.onUpdated(existing, reconciled);
-
-        return reconciled;
-    }
-
-    public Cell<?> merge(Cell<?> previous, Cell<?> insert)
-    {
-        if (insert == previous)
-            return insert;
-
-        long timeDelta = Math.abs(insert.timestamp() - previous.timestamp());
-        if (timeDelta < colUpdateTimeDelta)
-            colUpdateTimeDelta = timeDelta;
-        if (cloner != null)
-            insert = cloner.clone(insert);
-        dataSize += insert.dataSize() - previous.dataSize();
-        heapSize += insert.unsharedHeapSizeExcludingData() - previous.unsharedHeapSizeExcludingData();
-        return insert;
-    }
-
-    public ColumnData insert(ColumnData insert)
-    {
-        if (cloner != null)
-            insert = insert.clone(cloner);
-        dataSize += insert.dataSize();
-        heapSize += insert.unsharedHeapSizeExcludingData();
-        return insert;
-    }
-
-    @Override
-    public void delete(ColumnData existing)
-    {
-        dataSize -= existing.dataSize();
-        heapSize -= existing.unsharedHeapSizeExcludingData();
-    }
-
-    public void onAllocatedOnHeap(long heapSize)
-    {
-        this.heapSize += heapSize;
-    }
 
     public void reportAllocatedMemory()
     {
diff --git a/src/java/org/apache/cassandra/db/partitions/BasePartitionUpdater.java b/src/java/org/apache/cassandra/db/partitions/BasePartitionUpdater.java
new file mode 100644
index 000000000000..42090bf428d6
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/partitions/BasePartitionUpdater.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.partitions;
+
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.utils.memory.Cloner;
+
+public class BasePartitionUpdater implements ColumnData.PostReconciliationFunction
+{
+    final Cloner cloner;
+    public long dataSize = 0;
+    public long heapSize = 0;
+    public long colUpdateTimeDelta = Long.MAX_VALUE;
+
+    public BasePartitionUpdater(Cloner cloner)
+    {
+        this.cloner = cloner;
+    }
+
+    public Cell<?> merge(Cell<?> previous, Cell<?> insert)
+    {
+        if (insert == previous)
+            return insert;
+        long timeDelta = Math.abs(insert.timestamp() - previous.timestamp());
+        if (timeDelta < colUpdateTimeDelta)
+            colUpdateTimeDelta = timeDelta;
+        if (cloner != null)
+            insert = cloner.clone(insert);
+        dataSize += insert.dataSize() - previous.dataSize();
+        heapSize += insert.unsharedHeapSizeExcludingData() - previous.unsharedHeapSizeExcludingData();
+        return insert;
+    }
+
+    public ColumnData insert(ColumnData insert)
+    {
+        if (cloner != null)
+            insert = insert.clone(cloner);
+        dataSize += insert.dataSize();
+        heapSize += insert.unsharedHeapSizeExcludingData();
+        return insert;
+    }
+
+    public void delete(ColumnData existing)
+    {
+        dataSize -= existing.dataSize();
+        heapSize -= existing.unsharedHeapSizeExcludingData();
+    }
+
+    public void onAllocatedOnHeap(long heapSize)
+    {
+        this.heapSize += heapSize;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java b/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java
index d7a0171d9a20..f44c982c7488 100644
--- a/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java
@@ -43,9 +43,10 @@ public static FilteredPartition create(RowIterator iterator)
         return new FilteredPartition(iterator);
     }
 
+    @Override
     public RowIterator rowIterator()
     {
-        final Iterator<Row> iter = iterator();
+        final Iterator<Row> iter = super.rowIterator();
         return new RowIterator()
         {
             public TableMetadata metadata()
diff --git a/src/java/org/apache/cassandra/db/partitions/Partition.java b/src/java/org/apache/cassandra/db/partitions/Partition.java
index 8888104d95fe..9b6dace1d00e 100644
--- a/src/java/org/apache/cassandra/db/partitions/Partition.java
+++ b/src/java/org/apache/cassandra/db/partitions/Partition.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.db.partitions;
 
+import java.util.Iterator;
 import java.util.NavigableSet;
 
 import javax.annotation.Nullable;
@@ -50,10 +51,36 @@ public interface Partition
     public boolean isEmpty();
 
     /**
-     * Whether the partition object has rows. This may be false but partition still be non-empty if it has a deletion.
+     * Whether the partition object has any rows, excluding the static row.
+     * This may be false but partition still be non-empty if it has a deletion or a non-empty static row.
      */
     boolean hasRows();
 
+    /**
+     * Returns the number of rows in this partition, excluding the static row.
+     */
+    int rowCount();
+
+    /**
+     * Returns an iterator over the rows of this partition excluding the static row.
+     */
+    Iterator<Row> rowIterator();
+
+    /**
+     * Returns the collection of rows of this partition excluding the static row as an iterable.
+     */
+    default Iterable<Row> rows()
+    {
+        return this::rowIterator;
+    }
+
+    Row staticRow();
+
+    /**
+     * Returns the last non-static row in the partition.
+     */
+    Row lastRow();
+
     /**
      * Returns the row corresponding to the provided clustering, or null if there is not such row.
      *
@@ -78,4 +105,37 @@ public interface Partition
      * selected by the provided clusterings.
      */
     public UnfilteredRowIterator unfilteredIterator(ColumnFilter columns, NavigableSet<Clustering<?>> clusteringsInQueryOrder, boolean reversed);
+
+    static String toString(Partition p)
+    {
+        return toString(p, true);
+    }
+
+    static String toString(Partition p, boolean includeFullDetails)
+    {
+        StringBuilder sb = new StringBuilder();
+        if (includeFullDetails)
+        {
+            sb.append(String.format("[%s.%s] key=%s partition_deletion=%s columns=%s",
+                                    p.metadata().keyspace,
+                                    p.metadata().name,
+                                    p.metadata().partitionKeyType.getString(p.partitionKey().getKey()),
+                                    p.partitionLevelDeletion(),
+                                    p.columns()));
+        }
+        else
+        {
+            sb.append("key=").append(p.metadata().partitionKeyType.getString(p.partitionKey().getKey()));
+        }
+
+        if (p.staticRow() != Rows.EMPTY_STATIC_ROW)
+            sb.append("\n    ").append(p.staticRow().toString(p.metadata(), includeFullDetails));
+
+        try (UnfilteredRowIterator iter = p.unfilteredIterator())
+        {
+            while (iter.hasNext())
+                sb.append("\n    ").append(iter.next().toString(p.metadata(), includeFullDetails));
+        }
+        return sb.toString();
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java
index b8a86d5a1aa2..9f81c2409070 100644
--- a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java
+++ b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java
@@ -18,28 +18,48 @@
 package org.apache.cassandra.db.partitions;
 
 import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.transform.MorePartitions;
 import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.AbstractIterator;
 
 import org.apache.cassandra.db.SinglePartitionReadQuery;
 import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.utils.NoSpamLogger;
 
 public abstract class PartitionIterators
 {
+    private static final Logger logger = LoggerFactory.getLogger(PartitionIterators.class);
+
     private PartitionIterators() {}
 
     public static RowIterator getOnlyElement(final PartitionIterator iter, SinglePartitionReadQuery query)
     {
-        // If the query has no results, we'll get an empty iterator, but we still
-        // want a RowIterator out of this method, so we return an empty one.
-        RowIterator toReturn = iter.hasNext()
-                             ? iter.next()
-                             : EmptyIterators.row(query.metadata(),
-                                                  query.partitionKey(),
-                                                  query.clusteringIndexFilter().isReversed());
+        RowIterator toReturn;
+        try
+        {
+            // If the query has no results, we'll get an empty iterator, but we still
+            // want a RowIterator out of this method, so we return an empty one.
+            toReturn = iter.hasNext()
+                       ? iter.next()
+                       : EmptyIterators.row(query.metadata(),
+                                            query.partitionKey(),
+                                            query.clusteringIndexFilter().isReversed());
+        }
+        catch (RuntimeException e)
+        {
+            iter.close();
+            throw e;
+        }
 
         // Note that in general, we should wrap the result so that it's close method actually
         // close the whole PartitionIterator.
@@ -158,6 +178,57 @@ public RowIterator next()
         };
     }
 
+    /**
+     * Wraps the provided iterator to run a specified actions whenever a new partition or row is iterated over.
+     * The resulting iterator is tolerant to the provided actions throwing exceptions.
+     * The actions are allowed to fail and won't stop iteration, but that fact will be logged on ERROR level.
+     *
+     * The wrapper iterators do not delegate Object class methods to the wrapped ones (PartitionIterator and RowIterator)
+     *
+     * @param delegate the iterator to wrap
+     * @param onPartition the action to run when a new partition is iterated over
+     * @param onStaticRow the action to run when the partition has a static row
+     * @param onRow the action to run when a new row is iterated over
+     */
+    public static PartitionIterator filteredRowTrackingIterator(PartitionIterator delegate,
+                                                                Consumer<DecoratedKey> onPartition,
+                                                                Consumer<Row> onStaticRow,
+                                                                Consumer<Row> onRow)
+    {
+        return new PartitionIterator()
+        {
+            public void close()
+            {
+                delegate.close();
+            }
+
+            public boolean hasNext()
+            {
+                return delegate.hasNext();
+            }
+
+            public RowIterator next()
+            {
+                RowIterator next = delegate.next();
+                try
+                {
+                    onPartition.accept(next.partitionKey());
+                    if (!next.staticRow().isEmpty())
+                    {
+                        onStaticRow.accept(next.staticRow());
+                    }
+                }
+                catch (Throwable t)
+                {
+                    NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 60, TimeUnit.SECONDS,
+                                     "Tracking callback for read rows failed on new partition {}", next.partitionKey(), t);
+                }
+                return new RowTrackingIterator(next, onRow);
+            }
+        };
+    }
+
+
     private static class SingletonPartitionIterator extends AbstractIterator<RowIterator> implements PartitionIterator
     {
         private final RowIterator iterator;
@@ -182,4 +253,94 @@ public void close()
             iterator.close();
         }
     }
+
+    private static class RowTrackingIterator implements RowIterator
+    {
+        private final RowIterator delegate;
+        private final Consumer<Row> onRow;
+
+        RowTrackingIterator(RowIterator delegate, Consumer<Row> onRow)
+        {
+            this.delegate = delegate;
+            this.onRow = onRow;
+        }
+
+        @Override
+        public TableMetadata metadata()
+        {
+            return delegate.metadata();
+        }
+
+        @Override
+        public boolean isReverseOrder()
+        {
+            return delegate.isReverseOrder();
+        }
+
+        @Override
+        public RegularAndStaticColumns columns()
+        {
+            return delegate.columns();
+        }
+
+        @Override
+        public DecoratedKey partitionKey()
+        {
+            return delegate.partitionKey();
+        }
+
+        @Override
+        public Row staticRow()
+        {
+            return delegate.staticRow();
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            return delegate.hasNext();
+        }
+
+        @Override
+        public Row next()
+        {
+            Row next = delegate.next();
+            try
+            {
+                onRow.accept(next);
+            }
+            catch (Throwable t)
+            {
+                NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 60, TimeUnit.SECONDS,
+                                 "Tracking callback for read rows failed on row {}", next, t);
+
+            }
+            return next;
+        }
+
+        @Override
+        public void remove()
+        {
+            delegate.remove();
+        }
+
+        @Override
+        public void forEachRemaining(Consumer<? super Row> action)
+        {
+            delegate.forEachRemaining(action);
+        }
+
+        @Override
+        public void close()
+        {
+            delegate.close();
+        }
+
+        @Override
+        public boolean isEmpty()
+        {
+            return delegate.isEmpty();
+        }
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java b/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java
index 7c3ba150aefd..e98642968b0a 100644
--- a/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java
+++ b/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java
@@ -28,4 +28,4 @@ public interface PartitionStatisticsCollector
     void update(Cell<?> cell);
     void updateColumnSetPerRow(long columnSetInRow);
     void updateHasLegacyCounterShards(boolean hasLegacyCounterShards);
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
index 035cb0edd6d1..2b663077de7e 100644
--- a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
+++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
@@ -20,33 +20,45 @@
 import java.io.EOFException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
-import com.google.common.primitives.Ints;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.*;
+import net.openhft.chronicle.core.util.ThrowingFunction;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.SimpleBuilders;
+import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.exceptions.UnknownTableException;
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.utils.btree.BTree;
-import org.apache.cassandra.utils.btree.UpdateFunction;
 import org.apache.cassandra.utils.vint.VIntCoding;
 
 import static org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.IS_EMPTY;
@@ -65,172 +77,134 @@
  * is also a few static helper constructor methods for special cases ({@code emptyUpdate()},
  * {@code fullPartitionDelete} and {@code singleRowUpdate}).
  */
-public class PartitionUpdate extends AbstractBTreePartition
+public interface PartitionUpdate extends Partition
 {
-    protected static final Logger logger = LoggerFactory.getLogger(PartitionUpdate.class);
+    @SuppressWarnings("Convert2MethodRef")
+    public static final PartitionUpdateSerializer serializer = new PartitionUpdateSerializer(tableId -> Schema.instance.getExistingTableMetadata(tableId));
 
-    public static final PartitionUpdateSerializer serializer = new PartitionUpdateSerializer();
+    DeletionInfo deletionInfo();
 
-    private final BTreePartitionData holder;
-    private final DeletionInfo deletionInfo;
-    private final TableMetadata metadata;
+    /**
+     * The number of "operations" contained in the update.
+     * <p>
+     * This is used by {@code Memtable} to approximate how much work this update does. In practice, this
+     * count how many rows are updated and how many ranges are deleted by the partition update.
+     *
+     * @return the number of "operations" performed by the update.
+     */
+    int operationCount();
 
-    private final boolean canHaveShadowedData;
+    /**
+     * The size of the data contained in this update.
+     *
+     * @return the size of the data contained in this update.
+     */
+    int dataSize();
 
-    private PartitionUpdate(TableMetadata metadata,
-                            DecoratedKey key,
-                            BTreePartitionData holder,
-                            MutableDeletionInfo deletionInfo,
-                            boolean canHaveShadowedData)
-    {
-        super(key);
-        this.metadata = metadata;
-        this.holder = holder;
-        this.deletionInfo = deletionInfo;
-        this.canHaveShadowedData = canHaveShadowedData;
-    }
+    long unsharedHeapSize();
+
+    @Override
+    RegularAndStaticColumns columns();
+
+    @Override
+    EncodingStats stats();
+
+    Row staticRow();
+
+    int rowCount();
 
     /**
-     * Creates a empty immutable partition update.
-     *
-     * @param metadata the metadata for the created update.
-     * @param key the partition key for the created update.
+     * Validates the data contained in this update.
      *
-     * @return the newly created empty (and immutable) update.
+     * @throws org.apache.cassandra.serializers.MarshalException if some of the data contained in this update is corrupted.
      */
-    public static PartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey key)
+    default void validate()
     {
-        MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
-        BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
-        return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
+        for (Row row : rows())
+        {
+            metadata().comparator.validate(row.clustering());
+            for (ColumnData cd : row)
+                cd.validate();
+        }
     }
 
     /**
-     * Creates an immutable partition update that entirely deletes a given partition.
-     *
-     * @param metadata the metadata for the created update.
-     * @param key the partition key for the partition that the created update should delete.
-     * @param timestamp the timestamp for the deletion.
-     * @param nowInSec the current time in seconds to use as local deletion time for the partition deletion.
+     * The maximum timestamp used in this update.
      *
-     * @return the newly created partition deletion update.
+     * @return the maximum timestamp used in this update.
      */
-    public static PartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, long nowInSec)
-    {
-        MutableDeletionInfo deletionInfo = new MutableDeletionInfo(timestamp, nowInSec);
-        BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
-        return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
-    }
+    long maxTimestamp();
 
     /**
-     * Creates an immutable partition update that contains a single row update.
-     *
-     * @param metadata the metadata for the created update.
-     * @param key the partition key for the partition to update.
-     * @param row the row for the update (may be null).
-     * @param row the static row for the update (may be null).
+     * For an update on a counter table, returns a list containing a {@code CounterMark} for
+     * every counter contained in the update.
      *
-     * @return the newly created partition update containing only {@code row}.
+     * @return a list with counter marks for every counter in this update.
      */
-    public static PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row, Row staticRow)
+    List<CounterMark> collectCounterMarks();
+
+    default void validateIndexedColumns(ClientState state)
     {
-        MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
-        BTreePartitionData holder = new BTreePartitionData(
-            new RegularAndStaticColumns(
-                staticRow == null ? Columns.NONE : Columns.from(staticRow),
-                row == null ? Columns.NONE : Columns.from(row)
-            ),
-            row == null ? BTree.empty() : BTree.singleton(row),
-            deletionInfo,
-            staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow,
-            EncodingStats.NO_STATS
-        );
-        return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
+        IndexRegistry.obtain(metadata()).validate(this, state);
     }
 
-    /**
-     * Creates an immutable partition update that contains a single row update.
-     *
-     * @param metadata the metadata for the created update.
-     * @param key the partition key for the partition to update.
-     * @param row the row for the update (may be static).
-     *
-     * @return the newly created partition update containing only {@code row}.
-     */
-    public static PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row)
+    PartitionUpdate withUpdatedTimestamps(long timestamp);
+
+    static Builder builder(TableMetadata metadata, DecoratedKey partitionKey, RegularAndStaticColumns columns, int initialRowCapacity)
     {
-        return singleRowUpdate(metadata, key, row.isStatic() ? null : row, row.isStatic() ? row : null);
+        return metadata.partitionUpdateFactory().builder(metadata, partitionKey, columns, initialRowCapacity);
     }
 
-    /**
-     * Creates an immutable partition update that contains a single row update.
-     *
-     * @param metadata the metadata for the created update.
-     * @param key the partition key for the partition to update.
-     * @param row the row for the update.
-     *
-     * @return the newly created partition update containing only {@code row}.
-     */
-    public static PartitionUpdate singleRowUpdate(TableMetadata metadata, ByteBuffer key, Row row)
+    static Builder builder(TableMetadata metadata, ByteBuffer partitionKey, RegularAndStaticColumns columns, int initialRowCapacity)
     {
-        return singleRowUpdate(metadata, metadata.partitioner.decorateKey(key), row);
+        return builder(metadata, metadata.partitioner.decorateKey(partitionKey), columns, initialRowCapacity);
     }
 
-    /**
-     * Turns the given iterator into an update.
-     *
-     * @param iterator the iterator to turn into updates.
-     * @param filter the column filter used when querying {@code iterator}. This is used to make
-     * sure we don't include data for which the value has been skipped while reading (as we would
-     * then be writing something incorrect).
-     *
-     * Warning: this method does not close the provided iterator, it is up to
-     * the caller to close it.
-     */
-    public static PartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter)
+    static PartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey partitionKey)
     {
-        iterator = UnfilteredRowIterators.withOnlyQueriedData(iterator, filter);
-        BTreePartitionData holder = build(iterator, 16);
-        MutableDeletionInfo deletionInfo = (MutableDeletionInfo) holder.deletionInfo;
-        return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
+        return metadata.partitionUpdateFactory().emptyUpdate(metadata, partitionKey);
     }
 
-    /**
-     * Turns the given iterator into an update.
-     *
-     * @param iterator the iterator to turn into updates.
-     * @param filter the column filter used when querying {@code iterator}. This is used to make
-     * sure we don't include data for which the value has been skipped while reading (as we would
-     * then be writing something incorrect).
-     *
-     * Warning: this method does not close the provided iterator, it is up to
-     * the caller to close it.
-     */
-    public static PartitionUpdate fromIterator(RowIterator iterator, ColumnFilter filter)
+    static PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey valueKey, Row row)
     {
-        iterator = RowIterators.withOnlyQueriedData(iterator, filter);
-        MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
-        BTreePartitionData holder = build(iterator, deletionInfo, true);
-        return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
+        return metadata.partitionUpdateFactory().singleRowUpdate(metadata, valueKey, row);
     }
 
+    static PartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, long nowInSec)
+    {
+        return metadata.partitionUpdateFactory().fullPartitionDelete(metadata, key, timestamp, nowInSec);
+    }
 
-    public PartitionUpdate withOnlyPresentColumns()
+    static PartitionUpdate fullPartitionDelete(TableMetadata metadata, ByteBuffer key, long timestamp, long nowInSec)
     {
-        Set<ColumnMetadata> columnSet = new HashSet<>();
+        return fullPartitionDelete(metadata, metadata.partitioner.decorateKey(key), timestamp, nowInSec);
+    }
 
-        for (Row row : this)
-            for (ColumnData column : row)
-                columnSet.add(column.column());
+    static PartitionUpdate fromIterator(UnfilteredRowIterator partition, ColumnFilter filter)
+    {
+        return partition.metadata().partitionUpdateFactory().fromIterator(partition, filter);
+    }
 
-        RegularAndStaticColumns columns = RegularAndStaticColumns.builder().addAll(columnSet).build();
-        return new PartitionUpdate(this.metadata, this.partitionKey, this.holder.withColumns(columns), this.deletionInfo.mutableCopy(), false);
+    static PartitionUpdate merge(List<? extends PartitionUpdate> updates)
+    {
+        assert !updates.isEmpty();
+        return updates.get(0).metadata().partitionUpdateFactory().merge(updates);
     }
 
+    PartitionUpdate withOnlyPresentColumns();
 
-    protected boolean canHaveShadowedData()
+    /**
+     * Creates a new simple partition update builder.
+     *
+     * @param metadata the metadata for the table this is a partition of.
+     * @param partitionKeyValues the values for partition key columns identifying this partition. The values for each
+     * partition key column can be passed either directly as {@code ByteBuffer} or using a "native" value (int for
+     * Int32Type, string for UTF8Type, ...). It is also allowed to pass a single {@code DecoratedKey} value directly.
+     * @return a newly created builder.
+     */
+    static SimpleBuilder simpleBuilder(TableMetadata metadata, Object... partitionKeyValues)
     {
-        return canHaveShadowedData;
+        return new SimpleBuilders.PartitionUpdateBuilder(metadata, partitionKeyValues);
     }
 
     /**
@@ -241,7 +215,8 @@ protected boolean canHaveShadowedData()
      *
      * @return the deserialized update or {@code null} if {@code bytes == null}.
      */
-    public static PartitionUpdate fromBytes(ByteBuffer bytes, int version)
+    @SuppressWarnings("resource")
+    static PartitionUpdate fromBytes(ByteBuffer bytes, int version)
     {
         if (bytes == null)
             return null;
@@ -266,7 +241,7 @@ public static PartitionUpdate fromBytes(ByteBuffer bytes, int version)
      *
      * @return a newly allocated byte buffer containing the serialized update.
      */
-    public static ByteBuffer toBytes(PartitionUpdate update, int version)
+    static ByteBuffer toBytes(PartitionUpdate update, int version)
     {
         try (DataOutputBuffer out = new DataOutputBuffer())
         {
@@ -280,195 +255,10 @@ public static ByteBuffer toBytes(PartitionUpdate update, int version)
     }
 
     /**
-     * Creates a partition update that entirely deletes a given partition.
-     *
-     * @param metadata the metadata for the created update.
-     * @param key the partition key for the partition that the created update should delete.
-     * @param timestamp the timestamp for the deletion.
-     * @param nowInSec the current time in seconds to use as local deletion time for the partition deletion.
-     *
-     * @return the newly created partition deletion update.
-     */
-    public static PartitionUpdate fullPartitionDelete(TableMetadata metadata, ByteBuffer key, long timestamp, long nowInSec)
-    {
-        return fullPartitionDelete(metadata, metadata.partitioner.decorateKey(key), timestamp, nowInSec);
-    }
-
-    /**
-     * Merges the provided updates, yielding a new update that incorporates all those updates.
-     *
-     * @param updates the collection of updates to merge. This shouldn't be empty.
-     *
-     * @return a partition update that include (merge) all the updates from {@code updates}.
-     */
-    public static PartitionUpdate merge(List<PartitionUpdate> updates)
-    {
-        assert !updates.isEmpty();
-        final int size = updates.size();
-
-        if (size == 1)
-            return Iterables.getOnlyElement(updates);
-
-        List<UnfilteredRowIterator> asIterators = Lists.transform(updates, AbstractBTreePartition::unfilteredIterator);
-        return fromIterator(UnfilteredRowIterators.merge(asIterators), ColumnFilter.all(updates.get(0).metadata()));
-    }
-
-    // We override this, because the version in the super-class calls holder(), which build the update preventing
-    // further updates, but that's not necessary here and being able to check at least the partition deletion without
-    // "locking" the update is nice (and used in DataResolver.RepairMergeListener.MergeListener).
-    @Override
-    public DeletionInfo deletionInfo()
-    {
-        return deletionInfo;
-    }
-
-    /**
-     * The number of "operations" contained in the update.
-     * <p>
-     * This is used by {@code Memtable} to approximate how much work this update does. In practice, this
-     * count how many rows are updated and how many ranges are deleted by the partition update.
-     *
-     * @return the number of "operations" performed by the update.
-     */
-    public int operationCount()
-    {
-        return rowCount()
-             + (staticRow().isEmpty() ? 0 : 1)
-             + deletionInfo.rangeCount()
-             + (deletionInfo.getPartitionDeletion().isLive() ? 0 : 1);
-    }
-
-    /**
-     * The size of the data contained in this update.
-     *
-     * @return the size of the data contained in this update.
-     */
-    public int dataSize()
-    {
-        return Ints.saturatedCast(BTree.<Row>accumulate(holder.tree, (row, value) -> row.dataSize() + value, 0L)
-                + holder.staticRow.dataSize() + holder.deletionInfo.dataSize());
-    }
-
-    /**
-     * The size of the data contained in this update.
-     *
-     * @return the size of the data contained in this update.
-     */
-    public long unsharedHeapSize()
-    {
-        return BTree.<Row>accumulate(holder.tree, (row, value) -> row.unsharedHeapSize() + value, 0L)
-                + holder.staticRow.unsharedHeapSize() + holder.deletionInfo.unsharedHeapSize();
-    }
-
-    public TableMetadata metadata()
-    {
-        return metadata;
-    }
-
-    @Override
-    public RegularAndStaticColumns columns()
-    {
-        // The superclass implementation calls holder(), but that triggers a build of the PartitionUpdate. But since
-        // the columns are passed to the ctor, we know the holder always has the proper columns even if it doesn't have
-        // the built rows yet, so just bypass the holder() method.
-        return holder.columns;
-    }
-
-    protected BTreePartitionData holder()
-    {
-        return holder;
-    }
-
-    public EncodingStats stats()
-    {
-        return holder().stats;
-    }
-
-    /**
-     * Validates the data contained in this update.
-     *
-     * @throws org.apache.cassandra.serializers.MarshalException if some of the data contained in this update is corrupted.
-     */
-    public void validate()
-    {
-        for (Row row : this)
-        {
-            metadata().comparator.validate(row.clustering());
-            for (ColumnData cd : row)
-                cd.validate();
-        }
-    }
-
-    /**
-     * The maximum timestamp used in this update.
      *
-     * @return the maximum timestamp used in this update.
+     * @return the estimated number of rows affected by this mutation
      */
-    public long maxTimestamp()
-    {
-        long maxTimestamp = deletionInfo.maxTimestamp();
-        for (Row row : this)
-        {
-            maxTimestamp = Math.max(maxTimestamp, row.primaryKeyLivenessInfo().timestamp());
-            for (ColumnData cd : row)
-            {
-                if (cd.column().isSimple())
-                {
-                    maxTimestamp = Math.max(maxTimestamp, ((Cell<?>)cd).timestamp());
-                }
-                else
-                {
-                    ComplexColumnData complexData = (ComplexColumnData)cd;
-                    maxTimestamp = Math.max(maxTimestamp, complexData.complexDeletion().markedForDeleteAt());
-                    for (Cell<?> cell : complexData)
-                        maxTimestamp = Math.max(maxTimestamp, cell.timestamp());
-                }
-            }
-        }
-
-        if (this.holder.staticRow != null)
-        {
-            for (ColumnData cd : this.holder.staticRow.columnData())
-            {
-                if (cd.column().isSimple())
-                {
-                    maxTimestamp = Math.max(maxTimestamp, ((Cell<?>) cd).timestamp());
-                }
-                else
-                {
-                    ComplexColumnData complexData = (ComplexColumnData) cd;
-                    maxTimestamp = Math.max(maxTimestamp, complexData.complexDeletion().markedForDeleteAt());
-                    for (Cell<?> cell : complexData)
-                        maxTimestamp = Math.max(maxTimestamp, cell.timestamp());
-                }
-            }
-        }
-        return maxTimestamp;
-    }
-
-    /**
-     * For an update on a counter table, returns a list containing a {@code CounterMark} for
-     * every counter contained in the update.
-     *
-     * @return a list with counter marks for every counter in this update.
-     */
-    public List<CounterMark> collectCounterMarks()
-    {
-        assert metadata().isCounter();
-        // We will take aliases on the rows of this update, and update them in-place. So we should be sure the
-        // update is now immutable for all intent and purposes.
-        List<CounterMark> marks = new ArrayList<>();
-        addMarksForRow(staticRow(), marks);
-        for (Row row : this)
-            addMarksForRow(row, marks);
-        return marks;
-    }
-
-    /**
-     *
-     * @return the estimated number of rows affected by this mutation 
-     */
-    public int affectedRowCount()
+    default int affectedRowCount()
     {
         // If there is a partition-level deletion, we intend to delete at least one row.
         if (!partitionLevelDeletion().isLive())
@@ -492,7 +282,7 @@ public int affectedRowCount()
      *
      * @return the estimated total number of columns that either have live data or are covered by a delete
      */
-    public int affectedColumnCount()
+    default int affectedColumnCount()
     {
         // If there is a partition-level deletion, we intend to delete at least the columns of one row.
         if (!partitionLevelDeletion().isLive())
@@ -504,10 +294,10 @@ public int affectedColumnCount()
         if (deletionInfo().hasRanges())
             count += deletionInfo().rangeCount() * metadata().regularColumns().size();
 
-        for (Row row : this)
+        for (Row row : rows())
         {
             if (row.deletion().isLive())
-                // If the row is live, this will include simple tombstones as well as cells w/ actual data. 
+                // If the row is live, this will include simple tombstones as well as cells w/ actual data.
                 count += row.columnCount();
             else
                 // We have a row deletion, so account for the columns that might be deleted.
@@ -520,44 +310,6 @@ public int affectedColumnCount()
         return count;
     }
 
-    private static void addMarksForRow(Row row, List<CounterMark> marks)
-    {
-        for (Cell<?> cell : row.cells())
-        {
-            if (cell.isCounterCell())
-                marks.add(new CounterMark(row, cell.column(), cell.path()));
-        }
-    }
-
-    /**
-     * Creates a new simple partition update builder.
-     *
-     * @param metadata the metadata for the table this is a partition of.
-     * @param partitionKeyValues the values for partition key columns identifying this partition. The values for each
-     * partition key column can be passed either directly as {@code ByteBuffer} or using a "native" value (int for
-     * Int32Type, string for UTF8Type, ...). It is also allowed to pass a single {@code DecoratedKey} value directly.
-     * @return a newly created builder.
-     */
-    public static SimpleBuilder simpleBuilder(TableMetadata metadata, Object... partitionKeyValues)
-    {
-        return new SimpleBuilders.PartitionUpdateBuilder(metadata, partitionKeyValues);
-    }
-
-    public void validateIndexedColumns(ClientState state)
-    {
-        IndexRegistry.obtain(metadata()).validate(this, state);
-    }
-
-    @VisibleForTesting
-    public static PartitionUpdate unsafeConstruct(TableMetadata metadata,
-                                                  DecoratedKey key,
-                                                  BTreePartitionData holder,
-                                                  MutableDeletionInfo deletionInfo,
-                                                  boolean canHaveShadowedData)
-    {
-        return new PartitionUpdate(metadata, key, holder, deletionInfo, canHaveShadowedData);
-    }
-
     /**
      * Interface for building partition updates geared towards human.
      * <p>
@@ -705,51 +457,47 @@ public interface RangeTombstoneBuilder
         }
     }
 
-    public static class PartitionUpdateSerializer
+    class PartitionUpdateSerializer
     {
+        private final ThrowingFunction<? super TableId, ? extends TableMetadata, ? extends UnknownTableException> tableMetadataResolver;
+
+        public PartitionUpdateSerializer(ThrowingFunction<? super TableId, ? extends TableMetadata, ? extends UnknownTableException> tableMetadataResolver)
+        {
+            this.tableMetadataResolver = tableMetadataResolver;
+        }
+
         public void serialize(PartitionUpdate update, DataOutputPlus out, int version) throws IOException
         {
+            Preconditions.checkArgument(version != MessagingService.VERSION_DSE_68,
+                                        "Can't serialize to version " + version);
             try (UnfilteredRowIterator iter = update.unfilteredIterator())
             {
                 assert !iter.isReverseOrder();
 
-                update.metadata.id.serialize(out);
+                update.metadata().id.serialize(out);
                 UnfilteredRowIteratorSerializer.serializer.serialize(iter, null, out, version, update.rowCount());
             }
         }
 
         public PartitionUpdate deserialize(DataInputPlus in, int version, DeserializationHelper.Flag flag) throws IOException
         {
-            TableMetadata metadata = Schema.instance.getExistingTableMetadata(TableId.deserialize(in));
+            TableMetadata metadata = tableMetadataResolver.apply(TableId.deserialize(in));
+            if (version == MessagingService.VERSION_DSE_68)
+            {
+                // ignore maxTimestamp
+                in.readLong();
+            }
+            Factory factory = metadata.partitionUpdateFactory();
             UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(metadata, null, in, version, flag);
             if (header.isEmpty)
-                return emptyUpdate(metadata, header.key);
+                return factory.emptyUpdate(metadata, header.key);
 
             assert !header.isReversed;
             assert header.rowEstimate >= 0;
-
-            MutableDeletionInfo.Builder deletionBuilder = MutableDeletionInfo.builder(header.partitionDeletion, metadata.comparator, false);
-            Object[] rows;
-            try (BTree.FastBuilder<Row> builder = BTree.fastBuilder();
-                 UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, metadata, flag, header))
+            try (UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, metadata, flag, header))
             {
-                while (partition.hasNext())
-                {
-                    Unfiltered unfiltered = partition.next();
-                    if (unfiltered.kind() == Unfiltered.Kind.ROW)
-                        builder.add((Row)unfiltered);
-                    else
-                        deletionBuilder.add((RangeTombstoneMarker)unfiltered);
-                }
-                rows = builder.build();
+                return factory.fromIterator(partition);
             }
-
-            MutableDeletionInfo deletionInfo = deletionBuilder.build();
-            return new PartitionUpdate(metadata,
-                                       header.key,
-                                       new BTreePartitionData(header.sHeader.columns(), rows, deletionInfo, header.staticRow, header.sHeader.stats()),
-                                       deletionInfo,
-                                       false);
         }
 
         public static boolean isEmpty(ByteBuffer in, DeserializationHelper.Flag flag, DecoratedKey key) throws IOException
@@ -771,8 +519,9 @@ public long serializedSize(PartitionUpdate update, int version)
         {
             try (UnfilteredRowIterator iter = update.unfilteredIterator())
             {
-                return update.metadata.id.serializedSize()
-                     + UnfilteredRowIteratorSerializer.serializer.serializedSize(iter, null, version, update.rowCount());
+                return update.metadata().id.serializedSize()
+                       + (version == MessagingService.VERSION_DSE_68 ? TypeSizes.LONG_SIZE : 0)
+                       + UnfilteredRowIteratorSerializer.serializer.serializedSize(iter, null, version, update.rowCount());
             }
         }
     }
@@ -782,13 +531,13 @@ public long serializedSize(PartitionUpdate update, int version)
      * us to update the counter value based on the pre-existing value read during the read-before-write that counters
      * do. See {@link CounterMutation} to understand how this is used.
      */
-    public static class CounterMark
+    class CounterMark
     {
         private final Row row;
         private final ColumnMetadata column;
         private final CellPath path;
 
-        private CounterMark(Row row, ColumnMetadata column, CellPath path)
+        protected CounterMark(Row row, ColumnMetadata column, CellPath path)
         {
             this.row = row;
             this.column = column;
@@ -831,74 +580,8 @@ public void setValue(ByteBuffer value)
      *
      * This class is not thread safe, but the PartitionUpdate it produces is (since it is immutable).
      */
-    public static class Builder
+    interface Builder
     {
-        private final TableMetadata metadata;
-        private final DecoratedKey key;
-        private final MutableDeletionInfo deletionInfo;
-        private final boolean canHaveShadowedData;
-        private Object[] tree = BTree.empty();
-        private final BTree.Builder<Row> rowBuilder;
-        private Row staticRow = Rows.EMPTY_STATIC_ROW;
-        private final RegularAndStaticColumns columns;
-        private boolean isBuilt = false;
-
-        public Builder(TableMetadata metadata,
-                       DecoratedKey key,
-                       RegularAndStaticColumns columns,
-                       int initialRowCapacity,
-                       boolean canHaveShadowedData)
-        {
-            this(metadata, key, columns, initialRowCapacity, canHaveShadowedData, Rows.EMPTY_STATIC_ROW, MutableDeletionInfo.live(), BTree.empty());
-        }
-
-        private Builder(TableMetadata metadata,
-                       DecoratedKey key,
-                       RegularAndStaticColumns columns,
-                       int initialRowCapacity,
-                       boolean canHaveShadowedData,
-                       BTreePartitionData holder)
-        {
-            this(metadata, key, columns, initialRowCapacity, canHaveShadowedData, holder.staticRow, holder.deletionInfo, holder.tree);
-        }
-
-        private Builder(TableMetadata metadata,
-                        DecoratedKey key,
-                        RegularAndStaticColumns columns,
-                        int initialRowCapacity,
-                        boolean canHaveShadowedData,
-                        Row staticRow,
-                        DeletionInfo deletionInfo,
-                        Object[] tree)
-        {
-            this.metadata = metadata;
-            this.key = key;
-            this.columns = columns;
-            this.rowBuilder = rowBuilder(initialRowCapacity);
-            this.canHaveShadowedData = canHaveShadowedData;
-            this.deletionInfo = deletionInfo.mutableCopy();
-            this.staticRow = staticRow;
-            this.tree = tree;
-        }
-
-        public Builder(TableMetadata metadata, DecoratedKey key, RegularAndStaticColumns columnDefinitions, int size)
-        {
-            this(metadata, key, columnDefinitions, size, true);
-        }
-
-        public Builder(PartitionUpdate base, int initialRowCapacity)
-        {
-            this(base.metadata, base.partitionKey, base.columns(), initialRowCapacity, base.canHaveShadowedData, base.holder);
-        }
-
-        public Builder(TableMetadata metadata,
-                        ByteBuffer key,
-                        RegularAndStaticColumns columns,
-                        int initialRowCapacity)
-        {
-            this(metadata, metadata.partitioner.decorateKey(key), columns, initialRowCapacity, true);
-        }
-
         /**
          * Adds a row to this update.
          *
@@ -910,121 +593,49 @@ public Builder(TableMetadata metadata,
          *
          * @param row the row to add.
          */
-        public void add(Row row)
-        {
-            if (row.isEmpty())
-                return;
+        void add(Row row);
 
-            if (row.isStatic())
-            {
-                // this assert is expensive, and possibly of limited value; we should consider removing it
-                // or introducing a new class of assertions for test purposes
-                assert columns().statics.containsAll(row.columns()) : columns().statics + " is not superset of " + row.columns();
-                staticRow = staticRow.isEmpty()
-                            ? row
-                            : Rows.merge(staticRow, row);
-            }
-            else
-            {
-                // this assert is expensive, and possibly of limited value; we should consider removing it
-                // or introducing a new class of assertions for test purposes
-                assert columns().regulars.containsAll(row.columns()) : columns().regulars + " is not superset of " + row.columns();
-                rowBuilder.add(row);
-            }
-        }
+        void addPartitionDeletion(DeletionTime deletionTime);
 
-        public void addPartitionDeletion(DeletionTime deletionTime)
-        {
-            deletionInfo.add(deletionTime);
-        }
+        void add(RangeTombstone range);
 
-        public void add(RangeTombstone range)
-        {
-            deletionInfo.add(range, metadata.comparator);
-        }
+        DecoratedKey partitionKey();
 
-        public DecoratedKey partitionKey()
-        {
-            return key;
-        }
+        TableMetadata metadata();
 
-        public TableMetadata metadata()
-        {
-            return metadata;
-        }
+        PartitionUpdate build();
 
-        public PartitionUpdate build()
-        {
-            // assert that we are not calling build() several times
-            assert !isBuilt : "A PartitionUpdate.Builder should only get built once";
-            Object[] add = rowBuilder.build();
-            Object[] merged = BTree.<Row, Row, Row>update(tree, add, metadata.comparator,
-                                                          UpdateFunction.Simple.of(Rows::merge));
-
-            EncodingStats newStats = EncodingStats.Collector.collect(staticRow, BTree.iterator(merged), deletionInfo);
-
-            isBuilt = true;
-            return new PartitionUpdate(metadata,
-                                       partitionKey(),
-                                       new BTreePartitionData(columns,
-                                                              merged,
-                                                              deletionInfo,
-                                                              staticRow,
-                                                              newStats),
-                                       deletionInfo,
-                                       canHaveShadowedData);
-        }
+        RegularAndStaticColumns columns();
 
-        public RegularAndStaticColumns columns()
-        {
-            return columns;
-        }
+        DeletionTime partitionLevelDeletion();
+    }
 
-        public DeletionTime partitionLevelDeletion()
-        {
-            return deletionInfo.getPartitionDeletion();
-        }
+    interface Factory
+    {
+        Builder builder(TableMetadata metadata, DecoratedKey partitionKey, RegularAndStaticColumns columns, int initialRowCapacity);
+        PartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey partitionKey);
+        PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey valueKey, Row row);
+        PartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, long nowInSec);
+        PartitionUpdate fromIterator(UnfilteredRowIterator iterator);
+        PartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter);
 
-        private BTree.Builder<Row> rowBuilder(int initialCapacity)
-        {
-            return BTree.<Row>builder(metadata.comparator, initialCapacity)
-                   .setQuickResolver(Rows::merge);
-        }
         /**
-         * Modify this update to set every timestamp for live data to {@code newTimestamp} and
-         * every deletion timestamp to {@code newTimestamp - 1}.
-         *
-         * There is no reason to use that expect on the Paxos code path, where we need ensure that
-         * anything inserted use the ballot timestamp (to respect the order of update decided by
-         * the Paxos algorithm). We use {@code newTimestamp - 1} for deletions because tombstones
-         * always win on timestamp equality and we don't want to delete our own insertions
-         * (typically, when we overwrite a collection, we first set a complex deletion to delete the
-         * previous collection before adding new elements. If we were to set that complex deletion
-         * to the same timestamp that the new elements, it would delete those elements). And since
-         * tombstones always wins on timestamp equality, using -1 guarantees our deletion will still
-         * delete anything from a previous update.
+         * Merge the provided updates into a single update. The method must also work (possibly inefficiently) when the
+         * given updates do not match the type of this factory.
          */
-        public Builder updateAllTimestamp(long newTimestamp)
+        default PartitionUpdate merge(List<? extends PartitionUpdate> updates)
         {
-            deletionInfo.updateAllTimestamp(newTimestamp - 1);
-            tree = BTree.<Row, Row>transformAndFilter(tree, (x) -> x.updateAllTimestamp(newTimestamp));
-            staticRow = this.staticRow.updateAllTimestamp(newTimestamp);
-            return this;
-        }
+            assert !updates.isEmpty();
+            final int size = updates.size();
 
-        @Override
-        public String toString()
-        {
-            return "Builder{" +
-                   "metadata=" + metadata +
-                   ", key=" + key +
-                   ", deletionInfo=" + deletionInfo +
-                   ", canHaveShadowedData=" + canHaveShadowedData +
-                   ", staticRow=" + staticRow +
-                   ", columns=" + columns +
-                   ", isBuilt=" + isBuilt +
-                   '}';
-        }
+            if (size == 1)
+                return Iterables.getOnlyElement(updates);
 
+            List<UnfilteredRowIterator> asIterators = Lists.transform(updates, Partition::unfilteredIterator);
+            try (UnfilteredRowIterator merge = UnfilteredRowIterators.merge(asIterators))
+            {
+                return fromIterator(merge);
+            }
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/db/partitions/TrieBackedPartition.java b/src/java/org/apache/cassandra/db/partitions/TrieBackedPartition.java
new file mode 100644
index 000000000000..ad0c75b8662d
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/partitions/TrieBackedPartition.java
@@ -0,0 +1,709 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.partitions;
+
+import java.util.Iterator;
+import java.util.NavigableSet;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.MutableDeletionInfo;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowAndDeletionMergeIterator;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.db.tries.Direction;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.db.tries.TrieEntriesIterator;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.btree.BTree;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.memory.Cloner;
+import org.apache.cassandra.utils.memory.EnsureOnHeap;
+
+/**
+ * In-memory partition backed by a trie. The rows of the partition are values in the leaves of the trie, where the key
+ * to the row is only stored as the path to reach that leaf; static rows are also treated as a row with STATIC_CLUSTERING
+ * path; the deletion information is placed as a metadata object at the root of the trie -- this matches how Memtable
+ * stores partitions within the larger map, so that TrieBackedPartition objects can be created directly from Memtable
+ * tail tries.
+ *
+ * This object also holds the partition key, as well as some metadata (columns and statistics).
+ *
+ * Currently all descendants and instances of this class are immutable (even tail tries from mutable memtables are
+ * guaranteed to not change as we use forced copying below the partition level), though this may change in the future.
+ */
+public class TrieBackedPartition implements Partition
+{
+    /**
+     * If keys are below this length, we will use a recursive procedure for inserting data when building the backing
+     * trie.
+     */
+    @VisibleForTesting
+    public static final int MAX_RECURSIVE_KEY_LENGTH = 128;
+
+    public static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS50;
+
+    /** Pre-made path for STATIC_CLUSTERING, to avoid creating path object when querying static path. */
+    public static final ByteComparable STATIC_CLUSTERING_PATH = v -> ByteSource.oneByte(ClusteringPrefix.Kind.STATIC_CLUSTERING.asByteComparableValue(v));
+    /** Pre-made path for BOTTOM, to avoid creating path object when iterating rows. */
+    public static final ByteComparable BOTTOM_PATH = v -> ByteSource.oneByte(ClusteringPrefix.Kind.INCL_START_BOUND.asByteComparableValue(v));
+
+    /**
+     * The representation of a row stored at the leaf of a trie. Does not contain the row key.
+     *
+     * The methods toRow and copyToOnHeapRow combine this with a clustering for the represented Row.
+     */
+    public static class RowData
+    {
+        final Object[] columnsBTree;
+        final LivenessInfo livenessInfo;
+        final DeletionTime deletion;
+        final long minLocalDeletionTime;
+
+        RowData(Object[] columnsBTree, LivenessInfo livenessInfo, DeletionTime deletion)
+        {
+            this(columnsBTree, livenessInfo, deletion, BTreeRow.minDeletionTime(columnsBTree, livenessInfo, deletion));
+        }
+
+        RowData(Object[] columnsBTree, LivenessInfo livenessInfo, DeletionTime deletion, long minLocalDeletionTime)
+        {
+            this.columnsBTree = columnsBTree;
+            this.livenessInfo = livenessInfo;
+            this.deletion = deletion;
+            this.minLocalDeletionTime = minLocalDeletionTime;
+        }
+
+        Row toRow(Clustering<?> clustering)
+        {
+            return BTreeRow.create(clustering,
+                                   livenessInfo,
+                                   Row.Deletion.regular(deletion),
+                                   columnsBTree,
+                                   minLocalDeletionTime);
+        }
+
+        public int dataSize()
+        {
+            int dataSize = livenessInfo.dataSize() + deletion.dataSize();
+
+            return Ints.checkedCast(BTree.accumulate(columnsBTree, (ColumnData cd, long v) -> v + cd.dataSize(), dataSize));
+        }
+
+        public long unsharedHeapSizeExcludingData()
+        {
+            long heapSize = EMPTY_ROWDATA_SIZE
+                            + BTree.sizeOfStructureOnHeap(columnsBTree)
+                            + livenessInfo.unsharedHeapSize()
+                            + deletion.unsharedHeapSize();
+
+            return BTree.accumulate(columnsBTree, (ColumnData cd, long v) -> v + cd.unsharedHeapSizeExcludingData(), heapSize);
+        }
+
+        public String toString()
+        {
+            return "row " + livenessInfo + " size " + dataSize();
+        }
+
+        public RowData clone(Cloner cloner)
+        {
+            Object[] tree = BTree.<ColumnData, ColumnData>transform(columnsBTree, c -> c.clone(cloner));
+            return new RowData(tree, livenessInfo, deletion, minLocalDeletionTime);
+        }
+    }
+
+    private static final long EMPTY_ROWDATA_SIZE = ObjectSizes.measure(new RowData(null, null, null, 0));
+
+    protected final Trie<Object> trie;
+    protected final DecoratedKey partitionKey;
+    protected final TableMetadata metadata;
+    protected final RegularAndStaticColumns columns;
+    protected final EncodingStats stats;
+    protected final int rowCountIncludingStatic;
+    protected final boolean canHaveShadowedData;
+
+    public TrieBackedPartition(DecoratedKey partitionKey,
+                               RegularAndStaticColumns columns,
+                               EncodingStats stats,
+                               int rowCountIncludingStatic,
+                               Trie<Object> trie,
+                               TableMetadata metadata,
+                               boolean canHaveShadowedData)
+    {
+        this.partitionKey = partitionKey;
+        this.trie = trie;
+        this.metadata = metadata;
+        this.columns = columns;
+        this.stats = stats;
+        this.rowCountIncludingStatic = rowCountIncludingStatic;
+        this.canHaveShadowedData = canHaveShadowedData;
+        // There must always be deletion info metadata.
+        // Note: we can't use deletionInfo() because WithEnsureOnHeap's override is not yet set up.
+        assert trie.get(ByteComparable.EMPTY) != null;
+        assert stats != null;
+    }
+
+    public static TrieBackedPartition fromIterator(UnfilteredRowIterator iterator)
+    {
+        ContentBuilder builder = build(iterator, false);
+        return new TrieBackedPartition(iterator.partitionKey(),
+                                       iterator.columns(),
+                                       iterator.stats(),
+                                       builder.rowCountIncludingStatic(),
+                                       builder.trie(),
+                                       iterator.metadata(),
+                                       false);
+    }
+
+    protected static ContentBuilder build(UnfilteredRowIterator iterator, boolean collectDataSize)
+    {
+        try
+        {
+            ContentBuilder builder = new ContentBuilder(iterator.metadata(), iterator.partitionLevelDeletion(), iterator.isReverseOrder(), collectDataSize);
+
+            builder.addStatic(iterator.staticRow());
+
+            while (iterator.hasNext())
+                builder.addUnfiltered(iterator.next());
+
+            return builder.complete();
+        }
+        catch (TrieSpaceExhaustedException e)
+        {
+            throw new AssertionError(e);
+        }
+    }
+
+    /**
+     * Create a row with the given properties and content, making sure to copy all off-heap data to keep it alive when
+     * the given access mode requires it.
+     */
+    public static TrieBackedPartition create(DecoratedKey partitionKey,
+                                             RegularAndStaticColumns columnMetadata,
+                                             EncodingStats encodingStats,
+                                             int rowCountIncludingStatic,
+                                             Trie<Object> trie,
+                                             TableMetadata metadata,
+                                             EnsureOnHeap ensureOnHeap)
+    {
+        return ensureOnHeap == EnsureOnHeap.NOOP
+               ? new TrieBackedPartition(partitionKey, columnMetadata, encodingStats, rowCountIncludingStatic, trie, metadata, true)
+               : new WithEnsureOnHeap(partitionKey, columnMetadata, encodingStats, rowCountIncludingStatic, trie, metadata, true, ensureOnHeap);
+    }
+
+    class RowIterator extends TrieEntriesIterator<Object, Row>
+    {
+        public RowIterator(Trie<Object> trie, Direction direction)
+        {
+            super(trie, direction, RowData.class::isInstance);
+        }
+
+        @Override
+        protected Row mapContent(Object content, byte[] bytes, int byteLength)
+        {
+            var rd = (RowData) content;
+            return toRow(rd,
+                         metadata.comparator.clusteringFromByteComparable(
+                             ByteBufferAccessor.instance,
+                             ByteComparable.preencoded(BYTE_COMPARABLE_VERSION, bytes, 0, byteLength)));
+        }
+    }
+
+    private Iterator<Row> rowIterator(Trie<Object> trie, Direction direction)
+    {
+        return new RowIterator(trie, direction);
+    }
+
+    static RowData rowToData(Row row)
+    {
+        BTreeRow brow = (BTreeRow) row;
+        return new RowData(brow.getBTree(), row.primaryKeyLivenessInfo(), row.deletion().time(), brow.getMinLocalDeletionTime());
+    }
+
+    /**
+     * Conversion from RowData to Row. TrieBackedPartitionOnHeap overrides this to do the necessary copying
+     * (hence the non-static method).
+     */
+    Row toRow(RowData data, Clustering clustering)
+    {
+        return data.toRow(clustering);
+    }
+
+    /**
+     * Put the given unfiltered in the trie.
+     * @param comparator for converting key to byte-comparable
+     * @param useRecursive whether the key length is guaranteed short and recursive put can be used
+     * @param trie destination
+     * @param row content to put
+     */
+    protected static void putInTrie(ClusteringComparator comparator, boolean useRecursive, InMemoryTrie<Object> trie, Row row) throws TrieSpaceExhaustedException
+    {
+        trie.putSingleton(comparator.asByteComparable(row.clustering()), rowToData(row), NO_CONFLICT_RESOLVER, useRecursive);
+    }
+
+    /**
+     * Check if we can use recursive operations when putting a value in tries.
+     * True if all types in the clustering keys are fixed length, and total size is small enough.
+     */
+    protected static boolean useRecursive(ClusteringComparator comparator)
+    {
+        int length = 1; // terminator
+        for (AbstractType type : comparator.subtypes())
+            if (!type.isValueLengthFixed())
+                return false;
+            else
+                length += 1 + type.valueLengthIfFixed();    // separator + value
+
+        return length <= MAX_RECURSIVE_KEY_LENGTH;
+    }
+
+    public TableMetadata metadata()
+    {
+        return metadata;
+    }
+
+    public DecoratedKey partitionKey()
+    {
+        return partitionKey;
+    }
+
+    public DeletionTime partitionLevelDeletion()
+    {
+        return deletionInfo().getPartitionDeletion();
+    }
+
+    public RegularAndStaticColumns columns()
+    {
+        return columns;
+    }
+
+    public EncodingStats stats()
+    {
+        return stats;
+    }
+
+    public int rowCount()
+    {
+        return rowCountIncludingStatic - (hasStaticRow() ? 1 : 0);
+    }
+
+    public DeletionInfo deletionInfo()
+    {
+        return (DeletionInfo) trie.get(ByteComparable.EMPTY);
+    }
+
+    public ByteComparable path(ClusteringPrefix clustering)
+    {
+        return metadata.comparator.asByteComparable(clustering);
+    }
+
+    public Row staticRow()
+    {
+        RowData staticRow = (RowData) trie.get(STATIC_CLUSTERING_PATH);
+
+        if (staticRow != null)
+            return toRow(staticRow, Clustering.STATIC_CLUSTERING);
+        else
+            return Rows.EMPTY_STATIC_ROW;
+    }
+
+    public boolean isEmpty()
+    {
+        return rowCountIncludingStatic == 0 && deletionInfo().isLive();
+    }
+
+    private boolean hasStaticRow()
+    {
+        return trie.get(STATIC_CLUSTERING_PATH) != null;
+    }
+
+    public boolean hasRows()
+    {
+        return rowCountIncludingStatic > 1 || rowCountIncludingStatic > 0 && !hasStaticRow();
+    }
+
+    /**
+     * Provides read access to the trie for users that can take advantage of it directly (e.g. Memtable).
+     */
+    public Trie<Object> trie()
+    {
+        return trie;
+    }
+
+    private Trie<Object> nonStaticSubtrie()
+    {
+        // skip static row if present - the static clustering sorts before BOTTOM so that it's never included in
+        // any slices (we achieve this by using the byte ByteSource.EXCLUDED for its representation, which is lower
+        // than BOTTOM's ByteSource.LT_NEXT_COMPONENT).
+        return trie.subtrie(BOTTOM_PATH, null);
+    }
+
+    public Iterator<Row> rowIterator()
+    {
+        return rowIterator(nonStaticSubtrie(), Direction.FORWARD);
+    }
+
+    public Iterator<Row> rowsIncludingStatic()
+    {
+        return rowIterator(trie, Direction.FORWARD);
+    }
+
+    @Override
+    public Row lastRow()
+    {
+        Iterator<Row> reverseIterator = rowIterator(nonStaticSubtrie(), Direction.REVERSE);
+        return reverseIterator.hasNext() ? reverseIterator.next() : null;
+    }
+
+    public Row getRow(Clustering clustering)
+    {
+        RowData data = (RowData) trie.get(path(clustering));
+
+        DeletionInfo deletionInfo = deletionInfo();
+        RangeTombstone rt = deletionInfo.rangeCovering(clustering);
+
+        // The trie only contains rows, so it doesn't allow to directly account for deletion that should apply to row
+        // (the partition deletion or the deletion of a range tombstone that covers it). So if needs be, reuse the row
+        // deletion to carry the proper deletion on the row.
+        DeletionTime partitionDeletion = deletionInfo.getPartitionDeletion();
+        DeletionTime activeDeletion = partitionDeletion;
+        if (rt != null && rt.deletionTime().supersedes(activeDeletion))
+            activeDeletion = rt.deletionTime();
+
+        if (data == null)
+        {
+            // this means our partition level deletion supersedes all other deletions and we don't have to keep the row deletions
+            if (activeDeletion == partitionDeletion)
+                return null;
+            // no need to check activeDeletion.isLive here - if anything superseedes the partitionDeletion
+            // it must be non-live
+            return BTreeRow.emptyDeletedRow(clustering, Row.Deletion.regular(activeDeletion));
+        }
+
+        Row row = toRow(data, clustering);
+        if (!activeDeletion.isLive())
+            row = row.filter(ColumnFilter.selection(columns()), activeDeletion, true, metadata());
+        return row;
+    }
+
+    public UnfilteredRowIterator unfilteredIterator()
+    {
+        return unfilteredIterator(ColumnFilter.selection(columns()), Slices.ALL, false);
+    }
+
+    public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, Slices slices, boolean reversed)
+    {
+        Row staticRow = staticRow(selection, false);
+        if (slices.size() == 0)
+        {
+            DeletionTime partitionDeletion = deletionInfo().getPartitionDeletion();
+            return UnfilteredRowIterators.noRowsIterator(metadata(), partitionKey(), staticRow, partitionDeletion, reversed);
+        }
+
+        return slices.size() == 1
+               ? sliceIterator(selection, slices.get(0), reversed, staticRow)
+               : new SlicesIterator(selection, slices, reversed, staticRow);
+    }
+
+    public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, NavigableSet<Clustering<?>> clusteringsInQueryOrder, boolean reversed)
+    {
+        Row staticRow = staticRow(selection, false);
+        if (clusteringsInQueryOrder.isEmpty())
+        {
+            DeletionTime partitionDeletion = deletionInfo().getPartitionDeletion();
+            return UnfilteredRowIterators.noRowsIterator(metadata(), partitionKey(), staticRow, partitionDeletion, reversed);
+        }
+
+        Iterator<Row> rowIter = new AbstractIterator<Row>() {
+
+            Iterator<Clustering<?>> clusterings = clusteringsInQueryOrder.iterator();
+
+            @Override
+            protected Row computeNext()
+            {
+                while (clusterings.hasNext())
+                {
+                    Clustering<?> clustering = clusterings.next();
+                    Object rowData = trie.get(path(clustering));
+                    if (rowData instanceof RowData)
+                        return toRow((RowData) rowData, clustering);
+                }
+                return endOfData();
+            }
+        };
+
+        // not using DeletionInfo.rangeCovering(Clustering), because it returns the original range tombstone,
+        // but we need DeletionInfo.rangeIterator(Set<Clustering>) that generates tombstones based on given clustering bound.
+        Iterator<RangeTombstone> deleteIter = deletionInfo().rangeIterator(clusteringsInQueryOrder, reversed);
+
+        return merge(rowIter, deleteIter, selection, reversed, staticRow);
+    }
+
+    private UnfilteredRowIterator sliceIterator(ColumnFilter selection, Slice slice, boolean reversed, Row staticRow)
+    {
+        ClusteringBound start = slice.start();
+        ClusteringBound end = slice.end() == ClusteringBound.TOP ? null : slice.end();
+        Iterator<Row> rowIter = slice(start, end, reversed);
+        Iterator<RangeTombstone> deleteIter = deletionInfo().rangeIterator(slice, reversed);
+        return merge(rowIter, deleteIter, selection, reversed, staticRow);
+    }
+
+    private Iterator<Row> slice(ClusteringBound start, ClusteringBound end, boolean reversed)
+    {
+        ByteComparable endPath = end != null ? path(end) : null;
+        // use BOTTOM as bound to skip over static rows
+        ByteComparable startPath = start != null ? path(start) : BOTTOM_PATH;
+        return rowIterator(trie.subtrie(startPath, endPath), Direction.fromBoolean(reversed));
+    }
+
+    private Row staticRow(ColumnFilter columns, boolean setActiveDeletionToRow)
+    {
+        DeletionTime partitionDeletion = deletionInfo().getPartitionDeletion();
+        Row staticRow = staticRow();
+        if (columns.fetchedColumns().statics.isEmpty() || (staticRow.isEmpty() && partitionDeletion.isLive()))
+            return Rows.EMPTY_STATIC_ROW;
+
+        Row row = staticRow.filter(columns, partitionDeletion, setActiveDeletionToRow, metadata());
+        return row == null ? Rows.EMPTY_STATIC_ROW : row;
+    }
+
+    private RowAndDeletionMergeIterator merge(Iterator<Row> rowIter, Iterator<RangeTombstone> deleteIter,
+                                              ColumnFilter selection, boolean reversed, Row staticRow)
+    {
+        return new RowAndDeletionMergeIterator(metadata(), partitionKey(), deletionInfo().getPartitionDeletion(),
+                                               selection, staticRow, reversed, stats(),
+                                               rowIter, deleteIter, canHaveShadowedData);
+    }
+
+
+    @Override
+    public String toString()
+    {
+        return Partition.toString(this);
+    }
+
+    class SlicesIterator extends AbstractUnfilteredRowIterator
+    {
+        private final Slices slices;
+
+        private int idx;
+        private Iterator<Unfiltered> currentSlice;
+        private final ColumnFilter selection;
+
+        private SlicesIterator(ColumnFilter selection,
+                               Slices slices,
+                               boolean isReversed,
+                               Row staticRow)
+        {
+            super(TrieBackedPartition.this.metadata(), TrieBackedPartition.this.partitionKey(),
+                  TrieBackedPartition.this.partitionLevelDeletion(),
+                  selection.fetchedColumns(), staticRow, isReversed, TrieBackedPartition.this.stats());
+            this.selection = selection;
+            this.slices = slices;
+        }
+
+        protected Unfiltered computeNext()
+        {
+            while (true)
+            {
+                if (currentSlice == null)
+                {
+                    if (idx >= slices.size())
+                        return endOfData();
+
+                    int sliceIdx = isReverseOrder ? slices.size() - idx - 1 : idx;
+                    currentSlice = sliceIterator(selection, slices.get(sliceIdx), isReverseOrder, Rows.EMPTY_STATIC_ROW);
+                    idx++;
+                }
+
+                if (currentSlice.hasNext())
+                    return currentSlice.next();
+
+                currentSlice = null;
+            }
+        }
+    }
+
+
+    /**
+     * An snapshot of the current TrieBackedPartition data, copied on heap when retrieved.
+     */
+    private static final class WithEnsureOnHeap extends TrieBackedPartition
+    {
+        final DeletionInfo onHeapDeletion;
+        EnsureOnHeap ensureOnHeap;
+
+        public WithEnsureOnHeap(DecoratedKey partitionKey,
+                                RegularAndStaticColumns columns,
+                                EncodingStats stats,
+                                int rowCountIncludingStatic,
+                                Trie<Object> trie,
+                                TableMetadata metadata,
+                                boolean canHaveShadowedData,
+                                EnsureOnHeap ensureOnHeap)
+        {
+            super(partitionKey, columns, stats, rowCountIncludingStatic, trie, metadata, canHaveShadowedData);
+            this.ensureOnHeap = ensureOnHeap;
+            this.onHeapDeletion = ensureOnHeap.applyToDeletionInfo(super.deletionInfo());
+        }
+
+        @Override
+        public Row toRow(RowData data, Clustering clustering)
+        {
+            return ensureOnHeap.applyToRow(super.toRow(data, clustering));
+        }
+
+        @Override
+        public DeletionInfo deletionInfo()
+        {
+            return onHeapDeletion;
+        }
+    }
+
+    /**
+     * Resolver for operations with trie-backed partitions. We don't permit any overwrites/merges.
+     */
+    public static final InMemoryTrie.UpsertTransformer<Object, Object> NO_CONFLICT_RESOLVER =
+            (existing, update) ->
+            {
+                if (existing != null)
+                    throw new AssertionError("Unique rows expected.");
+                return update;
+            };
+
+    /**
+     * Helper class for constructing tries and deletion info from an iterator or flowable partition.
+     *
+     * Note: This is not collecting any stats or columns!
+     */
+    public static class ContentBuilder
+    {
+        final TableMetadata metadata;
+        final ClusteringComparator comparator;
+
+        private final MutableDeletionInfo.Builder deletionBuilder;
+        private final InMemoryTrie<Object> trie;
+
+        private final boolean useRecursive;
+        private final boolean collectDataSize;
+
+        private int rowCountIncludingStatic;
+        private long dataSize;
+
+        public ContentBuilder(TableMetadata metadata, DeletionTime partitionLevelDeletion, boolean isReverseOrder, boolean collectDataSize)
+        {
+            this.metadata = metadata;
+            this.comparator = metadata.comparator;
+
+            this.deletionBuilder = MutableDeletionInfo.builder(partitionLevelDeletion,
+                                                               comparator,
+                                                               isReverseOrder);
+            this.trie = InMemoryTrie.shortLived(BYTE_COMPARABLE_VERSION);
+
+            this.useRecursive = useRecursive(comparator);
+            this.collectDataSize = collectDataSize;
+
+            rowCountIncludingStatic = 0;
+            dataSize = 0;
+        }
+
+        public ContentBuilder addStatic(Row staticRow) throws TrieSpaceExhaustedException
+        {
+            if (!staticRow.isEmpty())
+                return addRow(staticRow);
+            else
+                return this;
+        }
+
+        public ContentBuilder addRow(Row row) throws TrieSpaceExhaustedException
+        {
+            putInTrie(comparator, useRecursive, trie, row);
+            ++rowCountIncludingStatic;
+            if (collectDataSize)
+                dataSize += row.dataSize();
+            return this;
+        }
+
+        public ContentBuilder addRangeTombstoneMarker(RangeTombstoneMarker unfiltered)
+        {
+            deletionBuilder.add(unfiltered);
+            return this;
+        }
+
+        public ContentBuilder addUnfiltered(Unfiltered unfiltered) throws TrieSpaceExhaustedException
+        {
+            if (unfiltered.kind() == Unfiltered.Kind.ROW)
+                return addRow((Row) unfiltered);
+            else
+                return addRangeTombstoneMarker((RangeTombstoneMarker) unfiltered);
+        }
+
+        public ContentBuilder complete() throws TrieSpaceExhaustedException
+        {
+            MutableDeletionInfo deletionInfo = deletionBuilder.build();
+            trie.putRecursive(ByteComparable.EMPTY, deletionInfo, NO_CONFLICT_RESOLVER);    // will throw if called more than once
+            // dataSize does not include the deletion info bytes
+            return this;
+        }
+
+        public Trie<Object> trie()
+        {
+            return trie;
+        }
+
+        public int rowCountIncludingStatic()
+        {
+            return rowCountIncludingStatic;
+        }
+
+        public int dataSize()
+        {
+            assert collectDataSize;
+            return Ints.saturatedCast(dataSize);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/partitions/TriePartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/TriePartitionUpdate.java
new file mode 100644
index 000000000000..bd5e120b9dd0
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/partitions/TriePartitionUpdate.java
@@ -0,0 +1,639 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.partitions;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
+import com.google.common.primitives.Ints;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.MutableDeletionInfo;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.btree.BTree;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * A trie-backed PartitionUpdate. Immutable.
+ * <p>
+ * Provides factories for simple variations (e.g. singleRowUpdate) and a mutable builder for constructing one.
+ * The builder holds a mutable trie to which content may be added in any order, also taking care of
+ * merging any duplicate rows, and keeping track of statistics and column coverage.
+ */
+public class TriePartitionUpdate extends TrieBackedPartition implements PartitionUpdate
+{
+    protected static final Logger logger = LoggerFactory.getLogger(TriePartitionUpdate.class);
+
+    public static final Factory FACTORY = new TrieFactory();
+
+    final int dataSize;
+
+    private TriePartitionUpdate(TableMetadata metadata,
+                                DecoratedKey key,
+                                RegularAndStaticColumns columns,
+                                EncodingStats stats,
+                                int rowCountIncludingStatic,
+                                int dataSize,
+                                Trie<Object> trie,
+                                boolean canHaveShadowedData)
+    {
+        super(key, columns, stats, rowCountIncludingStatic, trie, metadata, canHaveShadowedData);
+        this.dataSize = dataSize;
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (!(obj instanceof TriePartitionUpdate))
+            return false;
+
+        TriePartitionUpdate that = (TriePartitionUpdate) obj;
+        return partitionKey.equals(that.partitionKey)
+               && metadata().id.equals(that.metadata().id)
+               && deletionInfo().equals(that.deletionInfo())
+               && staticRow().equals(that.staticRow())
+               && Iterators.elementsEqual(rowIterator(), that.rowIterator());
+    }
+
+
+    private static InMemoryTrie<Object> newTrie(DeletionInfo deletion)
+    {
+        InMemoryTrie<Object> trie = InMemoryTrie.shortLived(BYTE_COMPARABLE_VERSION);
+        try
+        {
+            trie.putRecursive(ByteComparable.EMPTY, deletion, NO_CONFLICT_RESOLVER);
+        }
+        catch (TrieSpaceExhaustedException e)
+        {
+            throw new AssertionError(e);
+        }
+        return trie;
+    }
+
+    /**
+     * Creates a empty immutable partition update.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the created update.
+     *
+     * @return the newly created empty (and immutable) update.
+     */
+    public static TriePartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey key)
+    {
+        return new TriePartitionUpdate(metadata,
+                                       key,
+                                       RegularAndStaticColumns.NONE,
+                                       EncodingStats.NO_STATS,
+                                       0,
+                                       0,
+                                       newTrie(MutableDeletionInfo.live()),
+                                       false);
+    }
+
+    /**
+     * Creates an immutable partition update that entirely deletes a given partition.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the partition that the created update should delete.
+     * @param timestamp the timestamp for the deletion.
+     * @param nowInSec the current time in seconds to use as local deletion time for the partition deletion.
+     *
+     * @return the newly created partition deletion update.
+     */
+    public static TriePartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, long nowInSec)
+    {
+        MutableDeletionInfo deletion = new MutableDeletionInfo(timestamp, nowInSec);
+        return new TriePartitionUpdate(metadata,
+                                       key,
+                                       RegularAndStaticColumns.NONE,
+                                       new EncodingStats(timestamp, nowInSec, LivenessInfo.NO_TTL),
+                                       0,
+                                       0,
+                                       newTrie(deletion),
+                                       false);
+    }
+
+    /**
+     * Creates an immutable partition update that contains a single row update.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the partition to update.
+     * @param row the row for the update, may be a regular or static row and cannot be null.
+     *
+     * @return the newly created partition update containing only {@code row}.
+     */
+    public static TriePartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row)
+    {
+        EncodingStats stats = EncodingStats.Collector.forRow(row);
+        InMemoryTrie<Object> trie = newTrie(DeletionInfo.LIVE);
+
+        RegularAndStaticColumns columns;
+        if (row.isStatic())
+            columns = new RegularAndStaticColumns(Columns.from(row.columns()), Columns.NONE);
+        else
+            columns = new RegularAndStaticColumns(Columns.NONE, Columns.from(row.columns()));
+
+        try
+        {
+            putInTrie(metadata.comparator, useRecursive(metadata.comparator), trie, row);
+        }
+        catch (TrieSpaceExhaustedException e)
+        {
+            throw new AssertionError(e);
+        }
+
+        return new TriePartitionUpdate(metadata, key, columns, stats, 1, row.dataSize(), trie, false);
+    }
+
+    /**
+     * Creates an immutable partition update that contains a single row update.
+     *
+     * @param metadata the metadata for the created update.
+     * @param key the partition key for the partition to update.
+     * @param row the row for the update.
+     *
+     * @return the newly created partition update containing only {@code row}.
+     */
+    public static TriePartitionUpdate singleRowUpdate(TableMetadata metadata, ByteBuffer key, Row row)
+    {
+        return singleRowUpdate(metadata, metadata.partitioner.decorateKey(key), row);
+    }
+
+    /**
+     * Turns the given iterator into an update.
+     *
+     * @param iterator the iterator to turn into updates.
+     *
+     * Warning: this method does not close the provided iterator, it is up to
+     * the caller to close it.
+     */
+    @SuppressWarnings("resource")
+    public static TriePartitionUpdate fromIterator(UnfilteredRowIterator iterator)
+    {
+        ContentBuilder builder = build(iterator, true);
+
+        return new TriePartitionUpdate(iterator.metadata(),
+                                       iterator.partitionKey(),
+                                       iterator.columns(),
+                                       iterator.stats(),
+                                       builder.rowCountIncludingStatic(),
+                                       builder.dataSize(),
+                                       builder.trie(),
+                                       false);
+    }
+
+    public static TriePartitionUpdate asTrieUpdate(PartitionUpdate update)
+    {
+        if (update instanceof TriePartitionUpdate)
+            return (TriePartitionUpdate) update;
+
+        try (UnfilteredRowIterator iterator = update.unfilteredIterator())
+        {
+            return fromIterator(iterator);
+        }
+    }
+
+    public static Trie<Object> asMergableTrie(PartitionUpdate update)
+    {
+        return asTrieUpdate(update).trie.prefixedBy(update.partitionKey());
+    }
+
+    /**
+     * Modify this update to set every timestamp for live data to {@code newTimestamp} and
+     * every deletion timestamp to {@code newTimestamp - 1}.
+     *
+     * There is no reason to use that except on the Paxos code path, where we need to ensure that
+     * anything inserted uses the ballot timestamp (to respect the order of updates decided by
+     * the Paxos algorithm). We use {@code newTimestamp - 1} for deletions because tombstones
+     * always win on timestamp equality and we don't want to delete our own insertions
+     * (typically, when we overwrite a collection, we first set a complex deletion to delete the
+     * previous collection before adding new elements. If we were to set that complex deletion
+     * to the same timestamp that the new elements, it would delete those elements). And since
+     * tombstones always wins on timestamp equality, using -1 guarantees our deletion will still
+     * delete anything from a previous update.
+     */
+    @Override
+    public TriePartitionUpdate withUpdatedTimestamps(long newTimestamp)
+    {
+
+        InMemoryTrie<Object> t = InMemoryTrie.shortLived(BYTE_COMPARABLE_VERSION);
+        try
+        {
+            t.apply(trie, new InMemoryTrie.UpsertTransformer<Object, Object>()
+            {
+                public Object apply(Object shouldBeNull, Object o)
+                {
+                    assert shouldBeNull == null;
+                    if (o instanceof RowData)
+                        return applyRowData((RowData) o);
+                    else
+                        return applyDeletion((DeletionInfo) o);
+                }
+
+                public RowData applyRowData(RowData update)
+                {
+                    LivenessInfo newInfo = update.livenessInfo.isEmpty()
+                                           ? update.livenessInfo
+                                           : update.livenessInfo.withUpdatedTimestamp(newTimestamp);
+                    DeletionTime newDeletion = update.deletion.isLive()
+                                               ? DeletionTime.LIVE
+                                               : DeletionTime.build(newTimestamp - 1, update.deletion.localDeletionTime());
+
+                    return new RowData(BTree.transformAndFilter(update.columnsBTree,
+                                                                (ColumnData cd) -> cd.updateAllTimestamp(newTimestamp)),
+                                       newInfo, newDeletion);
+                }
+
+                public DeletionInfo applyDeletion(DeletionInfo update)
+                {
+                    if (update.isLive())
+                        return update;
+
+                    MutableDeletionInfo mdi = update.mutableCopy();
+                    mdi.updateAllTimestamp(newTimestamp - 1);
+                    return mdi;
+                }
+            }, x -> false);
+        }
+        catch (TrieSpaceExhaustedException e)
+        {
+            throw new AssertionError(e);
+        }
+        return new TriePartitionUpdate(metadata, partitionKey, columns, stats, rowCountIncludingStatic, dataSize, t, canHaveShadowedData);
+    }
+
+    /**
+     * The number of "operations" contained in the update.
+     * <p>
+     * This is used by {@code Memtable} to approximate how much work this update does. In practice, this
+     * count how many rows are updated and how many ranges are deleted by the partition update.
+     *
+     * @return the number of "operations" performed by the update.
+     */
+    @Override
+    public int operationCount()
+    {
+        return rowCountIncludingStatic
+             + deletionInfo().rangeCount()
+             + (deletionInfo().getPartitionDeletion().isLive() ? 0 : 1);
+    }
+
+    /**
+     * The size of the data contained in this update.
+     *
+     * @return the size of the data contained in this update.
+     */
+    @Override
+    public int dataSize()
+    {
+        return dataSize;
+    }
+
+    /**
+     * The size of the data contained in this update.
+     *
+     * @return the size of the data contained in this update.
+     */
+    @Override
+    public long unsharedHeapSize()
+    {
+        assert trie instanceof InMemoryTrie;
+        InMemoryTrie<Object> inMemoryTrie = (InMemoryTrie<Object>) trie;
+        long heapSize = inMemoryTrie.usedSizeOnHeap();
+        for (Object o : inMemoryTrie.values())
+        {
+            if (o instanceof RowData)
+                heapSize += ((RowData) o).unsharedHeapSizeExcludingData();
+            else
+                heapSize += ((DeletionInfo) o).unsharedHeapSize();
+        }
+        return heapSize;
+    }
+
+    /**
+     * Validates the data contained in this update.
+     *
+     * @throws org.apache.cassandra.serializers.MarshalException if some of the data contained in this update is corrupted.
+     */
+    @Override
+    public void validate()
+    {
+        for (Iterator<Row> it = rowsIncludingStatic(); it.hasNext();)
+        {
+            Row row = it.next();
+            metadata().comparator.validate(row.clustering());
+            for (ColumnData cd : row)
+                cd.validate();
+        }
+    }
+
+    /**
+     * The maximum timestamp used in this update.
+     *
+     * @return the maximum timestamp used in this update.
+     */
+    @Override
+    public long maxTimestamp()
+    {
+        long maxTimestamp = deletionInfo().maxTimestamp();
+        for (Iterator<Row> it = rowsIncludingStatic(); it.hasNext();)
+            maxTimestamp = Math.max(maxTimestamp, Rows.collectMaxTimestamp(it.next()));
+
+        return maxTimestamp;
+    }
+
+    /**
+     * For an update on a counter table, returns a list containing a {@code CounterMark} for
+     * every counter contained in the update.
+     *
+     * @return a list with counter marks for every counter in this update.
+     */
+    @Override
+    public List<CounterMark> collectCounterMarks()
+    {
+        assert metadata().isCounter();
+        // We will take aliases on the rows of this update, and update them in-place. So we should be sure the
+        // update is now immutable for all intent and purposes.
+        List<CounterMark> marks = new ArrayList<>();
+        for (Iterator<Row> it = rowsIncludingStatic(); it.hasNext();)
+        {
+            Row row = it.next();
+            addMarksForRow(row, marks);
+        }
+        return marks;
+    }
+
+    private static void addMarksForRow(Row row, List<CounterMark> marks)
+    {
+        for (Cell<?> cell : row.cells())
+        {
+            if (cell.isCounterCell())
+                marks.add(new CounterMark(row, cell.column(), cell.path()));
+        }
+    }
+
+    @Override
+    public PartitionUpdate withOnlyPresentColumns()
+    {
+        Set<ColumnMetadata> columnSet = new HashSet<>();
+
+        for (Row row : rows())
+            for (ColumnData column : row)
+                columnSet.add(column.column());
+
+        RegularAndStaticColumns columns = RegularAndStaticColumns.builder().addAll(columnSet).build();
+        return new TriePartitionUpdate(metadata, partitionKey, columns, stats, rowCountIncludingStatic, dataSize, trie, false);
+    }
+
+    /**
+     * Builder for PartitionUpdates
+     *
+     * This class is not thread safe, but the PartitionUpdate it produces is (since it is immutable).
+     */
+    public static class Builder implements PartitionUpdate.Builder
+    {
+        private final TableMetadata metadata;
+        private final DecoratedKey key;
+        private final MutableDeletionInfo deletionInfo;
+        private final boolean canHaveShadowedData;
+        private final RegularAndStaticColumns columns;
+        private final InMemoryTrie<Object> trie = InMemoryTrie.shortLived(BYTE_COMPARABLE_VERSION);
+        private final EncodingStats.Collector statsCollector = new EncodingStats.Collector();
+        private final boolean useRecursive;
+        private int rowCountIncludingStatic;
+        private long dataSize;
+
+        public Builder(TableMetadata metadata,
+                       DecoratedKey key,
+                       RegularAndStaticColumns columns)
+        {
+            this(metadata, key, columns, true, Rows.EMPTY_STATIC_ROW, DeletionInfo.LIVE);
+        }
+
+        private Builder(TableMetadata metadata,
+                        DecoratedKey key,
+                        RegularAndStaticColumns columns,
+                        boolean canHaveShadowedData,
+                        Row staticRow,
+                        DeletionInfo deletionInfo)
+        {
+            this.metadata = metadata;
+            this.key = key;
+            this.columns = columns;
+            this.canHaveShadowedData = canHaveShadowedData;
+            this.deletionInfo = deletionInfo.mutableCopy();
+            useRecursive = useRecursive(metadata.comparator);
+            rowCountIncludingStatic = 0;
+            dataSize = 0;
+            add(staticRow);
+        }
+
+        // This is wasteful, only to be used for testing.
+        @VisibleForTesting
+        public Builder(TriePartitionUpdate base)
+        {
+            this(base.metadata, base.partitionKey, base.columns(), base.canHaveShadowedData, Rows.EMPTY_STATIC_ROW, base.deletionInfo());
+            for (Iterator<Row> it = base.rowsIncludingStatic(); it.hasNext();)
+                add(it.next());
+        }
+
+        /**
+         * Adds a row to this update.
+         * <p>
+         * There is no particular assumption made on the order of row added to a partition update. It is further
+         * allowed to add the same row (more precisely, multiple row objects for the same clustering).
+         * <p>
+         * Note however that the columns contained in the added row must be a subset of the columns used when
+         * creating this update.
+         *
+         * @param row the row to add.
+         */
+        public void add(Row row)
+        {
+            if (row.isEmpty())
+                return;
+
+            // this assert is expensive, and possibly of limited value; we should consider removing it
+            // or introducing a new class of assertions for test purposes
+            assert (row.isStatic() ? columns().statics : columns().regulars).containsAll(row.columns())
+            : (row.isStatic() ? columns().statics : columns().regulars) + " is not superset of " + row.columns();
+
+            try
+            {
+                trie.putSingleton(metadata.comparator.asByteComparable(row.clustering()),
+                                  row,
+                                  this::merge,
+                                  useRecursive);
+            }
+            catch (TrieSpaceExhaustedException e)
+            {
+                throw new AssertionError(e);
+            }
+            Rows.collectStats(row, statsCollector);
+        }
+
+        public void addPartitionDeletion(DeletionTime deletionTime)
+        {
+            deletionInfo.add(deletionTime);
+        }
+
+        public void add(RangeTombstone range)
+        {
+            deletionInfo.add(range, metadata.comparator);
+        }
+
+        public DecoratedKey partitionKey()
+        {
+            return key;
+        }
+
+        public TableMetadata metadata()
+        {
+            return metadata;
+        }
+
+        public TriePartitionUpdate build()
+        {
+            try
+            {
+                trie.putRecursive(ByteComparable.EMPTY, deletionInfo, NO_CONFLICT_RESOLVER);
+            }
+            catch (TrieSpaceExhaustedException e)
+            {
+                throw new AssertionError(e);
+            }
+            deletionInfo.collectStats(statsCollector);
+            TriePartitionUpdate pu = new TriePartitionUpdate(metadata,
+                                                             partitionKey(),
+                                                             columns,
+                                                             statsCollector.get(),
+                                                             rowCountIncludingStatic,
+                                                             Ints.saturatedCast(dataSize),
+                                                             trie,
+                                                             canHaveShadowedData);
+
+            return pu;
+        }
+
+        RowData merge(Object existing, Row update)
+        {
+            if (existing != null)
+            {
+                // this is not expected to happen much, so going through toRow and the existing size is okay
+                RowData rowData = (RowData) existing;
+                update = Rows.merge(rowData.toRow(update.clustering()), update);
+                dataSize += update.dataSize() - rowData.dataSize();
+            }
+            else
+            {
+                ++rowCountIncludingStatic;
+                dataSize += update.dataSize();
+            }
+
+            return rowToData(update);
+        }
+
+        public RegularAndStaticColumns columns()
+        {
+            return columns;
+        }
+
+        public DeletionTime partitionLevelDeletion()
+        {
+            return deletionInfo.getPartitionDeletion();
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Builder{" +
+                   "metadata=" + metadata +
+                   ", key=" + key +
+                   ", deletionInfo=" + deletionInfo +
+                   ", canHaveShadowedData=" + canHaveShadowedData +
+                   ", columns=" + columns +
+                   '}';
+        }
+    }
+
+    public static class TrieFactory implements PartitionUpdate.Factory
+    {
+
+        @Override
+        public PartitionUpdate.Builder builder(TableMetadata metadata, DecoratedKey partitionKey, RegularAndStaticColumns columns, int initialRowCapacity)
+        {
+            return new TriePartitionUpdate.Builder(metadata, partitionKey, columns);
+        }
+
+        @Override
+        public PartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey partitionKey)
+        {
+            return TriePartitionUpdate.emptyUpdate(metadata, partitionKey);
+        }
+
+        @Override
+        public PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey valueKey, Row row)
+        {
+            return TriePartitionUpdate.singleRowUpdate(metadata, valueKey, row);
+        }
+
+        @Override
+        public PartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, long nowInSec)
+        {
+            return TriePartitionUpdate.fullPartitionDelete(metadata, key, timestamp, nowInSec);
+        }
+
+        @Override
+        public PartitionUpdate fromIterator(UnfilteredRowIterator iterator)
+        {
+            return TriePartitionUpdate.fromIterator(iterator);
+        }
+
+        @Override
+        public PartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter)
+        {
+            return TriePartitionUpdate.fromIterator(UnfilteredRowIterators.withOnlyQueriedData(iterator, filter));
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/partitions/TriePartitionUpdater.java b/src/java/org/apache/cassandra/db/partitions/TriePartitionUpdater.java
new file mode 100644
index 000000000000..832c38dbf3de
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/partitions/TriePartitionUpdater.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.partitions;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.db.memtable.TrieMemtable;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.memory.Cloner;
+
+import static org.apache.cassandra.db.partitions.TrieBackedPartition.RowData;
+
+/**
+ *  The function we provide to the trie utilities to perform any partition and row inserts and updates
+ */
+public final class TriePartitionUpdater
+extends BasePartitionUpdater
+implements InMemoryTrie.UpsertTransformerWithKeyProducer<Object, Object>
+{
+    private final UpdateTransaction indexer;
+    private final TableMetadata metadata;
+    private TrieMemtable.PartitionData currentPartition;
+    private final TrieMemtable.MemtableShard owner;
+    public int partitionsAdded = 0;
+
+    public TriePartitionUpdater(Cloner cloner,
+                                UpdateTransaction indexer,
+                                TableMetadata metadata,
+                                TrieMemtable.MemtableShard owner)
+    {
+        super(cloner);
+        this.indexer = indexer;
+        this.metadata = metadata;
+        this.owner = owner;
+    }
+
+    @Override
+    public Object apply(Object existing, Object update, InMemoryTrie.KeyProducer<Object> keyState)
+    {
+        if (update instanceof RowData)
+            return applyRow((RowData) existing, (RowData) update, keyState);
+        else if (update instanceof DeletionInfo)
+            return applyDeletion((TrieMemtable.PartitionData) existing, (DeletionInfo) update);
+        else
+            throw new AssertionError("Unexpected update type: " + update.getClass());
+    }
+
+    /**
+     * Called when a row needs to be copied to the Memtable trie.
+     *
+     * @param existing Existing RowData for this clustering, or null if there isn't any.
+     * @param insert RowData to be inserted.
+     * @param keyState Used to obtain the path through which this node was reached.
+     * @return the insert row, or the merged row, copied using our allocator
+     */
+    private RowData applyRow(RowData existing, RowData insert, InMemoryTrie.KeyProducer<Object> keyState)
+    {
+        if (existing == null)
+        {
+            RowData data = insert.clone(cloner);
+
+            if (indexer != UpdateTransaction.NO_OP)
+                indexer.onInserted(data.toRow(clusteringFor(keyState)));
+
+            this.dataSize += data.dataSize();
+            this.heapSize += data.unsharedHeapSizeExcludingData();
+            currentPartition.markInsertedRows(1);  // null pointer here means a problem in applyDeletion
+            return data;
+        }
+        else
+        {
+            // data and heap size are updated during merge through the PostReconciliationFunction interface
+            RowData reconciled = merge(existing, insert);
+
+            if (indexer != UpdateTransaction.NO_OP)
+            {
+                Clustering<?> clustering = clusteringFor(keyState);
+                indexer.onUpdated(existing.toRow(clustering), reconciled.toRow(clustering));
+            }
+
+            return reconciled;
+        }
+    }
+
+    private RowData merge(RowData existing, RowData update)
+    {
+
+        LivenessInfo livenessInfo = LivenessInfo.merge(update.livenessInfo, existing.livenessInfo);
+        DeletionTime deletion = DeletionTime.merge(update.deletion, existing.deletion);
+        if (deletion.deletes(livenessInfo))
+            livenessInfo = LivenessInfo.EMPTY;
+
+        Object[] tree = BTreeRow.mergeRowBTrees(this,
+                                                existing.columnsBTree, update.columnsBTree,
+                                                deletion, existing.deletion);
+        return new RowData(tree, livenessInfo, deletion);
+    }
+
+    private Clustering<?> clusteringFor(InMemoryTrie.KeyProducer<Object> keyState)
+    {
+        return metadata.comparator.clusteringFromByteComparable(
+            ByteArrayAccessor.instance,
+            ByteComparable.preencoded(TrieBackedPartition.BYTE_COMPARABLE_VERSION,
+                                      keyState.getBytes(TrieMemtable.IS_PARTITION_BOUNDARY)));
+    }
+
+    /**
+     * Called at the partition boundary to merge the existing and new metadata associated with the partition. This needs
+     * to update the deletion time with any new deletion introduced by the update, but also make sure that the
+     * statistics we track for the partition (dataSize) are updated for the changes caused by merging the update's rows
+     * (note that this is called _after_ the rows of the partition have been merged, on the return path of the
+     * recursion).
+     *
+     * @param existing Any partition data already associated with the partition.
+     * @param update The update, always non-null.
+     * @return the combined partition data, copying any updated deletion information to heap.
+     */
+    private TrieMemtable.PartitionData applyDeletion(TrieMemtable.PartitionData existing, DeletionInfo update)
+    {
+        if (indexer != UpdateTransaction.NO_OP)
+        {
+            if (!update.getPartitionDeletion().isLive())
+                indexer.onPartitionDeletion(update.getPartitionDeletion());
+            if (update.hasRanges())
+                update.rangeIterator(false).forEachRemaining(indexer::onRangeTombstone);
+        }
+
+        if (existing == null)
+        {
+            // Note: Always on-heap, regardless of cloner
+            TrieMemtable.PartitionData newRef = new TrieMemtable.PartitionData(update, owner);
+            this.heapSize += newRef.unsharedHeapSize();
+            ++this.partitionsAdded;
+            return currentPartition = newRef;
+        }
+
+        assert owner == existing.owner;
+        if (update.isLive() || !update.mayModify(existing))
+            return currentPartition = existing;
+
+        // Note: Always on-heap, regardless of cloner
+        TrieMemtable.PartitionData merged = new TrieMemtable.PartitionData(existing, update);
+        this.heapSize += merged.unsharedHeapSize() - existing.unsharedHeapSize();
+        return currentPartition = merged;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
index e68603c9f3dd..906c7a42b841 100644
--- a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
+++ b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
@@ -19,18 +19,30 @@
 
 import java.io.IOError;
 import java.io.IOException;
-import java.util.*;
-
-import org.apache.cassandra.db.*;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.db.transform.FilteredPartitions;
 import org.apache.cassandra.db.transform.MorePartitions;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.CloseableIterator;
 import org.apache.cassandra.utils.MergeIterator;
+import org.apache.cassandra.utils.Reducer;
 
 /**
  * Static methods to work with partition iterators.
@@ -128,7 +140,7 @@ public static UnfilteredPartitionIterator merge(final List<? extends UnfilteredP
 
         final boolean preserveOrder = listener != null && listener.preserveOrder();
 
-        final MergeIterator<UnfilteredRowIterator, UnfilteredRowIterator> merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer<UnfilteredRowIterator, UnfilteredRowIterator>()
+        final CloseableIterator<UnfilteredRowIterator> merged = MergeIterator.getCloseable(iterators, partitionComparator, new Reducer<UnfilteredRowIterator, UnfilteredRowIterator>()
         {
             private final List<UnfilteredRowIterator> toMerge = new ArrayList<>(iterators.size());
 
@@ -152,7 +164,7 @@ public void reduce(int idx, UnfilteredRowIterator current)
                 }
             }
 
-            protected UnfilteredRowIterator getReduced()
+            public UnfilteredRowIterator getReduced()
             {
                 UnfilteredRowIterators.MergeListener rowListener = listener == null
                                                                  ? null
@@ -178,7 +190,7 @@ protected UnfilteredRowIterator getReduced()
                 return UnfilteredRowIterators.merge(toMerge, rowListener);
             }
 
-            protected void onKeyChange()
+            public void onKeyChange()
             {
                 toMerge.clear();
                 if (preserveOrder)
@@ -226,7 +238,7 @@ public static UnfilteredPartitionIterator mergeLazily(final List<? extends Unfil
 
         final TableMetadata metadata = iterators.get(0).metadata();
 
-        final MergeIterator<UnfilteredRowIterator, UnfilteredRowIterator> merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer<UnfilteredRowIterator, UnfilteredRowIterator>()
+        final CloseableIterator<UnfilteredRowIterator> merged = MergeIterator.getCloseable(iterators, partitionComparator, new Reducer<UnfilteredRowIterator, UnfilteredRowIterator>()
         {
             private final List<UnfilteredRowIterator> toMerge = new ArrayList<>(iterators.size());
 
@@ -235,7 +247,7 @@ public void reduce(int idx, UnfilteredRowIterator current)
                 toMerge.add(current);
             }
 
-            protected UnfilteredRowIterator getReduced()
+            public UnfilteredRowIterator getReduced()
             {
                 return new LazilyInitializedUnfilteredRowIterator(toMerge.get(0).partitionKey())
                 {
@@ -246,7 +258,7 @@ protected UnfilteredRowIterator initializeIterator()
                 };
             }
 
-            protected void onKeyChange()
+            public void onKeyChange()
             {
                 toMerge.clear();
             }
diff --git a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
index 4e54d6ee7782..1793c46d5a34 100644
--- a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
+++ b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
@@ -20,13 +20,20 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
 
 import com.google.common.base.Predicate;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.RepairFinishedCompactionTask;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -37,10 +44,12 @@
 import org.apache.cassandra.repair.ValidationPartitionIterator;
 import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.repair.consistent.LocalSessions;
 import org.apache.cassandra.service.ActiveRepairService;
 
 public class CassandraTableRepairManager implements TableRepairManager
 {
+    private static final Logger logger = LoggerFactory.getLogger(CassandraTableRepairManager.class);
     private final ColumnFamilyStore cfs;
     private final SharedContext ctx;
 
@@ -68,9 +77,38 @@ public Future<?> submitValidation(Callable<Object> validation)
     }
 
     @Override
-    public void incrementalSessionCompleted(TimeUUID sessionID)
+    public synchronized void incrementalSessionCompleted(TimeUUID sessionID)
     {
-        CompactionManager.instance.submitBackground(cfs);
+        LocalSessions sessions = ActiveRepairService.instance().consistent.local;
+        if (sessions.isSessionInProgress(sessionID))
+            return;
+
+        Set<SSTableReader> pendingRepairSSTables = cfs.getPendingRepairSSTables(sessionID);
+        if (pendingRepairSSTables.isEmpty())
+            return;
+
+        logger.debug("Number of sstables in pending repair: {} for session {}", pendingRepairSSTables.size(), sessionID);
+        LifecycleTransaction txn = cfs.getTracker().tryModify(pendingRepairSSTables, OperationType.COMPACTION);
+        if (txn == null)
+            return;
+
+        boolean isTransient = false;
+        for (SSTableReader sstable : pendingRepairSSTables)
+        {
+            if (sstable.isTransient())
+            {
+                isTransient = true;
+                break;
+            }
+        }
+
+        long repairedAt = sessions.getFinalSessionRepairedAt(sessionID);
+        RepairFinishedCompactionTask task = new RepairFinishedCompactionTask(cfs,
+                                                                             txn,
+                                                                             sessionID,
+                                                                             repairedAt,
+                                                                             isTransient);
+        task.run();
     }
 
     @Override
@@ -95,6 +133,8 @@ public boolean apply(SSTableReader sstable)
         }
         catch (Exception ex)
         {
+            if (ex instanceof InterruptedException)
+                Thread.currentThread().interrupt();
             throw new RuntimeException(String.format("Unable to take a snapshot %s on %s.%s", name, cfs.metadata.keyspace, cfs.metadata.name), ex);
         }
 
diff --git a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
index a31a7038b069..8b5b39d5cbc7 100644
--- a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
+++ b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
@@ -31,17 +31,14 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.Maps;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.compaction.ActiveCompactionsTracker;
 import org.apache.cassandra.db.compaction.CompactionController;
 import org.apache.cassandra.db.compaction.CompactionIterator;
-import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
@@ -50,13 +47,14 @@
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.metrics.TopPartitionTracker;
+import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.repair.ValidationPartitionIterator;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Refs;
 
@@ -106,9 +104,9 @@ public static long getDefaultGcBefore(ColumnFamilyStore cfs, long nowInSec)
 
     private static class ValidationCompactionIterator extends CompactionIterator
     {
-        public ValidationCompactionIterator(List<ISSTableScanner> scanners, ValidationCompactionController controller, long nowInSec, ActiveCompactionsTracker activeCompactions, TopPartitionTracker.Collector topPartitionCollector)
+        public ValidationCompactionIterator(List<ISSTableScanner> scanners, ValidationCompactionController controller, long nowInSec, TopPartitionTracker.Collector topPartitionCollector)
         {
-            super(OperationType.VALIDATION, scanners, controller, nowInSec, nextTimeUUID(), activeCompactions, topPartitionCollector);
+            super(OperationType.VALIDATION, scanners, controller, nowInSec, nextTimeUUID(), topPartitionCollector, null);
         }
     }
 
@@ -165,7 +163,7 @@ else if (isIncremental)
     private final boolean isGlobalSnapshotValidation;
 
     private final boolean isSnapshotValidation;
-    private final AbstractCompactionStrategy.ScannerList scanners;
+    private final ScannerList scanners;
     private final ValidationCompactionController controller;
 
     private final CompactionIterator ci;
@@ -195,7 +193,7 @@ public CassandraValidationIterator(ColumnFamilyStore cfs, SharedContext ctx, Col
         }
         else
         {
-            if (!isIncremental)
+            if (!isIncremental && DatabaseDescriptor.enableMemtableAndCommitLog())
             {
                 // flush first so everyone is validating data that is as similar as possible
                 cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.VALIDATION);
@@ -220,8 +218,8 @@ public CassandraValidationIterator(ColumnFamilyStore cfs, SharedContext ctx, Col
                     cfs.getTableName());
 
         controller = new ValidationCompactionController(cfs, getDefaultGcBefore(cfs, nowInSec));
-        scanners = cfs.getCompactionStrategyManager().getScanners(sstables, ranges);
-        ci = new ValidationCompactionIterator(scanners.scanners, controller, nowInSec, CompactionManager.instance.active, topPartitionCollector);
+        scanners = cfs.getCompactionStrategyContainer().getScanners(sstables, ranges);
+        ci = new ValidationCompactionIterator(scanners.scanners, controller, nowInSec, topPartitionCollector);
 
         long allPartitions = 0;
         rangePartitionCounts = Maps.newHashMapWithExpectedSize(ranges.size());
@@ -247,7 +245,7 @@ public CassandraValidationIterator(ColumnFamilyStore cfs, SharedContext ctx, Col
     @Override
     public long getBytesRead()
     {
-        return ci.getBytesRead();
+        return ci.bytesRead();
     }
 
     @Override
@@ -282,6 +280,12 @@ public TableMetadata metadata()
         return cfs.metadata.get();
     }
 
+    @Override
+    public CompactionIterator getCompactionIterator()
+    {
+        return ci;
+    }
+
     @Override
     public boolean hasNext()
     {
diff --git a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
index c39a667be90d..94d4e76664cd 100644
--- a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
+++ b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
@@ -44,9 +44,9 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.compaction.CompactionInfo;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -147,8 +147,8 @@ public boolean apply(SSTableReader sstable)
                 }
                 return false;
             }
-            Collection<CompactionInfo> cis = CompactionManager.instance.active.getCompactionsForSSTable(sstable, OperationType.ANTICOMPACTION);
-            if (cis != null && !cis.isEmpty())
+            Collection<TableOperation.Progress> ops = CompactionManager.instance.active.getOperationsForSSTable(sstable, OperationType.ANTICOMPACTION);
+            if (ops != null && !ops.isEmpty())
             {
                 // todo: start tracking the parent repair session id that created the anticompaction to be able to give a better error messsage here:
                 StringBuilder sb = new StringBuilder();
@@ -157,8 +157,10 @@ public boolean apply(SSTableReader sstable)
                 sb.append(" has failed because it encountered intersecting sstables belonging to another incremental repair session. ");
                 sb.append("This is caused by starting multiple conflicting incremental repairs at the same time. ");
                 sb.append("Conflicting anticompactions: ");
-                for (CompactionInfo ci : cis)
-                    sb.append(ci.getTaskId() == null ? "no compaction id" : ci.getTaskId()).append(':').append(ci.getSSTables()).append(',');
+                for (TableOperation.Progress op : ops)
+                {
+                    sb.append(op.operationId() == null ? "no compaction id" : op.operationId()).append(':').append(op.sstables()).append(',');
+                }
                 throw new SSTableAcquisitionException(sb.toString());
             }
             return true;
@@ -216,7 +218,7 @@ private AcquireResult acquireTuple()
 
         protected AcquireResult acquireSSTables()
         {
-            return cfs.runWithCompactionsDisabled(this::acquireTuple, predicate, OperationType.ANTICOMPACTION, false, false, false);
+            return cfs.runWithCompactionsDisabled(this::acquireTuple, predicate, OperationType.ANTICOMPACTION, false, false, false, TableOperation.StopTrigger.ANTICOMPACTION);
         }
 
         public AcquireResult call()
diff --git a/src/java/org/apache/cassandra/db/rows/AbstractCell.java b/src/java/org/apache/cassandra/db/rows/AbstractCell.java
index 69ca0b1c315d..79a52b8631d7 100644
--- a/src/java/org/apache/cassandra/db/rows/AbstractCell.java
+++ b/src/java/org/apache/cassandra/db/rows/AbstractCell.java
@@ -171,6 +171,11 @@ public long maxTimestamp()
         return timestamp();
     }
 
+    public long minTimestamp()
+    {
+        return timestamp();
+    }
+
     public static <V1, V2> boolean equals(Cell<V1> left, Cell<V2> right)
     {
         return left.column().equals(right.column())
diff --git a/src/java/org/apache/cassandra/db/rows/ArrayCell.java b/src/java/org/apache/cassandra/db/rows/ArrayCell.java
index 07823d2be515..90c20e978c9f 100644
--- a/src/java/org/apache/cassandra/db/rows/ArrayCell.java
+++ b/src/java/org/apache/cassandra/db/rows/ArrayCell.java
@@ -127,7 +127,7 @@ public long unsharedHeapSizeExcludingData()
     }
 
     @Override
-    protected int localDeletionTimeAsUnsignedInt()
+    public int localDeletionTimeAsUnsignedInt()
     {
         return localDeletionTimeUnsignedInteger;
     }
diff --git a/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java b/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java
index ed6e39a5a299..40402fece5fd 100644
--- a/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java
+++ b/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java
@@ -56,4 +56,4 @@ public String toString(TableMetadata metadata)
     {
         return String.format("LowerBoundMarker %s", bound.toString(metadata));
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/db/rows/BTreeRow.java b/src/java/org/apache/cassandra/db/rows/BTreeRow.java
index 075a4f67fe6d..6c3dc33ef2bc 100644
--- a/src/java/org/apache/cassandra/db/rows/BTreeRow.java
+++ b/src/java/org/apache/cassandra/db/rows/BTreeRow.java
@@ -204,6 +204,34 @@ public <A> void apply(BiConsumer<A, ColumnData> function, A arg)
         BTree.apply(btree, function, arg);
     }
 
+    /**
+     * Computes the maximum timestamp for any data (deletion info, PK liveness or cell) in this row.
+     */
+    public long maxTimestamp()
+    {
+        long maxTs = Math.max(primaryKeyLivenessInfo().timestamp(), deletion().time().markedForDeleteAt());
+        return reduce(maxTs, (ts, cd) -> Math.max(ts, cd.maxTimestamp()));
+    }
+
+    /**
+     * Computes the minimum timestamp for any data (deletion info, PK liveness or cell) in this row.
+     */
+    public long minTimestamp()
+    {
+        long minTs = Long.MAX_VALUE;
+        if (!primaryKeyLivenessInfo().isEmpty())
+            minTs = Math.min(minTs, primaryKeyLivenessInfo().timestamp());
+        if (!deletion().isLive())
+            minTs = Math.min(minTs, deletion().time().markedForDeleteAt());
+
+        return reduce(minTs, (ts, cd) -> Math.min(ts, cd.minTimestamp()));
+    }
+
+    public <R> R reduce(R seed, BTree.ReduceFunction<R, ColumnData> reducer)
+    {
+        return BTree.reduce(btree, seed, reducer);
+    }
+
     public long accumulate(LongAccumulator<ColumnData> accumulator, long initialValue)
     {
         return BTree.accumulate(btree, accumulator, initialValue);
@@ -224,7 +252,7 @@ public <A> long accumulate(BiLongAccumulator<A, ColumnData> accumulator, A arg,
         return BTree.accumulate(btree, accumulator, arg, comparator, from, initialValue);
     }
 
-    private static long minDeletionTime(Object[] btree, LivenessInfo info, DeletionTime rowDeletion)
+    public static long minDeletionTime(Object[] btree, LivenessInfo info, DeletionTime rowDeletion)
     {
         long min = Math.min(minDeletionTime(info), minDeletionTime(rowDeletion));
         return BTree.<ColumnData>accumulate(btree, (cd, l) -> Math.min(l, minDeletionTime(cd)), min);
@@ -585,9 +613,7 @@ public static Row merge(BTreeRow existing,
         Object[] existingBtree = existing.btree;
         Object[] updateBtree = update.btree;
 
-        LivenessInfo existingInfo = existing.primaryKeyLivenessInfo();
-        LivenessInfo updateInfo = update.primaryKeyLivenessInfo();
-        LivenessInfo livenessInfo = existingInfo.supersedes(updateInfo) ? existingInfo : updateInfo;
+        LivenessInfo livenessInfo = LivenessInfo.merge(update.primaryKeyLivenessInfo(), existing.primaryKeyLivenessInfo());
 
         Row.Deletion rowDeletion = existing.deletion().supersedes(update.deletion()) ? existing.deletion() : update.deletion();
 
@@ -597,11 +623,19 @@ else if (rowDeletion.isShadowedBy(livenessInfo))
             rowDeletion = Row.Deletion.LIVE;
 
         DeletionTime deletion = rowDeletion.time();
+        Object[] tree = mergeRowBTrees(reconcileF, existingBtree, updateBtree, deletion, existing.deletion().time());
+        return new BTreeRow(existing.clustering, livenessInfo, rowDeletion, tree, minDeletionTime(tree, livenessInfo, deletion));
+    }
+
+    public static Object[] mergeRowBTrees(ColumnData.PostReconciliationFunction reconcileF,
+                                          Object[] existingBtree, Object[] updateBtree,
+                                          DeletionTime deletion, DeletionTime existingDeletion)
+    {
         try (ColumnData.Reconciler reconciler = ColumnData.reconciler(reconcileF, deletion))
         {
-            if (!rowDeletion.isLive())
+            if (!deletion.isLive())
             {
-                if (rowDeletion == existing.deletion())
+                if (deletion == existingDeletion)
                 {
                     updateBtree = BTree.transformAndFilter(updateBtree, reconciler::retain);
                 }
@@ -610,11 +644,23 @@ else if (rowDeletion.isShadowedBy(livenessInfo))
                     existingBtree = BTree.transformAndFilter(existingBtree, reconciler::retain);
                 }
             }
-            Object[] tree = BTree.update(existingBtree, updateBtree, ColumnData.comparator, reconciler);
-            return new BTreeRow(existing.clustering, livenessInfo, rowDeletion, tree, minDeletionTime(tree, livenessInfo, deletion));
+            return BTree.update(existingBtree, updateBtree, ColumnData.comparator, reconciler);
         }
     }
 
+    /**
+     * Exposed for TrieBackedPartition.
+     */
+    public Object[] getBTree()
+    {
+        return btree;
+    }
+
+    public long getMinLocalDeletionTime()
+    {
+        return minLocalDeletionTime;
+    }
+
     private class CellIterator extends AbstractIterator<Cell<?>>
     {
         private Iterator<ColumnData> columnData = iterator();
diff --git a/src/java/org/apache/cassandra/db/rows/BufferCell.java b/src/java/org/apache/cassandra/db/rows/BufferCell.java
index d6918533e868..85a2e3aeb458 100644
--- a/src/java/org/apache/cassandra/db/rows/BufferCell.java
+++ b/src/java/org/apache/cassandra/db/rows/BufferCell.java
@@ -159,7 +159,7 @@ public long unsharedHeapSizeExcludingData()
     }
 
     @Override
-    protected int localDeletionTimeAsUnsignedInt()
+    public int localDeletionTimeAsUnsignedInt()
     {
         return localDeletionTimeUnsignedInteger;
     }
diff --git a/src/java/org/apache/cassandra/db/rows/Cell.java b/src/java/org/apache/cassandra/db/rows/Cell.java
index d60fdda5a012..e630ec0c38a2 100644
--- a/src/java/org/apache/cassandra/db/rows/Cell.java
+++ b/src/java/org/apache/cassandra/db/rows/Cell.java
@@ -94,7 +94,7 @@ public static long getVersionedMaxDeletiontionTime()
             // The whole cluster is 2016, we're out of the 2038/2106 mixed cluster scenario. Shortcut to avoid the 'minClusterVersion' volatile read
             return Cell.MAX_DELETION_TIME;
         else
-            return MessagingService.instance().versions.minClusterVersion >= MessagingService.VERSION_50
+            return MessagingService.Version.supportsExtendedDeletionTime(MessagingService.instance().versions.minClusterVersion)
                    ? Cell.MAX_DELETION_TIME
                    : Cell.MAX_DELETION_TIME_2038_LEGACY_CAP;
     }
@@ -212,8 +212,8 @@ public final Cell<?> clone(Cloner cloner)
     @Override
     // Overrides super type to provide a more precise return type.
     public abstract Cell<?> purgeDataOlderThan(long timestamp);
-    
-    protected abstract int localDeletionTimeAsUnsignedInt();
+
+    public abstract int localDeletionTimeAsUnsignedInt();
 
     /**
      * Handle unsigned encoding and potentially invalid localDeletionTime.
@@ -226,9 +226,8 @@ public static long decodeLocalDeletionTime(long localDeletionTime, int ttl, Dese
         if (localDeletionTime < 0)
         {
             // Overflown signed int, decode to long. The result is guaranteed > ttl (and any signed int)
-            return helper.version < MessagingService.VERSION_50
-                   ? INVALID_DELETION_TIME
-                   : deletionTimeUnsignedIntegerToLong((int) localDeletionTime);
+            return MessagingService.Version.supportsExtendedDeletionTime(helper.version)
+                   ? deletionTimeUnsignedIntegerToLong((int) localDeletionTime) : INVALID_DELETION_TIME;
         }
 
         if (ttl == LivenessInfo.EXPIRED_LIVENESS_TTL)
@@ -257,7 +256,7 @@ public static long decodeLocalDeletionTime(long localDeletionTime, int ttl, Dese
      *   - [ value ]: the cell value, unless it has the HAS_EMPTY_VALUE_MASK.
      *   - [ path ]: the cell path if the column this is a cell of is complex.
      */
-    static class Serializer
+    public static class Serializer
     {
         private final static int IS_DELETED_MASK             = 0x01; // Whether the cell is a tombstone or not.
         private final static int IS_EXPIRING_MASK            = 0x02; // Whether the cell is expiring.
diff --git a/src/java/org/apache/cassandra/db/rows/ColumnData.java b/src/java/org/apache/cassandra/db/rows/ColumnData.java
index b9f19dc07fce..bc821c2166b7 100644
--- a/src/java/org/apache/cassandra/db/rows/ColumnData.java
+++ b/src/java/org/apache/cassandra/db/rows/ColumnData.java
@@ -291,4 +291,6 @@ public static void digest(Digest digest, ColumnData cd)
     public abstract ColumnData purgeDataOlderThan(long timestamp);
 
     public abstract long maxTimestamp();
+
+    public abstract long minTimestamp();
 }
diff --git a/src/java/org/apache/cassandra/db/rows/ColumnMetadataVersionComparator.java b/src/java/org/apache/cassandra/db/rows/ColumnMetadataVersionComparator.java
index 6b2d97c8370c..4b5403246b62 100644
--- a/src/java/org/apache/cassandra/db/rows/ColumnMetadataVersionComparator.java
+++ b/src/java/org/apache/cassandra/db/rows/ColumnMetadataVersionComparator.java
@@ -35,7 +35,7 @@
  *     cannot guarantee when that's fully done).</li>
  * </ul>
  */
-final class ColumnMetadataVersionComparator implements Comparator<ColumnMetadata>
+public final class ColumnMetadataVersionComparator implements Comparator<ColumnMetadata>
 {
     public static final Comparator<ColumnMetadata> INSTANCE = new ColumnMetadataVersionComparator();
 
diff --git a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java
index dea77413c09d..033dbf4919f2 100644
--- a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java
+++ b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java
@@ -70,6 +70,11 @@ public class ComplexColumnData extends ColumnData implements Iterable<Cell<?>>
         this.complexDeletion = complexDeletion;
     }
 
+    // Used by CNDB
+    public boolean hasCells() {
+        return !BTree.isEmpty(this.cells);
+    }
+
     public int cellsCount()
     {
         return BTree.size(cells);
@@ -80,6 +85,11 @@ public Cell<?> getCell(CellPath path)
         return (Cell<?>) BTree.<Object>find(cells, column.asymmetricCellPathComparator(), path);
     }
 
+    public <R> R reduce(R seed, BTree.ReduceFunction<R, Cell> reducer)
+    {
+        return BTree.reduce(cells, seed, reducer);
+    }
+
     public Cell<?> getCellByIndex(int idx)
     {
         return BTree.findByIndex(cells, idx);
@@ -272,6 +282,16 @@ public long maxTimestamp()
         return timestamp;
     }
 
+    public long minTimestamp()
+    {
+        long timestamp = complexDeletion.isLive()
+                         ? Long.MAX_VALUE
+                         : complexDeletion.markedForDeleteAt();
+        for (Cell cell : this)
+            timestamp = Math.min(timestamp, cell.timestamp());
+        return timestamp;
+    }
+
     // This is the partner in crime of ArrayBackedRow.setValue. The exact warning apply. The short
     // version is: "don't use that method".
     void setValue(CellPath path, ByteBuffer value)
diff --git a/src/java/org/apache/cassandra/db/rows/EncodingStats.java b/src/java/org/apache/cassandra/db/rows/EncodingStats.java
index d0f788ae5ae5..dd3f104cade1 100644
--- a/src/java/org/apache/cassandra/db/rows/EncodingStats.java
+++ b/src/java/org/apache/cassandra/db/rows/EncodingStats.java
@@ -97,19 +97,37 @@ public EncodingStats(long minTimestamp,
      */
     public EncodingStats mergeWith(EncodingStats that)
     {
-        long minTimestamp = this.minTimestamp == TIMESTAMP_EPOCH
-                            ? that.minTimestamp
-                            : (that.minTimestamp == TIMESTAMP_EPOCH ? this.minTimestamp : Math.min(this.minTimestamp, that.minTimestamp));
 
-        long minDelTime = this.minLocalDeletionTime == DELETION_TIME_EPOCH
-                         ? that.minLocalDeletionTime
-                         : (that.minLocalDeletionTime == DELETION_TIME_EPOCH ? this.minLocalDeletionTime : Math.min(this.minLocalDeletionTime, that.minLocalDeletionTime));
+        return new EncodingStats(mergeMinTimestamp(this.minTimestamp, that),
+                                 mergeMinLocalDeletionTime(this.minLocalDeletionTime, that),
+                                 mergeMinTTL(this.minTTL, that));
+    }
+
+    public static long mergeMinTimestamp(long minTimestamp, EncodingStats stats)
+    {
+        return minTimestamp == TIMESTAMP_EPOCH
+               ? stats.minTimestamp
+               : (stats.minTimestamp == TIMESTAMP_EPOCH
+                  ? minTimestamp
+                  : Math.min(minTimestamp, stats.minTimestamp));
+    }
 
-        int minTTL = this.minTTL == TTL_EPOCH
-                     ? that.minTTL
-                     : (that.minTTL == TTL_EPOCH ? this.minTTL : Math.min(this.minTTL, that.minTTL));
+    public static long mergeMinLocalDeletionTime(long minLocalDeletionTime, EncodingStats stats)
+    {
+        return minLocalDeletionTime == DELETION_TIME_EPOCH
+               ? stats.minLocalDeletionTime
+               : (stats.minLocalDeletionTime == DELETION_TIME_EPOCH
+                  ? minLocalDeletionTime
+                  : Math.min(minLocalDeletionTime, stats.minLocalDeletionTime));
+    }
 
-        return new EncodingStats(minTimestamp, minDelTime, minTTL);
+    public static int mergeMinTTL(int minTTL, EncodingStats stats)
+    {
+        return minTTL == TTL_EPOCH
+               ? stats.minTTL
+               : (stats.minTTL == TTL_EPOCH
+                  ? minTTL
+                  : Math.min(minTTL, stats.minTTL));
     }
 
     /**
@@ -265,6 +283,13 @@ public static EncodingStats collect(Row staticRow, Iterator<Row> rows, DeletionI
                 Rows.collectStats(rows.next(), collector);
             return collector.get();
         }
+
+        public static EncodingStats forRow(Row row)
+        {
+            Collector collector = new Collector();
+            Rows.collectStats(row, collector);
+            return collector.get();
+        }
     }
 
     public static class Serializer
diff --git a/src/java/org/apache/cassandra/db/rows/NativeCell.java b/src/java/org/apache/cassandra/db/rows/NativeCell.java
index a876e7d59577..1003e5573907 100644
--- a/src/java/org/apache/cassandra/db/rows/NativeCell.java
+++ b/src/java/org/apache/cassandra/db/rows/NativeCell.java
@@ -201,7 +201,7 @@ private boolean hasPath()
     }
 
     @Override
-    protected int localDeletionTimeAsUnsignedInt()
+    public int localDeletionTimeAsUnsignedInt()
     {
         return MemoryUtil.getInt(peer + DELETION);
     }
diff --git a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java
index 67f43c910c72..ae039a555e03 100644
--- a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java
+++ b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java
@@ -166,6 +166,18 @@ public long unsharedHeapSize()
         return EMPTY_SIZE + deletion.unsharedHeapSize();
     }
 
+    @Override
+    public long minTimestamp()
+    {
+        return deletion.markedForDeleteAt();
+    }
+
+    @Override
+    public long maxTimestamp()
+    {
+        return deletion.markedForDeleteAt();
+    }
+
     public String toString(TableMetadata metadata)
     {
         return String.format("Marker %s@%d/%d", bound.toString(metadata), deletion.markedForDeleteAt(), deletion.localDeletionTime());
diff --git a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java
index c36dcfdd55e6..e78eae6eefd8 100644
--- a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java
+++ b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java
@@ -199,6 +199,18 @@ public long unsharedHeapSize()
         return EMPTY_SIZE + startDeletion.unsharedHeapSize() + endDeletion.unsharedHeapSize();
     }
 
+    @Override
+    public long minTimestamp()
+    {
+        return Math.min(startDeletion.markedForDeleteAt(), endDeletion.markedForDeleteAt());
+    }
+
+    @Override
+    public long maxTimestamp()
+    {
+        return Math.max(startDeletion.markedForDeleteAt(), endDeletion.markedForDeleteAt());
+    }
+
     public String toString(TableMetadata metadata)
     {
         return String.format("Marker %s@%d/%d-%d/%d",
diff --git a/src/java/org/apache/cassandra/db/rows/Row.java b/src/java/org/apache/cassandra/db/rows/Row.java
index 5e0bbaf6edf7..eae51c13f413 100644
--- a/src/java/org/apache/cassandra/db/rows/Row.java
+++ b/src/java/org/apache/cassandra/db/rows/Row.java
@@ -17,13 +17,24 @@
  */
 package org.apache.cassandra.db.rows;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
 import org.apache.cassandra.cache.IMeasurableMemory;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DeletionPurger;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.LivenessInfo;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
@@ -32,6 +43,7 @@
 import org.apache.cassandra.utils.LongAccumulator;
 import org.apache.cassandra.utils.MergeIterator;
 import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.Reducer;
 import org.apache.cassandra.utils.SearchIterator;
 import org.apache.cassandra.utils.btree.BTree;
 import org.apache.cassandra.utils.memory.Cloner;
@@ -358,7 +370,7 @@ public interface Row extends Unfiltered, Iterable<ColumnData>, IMeasurableMemory
      * <p>
      * Currently, the only use of shadowable row deletions is Materialized Views, see CASSANDRA-10261.
      */
-    public static class Deletion
+    public static class Deletion implements IMeasurableMemory
     {
         public static final Deletion LIVE = new Deletion(DeletionTime.LIVE, false);
         private static final long EMPTY_SIZE = ObjectSizes.measure(DeletionTime.build(0, 0));
@@ -790,7 +802,7 @@ public Row[] mergedRows()
             return rows;
         }
 
-        private static class ColumnDataReducer extends MergeIterator.Reducer<ColumnData, ColumnData>
+        private static class ColumnDataReducer extends Reducer<ColumnData, ColumnData>
         {
             private ColumnMetadata column;
             private final List<ColumnData> versions;
@@ -835,7 +847,7 @@ private boolean useColumnMetadata(ColumnMetadata dataColumn)
                 return ColumnMetadataVersionComparator.INSTANCE.compare(column, dataColumn) < 0;
             }
 
-            protected ColumnData getReduced()
+            public ColumnData getReduced()
             {
                 if (column.isSimple())
                 {
@@ -883,14 +895,14 @@ protected ColumnData getReduced()
                 }
             }
 
-            protected void onKeyChange()
+            public void onKeyChange()
             {
                 column = null;
                 versions.clear();
             }
         }
 
-        private static class CellReducer extends MergeIterator.Reducer<Cell<?>, Cell<?>>
+        private static class CellReducer extends Reducer<Cell<?>, Cell<?>>
         {
             private DeletionTime activeDeletion;
             private Cell<?> merged;
@@ -907,12 +919,12 @@ public void reduce(int idx, Cell<?> cell)
                     merged = merged == null ? cell : Cells.reconcile(merged, cell);
             }
 
-            protected Cell<?> getReduced()
+            public Cell<?> getReduced()
             {
                 return merged;
             }
 
-            protected void onKeyChange()
+            public void onKeyChange()
             {
                 merged = null;
             }
diff --git a/src/java/org/apache/cassandra/db/rows/Rows.java b/src/java/org/apache/cassandra/db/rows/Rows.java
index df9ff5e28125..82cbaae304ad 100644
--- a/src/java/org/apache/cassandra/db/rows/Rows.java
+++ b/src/java/org/apache/cassandra/db/rows/Rows.java
@@ -17,16 +17,24 @@
  */
 package org.apache.cassandra.db.rows;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
 
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
 
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.SimpleBuilders;
+import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
 import org.apache.cassandra.utils.MergeIterator;
+import org.apache.cassandra.utils.Reducer;
 
 /**
  * Static utilities to work on Row objects.
@@ -97,9 +105,8 @@ private static int unpackColumnCount(long v)
      *
      * @param row the row for which to collect stats.
      * @param collector the stats collector.
-     * @return the total number of cells in {@code row}.
      */
-    public static int collectStats(Row row, PartitionStatisticsCollector collector)
+    public static void collectStats(Row row, PartitionStatisticsCollector collector)
     {
         assert !row.isEmpty();
 
@@ -109,7 +116,26 @@ public static int collectStats(Row row, PartitionStatisticsCollector collector)
         long result = row.accumulate(StatsAccumulation::accumulateOnColumnData, collector, 0);
 
         collector.updateColumnSetPerRow(StatsAccumulation.unpackColumnCount(result));
-        return StatsAccumulation.unpackCellCount(result);
+    }
+
+    public static long collectMaxTimestamp(Row row)
+    {
+        long maxTimestamp = row.primaryKeyLivenessInfo().timestamp();
+        for (ColumnData cd : row)
+        {
+            if (cd.column().isSimple())
+            {
+                maxTimestamp = Math.max(maxTimestamp, ((Cell<?>)cd).timestamp());
+            }
+            else
+            {
+                ComplexColumnData complexData = (ComplexColumnData)cd;
+                maxTimestamp = Math.max(maxTimestamp, complexData.complexDeletion().markedForDeleteAt());
+                for (Cell<?> cell : complexData)
+                    maxTimestamp = Math.max(maxTimestamp, cell.timestamp());
+            }
+        }
+        return maxTimestamp;
     }
 
     /**
@@ -145,7 +171,7 @@ public static void diff(RowDiffListener diffListener, Row merged, Row...inputs)
         for (Row row : inputs)
             inputIterators.add(row == null ? Collections.emptyIterator() : row.iterator());
 
-        Iterator<?> iter = MergeIterator.get(inputIterators, ColumnData.comparator, new MergeIterator.Reducer<ColumnData, Object>()
+        Iterator<?> iter = MergeIterator.get(inputIterators, ColumnData.comparator, new Reducer<ColumnData, Object>()
         {
             ColumnData mergedData;
             ColumnData[] inputDatas = new ColumnData[inputs.length];
@@ -157,7 +183,7 @@ public void reduce(int idx, ColumnData current)
                     inputDatas[idx - 1] = current;
             }
 
-            protected Object getReduced()
+            public Object getReduced()
             {
                 for (int i = 0 ; i != inputDatas.length ; i++)
                 {
@@ -219,7 +245,7 @@ else if (cmp < 0)
                 return null;
             }
 
-            protected void onKeyChange()
+            public void onKeyChange()
             {
                 mergedData = null;
                 Arrays.fill(inputDatas, null);
diff --git a/src/java/org/apache/cassandra/db/rows/Unfiltered.java b/src/java/org/apache/cassandra/db/rows/Unfiltered.java
index 4a90ded50d1f..3f344f22c88d 100644
--- a/src/java/org/apache/cassandra/db/rows/Unfiltered.java
+++ b/src/java/org/apache/cassandra/db/rows/Unfiltered.java
@@ -83,4 +83,16 @@ default boolean isRangeTombstoneMarker()
     {
         return kind() == Kind.RANGE_TOMBSTONE_MARKER;
     }
+
+    /**
+     * Minimum the timestamps of all data in the row or marker.
+     * Note: deletion times are timestamps too, e.g. the min and max timestamp of a range marker is its deletion time.
+     */
+    public long minTimestamp();
+
+    /**
+     * Maximum the timestamps of all data in the row or marker.
+     * Note: deletion times are timestamps too, e.g. the min and max timestamp of a range marker is its deletion time.
+     */
+    public long maxTimestamp();
 }
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
index 53a3ba37cbf8..93370fd14a7f 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
@@ -41,7 +41,6 @@
 import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.utils.IteratorWithLowerBound;
 
 /**
  * An unfiltered row iterator with a lower bound retrieved from either the global
@@ -51,15 +50,23 @@
  * the result is that if we don't need to access this sstable, i.e. due to the LIMIT conditon,
  * then we will not. See CASSANDRA-8180 for examples of why this is useful.
  */
-public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilteredRowIterator implements IteratorWithLowerBound<Unfiltered>
+public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilteredRowIterator
 {
+    enum State
+    {
+        LOWER_BOUND_NOT_REQUESTED,
+        LOWER_BOUND_REQUESTED,
+        LOWER_BOUND_PRODUCED,
+        PRODUCING_ITEMS;
+    }
+
     private final SSTableReader sstable;
     private final Slices slices;
     private final boolean isReverseOrder;
     private final ColumnFilter selectedColumns;
     private final SSTableReadsListener listener;
     private Optional<Unfiltered> lowerBoundMarker;
-    private boolean firstItemRetrieved;
+    private State state;
 
     public UnfilteredRowIteratorWithLowerBound(DecoratedKey partitionKey,
                                                SSTableReader sstable,
@@ -84,9 +91,20 @@ public UnfilteredRowIteratorWithLowerBound(DecoratedKey partitionKey,
         this.isReverseOrder = isReverseOrder;
         this.selectedColumns = selectedColumns;
         this.listener = listener;
-        this.firstItemRetrieved = false;
+        this.state = State.LOWER_BOUND_NOT_REQUESTED;
     }
 
+    /**
+     * Request that the iterator produce an artificial lower bound (i.e. an ineffective range tombstone that is used to
+     * delay opening the sstable until the iteration reaches the clustering range that the sstable covers).
+     */
+    public void requestLowerBound()
+    {
+        assert state == State.LOWER_BOUND_NOT_REQUESTED || state == State.LOWER_BOUND_REQUESTED;
+        state = State.LOWER_BOUND_REQUESTED;
+    }
+
+    @VisibleForTesting
     public Unfiltered lowerBound()
     {
         if (lowerBoundMarker != null)
@@ -126,21 +144,35 @@ protected UnfilteredRowIterator initializeIterator()
     @Override
     protected Unfiltered computeNext()
     {
-        Unfiltered ret = super.computeNext();
-        if (firstItemRetrieved)
-            return ret;
-
-        // Check that the lower bound is not bigger than the first item retrieved
-        firstItemRetrieved = true;
         Unfiltered lowerBound = lowerBound();
-        if (lowerBound != null && ret != null)
-            assert comparator().compare(lowerBound.clustering(), ret.clustering()) <= 0
-            : String.format("Lower bound [%s ]is bigger than first returned value [%s] for sstable %s",
-                            lowerBound.clustering().toString(metadata()),
-                            ret.toString(metadata()),
-                            sstable.getFilename());
-
-        return ret;
+        switch (state)
+        {
+            case LOWER_BOUND_REQUESTED:
+                if (lowerBound != null)
+                {
+                    state = State.LOWER_BOUND_PRODUCED;
+                    return lowerBound;
+                }
+                break;
+            case LOWER_BOUND_PRODUCED:
+                state = State.PRODUCING_ITEMS;
+                Unfiltered ret = super.computeNext();
+
+                // Check that the lower bound is not bigger than the first item retrieved
+                if (lowerBound != null && ret != null)
+                    assert comparator().compare(lowerBound.clustering(), ret.clustering()) <= 0
+                    : String.format("Lower bound [%s ]is bigger than first returned value [%s] for sstable %s",
+                                    lowerBound.clustering().toString(metadata()),
+                                    ret.toString(metadata()),
+                                    sstable.getFilename());
+
+                return ret;
+        }
+
+        // if the bound was not requested, was null, or we have already produced it and the first item, pass on all
+        // items from the source
+        state = State.PRODUCING_ITEMS;
+        return super.computeNext();
     }
 
     private Comparator<Clusterable> comparator()
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
index 7ccc6ff97077..c6e41a99ca13 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
@@ -33,12 +33,14 @@
 import org.apache.cassandra.db.transform.MoreRows;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.CloseableIterator;
 import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.IMergeIterator;
 import org.apache.cassandra.utils.MergeIterator;
+import org.apache.cassandra.utils.Reducer;
 
 /**
  * Static methods to work with atom iterators.
@@ -303,12 +305,12 @@ public Unfiltered next()
      * This is mainly used by scrubber to detect problems in sstables.
      *
      * @param iterator the partition to check.
-     * @param filename the name of the file the data is comming from.
+     * @param file the data is comming from.
      * @return an iterator that returns the same data than {@code iterator} but that
      * checks said data and throws a {@code CorruptedSSTableException} if it detects
      * invalid data.
      */
-    public static UnfilteredRowIterator withValidation(UnfilteredRowIterator iterator, final String filename)
+    public static UnfilteredRowIterator withValidation(UnfilteredRowIterator iterator, final File file)
     {
         class Validator extends Transformation
         {
@@ -341,7 +343,7 @@ private void validate(Unfiltered unfiltered)
                 }
                 catch (MarshalException me)
                 {
-                    throw new CorruptSSTableException(me, filename);
+                    throw new CorruptSSTableException(me, file);
                 }
             }
         }
@@ -397,7 +399,7 @@ public RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker)
      */
     private static class UnfilteredRowMergeIterator extends AbstractUnfilteredRowIterator
     {
-        private final IMergeIterator<Unfiltered, Unfiltered> mergeIterator;
+        private final CloseableIterator<Unfiltered> mergeIterator;
         private final MergeListener listener;
 
         private UnfilteredRowMergeIterator(TableMetadata metadata,
@@ -415,9 +417,21 @@ private UnfilteredRowMergeIterator(TableMetadata metadata,
                   reversed,
                   EncodingStats.merge(iterators, UnfilteredRowIterator::stats));
 
-            this.mergeIterator = MergeIterator.get(iterators,
-                                                   reversed ? metadata.comparator.reversed() : metadata.comparator,
-                                                   new MergeReducer(iterators.size(), reversed, listener));
+            // If merging more than 1 source, ask iterators to provide artificial lower bounds which will help to delay
+            // opening sstables until they are needed. The tomsbtone processing will throw these ineffective bounds away
+            // (they are in the form of range tombstone markers with DeletionTime.LIVE).
+            if (iterators.size() > 1)
+            {
+                for (UnfilteredRowIterator iter : iterators)
+                {
+                    if (iter instanceof UnfilteredRowIteratorWithLowerBound)
+                        ((UnfilteredRowIteratorWithLowerBound) iter).requestLowerBound();
+                }
+            }
+
+            this.mergeIterator = MergeIterator.getCloseable(iterators,
+                                                            reversed ? metadata.comparator.reversed() : metadata.comparator,
+                                                            new MergeReducer(iterators.size(), reversed, listener));
             this.listener = listener;
         }
 
@@ -540,7 +554,7 @@ public void close()
                 listener.close();
         }
 
-        private class MergeReducer extends MergeIterator.Reducer<Unfiltered, Unfiltered>
+        private class MergeReducer extends Reducer<Unfiltered, Unfiltered>
         {
             private final MergeListener listener;
 
@@ -557,7 +571,7 @@ private MergeReducer(int size, boolean reversed, MergeListener listener)
             }
 
             @Override
-            public boolean trivialReduceIsTrivial()
+            public boolean singleSourceReduceIsTrivial()
             {
                 // If we have a listener, we must signal it even when we have a single version
                 return listener == null;
@@ -572,7 +586,7 @@ public void reduce(int idx, Unfiltered current)
                     markerMerger.add(idx, (RangeTombstoneMarker)current);
             }
 
-            protected Unfiltered getReduced()
+            public Unfiltered getReduced()
             {
                 if (nextKind == Unfiltered.Kind.ROW)
                 {
@@ -590,7 +604,7 @@ protected Unfiltered getReduced()
                 }
             }
 
-            protected void onKeyChange()
+            public void onKeyChange()
             {
                 if (nextKind == Unfiltered.Kind.ROW)
                     rowMerger.clear();
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
index 2fcba1bce8ea..7cba7dcc906b 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
@@ -20,16 +20,24 @@
 import java.io.IOException;
 
 import net.nicoulaj.compilecommand.annotations.Inline;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundOrBoundary;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.io.util.TrackedDataInputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.db.rows.Row.Deletion;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.TrackedDataInputPlus;
 import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.utils.SearchIterator;
 import org.apache.cassandra.utils.WrappedException;
 
@@ -101,11 +109,11 @@ public class UnfilteredSerializer
      */
     private final static int END_OF_PARTITION     = 0x01; // Signal the end of the partition. Nothing follows a <flags> field with that flag.
     private final static int IS_MARKER            = 0x02; // Whether the encoded unfiltered is a marker or a row. All following markers applies only to rows.
-    private final static int HAS_TIMESTAMP        = 0x04; // Whether the encoded row has a timestamp (i.e. if row.partitionKeyLivenessInfo().hasTimestamp() == true).
-    private final static int HAS_TTL              = 0x08; // Whether the encoded row has some expiration info (i.e. if row.partitionKeyLivenessInfo().hasTTL() == true).
-    private final static int HAS_DELETION         = 0x10; // Whether the encoded row has some deletion info.
-    private final static int HAS_ALL_COLUMNS      = 0x20; // Whether the encoded row has all of the columns from the header present.
-    private final static int HAS_COMPLEX_DELETION = 0x40; // Whether the encoded row has some complex deletion for at least one of its columns.
+    public final static int HAS_TIMESTAMP        = 0x04; // Whether the encoded row has a timestamp (i.e. if row.partitionKeyLivenessInfo().hasTimestamp() == true).
+    public final static int HAS_TTL              = 0x08; // Whether the encoded row has some expiration info (i.e. if row.partitionKeyLivenessInfo().hasTTL() == true).
+    public final static int HAS_DELETION         = 0x10; // Whether the encoded row has some deletion info.
+    public final static int HAS_ALL_COLUMNS      = 0x20; // Whether the encoded row has all of the columns from the header present.
+    public final static int HAS_COMPLEX_DELETION = 0x40; // Whether the encoded row has some complex deletion for at least one of its columns.
     private final static int EXTENSION_FLAG       = 0x80; // If present, another byte is read containing the "extended flags" above.
 
     /*
@@ -582,9 +590,9 @@ public Row deserializeRowBody(DataInputPlus in,
 
             if (header.isForSSTable())
             {
-                long rowSize = in.readUnsignedVInt();
-                in.readUnsignedVInt(); // previous unfiltered size
+                int rowSize = Math.toIntExact(in.readUnsignedVInt());
                 in = new TrackedDataInputPlus(in, rowSize);
+                in.readUnsignedVInt(); // previous unfiltered size
             }
 
             LivenessInfo rowLiveness = LivenessInfo.EMPTY;
@@ -670,10 +678,10 @@ private void readComplexColumn(ColumnMetadata column, DataInputPlus in, Serializ
                 DeletionTime complexDeletion = header.readDeletionTime(in);
                 if (complexDeletion.localDeletionTime() < 0)
                 {
-                    if (helper.version < MessagingService.VERSION_50)
-                        complexDeletion = DeletionTime.build(complexDeletion.markedForDeleteAt(), Cell.INVALID_DELETION_TIME);
-                    else
+                    if (MessagingService.Version.supportsExtendedDeletionTime(helper.version))
                         complexDeletion = DeletionTime.build(complexDeletion.markedForDeleteAt(), Cell.deletionTimeUnsignedIntegerToLong((int) complexDeletion.localDeletionTime()));
+                    else
+                        complexDeletion = DeletionTime.build(complexDeletion.markedForDeleteAt(), Cell.INVALID_DELETION_TIME);
                 }
                 if (!helper.isDroppedComplexDeletion(complexDeletion))
                     builder.addComplexDeletion(column, complexDeletion);
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java
index 806a74a35c30..0300595b19ea 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java
@@ -47,12 +47,14 @@ public class CassandraCompressedStreamWriter extends CassandraStreamWriter
 
     private final CompressionInfo compressionInfo;
     private final long totalSize;
+    private final long onDiskOffset;
 
     public CassandraCompressedStreamWriter(SSTableReader sstable, CassandraStreamHeader header, StreamSession session)
     {
         super(sstable, header, session);
         this.compressionInfo = header.compressionInfo;
         this.totalSize = header.size();
+        this.onDiskOffset = sstable.getCompressionMetadata().chunkFor(sstable.getDataFileSliceDescriptor().sliceStart).offset;
     }
 
     @Override
@@ -84,7 +86,10 @@ public void write(StreamingDataOutputPlus out) throws IOException
                 while (bytesTransferred < length)
                 {
                     int toTransfer = (int) Math.min(CHUNK_SIZE, length - bytesTransferred);
-                    long position = section.start + bytesTransferred;
+                    // since we access the file directly (not through the rebufferer) we need to adjust the position
+                    // manually when dealing with a slice (see ZeroCopyMetadata); therefore we subtract the onDiskOffset
+                    // by which all the section positions are translated
+                    long position = section.start + bytesTransferred - onDiskOffset;
 
                     out.writeToChannel(bufferSupplier -> {
                         ByteBuffer outBuffer = bufferSupplier.get(toTransfer);
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
index 97c3b2d4f9e3..9ef105332c2a 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
@@ -22,6 +22,8 @@
 import java.util.Collection;
 import java.util.function.UnaryOperator;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,6 +57,8 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader
 {
     private static final Logger logger = LoggerFactory.getLogger(CassandraEntireSSTableStreamReader.class);
 
+    private static final boolean SKIP_MUTATING_STATS_AFTER_ZCS = CassandraRelevantProperties.SKIP_MUTATING_STATS_AFTER_ZCS.getBoolean();
+
     private final TableId tableId;
     private final StreamSession session;
     private final StreamMessageHeader messageHeader;
@@ -133,11 +137,22 @@ public SSTableMultiWriter read(DataInputPlus in) throws IOException
                              prettyPrintMemory(totalSize));
             }
 
-            UnaryOperator<StatsMetadata> transform = stats -> stats.mutateLevel(header.sstableLevel)
-                                                                   .mutateRepairedMetadata(messageHeader.repairedAt, messageHeader.pendingRepair, false);
-            String description = String.format("level %s and repairedAt time %s and pendingRepair %s",
-                                               header.sstableLevel, messageHeader.repairedAt, messageHeader.pendingRepair);
-            writer.descriptor.getMetadataSerializer().mutate(writer.descriptor, description, transform);
+            if (!SKIP_MUTATING_STATS_AFTER_ZCS)
+            {
+                UnaryOperator<StatsMetadata> transform = stats -> stats.mutateLevel(header.sstableLevel)
+                                                                       .mutateRepairedMetadata(messageHeader.repairedAt, messageHeader.pendingRepair, false);
+                String description = String.format("level %s and repairedAt time %s and pendingRepair %s",
+                                                   header.sstableLevel, messageHeader.repairedAt, messageHeader.pendingRepair);
+                writer.descriptor.getMetadataSerializer().mutate(writer.descriptor, description, transform);
+            }
+            else
+            {
+                logger.debug("[Stream #{}] Skipped mutating {} component from {} for sstable {} by config -Dcassandra.skip_mutating_stats_after_zcs",
+                             session.planId(),
+                             SSTableFormat.Components.STATS,
+                             session.peer,
+                             writer.descriptor);
+            }
             return writer;
         }
         catch (Throwable e)
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java
index 7572749d37e2..984046dad588 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java
@@ -42,7 +42,7 @@
  */
 public class CassandraOutgoingFile implements OutgoingStream
 {
-    private final Ref<SSTableReader> ref;
+    private final Ref<? extends SSTableReader> ref;
     private final long estimatedKeys;
     private final List<SSTableReader.PartitionPositionBounds> sections;
     private final String filename;
@@ -50,7 +50,7 @@ public class CassandraOutgoingFile implements OutgoingStream
     private final StreamOperation operation;
     private final CassandraStreamHeader header;
 
-    public CassandraOutgoingFile(StreamOperation operation, Ref<SSTableReader> ref,
+    public CassandraOutgoingFile(StreamOperation operation, Ref<? extends SSTableReader> ref,
                                  List<SSTableReader.PartitionPositionBounds> sections, List<Range<Token>> normalizedRanges,
                                  long estimatedKeys)
     {
@@ -102,7 +102,7 @@ public static CassandraOutgoingFile fromStream(OutgoingStream stream)
     }
 
     @VisibleForTesting
-    public Ref<SSTableReader> getRef()
+    public Ref<? extends SSTableReader> getRef()
     {
         return ref;
     }
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
index d122c9c78852..df084cefd0ab 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
@@ -143,7 +143,7 @@ else if (pendingRepair == ActiveRepairService.NO_PENDING_REPAIR)
                 List<Range<Token>> ranges = sstable.isRepaired() ? normalizedFullRanges : normalizedAllRanges;
                 List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(ranges);
 
-                Ref<SSTableReader> ref = refs.get(sstable);
+                Ref<? extends SSTableReader> ref = refs.get(sstable);
                 if (sections.isEmpty())
                 {
                     ref.release();
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
index ea911d629d4b..7bea8f5e490d 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
@@ -36,6 +36,7 @@
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.rows.DeserializationHelper;
 import org.apache.cassandra.db.rows.EncodingStats;
@@ -171,7 +172,7 @@ protected StreamDeserializer getDeserializer(TableMetadata metadata,
 
     protected SerializationHeader getHeader(TableMetadata metadata) throws UnknownColumnException
     {
-        return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader
+        return header != null? header.toHeader("stream from " + session.peer, metadata, inputVersion, false) : null; //pre-3.0 sstable have no SerializationHeader
     }
     protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat<?, ?> format) throws IOException
     {
@@ -183,7 +184,18 @@ protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize,
         Preconditions.checkState(streamReceiver instanceof CassandraStreamReceiver);
         LifecycleNewTracker lifecycleNewTracker = CassandraStreamReceiver.fromReceiver(session.getAggregator(tableId)).createLifecycleNewTracker();
 
-        RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, false, format, sstableLevel, totalSize, lifecycleNewTracker, getHeader(cfs.metadata()));
+        RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs,
+                                                                     estimatedKeys,
+                                                                     repairedAt,
+                                                                     pendingRepair,
+                                                                     false,
+                                                                     format,
+                                                                     // Commit log intervals for other nodes are not relevant and should not be copied
+                                                                     IntervalSet.empty(),
+                                                                     sstableLevel,
+                                                                     totalSize,
+                                                                     lifecycleNewTracker,
+                                                                     getHeader(cfs.metadata()));
         return writer;
     }
 
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
index 50f87c799ece..0c3683cb6e87 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
@@ -32,6 +32,7 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.WriteOptions;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
@@ -81,7 +82,7 @@ public CassandraStreamReceiver(ColumnFamilyStore cfs, StreamSession session, int
         this.session = session;
         // this is an "offline" transaction, as we currently manually expose the sstables once done;
         // this should be revisited at a later date, so that LifecycleTransaction manages all sstable state changes
-        this.txn = LifecycleTransaction.offline(OperationType.STREAM);
+        this.txn = LifecycleTransaction.offline(OperationType.STREAM, cfs.metadata);
         this.sstables = new ArrayList<>(totalFiles);
         this.requiresWritePath = requiresWritePath(cfs);
     }
@@ -141,6 +142,15 @@ public void trackNew(SSTable table)
                 }
             }
 
+            @Override
+            public void trackNewWritten(SSTable table)
+            {
+                synchronized (CassandraStreamReceiver.this)
+                {
+                    txn.trackNewWritten(table);
+                }
+            }
+
             @Override
             public void untrackNew(SSTable table)
             {
@@ -204,6 +214,7 @@ private void sendThroughWritePath(ColumnFamilyStore cfs, Collection<SSTableReade
         for (SSTableReader reader : readers)
         {
             Keyspace ks = Keyspace.open(reader.getKeyspaceName());
+            WriteOptions opts = WriteOptions.forStreaming(session.streamOperation(), writeCDCCommitLog);
             // When doing mutation-based repair we split each partition into smaller batches
             // ({@link Stream MAX_ROWS_PER_BATCH}) to avoid OOMing and generating heap pressure
             try (ISSTableScanner scanner = reader.getScanner();
@@ -216,10 +227,7 @@ private void sendThroughWritePath(ColumnFamilyStore cfs, Collection<SSTableReade
                     //
                     // If the CFS has CDC, however, these updates need to be written to the CommitLog
                     // so they get archived into the cdc_raw folder
-                    ks.apply(new Mutation(PartitionUpdate.fromIterator(throttledPartitions.next(), filter)),
-                             writeCDCCommitLog,
-                             true,
-                             false);
+                    ks.apply(new Mutation(PartitionUpdate.fromIterator(throttledPartitions.next(), filter)), opts);
                 }
             }
         }
@@ -255,7 +263,7 @@ public void finished()
 
                 // add sstables (this will build non-SSTable-attached secondary indexes too, see CASSANDRA-10130)
                 logger.debug("[Stream #{}] Received {} sstables from {} ({})", session.planId(), readers.size(), session.peer, readers);
-                cfs.addSSTables(readers);
+                cfs.addSSTables(readers, OperationType.STREAM);
 
                 //invalidate row and counter cache
                 if (cfs.isRowCacheEnabled() || cfs.metadata().isCounter())
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java
index cae6fefb2d2a..dc912c1bf031 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java
@@ -142,14 +142,14 @@ protected long totalSize()
     protected long write(ChannelProxy proxy, ChecksumValidator validator, StreamingDataOutputPlus output, long start, int transferOffset, int toTransfer, int bufferSize) throws IOException
     {
         // the count of bytes to read off disk
-        int minReadable = (int) Math.min(bufferSize, proxy.size() - start);
+        int minReadable = (int) Math.min(bufferSize, proxy.size() - (start - sstable.getDataFileSliceDescriptor().sliceStart));
 
         // this buffer will hold the data from disk. as it will be compressed on the fly by
         // AsyncChannelCompressedStreamWriter.write(ByteBuffer), we can release this buffer as soon as we can.
         ByteBuffer buffer = BufferPools.forNetworking().get(minReadable, BufferType.OFF_HEAP);
         try
         {
-            int readCount = proxy.read(buffer, start);
+            int readCount = proxy.read(buffer, start - sstable.getDataFileSliceDescriptor().sliceStart);
             assert readCount == minReadable : String.format("could not read required number of bytes from file to be streamed: read %d bytes, wanted %d bytes", readCount, minReadable);
             buffer.flip();
 
diff --git a/src/java/org/apache/cassandra/db/streaming/ComponentContext.java b/src/java/org/apache/cassandra/db/streaming/ComponentContext.java
index c03e7b4c3436..775b8843256c 100644
--- a/src/java/org/apache/cassandra/db/streaming/ComponentContext.java
+++ b/src/java/org/apache/cassandra/db/streaming/ComponentContext.java
@@ -26,12 +26,19 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 
+import java.util.Collections;
+
+/**
+ * Mutable SSTable components and their hardlinks to avoid concurrent sstable component modification
+ * during entire-sstable-streaming.
+ */
 public class ComponentContext implements AutoCloseable
 {
     private static final Logger logger = LoggerFactory.getLogger(ComponentContext.class);
@@ -47,6 +54,9 @@ private ComponentContext(Map<Component, File> hardLinks, ComponentManifest manif
 
     public static ComponentContext create(SSTable sstable)
     {
+        if (!DatabaseDescriptor.supportsHardlinksForEntireSSTableStreaming())
+            return new ComponentContext(Collections.emptyMap(), ComponentManifest.create(sstable));
+
         Descriptor descriptor = sstable.descriptor;
         Map<Component, File> hardLinks = new HashMap<>(1);
 
diff --git a/src/java/org/apache/cassandra/db/tries/CollectionMergeTrie.java b/src/java/org/apache/cassandra/db/tries/CollectionMergeTrie.java
index 033691049440..04f732627f4a 100644
--- a/src/java/org/apache/cassandra/db/tries/CollectionMergeTrie.java
+++ b/src/java/org/apache/cassandra/db/tries/CollectionMergeTrie.java
@@ -23,6 +23,8 @@
 
 import com.google.common.collect.Iterables;
 
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
 /**
  * A merged view of multiple tries.
  *
@@ -48,9 +50,9 @@ class CollectionMergeTrie<T> extends Trie<T>
     }
 
     @Override
-    protected Cursor<T> cursor()
+    protected Cursor<T> cursor(Direction direction)
     {
-        return new CollectionMergeCursor<>(resolver, inputs);
+        return new CollectionMergeCursor<>(resolver, direction, inputs);
     }
 
     /**
@@ -58,13 +60,13 @@ protected Cursor<T> cursor()
      * - its depth is greater, or
      * - its depth is equal, and the incoming transition is smaller.
      */
-    static <T> boolean greaterCursor(Cursor<T> c1, Cursor<T> c2)
+    static <T> boolean greaterCursor(Direction direction, Cursor<T> c1, Cursor<T> c2)
     {
         int c1depth = c1.depth();
         int c2depth = c2.depth();
         if (c1depth != c2depth)
             return c1depth < c2depth;
-        return c1.incomingTransition() > c2.incomingTransition();
+        return direction.lt(c2.incomingTransition(), c1.incomingTransition());
     }
 
     static <T> boolean equalCursor(Cursor<T> c1, Cursor<T> c2)
@@ -115,6 +117,7 @@ static <T> boolean equalCursor(Cursor<T> c1, Cursor<T> c2)
     static class CollectionMergeCursor<T> implements Cursor<T>
     {
         private final CollectionMergeResolver<T> resolver;
+        private final Direction direction;
 
         /**
          * The smallest cursor, tracked separately to improve performance in single-source sections of the trie.
@@ -133,9 +136,10 @@ static class CollectionMergeCursor<T> implements Cursor<T>
          */
         private final List<T> contents;
 
-        public CollectionMergeCursor(CollectionMergeResolver<T> resolver, Collection<? extends Trie<T>> inputs)
+        public CollectionMergeCursor(CollectionMergeResolver<T> resolver, Direction direction, Collection<? extends Trie<T>> inputs)
         {
             this.resolver = resolver;
+            this.direction = direction;
             int count = inputs.size();
             // Get cursors for all inputs. Put one of them in head and the rest in the heap.
             heap = new Cursor[count - 1];
@@ -143,7 +147,7 @@ public CollectionMergeCursor(CollectionMergeResolver<T> resolver, Collection<? e
             int i = -1;
             for (Trie<T> trie : inputs)
             {
-                Cursor<T> cursor = trie.cursor();
+                Cursor<T> cursor = trie.cursor(direction);
                 assert cursor.depth() == 0;
                 if (i >= 0)
                     heap[i] = cursor;
@@ -155,21 +159,26 @@ public CollectionMergeCursor(CollectionMergeResolver<T> resolver, Collection<? e
         }
 
         /**
-         * Interface for internal operations that can be applied to the equal top elements of the heap.
+         * Interface for internal operations that can be applied to selected top elements of the heap.
          */
         interface HeapOp<T>
         {
             void apply(CollectionMergeCursor<T> self, Cursor<T> cursor, int index);
+
+            default boolean shouldContinueWithChild(Cursor<T> child, Cursor<T> head)
+            {
+                return equalCursor(child, head);
+            }
         }
 
         /**
          * Apply a non-interfering operation, i.e. one that does not change the cursor state, to all inputs in the heap
-         * that are on equal position to the head.
-         * For interfering operations like advancing the cursors, use {@link #advanceEqualAndRestoreHeap(AdvancingHeapOp)}.
+         * that satisfy the {@link HeapOp#shouldContinueWithChild} condition (by default, being equal to the head).
+         * For interfering operations like advancing the cursors, use {@link #advanceSelectedAndRestoreHeap(AdvancingHeapOp)}.
          */
-        private void applyToEqualOnHeap(HeapOp<T> action)
+        private void applyToSelectedInHeap(HeapOp<T> action)
         {
-            applyToEqualElementsInHeap(action, 0);
+            applyToSelectedElementsInHeap(action, 0);
         }
 
         /**
@@ -195,35 +204,36 @@ default void apply(CollectionMergeCursor<T> self, Cursor<T> cursor, int index)
 
 
         /**
-         * Advance the state of all inputs in the heap that are on equal position as the head and restore the heap
-         * invariant.
+         * Advance the state of all inputs in the heap that satisfy the {@link HeapOp#shouldContinueWithChild} condition
+         * (by default, being equal to the head) and restore the heap invariant.
          */
-        private void advanceEqualAndRestoreHeap(AdvancingHeapOp<T> action)
+        private void advanceSelectedAndRestoreHeap(AdvancingHeapOp<T> action)
         {
-            applyToEqualElementsInHeap(action, 0);
+            applyToSelectedElementsInHeap(action, 0);
         }
 
         /**
-         * Apply an operation to all elements on the heap that are equal to the head. Descends recursively in the heap
-         * structure to all equal children and applies the operation on the way back.
-         *
+         * Apply an operation to all elements on the heap that satisfy, recursively through the heap hierarchy, the
+         * {@code shouldContinueWithChild} condition (being equal to the head by default). Descends recursively in the
+         * heap structure to all selected children and applies the operation on the way back.
+         * <p>
          * This operation can be something that does not change the cursor state (see {@link #content}) or an operation
          * that advances the cursor to a new state, wrapped in a {@link AdvancingHeapOp} ({@link #advance} or
-         * {@link #skipChildren}). The latter interface takes care of pushing elements down in the heap after advancing
+         * {@link #skipTo}). The latter interface takes care of pushing elements down in the heap after advancing
          * and restores the subheap state on return from each level of the recursion.
          */
-        private void applyToEqualElementsInHeap(HeapOp<T> action, int index)
+        private void applyToSelectedElementsInHeap(HeapOp<T> action, int index)
         {
             if (index >= heap.length)
                 return;
             Cursor<T> item = heap[index];
-            if (!equalCursor(item, head))
+            if (!action.shouldContinueWithChild(item, head))
                 return;
 
             // If the children are at the same position, they also need advancing and their subheap
             // invariant to be restored.
-            applyToEqualElementsInHeap(action, index * 2 + 1);
-            applyToEqualElementsInHeap(action, index * 2 + 2);
+            applyToSelectedElementsInHeap(action, index * 2 + 1);
+            applyToSelectedElementsInHeap(action, index * 2 + 2);
 
             // Apply the action. This is done on the reverse direction to give the action a chance to form proper
             // subheaps and combine them on processing the parent.
@@ -242,10 +252,10 @@ private void heapifyDown(Cursor<T> item, int index)
                 if (next >= heap.length)
                     break;
                 // Select the smaller of the two children to push down to.
-                if (next + 1 < heap.length && greaterCursor(heap[next], heap[next + 1]))
+                if (next + 1 < heap.length && greaterCursor(direction, heap[next], heap[next + 1]))
                     ++next;
                 // If the child is greater or equal, the invariant has been restored.
-                if (!greaterCursor(item, heap[next]))
+                if (!greaterCursor(direction, item, heap[next]))
                     break;
                 heap[index] = heap[next];
                 index = next;
@@ -263,7 +273,7 @@ private int maybeSwapHead(int headDepth)
         {
             int heap0Depth = heap[0].depth();
             if (headDepth > heap0Depth ||
-                (headDepth == heap0Depth && head.incomingTransition() <= heap[0].incomingTransition()))
+                (headDepth == heap0Depth && direction.le(head.incomingTransition(), heap[0].incomingTransition())))
                 return headDepth;   // head is still smallest
 
             // otherwise we need to swap heap and heap[0]
@@ -273,10 +283,15 @@ private int maybeSwapHead(int headDepth)
             return heap0Depth;
         }
 
+        boolean branchHasMultipleSources()
+        {
+            return equalCursor(heap[0], head);
+        }
+
         @Override
         public int advance()
         {
-            advanceEqualAndRestoreHeap(Cursor::advance);
+            advanceSelectedAndRestoreHeap(Cursor::advance);
             return maybeSwapHead(head.advance());
         }
 
@@ -285,7 +300,7 @@ public int advanceMultiple(TransitionsReceiver receiver)
         {
             // If the current position is present in just one cursor, we can safely descend multiple levels within
             // its branch as no one of the other tries has content for it.
-            if (equalCursor(heap[0], head))
+            if (branchHasMultipleSources())
                 return advance();   // More than one source at current position, do single-step advance.
 
             // If there are no children, i.e. the cursor ascends, we have to check if it's become larger than some
@@ -294,10 +309,36 @@ public int advanceMultiple(TransitionsReceiver receiver)
         }
 
         @Override
-        public int skipChildren()
+        public int skipTo(int skipDepth, int skipTransition)
         {
-            advanceEqualAndRestoreHeap(Cursor::skipChildren);
-            return maybeSwapHead(head.skipChildren());
+            // We need to advance all cursors that stand before the requested position.
+            // If a child cursor does not need to advance as it is greater than the skip position, neither of the ones
+            // below it in the heap hierarchy do as they can't have an earlier position.
+            class SkipTo implements AdvancingHeapOp<T>
+            {
+                @Override
+                public boolean shouldContinueWithChild(Cursor<T> child, Cursor<T> head)
+                {
+                    // When the requested position descends, the inplicit prefix bytes are those of the head cursor,
+                    // and thus we need to check against that if it is a match.
+                    if (equalCursor(child, head))
+                        return true;
+                    // Otherwise we can compare the child's position against a cursor advanced as requested, and need
+                    // to skip only if it would be before it.
+                    int childDepth = child.depth();
+                    return childDepth > skipDepth ||
+                           childDepth == skipDepth && direction.lt(child.incomingTransition(), skipTransition);
+                }
+
+                @Override
+                public void apply(Cursor<T> cursor)
+                {
+                    cursor.skipTo(skipDepth, skipTransition);
+                }
+            }
+
+            applyToSelectedElementsInHeap(new SkipTo(), 0);
+            return maybeSwapHead(head.skipTo(skipDepth, skipTransition));
         }
 
         @Override
@@ -312,10 +353,25 @@ public int incomingTransition()
             return head.incomingTransition();
         }
 
+        @Override
+        public Direction direction()
+        {
+            return direction;
+        }
+
+        @Override
+        public ByteComparable.Version byteComparableVersion()
+        {
+            return head.byteComparableVersion();
+        }
+
         @Override
         public T content()
         {
-            applyToEqualOnHeap(CollectionMergeCursor::collectContent);
+            if (!branchHasMultipleSources())
+                return head.content();
+
+            applyToSelectedInHeap(CollectionMergeCursor::collectContent);
             collectContent(head, -1);
 
             T toReturn;
@@ -341,6 +397,19 @@ private void collectContent(Cursor<T> item, int index)
             if (itemContent != null)
                 contents.add(itemContent);
         }
+
+        @Override
+        public Trie<T> tailTrie()
+        {
+            if (!branchHasMultipleSources())
+                return head.tailTrie();
+
+            List<Trie<T>> inputs = new ArrayList<>(heap.length);
+            inputs.add(head.tailTrie());
+            applyToSelectedInHeap((self, cursor, index) -> inputs.add(cursor.tailTrie()));
+
+            return new CollectionMergeTrie<>(inputs, resolver);
+        }
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/db/tries/Direction.java b/src/java/org/apache/cassandra/db/tries/Direction.java
new file mode 100644
index 000000000000..29f8e2b97b79
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/Direction.java
@@ -0,0 +1,181 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+/**
+ * Class used to specify the direction of iteration. Provides methods used to replace comparisons and values in typical
+ * loops and allow code to be written without explicit direction checks.
+ * <p>
+ * For example, iterating between l and r inclusive in forward direction is usually done as<br/>
+ * {@code for (int i = l; i <= r; ++i) ...}
+ * <p>
+ * To loop over them in the specified direction dir, the loop above would change to<br/>
+ * {@code for (int i = dir.select(l, r); dir.inLoop(i, l, r); i += dir.increase) ...}
+ */
+public enum Direction
+{
+    FORWARD(1)
+    {
+        public boolean inLoop(int index, int left, int right)
+        {
+            return index <= right;
+        }
+
+        public boolean lt(int a, int b)
+        {
+            return a < b;
+        }
+
+        public boolean le(int a, int b)
+        {
+            return a <= b;
+        }
+
+        public int min(int a, int b)
+        {
+            return Math.min(a, b);
+        }
+
+        public int max(int a, int b)
+        {
+            return Math.max(a, b);
+        }
+
+        public <T> T select(T forward, T reverse)
+        {
+            return forward;
+        }
+
+        public int select(int forward, int reverse)
+        {
+            return forward;
+        }
+
+        public boolean isForward()
+        {
+            return true;
+        }
+
+        public Direction opposite()
+        {
+            return REVERSE;
+        }
+    },
+    REVERSE(-1)
+    {
+        public boolean inLoop(int index, int left, int right)
+        {
+            return index >= left;
+        }
+
+        public boolean lt(int a, int b)
+        {
+            return a > b;
+        }
+
+        public boolean le(int a, int b)
+        {
+            return a >= b;
+        }
+
+        public int min(int a, int b)
+        {
+            return Math.max(a, b);
+        }
+
+        public int max(int a, int b)
+        {
+            return Math.min(a, b);
+        }
+
+        public <T> T select(T forward, T reverse)
+        {
+            return reverse;
+        }
+
+        public int select(int forward, int reverse)
+        {
+            return reverse;
+        }
+
+        public boolean isForward()
+        {
+            return false;
+        }
+
+        public Direction opposite()
+        {
+            return FORWARD;
+        }
+    };
+
+    /** Value that needs to be added to advance the iteration, i.e. value corresponding to 1 */
+    public final int increase;
+
+    Direction(int increase)
+    {
+        this.increase = increase;
+    }
+
+    /** Returns the result of the operation corresponding to a < b for the forward direction */
+    public abstract boolean lt(int a, int b);
+    /** Returns the result of the operation corresponding to a>b for the forward direction */
+    public boolean gt(int a, int b)
+    {
+        return lt(b, a);
+    }
+    /** Returns the result of the operation corresponding to a<=b for the forward direction */
+    public abstract boolean le(int a, int b);
+    /** Returns the result of the operation corresponding to a>=b for the forward direction */
+    public boolean ge(int a, int b)
+    {
+        return le(b, a);
+    }
+    /** Returns the result of the operation corresponding to min(a, b) for the forward direction */
+    public abstract int min(int a, int b);
+    /** Returns the result of the operation corresponding to max(a, b) for the forward direction */
+    public abstract int max(int a, int b);
+
+    /**
+     * Use the first argument in forward direction and the second in reverse, i.e. isForward() ? forward : reverse.
+     */
+    public abstract <T> T select(T forward, T reverse);
+
+    /**
+     * Use the first argument in forward direction and the second in reverse, i.e. isForward() ? forward : reverse.
+     */
+    public abstract int select(int forward, int reverse);
+
+    /**
+     * Helper to perform loops over possible values in the given direction. Returns whether the given index is still
+     * within bounds when iterating.
+     * <p>
+     * {@code for} loops implemented as<br/>
+     *   {@code for (int i = dir.select(l, r); dir.inLoop(i, l, r); i += dir.increase) ...}<br/>
+     * will iterate over all values between l and r inclusive in the specified direction.
+     */
+    public abstract boolean inLoop(int index, int left, int right);
+
+    public abstract boolean isForward();
+
+    public abstract Direction opposite();
+
+    public static Direction fromBoolean(boolean reversed)
+    {
+        return reversed ? REVERSE : FORWARD;
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java b/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java
index 88f5987a3380..ecddfd9544ef 100644
--- a/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java
@@ -35,24 +35,24 @@ public class InMemoryReadTrie<T> extends Trie<T>
     /*
     TRIE FORMAT AND NODE TYPES
 
-    The memtable trie uses five different types of nodes:
+    The in-memory trie uses five different types of nodes:
      - "leaf" nodes, which have content and no children;
      - single-transition "chain" nodes, which have exactly one child; while each node is a single transition, they are
-       called "chain" because multiple such transition are packed in a block.
+       called "chain" because multiple such transition are packed in a cell.
      - "sparse" nodes which have between two and six children;
      - "split" nodes for anything above six children;
      - "prefix" nodes that augment one of the other types (except leaf) with content.
 
-    The data for all nodes except leaf ones is stored in a contiguous 'node buffer' and laid out in blocks of 32 bytes.
-    A block only contains data for a single type of node, but there is no direct correspondence between block and node
+    The data for all nodes except leaf ones is stored in a contiguous 'node buffer' and laid out in cells of 32 bytes.
+    A cell only contains data for a single type of node, but there is no direct correspondence between cell and node
     in that:
-     - a single block can contain multiple "chain" nodes.
-     - a sparse node occupies exactly one block.
-     - a split node occupies a variable number of blocks.
-     - a prefix node can be placed in the same block as the node it augments, or in a separate block.
+     - a single cell can contain multiple "chain" nodes.
+     - a sparse node occupies exactly one cell.
+     - a split node occupies a variable number of cells.
+     - a prefix node can be placed in the same cell as the node it augments, or in a separate cell.
 
     Nodes are referenced in that buffer by an integer position/pointer, the 'node pointer'. Note that node pointers are
-    not pointing at the beginning of blocks, and we call 'pointer offset' the offset of the node pointer to the block it
+    not pointing at the beginning of cells, and we call 'pointer offset' the offset of the node pointer to the cell it
     points into. The value of a 'node pointer' is used to decide what kind of node is pointed:
 
      - If the pointer is negative, we have a leaf node. Since a leaf has no children, we need no data outside of its
@@ -62,12 +62,12 @@ public class InMemoryReadTrie<T> extends Trie<T>
 
      - If the 'pointer offset' is smaller than 28, we have a chain node with one transition. The transition character is
        the byte at the position pointed in the 'node buffer', and the child is pointed by:
-       - the integer value at offset 28 of the block pointed if the 'pointer offset' is 27
+       - the integer value at offset 28 of the cell pointed if the 'pointer offset' is 27
        - pointer + 1 (which is guaranteed to have offset smaller than 28, i.e. to be a chain node), otherwise
-       In other words, a chain block contains a sequence of characters that leads to the child whose address is at
-       offset 28. It may have between 1 and 28 characters depending on the pointer with which the block is entered.
+       In other words, a chain cell contains a sequence of characters that leads to the child whose address is at
+       offset 28. It may have between 1 and 28 characters depending on the pointer with which the cell is entered.
 
-     - If the 'pointer offset' is 30, we have a sparse node. The data of a sparse node occupies a full block and is laid
+     - If the 'pointer offset' is 30, we have a sparse node. The data of a sparse node occupies a full cell and is laid
        out as:
        - six pointers to children at offsets 0 to 24
        - six transition characters at offsets 24 to 30
@@ -82,27 +82,27 @@ allows iteration over the order word (which divides said word by 6 each step) to
      - If the 'pointer offset' is 28, the node is a split one. Split nodes are dense, meaning that there is a direct
        mapping between a transition character and the address of the associated pointer, and new children can easily be
        added in place.
-       Split nodes occupy multiple blocks, and a child is located by traversing 3 layers of pointers:
-       - the first pointer is within the top-level block (the one pointed by the pointer) and points to a "mid" block.
-         The top-level block has 4 such pointers to "mid" block, located between offset 16 and 32.
-       - the 2nd pointer is within the "mid" block and points to a "tail" block. A "mid" block has 8 such pointers
-         occupying the whole block.
-       - the 3rd pointer is with the "tail" block and is the actual child pointer. Like "mid" block, there are 8 such
+       Split nodes occupy multiple cells, and a child is located by traversing 3 layers of pointers:
+       - the first pointer is within the top-level cell (the one pointed by the pointer) and points to a "mid" cell.
+         The top-level cell has 4 such pointers to "mid" cell, located between offset 16 and 32.
+       - the 2nd pointer is within the "mid" cell and points to a "tail" cell. A "mid" cell has 8 such pointers
+         occupying the whole cell.
+       - the 3rd pointer is with the "tail" cell and is the actual child pointer. Like "mid" cell, there are 8 such
          pointers (so we finally address 4 * 8 * 8 = 256 children).
-       To find a child, we thus need to know the index of the pointer to follow within the top-level block, the index
-       of the one in the "mid" block and the index in the "tail" block. For that, we split the transition byte in a
+       To find a child, we thus need to know the index of the pointer to follow within the top-level cell, the index
+       of the one in the "mid" cell and the index in the "tail" cell. For that, we split the transition byte in a
        sequence of 2-3-3 bits:
-       - the first 2 bits are the index in the top-level block;
-       - the next 3 bits, the index in the "mid" block;
-       - and the last 3 bits the index in the "tail" block.
-       This layout allows the node to use the smaller fixed-size blocks (instead of 256*4 bytes for the whole character
-       space) and also leaves some room in the head block (the 16 first bytes) for additional information (which we can
+       - the first 2 bits are the index in the top-level cell;
+       - the next 3 bits, the index in the "mid" cell;
+       - and the last 3 bits the index in the "tail" cell.
+       This layout allows the node to use the smaller fixed-size cells (instead of 256*4 bytes for the whole character
+       space) and also leaves some room in the head cell (the 16 first bytes) for additional information (which we can
        use to store prefix nodes containing things like deletion times).
-       One split node may need up to 1 + 4 + 4*8 blocks (1184 bytes) to store all its children.
+       One split node may need up to 1 + 4 + 4*8 cells (1184 bytes) to store all its children.
 
      - If the pointer offset is 31, we have a prefix node. These are two types:
        -- Embedded prefix nodes occupy the free bytes in a chain or split node. The byte at offset 4 has the offset
-          within the 32-byte block for the augmented node.
+          within the 32-byte cell for the augmented node.
        -- Full prefix nodes have 0xFF at offset 4 and a pointer at 28, pointing to the augmented node.
        Both types contain an index for content at offset 0. The augmented node cannot be a leaf or NONE -- in the former
        case the leaf itself contains the content index, in the latter we use a leaf instead.
@@ -117,40 +117,39 @@ single transitions leading to a chain node, we can expand that node (attaching a
      (i.e. create a new node and remap the parent) to sparse with two children. When a six-child sparse node needs a new
      child, we switch to split.
 
-     Blocks currently are not reused, because we do not yet have a mechanism to tell when readers are done with blocks
-     they are referencing. This currently causes a very low overhead (because we change data in place with the only
-     exception of nodes needing to change type) and is planned to be addressed later.
+     Cells can be reused once they are no longer used and cannot be in the state of a concurrently running reader. See
+     MemoryAllocationStrategy for details.
 
      For further descriptions and examples of the mechanics of the trie, see InMemoryTrie.md.
      */
 
-    static final int BLOCK_SIZE = 32;
+    static final int CELL_SIZE = 32;
 
-    // Biggest block offset that can contain a pointer.
-    static final int LAST_POINTER_OFFSET = BLOCK_SIZE - 4;
+    // Biggest cell offset that can contain a pointer.
+    static final int LAST_POINTER_OFFSET = CELL_SIZE - 4;
 
     /*
-     Block offsets used to identify node types (by comparing them to the node 'pointer offset').
+     Cell offsets used to identify node types (by comparing them to the node 'pointer offset').
      */
 
-    // split node (dense, 2-3-3 transitions), laid out as 4 pointers to "mid" block, with has 8 pointers to "tail" block,
+    // split node (dense, 2-3-3 transitions), laid out as 4 pointers to "mid" cell, with has 8 pointers to "tail" cell,
     // which has 8 pointers to children
-    static final int SPLIT_OFFSET = BLOCK_SIZE - 4;
+    static final int SPLIT_OFFSET = CELL_SIZE - 4;
     // sparse node, unordered list of up to 6 transition, laid out as 6 transition pointers followed by 6 transition
     // bytes. The last two bytes contain an ordering of the transitions (in base-6) which is used for iteration. On
     // update the pointer is set last, i.e. during reads the node may show that a transition exists and list a character
     // for it, but pointer may still be null.
-    static final int SPARSE_OFFSET = BLOCK_SIZE - 2;
-    // min and max offset for a chain node. A block of chain node is laid out as a pointer at LAST_POINTER_OFFSET,
-    // preceded by characters that lead to it. Thus a full chain block contains BLOCK_SIZE-4 transitions/chain nodes.
+    static final int SPARSE_OFFSET = CELL_SIZE - 2;
+    // min and max offset for a chain node. A cell of chain node is laid out as a pointer at LAST_POINTER_OFFSET,
+    // preceded by characters that lead to it. Thus a full chain cell contains CELL_SIZE-4 transitions/chain nodes.
     static final int CHAIN_MIN_OFFSET = 0;
-    static final int CHAIN_MAX_OFFSET = BLOCK_SIZE - 5;
+    static final int CHAIN_MAX_OFFSET = CELL_SIZE - 5;
     // Prefix node, an intermediate node augmenting its child node with content.
-    static final int PREFIX_OFFSET = BLOCK_SIZE - 1;
+    static final int PREFIX_OFFSET = CELL_SIZE - 1;
 
     /*
-     Offsets and values for navigating in a block for particular node type. Those offsets are 'from the node pointer'
-     (not the block start) and can be thus negative since node pointers points towards the end of blocks.
+     Offsets and values for navigating in a cell for particular node type. Those offsets are 'from the node pointer'
+     (not the cell start) and can be thus negative since node pointers points towards the end of cells.
      */
 
     // Limit for the starting cell / sublevel (2 bits -> 4 pointers).
@@ -161,14 +160,14 @@ Block offsets used to identify node types (by comparing them to the node 'pointe
     static final int SPLIT_LEVEL_SHIFT = 3;
 
     static final int SPARSE_CHILD_COUNT = 6;
-    // Offset to the first child pointer of a spare node (laid out from the start of the block)
+    // Offset to the first child pointer of a spare node (laid out from the start of the cell)
     static final int SPARSE_CHILDREN_OFFSET = 0 - SPARSE_OFFSET;
     // Offset to the first transition byte of a sparse node (laid out after the child pointers)
     static final int SPARSE_BYTES_OFFSET = SPARSE_CHILD_COUNT * 4 - SPARSE_OFFSET;
     // Offset to the order word of a sparse node (laid out after the children (pointer + transition byte))
     static final int SPARSE_ORDER_OFFSET = SPARSE_CHILD_COUNT * 5 - SPARSE_OFFSET;  // 0
 
-    // Offset of the flag byte in a prefix node. In shared blocks, this contains the offset of the next node.
+    // Offset of the flag byte in a prefix node. In shared cells, this contains the offset of the next node.
     static final int PREFIX_FLAGS_OFFSET = 4 - PREFIX_OFFSET;
     // Offset of the content id
     static final int PREFIX_CONTENT_OFFSET = 0 - PREFIX_OFFSET;
@@ -178,7 +177,7 @@ Block offsets used to identify node types (by comparing them to the node 'pointe
     /**
      * Value used as null for node pointers.
      * No node can use this address (we enforce this by not allowing chain nodes to grow to position 0).
-     * Do not change this as the code relies there being a NONE placed in all bytes of the block that are not set.
+     * Do not change this as the code relies there being a NONE placed in all bytes of the cell that are not set.
      */
     static final int NONE = 0;
 
@@ -200,8 +199,8 @@ Block offsets used to identify node types (by comparing them to the node 'pointe
 
      The allocated space starts 256 bytes for the buffer and 16 entries for the content list.
 
-     Note that a buffer is not allowed to split 32-byte blocks (code assumes same buffer can be used for all bytes
-     inside the block).
+     Note that a buffer is not allowed to split 32-byte cells (code assumes same buffer can be used for all bytes
+     inside the cell).
      */
 
     static final int BUF_START_SHIFT = 8;
@@ -212,42 +211,44 @@ Block offsets used to identify node types (by comparing them to the node 'pointe
 
     static
     {
-        assert BUF_START_SIZE % BLOCK_SIZE == 0 : "Initial buffer size must fit a full block.";
+        assert BUF_START_SIZE % CELL_SIZE == 0 : "Initial buffer size must fit a full cell.";
     }
 
     final UnsafeBuffer[] buffers;
     final AtomicReferenceArray<T>[] contentArrays;
+    final ByteComparable.Version byteComparableVersion;
 
-    InMemoryReadTrie(UnsafeBuffer[] buffers, AtomicReferenceArray<T>[] contentArrays, int root)
+    InMemoryReadTrie(ByteComparable.Version byteComparableVersion, UnsafeBuffer[] buffers, AtomicReferenceArray<T>[] contentArrays, int root)
     {
+        this.byteComparableVersion = byteComparableVersion;
         this.buffers = buffers;
         this.contentArrays = contentArrays;
         this.root = root;
     }
 
     /*
-     Buffer, content list and block management
+     Buffer, content list and cell management
      */
-    int getChunkIdx(int pos, int minChunkShift, int minChunkSize)
+    int getBufferIdx(int pos, int minBufferShift, int minBufferSize)
     {
-        return 31 - minChunkShift - Integer.numberOfLeadingZeros(pos + minChunkSize);
+        return 31 - minBufferShift - Integer.numberOfLeadingZeros(pos + minBufferSize);
     }
 
-    int inChunkPointer(int pos, int chunkIndex, int minChunkSize)
+    int inBufferOffset(int pos, int bufferIndex, int minBufferSize)
     {
-        return pos + minChunkSize - (minChunkSize << chunkIndex);
+        return pos + minBufferSize - (minBufferSize << bufferIndex);
     }
 
-    UnsafeBuffer getChunk(int pos)
+    UnsafeBuffer getBuffer(int pos)
     {
-        int leadBit = getChunkIdx(pos, BUF_START_SHIFT, BUF_START_SIZE);
+        int leadBit = getBufferIdx(pos, BUF_START_SHIFT, BUF_START_SIZE);
         return buffers[leadBit];
     }
 
-    int inChunkPointer(int pos)
+    int inBufferOffset(int pos)
     {
-        int leadBit = getChunkIdx(pos, BUF_START_SHIFT, BUF_START_SIZE);
-        return inChunkPointer(pos, leadBit, BUF_START_SIZE);
+        int leadBit = getBufferIdx(pos, BUF_START_SHIFT, BUF_START_SIZE);
+        return inBufferOffset(pos, leadBit, BUF_START_SIZE);
     }
 
 
@@ -256,28 +257,39 @@ int inChunkPointer(int pos)
      */
     int offset(int pos)
     {
-        return pos & (BLOCK_SIZE - 1);
+        return pos & (CELL_SIZE - 1);
     }
 
     final int getUnsignedByte(int pos)
     {
-        return getChunk(pos).getByte(inChunkPointer(pos)) & 0xFF;
+        return getBuffer(pos).getByte(inBufferOffset(pos)) & 0xFF;
     }
 
-    final int getUnsignedShort(int pos)
+    final int getUnsignedShortVolatile(int pos)
     {
-        return getChunk(pos).getShort(inChunkPointer(pos)) & 0xFFFF;
+        return getBuffer(pos).getShortVolatile(inBufferOffset(pos)) & 0xFFFF;
     }
 
-    final int getInt(int pos)
+    /**
+     * Following a pointer must be done using a volatile read to enforce happens-before between reading the node we
+     * advance to and the preparation of that node that finishes in a volatile write of the pointer that makes it
+     * visible.
+     */
+    final int getIntVolatile(int pos)
     {
-        return getChunk(pos).getInt(inChunkPointer(pos));
+        return getBuffer(pos).getIntVolatile(inBufferOffset(pos));
     }
 
-    T getContent(int index)
+    /**
+     * Get the content for the given content pointer.
+     *
+     * @param id content pointer, encoded as ~index where index is the position in the content array.
+     * @return the current content value.
+     */
+    T getContent(int id)
     {
-        int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
-        int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
+        int leadBit = getBufferIdx(~id, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
+        int ofs = inBufferOffset(~id, leadBit, CONTENTS_START_SIZE);
         AtomicReferenceArray<T> array = contentArrays[leadBit];
         return array.get(ofs);
     }
@@ -302,9 +314,9 @@ boolean isNullOrLeaf(int node)
     }
 
     /**
-     * Returns the number of transitions in a chain block entered with the given pointer.
+     * Returns the number of transitions in a chain cell entered with the given pointer.
      */
-    private int chainBlockLength(int node)
+    private int chainCellLength(int node)
     {
         return LAST_POINTER_OFFSET - offset(node);
     }
@@ -328,7 +340,7 @@ int getChild(int node, int trans)
             case CHAIN_MAX_OFFSET:
                 if (trans != getUnsignedByte(node))
                     return NONE;
-                return getInt(node + 1);
+                return getIntVolatile(node + 1);
             default:
                 if (trans != getUnsignedByte(node))
                     return NONE;
@@ -344,10 +356,10 @@ protected int followContentTransition(int node)
         if (offset(node) == PREFIX_OFFSET)
         {
             int b = getUnsignedByte(node + PREFIX_FLAGS_OFFSET);
-            if (b < BLOCK_SIZE)
+            if (b < CELL_SIZE)
                 node = node - PREFIX_OFFSET + b;
             else
-                node = getInt(node + PREFIX_POINTER_OFFSET);
+                node = getIntVolatile(node + PREFIX_POINTER_OFFSET);
 
             assert node >= 0 && offset(node) != PREFIX_OFFSET;
         }
@@ -378,14 +390,14 @@ int advance(int node, int first, ByteSource rest)
                 if (getUnsignedByte(node++) != first)
                     return NONE;
                 // Check the rest of the bytes provided by the chain node
-                for (int length = chainBlockLength(node); length > 0; --length)
+                for (int length = chainCellLength(node); length > 0; --length)
                 {
                     first = rest.next();
                     if (getUnsignedByte(node++) != first)
                         return NONE;
                 }
                 // All bytes matched, node is now positioned on the child pointer. Follow it.
-                return getInt(node);
+                return getIntVolatile(node);
         }
     }
 
@@ -398,7 +410,7 @@ int getSparseChild(int node, int trans)
         {
             if (getUnsignedByte(node + SPARSE_BYTES_OFFSET + i) == trans)
             {
-                int child = getInt(node + SPARSE_CHILDREN_OFFSET + i * 4);
+                int child = getIntVolatile(node + SPARSE_CHILDREN_OFFSET + i * 4);
 
                 // we can't trust the transition character read above, because it may have been fetched before a
                 // concurrent update happened, and the update may have managed to modify the pointer by now.
@@ -412,7 +424,7 @@ int getSparseChild(int node, int trans)
     }
 
     /**
-     * Given a transition, returns the corresponding index (within the node block) of the pointer to the mid block of
+     * Given a transition, returns the corresponding index (within the node cell) of the pointer to the mid cell of
      * a split node.
      */
     int splitNodeMidIndex(int trans)
@@ -422,7 +434,7 @@ int splitNodeMidIndex(int trans)
     }
 
     /**
-     * Given a transition, returns the corresponding index (within the mid block) of the pointer to the tail block of
+     * Given a transition, returns the corresponding index (within the mid cell) of the pointer to the tail cell of
      * a split node.
      */
     int splitNodeTailIndex(int trans)
@@ -432,7 +444,7 @@ int splitNodeTailIndex(int trans)
     }
 
     /**
-     * Given a transition, returns the corresponding index (within the tail block) of the pointer to the child of
+     * Given a transition, returns the corresponding index (within the tail cell) of the pointer to the child of
      * a split node.
      */
     int splitNodeChildIndex(int trans)
@@ -446,14 +458,14 @@ int splitNodeChildIndex(int trans)
      */
     int getSplitChild(int node, int trans)
     {
-        int mid = getSplitBlockPointer(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
+        int mid = getSplitCellPointer(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
         if (isNull(mid))
             return NONE;
 
-        int tail = getSplitBlockPointer(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+        int tail = getSplitCellPointer(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
         if (isNull(tail))
             return NONE;
-        return getSplitBlockPointer(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+        return getSplitCellPointer(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
     }
 
     /**
@@ -462,25 +474,25 @@ int getSplitChild(int node, int trans)
     T getNodeContent(int node)
     {
         if (isLeaf(node))
-            return getContent(~node);
+            return getContent(node);
 
         if (offset(node) != PREFIX_OFFSET)
             return null;
 
-        int index = getInt(node + PREFIX_CONTENT_OFFSET);
-        return (index >= 0)
+        int index = getIntVolatile(node + PREFIX_CONTENT_OFFSET);
+        return (isLeaf(index))
                ? getContent(index)
                : null;
     }
 
-    int splitBlockPointerAddress(int node, int childIndex, int subLevelLimit)
+    int splitCellPointerAddress(int node, int childIndex, int subLevelLimit)
     {
         return node - SPLIT_OFFSET + (8 - subLevelLimit + childIndex) * 4;
     }
 
-    int getSplitBlockPointer(int node, int childIndex, int subLevelLimit)
+    int getSplitCellPointer(int node, int childIndex, int subLevelLimit)
     {
-        return getInt(splitBlockPointerAddress(node, childIndex, subLevelLimit));
+        return getIntVolatile(splitCellPointerAddress(node, childIndex, subLevelLimit));
     }
 
     /**
@@ -537,15 +549,18 @@ int depth(int backtrackDepth)
      * (i.e. it is positioned on a leaf node), it goes one level up the backtracking chain, where we are guaranteed to
      * have a remaining child to advance to. When there's nothing to backtrack to, the trie is exhausted.
      */
-    class MemtableCursor extends CursorBacktrackingState implements Cursor<T>
+    class InMemoryCursor extends CursorBacktrackingState implements Cursor<T>
     {
         private int currentNode;
+        private int currentFullNode;
         private int incomingTransition;
         private T content;
-        private int depth = -1;
+        private final Direction direction;
+        int depth = -1;
 
-        MemtableCursor()
+        InMemoryCursor(Direction direction)
         {
+            this.direction = direction;
             descendInto(root, -1);
         }
 
@@ -566,26 +581,54 @@ public int advanceMultiple(TransitionsReceiver receiver)
                 return advance();
 
             // Jump directly to the chain's child.
-            UnsafeBuffer chunk = getChunk(node);
-            int inChunkNode = inChunkPointer(node);
-            int bytesJumped = chainBlockLength(node) - 1;   // leave the last byte for incomingTransition
+            UnsafeBuffer buffer = getBuffer(node);
+            int inBufferNode = inBufferOffset(node);
+            int bytesJumped = chainCellLength(node) - 1;   // leave the last byte for incomingTransition
             if (receiver != null && bytesJumped > 0)
-                receiver.addPathBytes(chunk, inChunkNode, bytesJumped);
+                receiver.addPathBytes(buffer, inBufferNode, bytesJumped);
             depth += bytesJumped;    // descendInto will add one
-            inChunkNode += bytesJumped;
+            inBufferNode += bytesJumped;
 
-            // inChunkNode is now positioned on the last byte of the chain.
+            // inBufferNode is now positioned on the last byte of the chain.
             // Consume it to be the new state's incomingTransition.
-            int transition = chunk.getByte(inChunkNode++) & 0xFF;
-            // inChunkNode is now positioned on the child pointer.
-            int child = chunk.getInt(inChunkNode);
+            int transition = buffer.getByte(inBufferNode++) & 0xFF;
+            // inBufferNode is now positioned on the child pointer.
+            int child = buffer.getIntVolatile(inBufferNode);
             return descendInto(child, transition);
         }
 
         @Override
-        public int skipChildren()
+        public int skipTo(int skipDepth, int skipTransition)
         {
-            return backtrack();
+            if (skipDepth > depth)
+            {
+                // Descent requested. Jump to the given child transition or greater, and backtrack if there's no such.
+                assert skipDepth == depth + 1;
+                int advancedDepth = advanceToChildWithTarget(currentNode, skipTransition);
+                if (advancedDepth < 0)
+                    return backtrack();
+
+                assert advancedDepth == skipDepth;
+                return advancedDepth;
+            }
+
+            // Backtrack until we reach the requested depth. Note that we may have more than one entry for a given
+            // depth (split sublevels) and we ascend through them individually.
+            while (--backtrackDepth >= 0)
+            {
+                depth = depth(backtrackDepth);
+
+                if (depth < skipDepth - 1)
+                    return advanceToNextChild(node(backtrackDepth), data(backtrackDepth));
+
+                if (depth == skipDepth - 1)
+                {
+                    int advancedDepth = advanceToNextChildWithTarget(node(backtrackDepth), data(backtrackDepth), skipTransition);
+                    if (advancedDepth >= 0)
+                        return advancedDepth;
+                }
+            }
+            return exhausted();
         }
 
         @Override
@@ -606,10 +649,39 @@ public int incomingTransition()
             return incomingTransition;
         }
 
+        @Override
+        public Direction direction()
+        {
+            return direction;
+        }
+
+        @Override
+        public ByteComparable.Version byteComparableVersion()
+        {
+            return byteComparableVersion;
+        }
+
+        @Override
+        public Trie<T> tailTrie()
+        {
+            assert depth >= 0 : "tailTrie called on exhausted cursor";
+            return new InMemoryReadTrie<>(byteComparableVersion, buffers, contentArrays, currentFullNode);
+        }
+
+        private int exhausted()
+        {
+            depth = -1;
+            incomingTransition = -1;
+            currentFullNode = NONE;
+            currentNode = NONE;
+            content = null;
+            return -1;
+        }
+
         private int backtrack()
         {
             if (--backtrackDepth < 0)
-                return depth = -1;
+                return exhausted();
 
             depth = depth(backtrackDepth);
             return advanceToNextChild(node(backtrackDepth), data(backtrackDepth));
@@ -624,12 +696,28 @@ private int advanceToFirstChild(int node)
                 case SPLIT_OFFSET:
                     return descendInSplitSublevel(node, SPLIT_START_LEVEL_LIMIT, 0, SPLIT_LEVEL_SHIFT * 2);
                 case SPARSE_OFFSET:
-                    return nextValidSparseTransition(node, getUnsignedShort(node + SPARSE_ORDER_OFFSET));
+                    return nextValidSparseTransition(node, prepareOrderWord(node));
                 default:
                     return getChainTransition(node);
             }
         }
 
+        private int advanceToChildWithTarget(int node, int skipTransition)
+        {
+            if (isNullOrLeaf(node))
+                return -1;
+
+            switch (offset(node))
+            {
+                case SPLIT_OFFSET:
+                    return descendInSplitSublevelWithTarget(node, SPLIT_START_LEVEL_LIMIT, 0, SPLIT_LEVEL_SHIFT * 2, skipTransition);
+                case SPARSE_OFFSET:
+                    return advanceToSparseTransition(node, prepareOrderWord(node), skipTransition);
+                default:
+                    return advanceToChainTransition(node, skipTransition);
+            }
+        }
+
         private int advanceToNextChild(int node, int data)
         {
             assert (!isNullOrLeaf(node));
@@ -645,19 +733,34 @@ private int advanceToNextChild(int node, int data)
             }
         }
 
+        private int advanceToNextChildWithTarget(int node, int data, int transition)
+        {
+            assert (!isNullOrLeaf(node));
+
+            switch (offset(node))
+            {
+                case SPLIT_OFFSET:
+                    return advanceToSplitTransition(node, data, transition);
+                case SPARSE_OFFSET:
+                    return advanceToSparseTransition(node, data, transition);
+                default:
+                    throw new AssertionError("Unexpected node type in backtrack state.");
+            }
+        }
+
         /**
          * Descend into the sub-levels of a split node. Advances to the first child and creates backtracking entries
          * for the following ones. We use the bits of trans (lowest non-zero ones) to identify which sub-level an
          * entry refers to.
          *
-         * @param node The node or block id, must have offset SPLIT_OFFSET.
+         * @param node The node or cell id, must have offset SPLIT_OFFSET.
          * @param limit The transition limit for the current sub-level (4 for the start, 8 for the others).
          * @param collected The transition bits collected from the parent chain (e.g. 0x40 after following 1 on the top
          *                  sub-level).
          * @param shift This level's bit shift (6 for start, 3 for mid and 0 for tail).
          * @return the depth reached after descending.
          */
-        private int descendInSplitSublevel(int node, int limit, int collected, int shift)
+        int descendInSplitSublevel(int node, int limit, int collected, int shift)
         {
             while (true)
             {
@@ -665,9 +768,11 @@ private int descendInSplitSublevel(int node, int limit, int collected, int shift
                 int childIndex;
                 int child = NONE;
                 // find the first non-null child
-                for (childIndex = 0; childIndex < limit; ++childIndex)
+                for (childIndex = direction.select(0, limit - 1);
+                     direction.inLoop(childIndex, 0, limit - 1);
+                     childIndex += direction.increase)
                 {
-                    child = getSplitBlockPointer(node, childIndex, limit);
+                    child = getSplitCellPointer(node, childIndex, limit);
                     if (!isNull(child))
                         break;
                 }
@@ -691,111 +796,297 @@ private int descendInSplitSublevel(int node, int limit, int collected, int shift
         }
 
         /**
-         * Backtrack to a split sub-level. The level is identified by the lowest non-0 bits in trans.
+         * As above, but also makes sure that the descend selects a value at least as big as the given
+         * {@code minTransition}.
+         */
+        private int descendInSplitSublevelWithTarget(int node, int limit, int collected, int shift, int minTransition)
+        {
+            minTransition -= collected;
+            if (minTransition >= limit << shift || minTransition < 0)
+                return -1;
+
+            while (true)
+            {
+                assert offset(node) == SPLIT_OFFSET;
+                int childIndex;
+                int child = NONE;
+                boolean isExact = true;
+                // find the first non-null child beyond minTransition
+                for (childIndex = minTransition >> shift;
+                     direction.inLoop(childIndex, 0, limit - 1);
+                     childIndex += direction.increase)
+                {
+                    child = getSplitCellPointer(node, childIndex, limit);
+                    if (!isNull(child))
+                        break;
+                    isExact = false;
+                }
+                if (!isExact && (childIndex == limit || childIndex == -1))
+                    return -1;
+
+                // look for any more valid transitions and add backtracking if found
+                maybeAddSplitBacktrack(node, childIndex, limit, collected, shift);
+
+                // add the bits just found
+                collected |= childIndex << shift;
+                // descend to next sub-level or child
+                if (shift == 0)
+                    return descendInto(child, collected);
+
+                if (isExact)
+                    minTransition -= childIndex << shift;
+                else
+                    minTransition = direction.select(0, (1 << shift) - 1);
+
+                // continue with next sublevel; same as
+                // return descendInSplitSublevelWithTarget(child + SPLIT_OFFSET, 8, collected, shift - 3, minTransition)
+                node = child;
+                limit = SPLIT_OTHER_LEVEL_LIMIT;
+                shift -= SPLIT_LEVEL_SHIFT;
+            }
+        }
+
+        /**
+         * Backtrack to a split sub-level. The level is identified by the lowest non-0 bits in data.
          */
-        private int nextValidSplitTransition(int node, int trans)
+        int nextValidSplitTransition(int node, int data)
         {
-            assert trans >= 0 && trans <= 0xFF;
-            int childIndex = splitNodeChildIndex(trans);
-            if (childIndex > 0)
+            // Note: This is equivalent to return advanceToSplitTransition(node, data, data) but quicker.
+            assert data >= 0 && data <= 0xFF;
+            int childIndex = splitNodeChildIndex(data);
+            if (childIndex != direction.select(0, SPLIT_OTHER_LEVEL_LIMIT - 1))
             {
                 maybeAddSplitBacktrack(node,
                                        childIndex,
                                        SPLIT_OTHER_LEVEL_LIMIT,
-                                       trans & -(1 << (SPLIT_LEVEL_SHIFT * 1)),
+                                       data & -(1 << (SPLIT_LEVEL_SHIFT * 1)),
                                        SPLIT_LEVEL_SHIFT * 0);
-                int child = getSplitBlockPointer(node, childIndex, SPLIT_OTHER_LEVEL_LIMIT);
-                return descendInto(child, trans);
+                int child = getSplitCellPointer(node, childIndex, SPLIT_OTHER_LEVEL_LIMIT);
+                return descendInto(child, data);
             }
-            int tailIndex = splitNodeTailIndex(trans);
-            if (tailIndex > 0)
+            int tailIndex = splitNodeTailIndex(data);
+            if (tailIndex != direction.select(0, SPLIT_OTHER_LEVEL_LIMIT - 1))
             {
                 maybeAddSplitBacktrack(node,
                                        tailIndex,
                                        SPLIT_OTHER_LEVEL_LIMIT,
-                                       trans & -(1 << (SPLIT_LEVEL_SHIFT * 2)),
+                                       data & -(1 << (SPLIT_LEVEL_SHIFT * 2)),
                                        SPLIT_LEVEL_SHIFT * 1);
-                int tail = getSplitBlockPointer(node, tailIndex, SPLIT_OTHER_LEVEL_LIMIT);
+                int tail = getSplitCellPointer(node, tailIndex, SPLIT_OTHER_LEVEL_LIMIT);
                 return descendInSplitSublevel(tail,
                                               SPLIT_OTHER_LEVEL_LIMIT,
-                                              trans,
+                                              data & -(1 << SPLIT_LEVEL_SHIFT * 1),
                                               SPLIT_LEVEL_SHIFT * 0);
             }
-            int midIndex = splitNodeMidIndex(trans);
-            assert midIndex > 0;
+            int midIndex = splitNodeMidIndex(data);
+            assert midIndex != direction.select(0, SPLIT_START_LEVEL_LIMIT - 1);
             maybeAddSplitBacktrack(node,
                                    midIndex,
                                    SPLIT_START_LEVEL_LIMIT,
                                    0,
                                    SPLIT_LEVEL_SHIFT * 2);
-            int mid = getSplitBlockPointer(node, midIndex, SPLIT_START_LEVEL_LIMIT);
+            int mid = getSplitCellPointer(node, midIndex, SPLIT_START_LEVEL_LIMIT);
             return descendInSplitSublevel(mid,
                                           SPLIT_OTHER_LEVEL_LIMIT,
-                                          trans,
+                                          data & -(1 << SPLIT_LEVEL_SHIFT * 2),
                                           SPLIT_LEVEL_SHIFT * 1);
         }
 
+        /**
+         * Backtrack to a split sub-level and advance to given transition if it fits within the sublevel.
+         * The level is identified by the lowest non-0 bits in data as above.
+         */
+        private int advanceToSplitTransition(int node, int data, int skipTransition)
+        {
+            assert data >= 0 && data <= 0xFF;
+            if (direction.lt(skipTransition, data))
+                return nextValidSplitTransition(node, data); // already went over the target in lower sublevel, just advance
+
+            int childIndex = splitNodeChildIndex(data);
+            if (childIndex != direction.select(0, SPLIT_OTHER_LEVEL_LIMIT - 1))
+            {
+                int sublevelMask = -(1 << (SPLIT_LEVEL_SHIFT * 1));
+                int sublevelShift = SPLIT_LEVEL_SHIFT * 0;
+                int sublevelLimit = SPLIT_OTHER_LEVEL_LIMIT;
+                return descendInSplitSublevelWithTarget(node, sublevelLimit, data & sublevelMask, sublevelShift, skipTransition);
+            }
+            int tailIndex = splitNodeTailIndex(data);
+            if (tailIndex != direction.select(0, SPLIT_OTHER_LEVEL_LIMIT - 1))
+            {
+                int sublevelMask = -(1 << (SPLIT_LEVEL_SHIFT * 2));
+                int sublevelShift = SPLIT_LEVEL_SHIFT * 1;
+                int sublevelLimit = SPLIT_OTHER_LEVEL_LIMIT;
+                return descendInSplitSublevelWithTarget(node, sublevelLimit, data & sublevelMask, sublevelShift, skipTransition);
+            }
+            int sublevelMask = -(1 << 8);
+            int sublevelShift = SPLIT_LEVEL_SHIFT * 2;
+            int sublevelLimit = SPLIT_START_LEVEL_LIMIT;
+            return descendInSplitSublevelWithTarget(node, sublevelLimit, data & sublevelMask, sublevelShift, skipTransition);
+        }
+
         /**
          * Look for any further non-null transitions on this sub-level and, if found, add a backtracking entry.
          */
         private void maybeAddSplitBacktrack(int node, int startAfter, int limit, int collected, int shift)
         {
             int nextChildIndex;
-            for (nextChildIndex = startAfter + 1; nextChildIndex < limit; ++nextChildIndex)
+            for (nextChildIndex = startAfter + direction.increase;
+                 direction.inLoop(nextChildIndex, 0, limit - 1);
+                 nextChildIndex += direction.increase)
             {
-                if (!isNull(getSplitBlockPointer(node, nextChildIndex, limit)))
+                if (!isNull(getSplitCellPointer(node, nextChildIndex, limit)))
                     break;
             }
-            if (nextChildIndex < limit)
-                addBacktrack(node, collected | (nextChildIndex << shift), depth);
+            if (direction.inLoop(nextChildIndex, 0, limit - 1))
+            {
+                if (direction.isForward())
+                    addBacktrack(node, collected | (nextChildIndex << shift), depth);
+                else
+                {
+                    // The (((x + 1) << shift) - 1) adjustment will put all 1s in all lower bits
+                    addBacktrack(node, collected | ((((nextChildIndex + 1) << shift)) - 1), depth);
+                }
+            }
         }
 
+
         private int nextValidSparseTransition(int node, int data)
         {
-            UnsafeBuffer chunk = getChunk(node);
-            int inChunkNode = inChunkPointer(node);
-
             // Peel off the next index.
             int index = data % SPARSE_CHILD_COUNT;
             data = data / SPARSE_CHILD_COUNT;
 
+            UnsafeBuffer buffer = getBuffer(node);
+            int inBufferNode = inBufferOffset(node);
+
+            // If there are remaining transitions, add backtracking entry.
+            if (data != exhaustedOrderWord())
+                addBacktrack(node, data, depth);
+
+            // Follow the transition.
+            int child = buffer.getIntVolatile(inBufferNode + SPARSE_CHILDREN_OFFSET + index * 4);
+            int transition = buffer.getByte(inBufferNode + SPARSE_BYTES_OFFSET + index) & 0xFF;
+            return descendInto(child, transition);
+        }
+
+        /**
+         * Prepare the sparse node order word for iteration. For forward iteration, this means just reading it.
+         * For reverse, we also invert the data so that the peeling code above still works.
+         */
+        int prepareOrderWord(int node)
+        {
+            int fwdState = getUnsignedShortVolatile(node + SPARSE_ORDER_OFFSET);
+            if (direction.isForward())
+                return fwdState;
+            else
+            {
+                // Produce an inverted state word.
+
+                // One subtlety is that in forward order we know we can terminate the iteration when the state becomes
+                // 0 because 0 cannot be the largest child (we enforce 10 order for the first two children and then can
+                // only insert other digits in the word, thus 0 is always preceded by a 1 (not necessarily immediately)
+                // in the order word) and thus we can't confuse a completed iteration with one that still has the child
+                // at 0 to present.
+                // In reverse order 0 can be the last child that needs to be iterated (e.g. for two children the order
+                // word is always 10, which is 01 inverted; if we treat it exactly as the forward iteration, we will
+                // only list child 1 because we will interpret the state 0 after peeling the first digit as a completed
+                // iteration). To know when to stop we must thus use a different marker - since we know 1 is never the
+                // last child to be iterated in reverse order (because it is preceded by a 0 in the reversed order
+                // word), we can use another 1 as the termination marker. The generated number may not fit a 16-bit word
+                // any more, but that does not matter as we don't need to store it.
+                // For example, the code below translates 120 to 1021, and to iterate we peel the lower order digits
+                // until the iteration state becomes just 1.
+
+                int revState = 1;   // 1 can't be the smallest child
+                while (fwdState != 0)
+                {
+                    revState = revState * SPARSE_CHILD_COUNT + fwdState % SPARSE_CHILD_COUNT;
+                    fwdState /= SPARSE_CHILD_COUNT;
+                }
+
+                return revState;
+            }
+        }
+
+        /**
+         * Returns the state which marks the exhaustion of the order word.
+         */
+        int exhaustedOrderWord()
+        {
+            return direction.select(0, 1);
+        }
+
+        private int advanceToSparseTransition(int node, int data, int skipTransition)
+        {
+            UnsafeBuffer buffer = getBuffer(node);
+            int inBufferNode = inBufferOffset(node);
+            int index;
+            int transition;
+            do
+            {
+                // Peel off the next index.
+                index = data % SPARSE_CHILD_COUNT;
+                data = data / SPARSE_CHILD_COUNT;
+                transition = buffer.getByte(inBufferNode + SPARSE_BYTES_OFFSET + index) & 0xFF;
+            }
+            while (direction.lt(transition, skipTransition) && data != exhaustedOrderWord());
+            if (direction.lt(transition, skipTransition))
+                return -1;
+
             // If there are remaining transitions, add backtracking entry.
-            if (data > 0)
+            if (data != exhaustedOrderWord())
                 addBacktrack(node, data, depth);
 
             // Follow the transition.
-            int child = chunk.getInt(inChunkNode + SPARSE_CHILDREN_OFFSET + index * 4);
-            int transition = chunk.getByte(inChunkNode + SPARSE_BYTES_OFFSET + index) & 0xFF;
+            int child = buffer.getIntVolatile(inBufferNode + SPARSE_CHILDREN_OFFSET + index * 4);
             return descendInto(child, transition);
         }
 
         private int getChainTransition(int node)
         {
             // No backtracking needed.
-            UnsafeBuffer chunk = getChunk(node);
-            int inChunkNode = inChunkPointer(node);
-            int transition = chunk.getByte(inChunkNode) & 0xFF;
+            UnsafeBuffer buffer = getBuffer(node);
+            int inBufferNode = inBufferOffset(node);
+            int transition = buffer.getByte(inBufferNode) & 0xFF;
+            int next = node + 1;
+            if (offset(next) <= CHAIN_MAX_OFFSET)
+                return descendIntoChain(next, transition);
+            else
+                return descendInto(buffer.getIntVolatile(inBufferNode + 1), transition);
+        }
+
+        private int advanceToChainTransition(int node, int skipTransition)
+        {
+            // No backtracking needed.
+            UnsafeBuffer buffer = getBuffer(node);
+            int inBufferNode = inBufferOffset(node);
+            int transition = buffer.getByte(inBufferNode) & 0xFF;
+            if (direction.gt(skipTransition, transition))
+                return -1;
+
             int next = node + 1;
             if (offset(next) <= CHAIN_MAX_OFFSET)
                 return descendIntoChain(next, transition);
             else
-                return descendInto(chunk.getInt(inChunkNode + 1), transition);
+                return descendInto(buffer.getIntVolatile(inBufferNode + 1), transition);
         }
 
-        private int descendInto(int child, int transition)
+        int descendInto(int child, int transition)
         {
             ++depth;
             incomingTransition = transition;
             content = getNodeContent(child);
+            currentFullNode = child;
             currentNode = followContentTransition(child);
             return depth;
         }
 
-        private int descendIntoChain(int child, int transition)
+        int descendIntoChain(int child, int transition)
         {
             ++depth;
             incomingTransition = transition;
             content = null;
+            currentFullNode = child;
             currentNode = child;
             return depth;
         }
@@ -806,9 +1097,9 @@ private boolean isChainNode(int node)
         return !isNullOrLeaf(node) && offset(node) <= CHAIN_MAX_OFFSET;
     }
 
-    public MemtableCursor cursor()
+    public InMemoryCursor cursor(Direction direction)
     {
-        return new MemtableCursor();
+        return new InMemoryCursor(direction);
     }
 
     /*
@@ -819,10 +1110,11 @@ public MemtableCursor cursor()
      * Get the content mapped by the specified key.
      * Fast implementation using integer node addresses.
      */
+    @Override
     public T get(ByteComparable path)
     {
         int n = root;
-        ByteSource source = path.asComparableBytes(BYTE_COMPARABLE_VERSION);
+        ByteSource source = path.asComparableBytes(byteComparableVersion);
         while (!isNull(n))
         {
             int c = source.next();
@@ -840,6 +1132,11 @@ public boolean isEmpty()
         return isNull(root);
     }
 
+    public ByteComparable.Version byteComparableVersion()
+    {
+        return byteComparableVersion;
+    }
+
     /**
      * Override of dump to provide more detailed printout that includes the type of each node in the trie.
      * We do this via a wrapping cursor that returns a content string for the type of node for every node we return.
@@ -847,7 +1144,7 @@ public boolean isEmpty()
     @Override
     public String dump(Function<T, String> contentToString)
     {
-        MemtableCursor source = cursor();
+        InMemoryCursor source = cursor(Direction.FORWARD);
         class TypedNodesCursor implements Cursor<String>
         {
             @Override
@@ -864,9 +1161,9 @@ public int advanceMultiple(TransitionsReceiver receiver)
             }
 
             @Override
-            public int skipChildren()
+            public int skipTo(int skipDepth, int skipTransition)
             {
-                return source.skipChildren();
+                return source.skipTo(skipDepth, skipTransition);
             }
 
             @Override
@@ -881,6 +1178,24 @@ public int incomingTransition()
                 return source.incomingTransition();
             }
 
+            @Override
+            public Direction direction()
+            {
+                return source.direction();
+            }
+
+            @Override
+            public ByteComparable.Version byteComparableVersion()
+            {
+                return source.byteComparableVersion();
+            }
+
+            @Override
+            public Trie<String> tailTrie()
+            {
+                throw new AssertionError();
+            }
+
             @Override
             public String content()
             {
@@ -917,4 +1232,73 @@ public String content()
         }
         return process(new TrieDumper<>(Function.identity()), new TypedNodesCursor());
     }
+
+    /**
+     * For use in debugging, dump info about the given node.
+     */
+    @SuppressWarnings("unused")
+    String dumpNode(int node)
+    {
+        if (isNull(node))
+            return "NONE";
+        else if (isLeaf(node))
+            return "~" + (~node);
+        else
+        {
+            StringBuilder builder = new StringBuilder();
+            builder.append(node + " ");
+            switch (offset(node))
+            {
+                case SPARSE_OFFSET:
+                {
+                    builder.append("Sparse: ");
+                    for (int i = 0; i < SPARSE_CHILD_COUNT; ++i)
+                    {
+                        int child = getIntVolatile(node + SPARSE_CHILDREN_OFFSET + i * 4);
+                        if (child != NONE)
+                            builder.append(String.format("%02x", getUnsignedByte(node + SPARSE_BYTES_OFFSET + i)))
+                                   .append(" -> ")
+                                   .append(child)
+                                   .append('\n');
+                    }
+                    break;
+                }
+                case SPLIT_OFFSET:
+                {
+                    builder.append("Split: ");
+                    for (int i = 0; i < SPLIT_START_LEVEL_LIMIT; ++i)
+                    {
+                        int child = getIntVolatile(node - (SPLIT_START_LEVEL_LIMIT - 1 - i) * 4);
+                        if (child != NONE)
+                            builder.append(Integer.toBinaryString(i))
+                                   .append(" -> ")
+                                   .append(child)
+                                   .append('\n');
+                    }
+                    break;
+                }
+                case PREFIX_OFFSET:
+                {
+                    builder.append("Prefix: ");
+                    int flags = getUnsignedByte(node + PREFIX_FLAGS_OFFSET);
+                    final int content = getIntVolatile(node + PREFIX_CONTENT_OFFSET);
+                    builder.append(content < 0 ? "~" + (~content) : "" + content);
+                    int child = followContentTransition(node);
+                    builder.append(" -> ")
+                           .append(child);
+                    break;
+                }
+                default:
+                {
+                    builder.append("Chain: ");
+                    for (int i = 0; i < chainCellLength(node); ++i)
+                        builder.append(String.format("%02x", getUnsignedByte(node + i)));
+                    builder.append(" -> ")
+                           .append(getIntVolatile(node + chainCellLength(node)));
+                    break;
+                }
+            }
+            return builder.toString();
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java
index 9bda82057f9a..0492cfb78e9e 100644
--- a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java
@@ -19,11 +19,12 @@
 
 import java.nio.ByteBuffer;
 import java.util.Arrays;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 
 import com.google.common.annotations.VisibleForTesting;
+import java.util.function.Predicate;
+
+import com.google.common.base.Predicates;
 
 import org.agrona.concurrent.UnsafeBuffer;
 import org.apache.cassandra.config.CassandraRelevantProperties;
@@ -32,17 +33,42 @@
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 
-import org.github.jamm.MemoryMeterStrategy;
+import static org.github.jamm.MemoryMeterStrategy.MEMORY_LAYOUT;
 
 /**
  * In-memory trie built for fast modification and reads executing concurrently with writes from a single mutator thread.
- *
- * This class can currently only provide atomicity (i.e. reads seeing either the content before a write, or the
- * content after it; any read seeing the write enforcing any subsequent (i.e. started after it completed) reads to
- * also see it) for singleton writes (i.e. calls to {@link #putRecursive}, {@link #putSingleton} or {@link #apply}
- * with a singleton trie as argument).
- *
+ * <p>
+ * The main method for performing writes is {@link #apply(Trie, UpsertTransformer, Predicate)} which takes a trie as
+ * an argument and merges it into the current trie using the methods supplied by the given {@link UpsertTransformer},
+ * force copying anything below the points where the third argument returns true.
+ * </p><p>
+ * The predicate can be used to implement several forms of atomicity and consistency guarantees:
+ * <list>
+ * <li> if the predicate is {@code nf -> false}, neither atomicity nor sequential consistency is guaranteed - readers
+ *      can see any mixture of old and modified content
+ * <li> if the predicate is {@code nf -> true}, full sequential consistency will be provided, i.e. if a reader sees any
+ *      part of a modification, it will see all of it, and all the results of all previous modifications
+ * <li> if the predicate is {@code nf -> nf.isBranching()} the write will be atomic, i.e. either none or all of the
+ *      content of the merged trie will be visible by concurrent readers, but not sequentially consistent, i.e. there
+ *      may be writes that are not visible to a reader even when they precede writes that are visible.
+ * <li> if the predicate is {@code nf -> <some_test>(nf.content())} the write will be consistent below the identified
+ *      point (used e.g. by Memtable to ensure partition-level consistency)
+ * </list>
+ * </p><p>
+ * Additionally, the class provides several simpler write methods for efficiency and convenience:
+ * <list>
+ * <li> {@link #putRecursive(ByteComparable, Object, UpsertTransformer)} inserts a single value using a recursive walk.
+ *      It cannot provide consistency (single-path writes are always atomic). This is more efficient as it stores the
+ *      walk state in the stack rather than on the heap but can cause a {@code StackOverflowException}.
+ * <li> {@link #putSingleton(ByteComparable, Object, UpsertTransformer)} is a non-recursive version of the above, using
+ *      the {@code apply} machinery.
+ * <li> {@link #putSingleton(ByteComparable, Object, UpsertTransformer, boolean)} uses the fourth argument to choose
+ *      between the two methods above, where some external property can be used to decide if the keys are short enough
+ *      to permit recursive execution.
+ * </list>
+ * </p><p>
  * Because it uses 32-bit pointers in byte buffers, this trie has a fixed size limit of 2GB.
  */
 public class InMemoryTrie<T> extends InMemoryReadTrie<T>
@@ -60,9 +86,9 @@ public class InMemoryTrie<T> extends InMemoryReadTrie<T>
     {
         // Default threshold + 10% == 2 GB. This should give the owner enough time to react to the
         // {@link #reachedAllocatedSizeThreshold()} signal and switch this trie out before it fills up.
-        int limitInMB = CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE.getInt(2048 * 10 / 11);
+        int limitInMB = CassandraRelevantProperties.MEMTABLE_TRIE_SIZE_LIMIT.getInt(2048 * 10 / 11);
         if (limitInMB < 1 || limitInMB > 2047)
-            throw new AssertionError(CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE.getKey() +
+            throw new AssertionError(CassandraRelevantProperties.MEMTABLE_TRIE_SIZE_LIMIT.getKey() +
                                      " must be within 1 and 2047");
         ALLOCATED_SIZE_THRESHOLD = 1024 * 1024 * limitInMB;
     }
@@ -70,7 +96,10 @@ public class InMemoryTrie<T> extends InMemoryReadTrie<T>
     private int allocatedPos = 0;
     private int contentCount = 0;
 
-    private final BufferType bufferType;    // on or off heap
+    final BufferType bufferType;    // on or off heap
+    final MemoryAllocationStrategy cellAllocator;
+    final MemoryAllocationStrategy objectAllocator;
+
 
     // constants for space calculations
     private static final long EMPTY_SIZE_ON_HEAP;
@@ -79,74 +108,131 @@ public class InMemoryTrie<T> extends InMemoryReadTrie<T>
 
     static
     {
-        InMemoryTrie<Object> empty = new InMemoryTrie<>(BufferType.ON_HEAP);
+        // Measuring the empty size of long-lived tries, because these are the ones for which we want to track size.
+        InMemoryTrie<Object> empty = new InMemoryTrie<>(ByteComparable.Version.OSS50, BufferType.ON_HEAP, ExpectedLifetime.LONG, null);
         EMPTY_SIZE_ON_HEAP = ObjectSizes.measureDeep(empty);
-        empty = new InMemoryTrie<>(BufferType.OFF_HEAP);
+        empty = new InMemoryTrie<>(ByteComparable.Version.OSS50, BufferType.OFF_HEAP, ExpectedLifetime.LONG, null);
         EMPTY_SIZE_OFF_HEAP = ObjectSizes.measureDeep(empty);
     }
 
-    public InMemoryTrie(BufferType bufferType)
+    enum ExpectedLifetime
     {
-        super(new UnsafeBuffer[31 - BUF_START_SHIFT],  // last one is 1G for a total of ~2G bytes
+        SHORT, LONG
+    }
+
+    InMemoryTrie(ByteComparable.Version byteComparableVersion, BufferType bufferType, ExpectedLifetime lifetime, OpOrder opOrder)
+    {
+        super(byteComparableVersion,
+              new UnsafeBuffer[31 - BUF_START_SHIFT],  // last one is 1G for a total of ~2G bytes
               new AtomicReferenceArray[29 - CONTENTS_START_SHIFT],  // takes at least 4 bytes to write pointer to one content -> 4 times smaller than buffers
               NONE);
         this.bufferType = bufferType;
+
+        switch (lifetime)
+        {
+            case SHORT:
+                cellAllocator = new MemoryAllocationStrategy.NoReuseStrategy(new MemoryAllocationStrategy.Allocator()
+                {
+                    @Override
+                    public int allocate() throws TrieSpaceExhaustedException
+                    {
+                        return allocateNewCell();
+                    }
+                });
+                objectAllocator = new MemoryAllocationStrategy.NoReuseStrategy(new MemoryAllocationStrategy.Allocator()
+                {
+                    @Override
+                    public int allocate()
+                    {
+                        return allocateNewObject();
+                    }
+                });
+                break;
+            case LONG:
+                cellAllocator = new MemoryAllocationStrategy.OpOrderReuseStrategy(new MemoryAllocationStrategy.Allocator()
+                {
+                    @Override
+                    public int allocate() throws TrieSpaceExhaustedException
+                    {
+                        return allocateNewCell();
+                    }
+                }, opOrder);
+                objectAllocator = new MemoryAllocationStrategy.OpOrderReuseStrategy(new MemoryAllocationStrategy.Allocator()
+                {
+                    @Override
+                    public int allocate()
+                    {
+                        return allocateNewObject();
+                    }
+                }, opOrder);
+                break;
+            default:
+                throw new AssertionError();
+        }
     }
 
-    // Buffer, content list and block management
+    public static <T> InMemoryTrie<T> shortLived(ByteComparable.Version byteComparableVersion)
+    {
+        return new InMemoryTrie<>(byteComparableVersion, BufferType.ON_HEAP, ExpectedLifetime.SHORT, null);
+    }
 
-    /**
-     * Because we use buffers and 32-bit pointers, the trie cannot grow over 2GB of size. This exception is thrown if
-     * a trie operation needs it to grow over that limit.
-     *
-     * To avoid this problem, users should query {@link #reachedAllocatedSizeThreshold} from time to time. If the call
-     * returns true, they should switch to a new trie (e.g. by flushing a memtable) as soon as possible. The threshold
-     * is configurable, and is set by default to 10% under the 2GB limit to give ample time for the switch to happen.
-     */
-    public static class SpaceExhaustedException extends Exception
+    public static <T> InMemoryTrie<T> shortLived(ByteComparable.Version byteComparableVersion, BufferType bufferType)
     {
-        public SpaceExhaustedException()
-        {
-            super("The hard 2GB limit on trie size has been exceeded");
-        }
+        return new InMemoryTrie<>(byteComparableVersion, bufferType, ExpectedLifetime.SHORT, null);
     }
 
-    final void putInt(int pos, int value)
+    public static <T> InMemoryTrie<T> longLived(ByteComparable.Version byteComparableVersion, OpOrder opOrder)
     {
-        getChunk(pos).putInt(inChunkPointer(pos), value);
+        return longLived(byteComparableVersion, BufferType.OFF_HEAP, opOrder);
     }
 
-    final void putIntVolatile(int pos, int value)
+    public static <T> InMemoryTrie<T> longLived(ByteComparable.Version byteComparableVersion, BufferType bufferType, OpOrder opOrder)
     {
-        getChunk(pos).putIntVolatile(inChunkPointer(pos), value);
+        return new InMemoryTrie<>(byteComparableVersion, bufferType, ExpectedLifetime.LONG, opOrder);
     }
 
-    final void putShort(int pos, short value)
+
+    // Buffer, content list and cell management
+
+    private void putInt(int pos, int value)
     {
-        getChunk(pos).putShort(inChunkPointer(pos), value);
+        getBuffer(pos).putInt(inBufferOffset(pos), value);
     }
 
-    final void putShortVolatile(int pos, short value)
+    private void putIntVolatile(int pos, int value)
     {
-        getChunk(pos).putShort(inChunkPointer(pos), value);
+        getBuffer(pos).putIntVolatile(inBufferOffset(pos), value);
     }
 
-    final void putByte(int pos, byte value)
+    private void putShort(int pos, short value)
     {
-        getChunk(pos).putByte(inChunkPointer(pos), value);
+        getBuffer(pos).putShort(inBufferOffset(pos), value);
     }
 
+    private void putShortVolatile(int pos, short value)
+    {
+        getBuffer(pos).putShort(inBufferOffset(pos), value);
+    }
 
-    private int allocateBlock() throws SpaceExhaustedException
+    private void putByte(int pos, byte value)
+    {
+        getBuffer(pos).putByte(inBufferOffset(pos), value);
+    }
+
+    /**
+     * Allocate a new cell in the data buffers. This is called by the memory allocation strategy when it runs out of
+     * free cells to reuse.
+     */
+    private int allocateNewCell() throws TrieSpaceExhaustedException
     {
         // Note: If this method is modified, please run InMemoryTrieTest.testOver1GSize to verify it acts correctly
         // close to the 2G limit.
         int v = allocatedPos;
-        if (inChunkPointer(v) == 0)
+        if (inBufferOffset(v) == 0)
         {
-            int leadBit = getChunkIdx(v, BUF_START_SHIFT, BUF_START_SIZE);
+            int leadBit = getBufferIdx(v, BUF_START_SHIFT, BUF_START_SIZE);
             if (leadBit + BUF_START_SHIFT == 31)
-                throw new SpaceExhaustedException();
+                throw new TrieSpaceExhaustedException();
 
             ByteBuffer newBuffer = bufferType.allocate(BUF_START_SIZE << leadBit);
             buffers[leadBit] = new UnsafeBuffer(newBuffer);
@@ -155,34 +241,97 @@ private int allocateBlock() throws SpaceExhaustedException
             // that attached the new path.
         }
 
-        allocatedPos += BLOCK_SIZE;
+        allocatedPos += CELL_SIZE;
         return v;
     }
 
-    private int addContent(T value)
+    /**
+     * Allocate a cell to use for storing data. This uses the memory allocation strategy to reuse cells if any are
+     * available, or to allocate new cells using {@link #allocateNewCell}. Because some node types rely on cells being
+     * filled with 0 as initial state, any cell we get through the allocator must also be cleaned.
+     */
+    private int allocateCell() throws TrieSpaceExhaustedException
+    {
+        int cell = cellAllocator.allocate();
+        getBuffer(cell).setMemory(inBufferOffset(cell), CELL_SIZE, (byte) 0);
+        return cell;
+    }
+
+    private void recycleCell(int cell)
+    {
+        cellAllocator.recycle(cell & -CELL_SIZE);
+    }
+
+    /**
+     * Creates a copy of a given cell and marks the original for recycling. Used when a mutation needs to force-copy
+     * paths to ensure earlier states are still available for concurrent readers.
+     */
+    private int copyCell(int cell) throws TrieSpaceExhaustedException
+    {
+        int copy = cellAllocator.allocate();
+        getBuffer(copy).putBytes(inBufferOffset(copy), getBuffer(cell), inBufferOffset(cell & -CELL_SIZE), CELL_SIZE);
+        recycleCell(cell);
+        return copy | (cell & (CELL_SIZE - 1));
+    }
+
+    /**
+     * Allocate a new position in the object array. Used by the memory allocation strategy to allocate a content spot
+     * when it runs out of recycled positions.
+     */
+    private int allocateNewObject()
     {
         int index = contentCount++;
-        int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
-        int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
+        int leadBit = getBufferIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
         AtomicReferenceArray<T> array = contentArrays[leadBit];
         if (array == null)
         {
-            assert ofs == 0 : "Error in content arrays configuration.";
-            contentArrays[leadBit] = array = new AtomicReferenceArray<>(CONTENTS_START_SIZE << leadBit);
+            assert inBufferOffset(index, leadBit, CONTENTS_START_SIZE) == 0 : "Error in content arrays configuration.";
+            contentArrays[leadBit] = new AtomicReferenceArray<>(CONTENTS_START_SIZE << leadBit);
         }
-        array.lazySet(ofs, value); // no need for a volatile set here; at this point the item is not referenced
-                                   // by any node in the trie, and a volatile set will be made to reference it.
         return index;
     }
 
-    private void setContent(int index, T value)
+
+    /**
+     * Add a new content value.
+     *
+     * @return A content id that can be used to reference the content, encoded as ~index where index is the
+     *         position of the value in the content array.
+     */
+    private int addContent(T value) throws TrieSpaceExhaustedException
     {
-        int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
-        int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
+        int index = objectAllocator.allocate();
+        int leadBit = getBufferIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
+        int ofs = inBufferOffset(index, leadBit, CONTENTS_START_SIZE);
+        AtomicReferenceArray<T> array = contentArrays[leadBit];
+        // no need for a volatile set here; at this point the item is not referenced
+        // by any node in the trie, and a volatile set will be made to reference it.
+        array.setPlain(ofs, value);
+        return ~index;
+    }
+
+    /**
+     * Change the content associated with a given content id.
+     *
+     * @param id content id, encoded as ~index where index is the position in the content array
+     * @param value new content value to store
+     */
+    private void setContent(int id, T value)
+    {
+        int leadBit = getBufferIdx(~id, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
+        int ofs = inBufferOffset(~id, leadBit, CONTENTS_START_SIZE);
         AtomicReferenceArray<T> array = contentArrays[leadBit];
         array.set(ofs, value);
     }
 
+    private void releaseContent(int id)
+    {
+        objectAllocator.recycle(~id);
+    }
+
+    /**
+     * Called to clean up all buffers when the trie is known to no longer be needed.
+     */
     public void discardBuffers()
     {
         if (bufferType == BufferType.ON_HEAP)
@@ -195,6 +344,42 @@ public void discardBuffers()
         }
     }
 
+    private int copyIfOriginal(int node, int originalNode) throws TrieSpaceExhaustedException
+    {
+        return (node == originalNode)
+               ? copyCell(originalNode)
+               : node;
+    }
+
+    private int getOrAllocate(int pointerAddress, int offsetWhenAllocating) throws TrieSpaceExhaustedException
+    {
+        int child = getIntVolatile(pointerAddress);
+        if (child != NONE)
+            return child;
+
+        child = allocateCell() | offsetWhenAllocating;
+        // volatile writes not needed because this branch is not attached yet
+        putInt(pointerAddress, child);
+        return child;
+    }
+
+    private int getCopyOrAllocate(int pointerAddress, int originalChild, int offsetWhenAllocating) throws TrieSpaceExhaustedException
+    {
+        int child = getIntVolatile(pointerAddress);
+        if (child == originalChild)
+        {
+            if (originalChild == NONE)
+                child = allocateCell() | offsetWhenAllocating;
+            else
+                child = copyCell(originalChild);
+
+            // volatile writes not needed because this branch is not attached yet
+            putInt(pointerAddress, child);
+        }
+
+        return child;
+    }
+
     // Write methods
 
     // Write visibility model: writes are not volatile, with the exception of the final write before a call returns
@@ -209,8 +394,52 @@ public void discardBuffers()
      * Attach a child to the given non-content node. This may be an update for an existing branch, or a new child for
      * the node. An update _is_ required (i.e. this is only called when the newChild pointer is not the same as the
      * existing value).
+     * This method is called when the original node content must be preserved for concurrent readers (i.e. any cell to
+     * be modified needs to be copied first.)
+     *
+     * @param node pointer to the node to update or copy
+     * @param originalNode pointer to the node as it was before any updates in the current modification (i.e. apply
+     *                     call) were started. In other words, the node that is currently reachable by readers if they
+     *                     follow the same key, and which will become unreachable for new readers after this update
+     *                     completes. Used to avoid copying again if already done -- if node is already != originalNode
+     *                     (which is the case when a second or further child of a node is changed by an update),
+     *                     then node is currently not reachable and can be safely modified or completely overwritten.
+     * @param trans transition to modify/add
+     * @param newChild new child pointer
+     * @return pointer to the updated node
+     */
+    private int attachChildCopying(int node, int originalNode, int trans, int newChild) throws TrieSpaceExhaustedException
+    {
+        assert !isLeaf(node) : "attachChild cannot be used on content nodes.";
+
+        switch (offset(node))
+        {
+            case PREFIX_OFFSET:
+                assert false : "attachChild cannot be used on content nodes.";
+            case SPARSE_OFFSET:
+                // If the node is already copied (e.g. this is not the first child being modified), there's no need to copy
+                // it again.
+                return attachChildToSparseCopying(node, originalNode, trans, newChild);
+            case SPLIT_OFFSET:
+                // This call will copy the split node itself and any intermediate cells as necessary to make sure cells
+                // reachable from the original node are not modified.
+                return attachChildToSplitCopying(node, originalNode, trans, newChild);
+            default:
+                // chain nodes
+                return attachChildToChainCopying(node, originalNode, trans, newChild); // always copies
+        }
+    }
+
+    /**
+     * Attach a child to the given node. This may be an update for an existing branch, or a new child for the node.
+     * An update _is_ required (i.e. this is only called when the newChild pointer is not the same as the existing value).
+     *
+     * @param node pointer to the node to update or copy
+     * @param trans transition to modify/add
+     * @param newChild new child pointer
+     * @return pointer to the updated node; same as node if update was in-place
      */
-    private int attachChild(int node, int trans, int newChild) throws SpaceExhaustedException
+    private int attachChild(int node, int trans, int newChild) throws TrieSpaceExhaustedException
     {
         assert !isLeaf(node) : "attachChild cannot be used on content nodes.";
 
@@ -221,16 +450,7 @@ private int attachChild(int node, int trans, int newChild) throws SpaceExhausted
             case SPARSE_OFFSET:
                 return attachChildToSparse(node, trans, newChild);
             case SPLIT_OFFSET:
-                attachChildToSplit(node, trans, newChild);
-                return node;
-            case LAST_POINTER_OFFSET - 1:
-                // If this is the last character in a Chain block, we can modify the child in-place
-                if (trans == getUnsignedByte(node))
-                {
-                    putIntVolatile(node + 1, newChild);
-                    return node;
-                }
-                // else pass through
+                return attachChildToSplit(node, trans, newChild);
             default:
                 return attachChildToChain(node, trans, newChild);
         }
@@ -239,48 +459,95 @@ private int attachChild(int node, int trans, int newChild) throws SpaceExhausted
     /**
      * Attach a child to the given split node. This may be an update for an existing branch, or a new child for the node.
      */
-    private void attachChildToSplit(int node, int trans, int newChild) throws SpaceExhaustedException
+    private int attachChildToSplit(int node, int trans, int newChild) throws TrieSpaceExhaustedException
     {
-        int midPos = splitBlockPointerAddress(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
-        int mid = getInt(midPos);
+        int midPos = splitCellPointerAddress(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
+        int mid = getIntVolatile(midPos);
         if (isNull(mid))
         {
             mid = createEmptySplitNode();
-            int tailPos = splitBlockPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+            int tailPos = splitCellPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
             int tail = createEmptySplitNode();
-            int childPos = splitBlockPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+            int childPos = splitCellPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
             putInt(childPos, newChild);
             putInt(tailPos, tail);
             putIntVolatile(midPos, mid);
-            return;
+            return node;
         }
 
-        int tailPos = splitBlockPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
-        int tail = getInt(tailPos);
+        int tailPos = splitCellPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+        int tail = getIntVolatile(tailPos);
         if (isNull(tail))
         {
             tail = createEmptySplitNode();
-            int childPos = splitBlockPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+            int childPos = splitCellPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
             putInt(childPos, newChild);
             putIntVolatile(tailPos, tail);
-            return;
+            return node;
         }
 
-        int childPos = splitBlockPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+        int childPos = splitCellPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
         putIntVolatile(childPos, newChild);
+        return node;
+    }
+
+    /**
+     * Non-volatile version of attachChildToSplit. Used when the split node is not reachable yet (during the conversion
+     * from sparse).
+     */
+    private int attachChildToSplitNonVolatile(int node, int trans, int newChild) throws TrieSpaceExhaustedException
+    {
+        assert offset(node) == SPLIT_OFFSET : "Invalid split node in trie";
+        int midPos = splitCellPointerAddress(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
+        int mid = getOrAllocate(midPos, SPLIT_OFFSET);
+        assert offset(mid) == SPLIT_OFFSET : "Invalid split node in trie";
+        int tailPos = splitCellPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+        int tail = getOrAllocate(tailPos, SPLIT_OFFSET);
+        assert offset(tail) == SPLIT_OFFSET : "Invalid split node in trie";
+        int childPos = splitCellPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+        putInt(childPos, newChild);
+        return node;
+    }
+
+    /**
+     * Attach a child to the given split node, copying all modified content to enable atomic visibility
+     * of modification.
+     * This may be an update for an existing branch, or a new child for the node.
+     */
+    private int attachChildToSplitCopying(int node, int originalNode, int trans, int newChild) throws TrieSpaceExhaustedException
+    {
+        if (offset(originalNode) != SPLIT_OFFSET)  // includes originalNode == NONE
+            return attachChildToSplitNonVolatile(node, trans, newChild);
+
+        node = copyIfOriginal(node, originalNode);
+        assert offset(node) == SPLIT_OFFSET : "Invalid split node in trie";
+
+        int midPos = splitCellPointerAddress(0, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
+        int midOriginal = originalNode != NONE ? getIntVolatile(midPos + originalNode) : NONE;
+        int mid = getCopyOrAllocate(node + midPos, midOriginal, SPLIT_OFFSET);
+        assert offset(mid) == SPLIT_OFFSET : "Invalid split node in trie";
+
+        int tailPos = splitCellPointerAddress(0, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+        int tailOriginal = midOriginal != NONE ? getIntVolatile(tailPos + midOriginal) : NONE;
+        int tail = getCopyOrAllocate(mid + tailPos, tailOriginal, SPLIT_OFFSET);
+        assert offset(tail) == SPLIT_OFFSET : "Invalid split node in trie";
+
+        int childPos = splitCellPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+        putInt(childPos, newChild);
+        return node;
     }
 
     /**
      * Attach a child to the given sparse node. This may be an update for an existing branch, or a new child for the node.
      */
-    private int attachChildToSparse(int node, int trans, int newChild) throws SpaceExhaustedException
+    private int attachChildToSparse(int node, int trans, int newChild) throws TrieSpaceExhaustedException
     {
         int index;
         int smallerCount = 0;
         // first check if this is an update and modify in-place if so
         for (index = 0; index < SPARSE_CHILD_COUNT; ++index)
         {
-            if (isNull(getInt(node + SPARSE_CHILDREN_OFFSET + index * 4)))
+            if (isNull(getIntVolatile(node + SPARSE_CHILDREN_OFFSET + index * 4)))
                 break;
             final int existing = getUnsignedByte(node + SPARSE_BYTES_OFFSET + index);
             if (existing == trans)
@@ -296,22 +563,14 @@ else if (existing < trans)
         if (childCount == SPARSE_CHILD_COUNT)
         {
             // Node is full. Switch to split
-            int split = createEmptySplitNode();
-            for (int i = 0; i < SPARSE_CHILD_COUNT; ++i)
-            {
-                int t = getUnsignedByte(node + SPARSE_BYTES_OFFSET + i);
-                int p = getInt(node + SPARSE_CHILDREN_OFFSET + i * 4);
-                attachChildToSplitNonVolatile(split, t, p);
-            }
-            attachChildToSplitNonVolatile(split, trans, newChild);
-            return split;
+            return upgradeSparseToSplit(node, trans, newChild);
         }
 
         // Add a new transition. They are not kept in order, so append it at the first free position.
         putByte(node + SPARSE_BYTES_OFFSET + childCount, (byte) trans);
 
         // Update order word.
-        int order = getUnsignedShort(node + SPARSE_ORDER_OFFSET);
+        int order = getUnsignedShortVolatile(node + SPARSE_ORDER_OFFSET);
         int newOrder = insertInOrderWord(order, childCount, smallerCount);
 
         // Sparse nodes have two access modes: via the order word, when listing transitions, or directly to characters
@@ -331,9 +590,71 @@ else if (existing < trans)
         return node;
     }
 
+    /**
+     * Attach a child to the given sparse node. This may be an update for an existing branch, or a new child for the node.
+     * Resulting node is not reachable, no volatile set needed.
+     */
+    private int attachChildToSparseCopying(int node, int originalNode, int trans, int newChild) throws TrieSpaceExhaustedException
+    {
+        int index;
+        int smallerCount = 0;
+        // first check if this is an update and modify in-place if so
+        for (index = 0; index < SPARSE_CHILD_COUNT; ++index)
+        {
+            if (isNull(getIntVolatile(node + SPARSE_CHILDREN_OFFSET + index * 4)))
+                break;
+            final int existing = getUnsignedByte(node + SPARSE_BYTES_OFFSET + index);
+            if (existing == trans)
+            {
+                node = copyIfOriginal(node, originalNode);
+                putInt(node + SPARSE_CHILDREN_OFFSET + index * 4, newChild);
+                return node;
+            }
+            else if (existing < trans)
+                ++smallerCount;
+        }
+        int childCount = index;
+
+        if (childCount == SPARSE_CHILD_COUNT)
+        {
+            // Node is full. Switch to split.
+            // Note that even if node != originalNode, we still have to recycle it as it was a temporary one that will
+            // no longer be attached.
+            return upgradeSparseToSplit(node, trans, newChild);
+        }
+
+        node = copyIfOriginal(node, originalNode);
+
+        // Add a new transition. They are not kept in order, so append it at the first free position.
+        putByte(node + SPARSE_BYTES_OFFSET + childCount,  (byte) trans);
+
+        putInt(node + SPARSE_CHILDREN_OFFSET + childCount * 4, newChild);
+
+        // Update order word.
+        int order = getUnsignedShortVolatile(node + SPARSE_ORDER_OFFSET);
+        int newOrder = insertInOrderWord(order, childCount, smallerCount);
+        putShort(node + SPARSE_ORDER_OFFSET,  (short) newOrder);
+
+        return node;
+    }
+
+    private int upgradeSparseToSplit(int node, int trans, int newChild) throws TrieSpaceExhaustedException
+    {
+        int split = createEmptySplitNode();
+        for (int i = 0; i < SPARSE_CHILD_COUNT; ++i)
+        {
+            int t = getUnsignedByte(node + SPARSE_BYTES_OFFSET + i);
+            int p = getIntVolatile(node + SPARSE_CHILDREN_OFFSET + i * 4);
+            attachChildToSplitNonVolatile(split, t, p);
+        }
+        attachChildToSplitNonVolatile(split, trans, newChild);
+        recycleCell(node);
+        return split;
+    }
+
     /**
      * Insert the given newIndex in the base-6 encoded order word in the correct position with respect to the ordering.
-     *
+     * <p>
      * E.g.
      *   - insertOrderWord(120, 3, 0) must return 1203 (decimal 48*6 + 3)
      *   - insertOrderWord(120, 3, 1, ptr) must return 1230 (decimal 8*36 + 3*6 + 0)
@@ -352,61 +673,85 @@ private static int insertInOrderWord(int order, int newIndex, int smallerCount)
     }
 
     /**
-     * Non-volatile version of attachChildToSplit. Used when the split node is not reachable yet (during the conversion
-     * from sparse).
+     * Attach a child to the given chain node. This may be an update for an existing branch with different target
+     * address, or a second child for the node.
+     * This method always copies the node -- with the exception of updates that change the child of the last node in a
+     * chain cell with matching transition byte (which this method is not used for, see attachChild), modifications to
+     * chain nodes cannot be done in place, either because we introduce a new transition byte and have to convert from
+     * the single-transition chain type to sparse, or because we have to remap the child from the implicit node + 1 to
+     * something else.
      */
-    private void attachChildToSplitNonVolatile(int node, int trans, int newChild) throws SpaceExhaustedException
+    private int attachChildToChain(int node, int transitionByte, int newChild) throws TrieSpaceExhaustedException
     {
-        assert offset(node) == SPLIT_OFFSET : "Invalid split node in trie";
-        int midPos = splitBlockPointerAddress(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
-        int mid = getInt(midPos);
-        if (isNull(mid))
-        {
-            mid = createEmptySplitNode();
-            putInt(midPos, mid);
-        }
-
-        assert offset(mid) == SPLIT_OFFSET : "Invalid split node in trie";
-        int tailPos = splitBlockPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
-        int tail = getInt(tailPos);
-        if (isNull(tail))
+        int existingByte = getUnsignedByte(node);
+        if (transitionByte == existingByte)
         {
-            tail = createEmptySplitNode();
-            putInt(tailPos, tail);
+            // This is still a single path. Update child if possible (only if this is the last character in the chain).
+            if (offset(node) == LAST_POINTER_OFFSET - 1)
+            {
+                putIntVolatile(node + 1, newChild);
+                return node;
+            }
+            else
+            {
+                // This will only be called if new child is different from old, and the update is not on the final child
+                // where we can change it in place (see attachChild). We must always create something new.
+                // Note that since this is not the last character, we either still need this cell or we have already
+                // released it (a createSparseNode must have been called earlier).
+                // If the child is a chain, we can expand it (since it's a different value, its branch must be new and
+                // nothing can already reside in the rest of the cell).
+                return expandOrCreateChainNode(transitionByte, newChild);
+            }
         }
 
-        assert offset(tail) == SPLIT_OFFSET : "Invalid split node in trie";
-        int childPos = splitBlockPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
-        putInt(childPos, newChild);
+        // The new transition is different, so we no longer have only one transition. Change type.
+        return convertChainToSparse(node, existingByte, newChild, transitionByte);
     }
 
     /**
-     * Attach a child to the given chain node. This may be an update for an existing branch with different target
-     * address, or a second child for the node.
-     * This method always copies the node -- with the exception of updates that change the child of the last node in a
-     * chain block with matching transition byte (which this method is not used for, see attachChild), modifications to
-     * chain nodes cannot be done in place, either because we introduce a new transition byte and have to convert from
-     * the single-transition chain type to sparse, or because we have to remap the child from the implicit node + 1 to
-     * something else.
+     * Attach a child to the given chain node, when we are force-copying.
      */
-    private int attachChildToChain(int node, int transitionByte, int newChild) throws SpaceExhaustedException
+    private int attachChildToChainCopying(int node, int originalNode, int transitionByte, int newChild)
+    throws TrieSpaceExhaustedException
     {
         int existingByte = getUnsignedByte(node);
         if (transitionByte == existingByte)
         {
-            // This will only be called if new child is different from old, and the update is not on the final child
-            // where we can change it in place (see attachChild). We must always create something new.
-            // If the child is a chain, we can expand it (since it's a different value, its branch must be new and
-            // nothing can already reside in the rest of the block).
+            // This is still a single path.
+            // Make sure we release the cell if it will no longer be referenced (if we update last reference, the whole
+            // path has to move as the other nodes in this chain can't be remapped).
+            if (offset(node) == LAST_POINTER_OFFSET - 1)
+            {
+                assert node == originalNode;    // if we have already created a node, the character can't match what
+                                                // it was created with
+
+                recycleCell(node);
+            }
+
             return expandOrCreateChainNode(transitionByte, newChild);
         }
+        else
+        {
+            // The new transition is different, so we no longer have only one transition. Change type.
+            return convertChainToSparse(node, existingByte, newChild, transitionByte);
+        }
+    }
 
-        // The new transition is different, so we no longer have only one transition. Change type.
+    private int convertChainToSparse(int node, int existingByte, int newChild, int transitionByte)
+    throws TrieSpaceExhaustedException
+    {
         int existingChild = node + 1;
         if (offset(existingChild) == LAST_POINTER_OFFSET)
         {
-            existingChild = getInt(existingChild);
+            existingChild = getIntVolatile(existingChild);
+            // This was a chain with just one transition which will no longer be referenced.
+            // The cell may contain other characters/nodes leading to this, which are also guaranteed to be
+            // unreferenced.
+            // However, these leading nodes may still be in the parent path and will be needed until the
+            // mutation completes.
+            recycleCell(node);
         }
+        // Otherwise the sparse node we will now create references this cell, so it can't be recycled.
         return createSparseNode(existingByte, existingChild, transitionByte, newChild);
     }
 
@@ -419,7 +764,7 @@ private boolean isExpandableChain(int newChild)
     /**
      * Create a sparse node with two children.
      */
-    private int createSparseNode(int byte1, int child1, int byte2, int child2) throws SpaceExhaustedException
+    private int createSparseNode(int byte1, int child1, int byte2, int child2) throws TrieSpaceExhaustedException
     {
         assert byte1 != byte2 : "Attempted to create a sparse node with two of the same transition";
         if (byte1 > byte2)
@@ -430,7 +775,7 @@ private int createSparseNode(int byte1, int child1, int byte2, int child2) throw
             t = child1; child1 = child2; child2 = t;
         }
 
-        int node = allocateBlock() + SPARSE_OFFSET;
+        int node = allocateCell() + SPARSE_OFFSET;
         putByte(node + SPARSE_BYTES_OFFSET + 0,  (byte) byte1);
         putByte(node + SPARSE_BYTES_OFFSET + 1,  (byte) byte2);
         putInt(node + SPARSE_CHILDREN_OFFSET + 0 * 4, child1);
@@ -446,9 +791,9 @@ private int createSparseNode(int byte1, int child1, int byte2, int child2) throw
      * Note that to avoid creating inefficient tries with under-utilized chain nodes, this should only be called from
      * {@link #expandOrCreateChainNode} and other call-sites should call {@link #expandOrCreateChainNode}.
      */
-    private int createNewChainNode(int transitionByte, int newChild) throws SpaceExhaustedException
+    private int createNewChainNode(int transitionByte, int newChild) throws TrieSpaceExhaustedException
     {
-        int newNode = allocateBlock() + LAST_POINTER_OFFSET - 1;
+        int newNode = allocateCell() + LAST_POINTER_OFFSET - 1;
         putByte(newNode, (byte) transitionByte);
         putInt(newNode + 1, newChild);
         // Note: this does not need a volatile write as it is a new node, returning a new pointer, which needs to be
@@ -458,7 +803,7 @@ private int createNewChainNode(int transitionByte, int newChild) throws SpaceExh
 
     /** Like {@link #createNewChainNode}, but if the new child is already a chain node and has room, expand
      * it instead of creating a brand new node. */
-    private int expandOrCreateChainNode(int transitionByte, int newChild) throws SpaceExhaustedException
+    private int expandOrCreateChainNode(int transitionByte, int newChild) throws TrieSpaceExhaustedException
     {
         if (isExpandableChain(newChild))
         {
@@ -471,12 +816,12 @@ private int expandOrCreateChainNode(int transitionByte, int newChild) throws Spa
         return createNewChainNode(transitionByte, newChild);
     }
 
-    private int createEmptySplitNode() throws SpaceExhaustedException
+    private int createEmptySplitNode() throws TrieSpaceExhaustedException
     {
-        return allocateBlock() + SPLIT_OFFSET;
+        return allocateCell() + SPLIT_OFFSET;
     }
 
-    private int createPrefixNode(int contentIndex, int child, boolean isSafeChain) throws SpaceExhaustedException
+    private int createPrefixNode(int contentId, int child, boolean isSafeChain) throws TrieSpaceExhaustedException
     {
         assert !isNullOrLeaf(child) : "Prefix node cannot reference a childless node.";
 
@@ -488,22 +833,22 @@ private int createPrefixNode(int contentIndex, int child, boolean isSafeChain) t
             // Note: for chain nodes we have a risk that the node continues beyond the current point, in which case
             // creating the embedded node may overwrite information that is still needed by concurrent readers or the
             // mutation process itself.
-            node = (child & -BLOCK_SIZE) | PREFIX_OFFSET;
+            node = (child & -CELL_SIZE) | PREFIX_OFFSET;
             putByte(node + PREFIX_FLAGS_OFFSET, (byte) offset);
         }
         else
         {
             // Full prefix node
-            node = allocateBlock() + PREFIX_OFFSET;
+            node = allocateCell() + PREFIX_OFFSET;
             putByte(node + PREFIX_FLAGS_OFFSET, (byte) 0xFF);
             putInt(node + PREFIX_POINTER_OFFSET, child);
         }
 
-        putInt(node + PREFIX_CONTENT_OFFSET, contentIndex);
+        putInt(node + PREFIX_CONTENT_OFFSET, contentId);
         return node;
     }
 
-    private int updatePrefixNodeChild(int node, int child) throws SpaceExhaustedException
+    private int updatePrefixNodeChild(int node, int child, boolean forcedCopy) throws TrieSpaceExhaustedException
     {
         assert offset(node) == PREFIX_OFFSET : "updatePrefix called on non-prefix node";
         assert !isNullOrLeaf(child) : "Prefix node cannot reference a childless node.";
@@ -511,20 +856,30 @@ private int updatePrefixNodeChild(int node, int child) throws SpaceExhaustedExce
         // We can only update in-place if we have a full prefix node
         if (!isEmbeddedPrefixNode(node))
         {
-            // This attaches the child branch and makes it reachable -- the write must be volatile.
-            putIntVolatile(node + PREFIX_POINTER_OFFSET, child);
-            return node;
+            if (!forcedCopy)
+            {
+                // This attaches the child branch and makes it reachable -- the write must be volatile.
+                putIntVolatile(node + PREFIX_POINTER_OFFSET, child);
+                return node;
+            }
+            else
+            {
+                node = copyCell(node);
+                putInt(node + PREFIX_POINTER_OFFSET, child);
+                return node;
+            }
         }
         else
         {
-            int contentIndex = getInt(node + PREFIX_CONTENT_OFFSET);
-            return createPrefixNode(contentIndex, child, true);
+            // No need to recycle this cell because that is already done by the modification of the child
+            int contentId = getIntVolatile(node + PREFIX_CONTENT_OFFSET);
+            return createPrefixNode(contentId, child, true);
         }
     }
 
     private boolean isEmbeddedPrefixNode(int node)
     {
-        return getUnsignedByte(node + PREFIX_FLAGS_OFFSET) < BLOCK_SIZE;
+        return getUnsignedByte(node + PREFIX_FLAGS_OFFSET) < CELL_SIZE;
     }
 
     /**
@@ -539,49 +894,50 @@ private boolean isEmbeddedPrefixNode(int node)
      *                               applied; if the modifications were applied in-place, this will be the same as
      *                               existingPostContentNode, otherwise a completely different pointer; always a non-
      *                               content node
+     * @param forcedCopy whether or not we need to preserve all pre-existing data for concurrent readers
      * @return a node which has the children of updatedPostContentNode combined with the content of
      *         existingPreContentNode
      */
     private int preserveContent(int existingPreContentNode,
                                 int existingPostContentNode,
-                                int updatedPostContentNode) throws SpaceExhaustedException
+                                int updatedPostContentNode,
+                               boolean forcedCopy)
+    throws TrieSpaceExhaustedException
     {
         if (existingPreContentNode == existingPostContentNode)
             return updatedPostContentNode;     // no content to preserve
 
         if (existingPostContentNode == updatedPostContentNode)
+        {
+            assert !forcedCopy;
             return existingPreContentNode;     // child didn't change, no update necessary
+        }
 
         // else we have existing prefix node, and we need to reference a new child
         if (isLeaf(existingPreContentNode))
         {
-            return createPrefixNode(~existingPreContentNode, updatedPostContentNode, true);
+            return createPrefixNode(existingPreContentNode, updatedPostContentNode, true);
         }
 
         assert offset(existingPreContentNode) == PREFIX_OFFSET : "Unexpected content in non-prefix and non-leaf node.";
-        return updatePrefixNodeChild(existingPreContentNode, updatedPostContentNode);
+        return updatePrefixNodeChild(existingPreContentNode, updatedPostContentNode, forcedCopy);
     }
 
-    final ApplyState applyState = new ApplyState();
+    private final ApplyState applyState = new ApplyState();
 
     /**
      * Represents the state for an {@link #apply} operation. Contains a stack of all nodes we descended through
      * and used to update the nodes with any new data during ascent.
-     *
+     * <p>
      * To make this as efficient and GC-friendly as possible, we use an integer array (instead of is an object stack)
      * and we reuse the same object. The latter is safe because memtable tries cannot be mutated in parallel by multiple
      * writers.
      */
-    class ApplyState
+    private class ApplyState implements KeyProducer<T>
     {
         int[] data = new int[16 * 5];
         int currentDepth = -1;
 
-        void reset()
-        {
-            currentDepth = -1;
-        }
-
         /**
          * Pointer to the existing node before skipping over content nodes, i.e. this is either the same as
          * existingPostContentNode or a pointer to a prefix or leaf node whose child is existingPostContentNode.
@@ -636,96 +992,142 @@ void setTransition(int transition)
         {
             data[currentDepth * 5 + 3] = transition;
         }
+        int transitionAtDepth(int stackDepth)
+        {
+            return data[stackDepth * 5 + 3];
+        }
 
         /**
-         * The compiled content index. Needed because we can only access a cursor's content on the way down but we can't
+         * The compiled content id. Needed because we can only access a cursor's content on the way down but we can't
          * attach it until we ascend from the node.
          */
-        int contentIndex()
+        int contentId()
         {
             return data[currentDepth * 5 + 4];
         }
-        void setContentIndex(int value)
+        void setContentId(int value)
         {
             data[currentDepth * 5 + 4] = value;
         }
+        int contentIdAtDepth(int stackDepth)
+        {
+            return data[stackDepth * 5 + 4];
+        }
+
+        ApplyState start()
+        {
+            int existingFullNode = root;
+            currentDepth = 0;
+
+            descendInto(existingFullNode);
+            return this;
+        }
 
         /**
-         * Descend to a child node. Prepares a new entry in the stack for the node.
+         * Returns true if the depth signals mutation cursor is exhausted.
          */
-        <U> void descend(int transition, U mutationContent, final UpsertTransformer<T, U> transformer)
+        boolean advanceTo(int depth, int transition, int forcedCopyDepth) throws TrieSpaceExhaustedException
         {
-            int existingPreContentNode;
-            if (currentDepth < 0)
-                existingPreContentNode = root;
-            else
+            while (currentDepth > Math.max(0, depth - 1))
             {
-                setTransition(transition);
-                existingPreContentNode = isNull(existingPostContentNode())
-                                         ? NONE
-                                         : getChild(existingPostContentNode(), transition);
+                // There are no more children. Ascend to the parent state to continue walk.
+                attachAndMoveToParentState(forcedCopyDepth);
             }
+            if (depth == -1)
+                return true;
 
+            // We have a transition, get child to descend into
+            descend(transition);
+            return false;
+        }
+
+        /**
+         * Descend to a child node. Prepares a new entry in the stack for the node.
+         */
+        void descend(int transition)
+        {
+            setTransition(transition);
+            int existingPreContentNode = getChild(existingPreContentNode(), transition);
             ++currentDepth;
+            descendInto(existingPreContentNode);
+        }
+
+        private void descendInto(int existingPreContentNode)
+        {
             if (currentDepth * 5 >= data.length)
                 data = Arrays.copyOf(data, currentDepth * 5 * 2);
             setExistingPreContentNode(existingPreContentNode);
 
-            int existingContentIndex = -1;
+            int existingContentId = NONE;
             int existingPostContentNode;
             if (isLeaf(existingPreContentNode))
             {
-                existingContentIndex = ~existingPreContentNode;
+                existingContentId = existingPreContentNode;
                 existingPostContentNode = NONE;
             }
             else if (offset(existingPreContentNode) == PREFIX_OFFSET)
             {
-                existingContentIndex = getInt(existingPreContentNode + PREFIX_CONTENT_OFFSET);
+                existingContentId = getIntVolatile(existingPreContentNode + PREFIX_CONTENT_OFFSET);
                 existingPostContentNode = followContentTransition(existingPreContentNode);
             }
             else
                 existingPostContentNode = existingPreContentNode;
             setExistingPostContentNode(existingPostContentNode);
             setUpdatedPostContentNode(existingPostContentNode);
+            setContentId(existingContentId);
+        }
 
-            int contentIndex = updateContentIndex(mutationContent, existingContentIndex, transformer);
-            setContentIndex(contentIndex);
+        T getContent()
+        {
+            int contentId = contentId();
+            if (contentId == NONE)
+                return null;
+            return InMemoryTrie.this.getContent(contentId());
         }
 
-        /**
-         * Combine existing and new content.
-         */
-        private <U> int updateContentIndex(U mutationContent, int existingContentIndex, final UpsertTransformer<T, U> transformer)
+        void setContent(T content, boolean forcedCopy) throws TrieSpaceExhaustedException
         {
-            if (mutationContent != null)
+            int contentId = contentId();
+            if (contentId == NONE)
             {
-                if (existingContentIndex != -1)
-                {
-                    final T existingContent = getContent(existingContentIndex);
-                    T combinedContent = transformer.apply(existingContent, mutationContent);
-                    assert (combinedContent != null) : "Transformer cannot be used to remove content.";
-                    setContent(existingContentIndex, combinedContent);
-                    return existingContentIndex;
-                }
-                else
-                {
-                    T combinedContent = transformer.apply(null, mutationContent);
-                    assert (combinedContent != null) : "Transformer cannot be used to remove content.";
-                    return addContent(combinedContent);
-                }
+                if (content != null)
+                    setContentId(InMemoryTrie.this.addContent(content));
+            }
+            else if (content == null)
+            {
+                releaseContent(contentId);
+                setContentId(NONE);
+                // At this point we are not deleting branches on the way up, just making sure we don't hold on to
+                // references to content.
+            }
+            else if (content == InMemoryTrie.this.getContent(contentId))
+            {
+                // no changes, nothing to do
+            }
+            else if (forcedCopy)
+            {
+                releaseContent(contentId);
+                setContentId(InMemoryTrie.this.addContent(content));
             }
             else
-                return existingContentIndex;
+            {
+                InMemoryTrie.this.setContent(contentId, content);
+            }
         }
 
         /**
          * Attach a child to the current node.
          */
-        private void attachChild(int transition, int child) throws SpaceExhaustedException
+        private void attachChild(int transition, int child, boolean forcedCopy) throws TrieSpaceExhaustedException
         {
             int updatedPostContentNode = updatedPostContentNode();
             if (isNull(updatedPostContentNode))
                 setUpdatedPostContentNode(expandOrCreateChainNode(transition, child));
+            else if (forcedCopy)
+                setUpdatedPostContentNode(attachChildCopying(updatedPostContentNode,
+                                                             existingPostContentNode(),
+                                                             transition,
+                                                             child));
             else
                 setUpdatedPostContentNode(InMemoryTrie.this.attachChild(updatedPostContentNode,
                                                                         transition,
@@ -736,70 +1138,215 @@ private void attachChild(int transition, int child) throws SpaceExhaustedExcepti
          * Apply the collected content to a node. Converts NONE to a leaf node, and adds or updates a prefix for all
          * others.
          */
-        private int applyContent() throws SpaceExhaustedException
+        private int applyContent(boolean forcedCopy) throws TrieSpaceExhaustedException
         {
-            int contentIndex = contentIndex();
-            int updatedPostContentNode = updatedPostContentNode();
-            if (contentIndex == -1)
-                return updatedPostContentNode;
-
+            // Note: the old content id itself is already released by setContent. Here we must release any standalone
+            // prefix nodes that may reference it.
+            int contentId = contentId();
+            final int updatedPostContentNode = updatedPostContentNode();
+            final int existingPreContentNode = existingPreContentNode();
+            final int existingPostContentNode = existingPostContentNode();
+
+            // applyPrefixChange does not understand leaf nodes, handle upgrade from and to one explicitly.
             if (isNull(updatedPostContentNode))
-                return ~contentIndex;
+            {
+                if (existingPreContentNode != existingPostContentNode
+                    && !isNullOrLeaf(existingPreContentNode)
+                    && !isEmbeddedPrefixNode(existingPreContentNode))
+                    recycleCell(existingPreContentNode);
+                return contentId;   // also fine for contentId == NONE
+            }
 
-            int existingPreContentNode = existingPreContentNode();
-            int existingPostContentNode = existingPostContentNode();
+            if (isLeaf(existingPreContentNode))
+                return contentId != NONE
+                       ? createPrefixNode(contentId, updatedPostContentNode, true)
+                       : updatedPostContentNode;
+
+            return applyPrefixChange(updatedPostContentNode,
+                                     existingPreContentNode,
+                                     existingPostContentNode,
+                                     contentId,
+                                     forcedCopy);
+        }
+
+        private int applyPrefixChange(int updatedPostPrefixNode,
+                                      int existingPrePrefixNode,
+                                      int existingPostPrefixNode,
+                                      int prefixData,
+                                      boolean forcedCopy)
+        throws TrieSpaceExhaustedException
+        {
+            boolean prefixWasPresent = existingPrePrefixNode != existingPostPrefixNode;
+            boolean prefixWasEmbedded = prefixWasPresent && isEmbeddedPrefixNode(existingPrePrefixNode);
+            if (prefixData == NONE)
+            {
+                if (prefixWasPresent && !prefixWasEmbedded)
+                    recycleCell(existingPrePrefixNode);
+                return updatedPostPrefixNode;
+            }
 
-            // We can't update in-place if there was no preexisting prefix, or if the prefix was embedded and the target
-            // node must change.
-            if (existingPreContentNode == existingPostContentNode ||
-                isNull(existingPostContentNode) ||
-                isEmbeddedPrefixNode(existingPreContentNode) && updatedPostContentNode != existingPostContentNode)
-                return createPrefixNode(contentIndex, updatedPostContentNode, isNull(existingPostContentNode));
+            boolean childChanged = updatedPostPrefixNode != existingPostPrefixNode;
+            boolean dataChanged = !prefixWasPresent || prefixData != getIntVolatile(existingPrePrefixNode + PREFIX_CONTENT_OFFSET);
+            if (!childChanged && !dataChanged)
+                return existingPrePrefixNode;
+
+            if (forcedCopy)
+            {
+                if (!childChanged && prefixWasEmbedded)
+                {
+                    // If we directly create in this case, we will find embedding is possible and will overwrite the
+                    // previous value.
+                    // We could create a separate metadata node referencing the child, but in that case we'll
+                    // use two nodes while one suffices. Instead, copy the child and embed the new metadata.
+                    updatedPostPrefixNode = copyCell(existingPostPrefixNode);
+                }
+                else if (prefixWasPresent && !prefixWasEmbedded)
+                {
+                    recycleCell(existingPrePrefixNode);
+                    // otherwise cell is already recycled by the recycling of the child
+                }
+                return createPrefixNode(prefixData, updatedPostPrefixNode, isNull(existingPostPrefixNode));
+            }
+
+            // We can't update in-place if there was no preexisting prefix, or if the
+            // prefix was embedded and the target node must change.
+            if (!prefixWasPresent || prefixWasEmbedded && childChanged)
+                return createPrefixNode(prefixData, updatedPostPrefixNode, isNull(existingPostPrefixNode));
 
             // Otherwise modify in place
-            if (updatedPostContentNode != existingPostContentNode) // to use volatile write but also ensure we don't corrupt embedded nodes
-                putIntVolatile(existingPreContentNode + PREFIX_POINTER_OFFSET, updatedPostContentNode);
-            assert contentIndex == getInt(existingPreContentNode + PREFIX_CONTENT_OFFSET) : "Unexpected change of content index.";
-            return existingPreContentNode;
+            if (childChanged) // to use volatile write but also ensure we don't corrupt embedded nodes
+                putIntVolatile(existingPrePrefixNode + PREFIX_POINTER_OFFSET, updatedPostPrefixNode);
+            if (dataChanged)
+                putIntVolatile(existingPrePrefixNode + PREFIX_CONTENT_OFFSET, prefixData);
+            return existingPrePrefixNode;
         }
 
         /**
          * After a node's children are processed, this is called to ascend from it. This means applying the collected
          * content to the compiled updatedPostContentNode and creating a mapping in the parent to it (or updating if
          * one already exists).
-         * Returns true if still have work to do, false if the operation is completed.
          */
-        private boolean attachAndMoveToParentState() throws SpaceExhaustedException
+        void attachAndMoveToParentState(int forcedCopyDepth) throws TrieSpaceExhaustedException
         {
-            int updatedPreContentNode = applyContent();
-            int existingPreContentNode = existingPreContentNode();
+            int updatedFullNode = applyContent(currentDepth >= forcedCopyDepth);
+            int existingFullNode = existingPreContentNode();
             --currentDepth;
-            if (currentDepth == -1)
+
+            if (updatedFullNode != existingFullNode)
+                attachChild(transition(), updatedFullNode, currentDepth >= forcedCopyDepth);
+        }
+
+        /**
+         * Ascend and update the root at the end of processing.
+         */
+        void attachRoot(int forcedCopyDepth) throws TrieSpaceExhaustedException
+        {
+            int updatedPreContentNode = applyContent(0 >= forcedCopyDepth);
+            int existingPreContentNode = existingPreContentNode();
+            assert root == existingPreContentNode : "Unexpected change to root. Concurrent trie modification?";
+            if (updatedPreContentNode != existingPreContentNode)
             {
-                assert root == existingPreContentNode : "Unexpected change to root. Concurrent trie modification?";
-                if (updatedPreContentNode != existingPreContentNode)
-                {
-                    // Only write to root if they are different (value doesn't change, but
-                    // we don't want to invalidate the value in other cores' caches unnecessarily).
-                    root = updatedPreContentNode;
-                }
-                return false;
+                // Only write to root if they are different (value doesn't change, but
+                // we don't want to invalidate the value in other cores' caches unnecessarily).
+                root = updatedPreContentNode;
             }
-            if (updatedPreContentNode != existingPreContentNode)
-                attachChild(transition(), updatedPreContentNode);
-            return true;
         }
+
+        public byte[] getBytes()
+        {
+            int arrSize = currentDepth;
+            byte[] data = new byte[arrSize];
+            int pos = 0;
+            for (int i = 0; i < currentDepth; ++i)
+            {
+                int trans = transitionAtDepth(i);
+                data[pos++] = (byte) trans;
+            }
+            return data;
+        }
+
+        public byte[] getBytes(Predicate<T> shouldStop)
+        {
+            if (currentDepth == 0)
+                return new byte[0];
+
+            int arrSize = 1;
+            int i;
+            for (i = currentDepth - 1; i > 0; --i)
+            {
+                int content = contentIdAtDepth(i);
+                if (!isNull(content) && shouldStop.test(InMemoryTrie.this.getContent(content)))
+                    break;
+                ++arrSize;
+            }
+            assert i > 0 || arrSize == currentDepth; // if the loop covers the whole stack, the array must cover the full depth
+
+            byte[] data = new byte[arrSize];
+            int pos = 0;
+            for (; i < currentDepth; ++i)
+            {
+                int trans = transitionAtDepth(i);
+                data[pos++] = (byte) trans;
+            }
+            return data;
+        }
+
+        public ByteComparable.Version byteComparableVersion()
+        {
+            return byteComparableVersion;
+        }
+    }
+
+    public interface KeyProducer<T>
+    {
+        /**
+         * Get the bytes of the path leading to this node.
+         */
+        byte[] getBytes();
+
+        /**
+         * Get the bytes of the path leading to this node from the closest ancestor whose content, after any new inserts
+         * have been applied, satisfies the given predicate.
+         * Note that the predicate is not called for the current position, because its content is not yet prepared when
+         * the method is being called.
+         */
+        byte[] getBytes(Predicate<T> shouldStop);
+
+        ByteComparable.Version byteComparableVersion();
+    }
+
+    /**
+     * Somewhat similar to {@link Trie.MergeResolver}, this encapsulates logic to be applied whenever new content is
+     * being upserted into a {@link InMemoryTrie}. Unlike {@link Trie.MergeResolver}, {@link UpsertTransformer} will be
+     * applied no matter if there's pre-existing content for that trie key/path or not.
+     *
+     * @param <T> The content type for this {@link InMemoryTrie}.
+     * @param <U> The type of the new content being applied to this {@link InMemoryTrie}.
+     */
+    public interface UpsertTransformerWithKeyProducer<T, U>
+    {
+        /**
+         * Called when there's content in the updating trie.
+         *
+         * @param existing Existing content for this key, or null if there isn't any.
+         * @param update   The update, always non-null.
+         * @param keyState An interface that can be used to retrieve the path of the value being updated.
+         * @return The combined value to use.
+         */
+        T apply(T existing, U update, KeyProducer<T> keyState);
     }
 
     /**
-     * Somewhat similar to {@link MergeResolver}, this encapsulates logic to be applied whenever new content is being
-     * upserted into a {@link InMemoryTrie}. Unlike {@link MergeResolver}, {@link UpsertTransformer} will be applied no
-     * matter if there's pre-existing content for that trie key/path or not.
+     * Somewhat similar to {@link Trie.MergeResolver}, this encapsulates logic to be applied whenever new content is
+     * being upserted into a {@link InMemoryTrie}. Unlike {@link Trie.MergeResolver}, {@link UpsertTransformer} will be
+     * applied no matter if there's pre-existing content for that trie key/path or not.
+     * <p>
+     * A version of the above that does not use a {@link KeyProducer}.
      *
      * @param <T> The content type for this {@link InMemoryTrie}.
      * @param <U> The type of the new content being applied to this {@link InMemoryTrie}.
      */
-    public interface UpsertTransformer<T, U>
+    public interface UpsertTransformer<T, U> extends UpsertTransformerWithKeyProducer<T, U>
     {
         /**
          * Called when there's content in the updating trie.
@@ -809,6 +1356,121 @@ public interface UpsertTransformer<T, U>
          * @return The combined value to use. Cannot be null.
          */
         T apply(T existing, U update);
+
+        /**
+         * Version of the above that also provides the path of a value being updated.
+         *
+         * @param existing Existing content for this key, or null if there isn't any.
+         * @param update   The update, always non-null.
+         * @param keyState An interface that can be used to retrieve the path of the value being updated.
+         * @return The combined value to use. Cannot be null.
+         */
+        default T apply(T existing, U update, KeyProducer<T> keyState)
+        {
+            return apply(existing, update);
+        }
+    }
+
+    /**
+     * Interface providing features of the mutating node during mutation done using {@link #apply}.
+     * Effectively a subset of the {@link Trie.Cursor} interface which only permits operations that are safe to
+     * perform before iterating the children of the mutation node to apply the branch mutation.
+     *
+     * This is mainly used as an argument to predicates that decide when to copy substructure when modifying tries,
+     * which enables different kinds of atomicity and consistency guarantees.
+     *
+     * See the InMemoryTrie javadoc or InMemoryTrieThreadedTest for demonstration of the typical usages and what they
+     * achieve.
+     */
+    public interface NodeFeatures<T>
+    {
+        /**
+         * Whether or not the node has more than one descendant. If a checker needs mutations to be atomic, they can
+         * return true when this becomes true.
+         */
+        boolean isBranching();
+
+        /**
+         * The metadata associated with the node. If readers need to see a consistent view (i.e. where older updates
+         * cannot be missed if a new one is presented) below some specified point (e.g. within a partition), the checker
+         * should return true when it identifies that point.
+         */
+        T content();
+    }
+
+    private static class Mutation<T, U> implements NodeFeatures<U>
+    {
+        final UpsertTransformerWithKeyProducer<T, U> transformer;
+        final Predicate<NodeFeatures<U>> needsForcedCopy;
+        final Cursor<U> mutationCursor;
+        final InMemoryTrie<T>.ApplyState state;
+        int forcedCopyDepth;
+
+        Mutation(UpsertTransformerWithKeyProducer<T, U> transformer,
+                 Predicate<NodeFeatures<U>> needsForcedCopy,
+                 Cursor<U> mutationCursor,
+                 InMemoryTrie<T>.ApplyState state)
+        {
+            assert mutationCursor.depth() == 0 : "Unexpected non-fresh cursor.";
+            assert state.currentDepth == 0 : "Unexpected change to applyState. Concurrent trie modification?";
+            this.transformer = transformer;
+            this.needsForcedCopy = needsForcedCopy;
+            this.mutationCursor = mutationCursor;
+            this.state = state;
+        }
+
+        void apply() throws TrieSpaceExhaustedException
+        {
+            int depth = state.currentDepth;
+            while (true)
+            {
+                if (depth <= forcedCopyDepth)
+                    forcedCopyDepth = needsForcedCopy.test(this) ? depth : Integer.MAX_VALUE;
+
+                applyContent();
+
+                depth = mutationCursor.advance();
+                if (state.advanceTo(depth, mutationCursor.incomingTransition(), forcedCopyDepth))
+                    break;
+                assert state.currentDepth == depth : "Unexpected change to applyState. Concurrent trie modification?";
+            }
+        }
+
+        void applyContent() throws TrieSpaceExhaustedException
+        {
+            U content = mutationCursor.content();
+            if (content != null)
+            {
+                T existingContent = state.getContent();
+                T combinedContent = transformer.apply(existingContent, content, state);
+                state.setContent(combinedContent, // can be null
+                                 state.currentDepth >= forcedCopyDepth); // this is called at the start of processing
+            }
+        }
+
+
+        void complete() throws TrieSpaceExhaustedException
+        {
+            assert state.currentDepth == 0 : "Unexpected change to applyState. Concurrent trie modification?";
+            state.attachRoot(forcedCopyDepth);
+        }
+
+        @Override
+        public boolean isBranching()
+        {
+            // This is not very efficient, but we only currently use this option in tests.
+            // If it's needed for production use, isBranching should be implemented in the cursor interface.
+            Cursor<U> dupe = mutationCursor.tailTrie().cursor(Direction.FORWARD);
+            int childDepth = dupe.advance();
+            return childDepth > 0 &&
+                   dupe.skipTo(childDepth, dupe.incomingTransition() + 1) == childDepth;
+        }
+
+        @Override
+        public U content()
+        {
+            return mutationCursor.content();
+        }
     }
 
     /**
@@ -818,33 +1480,47 @@ public interface UpsertTransformer<T, U>
      * different than the element type for this memtable trie.
      * @param transformer a function applied to the potentially pre-existing value for the given key, and the new
      * value. Applied even if there's no pre-existing value in the memtable trie.
+     * @param needsForcedCopy a predicate which decides when to fully copy a branch to provide atomicity guarantees to
+     * concurrent readers. See NodeFeatures for details.
      */
-    public <U> void apply(Trie<U> mutation, final UpsertTransformer<T, U> transformer) throws SpaceExhaustedException
+    public <U> void apply(Trie<U> mutation,
+                          final UpsertTransformerWithKeyProducer<T, U> transformer,
+                          final Predicate<NodeFeatures<U>> needsForcedCopy)
+    throws TrieSpaceExhaustedException
     {
-        Cursor<U> mutationCursor = mutation.cursor();
-        assert mutationCursor.depth() == 0 : "Unexpected non-fresh cursor.";
-        ApplyState state = applyState;
-        state.reset();
-        state.descend(-1, mutationCursor.content(), transformer);
-        assert state.currentDepth == 0 : "Unexpected change to applyState. Concurrent trie modification?";
-
-        while (true)
+        try
         {
-            int depth = mutationCursor.advance();
-            while (state.currentDepth >= depth)
-            {
-                // There are no more children. Ascend to the parent state to continue walk.
-                if (!state.attachAndMoveToParentState())
-                {
-                    assert depth == -1 : "Unexpected change to applyState. Concurrent trie modification?";
-                    return;
-                }
-            }
-
-            // We have a transition, get child to descend into
-            state.descend(mutationCursor.incomingTransition(), mutationCursor.content(), transformer);
-            assert state.currentDepth == depth : "Unexpected change to applyState. Concurrent trie modification?";
+            Mutation<T, U> m = new Mutation<>(transformer,
+                                              needsForcedCopy,
+                                              mutation.cursor(Direction.FORWARD),
+                                              applyState.start());
+            m.apply();
+            m.complete();
+            completeMutation();
         }
+        catch (Throwable t)
+        {
+            abortMutation();
+            throw t;
+        }
+    }
+
+    /**
+     * Modify this trie to apply the mutation given in the form of a trie. Any content in the mutation will be resolved
+     * with the given function before being placed in this trie (even if there's no pre-existing content in this trie).
+     * @param mutation the mutation to be applied, given in the form of a trie. Note that its content can be of type
+     * different than the element type for this memtable trie.
+     * @param transformer a function applied to the potentially pre-existing value for the given key, and the new
+     * value. Applied even if there's no pre-existing value in the memtable trie.
+     * @param needsForcedCopy a predicate which decides when to fully copy a branch to provide atomicity guarantees to
+     * concurrent readers. See NodeFeatures for details.
+     */
+    public <U> void apply(Trie<U> mutation,
+                          final UpsertTransformer<T, U> transformer,
+                          final Predicate<NodeFeatures<U>> needsForcedCopy)
+    throws TrieSpaceExhaustedException
+    {
+        apply(mutation, (UpsertTransformerWithKeyProducer<T, U>) transformer, needsForcedCopy);
     }
 
     /**
@@ -861,9 +1537,9 @@ public <U> void apply(Trie<U> mutation, final UpsertTransformer<T, U> transforme
      */
     public <R> void putSingleton(ByteComparable key,
                                  R value,
-                                 UpsertTransformer<T, ? super R> transformer) throws SpaceExhaustedException
+                                 UpsertTransformer<T, ? super R> transformer) throws TrieSpaceExhaustedException
     {
-        apply(Trie.singleton(key, value), transformer);
+        apply(Trie.singleton(key, byteComparableVersion, value), transformer, Predicates.alwaysFalse());
     }
 
     /**
@@ -872,7 +1548,7 @@ public <R> void putSingleton(ByteComparable key,
     public <R> void putSingleton(ByteComparable key,
                                  R value,
                                  UpsertTransformer<T, ? super R> transformer,
-                                 boolean useRecursive) throws SpaceExhaustedException
+                                 boolean useRecursive) throws TrieSpaceExhaustedException
     {
         if (useRecursive)
             putRecursive(key, value, transformer);
@@ -892,14 +1568,23 @@ public <R> void putSingleton(ByteComparable key,
      * value (of a potentially different type), returning the final value that will stay in the memtable trie. Applied
      * even if there's no pre-existing value in the memtable trie.
      */
-    public <R> void putRecursive(ByteComparable key, R value, final UpsertTransformer<T, R> transformer) throws SpaceExhaustedException
+    public <R> void putRecursive(ByteComparable key, R value, final UpsertTransformer<T, R> transformer) throws TrieSpaceExhaustedException
     {
-        int newRoot = putRecursive(root, key.asComparableBytes(BYTE_COMPARABLE_VERSION), value, transformer);
-        if (newRoot != root)
-            root = newRoot;
+        try
+        {
+            int newRoot = putRecursive(root, key.asComparableBytes(byteComparableVersion), value, transformer);
+            if (newRoot != root)
+                root = newRoot;
+            completeMutation();
+        }
+        catch (Throwable t)
+        {
+            abortMutation();
+            throw t;
+        }
     }
 
-    private <R> int putRecursive(int node, ByteSource key, R value, final UpsertTransformer<T, R> transformer) throws SpaceExhaustedException
+    private <R> int putRecursive(int node, ByteSource key, R value, final UpsertTransformer<T, R> transformer) throws TrieSpaceExhaustedException
     {
         int transition = key.next();
         if (transition == ByteSource.END_OF_STREAM)
@@ -916,35 +1601,47 @@ private <R> int putRecursive(int node, ByteSource key, R value, final UpsertTran
                             ? attachChild(skippedContent, transition, newChild)  // Single path, no copying required
                             : expandOrCreateChainNode(transition, newChild);
 
-        return preserveContent(node, skippedContent, attachedChild);
+        return preserveContent(node, skippedContent, attachedChild, false);
     }
 
-    private <R> int applyContent(int node, R value, UpsertTransformer<T, R> transformer) throws SpaceExhaustedException
+    private <R> int applyContent(int node, R value, UpsertTransformer<T, R> transformer) throws TrieSpaceExhaustedException
     {
         if (isNull(node))
-            return ~addContent(transformer.apply(null, value));
+            return addContent(transformer.apply(null, value));
 
         if (isLeaf(node))
         {
-            int contentIndex = ~node;
-            setContent(contentIndex, transformer.apply(getContent(contentIndex), value));
+            int contentId = node;
+            setContent(contentId, transformer.apply(getContent(contentId), value));
             return node;
         }
 
         if (offset(node) == PREFIX_OFFSET)
         {
-            int contentIndex = getInt(node + PREFIX_CONTENT_OFFSET);
-            setContent(contentIndex, transformer.apply(getContent(contentIndex), value));
+            int contentId = getIntVolatile(node + PREFIX_CONTENT_OFFSET);
+            setContent(contentId, transformer.apply(getContent(contentId), value));
             return node;
         }
         else
             return createPrefixNode(addContent(transformer.apply(null, value)), node, false);
     }
 
+    private void completeMutation()
+    {
+        cellAllocator.completeMutation();
+        objectAllocator.completeMutation();
+    }
+
+    private void abortMutation()
+    {
+        cellAllocator.abortMutation();
+        objectAllocator.abortMutation();
+    }
+
     /**
      * Returns true if the allocation threshold has been reached. To be called by the the writing thread (ideally, just
      * after the write completes). When this returns true, the user should switch to a new trie as soon as feasible.
-     *
+     * <p>
      * The trie expects up to 10% growth above this threshold. Any growth beyond that may be done inefficiently, and
      * the trie will fail altogether when the size grows beyond 2G - 256 bytes.
      */
@@ -958,72 +1655,108 @@ public boolean reachedAllocatedSizeThreshold()
      * full.
      */
     @VisibleForTesting
-    int advanceAllocatedPos(int wantedPos) throws SpaceExhaustedException
+    int advanceAllocatedPos(int wantedPos) throws TrieSpaceExhaustedException
     {
         while (allocatedPos < wantedPos)
-            allocateBlock();
+            allocateCell();
         return allocatedPos;
     }
 
-    /** Returns the off heap size of the memtable trie itself, not counting any space taken by referenced content. */
-    public long sizeOffHeap()
+    /**
+     * For tests only! Returns the current allocation position.
+     */
+    @VisibleForTesting
+    int getAllocatedPos()
     {
-        return bufferType == BufferType.ON_HEAP ? 0 : allocatedPos;
+        return allocatedPos;
     }
 
-    /** Returns the on heap size of the memtable trie itself, not counting any space taken by referenced content. */
-    public long sizeOnHeap()
+    /**
+     * Returns the off heap size of the memtable trie itself, not counting any space taken by referenced content, or
+     * any space that has been allocated but is not currently in use (e.g. recycled cells or preallocated buffer).
+     * The latter means we are undercounting the actual usage, but the purpose of this reporting is to decide when
+     * to flush out e.g. a memtable and if we include the unused space we would almost always end up flushing out
+     * immediately after allocating a large buffer and not having a chance to use it. Counting only used space makes it
+     * possible to flush out before making these large allocations.
+     */
+    public long usedSizeOffHeap()
     {
-        return contentCount * MemoryMeterStrategy.MEMORY_LAYOUT.getReferenceSize() +
-               REFERENCE_ARRAY_ON_HEAP_SIZE * getChunkIdx(contentCount, CONTENTS_START_SHIFT, CONTENTS_START_SIZE) +
-               (bufferType == BufferType.ON_HEAP ? allocatedPos + EMPTY_SIZE_ON_HEAP : EMPTY_SIZE_OFF_HEAP) +
-               REFERENCE_ARRAY_ON_HEAP_SIZE * getChunkIdx(allocatedPos, BUF_START_SHIFT, BUF_START_SIZE);
+        return bufferType == BufferType.ON_HEAP ? 0 : usedBufferSpace();
     }
 
-    @Override
-    public Iterable<T> valuesUnordered()
+    /**
+     * Returns the on heap size of the memtable trie itself, not counting any space taken by referenced content, or
+     * any space that has been allocated but is not currently in use (e.g. recycled cells or preallocated buffer).
+     * The latter means we are undercounting the actual usage, but the purpose of this reporting is to decide when
+     * to flush out e.g. a memtable and if we include the unused space we would almost always end up flushing out
+     * immediately after allocating a large buffer and not having a chance to use it. Counting only used space makes it
+     * possible to flush out before making these large allocations.
+     */
+    public long usedSizeOnHeap()
     {
-        return () -> new Iterator<T>()
-        {
-            int idx = 0;
-
-            public boolean hasNext()
-            {
-                return idx < contentCount;
-            }
-
-            public T next()
-            {
-                if (!hasNext())
-                    throw new NoSuchElementException();
+        return usedObjectSpace() +
+               REFERENCE_ARRAY_ON_HEAP_SIZE * getBufferIdx(contentCount, CONTENTS_START_SHIFT, CONTENTS_START_SIZE) +
+               (bufferType == BufferType.ON_HEAP ? usedBufferSpace() + EMPTY_SIZE_ON_HEAP : EMPTY_SIZE_OFF_HEAP) +
+               REFERENCE_ARRAY_ON_HEAP_SIZE * getBufferIdx(allocatedPos, BUF_START_SHIFT, BUF_START_SIZE);
+    }
 
-                return getContent(idx++);
-            }
-        };
+    private long usedBufferSpace()
+    {
+        return allocatedPos - cellAllocator.indexCountInPipeline() * CELL_SIZE;
     }
 
-    public int valuesCount()
+    private long usedObjectSpace()
     {
-        return contentCount;
+        return (contentCount - objectAllocator.indexCountInPipeline()) * MEMORY_LAYOUT.getReferenceSize();
     }
 
-    public long unusedReservedMemory()
+    /**
+     * Returns the amount of memory that has been allocated for various buffers but isn't currently in use.
+     * The total on-heap space used by the trie is {@code usedSizeOnHeap() + unusedReservedOnHeapMemory()}.
+     */
+    @VisibleForTesting
+    public long unusedReservedOnHeapMemory()
     {
         int bufferOverhead = 0;
         if (bufferType == BufferType.ON_HEAP)
         {
             int pos = this.allocatedPos;
-            UnsafeBuffer buffer = getChunk(pos);
+            UnsafeBuffer buffer = getBuffer(pos);
             if (buffer != null)
-                bufferOverhead = buffer.capacity() - inChunkPointer(pos);
+                bufferOverhead = buffer.capacity() - inBufferOffset(pos);
+            bufferOverhead += cellAllocator.indexCountInPipeline() * CELL_SIZE;
         }
 
         int index = contentCount;
-        int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
-        int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
+        int leadBit = getBufferIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
+        int ofs = inBufferOffset(index, leadBit, CONTENTS_START_SIZE);
         AtomicReferenceArray<T> contentArray = contentArrays[leadBit];
-        int contentOverhead = ((contentArray != null ? contentArray.length() : 0) - ofs) * MemoryMeterStrategy.MEMORY_LAYOUT.getReferenceSize();
+        int contentOverhead = ((contentArray != null ? contentArray.length() : 0) - ofs);
+        contentOverhead += objectAllocator.indexCountInPipeline();
+        contentOverhead *= MEMORY_LAYOUT.getReferenceSize();
 
         return bufferOverhead + contentOverhead;
     }
+
+    /**
+     * Release all recycled content references, including the ones waiting in still incomplete recycling lists.
+     * This is a test method and can cause null pointer exceptions if used on a live trie.
+     * <p>
+     * If similar functionality is required for non-test purposes, a version of this should be developed that only
+     * releases references on barrier-complete lists.
+     */
+    @VisibleForTesting
+    public void releaseReferencesUnsafe()
+    {
+        for (int idx : objectAllocator.indexesInPipeline())
+            setContent(~idx, null);
+    }
+
+    /**
+     * Returns the number of values in the trie
+     */
+    public int valuesCount()
+    {
+        return contentCount;
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md
index 09c14087319b..1952d864e056 100644
--- a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md
@@ -26,6 +26,8 @@ The main features of its implementation are:
 - using nodes of several different types for efficiency
 - support for content on any node, including intermediate (prefix)
 - support for writes from a single mutator thread concurrent with multiple readers
+- various consistency and atomicity guarantees for readers
+- memory management, off-heap or on-heap
 - maximum trie size of 2GB
 
 
@@ -34,8 +36,7 @@ The main features of its implementation are:
 One of the main design drivers of the memtable trie is the desire to avoid on-heap storage and Java object management.
 The trie thus implements its own memory management for the structure of the trie (content is, at this time, still given
 as Java objects in a content array). The structure resides in one `UnsafeBuffer` (which can be on or off heap as
-desired) and is broken up in 32-byte "cells" (also called "blocks" in the code), which are the unit of allocation,
-update and reuse.
+desired) and is broken up in 32-byte "cells", which are the unit of allocation, update and reuse.
 
 Like all tries, `InMemoryTrie` is built from nodes and has a root pointer. The nodes reside in cells, but there is no
 1:1 correspondence between nodes and cells - some node types pack multiple in one cell, while other types require
@@ -282,14 +283,14 @@ offset|content|
 18 - 1B|pointer to child for ending 110|
 1C - 1F|pointer to child for ending 111|
 
-In any of the cell or pointer positions we can have `NONE`, meaning that such a child (or block of children) does not
+In any of the cell or pointer positions we can have `NONE`, meaning that such a child (or cell of children) does not
 exist. At minimum, a split node occupies 3 cells (one leading, one mid and one end), and at maximum &mdash;
 `1 + 4 + 4*8 = 37` cells i.e. `1184` bytes. If we could allocate contiguous arrays, a full split node would use `1024`
 bytes, thus this splitting can add ~15% overhead. However, real data often has additional structure that this can make
-use of to avoid creating some of the blocks, e.g. if the trie encodes US-ASCII or UTF-encoded strings where some
+use of to avoid creating some of the cells, e.g. if the trie encodes US-ASCII or UTF-encoded strings where some
 character ranges are not allowed at all, and others are prevalent. Another benefit is that to change a transition while
-preserving the previous state of the node for concurrent readers we have to only copy three blocks and not the entire
-range of children (applications of this will be given later).
+preserving the previous state of the node for concurrent readers we have to only copy three cells and not the entire
+range of children (applications of this will be given in the [Mutation](#mutation) section).
 
 As an example, suppose we need to add a `0x51` `Q` transition to `0x455` to the 6-children sparse node from the previous
 section. This will generate the following structure:
@@ -465,7 +466,7 @@ This substructure is a little more efficient than storing only one entry for the
 mid-to-tail links do not need to be followed for every new child) and also allows us to easily get the precise next 
 child and remove the backtracking entry when a cell has no further children.
 
-`InMemoryTrie` cursors also implement `advanceMultiple`, which jumps over intermediate nodes in `Chain` blocks:
+`InMemoryTrie` cursors also implement `advanceMultiple`, which jumps over intermediate nodes in `Chain` cells:
 
 ![graph](InMemoryTrie.md.wc2.svg)
 
@@ -540,7 +541,8 @@ Note that if we perform multiple mutations in sequence, and a reader happens to
 order), such reader may see only the mutation that is ahead of it _in iteration order_, which is not necessarily the
 mutation that happened first. For the example above, if we also inserted `trespass`, a reader thread that was paused
 at `0x018` in a forward traversal and wakes up after both insertions have completed will see `trespass`, but _will not_
-see `traverse` even though it was inserted earlier.
+see `traverse` even though it was inserted earlier. This inconsistency is often undesirable; we will describe a
+method of avoiding it in one of the next paragraphs.
 
 ### In-place modifications
 
@@ -701,6 +703,10 @@ iteration processes a child, we apply the update to the node, which may happen i
 the original `existingNode`, it was pointing to an unreachable copied node which will remain unreachable as we will only
 attach the newer version.
 
+For reasons to be described below, copying of existing reachable nodes may be enforced. The test `updatedNode
+ == existingNode` can be used to tell if the node is indeed reachable; if we have copied it already there is no need to
+ copy it again to update for a new child (note: we may still need to copy reachable mid or end cells in `Split` nodes).
+
 After all modifications coming as the result of application of child branches have been applied, we have an
 `updatedNode` that reflects all. As we ascend we apply that new value to the parent's `updatedNode`.
 
@@ -722,6 +728,52 @@ manages to attach `truck`;
 - a reading thread that iterated to `tree` (while `traverse` was not yet attached) and paused, will see `truck` if the
 mutating thread applies the update during the pause.
 
+### Atomicity
+
+Atomicity of writes is usually a desirable property. Atomicity means that readers can see either none of the contents
+of a mutation, or all of them, i.e. that they can never see some part of an update and miss another.
+
+We can achieve this by making sure that the application of a mutation has only one attachment point. This is always the
+case for single-path updates (`putRecursive` or `apply` where no mutation node has more than one child). We can achieve
+the same for branching updates if we "force-copy" all memtable trie nodes at or below the topmost branching node of
+the mutation trie.
+
+This ensures that any partially applied changes are only done in unreachable copy nodes, while concurrent readers
+continue working on the originals. Once the branch is fully prepared, we attach it using one in-place write which makes
+the whole of it visible.
+
+The example above with atomic writes will be done as
+
+![graph](InMemoryTrie.md.a2.svg)
+
+### Consistency
+
+The same idea can also be used to enforce sequential consistency, defined here as the property that all readers that see
+an update must also be able to see all updates that happened before it (alternatively, if a reader does not see an
+update, it will not see any update that was applied after it in the order of execution of the mutating thread).
+
+Inconsistencies happen because, while a reader is traversing through it, a branch can change at random places, some of
+which may be before or after the reader's position in iteration order. We can avoid this problem if we ensure that the
+snapshot the reader is operating on does not change.
+
+To do this, we must force-copy any node we update, until the modification proceeds to the root pointer, which we then
+update to the new value (i.e. we force the attachment point for the mutation to be the root pointer). Any reader who has
+already read the root pointer will not see any updates that apply after that point in time. Any reader who reads the new
+pointer will see everything that the mutation thread did until it wrote that pointer, i.e. the last mutation and all
+mutations that precede it.
+
+![graph](InMemoryTrie.md.a3.svg)
+
+At the end of the processing of this example, the root pointer is written volatile to the new value `0x13A`. Although we
+maintain a full snapshot of the trie, we did not need to copy all nodes, only the ones that were touched by the update
+(i.e. the extra space is proportional to the update size, not to the size of the recipient trie).
+
+Consistency can also be applied below a point selected by the user (e.g. below user-identifiable metadata). In this case
+the snapshot is preserved only for nodes at or below the identified point, i.e. force-copying applies at that level
+and below, and the attachment point of any update is above the identified point &mdash; readers who see the new link
+must also see anything that the mutation thread did below that point, including any mutation that preceded the last and
+all modified content in the protected branch.
+
 ### Handling prefix nodes
 
 The descriptions above were given without prefix nodes. Handling prefixes is just a little complication over the update
@@ -732,7 +784,7 @@ To do this we expand the state tracked to:
   nodes like a prefix with no child) and is the base for all child updates (i.e. it takes the role of
   `existingNode` in the descriptions above),
 - `updatedPostContentNode` which is the node as changed/copied after children modifications are applied,
-- `contentIndex` which is the index in the content array for the result of merging existing and newly introduced 
+- `contentIndex` which is the index in the content array for the result of merging existing and newly introduced
   content, (Note: The mutation content is only readable when the cursor enters the node, and we can only attach it when
   we ascend from it.)
 - `transition` remains as before.
@@ -751,3 +803,93 @@ When descending at `tree` we set `existingPreContentNode = ~1`, `existingPostCon
 Ascending back to add the child `~3`, we add a child to `NONE` and get `updatedPostContentNode = 0x0BB`. To then apply
 the existing content, we create the embedded prefix node `updatedPreContentNode = 0x0BF` with `contentIndex = 1` and
 pass that on to the recursion.
+
+### Memory management and cell reuse
+
+As mentioned in the beginning, in order to avoid long garbage collection pauses due to large long-lasting content in
+memtable tries such as the ones used to store database memtables, `InMemoryTrie` uses its own memory management, and
+can be used with on- or off-heap memory.
+
+The most important uses for the trie are long-lived ones, but there are also cases where we want to compose small
+short-lived tries, for example to store the result of a query, or to prepare a partition update before it is merged
+with the memtable. The two usecases are served most efficiently by using different allocation and reuse methods, which
+is why `InMemoryTrie`s offer two factory methods to create two different kinds of tries:
+
+- `InMemoryTrie.shortLived()` creates a trie that is expected to remain relatively small, to be used only for a short
+  period (e.g. the duration of a write or read request), and typically to be accessed by one thread only. These tries
+  reside on heap, because allocation and release of smaller buffers is done more efficiently using garbage collection, and
+  do not make any attempt to reclaim cells that become unused due to copying or type change. In tries that are accessed
+  by one thread only, mutations can safely be made without any atomicity or consistency concerns thus without any forced
+  copying, which means that the overhead of not reclaiming cells should be inconsequential.
+  `PartitionUpdate`s are an example of short-lived tries, where mutations are prepared before being sent to the commit log
+  and merged into a memtable.
+
+- `InMemoryTrie.longLived(OpOrder)` creates a trie which is expected to grow large, to remain in place for a long time,
+  and to be read by a multitude of threads concurrently with a single mutator. This kind of trie will usually be off-heap,
+  and will reclaim any cells that become unreferenced due to copying or type change. Because recycling cells requires the
+  trie/mutator to know if a reader can still be looking at cells that have become unreachable, long lived tries rely on an
+  `OpOrder` which readers must take a group from before reading the trie, and release when done.
+  `MemtableShard` (and in general database memtables) use long-lived tries, with the table's `readOrdering` (which all
+  reads already use) as the `OpOrder` signal when unreachable cells can be reused.
+
+The on/off-heap distinction is handled by the `BufferType` used in constructing the trie, while the recycling strategy
+is handled by the one of the two `MemtableAllocationStrategies`. `NoReuseStrategy`, used by the short-lived option is
+trivial, simply bump-allocating cells and objects from linear byte buffer and object array. `OpOrderReuseStrategy`
+handles the long-lived case and will be detailed below.
+
+The descriptions in this section talk about cells, but we apply exactly the same mechanisms for handling slots in the
+java object content array (with separate queues).
+
+#### Cell recycling in long-lived tries
+
+During the application of a mutation, the `InMemoryTrie` code knows which cells are being copied to another location and
+tells the allocation strategy that the cells are going to be freed (using a `recycleCell` or implied in `copyCell`).
+This does not mean that the old cell is already free, because:
+- (1) it is probably still reachable (if the process has not backtracked enough to attach the new cell to some
+  parent) by concurrent readers;
+- (2) the procedure may fail before the attachment point and the old cell may remain reachable even for this thread;
+- (3) it may still be needed by the mutator (e.g. a chain cell is freed when we recognize that the last node in the
+  chain needs to be moved, but the other nodes in the cell are still in the parent path for the mutation process); or
+- (4) concurrent readers may hold a pointer to the old cell, or a parent or child chain that leads to it.
+
+Thus, we can only recycle a cell when all four conditions are no longer possible. Once an attachment has been made, (1)
+and (2) are no longer possible (since attachment writes are volatile, all threads that visit that point at any time
+after the write _must_ see the new paths). (3) becomes impossible when the mutation completes. To make things simple,
+we use the completion of a mutation (signalled to the allocation strategy via a `completeMutation` call) as the point
+in time when all attachments are in place. To make sure (4) is no longer happening, the allocation strategy relies on
+the given `OpOrder` &mdash; when a barrier, issued at any point _after_ the `completeMutation` call, expires, no cells
+identified by the mutation as recyclable can be referenced in any current readers, because they must have followed the
+newly set paths and thus cannot have reached those cells.
+
+The allocation strategy implements this by maintaining several lists:
+- just-released cells, added in response to `recycleCell` calls and awaiting `completeMutation`
+- cells awaiting barrier, moved from the top list after `completeMutation`, for which a barrier has been issued,
+  awaiting the barrier to expire
+- reusable cells, moved from the list above after their barrier has expired
+
+For efficiency the allocation strategy does not work with individual cells, but rather in blocks of ~250. Newly
+allocated cells are taken from a `free` block. When a mutation releases cells, they are put in a `justReleased` block,
+and if the block is filled, another one is created and linked to form a queue. At mutation completion we do nothing if
+no block is yet completed; if one is, we issue a barrier and give it to the block (and any other completed blocks in
+the `justReleased` queue), with which we move it/them to the tail of an `awaitingBarrier` queue. The head of this queue
+is the oldest block of recycled cells and has the highest probability of having passed its barrier &mdash; if any block
+in the queue has an expired barrier, all previous ones also will (because of the logic of expiring barriers in
+`OpOrder`). Hence, when we need to allocate a new cell and the free block is empty, we check if that head block's barrier
+has expired, and if it has, we make that the new `free` block. If the barrier hasn't expired, there is no block of cells
+that is ready for recycling, thus we must refill the `free` block with new cells.
+
+Technically, the "reusable cells" list and "cells awaiting barrier" are in the same linked queue, which is effectively
+split in two parts by the property of having an expired barrier. Also, to simplify handling, the `free` block stands at
+the head of that queue &mdash; it plays the part of a sentinel block for the `awaitingBarrier` queue as we
+always have a block at `free`, thus `awaitingBarrierTail` can move to it when the queue becomes empty.
+
+![diagram](InMemoryTrie.md.recycling.svg)
+
+If an exception is thrown during a mutation, the `InMemoryTrie` code catches that exception and signals `abortMutation`
+to the strategy, which tells it that the cells the current call marked as recyclable will probably remain reachable
+and should be discarded; because the strategy works with blocks, it will actually discard everything in the
+`justReleased` block and queue. This may result in some cell waste &mdash; unreachable cells that cannot be recycled
+&mdash; if cells were allocated and/or an attachment was made before the exception is thrown. We don't expect this to
+happen often, but any users of tries that expect them to live indefinitely (unlike memtables which are flushed
+regularly; an example would be the chunk cache map when/if we switch it to `InMemoryTrie`) must ensure that exceptions
+cannot happen during mutation, otherwise waste can slowly accumulate to bring the node down.
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a2.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a2.svg
new file mode 100644
index 000000000000..553981e795bb
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a2.svg
@@ -0,0 +1,634 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+  - Licensed to the Apache Software Foundation (ASF) under one
+  - or more contributor license agreements.  See the NOTICE file
+  - distributed with this work for additional information
+  - regarding copyright ownership.  The ASF licenses this file
+  - to you under the Apache License, Version 2.0 (the
+  - "License"); you may not use this file except in compliance
+  - with the License.  You may obtain a copy of the License at
+  -
+  -     http://www.apache.org/licenses/LICENSE-2.0
+  -
+  - Unless required by applicable law or agreed to in writing, software
+  - distributed under the License is distributed on an "AS IS" BASIS,
+  - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  - See the License for the specific language governing permissions and
+  - limitations under the License.
+  -->
+
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+        "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# embedded DOT (plantuml) works in IDEA preview
+# but not on GitHub
+```plantuml
+digraph G {
+    { rank=same root -> start [style=invis] }
+    start [label="start/end"]
+
+    tractor [label = "contentArray[0]"]
+    tracto [label = "0x01B"]
+    tract [label = "0x01A"]
+    trac [label = "0x019"]
+    tra [label = "Multi\n0x018"; color = "lightgrey"; fontcolor = lightgray]
+
+    tree [label = "contentArray[1]"]
+    tre [label = "Multi\n0x03B"]
+    trie [label = "contentArray[2]"]
+    tri [label = "Multi\n0x05B"]
+
+    tr [label = "Sparse\n0x07E"; color = "lightgrey"; fontcolor = lightgray]
+
+    t [label = "0x09B"]
+    root [label = "Multi\n0x9A"]
+
+    root -> t [label = " t"]
+    t -> tr [label = " r"; color = "lightgrey"; fontcolor = lightgray]
+    trac -> tract [label = " t"]
+    tract -> tracto [label = " o"]
+    tracto -> tractor [label = " r"]
+
+    tr -> tra [label = " a"; color = "lightgrey"; fontcolor = lightgray]
+    tra -> trac [label = " c"; color = "lightgrey"; fontcolor = lightgray]
+
+    tr -> tre [label = " e"; color = "lightgrey"; fontcolor = lightgray]
+    tre -> tree [label = " e"]
+
+    tr -> tri [label = " i"; color = "lightgrey"; fontcolor = lightgray]
+    tri -> trie [label = " e"]
+
+    subgraph path {
+        edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+        node [color = "blue"; fontcolor="blue"]
+
+        start -> root
+
+        root -> t [label = " t"]
+        t -> tr [label = " r"]
+        tr -> tra [label = " a"]
+        tra -> trav [label = " v"]
+        trav -> trave [label = " e"]
+        trave -> traver [label = " r"]
+        traver -> travers [label = " s"]
+        travers -> traverse [label = " e"]
+
+        tra2 [label = "Sparse\n0x0DE"]
+        trav [label = "Multi\n0x0B8"]
+        trave [label = "0x0B9"]
+        traver [label = "0x0BA"]
+        travers [label = "0x0BB"]
+        traverse [label = "contentArray[3]"]
+
+        tr -> tru [label = " u"]
+        tru -> truc [label = " c"]
+        truc -> truck [label = " k"]
+
+        tr2 [label = "Sparse\n0x0FE"]
+        tru [label = "Multi\n0x11A"]
+        truc [label = "0x11B"]
+        truck [label = "contentArray[4]"]
+    }
+
+    {rank=same tr -> tr2 [style=invis]}
+    {rank=same tra -> tra2 -> tre -> tri -> tru [style=invis]}
+    {rank=same trac -> trav -> tree -> trie -> truc [style=invis]}
+
+    {
+        edge [color = "blue"]
+        t -> tr2 [label = " r"]
+        tr2 -> tre [label = " e"]
+        tr2 -> tri [label = " i"]
+
+        tr2 -> tra2 [label = " a"]
+        tra2 -> trac [label = " c"]
+        tra2 -> trav [label = " v"]
+        trav -> trave [label = " e"]
+        trave -> traver [label = " r"]
+        traver -> travers [label = " s"]
+        travers -> traverse [label = " e"]
+        tr2 -> tru [label = " u"]
+        tru -> truc [label = " c"]
+        truc -> truck [label = " k"]
+    }
+
+    subgraph back {
+        edge [color = "pink"; fontcolor="red"; arrowhead="vee"; constrain="false"]
+
+        traverse -> travers [label = " ~3"]
+        travers -> traver [label = "0x0BB"]
+        traver -> trave [label = "0x0BA"]
+        trave -> trav [label = "0x0B9"]
+        trav -> tra2 [label = "0x0B8"]
+        tra2 -> tr2 [label = "0x0DE"]
+        tr2 -> t [label = "0x07E"]
+        t -> root [label = "0x09B"]
+        root -> start [label = "0x09A"]
+
+        truck -> truc [label = "~4"]
+        truc -> tru [label = "0x11B"]
+        tru -> tr2 [label = "0x11A"]
+    }
+}
+```
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="757pt" height="846pt"
+     viewBox="0.00 0.00 757.47 845.73" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 841.7251)">
+        <title>G</title>
+        <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-841.7251 753.4738,-841.7251 753.4738,4 -4,4"/>
+        <!-- root -->
+        <g id="node1" class="node">
+            <title>root</title>
+            <ellipse fill="none" stroke="#000000" cx="352.2369" cy="-808.3095" rx="33.1337" ry="29.3315"/>
+            <text text-anchor="middle" x="352.2369" y="-812.5095" font-family="Times,serif" font-size="14.00" fill="#000000">Multi</text>
+            <text text-anchor="middle" x="352.2369" y="-795.7095" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+        </g>
+        <!-- start -->
+        <g id="node2" class="node">
+            <title>start</title>
+            <ellipse fill="none" stroke="#000000" cx="485.2369" cy="-808.3095" rx="44.0775" ry="18"/>
+            <text text-anchor="middle" x="485.2369" y="-804.1095" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+        </g>
+        <!-- root&#45;&gt;start -->
+        <!-- root&#45;&gt;start -->
+        <g id="edge55" class="edge">
+            <title>root&#45;&gt;start</title>
+            <path fill="none" stroke="#ffc0cb" d="M385.7984,-808.3095C399.3037,-808.3095 415.2339,-808.3095 430.3398,-808.3095"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="440.6261,-808.3095 430.6262,-812.8096 435.6261,-808.3095 430.6261,-808.3096 430.6261,-808.3096 430.6261,-808.3096 435.6261,-808.3095 430.6261,-803.8096 440.6261,-808.3095 440.6261,-808.3095"/>
+            <text text-anchor="middle" x="413.2509" y="-815.5095" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x09A</text>
+        </g>
+        <!-- t -->
+        <g id="node13" class="node">
+            <title>t</title>
+            <ellipse fill="none" stroke="#000000" cx="352.2369" cy="-708.0939" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="352.2369" y="-703.8939" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+        </g>
+        <!-- root&#45;&gt;t -->
+        <g id="edge2" class="edge">
+            <title>root&#45;&gt;t</title>
+            <path fill="none" stroke="#000000" d="M332.9564,-784.2612C325.7393,-772.3485 320.2768,-757.6462 324.8463,-744.0939 326.1636,-740.1869 328.0926,-736.3684 330.3269,-732.7663"/>
+            <polygon fill="#000000" stroke="#000000" points="333.3844,-734.5068 336.3357,-724.3313 327.683,-730.4454 333.3844,-734.5068"/>
+            <text text-anchor="middle" x="328.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+        </g>
+        <!-- root&#45;&gt;t -->
+        <g id="edge14" class="edge">
+            <title>root&#45;&gt;t</title>
+            <path fill="none" stroke="#add8e6" d="M352.2369,-778.7835C352.2369,-765.2984 352.2369,-749.4062 352.2369,-736.1092"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="352.2369,-726.0972 356.737,-736.0971 352.2369,-731.0972 352.237,-736.0972 352.237,-736.0972 352.237,-736.0972 352.2369,-731.0972 347.737,-736.0972 352.2369,-726.0972 352.2369,-726.0972"/>
+            <text text-anchor="middle" x="355.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+        </g>
+        <!-- start&#45;&gt;root -->
+        <g id="edge13" class="edge">
+            <title>start&#45;&gt;root</title>
+            <path fill="none" stroke="#add8e6" d="M441.4042,-805.6333C429.2938,-805.1818 416.1479,-804.9584 404.0184,-805.3095 401.3264,-805.3874 398.5568,-805.4881 395.7643,-805.6051"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="385.6504,-806.089 395.4239,-801.1161 390.6447,-805.85 395.639,-805.611 395.639,-805.611 395.639,-805.611 390.6447,-805.85 395.8541,-810.1058 385.6504,-806.089 385.6504,-806.089"/>
+        </g>
+        <!-- tractor -->
+        <g id="node3" class="node">
+            <title>tractor</title>
+            <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+        </g>
+        <!-- tracto -->
+        <g id="node4" class="node">
+            <title>tracto</title>
+            <ellipse fill="none" stroke="#000000" cx="74.2369" cy="-195.6" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="74.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+        </g>
+        <!-- tracto&#45;&gt;tractor -->
+        <g id="edge6" class="edge">
+            <title>tracto&#45;&gt;tractor</title>
+            <path fill="none" stroke="#000000" d="M74.0297,-177.2006C73.8934,-165.0949 73.7122,-149.0076 73.5575,-135.2674"/>
+            <polygon fill="#000000" stroke="#000000" points="77.0529,-134.8319 73.4404,-124.872 70.0533,-134.9108 77.0529,-134.8319"/>
+            <text text-anchor="middle" x="78.3172" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+        </g>
+        <!-- tract -->
+        <g id="node5" class="node">
+            <title>tract</title>
+            <ellipse fill="none" stroke="#000000" cx="78.2369" cy="-284.4" rx="37.1443" ry="18"/>
+            <text text-anchor="middle" x="78.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+        </g>
+        <!-- tract&#45;&gt;tracto -->
+        <g id="edge5" class="edge">
+            <title>tract&#45;&gt;tracto</title>
+            <path fill="none" stroke="#000000" d="M77.4081,-266.0006C76.8628,-253.8949 76.1381,-237.8076 75.5192,-224.0674"/>
+            <polygon fill="#000000" stroke="#000000" points="78.9975,-223.5044 75.0509,-213.672 72.0046,-223.8194 78.9975,-223.5044"/>
+            <text text-anchor="middle" x="82.4869" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+        </g>
+        <!-- trac -->
+        <g id="node6" class="node">
+            <title>trac</title>
+            <ellipse fill="none" stroke="#000000" cx="94.2369" cy="-384.6156" rx="35.3587" ry="18"/>
+            <text text-anchor="middle" x="94.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+        </g>
+        <!-- trac&#45;&gt;tract -->
+        <g id="edge4" class="edge">
+            <title>trac&#45;&gt;tract</title>
+            <path fill="none" stroke="#000000" d="M91.3022,-366.2345C88.9115,-351.2603 85.4955,-329.8643 82.756,-312.7055"/>
+            <polygon fill="#000000" stroke="#000000" points="86.1858,-311.9877 81.1529,-302.6646 79.2734,-313.0913 86.1858,-311.9877"/>
+            <text text-anchor="middle" x="89.9322" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+        </g>
+        <!-- trav -->
+        <g id="node14" class="node">
+            <title>trav</title>
+            <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+            <text text-anchor="middle" x="202.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#0000ff">Multi</text>
+            <text text-anchor="middle" x="202.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B8</text>
+        </g>
+        <!-- trac&#45;&gt;trav -->
+        <!-- tra -->
+        <g id="node7" class="node">
+            <title>tra</title>
+            <ellipse fill="none" stroke="#d3d3d3" cx="133.2369" cy="-496.2469" rx="36.125" ry="29.3315"/>
+            <text text-anchor="middle" x="133.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">Multi</text>
+            <text text-anchor="middle" x="133.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">0x018</text>
+        </g>
+        <!-- tra&#45;&gt;trac -->
+        <g id="edge8" class="edge">
+            <title>tra&#45;&gt;trac</title>
+            <path fill="none" stroke="#d3d3d3" d="M114.5188,-470.6739C110.3016,-463.8622 106.2666,-456.3098 103.5223,-448.8313 99.3758,-437.5318 97.0731,-424.4776 95.7973,-413.1021"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="99.2538,-412.4638 94.8622,-402.822 92.2826,-413.098 99.2538,-412.4638"/>
+            <text text-anchor="middle" x="109.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> c</text>
+        </g>
+        <!-- tra&#45;&gt;trav -->
+        <g id="edge17" class="edge">
+            <title>tra&#45;&gt;trav</title>
+            <path fill="none" stroke="#add8e6" d="M130.7527,-466.6197C131.0399,-455.1889 132.9734,-442.3975 138.7369,-432.0313 144.2651,-422.0881 152.8469,-413.6214 161.933,-406.7148"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="170.346,-400.8129 164.7439,-410.2398 166.2527,-403.6844 162.1595,-406.5559 162.1595,-406.5559 162.1595,-406.5559 166.2527,-403.6844 159.5751,-402.872 170.346,-400.8129 170.346,-400.8129"/>
+            <text text-anchor="middle" x="144.4869" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> v</text>
+        </g>
+        <!-- tra2 -->
+        <g id="node19" class="node">
+            <title>tra2</title>
+            <ellipse fill="none" stroke="#0000ff" cx="244.2369" cy="-496.2469" rx="39.2145" ry="29.3315"/>
+            <text text-anchor="middle" x="244.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#0000ff">Sparse</text>
+            <text text-anchor="middle" x="244.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0DE</text>
+        </g>
+        <!-- tra&#45;&gt;tra2 -->
+        <!-- tree -->
+        <g id="node8" class="node">
+            <title>tree</title>
+            <ellipse fill="none" stroke="#000000" cx="348.2369" cy="-384.6156" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="348.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+        </g>
+        <!-- trie -->
+        <g id="node10" class="node">
+            <title>trie</title>
+            <ellipse fill="none" stroke="#000000" cx="530.2369" cy="-384.6156" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="530.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+        </g>
+        <!-- tree&#45;&gt;trie -->
+        <!-- tre -->
+        <g id="node9" class="node">
+            <title>tre</title>
+            <ellipse fill="none" stroke="#000000" cx="357.2369" cy="-496.2469" rx="37.9306" ry="29.3315"/>
+            <text text-anchor="middle" x="357.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Multi</text>
+            <text text-anchor="middle" x="357.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+        </g>
+        <!-- tre&#45;&gt;tree -->
+        <g id="edge10" class="edge">
+            <title>tre&#45;&gt;tree</title>
+            <path fill="none" stroke="#000000" d="M354.8487,-466.6249C353.5073,-449.9873 351.8422,-429.334 350.5186,-412.9163"/>
+            <polygon fill="#000000" stroke="#000000" points="353.9973,-412.5103 349.7049,-402.8239 347.0199,-413.0729 353.9973,-412.5103"/>
+            <text text-anchor="middle" x="359.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- tri -->
+        <g id="node11" class="node">
+            <title>tri</title>
+            <ellipse fill="none" stroke="#000000" cx="469.2369" cy="-496.2469" rx="37.9306" ry="29.3315"/>
+            <text text-anchor="middle" x="469.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Multi</text>
+            <text text-anchor="middle" x="469.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+        </g>
+        <!-- tre&#45;&gt;tri -->
+        <!-- truc -->
+        <g id="node21" class="node">
+            <title>truc</title>
+            <ellipse fill="none" stroke="#0000ff" cx="676.2369" cy="-384.6156" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="676.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x11B</text>
+        </g>
+        <!-- trie&#45;&gt;truc -->
+        <!-- tri&#45;&gt;trie -->
+        <g id="edge12" class="edge">
+            <title>tri&#45;&gt;trie</title>
+            <path fill="none" stroke="#000000" d="M484.1592,-468.9388C493.6824,-451.511 505.9755,-429.0146 515.4687,-411.6418"/>
+            <polygon fill="#000000" stroke="#000000" points="518.6954,-413.0358 520.4193,-402.5821 512.5526,-409.6791 518.6954,-413.0358"/>
+            <text text-anchor="middle" x="508.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- tru -->
+        <g id="node20" class="node">
+            <title>tru</title>
+            <ellipse fill="none" stroke="#0000ff" cx="580.2369" cy="-496.2469" rx="38.0198" ry="29.3315"/>
+            <text text-anchor="middle" x="580.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#0000ff">Multi</text>
+            <text text-anchor="middle" x="580.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x11A</text>
+        </g>
+        <!-- tri&#45;&gt;tru -->
+        <!-- tr -->
+        <g id="node12" class="node">
+            <title>tr</title>
+            <ellipse fill="none" stroke="#d3d3d3" cx="292.2369" cy="-607.8782" rx="37.9027" ry="29.3315"/>
+            <text text-anchor="middle" x="292.2369" y="-612.0782" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">Sparse</text>
+            <text text-anchor="middle" x="292.2369" y="-595.2782" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">0x07E</text>
+        </g>
+        <!-- tr&#45;&gt;tra -->
+        <g id="edge7" class="edge">
+            <title>tr&#45;&gt;tra</title>
+            <path fill="none" stroke="#d3d3d3" d="M255.4131,-600.8857C228.7599,-594.257 193.2255,-582.0058 168.5223,-560.4626 160.1672,-553.1763 153.4115,-543.409 148.1345,-533.6943"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="151.1286,-531.8563 143.5242,-524.479 144.8683,-534.9882 151.1286,-531.8563"/>
+            <text text-anchor="middle" x="174.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> a</text>
+        </g>
+        <!-- tr&#45;&gt;tra -->
+        <g id="edge16" class="edge">
+            <title>tr&#45;&gt;tra</title>
+            <path fill="none" stroke="#add8e6" d="M261.3078,-590.8691C246.265,-582.25 228.1107,-571.3271 212.5223,-560.4626 196.6086,-549.3714 179.8017,-536.0186 165.7777,-524.3687"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="157.8699,-517.7362 168.4235,-520.7146 161.7008,-520.9493 165.5317,-524.1624 165.5317,-524.1624 165.5317,-524.1624 161.7008,-520.9493 162.6399,-527.6103 157.8699,-517.7362 157.8699,-517.7362"/>
+            <text text-anchor="middle" x="218.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> a</text>
+        </g>
+        <!-- tr&#45;&gt;tre -->
+        <g id="edge9" class="edge">
+            <title>tr&#45;&gt;tre</title>
+            <path fill="none" stroke="#d3d3d3" d="M278.1384,-580.3332C274.0811,-568.4595 272.2412,-554.7787 278.5223,-543.6626 286.3122,-529.876 299.8947,-519.6522 313.6068,-512.301"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="315.1568,-515.4391 322.5768,-507.8764 312.0602,-509.1613 315.1568,-515.4391"/>
+            <text text-anchor="middle" x="284.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> e</text>
+        </g>
+        <!-- tr&#45;&gt;tri -->
+        <g id="edge11" class="edge">
+            <title>tr&#45;&gt;tri</title>
+            <path fill="none" stroke="#d3d3d3" d="M318.6123,-586.768C338.5866,-570.9443 364.4656,-550.8354 375.8463,-543.6626 392.0492,-533.4505 410.7284,-523.6151 427.1033,-515.5752"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="428.963,-518.5637 436.4423,-511.0596 425.9158,-512.2617 428.963,-518.5637"/>
+            <text text-anchor="middle" x="379.9322" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> i</text>
+        </g>
+        <!-- tr&#45;&gt;tru -->
+        <g id="edge22" class="edge">
+            <title>tr&#45;&gt;tru</title>
+            <path fill="none" stroke="#add8e6" d="M324.6333,-592.5474C334.7055,-587.9001 345.8831,-582.8634 356.2369,-578.4626 376.4415,-569.8748 382.1813,-569.3927 402.2369,-560.4626 417.4529,-553.6873 420.0476,-549.2551 435.7369,-543.6626 470.2699,-531.3532 481.4995,-537.3829 516.2369,-525.6626 523.7321,-523.1337 531.5258,-520.0047 538.9887,-516.7402"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="548.2173,-512.5679 540.9591,-520.788 543.6613,-514.6277 539.1053,-516.6876 539.1053,-516.6876 539.1053,-516.6876 543.6613,-514.6277 537.2514,-512.5872 548.2173,-512.5679 548.2173,-512.5679"/>
+            <text text-anchor="middle" x="441.4869" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> u</text>
+        </g>
+        <!-- tr2 -->
+        <g id="node23" class="node">
+            <title>tr2</title>
+            <ellipse fill="none" stroke="#0000ff" cx="403.2369" cy="-607.8782" rx="37.9286" ry="29.3315"/>
+            <text text-anchor="middle" x="403.2369" y="-612.0782" font-family="Times,serif" font-size="14.00" fill="#0000ff">Sparse</text>
+            <text text-anchor="middle" x="403.2369" y="-595.2782" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0FE</text>
+        </g>
+        <!-- tr&#45;&gt;tr2 -->
+        <!-- t&#45;&gt;root -->
+        <g id="edge54" class="edge">
+            <title>t&#45;&gt;root</title>
+            <path fill="none" stroke="#ffc0cb" d="M357.0886,-726.2786C358.3802,-731.9426 359.5942,-738.2388 360.2369,-744.0939 361.1247,-752.1817 360.87,-760.8327 360.048,-769.081"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="358.7469,-779.2407 355.5537,-768.7501 359.3821,-774.2812 360.0173,-769.3217 360.0173,-769.3217 360.0173,-769.3217 359.3821,-774.2812 364.4808,-769.8934 358.7469,-779.2407 358.7469,-779.2407"/>
+            <text text-anchor="middle" x="379.9052" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x09B</text>
+        </g>
+        <!-- t&#45;&gt;tr -->
+        <g id="edge3" class="edge">
+            <title>t&#45;&gt;tr</title>
+            <path fill="none" stroke="#d3d3d3" d="M328.2023,-694.2178C320.033,-688.3735 311.5858,-680.8748 306.0763,-672.0939 301.4147,-664.6644 298.305,-655.9034 296.2354,-647.2813"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="299.6361,-646.4371 294.253,-637.3118 292.7705,-647.8024 299.6361,-646.4371"/>
+            <text text-anchor="middle" x="310.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> r</text>
+        </g>
+        <!-- t&#45;&gt;tr -->
+        <g id="edge15" class="edge">
+            <title>t&#45;&gt;tr</title>
+            <path fill="none" stroke="#add8e6" d="M341.7857,-690.6376C333.9669,-677.5782 323.0103,-659.2778 313.3877,-643.2055"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="308.2074,-634.5532 317.2052,-640.8214 310.7759,-638.8431 313.3443,-643.133 313.3443,-643.133 313.3443,-643.133 310.7759,-638.8431 309.4834,-645.4446 308.2074,-634.5532 308.2074,-634.5532"/>
+            <text text-anchor="middle" x="334.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+        </g>
+        <!-- t&#45;&gt;tr2 -->
+        <g id="edge34" class="edge">
+            <title>t&#45;&gt;tr2</title>
+            <path fill="none" stroke="#0000ff" d="M357.4304,-690.0268C360.7316,-679.6045 365.4414,-666.4045 371.0763,-655.2939 373.2668,-650.9747 375.7873,-646.5834 378.4332,-642.3036"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="381.4835,-644.032 383.9745,-633.7342 375.6054,-640.231 381.4835,-644.032"/>
+            <text text-anchor="middle" x="375.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+        </g>
+        <!-- trav&#45;&gt;tree -->
+        <!-- trave -->
+        <g id="node15" class="node">
+            <title>trave</title>
+            <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-284.4" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="202.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B9</text>
+        </g>
+        <!-- trav&#45;&gt;trave -->
+        <g id="edge18" class="edge">
+            <title>trav&#45;&gt;trave</title>
+            <path fill="none" stroke="#add8e6" d="M181.4692,-359.4476C176.9833,-352.6281 172.8815,-344.9707 170.5223,-337.2 168.3531,-330.0554 167.8843,-327.3851 170.5223,-320.4 172.1555,-316.0753 174.5712,-311.9338 177.3533,-308.0992"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="183.8676,-300.2431 180.9485,-310.8134 180.6761,-304.0921 177.4845,-307.941 177.4845,-307.941 177.4845,-307.941 180.6761,-304.0921 174.0205,-305.0686 183.8676,-300.2431 183.8676,-300.2431"/>
+            <text text-anchor="middle" x="176.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+        </g>
+        <!-- trav&#45;&gt;trave -->
+        <g id="edge40" class="edge">
+            <title>trav&#45;&gt;trave</title>
+            <path fill="none" stroke="#0000ff" d="M202.2369,-355.0897C202.2369,-341.6046 202.2369,-325.7123 202.2369,-312.4153"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="205.737,-312.4033 202.2369,-302.4033 198.737,-312.4034 205.737,-312.4033"/>
+            <text text-anchor="middle" x="207.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- trav&#45;&gt;tra2 -->
+        <g id="edge51" class="edge">
+            <title>trav&#45;&gt;tra2</title>
+            <path fill="none" stroke="#ffc0cb" d="M213.0163,-412.9388C215.4002,-419.2228 217.909,-425.8514 220.2369,-432.0313 223.4375,-440.5282 226.8619,-449.6636 230.0924,-458.3025"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="233.6352,-467.7845 225.9197,-459.992 231.8851,-463.1007 230.1351,-458.417 230.1351,-458.417 230.1351,-458.417 231.8851,-463.1007 234.3505,-456.8419 233.6352,-467.7845 233.6352,-467.7845"/>
+            <text text-anchor="middle" x="244.9052" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B8</text>
+        </g>
+        <!-- trave&#45;&gt;trav -->
+        <g id="edge50" class="edge">
+            <title>trave&#45;&gt;trav</title>
+            <path fill="none" stroke="#ffc0cb" d="M208.9024,-302.4722C210.6782,-308.1317 212.3487,-314.4535 213.2369,-320.4 214.4724,-328.6713 214.0766,-337.5213 212.8888,-345.926"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="211.1036,-355.888 208.4382,-345.2509 211.9856,-350.9664 212.8676,-346.0448 212.8676,-346.0448 212.8676,-346.0448 211.9856,-350.9664 217.297,-346.8386 211.1036,-355.888 211.1036,-355.888"/>
+            <text text-anchor="middle" x="232.9052" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B9</text>
+        </g>
+        <!-- traver -->
+        <g id="node16" class="node">
+            <title>traver</title>
+            <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-195.6" rx="38.8671" ry="18"/>
+            <text text-anchor="middle" x="202.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BA</text>
+        </g>
+        <!-- trave&#45;&gt;traver -->
+        <g id="edge19" class="edge">
+            <title>trave&#45;&gt;traver</title>
+            <path fill="none" stroke="#add8e6" d="M185.324,-268.32C180.3995,-262.5605 175.6543,-255.6784 173.0763,-248.4 170.5834,-241.3618 170.5834,-238.6382 173.0763,-231.6 174.4962,-227.5912 176.5735,-223.7026 178.9762,-220.0556"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="185.0164,-212.0415 182.5911,-222.7358 182.007,-216.0344 178.9975,-220.0273 178.9975,-220.0273 178.9975,-220.0273 182.007,-216.0344 175.4039,-217.3188 185.0164,-212.0415 185.0164,-212.0415"/>
+            <text text-anchor="middle" x="177.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+        </g>
+        <!-- trave&#45;&gt;traver -->
+        <g id="edge41" class="edge">
+            <title>trave&#45;&gt;traver</title>
+            <path fill="none" stroke="#0000ff" d="M202.2369,-266.0006C202.2369,-253.8949 202.2369,-237.8076 202.2369,-224.0674"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="205.737,-223.672 202.2369,-213.672 198.737,-223.6721 205.737,-223.672"/>
+            <text text-anchor="middle" x="206.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+        </g>
+        <!-- traver&#45;&gt;trave -->
+        <g id="edge49" class="edge">
+            <title>traver&#45;&gt;trave</title>
+            <path fill="none" stroke="#ffc0cb" d="M207.6937,-213.7509C209.1467,-219.4136 210.5127,-225.7174 211.2369,-231.6 212.1492,-239.0107 212.1492,-240.9893 211.2369,-248.4 210.9201,-250.9736 210.4804,-253.6279 209.9643,-256.2738"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="207.6937,-266.2491 205.5255,-255.4997 208.8035,-261.3738 209.9132,-256.4985 209.9132,-256.4985 209.9132,-256.4985 208.8035,-261.3738 214.301,-257.4973 207.6937,-266.2491 207.6937,-266.2491"/>
+            <text text-anchor="middle" x="232.4585" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BA</text>
+        </g>
+        <!-- travers -->
+        <g id="node17" class="node">
+            <title>travers</title>
+            <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-106.8" rx="38.305" ry="18"/>
+            <text text-anchor="middle" x="203.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BB</text>
+        </g>
+        <!-- traver&#45;&gt;travers -->
+        <g id="edge20" class="edge">
+            <title>traver&#45;&gt;travers</title>
+            <path fill="none" stroke="#add8e6" d="M183.9854,-179.6658C174.6965,-169.5397 166.3137,-155.9116 171.2923,-142.8 172.9385,-138.4645 175.3723,-134.3169 178.1749,-130.4793"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="184.7367,-122.6213 181.7812,-133.1814 181.5319,-126.4592 178.3271,-130.2971 178.3271,-130.2971 178.3271,-130.2971 181.5319,-126.4592 174.873,-127.4128 184.7367,-122.6213 184.7367,-122.6213"/>
+            <text text-anchor="middle" x="176.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> s</text>
+        </g>
+        <!-- traver&#45;&gt;travers -->
+        <g id="edge42" class="edge">
+            <title>traver&#45;&gt;travers</title>
+            <path fill="none" stroke="#0000ff" d="M202.4441,-177.2006C202.5804,-165.0949 202.7616,-149.0076 202.9163,-135.2674"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="206.4205,-134.9108 203.0334,-124.872 199.4209,-134.8319 206.4205,-134.9108"/>
+            <text text-anchor="middle" x="207.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> s</text>
+        </g>
+        <!-- travers&#45;&gt;traver -->
+        <g id="edge48" class="edge">
+            <title>travers&#45;&gt;traver</title>
+            <path fill="none" stroke="#ffc0cb" d="M209.0666,-124.8376C211.6638,-135.0186 213.8193,-147.9908 212.2369,-159.6 211.8717,-162.2793 211.3602,-165.0407 210.7599,-167.7863"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="208.2983,-177.4866 206.3963,-166.6869 209.5282,-172.6402 210.758,-167.7938 210.758,-167.7938 210.758,-167.7938 209.5282,-172.6402 215.1198,-168.9007 208.2983,-177.4866 208.2983,-177.4866"/>
+            <text text-anchor="middle" x="233.0735" y="-147" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BB</text>
+        </g>
+        <!-- traverse -->
+        <g id="node18" class="node">
+            <title>traverse</title>
+            <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-18" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="203.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#0000ff">contentArray[3]</text>
+        </g>
+        <!-- travers&#45;&gt;traverse -->
+        <g id="edge21" class="edge">
+            <title>travers&#45;&gt;traverse</title>
+            <path fill="none" stroke="#add8e6" d="M184.8676,-90.9569C179.5135,-85.2132 174.3474,-78.2805 171.5223,-70.8 168.8843,-63.8149 168.8843,-60.9851 171.5223,-54 172.8668,-50.4399 174.7415,-47.004 176.9144,-43.7612"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="183.203,-35.676 180.6155,-46.3323 180.1332,-39.6227 177.0634,-43.5695 177.0634,-43.5695 177.0634,-43.5695 180.1332,-39.6227 173.5114,-40.8067 183.203,-35.676 183.203,-35.676"/>
+            <text text-anchor="middle" x="177.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+        </g>
+        <!-- travers&#45;&gt;traverse -->
+        <g id="edge43" class="edge">
+            <title>travers&#45;&gt;traverse</title>
+            <path fill="none" stroke="#0000ff" d="M203.2369,-88.4006C203.2369,-76.2949 203.2369,-60.2076 203.2369,-46.4674"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="206.737,-46.072 203.2369,-36.072 199.737,-46.0721 206.737,-46.072"/>
+            <text text-anchor="middle" x="208.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- traverse&#45;&gt;travers -->
+        <g id="edge47" class="edge">
+            <title>traverse&#45;&gt;travers</title>
+            <path fill="none" stroke="#ffc0cb" d="M209.9024,-36.0722C211.6782,-41.7317 213.3487,-48.0535 214.2369,-54 215.3399,-61.3847 215.3399,-63.4153 214.2369,-70.8 213.8344,-73.4945 213.2713,-76.2661 212.6106,-79.0181"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="209.9024,-88.7278 208.2546,-77.8864 211.2458,-83.9116 212.5891,-79.0954 212.5891,-79.0954 212.5891,-79.0954 211.2458,-83.9116 216.9237,-80.3044 209.9024,-88.7278 209.9024,-88.7278"/>
+            <text text-anchor="middle" x="224.2732" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#ff0000"> ~3</text>
+        </g>
+        <!-- tra2&#45;&gt;trac -->
+        <g id="edge38" class="edge">
+            <title>tra2&#45;&gt;trac</title>
+            <path fill="none" stroke="#0000ff" d="M216.023,-475.2499C189.2207,-455.3034 149.1456,-425.4792 122.3699,-405.5525"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="124.2894,-402.6181 114.1775,-399.4556 120.1102,-408.2337 124.2894,-402.6181"/>
+            <text text-anchor="middle" x="183.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+        </g>
+        <!-- tra2&#45;&gt;tre -->
+        <!-- tra2&#45;&gt;trav -->
+        <g id="edge39" class="edge">
+            <title>tra2&#45;&gt;trav</title>
+            <path fill="none" stroke="#0000ff" d="M222.659,-471.5823C217.635,-464.6387 212.8503,-456.7959 209.7369,-448.8313 206.7154,-441.1019 204.8136,-432.4806 203.6368,-424.1275"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="207.1115,-423.7045 202.5344,-414.1494 200.1538,-424.4733 207.1115,-423.7045"/>
+            <text text-anchor="middle" x="215.4869" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> v</text>
+        </g>
+        <!-- tra2&#45;&gt;tr2 -->
+        <g id="edge52" class="edge">
+            <title>tra2&#45;&gt;tr2</title>
+            <path fill="none" stroke="#ffc0cb" d="M266.4769,-520.5821C278.9596,-533.4015 295.2687,-548.8312 311.5777,-560.4626 326.8195,-571.3328 344.8896,-581.2289 360.9398,-589.135"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="370.1198,-593.5523 359.1575,-593.2712 365.6143,-591.3843 361.1088,-589.2162 361.1088,-589.2162 361.1088,-589.2162 365.6143,-591.3843 363.06,-585.1613 370.1198,-593.5523 370.1198,-593.5523"/>
+            <text text-anchor="middle" x="332.0665" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0DE</text>
+        </g>
+        <!-- tru&#45;&gt;truc -->
+        <g id="edge23" class="edge">
+            <title>tru&#45;&gt;truc</title>
+            <path fill="none" stroke="#add8e6" d="M584.906,-467.0117C587.8872,-455.247 592.7225,-442.1094 600.5223,-432.0313 610.5674,-419.0519 625.2451,-408.7011 639.0092,-401.0235"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="647.8644,-396.3635 641.1106,-405.0028 643.4397,-398.692 639.0149,-401.0205 639.0149,-401.0205 639.0149,-401.0205 643.4397,-398.692 636.9193,-397.0383 647.8644,-396.3635 647.8644,-396.3635"/>
+            <text text-anchor="middle" x="605.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> c</text>
+        </g>
+        <!-- tru&#45;&gt;truc -->
+        <g id="edge45" class="edge">
+            <title>tru&#45;&gt;truc</title>
+            <path fill="none" stroke="#0000ff" d="M601.5424,-471.4723C617.5177,-452.8958 639.2774,-427.5931 655.1898,-409.0897"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="657.9656,-411.2298 661.8323,-401.3657 652.6583,-406.6656 657.9656,-411.2298"/>
+            <text text-anchor="middle" x="638.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+        </g>
+        <!-- tru&#45;&gt;tr2 -->
+        <g id="edge58" class="edge">
+            <title>tru&#45;&gt;tr2</title>
+            <path fill="none" stroke="#ffc0cb" d="M550.5144,-514.9925C520.6138,-533.8503 474.3748,-563.0126 441.6955,-583.6229"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="432.8627,-589.1936 438.9205,-580.0528 437.0919,-586.5264 441.321,-583.8591 441.321,-583.8591 441.321,-583.8591 437.0919,-586.5264 443.7216,-587.6653 432.8627,-589.1936 432.8627,-589.1936"/>
+            <text text-anchor="middle" x="521.2902" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x11A</text>
+        </g>
+        <!-- truc&#45;&gt;tru -->
+        <g id="edge57" class="edge">
+            <title>truc&#45;&gt;tru</title>
+            <path fill="none" stroke="#ffc0cb" d="M670.9731,-402.6462C666.3263,-416.3625 658.5213,-435.1464 647.2369,-448.8313 639.6641,-458.015 629.8999,-466.2538 620.1806,-473.1865"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="611.7071,-478.9336 617.4572,-469.5962 615.8452,-476.127 619.9832,-473.3204 619.9832,-473.3204 619.9832,-473.3204 615.8452,-476.127 622.5091,-477.0446 611.7071,-478.9336 611.7071,-478.9336"/>
+            <text text-anchor="middle" x="675.9052" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x11B</text>
+        </g>
+        <!-- truck -->
+        <g id="node22" class="node">
+            <title>truck</title>
+            <ellipse fill="none" stroke="#0000ff" cx="676.2369" cy="-284.4" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="676.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">contentArray[4]</text>
+        </g>
+        <!-- truc&#45;&gt;truck -->
+        <g id="edge24" class="edge">
+            <title>truc&#45;&gt;truck</title>
+            <path fill="none" stroke="#add8e6" d="M661.1247,-367.9416C654.3993,-359.4018 647.222,-348.4489 643.7369,-337.2 641.5272,-330.0678 641.0565,-327.369 643.7369,-320.4 645.1585,-316.7039 647.1547,-313.1507 649.4663,-309.8129"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="655.7148,-302.0002 652.9831,-312.6204 652.5918,-305.9049 649.4689,-309.8097 649.4689,-309.8097 649.4689,-309.8097 652.5918,-305.9049 645.9546,-306.999 655.7148,-302.0002 655.7148,-302.0002"/>
+            <text text-anchor="middle" x="648.4869" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> k</text>
+        </g>
+        <!-- truc&#45;&gt;truck -->
+        <g id="edge46" class="edge">
+            <title>truc&#45;&gt;truck</title>
+            <path fill="none" stroke="#0000ff" d="M676.2369,-366.2345C676.2369,-351.2603 676.2369,-329.8643 676.2369,-312.7055"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="679.737,-312.6645 676.2369,-302.6646 672.737,-312.6646 679.737,-312.6645"/>
+            <text text-anchor="middle" x="681.4869" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> k</text>
+        </g>
+        <!-- truck&#45;&gt;truc -->
+        <g id="edge56" class="edge">
+            <title>truck&#45;&gt;truc</title>
+            <path fill="none" stroke="#ffc0cb" d="M683.5061,-302.4275C685.4432,-308.0852 687.266,-314.4172 688.2369,-320.4 690.1842,-332.3997 688.3971,-345.6086 685.6614,-356.9103"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="682.9623,-366.6639 681.2924,-355.8259 684.2958,-361.845 685.6294,-357.0261 685.6294,-357.0261 685.6294,-357.0261 684.2958,-361.845 689.9664,-358.2263 682.9623,-366.6639 682.9623,-366.6639"/>
+            <text text-anchor="middle" x="695.5232" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#ff0000">~4</text>
+        </g>
+        <!-- tr2&#45;&gt;tre -->
+        <g id="edge35" class="edge">
+            <title>tr2&#45;&gt;tre</title>
+            <path fill="none" stroke="#0000ff" d="M394.3252,-579.1086C390.6132,-567.9144 386.0477,-555.063 381.2369,-543.6626 379.7068,-540.0366 378.0202,-536.3025 376.2719,-532.597"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="379.3778,-530.9808 371.8525,-523.523 373.0845,-534.0459 379.3778,-530.9808"/>
+            <text text-anchor="middle" x="393.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- tr2&#45;&gt;tri -->
+        <g id="edge36" class="edge">
+            <title>tr2&#45;&gt;tri</title>
+            <path fill="none" stroke="#0000ff" d="M428.3953,-585.4737C435.5741,-578.0764 442.8897,-569.4305 448.2369,-560.4626 452.9069,-552.6304 456.6772,-543.633 459.6635,-534.9006"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="463.0435,-535.8196 462.7083,-525.2301 456.3666,-533.7173 463.0435,-535.8196"/>
+            <text text-anchor="middle" x="459.9322" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+        </g>
+        <!-- tr2&#45;&gt;t -->
+        <g id="edge53" class="edge">
+            <title>tr2&#45;&gt;t</title>
+            <path fill="none" stroke="#ffc0cb" d="M394.4099,-636.7827C390.4471,-648.114 385.2917,-661.0259 379.2369,-672.0939 377.2684,-675.6922 374.9717,-679.3287 372.5643,-682.8363"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="366.4757,-691.1951 368.7261,-680.4626 369.4196,-687.1536 372.3635,-683.1121 372.3635,-683.1121 372.3635,-683.1121 369.4196,-687.1536 376.0008,-685.7616 366.4757,-691.1951 366.4757,-691.1951"/>
+            <text text-anchor="middle" x="405.5132" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x07E</text>
+        </g>
+        <!-- tr2&#45;&gt;tra2 -->
+        <g id="edge37" class="edge">
+            <title>tr2&#45;&gt;tra2</title>
+            <path fill="none" stroke="#0000ff" d="M371.5278,-591.76C361.3876,-587.0267 349.9987,-582.1393 339.2369,-578.4626 305.6867,-567.0003 287.0313,-585.0856 261.5223,-560.4626 254.6899,-553.8675 250.4636,-544.8815 247.874,-535.7218"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="251.2577,-534.8148 245.6605,-525.8192 244.4263,-536.3419 251.2577,-534.8148"/>
+            <text text-anchor="middle" x="267.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+        </g>
+        <!-- tr2&#45;&gt;tru -->
+        <g id="edge44" class="edge">
+            <title>tr2&#45;&gt;tru</title>
+            <path fill="none" stroke="#0000ff" d="M440.8715,-604.3719C473.582,-599.5098 520.4295,-587.9399 551.2369,-560.4626 558.9738,-553.5619 564.7664,-544.1924 569.054,-534.745"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="572.3451,-535.9426 572.8562,-525.3601 565.8573,-533.3141 572.3451,-535.9426"/>
+            <text text-anchor="middle" x="568.4869" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> u</text>
+        </g>
+    </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a3.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a3.svg
new file mode 100644
index 000000000000..7ca3661d9181
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a3.svg
@@ -0,0 +1,659 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+  - Licensed to the Apache Software Foundation (ASF) under one
+  - or more contributor license agreements.  See the NOTICE file
+  - distributed with this work for additional information
+  - regarding copyright ownership.  The ASF licenses this file
+  - to you under the Apache License, Version 2.0 (the
+  - "License"); you may not use this file except in compliance
+  - with the License.  You may obtain a copy of the License at
+  -
+  -     http://www.apache.org/licenses/LICENSE-2.0
+  -
+  - Unless required by applicable law or agreed to in writing, software
+  - distributed under the License is distributed on an "AS IS" BASIS,
+  - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  - See the License for the specific language governing permissions and
+  - limitations under the License.
+  -->
+
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+        "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# embedded DOT (plantuml) works in IDEA preview
+# but not on GitHub
+```plantuml
+digraph G {
+    start [label="start/end"]
+
+    tractor [label = "contentArray[0]"]
+    tracto [label = "0x01B"]
+    tract [label = "0x01A"]
+    trac [label = "0x019"]
+    tra [label = "Multi\n0x018"; color = "lightgrey"; fontcolor = lightgray]
+
+    tree [label = "contentArray[1]"]
+    tre [label = "Multi\n0x03B"]
+    trie [label = "contentArray[2]"]
+    tri [label = "Multi\n0x05B"]
+
+    tr [label = "Sparse\n0x07E"; color = "lightgrey"; fontcolor = lightgray]
+
+    t [label = "0x09B"; color = "lightgrey"; fontcolor = lightgray]
+    root [label = "Multi\n0x9A"; color = "lightgrey"; fontcolor = lightgray]
+
+    root -> t [label = " t"; color = "lightgrey"; fontcolor = lightgray]
+    t -> tr [label = " r"; color = "lightgrey"; fontcolor = lightgray]
+    trac -> tract [label = " t"]
+    tract -> tracto [label = " o"]
+    tracto -> tractor [label = " r"]
+
+    tr -> tra [label = " a"; color = "lightgrey"; fontcolor = lightgray]
+    tra -> trac [label = " c"; color = "lightgrey"; fontcolor = lightgray]
+
+    tr -> tre [label = " e"; color = "lightgrey"; fontcolor = lightgray]
+    tre -> tree [label = " e"]
+
+    tr -> tri [label = " i"; color = "lightgrey"; fontcolor = lightgray]
+    tri -> trie [label = " e"]
+
+    subgraph path {
+        edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+        node [color = "blue"; fontcolor="blue"]
+
+        start -> root
+
+        root -> t [label = " t"]
+        t -> tr [label = " r"]
+        tr -> tra [label = " a"]
+        tra -> trav [label = " v"]
+        trav -> trave [label = " e"]
+        trave -> traver [label = " r"]
+        traver -> travers [label = " s"]
+        travers -> traverse [label = " e"]
+
+        tra2 [label = "Sparse\n0x0DE"]
+        trav [label = "Multi\n0x0B8"]
+        trave [label = "0x0B9"]
+        traver [label = "0x0BA"]
+        travers [label = "0x0BB"]
+        traverse [label = "contentArray[3]"]
+
+        tr -> tru [label = " u"]
+        tru -> truc [label = " c"]
+        truc -> truck [label = " k"]
+
+        root2 [label = "Multi\n0x13A"]
+        t2 [label = "0x13B"]
+        tr2 [label = "Sparse\n0x0FE"]
+        tru [label = "Multi\n0x11A"]
+        truc [label = "0x11B"]
+        truck [label = "contentArray[4]"]
+    }
+
+    {rank=same root -> root2 [style=invis]}
+    {rank=same t -> t2 [style=invis]}
+    {rank=same tr -> tr2 [style=invis]}
+    {rank=same tra -> tra2 -> tre -> tri -> tru [style=invis]}
+    {rank=same trac -> trav -> tree -> trie -> truc [style=invis]}
+
+    {
+        edge [color = "blue"]
+        root2 -> t2 [label = " t"]
+        t2 -> tr2 [label = " r"]
+        tr2 -> tre [label = " e"]
+        tr2 -> tri [label = " i"]
+
+        tr2 -> tra2 [label = " a"]
+        tra2 -> trac [label = " c"]
+        tra2 -> trav [label = " v"]
+        trav -> trave [label = " e"]
+        trave -> traver [label = " r"]
+        traver -> travers [label = " s"]
+        travers -> traverse [label = " e"]
+        tr2 -> tru [label = " u"]
+        tru -> truc [label = " c"]
+        truc -> truck [label = " k"]
+    }
+
+    subgraph back {
+        edge [color = "pink"; fontcolor="red"; arrowhead="vee"; constrain="false"]
+
+        traverse -> travers [label = " ~3"]
+        travers -> traver [label = "0x0BB"]
+        traver -> trave [label = "0x0BA"]
+        trave -> trav [label = "0x0B9"]
+        trav -> tra2 [label = "0x0B8"]
+        tra2 -> tr2 [label = "0x0DE"]
+        tr2 -> t2 [label = "0x0FE"]
+        t2 -> root2 [label = "0x13B"]
+        root2 -> start [label = "0x13A"]
+
+        truck -> truc [label = "~4"]
+        truc -> tru [label = "0x11B"]
+        tru -> tr2 [label = "0x11A"]
+    }
+}
+```
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="757pt" height="935pt"
+     viewBox="0.00 0.00 757.47 934.53" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 930.5251)">
+        <title>G</title>
+        <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-930.5251 753.4738,-930.5251 753.4738,4 -4,4"/>
+        <!-- start -->
+        <g id="node1" class="node">
+            <title>start</title>
+            <ellipse fill="none" stroke="#000000" cx="318.2369" cy="-908.5251" rx="44.0775" ry="18"/>
+            <text text-anchor="middle" x="318.2369" y="-904.3251" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+        </g>
+        <!-- root -->
+        <g id="node13" class="node">
+            <title>root</title>
+            <ellipse fill="none" stroke="#d3d3d3" cx="265.2369" cy="-808.3095" rx="33.1337" ry="29.3315"/>
+            <text text-anchor="middle" x="265.2369" y="-812.5095" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">Multi</text>
+            <text text-anchor="middle" x="265.2369" y="-795.7095" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">0x9A</text>
+        </g>
+        <!-- start&#45;&gt;root -->
+        <g id="edge12" class="edge">
+            <title>start&#45;&gt;root</title>
+            <path fill="none" stroke="#add8e6" d="M308.7616,-890.6086C301.9417,-877.7133 292.5132,-859.8852 284.1748,-844.1185"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="279.4192,-835.1263 288.0723,-841.8625 281.7568,-839.5463 284.0944,-843.9663 284.0944,-843.9663 284.0944,-843.9663 281.7568,-839.5463 280.1164,-846.0701 279.4192,-835.1263 279.4192,-835.1263"/>
+        </g>
+        <!-- tractor -->
+        <g id="node2" class="node">
+            <title>tractor</title>
+            <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+        </g>
+        <!-- tracto -->
+        <g id="node3" class="node">
+            <title>tracto</title>
+            <ellipse fill="none" stroke="#000000" cx="74.2369" cy="-195.6" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="74.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+        </g>
+        <!-- tracto&#45;&gt;tractor -->
+        <g id="edge5" class="edge">
+            <title>tracto&#45;&gt;tractor</title>
+            <path fill="none" stroke="#000000" d="M74.0297,-177.2006C73.8934,-165.0949 73.7122,-149.0076 73.5575,-135.2674"/>
+            <polygon fill="#000000" stroke="#000000" points="77.0529,-134.8319 73.4404,-124.872 70.0533,-134.9108 77.0529,-134.8319"/>
+            <text text-anchor="middle" x="78.3172" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+        </g>
+        <!-- tract -->
+        <g id="node4" class="node">
+            <title>tract</title>
+            <ellipse fill="none" stroke="#000000" cx="78.2369" cy="-284.4" rx="37.1443" ry="18"/>
+            <text text-anchor="middle" x="78.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+        </g>
+        <!-- tract&#45;&gt;tracto -->
+        <g id="edge4" class="edge">
+            <title>tract&#45;&gt;tracto</title>
+            <path fill="none" stroke="#000000" d="M77.4081,-266.0006C76.8628,-253.8949 76.1381,-237.8076 75.5192,-224.0674"/>
+            <polygon fill="#000000" stroke="#000000" points="78.9975,-223.5044 75.0509,-213.672 72.0046,-223.8194 78.9975,-223.5044"/>
+            <text text-anchor="middle" x="82.4869" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+        </g>
+        <!-- trac -->
+        <g id="node5" class="node">
+            <title>trac</title>
+            <ellipse fill="none" stroke="#000000" cx="94.2369" cy="-384.6156" rx="35.3587" ry="18"/>
+            <text text-anchor="middle" x="94.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+        </g>
+        <!-- trac&#45;&gt;tract -->
+        <g id="edge3" class="edge">
+            <title>trac&#45;&gt;tract</title>
+            <path fill="none" stroke="#000000" d="M91.3022,-366.2345C88.9115,-351.2603 85.4955,-329.8643 82.756,-312.7055"/>
+            <polygon fill="#000000" stroke="#000000" points="86.1858,-311.9877 81.1529,-302.6646 79.2734,-313.0913 86.1858,-311.9877"/>
+            <text text-anchor="middle" x="89.9322" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+        </g>
+        <!-- trav -->
+        <g id="node14" class="node">
+            <title>trav</title>
+            <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+            <text text-anchor="middle" x="202.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#0000ff">Multi</text>
+            <text text-anchor="middle" x="202.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B8</text>
+        </g>
+        <!-- trac&#45;&gt;trav -->
+        <!-- tra -->
+        <g id="node6" class="node">
+            <title>tra</title>
+            <ellipse fill="none" stroke="#d3d3d3" cx="133.2369" cy="-496.2469" rx="36.125" ry="29.3315"/>
+            <text text-anchor="middle" x="133.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">Multi</text>
+            <text text-anchor="middle" x="133.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">0x018</text>
+        </g>
+        <!-- tra&#45;&gt;trac -->
+        <g id="edge7" class="edge">
+            <title>tra&#45;&gt;trac</title>
+            <path fill="none" stroke="#d3d3d3" d="M114.5188,-470.6739C110.3016,-463.8622 106.2666,-456.3098 103.5223,-448.8313 99.3758,-437.5318 97.0731,-424.4776 95.7973,-413.1021"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="99.2538,-412.4638 94.8622,-402.822 92.2826,-413.098 99.2538,-412.4638"/>
+            <text text-anchor="middle" x="109.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> c</text>
+        </g>
+        <!-- tra&#45;&gt;trav -->
+        <g id="edge16" class="edge">
+            <title>tra&#45;&gt;trav</title>
+            <path fill="none" stroke="#add8e6" d="M130.7527,-466.6197C131.0399,-455.1889 132.9734,-442.3975 138.7369,-432.0313 144.2651,-422.0881 152.8469,-413.6214 161.933,-406.7148"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="170.346,-400.8129 164.7439,-410.2398 166.2527,-403.6844 162.1595,-406.5559 162.1595,-406.5559 162.1595,-406.5559 166.2527,-403.6844 159.5751,-402.872 170.346,-400.8129 170.346,-400.8129"/>
+            <text text-anchor="middle" x="144.4869" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> v</text>
+        </g>
+        <!-- tra2 -->
+        <g id="node19" class="node">
+            <title>tra2</title>
+            <ellipse fill="none" stroke="#0000ff" cx="244.2369" cy="-496.2469" rx="39.2145" ry="29.3315"/>
+            <text text-anchor="middle" x="244.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#0000ff">Sparse</text>
+            <text text-anchor="middle" x="244.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0DE</text>
+        </g>
+        <!-- tra&#45;&gt;tra2 -->
+        <!-- tree -->
+        <g id="node7" class="node">
+            <title>tree</title>
+            <ellipse fill="none" stroke="#000000" cx="348.2369" cy="-384.6156" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="348.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+        </g>
+        <!-- trie -->
+        <g id="node9" class="node">
+            <title>trie</title>
+            <ellipse fill="none" stroke="#000000" cx="530.2369" cy="-384.6156" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="530.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+        </g>
+        <!-- tree&#45;&gt;trie -->
+        <!-- tre -->
+        <g id="node8" class="node">
+            <title>tre</title>
+            <ellipse fill="none" stroke="#000000" cx="357.2369" cy="-496.2469" rx="37.9306" ry="29.3315"/>
+            <text text-anchor="middle" x="357.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Multi</text>
+            <text text-anchor="middle" x="357.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+        </g>
+        <!-- tre&#45;&gt;tree -->
+        <g id="edge9" class="edge">
+            <title>tre&#45;&gt;tree</title>
+            <path fill="none" stroke="#000000" d="M354.8487,-466.6249C353.5073,-449.9873 351.8422,-429.334 350.5186,-412.9163"/>
+            <polygon fill="#000000" stroke="#000000" points="353.9973,-412.5103 349.7049,-402.8239 347.0199,-413.0729 353.9973,-412.5103"/>
+            <text text-anchor="middle" x="358.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- tri -->
+        <g id="node10" class="node">
+            <title>tri</title>
+            <ellipse fill="none" stroke="#000000" cx="469.2369" cy="-496.2469" rx="37.9306" ry="29.3315"/>
+            <text text-anchor="middle" x="469.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Multi</text>
+            <text text-anchor="middle" x="469.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+        </g>
+        <!-- tre&#45;&gt;tri -->
+        <!-- truc -->
+        <g id="node21" class="node">
+            <title>truc</title>
+            <ellipse fill="none" stroke="#0000ff" cx="676.2369" cy="-384.6156" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="676.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x11B</text>
+        </g>
+        <!-- trie&#45;&gt;truc -->
+        <!-- tri&#45;&gt;trie -->
+        <g id="edge11" class="edge">
+            <title>tri&#45;&gt;trie</title>
+            <path fill="none" stroke="#000000" d="M484.1592,-468.9388C493.6824,-451.511 505.9755,-429.0146 515.4687,-411.6418"/>
+            <polygon fill="#000000" stroke="#000000" points="518.6954,-413.0358 520.4193,-402.5821 512.5526,-409.6791 518.6954,-413.0358"/>
+            <text text-anchor="middle" x="508.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- tru -->
+        <g id="node20" class="node">
+            <title>tru</title>
+            <ellipse fill="none" stroke="#0000ff" cx="580.2369" cy="-496.2469" rx="38.0198" ry="29.3315"/>
+            <text text-anchor="middle" x="580.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#0000ff">Multi</text>
+            <text text-anchor="middle" x="580.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x11A</text>
+        </g>
+        <!-- tri&#45;&gt;tru -->
+        <!-- tr -->
+        <g id="node11" class="node">
+            <title>tr</title>
+            <ellipse fill="none" stroke="#d3d3d3" cx="285.2369" cy="-607.8782" rx="37.9027" ry="29.3315"/>
+            <text text-anchor="middle" x="285.2369" y="-612.0782" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">Sparse</text>
+            <text text-anchor="middle" x="285.2369" y="-595.2782" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">0x07E</text>
+        </g>
+        <!-- tr&#45;&gt;tra -->
+        <g id="edge6" class="edge">
+            <title>tr&#45;&gt;tra</title>
+            <path fill="none" stroke="#d3d3d3" d="M248.8847,-599.957C223.9257,-593.0009 191.3164,-580.7434 168.5223,-560.4626 160.3121,-553.1576 153.6248,-543.47 148.3676,-533.8397"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="151.389,-532.058 143.7623,-524.7038 145.1383,-535.2089 151.389,-532.058"/>
+            <text text-anchor="middle" x="174.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> a</text>
+        </g>
+        <!-- tr&#45;&gt;tra -->
+        <g id="edge15" class="edge">
+            <title>tr&#45;&gt;tra</title>
+            <path fill="none" stroke="#add8e6" d="M255.4057,-589.4049C242.0712,-580.8992 226.2901,-570.4861 212.5223,-560.4626 196.9603,-549.1328 180.3233,-535.8503 166.3281,-524.3146"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="158.4213,-517.7524 168.9903,-520.6762 162.2688,-520.9457 166.1164,-524.1389 166.1164,-524.1389 166.1164,-524.1389 162.2688,-520.9457 163.2424,-527.6017 158.4213,-517.7524 158.4213,-517.7524"/>
+            <text text-anchor="middle" x="218.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> a</text>
+        </g>
+        <!-- tr&#45;&gt;tre -->
+        <g id="edge8" class="edge">
+            <title>tr&#45;&gt;tre</title>
+            <path fill="none" stroke="#d3d3d3" d="M275.192,-579.1963C272.8368,-567.5207 272.5117,-554.3001 278.5223,-543.6626 286.3122,-529.876 299.8947,-519.6522 313.6068,-512.301"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="315.1568,-515.4391 322.5768,-507.8764 312.0602,-509.1613 315.1568,-515.4391"/>
+            <text text-anchor="middle" x="284.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> e</text>
+        </g>
+        <!-- tr&#45;&gt;tri -->
+        <g id="edge10" class="edge">
+            <title>tr&#45;&gt;tri</title>
+            <path fill="none" stroke="#d3d3d3" d="M312.9664,-587.9168C337.1705,-570.5446 369.409,-547.5491 375.8463,-543.6626 392.2423,-533.7635 410.9556,-523.9835 427.304,-515.9004"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="429.1719,-518.8833 436.6209,-511.3492 426.0994,-512.5936 429.1719,-518.8833"/>
+            <text text-anchor="middle" x="378.9322" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> i</text>
+        </g>
+        <!-- tr&#45;&gt;tru -->
+        <g id="edge21" class="edge">
+            <title>tr&#45;&gt;tru</title>
+            <path fill="none" stroke="#add8e6" d="M318.7909,-593.8777C330.6131,-588.971 344.003,-583.4434 356.2369,-578.4626 376.1874,-570.34 381.5406,-569.1837 401.2369,-560.4626 416.8313,-553.5577 419.6391,-549.2949 435.7369,-543.6626 470.3412,-531.5551 481.4995,-537.3829 516.2369,-525.6626 523.7321,-523.1337 531.5258,-520.0047 538.9887,-516.7402"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="548.2173,-512.5679 540.9591,-520.788 543.6613,-514.6277 539.1053,-516.6876 539.1053,-516.6876 539.1053,-516.6876 543.6613,-514.6277 537.2514,-512.5872 548.2173,-512.5679 548.2173,-512.5679"/>
+            <text text-anchor="middle" x="440.4869" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> u</text>
+        </g>
+        <!-- tr2 -->
+        <g id="node25" class="node">
+            <title>tr2</title>
+            <ellipse fill="none" stroke="#0000ff" cx="403.2369" cy="-607.8782" rx="37.9286" ry="29.3315"/>
+            <text text-anchor="middle" x="403.2369" y="-612.0782" font-family="Times,serif" font-size="14.00" fill="#0000ff">Sparse</text>
+            <text text-anchor="middle" x="403.2369" y="-595.2782" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0FE</text>
+        </g>
+        <!-- tr&#45;&gt;tr2 -->
+        <!-- t -->
+        <g id="node12" class="node">
+            <title>t</title>
+            <ellipse fill="none" stroke="#d3d3d3" cx="271.2369" cy="-708.0939" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="271.2369" y="-703.8939" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">0x09B</text>
+        </g>
+        <!-- t&#45;&gt;tr -->
+        <g id="edge2" class="edge">
+            <title>t&#45;&gt;tr</title>
+            <path fill="none" stroke="#d3d3d3" d="M270.39,-689.92C270.1973,-679.6962 270.4264,-666.7269 272.0763,-655.2939 272.4938,-652.4005 273.0349,-649.4334 273.659,-646.4641"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="277.1286,-647.0025 276.0419,-636.4636 270.3192,-645.3799 277.1286,-647.0025"/>
+            <text text-anchor="middle" x="276.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> r</text>
+        </g>
+        <!-- t&#45;&gt;tr -->
+        <g id="edge14" class="edge">
+            <title>t&#45;&gt;tr</title>
+            <path fill="none" stroke="#add8e6" d="M275.7537,-689.8645C277.0318,-684.1986 278.3222,-677.9127 279.2369,-672.0939 280.4839,-664.1608 281.5031,-655.6328 282.3243,-647.4657"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="283.2608,-637.3857 286.8163,-647.7591 282.7982,-642.3642 282.3356,-647.3428 282.3356,-647.3428 282.3356,-647.3428 282.7982,-642.3642 277.8549,-646.9265 283.2608,-637.3857 283.2608,-637.3857"/>
+            <text text-anchor="middle" x="286.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+        </g>
+        <!-- t2 -->
+        <g id="node24" class="node">
+            <title>t2</title>
+            <ellipse fill="none" stroke="#0000ff" cx="380.2369" cy="-708.0939" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="380.2369" y="-703.8939" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x13B</text>
+        </g>
+        <!-- t&#45;&gt;t2 -->
+        <!-- root&#45;&gt;t -->
+        <g id="edge1" class="edge">
+            <title>root&#45;&gt;t</title>
+            <path fill="none" stroke="#d3d3d3" d="M259.8774,-779.1183C258.6167,-768.1018 258.0876,-755.4822 259.8463,-744.0939 260.2634,-741.3931 260.8467,-738.6173 261.531,-735.8625"/>
+            <polygon fill="#d3d3d3" stroke="#d3d3d3" points="264.9241,-736.7275 264.3355,-726.149 258.1988,-734.7857 264.9241,-736.7275"/>
+            <text text-anchor="middle" x="263.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> t</text>
+        </g>
+        <!-- root&#45;&gt;t -->
+        <g id="edge13" class="edge">
+            <title>root&#45;&gt;t</title>
+            <path fill="none" stroke="#add8e6" d="M267.0046,-778.7835C267.812,-765.2984 268.7635,-749.4062 269.5596,-736.1092"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="270.159,-726.0972 274.0532,-736.3483 269.8601,-731.0882 269.5613,-736.0793 269.5613,-736.0793 269.5613,-736.0793 269.8601,-731.0882 265.0693,-735.8103 270.159,-726.0972 270.159,-726.0972"/>
+            <text text-anchor="middle" x="272.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+        </g>
+        <!-- root2 -->
+        <g id="node23" class="node">
+            <title>root2</title>
+            <ellipse fill="none" stroke="#0000ff" cx="372.2369" cy="-808.3095" rx="38.0198" ry="29.3315"/>
+            <text text-anchor="middle" x="372.2369" y="-812.5095" font-family="Times,serif" font-size="14.00" fill="#0000ff">Multi</text>
+            <text text-anchor="middle" x="372.2369" y="-795.7095" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x13A</text>
+        </g>
+        <!-- root&#45;&gt;root2 -->
+        <!-- trav&#45;&gt;tree -->
+        <!-- trave -->
+        <g id="node15" class="node">
+            <title>trave</title>
+            <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-284.4" rx="36.5824" ry="18"/>
+            <text text-anchor="middle" x="202.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B9</text>
+        </g>
+        <!-- trav&#45;&gt;trave -->
+        <g id="edge17" class="edge">
+            <title>trav&#45;&gt;trave</title>
+            <path fill="none" stroke="#add8e6" d="M181.4692,-359.4476C176.9833,-352.6281 172.8815,-344.9707 170.5223,-337.2 168.3531,-330.0554 167.8843,-327.3851 170.5223,-320.4 172.1555,-316.0753 174.5712,-311.9338 177.3533,-308.0992"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="183.8676,-300.2431 180.9485,-310.8134 180.6761,-304.0921 177.4845,-307.941 177.4845,-307.941 177.4845,-307.941 180.6761,-304.0921 174.0205,-305.0686 183.8676,-300.2431 183.8676,-300.2431"/>
+            <text text-anchor="middle" x="176.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+        </g>
+        <!-- trav&#45;&gt;trave -->
+        <g id="edge42" class="edge">
+            <title>trav&#45;&gt;trave</title>
+            <path fill="none" stroke="#0000ff" d="M202.2369,-355.0897C202.2369,-341.6046 202.2369,-325.7123 202.2369,-312.4153"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="205.737,-312.4033 202.2369,-302.4033 198.737,-312.4034 205.737,-312.4033"/>
+            <text text-anchor="middle" x="207.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- trav&#45;&gt;tra2 -->
+        <g id="edge53" class="edge">
+            <title>trav&#45;&gt;tra2</title>
+            <path fill="none" stroke="#ffc0cb" d="M213.0163,-412.9388C215.4002,-419.2228 217.909,-425.8514 220.2369,-432.0313 223.4375,-440.5282 226.8619,-449.6636 230.0924,-458.3025"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="233.6352,-467.7845 225.9197,-459.992 231.8851,-463.1007 230.1351,-458.417 230.1351,-458.417 230.1351,-458.417 231.8851,-463.1007 234.3505,-456.8419 233.6352,-467.7845 233.6352,-467.7845"/>
+            <text text-anchor="middle" x="244.9052" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B8</text>
+        </g>
+        <!-- trave&#45;&gt;trav -->
+        <g id="edge52" class="edge">
+            <title>trave&#45;&gt;trav</title>
+            <path fill="none" stroke="#ffc0cb" d="M208.9024,-302.4722C210.6782,-308.1317 212.3487,-314.4535 213.2369,-320.4 214.4724,-328.6713 214.0766,-337.5213 212.8888,-345.926"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="211.1036,-355.888 208.4382,-345.2509 211.9856,-350.9664 212.8676,-346.0448 212.8676,-346.0448 212.8676,-346.0448 211.9856,-350.9664 217.297,-346.8386 211.1036,-355.888 211.1036,-355.888"/>
+            <text text-anchor="middle" x="232.9052" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B9</text>
+        </g>
+        <!-- traver -->
+        <g id="node16" class="node">
+            <title>traver</title>
+            <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-195.6" rx="38.8671" ry="18"/>
+            <text text-anchor="middle" x="202.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BA</text>
+        </g>
+        <!-- trave&#45;&gt;traver -->
+        <g id="edge18" class="edge">
+            <title>trave&#45;&gt;traver</title>
+            <path fill="none" stroke="#add8e6" d="M185.324,-268.32C180.3995,-262.5605 175.6543,-255.6784 173.0763,-248.4 170.5834,-241.3618 170.5834,-238.6382 173.0763,-231.6 174.4962,-227.5912 176.5735,-223.7026 178.9762,-220.0556"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="185.0164,-212.0415 182.5911,-222.7358 182.007,-216.0344 178.9975,-220.0273 178.9975,-220.0273 178.9975,-220.0273 182.007,-216.0344 175.4039,-217.3188 185.0164,-212.0415 185.0164,-212.0415"/>
+            <text text-anchor="middle" x="177.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+        </g>
+        <!-- trave&#45;&gt;traver -->
+        <g id="edge43" class="edge">
+            <title>trave&#45;&gt;traver</title>
+            <path fill="none" stroke="#0000ff" d="M202.2369,-266.0006C202.2369,-253.8949 202.2369,-237.8076 202.2369,-224.0674"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="205.737,-223.672 202.2369,-213.672 198.737,-223.6721 205.737,-223.672"/>
+            <text text-anchor="middle" x="206.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+        </g>
+        <!-- traver&#45;&gt;trave -->
+        <g id="edge51" class="edge">
+            <title>traver&#45;&gt;trave</title>
+            <path fill="none" stroke="#ffc0cb" d="M207.6937,-213.7509C209.1467,-219.4136 210.5127,-225.7174 211.2369,-231.6 212.1492,-239.0107 212.1492,-240.9893 211.2369,-248.4 210.9201,-250.9736 210.4804,-253.6279 209.9643,-256.2738"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="207.6937,-266.2491 205.5255,-255.4997 208.8035,-261.3738 209.9132,-256.4985 209.9132,-256.4985 209.9132,-256.4985 208.8035,-261.3738 214.301,-257.4973 207.6937,-266.2491 207.6937,-266.2491"/>
+            <text text-anchor="middle" x="232.4585" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BA</text>
+        </g>
+        <!-- travers -->
+        <g id="node17" class="node">
+            <title>travers</title>
+            <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-106.8" rx="38.305" ry="18"/>
+            <text text-anchor="middle" x="203.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BB</text>
+        </g>
+        <!-- traver&#45;&gt;travers -->
+        <g id="edge19" class="edge">
+            <title>traver&#45;&gt;travers</title>
+            <path fill="none" stroke="#add8e6" d="M183.9854,-179.6658C174.6965,-169.5397 166.3137,-155.9116 171.2923,-142.8 172.9385,-138.4645 175.3723,-134.3169 178.1749,-130.4793"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="184.7367,-122.6213 181.7812,-133.1814 181.5319,-126.4592 178.3271,-130.2971 178.3271,-130.2971 178.3271,-130.2971 181.5319,-126.4592 174.873,-127.4128 184.7367,-122.6213 184.7367,-122.6213"/>
+            <text text-anchor="middle" x="176.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> s</text>
+        </g>
+        <!-- traver&#45;&gt;travers -->
+        <g id="edge44" class="edge">
+            <title>traver&#45;&gt;travers</title>
+            <path fill="none" stroke="#0000ff" d="M202.4441,-177.2006C202.5804,-165.0949 202.7616,-149.0076 202.9163,-135.2674"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="206.4205,-134.9108 203.0334,-124.872 199.4209,-134.8319 206.4205,-134.9108"/>
+            <text text-anchor="middle" x="207.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> s</text>
+        </g>
+        <!-- travers&#45;&gt;traver -->
+        <g id="edge50" class="edge">
+            <title>travers&#45;&gt;traver</title>
+            <path fill="none" stroke="#ffc0cb" d="M209.0666,-124.8376C211.6638,-135.0186 213.8193,-147.9908 212.2369,-159.6 211.8717,-162.2793 211.3602,-165.0407 210.7599,-167.7863"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="208.2983,-177.4866 206.3963,-166.6869 209.5282,-172.6402 210.758,-167.7938 210.758,-167.7938 210.758,-167.7938 209.5282,-172.6402 215.1198,-168.9007 208.2983,-177.4866 208.2983,-177.4866"/>
+            <text text-anchor="middle" x="233.0735" y="-147" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BB</text>
+        </g>
+        <!-- traverse -->
+        <g id="node18" class="node">
+            <title>traverse</title>
+            <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-18" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="203.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#0000ff">contentArray[3]</text>
+        </g>
+        <!-- travers&#45;&gt;traverse -->
+        <g id="edge20" class="edge">
+            <title>travers&#45;&gt;traverse</title>
+            <path fill="none" stroke="#add8e6" d="M184.8676,-90.9569C179.5135,-85.2132 174.3474,-78.2805 171.5223,-70.8 168.8843,-63.8149 168.8843,-60.9851 171.5223,-54 172.8668,-50.4399 174.7415,-47.004 176.9144,-43.7612"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="183.203,-35.676 180.6155,-46.3323 180.1332,-39.6227 177.0634,-43.5695 177.0634,-43.5695 177.0634,-43.5695 180.1332,-39.6227 173.5114,-40.8067 183.203,-35.676 183.203,-35.676"/>
+            <text text-anchor="middle" x="177.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+        </g>
+        <!-- travers&#45;&gt;traverse -->
+        <g id="edge45" class="edge">
+            <title>travers&#45;&gt;traverse</title>
+            <path fill="none" stroke="#0000ff" d="M203.2369,-88.4006C203.2369,-76.2949 203.2369,-60.2076 203.2369,-46.4674"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="206.737,-46.072 203.2369,-36.072 199.737,-46.0721 206.737,-46.072"/>
+            <text text-anchor="middle" x="208.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- traverse&#45;&gt;travers -->
+        <g id="edge49" class="edge">
+            <title>traverse&#45;&gt;travers</title>
+            <path fill="none" stroke="#ffc0cb" d="M209.9024,-36.0722C211.6782,-41.7317 213.3487,-48.0535 214.2369,-54 215.3399,-61.3847 215.3399,-63.4153 214.2369,-70.8 213.8344,-73.4945 213.2713,-76.2661 212.6106,-79.0181"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="209.9024,-88.7278 208.2546,-77.8864 211.2458,-83.9116 212.5891,-79.0954 212.5891,-79.0954 212.5891,-79.0954 211.2458,-83.9116 216.9237,-80.3044 209.9024,-88.7278 209.9024,-88.7278"/>
+            <text text-anchor="middle" x="224.2732" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#ff0000"> ~3</text>
+        </g>
+        <!-- tra2&#45;&gt;trac -->
+        <g id="edge40" class="edge">
+            <title>tra2&#45;&gt;trac</title>
+            <path fill="none" stroke="#0000ff" d="M216.023,-475.2499C189.2207,-455.3034 149.1456,-425.4792 122.3699,-405.5525"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="124.2894,-402.6181 114.1775,-399.4556 120.1102,-408.2337 124.2894,-402.6181"/>
+            <text text-anchor="middle" x="183.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+        </g>
+        <!-- tra2&#45;&gt;tre -->
+        <!-- tra2&#45;&gt;trav -->
+        <g id="edge41" class="edge">
+            <title>tra2&#45;&gt;trav</title>
+            <path fill="none" stroke="#0000ff" d="M222.659,-471.5823C217.635,-464.6387 212.8503,-456.7959 209.7369,-448.8313 206.7154,-441.1019 204.8136,-432.4806 203.6368,-424.1275"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="207.1115,-423.7045 202.5344,-414.1494 200.1538,-424.4733 207.1115,-423.7045"/>
+            <text text-anchor="middle" x="215.4869" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> v</text>
+        </g>
+        <!-- tra2&#45;&gt;tr2 -->
+        <g id="edge54" class="edge">
+            <title>tra2&#45;&gt;tr2</title>
+            <path fill="none" stroke="#ffc0cb" d="M266.4769,-520.5821C278.9596,-533.4015 295.2687,-548.8312 311.5777,-560.4626 326.8195,-571.3328 344.8896,-581.2289 360.9398,-589.135"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="370.1198,-593.5523 359.1575,-593.2712 365.6143,-591.3843 361.1088,-589.2162 361.1088,-589.2162 361.1088,-589.2162 365.6143,-591.3843 363.06,-585.1613 370.1198,-593.5523 370.1198,-593.5523"/>
+            <text text-anchor="middle" x="331.0665" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0DE</text>
+        </g>
+        <!-- tru&#45;&gt;truc -->
+        <g id="edge22" class="edge">
+            <title>tru&#45;&gt;truc</title>
+            <path fill="none" stroke="#add8e6" d="M584.906,-467.0117C587.8872,-455.247 592.7225,-442.1094 600.5223,-432.0313 610.5674,-419.0519 625.2451,-408.7011 639.0092,-401.0235"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="647.8644,-396.3635 641.1106,-405.0028 643.4397,-398.692 639.0149,-401.0205 639.0149,-401.0205 639.0149,-401.0205 643.4397,-398.692 636.9193,-397.0383 647.8644,-396.3635 647.8644,-396.3635"/>
+            <text text-anchor="middle" x="605.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> c</text>
+        </g>
+        <!-- tru&#45;&gt;truc -->
+        <g id="edge47" class="edge">
+            <title>tru&#45;&gt;truc</title>
+            <path fill="none" stroke="#0000ff" d="M601.5424,-471.4723C617.5177,-452.8958 639.2774,-427.5931 655.1898,-409.0897"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="657.9656,-411.2298 661.8323,-401.3657 652.6583,-406.6656 657.9656,-411.2298"/>
+            <text text-anchor="middle" x="638.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+        </g>
+        <!-- tru&#45;&gt;tr2 -->
+        <g id="edge60" class="edge">
+            <title>tru&#45;&gt;tr2</title>
+            <path fill="none" stroke="#ffc0cb" d="M550.5144,-514.9925C520.6138,-533.8503 474.3748,-563.0126 441.6955,-583.6229"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="432.8627,-589.1936 438.9205,-580.0528 437.0919,-586.5264 441.321,-583.8591 441.321,-583.8591 441.321,-583.8591 437.0919,-586.5264 443.7216,-587.6653 432.8627,-589.1936 432.8627,-589.1936"/>
+            <text text-anchor="middle" x="521.2902" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x11A</text>
+        </g>
+        <!-- truc&#45;&gt;tru -->
+        <g id="edge59" class="edge">
+            <title>truc&#45;&gt;tru</title>
+            <path fill="none" stroke="#ffc0cb" d="M670.9731,-402.6462C666.3263,-416.3625 658.5213,-435.1464 647.2369,-448.8313 639.6641,-458.015 629.8999,-466.2538 620.1806,-473.1865"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="611.7071,-478.9336 617.4572,-469.5962 615.8452,-476.127 619.9832,-473.3204 619.9832,-473.3204 619.9832,-473.3204 615.8452,-476.127 622.5091,-477.0446 611.7071,-478.9336 611.7071,-478.9336"/>
+            <text text-anchor="middle" x="675.9052" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x11B</text>
+        </g>
+        <!-- truck -->
+        <g id="node22" class="node">
+            <title>truck</title>
+            <ellipse fill="none" stroke="#0000ff" cx="676.2369" cy="-284.4" rx="73.4745" ry="18"/>
+            <text text-anchor="middle" x="676.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">contentArray[4]</text>
+        </g>
+        <!-- truc&#45;&gt;truck -->
+        <g id="edge23" class="edge">
+            <title>truc&#45;&gt;truck</title>
+            <path fill="none" stroke="#add8e6" d="M661.1247,-367.9416C654.3993,-359.4018 647.222,-348.4489 643.7369,-337.2 641.5272,-330.0678 641.0565,-327.369 643.7369,-320.4 645.1585,-316.7039 647.1547,-313.1507 649.4663,-309.8129"/>
+            <polygon fill="#add8e6" stroke="#add8e6" points="655.7148,-302.0002 652.9831,-312.6204 652.5918,-305.9049 649.4689,-309.8097 649.4689,-309.8097 649.4689,-309.8097 652.5918,-305.9049 645.9546,-306.999 655.7148,-302.0002 655.7148,-302.0002"/>
+            <text text-anchor="middle" x="648.4869" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> k</text>
+        </g>
+        <!-- truc&#45;&gt;truck -->
+        <g id="edge48" class="edge">
+            <title>truc&#45;&gt;truck</title>
+            <path fill="none" stroke="#0000ff" d="M676.2369,-366.2345C676.2369,-351.2603 676.2369,-329.8643 676.2369,-312.7055"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="679.737,-312.6645 676.2369,-302.6646 672.737,-312.6646 679.737,-312.6645"/>
+            <text text-anchor="middle" x="681.4869" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> k</text>
+        </g>
+        <!-- truck&#45;&gt;truc -->
+        <g id="edge58" class="edge">
+            <title>truck&#45;&gt;truc</title>
+            <path fill="none" stroke="#ffc0cb" d="M683.5061,-302.4275C685.4432,-308.0852 687.266,-314.4172 688.2369,-320.4 690.1842,-332.3997 688.3971,-345.6086 685.6614,-356.9103"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="682.9623,-366.6639 681.2924,-355.8259 684.2958,-361.845 685.6294,-357.0261 685.6294,-357.0261 685.6294,-357.0261 684.2958,-361.845 689.9664,-358.2263 682.9623,-366.6639 682.9623,-366.6639"/>
+            <text text-anchor="middle" x="695.5232" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#ff0000">~4</text>
+        </g>
+        <!-- root2&#45;&gt;start -->
+        <g id="edge57" class="edge">
+            <title>root2&#45;&gt;start</title>
+            <path fill="none" stroke="#ffc0cb" d="M357.5792,-835.5119C349.8381,-849.8782 340.3722,-867.4453 332.6656,-881.7478"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="327.891,-890.6086 328.6731,-879.6706 330.2628,-886.2069 332.6346,-881.8052 332.6346,-881.8052 332.6346,-881.8052 330.2628,-886.2069 336.5961,-883.9399 327.891,-890.6086 327.891,-890.6086"/>
+            <text text-anchor="middle" x="364.2902" y="-859.9251" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x13A</text>
+        </g>
+        <!-- root2&#45;&gt;t2 -->
+        <g id="edge35" class="edge">
+            <title>root2&#45;&gt;t2</title>
+            <path fill="none" stroke="#0000ff" d="M364.2537,-779.3628C362.2873,-768.2618 361.3673,-755.5211 363.8463,-744.0939 364.5092,-741.0382 365.4547,-737.9306 366.5615,-734.8864"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="369.8507,-736.0934 370.4972,-725.5183 363.3971,-733.3821 369.8507,-736.0934"/>
+            <text text-anchor="middle" x="366.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+        </g>
+        <!-- t2&#45;&gt;root2 -->
+        <g id="edge56" class="edge">
+            <title>t2&#45;&gt;root2</title>
+            <path fill="none" stroke="#ffc0cb" d="M378.7997,-726.0972C377.8593,-737.8779 376.5924,-753.7484 375.4225,-768.4033"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="374.5939,-778.7835 370.904,-768.4571 374.9918,-773.7994 375.3897,-768.8152 375.3897,-768.8152 375.3897,-768.8152 374.9918,-773.7994 379.8754,-769.1734 374.5939,-778.7835 374.5939,-778.7835"/>
+            <text text-anchor="middle" x="394.9052" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x13B</text>
+        </g>
+        <!-- t2&#45;&gt;tr2 -->
+        <g id="edge36" class="edge">
+            <title>t2&#45;&gt;tr2</title>
+            <path fill="none" stroke="#0000ff" d="M376.9714,-689.7532C375.7642,-679.4773 375.2844,-666.5113 378.0763,-655.2939 379.0487,-651.3869 380.4094,-647.4482 381.9998,-643.5998"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="385.287,-644.8318 386.3356,-634.2889 378.9412,-641.8767 385.287,-644.8318"/>
+            <text text-anchor="middle" x="381.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+        </g>
+        <!-- tr2&#45;&gt;tre -->
+        <g id="edge37" class="edge">
+            <title>tr2&#45;&gt;tre</title>
+            <path fill="none" stroke="#0000ff" d="M394.3252,-579.1086C390.6132,-567.9144 386.0477,-555.063 381.2369,-543.6626 379.7068,-540.0366 378.0202,-536.3025 376.2719,-532.597"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="379.3778,-530.9808 371.8525,-523.523 373.0845,-534.0459 379.3778,-530.9808"/>
+            <text text-anchor="middle" x="392.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+        </g>
+        <!-- tr2&#45;&gt;tri -->
+        <g id="edge38" class="edge">
+            <title>tr2&#45;&gt;tri</title>
+            <path fill="none" stroke="#0000ff" d="M428.3953,-585.4737C435.5741,-578.0764 442.8897,-569.4305 448.2369,-560.4626 452.9069,-552.6304 456.6772,-543.633 459.6635,-534.9006"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="463.0435,-535.8196 462.7083,-525.2301 456.3666,-533.7173 463.0435,-535.8196"/>
+            <text text-anchor="middle" x="459.9322" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+        </g>
+        <!-- tr2&#45;&gt;tra2 -->
+        <g id="edge39" class="edge">
+            <title>tr2&#45;&gt;tra2</title>
+            <path fill="none" stroke="#0000ff" d="M370.3864,-593.0877C358.5096,-588.0561 344.9047,-582.6514 332.2369,-578.4626 301.0365,-568.1458 283.7876,-583.6708 260.5223,-560.4626 253.8249,-553.7816 249.75,-544.7649 247.3038,-535.603"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="250.7083,-534.7836 245.2416,-525.7081 243.8556,-536.2118 250.7083,-534.7836"/>
+            <text text-anchor="middle" x="266.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+        </g>
+        <!-- tr2&#45;&gt;tru -->
+        <g id="edge46" class="edge">
+            <title>tr2&#45;&gt;tru</title>
+            <path fill="none" stroke="#0000ff" d="M440.8715,-604.3719C473.582,-599.5098 520.4295,-587.9399 551.2369,-560.4626 558.9738,-553.5619 564.7664,-544.1924 569.054,-534.745"/>
+            <polygon fill="#0000ff" stroke="#0000ff" points="572.3451,-535.9426 572.8562,-525.3601 565.8573,-533.3141 572.3451,-535.9426"/>
+            <text text-anchor="middle" x="568.4869" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> u</text>
+        </g>
+        <!-- tr2&#45;&gt;t2 -->
+        <g id="edge55" class="edge">
+            <title>tr2&#45;&gt;t2</title>
+            <path fill="none" stroke="#ffc0cb" d="M396.5462,-637.0309C393.433,-650.5957 389.7463,-666.6595 386.6663,-680.0797"/>
+            <polygon fill="#ffc0cb" stroke="#ffc0cb" points="384.3488,-690.1773 382.1998,-679.4241 385.4673,-685.304 386.5858,-680.4307 386.5858,-680.4307 386.5858,-680.4307 385.4673,-685.304 390.9718,-681.4373 384.3488,-690.1773 384.3488,-690.1773"/>
+            <text text-anchor="middle" x="409.9045" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0FE</text>
+        </g>
+    </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.recycling.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.recycling.svg
new file mode 100644
index 000000000000..7861ea54b28d
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.recycling.svg
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><!--
+  - Licensed to the Apache Software Foundation (ASF) under one
+  - or more contributor license agreements.  See the NOTICE file
+  - distributed with this work for additional information
+  - regarding copyright ownership.  The ASF licenses this file
+  - to you under the Apache License, Version 2.0 (the
+  - "License"); you may not use this file except in compliance
+  - with the License.  You may obtain a copy of the License at
+  -
+  -     http://www.apache.org/licenses/LICENSE-2.0
+  -
+  - Unless required by applicable law or agreed to in writing, software
+  - distributed under the License is distributed on an "AS IS" BASIS,
+  - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  - See the License for the specific language governing permissions and
+  - limitations under the License.
+  -->
+
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" contentScriptType="application/ecmascript" contentStyleType="text/css" height="346px" preserveAspectRatio="none" style="width:1155px;height:346px;" version="1.1" viewBox="0 0 1155 346" width="1155px" zoomAndPan="magnify"><defs><filter height="300%" id="finwmyjg3l1x5" width="300%" x="-1" y="-1"><feGaussianBlur result="blurOut" stdDeviation="2.0"/><feColorMatrix in="blurOut" result="blurOut2" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 .4 0"/><feOffset dx="4.0" dy="4.0" in="blurOut2" result="blurOut3"/><feBlend in="SourceGraphic" in2="blurOut3" mode="normal"/></filter></defs><g><rect fill="#FEFECE" filter="url(#finwmyjg3l1x5)" height="49.25" style="stroke: #A80036; stroke-width: 1.5;" width="76" x="250.5" y="180"/><text fill="#000000" font-family="sans-serif" font-size="12" lengthAdjust="spacingAndGlyphs" textLength="41" x="268" y="193.1387">block2</text><line style="stroke: #A80036; stroke-width: 1.5;" x1="251.5" x2="325.5" y1="197.9688" y2="197.9688"/><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="51" x="256.5" y="211.251">ready cells</text><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="64" x="256.5" y="222.8916">partially used</text><rect fill="#FEFECE" filter="url(#finwmyjg3l1x5)" height="49.25" style="stroke: #A80036; stroke-width: 1.5;" width="93" x="491.5" y="180"/><text fill="#000000" font-family="sans-serif" font-size="12" lengthAdjust="spacingAndGlyphs" textLength="41" x="517.5" y="193.1387">block3</text><line style="stroke: #A80036; stroke-width: 1.5;" x1="492.5" x2="583.5" y1="197.9688" y2="197.9688"/><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="51" x="497.5" y="211.251">ready cells</text><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="81" x="497.5" y="222.8916">expired barrier B</text><rect fill="#FEFECE" filter="url(#finwmyjg3l1x5)" height="49.25" style="stroke: #A80036; stroke-width: 1.5;" width="110" x="986" y="180"/><text fill="#000000" font-family="sans-serif" font-size="12" lengthAdjust="spacingAndGlyphs" textLength="41" x="1020.5" y="193.1387">block4</text><line style="stroke: #A80036; stroke-width: 1.5;" x1="987" x2="1095" y1="197.9688" y2="197.9688"/><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="98" x="992" y="211.251">cells awaiting barrier</text><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="75" x="992" y="222.8916">active barrier C</text><rect fill="#FEFECE" filter="url(#finwmyjg3l1x5)" height="49.25" style="stroke: #A80036; stroke-width: 1.5;" width="110" x="730.5" y="180"/><text fill="#000000" font-family="sans-serif" font-size="12" lengthAdjust="spacingAndGlyphs" textLength="41" x="765" y="193.1387">block5</text><line style="stroke: #A80036; stroke-width: 1.5;" x1="731.5" x2="839.5" y1="197.9688" y2="197.9688"/><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="98" x="736.5" y="211.251">cells awaiting barrier</text><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="75" x="736.5" y="222.8916">active barrier C</text><rect fill="#FEFECE" filter="url(#finwmyjg3l1x5)" height="60.8906" style="stroke: #A80036; stroke-width: 1.5;" width="76" x="500" y="84"/><text fill="#000000" font-family="sans-serif" font-size="12" lengthAdjust="spacingAndGlyphs" textLength="41" x="517.5" y="97.1387">block6</text><line style="stroke: #A80036; stroke-width: 1.5;" x1="501" x2="575" y1="101.9688" y2="101.9688"/><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="64" x="506" y="115.251">released cells</text><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="14" x="506" y="126.8916">full</text><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="47" x="506" y="138.5322">no barrier</text><rect fill="#FEFECE" filter="url(#finwmyjg3l1x5)" height="49.25" style="stroke: #A80036; stroke-width: 1.5;" width="76" x="250.5" y="90"/><text fill="#000000" font-family="sans-serif" font-size="12" lengthAdjust="spacingAndGlyphs" textLength="41" x="268" y="103.1387">block7</text><line style="stroke: #A80036; stroke-width: 1.5;" x1="251.5" x2="325.5" y1="107.9688" y2="107.9688"/><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="64" x="256.5" y="121.251">released cells</text><text fill="#000000" font-family="sans-serif" font-size="10" lengthAdjust="spacingAndGlyphs" textLength="63" x="256.5" y="132.8916">partially filled</text><ellipse cx="69.5919" cy="204.5236" fill="#FEFECE" filter="url(#finwmyjg3l1x5)" rx="22.0919" ry="14.5236" style="stroke: #A80036; stroke-width: 1.5;"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacingAndGlyphs" textLength="27" x="56.0919" y="209.1721">free</text><path d="M206,264.5 L206,304.7656 A0,0 0 0 0 206,304.7656 L371,304.7656 A0,0 0 0 0 371,304.7656 L371,274.5 L361,264.5 L292.5,264.5 L288.5,229.214 L284.5,264.5 L206,264.5 A0,0 0 0 0 206,264.5 " fill="#FBFB77" filter="url(#finwmyjg3l1x5)" style="stroke: #A80036; stroke-width: 1.0;"/><path d="M361,264.5 L361,274.5 L371,274.5 L361,264.5 " fill="#FBFB77" style="stroke: #A80036; stroke-width: 1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" font-style="italic" lengthAdjust="spacingAndGlyphs" textLength="73" x="212" y="281.5669">allocateCell</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="67" x="289" y="281.5669">takes cells</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="67" x="212" y="296.6997">from here.</text><path d="M444,264.5 L444,304.7656 A0,0 0 0 0 444,304.7656 L632,304.7656 A0,0 0 0 0 632,304.7656 L632,274.5 L622,264.5 L542,264.5 L538,229.214 L534,264.5 L444,264.5 A0,0 0 0 0 444,264.5 " fill="#FBFB77" filter="url(#finwmyjg3l1x5)" style="stroke: #A80036; stroke-width: 1.0;"/><path d="M622,264.5 L622,274.5 L632,274.5 L622,264.5 " fill="#FBFB77" style="stroke: #A80036; stroke-width: 1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" font-style="italic" lengthAdjust="spacingAndGlyphs" textLength="25" x="450" y="281.5669">free</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="91" x="479" y="281.5669">will move here</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="167" x="450" y="296.6997">when block2 is exhausted.</text><path d="M692,264 L692,334.5313 A0,0 0 0 0 692,334.5313 L879,334.5313 A0,0 0 0 0 879,334.5313 L879,274 L869,264 L789.5,264 L785.5,229.085 L781.5,264 L692,264 A0,0 0 0 0 692,264 " fill="#FBFB77" filter="url(#finwmyjg3l1x5)" style="stroke: #A80036; stroke-width: 1.0;"/><path d="M869,264 L869,274 L879,274 L869,264 " fill="#FBFB77" style="stroke: #A80036; stroke-width: 1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="137" x="698" y="281.0669">If block3 is exhausted</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="166" x="698" y="296.1997">before the barrier expires,</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="128" x="698" y="311.3325">it will be replenished</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="97" x="698" y="326.4653">with fresh cells.</text><ellipse cx="1041.1884" cy="126.3377" fill="#FEFECE" filter="url(#finwmyjg3l1x5)" rx="77.1884" ry="17.8377" style="stroke: #A80036; stroke-width: 1.5;"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacingAndGlyphs" textLength="124" x="979.1884" y="130.9861">awaitingBarrierTail</text><path d="M939,264 L939,319.3984 A0,0 0 0 0 939,319.3984 L1143,319.3984 A0,0 0 0 0 1143,319.3984 L1143,274 L1133,264 L1045,264 L1041,229.125 L1037,264 L939,264 A0,0 0 0 0 939,264 " fill="#FBFB77" filter="url(#finwmyjg3l1x5)" style="stroke: #A80036; stroke-width: 1.0;"/><path d="M1133,264 L1133,274 L1143,274 L1133,264 " fill="#FBFB77" style="stroke: #A80036; stroke-width: 1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="175" x="945" y="281.0669">Blocks 5 and 4 were moved</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="89" x="945" y="296.1997">together from</text><text fill="#000000" font-family="sans-serif" font-size="13" font-style="italic" lengthAdjust="spacingAndGlyphs" textLength="82" x="1038" y="296.1997">justReleased</text><text fill="#000000" font-family="sans-serif" font-size="13" font-style="italic" lengthAdjust="spacingAndGlyphs" textLength="4" x="1120" y="296.1997">,</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="183" x="945" y="311.3325">they share the same barrier.</text><ellipse cx="69.7036" cy="114.6407" fill="#FEFECE" filter="url(#finwmyjg3l1x5)" rx="63.7036" ry="15.1407" style="stroke: #A80036; stroke-width: 1.5;"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacingAndGlyphs" textLength="90" x="24.7036" y="119.2892">justReleased</text><path d="M193,30 L193,55.1328 A0,0 0 0 0 193,55.1328 L284.5,55.1328 L288.5,89.75 L292.5,55.1328 L384,55.1328 A0,0 0 0 0 384,55.1328 L384,40 L374,30 L193,30 A0,0 0 0 0 193,30 " fill="#FBFB77" filter="url(#finwmyjg3l1x5)" style="stroke: #A80036; stroke-width: 1.0;"/><path d="M374,30 L374,40 L384,40 L374,30 " fill="#FBFB77" style="stroke: #A80036; stroke-width: 1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" font-style="italic" lengthAdjust="spacingAndGlyphs" textLength="69" x="199" y="47.0669">recycleCell</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="97" x="272" y="47.0669">puts cells here.</text><path d="M461.5,8.5 L461.5,48.7656 A0,0 0 0 0 461.5,48.7656 L534,48.7656 L538,83.93 L542,48.7656 L614.5,48.7656 A0,0 0 0 0 614.5,48.7656 L614.5,18.5 L604.5,8.5 L461.5,8.5 A0,0 0 0 0 461.5,8.5 " fill="#FBFB77" filter="url(#finwmyjg3l1x5)" style="stroke: #A80036; stroke-width: 1.0;"/><path d="M604.5,8.5 L604.5,18.5 L614.5,18.5 L604.5,8.5 " fill="#FBFB77" style="stroke: #A80036; stroke-width: 1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="132" x="467.5" y="25.5669">This block is awaiting</text><text fill="#000000" font-family="sans-serif" font-size="13" font-style="italic" lengthAdjust="spacingAndGlyphs" textLength="116" x="467.5" y="40.6997">completeMutation</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacingAndGlyphs" textLength="4" x="583.5" y="40.6997">.</text><!--MD5=[887c525a89dcde46bea6320fdca2bddd]
+link free to block2--><path d="M91.702,204.5 C127.13,204.5 199.01,204.5 245.126,204.5 " fill="none" id="free-&gt;block2" style="stroke: #A80036; stroke-width: 1.0;"/><polygon fill="#A80036" points="250.34,204.5,241.34,200.5,245.34,204.5,241.34,208.5,250.34,204.5" style="stroke: #A80036; stroke-width: 1.0;"/><!--MD5=[d115d9281859d487cf971087884e9111]
+link block2 to block3--><path d="M326.553,204.5 C368.969,204.5 438.694,204.5 486.166,204.5 " fill="none" id="block2-&gt;block3" style="stroke: #A80036; stroke-width: 1.0;"/><polygon fill="#A80036" points="491.264,204.5,482.264,200.5,486.264,204.5,482.264,208.5,491.264,204.5" style="stroke: #A80036; stroke-width: 1.0;"/><!--MD5=[e8ababe78de7c4f379e18dfb0638b889]
+link block3 to block5--><path d="M584.601,204.5 C624.176,204.5 681.743,204.5 724.947,204.5 " fill="none" id="block3-&gt;block5" style="stroke: #A80036; stroke-width: 1.0;"/><polygon fill="#A80036" points="730.143,204.5,721.143,200.5,725.143,204.5,721.143,208.5,730.143,204.5" style="stroke: #A80036; stroke-width: 1.0;"/><!--MD5=[f55ff1b5719cb15b8711394b3b2e4abb]
+link block5 to block4--><path d="M840.571,204.5 C881.652,204.5 938.117,204.5 980.481,204.5 " fill="none" id="block5-&gt;block4" style="stroke: #A80036; stroke-width: 1.0;"/><polygon fill="#A80036" points="985.575,204.5,976.575,200.5,980.575,204.5,976.575,208.5,985.575,204.5" style="stroke: #A80036; stroke-width: 1.0;"/><!--MD5=[cc0afcbb00e3ad0cbb694adc5825b8e9]
+link awaitingBarrierTail to block4--><path d="M1041,144.829 C1041,154.761 1041,164.694 1041,174.626 " fill="none" id="awaitingBarrierTail-&gt;block4" style="stroke: #A80036; stroke-width: 1.0;"/><polygon fill="#A80036" points="1041,179.82,1045,170.82,1041,174.82,1037,170.82,1041,179.82" style="stroke: #A80036; stroke-width: 1.0;"/><!--MD5=[a4d9e142809aacbb1ce24060899a0ec9]
+link justReleased to block7--><path d="M133.265,114.5 C169.267,114.5 213.491,114.5 245.339,114.5 " fill="none" id="justReleased-&gt;block7" style="stroke: #A80036; stroke-width: 1.0;"/><polygon fill="#A80036" points="250.341,114.5,241.341,110.5,245.341,114.5,241.341,118.5,250.341,114.5" style="stroke: #A80036; stroke-width: 1.0;"/><!--MD5=[6cc67023ea77713d166dd2b12e568ea0]
+link block7 to block6--><path d="M326.553,114.5 C371.572,114.5 447.352,114.5 494.641,114.5 " fill="none" id="block7-&gt;block6" style="stroke: #A80036; stroke-width: 1.0;"/><polygon fill="#A80036" points="499.693,114.5,490.693,110.5,494.693,114.5,490.693,118.5,499.693,114.5" style="stroke: #A80036; stroke-width: 1.0;"/><!--MD5=[a6e4b6d6f9e243ea1ba58b1343d7b1e8]
+@startuml
+allowmixing
+  left to right direction
+  hide empty description
+
+  object block2
+  object block3
+  object block4
+  object block5
+  object block6
+  object block7
+
+  usecase free
+  free - -> block2
+
+
+  block2 - -> block3
+  block2 : ready cells
+  block2 : partially used
+  note bottom of block2 : <i>allocateCell</i> takes cells\nfrom here.
+
+  block3 - -> block5
+  block3 : ready cells
+  block3 : expired barrier B
+  note bottom of block3 : <i>free</i> will move here\nwhen block2 is exhausted.
+
+  block5 - -> block4
+  block5 : cells awaiting barrier
+  block5 : active barrier C
+  note bottom of block5 : If block3 is exhausted\nbefore the barrier expires,\nit will be replenished\nwith fresh cells.
+
+  usecase awaitingBarrierTail
+  awaitingBarrierTail -> block4
+
+  block4 : cells awaiting barrier
+  block4 : active barrier C
+  note bottom of block4 : Blocks 5 and 4 were moved\ntogether from <i>justReleased<i>,\nthey share the same barrier.
+
+  usecase justReleased
+  justReleased - -> block7
+  block7 - -> block6
+
+  block7 : released cells
+  block7 : partially filled
+  note top of block7 : <i>recycleCell</i> puts cells here.
+
+  block6 : released cells
+  block6 : full
+  block6 : no barrier
+  note top of block6 : This block is awaiting\n<i>completeMutation</i>.
+@enduml
+
+PlantUML version 1.2019.13(Tue Dec 10 17:18:29 UTC 2019)
+(GPL source distribution)
+Java Runtime: Java(TM) SE Runtime Environment
+JVM: Java HotSpot(TM) 64-Bit Server VM
+Java Version: 1.7.0_25-b15
+Operating System: Linux
+Default Encoding: UTF-8
+Language: en
+Country: US
+--></g></svg>
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/tries/MemoryAllocationStrategy.java b/src/java/org/apache/cassandra/db/tries/MemoryAllocationStrategy.java
new file mode 100644
index 000000000000..c34942097437
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/MemoryAllocationStrategy.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+/**
+ * Allocation strategy for buffers and arrays for InMemoryTrie's. Controls how space is allocated and reused.
+ */
+public interface MemoryAllocationStrategy
+{
+    /**
+     * Get a free index. This is either a new index, allocated via the passed index producer functions, or one that
+     * has been previously recycled.
+     */
+    int allocate() throws TrieSpaceExhaustedException;
+
+    /**
+     * Marks the given index for recycling.
+     *
+     * When the index is actually reused depends on the recycling strategy. In any case it cannot be before the current
+     * mutation is complete (because it may still be walking cells that have been moved), and any concurrent readers
+     * that have started before this cell has become unreachable must also have completed.
+     */
+    void recycle(int index);
+
+    /**
+     * To be called when a mutation completes. No new readers must be able to see recycled content at the time of this
+     * call (the paths for reaching them must have been overwritten via a volatile write; additionally, if the buffer
+     * has grown, the root variable (which is stored outside the buffer) must have accepted a volatile write).
+     * No recycled indexes can be made available for reuse before this is called, and before any readers started before
+     * this call have completed.
+     */
+    void completeMutation();
+
+    /**
+     * Called when a mutation is aborted because of an exception. This means that the indexes that were marked for
+     * recycling are still going to be in use (unless this is called a later separate completeMutation call may release
+     * and reuse them, causing corruption).
+     *
+     * Aborted mutations are not normal, and at this time we are not trying to ensure that a trie will behave at its
+     * best if an abort has taken place (i.e. it may take more space, be slower etc.), but it should still operate
+     * correctly.
+     */
+    void abortMutation();
+
+    /**
+     * Returns the number of indexes that have been claimed by the allocation strategy but are not currently in use
+     * (either because they are in various stages of recycling, or have yet to see first use).
+     */
+    long indexCountInPipeline();
+
+    /**
+     * Constructs a list of all the indexes that are in the recycling pipeline.
+     * Used to test available and unreachable indexes are the same thing.
+     */
+    @VisibleForTesting
+    IntArrayList indexesInPipeline();
+
+    interface Allocator
+    {
+        int allocate() throws TrieSpaceExhaustedException;
+
+        default void allocate(int[] indexList) throws TrieSpaceExhaustedException
+        {
+            for (int i = indexList.length - 1; i >= 0; --i)
+                indexList[i] = allocate();
+        }
+    }
+
+    /**
+     * Strategy for small short-lived tries, usually on-heap. This strategy does not reuse any indexes.
+     */
+    class NoReuseStrategy implements MemoryAllocationStrategy
+    {
+        final Allocator allocator;
+
+        public NoReuseStrategy(Allocator allocator)
+        {
+            this.allocator = allocator;
+        }
+
+        public int allocate() throws TrieSpaceExhaustedException
+        {
+            return allocator.allocate();
+        }
+
+        public void recycle(int index)
+        {
+            // No reuse, do nothing
+        }
+
+        public void completeMutation()
+        {
+            // No reuse, nothing to do
+        }
+
+        public void abortMutation()
+        {
+            // No reuse, nothing to do
+        }
+
+        @Override
+        public long indexCountInPipeline()
+        {
+            // No indexes recycled
+            return 0;
+        }
+
+        @Override
+        public IntArrayList indexesInPipeline()
+        {
+            return new IntArrayList();
+        }
+    }
+
+    /**
+     * Reuse strategy for large, long-lived tries. Recycles indexes when it knows that the mutation recycling
+     * them has completed, and all reads started no later than this completion have also completed (signalled by an
+     * OpOrder which the strategy assumes all readers subscribe to).
+     *
+     * The OpOrder recycling strategy holds queues of indexes available for recycling. The queues ar organized in blocks
+     * of REUSE_BLOCK_SIZE entries. The blocks move through the following stages:
+     * - Being filled with newly released indexes. In this stage they are at the head of the "justReleased" list. When
+     *   a block becomes full, a new block is created and attached to the head of the list.
+     * - Full, but the mutation that released one or more of the mutations in them has not yet completed. In this stage
+     *   they are attached to the "justReleased" list as the second or further block. When a mutationComplete is
+     *   received, all such blocks get issued a common OpOrder.Barrier and are attached to "awaitingBarrierTail" (which
+     *   is the tail of the "free" list).
+     * - Awaiting a barrier. In this stage they are in the "free" list after its head, closer to its
+     *   "awaitingBarrierTail", identified by the fact that their barrier has not yet expired. Note that the blocks are
+     *   put in the order in which their barriers are issued, thus if a block has an active barrier, all blocks that
+     *   follow it in the list also do.
+     * - Ready for use. In this stage they are still in the "free" list after its head, but their barrier has now
+     *   expired. All the indexes in such blocks can now be reused, and will be when the head of the list is exhausted.
+     * - Active free block at the head of the "free" list. This block is the one new allocations are served from. When
+     *   it is exhausted, we check if the next block's barrier has expired. If so, the "free" pointer moves to it.
+     *   If not, there's nothing to reuse as any blocks in the list still have an active barrier, thus we grab some new
+     *   memory and refill the block.
+     * - If a mutation is aborted by an error, we throw away all indexes in the "justReleased" list. This is done so
+     *   that none of the indexes that were marked for release, but whose parent chain may have remained in place,
+     *   making them reachable, are reused and corrupt the trie. This will leak some indexes (from earlier mutations in
+     *   the block and/or ones whose parents have already been moved), but we prefer not to pay the cost of identifying
+     *   the exact indexes that need to remain or be recycled.
+     *   We assume that exceptions while mutating are not normal and should not happen, and thus a temporary leak (e.g.
+     *   until the memtable is switched) is acceptable. Should this change (e.g. if a trie is used for the full lifetime
+     *   of the process or longer and exceptions are expected as part of its function), we can implement a reachability
+     *   walk to identify orphaned indexes and call it with some frequency after one or more exceptions have occured.
+     */
+    static class OpOrderReuseStrategy implements MemoryAllocationStrategy
+    {
+        /**
+         * Cells list holding indexes that are just recycled. When full, new one is allocated and linked.
+         *
+         * On mutationComplete, any full (in justReleased.nextList) lists get issued a barrier and are moved to
+         * awaitingBarrierTail.
+         */
+        IndexBlockList justReleased;
+
+        /**
+         * Tail of the "free and awaiting barrier" queue. This is reachable by following the links from free.
+         *
+         * Full lists are attached to this tail when their barrier is issued.
+         * Lists are consumed from the head when free becomes empty if the list at the head has an expired barrier.
+         */
+        IndexBlockList awaitingBarrierTail;
+
+        /**
+         * Current free list, head of the "free and awaiting barrier" queue. Allocations are served from here.
+         *
+         * Starts full, and when it is exhausted we check the barrier at the next linked block.
+         * If expired, update free to point to it (consuming one block from the queue).
+         * If not, re-fill the block by allocating a new set of REUSE_BLOCK_SIZE indexes.
+         */
+        IndexBlockList free;
+
+        /**
+         * Called to allocate a new block of indexes to distribute.
+         */
+        final Allocator allocator;
+        final OpOrder opOrder;
+
+        public OpOrderReuseStrategy(Allocator allocator, OpOrder opOrder)
+        {
+            this.allocator = allocator;
+            this.opOrder = opOrder;
+            justReleased = new IndexBlockList(null);
+            awaitingBarrierTail = free = new IndexBlockList(null);
+            free.count = 0;
+        }
+
+        @Override
+        public int allocate() throws TrieSpaceExhaustedException
+        {
+            if (free.count == 0)
+            {
+                IndexBlockList awaitingBarrierHead = free.nextList;
+                if (awaitingBarrierHead != null &&
+                    (awaitingBarrierHead.barrier == null || awaitingBarrierHead.barrier.allPriorOpsAreFinished()))
+                {
+                    // A block is ready for reuse. Switch to it.
+                    free = awaitingBarrierHead;
+                    // Index blocks only enter these lists when the justReleased block is filled. Sanity check that
+                    // the block is still full.
+                    assert free.count == free.indexes.length;
+                    // We could recycle/pool the IndexBlockList object that free was pointing to before this.
+                    // As the trie will create and drop many times more objects to end up filling one of these, the
+                    // potential impact does not appear to justify the extra complexity.
+                }
+                else
+                {
+                    // Nothing available for reuse. Grab more memory.
+                    allocator.allocate(free.indexes);
+                    free.count = free.indexes.length;
+                }
+            }
+
+            return free.indexes[--free.count];
+        }
+
+        @Override
+        public void recycle(int index)
+        {
+            if (justReleased.count == REUSE_BLOCK_SIZE)
+            {
+                // Block is full, allocate and attach a new one.
+                justReleased = new IndexBlockList(justReleased);
+            }
+
+            justReleased.indexes[justReleased.count++] = index;
+        }
+
+        @Override
+        public void completeMutation()
+        {
+            IndexBlockList toProcess = justReleased.nextList;
+            if (toProcess == null)
+                return;
+
+            // We have some completed blocks now, issue a barrier for them and move them to the
+            // "free and awaiting barrier" queue.
+            justReleased.nextList = null;
+
+            OpOrder.Barrier barrier = null;
+            if (opOrder != null)
+            {
+                barrier = opOrder.newBarrier();
+                barrier.issue();
+            }
+
+            IndexBlockList last = null;
+            for (IndexBlockList current = toProcess; current != null; current = current.nextList)
+            {
+                current.barrier = barrier;
+                last = current;
+            }
+
+            assert awaitingBarrierTail.nextList == null;
+            awaitingBarrierTail.nextList = toProcess;
+            awaitingBarrierTail = last;
+        }
+
+        @Override
+        public void abortMutation()
+        {
+            // Some of the releases in the justReleased queue may still be reachable cells.
+            // We don't have a way of telling which, so we have to remove everything.
+            justReleased.nextList = null;
+            justReleased.count = 0;
+        }
+
+        /**
+         * Returns the number of indexes that are somewhere in the recycling pipeline.
+         */
+        @Override
+        public long indexCountInPipeline()
+        {
+            long count = 0;
+            for (IndexBlockList list = justReleased; list != null; list = list.nextList)
+                count += list.count;
+            for (IndexBlockList list = free; list != null; list = list.nextList) // includes awaiting barrier
+                count += list.count;
+            return count;
+        }
+
+        @Override
+        public IntArrayList indexesInPipeline()
+        {
+            IntArrayList res = new IntArrayList((int) indexCountInPipeline(), -1);
+            for (IndexBlockList list = justReleased; list != null; list = list.nextList)
+                res.addAll(new IntArrayList(list.indexes, list.count, -1));
+            for (IndexBlockList list = free; list != null; list = list.nextList) // includes awaiting barrier
+                res.addAll(new IntArrayList(list.indexes, list.count, -1));
+            return res;
+        }
+    }
+
+
+    static final int REUSE_BLOCK_SIZE = 252; // array fits into 1k bytes
+
+    static class IndexBlockList
+    {
+        final int[] indexes;
+        int count;
+        OpOrder.Barrier barrier;
+        IndexBlockList nextList;
+
+        IndexBlockList(IndexBlockList next)
+        {
+            indexes = new int[REUSE_BLOCK_SIZE];
+            nextList = next;
+            count = 0;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/MergeTrie.java b/src/java/org/apache/cassandra/db/tries/MergeTrie.java
index f2807769844f..ffdfee4267e8 100644
--- a/src/java/org/apache/cassandra/db/tries/MergeTrie.java
+++ b/src/java/org/apache/cassandra/db/tries/MergeTrie.java
@@ -19,6 +19,8 @@
 
 import com.google.common.collect.Iterables;
 
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
 /**
  * A merged view of two tries.
  *
@@ -44,25 +46,27 @@ class MergeTrie<T> extends Trie<T>
     }
 
     @Override
-    protected Cursor<T> cursor()
+    protected Cursor<T> cursor(Direction direction)
     {
-        return new MergeCursor<>(resolver, t1, t2);
+        return new MergeCursor<>(resolver, direction, t1, t2);
     }
 
     static class MergeCursor<T> implements Cursor<T>
     {
         private final MergeResolver<T> resolver;
+        private final Direction direction;
         private final Cursor<T> c1;
         private final Cursor<T> c2;
 
         boolean atC1;
         boolean atC2;
 
-        MergeCursor(MergeResolver<T> resolver, Trie<T> t1, Trie<T> t2)
+        MergeCursor(MergeResolver<T> resolver, Direction direction, Trie<T> t1, Trie<T> t2)
         {
             this.resolver = resolver;
-            this.c1 = t1.cursor();
-            this.c2 = t2.cursor();
+            this.direction = direction;
+            this.c1 = t1.cursor(direction);
+            this.c2 = t2.cursor(direction);
             assert c1.depth() == 0;
             assert c2.depth() == 0;
             atC1 = atC2 = true;
@@ -76,10 +80,17 @@ public int advance()
         }
 
         @Override
-        public int skipChildren()
+        public int skipTo(int skipDepth, int skipTransition)
         {
-            return checkOrder(atC1 ? c1.skipChildren() : c1.depth(),
-                              atC2 ? c2.skipChildren() : c2.depth());
+            int c1depth = c1.depth();
+            int c2depth = c2.depth();
+            assert skipDepth <= c1depth + 1 || skipDepth <= c2depth + 1;
+            if (atC1 || skipDepth < c1depth || skipDepth == c1depth && direction.gt(skipTransition, c1.incomingTransition()))
+                c1depth = c1.skipTo(skipDepth, skipTransition);
+            if (atC2 || skipDepth < c2depth || skipDepth == c2depth && direction.gt(skipTransition, c2.incomingTransition()))
+                c2depth = c2.skipTo(skipDepth, skipTransition);
+
+            return checkOrder(c1depth, c2depth);
         }
 
         @Override
@@ -116,8 +127,9 @@ private int checkOrder(int c1depth, int c2depth)
             // c1depth == c2depth
             int c1trans = c1.incomingTransition();
             int c2trans = c2.incomingTransition();
-            atC1 = c1trans <= c2trans;
-            atC2 = c1trans >= c2trans;
+            atC1 = direction.le(c1trans, c2trans);
+            atC2 = direction.le(c2trans, c1trans);
+            assert atC1 | atC2;
             return c1depth;
         }
 
@@ -133,6 +145,21 @@ public int incomingTransition()
             return atC1 ? c1.incomingTransition() : c2.incomingTransition();
         }
 
+        @Override
+        public Direction direction()
+        {
+            return direction;
+        }
+
+        @Override
+        public ByteComparable.Version byteComparableVersion()
+        {
+            assert c1.byteComparableVersion() == c2.byteComparableVersion() :
+                "Merging cursors with different byteComparableVersions: " +
+                c1.byteComparableVersion() + " vs " + c2.byteComparableVersion();
+            return c1.byteComparableVersion();
+        }
+
         public T content()
         {
             T mc = atC2 ? c2.content() : null;
@@ -144,6 +171,19 @@ else if (nc == null)
             else
                 return resolver.resolve(nc, mc);
         }
+
+        @Override
+        public Trie<T> tailTrie()
+        {
+            if (atC1 && atC2)
+                return new MergeTrie<>(resolver, c1.tailTrie(), c2.tailTrie());
+            else if (atC1)
+                return c1.tailTrie();
+            else if (atC2)
+                return c2.tailTrie();
+            else
+                throw new AssertionError();
+        }
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/db/tries/PrefixedTrie.java b/src/java/org/apache/cassandra/db/tries/PrefixedTrie.java
new file mode 100644
index 000000000000..cf5f9dd63513
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/PrefixedTrie.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Prefixed trie. Represents the content of the given trie with the prefix prepended to all keys.
+ */
+public class PrefixedTrie<T> extends Trie<T>
+{
+    final ByteComparable prefix;
+    final Trie<T> trie;
+
+    public PrefixedTrie(ByteComparable prefix, Trie<T> trie)
+    {
+        this.prefix = prefix;
+        this.trie = trie;
+    }
+
+    @Override
+    protected Trie.Cursor<T> cursor(Direction direction)
+    {
+        Trie.Cursor<T> sourceCursor = trie.cursor(direction);
+        return new Cursor<>(prefix.asComparableBytes(sourceCursor.byteComparableVersion()), sourceCursor);
+    }
+
+    private static class Cursor<T> implements Trie.Cursor<T>
+    {
+        final Trie.Cursor<T> tail;
+        ByteSource prefixBytes;
+        int nextPrefixByte;
+        int incomingTransition;
+        int depthOfPrefix;
+
+        Cursor(ByteSource prefix, Trie.Cursor<T> tail)
+        {
+            this.tail = tail;
+            prefixBytes = prefix;
+            incomingTransition = -1;
+            nextPrefixByte = prefixBytes.next();
+            depthOfPrefix = 0;
+        }
+
+        int completeAdvanceInTail(int depthInTail)
+        {
+            if (depthInTail < 0)
+                return exhausted();
+
+            incomingTransition = tail.incomingTransition();
+            return depthInTail + depthOfPrefix;
+        }
+
+        boolean prefixDone()
+        {
+            return nextPrefixByte == ByteSource.END_OF_STREAM;
+        }
+
+        @Override
+        public int depth()
+        {
+            if (prefixDone())
+                return tail.depth() + depthOfPrefix;
+            else
+                return depthOfPrefix;
+        }
+
+        @Override
+        public int incomingTransition()
+        {
+            return incomingTransition;
+        }
+
+        @Override
+        public int advance()
+        {
+            if (prefixDone())
+                return completeAdvanceInTail(tail.advance());
+
+            ++depthOfPrefix;
+            incomingTransition = nextPrefixByte;
+            nextPrefixByte = prefixBytes.next();
+            return depthOfPrefix;
+        }
+
+        @Override
+        public int advanceMultiple(Trie.TransitionsReceiver receiver)
+        {
+            if (prefixDone())
+                return completeAdvanceInTail(tail.advanceMultiple(receiver));
+
+            while (!prefixDone())
+            {
+                receiver.addPathByte(incomingTransition);
+                ++depthOfPrefix;
+                incomingTransition = nextPrefixByte;
+                nextPrefixByte = prefixBytes.next();
+            }
+            return depthOfPrefix;
+        }
+
+        @Override
+        public int skipTo(int skipDepth, int skipTransition)
+        {
+            // regardless if we exhausted prefix, if caller asks for depth <= prefix depth, we're done.
+            if (skipDepth <= depthOfPrefix)
+                return exhausted();
+            if (prefixDone())
+                return completeAdvanceInTail(tail.skipTo(skipDepth - depthOfPrefix, skipTransition));
+            assert skipDepth == depthOfPrefix + 1 : "Invalid advance request to depth " + skipDepth + " to cursor at depth " + depthOfPrefix;
+            if (tail.direction().gt(skipTransition, nextPrefixByte))
+                return exhausted();
+            return advance();
+        }
+
+        private int exhausted()
+        {
+            incomingTransition = -1;
+            depthOfPrefix = -1;
+            nextPrefixByte = 0; // to make prefixDone() false so incomingTransition/depth/content are -1/-1/null
+            return depthOfPrefix;
+        }
+
+        public Direction direction()
+        {
+            return tail.direction();
+        }
+
+        public ByteComparable.Version byteComparableVersion()
+        {
+            return tail.byteComparableVersion();
+        }
+
+        @Override
+        public T content()
+        {
+            return prefixDone() ? tail.content() : null;
+        }
+
+        @Override
+        public Trie<T> tailTrie()
+        {
+            if (prefixDone())
+                return tail.tailTrie();
+            else
+            {
+                assert depthOfPrefix >= 0 : "tailTrie called on exhausted cursor";
+                if (!(prefixBytes instanceof ByteSource.Duplicatable))
+                    prefixBytes = ByteSource.duplicatable(prefixBytes);
+                ByteSource.Duplicatable duplicatableSource = (ByteSource.Duplicatable) prefixBytes;
+
+                return new PrefixedTrie<>(v -> duplicatableSource.duplicate(), tail.tailTrie());
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/SingletonTrie.java b/src/java/org/apache/cassandra/db/tries/SingletonTrie.java
index 0336a851ffab..e3eb62783ea1 100644
--- a/src/java/org/apache/cassandra/db/tries/SingletonTrie.java
+++ b/src/java/org/apache/cassandra/db/tries/SingletonTrie.java
@@ -26,26 +26,34 @@
 class SingletonTrie<T> extends Trie<T>
 {
     private final ByteComparable key;
+    private final ByteComparable.Version byteComparableVersion;
     private final T value;
 
-    SingletonTrie(ByteComparable key, T value)
+    SingletonTrie(ByteComparable key, ByteComparable.Version byteComparableVersion, T value)
     {
+        this.byteComparableVersion = byteComparableVersion;
         this.key = key;
         this.value = value;
     }
 
-    public Cursor cursor()
+    public Cursor cursor(Direction direction)
     {
-        return new Cursor();
+        return new Cursor(direction);
     }
 
     class Cursor implements Trie.Cursor<T>
     {
-        private final ByteSource src = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+        private final Direction direction;
+        private ByteSource src = key.asComparableBytes(byteComparableVersion);
         private int currentDepth = 0;
         private int currentTransition = -1;
         private int nextTransition = src.next();
 
+        public Cursor(Direction direction)
+        {
+            this.direction = direction;
+        }
+
         @Override
         public int advance()
         {
@@ -83,9 +91,17 @@ public int advanceMultiple(TransitionsReceiver receiver)
         }
 
         @Override
-        public int skipChildren()
+        public int skipTo(int skipDepth, int skipTransition)
         {
-            return currentDepth = -1;  // no alternatives
+            if (skipDepth <= currentDepth)
+            {
+                assert skipDepth < currentDepth || direction.gt(skipTransition, currentTransition);
+                return currentDepth = -1;  // no alternatives
+            }
+            if (direction.gt(skipTransition, nextTransition))
+                return currentDepth = -1;   // request is skipping over our path
+
+            return advance();
         }
 
         @Override
@@ -105,5 +121,27 @@ public int incomingTransition()
         {
             return currentTransition;
         }
+
+        @Override
+        public Direction direction()
+        {
+            return direction;
+        }
+
+        @Override
+        public ByteComparable.Version byteComparableVersion()
+        {
+            return byteComparableVersion;
+        }
+
+        @Override
+        public Trie<T> tailTrie()
+        {
+            if (!(src instanceof ByteSource.Duplicatable))
+                src = ByteSource.duplicatable(src);
+            ByteSource.Duplicatable duplicatableSource = (ByteSource.Duplicatable) src;
+
+            return new SingletonTrie(v -> duplicatableSource.duplicate(), byteComparableVersion, value);
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/db/tries/SlicedTrie.java b/src/java/org/apache/cassandra/db/tries/SlicedTrie.java
index 75ae3df27e10..c14f0adde620 100644
--- a/src/java/org/apache/cassandra/db/tries/SlicedTrie.java
+++ b/src/java/org/apache/cassandra/db/tries/SlicedTrie.java
@@ -61,170 +61,297 @@ public SlicedTrie(Trie<T> source, ByteComparable left, boolean includeLeft, Byte
         this.includeRight = includeRight;
     }
 
+    static ByteSource openAndMaybeAdd0(ByteComparable key, ByteComparable.Version byteComparableVersion, boolean shouldAdd0)
+    {
+        if (key == null)
+            return null;
+        ByteSource src = key.asComparableBytes(byteComparableVersion);
+        if (shouldAdd0)
+            return ByteSource.append(src, 0);
+        else
+            return src;
+    }
+
     @Override
-    protected Cursor<T> cursor()
+    protected Cursor<T> cursor(Direction direction)
+    {
+        Cursor<T> sourceCursor = source.cursor(direction);
+        // The cursor is left-inclusive and right-exclusive by default. If we need to change the inclusiveness, adjust
+        // the bound to the next possible value by adding a 00 byte at the end.
+        ByteSource leftSource = openAndMaybeAdd0(left, sourceCursor.byteComparableVersion(), !includeLeft);
+        ByteSource rightSource = openAndMaybeAdd0(right, sourceCursor.byteComparableVersion(), includeRight);
+
+        // Empty left bound is the same as having no left bound, adjust for that.
+        int leftNext = -1;
+        if (leftSource != null)
+        {
+            leftNext = leftSource.next();
+            if (leftNext == ByteSource.END_OF_STREAM)
+                leftSource = null;
+        }
+
+        // Empty right bound means the result can only be empty. Make things easier for the cursor by handling this.
+        int rightNext = -1;
+        if (rightSource != null)
+        {
+            rightNext = rightSource.next();
+            if (rightNext == ByteSource.END_OF_STREAM)
+            {
+                assert leftSource == null : "Invalid range " + sliceString();
+                return new Trie.EmptyCursor<>(direction, sourceCursor.byteComparableVersion());
+            }
+        }
+
+        return new SlicedCursor<>(sourceCursor,
+                                  leftSource,
+                                  leftNext,
+                                  rightSource,
+                                  rightNext,
+                                  direction);
+    }
+
+    String sliceString()
     {
-        return new SlicedCursor<>(this);
+        ByteComparable.Version version = source.cursor(Direction.FORWARD).byteComparableVersion();
+        return String.format("%s%s;%s%s",
+                             includeLeft ? "[" : "(",
+                             left.byteComparableAsString(version),
+                             right.byteComparableAsString(version),
+                             includeRight ? "]" : ")");
     }
 
     private enum State
     {
-        /** The cursor is still positioned on some prefix of the left bound. Content should not be produced. */
-        BEFORE_LEFT,
-        /** The cursor is positioned inside the range, i.e. beyond the left bound, possibly on a prefix of the right. */
+        /**
+         * The cursor is at the initial phase while it is walking prefixes of both bounds.
+         * Content is not to be reported.
+         */
+        COMMON_PREFIX,
+        /**
+         * The cursor is positioned on some prefix of the start bound, strictly before any prefix of the end bound in
+         * iteration order.
+         * Content should only be reported in the reverse direction (as these prefixes are prefixes of the right bound
+         * and included in the slice).
+         */
+        START_PREFIX,
+        /**
+         * The cursor is positioned inside the range, i.e. strictly between any prefixes of the start and end bounds.
+         * All content should be reported.
+         */
         INSIDE,
-        /** The cursor is positioned beyond the right bound. Exhaustion (depth -1) has been reported. */
-        AFTER_RIGHT
+        /**
+         * The cursor is positioned on some prefix of the end bound, strictly after any prefix of the start bound.
+         * Content should only be reported in the forward direction.
+         */
+        END_PREFIX,
+        /** The cursor is positioned beyond the end bound. Exhaustion (depth -1) has been reported. */
+        EXHAUSTED;
     }
 
     private static class SlicedCursor<T> implements Cursor<T>
     {
-        private final ByteSource left;
-        private final ByteSource right;
-        private final boolean includeLeft;
-        private final boolean excludeRight;
+        private ByteSource start;
+        private ByteSource end;
         private final Cursor<T> source;
+        private final Direction direction;
 
-        private State state;
-        private int leftNext;
-        private int leftNextDepth;
-        private int rightNext;
-        private int rightNextDepth;
+        State state;
+        int startNext;
+        int startNextDepth;
+        int endNext;
+        int endNextDepth;
 
-        public SlicedCursor(SlicedTrie<T> slicedTrie)
+        public SlicedCursor(Cursor<T> source,
+                            ByteSource leftSource,
+                            int leftNext,
+                            ByteSource rightSource,
+                            int rightNext,
+                            Direction direction)
         {
-            source = slicedTrie.source.cursor();
-            if (slicedTrie.left != null)
-            {
-                left = slicedTrie.left.asComparableBytes(BYTE_COMPARABLE_VERSION);
-                includeLeft = slicedTrie.includeLeft;
-                leftNext = left.next();
-                leftNextDepth = 1;
-                if (leftNext == ByteSource.END_OF_STREAM && includeLeft)
-                    state = State.INSIDE;
-                else
-                    state = State.BEFORE_LEFT;
-            }
-            else
-            {
-                left = null;
-                includeLeft = true;
-                state = State.INSIDE;
-            }
-
-            if (slicedTrie.right != null)
-            {
-                right = slicedTrie.right.asComparableBytes(BYTE_COMPARABLE_VERSION);
-                excludeRight = !slicedTrie.includeRight;
-                rightNext = right.next();
-                rightNextDepth = 1;
-                if (rightNext == ByteSource.END_OF_STREAM && excludeRight)
-                    state = State.BEFORE_LEFT;  // This is a hack, we are after the right bound but we don't want to
-                                                // report depth -1 yet. So just make sure root's content is not reported.
-            }
-            else
-            {
-                right = null;
-                excludeRight = true;
-                rightNextDepth = 0;
-            }
+            this.source = source;
+            this.direction = direction;
+            start = direction.select(leftSource, rightSource);
+            end = direction.select(rightSource, leftSource);
+            startNext = direction.select(leftNext, rightNext);
+            endNext = direction.select(rightNext, leftNext);
+            startNextDepth = start != null ? 1 : 0;
+            endNextDepth = end != null ? 1 : 0;
+            state = start != null
+                    ? end != null
+                      ? State.COMMON_PREFIX
+                      : State.START_PREFIX
+                    : end != null
+                      ? State.END_PREFIX
+                      : State.INSIDE;
         }
 
         @Override
         public int advance()
         {
-            assert (state != State.AFTER_RIGHT);
-
-            int newDepth = source.advance();
-            int transition = source.incomingTransition();
+            int newDepth;
+            int transition;
 
-            if (state == State.BEFORE_LEFT)
+            switch (state)
             {
-                // Skip any transitions before the left bound
-                while (newDepth == leftNextDepth && transition < leftNext)
-                {
-                    newDepth = source.skipChildren();
+                case COMMON_PREFIX:
+                case START_PREFIX:
+                    // Skip any transitions before the start bound
+                    newDepth = source.skipTo(startNextDepth, startNext);
                     transition = source.incomingTransition();
-                }
-
-                // Check if we are still following the left bound
-                if (newDepth == leftNextDepth && transition == leftNext)
-                {
-                    assert leftNext != ByteSource.END_OF_STREAM;
-                    leftNext = left.next();
-                    ++leftNextDepth;
-                    if (leftNext == ByteSource.END_OF_STREAM && includeLeft)
-                        state = State.INSIDE; // report the content on the left bound
-                }
-                else // otherwise we are beyond it
-                    state = State.INSIDE;
+                    return checkBothBounds(newDepth, transition);
+                case INSIDE:
+                case END_PREFIX:
+                    newDepth = source.advance();
+                    transition = source.incomingTransition();
+                    return checkEndBound(newDepth, transition);
+                default:
+                    throw new AssertionError();
             }
-
-            return checkRightBound(newDepth, transition);
         }
 
         private int markDone()
         {
-            state = State.AFTER_RIGHT;
+            state = State.EXHAUSTED;
             return -1;
         }
 
-        private int checkRightBound(int newDepth, int transition)
+        int checkBothBounds(int newDepth, int transition)
+        {
+            // Check if we are still following the start bound
+            if (newDepth == startNextDepth && transition == startNext)
+            {
+                assert startNext != ByteSource.END_OF_STREAM;
+                startNext = start.next();
+                ++startNextDepth;
+                State currState = state;
+                // In the forward direction the exact match for the left bound and all descendant states are
+                // included in the set.
+                // In the reverse direction we will instead use the -1 as target transition and thus ascend on
+                // the next advance (skipping the exact right bound and all its descendants).
+                if (startNext == ByteSource.END_OF_STREAM && direction.isForward())
+                    state = State.INSIDE; // checkEndBound may adjust this to END_PREFIX
+                if (currState == State.START_PREFIX)
+                    return newDepth;   // there is no need to check the end bound as we descended along a
+                                       // strictly earlier path
+            }
+            else // otherwise we are beyond the start bound
+                state = State.INSIDE; // checkEndBound may adjust this to END_PREFIX
+
+            return checkEndBound(newDepth, transition);
+        }
+
+        private int checkEndBound(int newDepth, int transition)
         {
             // Cursor positions compare by depth descending and transition ascending.
-            if (newDepth > rightNextDepth)
-                return newDepth;
-            if (newDepth < rightNextDepth)
+            if (newDepth > endNextDepth)
+                return newDepth;    // happy and quick path in the interior of the slice
+                                    // (state == State.INSIDE can be asserted here (we skip it for efficiency))
+            if (newDepth < endNextDepth)
                 return markDone();
-            // newDepth == rightDepth
-            if (transition < rightNext)
+            // newDepth == endDepth
+            if (direction.lt(transition, endNext))
+            {
+                adjustStateStrictlyBeforeEnd();
                 return newDepth;
-            if (transition > rightNext)
+            }
+            if (direction.lt(endNext, transition))
                 return markDone();
 
-            // Following right bound
-            rightNext = right.next();
-            ++rightNextDepth;
-            if (rightNext == ByteSource.END_OF_STREAM && excludeRight)
-                return markDone();  // do not report any content on the right bound
+            // Following end bound
+            endNext = end.next();
+            ++endNextDepth;
+            if (endNext == ByteSource.END_OF_STREAM)
+            {
+                // At the exact end bound.
+                if (direction.isForward())
+                {
+                    // In forward direction the right bound is not included in the slice.
+                    return markDone();
+                }
+                else
+                {
+                    // In reverse, the left bound and all its descendants are included, thus we use the -1 as limiting
+                    // transition. We can also see the bound as strictly ahead of our current position as the current
+                    // branch should be fully included.
+                    adjustStateStrictlyBeforeEnd();
+                }
+            }
+            else
+                adjustStateAtEndPrefix();
             return newDepth;
         }
 
+        private void adjustStateAtEndPrefix()
+        {
+            switch (state)
+            {
+                case INSIDE:
+                    state = State.END_PREFIX;
+                    break;
+            }
+        }
+
+        private void adjustStateStrictlyBeforeEnd()
+        {
+            switch (state)
+            {
+                case COMMON_PREFIX:
+                    state = State.START_PREFIX;
+                    break;
+                case END_PREFIX:
+                    state = State.INSIDE;
+                    break;
+            }
+        }
+
         @Override
         public int advanceMultiple(TransitionsReceiver receiver)
         {
             switch (state)
             {
-                case BEFORE_LEFT:
+                case COMMON_PREFIX:
+                case START_PREFIX:
+                case END_PREFIX:
                     return advance();   // descend only one level to be able to compare cursors correctly
                 case INSIDE:
                     int depth = source.depth();
-                    if (depth == rightNextDepth - 1)  // this is possible because right is already advanced;
-                        return advance();   // we need to check next byte against right boundary in this case
                     int newDepth = source.advanceMultiple(receiver);
                     if (newDepth > depth)
-                        return newDepth;    // successfully advanced
+                        return newDepth;    // successfully descended
                     // we ascended, check if we are still within boundaries
-                    return checkRightBound(newDepth, source.incomingTransition());
+                    return checkEndBound(newDepth, source.incomingTransition());
                 default:
                     throw new AssertionError();
             }
         }
 
         @Override
-        public int skipChildren()
+        public int skipTo(int skipDepth, int skipTransition)
         {
-            assert (state != State.AFTER_RIGHT);
+            // if skipping beyond end, we are done
+            if (skipDepth < endNextDepth || skipDepth == endNextDepth && direction.gt(skipTransition, endNext))
+                return markDone();
+            // if skipping before start, adjust request to skip to start
+            if (skipDepth == startNextDepth && direction.lt(skipTransition, startNext))
+                skipTransition = startNext;
 
-            // We are either inside or following the left bound. In the latter case ascend takes us beyond it.
-            state = State.INSIDE;
-            return checkRightBound(source.skipChildren(), source.incomingTransition());
+            switch (state)
+            {
+                case START_PREFIX:
+                case COMMON_PREFIX:
+                    return checkBothBounds(source.skipTo(skipDepth, skipTransition), source.incomingTransition());
+                case INSIDE:
+                case END_PREFIX:
+                    return checkEndBound(source.skipTo(skipDepth, skipTransition), source.incomingTransition());
+                default:
+                    throw new AssertionError("Cursor already exhaused.");
+            }
         }
 
         @Override
         public int depth()
         {
-            return state == State.AFTER_RIGHT ? -1 : source.depth();
+            return state == State.EXHAUSTED ? -1 : source.depth();
         }
 
         @Override
@@ -233,10 +360,99 @@ public int incomingTransition()
             return source.incomingTransition();
         }
 
+        @Override
+        public Direction direction()
+        {
+            return direction;
+        }
+
+        @Override
+        public ByteComparable.Version byteComparableVersion()
+        {
+            return source.byteComparableVersion();
+        }
+
         @Override
         public T content()
         {
-            return state == State.INSIDE ? source.content() : null;
+            switch (state)
+            {
+                case INSIDE:
+                    return source.content();
+                // Additionally, prefixes of the right bound (which are not prefixes of the left) need to be reported:
+                case START_PREFIX:
+                    // start prefixes in reverse direction (but making sure we don't report the exact match);
+                    return !direction.isForward() && startNext != ByteSource.END_OF_STREAM ? source.content() : null;
+                case END_PREFIX:
+                    // end prefixes in forward direction.
+                    return direction.isForward() ? source.content() : null;
+                default:
+                    return null;
+            }
+        }
+
+        @Override
+        public Trie<T> tailTrie()
+        {
+            final Trie<T> sourceTail = source.tailTrie();
+            switch (state)
+            {
+                case INSIDE:
+                    return sourceTail;
+                case COMMON_PREFIX:
+                    return makeTrie(sourceTail, duplicatableStart(), startNext, duplicatableEnd(), endNext, direction);
+                case START_PREFIX:
+                    return makeTrie(sourceTail, duplicatableStart(), startNext, null, -1, direction);
+                case END_PREFIX:
+                    return makeTrie(sourceTail, null, -1, duplicatableEnd(), endNext, direction);
+                default:
+                    throw new UnsupportedOperationException("tailTrie on a slice boundary");
+            }
+        }
+
+        private ByteSource.Duplicatable duplicatableStart()
+        {
+            if (start == null || start instanceof ByteSource.Duplicatable)
+                return (ByteSource.Duplicatable) start;
+            ByteSource.Duplicatable duplicatable = ByteSource.duplicatable(start);
+            start = duplicatable;
+            return duplicatable;
+        }
+
+        private ByteSource.Duplicatable duplicatableEnd()
+        {
+            if (end == null || end instanceof ByteSource.Duplicatable)
+                return (ByteSource.Duplicatable) end;
+            ByteSource.Duplicatable duplicatable = ByteSource.duplicatable(end);
+            end = duplicatable;
+            return duplicatable;
+        }
+
+
+        private static <T> Trie<T> makeTrie(Trie<T> source,
+                                            ByteSource.Duplicatable startSource,
+                                            int startNext,
+                                            ByteSource.Duplicatable endSource,
+                                            int endNext,
+                                            Direction direction)
+        {
+            ByteSource.Duplicatable leftSource = direction.select(startSource, endSource);
+            ByteSource.Duplicatable rightSource = direction.select(endSource, startSource);
+            int leftNext = direction.select(startNext, endNext);
+            int rightNext = direction.select(endNext, startNext);
+            return new Trie<T>()
+            {
+                @Override
+                protected Cursor<T> cursor(Direction direction)
+                {
+                    return new SlicedCursor<>(source.cursor(direction),
+                                              leftSource != null ? leftSource.duplicate() : null,
+                                              leftNext,
+                                              rightSource != null ? rightSource.duplicate() : null,
+                                              rightNext,
+                                              direction);
+                }
+            };
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/db/tries/Trie.java b/src/java/org/apache/cassandra/db/tries/Trie.java
index a139e08e67df..90006e52525b 100644
--- a/src/java/org/apache/cassandra/db/tries/Trie.java
+++ b/src/java/org/apache/cassandra/db/tries/Trie.java
@@ -28,31 +28,32 @@
 
 import org.agrona.DirectBuffer;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
 /**
  * Base class for tries.
- *
+ * <p>
  * Normal users of tries will only use the public methods, which provide various transformations of the trie, conversion
  * of its content to other formats (e.g. iterable of values), and several forms of processing.
- *
+ * <p>
  * For any unimplemented data extraction operations one can build on the {@link TrieEntriesWalker} (for-each processing)
  * and {@link TrieEntriesIterator} (to iterator) base classes, which provide the necessary mechanisms to handle walking
  * the trie.
- *
+ * <p>
  * The internal representation of tries using this interface is defined in the {@link Cursor} interface.
- *
+ * <p>
  * Cursors are a method of presenting the internal structure of a trie without representing nodes as objects, which is
  * still useful for performing the basic operations on tries (iteration, slicing/intersection and merging). A cursor
  * will list the nodes of a trie in order, together with information about the path that was taken to reach them.
- *
- * To begin traversal over a trie, one must retrieve a cursor by calling {@link #cursor()}. Because cursors are
+ * <p>
+ * To begin traversal over a trie, one must retrieve a cursor by calling {@link #cursor}. Because cursors are
  * stateful, the traversal must always proceed from one thread. Should concurrent reads be required, separate calls to
- * {@link #cursor()} must be made. Any modification that has completed before the construction of a cursor must be
+ * {@link #cursor} must be made. Any modification that has completed before the construction of a cursor must be
  * visible, but any later concurrent modifications may be presented fully, partially or not at all; this also means that
  * if multiple are made, the cursor may see any part of any subset of them.
- *
+ * <p>
  * Note: This model only supports depth-first traversals. We do not currently have a need for breadth-first walks.
- *
+ * <p>
  * See Trie.md for further description of the trie representation model.
  *
  * @param <T> The content type of the trie.
@@ -61,33 +62,38 @@ public abstract class Trie<T>
 {
     /**
      * A trie cursor.
-     *
+     * <p>
      * This is the internal representation of the trie, which enables efficient walks and basic operations (merge,
      * slice) on tries.
-     *
-     * The cursor represents the state of a walk over the nodes of trie. It provides three main features:
-     * - the current "depth" or descend-depth in the trie;
-     * - the "incomingTransition", i.e. the byte that was used to reach the current point;
-     * - the "content" associated with the current node,
+     * <p>
+     * The cursor represents the state of a walk over the nodes of trie. It provides three main features:<ul>
+     * <li>the current {@code depth} or descend-depth in the trie;</li>
+     * <li>the {@code incomingTransition}, i.e. the byte that was used to reach the current point;</li>
+     * <li>the {@code content} associated with the current node,</li>
+     * </ul>
      * and provides methods for advancing to the next position.  This is enough information to extract all paths, and
      * also to easily compare cursors over different tries that are advanced together. Advancing is always done in
      * order; if one imagines the set of nodes in the trie with their associated paths, a cursor may only advance from a
-     * node with a lexicographically smaller path to one with bigger. The "advance" operation moves to the immediate
-     * next, it is also possible to skip over some items e.g. all children of the current node ("skipChildren").
-     *
-     * Moving to the immediate next position in the lexicographic order is accomplished by:
-     * - if the current node has children, moving to its first child;
-     * - otherwise, ascend the parent chain and return the next child of the closest parent that still has any.
+     * node with a lexicographically smaller path to one with bigger. The {@code advance} operation moves to the immediate
+     * next, it is also possible to skip over some items e.g. all children of the current node ({@code skipChildren}).
+     * <p>
+     * Moving to the immediate next position in the lexicographic order is accomplished by:<ul>
+     * <li>if the current node has children, moving to its first child;</li>
+     * <li>otherwise, ascend the parent chain and return the next child of the closest parent that still has any.</li>
+     * </ul>
      * As long as the trie is not exhausted, advancing always takes one step down, from the current node, or from a node
-     * on the parent chain. By comparing the new depth (which "advance" also returns) with the one before the advance,
-     * one can tell if the former was the case (if newDepth == oldDepth + 1) and how many steps up we had to take
-     * (oldDepth + 1 - newDepth). When following a path down, the cursor will stop on all prefixes.
-     *
-     * When it is created the cursor is placed on the root node with depth() = 0, incomingTransition() = -1. Since
-     * tries can have mappings for empty, content() can possibly be non-null. It is not allowed for a cursor to start
-     * in exhausted state (i.e. with depth() = -1).
-     *
-     * For example, the following trie:
+     * on the parent chain. By comparing the new depth (which {@code advance} also returns) with the one before the advance,
+     * one can tell if the former was the case (if {@code newDepth == oldDepth + 1}) and how many steps up we had to take
+     * ({@code oldDepth + 1 - newDepth}). When following a path down, the cursor will stop on all prefixes.
+     * <p>
+     * When it is created the cursor is placed on the root node with {@code depth() = 0}, {@code incomingTransition() = -1}.
+     * Since tries can have mappings for empty, content() can possibly be non-null. The cursor is exhausted when it
+     * returns a depth of -1 (the operations that advance a cursor return the depth, and {@code depth()} will also
+     * return -1 if queried afterwards). It is not allowed for a cursor to start in exhausted state; once a cursor is
+     * exhausted, calling any of the advance methods or {@code tailTrie} is an error.
+     * <p>
+     * For example, the following trie:<br/>
+     * <pre>
      *  t
      *   r
      *    e
@@ -98,17 +104,20 @@ public abstract class Trie<T>
      *  w
      *   i
      *    n  *
-     * has nodes reachable with the paths
-     *  "", t, tr, tre, tree*, tri, trie*, trip*, w, wi, win*
-     * and the cursor will list them with the following (depth, incomingTransition) pairs:
-     *  (0, -1), (1, t), (2, r), (3, e), (4, e)*, (3, i), (4, e)*, (4, p)*, (1, w), (2, i), (3, n)*
-     *
+     * </pre>
+     * has nodes reachable with the paths<br/>
+     * &nbsp; "", t, tr, tre, tree*, tri, trie*, trip*, w, wi, win*<br/>
+     * and the cursor will list them with the following {@code (depth, incomingTransition)} pairs:<br/>
+     * &nbsp; (0, -1), (1, t), (2, r), (3, e), (4, e)*, (3, i), (4, e)*, (4, p)*, (1, w), (2, i), (3, n)*
+     * <p>
      * Because we exhaust transitions on bigger depths before we go the next transition on the smaller ones, when
-     * cursors are advanced together their positions can be easily compared using only the depth and incomingTransition:
-     * - one that is higher in depth is before one that is lower;
-     * - for equal depths, the one with smaller incomingTransition is first.
-     *
-     * If we consider walking the trie above in parallel with this:
+     * cursors are advanced together their positions can be easily compared using only the {@code depth} and
+     * {@code incomingTransition}:<ul>
+     * <li>one that is higher in depth is before one that is lower;</li>
+     * <li>for equal depths, the one with smaller incomingTransition is first.</li>
+     * </ul>
+     * If we consider walking the trie above in parallel with this:<br/>
+     * <pre>
      *  t
      *   r
      *    i
@@ -116,23 +125,31 @@ public abstract class Trie<T>
      *      k *
      *  u
      *   p *
-     * the combined iteration will proceed as follows:
-     *  (0, -1)+    (0, -1)+               cursors equal, advance both
-     *  (1, t)+     (1, t)+        t       cursors equal, advance both
-     *  (2, r)+     (2, r)+        tr      cursors equal, advance both
-     *  (3, e)+  <  (3, i)         tre     cursors not equal, advance smaller (3 = 3, e < i)
-     *  (4, e)+  <  (3, i)         tree*   cursors not equal, advance smaller (4 > 3)
-     *  (3, i)+     (3, i)+        tri     cursors equal, advance both
-     *  (4, e)   >  (4, c)+        tric    cursors not equal, advance smaller (4 = 4, e > c)
-     *  (4, e)   >  (5, k)+        trick*  cursors not equal, advance smaller (4 < 5)
-     *  (4, e)+  <  (1, u)         trie*   cursors not equal, advance smaller (4 > 1)
-     *  (4, p)+  <  (1, u)         trip*   cursors not equal, advance smaller (4 > 1)
-     *  (1, w)   >  (1, u)+        u       cursors not equal, advance smaller (1 = 1, w > u)
-     *  (1, w)   >  (2, p)+        up*     cursors not equal, advance smaller (1 < 2)
-     *  (1, w)+  <  (-1, -1)       w       cursors not equal, advance smaller (1 > -1)
-     *  (2, i)+  <  (-1, -1)       wi      cursors not equal, advance smaller (2 > -1)
-     *  (3, n)+  <  (-1, -1)       win*    cursors not equal, advance smaller (3 > -1)
-     *  (-1, -1)    (-1, -1)               both exhasted
+     * </pre>
+     * the combined iteration will proceed as follows:<pre>
+     *  (0, -1)+  (0, -1)+          cursors equal, advance both
+     *  (1, t)+   (1, t)+   t       cursors equal, advance both
+     *  (2, r)+   (2, r)+   tr      cursors equal, advance both
+     *  (3, e)+ < (3, i)    tre     cursors not equal, advance smaller (3 = 3, e < i)
+     *  (4, e)+ < (3, i)    tree*   cursors not equal, advance smaller (4 > 3)
+     *  (3, i)+   (3, i)+   tri     cursors equal, advance both
+     *  (4, e)  > (4, c)+   tric    cursors not equal, advance smaller (4 = 4, e > c)
+     *  (4, e)  > (5, k)+   trick*  cursors not equal, advance smaller (4 < 5)
+     *  (4, e)+ < (1, u)    trie*   cursors not equal, advance smaller (4 > 1)
+     *  (4, p)+ < (1, u)    trip*   cursors not equal, advance smaller (4 > 1)
+     *  (1, w)  > (1, u)+   u       cursors not equal, advance smaller (1 = 1, w > u)
+     *  (1, w)  > (2, p)+   up*     cursors not equal, advance smaller (1 < 2)
+     *  (1, w)+ < (-1, -1)  w       cursors not equal, advance smaller (1 > -1)
+     *  (2, i)+ < (-1, -1)  wi      cursors not equal, advance smaller (2 > -1)
+     *  (3, n)+ < (-1, -1)  win*    cursors not equal, advance smaller (3 > -1)
+     *  (-1, -1)  (-1, -1)          both exhasted
+     *  </pre>
+     * <p>
+     * Cursors are created with a direction (forward or reverse), which specifies the order in which a node's children
+     * are iterated (smaller first or larger first). Note that entries returned in reverse direction are in
+     * lexicographic order for the inverted alphabet, which is not the same as being presented in reverse. For example,
+     * a cursor for a trie containing "ab", "abc" and "cba", will visit the nodes in order "cba", "ab", "abc", i.e.
+     * prefixes will still be reported before their descendants.
      */
     protected interface Cursor<T>
     {
@@ -154,13 +171,23 @@ protected interface Cursor<T>
          */
         T content();
 
+        /**
+         * Returns the direction in which this cursor is progressing.
+         */
+        Direction direction();
+
+        /**
+         * Returns the byte-comparable version that this trie uses.
+         */
+        ByteComparable.Version byteComparableVersion();
+
         /**
          * Advance one position to the node whose associated path is next lexicographically.
-         * This can be either:
-         * - descending one level to the first child of the current node,
-         * - ascending to the closest parent that has remaining children, and then descending one level to its next
+         * This can be either:<ul>
+         * <li>descending one level to the first child of the current node,
+         * <li>ascending to the closest parent that has remaining children, and then descending one level to its next
          *   child.
-         *
+         * </ul>
          * It is an error to call this after the trie has already been exhausted (i.e. when depth() == -1);
          * for performance reasons we won't always check this.
          *
@@ -173,11 +200,11 @@ protected interface Cursor<T>
          * (e.g. when positioned on a chain node in a memtable trie). If the current node does not have children this
          * is exactly the same as advance(), otherwise it may take multiple steps down (but will not necessarily, even
          * if they exist).
-         *
+         * <p>
          * Note that if any positions are skipped, their content must be null.
-         *
+         * <p>
          * This is an optional optimization; the default implementation falls back to calling advance.
-         *
+         * <p>
          * It is an error to call this after the trie has already been exhausted (i.e. when depth() == -1);
          * for performance reasons we won't always check this.
          *
@@ -192,7 +219,7 @@ default int advanceMultiple(TransitionsReceiver receiver)
 
         /**
          * Advance all the way to the next node with non-null content.
-         *
+         * <p>
          * It is an error to call this after the trie has already been exhausted (i.e. when depth() == -1);
          * for performance reasons we won't always check this.
          *
@@ -221,18 +248,45 @@ default T advanceToContent(ResettingTransitionsReceiver receiver)
         }
 
         /**
-         * Ignore the current node's children and advance to the next child of the closest node on the parent chain that
-         * has any.
-         *
-         * It is an error to call this after the trie has already been exhausted (i.e. when depth() == -1);
-         * for performance reasons we won't always check this.
+         * Advance to the specified depth and incoming transition or the first valid position that is after the specified
+         * position. The inputs must be something that could be returned by a single call to {@link #advance} (i.e.
+         * {@code depth} must be <= current depth + 1, and {@code incomingTransition} must be higher than what the
+         * current state saw at the requested depth.
          *
          * @return the new depth, always <= previous depth; -1 if the trie is exhausted
          */
-        int skipChildren();
+        int skipTo(int skipDepth, int skipTransition);
+
+        /**
+         * Descend into the cursor with the given path.
+         *
+         * @return True if the descent is positioned at the end of the given path, false if the trie did not have a path
+         * for it. In the latter case the cursor is positioned at the first node that follows the given key in iteration
+         * order.
+         */
+        default boolean descendAlong(ByteSource bytes)
+        {
+            int next = bytes.next();
+            int depth = depth();
+            while (next != ByteSource.END_OF_STREAM)
+            {
+                if (skipTo(++depth, next) != depth || incomingTransition() != next)
+                    return false;
+                next = bytes.next();
+            }
+            return true;
+        }
+
+        /**
+         * Returns a tail trie, i.e. a trie whose root is the current position. Walking a tail trie will list all
+         * descendants of the current position with depth adjusted by the current depth.
+         * <p>
+         * It is an error to call tailTrie on an exhausted cursor.
+         */
+        Trie<T> tailTrie();
     }
 
-    protected abstract Cursor<T> cursor();
+    protected abstract Cursor<T> cursor(Direction direction);
 
     /**
      * Used by {@link Cursor#advanceMultiple} to feed the transitions taken.
@@ -269,9 +323,6 @@ protected interface Walker<T, R> extends ResettingTransitionsReceiver
         R complete();
     }
 
-    // Version of the byte comparable conversion to use for all operations
-    protected static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS50;
-
     /**
      * Adapter interface providing the methods a {@link Walker} to a {@link Consumer}, so that the latter can be used
      * with {@link #process}.
@@ -318,15 +369,24 @@ default void addPathBytes(DirectBuffer buffer, int pos, int count)
      */
     public void forEachValue(ValueConsumer<T> consumer)
     {
-        process(consumer);
+        process(consumer, Direction.FORWARD);
     }
 
     /**
      * Call the given consumer on all (path, content) pairs with non-null content in the trie in order.
      */
-    public void forEachEntry(BiConsumer<ByteComparable, T> consumer)
+    public void forEachEntry(BiConsumer<ByteComparable.Preencoded, T> consumer)
     {
-        process(new TrieEntriesWalker.WithConsumer<T>(consumer));
+        forEachEntry(Direction.FORWARD, consumer);
+    }
+
+    /**
+     * Call the given consumer on all (path, content) pairs with non-null content in the trie in order.
+     */
+    public void forEachEntry(Direction direction, BiConsumer<ByteComparable.Preencoded, T> consumer)
+    {
+        Cursor<T> cursor = cursor(direction);
+        process(new TrieEntriesWalker.WithConsumer<T>(consumer, cursor.byteComparableVersion()), cursor);
         // Note: we can't do the ValueConsumer trick here, because the implementation requires state and cannot be
         // implemented with default methods alone.
     }
@@ -334,9 +394,9 @@ public void forEachEntry(BiConsumer<ByteComparable, T> consumer)
     /**
      * Process the trie using the given Walker.
      */
-    public <R> R process(Walker<T, R> walker)
+    public <R> R process(Walker<T, R> walker, Direction direction)
     {
-        return process(walker, cursor());
+        return process(walker, cursor(direction));
     }
 
     static <T, R> R process(Walker<T, R> walker, Cursor<T> cursor)
@@ -354,6 +414,72 @@ static <T, R> R process(Walker<T, R> walker, Cursor<T> cursor)
         return walker.complete();
     }
 
+
+    /**
+     * Process the trie using the given ValueConsumer, skipping all branches below the top content-bearing node.
+     */
+    public Void forEachValueSkippingBranches(Direction direction, ValueConsumer<T> consumer)
+    {
+        return processSkippingBranches(consumer, cursor(direction));
+    }
+
+    /**
+     * Call the given consumer on all (path, content) pairs with non-null content in the trie in order, skipping all
+     * branches below the top content-bearing node.
+     */
+    public void forEachEntrySkippingBranches(Direction direction, BiConsumer<ByteComparable.Preencoded, T> consumer)
+    {
+        Cursor<T> cursor = cursor(direction);
+        processSkippingBranches(new TrieEntriesWalker.WithConsumer<T>(consumer, cursor.byteComparableVersion()), cursor);
+        // Note: we can't do the ValueConsumer trick here, because the implementation requires state and cannot be
+        // implemented with default methods alone.
+    }
+
+    /**
+     * Process the trie using the given Walker, skipping all branches below the top content-bearing node.
+     */
+    public <R> R processSkippingBranches(Walker<T, R> walker, Direction direction)
+    {
+        return processSkippingBranches(walker, cursor(direction));
+    }
+
+    static <T, R> R processSkippingBranches(Walker<T, R> walker, Cursor<T> cursor)
+    {
+        assert cursor.depth() == 0 : "The provided cursor has already been advanced.";
+        T content = cursor.content();   // handle content on the root node
+        if (content != null)
+        {
+            walker.content(content);
+            return walker.complete();
+        }
+        content = cursor.advanceToContent(walker);
+
+        while (content != null)
+        {
+            walker.content(content);
+            if (cursor.skipTo(cursor.depth(), cursor.incomingTransition() + cursor.direction().increase) < 0)
+                break;
+            walker.resetPathLength(cursor.depth() - 1);
+            walker.addPathByte(cursor.incomingTransition());
+            content = cursor.content();
+            if (content == null)
+                content = cursor.advanceToContent(walker);
+        }
+        return walker.complete();
+    }
+
+    /**
+     * Map-like get by key.
+     */
+    public T get(ByteComparable key)
+    {
+        Cursor<T> cursor = cursor(Direction.FORWARD);
+        if (cursor.descendAlong(key.asComparableBytes(cursor.byteComparableVersion())))
+            return cursor.content();
+        else
+            return null;
+    }
+
     /**
      * Constuct a textual representation of the trie.
      */
@@ -367,15 +493,15 @@ public String dump()
      */
     public String dump(Function<T, String> contentToString)
     {
-        return process(new TrieDumper<>(contentToString));
+        return process(new TrieDumper<>(contentToString), Direction.FORWARD);
     }
 
     /**
      * Returns a singleton trie mapping the given byte path to content.
      */
-    public static <T> Trie<T> singleton(ByteComparable b, T v)
+    public static <T> Trie<T> singleton(ByteComparable b, ByteComparable.Version byteComparableVersion, T v)
     {
-        return new SingletonTrie<>(b, v);
+        return new SingletonTrie<>(b, byteComparableVersion, v);
     }
 
     /**
@@ -400,19 +526,15 @@ public Trie<T> subtrie(ByteComparable left, boolean includeLeft, ByteComparable
     }
 
     /**
-     * Returns a view of the subtrie containing everything in this trie whose keys fall between the given boundaries,
-     * left-inclusive and right-exclusive.
+     * Returns a view of the subtrie containing everything in this trie whose keys fall between the given boundaries.
      * The view is live, i.e. any write to the source will be reflected in the subtrie.
      *
-     * This method will not check its arguments for correctness. The resulting trie may be empty or throw an exception
-     * if the right bound is smaller than the left.
-     *
-     * Equivalent to calling subtrie(left, true, right, false).
-     *
-     * @param left the left bound for the returned subtrie. If {@code null}, the resulting subtrie is not left-bounded.
-     * @param right the right bound for the returned subtrie. If {@code null}, the resulting subtrie is not right-bounded.
-     * @return a view of the subtrie containing all the keys of this trie falling between {@code left} (inclusively if
-     * {@code includeLeft}) and {@code right} (inclusively if {@code includeRight}).
+     * @param left the left bound for the returned subtrie, inclusive. If {@code null}, the resulting subtrie is not
+     *             left-bounded.
+     * @param right the right bound for the returned subtrie, exclusive. If {@code null}, the resulting subtrie is not
+     *              right-bounded.
+     * @return a view of the subtrie containing all the keys of this trie falling between {@code left} inclusively and
+     * {@code right} exclusively.
      */
     public Trie<T> subtrie(ByteComparable left, ByteComparable right)
     {
@@ -422,17 +544,57 @@ public Trie<T> subtrie(ByteComparable left, ByteComparable right)
     /**
      * Returns the ordered entry set of this trie's content as an iterable.
      */
-    public Iterable<Map.Entry<ByteComparable, T>> entrySet()
+    public Iterable<Map.Entry<ByteComparable.Preencoded, T>> entrySet()
     {
         return this::entryIterator;
     }
 
+    /**
+     * Returns the ordered entry set of this trie's content as an iterable.
+     */
+    public Iterable<Map.Entry<ByteComparable.Preencoded, T>> entrySet(Direction direction)
+    {
+        return () -> entryIterator(direction);
+    }
+
+    /**
+     * Returns the ordered entry set of this trie's content in an iterator.
+     */
+    public Iterator<Map.Entry<ByteComparable.Preencoded, T>> entryIterator()
+    {
+        return entryIterator(Direction.FORWARD);
+    }
+
     /**
      * Returns the ordered entry set of this trie's content in an iterator.
      */
-    public Iterator<Map.Entry<ByteComparable, T>> entryIterator()
+    public Iterator<Map.Entry<ByteComparable.Preencoded, T>> entryIterator(Direction direction)
     {
-        return new TrieEntriesIterator.AsEntries<>(this);
+        return new TrieEntriesIterator.AsEntries<>(cursor(direction));
+    }
+
+    /**
+     * Returns the ordered entry set of this trie's content in an iterable, filtered by the given type.
+     */
+    public <U extends T> Iterable<Map.Entry<ByteComparable.Preencoded, U>> filteredEntrySet(Class<U> clazz)
+    {
+        return filteredEntrySet(Direction.FORWARD, clazz);
+    }
+
+    /**
+     * Returns the ordered entry set of this trie's content in an iterable, filtered by the given type.
+     */
+    public <U extends T> Iterable<Map.Entry<ByteComparable.Preencoded, U>> filteredEntrySet(Direction direction, Class<U> clazz)
+    {
+        return () -> filteredEntryIterator(direction, clazz);
+    }
+
+    /**
+     * Returns the ordered entry set of this trie's content in an iterator, filtered by the given type.
+     */
+    public <U extends T> Iterator<Map.Entry<ByteComparable.Preencoded, U>> filteredEntryIterator(Direction direction, Class<U> clazz)
+    {
+        return new TrieEntriesIterator.AsEntriesFilteredByType<>(cursor(direction), clazz);
     }
 
     /**
@@ -443,12 +605,60 @@ public Iterable<T> values()
         return this::valueIterator;
     }
 
+    /**
+     * Returns the ordered set of values of this trie as an iterable.
+     */
+    public Iterable<T> values(Direction direction)
+    {
+        return direction.isForward() ? this::valueIterator : this::reverseValueIterator;
+    }
+
     /**
      * Returns the ordered set of values of this trie in an iterator.
      */
     public Iterator<T> valueIterator()
     {
-        return new TrieValuesIterator<>(this);
+        return valueIterator(Direction.FORWARD);
+    }
+
+    /**
+     * Returns the inversely ordered set of values of this trie in an iterator.
+     */
+    public Iterator<T> reverseValueIterator()
+    {
+        return valueIterator(Direction.REVERSE);
+    }
+
+    /**
+     * Returns the ordered set of values of this trie in an iterator.
+     */
+    public Iterator<T> valueIterator(Direction direction)
+    {
+        return new TrieValuesIterator<>(cursor(direction));
+    }
+
+    /**
+     * Returns the ordered set of values of this trie in an iterable, filtered by the given type.
+     */
+    public <U extends T> Iterable<U> filteredValues(Class<U> clazz)
+    {
+        return filteredValues(Direction.FORWARD, clazz);
+    }
+
+    /**
+     * Returns the ordered set of values of this trie in an iterable, filtered by the given type.
+     */
+    public <U extends T> Iterable<U> filteredValues(Direction direction, Class<U> clazz)
+    {
+        return () -> filteredValuesIterator(direction, clazz);
+    }
+
+    /**
+     * Returns the ordered set of values of this trie in an iterator, filtered by the given type.
+     */
+    public <U extends T> Iterator<U> filteredValuesIterator(Direction direction, Class<U> clazz)
+    {
+        return new TrieValuesIterator.FilteredByType<>(cursor(direction), clazz);
     }
 
     /**
@@ -537,7 +747,7 @@ public static <T> Trie<T> merge(Collection<? extends Trie<T>> sources, Collectio
         switch (sources.size())
         {
         case 0:
-            return empty();
+            throw new AssertionError();
         case 1:
             return sources.iterator().next();
         case 2:
@@ -563,7 +773,7 @@ public static <T> Trie<T> mergeDistinct(Collection<? extends Trie<T>> sources)
         switch (sources.size())
         {
         case 0:
-            return empty();
+            throw new AssertionError();
         case 1:
             return sources.iterator().next();
         case 2:
@@ -578,45 +788,107 @@ public static <T> Trie<T> mergeDistinct(Collection<? extends Trie<T>> sources)
         }
     }
 
-    private static final Trie<Object> EMPTY = new Trie<Object>()
+    /**
+     * Returns a Trie that is a view of this one, where the given prefix is prepended before the root.
+     */
+    public Trie<T> prefixedBy(ByteComparable prefix)
+    {
+        return new PrefixedTrie(prefix, this);
+    }
+
+    /**
+     * Returns an entry set containing all tail tree constructed at the points that contain content of
+     * the given type.
+     */
+    public Iterable<Map.Entry<ByteComparable.Preencoded, Trie<T>>> tailTries(Direction direction, Class<? extends T> clazz)
+    {
+        return () -> new TrieTailsIterator.AsEntries<>(cursor(direction), clazz);
+    }
+
+    /**
+     * Returns a trie that corresponds to the branch of this trie rooted at the given prefix.
+     * <p>
+     * The result will include the same values as {@code subtrie(prefix, nextBranch(prefix))}, but the keys in the
+     * resulting trie will not include the prefix. In other words,
+     *   {@code tailTrie(prefix).prefixedBy(prefix) = subtrie(prefix, nextBranch(prefix))}
+     * where nextBranch stands for the key adjusted by adding one at the last position.
+     */
+    public Trie<T> tailTrie(ByteComparable prefix)
     {
-        protected Cursor<Object> cursor()
+        Cursor<T> c = cursor(Direction.FORWARD);
+        if (c.descendAlong(prefix.asComparableBytes(c.byteComparableVersion())))
+            return c.tailTrie();
+        else
+            return null;
+    }
+
+    public static <T> Trie<T> empty(ByteComparable.Version byteComparableVersion)
+    {
+        return new Trie<T>()
         {
-            return new Cursor<Object>()
+            public Cursor<T> cursor(Direction dir)
             {
-                int depth = 0;
+                return new EmptyCursor<>(dir, byteComparableVersion);
+            }
+        };
+    }
 
-                public int advance()
-                {
-                    return depth = -1;
-                }
+    static class EmptyCursor<T> implements Cursor<T>
+    {
+        private final Direction direction;
+        private final ByteComparable.Version byteComparableVersion;
+        int depth;
 
-                public int skipChildren()
-                {
-                    return depth = -1;
-                }
+        public EmptyCursor(Direction direction, ByteComparable.Version byteComparableVersion)
+        {
+            this.direction = direction;
+            this.byteComparableVersion = byteComparableVersion;
+            depth = 0;
+        }
 
-                public int depth()
-                {
-                    return depth;
-                }
+        public int advance()
+        {
+            return depth = -1;
+        }
 
-                public Object content()
-                {
-                    return null;
-                }
+        public int skipTo(int skipDepth, int skipTransition)
+        {
+            return depth = -1;
+        }
 
-                public int incomingTransition()
-                {
-                    return -1;
-                }
-            };
+        public ByteComparable.Version byteComparableVersion()
+        {
+            if (byteComparableVersion != null)
+                return byteComparableVersion;
+            throw new AssertionError();
         }
-    };
 
-    @SuppressWarnings("unchecked")
-    public static <T> Trie<T> empty()
-    {
-        return (Trie<T>) EMPTY;
+        @Override
+        public Trie<T> tailTrie()
+        {
+            assert depth == 0 : "tailTrie called on exhausted cursor";
+            return empty(byteComparableVersion);
+        }
+
+        public int depth()
+        {
+            return depth;
+        }
+
+        public T content()
+        {
+            return null;
+        }
+
+        @Override
+        public Direction direction()
+        {
+            return direction;
+        }
+
+        public int incomingTransition()
+        {
+            return -1;
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/db/tries/Trie.md b/src/java/org/apache/cassandra/db/tries/Trie.md
index 4265871e7b9b..a482d7dc4a80 100644
--- a/src/java/org/apache/cassandra/db/tries/Trie.md
+++ b/src/java/org/apache/cassandra/db/tries/Trie.md
@@ -248,5 +248,15 @@ as soon as the source becomes larger than the right bound.
 implicit representation using a pair of `depth` and `incomingTransition` for each bound.
 
 In slices we can also use `advanceMultiple` when we are certain to be strictly inside the slice, i.e. beyond the
-left bound and before a prefix of the right bound. As above, descending to any depth in this case is safe as the
-result will remain smaller than the right bound.
\ No newline at end of file
+left bound and before the right bound. As above, descending to any depth in this case is safe as the
+result will remain smaller than the right bound.
+
+## Reverse iteration
+
+Tries and trie cursors support reverse iteration. Reverse trie iteration presents data in lexicographic order 
+using the inverted alphabet. This is not always the same as the reverse order of the data returned in the forward 
+direction; the latter is only guaranteed if the entries in the trie can contain no prefixes (i.e. the representation 
+is prefix-free like the byte-ordered type translations).
+
+This difference is imposed by the cursor interfaces which necessarily have to present parent nodes before their 
+children and do not preserve or present any state on ascent.
diff --git a/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java b/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java
index 7ab3e7de4628..99e3f764244d 100644
--- a/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java
+++ b/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java
@@ -20,6 +20,9 @@
 import java.util.AbstractMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.function.Predicate;
+
+import com.google.common.base.Predicates;
 
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
@@ -30,23 +33,33 @@
 public abstract class TrieEntriesIterator<T, V> extends TriePathReconstructor implements Iterator<V>
 {
     private final Trie.Cursor<T> cursor;
+    private final Predicate<T> predicate;
     T next;
     boolean gotNext;
 
-    protected TrieEntriesIterator(Trie<T> trie)
+    protected TrieEntriesIterator(Trie<T> trie, Direction direction, Predicate<T> predicate)
     {
-        cursor = trie.cursor();
+        this(trie.cursor(direction), predicate);
+    }
+
+    TrieEntriesIterator(Trie.Cursor<T> cursor, Predicate<T> predicate)
+    {
+        this.cursor = cursor;
+        this.predicate = predicate;
         assert cursor.depth() == 0;
         next = cursor.content();
-        gotNext = next != null;
+        gotNext = next != null && predicate.test(next);
     }
 
     public boolean hasNext()
     {
-        if (!gotNext)
+        while (!gotNext)
         {
             next = cursor.advanceToContent(this);
-            gotNext = true;
+            if (next != null)
+                gotNext = predicate.test(next);
+            else
+                gotNext = true;
         }
 
         return next != null;
@@ -54,33 +67,59 @@ public boolean hasNext()
 
     public V next()
     {
+        if (!hasNext())
+            throw new IllegalStateException("next without hasNext");
+
         gotNext = false;
         T v = next;
         next = null;
         return mapContent(v, keyBytes, keyPos);
     }
 
+    ByteComparable.Version byteComparableVersion()
+    {
+        return cursor.byteComparableVersion();
+    }
+
     protected abstract V mapContent(T content, byte[] bytes, int byteLength);
 
     /**
      * Iterator representing the content of the trie a sequence of (path, content) pairs.
      */
-    static class AsEntries<T> extends TrieEntriesIterator<T, Map.Entry<ByteComparable, T>>
+    static class AsEntries<T> extends TrieEntriesIterator<T, Map.Entry<ByteComparable.Preencoded, T>>
+    {
+        public AsEntries(Trie.Cursor<T> cursor)
+        {
+            super(cursor, Predicates.alwaysTrue());
+        }
+
+        @Override
+        protected Map.Entry<ByteComparable.Preencoded, T> mapContent(T content, byte[] bytes, int byteLength)
+        {
+            return toEntry(byteComparableVersion(), content, bytes, byteLength);
+        }
+    }
+
+    /**
+     * Iterator representing the content of the trie a sequence of (path, content) pairs.
+     */
+    static class AsEntriesFilteredByType<T, U extends T> extends TrieEntriesIterator<T, Map.Entry<ByteComparable.Preencoded, U>>
     {
-        public AsEntries(Trie<T> trie)
+        public AsEntriesFilteredByType(Trie.Cursor<T> cursor, Class<U> clazz)
         {
-            super(trie);
+            super(cursor, clazz::isInstance);
         }
 
         @Override
-        protected Map.Entry<ByteComparable, T> mapContent(T content, byte[] bytes, int byteLength)
+        @SuppressWarnings("unchecked")  // checked by the predicate
+        protected Map.Entry<ByteComparable.Preencoded, U> mapContent(T content, byte[] bytes, int byteLength)
         {
-            return toEntry(content, bytes, byteLength);
+            return toEntry(byteComparableVersion(), (U) content, bytes, byteLength);
         }
     }
 
-    static <T> java.util.Map.Entry<ByteComparable, T> toEntry(T content, byte[] bytes, int byteLength)
+    static <T> java.util.Map.Entry<ByteComparable.Preencoded, T> toEntry(ByteComparable.Version version, T content, byte[] bytes, int byteLength)
     {
-        return new AbstractMap.SimpleImmutableEntry<>(toByteComparable(bytes, byteLength), content);
+        return new AbstractMap.SimpleImmutableEntry<>(toByteComparable(version, bytes, byteLength), content);
     }
 }
diff --git a/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java b/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java
index ca06015733e0..362fe8f112b7 100644
--- a/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java
+++ b/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java
@@ -40,17 +40,19 @@ public void content(T content)
      */
     static class WithConsumer<T> extends TrieEntriesWalker<T, Void>
     {
-        private final BiConsumer<ByteComparable, T> consumer;
+        private final BiConsumer<ByteComparable.Preencoded, T> consumer;
+        private final ByteComparable.Version byteComparableVersion;
 
-        public WithConsumer(BiConsumer<ByteComparable, T> consumer)
+        public WithConsumer(BiConsumer<ByteComparable.Preencoded, T> consumer, ByteComparable.Version byteComparableVersion)
         {
             this.consumer = consumer;
+            this.byteComparableVersion = byteComparableVersion;
         }
 
         @Override
         protected void content(T content, byte[] bytes, int byteLength)
         {
-            consumer.accept(toByteComparable(bytes, byteLength), content);
+            consumer.accept(toByteComparable(byteComparableVersion, bytes, byteLength), content);
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java b/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java
index 4a9883fa006a..c59d126fe272 100644
--- a/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java
+++ b/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java
@@ -49,8 +49,9 @@ public void resetPathLength(int newLength)
         keyPos = newLength;
     }
 
-    static ByteComparable toByteComparable(byte[] bytes, int byteLength)
+    static ByteComparable.Preencoded toByteComparable(ByteComparable.Version byteComparableVersion, byte[] bytes, int byteLength)
     {
-        return ByteComparable.fixedLength(Arrays.copyOf(bytes, byteLength));
+        // Taking a copy here to make sure it does not get modified when the cursor advances.
+        return ByteComparable.preencoded(byteComparableVersion, Arrays.copyOf(bytes, byteLength));
     }
 }
diff --git a/src/java/org/apache/cassandra/db/tries/TrieSpaceExhaustedException.java b/src/java/org/apache/cassandra/db/tries/TrieSpaceExhaustedException.java
new file mode 100644
index 000000000000..d355a467ef29
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/TrieSpaceExhaustedException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+/**
+ * Because we use buffers and 32-bit pointers, the trie cannot grow over 2GB of size. This exception is thrown if
+ * a trie operation needs it to grow over that limit.
+ * <p>
+ * To avoid this problem, users should query {@link InMemoryTrie#reachedAllocatedSizeThreshold} from time to time. If
+ * the call returns true, they should switch to a new trie (e.g. by flushing a memtable) as soon as possible. The
+ * threshold is configurable, and is set by default to 10% under the 2GB limit to give ample time for the switch to
+ * happen.
+ */
+public class TrieSpaceExhaustedException extends Exception
+{
+    public TrieSpaceExhaustedException()
+    {
+        super("The hard 2GB limit on trie size has been exceeded");
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/TrieTailsIterator.java b/src/java/org/apache/cassandra/db/tries/TrieTailsIterator.java
new file mode 100644
index 000000000000..e15ce6548206
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/TrieTailsIterator.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.AbstractMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Iterator of trie entries that constructs tail tries for the content-bearing branches that satisfy the given predicate
+ * and skips over the returned branches.
+ */
+public abstract class TrieTailsIterator<T, V> extends TriePathReconstructor implements Iterator<V>
+{
+    final Trie.Cursor<T> cursor;
+    private final Predicate<T> predicate;
+    private T next;
+    private boolean gotNext;
+
+    protected TrieTailsIterator(Trie<T> trie, Direction direction, Predicate<T> predicate)
+    {
+        this.cursor = trie.cursor(direction);
+        this.predicate = predicate;
+        assert cursor.depth() == 0;
+    }
+
+    TrieTailsIterator(Trie.Cursor<T> cursor, Predicate<T> predicate)
+    {
+        this.cursor = cursor;
+        this.predicate = predicate;
+        assert cursor.depth() == 0;
+    }
+
+    public boolean hasNext()
+    {
+        if (!gotNext)
+        {
+            int depth = cursor.depth();
+            if (depth > 0)
+            {
+                // if we are not just starting, we have returned a branch and must skip over it
+                depth = cursor.skipTo(depth, cursor.incomingTransition() + cursor.direction().increase);
+                if (depth < 0)
+                    return false;
+                resetPathLength(depth - 1);
+                addPathByte(cursor.incomingTransition());
+            }
+
+            next = cursor.content();
+            if (next != null)
+                gotNext = predicate.test(next);
+
+            while (!gotNext)
+            {
+                next = cursor.advanceToContent(this);
+                if (next != null)
+                    gotNext = predicate.test(next);
+                else
+                    gotNext = true;
+            }
+        }
+
+        return next != null;
+    }
+
+    public V next()
+    {
+        gotNext = false;
+        T v = next;
+        next = null;
+        return mapContent(v, cursor.tailTrie(), keyBytes, keyPos);
+    }
+
+    ByteComparable.Version byteComparableVersion()
+    {
+        return cursor.byteComparableVersion();
+    }
+
+    protected abstract V mapContent(T value, Trie<T> tailTrie, byte[] bytes, int byteLength);
+
+    /**
+     * Iterator representing the selected content of the trie a sequence of {@code (path, tail)} pairs, where
+     * {@code tail} is the branch of the trie rooted at the selected content node (reachable by following
+     * {@code path}). The tail trie will have the selected content at its root.
+     */
+    static class AsEntries<T> extends TrieTailsIterator<T, Map.Entry<ByteComparable.Preencoded, Trie<T>>>
+    {
+        public AsEntries(Trie.Cursor<T> cursor, Class<? extends T> clazz)
+        {
+            super(cursor, clazz::isInstance);
+        }
+
+        @Override
+        protected Map.Entry<ByteComparable.Preencoded, Trie<T>> mapContent(T value, Trie<T> tailTrie, byte[] bytes, int byteLength)
+        {
+            ByteComparable.Preencoded key = toByteComparable(byteComparableVersion(), bytes, byteLength);
+            return new AbstractMap.SimpleImmutableEntry<>(key, tailTrie);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/TrieValuesIterator.java b/src/java/org/apache/cassandra/db/tries/TrieValuesIterator.java
index 29d3642b2e60..0a99c3ff0b99 100644
--- a/src/java/org/apache/cassandra/db/tries/TrieValuesIterator.java
+++ b/src/java/org/apache/cassandra/db/tries/TrieValuesIterator.java
@@ -28,9 +28,9 @@ class TrieValuesIterator<T> implements Iterator<T>
     T next;
     boolean gotNext;
 
-    protected TrieValuesIterator(Trie<T> trie)
+    protected TrieValuesIterator(Trie.Cursor<T> cursor)
     {
-        cursor = trie.cursor();
+        this.cursor = cursor;
         assert cursor.depth() == 0;
         next = cursor.content();
         gotNext = next != null;
@@ -49,9 +49,51 @@ public boolean hasNext()
 
     public T next()
     {
+        if (!hasNext())
+            throw new IllegalStateException("next without hasNext");
+
         gotNext = false;
         T v = next;
         next = null;
         return v;
     }
+
+    static class FilteredByType<T, U> implements Iterator<U>
+    {
+        private final Trie.Cursor<T> cursor;
+        T next;
+        boolean gotNext;
+        Class<U> clazz;
+
+        FilteredByType(Trie.Cursor<T> cursor, Class<U> clazz)
+        {
+            this.cursor = cursor;
+            this.clazz = clazz;
+            assert cursor.depth() == 0;
+            next = cursor.content();
+            gotNext = next != null && clazz.isInstance(next);
+        }
+
+        public boolean hasNext()
+        {
+            while (!gotNext)
+            {
+                next = cursor.advanceToContent(null);
+                gotNext = next == null || clazz.isInstance(next);
+            }
+
+            return next != null;
+        }
+
+        public U next()
+        {
+            if (!hasNext())
+                throw new IllegalStateException("next without hasNext");
+
+            gotNext = false;
+            T v = next;
+            next = null;
+            return (U) v;
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/db/view/TableViews.java b/src/java/org/apache/cassandra/db/view/TableViews.java
index a8ca7b7f6e44..e4c4c217417b 100644
--- a/src/java/org/apache/cassandra/db/view/TableViews.java
+++ b/src/java/org/apache/cassandra/db/view/TableViews.java
@@ -17,7 +17,14 @@
  */
 package org.apache.cassandra.db.view;
 
-import java.util.*;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
@@ -27,11 +34,36 @@
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.WriteOptions;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
@@ -123,7 +155,7 @@ public void truncateBlocking(CommitLogPosition replayAfter, long truncatedAt)
         for (ColumnFamilyStore viewCfs : allViewsCfs())
         {
             viewCfs.discardSSTables(truncatedAt);
-            SystemKeyspace.saveTruncationRecord(viewCfs, truncatedAt, replayAfter);
+            SystemKeyspace.saveTruncationRecord(viewCfs.metadata.id, truncatedAt, replayAfter);
         }
     }
 
@@ -140,7 +172,7 @@ public void removeByName(String viewName)
      * @param writeCommitLog whether we should write the commit log for the view updates.
      * @param baseComplete time from epoch in ms that the local base mutation was (or will be) completed
      */
-    public void pushViewReplicaUpdates(PartitionUpdate update, boolean writeCommitLog, AtomicLong baseComplete)
+    public void pushViewReplicaUpdates(PartitionUpdate update, WriteOptions writeOptions, AtomicLong baseComplete)
     {
         assert update.metadata().id.equals(baseTableMetadata.id);
 
@@ -167,7 +199,7 @@ public void pushViewReplicaUpdates(PartitionUpdate update, boolean writeCommitLo
         Keyspace.openAndGetStore(update.metadata()).metric.viewReadTime.update(nanoTime() - start, TimeUnit.NANOSECONDS);
 
         if (!mutations.isEmpty())
-            StorageProxy.mutateMV(update.partitionKey().getKey(), mutations, writeCommitLog, baseComplete, requestTime);
+            StorageProxy.mutateMV(update.partitionKey().getKey(), mutations, writeOptions, baseComplete, requestTime);
     }
 
 
@@ -428,7 +460,7 @@ private SinglePartitionReadCommand readExistingRowsCommand(PartitionUpdate updat
         NavigableSet<Clustering<?>> names;
         try (BTree.FastBuilder<Clustering<?>> namesBuilder = sliceBuilder == null ? BTree.fastBuilder() : null)
         {
-            for (Row row : updates)
+            for (Row row : updates.rows())
             {
                 // Don't read the existing state if we can prove the update won't affect any views
                 if (!affectsAnyViews(key, row, views))
diff --git a/src/java/org/apache/cassandra/db/view/View.java b/src/java/org/apache/cassandra/db/view/View.java
index 30bad17b3460..1f07bdc29d97 100644
--- a/src/java/org/apache/cassandra/db/view/View.java
+++ b/src/java/org/apache/cassandra/db/view/View.java
@@ -17,19 +17,27 @@
  */
 package org.apache.cassandra.db.view;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.stream.Collectors;
-
 import javax.annotation.Nullable;
 
 import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.selection.RawSelector;
 import org.apache.cassandra.cql3.selection.Selectable;
+import org.apache.cassandra.cql3.statements.SelectOptions;
 import org.apache.cassandra.cql3.statements.SelectStatement;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
@@ -37,8 +45,6 @@
 import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.FBUtilities;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * A View copies data from a base table into a view table which can be queried independently from the
@@ -174,11 +180,13 @@ SelectStatement getSelectStatement()
                                                  selectClause(),
                                                  definition.whereClause,
                                                  null,
-                                                 null);
+                                                 null,
+                                                 null,
+                                                 SelectOptions.EMPTY);
 
             rawSelect.setBindVariables(Collections.emptyList());
 
-            select = rawSelect.prepare(ClientState.forInternalCalls(), true);
+            select = rawSelect.prepare(ClientState.forInternalCalls(), true, Constants.IDENTITY_STRING_MAPPER);
         }
 
         return select;
diff --git a/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java b/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java
index 9a72c1e270a2..392a3a820dca 100644
--- a/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java
+++ b/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java
@@ -41,10 +41,11 @@
 import org.apache.cassandra.db.ReadQuery;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
+import org.apache.cassandra.db.WriteOptions;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
 import org.apache.cassandra.db.rows.Rows;
@@ -63,7 +64,7 @@
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
-public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<Long>
+public class ViewBuilderTask extends AbstractTableOperation implements Callable<Long>
 {
     private static final Logger logger = LoggerFactory.getLogger(ViewBuilderTask.class);
 
@@ -114,7 +115,7 @@ private void buildKey(DecoratedKey key)
                                                        .generateViewUpdates(Collections.singleton(view), data, empty, nowInSec, true);
 
             AtomicLong noBase = new AtomicLong(Long.MAX_VALUE);
-            mutations.forEachRemaining(m -> StorageProxy.mutateMV(key.getKey(), m, true, noBase, Dispatcher.RequestTime.forImmediateExecution()));
+            mutations.forEachRemaining(m -> StorageProxy.mutateMV(key.getKey(), m, WriteOptions.FOR_VIEW_BUILD, noBase, Dispatcher.RequestTime.forImmediateExecution()));
         }
     }
 
@@ -190,12 +191,12 @@ private void finish()
             // If it's stopped due to a compaction interruption we should throw that exception.
             // Otherwise we assume that the task has been stopped due to a schema update and we can finish successfully.
             if (isCompactionInterrupted)
-                throw new StoppedException(ksName, view.name, getCompactionInfo());
+                throw new StoppedException(ksName, view.name, getProgress(), trigger());
         }
     }
 
     @Override
-    public CompactionInfo getCompactionInfo()
+    public OperationProgress getProgress()
     {
         // we don't know the sstables at construction of ViewBuilderTask and we could change this to return once we know the
         // but since we basically only cancel view builds on truncation where we cancel all compactions anyway, this seems reasonable
@@ -204,17 +205,17 @@ public CompactionInfo getCompactionInfo()
         if (range.left.getPartitioner().splitter().isPresent())
         {
             long progress = prevToken == null ? 0 : Math.round(prevToken.getPartitioner().splitter().get().positionInRange(prevToken, range) * 1000);
-            return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, Unit.RANGES, compactionId);
+            return OperationProgress.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, Unit.RANGES, compactionId);
         }
 
         // When there is no splitter, estimate based on number of total keys but
         // take the max with keysBuilt + 1 to avoid having more completed than total
         long keysTotal = Math.max(keysBuilt + 1, baseCfs.estimatedKeysForRange(range));
-        return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, Unit.KEYS, compactionId);
+        return OperationProgress.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, Unit.KEYS, compactionId);
     }
 
     @Override
-    public void stop()
+    public void stop(StopTrigger trigger)
     {
         stop(true);
     }
@@ -247,9 +248,9 @@ static class StoppedException extends CompactionInterruptedException
     {
         private final String ksName, viewName;
 
-        private StoppedException(String ksName, String viewName, CompactionInfo info)
+        private StoppedException(String ksName, String viewName, OperationProgress info, TableOperation.StopTrigger trigger)
         {
-            super(info);
+            super(info, trigger);
             this.ksName = ksName;
             this.viewName = viewName;
         }
diff --git a/src/java/org/apache/cassandra/db/view/ViewUpdateGenerator.java b/src/java/org/apache/cassandra/db/view/ViewUpdateGenerator.java
index c49d0ceb0cf5..8b71e6b3dc4b 100644
--- a/src/java/org/apache/cassandra/db/view/ViewUpdateGenerator.java
+++ b/src/java/org/apache/cassandra/db/view/ViewUpdateGenerator.java
@@ -566,16 +566,22 @@ private void submitUpdate()
             return;
 
         DecoratedKey partitionKey = makeCurrentPartitionKey();
-        // We can't really know which columns of the view will be updated nor how many row will be updated for this key
-        // so we rely on hopefully sane defaults.
         PartitionUpdate.Builder update = updates.computeIfAbsent(partitionKey,
-                                                                 k -> new PartitionUpdate.Builder(viewMetadata,
-                                                                                                  partitionKey,
-                                                                                                  viewMetadata.regularAndStaticColumns(),
-                                                                                                  4));
+                                                                 k -> builderFor(viewMetadata, partitionKey));
         update.add(row);
     }
 
+    private static PartitionUpdate.Builder builderFor(TableMetadata viewMetadata,
+                                                      DecoratedKey partitionKey)
+    {
+        // We can't really know which columns of the view will be updated nor how many row will be updated for this key
+        // so we rely on hopefully sane defaults.
+        return viewMetadata.params.memtable.factory.partitionUpdateFactory().builder(viewMetadata,
+                                                                                     partitionKey,
+                                                                                     viewMetadata.regularAndStaticColumns(),
+                                                                                     4);
+    }
+
     private DecoratedKey makeCurrentPartitionKey()
     {
         ByteBuffer rawKey = viewMetadata.partitionKeyColumns().size() == 1
diff --git a/src/java/org/apache/cassandra/db/virtual/AbstractMutableVirtualTable.java b/src/java/org/apache/cassandra/db/virtual/AbstractMutableVirtualTable.java
index 7044312eac8d..bf421af491ab 100644
--- a/src/java/org/apache/cassandra/db/virtual/AbstractMutableVirtualTable.java
+++ b/src/java/org/apache/cassandra/db/virtual/AbstractMutableVirtualTable.java
@@ -62,7 +62,7 @@ public final void apply(PartitionUpdate update)
         ColumnValues partitionKey = ColumnValues.from(metadata(), update.partitionKey());
 
         if (update.deletionInfo().isLive())
-            update.forEach(row ->
+            update.rowIterator().forEachRemaining(row ->
             {
                 ColumnValues clusteringColumns = ColumnValues.from(metadata(), row.clustering());
 
diff --git a/src/java/org/apache/cassandra/db/virtual/CachesTable.java b/src/java/org/apache/cassandra/db/virtual/CachesTable.java
index 5a265e63304a..b08b9fbdcfd4 100644
--- a/src/java/org/apache/cassandra/db/virtual/CachesTable.java
+++ b/src/java/org/apache/cassandra/db/virtual/CachesTable.java
@@ -21,6 +21,7 @@
 import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.dht.LocalPartitioner;
 import org.apache.cassandra.metrics.CacheMetrics;
+import org.apache.cassandra.metrics.ChunkCacheMetrics;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.CacheService;
 
@@ -57,14 +58,27 @@ final class CachesTable extends AbstractVirtualTable
     private void addRow(SimpleDataSet result, String name, CacheMetrics metrics)
     {
         result.row(name)
-              .column(CAPACITY_BYTES, metrics.capacity.getValue())
-              .column(SIZE_BYTES, metrics.size.getValue())
-              .column(ENTRY_COUNT, metrics.entries.getValue())
-              .column(REQUEST_COUNT, metrics.requests.getCount())
-              .column(HIT_COUNT, metrics.hits.getCount())
-              .column(HIT_RATIO, metrics.hitRate.getValue())
-              .column(RECENT_REQUEST_RATE_PER_SECOND, (long) metrics.requests.getFifteenMinuteRate())
-              .column(RECENT_HIT_RATE_PER_SECOND, (long) metrics.hits.getFifteenMinuteRate());
+              .column(CAPACITY_BYTES, metrics.capacity())
+              .column(SIZE_BYTES, metrics.size())
+              .column(ENTRY_COUNT, metrics.entries())
+              .column(REQUEST_COUNT, metrics.requests())
+              .column(HIT_COUNT, metrics.hits())
+              .column(HIT_RATIO, metrics.hitRate())
+              .column(RECENT_REQUEST_RATE_PER_SECOND, (long) metrics.requestsFifteenMinuteRate())
+              .column(RECENT_HIT_RATE_PER_SECOND, (long) metrics.hitFifteenMinuteRate());
+    }
+
+    private void addRow(SimpleDataSet result, String name, ChunkCacheMetrics metrics)
+    {
+        result.row(name)
+              .column(CAPACITY_BYTES, metrics.capacity())
+              .column(SIZE_BYTES, metrics.size())
+              .column(ENTRY_COUNT, metrics.entries())
+              .column(REQUEST_COUNT, metrics.requests())
+              .column(HIT_COUNT, metrics.hits())
+              .column(HIT_RATIO, metrics.hitRate())
+              .column(RECENT_REQUEST_RATE_PER_SECOND, metrics.requestsFifteenMinuteRate())
+              .column(RECENT_HIT_RATE_PER_SECOND, metrics.hitFifteenMinuteRate());
     }
 
     public DataSet data()
diff --git a/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java b/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
index e2f38f8e9201..e48f5aba87b9 100644
--- a/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
+++ b/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
@@ -17,8 +17,8 @@
  */
 package org.apache.cassandra.db.virtual;
 
-import org.apache.cassandra.db.compaction.CompactionInfo;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.marshal.DoubleType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.LongType;
@@ -63,22 +63,22 @@ public DataSet data()
     {
         SimpleDataSet result = new SimpleDataSet(metadata());
 
-        for (CompactionInfo task : CompactionManager.instance.getSSTableTasks())
+        for (TableOperation.Progress task : CompactionManager.instance.getSSTableTasks())
         {
-            long completed = task.getCompleted();
-            long total = task.getTotal();
+            long completed = task.completed();
+            long total = task.total();
 
             double completionRatio = total == 0L ? 1.0 : (((double) completed) / total);
 
-            result.row(task.getKeyspace().orElse("*"),
-                       task.getTable().orElse("*"),
-                       task.getTaskId())
+            result.row(task.keyspace().orElse("*"),
+                       task.table().orElse("*"),
+                       task.operationId())
                   .column(COMPLETION_RATIO, completionRatio)
-                  .column(KIND, task.getTaskType().toString().toLowerCase())
+                  .column(KIND, task.operationType().toString().toLowerCase())
                   .column(PROGRESS, completed)
-                  .column(SSTABLES, task.getSSTables().size())
+                  .column(SSTABLES, task.sstables().size())
                   .column(TOTAL, total)
-                  .column(UNIT, task.getUnit().toString().toLowerCase())
+                  .column(UNIT, task.unit().toString().toLowerCase())
                   .column(TARGET_DIRECTORY, task.targetDirectory());
         }
 
diff --git a/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java b/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
index 7d6152bdc207..7622387a01c0 100644
--- a/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
+++ b/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
@@ -19,8 +19,13 @@
 
 import com.google.common.collect.ImmutableList;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.index.sai.virtual.IndexesSystemView;
+import org.apache.cassandra.index.sai.virtual.SSTablesSystemView;
 import org.apache.cassandra.index.sai.virtual.StorageAttachedIndexTables;
 
+import java.util.Collection;
+
 import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_VIEWS;
 
 public final class SystemViewsKeyspace extends VirtualKeyspace
@@ -29,8 +34,14 @@ public final class SystemViewsKeyspace extends VirtualKeyspace
 
     private SystemViewsKeyspace()
     {
-        super(VIRTUAL_VIEWS, new ImmutableList.Builder<VirtualTable>()
-                    .add(new CachesTable(VIRTUAL_VIEWS))
+        super(VIRTUAL_VIEWS, buildTables());
+    }
+
+    private static Collection<VirtualTable> buildTables()
+    {
+        ImmutableList.Builder<VirtualTable> tables = new ImmutableList.Builder<>();
+        if (CassandraRelevantProperties.SYSTEM_VIEWS_INCLUDE_ALL.getBoolean())
+            tables.add(new CachesTable(VIRTUAL_VIEWS))
                     .add(new ClientsTable(VIRTUAL_VIEWS))
                     .add(new SettingsTable(VIRTUAL_VIEWS))
                     .add(new SystemPropertiesTable(VIRTUAL_VIEWS))
@@ -39,6 +50,7 @@ private SystemViewsKeyspace()
                     .add(new InternodeOutboundTable(VIRTUAL_VIEWS))
                     .add(new InternodeInboundTable(VIRTUAL_VIEWS))
                     .add(new PendingHintsTable(VIRTUAL_VIEWS))
+                    .add(new SSTablesSystemView(VIRTUAL_VIEWS))
                     .addAll(TableMetricTables.getAll(VIRTUAL_VIEWS))
                     .add(new CredentialsCacheKeysTable(VIRTUAL_VIEWS))
                     .add(new JmxPermissionsCacheKeysTable(VIRTUAL_VIEWS))
@@ -54,7 +66,11 @@ private SystemViewsKeyspace()
                     .add(new SnapshotsTable(VIRTUAL_VIEWS))
                     .addAll(LocalRepairTables.getAll(VIRTUAL_VIEWS))
                     .addAll(CIDRFilteringMetricsTable.getAll(VIRTUAL_VIEWS))
-                    .addAll(StorageAttachedIndexTables.getAll(VIRTUAL_VIEWS))
-                    .build());
+                    .addAll(StorageAttachedIndexTables.getAll(VIRTUAL_VIEWS));
+        if (CassandraRelevantProperties.SYSTEM_VIEWS_INCLUDE_ALL.getBoolean()
+            || CassandraRelevantProperties.SYSTEM_VIEWS_INCLUDE_INDEXES.getBoolean())
+            tables.add(new IndexesSystemView(VIRTUAL_VIEWS));
+
+        return tables.build();
     }
 }
diff --git a/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java b/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java
index 5528c92011cc..4640adbdc116 100644
--- a/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java
+++ b/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java
@@ -68,14 +68,14 @@ public class TableMetricTables
     public static Collection<VirtualTable> getAll(String name)
     {
         return ImmutableList.of(
-            new LatencyTableMetric(name, "local_read_latency", t -> t.readLatency.latency),
-            new LatencyTableMetric(name, "local_scan_latency", t -> t.rangeLatency.latency),
-            new LatencyTableMetric(name, "coordinator_read_latency", t -> t.coordinatorReadLatency),
-            new LatencyTableMetric(name, "coordinator_scan_latency", t -> t.coordinatorScanLatency),
-            new LatencyTableMetric(name, "local_write_latency", t -> t.writeLatency.latency),
-            new LatencyTableMetric(name, "coordinator_write_latency", t -> t.coordinatorWriteLatency),
-            new HistogramTableMetric(name, "tombstones_per_read", t -> t.tombstoneScannedHistogram.cf),
-            new HistogramTableMetric(name, "rows_per_read", t -> t.liveScannedHistogram.cf),
+            new LatencyTableMetric(name, "local_read_latency", t -> t.readLatency.tableOrKeyspaceMetric().latency),
+            new LatencyTableMetric(name, "local_scan_latency", t -> t.rangeLatency.tableOrKeyspaceMetric().latency),
+            new LatencyTableMetric(name, "coordinator_read_latency", t -> t.coordinatorReadLatency.tableOrKeyspaceTimer()),
+            new LatencyTableMetric(name, "coordinator_scan_latency", t -> t.coordinatorScanLatency.tableOrKeyspaceTimer()),
+            new LatencyTableMetric(name, "local_write_latency", t -> t.writeLatency.tableOrKeyspaceMetric().latency),
+            new LatencyTableMetric(name, "coordinator_write_latency", t -> t.coordinatorWriteLatency.tableOrKeyspaceTimer()),
+            new HistogramTableMetric(name, "tombstones_per_read", t -> t.tombstoneScannedHistogram.tableOrKeyspaceHistogram()),
+            new HistogramTableMetric(name, "rows_per_read", t -> t.liveScannedHistogram.tableOrKeyspaceHistogram()),
             new StorageTableMetric(name, "disk_usage", (TableMetrics t) -> t.totalDiskSpaceUsed),
             new StorageTableMetric(name, "max_partition_size", (TableMetrics t) -> t.maxPartitionSize),
             new StorageTableMetric(name, "max_sstable_size", (TableMetrics t) -> t.maxSSTableSize),
diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java b/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java
index 8c3b5b4afda6..ff66335f23a5 100644
--- a/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java
+++ b/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java
@@ -27,6 +27,7 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.schema.TableId;
@@ -67,6 +68,12 @@ public String getKeyspaceName()
         return keyspaceName;
     }
 
+    @Override
+    public Keyspace getKeyspace()
+    {
+        return Keyspace.open(keyspaceName);
+    }
+
     @Override
     public Collection<TableId> getTableIds()
     {
diff --git a/src/java/org/apache/cassandra/dht/AbstractBounds.java b/src/java/org/apache/cassandra/dht/AbstractBounds.java
index 7a603b0a5dc0..71181e3b2bd1 100644
--- a/src/java/org/apache/cassandra/dht/AbstractBounds.java
+++ b/src/java/org/apache/cassandra/dht/AbstractBounds.java
@@ -55,6 +55,12 @@ public AbstractBounds(T left, T right)
         this.right = right;
     }
 
+    public static AbstractBounds<PartitionPosition> unbounded(IPartitioner partitioner)
+    {
+        return bounds(partitioner.getMinimumToken().minKeyBound(), true,
+                      partitioner.getMaximumToken().maxKeyBound(), true);
+    }
+
     /**
      * Given token T and AbstractBounds ?L,R?, returns Pair(?L,T], (T,R?),
      * where ? means that the same type of AbstractBounds is returned as the original.
diff --git a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
index e2371c09376d..9a1e07523f78 100644
--- a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
+++ b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
@@ -221,7 +221,7 @@ public double size(Token next)
         @Override
         public LongToken nextValidToken()
         {
-            return new LongToken(token + 1);
+            return new LongToken(token + 1);    // wraparound to MINIMUM if token is MAXIMUM
         }
 
         public LongToken decreaseSlightly()
diff --git a/src/java/org/apache/cassandra/dht/RandomPartitioner.java b/src/java/org/apache/cassandra/dht/RandomPartitioner.java
index a8fbe764d47d..91613b0a3d18 100644
--- a/src/java/org/apache/cassandra/dht/RandomPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/RandomPartitioner.java
@@ -273,7 +273,9 @@ public long getHeapSize()
 
         public Token nextValidToken()
         {
-            return new BigIntegerToken(token.add(BigInteger.ONE));
+            BigInteger next = token.equals(MAXIMUM) ? ZERO
+                                                    : token.add(BigInteger.ONE);
+            return new BigIntegerToken(next);
         }
 
         public double size(Token next)
diff --git a/src/java/org/apache/cassandra/dht/Range.java b/src/java/org/apache/cassandra/dht/Range.java
index 0ba6d2087092..c1b8b2e56cff 100644
--- a/src/java/org/apache/cassandra/dht/Range.java
+++ b/src/java/org/apache/cassandra/dht/Range.java
@@ -18,7 +18,15 @@
 package org.apache.cassandra.dht;
 
 import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 import java.util.function.Predicate;
 
 import com.google.common.collect.Iterables;
@@ -70,7 +78,7 @@ public static <T extends RingPosition<T>> boolean contains(T left, T right, T po
         }
     }
 
-    public boolean contains(Range<T> that)
+    public boolean contains(AbstractBounds<T> that)
     {
         if (this.left.equals(this.right))
         {
@@ -125,9 +133,36 @@ public boolean intersects(AbstractBounds<T> that)
             return intersects((Range<T>) that);
         if (that instanceof Bounds)
             return intersects((Bounds<T>) that);
+        if (that instanceof ExcludingBounds)
+            return intersects((ExcludingBounds<T>) that);
+        if (that instanceof IncludingExcludingBounds)
+            return intersects((IncludingExcludingBounds<T>) that);
+
         throw new UnsupportedOperationException("Intersection is only supported for Bounds and Range objects; found " + that.getClass());
     }
 
+    public boolean intersects(IncludingExcludingBounds<T> that)
+    {
+        if (!isWrapAround() && !that.right.isMinimum() && (this.left.compareTo(that.right) == 0))
+            return false;
+        else if (isWrapAround() && !that.right.isMinimum() && (this.right.compareTo(that.right) == 0))
+            return false;
+        return contains(that.left) || (!that.left.equals(that.right) && intersects(new Range<T>(that.left, that.right)));
+    }
+
+    public boolean intersects(ExcludingBounds<T> that)
+    {
+        if (!isWrapAround() &&
+            ((!that.right.isMinimum() && (this.left.compareTo(that.right) == 0)) ||
+             (this.right.compareTo(that.left) == 0)))
+            return false;
+        else if (isWrapAround() &&
+                 ((this.left.compareTo(that.left) == 0) ||
+                  (!that.right.isMinimum() && (this.right.compareTo(that.right) == 0))))
+            return false;
+        return contains(that.left) || (!that.left.equals(that.right) && intersects(new Range<T>(that.left, that.right)));
+    }
+
     /**
      * @param that range to check for intersection
      * @return true if the given range intersects with this range.
diff --git a/src/java/org/apache/cassandra/dht/RangeStreamer.java b/src/java/org/apache/cassandra/dht/RangeStreamer.java
index 9b7833b90aab..e9f362b8ebb9 100644
--- a/src/java/org/apache/cassandra/dht/RangeStreamer.java
+++ b/src/java/org/apache/cassandra/dht/RangeStreamer.java
@@ -41,11 +41,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.gms.FailureDetector;
-import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.EndpointsByRange;
@@ -85,7 +85,7 @@ public class RangeStreamer
                                                              (!Gossiper.instance.isEnabled() ||
                                                               (Gossiper.instance.getEndpointStateForEndpoint(replica.endpoint()) == null ||
                                                                Gossiper.instance.getEndpointStateForEndpoint(replica.endpoint()).isAlive())) &&
-                                                             FailureDetector.instance.isAlive(replica.endpoint());
+                                                             IFailureDetector.instance.isAlive(replica.endpoint());
 
     /* bootstrap tokens. can be null if replacing the node. */
     private final Collection<Token> tokens;
@@ -283,7 +283,7 @@ public RangeStreamer(TokenMetadata metadata,
                          int connectionsPerHost)
     {
         this(metadata, tokens, address, streamOperation, useStrictConsistency, snitch, stateStore,
-             FailureDetector.instance, connectSequentially, connectionsPerHost);
+             IFailureDetector.instance, connectSequentially, connectionsPerHost);
     }
 
     RangeStreamer(TokenMetadata metadata,
@@ -358,9 +358,13 @@ public void addRanges(String keyspaceName, ReplicaCollection<?> replicas)
             logger.info("{}: range {} exists on {} for keyspace {}", description, entry.getKey(), entry.getValue(), keyspaceName);
 
         Multimap<InetAddressAndPort, FetchReplica> workMap;
-        //Only use the optimized strategy if we don't care about strict sources, have a replication factor > 1, and no
-        //transient replicas.
-        if (useStrictSource || strat == null || strat.getReplicationFactor().allReplicas == 1 || strat.getReplicationFactor().hasTransientReplicas())
+        //Only use the optimized strategy if we don't care about strict sources, have a replication factor > 1, no
+        //transient replicas or HCD-84
+        if (CassandraRelevantProperties.SKIP_OPTIMAL_STREAMING_CANDIDATES_CALCULATION.getBoolean() ||
+            useStrictSource ||
+            strat == null ||
+            strat.getReplicationFactor().allReplicas == 1 ||
+            strat.getReplicationFactor().hasTransientReplicas())
         {
             workMap = convertPreferredEndpointsToWorkMap(fetchMap);
         }
@@ -389,7 +393,7 @@ public void addRanges(String keyspaceName, ReplicaCollection<?> replicas)
     private boolean useStrictSourcesForRanges(AbstractReplicationStrategy strat)
     {
         boolean res = useStrictConsistency && tokens != null;
-        
+
         if (res)
         {
             int nodes = 0;
@@ -404,10 +408,10 @@ private boolean useStrictSourcesForRanges(AbstractReplicationStrategy strat)
             }
             else
                 nodes = metadata.getSizeOfAllEndpoints();
-    
+
             res = nodes > strat.getReplicationFactor().allReplicas;
         }
-        
+
         return res;
     }
 
@@ -488,13 +492,15 @@ else if (useStrictConsistency)
                  final EndpointsForRange oldEndpoints = sorted.apply(rangeAddresses.get(range));
 
                  //Ultimately we populate this with whatever is going to be fetched from to satisfy toFetch
-                 //It could be multiple endpoints and we must fetch from all of them if they are there
+                 //It could be multiple endpoints, and we must fetch from all of them if they are there
                  //With transient replication and strict consistency this is to get the full data from a full replica and
                  //transient data from the transient replica losing data
                  EndpointsForRange sources;
+
                  //Due to CASSANDRA-5953 we can have a higher RF than we have endpoints.
                  //So we need to be careful to only be strict when endpoints == RF
                  boolean isStrictConsistencyApplicable = useStrictConsistency && (oldEndpoints.size() == strat.getReplicationFactor().allReplicas);
+
                  if (isStrictConsistencyApplicable)
                  {
                      EndpointsForRange strictEndpoints;
diff --git a/src/java/org/apache/cassandra/dht/Splitter.java b/src/java/org/apache/cassandra/dht/Splitter.java
index 53b4462221cd..ad890d323bbf 100644
--- a/src/java/org/apache/cassandra/dht/Splitter.java
+++ b/src/java/org/apache/cassandra/dht/Splitter.java
@@ -23,7 +23,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -118,10 +118,75 @@ public double positionInRange(Token token, Range<Token> range)
         return new BigDecimal(elapsedTokens(token, range)).divide(new BigDecimal(tokensInRange(range)), 3, BigDecimal.ROUND_HALF_EVEN).doubleValue();
     }
 
-    public List<Token> splitOwnedRanges(int parts, List<WeightedRange> weightedRanges, boolean dontSplitRanges)
+    /**
+     * How local ranges should be split
+     */
+    public enum SplitType
+    {
+        /** Local ranges should always be split, without attempting to keep them whole */
+        ALWAYS_SPLIT,
+        /** A first pass will try to avoid splitting ranges, but if there aren't enough parts,
+         * then ranges will be split in a second pass.
+         */
+        PREFER_WHOLE,
+        /** Ranges Should never be split */
+        ONLY_WHOLE
+    }
+
+    /**
+     * The result of a split operation, this is just a wrapper of the boundaries and the type
+     * of split that was done, i.e. if the local ranges were split or not. This is just so that
+     * we can test the algorithm.
+     */
+    public final static class SplitResult
+    {
+        public final List<Token> boundaries;
+        public final boolean rangesWereSplit;
+
+        SplitResult(List<Token> boundaries, boolean rangesWereSplit)
+        {
+            this.boundaries = boundaries;
+            this.rangesWereSplit = rangesWereSplit;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o)
+                return true;
+
+            if (!(o instanceof SplitResult))
+                return false;
+
+            SplitResult splitResult = (SplitResult) o;
+            return Objects.equals(boundaries, splitResult.boundaries)
+                   && Objects.equals(rangesWereSplit, splitResult.rangesWereSplit);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(boundaries, rangesWereSplit);
+        }
+    }
+
+    /**
+     * Split the local ranges into the specified number of parts.
+     *
+     * Depending on the parameter {@code splitType}, it may attempt to only merge the local ranges first,
+     * to see if this is sufficient to cover the requested number of parts. If it's not, it will then split
+     * existing ranges.
+     *
+     * @param parts the number of parts
+     * @param weightedRanges the local ranges owned by this node
+     * @param splitType how local ranges should be split, see {@link SplitType}
+     *
+     * @return the split result, which contains a list of tokens, one per part, and if the ranges were split or not
+     */
+    public SplitResult splitOwnedRanges(int parts, List<WeightedRange> weightedRanges, SplitType splitType)
     {
-        if (weightedRanges.isEmpty() || parts == 1)
-            return Collections.singletonList(partitioner.getMaximumToken());
+        if (weightedRanges.isEmpty() || parts <= 1)
+            return new SplitResult(Collections.singletonList(partitioner.getMaximumToken()), false);
 
         BigInteger totalTokens = BigInteger.ZERO;
         for (WeightedRange weightedRange : weightedRanges)
@@ -132,12 +197,23 @@ public List<Token> splitOwnedRanges(int parts, List<WeightedRange> weightedRange
         BigInteger perPart = totalTokens.divide(BigInteger.valueOf(parts));
         // the range owned is so tiny we can't split it:
         if (perPart.equals(BigInteger.ZERO))
-            return Collections.singletonList(partitioner.getMaximumToken());
-
-        if (dontSplitRanges)
-            return splitOwnedRangesNoPartialRanges(weightedRanges, perPart, parts);
+            return new SplitResult(Collections.singletonList(partitioner.getMaximumToken()), false);
 
         List<Token> boundaries = new ArrayList<>();
+
+        if (splitType != SplitType.ALWAYS_SPLIT)
+        {
+            // see if we can obtain a sufficient number of parts by only merging local ranges
+            boundaries = splitOwnedRangesNoPartialRanges(weightedRanges, perPart, parts);
+            // we were either able to obtain sufficient parts without splitting ranges or we should never split ranges
+            if (splitType == SplitType.ONLY_WHOLE || boundaries.size() == parts)
+                return new SplitResult(boundaries, false);
+            else
+                boundaries.clear();
+        }
+
+        // otherwise continue by splitting ranges
+
         BigInteger sum = BigInteger.ZERO;
         BigInteger tokensLeft = totalTokens;
         for (WeightedRange weightedRange : weightedRanges)
@@ -155,16 +231,20 @@ public List<Token> splitOwnedRanges(int parts, List<WeightedRange> weightedRange
                 sum = BigInteger.ZERO;
                 int partsLeft = parts - boundaries.size();
                 if (partsLeft == 0)
+                {
                     break;
+                }
                 else if (partsLeft == 1)
+                {
                     perPart = tokensLeft;
+                }
             }
             sum = sum.add(currentRangeWidth);
         }
         boundaries.set(boundaries.size() - 1, partitioner.getMaximumToken());
 
         assert boundaries.size() == parts : boundaries.size() + "!=" + parts + " " + boundaries + ":" + weightedRanges;
-        return boundaries;
+        return new SplitResult(boundaries, true);
     }
 
     private List<Token> splitOwnedRangesNoPartialRanges(List<WeightedRange> weightedRanges, BigInteger perPart, int parts)
@@ -238,28 +318,26 @@ public Set<Range<Token>> split(Collection<Range<Token>> ranges, int parts)
     }
 
     /**
-     * Splits the specified token range in at least {@code minParts} subranges, unless the range has not enough tokens
-     * in which case the range will be returned without splitting.
+     * Splits the specified token range in {@code parts} subranges, unless the range has not enough tokens in which case
+     * the range will be returned without splitting.
      *
      * @param range a token range
      * @param parts the number of subranges
-     * @return {@code parts} even subranges of {@code range}
+     * @return {@code parts} even subranges of {@code range}, or {@code range} if it is too small to be splitted
      */
     private Set<Range<Token>> split(Range<Token> range, int parts)
     {
-        // the range might not have enough tokens to split
-        BigInteger numTokens = tokensInRange(range);
-        if (BigInteger.valueOf(parts).compareTo(numTokens) > 0)
-            return Collections.singleton(range);
-
         Token left = range.left;
-        Set<Range<Token>> subranges = new HashSet<>(parts);
-        for (double i = 1; i <= parts; i++)
+        Set<Range<Token>> subranges = new LinkedHashSet<>(parts);
+
+        for (double i = 1; i < parts; i++)
         {
             Token right = partitioner.split(range.left, range.right, i / parts);
-            subranges.add(new Range<>(left, right));
+            if (!left.equals(right))
+                subranges.add(new Range<>(left, right));
             left = right;
         }
+        subranges.add(new Range<>(left, range.right));
         return subranges;
     }
 
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/IsolatedTokenAllocator.java b/src/java/org/apache/cassandra/dht/tokenallocator/IsolatedTokenAllocator.java
new file mode 100644
index 000000000000..39766e10f8fc
--- /dev/null
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/IsolatedTokenAllocator.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht.tokenallocator;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Maps;
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.locator.SimpleSnitch;
+import org.apache.cassandra.locator.TokenMetadata;
+
+/**
+ * A utility class that allocates additional tokens for a given {@link AbstractReplicationStrategy} by creating mock
+ * nodes and then allocating tokens for them. The source metadata and replication strategy are not modified. This
+ * class relies on the detail that the allocation of new tokens for bootstrapping nodes is deterministic.
+ */
+public class IsolatedTokenAllocator
+{
+    private static final Logger logger = LoggerFactory.getLogger(IsolatedTokenAllocator.class);
+
+    public static List<Token> allocateTokens(int additionalSplits, AbstractReplicationStrategy source)
+    {
+        Preconditions.checkArgument(additionalSplits > 0, "additionalSplits must be greater than zero");
+        Preconditions.checkNotNull(source);
+
+        List<Token> allocatedTokens = new ArrayList<>();
+        QuietAllocator allocator = new QuietAllocator(source);
+
+        // Distribute nodes among the racks in round-robin fashion in the order the user is supposed to start them.
+        var localDc = source.snitch.getLocalDatacenter();
+        // Get a list to consistently iterate over the racks as we allocate nodes. Need to clone the map in
+        // order to retreive the topology.
+        var localRacks = source.getTokenMetadata().cloneOnlyTokenMap().getTopology().getDatacenterRacks().get(localDc);
+        assert localRacks != null && !localRacks.isEmpty() : "No racks found for local datacenter " + localDc;
+        // Because we have RF=racks, we do not need to worry about the order of the racks. If we wnat to make anything
+        // else work, we probably need to know the order of the racks here to ensure we do it the same way each time.
+        // Issues could arise where we allocate a token for one rack, but that isn't the rack
+        // that bootstraps a new node next.
+        var racks = Iterators.cycle(localRacks.keySet());
+        int nodeId = 0;
+        while (allocatedTokens.size() < additionalSplits)
+        {
+            // Allocate tokens for current node, distributing tokens round-robin among the racks.
+            var newTokens = allocator.allocateTokensForNode(nodeId, racks.next());
+            int remainingTokensNeeded = additionalSplits - allocatedTokens.size();
+            if (newTokens.size() > remainingTokensNeeded)
+            {
+                var iter = newTokens.iterator();
+                for (int i = 0; i < remainingTokensNeeded; i++)
+                    allocatedTokens.add(iter.next());
+                return allocatedTokens;
+            }
+            else
+            {
+                allocatedTokens.addAll(newTokens);
+            }
+            nodeId++;
+        }
+        return allocatedTokens;
+    }
+
+    /**
+     * A token allocator that takes a source token metadata and replication strategy, but clones the source token
+     * metadata with a quiet snitch, so that added nodes are not communicated to the rest of the system.
+     */
+    private static class QuietAllocator
+    {
+        private final QuietSnitch quietSnitch;
+        private final TokenMetadata quietTokenMetadata;
+        private final TokenAllocation allocation;
+        private final Map<String, SummaryStatistics> lastCheckPoint = Maps.newHashMap();
+
+        private QuietAllocator(AbstractReplicationStrategy rs)
+        {
+            // Wrap the replication strategy's snitch with a quiet snitch
+            this.quietSnitch = new QuietSnitch(rs.snitch);
+            this.quietTokenMetadata = rs.getTokenMetadata().cloneWithNewSnitch(quietSnitch);
+            var numTokens = DatabaseDescriptor.getNumTokens();
+            this.allocation = TokenAllocation.create(quietTokenMetadata, rs, quietSnitch, numTokens);
+        }
+
+        private Collection<Token> allocateTokensForNode(int nodeId, String rackId)
+        {
+            // Update snitch and token metadata info to inform token allocation
+            InetAddressAndPort fakeNodeAddressAndPort = getLoopbackAddressWithPort(nodeId);
+            quietSnitch.nodeByRack.put(fakeNodeAddressAndPort, rackId);
+            quietTokenMetadata.updateTopology(fakeNodeAddressAndPort);
+
+            // Allocate tokens
+            Collection<Token> tokens = allocation.allocate(fakeNodeAddressAndPort);
+
+            // Validate ownership stats
+            validateAllocation(nodeId, rackId);
+
+            return tokens;
+        }
+
+        private void validateAllocation(int nodeId, String rackId)
+        {
+            SummaryStatistics newOwnership = allocation.getAllocationRingOwnership(SimpleSnitch.DATA_CENTER_NAME, rackId);
+            SummaryStatistics oldOwnership = lastCheckPoint.put(rackId, newOwnership);
+            if (oldOwnership != null)
+                logger.debug(String.format("Replicated node load in rack=%s before allocating node %d: %s.", rackId, nodeId,
+                                           TokenAllocation.statToString(oldOwnership)));
+            logger.debug(String.format("Replicated node load in rack=%s after allocating node %d: %s.", rackId, nodeId,
+                                       TokenAllocation.statToString(newOwnership)));
+            if (oldOwnership != null && oldOwnership.getStandardDeviation() != 0.0)
+            {
+                double stdDevGrowth = newOwnership.getStandardDeviation() - oldOwnership.getStandardDeviation();
+                if (stdDevGrowth > TokenAllocation.WARN_STDEV_GROWTH)
+                {
+                    logger.warn(String.format("Growth of %.2f%% in token ownership standard deviation after allocating node %d on rack %s above warning threshold of %d%%",
+                                              stdDevGrowth * 100, nodeId, rackId, (int)(TokenAllocation.WARN_STDEV_GROWTH * 100)));
+                }
+            }
+        }
+    }
+
+    /**
+     * A snitch that doesn't gossip.
+     */
+    private static class QuietSnitch implements IEndpointSnitch
+    {
+        private final Map<InetAddressAndPort, String> nodeByRack = new HashMap<>();
+        private final IEndpointSnitch fallbackSnitch;
+
+        QuietSnitch(IEndpointSnitch fallbackSnitch)
+        {
+            this.fallbackSnitch = fallbackSnitch;
+        }
+
+        @Override
+        public String getRack(InetAddressAndPort endpoint)
+        {
+            String result = nodeByRack.get(endpoint);
+            return result != null ? result : fallbackSnitch.getRack(endpoint);
+        }
+
+        @Override
+        public String getDatacenter(InetAddressAndPort endpoint)
+        {
+            // For our mocked endpoints, we return the local datacenter, otherwise we return the real datacenter
+            return nodeByRack.containsKey(endpoint)
+                   ? fallbackSnitch.getLocalDatacenter()
+                   : fallbackSnitch.getDatacenter(endpoint);
+        }
+
+        @Override
+        public <C extends ReplicaCollection<? extends C>> C sortedByProximity(InetAddressAndPort address, C addresses)
+        {
+            throw new NotImplementedException("sortedByProximity not implemented in QuietSnitch");
+        }
+
+        @Override
+        public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2)
+        {
+            throw new NotImplementedException("compareEndpoints not implemented in QuietSnitch");
+        }
+
+        @Override
+        public void gossiperStarting()
+        {
+            // This snitch doesn't gossip.
+        }
+
+        @Override
+        public boolean isWorthMergingForRangeQuery(ReplicaCollection<?> merged, ReplicaCollection<?> l1, ReplicaCollection<?> l2)
+        {
+            throw new NotImplementedException("isWorthMergingForRangeQuery not implemented in QuietSnitch");
+        }
+    }
+
+    private static InetAddressAndPort getLoopbackAddressWithPort(int port)
+    {
+        try
+        {
+            return InetAddressAndPort.getByAddressOverrideDefaults(InetAddress.getByName("127.0.0.1"), port);
+        }
+        catch (UnknownHostException e)
+        {
+            throw new IllegalStateException("Unexpected UnknownHostException", e);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocator.java b/src/java/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocator.java
index 255a2c95692f..a82720b3d1eb 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocator.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocator.java
@@ -26,6 +26,7 @@
 import java.util.NavigableMap;
 import java.util.PriorityQueue;
 import java.util.Queue;
+import java.util.function.Supplier;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -49,6 +50,14 @@ public NoReplicationTokenAllocator(NavigableMap<Token, Unit> sortedTokens,
         super(sortedTokens, strategy, partitioner);
     }
 
+    public NoReplicationTokenAllocator(NavigableMap<Token, Unit> sortedTokens,
+                                       ReplicationStrategy<Unit> strategy,
+                                       IPartitioner partitioner,
+                                       Supplier<Token> seedTokenSupplier)
+    {
+        super(sortedTokens, strategy, partitioner, seedTokenSupplier);
+    }
+
     /**
      * Construct the token ring as a CircularList of TokenInfo,
      * and populate the ownership of the UnitInfo's provided
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationStrategy.java b/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationStrategy.java
index 8cb5fe1cebaf..d127a5027b51 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationStrategy.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationStrategy.java
@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.dht.tokenallocator;
 
-interface ReplicationStrategy<Unit>
+public interface ReplicationStrategy<Unit>
 {
     int replicas();
 
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java
index 7e46b87855ce..35c9a681878c 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java
@@ -25,6 +25,7 @@
 import java.util.NavigableMap;
 import java.util.TreeMap;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
@@ -33,7 +34,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
@@ -51,13 +54,17 @@ public class TokenAllocation
     private static final Logger logger = LoggerFactory.getLogger(TokenAllocation.class);
     final TokenMetadata tokenMetadata;
     final AbstractReplicationStrategy replicationStrategy;
+    // In order for the IsolatedTokenAllocator to work correctly, we need to allow for a different snitch than the
+    // one provided by the replicationStrategy.
+    final IEndpointSnitch snitch;
     final int numTokens;
     final Map<String, Map<String, StrategyAdapter>> strategyByRackDc = new HashMap<>();
 
-    private TokenAllocation(TokenMetadata tokenMetadata, AbstractReplicationStrategy replicationStrategy, int numTokens)
+    private TokenAllocation(TokenMetadata tokenMetadata, AbstractReplicationStrategy replicationStrategy, IEndpointSnitch snitch, int numTokens)
     {
         this.tokenMetadata = tokenMetadata.cloneOnlyTokenMap();
         this.replicationStrategy = replicationStrategy;
+        this.snitch = snitch;
         this.numTokens = numTokens;
     }
 
@@ -77,6 +84,50 @@ public static Collection<Token> allocateTokens(final TokenMetadata tokenMetadata
         return create(DatabaseDescriptor.getEndpointSnitch(), tokenMetadata, replicas, numTokens).allocate(endpoint);
     }
 
+    // Used by CNDB TokenTracker
+    public static Collection<Token> allocateTokens(TokenMetadata tokenMetadata,
+                                                   IEndpointSnitch snitch,
+                                                   int localReplicationFactor,
+                                                   InetAddressAndPort endpoint,
+                                                   int numTokens,
+                                                   StrategyAdapter strategy)
+    {
+        return create(snitch, tokenMetadata, localReplicationFactor, numTokens).allocate(endpoint, strategy);
+    }
+
+    // Used by CNDB
+    // return the ratio of ownership for each endpoint
+    public static Map<InetAddressAndPort, Double> evaluateReplicatedOwnership(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs)
+    {
+        Map<InetAddressAndPort, Double> ownership = Maps.newHashMap();
+        List<Token> sortedTokens = tokenMetadata.sortedTokens();
+        if (sortedTokens.isEmpty())
+            return ownership;
+
+        Iterator<Token> it = sortedTokens.iterator();
+        Token current = it.next();
+        while (it.hasNext())
+        {
+            Token next = it.next();
+            addOwnership(tokenMetadata, rs, current, next, ownership);
+            current = next;
+        }
+        addOwnership(tokenMetadata, rs, current, sortedTokens.get(0), ownership);
+
+        return ownership;
+    }
+
+    private static void addOwnership(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs, Token current, Token next, Map<InetAddressAndPort, Double> ownership)
+    {
+        double size = current.size(next);
+        Token representative = current.getPartitioner().midpoint(current, next);
+        for (InetAddressAndPort n : rs.calculateNaturalReplicas(representative, tokenMetadata).endpoints())
+        {
+            Double v = ownership.get(n);
+            ownership.put(n, v != null ? v + size : size);
+        }
+    }
+
     static TokenAllocation create(IEndpointSnitch snitch, TokenMetadata tokenMetadata, int replicas, int numTokens)
     {
         // We create a fake NTS replication strategy with the specified RF in the local DC
@@ -84,25 +135,34 @@ static TokenAllocation create(IEndpointSnitch snitch, TokenMetadata tokenMetadat
         options.put(snitch.getLocalDatacenter(), Integer.toString(replicas));
         NetworkTopologyStrategy fakeReplicationStrategy = new NetworkTopologyStrategy(null, tokenMetadata, snitch, options);
 
-        TokenAllocation allocator = new TokenAllocation(tokenMetadata, fakeReplicationStrategy, numTokens);
+        TokenAllocation allocator = new TokenAllocation(tokenMetadata, fakeReplicationStrategy, snitch, numTokens);
         return allocator;
     }
 
     static TokenAllocation create(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs, int numTokens)
     {
-        return new TokenAllocation(tokenMetadata, rs, numTokens);
+        return new TokenAllocation(tokenMetadata, rs, rs.snitch, numTokens);
+    }
+
+    static TokenAllocation create(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs, IEndpointSnitch snitch, int numTokens)
+    {
+        return new TokenAllocation(tokenMetadata, rs, snitch, numTokens);
     }
 
     Collection<Token> allocate(InetAddressAndPort endpoint)
     {
-        StrategyAdapter strategy = getOrCreateStrategy(endpoint);
+        return allocate(endpoint, getOrCreateStrategy(endpoint));
+    }
+
+    private Collection<Token> allocate(InetAddressAndPort endpoint, StrategyAdapter strategy)
+    {
         Collection<Token> tokens = strategy.createAllocator().addUnit(endpoint, numTokens);
         tokens = strategy.adjustForCrossDatacenterClashes(tokens);
 
-        SummaryStatistics os = strategy.replicatedOwnershipStats();
+        SummaryStatistics os = replicatedOwnershipStats(strategy);
         tokenMetadata.updateNormalTokens(tokens, endpoint);
 
-        SummaryStatistics ns = strategy.replicatedOwnershipStats();
+        SummaryStatistics ns = replicatedOwnershipStats(strategy);
         logger.info("Selected tokens {}", tokens);
         logger.debug("Replicated node load in datacenter before allocation {}", statToString(os));
         logger.debug("Replicated node load in datacenter after allocation {}", statToString(ns));
@@ -124,20 +184,34 @@ static String statToString(SummaryStatistics stat)
 
     SummaryStatistics getAllocationRingOwnership(String datacenter, String rack)
     {
-        return getOrCreateStrategy(datacenter, rack).replicatedOwnershipStats();
+        return replicatedOwnershipStats(getOrCreateStrategy(datacenter, rack));
     }
 
+    @VisibleForTesting
     SummaryStatistics getAllocationRingOwnership(InetAddressAndPort endpoint)
     {
-        return getOrCreateStrategy(endpoint).replicatedOwnershipStats();
+        return replicatedOwnershipStats(getOrCreateStrategy(endpoint));
     }
 
-    abstract class StrategyAdapter implements ReplicationStrategy<InetAddressAndPort>
+    public static abstract class StrategyAdapter implements ReplicationStrategy<InetAddressAndPort>
     {
+        final TokenMetadata tokenMetadata;
+
+        public StrategyAdapter(TokenMetadata tokenMetadata)
+        {
+            this.tokenMetadata = tokenMetadata;
+        }
+
         // return true iff the provided endpoint occurs in the same virtual token-ring we are allocating for
         // i.e. the set of the nodes that share ownership with the node we are allocating
         // alternatively: return false if the endpoint's ownership is independent of the node we are allocating tokens for
-        abstract boolean inAllocationRing(InetAddressAndPort other);
+        public abstract boolean inAllocationRing(InetAddressAndPort other);
+
+        // Allows sub classes to override and provide custom partitioners
+        public IPartitioner partitioner()
+        {
+            return tokenMetadata.partitioner;
+        }
 
         final TokenAllocator<InetAddressAndPort> createAllocator()
         {
@@ -147,7 +221,7 @@ final TokenAllocator<InetAddressAndPort> createAllocator()
                 if (inAllocationRing(en.getValue()))
                     sortedTokens.put(en.getKey(), en.getValue());
             }
-            return TokenAllocatorFactory.createTokenAllocator(sortedTokens, this, tokenMetadata.partitioner);
+            return TokenAllocatorFactory.createTokenAllocator(sortedTokens, this, partitioner());
         }
 
         final Collection<Token> adjustForCrossDatacenterClashes(Collection<Token> tokens)
@@ -156,9 +230,9 @@ final Collection<Token> adjustForCrossDatacenterClashes(Collection<Token> tokens
 
             for (Token t : tokens)
             {
-                while (tokenMetadata.getEndpoint(t) != null)
+                InetAddressAndPort other;
+                while ((other = tokenMetadata.getEndpoint(t)) != null)
                 {
-                    InetAddressAndPort other = tokenMetadata.getEndpoint(t);
                     if (inAllocationRing(other))
                         throw new ConfigurationException(String.format("Allocated token %s already assigned to node %s. Is another node also allocating tokens?", t, other));
                     t = t.nextValidToken();
@@ -167,59 +241,42 @@ final Collection<Token> adjustForCrossDatacenterClashes(Collection<Token> tokens
             }
             return filtered;
         }
+    }
 
-        final SummaryStatistics replicatedOwnershipStats()
+    private SummaryStatistics replicatedOwnershipStats(StrategyAdapter strategy)
+    {
+        SummaryStatistics stat = new SummaryStatistics();
+        for (Map.Entry<InetAddressAndPort, Double> en : TokenAllocation.evaluateReplicatedOwnership(tokenMetadata, replicationStrategy).entrySet())
         {
-            SummaryStatistics stat = new SummaryStatistics();
-            for (Map.Entry<InetAddressAndPort, Double> en : evaluateReplicatedOwnership().entrySet())
-            {
-                // Filter only in the same allocation ring
-                if (inAllocationRing(en.getKey()))
-                    stat.addValue(en.getValue() / tokenMetadata.getTokens(en.getKey()).size());
-            }
-            return stat;
+            // Filter only in the same allocation ring
+            if (strategy.inAllocationRing(en.getKey()))
+                stat.addValue(en.getValue() / tokenMetadata.getTokens(en.getKey()).size());
         }
+        return stat;
+    }
 
-        // return the ratio of ownership for each endpoint
-        private Map<InetAddressAndPort, Double> evaluateReplicatedOwnership()
-        {
-            Map<InetAddressAndPort, Double> ownership = Maps.newHashMap();
-            List<Token> sortedTokens = tokenMetadata.sortedTokens();
-            if (sortedTokens.isEmpty())
-                return ownership;
-
-            Iterator<Token> it = sortedTokens.iterator();
-            Token current = it.next();
-            while (it.hasNext())
-            {
-                Token next = it.next();
-                addOwnership(current, next, ownership);
-                current = next;
-            }
-            addOwnership(current, sortedTokens.get(0), ownership);
+    private StrategyAdapter getOrCreateStrategy(InetAddressAndPort endpoint)
+    {
+        String dc = snitch.getDatacenter(endpoint);
+        String rack = snitch.getRack(endpoint);
 
-            return ownership;
+        try
+        {
+            return getOrCreateStrategy(dc, rack);
         }
-
-        private void addOwnership(Token current, Token next, Map<InetAddressAndPort, Double> ownership)
+        catch (ConfigurationException e)
         {
-            double size = current.size(next);
-            Token representative = current.getPartitioner().midpoint(current, next);
-            for (InetAddressAndPort n : replicationStrategy.calculateNaturalReplicas(representative, tokenMetadata).endpoints())
-            {
-                Double v = ownership.get(n);
-                ownership.put(n, v != null ? v + size : size);
-            }
+            if (CassandraRelevantProperties.USE_RANDOM_ALLOCATION_IF_NOT_SUPPORTED.getBoolean())
+                return createRandomStrategy(endpoint);
+
+            throw new ConfigurationException(
+                String.format("Algorithmic token allocation failed: the number of racks in datacenter %s is lower than its replication factor %d.\n" +
+                          "If you are starting a new datacenter, please make sure that the first %d nodes to start are from different racks.\n" +
+                          "If you wish to fall back to random token allocation, please use '" + CassandraRelevantProperties.USE_RANDOM_ALLOCATION_IF_NOT_SUPPORTED + "'.",
+                          dc, replicationStrategy.getReplicationFactor().allReplicas, replicationStrategy.getReplicationFactor().allReplicas));
         }
     }
 
-    private StrategyAdapter getOrCreateStrategy(InetAddressAndPort endpoint)
-    {
-        String dc = replicationStrategy.snitch.getDatacenter(endpoint);
-        String rack = replicationStrategy.snitch.getRack(endpoint);
-        return getOrCreateStrategy(dc, rack);
-    }
-
     private StrategyAdapter getOrCreateStrategy(String dc, String rack)
     {
         return strategyByRackDc.computeIfAbsent(dc, k -> new HashMap<>()).computeIfAbsent(rack, k -> createStrategy(dc, rack));
@@ -236,7 +293,7 @@ private StrategyAdapter createStrategy(String dc, String rack)
 
     private StrategyAdapter createStrategy(final SimpleStrategy rs)
     {
-        return createStrategy(rs.snitch, null, null, rs.getReplicationFactor().allReplicas, false);
+        return createStrategy(snitch, null, null, rs.getReplicationFactor().allReplicas, false);
     }
 
     private StrategyAdapter createStrategy(TokenMetadata tokenMetadata, NetworkTopologyStrategy strategy, String dc, String rack)
@@ -252,32 +309,56 @@ private StrategyAdapter createStrategy(TokenMetadata tokenMetadata, NetworkTopol
         if (replicas <= 1)
         {
             // each node is treated as separate and replicates once
-            return createStrategy(strategy.snitch, dc, null, 1, false);
+            return createStrategy(snitch, dc, null, 1, false);
         }
         else if (racks == replicas)
         {
             // each node is treated as separate and replicates once, with separate allocation rings for each rack
-            return createStrategy(strategy.snitch, dc, rack, 1, false);
+            return createStrategy(snitch, dc, rack, 1, false);
         }
         else if (racks > replicas)
         {
             // group by rack
-            return createStrategy(strategy.snitch, dc, null, replicas, true);
+            return createStrategy(snitch, dc, null, replicas, true);
         }
         else if (racks == 1)
         {
-            return createStrategy(strategy.snitch, dc, null, replicas, false);
+            return createStrategy(snitch, dc, null, replicas, false);
         }
 
         throw new ConfigurationException(String.format("Token allocation failed: the number of racks %d in datacenter %s is lower than its replication factor %d.",
                                                        racks, dc, replicas));
     }
 
+    private StrategyAdapter createRandomStrategy(InetAddressAndPort endpoint)
+    {
+        return new StrategyAdapter(this.tokenMetadata)
+        {
+            @Override
+            public int replicas()
+            {
+                return 1;
+            }
+
+            @Override
+            public Object getGroup(InetAddressAndPort unit)
+            {
+                return unit;
+            }
+
+            @Override
+            public boolean inAllocationRing(InetAddressAndPort other)
+            {
+                return endpoint.equals(other); // Make the algorithm believe this is the only node in the DC so it assigns tokens randomly.
+            }
+        };
+    }
+
     // a null dc will always return true for inAllocationRing(..)
     // a null rack will return true for inAllocationRing(..) for all nodes in the same dc
     private StrategyAdapter createStrategy(IEndpointSnitch snitch, String dc, String rack, int replicas, boolean groupByRack)
     {
-        return new StrategyAdapter()
+        return new StrategyAdapter(this.tokenMetadata)
         {
             @Override
             public int replicas()
diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorBase.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorBase.java
index 3d7e6b96560c..fcd80f9a5f46 100644
--- a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorBase.java
+++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorBase.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Random;
+import java.util.function.Supplier;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -38,14 +39,24 @@ public abstract class TokenAllocatorBase<Unit> implements TokenAllocator<Unit>
     final NavigableMap<Token, Unit> sortedTokens;
     final ReplicationStrategy<Unit> strategy;
     final IPartitioner partitioner;
+    final Supplier<Token> seedTokenSupplier;
 
     protected TokenAllocatorBase(NavigableMap<Token, Unit> sortedTokens,
-                             ReplicationStrategy<Unit> strategy,
-                             IPartitioner partitioner)
+                                 ReplicationStrategy<Unit> strategy,
+                                 IPartitioner partitioner)
+    {
+        this(sortedTokens, strategy, partitioner, partitioner::getRandomToken);
+    }
+
+    protected TokenAllocatorBase(NavigableMap<Token, Unit> sortedTokens,
+                                 ReplicationStrategy<Unit> strategy,
+                                 IPartitioner partitioner,
+                                 Supplier<Token> seedTokenSupplier)
     {
         this.sortedTokens = sortedTokens;
         this.strategy = strategy;
         this.partitioner = partitioner;
+        this.seedTokenSupplier = seedTokenSupplier;
     }
 
     public abstract int getReplicas();
@@ -107,9 +118,10 @@ Collection<Token> generateSplits(Unit newUnit, int numTokens, double minRatio, d
 
         if (sortedTokens.isEmpty())
         {
-            // Select a random start token. This has no effect on distribution, only on where the local ring is "centered".
+            // Select a start token using the configured seedTokenSupplier. By default, the token is random. It can also
+            // be supplied by the subclass. This has no effect on distribution, only on where the local ring is "centered".
             // Using a random start decreases the chances of clash with the tokens of other datacenters in the ring.
-            Token t = partitioner.getRandomToken();
+            Token t = seedTokenSupplier.get();
             tokens.add(t);
             sortedTokens.put(t, newUnit);
         }
diff --git a/src/java/org/apache/cassandra/exceptions/AlreadyExistsException.java b/src/java/org/apache/cassandra/exceptions/AlreadyExistsException.java
index 1829c5cb1f6c..f0b088356dc9 100644
--- a/src/java/org/apache/cassandra/exceptions/AlreadyExistsException.java
+++ b/src/java/org/apache/cassandra/exceptions/AlreadyExistsException.java
@@ -22,7 +22,7 @@ public class AlreadyExistsException extends ConfigurationException
     public final String ksName;
     public final String cfName;
 
-    private AlreadyExistsException(String ksName, String cfName, String msg)
+    public AlreadyExistsException(String ksName, String cfName, String msg)
     {
         super(ExceptionCode.ALREADY_EXISTS, msg);
         this.ksName = ksName;
diff --git a/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java b/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java
index 32cc014da160..e6b49b651f92 100644
--- a/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java
+++ b/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java
@@ -27,7 +27,12 @@ public class CasWriteTimeoutException extends WriteTimeoutException
 
     public CasWriteTimeoutException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor, int contentions)
     {
-        super(writeType, consistency, received, blockFor, String.format("CAS operation timed out: received %d of %d required responses after %d contention retries", received, blockFor, contentions));
+        this(writeType, consistency, received, blockFor, contentions, String.format("CAS operation timed out: received %d of %d required responses after %d contention retries", received, blockFor, contentions));
+    }
+
+    public CasWriteTimeoutException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor, int contentions, String message)
+    {
+        super(writeType, consistency, received, blockFor, message);
         this.contentions = contentions;
     }
 }
diff --git a/src/java/org/apache/cassandra/exceptions/IncompatibleSchemaException.java b/src/java/org/apache/cassandra/exceptions/IncompatibleSchemaException.java
index fe3a167b6f72..d2d3f00e0c31 100644
--- a/src/java/org/apache/cassandra/exceptions/IncompatibleSchemaException.java
+++ b/src/java/org/apache/cassandra/exceptions/IncompatibleSchemaException.java
@@ -19,10 +19,18 @@
 
 import java.io.IOException;
 
-public class IncompatibleSchemaException extends IOException
+public class IncompatibleSchemaException extends IOException implements InternalRequestExecutionException
 {
-    public IncompatibleSchemaException(String msg)
+    private final RequestFailureReason reason;
+
+    public IncompatibleSchemaException(RequestFailureReason reason, String msg)
     {
         super(msg);
+        this.reason = reason;
+    }
+
+    public RequestFailureReason getReason()
+    {
+        return reason;
     }
 }
diff --git a/src/java/org/apache/cassandra/exceptions/InternalRequestExecutionException.java b/src/java/org/apache/cassandra/exceptions/InternalRequestExecutionException.java
new file mode 100644
index 000000000000..e4cf5eeef7c0
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/InternalRequestExecutionException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.exceptions;
+
+/**
+ * Indicates an "expected" exception during the execution of a request on a
+ * replica.
+ * <p>
+ * This groups exceptions that can happen on replicas but aren't unexpected in
+ * the sense that the circumstance for it happening is understood and a result
+ * of a user error, which we simply couldn't detect on the coordinator.
+ * <p>
+ * Such failures include an index query while the index is not built yet, or a
+ * 'TombstoneOverwhelmingException' for instance.
+ */
+public interface InternalRequestExecutionException
+{
+    RequestFailureReason getReason();
+}
diff --git a/src/java/org/apache/cassandra/exceptions/InvalidColumnTypeException.java b/src/java/org/apache/cassandra/exceptions/InvalidColumnTypeException.java
new file mode 100644
index 000000000000..164a748ff339
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/InvalidColumnTypeException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.exceptions;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.marshal.AbstractType;
+
+/**
+ * Exception thrown when a configured column type is invalid.
+ */
+public class InvalidColumnTypeException extends ConfigurationException
+{
+    public InvalidColumnTypeException(ByteBuffer name,
+                                      AbstractType<?> invalidType,
+                                      String reason)
+    {
+        super(msg(name, invalidType, reason));
+    }
+
+    private static String msg(ByteBuffer name,
+                              AbstractType<?> invalidType,
+                              String reason)
+    {
+        return String.format("Invalid type %s for column %s: %s",
+                             invalidType.asCQL3Type().toSchemaString(),
+                             ColumnIdentifier.toCQLString(name),
+                             reason);
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
index ae5566104572..02ce028dc014 100644
--- a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
+++ b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
@@ -18,14 +18,18 @@
 package org.apache.cassandra.exceptions;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
+import org.apache.cassandra.index.IndexBuildInProgressException;
+import org.apache.cassandra.index.IndexNotAvailableException;
+import org.apache.cassandra.index.sai.utils.AbortedOperationException;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.vint.VIntCoding;
 
-import static java.lang.Math.max;
 import static org.apache.cassandra.net.MessagingService.VERSION_40;
 
 public enum RequestFailureReason
@@ -37,7 +41,13 @@ public enum RequestFailureReason
     READ_SIZE                (4),
     NODE_DOWN                (5),
     INDEX_NOT_AVAILABLE      (6),
-    READ_TOO_MANY_INDEXES    (7);
+    READ_TOO_MANY_INDEXES    (7),
+    // The following codes are not present in Apache Cassandra's RequestFailureReason
+    // We should add new codes in HCD (which do not exist in Apache Cassandra) only with big numbers, to avoid conflicts
+    UNKNOWN_COLUMN           (500),
+    UNKNOWN_TABLE            (501),
+    REMOTE_STORAGE_FAILURE   (502),
+    INDEX_BUILD_IN_PROGRESS  (503);
 
     public static final Serializer serializer = new Serializer();
 
@@ -48,26 +58,36 @@ public enum RequestFailureReason
         this.code = code;
     }
 
-    private static final RequestFailureReason[] codeToReasonMap;
+    public int codeForNativeProtocol()
+    {
+        // We explicitly indicated in the protocol spec that drivers should not error out on unknown code, and we
+        // currently support a superset of the OSS codes, so we don't yet worry about the version.
+        return code;
+    }
+
+    private static final Map<Integer, RequestFailureReason> codeToReasonMap = new HashMap<>();
+    private static final Map<Class<? extends Throwable>, RequestFailureReason> exceptionToReasonMap = new HashMap<>();
 
     static
     {
         RequestFailureReason[] reasons = values();
 
-        int max = -1;
-        for (RequestFailureReason r : reasons)
-            max = max(r.code, max);
-
-        RequestFailureReason[] codeMap = new RequestFailureReason[max + 1];
-
         for (RequestFailureReason reason : reasons)
         {
-            if (codeMap[reason.code] != null)
+            if (codeToReasonMap.put(reason.code, reason) != null)
                 throw new RuntimeException("Two RequestFailureReason-s that map to the same code: " + reason.code);
-            codeMap[reason.code] = reason;
         }
 
-        codeToReasonMap = codeMap;
+        exceptionToReasonMap.put(TombstoneOverwhelmingException.class, READ_TOO_MANY_TOMBSTONES);
+        exceptionToReasonMap.put(IncompatibleSchemaException.class, INCOMPATIBLE_SCHEMA);
+        exceptionToReasonMap.put(AbortedOperationException.class, TIMEOUT);
+        exceptionToReasonMap.put(IndexNotAvailableException.class, INDEX_NOT_AVAILABLE);
+        exceptionToReasonMap.put(UnknownColumnException.class, UNKNOWN_COLUMN);
+        exceptionToReasonMap.put(UnknownTableException.class, UNKNOWN_TABLE);
+        exceptionToReasonMap.put(IndexBuildInProgressException.class, INDEX_BUILD_IN_PROGRESS);
+
+        if (exceptionToReasonMap.size() != reasons.length-5)
+            throw new RuntimeException("A new RequestFailureReasons was probably added and you may need to update the exceptionToReasonMap");
     }
 
     public static RequestFailureReason fromCode(int code)
@@ -76,16 +96,18 @@ public static RequestFailureReason fromCode(int code)
             throw new IllegalArgumentException("RequestFailureReason code must be non-negative (got " + code + ')');
 
         // be forgiving and return UNKNOWN if we aren't aware of the code - for forward compatibility
-        return code < codeToReasonMap.length ? codeToReasonMap[code] : UNKNOWN;
+        return codeToReasonMap.getOrDefault(code, UNKNOWN);
     }
 
     public static RequestFailureReason forException(Throwable t)
     {
-        if (t instanceof TombstoneOverwhelmingException)
-            return READ_TOO_MANY_TOMBSTONES;
+        RequestFailureReason r = exceptionToReasonMap.get(t.getClass());
+        if (r != null)
+            return r;
 
-        if (t instanceof IncompatibleSchemaException)
-            return INCOMPATIBLE_SCHEMA;
+        for (Map.Entry<Class<? extends Throwable>, RequestFailureReason> entry : exceptionToReasonMap.entrySet())
+            if (entry.getKey().isInstance(t))
+                return entry.getValue();
 
         return UNKNOWN;
     }
diff --git a/src/java/org/apache/cassandra/exceptions/UnknownColumnException.java b/src/java/org/apache/cassandra/exceptions/UnknownColumnException.java
index 93a464e77e02..90548af8e8df 100644
--- a/src/java/org/apache/cassandra/exceptions/UnknownColumnException.java
+++ b/src/java/org/apache/cassandra/exceptions/UnknownColumnException.java
@@ -21,6 +21,6 @@ public final class UnknownColumnException extends IncompatibleSchemaException
 {
     public UnknownColumnException(String msg)
     {
-        super(msg);
+        super(RequestFailureReason.UNKNOWN_COLUMN, msg);
     }
 }
diff --git a/src/java/org/apache/cassandra/exceptions/UnknownKeyspaceException.java b/src/java/org/apache/cassandra/exceptions/UnknownKeyspaceException.java
new file mode 100644
index 000000000000..0fc4f988d278
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/UnknownKeyspaceException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.exceptions;
+
+public class UnknownKeyspaceException extends RuntimeException implements InternalRequestExecutionException
+{
+    public final String keyspaceName;
+
+    public UnknownKeyspaceException(String keyspaceName)
+    {
+        super("Could not find a keyspace " + keyspaceName);
+        this.keyspaceName = keyspaceName;
+    }
+
+    @Override
+    public RequestFailureReason getReason()
+    {
+        return RequestFailureReason.INCOMPATIBLE_SCHEMA;
+    }
+}
diff --git a/src/java/org/apache/cassandra/exceptions/UnknownTableException.java b/src/java/org/apache/cassandra/exceptions/UnknownTableException.java
index 3e9c77537061..128238631d75 100644
--- a/src/java/org/apache/cassandra/exceptions/UnknownTableException.java
+++ b/src/java/org/apache/cassandra/exceptions/UnknownTableException.java
@@ -25,7 +25,7 @@ public class UnknownTableException extends IncompatibleSchemaException
 
     public UnknownTableException(String msg, TableId id)
     {
-        super(msg);
+        super(RequestFailureReason.UNKNOWN_TABLE, msg);
         this.id = id;
     }
 }
diff --git a/src/java/org/apache/cassandra/gms/EndpointState.java b/src/java/org/apache/cassandra/gms/EndpointState.java
index 49847a3c710c..64bba8b03fd9 100644
--- a/src/java/org/apache/cassandra/gms/EndpointState.java
+++ b/src/java/org/apache/cassandra/gms/EndpointState.java
@@ -17,11 +17,17 @@
  */
 package org.apache.cassandra.gms;
 
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
-
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -30,14 +36,28 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.CassandraRelevantProperties;
+import com.google.common.base.Preconditions;
+
+import net.openhft.chronicle.core.util.ThrowingConsumer;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.nodes.LocalInfo;
+import org.apache.cassandra.nodes.NodeInfo;
 import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.NullableSerializer;
 
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.gms.ApplicationState.SCHEMA;
+import static org.apache.cassandra.gms.ApplicationState.TOKENS;
 
 /**
  * This abstraction represents both the HeartBeatState and the ApplicationState in an EndpointState
@@ -58,10 +78,11 @@ public class EndpointState
     /* fields below do not get serialized */
     private volatile long updateTimestamp;
     private volatile boolean isAlive;
+    private volatile Consumer<UnaryOperator<NodeInfo<?>>> updater;
 
     public EndpointState(HeartBeatState initialHbState)
     {
-        this(initialHbState, new EnumMap<ApplicationState, VersionedValue>(ApplicationState.class));
+        this(initialHbState, new EnumMap<>(ApplicationState.class));
     }
 
     public EndpointState(EndpointState other)
@@ -72,7 +93,7 @@ public EndpointState(EndpointState other)
     EndpointState(HeartBeatState initialHbState, Map<ApplicationState, VersionedValue> states)
     {
         hbState = initialHbState;
-        applicationState = new AtomicReference<Map<ApplicationState, VersionedValue>>(new EnumMap<>(states));
+        applicationState = new AtomicReference<>(new EnumMap<>(states));
         updateTimestamp = nanoTime();
         isAlive = true;
     }
@@ -83,6 +104,53 @@ public HeartBeatState getHeartBeatState()
         return hbState;
     }
 
+    public synchronized void maybeSetUpdater(Consumer<UnaryOperator<NodeInfo<?>>> updater)
+    {
+        Preconditions.checkNotNull(updater);
+        if (this.updater == null)
+            this.updater = updater;
+    }
+
+    public synchronized void maybeUpdate()
+    {
+        if (this.updater != null)
+            update(states());
+    }
+
+    public synchronized void maybeRemoveUpdater()
+    {
+        this.updater = null;
+    }
+
+    public void update(Set<Map.Entry<ApplicationState, VersionedValue>> entries)
+    {
+        Consumer<UnaryOperator<NodeInfo<?>>> updater = this.updater;
+        if (updater == null)
+            return;
+
+        List<ThrowingConsumer<NodeInfo<?>, UnknownHostException>> allUpdates = entries.stream()
+                                                                                      .map(e -> updateNodeInfo(e.getKey(), e.getValue()))
+                                                                                      .filter(Objects::nonNull)
+                                                                                      .collect(Collectors.toList());
+
+        if (!allUpdates.isEmpty())
+        {
+            updater.accept(info -> {
+                allUpdates.forEach(update -> {
+                    try
+                    {
+                        update.accept(info);
+                    }
+                    catch (UnknownHostException e)
+                    {
+                        throw Throwables.cleaned(e);
+                    }
+                });
+                return info;
+            });
+        }
+    }
+
     void setHeartBeatState(HeartBeatState newHbState)
     {
         updateTimestamp();
@@ -114,7 +182,7 @@ public void addApplicationStates(Map<ApplicationState, VersionedValue> values)
         addApplicationStates(values.entrySet());
     }
 
-    public void addApplicationStates(Set<Map.Entry<ApplicationState, VersionedValue>> values)
+    public synchronized void addApplicationStates(Set<Map.Entry<ApplicationState, VersionedValue>> values)
     {
         while (true)
         {
@@ -125,7 +193,16 @@ public void addApplicationStates(Set<Map.Entry<ApplicationState, VersionedValue>
                 copy.put(value.getKey(), value.getValue());
 
             if (applicationState.compareAndSet(orig, copy))
+            {
+                EnumMap<ApplicationState, VersionedValue> diff = new EnumMap<>(copy);
+                for (Map.Entry<ApplicationState, VersionedValue> entry : copy.entrySet())
+                {
+                    if (Objects.equals(entry.getValue(), orig.get(entry.getKey())))
+                        diff.remove(entry.getKey());
+                }
+                update(diff.entrySet());
                 return;
+            }
         }
     }
 
@@ -155,22 +232,23 @@ private boolean hasLegacyFields()
     private static Map<ApplicationState, VersionedValue> filterMajorVersion3LegacyApplicationStates(Map<ApplicationState, VersionedValue> states)
     {
         return states.entrySet().stream().filter(entry -> {
-                // Filter out pre-4.0 versions of data for more complete 4.0 versions
-                switch (entry.getKey())
-                {
-                    case INTERNAL_IP:
-                        return !states.containsKey(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
-                    case STATUS:
-                        return !states.containsKey(ApplicationState.STATUS_WITH_PORT);
-                    case RPC_ADDRESS:
-                        return !states.containsKey(ApplicationState.NATIVE_ADDRESS_AND_PORT);
-                    default:
-                        return true;
-                }
-            }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+            // Filter out pre-4.0 versions of data for more complete 4.0 versions
+            switch (entry.getKey())
+            {
+                case INTERNAL_IP:
+                    return !states.containsKey(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+                case STATUS:
+                    return !states.containsKey(ApplicationState.STATUS_WITH_PORT);
+                case RPC_ADDRESS:
+                    return !states.containsKey(ApplicationState.NATIVE_ADDRESS_AND_PORT);
+                default:
+                    return true;
+            }
+        }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
 
     /* getters and setters */
+
     /**
      * @return System.nanoTime() when state was updated last time.
      */
@@ -262,19 +340,65 @@ public String getStatus()
     @Nullable
     public UUID getSchemaVersion()
     {
-        VersionedValue applicationState = getApplicationState(ApplicationState.SCHEMA);
+        VersionedValue applicationState = getApplicationState(SCHEMA);
         return applicationState != null
                ? UUID.fromString(applicationState.value)
                : null;
     }
 
     @Nullable
-    public CassandraVersion getReleaseVersion()
+    public Collection<Token> getTokens(IPartitioner partitioner)
     {
-        VersionedValue applicationState = getApplicationState(ApplicationState.RELEASE_VERSION);
-        return applicationState != null
-               ? new CassandraVersion(applicationState.value)
-               : null;
+        VersionedValue value = getApplicationState(TOKENS);
+        return value != null ? getTokens(partitioner, value) : null;
+    }
+
+    @Nonnull
+    private static Collection<Token> getTokens(IPartitioner partitioner, @Nonnull VersionedValue value)
+    {
+        try
+        {
+            return TokenSerializer.deserialize(partitioner, new DataInputStream(new ByteArrayInputStream(value.toBytes())));
+        }
+        catch (IOException e)
+        {
+            throw Throwables.unchecked(e);
+        }
+    }
+
+    private static ThrowingConsumer<NodeInfo<?>, UnknownHostException> updateNodeInfo(ApplicationState state, VersionedValue value)
+    {
+        switch (state)
+        {
+            case TOKENS:
+                return info -> info.setTokens(getTokens(DatabaseDescriptor.getPartitioner(), value));
+            case HOST_ID:
+                return info -> info.setHostId(UUID.fromString(value.value));
+            case RELEASE_VERSION:
+                return info -> info.setReleaseVersion(new CassandraVersion(value.value));
+            case DC:
+                return info -> info.setDataCenter(value.value);
+            case RACK:
+                return info -> info.setRack(value.value);
+            case SCHEMA:
+                return info -> info.setSchemaVersion(UUID.fromString(value.value));
+            case INTERNAL_IP:
+                return info -> {
+                    if (info instanceof LocalInfo)
+                        ((LocalInfo) info).setListenAddressOnly(InetAddress.getByName(value.value), FBUtilities.getLocalAddressAndPort().getPort());
+                };
+            case INTERNAL_ADDRESS_AND_PORT:
+                return info -> {
+                    if (info instanceof LocalInfo)
+                        ((LocalInfo) info).setListenAddressAndPort(InetAddressAndPort.getByName(value.value));
+                };
+            case RPC_ADDRESS:
+                return info -> info.setNativeTransportAddressOnly(InetAddress.getByName(value.value), DatabaseDescriptor.getNativeTransportPort());
+            case NATIVE_ADDRESS_AND_PORT:
+                return info -> info.setNativeTransportAddressAndPort(InetAddressAndPort.getByName(value.value));
+            default:
+                return null;
+        }
     }
 
     public String toString()
diff --git a/src/java/org/apache/cassandra/gms/FailureDetector.java b/src/java/org/apache/cassandra/gms/FailureDetector.java
index 9cf7ad0392f7..6fb901b0ab00 100644
--- a/src/java/org/apache/cassandra/gms/FailureDetector.java
+++ b/src/java/org/apache/cassandra/gms/FailureDetector.java
@@ -32,7 +32,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Predicate;
 import javax.management.openmbean.CompositeData;
 import javax.management.openmbean.CompositeDataSupport;
 import javax.management.openmbean.CompositeType;
@@ -49,7 +48,6 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MBeanWrapper;
 
@@ -57,7 +55,6 @@
 import static org.apache.cassandra.config.CassandraRelevantProperties.FD_MAX_INTERVAL_MS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR;
 import static org.apache.cassandra.config.CassandraRelevantProperties.MAX_LOCAL_PAUSE_IN_MS;
-import static org.apache.cassandra.config.DatabaseDescriptor.newFailureDetector;
 import static org.apache.cassandra.utils.MonotonicClock.Global.preciseTime;
 
 /**
@@ -87,10 +84,6 @@ private static long getMaxLocalPause()
         return pause * 1000000L;
     }
 
-    public static final IFailureDetector instance = newFailureDetector();
-    public static final Predicate<InetAddressAndPort> isEndpointAlive = instance::isAlive;
-    public static final Predicate<Replica> isReplicaAlive = r -> isEndpointAlive.test(r.endpoint());
-
     // this is useless except to provide backwards compatibility in phi_convict_threshold,
     // because everyone seems pretty accustomed to the default of 8, and users who have
     // already tuned their phi_convict_threshold for their own environments won't need to
@@ -304,8 +297,8 @@ public boolean isAlive(InetAddressAndPort ep)
         // we could assert not-null, but having isAlive fail screws a node over so badly that
         // it's worth being defensive here so minor bugs don't cause disproportionate
         // badness.  (See CASSANDRA-1463 for an example).
-        if (epState == null)
-            logger.error("Unknown endpoint: " + ep, new IllegalArgumentException(""));
+        if (epState == null && Gossiper.instance.isEnabled())
+            logger.error("Unknown endpoint: " + ep, new IllegalArgumentException("Unknown endpoint: " + ep));
         return epState != null && epState.isAlive();
     }
 
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index 4ab5928cf991..ba950e4e320d 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -17,6 +17,9 @@
  */
 package org.apache.cassandra.gms;
 
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -41,10 +44,12 @@
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.BooleanSupplier;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -55,6 +60,7 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.utils.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -64,6 +70,7 @@
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
@@ -71,21 +78,16 @@
 import org.apache.cassandra.net.NoPayload;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.CassandraVersion;
-import org.apache.cassandra.utils.ExecutorUtils;
-import org.apache.cassandra.utils.ExpiringMemoizingSupplier;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.MBeanWrapper;
-import org.apache.cassandra.utils.NoSpamLogger;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.RecomputingSupplier;
 import org.apache.cassandra.utils.concurrent.NotScheduledFuture;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_GOSSIP_ENDPOINT_REMOVAL;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CLUSTER_VERSION_PROVIDER_CLASS_NAME;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CLUSTER_VERSION_PROVIDER_MIN_STABLE_DURATION;
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIPER_QUARANTINE_DELAY;
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIPER_SKIP_WAITING_TO_SETTLE;
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
@@ -192,87 +194,145 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean,
 
     private volatile long lastProcessedMessageAt = currentTimeMillis();
 
-    /**
-     * This property is initially set to {@code true} which means that we have no information about the other nodes.
-     * Once all nodes are on at least this node version, it becomes {@code false}, which means that we are not
-     * upgrading from the previous version (major, minor).
-     *
-     * This property and anything that checks it should be removed in 5.0
-     */
-    private volatile boolean upgradeInProgressPossible = true;
     private volatile boolean hasNodeWithUnknownVersion = false;
 
+    @VisibleForTesting
     public void clearUnsafe()
     {
         unreachableEndpoints.clear();
         liveEndpoints.clear();
         justRemovedEndpoints.clear();
         expireTimeEndpointMap.clear();
+        endpointStateMap.values().forEach(EndpointState::maybeRemoveUpdater);
         endpointStateMap.clear();
         endpointShadowStateMap.clear();
         seedsInShadowRound.clear();
+        Nodes.peers().get().forEach(peer -> Nodes.peers().remove(peer.getPeerAddressAndPort(), true, true));
     }
 
-    // returns true when the node does not know the existence of other nodes.
-    private static boolean isLoneNode(Map<InetAddressAndPort, EndpointState> epStates)
+    private class DefaultClusterVersionProvider implements IClusterVersionProvider
     {
-        return epStates.isEmpty() || epStates.keySet().equals(Collections.singleton(FBUtilities.getBroadcastAddressAndPort()));
-    }
+        // -1L means that the cluster may be in upgrading state; positive value is the timestamp when the cluster
+        // was detected as fully upgraded
+        private final AtomicLong notUpgradingSinceMillis = new AtomicLong(-1L);
 
-    private static final ExpiringMemoizingSupplier.Memoized<CassandraVersion> NO_UPGRADE_IN_PROGRESS = new ExpiringMemoizingSupplier.Memoized<>(null);
-    private static final ExpiringMemoizingSupplier.NotMemoized<CassandraVersion> CURRENT_NODE_VERSION = new ExpiringMemoizingSupplier.NotMemoized<>(SystemKeyspace.CURRENT_VERSION);
-    final Supplier<ExpiringMemoizingSupplier.ReturnValue<CassandraVersion>> upgradeFromVersionSupplier = () ->
-    {
-        // Once there are no prior version nodes we don't need to keep rechecking
-        if (!upgradeInProgressPossible)
-            return NO_UPGRADE_IN_PROGRESS;
+        // minimum time that needs to pass after the cluster is detected as fully upgraded
+        // to report that there is no upgrade in progress
+        private final long MIN_STABLE_DURATION_MS = CLUSTER_VERSION_PROVIDER_MIN_STABLE_DURATION.getLong();
 
-        CassandraVersion minVersion = SystemKeyspace.CURRENT_VERSION;
+        private final Supplier<ExpiringMemoizingSupplier.ReturnValue<CassandraVersion>> upgradeFromVersionSupplier = () ->
+        {
+            long notUpgradingSinceMillis = this.notUpgradingSinceMillis.get();
+            long stableDuration = notUpgradingSinceMillis < 0 ? -1 : Clock.Global.currentTimeMillis() - notUpgradingSinceMillis;
 
-        // Skip the round if the gossiper has not started yet
-        // Otherwise, upgradeInProgressPossible can be set to false wrongly.
-        // If we don't know any epstate we don't know anything about the cluster.
-        // If we only know about ourselves, we can assume that version is CURRENT_VERSION
-        if (!isEnabled() || isLoneNode(endpointStateMap))
-            return CURRENT_NODE_VERSION;
+            // The cluster is upgraded
+            if (stableDuration > 0)
+                return new ExpiringMemoizingSupplier.Memoized<>(SystemKeyspace.CURRENT_VERSION);
 
-        // Check the release version of all the peers it heard of. Not necessary the peer that it has/had contacted with.
-        hasNodeWithUnknownVersion = false;
-        for (Entry<InetAddressAndPort, EndpointState> entry : endpointStateMap.entrySet())
-        {
+            if (!isEnabled())
+            {
+                // start the stabilisation period by setting the current timestamp in notUpgradingSinceMillis
+                // if Gossiper is going to be enabled, it will be enabled quickly
+                if (DatabaseDescriptor.isDaemonInitialized())
+                {
+                    if (CassandraRelevantProperties.CLUSTER_VERSION_PROVIDER_SKIP_WAIT_FOR_GOSSIP.getBoolean())
+                        this.notUpgradingSinceMillis.compareAndSet(notUpgradingSinceMillis, Clock.Global.currentTimeMillis());
 
-            if (justRemovedEndpoints.containsKey(entry.getKey()))
-                continue;
+                    return new ExpiringMemoizingSupplier.NotMemoized<>(SystemKeyspace.CURRENT_VERSION);
+                }
+                else
+                {
+                    // it is not going to be enabled because we are not running in server mode
+                    if (this.notUpgradingSinceMillis.compareAndSet(notUpgradingSinceMillis, 0)) // set 0 to make it stable
+                        return new ExpiringMemoizingSupplier.Memoized<>(SystemKeyspace.CURRENT_VERSION);
+                    else
+                        return new ExpiringMemoizingSupplier.NotMemoized<>(SystemKeyspace.CURRENT_VERSION);
+                }
+            }
 
-            CassandraVersion version = getReleaseVersion(entry.getKey());
+            // Check the release version of all the peers it heard of. Not necessary the peer that it has/had contacted with.
+            CassandraVersion minVersion = SystemKeyspace.CURRENT_VERSION;
+            hasNodeWithUnknownVersion = false;
+            for (Entry<InetAddressAndPort, EndpointState> entry : endpointStateMap.entrySet())
+            {
 
-            // if it is dead state, we skip the version check
-            if (isDeadState(entry.getValue()))
-                continue;
-            //Raced with changes to gossip state, wait until next iteration
-            if (version == null)
-                hasNodeWithUnknownVersion = true;
-            else if (version.compareTo(minVersion) < 0)
-                minVersion = version;
-        }
+                if (justRemovedEndpoints.containsKey(entry.getKey()))
+                    continue;
 
-        if (minVersion.compareTo(SystemKeyspace.CURRENT_VERSION) < 0)
-            return new ExpiringMemoizingSupplier.Memoized<>(minVersion);
+                CassandraVersion version = getReleaseVersion(entry.getKey());
 
-        if (hasNodeWithUnknownVersion)
-            return new ExpiringMemoizingSupplier.NotMemoized<>(minVersion);
+                // if it is dead state, we skip the version check
+                if (isDeadState(entry.getValue()))
+                    continue;
+                //Raced with changes to gossip state, wait until next iteration
+                if (version == null)
+                    hasNodeWithUnknownVersion = true;
+                else if (version.compareTo(minVersion) < 0)
+                    minVersion = version;
+            }
 
-        upgradeInProgressPossible = false;
-        return NO_UPGRADE_IN_PROGRESS;
-    };
+            // remember the minimum version for the expiration duration
+            if (minVersion.compareTo(SystemKeyspace.CURRENT_VERSION) < 0)
+                return new ExpiringMemoizingSupplier.Memoized<>(minVersion);
 
-    private final Supplier<CassandraVersion> upgradeFromVersionMemoized = ExpiringMemoizingSupplier.memoizeWithExpiration(upgradeFromVersionSupplier, 1, TimeUnit.MINUTES);
+            // don't remember the minimum version and recheck whenever requested
+            if (hasNodeWithUnknownVersion)
+                return new ExpiringMemoizingSupplier.NotMemoized<>(minVersion);
 
+            // all hosts have known versions and == CURRENT_VERSION, we can stop checking - the cluster is fully upgraded
+            // start the stability period by setting the current timestamp in notUpgradingSinceMillis
+            if (this.notUpgradingSinceMillis.compareAndSet(notUpgradingSinceMillis, Clock.Global.currentTimeMillis()))
+                return new ExpiringMemoizingSupplier.Memoized<>(minVersion);
+            else
+                return new ExpiringMemoizingSupplier.NotMemoized<>(minVersion);
+        };
+
+        private final ExpiringMemoizingSupplier<CassandraVersion> minVersionMemoized = ExpiringMemoizingSupplier.memoizeWithExpiration(upgradeFromVersionSupplier, 60, TimeUnit.SECONDS);
+
+        @Override
+        public void reset()
+        {
+            notUpgradingSinceMillis.set(-1L);
+            minVersionMemoized.expire();
+        }
+
+        @Override
+        public CassandraVersion getMinClusterVersion()
+        {
+            return minVersionMemoized.get();
+        }
+
+        @Override
+        public boolean isUpgradeInProgress()
+        {
+            long notUpgradingSince = this.notUpgradingSinceMillis.get();
+            long stableDuration = notUpgradingSince < 0 ? -1 : Clock.Global.currentTimeMillis() - notUpgradingSince;
+            return stableDuration < MIN_STABLE_DURATION_MS;
+        }
+    }
+
+    // For testing only
     @VisibleForTesting
-    public void expireUpgradeFromVersion()
+    public void setNotUpgradingSinceMillisUnsafe(long notUpgradingSinceMillis)
     {
-        upgradeInProgressPossible = true;
-        ((ExpiringMemoizingSupplier<CassandraVersion>) upgradeFromVersionMemoized).expire();
+        ((DefaultClusterVersionProvider) clusterVersionProvider).notUpgradingSinceMillis.set(notUpgradingSinceMillis);
+    }
+
+    @VisibleForTesting
+    public final IClusterVersionProvider clusterVersionProvider;
+
+    private static IClusterVersionProvider maybeCustomClusterVersionProvider()
+    {
+        IClusterVersionProvider clusterVersionProvider = null;
+        String className = CLUSTER_VERSION_PROVIDER_CLASS_NAME.getString();
+        if (className != null)
+        {
+            clusterVersionProvider = FBUtilities.instanceOrConstruct(className,"Custom implementation of " + IClusterVersionProvider.class.getSimpleName());
+            if (clusterVersionProvider != null)
+                logger.info("Using custom implementation of {}: {} - {}", IClusterVersionProvider.class.getSimpleName(), className, clusterVersionProvider);
+        }
+
+        return clusterVersionProvider;
     }
 
     private static final boolean disableThreadValidation = GOSSIP_DISABLE_THREAD_VALIDATION.getBoolean();
@@ -394,13 +454,19 @@ public void run()
         }
     }
 
-    private final RecomputingSupplier<CassandraVersion> minVersionSupplier = new RecomputingSupplier<>(this::computeMinVersion, executor);
+    public Gossiper(boolean registerJmx)
+    {
+        this(registerJmx, maybeCustomClusterVersionProvider());
+    }
 
     @VisibleForTesting
-    public Gossiper(boolean registerJmx)
+    public Gossiper(boolean registerJmx, IClusterVersionProvider customClusterVersionProvider)
     {
+        this.clusterVersionProvider = Objects.requireNonNullElseGet(customClusterVersionProvider, DefaultClusterVersionProvider::new);
+        logger.info("Using cluster version provider {}: {}", this.clusterVersionProvider.getClass().getName(), clusterVersionProvider);
+
         /* register with the Failure Detector for receiving Failure detector events */
-        FailureDetector.instance.registerFailureDetectionEventListener(this);
+        IFailureDetector.instance.registerFailureDetectionEventListener(this);
 
         // Register this instance with JMX
         if (registerJmx)
@@ -410,26 +476,29 @@ public Gossiper(boolean registerJmx)
 
         subscribers.add(new IEndpointStateChangeSubscriber()
         {
+            @Override
             public void onJoin(InetAddressAndPort endpoint, EndpointState state)
-	    {
+            {
                 maybeRecompute(state);
             }
 
+            @Override
             public void onAlive(InetAddressAndPort endpoint, EndpointState state)
-	    {
+            {
                 maybeRecompute(state);
             }
 
             private void maybeRecompute(EndpointState state)
-	    {
+            {
                 if (state.getApplicationState(ApplicationState.RELEASE_VERSION) != null)
-                    minVersionSupplier.recompute();
+                    Gossiper.this.clusterVersionProvider.reset();
             }
 
+            @Override
             public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
             {
                 if (state == ApplicationState.RELEASE_VERSION)
-                    minVersionSupplier.recompute();
+                    Gossiper.this.clusterVersionProvider.reset();
             }
         });
     }
@@ -640,7 +709,7 @@ protected void markAsShutdown(InetAddressAndPort endpoint)
         epState.addApplicationState(ApplicationState.RPC_READY, StorageService.instance.valueFactory.rpcReady(false));
         epState.getHeartBeatState().forceHighestPossibleVersionUnsafe();
         markDead(endpoint, epState);
-        FailureDetector.instance.forceConviction(endpoint);
+        IFailureDetector.instance.forceConviction(endpoint);
         GossiperDiagnostics.markedAsShutdown(this, endpoint);
         for (IEndpointStateChangeSubscriber subscriber : subscribers)
             subscriber.onChange(endpoint, ApplicationState.STATUS_WITH_PORT, shutdown);
@@ -697,9 +766,10 @@ private void evictFromMembership(InetAddressAndPort endpoint)
     {
         checkProperThreadForStateMutation();
         unreachableEndpoints.remove(endpoint);
-        endpointStateMap.remove(endpoint);
+        removeEndpointState(endpoint);
+        Nodes.peers().remove(endpoint, true, true);
         expireTimeEndpointMap.remove(endpoint);
-        FailureDetector.instance.remove(endpoint);
+        IFailureDetector.instance.remove(endpoint);
         quarantineEndpoint(endpoint);
         if (logger.isDebugEnabled())
             logger.debug("evicting {} from gossip", endpoint);
@@ -728,6 +798,10 @@ public void removeEndpoint(InetAddressAndPort endpoint)
         if (disableEndpointRemoval)
             return;
 
+        endpointStateMap.computeIfPresent(endpoint, (key, value) -> {
+            value.maybeRemoveUpdater();
+            return value;
+        });
         liveEndpoints.remove(endpoint);
         unreachableEndpoints.remove(endpoint);
         MessagingService.instance().versions.reset(endpoint);
@@ -865,7 +939,7 @@ public void advertiseRemoving(InetAddressAndPort endpoint, UUID hostId, UUID loc
         states.put(ApplicationState.STATUS, StorageService.instance.valueFactory.removingNonlocal(hostId));
         states.put(ApplicationState.REMOVAL_COORDINATOR, StorageService.instance.valueFactory.removalCoordinator(localHostId));
         epState.addApplicationStates(states);
-        endpointStateMap.put(endpoint, epState);
+        putEndpointState(endpoint, epState);
     }
 
     /**
@@ -885,7 +959,7 @@ public void advertiseTokenRemoved(InetAddressAndPort endpoint, UUID hostId)
         epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.removedNonlocal(hostId, expireTime));
         logger.info("Completing removal of {}", endpoint);
         addExpireTimeForEndpoint(endpoint, expireTime);
-        endpointStateMap.put(endpoint, epState);
+        putEndpointState(endpoint, epState);
         // ensure at least one gossip round occurs before returning
         Uninterruptibles.sleepUninterruptibly(intervalInMillis * 2, TimeUnit.MILLISECONDS);
     }
@@ -957,9 +1031,128 @@ else if (newState.getHeartBeatState().getHeartBeatVersion() != heartbeat)
         });
     }
 
-    public boolean isKnownEndpoint(InetAddressAndPort endpoint)
+    public void reviveEndpoint(String address) throws UnknownHostException
     {
-        return endpointStateMap.containsKey(endpoint);
+        InetAddressAndPort endpoint = InetAddressAndPort.getByName(address);
+        EndpointState epState = endpointStateMap.get(endpoint);
+        logger.warn("Reviving {} via gossip", endpoint);
+
+        if (epState == null)
+            throw new RuntimeException("Cannot revive endpoint " + endpoint + ": no endpoint-state");
+
+        int generation = epState.getHeartBeatState().getGeneration();
+        int heartbeat = epState.getHeartBeatState().getHeartBeatVersion();
+
+        logger.info("Have endpoint-state for {}: status={}, generation={}, heartbeat={}",
+                endpoint, epState.getStatus(), generation, heartbeat);
+
+        if (!isSilentShutdownState(epState))
+            throw new RuntimeException("Cannot revive endpoint " + endpoint + ": not in a (silent) shutdown state: " + epState.getStatus());
+
+        if (FailureDetector.instance.isAlive(endpoint))
+            throw new RuntimeException("Cannot revive endpoint " + endpoint + ": still alive (failure-detector)");
+
+        logger.info("Sleeping for {}ms to ensure {} does not change", StorageService.RING_DELAY_MILLIS, endpoint);
+        Uninterruptibles.sleepUninterruptibly(StorageService.RING_DELAY_MILLIS, TimeUnit.MILLISECONDS);
+        // make sure the endpoint state did not change
+        EndpointState newState = endpointStateMap.get(endpoint);
+        if (newState == null)
+            throw new RuntimeException("Cannot revive endpoint " + endpoint + ": endpoint-state disappeared");
+        if (newState.getHeartBeatState().getGeneration() != generation)
+            throw new RuntimeException("Cannot revive endpoint " + endpoint + ": still alive, generation changed while trying to reviving it");
+        if (newState.getHeartBeatState().getHeartBeatVersion() != heartbeat)
+            throw new RuntimeException("Cannot revive endpoint " + endpoint + ": still alive, heartbeat changed while trying to reviving it");
+
+        epState.updateTimestamp(); // make sure we don't evict it too soon
+        epState.getHeartBeatState().forceNewerGenerationUnsafe();
+
+        // using the tokens from the endpoint-state as that is the real source of truth
+        Collection<Token> tokens = getTokensFromEndpointState(epState, DatabaseDescriptor.getPartitioner());
+        if (tokens == null || tokens.isEmpty())
+            throw new RuntimeException("Cannot revive endpoint " + endpoint + ": no tokens from TokenMetadata");
+
+        epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.normal(tokens));
+        epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.normal(tokens));
+        handleMajorStateChange(endpoint, epState);
+        Uninterruptibles.sleepUninterruptibly(intervalInMillis * 4, TimeUnit.MILLISECONDS);
+        logger.warn("Finished reviving {}, status={}, generation={}, heartbeat={}",
+                endpoint, epState.getStatus(), generation, heartbeat);
+    }
+
+    public void unsafeSetEndpointState(String address, String status) throws UnknownHostException
+    {
+        logger.warn("Forcibly changing gossip status of " + address + " to " + status);
+
+        InetAddressAndPort endpoint = InetAddressAndPort.getByName(address);
+        EndpointState epState = endpointStateMap.get(endpoint);
+
+        if (epState == null)
+            throw new RuntimeException("No state for endpoint " + endpoint);
+
+        int generation = epState.getHeartBeatState().getGeneration();
+        int heartbeat = epState.getHeartBeatState().getHeartBeatVersion();
+
+        logger.info("Have endpoint-state for {}: status={}, generation={}, heartbeat={}",
+                endpoint, epState.getStatus(), generation, heartbeat);
+
+        if (FailureDetector.instance.isAlive(endpoint))
+            throw new RuntimeException("Cannot update status for endpoint " + endpoint + ": still alive (failure-detector)");
+
+        Collection<Token> tokens = getTokensFromEndpointState(epState, DatabaseDescriptor.getPartitioner());
+
+        VersionedValue newStatus;
+        switch (status.toLowerCase())
+        {
+            case "hibernate":
+                newStatus = StorageService.instance.valueFactory.hibernate(true);
+                break;
+            case "normal":
+                newStatus = StorageService.instance.valueFactory.normal(tokens);
+                break;
+            case "left":
+                newStatus = StorageService.instance.valueFactory.left(tokens, computeExpireTime());
+                break;
+            case "shutdown":
+                newStatus = StorageService.instance.valueFactory.shutdown(true);
+                break;
+            default:
+                throw new IllegalArgumentException("Unknown status '" + status + '\'');
+        }
+
+        epState.updateTimestamp(); // make sure we don't evict it too soon
+        epState.getHeartBeatState().forceNewerGenerationUnsafe();
+
+        epState.addApplicationState(ApplicationState.STATUS, newStatus);
+        epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, newStatus);
+
+        handleMajorStateChange(endpoint, epState);
+
+        logger.warn("Forcibly changed gossip status of " + endpoint + " to " + newStatus);
+    }
+
+    public Collection<Token> getTokensFor(InetAddressAndPort endpoint, IPartitioner partitioner)
+    {
+        EndpointState state = getEndpointStateForEndpoint(endpoint);
+        if (state == null)
+            return Collections.emptyList();
+
+        return getTokensFromEndpointState(state, partitioner);
+    }
+
+    private Collection<Token> getTokensFromEndpointState(EndpointState state, IPartitioner partitioner)
+    {
+        try
+        {
+            VersionedValue versionedValue = state.getApplicationState(ApplicationState.TOKENS);
+            if (versionedValue == null)
+                return Collections.emptyList();
+
+            return TokenSerializer.deserialize(partitioner, new DataInputStream(new ByteArrayInputStream(versionedValue.toBytes())));
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
     }
 
     public int getCurrentGenerationNumber(InetAddressAndPort endpoint)
@@ -1124,7 +1317,7 @@ void doStatusCheck()
         long now = currentTimeMillis();
         long nowNano = nanoTime();
 
-        long pending = Stage.GOSSIP.executor().getPendingTaskCount();
+        long pending = Stage.GOSSIP.getPendingTaskCount();
         if (pending > 0 && lastProcessedMessageAt < now - 1000)
         {
             // if some new messages just arrived, give the executor some time to work on them
@@ -1144,7 +1337,7 @@ void doStatusCheck()
             if (endpoint.equals(getBroadcastAddressAndPort()))
                 continue;
 
-            FailureDetector.instance.interpret(endpoint);
+            IFailureDetector.instance.interpret(endpoint);
             EndpointState epState = endpointStateMap.get(endpoint);
             if (epState != null)
             {
@@ -1281,11 +1474,6 @@ long getLastProcessedMessageAt()
         return lastProcessedMessageAt;
     }
 
-    public UUID getHostId(InetAddressAndPort endpoint)
-    {
-        return getHostId(endpoint, endpointStateMap);
-    }
-
     public UUID getHostId(InetAddressAndPort endpoint, Map<InetAddressAndPort, EndpointState> epStates)
     {
         return UUID.fromString(epStates.get(endpoint).getApplicationState(ApplicationState.HOST_ID).value);
@@ -1389,7 +1577,7 @@ void notifyFailureDetector(InetAddressAndPort endpoint, EndpointState remoteEndp
         */
         if (localEndpointState != null)
         {
-            IFailureDetector fd = FailureDetector.instance;
+            IFailureDetector fd = IFailureDetector.instance;
             int localGeneration = localEndpointState.getHeartBeatState().getGeneration();
             int remoteGeneration = remoteEndpointState.getHeartBeatState().getGeneration();
             if (remoteGeneration > localGeneration)
@@ -1491,11 +1679,13 @@ private void silentlyMarkDead(InetAddressAndPort addr, EndpointState localState)
 
     /**
      * This method is called whenever there is a "big" change in ep state (a generation change for a known node).
+     * It is public as the state change simulation is needed in testing, otherwise should not be used directly.
      *
      * @param ep      endpoint
      * @param epState EndpointState for the endpoint
      */
-    private void handleMajorStateChange(InetAddressAndPort ep, EndpointState epState)
+    @VisibleForTesting
+    public void handleMajorStateChange(InetAddressAndPort ep, EndpointState epState)
     {
         checkProperThreadForStateMutation();
         EndpointState localEpState = endpointStateMap.get(ep);
@@ -1508,7 +1698,8 @@ private void handleMajorStateChange(InetAddressAndPort ep, EndpointState epState
         }
         if (logger.isTraceEnabled())
             logger.trace("Adding endpoint state for {}", ep);
-        endpointStateMap.put(ep, epState);
+
+        putEndpointState(ep, epState);
 
         if (localEpState != null)
         {   // the node restarted: it is up to the subscriber to take whatever action is necessary
@@ -1738,7 +1929,7 @@ else if (logger.isTraceEnabled())
             else
             {
                 // this is a new node, report it to the FD in case it is the first time we are seeing it AND it's not alive
-                FailureDetector.instance.report(ep);
+                IFailureDetector.instance.report(ep);
                 handleMajorStateChange(ep, remoteState);
             }
         }
@@ -1983,7 +2174,7 @@ public void start(int generationNbr, Map<ApplicationState, VersionedValue> prelo
         maybeInitializeLocalState(generationNbr);
         EndpointState localState = endpointStateMap.get(getBroadcastAddressAndPort());
         localState.addApplicationStates(preloadLocalStates);
-        minVersionSupplier.recompute();
+        clusterVersionProvider.reset();
 
         //notify snitches that Gossiper is about to start
         DatabaseDescriptor.getEndpointSnitch().gossiperStarting();
@@ -2165,7 +2356,7 @@ public void maybeInitializeLocalState(int generationNbr)
         HeartBeatState hbState = new HeartBeatState(generationNbr);
         EndpointState localState = new EndpointState(hbState);
         localState.markAlive();
-        endpointStateMap.putIfAbsent(getBroadcastAddressAndPort(), localState);
+        putEndpointStateIfAbsent(FBUtilities.getBroadcastAddressAndPort(), localState);
     }
 
     public void forceNewerGeneration()
@@ -2201,7 +2392,7 @@ public void addSavedEndpoint(InetAddressAndPort ep)
         }
 
         epState.markDead();
-        endpointStateMap.put(ep, epState);
+        putEndpointState(ep, epState);
         silentlyMarkDead(ep, epState);
         if (logger.isTraceEnabled())
             logger.trace("Adding saved endpoint {} {}", ep, epState.getHeartBeatState().getGeneration());
@@ -2251,6 +2442,7 @@ public void stop()
         EndpointState mystate = endpointStateMap.get(getBroadcastAddressAndPort());
         if (mystate != null && !isSilentShutdownState(mystate) && StorageService.instance.isJoined())
         {
+            // HCD-73 note: not using announceShutdown() here because we clone the EndpointState for the message payload
             logger.info("Announcing shutdown");
             addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.shutdown(true));
             addLocalApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.shutdown(true));
@@ -2267,6 +2459,21 @@ public void stop()
             scheduledGossipTask.cancel(false);
     }
 
+    /**
+     * This method sends the node shutdown status to all live endpoints.
+     * It does not close the gossiper itself.
+     */
+    public void announceShutdown()
+    {
+        logger.info("Announcing shutdown");
+        addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.shutdown(true));
+        addLocalApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.shutdown(true));
+        Message message = Message.out(Verb.GOSSIP_SHUTDOWN, noPayload);
+        for (InetAddressAndPort ep : liveEndpoints)
+            MessagingService.instance().send(message, ep);
+        Uninterruptibles.sleepUninterruptibly(SHUTDOWN_ANNOUNCE_DELAY_IN_MS.getInt(), TimeUnit.MILLISECONDS);
+    }
+
     public boolean isEnabled()
     {
         ScheduledFuture<?> scheduledGossipTask = this.scheduledGossipTask;
@@ -2337,7 +2544,7 @@ public void initializeUnreachableNodeUnsafe(InetAddressAndPort addr)
     {
         EndpointState state = new EndpointState(HeartBeatState.empty());
         state.markDead();
-        EndpointState oldState = endpointStateMap.putIfAbsent(addr, state);
+        EndpointState oldState = putEndpointStateIfAbsent(addr, state);
         if (null != oldState)
         {
             throw new RuntimeException("Attempted to initialize endpoint state for unreachable node, " +
@@ -2357,7 +2564,7 @@ public void initializeNodeUnsafe(InetAddressAndPort addr, UUID uuid, int netVers
         HeartBeatState hbState = new HeartBeatState(generationNbr);
         EndpointState newState = new EndpointState(hbState);
         newState.markAlive();
-        EndpointState oldState = endpointStateMap.putIfAbsent(addr, newState);
+        EndpointState oldState = putEndpointStateIfAbsent(addr, newState);
         EndpointState localState = oldState == null ? newState : oldState;
 
         // always add the version state
@@ -2375,6 +2582,8 @@ public void injectApplicationState(InetAddressAndPort endpoint, ApplicationState
     {
         EndpointState localState = endpointStateMap.get(endpoint);
         localState.addApplicationState(state, value);
+        localState.maybeSetUpdater(update -> Nodes.updateLocalOrPeer(endpoint, update, false));
+        localState.maybeUpdate();
     }
 
     public long getEndpointDowntime(String address) throws UnknownHostException
@@ -2509,7 +2718,7 @@ public boolean waitForSchemaAgreement(long maxWait, TimeUnit unit, BooleanSuppli
     public boolean hasMajorVersion3OrUnknownNodes()
     {
         return isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0) || // this is quite obvious
-               // however if we discovered only nodes at current version so far (in particular only this node),
+               // this is not so obvious:// however if we discovered only nodes at current version so far (in particular only this node),
                // but still there are nodes with unknown version, we also want to report that the cluster may have nodes at 3.x
                hasNodeWithUnknownVersion;
     }
@@ -2519,11 +2728,28 @@ public boolean hasMajorVersion3OrUnknownNodes()
      */
     public boolean isUpgradingFromVersionLowerThan(CassandraVersion referenceVersion)
     {
-        CassandraVersion v = upgradeFromVersionMemoized.get();
-        if (CassandraVersion.NULL_VERSION.equals(v) && scheduledGossipTask == null)
-            return false;
+        return getMinVersion().compareTo(referenceVersion) < 0;
+    }
 
-        return v != null && v.compareTo(referenceVersion) < 0;
+    /**
+     * This is a safe way to get the version we are upgrading from. It will return the current version if the cluster
+     * is not in the upgrade state. If the cluster is in upgrade state, it will return NULL_VERSION, if there is no
+     * information about the other nodes yet. Otherwise, it will just return the minimum cluster version.
+     */
+    public CassandraVersion getMinVersion()
+    {
+        CassandraVersion v = clusterVersionProvider.getMinClusterVersion();
+        assert v != null : "API contract violation: cluster version provider implementation should never return null";
+
+        if (!clusterVersionProvider.isUpgradeInProgress())
+            return v;
+
+        // we are in the upgrade state but since the minimum reported version is current version, we do not know
+        // anything about the other nodes
+        if (v.compareTo(SystemKeyspace.CURRENT_VERSION) < 0)
+            return v;
+        else
+            return CassandraVersion.NULL_VERSION;
     }
 
     private boolean nodesAgreeOnSchema(Collection<InetAddressAndPort> nodes)
@@ -2552,72 +2778,6 @@ public void stopShutdownAndWait(long timeout, TimeUnit unit) throws InterruptedE
         ExecutorUtils.shutdownAndWait(timeout, unit, executor);
     }
 
-    @Nullable
-    public CassandraVersion getMinVersion(long delay, TimeUnit timeUnit)
-    {
-        try
-        {
-            return minVersionSupplier.get(delay, timeUnit);
-        }
-        catch (TimeoutException e)
-        {
-            // Timeouts here are harmless: they won't cause reprepares and may only
-            // cause the old version of the hash to be kept for longer
-            return null;
-        }
-        catch (Throwable e)
-        {
-            logger.error("Caught an exception while waiting for min version", e);
-            return null;
-        }
-    }
-
-    @Nullable
-    private String getReleaseVersionString(InetAddressAndPort ep)
-    {
-        EndpointState state = getEndpointStateForEndpoint(ep);
-        if (state == null)
-            return null;
-
-        VersionedValue value = state.getApplicationState(ApplicationState.RELEASE_VERSION);
-        return value == null ? null : value.value;
-    }
-
-    private CassandraVersion computeMinVersion()
-    {
-        CassandraVersion minVersion = null;
-
-        for (InetAddressAndPort addr : Iterables.concat(Gossiper.instance.getLiveMembers(),
-                                                        Gossiper.instance.getUnreachableMembers()))
-        {
-            String versionString = getReleaseVersionString(addr);
-            // Raced with changes to gossip state, wait until next iteration
-            if (versionString == null)
-                return null;
-
-            CassandraVersion version;
-
-            try
-            {
-                version = new CassandraVersion(versionString);
-            }
-            catch (Throwable t)
-            {
-                JVMStabilityInspector.inspectThrowable(t);
-                String message = String.format("Can't parse version string %s", versionString);
-                logger.warn(message);
-                if (logger.isDebugEnabled())
-                    logger.debug(message, t);
-                return null;
-            }
-
-            if (minVersion == null || version.compareTo(minVersion) < 0)
-                minVersion = version;
-        }
-
-        return minVersion;
-    }
-
     @Override
     public boolean getLooseEmptyEnabled()
     {
@@ -2697,4 +2857,39 @@ public void unsafeSendLocalEndpointStateTo(InetAddressAndPort ep)
         Message<GossipDigestAck2> message = Message.out(Verb.GOSSIP_DIGEST_ACK2, digestAck2Message);
         MessagingService.instance().send(message, ep);
     }
+
+    private EndpointState putEndpointState(InetAddressAndPort endpoint, @Nonnull EndpointState state)
+    {
+        state.maybeSetUpdater(update -> Nodes.updateLocalOrPeer(endpoint, update, false));
+
+        EndpointState prev = endpointStateMap.put(endpoint, state);
+        if (prev != null && prev != state)
+            prev.maybeRemoveUpdater();
+
+        state.maybeUpdate();
+
+        return prev;
+    }
+
+    private EndpointState putEndpointStateIfAbsent(InetAddressAndPort endpoint, @Nonnull EndpointState state)
+    {
+        state.maybeSetUpdater(update -> Nodes.updateLocalOrPeer(endpoint, update, false));
+
+        EndpointState prev = endpointStateMap.putIfAbsent(endpoint, state);
+
+        if (prev != null && prev != state)
+            state.maybeRemoveUpdater();
+        else
+            state.maybeUpdate();
+
+        return prev;
+    }
+
+    private EndpointState removeEndpointState(InetAddressAndPort endpoint)
+    {
+        EndpointState removedState = endpointStateMap.remove(endpoint);
+        if (removedState != null)
+            removedState.maybeRemoveUpdater();
+        return removedState;
+    }
 }
diff --git a/src/java/org/apache/cassandra/gms/GossiperMBean.java b/src/java/org/apache/cassandra/gms/GossiperMBean.java
index 2d59e37f2d56..589aec70c471 100644
--- a/src/java/org/apache/cassandra/gms/GossiperMBean.java
+++ b/src/java/org/apache/cassandra/gms/GossiperMBean.java
@@ -29,8 +29,37 @@ public interface GossiperMBean
 
     public void unsafeAssassinateEndpoint(String address) throws UnknownHostException;
 
+    /**
+     * Do not call this method unless you know what you are doing.
+     * It will try extremely hard to obliterate any endpoint from the ring,
+     * even if it does not know about it. Sets gossip status to {@code left}.
+     *
+     * @param address endpoint to assassinate
+     */
     public void assassinateEndpoint(String address) throws UnknownHostException;
 
+    /**
+     * Do not call this method unless you know what you are doing.
+     * In case a node went into a hibernate state - i.e. replacing a node with the <em>same</em> address
+     * or bootstrapping a node without letting join the ring - and it's required to bring that node back
+     * to a normal status (e.g. for a failed replace operation), use this method.
+     * It can be called on any node, prefer a seed node, to set the status back to {@code normal}.
+     *
+     * @param address endpoint to revive
+     */
+    public void reviveEndpoint(String address) throws UnknownHostException;
+
+    /**
+     * Completely unsafe method to set the Gossip status of an endpoint.
+     * Primary intention is for testing only.
+     * The method will refuse the request if (and only if) {@link FailureDetector} - no further
+     * lifetime checks nor gossip state change safety barrier.
+     *
+     * @param address endpoint address
+     * @param status One of {@code hibernate}, {@code normal}, {@code left}, {@code shutdown}
+     */
+    public void unsafeSetEndpointState(String address, String status) throws UnknownHostException;
+
     public List<String> reloadSeeds();
 
     public List<String> getSeeds();
diff --git a/src/java/org/apache/cassandra/gms/IClusterVersionProvider.java b/src/java/org/apache/cassandra/gms/IClusterVersionProvider.java
new file mode 100644
index 000000000000..27d0b373efb4
--- /dev/null
+++ b/src/java/org/apache/cassandra/gms/IClusterVersionProvider.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.gms;
+
+import org.apache.cassandra.utils.CassandraVersion;
+
+public interface IClusterVersionProvider
+{
+    /**
+     * Returns the minimum Cassandra version of the nodes in the cluster. The method skips the nodes that have no or
+     * invalid version. However, if such nodes are present, {@link #isUpgradeInProgress()}} returns {@code true}.
+     */
+    CassandraVersion getMinClusterVersion();
+
+    /**
+     * Resets the provider to its initial state. This is called by the {@link Gossiper} when nodes join or becomes alive.
+     * It can be also called by the tests to verify the behaviour.
+     */
+    void reset();
+
+    /**
+     * Returns {@code true} if the cluster is in the middle of an upgrade. This is the case when the provider has
+     * detected that some nodes have no or invalid version. May also implement some grace period to avoid flapping.
+     */
+    boolean isUpgradeInProgress();
+}
diff --git a/src/java/org/apache/cassandra/gms/IFailureDetector.java b/src/java/org/apache/cassandra/gms/IFailureDetector.java
index 62fc97dbbaba..0c694e1647e4 100644
--- a/src/java/org/apache/cassandra/gms/IFailureDetector.java
+++ b/src/java/org/apache/cassandra/gms/IFailureDetector.java
@@ -17,7 +17,13 @@
  */
 package org.apache.cassandra.gms;
 
+import java.util.function.Predicate;
+
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_FAILURE_DETECTOR_PROPERTY;
 
 /**
  * An interface that provides an application with the ability
@@ -28,6 +34,14 @@
 
 public interface IFailureDetector
 {
+    IFailureDetector instance = CUSTOM_FAILURE_DETECTOR_PROPERTY.isPresent()
+                                ? FBUtilities.construct(CUSTOM_FAILURE_DETECTOR_PROPERTY.getString(),
+                                                        "Failure Detector")
+                                : new FailureDetector();
+
+    public static final Predicate<InetAddressAndPort> isEndpointAlive = instance::isAlive;
+    public static final Predicate<Replica> isReplicaAlive = r -> isEndpointAlive.test(r.endpoint());
+
     /**
      * Failure Detector's knowledge of whether a node is up or
      * down.
diff --git a/src/java/org/apache/cassandra/gms/IGossiper.java b/src/java/org/apache/cassandra/gms/IGossiper.java
index aa9d95a97d45..aa5a9db762f5 100644
--- a/src/java/org/apache/cassandra/gms/IGossiper.java
+++ b/src/java/org/apache/cassandra/gms/IGossiper.java
@@ -37,6 +37,9 @@ public interface IGossiper
     default CassandraVersion getReleaseVersion(InetAddressAndPort ep)
     {
         EndpointState state = getEndpointStateForEndpoint(ep);
-        return state != null ? state.getReleaseVersion() : null;
+        VersionedValue applicationState = state != null ? state.getApplicationState(ApplicationState.RELEASE_VERSION) : null;
+        return applicationState != null
+                ? new CassandraVersion(applicationState.value)
+                : null;
     }
 }
diff --git a/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java b/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java
index b57f898a7443..7bed57b3c3ae 100644
--- a/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java
+++ b/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java
@@ -26,7 +26,6 @@
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.io.util.*;
 import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.NativeLibrary;
 
 /**
  * A {@link RandomAccessReader} wrapper that calculates the CRC in place.
@@ -221,7 +220,7 @@ protected void readBuffer()
 
     public void tryUncacheRead()
     {
-        NativeLibrary.trySkipCache(getChannel().getFileDescriptor(), 0, getSourcePosition(), getPath());
+        getChannel().trySkipCache(0, getSourcePosition());
     }
 
     private void updateCrc()
@@ -245,9 +244,9 @@ public void close()
         channel.close();
     }
 
-    protected String getPath()
+    protected File getFile()
     {
-        return channel.filePath();
+        return channel.getFile();
     }
 
     public ChannelProxy getChannel()
diff --git a/src/java/org/apache/cassandra/hints/CompressedChecksummedDataInput.java b/src/java/org/apache/cassandra/hints/CompressedChecksummedDataInput.java
index 8236364077aa..b2419425f890 100644
--- a/src/java/org/apache/cassandra/hints/CompressedChecksummedDataInput.java
+++ b/src/java/org/apache/cassandra/hints/CompressedChecksummedDataInput.java
@@ -147,7 +147,7 @@ protected void readBuffer()
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, getPath());
+            throw new FSReadError(e, getFile());
         }
     }
 
@@ -163,7 +163,7 @@ public static ChecksummedDataInput upgradeInput(ChecksummedDataInput input, ICom
         long position = input.getPosition();
         input.close();
 
-        ChannelProxy channel = new ChannelProxy(input.getPath());
+        ChannelProxy channel = new ChannelProxy(input.getFile());
         try
         {
             return new CompressedChecksummedDataInput(channel, compressor, position);
diff --git a/src/java/org/apache/cassandra/hints/EncryptedChecksummedDataInput.java b/src/java/org/apache/cassandra/hints/EncryptedChecksummedDataInput.java
index ab788b9ad6cd..dbb972491167 100644
--- a/src/java/org/apache/cassandra/hints/EncryptedChecksummedDataInput.java
+++ b/src/java/org/apache/cassandra/hints/EncryptedChecksummedDataInput.java
@@ -128,7 +128,7 @@ protected void readBuffer()
         }
         catch (IOException ioe)
         {
-            throw new FSReadError(ioe, getPath());
+            throw new FSReadError(ioe, getFile());
         }
     }
 
@@ -137,7 +137,7 @@ public static ChecksummedDataInput upgradeInput(ChecksummedDataInput input, Ciph
         long position = input.getPosition();
         input.close();
 
-        ChannelProxy channel = new ChannelProxy(input.getPath());
+        ChannelProxy channel = new ChannelProxy(input.getFile());
         try
         {
             return new EncryptedChecksummedDataInput(channel, cipher, compressor, position);
diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java
index e2e74ee6f502..fe16134e88c8 100644
--- a/src/java/org/apache/cassandra/hints/Hint.java
+++ b/src/java/org/apache/cassandra/hints/Hint.java
@@ -20,14 +20,15 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Throwables;
 import com.google.common.primitives.Ints;
 
-import javax.annotation.Nullable;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.WriteOptions;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -95,7 +96,7 @@ public static Hint create(Mutation mutation, long creationTime, int gcgs)
     /**
      * Applies the contained mutation unless it's expired, filtering out any updates for truncated tables
      */
-    Future<?> applyFuture()
+    public Future<?> applyFuture()
     {
         if (isLive())
         {
@@ -106,7 +107,7 @@ Future<?> applyFuture()
                     filtered = filtered.without(id);
 
             if (!filtered.isEmpty())
-                return filtered.applyFuture();
+                return filtered.applyFuture(WriteOptions.FOR_HINT_REPLAY);
         }
 
         return ImmediateFuture.success(null);
@@ -125,6 +126,14 @@ void apply()
         }
     }
 
+    /**
+     * @return the mutation stored in the hint
+     */
+    public Mutation mutation()
+    {
+        return mutation;
+    }
+
     /**
      * @return the overall ttl of the hint - the minimum of all mutation's tables' gc gs now and at the time of creation
      */
diff --git a/src/java/org/apache/cassandra/hints/HintMessage.java b/src/java/org/apache/cassandra/hints/HintMessage.java
index 978ab4170191..f7b542ec025c 100644
--- a/src/java/org/apache/cassandra/hints/HintMessage.java
+++ b/src/java/org/apache/cassandra/hints/HintMessage.java
@@ -74,6 +74,21 @@ public final class HintMessage implements SerializableHintMessage
         this.unknownTableID = unknownTableID;
     }
 
+    public UUID hostId()
+    {
+        return hostId;
+    }
+
+    public Hint hint()
+    {
+        return hint;
+    }
+
+    public TableId unknownTableID()
+    {
+        return unknownTableID;
+    }
+
     public static class Serializer implements IVersionedAsymmetricSerializer<SerializableHintMessage, HintMessage>
     {
         public long serializedSize(SerializableHintMessage obj, int version)
@@ -96,7 +111,8 @@ else if (obj instanceof Encoded)
                 Encoded message = (Encoded) obj;
 
                 if (version != message.version)
-                    throw new IllegalArgumentException("serializedSize() called with non-matching version " + version);
+                    throw new IllegalArgumentException("serializedSize() called with non-matching version " + version +
+                                                       " for message with version " + message.version);
 
                 long size = UUIDSerializer.serializer.serializedSize(message.hostId, version);
                 size += TypeSizes.sizeofUnsignedVInt(message.hint.remaining());
diff --git a/src/java/org/apache/cassandra/hints/HintVerbHandler.java b/src/java/org/apache/cassandra/hints/HintVerbHandler.java
index 1fb04d995019..f11f29f19f9b 100644
--- a/src/java/org/apache/cassandra/hints/HintVerbHandler.java
+++ b/src/java/org/apache/cassandra/hints/HintVerbHandler.java
@@ -34,6 +34,9 @@
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_HINTS_HANDLER;
 
 /**
  * Verb handler used both for hint dispatch and streaming.
@@ -44,15 +47,19 @@
  */
 public final class HintVerbHandler implements IVerbHandler<HintMessage>
 {
-    public static final HintVerbHandler instance = new HintVerbHandler();
+    public static final IVerbHandler<HintMessage> instance = CUSTOM_HINTS_HANDLER.isPresent()
+                                                             ? FBUtilities.construct(CUSTOM_HINTS_HANDLER.getString(),
+                                                                                     "Custom Hint Verb Handler")
+                                                             : new HintVerbHandler();
 
     private static final Logger logger = LoggerFactory.getLogger(HintVerbHandler.class);
 
+    @Override
     public void doVerb(Message<HintMessage> message)
     {
         UUID hostId = message.payload.hostId;
         Hint hint = message.payload.hint;
-        InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
+        InetAddressAndPort address = HintsEndpointProvider.instance.endpointForHost(hostId);
 
         // If we see an unknown table id, it means the table, or one of the tables in the mutation, had been dropped.
         // In that case there is nothing we can really do, or should do, other than log it go on.
diff --git a/src/java/org/apache/cassandra/hints/HintsCatalog.java b/src/java/org/apache/cassandra/hints/HintsCatalog.java
index 6bc00309247a..acd364a02e68 100644
--- a/src/java/org/apache/cassandra/hints/HintsCatalog.java
+++ b/src/java/org/apache/cassandra/hints/HintsCatalog.java
@@ -35,7 +35,7 @@
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.INativeLibrary;
 import org.apache.cassandra.utils.SyncUtil;
 
 import static java.util.stream.Collectors.groupingBy;
@@ -148,17 +148,17 @@ void exciseStore(UUID hostId)
 
     void fsyncDirectory()
     {
-        int fd = NativeLibrary.tryOpenDirectory(hintsDirectory.absolutePath());
+        int fd = INativeLibrary.instance.tryOpenDirectory(hintsDirectory.toAbsolute());
         if (fd != -1)
         {
             try
             {
                 SyncUtil.trySync(fd);
-                NativeLibrary.tryCloseFD(fd);
+                INativeLibrary.instance.tryCloseFD(fd);
             }
             catch (FSError e) // trySync failed
             {
-                logger.error("Unable to sync directory {}", hintsDirectory.absolutePath(), e);
+                logger.error("Unable to sync directory {}", hintsDirectory.toAbsolute(), e);
                 FileUtils.handleFSErrorAndPropagate(e);
             }
         }
diff --git a/src/java/org/apache/cassandra/hints/HintsDescriptor.java b/src/java/org/apache/cassandra/hints/HintsDescriptor.java
index 4fce3fbbd081..37ef629c0656 100644
--- a/src/java/org/apache/cassandra/hints/HintsDescriptor.java
+++ b/src/java/org/apache/cassandra/hints/HintsDescriptor.java
@@ -18,7 +18,12 @@
 package org.apache.cassandra.hints;
 
 import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
@@ -33,9 +38,6 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
 import com.google.common.collect.ImmutableMap;
-
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileInputStreamPlus;
 import com.google.common.io.ByteStreams;
 import com.google.common.io.CountingOutputStream;
 import org.slf4j.Logger;
@@ -45,8 +47,12 @@
 import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.compress.ICompressor;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.CompressionParams;
 import org.apache.cassandra.security.EncryptionContext;
@@ -69,7 +75,10 @@ final class HintsDescriptor
     static final int VERSION_30 = 1;
     static final int VERSION_40 = 2;
     static final int VERSION_50 = 3;
-    static final int CURRENT_VERSION = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50;
+    static final int VERSION_DS_10 = MessagingService.VERSION_DS_10;
+    static final int VERSION_DS_11 = MessagingService.VERSION_DS_11;
+    static final int VERSION_DS_20 = MessagingService.VERSION_DS_20;
+    static final int CURRENT_VERSION = MessagingService.current_version;
 
     static final String COMPRESSION = "compression";
     static final String ENCRYPTION = "encryption";
@@ -86,9 +95,15 @@ final class HintsDescriptor
     final ImmutableMap<String, Object> parameters;
     final ParameterizedClass compressionConfig;
 
+    // It's set when HintsWriter closed for new hint file or
+    // when descriptor is deserialized from local hint file.
+    private volatile long dataSize = 0;
+
     private final Cipher cipher;
     private final ICompressor compressor;
 
+    private volatile Statistics statistics = EMPTY_STATS;
+
     HintsDescriptor(UUID hostId, int version, long timestamp, ImmutableMap<String, Object> parameters)
     {
         this.hostId = hostId;
@@ -192,6 +207,36 @@ static EncryptionData createEncryption(ImmutableMap<String, Object> params)
         }
     }
 
+    public void setDataSize(long length)
+    {
+        this.dataSize = length;
+    }
+
+    public long getDataSize()
+    {
+        return dataSize;
+    }
+
+    public void setStatistics(Statistics statistics)
+    {
+        this.statistics = statistics;
+    }
+
+    public Statistics statistics()
+    {
+        return statistics;
+    }
+
+    String statisticsFileName()
+    {
+        return statisticsFileName(hostId, timestamp, version);
+    }
+
+    static String statisticsFileName(UUID hostId, long timestamp, int version)
+    {
+        return String.format("%s-%s-%s-Statistics.hints", hostId, timestamp, version);
+    }
+
     private static final class EncryptionData
     {
         final Cipher cipher;
@@ -254,9 +299,15 @@ static int messagingVersion(int hintsVersion)
             case VERSION_30:
                 return MessagingService.VERSION_30;
             case VERSION_40:
-                return MessagingService.VERSION_40;
+                return MessagingService.Version.VERSION_40.value;
             case VERSION_50:
-                return MessagingService.VERSION_50;
+                return MessagingService.Version.VERSION_50.value;
+            case VERSION_DS_10:
+                return MessagingService.VERSION_DS_10;
+            case VERSION_DS_11:
+                return MessagingService.VERSION_DS_11;
+            case VERSION_DS_20:
+                return MessagingService.Version.VERSION_DS_20.value;
             default:
                 throw new AssertionError();
         }
@@ -271,7 +322,10 @@ static Optional<HintsDescriptor> readFromFileQuietly(Path path)
     {
         try (FileInputStreamPlus raf = new FileInputStreamPlus(path))
         {
-            return Optional.of(deserialize(raf));
+            HintsDescriptor descriptor = deserialize(raf);
+            descriptor.setDataSize(FileUtils.size(path));
+            descriptor.loadStatsComponent(path.getParent());
+            return Optional.of(descriptor);
         }
         catch (ChecksumMismatchException e)
         {
@@ -308,18 +362,6 @@ static void handleDescriptorIOE(IOException e, Path path)
         }
     }
 
-    static HintsDescriptor readFromFile(File path)
-    {
-        try (FileInputStreamPlus raf = new FileInputStreamPlus(path))
-        {
-            return deserialize(raf);
-        }
-        catch (IOException e)
-        {
-            throw new FSReadError(e, path);
-        }
-    }
-
     public boolean isCompressed()
     {
         return compressionConfig != null;
@@ -488,4 +530,73 @@ private static void validateCRC(int expected, int actual) throws IOException
         if (expected != actual)
             throw new ChecksumMismatchException("Hints Descriptor CRC Mismatch");
     }
+
+    @VisibleForTesting
+    void loadStatsComponent(Path hintsDirectory)
+    {
+        Path file = hintsDirectory.resolve(statisticsFileName());
+        try (InputStream inputStream = Files.newInputStream(file);
+             DataInputStream statsFile = new DataInputStream(inputStream))
+        {
+            this.statistics = Statistics.deserialize(statsFile);
+        }
+        catch (FileNotFoundException e)
+        {
+            // Statistics are only used for metrics; it's ok to ignore an absent component during upgrades
+            logger.warn("Cannot find stats component `{}` for hints descriptor, initialising with empty statistics.", file);
+            this.statistics = EMPTY_STATS;
+        }
+        catch (IOException e)
+        {
+            // Ignore error in case of corruption
+            logger.error("Cannot read stats component `{}` for hints descriptor, initialising with empty statistics.", file, e);
+            this.statistics = EMPTY_STATS;
+        }
+    }
+
+    void writeStatsComponent(Path directory)
+    {
+        File file = new File(directory, statisticsFileName());
+        try (DataOutputStream out = new DataOutputStream(Files.newOutputStream(file.toPath())))
+        {
+            statistics.serialize(out);
+        }
+        catch (IOException e)
+        {
+            throw new FSWriteError(e, file);
+        }
+    }
+
+    public static Statistics EMPTY_STATS = new Statistics(0);
+
+    public static class Statistics
+    {
+        private final long totalCount;
+
+        public Statistics(long totalCount)
+        {
+            this.totalCount = totalCount;
+        }
+
+        public long totalCount()
+        {
+            return totalCount;
+        }
+
+        public void serialize(DataOutput out) throws IOException
+        {
+            out.writeLong(totalCount);
+        }
+
+        public static Statistics deserialize(DataInput in) throws IOException
+        {
+            long totalCount = in.readLong();
+            return new Statistics(totalCount);
+        }
+
+        public static int serializedSize()
+        {
+            return Long.BYTES;
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java b/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
index 540f5bd85dc7..80a2776dfedd 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.hints;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
@@ -28,21 +29,20 @@
 import java.util.function.Supplier;
 
 import com.google.common.util.concurrent.RateLimiter;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 import org.apache.cassandra.utils.concurrent.Future;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_REWRITING_HINTS_ON_HOST_LEFT;
+
 /**
  * A multi-threaded (by default) executor for dispatching hints.
  *
@@ -55,10 +55,10 @@ final class HintsDispatchExecutor
     private final File hintsDirectory;
     private final ExecutorPlus executor;
     private final AtomicBoolean isPaused;
-    private final Predicate<InetAddressAndPort> isAlive;
+    private final Predicate<UUID> isAlive;
     private final Map<UUID, Future> scheduledDispatches;
 
-    HintsDispatchExecutor(File hintsDirectory, int maxThreads, AtomicBoolean isPaused, Predicate<InetAddressAndPort> isAlive)
+    HintsDispatchExecutor(File hintsDirectory, int maxThreads, AtomicBoolean isPaused, Predicate<UUID> isAlive)
     {
         this.hintsDirectory = hintsDirectory;
         this.isPaused = isPaused;
@@ -164,7 +164,7 @@ private TransferHintsTask(HintsCatalog catalog, Supplier<UUID> hostIdSupplier)
         public void run()
         {
             UUID hostId = hostIdSupplier.get();
-            InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
+            InetAddressAndPort address = HintsEndpointProvider.instance.endpointForHost(hostId);
             logger.info("Transferring all hints to {}: {}", address, hostId);
             if (transfer(hostId))
                 return;
@@ -209,16 +209,7 @@ private final class DispatchHintsTask implements Runnable
         {
             this.store = store;
             this.hostId = hostId;
-
-            // Rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
-            // Max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272), unless we are transferring
-            // hints during decomission rather than dispatching them to their final destination.
-            // The goal is to bound maximum hints traffic going towards a particular node from the rest of the cluster,
-            // not total outgoing hints traffic from this node. This is why the rate limiter is not shared between
-            // all the dispatch tasks (as there will be at most one dispatch task for a particular host id at a time).
-            int nodesCount = isTransfer ? 1 : Math.max(1, StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1);
-            double throttleInBytes = DatabaseDescriptor.getHintedHandoffThrottleInKiB() * 1024.0 / nodesCount;
-            this.rateLimiter = RateLimiter.create(throttleInBytes == 0 ? Double.MAX_VALUE : throttleInBytes);
+            this.rateLimiter = HintsRateLimiterFactory.instance.create(hostId);
         }
 
         DispatchHintsTask(HintsStore store, UUID hostId)
@@ -269,14 +260,21 @@ private void dispatch()
          */
         private boolean dispatch(HintsDescriptor descriptor)
         {
-            logger.trace("Dispatching hints file {}", descriptor.fileName());
+            logger.debug("Dispatching hints file {}", descriptor.fileName());
 
-            InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
+            InetAddressAndPort address = HintsEndpointProvider.instance.endpointForHost(hostId);
             if (address != null)
                 return deliver(descriptor, address);
 
             // address == null means the target no longer exist; find new home for each hint entry.
-            convert(descriptor);
+            if (SKIP_REWRITING_HINTS_ON_HOST_LEFT.getBoolean())
+            {
+                logger.debug("Host {} is no longer a member of cluster, dropping hints", hostId);
+                store.cleanUp(descriptor);
+                store.delete(descriptor);
+            }
+            else
+                convert(descriptor);
             return true;
         }
 
@@ -285,8 +283,16 @@ private boolean deliver(HintsDescriptor descriptor, InetAddressAndPort address)
             File file = descriptor.file(hintsDirectory);
             InputPosition offset = store.getDispatchOffset(descriptor);
 
-            BooleanSupplier shouldAbort = () -> !isAlive.test(address) || isPaused.get();
-            try (HintsDispatcher dispatcher = HintsDispatcher.create(file, rateLimiter, address, descriptor.hostId, shouldAbort))
+            BooleanSupplier shouldAbort = () -> !isAlive.test(descriptor.hostId) || isPaused.get();
+
+            Optional<Integer> optVersion = HintsEndpointProvider.instance.versionForEndpoint(address);
+            if (optVersion.isEmpty())
+            {
+                logger.debug("Cannot deliver handoff to endpoint {}: its version is unknown. This should be temporary.", address);
+                return false;
+            }
+
+            try (HintsDispatcher dispatcher = HintsDispatcher.create(file, rateLimiter, address, descriptor.hostId, optVersion.get(), shouldAbort))
             {
                 if (offset != null)
                     dispatcher.seek(offset);
@@ -348,4 +354,31 @@ public boolean hasScheduledDispatches()
     {
         return !scheduledDispatches.isEmpty();
     }
+
+    public void updateDispatcherConcurrency(int concurrency)
+    {
+        logger.info("updating HintsDispatchExecutor with new concurrency = {} (current value = {})", concurrency, executor.getCorePoolSize());
+        if (concurrency > executor.getCorePoolSize())
+        {
+            // we are increasing the value
+            executor.setMaximumPoolSize(concurrency);
+            executor.setCorePoolSize(concurrency);
+        }
+        else if (concurrency < executor.getCorePoolSize())
+        {
+            // we are reducing the value
+            executor.setCorePoolSize(concurrency);
+            executor.setMaximumPoolSize(concurrency);
+        }
+    }
+
+    public int getDispatcherCorePoolSize()
+    {
+        return executor.getCorePoolSize();
+    }
+
+    public int getDispatcherMaxPoolSize()
+    {
+        return executor.getMaximumPoolSize();
+    }
 }
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java b/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java
index 0dfc6e132311..a1b625a39d93 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java
@@ -19,9 +19,6 @@
 
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.schema.Schema;
-
 /**
  * A simple dispatch trigger that's being run every 10 seconds.
  *
@@ -62,7 +59,7 @@ public void run()
                .filter(store -> !isScheduled(store))
                .filter(HintsStore::isLive)
                .filter(store -> store.isWriting() || store.hasFiles())
-               .filter(store -> Schema.instance.isSameVersion(Gossiper.instance.getSchemaVersion(store.address())))
+               .filter(store -> HintsEndpointProvider.instance.isSameSchemaVersion(store.hostId))
                .forEach(this::schedule);
     }
 
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java
index b6273385435b..2685ad57b5fd 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java
@@ -73,10 +73,14 @@ private HintsDispatcher(HintsReader reader, UUID hostId, InetAddressAndPort addr
         this.abortRequested = abortRequested;
     }
 
-    static HintsDispatcher create(File file, RateLimiter rateLimiter, InetAddressAndPort address, UUID hostId, BooleanSupplier abortRequested)
+    static HintsDispatcher create(File file,
+                                  RateLimiter rateLimiter,
+                                  InetAddressAndPort address,
+                                  UUID hostId,
+                                  int peerMessagingVersion,
+                                  BooleanSupplier abortRequested)
     {
-        int messagingVersion = MessagingService.instance().versions.get(address);
-        HintsDispatcher dispatcher = new HintsDispatcher(HintsReader.open(file, rateLimiter), hostId, address, messagingVersion, abortRequested);
+        HintsDispatcher dispatcher = new HintsDispatcher(HintsReader.open(file, rateLimiter), hostId, address, peerMessagingVersion, abortRequested);
         HintDiagnostics.dispatcherCreated(dispatcher);
         return dispatcher;
     }
diff --git a/src/java/org/apache/cassandra/hints/HintsEndpointProvider.java b/src/java/org/apache/cassandra/hints/HintsEndpointProvider.java
new file mode 100644
index 000000000000..d8b58fc58b77
--- /dev/null
+++ b/src/java/org/apache/cassandra/hints/HintsEndpointProvider.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.util.Optional;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.EndpointMessagingVersions;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_HINTS_ENDPOINT_PROVIDER;
+
+/**
+ * Provide endpoint info and host info for hints. It's used by CNDB to support cross-region hints
+ */
+public interface HintsEndpointProvider
+{
+    static final Logger LOGGER = LoggerFactory.getLogger(HintsDispatcher.class);
+
+    HintsEndpointProvider instance = CUSTOM_HINTS_ENDPOINT_PROVIDER.isPresent()
+                                     ? FBUtilities.construct(CUSTOM_HINTS_ENDPOINT_PROVIDER.getString(),
+                                                             "Hinted Handoff Endpoint Provider")
+                                     : new DefaultHintsEndpointProvider();
+
+    boolean isSameSchemaVersion(UUID hostId);
+
+    boolean isAlive(UUID hostId);
+
+    InetAddressAndPort endpointForHost(UUID hostId);
+
+    UUID hostForEndpoint(InetAddressAndPort endpoint);
+
+    Optional<Integer> versionForEndpoint(InetAddressAndPort endpoint);
+
+    class DefaultHintsEndpointProvider implements HintsEndpointProvider
+    {
+        @Override
+        public InetAddressAndPort endpointForHost(UUID hostId)
+        {
+            return StorageService.instance.getEndpointForHostId(hostId);
+        }
+
+        @Override
+        public UUID hostForEndpoint(InetAddressAndPort endpoint)
+        {
+            return StorageService.instance.getHostIdForEndpoint(endpoint);
+        }
+
+        @Override
+        public Optional<Integer> versionForEndpoint(InetAddressAndPort endpoint)
+        {
+            EndpointMessagingVersions versions = MessagingService.instance().versions;
+            if (versions.knows(endpoint))
+            {
+                try
+                {
+                    return Optional.of(versions.getRaw(endpoint));
+                }
+                catch (Exception e)
+                {
+                    LOGGER.debug("Failed to get raw version for endpoint {}", endpoint, e);
+                }
+            }
+            return Optional.empty();
+        }
+
+        @Override
+        public boolean isSameSchemaVersion(UUID hostId)
+        {
+            InetAddressAndPort peer = this.endpointForHost(hostId);
+            return Schema.instance.isSameVersion(Gossiper.instance.getSchemaVersion(peer));
+        }
+
+        @Override
+        public boolean isAlive(UUID hostId)
+        {
+            InetAddressAndPort address = this.endpointForHost(hostId);
+            return address != null && IFailureDetector.instance.isAlive(address);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/hints/HintsRateLimiterFactory.java b/src/java/org/apache/cassandra/hints/HintsRateLimiterFactory.java
new file mode 100644
index 000000000000..c0f6c1bcfa8d
--- /dev/null
+++ b/src/java/org/apache/cassandra/hints/HintsRateLimiterFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+import java.util.UUID;
+
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_HINTS_RATE_LIMITER_FACTORY;
+
+/**
+ * The factory for creating {@link RateLimiter} for every hints dispatch task
+ */
+public interface HintsRateLimiterFactory
+{
+
+    HintsRateLimiterFactory instance = CUSTOM_HINTS_RATE_LIMITER_FACTORY.isPresent() ?
+                                       make(CUSTOM_HINTS_RATE_LIMITER_FACTORY.getString()):
+                                       new DefaultHintsRateLimiterFactory();
+
+    /**
+     * return {@link RateLimiter} for current dispatch task
+     */
+    RateLimiter create(UUID hostId);
+
+    class DefaultHintsRateLimiterFactory implements HintsRateLimiterFactory
+    {
+        DefaultHintsRateLimiterFactory()
+        {
+        }
+
+        @Override
+        public RateLimiter create(UUID hostId)
+        {
+            // rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
+            // max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272).
+            // the goal is to bound maximum hints traffic going towards a particular node from the rest of the cluster,
+            // not total outgoing hints traffic from this node - this is why the rate limiter is not shared between
+            // all the dispatch tasks (as there will be at most one dispatch task for a particular host id at a time).
+            int nodesCount = Math.max(1, StorageService.instance.getTokenMetadata().getSizeOfAllEndpoints() - 1);
+            int throttleInKB = DatabaseDescriptor.getHintedHandoffThrottleInKiB() / nodesCount;
+            return RateLimiter.create(throttleInKB == 0 ? Double.MAX_VALUE : throttleInKB * 1024);
+        }
+    }
+
+    static HintsRateLimiterFactory make(String customImpl)
+    {
+        try
+        {
+            return (HintsRateLimiterFactory) Class.forName(customImpl).newInstance();
+        }
+        catch (Throwable ex)
+        {
+            throw new IllegalStateException("Unknown Hinted Handoff Rate Limiter Factory: " + customImpl);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/hints/HintsReader.java b/src/java/org/apache/cassandra/hints/HintsReader.java
index fc6796b624ba..5cd148e92831 100644
--- a/src/java/org/apache/cassandra/hints/HintsReader.java
+++ b/src/java/org/apache/cassandra/hints/HintsReader.java
@@ -21,9 +21,9 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
-
 import javax.annotation.Nullable;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.primitives.Ints;
 import com.google.common.util.concurrent.RateLimiter;
 
@@ -33,7 +33,6 @@
 
 import org.apache.cassandra.exceptions.UnknownTableException;
 import org.apache.cassandra.io.FSReadError;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.AbstractIterator;
 
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@@ -78,6 +77,8 @@ static HintsReader open(File file, RateLimiter rateLimiter)
         try
         {
             HintsDescriptor descriptor = HintsDescriptor.deserialize(reader);
+            descriptor.setDataSize(file.length());
+            descriptor.loadStatsComponent(file.toPath().getParent());
             if (descriptor.isCompressed())
             {
                 // since the hints descriptor is always uncompressed, it needs to be read with the normal ChecksummedDataInput.
@@ -95,6 +96,7 @@ else if (descriptor.isEncrypted())
         }
     }
 
+    @VisibleForTesting
     static HintsReader open(File file)
     {
         return open(file, null);
@@ -242,7 +244,7 @@ private Hint readHint(int size) throws IOException
             catch (UnknownTableException e)
             {
                 logger.warn("Failed to read a hint for {}: {} - table with id {} is unknown in file {}",
-                            StorageService.instance.getEndpointForHostId(descriptor.hostId),
+                            HintsEndpointProvider.instance.endpointForHost(descriptor.hostId),
                             descriptor.hostId,
                             e.id,
                             descriptor.fileName());
@@ -256,7 +258,7 @@ private Hint readHint(int size) throws IOException
 
             // log a warning and skip the corrupted entry
             logger.warn("Failed to read a hint for {}: {} - digest mismatch for hint at position {} in file {}",
-                        StorageService.instance.getEndpointForHostId(descriptor.hostId),
+                        HintsEndpointProvider.instance.endpointForHost(descriptor.hostId),
                         descriptor.hostId,
                         input.getPosition() - size - 4,
                         descriptor.fileName());
diff --git a/src/java/org/apache/cassandra/hints/HintsService.java b/src/java/org/apache/cassandra/hints/HintsService.java
index 19989a6c87c7..bbfd9a790368 100644
--- a/src/java/org/apache/cassandra/hints/HintsService.java
+++ b/src/java/org/apache/cassandra/hints/HintsService.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -43,15 +44,12 @@
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
-import org.apache.cassandra.gms.FailureDetector;
-import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.EndpointsForToken;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.HintedHandoffMetrics;
 import org.apache.cassandra.metrics.StorageMetrics;
-import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.MBeanWrapper;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
@@ -95,11 +93,11 @@ public final class HintsService implements HintsServiceMBean
 
     private HintsService()
     {
-        this(FailureDetector.instance);
+        this(HintsEndpointProvider.instance::isAlive);
     }
 
     @VisibleForTesting
-    HintsService(IFailureDetector failureDetector)
+    HintsService(Predicate<UUID> isAlive)
     {
         File hintsDirectory = DatabaseDescriptor.getHintsDirectory();
         int maxDeliveryThreads = DatabaseDescriptor.getMaxHintsDeliveryThreads();
@@ -111,7 +109,7 @@ private HintsService()
         bufferPool = new HintsBufferPool(bufferSize, writeExecutor::flushBuffer);
 
         isDispatchPaused = new AtomicBoolean(true);
-        dispatchExecutor = new HintsDispatchExecutor(hintsDirectory, maxDeliveryThreads, isDispatchPaused, failureDetector::isAlive);
+        dispatchExecutor = new HintsDispatchExecutor(hintsDirectory, maxDeliveryThreads, isDispatchPaused, isAlive);
 
         // periodically empty the current content of the buffers
         int flushPeriod = DatabaseDescriptor.getHintsFlushPeriodInMS();
@@ -163,6 +161,9 @@ public void write(Collection<UUID> hostIds, Hint hint)
         if (isShutDown)
             throw new IllegalStateException("HintsService is shut down and can't accept new hints");
 
+        if (hostIds.isEmpty())
+            return;
+
         // we have to make sure that the HintsStore instances get properly initialized - otherwise dispatch will not trigger
         catalog.maybeLoadStores(hostIds);
 
@@ -187,8 +188,8 @@ public void write(UUID hostId, Hint hint)
      */
     void writeForAllReplicas(Hint hint)
     {
-        String keyspaceName = hint.mutation.getKeyspaceName();
-        Token token = hint.mutation.key().getToken();
+        String keyspaceName = hint.mutation().getKeyspaceName();
+        Token token = hint.mutation().key().getToken();
 
         EndpointsForToken replicas = ReplicaLayout.forTokenWriteLiveAndDown(Keyspace.open(keyspaceName), token).all();
 
@@ -196,7 +197,7 @@ void writeForAllReplicas(Hint hint)
         // than performing filters / translations 2x extra via Iterables.filter/transform
         List<UUID> hostIds = replicas.stream()
                 .filter(replica -> StorageProxy.shouldHint(replica, false))
-                .map(replica -> StorageService.instance.getHostIdForEndpoint(replica.endpoint()))
+                .map(replica -> HintsEndpointProvider.instance.hostForEndpoint(replica.endpoint()))
                 .collect(Collectors.toList());
 
         write(hostIds, hint);
@@ -347,7 +348,7 @@ public void deleteAllHintsForEndpoint(String address)
      */
     public void deleteAllHintsForEndpoint(InetAddressAndPort target)
     {
-        UUID hostId = StorageService.instance.getHostIdForEndpoint(target);
+        UUID hostId = HintsEndpointProvider.instance.hostForEndpoint(target);
         if (hostId == null)
             throw new IllegalArgumentException("Can't delete hints for unknown address " + target);
         catalog.deleteAllHints(hostId);
@@ -454,6 +455,41 @@ public long findOldestHintTimestamp(UUID hostId)
     {
         return catalog.get(hostId).findOldestHintTimestamp();
     }
+    
+    /**
+     * Get the total size in bytes of all the hints files on disk.
+     * @return total file size, in bytes
+     */
+    public long getTotalHintsSize()
+    {
+        return catalog.stores().mapToLong(HintsStore::getTotalFileSize).sum();
+    }
+
+    /**
+     * @return the number of all hint files on disk, including corrupted files
+     */
+    public int getTotalFilesNum()
+    {
+        return catalog.stores().mapToInt(HintsStore::getTotalFilesNum).sum();
+    }
+
+    /**
+     * @return the number of corrupted hint files on disk.
+     */
+    public int getCorruptedFilesNum()
+    {
+        return catalog.stores().mapToInt(HintsStore::getCorruptedFilesNum).sum();
+    }
+
+    /**
+     * Checks hints files total size on disk exceeds the total maximum.
+     * @return true if the max is exceeded
+     */
+    public boolean exceedsMaxHintsSize()
+    {
+        long maxTotalHintsSize = DatabaseDescriptor.getMaxHintsSizePerHost();
+        return maxTotalHintsSize > 0 && getTotalHintsSize() > maxTotalHintsSize;
+    }
 
     HintsCatalog getCatalog()
     {
@@ -473,4 +509,28 @@ public boolean isDispatchPaused()
     {
         return isDispatchPaused.get();
     }
+
+    @VisibleForTesting
+    public HintsDispatchExecutor dispatcherExecutor()
+    {
+        return dispatchExecutor;
+    }
+
+    // used by CNDB
+    public void updateDispatcherConcurrency(int concurrency)
+    {
+        dispatchExecutor.updateDispatcherConcurrency(concurrency);
+    }
+
+    // used by CNDB
+    public int getDispatcherCorePoolSize()
+    {
+        return dispatchExecutor.getDispatcherCorePoolSize();
+    }
+
+    // used by CNDB
+    public int getDispatcherMaxPoolSize()
+    {
+        return dispatchExecutor.getDispatcherMaxPoolSize();
+    }
 }
diff --git a/src/java/org/apache/cassandra/hints/HintsStore.java b/src/java/org/apache/cassandra/hints/HintsStore.java
index 969f37ae7f06..e1f745a53d21 100644
--- a/src/java/org/apache/cassandra/hints/HintsStore.java
+++ b/src/java/org/apache/cassandra/hints/HintsStore.java
@@ -18,13 +18,21 @@
 package org.apache.cassandra.hints;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.function.Predicate;
 
 import javax.annotation.Nullable;
+import java.util.stream.Stream;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
@@ -33,9 +41,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.HintsServiceMetrics;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.SyncUtil;
@@ -79,6 +87,9 @@ private HintsStore(UUID hostId, File hintsDirectory, ImmutableMap<String, Object
 
         //noinspection resource
         lastUsedTimestamp = descriptors.stream().mapToLong(d -> d.timestamp).max().orElse(0L);
+
+        long hintsNum = descriptors.stream().mapToLong(d -> d.statistics().totalCount()).sum();
+        HintsServiceMetrics.hintsOnDisk.inc(hintsNum);
     }
 
     static HintsStore create(UUID hostId, File hintsDirectory, ImmutableMap<String, Object> writerParams, List<HintsDescriptor> descriptors)
@@ -145,8 +156,7 @@ public long findOldestHintTimestamp()
 
     boolean isLive()
     {
-        InetAddressAndPort address = address();
-        return address != null && FailureDetector.instance.isAlive(address);
+        return HintsEndpointProvider.instance.isAlive(hostId);
     }
 
     HintsDescriptor poll()
@@ -177,6 +187,7 @@ void deleteAllHints()
         {
             cleanUp(descriptor);
             delete(descriptor);
+            HintsServiceMetrics.corruptedHintsOnDisk.dec(descriptor.statistics().totalCount());
         }
     }
 
@@ -229,7 +240,10 @@ void delete(HintsDescriptor descriptor)
     {
         File hintsFile = descriptor.file(hintsDirectory);
         if (hintsFile.tryDelete())
+        {
+            HintsServiceMetrics.hintsOnDisk.dec(descriptor.statistics().totalCount());
             logger.info("Deleted hint file {}", descriptor.fileName());
+        }
         else if (hintsFile.exists())
             logger.error("Failed to delete hint file {}", descriptor.fileName());
         else
@@ -237,6 +251,8 @@ else if (hintsFile.exists())
 
         //noinspection ResultOfMethodCallIgnored
         descriptor.checksumFile(hintsDirectory).tryDelete();
+        //noinspection ResultOfMethodCallIgnored
+        new File(hintsDirectory, descriptor.statisticsFileName()).tryDelete();
     }
 
     boolean hasFiles()
@@ -244,6 +260,12 @@ boolean hasFiles()
         return !dispatchDequeue.isEmpty();
     }
 
+    @VisibleForTesting
+    Stream<HintsDescriptor> descriptors()
+    {
+        return dispatchDequeue.stream();
+    }
+
     InputPosition getDispatchOffset(HintsDescriptor descriptor)
     {
         return dispatchPositions.get(descriptor);
@@ -261,15 +283,25 @@ long getTotalFileSize()
     {
         long total = 0;
         for (HintsDescriptor descriptor : Iterables.concat(dispatchDequeue, corruptedFiles))
-            total += descriptor.hintsFileSize(hintsDirectory);
+            total += descriptor.getDataSize();
 
         HintsWriter currentWriter = getWriter();
         if (null != currentWriter)
-            total += currentWriter.descriptor().hintsFileSize(hintsDirectory);
+            total += currentWriter.descriptor().getDataSize();
 
         return total;
     }
 
+    public int getTotalFilesNum()
+    {
+        return dispatchDequeue.size() + corruptedFiles.size();
+    }
+
+    public int getCorruptedFilesNum()
+    {
+        return corruptedFiles.size();
+    }
+
     void cleanUp(HintsDescriptor descriptor)
     {
         dispatchPositions.remove(descriptor);
@@ -279,6 +311,7 @@ void cleanUp(HintsDescriptor descriptor)
     void markCorrupted(HintsDescriptor descriptor)
     {
         corruptedFiles.add(descriptor);
+        HintsServiceMetrics.corruptedHintsOnDisk.inc(descriptor.statistics().totalCount());
     }
 
     /*
diff --git a/src/java/org/apache/cassandra/hints/HintsWriteExecutor.java b/src/java/org/apache/cassandra/hints/HintsWriteExecutor.java
index 2b0a434dd3ea..082cd596902a 100644
--- a/src/java/org/apache/cassandra/hints/HintsWriteExecutor.java
+++ b/src/java/org/apache/cassandra/hints/HintsWriteExecutor.java
@@ -23,15 +23,15 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.utils.concurrent.Future;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
@@ -75,6 +75,10 @@ void shutdownBlocking()
         {
             throw new AssertionError(e);
         }
+        finally
+        {
+            FileUtils.clean(writeBuffer);
+        }
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/hints/HintsWriter.java b/src/java/org/apache/cassandra/hints/HintsWriter.java
index ecee314249de..9bba5bf983d1 100644
--- a/src/java/org/apache/cassandra/hints/HintsWriter.java
+++ b/src/java/org/apache/cassandra/hints/HintsWriter.java
@@ -24,6 +24,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.CRC32;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -33,7 +34,8 @@
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputBufferFixed;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.metrics.HintsServiceMetrics;
+import org.apache.cassandra.utils.INativeLibrary;
 import org.apache.cassandra.utils.SyncUtil;
 import org.apache.cassandra.utils.Throwables;
 
@@ -55,6 +57,9 @@ class HintsWriter implements AutoCloseable
 
     private volatile long lastSyncPosition = 0L;
 
+    @VisibleForTesting
+    AtomicLong totalHintsWritten = new AtomicLong();
+
     protected HintsWriter(File directory, HintsDescriptor descriptor, File file, FileChannel channel, int fd, CRC32 globalCRC)
     {
         this.directory = directory;
@@ -70,7 +75,7 @@ static HintsWriter create(File directory, HintsDescriptor descriptor) throws IOE
         File file = descriptor.file(directory);
 
         FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
-        int fd = NativeLibrary.getfd(channel);
+        int fd = INativeLibrary.instance.getfd(channel);
 
         CRC32 crc = new CRC32();
 
@@ -120,6 +125,10 @@ public void close()
         perform(file, Throwables.FileOpType.WRITE, this::doFsync, channel::close);
 
         writeChecksum();
+        descriptor.setStatistics(new HintsDescriptor.Statistics(totalHintsWritten.get()));
+        descriptor.writeStatsComponent(file.toPath().getParent());
+
+        descriptor.setDataSize(file.length());
     }
 
     public void fsync()
@@ -170,6 +179,7 @@ final class Session implements AutoCloseable
 
         private final long initialSize;
         private long bytesWritten;
+        private long hintsWritten;
 
         Session(ByteBuffer buffer, long initialSize)
         {
@@ -200,6 +210,7 @@ long position()
         void append(ByteBuffer hint) throws IOException
         {
             bytesWritten += hint.remaining();
+            hintsWritten += 1;
 
             // if the hint to write won't fit in the aggregation buffer, flush it
             if (hint.remaining() > buffer.remaining())
@@ -257,9 +268,14 @@ void append(Hint hint) throws IOException
             }
 
             if (hintBuffer == buffer)
+            {
                 bytesWritten += totalSize;
+                hintsWritten += 1;
+            }
             else
+            {
                 append(hintBuffer.flip());
+            }
         }
 
         /**
@@ -272,6 +288,11 @@ public void close() throws IOException
             maybeFsync();
             maybeSkipCache();
             descriptor.hintsFileSize(position());
+            long hintsCnt = totalHintsWritten.addAndGet(hintsWritten);
+            descriptor.setDataSize(channel.position());
+            descriptor.setStatistics(new HintsDescriptor.Statistics(hintsCnt));
+            descriptor.writeStatsComponent(file.toPath().getParent());
+            HintsServiceMetrics.hintsOnDisk.inc(hintsWritten);
         }
 
         private void flushBuffer() throws IOException
@@ -299,7 +320,7 @@ private void maybeSkipCache()
             // don't skip page cache for tiny files, on the assumption that if they are tiny, the target node is probably
             // alive, and if so, the file will be closed and dispatched shortly (within a minute), and the file will be dropped.
             if (position >= DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024L)
-                NativeLibrary.trySkipCache(fd, 0, position - (position % PAGE_SIZE), file.path());
+                INativeLibrary.instance.trySkipCache(fd, 0, position - (position % PAGE_SIZE), file.path());
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java
index 8abc800e0f18..067ec285fb55 100644
--- a/src/java/org/apache/cassandra/index/Index.java
+++ b/src/java/org/apache/cassandra/index/Index.java
@@ -24,7 +24,7 @@
 import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -36,8 +36,6 @@
 import javax.annotation.Nullable;
 
 import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.restrictions.Restriction;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionTime;
@@ -56,6 +54,7 @@
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.internal.CollatedViewIndexBuilder;
+import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
@@ -182,11 +181,16 @@ public boolean supportsReads()
 
     /**
      * Provider of {@code SecondaryIndexBuilder} instances. See {@code getBuildTaskSupport} and
-     * {@code SecondaryIndexManager.buildIndexesBlocking} for more detail.
+     * {@code SecondaryIndexManager} for more detail.
      */
     interface IndexBuildingSupport
     {
         SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables, boolean isFullRebuild);
+
+        default List<SecondaryIndexBuilder> getParallelIndexBuildTasks(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables, boolean isFullRebuild)
+        {
+            return Collections.singletonList(getIndexBuildTask(cfs, indexes, sstables, isFullRebuild));
+        }
     }
 
     /**
@@ -245,6 +249,16 @@ default LoadType getSupportedLoadTypeOnFailure(boolean isInitialBuild)
         return isInitialBuild ? LoadType.WRITE : LoadType.ALL;
     }
 
+    /**
+     * Returns true if index initialization should be skipped, false if it should run
+     * (via {@link #getInitializationTask()}); defaults to skipping based on {@link IndexBuildDecider#onInitialBuild()}
+     * decision.
+     */
+    default boolean shouldSkipInitialization()
+    {
+        return IndexBuildDecider.instance.onInitialBuild().skipped();
+    }
+
     /**
      * Return a task to perform any initialization work when a new index instance is created.
      * This may involve costly operations such as (re)building the index, and is performed asynchronously
@@ -283,7 +297,7 @@ default LoadType getSupportedLoadTypeOnFailure(boolean isInitialBuild)
     /**
      * Unregister current index when it's removed from system
      *
-     * @param registry the index registry to register the instance with
+     * @param registry the index registry to unregister the instance with
      */
     default void unregister(IndexRegistry registry)
     {
@@ -333,6 +347,16 @@ public default Callable<?> getBlockingFlushTask(Memtable baseCfs)
      */
     public Callable<?> getInvalidateTask();
 
+    /**
+     * Return a task which unload the index, indicating it should no longer be considered usable.
+     * This should include an clean up and releasing of resources required without removing files.
+     * @return task to be executed by the index manager to invalidate the index.
+     */
+    default Callable<?> getUnloadTask()
+    {
+        return () -> null;
+    }
+
     /**
      * Return a task to truncate the index with the specified truncation timestamp.
      * Called when the base table is truncated.
@@ -444,6 +468,58 @@ default boolean filtersMultipleContains()
      */
     public AbstractType<?> customExpressionValueType();
 
+    /**
+     * If the index supports custom search expressions using the
+     * {@code SELECT * FROM table WHERE expr(index_name, expression)} syntax, this method should return a new
+     * {@link RowFilter.CustomExpression} for the specified expression value. Index implementations may provide their
+     * own implementations using method {@link RowFilter.CustomExpression#isSatisfiedBy(TableMetadata, DecoratedKey, Row)}
+     * to filter reconciled rows in the coordinator. Otherwise, the default implementation will accept all rows.
+     * See DB-2185 and DSP-16537 for further details.
+     *
+     * @param metadata the indexed table metadata
+     * @param value the custom expression value
+     * @return a custom index expression for the specified value
+     */
+    default RowFilter.CustomExpression customExpressionFor(TableMetadata metadata, ByteBuffer value)
+    {
+        return new RowFilter.CustomExpression(metadata, getIndexMetadata(), value);
+    }
+
+    /**
+     * Returns the write-time {@link Analyzer} for this index, if any. If the index doesn't transform the column values,
+     * this method will return an empty optional.
+     *
+     * @return the write-time transforming column value analyzer for the index, if any
+     */
+    default Optional<Analyzer> getIndexAnalyzer()
+    {
+        return Optional.empty();
+    }
+
+    /**
+     * Returns the query-time {@link Analyzer} for this index, if any. If the index doesn't transform the column values,
+     * this method will return an empty optional.
+     *
+     * @return the query-time transforming column value analyzer for the index, if any
+     */
+    default Optional<Analyzer> getQueryAnalyzer()
+    {
+        return Optional.empty();
+    }
+
+    /**
+     * Class representing a transformation of the indexed values done by the index.
+     * </p>
+     * This is used by the CQL operators when a filtering expression supported by an index is evaluated outside the
+     * index. It can be used to perform the same transformation on values that the index does when indexing. That way,
+     * the CQL operator can replicate the index behaviour when filtering results.
+     */
+    @FunctionalInterface
+    interface Analyzer
+    {
+        List<ByteBuffer> analyze(ByteBuffer value);
+    }
+
     /**
      * Transform an initial RowFilter into the filter that will still need to applied
      * to a set of Rows after the index has performed it's initial scan.
@@ -454,19 +530,7 @@ default boolean filtersMultipleContains()
      * @return the (hopefully) reduced filter that would still need to be applied after
      *         the index was used to narrow the initial result set
      */
-    public RowFilter getPostIndexQueryFilter(RowFilter filter);
-
-    /**
-     * Return a comparator that reorders query result before sending to client
-     *
-     * @param restriction restriction that requires current index
-     * @param options query options
-     * @return a comparator for post-query ordering; or null if not supported
-     */
-    default Comparator<ByteBuffer> getPostQueryOrdering(Restriction restriction, QueryOptions options)
-    {
-        return null;
-    }
+    RowFilter getPostIndexQueryFilter(RowFilter filter);
 
     /**
      * Return an estimate of the number of results this index is expected to return for any given
@@ -476,7 +540,7 @@ default Comparator<ByteBuffer> getPostQueryOrdering(Restriction restriction, Que
      *
      * @return the estimated average number of results a Searcher may return for any given query
      */
-    public long getEstimatedResultRows();
+    long getEstimatedResultRows();
 
     /**
      * Check if current index is queryable based on the index status.
@@ -505,16 +569,6 @@ default boolean isQueryable(Status status)
      */
     public void validate(PartitionUpdate update, ClientState state) throws InvalidRequestException;
 
-    /**
-     * Returns the SSTable-attached {@link Component}s created by this index.
-     *
-     * @return the SSTable components created by this index
-     */
-    default Set<Component> getComponents()
-    {
-        return Collections.emptySet();
-    }
-
     /*
      * Update processing
      */
@@ -644,10 +698,7 @@ default void finish() {}
 
     /**
      * Used to validate the various parameters of a supplied {@code}ReadCommand{@code},
-     * this is called prior to execution. In theory, any command instance may be checked
-     * by any {@code}Index{@code} instance, but in practice the index will be the one
-     * returned by a call to the {@code}getIndex(ColumnFamilyStore cfs){@code} method on
-     * the supplied command.
+     * this is called prior to execution.
      *
      * Custom index implementations should perform any validation of query expressions here and throw a meaningful
      * InvalidRequestException when any expression or other parameter is invalid.
@@ -660,6 +711,23 @@ default void validate(ReadCommand command) throws InvalidRequestException
     {
     }
 
+    /**
+     * Tells whether this index supports replica fitering protection or not.
+     *
+     * Replica filtering protection might need to run the query row filter in the coordinator to detect stale results.
+     * An index implementation will be compatible with this protection mechanism if it returns the same results for the
+     * row filter as CQL will return with {@code ALLOW FILTERING} and without using the index. This means that index
+     * implementations using custom query syntax or applying transformations to the indexed data won't support it.
+     * See CASSANDRA-8272 for further details.
+     *
+     * @param rowFilter rowFilter of query to decide if it supports replica filtering protection or not
+     * @return true if this index supports replica filtering protection, false otherwise
+     */
+    default boolean supportsReplicaFilteringProtection(RowFilter rowFilter)
+    {
+        return true;
+    }
+
     /**
      * Factory method for query time search helper.
      *
@@ -687,28 +755,16 @@ public interface Searcher
          * @return partitions from the base table matching the criteria of the search.
          */
         public UnfilteredPartitionIterator search(ReadExecutionController executionController);
-
-        /**
-         * Replica filtering protection may fetch data that doesn't match query conditions.
-         *
-         * On coordinator, we need to filter the replicas' responses again.
-         *
-         * This will not be called if {@link QueryPlan#supportsReplicaFilteringProtection(RowFilter)} returns false.
-         *
-         * @return filtered response that satisfied query conditions
-         */
-        default PartitionIterator filterReplicaFilteringProtection(PartitionIterator fullResponse)
-        {
-            return command().rowFilter().filter(fullResponse, command().metadata(), command().nowInSec());
-        }
     }
 
     /**
      * Class providing grouped operations for indexes that communicate with each other.
-     *
+     * </p>
      * Index implementations should provide a {@code Group} implementation calling to
-     * {@link SecondaryIndexManager#registerIndex(Index, Index.Group.Key, Supplier)} during index registering
-     * at {@link #register(IndexRegistry)} method.
+     * {@link IndexRegistry#registerIndex(Index, Key, Supplier)}
+     * at {@link #register(IndexRegistry)} method and provide {@code groupKey} calling to
+     * {@link IndexRegistry#unregisterIndex(Index, Key)} during index unregistering
+     * at {@link #unregister(IndexRegistry)} method
      */
     interface Group
     {
@@ -745,7 +801,7 @@ public int hashCode()
          *
          * @return the indexes that are members of this group
          */
-        Set<Index> getIndexes();
+        Set<? extends Index> getIndexes();
 
         /**
          * Adds the specified {@link Index} as a member of this group.
@@ -821,13 +877,13 @@ Indexer indexerFor(Predicate<Index> indexSelector,
         /**
          * Get flush observer to observe partition/cell events generated by flushing SSTable (memtable flush or compaction).
          *
-         * @param descriptor The descriptor of the sstable observer is requested for.
-         * @param tracker The {@link LifecycleNewTracker} associated with the SSTable being written
+         * @param descriptor    The descriptor of the sstable observer is requested for.
+         * @param tracker       The {@link LifecycleNewTracker} associated with the SSTable being written
          * @param tableMetadata The immutable metadata of the table at the moment the SSTable is flushed
-         *
+         * @param keyCount
          * @return SSTable flush observer.
          */
-        SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata);
+        SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata, long keyCount);
 
         /**
          * @param type index transaction type
@@ -845,11 +901,48 @@ default boolean handles(IndexTransaction.Type type)
         default void invalidate() { }
 
         /**
-         * Returns the SSTable-attached {@link Component}s created by this index group.
+         * Called when the table associated with this group has been unloaded. Implementations
+         * should dispose of any resources tied to the lifecycle of the {@link Group} without removing index files.
+         */
+        default void unload() { }
+
+        /**
+         * Returns the set of sstable-attached components that this group will create for a newly flushed sstable.
+         *
+         * Note that the result of this method is only valid for newly flushed/written sstables as the components
+         * returned will assume a version of {@link Version#latest()} and a generation of 0. SSTables for which some
+         * index have been rebuild may have index components that do not match what this method return in particular.
+         */
+        Set<Component> componentsForNewSSTable();
+
+        /**
+         * Return the set of sstable-attached components belonging to the group that are currently "active" for the
+         * provided sstable.
+         * <p>
+         * The "active" components are the components that are currently in use, meaning that if a given component
+         * of the sstable exists with multiple versions or generation on disk, only the most recent version/generation
+         * is the active one.
          *
-         * @return the SSTable components created by this group
+         * @param sstable the sstable to get components for.
+         * @return the set of the sstable-attached components of the provided sstable for this group.
          */
-        Set<Component> getComponents();
+        Set<Component> activeComponents(SSTableReader sstable);
+
+        /**
+         * @return true if this index group is capable of supporting multiple contains restrictions, false otherwise
+         */
+        default boolean supportsMultipleContains()
+        {
+            return false;
+        }
+
+        /**
+         * @return true is this index group supports disjunction queries of "a = 1 OR a = 2" or "a IN (1, 2)"
+         */
+        default boolean supportsDisjunction()
+        {
+            return false;
+        }
 
         /**
          * Validates all indexes in the group against the specified SSTables.
@@ -1024,6 +1117,14 @@ default boolean isTopK()
         {
             return false;
         }
+        
+        /**
+         * @return true if the indexes in this plan support querying multiple vnode ranges at once.
+         */
+        default boolean supportsMultiRangeReadCommand()
+        {
+            return false;
+        }
     }
 
     /*
diff --git a/src/java/org/apache/cassandra/index/IndexBuildDecider.java b/src/java/org/apache/cassandra/index/IndexBuildDecider.java
new file mode 100644
index 000000000000..56ec158f7305
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/IndexBuildDecider.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
+import org.apache.cassandra.utils.FBUtilities;
+
+public interface IndexBuildDecider
+{
+    IndexBuildDecider instance = CassandraRelevantProperties.CUSTOM_INDEX_BUILD_DECIDER.getString() == null ?
+                                 new IndexBuildDecider() {} :
+                                 FBUtilities.construct(CassandraRelevantProperties.CUSTOM_INDEX_BUILD_DECIDER.getString(), "custom index build decider");
+
+    enum Decision
+    {
+        /**
+         * index will be built synchronously
+         */
+        SYNC,
+        /**
+         * index will be built asynchronously
+         */
+        ASYNC,
+        /**
+         * index build will be skipped
+         */
+        NONE;
+
+        public boolean skipped()
+        {
+            return this == NONE;
+        }
+    }
+
+    /**
+     * CNDB overrides this method to skip building indexes for sstables.
+     *
+     * @return decision for index initial build {@link Index#getInitializationTask()}
+     */
+    default Decision onInitialBuild()
+    {
+        return Decision.SYNC;
+    }
+
+    /**
+     * CNDB overrides this method to mark index queryable if there is no sstables on writer.
+     *
+     * @return true if index should be queryable after {@link Index#getInitializationTask()}
+     */
+    default boolean isIndexQueryableAfterInitialBuild(ColumnFamilyStore cfs)
+    {
+        return true;
+    }
+
+    /**
+     * CNDB overrides this method to skip building indexes on writer when sstables are reloaded from remote storage
+     *
+     * @return decision for index initial build when receiving {@link SSTableListChangedNotification}
+     */
+    default Decision onSSTableListChanged(SSTableListChangedNotification notification)
+    {
+        return notification.operationType.equals(OperationType.REMOTE_RELOAD) ? Decision.ASYNC : Decision.NONE;
+    }
+
+    /**
+     * CNDB overrides this method to skip building indexes on writer when sstables are reloaded from remote storage
+     *
+     * @return decision for index initial build when receiving {@link SSTableAddedNotification}
+     */
+    default Decision onSSTableAdded(SSTableAddedNotification notification)
+    {
+        // SSTables associated to a memtable come from a flush, so their contents have already been indexed
+        if (notification.memtable().isPresent())
+            return Decision.NONE;
+
+        return notification.operationType == OperationType.REMOTE_RELOAD ? Decision.ASYNC : Decision.SYNC;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/IndexBuildInProgressException.java b/src/java/org/apache/cassandra/index/IndexBuildInProgressException.java
new file mode 100644
index 000000000000..ff4d31b77a24
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/IndexBuildInProgressException.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index;
+
+import org.apache.cassandra.exceptions.InternalRequestExecutionException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+
+/**
+ * Thrown if a secondary index is not currently available because it is building.
+ */
+public final class IndexBuildInProgressException extends RuntimeException implements InternalRequestExecutionException
+{
+    /**
+     * Creates a new <code>IndexIsBuildingException</code> for the specified index.
+     * @param index the index
+     */
+    public IndexBuildInProgressException(Index index)
+    {
+        super(String.format("The secondary index '%s' is not yet available as it is building", index.getIndexMetadata().name));
+    }
+
+    @Override
+    public RequestFailureReason getReason()
+    {
+        return RequestFailureReason.INDEX_BUILD_IN_PROGRESS;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/IndexNotAvailableException.java b/src/java/org/apache/cassandra/index/IndexNotAvailableException.java
index 5e5a753803ea..179b0cb3f508 100644
--- a/src/java/org/apache/cassandra/index/IndexNotAvailableException.java
+++ b/src/java/org/apache/cassandra/index/IndexNotAvailableException.java
@@ -18,10 +18,13 @@
 
 package org.apache.cassandra.index;
 
+import org.apache.cassandra.exceptions.InternalRequestExecutionException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+
 /**
  * Thrown if a secondary index is not currently available.
  */
-public final class IndexNotAvailableException extends RuntimeException
+public final class IndexNotAvailableException extends RuntimeException implements InternalRequestExecutionException
 {
     /**
      * Creates a new <code>IndexNotAvailableException</code> for the specified index.
@@ -31,4 +34,10 @@ public IndexNotAvailableException(Index index)
     {
         super(String.format("The secondary index '%s' is not yet available", index.getIndexMetadata().name));
     }
+
+    @Override
+    public RequestFailureReason getReason()
+    {
+        return RequestFailureReason.INDEX_NOT_AVAILABLE;
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/IndexRegistry.java b/src/java/org/apache/cassandra/index/IndexRegistry.java
index d29bb11db4b0..e448ba567fd3 100644
--- a/src/java/org/apache/cassandra/index/IndexRegistry.java
+++ b/src/java/org/apache/cassandra/index/IndexRegistry.java
@@ -25,6 +25,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
@@ -44,6 +45,7 @@
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
@@ -135,11 +137,7 @@ public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata)
             @Override
             public void register(IndexRegistry registry)
             {
-            }
 
-            @Override
-            public void unregister(IndexRegistry registry)
-            {
             }
 
             @Override
@@ -249,13 +247,19 @@ public Index.QueryPlan queryPlanFor(RowFilter rowFilter)
 
             @Nullable
             @Override
-            public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+            public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata, long keyCount)
+            {
+                return null;
+            }
+
+            @Override
+            public Set<Component> componentsForNewSSTable()
             {
                 return null;
             }
 
             @Override
-            public Set<Component> getComponents()
+            public Set<Component> activeComponents(SSTableReader sstable)
             {
                 return null;
             }
@@ -266,7 +270,6 @@ public void registerIndex(Index index, Index.Group.Key groupKey, Supplier<Index.
         {
         }
 
-        @Override
         public void unregisterIndex(Index index, Index.Group.Key groupKey)
         {
         }
@@ -303,18 +306,41 @@ public void validate(PartitionUpdate update, ClientState state)
 
     default void registerIndex(Index index)
     {
-        registerIndex(index, new Index.Group.Key(index), () -> new SingletonIndexGroup(index));
+        registerIndex(index, new Index.Group.Key(index), () -> new SingletonIndexGroup());
     }
-
     void registerIndex(Index index, Index.Group.Key groupKey, Supplier<Index.Group> groupSupplier);
-
     void unregisterIndex(Index index, Index.Group.Key groupKey);
-
     Collection<Index.Group> listIndexGroups();
 
     Index getIndex(IndexMetadata indexMetadata);
     Collection<Index> listIndexes();
 
+    default Optional<Index.Analyzer> getIndexAnalyzerFor(ColumnMetadata column, Operator operator)
+    {
+        return getAnalyzerFor(column, operator, Index::getIndexAnalyzer);
+    }
+
+    default Optional<Index.Analyzer> getQueryAnalyzerFor(ColumnMetadata column, Operator operator)
+    {
+        return getAnalyzerFor(column, operator, Index::getQueryAnalyzer);
+    }
+
+    default Optional<Index.Analyzer> getAnalyzerFor(ColumnMetadata column,
+                                                    Operator operator,
+                                                    Function<Index, Optional<Index.Analyzer>> analyzerGetter)
+    {
+        for (Index index : listIndexes())
+        {
+            if (index.supportsExpression(column, operator))
+            {
+                Optional<Index.Analyzer> analyzer = analyzerGetter.apply(index);
+                if (analyzer.isPresent())
+                    return analyzer;
+            }
+        }
+        return Optional.empty();
+    }
+
     Optional<Index> getBestIndexFor(RowFilter.Expression expression);
 
     /**
@@ -342,4 +368,77 @@ static IndexRegistry obtain(TableMetadata table)
 
         return table.isVirtual() ? EMPTY : Keyspace.openAndGetStore(table).indexManager;
     }
+
+    enum EqBehavior
+    {
+        EQ,
+        MATCH,
+        AMBIGUOUS
+    }
+
+    class EqBehaviorIndexes
+    {
+        public EqBehavior behavior;
+        public final Index eqIndex;
+        public final Index matchIndex;
+
+        private EqBehaviorIndexes(Index eqIndex, Index matchIndex, EqBehavior behavior)
+        {
+            this.eqIndex = eqIndex;
+            this.matchIndex = matchIndex;
+            this.behavior = behavior;
+        }
+
+        public static EqBehaviorIndexes eq(Index eqIndex)
+        {
+            return new EqBehaviorIndexes(eqIndex, null, EqBehavior.EQ);
+        }
+
+        public static EqBehaviorIndexes match(Index eqAndMatchIndex)
+        {
+            return new EqBehaviorIndexes(eqAndMatchIndex, eqAndMatchIndex, EqBehavior.MATCH);
+        }
+
+        public static EqBehaviorIndexes ambiguous(Index firstEqIndex, Index secondEqIndex)
+        {
+            return new EqBehaviorIndexes(firstEqIndex, secondEqIndex, EqBehavior.AMBIGUOUS);
+        }
+    }
+
+    /**
+     * @return
+     * - AMBIGUOUS if an index supports EQ and a different one supports both EQ and ANALYZER_MATCHES
+     * - MATCHES if an index supports both EQ and ANALYZER_MATCHES
+     * - otherwise EQ
+     */
+    default EqBehaviorIndexes getEqBehavior(ColumnMetadata cm)
+    {
+        Index eqOnlyIndex = null;
+        Index bothIndex = null;
+
+        for (Index index : listIndexes())
+        {
+            boolean supportsEq = index.supportsExpression(cm, Operator.EQ);
+            boolean supportsMatches = index.supportsExpression(cm, Operator.ANALYZER_MATCHES);
+            // This is an edge case due to the NON_DAEMON IndexRegistry, which doesn't have index metadata and
+            // which uses regular equality by convention.
+            boolean hasIndexMetadata = index.getIndexMetadata() != null;
+
+            if (supportsEq && supportsMatches && hasIndexMetadata)
+                bothIndex = index;
+            else if (supportsEq)
+                eqOnlyIndex = index;
+        }
+
+        // If we have one index supporting only EQ and another supporting both, return AMBIGUOUS
+        if (eqOnlyIndex != null && bothIndex != null)
+            return EqBehaviorIndexes.ambiguous(eqOnlyIndex, bothIndex);
+
+        // If we have an index supporting both EQ and MATCHES, return MATCHES
+        if (bothIndex != null)
+            return EqBehaviorIndexes.match(bothIndex);
+
+        // Otherwise return EQ
+        return EqBehaviorIndexes.eq(eqOnlyIndex == null ? bothIndex : eqOnlyIndex);
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/IndexStatusManager.java b/src/java/org/apache/cassandra/index/IndexStatusManager.java
index 1c0f5887db1a..e44033da69e5 100644
--- a/src/java/org/apache/cassandra/index/IndexStatusManager.java
+++ b/src/java/org/apache/cassandra/index/IndexStatusManager.java
@@ -89,6 +89,8 @@ public <E extends Endpoints<E>> E filterForQuery(E liveEndpoints, Keyspace keysp
         // UNKNOWN states are transient/rare; only a few replicas should have this state at any time. See CASSANDRA-19400
         Set<Replica> queryableNonSucceeded = new HashSet<>(4);
 
+        Map<InetAddressAndPort, Index.Status> indexStatusMap = new HashMap<>();
+
         E queryableEndpoints = liveEndpoints.filter(replica -> {
 
             boolean allBuilt = true;
@@ -96,7 +98,10 @@ public <E extends Endpoints<E>> E filterForQuery(E liveEndpoints, Keyspace keysp
             {
                 Index.Status status = getIndexStatus(replica.endpoint(), keyspace.getName(), index.getIndexMetadata().name);
                 if (!index.isQueryable(status))
+                {
+                    indexStatusMap.put(replica.endpoint(), status);
                     return false;
+                }
 
                 if (status != Index.Status.BUILD_SUCCEEDED)
                     allBuilt = false;
@@ -124,7 +129,14 @@ public <E extends Endpoints<E>> E filterForQuery(E liveEndpoints, Keyspace keysp
             {
                 Map<InetAddressAndPort, RequestFailureReason> failureReasons = new HashMap<>();
                 liveEndpoints.without(queryableEndpoints.endpoints())
-                             .forEach(replica -> failureReasons.put(replica.endpoint(), RequestFailureReason.INDEX_NOT_AVAILABLE));
+//                             .forEach(replica -> failureReasons.put(replica.endpoint(), RequestFailureReason.INDEX_NOT_AVAILABLE));
+                             .forEach(replica -> {
+                                 Index.Status status = indexStatusMap.get(replica.endpoint());
+                                 if (status == Index.Status.FULL_REBUILD_STARTED)
+                                     failureReasons.put(replica.endpoint(), RequestFailureReason.INDEX_BUILD_IN_PROGRESS);
+                                 else
+                                     failureReasons.put(replica.endpoint(), RequestFailureReason.INDEX_NOT_AVAILABLE);
+                             });
 
                 throw new ReadFailureException(level, filtered, required, false, failureReasons);
             }
diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexBuilder.java b/src/java/org/apache/cassandra/index/SecondaryIndexBuilder.java
index 73dc3345a250..eeaf0ce81d70 100644
--- a/src/java/org/apache/cassandra/index/SecondaryIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/SecondaryIndexBuilder.java
@@ -17,12 +17,12 @@
  */
 package org.apache.cassandra.index;
 
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 
 /**
  * Manages building an entire index from column family data. Runs on to compaction manager.
  */
-public abstract class SecondaryIndexBuilder extends CompactionInfo.Holder
+public abstract class SecondaryIndexBuilder extends AbstractTableOperation
 {
     public abstract void build();
 
diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
index ba4132654c8d..01e5a186d699 100644
--- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
+++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
@@ -19,8 +19,21 @@
 
 import java.io.UncheckedIOException;
 import java.lang.reflect.Constructor;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.StringJoiner;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -29,7 +42,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
-
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -41,6 +54,9 @@
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;  //checkstyle: permit this import
+import com.google.common.util.concurrent.ListenableFuture;  //checkstyle: permit this import
+import com.google.common.util.concurrent.SettableFuture;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,7 +65,23 @@
 import org.apache.cassandra.concurrent.FutureTask;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.MutableDeletionInfo;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.WriteContext;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
 import org.apache.cassandra.db.filter.ColumnFilter;
@@ -60,7 +92,14 @@
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowDiffListener;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.Index.IndexBuildingSupport;
 import org.apache.cassandra.index.internal.CassandraIndex;
@@ -69,9 +108,11 @@
 import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.index.transactions.UpdateTransaction;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.notifications.INotification;
 import org.apache.cassandra.notifications.INotificationConsumer;
 import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.Indexes;
@@ -81,7 +122,11 @@
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.concurrent.*;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
+import org.apache.cassandra.utils.concurrent.Promise;
+import org.apache.cassandra.utils.concurrent.Refs;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.config.CassandraRelevantProperties.FORCE_DEFAULT_INDEXING_PAGE_SIZE;
@@ -163,7 +208,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
     /**
      * The groups of all the registered indexes
      */
-    private final Map<Index.Group.Key, Index.Group> indexGroups = Maps.newConcurrentMap();
+    private final ConcurrentMap<Index.Group.Key, Index.Group> indexGroups = Maps.newConcurrentMap();
 
     /**
      * The count of pending index builds for each index.
@@ -233,15 +278,18 @@ private synchronized Future<Void> createIndex(IndexMetadata indexDef, boolean is
     @VisibleForTesting
     public Future<Void> buildIndex(final Index index)
     {
-        FutureTask<?> initialBuildTask = null;
+        FutureTask<?> initialBuildTask = new FutureTask<>(() -> null);
         // if the index didn't register itself, we can probably assume that no initialization needs to happen
         if (indexes.containsKey(index.getIndexMetadata().name))
         {
             try
             {
-                Callable<?> call = DatabaseDescriptor.isDaemonInitialized() ? index.getInitializationTask() : null;
-                if (call != null)
-                    initialBuildTask = new FutureTask<>(call);
+                if (!index.shouldSkipInitialization()) 
+                {
+                    Callable<?> call = index.getInitializationTask();
+                    if (call != null)
+                        initialBuildTask = new FutureTask<>(call);
+                }
             }
             catch (Throwable t)
             {
@@ -250,13 +298,6 @@ public Future<Void> buildIndex(final Index index)
             }
         }
 
-        // if there's no initialization, just mark as built and return:
-        if (initialBuildTask == null)
-        {
-            markIndexBuilt(index, true);
-            return ImmediateFuture.success(null);
-        }
-
         // otherwise run the initialization task asynchronously with a callback to mark it built or failed
         final Promise<Void> initialization = new AsyncPromise<>();
         // we want to ensure we invoke this task asynchronously, so we want to add our callback before submission
@@ -264,7 +305,8 @@ public Future<Void> buildIndex(final Index index)
         // This is because Keyspace.open("system") can transitively attempt to open Keyspace.open("system")
         initialBuildTask.addCallback(
             success -> {
-                markIndexBuilt(index, true);
+                if (IndexBuildDecider.instance.isIndexQueryableAfterInitialBuild(baseCfs))
+                    markIndexBuilt(index, true);
                 initialization.trySuccess(null);
             },
             failure -> {
@@ -300,7 +342,18 @@ public synchronized Future<?> addIndex(IndexMetadata indexDef, boolean isNewCF)
      */
     public boolean isIndexQueryable(Index index)
     {
-        return queryableIndexes.contains(index.getIndexMetadata().name);
+        return isIndexQueryable(index.getIndexMetadata().name);
+    }
+
+    /**
+     * Checks if the specified index is queryable.
+     *
+     * @param indexName name of the index
+     * @return <code>true</code> if the specified index is registered, <code>false</code> otherwise
+     */
+    public boolean isIndexQueryable(String indexName)
+    {
+        return queryableIndexes.contains(indexName);
     }
 
     /**
@@ -312,10 +365,22 @@ public boolean isIndexQueryable(Index index)
      */
     public void checkQueryability(Index.QueryPlan queryPlan)
     {
+        InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort();
+
         for (Index index : queryPlan.getIndexes())
         {
+            String indexName = index.getIndexMetadata().name;
+            Index.Status indexStatus = IndexStatusManager.instance.getIndexStatus(endpoint, keyspace.getName(), indexName);
+
             if (!isIndexQueryable(index))
+            {
+                // In Astra index can be queryable during index build, thus we need to check both not queryable and building
+                // Plus isQueryable is always true for non-SAI index implementations
+                if (indexStatus == Index.Status.FULL_REBUILD_STARTED)
+                    throw new IndexBuildInProgressException(index);
+
                 throw new IndexNotAvailableException(index);
+            }
         }
     }
 
@@ -345,17 +410,19 @@ public synchronized boolean isIndexBuilding(String indexName)
 
     public synchronized void removeIndex(String indexName)
     {
-        Index removedIndex = indexes.remove(indexName);
+        Index removed = indexes.remove(indexName);
+        logger.trace(removed == null ? "Index {} was not registered" : "Removed index {} from registry", indexName);
 
-        if (removedIndex != null)
+        if (null != removed)
         {
-            removedIndex.unregister(this);
+            removed.unregister(this);
 
-            markIndexRemoved(indexName);
-            executeBlocking(removedIndex.getInvalidateTask(), null);
+            markIndexRemoved(removed);
+            executeBlocking(removed.getInvalidateTask(), null);
         }
     }
 
+
     public Set<IndexMetadata> getDependentIndexes(ColumnMetadata column)
     {
         if (indexes.isEmpty())
@@ -374,7 +441,7 @@ public Set<IndexMetadata> getDependentIndexes(ColumnMetadata column)
      */
     public void markAllIndexesRemoved()
     {
-        getBuiltIndexNames().forEach(this::markIndexRemoved);
+        listIndexes().forEach(this::markIndexRemoved);
     }
 
     /**
@@ -499,14 +566,14 @@ public static String getIndexName(String cfName)
     }
 
     /**
-     * Validates all index groups against the specified SSTables. 
+     * Validates all index groups against the specified SSTables.
      *
      * @param sstables SSTables for which indexes in the group should be built
      * @param throwOnIncomplete whether to throw an error if any index in the group is incomplete
      * @param validateChecksum whether to validate checksum or not
      *
      * @return true if all indexes in all groups are complete and valid
-     *         false if an index in any group is incomplete and {@code throwOnIncomplete} is false 
+     *         false if an index in any group is incomplete and {@code throwOnIncomplete} is false
      *
      * @throws IllegalStateException if {@code throwOnIncomplete} is true and an index in any group is incomplete
      * @throws UncheckedIOException if there is a problem validating any on-disk component in any group
@@ -556,27 +623,28 @@ public void buildSSTableAttachedIndexesBlocking(Collection<SSTableReader> sstabl
         }
 
         // Schedule all index building tasks with callbacks to handle success and failure
-        List<Future<?>> futures = new ArrayList<>(byType.size());
+        List<ListenableFuture<?>> futures = new ArrayList<>(byType.size());
         byType.forEach((buildingSupport, groupedIndexes) ->
         {
-            SecondaryIndexBuilder builder = buildingSupport.getIndexBuildTask(baseCfs, groupedIndexes, sstables, false);
-            AsyncPromise<Object> build = new AsyncPromise<>();
-            CompactionManager.instance.submitIndexBuild(builder).addCallback(new FutureCallback<Object>()
+            List<SecondaryIndexBuilder> builders = buildingSupport.getParallelIndexBuildTasks(baseCfs, groupedIndexes, sstables, false);
+            List<ListenableFuture<?>> builderFutures = builders.stream().map(CompactionManager.instance::submitIndexBuild).collect(Collectors.toList());
+            final SettableFuture<Object> build = SettableFuture.create();
+            Futures.addCallback(Futures.allAsList(builderFutures), new FutureCallback<Object>()
             {
                 @Override
                 public void onFailure(Throwable t)
                 {
                     logger.warn("Failed to incrementally build indexes {}", getIndexNames(groupedIndexes));
-                    build.tryFailure(t);
+                    build.setException(t);
                 }
 
                 @Override
                 public void onSuccess(Object o)
                 {
                     logger.info("Incremental index build of {} completed", getIndexNames(groupedIndexes));
-                    build.trySuccess(o);
+                    build.set(o);
                 }
-            });
+            }, ImmediateExecutor.INSTANCE);
             futures.add(build);
         });
 
@@ -595,10 +663,22 @@ public void onSuccess(Object o)
      * @param isFullRebuild True if this method is invoked as a full index rebuild, false otherwise
      */
     @SuppressWarnings({"unchecked", "RedundantSuppression"})
-    private void buildIndexesBlocking(Collection<SSTableReader> sstables, Set<Index> indexes, boolean isFullRebuild)
+    public void buildIndexesBlocking(Collection<SSTableReader> sstables, Set<Index> indexes, boolean isFullRebuild)
+    {
+        FBUtilities.waitOnFuture(buildIndexesAsync(sstables, indexes, isFullRebuild));
+    }
+
+    /**
+     * Performs an asynchronous (re)indexing of the specified SSTables for the specified indexes.
+     *
+     * @param sstables the SSTables to be (re)indexed
+     * @param indexes the indexes to be (re)built for the specified SSTables
+     * @param isFullRebuild True if this method is invoked as a full index rebuild, false otherwise
+     */
+    private java.util.concurrent.Future<?> buildIndexesAsync(Collection<SSTableReader> sstables, Set<Index> indexes, boolean isFullRebuild)
     {
         if (indexes.isEmpty())
-            return;
+            return ImmediateFuture.success(null);
 
         // Mark all indexes as building: this step must happen first, because if any index can't be marked, the whole
         // process needs to abort
@@ -608,104 +688,108 @@ private void buildIndexesBlocking(Collection<SSTableReader> sstables, Set<Index>
         final Set<Index> builtIndexes = Sets.newConcurrentHashSet();
         final Set<Index> unbuiltIndexes = Sets.newConcurrentHashSet();
 
-        // Any exception thrown during index building that could be suppressed by the finally block
-        Exception accumulatedFail = null;
+        logger.info("Submitting index {} of {} for data in {}",
+                    isFullRebuild ? "recovery" : "build",
+                    commaSeparated(indexes),
+                    sstables.stream().map(SSTableReader::toString).collect(Collectors.joining(",")));
 
-        try
+        // Group all building tasks
+        Map<Index.IndexBuildingSupport, Set<Index>> byType = new HashMap<>();
+        for (Index index : indexes)
         {
-            logger.info("Submitting index {} of {} for data in {}",
-                        isFullRebuild ? "recovery" : "build",
-                        commaSeparated(indexes),
-                        sstables.stream().map(SSTableReader::toString).collect(Collectors.joining(",")));
-
-            // Group all building tasks
-            Map<Index.IndexBuildingSupport, Set<Index>> byType = new HashMap<>();
-            for (Index index : indexes)
-            {
-                IndexBuildingSupport buildOrRecoveryTask = isFullRebuild
-                                                           ? index.getBuildTaskSupport()
-                                                           : index.getRecoveryTaskSupport();
-                Set<Index> stored = byType.computeIfAbsent(buildOrRecoveryTask, i -> new HashSet<>());
-                stored.add(index);
-            }
+            IndexBuildingSupport buildOrRecoveryTask = isFullRebuild
+                                                       ? index.getBuildTaskSupport()
+                                                       : index.getRecoveryTaskSupport();
+            Set<Index> stored = byType.computeIfAbsent(buildOrRecoveryTask, i -> new HashSet<>());
+            stored.add(index);
+        }
 
-            // Schedule all index building tasks with a callback to mark them as built or failed
-            List<Future<?>> futures = new ArrayList<>(byType.size());
-            byType.forEach((buildingSupport, groupedIndexes) ->
+        // Schedule all index building tasks with a callback to mark them as built or failed
+        List<ListenableFuture<?>> futures = new ArrayList<>(byType.size());
+        byType.forEach((buildingSupport, groupedIndexes) ->
+                       {
+                           List<SecondaryIndexBuilder> builders = buildingSupport.getParallelIndexBuildTasks(baseCfs, groupedIndexes, sstables, isFullRebuild);
+                           List<ListenableFuture<?>> builderFutures = builders.stream().map(CompactionManager.instance::submitIndexBuild).collect(Collectors.toList());
+                           final SettableFuture<Object> build = SettableFuture.create();
+                           Futures.addCallback(Futures.allAsList(builderFutures), new FutureCallback<Object>()
                            {
-                               SecondaryIndexBuilder builder = buildingSupport.getIndexBuildTask(baseCfs, groupedIndexes, sstables, isFullRebuild);
-                               final AsyncPromise<Object> build = new AsyncPromise<>();
-                               CompactionManager.instance.submitIndexBuild(builder).addCallback(new FutureCallback<Object>()
+                               @Override
+                               public void onFailure(Throwable t)
                                {
-                                   @Override
-                                   public void onFailure(Throwable t)
-                                   {
-                                       logAndMarkIndexesFailed(groupedIndexes, t, false);
-                                       unbuiltIndexes.addAll(groupedIndexes);
-                                       build.tryFailure(t);
-                                   }
-
-                                   @Override
-                                   public void onSuccess(Object o)
-                                   {
-                                       groupedIndexes.forEach(i -> markIndexBuilt(i, isFullRebuild));
-                                       logger.info("Index build of {} completed", getIndexNames(groupedIndexes));
-                                       builtIndexes.addAll(groupedIndexes);
-                                       build.trySuccess(o);
-                                   }
-                               });
-                               futures.add(build);
-                           });
-
-            // Finally wait for the index builds to finish and flush the indexes that built successfully
-            FBUtilities.waitOnFutures(futures);
-        }
-        catch (Exception e)
-        {
-            accumulatedFail = e;
-            throw e;
-        }
-        finally
-        {
+                                   logAndMarkIndexesFailed(groupedIndexes, t, false);
+                                   unbuiltIndexes.addAll(groupedIndexes);
+                                   build.setException(t);
+                               }
+
+                               @Override
+                               public void onSuccess(Object o)
+                               {
+                                   groupedIndexes.forEach(i -> markIndexBuilt(i, isFullRebuild));
+                                   logger.info("Index build of {} completed", getIndexNames(groupedIndexes));
+                                   builtIndexes.addAll(groupedIndexes);
+                                   build.set(o);
+                               }
+                           }, ImmediateExecutor.INSTANCE);
+                           futures.add(build);
+                       });
+
+        ListenableFuture<List<Object>> allIndexBuilds = Futures.allAsList(futures);
+        SettableFuture finalResult = SettableFuture.create();
+        allIndexBuilds.addListener(() -> {
             try
             {
-                // Fail any indexes that couldn't be marked
-                Set<Index> failedIndexes = Sets.difference(indexes, Sets.union(builtIndexes, unbuiltIndexes));
-                if (!failedIndexes.isEmpty())
-                {
-                    logAndMarkIndexesFailed(failedIndexes, accumulatedFail, false);
-                }
-
-                // Flush all built indexes with an aynchronous callback to log the success or failure of the flush
-                flushIndexesBlocking(builtIndexes, new FutureCallback<>()
-                {
-                    final String indexNames = StringUtils.join(builtIndexes.stream()
-                                                                           .map(i -> i.getIndexMetadata().name)
-                                                                           .collect(Collectors.toList()), ',');
-
-                    @Override
-                    public void onFailure(Throwable ignored)
-                    {
-                        logger.info("Index flush of {} failed", indexNames);
-                    }
+                finalizeIndexBuild(indexes, builtIndexes, unbuiltIndexes);
+                finalResult.setFuture(allIndexBuilds);
+            }
+            catch (Exception ex)
+            {
+                finalResult.setException(ex);
+            }
+        }, ImmediateExecutor.INSTANCE);
+        return finalResult;
+    }
 
-                    @Override
-                    public void onSuccess(Object ignored)
-                    {
-                        logger.info("Index flush of {} completed", indexNames);
-                    }
-                });
+    private void finalizeIndexBuild(Set<Index> indexes, Set<Index> builtIndexes, Set<Index> unbuiltIndexes)
+    {
+        Exception accumulatedFail = null;
+        try
+        {
+            // Fail any indexes that couldn't be marked
+            Set<Index> failedIndexes = Sets.difference(indexes, Sets.union(builtIndexes, unbuiltIndexes));
+            if (!failedIndexes.isEmpty())
+            {
+                logAndMarkIndexesFailed(failedIndexes, accumulatedFail, false);
             }
-            catch (Exception e)
+
+            // Flush all built indexes with an aynchronous callback to log the success or failure of the flush
+            flushIndexesBlocking(builtIndexes, new FutureCallback()
             {
-                if (accumulatedFail != null)
+                String indexNames = StringUtils.join(builtIndexes.stream()
+                                                                 .map(i -> i.getIndexMetadata().name)
+                                                                 .collect(Collectors.toList()), ',');
+
+                @Override
+                public void onFailure(Throwable ignored)
                 {
-                    accumulatedFail.addSuppressed(e);
+                    logger.info("Index flush of {} failed", indexNames);
                 }
-                else
+
+                @Override
+                public void onSuccess(Object ignored)
                 {
-                    throw e;
+                    logger.info("Index flush of {} completed", indexNames);
                 }
+            });
+        }
+        catch (Exception e)
+        {
+            if (accumulatedFail != null)
+            {
+                accumulatedFail.addSuppressed(e);
+            }
+            else
+            {
+                throw e;
             }
         }
     }
@@ -786,12 +870,12 @@ public synchronized void markIndexesBuilding(Set<Index> indexes, boolean isFullR
      * @param index the index to be marked as built
      * @param isFullRebuild {@code true} if this method is invoked as a full index rebuild, {@code false} otherwise
      */
-    private synchronized void markIndexBuilt(Index index, boolean isFullRebuild)
+    public synchronized void markIndexBuilt(Index index, boolean isFullRebuild)
     {
         String indexName = index.getIndexMetadata().name;
         if (isFullRebuild)
             makeIndexQueryable(index, Index.Status.BUILD_SUCCEEDED);
-        
+
         AtomicInteger counter = inProgressBuilds.get(indexName);
         if (counter != null)
         {
@@ -833,6 +917,8 @@ private synchronized void markIndexFailed(Index index, boolean isInitialBuild)
 
             if (!index.getSupportedLoadTypeOnFailure(isInitialBuild).supportsReads() && queryableIndexes.remove(indexName))
                 logger.info("Index [{}] became not-queryable because of failed build.", indexName);
+
+            makeIndexNonQueryable(index, Index.Status.BUILD_FAILED);
         }
     }
 
@@ -849,10 +935,11 @@ private void logAndMarkIndexesFailed(Set<Index> indexes, Throwable indexBuildFai
     /**
      * Marks the specified index as removed.
      *
-     * @param indexName the index name
+     * @param index the index to be removed
      */
-    private synchronized void markIndexRemoved(String indexName)
+    private synchronized void markIndexRemoved(Index index)
     {
+        String indexName = index.getIndexMetadata().name;
         SystemKeyspace.setIndexRemoved(baseCfs.getKeyspaceName(), indexName);
         queryableIndexes.remove(indexName);
         writableIndexes.remove(indexName);
@@ -873,8 +960,8 @@ private Index createInstance(IndexMetadata indexDef)
         if (indexDef.isCustom())
         {
             assert indexDef.options != null;
-            // Get the fully qualified index class name from the index metadata
-            String className = indexDef.getIndexClassName();
+            // Find any aliases to the fully qualified index class name:
+            String className = IndexMetadata.expandAliases(indexDef.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
             assert !Strings.isNullOrEmpty(className);
 
             try
@@ -916,6 +1003,15 @@ public void dropAllIndexes(boolean dropData)
         indexGroups.forEach((key, group) -> group.invalidate());
     }
 
+    /**
+     * unload all indexes without removing index data
+     */
+    public void unloadAllIndexes()
+    {
+        executeAllBlocking(indexes.values().stream(), Index::getUnloadTask, null);
+        indexGroups.forEach((key, group) -> group.unload());
+    }
+
     @VisibleForTesting
     public void invalidateAllIndexesBlocking()
     {
@@ -943,7 +1039,7 @@ public void flushIndexesBlocking(Set<Index> indexes)
      */
     public void executePreJoinTasksBlocking(boolean hadBootstrap)
     {
-        logger.info("Executing pre-join{} tasks for: {}", hadBootstrap ? " post-bootstrap" : "", this.baseCfs);
+        logger.debug("Executing pre-join{} tasks for: {}", hadBootstrap ? " post-bootstrap" : "", this.baseCfs);
         executeAllBlocking(indexes.values().stream(), (index) ->
         {
             return index.getPreJoinTask(hadBootstrap);
@@ -1014,7 +1110,7 @@ public boolean hasIndexes()
         return !indexes.isEmpty();
     }
 
-    public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize)
+    public void indexPartition(DecoratedKey key, Set<Index> indexes, PageSize pageSize)
     {
         indexPartition(key, indexes, pageSize, baseCfs.metadata().regularAndStaticColumns());
     }
@@ -1027,7 +1123,7 @@ public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize)
      * @param pageSize the number of {@link Unfiltered} objects to process in a single page
      * @param columns the columns indexed by at least one of the supplied indexes
      */
-    public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize, RegularAndStaticColumns columns)
+    public void indexPartition(DecoratedKey key, Set<Index> indexes, PageSize pageSize, RegularAndStaticColumns columns)
     {
         if (logger.isTraceEnabled())
             logger.trace("Indexing partition {}", baseCfs.metadata().partitionKeyType.getString(key.getKey()));
@@ -1057,20 +1153,16 @@ public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize, R
 
                     try (UnfilteredRowIterator partition = page.next())
                     {
-                        Set<Index.Indexer> indexers = new HashSet<>(indexGroups.size());
-
-                        for (Index.Group g : indexGroups.values())
-                        {
-                            Index.Indexer indexerFor = g.indexerFor(indexes::contains,
-                                                                    key,
-                                                                    partition.columns(),
-                                                                    nowInSec,
-                                                                    ctx,
-                                                                    IndexTransaction.Type.UPDATE,
-                                                                    null);
-                            if (indexerFor != null)
-                                indexers.add(indexerFor);
-                        }
+                        Set<Index.Indexer> indexers = indexGroups.values().stream()
+                                                                 .map(g -> g.indexerFor(indexes::contains,
+                                                                                        key,
+                                                                                        partition.columns(),
+                                                                                        nowInSec,
+                                                                                        ctx,
+                                                                                        IndexTransaction.Type.UPDATE,
+                                                                                        null))
+                                                                 .filter(Objects::nonNull)
+                                                                 .collect(Collectors.toSet());
 
                         // Short-circuit empty partitions if static row is processed or isn't read
                         if (!readStatic && partition.isEmpty() && partition.staticRow().isEmpty())
@@ -1126,39 +1218,12 @@ public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize, R
     /**
      * Return the page size used when indexing an entire partition
      */
-    public int calculateIndexingPageSize()
+    public PageSize calculateIndexingPageSize()
     {
         if (FORCE_DEFAULT_INDEXING_PAGE_SIZE.getBoolean())
-            return DEFAULT_PAGE_SIZE;
-
-        double targetPageSizeInBytes = 32 * 1024 * 1024;
-        double meanPartitionSize = baseCfs.getMeanPartitionSize();
-        if (meanPartitionSize <= 0)
-            return DEFAULT_PAGE_SIZE;
-
-        int meanCellsPerPartition = baseCfs.getMeanEstimatedCellPerPartitionCount();
-        if (meanCellsPerPartition <= 0)
-            return DEFAULT_PAGE_SIZE;
-
-        int columnsPerRow = baseCfs.metadata().regularColumns().size();
-        if (columnsPerRow <= 0)
-            return DEFAULT_PAGE_SIZE;
+            return PageSize.inRows(DEFAULT_PAGE_SIZE);
 
-        int meanRowsPerPartition = meanCellsPerPartition / columnsPerRow;
-        double meanRowSize = meanPartitionSize / meanRowsPerPartition;
-
-        int pageSize = (int) Math.max(1, Math.min(DEFAULT_PAGE_SIZE, targetPageSizeInBytes / meanRowSize));
-
-        logger.trace("Calculated page size {} for indexing {}.{} ({}/{}/{}/{})",
-                     pageSize,
-                     baseCfs.metadata.keyspace,
-                     baseCfs.metadata.name,
-                     meanPartitionSize,
-                     meanCellsPerPartition,
-                     meanRowsPerPartition,
-                     meanRowSize);
-
-        return pageSize;
+        return PageSize.inBytes(32 * 1024 * 1024);
     }
 
     /**
@@ -1227,7 +1292,7 @@ public Index.QueryPlan getBestIndexQueryPlanFor(RowFilter rowFilter)
         if (indexes.isEmpty() || rowFilter.isEmpty())
             return null;
 
-        for (RowFilter.Expression expression : rowFilter)
+        for (RowFilter.Expression expression : rowFilter.expressions())
         {
             if (expression.isCustom())
             {
@@ -1341,7 +1406,7 @@ public void registerIndex(Index index, Index.Group.Key groupKey, Supplier<Index.
         // instantiate and add the index group if it hasn't been already added
         Index.Group group = indexGroups.computeIfAbsent(groupKey, k -> groupSupplier.get());
 
-        // add the created index to its group if it is not a singleton group
+        // add the created index to its group
         group.addIndex(index);
     }
 
@@ -1354,8 +1419,8 @@ public void unregisterIndex(Index removed, Index.Group.Key groupKey)
             // Remove the index from non-singleton groups...
             group.removeIndex(removed);
 
-            // if the group is a singleton or there are no more indexes left in the group, remove it
-            if (group.isSingleton() || group.getIndexes().isEmpty())
+            // if no more indexes left in the group, remove it
+            if (group.getIndexes().isEmpty())
             {
                 Index.Group removedGroup = indexGroups.remove(groupKey);
                 if (removedGroup != null)
@@ -1364,7 +1429,7 @@ public void unregisterIndex(Index removed, Index.Group.Key groupKey)
         }
     }
 
-    public Index getIndex(IndexMetadata metadata)
+    public Index getIndex(@Nonnull IndexMetadata metadata)
     {
         return indexes.get(metadata.name);
     }
@@ -1393,7 +1458,7 @@ public Index.Group getIndexGroup(Index.Group.Key key)
      * associated to any group
      */
     @Nullable
-    public Index.Group getIndexGroup(IndexMetadata metadata)
+    public Index.Group getIndexGroup(@Nonnull IndexMetadata metadata)
     {
         Index index = getIndex(metadata);
         return index == null ? null : getIndexGroup(index);
@@ -1428,23 +1493,18 @@ public UpdateTransaction newUpdateTransaction(PartitionUpdate update, WriteConte
         if (!hasIndexes())
             return UpdateTransaction.NO_OP;
 
-        List<Index.Indexer> indexers = new ArrayList<>(indexGroups.size());
+        Index.Indexer[] indexers = listIndexGroups().stream()
+                                                    .map(g -> g.indexerFor(writableIndexSelector(),
+                                                                           update.partitionKey(),
+                                                                           update.columns(),
+                                                                           nowInSec,
+                                                                           ctx,
+                                                                           IndexTransaction.Type.UPDATE,
+                                                                           memtable))
+                                                    .filter(Objects::nonNull)
+                                                    .toArray(Index.Indexer[]::new);
 
-        for (Index.Group g : indexGroups.values())
-        {
-            Index.Indexer indexer = g.indexerFor(writableIndexSelector(),
-                                                 update.partitionKey(),
-                                                 update.columns(),
-                                                 nowInSec,
-                                                 ctx,
-                                                 IndexTransaction.Type.UPDATE,
-                                                 memtable);
-            if (indexer != null)
-                indexers.add(indexer);
-        }
-
-        return indexers.isEmpty() ? UpdateTransaction.NO_OP
-                                  : new WriteTimeTransaction(indexers.toArray(Index.Indexer[]::new));
+        return indexers.length == 0 ? UpdateTransaction.NO_OP : new WriteTimeTransaction(indexers);
     }
 
     private Predicate<Index> writableIndexSelector()
@@ -1542,19 +1602,19 @@ public void onUpdated(Row existing, Row updated)
             // diff listener collates the columns to be added & removed from the indexes
             RowDiffListener diffListener = new RowDiffListener()
             {
-                public void onPrimaryKeyLivenessInfo(int i, Clustering<?> clustering, LivenessInfo merged, LivenessInfo original)
+                public void onPrimaryKeyLivenessInfo(int i, Clustering clustering, LivenessInfo merged, LivenessInfo original)
                 {
                 }
 
-                public void onDeletion(int i, Clustering<?> clustering, Row.Deletion merged, Row.Deletion original)
+                public void onDeletion(int i, Clustering clustering, Row.Deletion merged, Row.Deletion original)
                 {
                 }
 
-                public void onComplexDeletion(int i, Clustering<?> clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original)
+                public void onComplexDeletion(int i, Clustering clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original)
                 {
                 }
 
-                public void onCell(int i, Clustering<?> clustering, Cell<?> merged, Cell<?> original)
+                public void onCell(int i, Clustering clustering, Cell merged, Cell original)
                 {
                     if (merged != null && !merged.equals(original))
                         toInsert.addCell(merged);
@@ -1576,7 +1636,7 @@ public void commit()
                 indexer.finish();
         }
 
-        private <V1, V2> boolean shouldCleanupOldValue(Cell<V1> oldCell, Cell<V2> newCell)
+        private boolean shouldCleanupOldValue(Cell oldCell, Cell newCell)
         {
             // If either the value or timestamp is different, then we
             // should delete from the index. If not, then we can infer that
@@ -1587,7 +1647,7 @@ private <V1, V2> boolean shouldCleanupOldValue(Cell<V1> oldCell, Cell<V2> newCel
             // Completely identical cells (including expiring columns with
             // identical ttl & localExpirationTime) will not get this far due
             // to the oldCell.equals(newCell) in StandardUpdater.update
-            return !Cells.valueEqual(oldCell, newCell) || oldCell.timestamp() != newCell.timestamp();
+            return !oldCell.value().equals(newCell.value()) || oldCell.timestamp() != newCell.timestamp();
         }
     }
 
@@ -1639,27 +1699,27 @@ public void onRowMerge(Row merged, Row... versions)
             final Row.Builder[] builders = new Row.Builder[versions.length];
             RowDiffListener diffListener = new RowDiffListener()
             {
-                public void onPrimaryKeyLivenessInfo(int i, Clustering<?> clustering, LivenessInfo merged, LivenessInfo original)
+                public void onPrimaryKeyLivenessInfo(int i, Clustering clustering, LivenessInfo merged, LivenessInfo original)
                 {
                     if (original != null && (merged == null || !merged.isLive(nowInSec)))
                         getBuilder(i, clustering).addPrimaryKeyLivenessInfo(original);
                 }
 
-                public void onDeletion(int i, Clustering<?> clustering, Row.Deletion merged, Row.Deletion original)
+                public void onDeletion(int i, Clustering clustering, Row.Deletion merged, Row.Deletion original)
                 {
                 }
 
-                public void onComplexDeletion(int i, Clustering<?> clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original)
+                public void onComplexDeletion(int i, Clustering clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original)
                 {
                 }
 
-                public void onCell(int i, Clustering<?> clustering, Cell<?> merged, Cell<?> original)
+                public void onCell(int i, Clustering clustering, Cell merged, Cell original)
                 {
                     if (original != null && (merged == null || !merged.isLive(nowInSec)))
                         getBuilder(i, clustering).addCell(original);
                 }
 
-                private Row.Builder getBuilder(int index, Clustering<?> clustering)
+                private Row.Builder getBuilder(int index, Clustering clustering)
                 {
                     if (builders[index] == null)
                     {
@@ -1806,21 +1866,45 @@ private void executeAllBlocking(Stream<Index> indexers, Function<Index, Callable
         FBUtilities.waitOnFutures(waitFor);
     }
 
+    @Override
     public void handleNotification(INotification notification, Object sender)
     {
-        if (!indexes.isEmpty() && notification instanceof SSTableAddedNotification)
+        if (indexes.isEmpty())
+            return;
+
+        if (notification instanceof SSTableAddedNotification)
         {
             SSTableAddedNotification notice = (SSTableAddedNotification) notification;
 
-            // SSTables asociated to a memtable come from a flush, so their contents have already been indexed
+            // SSTables associated to a memtable come from a flush, so their contents have already been indexed
             if (notice.memtable().isEmpty())
-                buildIndexesBlocking(Lists.newArrayList(notice.added),
-                                     indexes.values()
-                                            .stream()
-                                            .filter(Index::shouldBuildBlocking)
-                                            .filter(i -> !i.isSSTableAttached())
-                                            .collect(Collectors.toSet()),
-                                     false);
+            {
+                IndexBuildDecider.Decision decision = IndexBuildDecider.instance.onSSTableAdded(notice);
+                build(decision, notice.added, i -> i.shouldBuildBlocking() && !i.isSSTableAttached());
+            }
+        }
+        else if (notification instanceof SSTableListChangedNotification)
+        {
+            SSTableListChangedNotification notice = (SSTableListChangedNotification) notification;
+
+            IndexBuildDecider.Decision decision = IndexBuildDecider.instance.onSSTableListChanged(notice);
+            build(decision, notice.added, Index::shouldBuildBlocking);
+        }
+    }
+
+    private void build(IndexBuildDecider.Decision decision, Iterable<SSTableReader> sstables, Predicate<Index> indexFilter)
+    {
+        if (decision == IndexBuildDecider.Decision.ASYNC)
+        {
+            buildIndexesAsync(Lists.newArrayList(sstables),
+                              indexes.values().stream().filter(indexFilter).collect(Collectors.toSet()),
+                              false);
+        }
+        else if (decision == IndexBuildDecider.Decision.SYNC)
+        {
+            buildIndexesBlocking(Lists.newArrayList(sstables),
+                                 indexes.values().stream().filter(indexFilter).collect(Collectors.toSet()),
+                                 false);
         }
     }
 
@@ -1833,9 +1917,6 @@ public static void shutdownAndWait(long timeout, TimeUnit units) throws Interrup
 
     public void makeIndexNonQueryable(Index index, Index.Status status)
     {
-        if (status == Index.Status.BUILD_SUCCEEDED)
-            throw new IllegalStateException("Index cannot be marked non-queryable with status " + status);
-
         String name = index.getIndexMetadata().name;
         if (indexes.get(name) == index)
         {
@@ -1847,9 +1928,6 @@ public void makeIndexNonQueryable(Index index, Index.Status status)
 
     public void makeIndexQueryable(Index index, Index.Status status)
     {
-        if (status != Index.Status.BUILD_SUCCEEDED)
-            throw new IllegalStateException("Index cannot be marked queryable with status " + status);
-
         String name = index.getIndexMetadata().name;
         if (indexes.get(name) == index)
         {
diff --git a/src/java/org/apache/cassandra/index/SingletonIndexGroup.java b/src/java/org/apache/cassandra/index/SingletonIndexGroup.java
index 162247fd743e..254ad6829674 100644
--- a/src/java/org/apache/cassandra/index/SingletonIndexGroup.java
+++ b/src/java/org/apache/cassandra/index/SingletonIndexGroup.java
@@ -25,16 +25,21 @@
 import java.util.Set;
 import java.util.function.Predicate;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.WriteContext;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.TableMetadata;
 
 /**
@@ -42,13 +47,11 @@
  */
 public class SingletonIndexGroup implements Index.Group
 {
-    private final Index delegate;
-    private final Set<Index> indexes;
+    private volatile Index delegate;
+    private final Set<Index> indexes = Sets.newConcurrentHashSet();
 
-    protected SingletonIndexGroup(Index delegate)
+    protected SingletonIndexGroup()
     {
-        this.delegate = delegate;
-        this.indexes = Collections.singleton(delegate);
     }
 
     @Override
@@ -62,6 +65,26 @@ public Index getIndex()
         return delegate;
     }
 
+    @Override
+    public void addIndex(Index index)
+    {
+        Preconditions.checkState(delegate == null);
+        // This class does not work for SAI because the `componentsForNewBuid` method would be incorrect (so more
+        // generally, it does not work for indexes that use dedicated sstable components, which is only SAI). See
+        // comments on `componentsForNewBuid` for more details.
+        Preconditions.checkState(!(index instanceof StorageAttachedIndex), "This shoudl not be used with SAI");
+        delegate = index;
+        indexes.add(index);
+    }
+
+    @Override
+    public void removeIndex(Index index)
+    {
+        Preconditions.checkState(containsIndex(index));
+        delegate = null;
+        indexes.clear();
+    }
+
     @Override
     public boolean containsIndex(Index index)
     {
@@ -77,25 +100,53 @@ public Index.Indexer indexerFor(Predicate<Index> indexSelector,
                                     IndexTransaction.Type transactionType,
                                     Memtable memtable)
     {
-        return indexSelector.test(delegate) ? delegate.indexerFor(key, columns, nowInSec, ctx, transactionType, memtable)
-                                            : null;
+        Preconditions.checkNotNull(delegate);
+        return indexSelector.test(delegate)
+               ? delegate.indexerFor(key, columns, nowInSec, ctx, transactionType, memtable)
+               : null;
     }
 
     @Override
     public Index.QueryPlan queryPlanFor(RowFilter rowFilter)
     {
-        return SingletonIndexQueryPlan.create(delegate, rowFilter);
+        Preconditions.checkNotNull(delegate);
+
+        // Indexes using a singleton group don't support disjunctions,
+        // so we only consider the top-level AND expressions for index selection.
+        for (RowFilter.Expression e : rowFilter.withoutDisjunctions().expressions())
+        {
+            if (delegate.supportsExpression(e.column(), e.operator()))
+                return new SingletonIndexQueryPlan(delegate, delegate.getPostIndexQueryFilter(rowFilter));
+        }
+
+        return null;
     }
 
     @Override
-    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata, long keyCount)
     {
+        Preconditions.checkNotNull(delegate);
         return delegate.getFlushObserver(descriptor, tracker);
     }
 
     @Override
-    public Set<Component> getComponents()
+    public Set<Component> componentsForNewSSTable()
+    {
+        // This class is only used for indexes that don't use per-sstable components, aka not-SAI (note that SASI uses
+        // some "file" per sstable, but it is not a `Component` in practice). We could add an equivalent
+        // `componentsForNewSSTable` method in `Index`, so that we can call `delegate.componentsForNewBuild` here, but
+        // this would kind of weird for SAI because of the per-sstable components: should they be returned by such
+        // method on `Index` or not? Tldr, for SAI, it's cleaner to deal with components created at the group level,
+        // which is what `StorageAttachedIndexGroup.componentsForNewBuild` does, and it's simpler to always use
+        // `StorageAttachedIndexGroup` for SAI, which is the case. So at that point, adding an
+        // `Index.componentsForNewSSTable` method would just be dead code, so let's avoid it.
+        return Collections.emptySet();
+    }
+
+    @Override
+    public Set<Component> activeComponents(SSTableReader sstable)
     {
-        return delegate.getComponents();
+        // Same rermarks as for `componentsForNewBuid`.
+        return Collections.emptySet();
     }
 }
diff --git a/src/java/org/apache/cassandra/index/SingletonIndexQueryPlan.java b/src/java/org/apache/cassandra/index/SingletonIndexQueryPlan.java
index b475cee145a3..f4d206b0e358 100644
--- a/src/java/org/apache/cassandra/index/SingletonIndexQueryPlan.java
+++ b/src/java/org/apache/cassandra/index/SingletonIndexQueryPlan.java
@@ -24,7 +24,6 @@
 import java.util.Collections;
 import java.util.Set;
 import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.filter.RowFilter;
@@ -42,18 +41,6 @@ protected SingletonIndexQueryPlan(Index index, RowFilter postIndexFilter)
         this.postIndexFilter = postIndexFilter;
     }
 
-    @Nullable
-    protected static SingletonIndexQueryPlan create(Index index, RowFilter rowFilter)
-    {
-        for (RowFilter.Expression e : rowFilter.getExpressions())
-        {
-            if (index.supportsExpression(e.column(), e.operator()))
-                return new SingletonIndexQueryPlan(index, index.getPostIndexQueryFilter(rowFilter));
-        }
-
-        return null;
-    }
-
     @Override
     public Set<Index> getIndexes()
     {
diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
index 20c1a0532864..c0349058a136 100644
--- a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
@@ -33,6 +33,8 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.TableOperation;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.schema.ColumnMetadata;
@@ -45,7 +47,6 @@
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.CollectionType;
-import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.dht.LocalPartitioner;
@@ -58,6 +59,10 @@
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.Type;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Refs;
@@ -80,10 +85,15 @@ public abstract class CassandraIndex implements Index
     protected ColumnMetadata indexedColumn;
     protected CassandraIndexFunctions functions;
 
+    private final RequestTracker requestTracker;
+    private final Context sensorContext;
+
     protected CassandraIndex(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
     {
         this.baseCfs = baseCfs;
         setMetadata(indexDef);
+        this.requestTracker = RequestTracker.instance;
+        this.sensorContext = Context.from(baseCfs.metadata());
     }
 
     /**
@@ -107,9 +117,9 @@ protected boolean supportsOperator(ColumnMetadata indexedColumn, Operator operat
      * @param path from the base data being indexed
      * @return a clustering prefix to be used to insert into the index table
      */
-    protected abstract <T> CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
-                                                           ClusteringPrefix<T> prefix,
-                                                           CellPath path);
+    protected abstract <T> ClusteringBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
+                                                                        ClusteringPrefix<T> prefix,
+                                                                        CellPath path);
 
     /**
      * Used at search time to convert a row in the index table into a simple struct containing the values required
@@ -217,7 +227,7 @@ public boolean isQueryable(Status status)
     @Override
     public void validate(ReadCommand command) throws InvalidRequestException
     {
-        Optional<RowFilter.Expression> target = getTargetExpression(command.rowFilter().getExpressions());
+        Optional<RowFilter.Expression> target = getTargetExpression(command.rowFilter());
 
         if (target.isPresent())
         {
@@ -277,23 +287,29 @@ public AbstractType<?> customExpressionValueType()
 
     public long getEstimatedResultRows()
     {
-        return indexCfs.getMeanRowCount();
+        return indexCfs.getMeanRowsPerPartition();
     }
 
     public RowFilter getPostIndexQueryFilter(RowFilter filter)
     {
-        return getTargetExpression(filter.getExpressions()).map(filter::without)
-                                                           .orElse(filter);
+        // This index doesn't support disjunctions, so if the query has any, we simply apply the entire filter.
+        return filter.containsDisjunctions() ? filter : getTargetExpression(filter).map(filter::without).orElse(filter);
     }
 
-    private Optional<RowFilter.Expression> getTargetExpression(List<RowFilter.Expression> expressions)
+    private Optional<RowFilter.Expression> getTargetExpression(RowFilter rowFilter)
     {
-        return expressions.stream().filter(this::supportsExpression).findFirst();
+        // This index doesn't support disjunctions, so we only consider the top-level AND expressions.
+        for (RowFilter.Expression expression : rowFilter.withoutDisjunctions().expressions())
+        {
+            if (supportsExpression(expression))
+                return Optional.of(expression);
+        }
+        return Optional.empty();
     }
 
     public Index.Searcher searcherFor(ReadCommand command)
     {
-        Optional<RowFilter.Expression> target = getTargetExpression(command.rowFilter().getExpressions());
+        Optional<RowFilter.Expression> target = getTargetExpression(command.rowFilter());
 
         if (target.isPresent())
         {
@@ -329,7 +345,7 @@ public void validate(PartitionUpdate update, ClientState state) throws InvalidRe
                 break;
             case REGULAR:
                 if (update.columns().regulars.contains(indexedColumn))
-                    validateRows(update);
+                    validateRows(update.rows());
                 break;
             case STATIC:
                 if (update.columns().statics.contains(indexedColumn))
@@ -343,7 +359,7 @@ public Indexer indexerFor(final DecoratedKey key,
                               final long nowInSec,
                               final WriteContext ctx,
                               final IndexTransaction.Type transactionType,
-                              Memtable memtable)
+                              final Memtable memtable)
     {
         /*
          * Indexes on regular and static columns (the non primary-key ones) only care about updates with live
@@ -448,6 +464,14 @@ private void indexCell(Clustering<?> clustering, Cell<?> cell)
                        cell,
                        LivenessInfo.withExpirationTime(cell.timestamp(), cell.ttl(), cell.localDeletionTime()),
                        ctx);
+
+                RequestSensors sensors = requestTracker.get();
+                if (sensors != null)
+                {
+                    sensors.registerSensor(sensorContext, Type.INDEX_WRITE_BYTES);
+                    // estimate the size of the index entry as the data size of the cell before indexing
+                    sensors.incrementSensor(sensorContext, Type.INDEX_WRITE_BYTES, cell.dataSize());
+                }
             }
 
             private void removeCells(Clustering<?> clustering, Iterable<Cell<?>> cells)
@@ -586,7 +610,7 @@ private void validatePartitionKey(DecoratedKey partitionKey) throws InvalidReque
     private void validateClusterings(PartitionUpdate update) throws InvalidRequestException
     {
         assert indexedColumn.isClusteringColumn();
-        for (Row row : update)
+        for (Row row : update.rows())
             validateIndexedValue(getIndexedValue(null, row.clustering(), null));
     }
 
@@ -659,7 +683,7 @@ private void invalidate()
     {
         // interrupt in-progress compactions
         Collection<ColumnFamilyStore> cfss = Collections.singleton(indexCfs);
-        CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true);
+        CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true, TableOperation.StopTrigger.INVALIDATE_INDEX);
         CompactionManager.instance.waitForCessation(cfss, (sstable) -> true);
         Keyspace.writeOrder.awaitNewBarrier();
         indexCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_REMOVED);
@@ -744,7 +768,7 @@ public static TableMetadata indexCfsMetadata(TableMetadata baseCfsMetadata, Inde
             TableMetadata.builder(baseCfsMetadata.keyspace, baseCfsMetadata.indexTableName(indexMetadata), baseCfsMetadata.id)
                          .kind(TableMetadata.Kind.INDEX)
                          .partitioner(new LocalPartitioner(indexedValueType))
-                         .addPartitionKeyColumn(indexedColumn.name, isCompatible ? indexedColumn.type : utils.getIndexedPartitionKeyType(indexedColumn))
+                         .addPartitionKeyColumn(indexedColumn.name, isCompatible ? indexedValueType : utils.getIndexedValueType(indexedColumn))
                          .addClusteringColumn("partition_key", isCompatible ? baseCfsMetadata.partitioner.partitionOrdering() : indexedTablePartitionKeyType);
 
         // Adding clustering columns, which depends on the index type.
diff --git a/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java b/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java
index 07bdc420ca07..31a1f55137b6 100644
--- a/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java
@@ -20,15 +20,15 @@
 import java.util.Collection;
 import java.util.Set;
 
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.RegularAndStaticColumns;
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.SecondaryIndexBuilder;
 import org.apache.cassandra.io.sstable.ReducingKeyIterator;
+import org.apache.cassandra.io.sstable.SSTableWatcher;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.utils.TimeUUID;
@@ -55,27 +55,29 @@ public CollatedViewIndexBuilder(ColumnFamilyStore cfs, Set<Index> indexers, Redu
         this.sstables = sstables;
     }
 
-    public CompactionInfo getCompactionInfo()
+    public OperationProgress getProgress()
     {
-        return new CompactionInfo(cfs.metadata(),
-                                  OperationType.INDEX_BUILD,
-                                  iter.getBytesRead(),
-                                  iter.getTotalBytes(),
-                                  compactionId,
-                                  sstables);
+        return new OperationProgress(cfs.metadata(),
+                                     OperationType.INDEX_BUILD,
+                                     iter.getBytesRead(),
+                                     iter.getTotalBytes(),
+                                     compactionId,
+                                     sstables);
     }
 
     public void build()
     {
         try
         {
-            int pageSize = cfs.indexManager.calculateIndexingPageSize();
+            for (SSTableReader sstable : sstables)
+                SSTableWatcher.instance.onIndexBuild(sstable, indexers);
+
+            PageSize pageSize = cfs.indexManager.calculateIndexingPageSize();
             RegularAndStaticColumns targetPartitionColumns = extractIndexedColumns();
-            
+
             while (iter.hasNext())
             {
-                if (isStopRequested())
-                    throw new CompactionInterruptedException(getCompactionInfo());
+                throwIfStopRequested();
                 DecoratedKey key = iter.next();
                 cfs.indexManager.indexPartition(key, indexers, pageSize, targetPartitionColumns);
             }
@@ -89,11 +91,11 @@ public void build()
     private RegularAndStaticColumns extractIndexedColumns()
     {
         RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder();
-        
+
         for (Index index : indexers)
         {
             boolean isPartitionIndex = true;
-            
+
             for (ColumnMetadata column : cfs.metadata().regularAndStaticColumns())
             {
                 if (index.dependsOn(column))
@@ -108,7 +110,7 @@ private RegularAndStaticColumns extractIndexedColumns()
             if (isPartitionIndex)
                 return cfs.metadata().regularAndStaticColumns();
         }
-        
+
         return builder.build();
     }
 }
diff --git a/src/java/org/apache/cassandra/index/internal/composites/ClusteringColumnIndex.java b/src/java/org/apache/cassandra/index/internal/composites/ClusteringColumnIndex.java
index 23cff3f8468c..fc15de8e45cd 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/ClusteringColumnIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/ClusteringColumnIndex.java
@@ -63,11 +63,11 @@ public ByteBuffer getIndexedValue(ByteBuffer partitionKey,
         return clustering.bufferAt(indexedColumn.position());
     }
 
-    public <T> CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
-                                                   ClusteringPrefix<T> prefix,
-                                                   CellPath path)
+    public <T> ClusteringBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
+                                                            ClusteringPrefix<T> prefix,
+                                                            CellPath path)
     {
-        CBuilder builder = CBuilder.create(getIndexComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(getIndexComparator());
         builder.add(partitionKey);
         for (int i = 0; i < Math.min(indexedColumn.position(), prefix.size()); i++)
             builder.add(prefix.get(i), prefix.accessor());
@@ -82,7 +82,7 @@ public IndexEntry decodeEntry(DecoratedKey indexedValue,
         int ckCount = baseCfs.metadata().clusteringColumns().size();
 
         Clustering<?> clustering = indexEntry.clustering();
-        CBuilder builder = CBuilder.create(baseCfs.getComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(baseCfs.getComparator());
         for (int i = 0; i < indexedColumn.position(); i++)
             builder.add(clustering, i + 1);
 
diff --git a/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndexBase.java b/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndexBase.java
index f0201e1effec..43196463e7b4 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndexBase.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/CollectionKeyIndexBase.java
@@ -48,11 +48,11 @@ public CollectionKeyIndexBase(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
         super(baseCfs, indexDef);
     }
 
-    public <T> CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
-                                                   ClusteringPrefix<T> prefix,
-                                                   CellPath path)
+    public <T> ClusteringBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
+                                                            ClusteringPrefix<T> prefix,
+                                                            CellPath path)
     {
-        CBuilder builder = CBuilder.create(getIndexComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(getIndexComparator());
         builder.add(partitionKey);
 
         // When indexing a static column, prefix will be empty but only the
@@ -74,7 +74,7 @@ public IndexEntry decodeEntry(DecoratedKey indexedValue,
         else
         {
             int count = 1 + baseCfs.metadata().clusteringColumns().size();
-            CBuilder builder = CBuilder.create(baseCfs.getComparator());
+            ClusteringBuilder builder = ClusteringBuilder.create(baseCfs.getComparator());
             for (int i = 0; i < count - 1; i++)
                 builder.add(clustering, i + 1);
             indexedEntryClustering = builder.build();
diff --git a/src/java/org/apache/cassandra/index/internal/composites/CollectionValueIndex.java b/src/java/org/apache/cassandra/index/internal/composites/CollectionValueIndex.java
index ed929f22b3c5..6707f664a592 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/CollectionValueIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/CollectionValueIndex.java
@@ -54,11 +54,11 @@ public ByteBuffer getIndexedValue(ByteBuffer partitionKey,
         return cellValue;
     }
 
-    public <T> CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
-                                                   ClusteringPrefix<T> prefix,
-                                                   CellPath path)
+    public <T> ClusteringBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
+                                                            ClusteringPrefix<T> prefix,
+                                                            CellPath path)
     {
-        CBuilder builder = CBuilder.create(getIndexComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(getIndexComparator());
         builder.add(partitionKey);
         for (int i = 0; i < prefix.size(); i++)
             builder.add(prefix.get(i), prefix.accessor());
@@ -81,7 +81,7 @@ public IndexEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry)
             indexedEntryClustering = Clustering.STATIC_CLUSTERING;
         else
         {
-            CBuilder builder = CBuilder.create(baseCfs.getComparator());
+            ClusteringBuilder builder = ClusteringBuilder.create(baseCfs.getComparator());
             for (int i = 0; i < baseCfs.getComparator().size(); i++)
                 builder.add(clustering, i + 1);
             indexedEntryClustering = builder.build();
diff --git a/src/java/org/apache/cassandra/index/internal/composites/PartitionKeyIndex.java b/src/java/org/apache/cassandra/index/internal/composites/PartitionKeyIndex.java
index b8235370b70c..adf8e483cf3b 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/PartitionKeyIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/PartitionKeyIndex.java
@@ -64,11 +64,11 @@ public ByteBuffer getIndexedValue(ByteBuffer partitionKey,
         return components[indexedColumn.position()];
     }
 
-    public <T> CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
-                                                   ClusteringPrefix<T> prefix,
-                                                   CellPath path)
+    public <T> ClusteringBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
+                                                            ClusteringPrefix<T> prefix,
+                                                            CellPath path)
     {
-        CBuilder builder = CBuilder.create(getIndexComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(getIndexComparator());
         builder.add(partitionKey);
         for (int i = 0; i < prefix.size(); i++)
             builder.add(prefix.get(i), prefix.accessor());
@@ -79,7 +79,7 @@ public IndexEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry)
     {
         int ckCount = baseCfs.metadata().clusteringColumns().size();
         Clustering<?> clustering = indexEntry.clustering();
-        CBuilder builder = CBuilder.create(baseCfs.getComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(baseCfs.getComparator());
         for (int i = 0; i < ckCount; i++)
             builder.add(clustering, i + 1);
 
diff --git a/src/java/org/apache/cassandra/index/internal/composites/RegularColumnIndex.java b/src/java/org/apache/cassandra/index/internal/composites/RegularColumnIndex.java
index 9dcb8dfcc4d9..9fd8596454db 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/RegularColumnIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/RegularColumnIndex.java
@@ -61,11 +61,11 @@ public ByteBuffer getIndexedValue(ByteBuffer partitionKey,
         return cellValue;
     }
 
-    public <T> CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
-                                               ClusteringPrefix<T> prefix,
-                                               CellPath path)
+    public <T> ClusteringBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
+                                                            ClusteringPrefix<T> prefix,
+                                                            CellPath path)
     {
-        CBuilder builder = CBuilder.create(getIndexComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(getIndexComparator());
         builder.add(partitionKey);
         for (int i = 0; i < prefix.size(); i++)
             builder.add(prefix.get(i), prefix.accessor());
@@ -87,7 +87,7 @@ public IndexEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry)
         else
         {
             ClusteringComparator baseComparator = baseCfs.getComparator();
-            CBuilder builder = CBuilder.create(baseComparator);
+            ClusteringBuilder builder = ClusteringBuilder.create(baseComparator);
             for (int i = 0; i < baseComparator.size(); i++)
                 builder.add(clustering, i + 1);
             indexedEntryClustering = builder.build();
diff --git a/src/java/org/apache/cassandra/index/internal/keys/KeysIndex.java b/src/java/org/apache/cassandra/index/internal/keys/KeysIndex.java
index 695fb67ef94f..bd89070e8789 100644
--- a/src/java/org/apache/cassandra/index/internal/keys/KeysIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/keys/KeysIndex.java
@@ -49,11 +49,11 @@ public TableMetadata.Builder addIndexClusteringColumns(TableMetadata.Builder bui
         return builder;
     }
 
-    protected <T> CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
-                                                      ClusteringPrefix<T> prefix,
-                                                      CellPath path)
+    protected <T> ClusteringBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
+                                                               ClusteringPrefix<T> prefix,
+                                                               CellPath path)
     {
-        CBuilder builder = CBuilder.create(getIndexComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(getIndexComparator());
         builder.add(partitionKey, ByteBufferAccessor.instance);
         return builder;
     }
diff --git a/src/java/org/apache/cassandra/index/sai/IndexContext.java b/src/java/org/apache/cassandra/index/sai/IndexContext.java
new file mode 100644
index 000000000000..171a71118676
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/IndexContext.java
@@ -0,0 +1,1016 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableSet;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
+import org.apache.cassandra.index.sai.disk.vector.VectorValidation;
+import org.apache.cassandra.index.sai.iterators.KeyRangeAntiJoinIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.memory.MemtableKeyRangeIterator;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+import org.apache.cassandra.index.sai.metrics.IndexMetrics;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.index.sai.view.IndexViewManager;
+import org.apache.cassandra.index.sai.view.View;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_ANALYZED_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_FROZEN_TERM_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_STRING_TERM_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_VECTOR_TERM_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_VALIDATE_MAX_TERM_SIZE_AT_COORDINATOR;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+/**
+ * Manage metadata for each column index.
+ */
+public class IndexContext
+{
+    private static final Logger logger = LoggerFactory.getLogger(IndexContext.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+
+    public static final int MAX_STRING_TERM_SIZE = SAI_MAX_STRING_TERM_SIZE.getInt() * 1024;
+    public static final int MAX_FROZEN_TERM_SIZE = SAI_MAX_FROZEN_TERM_SIZE.getInt() * 1024;
+    public static final int MAX_VECTOR_TERM_SIZE = SAI_MAX_VECTOR_TERM_SIZE.getInt() * 1024;
+    public static final int MAX_ANALYZED_SIZE = SAI_MAX_ANALYZED_SIZE.getInt() * 1024;
+    private static final String TERM_OVERSIZE_LOG_MESSAGE =
+    "Can't add term of column {} to index for key: {}, term size {} max allowed size {}.";
+    private static final String TERM_OVERSIZE_ERROR_MESSAGE =
+    "Term of column %s exceeds the byte limit for index. Term size %s. Max allowed size %s.";
+
+    private static final String ANALYZED_TERM_OVERSIZE_LOG_MESSAGE =
+    "Term's analyzed size for column {} exceeds the cumulative limit for index. Max allowed size {}.";
+    private static final String ANALYZED_TERM_OVERSIZE_ERROR_MESSAGE =
+    "Term's analyzed size for column %s exceeds the cumulative limit for index. Max allowed size %s.";
+
+    private static final Set<AbstractType<?>> EQ_ONLY_TYPES =
+            ImmutableSet.of(UTF8Type.instance, AsciiType.instance, BooleanType.instance, UUIDType.instance);
+
+    public static final String ENABLE_SEGMENT_COMPACTION_OPTION_NAME = "enable_segment_compaction";
+
+    private final AbstractType<?> partitionKeyType;
+    private final ClusteringComparator clusteringComparator;
+
+    private final String keyspace;
+    private final String table;
+    private final TableId tableId;
+    private final ColumnMetadata column;
+    private final IndexTarget.Type indexType;
+    private final AbstractType<?> validator;
+    private final ColumnFamilyStore cfs;
+
+    // Config can be null if the column context is "fake" (i.e. created for a filtering expression).
+    private final IndexMetadata config;
+    private final VectorSimilarityFunction vectorSimilarityFunction;
+
+    private final ConcurrentMap<Memtable, MemtableIndex> liveMemtables = new ConcurrentHashMap<>();
+
+    private final IndexViewManager viewManager;
+    private final IndexMetrics indexMetrics;
+    private final ColumnQueryMetrics columnQueryMetrics;
+    private final IndexWriterConfig indexWriterConfig;
+    private final boolean isAnalyzed;
+    private final boolean hasEuclideanSimilarityFunc;
+    private final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
+    private final AbstractAnalyzer.AnalyzerFactory queryAnalyzerFactory;
+    private final PrimaryKey.Factory primaryKeyFactory;
+
+    private final int maxTermSize;
+
+    private volatile boolean dropped = false;
+
+    public IndexContext(@Nonnull String keyspace,
+                        @Nonnull String table,
+                        @Nonnull TableId tableId,
+                        @Nonnull AbstractType<?> partitionKeyType,
+                        @Nonnull ClusteringComparator clusteringComparator,
+                        @Nonnull ColumnMetadata column,
+                        @Nonnull IndexTarget.Type indexType,
+                        IndexMetadata config,
+                        @Nonnull ColumnFamilyStore cfs)
+    {
+        this.keyspace = keyspace;
+        this.table = table;
+        this.tableId = tableId;
+        this.partitionKeyType = partitionKeyType;
+        this.clusteringComparator = clusteringComparator;
+        this.column = column;
+        this.indexType = indexType;
+        this.config = config;
+        this.viewManager = new IndexViewManager(this);
+        this.validator = TypeUtil.cellValueType(column, indexType);
+        this.cfs = cfs;
+        this.primaryKeyFactory = Version.latest().onDiskFormat().newPrimaryKeyFactory(clusteringComparator);
+
+        String columnName = column.name.toString();
+
+        if (config != null)
+        {
+            String fullIndexName = String.format("%s.%s.%s", this.keyspace, this.table, this.config.name);
+            this.indexWriterConfig = IndexWriterConfig.fromOptions(fullIndexName, validator, config.options);
+            this.isAnalyzed = AbstractAnalyzer.isAnalyzed(config.options);
+            this.analyzerFactory = AbstractAnalyzer.fromOptions(columnName, validator, config.options);
+            this.queryAnalyzerFactory = AbstractAnalyzer.hasQueryAnalyzer(config.options)
+                                        ? AbstractAnalyzer.fromOptionsQueryAnalyzer(validator, config.options)
+                                        : this.analyzerFactory;
+            this.vectorSimilarityFunction = indexWriterConfig.getSimilarityFunction();
+            this.hasEuclideanSimilarityFunc = vectorSimilarityFunction == VectorSimilarityFunction.EUCLIDEAN;
+
+            this.indexMetrics = new IndexMetrics(this);
+            this.columnQueryMetrics = isVector() ? new ColumnQueryMetrics.VectorIndexMetrics(keyspace, table, getIndexName()) :
+                                      isLiteral() ? new ColumnQueryMetrics.TrieIndexMetrics(keyspace, table, getIndexName())
+                                                  : new ColumnQueryMetrics.BKDIndexMetrics(keyspace, table, getIndexName());
+
+        }
+        else
+        {
+            this.indexWriterConfig = IndexWriterConfig.emptyConfig();
+            this.isAnalyzed = AbstractAnalyzer.isAnalyzed(Collections.emptyMap());
+            this.analyzerFactory = AbstractAnalyzer.fromOptions(columnName, validator, Collections.EMPTY_MAP);
+            this.queryAnalyzerFactory = this.analyzerFactory;
+            this.vectorSimilarityFunction = null;
+            this.hasEuclideanSimilarityFunc = false;
+
+            // null config indicates a "fake" index context. As such, it won't actually be used for indexing/accessing
+            // data, leaving these metrics unused. This also eliminates the overhead of creating these metrics on the
+            // query path.
+            this.indexMetrics = null;
+            this.columnQueryMetrics = null;
+        }
+
+        this.maxTermSize = isVector() ? MAX_VECTOR_TERM_SIZE
+                                      : isAnalyzed ? MAX_ANALYZED_SIZE
+                                                   : isFrozen() ? MAX_FROZEN_TERM_SIZE : MAX_STRING_TERM_SIZE;
+
+
+        logger.debug(logMessage("Initialized index context with index writer config: {}"), indexWriterConfig);
+    }
+
+    public AbstractType<?> keyValidator()
+    {
+        return partitionKeyType;
+    }
+
+    public PrimaryKey.Factory keyFactory()
+    {
+        return primaryKeyFactory;
+    }
+
+    public ClusteringComparator comparator()
+    {
+        return clusteringComparator;
+    }
+
+    public IndexMetrics getIndexMetrics()
+    {
+        return indexMetrics;
+    }
+
+    public ColumnQueryMetrics getColumnQueryMetrics()
+    {
+        return columnQueryMetrics;
+    }
+
+    public String getKeyspace()
+    {
+        return keyspace;
+    }
+    public String getTable()
+    {
+        return table;
+    }
+
+    public TableId getTableId()
+    {
+        return tableId;
+    }
+
+    public ColumnFamilyStore columnFamilyStore()
+    {
+        return cfs;
+    }
+
+    public IPartitioner getPartitioner()
+    {
+        return cfs.getPartitioner();
+    }
+
+    public void index(DecoratedKey key, Row row, Memtable mt, OpOrder.Group opGroup)
+    {
+        MemtableIndex target = liveMemtables.computeIfAbsent(mt, memtable -> MemtableIndex.createIndex(this, memtable));
+
+        long start = nanoTime();
+
+        if (isNonFrozenCollection())
+        {
+            Iterator<ByteBuffer> bufferIterator = getValuesOf(row, FBUtilities.nowInSeconds());
+            if (bufferIterator != null)
+            {
+                while (bufferIterator.hasNext())
+                {
+                    ByteBuffer value = bufferIterator.next();
+                    target.index(key, row.clustering(), value, mt, opGroup);
+                }
+            }
+        }
+        else
+        {
+            ByteBuffer value = getValueOf(key, row, FBUtilities.nowInSeconds());
+            target.index(key, row.clustering(), value, mt, opGroup);
+        }
+        indexMetrics.memtableIndexWriteLatency.update(nanoTime() - start, TimeUnit.NANOSECONDS);
+    }
+
+    /**
+     * Validate maximum term size for given row. Throw an exception when invalid.
+     */
+    public void validateMaxTermSizeForRow(DecoratedKey key, Row row)
+    {
+        AbstractAnalyzer analyzer = getAnalyzerFactory().create();
+        if (isNonFrozenCollection())
+        {
+            Iterator<ByteBuffer> bufferIterator = getValuesOf(row, FBUtilities.nowInSeconds());
+            while (bufferIterator != null && bufferIterator.hasNext())
+                validateMaxTermSizeForCell(analyzer, key, bufferIterator.next());
+        }
+        else
+        {
+            ByteBuffer value = getValueOf(key, row, FBUtilities.nowInSeconds());
+            validateMaxTermSizeForCell(analyzer, key, value);
+        }
+    }
+
+    private void validateMaxTermSizeForCell(AbstractAnalyzer analyzer, DecoratedKey key, @Nullable ByteBuffer cellBuffer)
+    {
+        if (cellBuffer == null || cellBuffer.remaining() == 0)
+            return;
+
+        analyzer.reset(cellBuffer);
+        try
+        {
+            if (analyzer.transformValue())
+            {
+                if (!validateCumulativeAnalyzedTermLimit(key, analyzer))
+                {
+                    var error = String.format(ANALYZED_TERM_OVERSIZE_ERROR_MESSAGE,
+                                              column.name, FBUtilities.prettyPrintMemory(maxTermSize));
+                    throw new InvalidRequestException(error);
+                }
+            }
+            else
+            {
+                while (analyzer.hasNext())
+                {
+                    var size = analyzer.next().remaining();
+                    if (!validateMaxTermSize(key, size))
+                    {
+                        var error = String.format(TERM_OVERSIZE_ERROR_MESSAGE,
+                                                  column.name,
+                                                  FBUtilities.prettyPrintMemory(size),
+                                                  FBUtilities.prettyPrintMemory(maxTermSize));
+                        throw new InvalidRequestException(error);
+                    }
+                }
+            }
+        }
+        finally
+        {
+            analyzer.end();
+        }
+    }
+
+
+    /**
+     * Validate maximum term size for given term
+     * @return true if given term is valid; otherwise false.
+     */
+    public boolean validateMaxTermSize(DecoratedKey key, ByteBuffer term)
+    {
+        return validateMaxTermSize(key, term.remaining());
+    }
+
+    private boolean validateMaxTermSize(DecoratedKey key, int termSize)
+    {
+        if (termSize > maxTermSize)
+        {
+            noSpamLogger.warn(logMessage(TERM_OVERSIZE_LOG_MESSAGE),
+                              getColumnName(),
+                              keyValidator().getString(key.getKey()),
+                              FBUtilities.prettyPrintMemory(termSize),
+                              FBUtilities.prettyPrintMemory(maxTermSize));
+            return false;
+        }
+
+        return true;
+    }
+
+    private boolean validateCumulativeAnalyzedTermLimit(DecoratedKey key, AbstractAnalyzer analyzer)
+    {
+        int bytesCount = 0;
+        // VSTODO anayzer.hasNext copies the byteBuffer, but we don't need that here.
+        while (analyzer.hasNext())
+        {
+            final ByteBuffer token = analyzer.next();
+            bytesCount += token.remaining();
+            if (bytesCount > maxTermSize)
+            {
+                noSpamLogger.warn(logMessage(ANALYZED_TERM_OVERSIZE_LOG_MESSAGE),
+                                  getColumnName(),
+                                  keyValidator().getString(key.getKey()),
+                                  FBUtilities.prettyPrintMemory(maxTermSize));
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public void update(DecoratedKey key, Row oldRow, Row newRow, Memtable memtable, OpOrder.Group opGroup)
+    {
+        MemtableIndex target = liveMemtables.get(memtable);
+        if (target == null)
+            return;
+
+        // Use 0 for nowInSecs to get the value(s) from the oldRow regardless of its liveness status. To get to this point,
+        // C* has already determined this is the current represntation of the oldRow in the memtable, and that means
+        // we need to add the newValue to the index and remove the oldValue from it, even if it has already expired via
+        // TTL.
+        if (isNonFrozenCollection())
+        {
+            Iterator<ByteBuffer> oldValues = getValuesOf(oldRow, 0);
+            Iterator<ByteBuffer> newValues = getValuesOf(newRow, FBUtilities.nowInSeconds());
+            target.update(key, oldRow.clustering(), oldValues, newValues, memtable, opGroup);
+        }
+        else
+        {
+            ByteBuffer oldValue = getValueOf(key, oldRow, 0);
+            ByteBuffer newValue = getValueOf(key, newRow, FBUtilities.nowInSeconds());
+            target.update(key, oldRow.clustering(), oldValue, newValue, memtable, opGroup);
+        }
+    }
+
+    public void renewMemtable(Memtable renewed)
+    {
+        // remove every index but the one that corresponds to the post-truncate Memtable
+        liveMemtables.keySet().removeIf(m -> m != renewed);
+    }
+
+    public void discardMemtable(Memtable discarded)
+    {
+        liveMemtables.remove(discarded);
+    }
+
+    public MemtableIndex getPendingMemtableIndex(LifecycleNewTracker tracker)
+    {
+        return liveMemtables.keySet().stream()
+                            .filter(m -> tracker.equals(m.getFlushTransaction()))
+                            .findFirst()
+                            .map(liveMemtables::get)
+                            .orElse(null);
+    }
+
+    // Returns an iterator for NEQ, NOT_CONTAINS_KEY, NOT_CONTAINS_VALUE, which
+    // 1. Either includes everything if the column type values can be truncated and
+    // thus the keys cannot be matched precisely,
+    // 2. or includes everything minus the keys matching the expression
+    // if the column type values cannot be truncated, i.e., matching the keys is always precise.
+    // (not matching precisely will lead to false negatives)
+    //
+    // keys k such that row(k) not contains v = (all keys) \ (keys k such that row(k) contains v)
+    //
+    // Note that rows in other indexes are not matched, so this can return false positives,
+    // but they are not a problem as post-filtering would get rid of them.
+    // The keys matched in other indexes cannot be safely subtracted
+    // as indexes may contain false positives caused by deletes and updates.
+    private KeyRangeIterator getNonEqIterator(QueryContext context, Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        KeyRangeIterator allKeys = scanMemtable(keyRange);
+        if (TypeUtil.supportsRounding(expression.validator))
+        {
+            return allKeys;
+        }
+        else
+        {
+            Expression negExpression = expression.negated();
+            KeyRangeIterator matchedKeys = searchMemtable(context, negExpression, keyRange, Integer.MAX_VALUE);
+            return KeyRangeAntiJoinIterator.create(allKeys, matchedKeys);
+        }
+    }
+
+    public KeyRangeIterator searchMemtable(QueryContext context, Expression expression, AbstractBounds<PartitionPosition> keyRange, int limit)
+    {
+        if (expression.getOp().isNonEquality())
+        {
+            return getNonEqIterator(context, expression, keyRange);
+        }
+
+        Collection<MemtableIndex> memtables = liveMemtables.values();
+
+        if (memtables.isEmpty())
+        {
+            return KeyRangeIterator.empty();
+        }
+
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
+
+        try
+        {
+            for (MemtableIndex index : memtables)
+            {
+                builder.add(index.search(context, expression, keyRange, limit));
+            }
+
+            return builder.build();
+        }
+        catch (Exception ex)
+        {
+            FileUtils.closeQuietly(builder.ranges());
+            throw ex;
+        }
+    }
+
+    private KeyRangeIterator scanMemtable(AbstractBounds<PartitionPosition> keyRange)
+    {
+        Collection<Memtable> memtables = liveMemtables.keySet();
+        if (memtables.isEmpty())
+        {
+            return KeyRangeIterator.empty();
+        }
+
+        KeyRangeIterator.Builder builder = KeyRangeUnionIterator.builder(memtables.size());
+
+        try
+        {
+            for (Memtable memtable : memtables)
+            {
+                KeyRangeIterator memtableIterator = new MemtableKeyRangeIterator(memtable, primaryKeyFactory, keyRange);
+                builder.add(memtableIterator);
+            }
+
+            return builder.build();
+        }
+        catch (Exception ex)
+        {
+            FileUtils.closeQuietly(builder.ranges());
+            throw ex;
+        }
+    }
+
+    // Search all memtables for all PrimaryKeys in list.
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderResultsBy(QueryContext context, List<PrimaryKey> source, Orderer orderer, int limit)
+    {
+        Collection<MemtableIndex> memtables = liveMemtables.values();
+
+        if (memtables.isEmpty())
+            return List.of();
+
+        List<CloseableIterator<PrimaryKeyWithSortKey>> result = new ArrayList<>(memtables.size());
+        try
+        {
+            for (MemtableIndex index : memtables)
+                result.add(index.orderResultsBy(context, source, orderer, limit));
+
+            return result;
+        }
+        catch (Exception ex)
+        {
+            FileUtils.closeQuietly(result);
+            throw ex;
+        }
+    }
+
+    public long liveMemtableWriteCount()
+    {
+        return liveMemtables.values().stream().mapToLong(MemtableIndex::writeCount).sum();
+    }
+
+    public long estimatedOnHeapMemIndexMemoryUsed()
+    {
+        return liveMemtables.values().stream().mapToLong(MemtableIndex::estimatedOnHeapMemoryUsed).sum();
+    }
+
+    public long estimatedOffHeapMemIndexMemoryUsed()
+    {
+        return liveMemtables.values().stream().mapToLong(MemtableIndex::estimatedOffHeapMemoryUsed).sum();
+    }
+
+    /**
+     * @return A set of SSTables which have attached to them invalid index components.
+     */
+    public Set<SSTableContext> onSSTableChanged(Collection<SSTableReader> oldSSTables,
+                                                Collection<SSTableReader> newSSTables,
+                                                Collection<SSTableContext> newContexts,
+                                                boolean validate)
+    {
+        return viewManager.update(oldSSTables, newSSTables, newContexts, validate);
+    }
+
+    public ColumnMetadata getDefinition()
+    {
+        return column;
+    }
+
+    public AbstractType<?> getValidator()
+    {
+        return validator;
+    }
+
+    public boolean isNonFrozenCollection()
+    {
+        return TypeUtil.isNonFrozenCollection(column.type);
+    }
+
+    public boolean isCollection()
+    {
+        return column.type.isCollection();
+    }
+
+    public boolean isFrozen()
+    {
+        return TypeUtil.isFrozen(column.type);
+    }
+
+    public String getColumnName()
+    {
+        return column.name.toString();
+    }
+
+    public String getIndexName()
+    {
+        return this.config == null ? null : config.name;
+    }
+
+    public int getIntOption(String name, int defaultValue)
+    {
+        String value = this.config.options.get(name);
+        if (value == null)
+            return defaultValue;
+
+        try
+        {
+            return Integer.parseInt(value);
+        }
+        catch (NumberFormatException e)
+        {
+            logger.error("Failed to parse index configuration " + name + " = " + value + " as integer");
+            return defaultValue;
+        }
+    }
+
+    public AbstractAnalyzer.AnalyzerFactory getAnalyzerFactory()
+    {
+        return analyzerFactory;
+    }
+
+    public AbstractAnalyzer.AnalyzerFactory getQueryAnalyzerFactory()
+    {
+        return queryAnalyzerFactory;
+    }
+
+    public IndexWriterConfig getIndexWriterConfig()
+    {
+        return indexWriterConfig;
+    }
+
+    public View getView()
+    {
+        return viewManager.getView();
+    }
+
+    /**
+     * @return total number of per-index open files
+     */
+    public int openPerIndexFiles()
+    {
+        return viewManager.getView().size() * Version.latest().onDiskFormat().openFilesPerIndex(this);
+    }
+
+    public void prepareSSTablesForRebuild(Collection<SSTableReader> sstablesToRebuild)
+    {
+        viewManager.prepareSSTablesForRebuild(sstablesToRebuild);
+    }
+
+    public boolean isIndexed()
+    {
+        return config != null && !dropped;
+    }
+
+    public boolean isDropped()
+    {
+        return dropped;
+    }
+
+    /**
+     * @return whether the column is analyzed, meaning it uses an analyzer that isn't no-op.
+     */
+    public boolean isAnalyzed()
+    {
+        return isAnalyzed;
+    }
+
+    /**
+     * Called when index is dropped. Mark all {@link SSTableIndex} as released and per-column index files
+     * will be removed when in-flight queries completed and {@code obsolete} is true.
+     *
+     * @param obsolete true if index files should be deleted after invalidate; false otherwise.
+     */
+    public void invalidate(boolean obsolete)
+    {
+        dropped = true;
+        liveMemtables.clear();
+        viewManager.invalidate(obsolete);
+        indexMetrics.release();
+        columnQueryMetrics.release();
+
+        analyzerFactory.close();
+        if (queryAnalyzerFactory != analyzerFactory)
+        {
+            queryAnalyzerFactory.close();
+        }
+    }
+
+    public ConcurrentMap<Memtable, MemtableIndex> getLiveMemtables()
+    {
+        return liveMemtables;
+    }
+
+    public @Nullable MemtableIndex getMemtableIndex(Memtable memtable)
+    {
+        return liveMemtables.get(memtable);
+    }
+
+    public @Nullable SSTableIndex getSSTableIndex(Descriptor descriptor)
+    {
+        return getView().getSSTableIndex(descriptor);
+    }
+
+    public boolean supports(Operator op)
+    {
+        if (op.isLike() || op == Operator.LIKE) return false;
+        // Analyzed columns store the indexed result, so we are unable to compute raw equality.
+        // The only supported operators are ANALYZER_MATCHES and BM25.
+        if (op == Operator.ANALYZER_MATCHES) return isAnalyzed;
+        // BM25 frequency calculations only work on non-collection columns because it assumes a 1:1 mapping from PrK
+        // to frequency, but collections have mulitple documents.
+        if (op == Operator.BM25) return isAnalyzed && !isCollection();
+
+        // If the column is analyzed and the operator is EQ, we need to check if the analyzer supports it.
+        if (op == Operator.EQ && isAnalyzed && !analyzerFactory.supportsEquals())
+            return false;
+
+        // ANN is only supported against vectors.
+        // BOUNDED_ANN is only supported against vectors with a Euclidean similarity function.
+        // Vector indexes only support ANN and BOUNDED_ANN
+        if (column.type instanceof VectorType)
+            return op == Operator.ANN || (op == Operator.BOUNDED_ANN && hasEuclideanSimilarityFunc);
+        if (op == Operator.ANN || op == Operator.BOUNDED_ANN)
+            return false;
+
+        // Only regular columns can be sorted by SAI (at least for now)
+        if (op == Operator.ORDER_BY_ASC || op == Operator.ORDER_BY_DESC)
+            return !isCollection()
+                   && column.isRegular()
+                   && !isAnalyzed
+                   && !(column.type instanceof InetAddressType  // Possible, but need to add decoding logic based on
+                                                                 // SAI's TypeUtil.encode method.
+                         || column.type instanceof DecimalType   // Currently truncates to 24 bytes
+                         || column.type instanceof IntegerType); // Currently truncates to 20 bytes
+
+        Expression.Op operator = Expression.Op.valueOf(op);
+        if (isNonFrozenCollection())
+        {
+            if (indexType == IndexTarget.Type.KEYS)
+                return operator == Expression.Op.CONTAINS_KEY
+                       || operator == Expression.Op.NOT_CONTAINS_KEY;
+            if (indexType == IndexTarget.Type.VALUES)
+                return operator == Expression.Op.CONTAINS_VALUE
+                       || operator == Expression.Op.NOT_CONTAINS_VALUE;
+            return indexType == IndexTarget.Type.KEYS_AND_VALUES &&
+                   (operator == Expression.Op.EQ || operator == Expression.Op.NOT_EQ || operator == Expression.Op.RANGE);
+        }
+        if (indexType == IndexTarget.Type.FULL)
+            return operator == Expression.Op.EQ;
+        AbstractType<?> validator = getValidator();
+        if (operator == Expression.Op.IN)
+            return true;
+        if (operator != Expression.Op.EQ && EQ_ONLY_TYPES.contains(validator)) return false;
+        // RANGE only applicable to non-literal indexes
+        return (operator != null) && !(TypeUtil.isLiteral(validator) && operator == Expression.Op.RANGE);
+    }
+
+    public ByteBuffer getValueOf(DecoratedKey key, Row row, long nowInSecs)
+    {
+        if (row == null)
+            return null;
+
+        switch (column.kind)
+        {
+            case PARTITION_KEY:
+                if (key == null)
+                    return null;
+                return partitionKeyType instanceof CompositeType
+                       ? CompositeType.extractComponent(key.getKey(), column.position())
+                       : key.getKey();
+            case CLUSTERING:
+                // skip indexing of static clustering when regular column is indexed
+                return row.isStatic() ? null : row.clustering().bufferAt(column.position());
+
+            // treat static cell retrieval the same was as regular
+            // only if row kind is STATIC otherwise return null
+            case STATIC:
+                if (!row.isStatic())
+                    return null;
+            case REGULAR:
+                Cell cell = row.getCell(column);
+                return cell == null || !cell.isLive(nowInSecs) ? null : cell.buffer();
+
+            default:
+                return null;
+        }
+    }
+
+    public Iterator<ByteBuffer> getValuesOf(Row row, long nowInSecs)
+    {
+        if (row == null)
+            return null;
+
+        switch (column.kind)
+        {
+            // treat static cell retrieval the same was as regular
+            // only if row kind is STATIC otherwise return null
+            case STATIC:
+                if (!row.isStatic())
+                    return null;
+            case REGULAR:
+                return TypeUtil.collectionIterator(validator, row.getComplexColumnData(column), column, indexType, nowInSecs);
+
+            default:
+                return null;
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("columnName", getColumnName())
+                          .add("indexName", getIndexName())
+                          .toString();
+    }
+
+    public boolean isLiteral()
+    {
+        return TypeUtil.isLiteral(getValidator());
+    }
+
+    public boolean isVector()
+    {
+        //VSTODO probably move this down to TypeUtils eventually
+        return getValidator().isVector();
+    }
+
+    public void validate(DecoratedKey key, Row row)
+    {
+        // Validate the size of the inserted term.
+        if (SAI_VALIDATE_MAX_TERM_SIZE_AT_COORDINATOR.getBoolean())
+            validateMaxTermSizeForRow(key, row);
+
+        // Verify vector is valid.
+        if (isVector())
+        {
+            float[] value = TypeUtil.decomposeVector(getValidator(), getValueOf(key, row, FBUtilities.nowInSeconds()));
+            if (value != null)
+                VectorValidation.validateIndexable(value, vectorSimilarityFunction);
+        }
+    }
+
+    public void validate(RowFilter rowFilter)
+    {
+        // Only vector indexes have requirements to validate right now.
+        if (!isVector())
+            return;
+        // Only iterate over the top level expressions because that is where the ANN expression is located.
+        for (RowFilter.Expression expression : rowFilter.root().expressions())
+            if (expression.operator() == Operator.ANN && expression.column().equals(column))
+            {
+                float[] value = TypeUtil.decomposeVector(getValidator(), expression.getIndexValue());
+                VectorValidation.validateIndexable(value, vectorSimilarityFunction);
+                // There is only one ANN expression per query.
+                return;
+            }
+    }
+
+
+    public boolean equals(Object obj)
+    {
+        if (obj == this)
+            return true;
+
+        if (!(obj instanceof IndexContext))
+            return false;
+
+        IndexContext other = (IndexContext) obj;
+
+        return Objects.equals(column, other.column) &&
+               Objects.equals(indexType, other.indexType) &&
+               Objects.equals(config, other.config) &&
+               Objects.equals(partitionKeyType, other.partitionKeyType) &&
+               Objects.equals(clusteringComparator, other.clusteringComparator);
+    }
+
+    public int hashCode()
+    {
+        return Objects.hash(column, indexType, config, partitionKeyType, clusteringComparator);
+    }
+
+    /**
+     * A helper method for constructing consistent log messages for specific column indexes.
+     *
+     * Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message
+     * "Flushing new index segment..." will produce...
+     *
+     * "[ks.tb.idx] Flushing new index segment..."
+     *
+     * @param message The raw content of a logging message, without information identifying it with an index.
+     *
+     * @return A log message with the proper keyspace, table and index name prepended to it.
+     */
+    public String logMessage(String message)
+    {
+        // Index names are unique only within a keyspace.
+        return String.format("[%s.%s.%s] %s", keyspace, table, config == null ? "?" : config.name, message);
+    }
+
+    /**
+     * @return the indexes that are built on the given SSTables on the left and corrupted indexes'
+     * corresponding contexts on the right
+     */
+    public Pair<Set<SSTableIndex>, Set<SSTableContext>> getBuiltIndexes(Collection<SSTableContext> sstableContexts, boolean validate)
+    {
+        Set<SSTableIndex> valid = ConcurrentHashMap.newKeySet();
+        Set<SSTableContext> invalid = ConcurrentHashMap.newKeySet();
+
+        sstableContexts.stream().parallel().forEach(context -> {
+            if (context.sstable.isMarkedCompacted())
+                return;
+
+            var perSSTableComponents = context.usedPerSSTableComponents();
+            var perIndexComponents = perSSTableComponents.indexDescriptor().perIndexComponents(this);
+            if (!perSSTableComponents.isComplete() || !perIndexComponents.isComplete())
+            {
+                logger.debug(logMessage("An on-disk index build for SSTable {} has not completed (per-index components={})."), context.descriptor(), perIndexComponents.all());
+                return;
+            }
+
+            try
+            {
+                if (validate)
+                {
+                    if (!perIndexComponents.validateComponents(context.sstable, cfs.getTracker(), false, false))
+                    {
+                        // Note that a precise warning is already logged by the validation if there is an issue.
+                        invalid.add(context);
+                        return;
+                    }
+                }
+
+                SSTableIndex index = new SSTableIndex(context, perIndexComponents);
+                long count = context.primaryKeyMapFactory().count();
+                logger.debug(logMessage("Successfully loaded index for SSTable {} with {} rows."), context.descriptor(), count);
+
+                // Try to add new index to the set, if set already has such index, we'll simply release and move on.
+                // This covers situation when SSTable collection has the same SSTable multiple
+                // times because we don't know what kind of collection it actually is.
+                if (!valid.add(index))
+                    index.release();
+            }
+            catch (Throwable e)
+            {
+                logger.error(logMessage("Failed to update per-column components for SSTable {}"), context.descriptor(), e);
+                invalid.add(context);
+            }
+        });
+
+        return Pair.create(valid, invalid);
+    }
+
+    /**
+     * @return the number of indexed rows in this index (aka. pair of term and rowId)
+     */
+    public long getCellCount()
+    {
+        return getView().getIndexes()
+                        .stream()
+                        .mapToLong(SSTableIndex::getRowCount)
+                        .sum();
+    }
+
+    /**
+     * @return the total size (in bytes) of per-column index components
+     */
+    public long diskUsage()
+    {
+        return getView().getIndexes()
+                        .stream()
+                        .mapToLong(SSTableIndex::sizeOfPerColumnComponents)
+                        .sum();
+    }
+
+    /**
+     * @return the total memory usage (in bytes) of per-column index on-disk data structure
+     */
+    public long indexFileCacheSize()
+    {
+        return getView().getIndexes()
+                        .stream()
+                        .mapToLong(SSTableIndex::indexFileCacheSize)
+                        .sum();
+    }
+
+    public IndexFeatureSet indexFeatureSet()
+    {
+        IndexFeatureSet.Accumulator accumulator = new IndexFeatureSet.Accumulator();
+        getView().getIndexes().stream().map(SSTableIndex::indexFeatureSet).forEach(set -> accumulator.accumulate(set));
+        return accumulator.complete();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/IndexValidation.java b/src/java/org/apache/cassandra/index/sai/IndexValidation.java
deleted file mode 100644
index edd9e0fd1c0a..000000000000
--- a/src/java/org/apache/cassandra/index/sai/IndexValidation.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai;
-
-public enum IndexValidation
-{
-    /**
-     * No validation to be performed
-     */
-    NONE,
-
-    /**
-     * Basic header/footer validation, but no data validation (fast)
-     */
-    HEADER_FOOTER,
-
-    /**
-     * Full validation with checksumming data (slow)
-     */
-    CHECKSUM
-
-}
diff --git a/src/java/org/apache/cassandra/index/sai/QueryContext.java b/src/java/org/apache/cassandra/index/sai/QueryContext.java
index e319730359ca..1d54526605b0 100644
--- a/src/java/org/apache/cassandra/index/sai/QueryContext.java
+++ b/src/java/org/apache/cassandra/index/sai/QueryContext.java
@@ -18,85 +18,245 @@
 
 package org.apache.cassandra.index.sai;
 
-import java.util.Collection;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.LongAdder;
 import javax.annotation.concurrent.NotThreadSafe;
 
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.exceptions.QueryCancelledException;
-import org.apache.cassandra.index.sai.plan.FilterTree;
-import org.apache.cassandra.index.sai.plan.QueryController;
-import org.apache.cassandra.utils.Clock;
+import com.google.common.annotations.VisibleForTesting;
 
-import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_TEST_DISABLE_TIMEOUT;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.utils.AbortedOperationException;
+import org.apache.cassandra.utils.MonotonicClock;
+
+import static java.lang.Math.max;
 
 /**
  * Tracks state relevant to the execution of a single query, including metrics and timeout monitoring.
- * <p>
- * Fields here are non-volatile, as they are accessed from a single thread.
  */
 @NotThreadSafe
 public class QueryContext
 {
-    private static final boolean DISABLE_TIMEOUT = SAI_TEST_DISABLE_TIMEOUT.getBoolean();
+    public static final boolean DISABLE_TIMEOUT = CassandraRelevantProperties.TEST_SAI_DISABLE_TIMEOUT.getBoolean();
 
-    private final ReadCommand readCommand;
-    private final long queryStartTimeNanos;
+    protected final long queryStartTimeNanos;
 
     public final long executionQuotaNano;
 
-    public long sstablesHit = 0;
-    public long segmentsHit = 0;
-    public long partitionsRead = 0;
-    public long rowsFiltered = 0;
+    private final LongAdder sstablesHit = new LongAdder();
+    private final LongAdder segmentsHit = new LongAdder();
+    private final LongAdder partitionsRead = new LongAdder();
+    private final LongAdder rowsPreFiltered = new LongAdder();
+    private final LongAdder rowsFiltered = new LongAdder();
+    private final LongAdder trieSegmentsHit = new LongAdder();
 
-    public long trieSegmentsHit = 0;
-    public long triePostingsSkips = 0;
-    public long triePostingsDecodes = 0;
+    private final LongAdder bkdPostingListsHit = new LongAdder();
+    private final LongAdder bkdSegmentsHit = new LongAdder();
 
-    public long balancedTreePostingListsHit = 0;
-    public long balancedTreeSegmentsHit = 0;
-    public long balancedTreePostingsSkips = 0;
-    public long balancedTreePostingsDecodes = 0;
+    private final LongAdder bkdPostingsSkips = new LongAdder();
+    private final LongAdder bkdPostingsDecodes = new LongAdder();
 
-    public boolean queryTimedOut = false;
+    private final LongAdder triePostingsSkips = new LongAdder();
+    private final LongAdder triePostingsDecodes = new LongAdder();
 
-    /**
-     * {@code true} if the local query for this context has matches from Memtable-attached indexes or indexes on
-     * unrepaired SSTables, and {@code false} otherwise. When this is {@code false}, {@link FilterTree} can ignore the
-     * coordinator suggestion to downgrade to non-strict filtering, potentially reducing the number of false positives.
-     *
-     * @see QueryController#getIndexQueryResults(Collection)
-     * */
-    public boolean hasUnrepairedMatches = false;
+    private final LongAdder queryTimeouts = new LongAdder();
+
+    private final LongAdder annGraphSearchLatency = new LongAdder();
 
-    private VectorQueryContext vectorContext;
+    private float annRerankFloor = 0.0f; // only called from single-threaded setup code
 
-    public QueryContext(ReadCommand readCommand, long executionQuotaMs)
+    private final LongAdder shadowedPrimaryKeyCount = new LongAdder();
+
+    // Determines the order of using indexes for filtering and sorting.
+    // Null means the query execution order hasn't been decided yet.
+    private FilterSortOrder filterSortOrder = null;
+
+    @VisibleForTesting
+    public QueryContext()
     {
-        this.readCommand = readCommand;
-        executionQuotaNano = TimeUnit.MILLISECONDS.toNanos(executionQuotaMs);
-        queryStartTimeNanos = Clock.Global.nanoTime();
+        this(DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.MILLISECONDS));
+    }
+
+    public QueryContext(long executionQuotaMs)
+    {
+        this.executionQuotaNano = TimeUnit.MILLISECONDS.toNanos(executionQuotaMs);
+        this.queryStartTimeNanos = MonotonicClock.Global.approxTime.now();
     }
 
     public long totalQueryTimeNs()
     {
-        return Clock.Global.nanoTime() - queryStartTimeNanos;
+        return MonotonicClock.Global.approxTime.now() - queryStartTimeNanos;
+    }
+
+    // setters
+    public void addSstablesHit(long val)
+    {
+        sstablesHit.add(val);
+    }
+    public void addSegmentsHit(long val) {
+        segmentsHit.add(val);
+    }
+    public void addPartitionsRead(long val)
+    {
+        partitionsRead.add(val);
+    }
+    public void addRowsFiltered(long val)
+    {
+        rowsFiltered.add(val);
+    }
+    public void addRowsPreFiltered(long val)
+    {
+        rowsPreFiltered.add(val);
+    }
+    public void addTrieSegmentsHit(long val)
+    {
+        trieSegmentsHit.add(val);
+    }
+    public void addBkdPostingListsHit(long val)
+    {
+        bkdPostingListsHit.add(val);
+    }
+    public void addBkdSegmentsHit(long val)
+    {
+        bkdSegmentsHit.add(val);
+    }
+    public void addBkdPostingsSkips(long val)
+    {
+        bkdPostingsSkips.add(val);
+    }
+    public void addBkdPostingsDecodes(long val)
+    {
+        bkdPostingsDecodes.add(val);
+    }
+    public void addTriePostingsSkips(long val)
+    {
+        triePostingsSkips.add(val);
+    }
+    public void addTriePostingsDecodes(long val)
+    {
+        triePostingsDecodes.add(val);
+    }
+    public void addQueryTimeouts(long val)
+    {
+        queryTimeouts.add(val);
+    }
+
+    public void addAnnGraphSearchLatency(long val)
+    {
+        annGraphSearchLatency.add(val);
+    }
+
+    public void setFilterSortOrder(FilterSortOrder filterSortOrder)
+    {
+        this.filterSortOrder = filterSortOrder;
+    }
+
+    // getters
+
+    public long sstablesHit()
+    {
+        return sstablesHit.longValue();
+    }
+    public long segmentsHit() {
+        return segmentsHit.longValue();
+    }
+    public long partitionsRead()
+    {
+        return partitionsRead.longValue();
+    }
+    public long rowsFiltered()
+    {
+        return rowsFiltered.longValue();
+    }
+    public long rowsPreFiltered()
+    {
+        return rowsPreFiltered.longValue();
+    }
+    public long trieSegmentsHit()
+    {
+        return trieSegmentsHit.longValue();
+    }
+    public long bkdPostingListsHit()
+    {
+        return bkdPostingListsHit.longValue();
+    }
+    public long bkdSegmentsHit()
+    {
+        return bkdSegmentsHit.longValue();
+    }
+    public long bkdPostingsSkips()
+    {
+        return bkdPostingsSkips.longValue();
+    }
+    public long bkdPostingsDecodes()
+    {
+        return bkdPostingsDecodes.longValue();
+    }
+    public long triePostingsSkips()
+    {
+        return triePostingsSkips.longValue();
+    }
+    public long triePostingsDecodes()
+    {
+        return triePostingsDecodes.longValue();
+    }
+    public long queryTimeouts()
+    {
+        return queryTimeouts.longValue();
+    }
+    public long annGraphSearchLatency()
+    {
+        return annGraphSearchLatency.longValue();
+    }
+
+    public FilterSortOrder filterSortOrder()
+    {
+        return filterSortOrder;
     }
 
     public void checkpoint()
     {
         if (totalQueryTimeNs() >= executionQuotaNano && !DISABLE_TIMEOUT)
         {
-            queryTimedOut = true;
-            throw new QueryCancelledException(readCommand);
+            addQueryTimeouts(1);
+            throw new AbortedOperationException();
         }
     }
 
-    public VectorQueryContext vectorContext()
+    public void addShadowed(long count)
     {
-        if (vectorContext == null)
-            vectorContext = new VectorQueryContext(readCommand);
-        return vectorContext;
+        shadowedPrimaryKeyCount.add(count);
+    }
+
+    /**
+     * @return shadowed primary keys, in ascending order
+     */
+    public long getShadowedPrimaryKeyCount()
+    {
+        return shadowedPrimaryKeyCount.longValue();
+    }
+
+    public float getAnnRerankFloor()
+    {
+        return annRerankFloor;
+    }
+
+    public void updateAnnRerankFloor(float observedFloor)
+    {
+        if (observedFloor < Float.POSITIVE_INFINITY)
+            annRerankFloor = max(annRerankFloor, observedFloor);
+    }
+
+    /**
+     * Determines the order of filtering and sorting operations.
+     * Currently used only by vector search.
+     */
+    public enum FilterSortOrder
+    {
+        /** First get the matching keys from the non-vector indexes, then use vector index to return the top K by similarity order */
+        SEARCH_THEN_ORDER,
+
+        /** First get the candidates in ANN order from the vector index, then fetch the rows and filter them until we find K matching the predicates */
+        SCAN_THEN_FILTER
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/README.md b/src/java/org/apache/cassandra/index/sai/README.md
index 36176a4c616d..a475b0c2222e 100644
--- a/src/java/org/apache/cassandra/index/sai/README.md
+++ b/src/java/org/apache/cassandra/index/sai/README.md
@@ -19,25 +19,25 @@
 # Storage-Attached Indexing
 
 ## Overview
-Storage-attached indexing is a column based local secondary index implementation for Cassandra.
+Storage-attached indexes are a new column-based secondary indexing apparatus for DSE.
 
-The project was inspired by SASI (SSTable-Attached Secondary Indexes) and retains some of its high-level
+The project was inspired by OSS SASI (SSTable-Attached Secondary Indexes) and retains some of its high-level
 architectural character (and even some actual code), but makes significant improvements in a number of areas:
 
 - The on-disk/SSTable index formats for both string and numeric data have been completely replaced. Strings are indexed
-  on disk using a byte-ordered trie data structure, while numeric types are indexed using a block-oriented balanced tree.
+  on disk using our proprietary on-disk byte-ordered trie data structure, while numeric types are indexed using Lucene's
+  balanced kd-tree.
 - While indexes continue to be managed at the column level from the user's perspective, the storage design at the column
   index level is row-based, with related offset and token information stored only once at the SSTable level. This
   drastically reduces our on-disk footprint when several columns are indexed on the same table.
-- Tracing, metrics, virtual table-based metadata and snapshot-based backup/restore are supported out of the box.
-- On-disk index components can be streamed completely when entire SSTable streaming is enabled.
-- Incremental index building is supported, and on-disk index components are included in snapshots.
+- The query path is synchronous and index searches run on IO threads.
+- Tracing, metrics, virtual table-based metadata, RLAC, and snapshot-based backup/restore are supported out of the box.
 
 Many similarities with standard secondary indexes remain:
 
 - The full set of C* consistency levels is supported for both reads and writes.
 - Index updates are synchronous with mutations and do not require any kind of read-before-write.
-- Global queries are implemented on the back of C* range reads.
+- Queries are implemented on the back of C* range reads.
 - Paging is supported.
 - Only token ordering of results is supported.
 - Index builds are visible to operators as compactions and are executed on compaction threads.
@@ -48,23 +48,37 @@ Many similarities with standard secondary indexes remain:
 
 The following short tutorial will get you up-and-running with storage-attached indexing.
 
-### Build and Start Cassandra
+### Build and Start DSE
 
-Follow the instructions to build and start Cassandra in README.asc in root folder of the Cassandra repository
+1.) Make sure you've created the following directories and given yourself permissions on them:
+
+`/var/log/cassandra`
+
+`/var/lib/cassandra`
+
+2.) From the bdp root directory, run the following commands:
+
+`./gradlew jar`
+
+`bin/dse cassandra`
+
+3.) When the node stabilizes, open up `cqlsh` from the bdp root directory.
+
+`bin/cqlsh`
 
 ### Create a Simple Data Model
 
 1.) Run the following DDL statements to create a table and two indexes:
 
-`CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'};`
+`CREATE KEYSPACE test WITH replication = {'class': 'NetworkTopologyStrategy' , 'Cassandra': '1'};`
 
 `USE test;`
 
 `CREATE TABLE person (id int, name text, age int, PRIMARY KEY (id));`
 
-`CREATE INDEX ON person (name) USING 'sai' WITH OPTIONS = {'case_sensitive': false};`
+`CREATE CUSTOM INDEX ON person (name) USING 'StorageAttachedIndex' WITH OPTIONS = {'case_sensitive': false};`
 
-`CREATE INDEX ON person (age) USING 'sai';`
+`CREATE CUSTOM INDEX ON person (age) USING 'StorageAttachedIndex';`
 
 2.) Add some data.
 
@@ -107,10 +121,9 @@ Follow the instructions to build and start Cassandra in README.asc in root folde
 - Zhao Yang
 - Jason Rutherglen
 - Maciej Zasada
-- Andres de la Peña
+- Andrew de la Peña
 - Mike Adamson
 - Zahir Patni
 - Tomek Lasica
 - Berenguer Blasi
 - Rocco Varela
-- Piotr Kołaczkowski
diff --git a/src/java/org/apache/cassandra/index/sai/SSTableContext.java b/src/java/org/apache/cassandra/index/sai/SSTableContext.java
index 96c53a228acb..3db41b4a6272 100644
--- a/src/java/org/apache/cassandra/index/sai/SSTableContext.java
+++ b/src/java/org/apache/cassandra/index/sai/SSTableContext.java
@@ -17,13 +17,11 @@
  */
 package org.apache.cassandra.index.sai;
 
-import java.util.Collections;
-
 import com.google.common.base.Objects;
 
 import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.utils.Throwables;
@@ -32,26 +30,28 @@
 import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
 
 /**
- * An {@link SSTableContext} is created for an individual sstable and is shared across column indexes to track per-sstable
- * index files.
- * <p>
- * The {@link SSTableContext} will be released when receiving a sstable removed notification, but its shared copies in
- * individual {@link SSTableIndex}es will be released when in-flight read requests complete.
+ * SSTableContext is created for individual sstable shared across indexes to track per-sstable index files.
+ *
+ * SSTableContext itself will be released when receiving sstable removed notification, but its shared copies in individual
+ * SSTableIndex will be released when in-flight read requests complete.
  */
 public class SSTableContext extends SharedCloseableImpl
 {
     public final SSTableReader sstable;
-    public final IndexDescriptor indexDescriptor;
+    private final IndexComponents.ForRead perSSTableComponents;
+    public final PrimaryKey.Factory primaryKeyFactory;
     public final PrimaryKeyMap.Factory primaryKeyMapFactory;
 
     private SSTableContext(SSTableReader sstable,
-                           IndexDescriptor indexDescriptor,
+                           IndexComponents.ForRead perSSTableComponents,
+                           PrimaryKey.Factory primaryKeyFactory,
                            PrimaryKeyMap.Factory primaryKeyMapFactory,
                            Cleanup cleanup)
     {
         super(cleanup);
         this.sstable = sstable;
-        this.indexDescriptor = indexDescriptor;
+        this.perSSTableComponents = perSSTableComponents;
+        this.primaryKeyFactory = primaryKeyFactory;
         this.primaryKeyMapFactory = primaryKeyMapFactory;
     }
 
@@ -59,16 +59,20 @@ private SSTableContext(SSTableContext copy)
     {
         super(copy);
         this.sstable = copy.sstable;
-        this.indexDescriptor = copy.indexDescriptor;
+        this.perSSTableComponents = copy.perSSTableComponents;
+        this.primaryKeyFactory = copy.primaryKeyFactory;
         this.primaryKeyMapFactory = copy.primaryKeyMapFactory;
     }
 
-    public static SSTableContext create(SSTableReader sstable)
+    @SuppressWarnings("resource")
+    public static SSTableContext create(SSTableReader sstable, IndexComponents.ForRead perSSTableComponents)
     {
+        var onDiskFormat = perSSTableComponents.onDiskFormat();
+        PrimaryKey.Factory primaryKeyFactory = onDiskFormat.newPrimaryKeyFactory(sstable.metadata().comparator);
+
         Ref<? extends SSTableReader> sstableRef = null;
         PrimaryKeyMap.Factory primaryKeyMapFactory = null;
 
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
         try
         {
             sstableRef = sstable.tryRef();
@@ -78,11 +82,11 @@ public static SSTableContext create(SSTableReader sstable)
                 throw new IllegalStateException("Couldn't acquire reference to the sstable: " + sstable);
             }
 
-            primaryKeyMapFactory = indexDescriptor.newPrimaryKeyMapFactory(sstable);
+            primaryKeyMapFactory = onDiskFormat.newPrimaryKeyMapFactory(perSSTableComponents, primaryKeyFactory, sstable);
 
-            Cleanup cleanup = new Cleanup(primaryKeyMapFactory, indexDescriptor, sstableRef);
+            Cleanup cleanup = new Cleanup(primaryKeyMapFactory, sstableRef);
 
-            return new SSTableContext(sstable, indexDescriptor, primaryKeyMapFactory, cleanup);
+            return new SSTableContext(sstable, perSSTableComponents, primaryKeyFactory, primaryKeyMapFactory, cleanup);
         }
         catch (Throwable t)
         {
@@ -91,22 +95,22 @@ public static SSTableContext create(SSTableReader sstable)
                 sstableRef.release();
             }
 
-            throw Throwables.unchecked(Throwables.close(t, Collections.singleton(primaryKeyMapFactory)));
+            throw Throwables.unchecked(Throwables.close(t, primaryKeyMapFactory));
         }
     }
 
-    @Override
-    public SSTableContext sharedCopy()
+    /**
+     * Returns the concrete on-disk perSStable components used by this context instance.
+     */
+    public IndexComponents.ForRead usedPerSSTableComponents()
     {
-        return new SSTableContext(this);
+        return perSSTableComponents;
     }
 
-    /**
-     * Returns a new {@link SSTableIndex} for a per-column index
-     */
-    public SSTableIndex newSSTableIndex(StorageAttachedIndex index)
+    @Override
+    public SSTableContext sharedCopy()
     {
-        return indexDescriptor.newSSTableIndex(this, index);
+        return new SSTableContext(this);
     }
 
     /**
@@ -117,12 +121,19 @@ public Descriptor descriptor()
         return sstable.descriptor;
     }
 
-    /**
-     * @return disk usage (in bytes) of per-sstable index files
-     */
-    public long diskUsage()
+    public SSTableReader sstable()
+    {
+        return sstable;
+    }
+
+    public PrimaryKey.Factory primaryKeyFactory()
+    {
+        return primaryKeyFactory;
+    }
+
+    public PrimaryKeyMap.Factory primaryKeyMapFactory()
     {
-        return indexDescriptor.sizeOnDiskOfPerSSTableComponents();
+        return primaryKeyMapFactory;
     }
 
     /**
@@ -130,7 +141,7 @@ public long diskUsage()
      */
     public int openFilesPerSSTable()
     {
-        return indexDescriptor.version.onDiskFormat().openFilesPerSSTableIndex(indexDescriptor.hasClustering());
+        return perSSTableComponents.onDiskFormat().openFilesPerSSTable();
     }
 
     @Override
@@ -159,15 +170,11 @@ public int hashCode()
     private static class Cleanup implements RefCounted.Tidy
     {
         private final PrimaryKeyMap.Factory primaryKeyMapFactory;
-        private final IndexDescriptor indexDescriptor;
         private final Ref<? extends SSTableReader> sstableRef;
 
-        private Cleanup(PrimaryKeyMap.Factory primaryKeyMapFactory,
-                        IndexDescriptor indexDescriptor,
-                        Ref<? extends SSTableReader> sstableRef)
+        private Cleanup(PrimaryKeyMap.Factory primaryKeyMapFactory, Ref<? extends SSTableReader> sstableRef)
         {
             this.primaryKeyMapFactory = primaryKeyMapFactory;
-            this.indexDescriptor = indexDescriptor;
             this.sstableRef = sstableRef;
         }
 
@@ -175,7 +182,7 @@ private Cleanup(PrimaryKeyMap.Factory primaryKeyMapFactory,
         public void tidy()
         {
             Throwable t = sstableRef.ensureReleased(null);
-            t = Throwables.close(t, Collections.singleton(primaryKeyMapFactory));
+            t = Throwables.close(t, primaryKeyMapFactory);
 
             Throwables.maybeFail(t);
         }
@@ -183,7 +190,7 @@ public void tidy()
         @Override
         public String name()
         {
-            return indexDescriptor.toString();
+            return null;
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/SSTableContextManager.java b/src/java/org/apache/cassandra/index/sai/SSTableContextManager.java
index b2df0f29c248..c0fff0304733 100644
--- a/src/java/org/apache/cassandra/index/sai/SSTableContextManager.java
+++ b/src/java/org/apache/cassandra/index/sai/SSTableContextManager.java
@@ -17,90 +17,149 @@
  */
 package org.apache.cassandra.index.sai;
 
+import java.lang.invoke.MethodHandles;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.Nullable;
 import javax.annotation.concurrent.ThreadSafe;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.utils.Pair;
 
 /**
- * Manages per-sstable {@link SSTableContext}s for {@link StorageAttachedIndexGroup}
+ * Manage per-sstable {@link SSTableContext} for {@link StorageAttachedIndexGroup}
  */
 @ThreadSafe
 public class SSTableContextManager
 {
-    private static final Logger logger = LoggerFactory.getLogger(SSTableContextManager.class);
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+    // Even though `SSTableContext` happens to point to its corresponding `IndexDescriptor` for convenience, we track
+    // the latter separately because we need to track descriptors before it is safe to build a context (we create
+    // a descriptor as soon as we start indexing a sstable to start tracking the added components, but can only create
+    // its context when the per-sstable components are complete).
+    private final ConcurrentHashMap<SSTableReader, IndexDescriptor> sstableDescriptors = new ConcurrentHashMap<>();
     private final ConcurrentHashMap<SSTableReader, SSTableContext> sstableContexts = new ConcurrentHashMap<>();
 
+    private final Tracker tracker;
+
+    SSTableContextManager(Tracker tracker)
+    {
+        this.tracker = tracker;
+    }
+
     /**
      * Initialize {@link SSTableContext}s if they are not already initialized.
      *
      * @param removed SSTables being removed
      * @param added SSTables being added
-     * @param validation Controls how indexes should be validated
+     * @param validate if true, header and footer will be validated.
      *
-     * @return a set of contexts for SSTables with valid per-SSTable components, and a set of
-     * SSTables with invalid or missing components
+     * @return if all the added (and still "live" at the time of this call) sstable with complete index build have
+     * valid per-sstable components, then an optional with the context for all those sstables. Otherwise, if any sstable
+     * has invalid/missing components, then an empty optional is returned (and all invalid sstable will have had their
+     * context removed, after a call to onInvalid).
      */
-    public Pair<Set<SSTableContext>, Set<SSTableReader>> update(Collection<SSTableReader> removed, Iterable<SSTableReader> added, IndexValidation validation)
+    @SuppressWarnings("resource")
+    public Optional<Set<SSTableContext>> update(Collection<SSTableReader> removed, Iterable<SSTableReader> added, boolean validate, Set<StorageAttachedIndex> indices)
     {
         release(removed);
 
         Set<SSTableContext> contexts = new HashSet<>();
-        Set<SSTableReader> invalid = new HashSet<>();
+        boolean hasInvalid = false;
 
         for (SSTableReader sstable : added)
         {
             if (sstable.isMarkedCompacted())
             {
+                logger.debug("Skipped tracking sstable {} because it's marked compacted", sstable);
                 continue;
             }
 
-            IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
-
-            if (!indexDescriptor.isPerSSTableIndexBuildComplete())
+            IndexDescriptor indexDescriptor = getOrLoadIndexDescriptor(sstable, indices);
+            var perSSTableComponents = indexDescriptor.perSSTableComponents();
+            if (!perSSTableComponents.isComplete())
             {
-                // Don't even try to validate or add the context if the completion marker is missing.
+                // This usually means no index has been built for that sstable yet (the alternative would be that we
+                // lost the completion marker when index build failed). Not point in running
+                // validation (it would fail), and we also don't want to add it to the returned contexts, since it's
+                // not ready yet. We know a future call of this method will be triggered for that sstable once the
+                // index finishes building.
+                logger.debug("Skipped tracking sstable {} because per sstable components are not complete (components={})", sstable, perSSTableComponents.all());
                 continue;
             }
 
             try
             {
                 // Only validate on restart or newly refreshed SSTable. Newly built files are unlikely to be corrupted.
-                if (!sstableContexts.containsKey(sstable) && !indexDescriptor.validatePerSSTableComponents(validation, true, false))
+                if (validate && !sstableContexts.containsKey(sstable) && !perSSTableComponents.validateComponents(sstable, tracker, true, false))
                 {
-                    invalid.add(sstable);
-                    removeInvalidSSTableContext(sstable);
+                    // Note that the validation already log details on the problem if it fails, so no reason to log further
+                    hasInvalid = true;
                     continue;
                 }
-                // ConcurrentHashMap#computeIfAbsent guarantees atomicity, so {@link SSTableContext#create(SSTableReader)}}
-                // is called at most once per key.
-                contexts.add(sstableContexts.computeIfAbsent(sstable, SSTableContext::create));
+                // ConcurrentHashMap#compute guarantees atomicity, so {@link SSTableContext#create(SSTableReader)}} is
+                // called at most once per key and underlying components.
+                contexts.add(sstableContexts.compute(sstable, (__, prevContext) -> computeUpdatedContext(sstable, prevContext, perSSTableComponents)));
             }
             catch (Throwable t)
             {
-                logger.warn(indexDescriptor.logMessage("Failed to update per-SSTable components for SSTable {}"), sstable.descriptor, t);
-                invalid.add(sstable);
-                removeInvalidSSTableContext(sstable);
+                logger.warn(indexDescriptor.logMessage("Unexpected error updating per-SSTable components for SSTable {}"), sstable.descriptor, t);
+                // We haven't been able to correctly set the context, so the index shouldn't be used, and we invalidate
+                // the components to ensure that's the case.
+                perSSTableComponents.invalidate(sstable, tracker);
+                hasInvalid = true;
+                remove(sstable);
             }
         }
 
-        return Pair.create(contexts, invalid);
+        return hasInvalid ? Optional.empty() : Optional.of(contexts);
+    }
+
+    private static SSTableContext computeUpdatedContext(SSTableReader reader, @Nullable SSTableContext previousContext, IndexComponents.ForRead perSSTableComponents)
+    {
+        // We can (and should) keep the previous context if both:
+        // 1. it exists
+        // 2. it uses a "complete" set of per-sstable components (not that we always initially create a `SSTableContext`
+        //    from a complete set, so if it is not complete, it means the previous components have been corrupted, and
+        //    we want to use the new one (a rebuild)).
+        // 3. it uses "up-to-date" per-sstable components.
+        if (previousContext != null && previousContext.usedPerSSTableComponents().isComplete() && previousContext.usedPerSSTableComponents().buildId().equals(perSSTableComponents.buildId()))
+            return previousContext;
+
+        // Now, if we create a new one, we should close the previous one if it exists.
+        // Note that `SSTableIndex` references `SSTableContext` through a `#sharedCopy() so even if there is still
+        // index referencing this context currently in use, this will not break ongoing queries.
+        if (previousContext != null)
+            previousContext.close();
+
+        return SSTableContext.create(reader, perSSTableComponents);
+    }
+
+    private void release(Collection<SSTableReader> toRelease)
+    {
+        toRelease.forEach(this::remove);
+    }
+
+    Collection<SSTableContext> allContexts()
+    {
+        return sstableContexts.values();
     }
 
-    public void release(Collection<SSTableReader> toRelease)
+    @VisibleForTesting
+    SSTableContext getContext(SSTableReader sstable)
     {
-        toRelease.stream().map(sstableContexts::remove).filter(Objects::nonNull).forEach(SSTableContext::close);
+        return sstableContexts.get(sstable);
     }
 
     /**
@@ -112,16 +171,19 @@ int openFiles()
     }
 
     /**
-     * @return total disk usage (in bytes) of all per-sstable index files
+     * @return total disk usage of all per-sstable index files
      */
     long diskUsage()
     {
-        return sstableContexts.values().stream().mapToLong(SSTableContext::diskUsage).sum();
+        return sstableContexts.values().stream()
+                              .mapToLong(ssTableContext -> ssTableContext.usedPerSSTableComponents().liveSizeOnDiskInBytes())
+                              .sum();
     }
 
-    Set<SSTableReader> sstables()
+    @VisibleForTesting
+    public boolean contains(SSTableReader sstable)
     {
-        return sstableContexts.keySet();
+        return sstableContexts.containsKey(sstable);
     }
 
     @VisibleForTesting
@@ -133,15 +195,33 @@ public int size()
     @VisibleForTesting
     public void clear()
     {
+        sstableDescriptors.clear();
         sstableContexts.values().forEach(SSTableContext::close);
         sstableContexts.clear();
     }
 
-    @SuppressWarnings("EmptyTryBlock")
-    private void removeInvalidSSTableContext(SSTableReader sstable)
+    @SuppressWarnings("resource")
+    private void remove(SSTableReader sstable)
     {
-        try (SSTableContext ignored = sstableContexts.remove(sstable))
-        {
-        }
+        sstableDescriptors.remove(sstable);
+        SSTableContext context = sstableContexts.remove(sstable);
+        if (context != null)
+            context.close();
+    }
+
+    IndexDescriptor getOrLoadIndexDescriptor(SSTableReader sstable, Set<StorageAttachedIndex> indices)
+    {
+        // If we have a SSTableReader, it means the sstable exists, and so if we don't have a descriptor for it,
+        // then create one now. Since the sstable exists, it also means that we will get notified if/when it
+        // is removed (in `update`), so we shouldn't "leak" descriptors.
+        return sstableDescriptors.computeIfAbsent(sstable, __ -> IndexDescriptor.load(sstable, contexts(indices)));
+    }
+
+    private static Set<IndexContext> contexts(Set<StorageAttachedIndex> indices)
+    {
+        Set<IndexContext> contexts = Sets.newHashSetWithExpectedSize(indices.size());
+        for (StorageAttachedIndex index : indices)
+            contexts.add(index.getIndexContext());
+        return contexts;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/SSTableIndex.java b/src/java/org/apache/cassandra/index/sai/SSTableIndex.java
new file mode 100644
index 000000000000..c885f672bf2c
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/SSTableIndex.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.disk.EmptyIndex;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMapIterator;
+import org.apache.cassandra.index.sai.disk.SearchableIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.Segment;
+import org.apache.cassandra.index.sai.iterators.KeyRangeAntiJoinIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.sstable.SSTableIdFactory;
+import org.apache.cassandra.io.sstable.SSTableWatcher;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.CloseableIterator;
+
+/**
+ * SSTableIndex is created for each column index on individual sstable to track per-column indexer.
+ */
+public class SSTableIndex
+{
+    private static final Logger logger = LoggerFactory.getLogger(SSTableIndex.class);
+
+    // sort sstable index by first key then last key
+    public static final Comparator<SSTableIndex> COMPARATOR = Comparator.comparing((SSTableIndex s) -> s.getSSTable().first)
+                                                                        .thenComparing(s -> s.getSSTable().last)
+                                                                        .thenComparing(s -> s.getSSTable().descriptor.id, SSTableIdFactory.COMPARATOR);
+
+    private final SSTableContext sstableContext;
+    private final IndexContext indexContext;
+    private final SSTableReader sstable;
+    private final SearchableIndex searchableIndex;
+    private final IndexComponents.ForRead perIndexComponents;
+
+    private final AtomicInteger references = new AtomicInteger(1);
+    private final AtomicBoolean indexWasDropped = new AtomicBoolean(false);
+
+    public SSTableIndex(SSTableContext sstableContext, IndexComponents.ForRead perIndexComponents)
+    {
+        assert perIndexComponents.context().getValidator() != null;
+        this.perIndexComponents = perIndexComponents;
+        this.searchableIndex = createSearchableIndex(sstableContext, perIndexComponents);
+
+        this.sstableContext = sstableContext.sharedCopy(); // this line must not be before any code that may throw
+        this.indexContext = perIndexComponents.context();
+        this.sstable = sstableContext.sstable;
+    }
+
+    private static SearchableIndex createSearchableIndex(SSTableContext sstableContext, IndexComponents.ForRead perIndexComponents)
+    {
+        if (CassandraRelevantProperties.SAI_INDEX_READS_DISABLED.getBoolean())
+        {
+            logger.info("Creating dummy (empty) index searcher for sstable {} as SAI index reads are disabled", sstableContext.sstable.descriptor);
+            return new EmptyIndex();
+        }
+
+        return perIndexComponents.onDiskFormat().newSearchableIndex(sstableContext, perIndexComponents);
+    }
+
+    public IndexContext getIndexContext()
+    {
+        return indexContext;
+    }
+
+    /**
+     * Returns the concrete on-disk perIndex components used by this index instance.
+     */
+    public IndexComponents.ForRead usedPerIndexComponents()
+    {
+        return perIndexComponents;
+    }
+
+    public SSTableContext getSSTableContext()
+    {
+        return sstableContext;
+    }
+
+    public List<Segment> getSegments()
+    {
+        return searchableIndex.getSegments();
+    }
+
+    public long indexFileCacheSize()
+    {
+        return searchableIndex.indexFileCacheSize();
+    }
+
+    /**
+     * @return number of indexed rows, note that rows may have been updated or removed in sstable.
+     */
+    public long getRowCount()
+    {
+        return searchableIndex.getRowCount();
+    }
+
+    public long estimateMatchingRowsCount(Expression predicate, AbstractBounds<PartitionPosition> keyRange)
+    {
+        return searchableIndex.estimateMatchingRowsCount(predicate, keyRange);
+    }
+
+    /**
+     * @return total size of per-column SAI components, in bytes
+     */
+    public long sizeOfPerColumnComponents()
+    {
+        return perIndexComponents.liveSizeOnDiskInBytes();
+    }
+
+    /**
+     * @return total size of per-sstable SAI components, in bytes
+     */
+    public long sizeOfPerSSTableComponents()
+    {
+        return sstableContext.usedPerSSTableComponents().liveSizeOnDiskInBytes();
+    }
+
+    /**
+     * @return the smallest possible sstable row id in this index.
+     */
+    public long minSSTableRowId()
+    {
+        return searchableIndex.minSSTableRowId();
+    }
+
+    /**
+     * @return the largest possible sstable row id in this index.
+     */
+    public long maxSSTableRowId()
+    {
+        return searchableIndex.maxSSTableRowId();
+    }
+
+    public ByteBuffer minTerm()
+    {
+        return searchableIndex.minTerm();
+    }
+
+    public ByteBuffer maxTerm()
+    {
+        return searchableIndex.maxTerm();
+    }
+
+    public DecoratedKey minKey()
+    {
+        return searchableIndex.minKey();
+    }
+
+    public DecoratedKey maxKey()
+    {
+        return searchableIndex.maxKey();
+    }
+
+    // Returns an iterator for NEQ, NOT_CONTAINS_KEY, NOT_CONTAINS_VALUE, which
+    // 1. Either includes everything if the column type values can be truncated and
+    // thus the keys cannot be matched precisely,
+    // 2. or includes everything minus the keys matching the expression
+    // if the column type values cannot be truncated, i.e., matching the keys is always precise.
+    // (not matching precisely will lead to false negatives)
+    //
+    // keys k such that row(k) not contains v = (all keys) \ (keys k such that row(k) contains v)
+    //
+    // Note that rows in other indexes are not matched, so this can return false positives,
+    // but they are not a problem as post-filtering would get rid of them.
+    // The keys matched in other indexes cannot be safely subtracted
+    // as indexes may contain false positives caused by deletes and updates.
+    private KeyRangeIterator getNonEqIterator(Expression expression,
+                                              AbstractBounds<PartitionPosition> keyRange,
+                                              QueryContext context,
+                                              boolean defer) throws IOException
+    {
+        KeyRangeIterator allKeys = allSSTableKeys(keyRange);
+        if (TypeUtil.supportsRounding(expression.validator))
+        {
+            return allKeys;
+        }
+        else
+        {
+            Expression negExpression = expression.negated();
+            KeyRangeIterator matchedKeys = searchableIndex.search(negExpression, keyRange, context, defer, Integer.MAX_VALUE);
+            return KeyRangeAntiJoinIterator.create(allKeys, matchedKeys);
+        }
+    }
+
+    public KeyRangeIterator search(Expression expression,
+                                   AbstractBounds<PartitionPosition> keyRange,
+                                   QueryContext context,
+                                   boolean defer,
+                                   int limit) throws IOException
+    {
+        if (expression.getOp().isNonEquality())
+        {
+            return getNonEqIterator(expression, keyRange, context, defer);
+        }
+
+        return searchableIndex.search(expression, keyRange, context, defer, limit);
+    }
+
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderBy(Orderer orderer,
+                                                                  Expression predicate,
+                                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                                  QueryContext context,
+                                                                  int limit,
+                                                                  long totalRows) throws IOException
+    {
+        return searchableIndex.orderBy(orderer, predicate, keyRange, context, limit, totalRows);
+    }
+
+    public void populateSegmentView(SimpleDataSet dataSet)
+    {
+        searchableIndex.populateSystemView(dataSet, sstable);
+    }
+
+    public Version getVersion()
+    {
+        return perIndexComponents.version();
+    }
+
+    public IndexFeatureSet indexFeatureSet()
+    {
+        return getVersion().onDiskFormat().indexFeatureSet();
+    }
+
+    public SSTableReader getSSTable()
+    {
+        return sstable;
+    }
+
+    public boolean reference()
+    {
+        while (true)
+        {
+            int n = references.get();
+            if (n <= 0)
+                return false;
+            if (references.compareAndSet(n, n + 1))
+            {
+                return true;
+            }
+        }
+    }
+
+    public boolean isReleased()
+    {
+        return references.get() <= 0;
+    }
+
+    public boolean isEmpty()
+    {
+        return searchableIndex instanceof EmptyIndex;
+    }
+
+    public void release()
+    {
+        int n = references.decrementAndGet();
+
+        if (n == 0)
+        {
+            FileUtils.closeQuietly(searchableIndex);
+            sstableContext.close();
+
+            /*
+             * When SSTable is removed, storage-attached index components will be automatically removed by LogTransaction.
+             * We only remove index components explicitly in case of index corruption or index rebuild if immutable
+             * components are not in use.
+             */
+            if (indexWasDropped.get())
+                SSTableWatcher.instance.onIndexDropped(sstable.metadata(), perIndexComponents.forWrite());
+        }
+    }
+
+    /**
+     * Indicates that this index has been dropped by the user, and so the underlying files can be safely removed.
+     */
+    public void markIndexDropped()
+    {
+        indexWasDropped.getAndSet(true);
+        release();
+    }
+
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SSTableIndex other = (SSTableIndex)o;
+        return Objects.equal(sstableContext, other.sstableContext) && Objects.equal(indexContext, other.indexContext);
+    }
+
+    public int hashCode()
+    {
+        return Objects.hashCode(sstableContext, indexContext);
+    }
+
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderResultsBy(QueryContext context, List<PrimaryKey> keys, Orderer orderer, int limit, long totalRows) throws IOException
+    {
+        return searchableIndex.orderResultsBy(context, keys, orderer, limit, totalRows);
+    }
+
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("column", indexContext.getColumnName())
+                          .add("sstable", sstable.descriptor)
+                          .add("totalRows", sstable.getTotalRows())
+                          .toString();
+    }
+
+    protected final KeyRangeIterator allSSTableKeys(AbstractBounds<PartitionPosition> keyRange) throws IOException
+    {
+        return PrimaryKeyMapIterator.create(sstableContext, keyRange);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
index b42a165f1111..84cbcf4482bb 100644
--- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
@@ -24,51 +24,46 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.NavigableMap;
 import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
 import java.util.function.BooleanSupplier;
 import java.util.stream.Collectors;
 
-import javax.annotation.Nullable;
-
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableSet;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.CqlBuilder;
 import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.QueryOptions;
-import org.apache.cassandra.cql3.restrictions.Restriction;
-import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.CassandraWriteContext;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RangeTombstone;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.WriteContext;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.guardrails.GuardrailViolatedException;
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.db.guardrails.MaxThreshold;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.Row;
@@ -79,27 +74,23 @@
 import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexBuildDecider;
 import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.index.SecondaryIndexBuilder;
+import org.apache.cassandra.index.SecondaryIndexManager;
 import org.apache.cassandra.index.TargetParser;
 import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.analyzer.AnalyzerEqOperatorSupport;
+import org.apache.cassandra.index.sai.analyzer.LuceneAnalyzer;
 import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.StorageAttachedIndexWriter;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
-import org.apache.cassandra.index.sai.memory.MemtableIndexManager;
-import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
-import org.apache.cassandra.index.sai.metrics.IndexMetrics;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.view.IndexViewManager;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.index.sai.view.View;
 import org.apache.cassandra.index.transactions.IndexTransaction;
-import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
-import org.apache.cassandra.io.sstable.SSTableIdFactory;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
@@ -108,57 +99,123 @@
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.FutureCombiner;
 import org.apache.cassandra.utils.concurrent.ImmediateFuture;
-import org.apache.cassandra.utils.concurrent.OpOrder;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_VALIDATE_TERMS_AT_COORDINATOR;
 import static org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig.MAX_TOP_K;
 
 public class StorageAttachedIndex implements Index
 {
     public static final String NAME = "sai";
-    
-    public static final String VECTOR_USAGE_WARNING = "SAI ANN indexes on vector columns are experimental and are not recommended for production use.\n" +
-                                                      "They don't yet support SELECT queries with:\n" +
-                                                      " * Consistency level higher than ONE/LOCAL_ONE.\n" +
-                                                      " * Paging.\n" +
-                                                      " * No LIMIT clauses.\n" +
-                                                      " * PER PARTITION LIMIT clauses.\n" +
-                                                      " * GROUP BY clauses.\n" +
-                                                      " * Aggregation functions.\n" +
-                                                      " * Filters on columns without a SAI index.";
-
-    public static final String VECTOR_NON_FLOAT_ERROR = "SAI ANN indexes are only allowed on vector columns with float elements";
-    public static final String VECTOR_1_DIMENSION_COSINE_ERROR = "Cosine similarity is not supported for single-dimension vectors";
-    public static final String VECTOR_MULTIPLE_DATA_DIRECTORY_ERROR = "SAI ANN indexes are not allowed on multiple data directories";
+    public static final String NGRAM_WITHOUT_QUERY_ANALYZER_WARNING =
+    "Using an ngram analyzer without defining a query_analyzer. " +
+    "This means that the same ngram analyzer will be applied to both indexed and queried column values. " +
+    "Applying ngram analysis to the queried values usually produces too many search tokens to be useful. " +
+    "The large number of tokens can also have a negative impact in performance. " +
+    "In most cases it's better to use a simpler query_analyzer such as the standard one.";
 
-    @VisibleForTesting
-    public static final String ANALYSIS_ON_KEY_COLUMNS_MESSAGE = "Analysis options are not supported on primary key columns, but found ";
+    private static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndex.class);
 
-    public static final String ANN_LIMIT_ERROR = "Use of ANN OF in an ORDER BY clause requires a LIMIT that is not greater than %s. LIMIT was %s";
+    private static final boolean VALIDATE_TERMS_AT_COORDINATOR = SAI_VALIDATE_TERMS_AT_COORDINATOR.getBoolean();
 
-    private static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndex.class);
+    private static class StorageAttachedIndexBuildingSupport implements IndexBuildingSupport
+    {
+        public NavigableMap<SSTableReader, Set<StorageAttachedIndex>> prepareSSTablesToBuild(StorageAttachedIndexGroup group,
+                                                       Set<Index> indexes,
+                                                       Collection<SSTableReader> sstablesToRebuild,
+                                                       boolean isFullRebuild)
+        {
+            NavigableMap<SSTableReader, Set<StorageAttachedIndex>> sstables = new TreeMap<>(SSTableReader.idComparator);
+
+            indexes.stream()
+                   .filter((i) -> i instanceof StorageAttachedIndex)
+                   .forEach((i) ->
+                            {
+                                StorageAttachedIndex sai = (StorageAttachedIndex) i;
+                                IndexContext indexContext = ((StorageAttachedIndex) i).getIndexContext();
+
+                                // If this is not a full manual index rebuild we can skip SSTables that already have an
+                                // attached index. Otherwise, we override any pre-existent index.
+                                Collection<SSTableReader> ss = sstablesToRebuild;
+                                if (!isFullRebuild)
+                                {
+                                    ss = sstablesToRebuild.stream()
+                                                          .filter(s -> !IndexDescriptor.isIndexBuildCompleteOnDisk(s, indexContext))
+                                                          .collect(Collectors.toList());
+                                }
+
+                                group.prepareIndexSSTablesForRebuild(ss, sai);
+
+                                ss.forEach((sstable) ->
+                                           {
+                                               Set<StorageAttachedIndex> toBuild = sstables.get(sstable);
+                                               if (toBuild == null) sstables.put(sstable, (toBuild = new HashSet<>()));
+                                               toBuild.add(sai);
+                                           });
+                            });
+
+            return sstables;
+        }
+
+        @Override
+        public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstablesToRebuild, boolean isFullRebuild)
+        {
+            StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+            NavigableMap<SSTableReader, Set<StorageAttachedIndex>> sstables = prepareSSTablesToBuild(group, indexes, sstablesToRebuild, isFullRebuild);
+            return new StorageAttachedIndexBuilder(group, sstables, isFullRebuild, false);
+        }
+
+        @Override
+        public List<SecondaryIndexBuilder> getParallelIndexBuildTasks(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstablesToRebuild, boolean isFullRebuild)
+        {
+            StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(cfs);
+            NavigableMap<SSTableReader, Set<StorageAttachedIndex>> sstables = prepareSSTablesToBuild(indexGroup, indexes, sstablesToRebuild, isFullRebuild);
+
+            List<List<SSTableReader>> groups = groupBySize(new ArrayList<>(sstables.keySet()), DatabaseDescriptor.getConcurrentCompactors());
+            List<SecondaryIndexBuilder> builders = new ArrayList<>();
 
-    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+            for (List<SSTableReader> group : groups)
+            {
+                SortedMap<SSTableReader, Set<StorageAttachedIndex>> current = new TreeMap<>(Comparator.comparing(sstable -> sstable.descriptor.id));
+                group.forEach(sstable -> current.put(sstable, sstables.get(sstable)));
+
+                builders.add(new StorageAttachedIndexBuilder(indexGroup, current, isFullRebuild, false));
+            }
+
+            logger.info("Creating {} parallel index builds over {} total sstables for {}...", builders.size(), sstables.size(), cfs.metadata());
 
-    public static final String TERM_OVERSIZE_MESSAGE = "Term in column '%s' for key '%s' is too large and cannot be indexed. (term size: %s)";
+            return builders;
+        }
+    }
 
     // Used to build indexes on newly added SSTables:
     private static final StorageAttachedIndexBuildingSupport INDEX_BUILDER_SUPPORT = new StorageAttachedIndexBuildingSupport();
 
-    private static final Set<String> VALID_OPTIONS = ImmutableSet.of(IndexTarget.TARGET_OPTION_NAME,
+    private static final Set<String> VALID_OPTIONS = ImmutableSet.of(NonTokenizingOptions.CASE_SENSITIVE,
+                                                                     NonTokenizingOptions.NORMALIZE,
+                                                                     NonTokenizingOptions.ASCII,
+                                                                     // For now, we leave this for backward compatibility even though it's not used
+                                                                     IndexContext.ENABLE_SEGMENT_COMPACTION_OPTION_NAME,
+                                                                     IndexTarget.TARGET_OPTION_NAME,
                                                                      IndexTarget.CUSTOM_INDEX_OPTION_NAME,
+                                                                     IndexWriterConfig.POSTING_LIST_LVL_MIN_LEAVES,
+                                                                     IndexWriterConfig.POSTING_LIST_LVL_SKIP_OPTION,
                                                                      IndexWriterConfig.MAXIMUM_NODE_CONNECTIONS,
                                                                      IndexWriterConfig.CONSTRUCTION_BEAM_WIDTH,
+                                                                     IndexWriterConfig.NEIGHBORHOOD_OVERFLOW,
+                                                                     IndexWriterConfig.ALPHA,
+                                                                     IndexWriterConfig.ENABLE_HIERARCHY,
                                                                      IndexWriterConfig.SIMILARITY_FUNCTION,
+                                                                     IndexWriterConfig.SOURCE_MODEL,
                                                                      IndexWriterConfig.OPTIMIZE_FOR,
-                                                                     NonTokenizingOptions.CASE_SENSITIVE,
-                                                                     NonTokenizingOptions.NORMALIZE,
-                                                                     NonTokenizingOptions.ASCII);
+                                                                     LuceneAnalyzer.INDEX_ANALYZER,
+                                                                     LuceneAnalyzer.QUERY_ANALYZER,
+                                                                     AnalyzerEqOperatorSupport.OPTION);
 
+    // this does not include vectors because each Vector declaration is a separate type instance
     public static final Set<CQL3Type> SUPPORTED_TYPES = ImmutableSet.of(CQL3Type.Native.ASCII, CQL3Type.Native.BIGINT, CQL3Type.Native.DATE,
                                                                         CQL3Type.Native.DOUBLE, CQL3Type.Native.FLOAT, CQL3Type.Native.INT,
                                                                         CQL3Type.Native.SMALLINT, CQL3Type.Native.TEXT, CQL3Type.Native.TIME,
@@ -170,44 +227,33 @@ public class StorageAttachedIndex implements Index
             ImmutableSet.of(OrderPreservingPartitioner.class, LocalPartitioner.class, ByteOrderedPartitioner.class, RandomPartitioner.class);
 
     private final ColumnFamilyStore baseCfs;
-    private final IndexMetadata indexMetadata;
-    private final IndexTermType indexTermType;
-    private final IndexIdentifier indexIdentifier;
-    private final IndexViewManager viewManager;
-    private final ColumnQueryMetrics columnQueryMetrics;
-    private final IndexWriterConfig indexWriterConfig;
-    @Nullable private final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
-    private final PrimaryKey.Factory primaryKeyFactory;
-    private final MemtableIndexManager memtableIndexManager;
-    private final IndexMetrics indexMetrics;
-    private final MaxThreshold maxTermSizeGuardrail;
-
-    // Tracks whether we've started the index build on initialization.
-    private volatile boolean initBuildStarted = false;
+    private final IndexMetadata config;
+    private final IndexContext indexContext;
+
+    // Tracks whether or not we've started the index build on initialization.
+    private volatile boolean canFlushFromMemtableIndex = false;
 
     // Tracks whether the index has been invalidated due to removal, a table drop, etc.
     private volatile boolean valid = true;
 
-    public StorageAttachedIndex(ColumnFamilyStore baseCfs, IndexMetadata indexMetadata)
+    /**
+     * Called via reflection from SecondaryIndexManager
+     */
+    public StorageAttachedIndex(ColumnFamilyStore baseCfs, IndexMetadata config)
     {
         this.baseCfs = baseCfs;
-        this.indexMetadata = indexMetadata;
+        this.config = config;
         TableMetadata tableMetadata = baseCfs.metadata();
-        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(tableMetadata, indexMetadata);
-        indexTermType = IndexTermType.create(target.left, tableMetadata.partitionKeyColumns(), target.right);
-        indexIdentifier = new IndexIdentifier(baseCfs.getKeyspaceName(), baseCfs.getTableName(), indexMetadata.name);
-        primaryKeyFactory = new PrimaryKey.Factory(tableMetadata.partitioner, tableMetadata.comparator);
-        indexWriterConfig = IndexWriterConfig.fromOptions(indexMetadata.name, indexTermType, indexMetadata.options);
-        viewManager = new IndexViewManager(this);
-        columnQueryMetrics = indexTermType.isLiteral() ? new ColumnQueryMetrics.TrieIndexMetrics(indexIdentifier)
-                                                       : new ColumnQueryMetrics.BalancedTreeIndexMetrics(indexIdentifier);
-        analyzerFactory = AbstractAnalyzer.fromOptions(indexTermType, indexMetadata.options);
-        memtableIndexManager = new MemtableIndexManager(this);
-        indexMetrics = new IndexMetrics(this, memtableIndexManager);
-        maxTermSizeGuardrail = indexTermType.isVector()
-                               ? Guardrails.saiVectorTermSize
-                               : (indexTermType.isFrozen() ? Guardrails.saiFrozenTermSize
-                                                           : Guardrails.saiStringTermSize);
+        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(tableMetadata, config);
+        this.indexContext = new IndexContext(tableMetadata.keyspace,
+                                             tableMetadata.name,
+                                             tableMetadata.id,
+                                             tableMetadata.partitionKeyType,
+                                             tableMetadata.comparator,
+                                             target.left,
+                                             target.right,
+                                             config,
+                                             baseCfs);
     }
 
     /**
@@ -255,54 +301,82 @@ public static Map<String, String> validateOptions(Map<String, String> options, T
             throw new InvalidRequestException("Failed to retrieve target column for: " + targetColumn);
         }
 
-        // In order to support different index targets on non-frozen map, ie. KEYS, VALUE, ENTRIES, we need to put index
-        // name as part of index file name instead of column name. We only need to check that the target is different
-        // between indexes. This will only allow indexes in the same column with a different IndexTarget.Type.
-        //
-        // Note that: "metadata.indexes" already includes current index
-        if (metadata.indexes.stream().filter(index -> index.getIndexClassName().equals(StorageAttachedIndex.class.getName()))
-                            .map(index -> TargetParser.parse(metadata, index.options.get(IndexTarget.TARGET_OPTION_NAME)))
-                            .filter(Objects::nonNull).filter(t -> t.equals(target)).count() > 1)
+        // Check for duplicate indexes considering both target and analyzer configuration
+        boolean isAnalyzed = AbstractAnalyzer.isAnalyzed(options);
+        long duplicateCount = metadata.indexes.stream()
+                                              .filter(index -> index.getIndexClassName().equals(StorageAttachedIndex.class.getName()))
+                                              .filter(index -> {
+                                                  // Indexes on the same column with different target (KEYS, VALUES, ENTRIES)
+                                                  // are allowed on non-frozen Maps
+                                                  var existingTarget = TargetParser.parse(metadata, index.options.get(IndexTarget.TARGET_OPTION_NAME));
+                                                  if (existingTarget == null || !existingTarget.equals(target))
+                                                      return false;
+                                                  // Also allow different indexes if one is analyzed and the other isn't
+                                                  return isAnalyzed == AbstractAnalyzer.isAnalyzed(index.options);
+                                              })
+                                              .count();
+        // >1 because "metadata.indexes" already includes current index
+        if (duplicateCount > 1)
+            throw new InvalidRequestException(String.format("Cannot create duplicate storage-attached index on column: %s", target.left));
+
+        // Analyzer is not supported against PK columns
+        if (isAnalyzed)
         {
-            throw new InvalidRequestException("Cannot create more than one storage-attached index on the same column: " + target.left);
+            for (ColumnMetadata column : metadata.primaryKeyColumns())
+            {
+                if (column.name.equals(target.left.name))
+                    logger.warn("Schema contains an invalid index analyzer on primary key column, allowed for backwards compatibility: " + target.left);
+            }
         }
 
-        Map<String, String> analysisOptions = AbstractAnalyzer.getAnalyzerOptions(options);
-        if (target.left.isPrimaryKeyColumn() && !analysisOptions.isEmpty())
+        AbstractType<?> type = TypeUtil.cellValueType(target.left, target.right);
+
+        // Validate analyzers by building them
+        try (AbstractAnalyzer.AnalyzerFactory analyzerFactory = AbstractAnalyzer.fromOptions(targetColumn, type, options))
         {
-            throw new InvalidRequestException(ANALYSIS_ON_KEY_COLUMNS_MESSAGE + new CqlBuilder().append(analysisOptions));
+            if (AbstractAnalyzer.hasQueryAnalyzer(options))
+                AbstractAnalyzer.fromOptionsQueryAnalyzer(type, options).close();
+            else if (analyzerFactory.isNGram())
+                ClientWarn.instance.warn(NGRAM_WITHOUT_QUERY_ANALYZER_WARNING);
         }
 
-        IndexTermType indexTermType = IndexTermType.create(target.left, metadata.partitionKeyColumns(), target.right);
-        AbstractAnalyzer.fromOptions(indexTermType, analysisOptions);
-        IndexWriterConfig config = IndexWriterConfig.fromOptions(null, indexTermType, options);
+        var config = IndexWriterConfig.fromOptions(null, type, options);
 
-        // If we are indexing map entries we need to validate the subtypes
-        if (indexTermType.isComposite())
+        // If we are indexing map entries we need to validate the sub-types
+        if (TypeUtil.isComposite(type))
         {
-            for (IndexTermType subType : indexTermType.subTypes())
+            for (AbstractType<?> subType : type.subTypes())
             {
-                if (!SUPPORTED_TYPES.contains(subType.asCQL3Type()) && !subType.isFrozen())
-                    throw new InvalidRequestException("Unsupported type: " + subType.asCQL3Type());
+                if (!SUPPORTED_TYPES.contains(subType.asCQL3Type()) && !TypeUtil.isFrozen(subType))
+                    throw new InvalidRequestException("Unsupported composite type for SAI: " + subType.asCQL3Type());
             }
         }
-        else if (!SUPPORTED_TYPES.contains(indexTermType.asCQL3Type()) && !indexTermType.isFrozen())
+        else if (type.isVector())
         {
-            throw new InvalidRequestException("Unsupported type: " + indexTermType.asCQL3Type());
+            if (type.valueLengthIfFixed() == 4 && config.getSimilarityFunction() == VectorSimilarityFunction.COSINE)
+                throw new InvalidRequestException("Cosine similarity is not supported for single-dimension vectors");
+
+            // vectors of fixed length types are fixed length too, so we can reject the index creation
+            // if that fixed length is over the max term size for vectors
+            if (type.isValueLengthFixed() && IndexContext.MAX_VECTOR_TERM_SIZE < type.valueLengthIfFixed())
+            {
+                AbstractType<?> elementType = ((VectorType<?>) type).elementType;
+                var error = String.format("Vector index created with %s will produce terms of %s, " +
+                                          "exceeding the max vector term size of %s. " +
+                                          "That sets an implicit limit of %d dimensions for %s vectors.",
+                                          type.asCQL3Type(),
+                                          FBUtilities.prettyPrintMemory(type.valueLengthIfFixed()),
+                                          FBUtilities.prettyPrintMemory(IndexContext.MAX_VECTOR_TERM_SIZE),
+                                          IndexContext.MAX_VECTOR_TERM_SIZE / elementType.valueLengthIfFixed(),
+                                          elementType.asCQL3Type());
+                // VSTODO until we can safely differentiate client and system requests, we can only log here
+                // Ticket for this: https://github.com/riptano/VECTOR-SEARCH/issues/85
+                logger.warn(error);
+            }
         }
-        // If this is a vector type we need to validate it for the current vector index constraints
-        else if (indexTermType.isVector())
+        else if (!SUPPORTED_TYPES.contains(type.asCQL3Type()) && !TypeUtil.isFrozen(type))
         {
-            if (!(indexTermType.vectorElementType() instanceof FloatType))
-                throw new InvalidRequestException(VECTOR_NON_FLOAT_ERROR);
-
-            if (indexTermType.vectorDimension() == 1 && config.getSimilarityFunction() == VectorSimilarityFunction.COSINE)
-                throw new InvalidRequestException(VECTOR_1_DIMENSION_COSINE_ERROR);
-
-            if (DatabaseDescriptor.getRawConfig().data_file_directories.length > 1)
-                throw new InvalidRequestException(VECTOR_MULTIPLE_DATA_DIRECTORY_ERROR);
-
-            ClientWarn.instance.warn(VECTOR_USAGE_WARNING);
+            throw new InvalidRequestException("Unsupported type for SAI: " + type.asCQL3Type());
         }
 
         return Collections.emptyMap();
@@ -324,236 +398,95 @@ public void unregister(IndexRegistry registry)
     @Override
     public IndexMetadata getIndexMetadata()
     {
-        return indexMetadata;
-    }
-
-    @Override
-    public Callable<?> getInitializationTask()
-    {
-        // New storage-attached indexes will be available for queries after on disk index data are built.
-        // Memtable data will be indexed via flushing triggered by schema change
-        // We only want to validate the index files if we are starting up
-        IndexValidation validation = StorageService.instance.isStarting() ? IndexValidation.HEADER_FOOTER : IndexValidation.NONE;
-        return () -> startInitialBuild(baseCfs, validation).get();
-    }
-
-    @Override
-    public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata)
-    {
-        return null;
-    }
-
-    @Override
-    public Callable<?> getBlockingFlushTask()
-    {
-        return null; // storage-attached indexes are flushed alongside memtable
-    }
-
-    @Override
-    public Callable<?> getInvalidateTask()
-    {
-        return () ->
-        {
-            // mark index as invalid, in-progress SSTableIndexWriters will abort
-            valid = false;
-
-            // in case of dropping table, SSTable indexes should already been removed by SSTableListChangedNotification.
-            Set<Component> toRemove = getComponents();
-            for (SSTableIndex sstableIndex : view().getIndexes())
-                sstableIndex.getSSTable().unregisterComponents(toRemove, baseCfs.getTracker());
-
-            viewManager.invalidate();
-            if (analyzerFactory != null)
-                analyzerFactory.close();
-            columnQueryMetrics.release();
-            memtableIndexManager.invalidate();
-            indexMetrics.release();
-            return null;
-        };
-    }
-
-    @Override
-    public Callable<?> getPreJoinTask(boolean hadBootstrap)
-    {
-        /*
-         * During bootstrap, streamed SSTable are already built for existing indexes via {@link StorageAttachedIndexBuildingSupport}
-         * from {@link org.apache.cassandra.streaming.StreamReceiveTask.OnCompletionRunnable}.
-         *
-         * For indexes created during bootstrapping, we don't have to block bootstrap for them.
-         */
-
-        return this::startPreJoinTask;
+        return config;
     }
 
     @Override
-    public Callable<?> getTruncateTask(long truncatedAt)
-    {
-        /*
-         * index files will be removed as part of base sstable lifecycle in {@link LogTransaction#delete(java.io.File)}
-         * asynchronously, but we need to mark the index queryable because if the truncation is during the initial
-         * build of the index it won't get marked queryable by the build.
-         */
-        return () -> {
-            logger.info(indexIdentifier.logMessage("Making index queryable during table truncation"));
-            baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
-            return null;
-        };
-    }
-
-    @Override
-    public boolean shouldBuildBlocking()
-    {
-        return true;
-    }
-
-    @Override
-    public boolean isSSTableAttached()
-    {
-        return true;
-    }
-
-    @Override
-    public Optional<ColumnFamilyStore> getBackingTable()
-    {
-        return Optional.empty();
-    }
-
-    @Override
-    public boolean dependsOn(ColumnMetadata column)
-    {
-        return indexTermType.dependsOn(column);
-    }
-
-    @Override
-    public boolean supportsExpression(ColumnMetadata column, Operator operator)
-    {
-        return dependsOn(column) && indexTermType.supports(operator);
-    }
-
-    @Override
-    public boolean filtersMultipleContains()
+    public boolean shouldSkipInitialization()
     {
+        // SAI performs partial initialization so it must always execute it; the actual index build is then still skipped
+        // if IndexBuildDecider.instance.onInitialBuild().skipped() is true.
         return false;
     }
 
     @Override
-    public AbstractType<?> customExpressionValueType()
-    {
-        return null;
-    }
-
-    @Override
-    public RowFilter getPostIndexQueryFilter(RowFilter filter)
-    {
-        // it should be executed from the SAI query plan, this is only used by the singleton index query plan
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Comparator<ByteBuffer> getPostQueryOrdering(Restriction restriction, QueryOptions options)
+    public Callable<?> getInitializationTask()
     {
-        // For now, only support ANN
-        assert restriction instanceof SingleColumnRestriction.AnnRestriction;
-
-        Preconditions.checkState(indexTermType.isVector());
-
-        SingleColumnRestriction.AnnRestriction annRestriction = (SingleColumnRestriction.AnnRestriction) restriction;
-        VectorSimilarityFunction function = indexWriterConfig.getSimilarityFunction();
-
-        float[] target = indexTermType.decomposeVector(annRestriction.value(options).duplicate());
-
-        return (leftBuf, rightBuf) -> {
-            float[] left = indexTermType.decomposeVector(leftBuf.duplicate());
-            double scoreLeft = function.compare(left, target);
-
-            float[] right = indexTermType.decomposeVector(rightBuf.duplicate());
-            double scoreRight = function.compare(right, target);
-            return Double.compare(scoreRight, scoreLeft); // descending order
-        };
+        IndexBuildDecider.Decision decision = IndexBuildDecider.instance.onInitialBuild();
+        // New storage-attached indexes will be available for queries after on disk index data are built.
+        // Memtable data will be indexed via flushing triggered by schema change
+        // We only want to validate the index files if we are starting up
+        return () -> startInitialBuild(baseCfs, StorageService.instance.isStarting(), decision.skipped()).get();
     }
 
-    @Override
-    public void validate(ReadCommand command) throws InvalidRequestException
+    private Future<?> startInitialBuild(ColumnFamilyStore baseCfs, boolean validate, boolean skipIndexBuild)
     {
-        if (!indexTermType.isVector())
-            return;
+        if (skipIndexBuild)
+        {
+            logger.info("Skipping initialization task for {}.{} after flushing memtable", baseCfs.metadata(), indexContext.getIndexName());
+            // Force another flush to make sure on disk index is generated for memtable data before marking it queryable.
+            // In case of offline scrub, there is no live memtables.
+            if (!baseCfs.getTracker().getView().liveMemtables.isEmpty())
+                baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED);
+
+            // even though we're skipping the index build, we still want to add any initial sstables that have indexes into SAI.
+            // Index will be queryable if all existing sstables have index files; otherwise non-queryable
+            Set<SSTableReader> sstables = baseCfs.getLiveSSTables();
+            StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(baseCfs);
+            indexGroup.onSSTableChanged(Collections.emptyList(), sstables, Collections.singleton(this), validate);
 
-        // to avoid overflow of the vector graph internal data structure and avoid OOM when filtering top-k
-        if (command.limits().count() > MAX_TOP_K)
-            throw new InvalidRequestException(String.format(ANN_LIMIT_ERROR, MAX_TOP_K, command.limits().count()));
-    }
+            // From now on, all memtable will have attached memtable index. It is now safe to flush indexes directly from flushing Memtables.
+            canFlushFromMemtableIndex = true;
+            return ImmediateFuture.success(null);
+        }
 
-    @Override
-    public long getEstimatedResultRows()
-    {
-        throw new UnsupportedOperationException("Use StorageAttachedIndexQueryPlan#getEstimatedResultRows() instead.");
-    }
+        if (baseCfs.indexManager.isIndexQueryable(this))
+        {
+            logger.debug(indexContext.logMessage("Skipping validation and building in initialization task, as pre-join has already made the storage attached index queryable..."));
+            canFlushFromMemtableIndex = true;
+            return ImmediateFuture.success(null);
+        }
 
-    @Override
-    public boolean isQueryable(Status status)
-    {
-        // consider unknown status as queryable, because gossip may not be up-to-date for newly joining nodes.
-        return status == Status.BUILD_SUCCEEDED || status == Status.UNKNOWN;
-    }
+        // stop in-progress compaction tasks to prevent compacted sstable not being index.
+        logger.debug(indexContext.logMessage("Stopping active compactions to make sure all sstables are indexed after initial build."));
+        CompactionManager.instance.interruptCompactionFor(Collections.singleton(baseCfs.metadata()),
+                                                          OperationType.REWRITES_SSTABLES,
+                                                          Predicates.alwaysTrue(),
+                                                          true,
+                                                          TableOperation.StopTrigger.INDEX_BUILD);
 
-    @Override
-    public void validate(PartitionUpdate update, ClientState state) throws InvalidRequestException
-    {
-        DecoratedKey key = update.partitionKey();
+        // Force another flush to make sure on disk index is generated for memtable data before marking it queryable.
+        // In case of offline scrub, there is no live memtables.
+        if (!baseCfs.getTracker().getView().liveMemtables.isEmpty())
+        {
+            baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED);
+        }
 
-        if (indexTermType.columnMetadata().isStatic())
-            validateTermSizeForRow(key, update.staticRow(), true, state);
-        else
-            for (Row row : update)
-                validateTermSizeForRow(key, row, true, state);
-    }
+        // From now on, all memtable will have attached memtable index. It is now safe to flush indexes directly from flushing Memtables.
+        canFlushFromMemtableIndex = true;
 
-    @Override
-    public Searcher searcherFor(ReadCommand command) throws InvalidRequestException
-    {
-        // searchers should be created from the query plan, this is only used by the singleton index query plan
-        throw new UnsupportedOperationException();
-    }
+        StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(baseCfs);
+        List<SSTableReader> nonIndexed = findNonIndexedSSTables(baseCfs, indexGroup, validate);
 
-    @Override
-    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker)
-    {
-        // flush observers should be created from the index group, this is only used by the singleton index group
-        throw new UnsupportedOperationException("Storage-attached index flush observers should never be created directly.");
-    }
+        if (nonIndexed.isEmpty())
+        {
+            return ImmediateFuture.success(null);
+        }
 
-    @Override
-    public Set<Component> getComponents()
-    {
-        return Version.LATEST.onDiskFormat()
-                             .perColumnIndexComponents(indexTermType)
-                             .stream()
-                             .map(c -> Version.LATEST.makePerIndexComponent(c, indexIdentifier))
-                             .collect(Collectors.toSet());
-    }
+        // split sorted sstables into groups with similar size and build each group in separate compaction thread
+        List<List<SSTableReader>> groups = groupBySize(nonIndexed, DatabaseDescriptor.getConcurrentCompactors());
+        List<Future<?>> futures = new ArrayList<>();
 
-    @Override
-    public Indexer indexerFor(DecoratedKey key,
-                              RegularAndStaticColumns columns,
-                              long nowInSec,
-                              WriteContext writeContext,
-                              IndexTransaction.Type transactionType,
-                              Memtable memtable)
-    {
-        if (transactionType == IndexTransaction.Type.UPDATE)
+        for (List<SSTableReader> group : groups)
         {
-            return new UpdateIndexer(key, memtable, writeContext);
-        }
+            SortedMap<SSTableReader, Set<StorageAttachedIndex>> current = new TreeMap<>(SSTableReader.idComparator);
+            group.forEach(sstable -> current.put(sstable, Collections.singleton(this)));
 
-        // we are only interested in the data from Memtable
-        // everything else is going to be handled by SSTableWriter observers
-        return null;
-    }
+            futures.add(CompactionManager.instance.submitIndexBuild(new StorageAttachedIndexBuilder(indexGroup, current, false, true)));
+        }
 
-    @Override
-    public IndexBuildingSupport getBuildTaskSupport()
-    {
-        return INDEX_BUILDER_SUPPORT;
+        logger.info(indexContext.logMessage("Submitting {} parallel initial index builds over {} total sstables..."), futures.size(), nonIndexed.size());
+        return FutureCombiner.allOf(futures);
     }
 
     /**
@@ -592,338 +525,256 @@ public static List<List<SSTableReader>> groupBySize(List<SSTableReader> toRebuil
         return groups;
     }
 
-    /**
-     * @return A set of SSTables which have attached to them invalid index components.
-     */
-    public Collection<SSTableContext> onSSTableChanged(Collection<SSTableReader> oldSSTables, Collection<SSTableContext> newSSTables, IndexValidation validation)
+    @Override
+    public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata)
     {
-        return viewManager.update(oldSSTables, newSSTables, validation);
+        return null;
     }
 
-    public void drop(Collection<SSTableReader> sstablesToRebuild)
+    @Override
+    public Callable<?> getBlockingFlushTask()
     {
-        viewManager.drop(sstablesToRebuild);
+        return null; // storage-attached indexes are flushed alongside memtable
     }
 
-    public MemtableIndexManager memtableIndexManager()
+    @Override
+    public Callable<?> getInvalidateTask()
     {
-        return memtableIndexManager;
-    }
+        return () ->
+        {
+            // mark index as invalid, in-progress SSTableIndexWriters will abort
+            valid = false;
 
-    public View view()
-    {
-        return viewManager.view();
-    }
+            // in case of dropping table, SSTable indexes should already been removed by SSTableListChangedNotification.
+            for (SSTableIndex sstableIndex : indexContext.getView().getIndexes())
+            {
+                var components = sstableIndex.usedPerIndexComponents();
+                sstableIndex.getSSTable().unregisterComponents(components.allAsCustomComponents(), baseCfs.getTracker());
+            }
 
-    public IndexTermType termType()
-    {
-        return indexTermType;
+            indexContext.invalidate(true);
+            return null;
+        };
     }
 
-    public IndexIdentifier identifier()
+    @Override
+    public Callable<?> getUnloadTask()
     {
-        return indexIdentifier;
+        return () ->
+        {
+            // mark index as invalid, in-progress SSTableIndexWriters will abort
+            valid = false;
+
+            indexContext.invalidate(false);
+            return null;
+        };
     }
 
-    public PrimaryKey.Factory keyFactory()
+    @Override
+    public Callable<?> getPreJoinTask(boolean hadBootstrap)
     {
-        return primaryKeyFactory;
+        /*
+         * During bootstrap, streamed SSTable are already built for existing indexes via {@link StorageAttachedIndexBuildingSupport}
+         * from {@link org.apache.cassandra.streaming.StreamReceiveTask.OnCompletionRunnable}.
+         *
+         * For indexes created during bootstrapping, we don't have to block bootstrap for them.
+         */
+
+        return this::startPreJoinTask;
     }
 
     @VisibleForTesting
-    public ColumnFamilyStore baseCfs()
+    public boolean canFlushFromMemtableIndex()
     {
-        return baseCfs;
+        return canFlushFromMemtableIndex;
     }
 
-    public IndexWriterConfig indexWriterConfig()
+    public BooleanSupplier isIndexValid()
     {
-        return indexWriterConfig;
+        return () -> valid;
     }
 
-    public boolean hasAnalyzer()
+    @SuppressWarnings("SameReturnValue")
+    private Future<?> startPreJoinTask()
     {
-        return analyzerFactory != null;
-    }
+        try
+        {
+            if (baseCfs.indexManager.isIndexQueryable(this))
+            {
+                logger.debug(indexContext.logMessage("Skipping validation in pre-join task, as the initialization task has already made the index queryable..."));
+                baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+                return null;
+            }
 
-    /**
-     * Returns an {@link AbstractAnalyzer} for use by write and query paths to transform
-     * literal values.
-     */
-    public AbstractAnalyzer analyzer()
-    {
-        assert analyzerFactory != null : "Index does not support string analysis";
-        return analyzerFactory.create();
-    }
+            StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(baseCfs);
+            Collection<SSTableReader> nonIndexed = findNonIndexedSSTables(baseCfs, group, true);
 
-    public IndexMetrics indexMetrics()
-    {
-        return indexMetrics;
-    }
+            if (nonIndexed.isEmpty())
+            {
+                // If the index is complete, mark it queryable before the node starts accepting requests:
+                baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+            }
+        }
+        catch (Throwable t)
+        {
+            logger.error(indexContext.logMessage("Failed in pre-join task!"), t);
+        }
 
-    public ColumnQueryMetrics columnQueryMetrics()
-    {
-        return columnQueryMetrics;
+        return null;
     }
 
-    public boolean isInitBuildStarted()
+    @Override
+    public Callable<?> getTruncateTask(long truncatedAt)
     {
-        return initBuildStarted;
+        /*
+         * index files will be removed as part of base sstable lifecycle in {@link LogTransaction#delete(java.io.File)}
+         * asynchronously, but we need to mark the index queryable because if the truncation is during the initial
+         * build of the index it won't get marked queryable by the build.
+         */
+        return () -> {
+            logger.info(indexContext.logMessage("Making index queryable during table truncation"));
+            baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+            return null;
+        };
     }
 
-    public BooleanSupplier isIndexValid()
+    @Override
+    public boolean shouldBuildBlocking()
     {
-        return () -> valid;
+        return true;
     }
 
-    public boolean hasClustering()
+    @Override
+    public boolean isSSTableAttached()
     {
-        return baseCfs.getComparator().size() > 0;
+        return true;
     }
 
-    /**
-     * @return the number of indexed rows in this index (aka. a pair of term and rowId)
-     */
-    public long cellCount()
+    @Override
+    public Optional<ColumnFamilyStore> getBackingTable()
     {
-        return view().getIndexes()
-                     .stream()
-                     .mapToLong(SSTableIndex::getRowCount)
-                     .sum();
+        return Optional.empty();
     }
 
-    /**
-     * @return total number of per-index open files
-     */
-    public int openPerColumnIndexFiles()
+    @Override
+    public boolean dependsOn(ColumnMetadata column)
     {
-        return viewManager.view().size() * Version.LATEST.onDiskFormat().openFilesPerColumnIndex();
+        return indexContext.getDefinition().compareTo(column) == 0;
     }
 
-    /**
-     * @return the total size (in bytes) of per-column index components
-     */
-    public long diskUsage()
+    @Override
+    public boolean supportsExpression(ColumnMetadata column, Operator operator)
     {
-        return view().getIndexes()
-                     .stream()
-                     .mapToLong(SSTableIndex::sizeOfPerColumnComponents)
-                     .sum();
+        return dependsOn(column) && indexContext.supports(operator);
     }
 
-    /**
-     * @return the total memory usage (in bytes) of per-column index on-disk data structure
-     */
-    public long indexFileCacheSize()
+    @Override
+    public boolean filtersMultipleContains()
     {
-        return view().getIndexes()
-                     .stream()
-                     .mapToLong(SSTableIndex::indexFileCacheSize)
-                     .sum();
+        return false;
     }
 
-    /**
-     * Removes this index from the {@code SecondaryIndexManager}'s set of queryable indexes.
-     */
-    public void makeIndexNonQueryable()
+    @Override
+    public AbstractType<?> customExpressionValueType()
     {
-        baseCfs.indexManager.makeIndexNonQueryable(this, Status.BUILD_FAILED);
-        logger.warn(indexIdentifier.logMessage("Storage-attached index is no longer queryable. Please restart this node to repair it."));
+        return null;
     }
 
-    /**
-     * Validate maximum term size for given row
-     */
-    public void validateTermSizeForRow(DecoratedKey key, Row row, boolean isClientMutation, ClientState state)
+    @Override
+    public Optional<Analyzer> getIndexAnalyzer()
     {
-        AbstractAnalyzer analyzer = hasAnalyzer() ? analyzer() : null;
-        if (indexTermType.isNonFrozenCollection())
-        {
-            Iterator<ByteBuffer> bufferIterator = indexTermType.valuesOf(row, FBUtilities.nowInSeconds());
-            while (bufferIterator != null && bufferIterator.hasNext())
-                validateTermSizeForCell(analyzer, key, bufferIterator.next(), isClientMutation, state);
-        }
-        else
-        {
-            ByteBuffer value = indexTermType.valueOf(key, row, FBUtilities.nowInSeconds());
-            validateTermSizeForCell(analyzer, key, value, isClientMutation, state);
-        }
+        return indexContext.isAnalyzed()
+               ? Optional.of(value -> analyze(indexContext.getAnalyzerFactory(), value))
+               : Optional.empty();
     }
 
-    private void validateTermSizeForCell(AbstractAnalyzer analyzer, DecoratedKey key, @Nullable ByteBuffer cellBuffer, boolean isClientMutation, ClientState state)
+    @Override
+    public Optional<Analyzer> getQueryAnalyzer()
     {
-        if (cellBuffer == null || cellBuffer.remaining() == 0)
-            return;
-
-        // analyzer should not return terms that are larger than the origin value.
-        if (!maxTermSizeGuardrail.warnsOn(cellBuffer.remaining(), null))
-            return;
-
-        if (analyzer != null)
-        {
-            analyzer.reset(cellBuffer.duplicate());
-            while (analyzer.hasNext())
-                validateTermSize(key, analyzer.next(), isClientMutation, state);
-        }
-        else
-        {
-            validateTermSize(key, cellBuffer.duplicate(), isClientMutation, state);
-        }
+        return indexContext.isAnalyzed()
+               ? Optional.of(value -> analyze(indexContext.getQueryAnalyzerFactory(), value))
+               : Optional.empty();
     }
 
-    /**
-     * @return true if the size of the given term is below the maximum term size, false otherwise
-     * 
-     * @throws GuardrailViolatedException if a client mutation contains a term that breaches the failure threshold
-     */
-    public boolean validateTermSize(DecoratedKey key, ByteBuffer term, boolean isClientMutation, ClientState state)
+    private static List<ByteBuffer> analyze(AbstractAnalyzer.AnalyzerFactory factory, ByteBuffer value)
     {
-        if (isClientMutation)
+        List<ByteBuffer> tokens = new ArrayList<>();
+        AbstractAnalyzer analyzer = factory.create();
+        try
         {
-            maxTermSizeGuardrail.guard(term.remaining(), indexTermType.columnName(), false, state);
-            return true;
+            analyzer.reset(value.duplicate());
+            while (analyzer.hasNext())
+                tokens.add(analyzer.next());
         }
-
-        if (maxTermSizeGuardrail.failsOn(term.remaining(), state))
+        finally
         {
-            String message = indexIdentifier.logMessage(String.format(TERM_OVERSIZE_MESSAGE,
-                                                                      indexTermType.columnName(),
-                                                                      key,
-                                                                      FBUtilities.prettyPrintMemory(term.remaining())));
-            noSpamLogger.warn(message);
-            return false;
+            analyzer.end();
         }
-
-        return true;
+        return tokens;
     }
 
     @Override
-    public String toString()
+    public RowFilter getPostIndexQueryFilter(RowFilter filter)
     {
-        return indexIdentifier.toString();
+        // it should be executed from the SAI query plan, this is only used by the singleton index query plan
+        throw new UnsupportedOperationException();
     }
 
     @Override
-    public boolean equals(Object obj)
+    public void validate(ReadCommand command) throws InvalidRequestException
     {
-        if (obj == this)
-            return true;
-
-        if (!(obj instanceof StorageAttachedIndex))
-            return false;
+        var indexQueryPlan = command.indexQueryPlan();
+        if (indexQueryPlan == null || !indexQueryPlan.isTopK())
+            return;
 
-        StorageAttachedIndex other = (StorageAttachedIndex) obj;
+        // to avoid overflow HNSW internal data structure and avoid OOM when filtering top-k
+        if (command.limits().isUnlimited() || command.limits().count() > MAX_TOP_K)
+            throw new InvalidRequestException(String.format("SAI based ORDER BY clause requires a LIMIT that is not greater than %s. LIMIT was %s",
+                                                            MAX_TOP_K, command.limits().isUnlimited() ? "NO LIMIT" : command.limits().count()));
 
-        return Objects.equals(indexTermType, other.indexTermType) &&
-               Objects.equals(indexMetadata, other.indexMetadata) &&
-               Objects.equals(baseCfs.getComparator(), other.baseCfs.getComparator());
+        indexContext.validate(command.rowFilter());
     }
 
     @Override
-    public int hashCode()
+    public long getEstimatedResultRows()
     {
-        return Objects.hash(indexTermType, indexMetadata, baseCfs.getComparator());
+        throw new UnsupportedOperationException("Use StorageAttachedIndexQueryPlan#getEstimatedResultRows() instead.");
     }
 
-    private Future<?> startInitialBuild(ColumnFamilyStore baseCfs, IndexValidation validation)
+    @Override
+    public boolean isQueryable(Status status)
     {
-        if (baseCfs.indexManager.isIndexQueryable(this))
-        {
-            logger.debug(indexIdentifier.logMessage("Skipping validation and building in initialization task, as pre-join has already made the storage-attached index queryable..."));
-            initBuildStarted = true;
-            return ImmediateFuture.success(null);
-        }
-
-        // stop in-progress compaction tasks to prevent compacted sstable not being indexed.
-        logger.debug(indexIdentifier.logMessage("Stopping active compactions to make sure all sstables are indexed after initial build."));
-        CompactionManager.instance.interruptCompactionFor(Collections.singleton(baseCfs.metadata()),
-                                                          ssTableReader -> true,
-                                                          true);
-
-        // Force another flush to make sure on disk index is generated for memtable data before marking it queryable.
-        // In the case of offline scrub, there are no live memtables.
-        if (!baseCfs.getTracker().getView().liveMemtables.isEmpty())
-        {
-            baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED);
-        }
-
-        // It is now safe to flush indexes directly from flushing Memtables.
-        initBuildStarted = true;
-
-        StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(baseCfs);
-
-        assert indexGroup != null : "Index group does not exist for table " + baseCfs.keyspace + '.' + baseCfs.name;
-
-        List<SSTableReader> nonIndexed = findNonIndexedSSTables(baseCfs, indexGroup, validation);
-
-        if (nonIndexed.isEmpty())
-            return ImmediateFuture.success(null);
-
-        // split sorted sstables into groups with similar size and build each group in separate compaction thread
-        List<List<SSTableReader>> groups = groupBySize(nonIndexed, DatabaseDescriptor.getConcurrentIndexBuilders());
-        List<Future<?>> futures = new ArrayList<>();
-
-        for (List<SSTableReader> group : groups)
-        {
-            SortedMap<SSTableReader, Set<StorageAttachedIndex>> current = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR));
-            group.forEach(sstable -> current.put(sstable, Collections.singleton(this)));
-
-            futures.add(CompactionManager.instance.submitIndexBuild(new StorageAttachedIndexBuilder(indexGroup, current, false, true)));
-        }
-
-        logger.info(indexIdentifier.logMessage("Submitting {} parallel initial index builds over {} total sstables..."), futures.size(), nonIndexed.size());
-        return FutureCombiner.allOf(futures);
+        return !CassandraRelevantProperties.SAI_INDEX_READS_DISABLED.getBoolean()
+            && (status == Status.BUILD_SUCCEEDED || status == Status.UNKNOWN);
     }
 
-    @SuppressWarnings("SameReturnValue")
-    private Future<?> startPreJoinTask()
+    @Override
+    public void validate(PartitionUpdate update, ClientState state) throws InvalidRequestException
     {
-        try
-        {
-            if (baseCfs.indexManager.isIndexQueryable(this))
-            {
-                logger.debug(indexIdentifier.logMessage("Skipping validation in pre-join task, as the initialization task has already made the index queryable..."));
-                baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
-                return null;
-            }
-
-            StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(baseCfs);
-
-            assert indexGroup != null : "Index group does not exist for table";
-
-            Collection<SSTableReader> nonIndexed = findNonIndexedSSTables(baseCfs, indexGroup, IndexValidation.HEADER_FOOTER);
-
-            if (nonIndexed.isEmpty())
-            {
-                // If the index is complete, mark it queryable before the node starts accepting requests:
-                baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
-            }
-        }
-        catch (Throwable t)
-        {
-            logger.error(indexIdentifier.logMessage("Failed in pre-join task!"), t);
-        }
+        if (!VALIDATE_TERMS_AT_COORDINATOR)
+            return;
 
-        return null;
+        DecoratedKey key = update.partitionKey();
+        for (Row row : update.rows())
+            indexContext.validate(key, row);
     }
-
     /**
      * This method is called by the startup tasks to find SSTables that don't have indexes. The method is
      * synchronized so that the view is unchanged between validation and the selection of non-indexed SSTables.
      *
      * @return a list SSTables without attached indexes
      */
-    private synchronized List<SSTableReader> findNonIndexedSSTables(ColumnFamilyStore baseCfs, StorageAttachedIndexGroup group, IndexValidation validation)
+    private synchronized List<SSTableReader> findNonIndexedSSTables(ColumnFamilyStore baseCfs, StorageAttachedIndexGroup group, boolean validate)
     {
         Set<SSTableReader> sstables = baseCfs.getLiveSSTables();
 
         // Initialize the SSTable indexes w/ valid existing components...
         assert group != null : "Missing index group on " + baseCfs.name;
-        group.onSSTableChanged(Collections.emptyList(), sstables, Collections.singleton(this), validation);
+        group.onSSTableChanged(Collections.emptyList(), sstables, Collections.singleton(this), validate);
 
         // ...then identify and rebuild the SSTable indexes that are missing.
         List<SSTableReader> nonIndexed = new ArrayList<>();
-        View view = viewManager.view();
+        View view = indexContext.getView();
 
         for (SSTableReader sstable : sstables)
         {
@@ -931,8 +782,9 @@ private synchronized List<SSTableReader> findNonIndexedSSTables(ColumnFamilyStor
             //   1. The current view does not contain the SSTable
             //   2. The SSTable is not marked compacted
             //   3. The column index does not have a completion marker
-            if (!view.containsSSTable(sstable) && !sstable.isMarkedCompacted() &&
-                !IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexIdentifier))
+            if (!view.containsSSTableIndex(sstable.descriptor)
+                && !sstable.isMarkedCompacted()
+                && !IndexDescriptor.isIndexBuildCompleteOnDisk(sstable, indexContext))
             {
                 nonIndexed.add(sstable);
             }
@@ -941,38 +793,112 @@ private synchronized List<SSTableReader> findNonIndexedSSTables(ColumnFamilyStor
         return nonIndexed;
     }
 
-    private class UpdateIndexer implements Index.Indexer
+    private class UpdateIndexer extends IndexerAdapter
     {
         private final DecoratedKey key;
-        private final Memtable memtable;
+        private final Memtable mt;
         private final WriteContext writeContext;
 
-        UpdateIndexer(DecoratedKey key, Memtable memtable, WriteContext writeContext)
+        UpdateIndexer(DecoratedKey key, Memtable mt, WriteContext writeContext)
         {
             this.key = key;
-            this.memtable = memtable;
+            this.mt = mt;
             this.writeContext = writeContext;
         }
 
         @Override
         public void insertRow(Row row)
         {
-            adjustMemtableSize(memtableIndexManager.index(key, row, memtable),
-                               CassandraWriteContext.fromContext(writeContext).getGroup());
+            indexContext.index(key, row, mt, CassandraWriteContext.fromContext(writeContext).getGroup());
         }
 
         @Override
         public void updateRow(Row oldRow, Row newRow)
         {
-            adjustMemtableSize(memtableIndexManager.update(key, oldRow, newRow, memtable),
-                               CassandraWriteContext.fromContext(writeContext).getGroup());
+            indexContext.update(key, oldRow, newRow, mt, CassandraWriteContext.fromContext(writeContext).getGroup());
         }
+    }
+
+    protected static abstract class IndexerAdapter implements Indexer
+    {
+        @Override
+        public void begin() { }
+
+        @Override
+        public void finish() { }
 
-        void adjustMemtableSize(long additionalSpace, OpOrder.Group opGroup)
+        @Override
+        public void partitionDelete(DeletionTime dt)
+        {
+        }
+
+        @Override
+        public void rangeTombstone(RangeTombstone rt)
+        {
+        }
+
+        @Override
+        public void removeRow(Row row)
+        {
+        }
+    }
+
+    @Override
+    public Searcher searcherFor(ReadCommand command) throws InvalidRequestException
+    {
+        // searchers should be created from the query plan, this is only used by the singleton index query plan
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker)
+    {
+        throw new UnsupportedOperationException("Storage-attached index flush observers should never be created directly.");
+    }
+
+    @Override
+    public Indexer indexerFor(DecoratedKey key,
+                              RegularAndStaticColumns columns,
+                              long nowInSec,
+                              WriteContext writeContext,
+                              IndexTransaction.Type transactionType,
+                              Memtable memtable)
+    {
+        if (transactionType == IndexTransaction.Type.UPDATE)
         {
-            // The memtable will assert if we try and reduce its memory usage so, for now, just don't tell it.
-            if (additionalSpace >= 0)
-                memtable.markExtraOnHeapUsed(additionalSpace, opGroup);
+            return new UpdateIndexer(key, memtable, writeContext);
         }
+
+        // we are only interested in the data from Memtable
+        // everything else is going to be handled by SSTableWriter observers
+        return null;
+    }
+
+    @Override
+    public IndexBuildingSupport getBuildTaskSupport()
+    {
+        return INDEX_BUILDER_SUPPORT;
+    }
+
+    public IndexContext getIndexContext()
+    {
+        return indexContext;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("%s.%s.%s", baseCfs.keyspace.getName(), baseCfs.name, config == null ? "?" : config.name);
+    }
+
+    /**
+     * Removes this index from the {@link SecondaryIndexManager}'s set of queryable indexes.
+     *
+     * This usually happens in response to an index writing failure from {@link StorageAttachedIndexWriter}.
+     */
+    public void makeIndexNonQueryable()
+    {
+        baseCfs.indexManager.makeIndexNonQueryable(this, Status.BUILD_FAILED);
+        logger.warn(indexContext.logMessage("Storage-attached index is no longer queryable. Please restart this node to repair it."));
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java
index 55f6381859cd..6790bf4f5ce5 100644
--- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java
@@ -21,36 +21,42 @@
 
 package org.apache.cassandra.index.sai;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import com.google.common.collect.Maps;
+
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.TimeUUID;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.compaction.CompactionInfo;
 import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.Tracker;
 import org.apache.cassandra.index.SecondaryIndexBuilder;
 import org.apache.cassandra.index.sai.disk.StorageAttachedIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.ComponentsBuildId;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.KeyIterator;
 import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
-import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTableWatcher;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.CountDownLatch;
 import org.apache.cassandra.utils.concurrent.Ref;
 
@@ -82,10 +88,7 @@ public class StorageAttachedIndexBuilder extends SecondaryIndexBuilder
     private long bytesProcessed = 0;
     private final long totalSizeInBytes;
 
-    StorageAttachedIndexBuilder(StorageAttachedIndexGroup group,
-                                SortedMap<SSTableReader, Set<StorageAttachedIndex>> sstables,
-                                boolean isFullRebuild,
-                                boolean isInitialBuild)
+    StorageAttachedIndexBuilder(StorageAttachedIndexGroup group, SortedMap<SSTableReader, Set<StorageAttachedIndex>> sstables, boolean isFullRebuild, boolean isInitialBuild)
     {
         this.group = group;
         this.metadata = group.metadata();
@@ -120,8 +123,7 @@ public void build()
         }
     }
 
-    private String logMessage(String message)
-    {
+    private String logMessage(String message) {
         return String.format("[%s.%s.*] %s", metadata.keyspace, metadata.name, message);
     }
 
@@ -131,6 +133,7 @@ private String logMessage(String message)
     private boolean indexSSTable(SSTableReader sstable, Set<StorageAttachedIndex> indexes)
     {
         logger.debug(logMessage("Starting index build on {}"), sstable.descriptor);
+        long startTimeNanos = Clock.Global.nanoTime();
 
         CountDownLatch perSSTableFileLock = null;
         StorageAttachedIndexWriter indexWriter = null;
@@ -142,19 +145,25 @@ private boolean indexSSTable(SSTableReader sstable, Set<StorageAttachedIndex> in
             return false;
         }
 
+        SSTableWatcher.instance.onIndexBuild(sstable, indexes);
+
+        IndexDescriptor indexDescriptor = group.descriptorFor(sstable);
+
+        Set<Component> replacedComponents = new HashSet<>();
+
         try (RandomAccessReader dataFile = sstable.openDataReader();
              LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.INDEX_BUILD, sstable))
         {
-            perSSTableFileLock = shouldWritePerSSTableFiles(sstable);
+            perSSTableFileLock = shouldWritePerSSTableFiles(sstable, indexDescriptor, replacedComponents);
             // If we were unable to get the per-SSTable file lock it means that the
-            // per-SSTable components are already being built, so we only want to
+            // per-SSTable components are already being built so we only want to
             // build the per-index components
             boolean perIndexComponentsOnly = perSSTableFileLock == null;
-            // remove existing per column index files instead of overwriting
-            IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
-            indexes.forEach(index -> indexDescriptor.deleteColumnIndex(index.termType(), index.identifier()));
+            for (StorageAttachedIndex index : indexes)
+                prepareForRebuild(indexDescriptor.perIndexComponents(index.getIndexContext()), replacedComponents);
 
-            indexWriter = StorageAttachedIndexWriter.createBuilderWriter(indexDescriptor, indexes, txn, perIndexComponentsOnly);
+            long keyCount = SSTableReader.getApproximateKeyCount(Set.of(sstable));
+            indexWriter = new StorageAttachedIndexWriter(indexDescriptor, metadata, indexes, txn, keyCount, perIndexComponentsOnly, group.table().metric);
 
             indexWriter.begin();
 
@@ -167,14 +176,14 @@ private boolean indexSSTable(SSTableReader sstable, Set<StorageAttachedIndex> in
                     if (isStopRequested())
                     {
                         logger.debug(indexDescriptor.logMessage("Index build has been stopped"));
-                        throw new CompactionInterruptedException(getCompactionInfo());
+                        throw new CompactionInterruptedException(getProgress(), trigger());
                     }
 
                     DecoratedKey key = keys.next();
+                    long position = sstable.getPosition(key, SSTableReader.Operator.EQ);
 
-                    indexWriter.startPartition(key, -1, -1);
+                    indexWriter.startPartition(key, position, position);
 
-                    long position = sstable.getPosition(key, SSTableReader.Operator.EQ);
                     dataFile.seek(position);
                     ByteBufferUtil.readWithShortLength(dataFile); // key
 
@@ -192,8 +201,11 @@ private boolean indexSSTable(SSTableReader sstable, Set<StorageAttachedIndex> in
                     previousBytesRead = bytesRead;
                 }
 
-                completeSSTable(indexWriter, sstable, indexes, perSSTableFileLock);
+                completeSSTable(txn, indexWriter, sstable, indexes, perSSTableFileLock, replacedComponents);
             }
+            long timeTaken = Clock.Global.nanoTime() - startTimeNanos;
+            group.table().metric.updateStorageAttachedIndexBuildTime(timeTaken);
+            logger.trace("Completed indexing sstable {} in {} seconds", sstable.descriptor, TimeUnit.NANOSECONDS.toSeconds(timeTaken));
 
             return false;
         }
@@ -243,46 +255,64 @@ else if (t instanceof CompactionInterruptedException)
     }
 
     @Override
-    public CompactionInfo getCompactionInfo()
+    public OperationProgress getProgress()
     {
-        return new CompactionInfo(metadata,
-                                  OperationType.INDEX_BUILD,
-                                  bytesProcessed,
-                                  totalSizeInBytes,
-                                  compactionId,
-                                  sstables.keySet());
+        return new OperationProgress(metadata,
+                                     OperationType.INDEX_BUILD,
+                                     bytesProcessed,
+                                     totalSizeInBytes,
+                                     compactionId,
+                                     sstables.keySet());
     }
 
     /**
-     * if the per sstable index files are already created, no need to write them again, unless found corrupted on rebuild
+     * if the per sstable index files are already created, not need to write it again, unless it's full rebuild.
      * if not created, try to acquire a lock, so only one builder will generate per sstable index files
      */
-    private CountDownLatch shouldWritePerSSTableFiles(SSTableReader sstable)
+    private CountDownLatch shouldWritePerSSTableFiles(SSTableReader sstable, IndexDescriptor indexDescriptor, Set<Component> replacedComponents)
     {
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
-
-        // if per-table files are incomplete, full rebuild is requested, or checksum fails
-        if (!indexDescriptor.isPerSSTableIndexBuildComplete()
-            || isFullRebuild
-            || !indexDescriptor.validatePerSSTableComponents(IndexValidation.CHECKSUM, true, false))
+        // if per-table files are incomplete or checksum failed during full rebuild.
+        if (!indexDescriptor.perSSTableComponents().isComplete() || isFullRebuild)
         {
             CountDownLatch latch = CountDownLatch.newCountDownLatch(1);
             if (inProgress.putIfAbsent(sstable, latch) == null)
             {
-                // lock owner should clean up existing per-SSTable files
-                group.deletePerSSTableFiles(Collections.singleton(sstable));
+                prepareForRebuild(indexDescriptor.perSSTableComponents(), replacedComponents);
                 return latch;
             }
         }
         return null;
     }
 
-    private void completeSSTable(SSTableFlushObserver indexWriter,
+    private static void prepareForRebuild(IndexComponents.ForRead components, Set<Component> replacedComponents)
+    {
+        // The current components are "replaced" (by "other" components) if the build create different components than
+        // the existing ones. This will happen in the following cases:
+        // 1. if we use immutable components, that's the point of immutable components.
+        // 2. when we do not use immutable components, the rebuild components will always be for the latest version and
+        // for generation 0, so if the current components are not for that specific built, then we won't be rebuilding
+        // the exact same components, and we're "replacing", not "overwriting" ()
+        //   a) the old components are from an older version: a new build will alawys be for `Version.latest()` and
+        //     so will create new files in that case (Note that "normally" we should not have non-0 generation in the
+        //     first place if immutable components are not used, but we handle this case to better support "downgrades"
+        //     where immutable components was enabled, but then disabled for some reason. If that happens, we still
+        //     want to ensure a new build removes the old files both from disk (happens below) and from the sstable TOC
+        //     (which is what `replacedComponents` is about)).
+        if (components.version().useImmutableComponentFiles() || !components.buildId().equals(ComponentsBuildId.forNewSSTable()))
+            replacedComponents.addAll(components.allAsCustomComponents());
+
+        if (!components.version().useImmutableComponentFiles())
+            components.forWrite().forceDeleteAllComponents();
+    }
+
+    private void completeSSTable(LifecycleTransaction txn,
+                                 StorageAttachedIndexWriter indexWriter,
                                  SSTableReader sstable,
                                  Set<StorageAttachedIndex> indexes,
-                                 CountDownLatch latch) throws InterruptedException
+                                 CountDownLatch latch,
+                                 Set<Component> replacedComponents) throws InterruptedException, IOException
     {
-        indexWriter.complete();
+        indexWriter.complete(sstable);
 
         if (latch != null)
         {
@@ -308,9 +338,19 @@ private void completeSSTable(SSTableFlushObserver indexWriter,
         }
 
         // register custom index components into existing sstables
-        sstable.registerComponents(StorageAttachedIndexGroup.getLiveComponents(sstable, existing), tracker);
-        Set<StorageAttachedIndex> incomplete = group.onSSTableChanged(Collections.emptyList(), Collections.singleton(sstable), existing, IndexValidation.NONE);
-
+        sstable.registerComponents(group.activeComponents(sstable), tracker);
+        if (!replacedComponents.isEmpty())
+            sstable.unregisterComponents(replacedComponents, tracker);
+
+        /**
+         * During memtable flush, it completes the transaction first which opens the flushed sstable,
+         * and then notify the new sstable to SAI. Here we should do the same.
+         */
+        txn.trackNewAttachedIndexFiles(sstable);
+        // there is nothing to commit. Close() effectively abort the transaction.
+        txn.close();
+
+        Set<StorageAttachedIndex> incomplete = group.onSSTableChanged(Collections.emptyList(), Collections.singleton(sstable), existing, false);
         if (!incomplete.isEmpty())
         {
             // If this occurs during an initial index build, there is only one index in play, and
@@ -319,7 +359,7 @@ private void completeSSTable(SSTableFlushObserver indexWriter,
             // set of indexes for a new added/streamed SSTables, we terminate pessimistically. In
             // other words, we abort the SSTable index write across all column indexes and mark
             // then non-queryable until a restart or other incremental rebuild occurs.
-            throw new RuntimeException(logMessage("Failed to update views on column indexes " + incomplete + " on indexes " + indexes + '.'));
+            throw new RuntimeException(logMessage("Failed to update views on column indexes " + incomplete + " on indexes " + indexes + "."));
         }
     }
 
@@ -342,11 +382,11 @@ private Set<StorageAttachedIndex> validateIndexes(Set<StorageAttachedIndex> inde
 
         if (!dropped.isEmpty())
         {
-            String droppedIndexes = dropped.stream().map(sai -> sai.identifier().indexName).collect(Collectors.toList()).toString();
+            String droppedIndexes = dropped.stream().map(sai -> sai.getIndexContext().getIndexName()).collect(Collectors.toList()).toString();
             if (isFullRebuild)
                 throw new RuntimeException(logMessage(String.format("%s are dropped, will stop index build.", droppedIndexes)));
             else
-                logger.debug(logMessage("Skip building dropped index {} on sstable {}"), droppedIndexes, descriptor.baseFile());
+                logger.debug(logMessage("Skip building dropped index {} on sstable {}"), droppedIndexes, descriptor.baseFileUri());
         }
 
         return existing;
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuildingSupport.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuildingSupport.java
deleted file mode 100644
index 7a13b4b186ab..000000000000
--- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuildingSupport.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai;
-
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.stream.Collectors;
-
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.SecondaryIndexBuilder;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.io.sstable.SSTableIdFactory;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-
-class StorageAttachedIndexBuildingSupport implements Index.IndexBuildingSupport
-{
-    @Override
-    public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs,
-                                                   Set<Index> indexes,
-                                                   Collection<SSTableReader> sstablesToRebuild,
-                                                   boolean isFullRebuild)
-    {
-        NavigableMap<SSTableReader, Set<StorageAttachedIndex>> sstables = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR));
-        StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
-
-        assert group != null : "Index group does not exist for table " + cfs.keyspace + '.' + cfs.name;
-
-        indexes.stream()
-               .filter((i) -> i instanceof StorageAttachedIndex)
-               .forEach((i) ->
-                        {
-                            StorageAttachedIndex sai = (StorageAttachedIndex) i;
-
-                            // If this is not a full manual index rebuild we can skip SSTables that already have an
-                            // attached index. Otherwise, we override any pre-existent index.
-                            Collection<SSTableReader> ss = sstablesToRebuild;
-                            if (!isFullRebuild)
-                            {
-                                ss = sstablesToRebuild.stream()
-                                                      .filter(s -> !IndexDescriptor.create(s).isPerColumnIndexBuildComplete(sai.identifier()))
-                                                      .collect(Collectors.toList());
-                            }
-
-                            group.dropIndexSSTables(ss, sai);
-
-                            ss.forEach(sstable -> sstables.computeIfAbsent(sstable, ignore -> new HashSet<>()).add(sai));
-                        });
-
-        return new StorageAttachedIndexBuilder(group, sstables, isFullRebuild, false);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
index 30d23f5de61d..bb9f47bea181 100644
--- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
@@ -17,12 +17,15 @@
  */
 package org.apache.cassandra.index.sai;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
@@ -30,7 +33,8 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.primitives.Ints;
+import com.google.common.collect.Lists;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,10 +48,10 @@
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
 import org.apache.cassandra.index.sai.disk.StorageAttachedIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.sai.metrics.IndexGroupMetrics;
 import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
 import org.apache.cassandra.index.sai.metrics.TableStateMetrics;
@@ -56,6 +60,7 @@
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTableWatcher;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.notifications.INotification;
 import org.apache.cassandra.notifications.INotificationConsumer;
@@ -64,7 +69,6 @@
 import org.apache.cassandra.notifications.SSTableAddedNotification;
 import org.apache.cassandra.notifications.SSTableListChangedNotification;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.Throwables;
 
 /**
@@ -80,18 +84,21 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
     private final TableQueryMetrics queryMetrics;
     private final TableStateMetrics stateMetrics;
     private final IndexGroupMetrics groupMetrics;
-    private final Set<StorageAttachedIndex> indexes = ConcurrentHashMap.newKeySet();
+
+    private final Set<StorageAttachedIndex> indices = ConcurrentHashMap.newKeySet();
     private final ColumnFamilyStore baseCfs;
 
     private final SSTableContextManager contextManager;
 
+
+
     StorageAttachedIndexGroup(ColumnFamilyStore baseCfs)
     {
         this.baseCfs = baseCfs;
         this.queryMetrics = new TableQueryMetrics(baseCfs.metadata());
         this.stateMetrics = new TableStateMetrics(baseCfs.metadata(), this);
         this.groupMetrics = new IndexGroupMetrics(baseCfs.metadata(), this);
-        this.contextManager = new SSTableContextManager();
+        this.contextManager = new SSTableContextManager(baseCfs.getTracker());
 
         Tracker tracker = baseCfs.getTracker();
         tracker.subscribe(this);
@@ -100,43 +107,56 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
     @Nullable
     public static StorageAttachedIndexGroup getIndexGroup(ColumnFamilyStore cfs)
     {
-        return (StorageAttachedIndexGroup) cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.GROUP_KEY);
+        return (StorageAttachedIndexGroup) cfs.indexManager.getIndexGroup(GROUP_KEY);
     }
 
     @Override
-    public Set<Index> getIndexes()
+    public Set<StorageAttachedIndex> getIndexes()
     {
-        return ImmutableSet.copyOf(indexes);
+        return ImmutableSet.copyOf(indices);
     }
 
     @Override
     public void addIndex(Index index)
     {
         assert index instanceof StorageAttachedIndex;
-        indexes.add((StorageAttachedIndex) index);
+        indices.add((StorageAttachedIndex) index);
     }
 
     @Override
     public void removeIndex(Index index)
     {
         assert index instanceof StorageAttachedIndex;
-        boolean removed = indexes.remove(index);
+        boolean removed = indices.remove(index);
         assert removed : "Cannot remove non-existing index " + index;
         /*
          * per index files are dropped via {@link StorageAttachedIndex#getInvalidateTask()}
          */
-        if (indexes.isEmpty())
+        if (indices.isEmpty())
         {
-            for (SSTableReader sstable : contextManager.sstables())
-                sstable.unregisterComponents(IndexDescriptor.create(sstable).getLivePerSSTableComponents(), baseCfs.getTracker());
-            deletePerSSTableFiles(baseCfs.getLiveSSTables());
+            // We unregister the per-sstable components first, then we clear the context, which closes all the contexts
+            // and unsure there is not more reference to it. When that's done, we can safely remove the component files
+            // on disk. Note that we copy the contexts list because we're going to clear the manager, and we need to
+            // make sure this does not clear the `contexts` collection below (since it exists to be used after the clear).
+            Collection<SSTableContext> contexts = new ArrayList<>(contextManager.allContexts());
+            contexts.forEach(context -> {
+                var components = context.usedPerSSTableComponents();
+                context.sstable.unregisterComponents(components.allAsCustomComponents(), baseCfs.getTracker());
+            });
+
+            contextManager.clear();
+
+            contexts.forEach(context -> {
+                SSTableWatcher.instance.onIndexDropped(baseCfs.metadata(), context.usedPerSSTableComponents().forWrite());
+            });
         }
     }
 
     @Override
     public void invalidate()
     {
-        // in case of removing last index from group, sstable contexts should already been removed by removeIndex
+        // in case of dropping table, sstable contexts should already been removed by SSTableListChangedNotification.
+        // in case of removing last index from group,  sstable contexts should already been removed by StorageAttachedIndexGroup#removeIndex
         queryMetrics.release();
         groupMetrics.release();
         stateMetrics.release();
@@ -144,16 +164,32 @@ public void invalidate()
     }
 
     @Override
-    @SuppressWarnings("SuspiciousMethodCalls")
-    public boolean containsIndex(Index index)
+    public void unload()
     {
-        return indexes.contains(index);
+        baseCfs.getTracker().unsubscribe(this);
+
+        contextManager.clear();
+        queryMetrics.release();
+        groupMetrics.release();
+        stateMetrics.release();
+    }
+
+    @Override
+    public boolean supportsMultipleContains()
+    {
+        return true;
     }
 
     @Override
-    public boolean isSingleton()
+    public boolean supportsDisjunction()
     {
-        return false;
+        return true;
+    }
+
+    @Override
+    public boolean containsIndex(Index index)
+    {
+        return index instanceof StorageAttachedIndex && indices.contains(index);
     }
 
     @Override
@@ -166,25 +202,34 @@ public Index.Indexer indexerFor(Predicate<Index> indexSelector,
                                     Memtable memtable)
     {
         final Set<Index.Indexer> indexers =
-                indexes.stream().filter(indexSelector)
-                       .map(i -> i.indexerFor(key, columns, nowInSec, ctx, transactionType, memtable))
-                       .filter(Objects::nonNull)
-                       .collect(Collectors.toSet());
+                indices.stream().filter(indexSelector)
+                                .map(i -> i.indexerFor(key, columns, nowInSec, ctx, transactionType, memtable))
+                                .filter(Objects::nonNull)
+                                .collect(Collectors.toSet());
 
-        return indexers.isEmpty() ? null : new Index.Indexer()
+        return indexers.isEmpty() ? null : new StorageAttachedIndex.IndexerAdapter()
         {
             @Override
             public void insertRow(Row row)
             {
-                for (Index.Indexer indexer : indexers)
-                    indexer.insertRow(row);
+                forEach(indexer -> indexer.insertRow(row));
             }
 
             @Override
             public void updateRow(Row oldRow, Row newRow)
             {
-                for (Index.Indexer indexer : indexers)
-                    indexer.updateRow(oldRow, newRow);
+                forEach(indexer -> indexer.updateRow(oldRow, newRow));
+            }
+
+            @Override
+            public void removeRow(Row row)
+            {
+                forEach(indexer -> indexer.removeRow(row));
+            }
+
+            private void forEach(Consumer<Index.Indexer> action)
+            {
+                indexers.forEach(action::accept);
             }
         };
     }
@@ -192,23 +237,23 @@ public void updateRow(Row oldRow, Row newRow)
     @Override
     public StorageAttachedIndexQueryPlan queryPlanFor(RowFilter rowFilter)
     {
-        return StorageAttachedIndexQueryPlan.create(baseCfs, queryMetrics, indexes, rowFilter);
+        return StorageAttachedIndexQueryPlan.create(baseCfs, queryMetrics, indices, rowFilter);
     }
 
     @Override
-    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata, long keyCount)
     {
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor, tableMetadata.partitioner, tableMetadata.comparator);
+        IndexDescriptor indexDescriptor = IndexDescriptor.empty(descriptor);
         try
         {
-            return StorageAttachedIndexWriter.createFlushObserverWriter(indexDescriptor, indexes, tracker);
+            return new StorageAttachedIndexWriter(indexDescriptor, tableMetadata, indices, tracker, keyCount, baseCfs.metric);
         }
         catch (Throwable t)
         {
             String message = "Unable to create storage-attached index writer on SSTable flush." +
                              " All indexes from this table are going to be marked as non-queryable and will need to be rebuilt.";
             logger.error(indexDescriptor.logMessage(message), t);
-            indexes.forEach(StorageAttachedIndex::makeIndexNonQueryable);
+            indices.forEach(StorageAttachedIndex::makeIndexNonQueryable);
             return null;
         }
     }
@@ -221,30 +266,31 @@ public boolean handles(IndexTransaction.Type type)
     }
 
     @Override
-    public Set<Component> getComponents()
+    public Set<Component> componentsForNewSSTable()
     {
-        return getComponents(indexes);
+        return IndexDescriptor.componentsForNewlyFlushedSSTable(indices);
     }
 
-    private Set<Component> getComponents(Collection<StorageAttachedIndex> indices)
+    @Override
+    public Set<Component> activeComponents(SSTableReader sstable)
     {
-        Set<Component> components = Version.LATEST.onDiskFormat()
-                                                  .perSSTableIndexComponents(baseCfs.metadata.get().comparator.size() > 0)
-                                                  .stream()
-                                                  .map(Version.LATEST::makePerSSTableComponent)
-                                                  .collect(Collectors.toSet());
-        indices.forEach(index -> components.addAll(index.getComponents()));
-        return components;
-    }
+        IndexDescriptor indexDescriptor = descriptorFor(sstable);
+        Set<Component> components = indexDescriptor
+                                    .perSSTableComponents()
+                                    .all()
+                                    .stream()
+                                    .map(IndexComponent::asCustomComponent)
+                                    .collect(Collectors.toSet());
+
+        for (StorageAttachedIndex index : indices)
+        {
+            indexDescriptor.perIndexComponents(index.getIndexContext())
+                           .all()
+                           .stream()
+                           .map(IndexComponent::asCustomComponent)
+                           .forEach(components::add);
+        }
 
-    // This differs from getComponents in that it only returns index components that exist on disk.
-    // It avoids errors being logged by the SSTable.readTOC method when we have an empty index.
-    @VisibleForTesting
-    public static Set<Component> getLiveComponents(SSTableReader sstable, Collection<StorageAttachedIndex> indices)
-    {
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
-        Set<Component> components = indexDescriptor.getLivePerSSTableComponents();
-        indices.forEach(index -> components.addAll(indexDescriptor.getLivePerIndexComponents(index.termType(), index.identifier())));
         return components;
     }
 
@@ -258,36 +304,30 @@ public void handleNotification(INotification notification, Object sender)
 
             // Avoid validation for index files just written following Memtable flush. Otherwise, the new SSTables have
             // come either from import, streaming, or a standalone tool, where they have also already been validated.
-            onSSTableChanged(Collections.emptySet(), notice.added, indexes, IndexValidation.NONE);
+            onSSTableChanged(Collections.emptySet(), Lists.newArrayList(notice.added), indices, false);
         }
         else if (notification instanceof SSTableListChangedNotification)
         {
             SSTableListChangedNotification notice = (SSTableListChangedNotification) notification;
 
             // Avoid validation for index files just written during compaction.
-            onSSTableChanged(notice.removed, notice.added, indexes, IndexValidation.NONE);
+            onSSTableChanged(notice.removed, notice.added, indices, false);
         }
         else if (notification instanceof MemtableRenewedNotification)
         {
-            indexes.forEach(index -> index.memtableIndexManager().renewMemtable(((MemtableRenewedNotification) notification).renewed));
+            indices.forEach(index -> index.getIndexContext().renewMemtable(((MemtableRenewedNotification) notification).renewed));
         }
         else if (notification instanceof MemtableDiscardedNotification)
         {
-            indexes.forEach(index -> index.memtableIndexManager().discardMemtable(((MemtableDiscardedNotification) notification).memtable));
+            indices.forEach(index -> index.getIndexContext().discardMemtable(((MemtableDiscardedNotification) notification).memtable));
         }
     }
 
-    void deletePerSSTableFiles(Collection<SSTableReader> sstables)
-    {
-        contextManager.release(sstables);
-        sstables.forEach(sstableReader -> IndexDescriptor.create(sstableReader).deletePerSSTableIndexComponents());
-    }
-
-    void dropIndexSSTables(Collection<SSTableReader> ss, StorageAttachedIndex index)
+    void prepareIndexSSTablesForRebuild(Collection<SSTableReader> ss, StorageAttachedIndex index)
     {
         try
         {
-            index.drop(ss);
+            index.getIndexContext().prepareSSTablesForRebuild(ss);
         }
         catch (Throwable t)
         {
@@ -304,23 +344,14 @@ void dropIndexSSTables(Collection<SSTableReader> ss, StorageAttachedIndex index)
      * @return the set of column indexes that were marked as non-queryable as a result of their per-SSTable index
      * files being corrupt or being unable to successfully update their views
      */
-    synchronized Set<StorageAttachedIndex> onSSTableChanged(Collection<SSTableReader> removed, Iterable<SSTableReader> added,
-                                                            Set<StorageAttachedIndex> indexes, IndexValidation validation)
+    public synchronized Set<StorageAttachedIndex> onSSTableChanged(Collection<SSTableReader> removed, Collection<SSTableReader> added,
+                                                            Set<StorageAttachedIndex> indexes, boolean validate)
     {
-        Pair<Set<SSTableContext>, Set<SSTableReader>> results = contextManager.update(removed, added, validation);
-
-        if (!results.right.isEmpty())
+        Optional<Set<SSTableContext>> optValid = contextManager.update(removed, added, validate, indices);
+        if (optValid.isEmpty())
         {
-            results.right.forEach(sstable -> {
-                IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
-                indexDescriptor.deletePerSSTableIndexComponents();
-                // Column indexes are invalid if their SSTable-level components are corrupted so delete
-                // their associated index files and mark them non-queryable.
-                indexes.forEach(index -> {
-                    indexDescriptor.deleteColumnIndex(index.termType(), index.identifier());
-                    index.makeIndexNonQueryable();
-                });
-            });
+            // This means at least one sstable had invalid per-sstable components, so mark all indexes non-queryable.
+            indices.forEach(StorageAttachedIndex::makeIndexNonQueryable);
             return indexes;
         }
 
@@ -328,13 +359,11 @@ synchronized Set<StorageAttachedIndex> onSSTableChanged(Collection<SSTableReader
 
         for (StorageAttachedIndex index : indexes)
         {
-            Collection<SSTableContext> invalid = index.onSSTableChanged(removed, results.left, validation);
+            Set<SSTableContext> invalid = index.getIndexContext().onSSTableChanged(removed, added, optValid.get(), validate);
 
             if (!invalid.isEmpty())
             {
-                // Delete the index files and mark the index non-queryable, as its view may be compromised,
-                // and incomplete, for our callers:
-                invalid.forEach(context -> context.indexDescriptor.deleteColumnIndex(index.termType(), index.identifier()));
+                // Mark the index non-queryable, as its view may be compromised, and incomplete, for our callers.
                 index.makeIndexNonQueryable();
                 incomplete.add(index);
             }
@@ -349,25 +378,28 @@ public boolean validateSSTableAttachedIndexes(Collection<SSTableReader> sstables
 
         for (SSTableReader sstable : sstables)
         {
-            IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
+            IndexDescriptor indexDescriptor = contextManager.getOrLoadIndexDescriptor(sstable, indices);
+            IndexComponents.ForRead perSSTableComponents = indexDescriptor.perSSTableComponents();
 
-            if (indexDescriptor.isPerSSTableIndexBuildComplete())
+            if (indexDescriptor.perSSTableComponents().isComplete())
             {
-                indexDescriptor.validatePerSSTableComponents(IndexValidation.CHECKSUM, validateChecksum, true);
+                perSSTableComponents.validateComponents(sstable, baseCfs.getTracker(), validateChecksum, true);
 
-                for (StorageAttachedIndex index : indexes)
+                for (StorageAttachedIndex index : indices)
                 {
-                    if (indexDescriptor.isPerColumnIndexBuildComplete(index.identifier()))
-                        indexDescriptor.validatePerIndexComponents(index.termType(), index.identifier(), IndexValidation.CHECKSUM, validateChecksum, true);
+                    IndexComponents.ForRead perIndexComponents = indexDescriptor.perIndexComponents(index.getIndexContext());
+
+                    if (perIndexComponents.isComplete())
+                        perIndexComponents.validateComponents(sstable, baseCfs.getTracker(), validateChecksum, true);
                     else if (throwOnIncomplete)
-                        throw new IllegalStateException(indexDescriptor.logMessage("Incomplete per-column index build for SSTable " + sstable.descriptor.toString()));
+                        throw new IllegalStateException(indexDescriptor.logMessage("Incomplete per-column index build for SSTable " + sstable.descriptor));
                     else
                         complete = false;
                 }
             }
             else if (throwOnIncomplete)
             {
-                throw new IllegalStateException(indexDescriptor.logMessage("Incomplete per-SSTable index build" + sstable.descriptor.toString()));
+                throw new IllegalStateException(indexDescriptor.logMessage("Incomplete per-SSTable index build" + sstable.descriptor));
             }
             else
             {
@@ -386,11 +418,11 @@ else if (throwOnIncomplete)
      */
     public int openIndexFiles()
     {
-        return contextManager.openFiles() + indexes.stream().mapToInt(StorageAttachedIndex::openPerColumnIndexFiles).sum();
+        return contextManager.openFiles() + indices.stream().mapToInt(index -> index.getIndexContext().openPerIndexFiles()).sum();
     }
 
     /**
-     * @return total disk usage (in bytes) of all per-sstable index files
+     * @return total disk usage of all per-sstable index files
      */
     public long diskUsage()
     {
@@ -402,7 +434,7 @@ public long diskUsage()
      */
     public int totalIndexBuildsInProgress()
     {
-        return (int) indexes.stream().filter(i -> baseCfs.indexManager.isIndexBuilding(i.getIndexMetadata().name)).count();
+        return (int) indices.stream().filter(i -> baseCfs.indexManager.isIndexBuilding(i.getIndexMetadata().name)).count();
     }
 
     /**
@@ -410,7 +442,7 @@ public int totalIndexBuildsInProgress()
      */
     public int totalQueryableIndexCount()
     {
-        return Ints.checkedCast(indexes.stream().filter(baseCfs.indexManager::isIndexQueryable).count());
+        return (int) indices.stream().filter(i -> baseCfs.indexManager.isIndexQueryable(i)).count();
     }
 
     /**
@@ -418,7 +450,7 @@ public int totalQueryableIndexCount()
      */
     public int totalIndexCount()
     {
-        return indexes.size();
+        return indices.size();
     }
 
     /**
@@ -426,7 +458,13 @@ public int totalIndexCount()
      */
     public long totalDiskUsage()
     {
-        return diskUsage() + indexes.stream().flatMap(index -> index.view().getIndexes().stream())
+        // Note that this only account the "active" files. That is, if we have old versions/generations or incomplete
+        // build still on disk, those won't be counted. Counting only "live" data here is consistent with the fact
+        // that `TableStateMetrics.diskUsagePercentageOfBaseTable` compare the number obtain from this to the base
+        // table "live" disk space use. But there is certainly a small risk for being misleading, and where base
+        // tables expose both a "liveDiskSpaceUsed" and "totalDiskSpaceUsed", SAI only exposes "diskUsageBytes", which
+        // has we just mentioned is the "live" usage. Might be worth improving at some point.
+        return diskUsage() + indices.stream().flatMap(i -> i.getIndexContext().getView().getIndexes().stream())
                                     .mapToLong(SSTableIndex::sizeOfPerColumnComponents).sum();
     }
 
@@ -446,6 +484,17 @@ public SSTableContextManager sstableContextManager()
         return contextManager;
     }
 
+    /**
+     * Returns the {@link IndexDescriptor} for the given {@link SSTableReader} (which must belong to the base table
+     * of this group).
+     * Note that this always return a non-null value, since all sstables must be indexed, but that descriptor could
+     * be "empty" if the sstable has never had an index built yet.
+     */
+    public IndexDescriptor descriptorFor(SSTableReader sstable)
+    {
+        return contextManager.getOrLoadIndexDescriptor(sstable, indices);
+    }
+
     /**
      * simulate index loading on restart with index file validation
      */
@@ -453,8 +502,8 @@ public SSTableContextManager sstableContextManager()
     public void unsafeReload()
     {
         contextManager.clear();
-        onSSTableChanged(baseCfs.getLiveSSTables(), Collections.emptySet(), indexes, IndexValidation.NONE);
-        onSSTableChanged(Collections.emptySet(), baseCfs.getLiveSSTables(), indexes, IndexValidation.HEADER_FOOTER);
+        onSSTableChanged(baseCfs.getLiveSSTables(), Collections.emptySet(), indices, false);
+        onSSTableChanged(Collections.emptySet(), baseCfs.getLiveSSTables(), indices, true);
     }
 
     /**
@@ -464,7 +513,7 @@ public void unsafeReload()
     public void reset()
     {
         contextManager.clear();
-        indexes.forEach(StorageAttachedIndex::makeIndexNonQueryable);
-        onSSTableChanged(baseCfs.getLiveSSTables(), Collections.emptySet(), indexes, IndexValidation.NONE);
+        indices.forEach(index -> index.makeIndexNonQueryable());
+        onSSTableChanged(baseCfs.getLiveSSTables(), Collections.emptySet(), indices, false);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/VECTOR.md b/src/java/org/apache/cassandra/index/sai/VECTOR.md
new file mode 100644
index 000000000000..7c9c8628efd5
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/VECTOR.md
@@ -0,0 +1,189 @@
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+# SAI Vector ANN Query Execution
+
+## Overview
+
+Vector search within SAI has taken two major forms to date. The first utilized PrimaryKey ordered iterators and was
+very sensitive to Shadowed Primary Keys as well as overwritten vectors for rows. The second utilized Score ordered
+iterators, which was able to handle these cases more gracefully.
+
+This document describes vector search using Score ordered iterators.
+
+## Storage-Attached Index Basics
+
+* We can create indexes on columns to support searching them without requiring `ALLOW FILTERING` and without requiring
+that they are part of the primary key
+* An indexed column index consists of local indexes for each memtable and each sstable segment within the table
+* Query execution scatters across each index to get the collection of Primary Keys that satisfy a predicate
+* Each sstable segment's index is immutable
+* Memtable indexes are mutable and are updated as the memtable is updated
+
+## Vector Index Basics
+
+* A vector index gives us the ability to search for similar vectors
+* We take advantage of the fact that each sstable segment is immutable and finite
+* If we take the top k vectors from each sstable segment, we can materialize them from storage and get the top k vectors
+  from the entire table (more on this later)
+* The `K` in `topK` is generally the `LIMIT` of the query, but can be larger (more on this later)
+
+## Query Types
+
+### Vector Only Query
+
+When a query is only limited by ANN, the query execution is scatter gather across all relevant vector indexes. The query
+results in a lazily evaluated iterator that materializes rows from storage in index score order, which can differ from
+the "global" score order in the case of updates.
+1. Eagerly query each sstable's and memtable's vector indexes producing local top k ordinals. Return them in best-first score order.
+2. Lazily map ordinals to row ids then to Primary Keys keeping them in descending (best-first) score order. 
+3. Merge the iterators while maintaining relative score order. This merge does not dedupe iterator elements.
+4. Materialize one row from storage at a time.
+5. Filter out deleted rows. Then, compute the vector similarity score. If the score is at least as good as the score computed by the index, the vector 
+   is in the global top k. If it is worse than the index's score, temporarily ignore that key. Finally, reorder into
+   Primary Key order.
+6. Return the global top k rows to the coordinator.
+
+```mermaid
+---
+title: "SELECT * FROM my.table ORDER BY vec ANN OF [...] LIMIT N"
+---
+graph LR
+    subgraph 1: Get topK
+      G[SSTable A\nVector Index]
+      H[SSTable B\nVector Index]
+      I[Memtable\nVector Index]
+    end
+    subgraph "2: Map"
+        X[Ordinal -> Row ID -> Scored PK]
+        Y[Ordinal -> Row ID -> Scored PK]
+        Z[Ordinal -> Scored PK]
+    end
+    subgraph 3: Merge
+        J
+    end
+    G -.-> X -.-> J[Merge\nIndex\nIterators]
+    H -.-> Y -.-> J
+    I -.-> Z -.-> J
+    subgraph "4: Materialize"
+        K[Unfiltered\nPartition\nIterator]
+    end
+    subgraph "5: Filter, Score, Reorder"
+        L[Global top k]
+    end
+    J -.-> K
+    K -.-> L
+    L ==> M[Coordinator]
+
+    subgraph Legend
+      direction LR
+      start1[ ] -.->|Score Order Iterator| stop1[ ]
+      style start1 height:0px;
+      style stop1 height:0px;
+      start2[ ] ==>|PrimaryKey Order Iterator| stop2[ ]
+      style start2 height:0px;
+      style stop2 height:0px;
+    end
+```
+
+Notes:
+* The flow is much lazier than before. Now, we only materialize the top k rows from storage, not every top k row from
+  every sstable segment and memtable.
+* Range queries on the Primary Key that do not require an index are supported and are considered ANN only.
+* `ALLOW FILTERING` is not supported.
+
+### Pre-fitered Boolean Predicates Combined with ANN Query
+
+When a query relies on non vector SAI indexes and an ANN ordering predicate, the query execution is more complex. The execution
+of query `SELECT * FROM my.table WHERE x = 1 AND y = 'foo' ORDER BY vec ANN OF [...] LIMIT 10` follows this path:
+1. Query each boolean predicate's index to get the Primary Keys that satisfy the predicate.
+2. Merge the results with a `RangeUnionIterator` that deduplicates results for the predicate and maintains PK ordering.
+3. Intersect the results with a `RangeIntersectionIterator` to get the Primary Keys that satisfy all boolean predicates.
+4. Materialize the Primary Keys that satisfy all boolean predicates.
+5. Map resulting Primary Keys back to row ids and search each vector index for the local top k ordinals, then map those to 
+Primary Keys. Ultimately producing a single score ordered iterator. **This is expensive.**
+6. Materialize one row from storage at a time.
+7. Filter out deleted rows and validate the row against the logical filter. If the row does not match the WHERE clause, ignore the result. Then,
+   compute the vector similarity score. If the score is at least as good as the score computed by the index, the vector
+   is in the global top k. If it is worse than the index's score, temporarily ignore that key. Finally, reorder into
+   Primary Key order.
+8. Return the global top k rows to the coordinator.
+
+```mermaid
+---
+title: "SELECT * FROM my.table WHERE A = 1 AND B = 'foo' ORDER BY vec ANN OF [...] LIMIT 10"
+---
+graph LR
+    subgraph Step 1 and 2: Query Boolean Predicates
+        subgraph Indexes on Column A
+            A[SSTable 1\nIndex] --A=1--> B[Range\nUnion\nIterator]
+            C[SSTable 2\nIndex] --A=1--> B
+            D[Memtable\nIndex] --A=1--> B
+        end
+        subgraph Indexes on Column B
+            M[SSTable 1\nIndex] --B='foo'--> N[Range\nUnion\nIterator]
+            P[SSTable 2\nIndex] --B='foo'--> N
+            O[Memtable\nIndex] --B='foo'--> N
+        end
+    end
+    subgraph Step 3: Find PKs\nMatching Both\nPredicates
+      N --> E[Range\nIntersection\nIterator]
+      B --> E
+    end
+    E --> F[Materialize\nALL\nPrimary Keys]
+    subgraph "Steps 4 & 5: Index on Column vec"
+        F --> G1[PK -> SSTable 1\nRowIds] --> G[SSTable 1\nVector Index] .-> X[Ordinal -> PK]
+        F --> H1[PK -> SSTable 2\nRowIds] --> H[SSTable 2\nVector Index] .-> Y[Ordinal -> PK]
+        F --> I[Memtable\nVector Index]
+        X -.-> J[Merge\nScored PKs\nPriority Queue]
+        Y -.-> J
+        I -..-> J
+    end
+    subgraph "Step 6: Materialize"
+      K[Unfiltered\nPartition\nIterator]
+    end
+    subgraph "Step 7: Filter, Score, Reorder"
+      L[Global top k]
+    end
+    J -.-> K[Unfiltered\nPartition\nIterator]
+    K -.-> L[Global top k]
+    L --> Z[Coordinator]
+
+    subgraph Legend
+      direction LR
+      start1[ ] -.->|Score Order Iterator| stop1[ ]
+      style start1 height:0px;
+      style stop1 height:0px;
+      start2[ ] -->|PrimaryKey Order Iterator| stop2[ ]
+      style start2 height:0px;
+      style stop2 height:0px;
+    end
+```
+
+### Post-fitered Boolean Predicates Combined with ANN Query
+
+Sometimes, the boolean predicates are expensive to evaluate using the pre-filtered approach described above. An
+alternate query execution path is to sort the results using ANN first, then filter the materialized rows using the
+boolean predicates. The execution of query `SELECT * FROM my.table WHERE x = 1 AND y = 'foo' ORDER BY vec ANN OF [...] LIMIT 10` 
+using a post-filtered approach follows the same path as the [Vector Only Query](#vector-only-query) with the exception
+that the "filter" in step 7 additionally applies the boolean predicates and filters out any rows that do not match.
+
+The primary cost of post-filtering is that we might materialize many rows before finding the ones that match the boolean
+predicates. As such, we have a cost based optimizer that helps determine which approach is best for a given query.
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/VectorQueryContext.java b/src/java/org/apache/cassandra/index/sai/VectorQueryContext.java
deleted file mode 100644
index f33686e13fd4..000000000000
--- a/src/java/org/apache/cassandra/index/sai/VectorQueryContext.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.TreeSet;
-
-import io.github.jbellis.jvector.util.Bits;
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.disk.v1.vector.DiskAnn;
-import org.apache.cassandra.index.sai.disk.v1.vector.OnHeapGraph;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-
-/**
- * This represents the state of a vector query. It is repsonsible for maintaining a list of any {@link PrimaryKey}s
- * that have been updated or deleted during a search of the indexes.
- * <p>
- * The number of {@link #shadowedPrimaryKeys} is compared before and after a search is performed. If it changes, it
- * means that a {@link PrimaryKey} was found to have been changed. In this case the whole search is repeated until the
- * counts match.
- * <p>
- * When this process has completed, a {@link Bits} array is generated. This is used by the vector graph search to
- * identify which nodes in the graph to include in the results.
- */
-public class VectorQueryContext
-{
-    private final int limit;
-    // Holds primary keys that are shadowed by expired TTL or row tombstone or range tombstone.
-    // They are populated by the StorageAttachedIndexSearcher during filtering. They are used to generate
-    // a bitset for the graph search to indicate graph nodes to ignore.
-    private TreeSet<PrimaryKey> shadowedPrimaryKeys;
-
-    public VectorQueryContext(ReadCommand readCommand)
-    {
-        this.limit = readCommand.limits().count();
-    }
-
-    public int limit()
-    {
-        return limit;
-    }
-
-    public void recordShadowedPrimaryKey(PrimaryKey primaryKey)
-    {
-        if (shadowedPrimaryKeys == null)
-            shadowedPrimaryKeys = new TreeSet<>();
-        shadowedPrimaryKeys.add(primaryKey);
-    }
-
-    // Returns true if the row ID will be included or false if the row ID will be shadowed
-    public boolean shouldInclude(long sstableRowId, PrimaryKeyMap primaryKeyMap)
-    {
-        return shadowedPrimaryKeys == null || !shadowedPrimaryKeys.contains(primaryKeyMap.primaryKeyFromRowId(sstableRowId));
-    }
-
-    public boolean shouldInclude(PrimaryKey pk)
-    {
-        return shadowedPrimaryKeys == null || !shadowedPrimaryKeys.contains(pk);
-    }
-
-    public boolean containsShadowedPrimaryKey(PrimaryKey primaryKey)
-    {
-        return shadowedPrimaryKeys != null && shadowedPrimaryKeys.contains(primaryKey);
-    }
-
-    /**
-     * @return shadowed primary keys, in ascending order
-     */
-    public NavigableSet<PrimaryKey> getShadowedPrimaryKeys()
-    {
-        if (shadowedPrimaryKeys == null)
-            return Collections.emptyNavigableSet();
-        return shadowedPrimaryKeys;
-    }
-
-    public Bits bitsetForShadowedPrimaryKeys(OnHeapGraph<PrimaryKey> graph)
-    {
-        if (shadowedPrimaryKeys == null)
-            return null;
-
-        return new IgnoredKeysBits(graph, shadowedPrimaryKeys);
-    }
-
-    public Bits bitsetForShadowedPrimaryKeys(SegmentMetadata metadata, PrimaryKeyMap primaryKeyMap, DiskAnn graph) throws IOException
-    {
-        Set<Integer> ignoredOrdinals = null;
-        try (var ordinalsView = graph.getOrdinalsView())
-        {
-            for (PrimaryKey primaryKey : getShadowedPrimaryKeys())
-            {
-                // not in current segment
-                if (primaryKey.compareTo(metadata.minKey) < 0 || primaryKey.compareTo(metadata.maxKey) > 0)
-                    continue;
-
-                long sstableRowId = primaryKeyMap.rowIdFromPrimaryKey(primaryKey);
-                if (sstableRowId == Long.MAX_VALUE) // not found
-                    continue;
-
-                int segmentRowId = Math.toIntExact(sstableRowId - metadata.rowIdOffset);
-                // not in segment yet
-                if (segmentRowId < 0)
-                    continue;
-                // end of segment
-                if (segmentRowId > metadata.maxSSTableRowId)
-                    break;
-
-                int ordinal = ordinalsView.getOrdinalForRowId(segmentRowId);
-                if (ordinal >= 0)
-                {
-                    if (ignoredOrdinals == null)
-                        ignoredOrdinals = new HashSet<>();
-                    ignoredOrdinals.add(ordinal);
-                }
-            }
-        }
-
-        if (ignoredOrdinals == null)
-            return null;
-
-        return new IgnoringBits(ignoredOrdinals, metadata);
-    }
-
-    private static class IgnoringBits implements Bits
-    {
-        private final Set<Integer> ignoredOrdinals;
-        private final int length;
-
-        public IgnoringBits(Set<Integer> ignoredOrdinals, SegmentMetadata metadata)
-        {
-            this.ignoredOrdinals = ignoredOrdinals;
-            this.length = 1 + Math.toIntExact(metadata.maxSSTableRowId - metadata.rowIdOffset);
-        }
-
-        @Override
-        public boolean get(int index)
-        {
-            return !ignoredOrdinals.contains(index);
-        }
-
-        @Override
-        public int length()
-        {
-            return length;
-        }
-    }
-
-    private static class IgnoredKeysBits implements Bits
-    {
-        private final OnHeapGraph<PrimaryKey> graph;
-        private final NavigableSet<PrimaryKey> ignored;
-
-        public IgnoredKeysBits(OnHeapGraph<PrimaryKey> graph, NavigableSet<PrimaryKey> ignored)
-        {
-            this.graph = graph;
-            this.ignored = ignored;
-        }
-
-        @Override
-        public boolean get(int ordinal)
-        {
-            var keys = graph.keysFromOrdinal(ordinal);
-            return keys.stream().anyMatch(k -> !ignored.contains(k));
-        }
-
-        @Override
-        public int length()
-        {
-            return graph.size();
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
index f9878c2f3ff4..2fe53fd042a9 100644
--- a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
@@ -1,3 +1,9 @@
+/*
+ * All changes to the original code are Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -18,22 +24,38 @@
 
 package org.apache.cassandra.index.sai.analyzer;
 
+import java.io.Closeable;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
-import java.util.stream.Collectors;
+import java.util.Set;
 
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+
+import org.slf4j.Logger;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.lucene.analysis.Analyzer;
 
 public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
 {
+    private static final Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractAnalyzer.class);
+
+    public static final Set<AbstractType<?>> ANALYZABLE_TYPES = ImmutableSet.of(UTF8Type.instance, AsciiType.instance);
+
     protected ByteBuffer next = null;
-    protected String nextLiteral = null;
+    String nextLiteral = null;
 
     /**
-     * @return true if index value is transformed, e.g. normalized or lower-cased or tokenized.
+     * @return true if index value is transformed, eg. normalized or lower-cased or tokenized.
      */
     public abstract boolean transformValue();
 
@@ -73,44 +95,155 @@ public void reset(ByteBuffer input)
         resetInternal(input);
     }
 
-    public interface AnalyzerFactory
+    public static boolean hasQueryAnalyzer(Map<String, String> options)
+    {
+       return options.containsKey(LuceneAnalyzer.QUERY_ANALYZER);
+    }
+
+    public interface AnalyzerFactory extends Closeable
     {
         AbstractAnalyzer create();
 
+        /**
+         * @return true if the analyzer supports EQ queries (see {@link AnalyzerEqOperatorSupport})
+         */
+        default boolean supportsEquals()
+        {
+            return true;
+        }
+
+        /**
+         * @return {@link true} if this analyzer configuration has a n-gram tokenizer or any of its filters is n-gram.
+         */
+        default boolean isNGram()
+        {
+            return false;
+        }
+
+        @Override
         default void close()
         {
         }
     }
 
-    public static AnalyzerFactory fromOptions(IndexTermType indexTermType, Map<String, String> options)
+    public static AnalyzerFactory fromOptionsQueryAnalyzer(final AbstractType<?> type, final Map<String, String> options)
+    {
+        final String json = options.get(LuceneAnalyzer.QUERY_ANALYZER);
+        return toAnalyzerFactory(json, type, options);
+    }
+
+    public static AnalyzerFactory toAnalyzerFactory(String json, final AbstractType<?> type, final Map<String, String> options) //throws Exception
     {
-        if (hasNonTokenizingOptions(options))
+        if (!TypeUtil.isIn(type, ANALYZABLE_TYPES))
         {
-            if (indexTermType.isString())
-            {
-                // validate options
-                NonTokenizingOptions.fromMap(options);
-                return () -> new NonTokenizingAnalyzer(indexTermType, options);
-            }
-            else
-            {
-                throw new InvalidRequestException("CQL type " + indexTermType.asCQL3Type() + " cannot be analyzed.");
-            }
+            logger.warn("CQL type {} cannot be analyzed options={}; using NoOpAnalyzer", type.asCQL3Type(), options);
+            return NoOpAnalyzer::new;
         }
 
-        return null;
+        try
+        {
+            Pair<Analyzer, LuceneCustomAnalyzerConfig> analyzerAndConfig = JSONAnalyzerParser.parse(json);
+            final Analyzer analyzer = analyzerAndConfig.left;
+            final boolean isNGram = analyzerAndConfig.right != null && analyzerAndConfig.right.isNGram();
+
+            return new AnalyzerFactory()
+            {
+                @Override
+                public void close()
+                {
+                    analyzer.close();
+                }
+
+                @Override
+                public AbstractAnalyzer create()
+                {
+                    return new LuceneAnalyzer(type, analyzer, options);
+                }
+
+                @Override
+                public boolean isNGram()
+                {
+                    return isNGram;
+                }
+
+                @Override
+                public boolean supportsEquals()
+                {
+                    return AnalyzerEqOperatorSupport.supportsEqualsFromOptions(options);
+                }
+            };
+        }
+        catch (InvalidRequestException ex)
+        {
+            throw ex;
+        }
+        catch (Exception ex)
+        {
+            throw new InvalidRequestException("CQL type " + type.asCQL3Type() + " cannot be analyzed options="+options, ex);
+        }
     }
 
-    private static boolean hasNonTokenizingOptions(Map<String, String> options)
-    {
-        return options.keySet().stream().anyMatch(NonTokenizingOptions::hasOption);
+    public static boolean isAnalyzed(Map<String, String> options) {
+        return options.containsKey(LuceneAnalyzer.INDEX_ANALYZER) || NonTokenizingOptions.hasNonDefaultOptions(options);
     }
 
-    public static Map<String, String> getAnalyzerOptions(Map<String, String> options)
+    public static AnalyzerFactory fromOptions(String target, AbstractType<?> type, Map<String, String> options)
     {
-        return options.entrySet().stream()
-                      .filter(e -> NonTokenizingOptions.hasOption(e.getKey()))
-                      .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-    }
+        boolean containsIndexAnalyzer = options.containsKey(LuceneAnalyzer.INDEX_ANALYZER);
+        boolean containsNonTokenizingOptions = NonTokenizingOptions.hasNonDefaultOptions(options);
+        boolean supportsEquals = AnalyzerEqOperatorSupport.supportsEqualsFromOptions(options);
+        if (containsIndexAnalyzer && containsNonTokenizingOptions)
+        {
+            logger.warn("Invalid combination of options for index_analyzer: {}", options);
+            var optionsToStrip = List.of(NonTokenizingOptions.CASE_SENSITIVE, NonTokenizingOptions.NORMALIZE, NonTokenizingOptions.ASCII);
+            options = Maps.filterKeys(options, k -> !optionsToStrip.contains(k));
+            logger.warn("Rewrote options to {}", options);
+        }
+        boolean containsQueryAnalyzer = options.containsKey(LuceneAnalyzer.QUERY_ANALYZER);
+        if (containsQueryAnalyzer && !containsIndexAnalyzer && !containsNonTokenizingOptions)
+        {
+            throw new InvalidRequestException("Cannot specify query_analyzer without an index_analyzer option or any" +
+                                              " combination of case_sensitive, normalize, or ascii options. options=" + options);
+        }
 
+        if ((containsIndexAnalyzer || containsNonTokenizingOptions) && type.isCollection() && !type.isMultiCell())
+            throw new InvalidRequestException("Cannot use an analyzer on " + target + " because it's a frozen collection.");
+
+        if (containsIndexAnalyzer)
+        {
+            String json = options.get(LuceneAnalyzer.INDEX_ANALYZER);
+            return toAnalyzerFactory(json, type, options);
+        }
+
+        if (containsNonTokenizingOptions)
+        {
+            if (TypeUtil.isIn(type, ANALYZABLE_TYPES))
+            {
+                // load NonTokenizingAnalyzer so it'll validate options
+                NonTokenizingAnalyzer a = new NonTokenizingAnalyzer(type, options);
+                a.end();
+                Map<String, String> finalOptions = options;
+
+                return new AnalyzerFactory()
+                {
+                    @Override
+                    public AbstractAnalyzer create()
+                    {
+                        return new NonTokenizingAnalyzer(type, finalOptions);
+                    }
+
+                    @Override
+                    public boolean supportsEquals()
+                    {
+                        return supportsEquals;
+                    }
+                };
+            }
+            else
+            {
+                throw new InvalidRequestException("CQL type " + type.asCQL3Type() + " cannot be analyzed.");
+            }
+        }
+        return NoOpAnalyzer::new;
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/AnalyzerEqOperatorSupport.java b/src/java/org/apache/cassandra/index/sai/analyzer/AnalyzerEqOperatorSupport.java
new file mode 100644
index 000000000000..30408c9b986f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/AnalyzerEqOperatorSupport.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.util.Arrays;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+/**
+ * Index config property for defining the behaviour of the equals operator (=) when the index is analyzed.
+ * </p>
+ * Analyzers transform the indexed value, so EQ queries using an analyzed index can return results different to those of
+ * an equivalent query without indexes. Having EQ queries returning different results depending on if/how the column is
+ * indexed can be confusing for users, so probably the safest approach is to reject EQ queries on analyzed indexes, and
+ * let users use the analyzer matches operator (:) instead. However, for backwards compatibility reasons, we should
+ * allow users to let equality queries behave same as match queries through this index config property. We use an enum
+ * value rather than a boolean to allow for future extensions.
+ */
+public class AnalyzerEqOperatorSupport
+{
+    public static final String OPTION = "equals_behaviour_when_analyzed";
+    public static final Value DEFAULT = Value.MATCH; // default to : behaviour for backwards compatibility
+
+    @VisibleForTesting
+    static final String NOT_ANALYZED_ERROR = "The behaviour of the equals operator (=) cannot be " +
+                                             "defined with the '" + OPTION + "' index option because " +
+                                             "the index is not analyzed.";
+
+    @VisibleForTesting
+    static final String WRONG_OPTION_ERROR = String.format("Invalid value for '%s' option. " +
+                                                           "Possible values are %s but found ",
+                                                           OPTION, Arrays.toString(Value.values()));
+
+    public static final String EQ_RESTRICTION_ON_ANALYZED_WARNING =
+    String.format("Column [%%s] is restricted by '=' and has an analyzed index [%%s] able to process those restrictions. " +
+                  "Analyzed indexes might process '=' restrictions in a way that is inconsistent with non-indexed queries. " +
+                  "While '=' is still supported on analyzed indexes for backwards compatibility, " +
+                  "it is recommended to use the ':' operator instead to prevent the ambiguity. " +
+                  "Future versions will remove support for '=' on analyzed indexes. " +
+                  "If you want to forbid the use of '=' on analyzed indexes now, " +
+                  "please use '%s':'%s' in the index options.",
+                  OPTION, Value.UNSUPPORTED.toString().toLowerCase());
+
+    public static final String EQ_AMBIGUOUS_ERROR =
+    String.format("Column [%%s] equality predicate is ambiguous.  It has both an analyzed index [%%s] configured with '%s':'%s', " +
+                  "and an un-analyzed index [%%s].  " +
+                  "To avoid ambiguity, drop the analyzed index and recreate it with option '%s':'%s'.",
+                  OPTION, Value.MATCH.toString().toLowerCase(), OPTION, Value.UNSUPPORTED.toString().toLowerCase());
+
+
+    public static final String LWT_CONDITION_ON_ANALYZED_WARNING =
+    "Index analyzers not applied to LWT conditions on columns [%s].";
+
+    public enum Value
+    {
+        /**
+         * The index won't support equality (=) expressions on analyzed indexes.
+         */
+        UNSUPPORTED,
+        /**
+         * Allow equality (=) expressions on analyzed indexes. They will behave same as match queries (:).
+         */
+        MATCH
+    }
+
+    public static boolean supportsEqualsFromOptions(Map<String, String> options)
+    {
+        return fromMap(options) == Value.MATCH;
+    }
+
+    public static Value fromMap(Map<String, String> options)
+    {
+        if (options == null || !options.containsKey(OPTION))
+            return DEFAULT;
+
+        if (!AbstractAnalyzer.isAnalyzed(options))
+            throw new InvalidRequestException(NOT_ANALYZED_ERROR);
+
+        String option = options.get(OPTION).toUpperCase();
+        try
+        {
+            return Value.valueOf(option);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new InvalidRequestException(WRONG_OPTION_ERROR + option);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/ArgsStringLoader.java b/src/java/org/apache/cassandra/index/sai/analyzer/ArgsStringLoader.java
new file mode 100644
index 000000000000..c12518268334
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/ArgsStringLoader.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.lucene.util.ResourceLoader;
+
+/**
+ * A resource loader that considers each passed string as the resource. This class allows us to configure stop words
+ * and synonyms as arguments in the 'args' parameter of the filter's configuration.
+ * Example: WITH OPTIONS = {'index_analyzer':'{"tokenizer":{"name" : "whitespace"}, "filters":[{"name":"stop", "args": {"words": "the, test"}}]}'}
+ * The above configuration will create a stop filter with the words "the" and "test". The args key name, e.g. words,
+ * is specific to the lucene component being configured. The delimiter can vary based on the filter, but appears to
+ * be comma delimited for most lucene components. Note that commas can be escaped with a backslash.
+ */
+public class ArgsStringLoader implements ResourceLoader
+{
+    @Override
+    public InputStream openResource(String s) throws IOException
+    {
+        return new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Override
+    public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
+        try {
+            return Class.forName(cname).asSubclass(expectedType);
+        } catch (Exception e) {
+            throw new RuntimeException("Cannot load class: " + cname, e);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/ByteLimitedMaterializer.java b/src/java/org/apache/cassandra/index/sai/analyzer/ByteLimitedMaterializer.java
new file mode 100644
index 000000000000..3a5d2f57e2d3
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/ByteLimitedMaterializer.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+/**
+ * Utility class for conditionally materializing a list of tokens given an analyzer and the term to be analyzed.
+ * If the cumulative size of the analyzed term exceeds the configured maximum, an empty list is returned to prevent
+ * excessive memory/disk usage for a single term.
+ */
+public class ByteLimitedMaterializer
+{
+    private static final Logger logger = LoggerFactory.getLogger(ByteLimitedMaterializer.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+    public static final String ANALYZED_TERM_OVERSIZE_MESSAGE = "Cannot add term's analyzed tokens of column {} to index" +
+                                                                " for key: {}, analzyed term size {}, but max allowed size {}.";
+
+    /**
+     * Using the configured analyzer, materialize the tokens for the given term. If the cumulative size of the analyzed
+     * term exceeds the configured maximum, an empty list is returned to prevent excessive memory/disk usage for a
+     * single term. If the analyzer does not transform the value, the analyzer's byte limit is ignored because indexes
+     * already have a limit on the size of a single term.
+     * @param analyzer the analyzer to use
+     * @param term the term to analyze
+     * @param indexContext the index context used for logging
+     * @param primaryKey the primary key of the row being indexed
+     * @return all the terms produced by the analyzer, or an empty list if the cumulative size of the analyzed term
+     * exceeds the configured maximum
+     */
+    public static List<ByteBuffer> materializeTokens(AbstractAnalyzer analyzer, ByteBuffer term, IndexContext indexContext, PrimaryKey primaryKey)
+    {
+        try
+        {
+            analyzer.reset(term);
+            if (!analyzer.transformValue())
+                return analyzer.hasNext() ? List.of(analyzer.next()) : List.of();
+
+            List<ByteBuffer> tokens = new ArrayList<>();
+            int bytesCount = 0;
+            while (analyzer.hasNext())
+            {
+                final ByteBuffer token = analyzer.next();
+                tokens.add(token);
+                bytesCount += token.remaining();
+                if (bytesCount >= IndexContext.MAX_ANALYZED_SIZE)
+                {
+                    noSpamLogger.warn(indexContext.logMessage(ANALYZED_TERM_OVERSIZE_MESSAGE),
+                                      indexContext.getColumnName(),
+                                      indexContext.keyValidator().getString(primaryKey.partitionKey().getKey()),
+                                      FBUtilities.prettyPrintMemory(bytesCount),
+                                      FBUtilities.prettyPrintMemory(IndexContext.MAX_ANALYZED_SIZE));
+                    return List.of();
+                }
+            }
+            return tokens;
+        }
+        finally
+        {
+            analyzer.end();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/JSONAnalyzerParser.java b/src/java/org/apache/cassandra/index/sai/analyzer/JSONAnalyzerParser.java
new file mode 100644
index 000000000000..e3aa3f349faa
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/JSONAnalyzerParser.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.io.IOException;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.analyzer.filter.BuiltInAnalyzers;
+import org.apache.cassandra.utils.Pair;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharFilterFactory;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenizerFactory;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
+import org.apache.lucene.analysis.ngram.NGramTokenizerFactory;
+
+import static org.apache.cassandra.utils.JsonUtils.JSON_OBJECT_MAPPER;
+
+public class JSONAnalyzerParser
+{
+    public static Pair<Analyzer, LuceneCustomAnalyzerConfig> parse(String json) throws IOException
+    {
+        Analyzer analyzer = matchBuiltInAnalzyer(json.toUpperCase());
+        if (analyzer != null)
+        {
+            return Pair.create(analyzer, null);
+        }
+
+        LuceneCustomAnalyzerConfig analyzerModel;
+        try
+        {
+            // Don't have built in analyzer, parse JSON
+            analyzerModel = JSON_OBJECT_MAPPER.readValue(json, LuceneCustomAnalyzerConfig.class);
+        }
+        catch (UnrecognizedPropertyException e)
+        {
+            throw new InvalidRequestException("Invalid field name '" + e.getPropertyName() + "' in analyzer config. Valid fields are: [tokenizer, filters, charFilters]");
+        }
+        catch (IOException e)
+        {
+            throw new InvalidRequestException("Invalid analyzer config: " + e.getMessage());
+        }
+
+        CustomAnalyzer.Builder builder = CustomAnalyzer.builder(new ArgsStringLoader());
+        // An ommitted tokenizer maps directly to the keyword tokenizer, which is an identity map on input terms
+        if (analyzerModel.getTokenizer() == null)
+        {
+            if (analyzerModel.getFilters().isEmpty() && analyzerModel.getCharFilters().isEmpty())
+            {
+                throw new InvalidRequestException("Analzyer config requires at least a tokenizer, a filter, or a charFilter, but none found. config=" + json);
+            }
+            builder.withTokenizer("keyword");
+        }
+        else
+        {
+            String name = analyzerModel.getTokenizer().getName();
+            try
+            {
+                // Validate before attempting to build the tokenizer so we can provide a more helpful error message.
+                // We use lookupClass because it does an internal lowercase to match the class name, which we cannot
+                // easily do because the list of available tokenizers is loaded via reflection.
+                TokenizerFactory.lookupClass(name);
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new InvalidRequestException("Unknown tokenizer '" + name + "'. Valid options: " + TokenizerFactory.availableTokenizers());
+            }
+
+            Map<String, String> args = analyzerModel.getTokenizer().getArgs();
+            try
+            {
+                builder.withTokenizer(name, applyTokenizerDefaults(name, args));
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new InvalidRequestException("Error configuring analyzer's tokenizer '" + name + "': " + e.getMessage());
+            }
+        }
+        for (LuceneClassNameAndArgs filter : analyzerModel.getFilters())
+        {
+            if (filter.getName() == null)
+                throw new InvalidRequestException("filter 'name' field is required for options=" + json);
+
+            try
+            {
+                // Validate before attempting to build the filter so we can provide a more helpful error message.
+                // We use lookupClass because it does an internal lowercase to match the class name, which we cannot
+                // easily do because the list of available tokenizers is loaded via reflection.
+                TokenFilterFactory.lookupClass(filter.getName());
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new InvalidRequestException("Unknown filter '" + filter.getName() + "'. Valid options: " + TokenFilterFactory.availableTokenFilters());
+            }
+
+            try
+            {
+                builder.addTokenFilter(filter.getName(), filter.getArgs());
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new InvalidRequestException("Error configuring analyzer's filter '" + filter.getName() + "': " + e.getMessage());
+            }
+        }
+
+        for (LuceneClassNameAndArgs charFilter : analyzerModel.getCharFilters())
+        {
+            if (charFilter.getName() == null)
+                throw new InvalidRequestException("charFilter 'name' field is required for options=" + json);
+
+            try
+            {
+                // Validate before attempting to build the charFilter so we can provide a more helpful error message.
+                // We use lookupClass because it does an internal lowercase to match the class name, which we cannot
+                // easily do because the list of available tokenizers is loaded via reflection.
+                CharFilterFactory.lookupClass(charFilter.getName());
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new InvalidRequestException("Unknown charFilter '" + charFilter.getName() + "'. Valid options: " + CharFilterFactory.availableCharFilters());
+            }
+
+            try
+            {
+                builder.addCharFilter(charFilter.getName(), charFilter.getArgs());
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new InvalidRequestException("Error configuring analyzer's charFilter '" + charFilter.getName() + "': " + e.getMessage());
+            }
+        }
+        return Pair.create(builder.build(), analyzerModel);
+    }
+
+    private static Analyzer matchBuiltInAnalzyer(String maybeAnalyzer)
+    {
+        for (BuiltInAnalyzers analyzer : BuiltInAnalyzers.values())
+        {
+            if (analyzer.name().equals(maybeAnalyzer))
+            {
+                return analyzer.getNewAnalyzer();
+            }
+        }
+        return null;
+    }
+
+    private static Map<String, String> applyTokenizerDefaults(String filterName, Map<String, String> args)
+    {
+        if (NGramTokenizerFactory.NAME.equalsIgnoreCase(filterName))
+        {
+            // Lucene's defaults are 1 and 2 respectively, which has a large memory overhead.
+            args.putIfAbsent("minGramSize", "3");
+            args.putIfAbsent("maxGramSize", "7");
+        }
+        return args;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/LuceneAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/LuceneAnalyzer.java
new file mode 100644
index 000000000000..0890e0333ca1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/LuceneAnalyzer.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.io.CharArrayReader;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import com.google.common.base.MoreObjects;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.disk.io.BytesRefUtil;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
+
+public class LuceneAnalyzer extends AbstractAnalyzer
+{
+    public static final String INDEX_ANALYZER = "index_analyzer";
+    public static final String QUERY_ANALYZER = "query_analyzer";
+    private AbstractType<?> type;
+    private boolean hasNext = false;
+
+    private final Analyzer analyzer;
+    private TokenStream tokenStream;
+    private final CharsRefBuilder charsBuilder = new CharsRefBuilder();
+    private TermToBytesRefAttribute termAttr;
+    private final BytesRefBuilder bytesBuilder = new BytesRefBuilder();
+    private final Map<String, String> options;
+
+    public LuceneAnalyzer(AbstractType<?> type, Analyzer analyzer, Map<String, String> options)
+    {
+        this.type = type;
+        this.analyzer = analyzer;
+        this.options = options;
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        if (tokenStream == null)
+        {
+            throw new IllegalStateException("resetInternal(ByteBuffer term) must be called prior to hasNext()");
+        }
+        try
+        {
+            hasNext = tokenStream.incrementToken();
+
+            if (hasNext)
+            {
+                final BytesRef br = termAttr.getBytesRef();
+                // TODO: should be able to reuse the bytes ref however
+                //       MemoryIndex#setMinMaxTerm requires a copy
+                //       getting the max term from the mem trie is best
+                next = ByteBuffer.wrap(BytesRef.deepCopyOf(br).bytes);
+            }
+            return hasNext;
+        }
+        catch (IOException ex)
+        {
+            throw Throwables.cleaned(ex);
+        }
+    }
+
+    @Override
+    public ByteBuffer next()
+    {
+        if (!hasNext)
+        {
+            throw new NoSuchElementException();
+        }
+        return next;
+    }
+
+    @Override
+    public void end()
+    {
+        if (tokenStream == null)
+            return;
+        try
+        {
+            try
+            {
+                tokenStream.end();
+            }
+            finally
+            {
+                tokenStream.close();
+            }
+        }
+        catch (IOException ex)
+        {
+            throw Throwables.cleaned(ex); // highly unlikely exception
+        }
+    }
+
+    @Override
+    public boolean transformValue()
+    {
+        return true;
+    }
+
+    @Override
+    protected void resetInternal(ByteBuffer input)
+    {
+        try
+        {
+            // the following uses a byte[] and char[] buffer to reduce object creation
+            BytesRefUtil.copyBufferToBytesRef(input, bytesBuilder);
+
+            charsBuilder.copyUTF8Bytes(bytesBuilder.get());
+
+            final CharsRef charsRef = charsBuilder.get();
+
+            // the field name doesn't matter here, it's an internal lucene thing
+            tokenStream = analyzer.tokenStream("field", new CharArrayReader(charsRef.chars, charsRef.offset, charsRef.length));
+            tokenStream.reset();
+            termAttr = tokenStream.getAttribute(TermToBytesRefAttribute.class);
+
+            this.hasNext = true;
+        }
+        catch (Exception ex)
+        {
+            throw Throwables.cleaned(ex);
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("type", type)
+                          .add("hasNext", hasNext)
+                          .add("analyzer", analyzer)
+                          .add("tokenStream", tokenStream)
+                          .add("termAttr", termAttr)
+                          .add("options", options)
+                          .toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/LuceneClassNameAndArgs.java b/src/java/org/apache/cassandra/index/sai/analyzer/LuceneClassNameAndArgs.java
new file mode 100644
index 000000000000..1b44b0238d35
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/LuceneClassNameAndArgs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * A class representing the name of a Lucene class and a map of arguments to pass as configuration.
+ */
+public class LuceneClassNameAndArgs
+{
+    private final String name;
+    private final Map<String, String> args;
+
+    public LuceneClassNameAndArgs(@JsonProperty("name") String name,
+                                  @JsonProperty("args") Map<String, String> args)
+    {
+        this.name = name;
+        this.args = args != null ? args : new HashMap<>();
+    }
+
+    public String getName()
+    {
+        return name;
+    }
+
+    public Map<String, String> getArgs()
+    {
+        return args;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/LuceneCustomAnalyzerConfig.java b/src/java/org/apache/cassandra/index/sai/analyzer/LuceneCustomAnalyzerConfig.java
new file mode 100644
index 000000000000..de8bfbe6091a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/LuceneCustomAnalyzerConfig.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class LuceneCustomAnalyzerConfig
+{
+    private final LuceneClassNameAndArgs tokenizer;
+    private final List<LuceneClassNameAndArgs> filters;
+    private final List<LuceneClassNameAndArgs> charFilters;
+
+    public LuceneCustomAnalyzerConfig(@JsonProperty("tokenizer") LuceneClassNameAndArgs tokenizer,
+                                      @JsonProperty("filters") List<LuceneClassNameAndArgs> filters,
+                                      @JsonProperty("charFilters") List<LuceneClassNameAndArgs> charFilters)
+    {
+        this.tokenizer = tokenizer;
+        this.filters = filters != null ? filters : List.of();
+        this.charFilters = charFilters != null ? charFilters : List.of();
+    }
+
+    public LuceneClassNameAndArgs getTokenizer()
+    {
+        return tokenizer;
+    }
+
+    public List<LuceneClassNameAndArgs> getFilters()
+    {
+        return filters;
+    }
+
+    public List<LuceneClassNameAndArgs> getCharFilters()
+    {
+        return charFilters;
+    }
+
+    /**
+     * @return {@link true} if this analyzer configuration has a n-gram tokenizer or any of its filters is n-gram.
+     */
+    public boolean isNGram()
+    {
+        if (getTokenizer().getName().equals("ngram"))
+            return true;
+
+        for (LuceneClassNameAndArgs filter : getFilters())
+        {
+            if (filter.getName().equals("ngram"))
+                return true;
+        }
+
+        return false;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java
new file mode 100644
index 000000000000..d8ae78ae6297
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.nio.ByteBuffer;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Default noOp tokenizer. The iterator will iterate only once
+ * returning the unmodified input
+ */
+public class NoOpAnalyzer extends AbstractAnalyzer
+{
+    private ByteBuffer input;
+    private boolean hasNext = false;
+
+    @SuppressWarnings("unused")
+    NoOpAnalyzer() {}
+
+    @Override
+    public boolean hasNext()
+    {
+        if (hasNext)
+        {
+            this.next = input;
+            this.hasNext = false;
+            return true;
+        }
+        this.next = null;
+        return false;
+    }
+
+    @Override
+    protected void resetInternal(ByteBuffer input)
+    {
+        this.input = input;
+        this.hasNext = true;
+    }
+
+    @Override
+    public boolean transformValue()
+    {
+        return false;
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this).toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java
index 30eedc08f39b..8a46dbbad595 100644
--- a/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java
@@ -25,36 +25,38 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.index.sai.analyzer.filter.BasicFilters;
-import org.apache.cassandra.index.sai.analyzer.filter.FilterPipeline;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.analyzer.filter.BasicResultFilters;
+import org.apache.cassandra.index.sai.analyzer.filter.FilterPipelineBuilder;
 import org.apache.cassandra.index.sai.analyzer.filter.FilterPipelineExecutor;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.analyzer.filter.FilterPipelineTask;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 /**
  * Analyzer that does *not* tokenize the input. Optionally will
- * apply filters for the input based on {@link NonTokenizingOptions}.
+ * apply filters for the input output as defined in analyzers options
  */
 public class NonTokenizingAnalyzer extends AbstractAnalyzer
 {
     private static final Logger logger = LoggerFactory.getLogger(NonTokenizingAnalyzer.class);
 
-    private final IndexTermType indexTermType;
-    private final NonTokenizingOptions options;
-    private final FilterPipeline filterPipeline;
+    private AbstractType<?> type;
+    private NonTokenizingOptions options;
+    private FilterPipelineTask filterPipeline;
 
     private ByteBuffer input;
     private boolean hasNext = false;
 
-    NonTokenizingAnalyzer(IndexTermType indexTermType, Map<String, String> options)
+    NonTokenizingAnalyzer(AbstractType<?> type, Map<String, String> options)
     {
-        this(indexTermType, NonTokenizingOptions.fromMap(options));
+        this(type, NonTokenizingOptions.fromMap(options));
     }
 
-    NonTokenizingAnalyzer(IndexTermType indexTermType, NonTokenizingOptions tokenizerOptions)
+    NonTokenizingAnalyzer(AbstractType<?> type, NonTokenizingOptions tokenizerOptions)
     {
-        this.indexTermType = indexTermType;
+        this.type = type;
         this.options = tokenizerOptions;
         this.filterPipeline = getFilterPipeline();
     }
@@ -63,19 +65,18 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer
     public boolean hasNext()
     {
         // check that we know how to handle the input, otherwise bail
-        if (!indexTermType.isString())
-            return false;
+        if (!TypeUtil.isIn(type, ANALYZABLE_TYPES)) return false;
 
         if (hasNext)
         {
             try
             {
-                String input = indexTermType.asString(this.input);
+                String input = type.getString(this.input);
 
                 if (input == null)
                 {
                     throw new MarshalException(String.format("'null' deserialized value for %s with %s",
-                                                             ByteBufferUtil.bytesToHex(this.input), indexTermType));
+                                                             ByteBufferUtil.bytesToHex(this.input), type));
                 }
 
                 String result = FilterPipelineExecutor.execute(filterPipeline, input);
@@ -88,13 +89,13 @@ public boolean hasNext()
                 }
 
                 nextLiteral = result;
-                next = indexTermType.fromString(result);
+                next = type.fromString(result);
 
                 return true;
             }
             catch (MarshalException e)
             {
-                logger.error("Failed to deserialize value with " + indexTermType, e);
+                logger.error("Failed to deserialize value with " + type, e);
                 return false;
             }
             finally
@@ -119,20 +120,26 @@ protected void resetInternal(ByteBuffer input)
         this.hasNext = true;
     }
 
-    private FilterPipeline getFilterPipeline()
+    private FilterPipelineTask getFilterPipeline()
     {
-        FilterPipeline builder = new FilterPipeline(new BasicFilters.NoOperation());
+        FilterPipelineBuilder builder = new FilterPipelineBuilder(new BasicResultFilters.NoOperation());
         
         if (!options.isCaseSensitive())
-            builder = builder.add("to_lower", new BasicFilters.LowerCase());
+        {
+            builder = builder.add("to_lower", new BasicResultFilters.LowerCase());
+        }
         
         if (options.isNormalized())
-            builder = builder.add("normalize", new BasicFilters.Normalize());
+        {
+            builder = builder.add("normalize", new BasicResultFilters.Normalize());
+        }
 
         if (options.isAscii())
-            builder = builder.add("ascii", new BasicFilters.Ascii());
+        {
+            builder = builder.add("ascii", new BasicResultFilters.Ascii());
+        }
         
-        return builder;
+        return builder.build();
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingOptions.java b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingOptions.java
index 66c7740a5393..bb823c5997a0 100644
--- a/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingOptions.java
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingOptions.java
@@ -30,6 +30,9 @@ public class NonTokenizingOptions
     public static final String NORMALIZE = "normalize";
     public static final String CASE_SENSITIVE = "case_sensitive";
     public static final String ASCII = "ascii";
+    static final boolean NORMALIZE_DEFAULT = false;
+    static final boolean CASE_SENSITIVE_DEFAULT = true;
+    static final boolean ASCII_DEFAULT = false;
 
     private boolean caseSensitive;
     private boolean normalized;
@@ -65,16 +68,11 @@ void setNormalized(boolean normalized)
         this.normalized = normalized;
     }
 
-    static boolean hasOption(String option)
-    {
-        return option.equals(NORMALIZE) || option.equals(CASE_SENSITIVE) || option.equals(ASCII);
-    }
-
     public static class OptionsBuilder
     {
-        private boolean caseSensitive = true;
-        private boolean normalized = false;
-        private boolean ascii = false;
+        private boolean caseSensitive = CASE_SENSITIVE_DEFAULT;
+        private boolean normalized = NORMALIZE_DEFAULT;
+        private boolean ascii = ASCII_DEFAULT;
 
         OptionsBuilder() {}
 
@@ -108,7 +106,7 @@ public NonTokenizingOptions build()
 
     public static NonTokenizingOptions getDefaultOptions()
     {
-        return fromMap(new HashMap<>(1));
+        return fromMap(new HashMap(1));
     }
 
     public static NonTokenizingOptions fromMap(Map<String, String> options)
@@ -148,7 +146,7 @@ private static boolean validateBoolean(String value, String option)
     {
         if (Strings.isNullOrEmpty(value))
         {
-            throw new InvalidRequestException("Empty value for boolean option '" + option + '\'');
+            throw new InvalidRequestException("Empty value for boolean option '" + option + "'");
         }
 
         if (!value.equalsIgnoreCase(Boolean.TRUE.toString()) && !value.equalsIgnoreCase(Boolean.FALSE.toString()))
@@ -158,4 +156,23 @@ private static boolean validateBoolean(String value, String option)
 
         return Boolean.parseBoolean(value);
     }
+
+    /**
+     * Returns true if any of the options are set to a non-default value. Can be used to determine whether the
+     * parameterized OPTIONS should be used to construct a {@link NonTokenizingAnalyzer}
+     * or a {@link NoOpAnalyzer} instance.
+     * @param options - index options
+     * @return true if and only if any of the options are set to a non-default value.
+     */
+    static boolean hasNonDefaultOptions(Map<String, String> options) {
+        return hasNonDefaultBooleanOption(options.get(CASE_SENSITIVE), CASE_SENSITIVE_DEFAULT)
+               || hasNonDefaultBooleanOption(options.get(NORMALIZE), NORMALIZE_DEFAULT)
+               || hasNonDefaultBooleanOption(options.get(ASCII), ASCII_DEFAULT);
+    }
+
+    private static boolean hasNonDefaultBooleanOption(String value, boolean defaultValue)
+    {
+        // Use string equality here to preven the need to parse the input string value.
+        return value != null && !Boolean.toString(defaultValue).equalsIgnoreCase(value);
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/README.md b/src/java/org/apache/cassandra/index/sai/analyzer/README.md
new file mode 100644
index 000000000000..f4c6099abe00
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/README.md
@@ -0,0 +1,160 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Configuring an SAI with an Analyzer
+
+Analyzers are built on the Lucene Java Analyzer API. The SAI uses the Lucene Java Analyzer API to transform text columns into tokens for indexing and querying. The SAI supports the use of built-in analyzers and custom analyzers.
+
+## Defining an Analyzer
+
+Analyzers have one `tokenizer`, a list of `filters`, and a list of `charFilters`. The `tokenizer` splits the input text into tokens. The `filters` and `charFilters` transform the tokens into a form that is suitable for indexing and querying. The `filters` and `charFilters` are applied in the order they are defined in the configuration. The `filters` and the `charFilters` are optional.
+
+## Configuration Formatting
+
+The `OPTIONS` configuration argument is formatted as a JSON object:
+
+```
+OPTIONS = { 'index_analyzer' : '<BUILT_IN_ANALYZER>' }
+```
+
+OR
+
+```
+OPTIONS = {
+  'index_analyzer':
+  {
+    "tokenizer" : {
+      "name" : "",
+      "args" : {}
+    },
+    "filters" : [
+      {
+        "name" : "", 
+        "args": {}
+      }
+    ], 
+    "charFilters" : [
+      {
+        "name" : "",
+        "args": {}
+      }
+    ]
+  }
+}
+```
+
+## Built-in Analyzers
+
+The following built-in analyzers are available:
+
+| Analyzer Name | Description from Lucene Java Docs                                                                        |
+|---------------|----------------------------------------------------------------------------------------------------------|
+| `standard`    | Filters `StandardTokenizer` output with `LowerCaseFilter`                                                |
+| `simple`      | Filters `LetterTokenizer` output with `LowerCaseFilter`                                                  |
+| `whitespace`  | Analyzer that uses `WhitespaceTokenizer`.                                                                |
+| `stop`        | Filters `LetterTokenizer` output with `LowerCaseFilter` and removes Lucene's default English stop words. |
+| `lowercase`   | Normalizes input by applying `LowerCaseFilter` (no additional tokenization is performed).                |
+| `keyword`     | Analyzer that uses `KeywordTokenizer`, which is an identity function on input values.                    |
+| `<language>`  | Analyzers for specific languages. For example, `english` and `french`.                                   |
+
+### Standard Analyzer
+
+Here is the custom analyzer configuration for the standard analyzer:
+
+```
+OPTIONS = {
+  'index_analyzer':
+  '{
+    "tokenizer" : {
+      "name" : "standard",
+      "args" : {}
+    },
+    "filters" : [
+      {
+        "name" : "lowercase", 
+        "args": {}
+      }
+    ], 
+    "charFilters" : []
+  }'
+}
+```
+
+### Simple Analyzer
+
+Here is the custom analyzer configuration for the simple analyzer:
+
+```
+OPTIONS = {
+  'index_analyzer':
+  '{
+    "tokenizer" : {
+      "name" : "letter",
+      "args" : {}
+    },
+    "filters" : [
+      {
+        "name" : "lowercase", 
+        "args": {}
+      }
+    ], 
+    "charFilters" : []
+  }'
+}
+```
+
+### Whitespace Analyzer
+
+Here is the custom analyzer configuration for the whitespace analyzer:
+
+```
+OPTIONS = {
+  'index_analyzer':
+  '{
+    "tokenizer" : {
+      "name" : "whitespace",
+      "args" : {}
+    },
+    "filters" : [], 
+    "charFilters" : []
+  }'
+}
+```
+
+### Lowercase Analyzer
+
+Here is the custom analyzer configuration for the lowercase analyzer:
+
+```
+OPTIONS = {
+ 'index_analyzer':
+  '{
+    "tokenizer" : {
+      "name" : "keyword",
+      "args" : {}
+    },
+    "filters" : [
+      {
+        "name" : "lowercase", 
+        "args": {}
+      }
+    ], 
+    "charFilters" : []
+  }'
+}
+```
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/filter/BasicResultFilters.java b/src/java/org/apache/cassandra/index/sai/analyzer/filter/BasicResultFilters.java
new file mode 100644
index 000000000000..b5ad225f65fe
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/filter/BasicResultFilters.java
@@ -0,0 +1,2007 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+import java.text.Normalizer;
+import java.util.Locale;
+
+/**
+ * Basic/General Token Filters
+ */
+public class BasicResultFilters
+{
+    private static final Locale DEFAULT_LOCALE = Locale.getDefault();
+
+    public static class LowerCase extends FilterPipelineTask
+    {
+        private final Locale locale;
+
+        public LowerCase()
+        {
+            this.locale = DEFAULT_LOCALE;
+        }
+
+        public String process(String input)
+        {
+            return input.toLowerCase(locale);
+        }
+    }
+
+    public static class Normalize extends FilterPipelineTask
+    {
+        public Normalize() { }
+
+        public String process(String input)
+        {
+            if (input == null) return null;
+            return Normalizer.isNormalized(input, Normalizer.Form.NFC) ? input : Normalizer.normalize(input, Normalizer.Form.NFC);
+        }
+    }
+
+    public static class Ascii extends FilterPipelineTask
+    {
+        public Ascii() { }
+
+        public String process(String input)
+        {
+            if (input == null) return null;
+            char[] inputChars = input.toCharArray();
+            // The output can (potentially) be 4 times the size of the input
+            char[] outputChars = new char[inputChars.length * 4];
+            int outputSize = foldToASCII(inputChars, 0, outputChars, 0, inputChars.length);
+            return new String(outputChars, 0, outputSize);
+        }
+    }
+
+    public static class NoOperation extends FilterPipelineTask
+    {
+        public String process(String input)
+        {
+            return input;
+        }
+    }
+
+    // copied from lucene org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter
+    public static final int foldToASCII(char input[], int inputPos, char output[], int outputPos, int length)
+    {
+        final int end = inputPos + length;
+        for (int pos = inputPos; pos < end ; ++pos) {
+            final char c = input[pos];
+
+            // Quick test: if it's not in range then just keep current character
+            if (c < '\u0080') {
+                output[outputPos++] = c;
+            } else {
+                switch (c) {
+                    case '\u00C0': // À  [LATIN CAPITAL LETTER A WITH GRAVE]
+                    case '\u00C1': // Á  [LATIN CAPITAL LETTER A WITH ACUTE]
+                    case '\u00C2': // Â  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX]
+                    case '\u00C3': // Ã  [LATIN CAPITAL LETTER A WITH TILDE]
+                    case '\u00C4': // Ä  [LATIN CAPITAL LETTER A WITH DIAERESIS]
+                    case '\u00C5': // Å  [LATIN CAPITAL LETTER A WITH RING ABOVE]
+                    case '\u0100': // Ā  [LATIN CAPITAL LETTER A WITH MACRON]
+                    case '\u0102': // Ă  [LATIN CAPITAL LETTER A WITH BREVE]
+                    case '\u0104': // Ą  [LATIN CAPITAL LETTER A WITH OGONEK]
+                    case '\u018F': // Ə  http://en.wikipedia.org/wiki/Schwa  [LATIN CAPITAL LETTER SCHWA]
+                    case '\u01CD': // Ǎ  [LATIN CAPITAL LETTER A WITH CARON]
+                    case '\u01DE': // Ǟ  [LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON]
+                    case '\u01E0': // Ǡ  [LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON]
+                    case '\u01FA': // Ǻ  [LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE]
+                    case '\u0200': // Ȁ  [LATIN CAPITAL LETTER A WITH DOUBLE GRAVE]
+                    case '\u0202': // Ȃ  [LATIN CAPITAL LETTER A WITH INVERTED BREVE]
+                    case '\u0226': // Ȧ  [LATIN CAPITAL LETTER A WITH DOT ABOVE]
+                    case '\u023A': // Ⱥ  [LATIN CAPITAL LETTER A WITH STROKE]
+                    case '\u1D00': // ᴀ  [LATIN LETTER SMALL CAPITAL A]
+                    case '\u1E00': // Ḁ  [LATIN CAPITAL LETTER A WITH RING BELOW]
+                    case '\u1EA0': // Ạ  [LATIN CAPITAL LETTER A WITH DOT BELOW]
+                    case '\u1EA2': // Ả  [LATIN CAPITAL LETTER A WITH HOOK ABOVE]
+                    case '\u1EA4': // Ấ  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE]
+                    case '\u1EA6': // Ầ  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE]
+                    case '\u1EA8': // Ẩ  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE]
+                    case '\u1EAA': // Ẫ  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE]
+                    case '\u1EAC': // Ậ  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW]
+                    case '\u1EAE': // Ắ  [LATIN CAPITAL LETTER A WITH BREVE AND ACUTE]
+                    case '\u1EB0': // Ằ  [LATIN CAPITAL LETTER A WITH BREVE AND GRAVE]
+                    case '\u1EB2': // Ẳ  [LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE]
+                    case '\u1EB4': // Ẵ  [LATIN CAPITAL LETTER A WITH BREVE AND TILDE]
+                    case '\u1EB6': // Ặ  [LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW]
+                    case '\u24B6': // Ⓐ  [CIRCLED LATIN CAPITAL LETTER A]
+                    case '\uFF21': // Ａ  [FULLWIDTH LATIN CAPITAL LETTER A]
+                        output[outputPos++] = 'A';
+                        break;
+                    case '\u00E0': // à  [LATIN SMALL LETTER A WITH GRAVE]
+                    case '\u00E1': // á  [LATIN SMALL LETTER A WITH ACUTE]
+                    case '\u00E2': // â  [LATIN SMALL LETTER A WITH CIRCUMFLEX]
+                    case '\u00E3': // ã  [LATIN SMALL LETTER A WITH TILDE]
+                    case '\u00E4': // ä  [LATIN SMALL LETTER A WITH DIAERESIS]
+                    case '\u00E5': // å  [LATIN SMALL LETTER A WITH RING ABOVE]
+                    case '\u0101': // ā  [LATIN SMALL LETTER A WITH MACRON]
+                    case '\u0103': // ă  [LATIN SMALL LETTER A WITH BREVE]
+                    case '\u0105': // ą  [LATIN SMALL LETTER A WITH OGONEK]
+                    case '\u01CE': // ǎ  [LATIN SMALL LETTER A WITH CARON]
+                    case '\u01DF': // ǟ  [LATIN SMALL LETTER A WITH DIAERESIS AND MACRON]
+                    case '\u01E1': // ǡ  [LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON]
+                    case '\u01FB': // ǻ  [LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE]
+                    case '\u0201': // ȁ  [LATIN SMALL LETTER A WITH DOUBLE GRAVE]
+                    case '\u0203': // ȃ  [LATIN SMALL LETTER A WITH INVERTED BREVE]
+                    case '\u0227': // ȧ  [LATIN SMALL LETTER A WITH DOT ABOVE]
+                    case '\u0250': // ɐ  [LATIN SMALL LETTER TURNED A]
+                    case '\u0259': // ə  [LATIN SMALL LETTER SCHWA]
+                    case '\u025A': // ɚ  [LATIN SMALL LETTER SCHWA WITH HOOK]
+                    case '\u1D8F': // ᶏ  [LATIN SMALL LETTER A WITH RETROFLEX HOOK]
+                    case '\u1D95': // ᶕ  [LATIN SMALL LETTER SCHWA WITH RETROFLEX HOOK]
+                    case '\u1E01': // ạ  [LATIN SMALL LETTER A WITH RING BELOW]
+                    case '\u1E9A': // ả  [LATIN SMALL LETTER A WITH RIGHT HALF RING]
+                    case '\u1EA1': // ạ  [LATIN SMALL LETTER A WITH DOT BELOW]
+                    case '\u1EA3': // ả  [LATIN SMALL LETTER A WITH HOOK ABOVE]
+                    case '\u1EA5': // ấ  [LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE]
+                    case '\u1EA7': // ầ  [LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE]
+                    case '\u1EA9': // ẩ  [LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE]
+                    case '\u1EAB': // ẫ  [LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE]
+                    case '\u1EAD': // ậ  [LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW]
+                    case '\u1EAF': // ắ  [LATIN SMALL LETTER A WITH BREVE AND ACUTE]
+                    case '\u1EB1': // ằ  [LATIN SMALL LETTER A WITH BREVE AND GRAVE]
+                    case '\u1EB3': // ẳ  [LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE]
+                    case '\u1EB5': // ẵ  [LATIN SMALL LETTER A WITH BREVE AND TILDE]
+                    case '\u1EB7': // ặ  [LATIN SMALL LETTER A WITH BREVE AND DOT BELOW]
+                    case '\u2090': // ₐ  [LATIN SUBSCRIPT SMALL LETTER A]
+                    case '\u2094': // ₔ  [LATIN SUBSCRIPT SMALL LETTER SCHWA]
+                    case '\u24D0': // ⓐ  [CIRCLED LATIN SMALL LETTER A]
+                    case '\u2C65': // ⱥ  [LATIN SMALL LETTER A WITH STROKE]
+                    case '\u2C6F': // Ɐ  [LATIN CAPITAL LETTER TURNED A]
+                    case '\uFF41': // ａ  [FULLWIDTH LATIN SMALL LETTER A]
+                        output[outputPos++] = 'a';
+                        break;
+                    case '\uA732': // Ꜳ  [LATIN CAPITAL LETTER AA]
+                        output[outputPos++] = 'A';
+                        output[outputPos++] = 'A';
+                        break;
+                    case '\u00C6': // Æ  [LATIN CAPITAL LETTER AE]
+                    case '\u01E2': // Ǣ  [LATIN CAPITAL LETTER AE WITH MACRON]
+                    case '\u01FC': // Ǽ  [LATIN CAPITAL LETTER AE WITH ACUTE]
+                    case '\u1D01': // ᴁ  [LATIN LETTER SMALL CAPITAL AE]
+                        output[outputPos++] = 'A';
+                        output[outputPos++] = 'E';
+                        break;
+                    case '\uA734': // Ꜵ  [LATIN CAPITAL LETTER AO]
+                        output[outputPos++] = 'A';
+                        output[outputPos++] = 'O';
+                        break;
+                    case '\uA736': // Ꜷ  [LATIN CAPITAL LETTER AU]
+                        output[outputPos++] = 'A';
+                        output[outputPos++] = 'U';
+                        break;
+                    case '\uA738': // Ꜹ  [LATIN CAPITAL LETTER AV]
+                    case '\uA73A': // Ꜻ  [LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR]
+                        output[outputPos++] = 'A';
+                        output[outputPos++] = 'V';
+                        break;
+                    case '\uA73C': // Ꜽ  [LATIN CAPITAL LETTER AY]
+                        output[outputPos++] = 'A';
+                        output[outputPos++] = 'Y';
+                        break;
+                    case '\u249C': // ⒜  [PARENTHESIZED LATIN SMALL LETTER A]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'a';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\uA733': // ꜳ  [LATIN SMALL LETTER AA]
+                        output[outputPos++] = 'a';
+                        output[outputPos++] = 'a';
+                        break;
+                    case '\u00E6': // æ  [LATIN SMALL LETTER AE]
+                    case '\u01E3': // ǣ  [LATIN SMALL LETTER AE WITH MACRON]
+                    case '\u01FD': // ǽ  [LATIN SMALL LETTER AE WITH ACUTE]
+                    case '\u1D02': // ᴂ  [LATIN SMALL LETTER TURNED AE]
+                        output[outputPos++] = 'a';
+                        output[outputPos++] = 'e';
+                        break;
+                    case '\uA735': // ꜵ  [LATIN SMALL LETTER AO]
+                        output[outputPos++] = 'a';
+                        output[outputPos++] = 'o';
+                        break;
+                    case '\uA737': // ꜷ  [LATIN SMALL LETTER AU]
+                        output[outputPos++] = 'a';
+                        output[outputPos++] = 'u';
+                        break;
+                    case '\uA739': // ꜹ  [LATIN SMALL LETTER AV]
+                    case '\uA73B': // ꜻ  [LATIN SMALL LETTER AV WITH HORIZONTAL BAR]
+                        output[outputPos++] = 'a';
+                        output[outputPos++] = 'v';
+                        break;
+                    case '\uA73D': // ꜽ  [LATIN SMALL LETTER AY]
+                        output[outputPos++] = 'a';
+                        output[outputPos++] = 'y';
+                        break;
+                    case '\u0181': // Ɓ  [LATIN CAPITAL LETTER B WITH HOOK]
+                    case '\u0182': // Ƃ  [LATIN CAPITAL LETTER B WITH TOPBAR]
+                    case '\u0243': // Ƀ  [LATIN CAPITAL LETTER B WITH STROKE]
+                    case '\u0299': // ʙ  [LATIN LETTER SMALL CAPITAL B]
+                    case '\u1D03': // ᴃ  [LATIN LETTER SMALL CAPITAL BARRED B]
+                    case '\u1E02': // Ḃ  [LATIN CAPITAL LETTER B WITH DOT ABOVE]
+                    case '\u1E04': // Ḅ  [LATIN CAPITAL LETTER B WITH DOT BELOW]
+                    case '\u1E06': // Ḇ  [LATIN CAPITAL LETTER B WITH LINE BELOW]
+                    case '\u24B7': // Ⓑ  [CIRCLED LATIN CAPITAL LETTER B]
+                    case '\uFF22': // Ｂ  [FULLWIDTH LATIN CAPITAL LETTER B]
+                        output[outputPos++] = 'B';
+                        break;
+                    case '\u0180': // ƀ  [LATIN SMALL LETTER B WITH STROKE]
+                    case '\u0183': // ƃ  [LATIN SMALL LETTER B WITH TOPBAR]
+                    case '\u0253': // ɓ  [LATIN SMALL LETTER B WITH HOOK]
+                    case '\u1D6C': // ᵬ  [LATIN SMALL LETTER B WITH MIDDLE TILDE]
+                    case '\u1D80': // ᶀ  [LATIN SMALL LETTER B WITH PALATAL HOOK]
+                    case '\u1E03': // ḃ  [LATIN SMALL LETTER B WITH DOT ABOVE]
+                    case '\u1E05': // ḅ  [LATIN SMALL LETTER B WITH DOT BELOW]
+                    case '\u1E07': // ḇ  [LATIN SMALL LETTER B WITH LINE BELOW]
+                    case '\u24D1': // ⓑ  [CIRCLED LATIN SMALL LETTER B]
+                    case '\uFF42': // ｂ  [FULLWIDTH LATIN SMALL LETTER B]
+                        output[outputPos++] = 'b';
+                        break;
+                    case '\u249D': // ⒝  [PARENTHESIZED LATIN SMALL LETTER B]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'b';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u00C7': // Ç  [LATIN CAPITAL LETTER C WITH CEDILLA]
+                    case '\u0106': // Ć  [LATIN CAPITAL LETTER C WITH ACUTE]
+                    case '\u0108': // Ĉ  [LATIN CAPITAL LETTER C WITH CIRCUMFLEX]
+                    case '\u010A': // Ċ  [LATIN CAPITAL LETTER C WITH DOT ABOVE]
+                    case '\u010C': // Č  [LATIN CAPITAL LETTER C WITH CARON]
+                    case '\u0187': // Ƈ  [LATIN CAPITAL LETTER C WITH HOOK]
+                    case '\u023B': // Ȼ  [LATIN CAPITAL LETTER C WITH STROKE]
+                    case '\u0297': // ʗ  [LATIN LETTER STRETCHED C]
+                    case '\u1D04': // ᴄ  [LATIN LETTER SMALL CAPITAL C]
+                    case '\u1E08': // Ḉ  [LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE]
+                    case '\u24B8': // Ⓒ  [CIRCLED LATIN CAPITAL LETTER C]
+                    case '\uFF23': // Ｃ  [FULLWIDTH LATIN CAPITAL LETTER C]
+                        output[outputPos++] = 'C';
+                        break;
+                    case '\u00E7': // ç  [LATIN SMALL LETTER C WITH CEDILLA]
+                    case '\u0107': // ć  [LATIN SMALL LETTER C WITH ACUTE]
+                    case '\u0109': // ĉ  [LATIN SMALL LETTER C WITH CIRCUMFLEX]
+                    case '\u010B': // ċ  [LATIN SMALL LETTER C WITH DOT ABOVE]
+                    case '\u010D': // č  [LATIN SMALL LETTER C WITH CARON]
+                    case '\u0188': // ƈ  [LATIN SMALL LETTER C WITH HOOK]
+                    case '\u023C': // ȼ  [LATIN SMALL LETTER C WITH STROKE]
+                    case '\u0255': // ɕ  [LATIN SMALL LETTER C WITH CURL]
+                    case '\u1E09': // ḉ  [LATIN SMALL LETTER C WITH CEDILLA AND ACUTE]
+                    case '\u2184': // ↄ  [LATIN SMALL LETTER REVERSED C]
+                    case '\u24D2': // ⓒ  [CIRCLED LATIN SMALL LETTER C]
+                    case '\uA73E': // Ꜿ  [LATIN CAPITAL LETTER REVERSED C WITH DOT]
+                    case '\uA73F': // ꜿ  [LATIN SMALL LETTER REVERSED C WITH DOT]
+                    case '\uFF43': // ｃ  [FULLWIDTH LATIN SMALL LETTER C]
+                        output[outputPos++] = 'c';
+                        break;
+                    case '\u249E': // ⒞  [PARENTHESIZED LATIN SMALL LETTER C]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'c';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u00D0': // Ð  [LATIN CAPITAL LETTER ETH]
+                    case '\u010E': // Ď  [LATIN CAPITAL LETTER D WITH CARON]
+                    case '\u0110': // Đ  [LATIN CAPITAL LETTER D WITH STROKE]
+                    case '\u0189': // Ɖ  [LATIN CAPITAL LETTER AFRICAN D]
+                    case '\u018A': // Ɗ  [LATIN CAPITAL LETTER D WITH HOOK]
+                    case '\u018B': // Ƌ  [LATIN CAPITAL LETTER D WITH TOPBAR]
+                    case '\u1D05': // ᴅ  [LATIN LETTER SMALL CAPITAL D]
+                    case '\u1D06': // ᴆ  [LATIN LETTER SMALL CAPITAL ETH]
+                    case '\u1E0A': // Ḋ  [LATIN CAPITAL LETTER D WITH DOT ABOVE]
+                    case '\u1E0C': // Ḍ  [LATIN CAPITAL LETTER D WITH DOT BELOW]
+                    case '\u1E0E': // Ḏ  [LATIN CAPITAL LETTER D WITH LINE BELOW]
+                    case '\u1E10': // Ḑ  [LATIN CAPITAL LETTER D WITH CEDILLA]
+                    case '\u1E12': // Ḓ  [LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW]
+                    case '\u24B9': // Ⓓ  [CIRCLED LATIN CAPITAL LETTER D]
+                    case '\uA779': // Ꝺ  [LATIN CAPITAL LETTER INSULAR D]
+                    case '\uFF24': // Ｄ  [FULLWIDTH LATIN CAPITAL LETTER D]
+                        output[outputPos++] = 'D';
+                        break;
+                    case '\u00F0': // ð  [LATIN SMALL LETTER ETH]
+                    case '\u010F': // ď  [LATIN SMALL LETTER D WITH CARON]
+                    case '\u0111': // đ  [LATIN SMALL LETTER D WITH STROKE]
+                    case '\u018C': // ƌ  [LATIN SMALL LETTER D WITH TOPBAR]
+                    case '\u0221': // ȡ  [LATIN SMALL LETTER D WITH CURL]
+                    case '\u0256': // ɖ  [LATIN SMALL LETTER D WITH TAIL]
+                    case '\u0257': // ɗ  [LATIN SMALL LETTER D WITH HOOK]
+                    case '\u1D6D': // ᵭ  [LATIN SMALL LETTER D WITH MIDDLE TILDE]
+                    case '\u1D81': // ᶁ  [LATIN SMALL LETTER D WITH PALATAL HOOK]
+                    case '\u1D91': // ᶑ  [LATIN SMALL LETTER D WITH HOOK AND TAIL]
+                    case '\u1E0B': // ḋ  [LATIN SMALL LETTER D WITH DOT ABOVE]
+                    case '\u1E0D': // ḍ  [LATIN SMALL LETTER D WITH DOT BELOW]
+                    case '\u1E0F': // ḏ  [LATIN SMALL LETTER D WITH LINE BELOW]
+                    case '\u1E11': // ḑ  [LATIN SMALL LETTER D WITH CEDILLA]
+                    case '\u1E13': // ḓ  [LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW]
+                    case '\u24D3': // ⓓ  [CIRCLED LATIN SMALL LETTER D]
+                    case '\uA77A': // ꝺ  [LATIN SMALL LETTER INSULAR D]
+                    case '\uFF44': // ｄ  [FULLWIDTH LATIN SMALL LETTER D]
+                        output[outputPos++] = 'd';
+                        break;
+                    case '\u01C4': // DŽ  [LATIN CAPITAL LETTER DZ WITH CARON]
+                    case '\u01F1': // DZ  [LATIN CAPITAL LETTER DZ]
+                        output[outputPos++] = 'D';
+                        output[outputPos++] = 'Z';
+                        break;
+                    case '\u01C5': // Dž  [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]
+                    case '\u01F2': // Dz  [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]
+                        output[outputPos++] = 'D';
+                        output[outputPos++] = 'z';
+                        break;
+                    case '\u249F': // ⒟  [PARENTHESIZED LATIN SMALL LETTER D]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'd';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0238': // ȸ  [LATIN SMALL LETTER DB DIGRAPH]
+                        output[outputPos++] = 'd';
+                        output[outputPos++] = 'b';
+                        break;
+                    case '\u01C6': // dž  [LATIN SMALL LETTER DZ WITH CARON]
+                    case '\u01F3': // dz  [LATIN SMALL LETTER DZ]
+                    case '\u02A3': // ʣ  [LATIN SMALL LETTER DZ DIGRAPH]
+                    case '\u02A5': // ʥ  [LATIN SMALL LETTER DZ DIGRAPH WITH CURL]
+                        output[outputPos++] = 'd';
+                        output[outputPos++] = 'z';
+                        break;
+                    case '\u00C8': // È  [LATIN CAPITAL LETTER E WITH GRAVE]
+                    case '\u00C9': // É  [LATIN CAPITAL LETTER E WITH ACUTE]
+                    case '\u00CA': // Ê  [LATIN CAPITAL LETTER E WITH CIRCUMFLEX]
+                    case '\u00CB': // Ë  [LATIN CAPITAL LETTER E WITH DIAERESIS]
+                    case '\u0112': // Ē  [LATIN CAPITAL LETTER E WITH MACRON]
+                    case '\u0114': // Ĕ  [LATIN CAPITAL LETTER E WITH BREVE]
+                    case '\u0116': // Ė  [LATIN CAPITAL LETTER E WITH DOT ABOVE]
+                    case '\u0118': // Ę  [LATIN CAPITAL LETTER E WITH OGONEK]
+                    case '\u011A': // Ě  [LATIN CAPITAL LETTER E WITH CARON]
+                    case '\u018E': // Ǝ  [LATIN CAPITAL LETTER REVERSED E]
+                    case '\u0190': // Ɛ  [LATIN CAPITAL LETTER OPEN E]
+                    case '\u0204': // Ȅ  [LATIN CAPITAL LETTER E WITH DOUBLE GRAVE]
+                    case '\u0206': // Ȇ  [LATIN CAPITAL LETTER E WITH INVERTED BREVE]
+                    case '\u0228': // Ȩ  [LATIN CAPITAL LETTER E WITH CEDILLA]
+                    case '\u0246': // Ɇ  [LATIN CAPITAL LETTER E WITH STROKE]
+                    case '\u1D07': // ᴇ  [LATIN LETTER SMALL CAPITAL E]
+                    case '\u1E14': // Ḕ  [LATIN CAPITAL LETTER E WITH MACRON AND GRAVE]
+                    case '\u1E16': // Ḗ  [LATIN CAPITAL LETTER E WITH MACRON AND ACUTE]
+                    case '\u1E18': // Ḙ  [LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW]
+                    case '\u1E1A': // Ḛ  [LATIN CAPITAL LETTER E WITH TILDE BELOW]
+                    case '\u1E1C': // Ḝ  [LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE]
+                    case '\u1EB8': // Ẹ  [LATIN CAPITAL LETTER E WITH DOT BELOW]
+                    case '\u1EBA': // Ẻ  [LATIN CAPITAL LETTER E WITH HOOK ABOVE]
+                    case '\u1EBC': // Ẽ  [LATIN CAPITAL LETTER E WITH TILDE]
+                    case '\u1EBE': // Ế  [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE]
+                    case '\u1EC0': // Ề  [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE]
+                    case '\u1EC2': // Ể  [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE]
+                    case '\u1EC4': // Ễ  [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE]
+                    case '\u1EC6': // Ệ  [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW]
+                    case '\u24BA': // Ⓔ  [CIRCLED LATIN CAPITAL LETTER E]
+                    case '\u2C7B': // ⱻ  [LATIN LETTER SMALL CAPITAL TURNED E]
+                    case '\uFF25': // Ｅ  [FULLWIDTH LATIN CAPITAL LETTER E]
+                        output[outputPos++] = 'E';
+                        break;
+                    case '\u00E8': // è  [LATIN SMALL LETTER E WITH GRAVE]
+                    case '\u00E9': // é  [LATIN SMALL LETTER E WITH ACUTE]
+                    case '\u00EA': // ê  [LATIN SMALL LETTER E WITH CIRCUMFLEX]
+                    case '\u00EB': // ë  [LATIN SMALL LETTER E WITH DIAERESIS]
+                    case '\u0113': // ē  [LATIN SMALL LETTER E WITH MACRON]
+                    case '\u0115': // ĕ  [LATIN SMALL LETTER E WITH BREVE]
+                    case '\u0117': // ė  [LATIN SMALL LETTER E WITH DOT ABOVE]
+                    case '\u0119': // ę  [LATIN SMALL LETTER E WITH OGONEK]
+                    case '\u011B': // ě  [LATIN SMALL LETTER E WITH CARON]
+                    case '\u01DD': // ǝ  [LATIN SMALL LETTER TURNED E]
+                    case '\u0205': // ȅ  [LATIN SMALL LETTER E WITH DOUBLE GRAVE]
+                    case '\u0207': // ȇ  [LATIN SMALL LETTER E WITH INVERTED BREVE]
+                    case '\u0229': // ȩ  [LATIN SMALL LETTER E WITH CEDILLA]
+                    case '\u0247': // ɇ  [LATIN SMALL LETTER E WITH STROKE]
+                    case '\u0258': // ɘ  [LATIN SMALL LETTER REVERSED E]
+                    case '\u025B': // ɛ  [LATIN SMALL LETTER OPEN E]
+                    case '\u025C': // ɜ  [LATIN SMALL LETTER REVERSED OPEN E]
+                    case '\u025D': // ɝ  [LATIN SMALL LETTER REVERSED OPEN E WITH HOOK]
+                    case '\u025E': // ɞ  [LATIN SMALL LETTER CLOSED REVERSED OPEN E]
+                    case '\u029A': // ʚ  [LATIN SMALL LETTER CLOSED OPEN E]
+                    case '\u1D08': // ᴈ  [LATIN SMALL LETTER TURNED OPEN E]
+                    case '\u1D92': // ᶒ  [LATIN SMALL LETTER E WITH RETROFLEX HOOK]
+                    case '\u1D93': // ᶓ  [LATIN SMALL LETTER OPEN E WITH RETROFLEX HOOK]
+                    case '\u1D94': // ᶔ  [LATIN SMALL LETTER REVERSED OPEN E WITH RETROFLEX HOOK]
+                    case '\u1E15': // ḕ  [LATIN SMALL LETTER E WITH MACRON AND GRAVE]
+                    case '\u1E17': // ḗ  [LATIN SMALL LETTER E WITH MACRON AND ACUTE]
+                    case '\u1E19': // ḙ  [LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW]
+                    case '\u1E1B': // ḛ  [LATIN SMALL LETTER E WITH TILDE BELOW]
+                    case '\u1E1D': // ḝ  [LATIN SMALL LETTER E WITH CEDILLA AND BREVE]
+                    case '\u1EB9': // ẹ  [LATIN SMALL LETTER E WITH DOT BELOW]
+                    case '\u1EBB': // ẻ  [LATIN SMALL LETTER E WITH HOOK ABOVE]
+                    case '\u1EBD': // ẽ  [LATIN SMALL LETTER E WITH TILDE]
+                    case '\u1EBF': // ế  [LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE]
+                    case '\u1EC1': // ề  [LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE]
+                    case '\u1EC3': // ể  [LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE]
+                    case '\u1EC5': // ễ  [LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE]
+                    case '\u1EC7': // ệ  [LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW]
+                    case '\u2091': // ₑ  [LATIN SUBSCRIPT SMALL LETTER E]
+                    case '\u24D4': // ⓔ  [CIRCLED LATIN SMALL LETTER E]
+                    case '\u2C78': // ⱸ  [LATIN SMALL LETTER E WITH NOTCH]
+                    case '\uFF45': // ｅ  [FULLWIDTH LATIN SMALL LETTER E]
+                        output[outputPos++] = 'e';
+                        break;
+                    case '\u24A0': // ⒠  [PARENTHESIZED LATIN SMALL LETTER E]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'e';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0191': // Ƒ  [LATIN CAPITAL LETTER F WITH HOOK]
+                    case '\u1E1E': // Ḟ  [LATIN CAPITAL LETTER F WITH DOT ABOVE]
+                    case '\u24BB': // Ⓕ  [CIRCLED LATIN CAPITAL LETTER F]
+                    case '\uA730': // ꜰ  [LATIN LETTER SMALL CAPITAL F]
+                    case '\uA77B': // Ꝼ  [LATIN CAPITAL LETTER INSULAR F]
+                    case '\uA7FB': // ꟻ  [LATIN EPIGRAPHIC LETTER REVERSED F]
+                    case '\uFF26': // Ｆ  [FULLWIDTH LATIN CAPITAL LETTER F]
+                        output[outputPos++] = 'F';
+                        break;
+                    case '\u0192': // ƒ  [LATIN SMALL LETTER F WITH HOOK]
+                    case '\u1D6E': // ᵮ  [LATIN SMALL LETTER F WITH MIDDLE TILDE]
+                    case '\u1D82': // ᶂ  [LATIN SMALL LETTER F WITH PALATAL HOOK]
+                    case '\u1E1F': // ḟ  [LATIN SMALL LETTER F WITH DOT ABOVE]
+                    case '\u1E9B': // ẛ  [LATIN SMALL LETTER LONG S WITH DOT ABOVE]
+                    case '\u24D5': // ⓕ  [CIRCLED LATIN SMALL LETTER F]
+                    case '\uA77C': // ꝼ  [LATIN SMALL LETTER INSULAR F]
+                    case '\uFF46': // ｆ  [FULLWIDTH LATIN SMALL LETTER F]
+                        output[outputPos++] = 'f';
+                        break;
+                    case '\u24A1': // ⒡  [PARENTHESIZED LATIN SMALL LETTER F]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'f';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\uFB00': // ff  [LATIN SMALL LIGATURE FF]
+                        output[outputPos++] = 'f';
+                        output[outputPos++] = 'f';
+                        break;
+                    case '\uFB03': // ffi  [LATIN SMALL LIGATURE FFI]
+                        output[outputPos++] = 'f';
+                        output[outputPos++] = 'f';
+                        output[outputPos++] = 'i';
+                        break;
+                    case '\uFB04': // ffl  [LATIN SMALL LIGATURE FFL]
+                        output[outputPos++] = 'f';
+                        output[outputPos++] = 'f';
+                        output[outputPos++] = 'l';
+                        break;
+                    case '\uFB01': // fi  [LATIN SMALL LIGATURE FI]
+                        output[outputPos++] = 'f';
+                        output[outputPos++] = 'i';
+                        break;
+                    case '\uFB02': // fl  [LATIN SMALL LIGATURE FL]
+                        output[outputPos++] = 'f';
+                        output[outputPos++] = 'l';
+                        break;
+                    case '\u011C': // Ĝ  [LATIN CAPITAL LETTER G WITH CIRCUMFLEX]
+                    case '\u011E': // Ğ  [LATIN CAPITAL LETTER G WITH BREVE]
+                    case '\u0120': // Ġ  [LATIN CAPITAL LETTER G WITH DOT ABOVE]
+                    case '\u0122': // Ģ  [LATIN CAPITAL LETTER G WITH CEDILLA]
+                    case '\u0193': // Ɠ  [LATIN CAPITAL LETTER G WITH HOOK]
+                    case '\u01E4': // Ǥ  [LATIN CAPITAL LETTER G WITH STROKE]
+                    case '\u01E5': // ǥ  [LATIN SMALL LETTER G WITH STROKE]
+                    case '\u01E6': // Ǧ  [LATIN CAPITAL LETTER G WITH CARON]
+                    case '\u01E7': // ǧ  [LATIN SMALL LETTER G WITH CARON]
+                    case '\u01F4': // Ǵ  [LATIN CAPITAL LETTER G WITH ACUTE]
+                    case '\u0262': // ɢ  [LATIN LETTER SMALL CAPITAL G]
+                    case '\u029B': // ʛ  [LATIN LETTER SMALL CAPITAL G WITH HOOK]
+                    case '\u1E20': // Ḡ  [LATIN CAPITAL LETTER G WITH MACRON]
+                    case '\u24BC': // Ⓖ  [CIRCLED LATIN CAPITAL LETTER G]
+                    case '\uA77D': // Ᵹ  [LATIN CAPITAL LETTER INSULAR G]
+                    case '\uA77E': // Ꝿ  [LATIN CAPITAL LETTER TURNED INSULAR G]
+                    case '\uFF27': // Ｇ  [FULLWIDTH LATIN CAPITAL LETTER G]
+                        output[outputPos++] = 'G';
+                        break;
+                    case '\u011D': // ĝ  [LATIN SMALL LETTER G WITH CIRCUMFLEX]
+                    case '\u011F': // ğ  [LATIN SMALL LETTER G WITH BREVE]
+                    case '\u0121': // ġ  [LATIN SMALL LETTER G WITH DOT ABOVE]
+                    case '\u0123': // ģ  [LATIN SMALL LETTER G WITH CEDILLA]
+                    case '\u01F5': // ǵ  [LATIN SMALL LETTER G WITH ACUTE]
+                    case '\u0260': // ɠ  [LATIN SMALL LETTER G WITH HOOK]
+                    case '\u0261': // ɡ  [LATIN SMALL LETTER SCRIPT G]
+                    case '\u1D77': // ᵷ  [LATIN SMALL LETTER TURNED G]
+                    case '\u1D79': // ᵹ  [LATIN SMALL LETTER INSULAR G]
+                    case '\u1D83': // ᶃ  [LATIN SMALL LETTER G WITH PALATAL HOOK]
+                    case '\u1E21': // ḡ  [LATIN SMALL LETTER G WITH MACRON]
+                    case '\u24D6': // ⓖ  [CIRCLED LATIN SMALL LETTER G]
+                    case '\uA77F': // ꝿ  [LATIN SMALL LETTER TURNED INSULAR G]
+                    case '\uFF47': // ｇ  [FULLWIDTH LATIN SMALL LETTER G]
+                        output[outputPos++] = 'g';
+                        break;
+                    case '\u24A2': // ⒢  [PARENTHESIZED LATIN SMALL LETTER G]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'g';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0124': // Ĥ  [LATIN CAPITAL LETTER H WITH CIRCUMFLEX]
+                    case '\u0126': // Ħ  [LATIN CAPITAL LETTER H WITH STROKE]
+                    case '\u021E': // Ȟ  [LATIN CAPITAL LETTER H WITH CARON]
+                    case '\u029C': // ʜ  [LATIN LETTER SMALL CAPITAL H]
+                    case '\u1E22': // Ḣ  [LATIN CAPITAL LETTER H WITH DOT ABOVE]
+                    case '\u1E24': // Ḥ  [LATIN CAPITAL LETTER H WITH DOT BELOW]
+                    case '\u1E26': // Ḧ  [LATIN CAPITAL LETTER H WITH DIAERESIS]
+                    case '\u1E28': // Ḩ  [LATIN CAPITAL LETTER H WITH CEDILLA]
+                    case '\u1E2A': // Ḫ  [LATIN CAPITAL LETTER H WITH BREVE BELOW]
+                    case '\u24BD': // Ⓗ  [CIRCLED LATIN CAPITAL LETTER H]
+                    case '\u2C67': // Ⱨ  [LATIN CAPITAL LETTER H WITH DESCENDER]
+                    case '\u2C75': // Ⱶ  [LATIN CAPITAL LETTER HALF H]
+                    case '\uFF28': // Ｈ  [FULLWIDTH LATIN CAPITAL LETTER H]
+                        output[outputPos++] = 'H';
+                        break;
+                    case '\u0125': // ĥ  [LATIN SMALL LETTER H WITH CIRCUMFLEX]
+                    case '\u0127': // ħ  [LATIN SMALL LETTER H WITH STROKE]
+                    case '\u021F': // ȟ  [LATIN SMALL LETTER H WITH CARON]
+                    case '\u0265': // ɥ  [LATIN SMALL LETTER TURNED H]
+                    case '\u0266': // ɦ  [LATIN SMALL LETTER H WITH HOOK]
+                    case '\u02AE': // ʮ  [LATIN SMALL LETTER TURNED H WITH FISHHOOK]
+                    case '\u02AF': // ʯ  [LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL]
+                    case '\u1E23': // ḣ  [LATIN SMALL LETTER H WITH DOT ABOVE]
+                    case '\u1E25': // ḥ  [LATIN SMALL LETTER H WITH DOT BELOW]
+                    case '\u1E27': // ḧ  [LATIN SMALL LETTER H WITH DIAERESIS]
+                    case '\u1E29': // ḩ  [LATIN SMALL LETTER H WITH CEDILLA]
+                    case '\u1E2B': // ḫ  [LATIN SMALL LETTER H WITH BREVE BELOW]
+                    case '\u1E96': // ẖ  [LATIN SMALL LETTER H WITH LINE BELOW]
+                    case '\u24D7': // ⓗ  [CIRCLED LATIN SMALL LETTER H]
+                    case '\u2C68': // ⱨ  [LATIN SMALL LETTER H WITH DESCENDER]
+                    case '\u2C76': // ⱶ  [LATIN SMALL LETTER HALF H]
+                    case '\uFF48': // ｈ  [FULLWIDTH LATIN SMALL LETTER H]
+                        output[outputPos++] = 'h';
+                        break;
+                    case '\u01F6': // Ƕ  http://en.wikipedia.org/wiki/Hwair  [LATIN CAPITAL LETTER HWAIR]
+                        output[outputPos++] = 'H';
+                        output[outputPos++] = 'V';
+                        break;
+                    case '\u24A3': // ⒣  [PARENTHESIZED LATIN SMALL LETTER H]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'h';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0195': // ƕ  [LATIN SMALL LETTER HV]
+                        output[outputPos++] = 'h';
+                        output[outputPos++] = 'v';
+                        break;
+                    case '\u00CC': // Ì  [LATIN CAPITAL LETTER I WITH GRAVE]
+                    case '\u00CD': // Í  [LATIN CAPITAL LETTER I WITH ACUTE]
+                    case '\u00CE': // Î  [LATIN CAPITAL LETTER I WITH CIRCUMFLEX]
+                    case '\u00CF': // Ï  [LATIN CAPITAL LETTER I WITH DIAERESIS]
+                    case '\u0128': // Ĩ  [LATIN CAPITAL LETTER I WITH TILDE]
+                    case '\u012A': // Ī  [LATIN CAPITAL LETTER I WITH MACRON]
+                    case '\u012C': // Ĭ  [LATIN CAPITAL LETTER I WITH BREVE]
+                    case '\u012E': // Į  [LATIN CAPITAL LETTER I WITH OGONEK]
+                    case '\u0130': // İ  [LATIN CAPITAL LETTER I WITH DOT ABOVE]
+                    case '\u0196': // Ɩ  [LATIN CAPITAL LETTER IOTA]
+                    case '\u0197': // Ɨ  [LATIN CAPITAL LETTER I WITH STROKE]
+                    case '\u01CF': // Ǐ  [LATIN CAPITAL LETTER I WITH CARON]
+                    case '\u0208': // Ȉ  [LATIN CAPITAL LETTER I WITH DOUBLE GRAVE]
+                    case '\u020A': // Ȋ  [LATIN CAPITAL LETTER I WITH INVERTED BREVE]
+                    case '\u026A': // ɪ  [LATIN LETTER SMALL CAPITAL I]
+                    case '\u1D7B': // ᵻ  [LATIN SMALL CAPITAL LETTER I WITH STROKE]
+                    case '\u1E2C': // Ḭ  [LATIN CAPITAL LETTER I WITH TILDE BELOW]
+                    case '\u1E2E': // Ḯ  [LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE]
+                    case '\u1EC8': // Ỉ  [LATIN CAPITAL LETTER I WITH HOOK ABOVE]
+                    case '\u1ECA': // Ị  [LATIN CAPITAL LETTER I WITH DOT BELOW]
+                    case '\u24BE': // Ⓘ  [CIRCLED LATIN CAPITAL LETTER I]
+                    case '\uA7FE': // ꟾ  [LATIN EPIGRAPHIC LETTER I LONGA]
+                    case '\uFF29': // Ｉ  [FULLWIDTH LATIN CAPITAL LETTER I]
+                        output[outputPos++] = 'I';
+                        break;
+                    case '\u00EC': // ì  [LATIN SMALL LETTER I WITH GRAVE]
+                    case '\u00ED': // í  [LATIN SMALL LETTER I WITH ACUTE]
+                    case '\u00EE': // î  [LATIN SMALL LETTER I WITH CIRCUMFLEX]
+                    case '\u00EF': // ï  [LATIN SMALL LETTER I WITH DIAERESIS]
+                    case '\u0129': // ĩ  [LATIN SMALL LETTER I WITH TILDE]
+                    case '\u012B': // ī  [LATIN SMALL LETTER I WITH MACRON]
+                    case '\u012D': // ĭ  [LATIN SMALL LETTER I WITH BREVE]
+                    case '\u012F': // į  [LATIN SMALL LETTER I WITH OGONEK]
+                    case '\u0131': // ı  [LATIN SMALL LETTER DOTLESS I]
+                    case '\u01D0': // ǐ  [LATIN SMALL LETTER I WITH CARON]
+                    case '\u0209': // ȉ  [LATIN SMALL LETTER I WITH DOUBLE GRAVE]
+                    case '\u020B': // ȋ  [LATIN SMALL LETTER I WITH INVERTED BREVE]
+                    case '\u0268': // ɨ  [LATIN SMALL LETTER I WITH STROKE]
+                    case '\u1D09': // ᴉ  [LATIN SMALL LETTER TURNED I]
+                    case '\u1D62': // ᵢ  [LATIN SUBSCRIPT SMALL LETTER I]
+                    case '\u1D7C': // ᵼ  [LATIN SMALL LETTER IOTA WITH STROKE]
+                    case '\u1D96': // ᶖ  [LATIN SMALL LETTER I WITH RETROFLEX HOOK]
+                    case '\u1E2D': // ḭ  [LATIN SMALL LETTER I WITH TILDE BELOW]
+                    case '\u1E2F': // ḯ  [LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE]
+                    case '\u1EC9': // ỉ  [LATIN SMALL LETTER I WITH HOOK ABOVE]
+                    case '\u1ECB': // ị  [LATIN SMALL LETTER I WITH DOT BELOW]
+                    case '\u2071': // ⁱ  [SUPERSCRIPT LATIN SMALL LETTER I]
+                    case '\u24D8': // ⓘ  [CIRCLED LATIN SMALL LETTER I]
+                    case '\uFF49': // ｉ  [FULLWIDTH LATIN SMALL LETTER I]
+                        output[outputPos++] = 'i';
+                        break;
+                    case '\u0132': // IJ  [LATIN CAPITAL LIGATURE IJ]
+                        output[outputPos++] = 'I';
+                        output[outputPos++] = 'J';
+                        break;
+                    case '\u24A4': // ⒤  [PARENTHESIZED LATIN SMALL LETTER I]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'i';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0133': // ij  [LATIN SMALL LIGATURE IJ]
+                        output[outputPos++] = 'i';
+                        output[outputPos++] = 'j';
+                        break;
+                    case '\u0134': // Ĵ  [LATIN CAPITAL LETTER J WITH CIRCUMFLEX]
+                    case '\u0248': // Ɉ  [LATIN CAPITAL LETTER J WITH STROKE]
+                    case '\u1D0A': // ᴊ  [LATIN LETTER SMALL CAPITAL J]
+                    case '\u24BF': // Ⓙ  [CIRCLED LATIN CAPITAL LETTER J]
+                    case '\uFF2A': // Ｊ  [FULLWIDTH LATIN CAPITAL LETTER J]
+                        output[outputPos++] = 'J';
+                        break;
+                    case '\u0135': // ĵ  [LATIN SMALL LETTER J WITH CIRCUMFLEX]
+                    case '\u01F0': // ǰ  [LATIN SMALL LETTER J WITH CARON]
+                    case '\u0237': // ȷ  [LATIN SMALL LETTER DOTLESS J]
+                    case '\u0249': // ɉ  [LATIN SMALL LETTER J WITH STROKE]
+                    case '\u025F': // ɟ  [LATIN SMALL LETTER DOTLESS J WITH STROKE]
+                    case '\u0284': // ʄ  [LATIN SMALL LETTER DOTLESS J WITH STROKE AND HOOK]
+                    case '\u029D': // ʝ  [LATIN SMALL LETTER J WITH CROSSED-TAIL]
+                    case '\u24D9': // ⓙ  [CIRCLED LATIN SMALL LETTER J]
+                    case '\u2C7C': // ⱼ  [LATIN SUBSCRIPT SMALL LETTER J]
+                    case '\uFF4A': // ｊ  [FULLWIDTH LATIN SMALL LETTER J]
+                        output[outputPos++] = 'j';
+                        break;
+                    case '\u24A5': // ⒥  [PARENTHESIZED LATIN SMALL LETTER J]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'j';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0136': // Ķ  [LATIN CAPITAL LETTER K WITH CEDILLA]
+                    case '\u0198': // Ƙ  [LATIN CAPITAL LETTER K WITH HOOK]
+                    case '\u01E8': // Ǩ  [LATIN CAPITAL LETTER K WITH CARON]
+                    case '\u1D0B': // ᴋ  [LATIN LETTER SMALL CAPITAL K]
+                    case '\u1E30': // Ḱ  [LATIN CAPITAL LETTER K WITH ACUTE]
+                    case '\u1E32': // Ḳ  [LATIN CAPITAL LETTER K WITH DOT BELOW]
+                    case '\u1E34': // Ḵ  [LATIN CAPITAL LETTER K WITH LINE BELOW]
+                    case '\u24C0': // Ⓚ  [CIRCLED LATIN CAPITAL LETTER K]
+                    case '\u2C69': // Ⱪ  [LATIN CAPITAL LETTER K WITH DESCENDER]
+                    case '\uA740': // Ꝁ  [LATIN CAPITAL LETTER K WITH STROKE]
+                    case '\uA742': // Ꝃ  [LATIN CAPITAL LETTER K WITH DIAGONAL STROKE]
+                    case '\uA744': // Ꝅ  [LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE]
+                    case '\uFF2B': // Ｋ  [FULLWIDTH LATIN CAPITAL LETTER K]
+                        output[outputPos++] = 'K';
+                        break;
+                    case '\u0137': // ķ  [LATIN SMALL LETTER K WITH CEDILLA]
+                    case '\u0199': // ƙ  [LATIN SMALL LETTER K WITH HOOK]
+                    case '\u01E9': // ǩ  [LATIN SMALL LETTER K WITH CARON]
+                    case '\u029E': // ʞ  [LATIN SMALL LETTER TURNED K]
+                    case '\u1D84': // ᶄ  [LATIN SMALL LETTER K WITH PALATAL HOOK]
+                    case '\u1E31': // ḱ  [LATIN SMALL LETTER K WITH ACUTE]
+                    case '\u1E33': // ḳ  [LATIN SMALL LETTER K WITH DOT BELOW]
+                    case '\u1E35': // ḵ  [LATIN SMALL LETTER K WITH LINE BELOW]
+                    case '\u24DA': // ⓚ  [CIRCLED LATIN SMALL LETTER K]
+                    case '\u2C6A': // ⱪ  [LATIN SMALL LETTER K WITH DESCENDER]
+                    case '\uA741': // ꝁ  [LATIN SMALL LETTER K WITH STROKE]
+                    case '\uA743': // ꝃ  [LATIN SMALL LETTER K WITH DIAGONAL STROKE]
+                    case '\uA745': // ꝅ  [LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE]
+                    case '\uFF4B': // ｋ  [FULLWIDTH LATIN SMALL LETTER K]
+                        output[outputPos++] = 'k';
+                        break;
+                    case '\u24A6': // ⒦  [PARENTHESIZED LATIN SMALL LETTER K]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'k';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0139': // Ĺ  [LATIN CAPITAL LETTER L WITH ACUTE]
+                    case '\u013B': // Ļ  [LATIN CAPITAL LETTER L WITH CEDILLA]
+                    case '\u013D': // Ľ  [LATIN CAPITAL LETTER L WITH CARON]
+                    case '\u013F': // Ŀ  [LATIN CAPITAL LETTER L WITH MIDDLE DOT]
+                    case '\u0141': // Ł  [LATIN CAPITAL LETTER L WITH STROKE]
+                    case '\u023D': // Ƚ  [LATIN CAPITAL LETTER L WITH BAR]
+                    case '\u029F': // ʟ  [LATIN LETTER SMALL CAPITAL L]
+                    case '\u1D0C': // ᴌ  [LATIN LETTER SMALL CAPITAL L WITH STROKE]
+                    case '\u1E36': // Ḷ  [LATIN CAPITAL LETTER L WITH DOT BELOW]
+                    case '\u1E38': // Ḹ  [LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON]
+                    case '\u1E3A': // Ḻ  [LATIN CAPITAL LETTER L WITH LINE BELOW]
+                    case '\u1E3C': // Ḽ  [LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW]
+                    case '\u24C1': // Ⓛ  [CIRCLED LATIN CAPITAL LETTER L]
+                    case '\u2C60': // Ⱡ  [LATIN CAPITAL LETTER L WITH DOUBLE BAR]
+                    case '\u2C62': // Ɫ  [LATIN CAPITAL LETTER L WITH MIDDLE TILDE]
+                    case '\uA746': // Ꝇ  [LATIN CAPITAL LETTER BROKEN L]
+                    case '\uA748': // Ꝉ  [LATIN CAPITAL LETTER L WITH HIGH STROKE]
+                    case '\uA780': // Ꞁ  [LATIN CAPITAL LETTER TURNED L]
+                    case '\uFF2C': // Ｌ  [FULLWIDTH LATIN CAPITAL LETTER L]
+                        output[outputPos++] = 'L';
+                        break;
+                    case '\u013A': // ĺ  [LATIN SMALL LETTER L WITH ACUTE]
+                    case '\u013C': // ļ  [LATIN SMALL LETTER L WITH CEDILLA]
+                    case '\u013E': // ľ  [LATIN SMALL LETTER L WITH CARON]
+                    case '\u0140': // ŀ  [LATIN SMALL LETTER L WITH MIDDLE DOT]
+                    case '\u0142': // ł  [LATIN SMALL LETTER L WITH STROKE]
+                    case '\u019A': // ƚ  [LATIN SMALL LETTER L WITH BAR]
+                    case '\u0234': // ȴ  [LATIN SMALL LETTER L WITH CURL]
+                    case '\u026B': // ɫ  [LATIN SMALL LETTER L WITH MIDDLE TILDE]
+                    case '\u026C': // ɬ  [LATIN SMALL LETTER L WITH BELT]
+                    case '\u026D': // ɭ  [LATIN SMALL LETTER L WITH RETROFLEX HOOK]
+                    case '\u1D85': // ᶅ  [LATIN SMALL LETTER L WITH PALATAL HOOK]
+                    case '\u1E37': // ḷ  [LATIN SMALL LETTER L WITH DOT BELOW]
+                    case '\u1E39': // ḹ  [LATIN SMALL LETTER L WITH DOT BELOW AND MACRON]
+                    case '\u1E3B': // ḻ  [LATIN SMALL LETTER L WITH LINE BELOW]
+                    case '\u1E3D': // ḽ  [LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW]
+                    case '\u24DB': // ⓛ  [CIRCLED LATIN SMALL LETTER L]
+                    case '\u2C61': // ⱡ  [LATIN SMALL LETTER L WITH DOUBLE BAR]
+                    case '\uA747': // ꝇ  [LATIN SMALL LETTER BROKEN L]
+                    case '\uA749': // ꝉ  [LATIN SMALL LETTER L WITH HIGH STROKE]
+                    case '\uA781': // ꞁ  [LATIN SMALL LETTER TURNED L]
+                    case '\uFF4C': // ｌ  [FULLWIDTH LATIN SMALL LETTER L]
+                        output[outputPos++] = 'l';
+                        break;
+                    case '\u01C7': // LJ  [LATIN CAPITAL LETTER LJ]
+                        output[outputPos++] = 'L';
+                        output[outputPos++] = 'J';
+                        break;
+                    case '\u1EFA': // Ỻ  [LATIN CAPITAL LETTER MIDDLE-WELSH LL]
+                        output[outputPos++] = 'L';
+                        output[outputPos++] = 'L';
+                        break;
+                    case '\u01C8': // Lj  [LATIN CAPITAL LETTER L WITH SMALL LETTER J]
+                        output[outputPos++] = 'L';
+                        output[outputPos++] = 'j';
+                        break;
+                    case '\u24A7': // ⒧  [PARENTHESIZED LATIN SMALL LETTER L]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'l';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u01C9': // lj  [LATIN SMALL LETTER LJ]
+                        output[outputPos++] = 'l';
+                        output[outputPos++] = 'j';
+                        break;
+                    case '\u1EFB': // ỻ  [LATIN SMALL LETTER MIDDLE-WELSH LL]
+                        output[outputPos++] = 'l';
+                        output[outputPos++] = 'l';
+                        break;
+                    case '\u02AA': // ʪ  [LATIN SMALL LETTER LS DIGRAPH]
+                        output[outputPos++] = 'l';
+                        output[outputPos++] = 's';
+                        break;
+                    case '\u02AB': // ʫ  [LATIN SMALL LETTER LZ DIGRAPH]
+                        output[outputPos++] = 'l';
+                        output[outputPos++] = 'z';
+                        break;
+                    case '\u019C': // Ɯ  [LATIN CAPITAL LETTER TURNED M]
+                    case '\u1D0D': // ᴍ  [LATIN LETTER SMALL CAPITAL M]
+                    case '\u1E3E': // Ḿ  [LATIN CAPITAL LETTER M WITH ACUTE]
+                    case '\u1E40': // Ṁ  [LATIN CAPITAL LETTER M WITH DOT ABOVE]
+                    case '\u1E42': // Ṃ  [LATIN CAPITAL LETTER M WITH DOT BELOW]
+                    case '\u24C2': // Ⓜ  [CIRCLED LATIN CAPITAL LETTER M]
+                    case '\u2C6E': // Ɱ  [LATIN CAPITAL LETTER M WITH HOOK]
+                    case '\uA7FD': // ꟽ  [LATIN EPIGRAPHIC LETTER INVERTED M]
+                    case '\uA7FF': // ꟿ  [LATIN EPIGRAPHIC LETTER ARCHAIC M]
+                    case '\uFF2D': // Ｍ  [FULLWIDTH LATIN CAPITAL LETTER M]
+                        output[outputPos++] = 'M';
+                        break;
+                    case '\u026F': // ɯ  [LATIN SMALL LETTER TURNED M]
+                    case '\u0270': // ɰ  [LATIN SMALL LETTER TURNED M WITH LONG LEG]
+                    case '\u0271': // ɱ  [LATIN SMALL LETTER M WITH HOOK]
+                    case '\u1D6F': // ᵯ  [LATIN SMALL LETTER M WITH MIDDLE TILDE]
+                    case '\u1D86': // ᶆ  [LATIN SMALL LETTER M WITH PALATAL HOOK]
+                    case '\u1E3F': // ḿ  [LATIN SMALL LETTER M WITH ACUTE]
+                    case '\u1E41': // ṁ  [LATIN SMALL LETTER M WITH DOT ABOVE]
+                    case '\u1E43': // ṃ  [LATIN SMALL LETTER M WITH DOT BELOW]
+                    case '\u24DC': // ⓜ  [CIRCLED LATIN SMALL LETTER M]
+                    case '\uFF4D': // ｍ  [FULLWIDTH LATIN SMALL LETTER M]
+                        output[outputPos++] = 'm';
+                        break;
+                    case '\u24A8': // ⒨  [PARENTHESIZED LATIN SMALL LETTER M]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'm';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u00D1': // Ñ  [LATIN CAPITAL LETTER N WITH TILDE]
+                    case '\u0143': // Ń  [LATIN CAPITAL LETTER N WITH ACUTE]
+                    case '\u0145': // Ņ  [LATIN CAPITAL LETTER N WITH CEDILLA]
+                    case '\u0147': // Ň  [LATIN CAPITAL LETTER N WITH CARON]
+                    case '\u014A': // Ŋ  http://en.wikipedia.org/wiki/Eng_(letter)  [LATIN CAPITAL LETTER ENG]
+                    case '\u019D': // Ɲ  [LATIN CAPITAL LETTER N WITH LEFT HOOK]
+                    case '\u01F8': // Ǹ  [LATIN CAPITAL LETTER N WITH GRAVE]
+                    case '\u0220': // Ƞ  [LATIN CAPITAL LETTER N WITH LONG RIGHT LEG]
+                    case '\u0274': // ɴ  [LATIN LETTER SMALL CAPITAL N]
+                    case '\u1D0E': // ᴎ  [LATIN LETTER SMALL CAPITAL REVERSED N]
+                    case '\u1E44': // Ṅ  [LATIN CAPITAL LETTER N WITH DOT ABOVE]
+                    case '\u1E46': // Ṇ  [LATIN CAPITAL LETTER N WITH DOT BELOW]
+                    case '\u1E48': // Ṉ  [LATIN CAPITAL LETTER N WITH LINE BELOW]
+                    case '\u1E4A': // Ṋ  [LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW]
+                    case '\u24C3': // Ⓝ  [CIRCLED LATIN CAPITAL LETTER N]
+                    case '\uFF2E': // Ｎ  [FULLWIDTH LATIN CAPITAL LETTER N]
+                        output[outputPos++] = 'N';
+                        break;
+                    case '\u00F1': // ñ  [LATIN SMALL LETTER N WITH TILDE]
+                    case '\u0144': // ń  [LATIN SMALL LETTER N WITH ACUTE]
+                    case '\u0146': // ņ  [LATIN SMALL LETTER N WITH CEDILLA]
+                    case '\u0148': // ň  [LATIN SMALL LETTER N WITH CARON]
+                    case '\u0149': // ʼn  [LATIN SMALL LETTER N PRECEDED BY APOSTROPHE]
+                    case '\u014B': // ŋ  http://en.wikipedia.org/wiki/Eng_(letter)  [LATIN SMALL LETTER ENG]
+                    case '\u019E': // ƞ  [LATIN SMALL LETTER N WITH LONG RIGHT LEG]
+                    case '\u01F9': // ǹ  [LATIN SMALL LETTER N WITH GRAVE]
+                    case '\u0235': // ȵ  [LATIN SMALL LETTER N WITH CURL]
+                    case '\u0272': // ɲ  [LATIN SMALL LETTER N WITH LEFT HOOK]
+                    case '\u0273': // ɳ  [LATIN SMALL LETTER N WITH RETROFLEX HOOK]
+                    case '\u1D70': // ᵰ  [LATIN SMALL LETTER N WITH MIDDLE TILDE]
+                    case '\u1D87': // ᶇ  [LATIN SMALL LETTER N WITH PALATAL HOOK]
+                    case '\u1E45': // ṅ  [LATIN SMALL LETTER N WITH DOT ABOVE]
+                    case '\u1E47': // ṇ  [LATIN SMALL LETTER N WITH DOT BELOW]
+                    case '\u1E49': // ṉ  [LATIN SMALL LETTER N WITH LINE BELOW]
+                    case '\u1E4B': // ṋ  [LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW]
+                    case '\u207F': // ⁿ  [SUPERSCRIPT LATIN SMALL LETTER N]
+                    case '\u24DD': // ⓝ  [CIRCLED LATIN SMALL LETTER N]
+                    case '\uFF4E': // ｎ  [FULLWIDTH LATIN SMALL LETTER N]
+                        output[outputPos++] = 'n';
+                        break;
+                    case '\u01CA': // NJ  [LATIN CAPITAL LETTER NJ]
+                        output[outputPos++] = 'N';
+                        output[outputPos++] = 'J';
+                        break;
+                    case '\u01CB': // Nj  [LATIN CAPITAL LETTER N WITH SMALL LETTER J]
+                        output[outputPos++] = 'N';
+                        output[outputPos++] = 'j';
+                        break;
+                    case '\u24A9': // ⒩  [PARENTHESIZED LATIN SMALL LETTER N]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'n';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u01CC': // nj  [LATIN SMALL LETTER NJ]
+                        output[outputPos++] = 'n';
+                        output[outputPos++] = 'j';
+                        break;
+                    case '\u00D2': // Ò  [LATIN CAPITAL LETTER O WITH GRAVE]
+                    case '\u00D3': // Ó  [LATIN CAPITAL LETTER O WITH ACUTE]
+                    case '\u00D4': // Ô  [LATIN CAPITAL LETTER O WITH CIRCUMFLEX]
+                    case '\u00D5': // Õ  [LATIN CAPITAL LETTER O WITH TILDE]
+                    case '\u00D6': // Ö  [LATIN CAPITAL LETTER O WITH DIAERESIS]
+                    case '\u00D8': // Ø  [LATIN CAPITAL LETTER O WITH STROKE]
+                    case '\u014C': // Ō  [LATIN CAPITAL LETTER O WITH MACRON]
+                    case '\u014E': // Ŏ  [LATIN CAPITAL LETTER O WITH BREVE]
+                    case '\u0150': // Ő  [LATIN CAPITAL LETTER O WITH DOUBLE ACUTE]
+                    case '\u0186': // Ɔ  [LATIN CAPITAL LETTER OPEN O]
+                    case '\u019F': // Ɵ  [LATIN CAPITAL LETTER O WITH MIDDLE TILDE]
+                    case '\u01A0': // Ơ  [LATIN CAPITAL LETTER O WITH HORN]
+                    case '\u01D1': // Ǒ  [LATIN CAPITAL LETTER O WITH CARON]
+                    case '\u01EA': // Ǫ  [LATIN CAPITAL LETTER O WITH OGONEK]
+                    case '\u01EC': // Ǭ  [LATIN CAPITAL LETTER O WITH OGONEK AND MACRON]
+                    case '\u01FE': // Ǿ  [LATIN CAPITAL LETTER O WITH STROKE AND ACUTE]
+                    case '\u020C': // Ȍ  [LATIN CAPITAL LETTER O WITH DOUBLE GRAVE]
+                    case '\u020E': // Ȏ  [LATIN CAPITAL LETTER O WITH INVERTED BREVE]
+                    case '\u022A': // Ȫ  [LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON]
+                    case '\u022C': // Ȭ  [LATIN CAPITAL LETTER O WITH TILDE AND MACRON]
+                    case '\u022E': // Ȯ  [LATIN CAPITAL LETTER O WITH DOT ABOVE]
+                    case '\u0230': // Ȱ  [LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON]
+                    case '\u1D0F': // ᴏ  [LATIN LETTER SMALL CAPITAL O]
+                    case '\u1D10': // ᴐ  [LATIN LETTER SMALL CAPITAL OPEN O]
+                    case '\u1E4C': // Ṍ  [LATIN CAPITAL LETTER O WITH TILDE AND ACUTE]
+                    case '\u1E4E': // Ṏ  [LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS]
+                    case '\u1E50': // Ṑ  [LATIN CAPITAL LETTER O WITH MACRON AND GRAVE]
+                    case '\u1E52': // Ṓ  [LATIN CAPITAL LETTER O WITH MACRON AND ACUTE]
+                    case '\u1ECC': // Ọ  [LATIN CAPITAL LETTER O WITH DOT BELOW]
+                    case '\u1ECE': // Ỏ  [LATIN CAPITAL LETTER O WITH HOOK ABOVE]
+                    case '\u1ED0': // Ố  [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE]
+                    case '\u1ED2': // Ồ  [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE]
+                    case '\u1ED4': // Ổ  [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE]
+                    case '\u1ED6': // Ỗ  [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE]
+                    case '\u1ED8': // Ộ  [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW]
+                    case '\u1EDA': // Ớ  [LATIN CAPITAL LETTER O WITH HORN AND ACUTE]
+                    case '\u1EDC': // Ờ  [LATIN CAPITAL LETTER O WITH HORN AND GRAVE]
+                    case '\u1EDE': // Ở  [LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE]
+                    case '\u1EE0': // Ỡ  [LATIN CAPITAL LETTER O WITH HORN AND TILDE]
+                    case '\u1EE2': // Ợ  [LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW]
+                    case '\u24C4': // Ⓞ  [CIRCLED LATIN CAPITAL LETTER O]
+                    case '\uA74A': // Ꝋ  [LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY]
+                    case '\uA74C': // Ꝍ  [LATIN CAPITAL LETTER O WITH LOOP]
+                    case '\uFF2F': // Ｏ  [FULLWIDTH LATIN CAPITAL LETTER O]
+                        output[outputPos++] = 'O';
+                        break;
+                    case '\u00F2': // ò  [LATIN SMALL LETTER O WITH GRAVE]
+                    case '\u00F3': // ó  [LATIN SMALL LETTER O WITH ACUTE]
+                    case '\u00F4': // ô  [LATIN SMALL LETTER O WITH CIRCUMFLEX]
+                    case '\u00F5': // õ  [LATIN SMALL LETTER O WITH TILDE]
+                    case '\u00F6': // ö  [LATIN SMALL LETTER O WITH DIAERESIS]
+                    case '\u00F8': // ø  [LATIN SMALL LETTER O WITH STROKE]
+                    case '\u014D': // ō  [LATIN SMALL LETTER O WITH MACRON]
+                    case '\u014F': // ŏ  [LATIN SMALL LETTER O WITH BREVE]
+                    case '\u0151': // ő  [LATIN SMALL LETTER O WITH DOUBLE ACUTE]
+                    case '\u01A1': // ơ  [LATIN SMALL LETTER O WITH HORN]
+                    case '\u01D2': // ǒ  [LATIN SMALL LETTER O WITH CARON]
+                    case '\u01EB': // ǫ  [LATIN SMALL LETTER O WITH OGONEK]
+                    case '\u01ED': // ǭ  [LATIN SMALL LETTER O WITH OGONEK AND MACRON]
+                    case '\u01FF': // ǿ  [LATIN SMALL LETTER O WITH STROKE AND ACUTE]
+                    case '\u020D': // ȍ  [LATIN SMALL LETTER O WITH DOUBLE GRAVE]
+                    case '\u020F': // ȏ  [LATIN SMALL LETTER O WITH INVERTED BREVE]
+                    case '\u022B': // ȫ  [LATIN SMALL LETTER O WITH DIAERESIS AND MACRON]
+                    case '\u022D': // ȭ  [LATIN SMALL LETTER O WITH TILDE AND MACRON]
+                    case '\u022F': // ȯ  [LATIN SMALL LETTER O WITH DOT ABOVE]
+                    case '\u0231': // ȱ  [LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON]
+                    case '\u0254': // ɔ  [LATIN SMALL LETTER OPEN O]
+                    case '\u0275': // ɵ  [LATIN SMALL LETTER BARRED O]
+                    case '\u1D16': // ᴖ  [LATIN SMALL LETTER TOP HALF O]
+                    case '\u1D17': // ᴗ  [LATIN SMALL LETTER BOTTOM HALF O]
+                    case '\u1D97': // ᶗ  [LATIN SMALL LETTER OPEN O WITH RETROFLEX HOOK]
+                    case '\u1E4D': // ṍ  [LATIN SMALL LETTER O WITH TILDE AND ACUTE]
+                    case '\u1E4F': // ṏ  [LATIN SMALL LETTER O WITH TILDE AND DIAERESIS]
+                    case '\u1E51': // ṑ  [LATIN SMALL LETTER O WITH MACRON AND GRAVE]
+                    case '\u1E53': // ṓ  [LATIN SMALL LETTER O WITH MACRON AND ACUTE]
+                    case '\u1ECD': // ọ  [LATIN SMALL LETTER O WITH DOT BELOW]
+                    case '\u1ECF': // ỏ  [LATIN SMALL LETTER O WITH HOOK ABOVE]
+                    case '\u1ED1': // ố  [LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE]
+                    case '\u1ED3': // ồ  [LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE]
+                    case '\u1ED5': // ổ  [LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE]
+                    case '\u1ED7': // ỗ  [LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE]
+                    case '\u1ED9': // ộ  [LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW]
+                    case '\u1EDB': // ớ  [LATIN SMALL LETTER O WITH HORN AND ACUTE]
+                    case '\u1EDD': // ờ  [LATIN SMALL LETTER O WITH HORN AND GRAVE]
+                    case '\u1EDF': // ở  [LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE]
+                    case '\u1EE1': // ỡ  [LATIN SMALL LETTER O WITH HORN AND TILDE]
+                    case '\u1EE3': // ợ  [LATIN SMALL LETTER O WITH HORN AND DOT BELOW]
+                    case '\u2092': // ₒ  [LATIN SUBSCRIPT SMALL LETTER O]
+                    case '\u24DE': // ⓞ  [CIRCLED LATIN SMALL LETTER O]
+                    case '\u2C7A': // ⱺ  [LATIN SMALL LETTER O WITH LOW RING INSIDE]
+                    case '\uA74B': // ꝋ  [LATIN SMALL LETTER O WITH LONG STROKE OVERLAY]
+                    case '\uA74D': // ꝍ  [LATIN SMALL LETTER O WITH LOOP]
+                    case '\uFF4F': // ｏ  [FULLWIDTH LATIN SMALL LETTER O]
+                        output[outputPos++] = 'o';
+                        break;
+                    case '\u0152': // Π [LATIN CAPITAL LIGATURE OE]
+                    case '\u0276': // ɶ  [LATIN LETTER SMALL CAPITAL OE]
+                        output[outputPos++] = 'O';
+                        output[outputPos++] = 'E';
+                        break;
+                    case '\uA74E': // Ꝏ  [LATIN CAPITAL LETTER OO]
+                        output[outputPos++] = 'O';
+                        output[outputPos++] = 'O';
+                        break;
+                    case '\u0222': // Ȣ  http://en.wikipedia.org/wiki/OU  [LATIN CAPITAL LETTER OU]
+                    case '\u1D15': // ᴕ  [LATIN LETTER SMALL CAPITAL OU]
+                        output[outputPos++] = 'O';
+                        output[outputPos++] = 'U';
+                        break;
+                    case '\u24AA': // ⒪  [PARENTHESIZED LATIN SMALL LETTER O]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'o';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0153': // œ  [LATIN SMALL LIGATURE OE]
+                    case '\u1D14': // ᴔ  [LATIN SMALL LETTER TURNED OE]
+                        output[outputPos++] = 'o';
+                        output[outputPos++] = 'e';
+                        break;
+                    case '\uA74F': // ꝏ  [LATIN SMALL LETTER OO]
+                        output[outputPos++] = 'o';
+                        output[outputPos++] = 'o';
+                        break;
+                    case '\u0223': // ȣ  http://en.wikipedia.org/wiki/OU  [LATIN SMALL LETTER OU]
+                        output[outputPos++] = 'o';
+                        output[outputPos++] = 'u';
+                        break;
+                    case '\u01A4': // Ƥ  [LATIN CAPITAL LETTER P WITH HOOK]
+                    case '\u1D18': // ᴘ  [LATIN LETTER SMALL CAPITAL P]
+                    case '\u1E54': // Ṕ  [LATIN CAPITAL LETTER P WITH ACUTE]
+                    case '\u1E56': // Ṗ  [LATIN CAPITAL LETTER P WITH DOT ABOVE]
+                    case '\u24C5': // Ⓟ  [CIRCLED LATIN CAPITAL LETTER P]
+                    case '\u2C63': // Ᵽ  [LATIN CAPITAL LETTER P WITH STROKE]
+                    case '\uA750': // Ꝑ  [LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER]
+                    case '\uA752': // Ꝓ  [LATIN CAPITAL LETTER P WITH FLOURISH]
+                    case '\uA754': // Ꝕ  [LATIN CAPITAL LETTER P WITH SQUIRREL TAIL]
+                    case '\uFF30': // Ｐ  [FULLWIDTH LATIN CAPITAL LETTER P]
+                        output[outputPos++] = 'P';
+                        break;
+                    case '\u01A5': // ƥ  [LATIN SMALL LETTER P WITH HOOK]
+                    case '\u1D71': // ᵱ  [LATIN SMALL LETTER P WITH MIDDLE TILDE]
+                    case '\u1D7D': // ᵽ  [LATIN SMALL LETTER P WITH STROKE]
+                    case '\u1D88': // ᶈ  [LATIN SMALL LETTER P WITH PALATAL HOOK]
+                    case '\u1E55': // ṕ  [LATIN SMALL LETTER P WITH ACUTE]
+                    case '\u1E57': // ṗ  [LATIN SMALL LETTER P WITH DOT ABOVE]
+                    case '\u24DF': // ⓟ  [CIRCLED LATIN SMALL LETTER P]
+                    case '\uA751': // ꝑ  [LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER]
+                    case '\uA753': // ꝓ  [LATIN SMALL LETTER P WITH FLOURISH]
+                    case '\uA755': // ꝕ  [LATIN SMALL LETTER P WITH SQUIRREL TAIL]
+                    case '\uA7FC': // ꟼ  [LATIN EPIGRAPHIC LETTER REVERSED P]
+                    case '\uFF50': // ｐ  [FULLWIDTH LATIN SMALL LETTER P]
+                        output[outputPos++] = 'p';
+                        break;
+                    case '\u24AB': // ⒫  [PARENTHESIZED LATIN SMALL LETTER P]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'p';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u024A': // Ɋ  [LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL]
+                    case '\u24C6': // Ⓠ  [CIRCLED LATIN CAPITAL LETTER Q]
+                    case '\uA756': // Ꝗ  [LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER]
+                    case '\uA758': // Ꝙ  [LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE]
+                    case '\uFF31': // Ｑ  [FULLWIDTH LATIN CAPITAL LETTER Q]
+                        output[outputPos++] = 'Q';
+                        break;
+                    case '\u0138': // ĸ  http://en.wikipedia.org/wiki/Kra_(letter)  [LATIN SMALL LETTER KRA]
+                    case '\u024B': // ɋ  [LATIN SMALL LETTER Q WITH HOOK TAIL]
+                    case '\u02A0': // ʠ  [LATIN SMALL LETTER Q WITH HOOK]
+                    case '\u24E0': // ⓠ  [CIRCLED LATIN SMALL LETTER Q]
+                    case '\uA757': // ꝗ  [LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER]
+                    case '\uA759': // ꝙ  [LATIN SMALL LETTER Q WITH DIAGONAL STROKE]
+                    case '\uFF51': // ｑ  [FULLWIDTH LATIN SMALL LETTER Q]
+                        output[outputPos++] = 'q';
+                        break;
+                    case '\u24AC': // ⒬  [PARENTHESIZED LATIN SMALL LETTER Q]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'q';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0239': // ȹ  [LATIN SMALL LETTER QP DIGRAPH]
+                        output[outputPos++] = 'q';
+                        output[outputPos++] = 'p';
+                        break;
+                    case '\u0154': // Ŕ  [LATIN CAPITAL LETTER R WITH ACUTE]
+                    case '\u0156': // Ŗ  [LATIN CAPITAL LETTER R WITH CEDILLA]
+                    case '\u0158': // Ř  [LATIN CAPITAL LETTER R WITH CARON]
+                    case '\u0210': // Ȓ  [LATIN CAPITAL LETTER R WITH DOUBLE GRAVE]
+                    case '\u0212': // Ȓ  [LATIN CAPITAL LETTER R WITH INVERTED BREVE]
+                    case '\u024C': // Ɍ  [LATIN CAPITAL LETTER R WITH STROKE]
+                    case '\u0280': // ʀ  [LATIN LETTER SMALL CAPITAL R]
+                    case '\u0281': // ʁ  [LATIN LETTER SMALL CAPITAL INVERTED R]
+                    case '\u1D19': // ᴙ  [LATIN LETTER SMALL CAPITAL REVERSED R]
+                    case '\u1D1A': // ᴚ  [LATIN LETTER SMALL CAPITAL TURNED R]
+                    case '\u1E58': // Ṙ  [LATIN CAPITAL LETTER R WITH DOT ABOVE]
+                    case '\u1E5A': // Ṛ  [LATIN CAPITAL LETTER R WITH DOT BELOW]
+                    case '\u1E5C': // Ṝ  [LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON]
+                    case '\u1E5E': // Ṟ  [LATIN CAPITAL LETTER R WITH LINE BELOW]
+                    case '\u24C7': // Ⓡ  [CIRCLED LATIN CAPITAL LETTER R]
+                    case '\u2C64': // Ɽ  [LATIN CAPITAL LETTER R WITH TAIL]
+                    case '\uA75A': // Ꝛ  [LATIN CAPITAL LETTER R ROTUNDA]
+                    case '\uA782': // Ꞃ  [LATIN CAPITAL LETTER INSULAR R]
+                    case '\uFF32': // Ｒ  [FULLWIDTH LATIN CAPITAL LETTER R]
+                        output[outputPos++] = 'R';
+                        break;
+                    case '\u0155': // ŕ  [LATIN SMALL LETTER R WITH ACUTE]
+                    case '\u0157': // ŗ  [LATIN SMALL LETTER R WITH CEDILLA]
+                    case '\u0159': // ř  [LATIN SMALL LETTER R WITH CARON]
+                    case '\u0211': // ȑ  [LATIN SMALL LETTER R WITH DOUBLE GRAVE]
+                    case '\u0213': // ȓ  [LATIN SMALL LETTER R WITH INVERTED BREVE]
+                    case '\u024D': // ɍ  [LATIN SMALL LETTER R WITH STROKE]
+                    case '\u027C': // ɼ  [LATIN SMALL LETTER R WITH LONG LEG]
+                    case '\u027D': // ɽ  [LATIN SMALL LETTER R WITH TAIL]
+                    case '\u027E': // ɾ  [LATIN SMALL LETTER R WITH FISHHOOK]
+                    case '\u027F': // ɿ  [LATIN SMALL LETTER REVERSED R WITH FISHHOOK]
+                    case '\u1D63': // ᵣ  [LATIN SUBSCRIPT SMALL LETTER R]
+                    case '\u1D72': // ᵲ  [LATIN SMALL LETTER R WITH MIDDLE TILDE]
+                    case '\u1D73': // ᵳ  [LATIN SMALL LETTER R WITH FISHHOOK AND MIDDLE TILDE]
+                    case '\u1D89': // ᶉ  [LATIN SMALL LETTER R WITH PALATAL HOOK]
+                    case '\u1E59': // ṙ  [LATIN SMALL LETTER R WITH DOT ABOVE]
+                    case '\u1E5B': // ṛ  [LATIN SMALL LETTER R WITH DOT BELOW]
+                    case '\u1E5D': // ṝ  [LATIN SMALL LETTER R WITH DOT BELOW AND MACRON]
+                    case '\u1E5F': // ṟ  [LATIN SMALL LETTER R WITH LINE BELOW]
+                    case '\u24E1': // ⓡ  [CIRCLED LATIN SMALL LETTER R]
+                    case '\uA75B': // ꝛ  [LATIN SMALL LETTER R ROTUNDA]
+                    case '\uA783': // ꞃ  [LATIN SMALL LETTER INSULAR R]
+                    case '\uFF52': // ｒ  [FULLWIDTH LATIN SMALL LETTER R]
+                        output[outputPos++] = 'r';
+                        break;
+                    case '\u24AD': // ⒭  [PARENTHESIZED LATIN SMALL LETTER R]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'r';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u015A': // Ś  [LATIN CAPITAL LETTER S WITH ACUTE]
+                    case '\u015C': // Ŝ  [LATIN CAPITAL LETTER S WITH CIRCUMFLEX]
+                    case '\u015E': // Ş  [LATIN CAPITAL LETTER S WITH CEDILLA]
+                    case '\u0160': // Š  [LATIN CAPITAL LETTER S WITH CARON]
+                    case '\u0218': // Ș  [LATIN CAPITAL LETTER S WITH COMMA BELOW]
+                    case '\u1E60': // Ṡ  [LATIN CAPITAL LETTER S WITH DOT ABOVE]
+                    case '\u1E62': // Ṣ  [LATIN CAPITAL LETTER S WITH DOT BELOW]
+                    case '\u1E64': // Ṥ  [LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE]
+                    case '\u1E66': // Ṧ  [LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE]
+                    case '\u1E68': // Ṩ  [LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE]
+                    case '\u24C8': // Ⓢ  [CIRCLED LATIN CAPITAL LETTER S]
+                    case '\uA731': // ꜱ  [LATIN LETTER SMALL CAPITAL S]
+                    case '\uA785': // ꞅ  [LATIN SMALL LETTER INSULAR S]
+                    case '\uFF33': // Ｓ  [FULLWIDTH LATIN CAPITAL LETTER S]
+                        output[outputPos++] = 'S';
+                        break;
+                    case '\u015B': // ś  [LATIN SMALL LETTER S WITH ACUTE]
+                    case '\u015D': // ŝ  [LATIN SMALL LETTER S WITH CIRCUMFLEX]
+                    case '\u015F': // ş  [LATIN SMALL LETTER S WITH CEDILLA]
+                    case '\u0161': // š  [LATIN SMALL LETTER S WITH CARON]
+                    case '\u017F': // ſ  http://en.wikipedia.org/wiki/Long_S  [LATIN SMALL LETTER LONG S]
+                    case '\u0219': // ș  [LATIN SMALL LETTER S WITH COMMA BELOW]
+                    case '\u023F': // ȿ  [LATIN SMALL LETTER S WITH SWASH TAIL]
+                    case '\u0282': // ʂ  [LATIN SMALL LETTER S WITH HOOK]
+                    case '\u1D74': // ᵴ  [LATIN SMALL LETTER S WITH MIDDLE TILDE]
+                    case '\u1D8A': // ᶊ  [LATIN SMALL LETTER S WITH PALATAL HOOK]
+                    case '\u1E61': // ṡ  [LATIN SMALL LETTER S WITH DOT ABOVE]
+                    case '\u1E63': // ṣ  [LATIN SMALL LETTER S WITH DOT BELOW]
+                    case '\u1E65': // ṥ  [LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE]
+                    case '\u1E67': // ṧ  [LATIN SMALL LETTER S WITH CARON AND DOT ABOVE]
+                    case '\u1E69': // ṩ  [LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE]
+                    case '\u1E9C': // ẜ  [LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE]
+                    case '\u1E9D': // ẝ  [LATIN SMALL LETTER LONG S WITH HIGH STROKE]
+                    case '\u24E2': // ⓢ  [CIRCLED LATIN SMALL LETTER S]
+                    case '\uA784': // Ꞅ  [LATIN CAPITAL LETTER INSULAR S]
+                    case '\uFF53': // ｓ  [FULLWIDTH LATIN SMALL LETTER S]
+                        output[outputPos++] = 's';
+                        break;
+                    case '\u1E9E': // ẞ  [LATIN CAPITAL LETTER SHARP S]
+                        output[outputPos++] = 'S';
+                        output[outputPos++] = 'S';
+                        break;
+                    case '\u24AE': // ⒮  [PARENTHESIZED LATIN SMALL LETTER S]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 's';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u00DF': // ß  [LATIN SMALL LETTER SHARP S]
+                        output[outputPos++] = 's';
+                        output[outputPos++] = 's';
+                        break;
+                    case '\uFB06': // st  [LATIN SMALL LIGATURE ST]
+                        output[outputPos++] = 's';
+                        output[outputPos++] = 't';
+                        break;
+                    case '\u0162': // Ţ  [LATIN CAPITAL LETTER T WITH CEDILLA]
+                    case '\u0164': // Ť  [LATIN CAPITAL LETTER T WITH CARON]
+                    case '\u0166': // Ŧ  [LATIN CAPITAL LETTER T WITH STROKE]
+                    case '\u01AC': // Ƭ  [LATIN CAPITAL LETTER T WITH HOOK]
+                    case '\u01AE': // Ʈ  [LATIN CAPITAL LETTER T WITH RETROFLEX HOOK]
+                    case '\u021A': // Ț  [LATIN CAPITAL LETTER T WITH COMMA BELOW]
+                    case '\u023E': // Ⱦ  [LATIN CAPITAL LETTER T WITH DIAGONAL STROKE]
+                    case '\u1D1B': // ᴛ  [LATIN LETTER SMALL CAPITAL T]
+                    case '\u1E6A': // Ṫ  [LATIN CAPITAL LETTER T WITH DOT ABOVE]
+                    case '\u1E6C': // Ṭ  [LATIN CAPITAL LETTER T WITH DOT BELOW]
+                    case '\u1E6E': // Ṯ  [LATIN CAPITAL LETTER T WITH LINE BELOW]
+                    case '\u1E70': // Ṱ  [LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW]
+                    case '\u24C9': // Ⓣ  [CIRCLED LATIN CAPITAL LETTER T]
+                    case '\uA786': // Ꞇ  [LATIN CAPITAL LETTER INSULAR T]
+                    case '\uFF34': // Ｔ  [FULLWIDTH LATIN CAPITAL LETTER T]
+                        output[outputPos++] = 'T';
+                        break;
+                    case '\u0163': // ţ  [LATIN SMALL LETTER T WITH CEDILLA]
+                    case '\u0165': // ť  [LATIN SMALL LETTER T WITH CARON]
+                    case '\u0167': // ŧ  [LATIN SMALL LETTER T WITH STROKE]
+                    case '\u01AB': // ƫ  [LATIN SMALL LETTER T WITH PALATAL HOOK]
+                    case '\u01AD': // ƭ  [LATIN SMALL LETTER T WITH HOOK]
+                    case '\u021B': // ț  [LATIN SMALL LETTER T WITH COMMA BELOW]
+                    case '\u0236': // ȶ  [LATIN SMALL LETTER T WITH CURL]
+                    case '\u0287': // ʇ  [LATIN SMALL LETTER TURNED T]
+                    case '\u0288': // ʈ  [LATIN SMALL LETTER T WITH RETROFLEX HOOK]
+                    case '\u1D75': // ᵵ  [LATIN SMALL LETTER T WITH MIDDLE TILDE]
+                    case '\u1E6B': // ṫ  [LATIN SMALL LETTER T WITH DOT ABOVE]
+                    case '\u1E6D': // ṭ  [LATIN SMALL LETTER T WITH DOT BELOW]
+                    case '\u1E6F': // ṯ  [LATIN SMALL LETTER T WITH LINE BELOW]
+                    case '\u1E71': // ṱ  [LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW]
+                    case '\u1E97': // ẗ  [LATIN SMALL LETTER T WITH DIAERESIS]
+                    case '\u24E3': // ⓣ  [CIRCLED LATIN SMALL LETTER T]
+                    case '\u2C66': // ⱦ  [LATIN SMALL LETTER T WITH DIAGONAL STROKE]
+                    case '\uFF54': // ｔ  [FULLWIDTH LATIN SMALL LETTER T]
+                        output[outputPos++] = 't';
+                        break;
+                    case '\u00DE': // Þ  [LATIN CAPITAL LETTER THORN]
+                    case '\uA766': // Ꝧ  [LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER]
+                        output[outputPos++] = 'T';
+                        output[outputPos++] = 'H';
+                        break;
+                    case '\uA728': // Ꜩ  [LATIN CAPITAL LETTER TZ]
+                        output[outputPos++] = 'T';
+                        output[outputPos++] = 'Z';
+                        break;
+                    case '\u24AF': // ⒯  [PARENTHESIZED LATIN SMALL LETTER T]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 't';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u02A8': // ʨ  [LATIN SMALL LETTER TC DIGRAPH WITH CURL]
+                        output[outputPos++] = 't';
+                        output[outputPos++] = 'c';
+                        break;
+                    case '\u00FE': // þ  [LATIN SMALL LETTER THORN]
+                    case '\u1D7A': // ᵺ  [LATIN SMALL LETTER TH WITH STRIKETHROUGH]
+                    case '\uA767': // ꝧ  [LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER]
+                        output[outputPos++] = 't';
+                        output[outputPos++] = 'h';
+                        break;
+                    case '\u02A6': // ʦ  [LATIN SMALL LETTER TS DIGRAPH]
+                        output[outputPos++] = 't';
+                        output[outputPos++] = 's';
+                        break;
+                    case '\uA729': // ꜩ  [LATIN SMALL LETTER TZ]
+                        output[outputPos++] = 't';
+                        output[outputPos++] = 'z';
+                        break;
+                    case '\u00D9': // Ù  [LATIN CAPITAL LETTER U WITH GRAVE]
+                    case '\u00DA': // Ú  [LATIN CAPITAL LETTER U WITH ACUTE]
+                    case '\u00DB': // Û  [LATIN CAPITAL LETTER U WITH CIRCUMFLEX]
+                    case '\u00DC': // Ü  [LATIN CAPITAL LETTER U WITH DIAERESIS]
+                    case '\u0168': // Ũ  [LATIN CAPITAL LETTER U WITH TILDE]
+                    case '\u016A': // Ū  [LATIN CAPITAL LETTER U WITH MACRON]
+                    case '\u016C': // Ŭ  [LATIN CAPITAL LETTER U WITH BREVE]
+                    case '\u016E': // Ů  [LATIN CAPITAL LETTER U WITH RING ABOVE]
+                    case '\u0170': // Ű  [LATIN CAPITAL LETTER U WITH DOUBLE ACUTE]
+                    case '\u0172': // Ų  [LATIN CAPITAL LETTER U WITH OGONEK]
+                    case '\u01AF': // Ư  [LATIN CAPITAL LETTER U WITH HORN]
+                    case '\u01D3': // Ǔ  [LATIN CAPITAL LETTER U WITH CARON]
+                    case '\u01D5': // Ǖ  [LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON]
+                    case '\u01D7': // Ǘ  [LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE]
+                    case '\u01D9': // Ǚ  [LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON]
+                    case '\u01DB': // Ǜ  [LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE]
+                    case '\u0214': // Ȕ  [LATIN CAPITAL LETTER U WITH DOUBLE GRAVE]
+                    case '\u0216': // Ȗ  [LATIN CAPITAL LETTER U WITH INVERTED BREVE]
+                    case '\u0244': // Ʉ  [LATIN CAPITAL LETTER U BAR]
+                    case '\u1D1C': // ᴜ  [LATIN LETTER SMALL CAPITAL U]
+                    case '\u1D7E': // ᵾ  [LATIN SMALL CAPITAL LETTER U WITH STROKE]
+                    case '\u1E72': // Ṳ  [LATIN CAPITAL LETTER U WITH DIAERESIS BELOW]
+                    case '\u1E74': // Ṵ  [LATIN CAPITAL LETTER U WITH TILDE BELOW]
+                    case '\u1E76': // Ṷ  [LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW]
+                    case '\u1E78': // Ṹ  [LATIN CAPITAL LETTER U WITH TILDE AND ACUTE]
+                    case '\u1E7A': // Ṻ  [LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS]
+                    case '\u1EE4': // Ụ  [LATIN CAPITAL LETTER U WITH DOT BELOW]
+                    case '\u1EE6': // Ủ  [LATIN CAPITAL LETTER U WITH HOOK ABOVE]
+                    case '\u1EE8': // Ứ  [LATIN CAPITAL LETTER U WITH HORN AND ACUTE]
+                    case '\u1EEA': // Ừ  [LATIN CAPITAL LETTER U WITH HORN AND GRAVE]
+                    case '\u1EEC': // Ử  [LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE]
+                    case '\u1EEE': // Ữ  [LATIN CAPITAL LETTER U WITH HORN AND TILDE]
+                    case '\u1EF0': // Ự  [LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW]
+                    case '\u24CA': // Ⓤ  [CIRCLED LATIN CAPITAL LETTER U]
+                    case '\uFF35': // Ｕ  [FULLWIDTH LATIN CAPITAL LETTER U]
+                        output[outputPos++] = 'U';
+                        break;
+                    case '\u00F9': // ù  [LATIN SMALL LETTER U WITH GRAVE]
+                    case '\u00FA': // ú  [LATIN SMALL LETTER U WITH ACUTE]
+                    case '\u00FB': // û  [LATIN SMALL LETTER U WITH CIRCUMFLEX]
+                    case '\u00FC': // ü  [LATIN SMALL LETTER U WITH DIAERESIS]
+                    case '\u0169': // ũ  [LATIN SMALL LETTER U WITH TILDE]
+                    case '\u016B': // ū  [LATIN SMALL LETTER U WITH MACRON]
+                    case '\u016D': // ŭ  [LATIN SMALL LETTER U WITH BREVE]
+                    case '\u016F': // ů  [LATIN SMALL LETTER U WITH RING ABOVE]
+                    case '\u0171': // ű  [LATIN SMALL LETTER U WITH DOUBLE ACUTE]
+                    case '\u0173': // ų  [LATIN SMALL LETTER U WITH OGONEK]
+                    case '\u01B0': // ư  [LATIN SMALL LETTER U WITH HORN]
+                    case '\u01D4': // ǔ  [LATIN SMALL LETTER U WITH CARON]
+                    case '\u01D6': // ǖ  [LATIN SMALL LETTER U WITH DIAERESIS AND MACRON]
+                    case '\u01D8': // ǘ  [LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE]
+                    case '\u01DA': // ǚ  [LATIN SMALL LETTER U WITH DIAERESIS AND CARON]
+                    case '\u01DC': // ǜ  [LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE]
+                    case '\u0215': // ȕ  [LATIN SMALL LETTER U WITH DOUBLE GRAVE]
+                    case '\u0217': // ȗ  [LATIN SMALL LETTER U WITH INVERTED BREVE]
+                    case '\u0289': // ʉ  [LATIN SMALL LETTER U BAR]
+                    case '\u1D64': // ᵤ  [LATIN SUBSCRIPT SMALL LETTER U]
+                    case '\u1D99': // ᶙ  [LATIN SMALL LETTER U WITH RETROFLEX HOOK]
+                    case '\u1E73': // ṳ  [LATIN SMALL LETTER U WITH DIAERESIS BELOW]
+                    case '\u1E75': // ṵ  [LATIN SMALL LETTER U WITH TILDE BELOW]
+                    case '\u1E77': // ṷ  [LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW]
+                    case '\u1E79': // ṹ  [LATIN SMALL LETTER U WITH TILDE AND ACUTE]
+                    case '\u1E7B': // ṻ  [LATIN SMALL LETTER U WITH MACRON AND DIAERESIS]
+                    case '\u1EE5': // ụ  [LATIN SMALL LETTER U WITH DOT BELOW]
+                    case '\u1EE7': // ủ  [LATIN SMALL LETTER U WITH HOOK ABOVE]
+                    case '\u1EE9': // ứ  [LATIN SMALL LETTER U WITH HORN AND ACUTE]
+                    case '\u1EEB': // ừ  [LATIN SMALL LETTER U WITH HORN AND GRAVE]
+                    case '\u1EED': // ử  [LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE]
+                    case '\u1EEF': // ữ  [LATIN SMALL LETTER U WITH HORN AND TILDE]
+                    case '\u1EF1': // ự  [LATIN SMALL LETTER U WITH HORN AND DOT BELOW]
+                    case '\u24E4': // ⓤ  [CIRCLED LATIN SMALL LETTER U]
+                    case '\uFF55': // ｕ  [FULLWIDTH LATIN SMALL LETTER U]
+                        output[outputPos++] = 'u';
+                        break;
+                    case '\u24B0': // ⒰  [PARENTHESIZED LATIN SMALL LETTER U]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'u';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u1D6B': // ᵫ  [LATIN SMALL LETTER UE]
+                        output[outputPos++] = 'u';
+                        output[outputPos++] = 'e';
+                        break;
+                    case '\u01B2': // Ʋ  [LATIN CAPITAL LETTER V WITH HOOK]
+                    case '\u0245': // Ʌ  [LATIN CAPITAL LETTER TURNED V]
+                    case '\u1D20': // ᴠ  [LATIN LETTER SMALL CAPITAL V]
+                    case '\u1E7C': // Ṽ  [LATIN CAPITAL LETTER V WITH TILDE]
+                    case '\u1E7E': // Ṿ  [LATIN CAPITAL LETTER V WITH DOT BELOW]
+                    case '\u1EFC': // Ỽ  [LATIN CAPITAL LETTER MIDDLE-WELSH V]
+                    case '\u24CB': // Ⓥ  [CIRCLED LATIN CAPITAL LETTER V]
+                    case '\uA75E': // Ꝟ  [LATIN CAPITAL LETTER V WITH DIAGONAL STROKE]
+                    case '\uA768': // Ꝩ  [LATIN CAPITAL LETTER VEND]
+                    case '\uFF36': // Ｖ  [FULLWIDTH LATIN CAPITAL LETTER V]
+                        output[outputPos++] = 'V';
+                        break;
+                    case '\u028B': // ʋ  [LATIN SMALL LETTER V WITH HOOK]
+                    case '\u028C': // ʌ  [LATIN SMALL LETTER TURNED V]
+                    case '\u1D65': // ᵥ  [LATIN SUBSCRIPT SMALL LETTER V]
+                    case '\u1D8C': // ᶌ  [LATIN SMALL LETTER V WITH PALATAL HOOK]
+                    case '\u1E7D': // ṽ  [LATIN SMALL LETTER V WITH TILDE]
+                    case '\u1E7F': // ṿ  [LATIN SMALL LETTER V WITH DOT BELOW]
+                    case '\u24E5': // ⓥ  [CIRCLED LATIN SMALL LETTER V]
+                    case '\u2C71': // ⱱ  [LATIN SMALL LETTER V WITH RIGHT HOOK]
+                    case '\u2C74': // ⱴ  [LATIN SMALL LETTER V WITH CURL]
+                    case '\uA75F': // ꝟ  [LATIN SMALL LETTER V WITH DIAGONAL STROKE]
+                    case '\uFF56': // ｖ  [FULLWIDTH LATIN SMALL LETTER V]
+                        output[outputPos++] = 'v';
+                        break;
+                    case '\uA760': // Ꝡ  [LATIN CAPITAL LETTER VY]
+                        output[outputPos++] = 'V';
+                        output[outputPos++] = 'Y';
+                        break;
+                    case '\u24B1': // ⒱  [PARENTHESIZED LATIN SMALL LETTER V]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'v';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\uA761': // ꝡ  [LATIN SMALL LETTER VY]
+                        output[outputPos++] = 'v';
+                        output[outputPos++] = 'y';
+                        break;
+                    case '\u0174': // Ŵ  [LATIN CAPITAL LETTER W WITH CIRCUMFLEX]
+                    case '\u01F7': // Ƿ  http://en.wikipedia.org/wiki/Wynn  [LATIN CAPITAL LETTER WYNN]
+                    case '\u1D21': // ᴡ  [LATIN LETTER SMALL CAPITAL W]
+                    case '\u1E80': // Ẁ  [LATIN CAPITAL LETTER W WITH GRAVE]
+                    case '\u1E82': // Ẃ  [LATIN CAPITAL LETTER W WITH ACUTE]
+                    case '\u1E84': // Ẅ  [LATIN CAPITAL LETTER W WITH DIAERESIS]
+                    case '\u1E86': // Ẇ  [LATIN CAPITAL LETTER W WITH DOT ABOVE]
+                    case '\u1E88': // Ẉ  [LATIN CAPITAL LETTER W WITH DOT BELOW]
+                    case '\u24CC': // Ⓦ  [CIRCLED LATIN CAPITAL LETTER W]
+                    case '\u2C72': // Ⱳ  [LATIN CAPITAL LETTER W WITH HOOK]
+                    case '\uFF37': // Ｗ  [FULLWIDTH LATIN CAPITAL LETTER W]
+                        output[outputPos++] = 'W';
+                        break;
+                    case '\u0175': // ŵ  [LATIN SMALL LETTER W WITH CIRCUMFLEX]
+                    case '\u01BF': // ƿ  http://en.wikipedia.org/wiki/Wynn  [LATIN LETTER WYNN]
+                    case '\u028D': // ʍ  [LATIN SMALL LETTER TURNED W]
+                    case '\u1E81': // ẁ  [LATIN SMALL LETTER W WITH GRAVE]
+                    case '\u1E83': // ẃ  [LATIN SMALL LETTER W WITH ACUTE]
+                    case '\u1E85': // ẅ  [LATIN SMALL LETTER W WITH DIAERESIS]
+                    case '\u1E87': // ẇ  [LATIN SMALL LETTER W WITH DOT ABOVE]
+                    case '\u1E89': // ẉ  [LATIN SMALL LETTER W WITH DOT BELOW]
+                    case '\u1E98': // ẘ  [LATIN SMALL LETTER W WITH RING ABOVE]
+                    case '\u24E6': // ⓦ  [CIRCLED LATIN SMALL LETTER W]
+                    case '\u2C73': // ⱳ  [LATIN SMALL LETTER W WITH HOOK]
+                    case '\uFF57': // ｗ  [FULLWIDTH LATIN SMALL LETTER W]
+                        output[outputPos++] = 'w';
+                        break;
+                    case '\u24B2': // ⒲  [PARENTHESIZED LATIN SMALL LETTER W]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'w';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u1E8A': // Ẋ  [LATIN CAPITAL LETTER X WITH DOT ABOVE]
+                    case '\u1E8C': // Ẍ  [LATIN CAPITAL LETTER X WITH DIAERESIS]
+                    case '\u24CD': // Ⓧ  [CIRCLED LATIN CAPITAL LETTER X]
+                    case '\uFF38': // Ｘ  [FULLWIDTH LATIN CAPITAL LETTER X]
+                        output[outputPos++] = 'X';
+                        break;
+                    case '\u1D8D': // ᶍ  [LATIN SMALL LETTER X WITH PALATAL HOOK]
+                    case '\u1E8B': // ẋ  [LATIN SMALL LETTER X WITH DOT ABOVE]
+                    case '\u1E8D': // ẍ  [LATIN SMALL LETTER X WITH DIAERESIS]
+                    case '\u2093': // ₓ  [LATIN SUBSCRIPT SMALL LETTER X]
+                    case '\u24E7': // ⓧ  [CIRCLED LATIN SMALL LETTER X]
+                    case '\uFF58': // ｘ  [FULLWIDTH LATIN SMALL LETTER X]
+                        output[outputPos++] = 'x';
+                        break;
+                    case '\u24B3': // ⒳  [PARENTHESIZED LATIN SMALL LETTER X]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'x';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u00DD': // Ý  [LATIN CAPITAL LETTER Y WITH ACUTE]
+                    case '\u0176': // Ŷ  [LATIN CAPITAL LETTER Y WITH CIRCUMFLEX]
+                    case '\u0178': // Ÿ  [LATIN CAPITAL LETTER Y WITH DIAERESIS]
+                    case '\u01B3': // Ƴ  [LATIN CAPITAL LETTER Y WITH HOOK]
+                    case '\u0232': // Ȳ  [LATIN CAPITAL LETTER Y WITH MACRON]
+                    case '\u024E': // Ɏ  [LATIN CAPITAL LETTER Y WITH STROKE]
+                    case '\u028F': // ʏ  [LATIN LETTER SMALL CAPITAL Y]
+                    case '\u1E8E': // Ẏ  [LATIN CAPITAL LETTER Y WITH DOT ABOVE]
+                    case '\u1EF2': // Ỳ  [LATIN CAPITAL LETTER Y WITH GRAVE]
+                    case '\u1EF4': // Ỵ  [LATIN CAPITAL LETTER Y WITH DOT BELOW]
+                    case '\u1EF6': // Ỷ  [LATIN CAPITAL LETTER Y WITH HOOK ABOVE]
+                    case '\u1EF8': // Ỹ  [LATIN CAPITAL LETTER Y WITH TILDE]
+                    case '\u1EFE': // Ỿ  [LATIN CAPITAL LETTER Y WITH LOOP]
+                    case '\u24CE': // Ⓨ  [CIRCLED LATIN CAPITAL LETTER Y]
+                    case '\uFF39': // Ｙ  [FULLWIDTH LATIN CAPITAL LETTER Y]
+                        output[outputPos++] = 'Y';
+                        break;
+                    case '\u00FD': // ý  [LATIN SMALL LETTER Y WITH ACUTE]
+                    case '\u00FF': // ÿ  [LATIN SMALL LETTER Y WITH DIAERESIS]
+                    case '\u0177': // ŷ  [LATIN SMALL LETTER Y WITH CIRCUMFLEX]
+                    case '\u01B4': // ƴ  [LATIN SMALL LETTER Y WITH HOOK]
+                    case '\u0233': // ȳ  [LATIN SMALL LETTER Y WITH MACRON]
+                    case '\u024F': // ɏ  [LATIN SMALL LETTER Y WITH STROKE]
+                    case '\u028E': // ʎ  [LATIN SMALL LETTER TURNED Y]
+                    case '\u1E8F': // ẏ  [LATIN SMALL LETTER Y WITH DOT ABOVE]
+                    case '\u1E99': // ẙ  [LATIN SMALL LETTER Y WITH RING ABOVE]
+                    case '\u1EF3': // ỳ  [LATIN SMALL LETTER Y WITH GRAVE]
+                    case '\u1EF5': // ỵ  [LATIN SMALL LETTER Y WITH DOT BELOW]
+                    case '\u1EF7': // ỷ  [LATIN SMALL LETTER Y WITH HOOK ABOVE]
+                    case '\u1EF9': // ỹ  [LATIN SMALL LETTER Y WITH TILDE]
+                    case '\u1EFF': // ỿ  [LATIN SMALL LETTER Y WITH LOOP]
+                    case '\u24E8': // ⓨ  [CIRCLED LATIN SMALL LETTER Y]
+                    case '\uFF59': // ｙ  [FULLWIDTH LATIN SMALL LETTER Y]
+                        output[outputPos++] = 'y';
+                        break;
+                    case '\u24B4': // ⒴  [PARENTHESIZED LATIN SMALL LETTER Y]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'y';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u0179': // Ź  [LATIN CAPITAL LETTER Z WITH ACUTE]
+                    case '\u017B': // Ż  [LATIN CAPITAL LETTER Z WITH DOT ABOVE]
+                    case '\u017D': // Ž  [LATIN CAPITAL LETTER Z WITH CARON]
+                    case '\u01B5': // Ƶ  [LATIN CAPITAL LETTER Z WITH STROKE]
+                    case '\u021C': // Ȝ  http://en.wikipedia.org/wiki/Yogh  [LATIN CAPITAL LETTER YOGH]
+                    case '\u0224': // Ȥ  [LATIN CAPITAL LETTER Z WITH HOOK]
+                    case '\u1D22': // ᴢ  [LATIN LETTER SMALL CAPITAL Z]
+                    case '\u1E90': // Ẑ  [LATIN CAPITAL LETTER Z WITH CIRCUMFLEX]
+                    case '\u1E92': // Ẓ  [LATIN CAPITAL LETTER Z WITH DOT BELOW]
+                    case '\u1E94': // Ẕ  [LATIN CAPITAL LETTER Z WITH LINE BELOW]
+                    case '\u24CF': // Ⓩ  [CIRCLED LATIN CAPITAL LETTER Z]
+                    case '\u2C6B': // Ⱬ  [LATIN CAPITAL LETTER Z WITH DESCENDER]
+                    case '\uA762': // Ꝣ  [LATIN CAPITAL LETTER VISIGOTHIC Z]
+                    case '\uFF3A': // Ｚ  [FULLWIDTH LATIN CAPITAL LETTER Z]
+                        output[outputPos++] = 'Z';
+                        break;
+                    case '\u017A': // ź  [LATIN SMALL LETTER Z WITH ACUTE]
+                    case '\u017C': // ż  [LATIN SMALL LETTER Z WITH DOT ABOVE]
+                    case '\u017E': // ž  [LATIN SMALL LETTER Z WITH CARON]
+                    case '\u01B6': // ƶ  [LATIN SMALL LETTER Z WITH STROKE]
+                    case '\u021D': // ȝ  http://en.wikipedia.org/wiki/Yogh  [LATIN SMALL LETTER YOGH]
+                    case '\u0225': // ȥ  [LATIN SMALL LETTER Z WITH HOOK]
+                    case '\u0240': // ɀ  [LATIN SMALL LETTER Z WITH SWASH TAIL]
+                    case '\u0290': // ʐ  [LATIN SMALL LETTER Z WITH RETROFLEX HOOK]
+                    case '\u0291': // ʑ  [LATIN SMALL LETTER Z WITH CURL]
+                    case '\u1D76': // ᵶ  [LATIN SMALL LETTER Z WITH MIDDLE TILDE]
+                    case '\u1D8E': // ᶎ  [LATIN SMALL LETTER Z WITH PALATAL HOOK]
+                    case '\u1E91': // ẑ  [LATIN SMALL LETTER Z WITH CIRCUMFLEX]
+                    case '\u1E93': // ẓ  [LATIN SMALL LETTER Z WITH DOT BELOW]
+                    case '\u1E95': // ẕ  [LATIN SMALL LETTER Z WITH LINE BELOW]
+                    case '\u24E9': // ⓩ  [CIRCLED LATIN SMALL LETTER Z]
+                    case '\u2C6C': // ⱬ  [LATIN SMALL LETTER Z WITH DESCENDER]
+                    case '\uA763': // ꝣ  [LATIN SMALL LETTER VISIGOTHIC Z]
+                    case '\uFF5A': // ｚ  [FULLWIDTH LATIN SMALL LETTER Z]
+                        output[outputPos++] = 'z';
+                        break;
+                    case '\u24B5': // ⒵  [PARENTHESIZED LATIN SMALL LETTER Z]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = 'z';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2070': // ⁰  [SUPERSCRIPT ZERO]
+                    case '\u2080': // ₀  [SUBSCRIPT ZERO]
+                    case '\u24EA': // ⓪  [CIRCLED DIGIT ZERO]
+                    case '\u24FF': // ⓿  [NEGATIVE CIRCLED DIGIT ZERO]
+                    case '\uFF10': // ０  [FULLWIDTH DIGIT ZERO]
+                        output[outputPos++] = '0';
+                        break;
+                    case '\u00B9': // ¹  [SUPERSCRIPT ONE]
+                    case '\u2081': // ₁  [SUBSCRIPT ONE]
+                    case '\u2460': // ①  [CIRCLED DIGIT ONE]
+                    case '\u24F5': // ⓵  [DOUBLE CIRCLED DIGIT ONE]
+                    case '\u2776': // ❶  [DINGBAT NEGATIVE CIRCLED DIGIT ONE]
+                    case '\u2780': // ➀  [DINGBAT CIRCLED SANS-SERIF DIGIT ONE]
+                    case '\u278A': // ➊  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ONE]
+                    case '\uFF11': // １  [FULLWIDTH DIGIT ONE]
+                        output[outputPos++] = '1';
+                        break;
+                    case '\u2488': // ⒈  [DIGIT ONE FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2474': // ⑴  [PARENTHESIZED DIGIT ONE]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u00B2': // ²  [SUPERSCRIPT TWO]
+                    case '\u2082': // ₂  [SUBSCRIPT TWO]
+                    case '\u2461': // ②  [CIRCLED DIGIT TWO]
+                    case '\u24F6': // ⓶  [DOUBLE CIRCLED DIGIT TWO]
+                    case '\u2777': // ❷  [DINGBAT NEGATIVE CIRCLED DIGIT TWO]
+                    case '\u2781': // ➁  [DINGBAT CIRCLED SANS-SERIF DIGIT TWO]
+                    case '\u278B': // ➋  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT TWO]
+                    case '\uFF12': // ２  [FULLWIDTH DIGIT TWO]
+                        output[outputPos++] = '2';
+                        break;
+                    case '\u2489': // ⒉  [DIGIT TWO FULL STOP]
+                        output[outputPos++] = '2';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2475': // ⑵  [PARENTHESIZED DIGIT TWO]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '2';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u00B3': // ³  [SUPERSCRIPT THREE]
+                    case '\u2083': // ₃  [SUBSCRIPT THREE]
+                    case '\u2462': // ③  [CIRCLED DIGIT THREE]
+                    case '\u24F7': // ⓷  [DOUBLE CIRCLED DIGIT THREE]
+                    case '\u2778': // ❸  [DINGBAT NEGATIVE CIRCLED DIGIT THREE]
+                    case '\u2782': // ➂  [DINGBAT CIRCLED SANS-SERIF DIGIT THREE]
+                    case '\u278C': // ➌  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT THREE]
+                    case '\uFF13': // ３  [FULLWIDTH DIGIT THREE]
+                        output[outputPos++] = '3';
+                        break;
+                    case '\u248A': // ⒊  [DIGIT THREE FULL STOP]
+                        output[outputPos++] = '3';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2476': // ⑶  [PARENTHESIZED DIGIT THREE]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '3';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2074': // ⁴  [SUPERSCRIPT FOUR]
+                    case '\u2084': // ₄  [SUBSCRIPT FOUR]
+                    case '\u2463': // ④  [CIRCLED DIGIT FOUR]
+                    case '\u24F8': // ⓸  [DOUBLE CIRCLED DIGIT FOUR]
+                    case '\u2779': // ❹  [DINGBAT NEGATIVE CIRCLED DIGIT FOUR]
+                    case '\u2783': // ➃  [DINGBAT CIRCLED SANS-SERIF DIGIT FOUR]
+                    case '\u278D': // ➍  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FOUR]
+                    case '\uFF14': // ４  [FULLWIDTH DIGIT FOUR]
+                        output[outputPos++] = '4';
+                        break;
+                    case '\u248B': // ⒋  [DIGIT FOUR FULL STOP]
+                        output[outputPos++] = '4';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2477': // ⑷  [PARENTHESIZED DIGIT FOUR]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '4';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2075': // ⁵  [SUPERSCRIPT FIVE]
+                    case '\u2085': // ₅  [SUBSCRIPT FIVE]
+                    case '\u2464': // ⑤  [CIRCLED DIGIT FIVE]
+                    case '\u24F9': // ⓹  [DOUBLE CIRCLED DIGIT FIVE]
+                    case '\u277A': // ❺  [DINGBAT NEGATIVE CIRCLED DIGIT FIVE]
+                    case '\u2784': // ➄  [DINGBAT CIRCLED SANS-SERIF DIGIT FIVE]
+                    case '\u278E': // ➎  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FIVE]
+                    case '\uFF15': // ５  [FULLWIDTH DIGIT FIVE]
+                        output[outputPos++] = '5';
+                        break;
+                    case '\u248C': // ⒌  [DIGIT FIVE FULL STOP]
+                        output[outputPos++] = '5';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2478': // ⑸  [PARENTHESIZED DIGIT FIVE]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '5';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2076': // ⁶  [SUPERSCRIPT SIX]
+                    case '\u2086': // ₆  [SUBSCRIPT SIX]
+                    case '\u2465': // ⑥  [CIRCLED DIGIT SIX]
+                    case '\u24FA': // ⓺  [DOUBLE CIRCLED DIGIT SIX]
+                    case '\u277B': // ❻  [DINGBAT NEGATIVE CIRCLED DIGIT SIX]
+                    case '\u2785': // ➅  [DINGBAT CIRCLED SANS-SERIF DIGIT SIX]
+                    case '\u278F': // ➏  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SIX]
+                    case '\uFF16': // ６  [FULLWIDTH DIGIT SIX]
+                        output[outputPos++] = '6';
+                        break;
+                    case '\u248D': // ⒍  [DIGIT SIX FULL STOP]
+                        output[outputPos++] = '6';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2479': // ⑹  [PARENTHESIZED DIGIT SIX]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '6';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2077': // ⁷  [SUPERSCRIPT SEVEN]
+                    case '\u2087': // ₇  [SUBSCRIPT SEVEN]
+                    case '\u2466': // ⑦  [CIRCLED DIGIT SEVEN]
+                    case '\u24FB': // ⓻  [DOUBLE CIRCLED DIGIT SEVEN]
+                    case '\u277C': // ❼  [DINGBAT NEGATIVE CIRCLED DIGIT SEVEN]
+                    case '\u2786': // ➆  [DINGBAT CIRCLED SANS-SERIF DIGIT SEVEN]
+                    case '\u2790': // ➐  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SEVEN]
+                    case '\uFF17': // ７  [FULLWIDTH DIGIT SEVEN]
+                        output[outputPos++] = '7';
+                        break;
+                    case '\u248E': // ⒎  [DIGIT SEVEN FULL STOP]
+                        output[outputPos++] = '7';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u247A': // ⑺  [PARENTHESIZED DIGIT SEVEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '7';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2078': // ⁸  [SUPERSCRIPT EIGHT]
+                    case '\u2088': // ₈  [SUBSCRIPT EIGHT]
+                    case '\u2467': // ⑧  [CIRCLED DIGIT EIGHT]
+                    case '\u24FC': // ⓼  [DOUBLE CIRCLED DIGIT EIGHT]
+                    case '\u277D': // ❽  [DINGBAT NEGATIVE CIRCLED DIGIT EIGHT]
+                    case '\u2787': // ➇  [DINGBAT CIRCLED SANS-SERIF DIGIT EIGHT]
+                    case '\u2791': // ➑  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT EIGHT]
+                    case '\uFF18': // ８  [FULLWIDTH DIGIT EIGHT]
+                        output[outputPos++] = '8';
+                        break;
+                    case '\u248F': // ⒏  [DIGIT EIGHT FULL STOP]
+                        output[outputPos++] = '8';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u247B': // ⑻  [PARENTHESIZED DIGIT EIGHT]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '8';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2079': // ⁹  [SUPERSCRIPT NINE]
+                    case '\u2089': // ₉  [SUBSCRIPT NINE]
+                    case '\u2468': // ⑨  [CIRCLED DIGIT NINE]
+                    case '\u24FD': // ⓽  [DOUBLE CIRCLED DIGIT NINE]
+                    case '\u277E': // ❾  [DINGBAT NEGATIVE CIRCLED DIGIT NINE]
+                    case '\u2788': // ➈  [DINGBAT CIRCLED SANS-SERIF DIGIT NINE]
+                    case '\u2792': // ➒  [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT NINE]
+                    case '\uFF19': // ９  [FULLWIDTH DIGIT NINE]
+                        output[outputPos++] = '9';
+                        break;
+                    case '\u2490': // ⒐  [DIGIT NINE FULL STOP]
+                        output[outputPos++] = '9';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u247C': // ⑼  [PARENTHESIZED DIGIT NINE]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '9';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2469': // ⑩  [CIRCLED NUMBER TEN]
+                    case '\u24FE': // ⓾  [DOUBLE CIRCLED NUMBER TEN]
+                    case '\u277F': // ❿  [DINGBAT NEGATIVE CIRCLED NUMBER TEN]
+                    case '\u2789': // ➉  [DINGBAT CIRCLED SANS-SERIF NUMBER TEN]
+                    case '\u2793': // ➓  [DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '0';
+                        break;
+                    case '\u2491': // ⒑  [NUMBER TEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '0';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u247D': // ⑽  [PARENTHESIZED NUMBER TEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '0';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u246A': // ⑪  [CIRCLED NUMBER ELEVEN]
+                    case '\u24EB': // ⓫  [NEGATIVE CIRCLED NUMBER ELEVEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '1';
+                        break;
+                    case '\u2492': // ⒒  [NUMBER ELEVEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u247E': // ⑾  [PARENTHESIZED NUMBER ELEVEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u246B': // ⑫  [CIRCLED NUMBER TWELVE]
+                    case '\u24EC': // ⓬  [NEGATIVE CIRCLED NUMBER TWELVE]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '2';
+                        break;
+                    case '\u2493': // ⒓  [NUMBER TWELVE FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '2';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u247F': // ⑿  [PARENTHESIZED NUMBER TWELVE]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '2';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u246C': // ⑬  [CIRCLED NUMBER THIRTEEN]
+                    case '\u24ED': // ⓭  [NEGATIVE CIRCLED NUMBER THIRTEEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '3';
+                        break;
+                    case '\u2494': // ⒔  [NUMBER THIRTEEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '3';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2480': // ⒀  [PARENTHESIZED NUMBER THIRTEEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '3';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u246D': // ⑭  [CIRCLED NUMBER FOURTEEN]
+                    case '\u24EE': // ⓮  [NEGATIVE CIRCLED NUMBER FOURTEEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '4';
+                        break;
+                    case '\u2495': // ⒕  [NUMBER FOURTEEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '4';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2481': // ⒁  [PARENTHESIZED NUMBER FOURTEEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '4';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u246E': // ⑮  [CIRCLED NUMBER FIFTEEN]
+                    case '\u24EF': // ⓯  [NEGATIVE CIRCLED NUMBER FIFTEEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '5';
+                        break;
+                    case '\u2496': // ⒖  [NUMBER FIFTEEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '5';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2482': // ⒂  [PARENTHESIZED NUMBER FIFTEEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '5';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u246F': // ⑯  [CIRCLED NUMBER SIXTEEN]
+                    case '\u24F0': // ⓰  [NEGATIVE CIRCLED NUMBER SIXTEEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '6';
+                        break;
+                    case '\u2497': // ⒗  [NUMBER SIXTEEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '6';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2483': // ⒃  [PARENTHESIZED NUMBER SIXTEEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '6';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2470': // ⑰  [CIRCLED NUMBER SEVENTEEN]
+                    case '\u24F1': // ⓱  [NEGATIVE CIRCLED NUMBER SEVENTEEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '7';
+                        break;
+                    case '\u2498': // ⒘  [NUMBER SEVENTEEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '7';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2484': // ⒄  [PARENTHESIZED NUMBER SEVENTEEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '7';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2471': // ⑱  [CIRCLED NUMBER EIGHTEEN]
+                    case '\u24F2': // ⓲  [NEGATIVE CIRCLED NUMBER EIGHTEEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '8';
+                        break;
+                    case '\u2499': // ⒙  [NUMBER EIGHTEEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '8';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2485': // ⒅  [PARENTHESIZED NUMBER EIGHTEEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '8';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2472': // ⑲  [CIRCLED NUMBER NINETEEN]
+                    case '\u24F3': // ⓳  [NEGATIVE CIRCLED NUMBER NINETEEN]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '9';
+                        break;
+                    case '\u249A': // ⒚  [NUMBER NINETEEN FULL STOP]
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '9';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2486': // ⒆  [PARENTHESIZED NUMBER NINETEEN]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '1';
+                        output[outputPos++] = '9';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2473': // ⑳  [CIRCLED NUMBER TWENTY]
+                    case '\u24F4': // ⓴  [NEGATIVE CIRCLED NUMBER TWENTY]
+                        output[outputPos++] = '2';
+                        output[outputPos++] = '0';
+                        break;
+                    case '\u249B': // ⒛  [NUMBER TWENTY FULL STOP]
+                        output[outputPos++] = '2';
+                        output[outputPos++] = '0';
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2487': // ⒇  [PARENTHESIZED NUMBER TWENTY]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '2';
+                        output[outputPos++] = '0';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u00AB': // «  [LEFT-POINTING DOUBLE ANGLE QUOTATION MARK]
+                    case '\u00BB': // »  [RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK]
+                    case '\u201C': // “  [LEFT DOUBLE QUOTATION MARK]
+                    case '\u201D': // ”  [RIGHT DOUBLE QUOTATION MARK]
+                    case '\u201E': // „  [DOUBLE LOW-9 QUOTATION MARK]
+                    case '\u2033': // ″  [DOUBLE PRIME]
+                    case '\u2036': // ‶  [REVERSED DOUBLE PRIME]
+                    case '\u275D': // ❝  [HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT]
+                    case '\u275E': // ❞  [HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT]
+                    case '\u276E': // ❮  [HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT]
+                    case '\u276F': // ❯  [HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT]
+                    case '\uFF02': // ＂  [FULLWIDTH QUOTATION MARK]
+                        output[outputPos++] = '"';
+                        break;
+                    case '\u2018': // ‘  [LEFT SINGLE QUOTATION MARK]
+                    case '\u2019': // ’  [RIGHT SINGLE QUOTATION MARK]
+                    case '\u201A': // ‚  [SINGLE LOW-9 QUOTATION MARK]
+                    case '\u201B': // ‛  [SINGLE HIGH-REVERSED-9 QUOTATION MARK]
+                    case '\u2032': // ′  [PRIME]
+                    case '\u2035': // ‵  [REVERSED PRIME]
+                    case '\u2039': // ‹  [SINGLE LEFT-POINTING ANGLE QUOTATION MARK]
+                    case '\u203A': // ›  [SINGLE RIGHT-POINTING ANGLE QUOTATION MARK]
+                    case '\u275B': // ❛  [HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT]
+                    case '\u275C': // ❜  [HEAVY SINGLE COMMA QUOTATION MARK ORNAMENT]
+                    case '\uFF07': // ＇  [FULLWIDTH APOSTROPHE]
+                        output[outputPos++] = '\'';
+                        break;
+                    case '\u2010': // ‐  [HYPHEN]
+                    case '\u2011': // ‑  [NON-BREAKING HYPHEN]
+                    case '\u2012': // ‒  [FIGURE DASH]
+                    case '\u2013': // –  [EN DASH]
+                    case '\u2014': // —  [EM DASH]
+                    case '\u207B': // ⁻  [SUPERSCRIPT MINUS]
+                    case '\u208B': // ₋  [SUBSCRIPT MINUS]
+                    case '\uFF0D': // －  [FULLWIDTH HYPHEN-MINUS]
+                        output[outputPos++] = '-';
+                        break;
+                    case '\u2045': // ⁅  [LEFT SQUARE BRACKET WITH QUILL]
+                    case '\u2772': // ❲  [LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT]
+                    case '\uFF3B': // ［  [FULLWIDTH LEFT SQUARE BRACKET]
+                        output[outputPos++] = '[';
+                        break;
+                    case '\u2046': // ⁆  [RIGHT SQUARE BRACKET WITH QUILL]
+                    case '\u2773': // ❳  [LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT]
+                    case '\uFF3D': // ］  [FULLWIDTH RIGHT SQUARE BRACKET]
+                        output[outputPos++] = ']';
+                        break;
+                    case '\u207D': // ⁽  [SUPERSCRIPT LEFT PARENTHESIS]
+                    case '\u208D': // ₍  [SUBSCRIPT LEFT PARENTHESIS]
+                    case '\u2768': // ❨  [MEDIUM LEFT PARENTHESIS ORNAMENT]
+                    case '\u276A': // ❪  [MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT]
+                    case '\uFF08': // （  [FULLWIDTH LEFT PARENTHESIS]
+                        output[outputPos++] = '(';
+                        break;
+                    case '\u2E28': // ⸨  [LEFT DOUBLE PARENTHESIS]
+                        output[outputPos++] = '(';
+                        output[outputPos++] = '(';
+                        break;
+                    case '\u207E': // ⁾  [SUPERSCRIPT RIGHT PARENTHESIS]
+                    case '\u208E': // ₎  [SUBSCRIPT RIGHT PARENTHESIS]
+                    case '\u2769': // ❩  [MEDIUM RIGHT PARENTHESIS ORNAMENT]
+                    case '\u276B': // ❫  [MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT]
+                    case '\uFF09': // ）  [FULLWIDTH RIGHT PARENTHESIS]
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u2E29': // ⸩  [RIGHT DOUBLE PARENTHESIS]
+                        output[outputPos++] = ')';
+                        output[outputPos++] = ')';
+                        break;
+                    case '\u276C': // ❬  [MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT]
+                    case '\u2770': // ❰  [HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT]
+                    case '\uFF1C': // ＜  [FULLWIDTH LESS-THAN SIGN]
+                        output[outputPos++] = '<';
+                        break;
+                    case '\u276D': // ❭  [MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT]
+                    case '\u2771': // ❱  [HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT]
+                    case '\uFF1E': // ＞  [FULLWIDTH GREATER-THAN SIGN]
+                        output[outputPos++] = '>';
+                        break;
+                    case '\u2774': // ❴  [MEDIUM LEFT CURLY BRACKET ORNAMENT]
+                    case '\uFF5B': // ｛  [FULLWIDTH LEFT CURLY BRACKET]
+                        output[outputPos++] = '{';
+                        break;
+                    case '\u2775': // ❵  [MEDIUM RIGHT CURLY BRACKET ORNAMENT]
+                    case '\uFF5D': // ｝  [FULLWIDTH RIGHT CURLY BRACKET]
+                        output[outputPos++] = '}';
+                        break;
+                    case '\u207A': // ⁺  [SUPERSCRIPT PLUS SIGN]
+                    case '\u208A': // ₊  [SUBSCRIPT PLUS SIGN]
+                    case '\uFF0B': // ＋  [FULLWIDTH PLUS SIGN]
+                        output[outputPos++] = '+';
+                        break;
+                    case '\u207C': // ⁼  [SUPERSCRIPT EQUALS SIGN]
+                    case '\u208C': // ₌  [SUBSCRIPT EQUALS SIGN]
+                    case '\uFF1D': // ＝  [FULLWIDTH EQUALS SIGN]
+                        output[outputPos++] = '=';
+                        break;
+                    case '\uFF01': // ！  [FULLWIDTH EXCLAMATION MARK]
+                        output[outputPos++] = '!';
+                        break;
+                    case '\u203C': // ‼  [DOUBLE EXCLAMATION MARK]
+                        output[outputPos++] = '!';
+                        output[outputPos++] = '!';
+                        break;
+                    case '\u2049': // ⁉  [EXCLAMATION QUESTION MARK]
+                        output[outputPos++] = '!';
+                        output[outputPos++] = '?';
+                        break;
+                    case '\uFF03': // ＃  [FULLWIDTH NUMBER SIGN]
+                        output[outputPos++] = '#';
+                        break;
+                    case '\uFF04': // ＄  [FULLWIDTH DOLLAR SIGN]
+                        output[outputPos++] = '$';
+                        break;
+                    case '\u2052': // ⁒  [COMMERCIAL MINUS SIGN]
+                    case '\uFF05': // ％  [FULLWIDTH PERCENT SIGN]
+                        output[outputPos++] = '%';
+                        break;
+                    case '\uFF06': // ＆  [FULLWIDTH AMPERSAND]
+                        output[outputPos++] = '&';
+                        break;
+                    case '\u204E': // ⁎  [LOW ASTERISK]
+                    case '\uFF0A': // ＊  [FULLWIDTH ASTERISK]
+                        output[outputPos++] = '*';
+                        break;
+                    case '\uFF0C': // ，  [FULLWIDTH COMMA]
+                        output[outputPos++] = ',';
+                        break;
+                    case '\uFF0E': // ．  [FULLWIDTH FULL STOP]
+                        output[outputPos++] = '.';
+                        break;
+                    case '\u2044': // ⁄  [FRACTION SLASH]
+                    case '\uFF0F': // ／  [FULLWIDTH SOLIDUS]
+                        output[outputPos++] = '/';
+                        break;
+                    case '\uFF1A': // ：  [FULLWIDTH COLON]
+                        output[outputPos++] = ':';
+                        break;
+                    case '\u204F': // ⁏  [REVERSED SEMICOLON]
+                    case '\uFF1B': // ；  [FULLWIDTH SEMICOLON]
+                        output[outputPos++] = ';';
+                        break;
+                    case '\uFF1F': // ？  [FULLWIDTH QUESTION MARK]
+                        output[outputPos++] = '?';
+                        break;
+                    case '\u2047': // ⁇  [DOUBLE QUESTION MARK]
+                        output[outputPos++] = '?';
+                        output[outputPos++] = '?';
+                        break;
+                    case '\u2048': // ⁈  [QUESTION EXCLAMATION MARK]
+                        output[outputPos++] = '?';
+                        output[outputPos++] = '!';
+                        break;
+                    case '\uFF20': // ＠  [FULLWIDTH COMMERCIAL AT]
+                        output[outputPos++] = '@';
+                        break;
+                    case '\uFF3C': // ＼  [FULLWIDTH REVERSE SOLIDUS]
+                        output[outputPos++] = '\\';
+                        break;
+                    case '\u2038': // ‸  [CARET]
+                    case '\uFF3E': // ＾  [FULLWIDTH CIRCUMFLEX ACCENT]
+                        output[outputPos++] = '^';
+                        break;
+                    case '\uFF3F': // ＿  [FULLWIDTH LOW LINE]
+                        output[outputPos++] = '_';
+                        break;
+                    case '\u2053': // ⁓  [SWUNG DASH]
+                    case '\uFF5E': // ～  [FULLWIDTH TILDE]
+                        output[outputPos++] = '~';
+                        break;
+                    default:
+                        output[outputPos++] = c;
+                        break;
+                }
+            }
+        }
+        return outputPos;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/filter/BuiltInAnalyzers.java b/src/java/org/apache/cassandra/index/sai/analyzer/filter/BuiltInAnalyzers.java
new file mode 100644
index 000000000000..d318030c5902
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/filter/BuiltInAnalyzers.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.ar.ArabicAnalyzer;
+import org.apache.lucene.analysis.bg.BulgarianAnalyzer;
+import org.apache.lucene.analysis.bn.BengaliAnalyzer;
+import org.apache.lucene.analysis.br.BrazilianAnalyzer;
+import org.apache.lucene.analysis.ca.CatalanAnalyzer;
+import org.apache.lucene.analysis.cjk.CJKAnalyzer;
+import org.apache.lucene.analysis.ckb.SoraniAnalyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.core.SimpleAnalyzer;
+import org.apache.lucene.analysis.core.StopAnalyzer;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
+import org.apache.lucene.analysis.cz.CzechAnalyzer;
+import org.apache.lucene.analysis.da.DanishAnalyzer;
+import org.apache.lucene.analysis.de.GermanAnalyzer;
+import org.apache.lucene.analysis.el.GreekAnalyzer;
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
+import org.apache.lucene.analysis.es.SpanishAnalyzer;
+import org.apache.lucene.analysis.et.EstonianAnalyzer;
+import org.apache.lucene.analysis.eu.BasqueAnalyzer;
+import org.apache.lucene.analysis.fa.PersianAnalyzer;
+import org.apache.lucene.analysis.fi.FinnishAnalyzer;
+import org.apache.lucene.analysis.fr.FrenchAnalyzer;
+import org.apache.lucene.analysis.ga.IrishAnalyzer;
+import org.apache.lucene.analysis.gl.GalicianAnalyzer;
+import org.apache.lucene.analysis.hi.HindiAnalyzer;
+import org.apache.lucene.analysis.hu.HungarianAnalyzer;
+import org.apache.lucene.analysis.hy.ArmenianAnalyzer;
+import org.apache.lucene.analysis.id.IndonesianAnalyzer;
+import org.apache.lucene.analysis.it.ItalianAnalyzer;
+import org.apache.lucene.analysis.lt.LithuanianAnalyzer;
+import org.apache.lucene.analysis.lv.LatvianAnalyzer;
+import org.apache.lucene.analysis.nl.DutchAnalyzer;
+import org.apache.lucene.analysis.no.NorwegianAnalyzer;
+import org.apache.lucene.analysis.pt.PortugueseAnalyzer;
+import org.apache.lucene.analysis.ro.RomanianAnalyzer;
+import org.apache.lucene.analysis.ru.RussianAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.sv.SwedishAnalyzer;
+import org.apache.lucene.analysis.th.ThaiAnalyzer;
+import org.apache.lucene.analysis.tr.TurkishAnalyzer;
+
+/**
+ * Built-in {@link Analyzer} implementations. These are provided to allow users to easily configure analyzers with
+ * a single word.
+ */
+public enum BuiltInAnalyzers
+{
+    STANDARD
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new StandardAnalyzer();
+        }
+    },
+    SIMPLE
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new SimpleAnalyzer();
+        }
+    },
+    WHITESPACE
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new WhitespaceAnalyzer();
+        }
+    },
+    STOP
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new StopAnalyzer(EnglishAnalyzer.getDefaultStopSet());
+        }
+    },
+    LOWERCASE
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            try
+            {
+                CustomAnalyzer.Builder builder = CustomAnalyzer.builder();
+                builder.withTokenizer("keyword");
+                builder.addTokenFilter("lowercase");
+                return builder.build();
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    },
+    KEYWORD
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            try
+            {
+                return new KeywordAnalyzer();
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    },
+    ARABIC
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new ArabicAnalyzer();
+        }
+    },
+    ARMENIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new ArmenianAnalyzer();
+        }
+    },
+    BASQUE
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new BasqueAnalyzer();
+        }
+    },
+     BENGALI
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new BengaliAnalyzer();
+         }
+     },
+     BRAZILIAN
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new BrazilianAnalyzer();
+         }
+     },
+     BULGARIAN
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new BulgarianAnalyzer();
+         }
+     },
+     CATALAN
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new CatalanAnalyzer();
+         }
+     },
+     CJK
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new CJKAnalyzer();
+         }
+     },
+     CZECH
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new CzechAnalyzer();
+         }
+     },
+     DANISH
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new DanishAnalyzer();
+         }
+     },
+     DUTCH
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new DutchAnalyzer();
+         }
+     },
+     ENGLISH
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new EnglishAnalyzer();
+         }
+     },
+     ESTONIAN
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new EstonianAnalyzer();
+         }
+     },
+     FINNISH
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new FinnishAnalyzer();
+         }
+     },
+     FRENCH
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new FrenchAnalyzer();
+         }
+     },
+     GALICIAN
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new GalicianAnalyzer();
+         }
+     },
+     GERMAN
+     {
+         public Analyzer getNewAnalyzer()
+         {
+             return new GermanAnalyzer();
+         }
+     },
+    GREEK
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new GreekAnalyzer();
+        }
+    },
+    HINDI
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new HindiAnalyzer();
+        }
+    },
+    HUNGARIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new HungarianAnalyzer();
+        }
+    },
+    INDONESIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new IndonesianAnalyzer();
+        }
+    },
+    IRISH
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new IrishAnalyzer();
+        }
+    },
+    ITALIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new ItalianAnalyzer();
+        }
+    },
+    LATVIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new LatvianAnalyzer();
+        }
+    },
+    LITHUANIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new LithuanianAnalyzer();
+        }
+    },
+    NORWEGIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new NorwegianAnalyzer();
+        }
+    },
+    PERSIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new PersianAnalyzer();
+        }
+    },
+    PORTUGUESE
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new PortugueseAnalyzer();
+        }
+    },
+    ROMANIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new RomanianAnalyzer();
+        }
+    },
+    RUSSIAN
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new RussianAnalyzer();
+        }
+    },
+    SORANI
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new SoraniAnalyzer();
+        }
+    },
+    SPANISH
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new SpanishAnalyzer();
+        }
+    },
+    SWEDISH
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new SwedishAnalyzer();
+        }
+    },
+    TURKISH
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new TurkishAnalyzer();
+        }
+    },
+    THAI
+    {
+        public Analyzer getNewAnalyzer()
+        {
+            return new ThaiAnalyzer();
+        }
+    },
+    ;
+
+    public abstract Analyzer getNewAnalyzer();
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineBuilder.java b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineBuilder.java
new file mode 100644
index 000000000000..3a6a72603df2
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineBuilder.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+/**
+ * Creates a Pipeline object for applying n pieces of logic
+ * from the provided methods to the builder in a guaranteed order
+ */
+public class FilterPipelineBuilder
+{
+    private final FilterPipelineTask parent;
+    private FilterPipelineTask current;
+
+    public FilterPipelineBuilder(FilterPipelineTask first)
+    {
+        this(first, first);
+    }
+
+    private FilterPipelineBuilder(FilterPipelineTask first, FilterPipelineTask current)
+    {
+        this.parent = first;
+        this.current = current;
+    }
+
+    public FilterPipelineBuilder add(String name, FilterPipelineTask nextTask)
+    {
+        this.current.setLast(name, nextTask);
+        this.current = nextTask;
+        return this;
+    }
+
+    public FilterPipelineTask build()
+    {
+        return this.parent;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineExecutor.java b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineExecutor.java
index c863f1e3cdb6..dcb04c2a98a4 100644
--- a/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineExecutor.java
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineExecutor.java
@@ -19,22 +19,26 @@
 package org.apache.cassandra.index.sai.analyzer.filter;
 
 /**
- * Executes all linked {@link FilterPipeline.Task}s serially on the provided input and returns a result
+ * Executes all linked Pipeline Tasks serially and returns
+ * output (if exists) from the executed logic
  */
 public class FilterPipelineExecutor
 {
-    public static String execute(FilterPipeline pipeline, String initialInput)
+    public static String execute(FilterPipelineTask task, String initialInput)
     {
-        FilterPipeline.Task currentTask = pipeline.head();
+        FilterPipelineTask taskPtr = task;
         String result = initialInput;
         
         while (true)
         {
-            result = currentTask.process(result);
-            currentTask = currentTask.next;
+            FilterPipelineTask taskGeneric = taskPtr;
+            result = taskGeneric.process(result);
+            taskPtr = taskPtr.next;
             
-            if (currentTask == null)
+            if (taskPtr == null)
+            {
                 return result;
+            }
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineTask.java b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineTask.java
new file mode 100644
index 000000000000..b80073e8b3c2
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineTask.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+/**
+ * A single task or set of work to process an input
+ * and return a single output. Maintains a link to the
+ * next task to be executed after itself
+ */
+public abstract class FilterPipelineTask
+{
+    private String name;
+    public FilterPipelineTask next;
+
+    void setLast(String name, FilterPipelineTask last)
+    {
+        if (last == this)
+            throw new IllegalArgumentException("provided last task [" + last.name + "] cannot be set to itself");
+
+        if (this.next == null)
+        {
+            this.next = last;
+            this.name = name;
+        }
+        else
+        {
+            this.next.setLast(name, last);
+        }
+    }
+
+    public abstract String process(String input);
+
+    public String getName()
+    {
+        return name;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/ByteSliceReader.java b/src/java/org/apache/cassandra/index/sai/disk/ByteSliceReader.java
new file mode 100644
index 000000000000..78660ecaa6ce
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/ByteSliceReader.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.util.ByteBlockPool;
+
+/* IndexInput that knows how to read the byte slices written
+ * by Posting and PostingVector.  We read the bytes in
+ * each slice until we hit the end of that slice at which
+ * point we read the forwarding address of the next slice
+ * and then jump to it.*/
+final class ByteSliceReader extends DataInput
+{
+    ByteBlockPool pool;
+    int bufferUpto;
+    byte[] buffer;
+    public int upto;
+    int limit;
+    int level;
+    public int bufferOffset;
+
+    public int endIndex;
+
+    public void init(ByteBlockPool pool, int startIndex, int endIndex)
+    {
+
+        assert endIndex - startIndex >= 0 : "startIndex=" + startIndex + " endIndex=" + endIndex;
+        assert startIndex >= 0;
+        assert endIndex >= 0;
+
+        this.pool = pool;
+        this.endIndex = endIndex;
+
+        level = 0;
+        bufferUpto = startIndex / ByteBlockPool.BYTE_BLOCK_SIZE;
+        bufferOffset = bufferUpto * ByteBlockPool.BYTE_BLOCK_SIZE;
+        buffer = pool.buffers[bufferUpto];
+        upto = startIndex & ByteBlockPool.BYTE_BLOCK_MASK;
+
+        final int firstSize = ByteBlockPool.LEVEL_SIZE_ARRAY[0];
+
+        if (startIndex + firstSize >= endIndex)
+        {
+            // There is only this one slice to read
+            limit = endIndex & ByteBlockPool.BYTE_BLOCK_MASK;
+        }
+        else
+            limit = upto + firstSize - 4;
+    }
+
+    public boolean eof()
+    {
+        assert upto + bufferOffset <= endIndex;
+        return upto + bufferOffset == endIndex;
+    }
+
+    @Override
+    public byte readByte()
+    {
+        assert !eof();
+        assert upto <= limit;
+        if (upto == limit)
+            nextSlice();
+        return buffer[upto++];
+    }
+
+    public long writeTo(DataOutput out) throws IOException
+    {
+        long size = 0;
+        while (true)
+        {
+            if (limit + bufferOffset == endIndex)
+            {
+                assert endIndex - bufferOffset >= upto;
+                out.write(buffer, upto, limit - upto);
+                size += limit - upto;
+                break;
+            }
+            else
+            {
+                out.write(buffer, upto, limit - upto);
+                size += limit - upto;
+                nextSlice();
+            }
+        }
+
+        return size;
+    }
+
+    public void nextSlice()
+    {
+
+        // Skip to our next slice
+        final int nextIndex = (int) BitUtil.VH_LE_INT.get(buffer, limit);
+
+        level = ByteBlockPool.NEXT_LEVEL_ARRAY[level];
+        final int newSize = ByteBlockPool.LEVEL_SIZE_ARRAY[level];
+
+        bufferUpto = nextIndex / ByteBlockPool.BYTE_BLOCK_SIZE;
+        bufferOffset = bufferUpto * ByteBlockPool.BYTE_BLOCK_SIZE;
+
+        buffer = pool.buffers[bufferUpto];
+        upto = nextIndex & ByteBlockPool.BYTE_BLOCK_MASK;
+
+        if (nextIndex + newSize >= endIndex)
+        {
+            // We are advancing to the final slice
+            assert endIndex - nextIndex > 0;
+            limit = endIndex - bufferOffset;
+        }
+        else
+        {
+            // This is not the final slice (subtract 4 for the
+            // forwarding address at the end of this new slice)
+            limit = upto + newSize - 4;
+        }
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len)
+    {
+        while (len > 0)
+        {
+            final int numLeft = limit - upto;
+            if (numLeft < len)
+            {
+                // Read entire slice
+                System.arraycopy(buffer, upto, b, offset, numLeft);
+                offset += numLeft;
+                len -= numLeft;
+                nextSlice();
+            }
+            else
+            {
+                // This slice is the last one
+                System.arraycopy(buffer, upto, b, offset, len);
+                upto += len;
+                break;
+            }
+        }
+    }
+
+    @Override
+    public void skipBytes(long l) throws IOException
+    {
+        while (l > 0)
+        {
+            final int numLeft = limit - upto;
+            if (numLeft < l)
+            {
+                // Skip entire slice
+                l -= numLeft;
+                nextSlice();
+            }
+            else
+            {
+                // This slice is the last one
+                upto += l;
+                break;
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/EmptyIndex.java b/src/java/org/apache/cassandra/index/sai/disk/EmptyIndex.java
new file mode 100644
index 000000000000..501d2c9833ed
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/EmptyIndex.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.v1.Segment;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.CloseableIterator;
+
+public class EmptyIndex implements SearchableIndex
+{
+    @Override
+    public long indexFileCacheSize()
+    {
+        return 0;
+    }
+
+    @Override
+    public long getRowCount()
+    {
+        return 0;
+    }
+
+    @Override
+    public long minSSTableRowId()
+    {
+        return -1;
+    }
+
+    @Override
+    public long maxSSTableRowId()
+    {
+        return -1;
+    }
+
+    @Override
+    public ByteBuffer minTerm()
+    {
+        return null;
+    }
+
+    @Override
+    public ByteBuffer maxTerm()
+    {
+        return null;
+    }
+
+    @Override
+    public DecoratedKey minKey()
+    {
+        return null;
+    }
+
+    @Override
+    public DecoratedKey maxKey()
+    {
+        return null;
+    }
+
+    @Override
+    public KeyRangeIterator search(Expression expression,
+                                   AbstractBounds<PartitionPosition> keyRange,
+                                   QueryContext context,
+                                   boolean defer,
+                                   int limit) throws IOException
+    {
+        return KeyRangeIterator.empty();
+    }
+
+    @Override
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderBy(Orderer orderer,
+                                                                  Expression slice,
+                                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                                  QueryContext context,
+                                                                  int limit,
+                                                                  long totalRows) throws IOException
+    {
+        return List.of();
+    }
+
+    @Override
+    public List<Segment> getSegments()
+    {
+        return List.of();
+    }
+
+    @Override
+    public void populateSystemView(SimpleDataSet dataSet, SSTableReader sstable)
+    {
+        // Empty indexes are not visible in the system view,
+        // as they don't really exist on disk (are not built).
+        // This is to keep backwards compatibility – before introducing
+        // this class, empty indexes weren't even included in the SAI View,
+        // so they did not appear in the system view as well.
+    }
+
+    @Override
+    public long estimateMatchingRowsCount(Expression predicate, AbstractBounds<PartitionPosition> keyRange)
+    {
+        return 0;
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        // EmptyIndex does not hold any resources
+    }
+
+    @Override
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderResultsBy(QueryContext context, List<PrimaryKey> keys, Orderer orderer, int limit, long totalRows) throws IOException
+    {
+        return List.of();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java b/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java
deleted file mode 100644
index 83887698fcf7..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.exceptions.QueryCancelledException;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.Throwables;
-
-public class IndexSearchResultIterator extends KeyRangeIterator
-{
-    private static final Logger logger = LoggerFactory.getLogger(IndexSearchResultIterator.class);
-
-    private final KeyRangeIterator union;
-
-    private IndexSearchResultIterator(KeyRangeIterator union, Runnable onClose)
-    {
-        super(union.getMinimum(), union.getMaximum(), union.getMaxKeys(), onClose);
-        this.union = union;
-    }
-
-    /**
-     * Builds a new {@link IndexSearchResultIterator} that wraps a {@link KeyRangeUnionIterator} over the
-     * results of searching the {@link org.apache.cassandra.index.sai.memory.MemtableIndex} and the {@link SSTableIndex}es.
-     */
-    public static IndexSearchResultIterator build(Expression expression,
-                                                  Collection<SSTableIndex> sstableIndexes,
-                                                  AbstractBounds<PartitionPosition> keyRange,
-                                                  QueryContext queryContext,
-                                                  boolean includeMemtables,
-                                                  Runnable onClose)
-    {
-        List<KeyRangeIterator> subIterators = new ArrayList<>(sstableIndexes.size() + (includeMemtables ? 1 : 0));
-
-        if (includeMemtables)
-        {
-            KeyRangeIterator memtableIterator = expression.getIndex().memtableIndexManager().searchMemtableIndexes(queryContext, expression, keyRange);
-            if (memtableIterator != null)
-                subIterators.add(memtableIterator);
-        }
-
-        for (SSTableIndex sstableIndex : sstableIndexes)
-        {
-            try
-            {
-                queryContext.checkpoint();
-                queryContext.sstablesHit++;
-
-                if (sstableIndex.isReleased())
-                    throw new IllegalStateException(sstableIndex.getIndexIdentifier().logMessage("Index was released from the view during the query"));
-
-                List<KeyRangeIterator> indexIterators = sstableIndex.search(expression, keyRange, queryContext);
-
-                if (!indexIterators.isEmpty())
-                    subIterators.addAll(indexIterators);
-            }
-            catch (Throwable e)
-            {
-                if (!(e instanceof QueryCancelledException))
-                    logger.debug(sstableIndex.getIndexIdentifier().logMessage(String.format("Failed search an index %s, aborting query.", sstableIndex.getSSTable())), e);
-
-                throw Throwables.cleaned(e);
-            }
-        }
-
-        KeyRangeIterator union = KeyRangeUnionIterator.build(subIterators, () -> {});
-        return new IndexSearchResultIterator(union, onClose);
-    }
-
-    public static IndexSearchResultIterator build(List<KeyRangeIterator> sstableIntersections,
-                                                  KeyRangeIterator memtableResults,
-                                                  Set<SSTableIndex> referencedIndexes,
-                                                  QueryContext queryContext,
-                                                  Runnable onClose)
-    {
-        queryContext.sstablesHit += referencedIndexes
-                                    .stream()
-                                    .map(SSTableIndex::getSSTable).collect(Collectors.toSet()).size();
-        queryContext.checkpoint();
-        KeyRangeIterator union = KeyRangeUnionIterator.builder(sstableIntersections.size() + 1, () -> {})
-                                                      .add(sstableIntersections)
-                                                      .add(memtableResults)
-                                                      .build();
-        return new IndexSearchResultIterator(union, onClose);
-    }
-
-    protected PrimaryKey computeNext()
-    {
-        return union.hasNext() ? union.next() : endOfData();
-    }
-
-    protected void performSkipTo(PrimaryKey nextKey)
-    {
-        union.skipTo(nextKey);
-    }
-
-    @Override
-    public void close()
-    {
-        super.close();
-        FileUtils.closeQuietly(union);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/IndexSearcherContext.java b/src/java/org/apache/cassandra/index/sai/disk/IndexSearcherContext.java
new file mode 100644
index 000000000000..0f26d1fd9ff6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/IndexSearcherContext.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+public class IndexSearcherContext
+{
+    final QueryContext context;
+    final PostingList postingList;
+
+    final PrimaryKey minimumKey;
+    final PrimaryKey maximumKey;
+    final long minSSTableRowId;
+    final long maxSSTableRowId;
+    final long segmentRowIdOffset;
+    final long maxPartitionOffset;
+
+    public IndexSearcherContext(PrimaryKey minimumKey,
+                                PrimaryKey maximumKey,
+                                long minSSTableRowId,
+                                long maxSSTableRowId,
+                                long segmentRowIdOffset,
+                                QueryContext context,
+                                PostingList postingList) throws IOException
+    {
+        this.context = context;
+        this.postingList = postingList;
+
+        this.segmentRowIdOffset = segmentRowIdOffset;
+
+        this.minimumKey = minimumKey;
+
+        // use segment's metadata for the range iterator, may not be accurate, but should not matter to performance.
+        this.maximumKey = maximumKey;
+
+        this.minSSTableRowId = minSSTableRowId;
+        this.maxSSTableRowId = maxSSTableRowId;
+        this.maxPartitionOffset = Long.MAX_VALUE;
+    }
+
+    public long getSegmentRowIdOffset()
+    {
+        return segmentRowIdOffset;
+    }
+
+    int count()
+    {
+        return postingList.size();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/MemtableTermsIterator.java b/src/java/org/apache/cassandra/index/sai/disk/MemtableTermsIterator.java
new file mode 100644
index 000000000000..75f37519a966
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/MemtableTermsIterator.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.index.sai.memory.RowMapping;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Iterator over a token range bounded segment of a Memtable index. Used to flush Memtable index segments to disk.
+ */
+public class MemtableTermsIterator implements TermsIterator
+{
+    private final ByteBuffer minTerm;
+    private final ByteBuffer maxTerm;
+    private final Iterator<Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>>> iterator;
+
+    private Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>> current;
+
+    private int maxSSTableRowId = -1;
+    private int minSSTableRowId = Integer.MAX_VALUE;
+
+    public MemtableTermsIterator(ByteBuffer minTerm,
+                                 ByteBuffer maxTerm,
+                                 Iterator<Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>>> iterator)
+    {
+        Preconditions.checkArgument(iterator != null);
+        this.minTerm = minTerm;
+        this.maxTerm = maxTerm;
+        this.iterator = iterator;
+    }
+
+    @Override
+    public ByteBuffer getMinTerm()
+    {
+        return minTerm;
+    }
+
+    @Override
+    public ByteBuffer getMaxTerm()
+    {
+        return maxTerm;
+    }
+
+    @Override
+    public void close() {}
+
+    @Override
+    public PostingList postings()
+    {
+        var list = current.right;
+
+        assert list.size() > 0;
+
+        final int minSegmentRowID = list.get(0).rowId;
+        final int maxSegmentRowID = list.get(list.size() - 1).rowId;
+
+        // Because we are working with postings from the memtable, there is only one segment, so segment row ids
+        // and sstable row ids are the same.
+        minSSTableRowId = Math.min(minSSTableRowId, minSegmentRowID);
+        maxSSTableRowId = Math.max(maxSSTableRowId, maxSegmentRowID);
+
+        var it = list.iterator();
+
+        return new PostingList()
+        {
+            int frequency;
+
+            @Override
+            public int nextPosting()
+            {
+                if (!it.hasNext())
+                {
+                    return END_OF_STREAM;
+                }
+
+                var rowIdWithFrequency = it.next();
+                frequency = rowIdWithFrequency.frequency;
+                return rowIdWithFrequency.rowId;
+            }
+
+            @Override
+            public int size()
+            {
+                return list.size();
+            }
+
+            @Override
+            public int frequency()
+            {
+                return frequency;
+            }
+
+            @Override
+            public int advance(int targetRowID)
+            {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        return iterator.hasNext();
+    }
+
+    @Override
+    public ByteComparable next()
+    {
+        current = iterator.next();
+        return current.left;
+    }
+
+    public long getMaxSSTableRowId()
+    {
+        return maxSSTableRowId;
+    }
+
+    public long getMinSSTableRowId()
+    {
+        return minSSTableRowId;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/ModernResettableByteBuffersIndexOutput.java b/src/java/org/apache/cassandra/index/sai/disk/ModernResettableByteBuffersIndexOutput.java
new file mode 100644
index 000000000000..0bef82c747cb
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/ModernResettableByteBuffersIndexOutput.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.oldlucene.ResettableByteBuffersIndexOutput;
+import org.apache.lucene.store.ByteBuffersDataInput;
+import org.apache.lucene.store.ByteBuffersDataOutput;
+import org.apache.lucene.store.ByteBuffersIndexInput;
+import org.apache.lucene.store.ByteBuffersIndexOutput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IndexInput;
+
+/***
+ * A wrapper around {@link ByteBuffersIndexOutput} that adds several methods that interact
+ * with the underlying delegate. This is "modern" in the sense that it uses the current Lucene
+ * dependency for its implementation of I/O. In particular, this means it cannot be used to write
+ * indexes/data compatible with the readers in older Lucene versions.
+ */
+public class ModernResettableByteBuffersIndexOutput extends ResettableByteBuffersIndexOutput
+{
+    private final ByteBuffersIndexOutput bbio;
+    private final ByteBuffersDataOutput delegate;
+
+    public ModernResettableByteBuffersIndexOutput(int expectedSize, String name)
+    {
+        super("", name, ByteOrder.LITTLE_ENDIAN);
+        delegate = new ByteBuffersDataOutput(expectedSize);
+        bbio = new ByteBuffersIndexOutput(delegate, "", name + "-bb");
+    }
+
+    public ByteBuffersDataInput toDataInput()
+    {
+        return delegate.toDataInput();
+    }
+
+    public IndexInput toIndexInput()
+    {
+        return new ByteBuffersIndexInput(toDataInput(), "");
+    }
+
+    public void copyTo(IndexOutput out) throws IOException
+    {
+        delegate.copyTo(out);
+    }
+
+    public int intSize() {
+        return Math.toIntExact(bbio.getFilePointer());
+    }
+
+    public byte[] toArrayCopy() {
+        return delegate.toArrayCopy();
+    }
+
+    public void reset()
+    {
+        delegate.reset();
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Resettable" + bbio.toString();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        bbio.close();
+    }
+
+    @Override
+    public long getFilePointer()
+    {
+        return bbio.getFilePointer();
+    }
+
+    @Override
+    public long getChecksum() throws IOException
+    {
+        return bbio.getChecksum();
+    }
+
+    @Override
+    public void writeByte(byte b) throws IOException
+    {
+        bbio.writeByte(b);
+    }
+
+    @Override
+    public void writeBytes(byte[] b, int offset, int length) throws IOException
+    {
+        bbio.writeBytes(b, offset, length);
+    }
+
+    @Override
+    public void writeBytes(byte[] b, int length) throws IOException
+    {
+        bbio.writeBytes(b, length);
+    }
+
+    @Override
+    public void writeInt(int i) throws IOException
+    {
+        bbio.writeInt(i);
+    }
+
+    @Override
+    public void writeShort(short i) throws IOException
+    {
+        bbio.writeShort(i);
+    }
+
+    @Override
+    public void writeLong(long i) throws IOException
+    {
+        bbio.writeLong(i);
+    }
+
+    @Override
+    public void writeString(String s) throws IOException
+    {
+        bbio.writeString(s);
+    }
+
+    @Override
+    public void copyBytes(DataInput input, long numBytes) throws IOException
+    {
+        bbio.copyBytes(input, numBytes);
+    }
+
+    @Override
+    public void writeMapOfStrings(Map<String, String> map) throws IOException
+    {
+        bbio.writeMapOfStrings(map);
+    }
+
+    @Override
+    public void writeSetOfStrings(Set<String> set) throws IOException
+    {
+        bbio.writeSetOfStrings(set);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/PerIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/PerIndexWriter.java
new file mode 100644
index 000000000000..13840fa0d658
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/PerIndexWriter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+
+import com.google.common.base.Stopwatch;
+
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+/**
+ * Creates an on-disk index for a given index.
+ */
+public interface PerIndexWriter
+{
+    /**
+     * The index components written on disk by this disk.
+     */
+    IndexComponents.ForWrite writtenComponents();
+
+    /**
+     * Adds a row to this index.
+     */
+    void addRow(PrimaryKey key, Row row, long sstableRowId) throws IOException;
+
+    /**
+     * Builds on-disk index data structures from accumulated data, moves them all to the filesystem, and fsync created files.
+     */
+    void complete(Stopwatch stopwatch) throws IOException;
+
+    /**
+     * Aborts accumulating data. Allows to clean up resources on error.
+     * 
+     * Note: Implementations should be idempotent, i.e. safe to call multiple times without producing undesirable side-effects.
+     */
+    void abort(Throwable cause);
+
+    IndexContext indexContext();
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/PerSSTableWriter.java b/src/java/org/apache/cassandra/index/sai/disk/PerSSTableWriter.java
new file mode 100644
index 000000000000..af323b91ecac
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/PerSSTableWriter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+
+import com.google.common.base.Stopwatch;
+
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+/**
+ * Writes all SSTable-attached index token and offset structures.
+ */
+public interface PerSSTableWriter
+{
+    public static final PerSSTableWriter NONE = (key) -> {};
+
+    default void startPartition(long position) throws IOException
+    {}
+
+    void nextRow(PrimaryKey primaryKey) throws IOException;
+
+    default void complete(Stopwatch stopwatch) throws IOException
+    {}
+
+    default void abort(Throwable accumulator)
+    {}
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/PostingList.java b/src/java/org/apache/cassandra/index/sai/disk/PostingList.java
new file mode 100644
index 000000000000..4959c6f0be6a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/PostingList.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.Closeable;
+import java.io.IOException;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * Interface for advancing on and consuming a posting list.
+ */
+@NotThreadSafe
+public interface PostingList extends Closeable
+{
+    PostingList EMPTY = new EmptyPostingList();
+
+    int OFFSET_NOT_FOUND = -1;
+    int END_OF_STREAM = Integer.MAX_VALUE;
+
+    @Override
+    default void close() throws IOException {}
+
+    /**
+     * Retrieves the next segment row ID, not including row IDs that have been returned by {@link #advance(int)}.
+     *
+     * @return next segment row ID
+     */
+    int nextPosting() throws IOException;
+
+    /**
+     * @return the number of occurrences of the term in the current row (the one most recently returned by nextPosting).
+     */
+    default int frequency()
+    {
+        return 1;
+    }
+
+    int size();
+
+    /**
+     * @return {@code true} if this posting list contains no postings
+     */
+    default boolean isEmpty()
+    {
+        return size() == 0;
+    }
+
+    /**
+     * Advances to the first row ID beyond the current that is greater than or equal to the
+     * target, and returns that row ID. Exhausts the iterator and returns {@link #END_OF_STREAM} if
+     * the target is greater than the highest row ID.
+     *
+     * Note: Callers must use the return value of this method before calling {@link #nextPosting()}, as calling
+     * that method will return the next posting, not the one to which we have just advanced.
+     *
+     * @param targetRowID target row ID to advance to
+     *
+     * @return first segment row ID which is >= the target row ID or {@link PostingList#END_OF_STREAM} if one does not exist
+     */
+    int advance(int targetRowID) throws IOException;
+
+    class EmptyPostingList implements PostingList
+    {
+        @Override
+        public int nextPosting() throws IOException
+        {
+            return END_OF_STREAM;
+        }
+
+        @Override
+        public int size()
+        {
+            return 0;
+        }
+
+        @Override
+        public int advance(int targetRowID) throws IOException
+        {
+            return END_OF_STREAM;
+        }
+    }
+
+    /**
+     * Returns a wrapper for this posting list that runs the specified {@link Closeable} when this posting list is closed,
+     * unless this posting list is empty, in which case the specified {@link Closeable} will be run immediately.
+     *
+     * @param onClose what to do on close
+     * @return a posting list that makes sure that {@code onClose} is run by the time it is closed.
+     */
+    default PostingList onClose(Closeable onClose) throws IOException
+    {
+        if (isEmpty())
+        {
+            onClose.close();
+            return EMPTY;
+        }
+
+        return new PostingListWithOnClose(this, onClose);
+    }
+
+    class PostingListWithOnClose implements PostingList
+    {
+        private final PostingList delegate;
+        private final Closeable onClose;
+
+        public PostingListWithOnClose(PostingList delegate, Closeable onClose)
+        {
+            this.delegate = delegate;
+            this.onClose = onClose;
+        }
+
+        @Override
+        public int size()
+        {
+            return delegate.size();
+        }
+
+        @Override
+        public int advance(int targetRowID) throws IOException
+        {
+            return delegate.advance(targetRowID);
+        }
+
+        @Override
+        public int nextPosting() throws IOException
+        {
+            return delegate.nextPosting();
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            delegate.close();
+            onClose.close();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/PostingListKeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/disk/PostingListKeyRangeIterator.java
new file mode 100644
index 000000000000..d500225a6fd7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/PostingListKeyRangeIterator.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.base.Stopwatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.AbortedOperationException;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Throwables;
+
+/**
+ * A range iterator based on {@link PostingList}.
+ *
+ * <ol>
+ *   <li> fetch next unique segment row id from posting list or skip to specific segment row id if {@link #skipTo(PrimaryKey)} is called </li>
+ *   <li> add segmentRowIdOffset to obtain the sstable row id </li>
+ *   <li> produce a {@link PrimaryKey} from {@link PrimaryKeyMap#primaryKeyFromRowId(long)} which is used
+ *       to avoid fetching duplicated keys due to partition-level indexing on wide partition schema.
+ *       <br/>
+ *       Note: in order to reduce disk access in multi-index query, partition keys will only be fetched for intersected tokens
+ *       in {@link org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher}.
+ *  </li>
+ * </ol>
+ *
+ */
+
+@NotThreadSafe
+public class PostingListKeyRangeIterator extends KeyRangeIterator
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private final Stopwatch timeToExhaust = Stopwatch.createStarted();
+    private final QueryContext queryContext;
+
+    private final PostingList postingList;
+    private final IndexContext indexContext;
+    private final PrimaryKeyMap primaryKeyMap;
+    private final IndexSearcherContext searcherContext;
+
+    private final AtomicBoolean isClosed = new AtomicBoolean(false);
+
+    private boolean needsSkipping = false;
+    private PrimaryKey skipToToken = null;
+    private long lastSegmentRowId = -1;
+
+    /**
+     * Create a direct PostingListKeyRangeIterator where the underlying PostingList is materialised
+     * immediately so the posting list size can be used.
+     */
+    public PostingListKeyRangeIterator(IndexContext indexContext,
+                                       PrimaryKeyMap primaryKeyMap,
+                                       IndexSearcherContext searcherContext)
+    {
+        super(searcherContext.minimumKey, searcherContext.maximumKey, searcherContext.count());
+
+        this.indexContext = indexContext;
+        this.primaryKeyMap = primaryKeyMap;
+        this.postingList = searcherContext.postingList;
+        this.searcherContext = searcherContext;
+        this.queryContext = this.searcherContext.context;
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        // If skipToToken is equal to nextKey, we take the nextKey because in practice, it is greater than or equal
+        // to the skipToToken. This is because token only PKs are considered equal to all PKs with the same token,
+        // and for a range query, we first skip on the token-only PK.
+        if (skipToToken != null && skipToToken.compareTo(nextKey) > 0)
+            return;
+
+        skipToToken = nextKey;
+        needsSkipping = true;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        try
+        {
+            queryContext.checkpoint();
+
+            // just end the iterator if we don't have a postingList or current segment is skipped
+            if (exhausted())
+                return endOfData();
+
+            long rowId = getNextRowId();
+            if (rowId == PostingList.END_OF_STREAM)
+                return endOfData();
+
+            var primaryKey = primaryKeyMap.primaryKeyFromRowId(rowId);
+            return new PrimaryKeyWithSource(primaryKey, primaryKeyMap.getSSTableId(), rowId);
+        }
+        catch (Throwable t)
+        {
+            if (!(t instanceof AbortedOperationException))
+                logger.error(indexContext.logMessage("Unable to provide next token!"), t);
+
+            throw Throwables.cleaned(t);
+        }
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        if (isClosed.compareAndSet(false, true))
+        {
+            if (logger.isTraceEnabled())
+            {
+                // timeToExhaust.stop() throws on already stopped stopwatch
+                final long closedInMills = timeToExhaust.stop().elapsed(TimeUnit.MILLISECONDS);
+                logger.trace(indexContext.logMessage("PostinListRangeIterator exhausted after {} ms"), closedInMills);
+            }
+
+            FileUtils.closeQuietly(postingList, primaryKeyMap);
+        }
+        else {
+            logger.warn("PostingListKeyRangeIterator is already closed",
+                        new IllegalStateException("PostingListKeyRangeIterator is already closed"));
+        }
+
+    }
+
+    private boolean exhausted()
+    {
+        return needsSkipping && skipToToken.compareTo(getMaximum()) > 0;
+    }
+
+    /**
+     * reads the next sstable row ID from the underlying posting list, potentially skipping to get there.
+     */
+    private long getNextRowId() throws IOException
+    {
+        long segmentRowId;
+        if (needsSkipping)
+        {
+            long targetSstableRowId;
+            if (skipToToken instanceof PrimaryKeyWithSource
+                && ((PrimaryKeyWithSource) skipToToken).getSourceSstableId().equals(primaryKeyMap.getSSTableId()))
+            {
+                targetSstableRowId = ((PrimaryKeyWithSource) skipToToken).getSourceRowId();
+            }
+            else
+            {
+                targetSstableRowId = primaryKeyMap.ceiling(skipToToken);
+                // skipToToken is larger than max token in token file
+                if (targetSstableRowId < 0)
+                {
+                    return PostingList.END_OF_STREAM;
+                }
+            }
+            int targetSegmentRowId = Math.toIntExact(targetSstableRowId - searcherContext.getSegmentRowIdOffset());
+            segmentRowId = postingList.advance(targetSegmentRowId);
+            needsSkipping = false;
+        }
+        else
+        {
+            do
+            {
+                segmentRowId = postingList.nextPosting();
+                // Do not produce a duplicate segment row id.
+            } while (segmentRowId == lastSegmentRowId && segmentRowId != PostingList.END_OF_STREAM);
+        }
+        lastSegmentRowId = segmentRowId;
+        return segmentRowId != PostingList.END_OF_STREAM
+               ? segmentRowId + searcherContext.getSegmentRowIdOffset()
+               : PostingList.END_OF_STREAM;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMap.java b/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMap.java
index 88e72efab7fa..2b56c8dc3d6a 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMap.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMap.java
@@ -22,13 +22,12 @@
 import java.io.IOException;
 
 import javax.annotation.concurrent.NotThreadSafe;
-import javax.annotation.concurrent.ThreadSafe;
 
-import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.sstable.SSTableId;
 
 /**
- * A bidirectional map of {@link PrimaryKey} to row ID. Implementations of this interface
+ * A bidirectional map of {@link PrimaryKey} to row Id. Implementations of this interface
  * are not expected to be threadsafe.
  */
 @NotThreadSafe
@@ -38,59 +37,91 @@ public interface PrimaryKeyMap extends Closeable
      * A factory for creating {@link PrimaryKeyMap} instances. Implementations of this
      * interface are expected to be threadsafe.
      */
-    @ThreadSafe
-    interface Factory extends Closeable
+    public interface Factory extends Closeable
     {
         /**
          * Creates a new {@link PrimaryKeyMap} instance
          *
          * @return a {@link PrimaryKeyMap}
-         * @throws IOException if the {@link PrimaryKeyMap} couldn't be created
+         * @throws IOException
          */
-        PrimaryKeyMap newPerSSTablePrimaryKeyMap() throws IOException;
+        PrimaryKeyMap newPerSSTablePrimaryKeyMap();
+
+        /**
+         * Returns the number of primary keys in the map. This is part of the factory because
+         * it can be retrieved without opening the map.
+         * @return the number of primary keys in the map
+         */
+        default long count()
+        {
+            try (PrimaryKeyMap map = newPerSSTablePrimaryKeyMap())
+            {
+                return map.count();
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
 
         @Override
-        default void close()
+        default void close() throws IOException
         {
         }
     }
 
     /**
-     * Returns a {@link PrimaryKey} for a row ID
+     * Returns the {@link SSTableId} associated with this {@link PrimaryKeyMap}
+     * @return an {@link SSTableId}
+     */
+    SSTableId<?> getSSTableId();
+
+    /**
+     * Returns a {@link PrimaryKey} for a row Id
      *
-     * @param sstableRowId the row ID to lookup
-     * @return the {@link PrimaryKey} associated with the row ID
+     * @param sstableRowId the row Id to lookup
+     * @return the {@link PrimaryKey} associated with the row Id
      */
     PrimaryKey primaryKeyFromRowId(long sstableRowId);
 
     /**
-     * Returns a row ID for a {@link PrimaryKey}
+     * Returns a row Id for a {@link PrimaryKey}. If there is no such term, returns the `-(next row id) - 1` where
+     * `next row id` is the row id of the next greatest {@link PrimaryKey} in the map.
      *
      * @param key the {@link PrimaryKey} to lookup
-     * @return the row ID associated with the {@link PrimaryKey}
+     * @return the row Id associated with the {@link PrimaryKey}
      */
-    long rowIdFromPrimaryKey(PrimaryKey key);
+    long exactRowIdOrInvertedCeiling(PrimaryKey key);
 
     /**
-     * Returns the first row ID of the nearest {@link Token} greater than or equal to the given {@link Token},
-     * or a negative value if not found
+     * Returns the sstable row id associated with the least {@link PrimaryKey} greater than or equal to the given
+     * {@link PrimaryKey}. If the {@link PrimaryKey} is a prefix of multiple {@link PrimaryKey}s in the map, e.g. it is
+     * just a token or a token and a partition key, the row id associated with the least {@link PrimaryKey} will be
+     * returned. If there is no {@link PrimaryKey} in the map that meets this definition, returns a negative value.
      *
-     * @param token the {@link Token} to lookup
-     * @return the ceiling row ID associated with the {@link Token} or a negative value
+     * @param key the {@link PrimaryKey} to lookup
+     * @return an sstable row id or a negative value if no row is found
      */
-    long ceiling(Token token);
+    long ceiling(PrimaryKey key);
 
     /**
-     * Returns the last row ID of the nearest {@link Token} less than or equal to the given {@link Token},
-     * or a negative value if the {@link Token} is at its minimum value
+     * Returns the sstable row id associated with the greatest {@link PrimaryKey} less than or equal to the given
+     * {@link PrimaryKey}. If the {@link PrimaryKey} is a prefix of multiple {@link PrimaryKey}s in the map, e.g. it is
+     * just a token or a token and a partition key, the row id associated with the greatest {@link PrimaryKey} will be
+     * returned. If there is no {@link PrimaryKey} in the map that meets this definition, returns a negative value.
      *
-     * @param token the {@link Token} to lookup
-     * @return the floor row ID associated with the {@link Token}
+     * @param key the {@link PrimaryKey} to lookup
+     * @return an sstable row id or a negative value if no row is found
+     */
+    long floor(PrimaryKey key);
+
+    /**
+     * Returns the number of primary keys in the map
      */
-    long floor(Token token);
+    long count();
 
     @Override
-    default void close()
+    default void close() throws IOException
     {
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMapIterator.java b/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMapIterator.java
new file mode 100644
index 000000000000..b8f7e87d58d3
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMapIterator.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Iterates keys in the {@link PrimaryKeyMap} of a SSTable.
+ * Iterating keys in the primary key map is faster than reading them from the SSTable data component
+ * because we don't deserialize all the other columns except the primary key columns.
+ * The primary key map components are also likely much smaller than the whole SSTable data component.
+ * <p>
+ * The keys are returned in token-clustering order.
+ */
+public final class PrimaryKeyMapIterator extends KeyRangeIterator
+{
+    // KeyFilter controls which keys we want to return from the iterator.
+    // This is a hack to make this iterator work correctly on schemas with static columns.
+    // If the table has static columns, the primary key map component may contain both keys with clustering
+    // and with no clustering. The keys of regular rows will likely have clustering and the keys associated with
+    // updates of the static columns will have no clustering. Hence, depending on the type of the queried column,
+    // we must return only all keys with clustering or only all keys with no clustering, but not mixed, or we may run
+    // into duplicate row issues. We also shouldn't return keys without clustering for regular rows that expect
+    // clustering information - as that would negate the row-awareness advantage.
+    private enum KeyFilter
+    {
+        ALL,   // return all keys, fast, but safe only if we know there are no mixed keys with and without clustering
+        KEYS_WITH_CLUSTERING     // return keys with clustering
+    }
+
+    private final PrimaryKeyMap keys;
+    private final KeyFilter filter;
+    private long currentRowId;
+
+
+    private PrimaryKeyMapIterator(PrimaryKeyMap keys, PrimaryKey min, PrimaryKey max, long startRowId, KeyFilter filter)
+    {
+        super(min, max, keys.count());
+        this.keys = keys;
+        this.filter = filter;
+        this.currentRowId = startRowId;
+    }
+
+    public static KeyRangeIterator create(SSTableContext ctx, AbstractBounds<PartitionPosition> keyRange) throws IOException
+    {
+        KeyFilter filter;
+        TableMetadata metadata = ctx.sstable().metadata();
+        // if not row-aware, we don't have clustering
+        var perSSTableComponents = ctx.usedPerSSTableComponents();
+        if (perSSTableComponents.onDiskFormat().indexFeatureSet().isRowAware() && metadata.hasStaticColumns())
+            filter = KeyFilter.KEYS_WITH_CLUSTERING;
+        else // the table doesn't consist anything we want to filter out, so let's use the cheap option
+            filter = KeyFilter.ALL;
+
+        if (perSSTableComponents.isEmpty())
+            return KeyRangeIterator.empty();
+
+        PrimaryKeyMap keys = ctx.primaryKeyMapFactory.newPerSSTablePrimaryKeyMap();
+        long count = keys.count();
+        if (keys.count() == 0)
+        {
+            keys.close();
+            return KeyRangeIterator.empty();
+        }
+
+        PrimaryKey.Factory pkFactory = ctx.primaryKeyFactory();
+        Token minToken = keyRange.left.getToken();
+        PrimaryKey minKeyBound = pkFactory.createTokenOnly(minToken);
+        PrimaryKey sstableMinKey = keys.primaryKeyFromRowId(0);
+        PrimaryKey sstableMaxKey = keys.primaryKeyFromRowId(count - 1);
+        PrimaryKey minKey = (minKeyBound.compareTo(sstableMinKey) > 0)
+                            ? minKeyBound
+                            : sstableMinKey;
+        long startRowId = minToken.isMinimum() ? 0 : keys.ceiling(minKey);
+        return new PrimaryKeyMapIterator(keys, sstableMinKey, sstableMaxKey, startRowId, filter);
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        this.currentRowId = keys.ceiling(nextKey);
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        while (currentRowId >= 0 && currentRowId < keys.count())
+        {
+            PrimaryKey key = keys.primaryKeyFromRowId(currentRowId++);
+            if (filter == KeyFilter.KEYS_WITH_CLUSTERING && key.hasEmptyClustering())
+                continue;
+            return key;
+        }
+        return endOfData();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        keys.close();
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyWithSource.java b/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyWithSource.java
new file mode 100644
index 000000000000..d303bd4d27c7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyWithSource.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+public class PrimaryKeyWithSource implements PrimaryKey
+{
+    private final PrimaryKey primaryKey;
+    private final SSTableId<?> sourceSstableId;
+    private final long sourceRowId;
+
+    public PrimaryKeyWithSource(PrimaryKey primaryKey, SSTableId<?> sstableId, long sstableRowId)
+    {
+        assert primaryKey != null : "Cannot construct a PrimaryKeyWithSource with a null primaryKey";
+        this.primaryKey = primaryKey;
+        this.sourceSstableId = sstableId;
+        this.sourceRowId = sstableRowId;
+    }
+
+    public long getSourceRowId()
+    {
+        return sourceRowId;
+    }
+
+    public SSTableId<?> getSourceSstableId()
+    {
+        return sourceSstableId;
+    }
+
+    @Override
+    public Token token()
+    {
+        return primaryKey.token();
+    }
+
+    @Override
+    public DecoratedKey partitionKey()
+    {
+        return primaryKey.partitionKey();
+    }
+
+    @Override
+    public Clustering clustering()
+    {
+        return primaryKey.clustering();
+    }
+
+    @Override
+    public PrimaryKey loadDeferred()
+    {
+        return primaryKey.loadDeferred();
+    }
+
+    @Override
+    public ByteSource asComparableBytes(ByteComparable.Version version)
+    {
+        return primaryKey.asComparableBytes(version);
+    }
+
+    @Override
+    public ByteSource asComparableBytesMinPrefix(ByteComparable.Version version)
+    {
+        return primaryKey.asComparableBytesMinPrefix(version);
+    }
+
+    @Override
+    public ByteSource asComparableBytesMaxPrefix(ByteComparable.Version version)
+    {
+        return primaryKey.asComparableBytesMaxPrefix(version);
+    }
+
+    @Override
+    public int compareTo(PrimaryKey o)
+    {
+        if (o instanceof PrimaryKeyWithSource)
+        {
+            var other = (PrimaryKeyWithSource) o;
+            if (sourceSstableId.equals(other.sourceSstableId))
+                return Long.compare(sourceRowId, other.sourceRowId);
+        }
+        return primaryKey.compareTo(o);
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (o instanceof PrimaryKeyWithSource)
+        {
+            var other = (PrimaryKeyWithSource) o;
+            if (sourceSstableId.equals(other.sourceSstableId))
+                return sourceRowId == other.sourceRowId;
+        }
+        return primaryKey.equals(o);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return primaryKey.hashCode();
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("%s (source sstable: %s, %s)", primaryKey, sourceSstableId, sourceRowId);
+    }
+
+    @Override
+    public long ramBytesUsed()
+    {
+        // Object header + 3 references (primaryKey, sourceSstableId) + long value
+        return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER +
+               2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF +
+               Long.BYTES +
+               primaryKey.ramBytesUsed();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/QueryEventListeners.java b/src/java/org/apache/cassandra/index/sai/disk/QueryEventListeners.java
new file mode 100644
index 000000000000..ba6a80265c66
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/QueryEventListeners.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+
+public class QueryEventListeners
+{
+    public static final QueryEventListener NO_OP = new BaseQueryEventListener();
+
+    public static final QueryEventListener.BKDIndexEventListener NO_OP_BKD_LISTENER = NO_OP.bkdIndexEventListener();
+
+    public static final QueryEventListener.TrieIndexEventListener NO_OP_TRIE_LISTENER = NO_OP.trieIndexEventListener();
+
+    public static final QueryEventListener.PostingListEventListener NO_OP_POSTINGS_LISTENER = new NoOpPostingListEventListener();
+
+    private static class BaseQueryEventListener implements QueryEventListener
+    {
+        @Override
+        public BKDIndexEventListener bkdIndexEventListener()
+        {
+            return NoOpBKDIndexEventListener.INSTANCE;
+        }
+
+        @Override
+        public TrieIndexEventListener trieIndexEventListener()
+        {
+            return NoOpTrieIndexEventListener.INSTANCE;
+        }
+
+        private enum NoOpTrieIndexEventListener implements TrieIndexEventListener
+        {
+            INSTANCE;
+
+            @Override
+            public void onSegmentHit() { }
+
+            @Override
+            public void onTraversalComplete(long traversalTotalTime, TimeUnit unit) { }
+
+            @Override
+            public PostingListEventListener postingListEventListener()
+            {
+                return NO_OP_POSTINGS_LISTENER;
+            }
+        }
+
+        private enum NoOpBKDIndexEventListener implements BKDIndexEventListener
+        {
+            INSTANCE;
+
+            @Override
+            public void onIntersectionComplete(long intersectionTotalTime, TimeUnit unit) { }
+
+            @Override
+            public void onIntersectionEarlyExit() { }
+
+            @Override
+            public void postingListsHit(int count) { }
+
+            @Override
+            public void onSegmentHit() { }
+
+            @Override
+            public PostingListEventListener postingListEventListener()
+            {
+                return NO_OP_POSTINGS_LISTENER;
+            }
+        }
+    }
+
+    public static class NoOpPostingListEventListener implements QueryEventListener.PostingListEventListener
+    {
+        @Override
+        public void onAdvance() { }
+
+        @Override
+        public void postingDecoded(long postingsDecoded) { }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/RAMPostingSlices.java b/src/java/org/apache/cassandra/index/sai/disk/RAMPostingSlices.java
new file mode 100644
index 000000000000..1313f81569ae
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/RAMPostingSlices.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.mutable.MutableValueInt;
+
+/**
+ * Encodes postings as variable integers into "slices" of byte blocks for efficient memory usage.
+ */
+class RAMPostingSlices
+{
+    static final int DEFAULT_TERM_DICT_SIZE = 1024;
+
+    /** Pool of byte blocks storing the actual posting data */
+    private final ByteBlockPool postingsPool;
+    /** true if we're also writing term frequencies for an analyzed index */
+    private final boolean includeFrequencies;
+
+    /** The starting positions of postings for each term.  Term id = index in array. */
+    private int[] postingStarts = new int[DEFAULT_TERM_DICT_SIZE];
+    /** The current write positions for each term's postings.  Term id = index in array. */
+    private int[] postingUptos = new int[DEFAULT_TERM_DICT_SIZE];
+    /** The number of postings for each term.  Term id = index in array. */
+    private int[] sizes = new int[DEFAULT_TERM_DICT_SIZE];
+
+    RAMPostingSlices(Counter memoryUsage, boolean includeFrequencies)
+    {
+        postingsPool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(memoryUsage));
+        this.includeFrequencies = includeFrequencies;
+    }
+
+    /**
+     * Creates and returns a PostingList for the given term ID.
+     */
+    PostingList postingList(int termID, final ByteSliceReader reader, long maxSegmentRowID)
+    {
+        initReader(reader, termID);
+
+        final MutableValueInt lastSegmentRowId = new MutableValueInt();
+
+        return new PostingList()
+        {
+            int frequency = Integer.MIN_VALUE;
+
+            @Override
+            public int nextPosting() throws IOException
+            {
+                if (reader.eof())
+                {
+                    frequency = Integer.MIN_VALUE;
+                    return PostingList.END_OF_STREAM;
+                }
+                else
+                {
+                    lastSegmentRowId.value += reader.readVInt();
+                    if (includeFrequencies)
+                        frequency = reader.readVInt();
+                    return lastSegmentRowId.value;
+                }
+            }
+
+            @Override
+            public int frequency()
+            {
+                if (!includeFrequencies)
+                    return 1;
+                if (frequency <= 0)
+                    throw new IllegalStateException("frequency() called before nextPosting()");
+                return frequency;
+            }
+
+            @Override
+            public int size()
+            {
+                return sizes[termID];
+            }
+
+            @Override
+            public int advance(int targetRowID)
+            {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    /**
+     * Initializes a ByteSliceReader for reading postings for a specific term.
+     */
+    void initReader(ByteSliceReader reader, int termID)
+    {
+        final int upto = postingUptos[termID];
+        reader.init(postingsPool, postingStarts[termID], upto);
+    }
+
+    /**
+     * Creates a new slice for storing postings for a given term ID.
+     * Grows the internal arrays if necessary and allocates a new block
+     * if the current block cannot accommodate a new slice.
+     */
+    void createNewSlice(int termID)
+    {
+        if (termID >= postingStarts.length - 1)
+        {
+            postingStarts = ArrayUtil.grow(postingStarts, termID + 1);
+            postingUptos = ArrayUtil.grow(postingUptos, termID + 1);
+            sizes = ArrayUtil.grow(sizes, termID + 1);
+        }
+
+        // the slice will not fit in the current block, create a new block
+        if ((ByteBlockPool.BYTE_BLOCK_SIZE - postingsPool.byteUpto) < ByteBlockPool.FIRST_LEVEL_SIZE)
+        {
+            postingsPool.nextBuffer();
+        }
+
+        final int upto = postingsPool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
+        postingStarts[termID] = upto + postingsPool.byteOffset;
+        postingUptos[termID] = upto + postingsPool.byteOffset;
+    }
+
+    void writePosting(int termID, int deltaRowId, int frequency)
+    {
+        assert termID >= 0 : termID;
+        assert deltaRowId >= 0 : deltaRowId;
+        writeVInt(termID, deltaRowId);
+
+        if (includeFrequencies)
+        {
+            assert frequency > 0 : frequency;
+            writeVInt(termID, frequency);
+        }
+
+        sizes[termID]++;
+    }
+
+    /**
+     * Writes a variable-length integer to the posting list for a given term.
+     * The integer is encoded using a variable-length encoding scheme where each
+     * byte uses 7 bits for the value and 1 bit to indicate if more bytes follow.
+     */
+    private void writeVInt(int termID, int i)
+    {
+        while ((i & ~0x7F) != 0)
+        {
+            writeByte(termID, (byte) ((i & 0x7f) | 0x80));
+            i >>>= 7;
+        }
+        writeByte(termID, (byte) i);
+    }
+
+    /**
+     * Writes a single byte to the posting list for a given term.
+     * If the current slice is full, it automatically allocates a new slice.
+     */
+    private void writeByte(int termID, byte b)
+    {
+        int upto = postingUptos[termID];
+        byte[] block = postingsPool.buffers[upto >> ByteBlockPool.BYTE_BLOCK_SHIFT];
+        assert block != null;
+        int offset = upto & ByteBlockPool.BYTE_BLOCK_MASK;
+        if (block[offset] != 0)
+        {
+            // End of slice; allocate a new one
+            offset = postingsPool.allocSlice(block, offset);
+            block = postingsPool.buffer;
+            postingUptos[termID] = offset + postingsPool.byteOffset;
+        }
+        block[offset] = b;
+        postingUptos[termID]++;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/RAMStringIndexer.java b/src/java/org/apache/cassandra/index/sai/disk/RAMStringIndexer.java
new file mode 100644
index 000000000000..40c7d8fa37a8
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/RAMStringIndexer.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.agrona.collections.Int2IntHashMap;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.Counter;
+
+/**
+ * Indexes strings into an on-heap inverted index to be flushed in an SSTable attached index later.
+ * For flushing use the PostingTerms interface.
+ */
+public class RAMStringIndexer
+{
+    @VisibleForTesting
+    public static int MAX_BLOCK_BYTE_POOL_SIZE = Integer.MAX_VALUE;
+    private final BytesRefHash termsHash;
+    private final RAMPostingSlices slices;
+    // counters need to be separate so that we can trigger flushes if either ByteBlockPool hits maximum size
+    private final Counter termsBytesUsed;
+    private final Counter slicesBytesUsed;
+
+    private int[] lastSegmentRowID = new int[RAMPostingSlices.DEFAULT_TERM_DICT_SIZE];
+
+    private final boolean writeFrequencies;
+    private final Int2IntHashMap docLengths = new Int2IntHashMap(Integer.MIN_VALUE);
+
+    public RAMStringIndexer(boolean writeFrequencies)
+    {
+        this.writeFrequencies = writeFrequencies;
+        termsBytesUsed = Counter.newCounter();
+        slicesBytesUsed = Counter.newCounter();
+
+        ByteBlockPool termsPool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(termsBytesUsed));
+
+        termsHash = new BytesRefHash(termsPool);
+
+        slices = new RAMPostingSlices(slicesBytesUsed, writeFrequencies);
+    }
+
+    public long estimatedBytesUsed()
+    {
+        return termsBytesUsed.get() + slicesBytesUsed.get();
+    }
+
+    public boolean requiresFlush()
+    {
+        // ByteBlockPool can't handle more than Integer.MAX_VALUE bytes. These are allocated in fixed-size chunks,
+        // and additions are guaranteed to be smaller than the chunks. This means that the last chunk allocation will
+        // be triggered by an addition, and the rest of the space in the final chunk will be wasted, as the bytesUsed
+        // counters track block allocation, not the size of additions. This means that we can't pass this check and then
+        // fail to add a term.
+        return termsBytesUsed.get() >= MAX_BLOCK_BYTE_POOL_SIZE || slicesBytesUsed.get() >= MAX_BLOCK_BYTE_POOL_SIZE;
+    }
+
+    public boolean isEmpty()
+    {
+        return docLengths.isEmpty();
+    }
+
+    public Int2IntHashMap getDocLengths()
+    {
+        return docLengths;
+    }
+
+    /**
+     * EXPENSIVE OPERATION due to sorting the terms, only call once.
+     */
+    // TODO: assert or throw and exception if getTermsWithPostings is called > 1
+    public TermsIterator getTermsWithPostings(ByteBuffer minTerm, ByteBuffer maxTerm, ByteComparable.Version byteComparableVersion)
+    {
+        final int[] sortedTermIDs = termsHash.sort();
+
+        final int valueCount = termsHash.size();
+        final ByteSliceReader sliceReader = new ByteSliceReader();
+
+        return new TermsIterator()
+        {
+            private int ordUpto = 0;
+            private final BytesRef br = new BytesRef();
+
+            @Override
+            public ByteBuffer getMinTerm()
+            {
+                return minTerm;
+            }
+
+            @Override
+            public ByteBuffer getMaxTerm()
+            {
+                return maxTerm;
+            }
+
+            public void close() {}
+
+            @Override
+            public PostingList postings()
+            {
+                int termID = sortedTermIDs[ordUpto - 1];
+                final int maxSegmentRowId = lastSegmentRowID[termID];
+                return slices.postingList(termID, sliceReader, maxSegmentRowId);
+            }
+
+            @Override
+            public boolean hasNext() {
+                return ordUpto < valueCount;
+            }
+
+            @Override
+            public ByteComparable next()
+            {
+                if (!hasNext())
+                    throw new NoSuchElementException();
+
+                termsHash.get(sortedTermIDs[ordUpto], br);
+                ordUpto++;
+                return asByteComparable(br.bytes, br.offset, br.length);
+            }
+
+            private ByteComparable asByteComparable(byte[] bytes, int offset, int length)
+            {
+                // The bytes were encoded when they were inserted into the termsHash.
+                return ByteComparable.preencoded(byteComparableVersion, bytes, offset, length);
+            }
+        };
+    }
+
+    /**
+     * @return bytes allocated.  may be zero if the (term, row) pair is a duplicate
+     */
+    public long addAll(List<BytesRef> terms, int segmentRowId)
+    {
+        long startBytes = estimatedBytesUsed();
+        Int2IntHashMap frequencies = new Int2IntHashMap(Integer.MIN_VALUE);
+        Int2IntHashMap deltas = new Int2IntHashMap(Integer.MIN_VALUE);
+
+        for (BytesRef term : terms)
+        {
+            int termID = termsHash.add(term);
+            boolean firstOccurrence = termID >= 0;
+
+            if (firstOccurrence)
+            {
+                // first time seeing this term in any row, create the term's first slice !
+                slices.createNewSlice(termID);
+                // grow the termID -> last segment array if necessary
+                if (termID >= lastSegmentRowID.length - 1)
+                    lastSegmentRowID = ArrayUtil.grow(lastSegmentRowID, termID + 1);
+                if (writeFrequencies)
+                    frequencies.put(termID, 1);
+            }
+            else
+            {
+                termID = (-termID) - 1;
+                // compaction should call this method only with increasing segmentRowIds
+                assert segmentRowId >= lastSegmentRowID[termID];
+                // increment frequency
+                if (writeFrequencies)
+                    frequencies.put(termID, frequencies.getOrDefault(termID, 0) + 1);
+                // Skip computing a delta if we've already seen this term in this row
+                if (segmentRowId == lastSegmentRowID[termID])
+                    continue;
+            }
+
+            // Compute the delta from the last time this term was seen, to this row
+            int delta = segmentRowId - lastSegmentRowID[termID];
+            // sanity check that we're advancing the row id, i.e. no duplicate entries.
+            assert firstOccurrence || delta > 0;
+            deltas.put(termID, delta);
+            lastSegmentRowID[termID] = segmentRowId;
+        }
+
+        // add the postings now that we know the frequencies
+        deltas.forEachInt((termID, delta) -> {
+            slices.writePosting(termID, delta, frequencies.get(termID));
+        });
+
+        docLengths.put(segmentRowId, terms.size());
+
+        return estimatedBytesUsed() - startBytes;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/ResettableByteBuffersIndexOutput.java b/src/java/org/apache/cassandra/index/sai/disk/ResettableByteBuffersIndexOutput.java
deleted file mode 100644
index 19430a50878c..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/ResettableByteBuffersIndexOutput.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.ByteBuffersIndexOutput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexOutput;
-
-/***
- * A wrapper around {@link ByteBuffersIndexOutput} that adds several methods that interact
- * with the underlying delegate.
- */
-public class ResettableByteBuffersIndexOutput extends IndexOutput
-{
-    private final ByteBuffersIndexOutput bbio;
-    private final ByteBuffersDataOutput delegate;
-
-    public ResettableByteBuffersIndexOutput(String name)
-    {
-        //TODO CASSANDRA-18280 to investigate the initial size allocation
-        this(128, name);
-    }
-
-    public ResettableByteBuffersIndexOutput(int expectedSize, String name)
-    {
-        super("", name);
-        delegate = new ByteBuffersDataOutput(expectedSize);
-        bbio = new ByteBuffersIndexOutput(delegate, "", name + "-bb");
-    }
-
-    public void copyTo(IndexOutput out) throws IOException
-    {
-        delegate.copyTo(out);
-    }
-
-    public int intSize() {
-        return Math.toIntExact(bbio.getFilePointer());
-    }
-
-    public byte[] toArrayCopy() {
-        return delegate.toArrayCopy();
-    }
-
-    public void reset()
-    {
-        delegate.reset();
-    }
-
-    @Override
-    public String toString()
-    {
-        return "Resettable" + bbio.toString();
-    }
-
-    @Override
-    public void close() throws IOException
-    {
-        bbio.close();
-    }
-
-    @Override
-    public long getFilePointer()
-    {
-        return bbio.getFilePointer();
-    }
-
-    @Override
-    public long getChecksum() throws IOException
-    {
-        return bbio.getChecksum();
-    }
-
-    @Override
-    public void writeByte(byte b) throws IOException
-    {
-        bbio.writeByte(b);
-    }
-
-    @Override
-    public void writeBytes(byte[] b, int offset, int length) throws IOException
-    {
-        bbio.writeBytes(b, offset, length);
-    }
-
-    @Override
-    public void writeBytes(byte[] b, int length) throws IOException
-    {
-        bbio.writeBytes(b, length);
-    }
-
-    @Override
-    public void writeInt(int i) throws IOException
-    {
-        bbio.writeInt(i);
-    }
-
-    @Override
-    public void writeShort(short i) throws IOException
-    {
-        bbio.writeShort(i);
-    }
-
-    @Override
-    public void writeLong(long i) throws IOException
-    {
-        bbio.writeLong(i);
-    }
-
-    @Override
-    public void writeString(String s) throws IOException
-    {
-        bbio.writeString(s);
-    }
-
-    @Override
-    public void copyBytes(DataInput input, long numBytes) throws IOException
-    {
-        bbio.copyBytes(input, numBytes);
-    }
-
-    @Override
-    public void writeMapOfStrings(Map<String, String> map) throws IOException
-    {
-        bbio.writeMapOfStrings(map);
-    }
-
-    @Override
-    public void writeSetOfStrings(Set<String> set) throws IOException
-    {
-        bbio.writeSetOfStrings(set);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/RowMapping.java b/src/java/org/apache/cassandra/index/sai/disk/RowMapping.java
deleted file mode 100644
index 2b91bc304bc6..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/RowMapping.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk;
-
-import java.util.Collections;
-import java.util.Iterator;
-
-import javax.annotation.concurrent.NotThreadSafe;
-
-import com.carrotsearch.hppc.LongArrayList;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.rows.RangeTombstoneMarker;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.tries.InMemoryTrie;
-import org.apache.cassandra.index.sai.memory.MemtableIndex;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.PrimaryKeys;
-import org.apache.cassandra.io.compress.BufferType;
-import org.apache.cassandra.utils.AbstractGuavaIterator;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-
-/**
- * In memory representation of {@link PrimaryKey} to row ID mappings which only contains
- * {@link Row} regardless of whether it's live or deleted. ({@link RangeTombstoneMarker} is not included.)
- * <p>
- * While this inherits the threading behaviour of {@link InMemoryTrie} of single-writer / multiple-reader,
- * since it is only used by {@link StorageAttachedIndexWriter}, which is not threadsafe, we can consider
- * this class not threadsafe as well.
- */
-@NotThreadSafe
-public class RowMapping
-{
-    private static final InMemoryTrie.UpsertTransformer<Long, Long> OVERWRITE_TRANSFORMER = (existing, update) -> update;
-
-    public static final RowMapping DUMMY = new RowMapping()
-    {
-        @Override
-        public Iterator<Pair<ByteComparable, LongArrayList>> merge(MemtableIndex index) { return Collections.emptyIterator(); }
-
-        @Override
-        public void complete() {}
-
-        @Override
-        public boolean isComplete()
-        {
-            return true;
-        }
-
-        @Override
-        public void add(PrimaryKey key, long sstableRowId) {}
-
-        @Override
-        public int get(PrimaryKey key)
-        {
-            return -1;
-        }
-    };
-
-    private final InMemoryTrie<Long> rowMapping = new InMemoryTrie<>(BufferType.OFF_HEAP);
-
-    private boolean complete = false;
-
-    private RowMapping()
-    {}
-
-    /**
-     * Create row mapping for FLUSH operation only.
-     */
-    public static RowMapping create(OperationType opType)
-    {
-        if (opType == OperationType.FLUSH)
-            return new RowMapping();
-        return DUMMY;
-    }
-
-    /**
-     * Link the term -> {@link PrimaryKeys} mappings from a provided {@link MemtableIndex} to
-     * the {@link PrimaryKey} -> row ID mappings maintained here in {@link #rowMapping} to produce
-     * mappings of terms to their postings lists.
-     *
-     * @param index a Memtable-attached column index
-     *
-     * @return an iterator of term -> postings list {@link Pair}s
-     */
-    public Iterator<Pair<ByteComparable, LongArrayList>> merge(MemtableIndex index)
-    {
-        assert complete : "RowMapping is not built.";
-
-        Iterator<Pair<ByteComparable, PrimaryKeys>> iterator = index.iterator();
-        return new AbstractGuavaIterator<>()
-        {
-            @Override
-            protected Pair<ByteComparable, LongArrayList> computeNext()
-            {
-                while (iterator.hasNext())
-                {
-                    Pair<ByteComparable, PrimaryKeys> pair = iterator.next();
-
-                    LongArrayList postings = null;
-                    Iterator<PrimaryKey> primaryKeys = pair.right.iterator();
-
-                    while (primaryKeys.hasNext())
-                    {
-                        Long sstableRowId = rowMapping.get(primaryKeys.next());
-
-                        // The in-memory index does not handle deletions, so it is possible to
-                        // have a primary key in the index that doesn't exist in the row mapping
-                        if (sstableRowId != null)
-                        {
-                            postings = postings == null ? new LongArrayList() : postings;
-                            postings.add(sstableRowId);
-                        }
-                    }
-                    if (postings != null)
-                        return Pair.create(pair.left, postings);
-                }
-                return endOfData();
-            }
-        };
-    }
-
-    /**
-     * Complete building in memory RowMapping, mark it as immutable.
-     */
-    public void complete()
-    {
-        assert !complete : "RowMapping can only be built once.";
-        this.complete = true;
-    }
-
-    public boolean isComplete()
-    {
-        return complete;
-    }
-
-    /**
-     * Include PrimaryKey to RowId mapping
-     */
-    public void add(PrimaryKey key, long sstableRowId) throws InMemoryTrie.SpaceExhaustedException
-    {
-        assert !complete : "Cannot modify and already built RowMapping.";
-        rowMapping.putSingleton(key, sstableRowId, OVERWRITE_TRANSFORMER);
-    }
-
-    /**
-     * Returns the SSTable row ID for a {@link PrimaryKey}
-     *
-     * @param key the {@link PrimaryKey}
-     * @return a valid SSTable row ID for the {@link PrimaryKey} or -1 if the {@link PrimaryKey} doesn't exist
-     * in the {@link RowMapping}
-     */
-    public int get(PrimaryKey key)
-    {
-        Long sstableRowId = rowMapping.get(key);
-        return sstableRowId == null ? -1 : Math.toIntExact(sstableRowId);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java
deleted file mode 100644
index aca92e6dd22a..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Comparator;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Objects;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.virtual.SimpleDataSet;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.SSTableContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.format.Version;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentOrdering;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.io.sstable.SSTableIdFactory;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-
-/**
- * A reference-counted container of a {@link SSTableReader} for each column index that:
- * <ul>
- *     <li>Manages references to the SSTable for each query</li>
- *     <li>Exposes a version agnostic searcher onto the column index</li>
- *     <li>Exposes the index metadata for the column index</li>
- * </ul>
- */
-public abstract class SSTableIndex implements SegmentOrdering
-{
-    private static final Logger logger = LoggerFactory.getLogger(SSTableIndex.class);
-
-    // sort sstable indexes by first key, then last key, then descriptor id
-    public static final Comparator<SSTableIndex> COMPARATOR = Comparator.comparing((SSTableIndex s) -> s.getSSTable().getFirst())
-                                                                        .thenComparing(s -> s.getSSTable().getLast())
-                                                                        .thenComparing(s -> s.getSSTable().descriptor.id, SSTableIdFactory.COMPARATOR);
-
-    protected final SSTableContext sstableContext;
-    protected final IndexTermType indexTermType;
-    protected final IndexIdentifier indexIdentifier;
-
-    private final AtomicInteger references = new AtomicInteger(1);
-    private final AtomicBoolean obsolete = new AtomicBoolean(false);
-
-    public SSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index)
-    {
-        this.sstableContext = sstableContext.sharedCopy(); // this line must not be before any code that may throw
-        this.indexTermType = index.termType();
-        this.indexIdentifier = index.identifier();
-    }
-
-    /**
-     * Returns the amount of memory occupied by the index when it is initially loaded.
-     * This is the amount of data loaded into internal memory buffers by the index and
-     * does include the class footprint overhead. It used by the index metrics.
-     */
-    public abstract long indexFileCacheSize();
-
-    /**
-     * Returns the number of indexed rows in the index. This comes from the index
-     * metadata created when the index was written and is used by the index metrics.
-     */
-    public abstract long getRowCount();
-
-    /**
-     * Returns the minimum indexed rowId for the index. This comes from the index
-     * metadata created when the index was written and is used by the index metrics.
-     */
-    public abstract long minSSTableRowId();
-
-    /**
-     * Returns the maximum indexed rowId for the index. This comes from the index
-     * metadata created when the index was written and is used by the index metrics.
-     */
-    public abstract long maxSSTableRowId();
-
-    /**
-     * Returns the minimum term held in the index based on the natural sort order of
-     * the index column type comparator. It comes from the index metadata created when
-     * the index was written and is used by the index metrics and used in queries to
-     * determine whether a term, or range or terms, exists in the index.
-     */
-    public abstract ByteBuffer minTerm();
-
-    /**
-     * Returns the maximum term held in the index based on the natural sort order of
-     * the index column type comparator. It comes from the index metadata created when
-     * the index was written and is used by the index metrics and used in queries to
-     * determine whether a term, or range or terms, exists in the index.
-     */
-    public abstract ByteBuffer maxTerm();
-
-    /**
-     * Returns the key bounds of the index. It is created from the minimum and
-     * maximum keys held in the metadata and is used to determine whether
-     * sstable indexes overlap or not.
-     */
-    public abstract AbstractBounds<PartitionPosition> bounds();
-
-    /**
-     * Perform a search on the index for a single expression and keyRange.
-     * <p>
-     * The result is a {@link List} of {@link KeyRangeIterator} because there will
-     * be a {@link KeyRangeIterator} for each segment in the index. The result
-     * will never be null but may be an empty {@link List}.
-     *
-     * @param expression The {@link Expression} to be searched for
-     * @param keyRange The {@code AbstractBounds<PartitionPosition>} defining the
-     *                 token range for the search
-     * @param context The {@link QueryContext} holding the per-query state
-     * @return a {@link List} of {@link KeyRangeIterator}s containing the results
-     * of the search
-     */
-    public abstract List<KeyRangeIterator> search(Expression expression,
-                                                  AbstractBounds<PartitionPosition> keyRange,
-                                                  QueryContext context) throws IOException;
-
-    /**
-     * Populates a virtual table using the index metadata owned by the index
-     */
-    public abstract void populateSegmentView(SimpleDataSet dataSet);
-
-    protected abstract void internalRelease();
-
-    /**
-     * @return total size of per-column index components, in bytes
-     */
-    public long sizeOfPerColumnComponents()
-    {
-        return sstableContext.indexDescriptor.sizeOnDiskOfPerIndexComponents(indexTermType, indexIdentifier);
-    }
-
-    public IndexTermType getIndexTermType()
-    {
-        return indexTermType;
-    }
-
-    public IndexIdentifier getIndexIdentifier()
-    {
-        return indexIdentifier;
-    }
-
-    public SSTableContext getSSTableContext()
-    {
-        return sstableContext;
-    }
-
-    public Version getVersion()
-    {
-        return sstableContext.indexDescriptor.version;
-    }
-
-    public SSTableReader getSSTable()
-    {
-        return sstableContext.sstable;
-    }
-
-    public boolean reference()
-    {
-        while (true)
-        {
-            int n = references.get();
-            if (n <= 0)
-                return false;
-            if (references.compareAndSet(n, n + 1))
-            {
-                return true;
-            }
-        }
-    }
-
-    public boolean isReleased()
-    {
-        return references.get() <= 0;
-    }
-
-    public void releaseQuietly()
-    {
-        try
-        {
-            release();
-        }
-        catch (Throwable e)
-        {
-            logger.error(indexIdentifier.logMessage("Failed to release index on SSTable {}"), getSSTable().descriptor, e);
-        }
-    }
-
-    public void release()
-    {
-        int n = references.decrementAndGet();
-
-        if (n == 0)
-        {
-            internalRelease();
-            sstableContext.close();
-
-            /*
-             * When SSTable is removed, storage-attached index components will be automatically removed by LogTransaction.
-             * We only remove index components explicitly in case of index corruption or index rebuild.
-             */
-            if (obsolete.get())
-            {
-                sstableContext.indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier);
-            }
-        }
-    }
-
-    public void markObsolete()
-    {
-        obsolete.getAndSet(true);
-        release();
-    }
-
-    @Override
-    public boolean equals(Object o)
-    {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        SSTableIndex other = (SSTableIndex)o;
-        return Objects.equal(sstableContext, other.sstableContext) &&
-               Objects.equal(indexTermType, other.indexTermType) &&
-               Objects.equal(indexIdentifier, other.indexIdentifier);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hashCode(sstableContext, indexTermType, indexIdentifier);
-    }
-
-    @Override
-    public String toString()
-    {
-        return MoreObjects.toStringHelper(this)
-                          .add("column", indexTermType.columnName())
-                          .add("sstable", sstableContext.sstable.descriptor)
-                          .add("minTerm", indexTermType.asString(minTerm()))
-                          .add("maxTerm", indexTermType.asString(maxTerm()))
-                          .add("totalRows", sstableContext.sstable.getTotalRows())
-                          .toString();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/SearchableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/SearchableIndex.java
new file mode 100644
index 000000000000..7aea633376ee
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/SearchableIndex.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.v1.Segment;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.CloseableIterator;
+
+/**
+ * This is used to abstract the index search between on-disk versions.
+ * Callers to this interface should be unaware of the on-disk version for
+ * the index.
+ *
+ * It is responsible for supplying metadata about the on-disk index. This is
+ * used during query time to help coordinate queries and is also returned
+ * by the virtual tables.
+ */
+public interface SearchableIndex extends Closeable
+{
+    public long indexFileCacheSize();
+
+    public long getRowCount();
+
+    public long minSSTableRowId();
+
+    public long maxSSTableRowId();
+
+    public ByteBuffer minTerm();
+
+    public ByteBuffer maxTerm();
+
+    public DecoratedKey minKey();
+
+    public DecoratedKey maxKey();
+
+    public KeyRangeIterator search(Expression expression,
+                                   AbstractBounds<PartitionPosition> keyRange,
+                                   QueryContext context,
+                                   boolean defer, int limit) throws IOException;
+
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderBy(Orderer orderer,
+                                                                  Expression slice,
+                                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                                  QueryContext context,
+                                                                  int limit,
+                                                                  long totalRows) throws IOException;
+
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderResultsBy(QueryContext context,
+                                                                         List<PrimaryKey> keys,
+                                                                         Orderer orderer,
+                                                                         int limit,
+                                                                         long totalRows) throws IOException;
+
+    List<Segment> getSegments();
+
+    public void populateSystemView(SimpleDataSet dataSet, SSTableReader sstable);
+
+    long estimateMatchingRowsCount(Expression predicate, AbstractBounds<PartitionPosition> keyRange);
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/StorageAttachedIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/StorageAttachedIndexWriter.java
index f35ae67f93c2..dd8177eec28d 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/StorageAttachedIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/StorageAttachedIndexWriter.java
@@ -18,89 +18,111 @@
 package org.apache.cassandra.index.sai.disk;
 
 import java.io.IOException;
+import java.lang.invoke.MethodHandles;
 import java.util.Collection;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import javax.annotation.concurrent.NotThreadSafe;
 
 import com.google.common.base.Stopwatch;
+import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Unfiltered;
-import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.memory.RowMapping;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.Throwables;
 
 /**
  * Writes all on-disk index structures attached to a given SSTable.
  */
-@NotThreadSafe
 public class StorageAttachedIndexWriter implements SSTableFlushObserver
 {
-    private static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndexWriter.class);
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
     private final IndexDescriptor indexDescriptor;
-    private final Collection<PerColumnIndexWriter> perIndexWriters;
-    private final PerSSTableIndexWriter perSSTableWriter;
+    private final PrimaryKey.Factory primaryKeyFactory;
+    private final Collection<StorageAttachedIndex> indices;
+    private final Collection<PerIndexWriter> perIndexWriters;
+    private final PerSSTableWriter perSSTableWriter;
     private final Stopwatch stopwatch = Stopwatch.createUnstarted();
     private final RowMapping rowMapping;
+    private final OperationType opType;
+    private final TableMetrics tableMetrics;
+
     private DecoratedKey currentKey;
     private boolean tokenOffsetWriterCompleted = false;
     private boolean aborted = false;
 
     private long sstableRowId = 0;
-
-    public static StorageAttachedIndexWriter createFlushObserverWriter(IndexDescriptor indexDescriptor,
-                                                                       Collection<StorageAttachedIndex> indexes,
-                                                                       LifecycleNewTracker lifecycleNewTracker) throws IOException
-    {
-        return new StorageAttachedIndexWriter(indexDescriptor, indexes, lifecycleNewTracker, false);
-
-    }
-
-    public static StorageAttachedIndexWriter createBuilderWriter(IndexDescriptor indexDescriptor,
-                                                                 Collection<StorageAttachedIndex> indexes,
-                                                                 LifecycleNewTracker lifecycleNewTracker,
-                                                                 boolean perIndexComponentsOnly) throws IOException
+    private long totalTimeSpent = 0;
+
+    public StorageAttachedIndexWriter(IndexDescriptor indexDescriptor,
+                                      TableMetadata tableMetadata,
+                                      Collection<StorageAttachedIndex> indices,
+                                      LifecycleNewTracker lifecycleNewTracker,
+                                      long keyCount,
+                                      TableMetrics tableMetrics) throws IOException
     {
-        return new StorageAttachedIndexWriter(indexDescriptor, indexes, lifecycleNewTracker, perIndexComponentsOnly);
+        this(indexDescriptor, tableMetadata, indices, lifecycleNewTracker, keyCount, false, tableMetrics);
     }
 
-    private StorageAttachedIndexWriter(IndexDescriptor indexDescriptor,
-                                       Collection<StorageAttachedIndex> indexes,
-                                       LifecycleNewTracker lifecycleNewTracker,
-                                       boolean perIndexComponentsOnly) throws IOException
+    public StorageAttachedIndexWriter(IndexDescriptor indexDescriptor,
+                                      TableMetadata tableMetadata,
+                                      Collection<StorageAttachedIndex> indices,
+                                      LifecycleNewTracker lifecycleNewTracker,
+                                      long keyCount,
+                                      boolean perIndexComponentsOnly,
+                                      TableMetrics tableMetrics) throws IOException
     {
+        // We always write at the latest version (through what that version is can be configured for specific cases)
+        var onDiskFormat = Version.latest().onDiskFormat();
         this.indexDescriptor = indexDescriptor;
-        this.rowMapping = RowMapping.create(lifecycleNewTracker.opType());
-        this.perIndexWriters = indexes.stream().map(index -> indexDescriptor.newPerColumnIndexWriter(index,
-                                                                                                     lifecycleNewTracker,
-                                                                                                     rowMapping))
+        // Note: I think there is a silent assumption here. That is, the PK factory we use here must be for the latest
+        // format version, because that is what `IndexContext.keyFactory` always uses (see ctor)
+        this.primaryKeyFactory = onDiskFormat.newPrimaryKeyFactory(tableMetadata.comparator);
+        this.indices = indices;
+        this.opType = lifecycleNewTracker.opType();
+        this.rowMapping = RowMapping.create(opType);
+        this.perIndexWriters = indices.stream().map(i -> onDiskFormat.newPerIndexWriter(i,
+                                                                                        indexDescriptor,
+                                                                                        lifecycleNewTracker,
+                                                                                        rowMapping,
+                                                                                        keyCount))
                                       .filter(Objects::nonNull) // a null here means the column had no data to flush
                                       .collect(Collectors.toList());
 
         // If the SSTable components are already being built by another index build then we don't want
-        // to build them again so use a null writer
-        this.perSSTableWriter = perIndexComponentsOnly ? PerSSTableIndexWriter.NONE : indexDescriptor.newPerSSTableIndexWriter();
+        // to build them again so use a NO-OP writer
+        this.perSSTableWriter = perIndexComponentsOnly
+                                ? PerSSTableWriter.NONE
+                                : onDiskFormat.newPerSSTableWriter(indexDescriptor);
+        this.tableMetrics = tableMetrics;
     }
 
     @Override
     public void begin()
     {
-        logger.debug(indexDescriptor.logMessage("Starting partition iteration for storage-attached index flush for SSTable {}..."), indexDescriptor.sstableDescriptor);
+        logger.trace(indexDescriptor.logMessage("Starting partition iteration for storage attached index flush for SSTable {}..."), indexDescriptor.descriptor);
         stopwatch.start();
     }
 
     @Override
-    public void startPartition(DecoratedKey key, long keyPosition, long keyPositionForSASI)
+    public void startPartition(DecoratedKey key, long position, long keyPositionForSASI)
     {
         if (aborted) return;
         
@@ -108,12 +130,14 @@ public void startPartition(DecoratedKey key, long keyPosition, long keyPositionF
 
         try
         {
-            perSSTableWriter.startPartition(key);
+            perSSTableWriter.startPartition(position);
         }
         catch (Throwable t)
         {
-            logger.error(indexDescriptor.logMessage("Failed to record a partition during an index build"), t);
+            logger.error(indexDescriptor.logMessage("Failed to record a partition start during an index build"), t);
             abort(t, true);
+            // fail compaction task or index build task if SAI failed
+            throw Throwables.unchecked(t);
         }
     }
 
@@ -128,12 +152,14 @@ public void nextUnfilteredCluster(Unfiltered unfiltered)
 
         try
         {
-            addRow((Row)unfiltered);
+            addRow((Row) unfiltered);
         }
         catch (Throwable t)
         {
             logger.error(indexDescriptor.logMessage("Failed to record a row during an index build"), t);
             abort(t, true);
+            // fail compaction task or index build task if SAI failed
+            throw Throwables.unchecked(t);
         }
     }
 
@@ -153,45 +179,94 @@ public void staticRow(Row staticRow)
         {
             logger.error(indexDescriptor.logMessage("Failed to record a static row during an index build"), t);
             abort(t, true);
+            // fail compaction task or index build task if SAI failed
+            throw Throwables.unchecked(t);
         }
     }
 
     @Override
-    public void complete()
+    public void complete(SSTable sstable)
     {
-        if (aborted) return;
-
-        long start = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-
-        logger.debug(indexDescriptor.logMessage("Completed partition iteration for index flush for SSTable {}. Elapsed time: {} ms"),
-                     indexDescriptor.sstableDescriptor, start);
-
+        long startComplete = Clock.Global.nanoTime();
         try
         {
-            perSSTableWriter.complete();
-            tokenOffsetWriterCompleted = true;
+            if (aborted) return;
+
+            long start = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
-            long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+            logger.trace(indexDescriptor.logMessage("Completed partition iteration for index flush for SSTable {}. Elapsed time: {} ms"),
+                         indexDescriptor.descriptor,
+                         start);
 
-            logger.debug(indexDescriptor.logMessage("Completed per-SSTable write for SSTable {}. Duration: {} ms. Total elapsed time: {} ms."),
-                         indexDescriptor.sstableDescriptor, elapsed - start, elapsed);
+            try
+            {
+                perSSTableWriter.complete(stopwatch);
+                tokenOffsetWriterCompleted = true;
+                long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+                logger.trace(indexDescriptor.logMessage("Completed per-SSTable write for SSTable {}. Duration: {} ms. Total elapsed time: {} ms."),
+                             indexDescriptor.descriptor,
+                             elapsed - start,
+                             elapsed);
 
-            start = elapsed;
+                start = elapsed;
 
-            rowMapping.complete();
+                rowMapping.complete();
 
-            for (PerColumnIndexWriter perIndexWriter : perIndexWriters)
+                for (PerIndexWriter perIndexWriter : perIndexWriters)
+                {
+                    perIndexWriter.complete(stopwatch);
+
+                    // The handling of components when we flush/compact is a tad backward: instead of registering the
+                    // components as we write them, all the components are collected beforehand in `SSTableWriter#create`,
+                    // which means this is a superset of possible components, but if any components are not written for
+                    // those reason, this needs to be fixed afterward. One case for SAI component for instance is empty
+                    // indexes: if a particular sstable has nothing indexed for a particular index, then only the completion
+                    // marker for that index is kept on disk but no other components, so we need to remove the components
+                    // that were "optimistically" added (and more generally, future index implementation may have some
+                    // components that are only optionally present based on specific conditions).
+                    // Note 1: for index build/rebuild on existing sstable, `SSTableWriter#create` is not used, and instead
+                    //   we do only register components written (see `StorageAttachedIndexBuilder#completeSSTable`).
+                    // Note 2: as hinted above, an alternative here would be to change the whole handling of components,
+                    //   registering components only as they are effectively written. This is a larger refactor, with some
+                    //   subtleties involved, so it is left as potential future work.
+                    if (opType == OperationType.FLUSH || opType == OperationType.COMPACTION)
+                    {
+                        var writtenComponents = perIndexWriter.writtenComponents().allAsCustomComponents();
+                        var registeredComponents = IndexDescriptor.perIndexComponentsForNewlyFlushedSSTable(perIndexWriter.indexContext());
+                        var toRemove = Sets.difference(registeredComponents, writtenComponents);
+                        if (!toRemove.isEmpty())
+                        {
+                            if (logger.isTraceEnabled())
+                            {
+                                logger.trace(indexDescriptor.logMessage("Removing optimistically added but not writen components from TOC of SSTable {} for index {}"),
+                                             indexDescriptor.descriptor,
+                                             perIndexWriter.indexContext().getIndexName());
+                            }
+
+                            // During flush, this happens as we finalize the sstable and before its size is tracked, so not
+                            // passing a tracker is correct and intended (there is nothing to update in the tracker).
+                            sstable.unregisterComponents(toRemove, null);
+                        }
+                    }
+                }
+                elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+                logger.trace(indexDescriptor.logMessage("Completed per-index writes for SSTable {}. Duration: {} ms. Total elapsed time: {} ms."),
+                             indexDescriptor.descriptor,
+                             elapsed - start,
+                             elapsed);
+            }
+            catch (Throwable t)
             {
-                perIndexWriter.complete(stopwatch);
+                logger.error(indexDescriptor.logMessage("Failed to complete an index build"), t);
+                abort(t, true);
+                // fail compaction task or index build task if SAI failed
+                throw Throwables.unchecked(t);
             }
-            elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-            logger.debug(indexDescriptor.logMessage("Completed per-index writes for SSTable {}. Duration: {} ms. Total elapsed time: {} ms."),
-                         indexDescriptor.sstableDescriptor, elapsed - start, elapsed);
         }
-        catch (Throwable t)
+        finally
         {
-            logger.error(indexDescriptor.logMessage("Failed to complete an index build"), t);
-            abort(t, true);
+            totalTimeSpent += (Clock.Global.nanoTime() - startComplete);
+            tableMetrics.updateStorageAttachedIndexWritingTime(totalTimeSpent, opType);
         }
     }
 
@@ -213,12 +288,13 @@ public void abort(Throwable accumulator)
      */
     public void abort(Throwable accumulator, boolean fromIndex)
     {
-        if (aborted) return;
+        if (aborted)
+            return;
 
-        // Mark the write operation aborted, so we can short-circuit any further operations on the component writers.
+        // Mark the write aborted, so we can short-circuit any further operations on the component writers.
         aborted = true;
         
-        for (PerColumnIndexWriter perIndexWriter : perIndexWriters)
+        for (PerIndexWriter perIndexWriter : perIndexWriters)
         {
             try
             {
@@ -236,26 +312,31 @@ public void abort(Throwable accumulator, boolean fromIndex)
         if (!tokenOffsetWriterCompleted)
         {
             // If the token/offset files have already been written successfully, they can be reused later. 
-            perSSTableWriter.abort();
+            perSSTableWriter.abort(accumulator);
         }
 
-        // If the abort was from an index error, propagate the error upstream so index builds, compactions, and 
+        // If the abort was from an index error, propagate the error upstream so index builds, compactions, and
         // flushes can handle it correctly.
         if (fromIndex)
             throw Throwables.unchecked(accumulator);
     }
 
-    private void addRow(Row row) throws IOException, InMemoryTrie.SpaceExhaustedException
+    private void addRow(Row row) throws IOException, TrieSpaceExhaustedException
     {
-        PrimaryKey primaryKey = indexDescriptor.hasClustering() ? indexDescriptor.primaryKeyFactory.create(currentKey, row.clustering())
-                                                                : indexDescriptor.primaryKeyFactory.create(currentKey);
+        // we are using System.nanoTime() instead of ApproximateTime.nanoTime() here because
+        // it is verify likely that this method takes microsecronds instead of milliseconds
+        // and ApproximateTime.nanoTime() precision is 2 milliseconds
+        long now = Clock.Global.nanoTime();
+        PrimaryKey primaryKey = primaryKeyFactory.create(currentKey, row.clustering());
         perSSTableWriter.nextRow(primaryKey);
         rowMapping.add(primaryKey, sstableRowId);
 
-        for (PerColumnIndexWriter w : perIndexWriters)
+        for (PerIndexWriter w : perIndexWriters)
         {
             w.addRow(primaryKey, row, sstableRowId);
         }
         sstableRowId++;
+
+        totalTimeSpent += (Clock.Global.nanoTime() - now);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/TermsIterator.java b/src/java/org/apache/cassandra/index/sai/disk/TermsIterator.java
new file mode 100644
index 000000000000..bd84d1228ecd
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/TermsIterator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Iterator to step through terms to obtain {@link PostingList} for the current term.
+ *
+ * Term enumerations are always ordered by their {@link ByteSource}.
+ */
+@NotThreadSafe
+public interface TermsIterator extends Iterator<ByteComparable>, Closeable
+{
+    /**
+     * Get {@link PostingList} for the current term.
+     */
+    PostingList postings() throws IOException;
+
+    /**
+     * Get the minimum term in the iterator. Due to legacy design, this is the term as represented on disk without
+     * any special encoding.
+     */
+    ByteBuffer getMinTerm();
+
+    /**
+     * Get the maximum term in the iterator. Due to legacy design, this is the term as represented on disk without
+     * any special encoding.
+     */
+    ByteBuffer getMaxTerm();
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/ComponentsBuildId.java b/src/java/org/apache/cassandra/index/sai/disk/format/ComponentsBuildId.java
new file mode 100644
index 000000000000..74c214a1004b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/ComponentsBuildId.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.util.Objects;
+import java.util.function.Predicate;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.index.sai.IndexContext;
+
+/**
+ * Identifies a particular build of a per-sstable or per-index group of SAI index components, aka a pair of the
+ * {@link Version} built and the generation.
+ */
+public class ComponentsBuildId implements Comparable<ComponentsBuildId>
+{
+    private static final ComponentsBuildId FOR_NEW_SSTABLE = ComponentsBuildId.latest(0);
+
+    private final Version version;
+    private final int generation;
+
+    private ComponentsBuildId(Version version, int generation)
+    {
+        this.version = version;
+        this.generation = generation;
+    }
+
+    public static ComponentsBuildId of(Version version, int generation)
+    {
+        return new ComponentsBuildId(version, generation);
+    }
+
+    public static ComponentsBuildId latest(int generation)
+    {
+        return of(Version.latest(), generation);
+    }
+
+    public static ComponentsBuildId forNewSSTable()
+    {
+        return FOR_NEW_SSTABLE;
+    }
+
+    public static ComponentsBuildId forNewBuild(@Nullable ComponentsBuildId previousBuild, Predicate<ComponentsBuildId> newBuildIsUsablePredicate)
+    {
+        Version version = Version.latest();
+        // If we're not using immutable components, we always use generation 0, and we're fine if that overrides existing files
+        if (!version.useImmutableComponentFiles())
+            return new ComponentsBuildId(version, 0);
+
+        // Otherwise, if there is no previous build or the new build is for a new version, then we can "tentatively"
+        // use generation 0, but if not, we need to bump the generation.
+        int generation = previousBuild != null && previousBuild.version.equals(version) ? previousBuild.generation + 1 : 0;
+        var candidate = new ComponentsBuildId(version, generation);
+
+        // Usually, the candidate above is fine, but we want to avoid overriding existing file (it's theoretically
+        // possible that the next generation was created at some other point, but then corrupted, and so we falled back
+        // on the previous generation but some of those file for the next generation still exists). So we check,
+        // repeatedly if that candidate is usable, incrementing the generation until we find one which is.
+        while (!newBuildIsUsablePredicate.test(candidate))
+            candidate = new ComponentsBuildId(version, ++generation);
+
+        return candidate;
+    }
+
+    public Version version()
+    {
+        return version;
+    }
+
+    public int generation()
+    {
+        return generation;
+    }
+
+    public String formatAsComponent(IndexComponentType indexComponentType, IndexContext indexContext)
+    {
+        return version.fileNameFormatter().format(indexComponentType, indexContext, generation);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (!(obj instanceof ComponentsBuildId))
+            return false;
+        ComponentsBuildId that = (ComponentsBuildId) obj;
+        return this.version.equals(that.version) && this.generation == that.generation;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(version, generation);
+    }
+
+    @Override
+    public int compareTo(ComponentsBuildId that)
+    {
+        if (this.version.equals(that.version))
+            return Integer.compare(generation, that.generation);
+
+        return this.version.onOrAfter(that.version) ? 1 : -1;
+    }
+
+    @Override
+    public String toString()
+    {
+        return version + "@" + generation;
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/DefaultIndexComponentDiscovery.java b/src/java/org/apache/cassandra/index/sai/disk/format/DefaultIndexComponentDiscovery.java
new file mode 100644
index 000000000000..b958e26b9bdc
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/DefaultIndexComponentDiscovery.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.format;
+
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public class DefaultIndexComponentDiscovery extends IndexComponentDiscovery
+{
+    @Override
+    public SSTableIndexComponentsState discoverComponents(SSTableReader sstable) {
+        Descriptor descriptor = sstable.getDescriptor();
+
+        // Older versions might not have all components in the TOC, we should not trust it (fix for CNDB-13582):
+        if (descriptor.version.version.compareTo("ca") < 0)
+           return discoverComponentsFromDiskFallback(descriptor);
+
+        SSTableIndexComponentsState groups = tryDiscoverComponentsFromTOC(descriptor);
+        return groups == null
+               ? discoverComponentsFromDiskFallback(descriptor)
+               : groups;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponent.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponent.java
index 21cd5cf455d7..5e37871f0cec 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponent.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponent.java
@@ -18,115 +18,67 @@
 
 package org.apache.cassandra.index.sai.disk.format;
 
-import java.util.regex.Pattern;
+import java.io.IOException;
+import java.nio.ByteOrder;
 
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
-import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryWriter;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
 import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.lucene.store.ChecksumIndexInput;
 
-import static org.apache.cassandra.index.sai.disk.format.Version.SAI_DESCRIPTOR;
-import static org.apache.cassandra.index.sai.disk.format.Version.SAI_SEPARATOR;
+public interface IndexComponent
+{
+    IndexComponents parent();
 
+    IndexComponentType componentType();
 
-/**
- * This is a definitive list of all the on-disk components for all versions
- */
-public enum IndexComponent
-{
-    /**
-     * Metadata for per-column index components
-     */
-    META("Meta"),
-
-    /**
-     * Balanced tree written by {@code BlockBalancedTreeWriter} indexes mappings of term to one or more segment row IDs
-     * (segment row ID = SSTable row ID - segment row ID offset).
-     */
-    BALANCED_TREE("BalancedTree"),
-
-    /**
-     * Term dictionary written by {@link TrieTermsDictionaryWriter} stores mappings of term and
-     * file pointer to posting block on posting file.
-     */
-    TERMS_DATA("TermsData"),
-
-    /**
-     * Product Quantization store used to store compressed vectors for the vector index
-     */
-    COMPRESSED_VECTORS("CompressedVectors"),
-
-    /**
-     * Stores postings written by {@link PostingsWriter}
-     */
-    POSTING_LISTS("PostingLists"),
-
-    /**
-     * If present indicates that the column index build completed successfully
-     */
-    COLUMN_COMPLETION_MARKER("ColumnComplete"),
-
-
-    // per-sstable components
-    /**
-     * An on-disk block packed index mapping rowIds to token values.
-     */
-    ROW_TO_TOKEN("RowToToken"),
-
-    /**
-     * An on-disk block packed index mapping rowIds to partitionIds.
-     */
-    ROW_TO_PARTITION("RowToPartition"),
-
-    /**
-     * An on-disk block packed index mapping partitionIds to the number of rows for the partition.
-     */
-    PARTITION_TO_SIZE("PartitionToSize"),
-
-    /**
-     * Prefix-compressed blocks of partition keys used for rowId to partition key lookups
-     */
-    PARTITION_KEY_BLOCKS("PartitionKeyBlocks"),
-
-    /**
-     * Encoded sequence of offsets to partition key blocks
-     */
-    PARTITION_KEY_BLOCK_OFFSETS("PartitionKeyBlockOffsets"),
-
-    /**
-     * Prefix-compressed blocks of clustering keys used for rowId to clustering key lookups
-     */
-    CLUSTERING_KEY_BLOCKS("ClusteringKeyBlocks"),
-
-    /**
-     * Encoded sequence of offsets to clustering key blocks
-     */
-    CLUSTERING_KEY_BLOCK_OFFSETS("ClusteringKeyBlockOffsets"),
-
-    /**
-     * Metadata for per-SSTable on-disk components.
-     */
-    GROUP_META("GroupMeta"),
-
-    /**
-     * If present indicates that the per-sstable index build completed successfully
-     */
-    GROUP_COMPLETION_MARKER("GroupComplete");
-
-    public final String name;
-    public final Component.Type type;
-
-    IndexComponent(String name)
+    ByteOrder byteOrder();
+
+    String fileNamePart();
+    Component asCustomComponent();
+    File file();
+
+    default boolean isCompletionMarker()
     {
-        this.name = name;
-        this.type = componentType(name);
+        return componentType() == parent().completionMarkerComponent();
     }
 
-    private static Component.Type componentType(String name)
+    interface ForRead extends IndexComponent
     {
-        String componentName = SAI_DESCRIPTOR + SAI_SEPARATOR + name;
-        String repr = Pattern.quote(SAI_DESCRIPTOR + SAI_SEPARATOR)
-                      + ".*"
-                      + Pattern.quote(SAI_SEPARATOR + name + ".db");
-        return Component.Type.create(componentName, repr, true, null);
+        @Override
+        IndexComponents.ForRead parent();
+
+        FileHandle createFileHandle();
+
+        /**
+         * Opens a file handle for the provided index component similarly to {@link #createFileHandle()},
+         * but this method shoud be called instead of the aforemented one if the access is done during index building, that is
+         * before the full index that this is a part of has been finalized.
+         * <p>
+         * The use of this method can allow specific storage providers, typically tiered storage ones, to distinguish accesses
+         * that happen "at index building time" from other accesses, as the related file may be in different tier of storage.
+         */
+        FileHandle createIndexBuildTimeFileHandle();
+
+        IndexInput openInput();
+
+        ChecksumIndexInput openCheckSummedInput();
+    }
+
+    interface ForWrite extends IndexComponent
+    {
+        @Override
+        IndexComponents.ForWrite parent();
+
+        default IndexOutputWriter openOutput() throws IOException
+        {
+            return openOutput(false);
+        }
+
+        IndexOutputWriter openOutput(boolean append) throws IOException;
+
+        void createEmpty() throws IOException;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponentDiscovery.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponentDiscovery.java
new file mode 100644
index 000000000000..05bf34c04833
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponentDiscovery.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.NoSuchFileException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.TOCComponent;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_CUSTOM_COMPONENTS_DISCOVERY_CLASS;
+import static org.apache.cassandra.index.sai.disk.format.SSTableIndexComponentsState.State.toMB;
+
+/**
+ * Handles "discovering" SAI index components files from disk for a given sstable.
+ * <p>
+ * This is used by {@link IndexDescriptor} and should rarely, if ever, be used directly, but it is exposed publicly to
+ * make the logic "pluggable" (typically for tiered-storage that may not store files directly on disk and thus require
+ * some specific abstraction).
+ */
+public abstract class IndexComponentDiscovery
+{
+    private static final Logger logger = LoggerFactory.getLogger(IndexComponentDiscovery.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+
+
+    // This works around potential (if very unlikely in our case) class-loading issues.
+    private static class LazyDiscoveryInitialization
+    {
+        private static final IndexComponentDiscovery instance = !SAI_CUSTOM_COMPONENTS_DISCOVERY_CLASS.isPresent()
+                                                        ? new DefaultIndexComponentDiscovery() {}
+                                                        : FBUtilities.construct(SAI_CUSTOM_COMPONENTS_DISCOVERY_CLASS.getString(), "SAI index components discovery");
+    }
+
+    public static IndexComponentDiscovery instance()
+    {
+        return LazyDiscoveryInitialization.instance;
+    }
+
+    /**
+     * Returns the set of groups of SAI components that should be used for the provided sstable.
+     * <p>
+     * Note that "discovery" in this method only means finding out the "build ID" (version and generation) that should
+     * be used for each group of components (per-sstable and per-index).
+     *
+     * @param sstable the sstable reader for which to discover components.
+     * @return the discovered {@link ComponentsBuildId} to use for both per-sstable and each per-index components. The
+     * returned build IDs should usually correspond to existing index components on disk but this is not a strong
+     * asumption: if some group of components corresponding to the returned build ID has no completion marker or is
+     * missing files, the group will not be usuable (and the corresponding index/indexes will not be usable) but this
+     * should be handled "gracefully" by callers.
+     */
+    public abstract SSTableIndexComponentsState discoverComponents(SSTableReader sstable);
+
+    protected static IndexComponentType completionMarker(@Nullable String name)
+    {
+        return name == null ? IndexComponentType.GROUP_COMPLETION_MARKER : IndexComponentType.COLUMN_COMPLETION_MARKER;
+    }
+
+    /**
+     * Tries reading the TOC file of the provided SSTable to discover its current SAI components.
+     *
+     * @param descriptor the SSTable to read the TOC file of.
+     * @return the discovered components, or `null` if the TOC file is missing or if it is corrupted in some way.
+     */
+    protected @Nullable SSTableIndexComponentsState tryDiscoverComponentsFromTOC(Descriptor descriptor)
+    {
+        Set<Component> componentsFromToc = readSAIComponentFromSSTableTOC(descriptor);
+        if (componentsFromToc == null)
+            return null;
+
+        // We collect all the version/generation for which we have files on disk for the per-sstable parts and every
+        // per-index found.
+        Map<String, StateBuilder> states = new HashMap<>();
+        Set<String> invalid = new HashSet<>();
+        for (Component component : componentsFromToc)
+        {
+            // We try parsing it as an SAI index name, and ignore if it doesn't match.
+            var opt = Version.tryParseFileName(component.name);
+            if (opt.isEmpty())
+                continue;
+
+            var parsed = opt.get();
+            String indexName = parsed.indexName;
+
+            if (invalid.contains(indexName))
+                continue;
+
+            var prev = states.computeIfAbsent(indexName, k -> new StateBuilder(parsed.buildId));
+            if (!prev.buildId.equals(parsed.buildId))
+            {
+                logger.error("Found multiple versions/generations of SAI components in TOC for SSTable {}: cannot load {}",
+                             descriptor, indexName == null ? "per-SSTable components" : "per-index components of " + indexName);
+
+                states.remove(indexName);
+                invalid.add(indexName);
+            }
+
+            prev.totalSizeInBytes += descriptor.fileFor(component).length();
+        }
+
+        return StateBuilder.convert(states);
+    }
+
+    private @Nullable Set<Component> readSAIComponentFromSSTableTOC(Descriptor descriptor)
+    {
+        try
+        {
+            // We skip the check for missing components on purpose: we do the existence check here because we want to
+            // know when it fails.
+            Set<Component> components = TOCComponent.loadTOC(descriptor, false);
+            Set<Component> SAIComponents = new HashSet<>();
+            for (Component component : components)
+            {
+                // We only care about SAI components, which are "custom"
+                if (component.type != SSTableFormat.Components.Types.CUSTOM)
+                    continue;
+
+                // And all start with "SAI" (the rest can depend on the version, but that part is common to all version)
+                if (!component.name.startsWith(Version.SAI_DESCRIPTOR))
+                    continue;
+
+                // Lastly, we check that the component file exists. If it doesn't, then we assume something is wrong
+                // with the TOC and we fall back to scanning the disk. This is admittedly a bit conservative, but
+                // we do have test data in `test/data/legacy-sai/aa` where the TOC is broken: it lists components that
+                // simply do not match the accompanying files (the index name differs), and it is unclear if this is
+                // just a mistake made while gathering the test data or if some old version used to write broken TOC
+                // for some reason (more precisely, it is hard to be entirely sure this isn't the later).
+                // Overall, there is no real reason for the TOC to list non-existing files (typically, when we remove
+                // an index, the TOC is rewritten to omit the removed component _before_ the files are deleted), so
+                // falling back conservatively feels reasonable.
+                if (!descriptor.fileFor(component).exists())
+                {
+                    noSpamLogger.warn("The TOC file for SSTable {} lists SAI component {} but it doesn't exists. Assuming the TOC is corrupted somehow and falling back on disk scanning (which may be slower)", descriptor, component.name);
+                    return null;
+                }
+
+                SAIComponents.add(component);
+            }
+            return SAIComponents;
+        }
+        catch (NoSuchFileException e)
+        {
+            // This is totally fine when we're building an `IndexDescriptor` for a new sstable that does not exist.
+            // But if the sstable exist, then that's less expected as we should have a TOC. But because we want to
+            // be somewhat resilient to losing the TOC and that historically the TOC hadn't been relyed on too strongly,
+            // we return `null` which trigger the fall-back path to scan disk.
+            if (descriptor.fileFor(SSTableFormat.Components.DATA).exists())
+            {
+                noSpamLogger.warn("SSTable {} exists (its data component exists) but it has no TOC file. Will use disk scanning to discover SAI components as fallback (which may be slower).", descriptor);
+                return null;
+            }
+
+            return Collections.emptySet();
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /**
+     * Scan disk to find all the SAI components for the provided descriptor that exists on disk. Then pick
+     * the approriate set of those components (the highest version/generation for which there is a completion marker).
+     * This should usually only be used ask a fallback because this will scan the whole table directory every time and
+     * can be a bit inefficient, especially when some tiered storage is used underneath where scanning a directory may
+     * be particularly expensive. And picking the most recent version/generation is usually the right thing to do, but
+     * may lack flexibility in some cases.
+     *
+     * @param descriptor the SSTable for which to discover components for.
+     * @return the discovered components. This is never {@code null}, but could well be empty if no SAI components are
+     * found.
+     */
+    protected SSTableIndexComponentsState discoverComponentsFromDiskFallback(Descriptor descriptor)
+    {
+        // For each "component group" (of each individual index, plus the per-sstable group), the "active" group is the
+        // one with the most recent build amongst complete ones. So we scan disk looking for completion markers (since
+        // that's what tell us a group is complete), and keep for each group the max build we find.
+        Map<String, Map<ComponentsBuildId, StateBuilder>> states = new HashMap<>();
+        PathUtils.forEach(descriptor.directory.toPath(), path -> {
+            String filename = path.getFileName().toString();
+            // First, we skip any file that do not belong to the sstable this is a descriptor for.
+            if (!filename.startsWith(descriptor.filenamePart()))
+                return;
+
+            Version.tryParseFileName(filename)
+                   .ifPresent(parsed -> {
+                       var forGroup = states.computeIfAbsent(parsed.indexName, k -> new HashMap<>());
+                       var state = forGroup.computeIfAbsent(parsed.buildId, k -> new StateBuilder(parsed.buildId));
+                       state.totalSizeInBytes += PathUtils.size(path);
+                       if (parsed.component == completionMarker(parsed.indexName))
+                           state.isComplete = true;
+                   });
+        });
+
+        Map<String, StateBuilder> maxStates = new HashMap<>();
+        for (var entry : states.entrySet())
+        {
+            entry.getValue()
+                 .values()
+                 .stream()
+                 .filter(s -> s.isComplete)
+                 .max(Comparator.comparing(s -> s.buildId))
+                 .ifPresent(max -> maxStates.put(entry.getKey(), max));
+        }
+
+        return StateBuilder.convert(maxStates);
+    }
+
+    private static class StateBuilder
+    {
+        private final ComponentsBuildId buildId;
+        private long totalSizeInBytes;
+        private boolean isComplete;
+
+        StateBuilder(ComponentsBuildId buildId)
+        {
+            this.buildId = buildId;
+        }
+
+        void addTo(SSTableIndexComponentsState.Builder builder, @Nullable String indexName)
+        {
+            if (indexName == null)
+                builder.addPerSSTable(buildId, toMB(totalSizeInBytes));
+            else
+                builder.addPerIndex(indexName, buildId, toMB(totalSizeInBytes));
+        }
+
+        static SSTableIndexComponentsState convert(Map<String, StateBuilder> states)
+        {
+            SSTableIndexComponentsState.Builder builder = SSTableIndexComponentsState.builder();
+            states.forEach((indexName, state) -> state.addTo(builder, indexName));
+            return builder.build();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponentType.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponentType.java
new file mode 100644
index 000000000000..bf38d6b8fbd8
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponentType.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * This is a definitive list of all the on-disk components for all versions
+ */
+public enum IndexComponentType
+{
+    /**
+     * Stores per-index metadata.
+     *
+     * V1
+     */
+    META("Meta"),
+    /**
+     * KDTree written by {@code BKDWriter} indexes mappings of term to one ore more segment row IDs
+     * (segment row ID = SSTable row ID - segment row ID offset).
+     *
+     * V1
+     */
+    KD_TREE("KDTree"),
+    KD_TREE_POSTING_LISTS("KDTreePostingLists"),
+
+    /**
+     * Vector index components
+     */
+    VECTOR("Vector"),
+    PQ("PQ"),
+
+    /**
+     * Term dictionary written by {@code TrieTermsDictionaryWriter} stores mappings of term and
+     * file pointer to posting block on posting file.
+     *
+     * V1
+     */
+    TERMS_DATA("TermsData"),
+    /**
+     * Stores postings written by {@code PostingsWriter}
+     *
+     * V1
+     */
+    POSTING_LISTS("PostingLists"),
+    /**
+     * If present indicates that the column index build completed successfully
+     *
+     * V1
+     */
+    COLUMN_COMPLETION_MARKER("ColumnComplete"),
+
+    // per-sstable components
+    /**
+     * Partition key token value for rows including row tombstone and static row. (access key is rowId)
+     *
+     * V1 V2
+     */
+    TOKEN_VALUES("TokenValues"),
+    /**
+     * Partition key offset in sstable data file for rows including row tombstone and static row. (access key is
+     * rowId)
+     *
+     * V1
+     */
+    OFFSETS_VALUES("OffsetsValues"),
+    /**
+     * An on-disk trie containing the primary keys used for looking up the rowId from a partition key
+     *
+     * V2
+     */
+    PRIMARY_KEY_TRIE("PrimaryKeyTrie"),
+    /**
+     * Prefix-compressed blocks of primary keys used for rowId to partition key lookups
+     *
+     * V2
+     */
+    PRIMARY_KEY_BLOCKS("PrimaryKeyBlocks"),
+    /**
+     * Encoded sequence of offsets to primary key blocks
+     *
+     * V2
+     */
+    PRIMARY_KEY_BLOCK_OFFSETS("PrimaryKeyBlockOffsets"),
+    /**
+     * Stores per-sstable metadata.
+     *
+     * V1
+     */
+    GROUP_META("GroupMeta"),
+    /**
+     * If present indicates that the per-sstable index build completed successfully
+     *
+     * V1 V2
+     */
+    GROUP_COMPLETION_MARKER("GroupComplete"),
+    
+    /**
+     * Stores document length information for BM25 scoring
+     */
+    DOC_LENGTHS("DocLengths");
+
+    public final String representation;
+
+    IndexComponentType(String representation)
+    {
+        this.representation = representation;
+    }
+
+    static final Map<String, IndexComponentType> byRepresentation = new HashMap<>();
+    static
+    {
+        for (IndexComponentType component : values())
+            byRepresentation.put(component.representation, component);
+    }
+
+    public static @Nullable IndexComponentType fromRepresentation(String representation)
+    {
+        return byRepresentation.get(representation);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponents.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponents.java
new file mode 100644
index 000000000000..e0cd0959e2be
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponents.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Represents a related group of concrete SAI components files which are either all the per-sstable components of a
+ * given sstable, or all the components of a particular column index (for a given sstable).
+ * <p>
+ * The members of a component group correspond to actual SAI component files on disk, and are components that are
+ * written together. A group is identified by:
+ * <ul>
+ *     <li>the sstable it is the components of (identified by the sstable {@link #descriptor()})</li>
+ *     <li>the index ({@link #context()}) for the components, which is {@code null} for the per-sstable components</li>
+ *     <li>the version the components are using. All the components of a group are on the same version</li>
+ *     <li>the generation, within the version, of the group/components. Generations are used when
+ *     {@link CassandraRelevantProperties#IMMUTABLE_SAI_COMPONENTS} is used, to avoid new builds to override (and thus
+ *     mutate) older builds. When the option of immutable components is not used, then the generation is fixed to 0.
+ *     See below for more details.</li>
+ * </ul>
+ * <p>
+ *
+ * <h2>Immutable components</h2>
+ *
+ * As mentioned above, when {@link CassandraRelevantProperties#IMMUTABLE_SAI_COMPONENTS} is enabled, existing components
+ * are not overwritten by new builds (the underlying intent is to allow rebuilding without stopping reads (to the
+ * rebuilt indexes). But unless a rebuild uses a different version than the existing components, the new components
+ * need "something" to distinguish them from the old ones, and it is what the generation provides.
+ * <p>
+ * The generation is specific to a component group, meaning that all the components of a group share the same
+ * generation (or to put it another way, if 2 components only differ by their generation, then they belong to different
+ * build and thus different groups), but that different groups can have different generations. For instance, if a
+ * specific index is rebuilt but without rebuilding the per-sstable components of each sstable, then after that rebuild
+ * the per-sstable groups will have generation 0, but the index groups will have generation 1.
+ * <p>
+ * When a sstable is "loaded", the "active" set of components to use is based on finding, for each kind of groups
+ * (per-sstable and per-index), the "complete" (see next paragraph) set of components with the highest version, and
+ * highest generation within that version.
+ * <p>
+ * A group of components may or may not be "complete" ({@link #isComplete()}): it is complete if the completion marker
+ * component for that group is present. Groups are temporarily incomplete during writing, but can also be more
+ * permanently incomplete for 2 main reasons: a build may fail mid-way, leaving one or more group incomplete, or we can
+ * have some corruption of the component files of some sort (corruption could here mean that a file is mistakenly
+ * deleted, or that the content of the file is corrupted somehow; the later triggers a removal of the corrupted file and
+ * of the completion marker, but not of the other component of the group). The bumping of generations takes incomplete
+ * groups into account, and so incomplete groups are not overridden either. Essentially, the generation used by a new
+ * build is always one more than the highest generation of any component found on disk (for the group in question, and
+ * the version we writting, usually {@link Version#latest()}).
+ */
+public interface IndexComponents
+{
+    /**
+     * SSTable this is the group of.
+     */
+    Descriptor descriptor();
+
+    /**
+     * The {@link IndexDescriptor} that created this group.
+     * <p>
+     * Note that {@link IndexDescriptor} essentially tracks the active and created groups for a sstable, and so a group
+     * always comes from a particular {@link IndexDescriptor} instance.
+     */
+    IndexDescriptor indexDescriptor();
+
+    /**
+     * Context of the group.
+     *
+     * @return the context of the index this is a group of, or {@code null} if the group is a per-sstable group.
+     */
+    @Nullable IndexContext context();
+
+    /**
+     * The build id of the components of this group.
+     */
+    ComponentsBuildId buildId();
+
+    /**
+     * Version used by the components of the groups (part of the {@link #buildId()} but exposed directly because often
+     * used).
+     */
+    default Version version()
+    {
+        return buildId().version();
+    }
+
+    /**
+     * The on-disk format used by the these components.
+     */
+    default OnDiskFormat onDiskFormat()
+    {
+        return version().onDiskFormat();
+    }
+
+    /**
+     * Whether that's a per-index group, that is one with components specific to a given index. Otherwise, it is a
+     * per-sstable group, that is one with components shared by all the SAI indexes (on that sstable).
+     */
+    default boolean isPerIndexGroup()
+    {
+        return context() != null;
+    }
+
+    /**
+     * The specific component "kind" used for storing the metadata of this group.
+     */
+    default IndexComponentType metadataComponent()
+    {
+        return isPerIndexGroup() ? IndexComponentType.META : IndexComponentType.GROUP_META;
+    }
+
+    /**
+     * The specific component "kind" used as a completion marker for this group.
+     */
+    default IndexComponentType completionMarkerComponent()
+    {
+        return isPerIndexGroup() ? IndexComponentType.COLUMN_COMPLETION_MARKER : IndexComponentType.GROUP_COMPLETION_MARKER;
+    }
+
+    default String logMessage(String message)
+    {
+        return indexDescriptor().logMessage(message);
+    }
+
+    /**
+     * Whether the provided component "kind" exists in this group.
+     */
+    boolean has(IndexComponentType component);
+
+    /**
+     * Whether this group is complete, meaning that is has a completion marker.
+     */
+    default boolean isComplete()
+    {
+        return has(completionMarkerComponent());
+    }
+
+    /**
+     * An empty group is one that is complete, but has only a completion marker and no other components.
+     */
+    boolean isEmpty();
+
+    /**
+     * The complete set of component types that are expected for this group version.
+     */
+    default Set<IndexComponentType> expectedComponentsForVersion()
+    {
+        return isPerIndexGroup()
+               ? onDiskFormat().perIndexComponentTypes(context())
+               : onDiskFormat().perSSTableComponentTypes();
+    }
+
+    default ByteComparable.Version byteComparableVersionFor(IndexComponentType component)
+    {
+        return version().byteComparableVersionFor(component, descriptor().version);
+    }
+
+    /**
+     * Specialisation of {@link IndexComponents} used when working with complete and active groups, and so mostly used
+     * for reading components.
+     */
+    interface ForRead extends IndexComponents
+    {
+        IndexComponent.ForRead get(IndexComponentType component);
+
+        /**
+         * The total size on disk used by the components of this group.
+         */
+        long liveSizeOnDiskInBytes();
+
+        Collection<IndexComponent.ForRead> all();
+
+        default Set<Component> allAsCustomComponents()
+        {
+            return all()
+                   .stream()
+                   .map(IndexComponent::asCustomComponent)
+                   .collect(Collectors.toSet());
+        }
+
+        /**
+         * Validates this group and its components.
+         * <p>
+         * This method both check that the group has all the components that it should have, and that the content of
+         * those components are, as far as this method can determine, valid.
+         * <p>
+         * If the group is invalid, then it will be invalidated by calling {@link #invalidate}. Specifics about what
+         * failed validation is also be logged.
+         *
+         * @param sstable the sstable object for which the component are validated (which should correspond to
+         *               {@code this.descriptor()}). This must be provided so if some components are invalid, they
+         *               can be unregistered.
+         * @param tracker the {@link Tracker} of the table (of the sstable/components). Like the sstable, this is used
+         *                as part of unregistering the components when they are invalid.
+         * @param validateChecksum if {@code true}, the checksum of the components will be validated. Otherwise, only
+         *                         basic checks on the header and footers will be performed.
+         * @param rethrow whether to throw an {@link UncheckedIOException} if the group is invalid
+         * @return whether the group is valid.
+         */
+        boolean validateComponents(SSTable sstable, Tracker tracker, boolean validateChecksum, boolean rethrow);
+
+        /**
+         * Marks the group as invalid/broken.
+         * <p>
+         * If it is an active group, it will remove it as active in the underlying {@link IndexDescriptor}. It will also
+         * at least delete the completion marker to ensure the group does not get used on any reload/restart.
+         * <p>
+         * Please note that this method is <b>already</b> called by {@link #validateComponents} if the group
+         * is found invalid: it is exposed here for case where we have reason to think the group is invalid but
+         * validation hasn't detected it for some reason.
+         *
+         * @param sstable the sstable object for which the component are invalidated (which should correspond to
+         *               {@code this.descriptor()}). This must be provided so the invalidated components are
+         *               unregistered.
+         * @param tracker the {@link Tracker} of the table (of the sstable/components). Like the sstable, this is used
+         *                as part of unregistering the components.
+         */
+        void invalidate(SSTable sstable, Tracker tracker);
+
+        /**
+         * Returns a {@link ForWrite} view of this group, mostly for calling {@link ForWrite#forceDeleteAllComponents()} at
+         * appropriate times.
+         */
+        ForWrite forWrite();
+    }
+
+    /**
+     * Specialisation of {@link IndexComponents} used when doing a new index build and thus writting a new group of
+     * components.
+     * <p>
+     * This extends {@link ForRead} because we sometimes read previously written components to write other ones in the group
+     */
+    interface ForWrite extends ForRead
+    {
+        /**
+         * Adds the provided component "kind" to this writer, or return the previously added one if it had already
+         * been added.
+         */
+        IndexComponent.ForWrite addOrGet(IndexComponentType component);
+
+        /**
+         * Delete the files of all the components in this writer (and remove them so that the group will be empty
+         * afterward).
+         */
+        void forceDeleteAllComponents();
+
+        /**
+         * Writes the completion marker file for the group on disk and, if said write succeeds, adds the components
+         * of this writer to the {@link IndexDescriptor} that created this writer (and so no additional components
+         * should be added to this writer after this call).
+         */
+        void markComplete() throws IOException;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java
index 4501aec7f9ce..7d3326d3446c 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java
@@ -20,470 +20,713 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteOrder;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.concurrent.TimeUnit;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
-import com.google.common.io.Files;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ClusteringComparator;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.IndexValidation;
-import org.apache.cassandra.index.sai.SSTableContext;
-import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
-import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.RowMapping;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
 import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
 import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.disk.oldlucene.EndiannessReverserChecksumIndexInput;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.lucene.store.IndexInput;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.lucene.store.BufferedChecksumIndexInput;
+import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.util.IOUtils;
 
 /**
- * The {@link IndexDescriptor} is an analog of the SSTable {@link Descriptor} and provides version
- * specific information about the on-disk state of a {@link StorageAttachedIndex}.
+ * The `IndexDescriptor` is an analog of the SSTable {@link Descriptor} and provides version
+ * specific information about the on-disk state of {@link StorageAttachedIndex}es.
  * <p>
- * The {@link IndexDescriptor} is primarily responsible for maintaining a view of the on-disk state
- * of an index for a specific {@link org.apache.cassandra.io.sstable.SSTable}.
+ * The `IndexDescriptor` is primarily responsible for maintaining a view of the on-disk state
+ * of the SAI indexes for a specific {@link org.apache.cassandra.io.sstable.SSTable}. It maintains mappings
+ * of the current on-disk components and files. It is responsible for opening files for use by
+ * writers and readers.
  * <p>
- * It is responsible for opening files for use by writers and readers.
+ * Each sstable has per-index components ({@link IndexComponentType}) associated with it, and also components
+ * that are shared by all indexes (notably, the components that make up the PrimaryKeyMap).
  * <p>
- * Its remaining responsibility is to act as a proxy to the {@link OnDiskFormat} associated with the
- * index {@link Version}.
+ * IndexDescriptor's remaining responsibility is to act as a proxy to the {@link OnDiskFormat}
+ * associated with the index {@link Version}.
  */
 public class IndexDescriptor
 {
-    private static final Logger logger = LoggerFactory.getLogger(IndexDescriptor.class);
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+
+    private static final ComponentsBuildId EMPTY_GROUP_MARKER = ComponentsBuildId.latest(-1);
+
+    // TODO Because indexes can be added at any time to existing data, the Version of a column index
+    // may not match the Version of the base sstable.  OnDiskFormat + IndexFeatureSet + IndexDescriptor
+    // was not designed with this in mind, leading to some awkwardness, notably in IFS where some features
+    // are per-sstable (`isRowAware`) and some are per-column (`hasVectorIndexChecksum`).
+
+    public final Descriptor descriptor;
 
-    public final Version version;
-    public final Descriptor sstableDescriptor;
-    public final ClusteringComparator clusteringComparator;
-    public final PrimaryKey.Factory primaryKeyFactory;
+    // The per-sstable components for this descriptor. This is never `null` in practice, but 1) it's a bit easier to
+    // initialize it outsides of the ctor, and 2) it can actually change upon calls to `reload`.
+    private IndexComponentsImpl perSSTable;
+    private final Map<IndexContext, IndexComponentsImpl> perIndexes = Maps.newHashMap();
 
-    private IndexDescriptor(Version version, Descriptor sstableDescriptor, IPartitioner partitioner, ClusteringComparator clusteringComparator)
+    private IndexDescriptor(Descriptor descriptor)
     {
-        this.version = version;
-        this.sstableDescriptor = sstableDescriptor;
-        this.clusteringComparator = clusteringComparator;
-        this.primaryKeyFactory = new PrimaryKey.Factory(partitioner, clusteringComparator);
+        this.descriptor = descriptor;
     }
 
-    public static IndexDescriptor create(Descriptor descriptor, IPartitioner partitioner, ClusteringComparator clusteringComparator)
+    public static IndexDescriptor empty(Descriptor descriptor)
     {
-        return new IndexDescriptor(Version.LATEST, descriptor, partitioner, clusteringComparator);
+        IndexDescriptor created = new IndexDescriptor(descriptor);
+        // Some code assumes that you can always at least call `perSSTableComponents()` and not get `null`, so we
+        // set it to an empty group here.
+        created.perSSTable = created.createEmptyGroup(null);
+        return created;
     }
 
-    public static IndexDescriptor create(SSTableReader sstable)
+    public static IndexDescriptor load(SSTableReader sstable, Set<IndexContext> indices)
     {
-        for (Version version : Version.ALL)
-        {
-            IndexDescriptor indexDescriptor = new IndexDescriptor(version,
-                                                                  sstable.descriptor,
-                                                                  sstable.getPartitioner(),
-                                                                  sstable.metadata().comparator);
-
-            if (version.onDiskFormat().isPerSSTableIndexBuildComplete(indexDescriptor))
-            {
-                return indexDescriptor;
-            }
-        }
-        return new IndexDescriptor(Version.LATEST,
-                                   sstable.descriptor,
-                                   sstable.getPartitioner(),
-                                   sstable.metadata().comparator);
+        SSTableIndexComponentsState discovered = IndexComponentDiscovery.instance().discoverComponents(sstable);
+        IndexDescriptor descriptor = new IndexDescriptor(sstable.descriptor);
+        descriptor.initialize(indices, discovered);
+        return descriptor;
     }
 
-    public boolean hasClustering()
+    private void initialize(Set<IndexContext> indices, SSTableIndexComponentsState discovered)
     {
-        return clusteringComparator.size() > 0;
+        this.perSSTable = initializeGroup(null, discovered.perSSTableBuild());
+        initializeIndexes(indices, discovered);
     }
 
-    public String componentName(IndexComponent indexComponent)
+    private void initializeIndexes(Set<IndexContext> indices, SSTableIndexComponentsState discovered)
     {
-        return version.fileNameFormatter().format(indexComponent, null);
+        for (var context : indices)
+            perIndexes.put(context, initializeGroup(context, discovered.perIndexBuild(context.getIndexName())));
     }
 
-    public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(SSTableReader sstable)
+    private Set<IndexComponentType> expectedComponentsForVersion(Version version, @Nullable IndexContext context)
     {
-        return version.onDiskFormat().newPrimaryKeyMapFactory(this, sstable);
+        return context == null
+               ? version.onDiskFormat().perSSTableComponentTypes()
+               : version.onDiskFormat().perIndexComponentTypes(context);
     }
 
-    public SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index)
+    private IndexComponentsImpl initializeGroup(@Nullable IndexContext context, @Nullable ComponentsBuildId buildId)
     {
-        return version.onDiskFormat().newSSTableIndex(sstableContext, index);
+        IndexComponentsImpl components;
+        if (buildId == null)
+        {
+            // Means there isn't a complete build for this context. We add some empty "group" as a marker.
+            components = createEmptyGroup(context);
+        }
+        else
+        {
+            components = new IndexComponentsImpl(context, buildId);
+            var expectedTypes = expectedComponentsForVersion(buildId.version(), context);
+            // Note that the "expected types" are actually a superset of the components we may have. In particular,
+            // when a particular index has no data for a particular sstable, the relevant components only have the
+            // metadata and completion marker components. So we check what exists.
+            expectedTypes.forEach(components::addIfExists);
+            components.sealed = true;
+
+            // We'll still track the group if it is incomplete because that's what discovery gave us, and all code know
+            // how to handle those, but this will mean some index won't be queriable because of this. Also, we'll only
+            // have incomplete groups if either 1) a build failed mid-way, 2) we detected some corrupted components and
+            // deleting the completion marker, or 3) we've lost the file, and all of those should be rare, so having
+            // a warning here feels appropriate.
+            if (!components.isComplete())
+            {
+                logger.warn("Discovered group of {} for SSTable {} has no completion marker and cannot be used. This will lead to some indexes not being queriable",
+                             context == null ? "per-SSTable SAI components" : "per-index SAI components of " + context.getIndexName(), descriptor);
+            }
+        }
+        return components;
     }
 
-    public PerSSTableIndexWriter newPerSSTableIndexWriter() throws IOException
+    private IndexComponentsImpl createEmptyGroup(@Nullable IndexContext context)
     {
-        return version.onDiskFormat().newPerSSTableIndexWriter(this);
+        return new IndexComponentsImpl(context, EMPTY_GROUP_MARKER);
     }
 
-    public PerColumnIndexWriter newPerColumnIndexWriter(StorageAttachedIndex index,
-                                                        LifecycleNewTracker tracker,
-                                                        RowMapping rowMapping)
+    /**
+     * The set of components _expected_ to be written for a newly flushed sstable given the provided set of indices.
+     * This includes both per-sstable and per-index components.
+     * <p>
+     * Please note that the final sstable may not contain all of these components, as some may be empty or not written
+     * due to the specific of the flush, but this should be a superset of the components written.
+     */
+    public static Set<Component> componentsForNewlyFlushedSSTable(Collection<StorageAttachedIndex> indices)
     {
-        return version.onDiskFormat().newPerColumnIndexWriter(index, this, tracker, rowMapping);
+        ComponentsBuildId buildId = ComponentsBuildId.forNewSSTable();
+        Set<Component> components = new HashSet<>();
+        for (IndexComponentType component : buildId.version().onDiskFormat().perSSTableComponentTypes())
+            components.add(customComponentFor(buildId, component, null));
+
+        for (StorageAttachedIndex index : indices)
+            addPerIndexComponentsForNewlyFlushedSSTable(components, buildId, index.getIndexContext());
+        return components;
     }
 
-    public boolean isPerSSTableIndexBuildComplete()
+    /**
+     * The set of per-index components _expected_ to be written for a newly flushed sstable for the provided index.
+     * <p>
+     * This is a subset of {@link #componentsForNewlyFlushedSSTable(Collection)} and has the same caveats.
+     */
+    public static Set<Component> perIndexComponentsForNewlyFlushedSSTable(IndexContext context)
     {
-        return version.onDiskFormat().isPerSSTableIndexBuildComplete(this);
+        return addPerIndexComponentsForNewlyFlushedSSTable(new HashSet<>(), ComponentsBuildId.forNewSSTable(), context);
     }
 
-    public boolean isPerColumnIndexBuildComplete(IndexIdentifier indexIdentifier)
+    private static Set<Component> addPerIndexComponentsForNewlyFlushedSSTable(Set<Component> addTo, ComponentsBuildId buildId, IndexContext context)
     {
-        return version.onDiskFormat().isPerColumnIndexBuildComplete(this, indexIdentifier);
+        for (IndexComponentType component : buildId.version().onDiskFormat().perIndexComponentTypes(context))
+            addTo.add(customComponentFor(buildId, component, context));
+        return addTo;
     }
 
-    public boolean hasComponent(IndexComponent indexComponent)
+    private static Component customComponentFor(ComponentsBuildId buildId, IndexComponentType componentType, @Nullable IndexContext context)
     {
-        return fileFor(indexComponent).exists();
+        return new Component(SSTableFormat.Components.Types.CUSTOM, buildId.formatAsComponent(componentType, context));
     }
 
-    public boolean hasComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
+    /**
+     * Given the indexes for the sstable this is a descriptor for, reload from disk to check if newer components are
+     * available.
+     * <p>
+     * This method is generally <b>not</b> safe to call concurrently with the other methods that modify the state
+     * of {@link IndexDescriptor}, which are {@link #newPerSSTableComponentsForWrite()} and
+     * {@link #newPerIndexComponentsForWrite(IndexContext)}. This method is in fact meant for tiered storage use-cases
+     * where (post-flush) index building is done on separate dedicated services, and this method allows to reload the
+     * result of such external services once it is made available locally.
+     *
+     * @param sstable the sstable reader to reload index components
+     * @param indices The set of indices to should part of the reloaded descriptor.
+     * @return this descriptor, for chaining purpose.
+     */
+    public IndexDescriptor reload(SSTableReader sstable, Set<IndexContext> indices)
     {
-        return fileFor(indexComponent, indexIdentifier).exists();
+        Preconditions.checkArgument(sstable.getDescriptor().equals(this.descriptor));
+        SSTableIndexComponentsState discovered = IndexComponentDiscovery.instance().discoverComponents(sstable);
+
+        // We want to make sure the descriptor only has data for the provided `indices` on reload, so we remove any
+        // index data that is not in the ones provided. This essentially make sure we don't hold up memory for
+        // dropped indexes.
+        for (IndexContext context : new HashSet<>(perIndexes.keySet()))
+        {
+            if (!indices.contains(context))
+                perIndexes.remove(context);
+        }
+
+        // Then reload data.
+        initialize(indices, discovered);
+        return this;
     }
 
-    public File fileFor(IndexComponent indexComponent)
+    public IndexComponents.ForRead perSSTableComponents()
     {
-        return createFile(indexComponent, null);
+        return perSSTable;
     }
 
-    public File fileFor(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
+    public IndexComponents.ForRead perIndexComponents(IndexContext context)
     {
-        return createFile(indexComponent, indexIdentifier);
+        var perIndex = perIndexes.get(context);
+        return perIndex == null ? createEmptyGroup(context) : perIndex;
     }
 
-    public boolean isIndexEmpty(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
+    public Set<IndexContext> includedIndexes()
     {
-        // The index is empty if the index build completed successfully in that both
-        // a GROUP_COMPLETION_MARKER companent and a COLUMN_COMPLETION_MARKER exist for
-        // the index and the number of per-index components is 1 indicating that only the
-        // COLUMN_COMPLETION_MARKER exists for the index, as this is the only file that
-        // will be written if the index is empty
-        return isPerColumnIndexBuildComplete(indexIdentifier) && numberOfPerIndexComponents(indexTermType, indexIdentifier) == 1;
+        return Collections.unmodifiableSet(perIndexes.keySet());
     }
 
-    public void createComponentOnDisk(IndexComponent component) throws IOException
+    public IndexComponents.ForWrite newPerSSTableComponentsForWrite()
     {
-        Files.touch(fileFor(component).toJavaIOFile());
+        return newComponentsForWrite(null, perSSTable);
     }
 
-    public void createComponentOnDisk(IndexComponent component, IndexIdentifier indexIdentifier) throws IOException
+    public IndexComponents.ForWrite newPerIndexComponentsForWrite(IndexContext context)
     {
-        Files.touch(fileFor(component, indexIdentifier).toJavaIOFile());
+        return newComponentsForWrite(context, perIndexes.get(context));
     }
 
-    public IndexInput openPerSSTableInput(IndexComponent indexComponent)
+    private IndexComponents.ForWrite newComponentsForWrite(@Nullable IndexContext context, IndexComponentsImpl currentComponents)
     {
-        File file = fileFor(indexComponent);
-        if (logger.isTraceEnabled())
-            logger.trace(logMessage("Opening blocking index input for file {} ({})"),
-                         file,
-                         FBUtilities.prettyPrintMemory(file.length()));
-
-        return IndexFileUtils.instance.openBlockingInput(file);
+        var currentBuildId = currentComponents == null ? null : currentComponents.buildId;
+        return new IndexComponentsImpl(context, ComponentsBuildId.forNewBuild(currentBuildId, candidateId -> {
+            // This checks that there is no existing files on disk we would overwrite by using `candidateId` for our
+            // new build.
+            IndexComponentsImpl candidate = new IndexComponentsImpl(context, candidateId);
+            boolean isUsable = candidate.expectedComponentsForVersion().stream().noneMatch(candidate::componentExistsOnDisk);
+            if (!isUsable)
+            {
+                noSpamLogger.warn(logMessage("Wanted to use generation {} for new build of {} SAI components of {}, but found some existing components on disk for that generation (maybe leftover from an incomplete/corrupted build?); trying next generation"),
+                                  candidateId.generation(),
+                                  context == null ? "per-SSTable" : "per-index",
+                                  descriptor);
+            }
+            return isUsable;
+        }));
     }
 
-    public IndexInput openPerIndexInput(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
+    /**
+     * Returns true if the per-column index components of the provided sstable have been built and are valid.
+     *
+     * @param sstable The sstable to check
+     * @param context The {@link IndexContext} for the index
+     * @return true if the per-column index components have been built and are complete
+     */
+    public static boolean isIndexBuildCompleteOnDisk(SSTableReader sstable, IndexContext context)
     {
-        final File file = fileFor(indexComponent, indexIdentifier);
-        if (logger.isTraceEnabled())
-            logger.trace(logMessage("Opening blocking index input for file {} ({})"),
-                         file,
-                         FBUtilities.prettyPrintMemory(file.length()));
-
-        return IndexFileUtils.instance.openBlockingInput(file);
+        IndexDescriptor descriptor = IndexDescriptor.load(sstable, Set.of(context));
+        return descriptor.perSSTableComponents().isComplete()
+               && descriptor.perIndexComponents(context).isComplete();
     }
 
-    public IndexOutputWriter openPerSSTableOutput(IndexComponent component) throws IOException
+    public boolean isIndexEmpty(IndexContext context)
     {
-        return openPerSSTableOutput(component, false);
+        return perSSTableComponents().isComplete() && perIndexComponents(context).isEmpty();
     }
 
-    public IndexOutputWriter openPerSSTableOutput(IndexComponent component, boolean append) throws IOException
+    @Override
+    public int hashCode()
     {
-        final File file = fileFor(component);
-
-        if (logger.isTraceEnabled())
-            logger.trace(logMessage("Creating SSTable attached index output for component {} on file {}..."),
-                         component,
-                         file);
+        return Objects.hash(descriptor, perSSTableComponents().version());
+    }
 
-        IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file);
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        IndexDescriptor other = (IndexDescriptor)o;
+        return Objects.equals(descriptor, other.descriptor) &&
+               Objects.equals(perSSTableComponents().version(), other.perSSTableComponents().version());
+    }
 
-        if (append)
-        {
-            writer.skipBytes(file.length());
-        }
+    @Override
+    public String toString()
+    {
+        return descriptor.toString() + "-SAI";
+    }
 
-        return writer;
+    public String logMessage(String message)
+    {
+        // Index names are unique only within a keyspace.
+        return String.format("[%s.%s.*] %s",
+                             descriptor.ksname,
+                             descriptor.cfname,
+                             message);
     }
 
-    public IndexOutputWriter openPerIndexOutput(IndexComponent indexComponent, IndexIdentifier indexIdentifier) throws IOException
+    private static void deleteComponentFile(File file)
     {
-        return openPerIndexOutput(indexComponent, indexIdentifier, false);
+        logger.debug("Deleting storage attached index component file {}", file);
+        try
+        {
+            IOUtils.deleteFilesIfExist(file.toPath());
+        }
+        catch (IOException e)
+        {
+            logger.warn("Unable to delete storage attached index component file {} due to {}.", file, e.getMessage(), e);
+        }
     }
 
-    public IndexOutputWriter openPerIndexOutput(IndexComponent component, IndexIdentifier indexIdentifier, boolean append) throws IOException
+    private class IndexComponentsImpl implements IndexComponents.ForWrite
     {
-        final File file = fileFor(component, indexIdentifier);
+        private final @Nullable IndexContext context;
+        private final ComponentsBuildId buildId;
 
-        if (logger.isTraceEnabled())
-            logger.trace(logMessage("Creating sstable attached index output for component {} on file {}..."), component, file);
+        private final Map<IndexComponentType, IndexComponentImpl> components = new EnumMap<>(IndexComponentType.class);
 
-        IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file);
+        // Mark groups that are being read/have been fully written, and thus should not have new components added.
+        // This is just to catch errors where we'd try to add a component after the completion marker was written.
+        private volatile boolean sealed;
 
-        if (append)
+        private IndexComponentsImpl(@Nullable IndexContext context, ComponentsBuildId buildId)
         {
-            writer.skipBytes(file.length());
+            this.context = context;
+            this.buildId = buildId;
         }
 
-        return writer;
-    }
-
-    public FileHandle createPerSSTableFileHandle(IndexComponent indexComponent, Throwables.DiscreteAction<?> cleanup)
-    {
-        try
+        private boolean componentExistsOnDisk(IndexComponentType component)
         {
-            final File file = fileFor(indexComponent);
-
-            if (logger.isTraceEnabled())
-                logger.trace(logMessage("Opening file handle for {} ({})"), file, FBUtilities.prettyPrintMemory(file.length()));
+            return new IndexComponentImpl(component).file().exists();
+        }
 
-            return new FileHandle.Builder(file).mmapped(true).complete();
+        @Override
+        public Descriptor descriptor()
+        {
+            return descriptor;
         }
-        catch (Throwable t)
+
+        @Override
+        public IndexDescriptor indexDescriptor()
         {
-            throw handleFileHandleCleanup(t, cleanup);
+            return IndexDescriptor.this;
         }
-    }
 
-    public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
-    {
-        return createPerIndexFileHandle(indexComponent, indexIdentifier, null);
-    }
+        @Nullable
+        @Override
+        public IndexContext context()
+        {
+            return context;
+        }
 
-    public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexIdentifier indexIdentifier, Throwables.DiscreteAction<?> cleanup)
-    {
-        try
+        @Override
+        public ComponentsBuildId buildId()
         {
-            final File file = fileFor(indexComponent, indexIdentifier);
+            return buildId;
+        }
 
-            if (logger.isTraceEnabled())
-                logger.trace(logMessage("Opening file handle for {} ({})"), file, FBUtilities.prettyPrintMemory(file.length()));
+        @Override
+        public boolean has(IndexComponentType component)
+        {
+            return components.containsKey(component);
+        }
 
-            return new FileHandle.Builder(file).mmapped(true).complete();
+        @Override
+        public boolean isEmpty()
+        {
+            return isComplete() && components.size() == 1;
         }
-        catch (Throwable t)
+
+        @Override
+        public Collection<IndexComponent.ForRead> all()
         {
-            throw handleFileHandleCleanup(t, cleanup);
+            return Collections.unmodifiableCollection(components.values());
         }
-    }
 
-    private RuntimeException handleFileHandleCleanup(Throwable t, Throwables.DiscreteAction<?> cleanup)
-    {
-        if (cleanup != null)
+        @Override
+        public boolean validateComponents(SSTable sstable, Tracker tracker, boolean validateChecksum, boolean rethrow)
         {
-            try
-            {
-                cleanup.perform();
-            }
-            catch (Exception e)
+            if (isEmpty())
+                return true;
+
+            boolean isValid = true;
+            for (IndexComponentType expected : expectedComponentsForVersion())
             {
-                return Throwables.unchecked(Throwables.merge(t, e));
+                var component = components.get(expected);
+                if (component == null)
+                {
+                    logger.warn(logMessage("Missing index component {} from SSTable {}"), expected, descriptor);
+                    isValid = false;
+                }
+                else
+                {
+                    try
+                    {
+                        onDiskFormat().validateIndexComponent(component, validateChecksum);
+                    }
+                    catch (UncheckedIOException e)
+                    {
+                        logger.warn(logMessage("Invalid/corrupted component {} for SSTable {}"), expected, descriptor);
+
+                        if (rethrow)
+                            throw e;
+
+                        if (CassandraRelevantProperties.DELETE_CORRUPT_SAI_COMPONENTS.getBoolean())
+                        {
+                            // We delete the corrupted file. Yes, this may break ongoing reads to that component, but
+                            // if something is wrong with the file, we're rather fail loudly from that point on than
+                            // risking reading and returning corrupted data.
+                            deleteComponentFile(component.file());
+                            // Note that invalidation will also delete the completion marker
+                        }
+                        else
+                        {
+                            logger.debug("Leaving believed-corrupt component {} of SSTable {} in place because {} is false",
+                                         expected, descriptor, CassandraRelevantProperties.DELETE_CORRUPT_SAI_COMPONENTS.getKey());
+                        }
+
+                        isValid = false;
+                    }
+                }
             }
+            if (!isValid)
+                invalidate(sstable, tracker);
+            return isValid;
         }
-        return Throwables.unchecked(t);
-    }
 
-    public Set<Component> getLivePerSSTableComponents()
-    {
-        return version.onDiskFormat()
-                      .perSSTableIndexComponents(hasClustering())
-                      .stream()
-                      .filter(c -> fileFor(c).exists())
-                      .map(version::makePerSSTableComponent)
-                      .collect(Collectors.toSet());
-    }
-
-    public Set<Component> getLivePerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
-    {
-        return version.onDiskFormat()
-                      .perColumnIndexComponents(indexTermType)
-                      .stream()
-                      .filter(c -> fileFor(c, indexIdentifier).exists())
-                      .map(c -> version.makePerIndexComponent(c, indexIdentifier))
-                      .collect(Collectors.toSet());
-    }
+        private void updateParentLink(IndexComponentsImpl update)
+        {
+            if (isPerIndexGroup())
+                perIndexes.put(context, update);
+            else
+                perSSTable = update;
+        }
 
-    public long sizeOnDiskOfPerSSTableComponents()
-    {
-        return version.onDiskFormat()
-                      .perSSTableIndexComponents(hasClustering())
-                      .stream()
-                      .map(this::fileFor)
-                      .filter(File::exists)
-                      .mapToLong(File::length)
-                      .sum();
-    }
+        @Override
+        public void invalidate(SSTable sstable, Tracker tracker)
+        {
+            // This rewrite the TOC to stop listing the components, which ensures that if the node is restarted,
+            // then discovery will use an empty group for that context (like we add at the end of this method).
+            sstable.unregisterComponents(allAsCustomComponents(), tracker);
+
+            // Also delete the completion marker, to make it clear the group of components shouldn't be used anymore.
+            // Note it's comparatively safe to do so in that the marker is never accessed during reads, so we cannot
+            // break ongoing operations here.
+            var marker = components.remove(completionMarkerComponent());
+            if (marker != null)
+                deleteComponentFile(marker.file());
+
+            // Keeping legacy behavior if immutable components is disabled.
+            if (!buildId.version().useImmutableComponentFiles() && CassandraRelevantProperties.DELETE_CORRUPT_SAI_COMPONENTS.getBoolean())
+                forceDeleteAllComponents();
+
+            // We replace ourselves by an explicitly empty group in the parent.
+            updateParentLink(createEmptyGroup(context));
+        }
 
-    public long sizeOnDiskOfPerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
-    {
-        return version.onDiskFormat()
-                      .perColumnIndexComponents(indexTermType)
-                      .stream()
-                      .map(c -> fileFor(c, indexIdentifier))
-                      .filter(File::exists)
-                      .mapToLong(File::length)
-                      .sum();
-    }
+        @Override
+        public ForWrite forWrite()
+        {
+            // The difference between Reader and Writer is just to make code cleaner and make it clear when we read
+            // components from when we write/modify them. But this concrete implementatation is both in practice.
+            return this;
+        }
 
-    @VisibleForTesting
-    public long sizeOnDiskOfPerIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
-    {
-        File componentFile = fileFor(indexComponent, indexIdentifier);
-        return componentFile.exists() ? componentFile.length() : 0;
-    }
+        @Override
+        public IndexComponent.ForRead get(IndexComponentType component)
+        {
+            IndexComponentImpl info = components.get(component);
+            Preconditions.checkNotNull(info, "SSTable %s has no %s component for build %s (context: %s)", descriptor, component, buildId, context);
+            return info;
+        }
 
-    @SuppressWarnings("BooleanMethodIsAlwaysInverted")
-    public boolean validatePerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier, IndexValidation validation, boolean validateChecksum, boolean rethrow)
-    {
-        if (validation == IndexValidation.NONE)
-            return true;
+        @Override
+        public long liveSizeOnDiskInBytes()
+        {
+            return components.values().stream().map(IndexComponentImpl::file).mapToLong(File::length).sum();
+        }
 
-        logger.info(logMessage("Validating per-column index components for {} for SSTable {} using mode {}"), indexIdentifier, sstableDescriptor.toString(), validation);
+        public void addIfExists(IndexComponentType component)
+        {
+            Preconditions.checkArgument(!sealed, "Should not add components for SSTable %s at this point; the completion marker has already been written", descriptor);
+            // When a sstable doesn't have any complete group, we use a marker empty one with a generation of -1:
+            Preconditions.checkArgument(buildId != EMPTY_GROUP_MARKER, "Should not be adding component to empty components");
+            components.computeIfAbsent(component, type -> {
+                var created = new IndexComponentImpl(type);
+                return created.file().exists() ? created : null;
+            });
+        }
 
-        try
+        @Override
+        public IndexComponent.ForWrite addOrGet(IndexComponentType component)
         {
-            version.onDiskFormat().validatePerColumnIndexComponents(this, indexTermType, indexIdentifier, validation == IndexValidation.CHECKSUM && validateChecksum);
-            return true;
+            Preconditions.checkArgument(!sealed, "Should not add components for SSTable %s at this point; the completion marker has already been written", descriptor);
+            // When a sstable doesn't have any complete group, we use a marker empty one with a generation of -1:
+            Preconditions.checkArgument(buildId != EMPTY_GROUP_MARKER, "Should not be adding component to empty components");
+            return components.computeIfAbsent(component, IndexComponentImpl::new);
         }
-        catch (UncheckedIOException e)
+
+        @Override
+        public void forceDeleteAllComponents()
         {
-            if (rethrow)
-                throw e;
-            else
-                return false;
+            components.values()
+                      .stream()
+                      .map(IndexComponentImpl::file)
+                      .forEach(IndexDescriptor::deleteComponentFile);
+            components.clear();
         }
-    }
 
-    @SuppressWarnings("BooleanMethodIsAlwaysInverted")
-    public boolean validatePerSSTableComponents(IndexValidation validation, boolean validateChecksum, boolean rethrow)
-    {
-        if (validation == IndexValidation.NONE)
-            return true;
+        @Override
+        public void markComplete() throws IOException
+        {
+            addOrGet(completionMarkerComponent()).createEmpty();
+            sealed = true;
+            // Until this call, the group is not attached to the parent. This create the link.
+            updateParentLink(this);
+        }
 
-        logger.info(logMessage("Validating per-sstable index components for SSTable {} using mode {}"), sstableDescriptor.toString(), validation);
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(descriptor, context, buildId);
+        }
 
-        try
+        @Override
+        public boolean equals(Object o)
         {
-            version.onDiskFormat().validatePerSSTableIndexComponents(this, validation == IndexValidation.CHECKSUM && validateChecksum);
-            return true;
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            IndexComponentsImpl that = (IndexComponentsImpl) o;
+            return Objects.equals(descriptor, that.descriptor())
+                   && Objects.equals(context, that.context)
+                   && Objects.equals(buildId, that.buildId);
         }
-        catch (UncheckedIOException e)
+
+        @Override
+        public String toString()
         {
-            if (rethrow)
-                throw e;
-            else
-                return false;
+            return String.format("%s components for %s (%s): %s",
+                                 context == null ? "Per-SSTable" : "Per-Index",
+                                 descriptor,
+                                 buildId,
+                                 components.values());
         }
-    }
 
-    public void deletePerSSTableIndexComponents()
-    {
-        version.onDiskFormat()
-               .perSSTableIndexComponents(hasClustering())
-               .stream()
-               .map(this::fileFor)
-               .filter(File::exists)
-               .forEach(this::deleteComponent);
-    }
+        private class IndexComponentImpl implements IndexComponent.ForRead, IndexComponent.ForWrite
+        {
+            private final IndexComponentType component;
 
-    public void deleteColumnIndex(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
-    {
-        version.onDiskFormat()
-               .perColumnIndexComponents(indexTermType)
-               .stream()
-               .map(c -> fileFor(c, indexIdentifier))
-               .filter(File::exists)
-               .forEach(this::deleteComponent);
-    }
+            private volatile String filenamePart;
+            private volatile File file;
 
-    @Override
-    public int hashCode()
-    {
-        return Objects.hashCode(sstableDescriptor, version);
-    }
+            private IndexComponentImpl(IndexComponentType component)
+            {
+                this.component = component;
+            }
 
-    @Override
-    public boolean equals(Object o)
-    {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        IndexDescriptor other = (IndexDescriptor)o;
-        return Objects.equal(sstableDescriptor, other.sstableDescriptor) &&
-               Objects.equal(version, other.version);
-    }
+            @Override
+            public IndexComponentsImpl parent()
+            {
+                return IndexComponentsImpl.this;
+            }
 
-    @Override
-    public String toString()
-    {
-        return sstableDescriptor.toString() + "-SAI";
-    }
+            @Override
+            public IndexComponentType componentType()
+            {
+                return component;
+            }
 
-    public String logMessage(String message)
-    {
-        // Index names are unique only within a keyspace.
-        return String.format("[%s.%s.*] %s",
-                             sstableDescriptor.ksname,
-                             sstableDescriptor.cfname,
-                             message);
-    }
+            @Override
+            public ByteOrder byteOrder()
+            {
+                return buildId.version().onDiskFormat().byteOrderFor(component, context);
+            }
 
-    private File createFile(IndexComponent component, IndexIdentifier indexIdentifier)
-    {
-        Component customComponent = version.makePerIndexComponent(component, indexIdentifier);
-        return sstableDescriptor.fileFor(customComponent);
-    }
+            @Override
+            public String fileNamePart()
+            {
+                // Not thread-safe, but not really the end of the world if called multiple time
+                if (filenamePart == null)
+                    filenamePart = buildId.formatAsComponent(component, context);
+                return filenamePart;
+            }
 
-    private long numberOfPerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
-    {
-        return version.onDiskFormat()
-                      .perColumnIndexComponents(indexTermType)
-                      .stream()
-                      .map(c -> fileFor(c, indexIdentifier))
-                      .filter(File::exists)
-                      .count();
-    }
+            @Override
+            public Component asCustomComponent()
+            {
+                return new Component(SSTableFormat.Components.Types.CUSTOM, fileNamePart());
+            }
 
-    private void deleteComponent(File file)
-    {
-        logger.debug(logMessage("Deleting storage-attached index component file {}"), file);
-        try
-        {
-            IOUtils.deleteFilesIfExist(file.toPath());
-        }
-        catch (IOException e)
-        {
-            logger.warn(logMessage("Unable to delete storage-attached index component file {} due to {}."), file, e.getMessage(), e);
+            @Override
+            public File file()
+            {
+                // Not thread-safe, but not really the end of the world if called multiple time
+                if (file == null)
+                    file = descriptor.fileFor(asCustomComponent());
+                return file;
+            }
+
+            @Override
+            public FileHandle createFileHandle()
+            {
+                var builder = StorageProvider.instance.fileHandleBuilderFor(this);
+                var b = builder.order(byteOrder());
+                return b.complete();
+            }
+
+            @Override
+            public FileHandle createIndexBuildTimeFileHandle()
+            {
+                final FileHandle.Builder builder = StorageProvider.instance.indexBuildTimeFileHandleBuilderFor(this);
+                return builder.order(byteOrder()).complete();
+            }
+
+            @Override
+            public IndexInput openInput()
+            {
+                return IndexFileUtils.instance().openBlockingInput(createFileHandle());
+            }
+
+            @Override
+            public ChecksumIndexInput openCheckSummedInput()
+            {
+                var indexInput = openInput();
+                return checksumIndexInput(indexInput);
+            }
+
+            /**
+             * Returns a ChecksumIndexInput that reads the indexInput in the correct endianness for the context.
+             * These files were written by the Lucene {@link org.apache.lucene.store.DataOutput}. When written by
+             * Lucene 7.5, {@link org.apache.lucene.store.DataOutput} wrote the file using big endian formatting.
+             * After the upgrade to Lucene 9, the {@link org.apache.lucene.store.DataOutput} writes in little endian
+             * formatting.
+             *
+             * @param indexInput The index input to read
+             * @return A ChecksumIndexInput that reads the indexInput in the correct endianness for the context
+             */
+            private ChecksumIndexInput checksumIndexInput(IndexInput indexInput)
+            {
+                if (buildId.version() == Version.AA)
+                    return new EndiannessReverserChecksumIndexInput(indexInput, buildId.version());
+                else
+                    return new BufferedChecksumIndexInput(indexInput);
+            }
+
+            @Override
+            public IndexOutputWriter openOutput(boolean append) throws IOException
+            {
+                File file = file();
+
+                if (logger.isTraceEnabled())
+                    logger.trace(this.parent().logMessage("Creating SSTable attached index output for component {} on file {}..."),
+                                 component,
+                                 file);
+
+                return IndexFileUtils.instance().openOutput(file, byteOrder(), append, buildId.version());
+            }
+
+            @Override
+            public void createEmpty() throws IOException
+            {
+                com.google.common.io.Files.touch(file().toJavaIOFile());
+            }
+
+            @Override
+            public int hashCode()
+            {
+                return Objects.hash(this.parent(), component);
+            }
+
+            @Override
+            public boolean equals(Object o)
+            {
+                if (this == o) return true;
+                if (o == null || getClass() != o.getClass()) return false;
+                IndexComponentImpl that = (IndexComponentImpl) o;
+                return Objects.equals(this.parent(), that.parent())
+                       && component == that.component;
+            }
+
+            @Override
+            public String toString()
+            {
+                return file().toString();
+            }
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexFeatureSet.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexFeatureSet.java
new file mode 100644
index 000000000000..46df62e39728
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexFeatureSet.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.format;
+
+/**
+ * The {@code IndexFeatureSet} represents the set of features available that are available
+ * to an {@code OnDiskFormat}.
+ *
+ * The baseline features included in the V1 on-disk format are not included in the feature set.
+ * Thus, V1 on-disk format features should only be added here if support for them is dropped in
+ * a future version.
+ */
+public interface IndexFeatureSet
+{
+    /**
+     * Returns whether the index supports row-awareness. Row-awareness means that the per-sstable
+     * index supports mapping rowID -> {@code PrimaryKey} where the {@code PrimaryKey} contains both
+     * partition key and clustering information.
+     *
+     * @return true if the index supports row-awareness
+     */
+    boolean isRowAware();
+
+    /**
+     * @return true if vector index files include a checksum at the end
+     */
+    boolean hasVectorIndexChecksum();
+
+    /**
+     * @return true if index metadata contains term histograms for fast cardinality estimation
+     */
+    boolean hasTermsHistogram();
+
+    /**
+     * The {@code Accumulator} is used to accumulate the {@link IndexFeatureSet} responses from
+     * multiple sources. This will include all the SSTables included in a query and all the indexes
+     * attached to those SSTables, added using {@link Accumulator#accumulate}.
+     * <p>
+     * The feature set of the latest version denoted by {@link Version#latest()}
+     * is implicitly added, so the result feature set will include only the features supported by the
+     * latest version.
+     * <p>
+     * The {@code Accumulator} creates an {@code IndexFeatureSet} this contains the features from
+     * all the associated feature sets where {@code false} is the highest priority. This means if any
+     * on-disk format on any SSTable doesn't support a feature then that feature isn't supported
+     * by the query.
+     */
+    class Accumulator
+    {
+        boolean isRowAware = true;
+        boolean hasVectorIndexChecksum = true;
+        boolean hasTermsHistogram = true;
+        boolean complete = false;
+
+        public Accumulator()
+        {
+            accumulate(Version.latest().onDiskFormat().indexFeatureSet());
+        }
+
+        /**
+         * Add another {@code IndexFeatureSet} to the accumulation
+         *
+         * @param indexFeatureSet the feature set to accumulate
+         */
+        public void accumulate(IndexFeatureSet indexFeatureSet)
+        {
+            assert !complete : "Cannot accumulate after complete has been called";
+            if (!indexFeatureSet.isRowAware())
+                isRowAware = false;
+            if (!indexFeatureSet.hasVectorIndexChecksum())
+                hasVectorIndexChecksum = false;
+            if (!indexFeatureSet.hasTermsHistogram())
+                hasTermsHistogram = false;
+        }
+
+        /**
+         * Complete the accumulation of feature sets and return the
+         * result of the accumulation.
+         *
+         * @return an {@link IndexFeatureSet} containing the accumulated feature set
+         */
+        public IndexFeatureSet complete()
+        {
+            complete = true;
+            return new IndexFeatureSet()
+            {
+                @Override
+                public boolean isRowAware()
+                {
+                    return isRowAware;
+                }
+
+                @Override
+                public boolean hasVectorIndexChecksum()
+                {
+                    return hasVectorIndexChecksum;
+                }
+
+                @Override
+                public boolean hasTermsHistogram()
+                {
+                    return hasTermsHistogram;
+                }
+            };
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java
index 30ba3b6295b0..fe0957772c1d 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java
@@ -20,149 +20,200 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.Set;
 
+import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SSTableContext;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
-import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
+import org.apache.cassandra.index.sai.disk.PerIndexWriter;
+import org.apache.cassandra.index.sai.disk.PerSSTableWriter;
 import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.RowMapping;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.disk.SearchableIndex;
+import org.apache.cassandra.index.sai.disk.v1.IndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.memory.RowMapping;
+import org.apache.cassandra.index.sai.memory.TrieMemtableIndex;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 /**
- * An interface to the on-disk format of an index. This provides format agnostic methods
+ * An interface to the on-disk format of an index. This provides format agnostics methods
  * to read and write an on-disk format.
- * <p>
+ *
  * The methods on this interface can be logically mapped into the following groups
  * based on their method parameters:
  * <ul>
  *     <li>Methods taking no parameters. These methods return static information about the
  *     format. This can include static information about the per-sstable components</li>
- *     <li>Methods taking an {@link IndexDescriptor}. These methods interact with the on-disk components, or
- *     return objects that will interact with the on-disk components, or return information about the on-disk
- *     components. If they take an {@link IndexTermType} and/or a {@link IndexIdentifier} as well they will be
- *     interacting with per-column index files; otherwise they will be interacting with per-sstable index files</li>
- *     <li>Methods taking an {@link IndexComponent}. These methods only interact with a single index component or
- *     set of index components</li>
+ *     <li>Methods taking just an {@link IndexContext}. These methods return static information
+ *     specific to the index. This can be information relating to the type of index being used</li>
+ *     <li>Methods taking an {@link IndexDescriptor}. These methods interact with the on-disk components or
+ *     return objects that will interact with the on-disk components or return information about the on-disk
+ *     components. If they take an {@link IndexContext} as well they will be interacting with per-index files
+ *     otherwise they will be interacting with per-sstable files</li>
+ *     <li>Methods taking an {@link IndexComponentType}. These methods only interact with a single component or
+ *     set of components</li>
  *
+ * To add a new version,
+ * (1) Create a new class, e.g. VXOnDiskFormat, that extends the previous version and overrides
+ *     methods relating to the new format and functionality
+ * (2) Wire it up in Version to its version string
  * </ul>
  */
 public interface OnDiskFormat
 {
     /**
-     * Returns a {@link PrimaryKeyMap.Factory} for the SSTable
+     * Returns the {@link IndexFeatureSet} for the on-disk format
      *
-     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable
-     * @param sstable The {@link SSTableReader} associated with the {@link IndexDescriptor}
+     * @return the index feature set
      */
-    PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexDescriptor indexDescriptor, SSTableReader sstable);
+    public IndexFeatureSet indexFeatureSet();
 
     /**
-     * Create a new {@link SSTableIndex} for an on-disk index.
+     * Returns the {@link PrimaryKey.Factory} for the on-disk format
      *
-     * @param sstableContext The {@link SSTableContext} holding the per-SSTable information for the index
-     * @param index The {@link StorageAttachedIndex}
-     * @return the new {@link SSTableIndex} for the on-disk index
+     * @param comparator
+     * @return the primary key factory
      */
-    SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index);
+    public PrimaryKey.Factory newPrimaryKeyFactory(ClusteringComparator comparator);
 
     /**
-     * Create a new {@link PerSSTableIndexWriter} to write the per-SSTable on-disk components of an index.
+     * Returns a {@link PrimaryKeyMap.Factory} for the SSTable
      *
-     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable
-     * @throws IOException if the writer couldn't be created
+     * @param perSSTableComponents The concrete sstable components to use for the factory
+     * @param primaryKeyFactory The {@link PrimaryKey.Factory} corresponding to the provided {@code perSSTableComponents}.
+     * @param sstable The {@link SSTableReader} associated with the per-sstable components
+     * @return a {@link PrimaryKeyMap.Factory} for the SSTable
+     * @throws IOException
      */
-    PerSSTableIndexWriter newPerSSTableIndexWriter(IndexDescriptor indexDescriptor) throws IOException;
+    public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexComponents.ForRead perSSTableComponents, PrimaryKey.Factory primaryKeyFactory, SSTableReader sstable) throws IOException;
 
     /**
-     * Create a new {@link PerColumnIndexWriter} to write the per-column on-disk components of an index. The {@link LifecycleNewTracker}
-     * is used to determine the type of index write about to happen this will either be an
-     * {@code OperationType.FLUSH} indicating that we are about to flush a {@link org.apache.cassandra.index.sai.memory.MemtableIndex}
-     * or one of the other operation types indicating that we will be writing from an existing SSTable
+     * Create a new {@link SearchableIndex} for an on-disk index. This is held by the {@SSTableIndex}
+     * and shared between queries.
      *
-     * @param index The {@link StorageAttachedIndex} holding the current index build status
-     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable
-     * @param tracker The {@link LifecycleNewTracker} for index build operation.
-     * @param rowMapping The {@link RowMapping} that is used to map rowID to {@code PrimaryKey} during the write operation
+     * @param sstableContext The {@link SSTableContext} holding the per-SSTable information for the index
+     * @param perIndexComponents The group of per-index sstable components to use/read for the returned index (which
+     *                           also link to the underlying {@link IndexContext} for the index).
+     * @return the created {@link SearchableIndex}.
      */
-    PerColumnIndexWriter newPerColumnIndexWriter(StorageAttachedIndex index,
-                                                 IndexDescriptor indexDescriptor,
-                                                 LifecycleNewTracker tracker,
-                                                 RowMapping rowMapping);
+    public SearchableIndex newSearchableIndex(SSTableContext sstableContext, IndexComponents.ForRead perIndexComponents);
 
-    /**
-     * Returns true if the per-sstable index components have been built and are valid.
-     *
-     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
-     */
-    boolean isPerSSTableIndexBuildComplete(IndexDescriptor indexDescriptor);
+    IndexSearcher newIndexSearcher(SSTableContext sstableContext,
+                                   IndexContext indexContext,
+                                   PerIndexFiles indexFiles,
+                                   SegmentMetadata segmentMetadata) throws IOException;
 
     /**
-     * Returns true if the per-column index components have been built and are valid.
+     * Create a new writer for the per-SSTable on-disk components of an index.
      *
-     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
-     * @param indexIdentifier The {@link IndexIdentifier} for the index
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable
+     * @return The {@link PerSSTableWriter} to write the per-SSTable on-disk components
+     * @throws IOException
      */
-    boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier);
+    public PerSSTableWriter newPerSSTableWriter(IndexDescriptor indexDescriptor) throws IOException;
 
     /**
-     * Validate all the per-SSTable on-disk components and throw if a component is not valid
-     *
-     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
-     * @param checksum {@code true} if the checksum should be tested as part of the validation
+     * Create a new writer for the per-index on-disk components of an index. The {@link LifecycleNewTracker}
+     * is used to determine the type of index write about to happen this will either be an
+     * {@code OperationType.FLUSH} indicating that we are about to flush a {@link TrieMemtableIndex}
+     * or one of the other operation types indicating that we will be writing from an existing SSTable
      *
-     * @throws UncheckedIOException if there is a problem validating any on-disk component
+     * @param index           The {@link StorageAttachedIndex} holding the current index build status
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable
+     * @param tracker         The {@link LifecycleNewTracker} for index build operation.
+     * @param rowMapping      The {@link RowMapping} that is used to map rowID to {@code PrimaryKey} during the write
+     * @param keyCount
+     * @return The {@link PerIndexWriter} that will write the per-index on-disk components
      */
-    void validatePerSSTableIndexComponents(IndexDescriptor indexDescriptor, boolean checksum);
+    public PerIndexWriter newPerIndexWriter(StorageAttachedIndex index,
+                                            IndexDescriptor indexDescriptor,
+                                            LifecycleNewTracker tracker,
+                                            RowMapping rowMapping, long keyCount);
 
     /**
-     * Validate all the per-column on-disk components and throw if a component is not valid
+     * Validate the provided on-disk components (that must be for this version).
      *
-     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
-     * @param indexTermType The {@link IndexTermType} of the index
-     * @param indexIdentifier The {@link IndexIdentifier} for the index
+     * @param component The component to validate
      * @param checksum {@code true} if the checksum should be tested as part of the validation
      *
      * @throws UncheckedIOException if there is a problem validating any on-disk component
      */
-    void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier, boolean checksum);
+    void validateIndexComponent(IndexComponent.ForRead component, boolean checksum);
 
     /**
-     * Returns the set of {@link IndexComponent} for the per-SSTable part of an index.
-     * This is a complete set of components that could exist on-disk. It does not imply that the
+     * Returns the set of {@link IndexComponentType} for the per-SSTable part of an index.
+     * This is a complete set of componentstypes that could exist on-disk. It does not imply that the
      * components currently exist on-disk.
-
-     * @param hasClustering true if the SSTable forms part of a table using clustering columns
+     *
+     * @return The set of {@link IndexComponentType} for the per-SSTable index
      */
-    Set<IndexComponent> perSSTableIndexComponents(boolean hasClustering);
+    public Set<IndexComponentType> perSSTableComponentTypes();
 
     /**
-     * Returns the set of {@link IndexComponent} for the per-column part of an index.
-     * This is a complete set of components that could exist on-disk. It does not imply that the
+     * Returns the set of {@link IndexComponentType} for the per-index part of an index.
+     * This is a complete set of component types that could exist on-disk. It does not imply that the
      * components currently exist on-disk.
      *
-     * @param indexTermType the {@link IndexTermType} of the index
+     * @param indexContext The {@link IndexContext} for the index
+     * @return The set of {@link IndexComponentType} for the per-index index
      */
-    Set<IndexComponent> perColumnIndexComponents(IndexTermType indexTermType);
+    default public Set<IndexComponentType> perIndexComponentTypes(IndexContext indexContext)
+    {
+        return perIndexComponentTypes(indexContext.getValidator());
+    }
+
+    public Set<IndexComponentType> perIndexComponentTypes(AbstractType<?> validator);
 
     /**
      * Return the number of open per-SSTable files that can be open during a query.
      * This is a static indication of the files that can be held open by an index
      * for queries. It is not a dynamic calculation.
      *
-     * @param hasClustering true if the SSTable forms part of a table using clustering columns
+     * @return The number of open per-SSTable files
      */
-    int openFilesPerSSTableIndex(boolean hasClustering);
+    public int openFilesPerSSTable();
 
     /**
-     * Return the number of open per-column index files that can be open during a query.
+     * Return the number of open per-index files that can be open during a query.
      * This is a static indication of the files that can be help open by an index
      * for queries. It is not a dynamic calculation.
+     *
+     * @param indexContext The {@link IndexContext} for the index
+     * @return The number of open per-index files
      */
-    int openFilesPerColumnIndex();
+    public int openFilesPerIndex(IndexContext indexContext);
+
+    /**
+     * Return the {@link ByteOrder} for the given {@link IndexComponentType} and {@link IndexContext}.
+     *
+     * @param component - The {@link IndexComponentType} for the index
+     * @param context   - The {@link IndexContext} for the index
+     * @return The {@link ByteOrder} for the file associated with the {@link IndexComponentType}
+     */
+    public ByteOrder byteOrderFor(IndexComponentType component, IndexContext context);
+
+    /**
+     * Encode the given {@link ByteBuffer} into a {@link ByteComparable} object based on the provided {@link AbstractType}
+     * for storage in the trie index. This is used for both in memory and on disk tries. This is valid for encoding
+     * terms to be inserted, search terms, and search bounds.
+     *
+     * @return The encoded {@link ByteComparable} object
+     */
+    ByteComparable encodeForTrie(ByteBuffer input, AbstractType<?> type);
+
+    /**
+     * Inverse of {@link #encodeForTrie(ByteBuffer, AbstractType)}
+     */
+    ByteBuffer decodeFromTrie(ByteComparable value, AbstractType<?> type);
+
+
+
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/SSTableIndexComponentsState.java b/src/java/org/apache/cassandra/index/sai/disk/format/SSTableIndexComponentsState.java
new file mode 100644
index 000000000000..7f2bc76c4072
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/SSTableIndexComponentsState.java
@@ -0,0 +1,538 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+/**
+ * Represents, for a sstable, the "state" (version and generation) of the index components it is using.
+ * <p>
+ * This class essentially store, for each "group" of index components (so for the per-sstable group, and for each index),
+ * a version and generation, identifying a particular build of each group. This is used by {@link IndexComponentDiscovery}
+ * to return which concrete components should be loaded for the sstable, but as this class is immutable, it can be
+ * used to figure changes to index files between two different times (by capturing the state before, and comparing to
+ * the state after); see {@link #indexWasUpdated} as an example.
+ * <p>
+ * As this state only reference the {@link ComponentsBuildId} of the components, it does not represent whether that
+ * build is complete/valid, and as such this class does not guarantee _in general_ that the builds it returns are
+ * usable, and whether it does depend on context. But some methods may explicitly return a state with only complete
+ * groups (like {@link #of(IndexDescriptor)}).
+ */
+public class SSTableIndexComponentsState
+{
+    public static final SSTableIndexComponentsState EMPTY = new SSTableIndexComponentsState(null, Map.of());
+
+    // The state of the per-sstable group of components, if they exist.
+    private final @Nullable State perSSTableState;
+
+    // The state for every "group" of per-index components keyed by the index name.
+    private final Map<String, State> perIndexStates;
+
+    private SSTableIndexComponentsState(@Nullable State perSSTableState, Map<String, State> perIndexStates)
+    {
+        Preconditions.checkNotNull(perIndexStates);
+        this.perSSTableState = perSSTableState;
+        this.perIndexStates = Collections.unmodifiableMap(perIndexStates);
+    }
+
+    /**
+     * Extracts the current state of a particular SSTable given its descriptor.
+     * <p>
+     * Please note that this method only include "complete" component groups in the state, and thus represents the
+     * "usable" groups. In particular, if the per-sstable group is not complete, the returned state will be empty.
+     *
+     * @param descriptor the index descriptor of the sstable for which to get the component state.
+     * @return the state of the sstable's complete components.
+     */
+    public static SSTableIndexComponentsState of(IndexDescriptor descriptor)
+    {
+        var perSSTable = descriptor.perSSTableComponents();
+        // If the per-sstable part is not complete, then nothing is complete.
+        if (!perSSTable.isComplete())
+            return EMPTY;
+
+        Map<String, State> perIndexStates = new HashMap<>();
+        for (IndexContext context : descriptor.includedIndexes())
+        {
+            var perIndex = descriptor.perIndexComponents(context);
+            if (perIndex.isComplete())
+                perIndexStates.put(context.getIndexName(), State.of(perIndex));
+        }
+        return new SSTableIndexComponentsState(State.of(perSSTable), perIndexStates);
+    }
+
+    /**
+     * Extracts the current index components state of a particular SSTable.
+     * <p>
+     * This method delegates to {@link #of(IndexDescriptor)}, so see that method for additional details.
+     *
+     * @param sstable the sstable for which to get the component state.
+     * @return the state of the sstable's complete index components. If the sstable belongs to a table that is not
+     * indexed (or not by SAI), then this will be {@link #EMPTY}.
+     *
+     * @throws IllegalStateException if the {@link org.apache.cassandra.db.ColumnFamilyStore} of the sstable cannot
+     * be found for some reason (it is necessary to retrieve the underlying {@link IndexDescriptor}). This may happen
+     * if this is called by an "offline" tool.
+     */
+    public static SSTableIndexComponentsState of(SSTableReader sstable)
+    {
+        ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(sstable.metadata().id);
+        if (cfs == null)
+            throw new IllegalStateException("Cannot find the ColumnFamilyStore for the sstable " + sstable);
+
+        StorageAttachedIndexGroup saiGroup = StorageAttachedIndexGroup.getIndexGroup(cfs);
+        // If the table is not indexed (at least by SAI), fine.
+        if (saiGroup == null)
+            return SSTableIndexComponentsState.EMPTY;
+
+        return SSTableIndexComponentsState.of(saiGroup.descriptorFor(sstable));
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    /**
+     * Returns a newly created builder initialized with the data of this state.
+     */
+    public Builder unbuild()
+    {
+        Builder builder = new Builder();
+        builder.addPerSSTable(perSSTableState);
+        perIndexStates.forEach(builder::addPerIndex);
+        return builder;
+    }
+
+    public boolean isEmpty()
+    {
+        return perSSTableState == null && perIndexStates.isEmpty();
+    }
+
+    public @Nullable State perSSTable()
+    {
+        return perSSTableState;
+    }
+
+    public @Nullable State perIndex(String indexName)
+    {
+        Preconditions.checkNotNull(indexName);
+        return perIndexStates.get(indexName);
+    }
+
+    public @Nullable ComponentsBuildId perSSTableBuild()
+    {
+        return perSSTableState == null ? null : perSSTableState.buildId;
+    }
+
+    public @Nullable ComponentsBuildId perIndexBuild(String indexName)
+    {
+        Preconditions.checkNotNull(indexName);
+        var state = perIndexStates.get(indexName);
+        return state == null ? null : state.buildId;
+    }
+
+    /**
+     * Returns whether the provided index have been updated since the given state.
+     * <p>
+     * Having been "updated" for this method means that builds of the components used by the index have changed.
+     * Importantly, this is true if _either_ the per-sstable components have changed, or that of the index itself,
+     * since every index uses the per-sstable components.
+     */
+    public boolean indexWasUpdated(SSTableIndexComponentsState stateBefore, String indexName)
+    {
+        Preconditions.checkNotNull(indexName);
+        return !Objects.equals(stateBefore.perSSTableBuild(), this.perSSTableBuild())
+                || !Objects.equals(stateBefore.perIndexBuild(indexName), this.perIndexBuild(indexName));
+    }
+
+    /**
+     * The set of the names of all the indexes for which the state has a build for.
+     * <p>
+     * This does not include anything regarding the per-sstable components.
+     */
+    public Set<String> includedIndexes()
+    {
+        return perIndexStates.keySet();
+    }
+
+    /**
+     * The total size (in MB) of all the components included in this state.
+     */
+    public long totalSizeInMB()
+    {
+        long total = perSSTableState == null ? 0 : perSSTableState.sizeInMB;
+        for (State state : perIndexStates.values())
+            total += state.sizeInMB;
+        return total;
+    }
+
+    /**
+     * Returns a diff between this state and the provided one which is assumed to be an earlier version.
+     *
+     * @param before the state to compare this state with.
+     * @return the diff between the 2 states.
+     */
+    public Diff diff(SSTableIndexComponentsState before)
+    {
+        boolean perSSTableModified = !Objects.equals(before.perSSTableBuild(), this.perSSTableBuild());
+        Set<String> modifiedIndexes = this.includedIndexes()
+                                          .stream()
+                                          .filter(index -> !Objects.equals(before.perIndexBuild(index), this.perIndexBuild(index)))
+                                          .collect(Collectors.toSet());
+        Set<String> removedIndexes = before.includedIndexes()
+                                           .stream()
+                                           .filter(index -> !this.perIndexStates.containsKey(index))
+                                           .collect(Collectors.toSet());
+        return new Diff(before, this, perSSTableModified, modifiedIndexes, removedIndexes);
+    }
+
+    /**
+     * Applies the provided diff to this state, if applicable.
+     * <p>
+     * The assumption of this method is that the state it is applied to is for the same sstable that the 2 states that
+     * were used to produce the provided diff.
+     * <p>
+     * The diff will apply successfully if for anything that is modified in the provided diff, the current state is
+     * equivalent to the "before" state of the diff. If that is not the case, an {@link UnapplicableDiffException} will
+     * be thrown. But for anything that was not modified by the diff, the current state will be kept as is. Note in
+     * particular that this means that if {@code this == diff.before}, then the result will be exactly {@code diff.after},
+     * but as long as {@code this} has only modifications (compared to {@code diff.before}) that are not in {@code diff},
+     * then the diff will still apply correctly.
+     * <p>
+     * In other word, this method allows to compute the expected result of some index builds represented by the diff
+     * to the "current" state as long as said "current" state is the "before" state of the diff plus some eventual
+     * concurrent modifications, as long as those concurrent modifications do not conflict with the ones of the diff.
+     *
+     * @param diff the diff to try to apply to this state.
+     * @return the result of applying the diff to this state, if successful.
+     *
+     * @throws UnapplicableDiffException if the diff cannot be applied to this state.
+     */
+    public SSTableIndexComponentsState tryApplyDiff(Diff diff)
+    {
+        if (diff.isEmpty())
+            return this;
+
+        Builder builder = builder();
+        builder.addPerSSTable(diff.perSSTableUpdated
+                              ? diffState(diff.before.perSSTableState, diff.after.perSSTableState, this.perSSTableState, () -> "per-sstable components build")
+                              : this.perSSTableState);
+
+        // Adds anything modified to the "modified" version, but making sure the diff "applies", meaning that the
+        // "current" state is still the origin of the diff.
+        for (String modified : diff.perIndexesUpdated)
+        {
+            builder.addPerIndex(modified, diffState(diff.before.perIndex(modified), diff.after.perIndex(modified), this.perIndex(modified), () -> "index " + modified + " components build"));
+        }
+        // Then mirror all the current index that were not modified, but skipping removed ones.
+        for (String index : includedIndexes())
+        {
+            // The `perIndexesUpdated` have already been handled above. And a removed index means the index has been
+            // dropped, so even if some concurrent build on the index happened concurrently, the index is still gone.
+            if (diff.perIndexesUpdated.contains(index) || diff.perIndexesRemoved.contains(index))
+                continue;
+
+            builder.addPerIndex(index, this.perIndex(index));
+        }
+        return builder.build();
+    }
+
+    private static State diffState(State diffBefore, State diffAfter, State current, Supplier<String> what)
+    {
+        // If current is `null`, but our "before" isn't, that means the index this is a component of has been dropped
+        // since the state we use to create the diff (and for the per-sstable components, it was the only index that
+        // was dropped). We want to handle a drop that happens concurrently of some build/rebuild of the same index,
+        // because it's impossible to completly prevent it anyway, and the result is simply that the index is not there
+        // anymore.
+        if (current == null && diffBefore != null)
+            return null;
+
+        if (!(Objects.equals(diffBefore, current)))
+            throw new UnapplicableDiffException("Current " + what.get() + " expected to be " + diffBefore + ", but was " + current);
+
+        return diffAfter;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(perSSTableState, perIndexStates);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (!(obj instanceof SSTableIndexComponentsState))
+            return false;
+
+        SSTableIndexComponentsState that = (SSTableIndexComponentsState) obj;
+        return Objects.equals(this.perSSTableState, that.perSSTableState)
+               && this.perIndexStates.equals(that.perIndexStates);
+    }
+
+    @Override
+    public String toString()
+    {
+        Stream<String> perIndex = perIndexStates.entrySet()
+                                                .stream()
+                                                .map(e -> e.getKey() + ": " + e.getValue());
+        Stream<String> all = perSSTableState == null
+                             ? perIndex
+                             : Stream.concat(Stream.of("<shared>: " + perSSTableState), perIndex);
+
+        return all.collect(Collectors.joining(", ", "{", "}"));
+    }
+
+    /**
+     * Represents the "state" for one "group" of components (so either the per-sstable one, or one of the per-index ones).
+     */
+    public static class State
+    {
+        /** The "build" (version and generaton) the components. */
+        public final ComponentsBuildId buildId;
+
+        /** The total size (in MB) of the components (we use MB because this is meant to be indicative, is enough
+         * precision in practice and is more human-readable). */
+        public final long sizeInMB;
+
+        private State(ComponentsBuildId buildId, long sizeInMB)
+        {
+            Preconditions.checkNotNull(buildId);
+            this.buildId = buildId;
+            this.sizeInMB = sizeInMB;
+        }
+
+        private static State of(IndexComponents.ForRead components)
+        {
+            return new State(components.buildId(), toMB(components.liveSizeOnDiskInBytes()));
+        }
+
+        public static long toMB(long bytes)
+        {
+            if (bytes == 0)
+                return 0;
+
+            // We avoid returning 0 unless the size is truly zero to avoid making it look like the components do not
+            // exist. Mostly a detail in practice but ...
+            return Math.max(bytes / 1024 / 1024, 1);
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (!(obj instanceof State))
+                return false;
+
+            State that = (State) obj;
+            return this.buildId.equals(that.buildId) && this.sizeInMB == that.sizeInMB;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(buildId, sizeInMB);
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s (%dMB)", buildId, sizeInMB);
+        }
+    }
+
+    /**
+     * Builder for {@link SSTableIndexComponentsState} instances.
+     * <p>
+     * This should primarily be used by implementations of {@link IndexComponentDiscovery} and tests, as the rest of
+     * the code should generally not build a state manually (and instead use methods like {@link SSTableIndexComponentsState#of}).
+     */
+    public static class Builder
+    {
+        private State perSSTableState;
+        // We use a linked map to preserve order of insertions. This is not crucial, but the overhead is negligible and
+        // in the case of tests, the predictability of entry order make things _a lot_ easier/natural.
+        private final Map<String, State> perIndexStates = new LinkedHashMap<>();
+        // This make extra sure we don't reuse a builder by accident as it is not safe to do so (we pass the map
+        // directly when we build the state). If one wants to reuse a builder, it should `copy` manually first.
+        private boolean built;
+
+        public Builder addPerSSTable(Version version, int generation, long sizeInMB)
+        {
+            return addPerSSTable(ComponentsBuildId.of(version, generation), sizeInMB);
+        }
+
+        public Builder addPerSSTable(ComponentsBuildId buildId, long sizeInMB)
+        {
+            return addPerSSTable(new State(buildId, sizeInMB));
+        }
+
+        public Builder addPerSSTable(State state)
+        {
+            Preconditions.checkState(!built, "Builder has already been used");
+            this.perSSTableState = state;
+            return this;
+        }
+
+        public Builder addPerIndex(String name, Version version, int generation, long sizeInMB)
+        {
+            return addPerIndex(name, ComponentsBuildId.of(version, generation), sizeInMB);
+        }
+
+        public Builder addPerIndex(String name, ComponentsBuildId buildId, long sizeInMB)
+        {
+            return addPerIndex(name, new State(buildId, sizeInMB));
+        }
+
+        public Builder addPerIndex(String name, State state)
+        {
+            Preconditions.checkState(!built, "Builder has already been used");
+            Preconditions.checkNotNull(name);
+            if (state != null)
+                perIndexStates.put(name, state);
+            return this;
+        }
+
+        public Builder removePerSSTable()
+        {
+            Preconditions.checkState(!built, "Builder has already been used");
+            perSSTableState = null;
+            return this;
+        }
+
+        public Builder removePerIndex(String name)
+        {
+            Preconditions.checkState(!built, "Builder has already been used");
+            Preconditions.checkNotNull(name);
+            perIndexStates.remove(name);
+            return this;
+        }
+
+        public Builder copy()
+        {
+            Builder copy = new Builder();
+            copy.perSSTableState = perSSTableState;
+            copy.perIndexStates.putAll(perIndexStates);
+            return copy;
+        }
+
+        public SSTableIndexComponentsState build()
+        {
+            built = true;
+            return new SSTableIndexComponentsState(perSSTableState, perIndexStates);
+        }
+    }
+
+    /**
+     * Represents the difference between two {@link SSTableIndexComponentsState} instances that are assumed snapshots
+     * of the same sstable at 2 different times.
+     */
+    public static class Diff
+    {
+        /** Older of the 2 states compared in this diff. */
+        public final SSTableIndexComponentsState before;
+        /** Newer of the 2 states compared in this diff. */
+        public final SSTableIndexComponentsState after;
+        /** Whether the per-sstable components were updated between the 2 states. */
+        public final boolean perSSTableUpdated;
+        /** Which per-index components were updated between the 2 states. */
+        public final Set<String> perIndexesUpdated;
+        /** Which per-index components were removed (where in {@link #before}) but not {@link #after}. */
+        public final Set<String> perIndexesRemoved;
+
+        private Diff(SSTableIndexComponentsState before, SSTableIndexComponentsState after, boolean perSSTableUpdated, Set<String> perIndexesUpdated, Set<String> perIndexesRemoved)
+        {
+            this.before = before;
+            this.after = after;
+            this.perSSTableUpdated = perSSTableUpdated;
+            this.perIndexesUpdated = Collections.unmodifiableSet(perIndexesUpdated);
+            this.perIndexesRemoved = Collections.unmodifiableSet(perIndexesRemoved);
+        }
+
+        /**
+         * Whether this diff is empty, meaning that no changes were detected between the 2 states.
+         */
+        public boolean isEmpty()
+        {
+            return !perSSTableUpdated && perIndexesUpdated.isEmpty() && perIndexesRemoved.isEmpty();
+        }
+
+        /**
+         * Whether the operation that created this diff (meaning, the operation(s) that happened on {@link #before}
+         * to create {@link #after}) left some "unused" components, meaning that new components (new version or
+         * generation) were created where previous one existed.
+         */
+        public boolean createsUnusedComponents()
+        {
+            // Removing any components left them "unused".
+            if (!perIndexesRemoved.isEmpty())
+                return true;
+
+            return (perSSTableUpdated && before.perSSTableState != null)
+                || perIndexesUpdated.stream().anyMatch(index -> before.perIndex(index) != null);
+        }
+
+        @Override
+        public String toString()
+        {
+            if (isEmpty())
+                return String.format("%s (no diff)", before);
+
+            List<String> updates = new ArrayList<>();
+            if (perSSTableUpdated)
+            {
+                if (after.perSSTableState == null)
+                    updates.add("-<shared>");
+                else
+                    updates.add("+<shared>");
+            }
+            for (String updated : perIndexesUpdated)
+                updates.add('+' + updated);
+            for (String removed : perIndexesRemoved)
+                updates.add('-' + removed);
+            return String.format("%s -> %s (%s)", before, after, String.join(" ", updates));
+        }
+    }
+
+    public static class UnapplicableDiffException extends RuntimeException
+    {
+        public UnapplicableDiffException(String message)
+        {
+            super(message);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/Version.java b/src/java/org/apache/cassandra/index/sai/disk/format/Version.java
index a536e2e91fbe..19d3d468e4c2 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/format/Version.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/Version.java
@@ -17,20 +17,30 @@
  */
 package org.apache.cassandra.index.sai.disk.format;
 
-import java.util.Comparator;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
-
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 
 import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 
 import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.index.sai.disk.v2.V2OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v4.V4OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v5.V5OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v6.V6OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v7.V7OnDiskFormat;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static com.google.common.base.Preconditions.checkArgument;
 
 /**
  * Format version of indexing component, denoted as [major][minor]. Same forward-compatibility rules apply as to
@@ -38,22 +48,34 @@
  */
 public class Version implements Comparable<Version>
 {
-    public static final String SAI_DESCRIPTOR = "SAI";
-    public static final String SAI_SEPARATOR = "+";
-
-    // Current version
-    public static final Version AA = new Version("aa", V1OnDiskFormat.instance, (c, i) -> defaultFileNameFormat(c, i, "aa"));
+    // 6.8 formats
+    public static final Version AA = new Version("aa", V1OnDiskFormat.instance, Version::aaFileNameFormat);
+    // Stargazer
+    public static final Version BA = new Version("ba", V2OnDiskFormat.instance, (c, i, g) -> stargazerFileNameFormat(c, i, g, "ba"));
+    // Converged Cassandra with JVector
+    public static final Version CA = new Version("ca", V3OnDiskFormat.instance, (c, i, g) -> stargazerFileNameFormat(c, i, g, "ca"));
+    // NOTE: use DB to prevent collisions with upstream file formats
+    // Encode trie entries using their AbstractType to ensure trie entries are sorted for range queries and are prefix free.
+    public static final Version DB = new Version("db", V4OnDiskFormat.instance, (c, i, g) -> stargazerFileNameFormat(c, i, g, "db"));
+    // revamps vector postings lists to cause fewer reads from disk
+    public static final Version DC = new Version("dc", V5OnDiskFormat.instance, (c, i, g) -> stargazerFileNameFormat(c, i, g, "dc"));
+    // histograms in index metadata
+    public static final Version EB = new Version("eb", V6OnDiskFormat.instance, (c, i, g) -> stargazerFileNameFormat(c, i, g, "eb"));
+    // term frequencies index component
+    public static final Version EC = new Version("ec", V7OnDiskFormat.instance, (c, i, g) -> stargazerFileNameFormat(c, i, g, "ec"));
 
-    // These should be added in reverse order so that the latest version is used first. Version matching tests
-    // are more likely to match the latest version, so we want to test that one first.
-    public static final SortedSet<Version> ALL = new TreeSet<>(Comparator.reverseOrder()) {{
-        add(AA);
-    }};
+    // These are in reverse-chronological order so that the latest version is first. Version matching tests
+    // are more likely to match the latest version so we want to test that one first.
+    public static final List<Version> ALL = Lists.newArrayList(EC, EB, DC, DB, CA, BA, AA);
 
     public static final Version EARLIEST = AA;
+    public static final Version VECTOR_EARLIEST = BA;
     // The latest version can be configured to be an earlier version to support partial upgrades that don't
-    // write newer versions of the on-disk formats.
-    public static final Version LATEST = CassandraRelevantProperties.SAI_LATEST_VERSION.convert(Version::parse);
+    // write newer versions of the on-disk formats. This is volatile rather than final so that tests may
+    // use reflection to change it and safely publish across threads.
+    private static volatile Version LATEST = parse(CassandraRelevantProperties.SAI_LATEST_VERSION.getString());
+
+    private static final Pattern GENERATION_PATTERN = Pattern.compile("\\d+");
 
     private final String version;
     private final OnDiskFormat onDiskFormat;
@@ -66,19 +88,63 @@ private Version(String version, OnDiskFormat onDiskFormat, FileNameFormatter fil
         this.fileNameFormatter = fileNameFormatter;
     }
 
-    public static Version parse(String versionString)
+    public static Version parse(String input)
     {
-        for (Version version : ALL)
-            if (version.version.equals(versionString))
-                return version;
-        throw new IllegalArgumentException("The version string " + versionString + " does not represent a valid SAI version. " +
-                                           "It should be one of " + ALL.stream().map(Version::toString).collect(Collectors.joining(", ")));
+        checkArgument(input != null);
+        checkArgument(input.length() == 2);
+        for (var v : ALL)
+        {
+            if (input.equals(v.version))
+                return v;
+        }
+        throw new IllegalArgumentException("Unrecognized SAI version string " + input);
     }
 
-    @Override
-    public int compareTo(Version other)
+    public static Version latest()
+    {
+        return LATEST;
+    }
+
+    /**
+     * Calculates the maximum allowed length for SAI index names to ensure generated filenames
+     * do not exceed the system's filename length limit (defined in {@link SchemaConstants#FILENAME_LENGTH}).
+     * This accounts for all additional components in the filename.
+     */
+    public static int calculateIndexNameAllowedLength()
+    {
+        int addedLength = getAddedLengthFromDescriptorAndVersion();
+        assert addedLength < SchemaConstants.FILENAME_LENGTH;
+        return SchemaConstants.FILENAME_LENGTH - addedLength;
+    }
+
+    /**
+     * Calculates the length of the added prefixes and suffixes from Descriptor constructor
+     * and {@link Version#stargazerFileNameFormat}.
+     *
+     * @return the length of the added prefixes and suffixes
+     */
+    private static int getAddedLengthFromDescriptorAndVersion()
     {
-        return version.compareTo(other.version);
+        // Prefixes and suffixes constructed by Version.stargazerFileNameFormat
+        int versionNameLength = latest().toString().length();
+        // room for up to 999 generations
+        int generationLength = 3 + SAI_SEPARATOR.length();
+        int addedLength = SAI_DESCRIPTOR.length()
+                          + versionNameLength
+                          + generationLength
+                          + IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS.representation.length()
+                          + SAI_SEPARATOR.length() * 3
+                          + EXTENSION.length();
+
+        // Prefixes from Descriptor constructor
+        int separatorLength = 1;
+        int indexVersionLength = 2;
+        int tableIdLength = 28;
+        addedLength += indexVersionLength
+                       + BtiFormat.NAME.length()
+                       + tableIdLength
+                       + separatorLength * 3;
+        return addedLength;
     }
 
     @Override
@@ -92,7 +158,7 @@ public boolean equals(Object o)
     {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        Version other = (Version)o;
+        Version other = (Version) o;
         return Objects.equal(version, other.version);
     }
 
@@ -112,46 +178,181 @@ public OnDiskFormat onDiskFormat()
         return onDiskFormat;
     }
 
-    public Component makePerSSTableComponent(IndexComponent indexComponent)
+    public FileNameFormatter fileNameFormatter()
     {
-        return indexComponent.type.createComponent(fileNameFormatter.format(indexComponent, null));
+        return fileNameFormatter;
     }
 
-    public Component makePerIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
+    public boolean useImmutableComponentFiles()
     {
-        return indexComponent.type.createComponent(fileNameFormatter.format(indexComponent, indexIdentifier));
+        // We only enable "immutable" components (meaning that new build don't delete or replace old versions) if the
+        // flag is set and even then, only starting at version CA. There is no reason to need it for older versions,
+        // and if older versions are involved, it means we likely want backward compatible behaviour.
+        return CassandraRelevantProperties.IMMUTABLE_SAI_COMPONENTS.getBoolean() && onOrAfter(Version.CA);
     }
 
-    public FileNameFormatter fileNameFormatter()
+    @Override
+    public int compareTo(Version other)
     {
-        return fileNameFormatter;
+        return this.version.compareTo(other.version);
     }
 
     public interface FileNameFormatter
     {
-        String format(IndexComponent indexComponent, IndexIdentifier indexIdentifier);
+        /**
+         * Format filename for given index component, context and generation.  Only the "component" part of the
+         * filename is returned (so the suffix of the full filename), not a full path.
+         */
+        default String format(IndexComponentType indexComponentType, IndexContext indexContext, int generation)
+        {
+            return format(indexComponentType, indexContext == null ? null : indexContext.getIndexName(), generation);
+        }
+
+        /**
+         * Format filename for given index component, index and generation.  Only the "component" part of the
+         * filename is returned (so the suffix of the full filename), not a full path.
+         *
+         * @param indexComponentType the type of the index component.
+         * @param indexName          the name of the index, or {@code null} for a per-sstable component.
+         * @param generation         the generation of the build of the component.
+         */
+        String format(IndexComponentType indexComponentType, @Nullable String indexName, int generation);
     }
 
     /**
-     * SAI default filename formatter. This is the current SAI on-disk filename format
-     * <p>
-     * Format: {@code <sstable descriptor>-SAI+<version>(+<index name>)+<component name>.db}
-     * Note: The index name is excluded for per-SSTable index files that are shared
-     * across all the per-column indexes for the SSTable.
+     * Try to parse the provided file name as a SAI component file name.
+     *
+     * @param filename the file name to try to parse.
+     * @return the information parsed from the provided file name if it can be sucessfully parsed, or an empty optional
+     * if the file name is not recognized as a SAI component file name for a supported version.
      */
-    private static String defaultFileNameFormat(IndexComponent indexComponent,
-                                                @Nullable IndexIdentifier indexIdentifier,
-                                                String version)
+    public static Optional<ParsedFileName> tryParseFileName(String filename)
+    {
+        if (!filename.endsWith(EXTENSION))
+            return Optional.empty();
+
+        // For flexibility, we handle both "full" filename, of the form "<descriptor>-SAI+....db", or just the component
+        // part, that is "SAI+....db". In the former, the following `lastIndexOf` will match, and we'll set
+        // `startOfComponent` at the begining of "SAI", and in the later it will not match and return -1, which, with
+        // the +1 will also be set at the begining of "SAI".
+        int startOfComponent = filename.lastIndexOf('-') + 1;
+
+        String componentStr = filename.substring(startOfComponent);
+        if (componentStr.startsWith("SAI_"))
+            return tryParseAAFileName(componentStr);
+        else if (componentStr.startsWith("SAI" + SAI_SEPARATOR))
+            return tryParseStargazerFileName(componentStr);
+        else
+            return Optional.empty();
+    }
+
+    public static class ParsedFileName
+    {
+        public final ComponentsBuildId buildId;
+        public final IndexComponentType component;
+        public final @Nullable String indexName;
+
+        private ParsedFileName(ComponentsBuildId buildId, IndexComponentType component, @Nullable String indexName)
+        {
+            this.buildId = buildId;
+            this.component = component;
+            this.indexName = indexName;
+        }
+    }
+
+    //
+    // Version.AA filename formatter. This is the old DSE 6.8 SAI on-disk filename format
+    //
+    // Format: <sstable descriptor>-SAI(_<index name>)_<component name>.db
+    //
+    private static final String VERSION_AA_PER_SSTABLE_FORMAT = "SAI_%s.db";
+    private static final String VERSION_AA_PER_INDEX_FORMAT = "SAI_%s_%s.db";
+
+    private static String aaFileNameFormat(IndexComponentType indexComponentType, @Nullable String indexName, int generation)
+    {
+        Preconditions.checkArgument(generation == 0, "Generation is not supported for AA version");
+        StringBuilder stringBuilder = new StringBuilder();
+
+        stringBuilder.append(indexName == null ? String.format(VERSION_AA_PER_SSTABLE_FORMAT, indexComponentType.representation)
+                                               : String.format(VERSION_AA_PER_INDEX_FORMAT, indexName, indexComponentType.representation));
+
+        return stringBuilder.toString();
+    }
+
+    private static Optional<ParsedFileName> tryParseAAFileName(String componentStr)
+    {
+        int lastSepIdx = componentStr.lastIndexOf('_');
+        if (lastSepIdx == -1)
+            return Optional.empty();
+
+        String indexComponentStr = componentStr.substring(lastSepIdx + 1, componentStr.length() - 3);
+        IndexComponentType indexComponentType = IndexComponentType.fromRepresentation(indexComponentStr);
+
+        String indexName = null;
+        int firstSepIdx = componentStr.indexOf('_');
+        if (firstSepIdx != -1 && firstSepIdx != lastSepIdx)
+            indexName = componentStr.substring(firstSepIdx + 1, lastSepIdx);
+
+        return Optional.of(new ParsedFileName(ComponentsBuildId.of(AA, 0), indexComponentType, indexName));
+    }
+
+    //
+    // Stargazer filename formatter. This is the current SAI on-disk filename format
+    //
+    // Format: <sstable descriptor>-SAI+<version>(+<generation>)(+<index name>)+<component name>.db
+    //
+    public static final String SAI_DESCRIPTOR = "SAI";
+    private static final String SAI_SEPARATOR = "+";
+    private static final String EXTENSION = ".db";
+
+    private static String stargazerFileNameFormat(IndexComponentType indexComponentType, @Nullable String indexName, int generation, String version)
     {
         StringBuilder stringBuilder = new StringBuilder();
 
         stringBuilder.append(SAI_DESCRIPTOR);
         stringBuilder.append(SAI_SEPARATOR).append(version);
-        if (indexIdentifier != null)
-            stringBuilder.append(SAI_SEPARATOR).append(indexIdentifier.indexName);
-        stringBuilder.append(SAI_SEPARATOR).append(indexComponent.name);
-        stringBuilder.append(Descriptor.EXTENSION);
+        if (generation > 0)
+            stringBuilder.append(SAI_SEPARATOR).append(generation);
+        if (indexName != null)
+            stringBuilder.append(SAI_SEPARATOR).append(indexName);
+        stringBuilder.append(SAI_SEPARATOR).append(indexComponentType.representation);
+        stringBuilder.append(EXTENSION);
 
         return stringBuilder.toString();
     }
+
+    public ByteComparable.Version byteComparableVersionFor(IndexComponentType component, org.apache.cassandra.io.sstable.format.Version sstableFormatVersion)
+    {
+        return this == AA && component == IndexComponentType.TERMS_DATA
+               ? sstableFormatVersion.getByteComparableVersion()
+               : TypeUtil.BYTE_COMPARABLE_VERSION;
+    }
+
+    private static Optional<ParsedFileName> tryParseStargazerFileName(String componentStr)
+    {
+        // We skip the beginning `SAI+` and ending `.db` parts.
+        String[] splits = componentStr.substring(4, componentStr.length() - 3).split("\\+");
+        if (splits.length < 2 || splits.length > 4)
+            return Optional.empty();
+
+        Version version = parse(splits[0]);
+        IndexComponentType indexComponentType = IndexComponentType.fromRepresentation(splits[splits.length - 1]);
+
+        int generation = 0;
+        String indexName = null;
+        if (splits.length > 2)
+        {
+            // If we have 4 parts, then we know we have both the generation and index name. If we have 3
+            // however, it means we have either one, but we don't know which, so we chekc if the additional
+            // part is a number or not to distinguish.
+            boolean hasGeneration = splits.length == 4 || GENERATION_PATTERN.matcher(splits[1]).matches();
+            boolean hasIndexName = splits.length == 4 || !hasGeneration;
+            if (hasGeneration)
+                generation = Integer.parseInt(splits[1]);
+            if (hasIndexName)
+                indexName = splits[splits.length - 2];
+        }
+
+        return Optional.of(new ParsedFileName(ComponentsBuildId.of(version, generation), indexComponentType, indexName));
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/BufferedChecksumIndexInput.java b/src/java/org/apache/cassandra/index/sai/disk/io/BufferedChecksumIndexInput.java
index 333868466ad1..6cb5fea30d91 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/io/BufferedChecksumIndexInput.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/BufferedChecksumIndexInput.java
@@ -28,7 +28,7 @@
  * This implementation of {@link ChecksumIndexInput} is based on {@link org.apache.lucene.store.BufferedChecksumIndexInput}
  * but uses custom checksum algorithm instead of the hardcoded {@code CRC32} in {@code BufferedChecksumIndexInput}.
  *
- * @see org.apache.cassandra.index.sai.disk.io.IndexFileUtils.ChecksummingWriter
+ * @see org.apache.cassandra.index.sai.disk.io.IndexFileUtils.IncrementalChecksumSequentialWriter
  */
 class BufferedChecksumIndexInput extends ChecksumIndexInput
 {
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/BytesRefUtil.java b/src/java/org/apache/cassandra/index/sai/disk/io/BytesRefUtil.java
new file mode 100644
index 000000000000..ccad8f85f51a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/BytesRefUtil.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.utils.FastByteOperations;
+import org.apache.lucene.util.BytesRefBuilder;
+
+public final class BytesRefUtil
+{
+    private BytesRefUtil() {}
+
+    public static void copyBufferToBytesRef(ByteBuffer buffer, BytesRefBuilder stringBuffer)
+    {
+        int length = buffer.remaining();
+        stringBuffer.clear();
+        stringBuffer.grow(length);
+        FastByteOperations.copy(buffer, buffer.position(), stringBuffer.bytes(), 0, buffer.remaining());
+        stringBuffer.setLength(length);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/CryptoUtils.java b/src/java/org/apache/cassandra/index/sai/disk/io/CryptoUtils.java
new file mode 100644
index 000000000000..46467dd999f5
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/CryptoUtils.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.index.sai.disk.oldlucene.ByteArrayIndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+
+public class CryptoUtils
+{
+
+    public static CompressionMetadata getCompressionMeta(SSTableReader ssTableReader)
+    {
+        return ssTableReader.compression ? ssTableReader.getCompressionMetadata() : null;
+    }
+
+    public static CompressionParams getCompressionParams(SSTableReader ssTableReader)
+    {
+        return getCompressionParams(getCompressionMeta(ssTableReader));
+    }
+
+    public static CompressionParams getCompressionParams(CompressionMetadata meta)
+    {
+        return meta != null ? meta.parameters : null;
+    }
+
+    //TODO Encryption tidyup
+//    public static ICompressor getEncryptionCompressor(CompressionParams compressionParams)
+//    {
+//        ICompressor compressor = compressionParams != null ? compressionParams.getSstableCompressor() : null;
+//        return compressor != null ? compressor.encryptionOnly() : null;
+//    }
+//
+//    public static boolean isCryptoEnabled(CompressionParams params)
+//    {
+//        ICompressor sstableCompressor = params != null ? params.getSstableCompressor() : null;
+//        return sstableCompressor != null && sstableCompressor.encryptionOnly() != null ? true : false;
+//    }
+
+    public static IndexInput uncompress(IndexInput input, ICompressor compressor) throws IOException
+    {
+        return uncompress(input, compressor,
+                          new BytesRef(new byte[16]), new BytesRef(new byte[16])
+        );
+    }
+
+    /**
+     * Takes an {@link IndexInput} with compressed/encrypted data and returns another {@link IndexInput} with
+     * that data uncompressed/decrypted.
+     */
+    public static IndexInput uncompress(IndexInput input, ICompressor compressor, BytesRef compBytes, BytesRef uncompBytes) throws IOException
+    {
+        final int uncompBytesLen = input.readVInt();
+        final int compBytesLength = input.readVInt();
+
+        assert compBytesLength > 0 : "uncompBytesLen="+uncompBytesLen+" compBytesLength="+compBytesLength;
+
+        compBytes.bytes = ArrayUtil.grow(compBytes.bytes, compBytesLength);
+
+        input.readBytes(compBytes.bytes, 0, compBytesLength);
+
+        if (uncompBytes.bytes == BytesRef.EMPTY_BYTES)
+        {
+            // if EMPTY_BYTES use an exact new byte array
+            uncompBytes.bytes = new byte[uncompBytesLen];
+            uncompBytes.length = uncompBytesLen;
+        }
+        else
+        {
+            uncompBytes.bytes = ArrayUtil.grow(uncompBytes.bytes, uncompBytesLen);
+            uncompBytes.length = uncompBytesLen;
+        }
+        compressor.uncompress(compBytes.bytes, 0, compBytesLength, uncompBytes.bytes, 0);
+
+        return new ByteArrayIndexInput("", uncompBytes.bytes, 0, uncompBytesLen, input.order());
+    }
+
+    public static void compress(BytesRef uncompBytes,
+                                IndexOutput out, ICompressor compressor) throws IOException
+    {
+       compress(uncompBytes, new BytesRef(new byte[16]), out, compressor);
+    }
+
+    public static void compress(BytesRef uncompBytes, BytesRef compBytes,
+                                IndexOutput out, ICompressor compressor) throws IOException
+    {
+        ByteBuffer input = ByteBuffer.wrap(uncompBytes.bytes, 0, uncompBytes.length);
+
+        final int initCompLen = compressor.initialCompressedBufferLength(uncompBytes.length);
+
+        compBytes.bytes = ArrayUtil.grow(compBytes.bytes, initCompLen);
+        compBytes.length = initCompLen;
+
+        ByteBuffer output = ByteBuffer.wrap(compBytes.bytes);
+
+        compressor.compress(input, output);
+
+        final int compLen = output.position();
+
+        compBytes.length = compLen;
+
+        assert uncompBytes.length > 0;
+        assert compLen > 0;
+
+        out.writeVInt(uncompBytes.length);
+        out.writeVInt(compLen);
+
+        out.writeBytes(compBytes.bytes, compLen);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/EmptyDirectory.java b/src/java/org/apache/cassandra/index/sai/disk/io/EmptyDirectory.java
new file mode 100644
index 000000000000..3176fcd6cf2a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/EmptyDirectory.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.Lock;
+
+/**
+ * Always empty directory. Any operations to create, delete or open index files are unsupported.
+ */
+public final class EmptyDirectory extends Directory
+{
+    public static final Directory INSTANCE = new EmptyDirectory();
+
+    @Override
+    public String[] listAll()
+    {
+        return new String[0];
+    }
+
+    @Override
+    public void close()
+    {
+        // no-op
+    }
+
+    @Override
+    public void deleteFile(String name)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long fileLength(String name)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public IndexOutput createOutput(String name, IOContext context)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void sync(Collection<String> names)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void syncMetaData()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void rename(String source, String dest)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public IndexInput openInput(String name, IOContext context)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Lock obtainLock(String name)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Set<String> getPendingDeletions() throws IOException
+    {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/FilterIndexInput.java b/src/java/org/apache/cassandra/index/sai/disk/io/FilterIndexInput.java
new file mode 100644
index 000000000000..8daa65bca150
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/FilterIndexInput.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.CorruptIndexException;
+
+public abstract class FilterIndexInput extends IndexInputReader
+{
+    private final IndexInputReader delegate;
+
+    protected FilterIndexInput(IndexInputReader delegate)
+    {
+        super(delegate.input, delegate.doOnClose);
+        this.delegate = delegate;
+    }
+
+    public IndexInput getDelegate()
+    {
+        return delegate;
+    }
+
+    @Override
+    public void close()
+    {
+        delegate.close();
+    }
+
+    @Override
+    public long getFilePointer()
+    {
+        return delegate.getFilePointer();
+    }
+
+    @Override
+    public void seek(long pos)
+    {
+        delegate.seek(pos);
+    }
+
+    @Override
+    public long length()
+    {
+        return delegate.length();
+    }
+
+    @Override
+    public IndexInput slice(String sliceDescription, long offset, long length) throws CorruptIndexException
+    {
+        return delegate.slice(sliceDescription, offset, length);
+    }
+
+    @Override
+    public byte readByte() throws IOException
+    {
+        return delegate.readByte();
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) throws IOException
+    {
+        delegate.readBytes(b, offset, len);
+    }
+
+    @Override
+    public String toString()
+    {
+        return delegate.toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/IndexFileUtils.java b/src/java/org/apache/cassandra/index/sai/disk/io/IndexFileUtils.java
index 2c203bdbce0d..3e184b920336 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/io/IndexFileUtils.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/IndexFileUtils.java
@@ -20,24 +20,40 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.function.Supplier;
+import java.nio.channels.FileChannel;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.zip.CRC32;
 import java.util.zip.CRC32C;
 import java.util.zip.Checksum;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import io.github.jbellis.jvector.disk.BufferedRandomAccessWriter;
+import net.nicoulaj.compilecommand.annotations.DontInline;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.io.util.SequentialWriter;
 import org.apache.cassandra.io.util.SequentialWriterOption;
 import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.codecs.CodecUtil;
 
 public class IndexFileUtils
 {
+    protected static final Logger logger = LoggerFactory.getLogger(IndexFileUtils.class);
+
     @VisibleForTesting
     public static final SequentialWriterOption DEFAULT_WRITER_OPTION = SequentialWriterOption.newBuilder()
                                                                                              .trickleFsync(DatabaseDescriptor.getTrickleFsync())
@@ -46,51 +62,84 @@ public class IndexFileUtils
                                                                                              .finishOnClose(true)
                                                                                              .build();
 
-    public static final IndexFileUtils instance = new IndexFileUtils(DEFAULT_WRITER_OPTION);
+    private static final IndexFileUtils instance = new IndexFileUtils(DEFAULT_WRITER_OPTION);
     private static final Supplier<Checksum> CHECKSUM_FACTORY = CRC32C::new;
+    private static final Supplier<Checksum> LEGACY_CHECKSUM_FACTORY = CRC32::new;
+    private static IndexFileUtils overrideInstance = null;
 
     private final SequentialWriterOption writerOption;
 
+    public static synchronized void setOverrideInstance(IndexFileUtils overrideInstance)
+    {
+        IndexFileUtils.overrideInstance = overrideInstance;
+    }
+
+    public static IndexFileUtils instance()
+    {
+        if (overrideInstance == null)
+            return instance;
+        else
+            return overrideInstance;
+    }
+
+    /**
+     * Remembers checksums of files so we don't have to recompute them from the beginning of the file whenever appending
+     * to a file. Keeps checksums with respective file lengths and footer checksums so we can detect file changes
+     * that don't go through this code, and we can evict stale entries.
+     */
+    private static final Cache<String, Guard<FileChecksum>> checksumCache = Caffeine.newBuilder()
+                                                                                    .maximumSize(4096)
+                                                                                    .build();
+
     @VisibleForTesting
     protected IndexFileUtils(SequentialWriterOption writerOption)
     {
         this.writerOption = writerOption;
     }
 
-    public IndexOutputWriter openOutput(File file)
+    public IndexOutputWriter openOutput(File file, ByteOrder order, boolean append, Version version) throws IOException
     {
         assert writerOption.finishOnClose() : "IndexOutputWriter relies on close() to sync with disk.";
-
-        return new IndexOutputWriter(new ChecksummingWriter(file, writerOption));
+        var checksumWriter = new IncrementalChecksumSequentialWriter(file, writerOption, version, append);
+        return new IndexOutputWriter(checksumWriter, order);
     }
 
-    public IndexOutputWriter openOutput(File file, boolean append) throws IOException
+    public BufferedRandomAccessWriter openRandomAccessOutput(File file, boolean append) throws IOException
     {
         assert writerOption.finishOnClose() : "IndexOutputWriter relies on close() to sync with disk.";
 
-        IndexOutputWriter indexOutputWriter = new IndexOutputWriter(new ChecksummingWriter(file, writerOption));
+        var out = new BufferedRandomAccessWriter(file.toPath());
         if (append)
-            indexOutputWriter.skipBytes(file.length());
+            out.seek(file.length());
 
-        return indexOutputWriter;
+        return out;
     }
 
-    public IndexInput openInput(FileHandle handle)
+    public IndexInputReader openInput(FileHandle handle)
     {
         return IndexInputReader.create(handle);
     }
 
-    public IndexInput openBlockingInput(File file)
+    public IndexInputReader openBlockingInput(FileHandle fileHandle)
     {
-        FileHandle fileHandle = new FileHandle.Builder(file).complete();
-        RandomAccessReader randomReader = fileHandle.createReader();
-
+        final RandomAccessReader randomReader = fileHandle.createReader();
         return IndexInputReader.create(randomReader, fileHandle::close);
     }
 
-    public static ChecksumIndexInput getBufferedChecksumIndexInput(IndexInput indexInput)
+    public static ChecksumIndexInput getBufferedChecksumIndexInput(org.apache.lucene.store.IndexInput indexInput, Version version)
+    {
+        return new BufferedChecksumIndexInput(indexInput, getChecksumFactory(version).get());
+    }
+
+    public static Supplier<Checksum> getChecksumFactory(Version version)
     {
-        return new BufferedChecksumIndexInput(indexInput, CHECKSUM_FACTORY.get());
+        // TODO Use the version to determine which checksum algorithm to use
+        return LEGACY_CHECKSUM_FACTORY;
+    }
+
+    public interface ChecksumWriter
+    {
+        long getChecksum();
     }
 
     /**
@@ -99,27 +148,325 @@ public static ChecksumIndexInput getBufferedChecksumIndexInput(IndexInput indexI
      * with {@link IndexOutputWriter}. This, in turn, is used in conjunction with {@link BufferedChecksumIndexInput}
      * to verify the checksum of the data read from the file, so they must share the same checksum algorithm.
      */
-    static class ChecksummingWriter extends SequentialWriter
+    static class IncrementalChecksumSequentialWriter extends SequentialWriter implements ChecksumWriter
     {
-        private final Checksum checksum = CHECKSUM_FACTORY.get();
+        private final Version version;
+        /** Protects the checksum so only one Writer can update it */
+        private Guard<FileChecksum> checksumGuard;
+        /** Current (running) checksum from the beginning of the file till the current position */
+        private FileChecksum checksum;
+        /** Remembers the checksum after closing this writer */
+        private long finalChecksum;
 
-        ChecksummingWriter(File file, SequentialWriterOption writerOption)
+        IncrementalChecksumSequentialWriter(File file, SequentialWriterOption writerOption, Version version, boolean append) throws IOException
         {
             super(file, writerOption);
+            this.version = version;
+
+            while (checksum == null)
+            {
+                checksumGuard = checksumCache.get(file.path(), s -> new Guard<>(new FileChecksum(version)));
+                checksum = checksumGuard.tryLock();
+
+                if (checksum == null)
+                {
+                    // If we're here this means some other Writer did not unlock the checksum object,
+                    // so we can't use the same checksum safely, as there is a slight probability it
+                    // is in active use. This is not necessarily a bug - e.g. it is also possible
+                    // the other writer was interrupted and the client code simply forgot to close() it.
+                    // Therefore, we'll get a new one, just to be safe.
+                    logger.warn("File {} still in use by another instance of {}", file, this.getClass().getSimpleName());
+                    checksumCache.invalidate(file.path());
+                }
+            }
+
+            if (append)
+            {
+                var fileLength = file.length();
+                skipBytes(fileLength);
+
+                // It is possible we didn't get a good checksum.
+                // We could have gotten a zero checksum because the cache has a limited size,
+                // or the file could have been changed in the meantime by another process, in which case the
+                // footer checksum would not match.
+                // However, we don't recalculate the checksum always, because it is very costly:
+                var footerChecksum = calculateFooterChecksum();
+                if (checksum.fileLength != fileLength || checksum.footerChecksum != footerChecksum)
+                {
+                    logger.warn("Length and checksum ({}, {}) of file {} does not match the length and checksum ({}, {}) in the checksum cache. " +
+                                "Recomputing the checksum from the beginning.",
+                                fileLength, footerChecksum, file, checksum.fileLength, checksum.footerChecksum);
+                    recalculateFileChecksum();
+                }
+            }
+            else
+            {
+                // We might be overwriting an existing file
+                checksum.reset();
+            }
+        }
+
+        /**
+         * Recalculates checksum for the file.
+         * <p>
+         * Useful when the file is opened for append and checksum will need to account for the existing data.
+         * e.g. if the file opened for append is a new file, then checksum start at 0 and goes from there with the writes.
+         * If the file opened for append is an existing file, without recalculating the checksum will start at 0
+         * and only account for appended data. Checksum validation will compare it to the checksum of the whole file and fail.
+         * Hence, for the existing files this method should be called to recalculate the checksum.
+         *
+         * @throws IOException if file read failed.
+         */
+        public void recalculateFileChecksum() throws IOException
+        {
+            checksum.reset();
+            if (!file.exists())
+                return;
+
+            try(FileChannel ch = StorageProvider.instance.writeTimeReadFileChannelFor(file))
+            {
+                if (ch.size() == 0)
+                    return;
+
+                final ByteBuffer buf = ByteBuffer.allocateDirect(65536);
+                int b = ch.read(buf);
+                while (b > 0)
+                {
+                    buf.flip();
+                    checksum.update(buf);
+                    buf.clear();
+                    b = ch.read(buf);
+                }
+            }
+
+            assert checksum.fileLength == position();
+        }
+
+        /**
+         * Returns the checksum of the footer of the index file.
+         * Those bytes contain the checksum for the whole file, so
+         * this checksum can be used to verify the integrity of the whole file.
+         * Note that this is not the same as checksum written in the file footer;
+         * this is done this way so we don't have to decode the footer here.
+         */
+        public long calculateFooterChecksum() throws IOException
+        {
+            Checksum footerChecksum = getChecksumFactory(version).get();
+            try (FileChannel ch = StorageProvider.instance.writeTimeReadFileChannelFor(file))
+            {
+                ch.position(Math.max(0, file.length() - CodecUtil.footerLength()));
+                final ByteBuffer buf = ByteBuffer.allocate(CodecUtil.footerLength());
+                int b = ch.read(buf);
+                while (b > 0)
+                {
+                    buf.flip();
+                    footerChecksum.update(buf);
+                    buf.clear();
+                    b = ch.read(buf);
+                }
+            }
+            return footerChecksum.getValue();
+        }
+
+        @Override
+        public void write(ByteBuffer src) throws IOException
+        {
+            ByteBuffer shallowCopy = src.slice().order(src.order());
+            super.write(src);
+            checksum.update(shallowCopy);
+        }
+
+        @Override
+        public void writeBoolean(boolean v) throws IOException
+        {
+            super.writeBoolean(v);
+            checksum.update(v ? 1 : 0);
+        }
+
+        @Override
+        public void writeByte(int b) throws IOException
+        {
+            super.writeByte(b);
+            checksum.update(b);
+        }
+
+        // Do not override write(byte[] b) to avoid double-counting bytes in the checksum.
+        // It just calls this method anyway.
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException
+        {
+            super.write(b, off, len);
+            checksum.update(b, off, len);
+        }
+
+        @Override
+        public void writeChar(int v) throws IOException
+        {
+            super.writeChar(v);
+            addTochecksum(v, 2);
+        }
+
+        @Override
+        public void writeInt(int v) throws IOException
+        {
+            super.writeInt(v);
+            addTochecksum(v, 4);
+        }
+
+        @Override
+        public void writeLong(long v) throws IOException
+        {
+            super.writeLong(v);
+            addTochecksum(v, 8);
+        }
+
+        public long getChecksum()
+        {
+            return checksum != null ? checksum.getValue() : finalChecksum;
+        }
+
+        // To avoid double-counting bytes in the checksum.
+        // Same as super's but calls super.writeByte
+        @DontInline
+        @Override
+        protected void writeSlow(long bytes, int count) throws IOException
+        {
+            int origCount = count;
+            if (ByteOrder.BIG_ENDIAN == buffer.order())
+                while (count > 0) super.writeByte((int) (bytes >>> (8 * --count)));
+            else
+                while (count > 0) super.writeByte((int) (bytes >>> (8 * (origCount - count--))));
+        }
+
+        private void addTochecksum(long bytes, int count)
+        {
+            int origCount = count;
+            if (ByteOrder.BIG_ENDIAN == buffer.order())
+                while (count > 0) checksum.update((int) (bytes >>> (8 * --count)));
+            else
+                while (count > 0) checksum.update((int) (bytes >>> (8 * (origCount - count--))));
         }
 
-        public long getChecksum() throws IOException
+        @Override
+        public void truncate(long toSize)
         {
-            flush();
-            return checksum.getValue();
+            if (toSize == 0)
+            {
+                checksum.reset();
+                super.truncate(toSize);
+            }
+
+            // this would invalidate the checksum
+            throw new UnsupportedOperationException("truncate to non zero length not supported");
         }
 
         @Override
-        protected void flushData()
+        public void close()
         {
-            ByteBuffer toAppend = buffer.duplicate().flip();
-            super.flushData();
-            checksum.update(toAppend);
+            try
+            {
+                super.close();
+            }
+            finally
+            {
+                try
+                {
+                    // Copy the checksum value to a field in order to make the checksum value available past close().
+                    // Release the FileChecksum object so it can be used by another writer.
+                    finalChecksum = checksum.getValue();
+                    checksum.footerChecksum = calculateFooterChecksum();
+                }
+                catch (IOException e)
+                {
+                    // mark the checksum as unusable
+                    // even though it stays in the cache, it won't ever match on the file length
+                    checksum.fileLength = -1;
+                }
+                finally
+                {
+                    checksumGuard.release();
+                    checksum = null;
+                }
+            }
         }
     }
+
+    /**
+     * A lightweight helper to guard against concurrent access to an object.
+     * Used when we know object should be owned by one owner at a time.
+     */
+    static class Guard<T>
+    {
+        final T inner;
+        final AtomicBoolean locked = new AtomicBoolean(false);
+
+        public Guard(T inner)
+        {
+            this.inner = inner;
+        }
+
+        /**
+         * Locks the object and returns it.
+         * If it was already locked, return null.
+         * @return protected object
+         */
+        public T tryLock()
+        {
+            return locked.compareAndSet(false, true)
+                   ? inner
+                   : null;
+        }
+
+        public void release()
+        {
+            locked.set(false);
+        }
+    }
+
+    /**
+     * Computes the checksum from the begining of a file.
+     * Keeps track of the number of bytes processed.
+     * We need the number of bytes so we can invalidate the checksum if the file was appended or truncated.
+     */
+    static class FileChecksum
+    {
+        long fileLength = 0;
+        long footerChecksum = 0;
+        final Checksum fileChecksum;
+
+        public FileChecksum(Version version)
+        {
+            fileChecksum = getChecksumFactory(version).get();
+        }
+
+        public void reset()
+        {
+            fileLength = 0;
+            fileChecksum.reset();
+        }
+
+        public void update(int b)
+        {
+            fileLength += 1;
+            fileChecksum.update(b);
+        }
+
+        public void update(byte[] b, int off, int len)
+        {
+            fileLength += len;
+            fileChecksum.update(b, off, len);
+        }
+
+        public void update(ByteBuffer b)
+        {
+            fileLength += b.remaining();
+            fileChecksum.update(b);
+        }
+
+        public long getValue()
+        {
+            return fileChecksum.getValue();
+        }
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/IndexInput.java b/src/java/org/apache/cassandra/index/sai/disk/io/IndexInput.java
new file mode 100644
index 000000000000..414c2dfb9219
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/IndexInput.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+/**
+ * A subclass of {@link org.apache.lucene.store.IndexInput} that provides access to the byte order of the underlying data.
+ */
+public abstract class IndexInput extends org.apache.lucene.store.IndexInput
+{
+    protected final ByteOrder order;
+
+    protected IndexInput(String resourceDescription, ByteOrder order)
+    {
+        super(resourceDescription);
+        this.order = order;
+    }
+
+    public ByteOrder order()
+    {
+        return order;
+    }
+
+    @Override
+    public abstract IndexInput slice(String sliceDescription, long offset, long length) throws IOException;
+
+
+    public final ByteBuffer readBytes() throws IOException
+    {
+        int len = readVInt();
+        byte[] bytes = new byte[len];
+        readBytes(bytes, 0, len);
+        return ByteBuffer.wrap(bytes);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/IndexInputReader.java b/src/java/org/apache/cassandra/index/sai/disk/io/IndexInputReader.java
index b97c727c6abf..7af5f59412dd 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/io/IndexInputReader.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/IndexInputReader.java
@@ -20,30 +20,20 @@
 
 import java.io.IOException;
 
+import org.apache.cassandra.io.compress.CorruptBlockException;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexInput;
 
-/**
- * This is a wrapper over a Cassandra {@link RandomAccessReader} that provides an {@link IndexInput}
- * interface for Lucene classes that need {@link IndexInput}. This is an optimisation because the
- * Lucene {@link DataInput} reads bytes one at a time whereas the {@link RandomAccessReader} is
- * optimised to read multibyte objects faster.
- */
 public class IndexInputReader extends IndexInput
 {
-    /**
-     * the byte order of `input`'s native readX operations doesn't matter,
-     * because we only use `readFully` and `readByte` methods. IndexInput calls these
-     * (via DataInput) with methods that enforce LittleEndian-ness.
-    */
-    private final RandomAccessReader input;
-    private final Runnable doOnClose;
-
-    private IndexInputReader(RandomAccessReader input, Runnable doOnClose)
+    protected final RandomAccessReader input;
+    protected final Runnable doOnClose;
+
+    protected IndexInputReader(RandomAccessReader input, Runnable doOnClose)
     {
-        super(input.getPath());
+        super(input.getFile().toString(), input.order());
         this.input = input;
         this.doOnClose = doOnClose;
     }
@@ -58,12 +48,18 @@ public static IndexInputReader create(RandomAccessReader input, Runnable doOnClo
         return new IndexInputReader(input, doOnClose);
     }
 
+    @SuppressWarnings("resource")
     public static IndexInputReader create(FileHandle handle)
     {
         RandomAccessReader reader = handle.createReader();
         return new IndexInputReader(reader, () -> {});
     }
 
+    public RandomAccessReader reader()
+    {
+        return input;
+    }
+
     @Override
     public byte readByte() throws IOException
     {
@@ -73,7 +69,65 @@ public byte readByte() throws IOException
     @Override
     public void readBytes(byte[] bytes, int off, int len) throws IOException
     {
-        input.readFully(bytes, off, len);
+        try
+        {
+            input.readFully(bytes, off, len);
+        }
+        catch (CorruptBlockException ex)
+        {
+            throw new CorruptIndexException(input.getFile().toString(), "Corrupted block", ex);
+        }
+    }
+
+    /**
+     * Using {@link RandomAccessReader#readShort()} directly is faster than {@link DataInput#readShort()} which calls
+     * {@link DataInput#readByte()} one by one
+     */
+    @Override
+    public short readShort() throws IOException
+    {
+        try
+        {
+            return input.readShort();
+        }
+        catch (CorruptBlockException ex)
+        {
+            throw new CorruptIndexException(input.getFile().toString(), "Corrupted block", ex);
+        }
+    }
+
+    /**
+     * Using {@link RandomAccessReader#readInt()} directly is faster than {@link DataInput#readInt()} which
+     * calls {@link DataInput#readByte()} one by one
+     */
+    @Override
+    public int readInt() throws IOException
+    {
+        try
+        {
+            return input.readInt();
+        }
+        catch (CorruptBlockException ex)
+        {
+            throw new CorruptIndexException(input.getFile().toString(), "Corrupted block", ex);
+        }
+    }
+
+    /**
+     * Using {@link RandomAccessReader#readLong()} directly is faster than {@link DataInput#readLong()} which
+     * calls {@link DataInput#readByte()} one by one
+     */
+    @Override
+    public long readLong() throws IOException
+    {
+        try
+        {
+            return input.readLong();
+        }
+        catch (CorruptBlockException ex)
+        {
+            throw new CorruptIndexException(input.getFile().toString(), "Corrupted block", ex);
+        }
     }
 
     @Override
@@ -108,8 +162,32 @@ public long length()
     }
 
     @Override
-    public IndexInput slice(String sliceDescription, long offset, long length)
+    public IndexInput slice(String sliceDescription, long offset, long length) throws CorruptIndexException
     {
-        throw new UnsupportedOperationException("Slice operations are not supported");
+        if (offset < 0 || length < 0 || offset + length > input.length())
+        {
+            throw new CorruptIndexException("Invalid slice! Offset: " + offset + ", Length: " + length + ", Input Length: " + input.length(), this);
+        }
+
+        return new IndexInputReader(input, doOnClose)
+        {
+            @Override
+            public void seek(long position)
+            {
+                input.seek(position + offset);
+            }
+
+            @Override
+            public long getFilePointer()
+            {
+                return input.getFilePointer() - offset;
+            }
+
+            @Override
+            public long length()
+            {
+                return length;
+            }
+        };
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutput.java b/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutput.java
new file mode 100644
index 000000000000..a57989088a43
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutput.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * A subclass of {@link org.apache.lucene.store.IndexOutput} that provides access to the byte order of the underlying data.
+ * This is used to select output implementations compatible with varying versions of Lucene.
+ */
+public abstract class IndexOutput extends org.apache.lucene.store.IndexOutput
+{
+    protected final ByteOrder order;
+
+    public IndexOutput(String resourceDescription, String name, ByteOrder order)
+    {
+        super(resourceDescription, name);
+        this.order = order;
+    }
+
+    public ByteOrder order()
+    {
+        return order;
+    }
+
+    public final void writeBytes(ByteBuffer buf) throws IOException
+    {
+        byte[] bytes = ByteBufferUtil.getArray(buf);
+        writeVInt(bytes.length);
+        writeBytes(bytes, 0, bytes.length);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutputWriter.java b/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutputWriter.java
index 4e801011da2c..b42688a83602 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutputWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutputWriter.java
@@ -18,7 +18,8 @@
 package org.apache.cassandra.index.sai.disk.io;
 
 import java.io.IOException;
-import javax.annotation.concurrent.NotThreadSafe;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteOrder;
 
 import com.google.common.base.MoreObjects;
 import org.slf4j.Logger;
@@ -26,28 +27,17 @@
 
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.SequentialWriter;
-import org.apache.lucene.store.IndexOutput;
 
-/**
- * This is a wrapper over a Cassandra {@link SequentialWriter} that provides a Lucene {@link IndexOutput}
- * interface for the Lucene index writers.
- */
-@NotThreadSafe
 public class IndexOutputWriter extends IndexOutput
 {
-    private static final Logger logger = LoggerFactory.getLogger(IndexOutputWriter.class);
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-    /**
-     * the byte order of `out`'s native writeX operations doesn't matter,
-     * because we only use `write(byte[])` and `writeByte` methods. IndexOutput calls these
-     * (via DataOutput) with methods that enforce LittleEndian-ness.
-    */
     private final SequentialWriter out;
     private boolean closed;
 
-    public IndexOutputWriter(SequentialWriter out)
+    public IndexOutputWriter(SequentialWriter out, ByteOrder order)
     {
-        super(out.getPath(), out.getPath());
+        super(out.getFile().toString(), out.getFile().name(), order);
         this.out = out;
     }
 
@@ -58,13 +48,13 @@ public void skipBytes(long length) throws IOException
 
     public File getFile()
     {
-        return new File(out.getPath());
+        return out.getFile();
     }
 
     @Override
-    public long getChecksum() throws IOException
+    public long getChecksum()
     {
-        return ((IndexFileUtils.ChecksummingWriter)out).getChecksum();
+        return ((IndexFileUtils.ChecksumWriter)out).getChecksum();
     }
 
     @Override
@@ -79,6 +69,56 @@ public void writeBytes(byte[] bytes, int offset, int len) throws IOException
         out.write(bytes, offset, len);
     }
 
+    @Override
+    public void writeInt(int v) throws IOException
+    {
+        if (order == ByteOrder.BIG_ENDIAN)
+        {
+            writeByte((byte) (v >>> 24));
+            writeByte((byte) (v >>> 16));
+            writeByte((byte) (v >>> 8));
+            writeByte((byte) v);
+        }
+        else
+        {
+            super.writeInt(v);
+        }
+    }
+
+    @Override
+    public void writeShort(short v) throws IOException
+    {
+        if (order == ByteOrder.BIG_ENDIAN)
+        {
+            writeByte((byte)(v >>> 8));
+            writeByte((byte) v);
+        }
+        else
+        {
+            super.writeShort(v);
+        }
+    }
+
+    @Override
+    public void writeLong(long v) throws IOException
+    {
+        if (order == ByteOrder.BIG_ENDIAN)
+        {
+            writeByte((byte)(v >>> 56));
+            writeByte((byte)(v >>> 48));
+            writeByte((byte)(v >>> 40));
+            writeByte((byte)(v >>> 32));
+            writeByte((byte)(v >>> 24));
+            writeByte((byte)(v >>> 16));
+            writeByte((byte)(v >>>  8));
+            writeByte((byte) v);
+        }
+        else
+        {
+            super.writeLong(v);
+        }
+    }
+
     @Override
     public void writeByte(byte b) throws IOException
     {
@@ -86,7 +126,7 @@ public void writeByte(byte b) throws IOException
     }
 
     @Override
-    public void close()
+    public void close() throws IOException
     {
         // IndexOutput#close contract allows any output to be closed multiple times,
         // and Lucene does it in few places. SequentialWriter can be closed once.
@@ -106,21 +146,15 @@ public void close()
     @Override
     public String toString()
     {
-        String checksum;
-        try {
-            checksum = String.valueOf(getChecksum());
-        } catch (IOException e) {
-            checksum = "unknown due to I/O error: " + e;
-        }
         return MoreObjects.toStringHelper(this)
-                          .add("path", out.getPath())
+                          .add("path", out.getFile())
                           .add("bytesWritten", getFilePointer())
-                          .add("crc", checksum)
+                          .add("crc", getChecksum())
                           .toString();
     }
 
     /**
-     * Returns {@link SequentialWriter} associated with this writer. Convenient when interacting with Cassandra codebase to
+     * Returns {@link SequentialWriter} associated with this writer. Convenient when interacting with DSE-DB codebase to
      * write files to disk. Note that all bytes written to the returned writer will still contribute to the checksum.
      *
      * @return {@link SequentialWriter} associated with this writer
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ByteArrayIndexInput.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ByteArrayIndexInput.java
new file mode 100644
index 000000000000..b24b28da29fd
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ByteArrayIndexInput.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteOrder;
+import java.util.Locale;
+
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+
+/**
+ * A {@link IndexInput} backed by a byte array.
+ *
+ * ByteBufferIndexInput is nominally the blessed replacement for this, but
+ * it's a pretty different API.
+ *
+ * @lucene.experimental
+ */
+public final class ByteArrayIndexInput extends IndexInput implements RandomAccessInput
+{
+    private byte[] bytes;
+
+    private final int offset;
+    private final int length;
+    private final boolean isBigEndian;
+
+    private int pos;
+
+    public ByteArrayIndexInput(String description, byte[] bytes, ByteOrder order) {
+        this(description, bytes, 0, bytes.length, order);
+    }
+
+    public ByteArrayIndexInput(String description, byte[] bytes, int offs, int length, ByteOrder order) {
+        super(description, order);
+        this.offset = offs;
+        this.bytes = bytes;
+        this.length = length;
+        this.pos = offs;
+        this.isBigEndian = order == ByteOrder.BIG_ENDIAN;
+    }
+
+    public long getFilePointer() {
+        return pos - offset;
+    }
+
+    public void seek(long pos) throws EOFException {
+        int newPos = Math.toIntExact(pos + offset);
+        try {
+            if (pos < 0 || pos > length) {
+                throw new EOFException();
+            }
+        } finally {
+            this.pos = newPos;
+        }
+    }
+
+    @Override
+    public long length() {
+        return length;
+    }
+
+    @Override
+    public short readShort() {
+        var b1 = bytes[pos++] & 0xFF;
+        var b2 = bytes[pos++] & 0xFF;
+        return isBigEndian
+               ? (short) (b1 << 8 | b2)
+               : (short) (b2 << 8 | b1);
+    }
+
+    @Override
+    public int readInt() {
+        var b1 = bytes[pos++] & 0xFF;
+        var b2 = bytes[pos++] & 0xFF;
+        var b3 = bytes[pos++] & 0xFF;
+        var b4 = bytes[pos++] & 0xFF;
+        
+        return isBigEndian
+               ? b1 << 24 | b2 << 16 | b3 << 8 | b4
+               : b4 << 24 | b3 << 16 | b2 << 8 | b1;
+    }
+
+    @Override
+    public long readLong()
+    {
+        int i1 = readInt();
+        int i2 = readInt();
+        return isBigEndian
+                ? (long) i1 << 32 | i2 & 0xFFFFFFFFL
+                : (long) i2 << 32 | i1 & 0xFFFFFFFFL;
+    }
+
+    // NOTE: AIOOBE not EOF if you read too much
+    @Override
+    public byte readByte() {
+        return bytes[pos++];
+    }
+
+    // NOTE: AIOOBE not EOF if you read too much
+    @Override
+    public void readBytes(byte[] b, int offset, int len) {
+        System.arraycopy(bytes, pos, b, offset, len);
+        pos += len;
+    }
+
+    @Override
+    public void close() {
+        bytes = null;
+    }
+
+    @Override
+    public IndexInput clone() {
+        ByteArrayIndexInput slice = slice("(cloned)" + toString(), 0, length());
+        try {
+            slice.seek(getFilePointer());
+        } catch (EOFException e) {
+            throw new UncheckedIOException(e);
+        }
+        return slice;
+    }
+
+    public ByteArrayIndexInput slice(String sliceDescription, long offset, long length) {
+        if (offset < 0 || length < 0 || offset + length > this.length) {
+            throw new IllegalArgumentException(String.format(Locale.ROOT,
+                                                             "slice(offset=%s, length=%s) is out of bounds: %s",
+                                                             offset, length, this));
+        }
+
+        return new ByteArrayIndexInput(sliceDescription,
+                                       this.bytes,
+                                       Math.toIntExact(this.offset + offset),
+                                       Math.toIntExact(length),
+                                       isBigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Override
+    public byte readByte(long pos) throws IOException {
+        return bytes[Math.toIntExact(offset + pos)];
+    }
+
+    @Override
+    public short readShort(long pos) throws IOException {
+        int i = Math.toIntExact(offset + pos);
+        var b1 = bytes[i] & 0xFF;
+        var b2 = bytes[i + 1] & 0xFF;
+        return isBigEndian
+                ? (short) (b1 << 8 | b2)
+                : (short) (b2 << 8 | b1);
+    }
+
+    @Override
+    public int readInt(long pos) throws IOException {
+        int i = Math.toIntExact(offset + pos);
+        var b1 = bytes[i] & 0xFF;
+        var b2 = bytes[i + 1] & 0xFF;
+        var b3 = bytes[i + 2] & 0xFF;
+        var b4 = bytes[i + 3] & 0xFF;
+        return isBigEndian
+                ? b1 << 24 | b2 << 16 | b3 << 8 | b4
+                : b4 << 24 | b3 << 16 | b2 << 8 | b1;
+    }
+
+    @Override
+    public long readLong(long pos) throws IOException {
+        int i = Math.toIntExact(offset + pos);
+        int b1 = readInt(i);
+        int b2 = readInt(i + 4);
+        return isBigEndian
+                ? (long) b1 << 32 | b2 & 0xFFFFFFFFL
+                : (long) b2 << 32 | b1 & 0xFFFFFFFFL;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ByteBuffersDataOutputAdapter.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ByteBuffersDataOutputAdapter.java
new file mode 100644
index 000000000000..948b60106606
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ByteBuffersDataOutputAdapter.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import org.apache.lucene.store.DataOutput;
+
+/**
+ * Minimal wrapper around Lucene's ByteBufferDataOutput/LegacyByteBufferDataOutput, which don't share an interface.
+ * We need this to call ByteBufferDataOutput-specific methods at callsites that could contain either type.
+ */
+public abstract class ByteBuffersDataOutputAdapter extends DataOutput
+{
+    public abstract void reset();
+    public abstract long size();
+    public abstract byte[] toArrayCopy();
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/DirectWriterAdapter.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/DirectWriterAdapter.java
new file mode 100644
index 000000000000..df31f3c0ca83
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/DirectWriterAdapter.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+
+/**
+ * Minimal wrapper around Lucene's DirectWriter/LegacyDirectWriter, which don't share an interface.
+ * We need this to write out AA version indexes using LegacyDirectWriter.
+ */
+public interface DirectWriterAdapter
+{
+    public void add(long l) throws IOException;
+    public void finish() throws IOException;
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/EndiannessReverserChecksumIndexInput.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/EndiannessReverserChecksumIndexInput.java
new file mode 100644
index 000000000000..b23135eb3fa2
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/EndiannessReverserChecksumIndexInput.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IndexInput;
+
+/**
+ * A {@link ChecksumIndexInput} wrapper that changes the endianness of the provided index output.
+ */
+public final class EndiannessReverserChecksumIndexInput extends ChecksumIndexInput {
+
+    private final ChecksumIndexInput in;
+
+    public EndiannessReverserChecksumIndexInput(IndexInput in, Version version) {
+        super("Endianness reverser Checksum Index Input wrapper");
+        this.in = IndexFileUtils.getBufferedChecksumIndexInput(in, version);
+    }
+
+    @Override
+    public long getChecksum() throws IOException {
+        return in.getChecksum();
+    }
+
+    @Override
+    public byte readByte() throws IOException {
+        return in.readByte();
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) throws IOException {
+        in.readBytes(b, offset, len);
+    }
+
+    @Override
+    public short readShort() throws IOException {
+        return Short.reverseBytes(in.readShort());
+    }
+
+    @Override
+    public int readInt() throws IOException {
+        return Integer.reverseBytes(in.readInt());
+    }
+
+    @Override
+    public long readLong() throws IOException {
+        return Long.reverseBytes(in.readLong());
+    }
+
+    @Override
+    public void close() throws IOException {
+        in.close();
+    }
+
+    @Override
+    public long getFilePointer() {
+        return in.getFilePointer();
+    }
+
+    @Override
+    public long length() {
+        return in.length();
+    }
+
+    @Override
+    public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+        throw new UnsupportedOperationException("This operation is not yet supported");
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataInput.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataInput.java
new file mode 100644
index 000000000000..d0ad53cd623a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataInput.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
+
+/**
+ * A {@link DataInput} implementing {@link RandomAccessInput} and reading data from a
+ * list of {@link ByteBuffer}s. This uses the big-endian byte ordering of Lucene 7.5.
+ * Note that this participates in the type hierarchy of the modern Lucene depencency,
+ * so DataInput methods that assume little-endianness must be overridden.
+ * This file was imported from the Apache Lucene project at commit b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df,
+ * tagged as releases/lucene-solr/7.5.0. The following modifications have been made to the original file:
+ * <ul>
+ * <li>Renamed from ByteBuffersDataInput to LegacyByteBuffersDataInput.</li>
+ * <li>Return types modified accordingly.</li>
+ * <li>skipBytes was added.</li>
+ * <li>readShort/readInt/readLong implemented as big-endian, since superclass implementations are little-endian.</li>
+ * <li>explicitly override readFloats/readInts/readLongs in case DataInput implementation changes.</li>
+ * </ul>
+ */
+public final class LegacyByteBuffersDataInput extends DataInput implements Accountable, RandomAccessInput
+{
+    private final ByteBuffer[] blocks;
+    private final int blockBits;
+    private final int blockMask;
+    private final long size;
+    private final long offset;
+
+    private long pos;
+
+    /**
+     * Read data from a set of contiguous buffers. All data buffers except for the last one
+     * must have an identical remaining number of bytes in the buffer (that is a power of two). The last
+     * buffer can be of an arbitrary remaining length.
+     */
+    public LegacyByteBuffersDataInput(List<ByteBuffer> buffers)
+    {
+        ensureAssumptions(buffers);
+
+        this.blocks = buffers.stream().map(buf -> buf.asReadOnlyBuffer()).toArray(ByteBuffer[]::new);
+
+        if (blocks.length == 1)
+        {
+            this.blockBits = 32;
+            this.blockMask = ~0;
+        }
+        else
+        {
+            final int blockBytes = determineBlockPage(buffers);
+            this.blockBits = Integer.numberOfTrailingZeros(blockBytes);
+            this.blockMask = (1 << blockBits) - 1;
+        }
+
+        this.size = Arrays.stream(blocks).mapToLong(block -> block.remaining()).sum();
+
+        // The initial "position" of this stream is shifted by the position of the first block.
+        this.offset = blocks[0].position();
+        this.pos = offset;
+    }
+
+    public long size()
+    {
+        return size;
+    }
+
+    @Override
+    public long ramBytesUsed()
+    {
+        // Return a rough estimation for allocated blocks. Note that we do not make
+        // any special distinction for what the type of buffer is (direct vs. heap-based).
+        return RamUsageEstimator.NUM_BYTES_OBJECT_REF * blocks.length +
+               Arrays.stream(blocks).mapToLong(buf -> buf.capacity()).sum();
+    }
+
+    @Override
+    public byte readByte() throws EOFException
+    {
+        try
+        {
+            ByteBuffer block = blocks[blockIndex(pos)];
+            byte v = block.get(blockOffset(pos));
+            pos++;
+            return v;
+        }
+        catch (IndexOutOfBoundsException e)
+        {
+            if (pos >= size())
+            {
+                throw new EOFException();
+            }
+            else
+            {
+                throw e; // Something is wrong.
+            }
+        }
+    }
+
+    /**
+     * Reads a specified number of floats into an array at the specified offset.
+     *
+     * @param floats the array to read bytes into
+     * @param offset the offset in the array to start storing floats
+     * @param len    the number of floats to read
+     */
+    @Override
+    public void readFloats(float[] floats, int offset, int len) throws IOException
+    {
+        Objects.checkFromIndexSize(offset, len, floats.length);
+        for (int i = 0; i < len; i++)
+        {
+            floats[offset + i] = Float.intBitsToFloat(readInt());
+        }
+    }
+
+    /**
+     * Read a specified number of longs.
+     *
+     * @lucene.experimental
+     */
+    public void readLongs(long[] dst, int offset, int length) throws IOException
+    {
+        Objects.checkFromIndexSize(offset, length, dst.length);
+        for (int i = 0; i < length; ++i)
+        {
+            dst[offset + i] = readLong();
+        }
+    }
+
+    /**
+     * Reads a specified number of ints into an array at the specified offset.
+     *
+     * @param dst    the array to read bytes into
+     * @param offset the offset in the array to start storing ints
+     * @param length the number of ints to read
+     */
+    public void readInts(int[] dst, int offset, int length) throws IOException
+    {
+        Objects.checkFromIndexSize(offset, length, dst.length);
+        for (int i = 0; i < length; ++i)
+        {
+            dst[offset + i] = readInt();
+        }
+    }
+
+    /**
+     * Reads exactly {@code len} bytes into the given buffer. The buffer must have
+     * enough remaining limit.
+     * <p>
+     * If there are fewer than {@code len} bytes in the input, {@link EOFException}
+     * is thrown.
+     */
+    public void readBytes(ByteBuffer buffer, int len) throws EOFException
+    {
+        try
+        {
+            while (len > 0)
+            {
+                ByteBuffer block = blocks[blockIndex(pos)].duplicate();
+                int blockOffset = blockOffset(pos);
+                block.position(blockOffset);
+                int chunk = Math.min(len, block.remaining());
+                if (chunk == 0)
+                {
+                    throw new EOFException();
+                }
+
+                // Update pos early on for EOF detection on output buffer, then try to get buffer content.
+                pos += chunk;
+                block.limit(blockOffset + chunk);
+                buffer.put(block);
+
+                len -= chunk;
+            }
+        }
+        catch (BufferUnderflowException | ArrayIndexOutOfBoundsException e)
+        {
+            if (pos >= size())
+            {
+                throw new EOFException();
+            }
+            else
+            {
+                throw e; // Something is wrong.
+            }
+        }
+    }
+
+    @Override
+    public void readBytes(byte[] arr, int off, int len) throws EOFException
+    {
+        try
+        {
+            while (len > 0)
+            {
+                ByteBuffer block = blocks[blockIndex(pos)].duplicate();
+                block.position(blockOffset(pos));
+                int chunk = Math.min(len, block.remaining());
+                if (chunk == 0)
+                {
+                    throw new EOFException();
+                }
+
+                // Update pos early on for EOF detection, then try to get buffer content.
+                pos += chunk;
+                block.get(arr, off, chunk);
+
+                len -= chunk;
+                off += chunk;
+            }
+        }
+        catch (BufferUnderflowException | ArrayIndexOutOfBoundsException e)
+        {
+            if (pos >= size())
+            {
+                throw new EOFException();
+            }
+            else
+            {
+                throw e; // Something is wrong.
+            }
+        }
+    }
+
+    @Override
+    public short readShort() throws IOException
+    {
+        return (short) (((readByte() & 0xFF) << 8) | (readByte() & 0xFF));
+    }
+
+    @Override
+    public int readInt() throws IOException
+    {
+        return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
+               | ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
+    }
+
+    @Override
+    public long readLong() throws IOException
+    {
+        return (((long) readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
+    }
+
+    @Override
+    public void skipBytes(long l) throws IOException
+    {
+        if (l < 0)
+        {
+            throw new IllegalArgumentException("l must be >= 0, got " + l);
+        }
+        if (l > size() - pos)
+        {
+            throw new EOFException();
+        }
+        pos += l;
+    }
+
+    @Override
+    public byte readByte(long pos)
+    {
+        pos += offset;
+        return blocks[blockIndex(pos)].get(blockOffset(pos));
+    }
+
+    @Override
+    public short readShort(long pos)
+    {
+        long absPos = offset + pos;
+        int blockOffset = blockOffset(absPos);
+        if (blockOffset + Short.BYTES <= blockMask)
+        {
+            return blocks[blockIndex(absPos)].getShort(blockOffset);
+        }
+        else
+        {
+            return (short) ((readByte(pos) & 0xFF) << 8 |
+                            (readByte(pos + 1) & 0xFF));
+        }
+    }
+
+    @Override
+    public int readInt(long pos)
+    {
+        long absPos = offset + pos;
+        int blockOffset = blockOffset(absPos);
+        if (blockOffset + Integer.BYTES <= blockMask)
+        {
+            return blocks[blockIndex(absPos)].getInt(blockOffset);
+        }
+        else
+        {
+            return ((readByte(pos)) << 24 |
+                    (readByte(pos + 1) & 0xFF) << 16 |
+                    (readByte(pos + 2) & 0xFF) << 8 |
+                    (readByte(pos + 3) & 0xFF));
+        }
+    }
+
+    @Override
+    public long readLong(long pos)
+    {
+        long absPos = offset + pos;
+        int blockOffset = blockOffset(absPos);
+        if (blockOffset + Long.BYTES <= blockMask)
+        {
+            return blocks[blockIndex(absPos)].getLong(blockOffset);
+        }
+        else
+        {
+            return (((long) readInt(pos)) << 32) | (readInt(pos + 4) & 0xFFFFFFFFL);
+        }
+    }
+
+    public long position()
+    {
+        return pos - offset;
+    }
+
+    public void seek(long position) throws EOFException
+    {
+        this.pos = position + offset;
+        if (position > size())
+        {
+            this.pos = size();
+            throw new EOFException();
+        }
+    }
+
+    public LegacyByteBuffersDataInput slice(long offset, long length)
+    {
+        if (offset < 0 || length < 0 || offset + length > this.size)
+        {
+            throw new IllegalArgumentException(String.format(Locale.ROOT,
+                                                             "slice(offset=%s, length=%s) is out of bounds: %s",
+                                                             offset, length, this));
+        }
+
+        return new LegacyByteBuffersDataInput(sliceBufferList(Arrays.asList(this.blocks), offset, length));
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format(Locale.ROOT,
+                             "%,d bytes, block size: %,d, blocks: %,d, position: %,d%s",
+                             size(),
+                             blockSize(),
+                             blocks.length,
+                             position(),
+                             offset == 0 ? "" : String.format(Locale.ROOT, " [offset: %,d]", offset));
+    }
+
+    private final int blockIndex(long pos)
+    {
+        return Math.toIntExact(pos >> blockBits);
+    }
+
+    private final int blockOffset(long pos)
+    {
+        return (int) pos & blockMask;
+    }
+
+    private int blockSize()
+    {
+        return 1 << blockBits;
+    }
+
+    private static final boolean isPowerOfTwo(int v)
+    {
+        return (v & (v - 1)) == 0;
+    }
+
+    private static void ensureAssumptions(List<ByteBuffer> buffers)
+    {
+        if (buffers.isEmpty())
+        {
+            throw new IllegalArgumentException("Buffer list must not be empty.");
+        }
+
+        if (buffers.size() == 1)
+        {
+            // Special case of just a single buffer, conditions don't apply.
+        }
+        else
+        {
+            final int blockPage = determineBlockPage(buffers);
+
+            // First buffer decides on block page length.
+            if (!isPowerOfTwo(blockPage))
+            {
+                throw new IllegalArgumentException("The first buffer must have power-of-two position() + remaining(): 0x"
+                                                   + Integer.toHexString(blockPage));
+            }
+
+            // Any block from 2..last-1 should have the same page size.
+            for (int i = 1, last = buffers.size() - 1; i < last; i++)
+            {
+                ByteBuffer buffer = buffers.get(i);
+                if (buffer.position() != 0)
+                {
+                    throw new IllegalArgumentException("All buffers except for the first one must have position() == 0: " + buffer);
+                }
+                if (i != last && buffer.remaining() != blockPage)
+                {
+                    throw new IllegalArgumentException("Intermediate buffers must share an identical remaining() power-of-two block size: 0x"
+                                                       + Integer.toHexString(blockPage));
+                }
+            }
+        }
+    }
+
+    static int determineBlockPage(List<ByteBuffer> buffers)
+    {
+        ByteBuffer first = buffers.get(0);
+        final int blockPage = Math.toIntExact((long) first.position() + first.remaining());
+        return blockPage;
+    }
+
+    private static List<ByteBuffer> sliceBufferList(List<ByteBuffer> buffers, long offset, long length)
+    {
+        ensureAssumptions(buffers);
+
+        if (buffers.size() == 1)
+        {
+            ByteBuffer cloned = buffers.get(0).asReadOnlyBuffer();
+            cloned.position(Math.toIntExact(cloned.position() + offset));
+            cloned.limit(Math.toIntExact(length + cloned.position()));
+            return Arrays.asList(cloned);
+        }
+        else
+        {
+            long absStart = buffers.get(0).position() + offset;
+            long absEnd = Math.toIntExact(absStart + length);
+
+            int blockBytes = LegacyByteBuffersDataInput.determineBlockPage(buffers);
+            int blockBits = Integer.numberOfTrailingZeros(blockBytes);
+            int blockMask = (1 << blockBits) - 1;
+
+            int endOffset = (int) absEnd & blockMask;
+
+            ArrayList<ByteBuffer> cloned =
+            buffers.subList(Math.toIntExact(absStart / blockBytes),
+                            Math.toIntExact(absEnd / blockBytes + (endOffset == 0 ? 0 : 1)))
+                   .stream()
+                   .map(buf -> buf.asReadOnlyBuffer())
+                   .collect(Collectors.toCollection(ArrayList::new));
+
+            if (endOffset == 0)
+            {
+                cloned.add(ByteBuffer.allocate(0));
+            }
+
+            cloned.get(0).position((int) absStart & blockMask);
+            cloned.get(cloned.size() - 1).limit(endOffset);
+            return cloned;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataOutput.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataOutput.java
new file mode 100644
index 000000000000..eb397a412486
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataOutput.java
@@ -0,0 +1,663 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.IntConsumer;
+import java.util.function.IntFunction;
+
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.UnicodeUtil;
+
+/**
+ * A {@link DataOutput} storing data in a list of {@link ByteBuffer}s. The data is written in big-endian byte
+ * order, as produced by Lucene 7.5. Note that this participates in the type hierarchy of the modern Lucene
+ * dependency, so it must carefully override DataOutput methods that would use the modern Lucene byte ordering of
+ * little-endian.
+ * This file was imported from the Apache Lucene project at commit b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df,
+ * tagged as releases/lucene-solr/7.5.0. The following modifications have been made to the original file:
+ * <ul>
+ * <li>Renamed from ByteBuffersDataOutput to LegacyByteBuffersDataOutput.</li>
+ * <li>Return types modified accordingly.</li>
+ * <li>toDataInput now returns a LegacyByteBuffersDataInput to match encodings.</li>
+ * <li>writeShort/writeInt/writeLong now use writeCrossBlock* implementations to avoid delegating to superclass.</li>
+ * </ul>
+ */
+public final class LegacyByteBuffersDataOutput extends DataOutput implements Accountable
+{
+    private final static ByteBuffer EMPTY = ByteBuffer.allocate(0);
+    private final static byte[] EMPTY_BYTE_ARRAY = {};
+
+    public final static IntFunction<ByteBuffer> ALLOCATE_BB_ON_HEAP = ByteBuffer::allocate;
+
+    /**
+     * A singleton instance of "no-reuse" buffer strategy.
+     */
+    public final static Consumer<ByteBuffer> NO_REUSE = (bb) -> {
+        throw new RuntimeException("reset() is not allowed on this buffer.");
+    };
+
+    /**
+     * An implementation of a {@link ByteBuffer} allocation and recycling policy.
+     * The blocks are recycled if exactly the same size is requested, otherwise
+     * they're released to be GCed.
+     */
+    public final static class ByteBufferRecycler
+    {
+        private final ArrayDeque<ByteBuffer> reuse = new ArrayDeque<>();
+        private final IntFunction<ByteBuffer> delegate;
+
+        public ByteBufferRecycler(IntFunction<ByteBuffer> delegate)
+        {
+            this.delegate = Objects.requireNonNull(delegate);
+        }
+
+        public ByteBuffer allocate(int size)
+        {
+            while (!reuse.isEmpty())
+            {
+                ByteBuffer bb = reuse.removeFirst();
+                // If we don't have a buffer of exactly the requested size, discard it.
+                if (bb.remaining() == size)
+                {
+                    return bb;
+                }
+            }
+
+            return delegate.apply(size);
+        }
+
+        public void reuse(ByteBuffer buffer)
+        {
+            buffer.rewind();
+            reuse.addLast(buffer);
+        }
+    }
+
+    public final static int DEFAULT_MIN_BITS_PER_BLOCK = 10; // 1024 B
+    public final static int DEFAULT_MAX_BITS_PER_BLOCK = 26; //   64 MB
+
+    /**
+     * Maximum number of blocks at the current {@link #blockBits} block size
+     * before we increase the block size (and thus decrease the number of blocks).
+     */
+    final static int MAX_BLOCKS_BEFORE_BLOCK_EXPANSION = 100;
+
+    /**
+     * Maximum block size: {@code 2^bits}.
+     */
+    private final int maxBitsPerBlock;
+
+    /**
+     * {@link ByteBuffer} supplier.
+     */
+    private final IntFunction<ByteBuffer> blockAllocate;
+
+    /**
+     * {@link ByteBuffer} recycler on {@link #reset}.
+     */
+    private final Consumer<ByteBuffer> blockReuse;
+
+    /**
+     * Current block size: {@code 2^bits}.
+     */
+    private int blockBits;
+
+    /**
+     * Blocks storing data.
+     */
+    private final ArrayDeque<ByteBuffer> blocks = new ArrayDeque<>();
+
+    /**
+     * The current-or-next write block.
+     */
+    private ByteBuffer currentBlock = EMPTY;
+
+    public LegacyByteBuffersDataOutput(long expectedSize)
+    {
+        this(computeBlockSizeBitsFor(expectedSize), DEFAULT_MAX_BITS_PER_BLOCK, ALLOCATE_BB_ON_HEAP, NO_REUSE);
+    }
+
+    public LegacyByteBuffersDataOutput()
+    {
+        this(DEFAULT_MIN_BITS_PER_BLOCK, DEFAULT_MAX_BITS_PER_BLOCK, ALLOCATE_BB_ON_HEAP, NO_REUSE);
+    }
+
+    public LegacyByteBuffersDataOutput(int minBitsPerBlock,
+                                       int maxBitsPerBlock,
+                                       IntFunction<ByteBuffer> blockAllocate,
+                                       Consumer<ByteBuffer> blockReuse)
+    {
+        if (minBitsPerBlock < 10 ||
+            minBitsPerBlock > maxBitsPerBlock ||
+            maxBitsPerBlock > 31)
+        {
+            throw new IllegalArgumentException(String.format(Locale.ROOT,
+                                                             "Invalid arguments: %s %s",
+                                                             minBitsPerBlock,
+                                                             maxBitsPerBlock));
+        }
+        this.maxBitsPerBlock = maxBitsPerBlock;
+        this.blockBits = minBitsPerBlock;
+        this.blockAllocate = Objects.requireNonNull(blockAllocate, "Block allocator must not be null.");
+        this.blockReuse = Objects.requireNonNull(blockReuse, "Block reuse must not be null.");
+    }
+
+    @Override
+    public void writeByte(byte b)
+    {
+        if (!currentBlock.hasRemaining())
+        {
+            appendBlock();
+        }
+        currentBlock.put(b);
+    }
+
+    @Override
+    public void writeBytes(byte[] src, int offset, int length)
+    {
+        assert length >= 0;
+        while (length > 0)
+        {
+            if (!currentBlock.hasRemaining())
+            {
+                appendBlock();
+            }
+
+            int chunk = Math.min(currentBlock.remaining(), length);
+            currentBlock.put(src, offset, chunk);
+            length -= chunk;
+            offset += chunk;
+        }
+    }
+
+    @Override
+    public void writeBytes(byte[] b, int length)
+    {
+        writeBytes(b, 0, length);
+    }
+
+    public void writeBytes(byte[] b)
+    {
+        writeBytes(b, 0, b.length);
+    }
+
+    public void writeBytes(ByteBuffer buffer)
+    {
+        buffer = buffer.duplicate();
+        int length = buffer.remaining();
+        while (length > 0)
+        {
+            if (!currentBlock.hasRemaining())
+            {
+                appendBlock();
+            }
+
+            int chunk = Math.min(currentBlock.remaining(), length);
+            buffer.limit(buffer.position() + chunk);
+            currentBlock.put(buffer);
+
+            length -= chunk;
+        }
+    }
+
+    /**
+     * Return a list of read-only view of {@link ByteBuffer} blocks over the
+     * current content written to the output.
+     */
+    public ArrayList<ByteBuffer> toBufferList()
+    {
+        ArrayList<ByteBuffer> result = new ArrayList<>(Math.max(blocks.size(), 1));
+        if (blocks.isEmpty())
+        {
+            result.add(EMPTY);
+        }
+        else
+        {
+            for (ByteBuffer bb : blocks)
+            {
+                bb = (ByteBuffer) bb.asReadOnlyBuffer().flip(); // cast for jdk8 (covariant in jdk9+)
+                result.add(bb);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Returns a list of writeable blocks over the (source) content buffers.
+     * <p>
+     * This method returns the raw content of source buffers that may change over the lifetime
+     * of this object (blocks can be recycled or discarded, for example). Most applications
+     * should favor calling {@link #toBufferList()} which returns a read-only <i>view</i> over
+     * the content of the source buffers.
+     * <p>
+     * The difference between {@link #toBufferList()} and {@link #toWriteableBufferList()} is that
+     * read-only view of source buffers will always return {@code false} from {@link ByteBuffer#hasArray()}
+     * (which sometimes may be required to avoid double copying).
+     */
+    public ArrayList<ByteBuffer> toWriteableBufferList()
+    {
+        ArrayList<ByteBuffer> result = new ArrayList<>(Math.max(blocks.size(), 1));
+        if (blocks.isEmpty())
+        {
+            result.add(EMPTY);
+        }
+        else
+        {
+            for (ByteBuffer bb : blocks)
+            {
+                bb = (ByteBuffer) bb.duplicate().flip(); // cast for jdk8 (covariant in jdk9+)
+                result.add(bb);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Return a {@link LegacyByteBuffersDataInput} for the set of current buffers ({@link #toBufferList()}).
+     */
+    public LegacyByteBuffersDataInput toDataInput()
+    {
+        return new LegacyByteBuffersDataInput(toBufferList());
+    }
+
+    /**
+     * Return a contiguous array with the current content written to the output. The returned
+     * array is always a copy (can be mutated).
+     */
+    public byte[] toArrayCopy()
+    {
+        if (blocks.isEmpty())
+        {
+            return EMPTY_BYTE_ARRAY;
+        }
+
+        // We could try to detect single-block, array-based ByteBuffer here
+        // and use Arrays.copyOfRange, but I don't think it's worth the extra
+        // instance checks.
+
+        byte[] arr = new byte[Math.toIntExact(size())];
+        int offset = 0;
+        for (ByteBuffer bb : toBufferList())
+        {
+            int len = bb.remaining();
+            bb.get(arr, offset, len);
+            offset += len;
+        }
+        return arr;
+    }
+
+    /**
+     * Copy the current content of this object into another {@link DataOutput}.
+     */
+    public void copyTo(DataOutput output) throws IOException
+    {
+        for (ByteBuffer bb : toBufferList())
+        {
+            if (bb.hasArray())
+            {
+                output.writeBytes(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining());
+            }
+            else
+            {
+                output.copyBytes(new LegacyByteBuffersDataInput(Arrays.asList(bb)), bb.remaining());
+            }
+        }
+    }
+
+    /**
+     * @return The number of bytes written to this output so far.
+     */
+    public long size()
+    {
+        long size = 0;
+        int blockCount = blocks.size();
+        if (blockCount >= 1)
+        {
+            int fullBlockSize = (blockCount - 1) * blockSize();
+            int lastBlockSize = blocks.getLast().position();
+            size = fullBlockSize + lastBlockSize;
+        }
+        return size;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format(Locale.ROOT,
+                             "%,d bytes, block size: %,d, blocks: %,d",
+                             size(),
+                             blockSize(),
+                             blocks.size());
+    }
+
+    // Specialized versions of writeXXX methods that break execution into
+    // fast/ slow path if the result would fall on the current block's
+    // boundary.
+    //
+    // We also remove the IOException from methods because it (theoretically)
+    // cannot be thrown from byte buffers.
+
+    @Override
+    public void writeShort(short v)
+    {
+        if (currentBlock.remaining() >= Short.BYTES)
+        {
+            currentBlock.putShort(v);
+        }
+        else
+        {
+            writeCrossBlockShort(v);
+        }
+    }
+
+    private void writeCrossBlockShort(short v)
+    {
+        writeByte((byte) (v >> 8));
+        writeByte((byte) v);
+    }
+
+    @Override
+    public void writeInt(int v)
+    {
+        if (currentBlock.remaining() >= Integer.BYTES)
+        {
+            currentBlock.putInt(v);
+        }
+        else
+        {
+            writeCrossBlockInt(v);
+        }
+    }
+
+    private void writeCrossBlockInt(int v)
+    {
+        writeByte((byte) (v >>> 24));
+        writeByte((byte) (v >>> 16));
+        writeByte((byte) (v >>> 8));
+        writeByte((byte) v);
+    }
+
+    @Override
+    public void writeLong(long v)
+    {
+        if (currentBlock.remaining() >= Long.BYTES)
+        {
+            currentBlock.putLong(v);
+        }
+        else
+        {
+            writeCrossBlockLong(v);
+        }
+    }
+
+    private void writeCrossBlockLong(long v)
+    {
+        writeByte((byte) (v >>> 56));
+        writeByte((byte) (v >>> 48));
+        writeByte((byte) (v >>> 40));
+        writeByte((byte) (v >>> 32));
+        writeByte((byte) (v >>> 24));
+        writeByte((byte) (v >>> 16));
+        writeByte((byte) (v >>> 8));
+        writeByte((byte) v);
+    }
+
+    @Override
+    public void writeString(String v)
+    {
+        try
+        {
+            final int MAX_CHARS_PER_WINDOW = 1024;
+            if (v.length() <= MAX_CHARS_PER_WINDOW)
+            {
+                final BytesRef utf8 = new BytesRef(v);
+                writeVInt(utf8.length);
+                writeBytes(utf8.bytes, utf8.offset, utf8.length);
+            }
+            else
+            {
+                writeVInt(UnicodeUtil.calcUTF16toUTF8Length(v, 0, v.length()));
+                final byte[] buf = new byte[UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR * MAX_CHARS_PER_WINDOW];
+                UTF16toUTF8(v, 0, v.length(), buf, (len) -> {
+                    writeBytes(buf, 0, len);
+                });
+            }
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    @Override
+    public void writeMapOfStrings(Map<String, String> map)
+    {
+        try
+        {
+            super.writeMapOfStrings(map);
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    @Override
+    public void writeSetOfStrings(Set<String> set)
+    {
+        try
+        {
+            super.writeSetOfStrings(set);
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    @Override
+    public long ramBytesUsed()
+    {
+        // Return a rough estimation for allocated blocks. Note that we do not make
+        // any special distinction for direct memory buffers.
+        return RamUsageEstimator.NUM_BYTES_OBJECT_REF * blocks.size() +
+               blocks.stream().mapToLong(buf -> buf.capacity()).sum();
+    }
+
+    /**
+     * This method resets this object to a clean (zero-size) state and
+     * publishes any currently allocated buffers for reuse to the reuse strategy
+     * provided in the constructor.
+     * <p>
+     * Sharing byte buffers for reads and writes is dangerous and will very likely
+     * lead to hard-to-debug issues, use with great care.
+     */
+    public void reset()
+    {
+        if (blockReuse != NO_REUSE)
+        {
+            blocks.stream().forEach(blockReuse);
+        }
+        blocks.clear();
+        currentBlock = EMPTY;
+    }
+
+    /**
+     * @return Returns a new {@link LegacyByteBuffersDataOutput} with the {@link #reset()} capability.
+     */
+    // TODO: perhaps we can move it out to an utility class (as a supplier of preconfigured instances?)
+    public static LegacyByteBuffersDataOutput newResettableInstance()
+    {
+        LegacyByteBuffersDataOutput.ByteBufferRecycler reuser = new LegacyByteBuffersDataOutput.ByteBufferRecycler(
+        LegacyByteBuffersDataOutput.ALLOCATE_BB_ON_HEAP);
+        return new LegacyByteBuffersDataOutput(
+        LegacyByteBuffersDataOutput.DEFAULT_MIN_BITS_PER_BLOCK,
+        LegacyByteBuffersDataOutput.DEFAULT_MAX_BITS_PER_BLOCK,
+        reuser::allocate,
+        reuser::reuse);
+    }
+
+    private int blockSize()
+    {
+        return 1 << blockBits;
+    }
+
+    private void appendBlock()
+    {
+        if (blocks.size() >= MAX_BLOCKS_BEFORE_BLOCK_EXPANSION && blockBits < maxBitsPerBlock)
+        {
+            rewriteToBlockSize(blockBits + 1);
+            if (blocks.getLast().hasRemaining())
+            {
+                return;
+            }
+        }
+
+        final int requiredBlockSize = 1 << blockBits;
+        currentBlock = blockAllocate.apply(requiredBlockSize);
+        assert currentBlock.capacity() == requiredBlockSize;
+        blocks.add(currentBlock);
+    }
+
+    private void rewriteToBlockSize(int targetBlockBits)
+    {
+        assert targetBlockBits <= maxBitsPerBlock;
+
+        // We copy over data blocks to an output with one-larger block bit size.
+        // We also discard references to blocks as we're copying to allow GC to
+        // clean up partial results in case of memory pressure.
+        LegacyByteBuffersDataOutput cloned = new LegacyByteBuffersDataOutput(targetBlockBits, targetBlockBits, blockAllocate, NO_REUSE);
+        ByteBuffer block;
+        while ((block = blocks.pollFirst()) != null)
+        {
+            block.flip();
+            cloned.writeBytes(block);
+            if (blockReuse != NO_REUSE)
+            {
+                blockReuse.accept(block);
+            }
+        }
+
+        assert blocks.isEmpty();
+        this.blockBits = targetBlockBits;
+        blocks.addAll(cloned.blocks);
+    }
+
+    private static int computeBlockSizeBitsFor(long bytes)
+    {
+        long powerOfTwo = BitUtil.nextHighestPowerOfTwo(bytes / MAX_BLOCKS_BEFORE_BLOCK_EXPANSION);
+        if (powerOfTwo == 0)
+        {
+            return DEFAULT_MIN_BITS_PER_BLOCK;
+        }
+
+        int blockBits = Long.numberOfTrailingZeros(powerOfTwo);
+        blockBits = Math.min(blockBits, DEFAULT_MAX_BITS_PER_BLOCK);
+        blockBits = Math.max(blockBits, DEFAULT_MIN_BITS_PER_BLOCK);
+        return blockBits;
+    }
+
+    // TODO: move this block-based conversion to UnicodeUtil.
+
+    private static final long HALF_SHIFT = 10;
+    private static final int SURROGATE_OFFSET =
+    Character.MIN_SUPPLEMENTARY_CODE_POINT -
+    (UnicodeUtil.UNI_SUR_HIGH_START << HALF_SHIFT) - UnicodeUtil.UNI_SUR_LOW_START;
+
+    /**
+     * A consumer-based UTF16-UTF8 encoder (writes the input string in smaller buffers.).
+     */
+    private static int UTF16toUTF8(final CharSequence s,
+                                   final int offset,
+                                   final int length,
+                                   byte[] buf,
+                                   IntConsumer bufferFlusher)
+    {
+        int utf8Len = 0;
+        int j = 0;
+        for (int i = offset, end = offset + length; i < end; i++)
+        {
+            final int chr = (int) s.charAt(i);
+
+            if (j + 4 >= buf.length)
+            {
+                bufferFlusher.accept(j);
+                utf8Len += j;
+                j = 0;
+            }
+
+            if (chr < 0x80)
+                buf[j++] = (byte) chr;
+            else if (chr < 0x800)
+            {
+                buf[j++] = (byte) (0xC0 | (chr >> 6));
+                buf[j++] = (byte) (0x80 | (chr & 0x3F));
+            }
+            else if (chr < 0xD800 || chr > 0xDFFF)
+            {
+                buf[j++] = (byte) (0xE0 | (chr >> 12));
+                buf[j++] = (byte) (0x80 | ((chr >> 6) & 0x3F));
+                buf[j++] = (byte) (0x80 | (chr & 0x3F));
+            }
+            else
+            {
+                // A surrogate pair. Confirm valid high surrogate.
+                if (chr < 0xDC00 && (i < end - 1))
+                {
+                    int utf32 = (int) s.charAt(i + 1);
+                    // Confirm valid low surrogate and write pair.
+                    if (utf32 >= 0xDC00 && utf32 <= 0xDFFF)
+                    {
+                        utf32 = (chr << 10) + utf32 + SURROGATE_OFFSET;
+                        i++;
+                        buf[j++] = (byte) (0xF0 | (utf32 >> 18));
+                        buf[j++] = (byte) (0x80 | ((utf32 >> 12) & 0x3F));
+                        buf[j++] = (byte) (0x80 | ((utf32 >> 6) & 0x3F));
+                        buf[j++] = (byte) (0x80 | (utf32 & 0x3F));
+                        continue;
+                    }
+                }
+                // Replace unpaired surrogate or out-of-order low surrogate
+                // with substitution character.
+                buf[j++] = (byte) 0xEF;
+                buf[j++] = (byte) 0xBF;
+                buf[j++] = (byte) 0xBD;
+            }
+        }
+
+        bufferFlusher.accept(j);
+        utf8Len += j;
+
+        return utf8Len;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataOutputAdapter.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataOutputAdapter.java
new file mode 100644
index 000000000000..fcd532c351e6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersDataOutputAdapter.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lucene.store.DataInput;
+
+/**
+ * Minimal wrapper around LegacyByteBufferDataOutput, to allow for mixed callsites of LegacyByteBufferDataOutput
+ * and ByteBufferDataOutput.
+ */
+public class LegacyByteBuffersDataOutputAdapter extends ByteBuffersDataOutputAdapter
+{
+    private LegacyByteBuffersDataOutput wrapped;
+
+    public LegacyByteBuffersDataOutputAdapter(long expectedSize)
+    {
+        wrapped = new LegacyByteBuffersDataOutput(expectedSize);
+    }
+
+    @Override
+    public void reset()
+    {
+        wrapped.reset();
+    }
+
+    @Override
+    public long size()
+    {
+        return wrapped.size();
+    }
+
+    @Override
+    public byte[] toArrayCopy()
+    {
+        return wrapped.toArrayCopy();
+    }
+
+    @Override
+    public void writeBytes(byte[] b, int length) throws IOException
+    {
+        wrapped.writeBytes(b, length);
+    }
+
+    @Override
+    public void writeInt(int i) throws IOException
+    {
+        wrapped.writeInt(i);
+    }
+
+    @Override
+    public void writeShort(short i) throws IOException
+    {
+        wrapped.writeShort(i);
+    }
+
+    @Override
+    public void writeLong(long i) throws IOException
+    {
+        wrapped.writeLong(i);
+    }
+
+    @Override
+    public void writeString(String s) throws IOException
+    {
+        wrapped.writeString(s);
+    }
+
+    @Override
+    public void copyBytes(DataInput input, long numBytes) throws IOException
+    {
+        wrapped.copyBytes(input, numBytes);
+    }
+
+    @Override
+    public void writeMapOfStrings(Map<String, String> map) throws IOException
+    {
+        wrapped.writeMapOfStrings(map);
+    }
+
+    @Override
+    public void writeSetOfStrings(Set<String> set) throws IOException
+    {
+        wrapped.writeSetOfStrings(set);
+    }
+
+    @Override
+    public void writeByte(byte b) throws IOException
+    {
+        wrapped.writeByte(b);
+    }
+
+    @Override
+    public void writeBytes(byte[] src, int offset, int length) throws IOException
+    {
+        wrapped.writeBytes(src, offset, length);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersIndexInput.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersIndexInput.java
new file mode 100644
index 000000000000..2204dba93db6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersIndexInput.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteOrder;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.RandomAccessInput;
+
+/**
+ * An {@link IndexInput} implementing {@link RandomAccessInput} and backed
+ * by a {@link LegacyByteBuffersDataInput}. Data is read in big-endian byte order,
+ * as produced by Lucene 7.5.
+ * This file was imported from the Apache Lucene project at commit b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df,
+ * tagged as releases/lucene-solr/7.5.0. The following modifications have been made to the original file:
+ * <ul>
+ * <li>Renamed from ByteBuffersIndexInput to LegacyByteBuffersIndexInput.</li>
+ * <li>Implements our IndexInput wrapper, which provides endianness.</li>
+ * <li>Wraps LegacyByteBuffersDataInput instead of ByteBuffersDataInput.</li>
+ * </ul>
+ */
+public final class LegacyByteBuffersIndexInput extends IndexInput implements RandomAccessInput
+{
+    private LegacyByteBuffersDataInput in;
+
+    public LegacyByteBuffersIndexInput(LegacyByteBuffersDataInput in, String resourceDescription)
+    {
+        super(resourceDescription, ByteOrder.BIG_ENDIAN);
+        this.in = in;
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        in = null;
+    }
+
+    @Override
+    public long getFilePointer()
+    {
+        ensureOpen();
+        return in.position();
+    }
+
+    @Override
+    public void seek(long pos) throws IOException
+    {
+        ensureOpen();
+        in.seek(pos);
+    }
+
+    @Override
+    public long length()
+    {
+        ensureOpen();
+        return in.size();
+    }
+
+    @Override
+    public LegacyByteBuffersIndexInput slice(String sliceDescription, long offset, long length) throws IOException
+    {
+        ensureOpen();
+        return new LegacyByteBuffersIndexInput(in.slice(offset, length),
+                                               "(sliced) offset=" + offset + ", length=" + length + " " + toString() + " [slice=" + sliceDescription + "]");
+    }
+
+    @Override
+    public byte readByte() throws IOException
+    {
+        ensureOpen();
+        return in.readByte();
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) throws IOException
+    {
+        ensureOpen();
+        in.readBytes(b, offset, len);
+    }
+
+    @Override
+    public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException
+    {
+        ensureOpen();
+        return slice("", offset, length);
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException
+    {
+        ensureOpen();
+        in.readBytes(b, offset, len, useBuffer);
+    }
+
+    @Override
+    public short readShort() throws IOException
+    {
+        ensureOpen();
+        return in.readShort();
+    }
+
+    @Override
+    public int readInt() throws IOException
+    {
+        ensureOpen();
+        return in.readInt();
+    }
+
+    @Override
+    public int readVInt() throws IOException
+    {
+        ensureOpen();
+        return in.readVInt();
+    }
+
+    @Override
+    public int readZInt() throws IOException
+    {
+        ensureOpen();
+        return in.readZInt();
+    }
+
+    @Override
+    public long readLong() throws IOException
+    {
+        ensureOpen();
+        return in.readLong();
+    }
+
+    @Override
+    public long readVLong() throws IOException
+    {
+        ensureOpen();
+        return in.readVLong();
+    }
+
+    @Override
+    public long readZLong() throws IOException
+    {
+        ensureOpen();
+        return in.readZLong();
+    }
+
+    @Override
+    public String readString() throws IOException
+    {
+        ensureOpen();
+        return in.readString();
+    }
+
+    @Override
+    public Map<String, String> readMapOfStrings() throws IOException
+    {
+        ensureOpen();
+        return in.readMapOfStrings();
+    }
+
+    @Override
+    public Set<String> readSetOfStrings() throws IOException
+    {
+        ensureOpen();
+        return in.readSetOfStrings();
+    }
+
+    @Override
+    public void skipBytes(long numBytes) throws IOException
+    {
+        ensureOpen();
+        super.skipBytes(numBytes);
+    }
+
+    @Override
+    public byte readByte(long pos) throws IOException
+    {
+        ensureOpen();
+        return in.readByte(pos);
+    }
+
+    @Override
+    public short readShort(long pos) throws IOException
+    {
+        ensureOpen();
+        return in.readShort(pos);
+    }
+
+    @Override
+    public int readInt(long pos) throws IOException
+    {
+        ensureOpen();
+        return in.readInt(pos);
+    }
+
+    @Override
+    public long readLong(long pos) throws IOException
+    {
+        ensureOpen();
+        return in.readLong(pos);
+    }
+
+    @Override
+    public IndexInput clone()
+    {
+        ensureOpen();
+        LegacyByteBuffersIndexInput cloned = new LegacyByteBuffersIndexInput(in.slice(0, in.size()), "(clone of) " + toString());
+        try
+        {
+            cloned.seek(getFilePointer());
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+        return cloned;
+    }
+
+    private void ensureOpen()
+    {
+        if (in == null)
+        {
+            throw new AlreadyClosedException("Already closed.");
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersIndexOutput.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersIndexOutput.java
new file mode 100644
index 000000000000..c3d08cd3e399
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyByteBuffersIndexOutput.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.DataInput;
+
+/**
+ * An {@link IndexOutput} writing to a {@link LegacyByteBuffersDataOutput}.
+ * This uses the big-endian byte ordering of Lucene 7.5 and is used to write indexes/data compatible with the
+ * readers in older Lucene versions.
+ * This file was imported from the Apache Lucene project at commit b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df,
+ * tagged as releases/lucene-solr/7.5.0. The following modifications have been made to the original file:
+ * <ul>
+ * <li>Renamed from ByteBuffersIndexOutput to LegacyByteBuffersIndexOutput.</li>
+ * <li>Implements our IndexOutput wrapper, which provides endianness.</li>
+ * <li>Wraps LegacyByteBuffersDataOutput instead of ByteBuffersDataOutput.</li>
+ * </ul>
+ */
+public final class LegacyByteBuffersIndexOutput extends IndexOutput
+{
+    private final Consumer<LegacyByteBuffersDataOutput> onClose;
+
+    private final Checksum checksum;
+    private long lastChecksumPosition;
+    private long lastChecksum;
+
+    private LegacyByteBuffersDataOutput delegate;
+
+    public LegacyByteBuffersIndexOutput(LegacyByteBuffersDataOutput delegate, String resourceDescription, String name)
+    {
+        this(delegate, resourceDescription, name, new CRC32(), null);
+    }
+
+    public LegacyByteBuffersIndexOutput(LegacyByteBuffersDataOutput delegate, String resourceDescription, String name,
+                                        Checksum checksum,
+                                        Consumer<LegacyByteBuffersDataOutput> onClose)
+    {
+        super(resourceDescription, name, ByteOrder.BIG_ENDIAN);
+        this.delegate = delegate;
+        this.checksum = checksum;
+        this.onClose = onClose;
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        // No special effort to be thread-safe here since IndexOutputs are not required to be thread-safe.
+        LegacyByteBuffersDataOutput local = delegate;
+        delegate = null;
+        if (local != null && onClose != null)
+        {
+            onClose.accept(local);
+        }
+    }
+
+    @Override
+    public long getFilePointer()
+    {
+        ensureOpen();
+        return delegate.size();
+    }
+
+    @Override
+    public long getChecksum() throws IOException
+    {
+        ensureOpen();
+
+        if (checksum == null)
+        {
+            throw new IOException("This index output has no checksum computing ability: " + toString());
+        }
+
+        // Compute checksum on the current content of the delegate.
+        //
+        // This way we can override more methods and pass them directly to the delegate for efficiency of writing,
+        // while allowing the checksum to be correctly computed on the current content of the output buffer (IndexOutput
+        // is per-thread, so no concurrent changes).
+        if (lastChecksumPosition != delegate.size())
+        {
+            lastChecksumPosition = delegate.size();
+            checksum.reset();
+            byte[] buffer = null;
+            for (ByteBuffer bb : delegate.toBufferList())
+            {
+                if (bb.hasArray())
+                {
+                    checksum.update(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining());
+                }
+                else
+                {
+                    if (buffer == null) buffer = new byte[1024 * 4];
+
+                    bb = bb.asReadOnlyBuffer();
+                    int remaining = bb.remaining();
+                    while (remaining > 0)
+                    {
+                        int len = Math.min(remaining, buffer.length);
+                        bb.get(buffer, 0, len);
+                        checksum.update(buffer, 0, len);
+                        remaining -= len;
+                    }
+                }
+            }
+            lastChecksum = checksum.getValue();
+        }
+        return lastChecksum;
+    }
+
+    @Override
+    public void writeByte(byte b) throws IOException
+    {
+        ensureOpen();
+        delegate.writeByte(b);
+    }
+
+    @Override
+    public void writeBytes(byte[] b, int offset, int length) throws IOException
+    {
+        ensureOpen();
+        delegate.writeBytes(b, offset, length);
+    }
+
+    @Override
+    public void writeBytes(byte[] b, int length) throws IOException
+    {
+        ensureOpen();
+        delegate.writeBytes(b, length);
+    }
+
+    @Override
+    public void writeInt(int i) throws IOException
+    {
+        ensureOpen();
+        delegate.writeInt(i);
+    }
+
+    @Override
+    public void writeShort(short i) throws IOException
+    {
+        ensureOpen();
+        delegate.writeShort(i);
+    }
+
+    @Override
+    public void writeLong(long i) throws IOException
+    {
+        ensureOpen();
+        delegate.writeLong(i);
+    }
+
+    @Override
+    public void writeString(String s) throws IOException
+    {
+        ensureOpen();
+        delegate.writeString(s);
+    }
+
+    @Override
+    public void copyBytes(DataInput input, long numBytes) throws IOException
+    {
+        ensureOpen();
+        delegate.copyBytes(input, numBytes);
+    }
+
+    @Override
+    public void writeMapOfStrings(Map<String, String> map) throws IOException
+    {
+        ensureOpen();
+        delegate.writeMapOfStrings(map);
+    }
+
+    @Override
+    public void writeSetOfStrings(Set<String> set) throws IOException
+    {
+        ensureOpen();
+        delegate.writeSetOfStrings(set);
+    }
+
+    private void ensureOpen()
+    {
+        if (delegate == null)
+        {
+            throw new AlreadyClosedException("Already closed.");
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyDirectWriterAdapter.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyDirectWriterAdapter.java
new file mode 100644
index 000000000000..620f3f35affb
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyDirectWriterAdapter.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+
+import org.apache.lucene.backward_codecs.packed.LegacyDirectWriter;
+
+/**
+ * Minimal wrapper around Lucene's LegacyDirectWriter, which doesn't share an interface with DirectWriter.
+ */
+public class LegacyDirectWriterAdapter implements DirectWriterAdapter
+{
+    private final LegacyDirectWriter delegate;
+
+    public LegacyDirectWriterAdapter(org.apache.lucene.store.DataOutput output, long numValues, int bitsPerValue)
+    {
+        this.delegate = LegacyDirectWriter.getInstance(output, numValues, bitsPerValue);
+    }
+
+    public void add(long l) throws IOException
+    {
+        delegate.add(l);
+    }
+
+    public void finish() throws IOException
+    {
+        delegate.finish();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyResettableByteBuffersIndexOutput.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyResettableByteBuffersIndexOutput.java
new file mode 100644
index 000000000000..c92028e20a48
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LegacyResettableByteBuffersIndexOutput.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.lucene.store.ByteBuffersIndexOutput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IndexInput;
+
+/***
+ * A wrapper around {@link ByteBuffersIndexOutput} that adds several methods that interact
+ * with the underlying delegate. This uses the big-endian byte ordering of Lucene 7.5 and
+ * is used to write indexes/data compatible with the readers in older Lucene versions.
+ */
+public class LegacyResettableByteBuffersIndexOutput extends ResettableByteBuffersIndexOutput
+{
+
+    private final LegacyByteBuffersIndexOutput bbio;
+    private final LegacyByteBuffersDataOutput delegate;
+
+    public LegacyResettableByteBuffersIndexOutput(int expectedSize, String name)
+    {
+        super("", name, ByteOrder.BIG_ENDIAN);
+        delegate = new LegacyByteBuffersDataOutput(expectedSize);
+        bbio = new LegacyByteBuffersIndexOutput(delegate, "", name + "-bb");
+    }
+
+    public LegacyByteBuffersDataInput toDataInput()
+    {
+        return delegate.toDataInput();
+    }
+
+    public IndexInput toIndexInput()
+    {
+        return new LegacyByteBuffersIndexInput(toDataInput(), "");
+    }
+
+    public void copyTo(IndexOutput out) throws IOException
+    {
+        delegate.copyTo(out);
+    }
+
+    public int intSize() {
+        return Math.toIntExact(bbio.getFilePointer());
+    }
+
+    public byte[] toArrayCopy() {
+        return delegate.toArrayCopy();
+    }
+
+    public void reset()
+    {
+        delegate.reset();
+    }
+
+    public String toString()
+    {
+        return "Resettable" + bbio.toString();
+    }
+
+    public void close() throws IOException
+    {
+        bbio.close();
+    }
+
+    public long getFilePointer()
+    {
+        return bbio.getFilePointer();
+    }
+
+    public long getChecksum() throws IOException
+    {
+        return bbio.getChecksum();
+    }
+
+    public void writeByte(byte b) throws IOException
+    {
+        bbio.writeByte(b);
+    }
+
+    public void writeBytes(byte[] b, int offset, int length) throws IOException
+    {
+        bbio.writeBytes(b, offset, length);
+    }
+
+    public void writeBytes(byte[] b, int length) throws IOException
+    {
+        bbio.writeBytes(b, length);
+    }
+
+    public void writeInt(int i) throws IOException
+    {
+        bbio.writeInt(i);
+    }
+
+    public void writeShort(short i) throws IOException
+    {
+        bbio.writeShort(i);
+    }
+
+    public void writeLong(long i) throws IOException
+    {
+        bbio.writeLong(i);
+    }
+
+    public void writeString(String s) throws IOException
+    {
+        bbio.writeString(s);
+    }
+
+    public void copyBytes(DataInput input, long numBytes) throws IOException
+    {
+        bbio.copyBytes(input, numBytes);
+    }
+
+    public void writeMapOfStrings(Map<String, String> map) throws IOException
+    {
+        bbio.writeMapOfStrings(map);
+    }
+
+    public void writeSetOfStrings(Set<String> set) throws IOException
+    {
+        bbio.writeSetOfStrings(set);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LuceneCompat.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LuceneCompat.java
new file mode 100644
index 000000000000..1df6d80cb798
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/LuceneCompat.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.nio.ByteOrder;
+
+import org.apache.cassandra.index.sai.disk.ModernResettableByteBuffersIndexOutput;
+import org.apache.cassandra.index.sai.utils.SeekingRandomAccessInput;
+import org.apache.lucene.backward_codecs.packed.LegacyDirectReader;
+import org.apache.lucene.backward_codecs.packed.LegacyDirectWriter;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.LongValues;
+import org.apache.lucene.util.packed.DirectReader;
+import org.apache.lucene.util.packed.DirectWriter;
+
+/**
+ * Compatibility layer for Lucene 7.5 and earlier.
+ */
+public class LuceneCompat
+{
+    public static LongValues directReaderGetInstance(SeekingRandomAccessInput slice, int bitsPerValue, long offset)
+    {
+        // Lucene 7.5 and earlier used big-endian ordering
+        return slice.order() == ByteOrder.LITTLE_ENDIAN ? DirectReader.getInstance(slice, bitsPerValue, offset)
+                                                        : LegacyDirectReader.getInstance(slice, bitsPerValue, offset);
+    }
+
+    public static DirectWriterAdapter directWriterGetInstance(ByteOrder order, DataOutput out, long numValues, int bitsPerValue)
+    {
+        // Lucene 7.5 and earlier used big-endian ordering
+        return order == ByteOrder.LITTLE_ENDIAN ? new ModernDirectWriterAdapter(out, numValues, bitsPerValue)
+                                                : new LegacyDirectWriterAdapter(out, numValues, bitsPerValue);
+    }
+
+    public static int directWriterUnsignedBitsRequired(ByteOrder order, long maxValue)
+    {
+        // Lucene 7.5 and earlier used big-endian ordering
+        return order == ByteOrder.LITTLE_ENDIAN ? DirectWriter.unsignedBitsRequired(maxValue)
+                                                : LegacyDirectWriter.unsignedBitsRequired(maxValue);
+    }
+
+    public static ResettableByteBuffersIndexOutput getResettableByteBuffersIndexOutput(ByteOrder order, int expectedSize, String name)
+    {
+        // Lucene 7.5 and earlier used big-endian ordering
+        return order == ByteOrder.LITTLE_ENDIAN ? new ModernResettableByteBuffersIndexOutput(expectedSize, name)
+                                                : new LegacyResettableByteBuffersIndexOutput(expectedSize, name);
+    }
+
+    public static ByteBuffersDataOutputAdapter getByteBuffersDataOutputAdapter(ByteOrder order, long expectedSize)
+    {
+        // Lucene 7.5 and earlier used big-endian ordering
+        return order == ByteOrder.LITTLE_ENDIAN ? new ModernByteBuffersDataOutputAdapter(expectedSize)
+                                                : new LegacyByteBuffersDataOutputAdapter(expectedSize);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ModernByteBuffersDataOutputAdapter.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ModernByteBuffersDataOutputAdapter.java
new file mode 100644
index 000000000000..eeb917c7c788
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ModernByteBuffersDataOutputAdapter.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lucene.store.ByteBuffersDataOutput;
+import org.apache.lucene.store.DataInput;
+
+/**
+ * Minimal wrapper around ByteBufferDataOutput, to allow for mixed callsites of LegacyByteBufferDataOutput
+ * and ByteBufferDataOutput.
+ */
+public class ModernByteBuffersDataOutputAdapter extends ByteBuffersDataOutputAdapter
+{
+    private ByteBuffersDataOutput wrapped;
+
+    public ModernByteBuffersDataOutputAdapter(long expectedSize)
+    {
+        wrapped = new ByteBuffersDataOutput(expectedSize);
+    }
+
+    @Override
+    public void reset()
+    {
+        wrapped.reset();
+    }
+
+    @Override
+    public long size()
+    {
+        return wrapped.size();
+    }
+
+    @Override
+    public byte[] toArrayCopy()
+    {
+        return wrapped.toArrayCopy();
+    }
+
+    @Override
+    public void writeBytes(byte[] b, int length) throws IOException
+    {
+        wrapped.writeBytes(b, length);
+    }
+
+    @Override
+    public void writeInt(int i) throws IOException
+    {
+        wrapped.writeInt(i);
+    }
+
+    @Override
+    public void writeShort(short i) throws IOException
+    {
+        wrapped.writeShort(i);
+    }
+
+    @Override
+    public void writeLong(long i) throws IOException
+    {
+        wrapped.writeLong(i);
+    }
+
+    @Override
+    public void writeString(String s) throws IOException
+    {
+        wrapped.writeString(s);
+    }
+
+    @Override
+    public void copyBytes(DataInput input, long numBytes) throws IOException
+    {
+        wrapped.copyBytes(input, numBytes);
+    }
+
+    @Override
+    public void writeMapOfStrings(Map<String, String> map) throws IOException
+    {
+        wrapped.writeMapOfStrings(map);
+    }
+
+    @Override
+    public void writeSetOfStrings(Set<String> set) throws IOException
+    {
+        wrapped.writeSetOfStrings(set);
+    }
+
+    @Override
+    public void writeByte(byte b) throws IOException
+    {
+        wrapped.writeByte(b);
+    }
+
+    @Override
+    public void writeBytes(byte[] src, int offset, int length) throws IOException
+    {
+        wrapped.writeBytes(src, offset, length);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ModernDirectWriterAdapter.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ModernDirectWriterAdapter.java
new file mode 100644
index 000000000000..ae822cc0aab1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ModernDirectWriterAdapter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.packed.DirectWriter;
+
+/**
+ * Minimal wrapper arount DirectWriter to allow it to be used in a common interface with LegacyDirectWriter.
+ */
+public class ModernDirectWriterAdapter implements DirectWriterAdapter
+{
+    private final org.apache.lucene.util.packed.DirectWriter delegate;
+
+    public ModernDirectWriterAdapter(DataOutput output, long numValues, int bitsPerValue)
+    {
+        this.delegate = DirectWriter.getInstance(output, numValues, bitsPerValue);
+    }
+
+    @Override
+    public void add(long l) throws IOException
+    {
+        delegate.add(l);
+    }
+
+    @Override
+    public void finish() throws IOException
+    {
+        delegate.finish();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/MutablePointValues.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/MutablePointValues.java
new file mode 100644
index 000000000000..d233e56b74ae
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/MutablePointValues.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.util.BytesRef;
+
+/** {@link PointValues} whose order of points can be changed.
+ *  This class is useful for codecs to optimize flush.
+ *  @lucene.internal */
+public abstract class MutablePointValues extends PointValues {
+
+  /** Sole constructor. */
+  protected MutablePointValues() {}
+
+  /** Set {@code packedValue} with a reference to the packed bytes of the i-th value. */
+  public abstract void getValue(int i, BytesRef packedValue);
+
+  /** Get the k-th byte of the i-th value. */
+  public abstract byte getByteAt(int i, int k);
+
+  /** Return the doc ID of the i-th value. */
+  public abstract int getDocID(int i);
+
+  /** Swap the i-th and j-th values. */
+  public abstract void swap(int i, int j);
+
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/MutablePointsReaderUtils.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/MutablePointsReaderUtils.java
new file mode 100644
index 000000000000..08d9e96c3585
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/MutablePointsReaderUtils.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.util.Arrays;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntroSelector;
+import org.apache.lucene.util.IntroSorter;
+import org.apache.lucene.util.MSBRadixSorter;
+import org.apache.lucene.util.RadixSelector;
+import org.apache.lucene.util.Selector;
+import org.apache.lucene.util.packed.PackedInts;
+
+/** Utility APIs for sorting and partitioning buffered points.
+ *
+ * @lucene.internal */
+public final class MutablePointsReaderUtils {
+
+  MutablePointsReaderUtils() {}
+
+  /** Sort the given {@link MutablePointValues} based on its packed value then doc ID. */
+  public static void sort(int maxDoc, int packedBytesLength,
+                          MutablePointValues reader, int from, int to) {
+    final int bitsPerDocId = PackedInts.bitsRequired(maxDoc - 1);
+    new MSBRadixSorter(packedBytesLength + (bitsPerDocId + 7) / 8) {
+
+      @Override
+      protected void swap(int i, int j) {
+        reader.swap(i, j);
+      }
+
+      @Override
+      protected int byteAt(int i, int k) {
+        if (k < packedBytesLength) {
+          return Byte.toUnsignedInt(reader.getByteAt(i, k));
+        } else {
+          final int shift = bitsPerDocId - ((k - packedBytesLength + 1) << 3);
+          return (reader.getDocID(i) >>> Math.max(0, shift)) & 0xff;
+        }
+      }
+
+      @Override
+      protected org.apache.lucene.util.Sorter getFallbackSorter(int k) {
+        return new IntroSorter() {
+
+          final BytesRef pivot = new BytesRef();
+          final BytesRef scratch = new BytesRef();
+          int pivotDoc;
+
+          @Override
+          protected void swap(int i, int j) {
+            reader.swap(i, j);
+          }
+
+          @Override
+          protected void setPivot(int i) {
+            reader.getValue(i, pivot);
+            pivotDoc = reader.getDocID(i);
+          }
+
+          @Override
+          protected int comparePivot(int j) {
+            if (k < packedBytesLength) {
+              reader.getValue(j, scratch);
+              int cmp = Arrays.compareUnsigned(pivot.bytes, pivot.offset + k, pivot.offset + k + packedBytesLength - k, scratch.bytes, scratch.offset + k, scratch.offset + k + packedBytesLength - k);
+              if (cmp != 0) {
+                return cmp;
+              }
+            }
+            return pivotDoc - reader.getDocID(j);
+          }
+        };
+      }
+
+    }.sort(from, to);
+  }
+
+  /** Sort points on the given dimension. */
+  public static void sortByDim(int sortedDim, int bytesPerDim, int[] commonPrefixLengths,
+                               MutablePointValues reader, int from, int to,
+                               BytesRef scratch1, BytesRef scratch2) {
+
+    // No need for a fancy radix sort here, this is called on the leaves only so
+    // there are not many values to sort
+    final int offset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];
+    final int numBytesToCompare = bytesPerDim - commonPrefixLengths[sortedDim];
+    new IntroSorter() {
+
+      final BytesRef pivot = scratch1;
+      int pivotDoc = -1;
+
+      @Override
+      protected void swap(int i, int j) {
+        reader.swap(i, j);
+      }
+
+      @Override
+      protected void setPivot(int i) {
+        reader.getValue(i, pivot);
+        pivotDoc = reader.getDocID(i);
+      }
+
+      @Override
+      protected int comparePivot(int j) {
+        reader.getValue(j, scratch2);
+        int cmp = Arrays.compareUnsigned(pivot.bytes, pivot.offset + offset, pivot.offset + offset + numBytesToCompare, scratch2.bytes, scratch2.offset + offset, scratch2.offset + offset + numBytesToCompare);
+        if (cmp == 0) {
+          cmp = pivotDoc - reader.getDocID(j);
+        }
+        return cmp;
+      }
+    }.sort(from, to);
+  }
+
+  /** Partition points around {@code mid}. All values on the left must be less
+   *  than or equal to it and all values on the right must be greater than or
+   *  equal to it. */
+  public static void partition(int maxDoc, int splitDim, int bytesPerDim, int commonPrefixLen,
+                               MutablePointValues reader, int from, int to, int mid,
+                               BytesRef scratch1, BytesRef scratch2) {
+    final int offset = splitDim * bytesPerDim + commonPrefixLen;
+    final int cmpBytes = bytesPerDim - commonPrefixLen;
+    final int bitsPerDocId = PackedInts.bitsRequired(maxDoc - 1);
+    new RadixSelector(cmpBytes + (bitsPerDocId + 7) / 8) {
+
+      @Override
+      protected Selector getFallbackSelector(int k) {
+        return new IntroSelector() {
+
+          final BytesRef pivot = scratch1;
+          int pivotDoc;
+
+          @Override
+          protected void swap(int i, int j) {
+            reader.swap(i, j);
+          }
+
+          @Override
+          protected void setPivot(int i) {
+            reader.getValue(i, pivot);
+            pivotDoc = reader.getDocID(i);
+          }
+
+          @Override
+          protected int comparePivot(int j) {
+            if (k < cmpBytes) {
+              reader.getValue(j, scratch2);
+              int cmp = Arrays.compareUnsigned(pivot.bytes, pivot.offset + offset + k, pivot.offset + offset + k + cmpBytes - k, scratch2.bytes, scratch2.offset + offset + k, scratch2.offset + offset + k + cmpBytes - k);
+              if (cmp != 0) {
+                return cmp;
+              }
+            }
+            return pivotDoc - reader.getDocID(j);
+          }
+        };
+      }
+
+      @Override
+      protected void swap(int i, int j) {
+        reader.swap(i, j);
+      }
+
+      @Override
+      protected int byteAt(int i, int k) {
+        if (k < cmpBytes) {
+          return Byte.toUnsignedInt(reader.getByteAt(i, offset + k));
+        } else {
+          final int shift = bitsPerDocId - ((k - cmpBytes + 1) << 3);
+          return (reader.getDocID(i) >>> Math.max(0, shift)) & 0xff;
+        }
+      }
+    }.select(from, to, mid);
+  }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ResettableByteBuffersIndexOutput.java b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ResettableByteBuffersIndexOutput.java
new file mode 100644
index 000000000000..be8ec7b62eff
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/oldlucene/ResettableByteBuffersIndexOutput.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+
+/**
+ * A wrapper around byte-buffer backed IndexOutputs that adds several methods that interact with the underlying
+ * delegate.
+ */
+
+public abstract class ResettableByteBuffersIndexOutput extends IndexOutput
+{
+
+    public ResettableByteBuffersIndexOutput(String resourceDescription, String name, ByteOrder order)
+    {
+        super(resourceDescription, name, order);
+    }
+
+    public abstract void copyTo(IndexOutput out) throws IOException;
+
+    public abstract byte[] toArrayCopy();
+
+    public abstract int intSize();
+
+    public abstract void reset();
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/ColumnCompletionMarkerUtil.java b/src/java/org/apache/cassandra/index/sai/disk/v1/ColumnCompletionMarkerUtil.java
deleted file mode 100644
index 6d3a3dd92795..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/ColumnCompletionMarkerUtil.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.io.IOException;
-
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-
-/**
- * Utility class for creating and reading the column completion marker, {@link IndexComponent#COLUMN_COMPLETION_MARKER}.
- * </p>
- * The file has a header and a footer, as written by {@link SAICodecUtils#writeHeader(IndexOutput)} and
- * {@link SAICodecUtils#writeFooter(IndexOutput)}. The only content of the file is a single byte indicating whether the
- * column index is empty or not. If the index is empty the completion marker will be the only per-index component.
- */
-public class ColumnCompletionMarkerUtil
-{
-    private static final byte EMPTY = (byte) 1;
-    private static final byte NOT_EMPTY = (byte) 0;
-
-    /**
-     * Creates a column index completion marker for the specified column index, storing in it whether the index is empty.
-     *
-     * @param descriptor the index descriptor
-     * @param indexIdentifier the column index identifier
-     * @param isEmpty whether the index is empty
-     */
-    public static void create(IndexDescriptor descriptor, IndexIdentifier indexIdentifier, boolean isEmpty) throws IOException
-    {
-        try (IndexOutputWriter output = descriptor.openPerIndexOutput(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier))
-        {
-            SAICodecUtils.writeHeader(output);
-            output.writeByte(isEmpty ? EMPTY : NOT_EMPTY);
-            SAICodecUtils.writeFooter(output);
-        }
-    }
-
-    /**
-     * Reads the column index completion marker and returns whether if the index is empty.
-     *
-     * @param descriptor the index descriptor
-     * @param indexIdentifier the column index identifier
-     * @return {@code true} if the index is empty, {@code false} otherwise.
-     */
-    public static boolean isEmptyIndex(IndexDescriptor descriptor, IndexIdentifier indexIdentifier) throws IOException
-    {
-        try (IndexInput input = descriptor.openPerIndexInput(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier))
-        {
-            SAICodecUtils.checkHeader(input); // consume header
-            return input.readByte() == EMPTY;
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/DirectReaders.java b/src/java/org/apache/cassandra/index/sai/disk/v1/DirectReaders.java
deleted file mode 100644
index 7b2353af6f41..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/DirectReaders.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.util.function.Supplier;
-
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.IndexInput;
-
-public class DirectReaders
-{
-    public static void checkBitsPerValue(int valuesBitsPerValue, IndexInput input, Supplier<String> source) throws CorruptIndexException
-    {
-        if (valuesBitsPerValue > 64)
-        {
-            String message = String.format("%s is corrupted: Bits per value for block offsets must be no more than 64 and is %d", source.get(), valuesBitsPerValue);
-            throw new CorruptIndexException(message, input);
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/DocLengthsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/DocLengthsReader.java
new file mode 100644
index 000000000000..0515ac48bc67
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/DocLengthsReader.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+
+@NotThreadSafe
+public class DocLengthsReader implements Closeable
+{
+    private final IndexInputReader input;
+    private final SegmentMetadata.ComponentMetadata componentMetadata;
+
+    public DocLengthsReader(FileHandle fileHandle, SegmentMetadata.ComponentMetadata componentMetadata)
+    {
+        this.input = IndexFileUtils.instance().openInput(fileHandle);
+        this.componentMetadata = componentMetadata;
+    }
+
+    public int get(int rowID) throws IOException
+    {
+        // Account for header size in offset calculation
+        long position = componentMetadata.offset + (long) rowID * Integer.BYTES;
+        if (position >= componentMetadata.offset + componentMetadata.length)
+            return 0;
+        input.seek(position);
+        return input.readInt();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        FileUtils.close(input);
+    }
+}
+
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/IndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/IndexSearcher.java
new file mode 100644
index 000000000000..3106c4abfa33
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/IndexSearcher.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import com.google.common.util.concurrent.Runnables;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.IndexSearcherContext;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.PostingListKeyRangeIterator;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.iterators.RowIdToPrimaryKeyWithSortKeyIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithByteComparable;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RowIdWithMeta;
+import org.apache.cassandra.index.sai.utils.SegmentOrdering;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.SortingIterator;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Abstract reader for individual segments of an on-disk index.
+ *
+ * Accepts shared resources (token/offset file readers), and uses them to perform lookups against on-disk data
+ * structures.
+ */
+public abstract class IndexSearcher implements Closeable, SegmentOrdering
+{
+    protected final PrimaryKeyMap.Factory primaryKeyMapFactory;
+    final PerIndexFiles indexFiles;
+    protected final SegmentMetadata metadata;
+    protected final IndexContext indexContext;
+
+    protected final ColumnFilter columnFilter;
+
+    protected IndexSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
+                            PerIndexFiles perIndexFiles,
+                            SegmentMetadata segmentMetadata,
+                            IndexContext indexContext)
+    {
+        this.primaryKeyMapFactory = primaryKeyMapFactory;
+        this.indexFiles = perIndexFiles;
+        this.metadata = segmentMetadata;
+        this.indexContext = indexContext;
+        columnFilter = ColumnFilter.selection(RegularAndStaticColumns.of(indexContext.getDefinition()));
+    }
+
+    /**
+     * @return memory usage of underlying on-disk data structure
+     */
+    public abstract long indexFileCacheSize();
+
+    /**
+     * Search on-disk index synchronously.  Used for WHERE clause predicates, including BOUNDED_ANN.
+     *
+     * @param expression   to filter on disk index
+     * @param keyRange     key range specific in read command, used by ANN index
+     * @param queryContext to track per sstable cache and per query metrics
+     * @param defer        create the iterator in a deferred state
+     * @return {@link KeyRangeIterator} that matches given expression
+     */
+    public abstract KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, boolean defer) throws IOException;
+
+    /**
+     * Order the rows by the given Orderer.  Used for ORDER BY clause when
+     * (1) the WHERE predicate is either a partition restriction or a range restriction on the index,
+     * (2) there is no WHERE predicate, or
+     * (3) the planner determines it is better to post-filter the ordered results by the predicate.
+     *
+     * @param orderer      the object containing the ordering logic
+     * @param slice        optional predicate to get a slice of the index
+     * @param keyRange     key range specific in read command, used by ANN index
+     * @param queryContext to track per sstable cache and per query metrics
+     * @param limit        the initial num of rows to returned, used by ANN index. More rows may be requested if filtering throws away more than expected!
+     * @return an iterator of {@link PrimaryKeyWithSortKey} in score order
+     */
+    public abstract CloseableIterator<PrimaryKeyWithSortKey> orderBy(Orderer orderer, Expression slice, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, int limit) throws IOException;
+
+    /**
+     * Order the rows by the given Orderer.  Used for ORDER BY clause when the WHERE predicates
+     * have been applied first, yielding a list of primary keys.  Again, `limit` is a planner hint for ANN to determine
+     * the initial number of results returned, not a maximum.
+     */
+    @Override
+    public CloseableIterator<PrimaryKeyWithSortKey> orderResultsBy(SSTableReader reader, QueryContext context, List<PrimaryKey> keys, Orderer orderer, int limit) throws IOException
+    {
+        return SortingIterator.createCloseable(
+            orderer.getComparator(),
+            keys,
+            key ->
+            {
+                var slices = Slices.with(indexContext.comparator(), Slice.make(key.clustering()));
+                // TODO if we end up needing to read the row still, is it better to store offset and use reader.unfilteredAt?
+                try (var iter = reader.rowIterator(key.partitionKey(), slices, columnFilter, false, SSTableReadsListener.NOOP_LISTENER))
+                {
+                    if (iter.hasNext())
+                    {
+                        var row = (Row) iter.next();
+                        assert !iter.hasNext();
+                        var cell = row.getCell(indexContext.getDefinition());
+                        if (cell == null)
+                            return null;
+                        // We encode the bytes to make sure they compare correctly.
+                        var byteComparable = encode(cell.buffer());
+                        return new PrimaryKeyWithByteComparable(indexContext, reader.descriptor.id, key, byteComparable);
+                    }
+                }
+                return null;
+            },
+            Runnables.doNothing()
+        );
+    }
+
+    private ByteComparable encode(ByteBuffer input)
+    {
+        return indexContext.isLiteral() ? v -> ByteSource.preencoded(input)
+                                        : v -> TypeUtil.asComparableBytes(input, indexContext.getValidator(), v);
+    }
+
+    protected KeyRangeIterator toPrimaryKeyIterator(PostingList postingList, QueryContext queryContext) throws IOException
+    {
+        if (postingList == null || postingList.size() == 0)
+            return KeyRangeIterator.empty();
+
+        IndexSearcherContext searcherContext = new IndexSearcherContext(metadata.minKey,
+                                                                        metadata.maxKey,
+                                                                        metadata.minSSTableRowId,
+                                                                        metadata.maxSSTableRowId,
+                                                                        metadata.segmentRowIdOffset,
+                                                                        queryContext,
+                                                                        postingList);
+        return new PostingListKeyRangeIterator(indexContext, primaryKeyMapFactory.newPerSSTablePrimaryKeyMap(), searcherContext);
+    }
+
+    protected CloseableIterator<PrimaryKeyWithSortKey> toMetaSortedIterator(CloseableIterator<? extends RowIdWithMeta> rowIdIterator, QueryContext queryContext) throws IOException
+    {
+        if (rowIdIterator == null || !rowIdIterator.hasNext())
+        {
+            FileUtils.closeQuietly(rowIdIterator);
+            return CloseableIterator.emptyIterator();
+        }
+
+        IndexSearcherContext searcherContext = new IndexSearcherContext(metadata.minKey,
+                                                                        metadata.maxKey,
+                                                                        metadata.minSSTableRowId,
+                                                                        metadata.maxSSTableRowId,
+                                                                        metadata.segmentRowIdOffset,
+                                                                        queryContext,
+                                                                        null);
+        var pkm = primaryKeyMapFactory.newPerSSTablePrimaryKeyMap();
+        return new RowIdToPrimaryKeyWithSortKeyIterator(indexContext,
+                                                        pkm.getSSTableId(),
+                                                        rowIdIterator,
+                                                        pkm,
+                                                        searcherContext);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java b/src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java
index fe9be1c98df1..9b6ee7483fd2 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java
@@ -21,11 +21,13 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
-import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.sai.disk.v1.vector.OptimizeFor;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.disk.vector.VectorSourceModel;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_VECTOR_SEARCH_MAX_TOP_K;
 
@@ -34,52 +36,139 @@
  */
 public class IndexWriterConfig
 {
-    public static final String MAXIMUM_NODE_CONNECTIONS = "maximum_node_connections";
-    public static final int MAXIMUM_MAXIMUM_NODE_CONNECTIONS = 512;
-    public static final int DEFAULT_MAXIMUM_NODE_CONNECTIONS = 16;
+    public static final String POSTING_LIST_LVL_MIN_LEAVES = "bkd_postings_min_leaves";
+    public static final String POSTING_LIST_LVL_SKIP_OPTION = "bkd_postings_skip";
+
+    private static final int DEFAULT_POSTING_LIST_MIN_LEAVES = 64;
+    private static final int DEFAULT_POSTING_LIST_LVL_SKIP = 3;
 
+    public static final String MAXIMUM_NODE_CONNECTIONS = "maximum_node_connections";
     public static final String CONSTRUCTION_BEAM_WIDTH = "construction_beam_width";
+    public static final String NEIGHBORHOOD_OVERFLOW = "neighborhood_overflow";
+    public static final String ALPHA = "alpha";
+    public static final String ENABLE_HIERARCHY = "enable_hierarchy";
+    public static final String SIMILARITY_FUNCTION = "similarity_function";
+    public static final String SOURCE_MODEL = "source_model";
+    public static final String OPTIMIZE_FOR = "optimize_for"; // unused, retained for compatibility w/ old schemas
+
+    public static final int MAXIMUM_MAXIMUM_NODE_CONNECTIONS = 512;
     public static final int MAXIMUM_CONSTRUCTION_BEAM_WIDTH = 3200;
+
+    public static final int DEFAULT_MAXIMUM_NODE_CONNECTIONS = 16;
     public static final int DEFAULT_CONSTRUCTION_BEAM_WIDTH = 100;
+    public static final boolean DEFAULT_ENABLE_HIERARCHY = false;
+
+    public static final int MAX_TOP_K = SAI_VECTOR_SEARCH_MAX_TOP_K.getInt();
 
-    public static final String SIMILARITY_FUNCTION = "similarity_function";
-    public static final VectorSimilarityFunction DEFAULT_SIMILARITY_FUNCTION = VectorSimilarityFunction.COSINE;
     public static final String validSimilarityFunctions = Arrays.stream(VectorSimilarityFunction.values())
                                                                 .map(Enum::name)
                                                                 .collect(Collectors.joining(", "));
 
-    public static final String OPTIMIZE_FOR = "optimize_for";
-    private static final OptimizeFor DEFAULT_OPTIMIZE_FOR = OptimizeFor.LATENCY;
-    private static final String validOptimizeFor = Arrays.stream(OptimizeFor.values())
-                                                         .map(Enum::name)
-                                                         .collect(Collectors.joining(", "));
+    private static final VectorSourceModel DEFAULT_SOURCE_MODEL = VectorSourceModel.OTHER;
 
-    public static final int MAX_TOP_K = SAI_VECTOR_SEARCH_MAX_TOP_K.getInt();
+    private static final IndexWriterConfig EMPTY_CONFIG = new IndexWriterConfig(null, -1, -1, -1, -1, null, DEFAULT_SOURCE_MODEL);
 
-    private static final IndexWriterConfig EMPTY_CONFIG = new IndexWriterConfig(-1, -1, null, null);
+    /**
+     * Fully qualified index name, in the format {@literal "<keyspace>.<table>.<index_name>"}.
+     */
+    private final String indexName;
 
-    // The maximum number of outgoing connections a node can have in a graph.
-    private final int maximumNodeConnections;
+    /**
+     * Skip, or the sampling interval, for selecting a bkd tree level that is eligible for an auxiliary posting list.
+     * Sampling starts from 0, but bkd tree root node is at level 1. For skip = 4, eligible levels are 4, 8, 12, etc (no
+     * level 0, because there is no node at level 0).
+     */
+    private final int bkdPostingsSkip;
 
-    // The size of the beam search used when finding nearest neighbours.
-    private final int constructionBeamWidth;
+    /**
+     * Min. number of reachable leaves for a given node to be eligible for an auxiliary posting list.
+     */
+    private final int bkdPostingsMinLeaves;
 
-    // Used to determine the search to determine the topK results. The score returned is used to order the topK results.
+    private final int maximumNodeConnections;
+    private final int constructionBeamWidth;
     private final VectorSimilarityFunction similarityFunction;
+    private final VectorSourceModel sourceModel;
+
+    private final Float neighborhoodOverflow; // default varies for in memory/compaction build
+    private final Float alpha; // default varies for in memory/compaction build
+    private final boolean enableHierarchy; // defaults to false
 
-    private final OptimizeFor optimizeFor;
+    public IndexWriterConfig(String indexName,
+                             int bkdPostingsSkip,
+                             int bkdPostingsMinLeaves)
+    {
+        this(indexName,
+             bkdPostingsSkip,
+             bkdPostingsMinLeaves,
+             DEFAULT_MAXIMUM_NODE_CONNECTIONS,
+             DEFAULT_CONSTRUCTION_BEAM_WIDTH,
+             DEFAULT_SOURCE_MODEL.defaultSimilarityFunction,
+             DEFAULT_SOURCE_MODEL
+        );
+    }
 
-    public IndexWriterConfig(int maximumNodeConnections,
+    public IndexWriterConfig(String indexName,
+                             int bkdPostingsSkip,
+                             int bkdPostingsMinLeaves,
+                             int maximumNodeConnections,
                              int constructionBeamWidth,
                              VectorSimilarityFunction similarityFunction,
-                             OptimizeFor optimizerFor)
+                             VectorSourceModel sourceModel)
     {
+        this(indexName, bkdPostingsSkip, bkdPostingsMinLeaves, maximumNodeConnections, constructionBeamWidth,
+             similarityFunction, sourceModel, null, null, false);
+    }
+
+    public IndexWriterConfig(String indexName,
+                             int bkdPostingsSkip,
+                             int bkdPostingsMinLeaves,
+                             int maximumNodeConnections,
+                             int constructionBeamWidth,
+                             VectorSimilarityFunction similarityFunction,
+                             VectorSourceModel sourceModel,
+                             Float neighborhoodOverflow,
+                             Float alpha,
+                             boolean enableHierarchy)
+    {
+        this.indexName = indexName;
+        this.bkdPostingsSkip = bkdPostingsSkip;
+        this.bkdPostingsMinLeaves = bkdPostingsMinLeaves;
         this.maximumNodeConnections = maximumNodeConnections;
         this.constructionBeamWidth = constructionBeamWidth;
         this.similarityFunction = similarityFunction;
-        this.optimizeFor = optimizerFor;
+        this.sourceModel = sourceModel;
+        this.neighborhoodOverflow = neighborhoodOverflow;
+        this.alpha = alpha;
+        this.enableHierarchy = enableHierarchy;
+    }
+
+    public String getIndexName()
+    {
+        return indexName;
+    }
+
+    public int getBkdPostingsMinLeaves()
+    {
+        return bkdPostingsMinLeaves;
+    }
+
+    public int getBkdPostingsSkip()
+    {
+        return bkdPostingsSkip;
     }
 
+    public int getAnnMaxDegree()
+    {
+        // For historical reasons (Lucene doubled the maximum node connections for its HNSW),
+        // maximumNodeConnections represents half of the graph degree, so double it
+        return 2 * maximumNodeConnections;
+    }
+
+    /** you should probably use getAnnMaxDegree instead */
+    /** @deprecated See https://github.com/datastax/cassandra/pull/1110 */
+    @Deprecated(since = "5.0")
+    @VisibleForTesting
     public int getMaximumNodeConnections()
     {
         return maximumNodeConnections;
@@ -95,31 +184,90 @@ public VectorSimilarityFunction getSimilarityFunction()
         return similarityFunction;
     }
 
-    public OptimizeFor getOptimizeFor()
+    public VectorSourceModel getSourceModel()
+    {
+        return sourceModel;
+    }
+
+    public float getNeighborhoodOverflow(float defaultValue)
+    {
+        return neighborhoodOverflow == null ? defaultValue : neighborhoodOverflow;
+    }
+
+    public float getAlpha(float defaultValue)
+    {
+        return alpha == null ? defaultValue : alpha;
+    }
+
+    public boolean isHierarchyEnabled()
     {
-        return optimizeFor;
+        return enableHierarchy;
     }
 
-    public static IndexWriterConfig fromOptions(String indexName, IndexTermType indexTermType, Map<String, String> options)
+    public static IndexWriterConfig fromOptions(String indexName, AbstractType<?> type, Map<String, String> options)
     {
+        int minLeaves = DEFAULT_POSTING_LIST_MIN_LEAVES;
+        int skip = DEFAULT_POSTING_LIST_LVL_SKIP;
         int maximumNodeConnections = DEFAULT_MAXIMUM_NODE_CONNECTIONS;
         int queueSize = DEFAULT_CONSTRUCTION_BEAM_WIDTH;
-        VectorSimilarityFunction similarityFunction = DEFAULT_SIMILARITY_FUNCTION;
-        OptimizeFor optimizeFor = DEFAULT_OPTIMIZE_FOR;
+        VectorSourceModel sourceModel = DEFAULT_SOURCE_MODEL;
+        VectorSimilarityFunction similarityFunction = sourceModel.defaultSimilarityFunction; // don't leave null in case no options at all are given
+
+        Float neighborhoodOverflow = null;
+        Float alpha = null;
+        boolean enableHierarchy = DEFAULT_ENABLE_HIERARCHY;
 
-        if (options.get(MAXIMUM_NODE_CONNECTIONS) != null ||
-            options.get(CONSTRUCTION_BEAM_WIDTH) != null ||
-            options.get(SIMILARITY_FUNCTION) != null ||
-            options.get(OPTIMIZE_FOR) != null)
+        if (options.get(POSTING_LIST_LVL_MIN_LEAVES) != null || options.get(POSTING_LIST_LVL_SKIP_OPTION) != null)
         {
-            if (!indexTermType.isVector())
-                throw new InvalidRequestException(String.format("CQL type %s cannot have vector options", indexTermType.asCQL3Type()));
+            if (TypeUtil.isLiteral(type))
+            {
+                throw new InvalidRequestException(String.format("CQL type %s cannot have auxiliary posting lists on index %s.", type.asCQL3Type(), indexName));
+            }
 
-            if (options.containsKey(MAXIMUM_NODE_CONNECTIONS))
+            for (Map.Entry<String, String> entry : options.entrySet())
             {
-                if (!CassandraRelevantProperties.SAI_VECTOR_ALLOW_CUSTOM_PARAMETERS.getBoolean())
-                    throw new InvalidRequestException(String.format("Maximum node connections cannot be set without enabling %s", CassandraRelevantProperties.SAI_VECTOR_ALLOW_CUSTOM_PARAMETERS.name()));
+                switch (entry.getKey())
+                {
+                    case POSTING_LIST_LVL_MIN_LEAVES:
+                    {
+                        minLeaves = Integer.parseInt(entry.getValue());
+
+                        if (minLeaves < 1)
+                        {
+                            throw new InvalidRequestException(String.format("Posting list min. leaves count can't be less than 1 on index %s.", indexName));
+                        }
+
+                        break;
+                    }
+
+                    case POSTING_LIST_LVL_SKIP_OPTION:
+                    {
+                        skip = Integer.parseInt(entry.getValue());
+
+                        if (skip < 1)
+                        {
+                            throw new InvalidRequestException(String.format("Posting list skip can't be less than 1 on index %s.", indexName));
+                        }
+
+                        break;
+                    }
+                }
+            }
+        }
+        else if (options.get(MAXIMUM_NODE_CONNECTIONS) != null ||
+                 options.get(CONSTRUCTION_BEAM_WIDTH) != null ||
+                 options.get(OPTIMIZE_FOR) != null ||
+                 options.get(SIMILARITY_FUNCTION) != null ||
+                 options.get(SOURCE_MODEL) != null ||
+                 options.get(NEIGHBORHOOD_OVERFLOW) != null ||
+                 options.get(ALPHA) != null ||
+                 options.get(ENABLE_HIERARCHY) != null)
+        {
+            if (!type.isVector())
+                throw new InvalidRequestException(String.format("CQL type %s cannot have vector options", type.asCQL3Type()));
 
+            if (options.containsKey(MAXIMUM_NODE_CONNECTIONS))
+            {
                 try
                 {
                     maximumNodeConnections = Integer.parseInt(options.get(MAXIMUM_NODE_CONNECTIONS));
@@ -134,9 +282,6 @@ public static IndexWriterConfig fromOptions(String indexName, IndexTermType inde
             }
             if (options.containsKey(CONSTRUCTION_BEAM_WIDTH))
             {
-                if (!CassandraRelevantProperties.SAI_VECTOR_ALLOW_CUSTOM_PARAMETERS.getBoolean())
-                    throw new InvalidRequestException(String.format("Construction beam width cannot be set without enabling %s", CassandraRelevantProperties.SAI_VECTOR_ALLOW_CUSTOM_PARAMETERS.name()));
-
                 try
                 {
                     queueSize = Integer.parseInt(options.get(CONSTRUCTION_BEAM_WIDTH));
@@ -149,6 +294,22 @@ public static IndexWriterConfig fromOptions(String indexName, IndexTermType inde
                 if (queueSize <= 0 || queueSize > MAXIMUM_CONSTRUCTION_BEAM_WIDTH)
                     throw new InvalidRequestException(String.format("Construction beam width for index %s cannot be <= 0 or > %s, was %s", indexName, MAXIMUM_CONSTRUCTION_BEAM_WIDTH, queueSize));
             }
+            if (options.containsKey(SOURCE_MODEL))
+            {
+                String option = options.get(SOURCE_MODEL).toUpperCase().replace("-", "_");
+                try
+                {
+                    sourceModel = VectorSourceModel.valueOf(option);
+                }
+                catch (IllegalArgumentException e)
+                {
+                    var validSourceModels = Arrays.stream(VectorSourceModel.values())
+                                                  .map(Enum::name)
+                                                  .collect(Collectors.joining(", "));
+                    throw new InvalidRequestException(String.format("source_model '%s' was not recognized for index %s. Valid values are: %s",
+                                                                    option, indexName, validSourceModels));
+                }
+            }
             if (options.containsKey(SIMILARITY_FUNCTION))
             {
                 String option = options.get(SIMILARITY_FUNCTION).toUpperCase();
@@ -162,21 +323,67 @@ public static IndexWriterConfig fromOptions(String indexName, IndexTermType inde
                                                                     option, indexName, validSimilarityFunctions));
                 }
             }
-            if (options.containsKey(OPTIMIZE_FOR))
+            else
+            {
+                similarityFunction = sourceModel.defaultSimilarityFunction;
+            }
+
+            if (options.containsKey(NEIGHBORHOOD_OVERFLOW))
             {
-                String option = options.get(OPTIMIZE_FOR).toUpperCase();
                 try
                 {
-                    optimizeFor = OptimizeFor.valueOf(option);
+                    neighborhoodOverflow = Float.parseFloat(options.get(NEIGHBORHOOD_OVERFLOW));
+                    if (neighborhoodOverflow < 1.0f)
+                        throw new InvalidRequestException(String.format("Neighborhood overflow for index %s must be >= 1.0, was %s",
+                                                                      indexName, neighborhoodOverflow));
                 }
-                catch (IllegalArgumentException e)
+                catch (NumberFormatException e)
                 {
-                    throw new InvalidRequestException(String.format("optimize_for '%s' was not recognized for index %s. Valid values are: %s",
-                                                                    option, indexName, validOptimizeFor));
+                    throw new InvalidRequestException(String.format("Neighborhood overflow %s is not a valid float for index %s",
+                                                                    options.get(NEIGHBORHOOD_OVERFLOW), indexName));
                 }
             }
+
+            if (options.containsKey(ALPHA))
+            {
+                try
+                {
+                    alpha = Float.parseFloat(options.get(ALPHA));
+                    if (alpha <= 0)
+                        throw new InvalidRequestException(String.format("Alpha for index %s must be > 0, was %s", 
+                                                                      indexName, alpha));
+                }
+                catch (NumberFormatException e)
+                {
+                    throw new InvalidRequestException(String.format("Alpha %s is not a valid float for index %s",
+                                                                  options.get(ALPHA), indexName));
+                }
+            }
+
+            if (options.containsKey(ENABLE_HIERARCHY))
+            {
+                String value = options.get(ENABLE_HIERARCHY).toLowerCase();
+                if (!value.equals("true") && !value.equals("false"))
+                    throw new InvalidRequestException(String.format("Enable hierarchy must be 'true' or 'false' for index %s, was '%s'",
+                                                                  indexName, value));
+                enableHierarchy = Boolean.parseBoolean(value);
+            }
         }
-        return new IndexWriterConfig(maximumNodeConnections, queueSize, similarityFunction, optimizeFor);
+
+        return new IndexWriterConfig(indexName, skip, minLeaves, maximumNodeConnections, queueSize,
+                                   similarityFunction, sourceModel, neighborhoodOverflow, alpha, enableHierarchy);
+    }
+
+    public static IndexWriterConfig defaultConfig(String indexName)
+    {
+        return new IndexWriterConfig(indexName,
+                                     DEFAULT_POSTING_LIST_LVL_SKIP,
+                                     DEFAULT_POSTING_LIST_MIN_LEAVES,
+                                     DEFAULT_MAXIMUM_NODE_CONNECTIONS,
+                                     DEFAULT_CONSTRUCTION_BEAM_WIDTH,
+                                     DEFAULT_SOURCE_MODEL.defaultSimilarityFunction,
+                                     DEFAULT_SOURCE_MODEL
+        );
     }
 
     public static IndexWriterConfig emptyConfig()
@@ -187,10 +394,15 @@ public static IndexWriterConfig emptyConfig()
     @Override
     public String toString()
     {
-        return String.format("IndexWriterConfig{%s=%d, %s=%d, %s=%s, %s=%s}",
+        return String.format("IndexWriterConfig{%s=%d, %s=%d, %s=%d, %s=%d, %s=%s, %s=%s, %s=%f, %s=%f, %s=%b}",
+                             POSTING_LIST_LVL_SKIP_OPTION, bkdPostingsSkip,
+                             POSTING_LIST_LVL_MIN_LEAVES, bkdPostingsMinLeaves,
                              MAXIMUM_NODE_CONNECTIONS, maximumNodeConnections,
                              CONSTRUCTION_BEAM_WIDTH, constructionBeamWidth,
                              SIMILARITY_FUNCTION, similarityFunction,
-                             OPTIMIZE_FOR, optimizeFor);
+                             SOURCE_MODEL, sourceModel,
+                             NEIGHBORHOOD_OVERFLOW, neighborhoodOverflow,
+                             ALPHA, alpha,
+                             ENABLE_HIERARCHY, enableHierarchy);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcher.java
new file mode 100644
index 000000000000..de1ea6264c79
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcher.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.base.MoreObjects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.postings.IntersectingPostingList;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.metrics.MulticastQueryEventListeners;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.BM25Utils;
+import org.apache.cassandra.index.sai.utils.BM25Utils.EagerDocTF;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithScore;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RowIdWithByteComparable;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.index.sai.disk.PostingList.END_OF_STREAM;
+
+/**
+ * Executes {@link Expression}s against the trie-based terms dictionary for an individual index segment.
+ */
+public class InvertedIndexSearcher extends IndexSearcher
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private final TermsReader reader;
+    private final QueryEventListener.TrieIndexEventListener perColumnEventListener;
+    private final Version version;
+    private final boolean filterRangeResults;
+    private final SSTableReader sstable;
+    private final SegmentMetadata.ComponentMetadata docLengthsMeta;
+    private final FileHandle docLengths;
+    private final long segmentRowIdOffset;
+
+    protected InvertedIndexSearcher(SSTableContext sstableContext,
+                                    PerIndexFiles perIndexFiles,
+                                    SegmentMetadata segmentMetadata,
+                                    IndexContext indexContext,
+                                    Version version,
+                                    boolean filterRangeResults) throws IOException
+    {
+        super(sstableContext.primaryKeyMapFactory(), perIndexFiles, segmentMetadata, indexContext);
+        this.sstable = sstableContext.sstable;
+
+        long root = metadata.getIndexRoot(IndexComponentType.TERMS_DATA);
+        assert root >= 0;
+
+        this.version = version;
+        this.filterRangeResults = filterRangeResults;
+        perColumnEventListener = (QueryEventListener.TrieIndexEventListener)indexContext.getColumnQueryMetrics();
+        this.segmentRowIdOffset = segmentMetadata.segmentRowIdOffset;
+        this.docLengthsMeta = segmentMetadata.componentMetadatas.getOptional(IndexComponentType.DOC_LENGTHS);
+        this.docLengths = docLengthsMeta == null ? null : indexFiles.docLengths();
+
+        Map<String,String> map = metadata.componentMetadatas.get(IndexComponentType.TERMS_DATA).attributes;
+        String footerPointerString = map.get(SAICodecUtils.FOOTER_POINTER);
+        long footerPointer = footerPointerString == null ? -1 : Long.parseLong(footerPointerString);
+
+        var perIndexComponents = perIndexFiles.usedPerIndexComponents();
+        reader = new TermsReader(indexContext,
+                                 indexFiles.termsData(),
+                                 perIndexComponents.byteComparableVersionFor(IndexComponentType.TERMS_DATA),
+                                 indexFiles.postingLists(),
+                                 root,
+                                 footerPointer,
+                                 version);
+    }
+
+    @Override
+    public long indexFileCacheSize()
+    {
+        // trie has no pre-allocated memory.
+        // TODO: Is this still the case now the trie isn't using the chunk cache?
+        return 0;
+    }
+
+    @SuppressWarnings("resource")
+    public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
+    {
+        PostingList postingList = searchPosting(exp, context);
+        return toPrimaryKeyIterator(postingList, context);
+    }
+
+    private PostingList searchPosting(Expression exp, QueryContext context)
+    {
+        if (logger.isTraceEnabled())
+            logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);
+
+        // We use the version to encode the search boundaries for the trie to ensure we use version appropriate bounds.
+        if (exp.getOp().isEquality() || exp.getOp() == Expression.Op.MATCH)
+        {
+            // Value is encoded in non-byte-comparable-version-specific fixed-length format.
+            final ByteComparable term = version.onDiskFormat().encodeForTrie(exp.lower.value.encoded, indexContext.getValidator());
+            QueryEventListener.TrieIndexEventListener listener = MulticastQueryEventListeners.of(context, perColumnEventListener);
+            return reader.exactMatch(term, listener, context);
+        }
+        else if (exp.getOp() == Expression.Op.RANGE)
+        {
+            QueryEventListener.TrieIndexEventListener listener = MulticastQueryEventListeners.of(context, perColumnEventListener);
+            var lower = exp.getEncodedLowerBoundByteComparable(version);
+            var upper = exp.getEncodedUpperBoundByteComparable(version);
+            return reader.rangeMatch(filterRangeResults ? exp : null, lower, upper, listener, context);
+        }
+        throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression: " + exp));
+    }
+
+    private Cell<?> readColumn(SSTableReader sstable, PrimaryKey primaryKey)
+    {
+        var dk = primaryKey.partitionKey();
+        var slices = Slices.with(indexContext.comparator(), Slice.make(primaryKey.clustering()));
+        try (var rowIterator = sstable.rowIterator(dk, slices, columnFilter, false, SSTableReadsListener.NOOP_LISTENER))
+        {
+            // primaryKey might not belong to this sstable, thus the iterator will be empty
+            if (rowIterator.isEmpty())
+                return null;
+            var unfiltered = rowIterator.next();
+            assert unfiltered.isRow() : unfiltered;
+            Row row = (Row) unfiltered;
+            return row.getCell(indexContext.getDefinition());
+        }
+    }
+
+    @Override
+    public CloseableIterator<PrimaryKeyWithSortKey> orderBy(Orderer orderer, Expression slice, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, int limit) throws IOException
+    {
+        if (!orderer.isBM25())
+        {
+            var iter = new RowIdWithTermsIterator(reader.allTerms(orderer.isAscending()));
+            return toMetaSortedIterator(iter, queryContext);
+        }
+        if (docLengthsMeta == null)
+            throw new InvalidRequestException(indexContext.getIndexName() + " does not support BM25 scoring until it is rebuilt");
+
+        // find documents that match each term
+        var queryTerms = orderer.getQueryTerms();
+        var postingLists = queryTerms.stream()
+                                     .collect(Collectors.toMap(Function.identity(), term ->
+        {
+            var encodedTerm = version.onDiskFormat().encodeForTrie(term, indexContext.getValidator());
+            var listener = MulticastQueryEventListeners.of(queryContext, perColumnEventListener);
+            var postings = reader.exactMatch(encodedTerm, listener, queryContext);
+            return postings == null ? PostingList.EMPTY : postings;
+        }));
+        // extract the match count for each
+        var documentFrequencies = postingLists.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> (long) e.getValue().size()));
+
+        var pkm = primaryKeyMapFactory.newPerSSTablePrimaryKeyMap();
+        var merged = IntersectingPostingList.intersect(postingLists);
+        var docLengthsReader = new DocLengthsReader(docLengths, docLengthsMeta);
+
+        // Wrap the iterator with resource management
+        var it = new AbstractIterator<BM25Utils.DocTF>() { // Anonymous class extends AbstractIterator
+            private boolean closed;
+
+            @Override
+            protected BM25Utils.DocTF computeNext()
+            {
+                try
+                {
+                    int rowId = merged.nextPosting();
+                    if (rowId == PostingList.END_OF_STREAM)
+                        return endOfData();
+                    // Reads from disk.
+                    int docLength = docLengthsReader.get(rowId); // segment-local rowid
+                    // We defer creating the primary key because it reads the token from disk, which is only needed
+                    // for the top rows just before they are materialized from disk, so we wait until after scoring
+                    // and sorting to read the token.
+                    return new LazyDocTF(pkm, segmentRowIdOffset + rowId, docLength, merged.frequencies());
+                }
+                catch (IOException e)
+                {
+                    throw new UncheckedIOException(e);
+                }
+            }
+
+            @Override
+            public void close()
+            {
+                if (closed) return;
+                closed = true;
+                FileUtils.closeQuietly(pkm, merged, docLengthsReader);
+            }
+        };
+        return bm25Internal(it, queryTerms, documentFrequencies);
+    }
+
+    private CloseableIterator<PrimaryKeyWithSortKey> bm25Internal(CloseableIterator<BM25Utils.DocTF> keyIterator,
+                                                                  List<ByteBuffer> queryTerms,
+                                                                  Map<ByteBuffer, Long> documentFrequencies)
+    {
+        var totalRows = sstable.getTotalRows();
+        // since doc frequencies can be an estimate from the index histogram, which does not have bounded error,
+        // cap frequencies to total rows so that the IDF term doesn't turn negative
+        var cappedFrequencies = documentFrequencies.entrySet().stream()
+                                                    .collect(Collectors.toMap(Map.Entry::getKey, e -> Math.min(e.getValue(), totalRows)));
+        var docStats = new BM25Utils.DocStats(cappedFrequencies, totalRows);
+        return BM25Utils.computeScores(keyIterator,
+                                       queryTerms,
+                                       docStats,
+                                       indexContext,
+                                       sstable.descriptor.id);
+    }
+
+    @Override
+    public CloseableIterator<PrimaryKeyWithSortKey> orderResultsBy(SSTableReader reader, QueryContext queryContext, List<PrimaryKey> keys, Orderer orderer, int limit) throws IOException
+    {
+        if (!orderer.isBM25())
+            return super.orderResultsBy(reader, queryContext, keys, orderer, limit);
+        if (docLengthsMeta == null)
+            throw new InvalidRequestException(indexContext.getIndexName() + " does not support BM25 scoring until it is rebuilt");
+
+        var queryTerms = orderer.getQueryTerms();
+        // compute documentFrequencies from either histogram or an index search
+        var documentFrequencies = new HashMap<ByteBuffer, Long>();
+        // any index new enough to support BM25 should also support histograms
+        assert metadata.version.onDiskFormat().indexFeatureSet().hasTermsHistogram();
+        for (ByteBuffer term : queryTerms)
+        {
+            long matches = metadata.estimateNumRowsMatching(new Expression(indexContext).add(Operator.ANALYZER_MATCHES, term));
+            documentFrequencies.put(term, matches);
+        }
+        var analyzer = indexContext.getAnalyzerFactory().create();
+        var it = keys.stream()
+                     .map(pk -> EagerDocTF.createFromDocument(pk, readColumn(sstable, pk), analyzer, queryTerms))
+                     .filter(Objects::nonNull)
+                     .iterator();
+        return bm25Internal(CloseableIterator.wrap(it), queryTerms, documentFrequencies);
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("indexContext", indexContext)
+                          .toString();
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(reader, docLengths);
+    }
+
+    /**
+     * An iterator that iterates over a source
+     */
+    private static class RowIdWithTermsIterator extends AbstractIterator<RowIdWithByteComparable>
+    {
+        private final TermsIterator source;
+        private PostingList currentPostingList = PostingList.EMPTY;
+        private ByteComparable currentTerm = null;
+
+        RowIdWithTermsIterator(TermsIterator source)
+        {
+            this.source = source;
+        }
+
+        @Override
+        protected RowIdWithByteComparable computeNext()
+        {
+            try
+            {
+                while (true)
+                {
+                    long nextPosting = currentPostingList.nextPosting();
+                    if (nextPosting != END_OF_STREAM)
+                        return new RowIdWithByteComparable(Math.toIntExact(nextPosting), currentTerm);
+
+                    if (!source.hasNext())
+                        return endOfData();
+
+                    currentTerm = source.next();
+                    FileUtils.closeQuietly(currentPostingList);
+                    currentPostingList = source.postings();
+                }
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(source, currentPostingList);
+        }
+    }
+
+    /**
+     * A {@link BM25Utils.DocTF} that is lazy in that it does not create the {@link PrimaryKey} until it is required.
+     */
+    private static class LazyDocTF implements BM25Utils.DocTF
+    {
+        private final PrimaryKeyMap pkm;
+        private final long sstableRowId;
+        private final int docLength;
+        private final Map<ByteBuffer, Integer> frequencies;
+
+        LazyDocTF(PrimaryKeyMap pkm, long sstableRowId, int docLength, Map<ByteBuffer, Integer> frequencies)
+        {
+            this.pkm = pkm;
+            this.sstableRowId = sstableRowId;
+            this.docLength = docLength;
+            this.frequencies = frequencies;
+        }
+
+        @Override
+        public int getTermFrequency(ByteBuffer term)
+        {
+            return frequencies.getOrDefault(term, 0);
+        }
+
+        @Override
+        public int termCount()
+        {
+            return docLength;
+        }
+
+        @Override
+        public PrimaryKeyWithSortKey primaryKey(IndexContext context, Memtable source, float score)
+        {
+            // Only sstables use this class, so this should never be called
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public PrimaryKeyWithSortKey primaryKey(IndexContext context, SSTableId<?> source, float score)
+        {
+            // We can eagerly get the token now, even though it might not technically be required until we know
+            // we have the best score. (Perhaps this should be lazy too?)
+            return new PrimaryKeyWithScore(context, source, pkm.primaryKeyFromRowId(sstableRowId), score);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/KDTreeIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/KDTreeIndexSearcher.java
new file mode 100644
index 000000000000..5911f2351014
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/KDTreeIndexSearcher.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+
+import com.google.common.base.MoreObjects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.BKDReader;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.metrics.MulticastQueryEventListeners;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RowIdWithByteComparable;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.index.sai.disk.v1.kdtree.BKDQueries.bkdQueryFrom;
+
+/**
+ * Executes {@link Expression}s against the kd-tree for an individual index segment.
+ */
+public class KDTreeIndexSearcher extends IndexSearcher
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private final BKDReader bkdReader;
+    private final QueryEventListener.BKDIndexEventListener perColumnEventListener;
+
+    KDTreeIndexSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
+                        PerIndexFiles perIndexFiles,
+                        SegmentMetadata segmentMetadata,
+                        IndexContext indexContext) throws IOException
+    {
+        super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext);
+
+        final long bkdPosition = metadata.getIndexRoot(IndexComponentType.KD_TREE);
+        assert bkdPosition >= 0;
+        final long postingsPosition = metadata.getIndexRoot(IndexComponentType.KD_TREE_POSTING_LISTS);
+        assert postingsPosition >= 0;
+
+        bkdReader = new BKDReader(indexContext,
+                                  indexFiles.kdtree(),
+                                  bkdPosition,
+                                  indexFiles.kdtreePostingLists(),
+                                  postingsPosition);
+        perColumnEventListener = (QueryEventListener.BKDIndexEventListener)indexContext.getColumnQueryMetrics();
+    }
+
+    @Override
+    public long indexFileCacheSize()
+    {
+        return bkdReader.memoryUsage();
+    }
+
+    @Override
+    public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
+    {
+        PostingList postingList = searchPosting(exp, context);
+        return toPrimaryKeyIterator(postingList, context);
+    }
+
+    private PostingList searchPosting(Expression exp, QueryContext context)
+    {
+        if (logger.isTraceEnabled())
+            logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);
+
+        if (exp.getOp().isEqualityOrRange())
+        {
+            final BKDReader.IntersectVisitor query = bkdQueryFrom(exp, bkdReader.getNumDimensions(), bkdReader.getBytesPerDimension());
+            QueryEventListener.BKDIndexEventListener listener = MulticastQueryEventListeners.of(context, perColumnEventListener);
+            return bkdReader.intersect(query, listener, context);
+        }
+        else
+        {
+            throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression during index query: " + exp));
+        }
+    }
+
+    public CloseableIterator<PrimaryKeyWithSortKey> orderBy(Orderer orderer, Expression slice, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, int limit) throws IOException
+    {
+        var query = slice != null && slice.getOp().isEqualityOrRange()
+                    ? bkdQueryFrom(slice, bkdReader.getNumDimensions(), bkdReader.getBytesPerDimension())
+                    : null;
+        var direction = orderer.isAscending() ? BKDReader.Direction.FORWARD : BKDReader.Direction.BACKWARD;
+        var iter = new RowIdIterator(bkdReader.iteratorState(direction, query));
+        return toMetaSortedIterator(iter, queryContext);
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("indexContext", indexContext)
+                          .add("count", bkdReader.getPointCount())
+                          .add("numDimensions", bkdReader.getNumDimensions())
+                          .add("bytesPerDimension", bkdReader.getBytesPerDimension())
+                          .toString();
+    }
+
+    @Override
+    public void close()
+    {
+        bkdReader.close();
+    }
+
+    private static class RowIdIterator extends AbstractGuavaIterator<RowIdWithByteComparable> implements CloseableIterator<RowIdWithByteComparable>
+    {
+        private final BKDReader.IteratorState iterator;
+        RowIdIterator(BKDReader.IteratorState iterator)
+        {
+            this.iterator = iterator;
+        }
+
+        @Override
+        public RowIdWithByteComparable computeNext()
+        {
+            if (!iterator.hasNext())
+                return endOfData();
+
+            var segmentRowId = iterator.next();
+            // We have to copy scratch to prevent it from being overwritten by the next call to computeNext()
+            var indexValue = new byte[iterator.scratch.length];
+            System.arraycopy(iterator.scratch, 0, indexValue, 0, iterator.scratch.length);
+            // We store the indexValue in an already encoded format, so we use the preencoded method here
+            // to avoid re-encoding it.
+            return new RowIdWithByteComparable(Math.toIntExact(segmentRowId),
+                                               ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION,
+                                                                         indexValue));
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(iterator);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/LongArray.java b/src/java/org/apache/cassandra/index/sai/disk/v1/LongArray.java
index 044e41b23ec7..fc1b2ad898ba 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/LongArray.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/LongArray.java
@@ -37,6 +37,15 @@ public interface LongArray extends Closeable
      */
     long length();
 
+
+    /**
+     * @param targetToken Token to look up.  Must not be smaller than previous value queried
+     *                    (the method is stateful)
+     * @return The row ID of the first token equal to or greater than the target,
+     *         or negative value if target token is greater than all tokens
+     */
+    long ceilingRowId(long targetToken);
+
     /**
      * Using the given value returns the first index corresponding to the value.
      *
@@ -75,10 +84,17 @@ public long length()
         }
 
         @Override
-        public long indexOf(long value)
+        public long ceilingRowId(long targetToken)
+        {
+            open();
+            return longArray.ceilingRowId(targetToken);
+        }
+
+        @Override
+        public long indexOf(long targetToken)
         {
             open();
-            return longArray.indexOf(value);
+            return longArray.indexOf(targetToken);
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java
index 0650e9b9d6dc..266100a76789 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java
@@ -18,135 +18,118 @@
 package org.apache.cassandra.index.sai.disk.v1;
 
 import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Stopwatch;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.carrotsearch.hppc.LongArrayList;
+import org.agrona.collections.Int2IntHashMap;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
-import org.apache.cassandra.index.sai.disk.RowMapping;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.bbtree.NumericIndexWriter;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentWriter;
-import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.MemtableTermsIterator;
+import org.apache.cassandra.index.sai.disk.PerIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.ImmutableOneDimPointValues;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.NumericIndexWriter;
+import org.apache.cassandra.index.sai.disk.v1.trie.InvertedIndexWriter;
+import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex;
 import org.apache.cassandra.index.sai.memory.MemtableIndex;
-import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
-import org.apache.cassandra.index.sai.metrics.IndexMetrics;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.memory.RowMapping;
+import org.apache.cassandra.index.sai.memory.TrieMemoryIndex;
+import org.apache.cassandra.index.sai.memory.TrieMemtableIndex;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 /**
  * Column index writer that flushes indexed data directly from the corresponding Memtable index, without buffering index
  * data in memory.
  */
-public class MemtableIndexWriter implements PerColumnIndexWriter
+public class MemtableIndexWriter implements PerIndexWriter
 {
-    private static final Logger logger = LoggerFactory.getLogger(MemtableIndexWriter.class);
-    private static final int NO_ROWS = -1;
-
-    private final IndexDescriptor indexDescriptor;
-    private final IndexTermType indexTermType;
-    private final IndexIdentifier indexIdentifier;
-    private final IndexMetrics indexMetrics;
-    private final MemtableIndex memtable;
-    private final RowMapping rowMapping;
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-    private PrimaryKey minKey;
-    private PrimaryKey maxKey;
-    private long maxSSTableRowId = NO_ROWS;
-    private int rowCount;
+    private final IndexComponents.ForWrite perIndexComponents;
+    private final MemtableIndex memtableIndex;
+    private final PrimaryKey.Factory pkFactory;
+    private final RowMapping rowMapping;
 
-    public MemtableIndexWriter(MemtableIndex memtable,
-                               IndexDescriptor indexDescriptor,
-                               IndexTermType indexTermType,
-                               IndexIdentifier indexIdentifier,
-                               IndexMetrics indexMetrics,
+    public MemtableIndexWriter(MemtableIndex memtableIndex,
+                               IndexComponents.ForWrite perIndexComponents,
+                               PrimaryKey.Factory pkFactory,
                                RowMapping rowMapping)
     {
         assert rowMapping != null && rowMapping != RowMapping.DUMMY : "Row mapping must exist during FLUSH.";
 
-        this.indexDescriptor = indexDescriptor;
-        this.indexTermType = indexTermType;
-        this.indexIdentifier = indexIdentifier;
-        this.indexMetrics = indexMetrics;
-        this.memtable = memtable;
+        this.perIndexComponents = perIndexComponents;
+        this.memtableIndex = memtableIndex;
+        this.pkFactory = pkFactory;
         this.rowMapping = rowMapping;
     }
 
+    @Override
+    public IndexContext indexContext()
+    {
+        return perIndexComponents.context();
+    }
+
+    @Override
+    public IndexComponents.ForWrite writtenComponents()
+    {
+        return perIndexComponents;
+    }
+
     @Override
     public void addRow(PrimaryKey key, Row row, long sstableRowId)
     {
         // Memtable indexes are flushed directly to disk with the aid of a mapping between primary
         // keys and row IDs in the flushing SSTable. This writer, therefore, does nothing in
-        // response to the flushing of individual rows except for keeping index-specific statistics.
-        boolean isStatic = indexTermType.columnMetadata().isStatic();
-
-        // Indexes on static columns should only track static rows, and indexes on non-static columns 
-        // should only track non-static rows. (Within a partition, the row ID for a static row will always
-        // come before any non-static row.) 
-        if (key.kind() == PrimaryKey.Kind.STATIC && isStatic || key.kind() != PrimaryKey.Kind.STATIC && !isStatic)
-        {
-            if (minKey == null)
-                minKey = key;
-            maxKey = key;
-            rowCount++;
-            maxSSTableRowId = Math.max(maxSSTableRowId, sstableRowId);
-        }
+        // response to the flushing of individual rows.
     }
 
     @Override
     public void abort(Throwable cause)
     {
-        if (cause == null)
-            // This commonly occurs when a Memtable has no rows to flush, and is harmless:
-            logger.debug(indexIdentifier.logMessage("Aborting index memtable flush for {}..."), indexDescriptor.sstableDescriptor);
-        else
-            logger.warn(indexIdentifier.logMessage("Aborting index memtable flush for {}..."), indexDescriptor.sstableDescriptor, cause);
-
-        indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier);
+        logger.warn(perIndexComponents.logMessage("Aborting index memtable flush for {}..."), perIndexComponents.descriptor(), cause);
+        perIndexComponents.forceDeleteAllComponents();
     }
 
     @Override
     public void complete(Stopwatch stopwatch) throws IOException
     {
-        assert rowMapping.isComplete() : "Cannot complete the memtable index writer because the row mapping is not complete";
-
         long start = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
         try
         {
-            if (maxSSTableRowId == -1 || memtable == null || memtable.isEmpty())
+            if (!rowMapping.hasRows() || (memtableIndex == null) || memtableIndex.isEmpty())
             {
-                logger.debug(indexIdentifier.logMessage("No indexed rows to flush from SSTable {}."), indexDescriptor.sstableDescriptor);
-                // Write a completion marker even though we haven't written anything to the index,
+                logger.debug(perIndexComponents.logMessage("No indexed rows to flush from SSTable {}."), perIndexComponents.descriptor());
+                // Write a completion marker even though we haven't written anything to the index
                 // so we won't try to build the index again for the SSTable
-                ColumnCompletionMarkerUtil.create(indexDescriptor, indexIdentifier, true);
-
+                perIndexComponents.markComplete();
                 return;
             }
 
-            if (indexTermType.isVector())
+            final DecoratedKey minKey = rowMapping.minKey.partitionKey();
+            final DecoratedKey maxKey = rowMapping.maxKey.partitionKey();
+
+            if (indexContext().isVector())
             {
-                flushVectorIndex(start, stopwatch);
+                flushVectorIndex(minKey, maxKey, start, stopwatch);
             }
             else
             {
-                final Iterator<Pair<ByteComparable, LongArrayList>> iterator = rowMapping.merge(memtable);
-
-                try (MemtableTermsIterator terms = new MemtableTermsIterator(memtable.getMinTerm(), memtable.getMaxTerm(), iterator))
+                var iterator = rowMapping.merge(memtableIndex);
+                try (MemtableTermsIterator terms = new MemtableTermsIterator(memtableIndex.getMinTerm(), memtableIndex.getMaxTerm(), iterator))
                 {
-                    long cellCount = flush(terms);
+                    long cellCount = flush(minKey, maxKey, indexContext().getValidator(), terms, rowMapping.maxSegmentRowId);
 
                     completeIndexFlush(cellCount, start, stopwatch);
                 }
@@ -154,80 +137,126 @@ public void complete(Stopwatch stopwatch) throws IOException
         }
         catch (Throwable t)
         {
-            logger.error(indexIdentifier.logMessage("Error while flushing index {}"), t.getMessage(), t);
-            indexMetrics.memtableIndexFlushErrors.inc();
+            logger.error(perIndexComponents.logMessage("Error while flushing index {}"), t.getMessage(), t);
+            indexContext().getIndexMetrics().memtableIndexFlushErrors.inc();
 
             throw t;
         }
     }
 
-    private long flush(MemtableTermsIterator terms) throws IOException
+    private long flush(DecoratedKey minKey, DecoratedKey maxKey, AbstractType<?> termComparator, MemtableTermsIterator terms, int maxSegmentRowId) throws IOException
     {
-        SegmentWriter writer = indexTermType.isLiteral() ? new LiteralIndexWriter(indexDescriptor, indexIdentifier)
-                                                         : new NumericIndexWriter(indexDescriptor,
-                                                                                  indexIdentifier,
-                                                                                  indexTermType.fixedSizeOf());
-
-        SegmentMetadata.ComponentMetadataMap indexMetas = writer.writeCompleteSegment(terms);
-        long numRows = writer.getNumberOfRows();
+        long numRows;
+        SegmentMetadataBuilder metadataBuilder = new SegmentMetadataBuilder(0, perIndexComponents);
+        SegmentMetadata.ComponentMetadataMap indexMetas;
+        if (TypeUtil.isLiteral(termComparator))
+        {
+            try (InvertedIndexWriter writer = new InvertedIndexWriter(perIndexComponents, writeFrequencies()))
+            {
+                // Convert PrimaryKey->length map to rowId->length using RowMapping
+                var docLengths = new Int2IntHashMap(Integer.MIN_VALUE);
+                Arrays.stream(((TrieMemtableIndex) memtableIndex).getRangeIndexes())
+                      .map(TrieMemoryIndex.class::cast)
+                      .forEach(trieMemoryIndex ->
+                          trieMemoryIndex.getDocLengths().forEach((pk, length) -> {
+                              int rowId = rowMapping.get(pk);
+                              if (rowId >= 0)
+                                  docLengths.put(rowId, (int) length);
+                          })
+                      );
+
+                indexMetas = writer.writeAll(metadataBuilder.intercept(terms), docLengths);
+                numRows = writer.getPostingsCount();
+            }
+        }
+        else
+        {
+            try (NumericIndexWriter writer = new NumericIndexWriter(perIndexComponents,
+                                                                    TypeUtil.fixedSizeOf(termComparator),
+                                                                    maxSegmentRowId,
+                                                                    // Due to stale entries in IndexMemtable, we may have more indexed rows than num of rowIds.
+                                                                    Integer.MAX_VALUE,
+                                                                    indexContext().getIndexWriterConfig()))
+            {
+                ImmutableOneDimPointValues values = ImmutableOneDimPointValues.fromTermEnum(terms, termComparator);
+                indexMetas = writer.writeAll(metadataBuilder.intercept(values));
+                numRows = writer.getPointCount();
+            }
+        }
 
         // If no rows were written we need to delete any created column index components
         // so that the index is correctly identified as being empty (only having a completion marker)
         if (numRows == 0)
         {
-            indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier);
+            perIndexComponents.forceDeleteAllComponents();
             return 0;
         }
 
-        // During index memtable flush, the data is sorted based on terms.
-        SegmentMetadata metadata = new SegmentMetadata(0,
-                                                       numRows,
-                                                       terms.getMinSSTableRowId(), terms.getMaxSSTableRowId(),
-                                                       minKey, maxKey, 
-                                                       terms.getMinTerm(), terms.getMaxTerm(),
-                                                       indexMetas);
+        metadataBuilder.setKeyRange(pkFactory.createPartitionKeyOnly(minKey), pkFactory.createPartitionKeyOnly(maxKey));
+        metadataBuilder.setRowIdRange(terms.getMinSSTableRowId(), terms.getMaxSSTableRowId());
+        metadataBuilder.setTermRange(terms.getMinTerm(), terms.getMaxTerm());
+        metadataBuilder.setComponentsMetadata(indexMetas);
+        SegmentMetadata metadata = metadataBuilder.build();
 
-        try (MetadataWriter metadataWriter = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier)))
+        try (MetadataWriter writer = new MetadataWriter(perIndexComponents))
         {
-            SegmentMetadata.write(metadataWriter, Collections.singletonList(metadata));
+            SegmentMetadata.write(writer, Collections.singletonList(metadata));
         }
 
         return numRows;
     }
 
-    private void flushVectorIndex(long startTime, Stopwatch stopwatch) throws IOException
+    private boolean writeFrequencies()
     {
-        SegmentMetadata.ComponentMetadataMap metadataMap = memtable.writeDirect(indexDescriptor, indexIdentifier, rowMapping::get);
-        completeIndexFlush(rowCount, startTime, stopwatch);
+        return indexContext().isAnalyzed() && Version.latest().onOrAfter(Version.EC);
+    }
+
+    private void flushVectorIndex(DecoratedKey minKey, DecoratedKey maxKey, long startTime, Stopwatch stopwatch) throws IOException
+    {
+        var vectorIndex = (VectorMemtableIndex) memtableIndex;
+
+        if (!vectorIndex.preFlush(rowMapping::get))
+        {
+            logger.debug(perIndexComponents.logMessage("Whole graph is deleted. Skipping index flush for {}."), perIndexComponents.descriptor());
+            perIndexComponents.markComplete();
+            return;
+        }
+
+        SegmentMetadata.ComponentMetadataMap metadataMap = vectorIndex.writeData(perIndexComponents);
 
         SegmentMetadata metadata = new SegmentMetadata(0,
-                                                       rowCount,
-                                                       0, maxSSTableRowId,
-                                                       minKey, maxKey, 
-                                                       ByteBufferUtil.bytes(0), ByteBufferUtil.bytes(0),
+                                                       rowMapping.size(), // TODO this isn't the right size metric.
+                                                       0,
+                                                       rowMapping.maxSegmentRowId,
+                                                       pkFactory.createPartitionKeyOnly(minKey),
+                                                       pkFactory.createPartitionKeyOnly(maxKey),
+                                                       ByteBufferUtil.bytes(0), // VSTODO by pass min max terms for vectors
+                                                       ByteBufferUtil.bytes(0), // VSTODO by pass min max terms for vectors
+                                                       null,
                                                        metadataMap);
 
-        try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier)))
+        try (MetadataWriter writer = new MetadataWriter(perIndexComponents))
         {
             SegmentMetadata.write(writer, Collections.singletonList(metadata));
         }
+
+        completeIndexFlush(rowMapping.size(), startTime, stopwatch);
     }
 
     private void completeIndexFlush(long cellCount, long startTime, Stopwatch stopwatch) throws IOException
     {
-        // create a completion marker indicating that the index is complete and not-empty
-        ColumnCompletionMarkerUtil.create(indexDescriptor, indexIdentifier, false);
+        perIndexComponents.markComplete();
 
-        indexMetrics.memtableIndexFlushCount.inc();
+        indexContext().getIndexMetrics().memtableIndexFlushCount.inc();
 
         long elapsedTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
-        logger.debug(indexIdentifier.logMessage("Completed flushing {} memtable index cells to SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
+        logger.debug(perIndexComponents.logMessage("Completed flushing {} memtable index cells to SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
                      cellCount,
-                     indexDescriptor.sstableDescriptor,
+                     perIndexComponents.descriptor(),
                      elapsedTime - startTime,
                      elapsedTime);
 
-        indexMetrics.memtableFlushCellsPerSecond.update((long) (cellCount * 1000.0 / Math.max(1, elapsedTime - startTime)));
+        indexContext().getIndexMetrics().memtableFlushCellsPerSecond.update((long) (cellCount * 1000.0 / Math.max(1, elapsedTime - startTime)));
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java
index e1c5c4f450d1..719da42276c5 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java
@@ -18,81 +18,91 @@
 package org.apache.cassandra.index.sai.disk.v1;
 
 import java.io.IOException;
+import java.nio.ByteOrder;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Supplier;
 import javax.annotation.concurrent.NotThreadSafe;
 
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
-import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.BytesRef;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.oldlucene.ByteArrayIndexInput;
 
 @NotThreadSafe
 public class MetadataSource
 {
-    private final Map<String, BytesRef> components;
+    private final Version version;
+    private final Map<String, Supplier<ByteArrayIndexInput>> components;
 
-    private MetadataSource(Map<String, BytesRef> components)
+    private MetadataSource(Version version, Map<String, Supplier<ByteArrayIndexInput>> components)
     {
+        this.version = version;
         this.components = components;
     }
 
-    public static MetadataSource loadGroupMetadata(IndexDescriptor indexDescriptor) throws IOException
+    public static MetadataSource loadMetadata(IndexComponents.ForRead components) throws IOException
     {
-        return MetadataSource.load(indexDescriptor.openPerSSTableInput(IndexComponent.GROUP_META));
+        IndexComponent.ForRead metadataComponent = components.get(components.metadataComponent());
+        try (var input = metadataComponent.openCheckSummedInput())
+        {
+            return MetadataSource.load(input, components.version(), metadataComponent.byteOrder());
+        }
     }
 
-    public static MetadataSource loadColumnMetadata(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
+    private static MetadataSource load(ChecksumIndexInput input, Version expectedVersion, ByteOrder order) throws IOException
     {
-        return MetadataSource.load(indexDescriptor.openPerIndexInput(IndexComponent.META, indexIdentifier));
-    }
+        Map<String, Supplier<ByteArrayIndexInput>> components = new HashMap<>();
+        Version version = SAICodecUtils.checkHeader(input);
+        if (version != expectedVersion)
+            throw new IllegalStateException("Unexpected version " + version + " in " + input + ", expected " + expectedVersion);
 
-    private static MetadataSource load(IndexInput indexInput) throws IOException
-    {
-        Map<String, BytesRef> components = new HashMap<>();
+        final int num = input.readInt();
 
-        try (ChecksumIndexInput input = IndexFileUtils.getBufferedChecksumIndexInput(indexInput))
+        for (int x = 0; x < num; x++)
         {
-            SAICodecUtils.checkHeader(input);
-            final int num = input.readInt();
-
-            for (int x = 0; x < num; x++)
+            if (input.length() == input.getFilePointer())
             {
-                if (input.length() == input.getFilePointer())
-                {
-                    // we should never get here, because we always add footer to the file
-                    throw new IllegalStateException("Unexpected EOF in " + input);
-                }
-
-                final String name = input.readString();
-                final int length = input.readInt();
-                final byte[] bytes = new byte[length];
-                input.readBytes(bytes, 0, length);
-
-                components.put(name, new BytesRef(bytes));
+                // we should never get here, because we always add footer to the file
+                throw new IllegalStateException("Unexpected EOF in " + input);
             }
 
-            SAICodecUtils.checkFooter(input);
+            final String name = input.readString();
+            final int length = input.readInt();
+            final byte[] bytes = new byte[length];
+            input.readBytes(bytes, 0, length);
+
+            components.put(name, () -> new ByteArrayIndexInput(name, bytes, order));
         }
 
-        return new MetadataSource(components);
+        SAICodecUtils.checkFooter(input);
+
+        return new MetadataSource(version, components);
     }
 
-    public DataInput get(String name)
+    public IndexInput get(IndexComponent component)
     {
-        BytesRef bytes = components.get(name);
+        return get(component.fileNamePart());
+    }
+
+    public IndexInput get(String name)
+    {
+        var supplier = components.get(name);
 
-        if (bytes == null)
+        if (supplier == null)
         {
             throw new IllegalArgumentException(String.format("Could not find component '%s'. Available properties are %s.",
                                                              name, components.keySet()));
         }
 
-        return new ByteArrayDataInput(bytes.bytes);
+        return supplier.get();
+    }
+
+    public Version getVersion()
+    {
+        return version;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataWriter.java
index 4b4dc7a98e99..7cc2df0ba118 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataWriter.java
@@ -19,41 +19,50 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.nio.ByteOrder;
 import java.util.HashMap;
 import java.util.Map;
 import javax.annotation.concurrent.NotThreadSafe;
 
-import org.apache.cassandra.index.sai.disk.ResettableByteBuffersIndexOutput;
-import org.apache.lucene.store.IndexOutput;
+import org.apache.cassandra.index.sai.disk.ModernResettableByteBuffersIndexOutput;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.oldlucene.LegacyResettableByteBuffersIndexOutput;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.lucene.util.BytesRef;
 
 @NotThreadSafe
 public class MetadataWriter implements Closeable
 {
+    private final Version version;
     private final IndexOutput output;
     private final Map<String, BytesRef> map = new HashMap<>();
 
-    public MetadataWriter(IndexOutput output)
+    public MetadataWriter(IndexComponents.ForWrite components) throws IOException
     {
-        this.output = output;
+        this.version = components.version();
+        this.output = components.addOrGet(components.metadataComponent()).openOutput();
     }
 
-    public Builder builder(String name)
+    public IndexOutput builder(String name)
     {
-        return new Builder(name);
-    }
-
-    public class Builder extends ResettableByteBuffersIndexOutput implements Closeable
-    {
-        private Builder(String name)
-        {
-            super(name);
-        }
-
-        @Override
-        public void close()
-        {
-            map.put(getName(), new BytesRef(toArrayCopy(), 0, intSize()));
+        if (output.order() == ByteOrder.BIG_ENDIAN) {
+            return new LegacyResettableByteBuffersIndexOutput(1024, name) {
+                @Override
+                public void close()
+                {
+                    map.put(getName(), new BytesRef(toArrayCopy(), 0, intSize()));
+                }
+            };
+        } else {
+            return new ModernResettableByteBuffersIndexOutput(1024, name) {
+                @Override
+                public void close()
+                {
+                    map.put(getName(), new BytesRef(toArrayCopy(), 0, intSize()));
+                }
+            };
         }
     }
 
@@ -82,4 +91,9 @@ public void close() throws IOException
             output.close();
         }
     }
+
+    public Version version()
+    {
+        return version;
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/PartitionAwarePrimaryKeyFactory.java b/src/java/org/apache/cassandra/index/sai/disk/v1/PartitionAwarePrimaryKeyFactory.java
new file mode 100644
index 000000000000..c1b47916229b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/PartitionAwarePrimaryKeyFactory.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.util.Objects;
+import java.util.function.Supplier;
+
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * A partition-aware {@link PrimaryKey.Factory}. This creates {@link PrimaryKey} instances that are
+ * sortable by {@link DecoratedKey} only.
+ */
+public class PartitionAwarePrimaryKeyFactory implements PrimaryKey.Factory
+{
+    @Override
+    public PrimaryKey createTokenOnly(Token token)
+    {
+        assert token != null;
+        return new PartitionAwarePrimaryKey(token, null, null);
+    }
+
+    @Override
+    public PrimaryKey createDeferred(Token token, Supplier<PrimaryKey> primaryKeySupplier)
+    {
+        assert token != null;
+        return new PartitionAwarePrimaryKey(token, null, primaryKeySupplier);
+    }
+
+    @Override
+    public PrimaryKey create(DecoratedKey partitionKey, Clustering clustering)
+    {
+        assert partitionKey != null;
+        return new PartitionAwarePrimaryKey(partitionKey.getToken(), partitionKey, null);
+    }
+
+    private class PartitionAwarePrimaryKey implements PrimaryKey
+    {
+        private final Token token;
+        private DecoratedKey partitionKey;
+        private Supplier<PrimaryKey> primaryKeySupplier;
+
+        private PartitionAwarePrimaryKey(Token token, DecoratedKey partitionKey, Supplier<PrimaryKey> primaryKeySupplier)
+        {
+            this.token = token;
+            this.partitionKey = partitionKey;
+            this.primaryKeySupplier = primaryKeySupplier;
+        }
+
+        @Override
+        public PrimaryKey loadDeferred()
+        {
+            if (primaryKeySupplier != null && partitionKey == null)
+            {
+                this.partitionKey = primaryKeySupplier.get().partitionKey();
+                primaryKeySupplier = null;
+            }
+            return this;
+        }
+
+        @Override
+        public Token token()
+        {
+            return this.token;
+        }
+
+        @Override
+        public DecoratedKey partitionKey()
+        {
+            loadDeferred();
+            return partitionKey;
+        }
+
+        @Override
+        public Clustering clustering()
+        {
+            return Clustering.EMPTY;
+        }
+
+        @Override
+        public ByteSource asComparableBytes(ByteComparable.Version version)
+        {
+            return asComparableBytes(version == ByteComparable.Version.LEGACY ? ByteSource.END_OF_STREAM : ByteSource.TERMINATOR, version, false);
+        }
+
+        @Override
+        public ByteSource asComparableBytesMinPrefix(ByteComparable.Version version)
+        {
+            return asComparableBytes(ByteSource.LT_NEXT_COMPONENT, version, true);
+        }
+
+        @Override
+        public ByteSource asComparableBytesMaxPrefix(ByteComparable.Version version)
+        {
+            return asComparableBytes(ByteSource.GT_NEXT_COMPONENT, version, true);
+        }
+
+        private ByteSource asComparableBytes(int terminator, ByteComparable.Version version, boolean isPrefix)
+        {
+            // Note: Unlike row-aware primary keys the asComparable method in for
+            // partition aware keys is only used on the write side so we do not need
+            // to enforce deferred loading here.
+            ByteSource tokenComparable = token.asComparableBytes(version);
+            ByteSource keyComparable = partitionKey == null ? null
+                                                            :ByteSource.of(partitionKey.getKey(), version);
+
+            // prefix doesn't include null components
+            if (isPrefix)
+            {
+                if (keyComparable == null)
+                    return ByteSource.withTerminator(terminator, tokenComparable);
+                else
+                    return ByteSource.withTerminator(terminator, tokenComparable, keyComparable);
+            }
+            return ByteSource.withTerminator(terminator, tokenComparable, keyComparable, null);
+        }
+
+        @Override
+        public long ramBytesUsed()
+        {
+            // Compute shallow size: object header + 4 references (3 declared + 1 implicit outer reference)
+            long shallowSize = RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 4L * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+            long preHashedDecoratedKeySize = partitionKey == null
+                                           ? 0
+                                           : RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
+                                             + 2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF // token and key references
+                                             + 2L * Long.BYTES;
+            return shallowSize + token.getHeapSize() + preHashedDecoratedKeySize;
+        }
+
+        @Override
+        public int compareTo(PrimaryKey o)
+        {
+            if (partitionKey == null || o.partitionKey() == null)
+                return token().compareTo(o.token());
+            return partitionKey.compareTo(o.partitionKey());
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(token);
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (obj instanceof PrimaryKey)
+                return compareTo((PrimaryKey)obj) == 0;
+            return false;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("TokenAwarePrimaryKey: { token: %s, partition: %s } ", token, partitionKey == null ? null : partitionKey);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/PartitionAwarePrimaryKeyMap.java b/src/java/org/apache/cassandra/index/sai/disk/v1/PartitionAwarePrimaryKeyMap.java
new file mode 100644
index 000000000000..ae09fc0e52d0
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/PartitionAwarePrimaryKeyMap.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.BlockPackedReader;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.MonotonicBlockPackedReader;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Throwables;
+
+/**
+ * A partition-aware {@link PrimaryKeyMap}
+ *
+ * This uses the following on-disk structures:
+ * <ul>
+ *     <li>Block-packed structure for rowId to token lookups using {@link BlockPackedReader}.
+ *     Uses component {@link IndexComponentType#TOKEN_VALUES} </li>
+ *     <li>Monotonic-block-packed structure for rowId to partition key offset lookups using {@link MonotonicBlockPackedReader}.
+ *     Uses component {@link IndexComponentType#OFFSETS_VALUES} </li>
+ * </ul>
+ *
+ * This uses a {@link IKeyFetcher} to read the {@link org.apache.cassandra.db.DecoratedKey} for a {@link PrimaryKey} from the
+ * sstable using the sstable offset provided by the monotonic-block-packed structure above.
+ */
+@NotThreadSafe
+public class PartitionAwarePrimaryKeyMap implements PrimaryKeyMap
+{
+    @ThreadSafe
+    public static class PartitionAwarePrimaryKeyMapFactory implements Factory
+    {
+        private final IndexComponents.ForRead perSSTableComponents;
+        private final LongArray.Factory tokenReaderFactory;
+        private final LongArray.Factory offsetReaderFactory;
+        private final MetadataSource metadata;
+        private final SSTableReader sstable;
+        private final IPartitioner partitioner;
+        private final PrimaryKey.Factory primaryKeyFactory;
+        private final SSTableId<?> sstableId;
+        private final long count;
+
+        private FileHandle token = null;
+        private FileHandle offset = null;
+
+        public PartitionAwarePrimaryKeyMapFactory(IndexComponents.ForRead perSSTableComponents, SSTableReader sstable, PrimaryKey.Factory primaryKeyFactory)
+        {
+            try
+            {
+                this.perSSTableComponents = perSSTableComponents;
+                this.metadata = MetadataSource.loadMetadata(perSSTableComponents);
+
+                IndexComponent.ForRead offsetsComponent = perSSTableComponents.get(IndexComponentType.OFFSETS_VALUES);
+                IndexComponent.ForRead tokensComponent = perSSTableComponents.get(IndexComponentType.TOKEN_VALUES);
+
+                NumericValuesMeta offsetsMeta = new NumericValuesMeta(this.metadata.get(offsetsComponent));
+                NumericValuesMeta tokensMeta = new NumericValuesMeta(this.metadata.get(tokensComponent));
+
+                count = tokensMeta.valueCount;
+                token = tokensComponent.createFileHandle();
+                offset = offsetsComponent.createFileHandle();
+
+                this.tokenReaderFactory = new BlockPackedReader(token, tokensMeta);
+                this.offsetReaderFactory = new MonotonicBlockPackedReader(offset, offsetsMeta);
+                this.partitioner = sstable.metadata().partitioner;
+                this.sstable = sstable;
+                this.primaryKeyFactory = primaryKeyFactory;
+                this.sstableId = sstable.getId();
+            }
+            catch (Throwable t)
+            {
+                throw Throwables.unchecked(Throwables.close(t, token, offset));
+            }
+        }
+
+        @Override
+        public PrimaryKeyMap newPerSSTablePrimaryKeyMap()
+        {
+            LongArray rowIdToToken = null;
+            LongArray rowIdToOffset = null;
+            IKeyFetcher keyFetcher = null;
+            try
+            {
+                rowIdToToken = new LongArray.DeferredLongArray(() -> tokenReaderFactory.open());
+                rowIdToOffset = new LongArray.DeferredLongArray(() -> offsetReaderFactory.open());
+                keyFetcher = sstable.openKeyFetcher(false);
+
+                return new PartitionAwarePrimaryKeyMap(rowIdToToken, rowIdToOffset, partitioner, keyFetcher, primaryKeyFactory, sstableId);
+            }
+            catch (RuntimeException | Error e)
+            {
+                Throwables.closeNonNullAndAddSuppressed(e, rowIdToToken, rowIdToOffset, keyFetcher);
+            }
+            return null;
+        }
+
+        @Override
+        public long count()
+        {
+            return count;
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            FileUtils.closeQuietly(offset, token);
+        }
+    }
+
+    private final LongArray rowIdToToken;
+    private final LongArray rowIdToOffset;
+    private final IPartitioner partitioner;
+    private final IKeyFetcher keyFetcher;
+    private final PrimaryKey.Factory primaryKeyFactory;
+    private final SSTableId<?> sstableId;
+    private final ByteBuffer tokenBuffer = ByteBuffer.allocate(Long.BYTES);
+
+    private PartitionAwarePrimaryKeyMap(LongArray rowIdToToken,
+                                        LongArray rowIdToOffset,
+                                        IPartitioner partitioner,
+                                        IKeyFetcher keyFetcher,
+                                        PrimaryKey.Factory primaryKeyFactory,
+                                        SSTableId<?> sstableId)
+    {
+        this.rowIdToToken = rowIdToToken;
+        this.rowIdToOffset = rowIdToOffset;
+        this.partitioner = partitioner;
+        this.keyFetcher = keyFetcher;
+        this.primaryKeyFactory = primaryKeyFactory;
+        this.sstableId = sstableId;
+    }
+
+    @Override
+    public SSTableId<?> getSSTableId()
+    {
+        return sstableId;
+    }
+
+    @Override
+    public PrimaryKey primaryKeyFromRowId(long sstableRowId)
+    {
+        tokenBuffer.putLong(rowIdToToken.get(sstableRowId));
+        tokenBuffer.rewind();
+        return primaryKeyFactory.createDeferred(partitioner.getTokenFactory().fromByteArray(tokenBuffer), () -> supplier(sstableRowId));
+    }
+
+    @Override
+    public long exactRowIdOrInvertedCeiling(PrimaryKey key)
+    {
+        return rowIdToToken.indexOf(key.token().getLongValue());
+    }
+
+    @Override
+    public long ceiling(PrimaryKey key)
+    {
+        var rowId = exactRowIdOrInvertedCeiling(key);
+        if (rowId >= 0)
+            return rowId;
+        if (rowId == Long.MIN_VALUE)
+            return -1;
+        else
+            return -rowId - 1;
+    }
+
+    @Override
+    public long floor(PrimaryKey key)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long count()
+    {
+        return rowIdToToken.length();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        FileUtils.closeQuietly(rowIdToToken, rowIdToOffset, keyFetcher);
+    }
+
+    private PrimaryKey supplier(long sstableRowId)
+    {
+        return primaryKeyFactory.createPartitionKeyOnly(keyFetcher.apply(rowIdToOffset.get(sstableRowId)));
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java b/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java
deleted file mode 100644
index 2e03d13b36ba..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.io.Closeable;
-import java.util.EnumMap;
-import java.util.Map;
-
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.FileUtils;
-
-/**
- * Maintains a mapping of {@link IndexComponent}s to associated {@link FileHandle}s for
- * read operations on the components. Users of this class are returned copies of the
- * {@link FileHandle}s using {@link FileHandle#sharedCopy()} so returned handles still
- * need to be closed by the user.
- */
-public class PerColumnIndexFiles implements Closeable
-{
-    private final Map<IndexComponent, FileHandle> files = new EnumMap<>(IndexComponent.class);
-    private final IndexDescriptor indexDescriptor;
-    private final IndexIdentifier indexIdentifier;
-
-    public PerColumnIndexFiles(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier)
-    {
-        this.indexDescriptor = indexDescriptor;
-        this.indexIdentifier = indexIdentifier;
-        for (IndexComponent component : indexDescriptor.version.onDiskFormat().perColumnIndexComponents(indexTermType))
-        {
-            if (component == IndexComponent.META || component == IndexComponent.COLUMN_COMPLETION_MARKER)
-                continue;
-            files.put(component, indexDescriptor.createPerIndexFileHandle(component, indexIdentifier, this::close));
-        }
-    }
-
-    public FileHandle termsData()
-    {
-        return getFile(IndexComponent.TERMS_DATA);
-    }
-
-    public FileHandle postingLists()
-    {
-        return getFile(IndexComponent.POSTING_LISTS);
-    }
-
-    public FileHandle balancedTree()
-    {
-        return getFile(IndexComponent.BALANCED_TREE);
-    }
-
-    public FileHandle compressedVectors()
-    {
-        return getFile(IndexComponent.COMPRESSED_VECTORS);
-    }
-
-    private FileHandle getFile(IndexComponent indexComponent)
-    {
-        FileHandle file = files.get(indexComponent);
-        if (file == null)
-            throw new IllegalArgumentException(String.format(indexIdentifier.logMessage("Component %s not found for SSTable %s"),
-                                                             indexComponent, indexDescriptor.sstableDescriptor));
-
-        return file.sharedCopy();
-    }
-
-    @Override
-    public void close()
-    {
-        FileUtils.closeQuietly(files.values());
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/PerIndexFiles.java b/src/java/org/apache/cassandra/index/sai/disk/v1/PerIndexFiles.java
new file mode 100644
index 000000000000..3df29c987edf
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/PerIndexFiles.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.io.UncheckedIOException;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.slf4j.Logger;
+
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+
+public class PerIndexFiles implements Closeable
+{
+    private static final Logger logger = org.slf4j.LoggerFactory.getLogger(PerIndexFiles.class);
+
+    private final Map<IndexComponentType, FileHandle> files = new EnumMap<>(IndexComponentType.class);
+    private final IndexComponents.ForRead perIndexComponents;
+
+    public PerIndexFiles(IndexComponents.ForRead perIndexComponents)
+    {
+        this.perIndexComponents = perIndexComponents;
+
+        var toOpen = new HashSet<>(perIndexComponents.expectedComponentsForVersion());
+        toOpen.remove(IndexComponentType.META);
+        toOpen.remove(IndexComponentType.COLUMN_COMPLETION_MARKER);
+
+        var componentsPresent = new HashSet<IndexComponentType>();
+        for (IndexComponentType component : toOpen)
+        {
+            try
+            {
+                files.put(component, perIndexComponents.get(component).createFileHandle());
+                componentsPresent.add(component);
+            }
+            catch (UncheckedIOException e)
+            {
+                // leave logging until we're done
+            }
+        }
+
+        logger.info("Components present for {} are {}", perIndexComponents.indexDescriptor(), componentsPresent);
+    }
+
+    public IndexComponents.ForRead usedPerIndexComponents()
+    {
+        return perIndexComponents;
+    }
+
+    /** It is the caller's responsibility to close the returned file handle. */
+    public FileHandle termsData()
+    {
+        return getFile(IndexComponentType.TERMS_DATA).sharedCopy();
+    }
+
+    /** It is the caller's responsibility to close the returned file handle. */
+    public FileHandle postingLists()
+    {
+        return getFile(IndexComponentType.POSTING_LISTS).sharedCopy();
+    }
+
+    /** It is the caller's responsibility to close the returned file handle. */
+    public FileHandle kdtree()
+    {
+        return getFile(IndexComponentType.KD_TREE).sharedCopy();
+    }
+
+    /** It is the caller's responsibility to close the returned file handle. */
+    public FileHandle kdtreePostingLists()
+    {
+        return getFile(IndexComponentType.KD_TREE_POSTING_LISTS).sharedCopy();
+    }
+
+    /** It is the caller's responsibility to close the returned file handle. */
+    public FileHandle vectors()
+    {
+        return getFile(IndexComponentType.VECTOR).sharedCopy();
+    }
+
+    /** It is the caller's responsibility to close the returned file handle. */
+    public FileHandle pq()
+    {
+        return getFile(IndexComponentType.PQ).sharedCopy();
+    }
+
+    /** It is the caller's responsibility to close the returned file handle. */
+    public FileHandle docLengths()
+    {
+        return getFile(IndexComponentType.DOC_LENGTHS).sharedCopy();
+    }
+
+    public FileHandle getFile(IndexComponentType indexComponentType)
+    {
+        FileHandle file = files.get(indexComponentType);
+        if (file == null)
+            throw new IllegalArgumentException(String.format(perIndexComponents.logMessage("Component %s not found for SSTable %s"),
+                                                             indexComponentType,
+                                                             perIndexComponents.descriptor()));
+
+        return file;
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(files.values());
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SAICodecUtils.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SAICodecUtils.java
deleted file mode 100644
index 58be96a5314c..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/SAICodecUtils.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.io.IOException;
-
-import org.apache.cassandra.index.sai.disk.format.Version;
-import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-
-import static org.apache.lucene.codecs.CodecUtil.CODEC_MAGIC;
-import static org.apache.lucene.codecs.CodecUtil.FOOTER_MAGIC;
-import static org.apache.lucene.codecs.CodecUtil.footerLength;
-import static org.apache.lucene.codecs.CodecUtil.readBEInt;
-import static org.apache.lucene.codecs.CodecUtil.readBELong;
-import static org.apache.lucene.codecs.CodecUtil.writeBEInt;
-import static org.apache.lucene.codecs.CodecUtil.writeBELong;
-
-public class SAICodecUtils
-{
-    // Lucene switched from big-endian to little-endian file format, but retained
-    // big-endian values in CodecUtils header and footer for compatibility.
-    // We follow their lead and use explicitly big-endian values here.
-
-    public static final String FOOTER_POINTER = "footerPointer";
-
-    public static void writeHeader(IndexOutput out) throws IOException
-    {
-        writeBEInt(out, CODEC_MAGIC);
-        out.writeString(Version.LATEST.toString());
-    }
-
-    public static void writeFooter(IndexOutput out) throws IOException
-    {
-        writeBEInt(out, FOOTER_MAGIC);
-        writeBEInt(out, 0);
-        writeChecksum(out);
-    }
-
-    public static void checkHeader(DataInput in) throws IOException
-    {
-        final int actualMagic = readBEInt(in);
-        if (actualMagic != CODEC_MAGIC)
-        {
-            throw new CorruptIndexException("codec header mismatch: actual header=" + actualMagic + " vs expected header=" + CODEC_MAGIC, in);
-        }
-        final Version actualVersion = Version.parse(in.readString());
-        if (!actualVersion.onOrAfter(Version.EARLIEST))
-        {
-            throw new IOException("Unsupported version: " + actualVersion);
-        }
-    }
-
-    public static void checkFooter(ChecksumIndexInput in) throws IOException
-    {
-        validateFooter(in, false);
-        long actualChecksum = in.getChecksum();
-        long expectedChecksum = readChecksum(in);
-        if (expectedChecksum != actualChecksum)
-        {
-            throw new CorruptIndexException("checksum failed (hardware problem?) : expected=" + Long.toHexString(expectedChecksum) +
-                                            " actual=" + Long.toHexString(actualChecksum), in);
-        }
-    }
-
-    public static void validate(IndexInput input) throws IOException
-    {
-        checkHeader(input);
-        validateFooterAndResetPosition(input);
-    }
-
-    public static void validate(IndexInput input, long footerPointer) throws IOException
-    {
-        checkHeader(input);
-
-        long current = input.getFilePointer();
-        input.seek(footerPointer);
-        validateFooter(input, true);
-
-        input.seek(current);
-    }
-
-    /**
-     * See {@link org.apache.lucene.codecs.CodecUtil#checksumEntireFile(org.apache.lucene.store.IndexInput)}.
-     * @param input IndexInput to validate.
-     * @throws IOException if a corruption is detected.
-     */
-    public static void validateChecksum(IndexInput input) throws IOException
-    {
-        IndexInput clone = input.clone();
-        clone.seek(0L);
-        ChecksumIndexInput in = IndexFileUtils.getBufferedChecksumIndexInput(clone);
-
-        assert in.getFilePointer() == 0L : in.getFilePointer() + " bytes already read from this input!";
-
-        if (in.length() < (long) footerLength())
-            throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), input);
-        else
-        {
-            in.seek(in.length() - (long) footerLength());
-            checkFooter(in);
-        }
-    }
-
-    // Copied from Lucene PackedInts as they are not public
-
-    public static int checkBlockSize(int blockSize, int minBlockSize, int maxBlockSize)
-    {
-        if (blockSize >= minBlockSize && blockSize <= maxBlockSize)
-        {
-            if ((blockSize & blockSize - 1) != 0)
-            {
-                throw new IllegalArgumentException("blockSize must be a power of two, got " + blockSize);
-            }
-            else
-            {
-                return Integer.numberOfTrailingZeros(blockSize);
-            }
-        }
-        else
-        {
-            throw new IllegalArgumentException("blockSize must be >= " + minBlockSize + " and <= " + maxBlockSize + ", got " + blockSize);
-        }
-    }
-
-    public static int numBlocks(long size, int blockSize)
-    {
-        if (size < 0)
-            throw new IllegalArgumentException("size cannot be negative");
-
-        int numBlocks = (int)(size / (long)blockSize) + (size % (long)blockSize == 0L ? 0 : 1);
-        if ((long)numBlocks * (long)blockSize < size)
-        {
-            throw new IllegalArgumentException("size is too large for this block size");
-        }
-        else
-        {
-            return numBlocks;
-        }
-    }
-
-    // Copied from Lucene BlockPackedReaderIterator as they are not public
-
-    /**
-     * Same as DataInput.readVLong but supports negative values
-     */
-    public static long readVLong(DataInput in) throws IOException
-    {
-        byte b = in.readByte();
-        if (b >= 0) return b;
-        long i = b & 0x7FL;
-        b = in.readByte();
-        i |= (b & 0x7FL) << 7;
-        if (b >= 0) return i;
-        b = in.readByte();
-        i |= (b & 0x7FL) << 14;
-        if (b >= 0) return i;
-        b = in.readByte();
-        i |= (b & 0x7FL) << 21;
-        if (b >= 0) return i;
-        b = in.readByte();
-        i |= (b & 0x7FL) << 28;
-        if (b >= 0) return i;
-        b = in.readByte();
-        i |= (b & 0x7FL) << 35;
-        if (b >= 0) return i;
-        b = in.readByte();
-        i |= (b & 0x7FL) << 42;
-        if (b >= 0) return i;
-        b = in.readByte();
-        i |= (b & 0x7FL) << 49;
-        if (b >= 0) return i;
-        b = in.readByte();
-        i |= (b & 0xFFL) << 56;
-        return i;
-    }
-
-    public static void validateFooterAndResetPosition(IndexInput in) throws IOException
-    {
-        long position = in.getFilePointer();
-        long fileLength = in.length();
-        long footerLength = footerLength();
-        long footerPosition = fileLength - footerLength;
-
-        if (footerPosition < 0)
-        {
-            throw new CorruptIndexException("invalid codec footer (file truncated?): file length=" + fileLength + ", footer length=" + footerLength, in);
-        }
-
-        in.seek(footerPosition);
-        validateFooter(in, false);
-        in.seek(position);
-    }
-
-    /**
-     * Copied from org.apache.lucene.codecs.CodecUtil.validateFooter(IndexInput).
-     *
-     * If the file is segmented then the footer can exist in the middle of the file
-     * so, we shouldn't check that the footer size is correct, we just check that the
-     * footer values are correct.
-     */
-    private static void validateFooter(IndexInput in, boolean segmented) throws IOException
-    {
-        long remaining = in.length() - in.getFilePointer();
-        long expected = footerLength();
-
-        if (!segmented)
-        {
-            if (remaining < expected)
-            {
-                throw new CorruptIndexException("misplaced codec footer (file truncated?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
-            }
-            else if (remaining > expected)
-            {
-                throw new CorruptIndexException("misplaced codec footer (file extended?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
-            }
-        }
-
-        final int magic = readBEInt(in);
-
-        if (magic != FOOTER_MAGIC)
-        {
-            throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in);
-        }
-
-        final int algorithmID = readBEInt(in);
-
-        if (algorithmID != 0)
-        {
-            throw new CorruptIndexException("codec footer mismatch: unknown algorithmID: " + algorithmID, in);
-        }
-    }
-
-    // Copied from Lucene CodecUtil as they are not public
-
-    /**
-     * Writes checksum value as a 64-bit long to the output.
-     * @throws IllegalStateException if CRC is formatted incorrectly (wrong bits set)
-     * @throws IOException if an i/o error occurs
-     */
-    private static void writeChecksum(IndexOutput output) throws IOException
-    {
-        long value = output.getChecksum();
-        if ((value & 0xFFFFFFFF00000000L) != 0)
-        {
-            throw new IllegalStateException("Illegal checksum: " + value + " (resource=" + output + ')');
-        }
-        writeBELong(output, value);
-    }
-
-    /**
-     * Reads checksum value as a 64-bit long from the input.
-     * @throws CorruptIndexException if CRC is formatted incorrectly (wrong bits set)
-     * @throws IOException if an i/o error occurs
-     */
-    private static long readChecksum(IndexInput input) throws IOException
-    {
-        long value = readBELong(input);
-        if ((value & 0xFFFFFFFF00000000L) != 0)
-        {
-            throw new CorruptIndexException("Illegal checksum: " + value, input);
-        }
-        return value;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableComponentsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableComponentsWriter.java
index b6e006584ac0..263ac680cd73 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableComponentsWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableComponentsWriter.java
@@ -15,115 +15,76 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.cassandra.index.sai.disk.v1;
 
 import java.io.IOException;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Stopwatch;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
+import org.apache.cassandra.index.sai.disk.PerSSTableWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
 import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
-import org.apache.cassandra.index.sai.disk.v1.keystore.KeyStoreWriter;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.lucene.util.IOUtils;
 
-public class SSTableComponentsWriter implements PerSSTableIndexWriter
+/**
+ * Writes all SSTable-attached index token and offset structures.
+ */
+public class SSTableComponentsWriter implements PerSSTableWriter
 {
     protected static final Logger logger = LoggerFactory.getLogger(SSTableComponentsWriter.class);
 
-    private final IndexDescriptor indexDescriptor;
-    private final MetadataWriter metadataWriter;
-    private final NumericValuesWriter partitionSizeWriter;
-    private final NumericValuesWriter partitionRowsWriter;
+    private final IndexComponents.ForWrite perSSTableComponents;
     private final NumericValuesWriter tokenWriter;
-    private final KeyStoreWriter partitionKeysWriter;
-    private final KeyStoreWriter clusteringKeysWriter;
+    private final NumericValuesWriter offsetWriter;
+    private final MetadataWriter metadataWriter;
 
-    private long partitionId = -1;
-    // This is used to record the number of rows in each partition
-    private long partitionRowCount = 0;
+    private long currentKeyPartitionOffset;
 
-    public SSTableComponentsWriter(IndexDescriptor indexDescriptor) throws IOException
+    public SSTableComponentsWriter(IndexComponents.ForWrite perSSTableComponents) throws IOException
     {
-        this.indexDescriptor = indexDescriptor;
-        this.metadataWriter = new MetadataWriter(indexDescriptor.openPerSSTableOutput(IndexComponent.GROUP_META));
-        this.tokenWriter = new NumericValuesWriter(indexDescriptor, IndexComponent.ROW_TO_TOKEN, metadataWriter, false);
-        this.partitionRowsWriter = new NumericValuesWriter(indexDescriptor, IndexComponent.ROW_TO_PARTITION, metadataWriter, true);
-        this.partitionSizeWriter = new NumericValuesWriter(indexDescriptor, IndexComponent.PARTITION_TO_SIZE, metadataWriter, false);
-        IndexOutputWriter partitionKeyBlocksWriter = indexDescriptor.openPerSSTableOutput(IndexComponent.PARTITION_KEY_BLOCKS);
-        NumericValuesWriter partitionKeyBlockOffsetWriter = new NumericValuesWriter(indexDescriptor, IndexComponent.PARTITION_KEY_BLOCK_OFFSETS, metadataWriter, true);
-        this.partitionKeysWriter = new KeyStoreWriter(indexDescriptor.componentName(IndexComponent.PARTITION_KEY_BLOCKS),
-                                                      metadataWriter,
-                                                      partitionKeyBlocksWriter,
-                                                      partitionKeyBlockOffsetWriter,
-                                                      CassandraRelevantProperties.SAI_SORTED_TERMS_PARTITION_BLOCK_SHIFT.getInt(),
-                                                      false);
-        if (indexDescriptor.hasClustering())
-        {
-            IndexOutputWriter clusteringKeyBlocksWriter = indexDescriptor.openPerSSTableOutput(IndexComponent.CLUSTERING_KEY_BLOCKS);
-            NumericValuesWriter clusteringKeyBlockOffsetWriter = new NumericValuesWriter(indexDescriptor, IndexComponent.CLUSTERING_KEY_BLOCK_OFFSETS, metadataWriter, true);
-            this.clusteringKeysWriter = new KeyStoreWriter(indexDescriptor.componentName(IndexComponent.CLUSTERING_KEY_BLOCKS),
-                                                           metadataWriter,
-                                                           clusteringKeyBlocksWriter,
-                                                           clusteringKeyBlockOffsetWriter,
-                                                           CassandraRelevantProperties.SAI_SORTED_TERMS_CLUSTERING_BLOCK_SHIFT.getInt(),
-                                                           true);
-        }
-        else
-        {
-            this.clusteringKeysWriter = null;
-        }
+        this.perSSTableComponents = perSSTableComponents;
+        this.metadataWriter = new MetadataWriter(perSSTableComponents);
+        this.tokenWriter = new NumericValuesWriter(perSSTableComponents.addOrGet(IndexComponentType.TOKEN_VALUES),
+                                                   metadataWriter, false);
+        this.offsetWriter = new NumericValuesWriter(perSSTableComponents.addOrGet(IndexComponentType.OFFSETS_VALUES),
+                                                    metadataWriter, true);
     }
 
     @Override
-    public void startPartition(DecoratedKey partitionKey) throws IOException
+    public void startPartition(long position)
     {
-        if (partitionId >= 0)
-            partitionSizeWriter.add(partitionRowCount);
-
-        partitionId++;
-        partitionRowCount = 0;
-        partitionKeysWriter.add(v -> ByteSource.of(partitionKey.getKey(), v));
-        if (indexDescriptor.hasClustering())
-            clusteringKeysWriter.startPartition();
+        currentKeyPartitionOffset = position;
     }
 
     @Override
     public void nextRow(PrimaryKey primaryKey) throws IOException
     {
-        tokenWriter.add(primaryKey.token().getLongValue());
-        partitionRowsWriter.add(partitionId);
-        partitionRowCount++;
-        if (indexDescriptor.hasClustering())
-            clusteringKeysWriter.add(indexDescriptor.clusteringComparator.asByteComparable(primaryKey.clustering()));
+        recordCurrentTokenOffset(primaryKey.token().getLongValue(), currentKeyPartitionOffset);
     }
 
     @Override
-    public void complete() throws IOException
+    public void complete(Stopwatch stopwatch) throws IOException
     {
-        try
-        {
-            partitionSizeWriter.add(partitionRowCount);
-            indexDescriptor.createComponentOnDisk(IndexComponent.GROUP_COMPLETION_MARKER);
-        }
-        finally
-        {
-            FileUtils.close(tokenWriter, partitionSizeWriter, partitionRowsWriter, partitionKeysWriter, clusteringKeysWriter, metadataWriter);
-        }
+        IOUtils.close(tokenWriter, offsetWriter, metadataWriter);
+        perSSTableComponents.markComplete();
     }
 
     @Override
-    public void abort()
+    public void abort(Throwable accumulator)
+    {
+        logger.debug(perSSTableComponents.logMessage("Aborting token/offset writer for {}..."), perSSTableComponents.descriptor());
+        perSSTableComponents.forceDeleteAllComponents();
+    }
+
+    @VisibleForTesting
+    public void recordCurrentTokenOffset(long tokenValue, long keyOffset) throws IOException
     {
-        logger.debug(indexDescriptor.logMessage("Aborting per-SSTable index component writer for {}..."), indexDescriptor.sstableDescriptor);
-        indexDescriptor.deletePerSSTableIndexComponents();
+        tokenWriter.add(tokenValue);
+        offsetWriter.add(keyOffset);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
index 58ee69a21565..5884b49501d6 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
@@ -26,52 +26,79 @@
 import java.util.function.BooleanSupplier;
 import javax.annotation.concurrent.NotThreadSafe;
 
+import com.google.common.base.Preconditions;
 import com.google.common.base.Stopwatch;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import io.github.jbellis.jvector.quantization.BinaryQuantization;
+import io.github.jbellis.jvector.quantization.ProductQuantization;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
-import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.PerIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter;
+import org.apache.cassandra.index.sai.disk.vector.CassandraDiskAnn;
+import org.apache.cassandra.index.sai.disk.vector.CassandraOnHeapGraph;
+import org.apache.cassandra.index.sai.disk.vector.VectorCompression.CompressionType;
 import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
  * Column index writer that accumulates (on-heap) indexed data from a compacted SSTable as it's being flushed to disk.
  */
 @NotThreadSafe
-public class SSTableIndexWriter implements PerColumnIndexWriter
+public class SSTableIndexWriter implements PerIndexWriter
 {
     private static final Logger logger = LoggerFactory.getLogger(SSTableIndexWriter.class);
 
-    private final IndexDescriptor indexDescriptor;
-    private final StorageAttachedIndex index;
+    private final IndexComponents.ForWrite perIndexComponents;
+    private final IndexContext indexContext;
     private final long nowInSec = FBUtilities.nowInSeconds();
-    private final AbstractAnalyzer analyzer;
     private final NamedMemoryLimiter limiter;
     private final BooleanSupplier isIndexValid;
-    private final List<SegmentMetadata> segments = new ArrayList<>();
+    private final long keyCount;
 
     private boolean aborted = false;
+
+    // segment writer
     private SegmentBuilder currentBuilder;
+    private final List<SegmentMetadata> segments = new ArrayList<>();
 
-    public SSTableIndexWriter(IndexDescriptor indexDescriptor,
-                              StorageAttachedIndex index,
-                              NamedMemoryLimiter limiter,
-                              BooleanSupplier isIndexValid)
+    public SSTableIndexWriter(IndexComponents.ForWrite perIndexComponents, NamedMemoryLimiter limiter, BooleanSupplier isIndexValid, long keyCount)
     {
-        this.indexDescriptor = indexDescriptor;
-        this.index = index;
-        this.analyzer = index.hasAnalyzer() ? index.analyzer() : null;
+        this.perIndexComponents = perIndexComponents;
+        this.indexContext = perIndexComponents.context();
+        Preconditions.checkNotNull(indexContext, "Provided components %s are the per-sstable ones, expected per-index ones", perIndexComponents);
         this.limiter = limiter;
         this.isIndexValid = isIndexValid;
+        this.keyCount = keyCount;
+    }
+
+    @Override
+    public IndexContext indexContext()
+    {
+        return indexContext;
+    }
+
+    @Override
+    public IndexComponents.ForWrite writtenComponents()
+    {
+        return perIndexComponents;
     }
 
     @Override
@@ -80,23 +107,29 @@ public void addRow(PrimaryKey key, Row row, long sstableRowId) throws IOExceptio
         if (maybeAbort())
             return;
 
-        if (index.termType().isNonFrozenCollection())
+        // This is to avoid duplicates (and also reduce space taken by indexes on static columns).
+        // An index on a static column indexes static rows only.
+        // An index on a non-static column indexes regular rows only.
+        if (indexContext.getDefinition().isStatic() != row.isStatic())
+            return;
+
+        if (indexContext.isNonFrozenCollection())
         {
-            Iterator<ByteBuffer> valueIterator = index.termType().valuesOf(row, nowInSec);
+            Iterator<ByteBuffer> valueIterator = indexContext.getValuesOf(row, nowInSec);
             if (valueIterator != null)
             {
                 while (valueIterator.hasNext())
                 {
                     ByteBuffer value = valueIterator.next();
-                    addTerm(index.termType().asIndexBytes(value.duplicate()), key, sstableRowId);
+                    addTerm(TypeUtil.asIndexBytes(value.duplicate(), indexContext.getValidator()), key, sstableRowId, indexContext.getValidator());
                 }
             }
         }
         else
         {
-            ByteBuffer value = index.termType().valueOf(key.partitionKey(), row, nowInSec);
+            ByteBuffer value = indexContext.getValueOf(key.partitionKey(), row, nowInSec);
             if (value != null)
-                addTerm(index.termType().asIndexBytes(value.duplicate()), key, sstableRowId);
+                addTerm(TypeUtil.asIndexBytes(value.duplicate(), indexContext.getValidator()), key, sstableRowId, indexContext.getValidator());
         }
     }
 
@@ -110,7 +143,7 @@ public void complete(Stopwatch stopwatch) throws IOException
         long elapsed;
 
         boolean emptySegment = currentBuilder == null || currentBuilder.isEmpty();
-        logger.debug(index.identifier().logMessage("Completing index flush with {}buffered data..."), emptySegment ? "no " : "");
+        logger.debug("Completing index flush with {}buffered data...", emptySegment ? "no " : "");
 
         try
         {
@@ -119,8 +152,8 @@ public void complete(Stopwatch stopwatch) throws IOException
             {
                 flushSegment();
                 elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-                logger.debug(index.identifier().logMessage("Completed flush of final segment for SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
-                             indexDescriptor.sstableDescriptor,
+                logger.debug("Completed flush of final segment for SSTable {}. Duration: {} ms. Total elapsed: {} ms",
+                             perIndexComponents.descriptor(),
                              elapsed - start,
                              elapsed);
             }
@@ -129,30 +162,34 @@ public void complete(Stopwatch stopwatch) throws IOException
             if (currentBuilder != null)
             {
                 long bytesAllocated = currentBuilder.totalBytesAllocated();
-                long globalBytesUsed = currentBuilder.release();
-                logger.debug(index.identifier().logMessage("Flushing final segment for SSTable {} released {}. Global segment memory usage now at {}."),
-                             indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
+                long globalBytesUsed = currentBuilder.release(indexContext);
+                logger.debug("Flushing final segment for SSTable {} released {}. Global segment memory usage now at {}",
+                             perIndexComponents.descriptor(), FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
             }
 
             writeSegmentsMetadata();
-
-            // write column index completion marker, indicating whether the index is empty
-            ColumnCompletionMarkerUtil.create(indexDescriptor, index.identifier(), segments.isEmpty());
+            perIndexComponents.markComplete();
         }
         finally
         {
-            index.indexMetrics().segmentsPerCompaction.update(segments.size());
-            segments.clear();
-            index.indexMetrics().compactionCount.inc();
+            if (indexContext.getIndexMetrics() != null)
+            {
+                indexContext.getIndexMetrics().segmentsPerCompaction.update(segments.size());
+                segments.clear();
+                indexContext.getIndexMetrics().compactionCount.inc();
+            }
         }
     }
 
     @Override
     public void abort(Throwable cause)
     {
+        if (aborted)
+            return;
+
         aborted = true;
 
-        logger.warn(index.identifier().logMessage("Aborting SSTable index flush for {}..."), indexDescriptor.sstableDescriptor, cause);
+        logger.warn("Aborting SSTable index flush for {}...", perIndexComponents.descriptor(), cause);
 
         // It's possible for the current builder to be unassigned after we flush a final segment.
         if (currentBuilder != null)
@@ -160,12 +197,15 @@ public void abort(Throwable cause)
             // If an exception is thrown out of any writer operation prior to successful segment
             // flush, we will end up here, and we need to free up builder memory tracked by the limiter:
             long allocated = currentBuilder.totalBytesAllocated();
-            long globalBytesUsed = currentBuilder.release();
-            logger.debug(index.identifier().logMessage("Aborting index writer for SSTable {} released {}. Global segment memory usage now at {}."),
-                         indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(allocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
+            long globalBytesUsed = currentBuilder.release(indexContext);
+            logger.debug("Aborting index writer for SSTable {} released {}. Global segment memory usage now at {}",
+                         perIndexComponents.descriptor(), FBUtilities.prettyPrintMemory(allocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
         }
 
-        indexDescriptor.deleteColumnIndex(index.termType(), index.identifier());
+        if (CassandraRelevantProperties.DELETE_CORRUPT_SAI_COMPONENTS.getBoolean())
+            perIndexComponents.forceDeleteAllComponents();
+        else
+            logger.debug("Skipping delete of index components after failure on index build of {}.{}", perIndexComponents.indexDescriptor(), indexContext);
     }
 
     /**
@@ -181,59 +221,41 @@ private boolean maybeAbort()
         if (isIndexValid.getAsBoolean())
             return false;
 
-        abort(new RuntimeException(String.format("index %s is dropped", index.identifier())));
+        abort(new RuntimeException(String.format("index %s is dropped", indexContext.getIndexName())));
         return true;
     }
 
-    private void addTerm(ByteBuffer term, PrimaryKey key, long sstableRowId) throws IOException
+    private void addTerm(ByteBuffer term, PrimaryKey key, long sstableRowId, AbstractType<?> type) throws IOException
     {
-        if (!index.validateTermSize(key.partitionKey(), term, false, null))
+        if (!indexContext.validateMaxTermSize(key.partitionKey(), term))
             return;
 
         if (currentBuilder == null)
         {
-            currentBuilder = newSegmentBuilder();
+            currentBuilder = newSegmentBuilder(sstableRowId);
         }
         else if (shouldFlush(sstableRowId))
         {
             flushSegment();
-            currentBuilder = newSegmentBuilder();
+            currentBuilder = newSegmentBuilder(sstableRowId);
         }
 
-        // Some types support empty byte buffers:
-        if (term.remaining() == 0 && !index.termType().indexType().allowsEmpty()) return;
+        if (term.remaining() == 0 && TypeUtil.skipsEmptyValue(indexContext.getValidator()))
+            return;
 
-        if (analyzer == null || !index.termType().isLiteral())
-        {
-            limiter.increment(currentBuilder.add(term, key, sstableRowId));
-        }
-        else
-        {
-            analyzer.reset(term);
-            try
-            {
-                while (analyzer.hasNext())
-                {
-                    ByteBuffer tokenTerm = analyzer.next();
-                    limiter.increment(currentBuilder.add(tokenTerm, key, sstableRowId));
-                }
-            }
-            finally
-            {
-                analyzer.end();
-            }
-        }
+        long allocated = currentBuilder.analyzeAndAdd(term, type, key, sstableRowId);
+        limiter.increment(allocated);
     }
 
     private boolean shouldFlush(long sstableRowId)
     {
-        // If we've hit the minimum flush size and, we've breached the global limit, flush a new segment:
+        // If we've hit the minimum flush size and we've breached the global limit, flush a new segment:
         boolean reachMemoryLimit = limiter.usageExceedsLimit() && currentBuilder.hasReachedMinimumFlushSize();
 
-        if (reachMemoryLimit)
+        if (currentBuilder.requiresFlush() || reachMemoryLimit)
         {
-            logger.debug(index.identifier().logMessage("Global limit of {} and minimum flush size of {} exceeded. " +
-                                                       "Current builder usage is {} for {} cells. Global Usage is {}. Flushing..."),
+            logger.debug("Global limit of {} and minimum flush size of {} exceeded. " +
+                         "Current builder usage is {} for {} rows. Global Usage is {}. Flushing...",
                          FBUtilities.prettyPrintMemory(limiter.limitBytes()),
                          FBUtilities.prettyPrintMemory(currentBuilder.getMinimumFlushBytes()),
                          FBUtilities.prettyPrintMemory(currentBuilder.totalBytesAllocated()),
@@ -241,32 +263,44 @@ private boolean shouldFlush(long sstableRowId)
                          FBUtilities.prettyPrintMemory(limiter.currentBytesUsed()));
         }
 
-        return reachMemoryLimit || currentBuilder.exceedsSegmentLimit(sstableRowId);
+        return reachMemoryLimit || currentBuilder.exceedsSegmentLimit(sstableRowId) || currentBuilder.requiresFlush();
     }
 
     private void flushSegment() throws IOException
     {
-        long start = Clock.Global.nanoTime();
+        currentBuilder.awaitAsyncAdditions();
+        if (currentBuilder.supportsAsyncAdd()
+            && currentBuilder.totalBytesAllocatedConcurrent.sum() > 1.1 * currentBuilder.totalBytesAllocated())
+        {
+            logger.warn("Concurrent memory usage is higher than estimated: {} vs {}",
+                        currentBuilder.totalBytesAllocatedConcurrent.sum(), currentBuilder.totalBytesAllocated());
+        }
 
+        // throw exceptions that occurred during async addInternal()
+        var ae = currentBuilder.getAsyncThrowable();
+        if (ae != null)
+            Throwables.throwAsUncheckedException(ae);
+
+        long start = nanoTime();
         try
         {
             long bytesAllocated = currentBuilder.totalBytesAllocated();
-
-            SegmentMetadata segmentMetadata = currentBuilder.flush(indexDescriptor);
-
-            long flushMillis = Math.max(1, TimeUnit.NANOSECONDS.toMillis(Clock.Global.nanoTime() - start));
+            SegmentMetadata segmentMetadata = currentBuilder.flush();
+            long flushMillis = Math.max(1, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start));
 
             if (segmentMetadata != null)
             {
                 segments.add(segmentMetadata);
 
                 double rowCount = segmentMetadata.numRows;
-                index.indexMetrics().compactionSegmentCellsPerSecond.update((long)(rowCount / flushMillis * 1000.0));
+                if (indexContext.getIndexMetrics() != null)
+                    indexContext.getIndexMetrics().compactionSegmentCellsPerSecond.update((long)(rowCount / flushMillis * 1000.0));
 
                 double segmentBytes = segmentMetadata.componentMetadatas.indexSize();
-                index.indexMetrics().compactionSegmentBytesPerSecond.update((long)(segmentBytes / flushMillis * 1000.0));
+                if (indexContext.getIndexMetrics() != null)
+                    indexContext.getIndexMetrics().compactionSegmentBytesPerSecond.update((long)(segmentBytes / flushMillis * 1000.0));
 
-                logger.debug(index.identifier().logMessage("Flushed segment with {} cells for a total of {} in {} ms."),
+                logger.debug("Flushed segment with {} cells for a total of {} in {} ms",
                              (long) rowCount, FBUtilities.prettyPrintMemory((long) segmentBytes), flushMillis);
             }
 
@@ -274,17 +308,19 @@ private void flushSegment() throws IOException
             // flush. Note that any failure that occurs before this (even in term addition) will
             // actuate this column writer's abort logic from the parent SSTable-level writer, and
             // that abort logic will release the current builder's memory against the limiter.
-            long globalBytesUsed = currentBuilder.release();
+            long globalBytesUsed = currentBuilder.release(indexContext);
             currentBuilder = null;
-            logger.debug(index.identifier().logMessage("Flushing index segment for SSTable {} released {}. Global segment memory usage now at {}."),
-                         indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
+            logger.debug("Flushing index segment for SSTable {} released {}. Global segment memory usage now at {}",
+                         perIndexComponents.descriptor(), FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
 
         }
         catch (Throwable t)
         {
-            logger.error(index.identifier().logMessage("Failed to build index for SSTable {}."), indexDescriptor.sstableDescriptor, t);
-            indexDescriptor.deleteColumnIndex(index.termType(), index.identifier());
-            index.indexMetrics().segmentFlushErrors.inc();
+            logger.error("Failed to build index for SSTable {}", perIndexComponents.descriptor(), t);
+            perIndexComponents.forceDeleteAllComponents();
+
+            indexContext.getIndexMetrics().segmentFlushErrors.inc();
+
             throw t;
         }
     }
@@ -294,7 +330,7 @@ private void writeSegmentsMetadata() throws IOException
         if (segments.isEmpty())
             return;
 
-        try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, index.identifier())))
+        try (MetadataWriter writer = new MetadataWriter(perIndexComponents))
         {
             SegmentMetadata.write(writer, segments);
         }
@@ -305,16 +341,101 @@ private void writeSegmentsMetadata() throws IOException
         }
     }
 
-    private SegmentBuilder newSegmentBuilder()
+    private SegmentBuilder newSegmentBuilder(long rowIdOffset) throws IOException
     {
-        SegmentBuilder builder = index.termType().isVector() ? new SegmentBuilder.VectorSegmentBuilder(index, limiter)
-                                                             : new SegmentBuilder.TrieSegmentBuilder(index, limiter);
+        SegmentBuilder builder;
+
+        if (indexContext.isVector())
+        {
+            int dimension = ((VectorType<?>) indexContext.getValidator()).dimension;
+            boolean bqPreferred = indexContext.getIndexWriterConfig().getSourceModel().compressionProvider.apply(dimension).type == CompressionType.BINARY_QUANTIZATION;
+
+            // if we have a PQ instance available, we can use it to build a CompactionGraph;
+            // otherwise, build on heap (which will create PQ for next time, if we have enough vectors)
+            var pqi = CassandraOnHeapGraph.getPqIfPresent(indexContext, vc -> vc.type == CompressionType.PRODUCT_QUANTIZATION);
+            // If no PQ instance available in indexes of completed sstables, check if we just wrote one in the previous segment
+            if (pqi == null && !segments.isEmpty())
+                pqi = maybeReadPqFromLastSegment();
+
+            if ((bqPreferred || pqi != null) && V3OnDiskFormat.ENABLE_LTM_CONSTRUCTION)
+            {
+                var compressor = bqPreferred ? new BinaryQuantization(dimension) : pqi.pq;
+                var unitVectors = bqPreferred ? false : pqi.unitVectors;
+                var allRowsHaveVectors = allRowsHaveVectorsInWrittenSegments(indexContext);
+                builder = new SegmentBuilder.VectorOffHeapSegmentBuilder(perIndexComponents, rowIdOffset, keyCount, compressor, unitVectors, allRowsHaveVectors, limiter);
+            }
+            else
+            {
+                // building on heap is the only way to get a PQ from nothing (CompactionGraph only knows how to fine-tune an existing one)
+                builder = new SegmentBuilder.VectorOnHeapSegmentBuilder(perIndexComponents, rowIdOffset, keyCount, limiter);
+            }
+        }
+        else if (indexContext.isLiteral())
+        {
+            builder = new SegmentBuilder.RAMStringSegmentBuilder(perIndexComponents, rowIdOffset, limiter);
+        }
+        else
+        {
+            builder = new SegmentBuilder.KDTreeSegmentBuilder(perIndexComponents, rowIdOffset, limiter, indexContext.getIndexWriterConfig());
+        }
 
         long globalBytesUsed = limiter.increment(builder.totalBytesAllocated());
-        logger.debug(index.identifier().logMessage("Created new segment builder while flushing SSTable {}. Global segment memory usage now at {}."),
-                     indexDescriptor.sstableDescriptor,
+        logger.debug("Created new segment builder while flushing SSTable {}. Global segment memory usage now at {}",
+                     perIndexComponents.descriptor(),
                      FBUtilities.prettyPrintMemory(globalBytesUsed));
 
         return builder;
     }
+
+    private static boolean allRowsHaveVectorsInWrittenSegments(IndexContext indexContext)
+    {
+        for (SSTableIndex index : indexContext.getView().getIndexes())
+        {
+            for (Segment segment : index.getSegments())
+            {
+                if (segment.getIndexSearcher() instanceof  V2VectorIndexSearcher)
+                    return true; // V2 doesn't know, so we err on the side of being optimistic.  See comments in CompactionGraph
+                var searcher = (V5VectorIndexSearcher) segment.getIndexSearcher();
+                var structure = searcher.getPostingsStructure();
+                if (structure == V5VectorPostingsWriter.Structure.ZERO_OR_ONE_TO_MANY)
+                    return false;
+            }
+        }
+        return true;
+    }
+
+    private CassandraOnHeapGraph.PqInfo maybeReadPqFromLastSegment() throws IOException
+    {
+        var pqComponent = perIndexComponents.get(IndexComponentType.PQ);
+        assert pqComponent != null; // we always have a PQ component even if it's not actually PQ compression
+
+        var fhBuilder = StorageProvider.instance.indexBuildTimeFileHandleBuilderFor(pqComponent);
+        try (var fh = fhBuilder.complete();
+             var reader = fh.createReader())
+        {
+            var sm = segments.get(segments.size() - 1);
+            long offset = sm.componentMetadatas.get(IndexComponentType.PQ).offset;
+            // close parallel to code in CassandraDiskANN constructor, but different enough
+            // (we only want the PQ codebook) that it's difficult to extract into a common method
+            reader.seek(offset);
+            boolean unitVectors;
+            if (reader.readInt() == CassandraDiskAnn.PQ_MAGIC)
+            {
+                reader.readInt(); // skip over version
+                unitVectors = reader.readBoolean();
+            }
+            else
+            {
+                unitVectors = true;
+                reader.seek(offset);
+            }
+            var compressionType = CompressionType.values()[reader.readByte()];
+            if (compressionType == CompressionType.PRODUCT_QUANTIZATION)
+            {
+                var pq = ProductQuantization.load(reader);
+                return new CassandraOnHeapGraph.PqInfo(pq, unitVectors, sm.numRows);
+            }
+        }
+        return null;
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/Segment.java b/src/java/org/apache/cassandra/index/sai/disk/v1/Segment.java
new file mode 100644
index 000000000000..38cdf3f1b11d
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/Segment.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+
+import org.slf4j.Logger;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RangeUtil;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.CloseableIterator;
+
+/**
+ * Each segment represents an on-disk index structure (kdtree/terms/postings) flushed by memory limit or token boundaries,
+ * or max segment rowId limit, because of lucene's limitation on 2B(Integer.MAX_VALUE). It also helps to reduce resource
+ * consumption for read requests as only segments that intersect with read request data range need to be loaded.
+ */
+public class Segment implements Closeable
+{
+    private static final Logger logger = org.slf4j.LoggerFactory.getLogger(Segment.class);
+
+    private final Token.KeyBound minKeyBound;
+    private final Token.KeyBound maxKeyBound;
+
+    // per sstable
+    final PrimaryKeyMap.Factory primaryKeyMapFactory;
+    // per-index
+    public final PerIndexFiles indexFiles;
+    // per-segment
+    public final SegmentMetadata metadata;
+    public final SSTableContext sstableContext;
+
+    private final IndexSearcher index;
+
+    public Segment(IndexContext indexContext, SSTableContext sstableContext, PerIndexFiles indexFiles, SegmentMetadata metadata) throws IOException
+    {
+        this.minKeyBound = metadata.minKey.token().minKeyBound();
+        this.maxKeyBound = metadata.maxKey.token().maxKeyBound();
+
+        this.sstableContext = sstableContext;
+        this.primaryKeyMapFactory = sstableContext.primaryKeyMapFactory();
+        this.indexFiles = indexFiles;
+        this.metadata = metadata;
+
+        var version = indexFiles.usedPerIndexComponents().version();
+        IndexSearcher searcher = version.onDiskFormat().newIndexSearcher(sstableContext, indexContext, indexFiles, metadata);
+        logger.info("Opened searcher {} for segment {} with row id meta ({},{},{},{}) for index [{}] on column [{}] at version {}",
+                    searcher.getClass().getSimpleName(),
+                    sstableContext.descriptor(),
+                    metadata.segmentRowIdOffset,
+                    metadata.numRows,
+                    metadata.minSSTableRowId,
+                    metadata.maxSSTableRowId,
+                    indexContext.getIndexName(),
+                    indexContext.getColumnName(),
+                    version);
+        this.index = searcher;
+    }
+
+    @VisibleForTesting
+    public Segment(PrimaryKeyMap.Factory primaryKeyMapFactory,
+                   PerIndexFiles indexFiles,
+                   SegmentMetadata metadata,
+                   AbstractType<?> columnType)
+    {
+        this.primaryKeyMapFactory = primaryKeyMapFactory;
+        this.indexFiles = indexFiles;
+        this.metadata = metadata;
+        this.minKeyBound = null;
+        this.maxKeyBound = null;
+        this.index = null;
+        this.sstableContext = null;
+    }
+
+    @VisibleForTesting
+    public Segment(Token minKey, Token maxKey)
+    {
+        this.primaryKeyMapFactory = null;
+        this.indexFiles = null;
+        this.metadata = null;
+        this.minKeyBound = minKey.minKeyBound();
+        this.maxKeyBound = maxKey.maxKeyBound();
+        this.index = null;
+        this.sstableContext = null;
+    }
+
+    /**
+     * @return true if current segment intersects with query key range
+     */
+    public boolean intersects(AbstractBounds<PartitionPosition> keyRange)
+    {
+        return RangeUtil.intersects(minKeyBound, maxKeyBound, keyRange);
+    }
+
+    public long indexFileCacheSize()
+    {
+        return index == null ? 0 : index.indexFileCacheSize();
+    }
+
+    /**
+     * Search on-disk index synchronously
+     *
+     * @param expression to filter on disk index
+     * @param keyRange   key range specific in read command, used by ANN index
+     * @param context    to track per sstable cache and per query metrics
+     * @param defer      create the iterator in a deferred state
+     * @param limit      the num of rows to returned, used by ANN index
+     * @return range iterator of {@link PrimaryKey} that matches given expression
+     */
+    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer, int limit) throws IOException
+    {
+        return index.search(expression, keyRange, context, defer);
+    }
+
+    /**
+     * Order the on-disk index synchronously and produce an iterator in score order
+     *
+     * @param orderer    to filter on disk index
+     * @param keyRange   key range specific in read command, used by ANN index
+     * @param context    to track per sstable cache and per query metrics
+     * @param limit      the num of rows to returned, used by ANN index
+     * @return an iterator of {@link PrimaryKeyWithSortKey} in score order
+     */
+    public CloseableIterator<PrimaryKeyWithSortKey> orderBy(Orderer orderer, Expression slice, AbstractBounds<PartitionPosition> keyRange, QueryContext context, int limit) throws IOException
+    {
+        return index.orderBy(orderer, slice, keyRange, context, limit);
+    }
+
+    public IndexSearcher getIndexSearcher()
+    {
+        return index;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Segment segment = (Segment) o;
+        return Objects.equal(metadata, segment.metadata);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(metadata);
+    }
+
+    public CloseableIterator<PrimaryKeyWithSortKey> orderResultsBy(QueryContext context, List<PrimaryKey> keys, Orderer orderer, int limit) throws IOException
+    {
+        return index.orderResultsBy(sstableContext.sstable, context, keys, orderer, limit);
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(index);
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Segment{metadata=%s}", metadata);
+    }
+
+    /**
+     * Estimate how many nodes the index will visit to find the top `limit` results
+     * given the number of candidates that match other predicates and taking into
+     * account the size of the index itself.  (The smaller
+     * the number of candidates, the more nodes we expect to visit just to find
+     * results that are in that set.)
+     */
+    public double estimateAnnSearchCost(Orderer orderer, int limit, int candidates)
+    {
+        V2VectorIndexSearcher searcher = (V2VectorIndexSearcher) getIndexSearcher();
+        int rerankK = orderer.rerankKFor(limit, searcher.getCompression());
+        return searcher.estimateAnnSearchCost(rerankK, candidates);
+    }
+
+    /**
+     * Returns a modified LIMIT (top k) to use with the ANN index that is proportional
+     * to the number of rows in this segment, relative to the total rows in the sstable.
+     */
+    public int proportionalAnnLimit(int limit, long totalRows)
+    {
+        if (!V3OnDiskFormat.REDUCE_TOPK_ACROSS_SSTABLES)
+            return limit;
+
+        // Note: it is tempting to think that we should max out results for the first segment
+        // since that's where we're establishing our rerank floor.  This *does* reduce the number
+        // of calls to resume, but it's 10-15% slower overall, so don't do it.
+        // if (context.getAnnRerankFloor() == 0 && V3OnDiskFormat.ENABLE_RERANK_FLOOR)
+        //    return limit;
+
+        // We expect the number of top results found in each segment to be proportional to its number of rows.
+        // (We don't pad this number more because resuming a search if we guess too low is very very inexpensive.)
+        long segmentRows = 1 + metadata.maxSSTableRowId - metadata.minSSTableRowId;
+        int proportionalLimit = (int) Math.ceil(limit * ((double) segmentRows / totalRows));
+        assert proportionalLimit >= 1 : proportionalLimit;
+        return proportionalLimit;
+    }
+
+    public long estimateMatchingRowsCount(Expression predicate, AbstractBounds<PartitionPosition> keyRange)
+    {
+        return metadata.estimateNumRowsMatching(predicate);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentBuilder.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentBuilder.java
new file mode 100644
index 000000000000..70997ccedcd1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentBuilder.java
@@ -0,0 +1,622 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
+import java.util.stream.Collectors;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.quantization.VectorCompressor;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.analyzer.ByteLimitedMaterializer;
+import org.apache.cassandra.index.sai.analyzer.NoOpAnalyzer;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.RAMStringIndexer;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.BKDTreeRamBuffer;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.MutableOneDimPointValues;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.NumericIndexWriter;
+import org.apache.cassandra.index.sai.disk.v1.trie.InvertedIndexWriter;
+import org.apache.cassandra.index.sai.disk.vector.CassandraOnHeapGraph;
+import org.apache.cassandra.index.sai.disk.vector.CompactionGraph;
+import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.metrics.QuickSlidingWindowReservoir;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.apache.lucene.util.BytesRef;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_TEST_LAST_VALID_SEGMENTS;
+import static org.apache.cassandra.utils.FBUtilities.busyWaitWhile;
+
+/**
+ * Creates an on-heap index data structure to be flushed to an SSTable index.
+ * <p>
+ * Not threadsafe, but does potentially make concurrent calls to addInternal by
+ * delegating them to an asynchronous executor.  This will be done when supportsAsyncAdd is true.
+ * Callers should check getAsyncThrowable when they are done adding rows to see if there was an error.
+ */
+@NotThreadSafe
+public abstract class SegmentBuilder
+{
+    private static final Logger logger = LoggerFactory.getLogger(SegmentBuilder.class);
+
+    /** for parallelism within a single compaction */
+    public static final ExecutorService compactionExecutor = executorFactory().configurePooled("SegmentBuilder", Runtime.getRuntime().availableProcessors())
+                                                                                .withQueueLimit(10 * Runtime.getRuntime().availableProcessors())
+                                                                                .withKeepAlive(1, TimeUnit.MINUTES)
+                                                                                .withThreadPriority(Thread.MIN_PRIORITY)
+                                                                                .build();
+
+    // Served as safe net in case memory limit is not triggered or when merger merges small segments..
+    public static final long LAST_VALID_SEGMENT_ROW_ID = ((long)Integer.MAX_VALUE / 2) - 1L;
+    private static long testLastValidSegmentRowId = SAI_TEST_LAST_VALID_SEGMENTS.getLong();
+
+    /** The number of column indexes being built globally. (Starts at one to avoid divide by zero.) */
+    public static final AtomicLong ACTIVE_BUILDER_COUNT = new AtomicLong(1);
+
+    /** Minimum flush size, dynamically updated as segment builds are started and completed/aborted. */
+    private static volatile long minimumFlushBytes;
+
+    protected final IndexComponents.ForWrite components;
+
+    final AbstractType<?> termComparator;
+    final AbstractAnalyzer analyzer;
+
+    // track memory usage for this segment so we can flush when it gets too big
+    private final NamedMemoryLimiter limiter;
+    long totalBytesAllocated;
+    // when we're adding terms asynchronously, totalBytesAllocated will be an approximation and this tracks the exact size
+    final LongAdder totalBytesAllocatedConcurrent = new LongAdder();
+
+    private final long lastValidSegmentRowID;
+
+    private boolean flushed = false;
+    private boolean active = true;
+
+    // segment metadata
+    private long minSSTableRowId = -1;
+    private long maxSSTableRowId = -1;
+    private long segmentRowIdOffset = 0;
+    int rowCount = 0;
+    int maxSegmentRowId = -1;
+    // in token order
+    private PrimaryKey minKey;
+    private PrimaryKey maxKey;
+    // in termComparator order
+    protected ByteBuffer minTerm;
+    protected ByteBuffer maxTerm;
+
+    protected final AtomicInteger updatesInFlight = new AtomicInteger(0);
+    protected final QuickSlidingWindowReservoir termSizeReservoir = new QuickSlidingWindowReservoir(100);
+    protected AtomicReference<Throwable> asyncThrowable = new AtomicReference<>();
+
+
+    public boolean requiresFlush()
+    {
+        return false;
+    }
+
+    public static class KDTreeSegmentBuilder extends SegmentBuilder
+    {
+        protected final byte[] buffer;
+        private final BKDTreeRamBuffer kdTreeRamBuffer;
+        private final IndexWriterConfig indexWriterConfig;
+
+        KDTreeSegmentBuilder(IndexComponents.ForWrite components, long rowIdOffset, NamedMemoryLimiter limiter, IndexWriterConfig indexWriterConfig)
+        {
+            super(components, rowIdOffset, limiter);
+
+            int typeSize = TypeUtil.fixedSizeOf(termComparator);
+            this.kdTreeRamBuffer = new BKDTreeRamBuffer(1, typeSize);
+            this.buffer = new byte[typeSize];
+            this.indexWriterConfig = indexWriterConfig;
+            totalBytesAllocated = kdTreeRamBuffer.ramBytesUsed();
+            totalBytesAllocatedConcurrent.add(totalBytesAllocated);}
+
+        public boolean isEmpty()
+        {
+            return kdTreeRamBuffer.numRows() == 0;
+        }
+
+        @Override
+        protected long addInternal(List<ByteBuffer> terms, int segmentRowId)
+        {
+            assert terms.size() == 1;
+            TypeUtil.toComparableBytes(terms.get(0), termComparator, buffer);
+            return kdTreeRamBuffer.addPackedValue(segmentRowId, new BytesRef(buffer));
+        }
+
+        @Override
+        protected void flushInternal(SegmentMetadataBuilder metadataBuilder) throws IOException
+        {
+            try (NumericIndexWriter writer = new NumericIndexWriter(components,
+                                                                    TypeUtil.fixedSizeOf(termComparator),
+                                                                    maxSegmentRowId,
+                                                                    rowCount,
+                                                                    indexWriterConfig))
+            {
+
+                MutableOneDimPointValues values = kdTreeRamBuffer.asPointValues();
+                var metadataMap = writer.writeAll(metadataBuilder.intercept(values));
+                metadataBuilder.setComponentsMetadata(metadataMap);
+            }
+        }
+
+        @Override
+        public boolean requiresFlush()
+        {
+            return kdTreeRamBuffer.requiresFlush();
+        }
+    }
+
+    public static class RAMStringSegmentBuilder extends SegmentBuilder
+    {
+        final RAMStringIndexer ramIndexer;
+        private final ByteComparable.Version byteComparableVersion;
+
+        RAMStringSegmentBuilder(IndexComponents.ForWrite components, long rowIdOffset, NamedMemoryLimiter limiter)
+        {
+            super(components, rowIdOffset, limiter);
+            this.byteComparableVersion = components.byteComparableVersionFor(IndexComponentType.TERMS_DATA);
+            ramIndexer = new RAMStringIndexer(writeFrequencies());
+            totalBytesAllocated = ramIndexer.estimatedBytesUsed();
+            totalBytesAllocatedConcurrent.add(totalBytesAllocated);
+        }
+
+        private boolean writeFrequencies()
+        {
+            return !(analyzer instanceof NoOpAnalyzer) && Version.latest().onOrAfter(Version.EC);
+        }
+
+        public boolean isEmpty()
+        {
+            return ramIndexer.isEmpty();
+        }
+
+        @Override
+        protected long addInternal(List<ByteBuffer> terms, int segmentRowId)
+        {
+            var bytesRefs = terms.stream()
+                                 .map(term -> components.onDiskFormat().encodeForTrie(term, termComparator))
+                                 .map(encodedTerm -> ByteSourceInverse.readBytes(encodedTerm.asComparableBytes(byteComparableVersion)))
+                                 .map(BytesRef::new)
+                                 .collect(Collectors.toList());
+            // ramIndexer is responsible for merging duplicate (term, row) pairs
+            return ramIndexer.addAll(bytesRefs, segmentRowId);
+        }
+
+        @Override
+        protected void flushInternal(SegmentMetadataBuilder metadataBuilder) throws IOException
+        {
+            try (InvertedIndexWriter writer = new InvertedIndexWriter(components, writeFrequencies()))
+            {
+                TermsIterator termsWithPostings = ramIndexer.getTermsWithPostings(minTerm, maxTerm, byteComparableVersion);
+                var docLengths = ramIndexer.getDocLengths();
+                var metadataMap = writer.writeAll(metadataBuilder.intercept(termsWithPostings), docLengths);
+                metadataBuilder.setComponentsMetadata(metadataMap);
+            }
+        }
+
+        @Override
+        public boolean requiresFlush()
+        {
+            return ramIndexer.requiresFlush();
+        }
+    }
+
+    public static class VectorOffHeapSegmentBuilder extends SegmentBuilder
+    {
+        private final CompactionGraph graphIndex;
+
+        public VectorOffHeapSegmentBuilder(IndexComponents.ForWrite components,
+                                           long rowIdOffset,
+                                           long keyCount,
+                                           VectorCompressor<?> compressor,
+                                           boolean unitVectors,
+                                           boolean allRowsHaveVectors,
+                                           NamedMemoryLimiter limiter)
+        {
+            super(components, rowIdOffset, limiter);
+            try
+            {
+                graphIndex = new CompactionGraph(components, compressor, unitVectors, keyCount, allRowsHaveVectors);
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+            totalBytesAllocated = graphIndex.ramBytesUsed();
+            totalBytesAllocatedConcurrent.add(totalBytesAllocated);
+        }
+
+        @Override
+        public boolean isEmpty()
+        {
+            return graphIndex.isEmpty();
+        }
+
+        @Override
+        protected long addInternal(List<ByteBuffer> terms, int segmentRowId)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected long addInternalAsync(List<ByteBuffer> terms, int segmentRowId)
+        {
+            assert terms.size() == 1;
+
+            // CompactionGraph splits adding a node into two parts:
+            // (1) maybeAddVector, which must be done serially because it writes to disk incrementally
+            // (2) addGraphNode, which may be done asynchronously
+            CompactionGraph.InsertionResult result;
+            try
+            {
+                result = graphIndex.maybeAddVector(terms.get(0), segmentRowId);
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+            if (result.vector == null)
+                return result.bytesUsed;
+
+            updatesInFlight.incrementAndGet();
+            compactionExecutor.submit(() -> {
+                try
+                {
+                    long bytesAdded = result.bytesUsed + graphIndex.addGraphNode(result);
+                    totalBytesAllocatedConcurrent.add(bytesAdded);
+                    termSizeReservoir.update(bytesAdded);
+                }
+                catch (Throwable th)
+                {
+                    asyncThrowable.compareAndExchange(null, th);
+                }
+                finally
+                {
+                    updatesInFlight.decrementAndGet();
+                }
+            });
+            // bytes allocated will be approximated immediately as the average of recently added terms,
+            // rather than waiting until the async update completes to get the exact value.  The latter could
+            // result in a dangerously large discrepancy between the amount of memory actually consumed
+            // and the amount the limiter knows about if the queue depth grows.
+            busyWaitWhile(() -> termSizeReservoir.size() == 0 && asyncThrowable.get() == null);
+            if (asyncThrowable.get() != null) {
+                throw new RuntimeException("Error adding term asynchronously", asyncThrowable.get());
+            }
+            return (long) termSizeReservoir.getMean();
+        }
+
+        @Override
+        protected void flushInternal(SegmentMetadataBuilder metadataBuilder) throws IOException
+        {
+            if (graphIndex.isEmpty())
+                return;
+            var componentsMetadata = graphIndex.flush();
+            metadataBuilder.setComponentsMetadata(componentsMetadata);
+        }
+
+        @Override
+        public boolean supportsAsyncAdd()
+        {
+            return true;
+        }
+
+        @Override
+        public boolean requiresFlush()
+        {
+            return graphIndex.requiresFlush();
+        }
+
+        @Override
+        long release(IndexContext indexContext)
+        {
+            try
+            {
+                graphIndex.close();
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+            return super.release(indexContext);
+        }
+    }
+
+    public static class VectorOnHeapSegmentBuilder extends SegmentBuilder
+    {
+        private final CassandraOnHeapGraph<Integer> graphIndex;
+
+        public VectorOnHeapSegmentBuilder(IndexComponents.ForWrite components, long rowIdOffset, long keyCount, NamedMemoryLimiter limiter)
+        {
+            super(components, rowIdOffset, limiter);
+            graphIndex = new CassandraOnHeapGraph<>(components.context(), false, null);
+            totalBytesAllocated = graphIndex.ramBytesUsed();
+            totalBytesAllocatedConcurrent.add(totalBytesAllocated);
+        }
+
+        @Override
+        public boolean isEmpty()
+        {
+            return graphIndex.isEmpty();
+        }
+
+        @Override
+        protected long addInternal(List<ByteBuffer> terms, int segmentRowId)
+        {
+            assert terms.size() == 1;
+            return graphIndex.add(terms.get(0), segmentRowId);
+        }
+
+        @Override
+        protected long addInternalAsync(List<ByteBuffer> terms, int segmentRowId)
+        {
+            updatesInFlight.incrementAndGet();
+            compactionExecutor.submit(() -> {
+                try
+                {
+                    long bytesAdded = addInternal(terms, segmentRowId);
+                    totalBytesAllocatedConcurrent.add(bytesAdded);
+                    termSizeReservoir.update(bytesAdded);
+                }
+                catch (Throwable th)
+                {
+                    asyncThrowable.compareAndExchange(null, th);
+                }
+                finally
+                {
+                    updatesInFlight.decrementAndGet();
+                }
+            });
+            // bytes allocated will be approximated immediately as the average of recently added terms,
+            // rather than waiting until the async update completes to get the exact value.  The latter could
+            // result in a dangerously large discrepancy between the amount of memory actually consumed
+            // and the amount the limiter knows about if the queue depth grows.
+            busyWaitWhile(() -> termSizeReservoir.size() == 0 && asyncThrowable.get() == null);
+            if (asyncThrowable.get() != null) {
+                throw new RuntimeException("Error adding term asynchronously", asyncThrowable.get());
+            }
+            return (long) termSizeReservoir.getMean();
+        }
+
+        @Override
+        protected void flushInternal(SegmentMetadataBuilder metadataBuilder) throws IOException
+        {
+            var shouldFlush = graphIndex.preFlush(p -> p);
+            // there are no deletes to worry about when building the index during compaction,
+            // and SegmentBuilder::flush checks for the empty index case before calling flushInternal
+            assert shouldFlush;
+            var componentsMetadata = graphIndex.flush(components);
+            metadataBuilder.setComponentsMetadata(componentsMetadata);
+        }
+
+        @Override
+        public boolean supportsAsyncAdd()
+        {
+            return true;
+        }
+    }
+
+    private SegmentBuilder(IndexComponents.ForWrite components, long rowIdOffset, NamedMemoryLimiter limiter)
+    {
+        IndexContext context = Objects.requireNonNull(components.context(), "IndexContext must be set on segment builder");
+        this.components = components;
+        this.termComparator = context.getValidator();
+        this.analyzer = context.getAnalyzerFactory().create();
+        this.limiter = limiter;
+        this.segmentRowIdOffset = rowIdOffset;
+        this.lastValidSegmentRowID = testLastValidSegmentRowId >= 0 ? testLastValidSegmentRowId : LAST_VALID_SEGMENT_ROW_ID;
+
+        minimumFlushBytes = limiter.limitBytes() / ACTIVE_BUILDER_COUNT.getAndIncrement();
+    }
+
+    public SegmentMetadata flush() throws IOException
+    {
+        assert !flushed;
+        flushed = true;
+
+        if (getRowCount() == 0)
+        {
+            logger.warn(components.logMessage("No rows to index during flush of SSTable {}."), components.descriptor());
+            return null;
+        }
+
+        SegmentMetadataBuilder metadataBuilder = new SegmentMetadataBuilder(segmentRowIdOffset, components);
+        metadataBuilder.setKeyRange(minKey, maxKey);
+        metadataBuilder.setRowIdRange(minSSTableRowId, maxSSTableRowId);
+        metadataBuilder.setTermRange(minTerm, maxTerm);
+
+        flushInternal(metadataBuilder);
+        return metadataBuilder.build();
+    }
+
+    public long analyzeAndAdd(ByteBuffer rawTerm, AbstractType<?> type, PrimaryKey key, long sstableRowId)
+    {
+        long totalSize = 0;
+        if (TypeUtil.isLiteral(type))
+        {
+            var terms = ByteLimitedMaterializer.materializeTokens(analyzer, rawTerm, components.context(), key);
+            totalSize += add(terms, key, sstableRowId);
+        }
+        else
+        {
+            totalSize += add(List.of(rawTerm), key, sstableRowId);
+        }
+        return totalSize;
+    }
+
+    private long add(List<ByteBuffer> terms, PrimaryKey key, long sstableRowId)
+    {
+        assert !flushed : "Cannot add to flushed segment.";
+        assert sstableRowId >= maxSSTableRowId;
+        minSSTableRowId = minSSTableRowId < 0 ? sstableRowId : minSSTableRowId;
+        maxSSTableRowId = sstableRowId;
+
+        assert maxKey == null || maxKey.compareTo(key) <= 0;
+        minKey = minKey == null ? key : minKey;
+        maxKey = key;
+
+        // Update term boundaries for all terms in this row
+        for (ByteBuffer term : terms)
+        {
+            minTerm = TypeUtil.min(term, minTerm, termComparator, Version.latest());
+            maxTerm = TypeUtil.max(term, maxTerm, termComparator, Version.latest());
+        }
+
+        rowCount++;
+
+        // segmentRowIdOffset should encode sstableRowId into Integer
+        int segmentRowId = Math.toIntExact(sstableRowId - segmentRowIdOffset);
+
+        if (segmentRowId == PostingList.END_OF_STREAM)
+            throw new IllegalArgumentException("Illegal segment row id: END_OF_STREAM found");
+
+        maxSegmentRowId = Math.max(maxSegmentRowId, segmentRowId);
+
+        long bytesAllocated;
+        if (supportsAsyncAdd())
+        {
+            // only vector indexing is done async and there can only be one term
+            assert terms.size() == 1;
+            bytesAllocated = addInternalAsync(terms, segmentRowId);
+        }
+        else
+        {
+            bytesAllocated = addInternal(terms, segmentRowId);
+        }
+
+        totalBytesAllocated += bytesAllocated;
+        return bytesAllocated;
+    }
+
+    protected long addInternalAsync(List<ByteBuffer> terms, int segmentRowId)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public boolean supportsAsyncAdd() {
+        return false;
+    }
+
+    public Throwable getAsyncThrowable()
+    {
+        return asyncThrowable.get();
+    }
+
+    public void awaitAsyncAdditions()
+    {
+        // addTerm is only called by the compaction thread, serially, so we don't need to worry about new
+        // terms being added while we're waiting -- updatesInFlight can only decrease
+        busyWaitWhile(() -> updatesInFlight.get() > 0);
+    }
+
+    long totalBytesAllocated()
+    {
+        return totalBytesAllocated;
+    }
+
+    boolean hasReachedMinimumFlushSize()
+    {
+        return totalBytesAllocated >= minimumFlushBytes;
+    }
+
+    long getMinimumFlushBytes()
+    {
+        return minimumFlushBytes;
+    }
+
+    /**
+     * This method does three things:
+     *
+     * 1.) It decrements active builder count and updates the global minimum flush size to reflect that.
+     * 2.) It releases the builder's memory against its limiter.
+     * 3.) It defensively marks the builder inactive to make sure nothing bad happens if we try to close it twice.
+     *
+     * @param indexContext
+     *
+     * @return the number of bytes currently used by the memory limiter
+     */
+    long release(IndexContext indexContext)
+    {
+        if (active)
+        {
+            minimumFlushBytes = limiter.limitBytes() / ACTIVE_BUILDER_COUNT.decrementAndGet();
+            long used = limiter.decrement(totalBytesAllocated);
+            active = false;
+            return used;
+        }
+
+        logger.warn(indexContext.logMessage("Attempted to release storage attached index segment builder memory after builder marked inactive."));
+        return limiter.currentBytesUsed();
+    }
+
+    public abstract boolean isEmpty();
+
+    protected abstract long addInternal(List<ByteBuffer> terms, int segmentRowId);
+
+    protected abstract void flushInternal(SegmentMetadataBuilder metadataBuilder) throws IOException;
+
+    int getRowCount()
+    {
+        return rowCount;
+    }
+
+    /**
+     * @return true if next SSTable row ID exceeds max segment row ID
+     */
+    boolean exceedsSegmentLimit(long ssTableRowId)
+    {
+        if (getRowCount() == 0)
+            return false;
+
+        // To handle the case where there are many non-indexable rows. eg. rowId-1 and rowId-3B are indexable,
+        // the rest are non-indexable. We should flush them as 2 separate segments, because rowId-3B is going
+        // to cause error in on-disk index structure with 2B limitation.
+        return ssTableRowId - segmentRowIdOffset > lastValidSegmentRowID;
+    }
+
+    @VisibleForTesting
+    public static void updateLastValidSegmentRowId(long lastValidSegmentRowID)
+    {
+        testLastValidSegmentRowId = lastValidSegmentRowID;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadata.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadata.java
new file mode 100644
index 000000000000..b8f4cd769873
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadata.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+import com.google.common.collect.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.ModernResettableByteBuffersIndexOutput;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.v6.TermsDistribution;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Multiple {@link SegmentMetadata} are stored in {@link IndexComponentType#META} file, each corresponds to an on-disk
+ * index segment.
+ */
+public class SegmentMetadata implements Comparable<SegmentMetadata>
+{
+    private static final String NAME = "SegmentMetadata";
+
+    public final Version version;
+
+    /**
+     * Used to retrieve sstableRowId which equals to offset plus segmentRowId.
+     */
+    public final long segmentRowIdOffset;
+
+    /**
+     * Min and max sstable rowId in current segment.
+     *
+     * For index generated by compaction, minSSTableRowId is the same as segmentRowIdOffset.
+     * But for flush, segmentRowIdOffset is taken from previous segment's maxSSTableRowId.
+     */
+    public final long minSSTableRowId;
+    public final long maxSSTableRowId;
+
+    /**
+     * number of indexed rows (aka. pair of term and segmentRowId) in current segment
+     */
+    public final long numRows;
+
+    /**
+     * Ordered by their token position in current segment
+     */
+    public final PrimaryKey minKey;
+    public final PrimaryKey maxKey;
+
+    /**
+     * Minimum and maximum indexed column value ordered by its {@link org.apache.cassandra.db.marshal.AbstractType}.
+     */
+    public final ByteBuffer minTerm;
+    public final ByteBuffer maxTerm;
+
+
+    /**
+     * Statistical distribution of term values, useful for estimating selectivity of queries against this segment.
+     */
+    public final TermsDistribution termsDistribution;
+
+    /**
+     * Root, offset, length for each index structure in the segment.
+     *
+     * Note: postings block offsets are stored in terms dictionary, no need to worry about its root.
+     */
+    public final ComponentMetadataMap componentMetadatas;
+
+    SegmentMetadata(long segmentRowIdOffset,
+                    long numRows,
+                    long minSSTableRowId,
+                    long maxSSTableRowId,
+                    PrimaryKey minKey,
+                    PrimaryKey maxKey,
+                    ByteBuffer minTerm,
+                    ByteBuffer maxTerm,
+                    TermsDistribution termsDistribution,
+                    ComponentMetadataMap componentMetadatas)
+    {
+        // numRows can exceed Integer.MAX_VALUE because it is the count of unique term and segmentRowId pairs.
+        Objects.requireNonNull(minKey);
+        Objects.requireNonNull(maxKey);
+        Objects.requireNonNull(minTerm);
+        Objects.requireNonNull(maxTerm);
+
+        this.version = Version.latest();
+        this.segmentRowIdOffset = segmentRowIdOffset;
+        this.minSSTableRowId = minSSTableRowId;
+        this.maxSSTableRowId = maxSSTableRowId;
+        this.numRows = numRows;
+        this.minKey = minKey;
+        this.maxKey = maxKey;
+        this.minTerm = minTerm;
+        this.maxTerm = maxTerm;
+        this.termsDistribution = termsDistribution;
+        this.componentMetadatas = componentMetadatas;
+    }
+
+    private static final Logger logger = LoggerFactory.getLogger(SegmentMetadata.class);
+
+    @SuppressWarnings("resource")
+    private SegmentMetadata(IndexInput input, IndexContext context, Version version) throws IOException
+    {
+        PrimaryKey.Factory primaryKeyFactory = context.keyFactory();
+        AbstractType<?> termsType = context.getValidator();
+
+        this.version = version;
+        this.segmentRowIdOffset = input.readLong();
+        this.numRows = input.readLong();
+        this.minSSTableRowId = input.readLong();
+        this.maxSSTableRowId = input.readLong();
+        this.minKey = primaryKeyFactory.createPartitionKeyOnly(DatabaseDescriptor.getPartitioner().decorateKey(readBytes(input)));
+        this.maxKey = primaryKeyFactory.createPartitionKeyOnly(DatabaseDescriptor.getPartitioner().decorateKey(readBytes(input)));
+        this.minTerm = readBytes(input);
+        this.maxTerm = readBytes(input);
+        TermsDistribution td = null;
+        if (version.onOrAfter(Version.EB))
+        {
+            int len = input.readInt();
+            long fp = input.getFilePointer();
+            if (len > 0)
+            {
+                td = TermsDistribution.read(input, termsType);
+                input.seek(fp + len);
+            }
+        }
+        this.termsDistribution = td;
+        this.componentMetadatas = new SegmentMetadata.ComponentMetadataMap(input);
+    }
+
+    @SuppressWarnings("resource")
+    public static List<SegmentMetadata> load(MetadataSource source, IndexContext context) throws IOException
+    {
+
+        IndexInput input = source.get(NAME);
+
+        int segmentCount = input.readVInt();
+
+        List<SegmentMetadata> segmentMetadata = new ArrayList<>(segmentCount);
+
+        for (int i = 0; i < segmentCount; i++)
+        {
+            segmentMetadata.add(new SegmentMetadata(input, context, source.getVersion()));
+        }
+
+        return segmentMetadata;
+    }
+
+    /**
+     * Writes disk metadata for the given segment list.
+     */
+    @SuppressWarnings("resource")
+    public static void write(MetadataWriter writer, List<SegmentMetadata> segments) throws IOException
+    {
+        try (IndexOutput output = writer.builder(NAME))
+        {
+            output.writeVInt(segments.size());
+
+            for (SegmentMetadata metadata : segments)
+            {
+                output.writeLong(metadata.segmentRowIdOffset);
+                output.writeLong(metadata.numRows);
+                output.writeLong(metadata.minSSTableRowId);
+                output.writeLong(metadata.maxSSTableRowId);
+
+                Stream.of(metadata.minKey.partitionKey().getKey(),
+                          metadata.maxKey.partitionKey().getKey(),
+                          metadata.minTerm, metadata.maxTerm).forEach(bb -> writeBytes(bb, output));
+
+                if (writer.version().onOrAfter(Version.EB))
+                {
+                    if (metadata.termsDistribution != null)
+                    {
+                        var tmp = new ModernResettableByteBuffersIndexOutput(1024, "");
+                        metadata.termsDistribution.write(tmp);
+                        output.writeInt(tmp.intSize());
+                        tmp.copyTo(output);
+                    }
+                    else
+                    {
+                        // some indexes, e.g. vector may have no terms distribution
+                        output.writeInt(0);
+                    }
+                }
+
+                metadata.componentMetadatas.write(output);
+            }
+        }
+    }
+
+    @Override
+    public int compareTo(SegmentMetadata other)
+    {
+        return Long.compare(this.segmentRowIdOffset, other.segmentRowIdOffset);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "SegmentMetadata{" +
+               "segmentRowIdOffset=" + segmentRowIdOffset +
+               ", minSSTableRowId=" + minSSTableRowId +
+               ", maxSSTableRowId=" + maxSSTableRowId +
+               ", numRows=" + numRows +
+               ", componentMetadatas=" + componentMetadatas +
+               '}';
+    }
+
+    public long estimateNumRowsMatching(Expression predicate)
+    {
+        if (termsDistribution == null)
+            throw new IllegalStateException("Terms distribution not available for " + this);
+
+
+        switch (predicate.getOp())
+        {
+            case MATCH:
+            case EQ:
+            case CONTAINS_KEY:
+            case CONTAINS_VALUE:
+            {
+                var value = asByteComparable(predicate.lower.value.encoded, predicate.validator);
+                return termsDistribution.estimateNumRowsMatchingExact(value);
+            }
+            case NOT_EQ:
+            case NOT_CONTAINS_KEY:
+            case NOT_CONTAINS_VALUE:
+            {
+                if (TypeUtil.supportsRounding(predicate.validator))
+                    return numRows;
+                else
+                {
+                    var value = asByteComparable(predicate.lower.value.encoded, predicate.validator);
+                    return numRows - termsDistribution.estimateNumRowsMatchingExact(value);
+                }
+            }
+            case RANGE:
+            {
+                var lower = predicate.lower != null ? asByteComparable(predicate.lower.value.encoded, predicate.validator) : null;
+                var upper = predicate.upper != null ? asByteComparable(predicate.upper.value.encoded, predicate.validator) : null;
+                boolean lowerInclusive = predicate.lower != null && predicate.lower.inclusive;
+                boolean upperInclusive = predicate.upper != null && predicate.upper.inclusive;
+                return termsDistribution.estimateNumRowsInRange(lower, lowerInclusive, upper, upperInclusive);
+            }
+            default:
+                throw new IllegalArgumentException("Unsupported expression: " + predicate);
+        }
+    }
+
+    private ByteComparable asByteComparable(ByteBuffer value, AbstractType<?> type)
+    {
+        if (TypeUtil.isLiteral(type))
+            return version.onDiskFormat().encodeForTrie(value, type);
+
+        byte[] buffer = new byte[TypeUtil.fixedSizeOf(type)];
+        TypeUtil.toComparableBytes(value, type, buffer);
+        return ByteComparable.preencoded(termsDistribution.byteComparableVersion, buffer);
+    }
+
+    private static ByteBuffer readBytes(IndexInput input) throws IOException
+    {
+        int len = input.readVInt();
+        byte[] bytes = new byte[len];
+        input.readBytes(bytes, 0, len);
+        return ByteBuffer.wrap(bytes);
+    }
+
+    static void writeBytes(ByteBuffer buf, IndexOutput out)
+    {
+        try
+        {
+            byte[] bytes = ByteBufferUtil.getArray(buf);
+            out.writeVInt(bytes.length);
+            out.writeBytes(bytes, 0, bytes.length);
+        }
+        catch (IOException ioe)
+        {
+            throw new RuntimeException(ioe);
+        }
+    }
+
+    long getIndexRoot(IndexComponentType indexComponentType)
+    {
+        return componentMetadatas.get(indexComponentType).root;
+    }
+
+    public int toSegmentRowId(long sstableRowId)
+    {
+        int segmentRowId = Math.toIntExact(sstableRowId - segmentRowIdOffset);
+
+        if (segmentRowId == PostingList.END_OF_STREAM)
+            throw new IllegalArgumentException("Illegal segment row id: END_OF_STREAM found");
+
+        return segmentRowId;
+    }
+
+    public static class ComponentMetadataMap
+    {
+        private final Map<IndexComponentType, ComponentMetadata> metas = new HashMap<>();
+
+        ComponentMetadataMap(IndexInput input) throws IOException
+        {
+            int size = input.readInt();
+
+            for (int i = 0; i < size; i++)
+            {
+                metas.put(IndexComponentType.valueOf(input.readString()), new ComponentMetadata(input));
+            }
+        }
+
+        public ComponentMetadataMap()
+        {
+        }
+
+        public void put(IndexComponentType indexComponentType, long root, long offset, long length)
+        {
+            metas.put(indexComponentType, new ComponentMetadata(root, offset, length));
+        }
+
+        public void put(IndexComponentType indexComponentType, long root, long offset, long length, Map<String, String> additionalMap)
+        {
+            metas.put(indexComponentType, new ComponentMetadata(root, offset, length, additionalMap));
+        }
+
+        private void write(IndexOutput output) throws IOException
+        {
+            output.writeInt(metas.size());
+
+            for (Map.Entry<IndexComponentType, ComponentMetadata> entry : metas.entrySet())
+            {
+                output.writeString(entry.getKey().name());
+                entry.getValue().write(output);
+            }
+        }
+
+        public ComponentMetadata get(IndexComponentType indexComponentType)
+        {
+            if (!metas.containsKey(indexComponentType))
+                throw new IllegalArgumentException(indexComponentType + " ComponentMetadata not found");
+
+            return metas.get(indexComponentType);
+        }
+
+        public ComponentMetadata getOptional(IndexComponentType indexComponentType)
+        {
+            return metas.get(indexComponentType);
+        }
+
+        public Map<String, Map<String, String>> asMap()
+        {
+            Map<String, Map<String, String>> metaAttributes = new HashMap<>();
+
+            for (Map.Entry<IndexComponentType, ComponentMetadata> entry : metas.entrySet())
+            {
+                String name = entry.getKey().name();
+                ComponentMetadata metadata = entry.getValue();
+
+                Map<String, String> componentAttributes = metadata.asMap();
+
+                assert !metaAttributes.containsKey(name) : "Found duplicate index type: " + name;
+                metaAttributes.put(name, componentAttributes);
+            }
+
+            return metaAttributes;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "ComponentMetadataMap{" +
+                   "metas=" + metas +
+                   '}';
+        }
+
+        public double indexSize()
+        {
+            return metas.values().stream().mapToLong(meta -> meta.length).sum();
+        }
+    }
+
+    public static class ComponentMetadata
+    {
+        public static final String ROOT = "Root";
+        public static final String OFFSET = "Offset";
+        public static final String LENGTH = "Length";
+
+        public final long root;
+        public final long offset;
+        public final long length;
+        public final Map<String,String> attributes;
+
+        public ComponentMetadata(long root, long offset, long length)
+        {
+            this.root = root;
+            this.offset = offset;
+            this.length = length;
+            this.attributes = Collections.emptyMap();
+        }
+
+        ComponentMetadata(long root, long offset, long length, Map<String, String> attributes)
+        {
+            this.root = root;
+            this.offset = offset;
+            this.length = length;
+            this.attributes = attributes;
+        }
+
+        ComponentMetadata(IndexInput input) throws IOException
+        {
+            this.root = input.readLong();
+            this.offset = input.readLong();
+            this.length = input.readLong();
+            int size = input.readInt();
+
+            attributes = new HashMap<>(size);
+            for (int x=0; x < size; x++)
+            {
+                String key = input.readString();
+                String value = input.readString();
+
+                attributes.put(key, value);
+            }
+        }
+
+        public void write(IndexOutput output) throws IOException
+        {
+            output.writeLong(root);
+            output.writeLong(offset);
+            output.writeLong(length);
+
+            output.writeInt(attributes.size());
+            for (Map.Entry<String,String> entry : attributes.entrySet())
+            {
+                output.writeString(entry.getKey());
+                output.writeString(entry.getValue());
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("ComponentMetadata{root=%d, offset=%d, length=%d, attributes=%s}", root, offset, length, attributes.toString());
+        }
+
+        public Map<String, String> asMap()
+        {
+            return ImmutableMap.<String, String>builder().putAll(attributes).put(OFFSET, Long.toString(offset)).put(LENGTH, Long.toString(length)).put(ROOT, Long.toString(root)).build();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadataBuilder.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadataBuilder.java
new file mode 100644
index 000000000000..26326fc91b6a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadataBuilder.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.MutableOneDimPointValues;
+import org.apache.cassandra.index.sai.disk.v6.TermsDistribution;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * {@link SegmentMetadata} contains a lot of information, so it got its own Builder.
+ * The builder is not only responsible for setting the fields, but also intercepts
+ * the index building process and records the {@link TermsDistribution}.
+ */
+@NotThreadSafe
+public class SegmentMetadataBuilder
+{
+    private static final String HISTOGRAM_SIZE_OPTION = "statistics.histogram_size";
+    private static final String MFT_COUNT_OPTION = "statistics.most_frequent_terms_count";
+
+    private final long segmentRowIdOffset;
+
+    private final List<Closeable> interceptors = new ArrayList<>();
+    private final ByteComparable.Version byteComparableVersion;
+
+    private boolean built = false;
+
+    private PrimaryKey minKey;
+    private PrimaryKey maxKey;
+
+    private long minRowId = -1;
+    private long maxRowId = -1;
+
+    private ByteBuffer minTerm;
+    private ByteBuffer maxTerm;
+
+    private long numRows;
+
+    private final TermsDistribution.Builder termsDistributionBuilder;
+
+    SegmentMetadata.ComponentMetadataMap metadataMap;
+
+    public SegmentMetadataBuilder(long segmentRowIdOffset, IndexComponents.ForWrite components)
+    {
+        IndexContext context = Objects.requireNonNull(components.context());
+        this.segmentRowIdOffset = segmentRowIdOffset;
+        this.byteComparableVersion = components.byteComparableVersionFor(IndexComponentType.TERMS_DATA);
+
+        int histogramSize = context.getIntOption(HISTOGRAM_SIZE_OPTION, 128);
+        int mostFrequentTermsCount = context.getIntOption(MFT_COUNT_OPTION, 128);
+        this.termsDistributionBuilder = new TermsDistribution.Builder(context.getValidator(), byteComparableVersion, histogramSize, mostFrequentTermsCount);
+    }
+
+    public void setKeyRange(@Nonnull PrimaryKey minKey, @Nonnull PrimaryKey maxKey)
+    {
+        assert minKey.compareTo(maxKey) <= 0: "minKey (" + minKey + ") must not be greater than (" + maxKey + ')';
+        this.minKey = minKey;
+        this.maxKey = maxKey;
+    }
+
+    public void setRowIdRange(long minRowId, long maxRowId)
+    {
+        assert minRowId <= maxRowId: "minRowId (" + minRowId + ") must not be greater than (" + maxRowId + ')';
+        this.minRowId = minRowId;
+        this.maxRowId = maxRowId;
+    }
+
+    /**
+     * Sets the term range of the data indexed by this segment.
+     * We need this method because we cannot automatically record min and max term. We need exact
+     * values but the values from the index are trucated to 20 bytes for some types like e.g. BigDecimals.
+     * <p>
+     * This method requires raw serializations of the term types, not bytecomparable encodings.
+     */
+    public void setTermRange(@Nonnull ByteBuffer minTerm, @Nonnull ByteBuffer maxTerm)
+    {
+        this.minTerm = minTerm;
+        this.maxTerm = maxTerm;
+    }
+
+    public void setComponentsMetadata(SegmentMetadata.ComponentMetadataMap metadataMap)
+    {
+        this.metadataMap = metadataMap;
+    }
+
+    /**
+     * Should be called whenever a point is added to the index.
+     * Points must be added in the index term order.
+     * @param term the term value
+     * @param rowCount the number of rows with this term value in the segment
+     */
+    void add(ByteComparable term, int rowCount)
+    {
+        if (built)
+            throw new IllegalStateException("Segment metadata already built, no more additions allowed");
+
+        numRows += rowCount;
+        termsDistributionBuilder.add(term, rowCount);
+    }
+
+    public @Nonnull SegmentMetadata build()
+    {
+        if (minRowId == -1 || maxRowId == -1)
+            throw new IllegalStateException("Segment row id range not set");
+        if (minKey == null || maxKey == null)
+            throw new IllegalStateException("Segment key range not set");
+        if (minTerm == null || maxTerm == null)
+            throw new IllegalStateException("Term range not set");
+
+        FileUtils.closeQuietly(interceptors);
+        built = true; // must be flipped after closing the interceptors, because they may push some data to us when closing
+
+        return new SegmentMetadata(segmentRowIdOffset,
+                                   numRows,
+                                   minRowId,
+                                   maxRowId,
+                                   minKey,
+                                   maxKey,
+                                   minTerm,
+                                   maxTerm,
+                                   termsDistributionBuilder.build(),
+                                   metadataMap);
+    }
+
+    /**
+     * Wraps a {@link TermsIterator} in such a way that while it is iterated it adds items to this builder.
+     * Used at index building time to build the {@link TermsDistribution}.
+     * @return a wrapped iterator which also implements {@link TermsIterator}.
+     */
+    public TermsIterator intercept(TermsIterator iterator)
+    {
+        TermsIteratorInterceptor interceptor = new TermsIteratorInterceptor(iterator, this);
+        interceptors.add(interceptor);
+        return interceptor;
+    }
+
+    /**
+     * Wraps a {@link MutableOneDimPointValues} in such a way that while it is iterated it adds items to this builder.
+     * Used at index building time to build the {@link TermsDistribution}.
+     * @return a wrapped iterator which also implements {@link MutableOneDimPointValues}.
+     */
+    public MutableOneDimPointValues intercept(MutableOneDimPointValues values)
+    {
+        MutableOneDimPointValuesInterceptor interceptor = new MutableOneDimPointValuesInterceptor(values, this);
+        interceptors.add(interceptor);
+        return interceptor;
+    }
+
+
+    private static class TermsIteratorInterceptor implements TermsIterator
+    {
+        final TermsIterator iterator;
+        final SegmentMetadataBuilder builder;
+
+        PostingList postings;
+        IOException exception;
+
+        public TermsIteratorInterceptor(TermsIterator iterator, SegmentMetadataBuilder builder)
+        {
+            this.iterator = iterator;
+            this.builder = builder;
+        }
+
+        @Override
+        public PostingList postings() throws IOException
+        {
+            maybeThrow();
+            return postings;
+        }
+
+        @Override
+        public ByteBuffer getMinTerm()
+        {
+            return iterator.getMinTerm();
+        }
+
+        @Override
+        public ByteBuffer getMaxTerm()
+        {
+            return iterator.getMaxTerm();
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            iterator.close();
+            maybeThrow();
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            return iterator.hasNext();
+        }
+
+        @Override
+        public ByteComparable next()
+        {
+            ByteComparable term = iterator.next();
+            try
+            {
+                postings = iterator.postings();
+            }
+            catch (IOException e)
+            {
+                exception = e;
+            }
+            builder.add(term, postings.size());
+            return term;
+        }
+
+        private void maybeThrow() throws IOException
+        {
+            if (exception != null)
+            {
+                IOException e = exception;
+                exception = null;
+                throw e;
+            }
+        }
+    }
+
+    private static class MutableOneDimPointValuesInterceptor extends MutableOneDimPointValues implements Closeable
+    {
+        final MutableOneDimPointValues values;
+        final SegmentMetadataBuilder builder;
+
+        byte[] lastTerm;
+        int count = 0;
+
+        public MutableOneDimPointValuesInterceptor(MutableOneDimPointValues values, SegmentMetadataBuilder builder)
+        {
+            this.values = values;
+            this.builder = builder;
+        }
+
+        @Override
+        public int getDocCount()
+        {
+            return values.getDocCount();
+        }
+
+        @Override
+        public long size()
+        {
+            return values.size();
+        }
+
+        @Override
+        public void getValue(int i, BytesRef packedValue)
+        {
+            values.getValue(i, packedValue);
+        }
+
+        @Override
+        public byte getByteAt(int i, int k)
+        {
+            return values.getByteAt(i, k);
+        }
+
+        @Override
+        public int getDocID(int i)
+        {
+            return values.getDocID(i);
+        }
+
+        @Override
+        public void swap(int i, int j)
+        {
+            values.swap(i, j);
+        }
+
+        @Override
+        public byte[] getMinPackedValue()
+        {
+            return values.getMinPackedValue();
+        }
+
+        @Override
+        public byte[] getMaxPackedValue()
+        {
+            return values.getMaxPackedValue();
+        }
+
+        @Override
+        public int getNumDimensions()
+        {
+            return values.getNumDimensions();
+        }
+
+        @Override
+        public int getBytesPerDimension()
+        {
+            return values.getBytesPerDimension();
+        }
+
+        @Override
+        public int getNumIndexDimensions() throws IOException
+        {
+            return values.getNumIndexDimensions();
+        }
+
+        @Override
+        public PointTree getPointTree() throws IOException
+        {
+            return values.getPointTree();
+        }
+
+        @Override
+        public void intersect(IntersectVisitor visitor) throws IOException
+        {
+            values.intersect((docId, term) -> {
+                if (!Arrays.equals(term, lastTerm))
+                {
+                    if (lastTerm != null)
+                        builder.add(ByteComparable.preencoded(builder.byteComparableVersion, lastTerm), count);
+
+
+                    count = 0;
+                    lastTerm = Arrays.copyOf(term, term.length);
+                }
+                count++;
+                visitor.visit(docId, term);
+            });
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            if (lastTerm != null)
+            {
+                builder.add(ByteComparable.preencoded(builder.byteComparableVersion, lastTerm), count);
+            }
+        }
+
+    }
+
+}
+
+
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SkinnyPrimaryKeyMap.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SkinnyPrimaryKeyMap.java
deleted file mode 100644
index a764eb6c8eb0..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/SkinnyPrimaryKeyMap.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.io.IOException;
-import java.util.Arrays;
-import javax.annotation.concurrent.NotThreadSafe;
-import javax.annotation.concurrent.ThreadSafe;
-
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.BlockPackedReader;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.MonotonicBlockPackedReader;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
-import org.apache.cassandra.index.sai.disk.v1.keystore.KeyLookupMeta;
-import org.apache.cassandra.index.sai.disk.v1.keystore.KeyLookup;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.Throwables;
-
-/**
- * A {@link PrimaryKeyMap} for skinny tables (those with no clustering columns).
- * <p>
- * This uses the following on-disk structures:
- * <ul>
- *     <li>A block-packed structure for rowId to token value lookups using {@link BlockPackedReader}.
- *     Uses the {@link IndexComponent#ROW_TO_TOKEN} component</li>
- *     <li>A monotonic block packed structure for rowId to partitionId lookups using {@link MonotonicBlockPackedReader}.
- *     Uses the {@link IndexComponent#ROW_TO_PARTITION} component</li>
- *     <li>A key store for rowId to {@link PrimaryKey} and {@link PrimaryKey} to rowId lookups using
- *     {@link KeyLookup}. Uses the {@link IndexComponent#PARTITION_KEY_BLOCKS} and
- *     {@link IndexComponent#PARTITION_KEY_BLOCK_OFFSETS} components</li>
- * </ul>
- *
- * While the {@link Factory} is threadsafe, individual instances of the {@link SkinnyPrimaryKeyMap}
- * are not.
- */
-@NotThreadSafe
-public class SkinnyPrimaryKeyMap implements PrimaryKeyMap
-{
-    @ThreadSafe
-    public static class Factory implements PrimaryKeyMap.Factory
-    {
-        protected final MetadataSource metadataSource;
-        protected final LongArray.Factory rowToTokenReaderFactory;
-        protected final LongArray.Factory rowToPartitionReaderFactory;
-        protected final KeyLookup partitionKeyReader;
-        protected final PrimaryKey.Factory primaryKeyFactory;
-
-        private final FileHandle rowToTokenFile;
-        private final FileHandle rowToPartitionFile;
-        private final FileHandle partitionKeyBlockOffsetsFile;
-        private final FileHandle partitionKeyBlocksFile;
-
-        public Factory(IndexDescriptor indexDescriptor)
-        {
-            this.rowToTokenFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.ROW_TO_TOKEN, this::close);
-            this.rowToPartitionFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.ROW_TO_PARTITION, this::close);
-            this.partitionKeyBlockOffsetsFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PARTITION_KEY_BLOCK_OFFSETS, this::close);
-            this.partitionKeyBlocksFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PARTITION_KEY_BLOCKS, this::close);
-            try
-            {
-                this.metadataSource = MetadataSource.loadGroupMetadata(indexDescriptor);
-                NumericValuesMeta tokensMeta = new NumericValuesMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.ROW_TO_TOKEN)));
-                this.rowToTokenReaderFactory = new BlockPackedReader(rowToTokenFile, tokensMeta);
-                NumericValuesMeta partitionsMeta = new NumericValuesMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.ROW_TO_PARTITION)));
-                this.rowToPartitionReaderFactory = new MonotonicBlockPackedReader(rowToPartitionFile, partitionsMeta);
-                NumericValuesMeta partitionKeyBlockOffsetsMeta = new NumericValuesMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.PARTITION_KEY_BLOCK_OFFSETS)));
-                KeyLookupMeta partitionKeysMeta = new KeyLookupMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.PARTITION_KEY_BLOCKS)));
-                this.partitionKeyReader = new KeyLookup(partitionKeyBlocksFile, partitionKeyBlockOffsetsFile, partitionKeysMeta, partitionKeyBlockOffsetsMeta);
-                this.primaryKeyFactory = indexDescriptor.primaryKeyFactory;
-            }
-            catch (Throwable t)
-            {
-                throw Throwables.unchecked(t);
-            }
-        }
-
-        @Override
-        @SuppressWarnings({"resource", "RedundantSuppression"}) // rowIdToToken, rowIdToPartitionId and cursor are closed by the SkinnyPrimaryKeyMap#close method
-        public PrimaryKeyMap newPerSSTablePrimaryKeyMap() throws IOException
-        {
-            LongArray rowIdToToken = new LongArray.DeferredLongArray(rowToTokenReaderFactory::open);
-            LongArray rowIdToPartitionId = new LongArray.DeferredLongArray(rowToPartitionReaderFactory::open);
-            return new SkinnyPrimaryKeyMap(rowIdToToken,
-                                           rowIdToPartitionId,
-                                           partitionKeyReader.openCursor(),
-                                           primaryKeyFactory);
-        }
-
-        @Override
-        public void close()
-        {
-            FileUtils.closeQuietly(Arrays.asList(rowToTokenFile, rowToPartitionFile, partitionKeyBlocksFile, partitionKeyBlockOffsetsFile));
-        }
-    }
-
-    protected final LongArray rowIdToTokenArray;
-    protected final LongArray rowIdToPartitionIdArray;
-    protected final KeyLookup.Cursor partitionKeyCursor;
-    protected final PrimaryKey.Factory primaryKeyFactory;
-
-    protected SkinnyPrimaryKeyMap(LongArray rowIdToTokenArray,
-                                  LongArray rowIdToPartitionIdArray,
-                                  KeyLookup.Cursor partitionKeyCursor,
-                                  PrimaryKey.Factory primaryKeyFactory)
-    {
-        this.rowIdToTokenArray = rowIdToTokenArray;
-        this.rowIdToPartitionIdArray = rowIdToPartitionIdArray;
-        this.partitionKeyCursor = partitionKeyCursor;
-        this.primaryKeyFactory = primaryKeyFactory;
-    }
-
-    @Override
-    public PrimaryKey primaryKeyFromRowId(long sstableRowId)
-    {
-        return primaryKeyFactory.create(readPartitionKey(sstableRowId));
-    }
-
-    @Override
-    public long rowIdFromPrimaryKey(PrimaryKey primaryKey)
-    {
-        long rowId = rowIdToTokenArray.indexOf(primaryKey.token().getLongValue());
-        // If the key is token only, the token is out of range, we are at the end of our keys, or we have skipped a token
-        // we can return straight away.
-        if (primaryKey.kind() == PrimaryKey.Kind.TOKEN ||
-            rowId < 0 ||
-            rowId + 1 == rowIdToTokenArray.length() || rowIdToTokenArray.get(rowId) != primaryKey.token().getLongValue())
-            return rowId;
-        // Otherwise we need to check for token collision.
-        return tokenCollisionDetection(primaryKey, rowId);
-    }
-
-    @Override
-    public long ceiling(Token token)
-    {
-        return rowIdToTokenArray.indexOf(token.getLongValue());
-    }
-
-    @Override
-    public long floor(Token token)
-    {
-        if (token.isMinimum())
-            return Long.MIN_VALUE;
-
-        return rowIdToTokenArray.indexOf(token.getLongValue());
-    }
-
-    @Override
-    public void close()
-    {
-        FileUtils.closeQuietly(Arrays.asList(partitionKeyCursor, rowIdToTokenArray, rowIdToPartitionIdArray));
-    }
-
-    // Look for token collision by if the ajacent token in the token array matches the
-    // current token. If we find a collision we need to compare the partition key instead.
-    protected long tokenCollisionDetection(PrimaryKey primaryKey, long rowId)
-    {
-        // Look for collisions while we haven't reached the end of the tokens and the tokens don't collide
-        while (rowId + 1 < rowIdToTokenArray.length() && primaryKey.token().getLongValue() == rowIdToTokenArray.get(rowId + 1))
-        {
-            // If we had a collision then see if the partition key for this row is >= to the lookup partition key
-            if (readPartitionKey(rowId).compareTo(primaryKey.partitionKey()) >= 0)
-                return rowId;
-
-            rowId++;
-        }
-        // Note: We would normally expect to get here without going into the while loop
-        return rowId;
-    }
-
-    protected DecoratedKey readPartitionKey(long sstableRowId)
-    {
-        return primaryKeyFactory.partitionKeyFromComparableBytes(partitionKeyCursor.seekToPointId(rowIdToPartitionIdArray.get(sstableRowId)));
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/TermsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/TermsReader.java
new file mode 100644
index 000000000000..4bbbc49cc815
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/TermsReader.java
@@ -0,0 +1,512 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
+import org.apache.cassandra.index.sai.disk.v1.postings.ScanningPostingsReader;
+import org.apache.cassandra.index.sai.disk.v1.trie.ReverseTrieTermsDictionaryReader;
+import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.AbortedOperationException;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+import static org.apache.cassandra.index.sai.utils.SAICodecUtils.validate;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+/**
+ * Synchronous reader of terms dictionary and postings lists to produce a {@link PostingList} with matching row ids.
+ *
+ * {@link #exactMatch(ByteComparable, QueryEventListener.TrieIndexEventListener, QueryContext)} does:
+ * <ul>
+ * <li>{@link TermQuery#lookupTermDictionary(ByteComparable)}: does term dictionary lookup to find the posting list file
+ * position</li>
+ * <li>{@link TermQuery#getPostingReader(long)}: reads posting list block summary and initializes posting read which
+ * reads the first block of the posting list into memory</li>
+ * </ul>
+ */
+public class TermsReader implements Closeable
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private final IndexContext indexContext;
+    private final FileHandle termDictionaryFile;
+    private final FileHandle postingsFile;
+    private final long termDictionaryRoot;
+    private final Version version;
+    private final ByteComparable.Version termDictionaryFileEncodingVersion;
+
+    public TermsReader(IndexContext indexContext,
+                       FileHandle termsData,
+                       ByteComparable.Version termsDataEncodingVersion,
+                       FileHandle postingLists,
+                       long root,
+                       long termsFooterPointer,
+                       Version version) throws IOException
+    {
+        this.indexContext = indexContext;
+        this.version = version;
+        termDictionaryFile = termsData;
+        postingsFile = postingLists;
+        termDictionaryRoot = root;
+        this.termDictionaryFileEncodingVersion = termsDataEncodingVersion;
+
+        try (final IndexInput indexInput = IndexFileUtils.instance().openInput(termDictionaryFile))
+        {
+            // if the pointer is -1 then this is a previous version of the index
+            // use the old way to validate the footer
+            // the footer pointer is used due to encrypted indexes padding extra bytes
+            if (termsFooterPointer == -1)
+            {
+                validate(indexInput);
+            }
+            else
+            {
+                validate(indexInput, termsFooterPointer);
+            }
+        }
+
+        try (final IndexInput indexInput = IndexFileUtils.instance().openInput(postingsFile))
+        {
+            validate(indexInput);
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        try
+        {
+            termDictionaryFile.close();
+        }
+        finally
+        {
+            postingsFile.close();
+        }
+    }
+
+    public TermsIterator allTerms()
+    {
+        return allTerms(true);
+    }
+
+    public TermsIterator allTerms(boolean ascending)
+    {
+        // blocking, since we use it only for segment merging for now
+        return ascending ? new TermsScanner(version, this.indexContext.getValidator())
+                         : new ReverseTermsScanner();
+    }
+
+    public PostingList exactMatch(ByteComparable term, QueryEventListener.TrieIndexEventListener perQueryEventListener, QueryContext context)
+    {
+        perQueryEventListener.onSegmentHit();
+        return new TermQuery(term, perQueryEventListener, context).execute();
+    }
+
+    /**
+     * Range query that uses the lower and upper bounds to retrieve the search results within the range. When
+     * the expression is not null, it post-filters results using the expression.
+     */
+    public PostingList rangeMatch(Expression exp, ByteComparable lower, ByteComparable upper, QueryEventListener.TrieIndexEventListener perQueryEventListener, QueryContext context)
+    {
+        perQueryEventListener.onSegmentHit();
+        return new RangeQuery(exp, lower, upper, perQueryEventListener, context).execute();
+    }
+
+    @VisibleForTesting
+    public class TermQuery
+    {
+        private final IndexInput postingsInput;
+        private final IndexInput postingsSummaryInput;
+        private final QueryEventListener.TrieIndexEventListener listener;
+        private final long lookupStartTime;
+        private final QueryContext context;
+
+        private ByteComparable term;
+
+        TermQuery(ByteComparable term, QueryEventListener.TrieIndexEventListener listener, QueryContext context)
+        {
+            this.listener = listener;
+            postingsInput = IndexFileUtils.instance().openInput(postingsFile);
+            postingsSummaryInput = IndexFileUtils.instance().openInput(postingsFile);
+            this.term = term;
+            lookupStartTime = nanoTime();
+            this.context = context;
+        }
+
+        public PostingList execute()
+        {
+            try
+            {
+                long postingOffset = lookupTermDictionary(term);
+                if (postingOffset == PostingList.OFFSET_NOT_FOUND)
+                {
+                    FileUtils.closeQuietly(postingsInput);
+                    FileUtils.closeQuietly(postingsSummaryInput);
+                    return null;
+                }
+
+                context.checkpoint();
+
+                // when posting is found, resources will be closed when posting reader is closed.
+                return getPostingReader(postingOffset);
+            }
+            catch (Throwable e)
+            {
+                //TODO Is there an equivalent of AOE in OS?
+                if (!(e instanceof AbortedOperationException))
+                    logger.error(indexContext.logMessage("Failed to execute term query"), e);
+
+                closeOnException();
+                throw Throwables.cleaned(e);
+            }
+        }
+
+        private void closeOnException()
+        {
+            FileUtils.closeQuietly(postingsInput);
+            FileUtils.closeQuietly(postingsSummaryInput);
+        }
+
+        public long lookupTermDictionary(ByteComparable term)
+        {
+            try (TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(termDictionaryFile.instantiateRebufferer(null), termDictionaryRoot, termDictionaryFileEncodingVersion))
+            {
+                final long offset = reader.exactMatch(term);
+
+                listener.onTraversalComplete(nanoTime() - lookupStartTime, TimeUnit.NANOSECONDS);
+
+                if (offset == TrieTermsDictionaryReader.NOT_FOUND)
+                    return PostingList.OFFSET_NOT_FOUND;
+
+                return offset;
+            }
+        }
+
+        public PostingsReader getPostingReader(long offset) throws IOException
+        {
+            PostingsReader.BlocksSummary header = new PostingsReader.BlocksSummary(postingsSummaryInput, offset);
+
+            return new PostingsReader(postingsInput, header, readFrequencies(), listener.postingListEventListener());
+        }
+    }
+
+    public class RangeQuery
+    {
+        private final QueryEventListener.TrieIndexEventListener listener;
+        private final long lookupStartTime;
+        private final QueryContext context;
+
+        private final Expression exp;
+        private final ByteComparable lower;
+        private final ByteComparable upper;
+
+        // When the exp is not null, we need to post filter the results
+        RangeQuery(Expression exp, ByteComparable lower, ByteComparable upper, QueryEventListener.TrieIndexEventListener listener, QueryContext context)
+        {
+            this.listener = listener;
+            this.exp = exp;
+            lookupStartTime = Clock.Global.nanoTime();
+            this.context = context;
+            this.lower = lower;
+            this.upper = upper;
+        }
+
+        public PostingList execute()
+        {
+            // Note: we always pass true for include start because we use the ByteComparable terminator above
+            // to selectively determine when we have a match on the first/last term. This is probably part of the API
+            // that could change, but it's been there for a bit, so we'll leave it for now.
+            try (TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(termDictionaryFile.instantiateRebufferer(null),
+                                                                                  termDictionaryRoot,
+                                                                                  lower,
+                                                                                  upper,
+                                                                                  true,
+                                                                                  exp != null,
+                                                                                  termDictionaryFileEncodingVersion))
+            {
+                if (!reader.hasNext())
+                    return PostingList.EMPTY;
+
+                context.checkpoint();
+                PostingList postings = exp == null
+                                       ? readAndMergePostings(reader)
+                                       : readFilterAndMergePosting(reader);
+
+                listener.onTraversalComplete(Clock.Global.nanoTime() - lookupStartTime, TimeUnit.NANOSECONDS);
+
+                return postings;
+            }
+            catch (Throwable e)
+            {
+                if (!(e instanceof AbortedOperationException))
+                    logger.error(indexContext.logMessage("Failed to execute term query"), e);
+
+                throw Throwables.cleaned(e);
+            }
+        }
+
+        /**
+         * Reads the posting lists for the matching terms and merges them into a single posting list.
+         * It assumes that the posting list for each term is sorted.
+         *
+         * @return the posting lists for the terms matching the query.
+         */
+        private PostingList readAndMergePostings(TrieTermsDictionaryReader reader) throws IOException
+        {
+            assert reader.hasNext();
+            ArrayList<PostingList> postingLists = new ArrayList<>();
+
+            // index inputs will be closed with the onClose method of the returned merged posting list
+            IndexInput postingsInput = IndexFileUtils.instance().openInput(postingsFile);
+            IndexInput postingsSummaryInput = IndexFileUtils.instance().openInput(postingsFile);
+
+            do
+            {
+                long postingsOffset = reader.nextAsLong();
+                var currentReader = currentReader(postingsInput, postingsSummaryInput, postingsOffset);
+
+                if (!currentReader.isEmpty())
+                    postingLists.add(currentReader);
+                else
+                    FileUtils.close(currentReader);
+            } while (reader.hasNext());
+
+            return MergePostingList.merge(postingLists)
+                                   .onClose(() -> FileUtils.close(postingsInput, postingsSummaryInput));
+        }
+
+        /**
+         * Reads the posting lists for the matching terms, apply the expression to filter results, and merge them into
+         * a single posting list. It assumes that the posting list for each term is sorted.
+         *
+         * @return the posting lists for the terms matching the query.
+         */
+        private PostingList readFilterAndMergePosting(TrieTermsDictionaryReader reader) throws IOException
+        {
+            assert reader.hasNext();
+            ArrayList<PostingList> postingLists = new ArrayList<>();
+
+            // index inputs will be closed with the onClose method of the returned merged posting list
+            IndexInput postingsInput = IndexFileUtils.instance().openInput(postingsFile);
+            IndexInput postingsSummaryInput = IndexFileUtils.instance().openInput(postingsFile);
+
+            do
+            {
+                Pair<ByteComparable, Long> nextTriePair = reader.next();
+                ByteSource mapEntry = nextTriePair.left.asComparableBytes(termDictionaryFileEncodingVersion);
+                long postingsOffset = nextTriePair.right;
+                byte[] nextBytes = ByteSourceInverse.readBytes(mapEntry);
+
+                if (exp.isSatisfiedBy(ByteBuffer.wrap(nextBytes)))
+                {
+                    var currentReader = currentReader(postingsInput, postingsSummaryInput, postingsOffset);
+
+                    if (!currentReader.isEmpty())
+                        postingLists.add(currentReader);
+                    else
+                        FileUtils.close(currentReader);
+                }
+            } while (reader.hasNext());
+
+            return MergePostingList.merge(postingLists)
+                                   .onClose(() -> FileUtils.close(postingsInput, postingsSummaryInput));
+        }
+
+        private PostingsReader currentReader(IndexInput postingsInput,
+                                             IndexInput postingsSummaryInput,
+                                             long postingsOffset) throws IOException
+        {
+            var blocksSummary = new PostingsReader.BlocksSummary(postingsSummaryInput,
+                                                                 postingsOffset,
+                                                                 PostingsReader.InputCloser.NOOP);
+            return new PostingsReader(postingsInput,
+                                      blocksSummary,
+                                      readFrequencies(),
+                                      listener.postingListEventListener(),
+                                      PostingsReader.InputCloser.NOOP);
+        }
+    }
+
+    private boolean readFrequencies()
+    {
+        return indexContext.isAnalyzed() && version.onOrAfter(Version.EC);
+    }
+
+    private class TermsScanner implements TermsIterator
+    {
+        private final TrieTermsDictionaryReader termsDictionaryReader;
+        private final ByteBuffer minTerm, maxTerm;
+        private Pair<ByteComparable, Long> entry;
+        private final IndexInput postingsInput;
+        private final IndexInput postingsSummaryInput;
+
+        private TermsScanner(Version version, AbstractType<?> type)
+        {
+            this.termsDictionaryReader = new TrieTermsDictionaryReader(termDictionaryFile.instantiateRebufferer(null), termDictionaryRoot, termDictionaryFileEncodingVersion);
+            this.postingsInput = IndexFileUtils.instance().openInput(postingsFile);
+            this.postingsSummaryInput = IndexFileUtils.instance().openInput(postingsFile);
+            // We decode based on the logic used to encode the min and max terms in the trie.
+            if (version.onOrAfter(Version.DB) && TypeUtil.isComposite(type))
+            {
+                this.minTerm = indexContext.getValidator().fromComparableBytes(ByteSource.peekable(termsDictionaryReader.getMinTerm().asComparableBytes(termDictionaryFileEncodingVersion)), termDictionaryFileEncodingVersion);
+                this.maxTerm = indexContext.getValidator().fromComparableBytes(ByteSource.peekable(termsDictionaryReader.getMaxTerm().asComparableBytes(termDictionaryFileEncodingVersion)), termDictionaryFileEncodingVersion);
+            }
+            else
+            {
+                this.minTerm = ByteBuffer.wrap(ByteSourceInverse.readBytes(termsDictionaryReader.getMinTerm().asComparableBytes(termDictionaryFileEncodingVersion)));
+                this.maxTerm = ByteBuffer.wrap(ByteSourceInverse.readBytes(termsDictionaryReader.getMaxTerm().asComparableBytes(termDictionaryFileEncodingVersion)));
+            }
+        }
+
+        @Override
+        @SuppressWarnings("resource")
+        public PostingList postings() throws IOException
+        {
+            assert entry != null;
+            var blockSummary = new PostingsReader.BlocksSummary(postingsSummaryInput, entry.right, PostingsReader.InputCloser.NOOP);
+            return new ScanningPostingsReader(postingsInput, blockSummary, readFrequencies());
+        }
+
+        @Override
+        public void close()
+        {
+            termsDictionaryReader.close();
+            FileUtils.closeQuietly(postingsInput);
+            FileUtils.closeQuietly(postingsSummaryInput);
+        }
+
+        @Override
+        public ByteBuffer getMinTerm()
+        {
+            return minTerm;
+        }
+
+        @Override
+        public ByteBuffer getMaxTerm()
+        {
+            return maxTerm;
+        }
+
+        @Override
+        public ByteComparable next()
+        {
+            if (termsDictionaryReader.hasNext())
+            {
+                entry = termsDictionaryReader.next();
+                return entry.left;
+            }
+            return null;
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            return termsDictionaryReader.hasNext();
+        }
+    }
+
+    private class ReverseTermsScanner implements TermsIterator
+    {
+        private final ReverseTrieTermsDictionaryReader iterator;
+        private Pair<ByteComparable, Long> entry;
+        private final IndexInput postingsInput;
+        private final IndexInput postingsSummaryInput;
+
+        private ReverseTermsScanner()
+        {
+            this.iterator = new ReverseTrieTermsDictionaryReader(termDictionaryFile.instantiateRebufferer(null), termDictionaryRoot);
+            this.postingsInput = IndexFileUtils.instance().openInput(postingsFile);
+            this.postingsSummaryInput = IndexFileUtils.instance().openInput(postingsFile);
+        }
+
+        @Override
+        @SuppressWarnings("resource")
+        public PostingList postings() throws IOException
+        {
+            assert entry != null;
+            var blockSummary = new PostingsReader.BlocksSummary(postingsSummaryInput, entry.right, PostingsReader.InputCloser.NOOP);
+            return new ScanningPostingsReader(postingsInput, blockSummary, readFrequencies());
+        }
+
+        @Override
+        public void close()
+        {
+            iterator.close();
+            FileUtils.closeQuietly(postingsInput);
+            FileUtils.closeQuietly(postingsSummaryInput);
+        }
+
+        @Override
+        public ByteBuffer getMinTerm()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ByteBuffer getMaxTerm()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ByteComparable next()
+        {
+            if (iterator.hasNext())
+            {
+                entry = iterator.next();
+                return entry.left;
+            }
+            return null;
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            return iterator.hasNext();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java
index 8d8266ac349e..e2ec3c8a05cb 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java
@@ -20,280 +20,311 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.EnumSet;
 import java.util.Set;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.codahale.metrics.Gauge;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SSTableContext;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
-import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
+import org.apache.cassandra.index.sai.disk.EmptyIndex;
+import org.apache.cassandra.index.sai.disk.PerIndexWriter;
+import org.apache.cassandra.index.sai.disk.PerSSTableWriter;
 import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.RowMapping;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.SearchableIndex;
 import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
 import org.apache.cassandra.index.sai.disk.format.OnDiskFormat;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.memory.RowMapping;
 import org.apache.cassandra.index.sai.metrics.AbstractMetrics;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
 import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.metrics.DefaultNameFactory;
 import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 import org.apache.lucene.store.IndexInput;
 
 import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
 
+/**
+ * The original SAI OnDiskFormat, found in DSE.  Because it has a simple token -> offsets map, queries
+ * against "wide partitions" are slow in proportion to the partition size, since we have to read
+ * the whole partition and post-filter the rows
+ */
 public class V1OnDiskFormat implements OnDiskFormat
 {
-    private static final Logger logger = LoggerFactory.getLogger(V1OnDiskFormat.class);
-
-    @VisibleForTesting
-    public static final Set<IndexComponent> SKINNY_PER_SSTABLE_COMPONENTS = EnumSet.of(IndexComponent.GROUP_COMPLETION_MARKER,
-                                                                                       IndexComponent.GROUP_META,
-                                                                                       IndexComponent.ROW_TO_TOKEN,
-                                                                                       IndexComponent.ROW_TO_PARTITION,
-                                                                                       IndexComponent.PARTITION_KEY_BLOCKS,
-                                                                                       IndexComponent.PARTITION_KEY_BLOCK_OFFSETS);
-
-    @VisibleForTesting
-    public static final Set<IndexComponent> WIDE_PER_SSTABLE_COMPONENTS = EnumSet.of(IndexComponent.GROUP_COMPLETION_MARKER,
-                                                                                     IndexComponent.GROUP_META,
-                                                                                     IndexComponent.ROW_TO_TOKEN,
-                                                                                     IndexComponent.ROW_TO_PARTITION,
-                                                                                     IndexComponent.PARTITION_TO_SIZE,
-                                                                                     IndexComponent.PARTITION_KEY_BLOCKS,
-                                                                                     IndexComponent.PARTITION_KEY_BLOCK_OFFSETS,
-                                                                                     IndexComponent.CLUSTERING_KEY_BLOCKS,
-                                                                                     IndexComponent.CLUSTERING_KEY_BLOCK_OFFSETS);
-
-    @VisibleForTesting
-    public static final Set<IndexComponent> LITERAL_COMPONENTS = EnumSet.of(IndexComponent.COLUMN_COMPLETION_MARKER,
-                                                                            IndexComponent.META,
-                                                                            IndexComponent.TERMS_DATA,
-                                                                            IndexComponent.POSTING_LISTS);
-    @VisibleForTesting
-    public static final Set<IndexComponent> NUMERIC_COMPONENTS = EnumSet.of(IndexComponent.COLUMN_COMPLETION_MARKER,
-                                                                            IndexComponent.META,
-                                                                            IndexComponent.BALANCED_TREE,
-                                                                            IndexComponent.POSTING_LISTS);
-
-    @VisibleForTesting
-    public static final Set<IndexComponent> VECTOR_COMPONENTS = EnumSet.of(IndexComponent.COLUMN_COMPLETION_MARKER,
-                                                                           IndexComponent.META,
-                                                                           IndexComponent.COMPRESSED_VECTORS,
-                                                                           IndexComponent.TERMS_DATA,
-                                                                           IndexComponent.POSTING_LISTS);
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private static final Set<IndexComponentType> PER_SSTABLE_COMPONENTS = EnumSet.of(IndexComponentType.GROUP_COMPLETION_MARKER,
+                                                                                     IndexComponentType.GROUP_META,
+                                                                                     IndexComponentType.TOKEN_VALUES,
+                                                                                     IndexComponentType.OFFSETS_VALUES);
+
+    private static final Set<IndexComponentType> LITERAL_COMPONENTS = EnumSet.of(IndexComponentType.COLUMN_COMPLETION_MARKER,
+                                                                                 IndexComponentType.META,
+                                                                                 IndexComponentType.TERMS_DATA,
+                                                                                 IndexComponentType.POSTING_LISTS);
+
+    public static final Set<IndexComponentType> NUMERIC_COMPONENTS = EnumSet.of(IndexComponentType.COLUMN_COMPLETION_MARKER,
+                                                                                IndexComponentType.META,
+                                                                                IndexComponentType.KD_TREE,
+                                                                                IndexComponentType.KD_TREE_POSTING_LISTS);
 
     /**
      * Global limit on heap consumed by all index segment building that occurs outside the context of Memtable flush.
-     * <p>
-     * Note that to avoid flushing small index segments, a segment is only flushed when
+     *
+     * Note that to avoid flushing extremly small index segments, a segment is only flushed when
      * both the global size of all building segments has breached the limit and the size of the
      * segment in question reaches (segment_write_buffer_space_mb / # currently building column indexes).
-     * <p>
+     *
      * ex. If there is only one column index building, it can buffer up to segment_write_buffer_space_mb.
-     * <p>
+     *
      * ex. If there is one column index building per table across 8 compactors, each index will be
      *     eligible to flush once it reaches (segment_write_buffer_space_mb / 8) MBs.
      */
-    public static final long SEGMENT_BUILD_MEMORY_LIMIT = DatabaseDescriptor.getSAISegmentWriteBufferSpace().toBytes();
+    public static final long SEGMENT_BUILD_MEMORY_LIMIT = 1024L * 1024L * DatabaseDescriptor.getSAISegmentWriteBufferSpace();
 
-    public static final NamedMemoryLimiter SEGMENT_BUILD_MEMORY_LIMITER = new NamedMemoryLimiter(SEGMENT_BUILD_MEMORY_LIMIT,
-                                                                                                 "Storage Attached Index Segment Builder");
+    public static final NamedMemoryLimiter SEGMENT_BUILD_MEMORY_LIMITER =
+    new NamedMemoryLimiter(SEGMENT_BUILD_MEMORY_LIMIT, "SSTable-attached Index Segment Builder");
 
     static
     {
+        logger.debug("Segment build memory limit set to {} bytes", prettyPrintMemory(SEGMENT_BUILD_MEMORY_LIMIT));
+
         CassandraMetricsRegistry.MetricName bufferSpaceUsed = DefaultNameFactory.createMetricName(AbstractMetrics.TYPE, "SegmentBufferSpaceUsedBytes", null);
         CassandraMetricsRegistry.Metrics.register(bufferSpaceUsed, (Gauge<Long>) SEGMENT_BUILD_MEMORY_LIMITER::currentBytesUsed);
 
         CassandraMetricsRegistry.MetricName bufferSpaceLimit = DefaultNameFactory.createMetricName(AbstractMetrics.TYPE, "SegmentBufferSpaceLimitBytes", null);
         CassandraMetricsRegistry.Metrics.register(bufferSpaceLimit, (Gauge<Long>) () -> SEGMENT_BUILD_MEMORY_LIMIT);
 
+        // Note: The active builder count starts at 1 to avoid dividing by zero.
         CassandraMetricsRegistry.MetricName buildsInProgress = DefaultNameFactory.createMetricName(AbstractMetrics.TYPE, "ColumnIndexBuildsInProgress", null);
-        CassandraMetricsRegistry.Metrics.register(buildsInProgress, (Gauge<Integer>) SegmentBuilder::getActiveBuilderCount);
+        CassandraMetricsRegistry.Metrics.register(buildsInProgress, (Gauge<Long>) () -> SegmentBuilder.ACTIVE_BUILDER_COUNT.get() - 1);
     }
 
     public static final V1OnDiskFormat instance = new V1OnDiskFormat();
 
+    private static final IndexFeatureSet v1IndexFeatureSet = new IndexFeatureSet()
+    {
+        @Override
+        public boolean isRowAware()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean hasVectorIndexChecksum()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean hasTermsHistogram()
+        {
+            return false;
+        }
+    };
+
     protected V1OnDiskFormat()
     {}
 
     @Override
-    public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexDescriptor indexDescriptor, SSTableReader sstable)
+    public IndexFeatureSet indexFeatureSet()
     {
-        return indexDescriptor.hasClustering() ? new WidePrimaryKeyMap.Factory(indexDescriptor, sstable)
-                                               : new SkinnyPrimaryKeyMap.Factory(indexDescriptor);
+        return v1IndexFeatureSet;
     }
 
     @Override
-    public SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index)
+    public PrimaryKey.Factory newPrimaryKeyFactory(ClusteringComparator comparator)
     {
-        return new V1SSTableIndex(sstableContext, index);
+        return new PartitionAwarePrimaryKeyFactory();
     }
 
     @Override
-    public PerSSTableIndexWriter newPerSSTableIndexWriter(IndexDescriptor indexDescriptor) throws IOException
+    public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexComponents.ForRead perSSTableComponents, PrimaryKey.Factory primaryKeyFactory, SSTableReader sstable) throws IOException
     {
-        return new SSTableComponentsWriter(indexDescriptor);
+        return new PartitionAwarePrimaryKeyMap.PartitionAwarePrimaryKeyMapFactory(perSSTableComponents, sstable, primaryKeyFactory);
     }
 
     @Override
-    public PerColumnIndexWriter newPerColumnIndexWriter(StorageAttachedIndex index,
-                                                        IndexDescriptor indexDescriptor,
-                                                        LifecycleNewTracker tracker,
-                                                        RowMapping rowMapping)
+    public SearchableIndex newSearchableIndex(SSTableContext sstableContext, IndexComponents.ForRead perIndexComponents)
     {
-        // If we're not flushing, or we haven't yet started the initialization build, flush from SSTable contents.
-        if (tracker.opType() != OperationType.FLUSH || !index.isInitBuildStarted())
-        {
-            NamedMemoryLimiter limiter = SEGMENT_BUILD_MEMORY_LIMITER;
-            logger.info(index.identifier().logMessage("Starting a compaction index build. Global segment memory usage: {}"),
-                        prettyPrintMemory(limiter.currentBytesUsed()));
-
-            return new SSTableIndexWriter(indexDescriptor, index, limiter, index.isIndexValid());
-        }
-
-        return new MemtableIndexWriter(index.memtableIndexManager().getPendingMemtableIndex(tracker),
-                                       indexDescriptor,
-                                       index.termType(),
-                                       index.identifier(),
-                                       index.indexMetrics(),
-                                       rowMapping);
+        return perIndexComponents.isEmpty()
+               ? new EmptyIndex()
+               : new V1SearchableIndex(sstableContext, perIndexComponents);
     }
 
     @Override
-    public boolean isPerSSTableIndexBuildComplete(IndexDescriptor indexDescriptor)
+    public IndexSearcher newIndexSearcher(SSTableContext sstableContext,
+                                          IndexContext indexContext,
+                                          PerIndexFiles indexFiles,
+                                          SegmentMetadata segmentMetadata) throws IOException
     {
-        return indexDescriptor.hasComponent(IndexComponent.GROUP_COMPLETION_MARKER);
+        if (indexContext.isLiteral())
+            // We filter because the CA format wrote maps acording to a different order than their abstract type.
+            return new InvertedIndexSearcher(sstableContext, indexFiles, segmentMetadata, indexContext, Version.AA, true);
+        return new KDTreeIndexSearcher(sstableContext.primaryKeyMapFactory(), indexFiles, segmentMetadata, indexContext);
     }
 
     @Override
-    public boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier)
+    public PerSSTableWriter newPerSSTableWriter(IndexDescriptor indexDescriptor) throws IOException
     {
-        return indexDescriptor.hasComponent(IndexComponent.GROUP_COMPLETION_MARKER) &&
-               indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier);
+        return new SSTableComponentsWriter(indexDescriptor.newPerSSTableComponentsForWrite());
     }
 
     @Override
-    public void validatePerSSTableIndexComponents(IndexDescriptor indexDescriptor, boolean checksum)
+    public PerIndexWriter newPerIndexWriter(StorageAttachedIndex index,
+                                            IndexDescriptor indexDescriptor,
+                                            LifecycleNewTracker tracker,
+                                            RowMapping rowMapping,
+                                            long keyCount)
     {
-        for (IndexComponent indexComponent : perSSTableIndexComponents(indexDescriptor.hasClustering()))
+        IndexContext context = index.getIndexContext();
+        IndexComponents.ForWrite perIndexComponents = indexDescriptor.newPerIndexComponentsForWrite(context);
+        // If we're not flushing or we haven't yet started the initialization build, flush from SSTable contents.
+        if (tracker.opType() != OperationType.FLUSH || !index.canFlushFromMemtableIndex())
         {
-            if (isNotBuildCompletionMarker(indexComponent))
-            {
-                validateIndexComponent(indexDescriptor, null, indexComponent, checksum);
-            }
+            NamedMemoryLimiter limiter = SEGMENT_BUILD_MEMORY_LIMITER;
+            logger.debug(index.getIndexContext().logMessage("Starting a compaction index build. Global segment memory usage: {}"),
+                         prettyPrintMemory(limiter.currentBytesUsed()));
+
+            return new SSTableIndexWriter(perIndexComponents, limiter, index.isIndexValid(), keyCount);
         }
+
+        return new MemtableIndexWriter(context.getPendingMemtableIndex(tracker),
+                                       perIndexComponents,
+                                       context.keyFactory(),
+                                       rowMapping);
     }
 
-    @Override
-    public void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier, boolean checksum)
+    protected Version getExpectedEarliestVersion(IndexContext context, IndexComponentType indexComponentType)
     {
-        // determine if the index is empty, which would be encoded in the column completion marker
-        boolean isEmptyIndex = false;
-        if (indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier))
+        Version earliest = Version.EARLIEST;
+        if (isVectorDataComponent(context, indexComponentType))
         {
-            // first validate the file...
-            validateIndexComponent(indexDescriptor, indexIdentifier, IndexComponent.COLUMN_COMPLETION_MARKER, checksum);
-
-            // ...then read to check if the index is empty
-            try
-            {
-                isEmptyIndex = ColumnCompletionMarkerUtil.isEmptyIndex(indexDescriptor, indexIdentifier);
-            }
-            catch (IOException e)
-            {
-                rethrowIOException(e);
-            }
+            if (!Version.latest().onOrAfter(Version.VECTOR_EARLIEST))
+                throw new IllegalStateException("Configured latest version " + Version.latest() + " is not compatible with vector index");
+            earliest = Version.VECTOR_EARLIEST;
         }
+        return earliest;
+    }
 
-        for (IndexComponent indexComponent : perColumnIndexComponents(indexTermType))
+    @Override
+    public void validateIndexComponent(IndexComponent.ForRead component, boolean checksum)
+    {
+        if (component.isCompletionMarker())
+            return;
+
+        // starting with v3, vector components include proper headers and checksum; skip for earlier versions
+        IndexContext context = component.parent().context();
+        if (isVectorDataComponent(context, component.componentType())
+            && !component.parent().onDiskFormat().indexFeatureSet().hasVectorIndexChecksum())
         {
-            if (!isEmptyIndex && isNotBuildCompletionMarker(indexComponent))
-            {
-                validateIndexComponent(indexDescriptor, indexIdentifier, indexComponent, checksum);
-            }
+            return;
         }
-    }
 
-    private static void validateIndexComponent(IndexDescriptor indexDescriptor,
-                                               IndexIdentifier indexContext,
-                                               IndexComponent indexComponent,
-                                               boolean checksum)
-    {
-        try (IndexInput input = indexContext == null
-                                ? indexDescriptor.openPerSSTableInput(indexComponent)
-                                : indexDescriptor.openPerIndexInput(indexComponent, indexContext))
+        Version earliest = getExpectedEarliestVersion(context, component.componentType());
+        try (IndexInput input = component.openInput())
         {
             if (checksum)
-                SAICodecUtils.validateChecksum(input);
+                SAICodecUtils.validateChecksum(input, earliest);
             else
-                SAICodecUtils.validate(input);
+                SAICodecUtils.validate(input, earliest);
         }
         catch (Exception e)
         {
-            logger.warn(indexDescriptor.logMessage("{} failed for index component {} on SSTable {}"),
-                        checksum ? "Checksum validation" : "Validation",
-                        indexComponent,
-                        indexDescriptor.sstableDescriptor);
-            rethrowIOException(e);
+            logger.warn(component.parent().logMessage("{} failed for index component {} on SSTable {}"),
+                        (checksum ? "Checksum validation" : "Validation"),
+                        component,
+                        component.parent().descriptor(),
+                        e);
+
+            if (e instanceof IOException)
+                throw new UncheckedIOException((IOException) e);
+            if (e.getCause() instanceof IOException)
+                throw new UncheckedIOException((IOException) e.getCause());
+            throw Throwables.unchecked(e);
         }
     }
 
-    private static void rethrowIOException(Exception e)
+    @Override
+    public Set<IndexComponentType> perSSTableComponentTypes()
+    {
+        return PER_SSTABLE_COMPONENTS;
+    }
+
+    @Override
+    public Set<IndexComponentType> perIndexComponentTypes(AbstractType<?> validator)
     {
-        if (e instanceof IOException)
-            throw new UncheckedIOException((IOException) e);
-        if (e.getCause() instanceof IOException)
-            throw new UncheckedIOException((IOException) e.getCause());
-        throw Throwables.unchecked(e);
+        if (TypeUtil.isLiteral(validator))
+            return LITERAL_COMPONENTS;
+        return NUMERIC_COMPONENTS;
     }
 
     @Override
-    public Set<IndexComponent> perSSTableIndexComponents(boolean hasClustering)
+    public int openFilesPerSSTable()
     {
-        return hasClustering ? WIDE_PER_SSTABLE_COMPONENTS : SKINNY_PER_SSTABLE_COMPONENTS;
+        return 2;
     }
 
     @Override
-    public Set<IndexComponent> perColumnIndexComponents(IndexTermType indexTermType)
+    public int openFilesPerIndex(IndexContext indexContext)
     {
-        return indexTermType.isVector() ? VECTOR_COMPONENTS : indexTermType.isLiteral() ? LITERAL_COMPONENTS : NUMERIC_COMPONENTS;
+        // For the V1 format there are always 2 open files per index - index (kdtree or terms) + postings
+        return 2;
     }
 
     @Override
-    public int openFilesPerSSTableIndex(boolean hasClustering)
+    public ByteOrder byteOrderFor(IndexComponentType indexComponentType, IndexContext context)
     {
-        // For the V1 format the number of open files depends on whether the table has clustering. For wide tables
-        // the number of open files will be 6 per SSTable - token values, partition sizes index, partition key blocks,
-        // partition key block offsets, clustering key blocks & clustering key block offsets and for skinny tables
-        // the number of files will be 4 per SSTable - token values, partition key sizes, partition key blocks &
-        // partition key block offsets.
-        return hasClustering ? 6 : 4;
+        return ByteOrder.BIG_ENDIAN;
     }
 
     @Override
-    public int openFilesPerColumnIndex()
+    public ByteComparable encodeForTrie(ByteBuffer input, AbstractType<?> type)
     {
-        // For the V1 format there are always 2 open files per index - index (balanced tree or terms) + auxiliary postings
-        // for the balanced tree and postings for the literal terms
-        return 2;
+        return TypeUtil.isLiteral(type) ? v -> ByteSource.preencoded(input)
+                                        : TypeUtil.asComparableBytes(input, type);
     }
 
-    protected boolean isNotBuildCompletionMarker(IndexComponent indexComponent)
+    @Override
+    public ByteBuffer decodeFromTrie(ByteComparable value, AbstractType<?> type)
     {
-        return indexComponent != IndexComponent.GROUP_COMPLETION_MARKER &&
-               indexComponent != IndexComponent.COLUMN_COMPLETION_MARKER;
+        return TypeUtil.isLiteral(type)
+               ? ByteBuffer.wrap(ByteSourceInverse.readBytes(value.asComparableBytes(ByteComparable.Version.OSS41)))
+               : TypeUtil.fromComparableBytes(value, type, ByteComparable.Version.OSS41);
+    }
+
+    /** vector data components (that did not have checksums before v3) */
+    private boolean isVectorDataComponent(IndexContext context, IndexComponentType indexComponentType)
+    {
+        if (context == null || !context.isVector())
+            return false;
+
+        return indexComponentType == IndexComponentType.VECTOR ||
+               indexComponentType == IndexComponentType.PQ ||
+               indexComponentType == IndexComponentType.TERMS_DATA ||
+               indexComponentType == IndexComponentType.POSTING_LISTS;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java
deleted file mode 100644
index 8813ab3154fc..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.common.collect.ImmutableList;
-
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.virtual.SimpleDataSet;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.SSTableContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
-import org.apache.cassandra.index.sai.disk.v1.segment.Segment;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.Throwables;
-
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.CELL_COUNT;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.COLUMN_NAME;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.COMPONENT_METADATA;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.END_TOKEN;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MAX_SSTABLE_ROW_ID;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MAX_TERM;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MIN_SSTABLE_ROW_ID;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MIN_TERM;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.START_TOKEN;
-import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.TABLE_NAME;
-
-/**
- * A version specific implementation of the {@link SSTableIndex} where the
- * index is segmented
- */
-public class V1SSTableIndex extends SSTableIndex
-{
-    private final ImmutableList<Segment> segments;
-    private final List<SegmentMetadata> metadatas;
-    private final AbstractBounds<PartitionPosition> bounds;
-    private final ByteBuffer minTerm;
-    private final ByteBuffer maxTerm;
-    private final long minSSTableRowId, maxSSTableRowId;
-    private final long numRows;
-
-    private PerColumnIndexFiles indexFiles;
-
-    public V1SSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index)
-    {
-        super(sstableContext, index);
-
-        try
-        {
-            this.indexFiles = new PerColumnIndexFiles(sstableContext.indexDescriptor, indexTermType, indexIdentifier);
-
-            ImmutableList.Builder<Segment> segmentsBuilder = ImmutableList.builder();
-
-            final MetadataSource source = MetadataSource.loadColumnMetadata(sstableContext.indexDescriptor, indexIdentifier);
-
-            metadatas = SegmentMetadata.load(source, sstableContext.indexDescriptor.primaryKeyFactory);
-
-            for (SegmentMetadata metadata : metadatas)
-            {
-                segmentsBuilder.add(new Segment(index, sstableContext, indexFiles, metadata));
-            }
-
-            segments = segmentsBuilder.build();
-            assert !segments.isEmpty();
-
-            DecoratedKey minKey = metadatas.get(0).minKey.partitionKey();
-            DecoratedKey maxKey = metadatas.get(metadatas.size() - 1).maxKey.partitionKey();
-
-            this.bounds = AbstractBounds.bounds(minKey, true, maxKey, true);
-
-            this.minTerm = metadatas.stream().map(m -> m.minTerm).min(indexTermType.comparator()).orElse(null);
-            this.maxTerm = metadatas.stream().map(m -> m.maxTerm).max(indexTermType.comparator()).orElse(null);
-
-            this.numRows = metadatas.stream().mapToLong(m -> m.numRows).sum();
-
-            this.minSSTableRowId = metadatas.get(0).minSSTableRowId;
-            this.maxSSTableRowId = metadatas.get(metadatas.size() - 1).maxSSTableRowId;
-        }
-        catch (Throwable t)
-        {
-            FileUtils.closeQuietly(indexFiles);
-            FileUtils.closeQuietly(sstableContext);
-            throw Throwables.unchecked(t);
-        }
-    }
-
-    @Override
-    public long indexFileCacheSize()
-    {
-        return segments.stream().mapToLong(Segment::indexFileCacheSize).sum();
-    }
-
-    @Override
-    public long getRowCount()
-    {
-        return numRows;
-    }
-
-    @Override
-    public long minSSTableRowId()
-    {
-        return minSSTableRowId;
-    }
-
-    @Override
-    public long maxSSTableRowId()
-    {
-        return maxSSTableRowId;
-    }
-
-    @Override
-    public ByteBuffer minTerm()
-    {
-        return minTerm;
-    }
-
-    @Override
-    public ByteBuffer maxTerm()
-    {
-        return maxTerm;
-    }
-
-    @Override
-    public AbstractBounds<PartitionPosition> bounds()
-    {
-        return bounds;
-    }
-
-    @Override
-    public List<KeyRangeIterator> search(Expression expression,
-                                         AbstractBounds<PartitionPosition> keyRange,
-                                         QueryContext context) throws IOException
-    {
-        List<KeyRangeIterator> segmentIterators = new ArrayList<>();
-
-        for (Segment segment : segments)
-        {
-            if (segment.intersects(keyRange))
-            {
-                segmentIterators.add(segment.search(expression, keyRange, context));
-            }
-        }
-
-        return segmentIterators;
-    }
-
-    @Override
-    public KeyRangeIterator limitToTopKResults(QueryContext context, List<PrimaryKey> primaryKeys, Expression expression) throws IOException
-    {
-        KeyRangeUnionIterator.Builder unionIteratorBuilder = KeyRangeUnionIterator.builder(segments.size());
-        for (Segment segment : segments)
-            unionIteratorBuilder.add(segment.limitToTopKResults(context, primaryKeys, expression));
-
-        return unionIteratorBuilder.build();
-    }
-
-    @Override
-    public void populateSegmentView(SimpleDataSet dataset)
-    {
-        SSTableReader sstable = getSSTable();
-        Token.TokenFactory tokenFactory = sstable.metadata().partitioner.getTokenFactory();
-
-        for (SegmentMetadata metadata : metadatas)
-        {
-            dataset.row(sstable.metadata().keyspace, indexIdentifier.indexName, sstable.getFilename(), metadata.rowIdOffset)
-                   .column(TABLE_NAME, sstable.descriptor.cfname)
-                   .column(COLUMN_NAME, indexTermType.columnName())
-                   .column(CELL_COUNT, metadata.numRows)
-                   .column(MIN_SSTABLE_ROW_ID, metadata.minSSTableRowId)
-                   .column(MAX_SSTABLE_ROW_ID, metadata.maxSSTableRowId)
-                   .column(START_TOKEN, tokenFactory.toString(metadata.minKey.token()))
-                   .column(END_TOKEN, tokenFactory.toString(metadata.maxKey.token()))
-                   .column(MIN_TERM, indexTermType.indexType().getSerializer().deserialize(metadata.minTerm).toString())
-                   .column(MAX_TERM, indexTermType.indexType().getSerializer().deserialize(metadata.maxTerm).toString())
-                   .column(COMPONENT_METADATA, metadata.componentMetadatas.asMap());
-        }
-    }
-
-    @Override
-    protected void internalRelease()
-    {
-        FileUtils.closeQuietly(indexFiles);
-        FileUtils.closeQuietly(segments);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/V1SearchableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/v1/V1SearchableIndex.java
new file mode 100644
index 000000000000..28f90d590f8c
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/V1SearchableIndex.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.SearchableIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.iterators.KeyRangeConcatIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.CELL_COUNT;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.COLUMN_NAME;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.COMPONENT_METADATA;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.END_TOKEN;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MAX_SSTABLE_ROW_ID;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MAX_TERM;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MIN_SSTABLE_ROW_ID;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MIN_TERM;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.START_TOKEN;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.TABLE_NAME;
+
+/**
+ * A version specific implementation of the {@link SearchableIndex} where the
+ * index is segmented
+ */
+public class V1SearchableIndex implements SearchableIndex
+{
+    private final IndexContext indexContext;
+    private final ImmutableList<Segment> segments;
+    private final List<SegmentMetadata> metadatas;
+    private final DecoratedKey minKey;
+    private final DecoratedKey maxKey; // in token order
+    private final ByteBuffer minTerm;
+    private final ByteBuffer maxTerm;
+    private final long minSSTableRowId, maxSSTableRowId;
+    private final long numRows;
+    private PerIndexFiles indexFiles;
+
+    public V1SearchableIndex(SSTableContext sstableContext, IndexComponents.ForRead perIndexComponents)
+    {
+        this.indexContext = perIndexComponents.context();
+        try
+        {
+            this.indexFiles = new PerIndexFiles(perIndexComponents);
+
+            ImmutableList.Builder<Segment> segmentsBuilder = ImmutableList.builder();
+
+            final MetadataSource source = MetadataSource.loadMetadata(perIndexComponents);
+
+            metadatas = SegmentMetadata.load(source, indexContext);
+
+            for (SegmentMetadata metadata : metadatas)
+            {
+                segmentsBuilder.add(new Segment(indexContext, sstableContext, indexFiles, metadata));
+            }
+
+            segments = segmentsBuilder.build();
+            assert !segments.isEmpty();
+
+            this.minKey = metadatas.get(0).minKey.partitionKey();
+            this.maxKey = metadatas.get(metadatas.size() - 1).maxKey.partitionKey();
+
+            var version = perIndexComponents.version();
+            this.minTerm = metadatas.stream().map(m -> m.minTerm).min(TypeUtil.comparator(indexContext.getValidator(), version)).orElse(null);
+            this.maxTerm = metadatas.stream().map(m -> m.maxTerm).max(TypeUtil.comparator(indexContext.getValidator(), version)).orElse(null);
+
+            this.numRows = metadatas.stream().mapToLong(m -> m.numRows).sum();
+
+            this.minSSTableRowId = metadatas.get(0).minSSTableRowId;
+            this.maxSSTableRowId = metadatas.get(metadatas.size() - 1).maxSSTableRowId;
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(indexFiles);
+            FileUtils.closeQuietly(sstableContext);
+            throw Throwables.unchecked(t);
+        }
+    }
+
+    @Override
+    public long indexFileCacheSize()
+    {
+        return segments.stream().mapToLong(Segment::indexFileCacheSize).sum();
+    }
+
+    @Override
+    public long getRowCount()
+    {
+        return numRows;
+    }
+
+    @Override
+    public long minSSTableRowId()
+    {
+        return minSSTableRowId;
+    }
+
+    @Override
+    public long maxSSTableRowId()
+    {
+        return maxSSTableRowId;
+    }
+
+    @Override
+    public ByteBuffer minTerm()
+    {
+        return minTerm;
+    }
+
+    @Override
+    public ByteBuffer maxTerm()
+    {
+        return maxTerm;
+    }
+
+    @Override
+    public DecoratedKey minKey()
+    {
+        return minKey;
+    }
+
+    @Override
+    public DecoratedKey maxKey()
+    {
+        return maxKey;
+    }
+
+    @Override
+    public KeyRangeIterator search(Expression expression,
+                                   AbstractBounds<PartitionPosition> keyRange,
+                                   QueryContext context,
+                                   boolean defer,
+                                   int limit) throws IOException
+    {
+        KeyRangeConcatIterator.Builder rangeConcatIteratorBuilder = KeyRangeConcatIterator.builder(segments.size());
+
+        try
+        {
+            for (Segment segment : segments)
+            {
+                if (segment.intersects(keyRange))
+                {
+                    rangeConcatIteratorBuilder.add(segment.search(expression, keyRange, context, defer, limit));
+                }
+            }
+
+            return rangeConcatIteratorBuilder.build();
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(rangeConcatIteratorBuilder.ranges());
+            throw t;
+        }
+    }
+
+    @Override
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderBy(Orderer orderer, Expression slice,
+                                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                                  QueryContext context,
+                                                                  int limit,
+                                                                  long totalRows) throws IOException
+    {
+        var iterators = new ArrayList<CloseableIterator<PrimaryKeyWithSortKey>>(segments.size());
+        try
+        {
+            for (Segment segment : segments)
+            {
+                if (segment.intersects(keyRange))
+                {
+                    // Note that the proportionality is not used when the user supplies a rerank_k value in the
+                    // ANN_OPTIONS map.
+                    var segmentLimit = segment.proportionalAnnLimit(limit, totalRows);
+                    iterators.add(segment.orderBy(orderer, slice, keyRange, context, segmentLimit));
+                }
+            }
+
+            return iterators;
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(iterators);
+            throw t;
+        }
+    }
+
+    @Override
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderResultsBy(QueryContext context, List<PrimaryKey> keys, Orderer orderer, int limit, long totalRows) throws IOException
+    {
+        var results = new ArrayList<CloseableIterator<PrimaryKeyWithSortKey>>(segments.size());
+        try
+        {
+            for (Segment segment : segments)
+            {
+                // Only pass the primary keys in a segment's range to the segment index.
+                var segmentKeys = getKeysInRange(keys, segment);
+                var segmentLimit = segment.proportionalAnnLimit(limit, totalRows);
+                results.add(segment.orderResultsBy(context, segmentKeys, orderer, segmentLimit));
+            }
+
+            return results;
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(results);
+            throw t;
+        }
+    }
+
+    @Override
+    public List<Segment> getSegments()
+    {
+        return segments;
+    }
+
+    @Override
+    public void populateSystemView(SimpleDataSet dataset, SSTableReader sstable)
+    {
+        Token.TokenFactory tokenFactory = sstable.metadata().partitioner.getTokenFactory();
+
+        for (SegmentMetadata metadata : metadatas)
+        {
+            String minTerm = indexContext.isVector() ? "N/A" : indexContext.getValidator().getSerializer().deserialize(metadata.minTerm).toString();
+            String maxTerm = indexContext.isVector() ? "N/A" : indexContext.getValidator().getSerializer().deserialize(metadata.maxTerm).toString();
+
+            dataset.row(sstable.metadata().keyspace, indexContext.getIndexName(), sstable.getFilename(), metadata.segmentRowIdOffset)
+                   .column(TABLE_NAME, sstable.descriptor.cfname)
+                   .column(COLUMN_NAME, indexContext.getColumnName())
+                   .column(CELL_COUNT, metadata.numRows)
+                   .column(MIN_SSTABLE_ROW_ID, metadata.minSSTableRowId)
+                   .column(MAX_SSTABLE_ROW_ID, metadata.maxSSTableRowId)
+                   .column(START_TOKEN, tokenFactory.toString(metadata.minKey.partitionKey().getToken()))
+                   .column(END_TOKEN, tokenFactory.toString(metadata.maxKey.partitionKey().getToken()))
+                   .column(MIN_TERM, minTerm)
+                   .column(MAX_TERM, maxTerm)
+                   .column(COMPONENT_METADATA, metadata.componentMetadatas.asMap());
+        }
+    }
+
+    @Override
+    public long estimateMatchingRowsCount(Expression predicate, AbstractBounds<PartitionPosition> keyRange)
+    {
+        long rowCount = 0;
+        for (Segment segment: segments)
+        {
+            long c = segment.estimateMatchingRowsCount(predicate, keyRange);
+            assert c >= 0 : "Estimated row count must not be negative: " + c + " (predicate: " + predicate + ')';
+            rowCount += c;
+        }
+        return rowCount;
+    }
+
+    /** Create a sublist of the keys within (inclusive) the segment's bounds */
+    protected List<PrimaryKey> getKeysInRange(List<PrimaryKey> keys, Segment segment)
+    {
+        int minIndex = findBoundaryIndex(keys, segment, true);
+        int maxIndex = findBoundaryIndex(keys, segment, false);
+        return keys.subList(minIndex, maxIndex);
+    }
+
+    private int findBoundaryIndex(List<PrimaryKey> keys, Segment segment, boolean findMin)
+    {
+        // The minKey and maxKey are sometimes just partition keys (not primary keys), so binarySearch
+        // may not return the index of the least/greatest match.
+        var key = findMin ? segment.metadata.minKey : segment.metadata.maxKey;
+        int index = Collections.binarySearch(keys, key);
+        if (index < 0)
+            return -index - 1;
+        if (findMin)
+        {
+            while (index > 0 && keys.get(index - 1).equals(key))
+                index--;
+        }
+        else
+        {
+            while (index < keys.size() - 1 && keys.get(index + 1).equals(key))
+                index++;
+            // We must include the PrimaryKey at the boundary
+            index++;
+        }
+        return index;
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        FileUtils.closeQuietly(indexFiles);
+        FileUtils.closeQuietly(segments);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/WidePrimaryKeyMap.java b/src/java/org/apache/cassandra/index/sai/disk/v1/WidePrimaryKeyMap.java
deleted file mode 100644
index c37681e268ef..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/WidePrimaryKeyMap.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.io.IOException;
-import java.util.Arrays;
-import javax.annotation.concurrent.NotThreadSafe;
-import javax.annotation.concurrent.ThreadSafe;
-
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.ClusteringComparator;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.BlockPackedReader;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
-import org.apache.cassandra.index.sai.disk.v1.keystore.KeyLookupMeta;
-import org.apache.cassandra.index.sai.disk.v1.keystore.KeyLookup;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.Throwables;
-
-/**
- * An extension of the {@link SkinnyPrimaryKeyMap} for wide tables (those with clustering columns).
- * <p>
- * This used the following additional on-disk structures to the {@link SkinnyPrimaryKeyMap}
- * <ul>
- *     <li>A block-packed structure for partitionId to partition size (number of rows in the partition) lookups using
- *     {@link BlockPackedReader}. Uses the {@link IndexComponent#PARTITION_TO_SIZE} component</li>
- *     <li>A key store for rowId to {@link Clustering} and {@link Clustering} to rowId lookups using
- *     {@link KeyLookup}. Uses the {@link IndexComponent#CLUSTERING_KEY_BLOCKS} and
- *     {@link IndexComponent#CLUSTERING_KEY_BLOCK_OFFSETS} components</li>
- * </ul>
- * While the {@link Factory} is threadsafe, individual instances of the {@link WidePrimaryKeyMap}
- * are not.
- */
-@NotThreadSafe
-public class WidePrimaryKeyMap extends SkinnyPrimaryKeyMap
-{
-    @ThreadSafe
-    public static class Factory extends SkinnyPrimaryKeyMap.Factory
-    {
-        private final ClusteringComparator clusteringComparator;
-        private final KeyLookup clusteringKeyReader;
-        private final LongArray.Factory partitionToSizeReaderFactory;
-        private final FileHandle clusteringKeyBlockOffsetsFile;
-        private final FileHandle clustingingKeyBlocksFile;
-        private final FileHandle partitionToSizeFile;
-
-        public Factory(IndexDescriptor indexDescriptor, SSTableReader sstable)
-        {
-            super(indexDescriptor);
-
-            this.clusteringKeyBlockOffsetsFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.CLUSTERING_KEY_BLOCK_OFFSETS, this::close);
-            this.clustingingKeyBlocksFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.CLUSTERING_KEY_BLOCKS, this::close);
-            this.partitionToSizeFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PARTITION_TO_SIZE, this::close);
-
-            try
-            {
-                this.clusteringComparator = indexDescriptor.clusteringComparator;
-                NumericValuesMeta partitionSizeMeta = new NumericValuesMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.PARTITION_TO_SIZE)));
-                this.partitionToSizeReaderFactory = new BlockPackedReader(partitionToSizeFile, partitionSizeMeta);
-                NumericValuesMeta clusteringKeyBlockOffsetsMeta = new NumericValuesMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.CLUSTERING_KEY_BLOCK_OFFSETS)));
-                KeyLookupMeta clusteringKeyMeta = new KeyLookupMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.CLUSTERING_KEY_BLOCKS)));
-                this.clusteringKeyReader = new KeyLookup(clustingingKeyBlocksFile, clusteringKeyBlockOffsetsFile, clusteringKeyMeta, clusteringKeyBlockOffsetsMeta);
-            }
-            catch (Throwable t)
-            {
-                throw Throwables.unchecked(t);
-            }
-        }
-
-        @Override
-        @SuppressWarnings({ "resource", "RedundantSuppression" }) // deferred long arrays and cursors are closed in the WidePrimaryKeyMap#close method
-        public PrimaryKeyMap newPerSSTablePrimaryKeyMap() throws IOException
-        {
-            LongArray rowIdToToken = new LongArray.DeferredLongArray(rowToTokenReaderFactory::open);
-            LongArray partitionIdToToken = new LongArray.DeferredLongArray(rowToPartitionReaderFactory::open);
-            LongArray partitionIdToSize = new LongArray.DeferredLongArray(partitionToSizeReaderFactory::open);
-
-            return new WidePrimaryKeyMap(rowIdToToken,
-                                         partitionIdToToken,
-                                         partitionIdToSize,
-                                         partitionKeyReader.openCursor(),
-                                         clusteringKeyReader.openCursor(),
-                                         primaryKeyFactory,
-                                         clusteringComparator);
-        }
-
-        @Override
-        public void close()
-        {
-            super.close();
-            FileUtils.closeQuietly(Arrays.asList(clustingingKeyBlocksFile, clusteringKeyBlockOffsetsFile, partitionToSizeFile));
-        }
-    }
-
-    private final LongArray partitionIdToSizeArray;
-    private final ClusteringComparator clusteringComparator;
-    private final KeyLookup.Cursor clusteringKeyCursor;
-
-    private WidePrimaryKeyMap(LongArray rowIdToTokenArray,
-                              LongArray rowIdToPartitionIdArray,
-                              LongArray partitionIdToSizeArray,
-                              KeyLookup.Cursor partitionKeyCursor,
-                              KeyLookup.Cursor clusteringKeyCursor,
-                              PrimaryKey.Factory primaryKeyFactory,
-                              ClusteringComparator clusteringComparator)
-    {
-        super(rowIdToTokenArray, rowIdToPartitionIdArray, partitionKeyCursor, primaryKeyFactory);
-
-        this.partitionIdToSizeArray = partitionIdToSizeArray;
-        this.clusteringComparator = clusteringComparator;
-        this.clusteringKeyCursor = clusteringKeyCursor;
-    }
-
-    @Override
-    public PrimaryKey primaryKeyFromRowId(long sstableRowId)
-    {
-        return primaryKeyFactory.create(readPartitionKey(sstableRowId), readClusteringKey(sstableRowId));
-    }
-
-    @Override
-    public long rowIdFromPrimaryKey(PrimaryKey primaryKey)
-    {
-        long rowId = rowIdToTokenArray.indexOf(primaryKey.token().getLongValue());
-
-        // If the key only has a token (initial range skip in the query), the token is out of range,
-        // or we have skipped a token, return the rowId from the token array.
-        if (primaryKey.kind() == PrimaryKey.Kind.TOKEN || rowId < 0 || rowIdToTokenArray.get(rowId) != primaryKey.token().getLongValue())
-            return rowId;
-
-        rowId = tokenCollisionDetection(primaryKey, rowId);
-
-        // Search the key store for the key in the same partition
-        return clusteringKeyCursor.clusteredSeekToKey(clusteringComparator.asByteComparable(primaryKey.clustering()), rowId, startOfNextPartition(rowId));
-    }
-
-    @Override
-    public long floor(Token token)
-    {
-        if (token.isMinimum())
-            return Long.MIN_VALUE;
-        long rowId = rowIdToTokenArray.indexOf(token.getLongValue());
-        return rowId < 0 ? rowId : startOfNextPartition(rowId) - 1;
-    }
-
-    @Override
-    public void close()
-    {
-        super.close();
-        FileUtils.closeQuietly(clusteringKeyCursor);
-    }
-
-    private Clustering<?> readClusteringKey(long sstableRowId)
-    {
-        return primaryKeyFactory.clusteringFromByteComparable(clusteringKeyCursor.seekToPointId(sstableRowId));
-    }
-
-    // Returns the rowId of the next partition or the number of rows if supplied rowId is in the last partition
-    private long startOfNextPartition(long rowId)
-    {
-        long partitionSize = partitionIdToSizeArray.get(rowIdToPartitionIdArray.get(rowId));
-        return partitionSize == -1 ? rowIdToPartitionIdArray.length() : rowId + partitionSize;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsIndex.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsIndex.java
deleted file mode 100644
index 4b87f4673acf..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsIndex.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.IOException;
-
-import com.carrotsearch.hppc.IntLongHashMap;
-import com.carrotsearch.hppc.IntLongMap;
-import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.RandomAccessReader;
-
-import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.validate;
-
-/**
- * Mapping between node ID and an offset to its auxiliary posting list (containing every row id from all leaves
- * reachable from that node. See {@link BlockBalancedTreePostingsWriter}).
- */
-class BlockBalancedTreePostingsIndex
-{
-    private final int size;
-    public final IntLongMap index = new IntLongHashMap();
-
-    BlockBalancedTreePostingsIndex(FileHandle postingsFileHandle, long filePosition) throws IOException
-    {
-        try (RandomAccessReader reader = postingsFileHandle.createReader();
-             IndexInputReader input = IndexInputReader.create(reader))
-        {
-            validate(input);
-            input.seek(filePosition);
-
-            size = input.readVInt();
-
-            for (int x = 0; x < size; x++)
-            {
-                final int node = input.readVInt();
-                final long filePointer = input.readVLong();
-
-                index.put(node, filePointer);
-            }
-        }
-    }
-
-    /**
-     * Returns <tt>true</tt> if given node ID has an auxiliary posting list.
-     */
-    boolean exists(int nodeID)
-    {
-        return index.containsKey(nodeID);
-    }
-
-    /**
-     * Returns an offset within the balanced tree postings file to the begining of the blocks summary of given node's auxiliary
-     * posting list.
-     *
-     * @throws IllegalArgumentException when given nodeID doesn't have an auxiliary posting list. Check first with
-     * {@link #exists(int)}
-     */
-    long getPostingsFilePointer(int nodeID)
-    {
-        return index.get(nodeID);
-    }
-
-    int size()
-    {
-        return size;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriter.java
deleted file mode 100644
index 590528e782a4..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriter.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Multimap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.agrona.collections.IntArrayList;
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList;
-import org.apache.cassandra.index.sai.disk.v1.postings.PackedLongsPostingList;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
-import org.apache.cassandra.index.sai.postings.PeekablePostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.packed.PackedLongValues;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-/**
- * Writes leaf postings and auxiliary posting lists for bbtree nodes. If a node has a posting list attached,
- * it will contain every row id from all leaves reachable from that node.
- * <p>
- * Writer is stateful, because it needs to collect data from the balanced tree data structure first to find set of eligible
- * nodes and leaf nodes reachable from them.
- * <p>
- * The leaf blocks are written in value order (in the order we pass them to the {@link BlockBalancedTreeWriter}).
- * This allows us to skip reading the leaves, instead just order leaf blocks by their offset in the index file,
- * and correlate them with buffered posting lists.
- */
-@NotThreadSafe
-public class BlockBalancedTreePostingsWriter implements BlockBalancedTreeWalker.TraversalCallback
-{
-    private static final Logger logger = LoggerFactory.getLogger(BlockBalancedTreePostingsWriter.class);
-
-    private final TreeMap<Long, Integer> leafOffsetToNodeID = new TreeMap<>(Long::compareTo);
-    private final Multimap<Integer, Integer> nodeToChildLeaves = HashMultimap.create();
-
-    /**
-     * Minimum number of reachable leaves for a given node to be eligible for an auxiliary posting list.
-     */
-    private final int minimumPostingsLeaves;
-    /**
-     * Skip, or the sampling interval, for selecting a balanced tree level that is eligible for an auxiliary posting list.
-     * Sampling starts from 0, but the balanced tree root node is at level 1. For skip = 4, eligible levels are 4, 8, 12, etc. (no
-     * level 0, because there is no node at level 0).
-     */
-    private final int postingsSkip;
-
-    int numNonLeafPostings = 0;
-    int numLeafPostings = 0;
-
-    public BlockBalancedTreePostingsWriter()
-    {
-        minimumPostingsLeaves = CassandraRelevantProperties.SAI_MINIMUM_POSTINGS_LEAVES.getInt();
-        postingsSkip = CassandraRelevantProperties.SAI_POSTINGS_SKIP.getInt();
-    }
-
-    /**
-     * Called when a leaf node is hit as we traverse the packed index.
-     *
-     * @param leafNodeID the current leaf node ID in the packed inded
-     * @param leafBlockFP the file pointer to the on-disk leaf block
-     * @param pathToRoot the path to the root leaf above this leaf. Contains all the intermediate leaf node IDs.
-     */
-    @Override
-    public void onLeaf(int leafNodeID, long leafBlockFP, IntArrayList pathToRoot)
-    {
-        checkArgument(!pathToRoot.containsInt(leafNodeID));
-        checkArgument(pathToRoot.isEmpty() || leafNodeID > pathToRoot.get(pathToRoot.size() - 1));
-
-        leafOffsetToNodeID.put(leafBlockFP, leafNodeID);
-        for (int i = 0; i < pathToRoot.size(); i++)
-        {
-            int level = i + 1;
-            if (isLevelEligibleForPostingList(level))
-            {
-                int nodeID = pathToRoot.get(i);
-                nodeToChildLeaves.put(nodeID, leafNodeID);
-            }
-        }
-    }
-
-    /**
-     * Writes merged posting lists for eligible internal nodes and leaf postings for each leaf in the tree.
-     * The merged postings list for an internal node contains all postings from the postings lists of leaf nodes
-     * in the subtree rooted at that node.
-     * <p>
-     * After writing out the postings, it writes a map of node ID -> postings file pointer for all
-     * nodes with an attached postings list. It then returns the file pointer to this map.
-     */
-    public long finish(IndexOutputWriter out, List<PackedLongValues> leafPostings, IndexIdentifier indexIdentifier) throws IOException
-    {
-        checkState(leafPostings.size() == leafOffsetToNodeID.size(),
-                   "Expected equal number of postings lists (%s) and leaf offsets (%s).",
-                   leafPostings.size(), leafOffsetToNodeID.size());
-
-        try (PostingsWriter postingsWriter = new PostingsWriter(out))
-        {
-            Iterator<PackedLongValues> postingsIterator = leafPostings.iterator();
-            Map<Integer, PackedLongValues> leafToPostings = new HashMap<>();
-            leafOffsetToNodeID.forEach((fp, nodeID) -> leafToPostings.put(nodeID, postingsIterator.next()));
-
-            long postingsRamBytesUsed = leafPostings.stream()
-                                                .mapToLong(PackedLongValues::ramBytesUsed)
-                                                .sum();
-
-            List<Integer> internalNodeIDs = nodeToChildLeaves.keySet()
-                                                             .stream()
-                                                             .filter(i -> nodeToChildLeaves.get(i).size() >= minimumPostingsLeaves)
-                                                             .collect(Collectors.toList());
-
-            Collection<Integer> leafNodeIDs = leafOffsetToNodeID.values();
-
-            logger.debug(indexIdentifier.logMessage("Writing posting lists for {} internal and {} leaf balanced tree nodes. Leaf postings memory usage: {}."),
-                         internalNodeIDs.size(), leafNodeIDs.size(), FBUtilities.prettyPrintMemory(postingsRamBytesUsed));
-
-            long startFP = out.getFilePointer();
-            Stopwatch flushTime = Stopwatch.createStarted();
-            TreeMap<Integer, Long> nodeIDToPostingsFilePointer = new TreeMap<>();
-            PriorityQueue<PeekablePostingList> postingLists = new PriorityQueue<>(minimumPostingsLeaves, Comparator.comparingLong(PeekablePostingList::peek));
-            for (int nodeID : Iterables.concat(internalNodeIDs, leafNodeIDs))
-            {
-                Collection<Integer> leaves = nodeToChildLeaves.get(nodeID);
-
-                if (leaves.isEmpty())
-                {
-                    leaves = Collections.singletonList(nodeID);
-                    numLeafPostings++;
-                }
-                else
-                {
-                    numNonLeafPostings++;
-                }
-
-                for (Integer leaf : leaves)
-                    postingLists.add(PeekablePostingList.makePeekable(new PackedLongsPostingList(leafToPostings.get(leaf))));
-
-                try (PostingList mergedPostingList = MergePostingList.merge(postingLists))
-                {
-                    long postingFilePosition = postingsWriter.write(mergedPostingList);
-                    // During compaction, we could end up with an empty postings due to deletions.
-                    // The writer will return a fp of -1 if no postings were written.
-                    if (postingFilePosition >= 0)
-                        nodeIDToPostingsFilePointer.put(nodeID, postingFilePosition);
-                }
-                postingLists.clear();
-            }
-            flushTime.stop();
-            logger.debug(indexIdentifier.logMessage("Flushed {} of posting lists for balanced tree nodes in {} ms."),
-                         FBUtilities.prettyPrintMemory(out.getFilePointer() - startFP),
-                         flushTime.elapsed(TimeUnit.MILLISECONDS));
-
-            long indexFilePointer = out.getFilePointer();
-            writeMap(nodeIDToPostingsFilePointer, out);
-            postingsWriter.complete();
-            return indexFilePointer;
-        }
-    }
-
-    private boolean isLevelEligibleForPostingList(int level)
-    {
-        return level > 1 && level % postingsSkip == 0;
-    }
-
-    private void writeMap(Map<Integer, Long> map, IndexOutput out) throws IOException
-    {
-        out.writeVInt(map.size());
-
-        for (Map.Entry<Integer, Long> e : map.entrySet())
-        {
-            out.writeVInt(e.getKey());
-            out.writeVLong(e.getValue());
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueries.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueries.java
deleted file mode 100644
index bb4b477c40d7..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueries.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.utils.ByteArrayUtil;
-import org.apache.lucene.index.PointValues.Relation;
-
-public class BlockBalancedTreeQueries
-{
-    private static final BlockBalancedTreeReader.IntersectVisitor MATCH_ALL = new BlockBalancedTreeReader.IntersectVisitor()
-    {
-        @Override
-        public boolean contains(byte[] packedValue)
-        {
-            return true;
-        }
-
-        @Override
-        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
-        {
-            return Relation.CELL_INSIDE_QUERY;
-        }
-    };
-
-    public static BlockBalancedTreeReader.IntersectVisitor balancedTreeQueryFrom(Expression expression, int bytesPerValue)
-    {
-        if (expression.lower() == null && expression.upper() == null)
-        {
-            return MATCH_ALL;
-        }
-
-        Bound lower = null ;
-        if (expression.lower() != null)
-        {
-            final byte[] lowerBound = toComparableBytes(bytesPerValue, expression.lower().value.encoded, expression.getIndexTermType());
-            lower = new Bound(lowerBound, !expression.lower().inclusive);
-        }
-
-        Bound upper = null;
-        if (expression.upper() != null)
-        {
-            final byte[] upperBound = toComparableBytes(bytesPerValue, expression.upper().value.encoded, expression.getIndexTermType());
-            upper = new Bound(upperBound, !expression.upper().inclusive);
-        }
-
-        return new RangeQueryVisitor(lower, upper);
-    }
-
-    private static byte[] toComparableBytes(int bytesPerDim, ByteBuffer value, IndexTermType indexTermType)
-    {
-        byte[] buffer = new byte[indexTermType.fixedSizeOf()];
-        assert buffer.length == bytesPerDim;
-        indexTermType.toComparableBytes(value, buffer);
-        return buffer;
-    }
-
-    private static class Bound
-    {
-        private final byte[] bound;
-        private final boolean exclusive;
-
-        Bound(byte[] bound, boolean exclusive)
-        {
-            this.bound = bound;
-            this.exclusive = exclusive;
-        }
-
-        boolean smallerThan(byte[] packedValue)
-        {
-            int cmp = compareTo(packedValue);
-            return cmp < 0 || (cmp == 0 && exclusive);
-        }
-
-        boolean greaterThan(byte[] packedValue)
-        {
-            int cmp = compareTo(packedValue);
-            return cmp > 0 || (cmp == 0 && exclusive);
-        }
-
-        private int compareTo(byte[] packedValue)
-        {
-            return ByteArrayUtil.compareUnsigned(bound, 0, packedValue, 0, bound.length);
-        }
-    }
-
-    private static class RangeQueryVisitor implements BlockBalancedTreeReader.IntersectVisitor
-    {
-        private final Bound lower;
-        private final Bound upper;
-
-        private RangeQueryVisitor(Bound lower, Bound upper)
-        {
-            this.lower = lower;
-            this.upper = upper;
-        }
-
-        @Override
-        public boolean contains(byte[] packedValue)
-        {
-            if (lower != null)
-            {
-                if (lower.greaterThan(packedValue))
-                {
-                    // value is too low, in this dimension
-                    return false;
-                }
-            }
-
-            if (upper != null)
-            {
-                return !upper.smallerThan(packedValue);
-            }
-
-            return true;
-        }
-
-        @Override
-        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
-        {
-            boolean crosses = false;
-
-            if (lower != null)
-            {
-                if (lower.greaterThan(maxPackedValue))
-                    return Relation.CELL_OUTSIDE_QUERY;
-
-                crosses = lower.greaterThan(minPackedValue);
-            }
-
-            if (upper != null)
-            {
-                if (upper.smallerThan(minPackedValue))
-                    return Relation.CELL_OUTSIDE_QUERY;
-
-                crosses |= upper.smallerThan(maxPackedValue);
-            }
-
-            return crosses ? Relation.CELL_CROSSES_QUERY : Relation.CELL_INSIDE_QUERY;
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReader.java
deleted file mode 100644
index 0f14db390b4f..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReader.java
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.Comparator;
-import java.util.PriorityQueue;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.base.Stopwatch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.exceptions.QueryCancelledException;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
-import org.apache.cassandra.index.sai.disk.io.SeekingRandomAccessInput;
-import org.apache.cassandra.index.sai.disk.v1.postings.FilteringPostingList;
-import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
-import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.postings.PeekablePostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.ByteArrayUtil;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.LongValues;
-import org.apache.lucene.util.packed.DirectReader;
-import org.apache.lucene.util.packed.DirectWriter;
-
-/**
- * Handles intersection of a point or point range with a block balanced tree previously written with
- * {@link BlockBalancedTreeWriter}.
- */
-public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements Closeable
-{
-    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-    private static final Comparator<PeekablePostingList> COMPARATOR = Comparator.comparingLong(PeekablePostingList::peek);
-
-    private final IndexIdentifier indexIdentifier;
-    private final FileHandle postingsFile;
-    private final BlockBalancedTreePostingsIndex postingsIndex;
-    private final int leafOrderMapBitsRequired;
-    /**
-     * Performs a blocking read.
-     */
-    public BlockBalancedTreeReader(IndexIdentifier indexIdentifier,
-                                   FileHandle treeIndexFile,
-                                   long treeIndexRoot,
-                                   FileHandle postingsFile,
-                                   long treePostingsRoot) throws IOException
-    {
-        super(treeIndexFile, treeIndexRoot);
-        this.indexIdentifier = indexIdentifier;
-        this.postingsFile = postingsFile;
-        this.postingsIndex = new BlockBalancedTreePostingsIndex(postingsFile, treePostingsRoot);
-        leafOrderMapBitsRequired = DirectWriter.unsignedBitsRequired(maxValuesInLeafNode - 1);
-    }
-
-    public int getBytesPerValue()
-    {
-        return bytesPerValue;
-    }
-
-    public long getPointCount()
-    {
-        return valueCount;
-    }
-
-    @Override
-    public void close()
-    {
-        super.close();
-        FileUtils.closeQuietly(postingsFile);
-    }
-
-    public PostingList intersect(IntersectVisitor visitor, QueryEventListener.BalancedTreeEventListener listener, QueryContext context)
-    {
-        Relation relation = visitor.compare(minPackedValue, maxPackedValue);
-
-        if (relation == Relation.CELL_OUTSIDE_QUERY)
-        {
-            listener.onIntersectionEarlyExit();
-            return null;
-        }
-
-        listener.onSegmentHit();
-        IndexInput treeInput = IndexFileUtils.instance.openInput(treeIndexFile);
-        IndexInput postingsInput = IndexFileUtils.instance.openInput(postingsFile);
-        IndexInput postingsSummaryInput = IndexFileUtils.instance.openInput(postingsFile);
-
-        Intersection intersection = relation == Relation.CELL_INSIDE_QUERY
-                                    ? new Intersection(treeInput, postingsInput, postingsSummaryInput, listener, context)
-                                    : new FilteringIntersection(treeInput, postingsInput, postingsSummaryInput, visitor, listener, context);
-
-        return intersection.execute();
-    }
-
-    /**
-     * Synchronous intersection of a point or point range with a block balanced tree previously written
-     * with {@link BlockBalancedTreeWriter}.
-     */
-    private class Intersection
-    {
-        private final Stopwatch queryExecutionTimer = Stopwatch.createStarted();
-        final QueryContext context;
-
-        final TraversalState state;
-        final IndexInput treeInput;
-        final IndexInput postingsInput;
-        final IndexInput postingsSummaryInput;
-        final QueryEventListener.BalancedTreeEventListener listener;
-        final PriorityQueue<PeekablePostingList> postingLists;
-
-        Intersection(IndexInput treeInput, IndexInput postingsInput, IndexInput postingsSummaryInput,
-                     QueryEventListener.BalancedTreeEventListener listener, QueryContext context)
-        {
-            this.state = newTraversalState();
-            this.treeInput = treeInput;
-            this.postingsInput = postingsInput;
-            this.postingsSummaryInput = postingsSummaryInput;
-            this.listener = listener;
-            this.context = context;
-            postingLists = new PriorityQueue<>(numLeaves, COMPARATOR);
-        }
-
-        public PostingList execute()
-        {
-            try
-            {
-                executeInternal();
-
-                FileUtils.closeQuietly(treeInput);
-
-                return mergePostings();
-            }
-            catch (Throwable t)
-            {
-                if (!(t instanceof QueryCancelledException))
-                    logger.error(indexIdentifier.logMessage("Balanced tree intersection failed on {}"), treeIndexFile.path(), t);
-
-                closeOnException();
-                throw Throwables.cleaned(t);
-            }
-        }
-
-        protected void executeInternal() throws IOException
-        {
-            collectPostingLists();
-        }
-
-        protected void closeOnException()
-        {
-            FileUtils.closeQuietly(treeInput);
-            FileUtils.closeQuietly(postingsInput);
-            FileUtils.closeQuietly(postingsSummaryInput);
-        }
-
-        protected PostingList mergePostings()
-        {
-            final long elapsedMicros = queryExecutionTimer.stop().elapsed(TimeUnit.MICROSECONDS);
-
-            listener.onIntersectionComplete(elapsedMicros, TimeUnit.MICROSECONDS);
-            listener.postingListsHit(postingLists.size());
-
-            if (postingLists.isEmpty())
-            {
-                FileUtils.closeQuietly(postingsInput);
-                FileUtils.closeQuietly(postingsSummaryInput);
-                return null;
-            }
-            else
-            {
-                if (logger.isTraceEnabled())
-                    logger.trace(indexIdentifier.logMessage("[{}] Intersection completed in {} microseconds. {} leaf and internal posting lists hit."),
-                                 treeIndexFile.path(), elapsedMicros, postingLists.size());
-                return MergePostingList.merge(postingLists, () -> FileUtils.close(postingsInput, postingsSummaryInput));
-            }
-        }
-
-        private void collectPostingLists() throws IOException
-        {
-            context.checkpoint();
-
-            // This will return true if the node is a child leaf that has postings or if there is postings for the
-            // entire subtree under a leaf
-            if (postingsIndex.exists(state.nodeID))
-            {
-                postingLists.add(initPostingReader(postingsIndex.getPostingsFilePointer(state.nodeID)));
-                return;
-            }
-
-            if (state.atLeafNode())
-                throw new CorruptIndexException(indexIdentifier.logMessage(String.format("Leaf node %s does not have balanced tree postings.", state.nodeID)), "");
-
-            // Recurse on left subtree:
-            state.pushLeft();
-            collectPostingLists();
-            state.pop();
-
-            // Recurse on right subtree:
-            state.pushRight();
-            collectPostingLists();
-            state.pop();
-        }
-
-        private PeekablePostingList initPostingReader(long offset) throws IOException
-        {
-            final PostingsReader.BlocksSummary summary = new PostingsReader.BlocksSummary(postingsSummaryInput, offset);
-            return PeekablePostingList.makePeekable(new PostingsReader(postingsInput, summary, listener.postingListEventListener()));
-        }
-    }
-
-    private class FilteringIntersection extends Intersection
-    {
-        private final IntersectVisitor visitor;
-        private final byte[] packedValue;
-        private final short[] origIndex;
-
-        FilteringIntersection(IndexInput treeInput, IndexInput postingsInput, IndexInput postingsSummaryInput,
-                              IntersectVisitor visitor, QueryEventListener.BalancedTreeEventListener listener, QueryContext context)
-        {
-            super(treeInput, postingsInput, postingsSummaryInput, listener, context);
-            this.visitor = visitor;
-            this.packedValue = new byte[bytesPerValue];
-            this.origIndex = new short[maxValuesInLeafNode];
-        }
-
-        @Override
-        public void executeInternal() throws IOException
-        {
-            collectPostingLists(minPackedValue, maxPackedValue);
-        }
-
-        private void collectPostingLists(byte[] minPackedValue, byte[] maxPackedValue) throws IOException
-        {
-            context.checkpoint();
-
-            final Relation r = visitor.compare(minPackedValue, maxPackedValue);
-
-            // This value range is fully outside the query shape: stop recursing
-            if (r == Relation.CELL_OUTSIDE_QUERY)
-                return;
-
-            if (r == Relation.CELL_INSIDE_QUERY)
-            {
-                // This value range is fully inside the query shape: recursively add all points from this node without filtering
-                super.collectPostingLists();
-                return;
-            }
-
-            if (state.atLeafNode())
-            {
-                if (state.nodeExists())
-                    filterLeaf();
-                return;
-            }
-
-            visitNode(minPackedValue, maxPackedValue);
-        }
-
-        private void filterLeaf() throws IOException
-        {
-            treeInput.seek(state.getLeafBlockFP());
-
-            int count = treeInput.readVInt();
-            int orderMapLength = treeInput.readVInt();
-            long orderMapPointer = treeInput.getFilePointer();
-
-            SeekingRandomAccessInput randomAccessInput = new SeekingRandomAccessInput(treeInput);
-            LongValues leafOrderMapReader = DirectReader.getInstance(randomAccessInput, leafOrderMapBitsRequired, orderMapPointer);
-            for (int index = 0; index < count; index++)
-            {
-                origIndex[index] = (short) Math.toIntExact(leafOrderMapReader.get(index));
-            }
-
-            // seek beyond the ordermap
-            treeInput.seek(orderMapPointer + orderMapLength);
-
-            FixedBitSet fixedBitSet = buildPostingsFilter(treeInput, count, visitor, origIndex);
-
-            if (postingsIndex.exists(state.nodeID) && fixedBitSet.cardinality() > 0)
-            {
-                long pointer = postingsIndex.getPostingsFilePointer(state.nodeID);
-                postingLists.add(initFilteringPostingReader(pointer, fixedBitSet));
-            }
-        }
-
-        void visitNode(byte[] minPackedValue, byte[] maxPackedValue) throws IOException
-        {
-            assert !state.atLeafNode() : "Cannot recurse down tree because nodeID " + state.nodeID + " is a leaf node";
-
-            byte[] splitValue = state.getSplitValue();
-
-            if (BlockBalancedTreeWriter.DEBUG)
-            {
-                // make sure cellMin <= splitValue <= cellMax:
-                assert ByteArrayUtil.compareUnsigned(minPackedValue, 0, splitValue, 0, bytesPerValue) <= 0 :"bytesPerValue=" + bytesPerValue;
-                assert ByteArrayUtil.compareUnsigned(maxPackedValue, 0, splitValue, 0, bytesPerValue) >= 0 : "bytesPerValue=" + bytesPerValue;
-            }
-
-            // Recurse on left subtree:
-            state.pushLeft();
-            collectPostingLists(minPackedValue, splitValue);
-            state.pop();
-
-            // Recurse on right subtree:
-            state.pushRight();
-            collectPostingLists(splitValue, maxPackedValue);
-            state.pop();
-        }
-
-        private PeekablePostingList initFilteringPostingReader(long offset, FixedBitSet filter) throws IOException
-        {
-            final PostingsReader.BlocksSummary summary = new PostingsReader.BlocksSummary(postingsSummaryInput, offset);
-            PostingsReader postingsReader = new PostingsReader(postingsInput, summary, listener.postingListEventListener());
-            return PeekablePostingList.makePeekable(new FilteringPostingList(filter, postingsReader));
-        }
-
-        private FixedBitSet buildPostingsFilter(IndexInput in, int count, IntersectVisitor visitor, short[] origIndex) throws IOException
-        {
-            int commonPrefixLength = readCommonPrefixLength(in);
-            return commonPrefixLength == bytesPerValue ? buildPostingsFilterForSingleValueLeaf(count, visitor, origIndex)
-                                                       : buildPostingsFilterForMultiValueLeaf(commonPrefixLength, in, count, visitor, origIndex);
-        }
-
-        private FixedBitSet buildPostingsFilterForMultiValueLeaf(int commonPrefixLength,
-                                                                 IndexInput in,
-                                                                 int count,
-                                                                 IntersectVisitor visitor,
-                                                                 short[] origIndex) throws IOException
-        {
-            // the byte at `compressedByteOffset` is compressed using run-length compression,
-            // other suffix bytes are stored verbatim
-            int compressedByteOffset = commonPrefixLength;
-            commonPrefixLength++;
-            int i;
-
-            FixedBitSet fixedBitSet = new FixedBitSet(maxValuesInLeafNode);
-
-            for (i = 0; i < count; )
-            {
-                packedValue[compressedByteOffset] = in.readByte();
-                final int runLen = Byte.toUnsignedInt(in.readByte());
-                for (int j = 0; j < runLen; ++j)
-                {
-                    in.readBytes(packedValue, commonPrefixLength, bytesPerValue - commonPrefixLength);
-                    final int rowIDIndex = origIndex[i + j];
-                    if (visitor.contains(packedValue))
-                        fixedBitSet.set(rowIDIndex);
-                }
-                i += runLen;
-            }
-            if (i != count)
-                throw new CorruptIndexException(String.format("Expected %d sub-blocks but read %d.", count, i), in);
-
-            return fixedBitSet;
-        }
-
-        private FixedBitSet buildPostingsFilterForSingleValueLeaf(int count, IntersectVisitor visitor, final short[] origIndex)
-        {
-            FixedBitSet fixedBitSet = new FixedBitSet(maxValuesInLeafNode);
-
-            // All the values in the leaf are the same, so we only
-            // need to visit once then set the bits for the relevant indexes
-            if (visitor.contains(packedValue))
-            {
-                for (int i = 0; i < count; ++i)
-                    fixedBitSet.set(origIndex[i]);
-            }
-            return fixedBitSet;
-        }
-
-        private int readCommonPrefixLength(IndexInput in) throws IOException
-        {
-            int prefixLength = in.readVInt();
-            if (prefixLength > 0)
-                in.readBytes(packedValue, 0, prefixLength);
-            return prefixLength;
-        }
-    }
-
-    /**
-     * We recurse the balanced tree, using a provided instance of this to guide the recursion.
-     */
-    public interface IntersectVisitor
-    {
-        /**
-         * Called for all values in a leaf cell that crosses the query.  The consumer should scrutinize the packedValue
-         * to decide whether to accept it. Values are visited in increasing order, and in the case of ties,
-         * in increasing order by segment row ID.
-         */
-        boolean contains(byte[] packedValue);
-
-        /**
-         * Called for non-leaf cells to test how the cell relates to the query, to
-         * determine how to further recurse down the tree.
-         */
-        Relation compare(byte[] minPackedValue, byte[] maxPackedValue);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeWalker.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeWalker.java
deleted file mode 100644
index 5a01b81f0971..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeWalker.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Arrays;
-
-import javax.annotation.concurrent.NotThreadSafe;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.agrona.collections.IntArrayList;
-import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.ByteArrayUtil;
-import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.BytesRef;
-
-/**
- * Base reader for a block balanced tree previously written with {@link BlockBalancedTreeWriter}.
- * <p>
- * Holds the index tree on heap and enables its traversal via {@link #traverse(TraversalCallback)}.
- */
-public class BlockBalancedTreeWalker implements Closeable
-{
-    final FileHandle treeIndexFile;
-    final int bytesPerValue;
-    final int numLeaves;
-    final int treeDepth;
-    final byte[] minPackedValue;
-    final byte[] maxPackedValue;
-    final long valueCount;
-    final int maxValuesInLeafNode;
-    final byte[] packedIndex;
-    final long memoryUsage;
-
-    BlockBalancedTreeWalker(FileHandle treeIndexFile, long treeIndexRoot)
-    {
-        this.treeIndexFile = treeIndexFile;
-
-        try (RandomAccessReader reader = treeIndexFile.createReader();
-             IndexInput indexInput = IndexInputReader.create(reader))
-        {
-            SAICodecUtils.validate(indexInput);
-            indexInput.seek(treeIndexRoot);
-
-            maxValuesInLeafNode = indexInput.readVInt();
-            bytesPerValue = indexInput.readVInt();
-
-            // Read index:
-            numLeaves = indexInput.readVInt();
-            assert numLeaves > 0;
-            treeDepth = indexInput.readVInt();
-            minPackedValue = new byte[bytesPerValue];
-            maxPackedValue = new byte[bytesPerValue];
-
-            indexInput.readBytes(minPackedValue, 0, bytesPerValue);
-            indexInput.readBytes(maxPackedValue, 0, bytesPerValue);
-
-            if (ByteArrayUtil.compareUnsigned(minPackedValue, 0, maxPackedValue, 0, bytesPerValue) > 0)
-            {
-                String message = String.format("Min packed value %s is > max packed value %s.",
-                                               new BytesRef(minPackedValue), new BytesRef(maxPackedValue));
-                throw new CorruptIndexException(message, indexInput);
-            }
-
-            valueCount = indexInput.readVLong();
-
-            int numBytes = indexInput.readVInt();
-            packedIndex = new byte[numBytes];
-            indexInput.readBytes(packedIndex, 0, numBytes);
-
-            memoryUsage = ObjectSizes.sizeOfArray(packedIndex) +
-                          ObjectSizes.sizeOfArray(minPackedValue) +
-                          ObjectSizes.sizeOfArray(maxPackedValue);
-        }
-        catch (Throwable t)
-        {
-            FileUtils.closeQuietly(treeIndexFile);
-            throw Throwables.unchecked(t);
-        }
-    }
-
-    @VisibleForTesting
-    public BlockBalancedTreeWalker(DataInput indexInput, long treeIndexRoot) throws IOException
-    {
-        treeIndexFile = null;
-
-        indexInput.skipBytes(treeIndexRoot);
-
-        maxValuesInLeafNode = indexInput.readVInt();
-        bytesPerValue = indexInput.readVInt();
-
-        // Read index:
-        numLeaves = indexInput.readVInt();
-        assert numLeaves > 0;
-        treeDepth = indexInput.readVInt();
-        minPackedValue = new byte[bytesPerValue];
-        maxPackedValue = new byte[bytesPerValue];
-
-        indexInput.readBytes(minPackedValue, 0, bytesPerValue);
-        indexInput.readBytes(maxPackedValue, 0, bytesPerValue);
-
-        if (ByteArrayUtil.compareUnsigned(minPackedValue, 0, maxPackedValue, 0, bytesPerValue) > 0)
-        {
-            String message = String.format("Min packed value %s is > max packed value %s.",
-                                           new BytesRef(minPackedValue), new BytesRef(maxPackedValue));
-            throw new CorruptIndexException(message, indexInput);
-        }
-
-        valueCount = indexInput.readVLong();
-
-        int numBytes = indexInput.readVInt();
-        packedIndex = new byte[numBytes];
-        indexInput.readBytes(packedIndex, 0, numBytes);
-
-        memoryUsage = ObjectSizes.sizeOfArray(packedIndex) +
-                      ObjectSizes.sizeOfArray(minPackedValue) +
-                      ObjectSizes.sizeOfArray(maxPackedValue);
-    }
-
-    public long memoryUsage()
-    {
-        return memoryUsage;
-    }
-
-    public TraversalState newTraversalState()
-    {
-        return new TraversalState();
-    }
-
-    @Override
-    public void close()
-    {
-        FileUtils.closeQuietly(treeIndexFile);
-    }
-
-    void traverse(TraversalCallback callback)
-    {
-        traverse(newTraversalState(), callback, new IntArrayList());
-    }
-
-    private void traverse(TraversalState state, TraversalCallback callback, IntArrayList pathToRoot)
-    {
-        if (state.atLeafNode())
-        {
-            // In the unbalanced case it's possible the left most node only has one child:
-            if (state.nodeExists())
-            {
-                callback.onLeaf(state.nodeID, state.getLeafBlockFP(), pathToRoot);
-            }
-        }
-        else
-        {
-            IntArrayList currentPath = new IntArrayList();
-            currentPath.addAll(pathToRoot);
-            currentPath.add(state.nodeID);
-
-            state.pushLeft();
-            traverse(state, callback, currentPath);
-            state.pop();
-
-            state.pushRight();
-            traverse(state, callback, currentPath);
-            state.pop();
-        }
-    }
-
-    interface TraversalCallback
-    {
-        void onLeaf(int leafNodeID, long leafBlockFP, IntArrayList pathToRoot);
-    }
-
-    /**
-     * This maintains the state for a traversal of the packed index. It is loaded once and can be resused
-     * by calling the reset method.
-     * <p>
-     * The packed index is a packed representation of a balanced tree and takes the form of a packed array of
-     * file pointer / split value pairs. Both the file pointers and split values are prefix compressed by tree level
-     * requiring us to maintain a stack of values for each level in the tree. The stack size is always the tree depth.
-     * <p>
-     * The tree is traversed by recursively following the left and then right subtrees under the current node. For the
-     * following tree (split values in square brackets):
-     * <pre>
-     *        1[16]
-     *       / \
-     *      /   \
-     *     2[8]  3[24]
-     *    / \   / \
-     *   4   5 6   7
-     * </pre>
-     * The traversal will be 1 -> 2 -> 4 -> 5 -> 3 -> 6 -> 7 with nodes 4, 5, 6 & 7 being leaf nodes.
-     * <p>
-     * Assuming the full range of values in the tree is 0 -> 32, the non-leaf nodes will represent the following
-     * values:
-     * <pre>
-     *         1[0-32]
-     *        /      \
-     *    2[0-16]   3[16-32]
-     * </pre>
-     */
-    @NotThreadSafe
-    final class TraversalState
-    {
-        // used to read the packed index byte[]
-        final ByteArrayDataInput dataInput;
-        // holds the minimum (left most) leaf block file pointer for each level we've recursed to:
-        final long[] leafBlockFPStack;
-        // holds the address, in the packed byte[] index, of the left-node of each level:
-        final int[] leftNodePositions;
-        // holds the address, in the packed byte[] index, of the right-node of each level:
-        final int[] rightNodePositions;
-        // holds the packed per-level split values; the run method uses this to save the cell min/max as it recurses:
-        final byte[][] splitValuesStack;
-
-        int nodeID;
-        int level;
-        @VisibleForTesting
-        int maxLevel;
-
-        private TraversalState()
-        {
-            nodeID = 1;
-            level = 0;
-            leafBlockFPStack = new long[treeDepth];
-            leftNodePositions = new int[treeDepth];
-            rightNodePositions = new int[treeDepth];
-            splitValuesStack = new byte[treeDepth][];
-            this.dataInput = new ByteArrayDataInput(packedIndex);
-            readNodeData(false);
-        }
-
-        public void pushLeft()
-        {
-            int nodePosition = leftNodePositions[level];
-            nodeID *= 2;
-            level++;
-            maxLevel = Math.max(maxLevel, level);
-            dataInput.setPosition(nodePosition);
-            readNodeData(true);
-        }
-
-        public void pushRight()
-        {
-            int nodePosition = rightNodePositions[level];
-            nodeID = nodeID * 2 + 1;
-            level++;
-            maxLevel = Math.max(maxLevel, level);
-            dataInput.setPosition(nodePosition);
-            readNodeData(false);
-        }
-
-        public void pop()
-        {
-            nodeID /= 2;
-            level--;
-        }
-
-        public boolean atLeafNode()
-        {
-            return nodeID >= numLeaves;
-        }
-
-        public boolean nodeExists()
-        {
-            return nodeID - numLeaves < numLeaves;
-        }
-
-        public long getLeafBlockFP()
-        {
-            return leafBlockFPStack[level];
-        }
-
-        public byte[] getSplitValue()
-        {
-            assert !atLeafNode();
-            return splitValuesStack[level];
-        }
-
-        private void readNodeData(boolean isLeft)
-        {
-            leafBlockFPStack[level] = level == 0 ? 0 : leafBlockFPStack[level - 1];
-
-            // read leaf block FP delta
-            if (!isLeft)
-                leafBlockFPStack[level] += dataInput.readVLong();
-
-            if (!atLeafNode())
-            {
-                // read prefix, firstDiffByteDelta encoded as int:
-                int code = dataInput.readVInt();
-                int prefix = code % (1 + bytesPerValue);
-                int suffix = bytesPerValue - prefix;
-
-                pushSplitValueStack();
-                if (suffix > 0)
-                {
-                    int firstDiffByteDelta = code / (1 + bytesPerValue);
-                    // If we are pushing to the left subtree then the delta will be negative
-                    if (isLeft)
-                        firstDiffByteDelta = -firstDiffByteDelta;
-                    int oldByte = splitValuesStack[level][prefix] & 0xFF;
-                    splitValuesStack[level][prefix] = (byte) (oldByte + firstDiffByteDelta);
-                    dataInput.readBytes(splitValuesStack[level], prefix + 1, suffix - 1);
-                }
-
-                int leftNumBytes = nodeID * 2 < numLeaves ? dataInput.readVInt() : 0;
-
-                leftNodePositions[level] = dataInput.getPosition();
-                rightNodePositions[level] = leftNodePositions[level] + leftNumBytes;
-            }
-        }
-
-        private void pushSplitValueStack()
-        {
-            if (splitValuesStack[level] == null)
-                splitValuesStack[level] = new byte[bytesPerValue];
-            if (level == 0)
-                Arrays.fill(splitValuesStack[level], (byte) 0);
-            else
-                System.arraycopy(splitValuesStack[level - 1], 0, splitValuesStack[level], 0, bytesPerValue);
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeWriter.java
deleted file mode 100644
index 0fa4180c0954..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeWriter.java
+++ /dev/null
@@ -1,767 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.annotation.concurrent.NotThreadSafe;
-
-import com.google.common.base.MoreObjects;
-
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.index.sai.disk.ResettableByteBuffersIndexOutput;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.utils.ByteArrayUtil;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IntroSorter;
-import org.apache.lucene.util.Sorter;
-import org.apache.lucene.util.bkd.BKDWriter;
-
-import static org.apache.cassandra.index.sai.postings.PostingList.END_OF_STREAM;
-
-/**
- * This is a specialisation of the Lucene {@link BKDWriter} that only writes a single dimension
- * balanced tree.
- * <p>
- * Recursively builds a block balanced tree to assign all incoming points to smaller
- * and smaller rectangles (cells) until the number of points in a given
- * rectangle is &lt;= <code>maxPointsInLeafNode</code>.  The tree is
- * fully balanced, which means the leaf nodes will have between 50% and 100% of
- * the requested <code>maxPointsInLeafNode</code>.  Values that fall exactly
- * on a cell boundary may be in either cell.
- * <p>
- * Visual representation of the disk format:
- * <pre>
- *
- * +========+=======================================+==================+========+
- * | HEADER | LEAF BLOCK LIST                       | BALANCED TREE    | FOOTER |
- * +========+================+=====+================+==================+========+
- *          | LEAF BLOCK (0) | ... | LEAF BLOCK (N) | VALUES PER LEAF  |
- *          +----------------+-----+----------------+------------------|
- *          | ORDER INDEX    |                      | BYTES PER VALUE  |
- *          +----------------+                      +------------------+
- *          | PREFIX         |                      | NUMBER OF LEAVES |
- *          +----------------+                      +------------------+
- *          | VALUES         |                      | MINIMUM VALUE    |
- *          +----------------+                      +------------------+
- *                                                  | MAXIMUM VALUE    |
- *                                                  +------------------+
- *                                                  | TOTAL VALUES     |
- *                                                  +------------------+
- *                                                  | INDEX TREE       |
- *                                                  +--------+---------+
- *                                                  | LENGTH | BYTES   |
- *                                                  +--------+---------+
- *  </pre>
- *
- * <p>
- * <b>NOTE</b>: This can write at most Integer.MAX_VALUE * <code>maxPointsInLeafNode</code> total points.
- * <p>
- * @see BKDWriter
- */
-@NotThreadSafe
-public class BlockBalancedTreeWriter
-{
-    // Enable to check that values are added to the tree in correct order and within bounds
-    public static final boolean DEBUG = CassandraRelevantProperties.SAI_TEST_BALANCED_TREE_DEBUG_ENABLED.getBoolean();
-
-    // Default maximum number of point in each leaf block
-    public static final int DEFAULT_MAX_POINTS_IN_LEAF_NODE = 1024;
-
-    private final int bytesPerValue;
-    private final int maxPointsInLeafNode;
-    private final byte[] minPackedValue;
-    private final byte[] maxPackedValue;
-    private long valueCount;
-
-    public BlockBalancedTreeWriter(int bytesPerValue, int maxPointsInLeafNode)
-    {
-        if (maxPointsInLeafNode <= 0)
-            throw new IllegalArgumentException("maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
-        if (maxPointsInLeafNode > ArrayUtil.MAX_ARRAY_LENGTH)
-            throw new IllegalArgumentException("maxPointsInLeafNode must be <= ArrayUtil.MAX_ARRAY_LENGTH (= " +
-                                               ArrayUtil.MAX_ARRAY_LENGTH + "); got " + maxPointsInLeafNode);
-
-        this.maxPointsInLeafNode = maxPointsInLeafNode;
-        this.bytesPerValue = bytesPerValue;
-
-        minPackedValue = new byte[bytesPerValue];
-        maxPackedValue = new byte[bytesPerValue];
-    }
-
-    public long getValueCount()
-    {
-        return valueCount;
-    }
-
-    public int getBytesPerValue()
-    {
-        return bytesPerValue;
-    }
-
-    public int getMaxPointsInLeafNode()
-    {
-        return maxPointsInLeafNode;
-    }
-
-    /**
-     * Write the sorted values from an {@link Iterator}.
-     * <p>
-     * @param treeOutput The {@link IndexOutput} to write the balanced tree to
-     * @param iterator An {@link Iterator} of {@link IndexEntry}s containing the terms and postings, sorted in term order
-     * @param callback The {@link Callback} used to record the leaf postings for each leaf
-     *
-     * @return The file pointer to the beginning of the balanced tree
-     */
-    public long write(IndexOutput treeOutput, Iterator<IndexEntry> iterator, final Callback callback) throws IOException
-    {
-        SAICodecUtils.writeHeader(treeOutput);
-
-        LeafWriter leafWriter = new LeafWriter(treeOutput, callback);
-
-        while (iterator.hasNext())
-        {
-            IndexEntry indexEntry = iterator.next();
-            long segmentRowId;
-            while ((segmentRowId = indexEntry.postingList.nextPosting()) != END_OF_STREAM)
-                leafWriter.add(indexEntry.term, segmentRowId);
-        }
-
-        valueCount = leafWriter.finish();
-
-        long treeFilePointer = valueCount == 0 ? -1 : treeOutput.getFilePointer();
-
-        // There is only any point in writing the balanced tree if any values were added
-        if (treeFilePointer >= 0)
-            writeBalancedTree(treeOutput, maxPointsInLeafNode, leafWriter.leafBlockStartValues, leafWriter.leafBlockFilePointers);
-
-        SAICodecUtils.writeFooter(treeOutput);
-
-        return treeFilePointer;
-    }
-
-    private void writeBalancedTree(IndexOutput out, int countPerLeaf, List<byte[]> leafBlockStartValues, List<Long> leafBlockFilePointer) throws IOException
-    {
-        int numInnerNodes = leafBlockStartValues.size();
-        byte[] splitValues = new byte[(1 + numInnerNodes) * bytesPerValue];
-        int treeDepth = recurseBalanceTree(1, 0, numInnerNodes, 1, splitValues, leafBlockStartValues);
-        long[] leafBlockFPs = leafBlockFilePointer.stream().mapToLong(l -> l).toArray();
-        byte[] packedIndex = packIndex(leafBlockFPs, splitValues);
-
-        out.writeVInt(countPerLeaf);
-        out.writeVInt(bytesPerValue);
-
-        out.writeVInt(leafBlockFPs.length);
-        out.writeVInt(Math.min(treeDepth, leafBlockFPs.length));
-
-        out.writeBytes(minPackedValue, 0, bytesPerValue);
-        out.writeBytes(maxPackedValue, 0, bytesPerValue);
-
-        out.writeVLong(valueCount);
-
-        out.writeVInt(packedIndex.length);
-        out.writeBytes(packedIndex, 0, packedIndex.length);
-    }
-
-    /**
-     * This can, potentially, be removed in the future by CASSANDRA-18597
-     */
-    private int recurseBalanceTree(int nodeID, int offset, int count, int treeDepth, byte[] splitValues, List<byte[]> leafBlockStartValues)
-    {
-        if (count == 1)
-        {
-            treeDepth++;
-            // Leaf index node
-            System.arraycopy(leafBlockStartValues.get(offset), 0, splitValues, nodeID * bytesPerValue, bytesPerValue);
-        }
-        else if (count > 1)
-        {
-            treeDepth++;
-            // Internal index node: binary partition of count
-            int countAtLevel = 1;
-            int totalCount = 0;
-            while (true)
-            {
-                int countLeft = count - totalCount;
-                if (countLeft <= countAtLevel)
-                {
-                    // This is the last level, possibly partially filled:
-                    int lastLeftCount = Math.min(countAtLevel / 2, countLeft);
-                    assert lastLeftCount >= 0;
-                    int leftHalf = (totalCount - 1) / 2 + lastLeftCount;
-
-                    int rootOffset = offset + leftHalf;
-
-                    System.arraycopy(leafBlockStartValues.get(rootOffset), 0, splitValues, nodeID * bytesPerValue, bytesPerValue);
-
-                    // TODO: we could optimize/specialize, when we know it's simply fully balanced binary tree
-                    // under here, to save this while loop on each recursion
-
-                    // Recurse left
-                    int leftTreeDepth = recurseBalanceTree(2 * nodeID, offset, leftHalf, treeDepth, splitValues, leafBlockStartValues);
-
-                    // Recurse right
-                    int rightTreeDepth = recurseBalanceTree(2 * nodeID + 1, rootOffset + 1, count - leftHalf - 1, treeDepth, splitValues, leafBlockStartValues);
-                    return Math.max(leftTreeDepth, rightTreeDepth);
-                }
-                totalCount += countAtLevel;
-                countAtLevel *= 2;
-            }
-        }
-        else
-        {
-            assert count == 0;
-        }
-        return treeDepth;
-    }
-
-    // Packs the two arrays, representing a balanced binary tree, into a compact byte[] structure.
-    private byte[] packIndex(long[] leafBlockFPs, byte[] splitValues) throws IOException
-    {
-        int numLeaves = leafBlockFPs.length;
-
-        // Possibly rotate the leaf block FPs, if the index is not a fully balanced binary tree (only happens
-        // if it was created by TreeWriter).  In this case the leaf nodes may straddle the two bottom
-        // levels of the binary tree:
-        if (numLeaves > 1)
-        {
-            int levelCount = 2;
-            while (true)
-            {
-                if (numLeaves >= levelCount && numLeaves <= 2 * levelCount)
-                {
-                    int lastLevel = 2 * (numLeaves - levelCount);
-                    assert lastLevel >= 0;
-                    if (lastLevel != 0)
-                    {
-                        // Last level is partially filled, so we must rotate the leaf FPs to match.  We do this here, after loading
-                        // at read-time, so that we can still delta code them on disk at write:
-                        long[] newLeafBlockFPs = new long[numLeaves];
-                        System.arraycopy(leafBlockFPs, lastLevel, newLeafBlockFPs, 0, leafBlockFPs.length - lastLevel);
-                        System.arraycopy(leafBlockFPs, 0, newLeafBlockFPs, leafBlockFPs.length - lastLevel, lastLevel);
-                        leafBlockFPs = newLeafBlockFPs;
-                    }
-                    break;
-                }
-
-                levelCount *= 2;
-            }
-        }
-
-        // Reused while packing the index
-        try (ResettableByteBuffersIndexOutput writeBuffer = new ResettableByteBuffersIndexOutput("PackedIndex"))
-        {
-            // This is the "file" we append the byte[] to:
-            List<byte[]> blocks = new ArrayList<>();
-            byte[] lastSplitValue = new byte[bytesPerValue];
-            int totalSize = recursePackIndex(writeBuffer, leafBlockFPs, splitValues, 0, blocks, 1, lastSplitValue, false);
-            // Compact the byte[] blocks into single byte index:
-            byte[] index = new byte[totalSize];
-            int upto = 0;
-            for (byte[] block : blocks)
-            {
-                System.arraycopy(block, 0, index, upto, block.length);
-                upto += block.length;
-            }
-            assert upto == totalSize;
-
-            return index;
-        }
-    }
-
-    /**
-     * lastSplitValue is the split value previously seen; we use this to prefix-code the split byte[] on each
-     * inner node
-     */
-    private int recursePackIndex(ResettableByteBuffersIndexOutput writeBuffer, long[] leafBlockFPs, byte[] splitValues,
-                                 long minBlockFP, List<byte[]> blocks, int nodeID, byte[] lastSplitValue, boolean isLeft) throws IOException
-    {
-        if (nodeID >= leafBlockFPs.length)
-        {
-            int leafID = nodeID - leafBlockFPs.length;
-
-            // In the unbalanced case it's possible the left most node only has one child:
-            if (leafID < leafBlockFPs.length)
-            {
-                long delta = leafBlockFPs[leafID] - minBlockFP;
-                if (isLeft)
-                {
-                    assert delta == 0;
-                    return 0;
-                }
-                else
-                {
-                    assert nodeID == 1 || delta > 0 : "nodeID=" + nodeID;
-                    writeBuffer.writeVLong(delta);
-                    return appendBlock(writeBuffer, blocks);
-                }
-            }
-            else
-            {
-                throw new IllegalStateException("Unbalanced tree");
-            }
-        }
-        else
-        {
-            long leftBlockFP;
-            if (!isLeft)
-            {
-                leftBlockFP = getLeftMostLeafBlockFP(leafBlockFPs, nodeID);
-                long delta = leftBlockFP - minBlockFP;
-                assert nodeID == 1 || delta > 0;
-                writeBuffer.writeVLong(delta);
-            }
-            else
-            {
-                // The left tree's left most leaf block FP is always the minimal FP:
-                leftBlockFP = minBlockFP;
-            }
-
-            int address = nodeID * bytesPerValue;
-
-            // find common prefix with last split value in this dim:
-            int prefix = 0;
-            for (; prefix < bytesPerValue; prefix++)
-            {
-                if (splitValues[address + prefix] != lastSplitValue[prefix])
-                {
-                    break;
-                }
-            }
-
-            int firstDiffByteDelta;
-            if (prefix < bytesPerValue)
-            {
-                firstDiffByteDelta = (splitValues[address + prefix] & 0xFF) - (lastSplitValue[prefix] & 0xFF);
-                // If this is left then we need to negate the delta
-                if (isLeft)
-                    firstDiffByteDelta = -firstDiffByteDelta;
-                assert firstDiffByteDelta > 0;
-            }
-            else
-            {
-                firstDiffByteDelta = 0;
-            }
-
-            // pack the prefix and delta first diff byte into a single vInt:
-            int code = (firstDiffByteDelta * (1 + bytesPerValue) + prefix);
-
-            writeBuffer.writeVInt(code);
-
-            // write the split value, prefix coded vs. our parent's split value:
-            int suffix = bytesPerValue - prefix;
-            byte[] savSplitValue = new byte[suffix];
-            if (suffix > 1)
-            {
-                writeBuffer.writeBytes(splitValues, address + prefix + 1, suffix - 1);
-            }
-
-            byte[] cmp = lastSplitValue.clone();
-
-            System.arraycopy(lastSplitValue, prefix, savSplitValue, 0, suffix);
-
-            // copy our split value into lastSplitValue for our children to prefix-code against
-            System.arraycopy(splitValues, address + prefix, lastSplitValue, prefix, suffix);
-
-            int numBytes = appendBlock(writeBuffer, blocks);
-
-            // placeholder for left-tree numBytes; we need this so that at search time if we only need to recurse into
-            // the right subtree we can quickly seek to its starting point
-            int idxSav = blocks.size();
-            blocks.add(null);
-
-            int leftNumBytes = recursePackIndex(writeBuffer, leafBlockFPs, splitValues, leftBlockFP, blocks, 2 * nodeID, lastSplitValue, true);
-
-            if (nodeID * 2 < leafBlockFPs.length)
-            {
-                writeBuffer.writeVInt(leftNumBytes);
-            }
-            else
-            {
-                assert leftNumBytes == 0 : "leftNumBytes=" + leftNumBytes;
-            }
-            int numBytes2 = Math.toIntExact(writeBuffer.getFilePointer());
-            byte[] bytes2 = writeBuffer.toArrayCopy();
-            writeBuffer.reset();
-            // replace our placeholder:
-            blocks.set(idxSav, bytes2);
-
-            int rightNumBytes = recursePackIndex(writeBuffer, leafBlockFPs, splitValues, leftBlockFP, blocks, 2 * nodeID + 1, lastSplitValue, false);
-
-            // restore lastSplitValue to what caller originally passed us:
-            System.arraycopy(savSplitValue, 0, lastSplitValue, prefix, suffix);
-
-            assert Arrays.equals(lastSplitValue, cmp);
-
-            return numBytes + numBytes2 + leftNumBytes + rightNumBytes;
-        }
-    }
-
-    /** Appends the current contents of writeBuffer as another block on the growing in-memory file */
-    private int appendBlock(ResettableByteBuffersIndexOutput writeBuffer, List<byte[]> blocks)
-    {
-        int pos = Math.toIntExact(writeBuffer.getFilePointer());
-        byte[] bytes = writeBuffer.toArrayCopy();
-        writeBuffer.reset();
-        blocks.add(bytes);
-        return pos;
-    }
-
-    private long getLeftMostLeafBlockFP(long[] leafBlockFPs, int nodeID)
-    {
-        // TODO: can we do this cheaper, e.g. a closed form solution instead of while loop?  Or
-        // change the recursion while packing the index to return this left-most leaf block FP
-        // from each recursion instead?
-        //
-        // Still, the overall cost here is minor: this method's cost is O(log(N)), and while writing
-        // we call it O(N) times (N = number of leaf blocks)
-        while (nodeID < leafBlockFPs.length)
-        {
-            nodeID *= 2;
-        }
-        int leafID = nodeID - leafBlockFPs.length;
-        long result = leafBlockFPs[leafID];
-        if (result < 0)
-        {
-            throw new AssertionError(result + " for leaf " + leafID);
-        }
-        return result;
-    }
-
-    interface Callback
-    {
-        void writeLeafPostings(RowIDAndIndex[] leafPostings, int offset, int count);
-    }
-
-    static class RowIDAndIndex
-    {
-        public int valueOrderIndex;
-        public long rowID;
-
-        @Override
-        public String toString()
-        {
-            return MoreObjects.toStringHelper(this)
-                              .add("valueOrderIndex", valueOrderIndex)
-                              .add("rowID", rowID)
-                              .toString();
-        }
-    }
-
-    /**
-     * Responsible for writing the leaf blocks at the beginning of the balanced tree index.
-     */
-    private class LeafWriter
-    {
-        private final IndexOutput treeOutput;
-        private final List<Long> leafBlockFilePointers = new ArrayList<>();
-        private final List<byte[]> leafBlockStartValues = new ArrayList<>();
-        private final byte[] leafValues = new byte[maxPointsInLeafNode * bytesPerValue];
-        private final long[] leafRowIDs = new long[maxPointsInLeafNode];
-        private final RowIDAndIndex[] rowIDAndIndexes = new RowIDAndIndex[maxPointsInLeafNode];
-        private final int[] orderIndex = new int[maxPointsInLeafNode];
-        private final Callback callback;
-        private final ByteBuffersDataOutput leafOrderIndexOutput = new ByteBuffersDataOutput(2 * 1024);
-        private final ByteBuffersDataOutput leafBlockOutput = new ByteBuffersDataOutput(32 * 1024);
-        private final byte[] packedValue = new byte[bytesPerValue];
-        private final byte[] lastPackedValue = new byte[bytesPerValue];
-
-        private long valueCount;
-        private int leafValueCount;
-        private long lastRowID;
-
-        LeafWriter(IndexOutput treeOutput, Callback callback)
-        {
-            assert callback != null : "Callback cannot be null in TreeWriter";
-
-            this.treeOutput = treeOutput;
-            this.callback = callback;
-
-            for (int x = 0; x < rowIDAndIndexes.length; x++)
-            {
-                rowIDAndIndexes[x] = new RowIDAndIndex();
-            }
-        }
-
-        /**
-         * Adds a value and row ID to the current leaf block. If the leaf block is full after the addition
-         * the current leaf block is written to disk.
-         */
-        void add(ByteComparable value, long rowID) throws IOException
-        {
-            ByteSourceInverse.copyBytes(value.asComparableBytes(ByteComparable.Version.OSS50), packedValue);
-
-            if (DEBUG)
-                valueInOrder(valueCount + leafValueCount, lastPackedValue, packedValue, 0, rowID, lastRowID);
-
-            System.arraycopy(packedValue, 0, leafValues, leafValueCount * bytesPerValue, bytesPerValue);
-            leafRowIDs[leafValueCount] = rowID;
-            leafValueCount++;
-
-            if (leafValueCount == maxPointsInLeafNode)
-            {
-                // We write a block once we hit exactly the max count
-                writeLeafBlock();
-                leafValueCount = 0;
-            }
-
-            if (DEBUG)
-                if ((lastRowID = rowID) < 0)
-                    throw new AssertionError("row id must be >= 0; got " + rowID);
-        }
-
-        /**
-         * Write a leaf block if we have unwritten values and return the total number of values added
-         */
-        public long finish() throws IOException
-        {
-            if (leafValueCount > 0)
-                writeLeafBlock();
-
-            return valueCount;
-        }
-
-        private void writeLeafBlock() throws IOException
-        {
-            assert leafValueCount != 0;
-            if (valueCount == 0)
-            {
-                System.arraycopy(leafValues, 0, minPackedValue, 0, bytesPerValue);
-            }
-            System.arraycopy(leafValues, (leafValueCount - 1) * bytesPerValue, maxPackedValue, 0, bytesPerValue);
-
-            valueCount += leafValueCount;
-
-            if (leafBlockFilePointers.size() > 0)
-            {
-                // Save the first (minimum) value in each leaf block except the first, to build the split value index in the end:
-                leafBlockStartValues.add(ArrayUtil.copyOfSubArray(leafValues, 0, bytesPerValue));
-            }
-            leafBlockFilePointers.add(treeOutput.getFilePointer());
-            checkMaxLeafNodeCount(leafBlockFilePointers.size());
-
-            // Find the common prefix between the first and last values in the block
-            int commonPrefixLength = bytesPerValue;
-            int offset = (leafValueCount - 1) * bytesPerValue;
-            for (int j = 0; j < bytesPerValue; j++)
-            {
-                if (leafValues[j] != leafValues[offset + j])
-                {
-                    commonPrefixLength = j;
-                    break;
-                }
-            }
-
-            treeOutput.writeVInt(leafValueCount);
-
-            for (int x = 0; x < leafValueCount; x++)
-            {
-                rowIDAndIndexes[x].valueOrderIndex = x;
-                rowIDAndIndexes[x].rowID = leafRowIDs[x];
-            }
-
-            final Sorter sorter = new IntroSorter()
-            {
-                RowIDAndIndex pivot;
-
-                @Override
-                protected void swap(int i, int j)
-                {
-                    RowIDAndIndex o = rowIDAndIndexes[i];
-                    rowIDAndIndexes[i] = rowIDAndIndexes[j];
-                    rowIDAndIndexes[j] = o;
-                }
-
-                @Override
-                protected void setPivot(int i)
-                {
-                    pivot = rowIDAndIndexes[i];
-                }
-
-                @Override
-                protected int comparePivot(int j)
-                {
-                    return Long.compare(pivot.rowID, rowIDAndIndexes[j].rowID);
-                }
-            };
-
-            sorter.sort(0, leafValueCount);
-
-            // write the leaf order index: leaf rowID -> orig index
-            leafOrderIndexOutput.reset();
-
-            // iterate in row ID order to get the row ID index for the given value order index
-            // place into an array to be written as packed ints
-            for (int x = 0; x < leafValueCount; x++)
-                orderIndex[rowIDAndIndexes[x].valueOrderIndex] = x;
-
-            LeafOrderMap.write(orderIndex, leafValueCount, maxPointsInLeafNode - 1, leafOrderIndexOutput);
-
-            treeOutput.writeVInt((int) leafOrderIndexOutput.size());
-            leafOrderIndexOutput.copyTo(treeOutput);
-
-            callback.writeLeafPostings(rowIDAndIndexes, 0, leafValueCount);
-
-            // Write the common prefix for the leaf block
-            writeCommonPrefix(treeOutput, commonPrefixLength);
-
-            // Write the run length encoded packed values for the leaf block
-            leafBlockOutput.reset();
-
-            if (DEBUG)
-                valuesInOrderAndBounds(leafValueCount,
-                                       ArrayUtil.copyOfSubArray(leafValues, 0, bytesPerValue),
-                                       ArrayUtil.copyOfSubArray(leafValues, (leafValueCount - 1) * bytesPerValue, leafValueCount * bytesPerValue),
-                                       leafRowIDs);
-
-            writeLeafBlockPackedValues(leafBlockOutput, commonPrefixLength, leafValueCount);
-
-            leafBlockOutput.copyTo(treeOutput);
-        }
-
-        private void checkMaxLeafNodeCount(int numLeaves)
-        {
-            if (bytesPerValue * (long) numLeaves > ArrayUtil.MAX_ARRAY_LENGTH)
-            {
-                throw new IllegalStateException("too many nodes; increase maxPointsInLeafNode (currently " + maxPointsInLeafNode + ") and reindex");
-            }
-        }
-
-        private void writeCommonPrefix(DataOutput treeOutput, int commonPrefixLength) throws IOException
-        {
-            treeOutput.writeVInt(commonPrefixLength);
-            if (commonPrefixLength > 0)
-                treeOutput.writeBytes(leafValues, 0, commonPrefixLength);
-        }
-
-        private void writeLeafBlockPackedValues(DataOutput out, int commonPrefixLength, int count) throws IOException
-        {
-            // If all the values are the same (e.g. the common prefix length == bytes per value) then we don't
-            // need to write anything. Otherwise, we run length compress the values to disk.
-            if (commonPrefixLength != bytesPerValue)
-            {
-                int compressedByteOffset = commonPrefixLength;
-                commonPrefixLength++;
-                for (int i = 0; i < count; )
-                {
-                    // do run-length compression on the byte at compressedByteOffset
-                    int runLen = runLen(i, Math.min(i + 0xff, count), compressedByteOffset);
-                    assert runLen <= 0xff;
-                    byte prefixByte = leafValues[i * bytesPerValue + compressedByteOffset];
-                    out.writeByte(prefixByte);
-                    out.writeByte((byte) runLen);
-                    writeLeafBlockPackedValuesRange(out, commonPrefixLength, i, i + runLen);
-                    i += runLen;
-                    assert i <= count;
-                }
-            }
-        }
-
-        private void writeLeafBlockPackedValuesRange(DataOutput out, int commonPrefixLength, int start, int end) throws IOException
-        {
-            for (int i = start; i < end; ++i)
-            {
-                out.writeBytes(leafValues, i * bytesPerValue + commonPrefixLength, bytesPerValue - commonPrefixLength);
-            }
-        }
-
-        private int runLen(int start, int end, int byteOffset)
-        {
-            byte b = leafValues[start * bytesPerValue + byteOffset];
-            for (int i = start + 1; i < end; ++i)
-            {
-                byte b2 = leafValues[i * bytesPerValue + byteOffset];
-                assert Byte.toUnsignedInt(b2) >= Byte.toUnsignedInt(b);
-                if (b != b2)
-                {
-                    return i - start;
-                }
-            }
-            return end - start;
-        }
-
-        // The following 3 methods are only used when DEBUG is true:
-
-        private void valueInBounds(byte[] packedValues, int packedValueOffset, byte[] minPackedValue, byte[] maxPackedValue)
-        {
-            if (ByteArrayUtil.compareUnsigned(packedValues,
-                                              packedValueOffset,
-                                              minPackedValue,
-                                              0,
-                                              bytesPerValue) < 0)
-            {
-                throw new AssertionError("value=" + new BytesRef(packedValues, packedValueOffset, bytesPerValue) +
-                                         " is < minPackedValue=" + new BytesRef(minPackedValue));
-            }
-
-            if (ByteArrayUtil.compareUnsigned(packedValues,
-                                              packedValueOffset,
-                                              maxPackedValue, 0,
-                                              bytesPerValue) > 0)
-            {
-                throw new AssertionError("value=" + new BytesRef(packedValues, packedValueOffset, bytesPerValue) +
-                                         " is > maxPackedValue=" + new BytesRef(maxPackedValue));
-            }
-        }
-
-        private void valuesInOrderAndBounds(int count, byte[] minPackedValue, byte[] maxPackedValue, long[] rowIds)
-        {
-            byte[] lastPackedValue = new byte[bytesPerValue];
-            long lastRowId = -1;
-            for (int i = 0; i < count; i++)
-            {
-                valueInOrder(i, lastPackedValue, leafValues, i * bytesPerValue, rowIds[i], lastRowId);
-                lastRowId = rowIds[i];
-
-                // Make sure this value does in fact fall within this leaf cell:
-                valueInBounds(leafValues, i * bytesPerValue, minPackedValue, maxPackedValue);
-            }
-        }
-
-        private void valueInOrder(long ord, byte[] lastPackedValue, byte[] packedValues, int packedValueOffset, long rowId, long lastRowId)
-        {
-            if (ord > 0)
-            {
-                int cmp = ByteArrayUtil.compareUnsigned(lastPackedValue, 0, packedValues, packedValueOffset, bytesPerValue);
-                if (cmp > 0)
-                {
-                    throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) +
-                                             " current value=" + new BytesRef(packedValues, packedValueOffset, bytesPerValue) +
-                                             " ord=" + ord);
-                }
-                if (cmp == 0 && rowId < lastRowId)
-                {
-                    throw new AssertionError("row IDs out of order: last rowID=" + lastRowId + " current rowID=" + rowId + " ord=" + ord);
-                }
-            }
-            System.arraycopy(packedValues, packedValueOffset, lastPackedValue, 0, bytesPerValue);
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/LeafOrderMap.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/LeafOrderMap.java
deleted file mode 100644
index 8fd5bf3dc2b7..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/LeafOrderMap.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.IOException;
-
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.packed.DirectWriter;
-
-class LeafOrderMap
-{
-    static void write(final int[] array, int length, int maxValue, final DataOutput out) throws IOException
-    {
-        final int bits = DirectWriter.unsignedBitsRequired(maxValue);
-        final DirectWriter writer = DirectWriter.getInstance(out, length, bits);
-        for (int i = 0; i < length; i++)
-        {
-            assert array[i] <= maxValue;
-
-            writer.add(array[i]);
-        }
-        writer.finish();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriter.java
deleted file mode 100644
index aedf64be1f71..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriter.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.MoreObjects;
-
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentWriter;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.packed.PackedInts;
-import org.apache.lucene.util.packed.PackedLongValues;
-
-/**
- * Specialized writer for values, that builds them into a {@link BlockBalancedTreeWriter} with auxiliary
- * posting lists on eligible tree levels.
- * <p>
- * Given a sorted input, the flush process is optimised because we don't need to buffer all point values to sort them.
- */
-public class NumericIndexWriter implements SegmentWriter
-{
-    public static final int MAX_POINTS_IN_LEAF_NODE = BlockBalancedTreeWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE;
-    private static final int DEFAULT_POSTINGS_SIZE = 128;
-
-    private final BlockBalancedTreeWriter writer;
-    private final IndexDescriptor indexDescriptor;
-    private final IndexIdentifier indexIdentifier;
-    private final int bytesPerValue;
-
-    public NumericIndexWriter(IndexDescriptor indexDescriptor,
-                              IndexIdentifier indexIdentifier,
-                              int bytesPerValue)
-    {
-        this(indexDescriptor, indexIdentifier, MAX_POINTS_IN_LEAF_NODE, bytesPerValue);
-    }
-
-    @VisibleForTesting
-    public NumericIndexWriter(IndexDescriptor indexDescriptor,
-                              IndexIdentifier indexIdentifier,
-                              int maxPointsInLeafNode,
-                              int bytesPerValue)
-    {
-        this.indexDescriptor = indexDescriptor;
-        this.indexIdentifier = indexIdentifier;
-        this.bytesPerValue = bytesPerValue;
-        this.writer = new BlockBalancedTreeWriter(bytesPerValue, maxPointsInLeafNode);
-    }
-
-    @Override
-    public String toString()
-    {
-        return MoreObjects.toStringHelper(this).add("indexName", indexIdentifier).add("bytesPerValue", bytesPerValue).toString();
-    }
-
-    private static class LeafCallback implements BlockBalancedTreeWriter.Callback
-    {
-        final List<PackedLongValues> leafPostings = new ArrayList<>(DEFAULT_POSTINGS_SIZE);
-
-        public int numLeaves()
-        {
-            return leafPostings.size();
-        }
-
-        @Override
-        public void writeLeafPostings(BlockBalancedTreeWriter.RowIDAndIndex[] leafPostings, int offset, int count)
-        {
-            PackedLongValues.Builder builder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
-
-            for (int i = offset; i < count; ++i)
-            {
-                builder.add(leafPostings[i].rowID);
-            }
-            this.leafPostings.add(builder.build());
-        }
-    }
-
-    @Override
-    public SegmentMetadata.ComponentMetadataMap writeCompleteSegment(Iterator<IndexEntry> iterator) throws IOException
-    {
-        long treePosition;
-
-        SegmentMetadata.ComponentMetadataMap components = new SegmentMetadata.ComponentMetadataMap();
-
-        LeafCallback leafCallback = new LeafCallback();
-
-        try (IndexOutput treeOutput = indexDescriptor.openPerIndexOutput(IndexComponent.BALANCED_TREE, indexIdentifier, true))
-        {
-            // The SSTable balanced tree component file is opened in append mode, so our offset is the current file pointer.
-            long treeOffset = treeOutput.getFilePointer();
-
-            treePosition = writer.write(treeOutput, iterator, leafCallback);
-
-            // If the treePosition is less than 0 then we didn't write any values out and the index is empty
-            if (treePosition < 0)
-                return components;
-
-            long treeLength = treeOutput.getFilePointer() - treeOffset;
-
-            Map<String, String> attributes = new LinkedHashMap<>();
-            attributes.put("max_points_in_leaf_node", Integer.toString(writer.getMaxPointsInLeafNode()));
-            attributes.put("num_leaves", Integer.toString(leafCallback.numLeaves()));
-            attributes.put("num_values", Long.toString(writer.getValueCount()));
-            attributes.put("bytes_per_value", Long.toString(writer.getBytesPerValue()));
-
-            components.put(IndexComponent.BALANCED_TREE, treePosition, treeOffset, treeLength, attributes);
-        }
-
-        try (BlockBalancedTreeWalker reader = new BlockBalancedTreeWalker(indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE,
-                                                                                                                   indexIdentifier,
-                                                                                                                   null),
-                                                                          treePosition);
-             IndexOutputWriter postingsOutput = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier, true))
-        {
-            long postingsOffset = postingsOutput.getFilePointer();
-
-            BlockBalancedTreePostingsWriter postingsWriter = new BlockBalancedTreePostingsWriter();
-            reader.traverse(postingsWriter);
-
-            // The balanced tree postings writer already writes its own header & footer.
-            long postingsPosition = postingsWriter.finish(postingsOutput, leafCallback.leafPostings, indexIdentifier);
-
-            Map<String, String> attributes = new LinkedHashMap<>();
-            attributes.put("num_leaf_postings", Integer.toString(postingsWriter.numLeafPostings));
-            attributes.put("num_non_leaf_postings", Integer.toString(postingsWriter.numNonLeafPostings));
-
-            long postingsLength = postingsOutput.getFilePointer() - postingsOffset;
-            components.put(IndexComponent.POSTING_LISTS, postingsPosition, postingsOffset, postingsLength, attributes);
-        }
-
-        return components;
-    }
-
-    @Override
-    public long getNumberOfRows()
-    {
-        return writer.getValueCount();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedReader.java
index 33f23acd15dd..a8bb3b0c14a7 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedReader.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedReader.java
@@ -17,75 +17,99 @@
  */
 package org.apache.cassandra.index.sai.disk.v1.bitpack;
 
-import javax.annotation.concurrent.NotThreadSafe;
-
-import org.apache.cassandra.index.sai.disk.io.SeekingRandomAccessInput;
+import com.carrotsearch.hppc.IntObjectHashMap;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
 import org.apache.cassandra.index.sai.disk.v1.LongArray;
-import org.apache.lucene.store.IndexInput;
+import org.apache.cassandra.index.sai.utils.SeekingRandomAccessInput;
 import org.apache.lucene.util.LongValues;
-import org.apache.lucene.util.packed.DirectReader;
 
-@NotThreadSafe
 public abstract class AbstractBlockPackedReader implements LongArray
 {
     private final int blockShift;
     private final int blockMask;
+    private final int blockSize;
     private final long valueCount;
-    private final byte[] blockBitsPerValue;
+    final byte[] blockBitsPerValue; // package protected for test access
     private final SeekingRandomAccessInput input;
+    private final IntObjectHashMap<LongValues> readers;
 
-    private long previousValue = Long.MIN_VALUE;
+    private long prevTokenValue = Long.MIN_VALUE;
     private long lastIndex; // the last index visited by token -> row ID searches
 
-    AbstractBlockPackedReader(IndexInput indexInput, byte[] blockBitsPerValue, int blockShift, int blockMask, long valueCount)
+    AbstractBlockPackedReader(IndexInput indexInput, byte[] blockBitsPerValue, int blockShift, int blockMask, long sstableRowId, long valueCount)
     {
         this.blockShift = blockShift;
         this.blockMask = blockMask;
+        this.blockSize = blockMask + 1;
         this.valueCount = valueCount;
         this.input = new SeekingRandomAccessInput(indexInput);
         this.blockBitsPerValue = blockBitsPerValue;
+        this.readers = new IntObjectHashMap<>();
+        // start searching tokens from current index segment
+        this.lastIndex = sstableRowId;
     }
 
     protected abstract long blockOffsetAt(int block);
 
     @Override
-    public long get(final long valueIndex)
+    public long get(final long index)
     {
-        if (valueIndex < 0 || valueIndex >= valueCount)
+        if (index < 0 || index >= valueCount)
         {
-            throw new IndexOutOfBoundsException(String.format("Index should be between [0, %d), but was %d.", valueCount, valueIndex));
+            throw new IndexOutOfBoundsException(String.format("Index should be between [0, %d), but was %d.", valueCount, index));
         }
 
-        int blockIndex = (int) (valueIndex >>> blockShift);
-        int inBlockIndex = (int) (valueIndex & blockMask);
-        byte bitsPerValue = blockBitsPerValue[blockIndex];
-        final LongValues subReader = bitsPerValue == 0 ? LongValues.ZEROES
-                                                       : DirectReader.getInstance(input, bitsPerValue, blockOffsetAt(blockIndex));
-        return delta(blockIndex, inBlockIndex) + subReader.get(inBlockIndex);
+        final int block = (int) (index >>> blockShift);
+        final int idx = (int) (index & blockMask);
+        return delta(block, idx) + getReader(block).get(idx);
     }
 
-    @Override
-    public long length()
+    private LongValues getReader(int block)
     {
-        return valueCount;
+        LongValues reader = readers.get(block);
+        if (reader == null)
+        {
+            reader = blockBitsPerValue[block] == 0 ? LongValues.ZEROES
+                                                   : LuceneCompat.directReaderGetInstance(input, blockBitsPerValue[block], blockOffsetAt(block));
+            readers.put(block, reader);
+        }
+        return reader;
     }
 
     @Override
-    public long indexOf(long value)
+    public long ceilingRowId(long targetValue)
     {
-        // If we are searching backwards, we need to reset the lastIndex. This is not normal since we normally move
-        // forwards when searching for tokens. We only (may) search backwards in vector searchs where we need the
-        // primary key ranges presented as row IDs.
-        if (value < previousValue)
-            lastIndex = 0;
-
         // already out of range
         if (lastIndex >= valueCount)
             return -1;
 
-        previousValue = value;
+        long rowId = findBlockRowId(targetValue);
+        lastIndex = rowId >= 0 ? rowId : -rowId - 1;
+        return lastIndex >= valueCount ? -1 : lastIndex;
+    }
 
-        int blockIndex = binarySearchBlockMinValues(value);
+    @Override
+    public long indexOf(long targetValue)
+    {
+        // already out of range
+        if (lastIndex >= valueCount)
+            return Long.MIN_VALUE;
+
+        long rowId = findBlockRowId(targetValue);
+        lastIndex = rowId >= 0 ? rowId : -rowId - 1;
+        return rowId >= valueCount ? Long.MIN_VALUE : rowId;
+    }
+
+    private long findBlockRowId(long targetValue)
+    {
+        // We keep track previous returned value in lastIndex, so searching backward will not return correct result.
+        // Also it's logically wrong to search backward during token iteration in PostingListKeyRangeIterator.
+        if (targetValue < prevTokenValue)
+            throw new IllegalArgumentException(String.format("%d is smaller than prev token value %d", targetValue, prevTokenValue));
+        prevTokenValue = targetValue;
+
+        int blockIndex = binarySearchBlockMinValues(targetValue);
 
         // We need to check next block's min value on an exact match.
         boolean exactMatch = blockIndex >= 0;
@@ -107,8 +131,7 @@ public long indexOf(long value)
         }
 
         // Find the global (not block-specific) index of the target token, which is equivalent to its row ID:
-        lastIndex = findBlockRowID(value, blockIndex, exactMatch);
-        return lastIndex >= valueCount ? -1 : lastIndex;
+        return findBlockRowID(targetValue, blockIndex, exactMatch);
     }
 
     /**
@@ -187,7 +210,7 @@ private long findBlockRowID(long targetValue, long blockIdx, boolean exactMatch)
         long low = Math.max(lastIndex, offset);
 
         // The high is either the last local index in the block, or something smaller if the block isn't full:
-        long high = Math.min(offset + blockMask + (exactMatch ? 1 : 0), valueCount - 1);
+        long high = Math.min(offset + blockSize - 1 + (exactMatch ? 1 : 0), valueCount - 1);
 
         return binarySearchBlock(targetValue, low, high);
     }
@@ -195,7 +218,7 @@ private long findBlockRowID(long targetValue, long blockIdx, boolean exactMatch)
     /**
      * binary search target value between low and high.
      *
-     * @return index if exact match is found, or *positive* insertion point if no exact match is found.
+     * @return index if exact match is found, or `-(insertion point) - 1` if no exact match is found.
      */
     private long binarySearchBlock(long target, long low, long high)
     {
@@ -232,7 +255,13 @@ else if (midVal > target)
         }
 
         // target not found
-        return low;
+        return -(low + 1);
+    }
+
+    @Override
+    public long length()
+    {
+        return valueCount;
     }
 
     abstract long delta(int block, int idx);
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedWriter.java
index 767217d88f01..01888b18cda8 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedWriter.java
@@ -19,11 +19,13 @@
 
 import java.io.IOException;
 
-import org.apache.cassandra.index.sai.disk.ResettableByteBuffersIndexOutput;
-import org.apache.lucene.store.IndexOutput;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.oldlucene.DirectWriterAdapter;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
+import org.apache.cassandra.index.sai.disk.oldlucene.ResettableByteBuffersIndexOutput;
 import org.apache.lucene.util.packed.DirectWriter;
 
-import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.checkBlockSize;
+import static org.apache.cassandra.index.sai.utils.SAICodecUtils.checkBlockSize;
 
 /**
  * Modified copy of {@code org.apache.lucene.util.packed.AbstractBlockPackedWriter} to use {@link DirectWriter} for
@@ -33,25 +35,31 @@ public abstract class AbstractBlockPackedWriter
 {
     static final int MIN_BLOCK_SIZE = 64;
     static final int MAX_BLOCK_SIZE = 1 << (30 - 3);
+    static final int MIN_VALUE_EQUALS_0 = 1;
+    static final int BPV_SHIFT = 1;
 
-    protected final IndexOutput indexOutput;
-    protected final long[] blockValues;
-    // This collects metadata specific to the block packed writer being used during the
-    // writing of the block packed data. This cached metadata is then written to the end
-    // of the data file when the block packed writer is finished.
-    protected final ResettableByteBuffersIndexOutput blockMetaWriter;
-
-    protected int blockIndex;
+    protected final IndexOutput out;
+    protected final long[] values;
+    protected int off;
     protected boolean finished;
+    
+    final ResettableByteBuffersIndexOutput blockMetaWriter;
 
-    AbstractBlockPackedWriter(IndexOutput indexOutput, int blockSize)
+    AbstractBlockPackedWriter(IndexOutput out, int blockSize)
     {
         checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
-        this.indexOutput = indexOutput;
-        this.blockMetaWriter = new ResettableByteBuffersIndexOutput(blockSize, "BlockPackedMeta");
-        blockValues = new long[blockSize];
+        this.out = out;
+        this.blockMetaWriter = LuceneCompat.getResettableByteBuffersIndexOutput(out.order(), 1024, "NumericValuesMeta");
+        values = new long[blockSize];
     }
 
+    private void checkNotFinished()
+    {
+        if (finished)
+        {
+            throw new IllegalStateException(String.format("[%s] Writer already finished!", out.getName()));
+        }
+    }
 
     /**
      * Append a new long.
@@ -59,13 +67,14 @@ public abstract class AbstractBlockPackedWriter
     public void add(long l) throws IOException
     {
         checkNotFinished();
-        if (blockIndex == blockValues.length)
+        if (off == values.length)
         {
             flush();
         }
-        blockValues[blockIndex++] = l;
+        values[off++] = l;
     }
 
+
     /**
      * Flush all buffered data to disk. This instance is not usable anymore
      * after this method has been called.
@@ -75,24 +84,24 @@ public void add(long l) throws IOException
     public long finish() throws IOException
     {
         checkNotFinished();
-        if (blockIndex > 0)
+        if (off > 0)
         {
             flush();
         }
-        final long fp = indexOutput.getFilePointer();
-        blockMetaWriter.copyTo(indexOutput);
+        final long fp = out.getFilePointer();
+        blockMetaWriter.copyTo(out);
         finished = true;
         return fp;
     }
 
-    protected abstract void flushBlock() throws IOException;
+    protected abstract void flush() throws IOException;
 
     void writeValues(int numValues, int bitsPerValue) throws IOException
     {
-        final DirectWriter writer = DirectWriter.getInstance(indexOutput, numValues, bitsPerValue);
+        final DirectWriterAdapter writer = LuceneCompat.directWriterGetInstance(out.order(), out, numValues, bitsPerValue);
         for (int i = 0; i < numValues; ++i)
         {
-            writer.add(blockValues[i]);
+            writer.add(values[i]);
         }
         writer.finish();
     }
@@ -107,18 +116,4 @@ void writeVLong(IndexOutput out, long i) throws IOException
         }
         out.writeByte((byte) i);
     }
-
-    private void flush() throws IOException
-    {
-        flushBlock();
-        blockIndex = 0;
-    }
-
-    private void checkNotFinished()
-    {
-        if (finished)
-        {
-            throw new IllegalStateException(String.format("[%s] Writer already finished!", indexOutput.getName()));
-        }
-    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedReader.java
index 50ce53d56cfa..50eb79ad43f5 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedReader.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedReader.java
@@ -19,18 +19,18 @@
 
 import java.io.IOException;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
 import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
-import org.apache.cassandra.index.sai.disk.v1.DirectReaders;
 import org.apache.cassandra.index.sai.disk.v1.LongArray;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.index.CorruptIndexException;
 
-import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.checkBlockSize;
-import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.numBlocks;
-import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.readVLong;
+import static org.apache.cassandra.index.sai.utils.SAICodecUtils.checkBlockSize;
+import static org.apache.cassandra.index.sai.utils.SAICodecUtils.numBlocks;
+import static org.apache.cassandra.index.sai.utils.SAICodecUtils.readVLong;
 import static org.apache.lucene.util.BitUtil.zigZagDecode;
 
 /**
@@ -46,6 +46,7 @@ public class BlockPackedReader implements LongArray.Factory
     private final long[] blockOffsets;
     private final long[] minValues;
 
+    @SuppressWarnings("resource")
     public BlockPackedReader(FileHandle file, NumericValuesMeta meta) throws IOException
     {
         this.file = file;
@@ -54,13 +55,12 @@ public BlockPackedReader(FileHandle file, NumericValuesMeta meta) throws IOExcep
 
         blockShift = checkBlockSize(meta.blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
         blockMask = meta.blockSize - 1;
-        int numBlocks = numBlocks(valueCount, meta.blockSize);
+        final int numBlocks = numBlocks(valueCount, meta.blockSize);
         blockBitsPerValue = new byte[numBlocks];
         blockOffsets = new long[numBlocks];
         minValues = new long[numBlocks];
 
-        try (RandomAccessReader reader = this.file.createReader();
-             IndexInputReader in = IndexInputReader.create(reader))
+        try (final IndexInputReader in = IndexInputReader.create(this.file.createReader()))
         {
             SAICodecUtils.validate(in);
             in.seek(meta.blockMetaOffset);
@@ -68,10 +68,12 @@ public BlockPackedReader(FileHandle file, NumericValuesMeta meta) throws IOExcep
             for (int i = 0; i < numBlocks; ++i)
             {
                 final int token = in.readByte() & 0xFF;
-                final int bitsPerValue = token >>> BlockPackedWriter.BPV_SHIFT;
-                int blockIndex = i;
-                DirectReaders.checkBitsPerValue(bitsPerValue, in, () -> String.format("Block %d", blockIndex));
-                if ((token & BlockPackedWriter.MIN_VALUE_EQUALS_0) == 0)
+                final int bitsPerValue = token >>> AbstractBlockPackedWriter.BPV_SHIFT;
+                if (bitsPerValue > 64)
+                {
+                    throw new CorruptIndexException(String.format("Block %d is corrupted. Bits per value should be no more than 64 and is %d.", i, bitsPerValue), in);
+                }
+                if ((token & AbstractBlockPackedWriter.MIN_VALUE_EQUALS_0) == 0)
                 {
                     long val = zigZagDecode(1L + readVLong(in));
                     minValues[i] = val;
@@ -95,11 +97,12 @@ public BlockPackedReader(FileHandle file, NumericValuesMeta meta) throws IOExcep
         }
     }
 
+    @VisibleForTesting
     @Override
     public LongArray open()
     {
-        IndexInput indexInput = IndexFileUtils.instance.openInput(file);
-        return new AbstractBlockPackedReader(indexInput, blockBitsPerValue, blockShift, blockMask, valueCount)
+        var indexInput = IndexFileUtils.instance().openInput(file);
+        return new AbstractBlockPackedReader(indexInput, blockBitsPerValue, blockShift, blockMask, 0, valueCount)
         {
             @Override
             protected long blockOffsetAt(int block)
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedWriter.java
index 9dcc29c6ab62..8c570932370c 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedWriter.java
@@ -19,45 +19,46 @@
 
 import java.io.IOException;
 
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.packed.DirectWriter;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.oldlucene.DirectWriterAdapter;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
 
 import static org.apache.lucene.util.BitUtil.zigZagEncode;
 
 /**
  * A writer for large sequences of longs.
  *
- * Modified copy of {@link org.apache.lucene.util.packed.BlockPackedWriter} to use {@link DirectWriter}
+ * Modified copy of {@link org.apache.lucene.util.packed.BlockPackedWriter} to use {@link DirectWriterAdapter}
  * for optimised reads that doesn't require seeking through the whole file to open a thread-exclusive reader.
  */
 public class BlockPackedWriter extends AbstractBlockPackedWriter
 {
-    static final int BPV_SHIFT = 1;
-    static final int MIN_VALUE_EQUALS_0 = 1;
-
     public BlockPackedWriter(IndexOutput out, int blockSize)
     {
         super(out, blockSize);
     }
 
     @Override
-    protected void flushBlock() throws IOException
+    protected void flush() throws IOException
     {
+        assert off > 0;
         long min = Long.MAX_VALUE, max = Long.MIN_VALUE;
-        for (int i = 0; i < blockIndex; ++i)
+        for (int i = 0; i < off; ++i)
         {
-            min = Math.min(blockValues[i], min);
-            max = Math.max(blockValues[i], max);
+            min = Math.min(values[i], min);
+            max = Math.max(values[i], max);
         }
 
-        long delta = max - min;
-        int bitsRequired = delta == 0 ? 0 : DirectWriter.unsignedBitsRequired(delta);
+        final long delta = max - min;
+        int bitsRequired = delta == 0 ? 0 : LuceneCompat.directWriterUnsignedBitsRequired(out.order(), delta);
 
-        int shiftedBitsRequired = (bitsRequired << BPV_SHIFT) | (min == 0 ? MIN_VALUE_EQUALS_0 : 0);
-        blockMetaWriter.writeByte((byte) shiftedBitsRequired);
+        final int token = (bitsRequired << BPV_SHIFT) | (min == 0 ? MIN_VALUE_EQUALS_0 : 0);
+        blockMetaWriter.writeByte((byte) token);
 
         if (min != 0)
         {
+            // TODO: the min values can be delta encoded since they are read linearly
+            // TODO: buffer the min values so they may be written as a single block
             writeVLong(blockMetaWriter, zigZagEncode(min) - 1);
         }
 
@@ -65,13 +66,15 @@ protected void flushBlock() throws IOException
         {
             if (min != 0)
             {
-                for (int i = 0; i < blockIndex; ++i)
+                for (int i = 0; i < off; ++i)
                 {
-                    blockValues[i] -= min;
+                    values[i] -= min;
                 }
             }
-            blockMetaWriter.writeVLong(indexOutput.getFilePointer());
-            writeValues(blockIndex, bitsRequired);
+            blockMetaWriter.writeVLong(out.getFilePointer());
+            writeValues(off, bitsRequired);
         }
+
+        off = 0;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedReader.java
index 02071b80525e..51d2cea02239 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedReader.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedReader.java
@@ -21,17 +21,15 @@
 
 import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
 import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
-import org.apache.cassandra.index.sai.disk.v1.DirectReaders;
 import org.apache.cassandra.index.sai.disk.v1.LongArray;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.util.packed.PackedInts;
 import org.apache.lucene.util.packed.PackedLongValues;
 
-import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.checkBlockSize;
-import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.numBlocks;
+import static org.apache.cassandra.index.sai.utils.SAICodecUtils.checkBlockSize;
+import static org.apache.cassandra.index.sai.utils.SAICodecUtils.numBlocks;
 
 /**
  * Provides non-blocking, random access to a stream written with {@link MonotonicBlockPackedWriter}.
@@ -47,6 +45,7 @@ public class MonotonicBlockPackedReader implements LongArray.Factory
     private final PackedLongValues minValues;
     private final float[] averages;
 
+    @SuppressWarnings("resource")
     public MonotonicBlockPackedReader(FileHandle file, NumericValuesMeta meta) throws IOException
     {
         this.valueCount = meta.valueCount;
@@ -59,8 +58,7 @@ public MonotonicBlockPackedReader(FileHandle file, NumericValuesMeta meta) throw
         blockBitsPerValue = new byte[numBlocks];
         this.file = file;
 
-        try (RandomAccessReader reader = this.file.createReader();
-             IndexInputReader in = IndexInputReader.create(reader))
+        try (final IndexInputReader in = IndexInputReader.create(this.file.createReader()))
         {
             SAICodecUtils.validate(in);
 
@@ -70,7 +68,10 @@ public MonotonicBlockPackedReader(FileHandle file, NumericValuesMeta meta) throw
                 minValuesBuilder.add(in.readZLong());
                 averages[i] = Float.intBitsToFloat(in.readInt());
                 final int bitsPerValue = in.readVInt();
-                DirectReaders.checkBitsPerValue(bitsPerValue, in, () -> "Postings list header");
+                if (bitsPerValue > 64)
+                {
+                    throw new CorruptIndexException(String.format("Block %d is corrupted. Bits per value should be no more than 64 and is %d.", i, bitsPerValue), in);
+                }
                 blockBitsPerValue[i] = (byte) bitsPerValue;
                 // when bitsPerValue is 0, block offset won't be used
                 blockOffsetsBuilder.add(bitsPerValue == 0 ? -1 : in.readVLong());
@@ -82,10 +83,11 @@ public MonotonicBlockPackedReader(FileHandle file, NumericValuesMeta meta) throw
     }
 
     @Override
+    @SuppressWarnings("resource")
     public LongArray open()
     {
-        final IndexInput indexInput = IndexFileUtils.instance.openInput(file);
-        return new AbstractBlockPackedReader(indexInput, blockBitsPerValue, blockShift, blockMask, valueCount)
+        var indexInput = IndexFileUtils.instance().openInput(file);
+        return new AbstractBlockPackedReader(indexInput, blockBitsPerValue, blockShift, blockMask, 0, valueCount)
         {
             @Override
             long delta(int block, int idx)
@@ -106,7 +108,13 @@ protected long blockOffsetAt(int block)
             }
 
             @Override
-            public long indexOf(long value)
+            public long ceilingRowId(long targetValue)
+            {
+               throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public long indexOf(long targetToken)
             {
                 throw new UnsupportedOperationException();
             }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedWriter.java
index 5845aabc77ca..316118d53673 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedWriter.java
@@ -19,17 +19,12 @@
 
 import java.io.IOException;
 
-import org.apache.lucene.store.IndexOutput;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
 import org.apache.lucene.util.packed.DirectWriter;
 
 /**
  * A writer for large monotonically increasing sequences of positive longs.
  *
- * The writer is optimised for monotonic sequences and stores values as a series of deltas
- * from an expected value. The expected value is calculated from the minimum value in the block and the average
- * delta for the block. This means that stored values are generally smaller and can be packed
- * into a smaller number of bits, allowing for larger block sizes.
- *
  * Modified copy of {@link org.apache.lucene.util.packed.MonotonicBlockPackedWriter} to use {@link DirectWriter} for
  * optimised reads that doesn't require seeking through the whole file to open a thread-exclusive reader.
  */
@@ -48,30 +43,32 @@ public void add(long l) throws IOException
     }
 
     @Override
-    protected void flushBlock() throws IOException
+    protected void flush() throws IOException
     {
-        final float averageDelta = blockIndex == 1 ? 0f : (float) (blockValues[blockIndex - 1] - blockValues[0]) / (blockIndex - 1);
-        long minimumValue = blockValues[0];
-        // adjust minimumValue so that all deltas will be positive
-        for (int index = 1; index < blockIndex; ++index)
+        assert off > 0;
+
+        final float avg = off == 1 ? 0f : (float) (values[off - 1] - values[0]) / (off - 1);
+        long min = values[0];
+        // adjust min so that all deltas will be positive
+        for (int i = 1; i < off; ++i)
         {
-            long actual = blockValues[index];
-            long expected = MonotonicBlockPackedReader.expected(minimumValue, averageDelta, index);
+            final long actual = values[i];
+            final long expected = MonotonicBlockPackedReader.expected(min, avg, i);
             if (expected > actual)
             {
-                minimumValue -= (expected - actual);
+                min -= (expected - actual);
             }
         }
 
         long maxDelta = 0;
-        for (int i = 0; i < blockIndex; ++i)
+        for (int i = 0; i < off; ++i)
         {
-            blockValues[i] = blockValues[i] - MonotonicBlockPackedReader.expected(minimumValue, averageDelta, i);
-            maxDelta = Math.max(maxDelta, blockValues[i]);
+            values[i] = values[i] - MonotonicBlockPackedReader.expected(min, avg, i);
+            maxDelta = Math.max(maxDelta, values[i]);
         }
 
-        blockMetaWriter.writeZLong(minimumValue);
-        blockMetaWriter.writeInt(Float.floatToIntBits(averageDelta));
+        blockMetaWriter.writeZLong(min);
+        blockMetaWriter.writeInt(Float.floatToIntBits(avg));
         if (maxDelta == 0)
         {
             blockMetaWriter.writeVInt(0);
@@ -80,8 +77,10 @@ protected void flushBlock() throws IOException
         {
             final int bitsRequired = DirectWriter.bitsRequired(maxDelta);
             blockMetaWriter.writeVInt(bitsRequired);
-            blockMetaWriter.writeVLong(indexOutput.getFilePointer());
-            writeValues(blockIndex, bitsRequired);
+            blockMetaWriter.writeVLong(out.getFilePointer());
+            writeValues(off, bitsRequired);
         }
+
+        off = 0;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesMeta.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesMeta.java
index 62da0292fb8d..cc9a0b0db926 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesMeta.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesMeta.java
@@ -19,7 +19,7 @@
 
 import java.io.IOException;
 
-import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 
 public class NumericValuesMeta
@@ -28,14 +28,21 @@ public class NumericValuesMeta
     public final int blockSize;
     public final long blockMetaOffset;
 
-    public NumericValuesMeta(DataInput input) throws IOException
+    public NumericValuesMeta(IndexInput input) throws IOException
     {
         valueCount = input.readLong();
         blockSize = input.readInt();
         blockMetaOffset = input.readVLong();
     }
 
-    public static void write(IndexOutput out, long valueCount, int blockSize, long blockMetaOffset) throws IOException
+    public NumericValuesMeta(long valueCount, int blockSize, long blockMetaOffset)
+    {
+        this.valueCount = valueCount;
+        this.blockSize = blockSize;
+        this.blockMetaOffset = blockMetaOffset;
+    }
+
+    public void write(IndexOutput out) throws IOException
     {
         out.writeLong(valueCount);
         out.writeInt(blockSize);
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesWriter.java
index 392146655d0c..d62355a2c32b 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesWriter.java
@@ -20,63 +20,64 @@
 import java.io.Closeable;
 import java.io.IOException;
 
-import javax.annotation.concurrent.NotThreadSafe;
-
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
 import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
 import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.lucene.store.IndexOutput;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_NUMERIC_VALUES_BLOCK_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_NUMERIC_VALUES_MONOTONIC_BLOCK_SIZE;
+
 
-@NotThreadSafe
 public class NumericValuesWriter implements Closeable
 {
-    public static final int MONOTONIC_BLOCK_SIZE = 16384;
-    public static final int BLOCK_SIZE = 128;
+    public static final int MONOTONIC_BLOCK_SIZE = SAI_NUMERIC_VALUES_MONOTONIC_BLOCK_SIZE.getInt();
+    public static final int BLOCK_SIZE = SAI_NUMERIC_VALUES_BLOCK_SIZE.getInt();
 
-    private final IndexOutput indexOutput;
+    private final IndexComponent.ForWrite components;
+    private final IndexOutput output;
     private final AbstractBlockPackedWriter writer;
     private final MetadataWriter metadataWriter;
-    private final String componentName;
     private final int blockSize;
     private long count = 0;
 
-    public NumericValuesWriter(IndexDescriptor indexDescriptor,
-                               IndexComponent indexComponent,
+    public NumericValuesWriter(IndexComponent.ForWrite components,
                                MetadataWriter metadataWriter,
                                boolean monotonic) throws IOException
     {
-        this(indexDescriptor, indexComponent, metadataWriter, monotonic, monotonic ? MONOTONIC_BLOCK_SIZE : BLOCK_SIZE);
+        this(components, metadataWriter, monotonic, monotonic ? MONOTONIC_BLOCK_SIZE : BLOCK_SIZE);
     }
 
-    public NumericValuesWriter(IndexDescriptor indexDescriptor,
-                               IndexComponent indexComponent,
+    public NumericValuesWriter(IndexComponent.ForWrite components,
                                MetadataWriter metadataWriter,
                                boolean monotonic,
                                int blockSize) throws IOException
     {
-        this.componentName = indexDescriptor.componentName(indexComponent);
-        this.indexOutput = indexDescriptor.openPerSSTableOutput(indexComponent);
-        SAICodecUtils.writeHeader(indexOutput);
-        this.writer = monotonic ? new MonotonicBlockPackedWriter(indexOutput, blockSize)
-                                : new BlockPackedWriter(indexOutput, blockSize);
+        this.components = components;
+        this.output = components.openOutput();
+        SAICodecUtils.writeHeader(output);
+
+        this.writer = monotonic ? new MonotonicBlockPackedWriter(output, blockSize)
+                                : new BlockPackedWriter(output, blockSize);
         this.metadataWriter = metadataWriter;
         this.blockSize = blockSize;
+
     }
 
     @Override
     public void close() throws IOException
     {
-        try (IndexOutput o = metadataWriter.builder(componentName))
+        try (IndexOutput o = metadataWriter.builder(components.fileNamePart()))
         {
-            long fp = writer.finish();
-            SAICodecUtils.writeFooter(indexOutput);
+            final long fp = writer.finish();
+            SAICodecUtils.writeFooter(output);
 
-            NumericValuesMeta.write(o, count, blockSize, fp);
+            NumericValuesMeta meta = new NumericValuesMeta(count, blockSize, fp);
+            meta.write(o);
         }
         finally
         {
-            indexOutput.close();
+            output.close();
         }
     }
 
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDPostingsIndex.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDPostingsIndex.java
new file mode 100644
index 000000000000..5a6756a56959
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDPostingsIndex.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+
+import com.carrotsearch.hppc.IntLongHashMap;
+import com.carrotsearch.hppc.IntLongMap;
+import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
+import org.apache.cassandra.io.util.FileHandle;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.cassandra.index.sai.utils.SAICodecUtils.validate;
+
+/**
+ * Mapping between node ID and an offset to its auxiliary posting list (containing every row id from all leaves
+ * reachable from that node. See {@link OneDimBKDPostingsWriter}).
+ */
+class BKDPostingsIndex
+{
+    private final int size;
+    public final IntLongMap index = new IntLongHashMap();
+
+    @SuppressWarnings("resource")
+    BKDPostingsIndex(FileHandle postingsFileHandle, long filePosition) throws IOException
+    {
+        try (final IndexInputReader input = IndexInputReader.create(postingsFileHandle.createReader()))
+        {
+            validate(input);
+            input.seek(filePosition);
+
+            size = input.readVInt();
+
+            for (int x = 0; x < size; x++)
+            {
+                final int node = input.readVInt();
+                final long filePointer = input.readVLong();
+
+                index.put(node, filePointer);
+            }
+        }
+    }
+
+    /**
+     * Returns <tt>true</tt> if given node ID has an auxiliary posting list.
+     */
+    boolean exists(int nodeID)
+    {
+        checkArgument(nodeID > 0);
+        return index.containsKey(nodeID);
+    }
+
+    /**
+     * Returns an offset within the bkd postings file to the begining of the blocks summary of given node's auxiliary
+     * posting list.
+     *
+     * @throws IllegalArgumentException when given nodeID doesn't have an auxiliary posting list. Check first with
+     * {@link #exists(int)}
+     */
+    long getPostingsFilePointer(int nodeID)
+    {
+        checkArgument(exists(nodeID));
+        return index.get(nodeID);
+    }
+
+    int size()
+    {
+        return size;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDQueries.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDQueries.java
new file mode 100644
index 000000000000..7b4b21f87ea8
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDQueries.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.lucene.index.PointValues.Relation;
+
+import static org.apache.lucene.index.PointValues.Relation.CELL_INSIDE_QUERY;
+
+public class BKDQueries
+{
+    private static final BKDReader.IntersectVisitor MATCH_ALL = new BKDReader.IntersectVisitor()
+    {
+        @Override
+        public boolean visit(byte[] packedValue)
+        {
+            return true;
+        }
+
+        @Override
+        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
+        {
+            return CELL_INSIDE_QUERY;
+        }
+    };
+
+    public static BKDReader.IntersectVisitor bkdQueryFrom(Expression expression, int numDim, int bytesPerDim)
+    {
+        if (expression.lower == null && expression.upper == null)
+        {
+            return MATCH_ALL;
+        }
+
+        Bound lower = null ;
+        if (expression.lower != null)
+        {
+            final byte[] lowerBound = toComparableBytes(numDim, bytesPerDim, expression.lower.value.encoded, expression.validator);
+            lower = new Bound(lowerBound, !expression.lower.inclusive);
+        }
+
+        Bound upper = null;
+        if (expression.upper != null)
+        {
+            final byte[] upperBound = toComparableBytes(numDim, bytesPerDim, expression.upper.value.encoded, expression.validator);
+            upper = new Bound(upperBound, !expression.upper.inclusive);
+        }
+
+        return new RangeQueryVisitor(numDim, bytesPerDim, lower, upper);
+    }
+
+    private static byte[] toComparableBytes(int numDim, int bytesPerDim, ByteBuffer value, AbstractType<?> type)
+    {
+        byte[] buffer = new byte[TypeUtil.fixedSizeOf(type)];
+        assert buffer.length == bytesPerDim * numDim;
+        TypeUtil.toComparableBytes(value, type, buffer);
+        return buffer;
+    }
+
+    private static abstract class RangeQuery implements BKDReader.IntersectVisitor
+    {
+        final int numDims;
+        final int bytesPerDim;
+
+        RangeQuery(int numDims, int bytesPerDim)
+        {
+            this.numDims = numDims;
+            this.bytesPerDim = bytesPerDim;
+        }
+
+        int compareUnsigned(byte[] packedValue, int dim, Bound bound)
+        {
+            final int offset = dim * bytesPerDim;
+            return Arrays.compareUnsigned(packedValue, offset, offset + bytesPerDim, bound.bound, offset, offset + bytesPerDim);
+        }
+    }
+
+    private static class Bound
+    {
+        private final byte[] bound;
+        private final boolean exclusive;
+
+        Bound(byte[] bound, boolean exclusive)
+        {
+            this.bound = bound;
+            this.exclusive = exclusive;
+        }
+
+        boolean smallerThan(int cmp)
+        {
+            return cmp > 0 || (cmp == 0 && exclusive);
+        }
+
+        boolean greaterThan(int cmp)
+        {
+            return cmp < 0 || (cmp == 0 && exclusive);
+        }
+    }
+
+    private static class RangeQueryVisitor extends RangeQuery
+    {
+        private final Bound lower;
+        private final Bound upper;
+
+        private RangeQueryVisitor(int numDims, int bytesPerDim, Bound lower, Bound upper)
+        {
+            super(numDims, bytesPerDim);
+            this.lower = lower;
+            this.upper = upper;
+        }
+
+        @Override
+        public boolean visit(byte[] packedValue)
+        {
+            for (int dim = 0; dim < numDims; dim++)
+            {
+                if (lower != null)
+                {
+                    int cmp = compareUnsigned(packedValue, dim, lower);
+                    if (lower.greaterThan(cmp))
+                    {
+                        // value is too low, in this dimension
+                        return false;
+                    }
+                }
+
+                if (upper != null)
+                {
+                    int cmp = compareUnsigned(packedValue, dim, upper);
+                    if (upper.smallerThan(cmp))
+                    {
+                        // value is too high, in this dimension
+                        return false;
+                    }
+                }
+            }
+
+            return true;
+        }
+
+        @Override
+        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
+        {
+            boolean crosses = false;
+
+            for (int dim = 0; dim < numDims; dim++)
+            {
+                if (lower != null)
+                {
+                    int maxCmp = compareUnsigned(maxPackedValue, dim, lower);
+                    if (lower.greaterThan(maxCmp))
+                        return Relation.CELL_OUTSIDE_QUERY;
+
+                    int minCmp = compareUnsigned(minPackedValue, dim, lower);
+                    crosses |= lower.greaterThan(minCmp);
+                }
+
+                if (upper != null)
+                {
+                    int minCmp = compareUnsigned(minPackedValue, dim, upper);
+                    if (upper.smallerThan(minCmp))
+                        return Relation.CELL_OUTSIDE_QUERY;
+
+                    int maxCmp = compareUnsigned(maxPackedValue, dim, upper);
+                    crosses |= upper.smallerThan(maxCmp);
+                }
+            }
+
+            if (crosses)
+            {
+                return Relation.CELL_CROSSES_QUERY;
+            }
+            else
+            {
+                return Relation.CELL_INSIDE_QUERY;
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDReader.java
new file mode 100644
index 000000000000..055441b71145
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDReader.java
@@ -0,0 +1,1008 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.base.Stopwatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.io.CryptoUtils;
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
+import org.apache.cassandra.index.sai.disk.v1.postings.FilteringPostingList;
+import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.utils.AbortedOperationException;
+import org.apache.cassandra.index.sai.utils.SeekingRandomAccessInput;
+import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.LongValues;
+
+/**
+ * Handles intersection of a multi-dimensional shape in byte[] space with a block KD-tree previously written with
+ * {@link BKDWriter}.
+ */
+public class BKDReader extends TraversingBKDReader implements Closeable
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    public enum Direction { FORWARD, BACKWARD }
+
+    private final IndexContext indexContext;
+    private final FileHandle postingsFile;
+    private final FileHandle kdtreeFile;
+    private final BKDPostingsIndex postingsIndex;
+    private final ICompressor compressor;
+
+    /**
+     * Performs a blocking read.
+     */
+    public BKDReader(IndexContext indexContext,
+                     FileHandle kdtreeFile,
+                     long bkdIndexRoot,
+                     FileHandle postingsFile,
+                     long bkdPostingsRoot) throws IOException
+    {
+        super(kdtreeFile, bkdIndexRoot);
+        this.indexContext = indexContext;
+        this.postingsFile = postingsFile;
+        this.kdtreeFile = kdtreeFile;
+        this.postingsIndex = new BKDPostingsIndex(postingsFile, bkdPostingsRoot);
+        this.compressor = null;
+    }
+
+    public interface DocMapper
+    {
+        int oldToNew(int rowID);
+    }
+
+    public IteratorState iteratorState(Direction direction, IntersectVisitor query) throws IOException
+    {
+        return new IteratorState(rowID -> rowID, direction, query);
+    }
+
+    @VisibleForTesting
+    public IteratorState iteratorState() throws IOException
+    {
+        return iteratorState(Direction.FORWARD, null);
+    }
+
+    public class IteratorState extends AbstractGuavaIterator<Integer> implements Comparable<IteratorState>, Closeable
+    {
+        public final byte[] scratch;
+
+        private final IndexInput bkdInput;
+        private final IndexInput bkdPostingsInput;
+        private final byte[] packedValues = new byte[maxPointsInLeafNode * packedBytesLength];
+        private final IntArrayList tempPostings = new IntArrayList();
+        private final int[] postings = new int[maxPointsInLeafNode];
+        private final DocMapper docMapper;
+        private final LeafCursor leafCursor;
+
+        private int leafPointCount;
+        private int leafPointIndex = -1;
+
+        private final Direction direction;
+        private final BKDReader.IntersectVisitor query;
+
+        public IteratorState(DocMapper docMapper, Direction direction, BKDReader.IntersectVisitor query) throws IOException
+        {
+            this.docMapper = docMapper;
+            this.direction = direction;
+            this.query = query;
+
+            scratch = new byte[packedBytesLength];
+
+            final long firstLeafFilePointer = getMinLeafBlockFP();
+            bkdInput = IndexFileUtils.instance().openInput(kdtreeFile);
+            bkdPostingsInput = IndexFileUtils.instance().openInput(postingsFile);
+            bkdInput.seek(firstLeafFilePointer);
+
+            leafCursor = new LeafCursor(direction, query);
+            leafPointCount = readLeaf(leafCursor.getFilePointer(), leafCursor.getNodeId(), bkdInput, packedValues, bkdPostingsInput, postings, tempPostings);
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(bkdInput, bkdPostingsInput);
+        }
+
+        @Override
+        public int compareTo(final IteratorState other)
+        {
+            final int cmp = Arrays.compareUnsigned(scratch, 0, packedBytesLength, other.scratch, 0, packedBytesLength);
+            if (cmp == 0)
+            {
+                final long rowid1 = next;
+                final long rowid2 = other.next;
+                return Long.compare(rowid1, rowid2);
+            }
+            return cmp;
+        }
+
+        @Override
+        protected Integer computeNext()
+        {
+            while (true)
+            {
+                if (leafPointIndex == leafPointCount - 1)
+                {
+                    if (!leafCursor.advance())
+                        return endOfData();
+
+                    try
+                    {
+                        int id = leafCursor.getNodeId();
+                        long fp = leafCursor.getFilePointer();
+                        leafPointCount = readLeaf(fp, id, bkdInput, packedValues, bkdPostingsInput, postings, tempPostings);
+                    }
+                    catch (IOException e)
+                    {
+                        logger.error("Failed to read leaf during BKDTree merger", e);
+                        throw new RuntimeException("Failed to read leaf during BKDTree merger", e);
+                    }
+                    leafPointIndex = -1;
+                }
+
+                leafPointIndex++;
+                // If we're ascending, we need to read the leaf from the start, otherwise we need to read it from the end
+                int pointer = direction == Direction.FORWARD ? leafPointIndex : leafPointCount - leafPointIndex - 1;
+
+                System.arraycopy(packedValues, pointer * packedBytesLength, scratch, 0, packedBytesLength);
+                if (query == null || query.visit(scratch))
+                    return docMapper.oldToNew(postings[pointer]);
+            }
+        }
+    }
+
+    @SuppressWarnings("resource")
+    public int readLeaf(long filePointer,
+                        int nodeID,
+                        final IndexInput bkdInput,
+                        final byte[] packedValues,
+                        final IndexInput bkdPostingsInput,
+                        int[] postings,
+                        IntArrayList tempPostings) throws IOException
+    {
+        bkdInput.seek(filePointer);
+        final int count = bkdInput.readVInt();
+        // loading doc ids occurred here prior
+        final int orderMapLength = bkdInput.readVInt();
+        final long orderMapPointer = bkdInput.getFilePointer();
+
+        // order of the values in the posting list
+        final short[] origIndex = new short[maxPointsInLeafNode];
+
+        final int[] commonPrefixLengths = new int[numDims];
+        final byte[] scratchPackedValue1 = new byte[packedBytesLength];
+
+        final SeekingRandomAccessInput randoInput = new SeekingRandomAccessInput(bkdInput);
+        LongValues orderMapReader = LuceneCompat.directReaderGetInstance(randoInput, bitsPerValue, orderMapPointer);
+        for (int x = 0; x < count; x++)
+        {
+            final short idx = LeafOrderMap.getValue(x, orderMapReader);
+            origIndex[x] = idx;
+        }
+
+        IndexInput leafInput = bkdInput;
+
+        // reused byte arrays for the decompression of leaf values
+        final BytesRef uncompBytes = new BytesRef(new byte[16]);
+        final BytesRef compBytes = new BytesRef(new byte[16]);
+
+        // seek beyond the ordermap
+        leafInput.seek(orderMapPointer + orderMapLength);
+
+        if (compressor != null)
+        {
+            // This should not throw WouldBlockException, even though we're on a TPC thread, because the
+            // secret key used by the underlying encryptor should be loaded at reader construction time.
+            leafInput = CryptoUtils.uncompress(bkdInput, compressor, compBytes, uncompBytes);
+        }
+
+        final IntersectVisitor visitor = new IntersectVisitor() {
+            int i = 0;
+
+            @Override
+            public boolean visit(byte[] packedValue)
+            {
+                System.arraycopy(packedValue, 0, packedValues, i * packedBytesLength, packedBytesLength);
+                i++;
+                return true;
+            }
+
+            @Override
+            public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+                return Relation.CELL_CROSSES_QUERY;
+            }
+        };
+
+        visitDocValues(commonPrefixLengths, scratchPackedValue1, leafInput, count, visitor, null, origIndex);
+
+        if (postingsIndex.exists(nodeID))
+        {
+            final long pointer = postingsIndex.getPostingsFilePointer(nodeID);
+            final PostingsReader.BlocksSummary summary = new PostingsReader.BlocksSummary(bkdPostingsInput, pointer);
+            final PostingsReader postingsReader = new PostingsReader(bkdPostingsInput, summary, QueryEventListener.PostingListEventListener.NO_OP);
+
+            tempPostings.clear();
+
+            // gather the postings into tempPostings
+            while (true)
+            {
+                final int rowid = postingsReader.nextPosting();
+                if (rowid == PostingList.END_OF_STREAM) break;
+                tempPostings.add(rowid);
+            }
+
+            // put the postings into the array according the origIndex
+            for (int x = 0; x < tempPostings.size(); x++)
+            {
+                int idx = origIndex[x];
+                final int rowid = tempPostings.get(idx);
+
+                postings[x] = rowid;
+            }
+        }
+        else
+        {
+            throw new IllegalStateException();
+        }
+        return count;
+    }
+
+    @Override
+    public void close()
+    {
+        try
+        {
+            super.close();
+        }
+        finally
+        {
+            FileUtils.closeQuietly(kdtreeFile, postingsFile);
+        }
+    }
+
+    @SuppressWarnings("resource")
+    public PostingList intersect(IntersectVisitor visitor, QueryEventListener.BKDIndexEventListener listener, QueryContext context)
+    {
+        Relation relation = visitor.compare(minPackedValue, maxPackedValue);
+
+        if (relation == Relation.CELL_OUTSIDE_QUERY)
+        {
+            listener.onIntersectionEarlyExit();
+            return PostingList.EMPTY;
+        }
+
+        listener.onSegmentHit();
+        IndexInput bkdInput = IndexFileUtils.instance().openInput(indexFile);
+        IndexInput postingsInput = IndexFileUtils.instance().openInput(postingsFile);
+        IndexInput postingsSummaryInput = IndexFileUtils.instance().openInput(postingsFile);
+        PackedIndexTree index = new PackedIndexTree();
+
+        Intersection completable =
+        relation == Relation.CELL_INSIDE_QUERY ?
+                new Intersection(bkdInput, postingsInput, postingsSummaryInput, index, listener, context) :
+                new FilteringIntersection(bkdInput, postingsInput, postingsSummaryInput, index, visitor, listener, context);
+
+        return completable.execute();
+    }
+
+    /**
+     * Synchronous intersection of an multi-dimensional shape in byte[] space with a block KD-tree
+     * previously written with {@link BKDWriter}.
+     */
+    class Intersection
+    {
+        private final Stopwatch queryExecutionTimer = Stopwatch.createStarted();
+        final QueryContext context;
+
+        final IndexInput bkdInput;
+        final SeekingRandomAccessInput bkdRandomInput;
+        final IndexInput postingsInput;
+        final IndexInput postingsSummaryInput;
+        final IndexTree index;
+        final QueryEventListener.BKDIndexEventListener listener;
+
+        Intersection(IndexInput bkdInput, IndexInput postingsInput, IndexInput postingsSummaryInput,
+                     IndexTree index, QueryEventListener.BKDIndexEventListener listener, QueryContext context)
+        {
+            this.bkdInput = bkdInput;
+            this.bkdRandomInput = new SeekingRandomAccessInput(bkdInput);
+            this.postingsInput = postingsInput;
+            this.postingsSummaryInput = postingsSummaryInput;
+            this.index = index;
+            this.listener = listener;
+            this.context = context;
+        }
+
+        public PostingList execute()
+        {
+            try
+            {
+                var postingLists = new ArrayList<PostingList>(100);
+                executeInternal(postingLists);
+
+                FileUtils.closeQuietly(bkdInput);
+
+                return mergePostings(postingLists);
+            }
+            catch (Throwable t)
+            {
+                if (!(t instanceof AbortedOperationException))
+                    logger.error(indexContext.logMessage("kd-tree intersection failed on {}"), indexFile.path(), t);
+
+                closeOnException();
+                throw Throwables.cleaned(t);
+            }
+        }
+
+        protected void executeInternal(final Collection<PostingList> postingLists) throws IOException
+        {
+            collectPostingLists(postingLists);
+        }
+
+        protected void closeOnException()
+        {
+            FileUtils.closeQuietly(bkdInput, postingsInput, postingsSummaryInput);
+        }
+
+        protected PostingList mergePostings(ArrayList<PostingList> postingLists) throws IOException
+        {
+            final long elapsedMicros = queryExecutionTimer.stop().elapsed(TimeUnit.MICROSECONDS);
+
+            listener.onIntersectionComplete(elapsedMicros, TimeUnit.MICROSECONDS);
+            listener.postingListsHit(postingLists.size());
+
+            if (!postingLists.isEmpty() && logger.isTraceEnabled())
+                logger.trace(indexContext.logMessage("[{}] Intersection completed in {} microseconds. {} leaf and internal posting lists hit."),
+                             indexFile.path(), elapsedMicros, postingLists.size());
+
+            return MergePostingList.merge(postingLists)
+                                   .onClose(() -> FileUtils.close(postingsInput, postingsSummaryInput));
+        }
+
+        public void collectPostingLists(Collection<PostingList> postingLists) throws IOException
+        {
+            context.checkpoint();
+
+            final int nodeID = index.getNodeID();
+
+            // if there is pre-built posting for entire subtree
+            if (postingsIndex.exists(nodeID))
+            {
+                postingLists.add(initPostingReader(postingsIndex.getPostingsFilePointer(nodeID)));
+                return;
+            }
+
+            Preconditions.checkState(!index.isLeafNode(), "Leaf node %s does not have kd-tree postings.", index.getNodeID());
+
+            // Recurse on left sub-tree:
+            index.pushLeft();
+            collectPostingLists(postingLists);
+            index.pop();
+
+            // Recurse on right sub-tree:
+            index.pushRight();
+            collectPostingLists(postingLists);
+            index.pop();
+        }
+
+        private PostingList initPostingReader(long offset) throws IOException
+        {
+            final PostingsReader.BlocksSummary summary = new PostingsReader.BlocksSummary(postingsSummaryInput, offset);
+            return new PostingsReader(postingsInput, summary, listener.postingListEventListener());
+        }
+    }
+
+    /**
+     * Modified copy of BKDReader#visitDocValues()
+     */
+    private int visitDocValues(int[] commonPrefixLengths,
+                               byte[] scratchPackedValue1,
+                               IndexInput in,
+                               int count,
+                               IntersectVisitor visitor,
+                               FixedBitSet[] holder,
+                               final short[] origIndex) throws IOException
+    {
+        readCommonPrefixes(commonPrefixLengths, scratchPackedValue1, in);
+
+        int compressedDim = readCompressedDim(in);
+        if (compressedDim == -1)
+        {
+            return visitRawDocValues(commonPrefixLengths, scratchPackedValue1, in, count, visitor, holder, origIndex);
+        }
+        else
+        {
+            return visitCompressedDocValues(commonPrefixLengths, scratchPackedValue1, in, count, visitor, compressedDim, holder, origIndex);
+        }
+    }
+
+    /**
+     * Modified copy of {@link org.apache.lucene.util.bkd.BKDReader#readCompressedDim(IndexInput)}
+     */
+    @SuppressWarnings("JavadocReference")
+    private int readCompressedDim(IndexInput in) throws IOException
+    {
+        int compressedDim = in.readByte();
+        if (compressedDim < -1 || compressedDim >= numDims)
+        {
+            throw new CorruptIndexException(String.format("Dimension should be in the range [-1, %d), but was %d.", numDims, compressedDim), in);
+        }
+        return compressedDim;
+    }
+
+    /**
+     * Modified copy of BKDReader#visitCompressedDocValues()
+     */
+    private int visitCompressedDocValues(int[] commonPrefixLengths,
+                                         byte[] scratchPackedValue,
+                                         IndexInput in,
+                                         int count,
+                                         IntersectVisitor visitor,
+                                         int compressedDim,
+                                         FixedBitSet[] holder,
+                                         final short[] origIndex) throws IOException
+    {
+        // the byte at `compressedByteOffset` is compressed using run-length compression,
+        // other suffix bytes are stored verbatim
+        final int compressedByteOffset = compressedDim * bytesPerDim + commonPrefixLengths[compressedDim];
+        commonPrefixLengths[compressedDim]++;
+        int i, collected = 0;
+
+        final FixedBitSet bitSet;
+        if (holder != null)
+        {
+            bitSet = new FixedBitSet(maxPointsInLeafNode);
+        }
+        else
+        {
+            bitSet = null;
+        }
+
+        for (i = 0; i < count; )
+        {
+            scratchPackedValue[compressedByteOffset] = in.readByte();
+            final int runLen = Byte.toUnsignedInt(in.readByte());
+            for (int j = 0; j < runLen; ++j)
+            {
+                for (int dim = 0; dim < numDims; dim++)
+                {
+                    int prefix = commonPrefixLengths[dim];
+                    in.readBytes(scratchPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
+                }
+                final int rowIDIndex = origIndex[i + j];
+                if (visitor.visit(scratchPackedValue))
+                {
+                    if (bitSet != null) bitSet.set(rowIDIndex);
+                    collected++;
+                }
+            }
+            i += runLen;
+        }
+        if (i != count)
+        {
+            throw new CorruptIndexException(String.format("Expected %d sub-blocks but read %d.", count, i), in);
+        }
+
+        if (holder != null)
+        {
+            holder[0] = bitSet;
+        }
+
+        return collected;
+    }
+
+    /**
+     * Modified copy of BKDReader#visitRawDocValues()
+     */
+    private int visitRawDocValues(int[] commonPrefixLengths,
+                                  byte[] scratchPackedValue,
+                                  IndexInput in,
+                                  int count,
+                                  IntersectVisitor visitor,
+                                  FixedBitSet[] holder,
+                                  final short[] origIndex) throws IOException
+    {
+        final FixedBitSet bitSet;
+        if (holder != null)
+        {
+            bitSet = new FixedBitSet(maxPointsInLeafNode);
+        }
+        else
+        {
+            bitSet = null;
+        }
+
+        int collected = 0;
+        for (int i = 0; i < count; ++i)
+        {
+            for (int dim = 0; dim < numDims; dim++)
+            {
+                int prefix = commonPrefixLengths[dim];
+                in.readBytes(scratchPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
+            }
+            final int rowIDIndex = origIndex[i];
+            if (visitor.visit(scratchPackedValue))
+            {
+                if (bitSet != null) bitSet.set(rowIDIndex);
+
+                collected++;
+            }
+        }
+        if (holder != null)
+        {
+            holder[0] = bitSet;
+        }
+        return collected;
+    }
+
+    /**
+     * Copy of BKDReader#readCommonPrefixes()
+     */
+    private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException
+    {
+        for (int dim = 0; dim < numDims; dim++)
+        {
+            int prefix = in.readVInt();
+            commonPrefixLengths[dim] = prefix;
+            if (prefix > 0)
+            {
+//                System.out.println("dim * bytesPerDim="+(dim * bytesPerDim)+" prefix="+prefix+" numDims="+numDims);
+                in.readBytes(scratchPackedValue, dim * bytesPerDim, prefix);
+            }
+        }
+    }
+
+    private class FilteringIntersection extends Intersection
+    {
+        private final IntersectVisitor visitor;
+        private final byte[] scratchPackedValue1;
+        private final int[] commonPrefixLengths;
+        private final short[] origIndex;
+
+        // reused byte arrays for the decompression of leaf values
+        private final BytesRef uncompBytes = new BytesRef(new byte[16]);
+        private final BytesRef compBytes = new BytesRef(new byte[16]);
+
+        FilteringIntersection(IndexInput bkdInput, IndexInput postingsInput, IndexInput postingsSummaryInput,
+                              IndexTree index, IntersectVisitor visitor,
+                              QueryEventListener.BKDIndexEventListener listener, QueryContext context)
+        {
+            super(bkdInput, postingsInput, postingsSummaryInput, index, listener, context);
+            this.visitor = visitor;
+            this.commonPrefixLengths = new int[numDims];
+            this.scratchPackedValue1 = new byte[packedBytesLength];
+            this.origIndex = new short[maxPointsInLeafNode];
+        }
+
+        @Override
+        public void executeInternal(final Collection<PostingList> postingLists) throws IOException
+        {
+            collectPostingLists(postingLists, minPackedValue, maxPackedValue);
+        }
+
+        public void collectPostingLists(Collection<PostingList> postingLists, byte[] cellMinPacked, byte[] cellMaxPacked) throws IOException
+        {
+            context.checkpoint();
+
+            final Relation r = visitor.compare(cellMinPacked, cellMaxPacked);
+
+            if (r == Relation.CELL_OUTSIDE_QUERY)
+            {
+                // This cell is fully outside of the query shape: stop recursing
+                return;
+            }
+
+            if (r == Relation.CELL_INSIDE_QUERY)
+            {
+                // This cell is fully inside of the query shape: recursively add all points in this cell without filtering
+                super.collectPostingLists(postingLists);
+                return;
+            }
+
+            if (index.isLeafNode())
+            {
+                if (index.nodeExists())
+                    filterLeaf(postingLists);
+                return;
+            }
+
+            visitNode(postingLists, cellMinPacked, cellMaxPacked);
+        }
+
+        @SuppressWarnings("resource")
+        void filterLeaf(Collection<PostingList> postingLists) throws IOException
+        {
+            bkdInput.seek(index.getLeafBlockFP());
+
+            final int count = bkdInput.readVInt();
+
+            // loading doc ids occurred here prior
+
+            final FixedBitSet[] holder = new FixedBitSet[1];
+
+            final int orderMapLength = bkdInput.readVInt();
+
+            final long orderMapPointer = bkdInput.getFilePointer();
+
+            LongValues orderMapReader = LuceneCompat.directReaderGetInstance(bkdRandomInput, bitsPerValue, orderMapPointer);
+            for (int x = 0; x < count; x++)
+            {
+                origIndex[x] = LeafOrderMap.getValue(x, orderMapReader);
+            }
+
+            // seek beyond the ordermap
+            bkdInput.seek(orderMapPointer + orderMapLength);
+
+            IndexInput leafInput = bkdInput;
+
+            if (compressor != null)
+            {
+                // This should not throw WouldBlockException, even though we're on a TPC thread, because the
+                // secret key used by the underlying encryptor should be loaded at reader construction time.
+                leafInput = CryptoUtils.uncompress(bkdInput, compressor, compBytes, uncompBytes);
+            }
+
+            visitDocValues(commonPrefixLengths, scratchPackedValue1, leafInput, count, visitor, holder, origIndex);
+
+            final int nodeID = index.getNodeID();
+
+            if (postingsIndex.exists(nodeID) && holder[0].cardinality() > 0)
+            {
+                final long pointer = postingsIndex.getPostingsFilePointer(nodeID);
+                postingLists.add(initFilteringPostingReader(pointer, holder[0]));
+            }
+        }
+
+        void visitNode(Collection<PostingList> postingLists, byte[] cellMinPacked, byte[] cellMaxPacked) throws IOException
+        {
+            int splitDim = index.getSplitDim();
+            assert splitDim >= 0 : "splitDim=" + splitDim;
+            assert splitDim < numDims;
+
+            byte[] splitPackedValue = index.getSplitPackedValue();
+            BytesRef splitDimValue = index.getSplitDimValue();
+            assert splitDimValue.length == bytesPerDim;
+
+            // make sure cellMin <= splitValue <= cellMax:
+            assert Arrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0 : "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
+            assert Arrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0 : "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
+
+            // Recurse on left sub-tree:
+            System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
+            System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim * bytesPerDim, bytesPerDim);
+
+            index.pushLeft();
+            collectPostingLists(postingLists, cellMinPacked, splitPackedValue);
+            index.pop();
+
+            // Restore the split dim value since it may have been overwritten while recursing:
+            System.arraycopy(splitPackedValue, splitDim * bytesPerDim, splitDimValue.bytes, splitDimValue.offset, bytesPerDim);
+            // Recurse on right sub-tree:
+            System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
+            System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim * bytesPerDim, bytesPerDim);
+            index.pushRight();
+            collectPostingLists(postingLists, splitPackedValue, cellMaxPacked);
+            index.pop();
+        }
+
+        private PostingList initFilteringPostingReader(long offset, FixedBitSet filter) throws IOException
+        {
+            final PostingsReader.BlocksSummary summary = new PostingsReader.BlocksSummary(postingsSummaryInput, offset);
+            return initFilteringPostingReader(filter, summary);
+        }
+
+        @SuppressWarnings("resource")
+        private PostingList initFilteringPostingReader(FixedBitSet filter, PostingsReader.BlocksSummary header) throws IOException
+        {
+            PostingsReader postingsReader = new PostingsReader(postingsInput, header, listener.postingListEventListener());
+            return new FilteringPostingList(filter, postingsReader);
+        }
+    }
+
+    public int getNumDimensions()
+    {
+        return numDims;
+    }
+
+    public int getBytesPerDimension()
+    {
+        return bytesPerDim;
+    }
+
+    public long getPointCount()
+    {
+        return pointCount;
+    }
+
+    /**
+     * We recurse the BKD tree, using a provided instance of this to guide the recursion.
+     */
+    public interface IntersectVisitor
+    {
+        /**
+         * Called for all values in a leaf cell that crosses the query.  The consumer
+         * should scrutinize the packedValue to decide whether to accept it.  In the 1D case,
+         * values are visited in increasing order, and in the case of ties, in increasing order
+         * by segment row ID.
+         */
+        boolean visit(byte[] packedValue);
+
+        /**
+         * Called for non-leaf cells to test how the cell relates to the query, to
+         * determine how to further recurse down the tree.
+         */
+        Relation compare(byte[] minPackedValue, byte[] maxPackedValue);
+    }
+
+    /**
+     * Iterates the leaves of the KD-tree forward or backwards.
+     * Makes no heap allocations on iteration.
+     */
+    private class LeafCursor
+    {
+        private final @Nullable IntersectVisitor query;
+        private final Direction direction;
+
+        // This is not just the index tree, but actually a tree + some state like current node pointer
+        // This remembers the current position of the cursor
+        private final PackedIndexTree tree;
+
+        // Remembers which nodes of the tree on the current path from the root were already fully explored.
+        // The set stores their level numbers.
+        //
+        // Because the index is a binary tree, a node can have at most 2 child nodes.
+        // When we visit the node for the first time, and we go down to its first child,
+        // and we see there is another child we must visit later,
+        // we consider this node as uncompleted (we're removing its level from this set).
+        // When we go back up to that node for the second time, we consult this set, and
+        // we see the node has one more child to visit. So we go down again to the second child, but this time we mark
+        // the node as complete, that is, we store its level in this set. So when we visit the node again for the third
+        // time, we know it's done, and we have to go up at least one more level.
+        //
+        // Note that we're storing levels, because we're interested only in the nodes on the current path from
+        // the root of the tree, as those are the only nodes that could be explored. A more obvious alternative
+        // would be to keep a set of all already visited node ids in the tree, but that would have worse memory
+        // complexity and would likely require a larger set and some heap allocations.
+        //
+        // Class invariant: this structure must contain up-to-date information
+        // for all the levels above the current level, up to the root.
+        private final BitSet completedLevels;
+
+        /**
+         * Creates the cursor over the KD-tree leaves and positions it on the first leaf
+         * appropriate for the given query and traversal direction.
+         * Even if the query does not match any data, the cursor is positioned on one of the tree leaves,
+         * so {@link #getFilePointer()} and {@link #getNodeId()} can be always called immediately after the construction.
+         *
+         * @param query restricts the leaves to the ones that might contain the data that match the query,
+         *              null query means the range is not restricted
+         */
+        LeafCursor(Direction direction, @Nullable IntersectVisitor query)
+        {
+            this.query = query;
+            this.direction = direction;
+
+            completedLevels = new BitSet(64); // physically impossible to have a tree bigger than 2^64 nodes
+            tree = new PackedIndexTree();  // this positions the tree at node id 1 and level 1 (not 0)
+
+            if (direction == Direction.FORWARD)
+                pushToMinLeaf(query);
+            else
+                pushToMaxLeaf(query);
+        }
+
+        /**
+         * Returns the id of the node the cursor is positioned at.
+         * Valid only immediately after construction or after a call to {@link #advance()} which returned {@code true}.
+         */
+        int getNodeId()
+        {
+            assert tree.isLeafNode() : "Cursor not on a leaf node; end of data reached";
+            return tree.nodeID;
+        }
+
+        /**
+         * Returns the file pointer of the node the cursor is positioned at
+         * Valid only immediately after construction or after a call to {@link #advance()} which returned {@code true}.
+         */
+        long getFilePointer()
+        {
+            assert tree.isLeafNode() : "Cursor not on a leaf node; end of data reached";
+            return tree.getLeafBlockFP();
+        }
+
+        /**
+         * Advances the cursor to the next leaf.
+         * If there are no more leaves in the tree at all, positions the index tree at node 0.
+         * If there exist leaves, but they are out of the query range, positions the index tree at a non-leaf node.
+         * Calling this again after the cursor reached the end of the data is not allowed.
+         *
+         * @return true if the cursor was moved to the next leaf, false if there are no more leaves to iterate
+         */
+        boolean advance()
+        {
+            assert tree.isLeafNode() : "Cursor not on a leaf node; end of data reached";
+
+            // Mark the current node as completed, so that the call to `popToFirstUncompletedLevel`
+            // won't stop on this level immediately but goes up instead.
+            completedLevels.set(tree.level);
+
+            // Go up to the closest parent node that has a child we haven't visited yet.
+            if (!popToFirstUncompletedLevel())
+                return false;
+
+            assert tree.nodeExists() : "Node does not exist";
+            assert !tree.isLeafNode() : "Expected a non-leaf node";
+            assert !completedLevels.get(tree.level) : "Expected an uncompleted node";
+
+            // Go to the next leaf
+            if (direction == Direction.FORWARD)
+            {
+                if (query != null && query.compare(tree.getSplitDimValue().bytes, maxPackedValue) == Relation.CELL_OUTSIDE_QUERY)
+                    return false;
+                pushRight();
+                pushToMinLeaf();
+            }
+            else // Direcion.BACKWARD
+            {
+                if (query != null && query.compare(minPackedValue, tree.getSplitDimValue().bytes) == Relation.CELL_OUTSIDE_QUERY)
+                    return false;
+                pushLeft();
+                pushToMaxLeaf();
+            }
+            assert tree.isLeafNode() : "Cursor ended up on a non-leaf node";
+            return true;
+        }
+
+        /**
+         * Goes up the tree until it finds the first node for which we haven't exhausted all the paths down.
+         *
+         * @return true if uncompleted node is found, false if it reaches the top of the tree
+         */
+        boolean popToFirstUncompletedLevel()
+        {
+            while (completedLevels.get(tree.level) && tree.level > 0)
+                tree.pop();
+
+            // 0 level is special; you cannot go down from level 0, so if we hit level 0, the traversal ended,
+            // so we must signal it to the caller by returning false
+            return tree.level != 0;
+        }
+
+        /**
+         * Positions the index on the left-most leaf.
+         */
+        void pushToMinLeaf()
+        {
+            pushToLeaf(Predicates.alwaysFalse());
+        }
+
+        /**
+         * Positions the index on the left-most leaf that intersects the query
+         */
+        void pushToMinLeaf(BKDReader.IntersectVisitor query)
+        {
+            pushToLeaf(split -> query != null && query.compare(minPackedValue, split) == Relation.CELL_OUTSIDE_QUERY);
+        }
+
+        /**
+         * Positions the index on the right-most leaf.
+         */
+        void pushToMaxLeaf()
+        {
+            pushToLeaf(Predicates.alwaysTrue());
+        }
+
+        /**
+         * Positions the index on the right-most leaf that intersects the query
+         */
+        void pushToMaxLeaf(BKDReader.IntersectVisitor query)
+        {
+            pushToLeaf(split -> query == null || query.compare(split, maxPackedValue) != Relation.CELL_OUTSIDE_QUERY);
+        }
+
+        /**
+         * Recursively goes down the KD-tree until it reaches a leaf node.
+         * At every non-leaf node, uses the provided function to decide the direction to go.
+         *
+         * @param shouldGoRight a function that takes the split point of a non-leaf node
+         *                      and returns true if the search path should follow to the right child
+         */
+        void pushToLeaf(Predicate<byte[]> shouldGoRight)
+        {
+            while (!tree.isLeafNode())
+            {
+                // It is tempting to call index.getSplitPackedValue(), but that would return an empty array.
+                // It looks the user of the PackedIndexTree is supposed to build the splitPackedValue by themselves
+                // by assembling them from the values provided by getSplitDimValue for each dimension.
+                // Caution: This won't work if we ever support more than 1 dimension.
+                // But for 1 dimension, splitDimValue is the whole value we need.
+                byte[] splitPackedValue = tree.getSplitDimValue().bytes;
+                boolean goRight = shouldGoRight.test(splitPackedValue);
+
+                if (goRight)
+                    pushRight();
+                else
+                    pushLeft();
+            }
+        }
+
+        /**
+         * Goes to the right child of the current node.
+         * Updates the status of completeness of the current level based on the direction of the traversal.
+         */
+        void pushRight()
+        {
+            // In FORWARD direction we process the left child before the right.
+            // In BACKWARD direction we process the right child before the left.
+            // Therefore, if we're going right in FORWARD direction, this node is completed.
+            // Otherwise, if we're going right in BACKWARD direction, the left child remains to be processed, so this
+            // node is uncompleted.
+            completedLevels.set(tree.level, direction == Direction.FORWARD);
+            tree.pushRight();
+        }
+
+        /**
+         * Goes to the left child of the current node.
+         * Updates the status of completeness of the current level based on the direction of the traversal.
+         */
+        void pushLeft()
+        {
+            // In FORWARD direction we process the left child before the right.
+            // In BACKWARD direction we process the right child before the left.
+            // Therefore, if we're going left in BACKWARD direction, this node is completed.
+            // Otherwise, if we're going left in FORWARD direction, the right child remains to be processed, so this
+            // node is uncompleted.
+            completedLevels.set(tree.level, direction == Direction.BACKWARD);
+            tree.pushLeft();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDTreeRamBuffer.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDTreeRamBuffer.java
new file mode 100644
index 000000000000..3107f51be2a6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDTreeRamBuffer.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointValues;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedLongValues;
+
+/**
+ * On-heap buffer for point values that provides a sortable view of itself as {@link MutablePointValues}.
+ */
+public class BKDTreeRamBuffer implements Accountable
+{
+    @VisibleForTesting
+    public static int MAX_BLOCK_BYTE_POOL_SIZE = Integer.MAX_VALUE;
+    // This counter should not be used to track any other allocations, as we use it to prevent block pool overflow
+    private final Counter blockBytesUsed;
+    private final ByteBlockPool bytes;
+    private final int pointDimensionCount, pointNumBytes;
+    private final int packedBytesLength;
+    private final byte[] packedValue;
+    private final PackedLongValues.Builder docIDsBuilder;
+    private int numPoints;
+    private int numRows;
+    private int lastSegmentRowID = -1;
+    private boolean closed = false;
+
+    public BKDTreeRamBuffer(int pointDimensionCount, int pointNumBytes)
+    {
+        this.blockBytesUsed = Counter.newCounter();
+        this.pointDimensionCount = pointDimensionCount;
+        this.pointNumBytes = pointNumBytes;
+
+        this.bytes = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(blockBytesUsed));
+
+        packedValue = new byte[pointDimensionCount * pointNumBytes];
+        packedBytesLength = pointDimensionCount * pointNumBytes;
+
+        docIDsBuilder = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
+    }
+
+    @Override
+    public long ramBytesUsed()
+    {
+        return docIDsBuilder.ramBytesUsed() + blockBytesUsed.get();
+    }
+
+    public boolean requiresFlush()
+    {
+        // ByteBlockPool can't handle more than Integer.MAX_VALUE bytes. These are allocated in fixed-size chunks,
+        // and additions are guaranteed to be smaller than the chunks. This means that the last chunk allocation will
+        // be triggered by an addition, and the rest of the space in the final chunk will be wasted, as the bytesUsed
+        // counters track block allocation, not the size of additions. This means that we can't pass this check and then
+        // fail to add a term.
+        return blockBytesUsed.get() >= MAX_BLOCK_BYTE_POOL_SIZE;
+    }
+
+    public int numRows()
+    {
+        return numRows;
+    }
+
+    public long addPackedValue(int segmentRowId, BytesRef value)
+    {
+        ensureOpen();
+        
+        if (value.length != packedBytesLength)
+        {
+            throw new IllegalArgumentException("The value has length=" + value.length + " but should be " + pointDimensionCount * pointNumBytes);
+        }
+
+        long startingBlockBytesUsed = blockBytesUsed.get();
+        long startingDocIDsBytesUsed = docIDsBuilder.ramBytesUsed();
+
+        docIDsBuilder.add(segmentRowId);
+        bytes.append(value);
+
+        if (segmentRowId != lastSegmentRowID)
+        {
+            numRows++;
+            lastSegmentRowID = segmentRowId;
+        }
+
+        numPoints++;
+
+        long docIDsAllocatedBytes = docIDsBuilder.ramBytesUsed() - startingDocIDsBytesUsed;
+        long blockAllocatedBytes = blockBytesUsed.get() - startingBlockBytesUsed;
+        
+        return docIDsAllocatedBytes + blockAllocatedBytes;
+    }
+
+    public MutableOneDimPointValues asPointValues()
+    {
+        ensureOpen();
+        // building packed longs is destructive
+        closed = true;
+        final PackedLongValues docIDs = docIDsBuilder.build();
+        return new MutableOneDimPointValues()
+        {
+            final int[] ords = new int[numPoints];
+
+            {
+                for (int i = 0; i < numPoints; ++i)
+                {
+                    ords[i] = i;
+                }
+            }
+
+            @Override
+            public void getValue(int i, BytesRef packedValue)
+            {
+                final long offset = (long) packedBytesLength * (long) ords[i];
+                packedValue.length = packedBytesLength;
+                bytes.setRawBytesRef(packedValue, offset);
+            }
+
+            @Override
+            public byte getByteAt(int i, int k)
+            {
+                byte[] a = new byte[1];
+                final long offset = (long) packedBytesLength * (long) ords[i] + (long) k;
+                bytes.readBytes(offset, a, 0, 1);
+                return a[0];
+            }
+
+            @Override
+            public int getDocID(int i)
+            {
+                return Math.toIntExact(docIDs.get(ords[i]));
+            }
+
+            @Override
+            public void swap(int i, int j)
+            {
+                int tmp = ords[i];
+                ords[i] = ords[j];
+                ords[j] = tmp;
+            }
+
+            @Override
+            public void intersect(IntersectVisitor visitor) throws IOException
+            {
+                final BytesRef scratch = new BytesRef();
+                for (int i = 0; i < numPoints; i++)
+                {
+                    getValue(i, scratch);
+                    assert scratch.length == packedValue.length;
+                    System.arraycopy(scratch.bytes, scratch.offset, packedValue, 0, packedBytesLength);
+                    visitor.visit(getDocID(i), packedValue);
+                }
+            }
+
+            @Override
+            public int getNumDimensions()
+            {
+                return pointDimensionCount;
+            }
+
+            @Override
+            public int getBytesPerDimension()
+            {
+                return pointNumBytes;
+            }
+
+            @Override
+            public long size()
+            {
+                return numPoints;
+            }
+
+            @Override
+            public int getDocCount()
+            {
+                return numRows;
+            }
+        };
+    }
+
+    private void ensureOpen()
+    {
+        Preconditions.checkState(!closed, "Expected open buffer.");
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDWriter.java
new file mode 100644
index 000000000000..59096b9ac00e
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDWriter.java
@@ -0,0 +1,1039 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.IntFunction;
+
+import com.google.common.base.MoreObjects;
+
+import org.apache.cassandra.index.sai.disk.io.CryptoUtils;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.oldlucene.ByteBuffersDataOutputAdapter;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
+import org.apache.cassandra.index.sai.disk.oldlucene.ResettableByteBuffersIndexOutput;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointValues;
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointsReaderUtils;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.IntroSorter;
+import org.apache.lucene.util.LongBitSet;
+import org.apache.lucene.util.Sorter;
+
+// TODO
+//   - allow variable length byte[] (across docs and dims), but this is quite a bit more hairy
+//   - we could also index "auto-prefix terms" here, and use better compression, and maybe only use for the "fully contained" case so we'd
+//     only index docIDs
+//   - the index could be efficiently encoded as an FST, so we don't have wasteful
+//     (monotonic) long[] leafBlockFPs; or we could use MonotonicLongValues ... but then
+//     the index is already plenty small: 60M OSM points --> 1.1 MB with 128 points
+//     per leaf, and you can reduce that by putting more points per leaf
+//   - we could use threads while building; the higher nodes are very parallelizable
+
+/**
+ * Recursively builds a block KD-tree to assign all incoming points in N-dim space to smaller
+ * and smaller N-dim rectangles (cells) until the number of points in a given
+ * rectangle is &lt;= <code>maxPointsInLeafNode</code>.  The tree is
+ * fully balanced, which means the leaf nodes will have between 50% and 100% of
+ * the requested <code>maxPointsInLeafNode</code>.  Values that fall exactly
+ * on a cell boundary may be in either cell.
+ *
+ * <p>The number of dimensions can be 1 to 8, but every byte[] value is fixed length.
+ *
+ * <p>
+ * See <a href="https://www.cs.duke.edu/~pankaj/publications/papers/bkd-sstd.pdf">this paper</a> for details.
+ *
+ * <p>This consumes heap during writing: it allocates a <code>LongBitSet(numPoints)</code>,
+ * and then uses up to the specified {@code maxMBSortInHeap} heap space for writing.
+ *
+ * <p>
+ * <b>NOTE</b>: This can write at most Integer.MAX_VALUE * <code>maxPointsInLeafNode</code> total points.
+ *
+ * @lucene.experimental
+ */
+
+public class BKDWriter implements Closeable
+{
+    /** How many bytes each docs takes in the fixed-width offline format */
+    private final int bytesPerDoc;
+
+    /** Default maximum number of point in each leaf block */
+    public static final int DEFAULT_MAX_POINTS_IN_LEAF_NODE = 1024;
+
+    /** Default maximum heap to use, before spilling to (slower) disk */
+    public static final float DEFAULT_MAX_MB_SORT_IN_HEAP = 16.0f;
+
+    /** Maximum number of dimensions */
+    public static final int MAX_DIMS = 8;
+
+    /** How many dimensions we are indexing */
+    protected final int numDims;
+
+    /** How many bytes each value in each dimension takes. */
+    protected final int bytesPerDim;
+
+    /** numDims * bytesPerDim */
+    protected final int packedBytesLength;
+
+    final BytesRef scratchBytesRef1 = new BytesRef();
+    final int[] commonPrefixLengths;
+
+    protected final LongBitSet docsSeen;
+
+    protected final int maxPointsInLeafNode;
+    private final int maxPointsSortInHeap;
+
+    /** Minimum per-dim values, packed */
+    protected final byte[] minPackedValue;
+
+    /** Maximum per-dim values, packed */
+    protected final byte[] maxPackedValue;
+
+    protected long pointCount;
+
+    /** true if we have so many values that we must write ords using long (8 bytes) instead of int (4 bytes) */
+    protected final boolean longOrds;
+
+    /** An upper bound on how many points the caller will add (includes deletions) */
+    private final long totalPointCount;
+
+    private final long maxDoc;
+
+    private final ICompressor compressor;
+    private final ByteOrder order;
+
+    // reused when writing leaf blocks
+    private final ByteBuffersDataOutputAdapter scratchOut;
+    private final ByteBuffersDataOutputAdapter scratchOut2;
+
+    public BKDWriter(long maxDoc, int numDims, int bytesPerDim,
+            int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, boolean singleValuePerDoc,
+            ICompressor compressor, ByteOrder order) throws IOException
+    {
+        this(maxDoc, numDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
+             totalPointCount > Integer.MAX_VALUE, compressor, order);
+    }
+
+    protected BKDWriter(long maxDoc, int numDims, int bytesPerDim,
+                        int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount,
+                        boolean singleValuePerDoc, boolean longOrds, ICompressor compressor,
+                        ByteOrder order) throws IOException
+    {
+        verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
+        // We use tracking dir to deal with removing files on exception, so each place that
+        // creates temp files doesn't need crazy try/finally/sucess logic:
+        this.maxPointsInLeafNode = maxPointsInLeafNode;
+        this.numDims = numDims;
+        this.bytesPerDim = bytesPerDim;
+        this.totalPointCount = totalPointCount;
+        this.maxDoc = maxDoc;
+        this.compressor = compressor;
+        this.order = order;
+        docsSeen = new LongBitSet(maxDoc);
+        packedBytesLength = numDims * bytesPerDim;
+
+        commonPrefixLengths = new int[numDims];
+
+        minPackedValue = new byte[packedBytesLength];
+        maxPackedValue = new byte[packedBytesLength];
+
+        // If we may have more than 1+Integer.MAX_VALUE values, then we must encode ords with long (8 bytes), else we can use int (4 bytes).
+        this.longOrds = longOrds;
+
+        // dimensional values (numDims * bytesPerDim) + ord (int or long) + docID (int)
+        if (singleValuePerDoc)
+        {
+            // Lucene only supports up to 2.1 docs, so we better not need longOrds in this case:
+            assert longOrds == false;
+            bytesPerDoc = packedBytesLength + Integer.BYTES;
+        }
+        else if (longOrds)
+        {
+            bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES;
+        }
+        else
+        {
+            bytesPerDoc = packedBytesLength + Integer.BYTES + Integer.BYTES;
+        }
+
+        // As we recurse, we compute temporary partitions of the data, halving the
+        // number of points at each recursion.  Once there are few enough points,
+        // we can switch to sorting in heap instead of offline (on disk).  At any
+        // time in the recursion, we hold the number of points at that level, plus
+        // all recursive halves (i.e. 16 + 8 + 4 + 2) so the memory usage is 2X
+        // what that level would consume, so we multiply by 0.5 to convert from
+        // bytes to points here.  Each dimension has its own sorted partition, so
+        // we must divide by numDims as wel.
+
+        maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDims));
+
+        // Finally, we must be able to hold at least the leaf node in heap during build:
+        if (maxPointsSortInHeap < maxPointsInLeafNode)
+        {
+            throw new IllegalArgumentException("maxMBSortInHeap=" + maxMBSortInHeap + " only allows for maxPointsSortInHeap=" + maxPointsSortInHeap + ", but this is less than maxPointsInLeafNode=" + maxPointsInLeafNode + "; either increase maxMBSortInHeap or decrease maxPointsInLeafNode");
+        }
+
+        scratchOut = LuceneCompat.getByteBuffersDataOutputAdapter(order, 32 * 1024);
+        scratchOut2 = LuceneCompat.getByteBuffersDataOutputAdapter(order, 2 * 1024);
+    }
+
+    public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount)
+    {
+        // We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use
+        // remaining 4 bits for another purpose later
+        if (numDims < 1 || numDims > MAX_DIMS)
+        {
+            throw new IllegalArgumentException("numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");
+        }
+        if (maxPointsInLeafNode <= 0)
+        {
+            throw new IllegalArgumentException("maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
+        }
+        if (maxPointsInLeafNode > ArrayUtil.MAX_ARRAY_LENGTH)
+        {
+            throw new IllegalArgumentException("maxPointsInLeafNode must be <= ArrayUtil.MAX_ARRAY_LENGTH (= " + ArrayUtil.MAX_ARRAY_LENGTH + "); got " + maxPointsInLeafNode);
+        }
+        if (maxMBSortInHeap < 0.0)
+        {
+            throw new IllegalArgumentException("maxMBSortInHeap must be >= 0.0 (got: " + maxMBSortInHeap + ")");
+        }
+        if (totalPointCount < 0)
+        {
+            throw new IllegalArgumentException("totalPointCount must be >=0 (got: " + totalPointCount + ")");
+        }
+    }
+
+    /** How many points have been added so far */
+    public long getPointCount()
+    {
+        return pointCount;
+    }
+
+    /**
+     * Write a field from a {@link MutablePointValues}. This way of writing
+     * points is faster than regular writes with BKDWriter#add since
+     * there is opportunity for reordering points before writing them to
+     * disk. This method does not use transient disk in order to reorder points.
+     */
+    public long writeField(IndexOutput out, MutableOneDimPointValues reader,
+                           final OneDimensionBKDWriterCallback callback) throws IOException
+    {
+        if (numDims == 1)
+        {
+            SAICodecUtils.writeHeader(out);
+            final long fp = writeField1Dim(out, reader, callback);
+            SAICodecUtils.writeFooter(out);
+            return fp;
+        }
+        else
+        {
+            throw new IllegalArgumentException("Only 1 dimension is supported.");
+        }
+    }
+
+    /* In the 1D case, we can simply sort points in ascending order and use the
+     * same writing logic as we use at merge time. */
+    private long writeField1Dim(IndexOutput out, MutableOneDimPointValues reader,
+                                OneDimensionBKDWriterCallback callback) throws IOException
+    {
+        // TODO: cast to int
+        if (reader.size() > 1)
+            MutablePointsReaderUtils.sort(Math.toIntExact(maxDoc), packedBytesLength, reader, 0, Math.toIntExact(reader.size()));
+
+        final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out, callback);
+
+        reader.intersect((docID, packedValue) -> oneDimWriter.add(packedValue, docID));
+
+        return oneDimWriter.finish();
+    }
+
+    interface OneDimensionBKDWriterCallback
+    {
+        void writeLeafDocs(int leafNum, RowIDAndIndex[] leafDocs, int offset, int count);
+    }
+
+    public static class RowIDAndIndex
+    {
+        public int valueOrderIndex;
+        public int rowID;
+
+        @Override
+        public String toString()
+        {
+            return MoreObjects.toStringHelper(this)
+                              .add("valueOrderIndex", valueOrderIndex)
+                              .add("rowID", rowID)
+                              .toString();
+        }
+    }
+
+    private class OneDimensionBKDWriter
+    {
+
+        final IndexOutput out;
+        final List<Long> leafBlockFPs = new ArrayList<>();
+        final List<byte[]> leafBlockStartValues = new ArrayList<>();
+        final byte[] leafValues = new byte[maxPointsInLeafNode * packedBytesLength];
+        final int[] leafDocs = new int[maxPointsInLeafNode];
+        private long valueCount;
+        private int leafCount;
+        final RowIDAndIndex[] rowIDAndIndexes = new RowIDAndIndex[maxPointsInLeafNode];
+        final int[] orderIndex = new int[maxPointsInLeafNode];
+        final OneDimensionBKDWriterCallback callback;
+
+        {
+            for (int x = 0; x < rowIDAndIndexes.length; x++)
+            {
+                rowIDAndIndexes[x] = new RowIDAndIndex();
+            }
+        }
+
+        OneDimensionBKDWriter(IndexOutput out, OneDimensionBKDWriterCallback callback)
+        {
+            if (numDims != 1)
+            {
+                throw new UnsupportedOperationException("numDims must be 1 but got " + numDims);
+            }
+            if (pointCount != 0)
+            {
+                throw new IllegalStateException("cannot mix add and merge");
+            }
+
+            this.out = out;
+            this.callback = callback;
+
+            lastPackedValue = new byte[packedBytesLength];
+        }
+
+        // for asserts
+        final byte[] lastPackedValue;
+        private long lastDocID;
+
+        void add(byte[] packedValue, int docID) throws IOException
+        {
+            assert valueInOrder(valueCount + leafCount,
+                                0, lastPackedValue, packedValue, 0, docID, lastDocID);
+
+            System.arraycopy(packedValue, 0, leafValues, leafCount * packedBytesLength, packedBytesLength);
+            leafDocs[leafCount] = docID;
+            docsSeen.set(docID);
+            leafCount++;
+
+            if (valueCount > totalPointCount)
+            {
+                throw new IllegalStateException("totalPointCount=" + totalPointCount + " was passed when we were created, but we just hit " + pointCount + " values");
+            }
+
+            if (leafCount == maxPointsInLeafNode)
+            {
+                // We write a block once we hit exactly the max count ... this is different from
+                // when we write N > 1 dimensional points where we write between max/2 and max per leaf block
+                writeLeafBlock();
+                leafCount = 0;
+            }
+
+            assert (lastDocID = docID) >= 0; // only assign when asserts are enabled
+        }
+
+        public long finish() throws IOException
+        {
+            if (leafCount > 0)
+            {
+                writeLeafBlock();
+                leafCount = 0;
+            }
+
+            if (valueCount == 0)
+            {
+                return -1;
+            }
+
+            pointCount = valueCount;
+
+            long indexFP = out.getFilePointer();
+
+            int numInnerNodes = leafBlockStartValues.size();
+
+            //System.out.println("BKDW: now rotate numInnerNodes=" + numInnerNodes + " leafBlockStarts=" + leafBlockStartValues.size());
+
+            byte[] index = new byte[(1 + numInnerNodes) * (1 + bytesPerDim)];
+            rotateToTree(1, 0, numInnerNodes, index, leafBlockStartValues);
+            long[] arr = new long[leafBlockFPs.size()];
+            for (int i = 0; i < leafBlockFPs.size(); i++)
+            {
+                arr[i] = leafBlockFPs.get(i);
+            }
+            writeIndex(out, maxPointsInLeafNode, arr, index);
+            return indexFP;
+        }
+
+        private void writeLeafBlock() throws IOException
+        {
+            assert leafCount != 0;
+            if (valueCount == 0)
+            {
+                System.arraycopy(leafValues, 0, minPackedValue, 0, packedBytesLength);
+            }
+            System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedBytesLength);
+
+            valueCount += leafCount;
+
+            if (leafBlockFPs.size() > 0)
+            {
+                // Save the first (minimum) value in each leaf block except the first, to build the split value index in the end:
+                leafBlockStartValues.add(ArrayUtil.copyOfSubArray(leafValues, 0, packedBytesLength));
+            }
+            leafBlockFPs.add(out.getFilePointer());
+            checkMaxLeafNodeCount(leafBlockFPs.size());
+
+            // Find per-dim common prefix:
+            int prefix = bytesPerDim;
+            int offset = (leafCount - 1) * packedBytesLength;
+            for (int j = 0; j < bytesPerDim; j++)
+            {
+                if (leafValues[j] != leafValues[offset + j])
+                {
+                    prefix = j;
+                    break;
+                }
+            }
+
+            commonPrefixLengths[0] = prefix;
+
+            assert scratchOut.size() == 0;
+
+            out.writeVInt(leafCount);
+
+            for (int x = 0; x < leafCount; x++)
+            {
+                rowIDAndIndexes[x].valueOrderIndex = x;
+                rowIDAndIndexes[x].rowID = leafDocs[x];
+            }
+
+            final Sorter sorter = new IntroSorter()
+            {
+                RowIDAndIndex pivot;
+
+                @Override
+                protected void swap(int i, int j)
+                {
+                    RowIDAndIndex o = rowIDAndIndexes[i];
+                    rowIDAndIndexes[i] = rowIDAndIndexes[j];
+                    rowIDAndIndexes[j] = o;
+                }
+
+                @Override
+                protected void setPivot(int i)
+                {
+                    pivot = rowIDAndIndexes[i];
+                }
+
+                @Override
+                protected int comparePivot(int j)
+                {
+                    return Long.compare(pivot.rowID, rowIDAndIndexes[j].rowID);
+                }
+            };
+
+            sorter.sort(0, leafCount);
+
+            // write leaf rowID -> orig index
+            scratchOut2.reset();
+
+            // iterate in row ID order to get the row ID index for the given value order index
+            // place into an array to be written as packed ints
+            for (int x = 0; x < leafCount; x++)
+            {
+                final int valueOrderIndex = rowIDAndIndexes[x].valueOrderIndex;
+                orderIndex[valueOrderIndex] = x;
+            }
+
+            LeafOrderMap.write(order, orderIndex, leafCount, maxPointsInLeafNode - 1, scratchOut2);
+
+            int scratchSize = Math.toIntExact(scratchOut2.size());
+            out.writeVInt(scratchSize);
+            out.writeBytes(scratchOut2.toArrayCopy(), 0, scratchSize);
+
+            if (callback != null) callback.writeLeafDocs(leafBlockFPs.size() - 1, rowIDAndIndexes, 0, leafCount);
+
+            writeCommonPrefixes(scratchOut, commonPrefixLengths, leafValues);
+
+            scratchBytesRef1.length = packedBytesLength;
+            scratchBytesRef1.bytes = leafValues;
+
+            final IntFunction<BytesRef> packedValues = (i) -> {
+                    scratchBytesRef1.offset = packedBytesLength * i;
+                    return scratchBytesRef1;
+            };
+            assert valuesInOrderAndBounds(leafCount, 0, ArrayUtil.copyOfSubArray(leafValues, 0, packedBytesLength),
+                                          ArrayUtil.copyOfSubArray(leafValues, (leafCount - 1) * packedBytesLength, leafCount * packedBytesLength),
+                                          packedValues, leafDocs, 0);
+
+            writeLeafBlockPackedValues(scratchOut, commonPrefixLengths, leafCount, 0, packedValues);
+
+            if (compressor == null)
+            {
+                out.writeBytes(scratchOut.toArrayCopy(), 0, Math.toIntExact(scratchOut.size()));
+            }
+            else
+            {
+                CryptoUtils.compress(new BytesRef(scratchOut.toArrayCopy(), 0, Math.toIntExact(scratchOut.size())), scratchBytesRef, out, compressor);
+            }
+            scratchOut.reset();
+        }
+    }
+
+    private final BytesRef scratchBytesRef = new BytesRef(new byte[128]);
+
+    // TODO: there must be a simpler way?
+    private void rotateToTree(int nodeID, int offset, int count, byte[] index, List<byte[]> leafBlockStartValues)
+    {
+        //System.out.println("ROTATE: nodeID=" + nodeID + " offset=" + offset + " count=" + count + " bpd=" + bytesPerDim + " index.length=" + index.length);
+        if (count == 1)
+        {
+            // Leaf index node
+            //System.out.println("  leaf index node");
+            //System.out.println("  index[" + nodeID + "] = blockStartValues[" + offset + "]");
+            System.arraycopy(leafBlockStartValues.get(offset), 0, index, nodeID * (1 + bytesPerDim) + 1, bytesPerDim);
+        }
+        else if (count > 1)
+        {
+            // Internal index node: binary partition of count
+            int countAtLevel = 1;
+            int totalCount = 0;
+            while (true)
+            {
+                int countLeft = count - totalCount;
+                //System.out.println("    cycle countLeft=" + countLeft + " coutAtLevel=" + countAtLevel);
+                if (countLeft <= countAtLevel)
+                {
+                    // This is the last level, possibly partially filled:
+                    int lastLeftCount = Math.min(countAtLevel / 2, countLeft);
+                    assert lastLeftCount >= 0;
+                    int leftHalf = (totalCount - 1) / 2 + lastLeftCount;
+
+                    int rootOffset = offset + leftHalf;
+          /*
+          System.out.println("  last left count " + lastLeftCount);
+          System.out.println("  leftHalf " + leftHalf + " rightHalf=" + (count-leftHalf-1));
+          System.out.println("  rootOffset=" + rootOffset);
+          */
+
+                    System.arraycopy(leafBlockStartValues.get(rootOffset), 0, index, nodeID * (1 + bytesPerDim) + 1, bytesPerDim);
+                    //System.out.println("  index[" + nodeID + "] = blockStartValues[" + rootOffset + "]");
+
+                    // TODO: we could optimize/specialize, when we know it's simply fully balanced binary tree
+                    // under here, to save this while loop on each recursion
+
+                    // Recurse left
+                    rotateToTree(2 * nodeID, offset, leftHalf, index, leafBlockStartValues);
+
+                    // Recurse right
+                    rotateToTree(2 * nodeID + 1, rootOffset + 1, count - leftHalf - 1, index, leafBlockStartValues);
+                    return;
+                }
+                totalCount += countAtLevel;
+                countAtLevel *= 2;
+            }
+        }
+        else
+        {
+            assert count == 0;
+        }
+    }
+
+    // useful for debugging:
+  /*
+  private void printPathSlice(String desc, PathSlice slice, int dim) throws IOException {
+    System.out.println("    " + desc + " dim=" + dim + " count=" + slice.count + ":");    
+    try(PointReader r = slice.writer.getReader(slice.start, slice.count)) {
+      int count = 0;
+      while (r.next()) {
+        byte[] v = r.packedValue();
+        System.out.println("      " + count + ": " + new BytesRef(v, dim*bytesPerDim, bytesPerDim));
+        count++;
+        if (count == slice.count) {
+          break;
+        }
+      }
+    }
+  }
+  */
+
+    private void checkMaxLeafNodeCount(int numLeaves)
+    {
+        if ((1 + bytesPerDim) * (long) numLeaves > ArrayUtil.MAX_ARRAY_LENGTH)
+        {
+            throw new IllegalStateException("too many nodes; increase maxPointsInLeafNode (currently " + maxPointsInLeafNode + ") and reindex");
+        }
+    }
+
+    /** Packs the two arrays, representing a balanced binary tree, into a compact byte[] structure. */
+    @SuppressWarnings("resource")
+    private byte[] packIndex(long[] leafBlockFPs, byte[] splitPackedValues) throws IOException
+    {
+
+        int numLeaves = leafBlockFPs.length;
+
+        // Possibly rotate the leaf block FPs, if the index not fully balanced binary tree (only happens
+        // if it was created by OneDimensionBKDWriter).  In this case the leaf nodes may straddle the two bottom
+        // levels of the binary tree:
+        if (numDims == 1 && numLeaves > 1)
+        {
+            int levelCount = 2;
+            while (true)
+            {
+                if (numLeaves >= levelCount && numLeaves <= 2 * levelCount)
+                {
+                    int lastLevel = 2 * (numLeaves - levelCount);
+                    assert lastLevel >= 0;
+                    if (lastLevel != 0)
+                    {
+                        // Last level is partially filled, so we must rotate the leaf FPs to match.  We do this here, after loading
+                        // at read-time, so that we can still delta code them on disk at write:
+                        long[] newLeafBlockFPs = new long[numLeaves];
+                        System.arraycopy(leafBlockFPs, lastLevel, newLeafBlockFPs, 0, leafBlockFPs.length - lastLevel);
+                        System.arraycopy(leafBlockFPs, 0, newLeafBlockFPs, leafBlockFPs.length - lastLevel, lastLevel);
+                        leafBlockFPs = newLeafBlockFPs;
+                    }
+                    break;
+                }
+
+                levelCount *= 2;
+            }
+        }
+
+        // Reused while packing the index
+        var writeBuffer = LuceneCompat.getResettableByteBuffersIndexOutput(order, 1024, "");
+
+        // This is the "file" we append the byte[] to:
+        List<byte[]> blocks = new ArrayList<>();
+        byte[] lastSplitValues = new byte[bytesPerDim * numDims];
+        //System.out.println("\npack index");
+        int totalSize = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, 0l, blocks, 1, lastSplitValues, new boolean[numDims], false);
+
+        // Compact the byte[] blocks into single byte index:
+        byte[] index = new byte[totalSize];
+        int upto = 0;
+        for (byte[] block : blocks)
+        {
+            System.arraycopy(block, 0, index, upto, block.length);
+            upto += block.length;
+        }
+        assert upto == totalSize;
+
+        return index;
+    }
+
+    /** Appends the current contents of writeBuffer as another block on the growing in-memory file */
+    private int appendBlock(ResettableByteBuffersIndexOutput writeBuffer, List<byte[]> blocks) throws IOException
+    {
+        int pos = writeBuffer.intSize();
+        blocks.add(writeBuffer.toArrayCopy());
+        writeBuffer.reset();
+        return pos;
+    }
+
+    /**
+     * lastSplitValues is per-dimension split value previously seen; we use this to prefix-code the split byte[] on each
+     * inner node
+     */
+    private int recursePackIndex(ResettableByteBuffersIndexOutput writeBuffer, long[] leafBlockFPs, byte[] splitPackedValues, long minBlockFP, List<byte[]> blocks,
+                                 int nodeID, byte[] lastSplitValues, boolean[] negativeDeltas, boolean isLeft) throws IOException
+    {
+        if (nodeID >= leafBlockFPs.length)
+        {
+            int leafID = nodeID - leafBlockFPs.length;
+            //System.out.println("recursePack leaf nodeID=" + nodeID);
+
+            // In the unbalanced case it's possible the left most node only has one child:
+            if (leafID < leafBlockFPs.length)
+            {
+                long delta = leafBlockFPs[leafID] - minBlockFP;
+                if (isLeft)
+                {
+                    assert delta == 0;
+                    return 0;
+                }
+                else
+                {
+                    assert nodeID == 1 || delta > 0 : "nodeID=" + nodeID;
+                    writeBuffer.writeVLong(delta);
+                    return appendBlock(writeBuffer, blocks);
+                }
+            }
+            else
+            {
+                return 0;
+            }
+        }
+        else
+        {
+            long leftBlockFP;
+            if (isLeft == false)
+            {
+                leftBlockFP = getLeftMostLeafBlockFP(leafBlockFPs, nodeID);
+                long delta = leftBlockFP - minBlockFP;
+                assert nodeID == 1 || delta > 0;
+                writeBuffer.writeVLong(delta);
+            }
+            else
+            {
+                // The left tree's left most leaf block FP is always the minimal FP:
+                leftBlockFP = minBlockFP;
+            }
+
+            int address = nodeID * (1 + bytesPerDim);
+            int splitDim = splitPackedValues[address++] & 0xff;
+
+            //System.out.println("recursePack inner nodeID=" + nodeID + " splitDim=" + splitDim + " splitValue=" + new BytesRef(splitPackedValues, address, bytesPerDim));
+
+            // find common prefix with last split value in this dim:
+            int prefix = 0;
+            for (; prefix < bytesPerDim; prefix++)
+            {
+                if (splitPackedValues[address + prefix] != lastSplitValues[splitDim * bytesPerDim + prefix])
+                {
+                    break;
+                }
+            }
+
+            //System.out.println("writeNodeData nodeID=" + nodeID + " splitDim=" + splitDim + " numDims=" + numDims + " bytesPerDim=" + bytesPerDim + " prefix=" + prefix);
+
+            int firstDiffByteDelta;
+            if (prefix < bytesPerDim)
+            {
+                //System.out.println("  delta byte cur=" + Integer.toHexString(splitPackedValues[address+prefix]&0xFF) + " prev=" + Integer.toHexString(lastSplitValues[splitDim * bytesPerDim + prefix]&0xFF) + " negated?=" + negativeDeltas[splitDim]);
+                firstDiffByteDelta = (splitPackedValues[address + prefix] & 0xFF) - (lastSplitValues[splitDim * bytesPerDim + prefix] & 0xFF);
+                if (negativeDeltas[splitDim])
+                {
+                    firstDiffByteDelta = -firstDiffByteDelta;
+                }
+                //System.out.println("  delta=" + firstDiffByteDelta);
+                assert firstDiffByteDelta > 0;
+            }
+            else
+            {
+                firstDiffByteDelta = 0;
+            }
+
+            // pack the prefix, splitDim and delta first diff byte into a single vInt:
+            int code = (firstDiffByteDelta * (1 + bytesPerDim) + prefix) * numDims + splitDim;
+
+            //System.out.println("  code=" + code);
+            //System.out.println("  splitValue=" + new BytesRef(splitPackedValues, address, bytesPerDim));
+
+            writeBuffer.writeVInt(code);
+
+            // write the split value, prefix coded vs. our parent's split value:
+            int suffix = bytesPerDim - prefix;
+            byte[] savSplitValue = new byte[suffix];
+            if (suffix > 1)
+            {
+                writeBuffer.writeBytes(splitPackedValues, address + prefix + 1, suffix - 1);
+            }
+
+            byte[] cmp = lastSplitValues.clone();
+
+            System.arraycopy(lastSplitValues, splitDim * bytesPerDim + prefix, savSplitValue, 0, suffix);
+
+            // copy our split value into lastSplitValues for our children to prefix-code against
+            System.arraycopy(splitPackedValues, address + prefix, lastSplitValues, splitDim * bytesPerDim + prefix, suffix);
+
+            int numBytes = appendBlock(writeBuffer, blocks);
+
+            // placeholder for left-tree numBytes; we need this so that at search time if we only need to recurse into the right sub-tree we can
+            // quickly seek to its starting point
+            int idxSav = blocks.size();
+            blocks.add(null);
+
+            boolean savNegativeDelta = negativeDeltas[splitDim];
+            negativeDeltas[splitDim] = true;
+
+            int leftNumBytes = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, leftBlockFP, blocks, 2 * nodeID, lastSplitValues, negativeDeltas, true);
+
+            if (nodeID * 2 < leafBlockFPs.length)
+            {
+                writeBuffer.writeVInt(leftNumBytes);
+            }
+            else
+            {
+                assert leftNumBytes == 0 : "leftNumBytes=" + leftNumBytes;
+            }
+            byte[] bytes2 = writeBuffer.toArrayCopy();
+            int numBytes2 = bytes2.length;
+            writeBuffer.reset();
+            // replace our placeholder:
+            blocks.set(idxSav, bytes2);
+
+            negativeDeltas[splitDim] = false;
+            int rightNumBytes = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, leftBlockFP, blocks, 2 * nodeID + 1, lastSplitValues, negativeDeltas, false);
+
+            negativeDeltas[splitDim] = savNegativeDelta;
+
+            // restore lastSplitValues to what caller originally passed us:
+            System.arraycopy(savSplitValue, 0, lastSplitValues, splitDim * bytesPerDim + prefix, suffix);
+
+            assert Arrays.equals(lastSplitValues, cmp);
+
+            return numBytes + numBytes2 + leftNumBytes + rightNumBytes;
+        }
+    }
+
+    private long getLeftMostLeafBlockFP(long[] leafBlockFPs, int nodeID)
+    {
+        // TODO: can we do this cheaper, e.g. a closed form solution instead of while loop?  Or
+        // change the recursion while packing the index to return this left-most leaf block FP
+        // from each recursion instead?
+        //
+        // Still, the overall cost here is minor: this method's cost is O(log(N)), and while writing
+        // we call it O(N) times (N = number of leaf blocks)
+        while (nodeID < leafBlockFPs.length)
+        {
+            nodeID *= 2;
+        }
+        int leafID = nodeID - leafBlockFPs.length;
+        long result = leafBlockFPs[leafID];
+        if (result < 0)
+        {
+            throw new AssertionError(result + " for leaf " + leafID);
+        }
+        return result;
+    }
+
+    private void writeIndex(IndexOutput out, int countPerLeaf, long[] leafBlockFPs, byte[] splitPackedValues) throws IOException
+    {
+        byte[] packedIndex = packIndex(leafBlockFPs, splitPackedValues);
+        writeIndex(out, countPerLeaf, leafBlockFPs.length, packedIndex);
+    }
+
+    private void writeIndex(IndexOutput out, int countPerLeaf, int numLeaves, byte[] packedIndex) throws IOException
+    {
+        out.writeVInt(numDims);
+        out.writeVInt(countPerLeaf);
+        out.writeVInt(bytesPerDim);
+
+        assert numLeaves > 0;
+        out.writeVInt(numLeaves);
+
+        if (compressor != null)
+        {
+            var ramOut = LuceneCompat.getResettableByteBuffersIndexOutput(order, 1024, "");
+            ramOut.writeBytes(minPackedValue, 0, packedBytesLength);
+            ramOut.writeBytes(maxPackedValue, 0, packedBytesLength);
+
+            CryptoUtils.compress(new BytesRef(ramOut.toArrayCopy(), 0, (int)ramOut.getFilePointer()), out, compressor);
+        }
+        else
+        {
+            out.writeBytes(minPackedValue, 0, packedBytesLength);
+            out.writeBytes(maxPackedValue, 0, packedBytesLength);
+        }
+
+        out.writeVLong(pointCount);
+        //TODO Changing disk format
+        out.writeVLong(docsSeen.cardinality());
+
+        if (compressor != null)
+        {
+            CryptoUtils.compress(new BytesRef(packedIndex, 0, packedIndex.length), out, compressor);
+        }
+        else
+        {
+            out.writeVInt(packedIndex.length);
+            out.writeBytes(packedIndex, 0, packedIndex.length);
+        }
+    }
+
+    private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLengths, int count, int sortedDim, IntFunction<BytesRef> packedValues) throws IOException
+    {
+        int prefixLenSum = Arrays.stream(commonPrefixLengths).sum();
+        if (prefixLenSum == packedBytesLength)
+        {
+            // all values in this block are equal
+            out.writeByte((byte) -1);
+        }
+        else
+        {
+            assert numDims == 1;
+
+            assert commonPrefixLengths[sortedDim] < bytesPerDim;
+            out.writeByte((byte) sortedDim);
+            int compressedByteOffset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];
+            commonPrefixLengths[sortedDim]++;
+            for (int i = 0; i < count; )
+            {
+                // do run-length compression on the byte at compressedByteOffset
+                int runLen = runLen(packedValues, i, Math.min(i + 0xff, count), compressedByteOffset);
+                assert runLen <= 0xff;
+                BytesRef first = packedValues.apply(i);
+                byte prefixByte = first.bytes[first.offset + compressedByteOffset];
+                out.writeByte(prefixByte);
+                out.writeByte((byte) runLen);
+                writeLeafBlockPackedValuesRange(out, commonPrefixLengths, i, i + runLen, packedValues);
+                i += runLen;
+                assert i <= count;
+            }
+        }
+    }
+
+    /**
+     * Return an array that contains the min and max values for the [offset, offset+length] interval
+     * of the given {@link BytesRef}s.
+     */
+    private static BytesRef[] computeMinMax(int count, IntFunction<BytesRef> packedValues, int offset, int length)
+    {
+        assert length > 0;
+        BytesRefBuilder min = new BytesRefBuilder();
+        BytesRefBuilder max = new BytesRefBuilder();
+        BytesRef first = packedValues.apply(0);
+        min.copyBytes(first.bytes, first.offset + offset, length);
+        max.copyBytes(first.bytes, first.offset + offset, length);
+        for (int i = 1; i < count; ++i)
+        {
+            BytesRef candidate = packedValues.apply(i);
+            if (Arrays.compareUnsigned(min.bytes(), 0, length, candidate.bytes, candidate.offset + offset, candidate.offset + offset + length) > 0)
+            {
+                min.copyBytes(candidate.bytes, candidate.offset + offset, length);
+            }
+            else if (Arrays.compareUnsigned(max.bytes(), 0, length, candidate.bytes, candidate.offset + offset, candidate.offset + offset + length) < 0)
+            {
+                max.copyBytes(candidate.bytes, candidate.offset + offset, length);
+            }
+        }
+        return new BytesRef[]{ min.get(), max.get() };
+    }
+
+    private void writeLeafBlockPackedValuesRange(DataOutput out, int[] commonPrefixLengths, int start, int end, IntFunction<BytesRef> packedValues) throws IOException
+    {
+        for (int i = start; i < end; ++i)
+        {
+            BytesRef ref = packedValues.apply(i);
+            assert ref.length == packedBytesLength;
+
+            for (int dim = 0; dim < numDims; dim++)
+            {
+                int prefix = commonPrefixLengths[dim];
+                out.writeBytes(ref.bytes, ref.offset + dim * bytesPerDim + prefix, bytesPerDim - prefix);
+            }
+        }
+    }
+
+    private static int runLen(IntFunction<BytesRef> packedValues, int start, int end, int byteOffset)
+    {
+        BytesRef first = packedValues.apply(start);
+        byte b = first.bytes[first.offset + byteOffset];
+        for (int i = start + 1; i < end; ++i)
+        {
+            BytesRef ref = packedValues.apply(i);
+            byte b2 = ref.bytes[ref.offset + byteOffset];
+            assert Byte.toUnsignedInt(b2) >= Byte.toUnsignedInt(b);
+            if (b != b2)
+            {
+                return i - start;
+            }
+        }
+        return end - start;
+    }
+
+    private void writeCommonPrefixes(DataOutput out, int[] commonPrefixes, byte[] packedValue) throws IOException
+    {
+        for (int dim = 0; dim < numDims; dim++)
+        {
+            out.writeVInt(commonPrefixes[dim]);
+            //System.out.println(commonPrefixes[dim] + " of " + bytesPerDim);
+            out.writeBytes(packedValue, dim * bytesPerDim, commonPrefixes[dim]);
+        }
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+
+    }
+
+    /** Called only in assert */
+    private boolean valueInBounds(BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue)
+    {
+        for (int dim = 0; dim < numDims; dim++)
+        {
+            int offset = bytesPerDim * dim;
+            if (Arrays.compareUnsigned(packedValue.bytes, packedValue.offset + offset, packedValue.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0)
+            {
+                return false;
+            }
+            if (Arrays.compareUnsigned(packedValue.bytes, packedValue.offset + offset, packedValue.offset + offset + bytesPerDim, maxPackedValue, offset, offset + bytesPerDim) > 0)
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    // only called from assert
+    private boolean valuesInOrderAndBounds(int count, int sortedDim, byte[] minPackedValue, byte[] maxPackedValue,
+                                           IntFunction<BytesRef> values, int[] docs, int docsOffset) throws IOException
+    {
+        byte[] lastPackedValue = new byte[packedBytesLength];
+        long lastDoc = -1;
+        for (int i = 0; i < count; i++)
+        {
+            BytesRef packedValue = values.apply(i);
+            assert packedValue.length == packedBytesLength;
+            assert valueInOrder(i, sortedDim, lastPackedValue, packedValue.bytes, packedValue.offset,
+                                docs[docsOffset + i], lastDoc);
+            lastDoc = docs[docsOffset + i];
+
+            // Make sure this value does in fact fall within this leaf cell:
+            assert valueInBounds(packedValue, minPackedValue, maxPackedValue);
+        }
+        return true;
+    }
+
+    // only called from assert
+    private boolean valueInOrder(long ord, int sortedDim, byte[] lastPackedValue, byte[] packedValue, int packedValueOffset,
+                                 long doc, long lastDoc)
+    {
+        int dimOffset = sortedDim * bytesPerDim;
+        if (ord > 0)
+        {
+            int cmp = Arrays.compareUnsigned(lastPackedValue, dimOffset, dimOffset + bytesPerDim, packedValue, packedValueOffset + dimOffset, packedValueOffset + dimOffset + bytesPerDim);
+            if (cmp > 0)
+            {
+                throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue, packedValueOffset, packedBytesLength) + " ord=" + ord);
+            }
+            if (cmp == 0 && doc < lastDoc)
+            {
+                throw new AssertionError("docs out of order: last doc=" + lastDoc + " current doc=" + doc + " ord=" + ord);
+            }
+        }
+        System.arraycopy(packedValue, packedValueOffset, lastPackedValue, 0, packedBytesLength);
+        return true;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValues.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValues.java
new file mode 100644
index 000000000000..0a81fb5260d4
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValues.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointValues;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.apache.lucene.util.bkd.BKDWriter;
+
+/**
+ * {@link MutablePointValues} that prevents buffered points from reordering, and always skips sorting phase in Lucene
+ * It's the responsibility of the underlying implementation to ensure that all points are correctly sorted.
+ * <p>
+ * It allows to take advantage of an optimised 1-dim writer {@link BKDWriter}
+ * (that is enabled only for {@link MutablePointValues}), and reduce number of times we sort point values.
+ */
+public class ImmutableOneDimPointValues extends MutableOneDimPointValues
+{
+    private final TermsIterator termEnum;
+    private final byte[] scratch;
+
+    private ImmutableOneDimPointValues(TermsIterator termEnum, AbstractType<?> termComparator)
+    {
+        this.termEnum = termEnum;
+        this.scratch = new byte[TypeUtil.fixedSizeOf(termComparator)];
+    }
+
+    public static ImmutableOneDimPointValues fromTermEnum(TermsIterator termEnum, AbstractType<?> termComparator)
+    {
+        return new ImmutableOneDimPointValues(termEnum, termComparator);
+    }
+
+    @Override
+    public void intersect(IntersectVisitor visitor) throws IOException
+    {
+        while (termEnum.hasNext())
+        {
+            ByteSourceInverse.readBytesMustFit(((ByteComparable.Preencoded) termEnum.next()).getPreencodedBytes(),
+                                               scratch);
+
+            try (final PostingList postings = termEnum.postings())
+            {
+                int segmentRowId;
+                while ((segmentRowId = postings.nextPosting()) != PostingList.END_OF_STREAM)
+                {
+                    visitor.visit(segmentRowId, scratch);
+                }
+            }
+        }
+    }
+
+    @Override
+    public int getBytesPerDimension()
+    {
+        return scratch.length;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/LeafOrderMap.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/LeafOrderMap.java
new file mode 100644
index 000000000000..d74ce09aefa5
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/LeafOrderMap.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+
+import org.apache.cassandra.index.sai.disk.oldlucene.DirectWriterAdapter;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.LongValues;
+
+public class LeafOrderMap
+{
+    /**
+     * Get the value at the given index from the reader, and cast it to a short. If the value is too large to fit in a
+     * short, an ArithmeticException is thrown.
+     * @param index the index to read from
+     * @param reader the reader to read from
+     * @return the value at the given index, cast to a short
+     */
+    public static short getValue(int index, LongValues reader)
+    {
+        var value = reader.get(index);
+        var result = (short) value;
+        if (result != value) {
+            throw new ArithmeticException("short overflow");
+        }
+        return result;
+    }
+
+    public static void write(ByteOrder order, final int[] array, int length, int maxValue, final DataOutput out) throws IOException
+    {
+        final int bits = LuceneCompat.directWriterUnsignedBitsRequired(order, maxValue);
+        final DirectWriterAdapter writer = LuceneCompat.directWriterGetInstance(order, out, length, bits);
+        for (int i = 0; i < length; i++)
+        {
+            assert array[i] <= maxValue;
+
+            writer.add(array[i]);
+        }
+        writer.finish();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/MutableOneDimPointValues.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/MutableOneDimPointValues.java
new file mode 100644
index 000000000000..88c3d217ff62
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/MutableOneDimPointValues.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointValues;
+import org.apache.lucene.util.BytesRef;
+
+public abstract class MutableOneDimPointValues extends MutablePointValues
+{
+    private static final byte[] EMPTY = new byte[0];
+
+    public abstract void intersect(IntersectVisitor visitor) throws IOException;
+
+    @Override
+    public int getDocCount()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long size()
+    {
+        // hack to skip sorting in Lucene
+        return 1;
+    }
+
+    @Override
+    public void getValue(int i, BytesRef packedValue)
+    {
+        // no-op
+    }
+
+    @Override
+    public byte getByteAt(int i, int k)
+    {
+        return 0;
+    }
+
+    @Override
+    public int getDocID(int i)
+    {
+        return 0;
+    }
+
+    @Override
+    public void swap(int i, int j)
+    {
+        throw new IllegalStateException("unexpected sorting");
+    }
+
+    @Override
+    public byte[] getMinPackedValue()
+    {
+        return EMPTY;
+    }
+
+    @Override
+    public byte[] getMaxPackedValue()
+    {
+        return EMPTY;
+    }
+
+    @Override
+    public int getNumDimensions()
+    {
+        return 1;
+    }
+
+    @Override
+    public int getBytesPerDimension()
+    {
+        return 0;
+    }
+
+    public interface IntersectVisitor
+    {
+        /** Called for all documents in a leaf cell that crosses the query.  The consumer
+         *  should scrutinize the packedValue to decide whether to accept it.  In the 1D case,
+         *  values are visited in increasing order, and in the case of ties, in increasing
+         *  docID order. */
+        void visit(int docID, byte[] packedValue) throws IOException;
+    }
+
+    @Override
+    public int getNumIndexDimensions() throws IOException
+    {
+        return 1;
+    }
+
+    @Override
+    public PointTree getPointTree() throws IOException
+    {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriter.java
new file mode 100644
index 000000000000..5d9a2f21f0b4
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriter.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.MoreObjects;
+
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointValues;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedLongValues;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+
+/**
+ * Specialized writer for 1-dim point values, that builds them into a BKD tree with auxiliary posting lists on eligible
+ * tree levels.
+ *
+ * Given sorted input {@link MutablePointValues}, 1-dim case allows to optimise flush process, because we don't need to
+ * buffer all point values to sort them.
+ */
+public class NumericIndexWriter implements Closeable
+{
+    public static final int MAX_POINTS_IN_LEAF_NODE = BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE;
+    private final BKDWriter writer;
+    private final IndexComponents.ForWrite components;
+    private final int bytesPerDim;
+
+    private final IndexWriterConfig config;
+
+    /**
+     * @param maxSegmentRowId maximum possible segment row ID, used to create `maxDoc` for kd-tree
+     * @param numRows must be greater than number of added rowIds, only used for validation.
+     */
+    public NumericIndexWriter(IndexComponents.ForWrite components,
+                              int bytesPerDim,
+                              int maxSegmentRowId,
+                              int numRows,
+                              IndexWriterConfig config) throws IOException
+    {
+        this(components, MAX_POINTS_IN_LEAF_NODE, bytesPerDim, maxSegmentRowId, numRows, config);
+    }
+
+    public NumericIndexWriter(IndexComponents.ForWrite components,
+                              int maxPointsInLeafNode,
+                              int bytesPerDim,
+                              int maxSegmentRowId,
+                              int numRows,
+                              IndexWriterConfig config) throws IOException
+    {
+        checkArgument(maxSegmentRowId >= 0,
+                      "[%s] maxRowId must be non-negative value, but got %s",
+                      config.getIndexName(), maxSegmentRowId);
+
+        checkArgument(numRows >= 0,
+                      "[$s] numRows must be non-negative value, but got %s",
+                      config.getIndexName(), numRows);
+
+        this.components = components;
+        this.bytesPerDim = bytesPerDim;
+        this.config = config;
+        this.writer = new BKDWriter(maxSegmentRowId + 1L,
+                                    1,
+                                    bytesPerDim,
+                                    maxPointsInLeafNode,
+                                    BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP,
+                                    numRows,
+                                    true, null,
+                                    components.addOrGet(IndexComponentType.KD_TREE).byteOrder());
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        IOUtils.close(writer);
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("bytesPerDim", bytesPerDim)
+                          .add("bufferedPoints", writer.getPointCount())
+                          .toString();
+    }
+
+    public static class LeafCallback implements BKDWriter.OneDimensionBKDWriterCallback
+    {
+        final List<PackedLongValues> postings = new ArrayList<>();
+
+        public int numLeaves()
+        {
+            return postings.size();
+        }
+
+        @Override
+        public void writeLeafDocs(int leafNum, BKDWriter.RowIDAndIndex[] sortedByRowID, int offset, int count)
+        {
+            final PackedLongValues.Builder builder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
+
+            for (int i = offset; i < count; ++i)
+            {
+                builder.add(sortedByRowID[i].rowID);
+            }
+            postings.add(builder.build());
+        }
+    }
+
+    /**
+     * Writes a k-d tree and posting lists from a {@link MutablePointValues}.
+     *
+     * @param values points to write
+     *
+     * @return metadata describing the location and size of this kd-tree in the overall SSTable kd-tree component file
+     */
+    public SegmentMetadata.ComponentMetadataMap writeAll(MutableOneDimPointValues values) throws IOException
+    {
+        long bkdPosition;
+        final SegmentMetadata.ComponentMetadataMap components = new SegmentMetadata.ComponentMetadataMap();
+
+        final LeafCallback leafCallback = new LeafCallback();
+
+        try (IndexOutput bkdOutput = this.components.addOrGet(IndexComponentType.KD_TREE).openOutput(true))
+        {
+            // The SSTable kd-tree component file is opened in append mode, so our offset is the current file pointer.
+            final long bkdOffset = bkdOutput.getFilePointer();
+
+            bkdPosition = writer.writeField(bkdOutput, values, leafCallback);
+
+            // If the bkdPosition is less than 0 then we didn't write any values out
+            // and the index is empty
+            if (bkdPosition < 0)
+                return components;
+
+            final long bkdLength = bkdOutput.getFilePointer() - bkdOffset;
+
+            Map<String, String> attributes = new LinkedHashMap<>();
+            attributes.put("max_points_in_leaf_node", Integer.toString(writer.maxPointsInLeafNode));
+            attributes.put("num_leaves", Integer.toString(leafCallback.numLeaves()));
+            attributes.put("num_points", Long.toString(writer.pointCount));
+            attributes.put("bytes_per_dim", Long.toString(writer.bytesPerDim));
+            attributes.put("num_dims", Long.toString(writer.numDims));
+
+            components.put(IndexComponentType.KD_TREE, bkdPosition, bkdOffset, bkdLength, attributes);
+        }
+
+        try (TraversingBKDReader reader = new TraversingBKDReader(this.components.get(IndexComponentType.KD_TREE).createIndexBuildTimeFileHandle(), bkdPosition);
+             IndexOutput postingsOutput = this.components.addOrGet(IndexComponentType.KD_TREE_POSTING_LISTS).openOutput(true))
+        {
+            final long postingsOffset = postingsOutput.getFilePointer();
+
+            final OneDimBKDPostingsWriter postingsWriter = new OneDimBKDPostingsWriter(leafCallback.postings, config, this.components::logMessage);
+            reader.traverse(postingsWriter);
+
+            // The kd-tree postings writer already writes its own header & footer.
+            final long postingsPosition = postingsWriter.finish(postingsOutput);
+
+            Map<String, String> attributes = new LinkedHashMap<>();
+            attributes.put("num_leaf_postings", Integer.toString(postingsWriter.numLeafPostings));
+            attributes.put("num_non_leaf_postings", Integer.toString(postingsWriter.numNonLeafPostings));
+
+            long postingsLength = postingsOutput.getFilePointer() - postingsOffset;
+            components.put(IndexComponentType.KD_TREE_POSTING_LISTS, postingsPosition, postingsOffset, postingsLength, attributes);
+        }
+
+        return components;
+    }
+
+    /**
+     * @return number of points added
+     */
+    public long getPointCount()
+    {
+        return writer.getPointCount();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/OneDimBKDPostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/OneDimBKDPostingsWriter.java
new file mode 100644
index 000000000000..b3cfa6c32b34
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/OneDimBKDPostingsWriter.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
+import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList;
+import org.apache.cassandra.index.sai.disk.v1.postings.PackedLongsPostingList;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.lucene.util.packed.PackedLongValues;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+/**
+ * Writes auxiliary posting lists for bkd tree nodes. If a node has a posting list attached, it will contain every row
+ * id
+ * from all leaves reachable from that node.
+ *
+ * Writer is stateful, because it needs to collect data from bkd index data structure first to find set of eligible
+ * nodes and leaf nodes reachable from them.
+ *
+ * This is an optimised writer for 1-dim points, where we know that leaf blocks are written in value order (in this
+ * order we pass them to the {@link BKDWriter}). That allows us to skip reading the leaves, instead just order leaf
+ * blocks by their offset in the index file, and correlate them with buffered posting lists. We can't make this
+ * assumption for multi-dim case.
+ */
+public class OneDimBKDPostingsWriter implements TraversingBKDReader.IndexTreeTraversalCallback
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private final List<PackedLongValues> postings;
+    private final TreeMap<Long, Integer> leafOffsetToNodeID = new TreeMap<>(Long::compareTo);
+    private final Multimap<Integer, Integer> nodeToChildLeaves = HashMultimap.create();
+
+    private final IndexWriterConfig config;
+    private final Function<String, String> logMessage;
+    int numNonLeafPostings = 0;
+    int numLeafPostings = 0;
+
+    OneDimBKDPostingsWriter(List<PackedLongValues> postings, IndexWriterConfig config, Function<String, String> logMessage)
+    {
+        this.postings = postings;
+        this.config = config;
+        this.logMessage = logMessage;
+    }
+
+    @Override
+    public void onLeaf(int leafNodeID, long leafBlockFP, IntArrayList pathToRoot)
+    {
+        checkArgument(!pathToRoot.containsInt(leafNodeID));
+        checkArgument(pathToRoot.isEmpty() || leafNodeID > pathToRoot.get(pathToRoot.size() - 1));
+
+        leafOffsetToNodeID.put(leafBlockFP, leafNodeID);
+        for (int i = 0; i < pathToRoot.size(); i++)
+        {
+            final int level = i + 1;
+            if (isLevelEligibleForPostingList(level))
+            {
+                final int nodeID = pathToRoot.get(i);
+                nodeToChildLeaves.put(nodeID, leafNodeID);
+            }
+        }
+    }
+
+    @SuppressWarnings("resource")
+    public long finish(IndexOutput out) throws IOException
+    {
+        checkState(postings.size() == leafOffsetToNodeID.size(),
+                   "Expected equal number of postings lists (%s) and leaf offsets (%s).",
+                   postings.size(), leafOffsetToNodeID.size());
+
+        final PostingsWriter postingsWriter = new PostingsWriter(out);
+
+        final Iterator<PackedLongValues> postingsIterator = postings.iterator();
+        final Map<Integer, PackedLongValues> leafToPostings = new HashMap<>();
+        leafOffsetToNodeID.forEach((fp, nodeID) -> leafToPostings.put(nodeID, postingsIterator.next()));
+
+        final long postingsRamBytesUsed = postings.stream()
+                                                  .mapToLong(PackedLongValues::ramBytesUsed)
+                                                  .sum();
+
+        final List<Integer> internalNodeIDs =
+                nodeToChildLeaves.keySet()
+                                 .stream()
+                                 .filter(i -> nodeToChildLeaves.get(i).size() >= config.getBkdPostingsMinLeaves())
+                                 .collect(Collectors.toList());
+
+        final Collection<Integer> leafNodeIDs = leafOffsetToNodeID.values();
+
+        logger.debug(logMessage.apply("Writing posting lists for {} internal and {} leaf kd-tree nodes. Leaf postings memory usage: {}."),
+                     internalNodeIDs.size(),
+                     leafNodeIDs.size(),
+                     FBUtilities.prettyPrintMemory(postingsRamBytesUsed));
+
+        final long startFP = out.getFilePointer();
+        final Stopwatch flushTime = Stopwatch.createStarted();
+        final TreeMap<Integer, Long> nodeIDToPostingsFilePointer = new TreeMap<>();
+        for (int nodeID : Iterables.concat(internalNodeIDs, leafNodeIDs))
+        {
+            Collection<Integer> leaves = nodeToChildLeaves.get(nodeID);
+
+            if (leaves.isEmpty())
+            {
+                leaves = Collections.singletonList(nodeID);
+                numLeafPostings++;
+            }
+            else
+            {
+                numNonLeafPostings++;
+            }
+
+            var postingLists = new ArrayList<PostingList>(leaves.size());
+            for (Integer leaf : leaves)
+                postingLists.add(new PackedLongsPostingList(leafToPostings.get(leaf)));
+
+            final PostingList mergedPostingList = MergePostingList.merge(postingLists);
+            final long postingFilePosition = postingsWriter.write(mergedPostingList);
+            // During compaction we could end up with an empty postings due to deletions.
+            // The writer will return a fp of -1 if no postings were written.
+            if (postingFilePosition >= 0)
+                nodeIDToPostingsFilePointer.put(nodeID, postingFilePosition);
+        }
+        flushTime.stop();
+        logger.debug(logMessage.apply("Flushed {} of posting lists for kd-tree nodes in {} ms."),
+                     FBUtilities.prettyPrintMemory(out.getFilePointer() - startFP),
+                     flushTime.elapsed(TimeUnit.MILLISECONDS));
+
+
+        final long indexFilePointer = out.getFilePointer();
+        writeMap(nodeIDToPostingsFilePointer, out);
+        postingsWriter.complete();
+        return indexFilePointer;
+    }
+
+    private boolean isLevelEligibleForPostingList(int level)
+    {
+        return level > 1 && level % config.getBkdPostingsSkip() == 0;
+    }
+
+    private void writeMap(Map<Integer, Long> map, IndexOutput out) throws IOException
+    {
+        out.writeVInt(map.size());
+
+        for (Map.Entry<Integer, Long> e : map.entrySet())
+        {
+            out.writeVInt(e.getKey());
+            out.writeVLong(e.getValue());
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/TraversingBKDReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/TraversingBKDReader.java
new file mode 100644
index 000000000000..e163352787d6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/TraversingBKDReader.java
@@ -0,0 +1,445 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.Closeable;
+import java.util.Arrays;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.MathUtil;
+
+/**
+ * Base reader for a block KD-tree previously written with {@link BKDWriter}.
+ *
+ * Holds index tree on heap and enables it's traversal via {@link #traverse(IndexTreeTraversalCallback)}.
+ */
+public class TraversingBKDReader implements Closeable
+{
+    final FileHandle indexFile;
+    final int bytesPerDim;
+    final int numLeaves;
+    final byte[] minPackedValue;
+    final byte[] maxPackedValue;
+    // Packed array of byte[] holding all split values in the full binary tree:
+    final byte[] packedIndex;
+    final long pointCount;
+    final int leafNodeOffset;
+    final int numDims;
+    final int maxPointsInLeafNode;
+    final int bitsPerValue;
+    final int packedBytesLength;
+
+    @SuppressWarnings("resource")
+    TraversingBKDReader(FileHandle indexFile, long root)
+    {
+        this.indexFile = indexFile;
+
+        try (final IndexInputReader in = IndexInputReader.create(indexFile.createReader()))
+        {
+            SAICodecUtils.validate(in);
+            in.seek(root);
+
+            numDims = in.readVInt();
+            maxPointsInLeafNode = in.readVInt();
+            bitsPerValue = LuceneCompat.directWriterUnsignedBitsRequired(in.order(), maxPointsInLeafNode - 1);
+            bytesPerDim = in.readVInt();
+            packedBytesLength = numDims * bytesPerDim;
+
+            // Read index:
+            numLeaves = in.readVInt();
+            assert numLeaves > 0;
+            leafNodeOffset = numLeaves;
+
+            minPackedValue = new byte[packedBytesLength];
+            maxPackedValue = new byte[packedBytesLength];
+
+            in.readBytes(minPackedValue, 0, packedBytesLength);
+            in.readBytes(maxPackedValue, 0, packedBytesLength);
+
+            for (int dim = 0; dim < numDims; dim++)
+            {
+                if (Arrays.compareUnsigned(minPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, maxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) > 0)
+                {
+                    String message = String.format("Min packed value %s is > max packed value %s for dimension %d.",
+                                                   new BytesRef(minPackedValue), new BytesRef(maxPackedValue), dim);
+                    throw new CorruptIndexException(message, in);
+                }
+            }
+
+            pointCount = in.readVLong();
+
+            // docCount, unused
+            in.readVLong();
+
+            int numBytes = in.readVInt();
+            packedIndex = new byte[numBytes];
+            in.readBytes(packedIndex, 0, numBytes);
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(indexFile);
+            throw Throwables.unchecked(t);
+        }
+    }
+
+    public long getMinLeafBlockFP()
+    {
+        if (packedIndex != null)
+        {
+            return new ByteArrayDataInput(packedIndex).readVLong();
+        }
+        else
+        {
+            throw new IllegalStateException();
+        }
+    }
+
+    public long memoryUsage()
+    {
+        return ObjectSizes.sizeOfArray(packedIndex)
+               + ObjectSizes.sizeOfArray(minPackedValue)
+               + ObjectSizes.sizeOfArray(maxPackedValue);
+    }
+
+    @Override
+    public void close()
+    {
+        indexFile.close();
+    }
+
+    interface IndexTreeTraversalCallback
+    {
+        void onLeaf(int leafNodeID, long leafBlockFP, IntArrayList pathToRoot);
+    }
+
+    /**
+     * Copy of BKDReader.IndexTree
+     */
+    abstract class IndexTree implements Cloneable
+    {
+        protected int nodeID;
+        // level is 1-based so that we can do level-1 w/o checking each time:
+        protected int level;
+        protected int splitDim;
+        protected final byte[][] splitPackedValueStack;
+
+        protected IndexTree()
+        {
+            int treeDepth = getTreeDepth();
+            splitPackedValueStack = new byte[treeDepth + 1][];
+            nodeID = 1;
+            level = 1;
+            splitPackedValueStack[level] = new byte[packedBytesLength];
+        }
+
+        public void pushLeft()
+        {
+            nodeID *= 2;
+            level++;
+            if (splitPackedValueStack[level] == null)
+            {
+                splitPackedValueStack[level] = new byte[packedBytesLength];
+            }
+        }
+
+        /** Clone, but you are not allowed to pop up past the point where the clone happened. */
+        public abstract IndexTree clone();
+
+        public void pushRight()
+        {
+            nodeID = nodeID * 2 + 1;
+            level++;
+            if (splitPackedValueStack[level] == null)
+            {
+                splitPackedValueStack[level] = new byte[packedBytesLength];
+            }
+        }
+
+        public void pop()
+        {
+            nodeID /= 2;
+            level--;
+            splitDim = -1;
+            //System.out.println("  pop nodeID=" + nodeID);
+        }
+
+        public boolean isLeafNode()
+        {
+            return nodeID >= leafNodeOffset;
+        }
+
+        public boolean nodeExists()
+        {
+            return nodeID - leafNodeOffset < leafNodeOffset;
+        }
+
+        public int getNodeID()
+        {
+            return nodeID;
+        }
+
+        public byte[] getSplitPackedValue()
+        {
+            assert !isLeafNode();
+            assert splitPackedValueStack[level] != null : "level=" + level;
+            return splitPackedValueStack[level];
+        }
+
+        /** Only valid after pushLeft or pushRight, not pop! */
+        public int getSplitDim()
+        {
+            assert !isLeafNode();
+            return splitDim;
+        }
+
+        /** Only valid after pushLeft or pushRight, not pop! */
+        public abstract BytesRef getSplitDimValue();
+
+        /** Only valid after pushLeft or pushRight, not pop! */
+        public abstract long getLeafBlockFP();
+    }
+
+
+    /**
+     * Copy of BKDReader.PackedIndexTree
+     */
+    final class PackedIndexTree extends IndexTree
+    {
+        // used to read the packed byte[]
+        private final ByteArrayDataInput in;
+        // holds the minimum (left most) leaf block file pointer for each level we've recursed to:
+        private final long[] leafBlockFPStack;
+        // holds the address, in the packed byte[] index, of the left-node of each level:
+        private final int[] leftNodePositions;
+        // holds the address, in the packed byte[] index, of the right-node of each level:
+        private final int[] rightNodePositions;
+        // holds the splitDim for each level:
+        private final int[] splitDims;
+        // true if the per-dim delta we read for the node at this level is a negative offset vs. the last split on this dim; this is a packed
+        // 2D array, i.e. to access array[level][dim] you read from negativeDeltas[level*numDims+dim].  this will be true if the last time we
+        // split on this dimension, we next pushed to the left sub-tree:
+        private final boolean[] negativeDeltas;
+        // holds the packed per-level split values; the run method uses this to save the cell min/max as it recurses:
+        private final byte[][] splitValuesStack;
+        // scratch value to return from getPackedValue:
+        private final BytesRef scratch;
+
+        PackedIndexTree()
+        {
+            int treeDepth = getTreeDepth();
+            leafBlockFPStack = new long[treeDepth + 1];
+            leftNodePositions = new int[treeDepth + 1];
+            rightNodePositions = new int[treeDepth + 1];
+            splitValuesStack = new byte[treeDepth + 1][];
+            splitDims = new int[treeDepth + 1];
+            negativeDeltas = new boolean[numDims * (treeDepth + 1)];
+
+            in = new ByteArrayDataInput(packedIndex);
+            splitValuesStack[0] = new byte[packedBytesLength];
+            readNodeData(false);
+            scratch = new BytesRef();
+            scratch.length = bytesPerDim;
+        }
+
+        @Override
+        public PackedIndexTree clone()
+        {
+            PackedIndexTree index = new PackedIndexTree();
+            index.nodeID = nodeID;
+            index.level = level;
+            index.splitDim = splitDim;
+            index.leafBlockFPStack[level] = leafBlockFPStack[level];
+            index.leftNodePositions[level] = leftNodePositions[level];
+            index.rightNodePositions[level] = rightNodePositions[level];
+            index.splitValuesStack[index.level] = splitValuesStack[index.level].clone();
+            System.arraycopy(negativeDeltas, level * numDims, index.negativeDeltas, level * numDims, numDims);
+            index.splitDims[level] = splitDims[level];
+            return index;
+        }
+
+        @Override
+        public void pushLeft()
+        {
+            int nodePosition = leftNodePositions[level];
+            super.pushLeft();
+            System.arraycopy(negativeDeltas, (level - 1) * numDims, negativeDeltas, level * numDims, numDims);
+            assert splitDim != -1;
+            negativeDeltas[level * numDims + splitDim] = true;
+            in.setPosition(nodePosition);
+            readNodeData(true);
+        }
+
+        @Override
+        public void pushRight()
+        {
+            int nodePosition = rightNodePositions[level];
+            super.pushRight();
+            System.arraycopy(negativeDeltas, (level - 1) * numDims, negativeDeltas, level * numDims, numDims);
+            assert splitDim != -1;
+            negativeDeltas[level * numDims + splitDim] = false;
+            in.setPosition(nodePosition);
+            readNodeData(false);
+        }
+
+        @Override
+        public void pop()
+        {
+            super.pop();
+            splitDim = splitDims[level];
+        }
+
+        @Override
+        public long getLeafBlockFP()
+        {
+            assert isLeafNode() : "nodeID=" + nodeID + " is not a leaf";
+            return leafBlockFPStack[level];
+        }
+
+        @Override
+        public BytesRef getSplitDimValue()
+        {
+            assert !isLeafNode();
+            scratch.bytes = splitValuesStack[level];
+            scratch.offset = splitDim * bytesPerDim;
+            return scratch;
+        }
+
+        private void readNodeData(boolean isLeft)
+        {
+
+            leafBlockFPStack[level] = leafBlockFPStack[level - 1];
+
+            // read leaf block FP delta
+            if (!isLeft)
+            {
+                leafBlockFPStack[level] += in.readVLong();
+            }
+
+            if (isLeafNode())
+            {
+                splitDim = -1;
+            }
+            else
+            {
+
+                // read split dim, prefix, firstDiffByteDelta encoded as int:
+                int code = in.readVInt();
+                splitDim = code % numDims;
+                splitDims[level] = splitDim;
+                code /= numDims;
+                int prefix = code % (1 + bytesPerDim);
+                int suffix = bytesPerDim - prefix;
+
+                if (splitValuesStack[level] == null)
+                {
+                    splitValuesStack[level] = new byte[packedBytesLength];
+                }
+                System.arraycopy(splitValuesStack[level - 1], 0, splitValuesStack[level], 0, packedBytesLength);
+                if (suffix > 0)
+                {
+                    int firstDiffByteDelta = code / (1 + bytesPerDim);
+                    if (negativeDeltas[level * numDims + splitDim])
+                    {
+                        firstDiffByteDelta = -firstDiffByteDelta;
+                    }
+                    int oldByte = splitValuesStack[level][splitDim * bytesPerDim + prefix] & 0xFF;
+                    splitValuesStack[level][splitDim * bytesPerDim + prefix] = (byte) (oldByte + firstDiffByteDelta);
+                    in.readBytes(splitValuesStack[level], splitDim * bytesPerDim + prefix + 1, suffix - 1);
+                }
+                else
+                {
+                    // our split value is == last split value in this dim, which can happen when there are many duplicate values
+                }
+
+                int leftNumBytes;
+                if (nodeID * 2 < leafNodeOffset)
+                {
+                    leftNumBytes = in.readVInt();
+                }
+                else
+                {
+                    leftNumBytes = 0;
+                }
+
+                leftNodePositions[level] = in.getPosition();
+                rightNodePositions[level] = leftNodePositions[level] + leftNumBytes;
+            }
+        }
+    }
+
+
+    void traverse(IndexTreeTraversalCallback callback)
+    {
+        traverse(callback,
+                 new PackedIndexTree(),
+                 new IntArrayList());
+    }
+
+    private void traverse(IndexTreeTraversalCallback callback,
+                          IndexTree index,
+                          IntArrayList pathToRoot)
+    {
+        if (index.isLeafNode())
+        {
+            // In the unbalanced case it's possible the left most node only has one child:
+            if (index.nodeExists())
+            {
+                callback.onLeaf(index.getNodeID(), index.getLeafBlockFP(), pathToRoot);
+            }
+        }
+        else
+        {
+            final int nodeID = index.getNodeID();
+            final IntArrayList currentPath = new IntArrayList();
+            currentPath.addAll(pathToRoot);
+            currentPath.add(nodeID);
+
+            index.pushLeft();
+            traverse(callback, index, currentPath);
+            index.pop();
+
+            index.pushRight();
+            traverse(callback, index, currentPath);
+            index.pop();
+        }
+    }
+
+    /**
+     * Copy of BKDReader#getTreeDepth()
+     */
+    private int getTreeDepth()
+    {
+        // First +1 because all the non-leave nodes makes another power
+        // of 2; e.g. to have a fully balanced tree with 4 leaves you
+        // need a depth=3 tree:
+
+        // Second +1 because MathUtil.log computes floor of the logarithm; e.g.
+        // with 5 leaves you need a depth=4 tree:
+        return MathUtil.log(numLeaves, 2) + 2;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookup.java b/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookup.java
deleted file mode 100644
index 439db532cd03..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookup.java
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.keystore;
-
-import java.io.IOException;
-import javax.annotation.Nonnull;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
-import org.apache.cassandra.index.sai.disk.v1.LongArray;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.MonotonicBlockPackedReader;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.utils.FastByteOperations;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-
-/**
- * Provides read access to an on-disk sequence of partition or clustering keys written by {@link KeyStoreWriter}.
- * <p>
- * Care has been taken to make this structure as efficient as possible.
- * Reading keys does not require allocating data heap buffers per each read operation.
- * Only one key at a time is loaded to memory.
- * Low complexity algorithms are used – a lookup of the key by point id is constant time,
- * and a lookup of the point id by the key is logarithmic.
- * <p>
- * Because the blocks are prefix compressed, random access applies only to the locating the whole block.
- * In order to jump to a concrete key inside the block, the block keys are iterated from the block beginning.
- *
- * @see KeyStoreWriter
- */
-@NotThreadSafe
-public class KeyLookup
-{
-    public static final String INDEX_OUT_OF_BOUNDS = "The target point id [%d] cannot be less than 0 or greater than or equal to the key count [%d]";
-
-    private final FileHandle keysFileHandle;
-    private final KeyLookupMeta keyLookupMeta;
-    private final LongArray.Factory keyBlockOffsetsFactory;
-
-    /**
-     * Creates a new reader based on its data components.
-     * <p>
-     * It does not own the components, so you must close them separately after you're done with the reader.
-     * @param keysFileHandle handle to the file with a sequence of prefix-compressed blocks
-     *                       each storing a fixed number of keys
-     * @param keysBlockOffsets handle to the file containing an encoded sequence of the file offsets pointing to the blocks
-     * @param keyLookupMeta metadata object created earlier by the writer
-     * @param keyBlockOffsetsMeta metadata object for the block offsets
-     */
-    public KeyLookup(@Nonnull FileHandle keysFileHandle,
-                     @Nonnull FileHandle keysBlockOffsets,
-                     @Nonnull KeyLookupMeta keyLookupMeta,
-                     @Nonnull NumericValuesMeta keyBlockOffsetsMeta) throws IOException
-    {
-        this.keysFileHandle = keysFileHandle;
-        this.keyLookupMeta = keyLookupMeta;
-        this.keyBlockOffsetsFactory = new MonotonicBlockPackedReader(keysBlockOffsets, keyBlockOffsetsMeta);
-    }
-
-    /**
-     * Opens a cursor over the keys stored in the keys file.
-     * <p>
-     * This will read the first key into the key buffer and point to the first point in the keys file.
-     * <p>
-     * The cursor is to be used in a single thread.
-     * The cursor is valid as long this object hasn't been closed.
-     * You must close the cursor when you no longer need it.
-     */
-    public @Nonnull Cursor openCursor() throws IOException
-    {
-        return new Cursor(keysFileHandle, keyBlockOffsetsFactory);
-    }
-
-    /**
-     * Allows reading the keys from the keys file.
-     * Can quickly seek to a random key by point id.
-     * <p>
-     * This object is stateful and not thread safe.
-     * It maintains a position to the current key as well as a buffer that can hold one key.
-     */
-    @NotThreadSafe
-    public class Cursor implements AutoCloseable
-    {
-        private final IndexInputReader keysInput;
-        private final int blockShift;
-        private final int blockMask;
-        private final boolean clustering;
-        private final long keysFilePointer;
-        private final LongArray blockOffsets;
-
-        // The key the cursor currently points to. Initially empty.
-        private final BytesRef currentKey;
-
-        // A temporary buffer used to hold the key at the start of the next block.
-        private final BytesRef nextBlockKey;
-
-        // The point id the cursor currently points to.
-        private long currentPointId;
-        private long currentBlockIndex;
-
-        Cursor(FileHandle keysFileHandle, LongArray.Factory blockOffsetsFactory) throws IOException
-        {
-            this.keysInput = IndexInputReader.create(keysFileHandle);
-            SAICodecUtils.validate(this.keysInput);
-            this.blockShift = this.keysInput.readVInt();
-            this.blockMask = (1 << this.blockShift) - 1;
-            this.clustering = this.keysInput.readByte() == 1;
-            this.keysFilePointer = this.keysInput.getFilePointer();
-            this.blockOffsets = new LongArray.DeferredLongArray(blockOffsetsFactory::open);
-            this.currentKey = new BytesRef(keyLookupMeta.maxKeyLength);
-            this.nextBlockKey = new BytesRef(keyLookupMeta.maxKeyLength);
-            keysInput.seek(keysFilePointer);
-            readKey(currentPointId, currentKey);
-        }
-
-        /**
-         * Positions the cursor on the target point id and reads the key at the target to the current key buffer.
-         * <p>
-         * It is allowed to position the cursor before the first item or after the last item;
-         * in these cases the internal buffer is cleared.
-         *
-         * @param pointId point id to lookup
-         * @return The {@link ByteSource} containing the key
-         * @throws IndexOutOfBoundsException if the target point id is less than -1 or greater than the number of keys
-         */
-        public @Nonnull ByteSource seekToPointId(long pointId)
-        {
-            if (pointId < 0 || pointId >= keyLookupMeta.keyCount)
-                throw new IndexOutOfBoundsException(String.format(INDEX_OUT_OF_BOUNDS, pointId, keyLookupMeta.keyCount));
-
-            if (pointId != currentPointId)
-            {
-                long blockIndex = pointId >>> blockShift;
-                // We need to reset the block if the block index has changed or the pointId < currentPointId.
-                // We can read forward in the same block without a reset, but we can't read backwards, and token
-                // collision can result in us moving backwards.
-                if (blockIndex != currentBlockIndex || pointId < currentPointId)
-                {
-                    currentBlockIndex = blockIndex;
-                    resetToCurrentBlock();
-                }
-            }
-            while (currentPointId < pointId)
-            {
-                currentPointId++;
-                readCurrentKey();
-                updateCurrentBlockIndex(currentPointId);
-            }
-
-            return ByteSource.fixedLength(currentKey.bytes, currentKey.offset, currentKey.length);
-        }
-
-        /**
-         * Finds the pointId for a clustering key within a range of pointIds. The start and end of the range must not
-         * exceed the number of keys available. The keys within the range are expected to be in lexographical order.
-         * <p>
-         * If the key is not in the block containing the start of the range a binary search is done to find
-         * the block containing the search key. That block is then searched to return the pointId that corresponds
-         * to the key that is either equal to or next highest to the search key.
-         *
-         * @param key The key to seek for with the partition
-         * @param startingPointId the inclusive starting point for the partition
-         * @param endingPointId the exclusive ending point for the partition.
-         *                      Note: this can be equal to the number of keys if this is the last partition
-         * @return a {@code long} representing the pointId of the key that is >= to the key passed to the method, or
-         *         -1 if the key passed is > all the keys.
-         */
-        public long clusteredSeekToKey(ByteComparable key, long startingPointId, long endingPointId)
-        {
-            assert clustering : "Cannot do a clustered seek to a key on non-clustered keys";
-
-            BytesRef searchKey = asBytesRef(key);
-
-            updateCurrentBlockIndex(startingPointId);
-            resetToCurrentBlock();
-
-            // We can return immediately if the currentPointId is within the requested partition range and the keys match
-            if (currentPointId >= startingPointId && currentPointId < endingPointId && compareKeys(currentKey, searchKey) == 0)
-                return currentPointId;
-
-            // Now do a binary search over the range if points between [lowSearchId, highSearchId)
-            long lowSearchId = startingPointId;
-            long highSearchId = endingPointId;
-
-            // We will keep going with the binary shift while the search consists of at least one block
-            while ((highSearchId - lowSearchId) >>> blockShift > 0)
-            {
-                long midSearchId = lowSearchId + (highSearchId - lowSearchId) / 2;
-
-                // See if the searchkey exists in the block containing the midSearchId or is above or below it
-                int position = moveToBlockAndCompareTo(midSearchId, searchKey);
-
-                if (position == 0)
-                {
-                    lowSearchId = currentPointId;
-                    break;
-                }
-
-                if (position < 0)
-                    highSearchId = midSearchId;
-                else
-                    lowSearchId = midSearchId;
-            }
-
-            updateCurrentBlockIndex(lowSearchId);
-            resetToCurrentBlock();
-
-            // Depending on where we are in the block we may need to move forwards to the starting point ID
-            while (currentPointId < startingPointId)
-            {
-                currentPointId++;
-                readCurrentKey();
-                updateCurrentBlockIndex(currentPointId);
-            }
-
-            // Move forward to the ending point ID, returning the point ID if we find our key
-            while (currentPointId < endingPointId)
-            {
-                if (compareKeys(currentKey, searchKey) >= 0)
-                    return currentPointId;
-
-                currentPointId++;
-                if (currentPointId == keyLookupMeta.keyCount)
-                    return -1;
-
-                readCurrentKey();
-                updateCurrentBlockIndex(currentPointId);
-            }
-            return endingPointId < keyLookupMeta.keyCount ? endingPointId : -1;
-        }
-
-        @VisibleForTesting
-        public void reset() throws IOException
-        {
-            currentPointId = 0;
-            currentBlockIndex = 0;
-            keysInput.seek(keysFilePointer);
-            readCurrentKey();
-        }
-
-        @Override
-        public void close()
-        {
-            keysInput.close();
-        }
-
-        // Move to a block and see if the key is in the block using compareTo logic to indicate the keys position
-        // relative to the block.
-        // Note: It is down to the caller to position the block after a call to this method.
-        private int moveToBlockAndCompareTo(long pointId, BytesRef key)
-        {
-            updateCurrentBlockIndex(pointId);
-            resetToCurrentBlock();
-
-            if (compareKeys(key, currentKey) < 0)
-                return -1;
-
-            // If we are in the last block we will assume for now that the key is in the last block and defer
-            // the final decision to later (if we can't find it).
-            if (currentBlockIndex == blockOffsets.length() -1)
-                return 0;
-
-            // Finish by getting the starting key of the next block and comparing that with the key.
-            keysInput.seek(blockOffsets.get(currentBlockIndex + 1) + keysFilePointer);
-            readKey((currentBlockIndex + 1) << blockShift, nextBlockKey);
-            return compareKeys(key, nextBlockKey) < 0 ? 0 : 1;
-        }
-
-        private void updateCurrentBlockIndex(long pointId)
-        {
-            currentBlockIndex = pointId >>> blockShift;
-        }
-
-        // Reset currentPointId and currentKey to be at the start of the block pointed to by currentBlockIndex.
-        private void resetToCurrentBlock()
-        {
-
-            keysInput.seek(blockOffsets.get(currentBlockIndex) + keysFilePointer);
-            currentPointId = currentBlockIndex << blockShift;
-            readCurrentKey();
-        }
-
-        private void readCurrentKey()
-        {
-            readKey(currentPointId, currentKey);
-        }
-
-        // Read the next key indicated by pointId.
-        //
-        // Note: pointId is only used to determine whether we are at the start of a block. It is
-        // important that resetPosition is called prior to multiple calls to readKey. It is
-        // easy to get out of position.
-        private void readKey(long pointId, BytesRef key)
-        {
-            try
-            {
-                int prefixLength;
-                int suffixLength;
-                if ((pointId & blockMask) == 0L)
-                {
-                    prefixLength = 0;
-                    suffixLength = keysInput.readVInt();
-                }
-                else
-                {
-                    // Read the prefix and suffix lengths following the compression mechanism described
-                    // in the KeyStoreWriterWriter. If the lengths contained in the starting byte are less
-                    // than the 4 bit maximum then nothing further is read. Otherwise, the lengths in the
-                    // following vints are added.
-                    int compressedLengths = Byte.toUnsignedInt(keysInput.readByte());
-                    prefixLength = compressedLengths & 0x0F;
-                    suffixLength = compressedLengths >>> 4;
-                    if (prefixLength == 15)
-                        prefixLength += keysInput.readVInt();
-                    if (suffixLength == 15)
-                        suffixLength += keysInput.readVInt();
-                }
-
-                assert prefixLength + suffixLength <= keyLookupMeta.maxKeyLength;
-                if (prefixLength + suffixLength > 0)
-                {
-                    key.length = prefixLength + suffixLength;
-                    // The currentKey is appended to as the suffix for the current key is
-                    // added to the existing prefix.
-                    keysInput.readBytes(key.bytes, prefixLength, suffixLength);
-                }
-            }
-            catch (IOException e)
-            {
-                throw Throwables.cleaned(e);
-            }
-        }
-
-        private int compareKeys(BytesRef left, BytesRef right)
-        {
-            return FastByteOperations.compareUnsigned(left.bytes, left.offset, left.offset + left.length,
-                                                      right.bytes, right.offset, right.offset + right.length);
-        }
-
-        private BytesRef asBytesRef(ByteComparable source)
-        {
-            BytesRefBuilder builder = new BytesRefBuilder();
-
-            ByteSource byteSource = source.asComparableBytes(ByteComparable.Version.OSS50);
-            int val;
-            while ((val = byteSource.next()) != ByteSource.END_OF_STREAM)
-                builder.append((byte) val);
-            return builder.get();
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookupMeta.java b/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookupMeta.java
deleted file mode 100644
index ac57e9a1a719..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookupMeta.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.keystore;
-
-import java.io.IOException;
-
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexOutput;
-
-/**
- * Metadata produced by {@link KeyStoreWriter}, needed by {@link KeyLookup}.
- */
-public class KeyLookupMeta
-{
-    public final long keyCount;
-    public final int maxKeyLength;
-
-    public KeyLookupMeta(DataInput input) throws IOException
-    {
-        this.keyCount = input.readLong();
-        this.maxKeyLength = input.readInt();
-    }
-
-    public static void write(IndexOutput output, long keyCount, int maxKeyLength) throws IOException
-    {
-        output.writeLong(keyCount);
-        output.writeInt(maxKeyLength);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyStoreWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyStoreWriter.java
deleted file mode 100644
index b95c355e7b76..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/keystore/KeyStoreWriter.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.keystore;
-
-import java.io.Closeable;
-import java.io.IOException;
-import javax.annotation.Nonnull;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.FastByteOperations;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.StringHelper;
-
-/**
- * Writes a sequence of partition keys or clustering keys for use with {@link KeyLookup}.
- * <p>
- * Partition keys are written unordered and clustering keys are written in ordered partitions determined by calls to
- * {@link #startPartition()}. In either case keys can be of varying lengths.
- * <p>
- * The {@link #blockShift} field is used to quickly determine the id of the current block
- * based on a point id or to check if we are exactly at the beginning of the block.
- * <p>
- * Keys are organized in blocks of (2 ^ {@link #blockShift}) keys.
- * <p>
- * The blocks should not be too small because they allow prefix compression of the keys except the first key in a block.
- * <p>
- * The blocks should not be too large because we can't just randomly jump to the key inside the block, but we have to
- * iterate through all the keys from the start of the block.
- *
- * @see KeyLookup
- */
-@NotThreadSafe
-public class KeyStoreWriter implements Closeable
-{
-    private final int blockShift;
-    private final int blockMask;
-    private final boolean clustering;
-    private final IndexOutput keysOutput;
-    private final NumericValuesWriter offsetsWriter;
-    private final String componentName;
-    private final MetadataWriter metadataWriter;
-
-    private BytesRefBuilder prevKey = new BytesRefBuilder();
-    private BytesRefBuilder tempKey = new BytesRefBuilder();
-
-    private final long bytesStartFP;
-
-    private boolean inPartition = false;
-    private int maxKeyLength = -1;
-    private long pointId = 0;
-
-    /**
-     * Creates a new writer.
-     * <p>
-     * It does not own the components, so you must close the components by yourself
-     * after you're done with the writer.
-     *
-     * @param componentName the component name for the {@link KeyLookupMeta}
-     * @param metadataWriter the {@link MetadataWriter} for storing the {@link KeyLookupMeta}
-     * @param keysOutput where to write the prefix-compressed keys
-     * @param keysBlockOffsets  where to write the offsets of each block of keys
-     * @param blockShift the block shift that is used to determine the block size
-     * @param clustering determines whether the keys will be written as ordered partitions
-     */
-    public KeyStoreWriter(String componentName,
-                          MetadataWriter metadataWriter,
-                          IndexOutput keysOutput,
-                          NumericValuesWriter keysBlockOffsets,
-                          int blockShift,
-                          boolean clustering) throws IOException
-    {
-        this.componentName = componentName;
-        this.metadataWriter = metadataWriter;
-        SAICodecUtils.writeHeader(keysOutput);
-        this.blockShift = blockShift;
-        this.blockMask = (1 << this.blockShift) - 1;
-        this.clustering = clustering;
-        this.keysOutput = keysOutput;
-        this.keysOutput.writeVInt(blockShift);
-        this.keysOutput.writeByte((byte ) (clustering ? 1 : 0));
-        this.bytesStartFP = keysOutput.getFilePointer();
-        this.offsetsWriter = keysBlockOffsets;
-    }
-
-    public void startPartition()
-    {
-        assert clustering : "Cannot start a partition on a non-clustering key store";
-
-        inPartition = false;
-    }
-
-    /**
-     * Appends a key at the end of the sequence.
-     *
-     * @throws IOException if write to disk fails
-     * @throws IllegalArgumentException if the key is not greater than the previous added key
-     */
-    public void add(final @Nonnull ByteComparable key) throws IOException
-    {
-        tempKey.clear();
-        copyBytes(key, tempKey);
-
-        BytesRef keyRef = tempKey.get();
-
-        if (clustering && inPartition)
-        {
-            if (compareKeys(keyRef, prevKey.get()) <= 0)
-                throw new IllegalArgumentException("Clustering keys must be in ascending lexographical order");
-        }
-
-        inPartition = true;
-
-        writeKey(keyRef);
-
-        maxKeyLength = Math.max(maxKeyLength, keyRef.length);
-
-        BytesRefBuilder temp = this.tempKey;
-        this.tempKey = this.prevKey;
-        this.prevKey = temp;
-
-        pointId++;
-    }
-
-    private void writeKey(BytesRef key) throws IOException
-    {
-        if ((pointId & blockMask) == 0)
-        {
-            offsetsWriter.add(keysOutput.getFilePointer() - bytesStartFP);
-
-            keysOutput.writeVInt(key.length);
-            keysOutput.writeBytes(key.bytes, key.offset, key.length);
-        }
-        else
-        {
-            int prefixLength = 0;
-            int suffixLength = 0;
-
-            // If the key is the same as the previous key then we use prefix and suffix lengths of 0.
-            // This means that we store a byte of 0 and don't write any data for the key.
-            if (compareKeys(prevKey.get(), key) != 0)
-            {
-                prefixLength = StringHelper.bytesDifference(prevKey.get(), key);
-                suffixLength = key.length - prefixLength;
-            }
-            // The prefix and suffix lengths are written as a byte followed by up to 2 vints. An attempt is
-            // made to compress the lengths into the byte (if prefix length < 15 and/or suffix length < 15).
-            // If either length exceeds the compressed byte maximum, it is written as a vint following the byte.
-            keysOutput.writeByte((byte) (Math.min(prefixLength, 15) | (Math.min(15, suffixLength) << 4)));
-
-            if (prefixLength + suffixLength > 0)
-            {
-                if (prefixLength >= 15)
-                    keysOutput.writeVInt(prefixLength - 15);
-                if (suffixLength >= 15)
-                    keysOutput.writeVInt(suffixLength - 15);
-
-                keysOutput.writeBytes(key.bytes, key.offset + prefixLength, key.length - prefixLength);
-            }
-        }
-    }
-
-    /**
-     * Flushes any in-memory buffers to the output streams.
-     * Does not close the output streams.
-     * No more writes are allowed.
-     */
-    @Override
-    public void close() throws IOException
-    {
-        try (IndexOutput output = metadataWriter.builder(componentName))
-        {
-            SAICodecUtils.writeFooter(keysOutput);
-            KeyLookupMeta.write(output, pointId, maxKeyLength);
-        }
-        finally
-        {
-            FileUtils.close(offsetsWriter, keysOutput);
-        }
-    }
-
-    private int compareKeys(BytesRef left, BytesRef right)
-    {
-        return FastByteOperations.compareUnsigned(left.bytes, left.offset, left.offset + left.length,
-                                                  right.bytes, right.offset, right.offset + right.length);
-    }
-
-    private void copyBytes(ByteComparable source, BytesRefBuilder dest)
-    {
-        ByteSource byteSource = source.asComparableBytes(ByteComparable.Version.OSS50);
-        int val;
-        while ((val = byteSource.next()) != ByteSource.END_OF_STREAM)
-            dest.append((byte) val);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/FilteringPostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/FilteringPostingList.java
index 9140e358c961..22433ee8d5c9 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/FilteringPostingList.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/FilteringPostingList.java
@@ -19,10 +19,12 @@
 
 import java.io.IOException;
 
-import org.apache.cassandra.index.sai.postings.OrdinalPostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
 import org.apache.lucene.util.FixedBitSet;
 
+
 /**
  * A wrapper that iterates over a delegate {@link PostingList}, filtering out postings at
  * positions that are not present in a provided filter.
@@ -37,12 +39,15 @@ public class FilteringPostingList implements PostingList
     public FilteringPostingList(FixedBitSet filter, OrdinalPostingList delegate)
     {
         cardinality = filter.cardinality();
+
+        Preconditions.checkArgument(cardinality > 0, "Filter must contain at least one match.");
+
         this.filter = filter;
         this.delegate = delegate;
     }
 
     @Override
-    public void close()
+    public void close() throws IOException
     {
         delegate.close();
     }
@@ -52,11 +57,11 @@ public void close()
      * @return the segment row ID of the next match
      */
     @Override
-    public long nextPosting() throws IOException
+    public int nextPosting() throws IOException
     {
         while (true)
         {
-            long segmentRowId = delegate.nextPosting();
+            int segmentRowId = delegate.nextPosting();
 
             if (segmentRowId == PostingList.END_OF_STREAM)
             {
@@ -71,23 +76,23 @@ public long nextPosting() throws IOException
     }
 
     @Override
-    public long size()
+    public int size()
     {
         return cardinality;
     }
 
     @Override
-    public  long advance(long targetRowID) throws IOException
+    public int advance(int targetRowID) throws IOException
     {
-        long segmentRowId = delegate.advance(targetRowID);
+        int segmentRowId = delegate.advance(targetRowID);
 
         if (segmentRowId == PostingList.END_OF_STREAM)
         {
             return PostingList.END_OF_STREAM;
         }
 
-        // these are always for leaf balanced tree postings so the max is 1024
-        position = (int)delegate.getOrdinal();
+        // these are always for leaf kdtree postings so the max is 1024
+        position = delegate.getOrdinal();
 
         // If the ordinal of the ID we just read satisfies the filter, just return it...
         if (filter.get(position - 1))
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/IntersectingPostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/IntersectingPostingList.java
new file mode 100644
index 000000000000..295796c9066f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/IntersectingPostingList.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * Performs intersection operations on multiple PostingLists, returning only postings
+ * that appear in all inputs.
+ */
+@NotThreadSafe
+public class IntersectingPostingList implements PostingList
+{
+    private final Map<ByteBuffer, PostingList> postingsByTerm;
+    private final List<PostingList> postingLists; // so we can access by ordinal in intersection code
+    private final int size;
+
+    private IntersectingPostingList(Map<ByteBuffer, PostingList> postingsByTerm)
+    {
+        if (postingsByTerm.isEmpty())
+            throw new AssertionError();
+        this.postingsByTerm = postingsByTerm;
+        this.postingLists = new ArrayList<>(postingsByTerm.values());
+        this.size = postingLists.stream()
+                                .mapToInt(PostingList::size)
+                                .min()
+                                .orElse(0);
+    }
+
+    /**
+     * @return the intersection of the provided term-posting list mappings
+     */
+    public static IntersectingPostingList intersect(Map<ByteBuffer, PostingList> postingsByTerm)
+    {
+        // TODO optimize cases where
+        // - we have a single postinglist
+        // - any posting list is empty (intersection also empty)
+        return new IntersectingPostingList(postingsByTerm);
+    }
+
+    @Override
+    public int nextPosting() throws IOException
+    {
+        return findNextIntersection(Integer.MIN_VALUE, false);
+    }
+
+    @Override
+    public int advance(int targetRowID) throws IOException
+    {
+        assert targetRowID >= 0 : targetRowID;
+        return findNextIntersection(targetRowID, true);
+    }
+
+    @Override
+    public int frequency()
+    {
+        // call frequencies() instead
+        throw new UnsupportedOperationException();
+    }
+
+    public Map<ByteBuffer, Integer> frequencies()
+    {
+        Map<ByteBuffer, Integer> result = new HashMap<>();
+        for (Map.Entry<ByteBuffer, PostingList> entry : postingsByTerm.entrySet())
+            result.put(entry.getKey(), entry.getValue().frequency());
+        return result;
+    }
+
+    private int findNextIntersection(int targetRowID, boolean isAdvance) throws IOException
+    {
+        int maxRowId = targetRowID;
+        int maxRowIdIndex = -1;
+
+        // Scan through all posting lists looking for a common row ID
+        for (int i = 0; i < postingLists.size(); i++)
+        {
+            // don't advance the sublist in which we found our current max
+            if (i == maxRowIdIndex)
+                continue;
+
+            // Advance this sublist to the current max, special casing the first one as needed
+            PostingList list = postingLists.get(i);
+            int rowId = (isAdvance || maxRowIdIndex >= 0)
+                        ? list.advance(maxRowId)
+                        : list.nextPosting();
+            if (rowId == END_OF_STREAM)
+                return END_OF_STREAM;
+
+            // Update maxRowId + index if we find a larger value, or this was the first sublist evaluated
+            if (rowId > maxRowId || maxRowIdIndex < 0)
+            {
+                maxRowId = rowId;
+                maxRowIdIndex = i;
+                i = -1; // restart the scan with new maxRowId
+            }
+        }
+
+        // Once we complete a full scan without finding a larger rowId, we've found an intersection
+        return maxRowId;
+    }
+
+    @Override
+    public int size()
+    {
+        return size;
+    }
+
+    @Override
+    public void close()
+    {
+        for (PostingList list : postingLists)
+            FileUtils.closeQuietly(list);
+    }
+}
+
+
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingList.java
index 39516d8f6b8d..a4aaa916368b 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingList.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingList.java
@@ -17,140 +17,81 @@
  */
 package org.apache.cassandra.index.sai.disk.v1.postings;
 
-import java.io.Closeable;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
-import java.util.PriorityQueue;
 import javax.annotation.concurrent.NotThreadSafe;
 
-import org.apache.cassandra.index.sai.postings.PeekablePostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.index.sai.disk.PostingList;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.IntMerger;
 
 import static com.google.common.base.Preconditions.checkArgument;
 
 /**
  * Merges multiple {@link PostingList} which individually contain unique items into a single list.
- * While the individual lists contain unique items, there can be duplicate items between lists so
- * the class also checks for duplicates and only returns unique items in sorted order.
  */
 @NotThreadSafe
-public class MergePostingList implements PostingList
+public class MergePostingList extends IntMerger<PostingList> implements PostingList
 {
-    private final PriorityQueue<PeekablePostingList> postingLists;
-    private final List<PeekablePostingList> temp;
-    private final Closeable onClose;
-    private final long minimum;
-    private final long maximum;
-    private final long size;
-    private long lastRowId = -1;
+    final int size;
 
-    private MergePostingList(PriorityQueue<PeekablePostingList> postingLists, Closeable onClose)
+    private MergePostingList(List<? extends PostingList> postingLists)
     {
-        this.temp = new ArrayList<>(postingLists.size());
-        this.onClose = onClose;
-        this.postingLists = postingLists;
-        long minimum = 0;
-        long maximum = 0;
+        super(postingLists, PostingList.class);
+        checkArgument(!postingLists.isEmpty());
         long totalPostings = 0;
         for (PostingList postingList : postingLists)
-        {
-            minimum = Math.min(minimum, postingList.minimum());
-            maximum = Math.max(maximum, postingList.maximum());
             totalPostings += postingList.size();
-        }
-        this.minimum = minimum;
-        this.maximum = maximum;
-        this.size = totalPostings;
-    }
-
-    public static PostingList merge(PriorityQueue<PeekablePostingList> postings, Closeable onClose)
-    {
-        checkArgument(!postings.isEmpty(), "Cannot merge an empty queue of posting lists");
-        return postings.size() > 1 ? new MergePostingList(postings, onClose) : postings.poll();
-    }
 
-    public static PostingList merge(PriorityQueue<PeekablePostingList> postings)
-    {
-        return merge(postings, () -> FileUtils.close(postings));
+        // We could technically "overflow" integer if enough row ids are duplicated in the source posting lists.
+        // The size does not affect correctness, so just use integer max if that happens.
+        this.size = (int) Math.min(totalPostings, Integer.MAX_VALUE);
     }
 
     public static PostingList merge(List<PostingList> postings)
     {
-        PriorityQueue<PeekablePostingList> postingsQueue = new PriorityQueue<>(postings.size(), Comparator.comparingLong(PeekablePostingList::peek));
-        postings.stream().map(PeekablePostingList::makePeekable).forEach(postingsQueue::add);
-        return merge(postingsQueue);
+        if (postings.isEmpty())
+            return PostingList.EMPTY;
+
+        if (postings.size() == 1)
+            return postings.get(0);
+
+        return new MergePostingList(postings);
     }
 
     @Override
-    public long minimum()
+    public int nextPosting() throws IOException
     {
-        return minimum;
+        return advance();
     }
 
     @Override
-    public long maximum()
+    public int advance(int targetRowID) throws IOException
     {
-        return maximum;
+        return skipTo(targetRowID);
     }
 
     @Override
-    public long nextPosting() throws IOException
+    public int size()
     {
-        while (!postingLists.isEmpty())
-        {
-            PeekablePostingList head = postingLists.poll();
-            long next = head.nextPosting();
-
-            if (next == END_OF_STREAM)
-            {
-                // skip current posting list
-                continue;
-            }
-
-            if (next > lastRowId)
-            {
-                lastRowId = next;
-                postingLists.add(head);
-                return next;
-            }
-            else if (next == lastRowId)
-            {
-                postingLists.add(head);
-            }
-        }
-
-        return PostingList.END_OF_STREAM;
+        return size;
     }
 
     @Override
-    public long advance(long targetRowID) throws IOException
+    public void close()
     {
-        temp.clear();
-
-        while (!postingLists.isEmpty())
-        {
-            PeekablePostingList peekable = postingLists.poll();
-            peekable.advanceWithoutConsuming(targetRowID);
-            if (peekable.peek() != PostingList.END_OF_STREAM)
-                temp.add(peekable);
-        }
-        postingLists.addAll(temp);
-
-        return nextPosting();
+        applyToAllSources(FileUtils::closeQuietly);
     }
 
     @Override
-    public long size()
+    public int advanceSource(PostingList s) throws IOException
     {
-        return size;
+        return s.nextPosting();
     }
 
     @Override
-    public void close()
+    protected int skipSource(PostingList s, int targetPosition) throws IOException
     {
-        FileUtils.closeQuietly(onClose);
+        return s.advance(targetPosition);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/postings/OrdinalPostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/OrdinalPostingList.java
similarity index 87%
rename from src/java/org/apache/cassandra/index/sai/postings/OrdinalPostingList.java
rename to src/java/org/apache/cassandra/index/sai/disk/v1/postings/OrdinalPostingList.java
index dbc01c6c228f..ebb4d44dd43e 100644
--- a/src/java/org/apache/cassandra/index/sai/postings/OrdinalPostingList.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/OrdinalPostingList.java
@@ -15,7 +15,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.index.sai.postings;
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
 
 public interface OrdinalPostingList extends PostingList
 {
@@ -23,5 +25,5 @@ public interface OrdinalPostingList extends PostingList
      *
      * @return the ordinal of the posting that will be returned on the next call to {@link #nextPosting()}
      */
-    long getOrdinal();
+    int getOrdinal();
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PackedLongsPostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PackedLongsPostingList.java
index 34a6ea8e1748..0239237c0e87 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PackedLongsPostingList.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PackedLongsPostingList.java
@@ -17,7 +17,9 @@
  */
 package org.apache.cassandra.index.sai.disk.v1.postings;
 
-import org.apache.cassandra.index.sai.postings.PostingList;
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
 import org.apache.lucene.util.packed.PackedLongValues;
 
 /**
@@ -35,11 +37,13 @@ public PackedLongsPostingList(PackedLongValues values)
     }
 
     @Override
-    public long nextPosting()
+    public int nextPosting()
     {
         if (iterator.hasNext())
         {
-            return iterator.next();
+            // This is assumed to be safe because we only insert segment row ids, which are always integers,
+            // into the packed longs object
+            return Math.toIntExact(iterator.next());
         }
         else
         {
@@ -48,13 +52,14 @@ public long nextPosting()
     }
 
     @Override
-    public long size()
+    public int size()
     {
-        return values.size();
+        // We know that the size of the packed longs object is less than or equal to Integer.MAX_VALUE
+        return Math.toIntExact(values.size());
     }
 
     @Override
-    public long advance(long targetRowID)
+    public int advance(int targetRowID) throws IOException
     {
         throw new UnsupportedOperationException();
     }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java
deleted file mode 100644
index 4f05e2a17159..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.postings;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.concurrent.TimeUnit;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import com.google.common.base.Stopwatch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.exceptions.QueryCancelledException;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcherContext;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.Throwables;
-
-/**
- * A key iterator based on a  {@link PostingList} derived from a single index segment.
- *
- * <ol>
- *   <li> fetch next segment row id from posting list or skip to specific segment row id if {@link #skipTo(PrimaryKey)} is called </li>
- *   <li> add {@link IndexSegmentSearcherContext#segmentRowIdOffset} to obtain the sstable row id </li>
- *   <li> produce a {@link PrimaryKey} from {@link PrimaryKeyMap#primaryKeyFromRowId(long)} which is used
- *       to avoid fetching duplicated keys due to partition-level indexing on wide partition schema.
- *       <br/>
- *       Note: in order to reduce disk access in multi-index query, partition keys will only be fetched for intersected tokens
- *       in {@link org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher}.
- *  </li>
- * </ol>
- *
- */
-
-@NotThreadSafe
-public class PostingListRangeIterator extends KeyRangeIterator
-{
-    private static final Logger logger = LoggerFactory.getLogger(PostingListRangeIterator.class);
-
-    private final Stopwatch timeToExhaust = Stopwatch.createStarted();
-    private final QueryContext queryContext;
-
-    private final PostingList postingList;
-    private final IndexIdentifier indexIdentifier;
-    private final PrimaryKeyMap primaryKeyMap;
-    private final long rowIdOffset;
-
-    private boolean needsSkipping = false;
-    private PrimaryKey skipToKey = null;
-
-    /**
-     * Create a direct PostingListRangeIterator where the underlying PostingList is materialised
-     * immediately so the posting list size can be used.
-     */
-    public PostingListRangeIterator(IndexIdentifier indexIdentifier,
-                                    PrimaryKeyMap primaryKeyMap,
-                                    IndexSegmentSearcherContext searcherContext)
-    {
-        super(searcherContext.minimumKey, searcherContext.maximumKey, searcherContext.count(), () -> {});
-
-        this.indexIdentifier = indexIdentifier;
-        this.primaryKeyMap = primaryKeyMap;
-        this.postingList = searcherContext.postingList;
-        this.rowIdOffset = searcherContext.segmentRowIdOffset;
-        this.queryContext = searcherContext.context;
-    }
-
-    @Override
-    protected void performSkipTo(PrimaryKey nextKey)
-    {
-        if (skipToKey != null && skipToKey.compareTo(nextKey) > 0)
-            return;
-
-        skipToKey = nextKey;
-        needsSkipping = true;
-    }
-
-    @Override
-    protected PrimaryKey computeNext()
-    {
-        try
-        {
-            queryContext.checkpoint();
-
-            // just end the iterator if we don't have a postingList or current segment is skipped
-            if (exhausted())
-                return endOfData();
-
-            long rowId = getNextRowId();
-            if (rowId == PostingList.END_OF_STREAM)
-                return endOfData();
-
-            return primaryKeyMap.primaryKeyFromRowId(rowId);
-        }
-        catch (Throwable t)
-        {
-            if (!(t instanceof QueryCancelledException))
-                logger.error(indexIdentifier.logMessage("Unable to provide next token!"), t);
-
-            FileUtils.closeQuietly(Arrays.asList(postingList, primaryKeyMap));
-            throw Throwables.cleaned(t);
-        }
-    }
-
-    @Override
-    public void close()
-    {
-        if (logger.isTraceEnabled())
-        {
-            final long exhaustedInMills = timeToExhaust.stop().elapsed(TimeUnit.MILLISECONDS);
-            logger.trace(indexIdentifier.logMessage("PostingListRangeIterator exhausted after {} ms"), exhaustedInMills);
-        }
-
-        FileUtils.closeQuietly(Arrays.asList(postingList, primaryKeyMap));
-    }
-
-    private boolean exhausted()
-    {
-        return needsSkipping && skipToKey.compareTo(getMaximum()) > 0;
-    }
-
-    /**
-     * reads the next sstable row ID from the underlying posting list, potentially skipping to get there.
-     */
-    private long getNextRowId() throws IOException
-    {
-        long segmentRowId;
-        if (needsSkipping)
-        {
-            long targetRowID = primaryKeyMap.rowIdFromPrimaryKey(skipToKey);
-            // skipToToken is larger than max token in token file
-            if (targetRowID < 0)
-            {
-                return PostingList.END_OF_STREAM;
-            }
-
-            segmentRowId = postingList.advance(targetRowID - rowIdOffset);
-
-            needsSkipping = false;
-        }
-        else
-        {
-            segmentRowId = postingList.nextPosting();
-        }
-
-        return segmentRowId != PostingList.END_OF_STREAM
-               ? segmentRowId + rowIdOffset
-               : PostingList.END_OF_STREAM;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsReader.java
index bbc360445c31..0ee2ab63b132 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsReader.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsReader.java
@@ -23,155 +23,231 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
-import org.apache.cassandra.index.sai.disk.io.SeekingRandomAccessInput;
-import org.apache.cassandra.index.sai.disk.v1.DirectReaders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
 import org.apache.cassandra.index.sai.disk.v1.LongArray;
 import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.postings.OrdinalPostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.index.sai.utils.SeekingRandomAccessInput;
 import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
 import org.apache.lucene.util.LongValues;
-import org.apache.lucene.util.packed.DirectReader;
 
 
 /**
  * Reads, decompresses and decodes postings lists written by {@link PostingsWriter}.
- * <p>
- * Holds exactly one posting block in memory at a time. Does binary search over skip table to find a postings block to
+ *
+ * Holds exactly one postings block in memory at a time. Does binary search over skip table to find a postings block to
  * load.
  */
 @NotThreadSafe
 public class PostingsReader implements OrdinalPostingList
 {
-    private final IndexInput input;
+    private static final Logger logger = LoggerFactory.getLogger(PostingsReader.class);
+
+    protected final IndexInput input;
+    protected final InputCloser runOnClose;
+    private final int blockEntries;
+    private final int numPostings;
+    private final LongArray blockOffsets;
+    private final LongArray blockMaxValues;
     private final SeekingRandomAccessInput seekingInput;
     private final QueryEventListener.PostingListEventListener listener;
+
+    // TODO: Expose more things through the summary, now that it's an actual field?
     private final BlocksSummary summary;
 
-    // Current block index
-    private int blockIndex;
-    // Current posting index within block
-    private int postingIndex;
-    private long totalPostingsRead;
-    private long actualPosting;
+    private int postingsBlockIdx;
+    private int blockIdx; // position in block
+    private int totalPostingsRead;
+    private int actualSegmentRowId;
 
-    private LongValues currentFoRValues;
-    private long postingsDecoded = 0;
+    private long currentPosition;
+    private LongValues currentFORValues;
+    private int postingsDecoded = 0;
+    private int currentFrequency = Integer.MIN_VALUE;
+    private final boolean readFrequencies;
 
     @VisibleForTesting
     public PostingsReader(IndexInput input, long summaryOffset, QueryEventListener.PostingListEventListener listener) throws IOException
     {
-        this(input, new BlocksSummary(input, summaryOffset), listener);
+        this(input, new BlocksSummary(input, summaryOffset, InputCloser.NOOP), listener);
     }
 
     public PostingsReader(IndexInput input, BlocksSummary summary, QueryEventListener.PostingListEventListener listener) throws IOException
     {
+        this(input, summary, false, listener, () -> {
+            try
+            {
+                input.close();
+            }
+            finally
+            {
+                summary.close();
+            }
+        });
+    }
+
+    public PostingsReader(IndexInput input, BlocksSummary summary, boolean readFrequencies, QueryEventListener.PostingListEventListener listener) throws IOException
+    {
+        this(input, summary, readFrequencies, listener, () -> {
+            try
+            {
+                input.close();
+            }
+            finally
+            {
+                summary.close();
+            }
+        });
+    }
+
+    public PostingsReader(IndexInput input, BlocksSummary summary, boolean readFrequencies, QueryEventListener.PostingListEventListener listener, InputCloser runOnClose) throws IOException
+    {
+        assert input instanceof IndexInputReader;
+        logger.trace("Opening postings reader for {}", input);
+        this.readFrequencies = readFrequencies;
         this.input = input;
         this.seekingInput = new SeekingRandomAccessInput(input);
+        this.blockOffsets = summary.offsets;
+        this.blockEntries = summary.blockEntries;
+        this.numPostings = summary.numPostings;
+        this.blockMaxValues = summary.maxValues;
         this.listener = listener;
+
         this.summary = summary;
+        this.runOnClose = runOnClose;
 
         reBuffer();
     }
 
     @Override
-    public long getOrdinal()
+    public int getOrdinal()
     {
         return totalPostingsRead;
     }
 
+    public interface InputCloser
+    {
+        InputCloser NOOP = () -> {};
+        void close() throws IOException;
+    }
+
     public static class BlocksSummary
     {
-        private final IndexInput input;
-        final int blockSize;
+        final int blockEntries;
         final int numPostings;
         final LongArray offsets;
         final LongArray maxValues;
 
+        private final InputCloser runOnClose;
+
         public BlocksSummary(IndexInput input, long offset) throws IOException
         {
-            this.input = input;
+            this(input, offset, input::close);
+        }
+
+        public BlocksSummary(IndexInput input, long offset, InputCloser runOnClose) throws IOException
+        {
+            this.runOnClose = runOnClose;
+
             input.seek(offset);
-            this.blockSize = input.readVInt();
-            //TODO This should need to change because we can potentially end up with postings of more than Integer.MAX_VALUE?
+            this.blockEntries = input.readVInt();
+            // This is the count of row ids in a single posting list. For now, a segment cannot have more than
+            // Integer.MAX_VALUE row ids, so it is safe to use an int here.
             this.numPostings = input.readVInt();
 
-            SeekingRandomAccessInput randomAccessInput = new SeekingRandomAccessInput(input);
-            int numBlocks = input.readVInt();
-            long maxBlockValuesLength = input.readVLong();
-            long maxBlockValuesOffset = input.getFilePointer() + maxBlockValuesLength;
+            final SeekingRandomAccessInput randomAccessInput = new SeekingRandomAccessInput(input);
+            final int numBlocks = input.readVInt();
+            final long maxBlockValuesLength = input.readVLong();
+            final long maxBlockValuesOffset = input.getFilePointer() + maxBlockValuesLength;
 
-            byte offsetBitsPerValue = input.readByte();
-            DirectReaders.checkBitsPerValue(offsetBitsPerValue, input, () -> "Postings list header");
-            LongValues lvOffsets = offsetBitsPerValue == 0 ? LongValues.ZEROES : DirectReader.getInstance(randomAccessInput, offsetBitsPerValue, input.getFilePointer());
-            this.offsets = new LongArrayReader(lvOffsets, numBlocks);
+            final byte offsetBitsPerValue = input.readByte();
+            if (offsetBitsPerValue > 64)
+            {
+                String message = String.format("Postings list header is corrupted: Bits per value for block offsets must be no more than 64 and is %d.", offsetBitsPerValue);
+                throw new CorruptIndexException(message, input);
+            }
+            this.offsets = new LongArrayReader(randomAccessInput, offsetBitsPerValue == 0 ? LongValues.ZEROES : LuceneCompat.directReaderGetInstance(randomAccessInput, offsetBitsPerValue, input.getFilePointer()), numBlocks);
 
             input.seek(maxBlockValuesOffset);
-            byte valuesBitsPerValue = input.readByte();
-            DirectReaders.checkBitsPerValue(valuesBitsPerValue, input, () -> "Postings list header");
-            LongValues lvValues = valuesBitsPerValue == 0 ? LongValues.ZEROES : DirectReader.getInstance(randomAccessInput, valuesBitsPerValue, input.getFilePointer());
-            this.maxValues = new LongArrayReader(lvValues, numBlocks);
+            final byte valuesBitsPerValue = input.readByte();
+            if (valuesBitsPerValue > 64)
+            {
+                String message = String.format("Postings list header is corrupted: Bits per value for values samples must be no more than 64 and is %d.", valuesBitsPerValue);
+                throw new CorruptIndexException(message, input);
+            }
+            this.maxValues = new LongArrayReader(randomAccessInput, valuesBitsPerValue == 0 ? LongValues.ZEROES : LuceneCompat.directReaderGetInstance(randomAccessInput, valuesBitsPerValue, input.getFilePointer()), numBlocks);
         }
 
-        void close()
+        void close() throws IOException
         {
-            FileUtils.closeQuietly(input);
+            runOnClose.close();
         }
 
         private static class LongArrayReader implements LongArray
         {
+            private final RandomAccessInput input;
             private final LongValues reader;
             private final int length;
 
-            private LongArrayReader(LongValues reader, int length)
+            private LongArrayReader(RandomAccessInput input, LongValues reader, int length)
             {
+                this.input = input;
                 this.reader = reader;
                 this.length = length;
             }
 
             @Override
-            public long get(long idx)
+            public long ceilingRowId(long value)
             {
-                return reader.get(idx);
+                throw new UnsupportedOperationException();
             }
 
             @Override
-            public long length()
+            public long indexOf(long targetToken)
             {
-                return length;
+                throw new UnsupportedOperationException();
             }
 
             @Override
-            public long indexOf(long value)
+            public long get(long idx)
             {
-                throw new UnsupportedOperationException();
+                return reader.get(idx);
+            }
+
+            @Override
+            public long length()
+            {
+                return length;
             }
         }
     }
 
     @Override
-    public void close()
+    public void close() throws IOException
     {
         listener.postingDecoded(postingsDecoded);
-        FileUtils.closeQuietly(input);
-        summary.close();
+        runOnClose.close();
     }
 
     @Override
-    public long size()
+    public int size()
     {
-        return summary.numPostings;
+        return numPostings;
     }
 
     /**
      * Advances to the first row ID beyond the current that is greater than or equal to the
      * target, and returns that row ID. Exhausts the iterator and returns {@link #END_OF_STREAM} if
      * the target is greater than the highest row ID.
-     * <p>
+     *
      * Does binary search over the skip table to find the next block to load into memory.
-     * <p>
+     *
      * Note: Callers must use the return value of this method before calling {@link #nextPosting()}, as calling
      * that method will return the next posting, not the one to which we have just advanced.
      *
@@ -180,17 +256,17 @@ public long size()
      * @return first segment row ID which is >= the target row ID or {@link PostingList#END_OF_STREAM} if one does not exist
      */
     @Override
-    public long advance(long targetRowID) throws IOException
+    public int advance(int targetRowID) throws IOException
     {
         listener.onAdvance();
-        int block = binarySearchBlocks(targetRowID);
+        int block = binarySearchBlock(targetRowID);
 
         if (block < 0)
         {
             block = -block - 1;
         }
 
-        if (blockIndex == block + 1)
+        if (postingsBlockIdx == block + 1)
         {
             // we're in the same block, just iterate through
             return slowAdvance(targetRowID);
@@ -202,11 +278,11 @@ public long advance(long targetRowID) throws IOException
         return slowAdvance(targetRowID);
     }
 
-    private long slowAdvance(long targetRowID) throws IOException
+    private int slowAdvance(int targetRowID) throws IOException
     {
-        while (totalPostingsRead < summary.numPostings)
+        while (totalPostingsRead < numPostings)
         {
-            long segmentRowId = peekNext();
+            int segmentRowId = peekNext();
 
             advanceOnePosition(segmentRowId);
 
@@ -218,70 +294,62 @@ private long slowAdvance(long targetRowID) throws IOException
         return END_OF_STREAM;
     }
 
-    // Perform a binary search of the blocks to the find the block index
-    // containing the targetRowID, or, in the case of a duplicate value
-    // crossing blocks, the preceeding block index
-    private int binarySearchBlocks(long targetRowID)
+    private int binarySearchBlock(long targetRowID)
     {
-        int lowBlockIndex = blockIndex - 1;
-        int highBlockIndex = Math.toIntExact(summary.maxValues.length()) - 1;
+        int low = postingsBlockIdx - 1;
+        int high = Math.toIntExact(blockMaxValues.length()) - 1;
 
         // in current block
-        if (lowBlockIndex <= highBlockIndex && targetRowID <= summary.maxValues.get(lowBlockIndex))
-            return lowBlockIndex;
+        if (low <= high && targetRowID <= blockMaxValues.get(low))
+            return low;
 
-        while (lowBlockIndex <= highBlockIndex)
+        while (low <= high)
         {
-            int midBlockIndex = lowBlockIndex + ((highBlockIndex - lowBlockIndex) >> 1) ;
+            int mid = low + ((high - low) >> 1) ;
 
-            long maxValueOfMidBlock = summary.maxValues.get(midBlockIndex);
+            long midVal = blockMaxValues.get(mid);
 
-            if (maxValueOfMidBlock < targetRowID)
+            if (midVal < targetRowID)
             {
-                lowBlockIndex = midBlockIndex + 1;
+                low = mid + 1;
             }
-            else if (maxValueOfMidBlock > targetRowID)
+            else if (midVal > targetRowID)
             {
-                highBlockIndex = midBlockIndex - 1;
+                high = mid - 1;
             }
             else
             {
-                // At this point the maximum value of the midway block matches our target.
-                //
-                // This following check is to see if we have a duplicate value in the last entry of the
-                // preceeding block. This check is only going to be successful if the entire current
-                // block is full of duplicates.
-                if (midBlockIndex > 0 && summary.maxValues.get(midBlockIndex - 1) == targetRowID)
+                // target found, but we need to check for duplicates
+                if (mid > 0 && blockMaxValues.get(mid - 1L) == targetRowID)
                 {
-                    // there is a duplicate in the preceeding block so restrict search to finish
-                    // at that block
-                    highBlockIndex = midBlockIndex - 1;
+                    // there are duplicates, pivot left
+                    high = mid - 1;
                 }
                 else
                 {
                     // no duplicates
-                    return midBlockIndex;
+                    return mid;
                 }
             }
         }
-        return -(lowBlockIndex + 1);  // target not found
+        return -(low + 1);  // target not found
     }
 
     private void lastPosInBlock(int block)
     {
         // blockMaxValues is integer only
-        actualPosting = summary.maxValues.get(block);
+        actualSegmentRowId = Math.toIntExact(blockMaxValues.get(block));
         //upper bound, since we might've advanced to the last block, but upper bound is enough
-        totalPostingsRead += (summary.blockSize - postingIndex) + (block - blockIndex + 1) * (long)summary.blockSize;
+        totalPostingsRead += (blockEntries - blockIdx) + (block - postingsBlockIdx + 1) * blockEntries;
 
-        blockIndex = block + 1;
-        postingIndex = summary.blockSize;
+        postingsBlockIdx = block + 1;
+        blockIdx = blockEntries;
     }
 
     @Override
-    public long nextPosting() throws IOException
+    public int nextPosting() throws IOException
     {
-        long next = peekNext();
+        final int next = peekNext();
         if (next != END_OF_STREAM)
         {
             advanceOnePosition(next);
@@ -289,73 +357,90 @@ public long nextPosting() throws IOException
         return next;
     }
 
-    private long peekNext() throws IOException
+    @VisibleForTesting
+    int getBlockEntries()
+    {
+        return blockEntries;
+    }
+
+    private int peekNext() throws IOException
     {
-        if (totalPostingsRead >= summary.numPostings)
+        if (totalPostingsRead >= numPostings)
         {
             return END_OF_STREAM;
         }
-        if (postingIndex == summary.blockSize)
+        if (blockIdx == blockEntries)
         {
             reBuffer();
         }
 
-        return actualPosting + nextFoRValue();
+        return actualSegmentRowId + nextRowDelta();
     }
 
-    private int nextFoRValue()
+    private int nextRowDelta()
     {
-        long id = currentFoRValues.get(postingIndex);
+        if (currentFORValues == null)
+        {
+            currentFrequency = Integer.MIN_VALUE;
+            return 0;
+        }
+
+        long offset = readFrequencies ? 2L * blockIdx : blockIdx;
+        long id = currentFORValues.get(offset);
+        if (readFrequencies)
+            currentFrequency = Math.toIntExact(currentFORValues.get(offset + 1));
         postingsDecoded++;
         return Math.toIntExact(id);
     }
 
-    private void advanceOnePosition(long nextPosting)
+    private void advanceOnePosition(int nextRowID)
     {
-        actualPosting = nextPosting;
+        actualSegmentRowId = nextRowID;
         totalPostingsRead++;
-        postingIndex++;
+        blockIdx++;
     }
 
     private void reBuffer() throws IOException
     {
-        long pointer = summary.offsets.get(blockIndex);
-        if (pointer < 4)
-        {
+        final long pointer = blockOffsets.get(postingsBlockIdx);
+        if (pointer < 4) {
             // the first 4 bytes must be CODEC_MAGIC
-            throw new CorruptIndexException(String.format("Invalid block offset %d for postings block idx %d", pointer, blockIndex), input);
+            throw new CorruptIndexException(String.format("Invalid block offset %d for postings block idx %d", pointer, postingsBlockIdx), input);
         }
+
         input.seek(pointer);
 
-        long left = summary.numPostings - totalPostingsRead;
+        final long left = numPostings - totalPostingsRead;
         assert left > 0;
 
         readFoRBlock(input);
 
-        blockIndex++;
-        postingIndex = 0;
+        postingsBlockIdx++;
+        blockIdx = 0;
     }
 
     private void readFoRBlock(IndexInput in) throws IOException
     {
-        if (blockIndex == 0)
-            actualPosting = in.readVLong();
-
-        byte bitsPerValue = in.readByte();
+        final byte bitsPerValue = in.readByte();
 
-        long currentPosition = in.getFilePointer();
+        currentPosition = in.getFilePointer();
 
         if (bitsPerValue == 0)
         {
             // If bitsPerValue is 0 then all the values in the block are the same
-            currentFoRValues = LongValues.ZEROES;
+            currentFORValues = LongValues.ZEROES;
             return;
         }
         else if (bitsPerValue > 64)
         {
             throw new CorruptIndexException(
-            String.format("Postings list #%s block is corrupted. Bits per value should be no more than 64 and is %d.", blockIndex, bitsPerValue), input);
+                    String.format("Postings list #%s block is corrupted. Bits per value should be no more than 64 and is %d.", postingsBlockIdx, bitsPerValue), input);
         }
-        currentFoRValues = DirectReader.getInstance(seekingInput, bitsPerValue, currentPosition);
+        currentFORValues = LuceneCompat.directReaderGetInstance(seekingInput, bitsPerValue, currentPosition);
+    }
+
+    @Override
+    public int frequency() {
+        return currentFrequency;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java
index 545710c8613a..5bf0decfb8e4 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java
@@ -24,27 +24,30 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.agrona.collections.LongArrayList;
-import org.apache.cassandra.index.sai.disk.ResettableByteBuffersIndexOutput;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.index.sai.disk.oldlucene.DirectWriterAdapter;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
+import org.apache.cassandra.index.sai.disk.oldlucene.ResettableByteBuffersIndexOutput;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.packed.DirectWriter;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.lang.Math.max;
+import static java.lang.Math.min;
+
 
 /**
  * Encodes, compresses and writes postings lists to disk.
- * <p>
- * All postings in the posting list are delta encoded, then deltas are divided into blocks for compression.
- * The deltas are based on the final value of the previous block. For the first block in the posting list
- * the first value in the block is written as a VLong prior to block delta encodings.
+ *
+ * All row IDs in the posting list are delta encoded, then deltas are divided into blocks for compression.
  * <p>
  * In packed blocks, longs are encoded with the same bit width (FoR compression). The block size (i.e. number of
  * longs inside block) is fixed (currently 128). Additionally blocks that are all the same value are encoded in an
@@ -56,77 +59,92 @@
  *
  * <p>
  * Packed blocks are favoured, meaning when the postings are long enough, {@link PostingsWriter} will try
- * to encode most data as a packed block. Take a term with 259 postings as an example, the first 256 postings are encoded
+ * to encode most data as a packed block. Take a term with 259 row IDs as an example, the first 256 IDs are encoded
  * as two packed blocks, while the remaining 3 are encoded as one VLong block.
  * </p>
  * <p>
- * Each posting list ends with a block summary containing metadata and a skip table, written right after all postings
- * blocks. Skip interval is the same as block size, and each skip entry points to the end of each block.
- * Skip table consist of block offsets and last values of each block, compressed as two FoR blocks.
+ * Each posting list ends with a meta section and a skip table, that are written right after all postings blocks. Skip
+ * interval is the same as block size, and each skip entry points to the end of each block.  Skip table consist of
+ * block offsets and maximum rowids of each block, compressed as two FoR blocks.
  * </p>
  *
  * Visual representation of the disk format:
  * <pre>
  *
- * +========+========================+=====+==============+===============+===============+=====+========================+========+
- * | HEADER | POSTINGS LIST (TERM 1)                                                      | ... | POSTINGS LIST (TERM N) | FOOTER |
- * +========+========================+=====+==============+===============+===============+=====+========================+========+
- *          | FIRST VALUE| FOR BLOCK (1)| ... | FOR BLOCK (N)| BLOCK SUMMARY              |
- *          +---------------------------+-----+--------------+---------------+------------+
- *                                                           | BLOCK SIZE    |            |
- *                                                           | LIST SIZE     | SKIP TABLE |
- *                                                           +---------------+------------+
- *                                                                           | BLOCKS POS.|
- *                                                                           | MAX VALUES |
- *                                                                           +------------+
+ * +========+========================+=====+==============+===============+============+=====+========================+========+
+ * | HEADER | POSTINGS LIST (TERM 1)                                                   | ... | POSTINGS LIST (TERM N) | FOOTER |
+ * +========+========================+=====+==============+===============+============+=====+========================+========+
+ *          | FOR BLOCK (1)          | ... | FOR BLOCK (N)| BLOCK SUMMARY              |
+ *          +------------------------+-----+--------------+---------------+------------+
+ *                                                        | BLOCK SIZE    |            |
+ *                                                        | LIST SIZE     | SKIP TABLE |
+ *                                                        +---------------+------------+
+ *                                                                        | BLOCKS POS.|
+ *                                                                        | MAX ROWIDS |
+ *                                                                        +------------+
  *
  *  </pre>
  */
 @NotThreadSafe
 public class PostingsWriter implements Closeable
 {
+    protected static final Logger logger = LoggerFactory.getLogger(PostingsWriter.class);
+
     // import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;
-    private final static int BLOCK_SIZE = 128;
+    private final static int BLOCK_ENTRIES = 128;
 
     private static final String POSTINGS_MUST_BE_SORTED_ERROR_MSG = "Postings must be sorted ascending, got [%s] after [%s]";
 
     private final IndexOutput dataOutput;
-    private final int blockSize;
+    private final int blockEntries;
     private final long[] deltaBuffer;
+    private final int[] freqBuffer; // frequency is capped at 255
     private final LongArrayList blockOffsets = new LongArrayList();
-    private final LongArrayList blockMaximumPostings = new LongArrayList();
-    private final ResettableByteBuffersIndexOutput inMemoryOutput = new ResettableByteBuffersIndexOutput("blockOffsets");
+    private final LongArrayList blockMaxIDs = new LongArrayList();
+    private final ResettableByteBuffersIndexOutput inMemoryOutput;
 
     private final long startOffset;
 
     private int bufferUpto;
-    private long firstPosting = Long.MIN_VALUE;
-    private long lastPosting = Long.MIN_VALUE;
-    private long maxDelta;
+    private long lastSegmentRowId;
+    // This number is the count of row ids written to the postings for this segment. Because a segment row id can be in
+    // multiple postings list for the segment, this number could exceed Integer.MAX_VALUE, so we use a long.
     private long totalPostings;
+    private final boolean writeFrequencies;
+
+    public PostingsWriter(IndexComponents.ForWrite components) throws IOException
+    {
+        this(components, BLOCK_ENTRIES);
+    }
 
-    public PostingsWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
+    public PostingsWriter(IndexComponents.ForWrite components, boolean writeFrequencies) throws IOException
     {
-        this(indexDescriptor, indexIdentifier, BLOCK_SIZE);
+        this(components.addOrGet(IndexComponentType.POSTING_LISTS).openOutput(true), BLOCK_ENTRIES, writeFrequencies);
     }
 
-    public PostingsWriter(IndexOutputWriter dataOutput) throws IOException
+
+    public PostingsWriter(IndexOutput dataOutput) throws IOException
     {
-        this(dataOutput, BLOCK_SIZE);
+        this(dataOutput, BLOCK_ENTRIES, false);
     }
 
     @VisibleForTesting
-    PostingsWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, int blockSize) throws IOException
+    PostingsWriter(IndexComponents.ForWrite components, int blockEntries) throws IOException
     {
-        this(indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier, true), blockSize);
+        this(components.addOrGet(IndexComponentType.POSTING_LISTS).openOutput(true), blockEntries, false);
     }
 
-    private PostingsWriter(IndexOutputWriter dataOutput, int blockSize) throws IOException
+    private PostingsWriter(IndexOutput dataOutput, int blockEntries, boolean writeFrequencies) throws IOException
     {
-        this.blockSize = blockSize;
+        assert dataOutput instanceof IndexOutputWriter;
+        logger.debug("Creating postings writer for output {}", dataOutput);
+        this.writeFrequencies = writeFrequencies;
+        this.blockEntries = blockEntries;
         this.dataOutput = dataOutput;
         startOffset = dataOutput.getFilePointer();
-        deltaBuffer = new long[blockSize];
+        deltaBuffer = new long[blockEntries];
+        freqBuffer = new int[blockEntries];
+        inMemoryOutput = LuceneCompat.getResettableByteBuffersIndexOutput(dataOutput.order(), 1024, "blockOffsets");
         SAICodecUtils.writeHeader(dataOutput);
     }
 
@@ -172,23 +190,22 @@ public long write(PostingList postings) throws IOException
         checkArgument(postings != null, "Expected non-null posting list.");
         checkArgument(postings.size() > 0, "Expected non-empty posting list.");
 
-        lastPosting = Long.MIN_VALUE;
         resetBlockCounters();
         blockOffsets.clear();
-        blockMaximumPostings.clear();
+        blockMaxIDs.clear();
 
-        long posting;
+        int segmentRowId;
         // When postings list are merged, we don't know exact size, just an upper bound.
         // We need to count how many postings we added to the block ourselves.
         int size = 0;
-        while ((posting = postings.nextPosting()) != PostingList.END_OF_STREAM)
+        while ((segmentRowId = postings.nextPosting()) != PostingList.END_OF_STREAM)
         {
-            writePosting(posting);
+            writePosting(segmentRowId, postings.frequency());
             size++;
             totalPostings++;
         }
-
-        assert size > 0 : "No postings were written";
+        if (size == 0)
+            return -1;
 
         finish();
 
@@ -202,63 +219,56 @@ public long getTotalPostings()
         return totalPostings;
     }
 
-    private void writePosting(long posting) throws IOException
-    {
-        if (lastPosting == Long.MIN_VALUE)
-        {
-            firstPosting = posting;
-            deltaBuffer[bufferUpto++] = 0;
-        }
-        else
-        {
-            if (posting < lastPosting)
-                throw new IllegalArgumentException(String.format(POSTINGS_MUST_BE_SORTED_ERROR_MSG, posting, lastPosting));
-            long delta = posting - lastPosting;
-            maxDelta = max(maxDelta, delta);
-            deltaBuffer[bufferUpto++] = delta;
-        }
-        lastPosting = posting;
+    private void writePosting(long segmentRowId, int freq) throws IOException {
+        if (!(segmentRowId >= lastSegmentRowId || lastSegmentRowId == 0))
+            throw new IllegalArgumentException(String.format(POSTINGS_MUST_BE_SORTED_ERROR_MSG, segmentRowId, lastSegmentRowId));
 
-        if (bufferUpto == blockSize)
-        {
-            addBlockToSkipTable();
-            writePostingsBlock();
+        assert freq > 0;
+        final long delta = segmentRowId - lastSegmentRowId;
+        deltaBuffer[bufferUpto] = delta;
+        freqBuffer[bufferUpto] = min(freq, 255);
+        bufferUpto++;
+
+        if (bufferUpto == blockEntries) {
+            addBlockToSkipTable(segmentRowId);
+            writePostingsBlock(bufferUpto);
             resetBlockCounters();
         }
+        lastSegmentRowId = segmentRowId;
     }
 
     private void finish() throws IOException
     {
         if (bufferUpto > 0)
         {
-            addBlockToSkipTable();
-            writePostingsBlock();
+            addBlockToSkipTable(lastSegmentRowId);
+
+            writePostingsBlock(bufferUpto);
         }
     }
 
     private void resetBlockCounters()
     {
-        firstPosting = Long.MIN_VALUE;
         bufferUpto = 0;
-        maxDelta = 0;
+        lastSegmentRowId = 0;
     }
 
-    private void addBlockToSkipTable()
+    private void addBlockToSkipTable(long maxSegmentRowID)
     {
         blockOffsets.add(dataOutput.getFilePointer());
-        blockMaximumPostings.add(lastPosting);
+        blockMaxIDs.add(maxSegmentRowID);
     }
 
     private void writeSummary(int exactSize) throws IOException
     {
-        dataOutput.writeVInt(blockSize);
+        dataOutput.writeVInt(blockEntries);
         dataOutput.writeVInt(exactSize);
         writeSkipTable();
     }
 
     private void writeSkipTable() throws IOException
     {
-        assert blockOffsets.size() == blockMaximumPostings.size();
+        assert blockOffsets.size() == blockMaxIDs.size();
         dataOutput.writeVInt(blockOffsets.size());
 
         // compressing offsets in memory first, to know the exact length (with padding)
@@ -267,34 +277,32 @@ private void writeSkipTable() throws IOException
         writeSortedFoRBlock(blockOffsets, inMemoryOutput);
         dataOutput.writeVLong(inMemoryOutput.getFilePointer());
         inMemoryOutput.copyTo(dataOutput);
-        writeSortedFoRBlock(blockMaximumPostings, dataOutput);
+        writeSortedFoRBlock(blockMaxIDs, dataOutput);
     }
 
-    private void writePostingsBlock() throws IOException
-    {
-        final int bitsPerValue = maxDelta == 0 ? 0 : DirectWriter.unsignedBitsRequired(maxDelta);
-
-        // If we have a first posting, indicating that this is the first block in the posting list
-        // then write it prior to the deltas.
-        if (firstPosting != Long.MIN_VALUE)
-            dataOutput.writeVLong(firstPosting);
-
+    private void writePostingsBlock(int entries) throws IOException {
+        // Find max value to determine bits needed
+        long maxValue = 0;
+        for (int i = 0; i < entries; i++) {
+            maxValue = max(maxValue, deltaBuffer[i]);
+            if (writeFrequencies)
+                maxValue = max(maxValue, freqBuffer[i]);
+        }
+        
+        // Use the maximum bits needed for either value type
+        final int bitsPerValue = maxValue == 0 ? 0 : LuceneCompat.directWriterUnsignedBitsRequired(dataOutput.order(), maxValue);
+        
         dataOutput.writeByte((byte) bitsPerValue);
-        if (bitsPerValue > 0)
-        {
-            final DirectWriter writer = DirectWriter.getInstance(dataOutput, blockSize, bitsPerValue);
-            for (int index = 0; index < bufferUpto; ++index)
-            {
-                writer.add(deltaBuffer[index]);
-            }
-            if (bufferUpto < blockSize)
-            {
-                // Pad the rest of the block with 0, so we don't write invalid
-                // values from previous blocks
-                for (int index = bufferUpto; index < blockSize; index++)
-                {
-                    writer.add(0);
-                }
+        if (bitsPerValue > 0) {
+            // Write interleaved [delta][freq] pairs
+            final DirectWriterAdapter writer = LuceneCompat.directWriterGetInstance(dataOutput.order(),
+                                                                                    dataOutput,
+                                                                                    writeFrequencies ? entries * 2L : entries,
+                                                                                    bitsPerValue);
+            for (int i = 0; i < entries; ++i) {
+                writer.add(deltaBuffer[i]);
+                if (writeFrequencies)
+                    writer.add(freqBuffer[i]);
             }
             writer.finish();
         }
@@ -302,14 +310,14 @@ private void writePostingsBlock() throws IOException
 
     private void writeSortedFoRBlock(LongArrayList values, IndexOutput output) throws IOException
     {
+        assert !values.isEmpty();
         final long maxValue = values.getLong(values.size() - 1);
 
-        assert values.size() > 0;
-        final int bitsPerValue = maxValue == 0 ? 0 : DirectWriter.unsignedBitsRequired(maxValue);
+        final int bitsPerValue = maxValue == 0 ? 0 : LuceneCompat.directWriterUnsignedBitsRequired(output.order(), maxValue);
         output.writeByte((byte) bitsPerValue);
         if (bitsPerValue > 0)
         {
-            final DirectWriter writer = DirectWriter.getInstance(output, values.size(), bitsPerValue);
+            final DirectWriterAdapter writer = LuceneCompat.directWriterGetInstance(output.order(), output, values.size(), bitsPerValue);
             for (int i = 0; i < values.size(); ++i)
             {
                 writer.add(values.getLong(i));
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/ReorderingPostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/ReorderingPostingList.java
new file mode 100644
index 000000000000..f43c7c4e8dce
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/ReorderingPostingList.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.io.IOException;
+import java.util.function.ToIntFunction;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.lucene.util.LongHeap;
+
+/**
+ * A posting list for ANN search results.  Transforms results from similarity order to rowId order.
+ */
+public class ReorderingPostingList implements PostingList
+{
+    private final LongHeap segmentRowIds;
+    private final int size;
+
+    public <T> ReorderingPostingList(CloseableIterator<T> source, ToIntFunction<T> rowIdTransformer)
+    {
+        segmentRowIds = new LongHeap(32);
+        int n = 0;
+        try (source)
+        {
+            while (source.hasNext())
+            {
+                segmentRowIds.push(rowIdTransformer.applyAsInt(source.next()));
+                n++;
+            }
+        }
+        this.size = n;
+    }
+
+    @Override
+    public int nextPosting() throws IOException
+    {
+        if (segmentRowIds.size() == 0)
+            return PostingList.END_OF_STREAM;
+        return (int) segmentRowIds.pop();
+    }
+
+    @Override
+    public int size()
+    {
+        return size;
+    }
+
+    @Override
+    public int advance(int targetRowID) throws IOException
+    {
+        int rowId;
+        do
+        {
+            rowId = nextPosting();
+        } while (rowId < targetRowID);
+        return rowId;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/ScanningPostingsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/ScanningPostingsReader.java
new file mode 100644
index 000000000000..5c9da6169e08
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/ScanningPostingsReader.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+
+/**
+ * An sub-class of the {@code PostingsReader} that does not allow the {@code PostingList} to be
+ * advanced and does not support mapping row ids to primary keys.
+ *
+ * It is used during index mergers to sequentially scan the postings in order using {@code nextPosting}.
+ */
+public class ScanningPostingsReader extends PostingsReader
+{
+    public ScanningPostingsReader(IndexInput input, BlocksSummary summary, boolean readFrequencies) throws IOException
+    {
+        super(input, summary, readFrequencies, QueryEventListener.PostingListEventListener.NO_OP, InputCloser.NOOP);
+    }
+
+    @Override
+    public int advance(int targetRowId)
+    {
+        throw new UnsupportedOperationException("Cannot advance a scanning postings reader");
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/VectorPostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/VectorPostingList.java
deleted file mode 100644
index cc3fc58665e8..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/VectorPostingList.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.postings;
-
-import java.io.IOException;
-import java.util.PrimitiveIterator;
-
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.lucene.util.LongHeap;
-
-/**
- * A {@link PostingList} for ANN search results. Transforms result from similarity order to row ID order.
- */
-public class VectorPostingList implements PostingList
-{
-    private final LongHeap segmentRowIds;
-    private final int size;
-    private final int visitedCount;
-
-    public VectorPostingList(PrimitiveIterator.OfInt source, int limit, int visitedCount)
-    {
-        this.visitedCount = visitedCount;
-        segmentRowIds = new LongHeap(Math.max(limit, 1));
-        int n = 0;
-        while (source.hasNext() && n++ < limit)
-            segmentRowIds.push(source.nextInt());
-        this.size = n;
-    }
-
-    @Override
-    public long nextPosting()
-    {
-        if (segmentRowIds.size() == 0)
-            return PostingList.END_OF_STREAM;
-        return segmentRowIds.pop();
-    }
-
-    @Override
-    public long size()
-    {
-        return size;
-    }
-
-    @Override
-    public long advance(long targetRowID) throws IOException
-    {
-        long rowId;
-        do
-        {
-            rowId = nextPosting();
-        } while (rowId < targetRowID);
-        return rowId;
-    }
-
-    public int getVisitedCount()
-    {
-        return visitedCount;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java
deleted file mode 100644
index 96f389d794c6..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingListRangeIterator;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.postings.PeekablePostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
-
-/**
- * Abstract reader for individual segments of an on-disk index.
- * <p>
- * Accepts shared resources (token/offset file readers), and uses them to perform lookups against on-disk data
- * structures.
- */
-public abstract class IndexSegmentSearcher implements SegmentOrdering, Closeable
-{
-    final PrimaryKeyMap.Factory primaryKeyMapFactory;
-    final PerColumnIndexFiles indexFiles;
-    final SegmentMetadata metadata;
-    final StorageAttachedIndex index;
-
-    IndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
-                         PerColumnIndexFiles perIndexFiles,
-                         SegmentMetadata segmentMetadata,
-                         StorageAttachedIndex index)
-    {
-        this.primaryKeyMapFactory = primaryKeyMapFactory;
-        this.indexFiles = perIndexFiles;
-        this.metadata = segmentMetadata;
-        this.index = index;
-    }
-
-    public static IndexSegmentSearcher open(PrimaryKeyMap.Factory primaryKeyMapFactory,
-                                            PerColumnIndexFiles indexFiles,
-                                            SegmentMetadata segmentMetadata,
-                                            StorageAttachedIndex index) throws IOException
-    {
-        if (index.termType().isVector())
-            return new VectorIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index);
-        else if (index.termType().isLiteral())
-            return new LiteralIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index);
-        else
-            return new NumericIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index);
-    }
-
-    /**
-     * @return memory usage of underlying on-disk data structure
-     */
-    public abstract long indexFileCacheSize();
-
-    /**
-     * Search on-disk index synchronously.
-     *
-     * @param expression to filter on disk index
-     * @param queryContext to track per sstable cache and per query metrics
-     *
-     * @return {@link KeyRangeIterator} with matches for the given expression
-     */
-    public abstract KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext) throws IOException;
-
-    KeyRangeIterator toPrimaryKeyIterator(PostingList postingList, QueryContext queryContext) throws IOException
-    {
-        if (postingList == null || postingList.size() == 0)
-            return KeyRangeIterator.empty();
-
-        IndexSegmentSearcherContext searcherContext = new IndexSegmentSearcherContext(metadata.minKey,
-                                                                                      metadata.maxKey,
-                                                                                      metadata.rowIdOffset,
-                                                                                      queryContext,
-                                                                                      PeekablePostingList.makePeekable(postingList));
-
-        return new PostingListRangeIterator(index.identifier(), primaryKeyMapFactory.newPerSSTablePrimaryKeyMap(), searcherContext);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcherContext.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcherContext.java
deleted file mode 100644
index 6cc5a1121ab9..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcherContext.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.postings.PeekablePostingList;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-public class IndexSegmentSearcherContext
-{
-    public final QueryContext context;
-    public final PeekablePostingList postingList;
-
-    public final PrimaryKey minimumKey;
-    public final PrimaryKey maximumKey;
-    public final long segmentRowIdOffset;
-
-    public IndexSegmentSearcherContext(PrimaryKey minimumKey,
-                                       PrimaryKey maximumKey,
-                                       long segmentRowIdOffset,
-                                       QueryContext context,
-                                       PeekablePostingList postingList)
-    {
-        this.context = context;
-        this.postingList = postingList;
-
-        this.segmentRowIdOffset = segmentRowIdOffset;
-
-        this.minimumKey = minimumKey;
-        this.maximumKey = maximumKey;
-    }
-
-    public long count()
-    {
-        return postingList.size();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java
deleted file mode 100644
index 3aa566e65ad6..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.IOException;
-import java.util.Map;
-
-import com.google.common.base.MoreObjects;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.metrics.MulticastQueryEventListeners;
-import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-
-/**
- * Executes {@link Expression}s against the trie-based terms dictionary for an individual index segment.
- */
-public class LiteralIndexSegmentSearcher extends IndexSegmentSearcher
-{
-    private static final Logger logger = LoggerFactory.getLogger(LiteralIndexSegmentSearcher.class);
-
-    private final LiteralIndexSegmentTermsReader reader;
-    private final QueryEventListener.TrieIndexEventListener perColumnEventListener;
-
-    LiteralIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
-                                PerColumnIndexFiles perIndexFiles,
-                                SegmentMetadata segmentMetadata,
-                                StorageAttachedIndex index) throws IOException
-    {
-        super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index);
-
-        long root = metadata.getIndexRoot(IndexComponent.TERMS_DATA);
-        assert root >= 0;
-
-        perColumnEventListener = (QueryEventListener.TrieIndexEventListener)index.columnQueryMetrics();
-
-        Map<String,String> map = metadata.componentMetadatas.get(IndexComponent.TERMS_DATA).attributes;
-        String footerPointerString = map.get(SAICodecUtils.FOOTER_POINTER);
-        long footerPointer = footerPointerString == null ? -1 : Long.parseLong(footerPointerString);
-
-        reader = new LiteralIndexSegmentTermsReader(index.identifier(), indexFiles.termsData(), indexFiles.postingLists(), root, footerPointer);
-    }
-
-    @Override
-    public long indexFileCacheSize()
-    {
-        // trie has no pre-allocated memory.
-        return 0;
-    }
-
-    @Override
-    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext) throws IOException
-    {
-        if (logger.isTraceEnabled())
-            logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), expression);
-
-        if (!expression.getIndexOperator().isEquality())
-            throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression: " + expression));
-
-        ByteComparable term = v -> index.termType().asComparableBytes(expression.lower().value.encoded, v);
-        QueryEventListener.TrieIndexEventListener listener = MulticastQueryEventListeners.of(queryContext, perColumnEventListener);
-        return toPrimaryKeyIterator(reader.exactMatch(term, listener, queryContext), queryContext);
-    }
-
-    @Override
-    public String toString()
-    {
-        return MoreObjects.toStringHelper(this).add("index", index).toString();
-    }
-
-    @Override
-    public void close()
-    {
-        reader.close();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java
deleted file mode 100644
index 6c6f81cddf3c..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.exceptions.QueryCancelledException;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
-import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader;
-import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.Clock;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.lucene.store.IndexInput;
-
-import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.validate;
-
-/**
- * Synchronous reader of terms dictionary and postings lists to produce a {@link PostingList} with matching row ids.
- *
- * {@link #exactMatch(ByteComparable, QueryEventListener.TrieIndexEventListener, QueryContext)} does:
- * <ul>
- * <li>{@link TermQuery#lookupPostingsOffset(ByteComparable)}: does term dictionary lookup to find the posting list file
- * position</li>
- * <li>{@link TermQuery#getPostingsReader(long)}: reads posting list block summary and initializes posting read which
- * reads the first block of the posting list into memory</li>
- * </ul>
- */
-public class LiteralIndexSegmentTermsReader implements Closeable
-{
-    private static final Logger logger = LoggerFactory.getLogger(LiteralIndexSegmentTermsReader.class);
-
-    private final IndexIdentifier indexIdentifier;
-    private final FileHandle termDictionaryFile;
-    private final FileHandle postingsFile;
-    private final long termDictionaryRoot;
-
-    public LiteralIndexSegmentTermsReader(IndexIdentifier indexIdentifier,
-                                          FileHandle termsData,
-                                          FileHandle postingLists,
-                                          long root,
-                                          long termsFooterPointer) throws IOException
-    {
-        this.indexIdentifier = indexIdentifier;
-        termDictionaryFile = termsData;
-        postingsFile = postingLists;
-        termDictionaryRoot = root;
-
-        try (final IndexInput indexInput = IndexFileUtils.instance.openInput(termDictionaryFile))
-        {
-            validate(indexInput, termsFooterPointer);
-        }
-
-        try (final IndexInput indexInput = IndexFileUtils.instance.openInput(postingsFile))
-        {
-            validate(indexInput);
-        }
-    }
-
-    @Override
-    public void close()
-    {
-        FileUtils.closeQuietly(termDictionaryFile);
-        FileUtils.closeQuietly(postingsFile);
-    }
-
-    public PostingList exactMatch(ByteComparable term, QueryEventListener.TrieIndexEventListener perQueryEventListener, QueryContext context)
-    {
-        perQueryEventListener.onSegmentHit();
-        return new TermQuery(term, perQueryEventListener, context).execute();
-    }
-
-    @VisibleForTesting
-    public class TermQuery
-    {
-        private final IndexInput postingsInput;
-        private final IndexInput postingsSummaryInput;
-        private final QueryEventListener.TrieIndexEventListener listener;
-        private final long lookupStartTime;
-        private final QueryContext context;
-        private final ByteComparable term;
-
-        TermQuery(ByteComparable term, QueryEventListener.TrieIndexEventListener listener, QueryContext context)
-        {
-            this.listener = listener;
-            postingsInput = IndexFileUtils.instance.openInput(postingsFile);
-            postingsSummaryInput = IndexFileUtils.instance.openInput(postingsFile);
-            this.term = term;
-            lookupStartTime = Clock.Global.nanoTime();
-            this.context = context;
-        }
-
-        public PostingList execute()
-        {
-            try
-            {
-                long postingOffset = lookupPostingsOffset(term);
-                if (postingOffset == PostingList.OFFSET_NOT_FOUND)
-                {
-                    FileUtils.closeQuietly(postingsInput);
-                    FileUtils.closeQuietly(postingsSummaryInput);
-                    return null;
-                }
-
-                context.checkpoint();
-
-                // when posting is found, resources will be closed when posting reader is closed.
-                return getPostingsReader(postingOffset);
-            }
-            catch (Throwable e)
-            {
-                if (!(e instanceof QueryCancelledException))
-                    logger.error(indexIdentifier.logMessage("Failed to execute term query"), e);
-
-                closeOnException();
-                throw Throwables.cleaned(e);
-            }
-        }
-
-        private void closeOnException()
-        {
-            FileUtils.closeQuietly(postingsInput);
-            FileUtils.closeQuietly(postingsSummaryInput);
-        }
-
-        public long lookupPostingsOffset(ByteComparable term)
-        {
-            try (TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(termDictionaryFile.instantiateRebufferer(null), termDictionaryRoot))
-            {
-                final long offset = reader.exactMatch(term);
-
-                listener.onTraversalComplete(Clock.Global.nanoTime() - lookupStartTime, TimeUnit.NANOSECONDS);
-
-                if (offset == TrieTermsDictionaryReader.NOT_FOUND)
-                    return PostingList.OFFSET_NOT_FOUND;
-
-                return offset;
-            }
-        }
-
-        public PostingsReader getPostingsReader(long offset) throws IOException
-        {
-            PostingsReader.BlocksSummary header = new PostingsReader.BlocksSummary(postingsSummaryInput, offset);
-
-            return new PostingsReader(postingsInput, header, listener.postingListEventListener());
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/NumericIndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/NumericIndexSegmentSearcher.java
deleted file mode 100644
index 137e635fd06c..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/NumericIndexSegmentSearcher.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-
-import com.google.common.base.MoreObjects;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
-import org.apache.cassandra.index.sai.disk.v1.bbtree.BlockBalancedTreeReader;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.metrics.MulticastQueryEventListeners;
-import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.lucene.index.CorruptIndexException;
-
-import static org.apache.cassandra.index.sai.disk.v1.bbtree.BlockBalancedTreeQueries.balancedTreeQueryFrom;
-
-/**
- * Executes {@link Expression}s against the balanced tree for an individual index segment.
- */
-public class NumericIndexSegmentSearcher extends IndexSegmentSearcher
-{
-    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-    private final BlockBalancedTreeReader treeReader;
-    private final QueryEventListener.BalancedTreeEventListener perColumnEventListener;
-
-    NumericIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
-                                PerColumnIndexFiles perIndexFiles,
-                                SegmentMetadata segmentMetadata,
-                                StorageAttachedIndex index) throws IOException
-    {
-        super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index);
-
-        final long treePosition = metadata.getIndexRoot(IndexComponent.BALANCED_TREE);
-        if (treePosition < 0)
-            throw new CorruptIndexException(index.identifier().logMessage("The tree position is less than zero."), IndexComponent.BALANCED_TREE.name);
-        final long postingsPosition = metadata.getIndexRoot(IndexComponent.POSTING_LISTS);
-        if (postingsPosition < 0)
-            throw new CorruptIndexException(index.identifier().logMessage("The postings position is less than zero."), IndexComponent.BALANCED_TREE.name);
-
-        treeReader = new BlockBalancedTreeReader(index.identifier(),
-                                                 indexFiles.balancedTree(),
-                                                 treePosition,
-                                                 indexFiles.postingLists(),
-                                                 postingsPosition);
-        perColumnEventListener = (QueryEventListener.BalancedTreeEventListener)index.columnQueryMetrics();
-    }
-
-    @Override
-    public long indexFileCacheSize()
-    {
-        return treeReader.memoryUsage();
-    }
-
-    @Override
-    public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context) throws IOException
-    {
-        if (logger.isTraceEnabled())
-            logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), exp);
-
-        if (exp.getIndexOperator().isEqualityOrRange())
-        {
-            final BlockBalancedTreeReader.IntersectVisitor query = balancedTreeQueryFrom(exp, treeReader.getBytesPerValue());
-            QueryEventListener.BalancedTreeEventListener listener = MulticastQueryEventListeners.of(context, perColumnEventListener);
-            return toPrimaryKeyIterator(treeReader.intersect(query, listener, context), context);
-        }
-        else
-        {
-            throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression during index query: " + exp));
-        }
-    }
-
-    @Override
-    public String toString()
-    {
-        return MoreObjects.toStringHelper(this)
-                          .add("index", index)
-                          .add("count", treeReader.getPointCount())
-                          .add("bytesPerValue", treeReader.getBytesPerValue())
-                          .toString();
-    }
-
-    @Override
-    public void close()
-    {
-        treeReader.close();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java
deleted file mode 100644
index 6e08551bd108..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.SSTableContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.util.FileUtils;
-
-/**
- * Each segment represents an on-disk index structure (balanced tree/terms/postings) flushed by memory limit or token boundaries.
- * It also helps to reduce resource consumption for read requests as only segments that intersect with read request data
- * range need to be loaded.
- */
-public class Segment implements SegmentOrdering, Closeable
-{
-    private final Token.KeyBound minKeyBound;
-    private final Token.KeyBound maxKeyBound;
-
-    // per sstable
-    final PrimaryKeyMap.Factory primaryKeyMapFactory;
-    // per-segment
-    public final SegmentMetadata metadata;
-
-    private final IndexSegmentSearcher index;
-
-    public Segment(StorageAttachedIndex index, SSTableContext sstableContext, PerColumnIndexFiles indexFiles, SegmentMetadata metadata) throws IOException
-    {
-        this.minKeyBound = metadata.minKey.token().minKeyBound();
-        this.maxKeyBound = metadata.maxKey.token().maxKeyBound();
-
-        this.primaryKeyMapFactory = sstableContext.primaryKeyMapFactory;
-        this.metadata = metadata;
-
-        this.index = IndexSegmentSearcher.open(primaryKeyMapFactory, indexFiles, metadata, index);
-    }
-
-    @VisibleForTesting
-    public Segment(Token minKey, Token maxKey)
-    {
-        this.primaryKeyMapFactory = null;
-        this.metadata = null;
-        this.minKeyBound = minKey.minKeyBound();
-        this.maxKeyBound = maxKey.maxKeyBound();
-        this.index = null;
-    }
-
-    /**
-     * @return true if current segment intersects with query key range
-     */
-    public boolean intersects(AbstractBounds<PartitionPosition> keyRange)
-    {
-        if (keyRange instanceof Range && ((Range<?>)keyRange).isWrapAround())
-            return keyRange.contains(minKeyBound) || keyRange.contains(maxKeyBound);
-
-        int cmp = keyRange.right.compareTo(minKeyBound);
-        // if right is minimum, it means right is the max token and bigger than maxKey.
-        // if right bound is less than minKeyBound, no intersection
-        if (!keyRange.right.isMinimum() && (!keyRange.inclusiveRight() && cmp == 0 || cmp < 0))
-            return false;
-
-        cmp = keyRange.left.compareTo(maxKeyBound);
-        // if left bound is bigger than maxKeyBound, no intersection
-        return (keyRange.isStartInclusive() || cmp != 0) && cmp <= 0;
-    }
-
-    public long indexFileCacheSize()
-    {
-        return index == null ? 0 : index.indexFileCacheSize();
-    }
-
-    /**
-     * Search on-disk index synchronously
-     *
-     * @param expression to filter on disk index
-     * @param context to track per sstable cache and per query metrics
-
-     * @return range iterator that matches given expression
-     */
-    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext context) throws IOException
-    {
-        return index.search(expression, keyRange, context);
-    }
-
-    @Override
-    public KeyRangeIterator limitToTopKResults(QueryContext context, List<PrimaryKey> primaryKeys, Expression expression) throws IOException
-    {
-        return index.limitToTopKResults(context, primaryKeys, expression);
-    }
-
-    @Override
-    public void close()
-    {
-        FileUtils.closeQuietly(index);
-    }
-
-    @Override
-    public String toString()
-    {
-        return String.format("Segment{metadata=%s}", metadata);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java
deleted file mode 100644
index fbc0775363cf..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.bbtree.NumericIndexWriter;
-import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
-import org.apache.cassandra.index.sai.disk.v1.vector.OnHeapGraph;
-import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-/**
- * Creates an on-heap index data structure to be flushed to an SSTable index.
- */
-@NotThreadSafe
-public abstract class SegmentBuilder
-{
-    private static final Logger logger = LoggerFactory.getLogger(SegmentBuilder.class);
-
-    // Served as safe net in case memory limit is not triggered or when merger merges small segments..
-    public static final long LAST_VALID_SEGMENT_ROW_ID = (Integer.MAX_VALUE / 2) - 1L;
-    private static long testLastValidSegmentRowId = -1;
-
-    /** The number of column indexes being built globally. (Starts at one to avoid divide by zero.) */
-    private static final AtomicInteger ACTIVE_BUILDER_COUNT = new AtomicInteger(0);
-
-    /** Minimum flush size, dynamically updated as segment builds are started and completed/aborted. */
-    private static volatile long minimumFlushBytes;
-    private final NamedMemoryLimiter limiter;
-    private final long lastValidSegmentRowID;
-    private boolean flushed = false;
-    private boolean active = true;
-    // segment metadata
-    private long minSSTableRowId = -1;
-    private long maxSSTableRowId = -1;
-    private long segmentRowIdOffset = 0;
-
-    // in token order
-    private PrimaryKey minKey;
-    private PrimaryKey maxKey;
-    // in termComparator order
-    private ByteBuffer minTerm;
-    private ByteBuffer maxTerm;
-
-    final StorageAttachedIndex index;
-    long totalBytesAllocated;
-    int rowCount = 0;
-    int maxSegmentRowId = -1;
-
-    public static class TrieSegmentBuilder extends SegmentBuilder
-    {
-        protected final SegmentTrieBuffer segmentTrieBuffer;
-
-        public TrieSegmentBuilder(StorageAttachedIndex index, NamedMemoryLimiter limiter)
-        {
-            super(index, limiter);
-
-            segmentTrieBuffer = new SegmentTrieBuffer();
-            totalBytesAllocated = segmentTrieBuffer.memoryUsed();
-        }
-
-        @Override
-        protected long addInternal(ByteBuffer term, int segmentRowId)
-        {
-            return segmentTrieBuffer.add(v -> index.termType().asComparableBytes(term, v), term.limit(), segmentRowId);
-        }
-
-        @Override
-        protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor) throws IOException
-        {
-            SegmentWriter writer = index.termType().isLiteral() ? new LiteralIndexWriter(indexDescriptor, index.identifier())
-                                                                : new NumericIndexWriter(indexDescriptor, index.identifier(), index.termType().fixedSizeOf());
-
-            return writer.writeCompleteSegment(segmentTrieBuffer.iterator());
-        }
-
-        @Override
-        public boolean isEmpty()
-        {
-            return segmentTrieBuffer.numRows() == 0;
-        }
-    }
-
-    public static class VectorSegmentBuilder extends SegmentBuilder
-    {
-        private final OnHeapGraph<Integer> graphIndex;
-
-        public VectorSegmentBuilder(StorageAttachedIndex index, NamedMemoryLimiter limiter)
-        {
-            super(index, limiter);
-            graphIndex = new OnHeapGraph<>(index.termType().indexType(), index.indexWriterConfig(), false);
-        }
-
-        @Override
-        public boolean isEmpty()
-        {
-            return graphIndex.isEmpty();
-        }
-
-        @Override
-        protected long addInternal(ByteBuffer term, int segmentRowId)
-        {
-            return graphIndex.add(term, segmentRowId, OnHeapGraph.InvalidVectorBehavior.IGNORE);
-        }
-
-        @Override
-        protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor) throws IOException
-        {
-            return graphIndex.writeData(indexDescriptor, index.identifier(), p -> p);
-        }
-    }
-
-    public static int getActiveBuilderCount()
-    {
-        return ACTIVE_BUILDER_COUNT.get();
-    }
-
-    private SegmentBuilder(StorageAttachedIndex index, NamedMemoryLimiter limiter)
-    {
-        this.index = index;
-        this.limiter = limiter;
-        lastValidSegmentRowID = testLastValidSegmentRowId >= 0 ? testLastValidSegmentRowId : LAST_VALID_SEGMENT_ROW_ID;
-
-        minimumFlushBytes = limiter.limitBytes() / ACTIVE_BUILDER_COUNT.incrementAndGet();
-    }
-
-    public SegmentMetadata flush(IndexDescriptor indexDescriptor) throws IOException
-    {
-        assert !flushed : "Cannot flush an already flushed segment";
-        flushed = true;
-
-        if (getRowCount() == 0)
-        {
-            logger.warn(index.identifier().logMessage("No rows to index during flush of SSTable {}."), indexDescriptor.sstableDescriptor);
-            return null;
-        }
-
-        SegmentMetadata.ComponentMetadataMap indexMetas = flushInternal(indexDescriptor);
-
-        return new SegmentMetadata(segmentRowIdOffset, rowCount, minSSTableRowId, maxSSTableRowId, minKey, maxKey, minTerm, maxTerm, indexMetas);
-    }
-
-    public long add(ByteBuffer term, PrimaryKey key, long sstableRowId)
-    {
-        assert !flushed : "Cannot add to a flushed segment.";
-        assert sstableRowId >= maxSSTableRowId;
-        minSSTableRowId = minSSTableRowId < 0 ? sstableRowId : minSSTableRowId;
-        maxSSTableRowId = sstableRowId;
-
-        assert maxKey == null || maxKey.compareTo(key) <= 0;
-        if (minKey == null)
-            minKey = key;
-        maxKey = key;
-
-        minTerm = index.termType().min(term, minTerm);
-        maxTerm = index.termType().max(term, maxTerm);
-
-        if (rowCount == 0)
-        {
-            // use first global rowId in the segment as segment rowId offset
-            segmentRowIdOffset = sstableRowId;
-        }
-
-        rowCount++;
-
-        // segmentRowIdOffset should encode sstableRowId into Integer
-        int segmentRowId = castToSegmentRowId(sstableRowId, segmentRowIdOffset);
-        maxSegmentRowId = Math.max(maxSegmentRowId, segmentRowId);
-
-        long bytesAllocated = addInternal(term, segmentRowId);
-        totalBytesAllocated += bytesAllocated;
-
-        return bytesAllocated;
-    }
-
-    public static int castToSegmentRowId(long sstableRowId, long segmentRowIdOffset)
-    {
-        return Math.toIntExact(sstableRowId - segmentRowIdOffset);
-    }
-
-    public long totalBytesAllocated()
-    {
-        return totalBytesAllocated;
-    }
-
-    public boolean hasReachedMinimumFlushSize()
-    {
-        return totalBytesAllocated >= minimumFlushBytes;
-    }
-
-    public long getMinimumFlushBytes()
-    {
-        return minimumFlushBytes;
-    }
-
-    /**
-     * This method does three things:
-     * <p>
-     * 1. It decrements active builder count and updates the global minimum flush size to reflect that.
-     * 2. It releases the builder's memory against its limiter.
-     * 3. It defensively marks the builder inactive to make sure nothing bad happens if we try to close it twice.
-     *
-     * @return the number of bytes used by the memory limiter after releasing this builder
-     */
-    public long release()
-    {
-        if (active)
-        {
-            minimumFlushBytes = limiter.limitBytes() / ACTIVE_BUILDER_COUNT.getAndDecrement();
-            long used = limiter.decrement(totalBytesAllocated);
-            active = false;
-            return used;
-        }
-
-        logger.warn(index.identifier().logMessage("Attempted to release storage-attached index segment builder memory after builder marked inactive."));
-        return limiter.currentBytesUsed();
-    }
-
-    public abstract boolean isEmpty();
-
-    protected abstract long addInternal(ByteBuffer term, int segmentRowId);
-
-    protected abstract SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor) throws IOException;
-
-    public int getRowCount()
-    {
-        return rowCount;
-    }
-
-    /**
-     * @return true if next SSTable row ID exceeds max segment row ID
-     */
-    public boolean exceedsSegmentLimit(long ssTableRowId)
-    {
-        if (getRowCount() == 0)
-            return false;
-
-        // To handle the case where there are many non-indexable rows. eg. rowId-1 and rowId-3B are indexable,
-        // the rest are non-indexable. We should flush them as 2 separate segments, because rowId-3B is going
-        // to cause error in on-disk index structure with 2B limitation.
-        return ssTableRowId - segmentRowIdOffset > lastValidSegmentRowID;
-    }
-
-    @VisibleForTesting
-    public static void updateLastValidSegmentRowId(long lastValidSegmentRowID)
-    {
-        testLastValidSegmentRowId = lastValidSegmentRowID;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentMetadata.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentMetadata.java
deleted file mode 100644
index 3ae1bd55d4de..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentMetadata.java
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Stream;
-
-import com.google.common.collect.ImmutableMap;
-
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.v1.MetadataSource;
-import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexOutput;
-
-/**
- * Multiple {@link SegmentMetadata} are stored in {@link IndexComponent#META} file, each corresponds to an on-disk
- * index segment.
- */
-public class SegmentMetadata
-{
-    private static final String NAME = "SegmentMetadata";
-
-    /**
-     * Used to retrieve sstableRowId which equals to offset plus segmentRowId.
-     */
-    public final long rowIdOffset;
-
-    /**
-     * Min and max sstable rowId in current segment.
-     * <p>
-     * For index generated by compaction, minSSTableRowId is the same as segmentRowIdOffset.
-     * But for flush, segmentRowIdOffset is taken from previous segment's maxSSTableRowId.
-     */
-    public final long minSSTableRowId;
-    public final long maxSSTableRowId;
-
-    /**
-     * number of indexed rows (aka. a pair of term and segmentRowId) in the current segment
-     */
-    public final long numRows;
-
-    /**
-     * Ordered by their token position in current segment
-     */
-    public final PrimaryKey minKey;
-    public final PrimaryKey maxKey;
-
-    /**
-     * Minimum and maximum indexed column value ordered by its {@link org.apache.cassandra.db.marshal.AbstractType}.
-     */
-    public final ByteBuffer minTerm;
-    public final ByteBuffer maxTerm;
-
-    /**
-     * Root, offset, length for each index structure in the segment.
-     * <p>
-     * Note: postings block offsets are stored in terms dictionary, no need to worry about its root.
-     */
-    public final ComponentMetadataMap componentMetadatas;
-
-    public SegmentMetadata(long rowIdOffset,
-                           long numRows,
-                           long minSSTableRowId,
-                           long maxSSTableRowId,
-                           PrimaryKey minKey,
-                           PrimaryKey maxKey,
-                           ByteBuffer minTerm,
-                           ByteBuffer maxTerm,
-                           ComponentMetadataMap componentMetadatas)
-    {
-        assert numRows < Integer.MAX_VALUE;
-        Objects.requireNonNull(minKey);
-        Objects.requireNonNull(maxKey);
-        Objects.requireNonNull(minTerm);
-        Objects.requireNonNull(maxTerm);
-
-        this.rowIdOffset = rowIdOffset;
-        this.minSSTableRowId = minSSTableRowId;
-        this.maxSSTableRowId = maxSSTableRowId;
-        this.numRows = numRows;
-        this.minKey = minKey;
-        this.maxKey = maxKey;
-        this.minTerm = minTerm;
-        this.maxTerm = maxTerm;
-        this.componentMetadatas = componentMetadatas;
-    }
-
-    private SegmentMetadata(DataInput input, PrimaryKey.Factory primaryKeyFactory) throws IOException
-    {
-        this.rowIdOffset = input.readLong();
-        this.numRows = input.readLong();
-        this.minSSTableRowId = input.readLong();
-        this.maxSSTableRowId = input.readLong();
-        this.minKey = primaryKeyFactory.fromComparableBytes(ByteSource.fixedLength(readBytes(input)));
-        this.maxKey = primaryKeyFactory.fromComparableBytes(ByteSource.fixedLength(readBytes(input)));
-        this.minTerm = readBytes(input);
-        this.maxTerm = readBytes(input);
-        this.componentMetadatas = new ComponentMetadataMap(input);
-    }
-
-    public int toSegmentRowId(long sstableRowId)
-    {
-        return Math.toIntExact(sstableRowId - rowIdOffset);
-    }
-
-    public static List<SegmentMetadata> load(MetadataSource source, PrimaryKey.Factory primaryKeyFactory) throws IOException
-    {
-        DataInput input = source.get(NAME);
-
-        int segmentCount = input.readVInt();
-
-        List<SegmentMetadata> segmentMetadata = new ArrayList<>(segmentCount);
-
-        for (int i = 0; i < segmentCount; i++)
-        {
-            segmentMetadata.add(new SegmentMetadata(input, primaryKeyFactory));
-        }
-
-        return segmentMetadata;
-    }
-
-    /**
-     * Writes disk metadata for the given segment list.
-     */
-    public static void write(MetadataWriter writer, List<SegmentMetadata> segments) throws IOException
-    {
-        try (IndexOutput output = writer.builder(NAME))
-        {
-            output.writeVInt(segments.size());
-
-            for (SegmentMetadata metadata : segments)
-            {
-                output.writeLong(metadata.rowIdOffset);
-                output.writeLong(metadata.numRows);
-                output.writeLong(metadata.minSSTableRowId);
-                output.writeLong(metadata.maxSSTableRowId);
-
-                Stream.of(ByteSourceInverse.readBytes(metadata.minKey.asComparableBytes(ByteComparable.Version.OSS50)),
-                          ByteSourceInverse.readBytes(metadata.maxKey.asComparableBytes(ByteComparable.Version.OSS50)))
-                      .forEach(b -> writeBytes(b, output));
-                Stream.of(metadata.minTerm, metadata.maxTerm).forEach(bb -> writeBytes(bb, output));
-
-                metadata.componentMetadatas.write(output);
-            }
-        }
-    }
-
-    @Override
-    public String toString()
-    {
-        return "SegmentMetadata{" +
-               "rowIdOffset=" + rowIdOffset +
-               ", minSSTableRowId=" + minSSTableRowId +
-               ", maxSSTableRowId=" + maxSSTableRowId +
-               ", numRows=" + numRows +
-               ", componentMetadatas=" + componentMetadatas +
-               '}';
-    }
-
-    private static ByteBuffer readBytes(DataInput input) throws IOException
-    {
-        int len = input.readInt();
-        byte[] bytes = new byte[len];
-        input.readBytes(bytes, 0, len);
-        return ByteBuffer.wrap(bytes);
-    }
-
-    private static void writeBytes(ByteBuffer buf, IndexOutput out)
-    {
-        try
-        {
-            byte[] bytes = ByteBufferUtil.getArray(buf);
-            out.writeInt(bytes.length);
-            out.writeBytes(bytes, 0, bytes.length);
-        }
-        catch (IOException e)
-        {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    private static void writeBytes(byte[] bytes, IndexOutput out)
-    {
-        try
-        {
-            out.writeInt(bytes.length);
-            out.writeBytes(bytes, 0, bytes.length);
-        }
-        catch (IOException ioe)
-        {
-            throw new RuntimeException(ioe);
-        }
-    }
-
-    long getIndexRoot(IndexComponent indexComponent)
-    {
-        return componentMetadatas.get(indexComponent).root;
-    }
-
-    public static class ComponentMetadataMap
-    {
-        private final Map<IndexComponent, ComponentMetadata> metas = new EnumMap<>(IndexComponent.class);
-
-        ComponentMetadataMap(DataInput input) throws IOException
-        {
-            int size = input.readInt();
-
-            for (int i = 0; i < size; i++)
-            {
-                metas.put(IndexComponent.valueOf(input.readString()), new ComponentMetadata(input));
-            }
-        }
-
-        public ComponentMetadataMap()
-        {
-        }
-
-        public void put(IndexComponent indexComponent, long root, long offset, long length)
-        {
-            metas.put(indexComponent, new ComponentMetadata(root, offset, length));
-        }
-
-        public void put(IndexComponent indexComponent, long root, long offset, long length, Map<String, String> additionalMap)
-        {
-            metas.put(indexComponent, new ComponentMetadata(root, offset, length, additionalMap));
-        }
-
-        private void write(IndexOutput output) throws IOException
-        {
-            output.writeInt(metas.size());
-
-            for (Map.Entry<IndexComponent, ComponentMetadata> entry : metas.entrySet())
-            {
-                output.writeString(entry.getKey().name());
-                entry.getValue().write(output);
-            }
-        }
-
-        public ComponentMetadata get(IndexComponent indexComponent)
-        {
-            if (!metas.containsKey(indexComponent))
-                throw new IllegalArgumentException(indexComponent + " ComponentMetadata not found");
-
-            return metas.get(indexComponent);
-        }
-
-        public Map<String, Map<String, String>> asMap()
-        {
-            Map<String, Map<String, String>> metaAttributes = new HashMap<>();
-
-            for (Map.Entry<IndexComponent, ComponentMetadata> entry : metas.entrySet())
-            {
-                String name = entry.getKey().name();
-                ComponentMetadata metadata = entry.getValue();
-
-                Map<String, String> componentAttributes = metadata.asMap();
-
-                assert !metaAttributes.containsKey(name) : "Found duplicate index type: " + name;
-                metaAttributes.put(name, componentAttributes);
-            }
-
-            return metaAttributes;
-        }
-
-        @Override
-        public String toString()
-        {
-            return "ComponentMetadataMap{" +
-                   "metas=" + metas +
-                   '}';
-        }
-
-        public double indexSize()
-        {
-            return metas.values().stream().mapToLong(meta -> meta.length).sum();
-        }
-    }
-
-    public static class ComponentMetadata
-    {
-        public static final String ROOT = "Root";
-        public static final String OFFSET = "Offset";
-        public static final String LENGTH = "Length";
-
-        public final long root;
-        public final long offset;
-        public final long length;
-        public final Map<String,String> attributes;
-
-        ComponentMetadata(long root, long offset, long length)
-        {
-            this.root = root;
-            this.offset = offset;
-            this.length = length;
-            this.attributes = Collections.emptyMap();
-        }
-
-        ComponentMetadata(long root, long offset, long length, Map<String, String> attributes)
-        {
-            this.root = root;
-            this.offset = offset;
-            this.length = length;
-            this.attributes = attributes;
-        }
-
-        ComponentMetadata(DataInput input) throws IOException
-        {
-            this.root = input.readLong();
-            this.offset = input.readLong();
-            this.length = input.readLong();
-            int size = input.readInt();
-
-            attributes = new HashMap<>(size);
-            for (int x=0; x < size; x++)
-            {
-                String key = input.readString();
-                String value = input.readString();
-
-                attributes.put(key, value);
-            }
-        }
-
-        public void write(IndexOutput output) throws IOException
-        {
-            output.writeLong(root);
-            output.writeLong(offset);
-            output.writeLong(length);
-
-            output.writeInt(attributes.size());
-            for (Map.Entry<String,String> entry : attributes.entrySet())
-            {
-                output.writeString(entry.getKey());
-                output.writeString(entry.getValue());
-            }
-        }
-
-        @Override
-        public String toString()
-        {
-            return String.format("ComponentMetadata{root=%d, offset=%d, length=%d, attributes=%s}", root, offset, length, attributes.toString());
-        }
-
-        public Map<String, String> asMap()
-        {
-            return ImmutableMap.<String, String>builder().putAll(attributes).put(OFFSET, Long.toString(offset)).put(LENGTH, Long.toString(length)).put(ROOT, Long.toString(root)).build();
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentOrdering.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentOrdering.java
deleted file mode 100644
index 616b0ea86d97..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentOrdering.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-/**
- * A {@link SegmentOrdering} orders and limits a list of {@link PrimaryKey}s.
- * <p>
- * When using {@link SegmentOrdering} there are several steps to
- * build the list of Primary Keys to be ordered and limited:
- * <p>
- * 1. Find all primary keys that match each non-ordering query predicate.
- * 2. Union and intersect the results of step 1 to build a single {@link KeyRangeIterator}
- *    ordered by {@link PrimaryKey}.
- * 3. Filter out any shadowed primary keys.
- * 4. Fan the primary keys from step 3 out to each sstable segment to order and limit each
- *    list of primary keys.
- * <p>
- * SegmentOrdering handles the fourth step.
- * <p>
- * Note: a segment ordering is only used when a query has both ordering and non-ordering predicates.
- * Where a query has only ordering predicates, the ordering is handled by
- * {@link org.apache.cassandra.index.sai.disk.SSTableIndex#search(Expression, AbstractBounds, QueryContext)}.
- */
-public interface SegmentOrdering
-{
-    /**
-     * Reorder, limit, and put back into original order the results from a single sstable
-     */
-    default KeyRangeIterator limitToTopKResults(QueryContext queryContext, List<PrimaryKey> primaryKeys, Expression expression) throws IOException
-    {
-        throw new UnsupportedOperationException();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentTrieBuffer.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentTrieBuffer.java
deleted file mode 100644
index 72c9add3132c..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentTrieBuffer.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.atomic.LongAdder;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.tries.InMemoryTrie;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.lucene.util.packed.PackedInts;
-import org.apache.lucene.util.packed.PackedLongValues;
-
-/**
- * On-heap buffer for values that provides a sorted view of itself as an {@link Iterator}.
- */
-@NotThreadSafe
-public class SegmentTrieBuffer
-{
-    private static final int MAX_RECURSIVE_TERM_LENGTH = 128;
-
-    private final InMemoryTrie<PackedLongValues.Builder> trie;
-    private final PostingsAccumulator postingsAccumulator;
-    private int numRows;
-
-    public SegmentTrieBuffer()
-    {
-        trie = new InMemoryTrie<>(DatabaseDescriptor.getMemtableAllocationType().toBufferType());
-        postingsAccumulator = new PostingsAccumulator();
-    }
-
-    public int numRows()
-    {
-        return numRows;
-    }
-
-    public long memoryUsed()
-    {
-        return trie.sizeOnHeap() + postingsAccumulator.heapAllocations();
-    }
-
-    public long add(ByteComparable term, int termLength, int segmentRowId)
-    {
-        final long initialSizeOnHeap = trie.sizeOnHeap();
-        final long reducerHeapSize = postingsAccumulator.heapAllocations();
-
-        try
-        {
-            trie.putSingleton(term, segmentRowId, postingsAccumulator, termLength <= MAX_RECURSIVE_TERM_LENGTH);
-        }
-        catch (InMemoryTrie.SpaceExhaustedException e)
-        {
-            throw Throwables.unchecked(e);
-        }
-
-        numRows++;
-        return (trie.sizeOnHeap() - initialSizeOnHeap) + (postingsAccumulator.heapAllocations() - reducerHeapSize);
-    }
-
-    public Iterator<IndexEntry> iterator()
-    {
-        var iterator = trie.entrySet().iterator();
-
-        return new Iterator<>()
-        {
-            @Override
-            public boolean hasNext()
-            {
-                return iterator.hasNext();
-            }
-
-            @Override
-            public IndexEntry next()
-            {
-                Map.Entry<ByteComparable, PackedLongValues.Builder> entry = iterator.next();
-                PackedLongValues postings = entry.getValue().build();
-                PackedLongValues.Iterator postingsIterator = postings.iterator();
-                return IndexEntry.create(entry.getKey(), new PostingList()
-                {
-                    @Override
-                    public long nextPosting()
-                    {
-                        if (postingsIterator.hasNext())
-                            return postingsIterator.next();
-                        return END_OF_STREAM;
-                    }
-
-                    @Override
-                    public long size()
-                    {
-                        return postings.size();
-                    }
-
-                    @Override
-                    public long advance(long targetRowID)
-                    {
-                        throw new UnsupportedOperationException();
-                    }
-                });
-            }
-        };
-    }
-
-    private static class PostingsAccumulator implements InMemoryTrie.UpsertTransformer<PackedLongValues.Builder, Integer>
-    {
-        private final LongAdder heapAllocations = new LongAdder();
-
-        @Override
-        public PackedLongValues.Builder apply(PackedLongValues.Builder existing, Integer rowID)
-        {
-            if (existing == null)
-            {
-                existing = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
-                heapAllocations.add(existing.ramBytesUsed());
-            }
-            long ramBefore = existing.ramBytesUsed();
-            existing.add(rowID);
-            heapAllocations.add(existing.ramBytesUsed() - ramBefore);
-            return existing;
-        }
-
-        long heapAllocations()
-        {
-            return heapAllocations.longValue();
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentWriter.java
deleted file mode 100644
index d30a84620717..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentWriter.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-
-public interface SegmentWriter
-{
-    /**
-     * Appends a set of terms and associated postings to their respective overall SSTable component files.
-     *
-     * @param indexEntryIterator an {@link Iterator} of {@link IndexEntry}s sorted in term order.
-     *
-     * @return metadata describing the location of this inverted index in the overall SSTable terms and postings component files
-     */
-    SegmentMetadata.ComponentMetadataMap writeCompleteSegment(Iterator<IndexEntry> indexEntryIterator) throws IOException;
-
-    /**
-     * Returns the number of rows written to the segment
-     *
-     * @return the number of rows
-     */
-    long getNumberOfRows();
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/VectorIndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/VectorIndexSegmentSearcher.java
deleted file mode 100644
index dc6b8fa7065a..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/VectorIndexSegmentSearcher.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.List;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.github.jbellis.jvector.util.Bits;
-import io.github.jbellis.jvector.util.SparseFixedBitSet;
-import org.agrona.collections.IntArrayList;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.VectorQueryContext;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
-import org.apache.cassandra.index.sai.disk.v1.vector.DiskAnn;
-import org.apache.cassandra.index.sai.disk.v1.vector.OptimizeFor;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.iterators.KeyRangeListIterator;
-import org.apache.cassandra.index.sai.memory.VectorMemoryIndex;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.AtomicRatio;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.RangeUtil;
-import org.apache.cassandra.tracing.Tracing;
-
-import static java.lang.Math.max;
-import static java.lang.Math.min;
-
-/**
- * Executes ANN search against a vector graph for an individual index segment.
- */
-public class VectorIndexSegmentSearcher extends IndexSegmentSearcher
-{
-    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-    private final DiskAnn graph;
-    private final int globalBruteForceRows;
-    private final AtomicRatio actualExpectedRatio = new AtomicRatio();
-    private final ThreadLocal<SparseFixedBitSet> cachedBitSets;
-    private final OptimizeFor optimizeFor;
-
-    VectorIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
-                               PerColumnIndexFiles perIndexFiles,
-                               SegmentMetadata segmentMetadata,
-                               StorageAttachedIndex index) throws IOException
-    {
-        super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index);
-        graph = new DiskAnn(segmentMetadata.componentMetadatas, perIndexFiles, index.indexWriterConfig());
-        cachedBitSets = ThreadLocal.withInitial(() -> new SparseFixedBitSet(graph.size()));
-        globalBruteForceRows = Integer.MAX_VALUE;
-        optimizeFor = index.indexWriterConfig().getOptimizeFor();
-    }
-
-    @Override
-    public long indexFileCacheSize()
-    {
-        return graph.ramBytesUsed();
-    }
-
-    @Override
-    public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context) throws IOException
-    {
-        int limit = context.vectorContext().limit();
-
-        if (logger.isTraceEnabled())
-            logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), exp);
-
-        if (exp.getIndexOperator() != Expression.IndexOperator.ANN)
-            throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression during ANN index query: " + exp));
-
-        int topK = optimizeFor.topKFor(limit);
-        BitsOrPostingList bitsOrPostingList = bitsOrPostingListForKeyRange(context.vectorContext(), keyRange, topK);
-        if (bitsOrPostingList.skipANN())
-            return toPrimaryKeyIterator(bitsOrPostingList.postingList(), context);
-
-        float[] queryVector = index.termType().decomposeVector(exp.lower().value.raw.duplicate());
-        var vectorPostings = graph.search(queryVector, topK, limit, bitsOrPostingList.getBits());
-        if (bitsOrPostingList.expectedNodesVisited >= 0)
-            updateExpectedNodes(vectorPostings.getVisitedCount(), bitsOrPostingList.expectedNodesVisited);
-        return toPrimaryKeyIterator(vectorPostings, context);
-    }
-
-    /**
-     * Return bit set we need to search the graph; otherwise return posting list to bypass the graph
-     */
-    private BitsOrPostingList bitsOrPostingListForKeyRange(VectorQueryContext context, AbstractBounds<PartitionPosition> keyRange, int limit) throws IOException
-    {
-        try (PrimaryKeyMap primaryKeyMap = primaryKeyMapFactory.newPerSSTablePrimaryKeyMap())
-        {
-            // not restricted
-            if (RangeUtil.coversFullRing(keyRange))
-                return new BitsOrPostingList(context.bitsetForShadowedPrimaryKeys(metadata, primaryKeyMap, graph));
-
-            // it will return the next row id if given key is not found.
-            long minSSTableRowId = primaryKeyMap.ceiling(keyRange.left.getToken());
-            // If we didn't find the first key, we won't find the last primary key either
-            if (minSSTableRowId < 0)
-                return new BitsOrPostingList(PostingList.EMPTY);
-            long maxSSTableRowId = getMaxSSTableRowId(primaryKeyMap, keyRange.right);
-
-            if (minSSTableRowId > maxSSTableRowId)
-                return new BitsOrPostingList(PostingList.EMPTY);
-
-            // if it covers entire segment, skip bit set
-            if (minSSTableRowId <= metadata.minSSTableRowId && maxSSTableRowId >= metadata.maxSSTableRowId)
-                return new BitsOrPostingList(context.bitsetForShadowedPrimaryKeys(metadata, primaryKeyMap, graph));
-
-            minSSTableRowId = Math.max(minSSTableRowId, metadata.minSSTableRowId);
-            maxSSTableRowId = min(maxSSTableRowId, metadata.maxSSTableRowId);
-
-            // If num of matches are not bigger than limit, skip ANN.
-            // (nRows should not include shadowed rows, but context doesn't break those out by segment,
-            // so we will live with the inaccuracy.)
-            var nRows = Math.toIntExact(maxSSTableRowId - minSSTableRowId + 1);
-            int maxBruteForceRows = min(globalBruteForceRows, maxBruteForceRows(limit, nRows, graph.size()));
-            logger.trace("Search range covers {} rows; max brute force rows is {} for sstable index with {} nodes, LIMIT {}",
-                         nRows, maxBruteForceRows, graph.size(), limit);
-            Tracing.trace("Search range covers {} rows; max brute force rows is {} for sstable index with {} nodes, LIMIT {}",
-                          nRows, maxBruteForceRows, graph.size(), limit);
-            if (nRows <= maxBruteForceRows)
-            {
-                IntArrayList postings = new IntArrayList(Math.toIntExact(nRows), -1);
-                for (long sstableRowId = minSSTableRowId; sstableRowId <= maxSSTableRowId; sstableRowId++)
-                {
-                    if (context.shouldInclude(sstableRowId, primaryKeyMap))
-                        postings.addInt(metadata.toSegmentRowId(sstableRowId));
-                }
-                return new BitsOrPostingList(new IntArrayPostingList(postings.toIntArray()));
-            }
-
-            // create a bitset of ordinals corresponding to the rows in the given key range
-            SparseFixedBitSet bits = bitSetForSearch();
-            boolean hasMatches = false;
-            try (var ordinalsView = graph.getOrdinalsView())
-            {
-                for (long sstableRowId = minSSTableRowId; sstableRowId <= maxSSTableRowId; sstableRowId++)
-                {
-                    if (context.shouldInclude(sstableRowId, primaryKeyMap))
-                    {
-                        int segmentRowId = metadata.toSegmentRowId(sstableRowId);
-                        int ordinal = ordinalsView.getOrdinalForRowId(segmentRowId);
-                        if (ordinal >= 0)
-                        {
-                            bits.set(ordinal);
-                            hasMatches = true;
-                        }
-                    }
-                }
-            }
-            catch (IOException e)
-            {
-                throw new RuntimeException(e);
-            }
-
-            if (!hasMatches)
-                return new BitsOrPostingList(PostingList.EMPTY);
-
-            return new BitsOrPostingList(bits, VectorMemoryIndex.expectedNodesVisited(limit, nRows, graph.size()));
-        }
-    }
-
-    private long getMaxSSTableRowId(PrimaryKeyMap primaryKeyMap, PartitionPosition right)
-    {
-        // if the right token is the minimum token, there is no upper bound on the keyRange and
-        // we can save a lookup by using the maxSSTableRowId
-        if (right.isMinimum())
-            return metadata.maxSSTableRowId;
-
-        long max = primaryKeyMap.floor(right.getToken());
-        if (max < 0)
-            return metadata.maxSSTableRowId;
-        return max;
-    }
-
-    private SparseFixedBitSet bitSetForSearch()
-    {
-        var bits = cachedBitSets.get();
-        bits.clear();
-        return bits;
-    }
-
-    @Override
-    public KeyRangeIterator limitToTopKResults(QueryContext context, List<PrimaryKey> primaryKeys, Expression expression) throws IOException
-    {
-        int limit = context.vectorContext().limit();
-        // VSTODO would it be better to do a binary search to find the boundaries?
-        List<PrimaryKey> keysInRange = primaryKeys.stream()
-                                                  .dropWhile(k -> k.compareTo(metadata.minKey) < 0)
-                                                  .takeWhile(k -> k.compareTo(metadata.maxKey) <= 0)
-                                                  .collect(Collectors.toList());
-        if (keysInRange.isEmpty())
-            return KeyRangeIterator.empty();
-        int topK = optimizeFor.topKFor(limit);
-        if (shouldUseBruteForce(topK, limit, keysInRange.size()))
-            return new KeyRangeListIterator(metadata.minKey, metadata.maxKey, keysInRange);
-
-        try (PrimaryKeyMap primaryKeyMap = primaryKeyMapFactory.newPerSSTablePrimaryKeyMap())
-        {
-            // the iterator represents keys from the whole table -- we'll only pull of those that
-            // are from our own token range, so we can use row ids to order the results by vector similarity.
-            var maxSegmentRowId = metadata.toSegmentRowId(metadata.maxSSTableRowId);
-            SparseFixedBitSet bits = bitSetForSearch();
-            var rowIds = new IntArrayList();
-            try (var ordinalsView = graph.getOrdinalsView())
-            {
-                for (PrimaryKey primaryKey : keysInRange)
-                {
-                    long sstableRowId = primaryKeyMap.rowIdFromPrimaryKey(primaryKey);
-                    // skip rows that are not in our segment (or more preciesely, have no vectors that were indexed)
-                    // or are not in this segment (exactRowIdForPrimaryKey returns a negative value for not found)
-                    if (sstableRowId < metadata.minSSTableRowId)
-                        continue;
-
-                    // if sstable row id has exceeded current ANN segment, stop
-                    if (sstableRowId > metadata.maxSSTableRowId)
-                        break;
-
-                    int segmentRowId = metadata.toSegmentRowId(sstableRowId);
-                    rowIds.add(segmentRowId);
-                    // VSTODO now that we know the size of keys evaluated, is it worth doing the brute
-                    // force check eagerly to potentially skip the PK to sstable row id to ordinal lookup?
-                    int ordinal = ordinalsView.getOrdinalForRowId(segmentRowId);
-                    if (ordinal >= 0)
-                        bits.set(ordinal);
-                }
-            }
-
-            if (shouldUseBruteForce(topK, limit, rowIds.size()))
-                return toPrimaryKeyIterator(new IntArrayPostingList(rowIds.toIntArray()), context);
-
-            // else ask the index to perform a search limited to the bits we created
-            float[] queryVector = index.termType().decomposeVector(expression.lower().value.raw.duplicate());
-            var results = graph.search(queryVector, topK, limit, bits);
-            updateExpectedNodes(results.getVisitedCount(), expectedNodesVisited(topK, maxSegmentRowId, graph.size()));
-            return toPrimaryKeyIterator(results, context);
-        }
-    }
-
-    private boolean shouldUseBruteForce(int topK, int limit, int numRows)
-    {
-        // if we have a small number of results then let TopK processor do exact NN computation
-        var maxBruteForceRows = min(globalBruteForceRows, maxBruteForceRows(topK, numRows, graph.size()));
-        logger.trace("SAI materialized {} rows; max brute force rows is {} for sstable index with {} nodes, LIMIT {}",
-                     numRows, maxBruteForceRows, graph.size(), limit);
-        Tracing.trace("SAI materialized {} rows; max brute force rows is {} for sstable index with {} nodes, LIMIT {}",
-                      numRows, maxBruteForceRows, graph.size(), limit);
-        return numRows <= maxBruteForceRows;
-    }
-
-    private int maxBruteForceRows(int limit, int nPermittedOrdinals, int graphSize)
-    {
-        int expectedNodes = expectedNodesVisited(limit, nPermittedOrdinals, graphSize);
-        // ANN index will do a bunch of extra work besides the full comparisons (performing PQ similarity for each edge);
-        // brute force from sstable will also do a bunch of extra work (going through trie index to look up row).
-        // VSTODO I'm not sure which one is more expensive (and it depends on things like sstable chunk cache hit ratio)
-        // so I'm leaving it as a 1:1 ratio for now.
-        return max(limit, expectedNodes);
-    }
-
-    private int expectedNodesVisited(int limit, int nPermittedOrdinals, int graphSize)
-    {
-        var observedRatio = actualExpectedRatio.getUpdateCount() >= 10 ? actualExpectedRatio.get() : 1.0;
-        return (int) (observedRatio * VectorMemoryIndex.expectedNodesVisited(limit, nPermittedOrdinals, graphSize));
-    }
-
-    private void updateExpectedNodes(int actualNodesVisited, int expectedNodesVisited)
-    {
-        assert expectedNodesVisited >= 0 : expectedNodesVisited;
-        assert actualNodesVisited >= 0 : actualNodesVisited;
-        if (actualNodesVisited >= 1000 && actualNodesVisited > 2 * expectedNodesVisited || expectedNodesVisited > 2 * actualNodesVisited)
-            logger.warn("Predicted visiting {} nodes, but actually visited {}", expectedNodesVisited, actualNodesVisited);
-        actualExpectedRatio.update(actualNodesVisited, expectedNodesVisited);
-    }
-
-    @Override
-    public String toString()
-    {
-        return MoreObjects.toStringHelper(this).add("index", index).toString();
-    }
-
-    @Override
-    public void close() throws IOException
-    {
-        graph.close();
-    }
-
-    private static class BitsOrPostingList
-    {
-        private final Bits bits;
-        private final int expectedNodesVisited;
-        private final PostingList postingList;
-
-        public BitsOrPostingList(@Nullable Bits bits, int expectedNodesVisited)
-        {
-            this.bits = bits;
-            this.expectedNodesVisited = expectedNodesVisited;
-            this.postingList = null;
-        }
-
-        public BitsOrPostingList(@Nullable Bits bits)
-        {
-            this.bits = bits;
-            this.postingList = null;
-            this.expectedNodesVisited = -1;
-        }
-
-        public BitsOrPostingList(PostingList postingList)
-        {
-            this.bits = null;
-            this.postingList = Preconditions.checkNotNull(postingList);
-            this.expectedNodesVisited = -1;
-        }
-
-        @Nullable
-        public Bits getBits()
-        {
-            Preconditions.checkState(!skipANN());
-            return bits;
-        }
-
-        public PostingList postingList()
-        {
-            Preconditions.checkState(skipANN());
-            return postingList;
-        }
-
-        public boolean skipANN()
-        {
-            return postingList != null;
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/DocLengthsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/DocLengthsWriter.java
new file mode 100644
index 000000000000..90dc6b5c6ff9
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/DocLengthsWriter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.trie;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.agrona.collections.Int2IntHashMap;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+
+/**
+ * Writes document length information to disk for use in text scoring
+ */
+public class DocLengthsWriter implements Closeable
+{
+    private final IndexOutputWriter output;
+
+    public DocLengthsWriter(IndexComponents.ForWrite components) throws IOException
+    {
+        this.output = components.addOrGet(IndexComponentType.DOC_LENGTHS).openOutput(true);
+        SAICodecUtils.writeHeader(output);
+    }
+
+    public void writeDocLengths(Int2IntHashMap lengths) throws IOException
+    {
+        // Calculate max row ID from doc lengths map
+        int maxRowId = -1;
+        for (var keyIterator = lengths.keySet().iterator(); keyIterator.hasNext(); )
+        {
+            int key = keyIterator.nextValue();
+            if (key > maxRowId)
+                maxRowId = key;
+        }
+
+        // write out the doc lengths in row order
+        for (int rowId = 0; rowId <= maxRowId; rowId++)
+        {
+            final int length = lengths.get(rowId);
+            output.writeInt(length == lengths.missingValue() ? 0 : length);
+        }
+
+        SAICodecUtils.writeFooter(output);
+    }
+
+    public long getFilePointer()
+    {
+        return output.getFilePointer();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        output.close();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/InvertedIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/InvertedIndexWriter.java
new file mode 100644
index 000000000000..a61df65f2239
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/InvertedIndexWriter.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.trie;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.commons.lang3.mutable.MutableLong;
+
+import org.agrona.collections.Int2IntHashMap;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Builds an on-disk inverted index structure: terms dictionary and postings lists.
+ */
+@NotThreadSafe
+public class InvertedIndexWriter implements Closeable
+{
+    private final TrieTermsDictionaryWriter termsDictionaryWriter;
+    private final PostingsWriter postingsWriter;
+    private final DocLengthsWriter docLengthsWriter;
+    private long postingsAdded;
+
+    public InvertedIndexWriter(IndexComponents.ForWrite components) throws IOException
+    {
+        this(components, false);
+    }
+
+    public InvertedIndexWriter(IndexComponents.ForWrite components, boolean writeFrequencies) throws IOException
+    {
+        this.termsDictionaryWriter = new TrieTermsDictionaryWriter(components);
+        this.postingsWriter = new PostingsWriter(components, writeFrequencies);
+        this.docLengthsWriter = Version.latest().onOrAfter(Version.EC) ? new DocLengthsWriter(components) : null;
+    }
+
+    /**
+     * Appends a set of terms and associated postings to their respective overall SSTable component files.
+     *
+     * @param terms an iterator of terms with their associated postings
+     *
+     * @return metadata describing the location of this inverted index in the overall SSTable
+     *         terms and postings component files
+     */
+    public SegmentMetadata.ComponentMetadataMap writeAll(TermsIterator terms, Int2IntHashMap docLengths) throws IOException
+    {
+        // Terms and postings writers are opened in append mode with pointers at the end of their respective files.
+        long termsOffset = termsDictionaryWriter.getStartOffset();
+        long postingsOffset = postingsWriter.getStartOffset();
+
+        while (terms.hasNext())
+        {
+            ByteComparable term = terms.next();
+            try (PostingList postings = terms.postings())
+            {
+                final long offset = postingsWriter.write(postings);
+                if (offset >= 0)
+                    termsDictionaryWriter.add(term, offset);
+            }
+        }
+        postingsAdded = postingsWriter.getTotalPostings();
+        MutableLong footerPointer = new MutableLong();
+        long termsRoot = termsDictionaryWriter.complete(footerPointer);
+        postingsWriter.complete();
+
+        long termsLength = termsDictionaryWriter.getFilePointer() - termsOffset;
+        long postingsLength = postingsWriter.getFilePointer() - postingsOffset;
+
+        SegmentMetadata.ComponentMetadataMap components = new SegmentMetadata.ComponentMetadataMap();
+
+        Map<String,String> map = new HashMap<>(2);
+        map.put(SAICodecUtils.FOOTER_POINTER, "" + footerPointer.getValue());
+
+        // Postings list file pointers are stored directly in TERMS_DATA, so a root is not needed.
+        components.put(IndexComponentType.POSTING_LISTS, -1, postingsOffset, postingsLength);
+        components.put(IndexComponentType.TERMS_DATA, termsRoot, termsOffset, termsLength, map);
+
+        // Write doc lengths
+        if (docLengthsWriter != null)
+        {
+            long docLengthsOffset = docLengthsWriter.getFilePointer();
+            docLengthsWriter.writeDocLengths(docLengths);
+            long docLengthsLength = docLengthsWriter.getFilePointer() - docLengthsOffset;
+            components.put(IndexComponentType.DOC_LENGTHS, -1, docLengthsOffset, docLengthsLength);
+        }
+
+        return components;
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        postingsWriter.close();
+        termsDictionaryWriter.close();
+        if (docLengthsWriter != null)
+            docLengthsWriter.close();
+    }
+
+    /**
+     * @return total number of row IDs added to posting lists
+     */
+    public long getPostingsCount()
+    {
+        return postingsAdded;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/LiteralIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/LiteralIndexWriter.java
deleted file mode 100644
index 616ef603a21d..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/LiteralIndexWriter.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.trie;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import org.apache.commons.lang3.mutable.MutableLong;
-
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentWriter;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.postings.PostingList;
-
-/**
- * Builds an on-disk inverted index structure: terms dictionary and postings lists.
- */
-@NotThreadSafe
-public class LiteralIndexWriter implements SegmentWriter
-{
-    private final IndexDescriptor indexDescriptor;
-    private final IndexIdentifier indexIdentifier;
-    private long postingsAdded;
-
-    public LiteralIndexWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier)
-    {
-        this.indexDescriptor = indexDescriptor;
-        this.indexIdentifier = indexIdentifier;
-    }
-
-    @Override
-    public SegmentMetadata.ComponentMetadataMap writeCompleteSegment(Iterator<IndexEntry> iterator) throws IOException
-    {
-        SegmentMetadata.ComponentMetadataMap components = new SegmentMetadata.ComponentMetadataMap();
-
-        try (TrieTermsDictionaryWriter termsDictionaryWriter = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier);
-             PostingsWriter postingsWriter = new PostingsWriter(indexDescriptor, indexIdentifier))
-        {
-            // Terms and postings writers are opened in append mode with pointers at the end of their respective files.
-            long termsOffset = termsDictionaryWriter.getStartOffset();
-            long postingsOffset = postingsWriter.getStartOffset();
-
-            while (iterator.hasNext())
-            {
-                IndexEntry indexEntry = iterator.next();
-                try (PostingList postings = indexEntry.postingList)
-                {
-                    long offset = postingsWriter.write(postings);
-                    termsDictionaryWriter.add(indexEntry.term, offset);
-                }
-            }
-            postingsAdded = postingsWriter.getTotalPostings();
-            MutableLong footerPointer = new MutableLong();
-            long termsRoot = termsDictionaryWriter.complete(footerPointer);
-            postingsWriter.complete();
-
-            long termsLength = termsDictionaryWriter.getFilePointer() - termsOffset;
-            long postingsLength = postingsWriter.getFilePointer() - postingsOffset;
-
-            Map<String, String> map = new HashMap<>(2);
-            map.put(SAICodecUtils.FOOTER_POINTER, footerPointer.getValue().toString());
-
-            // Postings list file pointers are stored directly in TERMS_DATA, so a root is not needed.
-            components.put(IndexComponent.POSTING_LISTS, -1, postingsOffset, postingsLength);
-            components.put(IndexComponent.TERMS_DATA, termsRoot, termsOffset, termsLength, map);
-        }
-        return components;
-    }
-
-    @Override
-    public long getNumberOfRows()
-    {
-        return postingsAdded;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/ReverseTrieTermsDictionaryReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/ReverseTrieTermsDictionaryReader.java
new file mode 100644
index 000000000000..ddf8fbcb20ab
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/ReverseTrieTermsDictionaryReader.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.trie;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.tries.ReverseValueIterator;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.SizedInts;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Page-aware reverse iterator reader for a trie terms dictionary written by {@link TrieTermsDictionaryWriter}.
+ */
+public class ReverseTrieTermsDictionaryReader extends ReverseValueIterator<ReverseTrieTermsDictionaryReader> implements Iterator<Pair<ByteComparable, Long>>
+{
+    public ReverseTrieTermsDictionaryReader(Rebufferer rebufferer, long root)
+    {
+        super(rebufferer, root, true, TypeUtil.BYTE_COMPARABLE_VERSION);
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        return super.hasNext();
+    }
+
+    @Override
+    public Pair<ByteComparable, Long> next()
+    {
+        return nextValue(this::getKeyAndPayload);
+    }
+
+    private Pair<ByteComparable, Long> getKeyAndPayload()
+    {
+        return Pair.create(collectedKey(), getPayload(buf, payloadPosition(), payloadFlags()));
+    }
+
+    private static long getPayload(ByteBuffer contents, int payloadPos, int bytes)
+    {
+        return SizedInts.read(contents, payloadPos, bytes);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryReader.java
index b0867da1e361..f4f6477fca7b 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryReader.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryReader.java
@@ -19,15 +19,17 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Iterator;
 import javax.annotation.concurrent.NotThreadSafe;
 
 import org.apache.cassandra.io.tries.SerializationNode;
 import org.apache.cassandra.io.tries.TrieNode;
 import org.apache.cassandra.io.tries.TrieSerializer;
-import org.apache.cassandra.io.tries.Walker;
+import org.apache.cassandra.io.tries.ValueIterator;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.io.util.Rebufferer;
 import org.apache.cassandra.io.util.SizedInts;
+import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
@@ -35,13 +37,27 @@
  * Page-aware random access reader for a trie terms dictionary written by {@link TrieTermsDictionaryWriter}.
  */
 @NotThreadSafe
-public class TrieTermsDictionaryReader extends Walker<TrieTermsDictionaryReader>
+public class TrieTermsDictionaryReader extends ValueIterator<TrieTermsDictionaryReader> implements Iterator<Pair<ByteComparable, Long>>
 {
     public static final long NOT_FOUND = -1;
 
-    public TrieTermsDictionaryReader(Rebufferer rebufferer, long root)
+    public TrieTermsDictionaryReader(Rebufferer rebufferer, long root, ByteComparable.Version version)
     {
-        super(rebufferer, root);
+        super(rebufferer, root, true, version);
+    }
+
+    /**
+     * Creates a reader for a trie terms dictionary range. See {@link ValueIterator} for details.
+     */
+    public TrieTermsDictionaryReader(Rebufferer source,
+                                     long root,
+                                     ByteComparable start,
+                                     ByteComparable end,
+                                     boolean inclStart,
+                                     boolean collecting,
+                                     ByteComparable.Version version)
+    {
+        super(source, root, start, end, inclStart ? LeftBoundTreatment.ADMIT_EXACT : LeftBoundTreatment.GREATER, collecting, version);
     }
 
     public static final TrieSerializer<Long, DataOutputPlus> trieSerializer = new TrieSerializer<>()
@@ -55,39 +71,150 @@ public int sizeofNode(SerializationNode<Long> node, long nodePosition)
         @Override
         public void write(DataOutputPlus dest, SerializationNode<Long> node, long nodePosition) throws IOException
         {
-            TrieNode type = TrieNode.typeFor(node, nodePosition);
-            Long payload = node.payload();
-            int payloadBits = sizeof(payload);
-            type.serialize(dest, node, payloadBits, nodePosition);
-
+            final TrieNode type = TrieNode.typeFor(node, nodePosition);
+            final Long payload = node.payload();
             if (payload != null)
+            {
+                final int payloadBits = SizedInts.nonZeroSize(payload);
+                type.serialize(dest, node, payloadBits, nodePosition);
                 SizedInts.write(dest, payload, payloadBits);
+            }
+            else
+            {
+                type.serialize(dest, node, 0, nodePosition);
+            }
         }
 
         private int sizeof(Long payload)
         {
-            return payload == null ? 0 : SizedInts.nonZeroSize(payload);
+            if (payload != null)
+            {
+                return SizedInts.nonZeroSize(payload);
+            }
+            return 0;
         }
     };
 
     public long exactMatch(ByteComparable key)
     {
-        // Since we are looking for an exact match we are always expecting the follow
-        // to return END_OF_STREAM if the key was found.
-        return follow(key) == ByteSource.END_OF_STREAM ? getCurrentPayload() : NOT_FOUND;
+        int b = follow(key);
+        if (b != ByteSource.END_OF_STREAM)
+        {
+            return NOT_FOUND;
+        }
+        return getCurrentPayload();
     }
 
-    private long getCurrentPayload()
+    /**
+     * Returns the position associated with the least term greater than or equal to the given key, or
+     * a negative value if there is no such term. In order to optimize the search, the trie is traversed
+     * statefully. Therefore, this method only returns correct results when called for increasing keys.
+     * Warning: ceiling is not idempotent. Calling ceiling() twice for the same key will return successive
+     * values instead of the same value. This is acceptable for the current usage of the method.
+     * @param key the prefix to traverse in the trie
+     * @return a position, if found, or a negative value if there is no such position
+     */
+    public long ceiling(ByteComparable key)
     {
-        return getPayloadAt(buf, payloadPosition(), payloadFlags());
+        skipTo(key, LeftBoundTreatment.ADMIT_EXACT);
+        return nextAsLong();
     }
 
-    private long getPayloadAt(ByteBuffer contents, int payloadPos, int bytes)
+    public long nextAsLong()
     {
-        if (bytes == 0)
+        return nextValueAsLong(this::getCurrentPayload, NOT_FOUND);
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        return super.hasNext();
+    }
+
+    @Override
+    public Pair<ByteComparable, Long> next()
+    {
+        return nextValue(this::getKeyAndPayload);
+    }
+
+    private Pair<ByteComparable, Long> getKeyAndPayload()
+    {
+        return Pair.create(collectedKey(), getCurrentPayload());
+    }
+
+    /**
+     * Returns the position associated with the greatest term less than or equal to the given key, or
+     * a negative value if there is no such term.
+     * @param key the prefix to traverse in the trie
+     * @return a position, if found, or a negative value if there is no such position
+     */
+    public long floor(ByteComparable key)
+    {
+        Long result = null;
+        try
+        {
+            result = prefixAndNeighbours(key, TrieTermsDictionaryReader::getPayload);
+        }
+        catch (IOException e)
         {
+            throw new RuntimeException(e);
+        }
+        if (result != null && result != NOT_FOUND)
+            return result;
+        if (lesserBranch == -1)
             return NOT_FOUND;
+        goMax(lesserBranch);
+        return getCurrentPayload();
+    }
+
+    public ByteComparable getMaxTerm()
+    {
+        final TransitionBytesCollector collector = new TransitionBytesCollector(byteComparableVersion);
+        go(root);
+        while (true)
+        {
+            int lastIdx = transitionRange() - 1;
+            long lastChild = transition(lastIdx);
+            if (lastIdx < 0)
+            {
+                return collector.toByteComparable();
+            }
+            collector.add(transitionByte(lastIdx));
+            go(lastChild);
+        }
+    }
+
+    public ByteComparable getMinTerm()
+    {
+        final TransitionBytesCollector collector = new TransitionBytesCollector(byteComparableVersion);
+        go(root);
+        while (true)
+        {
+            int payloadBits = payloadFlags();
+            if (payloadBits > 0)
+            {
+                return collector.toByteComparable();
+            }
+            collector.add(transitionByte(0));
+            go(transition(0));
         }
+    }
+
+    private long getCurrentPayload()
+    {
+        return getPayload(payloadPosition(), payloadFlags());
+    }
+
+    private long getPayload(int payloadPos, int bits)
+    {
+        return getPayload(buf, payloadPos, bits);
+    }
+
+    private static long getPayload(ByteBuffer contents, int payloadPos, int bytes)
+    {
+        if (bytes == 0)
+            return NOT_FOUND;
+
         return SizedInts.read(contents, payloadPos, bytes);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java
index df6b278a069f..767b04e96d44 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java
@@ -23,18 +23,16 @@
 
 import org.apache.commons.lang3.mutable.MutableLong;
 
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
 import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.io.tries.IncrementalDeepTrieWriterPageAware;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.cassandra.io.tries.IncrementalTrieWriter;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 /**
- * Writes terms dictionary to disk in a trie format (see {@link IncrementalTrieWriter}).
- * <p>
+ * Writes terms dictionary to disk in a trie format (see {@link IncrementalTrieWriter}.
+ *
  * Allows for variable-length keys. Trie values are 64-bit offsets to the posting file, pointing to the beginning of
  * summary block for that postings list.
  */
@@ -45,14 +43,15 @@ public class TrieTermsDictionaryWriter implements Closeable
     private final IndexOutputWriter termDictionaryOutput;
     private final long startOffset;
 
-    TrieTermsDictionaryWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
+    TrieTermsDictionaryWriter(IndexComponents.ForWrite components) throws IOException
     {
-        termDictionaryOutput = indexDescriptor.openPerIndexOutput(IndexComponent.TERMS_DATA, indexIdentifier, true);
+        termDictionaryOutput = components.addOrGet(IndexComponentType.TERMS_DATA).openOutput(true);
         startOffset = termDictionaryOutput.getFilePointer();
 
         SAICodecUtils.writeHeader(termDictionaryOutput);
         // we pass the output as SequentialWriter, but we keep IndexOutputWriter around to write footer on flush
-        termsDictionaryWriter = new IncrementalDeepTrieWriterPageAware<>(TrieTermsDictionaryReader.trieSerializer, termDictionaryOutput.asSequentialWriter());
+        var encodingVersion = components.byteComparableVersionFor(IndexComponentType.TERMS_DATA);
+        termsDictionaryWriter = IncrementalTrieWriter.open(TrieTermsDictionaryReader.trieSerializer, termDictionaryOutput.asSequentialWriter(), encodingVersion);
     }
 
     public void add(ByteComparable term, long postingListOffset) throws IOException
@@ -61,7 +60,7 @@ public void add(ByteComparable term, long postingListOffset) throws IOException
     }
 
     @Override
-    public void close()
+    public void close() throws IOException
     {
         termsDictionaryWriter.close();
         termDictionaryOutput.close();
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/CompactionVectorValues.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/CompactionVectorValues.java
deleted file mode 100644
index 8974752086dc..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/CompactionVectorValues.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import io.github.jbellis.jvector.util.RamUsageEstimator;
-import org.apache.cassandra.db.marshal.VectorType;
-import org.apache.cassandra.io.util.SequentialWriter;
-
-@NotThreadSafe
-public class CompactionVectorValues implements RamAwareVectorValues
-{
-    private final int dimension;
-    private final ArrayList<ByteBuffer> values = new ArrayList<>();
-    private final VectorType<Float> type;
-
-    public CompactionVectorValues(VectorType<Float> type)
-    {
-        this.dimension = type.dimension;
-        this.type = type;
-    }
-
-    @Override
-    public int size()
-    {
-        return values.size();
-    }
-
-    @Override
-    public int dimension()
-    {
-        return dimension;
-    }
-
-    @Override
-    public float[] vectorValue(int i)
-    {
-        return type.composeAsFloat(values.get(i));
-    }
-
-    /** return approximate bytes used by the new vector */
-    public long add(int ordinal, ByteBuffer value)
-    {
-        if (ordinal != values.size())
-            throw new IllegalArgumentException(String.format("CVV requires vectors to be added in ordinal order (%d given, expected %d)",
-                                                             ordinal, values.size()));
-        values.add(value);
-        return RamEstimation.concurrentHashMapRamUsed(1) + oneVectorBytesUsed();
-    }
-
-    @Override
-    public CompactionVectorValues copy()
-    {
-        return this;
-    }
-
-    public long write(SequentialWriter writer) throws IOException
-    {
-        writer.writeInt(size());
-        writer.writeInt(dimension());
-
-        for (var i = 0; i < size(); i++) {
-            var bb = values.get(i);
-            assert bb != null : "null vector at index " + i + " of " + size();
-            writer.write(bb);
-        }
-
-        return writer.position();
-    }
-
-    @Override
-    public boolean isValueShared()
-    {
-        return false;
-    }
-
-    private long oneVectorBytesUsed()
-    {
-        return RamUsageEstimator.NUM_BYTES_OBJECT_REF;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/ConcurrentVectorValues.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/ConcurrentVectorValues.java
deleted file mode 100644
index 89ae69251e0a..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/ConcurrentVectorValues.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import org.jctools.maps.NonBlockingHashMapLong;
-
-public class ConcurrentVectorValues implements RamAwareVectorValues
-{
-    private final int dimensions;
-    private final NonBlockingHashMapLong<float[]> values = new NonBlockingHashMapLong<>();
-
-    public ConcurrentVectorValues(int dimensions)
-    {
-        this.dimensions = dimensions;
-    }
-
-    @Override
-    public int size()
-    {
-        return values.size();
-    }
-
-    @Override
-    public int dimension()
-    {
-        return dimensions;
-    }
-
-    @Override
-    public float[] vectorValue(int i)
-    {
-        return values.get(i);
-    }
-
-    /** return approximate bytes used by the new vector */
-    public long add(int ordinal, float[] vector)
-    {
-        values.put(ordinal, vector);
-        return RamEstimation.concurrentHashMapRamUsed(1) + oneVectorBytesUsed();
-    }
-
-    @Override
-    public boolean isValueShared()
-    {
-        return false;
-    }
-
-    @Override
-    public ConcurrentVectorValues copy()
-    {
-        // no actual copy required because we always return distinct float[] for distinct vector ordinals
-        return this;
-    }
-
-    private long oneVectorBytesUsed()
-    {
-        return Integer.BYTES + Integer.BYTES + (long) dimension() * Float.BYTES;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskAnn.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskAnn.java
deleted file mode 100644
index 93f50df54f57..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskAnn.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.PrimitiveIterator;
-import java.util.stream.IntStream;
-
-import io.github.jbellis.jvector.disk.CachingGraphIndex;
-import io.github.jbellis.jvector.disk.OnDiskGraphIndex;
-import io.github.jbellis.jvector.graph.GraphSearcher;
-import io.github.jbellis.jvector.graph.NeighborSimilarity;
-import io.github.jbellis.jvector.graph.SearchResult;
-import io.github.jbellis.jvector.graph.SearchResult.NodeScore;
-import io.github.jbellis.jvector.pq.CompressedVectors;
-import io.github.jbellis.jvector.util.Bits;
-import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
-import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
-import org.apache.cassandra.index.sai.disk.v1.postings.VectorPostingList;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.tracing.Tracing;
-
-public class DiskAnn implements AutoCloseable
-{
-    private final FileHandle graphHandle;
-    private final OnDiskOrdinalsMap ordinalsMap;
-    private final CachingGraphIndex graph;
-    private final VectorSimilarityFunction similarityFunction;
-
-    // only one of these will be not null
-    private final CompressedVectors compressedVectors;
-
-    public DiskAnn(SegmentMetadata.ComponentMetadataMap componentMetadatas, PerColumnIndexFiles indexFiles, IndexWriterConfig config) throws IOException
-    {
-        similarityFunction = config.getSimilarityFunction();
-
-        SegmentMetadata.ComponentMetadata termsMetadata = componentMetadatas.get(IndexComponent.TERMS_DATA);
-        graphHandle = indexFiles.termsData();
-        graph = new CachingGraphIndex(new OnDiskGraphIndex<>(RandomAccessReaderAdapter.createSupplier(graphHandle), termsMetadata.offset));
-
-        long pqSegmentOffset = componentMetadatas.get(IndexComponent.COMPRESSED_VECTORS).offset;
-        try (var pqFileHandle = indexFiles.compressedVectors(); var reader = new RandomAccessReaderAdapter(pqFileHandle))
-        {
-            reader.seek(pqSegmentOffset);
-            var containsCompressedVectors = reader.readBoolean();
-            if (containsCompressedVectors)
-                compressedVectors = CompressedVectors.load(reader, reader.getFilePointer());
-            else
-                compressedVectors = null;
-        }
-
-        SegmentMetadata.ComponentMetadata postingListsMetadata = componentMetadatas.get(IndexComponent.POSTING_LISTS);
-        ordinalsMap = new OnDiskOrdinalsMap(indexFiles.postingLists(), postingListsMetadata.offset, postingListsMetadata.length);
-    }
-
-    public long ramBytesUsed()
-    {
-        return graph.ramBytesUsed();
-    }
-
-    public int size()
-    {
-        return graph.size();
-    }
-
-    /**
-     * @return Row IDs associated with the topK vectors near the query
-     */
-    public VectorPostingList search(float[] queryVector, int topK, int limit, Bits acceptBits)
-    {
-        OnHeapGraph.validateIndexable(queryVector, similarityFunction);
-
-        var view = graph.getView();
-        var searcher = new GraphSearcher.Builder<>(view).build();
-        NeighborSimilarity.ScoreFunction scoreFunction;
-        NeighborSimilarity.ReRanker<float[]> reRanker;
-        if (compressedVectors == null)
-        {
-            scoreFunction = (NeighborSimilarity.ExactScoreFunction)
-                            i -> similarityFunction.compare(queryVector, view.getVector(i));
-            reRanker = null;
-        }
-        else
-        {
-            scoreFunction = compressedVectors.approximateScoreFunctionFor(queryVector, similarityFunction);
-            reRanker = (i, map) -> similarityFunction.compare(queryVector, map.get(i));
-        }
-        var result = searcher.search(scoreFunction,
-                                     reRanker,
-                                     topK,
-                                     ordinalsMap.ignoringDeleted(acceptBits));
-        Tracing.trace("DiskANN search visited {} nodes to return {} results", result.getVisitedCount(), result.getNodes().length);
-        return annRowIdsToPostings(result, limit);
-    }
-
-    private class RowIdIterator implements PrimitiveIterator.OfInt, AutoCloseable
-    {
-        private final Iterator<NodeScore> it;
-        private final OnDiskOrdinalsMap.RowIdsView rowIdsView = ordinalsMap.getRowIdsView();
-
-        private OfInt segmentRowIdIterator = IntStream.empty().iterator();
-
-        public RowIdIterator(NodeScore[] results)
-        {
-            this.it = Arrays.stream(results).iterator();
-        }
-
-        @Override
-        public boolean hasNext()
-        {
-            while (!segmentRowIdIterator.hasNext() && it.hasNext())
-            {
-                try
-                {
-                    var ordinal = it.next().node;
-                    segmentRowIdIterator = Arrays.stream(rowIdsView.getSegmentRowIdsMatching(ordinal)).iterator();
-                }
-                catch (IOException e)
-                {
-                    throw new RuntimeException(e);
-                }
-            }
-            return segmentRowIdIterator.hasNext();
-        }
-
-        @Override
-        public int nextInt() {
-            if (!hasNext())
-                throw new NoSuchElementException();
-            return segmentRowIdIterator.nextInt();
-        }
-
-        @Override
-        public void close()
-        {
-            rowIdsView.close();
-        }
-    }
-
-    private VectorPostingList annRowIdsToPostings(SearchResult results, int limit)
-    {
-        try (var iterator = new RowIdIterator(results.getNodes()))
-        {
-            return new VectorPostingList(iterator, limit, results.getVisitedCount());
-        }
-    }
-
-    @Override
-    public void close() throws IOException
-    {
-        ordinalsMap.close();
-        graph.close();
-        graphHandle.close();
-    }
-
-    public OnDiskOrdinalsMap.OrdinalsView getOrdinalsView()
-    {
-        return ordinalsMap.getOrdinalsView();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskBinarySearch.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskBinarySearch.java
deleted file mode 100644
index 017ddfd7698c..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskBinarySearch.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-
-import java.util.function.Function;
-
-public class DiskBinarySearch
-{
-    /**
-     * Search for the target int between positions low and high, using the provided function
-     * to retrieve the int value at the given ordinal.
-     *
-     * Returns the position at which target is found.  Raises an exception if it is not found.
-     *
-     * This will not call f() after the target is found, so if f is performing disk seeks,
-     * it will leave the underlying reader at the position right after reading the target.
-     *
-     * @return index if target is found; otherwise return -1 if targer is not found
-     */
-    public static long searchInt(long low, long high, int target, Function<Long, Integer> f)
-    {
-        assert high < Long.MAX_VALUE >> 2 : "high is too large to avoid potential overflow: " + high;
-        assert low < high : "low must be less than high: " + low + " >= " + high;
-
-        while (low < high)
-        {
-            long i = low + (high - low) / 2;
-            int value = f.apply(i);
-            if (target == value)
-                return i;
-            else if (target > value)
-                low = i + 1;
-            else
-                high = i;
-        }
-        return -1;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnDiskOrdinalsMap.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnDiskOrdinalsMap.java
deleted file mode 100644
index 121ab01ed024..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnDiskOrdinalsMap.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import com.google.common.base.Preconditions;
-
-import io.github.jbellis.jvector.util.Bits;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.RandomAccessReader;
-
-public class OnDiskOrdinalsMap implements AutoCloseable
-{
-    private final FileHandle fh;
-    private final long ordToRowOffset;
-    private final long segmentEnd;
-    private final int size;
-    // the offset where we switch from recording ordinal -> rows, to row -> ordinal
-    private final long rowOrdinalOffset;
-    private final Set<Integer> deletedOrdinals;
-
-    public OnDiskOrdinalsMap(FileHandle fh, long segmentOffset, long segmentLength)
-    {
-        deletedOrdinals = new HashSet<>();
-
-        this.segmentEnd = segmentOffset + segmentLength;
-        this.fh = fh;
-        try (var reader = fh.createReader())
-        {
-            reader.seek(segmentOffset);
-            int deletedCount = reader.readInt();
-            for (var i = 0; i < deletedCount; i++)
-            {
-                deletedOrdinals.add(reader.readInt());
-            }
-
-            this.ordToRowOffset = reader.getFilePointer();
-            this.size = reader.readInt();
-            reader.seek(segmentEnd - 8);
-            this.rowOrdinalOffset = reader.readLong();
-            assert rowOrdinalOffset < segmentEnd : "rowOrdinalOffset " + rowOrdinalOffset + " is not less than segmentEnd " + segmentEnd;
-        }
-        catch (Exception e)
-        {
-            throw new RuntimeException("Error initializing OnDiskOrdinalsMap at segment " + segmentOffset, e);
-        }
-    }
-
-    public RowIdsView getRowIdsView()
-    {
-        return new RowIdsView();
-    }
-
-    public Bits ignoringDeleted(Bits acceptBits)
-    {
-        return BitsUtil.bitsIgnoringDeleted(acceptBits, deletedOrdinals);
-    }
-
-    public class RowIdsView implements AutoCloseable
-    {
-        final RandomAccessReader reader = fh.createReader();
-
-        public int[] getSegmentRowIdsMatching(int vectorOrdinal) throws IOException
-        {
-            Preconditions.checkArgument(vectorOrdinal < size, "vectorOrdinal %s is out of bounds %s", vectorOrdinal, size);
-
-            // read index entry
-            try
-            {
-                reader.seek(ordToRowOffset + 4L + vectorOrdinal * 8L);
-            }
-            catch (Exception e)
-            {
-                throw new RuntimeException(String.format("Error seeking to index offset for ordinal %d with ordToRowOffset %d",
-                                                         vectorOrdinal, ordToRowOffset), e);
-            }
-            var offset = reader.readLong();
-            // seek to and read rowIds
-            try
-            {
-                reader.seek(offset);
-            }
-            catch (Exception e)
-            {
-                throw new RuntimeException(String.format("Error seeking to rowIds offset for ordinal %d with ordToRowOffset %d",
-                                                         vectorOrdinal, ordToRowOffset), e);
-            }
-            var postingsSize = reader.readInt();
-            var rowIds = new int[postingsSize];
-            for (var i = 0; i < rowIds.length; i++)
-            {
-                rowIds[i] = reader.readInt();
-            }
-            return rowIds;
-        }
-
-        @Override
-        public void close()
-        {
-            reader.close();
-        }
-    }
-
-    public OrdinalsView getOrdinalsView()
-    {
-        return new OrdinalsView();
-    }
-
-    public class OrdinalsView implements AutoCloseable
-    {
-        final RandomAccessReader reader = fh.createReader();
-        private final long high = (segmentEnd - 8 - rowOrdinalOffset) / 8;
-
-        /**
-         * @return order if given row id is found; otherwise return -1
-         */
-        public int getOrdinalForRowId(int rowId) throws IOException
-        {
-            // Compute the offset of the start of the rowId to vectorOrdinal mapping
-            long index = DiskBinarySearch.searchInt(0, Math.toIntExact(high), rowId, i -> {
-                try
-                {
-                    long offset = rowOrdinalOffset + i * 8;
-                    reader.seek(offset);
-                    return reader.readInt();
-                }
-                catch (IOException e)
-                {
-                    throw new RuntimeException(e);
-                }
-            });
-
-            // not found
-            if (index < 0)
-                return -1;
-
-            return reader.readInt();
-        }
-
-        @Override
-        public void close()
-        {
-            reader.close();
-        }
-    }
-
-    @Override
-    public void close()
-    {
-        fh.close();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnHeapGraph.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnHeapGraph.java
deleted file mode 100644
index 903058108369..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnHeapGraph.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
-import java.util.stream.IntStream;
-
-import org.cliffc.high_scale_lib.NonBlockingHashMapLong;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.github.jbellis.jvector.disk.OnDiskGraphIndex;
-import io.github.jbellis.jvector.graph.GraphIndex;
-import io.github.jbellis.jvector.graph.GraphIndexBuilder;
-import io.github.jbellis.jvector.graph.GraphSearcher;
-import io.github.jbellis.jvector.graph.NeighborSimilarity;
-import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
-import io.github.jbellis.jvector.pq.CompressedVectors;
-import io.github.jbellis.jvector.pq.ProductQuantization;
-import io.github.jbellis.jvector.util.Bits;
-import io.github.jbellis.jvector.vector.VectorEncoding;
-import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.VectorType;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
-import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.io.util.SequentialWriter;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.lucene.util.StringHelper;
-
-public class OnHeapGraph<T>
-{
-    private static final Logger logger = LoggerFactory.getLogger(OnHeapGraph.class);
-
-    private final RamAwareVectorValues vectorValues;
-    private final GraphIndexBuilder<float[]> builder;
-    private final VectorType<?> vectorType;
-    private final VectorSimilarityFunction similarityFunction;
-    private final ConcurrentMap<float[], VectorPostings<T>> postingsMap;
-    private final NonBlockingHashMapLong<VectorPostings<T>> postingsByOrdinal;
-    private final AtomicInteger nextOrdinal = new AtomicInteger();
-    private volatile boolean hasDeletions;
-
-    /**
-     * @param termComparator the vector type
-     * @param indexWriterConfig
-     *
-     * Will create a concurrent object.
-     */
-    public OnHeapGraph(AbstractType<?> termComparator, IndexWriterConfig indexWriterConfig)
-    {
-        this(termComparator, indexWriterConfig, true);
-    }
-
-    /**
-     * @param termComparator the vector type
-     * @param indexWriterConfig the {@link IndexWriterConfig} for the graph
-     * @param concurrent should be true for memtables, false for compaction.  Concurrent allows us to search
-     *                   while building the graph; non-concurrent allows us to avoid synchronization costs.
-     */
-    @SuppressWarnings("unchecked")
-    public OnHeapGraph(AbstractType<?> termComparator, IndexWriterConfig indexWriterConfig, boolean concurrent)
-    {
-        this.vectorType = (VectorType<?>) termComparator;
-        vectorValues = concurrent
-                       ? new ConcurrentVectorValues(((VectorType<?>) termComparator).dimension)
-                       : new CompactionVectorValues(((VectorType<Float>) termComparator));
-        similarityFunction = indexWriterConfig.getSimilarityFunction();
-        // We need to be able to inexpensively distinguish different vectors, with a slower path
-        // that identifies vectors that are equal but not the same reference.  A comparison
-        // based Map (which only needs to look at vector elements until a difference is found)
-        // is thus a better option than hash-based (which has to look at all elements to compute the hash).
-        postingsMap = new ConcurrentSkipListMap<>(Arrays::compare);
-        postingsByOrdinal = new NonBlockingHashMapLong<>();
-
-        builder = new GraphIndexBuilder<>(vectorValues,
-                                          VectorEncoding.FLOAT32,
-                                          similarityFunction,
-                                          indexWriterConfig.getMaximumNodeConnections(),
-                                          indexWriterConfig.getConstructionBeamWidth(),
-                                          1.2f,
-                                          1.4f);
-    }
-
-    public int size()
-    {
-        return vectorValues.size();
-    }
-
-    public boolean isEmpty()
-    {
-        return postingsMap.values().stream().allMatch(VectorPostings::isEmpty);
-    }
-
-    /**
-     * @return the incremental bytes ysed by adding the given vector to the index
-     */
-    public long add(ByteBuffer term, T key, InvalidVectorBehavior behavior)
-    {
-        assert term != null && term.remaining() != 0;
-
-        var vector = vectorType.composeAsFloat(term);
-        if (behavior == InvalidVectorBehavior.IGNORE)
-        {
-            try
-            {
-                validateIndexable(vector, similarityFunction);
-            }
-            catch (InvalidRequestException e)
-            {
-                logger.trace("Ignoring invalid vector during index build against existing data: {}", vector, e);
-                return 0;
-            }
-        }
-        else
-        {
-            assert behavior == InvalidVectorBehavior.FAIL;
-            validateIndexable(vector, similarityFunction);
-        }
-
-        var bytesUsed = 0L;
-        VectorPostings<T> postings = postingsMap.get(vector);
-        // if the vector is already in the graph, all that happens is that the postings list is updated
-        // otherwise, we add the vector in this order:
-        // 1. to the postingsMap
-        // 2. to the vectorValues
-        // 3. to the graph
-        // This way, concurrent searches of the graph won't see the vector until it's visible
-        // in the other structures as well.
-        if (postings == null)
-        {
-            postings = new VectorPostings<>(key);
-            // since we are using ConcurrentSkipListMap, it is NOT correct to use computeIfAbsent here
-            if (postingsMap.putIfAbsent(vector, postings) == null)
-            {
-                // we won the race to add the new entry; assign it an ordinal and add to the other structures
-                var ordinal = nextOrdinal.getAndIncrement();
-                postings.setOrdinal(ordinal);
-                bytesUsed += RamEstimation.concurrentHashMapRamUsed(1); // the new posting Map entry
-                bytesUsed += (vectorValues instanceof ConcurrentVectorValues)
-                             ? ((ConcurrentVectorValues) vectorValues).add(ordinal, vector)
-                             : ((CompactionVectorValues) vectorValues).add(ordinal, term);
-                bytesUsed += VectorPostings.emptyBytesUsed() + VectorPostings.bytesPerPosting();
-                postingsByOrdinal.put(ordinal, postings);
-                bytesUsed += builder.addGraphNode(ordinal, vectorValues);
-                return bytesUsed;
-            }
-            else
-            {
-                postings = postingsMap.get(vector);
-            }
-        }
-        // postings list already exists, just add the new key (if it's not already in the list)
-        if (postings.add(key))
-        {
-            bytesUsed += VectorPostings.bytesPerPosting();
-        }
-
-        return bytesUsed;
-    }
-
-    // copied out of a Lucene PR -- hopefully committed soon
-    public static final float MAX_FLOAT32_COMPONENT = 1E17f;
-
-    public static void checkInBounds(float[] v)
-    {
-        for (int i = 0; i < v.length; i++)
-        {
-            if (!Float.isFinite(v[i]))
-            {
-                throw new IllegalArgumentException("non-finite value at vector[" + i + "]=" + v[i]);
-            }
-
-            if (Math.abs(v[i]) > MAX_FLOAT32_COMPONENT)
-            {
-                throw new IllegalArgumentException("Out-of-bounds value at vector[" + i + "]=" + v[i]);
-            }
-        }
-    }
-
-    public static void validateIndexable(float[] vector, VectorSimilarityFunction similarityFunction)
-    {
-        try
-        {
-            checkInBounds(vector);
-        }
-        catch (IllegalArgumentException e)
-        {
-            throw new InvalidRequestException(e.getMessage());
-        }
-
-        if (similarityFunction == VectorSimilarityFunction.COSINE)
-        {
-            for (int i = 0; i < vector.length; i++)
-            {
-                if (vector[i] != 0)
-                    return;
-            }
-            throw new InvalidRequestException("Zero vectors cannot be indexed or queried with cosine similarity");
-        }
-    }
-
-    public Collection<T> keysFromOrdinal(int node)
-    {
-        return postingsByOrdinal.get(node).getPostings();
-    }
-
-    public long remove(ByteBuffer term, T key)
-    {
-        assert term != null && term.remaining() != 0;
-
-        var vector = vectorType.composeAsFloat(term);
-        var postings = postingsMap.get(vector);
-        if (postings == null)
-        {
-            // it's possible for this to be called against a different memtable than the one
-            // the value was originally added to, in which case we do not expect to find
-            // the key among the postings for this vector
-            return 0;
-        }
-
-        hasDeletions = true;
-        return postings.remove(key);
-    }
-
-    /**
-     * @return keys (PrimaryKey or segment row id) associated with the topK vectors near the query
-     */
-    public PriorityQueue<T> search(float[] queryVector, int limit, Bits toAccept)
-    {
-        validateIndexable(queryVector, similarityFunction);
-
-        // search() errors out when an empty graph is passed to it
-        if (vectorValues.size() == 0)
-            return new PriorityQueue<>();
-
-        Bits bits = hasDeletions ? BitsUtil.bitsIgnoringDeleted(toAccept, postingsByOrdinal) : toAccept;
-        GraphIndex<float[]> graph = builder.getGraph();
-        var searcher = new GraphSearcher.Builder<>(graph.getView()).withConcurrentUpdates().build();
-        NeighborSimilarity.ExactScoreFunction scoreFunction = node2 -> vectorCompareFunction(queryVector, node2);
-        var result = searcher.search(scoreFunction, null, limit, bits);
-        Tracing.trace("ANN search visited {} in-memory nodes to return {} results", result.getVisitedCount(), result.getNodes().length);
-        var a = result.getNodes();
-        PriorityQueue<T> keyQueue = new PriorityQueue<>();
-        for (int i = 0; i < a.length; i++)
-            keyQueue.addAll(keysFromOrdinal(a[i].node));
-        return keyQueue;
-    }
-
-    public SegmentMetadata.ComponentMetadataMap writeData(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, Function<T, Integer> postingTransformer) throws IOException
-    {
-        int nInProgress = builder.insertsInProgress();
-        assert nInProgress == 0 : String.format("Attempting to write graph while %d inserts are in progress", nInProgress);
-        assert nextOrdinal.get() == builder.getGraph().size() : String.format("nextOrdinal %d != graph size %d -- ordinals should be sequential",
-                                                                              nextOrdinal.get(), builder.getGraph().size());
-        assert vectorValues.size() == builder.getGraph().size() : String.format("vector count %d != graph size %d",
-                                                                                vectorValues.size(), builder.getGraph().size());
-        assert postingsMap.keySet().size() == vectorValues.size() : String.format("postings map entry count %d != vector count %d",
-                                                                                  postingsMap.keySet().size(), vectorValues.size());
-        logger.debug("Writing graph with {} rows and {} distinct vectors", postingsMap.values().stream().mapToInt(VectorPostings::size).sum(), vectorValues.size());
-
-        try (var pqOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.COMPRESSED_VECTORS, indexIdentifier), true);
-             var postingsOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.POSTING_LISTS, indexIdentifier), true);
-             var indexOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.TERMS_DATA, indexIdentifier), true))
-        {
-            SAICodecUtils.writeHeader(pqOutput);
-            SAICodecUtils.writeHeader(postingsOutput);
-            SAICodecUtils.writeHeader(indexOutput);
-
-            // compute and write PQ
-            long pqOffset = pqOutput.getFilePointer();
-            long pqPosition = writePQ(pqOutput.asSequentialWriter());
-            long pqLength = pqPosition - pqOffset;
-
-            var deletedOrdinals = new HashSet<Integer>();
-            postingsMap.values().stream().filter(VectorPostings::isEmpty).forEach(vectorPostings -> deletedOrdinals.add(vectorPostings.getOrdinal()));
-            // remove ordinals that don't have corresponding row ids due to partition/range deletion
-            for (VectorPostings<T> vectorPostings : postingsMap.values())
-            {
-                vectorPostings.computeRowIds(postingTransformer);
-                if (vectorPostings.shouldAppendDeletedOrdinal())
-                    deletedOrdinals.add(vectorPostings.getOrdinal());
-            }
-            // write postings
-            long postingsOffset = postingsOutput.getFilePointer();
-            long postingsPosition = new VectorPostingsWriter<T>().writePostings(postingsOutput.asSequentialWriter(), vectorValues, postingsMap, deletedOrdinals);
-            long postingsLength = postingsPosition - postingsOffset;
-
-            // complete (internal clean up) and write the graph
-            builder.complete();
-            long termsOffset = indexOutput.getFilePointer();
-            OnDiskGraphIndex.write(builder.getGraph(), vectorValues, indexOutput.asSequentialWriter());
-            long termsLength = indexOutput.getFilePointer() - termsOffset;
-
-            // write footers/checksums
-            SAICodecUtils.writeFooter(pqOutput);
-            SAICodecUtils.writeFooter(postingsOutput);
-            SAICodecUtils.writeFooter(indexOutput);
-
-            // add components to the metadata map
-            SegmentMetadata.ComponentMetadataMap metadataMap = new SegmentMetadata.ComponentMetadataMap();
-            metadataMap.put(IndexComponent.TERMS_DATA, -1, termsOffset, termsLength, Map.of());
-            metadataMap.put(IndexComponent.POSTING_LISTS, -1, postingsOffset, postingsLength, Map.of());
-            Map<String, String> vectorConfigs = Map.of("SEGMENT_ID", ByteBufferUtil.bytesToHex(ByteBuffer.wrap(StringHelper.randomId())));
-            metadataMap.put(IndexComponent.COMPRESSED_VECTORS, -1, pqOffset, pqLength, vectorConfigs);
-            return metadataMap;
-        }
-    }
-
-    private float vectorCompareFunction(float[] queryVector, int node)
-    {
-        return similarityFunction.compare(queryVector, ((RandomAccessVectorValues<float[]>) vectorValues).vectorValue(node));
-    }
-
-    private long writePQ(SequentialWriter writer) throws IOException
-    {
-        // don't bother with PQ if there are fewer than 1K vectors
-        int M = vectorValues.dimension() / 2;
-        writer.writeBoolean(vectorValues.size() >= 1024);
-        if (vectorValues.size() < 1024)
-        {
-            logger.debug("Skipping PQ for only {} vectors", vectorValues.size());
-            return writer.position();
-        }
-
-        logger.debug("Computing PQ for {} vectors", vectorValues.size());
-        // limit the PQ computation and encoding to one index at a time -- goal during flush is to
-        // evict from memory ASAP so better to do the PQ build (in parallel) one at a time
-        ProductQuantization pq;
-        byte[][] encoded;
-        synchronized (OnHeapGraph.class)
-        {
-            // train PQ and encode
-            pq = ProductQuantization.compute(vectorValues, M, false);
-            assert !vectorValues.isValueShared();
-            encoded = IntStream.range(0, vectorValues.size())
-                               .parallel()
-                               .mapToObj(i -> pq.encode(vectorValues.vectorValue(i)))
-                               .toArray(byte[][]::new);
-        }
-        var cv = new CompressedVectors(pq, encoded);
-        // save
-        cv.write(writer);
-        return writer.position();
-    }
-
-    public enum InvalidVectorBehavior
-    {
-        IGNORE,
-        FAIL
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OptimizeFor.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OptimizeFor.java
deleted file mode 100644
index 51e58b37df78..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OptimizeFor.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import java.util.function.Function;
-
-import static java.lang.Math.pow;
-
-/**
- * Allows the vector index searches to be optimised for latency or recall. This is used by the
- * {@link org.apache.cassandra.index.sai.disk.v1.segment.VectorIndexSegmentSearcher} to determine how many results to ask the graph
- * to search for. If we are optimising for {@link #RECALL} we ask for more than the requested limit which
- * (since it will search deeper in the graph) will tend to surface slightly better results.
- */
-public enum OptimizeFor
-{
-    LATENCY(limit -> 0.979 + 4.021 * pow(limit, -0.761)), // f(1) =  5.0, f(100) = 1.1, f(1000) = 1.0
-    RECALL(limit -> 0.509 + 9.491 * pow(limit, -0.402));  // f(1) = 10.0, f(100) = 2.0, f(1000) = 1.1
-
-    private final Function<Integer, Double> limitMultiplier;
-
-    OptimizeFor(Function<Integer, Double> limitMultiplier)
-    {
-        this.limitMultiplier = limitMultiplier;
-    }
-
-    public int topKFor(int limit)
-    {
-        return (int)(limitMultiplier.apply(limit) * limit);
-    }
-
-    public static OptimizeFor fromString(String value)
-    {
-        return valueOf(value.toUpperCase());
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/RamEstimation.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/RamEstimation.java
deleted file mode 100644
index 4288a84e014a..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/RamEstimation.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import org.apache.lucene.util.RamUsageEstimator;
-
-public class RamEstimation
-{
-    /**
-     * @param externalNodeCount the size() of the ConcurrentHashMap
-     * @return an estimate of the number of bytes used
-     */
-    public static long concurrentHashMapRamUsed(int externalNodeCount) {
-        long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
-        long AH_BYTES = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
-        long CORES = Runtime.getRuntime().availableProcessors();
-
-        long chmNodeBytes =
-        REF_BYTES // node itself in Node[]
-        + 3L * REF_BYTES
-        + Integer.BYTES; // node internals
-        float chmLoadFactor = 0.75f; // this is hardcoded inside ConcurrentHashMap
-        // CHM has a striped counter Cell implementation, we expect at most one per core
-        long chmCounters = AH_BYTES + CORES * (REF_BYTES + Long.BYTES);
-
-        double nodeCount = externalNodeCount / chmLoadFactor;
-
-        return
-        (long) nodeCount * (chmNodeBytes + REF_BYTES)// nodes
-        + AH_BYTES // nodes array
-        + Long.BYTES
-        + 3 * Integer.BYTES
-        + 3 * REF_BYTES // extra internal fields
-        + chmCounters
-        + REF_BYTES; // the Map reference itself
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/RandomAccessReaderAdapter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/RandomAccessReaderAdapter.java
deleted file mode 100644
index b63d1a21fc6f..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/RandomAccessReaderAdapter.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.FloatBuffer;
-import java.nio.IntBuffer;
-
-import com.google.common.primitives.Ints;
-
-import io.github.jbellis.jvector.disk.ReaderSupplier;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.RandomAccessReader;
-
-public class RandomAccessReaderAdapter extends RandomAccessReader implements io.github.jbellis.jvector.disk.RandomAccessReader
-{
-    static ReaderSupplier createSupplier(FileHandle fileHandle)
-    {
-        return () -> new RandomAccessReaderAdapter(fileHandle);
-    }
-
-    RandomAccessReaderAdapter(FileHandle fileHandle)
-    {
-        super(fileHandle.instantiateRebufferer(null));
-    }
-
-    @Override
-    public void readFully(float[] dest) throws IOException
-    {
-        var bh = bufferHolder;
-        long position = getPosition();
-
-        FloatBuffer floatBuffer;
-        if (bh.offset() == 0 && position % Float.BYTES == 0)
-        {
-            // this is a separate code path because buffer() and asFloatBuffer() both allocate
-            // new and relatively expensive xBuffer objects, so we want to avoid doing that
-            // twice, where possible
-            floatBuffer = bh.floatBuffer();
-            floatBuffer.position(Ints.checkedCast(position / Float.BYTES));
-        }
-        else
-        {
-            // offset is non-zero, and probably not aligned to Float.BYTES, so
-            // set the position before converting to FloatBuffer.
-            var bb = bh.buffer();
-            bb.position(Ints.checkedCast(position - bh.offset()));
-            floatBuffer = bb.asFloatBuffer();
-        }
-
-        if (dest.length > floatBuffer.remaining())
-        {
-            // slow path -- desired slice is across region boundaries
-            var bb = ByteBuffer.allocate(Float.BYTES * dest.length);
-            readFully(bb);
-            floatBuffer = bb.asFloatBuffer();
-        }
-
-        floatBuffer.get(dest);
-        seek(position + (long) Float.BYTES * dest.length);
-    }
-
-    /**
-     * Read ints into an int[], starting at the current position.
-     *
-     * @param dest the array to read into
-     * @param offset the offset in the array at which to start writing ints
-     * @param count the number of ints to read
-     *
-     * Will change the buffer position.
-     */
-    @Override
-    public void read(int[] dest, int offset, int count) throws IOException
-    {
-        if (count == 0)
-            return;
-
-        var bh = bufferHolder;
-        long position = getPosition();
-
-        IntBuffer intBuffer;
-        if (bh.offset() == 0 && position % Integer.BYTES == 0)
-        {
-            // this is a separate code path because buffer() and asIntBuffer() both allocate
-            // new and relatively expensive xBuffer objects, so we want to avoid doing that
-            // twice, where possible
-            intBuffer = bh.intBuffer();
-            intBuffer.position(Ints.checkedCast(position / Integer.BYTES));
-        }
-        else
-        {
-            // offset is non-zero, and probably not aligned to Integer.BYTES, so
-            // set the position before converting to IntBuffer.
-            var bb = bh.buffer();
-            bb.position(Ints.checkedCast(position - bh.offset()));
-            intBuffer = bb.asIntBuffer();
-        }
-
-        if (count > intBuffer.remaining())
-        {
-            // slow path -- desired slice is across region boundaries
-            var bb = ByteBuffer.allocate(Integer.BYTES * count);
-            readFully(bb);
-            intBuffer = bb.asIntBuffer();
-        }
-
-        intBuffer.get(dest, offset, count);
-        seek(position + (long) Integer.BYTES * count);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/VectorPostings.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/VectorPostings.java
deleted file mode 100644
index 468c85037ebc..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/VectorPostings.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.function.Function;
-
-import com.google.common.base.Preconditions;
-
-import org.agrona.collections.IntArrayList;
-import org.apache.lucene.util.RamUsageEstimator;
-
-public class VectorPostings<T>
-{
-    private final List<T> postings;
-    private volatile int ordinal = -1;
-
-    private volatile IntArrayList rowIds;
-
-    public VectorPostings(T firstKey)
-    {
-        // we expect that the overwhelmingly most common cardinality will be 1, so optimize for reads
-        postings = new CopyOnWriteArrayList<>(List.of(firstKey));
-    }
-
-    /**
-     * Split out from constructor only to make dealing with concurrent inserts easier for CassandraOnHeapGraph.
-     * Should be called at most once per instance.
-     */
-    void setOrdinal(int ordinal)
-    {
-        assert this.ordinal == -1 : String.format("ordinal already set to %d; attempted to set to %d", this.ordinal, ordinal);
-        this.ordinal = ordinal;
-    }
-
-    public boolean add(T key)
-    {
-        for (T existing : postings)
-            if (existing.equals(key))
-                return false;
-        postings.add(key);
-        return true;
-    }
-
-    /**
-     * @return true if current ordinal is removed by partition/range deletion.
-     * Must be called after computeRowIds.
-     */
-    public boolean shouldAppendDeletedOrdinal()
-    {
-        return !postings.isEmpty() && (rowIds != null && rowIds.isEmpty());
-    }
-
-    /**
-     * Compute the rowIds corresponding to the {@code <T>} keys in this postings list.
-     */
-    public void computeRowIds(Function<T, Integer> postingTransformer)
-    {
-        Preconditions.checkState(rowIds == null);
-
-        IntArrayList ids = new IntArrayList(postings.size(), -1);
-        for (T key : postings)
-        {
-            int rowId = postingTransformer.apply(key);
-            // partition deletion and range deletion won't trigger index update. There is no row id for given key during flush
-            if (rowId >= 0)
-                ids.add(rowId);
-        }
-
-        rowIds = ids;
-    }
-
-    /**
-     * @return rowIds corresponding to the {@code <T>} keys in this postings list.
-     * Must be called after computeRowIds.
-     */
-    public IntArrayList getRowIds()
-    {
-        Preconditions.checkNotNull(rowIds);
-        return rowIds;
-    }
-
-    public long remove(T key)
-    {
-        long bytesUsed = ramBytesUsed();
-        postings.remove(key);
-        return bytesUsed - ramBytesUsed();
-    }
-
-    public long ramBytesUsed()
-    {
-        return emptyBytesUsed() + postings.size() * bytesPerPosting();
-    }
-
-    public static long emptyBytesUsed()
-    {
-        long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
-        long AH_BYTES = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
-        return Integer.BYTES + REF_BYTES + AH_BYTES;
-    }
-
-    // we can't do this exactly without reflection, because keys could be Long or PrimaryKey.
-    // PK is larger, so we'll take that and return an upper bound.
-    // we already count the float[] vector in vectorValues, so leave it out here
-    public static long bytesPerPosting()
-    {
-        long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
-        return REF_BYTES
-               + 2 * Long.BYTES // hashes in PreHashedDecoratedKey
-               + REF_BYTES; // key ByteBuffer, this is used elsewhere, so we don't take the deep size
-    }
-
-    public int size()
-    {
-        return postings.size();
-    }
-
-    public List<T> getPostings()
-    {
-        return postings;
-    }
-
-    public boolean isEmpty()
-    {
-        return postings.isEmpty();
-    }
-
-    public int getOrdinal()
-    {
-        assert ordinal >= 0 : "ordinal not set";
-        return ordinal;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/VectorPostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/VectorPostingsWriter.java
deleted file mode 100644
index fd92bf656dd7..000000000000
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/VectorPostingsWriter.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.vector;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.cassandra.io.util.SequentialWriter;
-import org.apache.cassandra.utils.Pair;
-
-public class VectorPostingsWriter<T>
-{
-    public long writePostings(SequentialWriter writer,
-                              RamAwareVectorValues vectorValues,
-                              Map<float[], VectorPostings<T>> postingsMap,
-                              Set<Integer> deletedOrdinals) throws IOException
-    {
-        writeDeletedOrdinals(writer, deletedOrdinals);
-        writeNodeOrdinalToRowIdMapping(writer, vectorValues, postingsMap);
-        writeRowIdToNodeOrdinalMapping(writer, vectorValues, postingsMap);
-
-        return writer.position();
-    }
-
-    private void writeDeletedOrdinals(SequentialWriter writer, Set<Integer> deletedOrdinals) throws IOException
-    {
-        writer.writeInt(deletedOrdinals.size());
-        for (var ordinal : deletedOrdinals) {
-            writer.writeInt(ordinal);
-        }
-    }
-
-    public void writeNodeOrdinalToRowIdMapping(SequentialWriter writer,
-                                               RamAwareVectorValues vectorValues,
-                                               Map<float[], VectorPostings<T>> postingsMap) throws IOException
-    {
-        long ordToRowOffset = writer.getOnDiskFilePointer();
-
-        // total number of vectors
-        writer.writeInt(vectorValues.size());
-
-        // Write the offsets of the postings for each ordinal
-        var offsetsStartAt = ordToRowOffset + 4L + 8L * vectorValues.size();
-        var nextOffset = offsetsStartAt;
-        for (var i = 0; i < vectorValues.size(); i++) {
-            // (ordinal is implied; don't need to write it)
-            writer.writeLong(nextOffset);
-            var rowIds = postingsMap.get(vectorValues.vectorValue(i)).getRowIds();
-            nextOffset += 4 + (rowIds.size() * 4L); // 4 bytes for size and 4 bytes for each integer in the list
-        }
-        assert writer.position() == offsetsStartAt : "writer.position()=" + writer.position() + " offsetsStartAt=" + offsetsStartAt;
-
-        // Write postings lists
-        for (var i = 0; i < vectorValues.size(); i++) {
-            VectorPostings<T> postings = postingsMap.get(vectorValues.vectorValue(i));
-
-            var rowIds = postings.getRowIds();
-            writer.writeInt(rowIds.size());
-            for (int r = 0; r < rowIds.size(); r++)
-                writer.writeInt(rowIds.getInt(r));
-        }
-        assert writer.position() == nextOffset;
-    }
-
-    public void writeRowIdToNodeOrdinalMapping(SequentialWriter writer,
-                                               RamAwareVectorValues vectorValues,
-                                               Map<float[], VectorPostings<T>> postingsMap) throws IOException
-    {
-        List<Pair<Integer, Integer>> pairs = new ArrayList<>();
-
-        // Collect all (rowId, vectorOrdinal) pairs
-        for (var i = 0; i < vectorValues.size(); i++) {
-            var rowIds = postingsMap.get(vectorValues.vectorValue(i)).getRowIds();
-            for (int r = 0; r < rowIds.size(); r++)
-                pairs.add(Pair.create(rowIds.getInt(r), i));
-        }
-
-        // Sort the pairs by rowId
-        pairs.sort(Comparator.comparingInt(Pair::left));
-
-        // Write the pairs to the file
-        long startOffset = writer.position();
-        for (var pair : pairs) {
-            writer.writeInt(pair.left);
-            writer.writeInt(pair.right);
-        }
-
-        // write the position of the beginning of rowid -> ordinals mappings to the end
-        writer.writeLong(startOffset);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/DiskBinarySearch.java b/src/java/org/apache/cassandra/index/sai/disk/v2/DiskBinarySearch.java
new file mode 100644
index 000000000000..4e0080bfead7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/DiskBinarySearch.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+
+public class DiskBinarySearch
+{
+    /**
+     * A function that takes a primitive long and returns a primitive int.
+     */
+    @FunctionalInterface
+    public interface LongIntFunction
+    {
+        int apply(long i);
+    }
+
+    /**
+     * Search for the target int between positions low and high, using the provided function
+     * to retrieve the int value at the given ordinal.
+     *
+     * Returns the position at which target is found.  Raises an exception if it is not found.
+     *
+     * This will not call f() after the target is found, so if f is performing disk seeks,
+     * it will leave the underlying reader at the position right after reading the target.
+     *
+     * @return index if target is found; otherwise return -1 if targer is not found
+     */
+    public static long searchInt(long low, long high, int target, LongIntFunction f)
+    {
+        return search(low, high, target, false, f);
+    }
+
+    /**
+     * Similar to searchInt but returns index of a value greater or equal to the target, -1 if not found.
+     */
+    public static long searchFloor(long low, long high, int target, LongIntFunction f)
+    {
+        return search(low, high, target, true, f);
+    }
+
+    private static long search(long low, long high, int target, boolean floorSearch, LongIntFunction f)
+    {
+        assert high < Long.MAX_VALUE >> 2 : "high is too large to avoid potential overflow: " + high;
+        assert low < high : "low must be less than high: " + low + " >= " + high;
+
+        int value = Integer.MIN_VALUE;
+        long i = low;
+
+        while (low < high)
+        {
+            i = low + (high - low) / 2;
+            value = f.apply(i);
+
+            if (target == value)
+                return i;
+
+            if (target > value)
+                low = i + 1;
+            else
+                high = i;
+        }
+        return floorSearch && value >= target ? i : -1;
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyFactory.java b/src/java/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyFactory.java
new file mode 100644
index 000000000000..cba1e1124ce3
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyFactory.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * A row-aware {@link PrimaryKey.Factory}. This creates {@link PrimaryKey} instances that are
+ * sortable by {@link DecoratedKey} and {@link Clustering}.
+ */
+public class RowAwarePrimaryKeyFactory implements PrimaryKey.Factory
+{
+    private final ClusteringComparator clusteringComparator;
+    private final boolean hasEmptyClustering;
+
+
+    public RowAwarePrimaryKeyFactory(ClusteringComparator clusteringComparator)
+    {
+        this.clusteringComparator = clusteringComparator;
+        this.hasEmptyClustering = clusteringComparator.size() == 0;
+    }
+
+    @Override
+    public PrimaryKey createTokenOnly(Token token)
+    {
+        return new RowAwarePrimaryKey(token, null, null, null);
+    }
+
+    @Override
+    public PrimaryKey createDeferred(Token token, Supplier<PrimaryKey> primaryKeySupplier)
+    {
+        return new RowAwarePrimaryKey(token, null, null, primaryKeySupplier);
+    }
+
+    @Override
+    public PrimaryKey create(DecoratedKey partitionKey, Clustering clustering)
+    {
+        return new RowAwarePrimaryKey(partitionKey.getToken(), partitionKey, clustering, null);
+    }
+
+    private class RowAwarePrimaryKey implements PrimaryKey
+    {
+        private Token token;
+        private DecoratedKey partitionKey;
+        private Clustering clustering;
+        private Supplier<PrimaryKey> primaryKeySupplier;
+
+        private RowAwarePrimaryKey(Token token, DecoratedKey partitionKey, Clustering clustering, Supplier<PrimaryKey> primaryKeySupplier)
+        {
+            this.token = token;
+            this.partitionKey = partitionKey;
+            this.clustering = clustering;
+            this.primaryKeySupplier = primaryKeySupplier;
+        }
+
+        @Override
+        public Token token()
+        {
+            return token;
+        }
+
+        @Override
+        public DecoratedKey partitionKey()
+        {
+            loadDeferred();
+            return partitionKey;
+        }
+
+        @Override
+        public Clustering clustering()
+        {
+            loadDeferred();
+            return clustering;
+        }
+
+        @Override
+        public PrimaryKey loadDeferred()
+        {
+            if (primaryKeySupplier != null && partitionKey == null)
+            {
+                PrimaryKey deferredPrimaryKey = primaryKeySupplier.get();
+                this.partitionKey = deferredPrimaryKey.partitionKey();
+                this.clustering = deferredPrimaryKey.clustering();
+                primaryKeySupplier = null;
+            }
+            return this;
+        }
+
+        @Override
+        public ByteSource asComparableBytes(ByteComparable.Version version)
+        {
+            return asComparableBytes(version == ByteComparable.Version.LEGACY ? ByteSource.END_OF_STREAM : ByteSource.TERMINATOR, version, false);
+        }
+
+        @Override
+        public ByteSource asComparableBytesMinPrefix(ByteComparable.Version version)
+        {
+            return asComparableBytes(ByteSource.LT_NEXT_COMPONENT, version, true);
+        }
+
+        @Override
+        public ByteSource asComparableBytesMaxPrefix(ByteComparable.Version version)
+        {
+            return asComparableBytes(ByteSource.GT_NEXT_COMPONENT, version, true);
+        }
+
+        private ByteSource asComparableBytes(int terminator, ByteComparable.Version version, boolean isPrefix)
+        {
+            // We need to make sure that the key is loaded before returning a
+            // byte comparable representation. If we don't we won't get a correct
+            // comparison because we potentially won't be using the partition key
+            // and clustering for the lookup
+            loadDeferred();
+
+            ByteSource tokenComparable = token.asComparableBytes(version);
+            ByteSource keyComparable = partitionKey == null ? null
+                                                            : ByteSource.of(partitionKey.getKey(), version);
+
+            // It is important that the ClusteringComparator.asBytesComparable method is used
+            // to maintain the correct clustering sort order
+            ByteSource clusteringComparable = clusteringComparator.size() == 0 ||
+                                              clustering == null ||
+                                              clustering.isEmpty() ? null
+                                                                   : clusteringComparator.asByteComparable(clustering)
+                                                                                         .asComparableBytes(version);
+
+            // prefix doesn't include null components
+            if (isPrefix)
+            {
+                if (keyComparable == null)
+                    return ByteSource.withTerminator(terminator, tokenComparable);
+                else if (clusteringComparable == null)
+                    return ByteSource.withTerminator(terminator, tokenComparable, keyComparable);
+            }
+            return ByteSource.withTerminator(terminator, tokenComparable, keyComparable, clusteringComparable);
+        }
+
+        @Override
+        public int compareTo(PrimaryKey o)
+        {
+            int cmp = token().compareTo(o.token());
+
+            // If the tokens don't match then we don't need to compare any more of the key.
+            // Otherwise if this key has no deferred loader and it's partition key is null
+            // or the other partition key is null then one or both of the keys
+            // are token only so we can only compare tokens
+            if ((cmp != 0) || (primaryKeySupplier == null && partitionKey == null) || o.partitionKey() == null)
+                return cmp;
+
+            // Next compare the partition keys. If they are not equal or
+            // this is a single row partition key or there are no
+            // clusterings then we can return the result of this without
+            // needing to compare the clusterings
+            cmp = partitionKey().compareTo(o.partitionKey());
+            if (cmp != 0 || hasEmptyClustering() || o.hasEmptyClustering())
+                return cmp;
+            return clusteringComparator.compare(clustering(), o.clustering());
+        }
+
+        @Override
+        public int hashCode()
+        {
+            if (hasEmptyClustering)
+                return Objects.hash(token);
+            return Objects.hash(token, clustering());
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (obj instanceof PrimaryKey)
+                return compareTo((PrimaryKey)obj) == 0;
+            return false;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("RowAwarePrimaryKey: { token: %s, partition: %s, clustering: %s:%s} ",
+                                 token,
+                                 partitionKey,
+                                 clustering == null ? null : clustering.kind(),
+                                 clustering == null ? null :String.join(",", Arrays.stream(clustering.getBufferArray())
+                                                                                   .map(ByteBufferUtil::bytesToHex)
+                                                                                   .collect(Collectors.toList())));
+        }
+
+        @Override
+        public long ramBytesUsed()
+        {
+            // Object header + 4 references (token, partitionKey, clustering, primaryKeySupplier) + implicit outer reference
+            long size = RamUsageEstimator.NUM_BYTES_OBJECT_HEADER +
+                       5L * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+            
+            if (token != null)
+                size += token.getHeapSize();
+            if (partitionKey != null)
+                size += RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 
+                       2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF + // token and key references  
+                       2L * Long.BYTES;
+            // We don't count clustering size here as it's managed elsewhere
+            return size;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyMap.java b/src/java/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyMap.java
new file mode 100644
index 000000000000..2f1681abef34
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyMap.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.apache.cassandra.index.sai.disk.v1.MetadataSource;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.BlockPackedReader;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
+import org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsMeta;
+import org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsReader;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+/**
+ * A row-aware {@link PrimaryKeyMap}
+ *
+ * This uses the following on-disk structures:
+ * <ul>
+ *     <li>Block-packed structure for rowId to token lookups using {@link BlockPackedReader}.
+ *     Uses component {@link IndexComponentType#TOKEN_VALUES} </li>
+ *     <li>A sorted-terms structure for rowId to {@link PrimaryKey} and {@link PrimaryKey} to rowId lookups using
+ *     {@link SortedTermsReader}. Uses components {@link IndexComponentType#PRIMARY_KEY_TRIE}, {@link IndexComponentType#PRIMARY_KEY_BLOCKS},
+ *     {@link IndexComponentType#PRIMARY_KEY_BLOCK_OFFSETS}</li>
+ * </ul>
+ *
+ * While the {@link RowAwarePrimaryKeyMapFactory} is threadsafe, individual instances of the {@link RowAwarePrimaryKeyMap}
+ * are not.
+ */
+@NotThreadSafe
+public class RowAwarePrimaryKeyMap implements PrimaryKeyMap
+{
+    @ThreadSafe
+    public static class RowAwarePrimaryKeyMapFactory implements Factory
+    {
+        private final IndexComponents.ForRead perSSTableComponents;
+        private final LongArray.Factory tokenReaderFactory;
+        private final SortedTermsReader sortedTermsReader;
+        private final long count;
+        private FileHandle token = null;
+        private FileHandle termsDataBlockOffsets = null;
+        private FileHandle termsData = null;
+        private FileHandle termsTrie = null;
+        private final IPartitioner partitioner;
+        private final ClusteringComparator clusteringComparator;
+        private final PrimaryKey.Factory primaryKeyFactory;
+        private final SSTableId<?> sstableId;
+
+        public RowAwarePrimaryKeyMapFactory(IndexComponents.ForRead perSSTableComponents, PrimaryKey.Factory primaryKeyFactory, SSTableReader sstable)
+        {
+            try
+            {
+                this.perSSTableComponents = perSSTableComponents;
+                MetadataSource metadataSource = MetadataSource.loadMetadata(perSSTableComponents);
+                NumericValuesMeta tokensMeta = new NumericValuesMeta(metadataSource.get(perSSTableComponents.get(IndexComponentType.TOKEN_VALUES)));
+                count = tokensMeta.valueCount;
+                SortedTermsMeta sortedTermsMeta = new SortedTermsMeta(metadataSource.get(perSSTableComponents.get(IndexComponentType.PRIMARY_KEY_BLOCKS)));
+                NumericValuesMeta blockOffsetsMeta = new NumericValuesMeta(metadataSource.get(perSSTableComponents.get(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS)));
+
+                token = perSSTableComponents.get(IndexComponentType.TOKEN_VALUES).createFileHandle();
+                this.tokenReaderFactory = new BlockPackedReader(token, tokensMeta);
+                this.termsDataBlockOffsets = perSSTableComponents.get(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS).createFileHandle();
+                this.termsData = perSSTableComponents.get(IndexComponentType.PRIMARY_KEY_BLOCKS).createFileHandle();
+                this.termsTrie = perSSTableComponents.get(IndexComponentType.PRIMARY_KEY_TRIE).createFileHandle();
+                this.sortedTermsReader = new SortedTermsReader(termsData, termsDataBlockOffsets, termsTrie, sortedTermsMeta, blockOffsetsMeta);
+                this.partitioner = sstable.metadata().partitioner;
+                this.primaryKeyFactory = primaryKeyFactory;
+                this.clusteringComparator = sstable.metadata().comparator;
+                this.sstableId = sstable.getId();
+            }
+            catch (Throwable t)
+            {
+                throw Throwables.unchecked(Throwables.close(t, token, termsData, termsDataBlockOffsets, termsTrie));
+            }
+        }
+
+        @Override
+        public PrimaryKeyMap newPerSSTablePrimaryKeyMap()
+        {
+            final LongArray rowIdToToken = new LongArray.DeferredLongArray(() -> tokenReaderFactory.open());
+            try
+            {
+                return new RowAwarePrimaryKeyMap(rowIdToToken,
+                                                 sortedTermsReader,
+                                                 sortedTermsReader.openCursor(),
+                                                 partitioner,
+                                                 primaryKeyFactory,
+                                                 clusteringComparator,
+                                                 sstableId);
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        @Override
+        public long count()
+        {
+            return count;
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            FileUtils.closeQuietly(token, termsData, termsDataBlockOffsets, termsTrie);
+        }
+    }
+
+    private final LongArray rowIdToToken;
+    private final SortedTermsReader sortedTermsReader;
+    private final SortedTermsReader.Cursor cursor;
+    private final IPartitioner partitioner;
+    private final PrimaryKey.Factory primaryKeyFactory;
+    private final ClusteringComparator clusteringComparator;
+    private final SSTableId<?> sstableId;
+    private final ByteBuffer tokenBuffer = ByteBuffer.allocate(Long.BYTES);
+
+    private RowAwarePrimaryKeyMap(LongArray rowIdToToken,
+                                  SortedTermsReader sortedTermsReader,
+                                  SortedTermsReader.Cursor cursor,
+                                  IPartitioner partitioner,
+                                  PrimaryKey.Factory primaryKeyFactory,
+                                  ClusteringComparator clusteringComparator,
+                                  SSTableId<?> sstableId)
+    {
+        this.rowIdToToken = rowIdToToken;
+        this.sortedTermsReader = sortedTermsReader;
+        this.cursor = cursor;
+        this.partitioner = partitioner;
+        this.primaryKeyFactory = primaryKeyFactory;
+        this.clusteringComparator = clusteringComparator;
+        this.sstableId = sstableId;
+    }
+
+    @Override
+    public SSTableId<?> getSSTableId()
+    {
+        return sstableId;
+    }
+
+    public long count()
+    {
+        return rowIdToToken.length();
+    }
+
+    @Override
+    public PrimaryKey primaryKeyFromRowId(long sstableRowId)
+    {
+        tokenBuffer.putLong(rowIdToToken.get(sstableRowId));
+        tokenBuffer.rewind();
+        return primaryKeyFactory.createDeferred(partitioner.getTokenFactory().fromByteArray(tokenBuffer), () -> supplier(sstableRowId));
+    }
+
+    private long skinnyExactRowIdOrInvertedCeiling(PrimaryKey key)
+    {
+        // Fast path when there is no clustering, i.e., there is one row per partition.
+        // (The reason we don't just make the Factory return a PartitionAware map for this case
+        // is that it reads partition keys directly from the sstable using the offsets file.
+        // While this worked in BDP, it was not efficient and caused problems because the
+        // sstable reader was using 64k page sizes, and this caused page cache thrashing.
+        long rowId = rowIdToToken.indexOf(key.token().getLongValue());
+        if (rowId < 0)
+            // No match found, return the inverted ceiling
+            return rowId;
+        // The first index might not have been the correct match in the case of token collisions.
+        return tokenCollisionDetection(key, rowId);
+    }
+
+    /**
+     * Returns a row Id for a {@link PrimaryKey}. If there is no such term, returns the `-(next row id) - 1` where
+     * `next row id` is the row id of the next greatest {@link PrimaryKey} in the map.
+     * @param key the {@link PrimaryKey} to lookup
+     * @return a row id
+     */
+    @Override
+    public long exactRowIdOrInvertedCeiling(PrimaryKey key)
+    {
+        if (clusteringComparator.size() == 0)
+            return skinnyExactRowIdOrInvertedCeiling(key);
+
+        long pointId = cursor.getExactPointId(v -> key.asComparableBytes(v));
+        if (pointId >= 0)
+            return pointId;
+        long ceiling = cursor.ceiling(v -> key.asComparableBytesMinPrefix(v));
+        // Use min value since -(Long.MIN_VALUE) - 1 == Long.MAX_VALUE.
+        return ceiling < 0 ? Long.MIN_VALUE : -ceiling - 1;
+    }
+
+    @Override
+    public long ceiling(PrimaryKey key)
+    {
+        if (clusteringComparator.size() == 0)
+        {
+            long rowId = skinnyExactRowIdOrInvertedCeiling(key);
+            if (rowId >= 0)
+                return rowId;
+            else
+                if (rowId == Long.MIN_VALUE)
+                    return -1;
+                else
+                    return -rowId - 1;
+        }
+
+        return cursor.ceiling(key::asComparableBytesMinPrefix);
+    }
+
+    @Override
+    public long floor(PrimaryKey key)
+    {
+        return cursor.floor(key::asComparableBytesMaxPrefix);
+    }
+
+
+    @Override
+    public void close() throws IOException
+    {
+        FileUtils.closeQuietly(cursor, rowIdToToken);
+    }
+
+    private PrimaryKey supplier(long sstableRowId)
+    {
+        try
+        {
+            cursor.seekToPointId(sstableRowId);
+            ByteSource.Peekable peekable = ByteSource.peekable(cursor.term().asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION));
+
+            Token token = partitioner.getTokenFactory().fromComparableBytes(ByteSourceInverse.nextComponentSource(peekable),
+                                                                            TypeUtil.BYTE_COMPARABLE_VERSION);
+            byte[] keyBytes = ByteSourceInverse.getUnescapedBytes(ByteSourceInverse.nextComponentSource(peekable));
+
+            if (keyBytes == null)
+                return primaryKeyFactory.createTokenOnly(token);
+
+            DecoratedKey partitionKey = new BufferDecoratedKey(token, ByteBuffer.wrap(keyBytes));
+
+            Clustering clustering = clusteringComparator.size() == 0
+                                    ? Clustering.EMPTY
+                                    : clusteringComparator.clusteringFromByteComparable(ByteBufferAccessor.instance,
+                                                                                        v -> ByteSourceInverse.nextComponentSource(peekable),
+                                                                                        TypeUtil.BYTE_COMPARABLE_VERSION);
+
+            return primaryKeyFactory.create(partitionKey, clustering);
+        }
+        catch (IOException e)
+        {
+            throw Throwables.cleaned(e);
+        }
+    }
+
+    // Look for token collision by if the ajacent token in the token array matches the
+    // current token. If we find a collision we need to compare the partition key instead.
+    protected long tokenCollisionDetection(PrimaryKey primaryKey, long rowId)
+    {
+        // Look for collisions while we haven't reached the end of the tokens and the tokens don't collide
+        while (rowId + 1 < rowIdToToken.length() && primaryKey.token().getLongValue() == rowIdToToken.get(rowId + 1))
+        {
+            // If we had a collision then see if the partition key for this row is >= to the lookup partition key
+            if (primaryKeyFromRowId(rowId).compareTo(primaryKey) >= 0)
+                return rowId;
+
+            rowId++;
+        }
+        // Note: We would normally expect to get here without going into the while loop
+        return rowId;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/SSTableComponentsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v2/SSTableComponentsWriter.java
new file mode 100644
index 000000000000..f4bfe3c34e0e
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/SSTableComponentsWriter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.io.IOException;
+
+import com.google.common.base.Stopwatch;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.index.sai.disk.PerSSTableWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
+import org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsWriter;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.lucene.util.IOUtils;
+
+public class SSTableComponentsWriter implements PerSSTableWriter
+{
+    protected static final Logger logger = LoggerFactory.getLogger(SSTableComponentsWriter.class);
+
+    private final IndexComponents.ForWrite perSSTableComponents;
+    private final MetadataWriter metadataWriter;
+    private final NumericValuesWriter tokenWriter;
+    private final NumericValuesWriter blockFPWriter;
+    private final SortedTermsWriter sortedTermsWriter;
+
+    public SSTableComponentsWriter(IndexComponents.ForWrite perSSTableComponents) throws IOException
+    {
+        this.perSSTableComponents = perSSTableComponents;
+        this.metadataWriter = new MetadataWriter(perSSTableComponents);
+        this.tokenWriter = new NumericValuesWriter(perSSTableComponents.addOrGet(IndexComponentType.TOKEN_VALUES),
+                                                   metadataWriter, false);
+
+        this.blockFPWriter = new NumericValuesWriter(perSSTableComponents.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS),
+                                                     metadataWriter, true);
+        this.sortedTermsWriter = new SortedTermsWriter(perSSTableComponents.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCKS),
+                                                       metadataWriter,
+                                                       blockFPWriter,
+                                                       perSSTableComponents.addOrGet(IndexComponentType.PRIMARY_KEY_TRIE));
+    }
+
+    @Override
+    public void nextRow(PrimaryKey primaryKey) throws IOException
+    {
+        tokenWriter.add(primaryKey.token().getLongValue());
+        sortedTermsWriter.add(v -> primaryKey.asComparableBytes(v));
+    }
+
+    @Override
+    public void complete(Stopwatch stopwatch) throws IOException
+    {
+        IOUtils.close(tokenWriter, sortedTermsWriter, metadataWriter);
+        perSSTableComponents.markComplete();
+    }
+
+    @Override
+    public void abort(Throwable accumulator)
+    {
+        logger.debug(perSSTableComponents.logMessage("Aborting per-SSTable index component writer for {}..."), perSSTableComponents.descriptor());
+        perSSTableComponents.forceDeleteAllComponents();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/V2InvertedIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v2/V2InvertedIndexSearcher.java
new file mode 100644
index 000000000000..c9e471d2116c
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/V2InvertedIndexSearcher.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.InvertedIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+
+/**
+ * The key override for this class is the use of {@link Version#BA}.
+ */
+class V2InvertedIndexSearcher extends InvertedIndexSearcher
+{
+    V2InvertedIndexSearcher(SSTableContext sstableContext,
+                            PerIndexFiles perIndexFiles,
+                            SegmentMetadata segmentMetadata,
+                            IndexContext indexContext) throws IOException
+    {
+        // We filter because the CA format wrote maps acording to a different order than their abstract type.
+        super(sstableContext, perIndexFiles, segmentMetadata, indexContext, Version.BA, true);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/V2OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v2/V2OnDiskFormat.java
new file mode 100644
index 000000000000..8ad67fcfd9e1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/V2OnDiskFormat.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteOrder;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.PerSSTableWriter;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
+import org.apache.cassandra.index.sai.disk.v1.IndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+/**
+ * Updates SAI OnDiskFormat to include full PK -> offset mapping, and adds vector components.
+ */
+public class V2OnDiskFormat extends V1OnDiskFormat
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private static final Set<IndexComponentType> PER_SSTABLE_COMPONENTS = EnumSet.of(IndexComponentType.GROUP_COMPLETION_MARKER,
+                                                                                     IndexComponentType.GROUP_META,
+                                                                                     IndexComponentType.TOKEN_VALUES,
+                                                                                     IndexComponentType.PRIMARY_KEY_TRIE,
+                                                                                     IndexComponentType.PRIMARY_KEY_BLOCKS,
+                                                                                     IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS);
+
+    public static final Set<IndexComponentType> VECTOR_COMPONENTS_V2 = EnumSet.of(IndexComponentType.COLUMN_COMPLETION_MARKER,
+                                                                                  IndexComponentType.META,
+                                                                                  IndexComponentType.VECTOR,
+                                                                                  IndexComponentType.TERMS_DATA,
+                                                                                  IndexComponentType.POSTING_LISTS);
+
+    public static final V2OnDiskFormat instance = new V2OnDiskFormat();
+
+    private static final IndexFeatureSet v2IndexFeatureSet = new IndexFeatureSet()
+    {
+        @Override
+        public boolean isRowAware()
+        {
+            return true;
+        }
+
+        @Override
+        public boolean hasVectorIndexChecksum()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean hasTermsHistogram()
+        {
+            return false;
+        }
+    };
+
+    protected V2OnDiskFormat()
+    {}
+
+    @Override
+    public IndexFeatureSet indexFeatureSet()
+    {
+        return v2IndexFeatureSet;
+    }
+
+    @Override
+    public PrimaryKey.Factory newPrimaryKeyFactory(ClusteringComparator comparator)
+    {
+        return new RowAwarePrimaryKeyFactory(comparator);
+    }
+
+    @Override
+    public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexComponents.ForRead perSSTableComponents, PrimaryKey.Factory primaryKeyFactory, SSTableReader sstable)
+    {
+        return new RowAwarePrimaryKeyMap.RowAwarePrimaryKeyMapFactory(perSSTableComponents, primaryKeyFactory, sstable);
+    }
+
+    @Override
+    public IndexSearcher newIndexSearcher(SSTableContext sstableContext,
+                                          IndexContext indexContext,
+                                          PerIndexFiles indexFiles,
+                                          SegmentMetadata segmentMetadata) throws IOException
+    {
+        if (indexContext.isVector())
+            throw new IllegalStateException("V2 (HNSW) vector index support has been removed");
+        if (indexContext.isLiteral())
+            return new V2InvertedIndexSearcher(sstableContext, indexFiles, segmentMetadata, indexContext);
+        return super.newIndexSearcher(sstableContext, indexContext, indexFiles, segmentMetadata);
+    }
+
+    @Override
+    public PerSSTableWriter newPerSSTableWriter(IndexDescriptor indexDescriptor) throws IOException
+    {
+        return new SSTableComponentsWriter(indexDescriptor.newPerSSTableComponentsForWrite());
+    }
+
+    @Override
+    public Set<IndexComponentType> perIndexComponentTypes(AbstractType<?> validator)
+    {
+        if (validator.isVector())
+            return VECTOR_COMPONENTS_V2;
+        return super.perIndexComponentTypes(validator);
+    }
+
+    @Override
+    public Set<IndexComponentType> perSSTableComponentTypes()
+    {
+        return PER_SSTABLE_COMPONENTS;
+    }
+
+    @Override
+    public int openFilesPerSSTable()
+    {
+        return 4;
+    }
+
+    @Override
+    public ByteOrder byteOrderFor(IndexComponentType indexComponentType, IndexContext context)
+    {
+        // The little-endian files are written by Lucene, and the upgrade to Lucene 9 switched the byte order from big to little.
+        switch (indexComponentType)
+        {
+            case META:
+            case GROUP_META:
+            case TOKEN_VALUES:
+            case PRIMARY_KEY_BLOCK_OFFSETS:
+            case KD_TREE:
+            case KD_TREE_POSTING_LISTS:
+                return ByteOrder.LITTLE_ENDIAN;
+            case POSTING_LISTS:
+                return (context != null && context.isVector()) ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;
+            default:
+                return ByteOrder.BIG_ENDIAN;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/V2OnDiskOrdinalsMap.java b/src/java/org/apache/cassandra/index/sai/disk/v2/V2OnDiskOrdinalsMap.java
new file mode 100644
index 000000000000..8fbe45fd6ab5
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/V2OnDiskOrdinalsMap.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.PrimitiveIterator;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.util.Bits;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter.Structure;
+import org.apache.cassandra.index.sai.disk.vector.BitsUtil;
+import org.apache.cassandra.index.sai.disk.vector.OnDiskOrdinalsMap;
+import org.apache.cassandra.index.sai.disk.vector.OrdinalsView;
+import org.apache.cassandra.index.sai.disk.vector.RowIdsView;
+import org.apache.cassandra.index.sai.utils.SingletonIntIterator;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+public class V2OnDiskOrdinalsMap implements OnDiskOrdinalsMap
+{
+    private static final Logger logger = LoggerFactory.getLogger(V2OnDiskOrdinalsMap.class);
+
+    private final OrdinalsView fastOrdinalsView;
+    private static final OneToOneRowIdsView ONE_TO_ONE_ROW_IDS_VIEW = new OneToOneRowIdsView();
+    private final FileHandle fh;
+    private final long ordToRowOffset;
+    private final long segmentEnd;
+    private final int size;
+    // the offset where we switch from recording ordinal -> rows, to row -> ordinal
+    private final long rowOrdinalOffset;
+    private final Set<Integer> deletedOrdinals;
+
+    private final boolean canFastMapOrdinalsView;
+    private final boolean canFastMapRowIdsView;
+
+    public V2OnDiskOrdinalsMap(FileHandle fh, long segmentOffset, long segmentLength)
+    {
+        deletedOrdinals = new HashSet<>();
+
+        this.segmentEnd = segmentOffset + segmentLength;
+        this.fh = fh;
+        try (var reader = fh.createReader())
+        {
+            reader.seek(segmentOffset);
+            int deletedCount = reader.readInt();
+            for (var i = 0; i < deletedCount; i++)
+            {
+                int ordinal = reader.readInt();
+                deletedOrdinals.add(ordinal);
+            }
+
+            this.ordToRowOffset = reader.getFilePointer();
+            this.size = reader.readInt();
+            reader.seek(segmentEnd - 8);
+            this.rowOrdinalOffset = reader.readLong();
+
+            // When rowOrdinalOffset + 8 is equal to segmentEnd, the segment has no postings. Therefore,
+            // we use the EmptyView. That case does not get a fastRowIdsView because we only hit that code after
+            // getting ordinals from the graph, and an EmptyView will not produce any ordinals to search. Importantly,
+            // the file format for the RowIdsView is correct, even if there are no postings.
+            this.canFastMapRowIdsView = deletedCount == -1;
+            this.canFastMapOrdinalsView = deletedCount == -1 || rowOrdinalOffset + 8 == segmentEnd;
+            this.fastOrdinalsView = deletedCount == -1 ? new OneToOneOrdinalsView(size) : new EmptyOrdinalsView();
+            assert rowOrdinalOffset < segmentEnd : "rowOrdinalOffset " + rowOrdinalOffset + " is not less than segmentEnd " + segmentEnd;
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException("Error initializing OnDiskOrdinalsMap at segment " + segmentOffset, e);
+        }
+    }
+
+    @Override
+    public Structure getStructure()
+    {
+        return canFastMapOrdinalsView ? Structure.ONE_TO_ONE : Structure.ZERO_OR_ONE_TO_MANY;
+    }
+
+    @Override
+    public long cachedBytesUsed()
+    {
+        return 0;
+    }
+
+    @Override
+    public RowIdsView getRowIdsView()
+    {
+        if (canFastMapRowIdsView) {
+            return ONE_TO_ONE_ROW_IDS_VIEW;
+        }
+
+        return new FileReadingRowIdsView();
+    }
+
+    @Override
+    public Bits ignoringDeleted(Bits acceptBits)
+    {
+        return BitsUtil.bitsIgnoringDeleted(acceptBits, deletedOrdinals);
+    }
+
+    private class FileReadingRowIdsView implements RowIdsView
+    {
+        RandomAccessReader reader = fh.createReader();
+
+        @Override
+        public PrimitiveIterator.OfInt getSegmentRowIdsMatching(int vectorOrdinal) throws IOException
+        {
+            Preconditions.checkArgument(vectorOrdinal < size, "vectorOrdinal %s is out of bounds %s", vectorOrdinal, size);
+
+            // read index entry
+            try
+            {
+                reader.seek(ordToRowOffset + 4L + vectorOrdinal * 8L);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(String.format("Error seeking to index offset for ordinal %d with ordToRowOffset %d",
+                                                         vectorOrdinal, ordToRowOffset), e);
+            }
+            var offset = reader.readLong();
+            // seek to and read rowIds
+            try
+            {
+                reader.seek(offset);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(String.format("Error seeking to rowIds offset for ordinal %d with ordToRowOffset %d",
+                                                         vectorOrdinal, ordToRowOffset), e);
+            }
+            var postingsSize = reader.readInt();
+
+            // Optimize for the most common case
+            if (postingsSize == 1)
+                return new SingletonIntIterator(reader.readInt());
+
+            var rowIds = new int[postingsSize];
+            for (var i = 0; i < rowIds.length; i++)
+            {
+                rowIds[i] = reader.readInt();
+            }
+            return Arrays.stream(rowIds).iterator();
+        }
+
+        @Override
+        public void close()
+        {
+            reader.close();
+        }
+    }
+
+    @Override
+    public OrdinalsView getOrdinalsView()
+    {
+        if (canFastMapOrdinalsView) {
+            return fastOrdinalsView;
+        }
+
+        return new FileReadingOrdinalsView();
+    }
+
+    /**
+     * not thread safe
+     */
+    private class FileReadingOrdinalsView implements OrdinalsView
+    {
+        RandomAccessReader reader = fh.createReader();
+        private final long high = (segmentEnd - 8 - rowOrdinalOffset) / 8;
+        private int lastFoundRowId = -1;
+        private long lastFoundRowIdIndex = -1;
+
+        private int lastRowId = -1;
+
+        /**
+         * @return order if given row id is found; otherwise return -1
+         * rowId must increase
+         */
+        @Override
+        public int getOrdinalForRowId(int rowId) throws IOException
+        {
+            if (rowId <= lastRowId)
+                throw new IllegalArgumentException("rowId " + rowId + " is less than or equal to lastRowId " + lastRowId);
+            lastRowId = rowId;
+
+            if (rowId < lastFoundRowId) // skipped row, no need to search
+                return -1;
+
+            long low = 0;
+            if (lastFoundRowId > -1 && lastFoundRowIdIndex < high)
+            {
+                low = lastFoundRowIdIndex;
+
+                if (lastFoundRowId == rowId) // "lastFoundRowId + 1 == rowId" case that returned -1 likely moved use here
+                {
+                    long offset = rowOrdinalOffset + lastFoundRowIdIndex * 8;
+                    reader.seek(offset);
+                    int foundRowId = reader.readInt();
+                    assert foundRowId == rowId : "expected rowId " + rowId + " but found " + foundRowId;
+                    return reader.readInt();
+                }
+                else if (lastFoundRowId + 1 == rowId) // sequential read, skip binary search
+                {
+                    long offset = rowOrdinalOffset + (lastFoundRowIdIndex + 1) * 8;
+                    reader.seek(offset);
+                    int foundRowId = reader.readInt();
+                    lastFoundRowId = foundRowId;
+                    lastFoundRowIdIndex++;
+                    if (foundRowId == rowId)
+                        return reader.readInt();
+                    else
+                        return -1;
+                }
+            }
+            final AtomicLong lastRowIdIndex = new AtomicLong(-1L);
+            // Compute the offset of the start of the rowId to vectorOrdinal mapping
+            long index = DiskBinarySearch.searchInt(low, high, rowId, i -> {
+                try
+                {
+                    lastRowIdIndex.set(i);
+                    long offset = rowOrdinalOffset + i * 8;
+                    reader.seek(offset);
+                    return reader.readInt();
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            });
+
+            // not found
+            if (index < 0)
+                return -1;
+
+            lastFoundRowId = rowId;
+            lastFoundRowIdIndex = lastRowIdIndex.get();
+            return reader.readInt();
+        }
+
+        @Override
+        public void forEachOrdinalInRange(int startRowId, int endRowId, OrdinalConsumer consumer) throws IOException
+        {
+            long start = DiskBinarySearch.searchFloor(0, high, startRowId, i -> {
+                try
+                {
+                    long offset = rowOrdinalOffset + i * 8;
+                    reader.seek(offset);
+                    return reader.readInt();
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            });
+
+            if (start < 0 || start >= high)
+                return;
+
+            reader.seek(rowOrdinalOffset + start * 8);
+            // sequential read without seeks should be fast, we expect OS to prefetch data from the disk
+            // binary search for starting offset of min rowid >= startRowId unlikely to be faster
+            for (long idx = start; idx < high; idx ++)
+            {
+                int rowId = reader.readInt();
+                if (rowId > endRowId)
+                    break;
+
+                int ordinal = reader.readInt();
+                if (rowId >= startRowId)
+                    consumer.accept(rowId, ordinal);
+            }
+        }
+
+        @Override
+        public void close()
+        {
+            reader.close();
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        fh.close();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/V2VectorIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v2/V2VectorIndexSearcher.java
new file mode 100644
index 000000000000..b53ff7460eaf
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/V2VectorIndexSearcher.java
@@ -0,0 +1,639 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.graph.NodeQueue;
+import io.github.jbellis.jvector.quantization.CompressedVectors;
+import io.github.jbellis.jvector.quantization.ProductQuantization;
+import io.github.jbellis.jvector.util.BitSet;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.util.BoundedLongHeap;
+import io.github.jbellis.jvector.util.SparseBits;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyWithSource;
+import org.apache.cassandra.index.sai.disk.v1.IndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v1.postings.ReorderingPostingList;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter;
+import org.apache.cassandra.index.sai.disk.vector.BruteForceRowIdIterator;
+import org.apache.cassandra.index.sai.disk.vector.CassandraDiskAnn;
+import org.apache.cassandra.index.sai.disk.vector.CloseableReranker;
+import org.apache.cassandra.index.sai.disk.vector.NodeQueueRowIdIterator;
+import org.apache.cassandra.index.sai.disk.vector.VectorCompression;
+import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.plan.Plan.CostCoefficients;
+import org.apache.cassandra.index.sai.utils.SegmentRowIdOrdinalPairs;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RangeUtil;
+import org.apache.cassandra.index.sai.utils.RowIdWithMeta;
+import org.apache.cassandra.index.sai.utils.RowIdWithScore;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.metrics.LinearFit;
+import org.apache.cassandra.metrics.PairedSlidingWindowReservoir;
+import org.apache.cassandra.metrics.QuickSlidingWindowReservoir;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.CloseableIterator;
+
+import static java.lang.Math.ceil;
+import static java.lang.Math.min;
+import static org.apache.cassandra.index.sai.plan.Plan.hrs;
+
+/**
+ * Executes ann search against the graph for an individual index segment.
+ */
+public class V2VectorIndexSearcher extends IndexSearcher
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    /**
+     * Only allow brute force if fewer than this many rows are involved.
+     * Not final so test can inject its own setting.
+     */
+    @VisibleForTesting
+    public static int GLOBAL_BRUTE_FORCE_ROWS = Integer.MAX_VALUE;
+    /**
+     * How much more expensive is brute forcing the comparisons than going through the index?
+     * (brute force needs to go through the full read path to pull out the vectors from the row)
+     */
+    @VisibleForTesting
+    public static double BRUTE_FORCE_EXPENSE_FACTOR = DatabaseDescriptor.getAnnBruteForceExpenseFactor();
+
+    protected final CassandraDiskAnn graph;
+    private final PrimaryKey.Factory keyFactory;
+    private final PairedSlidingWindowReservoir expectedActualNodesVisited = new PairedSlidingWindowReservoir(20);
+    private final ThreadLocal<SparseBits> cachedBits;
+    private final ColumnQueryMetrics.VectorIndexMetrics columnQueryMetrics;
+
+    protected V2VectorIndexSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
+                                    PerIndexFiles perIndexFiles,
+                                    SegmentMetadata segmentMetadata,
+                                    IndexContext indexContext,
+                                    CassandraDiskAnn graph)
+    {
+        super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext);
+        this.graph = graph;
+        this.keyFactory = PrimaryKey.factory(indexContext.comparator(), indexContext.indexFeatureSet());
+        this.cachedBits = ThreadLocal.withInitial(SparseBits::new);
+        this.columnQueryMetrics = (ColumnQueryMetrics.VectorIndexMetrics) indexContext.getColumnQueryMetrics();
+    }
+
+    @Override
+    public long indexFileCacheSize()
+    {
+        return graph.ramBytesUsed();
+    }
+
+    public VectorCompression getCompression()
+    {
+        return graph.getCompression();
+    }
+
+    public ProductQuantization getPQ()
+    {
+        return graph.getPQ();
+    }
+
+    @Override
+    public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
+    {
+        PostingList results = searchPosting(context, exp, keyRange);
+        return toPrimaryKeyIterator(results, context);
+    }
+
+    private PostingList searchPosting(QueryContext context, Expression exp, AbstractBounds<PartitionPosition> keyRange) throws IOException
+    {
+        if (logger.isTraceEnabled())
+            logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);
+
+        if (exp.getOp() != Expression.Op.BOUNDED_ANN)
+            throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression during BOUNDED_ANN index query: " + exp));
+
+        var queryVector = vts.createFloatVector(exp.lower.value.vector);
+
+        // this is a thresholded query, so pass graph.size() as top k to get all results satisfying the threshold
+        var result = searchInternal(keyRange, context, queryVector, graph.size(), graph.size(), exp.getEuclideanSearchThreshold());
+        return new ReorderingPostingList(result, RowIdWithMeta::getSegmentRowId);
+    }
+
+    @Override
+    public CloseableIterator<PrimaryKeyWithSortKey> orderBy(Orderer orderer, Expression slice, AbstractBounds<PartitionPosition> keyRange, QueryContext context, int limit) throws IOException
+    {
+        if (logger.isTraceEnabled())
+            logger.trace(indexContext.logMessage("Searching on expression '{}'..."), orderer);
+
+        if (!orderer.isANN())
+            throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression during ANN index query: " + orderer));
+
+        int rerankK = orderer.rerankKFor(limit, graph.getCompression());
+        var queryVector = vts.createFloatVector(orderer.getVectorTerm());
+
+        var result = searchInternal(keyRange, context, queryVector, limit, rerankK, 0);
+        return toMetaSortedIterator(result, context);
+    }
+
+    /**
+     * Find the closest `limit` neighbors to the given query vector, using a coarse search pass for `rerankK`
+     * candidates.  May decide to use brute force instead of the index.
+     * @param keyRange the key range to search
+     * @param context the query context
+     * @param queryVector the query vector
+     * @param limit the limit for the query
+     * @param rerankK the amplified limit for the query to get more accurate results
+     * @param threshold the threshold for the query. When the threshold is greater than 0 and brute force logic is used,
+     *                  the results will be filtered by the threshold.
+     */
+    private CloseableIterator<RowIdWithScore> searchInternal(AbstractBounds<PartitionPosition> keyRange,
+                                                             QueryContext context,
+                                                             VectorFloat<?> queryVector,
+                                                             int limit,
+                                                             int rerankK,
+                                                             float threshold) throws IOException
+    {
+        try (PrimaryKeyMap primaryKeyMap = primaryKeyMapFactory.newPerSSTablePrimaryKeyMap())
+        {
+            // not restricted
+            if (RangeUtil.coversFullRing(keyRange))
+            {
+                var estimate = estimateCost(rerankK, graph.size());
+                return graph.search(queryVector, limit, rerankK, threshold, Bits.ALL, context, estimate::updateStatistics);
+            }
+
+            PrimaryKey firstPrimaryKey = keyFactory.createTokenOnly(keyRange.left.getToken());
+
+            // it will return the next row id if given key is not found.
+            long minSSTableRowId = primaryKeyMap.ceiling(firstPrimaryKey);
+            // If we didn't find the first key, we won't find the last primary key either
+            if (minSSTableRowId < 0)
+                return CloseableIterator.emptyIterator();
+            long maxSSTableRowId = getMaxSSTableRowId(primaryKeyMap, keyRange.right);
+
+            if (minSSTableRowId > maxSSTableRowId)
+                return CloseableIterator.emptyIterator();
+
+            // if the range covers the entire segment, skip directly to an index search
+            if (minSSTableRowId <= metadata.minSSTableRowId && maxSSTableRowId >= metadata.maxSSTableRowId)
+                return graph.search(queryVector, limit, rerankK, threshold, Bits.ALL, context, visited -> {});
+
+            minSSTableRowId = Math.max(minSSTableRowId, metadata.minSSTableRowId);
+            maxSSTableRowId = min(maxSSTableRowId, metadata.maxSSTableRowId);
+
+            // Upper-bound cost based on maximum possible rows included
+            int nRows = Math.toIntExact(maxSSTableRowId - minSSTableRowId + 1);
+            var initialCostEstimate = estimateCost(rerankK, nRows);
+            Tracing.logAndTrace(logger, "Search range covers {} rows in index of {} nodes; estimate for LIMIT {} is {}",
+                                nRows, graph.size(), rerankK, initialCostEstimate);
+            // if the range spans a small number of rows, then generate scores from the sstable rows instead of searching the index
+            int startSegmentRowId = metadata.toSegmentRowId(minSSTableRowId);
+            int endSegmentRowId = metadata.toSegmentRowId(maxSSTableRowId);
+            if (initialCostEstimate.shouldUseBruteForce())
+            {
+                var maxSize = endSegmentRowId - startSegmentRowId + 1;
+                var segmentOrdinalPairs = new SegmentRowIdOrdinalPairs(maxSize);
+                try (var ordinalsView = graph.getOrdinalsView())
+                {
+                    ordinalsView.forEachOrdinalInRange(startSegmentRowId, endSegmentRowId, segmentOrdinalPairs::add);
+                }
+
+                // When we have a threshold, we only need to filter the results, not order them, because it means we're
+                // evaluating a boolean predicate in the SAI pipeline that wants to collate by PK
+                if (threshold > 0)
+                    return filterByBruteForce(queryVector, segmentOrdinalPairs, threshold);
+                else
+                    return orderByBruteForce(queryVector, segmentOrdinalPairs, limit, rerankK);
+            }
+
+            // create a bitset of ordinals corresponding to the rows in the given key range
+            final Bits bits;
+            try (var ordinalsView = graph.getOrdinalsView())
+            {
+                bits = ordinalsView.buildOrdinalBits(startSegmentRowId, endSegmentRowId, this::bitSetForSearch);
+            }
+            // the set of ordinals may be empty if no rows in the range had a vector associated with them
+            int cardinality = bits instanceof SparseBits ? ((SparseBits) bits).cardinality() : ((BitSet) bits).cardinality();
+            if (cardinality == 0)
+                return CloseableIterator.emptyIterator();
+            // Rows are many-to-one wrt index ordinals, so the actual number of ordinals involved (`cardinality`)
+            // could be less than the number of rows in the range (`nRows`).  In that case we should update the cost
+            // so that we don't pollute the planner with incorrectly pessimistic estimates.
+            //
+            // Technically, we could also have another `shouldUseBruteForce` branch here, but we don't have
+            // the code to generate rowids from ordinals, and it's a rare enough case that it doesn't seem worth
+            // the trouble to add it.
+            var betterCostEstimate = estimateCost(rerankK, cardinality);
+
+            return graph.search(queryVector, limit, rerankK, threshold, bits, context, betterCostEstimate::updateStatistics);
+        }
+    }
+
+    private CloseableIterator<RowIdWithScore> orderByBruteForce(VectorFloat<?> queryVector, SegmentRowIdOrdinalPairs segmentOrdinalPairs, int limit, int rerankK) throws IOException
+    {
+        // If we use compressed vectors, we still have to order rerankK results using full resolution similarity
+        // scores, so only use the compressed vectors when there are enough vectors to make it worthwhile.
+        double twoPassCost = segmentOrdinalPairs.size() * CostCoefficients.ANN_SIMILARITY_COST
+                             + rerankK * hrs(CostCoefficients.ANN_SCORED_KEY_COST);
+        double onePassCost = segmentOrdinalPairs.size() * hrs(CostCoefficients.ANN_SCORED_KEY_COST);
+        if (graph.getCompressedVectors() != null && twoPassCost < onePassCost)
+            return orderByBruteForce(graph.getCompressedVectors(), queryVector, segmentOrdinalPairs, limit, rerankK);
+        return orderByBruteForce(queryVector, segmentOrdinalPairs);
+    }
+
+    /**
+     * Materialize the compressed vectors for the given segment row ids, put them into a priority queue ordered by
+     * approximate similarity score, and then pass to the {@link BruteForceRowIdIterator} to lazily resolve the
+     * full resolution ordering as needed.
+     */
+    private CloseableIterator<RowIdWithScore> orderByBruteForce(CompressedVectors cv,
+                                                                VectorFloat<?> queryVector,
+                                                                SegmentRowIdOrdinalPairs segmentOrdinalPairs,
+                                                                int limit,
+                                                                int rerankK) throws IOException
+    {
+        // Use the jvector NodeQueue to avoid unnecessary object allocations since this part of the code operates on
+        // many rows.
+        var approximateScores = new NodeQueue(new BoundedLongHeap(segmentOrdinalPairs.size()), NodeQueue.Order.MAX_HEAP);
+        var similarityFunction = indexContext.getIndexWriterConfig().getSimilarityFunction();
+        var scoreFunction = cv.precomputedScoreFunctionFor(queryVector, similarityFunction);
+
+        // Store the index of the (rowId, ordinal) pair from the segmentOrdinalPairs in the NodeQueue so that we can
+        // retrieve both values with O(1) lookup when we need to resolve the full resolution score in the
+        // BruteForceRowIdIterator.
+        segmentOrdinalPairs.forEachIndexOrdinalPair((i, ordinal) -> {
+            approximateScores.push(i, scoreFunction.similarityTo(ordinal));
+        });
+        columnQueryMetrics.onBruteForceNodesVisited(segmentOrdinalPairs.size());
+        var reranker = new CloseableReranker(similarityFunction, queryVector, graph.getView());
+        return new BruteForceRowIdIterator(approximateScores, segmentOrdinalPairs, reranker, limit, rerankK, columnQueryMetrics);
+    }
+
+    /**
+     * Produces a correct ranking of the rows in the given segment. Because this graph does not have compressed
+     * vectors, read all vectors and put them into a priority queue to rank them lazily. It is assumed that the whole
+     * PQ will often not be needed.
+     */
+    private CloseableIterator<RowIdWithScore> orderByBruteForce(VectorFloat<?> queryVector, SegmentRowIdOrdinalPairs segmentOrdinalPairs) throws IOException
+    {
+        var scoredRowIds = new NodeQueue(new BoundedLongHeap(segmentOrdinalPairs.size()), NodeQueue.Order.MAX_HEAP);
+        try (var vectorsView = graph.getView())
+        {
+            var similarityFunction = indexContext.getIndexWriterConfig().getSimilarityFunction();
+            var esf = vectorsView.rerankerFor(queryVector, similarityFunction);
+            // Because the scores are exact, we only store the rowid, score pair.
+            segmentOrdinalPairs.forEachSegmentRowIdOrdinalPair((segmentRowId, ordinal) -> {
+                scoredRowIds.push(segmentRowId, esf.similarityTo(ordinal));
+            });
+            columnQueryMetrics.onBruteForceNodesReranked(segmentOrdinalPairs.size());
+            return new NodeQueueRowIdIterator(scoredRowIds);
+        }
+    }
+
+    /**
+     * Materialize the full resolution vector for each row id, compute the similarity score, filter
+     * out rows that do not meet the threshold, and then return them in an iterator.
+     * NOTE: because the threshold is not used for ordering, the result is returned in PK order, not score order.
+     */
+    private CloseableIterator<RowIdWithScore> filterByBruteForce(VectorFloat<?> queryVector,
+                                                                 SegmentRowIdOrdinalPairs segmentOrdinalPairs,
+                                                                 float threshold) throws IOException
+    {
+        var results = new ArrayList<RowIdWithScore>(segmentOrdinalPairs.size());
+        try (var vectorsView = graph.getView())
+        {
+            var similarityFunction = indexContext.getIndexWriterConfig().getSimilarityFunction();
+            var esf = vectorsView.rerankerFor(queryVector, similarityFunction);
+            segmentOrdinalPairs.forEachSegmentRowIdOrdinalPair((segmentRowId, ordinal) -> {
+                var score = esf.similarityTo(ordinal);
+                if (score >= threshold)
+                    results.add(new RowIdWithScore(segmentRowId, score));
+            });
+            columnQueryMetrics.onBruteForceNodesReranked(segmentOrdinalPairs.size());
+        }
+        return CloseableIterator.wrap(results.iterator());
+    }
+
+    private long getMaxSSTableRowId(PrimaryKeyMap primaryKeyMap, PartitionPosition right)
+    {
+        // if the right token is the minimum token, there is no upper bound on the keyRange and
+        // we can save a lookup by using the maxSSTableRowId
+        if (right.isMinimum())
+            return metadata.maxSSTableRowId;
+
+        PrimaryKey lastPrimaryKey = keyFactory.createTokenOnly(right.getToken());
+        long max = primaryKeyMap.floor(lastPrimaryKey);
+        if (max < 0)
+            return metadata.maxSSTableRowId;
+        return max;
+    }
+
+    public V5VectorPostingsWriter.Structure getPostingsStructure()
+    {
+        return graph.getPostingsStructure();
+    }
+
+    private class CostEstimate
+    {
+        private final int candidates;
+        private final int rawExpectedNodesVisited;
+        private final int expectedNodesVisited;
+
+        public CostEstimate(int candidates, int rawExpectedNodesVisited, int expectedNodesVisited)
+        {
+            assert rawExpectedNodesVisited >= 0 : rawExpectedNodesVisited;
+            assert expectedNodesVisited >= 0 : expectedNodesVisited;
+
+            this.candidates = candidates;
+            this.rawExpectedNodesVisited = rawExpectedNodesVisited;
+            this.expectedNodesVisited = expectedNodesVisited;
+        }
+
+        public boolean shouldUseBruteForce()
+        {
+            if (candidates > GLOBAL_BRUTE_FORCE_ROWS)
+                return false;
+            return bruteForceCost() <= indexScanCost();
+        }
+
+        private double indexScanCost()
+        {
+            return expectedNodesVisited
+                   * (CostCoefficients.ANN_SIMILARITY_COST + hrs(CostCoefficients.ANN_EDGELIST_COST) / graph.maxDegree());
+        }
+
+        private double bruteForceCost()
+        {
+            // VSTODO we don't have rerankK available here, so we only calculate the two pass cost
+            // out of the options in orderByBruteForce.  (The rerank cost is roughly equal for both
+            // indexScanCost and bruteForceCost so we can leave it out of both.)
+            return candidates * CostCoefficients.ANN_SIMILARITY_COST;
+        }
+
+        public void updateStatistics(int actualNodesVisited)
+        {
+            assert actualNodesVisited >= 0 : actualNodesVisited;
+            expectedActualNodesVisited.update(rawExpectedNodesVisited, actualNodesVisited);
+
+            if (actualNodesVisited >= 1000 && (actualNodesVisited > 2 * expectedNodesVisited || actualNodesVisited < 0.5 * expectedNodesVisited))
+                Tracing.logAndTrace(logger, "Predicted visiting {} nodes ({} raw), but actually visited {}",
+                                    expectedNodesVisited, rawExpectedNodesVisited, actualNodesVisited);
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("{brute force(%d) = %.2f, index scan(%d) = %.2f}",
+                                 candidates, bruteForceCost(), expectedNodesVisited, indexScanCost());
+        }
+
+        public double cost()
+        {
+            return min(bruteForceCost(), indexScanCost());
+        }
+    }
+
+    public double estimateAnnSearchCost(int rerankK, int candidates)
+    {
+        var estimate = estimateCost(rerankK, candidates);
+        return estimate.cost();
+    }
+
+    private CostEstimate estimateCost(int rerankK, int candidates)
+    {
+        int rawExpectedNodes = getRawExpectedNodes(rerankK, candidates);
+        // update the raw expected value with a linear interpolation based on observed data
+        var observedValues = expectedActualNodesVisited.getSnapshot().values;
+        int expectedNodes;
+        if (observedValues.length >= 10)
+        {
+            var interceptSlope = LinearFit.interceptSlopeFor(observedValues);
+            expectedNodes = (int) (interceptSlope.left + interceptSlope.right * rawExpectedNodes);
+        }
+        else
+        {
+            expectedNodes = rawExpectedNodes;
+        }
+
+        int sanitizedEstimate = VectorMemtableIndex.ensureSaneEstimate(expectedNodes, rerankK, graph.size());
+        return new CostEstimate(candidates, rawExpectedNodes, sanitizedEstimate);
+    }
+
+    private SparseBits bitSetForSearch()
+    {
+        var bits = cachedBits.get();
+        bits.clear();
+        return bits;
+    }
+
+    @Override
+    public CloseableIterator<PrimaryKeyWithSortKey> orderResultsBy(SSTableReader reader,
+                                                                             QueryContext context,
+                                                                             List<PrimaryKey> keys,
+                                                                             Orderer orderer,
+                                                                             int limit) throws IOException
+    {
+        if (keys.isEmpty())
+            return CloseableIterator.emptyIterator();
+
+        int rerankK = orderer.rerankKFor(limit, graph.getCompression());
+        // Convert PKs to segment row ids and map to ordinals, skipping any that don't exist in this segment
+        var segmentOrdinalPairs = flatmapPrimaryKeysToBitsAndRows(keys);
+        var numRows = segmentOrdinalPairs.size();
+        final CostEstimate cost = estimateCost(rerankK, numRows);
+        Tracing.logAndTrace(logger, "{} relevant rows out of {} in range in index of {} nodes; estimate for LIMIT {} is {}",
+                            numRows, keys.size(), graph.size(), limit, cost);
+        if (numRows == 0)
+            return CloseableIterator.emptyIterator();
+
+        if (cost.shouldUseBruteForce())
+        {
+            // brute force using the in-memory compressed vectors to cut down the number of results returned
+            var queryVector = vts.createFloatVector(orderer.getVectorTerm());
+            return toMetaSortedIterator(this.orderByBruteForce(queryVector, segmentOrdinalPairs, limit, rerankK), context);
+        }
+        // Create bits from the mapping
+        var bits = bitSetForSearch();
+        segmentOrdinalPairs.forEachOrdinal(bits::set);
+        // else ask the index to perform a search limited to the bits we created
+        var queryVector = vts.createFloatVector(orderer.getVectorTerm());
+        var results = graph.search(queryVector, limit, rerankK, 0, bits, context, cost::updateStatistics);
+        return toMetaSortedIterator(results, context);
+    }
+
+
+    /**
+     * Build a mapping of segment row id to ordinal for the given primary keys, skipping any that don't exist in this
+     * segment.
+     * @param keysInRange the primary keys to map
+     * @return a mapping of segment row id to ordinal
+     * @throws IOException
+     */
+    private SegmentRowIdOrdinalPairs flatmapPrimaryKeysToBitsAndRows(List<PrimaryKey> keysInRange) throws IOException
+    {
+        var segmentOrdinalPairs = new SegmentRowIdOrdinalPairs(keysInRange.size());
+        int lastSegmentRowId = -1;
+        try (var primaryKeyMap = primaryKeyMapFactory.newPerSSTablePrimaryKeyMap();
+             var ordinalsView = graph.getOrdinalsView())
+        {
+            // track whether we are saving comparisons by using binary search to skip ahead
+            // (if most of the keys belong to this sstable, bsearch will actually be slower)
+            var comparisonsSavedByBsearch = new QuickSlidingWindowReservoir(10);
+            boolean preferSeqScanToBsearch = false;
+
+            for (int i = 0; i < keysInRange.size();)
+            {
+                // turn the pk back into a row id, with a fast path for the case where the pk is from this sstable
+                var primaryKey = keysInRange.get(i);
+                long sstableRowId;
+                if (primaryKey instanceof PrimaryKeyWithSource
+                    && ((PrimaryKeyWithSource) primaryKey).getSourceSstableId().equals(primaryKeyMap.getSSTableId()))
+                    sstableRowId = ((PrimaryKeyWithSource) primaryKey).getSourceRowId();
+                else
+                    sstableRowId = primaryKeyMap.exactRowIdOrInvertedCeiling(primaryKey);
+
+                if (sstableRowId < 0)
+                {
+                    // The given PK doesn't exist in this sstable, so sstableRowId represents the negation
+                    // of the next-highest.  Turn that back into a PK so we can skip ahead in keysInRange.
+                    long ceilingRowId = - sstableRowId - 1;
+                    if (ceilingRowId > metadata.maxSSTableRowId)
+                    {
+                        // The next greatest primary key is greater than all the primary keys in this segment
+                        break;
+                    }
+                    var ceilingPrimaryKey = primaryKeyMap.primaryKeyFromRowId(ceilingRowId);
+
+                    boolean ceilingPrimaryKeyMatchesKeyInRange = false;
+                    // adaptively choose either seq scan or bsearch to skip ahead in keysInRange until
+                    // we find one at least as large as the ceiling key
+                    if (preferSeqScanToBsearch)
+                    {
+                        int keysToSkip = 1; // We already know that the PK at index i is not equal to the ceiling PK.
+                        int cmp = 1; // Need to initialize. The value is irrelevant.
+                        for ( ; i + keysToSkip < keysInRange.size(); keysToSkip++)
+                        {
+                            var nextPrimaryKey = keysInRange.get(i + keysToSkip);
+                            cmp = nextPrimaryKey.compareTo(ceilingPrimaryKey);
+                            if (cmp >= 0)
+                                break;
+                        }
+                        comparisonsSavedByBsearch.update(keysToSkip - (int) ceil(logBase2(keysInRange.size() - i)));
+                        i += keysToSkip;
+                        ceilingPrimaryKeyMatchesKeyInRange = cmp == 0;
+                    }
+                    else
+                    {
+                        // Use a sublist to only search the remaining primary keys in range.
+                        var keysRemaining = keysInRange.subList(i, keysInRange.size());
+                        int nextIndexForCeiling = Collections.binarySearch(keysRemaining, ceilingPrimaryKey);
+                        if (nextIndexForCeiling < 0)
+                            // We got: -(insertion point) - 1. Invert it so we get the insertion point.
+                            nextIndexForCeiling = -nextIndexForCeiling - 1;
+                        else
+                            ceilingPrimaryKeyMatchesKeyInRange = true;
+
+                        comparisonsSavedByBsearch.update(nextIndexForCeiling - (int) ceil(logBase2(keysRemaining.size())));
+                        i += nextIndexForCeiling;
+                    }
+
+                    // update our estimate
+                    preferSeqScanToBsearch = comparisonsSavedByBsearch.size() >= 10
+                                             && comparisonsSavedByBsearch.getMean() < 0;
+                    if (ceilingPrimaryKeyMatchesKeyInRange)
+                        sstableRowId = ceilingRowId;
+                    else
+                        continue; // without incrementing i further. ceilingPrimaryKey is less than the PK at index i.
+                }
+                // Increment here to simplify the sstableRowId < 0 logic.
+                i++;
+
+                // During compaction, the SegmentMetadata is written based on the rows with vector values. Therefore,
+                // we can find a row that has a row id but is outside the min/max range of the segment. We can ignore
+                // these rows here and skip the row id to ordinal conversion that would result in a -1 ordinal.
+                if (sstableRowId < metadata.minSSTableRowId || sstableRowId > metadata.maxSSTableRowId)
+                    continue;
+
+                // convert the global row id to segment row id and from segment row id to graph ordinal
+                int segmentRowId = metadata.toSegmentRowId(sstableRowId);
+                // This requirement is required by the ordinals view. There are cases where we have broken this
+                // requirement, and in order to make future debugging easier, we check here and throw an exception
+                // with additional detail.
+                if (segmentRowId <= lastSegmentRowId)
+                    throw new IllegalStateException("Row ids must ascend monotonically. Got " + segmentRowId + " after " + lastSegmentRowId
+                                                    + " for " + primaryKey + " on sstable " + primaryKeyMap.getSSTableId());
+                lastSegmentRowId = segmentRowId;
+                int ordinal = ordinalsView.getOrdinalForRowId(segmentRowId);
+                if (ordinal >= 0)
+                    segmentOrdinalPairs.add(segmentRowId, ordinal);
+            }
+        }
+        return segmentOrdinalPairs;
+    }
+
+    public static double logBase2(double number) {
+        return Math.log(number) / Math.log(2);
+    }
+
+    private int getRawExpectedNodes(int rerankK, int nPermittedOrdinals)
+    {
+        return VectorMemtableIndex.expectedNodesVisited(rerankK, nPermittedOrdinals, graph.size());
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("indexContext", indexContext)
+                          .toString();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        graph.close();
+    }
+
+    public boolean containsUnitVectors()
+    {
+        return graph.containsUnitVectors();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/V2VectorPostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v2/V2VectorPostingsWriter.java
new file mode 100644
index 000000000000..8d1b55555408
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/V2VectorPostingsWriter.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.function.IntUnaryOperator;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+
+import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import org.agrona.collections.Int2IntHashMap;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter;
+import org.apache.cassandra.index.sai.disk.vector.VectorPostings;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.utils.Pair;
+
+public class V2VectorPostingsWriter<T>
+{
+    // true if vectors rows are 1:1 (all vectors are associated with exactly 1 row, and each row has a non-null vector)
+    private final boolean oneToOne;
+    // the size of the post-cleanup graph (so NOT necessarily the same as the VectorValues size, which contains entries for obsoleted ordinals)
+    private final int graphSize;
+    // given a "new" ordinal (0..size), return the ordinal it corresponds to in the original graph and VectorValues
+    private final IntUnaryOperator newToOldMapper;
+
+    public V2VectorPostingsWriter(boolean oneToOne, int graphSize, IntUnaryOperator mapper) {
+        this.oneToOne = oneToOne;
+        this.graphSize = graphSize;
+        this.newToOldMapper = mapper;
+    }
+
+    public long writePostings(SequentialWriter writer,
+                              RandomAccessVectorValues vectorValues,
+                              Map<? extends VectorFloat<?>, ? extends VectorPostings<T>> postingsMap,
+                              Set<Integer> deletedOrdinals) throws IOException
+    {
+        writeDeletedOrdinals(writer, deletedOrdinals);
+        writeNodeOrdinalToRowIdMapping(writer, vectorValues, postingsMap);
+        writeRowIdToNodeOrdinalMapping(writer, vectorValues, postingsMap);
+
+        return writer.position();
+    }
+
+    private void writeDeletedOrdinals(SequentialWriter writer, Set<Integer> deletedOrdinals) throws IOException
+    {
+        if (oneToOne) {
+            assert deletedOrdinals.isEmpty();
+            // -1 indicates that fast mapping of ordinal to rowId can be used
+            writer.writeInt(-1);
+            return;
+        }
+
+        writer.writeInt(deletedOrdinals.size());
+        for (var ordinal : deletedOrdinals) {
+            writer.writeInt(ordinal);
+        }
+    }
+
+    public void writeNodeOrdinalToRowIdMapping(SequentialWriter writer,
+                                               RandomAccessVectorValues vectorValues,
+                                               Map<? extends VectorFloat<?>, ? extends VectorPostings<T>> postingsMap) throws IOException
+    {
+        long ordToRowOffset = writer.getOnDiskFilePointer();
+
+        // total number of vectors
+        writer.writeInt(graphSize);
+
+        // Write the offsets of the postings for each ordinal
+        var offsetsStartAt = ordToRowOffset + 4L + 8L * graphSize;
+        var nextOffset = offsetsStartAt;
+        for (var i = 0; i < graphSize; i++) {
+            // (ordinal is implied; don't need to write it)
+            writer.writeLong(nextOffset);
+            int postingListSize;
+            if (oneToOne)
+            {
+                postingListSize = 1;
+            }
+            else
+            {
+                var originalOrdinal = newToOldMapper.applyAsInt(i);
+                var rowIds = postingsMap.get(vectorValues.getVector(originalOrdinal)).getRowIds();
+                postingListSize = rowIds.size();
+            }
+            nextOffset += 4 + (postingListSize * 4L); // 4 bytes for size and 4 bytes for each integer in the list
+        }
+        assert writer.position() == offsetsStartAt : "writer.position()=" + writer.position() + " offsetsStartAt=" + offsetsStartAt;
+
+        // Write postings lists
+        for (var i = 0; i < graphSize; i++) {
+            if (oneToOne)
+            {
+                writer.writeInt(1);
+                writer.writeInt(i);
+            }
+            else
+            {
+                var originalOrdinal = newToOldMapper.applyAsInt(i);
+                var rowIds = postingsMap.get(vectorValues.getVector(originalOrdinal)).getRowIds();
+                writer.writeInt(rowIds.size());
+                for (int r = 0; r < rowIds.size(); r++)
+                    writer.writeInt(rowIds.getInt(r));
+            }
+        }
+        assert writer.position() == nextOffset;
+    }
+
+    public void writeRowIdToNodeOrdinalMapping(SequentialWriter writer,
+                                               RandomAccessVectorValues vectorValues,
+                                               Map<? extends VectorFloat<?>, ? extends VectorPostings<T>> postingsMap) throws IOException
+    {
+        long startOffset = writer.position();
+
+        if (oneToOne)
+        {
+            for (var i = 0; i < graphSize; i++)
+            {
+                writer.writeInt(i);
+                writer.writeInt(i);
+            }
+        }
+        else
+        {
+            // Collect all (rowId, vectorOrdinal) pairs
+            List<Pair<Integer, Integer>> pairs = new ArrayList<>();
+            for (var newOrdinal = 0; newOrdinal < graphSize; newOrdinal++) {
+                int oldOrdinal = newToOldMapper.applyAsInt(newOrdinal);
+                // if it's an on-disk Map then this is an expensive assert, only do it when in memory
+                if (postingsMap instanceof ConcurrentSkipListMap)
+                    assert postingsMap.get(vectorValues.getVector(oldOrdinal)).getOrdinal() == oldOrdinal;
+
+                var rowIds = postingsMap.get(vectorValues.getVector(oldOrdinal)).getRowIds();
+                for (int r = 0; r < rowIds.size(); r++)
+                    pairs.add(Pair.create(rowIds.getInt(r), newOrdinal));
+            }
+
+            // Sort the pairs by rowId
+            pairs.sort(Comparator.comparingInt(Pair::left));
+
+            // Write the pairs to the file
+            for (var pair : pairs) {
+                writer.writeInt(pair.left);
+                writer.writeInt(pair.right);
+            }
+        }
+
+        // write the position of the beginning of rowid -> ordinals mappings to the end
+        writer.writeLong(startOffset);
+    }
+
+    /**
+     * @return a map of vector ordinal to row id and the largest rowid, or null if the vectors are not 1:1 with rows
+     */
+    private static <T> Pair<BiMap<Integer, Integer>, Integer> buildOrdinalMap(Map<VectorFloat<?>, ? extends VectorPostings<T>> postingsMap)
+    {
+        BiMap<Integer, Integer> ordinalMap = HashBiMap.create();
+        int minRow = Integer.MAX_VALUE;
+        int maxRow = Integer.MIN_VALUE;
+        for (VectorPostings<T> vectorPostings : postingsMap.values())
+        {
+            if (vectorPostings.getRowIds().size() != 1)
+            {
+                // multiple rows associated with this vector
+                return null;
+            }
+            int rowId = vectorPostings.getRowIds().getInt(0);
+            int ordinal = vectorPostings.getOrdinal();
+            minRow = Math.min(minRow, rowId);
+            maxRow = Math.max(maxRow, rowId);
+            assert !ordinalMap.containsKey(ordinal); // vector <-> ordinal should be unique
+            ordinalMap.put(ordinal, rowId);
+        }
+
+        if (minRow != 0 || maxRow != postingsMap.values().size() - 1)
+        {
+            // not every row had a vector associated with it
+            return null;
+        }
+        return Pair.create(ordinalMap, maxRow);
+    }
+
+    public static <T> V5VectorPostingsWriter.RemappedPostings remapForMemtable(Map<VectorFloat<?>, ? extends VectorPostings<T>> postingsMap,
+                                                                               boolean containsDeletes)
+    {
+        var p = buildOrdinalMap(postingsMap);
+        int maxNewOrdinal = postingsMap.size() - 1; // no in-graph deletes in v2
+        if (p == null || containsDeletes)
+            return V5VectorPostingsWriter.createGenericIdentityMapping(postingsMap);
+
+        var ordinalMap = p.left;
+        var maxRow = p.right;
+        return new V5VectorPostingsWriter.RemappedPostings(V5VectorPostingsWriter.Structure.ONE_TO_ONE,
+                                                           maxNewOrdinal,
+                                                           maxRow,
+                                                           ordinalMap,
+                                                           new Int2IntHashMap(Integer.MIN_VALUE),
+                                                           new V5VectorPostingsWriter.BiMapMapper(maxNewOrdinal, ordinalMap));
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsMeta.java b/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsMeta.java
new file mode 100644
index 000000000000..4350eb564209
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsMeta.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2.sortedterms;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+
+/**
+ * Metadata produced by {@link SortedTermsWriter}, needed by {@link SortedTermsReader}.
+ */
+public class SortedTermsMeta
+{
+    public final long trieFP;
+    /** Number of terms */
+    public final long count;
+    public final int maxTermLength;
+
+    public SortedTermsMeta(IndexInput input) throws IOException
+    {
+        this.trieFP = input.readLong();
+        this.count = input.readLong();
+        this.maxTermLength = input.readInt();
+    }
+
+    public SortedTermsMeta(long trieFP, long count, int maxTermLength)
+    {
+        this.trieFP = trieFP;
+        this.count = count;
+        this.maxTermLength = maxTermLength;
+    }
+
+    public void write(IndexOutput output) throws IOException
+    {
+        output.writeLong(trieFP);
+        output.writeLong(count);
+        output.writeInt(maxTermLength);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsReader.java
new file mode 100644
index 000000000000..f8449dcca67e
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsReader.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2.sortedterms;
+
+import java.io.IOException;
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.NotThreadSafe;
+import javax.annotation.concurrent.ThreadSafe;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.MonotonicBlockPackedReader;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
+import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
+
+import static org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsWriter.TERMS_DICT_BLOCK_MASK;
+import static org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsWriter.TERMS_DICT_BLOCK_SHIFT;
+
+/**
+ * Provides read access to a sorted on-disk sequence of terms.
+ * <p>
+ * Offers the following features:
+ * <ul>
+ *     <li>forward iterating over all terms sequentially with a cursor</li>
+ *     <li>constant-time look up of the term at a given point id</li>
+ *     <li>log-time lookup of the point id of a term</li>
+ * </ul>
+ * <p>
+ * Care has been taken to make this structure as efficient as possible.
+ * Reading terms does not require allocating data heap buffers per each read operation.
+ * Only one term at a time is loaded to memory.
+ * Low complexity algorithms are used – a lookup of the term by point id is constant time,
+ * and a lookup of the point id by the term is logarithmic.
+ *
+ * <p>
+ * Because the blocks are prefix compressed, random access applies only to the locating the whole block.
+ * In order to jump to a concrete term inside the block, the block terms are iterated from the block beginning.
+ * Expect random access by {@link Cursor#seekToPointId(long)} to be slower
+ * than just moving to the next term with {@link Cursor#advance()}.
+ * <p>
+ * For documentation of the underlying on-disk data structures, see the package documentation.
+ *
+ * @see SortedTermsWriter
+ * @see org.apache.cassandra.index.sai.disk.v2.sortedterms
+ */
+@ThreadSafe
+public class SortedTermsReader
+{
+    private final FileHandle termsData;
+    private final SortedTermsMeta meta;
+    private final FileHandle termsTrie;
+    private final LongArray.Factory blockOffsetsFactory;
+
+    /**
+     * Creates a new reader based on its data components.
+     * <p>
+     * It does not own the components, so you must close them separately after you're done with the reader.
+     * @param termsData handle to the file with a sequence of prefix-compressed blocks
+     *                  each storing a fixed number of terms
+     * @param termsDataBlockOffsets handle to the file containing an encoded sequence of the file offsets pointing to the blocks
+     * @param termsTrie handle to the file storing the trie with the term-to-point-id mapping
+     * @param meta metadata object created earlier by the writer
+     * @param blockOffsetsMeta metadata object for the block offsets
+     */
+    public SortedTermsReader(@Nonnull FileHandle termsData,
+                             @Nonnull FileHandle termsDataBlockOffsets,
+                             @Nonnull FileHandle termsTrie,
+                             @Nonnull SortedTermsMeta meta,
+                             @Nonnull NumericValuesMeta blockOffsetsMeta) throws IOException
+    {
+        this.termsData = termsData;
+        this.termsTrie = termsTrie;
+        try (IndexInput trieInput = IndexInputReader.create(termsTrie))
+        {
+            SAICodecUtils.validate(trieInput);
+        }
+        this.meta = meta;
+        this.blockOffsetsFactory = new MonotonicBlockPackedReader(termsDataBlockOffsets, blockOffsetsMeta);
+    }
+
+    /**
+     * Returns the total number of terms.
+     */
+    public long count()
+    {
+        return meta.count;
+    }
+
+    /**
+     * Opens a cursor over the terms stored in the terms file.
+     * <p>
+     * This does not read any data yet.
+     * The cursor is initially positioned before the first item.
+     * <p>
+     * The cursor is to be used in a single thread.
+     * The cursor is valid as long this object hasn't been closed.
+     * You must close the cursor when you no longer need it.
+     */
+    public @Nonnull Cursor openCursor() throws IOException
+    {
+        return new Cursor(termsData, blockOffsetsFactory);
+    }
+
+    /**
+     * Allows reading the terms from the terms file.
+     * Can quickly seek to a random term by <code>pointId</code>.
+     * <p>
+     * This object is stateful and not thread safe.
+     * It maintains a position to the current term as well as a buffer that can hold one term.
+     */
+    @NotThreadSafe
+    public class Cursor implements AutoCloseable
+    {
+        private final IndexInputReader termsData;
+        private final long termsDataFp;
+        private final LongArray blockOffsets;
+
+        // The term the cursor currently points to. Initially empty.
+        private final BytesRef currentTerm;
+
+        // The point id the cursor currently points to. -1 means before the first item.
+        private long pointId = -1;
+
+        private TrieTermsDictionaryReader reader;
+
+        Cursor(FileHandle termsData, LongArray.Factory blockOffsetsFactory) throws IOException
+        {
+            try
+            {
+                this.termsData = IndexInputReader.create(termsData);
+                SAICodecUtils.validate(this.termsData);
+                this.termsDataFp = this.termsData.getFilePointer();
+                this.blockOffsets = new LongArray.DeferredLongArray(blockOffsetsFactory::open);
+                this.currentTerm = new BytesRef(Math.max(meta.maxTermLength, 0));  // maxTermLength can be negative if meta.count == 0
+                this.reader = new TrieTermsDictionaryReader(termsTrie.instantiateRebufferer(null), meta.trieFP, TypeUtil.BYTE_COMPARABLE_VERSION);
+            }
+            catch (Throwable t)
+            {
+                if (termsData != null)
+                    termsData.close();
+                throw t;
+            }
+        }
+
+        /**
+         * Returns the point id (ordinal) associated with the least term greater than or equal to the given term, or
+         * a negative value if there is no such term.
+         * @param term
+         * @return
+         */
+        public long ceiling(@Nonnull ByteComparable term)
+        {
+            Preconditions.checkNotNull(term, "term null");
+            return reader.ceiling(term);
+        }
+
+        /**
+         * Returns the point id (ordinal) of the target term or a negative value if there is no such term.
+         * Complexity of this operation is O(log n).
+         *
+         * @param term target term to lookup
+         */
+        public long getExactPointId(@Nonnull ByteComparable term)
+        {
+            Preconditions.checkNotNull(term, "term null");
+            return reader.exactMatch(term);
+        }
+
+        /**
+         * Returns the point id (ordinal) associated with the greatest term less than or equal to the given term, or
+         * a negative value if there is no such term.
+         * Complexity of this operation is O(log n).
+         *
+         * @param term target term to lookup
+         */
+        public long floor(@Nonnull ByteComparable term)
+        {
+            Preconditions.checkNotNull(term, "term null");
+            return reader.floor(term);
+        }
+
+        /**
+         * Returns the number of terms
+         */
+        public long count()
+        {
+            return SortedTermsReader.this.count();
+        }
+
+        /**
+         * Returns the current position of the cursor.
+         * Initially, before the first call to {@link Cursor#advance}, the cursor is positioned at -1.
+         * After reading all the items, the cursor is positioned at index one
+         * greater than the position of the last item.
+         */
+        public long pointId()
+        {
+            return pointId;
+        }
+
+        /**
+         * Returns the current term data as <code>ByteComparable</code> referencing the internal term buffer.
+         * The term data stored behind that reference is valid only until the next call to
+         * {@link Cursor#advance} or {@link Cursor#seekToPointId(long)}.
+         */
+        public @Nonnull ByteComparable term()
+        {
+            return ByteComparable.preencoded(reader.byteComparableVersion, currentTerm.bytes, currentTerm.offset, currentTerm.length);
+        }
+
+        /**
+         * Advances the cursor to the next term and reads it into the current term buffer.
+         * <p>
+         * If there are no more available terms, clears the term buffer and the cursor's position will point to the
+         * one behind the last item.
+         * <p>
+         * This method has constant time complexity.
+         *
+         * @return true if the cursor was advanced successfully, false if the end of file was reached
+         * @throws IOException if a read from the terms file fails
+         */
+        public boolean advance() throws IOException
+        {
+            if (pointId >= meta.count || ++pointId >= meta.count)
+            {
+                currentTerm.length = 0;
+                return false;
+            }
+
+            int prefixLength;
+            int suffixLength;
+            if ((pointId & TERMS_DICT_BLOCK_MASK) == 0L)
+            {
+                prefixLength = 0;
+                suffixLength = termsData.readVInt();
+            }
+            else
+            {
+                final int token = Byte.toUnsignedInt(termsData.readByte());
+                prefixLength = token & 0x0F;
+                suffixLength = 1 + (token >>> 4);
+                if (prefixLength == 15)
+                    prefixLength += termsData.readVInt();
+                if (suffixLength == 16)
+                    suffixLength += termsData.readVInt();
+            }
+
+            assert prefixLength + suffixLength <= meta.maxTermLength;
+            currentTerm.length = prefixLength + suffixLength;
+            termsData.readBytes(currentTerm.bytes, prefixLength, suffixLength);
+            return true;
+        }
+
+        /**
+         * Positions the cursor on the target point id and reads the term at target to the current term buffer.
+         * <p>
+         * It is allowed to position the cursor before the first item or after the last item;
+         * in these cases the internal buffer is cleared.
+         * <p>
+         * This method has constant complexity.
+         *
+         * @param target point id to lookup
+         * @throws IOException if a seek and read from the terms file fails
+         * @throws IndexOutOfBoundsException if the target point id is less than -1 or greater than {@link Cursor#count}.
+         */
+        public void seekToPointId(long target) throws IOException
+        {
+            if (target < -1 || target > meta.count)
+                throw new IndexOutOfBoundsException();
+
+            if (target == -1 || target == meta.count)
+            {
+                termsData.seek(termsDataFp);   // matters only if target is -1
+                pointId = target;
+                currentTerm.length = 0;
+            }
+            else
+            {
+                final long blockIndex = target >>> TERMS_DICT_BLOCK_SHIFT;
+                final long blockAddress = blockOffsets.get(blockIndex);
+                termsData.seek(blockAddress + termsDataFp);
+                pointId = (blockIndex << TERMS_DICT_BLOCK_SHIFT) - 1;
+                while (pointId < target)
+                {
+                    boolean advanced = advance();
+                    assert advanced : "unexpected eof";   // must return true because target is in range
+                }
+            }
+        }
+
+        /**
+         * Resets the cursor to its initial position before the first item.
+         */
+        public void reset() throws IOException
+        {
+            seekToPointId(-1);
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            blockOffsets.close();
+            termsData.close();
+            this.reader.close();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsWriter.java
new file mode 100644
index 000000000000..6c18f7979040
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsWriter.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2.sortedterms;
+
+import java.io.Closeable;
+import java.io.IOException;
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.base.Preconditions;
+
+import io.micrometer.core.lang.NonNull;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
+import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.tries.IncrementalTrieWriter;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.StringHelper;
+
+import static org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader.trieSerializer;
+
+/**
+ * Writes an ordered sequence of terms for use with {@link SortedTermsReader}.
+ * <p>
+ * Terms must be added in lexicographical ascending order.
+ * Terms can be of varying lengths.
+ *
+ * <p>
+ * Important implementation note: SAI blocked packed readers are slow,
+ * and Lucene MonotonicBlockPackedReader is slow.  Using them
+ * will cause this class to slow considerably.
+ *
+ * For documentation of the underlying on-disk data structures, see the package documentation.
+ *
+ * @see SortedTermsReader
+ * @see org.apache.cassandra.index.sai.disk.v2.sortedterms
+ */
+@NotThreadSafe
+public class SortedTermsWriter implements Closeable
+{
+    // The TERMS_DICT_ constants allow for quickly determining the id of the current block based on a point id
+    // or to check if we are exactly at the beginning of the block.
+    // Terms data are organized in blocks of (2 ^ TERMS_DICT_BLOCK_SHIFT) terms.
+    // The blocks should not be too small because they allow prefix compression of
+    // the terms except the first term in a block.
+    // The blocks should not be too large because we can't just ranfomly jump to the term inside the block,
+    // but we have to iterate through all the terms from the start of the block.
+    static final int TERMS_DICT_BLOCK_SHIFT = 4;
+    static final int TERMS_DICT_BLOCK_SIZE = 1 << TERMS_DICT_BLOCK_SHIFT;
+    static final int TERMS_DICT_BLOCK_MASK = TERMS_DICT_BLOCK_SIZE - 1;
+
+    static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16;
+
+    private final IncrementalTrieWriter<Long> trieWriter;
+    private final IndexOutputWriter trieOutput;
+    private final IndexOutput termsOutput;
+    private final NumericValuesWriter offsetsWriter;
+    private final String componentName;
+    private final MetadataWriter metadataWriter;
+
+    private BytesRefBuilder prevTerm = new BytesRefBuilder();
+    private BytesRefBuilder tempTerm = new BytesRefBuilder();
+
+    private final long bytesStartFP;
+
+    private int maxLength = -1;
+    private long pointId = 0;
+
+    /**
+     * Creates a new writer.
+     * <p>
+     * It does not own the components, so you must close the components by yourself
+     * after you're done with the writer.
+     *
+     * @param termsDataComponent component builder for the prefix-compressed terms data
+     * @param metadataWriter the MetadataWriter for storing the SortedTermsMeta
+     * @param termsDataBlockOffsets  where to write the offsets of each block of terms data
+     * @param trieComponent component where to write the trie that maps the terms to point ids
+     */
+    public SortedTermsWriter(@NonNull IndexComponent.ForWrite termsDataComponent,
+                             @NonNull MetadataWriter metadataWriter,
+                             @Nonnull NumericValuesWriter termsDataBlockOffsets,
+                             @Nonnull IndexComponent.ForWrite trieComponent) throws IOException
+    {
+        this.componentName = termsDataComponent.fileNamePart();
+        this.metadataWriter = metadataWriter;
+        this.trieOutput = trieComponent.openOutput();
+        SAICodecUtils.writeHeader(this.trieOutput);
+        this.trieWriter = IncrementalTrieWriter.open(trieSerializer, trieOutput.asSequentialWriter(), TypeUtil.BYTE_COMPARABLE_VERSION);
+        this.termsOutput = termsDataComponent.openOutput();
+        SAICodecUtils.writeHeader(termsOutput);
+        this.bytesStartFP = termsOutput.getFilePointer();
+        this.offsetsWriter = termsDataBlockOffsets;
+    }
+
+    /**
+     * Appends a term at the end of the sequence.
+     * Terms must be added in lexicographic order.
+     *
+     * @throws IOException if write to disk fails
+     * @throws IllegalArgumentException if the term is not greater than the previous added term
+     */
+    public void add(final @Nonnull ByteComparable term) throws IOException
+    {
+        tempTerm.clear();
+        copyBytes(term, tempTerm);
+
+        final BytesRef termRef = tempTerm.get();
+        final BytesRef prevTermRef = this.prevTerm.get();
+
+        Preconditions.checkArgument(prevTermRef.length == 0 || prevTermRef.compareTo(termRef) < 0,
+                                    "Terms must be added in lexicographic ascending order.");
+        writeTermData(termRef);
+        writeTermToTrie(term);
+
+        maxLength = Math.max(maxLength, termRef.length);
+        swapTempWithPrevious();
+        pointId++;
+    }
+
+    private void writeTermToTrie(ByteComparable term) throws IOException
+    {
+        trieWriter.add(term, pointId);
+    }
+
+    private void writeTermData(BytesRef term) throws IOException
+    {
+        if ((pointId & TERMS_DICT_BLOCK_MASK) == 0)
+        {
+            offsetsWriter.add(termsOutput.getFilePointer() - bytesStartFP);
+
+            termsOutput.writeVInt(term.length);
+            termsOutput.writeBytes(term.bytes, term.offset, term.length);
+        }
+        else
+        {
+            final int prefixLength = StringHelper.bytesDifference(prevTerm.get(), term);
+            final int suffixLength = term.length - prefixLength;
+            assert suffixLength > 0: "terms must be unique";
+
+            termsOutput.writeByte((byte) (Math.min(prefixLength, 15) | (Math.min(15, suffixLength - 1) << 4)));
+            if (prefixLength >= 15)
+                termsOutput.writeVInt(prefixLength - 15);
+            if (suffixLength >= 16)
+                termsOutput.writeVInt(suffixLength - 16);
+
+            termsOutput.writeBytes(term.bytes, term.offset + prefixLength, term.length - prefixLength);
+        }
+    }
+
+    /**
+     * Flushes any in-memory buffers to the output streams.
+     * Does not close the output streams.
+     * No more writes are allowed.
+     */
+    @Override
+    public void close() throws IOException
+    {
+        try (IndexOutput output = metadataWriter.builder(componentName))
+        {
+            final long trieFP = this.trieWriter.complete();
+            SAICodecUtils.writeFooter(trieOutput);
+            SAICodecUtils.writeFooter(termsOutput);
+            SortedTermsMeta sortedTermsMeta = new SortedTermsMeta(trieFP, pointId, maxLength);
+            sortedTermsMeta.write(output);
+        }
+        finally
+        {
+            FileUtils.closeQuietly(trieWriter, trieOutput, termsOutput, offsetsWriter);
+        }
+    }
+
+    /**
+     * Copies bytes from source to dest.
+     */
+    private void copyBytes(ByteComparable source, BytesRefBuilder dest)
+    {
+        ByteSource byteSource = source.asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION);
+        int val;
+        while ((val = byteSource.next()) != ByteSource.END_OF_STREAM)
+            dest.append((byte) val);
+    }
+
+    /**
+     * Swaps <code>this.temp</code> with <code>this.previous</code>.
+     * It is faster to swap the pointers instead of copying the data.
+     */
+    private void swapTempWithPrevious()
+    {
+        BytesRefBuilder temp = this.tempTerm;
+        this.tempTerm = this.prevTerm;
+        this.prevTerm = temp;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/package-info.java b/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/package-info.java
new file mode 100644
index 000000000000..d5f1cf9f3eef
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v2/sortedterms/package-info.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Space-efficient on-disk data structure for storing a sorted sequence of terms.
+ * Provides efficient lookup of terms by their point id, as well as locating them by contents.
+ * <p>
+ * All the code in the package uses the following teminology:
+ * <ul>
+ *     <li>Term: arbitrary data provided by the user as a bunch of bytes. Terms can be of variable length.</li>
+ *     <li>Point id: the ordinal position of a term in the sequence, 0-based.</li>
+ * </ul>
+ *
+ * Terms are stored in <code>ByteComparable</code> strictly ascending order.
+ * Duplicates are not allowed.
+ *
+ * <p>
+ * The structure is immutable, i.e. cannot be modified nor appended after writing to disk is completed.
+ * You build it by adding terms in the ascending order using
+ * {@link org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsWriter}.
+ * Once saved to disk, you can open it for lookups with
+ * {@link org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsReader}.
+ *
+ * <p>
+ * The data structure comprises of the following components, each stored in a separate file:
+ * <ul>
+ *     <li>terms data, organized as a sequence of prefix-compressed blocks each storing 16 terms</li>
+ *     <li>a monotonic list of file offsets of the blocks; this component allows to quickly locate the block
+ *     that contains the term with a given point id</li>
+ *     <li>a trie indexed by terms, with a long payload for the point id,
+ *     to quickly locate the point id of a term by the term contents
+ *     </li>
+ * </ul>
+ * </p>
+ *
+ * The implementation has been based on code from Lucene version 7.5 SortedDocValues.
+ * Prefix compression and bitpacking are used extensively to save space.
+ */
+package org.apache.cassandra.index.sai.disk.v2.sortedterms;
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v3/V3InvertedIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v3/V3InvertedIndexSearcher.java
new file mode 100644
index 000000000000..d62394673857
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v3/V3InvertedIndexSearcher.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v3;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.InvertedIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+
+/**
+ * The key override for this class is the use of {@link Version#CA}.
+ */
+class V3InvertedIndexSearcher extends InvertedIndexSearcher
+{
+    V3InvertedIndexSearcher(SSTableContext sstableContext,
+                            PerIndexFiles perIndexFiles,
+                            SegmentMetadata segmentMetadata,
+                            IndexContext indexContext) throws IOException
+    {
+        // We filter because the CA format wrote maps acording to a different order than their abstract type.
+        super(sstableContext, perIndexFiles, segmentMetadata, indexContext, Version.CA, true);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v3/V3OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v3/V3OnDiskFormat.java
new file mode 100644
index 000000000000..05789de225e7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v3/V3OnDiskFormat.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v3;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
+import org.apache.cassandra.index.sai.disk.v1.IndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v2.V2OnDiskFormat;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_ENABLE_EDGES_CACHE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_ENABLE_JVECTOR_DELETES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_ENABLE_LTM_CONSTRUCTION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_ENABLE_RERANK_FLOOR;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_JVECTOR_VERSION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_REDUCE_TOPK_ACROSS_SSTABLES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_WRITE_JVECTOR3_FORMAT;
+
+/**
+ * Different vector components compared to V2OnDiskFormat (supporting DiskANN/jvector instead of HNSW/lucene).
+ */
+public class V3OnDiskFormat extends V2OnDiskFormat
+{
+    public static final boolean REDUCE_TOPK_ACROSS_SSTABLES = SAI_REDUCE_TOPK_ACROSS_SSTABLES.getBoolean();
+    public static final boolean ENABLE_RERANK_FLOOR = SAI_ENABLE_RERANK_FLOOR.getBoolean();
+    public static final boolean ENABLE_EDGES_CACHE = SAI_ENABLE_EDGES_CACHE.getBoolean();
+    public static final boolean ENABLE_JVECTOR_DELETES = SAI_ENABLE_JVECTOR_DELETES.getBoolean();
+
+    public static volatile boolean WRITE_JVECTOR3_FORMAT = SAI_WRITE_JVECTOR3_FORMAT.getBoolean();
+    public static final boolean ENABLE_LTM_CONSTRUCTION = SAI_ENABLE_LTM_CONSTRUCTION.getBoolean();
+
+    // These are built to be backwards and forwards compatible. Not final only for testing.
+    public static int JVECTOR_VERSION = SAI_JVECTOR_VERSION.getInt();
+    static
+    {
+        // JVector 3 is not compatible with the latest jvector changes, so we fail fast if the config is enabled.
+        assert JVECTOR_VERSION != 3 : "JVector version 3 is no longer suppoerted";
+        assert !WRITE_JVECTOR3_FORMAT : "JVector version 3 is no longer suppoerted";
+    }
+
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    public static final V3OnDiskFormat instance = new V3OnDiskFormat();
+
+    public static final Set<IndexComponentType> VECTOR_COMPONENTS_V3 = EnumSet.of(IndexComponentType.COLUMN_COMPLETION_MARKER,
+                                                                                  IndexComponentType.META,
+                                                                                  IndexComponentType.PQ,
+                                                                                  IndexComponentType.TERMS_DATA,
+                                                                                  IndexComponentType.POSTING_LISTS);
+
+    private static final IndexFeatureSet v3IndexFeatureSet = new IndexFeatureSet()
+    {
+        @Override
+        public boolean isRowAware()
+        {
+            return true;
+        }
+
+        @Override
+        public boolean hasVectorIndexChecksum()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean hasTermsHistogram()
+        {
+            return false;
+        }
+    };
+
+    @Override
+    public IndexFeatureSet indexFeatureSet()
+    {
+        return v3IndexFeatureSet;
+    }
+
+    @Override
+    public IndexSearcher newIndexSearcher(SSTableContext sstableContext,
+                                          IndexContext indexContext,
+                                          PerIndexFiles indexFiles,
+                                          SegmentMetadata segmentMetadata) throws IOException
+    {
+        if (indexContext.isVector())
+            return new V3VectorIndexSearcher(sstableContext, indexFiles, segmentMetadata, indexContext);
+        if (indexContext.isLiteral())
+            return new V3InvertedIndexSearcher(sstableContext, indexFiles, segmentMetadata, indexContext);
+        return super.newIndexSearcher(sstableContext, indexContext, indexFiles, segmentMetadata);
+    }
+
+    @Override
+    public Set<IndexComponentType> perIndexComponentTypes(AbstractType<?> validator)
+    {
+        // VSTODO add checksums and actual validation
+        if (validator.isVector())
+            return VECTOR_COMPONENTS_V3;
+        return super.perIndexComponentTypes(validator);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v3/V3VectorIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v3/V3VectorIndexSearcher.java
new file mode 100644
index 000000000000..ff03a24e77d7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v3/V3VectorIndexSearcher.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v3;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v2.V2OnDiskOrdinalsMap;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher;
+import org.apache.cassandra.index.sai.disk.vector.CassandraDiskAnn;
+
+/**
+ * Executes ann search against the graph for an individual index segment.
+ */
+public class V3VectorIndexSearcher extends V2VectorIndexSearcher
+{
+    public V3VectorIndexSearcher(SSTableContext sstableContext,
+                                 PerIndexFiles perIndexFiles,
+                                 SegmentMetadata segmentMetadata,
+                                 IndexContext indexContext) throws IOException
+    {
+        super(sstableContext.primaryKeyMapFactory(),
+              perIndexFiles,
+              segmentMetadata,
+              indexContext,
+              new CassandraDiskAnn(sstableContext, segmentMetadata.componentMetadatas, perIndexFiles, indexContext, V2OnDiskOrdinalsMap::new));
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v4/V4InvertedIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v4/V4InvertedIndexSearcher.java
new file mode 100644
index 000000000000..f819b6e1ee6f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v4/V4InvertedIndexSearcher.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v4;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.InvertedIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+
+/**
+ * The key override for this class is the use of {@link Version#DB}, which allows us to skip filtering range results.
+ */
+class V4InvertedIndexSearcher extends InvertedIndexSearcher
+{
+    V4InvertedIndexSearcher(SSTableContext sstableContext,
+                            PerIndexFiles perIndexFiles,
+                            SegmentMetadata segmentMetadata,
+                            IndexContext indexContext) throws IOException
+    {
+        super(sstableContext, perIndexFiles, segmentMetadata, indexContext, segmentMetadata.version, false);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v4/V4OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v4/V4OnDiskFormat.java
new file mode 100644
index 000000000000..705f893c9fef
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v4/V4OnDiskFormat.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v4;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.v1.IndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+public class V4OnDiskFormat extends V3OnDiskFormat
+{
+    public static final V4OnDiskFormat instance = new V4OnDiskFormat();
+
+    @Override
+    public IndexSearcher newIndexSearcher(SSTableContext sstableContext,
+                                          IndexContext indexContext,
+                                          PerIndexFiles indexFiles,
+                                          SegmentMetadata segmentMetadata) throws IOException
+    {
+        if (indexContext.isVector())
+            return super.newIndexSearcher(sstableContext, indexContext, indexFiles, segmentMetadata);
+        if (indexContext.isLiteral())
+            return new V4InvertedIndexSearcher(sstableContext, indexFiles, segmentMetadata, indexContext);
+        return super.newIndexSearcher(sstableContext, indexContext, indexFiles, segmentMetadata);
+    }
+
+    @Override
+    public ByteComparable encodeForTrie(ByteBuffer input, AbstractType<?> type)
+    {
+        // Composite types use their individual type to ensure they sorted correctly in the trie so we can do
+        // range queries over entries.
+        return TypeUtil.isLiteral(type) && !TypeUtil.isComposite(type)
+               ? v -> ByteSource.preencoded(input)
+               : TypeUtil.asComparableBytes(input, type);
+    }
+
+    @Override
+    public ByteBuffer decodeFromTrie(ByteComparable value, AbstractType<?> type)
+    {
+        return TypeUtil.isLiteral(type) && !TypeUtil.isComposite(type)
+               ? ByteBuffer.wrap(ByteSourceInverse.readBytes(value.asComparableBytes(ByteComparable.Version.OSS41)))
+               : TypeUtil.fromComparableBytes(value, type, ByteComparable.Version.OSS41);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v5/V5OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v5/V5OnDiskFormat.java
new file mode 100644
index 000000000000..8b16ff5f55f9
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v5/V5OnDiskFormat.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v5;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.IndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v4.V4OnDiskFormat;
+
+public class V5OnDiskFormat extends V4OnDiskFormat
+{
+    public static final V5OnDiskFormat instance = new V5OnDiskFormat();
+
+    public static boolean writeV5VectorPostings()
+    {
+        return Version.latest().onOrAfter(Version.DC);
+    }
+
+    @Override
+    public IndexSearcher newIndexSearcher(SSTableContext sstableContext,
+                                          IndexContext indexContext,
+                                          PerIndexFiles indexFiles,
+                                          SegmentMetadata segmentMetadata) throws IOException
+    {
+        if (indexContext.isVector())
+            return new V5VectorIndexSearcher(sstableContext, indexFiles, segmentMetadata, indexContext);
+        return super.newIndexSearcher(sstableContext, indexContext, indexFiles, segmentMetadata);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v5/V5OnDiskOrdinalsMap.java b/src/java/org/apache/cassandra/index/sai/disk/v5/V5OnDiskOrdinalsMap.java
new file mode 100644
index 000000000000..677abff6450f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v5/V5OnDiskOrdinalsMap.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v5;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.PrimitiveIterator;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.index.sai.disk.vector.OnDiskOrdinalsMap;
+import org.apache.cassandra.index.sai.disk.vector.OrdinalsView;
+import org.apache.cassandra.index.sai.disk.vector.RowIdsView;
+import org.apache.cassandra.index.sai.utils.SingletonIntIterator;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+
+public class V5OnDiskOrdinalsMap implements OnDiskOrdinalsMap
+{
+    private static final Logger logger = LoggerFactory.getLogger(V5OnDiskOrdinalsMap.class);
+
+    private static final OneToOneRowIdsView ONE_TO_ONE_ROW_IDS_VIEW = new OneToOneRowIdsView();
+    private static final EmptyOrdinalsView EMPTY_ORDINALS_VIEW = new EmptyOrdinalsView();
+    private static final EmptyRowIdsView EMPTY_ROW_IDS_VIEW = new EmptyRowIdsView();
+
+    private final FileHandle fh;
+    private final long ordToRowOffset;
+    private final long segmentEnd;
+    private final int maxOrdinal;
+    private final int maxRowId;
+    private final long rowToOrdinalOffset;
+    @VisibleForTesting
+    final V5VectorPostingsWriter.Structure structure;
+
+    private final Supplier<OrdinalsView> ordinalsViewSupplier;
+    private final Supplier<RowIdsView> rowIdsViewSupplier;
+
+    // cached values for OneToMany structure
+    private Int2ObjectHashMap<int[]> extraRowsByOrdinal = null;
+    private int[] extraRowIds = null;
+    private int[] extraOrdinals = null;
+
+
+    public V5OnDiskOrdinalsMap(FileHandle fh, long segmentOffset, long segmentLength)
+    {
+        this.segmentEnd = segmentOffset + segmentLength;
+        this.fh = fh;
+        try (var reader = fh.createReader())
+        {
+            reader.seek(segmentOffset);
+            int magic = reader.readInt();
+            if (magic != V5VectorPostingsWriter.MAGIC)
+            {
+                throw new RuntimeException("Invalid magic number in V5OnDiskOrdinalsMap");
+            }
+            this.structure = V5VectorPostingsWriter.Structure.values()[reader.readInt()];
+            this.maxOrdinal = reader.readInt();
+            this.maxRowId = reader.readInt();
+            this.ordToRowOffset = reader.getFilePointer();
+            if (structure == V5VectorPostingsWriter.Structure.ONE_TO_ONE)
+            {
+                this.rowToOrdinalOffset = segmentEnd;
+            }
+            else
+            {
+                reader.seek(segmentEnd - 8);
+                this.rowToOrdinalOffset = reader.readLong();
+            }
+
+            if (maxOrdinal < 0)
+            {
+                this.rowIdsViewSupplier = () -> EMPTY_ROW_IDS_VIEW;
+                this.ordinalsViewSupplier = () -> EMPTY_ORDINALS_VIEW;
+            }
+            else if (structure == V5VectorPostingsWriter.Structure.ONE_TO_ONE)
+            {
+                this.rowIdsViewSupplier = () -> ONE_TO_ONE_ROW_IDS_VIEW;
+                this.ordinalsViewSupplier = () -> new OneToOneOrdinalsView(maxOrdinal + 1);
+            }
+            else if (structure == V5VectorPostingsWriter.Structure.ONE_TO_MANY)
+            {
+                cacheExtraRowIds(reader);
+                cacheExtraRowOrdinals(reader);
+                this.rowIdsViewSupplier = OneToManyRowIdsView::new;
+                this.ordinalsViewSupplier = OneToManyOrdinalsView::new;
+            }
+            else
+            {
+                this.rowIdsViewSupplier = GenericRowIdsView::new;
+                this.ordinalsViewSupplier = GenericOrdinalsView::new;
+            }
+
+            assert rowToOrdinalOffset <= segmentEnd : "rowOrdinalOffset " + rowToOrdinalOffset + " is not less than or equal to segmentEnd " + segmentEnd;
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException("Error initializing OnDiskOrdinalsMap at segment " + segmentOffset, e);
+        }
+    }
+
+    @Override
+    public V5VectorPostingsWriter.Structure getStructure()
+    {
+        return structure;
+    }
+
+    private void cacheExtraRowIds(RandomAccessReader reader) throws IOException
+    {
+        extraRowsByOrdinal = new Int2ObjectHashMap<>();
+        reader.seek(ordToRowOffset);
+        int entryCount = reader.readInt();
+        for (int i = 0; i < entryCount; i++)
+        {
+            int ordinal = reader.readInt();
+            int postingsSize = reader.readInt();
+            if (postingsSize > 0)
+                postingsSize++; // add the ordinal itself
+            int[] rowIds = new int[postingsSize];
+            if (postingsSize > 0)
+            {
+                rowIds[0] = ordinal;
+                for (int j = 1; j < postingsSize; j++)
+                    rowIds[j] = reader.readInt();
+            }
+            extraRowsByOrdinal.put(ordinal, rowIds);
+        }
+    }
+
+    private void cacheExtraRowOrdinals(RandomAccessReader reader) throws IOException
+    {
+        var extraRowIdsList = new IntArrayList();
+        var extraOrdinalsList = new IntArrayList();
+        reader.seek(rowToOrdinalOffset);
+        while (reader.getFilePointer() < segmentEnd - 8)
+        {
+            extraRowIdsList.add(reader.readInt());
+            extraOrdinalsList.add(reader.readInt());
+        }
+
+        extraRowIds = extraRowIdsList.toIntArray();
+        extraOrdinals = extraOrdinalsList.toIntArray();
+    }
+
+    public RowIdsView getRowIdsView()
+    {
+        return rowIdsViewSupplier.get();
+    }
+
+    private class GenericRowIdsView implements RowIdsView
+    {
+        RandomAccessReader reader = fh.createReader();
+
+        @Override
+        public PrimitiveIterator.OfInt getSegmentRowIdsMatching(int vectorOrdinal) throws IOException
+        {
+            Preconditions.checkArgument(vectorOrdinal <= maxOrdinal, "vectorOrdinal %s is out of bounds %s", vectorOrdinal, maxOrdinal);
+
+            // read index entry
+            try
+            {
+                reader.seek(ordToRowOffset + vectorOrdinal * 8L);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(String.format("Error seeking to index offset for ordinal %d with ordToRowOffset %d",
+                                                         vectorOrdinal, ordToRowOffset), e);
+            }
+            var offset = reader.readLong();
+            // seek to and read rowIds
+            try
+            {
+                reader.seek(offset);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(String.format("Error seeking to rowIds offset for ordinal %d with ordToRowOffset %d",
+                                                         vectorOrdinal, ordToRowOffset), e);
+            }
+            var postingsSize = reader.readInt();
+
+            // Optimize for the most common case
+            if (postingsSize == 1)
+                return new SingletonIntIterator(reader.readInt());
+
+            var rowIds = new int[postingsSize];
+            for (var i = 0; i < rowIds.length; i++)
+            {
+                rowIds[i] = reader.readInt();
+            }
+            return Arrays.stream(rowIds).iterator();
+        }
+
+        @Override
+        public void close()
+        {
+            reader.close();
+        }
+    }
+
+    public OrdinalsView getOrdinalsView()
+    {
+        return ordinalsViewSupplier.get();
+    }
+
+    @NotThreadSafe
+    private class GenericOrdinalsView implements OrdinalsView
+    {
+        RandomAccessReader reader = fh.createReader();
+
+        /**
+         * @return ordinal if given row id is found; otherwise return -1
+         * rowId must increase
+         */
+        @Override
+        public int getOrdinalForRowId(int rowId) throws IOException
+        {
+            long offset = rowToOrdinalOffset + (long) rowId * 8;
+            if (offset >= segmentEnd - 8)
+                return -1;
+
+            reader.seek(offset);
+            int foundRowId = reader.readInt();
+            assert foundRowId == rowId : "foundRowId=" + foundRowId + " instead of rowId=" + rowId;
+
+            return reader.readInt();
+        }
+
+        @Override
+        public void forEachOrdinalInRange(int startRowId, int endRowId, OrdinalConsumer consumer) throws IOException
+        {
+            long startOffset = max(rowToOrdinalOffset, rowToOrdinalOffset + (long) startRowId * 8);
+            if (startOffset >= segmentEnd - 8)
+                return;  // start rowid is larger than any rowId that has an associated vector ordinal
+
+            reader.seek(startOffset);
+
+            while (reader.getFilePointer() < segmentEnd - 8)
+            {
+                int rowId = reader.readInt();
+                int ordinal = reader.readInt();
+
+                if (rowId > endRowId)
+                    break;
+
+                if (ordinal != -1)
+                    consumer.accept(rowId, ordinal);
+            }
+        }
+
+        @Override
+        public void close()
+        {
+            reader.close();
+        }
+    }
+
+    public void close()
+    {
+        fh.close();
+    }
+
+    private class OneToManyRowIdsView implements RowIdsView
+    {
+        @Override
+        public PrimitiveIterator.OfInt getSegmentRowIdsMatching(int ordinal)
+        {
+            Preconditions.checkArgument(ordinal <= maxOrdinal, "vectorOrdinal %s is out of bounds %s", ordinal, maxOrdinal);
+
+            int[] rowIds = extraRowsByOrdinal.get(ordinal);
+            // no entry means there is just one rowid matching the ordinal
+            if (rowIds == null)
+                return new SingletonIntIterator(ordinal);
+            // zero-length entry means it's a hole
+            if (rowIds.length == 0)
+                return IntStream.empty().iterator();
+            // otherwise return the rowIds
+            return Arrays.stream(rowIds).iterator();
+        }
+
+        @Override
+        public void close()
+        {
+            // no-op
+        }
+    }
+
+    private class OneToManyOrdinalsView implements OrdinalsView {
+        @Override
+        public int getOrdinalForRowId(int rowId)
+        {
+            assert rowId >= 0 : rowId;
+            if (rowId > maxRowId) {
+                return -1;
+            }
+
+            int index = Arrays.binarySearch(extraRowIds, rowId);
+            if (index >= 0) {
+                // Found in extra rows
+                return extraOrdinals[index];
+            }
+
+            // If it's not an "extra" row then the ordinal is the same as the rowId
+            return rowId;
+        }
+
+        @Override
+        public void forEachOrdinalInRange(int startRowId, int endRowId, OrdinalConsumer consumer) throws IOException
+        {
+            int rawIndex = Arrays.binarySearch(extraRowIds, startRowId);
+            int extraIndex = rawIndex >= 0 ? rawIndex : -rawIndex - 1;
+            for (int rowId = max(0, startRowId); rowId <= min(endRowId, maxRowId); rowId++)
+            {
+                if (extraIndex < extraRowIds.length && extraRowIds[extraIndex] == rowId)
+                {
+                    consumer.accept(extraRowIds[extraIndex], extraOrdinals[extraIndex]);
+                    extraIndex++;
+                }
+                else
+                {
+                    consumer.accept(rowId, rowId);
+                }
+            }
+        }
+
+        @Override
+        public void close() {
+            // no-op
+        }
+    }
+
+    @Override
+    public long cachedBytesUsed()
+    {
+        if (structure != V5VectorPostingsWriter.Structure.ONE_TO_MANY) {
+            return 0;
+        }
+
+        long bytes = 0;
+        if (extraRowIds != null) {
+            bytes += extraRowIds.length * 4L;
+        }
+        if (extraOrdinals != null) {
+            bytes += extraOrdinals.length * 4L;
+        }
+        if (extraRowsByOrdinal != null) {
+            for (int[] rowIds : extraRowsByOrdinal.values()) {
+                bytes += rowIds.length * 4L;
+            }
+        }
+        return bytes;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v5/V5VectorIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v5/V5VectorIndexSearcher.java
new file mode 100644
index 000000000000..48634e33c6ef
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v5/V5VectorIndexSearcher.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v5;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher;
+import org.apache.cassandra.index.sai.disk.vector.CassandraDiskAnn;
+
+/**
+ * Executes ann search against the graph for an individual index segment.
+ */
+public class V5VectorIndexSearcher extends V2VectorIndexSearcher
+{
+    public V5VectorIndexSearcher(SSTableContext sstableContext,
+                                 PerIndexFiles perIndexFiles,
+                                 SegmentMetadata segmentMetadata,
+                                 IndexContext indexContext) throws IOException
+    {
+        // inherits from V2 instead of V3 because the difference between V5 and V3 is the OnDiskOrdinalsMap that they use
+        super(sstableContext.primaryKeyMapFactory(),
+              perIndexFiles,
+              segmentMetadata,
+              indexContext,
+              new CassandraDiskAnn(sstableContext, segmentMetadata.componentMetadatas, perIndexFiles, indexContext, V5OnDiskOrdinalsMap::new));
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v5/V5VectorPostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v5/V5VectorPostingsWriter.java
new file mode 100644
index 000000000000..27e2317b057f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v5/V5VectorPostingsWriter.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v5;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.IntPredicate;
+import java.util.function.IntUnaryOperator;
+import java.util.stream.IntStream;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
+import io.github.jbellis.jvector.graph.disk.OrdinalMapper;
+import io.github.jbellis.jvector.util.FixedBitSet;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import org.agrona.collections.Int2IntHashMap;
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.index.sai.disk.vector.VectorPostings;
+import org.apache.cassandra.io.util.SequentialWriter;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+
+public class V5VectorPostingsWriter<T>
+{
+    private static final Logger logger = LoggerFactory.getLogger(V5VectorPostingsWriter.class);
+
+    /**
+     * Write a one-to-many mapping if the number of "holes" in the resulting ordinal sequence
+     * is less than this fraction of the total rows.  Holes have two effects that make us not
+     * want to overuse them:
+     * (1) We read the list of rowids associated with the holes into memory
+     * (2) The holes make the terms component (the vector index) less cache-efficient
+     * <br>
+     * In the Cohere wikipedia dataset, we observe 0.014% vectors with multiple rows, so this
+     * almost two orders of magnitude higher than the observed rate of holes in the same dataset.
+     */
+    @VisibleForTesting
+    public static double GLOBAL_HOLES_ALLOWED = 0.01;
+
+    public static int MAGIC = 0x90571265; // POSTINGS
+
+    public enum Structure
+    {
+        /**
+         * The mapping from vector ordinals to row ids is a bijection, i.e. each vector has exactly one row associated
+         * with it and each row has exactly one vector associated with it.  No additional mappings need to be written,
+         * and reads can happen without consulting disk.
+         */
+        ONE_TO_ONE,
+
+        /**
+         * Every row has a vector and at least one vector has multiple rows. The ratio of rows without a unique vector
+         * to total rows is smaller than {@link #GLOBAL_HOLES_ALLOWED}.  Only special cases (where the row id
+         * cannot be mapped to the same vector ordinal) are written; since this is a small fraction of total
+         * rows, these special cases are read into memory and reads can happen without consulting disk.
+         */
+        ONE_TO_MANY,
+
+        /**
+         * Either:
+         * 1. There is at least one row without a vector, or
+         * 2. The mapping would be {@link #ONE_TO_MANY}, but the ratio of rows without a unique vector to total rows is larger
+         *    than {@link #GLOBAL_HOLES_ALLOWED}.
+         * Explicit mappings from each row id to vector ordinal and vice versa are written.  Reads must consult disk.
+         */
+        ZERO_OR_ONE_TO_MANY
+    }
+
+    private final RemappedPostings remappedPostings;
+
+    /**
+     * If Structure is ONE_TO_MANY then extraPostings should be the rowid -> ordinal map for the "extra" rows
+     * as determined by CassandraOnHeapGraph::buildOrdinalMap; otherwise it should be null
+     */
+    public V5VectorPostingsWriter(RemappedPostings remappedPostings)
+    {
+        this.remappedPostings = remappedPostings;
+    }
+
+    /**
+     * This method describes the mapping done during construction of the graph so that we can easily create
+     * an appropriate V5VectorPostingsWriter.  No ordinal remapping is performed because (V5) compaction writes
+     * vectors to disk as they are added to the graph, so there is no opportunity to reorder the way there is
+     * in a Memtable index.
+     */
+    public static RemappedPostings describeForCompaction(Structure structure, int graphSize, Map<VectorFloat<?>, VectorPostings.CompactionVectorPostings> postingsMap)
+    {
+        assert !postingsMap.isEmpty(); // flush+compact should skip writing an index component in this case
+
+        if (structure == Structure.ONE_TO_ONE)
+        {
+            return new RemappedPostings(Structure.ONE_TO_ONE,
+                                        graphSize - 1,
+                                        graphSize - 1,
+                                        null,
+                                        null,
+                                        new OrdinalMapper.IdentityMapper(graphSize - 1));
+        }
+
+        if (structure == Structure.ONE_TO_MANY)
+        {
+            // compute maxOldOrdinal, maxRow, and extraOrdinals from the postingsMap
+            int maxOldOrdinal = Integer.MIN_VALUE;
+            int maxRow = Integer.MIN_VALUE;
+            var extraOrdinals = new Int2IntHashMap(Integer.MIN_VALUE);
+            for (var entry : postingsMap.entrySet())
+            {
+                var postings = entry.getValue();
+                int ordinal = postings.getOrdinal();
+
+                maxOldOrdinal = Math.max(maxOldOrdinal, ordinal);
+                var rowIds = postings.getRowIds();
+                assert ordinal == rowIds.getInt(0); // synthetic ordinals not allowed in ONE_TO_MANY
+                for (int i = 0; i < rowIds.size(); i++)
+                {
+                    int rowId = rowIds.getInt(i);
+                    maxRow = Math.max(maxRow, rowId);
+                    if (i > 0)
+                        extraOrdinals.put(rowId, ordinal);
+                }
+            }
+
+            var skippedOrdinals = extraOrdinals.keySet();
+            return new RemappedPostings(Structure.ONE_TO_MANY,
+                                        maxOldOrdinal,
+                                        maxRow,
+                                        null,
+                                        extraOrdinals,
+                                        new OmissionAwareIdentityMapper(maxOldOrdinal, skippedOrdinals::contains));
+        }
+
+        assert structure == Structure.ZERO_OR_ONE_TO_MANY : structure;
+        return createGenericIdentityMapping(postingsMap);
+    }
+
+    public long writePostings(SequentialWriter writer,
+                              RandomAccessVectorValues vectorValues,
+                              Map<? extends VectorFloat<?>, ? extends VectorPostings<T>> postingsMap) throws IOException
+    {
+        var structure = remappedPostings.structure;
+
+        writer.writeInt(MAGIC);
+        writer.writeInt(structure.ordinal());
+        writer.writeInt(remappedPostings.maxNewOrdinal);
+        writer.writeInt(remappedPostings.maxRowId);
+
+        if (structure == Structure.ONE_TO_ONE || remappedPostings.maxNewOrdinal < 0)
+        {
+            // nothing more to do
+        }
+        else if (structure == Structure.ONE_TO_MANY)
+        {
+            writeOneToManyOrdinalMapping(writer);
+            writeOneToManyRowIdMapping(writer);
+        }
+        else
+        {
+            assert structure == Structure.ZERO_OR_ONE_TO_MANY;
+            writeGenericOrdinalToRowIdMapping(writer, vectorValues, postingsMap);
+            writeGenericRowIdMapping(writer, vectorValues, postingsMap);
+        }
+
+        return writer.position();
+    }
+
+    private void writeOneToManyOrdinalMapping(SequentialWriter writer) throws IOException
+    {
+        // make sure we're in the right branch
+        assert !remappedPostings.extraPostings.isEmpty();
+
+        // Create a map of (original) ordinals to their extra rowids
+        var ordinalToExtraRowIds = new Int2ObjectHashMap<IntArrayList>();
+        for (var entry : remappedPostings.extraPostings.entrySet()) {
+            int rowId = entry.getKey();
+            int ordinal = entry.getValue();
+            ordinalToExtraRowIds.computeIfAbsent(ordinal, k -> new IntArrayList()).add(rowId);
+        }
+
+        // Write the ordinals and their extra rowids
+        int holeCount = (int) IntStream.range(0, remappedPostings.maxNewOrdinal + 1)
+                                       .map(remappedPostings.ordinalMapper::newToOld)
+                                       .filter(i -> i == OrdinalMapper.OMITTED)
+                                       .count();
+        writer.writeInt(holeCount + ordinalToExtraRowIds.size());
+        int entries = 0;
+        for (int newOrdinal = 0; newOrdinal <= remappedPostings.maxNewOrdinal; newOrdinal++) {
+            // write the "holes" so they are not incorrectly associated with the corresponding rowId
+            int oldOrdinal = remappedPostings.ordinalMapper.newToOld(newOrdinal);
+            if (oldOrdinal == OrdinalMapper.OMITTED)
+            {
+                writer.writeInt(newOrdinal);
+                writer.writeInt(0);
+                entries++;
+                continue;
+            }
+
+            // write the ordinals with multiple rows
+            var extraRowIds = ordinalToExtraRowIds.get(oldOrdinal);
+            if (extraRowIds != null)
+            {
+                writer.writeInt(newOrdinal);
+                writer.writeInt(extraRowIds.size());
+                for (int rowId : extraRowIds) {
+                    writer.writeInt(rowId);
+                }
+                entries++;
+            }
+        }
+        assert entries == holeCount + ordinalToExtraRowIds.size();
+    }
+
+    private void writeOneToManyRowIdMapping(SequentialWriter writer) throws IOException
+    {
+        long startOffset = writer.position();
+
+        // make sure we're in the right branch
+        assert !remappedPostings.extraPostings.isEmpty();
+
+        // sort the extra rowids.  this boxes, but there isn't a good way to avoid that
+        var extraRowIds = remappedPostings.extraPostings.keySet().stream().sorted().mapToInt(i -> i).toArray();
+        // only write the extra postings, everything else can be determined from those
+        int lastExtraRowId = -1;
+        for (int i = 0; i < extraRowIds.length; i++)
+        {
+            int rowId = extraRowIds[i];
+            int originalOrdinal = remappedPostings.extraPostings.get(rowId);
+            writer.writeInt(rowId);
+            writer.writeInt(remappedPostings.ordinalMapper.oldToNew(originalOrdinal));
+            // validate that we do in fact have contiguous rowids in the non-extra mapping
+            assert IntStream.range(lastExtraRowId + 1, rowId)
+                            .allMatch(j -> remappedPostings.ordinalMapper.newToOld(j) != OrdinalMapper.OMITTED) : "Non-contiguous rowids found in non-extra mapping";
+            lastExtraRowId = rowId;
+        }
+
+        // Write the position of the beginning of rowid -> ordinals mappings to the end
+        writer.writeLong(startOffset);
+    }
+
+    // VSTODO add missing row information to remapping so we don't have to go through the vectorValues again
+    public void writeGenericOrdinalToRowIdMapping(SequentialWriter writer,
+                                                  RandomAccessVectorValues vectorValues,
+                                                  Map<? extends VectorFloat<?>, ? extends VectorPostings<T>> postingsMap) throws IOException
+    {
+        long ordToRowOffset = writer.getOnDiskFilePointer();
+
+        var newToOldMapper = (IntUnaryOperator) remappedPostings.ordinalMapper::newToOld;
+        int ordinalCount = remappedPostings.maxNewOrdinal + 1; // may include unmapped ordinals
+        // Write the offsets of the postings for each ordinal
+        var offsetsStartAt = ordToRowOffset + 8L * ordinalCount;
+        var nextOffset = offsetsStartAt;
+        for (var i = 0; i < ordinalCount; i++) {
+            // (ordinal is implied; don't need to write it)
+            writer.writeLong(nextOffset);
+            int originalOrdinal = newToOldMapper.applyAsInt(i);
+            int postingListSize;
+            if (originalOrdinal == OrdinalMapper.OMITTED)
+            {
+                assert remappedPostings.structure == Structure.ZERO_OR_ONE_TO_MANY;
+                postingListSize = 0;
+            }
+            else
+            {
+                var rowIds = postingsMap.get(vectorValues.getVector(originalOrdinal)).getRowIds();
+                postingListSize = rowIds.size();
+            }
+            nextOffset += 4 + (postingListSize * 4L); // 4 bytes for size and 4 bytes for each integer in the list
+        }
+        assert writer.position() == offsetsStartAt : "writer.position()=" + writer.position() + " offsetsStartAt=" + offsetsStartAt;
+
+        // Write postings lists
+        for (var i = 0; i < ordinalCount; i++) {
+            int originalOrdinal = newToOldMapper.applyAsInt(i);
+            if (originalOrdinal == OrdinalMapper.OMITTED)
+            {
+                assert remappedPostings.structure == Structure.ZERO_OR_ONE_TO_MANY;
+                writer.writeInt(0);
+                continue;
+            }
+            var rowIds = postingsMap.get(vectorValues.getVector(originalOrdinal)).getRowIds();
+            writer.writeInt(rowIds.size());
+            for (int r = 0; r < rowIds.size(); r++)
+                writer.writeInt(rowIds.getInt(r));
+        }
+        assert writer.position() == nextOffset;
+    }
+
+    public void writeGenericRowIdMapping(SequentialWriter writer,
+                                         RandomAccessVectorValues vectorValues,
+                                         Map<? extends VectorFloat<?>, ? extends VectorPostings<T>> postingsMap) throws IOException
+    {
+        long startOffset = writer.position();
+
+        // Create a Map of rowId -> ordinal
+        int maxRowId = -1;
+        var rowIdToOrdinalMap = new Int2IntHashMap(remappedPostings.maxNewOrdinal, 0.65f, OrdinalMapper.OMITTED);
+        for (int i = 0; i <= remappedPostings.maxNewOrdinal; i++) {
+            int ord = remappedPostings.ordinalMapper.newToOld(i);
+            if (ord == OrdinalMapper.OMITTED)
+                continue;
+
+            var rowIds = postingsMap.get(vectorValues.getVector(ord)).getRowIds();
+            for (int r = 0; r < rowIds.size(); r++)
+            {
+                var rowId = rowIds.getInt(r);
+                rowIdToOrdinalMap.put(rowId, i);
+                maxRowId = max(maxRowId, rowId);
+            }
+        }
+
+        // Write rowId -> ordinal mappings, filling in missing rowIds with -1
+        for (int currentRowId = 0; currentRowId <= maxRowId; currentRowId++) {
+            writer.writeInt(currentRowId);
+            if (rowIdToOrdinalMap.containsKey(currentRowId))
+                writer.writeInt(rowIdToOrdinalMap.get(currentRowId));
+            else
+                writer.writeInt(-1); // no corresponding ordinal
+        }
+
+        // write the position of the beginning of rowid -> ordinals mappings to the end
+        writer.writeLong(startOffset);
+    }
+
+    /**
+     * RemappedPostings is a
+     *   - BiMap of original vector ordinal to the first row id it is associated with
+     *   - Map of row id to original vector ordinal for rows that are NOT the first row associated with their vector
+     * <p>
+     * Example, using digits as ordianls and letters as row ids.  Postings map contains
+     * 0 -> B, C
+     * 1 -> A
+     * 2 -> D
+     * <p>
+     * The returned ordinalMap would be {0 <-> B, 1 <-> A, 2 <-> D} and the extraPostings would be {C -> 0}
+     */
+    public static class RemappedPostings
+    {
+        /** relationship of vector ordinals to row ids */
+        public final Structure structure;
+        /** the largest vector ordinal in the postings (inclusive) */
+        public final int maxNewOrdinal;
+        /** the largest rowId in the postings (inclusive) */
+        public final int maxRowId;
+        /** map from rowId to [original] vector ordinal */
+        @Nullable
+        private final Int2IntHashMap extraPostings;
+        /** public api */
+        public final OrdinalMapper ordinalMapper;
+
+        /** visible for V2VectorPostingsWriter.remapPostings, everyone else should use factory methods */
+        public RemappedPostings(Structure structure, int maxNewOrdinal, int maxRowId, BiMap<Integer, Integer> ordinalMap, Int2IntHashMap extraPostings, OrdinalMapper ordinalMapper)
+        {
+            this.structure = structure;
+            this.maxNewOrdinal = maxNewOrdinal;
+            this.maxRowId = maxRowId;
+            this.extraPostings = extraPostings;
+            this.ordinalMapper = ordinalMapper;
+        }
+    }
+
+    /**
+     * @see RemappedPostings
+     */
+    public static <T> RemappedPostings remapForMemtable(Map<VectorFloat<?>, ? extends VectorPostings<T>> postingsMap)
+    {
+        assert V5OnDiskFormat.writeV5VectorPostings();
+
+        BiMap<Integer, Integer> ordinalMap = HashBiMap.create();
+        Int2IntHashMap extraPostings = new Int2IntHashMap(Integer.MIN_VALUE);
+        int minRow = Integer.MAX_VALUE;
+        int maxRow = Integer.MIN_VALUE;
+        int maxNewOrdinal = Integer.MIN_VALUE;
+        int maxOldOrdinal = Integer.MIN_VALUE;
+        int totalRowsAssigned = 0;
+
+        // build the ordinalMap and extraPostings
+        for (var vectorPostings : postingsMap.values())
+        {
+            assert !vectorPostings.isEmpty(); // deleted vectors should be cleaned out before remapping
+            var a = vectorPostings.getRowIds().toIntArray();
+            Arrays.sort(a);
+            int rowId = a[0];
+            int oldOrdinal = vectorPostings.getOrdinal();
+            maxOldOrdinal = max(maxOldOrdinal, oldOrdinal);
+            minRow = min(minRow, rowId);
+            maxRow = max(maxRow, a[a.length - 1]);
+            assert !ordinalMap.containsKey(oldOrdinal); // vector <-> ordinal should be unique
+            ordinalMap.put(oldOrdinal, rowId);
+            maxNewOrdinal = max(maxNewOrdinal, rowId);
+            totalRowsAssigned += a.length; // all row ids should also be unique, but we can't easily check that
+            if (a.length > 1)
+            {
+                for (int i = 1; i < a.length; i++)
+                    extraPostings.put(a[i], oldOrdinal);
+            }
+        }
+        assert totalRowsAssigned == 0 || totalRowsAssigned <= maxRow + 1: "rowids are not unique -- " + totalRowsAssigned + " >= " + maxRow;
+
+        // derive the correct structure
+        Structure structure;
+        if (totalRowsAssigned > 0 && (minRow != 0 || totalRowsAssigned < maxRow + 1))
+        {
+            logger.debug("Not all rows are assigned vectors, cannot remap one-to-many");
+            structure = Structure.ZERO_OR_ONE_TO_MANY;
+        }
+        else
+        {
+            logger.debug("Remapped postings include {} unique vectors and {} 'extra' rows sharing them", ordinalMap.size(), extraPostings.size());
+            structure = extraPostings.isEmpty()
+                      ? Structure.ONE_TO_ONE
+                      : Structure.ONE_TO_MANY;
+            // override one-to-many to generic if there are too many holes
+            if (structure == Structure.ONE_TO_MANY && extraPostings.size() > max(1, GLOBAL_HOLES_ALLOWED * maxRow))
+                structure = Structure.ZERO_OR_ONE_TO_MANY;
+        }
+
+        // create the mapping
+        if (structure == Structure.ZERO_OR_ONE_TO_MANY)
+            return createGenericRenumberedMapping(ordinalMap.keySet(), maxOldOrdinal, maxRow);
+        var ordinalMapper = new BiMapMapper(maxNewOrdinal, ordinalMap);
+        return new RemappedPostings(structure, maxNewOrdinal, maxRow, ordinalMap, extraPostings, ordinalMapper);
+    }
+
+    /**
+     * return an exhaustive zero-to-many mapping with the live ordinals renumbered sequentially
+     */
+    private static RemappedPostings createGenericRenumberedMapping(Set<Integer> liveOrdinals, int maxOldOrdinal, int maxRow)
+    {
+        var oldToNew = new Int2IntHashMap(maxOldOrdinal, 0.65f, Integer.MIN_VALUE);
+        int nextOrdinal = 0;
+        for (int i = 0; i <= maxOldOrdinal; i++) {
+            if (liveOrdinals.contains(i))
+                oldToNew.put(i, nextOrdinal++);
+        }
+        return new RemappedPostings(Structure.ZERO_OR_ONE_TO_MANY,
+                                    nextOrdinal - 1,
+                                    maxRow,
+                                    null,
+                                    null,
+                                    new OrdinalMapper.MapMapper(oldToNew));
+    }
+
+    /**
+     * return an exhaustive zero-to-many mapping with no renumbering
+     */
+    public static <T> RemappedPostings createGenericIdentityMapping(Map<VectorFloat<?>, ? extends VectorPostings<T>> postingsMap)
+    {
+        var maxOldOrdinal = postingsMap.values().stream().mapToInt(VectorPostings::getOrdinal).max().orElseThrow();
+        int maxRow = postingsMap.values().stream().flatMap(p -> p.getRowIds().stream()).mapToInt(i -> i).max().orElseThrow();
+        var presentOrdinals = new FixedBitSet(maxOldOrdinal + 1);
+        for (var entry : postingsMap.entrySet())
+            presentOrdinals.set(entry.getValue().getOrdinal());
+        return new RemappedPostings(Structure.ZERO_OR_ONE_TO_MANY,
+                                    maxOldOrdinal,
+                                    maxRow,
+                                    null,
+                                    null,
+                                    new OmissionAwareIdentityMapper(maxOldOrdinal, i -> !presentOrdinals.get(i)));
+    }
+
+    public static class BiMapMapper implements OrdinalMapper
+    {
+        private final int maxOrdinal;
+        private final BiMap<Integer, Integer> ordinalMap;
+
+        public BiMapMapper(int maxNewOrdinal, BiMap<Integer, Integer> ordinalMap)
+        {
+            this.maxOrdinal = maxNewOrdinal;
+            this.ordinalMap = ordinalMap;
+        }
+
+        @Override
+        public int maxOrdinal()
+        {
+            return maxOrdinal;
+        }
+
+        @Override
+        public int oldToNew(int i)
+        {
+            return ordinalMap.get(i);
+        }
+
+        @Override
+        public int newToOld(int i)
+        {
+            return ordinalMap.inverse().getOrDefault(i, OMITTED);
+        }
+    }
+
+    private static class OmissionAwareIdentityMapper implements OrdinalMapper
+    {
+        private final int maxVectorOrdinal;
+        private final IntPredicate toSkip;
+
+        public OmissionAwareIdentityMapper(int maxVectorOrdinal, IntPredicate toSkip)
+        {
+            this.maxVectorOrdinal = maxVectorOrdinal;
+            this.toSkip = toSkip;
+        }
+
+        @Override
+        public int maxOrdinal()
+        {
+            return maxVectorOrdinal;
+        }
+
+        @Override
+        public int oldToNew(int i)
+        {
+            return i;
+        }
+
+        @Override
+        public int newToOld(int i)
+        {
+            return toSkip.test(i) ? OrdinalMapper.OMITTED : i;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v6/TermsDistribution.java b/src/java/org/apache/cassandra/index/sai/disk/v6/TermsDistribution.java
new file mode 100644
index 000000000000..05dae88387d7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v6/TermsDistribution.java
@@ -0,0 +1,586 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v6;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.PriorityQueue;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.Immutable;
+import javax.annotation.concurrent.NotThreadSafe;
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.NumberType;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+/**
+ * Approximates a statistical distribution of term values in a sstable index segment.
+ * It is used to quickly estimate how many rows match a given term value or a range of values,
+ * without performing the search using the index (which would be more costly).
+ * <p>
+ * Comprises a histogram and a most frequent term table.
+ * <p>
+ * To build instances of this class, use the nested {@link Builder} class.
+ *
+ * @see SegmentMetadata
+ */
+@ThreadSafe
+@Immutable
+public class TermsDistribution
+{
+    // Special virtual bucket placed before all the other buckets of the histogram.
+    // Can be considered a bucket at index -1. The existence of this instance allows us to never return null
+    // when looking up the bucket by an index and simplifies the code.
+    private static final Bucket MIN_BUCKET = new Bucket(null, 0, 0);
+
+    private static final int MAGIC = 0xd57a75;  // STATS ;)
+
+    public final AbstractType<?> termType;
+    public final Version indexVersion;
+    public final ByteComparable.Version byteComparableVersion;
+
+    final ByteComparable minTerm;
+    final ByteComparable maxTerm;
+    final List<Bucket> histogram;
+    final NavigableMap<ByteComparable, Long> mostFrequentTerms;
+
+    final long numPoints;
+    final long numRows;
+
+    private TermsDistribution(AbstractType<?> termType,
+                              List<Bucket> histogram,
+                              NavigableMap<ByteComparable, Long> mostFrequentTerms,
+                              Version indexVersion,
+                              ByteComparable.Version byteComparableVersion)
+    {
+        this.termType = termType;
+        this.indexVersion = indexVersion;
+        this.byteComparableVersion = byteComparableVersion;
+        this.histogram = histogram;
+        this.mostFrequentTerms = mostFrequentTerms;
+
+        this.numRows = histogram.isEmpty() ? 0 : histogram.get(histogram.size() - 1).cumulativeRowCount;
+        this.numPoints = histogram.isEmpty() ? 0 : histogram.get(histogram.size() - 1).cumulativePointCount;
+        this.minTerm = histogram.isEmpty() ? null : histogram.get(0).term;
+        this.maxTerm = histogram.isEmpty() ? null : histogram.get(histogram.size() - 1).term;
+    }
+
+    /**
+     * Estimates the number of values equal to the given term.
+     *
+     * @param term term encoded as byte-comparable the same way as stored by the index on-disk
+     */
+    public long estimateNumRowsMatchingExact(ByteComparable term)
+    {
+        Long count = mostFrequentTerms.get(term);
+        if (count != null)
+            return count;
+
+        int index = indexOfBucketContaining(term);
+        Bucket low = getBucket(index - 1);
+        Bucket high = getBucket(index);
+
+        // The histogram buckets include all most frequent terms,
+        // but if we're here, we know our term is *not* any of the frequent values.
+        // Therefore, we should subtract frequent values to get better precision:
+        var mft = mostFrequentTermsInRange(low.term, high.term);
+
+        long points = high.cumulativePointCount - low.cumulativePointCount - mft.size();
+        long rows = high.cumulativeRowCount - low.cumulativeRowCount - sumValues(mft);
+        return rows == 0 ? 0 : Math.round((double) rows / points);
+    }
+
+    /**
+     * Estimates the number of rows with a value in given range.
+     * Allows to specify inclusiveness/exclusiveness of bounds.
+     * Bounds must be encoded as byte-comparable the same way as stored by the index on-disk.
+     */
+    public long estimateNumRowsInRange(ByteComparable min, boolean minInclusive, ByteComparable max, boolean maxInclusive)
+    {
+        long rowCount = estimateNumRowsInRange(min, max);
+
+        if (minInclusive && min != null)
+            rowCount += estimateNumRowsMatchingExact(min);
+        if (!maxInclusive && max != null)
+            rowCount = Math.max(0, rowCount - estimateNumRowsMatchingExact(max));
+
+        return rowCount;
+    }
+
+    /**
+     * Estimates the number of rows with a value in given range.
+     * Bounds must be encoded as byte-comparable the same way as stored by the index on-disk.
+     *
+     * @param min exclusive minimum bound
+     * @param max inclusive maximum bound
+     */
+    public long estimateNumRowsInRange(ByteComparable min, ByteComparable max)
+    {
+        Bucket low = (min != null) ? interpolate(min) : getBucket(-1);
+        Bucket high = (max != null) ? interpolate(max) : getBucket(histogram.size());
+        return Math.max(0, high.cumulativeRowCount - low.cumulativeRowCount);
+    }
+
+    /**
+     * Returns cumulative point count and cumulative row count for given term
+     * by linear interpolation of two adjacent histogram buckets.
+     * <p>
+     * The information from the most frequent terms map is also included,
+     * so if any of the frequent terms are lower or equal to the given term, their
+     * row counts will be also added.
+     * <p>
+     *
+     * Example - Let's assume the following histogram:
+     * <pre>
+     * bucket index  | term      |  cumulativePointCount  |  cumulativeRowCount
+     * --------------+-----------+------------------------+----------------------
+     * -1            |  null     |                     0  |                   0
+     *  0            |  "2.0"    |                   100  |               10000
+     *  1            |  "3.0"    |                   140  |               20000
+     * </pre>
+     * The results of calling this function are as follows:
+     * <pre>
+     * interpolate("1.0") = Bucket("1.0", 0, 0)
+     * interpolate("1.9") = Bucket("1.9", 0, 0)
+     * interpolate("2.0") = Bucket("2.0", 100, 10000)
+     * interpolate("2.5") = Bucket("2.5", 120, 15000)
+     * interpolate("3.0") = Bucket("3.0", 140, 20000)
+     * interpolate("4.0") = Bucket("4.0", 140, 20000)
+     * </pre>
+     */
+    private @Nonnull Bucket interpolate(@Nonnull ByteComparable term)
+    {
+        int bucketIndex = indexOfBucketContaining(term);
+        Bucket bucket = getBucket(bucketIndex);
+        Bucket prevBucket = getBucket(bucketIndex - 1);
+
+        if (prevBucket.term == null)
+            return new Bucket(term, bucket.cumulativePointCount, bucket.cumulativeRowCount);
+
+        ByteComparable bucketMinTerm = prevBucket.term;
+        ByteComparable bucketMaxTerm = bucket.term;
+
+        BigDecimal bucketMinValue = toBigDecimal(bucketMinTerm);
+        BigDecimal bucketMaxValue = toBigDecimal(bucketMaxTerm);
+
+        // Estimate the fraction of the bucket on the left side of the term.
+        // We assume terms are distributed evenly.
+        BigDecimal termValue = toBigDecimal(term).min(bucketMaxValue).max(bucketMinValue);
+        double termDistance = termValue.subtract(bucketMinValue).doubleValue();
+        double bucketSize = bucketMaxValue.subtract(bucketMinValue).doubleValue();
+
+        // Edge case: this can theoretically happen if our big decimals have insufficient resolution
+        // to distinguish terms. If we didn't return early in this case,
+        // the later interpolation logic would divide by 0.
+        if (bucketSize < Double.MIN_NORMAL)
+            return new Bucket(term, bucket.cumulativePointCount, bucket.cumulativeRowCount);
+
+        double fraction = termDistance / bucketSize;
+        assert fraction >= 0.0 && fraction <= 1.0: "Invalid fraction value: " + fraction;
+
+        // Total number of points and rows in this bucket:
+        long pointCount = bucket.cumulativePointCount - prevBucket.cumulativePointCount;
+        long rowCount = bucket.cumulativeRowCount - prevBucket.cumulativeRowCount;
+
+        // We need those to include precise information about most frequent terms in the calculation.
+        // For most frequent terms we know the exact number of rows, so if we're matching any most frequent
+        // terms, those will be added at the end to the final row count estimate.
+        SortedMap<ByteComparable, Long> bucketMft = mostFrequentTermsInRange(prevBucket.term, bucket.term);
+        SortedMap<ByteComparable, Long> matchedMft = mostFrequentTermsInRange(prevBucket.term, term);
+        long matchedMftPointCount = matchedMft.size();
+        long matchedMftRowCount = sumValues(matchedMft);
+
+        // We likely don't have the information on all the points in the MFT table.
+        // Compute the average number of rows per point for all the non-MFT points, that is
+        // as if all the most frequent terms didn't exist.
+        // Then we'll multiply this value by the number of matching non-MFT points to get
+        // a reasonable row count estimate for the non-MFT points.
+        long nonMftPointCount = pointCount - bucketMft.size();
+        long nonMftRowCount = rowCount - sumValues(bucketMft);
+        assert nonMftPointCount >= 0 : "point count cannot be negative";
+        assert nonMftRowCount >= 0 : "row count cannot be negative";
+        double rowsPerPoint = nonMftPointCount == 0 ? 0.0 : (double) nonMftRowCount / nonMftPointCount;
+
+        // We assume points are distributed evenly; therefore we use total pointCount here:
+        double matchedPointCount = fraction * pointCount;
+
+        double matchedNonMftRowCount = Math.max(0.0, matchedPointCount - matchedMftPointCount) * rowsPerPoint;
+        double matchedRowCount = matchedNonMftRowCount + matchedMftRowCount;
+
+        long cumulativePointCount = prevBucket.cumulativePointCount + Math.round(matchedPointCount);
+        long cumulativeRowCount = prevBucket.cumulativeRowCount + Math.round(matchedRowCount);
+        return new Bucket(term, cumulativePointCount, cumulativeRowCount);
+    }
+
+
+    /**
+     * @see #toBigDecimal(ByteComparable, AbstractType, Version, ByteComparable.Version)
+     */
+    private BigDecimal toBigDecimal(ByteComparable value)
+    {
+        return toBigDecimal(value, termType, indexVersion, byteComparableVersion);
+    }
+
+    /**
+     * Converts the term value stored in the index to a big decimal value. Preserves order.
+     * If the type represents a number, the correspondence is linear.
+     * For non-number types, it reinterprets a bytecomparable serialization as a number,
+     * so it is not necessarily linear, but still preserves the order.
+     */
+    public static BigDecimal toBigDecimal(ByteComparable value,
+                                          AbstractType<?> termType,
+                                          Version indexVersion,
+                                          ByteComparable.Version byteComparableVersion)
+    {
+        if (termType instanceof NumberType)
+        {
+            // For numbers we decode the number back to the raw C* representation and then convert it to BigDecimal
+            var numberType = (NumberType<?>) termType;
+            var saiEncoded = indexVersion.onDiskFormat().decodeFromTrie(value, termType);
+            var raw = TypeUtil.decode(saiEncoded, termType);
+            return DecimalType.instance.toBigDecimal(numberType.compose(raw));
+        }
+
+        // For non numbers we just reinterpret the bytecomparable representation as decimal of fixed width.
+        // Therefore, we don't need to decode anything.
+        byte[] fixedLengthBytes = Arrays.copyOf(ByteSourceInverse.readBytes(value.asComparableBytes(byteComparableVersion)), 20);
+        return new BigDecimal(new BigInteger(fixedLengthBytes));
+    }
+
+    /**
+     * Finds the bucket at given index.
+     * Saturates at edges, so never returns null.
+     * If index is negative, returns {@link this#MIN_BUCKET}.
+     * If index >= histogram.size(), returns the last (highest) bucket.
+     */
+    private @Nonnull Bucket getBucket(int index)
+    {
+        if (index < 0 || histogram.isEmpty())
+            return MIN_BUCKET;
+        if (index >= histogram.size())
+            return histogram.get(histogram.size() - 1);
+
+        return histogram.get(index);
+    }
+
+    /**
+     * Returns the index of the highest bucket whose term value is equal or greater than the given value.
+     * If the value is lower than {@link this#minTerm}, returns -1.
+     * If the value is higher than {@link this#maxTerm}, returns {@code histogram.size()}.
+     */
+    private int indexOfBucketContaining(@Nonnull ByteComparable b)
+    {
+        Bucket needle = new Bucket(b, 0, 0);
+        int index = Collections.binarySearch(histogram, needle, (b1, b2) -> ByteComparable.compare(b1.term, b2.term, byteComparableVersion));
+        return (index >= -1) ? index : -(index + 1);
+    }
+
+    /**
+     * Helper function to return the sum of values in a map
+     */
+    private static long sumValues(Map<?, Long> map)
+    {
+        return map.values().stream().mapToLong(Long::longValue).sum();
+    }
+
+    /**
+     * Returns a subtree of {@code mostFrequentTerms} map with values between given range.
+     * A null term means a term before the lowest term.
+     *
+     * @param min exclusive lower bound
+     * @param max inclusive upper bound
+     */
+    private SortedMap<ByteComparable, Long>  mostFrequentTermsInRange(@Nullable ByteComparable min, @Nullable ByteComparable max)
+    {
+        if (max == null)
+            return Collections.emptySortedMap();
+        if (min == null)
+            return mostFrequentTerms.headMap(max);
+
+        return mostFrequentTerms.subMap(min, false, max, true);
+    }
+
+    public void write(IndexOutput out) throws IOException
+    {
+        out.writeInt(MAGIC);
+
+        // Reserved for future use.
+        // Writing a few zeroes doesn't cost us much, and we could use those for flags or other important
+        // stuff in the future, so we can keep backwards compatibility between minor index versions
+        out.writeLong(0);
+        out.writeLong(0);
+        out.writeLong(0);
+        out.writeLong(0);
+
+        out.writeString(indexVersion.toString());
+        out.writeString(byteComparableVersion.toString());
+        out.writeShort((short) histogram.size());
+        for (Bucket b : histogram)
+        {
+            var term = ByteBuffer.wrap(b.term.asByteComparableArray(byteComparableVersion));
+            out.writeBytes(term);
+            out.writeVLong(b.cumulativePointCount);
+            out.writeVLong(b.cumulativeRowCount);
+        }
+        out.writeShort((short) mostFrequentTerms.size());
+        for (Map.Entry<ByteComparable, Long> entry : mostFrequentTerms.entrySet())
+        {
+            var term = ByteBuffer.wrap(entry.getKey().asByteComparableArray(byteComparableVersion));
+            out.writeBytes(term);
+            out.writeVLong(entry.getValue());
+        }
+    }
+
+    public static TermsDistribution read(IndexInput input, AbstractType<?> termType) throws IOException
+    {
+        long magic = input.readInt();
+        if (magic != MAGIC)
+            throw new IOException(String.format(
+                "Invalid TermsDistribution header. Expected MAGIC = 0x%08x but read 0x%08x instead", MAGIC, magic));
+
+        input.readLong();  // reserved
+        input.readLong();  // reserved
+        input.readLong();  // reserved
+        input.readLong();  // reserved
+
+        Version indexVersion = decodeIndexVersion(input.readString());
+        ByteComparable.Version bcVersion = decodeByteComparableVersion(input.readString());
+
+        int bucketCount = input.readShort();
+        if (bucketCount < 0)
+            throw new IOException("Number of buckets cannot be negative: " + bucketCount);
+
+        List<Bucket> buckets = new ArrayList<>(bucketCount);
+        for (int i = 0; i < bucketCount; i++)
+        {
+            ByteBuffer termBytes = input.readBytes();
+            ByteComparable term = ByteComparable.preencoded(bcVersion, termBytes);
+            long cumulativePointCount = input.readVLong();
+            long cumulativeRowCount = input.readVLong();
+            buckets.add(new Bucket(term, cumulativePointCount, cumulativeRowCount));
+        }
+
+        int mostFrequentTermsCount = input.readShort();
+        if (mostFrequentTermsCount < 0)
+            throw new IOException("Number of most frequent terms cannot be negative: " + mostFrequentTermsCount);
+
+        NavigableMap<ByteComparable, Long> mostFrequentTerms = new TreeMap<>((b1, b2) -> ByteComparable.compare(b1, b2, bcVersion));
+        for (int i = 0; i < mostFrequentTermsCount; i++)
+        {
+            ByteBuffer termBytes = input.readBytes();
+            ByteComparable term = ByteComparable.preencoded(bcVersion, termBytes);
+            long rowCount = input.readVLong();
+            mostFrequentTerms.put(term, rowCount);
+        }
+
+        return new TermsDistribution(termType, buckets, mostFrequentTerms, indexVersion, bcVersion);
+    }
+
+    private static ByteComparable.Version decodeByteComparableVersion(String versionStr) throws IOException
+    {
+        try
+        {
+            return ByteComparable.Version.valueOf(versionStr);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new IOException("Unrecognized ByteComparable version " + versionStr);
+        }
+    }
+
+    private static Version decodeIndexVersion(String versionStr) throws IOException
+    {
+        try
+        {
+            return Version.parse(versionStr);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new IOException("Unrecognized index version " + versionStr);
+        }
+    }
+
+
+    @NotThreadSafe
+    public static class Builder
+    {
+        final AbstractType<?> termType;
+        final ByteComparable.Version byteComparableVersion;
+        final int histogramSize;
+        final int mostFrequentTermsTableSize;
+
+        long maxRowsPerBucket;
+
+        List<Bucket> buckets = new ArrayList<>();
+        PriorityQueue<Point> mostFrequentTerms = new PriorityQueue<>();
+
+        ByteComparable lastTerm;
+        long cumulativePointCount;
+        long cumulativeRowCount;
+
+        public Builder(AbstractType<?> termType,
+                       ByteComparable.Version byteComparableVersion,
+                       int histogramSize,
+                       int mostFrequentTermsTableSize)
+        {
+            this.termType = termType;
+            this.byteComparableVersion = byteComparableVersion;
+            this.histogramSize = histogramSize;
+            this.mostFrequentTermsTableSize = mostFrequentTermsTableSize;
+
+            // Let's start with adding buckets for every point.
+            // This will be corrected to a higher value once the histogram gets too large and we'll do shrinking.
+            this.maxRowsPerBucket = 1;
+        }
+
+        /**
+         * Adds a point to the histogram.
+         * Terms must be added in ascending order of term values matching the order of the index.
+         * Terms must be encoded as byte-comparable, because they are compared lexicographically by unsigned bytes.
+         * If the order is not preserved, the behavior is undefined.
+         *
+         * @param term encoded term
+         */
+        public void add(ByteComparable term, long rowCount)
+        {
+            mostFrequentTerms.add(new Point(term, rowCount));
+            if (mostFrequentTerms.size() > mostFrequentTermsTableSize)
+                mostFrequentTerms.poll();
+
+            cumulativePointCount += 1;
+            cumulativeRowCount += rowCount;
+            lastTerm = term;
+
+            if (buckets.isEmpty() || cumulativeRowCount > buckets.get(buckets.size() - 1).cumulativeRowCount + maxRowsPerBucket)
+            {
+                buckets.add(new Bucket(lastTerm, cumulativePointCount, cumulativeRowCount));
+                lastTerm = null;
+
+                if (buckets.size() > histogramSize * 2)
+                    shrink();
+            }
+        }
+
+        public TermsDistribution build()
+        {
+            if (lastTerm != null)
+                buckets.add(new Bucket(lastTerm, cumulativePointCount, cumulativeRowCount));
+
+            shrink();
+
+            var mft = new TreeMap<ByteComparable, Long>((b1, b2) -> ByteComparable.compare(b1, b2, byteComparableVersion));
+            for (Point point : mostFrequentTerms) {
+                mft.put(point.term, point.rowCount);
+            }
+
+            return new TermsDistribution(termType, buckets, mft, Version.latest(), byteComparableVersion);
+        }
+
+        /**
+         * Shrinks the histogram to fit in the histogramSize limit, by removing some points.
+         * Tries to keep uniform granulatiry in terms of the number of rows.
+         * Runs in O(n) time.
+         * Needed because in some cases we don't know the number of points added to the histogram in advance,
+         * so we have to build it incrementally.
+         */
+        private void shrink()
+        {
+            if (buckets.size() < histogramSize)
+                return;
+
+            maxRowsPerBucket = buckets.get(buckets.size() - 1).cumulativeRowCount / histogramSize;
+            int targetIndex = 1;
+            for (int candidateIndex = 1; candidateIndex < buckets.size(); candidateIndex++)
+            {
+                Bucket last = buckets.get(targetIndex - 1);
+                Bucket candidate = buckets.get(candidateIndex);
+                if (candidate.cumulativeRowCount - last.cumulativeRowCount > maxRowsPerBucket || candidateIndex == buckets.size() - 1)
+                {
+                    buckets.set(targetIndex, candidate);
+                    targetIndex++;
+                }
+            }
+            buckets.subList(targetIndex, buckets.size()).clear();
+        }
+    }
+
+    /**
+     * A histogram bucket - keeps the cumulative point and row counts for all the terms smaller or equal given term.
+     */
+    @ThreadSafe
+    @Immutable
+    static class Bucket
+    {
+        final ByteComparable term;
+        final long cumulativePointCount;
+        final long cumulativeRowCount;
+
+        Bucket(ByteComparable term, long cumulativePointCount, long cumulativeRowCount)
+        {
+            this.term = term;
+            this.cumulativePointCount = cumulativePointCount;
+            this.cumulativeRowCount = cumulativeRowCount;
+        }
+    }
+
+    /**
+     * A helper class for building the most frequent terms queue.
+     * Associates the term with the row count and provides a natural ordering by row count.
+     */
+    static class Point implements Comparable<Point>
+    {
+        final ByteComparable term;
+        final long rowCount;
+
+        Point(ByteComparable term, long rowCount)
+        {
+            this.term = term;
+            this.rowCount = rowCount;
+        }
+
+        @Override
+        public int compareTo(Point o)
+        {
+            return Long.compare(rowCount, o.rowCount);
+        }
+    }
+
+}
+
+
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v6/V6OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v6/V6OnDiskFormat.java
new file mode 100644
index 000000000000..98a4eb356a5c
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v6/V6OnDiskFormat.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v6;
+
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
+import org.apache.cassandra.index.sai.disk.v5.V5OnDiskFormat;
+
+public class V6OnDiskFormat extends V5OnDiskFormat
+{
+    public static final V6OnDiskFormat instance = new V6OnDiskFormat();
+
+    private static final IndexFeatureSet v6IndexFeatureSet = new IndexFeatureSet()
+    {
+        @Override
+        public boolean isRowAware()
+        {
+            return true;
+        }
+
+        @Override
+        public boolean hasVectorIndexChecksum()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean hasTermsHistogram()
+        {
+            return true;
+        }
+    };
+
+    @Override
+    public IndexFeatureSet indexFeatureSet()
+    {
+        return v6IndexFeatureSet;
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v7/V7OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v7/V7OnDiskFormat.java
new file mode 100644
index 000000000000..5457685cf1b0
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v7/V7OnDiskFormat.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v7;
+
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v6.V6OnDiskFormat;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+
+public class V7OnDiskFormat extends V6OnDiskFormat
+{
+    public static final V7OnDiskFormat instance = new V7OnDiskFormat();
+
+    private static final Set<IndexComponentType> LITERAL_COMPONENTS = EnumSet.of(IndexComponentType.COLUMN_COMPLETION_MARKER,
+                                                                                 IndexComponentType.META,
+                                                                                 IndexComponentType.TERMS_DATA,
+                                                                                 IndexComponentType.POSTING_LISTS,
+                                                                                 IndexComponentType.DOC_LENGTHS);
+
+    @Override
+    public Set<IndexComponentType> perIndexComponentTypes(AbstractType<?> validator)
+    {
+        if (validator.isVector())
+            return V3OnDiskFormat.VECTOR_COMPONENTS_V3;
+        if (TypeUtil.isLiteral(validator))
+            return LITERAL_COMPONENTS;
+        return V1OnDiskFormat.NUMERIC_COMPONENTS;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/AutoResumingNodeScoreIterator.java b/src/java/org/apache/cassandra/index/sai/disk/vector/AutoResumingNodeScoreIterator.java
new file mode 100644
index 000000000000..ab586b0ce025
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/AutoResumingNodeScoreIterator.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.function.IntConsumer;
+
+import io.github.jbellis.jvector.graph.GraphSearcher;
+import io.github.jbellis.jvector.graph.SearchResult;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.AbstractIterator;
+
+import static java.lang.Math.max;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+/**
+ * An iterator over {@link SearchResult.NodeScore} backed by a {@link SearchResult} that resumes search
+ * when the backing {@link SearchResult} is exhausted.
+ */
+public class AutoResumingNodeScoreIterator extends AbstractIterator<SearchResult.NodeScore>
+{
+    private final GraphSearcher searcher;
+    private final GraphSearcherAccessManager accessManager;
+    private final int limit;
+    private final int rerankK;
+    private final boolean inMemory;
+    private final String source;
+    private final QueryContext context;
+    private final ColumnQueryMetrics.VectorIndexMetrics columnQueryMetrics;
+    private final IntConsumer nodesVisitedConsumer;
+    private Iterator<SearchResult.NodeScore> nodeScores;
+    private int cumulativeNodesVisited;
+
+    /**
+     * Create a new {@link AutoResumingNodeScoreIterator} that iterates over the provided {@link SearchResult}.
+     * If the {@link SearchResult} is consumed, it retrieves the next {@link SearchResult} until the search returns
+     * no more results.
+     * @param searcher the {@link GraphSearcher} to use to resume search.
+     * @param result the first {@link SearchResult} to iterate over
+     * @param context the {@link QueryContext} to use to record metrics
+     * @param columnQueryMetrics object to record metrics
+     * @param nodesVisitedConsumer a consumer that accepts the total number of nodes visited
+     * @param limit the limit to pass to the {@link GraphSearcher} when resuming search
+     * @param rerankK the rerankK to pass to the {@link GraphSearcher} when resuming search
+     * @param inMemory whether the graph is in memory or on disk (used for trace logging)
+     * @param source the source of the search (used for trace logging)
+     */
+    public AutoResumingNodeScoreIterator(GraphSearcher searcher,
+                                         GraphSearcherAccessManager accessManager,
+                                         SearchResult result,
+                                         QueryContext context,
+                                         ColumnQueryMetrics.VectorIndexMetrics columnQueryMetrics,
+                                         IntConsumer nodesVisitedConsumer,
+                                         int limit,
+                                         int rerankK,
+                                         boolean inMemory,
+                                         String source)
+    {
+        this.searcher = searcher;
+        this.accessManager = accessManager;
+        this.nodeScores = Arrays.stream(result.getNodes()).iterator();
+        this.context = context;
+        this.columnQueryMetrics = columnQueryMetrics;
+        this.cumulativeNodesVisited = 0;
+        this.nodesVisitedConsumer = nodesVisitedConsumer;
+        this.limit = max(1, limit / 2); // we shouldn't need as many results on resume
+        this.rerankK = rerankK;
+        this.inMemory = inMemory;
+        this.source = source;
+    }
+
+    @Override
+    protected SearchResult.NodeScore computeNext()
+    {
+        if (nodeScores.hasNext())
+            return nodeScores.next();
+
+        long start = nanoTime();
+
+        // Search deeper into the graph
+        var nextResult = searcher.resume(limit, rerankK);
+
+        // Record metrics
+        long elapsed = nanoTime() - start;
+        columnQueryMetrics.onSearchResult(nextResult, elapsed, true);
+        context.addAnnGraphSearchLatency(elapsed);
+        cumulativeNodesVisited += nextResult.getVisitedCount();
+
+        if (Tracing.isTracing())
+        {
+            String msg = inMemory ? "Memory based ANN resume for {}/{} visited {} nodes, reranked {} to return {} results from {}"
+                                  : "Disk based ANN resume for {}/{} visited {} nodes, reranked {} to return {} results from {}";
+            Tracing.trace(msg, limit, rerankK, nextResult.getVisitedCount(), nextResult.getRerankedCount(), nextResult.getNodes().length, source);
+        }
+
+        // If the next result is empty, we are done searching.
+        nodeScores = Arrays.stream(nextResult.getNodes()).iterator();
+        return nodeScores.hasNext() ? nodeScores.next() : endOfData();
+    }
+
+    @Override
+    public void close()
+    {
+        nodesVisitedConsumer.accept(cumulativeNodesVisited);
+        accessManager.release();
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/BitsUtil.java b/src/java/org/apache/cassandra/index/sai/disk/vector/BitsUtil.java
similarity index 78%
rename from src/java/org/apache/cassandra/index/sai/disk/v1/vector/BitsUtil.java
rename to src/java/org/apache/cassandra/index/sai/disk/vector/BitsUtil.java
index 40aff6b49ee3..86c8741cc3ce 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/BitsUtil.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/BitsUtil.java
@@ -16,13 +16,12 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.index.sai.disk.v1.vector;
+package org.apache.cassandra.index.sai.disk.vector;
 
 import java.util.Set;
 
-import org.cliffc.high_scale_lib.NonBlockingHashMapLong;
-
 import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.util.DenseIntMap;
 
 public class BitsUtil
 {
@@ -30,12 +29,12 @@ public static Bits bitsIgnoringDeleted(Bits toAccept, Set<Integer> deletedOrdina
     {
         return deletedOrdinals.isEmpty()
                ? toAccept
-               : toAccept == null ? new NoDeletedBits(deletedOrdinals) : new NoDeletedIntersectingBits(toAccept, deletedOrdinals);
+               : toAccept == Bits.ALL ? new NoDeletedBits(deletedOrdinals) : new NoDeletedIntersectingBits(toAccept, deletedOrdinals);
     }
 
-    public static <T> Bits bitsIgnoringDeleted(Bits toAccept, NonBlockingHashMapLong<VectorPostings<T>> postings)
+    public static <T> Bits bitsIgnoringDeleted(Bits toAccept, DenseIntMap<VectorPostings<T>> postings)
     {
-        return toAccept == null ? new NoDeletedPostings<>(postings) : new NoDeletedIntersectingPostings<>(toAccept, postings);
+        return toAccept == Bits.ALL ? new NoDeletedPostings(postings) : new NoDeletedIntersectingPostings(toAccept, postings);
     }
 
     private static abstract class BitsWithoutLength implements Bits, org.apache.lucene.util.Bits
@@ -84,9 +83,9 @@ public boolean get(int i)
 
     private static class NoDeletedPostings<T> extends BitsWithoutLength
     {
-        private final NonBlockingHashMapLong<VectorPostings<T>> postings;
+        private final DenseIntMap<VectorPostings<T>> postings;
 
-        public NoDeletedPostings(NonBlockingHashMapLong<VectorPostings<T>> postings)
+        public NoDeletedPostings(DenseIntMap<VectorPostings<T>> postings)
         {
             this.postings = postings;
         }
@@ -103,9 +102,9 @@ public boolean get(int i)
     private static class NoDeletedIntersectingPostings<T> extends BitsWithoutLength
     {
         private final Bits toAccept;
-        private final NonBlockingHashMapLong<VectorPostings<T>> postings;
+        private final DenseIntMap<VectorPostings<T>> postings;
 
-        public NoDeletedIntersectingPostings(Bits toAccept, NonBlockingHashMapLong<VectorPostings<T>> postings)
+        public NoDeletedIntersectingPostings(Bits toAccept, DenseIntMap<VectorPostings<T>> postings)
         {
             this.toAccept = toAccept;
             this.postings = postings;
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/BruteForceRowIdIterator.java b/src/java/org/apache/cassandra/index/sai/disk/vector/BruteForceRowIdIterator.java
new file mode 100644
index 000000000000..e2b2aa22b93f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/BruteForceRowIdIterator.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import io.github.jbellis.jvector.graph.NodeQueue;
+import io.github.jbellis.jvector.util.BoundedLongHeap;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+import org.apache.cassandra.index.sai.utils.SegmentRowIdOrdinalPairs;
+import org.apache.cassandra.index.sai.utils.RowIdWithMeta;
+import org.apache.cassandra.index.sai.utils.RowIdWithScore;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.SortingIterator;
+
+
+/**
+ * An iterator over {@link RowIdWithMeta} that lazily consumes from a {@link SortingIterator} of
+ * {@link RowWithApproximateScore}.
+ * <p>
+ * The idea is that we maintain the same level of accuracy as we would get from a graph search, by re-ranking the top
+ * `k` best approximate scores at a time with the full resolution vectors to return the top `limit`.
+ * <p>
+ * For example, suppose that limit=3 and k=5 and we have ten elements.  After our first re-ranking batch, we have
+ *   ABDEF?????
+ * We will return A, B, and D; if more elements are requested, we will re-rank another 5 (so three more, including
+ * the two remaining from the first batch).  Here we uncover C, G, and H, and order them appropriately:
+ *      CEFGH??
+ * This illustrates that, also like a graph search, we only guarantee ordering of results within a re-ranking batch,
+ * not globally.
+ * <p>
+ * Note that we deliberately do not fetch new items from the approximate list until the first batch of `limit`-many
+ * is consumed. We do this because we expect that most often the first limit-many will pass the final verification
+ * and only query more if some didn't (e.g. because the vector was deleted in a newer sstable).
+ * <p>
+ * As an implementation detail, we use a heap to maintain state rather than a List and sorting.
+ */
+public class BruteForceRowIdIterator extends AbstractIterator<RowIdWithScore>
+{
+    // We use two binary heaps (NodeQueue) because we do not need an eager ordering of
+    // these results. Depending on how many sstables the query hits and the relative scores of vectors from those
+    // sstables, we may not need to return more than the first handful of scores.
+    // Heap with compressed vector scores
+    private final NodeQueue approximateScoreQueue;
+    private final SegmentRowIdOrdinalPairs segmentOrdinalPairs;
+    // Use the jvector NodeQueue to avoid unnecessary object allocations
+    private final NodeQueue exactScoreQueue;
+    private final CloseableReranker reranker;
+    private final int topK;
+    private final int limit;
+    private final ColumnQueryMetrics.VectorIndexMetrics columnQueryMetrics;
+    private int rerankedCount;
+
+    /**
+     * @param approximateScoreQueue A heap of indexes ordered by their approximate similarity scores
+     * @param segmentOrdinalPairs A mapping from the index in the approximateScoreQueue to the node's rowId and ordinal
+     * @param reranker A function that takes a graph ordinal and returns the exact similarity score
+     * @param limit The query limit
+     * @param topK The number of vectors to resolve and score before returning results
+     * @param columnQueryMetrics object to record metrics
+     */
+    public BruteForceRowIdIterator(NodeQueue approximateScoreQueue,
+                                   SegmentRowIdOrdinalPairs segmentOrdinalPairs,
+                                   CloseableReranker reranker,
+                                   int limit,
+                                   int topK,
+                                   ColumnQueryMetrics.VectorIndexMetrics columnQueryMetrics)
+    {
+        this.approximateScoreQueue = approximateScoreQueue;
+        this.segmentOrdinalPairs = segmentOrdinalPairs;
+        this.exactScoreQueue = new NodeQueue(new BoundedLongHeap(topK), NodeQueue.Order.MAX_HEAP);
+        this.reranker = reranker;
+        assert topK >= limit : "topK must be greater than or equal to limit. Found: " + topK + " < " + limit;
+        this.limit = limit;
+        this.topK = topK;
+        this.columnQueryMetrics = columnQueryMetrics;
+        this.rerankedCount = topK; // placeholder to kick off computeNext
+    }
+
+    @Override
+    protected RowIdWithScore computeNext() {
+        int consumed = rerankedCount - exactScoreQueue.size();
+        if (consumed >= limit) {
+            int exactComparisons = 0;
+            // Refill the exactScoreQueue until it reaches topK exact scores, or the approximate score queue is empty
+            while (approximateScoreQueue.size() > 0 && exactScoreQueue.size() < topK) {
+                int segmentOrdinalIndex = approximateScoreQueue.pop();
+                int rowId = segmentOrdinalPairs.getSegmentRowId(segmentOrdinalIndex);
+                int ordinal = segmentOrdinalPairs.getOrdinal(segmentOrdinalIndex);
+                float score = reranker.similarityTo(ordinal);
+                exactComparisons++;
+                exactScoreQueue.push(rowId, score);
+            }
+            columnQueryMetrics.onBruteForceNodesReranked(exactComparisons);
+            rerankedCount = exactScoreQueue.size();
+        }
+        if (exactScoreQueue.size() == 0)
+            return endOfData();
+
+        float score = exactScoreQueue.topScore();
+        int rowId = exactScoreQueue.pop();
+        return new RowIdWithScore(rowId, score);
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(reranker);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/CassandraDiskAnn.java b/src/java/org/apache/cassandra/index/sai/disk/vector/CassandraDiskAnn.java
new file mode 100644
index 000000000000..748cf48e582b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/CassandraDiskAnn.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.function.IntConsumer;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.graph.GraphIndex;
+import io.github.jbellis.jvector.graph.GraphSearcher;
+import io.github.jbellis.jvector.graph.disk.feature.FeatureId;
+import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex;
+import io.github.jbellis.jvector.graph.similarity.SearchScoreProvider;
+import io.github.jbellis.jvector.quantization.BQVectors;
+import io.github.jbellis.jvector.quantization.CompressedVectors;
+import io.github.jbellis.jvector.quantization.PQVectors;
+import io.github.jbellis.jvector.quantization.ProductQuantization;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.util.ExplicitThreadLocal;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter.Structure;
+import org.apache.cassandra.index.sai.disk.vector.CassandraOnHeapGraph.PQVersion;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+import org.apache.cassandra.index.sai.utils.RowIdWithScore;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.CloseableIterator;
+
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+
+public class CassandraDiskAnn
+{
+    private static final Logger logger = LoggerFactory.getLogger(CassandraDiskAnn.class.getName());
+
+    public static final int PQ_MAGIC = 0xB011A61C; // PQ_MAGIC, with a lot of liberties taken
+    protected final PerIndexFiles indexFiles;
+    private final ColumnQueryMetrics.VectorIndexMetrics columnQueryMetrics;
+    protected final SegmentMetadata.ComponentMetadataMap componentMetadatas;
+
+    private final SSTableId<?> source;
+    private final FileHandle graphHandle;
+    private final OnDiskOrdinalsMap ordinalsMap;
+    private final Set<FeatureId> features;
+    private final GraphIndex graph;
+    private final VectorSimilarityFunction similarityFunction;
+    @Nullable
+    private final CompressedVectors compressedVectors;
+    @Nullable
+    private final ProductQuantization pq;
+    private final VectorCompression compression;
+    final boolean pqUnitVectors;
+
+    private final ExplicitThreadLocal<GraphSearcherAccessManager> searchers;
+
+    public CassandraDiskAnn(SSTableContext sstableContext, SegmentMetadata.ComponentMetadataMap componentMetadatas, PerIndexFiles indexFiles, IndexContext context, OrdinalsMapFactory omFactory) throws IOException
+    {
+        this.source = sstableContext.sstable().getId();
+        this.componentMetadatas = componentMetadatas;
+        this.indexFiles = indexFiles;
+        this.columnQueryMetrics = (ColumnQueryMetrics.VectorIndexMetrics) context.getColumnQueryMetrics();
+
+        similarityFunction = context.getIndexWriterConfig().getSimilarityFunction();
+
+        SegmentMetadata.ComponentMetadata termsMetadata = this.componentMetadatas.get(IndexComponentType.TERMS_DATA);
+        graphHandle = indexFiles.termsData();
+        var rawGraph = OnDiskGraphIndex.load(graphHandle::createReader, termsMetadata.offset);
+        features = rawGraph.getFeatureSet();
+        graph = rawGraph;
+
+        long pqSegmentOffset = this.componentMetadatas.get(IndexComponentType.PQ).offset;
+        try (var pqFile = indexFiles.pq();
+             var reader = pqFile.createReader())
+        {
+            reader.seek(pqSegmentOffset);
+            var version = PQVersion.V0;
+            if (reader.readInt() == PQ_MAGIC)
+            {
+                version = PQVersion.values()[reader.readInt()];
+                assert PQVersion.V1.compareTo(version) >= 0 : String.format("Old PQ version %s written with PQ_MAGIC!?", version);
+                pqUnitVectors = reader.readBoolean();
+            }
+            else
+            {
+                pqUnitVectors = true;
+                reader.seek(pqSegmentOffset);
+            }
+
+            VectorCompression.CompressionType compressionType = VectorCompression.CompressionType.values()[reader.readByte()];
+            if (features.contains(FeatureId.FUSED_ADC))
+            {
+                assert compressionType == VectorCompression.CompressionType.PRODUCT_QUANTIZATION;
+                compressedVectors = null;
+                // don't load full PQVectors, all we need is the metadata from the PQ at the start
+                pq = ProductQuantization.load(reader);
+                compression = new VectorCompression(VectorCompression.CompressionType.PRODUCT_QUANTIZATION,
+                                                    rawGraph.getDimension() * Float.BYTES,
+                                                    pq.compressedVectorSize());
+            }
+            else
+            {
+                if (compressionType == VectorCompression.CompressionType.PRODUCT_QUANTIZATION)
+                {
+                    compressedVectors = PQVectors.load(reader, reader.getFilePointer());
+                    pq = ((PQVectors) compressedVectors).getCompressor();
+                    compression = new VectorCompression(compressionType,
+                                                        compressedVectors.getOriginalSize(),
+                                                        compressedVectors.getCompressedSize());
+                }
+                else if (compressionType == VectorCompression.CompressionType.BINARY_QUANTIZATION)
+                {
+                    compressedVectors = BQVectors.load(reader, reader.getFilePointer());
+                    pq = null;
+                    compression = new VectorCompression(compressionType,
+                                                        compressedVectors.getOriginalSize(),
+                                                        compressedVectors.getCompressedSize());
+                }
+                else
+                {
+                    compressedVectors = null;
+                    pq = null;
+                    compression = VectorCompression.NO_COMPRESSION;
+                }
+            }
+        }
+
+        SegmentMetadata.ComponentMetadata postingListsMetadata = this.componentMetadatas.get(IndexComponentType.POSTING_LISTS);
+        ordinalsMap = omFactory.create(indexFiles.postingLists(), postingListsMetadata.offset, postingListsMetadata.length);
+        if (ordinalsMap.getStructure() == Structure.ZERO_OR_ONE_TO_MANY)
+            logger.warn("Index {} has structure ZERO_OR_ONE_TO_MANY, which requires on reading the on disk row id" +
+                        " to ordinal mapping for each search. This will be slower.", source);
+
+        searchers = ExplicitThreadLocal.withInitial(() -> new GraphSearcherAccessManager(new GraphSearcher(graph)));
+
+        // Record metrics for this graph
+        columnQueryMetrics.onGraphLoaded(compressedVectors == null ? 0 : compressedVectors.ramBytesUsed(),
+                                         ordinalsMap.cachedBytesUsed(),
+                                         graph.size(0));
+    }
+
+    public Structure getPostingsStructure()
+    {
+        return ordinalsMap.getStructure();
+    }
+
+    @FunctionalInterface
+    public interface OrdinalsMapFactory {
+        OnDiskOrdinalsMap create(FileHandle handle, long offset, long length);
+    }
+
+    public ProductQuantization getPQ()
+    {
+        assert compression.type == VectorCompression.CompressionType.PRODUCT_QUANTIZATION;
+        assert pq != null;
+        return pq;
+    }
+
+    public long ramBytesUsed()
+    {
+        return graph.ramBytesUsed();
+    }
+
+    public int size()
+    {
+        // The base layer of the graph has all nodes.
+        return graph.size(0);
+    }
+
+    /**
+     * @param queryVector the query vector
+     * @param limit the number of results to look for in the index (>= limit)
+     * @param rerankK the number of results to look for in the index (>= limit)
+     * @param threshold the minimum similarity score to accept
+     * @param acceptBits a Bits indicating which row IDs are acceptable, or null if no constraints
+     * @param context unused (vestige from HNSW, retained in signature to allow calling both easily)
+     * @param nodesVisitedConsumer a consumer that will be called with the number of nodes visited during the search
+     * @return Iterator of Row IDs associated with the vectors near the query. If a threshold is specified, only vectors
+     * with a similarity score >= threshold will be returned.
+     */
+    public CloseableIterator<RowIdWithScore> search(VectorFloat<?> queryVector,
+                                                    int limit,
+                                                    int rerankK,
+                                                    float threshold,
+                                                    Bits acceptBits,
+                                                    QueryContext context,
+                                                    IntConsumer nodesVisitedConsumer)
+    {
+        VectorValidation.validateIndexable(queryVector, similarityFunction);
+
+        var graphAccessManager = searchers.get();
+        var searcher = graphAccessManager.get();
+        try
+        {
+            var view = (GraphIndex.ScoringView) searcher.getView();
+            SearchScoreProvider ssp;
+            // FusedADC can no longer be written due to jvector upgrade. However, it's possible these index files
+            // still exist, so we have to support them.
+            if (features.contains(FeatureId.FUSED_ADC))
+            {
+                var asf = view.approximateScoreFunctionFor(queryVector, similarityFunction);
+                var rr = view.rerankerFor(queryVector, similarityFunction);
+                ssp = new SearchScoreProvider(asf, rr);
+            }
+            else if (compressedVectors == null)
+            {
+                ssp = new SearchScoreProvider(view.rerankerFor(queryVector, similarityFunction));
+            }
+            else
+            {
+                // unit vectors defined with dot product should switch to cosine similarity for compressed
+                // comparisons, since the compression does not maintain unit length
+                var sf = pqUnitVectors && similarityFunction == VectorSimilarityFunction.DOT_PRODUCT
+                         ? VectorSimilarityFunction.COSINE
+                         : similarityFunction;
+                var asf = compressedVectors.precomputedScoreFunctionFor(queryVector, sf);
+                var rr = view.rerankerFor(queryVector, similarityFunction);
+                ssp = new SearchScoreProvider(asf, rr);
+            }
+            long start = nanoTime();
+            var result = searcher.search(ssp, limit, rerankK, threshold, context.getAnnRerankFloor(), ordinalsMap.ignoringDeleted(acceptBits));
+            long elapsed = nanoTime() - start;
+            if (V3OnDiskFormat.ENABLE_RERANK_FLOOR)
+                context.updateAnnRerankFloor(result.getWorstApproximateScoreInTopK());
+            Tracing.trace("DiskANN search for {}/{} visited {} nodes, reranked {} to return {} results from {}",
+                          limit, rerankK, result.getVisitedCount(), result.getRerankedCount(), result.getNodes().length, source);
+            columnQueryMetrics.onSearchResult(result, elapsed, false);
+            context.addAnnGraphSearchLatency(elapsed);
+            if (threshold > 0)
+            {
+                // Threshold based searches are comprehensive and do not need to resume the search.
+                graphAccessManager.release();
+                nodesVisitedConsumer.accept(result.getVisitedCount());
+                var nodeScores = CloseableIterator.wrap(Arrays.stream(result.getNodes()).iterator());
+                return new NodeScoreToRowIdWithScoreIterator(nodeScores, ordinalsMap.getRowIdsView());
+            }
+            else
+            {
+                var nodeScores = new AutoResumingNodeScoreIterator(searcher, graphAccessManager, result, context, columnQueryMetrics, nodesVisitedConsumer, limit, rerankK, false, source.toString());
+                return new NodeScoreToRowIdWithScoreIterator(nodeScores, ordinalsMap.getRowIdsView());
+            }
+        }
+        catch (Throwable t)
+        {
+            // If we don't release it, we'll never be able to aquire it, so catch and rethrow Throwable.
+            graphAccessManager.forceRelease();
+            throw t;
+        }
+    }
+
+    public VectorCompression getCompression()
+    {
+        return compression;
+    }
+
+    public CompressedVectors getCompressedVectors()
+    {
+        return compressedVectors;
+    }
+
+    public void close() throws IOException
+    {
+        FileUtils.close(ordinalsMap, searchers, graph, graphHandle);
+        columnQueryMetrics.onGraphClosed(compressedVectors == null ? 0 : compressedVectors.ramBytesUsed(),
+                                         ordinalsMap.cachedBytesUsed(),
+                                         graph.size(0));
+    }
+
+    public OrdinalsView getOrdinalsView()
+    {
+        return ordinalsMap.getOrdinalsView();
+    }
+
+    public GraphIndex.ScoringView getView()
+    {
+        return (GraphIndex.ScoringView) graph.getView();
+    }
+
+    public boolean containsUnitVectors()
+    {
+        return pqUnitVectors;
+    }
+
+    public int maxDegree()
+    {
+        return graph.maxDegree();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/CassandraOnHeapGraph.java b/src/java/org/apache/cassandra/index/sai/disk/vector/CassandraOnHeapGraph.java
new file mode 100644
index 000000000000..6356286894f0
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/CassandraOnHeapGraph.java
@@ -0,0 +1,702 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.function.IntUnaryOperator;
+import java.util.function.ToIntFunction;
+import java.util.stream.IntStream;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.cliffc.high_scale_lib.NonBlockingHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.graph.GraphIndexBuilder;
+import io.github.jbellis.jvector.graph.GraphSearcher;
+import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
+import io.github.jbellis.jvector.graph.SearchResult;
+import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndexWriter;
+import io.github.jbellis.jvector.graph.disk.OrdinalMapper;
+import io.github.jbellis.jvector.graph.disk.feature.Feature;
+import io.github.jbellis.jvector.graph.disk.feature.FeatureId;
+import io.github.jbellis.jvector.graph.disk.feature.InlineVectors;
+import io.github.jbellis.jvector.graph.similarity.SearchScoreProvider;
+import io.github.jbellis.jvector.quantization.BinaryQuantization;
+import io.github.jbellis.jvector.quantization.CompressedVectors;
+import io.github.jbellis.jvector.quantization.ProductQuantization;
+import io.github.jbellis.jvector.quantization.VectorCompressor;
+import io.github.jbellis.jvector.util.Accountable;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.util.DenseIntMap;
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import io.github.jbellis.jvector.vector.ArrayVectorFloat;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorUtil;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.agrona.collections.IntHashSet;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.v1.Segment;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorPostingsWriter;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v5.V5OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter.Structure;
+import org.apache.cassandra.index.sai.disk.vector.VectorCompression.CompressionType;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.lucene.util.StringHelper;
+
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat.JVECTOR_VERSION;
+
+public class CassandraOnHeapGraph<T> implements Accountable
+{
+    // Cassandra's PQ features, independent of JVector's
+    public enum PQVersion {
+        V0, // initial version
+        V1, // includes unit vector calculation
+    }
+
+    /** minimum number of rows to perform PQ codebook generation */
+    public static final int MIN_PQ_ROWS = 1024;
+
+    private static final Logger logger = LoggerFactory.getLogger(CassandraOnHeapGraph.class);
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    // We use the metable reference for easier tracing.
+    private final String source;
+    private final ColumnQueryMetrics.VectorIndexMetrics columnQueryMetrics;
+    private final ConcurrentVectorValues vectorValues;
+    private final GraphIndexBuilder builder;
+    private final VectorType.VectorSerializer serializer;
+    private final VectorSimilarityFunction similarityFunction;
+    private final ConcurrentMap<VectorFloat<?>, VectorPostings<T>> postingsMap;
+    private final DenseIntMap<VectorPostings<T>> postingsByOrdinal;
+    private final NonBlockingHashMap<T, VectorFloat<?>> vectorsByKey;
+    private final AtomicInteger nextOrdinal = new AtomicInteger();
+    private final VectorSourceModel sourceModel;
+    private final InvalidVectorBehavior invalidVectorBehavior;
+    private final IntHashSet deletedOrdinals;
+    private volatile boolean hasDeletions;
+
+    // we don't need to explicitly close these since only on-heap resources are involved
+    private final ThreadLocal<GraphSearcherAccessManager> searchers;
+
+    /**
+     * @param forSearching if true, vectorsByKey will be initialized and populated with vectors as they are added
+     */
+    public CassandraOnHeapGraph(IndexContext context, boolean forSearching, Memtable memtable)
+    {
+        this.source = memtable == null
+                      ? "null"
+                      : memtable.getClass().getSimpleName() + '@' + Integer.toHexString(memtable.hashCode());
+        this.columnQueryMetrics = (ColumnQueryMetrics.VectorIndexMetrics) context.getColumnQueryMetrics();
+        var indexConfig = context.getIndexWriterConfig();
+        var termComparator = context.getValidator();
+        serializer = (VectorType.VectorSerializer) termComparator.getSerializer();
+        var dimension = ((VectorType<?>) termComparator).dimension;
+        vectorValues = new ConcurrentVectorValues(dimension);
+        similarityFunction = indexConfig.getSimilarityFunction();
+        sourceModel = indexConfig.getSourceModel();
+        // We need to be able to inexpensively distinguish different vectors, with a slower path
+        // that identifies vectors that are equal but not the same reference.  A comparison-
+        // based Map (which only needs to look at vector elements until a difference is found)
+        // is thus a better option than hash-based (which has to look at all elements to compute the hash).
+        postingsMap = new ConcurrentSkipListMap<>((a, b) -> {
+            return Arrays.compare(((ArrayVectorFloat) a).get(), ((ArrayVectorFloat) b).get());
+        });
+        postingsByOrdinal = new DenseIntMap<>(1024);
+        deletedOrdinals = new IntHashSet();
+        vectorsByKey = forSearching ? new NonBlockingHashMap<>() : null;
+        invalidVectorBehavior = forSearching ? InvalidVectorBehavior.FAIL : InvalidVectorBehavior.IGNORE;
+
+        // This is only a warning since it's not a fatal error to write without hierarchy
+        if (indexConfig.isHierarchyEnabled() && V3OnDiskFormat.JVECTOR_VERSION < 4)
+            logger.warn("Hierarchical graphs configured but node configured with V3OnDiskFormat.JVECTOR_VERSION {}. " +
+                        "Skipping setting for {}", V3OnDiskFormat.JVECTOR_VERSION, indexConfig.getIndexName());
+
+        builder = new GraphIndexBuilder(vectorValues,
+                                        similarityFunction,
+                                        indexConfig.getAnnMaxDegree(),
+                                        indexConfig.getConstructionBeamWidth(),
+                                        indexConfig.getNeighborhoodOverflow(1.0f), // no overflow means add will be a bit slower but flush will be faster
+                                        indexConfig.getAlpha(dimension > 3 ? 1.2f : 2.0f),
+                                        indexConfig.isHierarchyEnabled() && V3OnDiskFormat.JVECTOR_VERSION >= 4);
+        searchers = ThreadLocal.withInitial(() -> new GraphSearcherAccessManager(new GraphSearcher(builder.getGraph())));
+    }
+
+    public int size()
+    {
+        return vectorValues.size();
+    }
+
+    public boolean isEmpty()
+    {
+        return postingsMap.values().stream().allMatch(VectorPostings::isEmpty);
+    }
+
+    /**
+     * @return the ordinal of the vector in the graph, or -1 if the vector is not in the graph
+     */
+    public int getOrdinal(VectorFloat<?> vector)
+    {
+        VectorPostings<T> postings = postingsMap.get(vector);
+        // There is a small race from when the postings list is created to when it is assigned an ordinal,
+        // so we do not assert that the ordinal is set here
+        return postings == null ? -1 : postings.getOrdinal(false);
+    }
+
+    /**
+     * @return the incremental bytes used by adding the given vector to the index
+     */
+    public long add(ByteBuffer term, T key)
+    {
+        assert term != null && term.remaining() != 0;
+
+        var vector = vts.createFloatVector(serializer.deserializeFloatArray(term));
+        // Validate the vector.  Almost always, this is called at insert time (which sets invalid behavior to FAIL,
+        // resulting in the insert being aborted if the vector is invalid), or while writing out an sstable
+        // from flush or compaction (which sets invalid behavior to IGNORE, since we can't just rip existing data out of
+        // the table).
+        //
+        // However, it's also possible for this to be called during commitlog replay if the node previously crashed
+        // AFTER processing CREATE INDEX, but BEFORE flushing active memtables.  Commitlog replay will then follow
+        // the normal insert code path, (which would set behavior to FAIL) so we special-case it here; see VECTOR-269.
+        var behavior = invalidVectorBehavior;
+        if (!StorageService.instance.isInitialized())
+            behavior = InvalidVectorBehavior.IGNORE; // we're replaying the commitlog so force IGNORE
+        if (behavior == InvalidVectorBehavior.IGNORE)
+        {
+            try
+            {
+                VectorValidation.validateIndexable(vector, similarityFunction);
+            }
+            catch (InvalidRequestException e)
+            {
+                if (StorageService.instance.isInitialized())
+                    logger.trace("Ignoring invalid vector during index build against existing data: {}", (Object) e);
+                else
+                    logger.trace("Ignoring invalid vector during commitlog replay: {}", (Object) e);
+                return 0;
+            }
+        }
+        else
+        {
+            assert behavior == InvalidVectorBehavior.FAIL;
+            VectorValidation.validateIndexable(vector, similarityFunction);
+        }
+
+        var bytesUsed = 0L;
+
+        // Store a cached reference to the vector for brute force computations later. There is a small race
+        // condition here: if inserts for the same PrimaryKey add different vectors, vectorsByKey might
+        // become out of sync with the graph.
+        if (vectorsByKey != null)
+        {
+            vectorsByKey.put(key, vector);
+            // The size of the entries themselves are counted below, so just count the two extra references
+            bytesUsed += RamUsageEstimator.NUM_BYTES_OBJECT_REF * 2L;
+        }
+
+        VectorPostings<T> postings = postingsMap.get(vector);
+        // if the vector is already in the graph, all that happens is that the postings list is updated
+        // otherwise, we add the vector in this order:
+        // 1. to the postingsMap
+        // 2. to the vectorValues
+        // 3. to the graph
+        // This way, concurrent searches of the graph won't see the vector until it's visible
+        // in the other structures as well.
+        if (postings == null)
+        {
+            postings = new VectorPostings<>(key);
+            // since we are using ConcurrentSkipListMap, it is NOT correct to use computeIfAbsent here
+            if (postingsMap.putIfAbsent(vector, postings) == null)
+            {
+                // we won the race to add the new entry; assign it an ordinal and add to the other structures
+                var ordinal = nextOrdinal.getAndIncrement();
+                postings.setOrdinal(ordinal);
+                bytesUsed += RamEstimation.concurrentHashMapRamUsed(1); // the new posting Map entry
+                bytesUsed += vectorValues.add(ordinal, vector);
+                bytesUsed += postings.ramBytesUsed();
+                var success = postingsByOrdinal.compareAndPut(ordinal, null, postings);
+                assert success : "postingsByOrdinal already contains an entry for ordinal " + ordinal;
+                bytesUsed += builder.addGraphNode(ordinal, vector);
+                return bytesUsed;
+            }
+            else
+            {
+                postings = postingsMap.get(vector);
+            }
+        }
+        // postings list already exists, just add the new key (if it's not already in the list)
+        if (postings.add(key))
+        {
+            bytesUsed += postings.bytesPerPosting();
+        }
+
+        return bytesUsed;
+    }
+
+    public Collection<T> keysFromOrdinal(int node)
+    {
+        return postingsByOrdinal.get(node).getPostings();
+    }
+
+    public VectorFloat<?> vectorForKey(T key)
+    {
+        if (vectorsByKey == null)
+            throw new IllegalStateException("vectorsByKey is not initialized");
+        return vectorsByKey.get(key);
+    }
+
+    public void remove(ByteBuffer term, T key)
+    {
+        assert term != null && term.remaining() != 0;
+
+        var rawVector = serializer.deserializeFloatArray(term);
+        VectorFloat<?> v = vts.createFloatVector(rawVector);
+        var postings = postingsMap.get(v);
+        if (postings == null)
+        {
+            // it's possible for this to be called against a different memtable than the one
+            // the value was originally added to, in which case we do not expect to find
+            // the key among the postings for this vector
+            return;
+        }
+
+        hasDeletions = true;
+        postings.remove(key);
+        if (vectorsByKey != null)
+            // On updates to a row, we call add then remove, so we must pass the key's value to ensure we only remove
+            // the deleted vector from vectorsByKey
+            vectorsByKey.remove(key, v);
+    }
+
+    /**
+     * @return an itererator over {@link PrimaryKeyWithSortKey} in the graph's {@link SearchResult} order
+     */
+    public CloseableIterator<SearchResult.NodeScore> search(QueryContext context, VectorFloat<?> queryVector, int limit, int rerankK, float threshold, Bits toAccept)
+    {
+        VectorValidation.validateIndexable(queryVector, similarityFunction);
+
+        // search() errors out when an empty graph is passed to it
+        if (vectorValues.size() == 0)
+            return CloseableIterator.emptyIterator();
+
+        Bits bits = hasDeletions ? BitsUtil.bitsIgnoringDeleted(toAccept, postingsByOrdinal) : toAccept;
+        var graphAccessManager = searchers.get();
+        var searcher = graphAccessManager.get();
+        try
+        {
+            var ssf = SearchScoreProvider.exact(queryVector, similarityFunction, vectorValues);
+            long start = nanoTime();
+            var result = searcher.search(ssf, limit, rerankK, threshold, 0.0f, bits);
+            long elapsed = nanoTime() - start;
+            Tracing.trace("ANN search for {}/{} visited {} nodes, reranked {} to return {} results from {}",
+                          limit, rerankK, result.getVisitedCount(), result.getRerankedCount(), result.getNodes().length, source);
+            columnQueryMetrics.onSearchResult(result, elapsed, false);
+            context.addAnnGraphSearchLatency(elapsed);
+            if (threshold > 0)
+            {
+                // Threshold based searches do not support resuming the search.
+                graphAccessManager.release();
+                return CloseableIterator.wrap(Arrays.stream(result.getNodes()).iterator());
+            }
+            return new AutoResumingNodeScoreIterator(searcher, graphAccessManager, result, context, columnQueryMetrics, visited -> {}, limit, rerankK, true, source);
+        }
+        catch (Throwable t)
+        {
+            // If we don't release it, we'll never be able to aquire it, so catch and rethrow Throwable.
+            graphAccessManager.forceRelease();
+            throw t;
+        }
+    }
+
+    /**
+     * Prepare for flushing by doing a bunch of housekeeping:
+     * 1. Compute row ids for each vector in the postings map
+     * 2. Remove any vectors that are no longer in use and populate `deletedOrdinals`, including for range deletions
+     * 3. Return true if the caller should proceed to invoke flush, or false if everything was deleted
+     * <p>
+     * This is split out from flush per se because of (3); we don't want to flush empty
+     * index segments, but until we do (1) and (2) we don't know if the segment is empty.
+     */
+    public boolean preFlush(ToIntFunction<T> postingTransformer)
+    {
+        var it = postingsMap.entrySet().iterator();
+        while (it.hasNext()) {
+            var entry = it.next();
+            var vp = entry.getValue();
+            vp.computeRowIds(postingTransformer);
+            if (vp.isEmpty() || vp.shouldAppendDeletedOrdinal())
+                deletedOrdinals.add(vp.getOrdinal());
+        }
+        return deletedOrdinals.size() < builder.getGraph().size();
+    }
+
+    public SegmentMetadata.ComponentMetadataMap flush(IndexComponents.ForWrite perIndexComponents) throws IOException
+    {
+        int nInProgress = builder.insertsInProgress();
+        assert nInProgress == 0 : String.format("Attempting to write graph while %d inserts are in progress", nInProgress);
+        assert nextOrdinal.get() == builder.getGraph().size() : String.format("nextOrdinal %d != graph size %d -- ordinals should be sequential",
+                                                                              nextOrdinal.get(), builder.getGraph().size());
+        assert vectorValues.size() == builder.getGraph().size() : String.format("vector count %d != graph size %d",
+                                                                                vectorValues.size(), builder.getGraph().size());
+        logger.debug("Writing graph with {} rows and {} distinct vectors", postingsMap.values().stream().mapToInt(VectorPostings::size).sum(), vectorValues.size());
+
+        // compute the remapping of old ordinals to new (to fill in holes from deletion and/or to create a
+        // closer correspondance to rowids, simplifying postings lookups later)
+        V5VectorPostingsWriter.RemappedPostings remappedPostings;
+        if (V5OnDiskFormat.writeV5VectorPostings())
+        {
+            // remove postings corresponding to marked-deleted vectors
+            var it = postingsMap.entrySet().iterator();
+            while (it.hasNext()) {
+                var entry = it.next();
+                var vp = entry.getValue();
+                if (deletedOrdinals.contains(vp.getOrdinal()))
+                    it.remove();
+            }
+
+            assert postingsMap.keySet().size() + deletedOrdinals.size() == vectorValues.size()
+            : String.format("postings map entry count %d + deleted count %d != vector count %d",
+                            postingsMap.keySet().size(), deletedOrdinals.size(), vectorValues.size());
+            // remove deleted ordinals from the graph.  this is not done at remove() time, because the same vector
+            // could be added back again, "undeleting" the ordinal, and the concurrency gets tricky
+            deletedOrdinals.stream().parallel().forEach(builder::markNodeDeleted);
+            deletedOrdinals.clear();
+            builder.cleanup();
+            remappedPostings = V5VectorPostingsWriter.remapForMemtable(postingsMap);
+        }
+        else
+        {
+            assert postingsMap.keySet().size() == vectorValues.size() : String.format("postings map entry count %d != vector count %d",
+                                                                                      postingsMap.keySet().size(), vectorValues.size());
+            builder.cleanup();
+            remappedPostings = V2VectorPostingsWriter.remapForMemtable(postingsMap, !deletedOrdinals.isEmpty());
+        }
+
+        OrdinalMapper ordinalMapper = remappedPostings.ordinalMapper;
+
+        IndexComponent.ForWrite termsDataComponent = perIndexComponents.addOrGet(IndexComponentType.TERMS_DATA);
+        var indexFile = termsDataComponent.file();
+        long termsOffset = SAICodecUtils.headerSize();
+        if (indexFile.exists())
+            termsOffset += indexFile.length();
+        try (var pqOutput = perIndexComponents.addOrGet(IndexComponentType.PQ).openOutput(true);
+             var postingsOutput = perIndexComponents.addOrGet(IndexComponentType.POSTING_LISTS).openOutput(true);
+             var indexWriter = new OnDiskGraphIndexWriter.Builder(builder.getGraph(), indexFile.toPath())
+                               .withVersion(JVECTOR_VERSION)
+                               .withMapper(ordinalMapper)
+                               .with(new InlineVectors(vectorValues.dimension()))
+                               .withStartOffset(termsOffset)
+                               .build())
+        {
+            SAICodecUtils.writeHeader(pqOutput);
+            SAICodecUtils.writeHeader(postingsOutput);
+            indexWriter.getOutput().seek(indexFile.length()); // position at the end of the previous segment before writing our own header
+            SAICodecUtils.writeHeader(SAICodecUtils.toLuceneOutput(indexWriter.getOutput()));
+            assert indexWriter.getOutput().position() == termsOffset : "termsOffset " + termsOffset + " != " + indexWriter.getOutput().position();
+
+            // compute and write PQ
+            long pqOffset = pqOutput.getFilePointer();
+            long pqPosition = writePQ(pqOutput.asSequentialWriter(), remappedPostings, perIndexComponents.context());
+            long pqLength = pqPosition - pqOffset;
+
+            // write postings
+            long postingsOffset = postingsOutput.getFilePointer();
+            long postingsPosition;
+            if (V5OnDiskFormat.writeV5VectorPostings())
+            {
+                assert deletedOrdinals.isEmpty(); // V5 format does not support recording deleted ordinals
+                postingsPosition = new V5VectorPostingsWriter<T>(remappedPostings)
+                                   .writePostings(postingsOutput.asSequentialWriter(), vectorValues, postingsMap);
+            }
+            else
+            {
+                IntUnaryOperator newToOldMapper = remappedPostings.ordinalMapper::newToOld;
+                postingsPosition = new V2VectorPostingsWriter<T>(remappedPostings.structure == Structure.ONE_TO_ONE, builder.getGraph().size(), newToOldMapper)
+                                   .writePostings(postingsOutput.asSequentialWriter(), vectorValues, postingsMap, deletedOrdinals);
+            }
+            long postingsLength = postingsPosition - postingsOffset;
+
+            // write the graph
+            var start = nanoTime();
+            var suppliers = Feature.singleStateFactory(FeatureId.INLINE_VECTORS, nodeId -> new InlineVectors.State(vectorValues.getVector(nodeId)));
+            indexWriter.write(suppliers);
+            SAICodecUtils.writeFooter(indexWriter.getOutput(), indexWriter.checksum());
+            logger.info("Writing graph took {}ms", (nanoTime() - start) / 1_000_000);
+            long termsLength = indexWriter.getOutput().position() - termsOffset;
+
+            // write remaining footers/checksums
+            SAICodecUtils.writeFooter(pqOutput);
+            SAICodecUtils.writeFooter(postingsOutput);
+
+            // add components to the metadata map
+            return createMetadataMap(termsOffset, termsLength, postingsOffset, postingsLength, pqOffset, pqLength);
+        }
+    }
+
+    static SegmentMetadata.ComponentMetadataMap createMetadataMap(long termsOffset, long termsLength, long postingsOffset, long postingsLength, long pqOffset, long pqLength)
+    {
+        SegmentMetadata.ComponentMetadataMap metadataMap = new SegmentMetadata.ComponentMetadataMap();
+        metadataMap.put(IndexComponentType.TERMS_DATA, -1, termsOffset, termsLength, Map.of());
+        metadataMap.put(IndexComponentType.POSTING_LISTS, -1, postingsOffset, postingsLength, Map.of());
+        Map<String, String> vectorConfigs = Map.of("SEGMENT_ID", ByteBufferUtil.bytesToHex(ByteBuffer.wrap(StringHelper.randomId())));
+        metadataMap.put(IndexComponentType.PQ, -1, pqOffset, pqLength, vectorConfigs);
+        return metadataMap;
+    }
+
+    /**
+     * Return the best previous CompressedVectors for this column that matches the `matcher` predicate.
+     * "Best" means the most recent one that hits the row count target of {@link ProductQuantization#MAX_PQ_TRAINING_SET_SIZE},
+     * or the one with the most rows if none are larger than that.
+     */
+    public static PqInfo getPqIfPresent(IndexContext indexContext, Function<VectorCompression, Boolean> matcher)
+    {
+        // Retrieve the first compressed vectors for a segment with at least MAX_PQ_TRAINING_SET_SIZE rows
+        // or the one with the most rows if none reach that size
+        var indexes = new ArrayList<>(indexContext.getView().getIndexes());
+        indexes.sort(Comparator.comparing(SSTableIndex::getSSTable, CompactionSSTable.maxTimestampDescending));
+
+        PqInfo cvi = null;
+        long maxRows = 0;
+        for (SSTableIndex index : indexes)
+        {
+            for (Segment segment : index.getSegments())
+            {
+                if (segment.metadata.numRows < maxRows)
+                    continue;
+
+                var searcher = (V2VectorIndexSearcher) segment.getIndexSearcher();
+                var cv = searcher.getCompression();
+                if (matcher.apply(cv))
+                {
+                    // We can exit now because we won't find a better candidate
+                    var candidate = new PqInfo(searcher.getPQ(), searcher.containsUnitVectors(), segment.metadata.numRows);
+                    if (segment.metadata.numRows >= ProductQuantization.MAX_PQ_TRAINING_SET_SIZE)
+                        return candidate;
+
+                    cvi = candidate;
+                    maxRows = segment.metadata.numRows;
+                }
+            }
+        }
+        return cvi;
+    }
+
+    private long writePQ(SequentialWriter writer, V5VectorPostingsWriter.RemappedPostings remapped, IndexContext indexContext) throws IOException
+    {
+        var preferredCompression = sourceModel.compressionProvider.apply(vectorValues.dimension());
+
+        // Build encoder and compress vectors
+        VectorCompressor<?> compressor; // will be null if we can't compress
+        CompressedVectors cv = null;
+        boolean containsUnitVectors;
+        // limit the PQ computation and encoding to one index at a time -- goal during flush is to
+        // evict from memory ASAP so better to do the PQ build (in parallel) one at a time
+        synchronized (CassandraOnHeapGraph.class)
+        {
+            // build encoder (expensive for PQ, cheaper for BQ)
+            if (preferredCompression.type == CompressionType.PRODUCT_QUANTIZATION)
+            {
+                var pqi = getPqIfPresent(indexContext, preferredCompression::equals);
+                compressor = computeOrRefineFrom(pqi, preferredCompression);
+            }
+            else
+            {
+                assert preferredCompression.type == CompressionType.BINARY_QUANTIZATION : preferredCompression.type;
+                compressor = BinaryQuantization.compute(vectorValues);
+            }
+            assert !vectorValues.isValueShared();
+            // encode (compress) the vectors to save
+            if (compressor != null)
+                cv = compressor.encodeAll(new RemappedVectorValues(remapped, remapped.maxNewOrdinal, vectorValues));
+
+            containsUnitVectors = IntStream.range(0, vectorValues.size())
+                                           .parallel()
+                                           .mapToObj(vectorValues::getVector)
+                                           .allMatch(v -> Math.abs(VectorUtil.dotProduct(v, v) - 1.0f) < 0.01);
+        }
+
+        var actualType = compressor == null ? CompressionType.NONE : preferredCompression.type;
+        writePqHeader(writer, containsUnitVectors, actualType);
+        if (actualType == CompressionType.NONE)
+            return writer.position();
+
+        // save (outside the synchronized block, this is io-bound not CPU)
+        cv.write(writer, JVECTOR_VERSION);
+        return writer.position();
+    }
+
+    static void writePqHeader(DataOutput writer, boolean unitVectors, CompressionType type)
+    throws IOException
+    {
+        if (V3OnDiskFormat.JVECTOR_VERSION >= 3)
+        {
+            // version and optional fields
+            writer.writeInt(CassandraDiskAnn.PQ_MAGIC);
+            writer.writeInt(PQVersion.V1.ordinal());
+            writer.writeBoolean(unitVectors);
+        }
+
+        // write the compression type
+        writer.writeByte(type.ordinal());
+    }
+
+    ProductQuantization computeOrRefineFrom(PqInfo existingInfo, VectorCompression preferredCompression)
+    {
+        if (existingInfo == null)
+        {
+            // no previous PQ, compute a new one if we have enough rows to do it
+            if (vectorValues.size() < MIN_PQ_ROWS)
+                return null;
+            else
+                return ProductQuantization.compute(vectorValues, preferredCompression.getCompressedSize(), 256, false);
+        }
+
+        // use the existing one unmodified if we either don't have enough rows to fine-tune, or
+        // the existing one was built with a large enough set
+        var existingPQ = existingInfo.pq;
+        if (vectorValues.size() < MIN_PQ_ROWS || existingInfo.rowCount >= ProductQuantization.MAX_PQ_TRAINING_SET_SIZE)
+            return existingPQ;
+
+        // refine the existing one
+        return existingPQ.refine(vectorValues);
+    }
+
+    public long ramBytesUsed()
+    {
+        return postingsBytesUsed() + vectorValues.ramBytesUsed() + builder.getGraph().ramBytesUsed();
+    }
+
+    private long postingsBytesUsed()
+    {
+        return RamEstimation.denseIntMapRamUsed(postingsByOrdinal.size())
+               + 3 * RamEstimation.concurrentHashMapRamUsed(postingsMap.size()) // CSLM is much less efficient than CHM
+               + postingsMap.values().stream().mapToLong(VectorPostings::ramBytesUsed).sum();
+    }
+
+    public enum InvalidVectorBehavior
+    {
+        IGNORE,
+        FAIL
+    }
+
+    public static class PqInfo
+    {
+        public final ProductQuantization pq;
+        /** an empty Optional indicates that the index was written with an older version that did not record this information */
+        public final boolean unitVectors;
+        public final long rowCount;
+
+        public PqInfo(ProductQuantization pq, boolean unitVectors, long rowCount)
+        {
+            this.pq = pq;
+            this.unitVectors = unitVectors;
+            this.rowCount = rowCount;
+        }
+    }
+
+    /** ensures that the graph is connected -- normally not necessary but it can help tests reason about the state */
+    @VisibleForTesting
+    public void cleanup()
+    {
+        builder.cleanup();
+    }
+
+    /**
+     * A simple wrapper that remaps the ordinals in the vector values to the new ordinals
+     */
+    private static class RemappedVectorValues implements RandomAccessVectorValues
+    {
+        final V5VectorPostingsWriter.RemappedPostings remapped;
+        final int maxNewOrdinal;
+        final RandomAccessVectorValues vectorValues;
+
+        RemappedVectorValues(V5VectorPostingsWriter.RemappedPostings remapped, int maxNewOrdinal, RandomAccessVectorValues vectorValues)
+        {
+            this.remapped = remapped;
+            this.maxNewOrdinal = maxNewOrdinal;
+            this.vectorValues = vectorValues;
+        }
+
+        @Override
+        public int size()
+        {
+            return maxNewOrdinal + 1;
+        }
+
+        @Override
+        public int dimension()
+        {
+            return vectorValues.dimension();
+        }
+
+        @Override
+        public VectorFloat<?> getVector(int i)
+        {
+            var oldOrdinal = remapped.ordinalMapper.newToOld(i);
+            return oldOrdinal == OrdinalMapper.OMITTED ? null : vectorValues.getVector(oldOrdinal);
+        }
+
+        @Override
+        public boolean isValueShared()
+        {
+            return vectorValues.isValueShared();
+        }
+
+        @Override
+        public RandomAccessVectorValues copy()
+        {
+            return new RemappedVectorValues(remapped, maxNewOrdinal, vectorValues.copy());
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/CloseableReranker.java b/src/java/org/apache/cassandra/index/sai/disk/vector/CloseableReranker.java
new file mode 100644
index 000000000000..b85b33c81b55
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/CloseableReranker.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.Closeable;
+
+import io.github.jbellis.jvector.graph.GraphIndex;
+import io.github.jbellis.jvector.graph.similarity.ScoreFunction;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * An ExactScoreFunction that closes the underlying {@link GraphIndex.ScoringView} when closed.
+ */
+public class CloseableReranker implements ScoreFunction.ExactScoreFunction, Closeable
+{
+    private final GraphIndex.ScoringView view;
+    private final ExactScoreFunction scoreFunction;
+
+    public CloseableReranker(VectorSimilarityFunction similarityFunction, VectorFloat<?> queryVector, GraphIndex.ScoringView view)
+    {
+        this.view = view;
+        this.scoreFunction = view.rerankerFor(queryVector, similarityFunction);
+    }
+
+    @Override
+    public float similarityTo(int i)
+    {
+        return scoreFunction.similarityTo(i);
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(view);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/CompactionGraph.java b/src/java/org/apache/cassandra/index/sai/disk/vector/CompactionGraph.java
new file mode 100644
index 000000000000..5475c996ae0e
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/CompactionGraph.java
@@ -0,0 +1,536 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.IntStream;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.graph.GraphIndexBuilder;
+import io.github.jbellis.jvector.graph.ListRandomAccessVectorValues;
+import io.github.jbellis.jvector.graph.disk.feature.Feature;
+import io.github.jbellis.jvector.graph.disk.feature.FeatureId;
+import io.github.jbellis.jvector.graph.disk.feature.InlineVectors;
+import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex;
+import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndexWriter;
+import io.github.jbellis.jvector.graph.disk.OrdinalMapper;
+import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider;
+import io.github.jbellis.jvector.quantization.BQVectors;
+import io.github.jbellis.jvector.quantization.BinaryQuantization;
+import io.github.jbellis.jvector.quantization.MutableBQVectors;
+import io.github.jbellis.jvector.quantization.MutableCompressedVectors;
+import io.github.jbellis.jvector.quantization.MutablePQVectors;
+import io.github.jbellis.jvector.quantization.PQVectors;
+import io.github.jbellis.jvector.quantization.ProductQuantization;
+import io.github.jbellis.jvector.quantization.VectorCompressor;
+import io.github.jbellis.jvector.util.Accountable;
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import net.openhft.chronicle.bytes.Bytes;
+import net.openhft.chronicle.hash.serialization.BytesReader;
+import net.openhft.chronicle.hash.serialization.BytesWriter;
+import net.openhft.chronicle.map.ChronicleMap;
+import net.openhft.chronicle.map.ChronicleMapBuilder;
+import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.agrona.collections.Int2ObjectHashMap;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorPostingsWriter;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v5.V5OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter.Structure;
+import org.apache.cassandra.index.sai.disk.vector.VectorPostings.CompactionVectorPostings;
+import org.apache.cassandra.index.sai.utils.LowPriorityThreadFactory;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat.JVECTOR_VERSION;
+
+public class CompactionGraph implements Closeable, Accountable
+{
+    private static final Logger logger = LoggerFactory.getLogger(CompactionGraph.class);
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    private static final ForkJoinPool compactionFjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), // checkstyle: permit this instantiation
+                                                                       new LowPriorityThreadFactory(),
+                                                                       null,
+                                                                       false);
+    // see comments to JVector PhysicalCoreExecutor -- HT tends to cause contention for the SIMD units
+    private static final ForkJoinPool compactionSimdPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors() / 2, // checkstyle: permit this instantiation
+                                                                            new LowPriorityThreadFactory(),
+                                                                            null,
+                                                                            false);
+
+    @VisibleForTesting
+    public static int PQ_TRAINING_SIZE = ProductQuantization.MAX_PQ_TRAINING_SET_SIZE;
+
+    private final VectorType.VectorSerializer serializer;
+    private final VectorSimilarityFunction similarityFunction;
+    private final ChronicleMap<VectorFloat<?>, CompactionVectorPostings> postingsMap;
+    private final IndexComponents.ForWrite perIndexComponents;
+    private final IndexContext context;
+    private final boolean unitVectors;
+    private final int postingsEntriesAllocated;
+    private final File postingsFile;
+    private final File termsFile;
+    private final int dimension;
+    private Structure postingsStructure;
+    private OnDiskGraphIndexWriter writer;
+    private final long termsOffset;
+    private int lastRowId = -1;
+    // if `useSyntheticOrdinals` is true then we use `nextOrdinal` to avoid holes, otherwise use rowId as source of ordinals
+    private final boolean useSyntheticOrdinals;
+    private int nextOrdinal = 0;
+
+    // protects the fine-tuning changes (done in maybeAddVector) from addGraphNode threads
+    // (and creates happens-before events so we don't need to mark the other fields volatile)
+    private final ReadWriteLock trainingLock = new ReentrantReadWriteLock();
+    private boolean pqFinetuned = false;
+    // not final; will be updated to different objects after fine-tuning
+    private VectorCompressor<?> compressor;
+    private MutableCompressedVectors compressedVectors;
+    private GraphIndexBuilder builder;
+
+    public CompactionGraph(IndexComponents.ForWrite perIndexComponents, VectorCompressor<?> compressor, boolean unitVectors, long keyCount, boolean allRowsHaveVectors) throws IOException
+    {
+        this.perIndexComponents = perIndexComponents;
+        this.context = perIndexComponents.context();
+        this.unitVectors = unitVectors;
+        var indexConfig = context.getIndexWriterConfig();
+        var termComparator = context.getValidator();
+        dimension = ((VectorType<?>) termComparator).dimension;
+
+        // We need to tell Chronicle Map (CM) how many entries to expect.  it's critical not to undercount,
+        // or CM will crash.  However, we don't want to just pass in a max entries count of 2B, since it eagerly
+        // allocated segments for that many entries, which takes about 25s.
+        //
+        // If our estimate turns out to be too small, it's not the end of the world, we'll flush this segment
+        // and start another to avoid crashing CM.  But we'd rather not do this because the whole goal of
+        // CompactionGraph is to write one segment only.
+        var dd = perIndexComponents.descriptor();
+        var rowsPerKey = max(1, Keyspace.open(dd.ksname).getColumnFamilyStore(dd.cfname).getMeanRowsPerPartition());
+        long estimatedRows = (long) (1.1 * keyCount * rowsPerKey); // 10% fudge factor
+        int maxRowsInGraph = Integer.MAX_VALUE - 100_000; // leave room for a few more async additions until we flush
+        postingsEntriesAllocated = max(1000, (int) min(estimatedRows, maxRowsInGraph));
+
+        serializer = (VectorType.VectorSerializer) termComparator.getSerializer();
+        similarityFunction = indexConfig.getSimilarityFunction();
+        postingsStructure = Structure.ONE_TO_ONE; // until proven otherwise
+        this.compressor = compressor;
+        // `allRowsHaveVectors` only tells us about data for which we have already built indexes; if we
+        // are adding previously unindexed data then we could still encounter rows with null vectors,
+        // so this is just a best guess.  If the guess is wrong then the penalty is that we end up
+        // with "holes" in the ordinal sequence (and pq and data files) which we would prefer to avoid
+        // (hence the effort to predict `allRowsHaveVectors`) but will not cause correctness issues,
+        // and the next compaction will fill in the holes.
+        this.useSyntheticOrdinals = !V5OnDiskFormat.writeV5VectorPostings() || !allRowsHaveVectors;
+
+        // the extension here is important to signal to CFS.scrubDataDirectories that it should be removed if present at restart
+        Component tmpComponent = new Component(SSTableFormat.Components.Types.CUSTOM, "chronicle" + Descriptor.TMP_EXT);
+        postingsFile = dd.fileFor(tmpComponent);
+        postingsMap = ChronicleMapBuilder.of((Class<VectorFloat<?>>) (Class) VectorFloat.class, (Class<CompactionVectorPostings>) (Class) CompactionVectorPostings.class)
+                                         .averageKeySize(dimension * Float.BYTES)
+                                         .averageValueSize(VectorPostings.emptyBytesUsed() + RamUsageEstimator.NUM_BYTES_OBJECT_REF + 2 * Integer.BYTES)
+                                         .keyMarshaller(new VectorFloatMarshaller())
+                                         .valueMarshaller(new VectorPostings.Marshaller())
+                                         .entries(postingsEntriesAllocated)
+                                         .createPersistedTo(postingsFile.toJavaIOFile());
+
+        // VSTODO add LVQ
+        BuildScoreProvider bsp;
+        if (compressor instanceof ProductQuantization)
+        {
+            compressedVectors = new MutablePQVectors((ProductQuantization) compressor);
+            bsp = BuildScoreProvider.pqBuildScoreProvider(similarityFunction, (PQVectors) compressedVectors);
+        }
+        else if (compressor instanceof BinaryQuantization)
+        {
+            var bq = new BinaryQuantization(dimension);
+            compressedVectors = new MutableBQVectors(bq);
+            bsp = BuildScoreProvider.bqBuildScoreProvider((BQVectors) compressedVectors);
+        }
+        else
+        {
+            throw new IllegalArgumentException("Unsupported compressor: " + compressor);
+        }
+        if (indexConfig.isHierarchyEnabled() && V3OnDiskFormat.JVECTOR_VERSION < 4)
+            logger.warn("Hierarchical graphs configured but node configured with V3OnDiskFormat.JVECTOR_VERSION {}. " +
+                        "Skipping setting for {}", V3OnDiskFormat.JVECTOR_VERSION, indexConfig.getIndexName());
+
+        builder = new GraphIndexBuilder(bsp,
+                                        dimension,
+                                        indexConfig.getAnnMaxDegree(),
+                                        indexConfig.getConstructionBeamWidth(),
+                                        indexConfig.getNeighborhoodOverflow(1.2f),
+                                        indexConfig.getAlpha(dimension > 3 ? 1.2f : 1.4f),
+                                        indexConfig.isHierarchyEnabled() && V3OnDiskFormat.JVECTOR_VERSION >= 4,
+                                        compactionSimdPool, compactionFjp);
+
+        termsFile = perIndexComponents.addOrGet(IndexComponentType.TERMS_DATA).file();
+        termsOffset = (termsFile.exists() ? termsFile.length() : 0)
+                      + SAICodecUtils.headerSize();
+        // placeholder writer, will be replaced at flush time when we finalize the index contents
+        writer = createTermsWriterBuilder().withMapper(new OrdinalMapper.IdentityMapper(maxRowsInGraph)).build();
+        writer.getOutput().seek(termsFile.length()); // position at the end of the previous segment before writing our own header
+        SAICodecUtils.writeHeader(SAICodecUtils.toLuceneOutput(writer.getOutput()));
+    }
+
+    private OnDiskGraphIndexWriter.Builder createTermsWriterBuilder() throws IOException
+    {
+        return new OnDiskGraphIndexWriter.Builder(builder.getGraph(), termsFile.toPath())
+               .withStartOffset(termsOffset)
+               .with(new InlineVectors(dimension))
+               .withVersion(JVECTOR_VERSION);
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        // this gets called in `finally` blocks, so use closeQuietly to avoid generating additional exceptions
+        FileUtils.closeQuietly(writer);
+        FileUtils.closeQuietly(postingsMap);
+        Files.delete(postingsFile.toJavaIOFile().toPath());
+    }
+
+    public int size()
+    {
+        return builder.getGraph().size();
+    }
+
+    public boolean isEmpty()
+    {
+        return postingsMap.values().stream().allMatch(VectorPostings::isEmpty);
+    }
+
+    /**
+     * @return the result of adding the given (vector) term; see {@link InsertionResult}
+     */
+    public InsertionResult maybeAddVector(ByteBuffer term, int segmentRowId) throws IOException
+    {
+        assert term != null && term.remaining() != 0;
+
+        var vector = vts.createFloatVector(serializer.deserializeFloatArray(term));
+        // Validate the vector.  Since we are compacting, invalid vectors are ignored instead of failing the operation.
+        try
+        {
+            VectorValidation.validateIndexable(vector, similarityFunction);
+        }
+        catch (InvalidRequestException e)
+        {
+            if (StorageService.instance.isInitialized())
+                logger.trace("Ignoring invalid vector during index build against existing data: {}", (Object) e);
+            else
+                logger.trace("Ignoring invalid vector during commitlog replay: {}", (Object) e);
+            return new InsertionResult(0);
+        }
+
+        // if we don't see sequential rowids, it means the skipped row(s) have null vectors
+        if (segmentRowId != lastRowId + 1)
+            postingsStructure = Structure.ZERO_OR_ONE_TO_MANY;
+        lastRowId = segmentRowId;
+
+        var bytesUsed = 0L;
+        var postings = postingsMap.get(vector);
+        if (postings == null)
+        {
+            // add a new entry
+            // this all runs on the same compaction thread, so we don't need to worry about concurrency
+            int ordinal = useSyntheticOrdinals ? nextOrdinal++ : segmentRowId;
+            postings = new CompactionVectorPostings(ordinal, segmentRowId);
+            postingsMap.put(vector, postings);
+
+            // fine-tune the PQ if we've collected enough vectors
+            if (compressor instanceof ProductQuantization && !pqFinetuned && postingsMap.size() >= PQ_TRAINING_SIZE)
+            {
+                // walk the on-disk Postings once to build (1) a dense list of vectors with no missing entries or zeros
+                // and (2) a map of vectors keyed by ordinal
+                var trainingVectors = new ArrayList<VectorFloat<?>>(postingsMap.size());
+                var vectorsByOrdinal = new Int2ObjectHashMap<VectorFloat<?>>();
+                postingsMap.forEach((v, p) -> {
+                    var vectorClone = v.copy();
+                    trainingVectors.add(vectorClone);
+                    vectorsByOrdinal.put(p.getOrdinal(), vectorClone);
+                });
+
+                // lock the addGraphNode threads out so they don't try to use old pq codepoints against the new codebook
+                trainingLock.writeLock().lock();
+                try
+                {
+                    // Fine tune the pq codebook
+                    compressor = ((ProductQuantization) compressor).refine(new ListRandomAccessVectorValues(trainingVectors, dimension));
+                    trainingVectors.clear(); // don't need these anymore so let GC reclaim if it wants to
+
+                    // re-encode the vectors added so far
+                    int encodedVectorCount = compressedVectors.count();
+                    compressedVectors = new MutablePQVectors((ProductQuantization) compressor);
+                    compactionFjp.submit(() -> {
+                        IntStream.range(0, encodedVectorCount)
+                                 .parallel()
+                                 .forEach(i -> {
+                                     var v = vectorsByOrdinal.get(i);
+                                     if (v == null)
+                                         compressedVectors.setZero(i);
+                                     else
+                                         compressedVectors.encodeAndSet(i, v);
+                                 });
+                    }).join();
+
+                    // Keep the existing edges but recompute their scores
+                    builder = GraphIndexBuilder.rescore(builder, BuildScoreProvider.pqBuildScoreProvider(similarityFunction, (PQVectors) compressedVectors));
+                }
+                finally
+                {
+                    trainingLock.writeLock().unlock();
+                }
+                pqFinetuned = true;
+            }
+
+            writer.writeInline(ordinal, Feature.singleState(FeatureId.INLINE_VECTORS, new InlineVectors.State(vector)));
+            // Fill in any holes in the pqVectors (setZero has the side effect of increasing the count)
+            while (compressedVectors.count() < ordinal)
+                compressedVectors.setZero(compressedVectors.count());
+            compressedVectors.encodeAndSet(ordinal, vector);
+
+            bytesUsed += postings.ramBytesUsed();
+            return new InsertionResult(bytesUsed, ordinal, vector);
+        }
+
+        // postings list already exists, just add the new key
+        if (postingsStructure == Structure.ONE_TO_ONE)
+            postingsStructure = Structure.ONE_TO_MANY;
+        var newPosting = postings.add(segmentRowId);
+        assert newPosting;
+        bytesUsed += postings.bytesPerPosting();
+        postingsMap.put(vector, postings); // re-serialize to disk
+
+        return new InsertionResult(bytesUsed);
+    }
+
+    public long addGraphNode(InsertionResult result)
+    {
+        trainingLock.readLock().lock();
+        try
+        {
+            return builder.addGraphNode(result.ordinal, result.vector);
+        }
+        finally
+        {
+            trainingLock.readLock().unlock();
+        }
+    }
+
+    public SegmentMetadata.ComponentMetadataMap flush() throws IOException
+    {
+        // header is required to write the postings, but we need to recreate the writer after that with an accurate OrdinalMapper
+        writer.writeHeader();
+        writer.close();
+
+        int nInProgress = builder.insertsInProgress();
+        assert nInProgress == 0 : String.format("Attempting to write graph while %d inserts are in progress", nInProgress);
+        assert !useSyntheticOrdinals || nextOrdinal == builder.getGraph().size() : String.format("nextOrdinal %d != graph size %d -- ordinals should be sequential",
+                                                                                                 nextOrdinal, builder.getGraph().size());
+        assert compressedVectors.count() == builder.getGraph().getIdUpperBound() : String.format("Largest vector id %d != largest graph id %d",
+                                                                                                 compressedVectors.count(), builder.getGraph().getIdUpperBound());
+        assert postingsMap.keySet().size() == builder.getGraph().size() : String.format("postings map entry count %d != vector count %d",
+                                                                                        postingsMap.keySet().size(), builder.getGraph().size());
+        if (logger.isDebugEnabled())
+        {
+            logger.debug("Writing graph with {} rows and {} distinct vectors",
+                         postingsMap.values().stream().mapToInt(VectorPostings::size).sum(), builder.getGraph().size());
+            logger.debug("Estimated size is {} + {}", compressedVectors.ramBytesUsed(), builder.getGraph().ramBytesUsed());
+        }
+
+        try (var postingsOutput = perIndexComponents.addOrGet(IndexComponentType.POSTING_LISTS).openOutput(true);
+             var pqOutput = perIndexComponents.addOrGet(IndexComponentType.PQ).openOutput(true))
+        {
+            SAICodecUtils.writeHeader(postingsOutput);
+            SAICodecUtils.writeHeader(pqOutput);
+
+            // write PQ (time to do this is negligible, don't bother doing it async)
+            long pqOffset = pqOutput.getFilePointer();
+            CassandraOnHeapGraph.writePqHeader(pqOutput.asSequentialWriter(), unitVectors, VectorCompression.CompressionType.PRODUCT_QUANTIZATION);
+            compressedVectors.write(pqOutput.asSequentialWriter(), JVECTOR_VERSION); // VSTODO old version until we add APQ
+            long pqLength = pqOutput.getFilePointer() - pqOffset;
+
+            // write postings asynchronously while we run cleanup()
+            var ordinalMapper = new AtomicReference<OrdinalMapper>();
+            long postingsOffset = postingsOutput.getFilePointer();
+            var es = ExecutorFactory.Global.executorFactory().sequential("CompactionGraphPostingsWriter");
+            long postingsLength;
+            try (var indexHandle = perIndexComponents.get(IndexComponentType.TERMS_DATA).createIndexBuildTimeFileHandle();
+                 var index = OnDiskGraphIndex.load(indexHandle::createReader, termsOffset))
+            {
+                var postingsFuture = es.submit(() -> {
+                    // V2 doesn't support ONE_TO_MANY so force it to ZERO_OR_ONE_TO_MANY if necessary;
+                    // similarly, if we've been using synthetic ordinals then we can't map to ONE_TO_MANY
+                    // (ending up at ONE_TO_MANY when the source sstables were not is unusual, but possible,
+                    // if a row with null vector in sstable A gets updated with a vector in sstable B)
+                    if (postingsStructure == Structure.ONE_TO_MANY
+                        && (!V5OnDiskFormat.writeV5VectorPostings() || useSyntheticOrdinals))
+                    {
+                        postingsStructure = Structure.ZERO_OR_ONE_TO_MANY;
+                    }
+                    var rp = V5VectorPostingsWriter.describeForCompaction(postingsStructure,
+                                                                          builder.getGraph().size(),
+                                                                          postingsMap);
+                    ordinalMapper.set(rp.ordinalMapper);
+                    try (var view = index.getView())
+                    {
+                        if (V5OnDiskFormat.writeV5VectorPostings())
+                        {
+                            return new V5VectorPostingsWriter<Integer>(rp).writePostings(postingsOutput.asSequentialWriter(), view, postingsMap);
+                        }
+                        else
+                        {
+                            assert postingsStructure == Structure.ONE_TO_ONE || postingsStructure == Structure.ZERO_OR_ONE_TO_MANY;
+                            return new V2VectorPostingsWriter<Integer>(postingsStructure == Structure.ONE_TO_ONE, builder.getGraph().size(), rp.ordinalMapper::newToOld)
+                                   .writePostings(postingsOutput.asSequentialWriter(), view, postingsMap, Set.of());
+                        }
+                    }
+                });
+
+                // complete internal graph clean up
+                builder.cleanup();
+
+                // wait for postings to finish writing and clean up related resources
+                long postingsEnd = postingsFuture.get();
+                postingsLength = postingsEnd - postingsOffset;
+                es.shutdown();
+            }
+
+            // Recreate the writer with the final ordinalMapper
+            writer = createTermsWriterBuilder().withMapper(ordinalMapper.get()).build();
+
+            // write the graph edge lists and optionally fused adc features
+            var start = nanoTime();
+            // Required becuase jvector 3 wrote the fused adc map here. We no longer write jvector 3, but we still
+            // write out the empty map.
+            writer.write(Map.of());
+            SAICodecUtils.writeFooter(writer.getOutput(), writer.checksum());
+            logger.info("Writing graph took {}ms", (nanoTime() - start) / 1_000_000);
+            long termsLength = writer.getOutput().position() - termsOffset;
+
+            // write remaining footers/checksums
+            SAICodecUtils.writeFooter(pqOutput);
+            SAICodecUtils.writeFooter(postingsOutput);
+
+            // add components to the metadata map
+            return CassandraOnHeapGraph.createMetadataMap(termsOffset, termsLength, postingsOffset, postingsLength, pqOffset, pqLength);
+        }
+        catch (ExecutionException | InterruptedException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public long ramBytesUsed()
+    {
+        return compressedVectors.ramBytesUsed() + builder.getGraph().ramBytesUsed();
+    }
+
+    public boolean requiresFlush()
+    {
+        return builder.getGraph().size() >= postingsEntriesAllocated;
+    }
+
+    private static class VectorFloatMarshaller implements BytesReader<VectorFloat<?>>, BytesWriter<VectorFloat<?>> {
+        @Override
+        public void write(Bytes out, VectorFloat<?> vector) {
+            out.writeInt(vector.length());
+            for (int i = 0; i < vector.length(); i++) {
+                out.writeFloat(vector.get(i));
+            }
+        }
+
+        @Override
+        public VectorFloat<?> read(Bytes in, VectorFloat<?> using) {
+            int length = in.readInt();
+            if (using == null) {
+                float[] data = new float[length];
+                for (int i = 0; i < length; i++) {
+                    data[i] = in.readFloat();
+                }
+                return vts.createFloatVector(data);
+            }
+
+            for (int i = 0; i < length; i++) {
+                using.set(i, in.readFloat());
+            }
+            return using;
+        }
+    }
+
+    /**
+     * AddResult is a container for the result of maybeAddVector.  If this call resulted in a new
+     * vector being added to the graph, then `ordinal` and `vector` fields will be populated, otherwise
+     * they will be null.
+     * <p>
+     * bytesUsed is always populated and always non-negative (it will be smaller, but not zero,
+     * when adding a vector that already exists in the graph to a new row).
+     */
+    public static class InsertionResult
+    {
+        public final long bytesUsed;
+        public final Integer ordinal;
+        public final VectorFloat<?> vector;
+
+        public InsertionResult(long bytesUsed, Integer ordinal, VectorFloat<?> vector)
+        {
+            this.bytesUsed = bytesUsed;
+            this.ordinal = ordinal;
+            this.vector = vector;
+        }
+
+        public InsertionResult(long bytesUsed)
+        {
+            this(bytesUsed, null, null);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/ConcurrentVectorValues.java b/src/java/org/apache/cassandra/index/sai/disk/vector/ConcurrentVectorValues.java
new file mode 100644
index 000000000000..db77f95dd797
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/ConcurrentVectorValues.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.FloatBuffer;
+import java.util.function.IntUnaryOperator;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.github.jbellis.jvector.util.DenseIntMap;
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import io.github.jbellis.jvector.vector.ArrayVectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import org.apache.cassandra.io.util.SequentialWriter;
+
+public class ConcurrentVectorValues implements RamAwareVectorValues
+{
+    private final int dimensions;
+    private final DenseIntMap<VectorFloat<?>> values = new DenseIntMap<>(1024);
+
+    public ConcurrentVectorValues(int dimensions)
+    {
+        this.dimensions = dimensions;
+    }
+
+    @Override
+    public int size()
+    {
+        return values.size();
+    }
+
+    @Override
+    public int dimension()
+    {
+        return dimensions;
+    }
+
+    @Override
+    public VectorFloat<?> getVector(int i)
+    {
+        return values.get(i);
+    }
+
+    /** return approximate bytes used by the new vector */
+    public long add(int ordinal, VectorFloat<?> vector)
+    {
+        if (!values.compareAndPut(ordinal, null, vector))
+            throw new IllegalStateException("Vector already exists for ordinal " + ordinal);
+        return RamUsageEstimator.NUM_BYTES_OBJECT_REF + oneVectorBytesUsed();
+    }
+
+    @Override
+    public boolean isValueShared()
+    {
+        return false;
+    }
+
+    @Override
+    public ConcurrentVectorValues copy()
+    {
+        // no actual copy required because we always return distinct float[] for distinct vector ordinals
+        return this;
+    }
+
+    public long ramBytesUsed()
+    {
+        long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+        return 2 * REF_BYTES
+               + RamEstimation.denseIntMapRamUsed(values.size())
+               + values.size() * oneVectorBytesUsed();
+    }
+
+    private long oneVectorBytesUsed()
+    {
+        return Integer.BYTES + Integer.BYTES + (long) dimension() * Float.BYTES;
+    }
+
+    @VisibleForTesting
+    public long write(SequentialWriter writer, IntUnaryOperator ordinalMapper) throws IOException
+    {
+        writer.writeInt(size());
+        writer.writeInt(dimension());
+
+        for (var i = 0; i < size(); i++) {
+            int ord = ordinalMapper.applyAsInt(i);
+            var fb = FloatBuffer.wrap(((ArrayVectorFloat) values.get(ord)).get());
+            var bb = ByteBuffer.allocate(fb.capacity() * Float.BYTES);
+            bb.asFloatBuffer().put(fb);
+            writer.write(bb);
+        }
+
+        return writer.position();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/GraphSearcherAccessManager.java b/src/java/org/apache/cassandra/index/sai/disk/vector/GraphSearcherAccessManager.java
new file mode 100644
index 000000000000..0dc514a22848
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/GraphSearcherAccessManager.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import io.github.jbellis.jvector.graph.GraphSearcher;
+
+/**
+ * Manages access to a {@link GraphSearcher} instance, validating that we respect the contract of GraphSearcher
+ * to only use it in a single search at a time.
+ */
+@NotThreadSafe
+public class GraphSearcherAccessManager
+{
+    private final GraphSearcher searcher;
+    private final AtomicBoolean locked;
+
+    public GraphSearcherAccessManager(GraphSearcher searcher)
+    {
+        this.searcher = searcher;
+        this.locked = new AtomicBoolean(false);
+    }
+
+    /**
+     * Get the {@link GraphSearcher} instance, locking it to the current in-progress search.
+     */
+    public GraphSearcher get()
+    {
+        if (!locked.compareAndSet(false, true))
+            throw new IllegalStateException("GraphAccessManager is already locked");
+        return searcher;
+    }
+
+    /**
+     * Release the {@link GraphSearcher} instance, allowing it to be used in another search.
+     */
+    public void release()
+    {
+        if (!locked.compareAndSet(true, false))
+            throw new IllegalStateException("GraphAccessManager is already unlocked");
+    }
+
+    /**
+     * Release the {@link GraphSearcher} instance, allowing it to be used in another search,
+     * without confirming its state. Inteaded for use in exceptional code paths.
+     */
+    public void forceRelease()
+    {
+        locked.set(false);
+    }
+
+    /**
+     * Close the {@link GraphSearcher} instance.  It cannot be used again after being closed.
+     */
+    public void close() throws IOException
+    {
+        searcher.close();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/NodeQueueRowIdIterator.java b/src/java/org/apache/cassandra/index/sai/disk/vector/NodeQueueRowIdIterator.java
new file mode 100644
index 000000000000..e47733158edb
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/NodeQueueRowIdIterator.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import io.github.jbellis.jvector.graph.NodeQueue;
+import org.apache.cassandra.index.sai.utils.RowIdWithScore;
+import org.apache.cassandra.utils.AbstractIterator;
+
+/**
+ * An iterator over {@link RowIdWithScore} that lazily consumes a {@link NodeQueue}.
+ */
+public class NodeQueueRowIdIterator extends AbstractIterator<RowIdWithScore>
+{
+    private final NodeQueue scoreQueue;
+
+    public NodeQueueRowIdIterator(NodeQueue scoreQueue)
+    {
+        this.scoreQueue = scoreQueue;
+    }
+
+    @Override
+    protected RowIdWithScore computeNext()
+    {
+        if (scoreQueue.size() == 0)
+            return endOfData();
+        float score = scoreQueue.topScore();
+        int rowId = scoreQueue.pop();
+        return new RowIdWithScore(rowId, score);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/NodeScoreToRowIdWithScoreIterator.java b/src/java/org/apache/cassandra/index/sai/disk/vector/NodeScoreToRowIdWithScoreIterator.java
new file mode 100644
index 000000000000..958420c2175a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/NodeScoreToRowIdWithScoreIterator.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.util.PrimitiveIterator;
+import java.util.stream.IntStream;
+
+import io.github.jbellis.jvector.graph.SearchResult;
+import org.apache.cassandra.index.sai.utils.RowIdWithScore;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CloseableIterator;
+
+/**
+ * An iterator over {@link RowIdWithScore} sorted by score descending. The iterator converts ordinals (node ids) to
+ * segment row ids and pairs them with the score given by the index.
+ */
+public class NodeScoreToRowIdWithScoreIterator extends AbstractIterator<RowIdWithScore>
+{
+    private final CloseableIterator<SearchResult.NodeScore> nodeScores;
+    private final RowIdsView rowIdsView;
+
+    private PrimitiveIterator.OfInt segmentRowIdIterator = IntStream.empty().iterator();
+    private float currentScore;
+
+    public NodeScoreToRowIdWithScoreIterator(CloseableIterator<SearchResult.NodeScore> nodeScores,
+                                             RowIdsView rowIdsView)
+    {
+        this.nodeScores = nodeScores;
+        this.rowIdsView = rowIdsView;
+    }
+
+    @Override
+    protected RowIdWithScore computeNext()
+    {
+        try
+        {
+            if (segmentRowIdIterator.hasNext())
+                return new RowIdWithScore(segmentRowIdIterator.nextInt(), currentScore);
+
+            while (nodeScores.hasNext())
+            {
+                SearchResult.NodeScore result = nodeScores.next();
+                currentScore = result.score;
+                var ordinal = result.node;
+                segmentRowIdIterator = rowIdsView.getSegmentRowIdsMatching(ordinal);
+                if (segmentRowIdIterator.hasNext())
+                    return new RowIdWithScore(segmentRowIdIterator.nextInt(), currentScore);
+            }
+            return endOfData();
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(rowIdsView, nodeScores);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/OnDiskOrdinalsMap.java b/src/java/org/apache/cassandra/index/sai/disk/vector/OnDiskOrdinalsMap.java
new file mode 100644
index 000000000000..87d22d785718
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/OnDiskOrdinalsMap.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.util.PrimitiveIterator;
+import java.util.function.Supplier;
+
+import io.github.jbellis.jvector.util.BitSet;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.util.SparseBits;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter;
+import org.apache.cassandra.index.sai.utils.SingletonIntIterator;
+
+public interface OnDiskOrdinalsMap extends AutoCloseable
+{
+    /** maps from vector ordinals returned by index search to rowids in the sstable */
+    RowIdsView getRowIdsView();
+
+    default Bits ignoringDeleted(Bits acceptBits) {
+        return acceptBits;
+    }
+
+    /** maps from rowids to their associated ordinals, for setting up the ordinals-to-accept in a restricted search */
+    OrdinalsView getOrdinalsView();
+
+    void close();
+
+    V5VectorPostingsWriter.Structure getStructure();
+
+    /**
+     * Ignoring the constant overhead of the object, return the variable overhead of the object. This helps
+     * identify the cost of caching.
+     */
+    long cachedBytesUsed();
+
+    class OneToOneRowIdsView implements RowIdsView {
+
+        @Override
+        public PrimitiveIterator.OfInt getSegmentRowIdsMatching(int vectorOrdinal) throws IOException
+        {
+            return new SingletonIntIterator(vectorOrdinal);
+        }
+
+        @Override
+        public void close()
+        {
+            // noop
+        }
+    }
+
+    class EmptyRowIdsView implements RowIdsView
+    {
+        @Override
+        public PrimitiveIterator.OfInt getSegmentRowIdsMatching(int vectorOrdinal) throws IOException
+        {
+            return new PrimitiveIterator.OfInt()
+            {
+                @Override
+                public int nextInt()
+                {
+                    throw new IllegalStateException();
+                }
+
+                @Override
+                public boolean hasNext()
+                {
+                    return false;
+                }
+            };
+        }
+
+        @Override
+        public void close()
+        {
+            // noop
+        }
+    }
+
+    /**
+     * An OrdinalsView that always returns -1 for all rowIds. This is used when the segment has no postings, which
+     * can happen if all the graph's ordinals are in the deletedOrdinals set.
+     */
+    class EmptyOrdinalsView implements OrdinalsView
+    {
+        @Override
+        public int getOrdinalForRowId(int rowId) throws IOException
+        {
+            return -1;
+        }
+
+        @Override
+        public void forEachOrdinalInRange(int startRowId, int endRowId, OrdinalConsumer consumer) throws IOException
+        {
+            // noop
+        }
+
+        @Override
+        public Bits buildOrdinalBits(int startRowId, int endRowId, Supplier<SparseBits> bitsSupplier) throws IOException
+        {
+            // Get an empty bitset
+            return bitsSupplier.get();
+        }
+
+        @Override
+        public void close()
+        {
+            // noop
+        }
+    }
+
+    /** Bits matching the given range, inclusively. */
+    class MatchRangeBits extends BitSet
+    {
+        final int lowerBound;
+        final int upperBound;
+
+        public MatchRangeBits(int lowerBound, int upperBound) {
+            // bitset is empty if lowerBound > upperBound
+            this.lowerBound = lowerBound;
+            this.upperBound = upperBound;
+        }
+
+        @Override
+        public boolean get(int index) {
+            return lowerBound <= index && index <= upperBound;
+        }
+
+        @Override
+        public int length() {
+            if (lowerBound > upperBound)
+                return 0;
+            return upperBound - lowerBound + 1;
+        }
+
+        @Override
+        public void set(int i)
+        {
+            throw new UnsupportedOperationException("not supported");
+        }
+
+        @Override
+        public boolean getAndSet(int i)
+        {
+            throw new UnsupportedOperationException("not supported");
+        }
+
+        @Override
+        public void clear(int i)
+        {
+            throw new UnsupportedOperationException("not supported");
+        }
+
+        @Override
+        public void clear(int i, int i1)
+        {
+            throw new UnsupportedOperationException("not supported");
+        }
+
+        @Override
+        public int cardinality()
+        {
+            return length();
+        }
+
+        @Override
+        public int approximateCardinality()
+        {
+            return length();
+        }
+
+        @Override
+        public int prevSetBit(int i)
+        {
+            throw new UnsupportedOperationException("not supported");
+        }
+
+        @Override
+        public int nextSetBit(int i)
+        {
+            throw new UnsupportedOperationException("not supported");
+        }
+
+        @Override
+        public long ramBytesUsed()
+        {
+            return 2 * Integer.BYTES;
+        }
+    }
+
+    class OneToOneOrdinalsView implements OrdinalsView
+    {
+        // The number of ordinals in the segment. If we see a rowId greater than or equal to this, we know it's not in
+        // the graph.
+        private final int size;
+
+        public OneToOneOrdinalsView(int size)
+        {
+            this.size = size;
+        }
+
+        @Override
+        public int getOrdinalForRowId(int rowId) throws IOException
+        {
+            if (rowId >= size)
+                return -1;
+            return rowId;
+        }
+
+        @Override
+        public void forEachOrdinalInRange(int startRowId, int endRowId, OrdinalConsumer consumer) throws IOException
+        {
+            // risk of overflow
+            assert endRowId < Integer.MAX_VALUE : "endRowId must be less than Integer.MAX_VALUE";
+            assert endRowId >= startRowId : "endRowId must be greater than or equal to startRowId";
+
+            int start = Math.max(startRowId, 0);
+            int end = Math.min(endRowId + 1, size);
+            for (int rowId = start; rowId < end; rowId++)
+                consumer.accept(rowId, rowId);
+        }
+
+        @Override
+        public BitSet buildOrdinalBits(int startRowId, int endRowId, Supplier<SparseBits> unused) throws IOException
+        {
+            int start = Math.max(startRowId, 0);
+            int end = Math.min(endRowId, size - 1);
+
+            return new MatchRangeBits(start, end);
+        }
+
+        @Override
+        public void close()
+        {
+            // noop
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/OptimizeFor.java b/src/java/org/apache/cassandra/index/sai/disk/vector/OptimizeFor.java
new file mode 100644
index 000000000000..7c6ab72f6cb7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/OptimizeFor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+public enum OptimizeFor
+{
+    LATENCY,
+    RECALL;
+
+    public static OptimizeFor fromString(String value)
+    {
+        return valueOf(value.toUpperCase());
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/OrdinalsView.java b/src/java/org/apache/cassandra/index/sai/disk/vector/OrdinalsView.java
new file mode 100644
index 000000000000..60890730a7b7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/OrdinalsView.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.util.SparseBits;
+
+public interface OrdinalsView extends AutoCloseable
+{
+    interface OrdinalConsumer
+    {
+        void accept(int rowId, int ordinal) throws IOException;
+    }
+
+    /** return the vector ordinal associated with the given row, or -1 if no vectors are associated with it */
+    int getOrdinalForRowId(int rowId) throws IOException;
+
+    /**
+     * iterates over all ordinals in the view.  order of iteration is undefined. Only calls consumer for valid mappings
+     * from row id to ordinal.
+     */
+    void forEachOrdinalInRange(int startRowId, int endRowId, OrdinalConsumer consumer) throws IOException;
+
+    default Bits buildOrdinalBits(int startRowId, int endRowId, Supplier<SparseBits> bitsSupplier) throws IOException
+    {
+        var bits = bitsSupplier.get();
+        this.forEachOrdinalInRange(startRowId, endRowId, (segmentRowId, ordinal) -> {
+            bits.set(ordinal);
+        });
+        return bits;
+    }
+
+    @Override
+    void close();
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/RamAwareVectorValues.java b/src/java/org/apache/cassandra/index/sai/disk/vector/RamAwareVectorValues.java
similarity index 90%
rename from src/java/org/apache/cassandra/index/sai/disk/v1/vector/RamAwareVectorValues.java
rename to src/java/org/apache/cassandra/index/sai/disk/vector/RamAwareVectorValues.java
index 4e76b443f4cb..d7ee9b0e7d9e 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/RamAwareVectorValues.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/RamAwareVectorValues.java
@@ -16,11 +16,11 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.index.sai.disk.v1.vector;
+package org.apache.cassandra.index.sai.disk.vector;
 
 import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
 
-public interface RamAwareVectorValues extends RandomAccessVectorValues<float[]>
+public interface RamAwareVectorValues extends RandomAccessVectorValues
 {
-    float[] vectorValue(int i);
+    long ramBytesUsed();
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/RamEstimation.java b/src/java/org/apache/cassandra/index/sai/disk/vector/RamEstimation.java
new file mode 100644
index 000000000000..bb1bee76a5f1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/RamEstimation.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+
+public class RamEstimation
+{
+    /**
+     * @param externalNodeCount the size() of the ConcurrentHashMap
+     * @return an estimate of the number of bytes used
+     */
+    public static long concurrentHashMapRamUsed(int externalNodeCount) {
+        long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+        long AH_BYTES = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+        long CORES = Runtime.getRuntime().availableProcessors();
+
+        long chmNodeBytes =
+        REF_BYTES // node itself in Node[]
+        + 3L * REF_BYTES
+        + Integer.BYTES; // node internals
+        float chmLoadFactor = 0.75f; // this is hardcoded inside ConcurrentHashMap
+        // CHM has a striped counter Cell implementation, we expect at most one per core
+        long chmCounters = AH_BYTES + CORES * (REF_BYTES + Long.BYTES);
+
+        double nodeCount = externalNodeCount / chmLoadFactor;
+
+        return
+        (long) nodeCount * (chmNodeBytes + REF_BYTES)// nodes
+        + AH_BYTES // nodes array
+        + Long.BYTES
+        + 3 * Integer.BYTES
+        + 3 * REF_BYTES // extra internal fields
+        + chmCounters
+        + REF_BYTES; // the Map reference itself
+    }
+
+    /**
+     * @param elementCount the size() of the DenseIntMap
+     * @return an estimate of the number of bytes used by a DenseIntMap
+     */
+    public static long denseIntMapRamUsed(int elementCount) {
+        long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+        long AH_BYTES = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+        long RWLOCK_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 3 * REF_BYTES; // Approx. size for ReadWriteLock
+        long ATOMIC_INT_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + Integer.BYTES + REF_BYTES; // AtomicInteger overhead
+
+        // Find power of 2 greater than or equal to elementCount
+        int capacity = 1;
+        while (capacity < elementCount) {
+            capacity <<= 1;
+        }
+
+        // Calculate size for AtomicReferenceArray with for capacity elements
+        long atomicRefArrayBytes = AH_BYTES + capacity * REF_BYTES;
+
+        return RWLOCK_BYTES // Size of the ReadWriteLock object
+               + ATOMIC_INT_BYTES // Size of the AtomicInteger
+               + atomicRefArrayBytes; // Size of the AtomicReferenceArray structure
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/RowIdsView.java b/src/java/org/apache/cassandra/index/sai/disk/vector/RowIdsView.java
new file mode 100644
index 000000000000..16d5db1ee6fe
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/RowIdsView.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.util.PrimitiveIterator;
+
+public interface RowIdsView extends AutoCloseable
+{
+    PrimitiveIterator.OfInt getSegmentRowIdsMatching(int vectorOrdinal) throws IOException;
+
+    @Override
+    void close();
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/VectorCompression.java b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorCompression.java
new file mode 100644
index 000000000000..33f3bd1b998d
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorCompression.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.util.Objects;
+
+public class VectorCompression
+{
+    public static final VectorCompression NO_COMPRESSION = new VectorCompression(CompressionType.NONE, -1, -1);
+
+    public final CompressionType type;
+    private final int originalSize; // in bytes
+    private final int compressedSize; // in bytes
+
+    public VectorCompression(CompressionType type, int dimension, double ratio)
+    {
+        this.type = type;
+        this.originalSize = dimension * Float.BYTES;
+        this.compressedSize = (int) (originalSize * ratio);
+    }
+
+    public VectorCompression(CompressionType type, int originalSize, int compressedSize)
+    {
+        this.type = type;
+        this.originalSize = originalSize;
+        this.compressedSize = compressedSize;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        VectorCompression that = (VectorCompression) o;
+        if (type == CompressionType.NONE)
+            return that.type == CompressionType.NONE;
+        return originalSize == that.originalSize && compressedSize == that.compressedSize && type == that.type;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(type, getOriginalSize(), getCompressedSize());
+    }
+
+    public String toString()
+    {
+        return String.format("VectorCompression(%s, %d->%d)", type, originalSize, compressedSize);
+    }
+
+    public int getOriginalSize()
+    {
+        if (type == CompressionType.NONE)
+            throw new UnsupportedOperationException();
+        return originalSize;
+    }
+
+    public int getCompressedSize()
+    {
+        if (type == CompressionType.NONE)
+            throw new UnsupportedOperationException();
+        return compressedSize;
+    }
+
+    public enum CompressionType
+    {
+        NONE,
+        PRODUCT_QUANTIZATION,
+        BINARY_QUANTIZATION
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/VectorMemtableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorMemtableIndex.java
new file mode 100644
index 000000000000..46827c87ccbf
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorMemtableIndex.java
@@ -0,0 +1,604 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+import java.util.function.ToIntFunction;
+import javax.annotation.Nullable;
+
+import com.google.common.util.concurrent.Runnables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.graph.SearchResult;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.agrona.collections.IntHashSet;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.memory.MemoryIndex;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithScore;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RangeUtil;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.SortingIterator;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static java.lang.Math.log;
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.lang.Math.pow;
+
+public class VectorMemtableIndex implements MemtableIndex
+{
+    private static final Logger logger = LoggerFactory.getLogger(VectorMemtableIndex.class);
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+    public static int GLOBAL_BRUTE_FORCE_ROWS = Integer.MAX_VALUE; // not final so test can inject its own setting
+
+    private final IndexContext indexContext;
+    private final ColumnQueryMetrics.VectorIndexMetrics columnQueryMetrics;
+    private final CassandraOnHeapGraph<PrimaryKey> graph;
+    private final LongAdder writeCount = new LongAdder();
+    private final LongAdder overwriteCount = new LongAdder();
+    private final LongAdder removedCount = new LongAdder();
+
+    private PrimaryKey minimumKey;
+    private PrimaryKey maximumKey;
+
+    private final NavigableSet<PrimaryKey> primaryKeys = new ConcurrentSkipListSet<>();
+    private final Memtable mt;
+
+    public VectorMemtableIndex(IndexContext indexContext, Memtable mt)
+    {
+        this.indexContext = indexContext;
+        this.columnQueryMetrics = (ColumnQueryMetrics.VectorIndexMetrics) indexContext.getColumnQueryMetrics();
+        this.graph = new CassandraOnHeapGraph<>(indexContext, true, mt);
+        this.mt = mt;
+    }
+
+    @Override
+    public Memtable getMemtable()
+    {
+        return mt;
+    }
+
+    @Override
+    public void index(DecoratedKey key, Clustering clustering, ByteBuffer value, Memtable memtable, OpOrder.Group opGroup)
+    {
+        if (value == null || value.remaining() == 0)
+            return;
+
+        var primaryKey = indexContext.keyFactory().create(key, clustering);
+        long allocatedBytes = index(primaryKey, value);
+        memtable.markExtraOnHeapUsed(allocatedBytes, opGroup);
+    }
+
+    private long index(PrimaryKey primaryKey, ByteBuffer value)
+    {
+        if (value == null || value.remaining() == 0)
+            return 0;
+
+        updateKeyBounds(primaryKey);
+
+        writeCount.increment();
+        primaryKeys.add(primaryKey);
+        return graph.add(value, primaryKey);
+    }
+
+    @Override
+    public void update(DecoratedKey key, Clustering clustering, ByteBuffer oldValue, ByteBuffer newValue, Memtable memtable, OpOrder.Group opGroup)
+    {
+        int oldRemaining = oldValue == null ? 0 : oldValue.remaining();
+        int newRemaining = newValue == null ? 0 : newValue.remaining();
+        if (oldRemaining == 0 && newRemaining == 0)
+            return;
+
+        boolean different;
+        if (oldRemaining != newRemaining)
+        {
+            assert oldRemaining == 0 || newRemaining == 0; // one of them is null
+            different = true;
+        }
+        else
+        {
+            different = indexContext.getValidator().compare(oldValue, newValue) != 0;
+        }
+
+        if (different)
+        {
+            var primaryKey = indexContext.keyFactory().create(key, clustering);
+            // update bounds because only rows with vectors are included in the key bounds,
+            // so if the vector was null before, we won't have included it
+            updateKeyBounds(primaryKey);
+
+            // make the changes in this order so we don't have a window where the row is not in the index at all
+            if (newRemaining > 0)
+            {
+                graph.add(newValue, primaryKey);
+                overwriteCount.increment();
+            }
+            if (oldRemaining > 0)
+                graph.remove(oldValue, primaryKey);
+
+            // remove primary key if it's no longer indexed
+            if (newRemaining <= 0 && oldRemaining > 0)
+            {
+                primaryKeys.remove(primaryKey);
+                removedCount.increment();
+            }
+        }
+    }
+
+    @Override
+    public void update(DecoratedKey key, Clustering clustering, Iterator<ByteBuffer> oldValues, Iterator<ByteBuffer> newValues, Memtable memtable, OpOrder.Group opGroup)
+    {
+        throw new UnsupportedOperationException("Vector index does not support multi-value updates");
+    }
+
+    private void updateKeyBounds(PrimaryKey primaryKey) {
+        if (minimumKey == null)
+            minimumKey = primaryKey;
+        else if (primaryKey.compareTo(minimumKey) < 0)
+            minimumKey = primaryKey;
+        if (maximumKey == null)
+            maximumKey = primaryKey;
+        else if (primaryKey.compareTo(maximumKey) > 0)
+            maximumKey = primaryKey;
+    }
+
+    @Override
+    public KeyRangeIterator search(QueryContext context, Expression expr, AbstractBounds<PartitionPosition> keyRange, int limit)
+    {
+        if (expr.getOp() != Expression.Op.BOUNDED_ANN)
+            throw new IllegalArgumentException(indexContext.logMessage("Only BOUNDED_ANN is supported, received: " + expr));
+        var qv = vts.createFloatVector(expr.lower.value.vector);
+        float threshold = expr.getEuclideanSearchThreshold();
+
+        SortingIterator.Builder<PrimaryKey> keyQueue;
+        try (var pkIterator = searchInternal(context, qv, keyRange, graph.size(), graph.size(), threshold))
+        {
+            keyQueue = new SortingIterator.Builder<>();
+            while (pkIterator.hasNext())
+                keyQueue.add(pkIterator.next().primaryKey());
+        }
+
+        if (keyQueue.size() == 0)
+            return KeyRangeIterator.empty();
+        return new ReorderingKeyRangeIterator(keyQueue.build(Comparator.naturalOrder()), keyQueue.size());
+    }
+
+    @Override
+    public long estimateMatchingRowsCount(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        // For BOUNDED_ANN we use the old way of estimating cardinality - by running the search.
+        throw new UnsupportedOperationException("Cardinality estimation not supported by vector indexes");
+    }
+
+    @Override
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderBy(QueryContext context,
+                                                                  Orderer orderer,
+                                                                  Expression slice,
+                                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                                  int limit)
+    {
+        assert slice == null : "ANN does not support index slicing";
+        assert orderer.isANN() : "Only ANN is supported for vector search, received " + orderer.operator;
+
+        var qv = vts.createFloatVector(orderer.getVectorTerm());
+        var rerankK = orderer.rerankKFor(limit, VectorCompression.NO_COMPRESSION);
+
+        return List.of(searchInternal(context, qv, keyRange, limit, rerankK, 0));
+    }
+
+    private CloseableIterator<PrimaryKeyWithSortKey> searchInternal(QueryContext context,
+                                                                  VectorFloat<?> queryVector,
+                                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                                  int limit,
+                                                                  int rerankK,
+                                                                  float threshold)
+    {
+        Bits bits;
+        if (RangeUtil.coversFullRing(keyRange))
+        {
+            bits = Bits.ALL;
+        }
+        else
+        {
+            // if left bound is MIN_BOUND or KEY_BOUND, we need to include all token-only PrimaryKeys with same token
+            boolean leftInclusive = keyRange.left.kind() != PartitionPosition.Kind.MAX_BOUND;
+            // if right bound is MAX_BOUND or KEY_BOUND, we need to include all token-only PrimaryKeys with same token
+            boolean rightInclusive = keyRange.right.kind() != PartitionPosition.Kind.MIN_BOUND;
+            // if right token is MAX (Long.MIN_VALUE), there is no upper bound
+            boolean isMaxToken = keyRange.right.getToken().isMinimum(); // max token
+
+            PrimaryKey left = indexContext.keyFactory().createTokenOnly(keyRange.left.getToken()); // lower bound
+            PrimaryKey right = isMaxToken ? null : indexContext.keyFactory().createTokenOnly(keyRange.right.getToken()); // upper bound
+
+            NavigableSet<PrimaryKey> resultKeys = isMaxToken ? primaryKeys.tailSet(left, leftInclusive)
+                                                             : primaryKeys.subSet(left, leftInclusive, right, rightInclusive);
+
+            if (resultKeys.isEmpty())
+                return CloseableIterator.emptyIterator();
+
+            int bruteForceRows = maxBruteForceRows(rerankK, resultKeys.size(), graph.size());
+            logger.trace("Search range covers {} rows; max brute force rows is {} for memtable index with {} nodes, rerankK {}, LIMIT {}",
+                         resultKeys.size(), bruteForceRows, graph.size(), rerankK, limit);
+            Tracing.trace("Search range covers {} rows; max brute force rows is {} for memtable index with {} nodes, rerankK {}, LIMIT {}",
+                          resultKeys.size(), bruteForceRows, graph.size(), rerankK, limit);
+            if (resultKeys.size() <= bruteForceRows)
+            {
+                // When we have a threshold, we only need to filter the results, not order them, because it means we're
+                // evaluating a boolean predicate in the SAI pipeline that wants to collate by PK
+                if (threshold > 0)
+                    return filterByBruteForce(queryVector, threshold, resultKeys);
+                else
+                    return orderByBruteForce(queryVector, resultKeys);
+            }
+            else
+            {
+                bits = new KeyRangeFilteringBits(keyRange);
+            }
+        }
+
+        var nodeScoreIterator = graph.search(context, queryVector, limit, rerankK, threshold, bits);
+        return new NodeScoreToScoredPrimaryKeyIterator(nodeScoreIterator);
+    }
+
+
+    @Override
+    public CloseableIterator<PrimaryKeyWithSortKey> orderResultsBy(QueryContext context, List<PrimaryKey> keys, Orderer orderer, int limit)
+    {
+        if (minimumKey == null)
+            // This case implies maximumKey is empty too.
+            return CloseableIterator.emptyIterator();
+
+        assert orderer.isANN() : "Only ANN is supported for vector search, received " + orderer;
+        // Compute the keys that exist in the current memtable and their corresponding graph ordinals
+        var keysInGraph = new HashSet<PrimaryKey>();
+        var relevantOrdinals = new IntHashSet();
+        keys.stream()
+            .dropWhile(k -> k.compareTo(minimumKey) < 0)
+            .takeWhile(k -> k.compareTo(maximumKey) <= 0)
+            .forEach(k ->
+        {
+            var v = graph.vectorForKey(k);
+            if (v == null)
+                return;
+            var i = graph.getOrdinal(v);
+            if (i < 0)
+                // might happen if the vector and/or its postings have been removed in the meantime between getting the
+                // vector and getting the ordinal (graph#vectorForKey and graph#getOrdinal are not synchronized)
+                return;
+            keysInGraph.add(k);
+            relevantOrdinals.add(i);
+        });
+
+        int rerankK = orderer.rerankKFor(limit, VectorCompression.NO_COMPRESSION);
+        int maxBruteForceRows = maxBruteForceRows(rerankK, relevantOrdinals.size(), graph.size());
+        Tracing.logAndTrace(logger, "{} rows relevant to current memtable out of {} materialized by SAI; max brute force rows is {} for memtable index with {} nodes, rerankK {}",
+                            relevantOrdinals.size(), keys.size(), maxBruteForceRows, graph.size(), rerankK);
+
+        // convert the expression value to query vector
+        var qv = vts.createFloatVector(orderer.getVectorTerm());
+        // brute force path
+        if (keysInGraph.size() <= maxBruteForceRows)
+        {
+            if (keysInGraph.isEmpty())
+                return CloseableIterator.emptyIterator();
+            return orderByBruteForce(qv, keysInGraph);
+        }
+        // indexed path
+        var nodeScoreIterator = graph.search(context, qv, limit, rerankK, 0, relevantOrdinals::contains);
+        return new NodeScoreToScoredPrimaryKeyIterator(nodeScoreIterator);
+    }
+
+    /**
+     * Filter the keys in the provided set by comparing their vectors to the query vector and returning only those
+     * that have a similarity score >= the provided threshold.
+     * NOTE: because the threshold is not used for ordering, the result is returned in PK order, not score order.
+     * @param queryVector the query vector
+     * @param threshold the minimum similarity score to accept
+     * @param keys the keys to filter
+     * @return an iterator over the keys that pass the filter in PK order
+     */
+    private CloseableIterator<PrimaryKeyWithSortKey> filterByBruteForce(VectorFloat<?> queryVector, float threshold, NavigableSet<PrimaryKey> keys)
+    {
+        columnQueryMetrics.onBruteForceNodesReranked(keys.size());
+        // Keys are already ordered in ascending PK order, so just use an ArrayList to collect the results.
+        var results = new ArrayList<PrimaryKeyWithSortKey>(keys.size());
+        scoreKeysAndAddToCollector(queryVector, keys, threshold, results);
+        return CloseableIterator.wrap(results.iterator());
+    }
+
+    private CloseableIterator<PrimaryKeyWithSortKey> orderByBruteForce(VectorFloat<?> queryVector, Collection<PrimaryKey> keys)
+    {
+        columnQueryMetrics.onBruteForceNodesReranked(keys.size());
+        // Use a sorting iterator because we often don't need to consume the entire iterator
+        var similarityFunction = indexContext.getIndexWriterConfig().getSimilarityFunction();
+        return SortingIterator.createCloseable(Comparator.naturalOrder(),
+                                               keys,
+                                               key -> scoreKey(similarityFunction, queryVector, key, 0),
+                                               Runnables.doNothing());
+    }
+
+    private void scoreKeysAndAddToCollector(VectorFloat<?> queryVector,
+                                            Collection<PrimaryKey> keys,
+                                            float threshold,
+                                            Collection<PrimaryKeyWithSortKey> collector)
+    {
+        var similarityFunction = indexContext.getIndexWriterConfig().getSimilarityFunction();
+        for (var key : keys)
+        {
+            var scored = scoreKey(similarityFunction, queryVector, key, threshold);
+            if (scored != null)
+                collector.add(scored);
+        }
+    }
+
+    private PrimaryKeyWithScore scoreKey(VectorSimilarityFunction similarityFunction, VectorFloat<?> queryVector, PrimaryKey key, float threshold)
+    {
+        var vector = graph.vectorForKey(key);
+        if (vector == null)
+            return null;
+        var score = similarityFunction.compare(queryVector, vector);
+        if (score < threshold)
+            return null;
+        return new PrimaryKeyWithScore(indexContext, mt, key, score);
+    }
+
+    private int maxBruteForceRows(int rerankK, int nPermittedOrdinals, int graphSize)
+    {
+        int expectedNodesVisited = expectedNodesVisited(rerankK, nPermittedOrdinals, graphSize);
+        return min(max(rerankK, expectedNodesVisited), GLOBAL_BRUTE_FORCE_ROWS);
+    }
+
+    public int estimateAnnNodesVisited(int rerankK, int nPermittedOrdinals)
+    {
+        return expectedNodesVisited(rerankK, nPermittedOrdinals, graph.size());
+    }
+
+    /**
+     * All parameters must be greater than zero.  nPermittedOrdinals may be larger than graphSize.
+     * <p>
+     * Returns the expected number of nodes visited by an ANN search.
+     * !!!
+     * !!! "Visted" means we compute the coarse similarity with the query vector.  This is
+     * !!! roughly `degree` times larger than the number of nodes whose edge lists we load!
+     * !!!
+     */
+    public static int expectedNodesVisited(int rerankK, int nPermittedOrdinals, int graphSize)
+    {
+        var K = rerankK;
+        var B = min(nPermittedOrdinals, graphSize);
+        var N = graphSize;
+        // These constants come from running many searches on a variety of datasets and graph sizes.
+        // * It is very consistent that the visited count is slightly less than linear wrt K, for both
+        //   unconstrained (B = N) and constrained (B < N) searches.
+        // * The behavior wrt B is hard to characterize.  Graphing the result F vs N/B shows ranges of
+        //   growth very close to linear, interspersed with sharp jumps up to a higher visit count.  Overall,
+        //   approximating it as linear is in the right ballpark.
+        // * For unconstrained searches, the visited count is closest to log(N) but for constrained searches
+        //   it is closer to log(N)**2 (or a higher exponent), perhaps as a result of N/B being too small.
+        //
+        // If we need to make this even more accurate, the relationship to B and to log(N) may be the best
+        // places to start.
+        var raw = (int) (100 + 0.025 * pow(log(N), 2) * pow(K, 0.95) * ((double) N / B));
+        return ensureSaneEstimate(raw, rerankK, graphSize);
+    }
+
+    public static int ensureSaneEstimate(int rawEstimate, int rerankK, int graphSize)
+    {
+        // we will always visit at least min(rerankK, graphSize) nodes, and we can't visit more nodes than exist in the graph
+        return min(max(rawEstimate, min(rerankK, graphSize)), graphSize);
+    }
+
+    @Override
+    public Iterator<Pair<ByteComparable.Preencoded, List<MemoryIndex.PkWithFrequency>>> iterator(DecoratedKey min, DecoratedKey max)
+    {
+        // This method is only used when merging an in-memory index with a RowMapping. This is done a different
+        // way with the graph using the writeData method below.
+        throw new UnsupportedOperationException();
+    }
+
+    /** returns true if the index is non-empty and should be flushed */
+    public boolean preFlush(ToIntFunction<PrimaryKey> ordinalMapper)
+    {
+        return graph.preFlush(ordinalMapper);
+    }
+
+    public int size()
+    {
+        return graph.size();
+    }
+
+    public SegmentMetadata.ComponentMetadataMap writeData(IndexComponents.ForWrite perIndexComponents) throws IOException
+    {
+        // Note that range deletions won't show up in the removed count, which is why it's just named removedCount and
+        // not deleted count.
+        logger.debug("Writing {} nodes to disk after {} inserts, {} overwrites, and {} removals for {}", graph.size(),
+                     writeCount.longValue(), overwriteCount.longValue(), removedCount.longValue(), perIndexComponents.descriptor());
+        return graph.flush(perIndexComponents);
+    }
+
+    @Override
+    public long writeCount()
+    {
+        return writeCount.longValue() + overwriteCount.longValue();
+    }
+
+    @Override
+    public long estimatedOnHeapMemoryUsed()
+    {
+        return graph.ramBytesUsed();
+    }
+
+    @Override
+    public long estimatedOffHeapMemoryUsed()
+    {
+        return 0;
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return graph.isEmpty();
+    }
+
+    @Nullable
+    @Override
+    public ByteBuffer getMinTerm()
+    {
+        return null;
+    }
+
+    @Nullable
+    @Override
+    public ByteBuffer getMaxTerm()
+    {
+        return null;
+    }
+
+    /*
+     * A {@link Bits} implementation that filters out all ordinals that do not correspond to a {@link PrimaryKey}
+     * in the provided {@link AbstractBounds<PartitionPosition>}.
+     */
+    private class KeyRangeFilteringBits implements Bits
+    {
+        private final AbstractBounds<PartitionPosition> keyRange;
+
+        public KeyRangeFilteringBits(AbstractBounds<PartitionPosition> keyRange)
+        {
+            this.keyRange = keyRange;
+        }
+
+        @Override
+        public boolean get(int ordinal)
+        {
+            var keys = graph.keysFromOrdinal(ordinal);
+            return keys.stream().anyMatch(k -> keyRange.contains(k.partitionKey()));
+        }
+    }
+
+    private class ReorderingKeyRangeIterator extends KeyRangeIterator
+    {
+        private final SortingIterator<PrimaryKey> keyQueue;
+
+        ReorderingKeyRangeIterator(SortingIterator<PrimaryKey> keyQueue, int expectedSize)
+        {
+            super(minimumKey, maximumKey, expectedSize);
+            this.keyQueue = keyQueue;
+        }
+
+        @Override
+        protected void performSkipTo(PrimaryKey nextKey)
+        {
+            keyQueue.skipTo(nextKey);
+        }
+
+        @Override
+        public void close() {}
+
+        @Override
+        protected PrimaryKey computeNext()
+        {
+            if (!keyQueue.hasNext())
+                return endOfData();
+            return keyQueue.next();
+        }
+    }
+
+    /**
+     * An iterator over {@link PrimaryKeyWithSortKey} sorted by score descending. The iterator converts ordinals (node ids)
+     * to {@link PrimaryKey}s and pairs them with the score given by the index.
+     */
+    private class NodeScoreToScoredPrimaryKeyIterator extends AbstractIterator<PrimaryKeyWithSortKey>
+    {
+        private final CloseableIterator<SearchResult.NodeScore> nodeScores;
+        private Iterator<PrimaryKeyWithScore> primaryKeysForNode = Collections.emptyIterator();
+
+        NodeScoreToScoredPrimaryKeyIterator(CloseableIterator<SearchResult.NodeScore> nodeScores)
+        {
+            this.nodeScores = nodeScores;
+        }
+
+        @Override
+        protected PrimaryKeyWithSortKey computeNext()
+        {
+            if (primaryKeysForNode.hasNext())
+                return primaryKeysForNode.next();
+
+            while (nodeScores.hasNext())
+            {
+                SearchResult.NodeScore nodeScore = nodeScores.next();
+                primaryKeysForNode = graph.keysFromOrdinal(nodeScore.node)
+                                          .stream()
+                                          .map(pk -> new PrimaryKeyWithScore(indexContext, mt, pk, nodeScore.score))
+                                          .iterator();
+                if (primaryKeysForNode.hasNext())
+                    return primaryKeysForNode.next();
+            }
+
+            return endOfData();
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(nodeScores);
+        }
+    }
+
+    /** ensures that the graph is connected -- normally not necessary but it can help tests reason about the state */
+    public void cleanup()
+    {
+        graph.cleanup();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/VectorPostings.java b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorPostings.java
new file mode 100644
index 000000000000..f60b096344fc
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorPostings.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.ToIntFunction;
+
+import com.google.common.base.Preconditions;
+
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import net.openhft.chronicle.bytes.Bytes;
+import net.openhft.chronicle.hash.serialization.BytesReader;
+import net.openhft.chronicle.hash.serialization.BytesWriter;
+import org.agrona.collections.IntArrayList;
+
+public class VectorPostings<T>
+{
+    // we expect that the overwhelmingly most common cardinality will be 1, so optimize for reads using COWAL
+    final CopyOnWriteArrayList<T> postings;
+    volatile int ordinal = -1;
+
+    private volatile IntArrayList rowIds; // initially null; gets filled in on flush by computeRowIds
+
+    public VectorPostings(T firstKey)
+    {
+        postings = new CopyOnWriteArrayList<>(List.of(firstKey));
+    }
+
+    public VectorPostings(List<T> raw)
+    {
+        postings = new CopyOnWriteArrayList<>(raw);
+    }
+
+    /**
+     * Split out from constructor only to make dealing with concurrent inserts easier for CassandraOnHeapGraph.
+     * Should be called at most once per instance.
+     */
+    public void setOrdinal(int ordinal)
+    {
+        assert this.ordinal == -1 : String.format("ordinal already set to %d; attempted to set to %d", this.ordinal, ordinal);
+        this.ordinal = ordinal;
+    }
+
+    public boolean add(T key)
+    {
+        for (T existing : postings)
+            if (existing.equals(key))
+                return false;
+        postings.add(key);
+        return true;
+    }
+
+    /**
+     * @return true if current ordinal is removed by partition/range deletion.
+     * Must be called after computeRowIds.
+     */
+    public boolean shouldAppendDeletedOrdinal()
+    {
+        return !postings.isEmpty() && (rowIds != null && rowIds.isEmpty());
+    }
+
+    /**
+     * Compute the rowIds corresponding to the < T > keys in this postings list.
+     */
+    public void computeRowIds(ToIntFunction<T> postingTransformer)
+    {
+        Preconditions.checkState(rowIds == null);
+
+        IntArrayList ids = new IntArrayList(postings.size(), -1);
+        for (T key : postings)
+        {
+            int rowId = postingTransformer.applyAsInt(key);
+            // partition deletion and range deletion won't trigger index update. There is no row id for given key during flush
+            if (rowId >= 0)
+                ids.add(rowId);
+        }
+
+        rowIds = ids;
+    }
+
+    /**
+     * @return rowIds corresponding to the < T > keys in this postings list.
+     * Must be called after computeRowIds.
+     */
+    public IntArrayList getRowIds()
+    {
+        Preconditions.checkNotNull(rowIds);
+        return rowIds;
+    }
+
+    public void remove(T key)
+    {
+        postings.remove(key);
+    }
+
+    public long ramBytesUsed()
+    {
+        return emptyBytesUsed() + postings.size() * bytesPerPosting();
+    }
+
+    public static long emptyBytesUsed()
+    {
+        long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+        long AH_BYTES = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+        return Integer.BYTES + REF_BYTES + AH_BYTES;
+    }
+
+    // we can't do this exactly without reflection, because keys could be Integer or PrimaryKey.
+    // PK is larger, so we'll take that and return an upper bound.
+    // we already count the float[] vector in vectorValues, so leave it out here
+    public long bytesPerPosting()
+    {
+        long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+        return REF_BYTES
+               + 2 * Long.BYTES // hashes in PreHashedDecoratedKey
+               + REF_BYTES; // key ByteBuffer, this is used elsewhere so we don't take the deep size
+    }
+
+    public int size()
+    {
+        return postings.size();
+    }
+
+    public List<T> getPostings()
+    {
+        return postings;
+    }
+
+    public boolean isEmpty()
+    {
+        return postings.isEmpty();
+    }
+
+    public int getOrdinal()
+    {
+        return getOrdinal(true);
+    }
+
+    public int getOrdinal(boolean assertSet)
+    {
+        assert !assertSet || ordinal >= 0 : "ordinal not set";
+        return ordinal;
+    }
+
+    public static class CompactionVectorPostings extends VectorPostings<Integer> {
+        public CompactionVectorPostings(int ordinal, List<Integer> raw)
+        {
+            super(raw);
+            this.ordinal = ordinal;
+        }
+
+        public CompactionVectorPostings(int ordinal, int firstKey)
+        {
+            super(firstKey);
+            this.ordinal = ordinal;
+        }
+
+        @Override
+        public void setOrdinal(int ordinal)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public IntArrayList getRowIds()
+        {
+            var L = new IntArrayList(size(), -1);
+            for (var i : postings)
+                L.addInt(i);
+            return L;
+        }
+
+        // CVP always contains int keys, so we don't have to be pessimistic on size like super does
+        @Override
+        public long bytesPerPosting()
+        {
+            long REF_BYTES = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+            return REF_BYTES + Integer.BYTES;
+        }
+    }
+
+    static class Marshaller implements BytesReader<CompactionVectorPostings>, BytesWriter<CompactionVectorPostings>
+    {
+        @Override
+        public void write(Bytes out, CompactionVectorPostings postings) {
+            out.writeInt(postings.ordinal);
+            out.writeInt(postings.size());
+            for (Integer posting : postings.getPostings()) {
+                out.writeInt(posting);
+            }
+        }
+
+        @Override
+        public CompactionVectorPostings read(Bytes in, CompactionVectorPostings using) {
+            int ordinal = in.readInt();
+            int size = in.readInt();
+            assert size >= 0 : size;
+            CompactionVectorPostings cvp;
+            if (size == 1) {
+                cvp = new CompactionVectorPostings(ordinal, in.readInt());
+            }
+            else
+            {
+                var postingsList = new IntArrayList(size, -1);
+                for (int i = 0; i < size; i++)
+                {
+                    postingsList.add(in.readInt());
+                }
+                cvp = new CompactionVectorPostings(ordinal, postingsList);
+            }
+            return cvp;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/VectorSourceModel.java b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorSourceModel.java
new file mode 100644
index 000000000000..bf08896591dd
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorSourceModel.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+
+import static io.github.jbellis.jvector.vector.VectorSimilarityFunction.COSINE;
+import static io.github.jbellis.jvector.vector.VectorSimilarityFunction.DOT_PRODUCT;
+import static java.lang.Math.max;
+import static java.lang.Math.pow;
+import static org.apache.cassandra.index.sai.disk.vector.VectorCompression.CompressionType.BINARY_QUANTIZATION;
+import static org.apache.cassandra.index.sai.disk.vector.VectorCompression.CompressionType.NONE;
+import static org.apache.cassandra.index.sai.disk.vector.VectorCompression.CompressionType.PRODUCT_QUANTIZATION;
+
+public enum VectorSourceModel
+{
+    ADA002((dimension) -> new VectorCompression(PRODUCT_QUANTIZATION, dimension, 0.125), 1.25),
+    OPENAI_V3_SMALL((dimension) -> new VectorCompression(PRODUCT_QUANTIZATION, dimension, 0.0625), 1.5),
+    OPENAI_V3_LARGE((dimension) -> new VectorCompression(PRODUCT_QUANTIZATION, dimension, 0.0625), 1.25),
+    BERT(COSINE, (dimension) -> new VectorCompression(PRODUCT_QUANTIZATION, dimension, 0.25), __ -> 1.0),
+    GECKO((dimension) -> new VectorCompression(PRODUCT_QUANTIZATION, dimension, 0.125), 1.25),
+    NV_QA_4((dimension) -> new VectorCompression(PRODUCT_QUANTIZATION, dimension, 0.125), 1.25),
+    COHERE_V3((dimension) -> new VectorCompression(PRODUCT_QUANTIZATION, dimension, 0.0625), 1.25),
+
+    OTHER(COSINE, VectorSourceModel::genericCompressionFor, VectorSourceModel::genericOverquery);
+
+    /**
+     * Default similarity function for this model.
+     */
+    public final VectorSimilarityFunction defaultSimilarityFunction;
+    /**
+     * Compression provider optimized for this model.
+     */
+    public final Function<Integer, VectorCompression> compressionProvider;
+    /**
+     * Factor by which to multiply the top K requested by to search deeper in the graph.
+     * This is IN ADDITION to the tapered 2x applied by OverqueryUtils.
+     */
+    public final Function<VectorCompression, Double> overqueryProvider;
+
+    VectorSourceModel(Function<Integer, VectorCompression> compressionProvider, double overqueryFactor)
+    {
+        this(DOT_PRODUCT, compressionProvider, __ -> overqueryFactor);
+    }
+
+    VectorSourceModel(VectorSimilarityFunction defaultSimilarityFunction,
+                      Function<Integer, VectorCompression> compressionProvider,
+                      Function<VectorCompression, Double> overqueryProvider)
+    {
+        this.defaultSimilarityFunction = defaultSimilarityFunction;
+        this.compressionProvider = compressionProvider;
+        this.overqueryProvider = overqueryProvider;
+    }
+
+    public static VectorSourceModel fromString(String value)
+    {
+        return valueOf(value.toUpperCase());
+    }
+
+    private static VectorCompression genericCompressionFor(int dimension)
+    {
+        // Model is unspecified / unknown, so we guess.
+        return new VectorCompression(PRODUCT_QUANTIZATION, dimension * Float.BYTES, defaultPQBytesFor(dimension));
+    }
+
+    private static int defaultPQBytesFor(int originalDimension)
+    {
+        // the idea here is that higher dimensions compress well, but not so well that we should use fewer bits
+        // than a lower-dimension vector, which is what you could get with cutoff points to switch between (e.g.)
+        // D*0.5 and D*0.25.  Thus, the following ensures that bytes per vector is strictly increasing with D.
+        int compressedBytes;
+        if (originalDimension <= 32) {
+            // We are compressing from 4-byte floats to single-byte codebook indexes,
+            // so this represents compression of 4x
+            // * GloVe-25 needs 25 BPV to achieve good recall
+            compressedBytes = originalDimension;
+        }
+        else if (originalDimension <= 64) {
+            // * GloVe-50 performs fine at 25
+            compressedBytes = 32;
+        }
+        else if (originalDimension <= 200) {
+            // * GloVe-100 and -200 perform well at 50 and 100 BPV, respectively
+            compressedBytes = (int) (originalDimension * 0.5);
+        }
+        else if (originalDimension <= 400) {
+            // * NYTimes-256 actually performs fine at 64 BPV but we'll be conservative
+            //   since we don't want BPV to decrease
+            compressedBytes = 100;
+        }
+        else if (originalDimension <= 768) {
+            // allow BPV to increase linearly up to 192
+            compressedBytes = (int) (originalDimension * 0.25);
+        }
+        else if (originalDimension <= 1536) {
+            // * ada002 vectors have good recall even at 192 BPV = compression of 32x
+            compressedBytes = 192;
+        }
+        else {
+            // We have not tested recall with larger vectors than this, let's let it increase linearly
+            compressedBytes = (int) (originalDimension * 0.125);
+        }
+        return compressedBytes;
+    }
+
+    private static double genericOverquery(VectorCompression vc)
+    {
+        assert vc != null;
+        // we compress extra-large vectors more aggressively, so we need to bump up the limit for those.
+        if (vc.type == BINARY_QUANTIZATION)
+            return 2.0;
+        else if ((double) vc.getOriginalSize() / vc.getCompressedSize() > 16.0)
+            return 1.5;
+        else
+            return 1.0;
+    }
+
+    /**
+     * @param limit the number of results the user asked for
+     * @param vc compression information about vectors being queried
+     * @return the topK >= `limit` results to ask the index to search for, forcing
+     * the greedy search deeper into the graph.  This serves two purposes:
+     * 1. Smoothes out the relevance difference between small LIMIT and large
+     * 2. Compensates for using lossily-compressed vectors during the search
+     */
+    public int rerankKFor(int limit, VectorCompression vc)
+    {
+        // if the vectors are uncompressed, bump up the limit a bit to start with but decay it rapidly
+        if (vc.type == NONE)
+        {
+            var n = max(1.0, 0.979 + 4.021 * pow(limit, -0.761)); // f(1) =  5.0, f(100) = 1.1, f(1000) = 1.0
+            return (int) (n * limit);
+        }
+
+        // Most compressed vectors should be queried at ~2x as much as uncompressed vectors.  (Our compression
+        // is tuned so that this should give us approximately the same recall as using uncompressed.)
+        // Again, we do want this to decay as we go to very large limits.
+        var n = tapered2x(limit);
+
+        // per-model adjustment on top of the ~2x factor
+        int originalDimension = vc.getOriginalSize() / 4;
+        if (compressionProvider.apply(originalDimension).equals(vc))
+        {
+            n *= overqueryProvider.apply(vc);
+        }
+        else
+        {
+            // we're using an older CV that wasn't created with the currently preferred parameters,
+            // so use the generic defaults instead
+            n *= OTHER.overqueryProvider.apply(vc);
+        }
+
+        return (int) (n * limit);
+    }
+
+    @VisibleForTesting
+    static double tapered2x(int limit)
+    {
+        return max(1.0, 0.509 + 9.491 * pow(limit, -0.402)); // f(1) = 10.0, f(100) = 2.0, f(1000) = 1.1
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/VectorValidation.java b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorValidation.java
new file mode 100644
index 000000000000..b3130f8e3423
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorValidation.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+public class VectorValidation
+{
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    // chosen to make sure dot products don't overflow
+    public static final float MAX_FLOAT32_COMPONENT = 1E17f;
+
+    public static void checkInBounds(VectorFloat<?> v)
+    {
+        for (int i = 0; i < v.length(); i++)
+        {
+            if (!Float.isFinite(v.get(i)))
+            {
+                throw new IllegalArgumentException("non-finite value at vector[" + i + "]=" + v.get(i));
+            }
+
+            if (Math.abs(v.get(i)) > MAX_FLOAT32_COMPONENT)
+            {
+                throw new IllegalArgumentException("Out-of-bounds value at vector[" + i + "]=" + v.get(i));
+            }
+        }
+    }
+
+    /** use with caution, it allocates a temporary VectorFloat */
+    public static void validateIndexable(float[] raw, VectorSimilarityFunction similarityFunction)
+    {
+        validateIndexable(vts.createFloatVector(raw), similarityFunction);
+    }
+
+    public static void validateIndexable(VectorFloat<?> vector, VectorSimilarityFunction similarityFunction)
+    {
+        try
+        {
+            checkInBounds(vector);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new InvalidRequestException(e.getMessage());
+        }
+
+        if (similarityFunction == VectorSimilarityFunction.COSINE)
+        {
+            if (isEffectivelyZero(vector))
+                throw new InvalidRequestException("Zero and near-zero vectors cannot be indexed or queried with cosine similarity");
+        }
+    }
+
+    public static boolean isEffectivelyZero(VectorFloat<?> vector)
+    {
+        for (int i = 0; i < vector.length(); i++)
+        {
+            if (vector.get(i) < -1E-6 || vector.get(i) > 1E-6)
+                return false;
+        }
+        return true;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeAntiJoinIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeAntiJoinIterator.java
new file mode 100644
index 000000000000..6743f0f5cc9f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeAntiJoinIterator.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.iterators;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * An iterator wrapper that wraps two iterators (left and right) and returns the primary keys from the left iterator
+ * that do not match the primary keys from the right iterator. The keys returned by the wrapped iterators must
+ * follow token-clustering order.
+ */
+public class KeyRangeAntiJoinIterator extends KeyRangeIterator
+{
+    final KeyRangeIterator left;
+    final KeyRangeIterator right;
+
+    private PrimaryKey nextKeyToSkip = null;
+
+    private KeyRangeAntiJoinIterator(KeyRangeIterator left, KeyRangeIterator right)
+    {
+        super(left.getMinimum(), left.getMaximum(), left.getMaxKeys());
+        this.left = left;
+        this.right = right;
+    }
+
+    public static KeyRangeAntiJoinIterator create(KeyRangeIterator left, KeyRangeIterator right)
+    {
+        return new KeyRangeAntiJoinIterator(left, right);
+    }
+
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        left.skipTo(nextKey);
+
+        if (nextKeyToSkip == null || nextKeyToSkip.compareTo(nextKey) < 0)
+            right.skipTo(nextKey);
+    }
+
+    public void close() throws IOException
+    {
+        FileUtils.close(left, right);
+    }
+
+    protected PrimaryKey computeNext()
+    {
+        if (nextKeyToSkip == null)
+            nextKeyToSkip = right.nextOrNull();
+
+        PrimaryKey key = left.nextOrNull();
+        int cmp = compare(key, nextKeyToSkip);
+
+        while (key != null && cmp >= 0)
+        {
+            if (cmp == 0)
+            {
+                key = left.nextOrNull();
+            }
+            else
+            {
+                right.skipTo(key);
+            }
+            nextKeyToSkip = right.nextOrNull();
+            cmp = compare(key, nextKeyToSkip);
+        }
+
+        return key != null ? key : endOfData();
+    }
+
+    private int compare(PrimaryKey key1, PrimaryKey key2)
+    {
+        return (key1 == null || key2 == null) ? -1 : key1.compareTo(key2);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeCollectionIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeCollectionIterator.java
new file mode 100644
index 000000000000..a8d86441c4ed
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeCollectionIterator.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.iterators;
+
+import java.util.List;
+import java.util.SortedSet;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+/**
+ * A {@link KeyRangeIterator} that iterates over a collection of {@link PrimaryKey}s without modifying the underlying list.
+ */
+public class KeyRangeCollectionIterator extends KeyRangeIterator
+{
+    private final PeekingIterator<PrimaryKey> keyQueue;
+
+    /**
+     * Create a new {@link KeyRangeCollectionIterator} that iterates over the provided list of keys.
+     * @param minimumKey the minimum key for the provided list of keys
+     * @param maximumKey the maximum key for the provided list of keys
+     * @param keys the list of keys to iterate over
+     */
+    public KeyRangeCollectionIterator(PrimaryKey minimumKey, PrimaryKey maximumKey, List<PrimaryKey> keys)
+    {
+        super(minimumKey, maximumKey, keys.size());
+        this.keyQueue = Iterators.peekingIterator(keys.iterator());
+    }
+
+    /**
+     * Create a new {@link KeyRangeCollectionIterator} that iterates over the provided set of keys.
+     * @param keys the sorted set of keys to iterate over
+     */
+    public KeyRangeCollectionIterator(SortedSet<PrimaryKey> keys)
+    {
+        super(keys.first(), keys.last(), keys.size());
+        this.keyQueue = Iterators.peekingIterator(keys.iterator());
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        while (keyQueue.hasNext())
+        {
+            if (keyQueue.peek().compareTo(nextKey) >= 0)
+                break;
+            keyQueue.next();
+        }
+    }
+
+    @Override
+    public void close() {}
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        return keyQueue.hasNext() ? keyQueue.next() : endOfData();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeConcatIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeConcatIterator.java
index bb83f1ea81c4..800ca7bccdb4 100644
--- a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeConcatIterator.java
+++ b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeConcatIterator.java
@@ -17,175 +17,152 @@
  */
 package org.apache.cassandra.index.sai.iterators;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 
-import com.google.common.annotations.VisibleForTesting;
-
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.io.util.FileUtils;
 
 /**
- * {@link KeyRangeConcatIterator} takes a list of sorted range iterators and concatenates them, leaving duplicates in
+ * {@link KeyRangeConcatIterator} takes a list of sorted range iterator and concatenates them, leaving duplicates in
  * place, to produce a new stably sorted iterator. Duplicates are eliminated later in
  * {@link org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher}
  * as results from multiple SSTable indexes and their respective segments are consumed.
- * <p>
+ *
  * ex. (1, 2, 3) + (3, 3, 4, 5) -> (1, 2, 3, 3, 3, 4, 5)
  * ex. (1, 2, 2, 3) + (3, 4, 4, 6, 6, 7) -> (1, 2, 2, 3, 3, 4, 4, 6, 6, 7)
+ *
  */
 public class KeyRangeConcatIterator extends KeyRangeIterator
 {
-    public static final String MUST_BE_SORTED_ERROR = "RangeIterator must be sorted, previous max: %s, next min: %s";
-    private final List<KeyRangeIterator> ranges;
+    private final Iterator<KeyRangeIterator> ranges;
+    private KeyRangeIterator currentRange;
+    private final List<KeyRangeIterator> toRelease;
 
-    private int current;
-
-    protected KeyRangeConcatIterator(KeyRangeIterator.Builder.Statistics statistics, List<KeyRangeIterator> ranges, Runnable onClose)
+    protected KeyRangeConcatIterator(KeyRangeIterator.Builder.Statistics statistics, List<KeyRangeIterator> ranges)
     {
-        super(statistics, onClose);
+        super(statistics);
 
         if (ranges.isEmpty())
             throw new IllegalArgumentException("Cannot concatenate empty list of ranges");
-
-        this.current = 0;
-        this.ranges = ranges;
+        this.ranges = ranges.iterator();
+        currentRange = this.ranges.next();
+        this.toRelease = ranges;
     }
 
     @Override
-    protected void performSkipTo(PrimaryKey nextKey)
+    protected void performSkipTo(PrimaryKey primaryKey)
     {
-        while (current < ranges.size())
+        while (true)
         {
-            KeyRangeIterator currentIterator = ranges.get(current);
-
-            if (currentIterator.hasNext() && currentIterator.peek().compareTo(nextKey) >= 0)
-                break;
-
-            if (currentIterator.getMaximum().compareTo(nextKey) >= 0)
+            if (currentRange.getMaximum().compareTo(primaryKey) >= 0)
             {
-                currentIterator.skipTo(nextKey);
-                break;
+                currentRange.skipTo(primaryKey);
+                return;
             }
-
-            current++;
+            if (!ranges.hasNext())
+            {
+                currentRange.skipTo(primaryKey);
+                return;
+            }
+            currentRange = ranges.next();
         }
     }
 
     @Override
     protected PrimaryKey computeNext()
     {
-        while (current < ranges.size())
+        while (!currentRange.hasNext())
         {
-            KeyRangeIterator currentIterator = ranges.get(current);
-
-            if (currentIterator.hasNext())
-                return currentIterator.next();
+            if (!ranges.hasNext())
+                return endOfData();
 
-            current++;
+            currentRange = ranges.next();
         }
-
-        return endOfData();
+        return currentRange.next();
     }
 
     @Override
-    public void close()
+    public void close() throws IOException
     {
-        super.close();
-
         // due to lazy key fetching, we cannot close iterator immediately
-        FileUtils.closeQuietly(ranges);
+        toRelease.forEach(FileUtils::closeQuietly);
+    }
+
+    public static Builder builder()
+    {
+        return builder(1);
     }
 
     public static Builder builder(int size)
     {
-        return builder(size, () -> {});
+        return new Builder(size);
     }
 
-    public static Builder builder(int size, Runnable onClose)
+    public static KeyRangeIterator build(List<KeyRangeIterator> tokens)
     {
-        return new Builder(size, onClose);
+        return new Builder(tokens.size()).add(tokens).build();
     }
 
-    @VisibleForTesting
     public static class Builder extends KeyRangeIterator.Builder
     {
         // We can use a list because the iterators are already in order
-        private final List<KeyRangeIterator> ranges;
+        private final List<KeyRangeIterator> rangeIterators;
+        public Builder(int size)
+        {
+            super(IteratorType.CONCAT);
+            this.rangeIterators = new ArrayList<>(size);
+        }
 
-        Builder(int size, Runnable onClose)
+        @Override
+        public int rangeCount()
+        {
+            return rangeIterators.size();
+        }
+
+        @Override
+        public Collection<KeyRangeIterator> ranges()
         {
-            super(new ConcatStatistics(), onClose);
-            this.ranges = new ArrayList<>(size);
+            return rangeIterators;
         }
 
         @Override
-        public KeyRangeIterator.Builder add(KeyRangeIterator range)
+        public Builder add(KeyRangeIterator range)
         {
             if (range == null)
                 return this;
 
             if (range.getMaxKeys() > 0)
-                ranges.add(range);
+            {
+                rangeIterators.add(range);
+                statistics.update(range);
+            }
             else
                 FileUtils.closeQuietly(range);
-            statistics.update(range);
 
             return this;
         }
 
         @Override
-        public int rangeCount()
+        public KeyRangeIterator.Builder add(List<KeyRangeIterator> ranges)
         {
-            return ranges.size();
-        }
+            if (ranges == null || ranges.isEmpty())
+                return this;
 
-        @Override
-        public void cleanup()
-        {
-            super.cleanup();
-            FileUtils.closeQuietly(ranges);
+            ranges.forEach(this::add);
+            return this;
         }
 
-        @Override
         protected KeyRangeIterator buildIterator()
         {
             if (rangeCount() == 0)
-            {
-                onClose.run();
                 return empty();
-            }
             if (rangeCount() == 1)
-            {
-                KeyRangeIterator single = ranges.get(0);
-                single.setOnClose(onClose);
-                return single;
-            }
-
-            return new KeyRangeConcatIterator(statistics, ranges, onClose);
-        }
-    }
-
-    private static class ConcatStatistics extends KeyRangeIterator.Builder.Statistics
-    {
-        @Override
-        public void update(KeyRangeIterator range)
-        {
-            // range iterators should be sorted, but previous max must not be greater than next min.
-            if (range.getMaxKeys() > 0)
-            {
-                if (count == 0)
-                {
-                    min = range.getMinimum();
-                }
-                else if (count > 0 && max.compareTo(range.getMinimum()) > 0)
-                {
-                    throw new IllegalArgumentException(String.format(MUST_BE_SORTED_ERROR, max, range.getMinimum()));
-                }
-
-                max = range.getMaximum();
-                count += range.getMaxKeys();
-            }
+                return rangeIterators.get(0);
+            return new KeyRangeConcatIterator(statistics, rangeIterators);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeIntersectionIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeIntersectionIterator.java
index 5391dd75edb7..9f1c27c81a5f 100644
--- a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeIntersectionIterator.java
+++ b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeIntersectionIterator.java
@@ -17,229 +17,180 @@
  */
 package org.apache.cassandra.index.sai.iterators;
 
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
-import java.util.Comparator;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
-import java.util.stream.Collectors;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.PrimaryKey.Kind;
 import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.tracing.Tracing;
 
-import javax.annotation.Nullable;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT;
 
 /**
  * A simple intersection iterator that makes no real attempts at optimising the iteration apart from
- * initially sorting the ranges. This implementation also supports an intersection limit via
- * {@code CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT} which limits the number of ranges that will 
- * be included in the intersection. This currently defaults to 2.
- * <p> 
- * Intersection only works for ranges that are compatible according to {@link PrimaryKey.Kind#isIntersectable(Kind)}.
+ * initially sorting the ranges. This implementation also supports an intersection limit which limits
+ * the number of ranges that will be included in the intersection. This currently defaults to 2.
  */
 public class KeyRangeIntersectionIterator extends KeyRangeIterator
 {
-    private static final Logger logger = LoggerFactory.getLogger(KeyRangeIntersectionIterator.class);
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    // The cassandra.sai.intersection_clause_limit (default: 2) controls the maximum number of range iterator that
+    // will be used in the final intersection of a query operation.
+    public static final int INTERSECTION_CLAUSE_LIMIT = SAI_INTERSECTION_CLAUSE_LIMIT.getInt();
 
     static
     {
-        logger.info(String.format("Storage attached index intersection clause limit is %d", CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT.getInt()));
+        logger.info(String.format("Storage attached index intersection clause limit is %d", INTERSECTION_CLAUSE_LIMIT));
     }
 
-    private final List<KeyRangeIterator> ranges;
-    private PrimaryKey highestKey;
+    public static boolean shouldDefer(int numberOfExpressions)
+    {
+        return (INTERSECTION_CLAUSE_LIMIT <= 0) || (numberOfExpressions <= INTERSECTION_CLAUSE_LIMIT);
+    }
 
-    private KeyRangeIntersectionIterator(Builder.Statistics statistics, List<KeyRangeIterator> ranges, Runnable onClose)
+    public final List<KeyRangeIterator> ranges;
+    private final int[] rangeStats;
+
+    private KeyRangeIntersectionIterator(Builder.Statistics statistics, List<KeyRangeIterator> ranges)
     {
-        super(statistics, onClose);
+        super(statistics);
         this.ranges = ranges;
-        this.highestKey = null;
+        this.rangeStats = new int[ranges.size()];
     }
 
-    @Override
     protected PrimaryKey computeNext()
     {
-        if (highestKey == null)
-            highestKey = computeHighestKey();
+        // The highest primary key seen on any range iterator so far.
+        // It can become null when we reach the end of the iterator.
+        PrimaryKey highestKey = ranges.get(0).hasNext() ? ranges.get(0).next() : null;
+        // Index of the range iterator that has advanced beyond the others
+        int alreadyAdvanced = 0;
+        rangeStats[0]++;
 
         outer:
-        // After advancing one iterator, we must try to advance all the other iterators that got behind,
-        // so they catch up to it. Note that we will not advance the iterators for static columns
-        // as long as they point to the partition of the highest key. (This is because STATIC primary keys 
-        // compare to other keys only by partition.) This loop continues until all iterators point to the same key,
-        // or if we run out of keys on any of them, or if we exceed the maximum key.
-        // There is no point in iterating after maximum, because no keys will match beyond that point.
-        while (highestKey != null && highestKey.compareTo(getMaximum()) <= 0)
+        while (highestKey != null)
         {
-            // Try to advance all iterators to the highest key seen so far.
+            // Try advance all iterators to the highest key seen so far.
             // Once this inner loop finishes normally, all iterators are guaranteed to be at the same value.
-            for (KeyRangeIterator range : ranges)
+            for (int index = 0; index < ranges.size(); index++)
             {
-                if (!range.hasNext())
-                    return endOfData();
-
-                if (range.peek().compareTo(highestKey) < 0)
+                if (index != alreadyAdvanced)
                 {
-                    // If we advance a STATIC key, then we must advance it to the same partition as the highestKey.
-                    // Advancing a STATIC key to a WIDE key directly (without throwing away the clustering) would
-                    // go too far, as WIDE keys are stored after STATIC in the posting list.
-                    PrimaryKey nextKey = range.peek().kind() == Kind.STATIC
-                                         ? skipAndPeek(range, highestKey.toStatic())
-                                         : skipAndPeek(range, highestKey);
-
-                    // We use strict comparison here, since it orders WIDE primary keys after STATIC primary keys
-                    // in the same partition. When WIDE keys are present, we want to return them rather than STATIC
-                    // keys to avoid retrieving and post-filtering entire partitions.
-                    if (nextKey == null || nextKey.compareToStrict(highestKey) > 0)
+                    KeyRangeIterator range = ranges.get(index);
+                    PrimaryKey nextKey = nextOrNull(range, highestKey);
+                    rangeStats[index]++;
+                    int comparisonResult;
+                    if (nextKey == null || (comparisonResult = nextKey.compareTo(highestKey)) > 0)
                     {
                         // We jumped over the highest key seen so far, so make it the new highest key.
                         highestKey = nextKey;
-
-                        // This iterator jumped over, so the other iterators might be lagging behind now,
+                        // Remember this iterator to avoid advancing it again, because it is already at the highest key
+                        alreadyAdvanced = index;
+                        // This iterator jumped over, so the other iterators are lagging behind now,
                         // including the ones already advanced in the earlier cycles of the inner loop.
-                        // Therefore, restart the inner loop in order to advance the lagging iterators.
+                        // Therefore, restart the inner loop in order to advance
+                        // the other iterators except this one to match the new highest key.
                         continue outer;
                     }
-                    assert nextKey.compareTo(highestKey) == 0 :
-                        String.format("Skipped to a key smaller than the target! " +
-                                      "iterator: %s, target key: %s, returned key: %s", range, highestKey, nextKey);
+                    assert comparisonResult == 0 :
+                           String.format("skipTo skipped to an item smaller than the target; " +
+                                         "iterator: %s, target key: %s, returned key: %s", range, highestKey, nextKey);
                 }
             }
+            // If we reached here, next() has been called at least once on each range iterator and
+            // the last call to next() on each iterator returned a value equal to the highestKey.
 
-            // If we get here, all iterators have been advanced to the same key. When STATIC and WIDE keys are
-            // mixed, this means WIDE keys point to exactly the same row, and STATIC keys the same partition.
-            PrimaryKey result = highestKey;
-
-            // Advance one iterator to the next key and remember the key as the highest seen so far.
-            // It can become null when we reach the end of the iterator.
-            // If there are both static and non-static keys being iterated here, we advance a non-static one,
-            // regardless of the order of ranges in the ranges list.
-            highestKey = advanceOneRange();
-
-            // If we get here, all iterators have been advanced to the same key. When STATIC and WIDE keys are
-            // mixed, this means WIDE keys point to exactly the same row, and STATIC keys the same partition.
-            return result;
-        }
-
-        return endOfData();
-    }
+            // Move the iterator that was called the least times to the start of the list.
+            // This is an optimisation assuming that iterator is likely a more selective one.
+            // E.g.if the first range produces (1, 2, 3, ... 100) and the second one (10, 20, 30, .. 100)
+            // we'd want to start with the second.
+            int idxOfSmallest = getIdxOfSmallest(rangeStats);
 
-    /**
-     * Advances the iterator of one range to the next item, which becomes the highest seen so far.
-     * Iterators pointing to STATIC keys are advanced only if no non-STATIC keys have been advanced.
-     *
-     * @return the next highest key or null if the iterator has reached the end
-     */
-    private @Nullable PrimaryKey advanceOneRange()
-    {
-        for (KeyRangeIterator range : ranges)
-            if (range.peek().kind() != Kind.STATIC)
-            {
-                range.next();
-                return range.hasNext() ? range.peek() : null;
-            }
-        
-        for (KeyRangeIterator range : ranges)
-            if (range.peek().kind() == Kind.STATIC)
+            if (idxOfSmallest != 0)
             {
-                range.next();
-                return range.hasNext() ? range.peek() : null;
+                Collections.swap(ranges, 0, idxOfSmallest);
+                // swap stats as well
+                int a = rangeStats[0];
+                int b = rangeStats[idxOfSmallest];
+                rangeStats[0] = b;
+                rangeStats[idxOfSmallest] = a;
             }
 
-        throw new IllegalStateException("There should be at least one range to advance!");
+            return highestKey;
+        }
+        return endOfData();
     }
 
-    private @Nullable PrimaryKey computeHighestKey()
+    private static int getIdxOfSmallest(int[] rangeStats)
     {
-        PrimaryKey max = getMinimum();
-        for (KeyRangeIterator range : ranges)
+        int idxOfSmallest = 0;
+        for (int i = 1; i < rangeStats.length; i++)
         {
-            if (!range.hasNext())
-                return null;
-            if (range.peek().compareToStrict(max) > 0)
-                max = range.peek();
+            if (rangeStats[i] < rangeStats[idxOfSmallest])
+                idxOfSmallest = i;
         }
-        return max;
+        return idxOfSmallest;
     }
 
-    @Override
-    protected void performSkipTo(PrimaryKey nextKey)
+    protected void performSkipTo(PrimaryKey nextToken)
     {
         // Resist the temptation to call range.hasNext before skipTo: this is a pessimisation, hasNext will invoke
         // computeNext under the hood, which is an expensive operation to produce a value that we plan to throw away.
         // Instead, it is the responsibility of the child iterators to make skipTo fast when the iterator is exhausted.
-        for (KeyRangeIterator range : ranges)
-            range.skipTo(nextKey);
-
-        // Force recomputing the highest key on the next call to computeNext()
-        highestKey = null;
-    }
-
-    @Override
-    public void close()
-    {
-        super.close();
-        FileUtils.closeQuietly(ranges);
+        for (var range : ranges)
+            range.skipTo(nextToken);
     }
 
     /**
      * Fetches the next available item from the iterator, such that the item is not lower than the given key.
      * If no such items are available, returns null.
      */
-    private PrimaryKey skipAndPeek(KeyRangeIterator iterator, PrimaryKey minKey)
+    private PrimaryKey nextOrNull(KeyRangeIterator iterator, PrimaryKey minKey)
     {
         iterator.skipTo(minKey);
-        return iterator.hasNext() ? iterator.peek() : null;
+        return iterator.hasNext() ? iterator.next() : null;
     }
 
-    public static Builder builder(int size, int limit)
+    public void close() throws IOException
     {
-        return builder(size, limit, () -> {});
+        ranges.forEach(FileUtils::closeQuietly);
     }
 
-    public static Builder builder(int size, Runnable onClose)
+    public static Builder builder(List<KeyRangeIterator> ranges)
     {
-        return new Builder(size, onClose);
+        var builder = new Builder(ranges.size());
+        for (var range : ranges)
+            builder.add(range);
+        return builder;
     }
 
-    @VisibleForTesting
-    public static Builder builder(int size, int limit, Runnable onClose)
+    public static Builder builder(int size)
     {
-        return new Builder(size, limit, onClose);
+        return new Builder(size);
+    }
+
+    public static Builder builder()
+    {
+        return builder(4);
     }
 
-    @VisibleForTesting
     public static class Builder extends KeyRangeIterator.Builder
     {
-        // This controls the maximum number of range iterators that will be used in the final
-        // intersection of a query operation. It is set from cassandra.sai.intersection_clause_limit
-        // and defaults to 2
-        private final int limit;
-        // tracks if any of the added ranges are disjoint with the other ranges, which is useful
-        // in case of intersection, as it gives a direct answer whether the iterator is going
-        // to produce any results.
-        private boolean isDisjoint;
-
-        protected final List<KeyRangeIterator> rangeIterators;
-
-        Builder(int size, Runnable onClose)
-        {
-            this(size, CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT.getInt(), onClose);
-        }
+        protected List<KeyRangeIterator> rangeIterators;
 
-        Builder(int size, int limit, Runnable onClose)
+        private Builder(int size)
         {
-            super(new IntersectionStatistics(), onClose);
+            super(IteratorType.INTERSECTION);
             rangeIterators = new ArrayList<>(size);
-            this.limit = limit;
         }
 
-        @Override
         public KeyRangeIterator.Builder add(KeyRangeIterator range)
         {
             if (range == null)
@@ -249,151 +200,46 @@ public KeyRangeIterator.Builder add(KeyRangeIterator range)
                 rangeIterators.add(range);
             else
                 FileUtils.closeQuietly(range);
+            statistics.update(range);
 
-            updateStatistics(statistics, range);
+            return this;
+        }
 
+        public KeyRangeIterator.Builder add(List<KeyRangeIterator> ranges)
+        {
+            if (ranges == null || ranges.isEmpty())
+                return this;
+
+            ranges.forEach(this::add);
             return this;
         }
 
-        @Override
         public int rangeCount()
         {
             return rangeIterators.size();
         }
 
         @Override
-        public void cleanup()
+        public Collection<KeyRangeIterator> ranges()
         {
-            super.cleanup();
-            FileUtils.closeQuietly(rangeIterators);
+            return rangeIterators;
         }
 
-        @Override
         protected KeyRangeIterator buildIterator()
         {
-            rangeIterators.sort(Comparator.comparingLong(KeyRangeIterator::getMaxKeys));
-            int initialSize = rangeIterators.size();
-            // all ranges will be included
-            if (limit >= rangeIterators.size() || limit <= 0)
-                return buildIterator(statistics, rangeIterators);
-
-            // Apply most selective iterators during intersection, because larger number of iterators will result lots of disk seek.
-            Statistics selectiveStatistics = new IntersectionStatistics();
-            isDisjoint = false;
-            for (int i = rangeIterators.size() - 1; i >= 0 && i >= limit; i--)
-                FileUtils.closeQuietly(rangeIterators.remove(i));
-
-            rangeIterators.forEach(range -> updateStatistics(selectiveStatistics, range));
-
-            if (Tracing.isTracing())
-                Tracing.trace("Selecting {} {} of {} out of {} indexes",
-                              rangeIterators.size(),
-                              rangeIterators.size() > 1 ? "indexes with cardinalities" : "index with cardinality",
-                              rangeIterators.stream().map(KeyRangeIterator::getMaxKeys).map(Object::toString).collect(Collectors.joining(", ")),
-                              initialSize);
-
-            return buildIterator(selectiveStatistics, rangeIterators);
-        }
-
-        public boolean isDisjoint()
-        {
-            return isDisjoint;
-        }
-
-        private KeyRangeIterator buildIterator(Statistics statistics, List<KeyRangeIterator> ranges)
-        {
-            // if the ranges are disjoint, or we have an intersection with an empty set,
+            // if the range is disjoint or we have an intersection with an empty set,
             // we can simply return an empty iterator, because it's not going to produce any results.
-            if (isDisjoint)
+            if (statistics.isEmptyOrDisjoint())
             {
-                FileUtils.closeQuietly(ranges);
-                onClose.run();
+                // release posting lists
+                FileUtils.closeQuietly(rangeIterators);
                 return KeyRangeIterator.empty();
             }
 
-            if (ranges.size() == 1)
-            {
-                KeyRangeIterator single = ranges.get(0);
-                single.setOnClose(onClose);
-                return single;
-            }
-
-            // Make sure intersection is supported on the ranges provided:
-            PrimaryKey.Kind firstKind = null;
-            
-            for (KeyRangeIterator range : ranges)
-            {
-                PrimaryKey key;
-                if(range.hasNext())
-                    key = range.peek();
-                else
-                    key = range.getMaximum();
-
-                if (key != null)
-                    if (firstKind == null)
-                        firstKind = key.kind();
-                    else if (!firstKind.isIntersectable(key.kind()))
-                        throw new IllegalArgumentException("Cannot intersect " + firstKind + " and " + key.kind() + " ranges!");
-            }
-
-            return new KeyRangeIntersectionIterator(statistics, ranges, onClose);
-        }
-
-        private void updateStatistics(Statistics statistics, KeyRangeIterator range)
-        {
-            statistics.update(range);
-            isDisjoint |= isDisjointInternal(statistics.min, statistics.max, range);
-        }
-    }
-
-    private static class IntersectionStatistics extends KeyRangeIterator.Builder.Statistics
-    {
-        private boolean empty = true;
+            if (rangeCount() == 1)
+                return rangeIterators.get(0);
 
-        @Override
-        public void update(KeyRangeIterator range)
-        {
-            // minimum of the intersection is the biggest minimum of individual iterators
-            min = nullSafeMax(min, range.getMinimum());
-            // maximum of the intersection is the smallest maximum of individual iterators
-            max = nullSafeMin(max, range.getMaximum());
-            if (empty)
-            {
-                empty = false;
-                count = range.getMaxKeys();
-            }
-            else
-            {
-                count = Math.min(count, range.getMaxKeys());
-            }
+            return new KeyRangeIntersectionIterator(statistics, rangeIterators);
         }
     }
-
-    @VisibleForTesting
-    protected static boolean isDisjoint(KeyRangeIterator a, KeyRangeIterator b)
-    {
-        return isDisjointInternal(a.peek(), a.getMaximum(), b);
-    }
-
-    /**
-     * Ranges are overlapping the following cases:
-     * <p>
-     *   * When they have a common subrange:
-     * <p>
-     *   min       b.current      max          b.max
-     *   +---------|--------------+------------|
-     * <p>
-     *   b.current      min       max          b.max
-     *   |--------------+---------+------------|
-     * <p>
-     *   min        b.current     b.max        max
-     *   +----------|-------------|------------+
-     * <p>
-     *
-     *  If either range is empty, they're disjoint.
-     */
-    private static boolean isDisjointInternal(PrimaryKey min, PrimaryKey max, KeyRangeIterator b)
-    {
-        return min == null || max == null || b.getMaxKeys() == 0 || min.compareTo(b.getMaximum()) > 0 || (b.hasNext() && b.peek().compareTo(max) > 0);
-    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeIterator.java
index 52c390770b2e..b6da42ba443e 100644
--- a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeIterator.java
+++ b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeIterator.java
@@ -18,63 +18,46 @@
 package org.apache.cassandra.index.sai.iterators;
 
 import java.io.Closeable;
+import java.util.Collection;
+import java.util.List;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
 
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.utils.AbstractGuavaIterator;
-
-import javax.annotation.concurrent.NotThreadSafe;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
 
 /**
- * An abstract implementation of {@link AbstractGuavaIterator} that supports the building and management of
- * concatanation, union and intersection iterators.
- * <p>
  * Range iterators contain primary keys, in sorted order, with no duplicates.  They also
  * know their minimum and maximum keys, and an upper bound on the number of keys they contain.
- * <p>
- * Only certain methods are designed to be overriden.  The others are marked private or final.
  */
-@NotThreadSafe
 public abstract class KeyRangeIterator extends AbstractGuavaIterator<PrimaryKey> implements Closeable
 {
+    private static final Builder.EmptyRangeIterator EMPTY = new Builder.EmptyRangeIterator();
+
     private final PrimaryKey min, max;
     private final long count;
-    private Runnable onClose;
 
-    protected KeyRangeIterator(Builder.Statistics statistics, Runnable onClose)
+    protected KeyRangeIterator(Builder.Statistics statistics)
     {
-        this(statistics.min, statistics.max, statistics.count, onClose);
+        this(statistics.min, statistics.max, statistics.tokenCount);
     }
 
-    public KeyRangeIterator(KeyRangeIterator range, Runnable onClose)
+    public KeyRangeIterator(KeyRangeIterator range)
     {
-        this(range == null ? null : range.min,
-             range == null ? null : range.max,
-             range == null ? -1 : range.count,
-             onClose);
+        this(range == null ? null : range.min, range == null ? null : range.max, range == null ? -1 : range.count);
     }
 
     public KeyRangeIterator(PrimaryKey min, PrimaryKey max, long count)
     {
-        this(min, max, count, () -> {});
-    }
-
-    public KeyRangeIterator(PrimaryKey min, PrimaryKey max, long count, Runnable onClose)
-    {
-        boolean isComplete = min != null && max != null && count != 0;
-        boolean isEmpty = min == null && max == null && (count == 0 || count == -1);
-        Preconditions.checkArgument(isComplete || isEmpty, "Range: [%s,%s], Count: %d", min, max, count);
-
-        if (isEmpty)
-          endOfData();
+        if (min == null || max == null || count == 0)
+        {
+            assert min == null && max == null && (count == 0 || count == -1) : min + " - " + max + " " + count;
+            endOfData();
+        }
 
         this.min = min;
         this.max = max;
         this.count = count;
-        this.onClose = onClose;
     }
 
     public final PrimaryKey getMinimum()
@@ -95,76 +78,58 @@ public final long getMaxKeys()
         return count;
     }
 
+    public final PrimaryKey nextOrNull()
+    {
+        return hasNext() ? next() : null;
+    }
+
     /**
-     * When called, this iterator's current position will
+     * When called, this iterators current position will
      * be skipped forwards until finding either:
-     *   1) an element equal to or bigger than nextKey
+     *   1) an element equal to or bigger than next
      *   2) the end of the iterator
      *
-     * @param nextKey value to skip the iterator forward until matching
+     * @param nextToken value to skip the iterator forward until matching
      */
-    public final void skipTo(PrimaryKey nextKey)
+    public final void skipTo(PrimaryKey nextToken)
     {
         if (state == State.DONE)
             return;
 
-        if (state == State.READY && next.compareTo(nextKey) >= 0)
-            return;
-
-        if (max.compareTo(nextKey) < 0)
-        {
-            endOfData();
+        if (state == State.READY && next.compareTo(nextToken) >= 0)
             return;
-        }
 
-        performSkipTo(nextKey);
+        performSkipTo(nextToken);
         state = State.NOT_READY;
     }
 
     /**
-     * Skip to nextKey.
-     * <p>
-     * That is, implementations should set up the iterator state such that
-     * calling computeNext() will return nextKey if present,
-     * or the first one after it if not present.
+     * Skip up to nextKey, but leave the internal state in a position where
+     * calling computeNext() will return nextKey or the first one after it.
      */
     protected abstract void performSkipTo(PrimaryKey nextKey);
 
-    public void setOnClose(Runnable onClose)
-    {
-        this.onClose = onClose;
-    }
-
-    @Override
-    public void close()
-    {
-        onClose.run();
-    }
-
     public static KeyRangeIterator empty()
     {
-        return EmptyRangeIterator.instance;
+        return EMPTY;
     }
 
-    private static class EmptyRangeIterator extends KeyRangeIterator
-    {
-        static final KeyRangeIterator instance = new EmptyRangeIterator();
-        EmptyRangeIterator() { super(null, null, 0, () -> {}); }
-        public PrimaryKey computeNext() { return endOfData(); }
-        protected void performSkipTo(PrimaryKey nextKey) { }
-        public void close() { }
-    }
-
-    @VisibleForTesting
     public static abstract class Builder
     {
+        public enum IteratorType
+        {
+            CONCAT,
+            UNION,
+            INTERSECTION
+        }
+
+        @VisibleForTesting
         protected final Statistics statistics;
-        protected final Runnable onClose;
 
-        public Builder(Statistics statistics, Runnable onClose)
+
+        public Builder(IteratorType type)
         {
-            this.statistics = statistics;
-            this.onClose = onClose;
+            statistics = new Statistics(type);
         }
 
         public PrimaryKey getMinimum()
@@ -177,66 +142,189 @@ public PrimaryKey getMaximum()
             return statistics.max;
         }
 
-        public long getCount()
+        public long getTokenCount()
         {
-            return statistics.count;
+            return statistics.tokenCount;
         }
 
-        public Builder add(Iterable<KeyRangeIterator> ranges)
-        {
-            if (ranges == null || Iterables.isEmpty(ranges))
-                return this;
+        public abstract int rangeCount();
 
-            ranges.forEach(this::add);
-            return this;
-        }
+        public abstract Collection<KeyRangeIterator> ranges();
+
+        // Implementation takes ownership of the range iterator. If the implementation decides not to include it, such
+        // that `rangeCount` may return 0, it must close the range iterator.
+        public abstract Builder add(KeyRangeIterator range);
+
+        public abstract Builder add(List<KeyRangeIterator> ranges);
 
         public final KeyRangeIterator build()
         {
             if (rangeCount() == 0)
-            {
-                onClose.run();
-                return empty();
-            }
+                return new EmptyRangeIterator();
             else
-            {
                 return buildIterator();
-            }
         }
 
-        public abstract Builder add(KeyRangeIterator range);
-
-        public abstract int rangeCount();
-
-        public void cleanup()
+        public static class EmptyRangeIterator extends KeyRangeIterator
         {
-            onClose.run();
+            EmptyRangeIterator() { super(null, null, 0); }
+            public org.apache.cassandra.index.sai.utils.PrimaryKey computeNext() { return endOfData(); }
+            protected void performSkipTo(org.apache.cassandra.index.sai.utils.PrimaryKey nextToken) { }
+            public void close() { }
         }
 
         protected abstract KeyRangeIterator buildIterator();
 
-        public static abstract class Statistics
+        public static class Statistics
         {
-            protected PrimaryKey min, max;
-            protected long count;
+            protected final IteratorType iteratorType;
+
+            protected org.apache.cassandra.index.sai.utils.PrimaryKey min, max;
+            protected long tokenCount;
+
+            // iterator with the least number of items
+            protected KeyRangeIterator minRange;
+            // iterator with the most number of items
+            protected KeyRangeIterator maxRange;
+
+
+            private boolean hasRange = false;
+
+            public Statistics(IteratorType iteratorType)
+            {
+                this.iteratorType = iteratorType;
+            }
+
+            /**
+             * Update statistics information with the given range.
+             *
+             * Updates min/max of the combined range, token count and
+             * tracks range with the least/most number of tokens.
+             *
+             * @param range The range to update statistics with.
+             */
+            public void update(KeyRangeIterator range)
+            {
+                switch (iteratorType)
+                {
+                    case CONCAT:
+                        // range iterators should be sorted, but previous max must not be greater than next min.
+                        if (range.getMaxKeys() > 0)
+                        {
+                            if (tokenCount == 0)
+                            {
+                                min = range.getMinimum();
+                            }
+                            else if (tokenCount > 0 && max.compareTo(range.getMinimum()) > 0)
+                            {
+                                throw new IllegalArgumentException("KeyRangeIterator must be sorted, previous max: " + max + ", next min: " + range.getMinimum());
+                            }
+
+                            max = range.getMaximum();
+                        }
+                        tokenCount += range.getMaxKeys();
+                        break;
+
+                    case UNION:
+                        min = nullSafeMin(min, range.getMinimum());
+                        max = nullSafeMax(max, range.getMaximum());
+                        tokenCount += range.getMaxKeys();
+                        break;
+
+                    case INTERSECTION:
+                        // minimum of the intersection is the biggest minimum of individual iterators
+                        min = nullSafeMax(min, range.getMinimum());
+                        // maximum of the intersection is the smallest maximum of individual iterators
+                        max = nullSafeMin(max, range.getMaximum());
+                        if (hasRange)
+                            tokenCount = Math.min(tokenCount, range.getMaxKeys());
+                        else
+                            tokenCount = range.getMaxKeys();
+
+                        break;
+
+                    default:
+                        throw new IllegalStateException("Unknown iterator type: " + iteratorType);
+                }
+
+                minRange = minRange == null ? range : min(minRange, range);
+                maxRange = maxRange == null ? range : max(maxRange, range);
+
+                hasRange = true;
+            }
+
+            private KeyRangeIterator min(KeyRangeIterator a, KeyRangeIterator b)
+            {
+                return a.getMaxKeys() > b.getMaxKeys() ? b : a;
+            }
+
+            private KeyRangeIterator max(KeyRangeIterator a, KeyRangeIterator b)
+            {
+                return a.getMaxKeys() > b.getMaxKeys() ? a : b;
+            }
 
-            public abstract void update(KeyRangeIterator range);
+            /**
+             * Returns true if the final range is not going to produce any results,
+             * so we can cleanup range storage and never added anything to it.
+             */
+            public boolean isEmptyOrDisjoint()
+            {
+                // max < min if intersected ranges are disjoint
+                return tokenCount == 0 || min.compareTo(max) > 0;
+            }
+
+            public double sizeRatio()
+            {
+                return minRange.getMaxKeys() * 1d / maxRange.getMaxKeys();
+            }
         }
     }
 
-    protected static PrimaryKey nullSafeMin(PrimaryKey a, PrimaryKey b)
+    @VisibleForTesting
+    protected static <U extends Comparable<U>> boolean isOverlapping(KeyRangeIterator a, KeyRangeIterator b)
+    {
+        return isOverlapping(a.peek(), a.getMaximum(), b);
+    }
+
+    /**
+     * Ranges are overlapping the following cases:
+     *
+     *   * When they have a common subrange:
+     *
+     *   min       b.current      max          b.max
+     *   +---------|--------------+------------|
+     *
+     *   b.current      min       max          b.max
+     *   |--------------+---------+------------|
+     *
+     *   min        b.current     b.max        max
+     *   +----------|-------------|------------+
+     *
+     *
+     *  If either range is empty, they're disjoint.
+     */
+    @VisibleForTesting
+    protected static boolean isOverlapping(PrimaryKey min, PrimaryKey max, KeyRangeIterator b)
+    {
+        return (min != null && max != null) &&
+               b.hasNext() && min.compareTo(b.getMaximum()) <= 0 && b.peek().compareTo(max) <= 0;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T extends Comparable> T nullSafeMin(T a, T b)
     {
         if (a == null) return b;
         if (b == null) return a;
 
-        return a.compareToStrict(b) > 0 ? b : a;
+        return a.compareTo(b) > 0 ? b : a;
     }
 
-    protected static PrimaryKey nullSafeMax(PrimaryKey a, PrimaryKey b)
+    @SuppressWarnings("unchecked")
+    private static <T extends Comparable> T nullSafeMax(T a, T b)
     {
         if (a == null) return b;
         if (b == null) return a;
 
-        return a.compareToStrict(b) > 0 ? a : b;
+        return a.compareTo(b) > 0 ? a : b;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeLazyIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeLazyIterator.java
new file mode 100644
index 000000000000..41dd2e92e58c
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeLazyIterator.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.iterators;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+/**
+ * Delays creating an iterator to the first use.
+ */
+public class KeyRangeLazyIterator extends KeyRangeIterator
+{
+    private KeyRangeIterator inner;
+    private final Supplier<KeyRangeIterator> factory;
+
+    public KeyRangeLazyIterator(Supplier<KeyRangeIterator> factory, PrimaryKey min, PrimaryKey max, long count)
+    {
+        super(min, max, count);
+        this.factory = factory;
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        maybeInitialize();
+        inner.skipTo(nextKey);
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        maybeInitialize();
+        return inner.hasNext() ? inner.next() : endOfData();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        if (inner != null)
+            inner.close();
+    }
+
+    private void maybeInitialize()
+    {
+        if (inner == null)
+        {
+            inner = factory.get();
+            assert inner != null;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeListIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeListIterator.java
deleted file mode 100644
index 334e740db87f..000000000000
--- a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeListIterator.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.iterators;
-
-import java.util.List;
-
-import com.google.common.collect.Iterators;
-import com.google.common.collect.PeekingIterator;
-
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-/**
- * A {@link KeyRangeIterator} that iterates over a list of {@link PrimaryKey}s without modifying the underlying list.
- */
-public class KeyRangeListIterator extends KeyRangeIterator
-{
-    private final PeekingIterator<PrimaryKey> keyQueue;
-
-    /**
-     * Create a new {@link KeyRangeListIterator} that iterates over the provided list of keys.
-     *
-     * @param minimumKey the minimum key for the provided list of keys
-     * @param maximumKey the maximum key for the provided list of keys
-     * @param keys the list of keys to iterate over
-     */
-    public KeyRangeListIterator(PrimaryKey minimumKey, PrimaryKey maximumKey, List<PrimaryKey> keys)
-    {
-        super(minimumKey, maximumKey, keys.size());
-        this.keyQueue = Iterators.peekingIterator(keys.iterator());
-    }
-
-    @Override
-    protected void performSkipTo(PrimaryKey nextKey)
-    {
-        while (keyQueue.hasNext())
-        {
-            if (keyQueue.peek().compareTo(nextKey) >= 0)
-                break;
-            keyQueue.next();
-        }
-    }
-
-    @Override
-    public void close() {}
-
-    @Override
-    protected PrimaryKey computeNext()
-    {
-        return keyQueue.hasNext() ? keyQueue.next() : endOfData();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeOrderingIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeOrderingIterator.java
deleted file mode 100644
index c63c79fddb3c..000000000000
--- a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeOrderingIterator.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.iterators;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Function;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.util.FileUtils;
-
-/**
- * An iterator that consumes a chunk of {@link PrimaryKey}s from the {@link KeyRangeIterator}, passes them to the
- * {@link Function} to filter the chunk of {@link PrimaryKey}s and then pass the results to next consumer.
- * The PKs are currently returned in {@link PrimaryKey} order, but that contract may change.
- */
-@NotThreadSafe
-public class KeyRangeOrderingIterator extends KeyRangeIterator
-{
-    private final KeyRangeIterator input;
-    private final int chunkSize;
-    private final Function<List<PrimaryKey>, KeyRangeIterator> nextRangeFunction;
-    private final ArrayList<PrimaryKey> nextKeys;
-    private KeyRangeIterator nextIterator;
-
-    public KeyRangeOrderingIterator(KeyRangeIterator input, int chunkSize, Function<List<PrimaryKey>, KeyRangeIterator> nextRangeFunction)
-    {
-        super(input, () -> {});
-        this.input = input;
-        this.chunkSize = chunkSize;
-        this.nextRangeFunction = nextRangeFunction;
-        this.nextKeys = new ArrayList<>(chunkSize);
-    }
-
-    @Override
-    public PrimaryKey computeNext()
-    {
-        if (nextIterator == null || !nextIterator.hasNext())
-        {
-            do
-            {
-                if (!input.hasNext())
-                    return endOfData();
-                nextKeys.clear();
-                do
-                {
-                    nextKeys.add(input.next());
-                }
-                while (nextKeys.size() < chunkSize && input.hasNext());
-                // Get the next iterator before closing this one to prevent releasing the resource.
-                var previousIterator = nextIterator;
-                // If this results in an exception, previousIterator is closed in close() method.
-                nextIterator = nextRangeFunction.apply(nextKeys);
-                if (previousIterator != null)
-                    FileUtils.closeQuietly(previousIterator);
-                // nextIterator might not have any rows due to shadowed primary keys
-            }
-            while (!nextIterator.hasNext());
-        }
-        return nextIterator.next();
-    }
-
-    @Override
-    protected void performSkipTo(PrimaryKey nextToken)
-    {
-        input.skipTo(nextToken);
-        if (nextIterator != null)
-            nextIterator.skipTo(nextToken);
-    }
-
-    public void close()
-    {
-        FileUtils.closeQuietly(input);
-        FileUtils.closeQuietly(nextIterator);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeTermIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeTermIterator.java
new file mode 100644
index 000000000000..83e695eef2f4
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeTermIterator.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.iterators;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.AbortedOperationException;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Throwables;
+
+/**
+ * KeyRangeTermIterator wraps KeyRangeUnionIterator with code that tracks and releases the referenced indexes,
+ * and adds timeout checkpoints around expensive operations.
+ */
+public class KeyRangeTermIterator extends KeyRangeIterator
+{
+    private static final Logger logger = LoggerFactory.getLogger(KeyRangeTermIterator.class);
+
+    private final QueryContext context;
+
+    private final KeyRangeIterator union;
+    private final Set<SSTableIndex> referencedIndexes;
+
+    private KeyRangeTermIterator(KeyRangeIterator union, Set<SSTableIndex> referencedIndexes, QueryContext queryContext)
+    {
+        super(union.getMinimum(), union.getMaximum(), union.getMaxKeys());
+
+        this.union = union;
+        this.referencedIndexes = referencedIndexes;
+        this.context = queryContext;
+
+        for (SSTableIndex index : referencedIndexes)
+        {
+            boolean success = index.reference();
+            // Won't happen, because the indexes we get here must be already referenced by the query view
+            assert success : "Failed to reference the index " + index;
+        }
+    }
+
+
+    @SuppressWarnings("resource")
+    public static KeyRangeTermIterator build(final Expression e, Set<SSTableIndex> perSSTableIndexes, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, boolean defer, int limit)
+    {
+        KeyRangeIterator rangeIterator = buildRangeIterator(e, perSSTableIndexes, keyRange, queryContext, defer, limit);
+        return new KeyRangeTermIterator(rangeIterator, perSSTableIndexes, queryContext);
+    }
+
+    private static KeyRangeIterator buildRangeIterator(final Expression e, Set<SSTableIndex> perSSTableIndexes, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, boolean defer, int limit)
+    {
+        final List<KeyRangeIterator> tokens = new ArrayList<>(1 + perSSTableIndexes.size());
+
+        KeyRangeIterator memtableIterator = e.context.searchMemtable(queryContext, e, keyRange, limit);
+        if (memtableIterator != null)
+            tokens.add(memtableIterator);
+
+        for (final SSTableIndex index : perSSTableIndexes)
+        {
+            try
+            {
+                queryContext.checkpoint();
+                queryContext.addSstablesHit(1);
+                assert !index.isReleased();
+
+                KeyRangeIterator keyIterator = index.search(e, keyRange, queryContext, defer, limit);
+
+                if (keyIterator == null || !keyIterator.hasNext())
+                    continue;
+
+                tokens.add(keyIterator);
+            }
+            catch (Throwable e1)
+            {
+                if (logger.isDebugEnabled() && !(e1 instanceof AbortedOperationException))
+                    logger.debug(String.format("Failed search an index %s, skipping.", index.getSSTable()), e1);
+
+                // Close the iterators that were successfully opened before the error
+                FileUtils.closeQuietly(tokens);
+
+                throw Throwables.cleaned(e1);
+            }
+        }
+
+        return KeyRangeUnionIterator.build(tokens);
+    }
+
+    protected PrimaryKey computeNext()
+    {
+        try
+        {
+            return union.hasNext() ? union.next() : endOfData();
+        }
+        finally
+        {
+            context.checkpoint();
+        }
+    }
+
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        try
+        {
+            union.skipTo(nextKey);
+        }
+        finally
+        {
+            context.checkpoint();
+        }
+    }
+
+    public void close()
+    {
+        FileUtils.closeQuietly(union);
+        referencedIndexes.forEach(KeyRangeTermIterator::releaseQuietly);
+    }
+
+    private static void releaseQuietly(SSTableIndex index)
+    {
+        try
+        {
+            index.release();
+        }
+        catch (Throwable e)
+        {
+            logger.error(String.format("Failed to release index %s", index.getSSTable()), e);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeUnionIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeUnionIterator.java
index 7bf73ab53453..bdb27028a7fe 100644
--- a/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeUnionIterator.java
+++ b/src/java/org/apache/cassandra/index/sai/iterators/KeyRangeUnionIterator.java
@@ -17,136 +17,98 @@
  */
 package org.apache.cassandra.index.sai.iterators;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.io.util.FileUtils;
 
 /**
- * Range Union Iterator is used to return sorted stream of elements from multiple RangeIterator instances.
+ * Range Union Iterator is used to return sorted stream of elements from multiple KeyRangeIterator instances.
  */
+@SuppressWarnings("resource")
 public class KeyRangeUnionIterator extends KeyRangeIterator
 {
-    private final List<KeyRangeIterator> ranges;
-    private final List<KeyRangeIterator> candidates;
+    public final List<KeyRangeIterator> ranges;
 
-    private KeyRangeUnionIterator(Builder.Statistics statistics, List<KeyRangeIterator> ranges, Runnable onClose)
+    private KeyRangeUnionIterator(Builder.Statistics statistics, List<KeyRangeIterator> ranges)
     {
-        super(statistics, onClose);
-        this.ranges = ranges;
-        this.candidates = new ArrayList<>(ranges.size());
+        super(statistics);
+        this.ranges = new ArrayList<>(ranges);
     }
 
-    @Override
     public PrimaryKey computeNext()
     {
-        // the design is to find the next best value from all the ranges,
-        // and then advance all the ranges that have the same value.
-        candidates.clear();
-        PrimaryKey candidateKey = null;
+        // Keep track of the next best candidate. If another candidate has the same value, advance it to prevent
+        // duplicate results. This design avoids unnecessary list operations.
+        KeyRangeIterator candidate = null;
         for (KeyRangeIterator range : ranges)
         {
             if (!range.hasNext())
                 continue;
 
-            if (candidateKey == null)
+            if (candidate == null)
             {
-                candidateKey = range.peek();
-                candidates.add(range);
+                candidate = range;
             }
             else
             {
-                PrimaryKey peeked = range.peek();
-    
-                int cmp = candidateKey.compareTo(peeked);
-
+                int cmp = candidate.peek().compareTo(range.peek());
                 if (cmp == 0)
-                {
-                    // Replace any existing candidate key if this one is STATIC:
-                    if (peeked.kind() == PrimaryKey.Kind.STATIC)
-                        candidateKey = peeked;
-
-                    candidates.add(range);
-                }
+                    range.next();
                 else if (cmp > 0)
-                {
-                    // we found a new best candidate, throw away the old ones
-                    candidates.clear();
-                    candidateKey = peeked;
-                    candidates.add(range);
-                }
-                // else, existing candidate is less than the next in this range
+                    candidate = range;
             }
         }
-        if (candidates.isEmpty())
+        if (candidate == null)
             return endOfData();
-
-        for (KeyRangeIterator candidate : candidates)
-        {
-            do
-            {
-                // Consume the remaining values equal to the candidate key:
-                candidate.next();
-            }
-            while (candidate.hasNext() && candidate.peek().compareTo(candidateKey) == 0);
-        }
-
-        return candidateKey;
+        return candidate.next();
     }
 
-    @Override
     protected void performSkipTo(PrimaryKey nextKey)
     {
+        // Resist the temptation to call range.hasNext before skipTo: this is a pessimisation, hasNext will invoke
+        // computeNext under the hood, which is an expensive operation to produce a value that we plan to throw away.
+        // Instead, it is the responsibility of the child iterators to make skipTo fast when the iterator is exhausted.
         for (KeyRangeIterator range : ranges)
-        {
-            if (range.hasNext())
-                range.skipTo(nextKey);
-        }
+            range.skipTo(nextKey);
     }
 
-    @Override
-    public void close()
+    public void close() throws IOException
     {
-        super.close();
-
         // Due to lazy key fetching, we cannot close iterator immediately
-        FileUtils.closeQuietly(ranges);
+        ranges.forEach(FileUtils::closeQuietly);
     }
 
     public static Builder builder(int size)
     {
-        return builder(size, () -> {});
+        return new Builder(size);
     }
 
-    public static Builder builder(int size, Runnable onClose)
+    public static Builder builder()
     {
-        return new Builder(size, onClose);
+        return builder(10);
     }
 
-    public static KeyRangeIterator build(List<KeyRangeIterator> keys, Runnable onClose)
-    {
-        return new Builder(keys.size(), onClose).add(keys).build();
-    }
-    public static KeyRangeIterator build(List<KeyRangeIterator> keys)
+    public static KeyRangeIterator build(Iterable<KeyRangeIterator> tokens)
     {
-        return build(keys, () -> {});
+        return KeyRangeUnionIterator.builder(Iterables.size(tokens)).add(tokens).build();
     }
 
-    @VisibleForTesting
     public static class Builder extends KeyRangeIterator.Builder
     {
-        protected final List<KeyRangeIterator> rangeIterators;
+        protected List<KeyRangeIterator> rangeIterators;
 
-        Builder(int size, Runnable onClose)
+        public Builder(int size)
         {
-            super(new UnionStatistics(), onClose);
+            super(IteratorType.UNION);
             this.rangeIterators = new ArrayList<>(size);
         }
 
-        @Override
         public KeyRangeIterator.Builder add(KeyRangeIterator range)
         {
             if (range == null)
@@ -158,48 +120,52 @@ public KeyRangeIterator.Builder add(KeyRangeIterator range)
                 statistics.update(range);
             }
             else
-            {
                 FileUtils.closeQuietly(range);
-            }
 
             return this;
         }
 
         @Override
+        public KeyRangeIterator.Builder add(List<KeyRangeIterator> ranges)
+        {
+            if (ranges == null || ranges.isEmpty())
+                return this;
+
+            ranges.forEach(this::add);
+            return this;
+        }
+
+        public KeyRangeIterator.Builder add(Iterable<KeyRangeIterator> ranges)
+        {
+            if (ranges == null || Iterables.isEmpty(ranges))
+                return this;
+
+            ranges.forEach(this::add);
+            return this;
+        }
+
         public int rangeCount()
         {
             return rangeIterators.size();
         }
 
         @Override
-        public void cleanup()
+        public Collection<KeyRangeIterator> ranges()
         {
-            super.cleanup();
-            FileUtils.closeQuietly(rangeIterators);
+            return rangeIterators;
         }
 
-        @Override
         protected KeyRangeIterator buildIterator()
         {
-            if (rangeCount() == 1)
+            switch (rangeCount())
             {
-                KeyRangeIterator single = rangeIterators.get(0);
-                single.setOnClose(onClose);
-                return single;
-            }
-
-            return new KeyRangeUnionIterator(statistics, rangeIterators, onClose);
-        }
-    }
+                case 1:
+                    return rangeIterators.get(0);
 
-    private static class UnionStatistics extends KeyRangeIterator.Builder.Statistics
-    {
-        @Override
-        public void update(KeyRangeIterator range)
-        {
-            min = nullSafeMin(min, range.getMinimum());
-            max = nullSafeMax(max, range.getMaximum());
-            count += range.getMaxKeys();
+                default:
+                    //TODO Need to test whether an initial sort improves things
+                    return new KeyRangeUnionIterator(statistics, rangeIterators);
+            }
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/iterators/RowIdToPrimaryKeyWithSortKeyIterator.java b/src/java/org/apache/cassandra/index/sai/iterators/RowIdToPrimaryKeyWithSortKeyIterator.java
new file mode 100644
index 000000000000..3f3c6dd07eb6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/iterators/RowIdToPrimaryKeyWithSortKeyIterator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.iterators;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.IndexSearcherContext;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RowIdWithMeta;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CloseableIterator;
+
+/**
+ * An iterator over scored primary keys ordered by the score descending
+ * Not skippable.
+ */
+public class RowIdToPrimaryKeyWithSortKeyIterator extends AbstractIterator<PrimaryKeyWithSortKey>
+{
+    private final IndexContext indexContext;
+    private final SSTableId<?> sstableId;
+    private final PrimaryKeyMap primaryKeyMap;
+    private final CloseableIterator<? extends RowIdWithMeta> scoredRowIdIterator;
+    private final IndexSearcherContext searcherContext;
+
+    public RowIdToPrimaryKeyWithSortKeyIterator(IndexContext indexContext,
+                                                SSTableId<?> sstableId,
+                                                CloseableIterator<? extends RowIdWithMeta> scoredRowIdIterator,
+                                                PrimaryKeyMap primaryKeyMap,
+                                                IndexSearcherContext context)
+    {
+        this.indexContext = indexContext;
+        this.sstableId = sstableId;
+        this.scoredRowIdIterator = scoredRowIdIterator;
+        this.primaryKeyMap = primaryKeyMap;
+        this.searcherContext = context;
+    }
+
+    @Override
+    protected PrimaryKeyWithSortKey computeNext()
+    {
+        if (!scoredRowIdIterator.hasNext())
+            return endOfData();
+        var rowIdWithMeta = scoredRowIdIterator.next();
+        return rowIdWithMeta.buildPrimaryKeyWithSortKey(indexContext, sstableId, primaryKeyMap, searcherContext.getSegmentRowIdOffset());
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(primaryKeyMap);
+        FileUtils.closeQuietly(scoredRowIdIterator);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/FilteringInMemoryKeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/memory/FilteringInMemoryKeyRangeIterator.java
deleted file mode 100644
index d54823546a96..000000000000
--- a/src/java/org/apache/cassandra/index/sai/memory/FilteringInMemoryKeyRangeIterator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.memory;
-
-import java.util.SortedSet;
-
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-/**
- * An {@link InMemoryKeyRangeIterator} that filters the returned {@link PrimaryKey}s based on the provided keyRange
- */
-public class FilteringInMemoryKeyRangeIterator extends InMemoryKeyRangeIterator
-{
-    private final AbstractBounds<PartitionPosition> keyRange;
-
-    public FilteringInMemoryKeyRangeIterator(SortedSet<PrimaryKey> keys, AbstractBounds<PartitionPosition> keyRange)
-    {
-        super(keys);
-        this.keyRange = keyRange;
-    }
-
-    @Override
-    protected PrimaryKey computeNext()
-    {
-        PrimaryKey key = computeNextKey();
-        while (key != null && !keyRange.contains(key.partitionKey()))
-            key = computeNextKey();
-        return key == null ? endOfData() : key;
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/FilteringKeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/memory/FilteringKeyRangeIterator.java
new file mode 100644
index 000000000000..bda41c727571
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/memory/FilteringKeyRangeIterator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.memory;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+/**
+ * A {@link KeyRangeIterator} that filters the returned {@link PrimaryKey}s based on the provided keyRange
+ */
+public class FilteringKeyRangeIterator extends KeyRangeIterator
+{
+    private final AbstractBounds<PartitionPosition> keyRange;
+    private final KeyRangeIterator source;
+
+    public FilteringKeyRangeIterator(KeyRangeIterator source, AbstractBounds<PartitionPosition> keyRange)
+    {
+        super(source.getMinimum(), source.getMaximum(), source.getMaxKeys());
+        this.keyRange = keyRange;
+        this.source = source;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        while (source.hasNext())
+        {
+            PrimaryKey key = source.next();
+            if (keyRange.contains(key.partitionKey()))
+                return key;
+        }
+        return endOfData();
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        source.skipTo(nextKey);
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIterator.java
deleted file mode 100644
index d502afebddda..000000000000
--- a/src/java/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIterator.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.memory;
-
-import java.util.PriorityQueue;
-import java.util.SortedSet;
-import javax.annotation.concurrent.NotThreadSafe;
-
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-@NotThreadSafe
-public class InMemoryKeyRangeIterator extends KeyRangeIterator
-{
-    private final PriorityQueue<PrimaryKey> keys;
-    private final boolean uniqueKeys;
-    private PrimaryKey lastKey;
-
-    /**
-     * An in-memory {@link KeyRangeIterator} that uses a {@link PriorityQueue} built from a {@link SortedSet}
-     * which has no duplication as its backing store.
-     */
-    public InMemoryKeyRangeIterator(SortedSet<PrimaryKey> keys)
-    {
-        super(keys.first(), keys.last(), keys.size(), () -> {});
-        this.keys = new PriorityQueue<>(keys);
-        this.uniqueKeys = true;
-    }
-
-    /**
-     * An in-memory {@link KeyRangeIterator} that uses a {@link PriorityQueue} which may
-     * contain duplicated keys as its backing store.
-     */
-    public InMemoryKeyRangeIterator(PrimaryKey min, PrimaryKey max, PriorityQueue<PrimaryKey> keys)
-    {
-        super(min, max, keys.size(), () -> {});
-        this.keys = keys;
-        this.uniqueKeys = false;
-    }
-
-    @Override
-    protected PrimaryKey computeNext()
-    {
-        PrimaryKey key = computeNextKey();
-        return key == null ? endOfData() : key;
-    }
-
-    protected PrimaryKey computeNextKey()
-    {
-        PrimaryKey next = null;
-
-        while (!keys.isEmpty())
-        {
-            PrimaryKey key = keys.poll();
-            if (uniqueKeys)
-                return key;
-
-            if (lastKey == null || lastKey.compareTo(key) != 0)
-            {
-                next = key;
-                lastKey = key;
-                break;
-            }
-        }
-
-        return next;
-    }
-
-    @Override
-    protected void performSkipTo(PrimaryKey nextKey)
-    {
-        while (!keys.isEmpty())
-        {
-            PrimaryKey key = keys.peek();
-            if (key.compareTo(nextKey) >= 0)
-                break;
-
-            // consume smaller key
-            keys.poll();
-        }
-    }
-
-    @Override
-    public void close()
-    {}
-}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java
index 4307727b88ae..4f1fe7724198 100644
--- a/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java
+++ b/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java
@@ -18,43 +18,67 @@
 
 package org.apache.cassandra.index.sai.memory;
 
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.LongConsumer;
+
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
 import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.utils.CloseableIterator;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.function.Function;
-
-public abstract class MemoryIndex implements MemtableOrdering
+public abstract class MemoryIndex
 {
-    protected final StorageAttachedIndex index;
+    protected final IndexContext indexContext;
 
-    protected MemoryIndex(StorageAttachedIndex index)
+    protected MemoryIndex(IndexContext indexContext)
     {
-        this.index = index;
+        this.indexContext = indexContext;
     }
 
-    public abstract long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value);
+    public abstract void add(DecoratedKey key,
+                             Clustering clustering,
+                             ByteBuffer value,
+                             LongConsumer onHeapAllocationsTracker,
+                             LongConsumer offHeapAllocationsTracker);
 
-    public abstract long update(DecoratedKey key, Clustering<?> clustering, ByteBuffer oldValue, ByteBuffer newValue);
+    /**
+     * Update the index value for the given key and clustering by removing the old value and adding the new value.
+     * This is meant to be used when the indexed column is any type other than a non-frozen collection.
+     */
+    public abstract void update(DecoratedKey key,
+                                Clustering clustering,
+                                ByteBuffer oldValue,
+                                ByteBuffer newValue,
+                                LongConsumer onHeapAllocationsTracker,
+                                LongConsumer offHeapAllocationsTracker);
 
-    public abstract KeyRangeIterator search(QueryContext queryContext, Expression expression, AbstractBounds<PartitionPosition> keyRange);
+    /**
+     * Update the index value for the given key and clustering by removing the old values and adding the new values.
+     * This is meant to be used when the indexed column is a non-frozen collection.
+     */
+    public abstract void update(DecoratedKey key,
+                                Clustering clustering,
+                                Iterator<ByteBuffer> oldValues,
+                                Iterator<ByteBuffer> newValues,
+                                LongConsumer onHeapAllocationsTracker,
+                                LongConsumer offHeapAllocationsTracker);
+
+    public abstract CloseableIterator<PrimaryKeyWithSortKey> orderBy(Orderer orderer, Expression slice);
+
+    public abstract KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange);
 
-    public abstract boolean isEmpty();
+    public abstract long estimateMatchingRowsCount(Expression expression, AbstractBounds<PartitionPosition> keyRange);
 
     public abstract ByteBuffer getMinTerm();
 
@@ -63,9 +87,17 @@ protected MemoryIndex(StorageAttachedIndex index)
     /**
      * Iterate all Term->PrimaryKeys mappings in sorted order
      */
-    public abstract Iterator<Pair<ByteComparable, PrimaryKeys>> iterator();
+    public abstract Iterator<Pair<ByteComparable.Preencoded, List<PkWithFrequency>>> iterator();
 
-    public abstract SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor,
-                                                                     IndexIdentifier indexIdentifier,
-                                                                     Function<PrimaryKey, Integer> postingTransformer) throws IOException;
+    public static class PkWithFrequency
+    {
+        public final PrimaryKey pk;
+        public final int frequency;
+
+        public PkWithFrequency(PrimaryKey pk, int frequency)
+        {
+            this.pk = pk;
+            this.frequency = frequency;
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java
index f0f2ea36ad6c..f2d55017e818 100644
--- a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java
+++ b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java
@@ -18,104 +18,69 @@
 
 package org.apache.cassandra.index.sai.memory;
 
-import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.atomic.LongAdder;
-import java.util.function.Function;
+import javax.annotation.Nullable;
 
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex;
 import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
 import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.index.sai.utils.MemtableOrdering;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 
-public class MemtableIndex implements MemtableOrdering
+public interface MemtableIndex extends MemtableOrdering
 {
-    private final MemoryIndex memoryIndex;
-    private final LongAdder writeCount = new LongAdder();
-    private final LongAdder estimatedMemoryUsed = new LongAdder();
-    private final AbstractType<?> type;
+    Memtable getMemtable();
 
-    public MemtableIndex(StorageAttachedIndex index)
-    {
-        this.memoryIndex = index.termType().isVector() ? new VectorMemoryIndex(index) : new TrieMemoryIndex(index);
-        this.type = index.termType().indexType();
-    }
+    long writeCount();
 
-    public long writeCount()
-    {
-        return writeCount.sum();
-    }
+    long estimatedOnHeapMemoryUsed();
 
-    public long estimatedMemoryUsed()
-    {
-        return estimatedMemoryUsed.sum();
-    }
+    long estimatedOffHeapMemoryUsed();
 
-    public boolean isEmpty()
-    {
-        return memoryIndex.isEmpty();
-    }
+    boolean isEmpty();
 
-    public ByteBuffer getMinTerm()
-    {
-        return memoryIndex.getMinTerm();
-    }
+    // Returns the minimum indexed term in the combined memory indexes.
+    // This can be null if the indexed memtable was empty. Users of the
+    // {@code MemtableIndex} requiring a non-null minimum term should
+    // use the {@link MemtableIndex#isEmpty} method.
+    // Note: Individual index shards can return null here if the index
+    // didn't receive any terms within the token range of the shard
+    @Nullable
+    ByteBuffer getMinTerm();
 
-    public ByteBuffer getMaxTerm()
-    {
-        return memoryIndex.getMaxTerm();
-    }
+    // Returns the maximum indexed term in the combined memory indexes.
+    // This can be null if the indexed memtable was empty. Users of the
+    // {@code MemtableIndex} requiring a non-null maximum term should
+    // use the {@link MemtableIndex#isEmpty} method.
+    // Note: Individual index shards can return null here if the index
+    // didn't receive any terms within the token range of the shard
+    @Nullable
+    ByteBuffer getMaxTerm();
 
-    public long index(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
-    {
-        if (value == null || (value.remaining() == 0 && !type.allowsEmpty()))
-            return 0;
+    void index(DecoratedKey key, Clustering clustering, ByteBuffer value, Memtable memtable, OpOrder.Group opGroup);
 
-        long ram = memoryIndex.add(key, clustering, value);
-        writeCount.increment();
-        estimatedMemoryUsed.add(ram);
-        return ram;
-    }
+    void update(DecoratedKey key, Clustering clustering, ByteBuffer oldValue, ByteBuffer newValue, Memtable memtable, OpOrder.Group opGroup);
+    void update(DecoratedKey key, Clustering clustering, Iterator<ByteBuffer> oldValues, Iterator<ByteBuffer> newValues, Memtable memtable, OpOrder.Group opGroup);
 
-    public long update(DecoratedKey key, Clustering<?> clustering, ByteBuffer oldValue, ByteBuffer newValue)
-    {
-        return memoryIndex.update(key, clustering, oldValue, newValue);
-    }
+    KeyRangeIterator search(QueryContext queryContext, Expression expression, AbstractBounds<PartitionPosition> keyRange, int limit);
 
-    public KeyRangeIterator search(QueryContext queryContext, Expression expression, AbstractBounds<PartitionPosition> keyRange)
-    {
-        return memoryIndex.search(queryContext, expression, keyRange);
-    }
+    long estimateMatchingRowsCount(Expression expression, AbstractBounds<PartitionPosition> keyRange);
 
-    public Iterator<Pair<ByteComparable, PrimaryKeys>> iterator()
-    {
-        return memoryIndex.iterator();
-    }
-
-    public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor,
-                                                            IndexIdentifier indexIdentifier,
-                                                            Function<PrimaryKey, Integer> postingTransformer) throws IOException
-    {
-        return memoryIndex.writeDirect(indexDescriptor, indexIdentifier, postingTransformer);
-    }
+    Iterator<Pair<ByteComparable.Preencoded, List<MemoryIndex.PkWithFrequency>>> iterator(DecoratedKey min, DecoratedKey max);
 
-    @Override
-    public KeyRangeIterator limitToTopResults(List<PrimaryKey> primaryKeys, Expression expression, int limit)
+    static MemtableIndex createIndex(IndexContext indexContext, Memtable mt)
     {
-        return memoryIndex.limitToTopResults(primaryKeys, expression, limit);
+        return indexContext.isVector() ? new VectorMemtableIndex(indexContext, mt) : new TrieMemtableIndex(indexContext, mt);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndexManager.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndexManager.java
deleted file mode 100644
index 36e48a04b1be..000000000000
--- a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndexManager.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.memory;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.utils.Clock;
-import org.apache.cassandra.utils.FBUtilities;
-
-public class MemtableIndexManager
-{
-    private final StorageAttachedIndex index;
-    private final ConcurrentMap<Memtable, MemtableIndex> liveMemtableIndexMap;
-
-    public MemtableIndexManager(StorageAttachedIndex index)
-    {
-        this.index = index;
-        this.liveMemtableIndexMap = new ConcurrentHashMap<>();
-    }
-
-    public long index(DecoratedKey key, Row row, Memtable mt)
-    {
-        MemtableIndex current = liveMemtableIndexMap.get(mt);
-
-        // We expect the relevant IndexMemtable to be present most of the time, so only make the
-        // call to computeIfAbsent() if it's not. (see https://bugs.openjdk.java.net/browse/JDK-8161372)
-        MemtableIndex target = (current != null)
-                               ? current
-                               : liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex(index));
-
-        long start = Clock.Global.nanoTime();
-
-        long bytes = 0;
-
-        if (index.termType().isNonFrozenCollection())
-        {
-            Iterator<ByteBuffer> bufferIterator = index.termType().valuesOf(row, FBUtilities.nowInSeconds());
-            if (bufferIterator != null)
-            {
-                while (bufferIterator.hasNext())
-                {
-                    ByteBuffer value = bufferIterator.next();
-                    bytes += target.index(key, row.clustering(), value);
-                }
-            }
-        }
-        else
-        {
-            ByteBuffer value = index.termType().valueOf(key, row, FBUtilities.nowInSeconds());
-            bytes += target.index(key, row.clustering(), value);
-        }
-        index.indexMetrics().memtableIndexWriteLatency.update(Clock.Global.nanoTime() - start, TimeUnit.NANOSECONDS);
-        return bytes;
-    }
-
-    public long update(DecoratedKey key, Row oldRow, Row newRow, Memtable memtable)
-    {
-        if (!index.termType().isVector())
-        {
-            return index(key, newRow, memtable);
-        }
-
-        MemtableIndex target = liveMemtableIndexMap.get(memtable);
-        if (target == null)
-            return 0;
-
-        ByteBuffer oldValue = index.termType().valueOf(key, oldRow, FBUtilities.nowInSeconds());
-        ByteBuffer newValue = index.termType().valueOf(key, newRow, FBUtilities.nowInSeconds());
-        return target.update(key, oldRow.clustering(), oldValue, newValue);
-    }
-
-    public void renewMemtable(Memtable renewed)
-    {
-        for (Memtable memtable : liveMemtableIndexMap.keySet())
-        {
-            // remove every index but the one that corresponds to the post-truncate Memtable
-            if (renewed != memtable)
-            {
-                liveMemtableIndexMap.remove(memtable);
-            }
-        }
-    }
-
-    public void discardMemtable(Memtable discarded)
-    {
-        liveMemtableIndexMap.remove(discarded);
-    }
-
-    @Nullable
-    public MemtableIndex getPendingMemtableIndex(LifecycleNewTracker tracker)
-    {
-        return liveMemtableIndexMap.keySet().stream()
-                                   .filter(m -> tracker.equals(m.getFlushTransaction()))
-                                   .findFirst()
-                                   .map(liveMemtableIndexMap::get)
-                                   .orElse(null);
-    }
-
-    public KeyRangeIterator searchMemtableIndexes(QueryContext queryContext, Expression e, AbstractBounds<PartitionPosition> keyRange)
-    {
-        Collection<MemtableIndex> memtableIndexes = liveMemtableIndexMap.values();
-
-        if (memtableIndexes.isEmpty())
-        {
-            return KeyRangeIterator.empty();
-        }
-
-        KeyRangeIterator.Builder builder = KeyRangeUnionIterator.builder(memtableIndexes.size());
-
-        for (MemtableIndex memtableIndex : memtableIndexes)
-        {
-            builder.add(memtableIndex.search(queryContext, e, keyRange));
-        }
-
-        return builder.build();
-    }
-
-    public KeyRangeIterator limitToTopResults(QueryContext context, List<PrimaryKey> source, Expression e)
-    {
-        Collection<MemtableIndex> memtables = liveMemtableIndexMap.values();
-
-        if (memtables.isEmpty())
-        {
-            return KeyRangeIterator.empty();
-        }
-
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(memtables.size());
-
-        for (MemtableIndex index : memtables)
-        {
-            builder.add(index.limitToTopResults(source, e, context.vectorContext().limit()));
-        }
-
-        return builder.build();
-    }
-
-    public long liveMemtableWriteCount()
-    {
-        return liveMemtableIndexMap.values().stream().mapToLong(MemtableIndex::writeCount).sum();
-    }
-
-    public long estimatedMemIndexMemoryUsed()
-    {
-        return liveMemtableIndexMap.values().stream().mapToLong(MemtableIndex::estimatedMemoryUsed).sum();
-    }
-
-    @VisibleForTesting
-    public int size()
-    {
-        return liveMemtableIndexMap.size();
-    }
-
-    public void invalidate()
-    {
-        liveMemtableIndexMap.clear();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableKeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableKeyRangeIterator.java
new file mode 100644
index 000000000000..df2466cc0a4f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/memory/MemtableKeyRangeIterator.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Iterates over primary keys in a memtable
+ */
+public class MemtableKeyRangeIterator extends KeyRangeIterator
+{
+    private final Memtable memtable;
+    private final PrimaryKey.Factory pkFactory;
+    private final AbstractBounds<PartitionPosition> keyRange;
+    private final ColumnFilter columns;
+    private UnfilteredPartitionIterator partitionIterator;
+    private UnfilteredRowIterator rowIterator;
+
+    public MemtableKeyRangeIterator(Memtable memtable,
+                                    PrimaryKey.Factory pkFactory,
+                                    AbstractBounds<PartitionPosition> keyRange)
+    {
+        super(minKey(memtable, pkFactory),
+              maxKey(memtable, pkFactory),
+              memtable.operationCount());
+
+        TableMetadata metadata = memtable.metadata();
+        this.memtable = memtable;
+        this.pkFactory = pkFactory;
+        this.keyRange = keyRange;
+        this.columns = ColumnFilter.selectionBuilder()
+                                           .addAll(metadata.partitionKeyColumns())
+                                           .addAll(metadata.clusteringColumns())
+                                           .addAll(metadata.regularColumns())
+                                           .build();
+
+        DataRange dataRange = new DataRange(keyRange, new ClusteringIndexSliceFilter(Slices.ALL, false));
+        this.partitionIterator = memtable.partitionIterator(columns, dataRange, SSTableReadsListener.NOOP_LISTENER);
+        this.rowIterator = null;
+    }
+
+    private static PrimaryKey minKey(Memtable memtable, PrimaryKey.Factory factory)
+    {
+        DecoratedKey pk = memtable.minPartitionKey();
+        return pk != null ? factory.createPartitionKeyOnly(pk) : null;
+    }
+
+    private static PrimaryKey maxKey(Memtable memtable, PrimaryKey.Factory factory)
+    {
+        DecoratedKey pk = memtable.maxPartitionKey();
+        return pk != null ? factory.createPartitionKeyOnly(pk) : null;
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        PartitionPosition start = nextKey.partitionKey() != null
+                                  ? nextKey.partitionKey()
+                                  : nextKey.token().minKeyBound();
+        if (!keyRange.right.isMinimum() && start.compareTo(keyRange.right) > 0)
+        {
+            partitionIterator = EmptyIterators.unfilteredPartition(memtable.metadata());
+            rowIterator = null;
+            return;
+        }
+
+        AbstractBounds<PartitionPosition> partitionBounds = AbstractBounds.bounds(start, true, keyRange.right, true);
+        DataRange dataRange = new DataRange(partitionBounds, new ClusteringIndexSliceFilter(Slices.ALL, false));
+        FileUtils.closeQuietly(partitionIterator);
+        partitionIterator = memtable.partitionIterator(columns, dataRange, SSTableReadsListener.NOOP_LISTENER);
+        if (partitionIterator.hasNext())
+        {
+            this.rowIterator = partitionIterator.next();
+            if (!nextKey.hasEmptyClustering() && rowIterator.partitionKey().equals(nextKey.partitionKey()))
+            {
+                Slice slice = Slice.make(nextKey.clustering(), Clustering.EMPTY);
+                Slices slices = Slices.with(memtable.metadata().comparator, slice);
+                FileUtils.closeQuietly(rowIterator);
+                rowIterator = memtable.getPartition(nextKey.partitionKey()).unfilteredIterator(columns, slices, false);
+            }
+        }
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        FileUtils.close(partitionIterator, rowIterator);
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        while (hasNextRow(rowIterator) || partitionIterator.hasNext())
+        {
+            if (!hasNextRow(rowIterator))
+            {
+                FileUtils.closeQuietly(rowIterator);
+                rowIterator = partitionIterator.next();
+                continue;
+            }
+
+            Unfiltered unfiltered = rowIterator.next();
+            if (unfiltered.isRow())
+            {
+                Row row = (Row) unfiltered;
+                return pkFactory.create(rowIterator.partitionKey(), row.clustering());
+            }
+        }
+        return endOfData();
+    }
+
+    private static boolean hasNextRow(UnfilteredRowIterator rowIterator)
+    {
+        return rowIterator != null && rowIterator.hasNext();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableOrdering.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableOrdering.java
deleted file mode 100644
index bd084a546b55..000000000000
--- a/src/java/org/apache/cassandra/index/sai/memory/MemtableOrdering.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.memory;
-
-import java.util.List;
-
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-/**
- * Analogue of {@link org.apache.cassandra.index.sai.disk.v1.segment.SegmentOrdering}, but for memtables.
- */
-public interface MemtableOrdering
-{
-    /**
-     * Filter the given list of {@code PrimaryKey} results to the top `limit` results corresponding to the given expression,
-     * Returns an iterator over the results that is put back in token order.
-     * <p>
-     * Assumes that the given list spans the same rows as the implementing index's segment.
-     */
-    default KeyRangeIterator limitToTopResults(List<PrimaryKey> primaryKeys, Expression expression, int limit)
-    {
-        throw new UnsupportedOperationException();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableTermsIterator.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableTermsIterator.java
deleted file mode 100644
index 638b0d218339..000000000000
--- a/src/java/org/apache/cassandra/index/sai/memory/MemtableTermsIterator.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.memory;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-import com.google.common.base.Preconditions;
-
-import com.carrotsearch.hppc.LongArrayList;
-import com.carrotsearch.hppc.cursors.LongCursor;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.index.sai.utils.TermsIterator;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-
-/**
- * Iterator over a token range bounded segment of a Memtable index. Used to flush Memtable index segments to disk.
- */
-public class MemtableTermsIterator implements TermsIterator
-{
-    private final ByteBuffer minTerm;
-    private final ByteBuffer maxTerm;
-    private final Iterator<Pair<ByteComparable, LongArrayList>> iterator;
-
-    private Pair<ByteComparable, LongArrayList> current;
-
-    private long maxSSTableRowId = -1;
-    private long minSSTableRowId = Long.MAX_VALUE;
-
-    public MemtableTermsIterator(ByteBuffer minTerm,
-                                 ByteBuffer maxTerm,
-                                 Iterator<Pair<ByteComparable, LongArrayList>> iterator)
-    {
-        Preconditions.checkArgument(iterator != null);
-        this.minTerm = minTerm;
-        this.maxTerm = maxTerm;
-        this.iterator = iterator;
-    }
-
-    @Override
-    public ByteBuffer getMinTerm()
-    {
-        return minTerm;
-    }
-
-    @Override
-    public ByteBuffer getMaxTerm()
-    {
-        return maxTerm;
-    }
-
-    @Override
-    public void close() {}
-
-    @Override
-    public boolean hasNext()
-    {
-        return iterator.hasNext();
-    }
-
-    @Override
-    public IndexEntry next()
-    {
-        current = iterator.next();
-        return IndexEntry.create(current.left, postings());
-    }
-
-    public long getMaxSSTableRowId()
-    {
-        return maxSSTableRowId;
-    }
-
-    public long getMinSSTableRowId()
-    {
-        return minSSTableRowId;
-    }
-
-    private PostingList postings()
-    {
-        final LongArrayList list = current.right;
-
-        assert list.size() > 0;
-
-        final long minSegmentRowID = list.get(0);
-        final long maxSegmentRowID = list.get(list.size() - 1);
-
-        minSSTableRowId = Math.min(minSSTableRowId, minSegmentRowID);
-        maxSSTableRowId = Math.max(maxSSTableRowId, maxSegmentRowID);
-
-        final Iterator<LongCursor> it = list.iterator();
-
-        return new PostingList()
-        {
-            @Override
-            public long nextPosting()
-            {
-                if (!it.hasNext())
-                {
-                    return END_OF_STREAM;
-                }
-
-                return it.next().value;
-            }
-
-            @Override
-            public long size()
-            {
-                return list.size();
-            }
-
-            @Override
-            public long advance(long targetRowID)
-            {
-                throw new UnsupportedOperationException();
-            }
-        };
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/RowMapping.java b/src/java/org/apache/cassandra/index/sai/memory/RowMapping.java
new file mode 100644
index 000000000000..a0bc124c4ae7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/memory/RowMapping.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.memory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * In memory representation of {@link PrimaryKey} to row ID mappings which only contains
+ * {@link Row} regardless it's live or deleted. ({@link RangeTombstoneMarker} is not included.)
+ *
+ * For JBOD, we can make use of sstable min/max partition key to filter irrelevant {@link TrieMemtableIndex} subranges.
+ * For Tiered Storage, in most cases, it flushes to tiered 0.
+ */
+public class RowMapping
+{
+    public static final RowMapping DUMMY = new RowMapping()
+    {
+        @Override
+        public Iterator<Pair<ByteComparable.Preencoded, List<RowIdWithFrequency>>> merge(MemtableIndex index) { return Collections.emptyIterator(); }
+
+        @Override
+        public void complete() {}
+
+        @Override
+        public void add(PrimaryKey key, long sstableRowId) {}
+
+        @Override
+        public int get(PrimaryKey key)
+        {
+            return -1;
+        }
+
+        @Override
+        public int size()
+        {
+            return 0;
+        }
+    };
+
+    private final InMemoryTrie<Integer> rowMapping = InMemoryTrie.shortLived(TypeUtil.BYTE_COMPARABLE_VERSION);
+
+    private volatile boolean complete = false;
+
+    public PrimaryKey minKey;
+    public PrimaryKey maxKey;
+
+    public int maxSegmentRowId = -1;
+
+    public int count;
+
+    private RowMapping()
+    {}
+
+    /**
+     * Create row mapping for FLUSH operation only.
+     */
+    public static RowMapping create(OperationType opType)
+    {
+        if (opType == OperationType.FLUSH)
+            return new RowMapping();
+        return DUMMY;
+    }
+
+    public static class RowIdWithFrequency {
+        public final int rowId;
+        public final int frequency;
+
+        public RowIdWithFrequency(int rowId, int frequency) {
+            this.rowId = rowId;
+            this.frequency = frequency;
+        }
+    }
+
+    /**
+     * Merge IndexMemtable(index term to PrimaryKeys mappings) with row mapping of a sstable
+     * (PrimaryKey to RowId mappings).
+     *
+     * @param index a Memtable-attached column index
+     *
+     * @return iterator of index term to postings mapping exists in the sstable
+     */
+    public Iterator<Pair<ByteComparable.Preencoded, List<RowIdWithFrequency>>> merge(MemtableIndex index)
+    {
+        assert complete : "RowMapping is not built.";
+
+        var it = index.iterator(minKey.partitionKey(), maxKey.partitionKey());
+        return new AbstractGuavaIterator<>()
+        {
+            @Override
+            protected Pair<ByteComparable.Preencoded, List<RowIdWithFrequency>> computeNext()
+            {
+                while (it.hasNext())
+                {
+                    var pair = it.next();
+
+                    List<RowIdWithFrequency> postings = null;
+                    var primaryKeysWithFreq = pair.right;
+
+                    for (var pkWithFreq : primaryKeysWithFreq)
+                    {
+                        ByteComparable byteComparable = pkWithFreq.pk::asComparableBytes;
+                        Integer segmentRowId = rowMapping.get(byteComparable);
+
+                        if (segmentRowId != null)
+                        {
+                            postings = postings == null ? new ArrayList<>() : postings;
+                            postings.add(new RowIdWithFrequency(segmentRowId, pkWithFreq.frequency));
+                        }
+                    }
+                    if (postings != null && !postings.isEmpty())
+                        return Pair.create(pair.left, postings);
+                }
+                return endOfData();
+            }
+        };
+    }
+
+    /**
+     * Complete building in memory RowMapping, mark it as immutable.
+     */
+    public void complete()
+    {
+        assert !complete : "RowMapping can only be built once.";
+        this.complete = true;
+    }
+
+    /**
+     * Include PrimaryKey to RowId mapping
+     */
+    public void add(PrimaryKey key, long sstableRowId) throws TrieSpaceExhaustedException
+    {
+        assert !complete : "Cannot modify built RowMapping.";
+
+        if (sstableRowId > Integer.MAX_VALUE)
+            throw new IllegalArgumentException("RowId must be less than or equal to Integer.MAX_VALUE");
+
+        // We only build this mapping for memtables, and because those only have a single segment, we know
+        // that the segment row id is the same as the sstable row id.
+        int segmentRowId = (int) sstableRowId;
+
+        ByteComparable byteComparable = v -> key.asComparableBytes(v);
+        rowMapping.putSingleton(byteComparable, segmentRowId, (existing, neww) -> neww);
+
+        maxSegmentRowId = Math.max(maxSegmentRowId, segmentRowId);
+
+        // data is written in token sorted order
+        if (minKey == null)
+            minKey = key;
+        maxKey = key;
+        count++;
+    }
+
+    public int get(PrimaryKey key)
+    {
+        Integer sstableRowId = rowMapping.get(v -> key.asComparableBytes(v));
+        return sstableRowId == null ? -1 : sstableRowId;
+    }
+
+    public int size()
+    {
+        return count;
+    }
+
+    public boolean hasRows()
+    {
+        return size() > 0;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
index c8d32a8386c0..8ec72d5f8d91 100644
--- a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
+++ b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
@@ -1,3 +1,9 @@
+/*
+ * All changes to the original code are Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -18,126 +24,263 @@
 
 package org.apache.cassandra.index.sai.memory;
 
+import java.io.IOException;
+import java.math.BigDecimal;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
-import java.util.PriorityQueue;
+import java.util.Set;
 import java.util.SortedSet;
 import java.util.concurrent.atomic.LongAdder;
-import java.util.function.Function;
+import java.util.function.LongConsumer;
+import javax.annotation.Nullable;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import io.github.jbellis.jvector.util.RamUsageEstimator;
 import io.netty.util.concurrent.FastThreadLocal;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.memtable.TrieMemtable;
 import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Direction;
 import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
 import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.disk.v6.TermsDistribution;
 import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
+import org.apache.cassandra.index.sai.plan.Orderer;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithByteComparable;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
 import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.BinaryHeap;
+import org.apache.cassandra.utils.CloseableIterator;
 import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
-/**
- * This is an in-memory index using the {@link InMemoryTrie} to store a {@link ByteComparable}
- * representation of the indexed values. Data is stored on-heap or off-heap and follows the
- * settings of the {@link TrieMemtable} to determine where.
- */
 public class TrieMemoryIndex extends MemoryIndex
 {
     private static final Logger logger = LoggerFactory.getLogger(TrieMemoryIndex.class);
+    private static final int MINIMUM_QUEUE_SIZE = 128;
     private static final int MAX_RECURSIVE_KEY_LENGTH = 128;
 
     private final InMemoryTrie<PrimaryKeys> data;
-    private final PrimaryKeysReducer primaryKeysReducer;
+    private final LongAdder primaryKeysHeapAllocations;
+    private final PrimaryKeysAccumulator primaryKeysAccumulator;
+    private final PrimaryKeysRemover primaryKeysRemover;
+    private final boolean analyzerTransformsValue;
+    private final Map<PrimaryKey, Integer> docLengths = new HashMap<>();
+
+    private final Memtable memtable;
+    private AbstractBounds<PartitionPosition> keyBounds;
 
     private ByteBuffer minTerm;
     private ByteBuffer maxTerm;
 
-    public TrieMemoryIndex(StorageAttachedIndex index)
+    private static final FastThreadLocal<Integer> lastQueueSize = new FastThreadLocal<Integer>()
+    {
+        protected Integer initialValue()
+        {
+            return MINIMUM_QUEUE_SIZE;
+        }
+    };
+
+    @VisibleForTesting
+    public TrieMemoryIndex(IndexContext indexContext)
     {
-        super(index);
-        this.data = new InMemoryTrie<>(TrieMemtable.BUFFER_TYPE);
-        this.primaryKeysReducer = new PrimaryKeysReducer();
+        this(indexContext, null, AbstractBounds.unbounded(indexContext.getPartitioner()));
     }
 
-    /**
-     * Adds an index value to the in-memory index
-     *
-     * @param key partition key for the indexed value
-     * @param clustering clustering for the indexed value
-     * @param value indexed value
-     * @return amount of heap allocated by the new value
-     */
-    @Override
-    public synchronized long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
+    public TrieMemoryIndex(IndexContext indexContext, Memtable memtable, AbstractBounds<PartitionPosition> keyBounds)
     {
-        value = index.termType().asIndexBytes(value);
-        final PrimaryKey primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering)
-                                                            : index.keyFactory().create(key);
-        final long initialSizeOnHeap = data.sizeOnHeap();
-        final long initialSizeOffHeap = data.sizeOffHeap();
-        final long reducerHeapSize = primaryKeysReducer.heapAllocations();
+        super(indexContext);
+        this.keyBounds = keyBounds;
+        this.primaryKeysHeapAllocations = new LongAdder();
+        this.primaryKeysAccumulator = new PrimaryKeysAccumulator(primaryKeysHeapAllocations);
+        this.primaryKeysRemover = new PrimaryKeysRemover(primaryKeysHeapAllocations);
+        this.analyzerTransformsValue = indexContext.getAnalyzerFactory().create().transformValue();
+        this.data = InMemoryTrie.longLived(TypeUtil.byteComparableVersionForTermsData(), TrieMemtable.BUFFER_TYPE, indexContext.columnFamilyStore().readOrdering());
+        this.memtable = memtable;
+    }
+
+    public synchronized Map<PrimaryKey, Integer> getDocLengths()
+    {
+        return docLengths;
+    }
 
-        if (index.hasAnalyzer())
+    public synchronized void add(DecoratedKey key,
+                                 Clustering clustering,
+                                 ByteBuffer value,
+                                 LongConsumer onHeapAllocationsTracker,
+                                 LongConsumer offHeapAllocationsTracker)
+    {
+        final PrimaryKey primaryKey = indexContext.keyFactory().create(key, clustering);
+        applyTransformer(primaryKey, value, onHeapAllocationsTracker, offHeapAllocationsTracker, primaryKeysAccumulator);
+    }
+
+    public synchronized void update(DecoratedKey key,
+                                    Clustering clustering,
+                                    ByteBuffer oldValue,
+                                    ByteBuffer newValue,
+                                    LongConsumer onHeapAllocationsTracker,
+                                    LongConsumer offHeapAllocationsTracker)
+    {
+        final PrimaryKey primaryKey = indexContext.keyFactory().create(key, clustering);
+        try
         {
-            AbstractAnalyzer analyzer = index.analyzer();
-            try
+            if (analyzerTransformsValue)
             {
-                analyzer.reset(value);
-                while (analyzer.hasNext())
-                {
-                    addTerm(primaryKey, analyzer.next());
-                }
+                // Because an update can add and remove the same term, we collect the set of the seen PrimaryKeys
+                // objects touched by the new values and pass it to the remover to prevent removing the PrimaryKey from
+                // the PrimaryKeys object if it was updated during the add part of this update.
+                var seenPrimaryKeys = new HashSet<PrimaryKeys>();
+                primaryKeysAccumulator.setSeenPrimaryKeys(seenPrimaryKeys);
+                primaryKeysRemover.setSeenPrimaryKeys(seenPrimaryKeys);
             }
-            finally
+
+            // Add before removing to prevent a period where the value is not available in the index
+            if (newValue != null && newValue.hasRemaining())
+                applyTransformer(primaryKey, newValue, onHeapAllocationsTracker, offHeapAllocationsTracker, primaryKeysAccumulator);
+            if (oldValue != null && oldValue.hasRemaining())
+                applyTransformer(primaryKey, oldValue, onHeapAllocationsTracker, offHeapAllocationsTracker, primaryKeysRemover);
+        }
+        finally
+        {
+            // Return the accumulator and remover to their default state.
+            primaryKeysAccumulator.setSeenPrimaryKeys(null);
+            primaryKeysRemover.setSeenPrimaryKeys(null);
+        }
+    }
+
+    public synchronized void update(DecoratedKey key,
+                                    Clustering clustering,
+                                    Iterator<ByteBuffer> oldValues,
+                                    Iterator<ByteBuffer> newValues,
+                                    LongConsumer onHeapAllocationsTracker,
+                                    LongConsumer offHeapAllocationsTracker)
+    {
+        final PrimaryKey primaryKey = indexContext.keyFactory().create(key, clustering);
+        try
+        {
+            // Because an update can add and remove the same term, we collect the set of the seen PrimaryKeys
+            // objects touched by the new values and pass it to the remover to prevent removing the PrimaryKey from
+            // the PrimaryKeys object if it was updated during the add part of this update.
+            var seenPrimaryKeys = new HashSet<PrimaryKeys>();
+            primaryKeysAccumulator.setSeenPrimaryKeys(seenPrimaryKeys);
+            primaryKeysRemover.setSeenPrimaryKeys(seenPrimaryKeys);
+
+            // Add before removing to prevent a period where the values are not available in the index
+            while (newValues != null && newValues.hasNext())
+            {
+                ByteBuffer newValue = newValues.next();
+                if (newValue != null && newValue.hasRemaining())
+                    applyTransformer(primaryKey, newValue, onHeapAllocationsTracker, offHeapAllocationsTracker, primaryKeysAccumulator);
+            }
+
+            while (oldValues != null && oldValues.hasNext())
             {
-                analyzer.end();
+                ByteBuffer oldValue = oldValues.next();
+                if (oldValue != null && oldValue.hasRemaining())
+                    applyTransformer(primaryKey, oldValue, onHeapAllocationsTracker, offHeapAllocationsTracker, primaryKeysRemover);
             }
         }
-        else
+        finally
         {
-            addTerm(primaryKey, value);
+            // Return the accumulator and remover to their default state.
+            primaryKeysAccumulator.setSeenPrimaryKeys(null);
+            primaryKeysRemover.setSeenPrimaryKeys(null);
         }
-        long onHeap = data.sizeOnHeap();
-        long offHeap = data.sizeOffHeap();
-        long heapAllocations = primaryKeysReducer.heapAllocations();
-        return (onHeap - initialSizeOnHeap) + (offHeap - initialSizeOffHeap) + (heapAllocations - reducerHeapSize);
     }
 
-    @Override
-    public long update(DecoratedKey key, Clustering<?> clustering, ByteBuffer oldValue, ByteBuffer newValue)
+    private void applyTransformer(PrimaryKey primaryKey,
+                                  ByteBuffer value,
+                                  LongConsumer onHeapAllocationsTracker,
+                                  LongConsumer offHeapAllocationsTracker,
+                                  InMemoryTrie.UpsertTransformer<PrimaryKeys, PrimaryKey> transformer)
     {
-        throw new UnsupportedOperationException();
+        AbstractAnalyzer analyzer = indexContext.getAnalyzerFactory().create();
+        try
+        {
+            value = TypeUtil.asIndexBytes(value, indexContext.getValidator());
+            analyzer.reset(value);
+            final long initialSizeOnHeap = data.usedSizeOnHeap();
+            final long initialSizeOffHeap = data.usedSizeOffHeap();
+            final long initialPrimaryKeysHeapAllocations = primaryKeysHeapAllocations.longValue();
+
+            int tokenCount = 0;
+            while (analyzer.hasNext())
+            {
+                final ByteBuffer term = analyzer.next();
+                if (!indexContext.validateMaxTermSize(primaryKey.partitionKey(), term))
+                    continue;
+
+                tokenCount++;
+
+                // Note that this term is already encoded once by the TypeUtil.encode call above.
+                setMinMaxTerm(term.duplicate());
+
+                final ByteComparable encodedTerm = asByteComparable(term.duplicate());
+
+                try
+                {
+                    data.putSingleton(encodedTerm, primaryKey, transformer, term.limit() <= MAX_RECURSIVE_KEY_LENGTH);
+                }
+                catch (TrieSpaceExhaustedException e)
+                {
+                    Throwables.throwAsUncheckedException(e);
+                }
+            }
+
+            Object prev = docLengths.put(primaryKey, tokenCount);
+            if (prev != null)
+            {
+                // heap used for doc lengths
+                long heapUsed = RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY
+                                + primaryKey.ramBytesUsed() // TODO do we count these bytes?
+                                + Integer.BYTES;
+                onHeapAllocationsTracker.accept(heapUsed);
+            }
+
+            // memory used by the trie
+            onHeapAllocationsTracker.accept((data.usedSizeOnHeap() - initialSizeOnHeap) +
+                                            (primaryKeysHeapAllocations.longValue() - initialPrimaryKeysHeapAllocations));
+            offHeapAllocationsTracker.accept(data.usedSizeOffHeap() - initialSizeOffHeap);
+        }
+        finally
+        {
+            analyzer.end();
+        }
     }
 
-    /**
-     * Search for an expression in the in-memory index within the {@link AbstractBounds} defined
-     * by keyRange. This can either be an exact match or a range match.
-     * <p>
-     * @param expression the {@link Expression} to search for
-     * @param keyRange the {@link AbstractBounds} containing the key range to restrict the search to
-     * @return a {@link KeyRangeIterator} containing the search results
-     */
-    public KeyRangeIterator search(QueryContext queryContext, Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    @Override
+    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange)
     {
         if (logger.isTraceEnabled())
             logger.trace("Searching memtable index on expression '{}'...", expression);
 
-        switch (expression.getIndexOperator())
+        switch (expression.getOp())
         {
+            case MATCH:
             case EQ:
             case CONTAINS_KEY:
             case CONTAINS_VALUE:
@@ -149,16 +292,10 @@ public KeyRangeIterator search(QueryContext queryContext, Expression expression,
         }
     }
 
-    /**
-     * Returns an {@link Iterator} over the entire dataset contained in the trie. This is used
-     * when the index is flushed to disk.
-     *
-     * @return the iterator containing the trie data
-     */
     @Override
-    public Iterator<Pair<ByteComparable, PrimaryKeys>> iterator()
+    public Iterator<Pair<ByteComparable.Preencoded, List<PkWithFrequency>>> iterator()
     {
-        Iterator<Map.Entry<ByteComparable, PrimaryKeys>> iterator = data.entrySet().iterator();
+        Iterator<Map.Entry<ByteComparable.Preencoded, PrimaryKeys>> iterator = data.entrySet().iterator();
         return new Iterator<>()
         {
             @Override
@@ -168,162 +305,513 @@ public boolean hasNext()
             }
 
             @Override
-            public Pair<ByteComparable, PrimaryKeys> next()
+            public Pair<ByteComparable.Preencoded, List<PkWithFrequency>> next()
             {
-                Map.Entry<ByteComparable, PrimaryKeys> entry = iterator.next();
-                return Pair.create(entry.getKey(), entry.getValue());
+                Map.Entry<ByteComparable.Preencoded, PrimaryKeys> entry = iterator.next();
+                var pairs = new ArrayList<PkWithFrequency>(entry.getValue().size());
+                Iterators.addAll(pairs, entry.getValue().iterator());
+                return Pair.create(entry.getKey(), pairs);
             }
         };
     }
 
-    @Override
-    public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor,
-                                                            IndexIdentifier indexIdentifier,
-                                                            Function<PrimaryKey, Integer> postingTransformer)
+    @VisibleForTesting
+    long estimatedTrieValuesMemoryUsed()
     {
-        throw new UnsupportedOperationException();
+        return primaryKeysHeapAllocations.longValue();
     }
 
     @Override
-    public boolean isEmpty()
+    public CloseableIterator<PrimaryKeyWithSortKey> orderBy(Orderer orderer, @Nullable Expression slice)
     {
-        return minTerm == null;
+        if (data.isEmpty())
+            return CloseableIterator.emptyIterator();
+
+        Trie<PrimaryKeys> subtrie = getSubtrie(slice);
+        var iter = subtrie.entrySet(orderer.isAscending() ? Direction.FORWARD : Direction.REVERSE).iterator();
+        return new AllTermsIterator(iter);
     }
 
-    @Override
-    public ByteBuffer getMinTerm()
+    private ByteComparable asByteComparable(ByteBuffer input)
     {
-        return minTerm;
+        return Version.latest().onDiskFormat().encodeForTrie(input, indexContext.getValidator());
     }
 
-    @Override
-    public ByteBuffer getMaxTerm()
+    public KeyRangeIterator exactMatch(Expression expression, AbstractBounds<PartitionPosition> keyRange)
     {
-        return maxTerm;
+        final ByteComparable prefix = expression.lower == null ? ByteComparable.EMPTY : asByteComparable(expression.lower.value.encoded);
+        final PrimaryKeys primaryKeys = data.get(prefix);
+        if (primaryKeys == null)
+        {
+            return KeyRangeIterator.empty();
+        }
+        return new FilteringKeyRangeIterator(new SortedSetKeyRangeIterator(primaryKeys.keys()), keyRange);
     }
 
-    private void addTerm(PrimaryKey primaryKey, ByteBuffer term)
+    /**
+     * Accumulator that adds a primary key to the primary keys set.
+     */
+    static class PrimaryKeysAccumulator implements InMemoryTrie.UpsertTransformer<PrimaryKeys, PrimaryKey>
     {
-        if (index.validateTermSize(primaryKey.partitionKey(), term, false, null))
+        private final LongAdder heapAllocations;
+        private HashSet<PrimaryKeys> seenPrimaryKeys;
+
+        PrimaryKeysAccumulator(LongAdder heapAllocations)
         {
-            setMinMaxTerm(term.duplicate());
+            this.heapAllocations = heapAllocations;
+        }
 
-            final ByteComparable comparableBytes = asComparableBytes(term);
+        /**
+         * Set the PrimaryKeys set to check for each PrimaryKeys object updated by this transformer.
+         * Warning: This method is not thread-safe and should only be called from within the synchronized block
+         * of the TrieMemoryIndex class.
+         * @param seenPrimaryKeys the set of PrimaryKeys objects updated so far
+         */
+        private void setSeenPrimaryKeys(HashSet<PrimaryKeys> seenPrimaryKeys)
+        {
+            this.seenPrimaryKeys = seenPrimaryKeys;
+        }
 
-            try
-            {
-                if (term.limit() <= MAX_RECURSIVE_KEY_LENGTH)
-                {
-                    data.putRecursive(comparableBytes, primaryKey, primaryKeysReducer);
-                }
-                else
-                {
-                    data.apply(Trie.singleton(comparableBytes, primaryKey), primaryKeysReducer);
-                }
-            }
-            catch (InMemoryTrie.SpaceExhaustedException e)
+        @Override
+        public PrimaryKeys apply(PrimaryKeys existing, PrimaryKey neww)
+        {
+            if (existing == null)
             {
-                throw new RuntimeException(e);
+                existing = new PrimaryKeys();
+                heapAllocations.add(PrimaryKeys.unsharedHeapSize());
             }
+
+            // If we are tracking PrimaryKeys via the seenPrimaryKeys set, then we need to reset the
+            // counter on the first time seeing each PrimaryKeys object since an update means that the
+            // frequency should be reset.
+            boolean shouldResetFrequency = false;
+            if (seenPrimaryKeys != null)
+                shouldResetFrequency = seenPrimaryKeys.add(existing);
+
+            long bytesAdded = shouldResetFrequency ? existing.addAndResetFrequency(neww)
+                                                   : existing.addAndIncrementFrequency(neww);
+            heapAllocations.add(bytesAdded);
+            return existing;
         }
     }
 
-    private void setMinMaxTerm(ByteBuffer term)
+    /**
+     * Transformer that removes a primary key from the primary keys set, if present.
+     */
+    static class PrimaryKeysRemover implements InMemoryTrie.UpsertTransformer<PrimaryKeys, PrimaryKey>
     {
-        assert term != null;
+        private final LongAdder heapAllocations;
+        private Set<PrimaryKeys> seenPrimaryKeys;
 
-        minTerm = index.termType().min(term, minTerm);
-        maxTerm = index.termType().max(term, maxTerm);
-    }
+        PrimaryKeysRemover(LongAdder heapAllocations)
+        {
+            this.heapAllocations = heapAllocations;
+        }
 
-    private ByteComparable asComparableBytes(ByteBuffer input)
-    {
-        return version -> index.termType().asComparableBytes(input, version);
+        /**
+         * Set the set of seenPrimaryKeys.
+         * Warning: This method is not thread-safe and should only be called from within the synchronized block
+         * of the TrieMemoryIndex class.
+         * @param seenPrimaryKeys
+         */
+        private void setSeenPrimaryKeys(Set<PrimaryKeys> seenPrimaryKeys)
+        {
+            this.seenPrimaryKeys = seenPrimaryKeys;
+        }
+
+        @Override
+        public PrimaryKeys apply(PrimaryKeys existing, PrimaryKey neww)
+        {
+            if (existing == null)
+                return null;
+
+            // This PrimaryKeys object was already seen during the add part of this update,
+            // so we skip removing the PrimaryKey from the PrimaryKeys class.
+            if (seenPrimaryKeys != null && seenPrimaryKeys.contains(existing))
+                return existing;
+
+            heapAllocations.add(existing.remove(neww));
+            if (!existing.isEmpty())
+                return existing;
+
+            heapAllocations.add(-PrimaryKeys.unsharedHeapSize());
+            return null;
+        }
     }
 
-    private KeyRangeIterator exactMatch(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    /**
+     * A sorting iterator over items that can either be singleton PrimaryKey or a SortedSetKeyRangeIterator.
+     */
+    static class SortingSingletonOrSetIterator extends BinaryHeap
     {
-        ByteComparable comparableMatch = expression.lower() == null ? ByteComparable.EMPTY
-                                                                    : asComparableBytes(expression.lower().value.encoded);
-        PrimaryKeys primaryKeys = data.get(comparableMatch);
-        return primaryKeys == null ? KeyRangeIterator.empty()
-                                   : new FilteringInMemoryKeyRangeIterator(primaryKeys.keys(), keyRange);
+        public SortingSingletonOrSetIterator(Collection<Object> data)
+        {
+            super(data.toArray());
+            heapify();
+        }
+
+        @Override
+        protected boolean greaterThan(Object a, Object b)
+        {
+            if (a == null || b == null)
+                return b != null;
+
+            return peek(a).compareTo(peek(b)) > 0;
+        }
+
+        public PrimaryKey nextOrNull()
+        {
+            Object key = top();
+            if (key == null)
+                return null;
+            PrimaryKey result = peek(key);
+            assert result != null;
+            replaceTop(advanceItem(key));
+            return result;
+        }
+
+        public void skipTo(PrimaryKey target)
+        {
+            advanceTo(target);
+        }
+
+        /**
+         * Advance the given keys object to the next key.
+         * If the keys object contains a single key, null is returned.
+         * If the keys object contains more than one key, the first key is dropped and the iterator to the
+         * remaining keys is returned.
+         */
+        @Override
+        protected @Nullable Object advanceItem(Object keys)
+        {
+            if (keys instanceof PrimaryKey)
+                return null;
+
+            SortedSetKeyRangeIterator iterator = (SortedSetKeyRangeIterator) keys;
+            assert iterator.hasNext();
+            iterator.next();
+            return iterator.hasNext() ? iterator : null;
+        }
+
+        /**
+         * Advance the given keys object to the first element that is greater than or equal to the target key.
+         * This is only called when the given item is known to be before the target key.
+         * If the keys object contains a single key, null is returned.
+         * If the keys object contains more than one key, it is skipped to the given target and the iterator to the
+         * remaining keys is returned.
+         */
+        @Override
+        protected @Nullable Object advanceItemTo(Object keys, Object target)
+        {
+            if (keys instanceof PrimaryKey)
+                return null;
+
+            SortedSetKeyRangeIterator iterator = (SortedSetKeyRangeIterator) keys;
+            iterator.skipTo((PrimaryKey) target);
+            return iterator.hasNext() ? iterator : null;
+        }
+
+        /**
+         * Resolve a keys object to either its singleton value or the current element in the iterator.
+         */
+        static PrimaryKey peek(Object keys)
+        {
+            if (keys instanceof PrimaryKey)
+                return (PrimaryKey) keys;
+            if (keys instanceof SortedSetKeyRangeIterator)
+                return ((SortedSetKeyRangeIterator) keys).peek();
+
+            throw new AssertionError("Unreachable");
+        }
     }
 
-    private static class Collector
+    static class MergingKeyRangeIterator extends KeyRangeIterator
     {
-        private static final int MINIMUM_QUEUE_SIZE = 128;
+        // A sorting iterator of items that can be either singletons or SortedSetKeyRangeIterator
+        SortingSingletonOrSetIterator keySets;  // class invariant: each object placed in this queue contains at least one key
 
-        // Maintain the last queue size used on this index to use for the next range match.
-        // This allows for receiving a stream of wide range queries where the queue size
-        // is larger than we would want to default the size to.
-        // TODO Investigate using a decaying histogram here to avoid the effect of outliers.
-        private static final FastThreadLocal<Integer> lastQueueSize = new FastThreadLocal<>()
+        MergingKeyRangeIterator(Collection<Object> keySets,
+                                PrimaryKey minKey,
+                                PrimaryKey maxKey,
+                                long count)
         {
-            protected Integer initialValue()
-            {
-                return MINIMUM_QUEUE_SIZE;
-            }
-        };
+            super(minKey, maxKey, count);
+
+            this.keySets = new SortingSingletonOrSetIterator(keySets);
+        }
+
+        static Builder builder(AbstractBounds<PartitionPosition> keyRange, PrimaryKey.Factory factory, int capacity)
+        {
+            return new Builder(keyRange, factory, capacity);
+        }
 
-        PrimaryKey minimumKey = null;
-        PrimaryKey maximumKey = null;
-        final PriorityQueue<PrimaryKey> mergedKeys = new PriorityQueue<>(lastQueueSize.get());
+        @Override
+        protected void performSkipTo(PrimaryKey nextKey)
+        {
+            keySets.skipTo(nextKey);
+        }
 
-        final AbstractBounds<PartitionPosition> keyRange;
+        @Override
+        protected PrimaryKey computeNext()
+        {
+            PrimaryKey result = keySets.nextOrNull();
+            if (result == null)
+                return endOfData();
+            else
+                return result;
+        }
 
-        public Collector(AbstractBounds<PartitionPosition> keyRange)
+        @Override
+        public void close() throws IOException
         {
-            this.keyRange = keyRange;
         }
 
-        public void processContent(PrimaryKeys keys)
+        static class Builder
         {
-            if (keys.isEmpty())
-                return;
+            final List<Object> keySets;
 
-            SortedSet<PrimaryKey> primaryKeys = keys.keys();
+            private final PrimaryKey min;
+            private final PrimaryKey max;
+            private long count;
 
-            // shortcut to avoid generating iterator
-            if (primaryKeys.size() == 1)
+
+            Builder(AbstractBounds<PartitionPosition> keyRange, PrimaryKey.Factory factory, int capacity)
             {
-                processKey(primaryKeys.first());
-                return;
+                this.min = factory.createTokenOnly(keyRange.left.getToken());
+                this.max = factory.createTokenOnly(keyRange.right.getToken());
+                this.keySets = new ArrayList<>(capacity);
             }
 
-            // skip entire partition keys if they don't overlap
-            if (!keyRange.right.isMinimum() && primaryKeys.first().partitionKey().compareTo(keyRange.right) > 0
-                || primaryKeys.last().partitionKey().compareTo(keyRange.left) < 0)
-                return;
+            public void add(PrimaryKeys primaryKeys)
+            {
+                if (primaryKeys.isEmpty())
+                    return;
+
+                int size = primaryKeys.size();
+                SortedSet<PrimaryKey> keys = primaryKeys.keys();
+                if (size == 1)
+                    keySets.add(keys.first());
+                else
+                    keySets.add(new SortedSetKeyRangeIterator(keys, min, max, size));
+
+                count += size;
+            }
+
+            public int size()
+            {
+                return keySets.size();
+            }
+
+            public boolean isEmpty()
+            {
+                return keySets.isEmpty();
+            }
+
+            public MergingKeyRangeIterator build()
+            {
+                return new MergingKeyRangeIterator(keySets, min, max, count);
+            }
+        }
+    }
 
-            primaryKeys.forEach(this::processKey);
+    static class SortedSetKeyRangeIterator extends KeyRangeIterator
+    {
+        private SortedSet<PrimaryKey> primaryKeySet;
+        private Iterator<PrimaryKey> iterator;
+        private PrimaryKey lastComputedKey;
+
+        public SortedSetKeyRangeIterator(SortedSet<PrimaryKey> source)
+        {
+            super(source.first(), source.last(), source.size());
+            this.primaryKeySet = source;
         }
 
-        public void updateLastQueueSize()
+        private SortedSetKeyRangeIterator(SortedSet<PrimaryKey> source, PrimaryKey min, PrimaryKey max, long count)
         {
-            lastQueueSize.set(Math.max(MINIMUM_QUEUE_SIZE, mergedKeys.size()));
+            super(min, max, count);
+            this.primaryKeySet = source;
         }
 
-        private void processKey(PrimaryKey key)
+
+        @Override
+        protected PrimaryKey computeNext()
         {
-            if (keyRange.contains(key.partitionKey()))
-            {
-                mergedKeys.add(key);
+            // Skip can be called multiple times in a row, so defer iterator creation until needed
+            if (iterator == null)
+                iterator = primaryKeySet.iterator();
+            lastComputedKey = iterator.hasNext() ? iterator.next() : endOfData();
+            return lastComputedKey;
+        }
 
-                minimumKey = minimumKey == null ? key : key.compareTo(minimumKey) < 0 ? key : minimumKey;
-                maximumKey = maximumKey == null ? key : key.compareTo(maximumKey) > 0 ? key : maximumKey;
-            }
+        @Override
+        protected void performSkipTo(PrimaryKey nextKey)
+        {
+            // Avoid going backwards
+            if (lastComputedKey != null && nextKey.compareTo(lastComputedKey) <= 0)
+                return;
+
+            primaryKeySet = primaryKeySet.tailSet(nextKey);
+            iterator = null;
+        }
+
+        @Override
+        public void close() throws IOException
+        {
         }
     }
 
     private KeyRangeIterator rangeMatch(Expression expression, AbstractBounds<PartitionPosition> keyRange)
     {
+        Trie<PrimaryKeys> subtrie = getSubtrie(expression);
+
+        var capacity = Math.max(MINIMUM_QUEUE_SIZE, lastQueueSize.get());
+        var mergingIteratorBuilder = MergingKeyRangeIterator.builder(keyBounds, indexContext.keyFactory(), capacity);
+        lastQueueSize.set(mergingIteratorBuilder.size());
+
+        if (!Version.latest().onOrAfter(Version.DB) && TypeUtil.isComposite(expression.validator))
+            subtrie.entrySet().forEach(entry -> {
+                // Before version DB, we encoded composite types using a non order-preserving function. In order to
+                // perform a range query on a map, we use the bounds to get all entries for a given map key and then
+                // only keep the map entries that satisfy the expression.
+                assert entry.getKey().encodingVersion() == TypeUtil.BYTE_COMPARABLE_VERSION || Version.latest() == Version.AA;
+                byte[] key = ByteSourceInverse.readBytes(entry.getKey().getPreencodedBytes());
+                if (expression.isSatisfiedBy(ByteBuffer.wrap(key)))
+                    mergingIteratorBuilder.add(entry.getValue());
+            });
+        else
+            subtrie.values().forEach(mergingIteratorBuilder::add);
+
+        return mergingIteratorBuilder.isEmpty()
+               ? KeyRangeIterator.empty()
+               : new FilteringKeyRangeIterator(mergingIteratorBuilder.build(), keyRange);
+    }
+
+    @Override
+    public long estimateMatchingRowsCount(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        switch (expression.getOp())
+        {
+            case MATCH:
+            case EQ:
+            case CONTAINS_KEY:
+            case CONTAINS_VALUE:
+                return estimateNumRowsMatchingExact(expression);
+            case NOT_EQ:
+            case NOT_CONTAINS_KEY:
+            case NOT_CONTAINS_VALUE:
+                if (TypeUtil.supportsRounding(expression.validator))
+                    return Memtable.estimateRowCount(memtable);
+                else
+                    // need to clamp at 0, because row count is imprecise
+                    return Math.max(0, Memtable.estimateRowCount(memtable) - estimateNumRowsMatchingExact(expression));
+            case RANGE:
+                return estimateNumRowsMatchingRange(expression);
+            default:
+                throw new IllegalArgumentException("Unsupported expression: " + expression);
+        }
+    }
+
+
+    private int estimateNumRowsMatchingExact(Expression expression)
+    {
+        final ByteComparable prefix = expression.lower == null ? ByteComparable.EMPTY : asByteComparable(expression.lower.value.encoded);
+        final PrimaryKeys primaryKeys = data.get(prefix);
+        return primaryKeys == null ? 0 : primaryKeys.size();
+    }
+
+    private long estimateNumRowsMatchingRange(Expression expression)
+    {
+        final Trie<PrimaryKeys> subtrie = getSubtrie(expression);
+
+        // We could compute the number of matching rows by iterating the subtrie
+        // and summing the sizes of PrimaryKeys collections. But this could be very costly
+        // if the subtrie is large. Instead, we iterate a limited number of entries, and then we
+        // check how far we got by inspecting the term and comparing it to the start term and the end term.
+        // For now, we assume that term values are distributed uniformly.
+
+        var iterator = subtrie.entryIterator();
+        if (!iterator.hasNext())
+            return 0;
+
+        AbstractType<?> termType = indexContext.getValidator();
+        ByteBuffer endTerm = expression.upper != null && TypeUtil.compare(expression.upper.value.encoded, maxTerm, termType, Version.latest()) < 0
+                             ? expression.upper.value.encoded
+                             : maxTerm;
+
+        long pointCount = 0;
+        long keyCount = 0;
+
+        ByteComparable startTerm = null;
+        ByteComparable currentTerm = null;
+
+        while (iterator.hasNext() && pointCount < 64)
+        {
+            var entry = iterator.next();
+            pointCount += 1;
+            keyCount += entry.getValue().size();
+            currentTerm = entry.getKey();
+            if (startTerm == null)
+                startTerm = currentTerm;
+        }
+        assert currentTerm != null;
+
+        // We iterated all points matched by the query, so keyCount contains the exact value of keys.
+        // This is a happy path, because the returned value will be accurate.
+        if (!iterator.hasNext())
+            return keyCount;
+
+        // There are some points remaining; let's estimate their count by extrapolation.
+        // Express the distance we iterated as a double value and the whole subtrie range also as a double.
+        // Then the ratio of those two values would give us a hint on how many total points there
+        // are in the subtrie. This should be fairly accurate assuming values are distributed uniformly.
+        BigDecimal startValue = toBigDecimal(startTerm);
+        BigDecimal endValue = toBigDecimal(endTerm);
+        BigDecimal currentValue = toBigDecimal(currentTerm);
+        double totalDistance = endValue.subtract(startValue).doubleValue() + Double.MIN_NORMAL;
+        double iteratedDistance = currentValue.subtract(startValue).doubleValue() + Double.MIN_NORMAL;
+        assert totalDistance > 0.0;
+        assert iteratedDistance > 0.0;
+
+        double extrapolatedPointCount = Math.min((pointCount - 1) * (totalDistance / iteratedDistance), this.data.valuesCount());
+        double keysPerPoint = (double) keyCount / pointCount;
+        return (long) (extrapolatedPointCount * keysPerPoint);
+    }
+
+    /**
+     * Converts the term to a BigDecimal in a way that it keeps the sort order
+     * (so terms comparing larger yield larger numbers).
+     * Works on raw representation (as passed to the index).
+     *
+     * @see #toBigDecimal(ByteComparable)
+     */
+    private BigDecimal toBigDecimal(ByteBuffer endTerm)
+    {
+        ByteComparable bc = Version.latest().onDiskFormat().encodeForTrie(endTerm, indexContext.getValidator());
+        return toBigDecimal(bc);
+    }
+
+    /**
+     * Converts the term to a BigDecimal in a way that it keeps the sort order
+     * (so terms comparing larger yield larger numbers).
+     * @see TermsDistribution#toBigDecimal(ByteComparable, AbstractType, Version, ByteComparable.Version)
+     */
+    private BigDecimal toBigDecimal(ByteComparable term)
+    {
+        AbstractType<?> type = indexContext.getValidator();
+        return TermsDistribution.toBigDecimal(term, type, Version.latest(), TypeUtil.BYTE_COMPARABLE_VERSION);
+    }
+
+    private Trie<PrimaryKeys> getSubtrie(@Nullable Expression expression)
+    {
+        if (expression == null)
+            return data;
+
         ByteComparable lowerBound, upperBound;
         boolean lowerInclusive, upperInclusive;
-        if (expression.lower() != null)
+        if (expression.lower != null)
         {
-            lowerBound = asComparableBytes(expression.lower().value.encoded);
-            lowerInclusive = expression.lower().inclusive;
+            lowerBound = expression.getEncodedLowerBoundByteComparable(Version.latest());
+            lowerInclusive = expression.lower.inclusive;
         }
         else
         {
@@ -331,10 +819,10 @@ private KeyRangeIterator rangeMatch(Expression expression, AbstractBounds<Partit
             lowerInclusive = false;
         }
 
-        if (expression.upper() != null)
+        if (expression.upper != null)
         {
-            upperBound = asComparableBytes(expression.upper().value.encoded);
-            upperInclusive = expression.upper().inclusive;
+            upperBound = expression.getEncodedUpperBoundByteComparable(Version.latest());
+            upperInclusive = expression.upper.inclusive;
         }
         else
         {
@@ -342,23 +830,10 @@ private KeyRangeIterator rangeMatch(Expression expression, AbstractBounds<Partit
             upperInclusive = false;
         }
 
-        Collector cd = new Collector(keyRange);
-
-        data.subtrie(lowerBound, lowerInclusive, upperBound, upperInclusive)
-            .values()
-            .forEach(cd::processContent);
-
-        if (cd.mergedKeys.isEmpty())
-        {
-            return KeyRangeIterator.empty();
-        }
-
-        cd.updateLastQueueSize();
-
-        return new InMemoryKeyRangeIterator(cd.minimumKey, cd.maximumKey, cd.mergedKeys);
+        return data.subtrie(lowerBound, lowerInclusive, upperBound, upperInclusive);
     }
 
-    private static class PrimaryKeysReducer implements InMemoryTrie.UpsertTransformer<PrimaryKeys, PrimaryKey>
+    private class PrimaryKeysReducer implements InMemoryTrie.UpsertTransformer<PrimaryKeys, PrimaryKey>
     {
         private final LongAdder heapAllocations = new LongAdder();
 
@@ -370,7 +845,7 @@ public PrimaryKeys apply(PrimaryKeys existing, PrimaryKey neww)
                 existing = new PrimaryKeys();
                 heapAllocations.add(existing.unsharedHeapSize());
             }
-            heapAllocations.add(existing.add(neww));
+            heapAllocations.add(existing.addAndIncrementFrequency(neww));
             return existing;
         }
 
@@ -379,4 +854,64 @@ long heapAllocations()
             return heapAllocations.longValue();
         }
     }
+
+    @Override
+    public ByteBuffer getMinTerm()
+    {
+        return minTerm;
+    }
+
+    @Override
+    public ByteBuffer getMaxTerm()
+    {
+        return maxTerm;
+    }
+
+    private void setMinMaxTerm(ByteBuffer term)
+    {
+        assert term != null;
+
+        // Note that an update to a term could make these inaccurate, but they err in the correct direction.
+        // An alternative solution could use the trie to find the min/max term, but the trie has ByteComparable
+        // objects, not the ByteBuffer, and we would need to implement a custom decoder to undo the encodeForTrie
+        // mapping.
+        minTerm = TypeUtil.min(term, minTerm, indexContext.getValidator(), Version.latest());
+        maxTerm = TypeUtil.max(term, maxTerm, indexContext.getValidator(), Version.latest());
+    }
+
+    /**
+     * Iterator that provides ordered access to all indexed terms and their associated primary keys
+     * in the TrieMemoryIndex. For each term in the index, yields PrimaryKeyWithSortKey objects that
+     * combine a primary key with its associated term.
+     * <p>
+     * A more verbose name could be KeysMatchingTermsByTermIterator.
+     */
+    private class AllTermsIterator extends AbstractIterator<PrimaryKeyWithSortKey>
+    {
+        private final Iterator<Map.Entry<ByteComparable.Preencoded, PrimaryKeys>> iterator;
+        private Iterator<PrimaryKey> primaryKeysIterator = CloseableIterator.emptyIterator();
+        private ByteComparable.Preencoded byteComparableTerm = null;
+
+        public AllTermsIterator(Iterator<Map.Entry<ByteComparable.Preencoded, PrimaryKeys>> iterator)
+        {
+            this.iterator = iterator;
+        }
+
+        @Override
+        protected PrimaryKeyWithSortKey computeNext()
+        {
+            assert memtable != null;
+            if (primaryKeysIterator.hasNext())
+                return new PrimaryKeyWithByteComparable(indexContext, memtable, primaryKeysIterator.next(), byteComparableTerm);
+
+            if (iterator.hasNext())
+            {
+                var entry = iterator.next();
+                primaryKeysIterator = entry.getValue().keys().iterator();
+                byteComparableTerm = entry.getKey();
+                return new PrimaryKeyWithByteComparable(indexContext, memtable, primaryKeysIterator.next(), byteComparableTerm);
+            }
+            return endOfData();
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/memory/TrieMemtableIndex.java b/src/java/org/apache/cassandra/index/sai/memory/TrieMemtableIndex.java
new file mode 100644
index 000000000000..4404c9be6ca4
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/memory/TrieMemtableIndex.java
@@ -0,0 +1,563 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.LongAdder;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Streams;
+import com.google.common.util.concurrent.Runnables;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.AbstractShardedMemtable;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.memtable.ShardBoundaries;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.iterators.KeyRangeConcatIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIntersectionIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeLazyIterator;
+import org.apache.cassandra.index.sai.memory.MemoryIndex.PkWithFrequency;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.BM25Utils;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithByteComparable;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.Type;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.MergeIterator;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.Reducer;
+import org.apache.cassandra.utils.SortingIterator;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static org.apache.cassandra.io.sstable.SSTableReadsListener.NOOP_LISTENER;
+
+public class TrieMemtableIndex implements MemtableIndex
+{
+    private final ShardBoundaries boundaries;
+    private final MemoryIndex[] rangeIndexes;
+    private final IndexContext indexContext;
+    private final AbstractType<?> validator;
+    private final LongAdder writeCount = new LongAdder();
+    private final LongAdder estimatedOnHeapMemoryUsed = new LongAdder();
+    private final LongAdder estimatedOffHeapMemoryUsed = new LongAdder();
+
+    private final Memtable memtable;
+    private final Context sensorContext;
+    private final RequestTracker requestTracker;
+
+    public TrieMemtableIndex(IndexContext indexContext, Memtable memtable)
+    {
+        this(indexContext, memtable, AbstractShardedMemtable.getDefaultShardCount());
+    }
+
+    @VisibleForTesting
+    public TrieMemtableIndex(IndexContext indexContext, Memtable memtable, int shardCount)
+    {
+        this.boundaries = indexContext.columnFamilyStore().localRangeSplits(shardCount);
+        this.rangeIndexes = new MemoryIndex[boundaries.shardCount()];
+        this.indexContext = indexContext;
+        this.validator = indexContext.getValidator();
+        this.memtable = memtable;
+        for (int shard = 0; shard < boundaries.shardCount(); shard++)
+        {
+            this.rangeIndexes[shard] = new TrieMemoryIndex(indexContext, memtable, boundaries.getBounds(shard));
+        }
+        this.sensorContext = Context.from(indexContext);
+        this.requestTracker = RequestTracker.instance;
+    }
+
+    @Override
+    public Memtable getMemtable()
+    {
+        return memtable;
+    }
+
+    @VisibleForTesting
+    public int shardCount()
+    {
+        return rangeIndexes.length;
+    }
+
+    @Override
+    public long writeCount()
+    {
+        return writeCount.sum();
+    }
+
+    @Override
+    public long estimatedOnHeapMemoryUsed()
+    {
+        return estimatedOnHeapMemoryUsed.sum();
+    }
+
+    @Override
+    public long estimatedOffHeapMemoryUsed()
+    {
+        return estimatedOffHeapMemoryUsed.sum();
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return getMinTerm() == null;
+    }
+
+    // Returns the minimum indexed term in the combined memory indexes.
+    // This can be null if the indexed memtable was empty. Users of the
+    // {@code MemtableIndex} requiring a non-null minimum term should
+    // use the {@link MemtableIndex#isEmpty} method.
+    // Note: Individual index shards can return null here if the index
+    // didn't receive any terms within the token range of the shard
+    @Override
+    @Nullable
+    public ByteBuffer getMinTerm()
+    {
+        return Arrays.stream(rangeIndexes)
+                     .map(MemoryIndex::getMinTerm)
+                     .filter(Objects::nonNull)
+                     .reduce((a, b) -> TypeUtil.min(a, b, validator, Version.latest()))
+                     .orElse(null);
+    }
+
+    // Returns the maximum indexed term in the combined memory indexes.
+    // This can be null if the indexed memtable was empty. Users of the
+    // {@code MemtableIndex} requiring a non-null maximum term should
+    // use the {@link MemtableIndex#isEmpty} method.
+    // Note: Individual index shards can return null here if the index
+    // didn't receive any terms within the token range of the shard
+    @Override
+    @Nullable
+    public ByteBuffer getMaxTerm()
+    {
+        return Arrays.stream(rangeIndexes)
+                     .map(MemoryIndex::getMaxTerm)
+                     .filter(Objects::nonNull)
+                     .reduce((a, b) -> TypeUtil.max(a, b, validator, Version.latest()))
+                     .orElse(null);
+    }
+
+    @Override
+    public void index(DecoratedKey key, Clustering clustering, ByteBuffer value, Memtable memtable, OpOrder.Group opGroup)
+    {
+        if (value == null || (value.remaining() == 0 && TypeUtil.skipsEmptyValue(validator)))
+            return;
+
+        RequestSensors sensors = requestTracker.get();
+        if (sensors != null)
+            sensors.registerSensor(sensorContext, Type.INDEX_WRITE_BYTES);
+        rangeIndexes[boundaries.getShardForKey(key)].add(key,
+                                                         clustering,
+                                                         value,
+                                                         allocatedBytes -> {
+                                                             memtable.markExtraOnHeapUsed(allocatedBytes, opGroup);
+                                                             estimatedOnHeapMemoryUsed.add(allocatedBytes);
+                                                             if (sensors != null)
+                                                                 sensors.incrementSensor(sensorContext, Type.INDEX_WRITE_BYTES, allocatedBytes);
+                                                         },
+                                                         allocatedBytes -> {
+                                                             memtable.markExtraOffHeapUsed(allocatedBytes, opGroup);
+                                                             estimatedOffHeapMemoryUsed.add(allocatedBytes);
+                                                             if (sensors != null)
+                                                                 sensors.incrementSensor(sensorContext, Type.INDEX_WRITE_BYTES, allocatedBytes);
+                                                         });
+        writeCount.increment();
+    }
+
+    @Override
+    public void update(DecoratedKey key, Clustering clustering, ByteBuffer oldValue, ByteBuffer newValue, Memtable memtable, OpOrder.Group opGroup)
+    {
+        int oldRemaining = oldValue == null ? 0 : oldValue.remaining();
+        int newRemaining = newValue == null ? 0 : newValue.remaining();
+        if (oldRemaining == 0 && newRemaining == 0)
+            return;
+
+        if (oldRemaining == newRemaining && validator.compare(oldValue, newValue) == 0)
+            return;
+
+        // The terms inserted into the index could still be the same in the case of certain analyzer configs.
+        // We don't know yet though, and instead of eagerly determining it, we leave it to the index to handle it.
+        rangeIndexes[boundaries.getShardForKey(key)].update(key,
+                                                            clustering,
+                                                            oldValue,
+                                                            newValue,
+                                                            allocatedBytes -> {
+                                                                memtable.markExtraOnHeapUsed(allocatedBytes, opGroup);
+                                                                estimatedOnHeapMemoryUsed.add(allocatedBytes);
+                                                                },
+                                                            allocatedBytes -> {
+                                                                memtable.markExtraOffHeapUsed(allocatedBytes, opGroup);
+                                                                estimatedOffHeapMemoryUsed.add(allocatedBytes);
+                                                            });
+        writeCount.increment();
+    }
+
+    @Override
+    public void update(DecoratedKey key, Clustering clustering, Iterator<ByteBuffer> oldValues, Iterator<ByteBuffer> newValues, Memtable memtable, OpOrder.Group opGroup)
+    {
+        // We defer on comparing old and new values here. Instead, we rely on the index to do the comparison and then
+        // have custom logic in the aggregator to ensure that we properly add/keep new values and remove old values
+        // that are not present in the new values.
+        rangeIndexes[boundaries.getShardForKey(key)].update(key,
+                                                            clustering,
+                                                            oldValues,
+                                                            newValues,
+                                                            allocatedBytes -> {
+                                                                memtable.markExtraOnHeapUsed(allocatedBytes, opGroup);
+                                                                estimatedOnHeapMemoryUsed.add(allocatedBytes);
+                                                            },
+                                                            allocatedBytes -> {
+                                                                memtable.markExtraOffHeapUsed(allocatedBytes, opGroup);
+                                                                estimatedOffHeapMemoryUsed.add(allocatedBytes);
+                                                            });
+        writeCount.increment();
+    }
+
+    public KeyRangeIterator search(QueryContext queryContext, Expression expression, AbstractBounds<PartitionPosition> keyRange, int limit)
+    {
+        int startShard = boundaries.getShardForToken(keyRange.left.getToken());
+        int endShard = keyRange.right.isMinimum() ? boundaries.shardCount() - 1 : boundaries.getShardForToken(keyRange.right.getToken());
+
+        KeyRangeConcatIterator.Builder builder = KeyRangeConcatIterator.builder(endShard - startShard + 1);
+
+        // We want to run the search on the first shard only to get the estimate on the number of matching keys.
+        // But we don't want to run the search on the other shards until the user polls more items from the
+        // result iterator. Therefore, the first shard search is special - we run the search eagerly,
+        // but the rest of the iterators are create lazily in the loop below.
+        assert rangeIndexes[startShard] != null;
+        KeyRangeIterator firstIterator = rangeIndexes[startShard].search(expression, keyRange);
+        // Assume all shards are the same size, but we must not pass 0 because of some checks in KeyRangeIterator
+        // that assume 0 means empty iterator and could fail.
+        var keyCount = Math.max(1, firstIterator.getMaxKeys());
+        builder.add(firstIterator);
+
+        // Prepare the search on the remaining shards, but wrap them in KeyRangeLazyIterator, so they don't run
+        // until the user exhaust the results given from the first shard.
+        for (int shard  = startShard + 1; shard <= endShard; ++shard)
+        {
+            assert rangeIndexes[shard] != null;
+            var index = rangeIndexes[shard];
+            var shardRange = boundaries.getBounds(shard);
+            var minKey = index.indexContext.keyFactory().createTokenOnly(shardRange.left.getToken());
+            var maxKey = index.indexContext.keyFactory().createTokenOnly(shardRange.right.getToken());
+            builder.add(new KeyRangeLazyIterator(() -> index.search(expression, keyRange), minKey, maxKey, keyCount));
+        }
+
+        return builder.build();
+    }
+
+    @Override
+    public List<CloseableIterator<PrimaryKeyWithSortKey>> orderBy(QueryContext queryContext,
+                                                                  Orderer orderer,
+                                                                  Expression slice,
+                                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                                  int limit)
+    {
+        int startShard = boundaries.getShardForToken(keyRange.left.getToken());
+        int endShard = keyRange.right.isMinimum() ? boundaries.shardCount() - 1 : boundaries.getShardForToken(keyRange.right.getToken());
+
+        if (!orderer.isBM25())
+        {
+            var iterators = new ArrayList<CloseableIterator<PrimaryKeyWithSortKey>>(endShard - startShard + 1);
+            for (int shard = startShard; shard <= endShard; ++shard)
+            {
+                assert rangeIndexes[shard] != null;
+                iterators.add(rangeIndexes[shard].orderBy(orderer, slice));
+            }
+            return iterators;
+        }
+
+        // BM25
+        var queryTerms = orderer.getQueryTerms();
+
+        // Intersect iterators to find documents containing all terms
+        var termIterators = keyIteratorsPerTerm(queryContext, keyRange, queryTerms);
+        var intersectedIterator = KeyRangeIntersectionIterator.builder(termIterators).build();
+
+        // Compute BM25 scores
+        var docStats = computeDocumentFrequencies(queryContext, queryTerms);
+        var analyzer = indexContext.getAnalyzerFactory().create();
+        var it = Streams.stream(intersectedIterator)
+                 .map(pk -> BM25Utils.EagerDocTF.createFromDocument(pk, getCellForKey(pk), analyzer, queryTerms))
+                 .filter(Objects::nonNull)
+                 .iterator();
+
+        return List.of(BM25Utils.computeScores(CloseableIterator.wrap(it),
+                                               queryTerms,
+                                               docStats,
+                                               indexContext,
+                                               memtable));
+    }
+
+    private List<KeyRangeIterator> keyIteratorsPerTerm(QueryContext queryContext, AbstractBounds<PartitionPosition> keyRange, List<ByteBuffer> queryTerms)
+    {
+        List<KeyRangeIterator> termIterators = new ArrayList<>(queryTerms.size());
+        for (ByteBuffer term : queryTerms)
+        {
+            Expression expr = new Expression(indexContext);
+            expr.add(Operator.ANALYZER_MATCHES, term);
+            KeyRangeIterator iterator = search(queryContext, expr, keyRange, Integer.MAX_VALUE);
+            termIterators.add(iterator);
+        }
+        return termIterators;
+    }
+
+    @Override
+    public long estimateMatchingRowsCount(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        int startShard = boundaries.getShardForToken(keyRange.left.getToken());
+        int endShard = keyRange.right.isMinimum() ? boundaries.shardCount() - 1 : boundaries.getShardForToken(keyRange.right.getToken());
+        return rangeIndexes[startShard].estimateMatchingRowsCount(expression, keyRange) * (endShard - startShard + 1);
+    }
+
+    @Override
+    public CloseableIterator<PrimaryKeyWithSortKey> orderResultsBy(QueryContext queryContext, List<PrimaryKey> keys, Orderer orderer, int limit)
+    {
+        if (keys.isEmpty())
+            return CloseableIterator.emptyIterator();
+
+        if (!orderer.isBM25())
+        {
+            return SortingIterator.createCloseable(
+                orderer.getComparator(),
+                keys,
+                key ->
+                {
+                    var partition = memtable.getPartition(key.partitionKey());
+                    if (partition == null)
+                        return null;
+                    var row = partition.getRow(key.clustering());
+                    if (row == null)
+                        return null;
+                    var cell = row.getCell(indexContext.getDefinition());
+                    if (cell == null)
+                        return null;
+
+                    // We do two kinds of encoding... it'd be great to make this more straight forward, but this is what
+                    // we have for now. I leave it to the reader to inspect the two methods to see the nuanced differences.
+                    var encoding = encode(TypeUtil.asIndexBytes(cell.buffer(), validator));
+                    return new PrimaryKeyWithByteComparable(indexContext, memtable, key, encoding);
+                },
+                Runnables.doNothing()
+            );
+        }
+
+        // BM25
+        var analyzer = indexContext.getAnalyzerFactory().create();
+        var queryTerms = orderer.getQueryTerms();
+        var docStats = computeDocumentFrequencies(queryContext, queryTerms);
+        var it = keys.stream()
+                     .map(pk -> BM25Utils.EagerDocTF.createFromDocument(pk, getCellForKey(pk), analyzer, queryTerms))
+                     .filter(Objects::nonNull)
+                     .iterator();
+        return BM25Utils.computeScores(CloseableIterator.wrap(it),
+                                       queryTerms,
+                                       docStats,
+                                       indexContext,
+                                       memtable);
+    }
+
+    /**
+     * Count document frequencies for each term using brute force
+     */
+    private BM25Utils.DocStats computeDocumentFrequencies(QueryContext queryContext, List<ByteBuffer> queryTerms)
+    {
+        var termIterators = keyIteratorsPerTerm(queryContext, Bounds.unbounded(indexContext.getPartitioner()), queryTerms);
+        var documentFrequencies = new HashMap<ByteBuffer, Long>();
+        for (int i = 0; i < queryTerms.size(); i++)
+        {
+            // KeyRangeIterator.getMaxKeys is not accurate enough, we have to count them
+            long keys = 0;
+            for (var it = termIterators.get(i); it.hasNext(); )
+            {
+                PrimaryKey pk = it.next();
+                Cell<?> cellForKey = getCellForKey(pk);
+                if (cellForKey == null)
+                    // skip deleted rows
+                    continue;
+                keys++;
+            }
+            documentFrequencies.put(queryTerms.get(i), keys);
+        }
+        long docCount = 0;
+
+        // count all documents in the queried column
+        try (var it = memtable.partitionIterator(ColumnFilter.selection(RegularAndStaticColumns.of(indexContext.getDefinition())),
+                                                     DataRange.allData(memtable.metadata().partitioner), NOOP_LISTENER))
+        {
+            while (it.hasNext())
+            {
+                var partitions = it.next();
+                while (partitions.hasNext())
+                {
+                    var unfiltered = partitions.next();
+                    if (!unfiltered.isRow())
+                        continue;
+                    var row = (Row) unfiltered;
+                    var cell = row.getCell(indexContext.getDefinition());
+                    if (cell == null)
+                        continue;
+
+                    docCount++;
+                }
+            }
+        }
+        return new BM25Utils.DocStats(documentFrequencies, docCount);
+    }
+
+    @Nullable
+    private org.apache.cassandra.db.rows.Cell<?> getCellForKey(PrimaryKey key)
+    {
+        var partition = memtable.getPartition(key.partitionKey());
+        if (partition == null)
+            return null;
+        var row = partition.getRow(key.clustering());
+        if (row == null)
+            return null;
+        return row.getCell(indexContext.getDefinition());
+    }
+
+    private ByteComparable encode(ByteBuffer input)
+    {
+        return Version.latest().onDiskFormat().encodeForTrie(input, indexContext.getValidator());
+    }
+
+    /**
+     * NOTE: returned data may contain partition key not within the provided min and max which are only used to find
+     * corresponding subranges. We don't do filtering here to avoid unnecessary token comparison. In case of JBOD,
+     * min/max should align exactly at token boundaries. In case of tiered-storage, keys within min/max may not
+     * belong to the given sstable.
+     *
+     * @param min minimum partition key used to find min subrange
+     * @param max maximum partition key used to find max subrange
+     *
+     * @return iterator of indexed term to primary keys mapping in sorted by indexed term and primary key.
+     */
+    @Override
+    public Iterator<Pair<ByteComparable.Preencoded, List<PkWithFrequency>>> iterator(DecoratedKey min, DecoratedKey max)
+    {
+        int minSubrange = min == null ? 0 : boundaries.getShardForKey(min);
+        int maxSubrange = max == null ? rangeIndexes.length - 1 : boundaries.getShardForKey(max);
+
+        List<Iterator<Pair<ByteComparable.Preencoded, List<PkWithFrequency>>>> rangeIterators = new ArrayList<>(maxSubrange - minSubrange + 1);
+        for (int i = minSubrange; i <= maxSubrange; i++)
+            rangeIterators.add(rangeIndexes[i].iterator());
+
+        return MergeIterator.get(rangeIterators,
+                                 (o1, o2) -> ByteComparable.compare(o1.left, o2.left),
+                                 new PrimaryKeysMergeReducer(rangeIterators.size()));
+    }
+
+    /**
+     * Used to merge sorted primary keys from multiple TrieMemoryIndex shards for a given indexed term.
+     * For each term that appears in multiple shards, the reducer:
+     * 1. Receives exactly one call to reduce() per shard containing that term
+     * 2. Merges all the primary keys for that term via getReduced()
+     * 3. Resets state via onKeyChange() before processing the next term
+     * <p>
+     * While this follows the Reducer pattern, its "reduction" operation is a simple merge since each term
+     * appears at most once per shard, and each key will only be found in a given shard, so there are no values to aggregate;
+     * we simply combine and sort the primary keys from each shard that contains the term.
+     */
+    private static class PrimaryKeysMergeReducer extends Reducer<Pair<ByteComparable.Preencoded, List<PkWithFrequency>>, Pair<ByteComparable.Preencoded, List<PkWithFrequency>>>
+    {
+        private final Pair<ByteComparable.Preencoded, List<PkWithFrequency>>[] rangeIndexEntriesToMerge;
+        private final Comparator<PrimaryKey> comparator;
+
+        private ByteComparable.Preencoded term;
+
+        @SuppressWarnings("unchecked")
+            // The size represents the number of range indexes that have been selected for the merger
+        PrimaryKeysMergeReducer(int size)
+        {
+            this.rangeIndexEntriesToMerge = new Pair[size];
+            this.comparator = PrimaryKey::compareTo;
+        }
+
+        @Override
+        // Receive the term entry for a range index. This should only be called once for each
+        // range index before reduction.
+        public void reduce(int index, Pair<ByteComparable.Preencoded, List<PkWithFrequency>> termPair)
+        {
+            Preconditions.checkArgument(rangeIndexEntriesToMerge[index] == null, "Terms should be unique in the memory index");
+
+            rangeIndexEntriesToMerge[index] = termPair;
+            if (termPair != null && term == null)
+                term = termPair.left;
+        }
+
+        @Override
+        // Return a merger of the term keys for the term.
+        public Pair<ByteComparable.Preencoded, List<PkWithFrequency>> getReduced()
+        {
+            Preconditions.checkArgument(term != null, "The term must exist in the memory index");
+
+            var merged = new ArrayList<PkWithFrequency>();
+            for (var p : rangeIndexEntriesToMerge)
+                if (p != null && p.right != null)
+                    merged.addAll(p.right);
+
+            merged.sort((o1, o2) -> comparator.compare(o1.pk, o2.pk));
+            return Pair.create(term, merged);
+        }
+
+        @Override
+        public void onKeyChange()
+        {
+            Arrays.fill(rangeIndexEntriesToMerge, null);
+            term = null;
+        }
+    }
+
+    @VisibleForTesting
+    public MemoryIndex[] getRangeIndexes()
+    {
+        return rangeIndexes;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/VectorMemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/VectorMemoryIndex.java
deleted file mode 100644
index bea5cb877fa0..000000000000
--- a/src/java/org/apache/cassandra/index/sai/memory/VectorMemoryIndex.java
+++ /dev/null
@@ -1,375 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.memory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListSet;
-import java.util.concurrent.atomic.LongAdder;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-
-import io.github.jbellis.jvector.util.Bits;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.VectorQueryContext;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.disk.v1.vector.OnHeapGraph;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.iterators.KeyRangeListIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.PrimaryKeys;
-import org.apache.cassandra.index.sai.utils.RangeUtil;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-
-import static java.lang.Math.log;
-import static java.lang.Math.max;
-import static java.lang.Math.min;
-import static java.lang.Math.pow;
-
-public class VectorMemoryIndex extends MemoryIndex
-{
-    private final OnHeapGraph<PrimaryKey> graph;
-    private final LongAdder writeCount = new LongAdder();
-
-    private PrimaryKey minimumKey;
-    private PrimaryKey maximumKey;
-
-    private final NavigableSet<PrimaryKey> primaryKeys = new ConcurrentSkipListSet<>();
-
-    public VectorMemoryIndex(StorageAttachedIndex index)
-    {
-        super(index);
-        this.graph = new OnHeapGraph<>(index.termType().indexType(), index.indexWriterConfig());
-    }
-
-    @Override
-    public synchronized long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
-    {
-        if (value == null || value.remaining() == 0 || !index.validateTermSize(key, value, false, null))
-            return 0;
-
-        var primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering)
-                                               : index.keyFactory().create(key);
-        return index(primaryKey, value);
-    }
-
-    private long index(PrimaryKey primaryKey, ByteBuffer value)
-    {
-        updateKeyBounds(primaryKey);
-
-        writeCount.increment();
-        primaryKeys.add(primaryKey);
-        return graph.add(value, primaryKey, OnHeapGraph.InvalidVectorBehavior.FAIL);
-    }
-
-    @Override
-    public long update(DecoratedKey key, Clustering<?> clustering, ByteBuffer oldValue, ByteBuffer newValue)
-    {
-        int oldRemaining = oldValue == null ? 0 : oldValue.remaining();
-        int newRemaining = newValue == null ? 0 : newValue.remaining();
-        if (oldRemaining == 0 && newRemaining == 0)
-            return 0;
-
-        boolean different;
-        if (oldRemaining != newRemaining)
-        {
-            assert oldRemaining == 0 || newRemaining == 0; // one of them is null
-            different = true;
-        }
-        else
-        {
-            different = index.termType().compare(oldValue, newValue) != 0;
-        }
-
-        long bytesUsed = 0;
-        if (different)
-        {
-            var primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering)
-                                                   : index.keyFactory().create(key);
-            // update bounds because only rows with vectors are included in the key bounds,
-            // so if the vector was null before, we won't have included it
-            updateKeyBounds(primaryKey);
-
-            // make the changes in this order, so we don't have a window where the row is not in the index at all
-            if (newRemaining > 0)
-                bytesUsed += graph.add(newValue, primaryKey, OnHeapGraph.InvalidVectorBehavior.FAIL);
-            if (oldRemaining > 0)
-                bytesUsed -= graph.remove(oldValue, primaryKey);
-
-            // remove primary key if it's no longer indexed
-            if (newRemaining <= 0 && oldRemaining > 0)
-                primaryKeys.remove(primaryKey);
-        }
-        return bytesUsed;
-    }
-
-    private void updateKeyBounds(PrimaryKey primaryKey)
-    {
-        if (minimumKey == null)
-            minimumKey = primaryKey;
-        else if (primaryKey.compareTo(minimumKey) < 0)
-            minimumKey = primaryKey;
-        if (maximumKey == null)
-            maximumKey = primaryKey;
-        else if (primaryKey.compareTo(maximumKey) > 0)
-            maximumKey = primaryKey;
-    }
-
-    @Override
-    public KeyRangeIterator search(QueryContext queryContext, Expression expr, AbstractBounds<PartitionPosition> keyRange)
-    {
-        assert expr.getIndexOperator() == Expression.IndexOperator.ANN : "Only ANN is supported for vector search, received " + expr.getIndexOperator();
-
-        VectorQueryContext vectorQueryContext = queryContext.vectorContext();
-
-        var buffer = expr.lower().value.raw;
-        float[] qv = index.termType().decomposeVector(buffer);
-
-        Bits bits;
-        if (!RangeUtil.coversFullRing(keyRange))
-        {
-            // if left bound is MIN_BOUND or KEY_BOUND, we need to include all token-only PrimaryKeys with same token
-            boolean leftInclusive = keyRange.left.kind() != PartitionPosition.Kind.MAX_BOUND;
-            // if right bound is MAX_BOUND or KEY_BOUND, we need to include all token-only PrimaryKeys with same token
-            boolean rightInclusive = keyRange.right.kind() != PartitionPosition.Kind.MIN_BOUND;
-            // if right token is MAX (Long.MIN_VALUE), there is no upper bound
-            boolean isMaxToken = keyRange.right.getToken().isMinimum(); // max token
-
-            PrimaryKey left = index.keyFactory().create(keyRange.left.getToken()); // lower bound
-            PrimaryKey right = isMaxToken ? null : index.keyFactory().create(keyRange.right.getToken()); // upper bound
-
-            Set<PrimaryKey> resultKeys = isMaxToken ? primaryKeys.tailSet(left, leftInclusive) : primaryKeys.subSet(left, leftInclusive, right, rightInclusive);
-            if (!vectorQueryContext.getShadowedPrimaryKeys().isEmpty())
-                resultKeys = resultKeys.stream().filter(pk -> !vectorQueryContext.containsShadowedPrimaryKey(pk)).collect(Collectors.toSet());
-
-            if (resultKeys.isEmpty())
-                return KeyRangeIterator.empty();
-
-            int bruteForceRows = maxBruteForceRows(vectorQueryContext.limit(), resultKeys.size(), graph.size());
-            Tracing.trace("Search range covers {} rows; max brute force rows is {} for memtable index with {} nodes, LIMIT {}",
-                          resultKeys.size(), bruteForceRows, graph.size(), vectorQueryContext.limit());
-            if (resultKeys.size() < Math.max(vectorQueryContext.limit(), bruteForceRows))
-                return new ReorderingRangeIterator(new PriorityQueue<>(resultKeys));
-            else
-                bits = new KeyRangeFilteringBits(keyRange, vectorQueryContext.bitsetForShadowedPrimaryKeys(graph));
-        }
-        else
-        {
-            // partition/range deletion won't trigger index update, so we have to filter shadow primary keys in memtable index
-            bits = queryContext.vectorContext().bitsetForShadowedPrimaryKeys(graph);
-        }
-
-        var keyQueue = graph.search(qv, queryContext.vectorContext().limit(), bits);
-        if (keyQueue.isEmpty())
-            return KeyRangeIterator.empty();
-        return new ReorderingRangeIterator(keyQueue);
-    }
-
-    @Override
-    public KeyRangeIterator limitToTopResults(List<PrimaryKey> primaryKeys, Expression expression, int limit)
-    {
-        if (minimumKey == null)
-            // This case implies maximumKey is empty too.
-            return KeyRangeIterator.empty();
-
-        List<PrimaryKey> results = primaryKeys.stream()
-                                              .dropWhile(k -> k.compareTo(minimumKey) < 0)
-                                              .takeWhile(k -> k.compareTo(maximumKey) <= 0)
-                                              .collect(Collectors.toList());
-
-        int maxBruteForceRows = maxBruteForceRows(limit, results.size(), graph.size());
-        Tracing.trace("SAI materialized {} rows; max brute force rows is {} for memtable index with {} nodes, LIMIT {}",
-                      results.size(), maxBruteForceRows, graph.size(), limit);
-        if (results.size() <= maxBruteForceRows)
-        {
-            if (results.isEmpty())
-                return KeyRangeIterator.empty();
-            return new KeyRangeListIterator(minimumKey, maximumKey, results);
-        }
-
-        ByteBuffer buffer = expression.lower().value.raw;
-        float[] qv = index.termType().decomposeVector(buffer);
-        var bits = new KeyFilteringBits(results);
-        var keyQueue = graph.search(qv, limit, bits);
-        if (keyQueue.isEmpty())
-            return KeyRangeIterator.empty();
-        return new ReorderingRangeIterator(keyQueue);
-    }
-
-    private int maxBruteForceRows(int limit, int nPermittedOrdinals, int graphSize)
-    {
-        int expectedNodesVisited = expectedNodesVisited(limit, nPermittedOrdinals, graphSize);
-        int expectedComparisons = index.indexWriterConfig().getMaximumNodeConnections() * expectedNodesVisited;
-        // in-memory comparisons are cheaper than pulling a row off disk and then comparing
-        // VSTODO this is dramatically oversimplified
-        // larger dimension should increase this, because comparisons are more expensive
-        // lower chunk cache hit ratio should decrease this, because loading rows is more expensive
-        double memoryToDiskFactor = 0.25;
-        return (int) max(limit, memoryToDiskFactor * expectedComparisons);
-    }
-
-    /**
-     * All parameters must be greater than zero.  nPermittedOrdinals may be larger than graphSize.
-     */
-    public static int expectedNodesVisited(int limit, int nPermittedOrdinals, int graphSize)
-    {
-        // constants are computed by Code Interpreter based on observed comparison counts in tests
-        // https://chat.openai.com/share/2b1d7195-b4cf-4a45-8dce-1b9b2f893c75
-        var sizeRestriction = min(nPermittedOrdinals, graphSize);
-        var raw = (int) (0.7 * pow(log(graphSize), 2) *
-                         pow(graphSize, 0.33) *
-                         pow(log(limit), 2) *
-                         pow(log((double) graphSize / sizeRestriction), 2) / pow(sizeRestriction, 0.13));
-        // we will always visit at least min(limit, graphSize) nodes, and we can't visit more nodes than exist in the graph
-        return min(max(raw, min(limit, graphSize)), graphSize);
-    }
-
-    @Override
-    public Iterator<Pair<ByteComparable, PrimaryKeys>> iterator()
-    {
-        // This method is only used when merging an in-memory index with a RowMapping. This is done a different
-        // way with the graph using the writeData method below.
-        throw new UnsupportedOperationException();
-    }
-
-    public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor,
-                                                            IndexIdentifier indexIdentifier,
-                                                            Function<PrimaryKey, Integer> postingTransformer) throws IOException
-    {
-        return graph.writeData(indexDescriptor, indexIdentifier, postingTransformer);
-    }
-
-    @Override
-    public boolean isEmpty()
-    {
-        return graph.isEmpty();
-    }
-
-    @Nullable
-    @Override
-    public ByteBuffer getMinTerm()
-    {
-        return null;
-    }
-
-    @Nullable
-    @Override
-    public ByteBuffer getMaxTerm()
-    {
-        return null;
-    }
-
-    private class KeyRangeFilteringBits implements Bits
-    {
-        private final AbstractBounds<PartitionPosition> keyRange;
-        @Nullable
-        private final Bits bits;
-
-        public KeyRangeFilteringBits(AbstractBounds<PartitionPosition> keyRange, @Nullable Bits bits)
-        {
-            this.keyRange = keyRange;
-            this.bits = bits;
-        }
-
-        @Override
-        public boolean get(int ordinal)
-        {
-            if (bits != null && !bits.get(ordinal))
-                return false;
-
-            var keys = graph.keysFromOrdinal(ordinal);
-            return keys.stream().anyMatch(k -> keyRange.contains(k.partitionKey()));
-        }
-
-        @Override
-        public int length()
-        {
-            return graph.size();
-        }
-    }
-
-    private class ReorderingRangeIterator extends KeyRangeIterator
-    {
-        private final PriorityQueue<PrimaryKey> keyQueue;
-
-        ReorderingRangeIterator(PriorityQueue<PrimaryKey> keyQueue)
-        {
-            super(minimumKey, maximumKey, keyQueue.size());
-            this.keyQueue = keyQueue;
-        }
-
-        @Override
-        // VSTODO maybe we can abuse "current" to avoid having to pop and re-add the last skipped key
-        protected void performSkipTo(PrimaryKey nextKey)
-        {
-            while (!keyQueue.isEmpty() && keyQueue.peek().compareTo(nextKey) < 0)
-                keyQueue.poll();
-        }
-
-        @Override
-        public void close() {}
-
-        @Override
-        protected PrimaryKey computeNext()
-        {
-            if (keyQueue.isEmpty())
-                return endOfData();
-            return keyQueue.poll();
-        }
-    }
-
-    private class KeyFilteringBits implements Bits
-    {
-        private final List<PrimaryKey> results;
-
-        public KeyFilteringBits(List<PrimaryKey> results)
-        {
-            this.results = results;
-        }
-
-        @Override
-        public boolean get(int i)
-        {
-            var pk = graph.keysFromOrdinal(i);
-            return results.stream().anyMatch(pk::contains);
-        }
-
-        @Override
-        public int length()
-        {
-            return results.size();
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java
index 1bb0a7666ac3..9d0568f2028f 100644
--- a/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java
+++ b/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java
@@ -20,7 +20,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.metrics.DefaultNameFactory;
 
@@ -36,11 +35,6 @@ public abstract class AbstractMetrics
     private final String scope;
     protected final List<CassandraMetricsRegistry.MetricName> tracked = new ArrayList<>();
 
-    AbstractMetrics(IndexIdentifier indexIdentifier, String scope)
-    {
-        this(indexIdentifier.keyspaceName, indexIdentifier.tableName, indexIdentifier.indexName, scope);
-    }
-
     AbstractMetrics(String keyspace, String table, String scope)
     {
         this(keyspace, table, null, scope);
@@ -48,7 +42,7 @@ public abstract class AbstractMetrics
 
     AbstractMetrics(String keyspace, String table, String index, String scope)
     {
-        assert keyspace != null && table != null : "SAI metrics must include keyspace and table";
+        assert keyspace != null && table != null : "SAI metrics must include table metadata";
         this.keyspace = keyspace;
         this.table = table;
         this.index = index;
@@ -68,12 +62,12 @@ protected CassandraMetricsRegistry.MetricName createMetricName(String name)
 
     protected CassandraMetricsRegistry.MetricName createMetricName(String name, String scope)
     {
-        String metricScope = keyspace + '.' + table;
+        String metricScope = keyspace + "." + table;
         if (index != null)
         {
-            metricScope += '.' + index;
+            metricScope += "." + index;
         }
-        metricScope += '.' + scope;
+        metricScope += "." + scope;
 
         CassandraMetricsRegistry.MetricName metricName = new CassandraMetricsRegistry.MetricName(DefaultNameFactory.GROUP_NAME,
                                                                                                  TYPE, name, metricScope, createMBeanName(name, scope));
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java
index f922231d0b37..6a38d9ae2a4f 100644
--- a/src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java
+++ b/src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java
@@ -18,18 +18,20 @@
 package org.apache.cassandra.index.sai.metrics;
 
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.LongAdder;
 
+import com.codahale.metrics.Counter;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Timer;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
+import io.github.jbellis.jvector.graph.SearchResult;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
 public abstract class ColumnQueryMetrics extends AbstractMetrics
 {
-    protected ColumnQueryMetrics(IndexIdentifier indexIdentifier)
+    protected ColumnQueryMetrics(String keyspace, String table, String indexName)
     {
-        super(indexIdentifier, "ColumnQueryMetrics");
+        super(keyspace, table, indexName, "ColumnQueryMetrics");
     }
 
     public static class TrieIndexMetrics extends ColumnQueryMetrics implements QueryEventListener.TrieIndexEventListener
@@ -43,9 +45,9 @@ public static class TrieIndexMetrics extends ColumnQueryMetrics implements Query
 
         private final QueryEventListener.PostingListEventListener postingsListener;
 
-        public TrieIndexMetrics(IndexIdentifier indexIdentifier)
+        public TrieIndexMetrics(String keyspace, String table, String indexName)
         {
-            super(indexIdentifier);
+            super(keyspace, table, indexName);
 
             termsTraversalTotalTime = Metrics.timer(createMetricName("TermsLookupLatency"));
 
@@ -70,12 +72,12 @@ public QueryEventListener.PostingListEventListener postingListEventListener()
         }
     }
 
-    public static class BalancedTreeIndexMetrics extends ColumnQueryMetrics implements QueryEventListener.BalancedTreeEventListener
+    public static class BKDIndexMetrics extends ColumnQueryMetrics implements QueryEventListener.BKDIndexEventListener
     {
-        private static final String BALANCED_TREE_POSTINGS_TYPE = "BalancedTreePostings";
+        private static final String BKD_POSTINGS_TYPE = "KDTreePostings";
 
         /**
-         * Balanced Tree index metrics.
+         * BKD index metrics.
          */
         private final Timer intersectionLatency;
         private final Meter postingsNumPostings;
@@ -83,16 +85,16 @@ public static class BalancedTreeIndexMetrics extends ColumnQueryMetrics implemen
 
         private final QueryEventListener.PostingListEventListener postingsListener;
 
-        public BalancedTreeIndexMetrics(IndexIdentifier indexIdentifier)
+        public BKDIndexMetrics(String keyspace, String table, String indexName)
         {
-            super(indexIdentifier);
+            super(keyspace, table, indexName);
 
-            intersectionLatency = Metrics.timer(createMetricName("BalancedTreeIntersectionLatency"));
-            intersectionEarlyExits = Metrics.meter(createMetricName("BalancedTreeIntersectionEarlyExits"));
+            intersectionLatency = Metrics.timer(createMetricName("KDTreeIntersectionLatency"));
+            intersectionEarlyExits = Metrics.meter(createMetricName("KDTreeIntersectionEarlyExits"));
 
-            postingsNumPostings = Metrics.meter(createMetricName("NumPostings", BALANCED_TREE_POSTINGS_TYPE));
+            postingsNumPostings = Metrics.meter(createMetricName("NumPostings", BKD_POSTINGS_TYPE));
 
-            Meter postingDecodes = Metrics.meter(createMetricName("PostingDecodes", BALANCED_TREE_POSTINGS_TYPE));
+            Meter postingDecodes = Metrics.meter(createMetricName("PostingDecodes", BKD_POSTINGS_TYPE));
 
             postingsListener = new PostingListEventsMetrics(postingDecodes);
         }
@@ -143,4 +145,102 @@ public void postingDecoded(long postingsDecoded)
             postingDecodes.mark(postingsDecoded);
         }
     }
+
+    /**
+     * Example VectorIndexMetrics for tracking ANN/Vector index–related metrics.
+     * You will also need a corresponding QueryEventListener implementation
+     * (e.g. VectorIndexEventListener) that calls these methods.
+     */
+    public static class VectorIndexMetrics extends ColumnQueryMetrics implements QueryEventListener.VectorIndexEventListener
+    {
+        // Vector index meatrics
+        // Note that the counters will essentially give us a number of operations per second. We lose the notion
+        // of per query counts, but we can back into an average by dividing by the number of queries.
+        public final Counter annNodesVisited;
+        public final Counter annNodesReranked;
+        public final Counter annNodesExpanded;
+        public final Counter annNodesExpandedBaseLayer;
+        public final Counter annGraphSearches;
+        public final Counter annGraphResumes;
+        public final Timer   annGraphSearchLatency; // Note that this timer measures individual graph search latency
+
+        public final Counter bruteForceNodesVisited;
+        public final Counter bruteForceNodesReranked;
+
+        // While not query metrics, these are vector specific metrics for the column.
+        public final LongAdder quantizationMemoryBytes;
+        public final LongAdder ordinalsMapMemoryBytes;
+        public final LongAdder onDiskGraphsCount;
+        public final LongAdder onDiskGraphVectorsCount;
+
+        public VectorIndexMetrics(String keyspace, String table, String indexName)
+        {
+            super(keyspace, table, indexName);
+
+            // Initialize Counters and Timer for ANN search
+            annNodesVisited = Metrics.counter(createMetricName("ANNNodesVisited"));
+            annNodesReranked = Metrics.counter(createMetricName("ANNNodesReranked"));
+            annNodesExpanded = Metrics.counter(createMetricName("ANNNodesExpanded"));
+            annNodesExpandedBaseLayer = Metrics.counter(createMetricName("ANNNodesExpandedBaseLayer"));
+            annGraphSearches = Metrics.counter(createMetricName("ANNGraphSearches"));
+            annGraphResumes = Metrics.counter(createMetricName("ANNGraphResumes"));
+            annGraphSearchLatency = Metrics.timer(createMetricName("ANNGraphSearchLatency"));
+
+            // Initialize Counters for brute-force fallback (if applicable)
+            bruteForceNodesVisited = Metrics.counter(createMetricName("BruteForceNodesVisited"));
+            bruteForceNodesReranked = Metrics.counter(createMetricName("BruteForceNodesReranked"));
+
+            // Initialize Gauge for PQ bytes. Ignoring codahale metrics for now.
+            quantizationMemoryBytes = new LongAdder();
+            ordinalsMapMemoryBytes = new LongAdder();
+            onDiskGraphVectorsCount = new LongAdder();
+            onDiskGraphsCount = new LongAdder();
+        }
+
+        @Override
+        public void onGraphLoaded(long quantizationBytes, long ordinalsMapCachedBytes, long vectorsLoaded)
+        {
+            this.quantizationMemoryBytes.add(quantizationBytes);
+            this.ordinalsMapMemoryBytes.add(ordinalsMapCachedBytes);
+            this.onDiskGraphVectorsCount.add(vectorsLoaded);
+            this.onDiskGraphsCount.increment();
+        }
+
+        @Override
+        public void onGraphClosed(long quantizationBytes, long ordinalsMapCachedBytes, long vectorsLoaded)
+        {
+            this.quantizationMemoryBytes.add(-quantizationBytes);
+            this.ordinalsMapMemoryBytes.add(-ordinalsMapCachedBytes);
+            this.onDiskGraphVectorsCount.add(-vectorsLoaded);
+            this.onDiskGraphsCount.decrement();
+        }
+
+        @Override
+        public void onSearchResult(SearchResult result, long latencyNs, boolean isResume)
+        {
+            annNodesVisited.inc(result.getVisitedCount());
+            annNodesReranked.inc(result.getRerankedCount());
+            annNodesExpanded.inc(result.getExpandedCount());
+            annNodesExpandedBaseLayer.inc(result.getExpandedCountBaseLayer());
+            annGraphSearchLatency.update(latencyNs, TimeUnit.NANOSECONDS);
+            if (isResume)
+                annGraphResumes.inc();
+            else
+                annGraphSearches.inc();
+        }
+
+        // These are the approximate similarity comparisons
+        @Override
+        public void onBruteForceNodesVisited(int visited)
+        {
+            bruteForceNodesVisited.inc(visited);
+        }
+
+        // These are the exact similarity comparisons
+        @Override
+        public void onBruteForceNodesReranked(int visited)
+        {
+            bruteForceNodesReranked.inc(visited);
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/IndexGroupMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/IndexGroupMetrics.java
index f6545760b7aa..0e1bf49b2485 100644
--- a/src/java/org/apache/cassandra/index/sai/metrics/IndexGroupMetrics.java
+++ b/src/java/org/apache/cassandra/index/sai/metrics/IndexGroupMetrics.java
@@ -25,11 +25,15 @@
 
 public class IndexGroupMetrics extends AbstractMetrics
 {
+    public final Gauge openIndexFiles;
+    public final Gauge diskUsedBytes;
+
     public IndexGroupMetrics(TableMetadata table, StorageAttachedIndexGroup group)
     {
         super(table.keyspace, table.name, "IndexGroupMetrics");
 
-        Metrics.register(createMetricName("OpenIndexFiles"), (Gauge<Integer>) group::openIndexFiles);
-        Metrics.register(createMetricName("DiskUsedBytes"), (Gauge<Long>) group::diskUsage);
+        openIndexFiles = Metrics.register(createMetricName("OpenIndexFiles"), group::openIndexFiles);
+
+        diskUsedBytes = Metrics.register(createMetricName("DiskUsedBytes"), group::diskUsage);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java
index 575fb79b9b9d..07ae213f8e75 100644
--- a/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java
+++ b/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java
@@ -21,42 +21,51 @@
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.memory.MemtableIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
 public class IndexMetrics extends AbstractMetrics
 {
     public final Timer memtableIndexWriteLatency;
-
+    
+    public final Gauge ssTableCellCount;
+    public final Gauge liveMemtableIndexWriteCount;
+    public final Gauge diskUsedBytes;
+    public final Gauge memtableOnHeapIndexBytes;
+    public final Gauge memtableOffHeapIndexBytes;
+    public final Gauge indexFileCacheBytes;
+    
     public final Counter memtableIndexFlushCount;
     public final Counter compactionCount;
     public final Counter memtableIndexFlushErrors;
     public final Counter segmentFlushErrors;
-
+    public final Counter queriesCount;
+    
     public final Histogram memtableFlushCellsPerSecond;
     public final Histogram segmentsPerCompaction;
     public final Histogram compactionSegmentCellsPerSecond;
     public final Histogram compactionSegmentBytesPerSecond;
 
-    public IndexMetrics(StorageAttachedIndex index, MemtableIndexManager memtableIndexManager)
+    public IndexMetrics(IndexContext context)
     {
-        super(index.identifier(), "IndexMetrics");
+        super(context.getKeyspace(), context.getTable(), context.getIndexName(), "IndexMetrics");
 
         memtableIndexWriteLatency = Metrics.timer(createMetricName("MemtableIndexWriteLatency"));
         compactionSegmentCellsPerSecond = Metrics.histogram(createMetricName("CompactionSegmentCellsPerSecond"), false);
         compactionSegmentBytesPerSecond = Metrics.histogram(createMetricName("CompactionSegmentBytesPerSecond"), false);
         memtableFlushCellsPerSecond = Metrics.histogram(createMetricName("MemtableIndexFlushCellsPerSecond"), false);
         segmentsPerCompaction = Metrics.histogram(createMetricName("SegmentsPerCompaction"), false);
+        ssTableCellCount = Metrics.register(createMetricName("SSTableCellCount"), context::getCellCount);
         memtableIndexFlushCount = Metrics.counter(createMetricName("MemtableIndexFlushCount"));
         compactionCount = Metrics.counter(createMetricName("CompactionCount"));
         memtableIndexFlushErrors = Metrics.counter(createMetricName("MemtableIndexFlushErrors"));
         segmentFlushErrors = Metrics.counter(createMetricName("CompactionSegmentFlushErrors"));
-        Metrics.register(createMetricName("SSTableCellCount"), (Gauge<Long>) index::cellCount);
-        Metrics.register(createMetricName("LiveMemtableIndexWriteCount"), (Gauge<Long>) memtableIndexManager::liveMemtableWriteCount);
-        Metrics.register(createMetricName("MemtableIndexBytes"), (Gauge<Long>) memtableIndexManager::estimatedMemIndexMemoryUsed);
-        Metrics.register(createMetricName("DiskUsedBytes"), (Gauge<Long>) index::diskUsage);
-        Metrics.register(createMetricName("IndexFileCacheBytes"), (Gauge<Long>) index::indexFileCacheSize);
+        queriesCount = Metrics.counter(createMetricName("QueriesCount"));
+        liveMemtableIndexWriteCount = Metrics.register(createMetricName("LiveMemtableIndexWriteCount"), context::liveMemtableWriteCount);
+        memtableOnHeapIndexBytes = Metrics.register(createMetricName("MemtableOnHeapIndexBytes"), context::estimatedOnHeapMemIndexMemoryUsed);
+        memtableOffHeapIndexBytes = Metrics.register(createMetricName("MemtableOffHeapIndexBytes"), context::estimatedOffHeapMemIndexMemoryUsed);
+        diskUsedBytes = Metrics.register(createMetricName("DiskUsedBytes"), context::diskUsage);
+        indexFileCacheBytes = Metrics.register(createMetricName("IndexFileCacheBytes"), context::indexFileCacheSize);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/MulticastQueryEventListeners.java b/src/java/org/apache/cassandra/index/sai/metrics/MulticastQueryEventListeners.java
index 5495f2f2322c..1be76175a130 100644
--- a/src/java/org/apache/cassandra/index/sai/metrics/MulticastQueryEventListeners.java
+++ b/src/java/org/apache/cassandra/index/sai/metrics/MulticastQueryEventListeners.java
@@ -28,9 +28,9 @@ public static QueryEventListener.TrieIndexEventListener of(QueryContext ctx, Que
         return new Multicast2TrieIndexEventListener(ctx, listener);
     }
 
-    public static QueryEventListener.BalancedTreeEventListener of(QueryContext ctx, QueryEventListener.BalancedTreeEventListener listener)
+    public static QueryEventListener.BKDIndexEventListener of(QueryContext ctx, QueryEventListener.BKDIndexEventListener listener)
     {
-        return new Multicast2BalancedTreeEventListener(ctx, listener);
+        return new Multicast2BKDIndexEventListener(ctx, listener);
     }
 
     public static class Multicast2TrieIndexEventListener implements QueryEventListener.TrieIndexEventListener
@@ -49,8 +49,8 @@ private Multicast2TrieIndexEventListener(QueryContext ctx, QueryEventListener.Tr
         @Override
         public void onSegmentHit()
         {
-            ctx.segmentsHit++;
-            ctx.trieSegmentsHit++;
+            ctx.addSegmentsHit(1);
+            ctx.addTrieSegmentsHit(1);
             listener.onSegmentHit();
         }
 
@@ -67,17 +67,17 @@ public QueryEventListener.PostingListEventListener postingListEventListener()
         }
     }
 
-    public static class Multicast2BalancedTreeEventListener implements QueryEventListener.BalancedTreeEventListener
+    public static class Multicast2BKDIndexEventListener implements QueryEventListener.BKDIndexEventListener
     {
         private final QueryContext ctx;
-        private final QueryEventListener.BalancedTreeEventListener listener;
-        private final Multicast2BalancedTreePostingListEventListener postingListEventListener;
+        private final QueryEventListener.BKDIndexEventListener listener;
+        private final Multicast2BKDPostingListEventListener postingListEventListener;
 
-        private Multicast2BalancedTreeEventListener(QueryContext ctx, QueryEventListener.BalancedTreeEventListener listener)
+        private Multicast2BKDIndexEventListener(QueryContext ctx, QueryEventListener.BKDIndexEventListener listener)
         {
             this.ctx = ctx;
             this.listener = listener;
-            this.postingListEventListener = new Multicast2BalancedTreePostingListEventListener(ctx, listener.postingListEventListener());
+            this.postingListEventListener = new Multicast2BKDPostingListEventListener(ctx, listener.postingListEventListener());
         }
 
         @Override
@@ -95,15 +95,15 @@ public void onIntersectionEarlyExit()
         @Override
         public void postingListsHit(int count)
         {
-            ctx.balancedTreePostingListsHit++;
+            ctx.addBkdPostingListsHit(1);
             listener.postingListsHit(count);
         }
 
         @Override
         public void onSegmentHit()
         {
-            ctx.segmentsHit++;
-            ctx.balancedTreeSegmentsHit++;
+            ctx.addSegmentsHit(1);
+            ctx.addBkdSegmentsHit(1);
             listener.onSegmentHit();
         }
 
@@ -114,12 +114,12 @@ public QueryEventListener.PostingListEventListener postingListEventListener()
         }
     }
 
-    public static class Multicast2BalancedTreePostingListEventListener implements QueryEventListener.PostingListEventListener
+    public static class Multicast2BKDPostingListEventListener implements QueryEventListener.PostingListEventListener
     {
         private final QueryContext ctx;
         private final QueryEventListener.PostingListEventListener listener;
 
-        Multicast2BalancedTreePostingListEventListener(QueryContext ctx, QueryEventListener.PostingListEventListener listener)
+        Multicast2BKDPostingListEventListener(QueryContext ctx, QueryEventListener.PostingListEventListener listener)
         {
             this.ctx = ctx;
             this.listener = listener;
@@ -128,14 +128,14 @@ public static class Multicast2BalancedTreePostingListEventListener implements Qu
         @Override
         public void onAdvance()
         {
-            ctx.balancedTreePostingsSkips++;
+            ctx.addBkdPostingsSkips(1);
             listener.onAdvance();
         }
 
         @Override
         public void postingDecoded(long postingDecoded)
         {
-            ctx.balancedTreePostingsDecodes += postingDecoded;
+            ctx.addBkdPostingsDecodes(postingDecoded);
             listener.postingDecoded(postingDecoded);
         }
     }
@@ -154,14 +154,14 @@ public static class Multicast2TriePostingListEventListener implements QueryEvent
         @Override
         public void onAdvance()
         {
-            ctx.triePostingsSkips++;
+            ctx.addTriePostingsSkips(1);
             listener.onAdvance();
         }
 
         @Override
         public void postingDecoded(long postingDecoded)
         {
-            ctx.triePostingsDecodes += postingDecoded;
+            ctx.addTriePostingsDecodes(postingDecoded);
             listener.postingDecoded(postingDecoded);
         }
     }
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/QueryEventListener.java b/src/java/org/apache/cassandra/index/sai/metrics/QueryEventListener.java
index db583d8b402f..41257169a0ab 100644
--- a/src/java/org/apache/cassandra/index/sai/metrics/QueryEventListener.java
+++ b/src/java/org/apache/cassandra/index/sai/metrics/QueryEventListener.java
@@ -19,18 +19,30 @@
 
 import java.util.concurrent.TimeUnit;
 
+import io.github.jbellis.jvector.graph.SearchResult;
+
 /**
  * Listener that gets notified during storage-attached index query execution.
  */
 public interface QueryEventListener
 {
     /**
-     * Collector for balanced tree file related metrics.
+     * Returns listener for bkd index events.
+     */
+    BKDIndexEventListener bkdIndexEventListener();
+
+    /**
+     * Returns listener for trie index events.
      */
-    interface BalancedTreeEventListener
+    TrieIndexEventListener trieIndexEventListener();
+
+    /**
+     * Collector for kd-tree index file related metrics.
+     */
+    interface BKDIndexEventListener
     {
         /**
-         * Per-segment balanced tree index intersection time in given units. Recorded when intersection completes.
+         * Per-segment kd-tree index intersection time in given units. Recorded when intersection completes.
          */
         void onIntersectionComplete(long intersectionTotalTime, TimeUnit unit);
 
@@ -40,17 +52,17 @@ interface BalancedTreeEventListener
         void onIntersectionEarlyExit();
 
         /**
-         * How many balanced tree posting list were matched during the intersection.
+         * How many bkd posting list were matched during the intersection.
          */
         void postingListsHit(int count);
 
         /**
-         * When query potentially matches value range within a segment, and we need to do a traversal.
+         * When query potentially matches value range within a segment and we need to do a traversal.
          */
         void onSegmentHit();
 
         /**
-         * Returns events listener for balanced tree postings.
+         * Returns events listener for bkd postings.
          */
         PostingListEventListener postingListEventListener();
     }
@@ -58,7 +70,7 @@ interface BalancedTreeEventListener
     interface TrieIndexEventListener
     {
         /**
-         * When query potentially matches value range within a segment, and we need to do a traversal.
+         * When query potentially matches value range within a segment and we need to do a traversal.
          */
         void onSegmentHit();
 
@@ -93,12 +105,27 @@ interface PostingListEventListener
             @Override
             public void onAdvance()
             {
+
             }
 
             @Override
             public void postingDecoded(long postingsDecoded)
             {
+
             }
         };
     }
+
+    interface VectorIndexEventListener
+    {
+        void onGraphLoaded(long quantizationBytes, long ordinalsMapCachedBytes, long vectorsLoaded);
+
+        void onGraphClosed(long pqBytes, long ordinalsMapCachedBytes, long vectorsLoaded);
+
+        void onSearchResult(SearchResult result, long latencyNs, boolean isResume);
+
+        void onBruteForceNodesVisited(int visited);
+
+        void onBruteForceNodesReranked(int visited);
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java
index 987c70ef75fe..059080490269 100644
--- a/src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java
+++ b/src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java
@@ -41,6 +41,9 @@ public class TableQueryMetrics extends AbstractMetrics
     private final Counter totalRowsFiltered;
     private final Counter totalQueriesCompleted;
 
+    private final Counter sortThenFilterQueriesCompleted;
+    private final Counter filterThenSortQueriesCompleted;
+
     public TableQueryMetrics(TableMetadata table)
     {
         super(table.keyspace, table.name, TABLE_QUERY_METRIC_TYPE);
@@ -53,12 +56,19 @@ public TableQueryMetrics(TableMetadata table)
         totalRowsFiltered = Metrics.counter(createMetricName("TotalRowsFiltered"));
         totalQueriesCompleted = Metrics.counter(createMetricName("TotalQueriesCompleted"));
         totalQueryTimeouts = Metrics.counter(createMetricName("TotalQueryTimeouts"));
+
+        sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
+        filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
     }
 
     public void record(QueryContext queryContext)
     {
-        if (queryContext.queryTimedOut)
+        if (queryContext.queryTimeouts() > 0)
+        {
+            assert queryContext.queryTimeouts() == 1;
+
             totalQueryTimeouts.inc();
+        }
 
         perQueryMetrics.record(queryContext);
     }
@@ -82,14 +92,17 @@ public class PerQueryMetrics extends AbstractMetrics
         private final Histogram rowsFiltered;
 
         /**
-         * Balanced tree index metrics.
+         * BKD index metrics.
          */
-        private final Histogram balancedTreePostingsNumPostings;
+        private final Histogram kdTreePostingsNumPostings;
         /**
-         * Balanced tree index posting lists metrics.
+         * BKD index posting lists metrics.
          */
-        private final Histogram balancedTreePostingsSkips;
-        private final Histogram balancedTreePostingsDecodes;
+        private final Histogram kdTreePostingsSkips;
+        private final Histogram kdTreePostingsDecodes;
+
+        /** Shadowed keys scan metrics **/
+        private final Histogram shadowedKeysScannedHistogram;
 
         /**
          * Trie index posting lists metrics.
@@ -97,6 +110,11 @@ public class PerQueryMetrics extends AbstractMetrics
         private final Histogram postingsSkips;
         private final Histogram postingsDecodes;
 
+        /**
+         * Cumulative time spent searching ANN graph.
+         */
+        private final Timer annGraphSearchLatency;
+
         public PerQueryMetrics(TableMetadata table)
         {
             super(table.keyspace, table.name, "PerQuery");
@@ -106,30 +124,40 @@ public PerQueryMetrics(TableMetadata table)
             sstablesHit = Metrics.histogram(createMetricName("SSTableIndexesHit"), false);
             segmentsHit = Metrics.histogram(createMetricName("IndexSegmentsHit"), false);
 
-            balancedTreePostingsSkips = Metrics.histogram(createMetricName("BalancedTreePostingsSkips"), false);
+            kdTreePostingsSkips = Metrics.histogram(createMetricName("KDTreePostingsSkips"), false);
 
-            balancedTreePostingsNumPostings = Metrics.histogram(createMetricName("BalancedTreePostingsNumPostings"), false);
-            balancedTreePostingsDecodes = Metrics.histogram(createMetricName("BalancedTreePostingsDecodes"), false);
+            kdTreePostingsNumPostings = Metrics.histogram(createMetricName("KDTreePostingsNumPostings"), false);
+            kdTreePostingsDecodes = Metrics.histogram(createMetricName("KDTreePostingsDecodes"), false);
 
             postingsSkips = Metrics.histogram(createMetricName("PostingsSkips"), false);
             postingsDecodes = Metrics.histogram(createMetricName("PostingsDecodes"), false);
 
             partitionReads = Metrics.histogram(createMetricName("PartitionReads"), false);
             rowsFiltered = Metrics.histogram(createMetricName("RowsFiltered"), false);
+
+            shadowedKeysScannedHistogram = Metrics.histogram(createMetricName("ShadowedKeysScannedHistogram"), false);
+
+            // Key vector metrics that translate to performance
+            annGraphSearchLatency = Metrics.timer(createMetricName("ANNGraphSearchLatency"));
         }
 
         private void recordStringIndexCacheMetrics(QueryContext events)
         {
-            postingsSkips.update(events.triePostingsSkips);
-            postingsDecodes.update(events.triePostingsDecodes);
+            postingsSkips.update(events.triePostingsSkips());
+            postingsDecodes.update(events.triePostingsDecodes());
         }
 
         private void recordNumericIndexCacheMetrics(QueryContext events)
         {
-            balancedTreePostingsNumPostings.update(events.balancedTreePostingListsHit);
+            kdTreePostingsNumPostings.update(events.bkdPostingListsHit());
 
-            balancedTreePostingsSkips.update(events.balancedTreePostingsSkips);
-            balancedTreePostingsDecodes.update(events.balancedTreePostingsDecodes);
+            kdTreePostingsSkips.update(events.bkdPostingsSkips());
+            kdTreePostingsDecodes.update(events.bkdPostingsDecodes());
+        }
+
+        private void recordVectorIndexMetrics(QueryContext queryContext)
+        {
+            annGraphSearchLatency.update(queryContext.annGraphSearchLatency(), TimeUnit.NANOSECONDS);
         }
 
         public void record(QueryContext queryContext)
@@ -138,32 +166,60 @@ public void record(QueryContext queryContext)
             queryLatency.update(totalQueryTimeNs, TimeUnit.NANOSECONDS);
             final long queryLatencyMicros = TimeUnit.NANOSECONDS.toMicros(totalQueryTimeNs);
 
-            sstablesHit.update(queryContext.sstablesHit);
-            segmentsHit.update(queryContext.segmentsHit);
+            final long ssTablesHit = queryContext.sstablesHit();
+            final long segmentsHit = queryContext.segmentsHit();
+            final long partitionsRead = queryContext.partitionsRead();
+            final long rowsFiltered = queryContext.rowsFiltered();
+            final long rowsPreFiltered = queryContext.rowsFiltered();
 
-            partitionReads.update(queryContext.partitionsRead);
-            totalPartitionReads.inc(queryContext.partitionsRead);
+            sstablesHit.update(ssTablesHit);
+            this.segmentsHit.update(segmentsHit);
 
-            rowsFiltered.update(queryContext.rowsFiltered);
-            totalRowsFiltered.inc(queryContext.rowsFiltered);
+            partitionReads.update(partitionsRead);
+            totalPartitionReads.inc(partitionsRead);
+
+            this.rowsFiltered.update(rowsFiltered);
+            totalRowsFiltered.inc(rowsFiltered);
+
+            if (queryContext.filterSortOrder() == QueryContext.FilterSortOrder.SCAN_THEN_FILTER)
+                sortThenFilterQueriesCompleted.inc();
+            else if (queryContext.filterSortOrder() == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
+                filterThenSortQueriesCompleted.inc();
 
             if (Tracing.isTracing())
             {
-                Tracing.trace("Index query accessed memtable indexes, {}, and {}, post-filtered {} in {}, and took {} microseconds.",
-                              pluralize(queryContext.sstablesHit, "SSTable index", "es"), pluralize(queryContext.segmentsHit, "segment", "s"),
-                              pluralize(queryContext.rowsFiltered, "row", "s"), pluralize(queryContext.partitionsRead, "partition", "s"),
-                              queryLatencyMicros);
+                if (queryContext.filterSortOrder() == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
+                {
+                    Tracing.trace("Index query accessed memtable indexes, {}, and {}, selected {} before ranking, post-filtered {} in {}, and took {} microseconds.",
+                                  pluralize(ssTablesHit, "SSTable index", "es"),
+                                  pluralize(segmentsHit, "segment", "s"),
+                                  pluralize(rowsPreFiltered, "row", "s"),
+                                  pluralize(rowsFiltered, "row", "s"),
+                                  pluralize(partitionsRead, "partition", "s"),
+                                  queryLatencyMicros);
+                }
+                else
+                {
+                    Tracing.trace("Index query accessed memtable indexes, {}, and {}, post-filtered {} in {}, and took {} microseconds.",
+                                  pluralize(ssTablesHit, "SSTable index", "es"),
+                                  pluralize(segmentsHit, "segment", "s"),
+                                  pluralize(rowsFiltered, "row", "s"),
+                                  pluralize(partitionsRead, "partition", "s"),
+                                  queryLatencyMicros);
+                }
             }
 
-            if (queryContext.trieSegmentsHit > 0)
-            {
+            if (queryContext.trieSegmentsHit() > 0)
                 recordStringIndexCacheMetrics(queryContext);
-            }
-
-            if (queryContext.balancedTreeSegmentsHit > 0)
-            {
+            if (queryContext.bkdSegmentsHit() > 0)
                 recordNumericIndexCacheMetrics(queryContext);
-            }
+            // If ann brute forced the whole search, this is 0. We don't measure brute force latency. Maybe we should?
+            // At the very least, we collect brute force comparison metrics, which should give a reasonable indicator
+            // of work done.
+            if (queryContext.annGraphSearchLatency() > 0)
+                recordVectorIndexMetrics(queryContext);
+
+            shadowedKeysScannedHistogram.update(queryContext.getShadowedPrimaryKeyCount());
 
             totalQueriesCompleted.inc();
         }
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/TableStateMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/TableStateMetrics.java
index f7b64055206d..6508c8db4fb6 100644
--- a/src/java/org/apache/cassandra/index/sai/metrics/TableStateMetrics.java
+++ b/src/java/org/apache/cassandra/index/sai/metrics/TableStateMetrics.java
@@ -28,19 +28,25 @@ public class TableStateMetrics extends AbstractMetrics
 {
     public static final String TABLE_STATE_METRIC_TYPE = "TableStateMetrics";
 
+    private final Gauge diskUsageBytes;
+    private final Gauge diskUsagePercentageOfBaseTable;
+    private final Gauge totalIndexCount;
+    private final Gauge totalIndexBuildsInProgress;
+    private final Gauge totalQueryableIndexCount;
+
     public TableStateMetrics(TableMetadata table, StorageAttachedIndexGroup group)
     {
         super(table.keyspace, table.name, TABLE_STATE_METRIC_TYPE);
-        Metrics.register(createMetricName("DiskUsedBytes"), (Gauge<Long>) group::totalDiskUsage);
-        Metrics.register(createMetricName("DiskPercentageOfBaseTable"), (Gauge<Double>) new RatioGauge() {
+        totalQueryableIndexCount = Metrics.register(createMetricName("TotalQueryableIndexCount"), group::totalQueryableIndexCount);
+        totalIndexCount = Metrics.register(createMetricName("TotalIndexCount"), group::totalIndexCount);
+        totalIndexBuildsInProgress = Metrics.register(createMetricName("TotalIndexBuildsInProgress"), group::totalIndexBuildsInProgress);
+        diskUsageBytes = Metrics.register(createMetricName("DiskUsedBytes"), group::totalDiskUsage);
+        diskUsagePercentageOfBaseTable = Metrics.register(createMetricName("DiskPercentageOfBaseTable"), new RatioGauge() {
             @Override
             protected Ratio getRatio()
             {
                 return Ratio.of(group.totalDiskUsage(), group.table().metric.liveDiskSpaceUsed.getCount());
             }
         });
-        Metrics.register(createMetricName("TotalIndexCount"), (Gauge<Integer>) group::totalIndexCount);
-        Metrics.register(createMetricName("TotalQueryableIndexCount"), (Gauge<Integer>) group::totalQueryableIndexCount);
-        Metrics.register(createMetricName("TotalIndexBuildsInProgress"), (Gauge<Integer>) group::totalIndexBuildsInProgress);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/plan/Expression.java b/src/java/org/apache/cassandra/index/sai/plan/Expression.java
index 262fc3a9cc4c..8748c2a318a4 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/Expression.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/Expression.java
@@ -1,3 +1,9 @@
+/*
+ * All changes to the original code are Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -19,87 +25,89 @@
 package org.apache.cassandra.index.sai.plan;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Objects;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-
-/**
- * An {@link Expression} is an internal representation of an index query operation. They are built from
- * CQL {@link Operator} and {@link ByteBuffer} value pairs for a single column.
- * <p>
- * Each {@link Expression} consists of an {@link IndexOperator} and optional lower and upper {@link Bound}s.
- * <p>
- * The {@link IndexedExpression} has a backing {@link StorageAttachedIndex} for the index query but order to support
- * CQL expressions on columns that do not have indexes or use operators that are not supported by the index there is
- * an {@link UnindexedExpression} that does not provide a {@link StorageAttachedIndex} and can only be used for
- * post-filtering
- */
-public abstract class Expression
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.utils.GeoUtil;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.lucene.util.SloppyMath;
+
+public class Expression
 {
-    Logger logger = LoggerFactory.getLogger(Expression.class);
-
-    private final IndexTermType indexTermType;
-    protected IndexOperator operator;
+    private static final Logger logger = LoggerFactory.getLogger(Expression.class);
 
-    public Bound lower, upper;
-    // The upperInclusive and lowerInclusive flags are maintained separately to the inclusive flags
-    // in the upper and lower bounds because the upper and lower bounds have their inclusivity relaxed
-    // if the datatype being filtered is rounded in the index. These flags are used in the post-filtering
-    // process to remove values equal to the bounds.
-    public boolean upperInclusive, lowerInclusive;
-
-    Expression(IndexTermType indexTermType)
+    public enum Op
     {
-        this.indexTermType = indexTermType;
-    }
+        EQ, MATCH, PREFIX, NOT_EQ, RANGE,
+        CONTAINS_KEY, CONTAINS_VALUE,
+        NOT_CONTAINS_VALUE, NOT_CONTAINS_KEY,
+        IN, ORDER_BY, BOUNDED_ANN;
 
-    public static Expression create(StorageAttachedIndex index)
-    {
-        return new IndexedExpression(index);
-    }
-
-    public static Expression create(IndexTermType indexTermType)
-    {
-        return new UnindexedExpression(indexTermType);
-    }
-
-    public static boolean supportsOperator(Operator operator)
-    {
-        return IndexOperator.valueOf(operator) != null;
-    }
-
-    public enum IndexOperator
-    {
-        EQ, RANGE, CONTAINS_KEY, CONTAINS_VALUE, ANN;
-
-        public static IndexOperator valueOf(Operator operator)
+        public static Op valueOf(Operator operator)
         {
             switch (operator)
             {
                 case EQ:
                     return EQ;
 
+                case NEQ:
+                    return NOT_EQ;
+
                 case CONTAINS:
                     return CONTAINS_VALUE; // non-frozen map: value contains term;
 
                 case CONTAINS_KEY:
                     return CONTAINS_KEY; // non-frozen map: value contains key term;
 
+                case NOT_CONTAINS:
+                    return NOT_CONTAINS_VALUE;
+
+                case NOT_CONTAINS_KEY:
+                    return NOT_CONTAINS_KEY;
+
                 case LT:
                 case GT:
                 case LTE:
                 case GTE:
                     return RANGE;
 
+                case LIKE_PREFIX:
+                    return PREFIX;
+
+                case LIKE_MATCHES:
+                case ANALYZER_MATCHES:
+                    return MATCH;
+
+                case IN:
+                    return IN;
+
                 case ANN:
-                    return ANN;
+                case BM25:
+                case ORDER_BY_ASC:
+                case ORDER_BY_DESC:
+                    return ORDER_BY;
+
+                case BOUNDED_ANN:
+                    return BOUNDED_ANN;
 
                 default:
                     return null;
@@ -115,44 +123,52 @@ public boolean isEqualityOrRange()
         {
             return isEquality() || this == RANGE;
         }
-    }
 
-    public abstract boolean isNotIndexed();
+        public boolean isNonEquality()
+        {
+            return this == NOT_EQ || this == NOT_CONTAINS_KEY || this == NOT_CONTAINS_VALUE;
+        }
 
-    public abstract StorageAttachedIndex getIndex();
+        public boolean isContains()
+        {
+            return this == CONTAINS_KEY
+                   || this == CONTAINS_VALUE
+                   || this == NOT_CONTAINS_KEY
+                   || this == NOT_CONTAINS_VALUE;
+        }
+    }
 
-    abstract boolean hasAnalyzer();
+    public final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
 
-    abstract AbstractAnalyzer getAnalyzer();
+    public final IndexContext context;
+    public final AbstractType<?> validator;
 
-    public IndexOperator getIndexOperator()
-    {
-        return operator;
-    }
+    @VisibleForTesting
+    protected Op operation;
 
-    public IndexTermType getIndexTermType()
-    {
-        return indexTermType;
-    }
+    public Bound lower, upper;
+    private float boundedAnnEuclideanDistanceThreshold = 0;
+    private float searchRadiusMeters = 0;
+    private float searchRadiusDegreesSquared = 0;
+    public int topK;
+    // These variables are only meant to be used for final validation of the range search. They are not
+    // meant to be used when searching the index. See the 'add' method below for additional explanation.
+    private boolean upperInclusive, lowerInclusive;
+
+    final List<ByteBuffer> exclusions = new ArrayList<>();
 
-    public Bound lower()
+    public Expression(IndexContext indexContext)
     {
-        return lower;
+        this.context = indexContext;
+        this.analyzerFactory = indexContext.getAnalyzerFactory();
+        this.validator = indexContext.getValidator();
     }
 
-    public Bound upper()
+    public boolean isLiteral()
     {
-        return upper;
+        return context.isLiteral();
     }
 
-    /**
-     * This adds an operation to the current {@link Expression} instance and
-     * returns the current instance.
-     *
-     * @param op the CQL3 operation
-     * @param value the expression value
-     * @return the current expression with the added operation
-     */
     public Expression add(Operator op, ByteBuffer value)
     {
         boolean lowerInclusive, upperInclusive;
@@ -160,19 +176,38 @@ public Expression add(Operator op, ByteBuffer value)
         // range search is always inclusive, otherwise we run the risk of
         // missing values that are within the exclusive range but are rejected
         // because their rounded value is the same as the value being queried.
-        lowerInclusive = upperInclusive = indexTermType.supportsRounding();
+        lowerInclusive = upperInclusive = TypeUtil.supportsRounding(validator);
         switch (op)
         {
+            case LIKE_PREFIX:
+            case LIKE_MATCHES:
+            case ANALYZER_MATCHES:
             case EQ:
             case CONTAINS:
             case CONTAINS_KEY:
-                lower = new Bound(value, indexTermType, true);
+            case NOT_CONTAINS:
+            case NOT_CONTAINS_KEY:
+                lower = new Bound(value, validator, true);
                 upper = lower;
-                operator = IndexOperator.valueOf(op);
+                operation = Op.valueOf(op);
+                break;
+
+            case NEQ:
+                // index expressions are priority sorted
+                // and NOT_EQ is the lowest priority, which means that operation type
+                // is always going to be set before reaching it in case of RANGE or EQ.
+                if (operation == null)
+                {
+                    operation = Op.NOT_EQ;
+                    lower = new Bound(value, validator, true);
+                    upper = lower;
+                }
+                else
+                    exclusions.add(value);
                 break;
 
             case LTE:
-                if (indexTermType.isReversed())
+                if (context.getDefinition().isReversedType())
                 {
                     this.lowerInclusive = true;
                     lowerInclusive = true;
@@ -183,15 +218,15 @@ public Expression add(Operator op, ByteBuffer value)
                     upperInclusive = true;
                 }
             case LT:
-                operator = IndexOperator.RANGE;
-                if (indexTermType.isReversed())
-                    lower = new Bound(value, indexTermType, lowerInclusive);
+                operation = Op.RANGE;
+                if (context.getDefinition().isReversedType())
+                    lower = new Bound(value, validator, lowerInclusive);
                 else
-                    upper = new Bound(value, indexTermType, upperInclusive);
+                    upper = new Bound(value, validator, upperInclusive);
                 break;
 
             case GTE:
-                if (indexTermType.isReversed())
+                if (context.getDefinition().isReversedType())
                 {
                     this.upperInclusive = true;
                     upperInclusive = true;
@@ -202,56 +237,81 @@ public Expression add(Operator op, ByteBuffer value)
                     lowerInclusive = true;
                 }
             case GT:
-                operator = IndexOperator.RANGE;
-                if (indexTermType.isReversed())
-                    upper = new Bound(value, indexTermType, upperInclusive);
+                operation = Op.RANGE;
+                if (context.getDefinition().isReversedType())
+                    upper = new Bound(value, validator,  upperInclusive);
                 else
-                    lower = new Bound(value, indexTermType, lowerInclusive);
+                    lower = new Bound(value, validator, lowerInclusive);
+                break;
+            case BOUNDED_ANN:
+                operation = Op.BOUNDED_ANN;
+                lower = new Bound(value, validator, true);
+                assert upper != null;
+                searchRadiusMeters = FloatType.instance.compose(upper.value.raw);
+                boundedAnnEuclideanDistanceThreshold = GeoUtil.amplifiedEuclideanSimilarityThreshold(lower.value.vector, searchRadiusMeters);
                 break;
             case ANN:
-                operator = IndexOperator.ANN;
-                lower = new Bound(value, indexTermType, true);
-                upper = lower;
+            case BM25:
+            case ORDER_BY_ASC:
+            case ORDER_BY_DESC:
+                // If we alread have an operation on the column, we don't need to set the ORDER_BY op because
+                // it is only used to force validation on a column, and the presence of another operation will do that.
+                if (operation == null)
+                    operation = Op.ORDER_BY;
                 break;
             default:
-                throw new IllegalArgumentException("Index does not support the " + op + " operator");
+                throw new UnsupportedOperationException("Unsupported operator: " + op);
         }
 
+        assert operation != null;
+
         return this;
     }
 
-    /**
-     * Used in post-filtering to determine is an indexed value matches the expression
-     */
+    // VSTODO seems like we could optimize for CompositeType here since we know we have a key match
     public boolean isSatisfiedBy(ByteBuffer columnValue)
     {
-        // If the expression represents an ANN ordering then we return true because the actual result
-        // is approximate and will rarely / never match the expression value
-        if (indexTermType.isVector())
+        if (columnValue == null)
+            return false;
+
+        // ORDER_BY is not indepently verifiable, so we always return true
+        if (operation == Op.ORDER_BY)
             return true;
 
-        if (!indexTermType.isValid(columnValue))
+        if (!TypeUtil.isValid(columnValue, validator))
         {
-            logger.error("Value is not valid for indexed column {} with {}", indexTermType.columnName(), indexTermType.indexType());
+            logger.error(context.logMessage("Value is not valid for indexed column {} with {}"), context.getColumnName(), validator);
             return false;
         }
 
-        Value value = new Value(columnValue, indexTermType);
+        Value value = new Value(columnValue, validator);
+
+        if (operation == Op.BOUNDED_ANN)
+        {
+            double haversineDistance = SloppyMath.haversinMeters(lower.value.vector[0], lower.value.vector[1], value.vector[0], value.vector[1]);
+            return upperInclusive ? haversineDistance <= searchRadiusMeters : haversineDistance < searchRadiusMeters;
+        }
 
         if (lower != null)
         {
             // suffix check
-            if (indexTermType.isLiteral())
-                return validateStringValue(value.raw, lower.value.raw);
+            if (TypeUtil.isLiteral(validator))
+            {
+                if (!validateStringValue(value.raw, lower.value.raw))
+                    return false;
+            }
             else
             {
                 // range or (not-)equals - (mainly) for numeric values
-                int cmp = indexTermType.comparePostFilter(lower.value, value);
+                int cmp = TypeUtil.comparePostFilter(lower.value, value, validator);
 
-                // in case of EQ lower == upper
-                if (operator == IndexOperator.EQ || operator == IndexOperator.CONTAINS_KEY || operator == IndexOperator.CONTAINS_VALUE)
+                // in case of (NOT_)EQ lower == upper
+                if (operation == Op.EQ || operation == Op.CONTAINS_KEY || operation == Op.CONTAINS_VALUE)
                     return cmp == 0;
 
+                if (operation == Op.NOT_EQ || operation == Op.NOT_CONTAINS_KEY || operation == Op.NOT_CONTAINS_VALUE)
+                    return cmp != 0;
+
                 if (cmp > 0 || (cmp == 0 && !lowerInclusive))
                     return false;
             }
@@ -260,60 +320,183 @@ public boolean isSatisfiedBy(ByteBuffer columnValue)
         if (upper != null && lower != upper)
         {
             // string (prefix or suffix) check
-            if (indexTermType.isLiteral())
-                return validateStringValue(value.raw, upper.value.raw);
+            if (TypeUtil.isLiteral(validator))
+            {
+                if (!validateStringValue(value.raw, upper.value.raw))
+                    return false;
+            }
             else
             {
                 // range - mainly for numeric values
-                int cmp = indexTermType.comparePostFilter(upper.value, value);
-                return (cmp > 0 || (cmp == 0 && upperInclusive));
+                int cmp = TypeUtil.comparePostFilter(upper.value, value, validator);
+                if (cmp < 0 || (cmp == 0 && !upperInclusive))
+                    return false;
             }
         }
 
+        // as a last step let's check exclusions for the given field,
+        // this covers EQ/RANGE with exclusions.
+        for (ByteBuffer term : exclusions)
+        {
+            if (TypeUtil.isLiteral(validator) && validateStringValue(value.raw, term) ||
+                TypeUtil.comparePostFilter(new Value(term, validator), value, validator) == 0)
+                return false;
+        }
+
         return true;
     }
 
+    /**
+     * Returns the lower bound of the expression as a ByteComparable with an encoding based on the version and the
+     * validator.
+     * @param version the version of the index
+     * @return
+     */
+    public ByteComparable getEncodedLowerBoundByteComparable(Version version)
+    {
+        // Note: this value was encoded using the TypeUtil.encode method, but it wasn't
+        var bound = getPartiallyEncodedLowerBound(version);
+        if (bound == null)
+            return null;
+        // If the lower bound is inclusive, we use the LT_NEXT_COMPONENT terminator to make sure the bound is not a
+        // prefix of some other key. This ensures reverse iteration works correctly too.
+        var terminator = lower.inclusive ? ByteSource.LT_NEXT_COMPONENT : ByteSource.GT_NEXT_COMPONENT;
+        return getBoundByteComparable(bound, version, terminator);
+    }
+
+    /**
+     * Returns the upper bound of the expression as a ByteComparable with an encoding based on the version and the
+     * validator.
+     * @param version the version of the index
+     * @return
+     */
+    public ByteComparable getEncodedUpperBoundByteComparable(Version version)
+    {
+        var bound = getPartiallyEncodedUpperBound(version);
+        if (bound == null)
+            return null;
+        // If the upper bound is inclusive, we use the LT_NEXT_COMPONENT terminator to make sure the bound is not a
+        // prefix of some other key. This ensures reverse iteration works correctly too.
+        var terminator = upper.inclusive ? ByteSource.GT_NEXT_COMPONENT : ByteSource.LT_NEXT_COMPONENT;
+        return getBoundByteComparable(bound, version, terminator);
+    }
+
+    // This call encodes the byte buffer into a ByteComparable object based on the version of the index, the validator,
+    // and whether the expression is in memory or on disk.
+    private ByteComparable getBoundByteComparable(ByteBuffer unencodedBound, Version version, int terminator)
+    {
+        if (TypeUtil.isComposite(validator) && version.onOrAfter(Version.DB))
+            // Note that for ranges that have one unrestricted bound, we technically do not need the terminator
+            // because we use the 0 or the 1 at the end of the first component as the bound. However, it works
+            // with the terminator, so we use it for simplicity.
+            return TypeUtil.asComparableBytes(unencodedBound, terminator, (CompositeType) validator);
+        else
+            return version.onDiskFormat().encodeForTrie(unencodedBound, validator);
+    }
+
+    /**
+     * This is partially encoded because it uses the {@link TypeUtil#encode(ByteBuffer, AbstractType)} method on the
+     * {@link ByteBuffer}, but it does not apply the validator's encoding. We do this because we apply
+     * {@link TypeUtil#encode(ByteBuffer, AbstractType)} before we find the min/max on an index and this method is
+     * exposed publicly for determining if a bound is within an index's min/max.
+     * @param version
+     * @return
+     */
+    public ByteBuffer getPartiallyEncodedLowerBound(Version version)
+    {
+        return getBound(lower, true, version);
+    }
+
+    /**
+     * This is partially encoded because it uses the {@link TypeUtil#encode(ByteBuffer, AbstractType)} method on the
+     * {@link ByteBuffer}, but it does not apply the validator's encoding. We do this because we apply
+     * {@link TypeUtil#encode(ByteBuffer, AbstractType)} before we find the min/max on an index and this method is
+     * exposed publicly for determining if a bound is within an index's min/max.
+     * @param version
+     * @return
+     */
+    public ByteBuffer getPartiallyEncodedUpperBound(Version version)
+    {
+        return getBound(upper, false, version);
+    }
+
+    private ByteBuffer getBound(Bound bound, boolean isLowerBound, Version version)
+    {
+        if (bound == null)
+            return null;
+        // Composite types are currently only used in maps.
+        // Before DB, we need to extract the first component of the composite type to use as the trie search prefix.
+        // After DB, we can use the encoded value directly because the trie is encoded in order so the range
+        // correctly gets all relevant values.
+        if (!version.onOrAfter(Version.DB) && validator instanceof CompositeType)
+            return CompositeType.extractFirstComponentAsTrieSearchPrefix(bound.value.encoded, isLowerBound);
+        return bound.value.encoded;
+    }
+
+    public boolean isSatisfiedBy(Iterator<ByteBuffer> values)
+    {
+        if (values == null)
+            values = Collections.emptyIterator();
+
+        boolean success = operation.isNonEquality();
+        while (values.hasNext())
+        {
+            ByteBuffer v = values.next();
+            if (isSatisfiedBy(v) ^ success)
+                return !success;
+        }
+        return success;
+    }
+
     private boolean validateStringValue(ByteBuffer columnValue, ByteBuffer requestedValue)
     {
-        if (hasAnalyzer())
+        AbstractAnalyzer analyzer = analyzerFactory.create();
+        analyzer.reset(columnValue);
+        try
         {
-            AbstractAnalyzer analyzer = getAnalyzer();
-            analyzer.reset(columnValue.duplicate());
-            try
+            while (analyzer.hasNext())
             {
-                while (analyzer.hasNext())
+                final ByteBuffer term = analyzer.next();
+
+                boolean isMatch = false;
+                switch (operation)
                 {
-                    if (termMatches(analyzer.next(), requestedValue))
-                        return true;
+                    case EQ:
+                    case MATCH:
+                        // Operation.isSatisfiedBy handles conclusion on !=,
+                        // here we just need to make sure that term matched it
+                    case CONTAINS_KEY:
+                    case CONTAINS_VALUE:
+                        isMatch = validator.compare(term, requestedValue) == 0;
+                        break;
+                    case NOT_EQ:
+                    case NOT_CONTAINS_KEY:
+                    case NOT_CONTAINS_VALUE:
+                        isMatch = validator.compare(term, requestedValue) != 0;
+                        break;
+                    case RANGE:
+                        isMatch = isLowerSatisfiedBy(term) && isUpperSatisfiedBy(term);
+                        break;
+
+                    case PREFIX:
+                        isMatch = ByteBufferUtil.startsWith(term, requestedValue);
+                        break;
                 }
-                return false;
-            }
-            finally
-            {
-                analyzer.end();
+
+                if (isMatch)
+                    return true;
             }
+            return false;
         }
-        else
+        finally
         {
-            return termMatches(columnValue, requestedValue);
+            analyzer.end();
         }
     }
 
-    private boolean termMatches(ByteBuffer term, ByteBuffer requestedValue)
+    public Op getOp()
     {
-        boolean isMatch = false;
-        switch (operator)
-        {
-            case EQ:
-            case CONTAINS_KEY:
-            case CONTAINS_VALUE:
-                isMatch = indexTermType.compare(term, requestedValue) == 0;
-                break;
-            case RANGE:
-                isMatch = isLowerSatisfiedBy(term) && isUpperSatisfiedBy(term);
-                break;
-        }
-        return isMatch;
+        return operation;
     }
 
     private boolean hasLower()
@@ -331,7 +514,7 @@ private boolean isLowerSatisfiedBy(ByteBuffer value)
         if (!hasLower())
             return true;
 
-        int cmp = indexTermType.indexType().compare(value, lower.value.raw);
+        int cmp = validator.compare(value, lower.value.raw);
         return cmp > 0 || cmp == 0 && lower.inclusive;
     }
 
@@ -340,31 +523,41 @@ private boolean isUpperSatisfiedBy(ByteBuffer value)
         if (!hasUpper())
             return true;
 
-        int cmp = indexTermType.indexType().compare(value, upper.value.raw);
+        int cmp = validator.compare(value, upper.value.raw);
         return cmp < 0 || cmp == 0 && upper.inclusive;
     }
 
-    @Override
+    public float getEuclideanSearchThreshold()
+    {
+        return boundedAnnEuclideanDistanceThreshold;
+    }
+
     public String toString()
     {
-        return String.format("Expression{name: %s, op: %s, lower: (%s, %s), upper: (%s, %s)}",
-                             indexTermType.columnName(),
-                             operator,
-                             lower == null ? "null" : indexTermType.asString(lower.value.raw),
+        return String.format("Expression{name: %s, op: %s, lower: (%s, %s), upper: (%s, %s), exclusions: %s}",
+                             context.getColumnName(),
+                             operation,
+                             lower == null ? "null" : validator.getString(lower.value.raw),
                              lower != null && lower.inclusive,
-                             upper == null ? "null" : indexTermType.asString(upper.value.raw),
-                             upper != null && upper.inclusive);
+                             upper == null ? "null" : validator.getString(upper.value.raw),
+                             upper != null && upper.inclusive,
+                             Iterators.toString(Iterators.transform(exclusions.iterator(), validator::getString)));
+    }
+
+    public String getIndexName()
+    {
+        return context.getIndexName();
     }
 
-    @Override
     public int hashCode()
     {
-        return new HashCodeBuilder().append(indexTermType)
-                                    .append(operator)
-                                    .append(lower).append(upper).build();
+        return new HashCodeBuilder().append(context.getColumnName())
+                                    .append(operation)
+                                    .append(validator)
+                                    .append(lower).append(upper)
+                                    .append(exclusions).build();
     }
 
-    @Override
     public boolean equals(Object other)
     {
         if (!(other instanceof Expression))
@@ -375,77 +568,38 @@ public boolean equals(Object other)
 
         Expression o = (Expression) other;
 
-        return Objects.equals(indexTermType, o.indexTermType)
-               && operator == o.operator
-               && Objects.equals(lower, o.lower)
-               && Objects.equals(upper, o.upper);
+        return Objects.equals(context.getColumnName(), o.context.getColumnName())
+                && validator.equals(o.validator)
+                && operation == o.operation
+                && Objects.equals(lower, o.lower)
+                && Objects.equals(upper, o.upper)
+                && exclusions.equals(o.exclusions);
     }
 
-    public static class IndexedExpression extends Expression
-    {
-        private final StorageAttachedIndex index;
-
-        public IndexedExpression(StorageAttachedIndex index)
-        {
-            super(index.termType());
-            this.index = index;
-        }
-
-        @Override
-        public boolean isNotIndexed()
-        {
-            return false;
-        }
-
-        @Override
-        public StorageAttachedIndex getIndex()
-        {
-            return index;
-        }
-
-        @Override
-        boolean hasAnalyzer()
-        {
-            return index.hasAnalyzer();
-        }
-
-        @Override
-        AbstractAnalyzer getAnalyzer()
-        {
-            return index.analyzer();
-        }
-    }
-
-    public static class UnindexedExpression extends Expression
+    /**
+     * Returns an expression that matches keys not matched by this expression.
+     */
+    public Expression negated()
     {
-        private UnindexedExpression(IndexTermType indexTermType)
-        {
-            super(indexTermType);
-        }
+        Expression result = new Expression(context);
+        result.lower = lower;
+        result.upper = upper;
 
-        @Override
-        public boolean isNotIndexed()
-        {
-            return true;
-        }
-
-        @Override
-        public StorageAttachedIndex getIndex()
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        boolean hasAnalyzer()
+        switch (operation)
         {
-            return false;
-        }
-
-        @Override
-        AbstractAnalyzer getAnalyzer()
-        {
-            throw new UnsupportedOperationException();
+            case NOT_EQ:
+                result.operation = Op.EQ;
+                break;
+            case NOT_CONTAINS_KEY:
+                result.operation = Op.CONTAINS_KEY;
+                break;
+            case NOT_CONTAINS_VALUE:
+                result.operation = Op.CONTAINS_VALUE;
+                break;
+            default:
+                throw new UnsupportedOperationException(String.format("Negation of operator %s not supported", operation));
         }
+        return result;
     }
 
     /**
@@ -456,10 +610,17 @@ public static class Value
         public final ByteBuffer raw;
         public final ByteBuffer encoded;
 
-        public Value(ByteBuffer value, IndexTermType indexTermType)
+        /**
+         * The native representation of our vector indexes is float[], so we cache that here as well
+         * to avoid repeated expensive conversions.  Always null for non-vector types.
+         */
+        public final float[] vector;
+
+        public Value(ByteBuffer value, AbstractType<?> type)
         {
             this.raw = value;
-            this.encoded = indexTermType.asIndexBytes(value);
+            this.encoded = TypeUtil.asIndexBytes(value, type);
+            this.vector = type.isVector() ? TypeUtil.decomposeVector(type, raw) : null;
         }
 
         @Override
@@ -487,9 +648,9 @@ public static class Bound
         public final Value value;
         public final boolean inclusive;
 
-        public Bound(ByteBuffer value, IndexTermType indexTermType, boolean inclusive)
+        public Bound(ByteBuffer value, AbstractType<?> type, boolean inclusive)
         {
-            this.value = new Value(value, indexTermType);
+            this.value = new Value(value, type);
             this.inclusive = inclusive;
         }
 
diff --git a/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java b/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java
index 4107fad2d271..84c6d351f95d 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java
@@ -19,42 +19,44 @@
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.ListIterator;
+import java.util.Set;
 
 import com.google.common.collect.ListMultimap;
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.ColumnMetadata.Kind;
 import org.apache.cassandra.utils.FBUtilities;
 
-import static org.apache.cassandra.index.sai.plan.Operation.BooleanOperator;
+import static org.apache.cassandra.index.sai.plan.Operation.OperationType;
 
 /**
  * Tree-like structure to filter base table data using indexed expressions and non-user-defined filters.
- * <p>
+ *
  * This is needed because:
  * 1. SAI doesn't index tombstones, base data may have been shadowed.
- * 2. Replica filter protecting may fetch data that doesn't match index expressions.
+ * 2. SAI indexes partition offset, not all rows in partition match index condition.
+ * 3. Replica filter protecting may fetch data that doesn't match index expressions.
  */
 public class FilterTree
 {
-    protected final BooleanOperator baseOperator;
+    protected final OperationType op;
     protected final ListMultimap<ColumnMetadata, Expression> expressions;
     protected final List<FilterTree> children = new ArrayList<>();
-    private final boolean isStrict;
-    private final QueryContext context;
 
-    FilterTree(BooleanOperator baseOperator, ListMultimap<ColumnMetadata, Expression> expressions, boolean isStrict, QueryContext context)
+    FilterTree(OperationType operation,
+               ListMultimap<ColumnMetadata, Expression> expressions)
     {
-        this.baseOperator = baseOperator;
+        this.op = operation;
         this.expressions = expressions;
-        this.isStrict = isStrict;
-        this.context = context;
     }
 
     void addChild(FilterTree child)
@@ -62,101 +64,91 @@ void addChild(FilterTree child)
         children.add(child);
     }
 
-    /**
-     * @return true if this node of the tree or any of its children filter a non-static column
-     */
-    public boolean restrictsNonStaticRow()
+    public boolean isSatisfiedBy(DecoratedKey key, Unfiltered currentCluster, Row staticRow)
     {
-        for (ColumnMetadata column : expressions.keySet())
-            if (!column.isStatic())
-                return true;
-
-        for (FilterTree child : children)
-            if (child.restrictsNonStaticRow())
-                return true;
-
-        return false;
-    }
+        boolean result = localSatisfiedBy(key, currentCluster, staticRow);
 
-    public boolean isSatisfiedBy(DecoratedKey key, Row row, Row staticRow)
-    {
-        boolean result = localSatisfiedBy(key, row, staticRow);
+        if (shouldReturnNow(result))
+            return result;
 
         for (FilterTree child : children)
-            result = baseOperator.apply(result, child.isSatisfiedBy(key, row, staticRow));
+        {
+            result = op.apply(result, child.isSatisfiedBy(key, currentCluster, staticRow));
+            if (shouldReturnNow(result))
+                return result;
+        }
 
         return result;
     }
 
-    private boolean localSatisfiedBy(DecoratedKey key, Row row, Row staticRow)
+    private boolean localSatisfiedBy(DecoratedKey key, Unfiltered currentCluster, Row staticRow)
     {
-        if (row == null)
+        if (currentCluster == null || !currentCluster.isRow())
             return false;
 
         final long now = FBUtilities.nowInSeconds();
-        // Downgrade AND to OR unless the coordinator indicates strict filtering is safe or all matches are repaired:
-        BooleanOperator localOperator = (isStrict || !context.hasUnrepairedMatches) ? baseOperator : BooleanOperator.OR;
-        boolean result = localOperator == BooleanOperator.AND;
+        boolean result = op == OperationType.AND;
 
         Iterator<ColumnMetadata> columnIterator = expressions.keySet().iterator();
-        while (columnIterator.hasNext())
+        while(columnIterator.hasNext())
         {
             ColumnMetadata column = columnIterator.next();
-            Row localRow = column.kind == Kind.STATIC ? staticRow : row;
+            Row row = column.kind == Kind.STATIC ? staticRow : (Row)currentCluster;
 
-            // If there is a column with multiple expressions that can mean an OR, or (in the case of map
+            // If there is a column with multiple expressions that can mean an OR or (in the case of map
             // collections) it can mean different map indexes.
             List<Expression> filters = expressions.get(column);
 
             // We do a reverse iteration over the filters because NOT_EQ operations will be at the end
-            // of the filter list, and we want to check them first.
+            // of the filter list and we want to check them first.
             ListIterator<Expression> filterIterator = filters.listIterator(filters.size());
-            while (filterIterator.hasPrevious())
+            while(filterIterator.hasPrevious())
             {
                 Expression filter = filterIterator.previous();
 
-                if (filter.getIndexTermType().isNonFrozenCollection())
+                if (TypeUtil.isNonFrozenCollection(column.type))
                 {
-                    Iterator<ByteBuffer> valueIterator = filter.getIndexTermType().valuesOf(localRow, now);
-                    result = localOperator.apply(result, collectionMatch(valueIterator, filter));
+                    Iterator<ByteBuffer> valueIterator = filter.context.getValuesOf(row, now);
+                    result = op.apply(result, filter.isSatisfiedBy(valueIterator));
                 }
                 else
                 {
-                    ByteBuffer value = filter.getIndexTermType().valueOf(key, localRow, now);
-                    result = localOperator.apply(result, singletonMatch(value, filter));
+                    ByteBuffer value = filter.context.getValueOf(key, row, now);
+                    result = op.apply(result, filter.isSatisfiedBy(value));
                 }
 
-                // If the operation is an AND then exit early if we get a single false
-                if ((localOperator == BooleanOperator.AND) && !result)
-                    return false;
-
-                // If the operation is an OR then exit early if we get a single true
-                if (localOperator == BooleanOperator.OR && result)
-                    return true;
+                if (shouldReturnNow(result))
+                    return result;
             }
         }
         return result;
     }
 
-    private boolean singletonMatch(ByteBuffer value, Expression filter)
-    {
-        return value != null && filter.isSatisfiedBy(value);
+    /**
+     * When evaluating an AND expression, if the current result is false, we can return immediately.
+     * When evaluating an OR expression, if the current result is true, we can return immediately.
+     * @param result the current result
+     * @return true if it is valid to return the current result
+     */
+    private boolean shouldReturnNow(boolean result) {
+        return (op == OperationType.AND && !result) || (op == OperationType.OR && result);
     }
 
-    private boolean collectionMatch(Iterator<ByteBuffer> valueIterator, Expression filter)
+    /**
+     * @return the number of unique SSTable indexes that are referenced by the expressions in this filter tree.
+     */
+    public int numSSTableIndexes()
     {
-        if (valueIterator == null)
-            return false;
-
-        while (valueIterator.hasNext())
-        {
-            ByteBuffer value = valueIterator.next();
-            if (value == null)
-                continue;
+        Set<SSTableIndex> referencedIndexes = new HashSet<>();
+        sstableIndexes(referencedIndexes);
+        return referencedIndexes.size();
+    }
 
-            if (filter.isSatisfiedBy(value))
-                return true;
-        }
-        return false;
+    private void sstableIndexes(Set<SSTableIndex> indexes)
+    {
+        for (Expression expression : expressions.values())
+            indexes.addAll(expression.context.getView().getIndexes());
+        for (FilterTree child : children)
+            child.sstableIndexes(indexes);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/plan/Operation.java b/src/java/org/apache/cassandra/index/sai/plan/Operation.java
index c7f313863ceb..ef8583e80852 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/Operation.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/Operation.java
@@ -21,9 +21,9 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
-import java.util.function.BiFunction;
-import java.util.stream.Collectors;
+import java.util.Map;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ArrayListMultimap;
@@ -31,45 +31,48 @@
 import com.google.common.collect.ListMultimap;
 
 import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.utils.TreeFormatter;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.serializers.ListSerializer;
 
 public class Operation
 {
-    public enum BooleanOperator
+    public enum OperationType
     {
-        AND((a, b) -> a & b),
-        OR((a, b) -> a | b);
-
-        private final BiFunction<Boolean, Boolean, Boolean> func;
-
-        BooleanOperator(BiFunction<Boolean, Boolean, Boolean> func)
-        {
-            this.func = func;
-        }
+        AND, OR;
 
         public boolean apply(boolean a, boolean b)
         {
-            return func.apply(a, b);
+            switch (this)
+            {
+                case OR:
+                    return a | b;
+
+                case AND:
+                    return a & b;
+
+                default:
+                    throw new AssertionError();
+            }
         }
     }
 
     @VisibleForTesting
-    protected static ListMultimap<ColumnMetadata, Expression> buildIndexExpressions(QueryController queryController,
-                                                                                    List<RowFilter.Expression> expressions)
+    protected static ListMultimap<ColumnMetadata, Expression> analyzeGroup(QueryController controller,
+                                                                           OperationType op,
+                                                                           List<RowFilter.Expression> expressions)
     {
         ListMultimap<ColumnMetadata, Expression> analyzed = ArrayListMultimap.create();
+        Map<ColumnMetadata, Boolean> columnIsMultiExpression = new HashMap<>();
 
-        // sort all the expressions in the operation by name and priority of the logical operator
+        // sort all of the expressions in the operation by name and priority of the logical operator
         // this gives us an efficient way to handle inequality and combining into ranges without extra processing
         // and converting expressions from one type to another.
         expressions.sort((a, b) -> {
@@ -77,67 +80,76 @@ protected static ListMultimap<ColumnMetadata, Expression> buildIndexExpressions(
             return cmp == 0 ? -Integer.compare(getPriority(a.operator()), getPriority(b.operator())) : cmp;
         });
 
-        for (final RowFilter.Expression expression : expressions)
+        for (final RowFilter.Expression e : expressions)
         {
-            if (Expression.supportsOperator(expression.operator()))
-            {
-                StorageAttachedIndex index = queryController.indexFor(expression);
-
-                List<Expression> perColumn = analyzed.get(expression.column());
-
-                if (index == null)
-                    buildUnindexedExpression(queryController, expression, perColumn);
-                else
-                    buildIndexedExpression(index, expression, perColumn);
-            }
-        }
-
-        return analyzed;
-    }
-
-    private static void buildUnindexedExpression(QueryController queryController,
-                                                 RowFilter.Expression expression,
-                                                 List<Expression> perColumn)
-    {
-        IndexTermType indexTermType = IndexTermType.create(expression.column(),
-                                                           queryController.metadata().partitionKeyColumns(),
-                                                           determineIndexTargetType(expression));
-        if (indexTermType.isMultiExpression(expression))
-        {
-            perColumn.add(Expression.create(indexTermType).add(expression.operator(), expression.getIndexValue().duplicate()));
-        }
-        else
-        {
-            Expression range;
-            if (perColumn.size() == 0)
-            {
-                range = Expression.create(indexTermType);
-                perColumn.add(range);
-            }
-            else
-            {
-                range = Iterables.getLast(perColumn);
-            }
-            range.add(expression.operator(), expression.getIndexValue().duplicate());
-        }
-    }
+            IndexContext indexContext = controller.getContext(e);
+            List<Expression> perColumn = analyzed.get(e.column());
 
-    private static void buildIndexedExpression(StorageAttachedIndex index, RowFilter.Expression expression, List<Expression> perColumn)
-    {
-        if (index.hasAnalyzer())
-        {
-            AbstractAnalyzer analyzer = index.analyzer();
+            AbstractAnalyzer.AnalyzerFactory analyzerFactory = indexContext.getQueryAnalyzerFactory();
+            AbstractAnalyzer analyzer = analyzerFactory.create();
             try
             {
-                analyzer.reset(expression.getIndexValue().duplicate());
+                analyzer.reset(e.getIndexValue());
+
+                // EQ/LIKE_*/NOT_EQ can have multiple expressions e.g. text = "Hello World",
+                // becomes text = "Hello" AND text = "World" because "space" is always interpreted as a split point (by analyzer),
+                // CONTAINS/CONTAINS_KEY are always treated as multiple expressions since they currently only targetting
+                // collections, NOT_EQ is made an independent expression only in case of pre-existing multiple EQ expressions, or
+                // if there is no EQ operations and NOT_EQ is met or a single NOT_EQ expression present,
+                // in such case we know exactly that there would be no more EQ/RANGE expressions for given column
+                // since NOT_EQ has the lowest priority.
+                boolean isMultiExpression = columnIsMultiExpression.getOrDefault(e.column(), Boolean.FALSE);
+                switch (e.operator())
+                {
+                    // case BM25: leave it at the default of `false`
+                    case EQ:
+                        // EQ operator will always be a multiple expression because it is being used by map entries
+                        isMultiExpression = indexContext.isNonFrozenCollection();
 
-                if (index.termType().isMultiExpression(expression))
+                        // EQ wil behave like ANALYZER_MATCHES for analyzed columns if the analyzer supports EQ queries
+                        isMultiExpression |= indexContext.isAnalyzed() && analyzerFactory.supportsEquals();
+                        break;
+                    case CONTAINS:
+                    case CONTAINS_KEY:
+                    case NOT_CONTAINS:
+                    case NOT_CONTAINS_KEY:
+                    case LIKE_PREFIX:
+                    case LIKE_MATCHES:
+                    case ANALYZER_MATCHES:
+                        isMultiExpression = true;
+                        break;
+                    case NEQ:
+                        // NEQ operator will always be a multiple expression if it is the only operator
+                        // (e.g. multiple NEQ expressions)
+                        isMultiExpression = isMultiExpression || perColumn.isEmpty();
+                        break;
+                }
+                columnIsMultiExpression.put(e.column(), isMultiExpression);
+
+                if (isMultiExpression)
                 {
-                    while (analyzer.hasNext())
+                    if (!analyzer.hasNext())
                     {
-                        final ByteBuffer token = analyzer.next();
-                        perColumn.add(Expression.create(index).add(expression.operator(), token.duplicate()));
+                        perColumn.add(new Expression(indexContext).add(e.operator(), ByteBuffer.allocate(0)));
                     }
+                    else
+                    {
+                        // The hasNext implementation has a side effect, so we need to call next before calling hasNext
+                        do
+                        {
+                            final ByteBuffer token = analyzer.next();
+                            perColumn.add(new Expression(indexContext).add(e.operator(), token.duplicate()));
+                        }
+                        while (analyzer.hasNext());
+                    }
+                }
+                else if (e instanceof RowFilter.GeoDistanceExpression)
+                {
+                    var distance = ((RowFilter.GeoDistanceExpression) e);
+                    var expression = new Expression(indexContext)
+                                     .add(distance.getDistanceOperator(), distance.getDistance().duplicate())
+                                     .add(Operator.BOUNDED_ANN, e.getIndexValue().duplicate());
+                    perColumn.add(expression);
                 }
                 else
                 // "range" or not-equals operator, combines both bounds together into the single expression,
@@ -145,9 +157,9 @@ private static void buildIndexedExpression(StorageAttachedIndex index, RowFilter
                 // not-equals is combined with the range iff operator is AND.
                 {
                     Expression range;
-                    if (perColumn.size() == 0)
+                    if (perColumn.size() == 0 || op != OperationType.AND || e instanceof RowFilter.MapComparisonExpression)
                     {
-                        range = Expression.create(index);
+                        range = new Expression(indexContext);
                         perColumn.add(range);
                     }
                     else
@@ -155,17 +167,40 @@ private static void buildIndexedExpression(StorageAttachedIndex index, RowFilter
                         range = Iterables.getLast(perColumn);
                     }
 
-                    if (index.termType().isLiteral())
+                    if (!TypeUtil.isLiteral(indexContext.getValidator()))
                     {
-                        while (analyzer.hasNext())
-                        {
-                            ByteBuffer term = analyzer.next();
-                            range.add(expression.operator(), term.duplicate());
+                        range.add(e.operator(), e.getIndexValue().duplicate());
+                    }
+                    else if (e instanceof RowFilter.MapComparisonExpression)
+                    {
+                        var map = (RowFilter.MapComparisonExpression) e;
+                        var operator = map.operator();
+                        switch (operator) {
+                            case EQ:
+                            case NEQ:
+                                range.add(operator, map.getIndexValue().duplicate());
+                                break;
+                            case GT:
+                            case GTE:
+                                range.add(operator, map.getLowerBound().duplicate());
+                                range.add(Operator.LTE, map.getUpperBound().duplicate());
+                                break;
+                            case LT:
+                            case LTE:
+                                range.add(Operator.GTE, map.getLowerBound().duplicate());
+                                range.add(operator, map.getUpperBound().duplicate());
+                                break;
+                            default:
+                                throw new InvalidRequestException("Unexpected operator: " + operator);
                         }
                     }
                     else
                     {
-                        range.add(expression.operator(), expression.getIndexValue().duplicate());
+                        while (analyzer.hasNext())
+                        {
+                            ByteBuffer term = analyzer.next();
+                            range.add(e.operator(), term.duplicate());
+                        }
                     }
                 }
             }
@@ -174,123 +209,52 @@ private static void buildIndexedExpression(StorageAttachedIndex index, RowFilter
                 analyzer.end();
             }
         }
-        else
-        {
-            if (index.termType().isMultiExpression(expression))
-            {
-                perColumn.add(Expression.create(index).add(expression.operator(), expression.getIndexValue().duplicate()));
-            }
-            else
-            {
-                Expression range;
-                if (perColumn.size() == 0)
-                {
-                    range = Expression.create(index);
-                    perColumn.add(range);
-                }
-                else
-                {
-                    range = Iterables.getLast(perColumn);
-                }
-                range.add(expression.operator(), expression.getIndexValue().duplicate());
-            }
-        }
-    }
 
-    /**
-     * Determines the {@link IndexTarget.Type} for the expression. In this case we are only interested in map types and
-     * the operator being used in the expression.
-     */
-    private static IndexTarget.Type determineIndexTargetType(RowFilter.Expression expression)
-    {
-        AbstractType<?> type  = expression.column().type;
-        IndexTarget.Type indexTargetType = IndexTarget.Type.SIMPLE;
-        if (type.isCollection() && type.isMultiCell())
-        {
-            CollectionType<?> collection = ((CollectionType<?>) type);
-            if (collection.kind == CollectionType.Kind.MAP)
-            {
-                switch (expression.operator())
-                {
-                    case EQ:
-                        indexTargetType = IndexTarget.Type.KEYS_AND_VALUES;
-                        break;
-                    case CONTAINS:
-                        indexTargetType = IndexTarget.Type.VALUES;
-                        break;
-                    case CONTAINS_KEY:
-                        indexTargetType = IndexTarget.Type.KEYS;
-                        break;
-                    default:
-                        throw new InvalidRequestException("Invalid operator");
-                }
-            }
-        }
-        return indexTargetType;
+        return analyzed;
     }
 
-    private static int getPriority(Operator op)
+    private static int getPriority(org.apache.cassandra.cql3.Operator op)
     {
         switch (op)
         {
             case EQ:
+                return 7;
+
             case CONTAINS:
             case CONTAINS_KEY:
+                return 6;
+
+            case LIKE_PREFIX:
+            case LIKE_MATCHES:
                 return 5;
 
             case GTE:
             case GT:
-                return 3;
+                return 4;
 
             case LTE:
             case LT:
+                return 3;
+
+            case NOT_CONTAINS:
+            case NOT_CONTAINS_KEY:
                 return 2;
 
+            case NEQ:
+                return 1;
+
             default:
                 return 0;
         }
     }
 
-    /**
-     * Converts expressions into filter tree for query.
-     *
-     * @return a KeyRangeIterator over the index query results
-     */
-    static KeyRangeIterator buildIterator(QueryController controller)
-    {
-        var orderings = controller.indexFilter().getExpressions()
-                                  .stream().filter(e -> e.operator() == Operator.ANN).collect(Collectors.toList());
-        assert orderings.size() <= 1;
-        if (controller.indexFilter().getExpressions().size() == 1 && orderings.size() == 1)
-            // If we only have one expression, we just use the ANN index to order and limit.
-            return controller.getTopKRows(orderings.get(0));
-        var iterator = Node.buildTree(controller.indexFilter()).analyzeTree(controller).rangeIterator(controller);
-        if (orderings.isEmpty())
-            return iterator;
-        return controller.getTopKRows(iterator, orderings.get(0));
-    }
-
-    /**
-     * Converts expressions into filter tree (which is currently just a single AND).
-     * <p>
-     * Filter tree allows us to do a couple of important optimizations
-     * namely, group flattening for AND operations (query rewrite), expression bounds checks,
-     * "satisfies by" checks for resulting rows with an early exit.
-     *
-     * @return root of the filter tree.
-     */
-    static FilterTree buildFilter(QueryController controller, boolean strict)
-    {
-        return Node.buildTree(controller.indexFilter()).buildFilter(controller, strict);
-    }
-
-    static abstract class Node
+    public static abstract class Node
     {
         ListMultimap<ColumnMetadata, Expression> expressionMap;
 
         boolean canFilter()
         {
-            return (expressionMap != null && !expressionMap.isEmpty()) || !children().isEmpty();
+            return (expressionMap != null && !expressionMap.isEmpty()) || !children().isEmpty() ;
         }
 
         List<Node> children()
@@ -308,61 +272,95 @@ RowFilter.Expression expression()
             throw new UnsupportedOperationException();
         }
 
-        abstract void analyze(List<RowFilter.Expression> expressionList, QueryController controller);
+        /**
+         * Analyze the tree, potentially flattening it and storing the result in expressionMap.
+         */
+        abstract void analyze(QueryController controller);
 
-        abstract FilterTree filterTree(boolean strict, QueryContext context);
+        abstract FilterTree filterTree();
 
-        abstract KeyRangeIterator rangeIterator(QueryController controller);
+        abstract Plan.KeysIteration plan(QueryController controller);
 
-        static Node buildTree(RowFilter filterOperation)
+        static Node buildTree(QueryController controller, List<RowFilter.Expression> expressions, List<RowFilter.FilterElement> children, boolean isDisjunction)
         {
-            OperatorNode node = new AndNode();
-            for (RowFilter.Expression expression : filterOperation.getExpressions())
-                node.add(buildExpression(expression));
+            OperatorNode node = isDisjunction ? new OrNode() : new AndNode();
+            for (RowFilter.Expression expression : expressions)
+                node.add(buildExpression(controller, expression, isDisjunction));
+            for (RowFilter.FilterElement child : children)
+                node.add(buildTree(controller, child));
             return node;
         }
 
-        static Node buildExpression(RowFilter.Expression expression)
+        static Node buildTree(QueryController controller, RowFilter.FilterElement filterOperation)
         {
-            return new ExpressionNode(expression);
+            return buildTree(controller, filterOperation.expressions(), filterOperation.children(), filterOperation.isDisjunction());
+        }
+
+        static Node buildExpression(QueryController controller, RowFilter.Expression expression, boolean isDisjunction)
+        {
+            if (expression.operator() == Operator.IN)
+            {
+                OperatorNode node = new OrNode();
+                int size = ListSerializer.readCollectionSize(expression.getIndexValue(), ByteBufferAccessor.instance);
+                int offset = ListSerializer.sizeOfCollectionSize();
+                for (int index = 0; index < size; index++)
+                {
+                    node.add(new ExpressionNode(new RowFilter.SimpleExpression(expression.column(),
+                                                                               Operator.EQ,
+                                                                               ListSerializer.readValue(expression.getIndexValue(),
+                                                                                                        ByteBufferAccessor.instance,
+                                                                                                        offset),
+                                                                               expression.indexAnalyzer(),
+                                                                               expression.queryAnalyzer(),
+                                                                               expression.annOptions())));
+                    offset += TypeSizes.INT_SIZE + ByteBufferAccessor.instance.getInt(expression.getIndexValue(), offset);
+                }
+                if (node.children().size() == 1)
+                    return node.children().get(0);
+                if (node.children().isEmpty())
+                    return new EmptyNode();
+                return node;
+            }
+            else if (isDisjunction && (expression.operator() == Operator.ANALYZER_MATCHES ||
+                                       expression.operator() == Operator.EQ && controller.getContext(expression).isAnalyzed()))
+            {
+                // In case of having a tokenizing query_analyzer (such as NGram) with OR, we need to split the
+                // expression into multiple expressions and intersect them.
+                // The additional node in case of no tokenization will be taken care of in Plan.Factory#intersection()
+                OperatorNode node = new AndNode();
+                node.add(new ExpressionNode(expression));
+                return node;
+            }
+            else
+                return new ExpressionNode(expression);
         }
 
         Node analyzeTree(QueryController controller)
         {
-            List<RowFilter.Expression> expressionList = new ArrayList<>();
-            doTreeAnalysis(this, expressionList, controller);
-            if (!expressionList.isEmpty())
-                this.analyze(expressionList, controller);
+            analyze(controller);
             return this;
         }
 
-        void doTreeAnalysis(Node node, List<RowFilter.Expression> expressions, QueryController controller)
+        @VisibleForTesting
+        FilterTree buildFilter(QueryController controller)
         {
-            if (node.children().isEmpty())
-                expressions.add(node.expression());
-            else
-            {
-                List<RowFilter.Expression> expressionList = new ArrayList<>();
-                for (Node child : node.children())
-                    doTreeAnalysis(child, expressionList, controller);
-                node.analyze(expressionList, controller);
-            }
+            analyze(controller);
+            return filterTree();
         }
 
-        FilterTree buildFilter(QueryController controller, boolean isStrict)
+        /**
+         * Formats the whole operation tree as a pretty tree.
+         */
+        public final String toStringRecursive()
         {
-            analyzeTree(controller);
-            FilterTree tree = filterTree(isStrict, controller.queryContext);
-            for (Node child : children())
-                if (child.canFilter())
-                    tree.addChild(child.buildFilter(controller, isStrict));
-            return tree;
+            TreeFormatter<Node> formatter = new TreeFormatter<>(Node::toString, Node::children);
+            return formatter.format(this);
         }
     }
 
     static abstract class OperatorNode extends Node
     {
-        final List<Node> children = new ArrayList<>();
+        List<Node> children = new ArrayList<>();
 
         @Override
         public List<Node> children()
@@ -375,52 +373,107 @@ public void add(Node child)
         {
             children.add(child);
         }
-    }
 
-    static class AndNode extends OperatorNode
-    {
+        abstract protected OperationType operationType();
+        abstract protected Plan.Builder planBuilder(QueryController controller);
+
+        // expression list is the children that are leaf nodes... we could figure that out here...
         @Override
-        public void analyze(List<RowFilter.Expression> expressionList, QueryController controller)
+        public void analyze(QueryController controller)
         {
-            expressionMap = buildIndexExpressions(controller, expressionList);
+            // This operation flattens the tree where possible and stores the result in expressionMap
+            List<RowFilter.Expression> expressionList = new ArrayList<>();
+            for (Node child : children)
+            {
+                if (child instanceof ExpressionNode)
+                    expressionList.add(child.expression());
+                else
+                    child.analyze(controller);
+            }
+            expressionMap = analyzeGroup(controller, operationType(), expressionList);
         }
 
         @Override
-        FilterTree filterTree(boolean isStrict, QueryContext context)
+        FilterTree filterTree()
         {
-            return new FilterTree(BooleanOperator.AND, expressionMap, isStrict, context);
+            assert expressionMap != null;
+            var tree = new FilterTree(operationType(), expressionMap);
+            for (Node child : children())
+                if (child.canFilter())
+                    tree.addChild(child.filterTree());
+            return tree;
         }
 
         @Override
-        KeyRangeIterator rangeIterator(QueryController controller)
+        Plan.KeysIteration plan(QueryController controller)
         {
-            KeyRangeIterator.Builder builder = controller.getIndexQueryResults(expressionMap.values());
+            var builder = planBuilder(controller);
+            if (!expressionMap.isEmpty())
+                controller.buildPlanForExpressions(builder, expressionMap.values());
             for (Node child : children)
-            {
-                boolean canFilter = child.canFilter();
-                if (canFilter)
-                    builder.add(child.rangeIterator(controller));
-            }
+                if (child.canFilter())
+                    builder.add(child.plan(controller));
             return builder.build();
         }
     }
 
-    static class ExpressionNode extends Node
+    public static class AndNode extends OperatorNode
     {
-        final RowFilter.Expression expression;
+        @Override
+        protected OperationType operationType()
+        {
+            return OperationType.AND;
+        }
 
         @Override
-        public void analyze(List<RowFilter.Expression> expressionList, QueryController controller)
+        protected Plan.Builder planBuilder(QueryController controller)
         {
-            expressionMap = buildIndexExpressions(controller, expressionList);
-            assert expressionMap.size() == 1 : "Expression nodes should only have a single expression!";
+            return controller.planFactory.intersectionBuilder();
         }
 
         @Override
-        FilterTree filterTree(boolean isStrict, QueryContext context)
+        public String toString()
         {
-            // There should only be one expression, so AND/OR would both work here. 
-            return new FilterTree(BooleanOperator.AND, expressionMap, isStrict, context);
+            return "AndNode";
+        }
+    }
+
+    public static class OrNode extends OperatorNode
+    {
+        @Override
+        protected OperationType operationType()
+        {
+            return OperationType.OR;
+        }
+
+        @Override
+        protected Plan.Builder planBuilder(QueryController controller)
+        {
+            return controller.planFactory.unionBuilder();
+        }
+
+        @Override
+        public String toString()
+        {
+            return "OrNode";
+        }
+    }
+
+    public static class ExpressionNode extends Node
+    {
+        RowFilter.Expression expression;
+
+        @Override
+        public void analyze(QueryController controller)
+        {
+            expressionMap = analyzeGroup(controller, OperationType.AND, Collections.singletonList(expression));
+        }
+
+        @Override
+        FilterTree filterTree()
+        {
+            assert expressionMap != null;
+            return new FilterTree(OperationType.AND, expressionMap);
         }
 
         public ExpressionNode(RowFilter.Expression expression)
@@ -435,11 +488,54 @@ public RowFilter.Expression expression()
         }
 
         @Override
-        KeyRangeIterator rangeIterator(QueryController controller)
+        Plan.KeysIteration plan(QueryController controller)
         {
             assert canFilter() : "Cannot process query with no expressions";
+            Plan.Builder builder = controller.planFactory.intersectionBuilder();
+            controller.buildPlanForExpressions(builder, expressionMap.values());
+            return builder.build();
+        }
 
-            return controller.getIndexQueryResults(expressionMap.values()).build();
+        @Override
+        public String toString()
+        {
+            return "ExpressionNode{expression=" + expression + '}';
         }
     }
+
+    public static class EmptyNode extends Node
+    {
+        // A FilterTree that filters out all rows
+        private static final FilterTree EMPTY_TREE = new FilterTree(OperationType.OR, ArrayListMultimap.create());
+
+        @Override
+        boolean canFilter()
+        {
+            return true;
+        }
+
+        @Override
+        void analyze(QueryController controller)
+        {
+        }
+
+        @Override
+        FilterTree filterTree()
+        {
+            return EMPTY_TREE;
+        }
+
+        @Override
+        Plan.KeysIteration plan(QueryController controller)
+        {
+            return controller.planFactory.nothing;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "EmptyNode";
+        }
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/index/sai/plan/Orderer.java b/src/java/org/apache/cassandra/index/sai/plan/Orderer.java
new file mode 100644
index 000000000000..7fd70502ce4f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/Orderer.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.vector.VectorCompression;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+
+/**
+ * An SAI Orderer represents an index based order by clause.
+ */
+public class Orderer
+{
+    // The list of operators that are valid for order by clauses.
+    static final EnumSet<Operator> ORDER_BY_OPERATORS = EnumSet.of(Operator.ANN,
+                                                                   Operator.BM25,
+                                                                   Operator.ORDER_BY_ASC,
+                                                                   Operator.ORDER_BY_DESC);
+
+    public final IndexContext context;
+    public final Operator operator;
+    public final ByteBuffer term;
+
+    // Vector search parameters
+    private float[] vector;
+    private final Integer rerankK;
+
+    // BM25 search parameter
+    private List<ByteBuffer> queryTerms;
+
+    /**
+     * Create an orderer for the given index context, operator, and term.
+     * @param context the index context, used to build the view of memtables and sstables for query execution.
+     * @param operator the operator for the order by clause.
+     * @param term the term to order by (not always relevant)
+     * @param rerankK optional rerank K parameter for ANN queries
+     */
+    public Orderer(IndexContext context, Operator operator, ByteBuffer term, @Nullable Integer rerankK)
+    {
+        this.context = context;
+        assert ORDER_BY_OPERATORS.contains(operator) : "Invalid operator for order by clause " + operator;
+        this.operator = operator;
+        this.rerankK = rerankK;
+        this.term = term;
+    }
+
+    public String getIndexName()
+    {
+        return context.getIndexName();
+    }
+
+    public boolean isAscending()
+    {
+        // Note: ANN is always descending.
+        return operator == Operator.ORDER_BY_ASC;
+    }
+
+    public Comparator<? super PrimaryKeyWithSortKey> getComparator()
+    {
+        // ANN/BM25's PrimaryKeyWithSortKey is always descending, so we use the natural order for the priority queue
+        return (isAscending() || isANN() || isBM25()) ? Comparator.naturalOrder() : Comparator.reverseOrder();
+    }
+
+    public boolean isLiteral()
+    {
+        return context.isLiteral();
+    }
+
+    public boolean isANN()
+    {
+        return operator == Operator.ANN;
+    }
+
+    /**
+     * Provide rerankK for ANN queries. Use the user provided rerankK if available, otherwise use the model's default
+     * based on the limit and compression type.
+     *
+     * @param limit the query limit or the proportional segment limit to use when calculating a reasonable rerankK
+     *              default value
+     * @param vc the compression type of the vectors in the index
+     * @return the rerankK value to use in ANN search
+     */
+    public int rerankKFor(int limit, VectorCompression vc)
+    {
+        assert isANN() : "rerankK is only valid for ANN queries";
+        return rerankK != null
+               ? rerankK
+               : context.getIndexWriterConfig().getSourceModel().rerankKFor(limit, vc);
+    }
+
+    public boolean isBM25()
+    {
+        return operator == Operator.BM25;
+    }
+
+    @Nullable
+    public static Orderer from(SecondaryIndexManager indexManager, RowFilter filter)
+    {
+        var expressions = filter.root().expressions().stream().filter(Orderer::isFilterExpressionOrderer).collect(Collectors.toList());
+        if (expressions.isEmpty())
+            return null;
+        var orderExpression = expressions.get(0);
+        var index = indexManager.getBestIndexFor(orderExpression, StorageAttachedIndex.class)
+                                .orElseThrow(() -> new IllegalStateException("No index found for order by clause"));
+
+        // Null if not specified explicitly in the CQL query.
+        Integer rerankK = filter.annOptions().rerankK;
+        return new Orderer(index.getIndexContext(), orderExpression.operator(), orderExpression.getIndexValue(), rerankK);
+    }
+
+    public static boolean isFilterExpressionOrderer(RowFilter.Expression expression)
+    {
+        return ORDER_BY_OPERATORS.contains(expression.operator());
+    }
+
+    @Override
+    public String toString()
+    {
+        String direction = isAscending() ? "ASC" : "DESC";
+        String rerankInfo = rerankK != null ? String.format(" (rerank_k=%d)", rerankK) : "";
+        if (isANN())
+            return context.getColumnName() + " ANN OF " + Arrays.toString(getVectorTerm()) + ' ' + direction + rerankInfo;
+        if (isBM25())
+            return context.getColumnName() + " BM25 OF " + TypeUtil.getString(term, context.getValidator()) + ' ' + direction;
+        return context.getColumnName() + ' ' + direction;
+    }
+
+    public float[] getVectorTerm()
+    {
+        if (vector == null)
+            vector = TypeUtil.decomposeVector(context.getValidator(), term);
+        return vector;
+    }
+
+    public List<ByteBuffer> getQueryTerms()
+    {
+        if (queryTerms != null)
+            return queryTerms;
+
+        var queryAnalyzer = context.getQueryAnalyzerFactory().create();
+        // Split query into terms
+        var uniqueTerms = new HashSet<ByteBuffer>();
+        queryAnalyzer.reset(term);
+        try
+        {
+            queryAnalyzer.forEachRemaining(uniqueTerms::add);
+        }
+        finally
+        {
+            queryAnalyzer.end();
+        }
+        queryTerms = new ArrayList<>(uniqueTerms);
+        return queryTerms;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/Plan.java b/src/java/org/apache/cassandra/index/sai/plan/Plan.java
new file mode 100644
index 000000000000..44f059c3aad6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/Plan.java
@@ -0,0 +1,2292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.*;
+import java.util.function.DoubleSupplier;
+import java.util.function.Function;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIntersectionIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TreeFormatter;
+import org.apache.cassandra.io.util.FileUtils;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.lang.Math.round;
+import static org.apache.cassandra.index.sai.plan.Plan.CostCoefficients.*;
+
+/**
+ * The common base class for query execution plan nodes.
+ * The top-level node is considered to be the execution plan of the query.
+ *
+ * <h1>Structure</h1>
+ * A query plan is an immutable tree constisting of nodes representing physical data operations,
+ * e.g. index scans, intersections, unions, filtering, limiting, sorting, etc.
+ * Nodes of type {@link KeysIteration} operate on streams of keys, and nodes of type {@link RowsIteration} operate
+ * on streams of rows. You should build a plan bottom-up by using static methods in {@link Plan.Factory}.
+ * Nodes don't have pointers to parent nodes on purpose – this way multiple plans can share subtrees.
+ *
+ * <h1>Cost estimation</h1>
+ * A plan can estimate its execution cost and result set size which is useful to select the best plan among
+ * the semantically equivalent candidate plans. Operations represented by nodes may be pipelined, so their actual
+ * runtime cost may depend on how many rows are read from the top level node. Upon construction, each plan node
+ * gets an {@link Access} object which describes the way how the node results are going to be used by the parent nodes:
+ * how many rows will be requested or what skip operations are going to be performed on the iterator.
+ * The access objects get propagated down the tree to the leaves. This way we get an accurate cost of execution
+ * at the leave nodes, taking into account any top-level limit or intersections.
+ * <p>
+ * Some nodes cannot be pipelined, e.g. nodes that represent sorting. To make cost estimation for such nodes possible,
+ * each node maintains an initial cost (initCost) of the operation - that is the cost of preparation before the first
+ * result row or key can be returned. Sorting nodes can have that cost very high.
+ *
+ * <h1>Optimization</h1>
+ * This class also offers a few methods for modifying the plans (e.g. removing nodes) and a method allowing
+ * to automatically improve the plan – see {@link #optimize()}. Whenever we talk about "modification" or "updates"
+ * we always mean constructing a new plan. All updates are non-destructive. Each node has a unique numeric
+ * identifier in the tree. Because a modification requires creating some new nodes, identifiers allow to find
+ * corresponding nodes in the modified plan, even if they addresses changed (they are different java objects).
+ *
+ * <h1>Execution</h1>
+ * The plan tree may store additional context information to be executable, i.e. to produce the iterator over the result
+ * keys or rows - see {@link KeysIteration#execute}. However, the purpose of the plan nodes is not to perform
+ * the actual computation of the result set. Instead, it should delegate the control to other modules responsible
+ * for data retrieval. The plan only sets up the execution, but must not contain the execution logic.
+ * For the sake of good testability, plan trees must be creatable, estimatable and optimizable also without
+ * creating any of the objects used by the execution engine.
+ *
+ * <h1>Example</h1>
+ * The CQL query
+ * <pre>
+ * SELECT * FROM table WHERE  a < 0.01 AND b < 0.2 LIMIT 10
+ * </pre>
+ *
+ * can be represented by the following query execution plan:
+ * <pre>
+ * Limit 10 (rows: 10.0, cost/row: 265.2, cost: 80.0..2732.2)
+ *  └─ Filter a < 0.2 AND b < 0.01 (sel: 1.000000000) (rows: 10.0, cost/row: 265.2, cost: 80.0..2732.2)
+ *      └─ Fetch (rows: 10.0, cost/row: 265.2, cost: 80.0..2732.2)
+ *          └─ Intersection (keys: 10.0, cost/key: 58.2, cost: 80.0..662.1)
+ *              ├─ NumericIndexScan of vector_b_idx using Expression{ ... } (sel: 0.010010000, step: 1.0) (keys: 50.2, cost/key: 1.0, cost: 40.0..90.2)
+ *              └─ NumericIndexScan of vector_a_idx using Expression{ ... } (sel: 0.199230000, step: 19.9) (keys: 50.2, cost/key: 10.6, cost: 40.0..571.9)
+ * </pre>
+ */
+@NotThreadSafe
+abstract public class Plan
+{
+    private static final Logger logger = LoggerFactory.getLogger(Plan.class);
+
+    @VisibleForTesting
+    static DoubleSupplier hitRateSupplier = () -> {
+        // cache hit rate with reasonable defaults if we have no data
+        double hitRate = ChunkCache.instance == null ? 1.0 : ChunkCache.instance.metrics.hitRate();
+        return Double.isFinite(hitRate) ? hitRate : 1.0;
+    };
+
+    /**
+     * Identifier of the plan tree node.
+     * Used to identify the nodes of the plan.
+     * Preserved during plan transformations.
+     * <p>
+     * Identifiers are more useful than object's identity (address) because plans can be transformed functionally
+     * and as the result of that process we may get new node objects.
+     * Identifiers allow us to match nodes in the transformed plan to the original.
+     */
+    final int id;
+
+    /**
+     * Reference to the factory gives access to common data shared among all nodes,
+     * e.g. total number of keys in the table and the cost parameters.
+     * It also allows to modify plan trees, e.g. create new nodes or recreate this node with different parameters.
+     */
+    final Factory factory;
+
+    /**
+     * Describes how this node is going to be used.
+     * Very likely affects the cost.
+     */
+    final Access access;
+
+    /**
+     * Lazily caches the estimated fraction of the table data that the result of this plan is expected to match.
+     */
+    private double selectivity = -1;
+
+
+    private Plan(Factory factory, int id, Access access)
+    {
+        this.id = id;
+        this.factory = factory;
+        this.access = access;
+    }
+
+    /**
+     * Returns the order of the keys / rows returned by this plan.
+     */
+    protected abstract @Nullable Orderer ordering();
+
+    /** selectivity comparisons to 0 will probably cause bugs, use this instead */
+    protected static boolean isEffectivelyZero(double a) {
+        assert a >= 0;
+        return a < 1e-9;
+    }
+
+    /** dividing by extremely tiny numbers can cause overflow so clamp the minimum to 1e-9 */
+    protected static double boundedSelectivity(double selectivity) {
+        assert 0 <= selectivity && selectivity <= 1.0;
+        return Math.max(1e-9, selectivity);
+    }
+
+    /**
+     * Returns a new list containing subplans of this node.
+     * The list can be later freely modified by the caller and does not affect the original plan.
+     * <p>
+     * Performance warning: This allocates a fresh list on the heap.
+     * If you only want to iterate the subplan nodes, it is recommended to use {@link #forEachSubplan(Function)}
+     * or {@link #withUpdatedSubplans(Function)} which offer better performance and less GC pressure.
+     */
+    final List<Plan> subplans()
+    {
+        List<Plan> result = new ArrayList<>();
+        forEachSubplan(subplan -> {
+            result.add(subplan);
+            return ControlFlow.Continue;
+        });
+        return result;
+    }
+
+    /**
+     * Returns a new list of nodes of given type.
+     * The tree is traversed in depth-first order.
+     * This node is included in the search.
+     * The list can be later freely modified by the caller and does not affect the original plan.
+     * <p>
+     * Performance warning: This allocates a fresh list on the heap.
+     * If you only want to iterate the subplan nodes, it is recommended to use {@link #forEachSubplan(Function)}
+     * which should offer better performance and less GC pressure.
+     */
+    @SuppressWarnings("unchecked")
+    final <T extends Plan> List<T> nodesOfType(Class<T> nodeType)
+    {
+        List<T> result = new ArrayList<>();
+        forEach(node -> {
+            if (nodeType.isAssignableFrom(node.getClass()))
+                result.add((T) node);
+            return ControlFlow.Continue;
+        });
+        return result;
+    }
+
+    /**
+     * Returns the first node of the given type.
+     * Searches the tree in depth-first order.
+     * This node is included in the search.
+     * If node of given type is not found, returns null.
+     */
+    @SuppressWarnings("unchecked")
+    final <T extends Plan> @Nullable T firstNodeOfType(Class<T> nodeType)
+    {
+        Plan[] result = new Plan[] { null };
+        forEach(node -> {
+            if (nodeType.isAssignableFrom(node.getClass()))
+            {
+                result[0] = node;
+                return ControlFlow.Break;
+            }
+            return ControlFlow.Continue;
+        });
+        return (T) result[0];
+    }
+
+    /**
+     * Calls a function recursively for each node of given type in the tree.
+     * If the function returns {@link ControlFlow#Break} then the traversal is aborted.
+     * @return {@link ControlFlow#Continue} if traversal hasn't been aborted, {@link ControlFlow#Break} otherwise.
+     */
+    final ControlFlow forEach(Function<Plan, ControlFlow> function)
+    {
+        return (function.apply(this) == ControlFlow.Continue)
+               ? forEachSubplan(subplan -> subplan.forEach(function))
+               : ControlFlow.Break;
+    }
+
+    /**
+     * Calls a function for each child node of this plan.
+     * The function should return {@link ControlFlow#Continue} to indicate the iteration should be continued
+     * and {@link ControlFlow#Break} to abort it.
+     *
+     * @return the value returned by the last invocation of the function
+     */
+    abstract ControlFlow forEachSubplan(Function<Plan, ControlFlow> function);
+
+
+    /** Controls tree traversals, see {@link #forEach(Function)} and {@link #forEachSubplan(Function)}  */
+    enum ControlFlow { Continue, Break }
+
+    /**
+     * Runs the updater function on each subplan and if the updater returns a new subplan, then reconstructs this
+     * plan from the modified subplans.
+     * <p>
+     * Accepting a list of sub-plans would be a valid alternative design of this API,
+     * but that would require constructing a list on the heap by the caller for each updated node,
+     * and that would be potentially wasteful as most of the node types have at most one subplan and don't use
+     * lists internally.
+     *
+     * @param updater a function to be called on each subplan; if no update is needed, should return the argument
+     * @return a new plan if any of the subplans has been replaced, this otherwise
+     */
+    protected abstract Plan withUpdatedSubplans(Function<Plan, Plan> updater);
+
+    /**
+     * Returns an object describing detailed cost information about running this plan.
+     * The actual type of the Cost depends in practice on the type of the result set returned by the node.
+     * The results of this method are supposed to be cached. The method is idempotent.
+     * The cost usually depends on the Access value.
+     */
+    protected abstract Cost cost();
+
+    /**
+     * Estimates the probability of a random key or row of the table to be included in the result set
+     * if the result was iterated fully with no skipping and if it did not have any limits.
+     * This property is independent of the way how result set is used.
+     */
+    protected abstract double estimateSelectivity();
+
+    /**
+     * Formats the whole plan as a pretty tree
+     */
+    public final String toStringRecursive()
+    {
+        TreeFormatter<Plan> formatter = new TreeFormatter<>(Plan::toString, Plan::subplans);
+        return formatter.format(this);
+    }
+
+    /**
+     * Returns the string representation of this node only
+     */
+    public final String toString()
+    {
+        String title = title();
+        String description = description();
+        return (title.isEmpty())
+               ? String.format("%s (%s)\n%s", getClass().getSimpleName(), cost(), description).stripTrailing()
+               : String.format("%s %s (%s)\n%s", getClass().getSimpleName(), title, cost(), description).stripTrailing();
+    }
+
+    /**
+     * Returns additional information specific to the node displayed in the first line.
+     * The information is included in the output of {@link #toString()} and {@link #toStringRecursive()}.
+     * It is up to subclasses to implement it.
+     */
+    protected String title()
+    {
+        return "";
+    }
+
+    /**
+     * Returns additional information specific to the node, displayed below the title.
+     * The information is included in the output of {@link #toString()} and {@link #toStringRecursive()}.
+     * It is up to subclasses to implement it.
+     */
+    protected String description()
+    {
+        return "";
+    }
+
+    /**
+     * Returns the index context if the plan node uses one.
+     * Need to be overridden by nodes that use an index.
+     * Non-recursive.
+     */
+    protected @Nullable IndexContext getIndexContext()
+    {
+        // By default, a node does not contain an index.
+        return null;
+    }
+
+    /**
+     * Returns an optimized plan.
+     * <p>
+     * The current optimization algorithm repeatedly cuts down one leaf of the plan tree
+     * and recomputes the nodes above it. Then it returns the best plan from candidates obtained that way.
+     * The expected running time is proportional to the height of the plan tree multiplied by the number of the leaves.
+     */
+    public final Plan optimize()
+    {
+        if (logger.isTraceEnabled())
+            logger.trace("Optimizing plan:\n{}", this.toStringRecursive());
+
+        Plan bestPlanSoFar = this;
+        List<Leaf> leaves = nodesOfType(Leaf.class);
+
+        // Remove leaves one by one, starting from the ones with the worst selectivity
+        leaves.sort(Comparator.comparingDouble(Plan::selectivity).reversed());
+        for (Leaf leaf : leaves)
+        {
+            Plan candidate = bestPlanSoFar.removeRestriction(leaf.id);
+            if (logger.isTraceEnabled())
+                logger.trace("Candidate query plan:\n{}", candidate.toStringRecursive());
+
+            if (candidate.fullCost() <= bestPlanSoFar.fullCost())
+                bestPlanSoFar = candidate;
+        }
+
+        if (logger.isTraceEnabled())
+            logger.trace("Optimized plan:\n{}", bestPlanSoFar.toStringRecursive());
+        return bestPlanSoFar;
+    }
+
+    /**
+     * Modifies all intersections to not intersect more clauses than the given limit.
+     * Retains the most selective clauses.
+     */
+    public final Plan limitIntersectedClauses(int clauseLimit)
+    {
+        Plan result = this;
+        if (result instanceof Intersection)
+        {
+            Plan.Intersection intersection = (Plan.Intersection) result;
+            result = intersection.stripSubplans(clauseLimit);
+        }
+        return result.withUpdatedSubplans(p -> p.limitIntersectedClauses(clauseLimit));
+    }
+
+    /** Returns true if the plan contains a node matching the condition */
+    final boolean contains(Function<Plan, Boolean> condition)
+    {
+        ControlFlow res = forEach(node -> (condition.apply(node)) ? ControlFlow.Break : ControlFlow.Continue);
+        return res == ControlFlow.Break;
+    }
+
+    /**
+     * Returns a new plan with the given node filtering restriction removed.
+     * Searches for the subplan to remove recursively down the tree.
+     * If the new plan is different, its estimates are also recomputed.
+     * If *this* plan matches the id, then the {@link Everything} node is returned.
+     *
+     * <p>
+     * The purpose of this method is to optimise the plan.
+     * Sometimes not doing an intersection and post-filtering instead can be faster, so by removing child nodes from
+     * intersections we can potentially get a better plan.
+     */
+    final Plan removeRestriction(int id)
+    {
+        if (this.id != id)
+            return withUpdatedSubplans(subplan -> subplan.removeRestriction(id));
+
+        // If id is the same, replace this node with "everything"
+        // because a query with no filter expression returns all rows
+        // (removing restrictions should widen the result set).
+        // Beware we must not remove ordering because that would change the semantics of the query.
+        Orderer ordering = this.ordering();
+        return (ordering != null)
+               ? factory.sort(factory.everything, ordering)
+               : factory.everything;
+    }
+
+    /**
+     * Returns the estimated cost of preparation steps
+     * that must be done before returning the first row / key
+     */
+    public final double initCost()
+    {
+        return cost().initCost();
+    }
+
+    public final double iterCost()
+    {
+        return cost().iterCost();
+    }
+
+    /**
+     * Returns the estimated cost of running the plan to completion, i.e. exhausting
+     * the key or row iterator returned by it
+     */
+    public final double fullCost()
+    {
+        return cost().fullCost();
+    }
+
+    /**
+     * Returns the estimated fraction of the table data that the result of this plan is expected to match
+     */
+    public final double selectivity()
+    {
+        if (selectivity == -1)
+            selectivity = estimateSelectivity();
+        assert 0.0 <= selectivity && selectivity <= 1.0 : "Invalid selectivity: " + selectivity;
+        return selectivity;
+    }
+
+    protected interface Cost
+    {
+        /**
+         * Initialization cost: cannot be reduced later.
+         */
+        double initCost();
+
+        /**
+         * Cost to iterate over all the expected keys or rows.  May be reduced by LIMIT.
+         */
+        double iterCost();
+
+        default double fullCost()
+        {
+            return initCost() + iterCost();
+        }
+    }
+
+    protected static final class KeysIterationCost implements Cost
+    {
+        final double expectedKeys;
+        final double initCost;
+        final double iterCost;
+
+        /**
+         * @param expectedKeys number of keys expected to be iterated over
+         * @param initCost cost to set up the iteration
+         * @param iterCost *total* cost of iterating over the expected number of keys
+         */
+        public KeysIterationCost(double expectedKeys, double initCost, double iterCost)
+        {
+            this.expectedKeys = expectedKeys;
+            this.initCost = initCost;
+            this.iterCost = iterCost;
+        }
+
+        @Override
+        public double initCost()
+        {
+            return initCost;
+        }
+
+        @Override
+        public double iterCost()
+        {
+            return iterCost;
+        }
+
+        public double costPerKey()
+        {
+            return expectedKeys == 0 ? 0.0 : iterCost / expectedKeys;
+        }
+
+        public String toString()
+        {
+            return String.format("keys: %.1f, cost/key: %.1f, cost: %.1f..%.1f",
+                                 expectedKeys, costPerKey(), initCost, fullCost());
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            KeysIterationCost that = (KeysIterationCost) o;
+            return Double.compare(expectedKeys, that.expectedKeys) == 0
+                   && Double.compare(initCost, that.initCost) == 0
+                   && Double.compare(iterCost, that.iterCost) == 0;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(expectedKeys, initCost, iterCost);
+        }
+    }
+
+    protected static final class RowsIterationCost implements Cost
+    {
+        final double expectedRows;
+        final double initCost;
+        final double iterCost;
+
+        public RowsIterationCost(double expectedRows, double initCost, double iterCost)
+        {
+            this.expectedRows = expectedRows;
+            this.initCost = initCost;
+            this.iterCost = iterCost;
+        }
+
+        @Override
+        public double initCost()
+        {
+            return initCost;
+        }
+
+        @Override
+        public double iterCost()
+        {
+            return iterCost;
+        }
+
+        public double costPerRow()
+        {
+            return expectedRows == 0 ? 0.0 : iterCost / expectedRows;
+        }
+
+        public String toString()
+        {
+            return String.format("rows: %.1f, cost/row: %.1f, cost: %.1f..%.1f",
+                                 expectedRows, costPerRow(), initCost, fullCost());
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            RowsIterationCost that = (RowsIterationCost) o;
+            return Double.compare(expectedRows, that.expectedRows) == 0
+                   && Double.compare(initCost, that.initCost) == 0
+                   && Double.compare(iterCost, that.iterCost) == 0;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(expectedRows, initCost, iterCost);
+        }
+    }
+
+    /**
+     * Common base class for all plan nodes that iterate over primary keys.
+     */
+    public abstract static class KeysIteration extends Plan
+    {
+        /**
+         * Caches the estimated cost to avoid frequent recomputation
+         */
+        private KeysIterationCost cost;
+
+        protected KeysIteration(Factory factory, int id, Access access)
+        {
+            super(factory, id, access);
+        }
+
+        @Override
+        protected final KeysIterationCost cost()
+        {
+            if (cost == null)
+                cost = estimateCost();
+            return cost;
+        }
+
+        protected abstract KeysIterationCost estimateCost();
+
+        /**
+         * Executes the operation represented by this node.
+         * The node itself isn't supposed for doing the actual work, but rather serves as a director which
+         * delegates the work to the query controller through the passed Executor.
+         *
+         * @param executor does all the hard work like fetching keys from the indexes or ANN sort
+         */
+        protected abstract Iterator<? extends PrimaryKey> execute(Executor executor);
+
+        protected abstract KeysIteration withAccess(Access patterns);
+
+        final double expectedKeys()
+        {
+            return cost().expectedKeys;
+        }
+
+        final double costPerKey()
+        {
+            return cost().costPerKey();
+        }
+
+    }
+
+    /**
+     * Leaves of the plan tree cannot have subplans.
+     * This class exists purely for DRY purpose.
+     */
+    abstract static class Leaf extends KeysIteration
+    {
+        protected Leaf(Factory factory, int id, Access accesses)
+        {
+            super(factory, id, accesses);
+        }
+
+        @Override
+        protected ControlFlow forEachSubplan(Function<Plan, ControlFlow> function)
+        {
+            return ControlFlow.Continue;
+        }
+
+        @Override
+        protected final Plan withUpdatedSubplans(Function<Plan, Plan> updater)
+        {
+            // There are no subplans so it is a noop
+            return this;
+        }
+    }
+
+    /**
+     * Represents an index scan that returns an empty range
+     */
+    static class Nothing extends Leaf
+    {
+        protected Nothing(int id, Factory factory)
+        {
+            super(factory, id, null);
+        }
+
+        @Nonnull
+        @Override
+        protected KeysIterationCost estimateCost()
+        {
+            return new KeysIterationCost(0, 0.0, 0.0);
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return null;
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            return 0;
+        }
+
+        @Override
+        protected KeyRangeIterator execute(Executor executor)
+        {
+            return KeyRangeIterator.empty();
+        }
+
+        @Override
+        protected Nothing withAccess(Access patterns)
+        {
+            // limit does not matter for Nothing node because it always returns 0 keys
+            return this;
+        }
+    }
+
+    /**
+     * Represents an index scan that returns all keys in the table.
+     * This is a virtual node that has no real representation in the database system.
+     * It is useful in query optimization.
+     */
+    static class Everything extends Leaf
+    {
+        protected Everything(int id, Factory factory, Access accesses)
+        {
+            super(factory, id, accesses);
+        }
+
+        @Nonnull
+        @Override
+        protected KeysIterationCost estimateCost()
+        {
+            // We set the cost to infinity so this node is never present in the optimized plan.
+            // We don't want to have those nodes in the final plan,
+            // because currently we have no way to execute it efficiently.
+            // In the future we may want to change it, when we have a way to return all rows without using an index.
+            return new KeysIterationCost(access.expectedAccessCount(factory.tableMetrics.rows),
+                                         Double.POSITIVE_INFINITY,
+                                         Double.POSITIVE_INFINITY);
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return null;
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            return 1.0;
+        }
+
+        @Override
+        protected KeyRangeIterator execute(Executor executor)
+        {
+            // Not supported because it doesn't make a lot of sense.
+            // A direct scan of table data would be certainly faster.
+            // Everything node is not supposed to be executed. However, it is useful for analyzing various plans,
+            // e.g. we may get such node after removing some nodes from a valid, executable plan.
+            throw new UnsupportedOperationException("Returning an iterator over all keys is not supported.");
+        }
+
+        @Override
+        protected Everything withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new Everything(id, factory, access);
+        }
+    }
+
+    abstract static class IndexScan extends Leaf
+    {
+        @Nullable
+        protected final Expression predicate;
+        @Nullable
+        protected final Orderer ordering;
+
+        protected final long matchingKeysCount;
+
+        public IndexScan(Factory factory, int id, Expression predicate, long matchingKeysCount, Access access, Orderer ordering)
+        {
+            super(factory, id, access);
+            Preconditions.checkArgument(predicate != null || ordering != null,
+                                        "Either predicate or ordering must be set");
+            Preconditions.checkArgument(predicate == null
+                                        || ordering == null
+                                        || predicate.getIndexName().equals(ordering.getIndexName()),
+                                        "Ordering must use the same index as the predicate");
+            this.predicate = predicate;
+            // If we match by equality, ordering makes no sense because all term values would be the same.
+            this.ordering = (predicate == null || predicate.getOp() != Expression.Op.EQ) ? ordering : null;
+            this.matchingKeysCount = matchingKeysCount;
+        }
+
+        @Override
+        protected final String title()
+        {
+            return String.format("of %s (sel: %.9f, step: %.1f)",
+                                 getIndexName(), selectivity(), access.meanDistance());
+        }
+
+        @Override
+        protected String description()
+        {
+            StringBuilder sb = new StringBuilder();
+            if (predicate != null)
+            {
+                sb.append("predicate: ");
+                sb.append(predicate);
+                sb.append('\n');
+            }
+            if (ordering != null)
+            {
+                sb.append("ordering: ");
+                sb.append(ordering);
+                sb.append('\n');
+            }
+            return sb.toString();
+        }
+
+        @Nullable
+        @Override
+        protected final Orderer ordering()
+        {
+            return ordering;
+        }
+
+        @Override
+        protected final KeysIterationCost estimateCost()
+        {
+            double expectedKeys = access.expectedAccessCount(matchingKeysCount);
+            double costPerKey = access.unitCost(SAI_KEY_COST, this::estimateCostPerSkip);
+            // we hit-rate-scale the open cost, but not the per-key cost, under the assumption that
+            // readahead against the postings file will mostly amortize the penalty when hitting disk per key
+            double initCost = hrs(SAI_OPEN_COST) * factory.tableMetrics.sstables;
+            double iterCost = expectedKeys * costPerKey;
+            return new KeysIterationCost(expectedKeys, initCost, iterCost);
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            return factory.tableMetrics.rows > 0
+                   ? ((double) matchingKeysCount / factory.tableMetrics.rows)
+                   : 0.0;
+        }
+
+        private double estimateCostPerSkip(double step)
+        {
+            // This is the first very rough approximation of the cost model for skipTo operation.
+            // It is likely not a very accurate model.
+            // We know for sure that the cost goes up the bigger the skip distance (= the more key we skip over)
+            // and also the bigger the merged posting list is. A range scan of the numeric
+            // index may require merging posting lists from many index nodes. Intuitively, the more keys we match,
+            // the higher number of posting lists are merged. Also, the further we skip, the higher number
+            // of posting lists must be advanced, and we're also more likely to hit a non-cached chunk.
+            // From a few experiments I did, I conclude those costs grow sublinearly.
+            // In the future we probably will need to take more index metrics into account
+            // (e.g. number of distinct values).
+
+            double keysPerSSTable = (double) matchingKeysCount / factory.tableMetrics.sstables;
+
+            double skipCostFactor;
+            double postingsCountFactor;
+            double postingsCountExponent;
+            double skipDistanceFactor;
+            double skipDistanceExponent;
+            if (predicate == null || predicate.getOp() == Expression.Op.RANGE)
+            {
+                skipCostFactor = RANGE_SCAN_SKIP_COST;
+                postingsCountFactor = RANGE_SCAN_SKIP_COST_POSTINGS_COUNT_FACTOR;
+                postingsCountExponent = RANGE_SCAN_SKIP_COST_POSTINGS_COUNT_EXPONENT;
+                skipDistanceFactor = RANGE_SCAN_SKIP_COST_DISTANCE_FACTOR;
+                skipDistanceExponent = RANGE_SCAN_SKIP_COST_DISTANCE_EXPONENT;
+            }
+            else
+            {
+                skipCostFactor = POINT_LOOKUP_SKIP_COST;
+                postingsCountFactor = 0.0;
+                postingsCountExponent = 1.0;
+                skipDistanceFactor = POINT_LOOKUP_SKIP_COST_DISTANCE_FACTOR;
+                skipDistanceExponent = POINT_LOOKUP_SKIP_COST_DISTANCE_EXPONENT;
+            }
+
+            // divide by exponent so the derivative at 1.0 equals postingsCountFactor
+            double dKeys = postingsCountFactor / postingsCountExponent;
+            double postingsCountPenalty = dKeys * Math.pow(keysPerSSTable, postingsCountExponent);
+
+            // divide by exponent so the derivative at 1.0 equals skipDistanceFactor
+            double dPostings = skipDistanceFactor / skipDistanceExponent;
+            double distancePenalty = dPostings * Math.pow(step, skipDistanceExponent);
+
+            return skipCostFactor
+                   * (1.0 + distancePenalty)
+                   * (1.0 + postingsCountPenalty)
+                   * factory.tableMetrics.sstables;
+        }
+
+        @Override
+        protected Iterator<? extends PrimaryKey> execute(Executor executor)
+        {
+            return (ordering != null)
+                ? executor.getTopKRows(predicate, max(1, round((float) access.expectedAccessCount(factory.tableMetrics.rows))))
+                : executor.getKeysFromIndex(predicate);
+        }
+
+        public String getIndexName()
+        {
+            assert predicate != null || ordering != null;
+            return predicate != null ? predicate.getIndexName() : ordering.getIndexName();
+        }
+
+        @Override
+        final protected IndexContext getIndexContext()
+        {
+            assert predicate != null || ordering != null;
+            return predicate != null ? predicate.context : ordering.context;
+        }
+    }
+    /**
+     * Represents a scan over a numeric storage attached index.
+     */
+    static class NumericIndexScan extends IndexScan
+    {
+        public NumericIndexScan(Factory factory, int id, Expression predicate, long matchingKeysCount, Access access, Orderer ordering)
+        {
+            super(factory, id, predicate, matchingKeysCount, access, ordering);
+        }
+
+        @Override
+        protected NumericIndexScan withAccess(Access access)
+        {
+            return Objects.equals(this.access, access)
+                   ? this
+                   : new NumericIndexScan(factory, id, predicate, matchingKeysCount, access, ordering);
+        }
+    }
+
+    /**
+     * Represents a scan over a literal storage attached index
+     */
+    static class LiteralIndexScan extends IndexScan
+    {
+        public LiteralIndexScan(Factory factory, int id, Expression predicate, long matchingKeysCount, Access access, Orderer ordering)
+        {
+            super(factory, id, predicate, matchingKeysCount, access, ordering);
+        }
+
+        @Override
+        protected LiteralIndexScan withAccess(Access access)
+        {
+            return Objects.equals(this.access, access)
+                   ? this
+                   : new LiteralIndexScan(factory, id, predicate, matchingKeysCount, this.access, ordering);
+        }
+    }
+
+    /**
+     * Union of multiple primary key streams.
+     * This is a fairly cheap operation - its cost is basically a sum of costs of the subplans.
+     */
+    static final class Union extends KeysIteration
+    {
+        private final LazyTransform<List<KeysIteration>> subplansSupplier;
+
+        Union(Factory factory, int id, List<KeysIteration> subplans, Access access)
+        {
+            super(factory, id, access);
+            Preconditions.checkArgument(!subplans.isEmpty(), "Subplans must not be empty");
+
+            // We propagate Access lazily just before we need the subplans.
+            // This is because there may be several requests to change the access pattern from the top,
+            // and we don't want to reconstruct the whole subtree each time
+            this.subplansSupplier = new LazyTransform<>(subplans, this::propagateAccess);
+        }
+
+        /**
+         * Adjusts the counts for each subplan to account for the other subplans.
+         * As explained in `estimateSelectivity`, the union of (for instance) two subplans
+         * that each select 50% of the keys is 75%, not 100%.  Thus, we need to reduce the counts
+         * to remove estimated overlapping keys.
+         */
+        private List<KeysIteration> propagateAccess(List<KeysIteration> subplans)
+        {
+            if (isEffectivelyZero(selectivity()))
+            {
+                // all subplan selectivity should also be ~0
+                for (var subplan: subplans)
+                    assert isEffectivelyZero(subplan.selectivity());
+                return subplans;
+            }
+
+            ArrayList<KeysIteration> newSubplans = new ArrayList<>(subplans.size());
+            for (KeysIteration subplan : subplans)
+            {
+                Access access = this.access.scaleCount(subplan.selectivity() / selectivity());
+                newSubplans.add(subplan.withAccess(access));
+            }
+            return newSubplans;
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            // Assume independence (lack of correlation) of subplans.
+            // We multiply the probabilities of *not* selecting a key.
+            // Because selectivity is usage-independent, we can use the original subplans,
+            // to avoid forcing pushdown of Access information down.
+            double inverseSelectivity = 1.0;
+            for (KeysIteration plan : subplansSupplier.orig)
+                inverseSelectivity *= (1.0 - plan.selectivity());
+            return 1.0 - inverseSelectivity;
+        }
+
+
+        @Override
+        protected ControlFlow forEachSubplan(Function<Plan, ControlFlow> function)
+        {
+            for (Plan s : subplansSupplier.get())
+            {
+                if (function.apply(s) == ControlFlow.Break)
+                    return ControlFlow.Break;
+            }
+            return ControlFlow.Continue;
+        }
+
+        @Override
+        protected Plan withUpdatedSubplans(Function<Plan, Plan> updater)
+        {
+            List<KeysIteration> subplans = subplansSupplier.get();
+            ArrayList<KeysIteration> newSubplans = new ArrayList<>(subplans.size());
+            for (Plan subplan : subplans)
+                newSubplans.add((KeysIteration) updater.apply(subplan));
+
+            return newSubplans.equals(subplans)
+                   ? this
+                   : factory.union(newSubplans, id).withAccess(access);
+        }
+
+        @Override
+        protected Union withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new Union(factory, id, subplansSupplier.orig, access);
+        }
+
+        @Override
+        protected KeysIterationCost estimateCost()
+        {
+            double initCost = 0.0;
+            double iterCost = 0.0;
+            List<KeysIteration> subplans = subplansSupplier.get();
+            for (int i = 0; i < subplans.size(); i++)
+            {
+                KeysIteration subplan = subplans.get(i);
+                // Initialization must be done all branches before we can start iterating
+                initCost += subplan.initCost();
+                iterCost += subplan.iterCost();
+            }
+            double expectedKeys = access.expectedAccessCount(factory.tableMetrics.rows * selectivity());
+            return new KeysIterationCost(expectedKeys, initCost, iterCost);
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return subplansSupplier.get().get(0).ordering();
+        }
+
+        @Override
+        protected KeyRangeIterator execute(Executor executor)
+        {
+            KeyRangeIterator.Builder builder = KeyRangeUnionIterator.builder();
+            try
+            {
+                for (KeysIteration plan : subplansSupplier.get())
+                    builder.add((KeyRangeIterator) plan.execute(executor));
+                return builder.build();
+            }
+            catch (Throwable t)
+            {
+                FileUtils.closeQuietly(builder.ranges());
+                throw t;
+            }
+        }
+    }
+
+    /**
+     * Intersection of multiple primary key streams.
+     * This is quite complex operation where many keys from all underlying streams must be read in order
+     * to return one matching key. Therefore, expect the cost of this operation to be significantly higher than
+     * the costs of the subplans.
+     */
+    static final class Intersection extends KeysIteration
+    {
+        private final LazyTransform<List<KeysIteration>> subplansSupplier;
+
+        private Intersection(Factory factory, int id, List<KeysIteration> subplans, Access access)
+        {
+            super(factory, id, access);
+            Preconditions.checkArgument(!subplans.isEmpty(), "Subplans must not be empty");
+
+            // We propagate Access lazily just before we need the subplans.
+            // This is because there may be several requests to change the access pattern from the top,
+            // and we don't want to reconstruct the whole subtree each time
+            this.subplansSupplier = new LazyTransform<>(subplans, this::propagateAccess);
+        }
+
+        /**
+         * In an intersection operation, the goal is to find the common elements between the results
+         * of multiple subplans.  This requires taking into account not only the selectivity but also
+         * the match probabilities between subplans.
+         * <p>
+         * VSTODO explain what's going on in more detail.
+         */
+        private ArrayList<KeysIteration> propagateAccess(List<KeysIteration> subplans)
+        {
+            double loops = subplans.get(0).selectivity() / boundedSelectivity(selectivity());
+            ArrayList<KeysIteration> newSubplans = new ArrayList<>(subplans.size());
+            KeysIteration s0 = subplans.get(0).withAccess(access.scaleDistance(loops).convolute(loops, 1.0));
+            newSubplans.add(s0);
+
+            // We may run out of keys while iterating the first iterator, and then we just break the loop early
+            loops = Math.min(s0.expectedKeys(), loops);
+
+            double matchProbability = 1.0;
+            for (int i = 1; i < subplans.size(); i++)
+            {
+                KeysIteration subplan = subplans.get(i);
+                double cumulativeSelectivity = subplans.get(0).selectivity() * matchProbability;
+                double skipDistance = subplan.selectivity() / boundedSelectivity(cumulativeSelectivity);
+                Access subAccess = access.scaleDistance(subplan.selectivity() / boundedSelectivity(selectivity()))
+                                         .convolute(loops * matchProbability, skipDistance)
+                                         .forceSkip();
+                newSubplans.add(subplan.withAccess(subAccess));
+                matchProbability *= subplan.selectivity();
+            }
+            return newSubplans;
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            double selectivity = 1.0;
+            for (KeysIteration plan : subplansSupplier.orig)
+                selectivity *= plan.selectivity();
+            return selectivity;
+        }
+
+        @Override
+        protected ControlFlow forEachSubplan(Function<Plan, ControlFlow> function)
+        {
+            for (Plan s : subplansSupplier.get())
+            {
+                if (function.apply(s) == ControlFlow.Break)
+                    return ControlFlow.Break;
+            }
+            return ControlFlow.Continue;
+        }
+
+        @Override
+        protected Plan withUpdatedSubplans(Function<Plan, Plan> updater)
+        {
+            List<KeysIteration> subplans = subplansSupplier.get();
+            ArrayList<KeysIteration> newSubplans = new ArrayList<>(subplans.size());
+            for (Plan subplan : subplans)
+                newSubplans.add((KeysIteration) updater.apply(subplan));
+
+            return newSubplans.equals(subplans)
+                   ? this
+                   : factory.intersection(newSubplans, id).withAccess(access);
+        }
+
+        @Override
+        protected Intersection withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new Intersection(factory, id, subplansSupplier.orig, access);
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return subplansSupplier.get().get(0).ordering();
+        }
+
+        @Override
+        protected KeysIterationCost estimateCost()
+        {
+            List<KeysIteration> subplans = subplansSupplier.get();
+            assert !subplans.isEmpty() : "Expected at least one subplan here. An intersection of 0 plans should have been optimized out.";
+
+            double initCost = 0.0;
+            double iterCost = 0.0;
+            for (KeysIteration subplan : subplans)
+            {
+                initCost += subplan.initCost();
+                iterCost += subplan.iterCost();
+            }
+            double expectedKeyCount = access.expectedAccessCount(factory.tableMetrics.rows * selectivity());
+            return new KeysIterationCost(expectedKeyCount, initCost, iterCost);
+        }
+
+        @Override
+        protected KeyRangeIterator execute(Executor executor)
+        {
+            KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder();
+            try
+            {
+                for (KeysIteration plan : subplansSupplier.get())
+                    builder.add((KeyRangeIterator) plan.execute(executor));
+
+                return builder.build();
+            }
+            catch (Throwable t)
+            {
+                FileUtils.closeQuietly(builder.ranges());
+                throw t;
+            }
+        }
+
+        /**
+         * Limits the number of intersected subplans
+         */
+        public Plan stripSubplans(int clauseLimit)
+        {
+            if (subplansSupplier.orig.size() <= clauseLimit)
+                return this;
+            List<Plan.KeysIteration> newSubplans = new ArrayList<>(subplansSupplier.orig.subList(0, clauseLimit));
+            return factory.intersection(newSubplans, id).withAccess(access);
+        }
+    }
+
+    /**
+     * Sorts keys in ANN order.
+     * Must fetch all keys from the source before sorting, so it has a high initial cost.
+     */
+    static final class KeysSort extends KeysIteration
+    {
+        private final KeysIteration source;
+        final Orderer ordering;
+
+        KeysSort(Factory factory, int id, KeysIteration source, Access access, Orderer ordering)
+        {
+            super(factory, id, access);
+            this.source = source;
+            this.ordering = ordering;
+        }
+
+        @Override
+        protected ControlFlow forEachSubplan(Function<Plan, ControlFlow> function)
+        {
+            return function.apply(source);
+        }
+
+        @Override
+        protected Plan withUpdatedSubplans(Function<Plan, Plan> updater)
+        {
+            return factory.sort((KeysIteration) updater.apply(source), ordering, id).withAccess(access);
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            return source.selectivity();
+        }
+
+        @Override
+        protected KeysIterationCost estimateCost()
+        {
+            if (ordering.isANN())
+                return estimateAnnSortCost();
+            else if (ordering.isBM25())
+                return estimateBm25SortCost();
+            else
+                return estimateGlobalSortCost();
+        }
+
+        private KeysIterationCost estimateAnnSortCost()
+        {
+            double expectedKeys = access.expectedAccessCount(source.expectedKeys());
+            int expectedKeysInt = max(1, (int) Math.ceil(expectedKeys));
+            int expectedSourceKeysInt = max(1, (int) Math.ceil(source.expectedKeys()));
+            double initCost = ANN_SORT_OPEN_COST * factory.tableMetrics.sstables
+                              + source.fullCost()
+                              + source.expectedKeys() * CostCoefficients.ANN_SORT_KEY_COST;
+            double searchCost = factory.costEstimator.estimateAnnSearchCost(ordering,
+                                                                            expectedKeysInt,
+                                                                            expectedSourceKeysInt);
+            return new KeysIterationCost(expectedKeys, initCost, searchCost);
+        }
+
+        private KeysIterationCost estimateBm25SortCost()
+        {
+            double expectedKeys = access.expectedAccessCount(source.expectedKeys());
+
+            int termCount = ordering.getQueryTerms().size();
+            // all of the cost for BM25 is up front since the index doesn't give us the information we need
+            // to return results in order, in isolation.  The big cost is reading the indexed cells out of
+            // the sstables.
+            // VSTODO if we had stats on cell size _per column_ we could usefully include ROW_BYTE_COST
+            double initCost = source.fullCost()
+                              + source.expectedKeys() * (hrs(ROW_CELL_COST) + ROW_CELL_COST)
+                              + termCount * BM25_SCORE_COST;
+            return new KeysIterationCost(expectedKeys, initCost, 0);
+        }
+
+        private KeysIterationCost estimateGlobalSortCost()
+        {
+            return new KeysIterationCost(source.expectedKeys(),
+                                         source.fullCost() + source.expectedKeys() * hrs(ROW_COST),
+                                         source.expectedKeys() * SAI_KEY_COST);
+
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return ordering;
+        }
+
+        @Override
+        protected Iterator<? extends PrimaryKey> execute(Executor executor)
+        {
+            KeyRangeIterator sourceIterator = (KeyRangeIterator) source.execute(executor);
+            int softLimit = max(1, round((float) access.expectedAccessCount(factory.tableMetrics.rows)));
+            return executor.getTopKRows(sourceIterator, softLimit);
+        }
+
+        @Override
+        protected KeysSort withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new KeysSort(factory, id, source, access, ordering);
+        }
+    }
+
+    /**
+     * Base class for index scans that return results in a computed order (ANN, BM25)
+     * rather than the natural index order.
+     */
+    abstract static class ScoredIndexScan extends Leaf
+    {
+        final Orderer ordering;
+
+        protected ScoredIndexScan(Factory factory, int id, Access access, Orderer ordering)
+        {
+            super(factory, id, access);
+            this.ordering = ordering;
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return ordering;
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            return 1.0;
+        }
+
+        @Override
+        protected Iterator<? extends PrimaryKey> execute(Executor executor)
+        {
+            int softLimit = max(1, round((float) access.expectedAccessCount(factory.tableMetrics.rows)));
+            return executor.getTopKRows((Expression) null, softLimit);
+        }
+    }
+
+    /**
+     * Returns all keys in ANN order.
+     * Contrary to {@link KeysSort}, there is no input node here and the output is generated lazily.
+     */
+    final static class AnnIndexScan extends ScoredIndexScan
+    {
+        protected AnnIndexScan(Factory factory, int id, Access access, Orderer ordering)
+        {
+            super(factory, id, access, ordering);
+        }
+
+        @Override
+        protected KeysIterationCost estimateCost()
+        {
+            double expectedKeys = access.expectedAccessCount(factory.tableMetrics.rows);
+            int expectedKeysInt = Math.max(1, (int) Math.ceil(expectedKeys));
+            double searchCost = factory.costEstimator.estimateAnnSearchCost(ordering,
+                                                                            expectedKeysInt,
+                                                                            factory.tableMetrics.rows);
+            double initCost = 0; // negligible
+            return new KeysIterationCost(expectedKeys, initCost, searchCost);
+        }
+
+        @Override
+        protected KeysIteration withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new AnnIndexScan(factory, id, access, ordering);
+        }
+
+        @Nullable
+        @Override
+        protected IndexContext getIndexContext()
+        {
+            return ordering.context;
+        }
+    }
+
+    /**
+     * Returns all keys in BM25 order.
+     * Like AnnIndexScan, this generates results lazily without an input node.
+     */
+    final static class Bm25IndexScan extends ScoredIndexScan
+    {
+        protected Bm25IndexScan(Factory factory, int id, Access access, Orderer ordering)
+        {
+            super(factory, id, access, ordering);
+        }
+
+        @Nonnull
+        @Override
+        protected KeysIterationCost estimateCost()
+        {
+            double expectedKeys = access.expectedAccessCount(factory.tableMetrics.rows);
+            int expectedKeysInt = Math.max(1, (int) Math.ceil(expectedKeys));
+
+            int termCount = ordering.getQueryTerms().size();
+            double initCost = expectedKeysInt * (hrs(ROW_CELL_COST) + ROW_CELL_COST)
+                              + termCount * BM25_SCORE_COST;
+
+            return new KeysIterationCost(expectedKeys, initCost, 0);
+        }
+
+        @Override
+        protected KeysIteration withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new Bm25IndexScan(factory, id, access, ordering);
+        }
+
+        @Override
+        protected IndexContext getIndexContext()
+        {
+            return ordering.context;
+        }
+    }
+
+
+    abstract public static class RowsIteration extends Plan
+    {
+        private RowsIterationCost cost;
+
+        private RowsIteration(Factory factory, int id, Access access)
+        {
+            super(factory, id, access);
+        }
+
+        @Override
+        protected RowsIterationCost cost()
+        {
+            if (cost == null)
+                cost = estimateCost();
+            return cost;
+        }
+
+        protected abstract RowsIterationCost estimateCost();
+
+        protected abstract RowsIteration withAccess(Access patterns);
+
+        final double costPerRow()
+        {
+            return cost().costPerRow();
+        }
+
+        final double expectedRows()
+        {
+            return cost().expectedRows;
+        }
+    }
+
+    /**
+     * Retrieves rows from storage based on the stream of primary keys
+     */
+    static final class Fetch extends RowsIteration
+    {
+        private final LazyTransform<KeysIteration> source;
+
+        private Fetch(Factory factory, int id, KeysIteration keysIteration, Access access)
+        {
+            super(factory, id, access);
+            this.source = new LazyTransform<>(keysIteration, k -> k.withAccess(access));
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return source.get().ordering();
+        }
+
+        @Override
+        protected ControlFlow forEachSubplan(Function<Plan, ControlFlow> function)
+        {
+            return function.apply(source.get());
+        }
+
+        @Override
+        protected Fetch withUpdatedSubplans(Function<Plan, Plan> updater)
+        {
+            Plan.KeysIteration updatedSource = (KeysIteration) updater.apply(source.get());
+            return updatedSource == source.get() ? this : new Fetch(factory, id, updatedSource, access);
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            return source.orig.selectivity();
+        }
+
+        @Override
+        protected RowsIterationCost estimateCost()
+        {
+            // VSTODO this assumes we will need to deserialize the entire row for any fetch.
+            // For vector rows where we need to check a non-vector field for a predicate,
+            // this is a very pessimistic assumption since the vectors (that we don't read)
+            // are by far the majority of the row size.
+            double rowFetchCost = hrs(CostCoefficients.ROW_COST)
+                                  + CostCoefficients.ROW_CELL_COST * factory.tableMetrics.avgCellsPerRow
+                                  + CostCoefficients.ROW_BYTE_COST * factory.tableMetrics.avgBytesPerRow;
+
+            KeysIteration src = source.get();
+            double expectedKeys = access.expectedAccessCount(src.expectedKeys());
+            return new RowsIterationCost(expectedKeys,
+                                         src.initCost(),
+                                         src.iterCost() + expectedKeys * rowFetchCost);
+        }
+
+        @Override
+        protected Fetch withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new Fetch(factory, id, source.orig, access);
+        }
+    }
+
+    /**
+     * Filters rows.
+     * In order to return one row in the result set it may need to retrieve many rows from the source node.
+     * Hence, it will typically have higher cost-per-row than the source node, and will return fewer rows.
+     */
+    static class Filter extends RowsIteration
+    {
+        private final RowFilter filter;
+        private final LazyTransform<RowsIteration> source;
+        private final double targetSelectivity;
+
+        Filter(Factory factory, int id, RowFilter filter, RowsIteration source, double targetSelectivity, Access access)
+        {
+            super(factory, id, access);
+            this.filter = filter;
+            this.source = new LazyTransform<>(source, this::propagateAccess);
+            this.targetSelectivity = targetSelectivity;
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return source.get().ordering();
+        }
+
+        /**
+         * Scale the access pattern of the source to reflect that we will need
+         * to keep pulling rows from it until the Filter is satisfied.
+         */
+        private RowsIteration propagateAccess(RowsIteration source)
+        {
+            Access scaledAccess = access.scaleCount(source.selectivity() / boundedSelectivity(targetSelectivity));
+            return source.withAccess(scaledAccess);
+        }
+
+        @Override
+        protected ControlFlow forEachSubplan(Function<Plan, ControlFlow> function)
+        {
+            return function.apply(source.get());
+        }
+
+        @Override
+        protected Plan withUpdatedSubplans(Function<Plan, Plan> updater)
+        {
+            Plan.RowsIteration updatedSource = (RowsIteration) updater.apply(source.get());
+            return updatedSource == source.get()
+                   ? this
+                   : new Filter(factory, id, filter, updatedSource, targetSelectivity, access);
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            return targetSelectivity;
+        }
+
+        @Override
+        protected RowsIterationCost estimateCost()
+        {
+            double expectedRows = access.expectedAccessCount(factory.tableMetrics.rows * targetSelectivity);
+            return new RowsIterationCost(expectedRows,
+                                         source.get().initCost(),
+                                         source.get().iterCost());
+        }
+
+        @Override
+        protected Filter withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new Filter(factory, id, filter, source.orig, targetSelectivity, access);
+        }
+
+        @Override
+        protected String title()
+        {
+            return String.format("%s (sel: %.9f)", filter, selectivity() / source.get().selectivity());
+        }
+    }
+
+    /**
+     * Limits the number of returned rows to a fixed number.
+     * Unlike {@link Filter} it does not affect the cost-per-row.
+     */
+    static class Limit extends RowsIteration
+    {
+        private final LazyTransform<RowsIteration> source;
+        final int limit;
+
+        private Limit(Factory factory, int id, RowsIteration source, int limit, Access access)
+        {
+            super(factory, id, access);
+            this.limit = limit;
+            this.source = new LazyTransform<>(source, s -> s.withAccess(access.limit(limit)));
+        }
+
+        @Nullable
+        @Override
+        protected Orderer ordering()
+        {
+            return source.get().ordering();
+        }
+
+        @Override
+        protected ControlFlow forEachSubplan(Function<Plan, ControlFlow> function)
+        {
+            return function.apply(source.get());
+        }
+
+        @Override
+        protected Plan withUpdatedSubplans(Function<Plan, Plan> updater)
+        {
+            Plan.RowsIteration updatedSource = (RowsIteration) updater.apply(source.get());
+            return updatedSource == source.get() ? this : new Limit(factory, id, updatedSource, limit, access);
+        }
+
+        @Override
+        protected double estimateSelectivity()
+        {
+            return source.orig.selectivity();
+        }
+
+        @Override
+        protected RowsIterationCost estimateCost()
+        {
+            RowsIteration src = source.get();
+            double expectedRows = access.expectedAccessCount(src.expectedRows());
+            double iterCost = (limit >= src.expectedRows())
+                              ? src.iterCost()
+                              : src.iterCost() * limit / src.expectedRows();
+            return new RowsIterationCost(expectedRows, src.initCost(), iterCost);
+        }
+
+        @Override
+        protected RowsIteration withAccess(Access access)
+        {
+            return Objects.equals(access, this.access)
+                   ? this
+                   : new Limit(factory, id, source.orig, limit, access);
+        }
+
+        @Override
+        protected String title()
+        {
+            return "" + limit;
+        }
+    }
+
+    /**
+     * Constructs plan nodes.
+     * Contains data common for all plan nodes.
+     * Performs very lightweight local optimizations.
+     * E.g. requesting an intersection/union of only one subplan will result in returning the subplan directly
+     * and no intersection/union will be created.
+     */
+    @NotThreadSafe
+    public static final class Factory
+    {
+        /** Table metrics that affect cost estimates, e.g. row count, sstable count etc */
+        public final TableMetrics tableMetrics;
+
+        public final CostEstimator costEstimator;
+
+        /** A plan returning no keys */
+        public final KeysIteration nothing;
+
+        /** A plan returning all keys in the table */
+        public final KeysIteration everything;
+
+        /** Default access pattern is to read all rows/keys without skipping until the end of the iterator */
+        private final Access defaultAccess;
+
+        /** Id of the next new node created by this factory */
+        private int nextId = 0;
+
+        /**
+         * Creates a factory that produces Plan nodes.
+         * @param tableMetrics allows the planner to adapt the cost estimates to the actual amount of data stored in the table
+         */
+        public Factory(TableMetrics tableMetrics, CostEstimator costEstimator)
+        {
+            this.tableMetrics = tableMetrics;
+            this.costEstimator = costEstimator;
+            this.nothing = new Nothing(-1, this);
+            this.defaultAccess = Access.sequential(tableMetrics.rows);
+            this.everything = new Everything(-1, this, defaultAccess);
+        }
+
+        /**
+         * Constructs a plan node representing a direct scan of an index.
+         *
+         * @param predicate         the expression matching the rows that we want to search in the index;
+         *                          this is needed for identifying this node, it doesn't affect the cost
+         * @param matchingKeysCount the number of row keys expected to be returned by the index scan,
+         *                          i.e. keys of rows that match the search predicate
+         */
+        public KeysIteration indexScan(@Nullable Expression predicate, long matchingKeysCount)
+        {
+            Preconditions.checkArgument(matchingKeysCount >= 0, "matchingKeyCount must not be negative");
+            Preconditions.checkArgument(matchingKeysCount <= tableMetrics.rows, "matchingKeyCount must not exceed totalKeyCount");
+            return indexScan(predicate, matchingKeysCount, null, nextId++);
+        }
+
+        private KeysIteration indexScan(Expression predicate, long matchingKeysCount, Orderer ordering, int id)
+        {
+            if (predicate == null && ordering == null)
+            {
+                assert matchingKeysCount == tableMetrics.rows;
+                return everything;
+            }
+
+            if (ordering != null)
+                if (ordering.isANN())
+                    return new AnnIndexScan(this, id, defaultAccess, ordering);
+                else if (ordering.isBM25())
+                    return new Bm25IndexScan(this, id, defaultAccess, ordering);
+                else if (ordering.isLiteral())
+                    return new LiteralIndexScan(this, id, predicate, matchingKeysCount, defaultAccess, ordering);
+                else
+                    return new NumericIndexScan(this, id, predicate, matchingKeysCount, defaultAccess, ordering);
+
+            Preconditions.checkNotNull(predicate, "predicate must not be null");
+            Preconditions.checkArgument(matchingKeysCount >= 0, "matchingKeyCount must not be negative");
+            Preconditions.checkArgument(matchingKeysCount <= tableMetrics.rows, "matchingKeyCount must not exceed totalKeyCount");
+            return predicate.isLiteral()
+                   ? new LiteralIndexScan(this, id, predicate, matchingKeysCount, defaultAccess, null)
+                   : new NumericIndexScan(this, id, predicate, matchingKeysCount, defaultAccess, null);
+        }
+
+        public KeysIteration fullIndexScan(IndexContext context)
+        {
+            Expression everythingExpression = new Expression(context);
+            everythingExpression.operation = Expression.Op.RANGE;
+            return indexScan(everythingExpression, tableMetrics.rows);
+        }
+
+        /**
+         * Constructs a plan node representing a union of two key sets.
+         * @param subplans a list of subplans for unioned key sets
+         */
+        public KeysIteration union(List<KeysIteration> subplans)
+        {
+            return union(subplans, nextId++);
+        }
+
+        private KeysIteration union(List<KeysIteration> subplans, int id)
+        {
+            if (subplans.contains(everything))
+                return everything;
+            if (subplans.contains(nothing))
+                subplans.removeIf(s -> s == nothing);
+            if (subplans.size() == 1)
+                return subplans.get(0);
+            if (subplans.isEmpty())
+                return nothing;
+
+            return new Union(this, id, subplans, defaultAccess);
+        }
+
+        /**
+         * Constructs a plan node representing an intersection of key sets.
+         * The subplans will be sorted by selectivity from the most selective to the least selective ones.
+         * @param subplans a list of subplans for intersected key sets
+         */
+        public KeysIteration intersection(List<KeysIteration> subplans)
+        {
+            return intersection(subplans, nextId++);
+        }
+
+        private KeysIteration intersection(List<KeysIteration> subplans, int id)
+        {
+            if (subplans.contains(nothing))
+                return nothing;
+            if (subplans.contains(everything))
+                subplans.removeIf(c -> c == everything);
+            if (subplans.size() == 1)
+                return subplans.get(0);
+            if (subplans.isEmpty())
+                return everything;
+
+            subplans.sort(Comparator.comparing(KeysIteration::selectivity));
+            return new Intersection(this, id, subplans, defaultAccess);
+        }
+
+        public Builder unionBuilder()
+        {
+            return new Builder(this, Operation.OperationType.OR);
+        }
+
+        public Builder intersectionBuilder()
+        {
+            return new Builder(this, Operation.OperationType.AND);
+        }
+
+        /**
+         * Constructs a node that sorts keys using an index
+         */
+        public KeysIteration sort(@Nonnull KeysIteration source, @Nonnull Orderer ordering)
+        {
+            return sort(source, ordering, nextId++);
+        }
+
+        private KeysIteration sort(@Nonnull KeysIteration source, @Nonnull Orderer ordering, int id)
+        {
+            if (source instanceof IndexScan)
+            {
+                // Optimization
+                // If we want to sort on the same column as the index scan we already have,
+                // then we collapse sorting with filtering in a single plan node as the index
+                // is already sorted.
+                IndexScan indexScan = (IndexScan) source;
+                if (indexScan.getIndexName().equals(ordering.getIndexName()))
+                    return indexScan(indexScan.predicate, indexScan.matchingKeysCount, ordering, id);
+            }
+
+            return (source instanceof Everything)
+                   ? indexScan(null, tableMetrics.rows, ordering, id)
+                   : new KeysSort(this, id, source, defaultAccess, ordering);
+        }
+
+        /**
+         * Constructs a node that lazily fetches the rows from storage, based on the primary key iterator.
+         */
+        public RowsIteration fetch(@Nonnull KeysIteration keysIterationPlan)
+        {
+            return new Fetch(this, nextId++, keysIterationPlan, defaultAccess);
+        }
+
+        /**
+         * Constructs a filter node with fixed target selectivity set to the selectivity of the source node.
+         * @see Plan.Factory#filter
+         */
+        public RowsIteration recheckFilter(@Nonnull RowFilter filter, @Nonnull RowsIteration source)
+        {
+            return new Filter(this, nextId++, filter, source, source.selectivity(), defaultAccess);
+        }
+
+        /**
+         * Constructs a filter node with fixed target selectivity.
+         * <p>
+         * Fixed target selectivity means that the expected number of rows returned by this node is always
+         * targetSelectivity/totalRows, regardless of the number of the input rows.
+         * Changing the number of the input rows by replacing the subplan
+         * with a subplan of different selectivity does not cause this node to return a different number
+         * of rows (however, it may change the cost per row estimate).
+         * </p><p>
+         * This property is useful for constructing so-called "recheck filters" – filters that
+         * are not any weaker than the filters in the subplan. If a recheck filter is present, we can freely reduce
+         * selectivity of the subplan by e.g. removing intersection nodes, and we still get exactly same number of rows
+         * in the result set.
+         * </p>
+         * @param filter defines which rows are accepted
+         * @param source source plan providing the input rows
+         * @param targetSelectivity a value in range [0.0, 1.0], but not greater than the selectivity of source
+         */
+        public RowsIteration filter(@Nonnull RowFilter filter, @Nonnull RowsIteration source, double targetSelectivity)
+        {
+            Preconditions.checkArgument(targetSelectivity >= 0.0, "selectivity must not be negative");
+            Preconditions.checkArgument(targetSelectivity <= source.selectivity(), "selectivity must not exceed source selectivity of " + source.selectivity());
+            return new Filter(this, nextId++, filter, source, targetSelectivity, defaultAccess);
+        }
+
+        /**
+         * Constructs a plan node that fetches only a limited number of rows.
+         * It is likely going to have lower fullCost than the fullCost of its input.
+         */
+        public RowsIteration limit(@Nonnull RowsIteration source, int limit)
+        {
+            return new Limit(this, nextId++, source, limit, defaultAccess);
+        }
+    }
+
+    public static class TableMetrics
+    {
+        public final long rows;
+        public final double avgCellsPerRow;
+        public final double avgBytesPerRow;
+        public final int sstables;
+
+        public TableMetrics(long rows, double avgCellsPerRow, double avgBytesPerRow, int sstables)
+        {
+            this.rows = rows;
+            this.avgCellsPerRow = avgCellsPerRow;
+            this.avgBytesPerRow = avgBytesPerRow;
+            this.sstables = sstables;
+        }
+    }
+
+    /**
+     * Executes the plan
+     */
+    public interface Executor
+    {
+        Iterator<? extends PrimaryKey> getKeysFromIndex(Expression predicate);
+        Iterator<? extends PrimaryKey> getTopKRows(Expression predicate, int softLimit);
+        Iterator<? extends PrimaryKey> getTopKRows(KeyRangeIterator keys, int softLimit);
+    }
+
+    /**
+     * Outsources more complex cost estimates to external components.
+     * Some components may collect stats on previous data execution and deliver more accurate estimates based
+     * on that state.
+     */
+    public interface CostEstimator
+    {
+        /**
+         * Returns the expected number of ANN index nodes that must be visited to get the list of candidates for top K.
+         *
+         * @param ordering   allows to identify the proper index
+         * @param limit      number of rows to fetch; must be > 0
+         * @param candidates number of candidate rows that satisfy the expression predicates
+         */
+        double estimateAnnSearchCost(Orderer ordering, int limit, long candidates);
+    }
+
+    /**
+     * Data-independent cost coefficients.
+     * They are likely going to change whenever storage engine algorithms change.
+     */
+    public static class CostCoefficients
+    {
+        /** The constant cost of performing skipTo on posting lists returned from range scans */
+        public final static double RANGE_SCAN_SKIP_COST = 0.2;
+
+        /** The coefficient controlling the increase of the skip cost with the distance of the skip. */
+        public final static double RANGE_SCAN_SKIP_COST_DISTANCE_FACTOR = 0.1;
+        public final static double RANGE_SCAN_SKIP_COST_DISTANCE_EXPONENT = 0.5;
+
+        /** The coefficient controlling the increase of the skip cost with the total size of the posting list. */
+        public final static double RANGE_SCAN_SKIP_COST_POSTINGS_COUNT_FACTOR = 0.03;
+        public final static double RANGE_SCAN_SKIP_COST_POSTINGS_COUNT_EXPONENT = 0.33;
+
+        /** The constant cost of performing skipTo on literal indexes */
+        public final static double POINT_LOOKUP_SKIP_COST = 0.5;
+
+        /** The coefficient controlling the increase of the skip cost with the total size of the posting list for point lookup queries. */
+        public final static double POINT_LOOKUP_SKIP_COST_DISTANCE_FACTOR = 0.1;
+        public final static double POINT_LOOKUP_SKIP_COST_DISTANCE_EXPONENT = 0.5;
+
+        /** Cost to open the per-sstable index, read metadata and obtain the iterators.  Affected by cache hit rate. */
+        public final static double SAI_OPEN_COST = 1500.0;
+
+        /** Cost to advance the index iterator to the next key and load the key. Common for literal and numeric indexes. */
+        public final static double SAI_KEY_COST = 0.1;
+
+        /** Cost to begin processing PKs into index ordinals for estimateAnnSortCost */
+        // DC introduced the one-to-many ordinal mapping optimization
+        public final static double ANN_SORT_OPEN_COST = Version.latest().onOrAfter(Version.DC) ? 370 : 4200;
+
+        /** Additional overhead needed to process each input key fed to the ANN index searcher */
+        // DC introduced the one-to-many ordinal mapping optimization
+        public final static double ANN_SORT_KEY_COST = Version.latest().onOrAfter(Version.DC) ? 0.03 : 0.2;
+
+        /** Cost to get a scored key from DiskANN (~rerank cost). Affected by cache hit rate */
+        public final static double ANN_SCORED_KEY_COST = 15;
+
+        /** Cost to perform a coarse (PQ or BQ) in-memory similarity computation */
+        public final static double ANN_SIMILARITY_COST = 0.5;
+
+        /** Cost to load the neighbor list for a DiskANN node. Affected by cache hit rate */
+        public final static double ANN_EDGELIST_COST = 20.0;
+
+        /** Cost to fetch one row from storage. Affected by cache hit rate */
+        public final static double ROW_COST = 100.0;
+
+        /** Additional cost added to row fetch cost per each row cell */
+        public final static double ROW_CELL_COST = 0.4;
+
+        /** Additional cost added to row fetch cost per each serialized byte of the row */
+        public final static double ROW_BYTE_COST = 0.005;
+
+        /** Cost to perform BM25 scoring, per query term */
+        public final static double BM25_SCORE_COST = 0.5;
+    }
+
+    /** Convenience builder for building intersection and union nodes */
+    public static class Builder
+    {
+        final Factory factory;
+        final Operation.OperationType type;
+        final List<KeysIteration> subplans;
+
+        Builder(Factory context, Operation.OperationType type)
+        {
+            this.factory = context;
+            this.type = type;
+            this.subplans = new ArrayList<>(4);
+        }
+
+        public Builder add(KeysIteration subplan)
+        {
+            subplans.add(subplan);
+            return this;
+        }
+
+        public KeysIteration build()
+        {
+            if (type == Operation.OperationType.AND)
+                return factory.intersection(subplans);
+            if (type == Operation.OperationType.OR)
+                return factory.union(subplans);
+
+            // Should never hit this
+            throw new AssertionError("Unexpected builder type: " + type);
+        }
+    }
+
+    /** hit-rate-scale the raw cost */
+    public static double hrs(double raw)
+    {
+        double multiplier = min(1000.0, 1 / hitRateSupplier.getAsDouble());
+        return raw * multiplier;
+    }
+
+    /**
+     * Describes the expected data access patterns for a plan node.
+     * <br>
+     * Each access pattern is assumed to follow uniform distribution and
+     * is represented by a pair of values:
+     * - a count (number of expected occurrences) and
+     * - an average distance to next occurrence (skip distance).
+     * For performance, these are split into arrays of the primitives.
+     * <br>
+     * For example, given:
+     *     counts = [100, 50, 10]
+     *     distances = [1.0, 2.0, 5.0]
+     * This represents:
+     *     - 100 sequential accesses (distance 1.0)
+     *     - 50 accesses with a skip distance of 2.0
+     *     - 10 accesses with a skip distance of 5.0
+     * <br>
+     * This information is used to optimize query execution plans by predicting
+     * how data will be accessed.
+     */
+    protected static final class Access
+    {
+        /** Represents an empty access pattern. */
+        final static Access EMPTY = Access.sequential(0);
+
+        /**
+         * Array of expected occurrence counts for each access pattern.
+         * Each element represents the number of times a particular access pattern
+         * is expected to occur.
+         */
+        final double[] counts;
+
+        /**
+         * Array of skip distances for each access pattern.
+         * Each element represents the skip distance for a particular access pattern.
+         * A distance of 1.0 indicates sequential access.  Smaller distances than 1.0 do not make sense.
+         */
+        final double[] distances;
+
+        /**
+         * The total count of expected accesses across all patterns.
+         * This is the sum of all elements in the counts array.
+         */
+        final double totalCount;
+
+        /**
+         * The total weighted distance of all access patterns.
+         * Calculated as the sum of (count * distance) for all patterns.
+         */
+        final double totalDistance;
+
+        /**
+         * Flag indicating whether to force the use of skip operations.
+         * When true, skip operations are used even for small distances (1.0).
+         */
+        final boolean forceSkip;
+
+        private Access(double[] count, double[] distance, boolean forceSkip)
+        {
+            assert count.length == distance.length;
+            this.counts = count;
+            this.distances = distance;
+            this.forceSkip = forceSkip;
+
+            double totalDistance = 0.0;
+            double totalCount = 0.0;
+            for (int i = 0; i < counts.length; i++)
+            {
+                totalCount += counts[i];
+                totalDistance += counts[i] * distances[i];
+            }
+
+            this.totalDistance = totalDistance;
+            this.totalCount = totalCount;
+        }
+
+        static Access sequential(double count)
+        {
+            return new Access(new double[] { count }, new double[] { 1.0 }, false);
+        }
+
+        /** Scales the counts so that the total count does not exceed given limit */
+        Access limit(long limit)
+        {
+            double totalCount = 0.0;
+            for (int i = 0; i < counts.length; i++)
+                totalCount += counts[i];
+
+            return limit > totalCount
+                   ? this
+                   : this.scaleCount(limit / totalCount);
+        }
+
+        /** Multiplies all counts by a constant without changing the distribution */
+        Access scaleCount(double factor)
+        {
+            assert Double.isFinite(factor) : "Count multiplier must not be finite; got " + factor;
+
+            double[] counts = Arrays.copyOf(this.counts, this.counts.length);
+            double[] skipDistances = Arrays.copyOf(this.distances, this.distances.length);
+            for (int i = 0; i < counts.length; i++)
+                counts[i] *= factor;
+            return new Access(counts, skipDistances, forceSkip);
+        }
+
+        /**
+         * Multiplies all skip distances by a constant
+         * (if constant is > 1, it spreads accesses further away from each other)
+         */
+        Access scaleDistance(double factor)
+        {
+            assert Double.isFinite(factor) : "Distance multiplier must not be finite; got " + factor;
+
+            double[] counts = Arrays.copyOf(this.counts, this.counts.length);
+            double[] skipDistances = Arrays.copyOf(this.distances, this.distances.length);
+            for (int i = 0; i < counts.length; i++)
+                skipDistances[i] *= factor;
+            return new Access(counts, skipDistances, forceSkip);
+        }
+
+        /**
+         * Returns a new Access pattern derived by applying a repeated access pattern to the current one,
+         * to represent the effect of intersecting with another predicate.  That is, given "x intersect y,"
+         * we apply `convolute` to y's Access pattern to account for the skips introduced by x.
+         * <p>
+         * Example (a star denotes a single access):
+         * <pre>
+         * Access.sequential(4).scaleDistance(6):
+         * *     *     *     *
+         * Access.sequential(4).scaleDistance(6).convolute(3, 1):
+         * ***   ***   ***   ***
+         * </pre>
+         * */
+        Access convolute(double count, double skipDistance)
+        {
+            assert !Double.isNaN(count) : "Count must not be NaN";
+            assert !Double.isNaN(skipDistance) : "Skip distance must not be NaN";
+
+            if (count <= 1.0)
+                return scaleCount(count);
+
+            double[] counts = Arrays.copyOf(this.counts, this.counts.length + 1);
+            double[] skipDistances = Arrays.copyOf(this.distances, this.distances.length + 1);
+
+            counts[counts.length - 1] = (count - 1) * totalCount;
+            skipDistances[skipDistances.length - 1] = skipDistance;
+
+            // Because we added new accesses, we need to adjust the distance of the remaining points
+            // in a way that the total distance stays the same:
+            for (int i = 0; i < skipDistances.length - 1; i++)
+                skipDistances[i] -= (count - 1) * skipDistance;
+
+            return new Access(counts, skipDistances, forceSkip);
+        }
+
+        /** Forces using skipTo cost even if skipping distance is not greater than 1 item */
+        Access forceSkip()
+        {
+            return new Access(counts, distances, true);
+        }
+
+        /** Returns the total expected number of items (rows or keys) to be retrieved from the node */
+        double expectedAccessCount(double availableCount)
+        {
+            return totalCount == 0 || totalDistance <= availableCount
+                   ? totalCount
+                   : availableCount / totalDistance * totalCount;
+        }
+
+        /**
+         * Computes the expected cost of fetching one item (row or key).
+         * This is computed as an arithmetic mean of costs of skipping by each distance, weighted by counts.
+         * @param nextCost the cost of fetching one item from the plan node as a function of the skip distance
+         *                 (measured in rows or keys)
+         */
+        double unitCost(double nextCost, Function<Double, Double> skipCostFn)
+        {
+            if (totalCount == 0)
+                return 0.0;  // we don't want NaNs ;)
+
+            double totalCost = 0.0;
+            double totalWeight = 0.0;
+            for (int i = 0; i < counts.length; i++)
+            {
+                double skipCost = (distances[i] > 1.0 || forceSkip) ? skipCostFn.apply(distances[i]) : 0.0;
+                totalCost += counts[i] * (nextCost + skipCost);
+                totalWeight += counts[i];
+            }
+            return totalCost / totalWeight;
+        }
+
+        public double meanDistance()
+        {
+            return totalCount > 0.0 ? totalDistance / totalCount : 0.0;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Access that = (Access) o;
+            return Arrays.equals(counts, that.counts) && Arrays.equals(distances, that.distances);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(Arrays.hashCode(counts), Arrays.hashCode(distances));
+        }
+    }
+
+    /**
+     * Applies given function to given object lazily, only when the result is needed.
+     * Caches the result for subsequent executions.
+     */
+    static class LazyTransform<T>
+    {
+        final T orig;
+        final Function<T, T> transform;
+        private T result;
+
+        LazyTransform(T orig, Function<T, T> transform)
+        {
+            this.orig = orig;
+            this.transform = transform;
+        }
+
+        public T get()
+        {
+            if (result == null)
+                result = transform.apply(orig);
+            return result;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/QueryController.java b/src/java/org/apache/cassandra/index/sai/plan/QueryController.java
index 4f8efb824814..ae88f185d097 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/QueryController.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/QueryController.java
@@ -18,23 +18,33 @@
 
 package org.apache.cassandra.index.sai.plan;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import javax.annotation.Nullable;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.MessageParams;
+import org.apache.cassandra.db.MultiRangeReadCommand;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.PartitionRangeReadCommand;
 import org.apache.cassandra.db.ReadCommand;
@@ -42,63 +52,136 @@
 import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.filter.ClusteringIndexFilter;
 import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.BaseRowIterator;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.VectorQueryContext;
-import org.apache.cassandra.index.sai.disk.IndexSearchResultIterator;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
-import org.apache.cassandra.index.sai.iterators.KeyRangeConcatIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
+import org.apache.cassandra.index.sai.disk.v1.Segment;
+import org.apache.cassandra.index.sai.disk.vector.VectorCompression;
+import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex;
 import org.apache.cassandra.index.sai.iterators.KeyRangeIntersectionIterator;
 import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.iterators.KeyRangeOrderingIterator;
-import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeTermIterator;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.utils.AbortedOperationException;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.net.ParamType;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RowWithSourceTable;
+import org.apache.cassandra.index.sai.utils.RangeUtil;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.index.sai.view.View;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.CloseableIterator;
 import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.MergeIterator;
 import org.apache.cassandra.utils.Throwables;
 
-import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_VECTOR_SEARCH_ORDER_CHUNK_SIZE;
+import static java.lang.Math.max;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_QUERY_OPT_LEVEL;
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
 
-public class QueryController
+public class QueryController implements Plan.Executor, Plan.CostEstimator
 {
-    final QueryContext queryContext;
+    public static final String INDEX_MAY_HAVE_BEEN_DROPPED = "An index may have been dropped. " +
+                                                             StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE;
+    private static final Logger logger = LoggerFactory.getLogger(QueryController.class);
+
+    /**
+     * Controls whether we optimize query plans.
+     * 0 disables the optimizer. As a side effect, hybrid ANN queries will default to FilterSortOrder.SCAN_THEN_FILTER.
+     * 1 enables the optimizer.
+     * Note: the config is not final to simplify testing.
+     */
+    @VisibleForTesting
+    public static int QUERY_OPT_LEVEL = SAI_QUERY_OPT_LEVEL.getInt();
 
     private final ColumnFamilyStore cfs;
     private final ReadCommand command;
-    private final RowFilter indexFilter;
+    private final Orderer orderer;
+    private final QueryContext queryContext;
+    private final IndexFeatureSet indexFeatureSet;
     private final List<DataRange> ranges;
     private final AbstractBounds<PartitionPosition> mergeRange;
+
     private final PrimaryKey.Factory keyFactory;
     private final PrimaryKey firstPrimaryKey;
-    private final PrimaryKey lastPrimaryKey;
-    private final int orderChunkSize;
 
+    final Plan.Factory planFactory;
+
+    /**
+     * Holds the primary key iterators for indexed expressions in the query (i.e. leaves of the expression tree).
+     * We will construct the final iterator from those.
+     * We need a MultiMap because the same Expression can occur more than once in a query.
+     * <p>
+     * Longer explanation why this is needed:
+     * In order to construct a Plan for a query, we need predicate selectivity estimates. But at the moment
+     * of writing this code, the only way to estimate an index predicate selectivity is to look at the posting
+     * list(s) in the index, by obtaining a {@link KeyRangeIterator} and callling {@link KeyRangeIterator#getMaxKeys()} on it.
+     * Hence, we need to create the iterators before creating the Plan.
+     * But later when we assemble the final key iterator according to the optimized Plan, we need those iterators
+     * again. In order to avoid recreating them, which would be costly, we just keep them here in this map.
+     */
+    private final Multimap<Expression, KeyRangeIterator> keyIterators = ArrayListMultimap.create();
+
+    private final Map<IndexContext, QueryView> queryViews = new HashMap<>();
+
+    static
+    {
+        logger.info(String.format("Query plan optimization is %s (level = %d)",
+                                  QUERY_OPT_LEVEL > 0 ? "enabled" : "disabled",
+                                  QUERY_OPT_LEVEL));
+    }
+
+    @VisibleForTesting
     public QueryController(ColumnFamilyStore cfs,
                            ReadCommand command,
-                           RowFilter indexFilter,
+                           IndexFeatureSet indexFeatureSet,
+                           QueryContext queryContext)
+    {
+        this(cfs, command, null, indexFeatureSet, queryContext);
+    }
+
+    public QueryController(ColumnFamilyStore cfs,
+                           ReadCommand command,
+                           Orderer orderer,
+                           IndexFeatureSet indexFeatureSet,
                            QueryContext queryContext)
     {
         this.cfs = cfs;
         this.command = command;
+        this.orderer = orderer;
         this.queryContext = queryContext;
-        this.indexFilter = indexFilter;
+        this.indexFeatureSet = indexFeatureSet;
         this.ranges = dataRanges(command);
         DataRange first = ranges.get(0);
         DataRange last = ranges.get(ranges.size() - 1);
         this.mergeRange = ranges.size() == 1 ? first.keyRange() : first.keyRange().withNewRight(last.keyRange().right);
-        this.keyFactory = new PrimaryKey.Factory(cfs.getPartitioner(), cfs.getComparator());
-        this.firstPrimaryKey = keyFactory.create(mergeRange.left.getToken());
-        this.lastPrimaryKey = keyFactory.create(mergeRange.right.getToken());
-        this.orderChunkSize = SAI_VECTOR_SEARCH_ORDER_CHUNK_SIZE.getInt();
+
+        this.keyFactory = PrimaryKey.factory(cfs.metadata().comparator, indexFeatureSet);
+        this.firstPrimaryKey = keyFactory.createTokenOnly(mergeRange.left.getToken());
+        var tableMetrics = new Plan.TableMetrics(estimateTotalAvailableRows(ranges),
+                                                 avgCellsPerRow(),
+                                                 avgRowSizeInBytes(),
+                                                 cfs.getLiveSSTables().size());
+        this.planFactory = new Plan.Factory(tableMetrics, this);
     }
 
     public PrimaryKey.Factory primaryKeyFactory()
@@ -106,305 +189,597 @@ public PrimaryKey.Factory primaryKeyFactory()
         return keyFactory;
     }
 
-    public PrimaryKey firstPrimaryKeyInRange()
+    public PrimaryKey firstPrimaryKey()
     {
         return firstPrimaryKey;
     }
 
-    public PrimaryKey lastPrimaryKeyInRange()
-    {
-        return lastPrimaryKey;
-    }
 
     public TableMetadata metadata()
     {
         return command.metadata();
     }
 
-    public RowFilter indexFilter()
+    public ReadCommand command()
     {
-        return this.indexFilter;
+        return command;
     }
-    
-    public boolean usesStrictFiltering()
+
+    RowFilter.FilterElement filterOperation()
     {
-        return command.rowFilter().isStrict();
+        // NOTE: we cannot remove the order by filter expression here yet because it is used in the FilterTree class
+        // to filter out shadowed rows.
+        return this.command.rowFilter().root();
     }
 
     /**
      * @return token ranges used in the read command
      */
-    public List<DataRange> dataRanges()
+    List<DataRange> dataRanges()
     {
         return ranges;
     }
 
-    @Nullable
-    public StorageAttachedIndex indexFor(RowFilter.Expression expression)
+    /**
+     * Note: merged range may contain subrange that no longer belongs to the local node after range movement.
+     * It should only be used as an optimization to reduce search space. Use {@link #dataRanges()} instead to filter data.
+     *
+     * @return merged token range
+     */
+    AbstractBounds<PartitionPosition> mergeRange()
+    {
+        return mergeRange;
+    }
+
+    /**
+     * @return indexed {@code ColumnContext} if index is found; otherwise return non-indexed {@code ColumnContext}.
+     */
+    public IndexContext getContext(RowFilter.Expression expression)
     {
-        return cfs.indexManager.getBestIndexFor(expression, StorageAttachedIndex.class).orElse(null);
+        StorageAttachedIndex index = getBestIndexFor(expression);
+
+        if (index != null)
+            return index.getIndexContext();
+
+        return new IndexContext(cfs.metadata().keyspace,
+                                cfs.metadata().name,
+                                cfs.metadata().id,
+                                cfs.metadata().partitionKeyType,
+                                cfs.metadata().comparator,
+                                expression.column(),
+                                determineIndexTargetType(expression),
+                                null,
+                                cfs);
     }
 
-    public boolean hasAnalyzer(RowFilter.Expression expression)
+    /**
+     * Determines the {@link IndexTarget.Type} for the expression. In this case we are only interested in map types and
+     * the operator being used in the expression.
+     */
+    public static IndexTarget.Type determineIndexTargetType(RowFilter.Expression expression)
     {
-        StorageAttachedIndex index = indexFor(expression);
-        return index != null && index.hasAnalyzer();
+        AbstractType<?> type  = expression.column().type;
+        IndexTarget.Type indexTargetType = IndexTarget.Type.SIMPLE;
+        if (type.isCollection() && type.isMultiCell())
+        {
+            CollectionType<?> collection = ((CollectionType<?>) type);
+            if (collection.kind == CollectionType.Kind.MAP)
+            {
+                Operator operator = expression.operator();
+                switch (operator)
+                {
+                    case EQ:
+                    case NEQ:
+                    case LT:
+                    case LTE:
+                    case GT:
+                    case GTE:
+                        indexTargetType = IndexTarget.Type.KEYS_AND_VALUES;
+                        break;
+                    case CONTAINS:
+                    case NOT_CONTAINS:
+                        indexTargetType = IndexTarget.Type.VALUES;
+                        break;
+                    case CONTAINS_KEY:
+                    case NOT_CONTAINS_KEY:
+                        indexTargetType = IndexTarget.Type.KEYS;
+                        break;
+                    default:
+                        throw new InvalidRequestException("Invalid operator " + operator + " for map type");
+                }
+            }
+        }
+        return indexTargetType;
     }
 
-    public UnfilteredRowIterator queryStorage(PrimaryKey key, ReadExecutionController executionController)
+    /**
+     * Get an iterator over the rows for this partition key. Builds a search view that includes all memtables and all
+     * {@link SSTableSet#LIVE} sstables.
+     * @param key
+     * @param executionController
+     * @return
+     */
+    public UnfilteredRowIterator getPartition(PrimaryKey key, ReadExecutionController executionController)
     {
         if (key == null)
             throw new IllegalArgumentException("non-null key required");
 
-        SinglePartitionReadCommand partition = SinglePartitionReadCommand.create(cfs.metadata(),
-                                                                                 command.nowInSec(),
-                                                                                 command.columnFilter(),
-                                                                                 RowFilter.none(),
-                                                                                 DataLimits.NONE,
-                                                                                 key.partitionKey(),
-                                                                                 makeFilter(key));
-
+        SinglePartitionReadCommand partition = getPartitionReadCommand(key, executionController);
         return partition.queryMemtableAndDisk(cfs, executionController);
     }
 
     /**
-     * Build a {@link KeyRangeIterator.Builder} from the given list of {@link Expression}s.
-     * <p>
-     * This is achieved by creating an on-disk view of the query that maps the expressions to
-     * the {@link SSTableIndex}s that will satisfy the expression.
-     * <p>
-     * Each (expression, SSTable indexes) pair is then passed to
-     * {@link IndexSearchResultIterator#build(Expression, Collection, AbstractBounds, QueryContext, boolean, Runnable)}
-     * to search the in-memory index associated with the expression and the SSTable indexes, the results of
-     * which are unioned and returned.
-     * <p>
-     * The results from each call to {@link IndexSearchResultIterator#build(Expression, Collection, AbstractBounds, QueryContext, boolean, Runnable)}
-     * are added to a {@link KeyRangeIntersectionIterator} and returned if strict filtering is allowed.
-     * <p>
-     * If strict filtering is not allowed, indexes are split into two groups according to the repaired status of their 
-     * backing SSTables. Results from searches over the repaired group are added to a 
-     * {@link KeyRangeIntersectionIterator}, which is then added, along with results from searches on the unrepaired
-     * set, to a top-level {@link KeyRangeUnionIterator}, and returned. This is done to ensure that AND queries do not
-     * prematurely filter out matches on un-repaired partial updates. Post-filtering must also take this into
-     * account. (see {@link FilterTree#isSatisfiedBy(DecoratedKey, Row, Row)}) Note that Memtable-attached 
-     * indexes are treated as part of the unrepaired set.
+     * Get an iterator over the rows for this partition key. Restrict the search to the specified view.
+     * @param key
+     * @param executionController
+     * @return
      */
-    public KeyRangeIterator.Builder getIndexQueryResults(Collection<Expression> expressions)
+    public UnfilteredRowIterator getPartition(PrimaryKey key, ColumnFamilyStore.ViewFragment view, ReadExecutionController executionController)
     {
-        // VSTODO move ANN out of expressions and into its own abstraction? That will help get generic ORDER BY support
-        expressions = expressions.stream().filter(e -> e.getIndexOperator() != Expression.IndexOperator.ANN).collect(Collectors.toList());
+        if (key == null)
+            throw new IllegalArgumentException("non-null key required");
 
-        QueryViewBuilder.QueryView queryView = new QueryViewBuilder(expressions, mergeRange).build();
-        Runnable onClose = () -> queryView.referencedIndexes.forEach(SSTableIndex::releaseQuietly);
-        KeyRangeIterator.Builder builder = command.rowFilter().isStrict()
-                                           ? KeyRangeIntersectionIterator.builder(expressions.size(), onClose)
-                                           : KeyRangeUnionIterator.builder(expressions.size(), onClose);
+        SinglePartitionReadCommand partition = getPartitionReadCommand(key, executionController);
 
-        try
+        // Class to transform the row to include its source table.
+        Function<Object, Transformation<BaseRowIterator<?>>> rowTransformer = (Object sourceTable) -> new Transformation<>()
         {
-            maybeTriggerGuardrails(queryView);
-
-            if (command.rowFilter().isStrict())
+            @Override
+            protected Row applyToRow(Row row)
             {
-                // If strict filtering is enabled, evaluate indexes for both repaired and un-repaired SSTables together.
-                // This usually means we are making this local index query in the context of a user query that reads 
-                // from a single replica and thus can safely perform local intersections.
-                for (Pair<Expression, Collection<SSTableIndex>> queryViewPair : queryView.view)
-                    builder.add(IndexSearchResultIterator.build(queryViewPair.left, queryViewPair.right, mergeRange, queryContext, true, () -> {}));
+                return new RowWithSourceTable(row, sourceTable);
             }
-            else
-            {
-                KeyRangeIterator.Builder repairedBuilder = KeyRangeIntersectionIterator.builder(expressions.size(), () -> {});
+        };
 
-                for (Pair<Expression, Collection<SSTableIndex>> queryViewPair : queryView.view)
-                {
-                    // The initial sizes here reflect little more than an effort to avoid resizing for 
-                    // partition-restricted searches w/ LCS:
-                    List<SSTableIndex> repaired = new ArrayList<>(5);
-                    List<SSTableIndex> unrepaired = new ArrayList<>(5);
-
-                    // Split SSTable indexes into repaired and un-reparired:
-                    for (SSTableIndex index : queryViewPair.right)
-                        if (index.getSSTable().isRepaired())
-                            repaired.add(index);
-                        else
-                            unrepaired.add(index);
-
-                    // Always build an iterator for the un-repaired set, given this must include Memtable indexes...  
-                    IndexSearchResultIterator unrepairedIterator =
-                            IndexSearchResultIterator.build(queryViewPair.left, unrepaired, mergeRange, queryContext, true, () -> {});
-
-                    // ...but ignore it if our combined results are empty.
-                    if (unrepairedIterator.getMaxKeys() > 0)
-                    {
-                        builder.add(unrepairedIterator);
-                        queryContext.hasUnrepairedMatches = true;
-                    }
-                    else
-                    {
-                        // We're not going to use this, so release the resources it holds.
-                        unrepairedIterator.close();
-                    }
-
-                    // ...then only add an iterator to the repaired intersection if repaired SSTable indexes exist. 
-                    if (!repaired.isEmpty())
-                        repairedBuilder.add(IndexSearchResultIterator.build(queryViewPair.left, repaired, mergeRange, queryContext, false, () -> {}));
-                }
+        return partition.queryMemtableAndDisk(cfs, view, rowTransformer, executionController);
+    }
 
-                if (repairedBuilder.rangeCount() > 0)
-                    builder.add(repairedBuilder.build());
-            }
+    public SinglePartitionReadCommand getPartitionReadCommand(PrimaryKey key, ReadExecutionController executionController)
+    {
+        if (key == null)
+            throw new IllegalArgumentException("non-null key required");
+
+        return SinglePartitionReadCommand.create(cfs.metadata(),
+                                                 command.nowInSec(),
+                                                 command.columnFilter(),
+                                                 RowFilter.none(),
+                                                 DataLimits.NONE,
+                                                 key.partitionKey(),
+                                                 makeFilter(key));
+    }
+
+    private void updateIndexMetricsQueriesCount(Plan plan)
+    {
+        HashSet<IndexContext> queriedIndexesContexts = new HashSet<>();
+        plan.forEach(node -> {
+            IndexContext indexContext = node.getIndexContext();
+            if (indexContext != null)
+                queriedIndexesContexts.add(indexContext);
+            return Plan.ControlFlow.Continue;
+        });
+        queriedIndexesContexts.forEach(indexContext ->
+                                       indexContext.getIndexMetrics().queriesCount.inc());
+    }
+
+    Plan buildPlan()
+    {
+        Plan.KeysIteration keysIterationPlan = buildKeysIterationPlan();
+        Plan.RowsIteration rowsIteration = planFactory.fetch(keysIterationPlan);
+        rowsIteration = planFactory.recheckFilter(command.rowFilter(), rowsIteration);
+        rowsIteration = planFactory.limit(rowsIteration, command.limits().rows());
+
+        // Limit the number of intersected clauses before optimizing so we reduce the size of the
+        // plan given to the optimizer and hence reduce the plan search space and speed up optimization.
+        // It is possible that some index operators like ':' expand to a huge number of MATCH predicates
+        // (see CNDB-10085) and could overload the optimizer.
+        // The intersected subplans are ordered by selectivity in the way the best ones are at the beginning
+        // of the list, therefore this limit is unlikely to remove good branches of the tree.
+        // The limit here is higher than the final limit, so that the optimizer has a bit more freedom
+        // in which predicates it leaves in the plan and the probability of accidentally removing a good branch
+        // here is even lower.
+        Plan plan = rowsIteration.limitIntersectedClauses(KeyRangeIntersectionIterator.INTERSECTION_CLAUSE_LIMIT * 3);
+
+        if (QUERY_OPT_LEVEL > 0)
+            plan = plan.optimize();
+
+        plan = plan.limitIntersectedClauses(KeyRangeIntersectionIterator.INTERSECTION_CLAUSE_LIMIT);
+
+        if (plan.contains(node -> node instanceof Plan.AnnIndexScan))
+            queryContext.setFilterSortOrder(QueryContext.FilterSortOrder.SCAN_THEN_FILTER);
+        if (plan.contains(node -> node instanceof Plan.KeysSort))
+            queryContext.setFilterSortOrder(QueryContext.FilterSortOrder.SEARCH_THEN_ORDER);
+
+        updateIndexMetricsQueriesCount(plan);
+
+        if (logger.isTraceEnabled())
+            logger.trace("Query execution plan:\n" + plan.toStringRecursive());
+
+        if (Tracing.isTracing())
+        {
+            Tracing.trace("Query execution plan:\n" + plan.toStringRecursive());
+            List<Plan.IndexScan> origIndexScans = keysIterationPlan.nodesOfType(Plan.IndexScan.class);
+            List<Plan.IndexScan> selectedIndexScans = plan.nodesOfType(Plan.IndexScan.class);
+            Tracing.trace("Selecting {} {} of {} out of {} indexes",
+                          selectedIndexScans.size(),
+                          selectedIndexScans.size() > 1 ? "indexes with cardinalities" : "index with cardinality",
+                          selectedIndexScans.stream().map(s -> "" + ((long) s.expectedKeys())).collect(Collectors.joining(", ")),
+                          origIndexScans.size());
         }
-        catch (Throwable t)
+        return plan;
+    }
+
+    private Plan.KeysIteration buildKeysIterationPlan()
+    {
+        // Remove the ORDER BY filter expression from the filter tree, as it is added below.
+        var filterElement = filterOperation().filter(e -> !Orderer.isFilterExpressionOrderer(e));
+        Plan.KeysIteration keysIterationPlan = Operation.Node.buildTree(this, filterElement)
+                                                             .analyzeTree(this)
+                                                             .plan(this);
+
+        // Because the orderer has a specific queue view
+        if (orderer != null)
+            keysIterationPlan = planFactory.sort(keysIterationPlan, orderer);
+
+        // This would mean we have no WHERE nor ANN clauses at all; this can happen in case an index was dropped after the
+        // query was initiated
+        if (keysIterationPlan == planFactory.everything)
+            throw invalidRequest(INDEX_MAY_HAVE_BEEN_DROPPED);
+
+        return keysIterationPlan;
+    }
+
+
+    public Iterator<? extends PrimaryKey> buildIterator(Plan plan)
+    {
+        try
         {
-            // all sstable indexes in view have been referenced, need to clean up when exception is thrown
-            builder.cleanup();
-            throw t;
+            Plan.KeysIteration keysIteration = plan.firstNodeOfType(Plan.KeysIteration.class);
+            assert keysIteration != null : "No index scan found";
+            return keysIteration.execute(this);
+        }
+        finally
+        {
+            // Because we optimize the plan, it is possible that there exist iterators that we
+            // constructed but which weren't used by the final plan.
+            // Let's close them here, so they don't hold the resources.
+            closeUnusedIterators();
         }
-        return builder;
     }
 
-    private void maybeTriggerGuardrails(QueryViewBuilder.QueryView queryView)
+    /**
+     * Creates an iterator over keys of rows that match given WHERE predicate.
+     * Does not cache the iterator!
+     */
+    private KeyRangeIterator buildIterator(Expression predicate)
     {
-        int referencedIndexes = queryView.referencedIndexes.size();
+        QueryView view = getQueryView(predicate.context);
+        return KeyRangeTermIterator.build(predicate, view.referencedIndexes, mergeRange, queryContext, false, Integer.MAX_VALUE);
+    }
+
+    /**
+     * Creates a consistent view of indexes.
+     * Invocations are memorized - multiple calls for the same context return the same view.
+     * The views are kept for the lifetime of this {@code QueryController}.
+     */
+    QueryView getQueryView(IndexContext context) throws QueryView.Builder.MissingIndexException
+    {
+        return queryViews.computeIfAbsent(context,
+                                          c -> new QueryView.Builder(c, mergeRange, queryContext).build());
+
+    }
+
 
-        if (Guardrails.saiSSTableIndexesPerQuery.failsOn(referencedIndexes, null))
+    private float avgCellsPerRow()
+    {
+        long cells = 0;
+        long rows = 0;
+        for (SSTableReader sstable : cfs.getLiveSSTables())
         {
-            String msg = String.format("Query %s attempted to read from too many indexes (%s) but max allowed is %s; " +
-                                       "query aborted (see sai_sstable_indexes_per_query_fail_threshold)",
-                                       command.toCQLString(),
-                                       referencedIndexes,
-                                       Guardrails.CONFIG_PROVIDER.getOrCreate(null).getSaiSSTableIndexesPerQueryFailThreshold());
-            Tracing.trace(msg);
-            MessageParams.add(ParamType.TOO_MANY_REFERENCED_INDEXES_FAIL, referencedIndexes);
-            throw new QueryReferencingTooManyIndexesException(msg);
+            rows += sstable.getTotalRows();
+            cells += sstable.getEstimatedCellPerPartitionCount().mean() * sstable.getEstimatedCellPerPartitionCount().count();
         }
-        else if (Guardrails.saiSSTableIndexesPerQuery.warnsOn(referencedIndexes, null))
+        return rows == 0 ? 0.0f : ((float) cells) / rows;
+    }
+
+    private float avgRowSizeInBytes()
+    {
+        long totalLength = 0;
+        long rows = 0;
+        for (SSTableReader sstable : cfs.getLiveSSTables())
         {
-            MessageParams.add(ParamType.TOO_MANY_REFERENCED_INDEXES_WARN, referencedIndexes);
+            rows += sstable.getTotalRows();
+            totalLength += sstable.uncompressedLength();
         }
+        return rows == 0 ? 0.0f : ((float) totalLength) / rows;
+    }
+
+
+    public FilterTree buildFilter()
+    {
+        return Operation.Node.buildTree(this, filterOperation()).analyzeTree(this).filterTree();
+    }
+
+    private Plan.KeysIteration buildHalfRangeFromInequality(Expression originPredicate, Operator op)
+    {
+        assert originPredicate.getOp() == Expression.Op.NOT_EQ : "assumes inequality";
+        assert originPredicate.lower.value == originPredicate.upper.value : "assumes lower and upper are the same in inequality";
+
+        Expression halfRange = new Expression(originPredicate.context);
+        halfRange.add(op, originPredicate.lower.value.raw);
+        long matchingRowCount = Math.min(estimateMatchingRowCount(halfRange), planFactory.tableMetrics.rows);
+        return planFactory.indexScan(halfRange, matchingRowCount);
     }
 
     /**
-     * Returns whether this query is not selecting the {@link PrimaryKey}.
-     * The query does not select the key if both of the following statements are false:
-     *  1. The table associated with the query is not using clustering keys
-     *  2. The clustering index filter for the command wants the row.
-     * <p>
-     *  Item 2 is important in paged queries where the {@link org.apache.cassandra.db.filter.ClusteringIndexSliceFilter} for
-     *  subsequent paged queries may not select rows that are returned by the index
-     *  search because that is initially partition based.
+     * Builds a plan for a restriction with inequality. It's implemented as
+     * union of two ranges, before the value and after the value.
+     * If the column type is truncatable, e.g., BigInteger or BigDecimal,
+     * then it returns a full index scan, since the ranges might result
+     * in false negatives when a truncated value is equivalent to
+     * the value to exclude.
+     * @param predicate Inequality expression with indexContext
+     * @return A plan on the index, which can also result false positives.
+     */
+    private Plan.KeysIteration buildInequalityPlan(Expression predicate)
+    {
+        assert predicate.getOp()== Expression.Op.NOT_EQ : "Only inequality predicate is expected";
+
+        if (TypeUtil.supportsRounding(predicate.validator))
+            return planFactory.fullIndexScan(predicate.context);
+        else
+        {
+            Plan.KeysIteration left = buildHalfRangeFromInequality(predicate, Operator.LT);
+            Plan.KeysIteration right = buildHalfRangeFromInequality(predicate, Operator.GT);
+            return planFactory.union(new ArrayList<>(Arrays.asList(left, right)));
+        }
+    }
+
+    /**
+     * Build a {@link Plan} from the given list of expressions by applying given operation (OR/AND).
+     * Building of such builder involves index search, results of which are persisted in the internal resources list
      *
-     * @param key The {@link PrimaryKey} to be tested
-     * @return true if the key is not selected by the query
+     * @param builder The plan node builder which receives the built index scans
+     * @param expressions The expressions to build the plan from
      */
-    public boolean doesNotSelect(PrimaryKey key)
+    void buildPlanForExpressions(Plan.Builder builder, Collection<Expression> expressions)
     {
-        return key.kind() == PrimaryKey.Kind.WIDE && !command.clusteringIndexFilter(key.partitionKey()).selects(key.clustering());
+        Operation.OperationType op = builder.type;
+        assert !expressions.isEmpty() : "expressions should not be empty for " + op + " in " + command.rowFilter().root();
+
+        assert !expressions.stream().anyMatch(e -> e.operation == Expression.Op.ORDER_BY);
+
+        // we cannot use indexes with OR if we have a mix of indexed and non-indexed columns (see CNDB-10142)
+        if (op == Operation.OperationType.OR && !expressions.stream().allMatch(e -> e.context.isIndexed()))
+        {
+            builder.add(planFactory.everything);
+            return;
+        }
+
+        for (Expression expression : expressions)
+        {
+            if (expression.context.isIndexed())
+            {
+                if ( expression.getOp() == Expression.Op.NOT_EQ)
+                    builder.add(buildInequalityPlan(expression));
+                else
+                {
+                    long expectedMatchingRowCount = Math.min(estimateMatchingRowCount(expression), planFactory.tableMetrics.rows);
+                    builder.add(planFactory.indexScan(expression, expectedMatchingRowCount));
+                }
+            }
+        }
     }
 
-    // This is an ANN only query
-    public KeyRangeIterator getTopKRows(RowFilter.Expression expression)
+    @Override
+    public Iterator<? extends PrimaryKey> getKeysFromIndex(Expression predicate)
     {
-        assert expression.operator() == Operator.ANN;
-        StorageAttachedIndex index = indexFor(expression);
-        assert index != null;
-        var planExpression = Expression.create(index).add(Operator.ANN, expression.getIndexValue().duplicate());
-        // search memtable before referencing sstable indexes; otherwise we may miss newly flushed memtable index
-        KeyRangeIterator memtableResults = index.memtableIndexManager().searchMemtableIndexes(queryContext, planExpression, mergeRange);
+        Collection<KeyRangeIterator> rangeIterators = keyIterators.get(predicate);
+        // This will be non-empty only if we created the iterator as part of the query planning process.
+        if (!rangeIterators.isEmpty())
+        {
+            KeyRangeIterator iterator = rangeIterators.iterator().next();
+            keyIterators.remove(predicate, iterator);  // remove so we never accidentally reuse the same iterator
+            return iterator;
+        }
 
-        QueryViewBuilder.QueryView queryView = new QueryViewBuilder(Collections.singleton(planExpression), mergeRange).build();
-        Runnable onClose = () -> queryView.referencedIndexes.forEach(SSTableIndex::releaseQuietly);
+        return buildIterator(predicate);
+    }
 
+    /**
+     * Use the configured {@link Orderer} to create an iterator that sorts the whole table by a specific column.
+     */
+    @Override
+    public CloseableIterator<PrimaryKeyWithSortKey> getTopKRows(Expression predicate, int softLimit)
+    {
+        List<CloseableIterator<PrimaryKeyWithSortKey>> memtableResults = new ArrayList<>();
         try
         {
-            List<KeyRangeIterator> sstableIntersections = queryView.view
-                                                                   .stream()
-                                                                   .map(this::createRowIdIterator)
-                                                                   .collect(Collectors.toList());
-
-            return IndexSearchResultIterator.build(sstableIntersections, memtableResults, queryView.referencedIndexes, queryContext, onClose);
+            QueryView view = getQueryView(orderer.context);
+            for (MemtableIndex index : view.memtableIndexes)
+                memtableResults.addAll(index.orderBy(queryContext, orderer, predicate, mergeRange, softLimit));
+
+            var totalRows = view.getTotalSStableRows();
+            SSTableSearcher searcher = index -> index.orderBy(orderer, predicate, mergeRange, queryContext, softLimit, totalRows);
+            var sstableResults = searchSSTables(view, searcher);
+            sstableResults.addAll(memtableResults);
+            return MergeIterator.getNonReducingCloseable(sstableResults, orderer.getComparator());
+        }
+        catch (QueryView.Builder.MissingIndexException e)
+        {
+            if (orderer.context.isDropped())
+                throw invalidRequest(TopKProcessor.INDEX_MAY_HAVE_BEEN_DROPPED);
+            else
+                throw new IllegalStateException("Index not found but hasn't been dropped", e);
         }
         catch (Throwable t)
         {
-            // all sstable indexes in view have been referenced, need to clean up when exception is thrown
-            onClose.run();
+            FileUtils.closeQuietly(memtableResults);
             throw t;
         }
     }
 
-    // This is a hybrid query. We apply all other predicates before ordering and limiting.
-    public KeyRangeIterator getTopKRows(KeyRangeIterator source, RowFilter.Expression expression)
+    /**
+     * Use the configured {@link Orderer} to sort the rows from the given source iterator.
+     */
+    public CloseableIterator<PrimaryKeyWithSortKey> getTopKRows(KeyRangeIterator source, int softLimit)
     {
-        return new KeyRangeOrderingIterator(source, orderChunkSize, list -> this.getTopKRows(list, expression));
+        try
+        {
+            var primaryKeys = materializeKeys(source);
+            if (primaryKeys.isEmpty())
+            {
+                FileUtils.closeQuietly(source);
+                return CloseableIterator.emptyIterator();
+            }
+            var result = getTopKRows(primaryKeys, softLimit);
+            // We cannot close the source iterator eagerly because it produces partially loaded PrimaryKeys
+            // that might not be needed until a deeper search into the ordering index, which happens after
+            // we exit this block.
+            return CloseableIterator.withOnClose(result, source);
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(source);
+            throw t;
+        }
     }
 
-    private KeyRangeIterator getTopKRows(List<PrimaryKey> rawSourceKeys, RowFilter.Expression expression)
+    /**
+     * Materialize the keys from the given source iterator. If there is a meaningful {@link #mergeRange}, the keys
+     * are filtered to only include those within the range. Note: does not close the source iterator.
+     * @param source The source iterator to materialize keys from.
+     * @return The list of materialized keys within the {@link #mergeRange}.
+     */
+    private List<PrimaryKey> materializeKeys(KeyRangeIterator source)
     {
-        VectorQueryContext vectorQueryContext = queryContext.vectorContext();
-        // Filter out PKs now. Each PK is passed to every segment of the ANN index, so filtering shadowed keys
-        // eagerly can save some work when going from PK to row id for on disk segments.
-        // Since the result is shared with multiple streams, we use an unmodifiable list.
-        var sourceKeys = rawSourceKeys.stream().filter(vectorQueryContext::shouldInclude).collect(Collectors.toList());
-        StorageAttachedIndex index = indexFor(expression);
-        assert index != null : "Cannot do ANN ordering on an unindexed column";
-        var planExpression = Expression.create(index);
-        planExpression.add(Operator.ANN, expression.getIndexValue().duplicate());
-
-        // search memtable before referencing sstable indexes; otherwise we may miss newly flushed memtable index
-        KeyRangeIterator memtableResults = index.memtableIndexManager().limitToTopResults(queryContext, sourceKeys, planExpression);
-        QueryViewBuilder.QueryView queryView = new QueryViewBuilder(Collections.singleton(planExpression), mergeRange).build();
-        Runnable onClose = () -> queryView.referencedIndexes.forEach(SSTableIndex::releaseQuietly);
+        // Skip to the first key in the range
+        source.skipTo(primaryKeyFactory().createTokenOnly(mergeRange.left.getToken()));
+        if (!source.hasNext())
+            return List.of();
+
+        var maxToken = primaryKeyFactory().createTokenOnly(mergeRange.right.getToken());
+        var hasLimitingMaxToken = !maxToken.token().isMinimum() && maxToken.compareTo(source.getMaximum()) < 0;
+        List<PrimaryKey> primaryKeys = new ArrayList<>();
+        while (source.hasNext())
+        {
+            var next = source.next();
+            if (hasLimitingMaxToken && next.compareTo(maxToken) > 0)
+                break;
+            primaryKeys.add(next);
+        }
+        return primaryKeys;
+    }
 
+    private CloseableIterator<PrimaryKeyWithSortKey> getTopKRows(List<PrimaryKey> sourceKeys, int softLimit)
+    {
+        Tracing.logAndTrace(logger, "SAI predicates produced {} keys", sourceKeys.size());
+        List<CloseableIterator<PrimaryKeyWithSortKey>> memtableResults = null;
         try
         {
-            List<KeyRangeIterator> sstableIntersections = queryView.view
-                                                                   .stream()
-                                                                   .flatMap(pair -> pair.right.stream())
-                                                                   .map(idx -> {
-                                                                       try
-                                                                       {
-                                                                           return idx.limitToTopKResults(queryContext, sourceKeys, planExpression);
-                                                                       }
-                                                                       catch (IOException e)
-                                                                       {
-                                                                           throw new UncheckedIOException(e);
-                                                                       }
-                                                                   })
-                                                                   .collect(Collectors.toList());
-
-            return IndexSearchResultIterator.build(sstableIntersections, memtableResults, queryView.referencedIndexes, queryContext, onClose);
+            QueryView view = getQueryView(orderer.context);
+            memtableResults = view.memtableIndexes.stream()
+                                                  .map(index -> index.orderResultsBy(queryContext,
+                                                                                     sourceKeys,
+                                                                                     orderer,
+                                                                                     softLimit))
+                                                  .collect(Collectors.toList());
+            var totalRows = view.getTotalSStableRows();
+            SSTableSearcher ssTableSearcher = index -> index.orderResultsBy(queryContext,
+                                                                            sourceKeys,
+                                                                            orderer,
+                                                                            softLimit,
+                                                                            totalRows);
+            var sstableScoredPrimaryKeyIterators = searchSSTables(view, ssTableSearcher);
+            sstableScoredPrimaryKeyIterators.addAll(memtableResults);
+            return MergeIterator.getNonReducingCloseable(sstableScoredPrimaryKeyIterators, orderer.getComparator());
+        }
+        catch (QueryView.Builder.MissingIndexException e)
+        {
+            if (orderer.context.isDropped())
+                throw invalidRequest(TopKProcessor.INDEX_MAY_HAVE_BEEN_DROPPED);
+            else
+                throw new IllegalStateException("Index not found but hasn't been dropped", e);
         }
         catch (Throwable t)
         {
-            // all sstable indexes in view have been referenced, need to clean up when exception is thrown
-            onClose.run();
+            if (memtableResults != null)
+                FileUtils.closeQuietly(memtableResults);
             throw t;
         }
+
+    }
+
+
+    @FunctionalInterface
+    interface SSTableSearcher
+    {
+        List<CloseableIterator<PrimaryKeyWithSortKey>> search(SSTableIndex index) throws Exception;
     }
 
     /**
-     * Create row id iterator from different indexes' on-disk searcher of the same sstable
+     * Create the list of iterators over {@link PrimaryKeyWithSortKey} from the given {@link QueryView}.
+     * @param queryView The view to use to create the iterators.
+     * @return The list of iterators over {@link PrimaryKeyWithSortKey}.
      */
-    private KeyRangeIterator createRowIdIterator(Pair<Expression, Collection<SSTableIndex>> indexExpression)
-    {
-        var subIterators = indexExpression.right
-                           .stream()
-                           .map(index ->
-                                {
-                                    try
-                                    {
-                                        List<KeyRangeIterator> iterators = index.search(indexExpression.left, mergeRange, queryContext);
-                                        // concat the result from multiple segments for the same index
-                                        return KeyRangeConcatIterator.builder(iterators.size()).add(iterators).build();
-                                    }
-                                    catch (Throwable ex)
-                                    {
-                                        throw Throwables.cleaned(ex);
-                                    }
-                                }).collect(Collectors.toList());
-
-        return KeyRangeUnionIterator.build(subIterators);
+    private List<CloseableIterator<PrimaryKeyWithSortKey>> searchSSTables(QueryView queryView, SSTableSearcher searcher)
+    {
+        List<CloseableIterator<PrimaryKeyWithSortKey>> results = new ArrayList<>();
+        for (var index : queryView.referencedIndexes)
+        {
+            try
+            {
+                var iterators = searcher.search(index);
+                results.addAll(iterators);
+            }
+            catch (Throwable ex)
+            {
+                // Close any iterators that were successfully opened before the exception
+                FileUtils.closeQuietly(results);
+                if (logger.isDebugEnabled() && !(ex instanceof AbortedOperationException))
+                {
+                    var msg = String.format("Failed search on index %s, aborting query.", index.getSSTable());
+                    logger.debug(index.getIndexContext().logMessage(msg), ex);
+                }
+                throw Throwables.cleaned(ex);
+            }
+        }
+        return results;
+    }
+
+    public IndexFeatureSet indexFeatureSet()
+    {
+        return indexFeatureSet;
+    }
+
+    public Orderer getOrderer()
+    {
+        return orderer;
+    }
+
+    /**
+     * Returns whether this query is selecting the {@link PrimaryKey}.
+     * The query selects the key if any of the following statements is true:
+     *  1. The query is not row-aware
+     *  2. The table associated with the query is not using clustering keys
+     *  3. The clustering index filter for the command wants the row.
+     *
+     *  Item 3 is important in paged queries where the {@link ClusteringIndexSliceFilter} for
+     *  subsequent paged queries may not select rows that are returned by the index
+     *  search because that is initially partition based.
+     *
+     * @param key The {@link PrimaryKey} to be tested
+     * @return true if the key is selected by the query
+     */
+    public boolean selects(PrimaryKey key)
+    {
+        return !indexFeatureSet.isRowAware() ||
+               key.hasEmptyClustering() ||
+               command.clusteringIndexFilter(key.partitionKey()).selects(key.clustering());
+    }
+
+    private StorageAttachedIndex getBestIndexFor(RowFilter.Expression expression)
+    {
+        return cfs.indexManager.getBestIndexFor(expression, StorageAttachedIndex.class).orElse(null);
     }
 
     // Note: This method assumes that the selects method has already been called for the
@@ -413,19 +788,58 @@ private ClusteringIndexFilter makeFilter(PrimaryKey key)
     {
         ClusteringIndexFilter clusteringIndexFilter = command.clusteringIndexFilter(key.partitionKey());
 
-        assert cfs.metadata().comparator.size() == 0 && !key.kind().hasClustering ||
-               cfs.metadata().comparator.size() > 0 && key.kind().hasClustering :
-               "PrimaryKey " + key + " clustering does not match table. There should be a clustering of size " + cfs.metadata().comparator.size();
-
-        // If we have skinny partitions or the key is for a static row then we need to get the partition as
-        // requested by the original query.
-        if (cfs.metadata().comparator.size() == 0 || key.kind() == PrimaryKey.Kind.STATIC)
+        if (!indexFeatureSet.isRowAware() || key.hasEmptyClustering())
             return clusteringIndexFilter;
         else
             return new ClusteringIndexNamesFilter(FBUtilities.singleton(key.clustering(), cfs.metadata().comparator),
                                                   clusteringIndexFilter.isReversed());
     }
 
+    /**
+     * Used to release all resources and record metrics when query finishes.
+     */
+    public void finish()
+    {
+        closeUnusedIterators();
+        closeQueryViews();
+    }
+
+    /**
+     * Releases all resources and does not record the metrics.
+     */
+    public void abort()
+    {
+        closeUnusedIterators();
+        closeQueryViews();
+    }
+
+    private void closeUnusedIterators()
+    {
+        Iterator<Map.Entry<Expression, KeyRangeIterator>> entries = keyIterators.entries().iterator();
+        while (entries.hasNext())
+        {
+            FileUtils.closeQuietly(entries.next().getValue());
+            entries.remove();
+        }
+    }
+
+    /**
+     * Try to reference all SSTableIndexes before querying on disk indexes.
+     *
+     * If we attempt to proceed into {@link KeyRangeTermIterator#build(Expression, Set, AbstractBounds, QueryContext, boolean, int)}
+     * without first referencing all indexes, a concurrent compaction may decrement one or more of their backing
+     * SSTable {@link Ref} instances. This will allow the {@link SSTableIndex} itself to be released and will fail the query.
+     */
+    private void closeQueryViews()
+    {
+        Iterator<Map.Entry<IndexContext, QueryView>> entries = queryViews.entrySet().iterator();
+        while (entries.hasNext())
+        {
+            entries.next().getValue().close();
+            entries.remove();
+        }
+    }
+
     /**
      * Returns the {@link DataRange} list covered by the specified {@link ReadCommand}.
      *
@@ -436,15 +850,141 @@ private static List<DataRange> dataRanges(ReadCommand command)
     {
         if (command instanceof SinglePartitionReadCommand)
         {
-            return Lists.newArrayList(command.dataRange());
+            SinglePartitionReadCommand cmd = (SinglePartitionReadCommand) command;
+            DecoratedKey key = cmd.partitionKey();
+            return Lists.newArrayList(new DataRange(new Bounds<>(key, key), cmd.clusteringIndexFilter()));
         }
         else if (command instanceof PartitionRangeReadCommand)
         {
             return Lists.newArrayList(command.dataRange());
         }
+        else if (command instanceof MultiRangeReadCommand)
+        {
+            MultiRangeReadCommand cmd = (MultiRangeReadCommand) command;
+            return cmd.ranges();
+        }
         else
         {
             throw new AssertionError("Unsupported read command type: " + command.getClass().getName());
         }
     }
+
+    /**
+     * Returns the total count of rows in the sstables which overlap with any of the given ranges
+     * and all live memtables.
+     */
+    private long estimateTotalAvailableRows(List<DataRange> ranges)
+    {
+        long rows = 0;
+
+        for (Memtable memtable : cfs.getAllMemtables())
+            rows += Memtable.estimateRowCount(memtable);
+
+        for (SSTableReader sstable : cfs.getLiveSSTables())
+            for (DataRange range : ranges)
+                if (RangeUtil.intersects(sstable, range.keyRange()))
+                    rows += sstable.getTotalRows();
+
+        return rows;
+    }
+
+    /**
+     * Estimates how many rows match the predicate.
+     * There are no guarantees. The returned value may come with a significant estimation error.
+     * You must not rely on this except for query optimization purposes.
+     */
+    private long estimateMatchingRowCount(Expression predicate)
+    {
+        switch (predicate.getOp())
+        {
+            case EQ:
+            case MATCH:
+            case CONTAINS_KEY:
+            case CONTAINS_VALUE:
+            case NOT_EQ:
+            case NOT_CONTAINS_KEY:
+            case NOT_CONTAINS_VALUE:
+            case RANGE:
+                return (indexFeatureSet.hasTermsHistogram())
+                       ? estimateMatchingRowCountUsingHistograms(predicate)
+                       : estimateMatchingRowCountUsingIndex(predicate);
+            default:
+                return estimateMatchingRowCountUsingIndex(predicate);
+        }
+    }
+
+    /**
+     * Estimates the number of matching rows by consulting the terms histograms on the indexes.
+     * This is faster but the histograms are not available on indexes before V6.
+     */
+    private long estimateMatchingRowCountUsingHistograms(Expression predicate)
+    {
+        assert indexFeatureSet.hasTermsHistogram();
+        var context = predicate.context;
+
+        Collection<MemtableIndex> memtables = context.getLiveMemtables().values();
+        long rowCount = 0;
+        for (MemtableIndex index : memtables)
+            rowCount += index.estimateMatchingRowsCount(predicate, mergeRange);
+
+        var queryView = context.getView();
+        for (SSTableIndex index : queryView.getIndexes())
+            rowCount += index.estimateMatchingRowsCount(predicate, mergeRange);
+
+        return rowCount;
+    }
+
+    /**
+     * Legacy way of estimating predicate selectivity.
+     * Runs the search on the index and returns the size of the iterator.
+     * Caches the iterator for future use, to avoid doing search twice.
+     */
+    private long estimateMatchingRowCountUsingIndex(Expression predicate)
+    {
+        // For older indexes we don't have histograms, so we need to construct the iterator
+        // and ask for the posting list size.
+        KeyRangeIterator iterator = buildIterator(predicate);
+
+        // We're not going to consume the iterator here, so memorize it for future uses.
+        // It can be used when executing the plan.
+        keyIterators.put(predicate, iterator);
+        return iterator.getMaxKeys();
+    }
+
+
+    @Override
+    public double estimateAnnSearchCost(Orderer orderer, int limit, long candidates)
+    {
+        Preconditions.checkArgument(limit > 0, "limit must be > 0");
+
+        IndexContext context = orderer.context;
+        Collection<MemtableIndex> memtables = context.getLiveMemtables().values();
+        View queryView = context.getView();
+
+        int memoryRerankK = orderer.rerankKFor(limit, VectorCompression.NO_COMPRESSION);
+        double cost = 0;
+        for (MemtableIndex index : memtables)
+        {
+            // FIXME convert nodes visited to search cost
+            int memtableCandidates = (int) Math.min(Integer.MAX_VALUE, candidates);
+            cost += ((VectorMemtableIndex) index).estimateAnnNodesVisited(memoryRerankK, memtableCandidates);
+        }
+
+        long totalRows = 0;
+        for (SSTableIndex index : queryView.getIndexes())
+            totalRows += index.getSSTable().getTotalRows();
+
+        for (SSTableIndex index : queryView.getIndexes())
+        {
+            for (Segment segment : index.getSegments())
+            {
+                if (!segment.intersects(mergeRange))
+                    continue;
+                int segmentLimit = segment.proportionalAnnLimit(limit, totalRows);
+                int segmentCandidates = max(1, (int) (candidates * (double) segment.metadata.numRows / totalRows));
+                cost += segment.estimateAnnSearchCost(orderer, segmentLimit, segmentCandidates);
+            }
+        }
+        return cost;
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/plan/QueryView.java b/src/java/org/apache/cassandra/index/sai/plan/QueryView.java
new file mode 100644
index 000000000000..4479d9e18899
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/QueryView.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.utils.RangeUtil;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+
+public class QueryView implements AutoCloseable
+{
+    final ColumnFamilyStore.RefViewFragment view;
+    final Set<SSTableIndex> referencedIndexes;
+    final Set<MemtableIndex> memtableIndexes;
+    final IndexContext indexContext;
+
+    public QueryView(ColumnFamilyStore.RefViewFragment view,
+                     Set<SSTableIndex> referencedIndexes,
+                     Set<MemtableIndex> memtableIndexes,
+                     IndexContext indexContext)
+    {
+
+        this.view = view;
+        this.referencedIndexes = referencedIndexes;
+        this.memtableIndexes = memtableIndexes;
+        this.indexContext = indexContext;
+    }
+
+    @Override
+    public void close()
+    {
+        view.release();
+        referencedIndexes.forEach(SSTableIndex::release);
+    }
+
+    /**
+     * Returns the total count of rows in all sstables in this view
+     */
+    public long getTotalSStableRows()
+    {
+        return view.sstables.stream().mapToLong(SSTableReader::getTotalRows).sum();
+    }
+
+    /**
+     * Build a query specific view of the memtables, sstables, and indexes for a query.
+     * For use with SAI ordered queries to ensure that the view is consistent over the lifetime of the query,
+     * which is particularly important for validation of a cell's source memtable/sstable.
+     */
+    static class Builder
+    {
+        private static final Logger logger = LoggerFactory.getLogger(Builder.class);
+
+        private final ColumnFamilyStore cfs;
+        private final IndexContext indexContext;
+        private final AbstractBounds<PartitionPosition> range;
+        private final QueryContext queryContext;
+
+        Builder(IndexContext indexContext, AbstractBounds<PartitionPosition> range, QueryContext queryContext)
+        {
+            this.cfs = indexContext.columnFamilyStore();
+            this.indexContext = indexContext;
+            this.range = range;
+            this.queryContext = queryContext;
+        }
+
+        /**
+         * Denotes a situation when there exist no index for an active memtable or sstable.
+         * This can happen e.g. when the index gets dropped while running the query.
+         */
+        static class MissingIndexException extends RuntimeException
+        {
+            final IndexContext context;
+            final String dataObjectName;
+
+            private MissingIndexException(IndexContext context, String dataObjectName)
+            {
+                super();
+                this.context = context;
+                this.dataObjectName = dataObjectName;
+            }
+
+            public static MissingIndexException forSSTable(IndexContext context, Descriptor descriptor)
+            {
+                return new MissingIndexException(context, "sstable " + descriptor);
+            }
+
+            public static MissingIndexException forMemtable(IndexContext context, Memtable memtable)
+            {
+                return new MissingIndexException(context, "memtable " + memtable);
+            }
+
+            @Override
+            public String getMessage()
+            {
+                return "Index " + context.getIndexName() + " not found for " + dataObjectName;
+            }
+        }
+
+        /**
+         * Acquire references to all the memtables, memtable indexes, sstables, and sstable indexes required for the
+         * given expression.
+         */
+        protected QueryView build() throws MissingIndexException
+        {
+            var referencedIndexes = new HashSet<SSTableIndex>();
+            ColumnFamilyStore.RefViewFragment refViewFragment = null;
+
+            // We must use the canonical view in order for the equality check for source sstable/memtable
+            // to work correctly.
+            var filter = RangeUtil.coversFullRing(range)
+                         ? View.selectFunction(SSTableSet.CANONICAL)
+                         : View.select(SSTableSet.CANONICAL, s -> RangeUtil.intersects(s, range));
+
+
+            try
+            {
+                // Keeps track of which memtables we've already tried to match the index to.
+                // If we fail to match the index to the memtable for the first time, we have to retry
+                // because the memtable could be flushed and its index removed between the moment we
+                // got the view and the moment we did the lookup.
+                // If we get the same memtable in the view again, and there is no index,
+                // then the missing index is not due to a concurrent modification, but it doesn't contain indexed
+                // data, so we can ignore it.
+                var processedMemtables = new HashSet<Memtable>();
+
+
+                var start = MonotonicClock.Global.approxTime.now();
+                Memtable unmatchedMemtable = null;
+                Descriptor unmatchedSStable = null;
+
+                // This loop will spin only if there is a mismatch between the view managed by IndexViewManager
+                // and the view managed by Cassandra Tracker. Such a mismatch can happen at the moment when
+                // the sstable or memtable sets are updated, e.g. on flushes or compactions. The mismatch
+                // should last only until all Tracker notifications get processed by SAI
+                // (which doesn't involve I/O and should be very fast). We expect the mismatch to resolve in order
+                // of nanoceconds, but the timeout is large enough just in case of unpredictable performance hiccups.
+                outer:
+                while (!MonotonicClock.Global.approxTime.isAfter(start + TimeUnit.MILLISECONDS.toNanos(2000)))
+                {
+                    // cleanup after the previous iteration if we're retrying
+                    release(referencedIndexes);
+                    release(refViewFragment);
+
+                    // Prevent exceeding the query timeout
+                    queryContext.checkpoint();
+
+                    // Lock a consistent view of memtables and sstables.
+                    // A consistent view is required for correctness of order by and vector queries.
+                    refViewFragment = cfs.selectAndReference(filter);
+                    var indexView = indexContext.getView();
+
+                    // Lookup the indexes corresponding to memtables:
+                    var memtableIndexes = new HashSet<MemtableIndex>();
+                    for (Memtable memtable : refViewFragment.memtables)
+                    {
+                        // Empty memtables have no index but that's not a problem, we can ignore them.
+                        if (memtable.getLiveDataSize() == 0)
+                            continue;
+
+                        MemtableIndex index = indexContext.getMemtableIndex(memtable);
+                        if (index != null)
+                        {
+                            memtableIndexes.add(index);
+                        }
+                        else if (indexContext.isDropped())
+                        {
+                            // Index was dropped deliberately by the user.
+                            // We cannot recover here.
+                            throw MissingIndexException.forMemtable(indexContext, memtable);
+                        }
+                        else if (!processedMemtables.contains(memtable))
+                        {
+                            // We can end up here if a flush happened right after we referenced the refViewFragment
+                            // but before looking up the memtable index.
+                            // In that case, we need to retry with the updated view
+                            // (we expect the updated view to not contain this memtable).
+
+                            // Remember this metable to protect from infinite looping in case we have a permanent
+                            // inconsistency between the index set and the memtable set.
+                            processedMemtables.add(memtable);
+
+                            unmatchedMemtable = memtable;
+                            continue outer;
+                        }
+                        // If the memtable was non-empty, the index context hasn't been dropped, but the
+                        // index doesn't exist on the second attempt, then his means there is no indexed data
+                        // in that memtable. In this case we just continue without it.
+                        // Memtable indexes are created lazily, on the first insert, therefore a missing index
+                        // is a normal situation.
+                    }
+
+                    // Lookup and reference the indexes corresponding to the sstables:
+                    for (SSTableReader sstable : refViewFragment.sstables)
+                    {
+                        // Empty sstables are ok to not have the index.
+                        if (sstable.getTotalRows() == 0)
+                            continue;
+
+                        // If the IndexViewManager never saw this sstable, then we need to spin.
+                        // Let's hope in the next iteration we get the indexView based on the same sstable set
+                        // as the refViewFragment.
+                        if (!indexView.isAwareOfSSTable(sstable.descriptor))
+                        {
+                            if (MonotonicClock.Global.approxTime.isAfter(start + 100))
+                                NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS,
+                                                 "Spinning trying to get the index for sstable {} because index view is out of sync", sstable.descriptor);
+
+                            unmatchedSStable = sstable.descriptor;
+                            continue outer;
+                        }
+
+                        SSTableIndex index = indexView.getSSTableIndex(sstable.descriptor);
+
+                        // The IndexViewManager got the update about this sstable, but there is no index for the sstable
+                        // (e.g. index was dropped or got corrupt, etc.). In this case retrying won't fix it.
+                        if (index == null)
+                            throw MissingIndexException.forSSTable(indexContext, sstable.descriptor);
+
+                        if (!indexInRange(index))
+                            continue;
+
+                        // It is unlikely but possible the index got unreferenced just between the moment we grabbed the
+                        // refViewFragment and getting here. In that case we won't be able to reference it and we have
+                        // to retry.
+                        if (!index.reference())
+                        {
+
+                            if (MonotonicClock.Global.approxTime.isAfter(start + 100))
+                                NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS,
+                                                 "Spinning trying to get the index for sstable {} because index was released", sstable.descriptor);
+
+                            unmatchedSStable = sstable.descriptor;
+                            continue outer;
+                        }
+
+                        referencedIndexes.add(index);
+                    }
+
+                    // freeze referencedIndexes and memtableIndexes, so we can safely give access to them
+                    // without risking something messes them up
+                    // (this was added after KeyRangeTermIterator messed them up which led to a bug)
+                    return new QueryView(refViewFragment,
+                                         Collections.unmodifiableSet(referencedIndexes),
+                                         Collections.unmodifiableSet(memtableIndexes),
+                                         indexContext);
+                }
+
+                if (unmatchedMemtable != null)
+                    throw MissingIndexException.forMemtable(indexContext, unmatchedMemtable);
+                if (unmatchedSStable != null)
+                    throw MissingIndexException.forSSTable(indexContext, unmatchedSStable);
+
+                // This should be unreachable, because whenever we retry, we always set unmatchedMemtable
+                // or unmatchedSSTable, so we'd log a better message above.
+                throw new AssertionError("Failed to build QueryView for index " + indexContext.getIndexName());
+            }
+            catch (MissingIndexException e)
+            {
+                release(referencedIndexes);
+                release(refViewFragment);
+                throw e;
+            }
+            finally
+            {
+                if (Tracing.isTracing())
+                {
+                    var groupedIndexes = referencedIndexes.stream().collect(
+                    Collectors.groupingBy(i -> i.getIndexContext().getIndexName(), Collectors.counting()));
+                    var summary = groupedIndexes.entrySet().stream()
+                                                .map(e -> String.format("%s (%s sstables)", e.getKey(), e.getValue()))
+                                                .collect(Collectors.joining(", "));
+                    Tracing.trace("Querying storage-attached indexes {}", summary);
+                }
+            }
+        }
+
+        private void release(ColumnFamilyStore.RefViewFragment refViewFragment)
+        {
+            if (refViewFragment != null)
+                refViewFragment.release();
+        }
+
+        private void release(Collection<SSTableIndex> indexes)
+        {
+            for (var index : indexes)
+                index.release();
+            indexes.clear();
+        }
+
+        // I've removed the concept of "most selective index" since we don't actually have per-sstable
+        // statistics on that; it looks like it was only used to check bounds overlap, so computing
+        // an actual global bounds should be an improvement.  But computing global bounds as an intersection
+        // of individual bounds is messy because you can end up with more than one range.
+        private boolean indexInRange(SSTableIndex index)
+        {
+            SSTableReader sstable = index.getSSTable();
+            if (range instanceof Bounds && range.left.equals(range.right) && (!range.left.isMinimum()) && range.left instanceof DecoratedKey)
+            {
+                if (!((SSTableReaderWithFilter) sstable).getFilter().isPresent((DecoratedKey)range.left))
+                    return false;
+            }
+            return range.left.compareTo(sstable.last) <= 0 && (range.right.isMinimum() || sstable.first.compareTo(range.right) <= 0);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java b/src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java
index 81aa2ab1e9ad..6c1fe699d0b5 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java
@@ -18,104 +18,208 @@
 
 package org.apache.cassandra.index.sai.plan;
 
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.googlecode.concurrenttrees.common.Iterables;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
-import org.apache.cassandra.index.sai.view.View;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.utils.RangeUtil;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.NoSpamLogger;
 
 /**
- * Build a query specific view of the on-disk indexes for a query. This will return a
- * {@link Collection} of {@link Expression} and {@link SSTableIndex}s that represent
- * the on-disk data for a query.
- * <p>
- * The query view will include all the indexed expressions even if they don't have any
- * on-disk data. This in necessary because the query view is used to query in-memory
- * data as well as the attached on-disk indexes.
+ * Build a query specific view of the memtables, sstables, and indexes for a query.
+ * For use with SAI ordered queries to ensure that the view is consistent over the lifetime of the query,
+ * which is particularly important for validation of a cell's source memtable/sstable.
  */
 public class QueryViewBuilder
 {
-    private final Collection<Expression> expressions;
+    private static final Logger logger = LoggerFactory.getLogger(QueryViewBuilder.class);
+
+    private final ColumnFamilyStore cfs;
+    private final Orderer orderer;
     private final AbstractBounds<PartitionPosition> range;
+    private final QueryContext queryContext;
 
-    QueryViewBuilder(Collection<Expression> expressions, AbstractBounds<PartitionPosition> range)
+    QueryViewBuilder(ColumnFamilyStore cfs, Orderer orderer, AbstractBounds<PartitionPosition> range, QueryContext queryContext)
     {
-        this.expressions = expressions;
+        this.cfs = cfs;
+        this.orderer = orderer;
         this.range = range;
+        this.queryContext = queryContext;
     }
 
-    public static class QueryView
+    public static class QueryView implements AutoCloseable
     {
-        public final Collection<Pair<Expression, Collection<SSTableIndex>>> view;
-        public final Set<SSTableIndex> referencedIndexes;
-
-        public QueryView(Collection<Pair<Expression, Collection<SSTableIndex>>> view, Set<SSTableIndex> referencedIndexes)
+        final ColumnFamilyStore.RefViewFragment view;
+        final Set<SSTableIndex> referencedIndexes;
+        final Set<MemtableIndex> memtableIndexes;
+        final Orderer orderer;
+
+        public QueryView(ColumnFamilyStore.RefViewFragment view,
+                         Set<SSTableIndex> referencedIndexes,
+                         Set<MemtableIndex> memtableIndexes,
+                         Orderer orderer)
         {
             this.view = view;
             this.referencedIndexes = referencedIndexes;
+            this.memtableIndexes = memtableIndexes;
+            this.orderer = orderer;
+        }
+
+        @Override
+        public void close()
+        {
+            view.release();
+            referencedIndexes.forEach(SSTableIndex::release);
+        }
+
+        /**
+         * Returns the total count of rows in all sstables in this view
+         */
+        public long getTotalSStableRows()
+        {
+            return view.sstables.stream().mapToLong(SSTableReader::getTotalRows).sum();
         }
     }
 
+    /**
+     * Acquire references to all the memtables, memtable indexes, sstables, and sstable indexes required for the
+     * given expression.
+     * <p>
+     * Will retry if the active sstables change concurrently.
+     */
     protected QueryView build()
     {
-        Set<SSTableIndex> referencedIndexes = new HashSet<>();
-        while (true)
+        var referencedIndexes = new HashSet<SSTableIndex>();
+        long failingSince = -1L;
+        try
         {
-            referencedIndexes.clear();
-            boolean failed = false;
-
-            Collection<Pair<Expression, Collection<SSTableIndex>>> view = getQueryView(expressions);
-            for (SSTableIndex index : view.stream().map(pair -> pair.right).flatMap(Collection::stream).collect(Collectors.toList()))
+            outer:
+            while (true)
             {
-                if (index.reference())
-                    referencedIndexes.add(index);
-                else
-                    failed = true;
+                // Prevent an infinite loop
+                queryContext.checkpoint();
+
+                // Acquire live memtable index and memtable references first to avoid missing an sstable due to flush.
+                // Copy the memtable indexes to avoid concurrent modification.
+                var memtableIndexes = new HashSet<>(orderer.context.getLiveMemtables().values());
+
+                // We must use the canonical view in order for the equality check for source sstable/memtable
+                // to work correctly.
+                var filter = RangeUtil.coversFullRing(range)
+                             ? View.selectFunction(SSTableSet.CANONICAL)
+                             : View.select(SSTableSet.CANONICAL, s -> RangeUtil.intersects(s, range));
+                var refViewFragment = cfs.selectAndReference(filter);
+                var memtables = Iterables.toSet(refViewFragment.memtables);
+                // Confirm that all the memtables associated with the memtable indexes we already have are still live.
+                // There might be additional memtables that are not associated with the expression because tombstones
+                // are not indexed.
+                for (MemtableIndex memtableIndex : memtableIndexes)
+                {
+                    if (!memtables.contains(memtableIndex.getMemtable()))
+                    {
+                        refViewFragment.release();
+                        continue outer;
+                    }
+                }
+
+                Set<SSTableIndex> indexes = getIndexesForExpression(orderer);
+                // Attempt to reference each of the indexes, and thn confirm that the sstable associated with the index
+                // is in the refViewFragment. If it isn't in the refViewFragment, we will get incorrect results, so
+                // we release the indexes and refViewFragment and try again.
+                for (SSTableIndex index : indexes)
+                {
+                    var success = index.reference();
+                    if (success)
+                        referencedIndexes.add(index);
+
+                    if (!success || !refViewFragment.sstables.contains(index.getSSTable()))
+                    {
+                        referencedIndexes.forEach(SSTableIndex::release);
+                        referencedIndexes.clear();
+                        refViewFragment.release();
+
+                        // Log about the failures
+                        if (failingSince <= 0)
+                        {
+                            failingSince = Clock.Global.nanoTime();
+                        }
+                        else if (Clock.Global.nanoTime() - failingSince > TimeUnit.MILLISECONDS.toNanos(100))
+                        {
+                            failingSince = Clock.Global.nanoTime();
+                            if (success)
+                                NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS,
+                                                 "Spinning trying to capture index reader for {}, but it was released.", index);
+                            else
+                                NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS,
+                                                 "Spinning trying to capture readers for {}, but : {}, ", refViewFragment.sstables, index.getSSTable());
+                        }
+                        continue outer;
+                    }
+                }
+                return new QueryView(refViewFragment, referencedIndexes, memtableIndexes, orderer);
             }
-
-            if (failed)
-                referencedIndexes.forEach(SSTableIndex::release);
-            else
-                return new QueryView(view, referencedIndexes);
         }
-    }
-
-    private Collection<Pair<Expression, Collection<SSTableIndex>>> getQueryView(Collection<Expression> expressions)
-    {
-        List<Pair<Expression, Collection<SSTableIndex>>> queryView = new ArrayList<>();
-
-        for (Expression expression : expressions)
+        finally
         {
-            // Non-index column query should only act as FILTER BY for satisfiedBy(Row) method
-            // because otherwise it likely to go through the whole index.
-            if (expression.isNotIndexed())
-                continue;
-
-            // Select all the sstable indexes that have a term range that is satisfied by this expression and 
-            // overlap with the key range being queried.
-            View view = expression.getIndex().view();
-            queryView.add(Pair.create(expression, selectIndexesInRange(view.match(expression))));
+            if (Tracing.isTracing())
+            {
+                var groupedIndexes = referencedIndexes.stream().collect(
+                    Collectors.groupingBy(i -> i.getIndexContext().getIndexName(), Collectors.counting()));
+                var summary = groupedIndexes.entrySet().stream()
+                                            .map(e -> String.format("%s (%s sstables)", e.getKey(), e.getValue()))
+                                            .collect(Collectors.joining(", "));
+                Tracing.trace("Querying storage-attached indexes {}", summary);
+            }
         }
-
-        return queryView;
     }
 
-    private List<SSTableIndex> selectIndexesInRange(Collection<SSTableIndex> indexes)
+    /**
+     * Get the index
+     */
+    private Set<SSTableIndex> getIndexesForExpression(Orderer orderer)
     {
-        return indexes.stream().filter(this::indexInRange).sorted(SSTableIndex.COMPARATOR).collect(Collectors.toList());
+        if (!orderer.context.isIndexed())
+            throw new IllegalArgumentException("Expression is not indexed");
+
+        // Get all the indexes in the range.
+        return orderer.context.getView().getIndexes().stream().filter(this::indexInRange).collect(Collectors.toSet());
     }
 
+    // I've removed the concept of "most selective index" since we don't actually have per-sstable
+    // statistics on that; it looks like it was only used to check bounds overlap, so computing
+    // an actual global bounds should be an improvement.  But computing global bounds as an intersection
+    // of individual bounds is messy because you can end up with more than one range.
     private boolean indexInRange(SSTableIndex index)
     {
         SSTableReader sstable = index.getSSTable();
-        return range.left.compareTo(sstable.getLast()) <= 0 && (range.right.isMinimum() || sstable.getFirst().compareTo(range.right) <= 0);
+        if (range instanceof Bounds && range.left.equals(range.right) && (!range.left.isMinimum()) && range.left instanceof DecoratedKey)
+        {
+            if (sstable instanceof SSTableReaderWithFilter)
+            {
+                SSTableReaderWithFilter sstableWithFilter = (SSTableReaderWithFilter) sstable;
+                if (!sstableWithFilter.getFilter().isPresent((DecoratedKey) range.left))
+                    return false;
+            }
+        }
+        return range.left.compareTo(sstable.last) <= 0 && (range.right.isMinimum() || sstable.first.compareTo(range.right) <= 0);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java
index e352f7fcdd55..1abe0aa9e863 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java
@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.index.sai.plan;
 
-import java.util.Map;
+import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
@@ -27,89 +27,154 @@
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
 import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
-import org.apache.cassandra.schema.TableMetadata;
 
 public class StorageAttachedIndexQueryPlan implements Index.QueryPlan
 {
     public static final String UNSUPPORTED_NON_STRICT_OPERATOR =
-            "Operator %s is only supported in intersections for reads that do not require replica reconciliation.";
+    "Operator %s is only supported in intersections for reads that do not require replica reconciliation.";
 
     private final ColumnFamilyStore cfs;
     private final TableQueryMetrics queryMetrics;
+
+    /**
+     * postIndexFilter comprised by those expressions in the read command row filter that can't be handled by
+     * {@link FilterTree#isSatisfiedBy(DecoratedKey, Unfiltered, Row)}. That includes expressions targeted
+     * at {@link RowFilter.UserExpression}s like those used by RLAC.
+     */
     private final RowFilter postIndexFilter;
-    private final RowFilter indexFilter;
     private final Set<Index> indexes;
-    private final boolean isTopK;
+    private final IndexFeatureSet indexFeatureSet;
+    private final Orderer orderer;
 
     private StorageAttachedIndexQueryPlan(ColumnFamilyStore cfs,
                                           TableQueryMetrics queryMetrics,
-                                          RowFilter postIndexFilter,
-                                          RowFilter indexFilter,
-                                          ImmutableSet<Index> indexes)
+                                          RowFilter filter,
+                                          ImmutableSet<Index> indexes,
+                                          IndexFeatureSet indexFeatureSet)
     {
         this.cfs = cfs;
         this.queryMetrics = queryMetrics;
-        this.postIndexFilter = postIndexFilter;
-        this.indexFilter = indexFilter;
+        this.postIndexFilter = filter.restrict(RowFilter.Expression::isUserDefined);
         this.indexes = indexes;
-        this.isTopK = indexes.stream().anyMatch(i -> i instanceof StorageAttachedIndex && ((StorageAttachedIndex) i).termType().isVector());
+        this.indexFeatureSet = indexFeatureSet;
+        this.orderer = Orderer.from(cfs.getIndexManager(), filter);
     }
 
     @Nullable
     public static StorageAttachedIndexQueryPlan create(ColumnFamilyStore cfs,
                                                        TableQueryMetrics queryMetrics,
-                                                       Set<StorageAttachedIndex> indexes,
-                                                       RowFilter filter)
+                                                       Set<StorageAttachedIndex> allIndexes,
+                                                       RowFilter rowFilter)
     {
-        ImmutableSet.Builder<Index> selectedIndexesBuilder = ImmutableSet.builder();
+        // collect the indexes that can be used with the provided row filter
+        Set<StorageAttachedIndex> selectedIndexes = new HashSet<>();
+        if (!selectedIndexes(rowFilter.root(), allIndexes, selectedIndexes))
+            return null;
 
-        RowFilter preIndexFilter = filter;
-        RowFilter postIndexFilter = filter;
+        // collect the features of the selected indexes
+        IndexFeatureSet.Accumulator accumulator = new IndexFeatureSet.Accumulator();
+        for (StorageAttachedIndex index : selectedIndexes)
+            accumulator.accumulate(index.getIndexContext().indexFeatureSet());
 
-        for (RowFilter.Expression expression : filter)
+        return new StorageAttachedIndexQueryPlan(cfs,
+                                                 queryMetrics,
+                                                 rowFilter,
+                                                 ImmutableSet.copyOf(selectedIndexes),
+                                                 accumulator.complete());
+    }
+
+    /**
+     * Collects the indexes that can be used with the specified filtering tree without doing a full index scan.
+     * </p>
+     * The selected indexes are those that can satisfy at least one of the expressions of the filter, and that
+     * aren't part of an OR operation that contains not indexed expressions, unless that OR operation is nested inside
+     * an AND operation that has at least one indexed operation.
+     * </p>
+     * For example, for {@code x AND y} we can use any index in {@code x}, {@code y}, or both.
+     * </p>
+     * For {@code x OR y} we can't use a single index on {@code x} or {@code y} because we would need to do a full index
+     * scan because of the unidexed expression. However, if both columns were indexed, we could use those two indexes.
+     * </p>
+     * For {@code (x OR y) AND z}, where {@code x} and {@code z} are indexed, we can use the index on {@code z}, even
+     * though we will ignore the index on {@code x}.
+     *
+     * @param element a row filter tree node
+     * @param allIndexes all the indexes in the index group
+     * @param selectedIndexes the set of indexes where we'll add those indexes can be used with the specified expression
+     * @return {@code true} if this has collected any indexes, {@code false} otherwise
+     */
+    private static boolean selectedIndexes(RowFilter.FilterElement element,
+                                           Set<StorageAttachedIndex> allIndexes,
+                                           Set<StorageAttachedIndex> selectedIndexes)
+    {
+        if (element.isDisjunction()) // OR, all restrictions should have an index
         {
-            // We ignore any expressions here (currently IN and user-defined expressions) where we don't have a way to
-            // translate their #isSatifiedBy method, they will be included in the filter returned by 
-            // QueryPlan#postIndexQueryFilter(). If strict filtering is not allowed, we must reject the query until the
-            // expression(s) in question are compatible with #isSatifiedBy.
-            //
-            // Note: For both the pre- and post-filters we need to check that the expression exists before removing it
-            // because the without method assert if the expression doesn't exist. This can be the case if we are given
-            // a duplicate expression - a = 1 and a = 1. The without method removes all instances of the expression.
-            if (expression.operator().isIN() || expression.isUserDefined())
+            Set<StorageAttachedIndex> orIndexes = new HashSet<>();
+            for (RowFilter.Expression expression : element.expressions())
             {
-                if (!filter.isStrict())
-                    throw new InvalidRequestException(String.format(UNSUPPORTED_NON_STRICT_OPERATOR, expression.operator()));
-
-                if (preIndexFilter.getExpressions().contains(expression))
-                    preIndexFilter = preIndexFilter.without(expression);
-                continue;
+                if (!selectedIndexes(expression, allIndexes, orIndexes))
+                    return false;
             }
-
-            if (postIndexFilter.getExpressions().contains(expression))
-                postIndexFilter = postIndexFilter.without(expression);
-
-            for (StorageAttachedIndex index : indexes)
+            for (RowFilter.FilterElement child : element.children())
             {
-                if (index.supportsExpression(expression.column(), expression.operator()))
-                {
-                    selectedIndexesBuilder.add(index);
-                }
+                if (!selectedIndexes(child, allIndexes, orIndexes))
+                    return false;
             }
+            selectedIndexes.addAll(orIndexes);
+            return !orIndexes.isEmpty();
         }
+        else // AND, only one restriction needs to have an index
+        {
+            boolean hasIndex = false;
+            for (RowFilter.Expression expression : element.expressions())
+            {
+                hasIndex |= selectedIndexes(expression, allIndexes, selectedIndexes);
+            }
+            for (RowFilter.FilterElement child : element.children())
+            {
+                hasIndex |= selectedIndexes(child, allIndexes, selectedIndexes);
+            }
+            return hasIndex;
+        }
+    }
 
-        ImmutableSet<Index> selectedIndexes = selectedIndexesBuilder.build();
-        if (selectedIndexes.isEmpty())
-            return null;
+    /**
+     * Collects the indexes that can be used with the specified expression.
+     *
+     * @param expression a row filter expression
+     * @param allIndexes all the indexes in the index group
+     * @param selectedIndexes the set of indexes where we'll add those indexes can be used with the specified expression
+     * @return {@code true} if this has collected any indexes, {@code false} otherwise
+     */
+    private static boolean selectedIndexes(RowFilter.Expression expression,
+                                           Set<StorageAttachedIndex> allIndexes,
+                                           Set<StorageAttachedIndex> selectedIndexes)
+    {
+        // we ignore user-defined expressions here because we don't have a way to translate their #isSatifiedBy
+        // method, they will be included in the filter returned by QueryPlan#postIndexQueryFilter()
+        if (expression.isUserDefined())
+            return false;
 
-        return new StorageAttachedIndexQueryPlan(cfs, queryMetrics, postIndexFilter, preIndexFilter, selectedIndexes);
+        boolean hasIndex = false;
+        for (StorageAttachedIndex index : allIndexes)
+        {
+            if (index.supportsExpression(expression.column(), expression.operator()))
+            {
+                selectedIndexes.add(index);
+                hasIndex = true;
+            }
+        }
+        return hasIndex;
     }
 
     @Override
@@ -139,7 +204,8 @@ public Index.Searcher searcherFor(ReadCommand command)
         return new StorageAttachedIndexSearcher(cfs,
                                                 queryMetrics,
                                                 command,
-                                                indexFilter,
+                                                orderer,
+                                                indexFeatureSet,
                                                 DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.MILLISECONDS));
     }
 
@@ -153,13 +219,11 @@ public Function<PartitionIterator, PartitionIterator> postProcessor(ReadCommand
             return partitions -> partitions;
 
         // in case of top-k query, filter out rows that are not actually global top-K
-        return partitions -> (PartitionIterator) new VectorTopKProcessor(command).filter(partitions);
+        return partitions -> new TopKProcessor(command).reorder(partitions);
     }
 
     /**
-     * @return a filter with all the expressions that are user-defined or for a non-indexed partition key column
-     * <p>
-     * (currently index on partition columns is not supported, see {@link StorageAttachedIndex#validateOptions(Map, TableMetadata)})
+     * @return a filter with all the expressions that are user-defined
      */
     @Override
     public RowFilter postIndexQueryFilter()
@@ -167,9 +231,15 @@ public RowFilter postIndexQueryFilter()
         return postIndexFilter;
     }
 
+    @Override
+    public boolean supportsMultiRangeReadCommand()
+    {
+        return true;
+    }
+
     @Override
     public boolean isTopK()
     {
-        return isTopK;
+        return orderer != null;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
index 279f0d82e2de..ac53e8e64462 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
@@ -18,59 +18,86 @@
 
 package org.apache.cassandra.index.sai.plan;
 
+import java.io.IOError;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.NoSuchElementException;
 import java.util.concurrent.TimeUnit;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Queue;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.MessageParams;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.ReadExecutionController;
-import org.apache.cassandra.db.RegularAndStaticColumns;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.marshal.FloatType;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.ColumnData;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.RowIterator;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.RequestTimeoutException;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
 import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithScore;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.apache.cassandra.index.sai.utils.RangeUtil;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.net.ParamType;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.btree.BTree;
 
 public class StorageAttachedIndexSearcher implements Index.Searcher
 {
+    protected static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndexSearcher.class);
+
     private final ReadCommand command;
-    private final QueryController queryController;
+    private final QueryController controller;
     private final QueryContext queryContext;
     private final TableQueryMetrics tableQueryMetrics;
 
     public StorageAttachedIndexSearcher(ColumnFamilyStore cfs,
                                         TableQueryMetrics tableQueryMetrics,
                                         ReadCommand command,
-                                        RowFilter indexFilter,
+                                        Orderer orderer,
+                                        IndexFeatureSet indexFeatureSet,
                                         long executionQuotaMs)
     {
         this.command = command;
-        this.queryContext = new QueryContext(command, executionQuotaMs);
-        this.queryController = new QueryController(cfs, command, indexFilter, queryContext);
+        this.queryContext = new QueryContext(executionQuotaMs);
+        this.controller = new QueryController(cfs, command, orderer, indexFeatureSet, queryContext);
         this.tableQueryMetrics = tableQueryMetrics;
     }
 
@@ -81,82 +108,145 @@ public ReadCommand command()
     }
 
     @Override
-    public PartitionIterator filterReplicaFilteringProtection(PartitionIterator fullResponse)
+    @SuppressWarnings("unchecked")
+    public UnfilteredPartitionIterator search(ReadExecutionController executionController) throws RequestTimeoutException
     {
-        for (RowFilter.Expression expression : queryController.indexFilter())
+        int retries = 0;
+        while (true)
         {
-            if (queryController.hasAnalyzer(expression))
-                return applyIndexFilter(fullResponse, Operation.buildFilter(queryController, true), queryContext);
-        }
+            try
+            {
+                FilterTree filterTree = analyzeFilter();
+                maybeTriggerReferencedIndexesGuardrail(filterTree);
 
-        // if no analyzer does transformation
-        return Index.Searcher.super.filterReplicaFilteringProtection(fullResponse);
-    }
+                Plan plan = controller.buildPlan();
+                Iterator<? extends PrimaryKey> keysIterator = controller.buildIterator(plan);
 
-    @Override
-    public UnfilteredPartitionIterator search(ReadExecutionController executionController) throws RequestTimeoutException
-    {
-        if (!command.isTopK())
-            return new ResultRetriever(executionController, false);
-        else
-        {
-            Supplier<ResultRetriever> resultSupplier = () -> new ResultRetriever(executionController, true);
+                // Can't check for `command.isTopK()` because the planner could optimize sorting out
+                Orderer ordering = plan.ordering();
+                if (ordering == null)
+                {
+                    assert keysIterator instanceof KeyRangeIterator;
+                    return new ResultRetriever((KeyRangeIterator) keysIterator, filterTree, executionController);
+                }
 
-            // VSTODO performance: if there is shadowed primary keys, we have to at least query twice.
-            //  First time to find out there are shadow keys, second time to find out there are no more shadow keys.
-            while (true)
+                assert !(keysIterator instanceof KeyRangeIterator);
+                var scoredKeysIterator = (CloseableIterator<PrimaryKeyWithSortKey>) keysIterator;
+                var result = new ScoreOrderedResultRetriever(scoredKeysIterator, filterTree, executionController,
+                                                             command.limits().count(),
+                                                             ordering.context.getDefinition());
+                return new TopKProcessor(command).filter(result);
+            }
+            catch (QueryView.Builder.MissingIndexException e)
             {
-                long lastShadowedKeysCount = queryContext.vectorContext().getShadowedPrimaryKeys().size();
-                ResultRetriever result = resultSupplier.get();
-                UnfilteredPartitionIterator topK = (UnfilteredPartitionIterator) new VectorTopKProcessor(command).filter(result);
+                // If an index was dropped while we were preparing the plan or between preparing the plan
+                // and creating the result retriever, we can retry without that index,
+                // because there may be other indexes that could be used to run the query.
+                // And if there are no good indexes left, we'd get a good contextual request error message.
+                if (e.context.isDropped() && retries < 8)
+                {
+                    logger.debug("Index " + e.context.getIndexName() + " dropped while preparing the query plan. Retrying.");
+                    retries++;
+                    continue;
+                }
 
-                long currentShadowedKeysCount = queryContext.vectorContext().getShadowedPrimaryKeys().size();
-                if (lastShadowedKeysCount == currentShadowedKeysCount)
-                    return topK;
+                // If we end up here, this is either a bug or a problem with an index (corrupted / missing components?).
+                controller.abort();
+                // Throwing IOError here because we want the coordinator to handle it as any other serious storage error
+                // and report it up to the user as failed query. It is better to fail than to return an incomplete
+                // result set.
+                throw new IOError(e);
             }
+            catch (Throwable t)
+            {
+                controller.abort();
+                throw t;
+            }
+        }
+    }
+
+    private void maybeTriggerReferencedIndexesGuardrail(FilterTree filterTree)
+    {
+        if (!Guardrails.saiSSTableIndexesPerQuery.enabled())
+            return;
+
+        int numReferencedIndexes = filterTree.numSSTableIndexes();
+
+        if (Guardrails.saiSSTableIndexesPerQuery.failsOn(numReferencedIndexes, null))
+        {
+            String msg = String.format("Query %s attempted to read from too many indexes (%s) but max allowed is %s; " +
+                                       "query aborted (see sai_sstable_indexes_per_query_fail_threshold)",
+                                       command.toCQLString(),
+                                       numReferencedIndexes,
+                                       Guardrails.CONFIG_PROVIDER.getOrCreate(null).getSaiSSTableIndexesPerQueryFailThreshold());
+            Tracing.trace(msg);
+            MessageParams.add(ParamType.TOO_MANY_REFERENCED_INDEXES_FAIL, numReferencedIndexes);
+            throw new QueryReferencingTooManyIndexesException(msg);
         }
+        else if (Guardrails.saiSSTableIndexesPerQuery.warnsOn(numReferencedIndexes, null))
+        {
+            MessageParams.add(ParamType.TOO_MANY_REFERENCED_INDEXES_WARN, numReferencedIndexes);
+        }
+    }
+
+    /**
+     * Converts expressions into filter tree (which is currently just a single AND).
+     * </p>
+     * Filter tree allows us to do a couple of important optimizations
+     * namely, group flattening for AND operations (query rewrite), expression bounds checks,
+     * "satisfies by" checks for resulting rows with an early exit.
+     *
+     * @return root of the filter tree.
+     */
+    private FilterTree analyzeFilter()
+    {
+        return controller.buildFilter();
     }
 
     private class ResultRetriever extends AbstractIterator<UnfilteredRowIterator> implements UnfilteredPartitionIterator
     {
         private final PrimaryKey firstPrimaryKey;
-        private final PrimaryKey lastPrimaryKey;
         private final Iterator<DataRange> keyRanges;
         private AbstractBounds<PartitionPosition> currentKeyRange;
 
-        private final KeyRangeIterator resultKeyIterator;
+        private final KeyRangeIterator operation;
         private final FilterTree filterTree;
         private final ReadExecutionController executionController;
         private final PrimaryKey.Factory keyFactory;
-        private final boolean topK;
 
         private PrimaryKey lastKey;
 
-        private ResultRetriever(ReadExecutionController executionController,
-                                boolean topK)
+        private ResultRetriever(KeyRangeIterator operation,
+                                FilterTree filterTree,
+                                ReadExecutionController executionController)
         {
-            this.keyRanges = queryController.dataRanges().iterator();
+            this.keyRanges = controller.dataRanges().iterator();
             this.currentKeyRange = keyRanges.next().keyRange();
-            this.resultKeyIterator = Operation.buildIterator(queryController);
-            this.filterTree = Operation.buildFilter(queryController, queryController.usesStrictFiltering());
+
+            this.operation = operation;
+            this.filterTree = filterTree;
             this.executionController = executionController;
-            this.keyFactory = queryController.primaryKeyFactory();
-            this.firstPrimaryKey = queryController.firstPrimaryKeyInRange();
-            this.lastPrimaryKey = queryController.lastPrimaryKeyInRange();
-            this.topK = topK;
+            this.keyFactory = controller.primaryKeyFactory();
+
+            this.firstPrimaryKey = controller.firstPrimaryKey();
         }
 
         @Override
         public UnfilteredRowIterator computeNext()
         {
-            if (resultKeyIterator == null)
+            // IMPORTANT: The correctness of the entire query pipeline relies on the fact that we consume a token
+            // and materialize its keys before moving on to the next token in the flow. This sequence must not be broken
+            // with toList() or similar. (Both the union and intersection flow constructs, to avoid excessive object
+            // allocation, reuse their token mergers as they process individual positions on the ring.)
+
+            if (operation == null)
                 return endOfData();
 
             // If being called for the first time, skip to the beginning of the range.
             // We can't put this code in the constructor because it may throw and the caller
             // may not be prepared for that.
             if (lastKey == null)
-                resultKeyIterator.skipTo(firstPrimaryKey);
+                operation.skipTo(firstPrimaryKey);
 
             // Theoretically we wouldn't need this if the caller of computeNext always ran the
             // returned iterators to the completion. Unfortunately, we have no control over the caller behavior here.
@@ -172,7 +262,7 @@ public UnfilteredRowIterator computeNext()
 
         /**
          * Tries to obtain a row iterator for one of the supplied keys by repeatedly calling
-         * {@link ResultRetriever#queryStorageAndFilter} until it gives a non-null result.
+         * {@link ResultRetriever#apply} until it gives a non-null result.
          * The keySupplier should return the next key with every call to get() and
          * null when there are no more keys to try.
          *
@@ -186,7 +276,7 @@ public UnfilteredRowIterator computeNext()
                 PrimaryKey key = keySupplier.get();
                 if (key == null)
                     return null;
-                iterator = queryStorageAndFilter(key);
+                iterator = apply(key);
             }
             return iterator;
         }
@@ -211,8 +301,15 @@ public UnfilteredRowIterator computeNext()
                 }
                 else
                 {
-                    // key either before the current range, so let's move the key forward
-                    skipTo(currentKeyRange.left.getToken());
+                    // the following condition may be false if currentKeyRange.left is not inclusive,
+                    // and key == currentKeyRange.left; in this case we should not try to skipTo the beginning
+                    // of the range because that would be requesting the key to go backwards
+                    // (in some implementations, skipTo can go backwards, and we don't want that)
+                    if (currentKeyRange.left.getToken().compareTo(key.token()) > 0)
+                    {
+                        // key before the current range, so let's move the key forward
+                        skipTo(currentKeyRange.left.getToken());
+                    }
                     key = nextKey();
                 }
             }
@@ -224,14 +321,14 @@ public UnfilteredRowIterator computeNext()
          * If the next key falls out of the current key range, it skips to the next key range, and so on.
          * If no more keys acceptd by the controller are available, returns null.
          */
-        private @Nullable PrimaryKey nextSelectedKeyInRange()
+         private @Nullable PrimaryKey nextSelectedKeyInRange()
         {
             PrimaryKey key;
             do
             {
                 key = nextKeyInRange();
             }
-            while (key != null && queryController.doesNotSelect(key));
+            while (key != null && !controller.selects(key));
             return key;
         }
 
@@ -252,14 +349,17 @@ public UnfilteredRowIterator computeNext()
             PrimaryKey key;
             do
             {
-                if (!resultKeyIterator.hasNext())
+                if (!operation.hasNext())
+                    return null;
+                PrimaryKey minKey = operation.peek();
+                if (!minKey.token().equals(partitionKey.getToken()))
                     return null;
-                if (!resultKeyIterator.peek().partitionKey().equals(partitionKey))
+                if (minKey.partitionKey() != null && !minKey.partitionKey().equals(partitionKey))
                     return null;
 
                 key = nextKey();
             }
-            while (key != null && queryController.doesNotSelect(key));
+            while (key != null && !controller.selects(key));
             return key;
         }
 
@@ -269,18 +369,7 @@ public UnfilteredRowIterator computeNext()
          */
         private @Nullable PrimaryKey nextKey()
         {
-            if (!resultKeyIterator.hasNext())
-                return null;
-            PrimaryKey key = resultKeyIterator.next();
-            return isWithinUpperBound(key) ? key : null;
-        }
-
-        /**
-         * Returns true if the key is not greater than lastPrimaryKey
-         */
-        private boolean isWithinUpperBound(PrimaryKey key)
-        {
-            return lastPrimaryKey.token().isMinimum() || lastPrimaryKey.compareTo(key) >= 0;
+            return operation.hasNext() ? operation.next() : null;
         }
 
         /**
@@ -296,7 +385,7 @@ private boolean isWithinUpperBound(PrimaryKey key)
          */
         private void skipTo(@Nonnull Token token)
         {
-            resultKeyIterator.skipTo(keyFactory.create(token));
+            operation.skipTo(keyFactory.createTokenOnly(token));
         }
 
         /**
@@ -307,8 +396,8 @@ private void skipToNextPartition()
             if (lastKey == null)
                 return;
             DecoratedKey lastPartitionKey = lastKey.partitionKey();
-            while (resultKeyIterator.hasNext() && resultKeyIterator.peek().partitionKey().equals(lastPartitionKey))
-                resultKeyIterator.next();
+            while (operation.hasNext() && operation.peek().partitionKey().equals(lastPartitionKey))
+                operation.next();
         }
 
 
@@ -316,8 +405,8 @@ private void skipToNextPartition()
          * Returns an iterator over the rows in the partition associated with the given iterator.
          * Initially, it retrieves the rows from the given iterator until it runs out of data.
          * Then it iterates the primary keys obtained from the index until the end of the partition
-         * and lazily constructs new row itertors for each of the keys. At a given time, only one row iterator is open.
-         * <p>
+         * and lazily constructs new row itertors for each of the key. At a given time, only one row iterator is open.
+         * </p>
          * The rows are retrieved in the order of primary keys provided by the underlying index.
          * The iterator is complete when the next key to be fetched belongs to different partition
          * (but the iterator does not consume that key).
@@ -326,13 +415,14 @@ private void skipToNextPartition()
          */
         private @Nonnull UnfilteredRowIterator iteratePartition(@Nonnull UnfilteredRowIterator startIter)
         {
-            return new AbstractUnfilteredRowIterator(startIter.metadata(),
-                                                     startIter.partitionKey(),
-                                                     startIter.partitionLevelDeletion(),
-                                                     startIter.columns(),
-                                                     startIter.staticRow(),
-                                                     startIter.isReverseOrder(),
-                                                     startIter.stats())
+            return new AbstractUnfilteredRowIterator(
+                startIter.metadata(),
+                startIter.partitionKey(),
+                startIter.partitionLevelDeletion(),
+                startIter.columns(),
+                startIter.staticRow(),
+                startIter.isReverseOrder(),
+                startIter.stats())
             {
                 private UnfilteredRowIterator currentIter = startIter;
                 private final DecoratedKey partitionKey = startIter.partitionKey();
@@ -359,226 +449,414 @@ public void close()
             };
         }
 
-        private UnfilteredRowIterator queryStorageAndFilter(PrimaryKey key)
+        public UnfilteredRowIterator apply(PrimaryKey key)
         {
-            // Key reads are lazy, delayed all the way to this point. Skip if we've already seen this one:
-            if (key.equals(lastKey))
+            // Key reads are lazy, delayed all the way to this point.
+            // We don't want key.equals(lastKey) because some PrimaryKey implementations consider more than just
+            // partition key and clustering for equality. This can break lastKey skipping, which is necessary for
+            // correctness when PrimaryKey doesn't have a clustering (as otherwise, the same partition may get
+            // filtered and considered as a result multiple times).
+            // we need a non-null partitionKey here, as we want to construct a SinglePartitionReadCommand
+            Preconditions.checkNotNull(key.partitionKey(), "Partition key must not be null");
+            if (lastKey != null && key.partitionKey().equals(lastKey.partitionKey()) && key.clustering().equals(lastKey.clustering()))
                 return null;
 
             lastKey = key;
             long startTimeNanos = Clock.Global.nanoTime();
 
-            try (UnfilteredRowIterator partition = queryController.queryStorage(key, executionController))
-            {
-                queryContext.partitionsRead++;
-                queryContext.checkpoint();
+            UnfilteredRowIterator partition = controller.getPartition(key, executionController);
+            queryContext.addPartitionsRead(1);
+            queryContext.checkpoint();
+            UnfilteredRowIterator filtered = applyIndexFilter(partition, filterTree, queryContext);
 
-                UnfilteredRowIterator filtered = applyIndexFilter(key, partition, filterTree);
+            // Note that we record the duration of the read after post-filtering, which actually
+            // materializes the rows from disk.
+            tableQueryMetrics.postFilteringReadLatency.update(Clock.Global.nanoTime() - startTimeNanos, TimeUnit.NANOSECONDS);
 
-                // Note that we record the duration of the read after post-filtering, which actually 
-                // materializes the rows from disk.
-                tableQueryMetrics.postFilteringReadLatency.update(Clock.Global.nanoTime() - startTimeNanos, TimeUnit.NANOSECONDS);
+            return filtered;
+        }
 
-                return filtered;
-            }
+        @Override
+        public TableMetadata metadata()
+        {
+            return controller.metadata();
         }
 
-        private UnfilteredRowIterator applyIndexFilter(PrimaryKey key, UnfilteredRowIterator partition, FilterTree tree)
+        @Override
+        public void close()
         {
-            Row staticRow = partition.staticRow();
-            List<Unfiltered> matchingRows = new ArrayList<>();
-            boolean hasMatch = false;
+            FileUtils.closeQuietly(operation);
+            controller.finish();
+            if (tableQueryMetrics != null)
+                tableQueryMetrics.record(queryContext);
+        }
+    }
 
-            // We need to filter the partition rows before filtering on the static row. If this is done in the other
-            // order then we get incorrect results if we are filtering on a partition key index on a table with a
-            // composite partition key.
-            while (partition.hasNext())
-            {
-                Unfiltered unfiltered = partition.next();
+    /**
+     * A result retriever that consumes an iterator primary keys sorted by some score, materializes the row for each
+     * primary key (currently, each primary key is required to be fully qualified and should only point to one row),
+     * apply the filter tree to the row to test that the real row satisfies the WHERE clause, and finally tests
+     * that the row is valid for the ORDER BY clause. The class performs some optimizations to avoid materializing
+     * rows unnecessarily. See the class for more details.
+     * <p>
+     * The resulting {@link UnfilteredRowIterator} objects are not guaranteed to be in any particular order. It is
+     * the responsibility of the caller to sort the results if necessary.
+     */
+    public class ScoreOrderedResultRetriever extends AbstractIterator<UnfilteredRowIterator> implements UnfilteredPartitionIterator
+    {
+        private final ColumnFamilyStore.RefViewFragment view;
+        private final List<AbstractBounds<PartitionPosition>> keyRanges;
+        private final boolean coversFullRing;
+        private final CloseableIterator<PrimaryKeyWithSortKey> scoredPrimaryKeyIterator;
+        private final FilterTree filterTree;
+        private final ReadExecutionController executionController;
 
-                if (unfiltered.isRow())
-                {
-                    queryContext.rowsFiltered++;
+        private final HashSet<PrimaryKey> processedKeys;
+        private final Queue<UnfilteredRowIterator> pendingRows;
 
-                    if (tree.isSatisfiedBy(partition.partitionKey(), (Row) unfiltered, staticRow))
-                    {
-                        matchingRows.add(unfiltered);
-                        hasMatch = true;
-                    }
-                }
-            }
+        // Null indicates we are not sending the synthetic score column to the coordinator
+        @Nullable
+        private final ColumnMetadata syntheticScoreColumn;
 
-            if (!hasMatch)
-            {
-                queryContext.rowsFiltered++;
+        // The limit requested by the query. We cannot load more than softLimit rows in bulk because we only want
+        // to fetch the topk rows where k is the limit. However, we allow the iterator to fetch more rows than the
+        // soft limit to avoid confusing behavior. When the softLimit is reached, the iterator will fetch one row
+        // at a time.
+        private final int softLimit;
+        private int returnedRowCount = 0;
 
-                if (tree.isSatisfiedBy(key.partitionKey(), staticRow, staticRow))
-                    hasMatch = true;
-            }
+        private ScoreOrderedResultRetriever(CloseableIterator<PrimaryKeyWithSortKey> scoredPrimaryKeyIterator,
+                                            FilterTree filterTree,
+                                            ReadExecutionController executionController, int limit,
+                                            ColumnMetadata orderedColumn)
+        {
+            IndexContext context = controller.getOrderer().context;
+            this.view = controller.getQueryView(context).view;
+            this.keyRanges = controller.dataRanges().stream().map(DataRange::keyRange).collect(Collectors.toList());
+            this.coversFullRing = keyRanges.size() == 1 && RangeUtil.coversFullRing(keyRanges.get(0));
 
-            if (!hasMatch)
-            {
-                // shadowed by expired TTL or row tombstone or range tombstone
-                if (topK)
-                    queryContext.vectorContext().recordShadowedPrimaryKey(key);
+            this.scoredPrimaryKeyIterator = scoredPrimaryKeyIterator;
+            this.filterTree = filterTree;
+            this.executionController = executionController;
 
-                // If there are no matches, return an empty partition. If reconciliation is required at the
-                // coordinator, replica filtering protection may make a second round trip to complete its view
-                // of the partition.
-                return null;
-            }
+            this.processedKeys = new HashSet<>(limit);
+            this.pendingRows = new ArrayDeque<>(limit);
+            this.softLimit = limit;
 
-            // Return all matches found, along with the static row... 
-            return new PartitionIterator(partition, staticRow, matchingRows.iterator());
+            // When +score is added on the coordinator side, it's represented as a PrecomputedColumnFilter
+            // even in a 'SELECT *' because WCF is not capable of representing synthetic columns.
+            // This can be simplified when we remove ANN_USE_SYNTHETIC_SCORE
+            var tempColumn = ColumnMetadata.syntheticScoreColumn(orderedColumn, FloatType.instance);
+            var isScoreFetched = controller.command().columnFilter().fetchesExplicitly(tempColumn);
+            this.syntheticScoreColumn = isScoreFetched ? tempColumn : null;
         }
 
-        private class PartitionIterator extends AbstractUnfilteredRowIterator
+        @Override
+        public UnfilteredRowIterator computeNext()
         {
-            private final Iterator<Unfiltered> rows;
+            if (pendingRows.isEmpty())
+                fillPendingRows();
+            returnedRowCount++;
+            // Because we know ordered keys are fully qualified, we do not iterate partitions
+            return !pendingRows.isEmpty() ? pendingRows.poll() : endOfData();
+        }
 
-            public PartitionIterator(UnfilteredRowIterator partition, Row staticRow, Iterator<Unfiltered> rows)
+        /**
+         * Fills the pendingRows queue to generate a queue of row iterators for the supplied keys by repeatedly calling
+         * {@link #readAndValidatePartition} until it gives enough non-null results.
+         */
+        private void fillPendingRows()
+        {
+            // Group PKs by source sstable/memtable
+            var groupedKeys = new HashMap<PrimaryKey, List<PrimaryKeyWithSortKey>>();
+            // We always want to get at least 1.
+            int rowsToRetrieve = Math.max(1, softLimit - returnedRowCount);
+            // We want to get the first unique `rowsToRetrieve` keys to materialize
+            // Don't pass the priority queue here because it is more efficient to add keys in bulk
+            fillKeys(groupedKeys, rowsToRetrieve, null);
+            // Sort the primary keys by PrK order, just in case that helps with cache and disk efficiency
+            var primaryKeyPriorityQueue = new PriorityQueue<>(groupedKeys.keySet());
+
+            // drain groupedKeys into pendingRows
+            while (!groupedKeys.isEmpty())
             {
-                super(partition.metadata(),
-                      partition.partitionKey(),
-                      partition.partitionLevelDeletion(),
-                      partition.columns(),
-                      staticRow,
-                      partition.isReverseOrder(),
-                      partition.stats());
+                var pk = primaryKeyPriorityQueue.poll();
+                var sourceKeys = groupedKeys.remove(pk);
+                var partitionIterator = readAndValidatePartition(pk, sourceKeys);
+                if (partitionIterator != null)
+                    pendingRows.add(partitionIterator);
+                else
+                    // The current primaryKey did not produce a partition iterator. We know the caller will need
+                    // `rowsToRetrieve` rows, so we get the next unique key and add it to the queue.
+                    fillKeys(groupedKeys, 1, primaryKeyPriorityQueue);
+            }
+        }
 
-                this.rows = rows;
+        /**
+         * Fills the `groupedKeys` Map with the next `count` unique primary keys that are in the keys produced by calling
+         * {@link #nextSelectedKeyInRange()}. We map PrimaryKey to {@literal List<PrimaryKeyWithSortKey>} because the same
+         * primary key can be in the result set multiple times, but with different source tables.
+         * @param groupedKeys the map to fill
+         * @param count the number of unique PrimaryKeys to consume from the iterator
+         * @param primaryKeyPriorityQueue the priority queue to add new keys to. If the queue is null, we do not add
+         *                                keys to the queue.
+         */
+        private void fillKeys(Map<PrimaryKey, List<PrimaryKeyWithSortKey>> groupedKeys, int count, PriorityQueue<PrimaryKey> primaryKeyPriorityQueue)
+        {
+            int initialSize = groupedKeys.size();
+            while (groupedKeys.size() - initialSize < count)
+            {
+                var primaryKeyWithSortKey = nextSelectedKeyInRange();
+                if (primaryKeyWithSortKey == null)
+                    return;
+                var nextPrimaryKey = primaryKeyWithSortKey.primaryKey();
+                var accumulator = groupedKeys.computeIfAbsent(nextPrimaryKey, k -> new ArrayList<>());
+                if (primaryKeyPriorityQueue != null && accumulator.isEmpty())
+                    primaryKeyPriorityQueue.add(nextPrimaryKey);
+                accumulator.add(primaryKeyWithSortKey);
             }
+        }
 
-            @Override
-            protected Unfiltered computeNext()
+        /**
+         * Determine if the key is in one of the queried key ranges. We do not iterate through results in
+         * {@link PrimaryKey} order, so we have to check each range.
+         *
+         * @param key a partition key
+         * @return true if the key is in one of the queried key ranges
+         */
+        private boolean isInRange(DecoratedKey key)
+        {
+            if (coversFullRing)
+                return true;
+
+            for (AbstractBounds<PartitionPosition> range : keyRanges)
+                if (range.contains(key))
+                    return true;
+            return false;
+        }
+
+        /**
+         * Returns the next available key contained by one of the keyRanges and selected by the queryController.
+         * If the next key falls out of the current key range, it skips to the next key range, and so on.
+         * If no more keys acceptd by the controller are available, returns null.
+         */
+        private @Nullable PrimaryKeyWithSortKey nextSelectedKeyInRange()
+        {
+            while (scoredPrimaryKeyIterator.hasNext())
+            {
+                var key = scoredPrimaryKeyIterator.next();
+                if (isInRange(key.partitionKey()) && controller.selects(key))
+                    return key;
+            }
+            return null;
+        }
+
+        /**
+         * Reads and validates a partition for a given primary key against its sources.
+         * <p>
+         * @param pk The primary key of the partition to read and validate
+         * @param sourceKeys A list of PrimaryKeyWithSortKey objects associated with the primary key.
+         *                   Multiple sort keys can exist for the same primary key when data comes from different
+         *                   sstables or memtables.
+         *
+         * @return An UnfilteredRowIterator containing the validated partition data, or null if:
+         *         - The key has already been processed
+         *         - The partition does not pass index filters
+         *         - The partition contains no valid rows
+         *         - The row data does not match the index metadata for any of the provided primary keys
+         */
+        public UnfilteredRowIterator readAndValidatePartition(PrimaryKey pk, List<PrimaryKeyWithSortKey> sourceKeys)
+        {
+            // If we've already processed the key, we can skip it. Because the score ordered iterator does not
+            // deduplicate rows, we could see dupes if a row is in the ordering index multiple times. This happens
+            // in the case of dupes and of overwrites.
+            if (processedKeys.contains(pk))
+                return null;
+
+            try (UnfilteredRowIterator partition = controller.getPartition(pk, view, executionController))
             {
-                return rows.hasNext() ? rows.next() : endOfData();
+                queryContext.addPartitionsRead(1);
+                queryContext.checkpoint();
+                var staticRow = partition.staticRow();
+                UnfilteredRowIterator clusters = applyIndexFilter(partition, filterTree, queryContext);
+
+                if (clusters == null || !clusters.hasNext())
+                {
+                    processedKeys.add(pk);
+                    return null;
+                }
+
+                var now = FBUtilities.nowInSeconds();
+                boolean isRowValid = false;
+                var row = clusters.next();
+                assert !clusters.hasNext() : "Expected only one row per partition";
+                if (!row.isRangeTombstoneMarker())
+                {
+                    for (PrimaryKeyWithSortKey sourceKey : sourceKeys)
+                    {
+                        // Each of these primary keys are equal, but they have different source tables. Therefore,
+                        // we check to see if the row is valid for any of them, and if it is, we return the row.
+                        if (sourceKey.isIndexDataValid((Row) row, now))
+                        {
+                            isRowValid = true;
+                            // We can only count the pk as processed once we know it was valid for one of the
+                            // scored keys.
+                            processedKeys.add(pk);
+                            break;
+                        }
+                    }
+                }
+                return isRowValid ? new PrimaryKeyIterator(partition, staticRow, row, sourceKeys, syntheticScoreColumn)
+                                  : null;
             }
         }
 
         @Override
         public TableMetadata metadata()
         {
-            return queryController.metadata();
+            return controller.metadata();
         }
 
         @Override
         public void close()
         {
-            FileUtils.closeQuietly(resultKeyIterator);
-            if (tableQueryMetrics != null) tableQueryMetrics.record(queryContext);
+            FileUtils.closeQuietly(scoredPrimaryKeyIterator);
+            controller.finish();
+            if (tableQueryMetrics != null)
+                tableQueryMetrics.record(queryContext);
         }
-    }
 
-    /**
-     * Used by {@link StorageAttachedIndexSearcher#filterReplicaFilteringProtection} to filter rows for columns that
-     * have transformations so won't get handled correctly by the row filter.
-     */
-    private static PartitionIterator applyIndexFilter(PartitionIterator response, FilterTree tree, QueryContext context)
-    {
-        return new PartitionIterator()
+        public class PrimaryKeyIterator extends AbstractUnfilteredRowIterator
         {
-            @Override
-            public void close()
-            {
-                response.close();
-            }
-
-            @Override
-            public boolean hasNext()
-            {
-                return response.hasNext();
-            }
+            private boolean consumed = false;
+            private final Unfiltered row;
 
-            @Override
-            public RowIterator next()
+            public PrimaryKeyIterator(UnfilteredRowIterator partition, Row staticRow, Unfiltered content, List<PrimaryKeyWithSortKey> primaryKeysWithScore, ColumnMetadata syntheticScoreColumn)
             {
-                RowIterator delegate = response.next();
-                Row staticRow = delegate.staticRow();
-
-                // If we only restrict static columns, and we pass the filter, simply pass through the delegate, as all
-                // non-static rows are matches. If we fail on the filter, no rows are matches, so return nothing.
-                if (!tree.restrictsNonStaticRow())
-                    return tree.isSatisfiedBy(delegate.partitionKey(), staticRow, staticRow) ? delegate : null;
+                super(partition.metadata(),
+                      partition.partitionKey(),
+                      partition.partitionLevelDeletion(),
+                      partition.columns(),
+                      staticRow,
+                      partition.isReverseOrder(),
+                      partition.stats());
 
-                return new RowIterator()
+                assert !primaryKeysWithScore.isEmpty();
+                var isScoredRow = primaryKeysWithScore.get(0) instanceof PrimaryKeyWithScore;
+                if (!content.isRow() || !isScoredRow)
                 {
-                    Row next;
+                    this.row = content;
+                    return;
+                }
 
-                    @Override
-                    public TableMetadata metadata()
-                    {
-                        return delegate.metadata();
-                    }
 
-                    @Override
-                    public boolean isReverseOrder()
-                    {
-                        return delegate.isReverseOrder();
-                    }
+                if (syntheticScoreColumn == null)
+                {
+                    this.row = content;
+                    return;
+                }
 
-                    @Override
-                    public RegularAndStaticColumns columns()
-                    {
-                        return delegate.columns();
-                    }
+                // Clone the original Row
+                Row originalRow = (Row) content;
+                ArrayList<ColumnData> columnData = new ArrayList<>(originalRow.columnCount() + 1);
+                columnData.addAll(originalRow.columnData());
+
+                // inject +score as a new column
+                var pkWithScore = (PrimaryKeyWithScore) primaryKeysWithScore.get(0);
+                columnData.add(BufferCell.live(syntheticScoreColumn,
+                                               FBUtilities.nowInSeconds(),
+                                               FloatType.instance.decompose(pkWithScore.indexScore)));
+
+                this.row = BTreeRow.create(originalRow.clustering(),
+                                           originalRow.primaryKeyLivenessInfo(),
+                                           originalRow.deletion(),
+                                           BTree.builder(ColumnData.comparator)
+                                                .auto(true)
+                                                .addAll(columnData)
+                                                .build());
+            }
 
-                    @Override
-                    public DecoratedKey partitionKey()
-                    {
-                        return delegate.partitionKey();
-                    }
+            @Override
+            protected Unfiltered computeNext()
+            {
+                if (consumed)
+                    return endOfData();
+                consumed = true;
+                return row;
+            }
+        }
+    }
 
-                    @Override
-                    public Row staticRow()
-                    {
-                        return staticRow;
-                    }
+    private static UnfilteredRowIterator applyIndexFilter(UnfilteredRowIterator partition, FilterTree tree, QueryContext queryContext)
+    {
+        FilteringPartitionIterator filtered = new FilteringPartitionIterator(partition, tree, queryContext);
+        if (!filtered.hasNext() && !filtered.matchesStaticRow())
+        {
+            // shadowed by expired TTL or row tombstone or range tombstone
+            queryContext.addShadowed(1);
+            filtered.close();
+            return null;
+        }
+        return filtered;
+    }
 
-                    @Override
-                    public void close()
-                    {
-                        delegate.close();
-                    }
+    /**
+     * Filters the rows in the partition so that only non-static rows that match given filter are returned.
+     */
+    private static class FilteringPartitionIterator extends AbstractUnfilteredRowIterator
+    {
+        private final FilterTree filter;
+        private final QueryContext queryContext;
+        private final UnfilteredRowIterator rows;
 
-                    private Row computeNext()
-                    {
-                        while (delegate.hasNext())
-                        {
-                            Row row = delegate.next();
-                            context.rowsFiltered++;
-                            if (tree.isSatisfiedBy(delegate.partitionKey(), row, staticRow))
-                                return row;
-                        }
-                        return null;
-                    }
+        private final DecoratedKey key;
+        private final Row staticRow;
 
-                    private Row loadNext()
-                    {
-                        if (next == null)
-                            next = computeNext();
-                        return next;
-                    }
+        public FilteringPartitionIterator(UnfilteredRowIterator partition, FilterTree filter, QueryContext queryContext)
+        {
+            super(partition.metadata(),
+                  partition.partitionKey(),
+                  partition.partitionLevelDeletion(),
+                  partition.columns(),
+                  partition.staticRow(),
+                  partition.isReverseOrder(),
+                  partition.stats());
+
+            this.rows = partition;
+            this.filter = filter;
+            this.queryContext = queryContext;
+            this.key = partition.partitionKey();
+            this.staticRow = partition.staticRow();
+        }
 
-                    @Override
-                    public boolean hasNext()
-                    {
-                        return loadNext() != null;
-                    }
+        public boolean matchesStaticRow()
+        {
+            queryContext.addRowsFiltered(1);
+            return filter.isSatisfiedBy(key, staticRow, staticRow);
+        }
 
-                    @Override
-                    public Row next()
-                    {
-                        Row result = loadNext();
-                        next = null;
+        @Override
+        protected Unfiltered computeNext()
+        {
+            while (rows.hasNext())
+            {
+                Unfiltered row = rows.next();
+                queryContext.addRowsFiltered(1);
 
-                        if (result == null)
-                            throw new NoSuchElementException();
+                if (!row.isRow() || ((Row)row).isStatic())
+                    continue;
 
-                        return result;
-                    }
-                };
+                if (filter.isSatisfiedBy(key, row, staticRow))
+                    return row;
             }
-        };
+            return endOfData();
+        }
+
+        @Override
+        public void close()
+        {
+            super.close();
+            rows.close();
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/plan/TopKProcessor.java b/src/java/org/apache/cassandra/index/sai/plan/TopKProcessor.java
new file mode 100644
index 000000000000..357fb5f0d436
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/TopKProcessor.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.tuple.Triple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.BaseRowIterator;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher.ScoreOrderedResultRetriever;
+import org.apache.cassandra.index.sai.utils.InMemoryPartitionIterator;
+import org.apache.cassandra.index.sai.utils.InMemoryUnfilteredPartitionIterator;
+import org.apache.cassandra.index.sai.utils.PartitionInfo;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TopKSelector;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
+/**
+ * Processor applied to SAI based ORDER BY queries.
+ *
+ *  * On a replica:
+ *  *  - filter(ScoreOrderedResultRetriever) is used to collect up to the top-K rows.
+ *  *  - We store any tombstones as well, to avoid losing them during coordinator reconciliation.
+ *  *  - The result is returned in PK order so that coordinator can merge from multiple replicas.
+ *
+ * On a coordinator:
+ *  - reorder(PartitionIterator) is used to consume all rows from the provided partitions,
+ *    compute the order based on either a column ordering or a similarity score, and keep top-K.
+ *  - The result is returned in score/sortkey order.
+ */
+public class TopKProcessor
+{
+    public static final String INDEX_MAY_HAVE_BEEN_DROPPED = "An index may have been dropped. Ordering on non-clustering " +
+                                                             "column requires the column to be indexed";
+    protected static final Logger logger = LoggerFactory.getLogger(TopKProcessor.class);
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    private final ReadCommand command;
+    private final IndexContext indexContext;
+    private final RowFilter.Expression expression;
+    private final VectorFloat<?> queryVector;
+    private final ColumnMetadata scoreColumn;
+
+    private final int limit;
+
+    public TopKProcessor(ReadCommand command)
+    {
+        this.command = command;
+
+        Pair<IndexContext, RowFilter.Expression> indexAndExpression = findTopKIndexContext();
+        // this can happen in case an index was dropped after the query was initiated
+        if (indexAndExpression == null)
+            throw invalidRequest(INDEX_MAY_HAVE_BEEN_DROPPED);
+
+        this.indexContext = indexAndExpression.left;
+        this.expression = indexAndExpression.right;
+        if (expression.operator() == Operator.ANN && !SelectStatement.ANN_USE_SYNTHETIC_SCORE)
+            this.queryVector = vts.createFloatVector(TypeUtil.decomposeVector(indexContext, expression.getIndexValue().duplicate()));
+        else
+            this.queryVector = null;
+        this.limit = command.limits().count();
+        this.scoreColumn = ColumnMetadata.syntheticScoreColumn(expression.column(), FloatType.instance);
+    }
+
+    /**
+     * Sort the specified filtered rows according to the {@code ORDER BY} clause and keep the first {@link #limit} rows.
+     * Called on the coordinator side.
+     *
+     * @param partitions the partitions collected by the coordinator. It will be closed as a side-effect.
+     * @return the provided rows, sorted and trimmed to {@link #limit} rows
+     */
+    public PartitionIterator reorder(PartitionIterator partitions)
+    {
+        // We consume the partitions iterator and create a new one. Use a try-with-resources block to ensure the
+        // original iterator is closed. We do not expect exceptions here, but if they happen, we want to make sure the
+        // original iterator is closed to prevent leaking resources, which could compound the effect of an exception.
+        try (partitions)
+        {
+            Comparator<Triple<PartitionInfo, Row, ?>> comparator = comparator()
+                   .thenComparing(Triple::getLeft, Comparator.comparing(pi -> pi.key))
+                   .thenComparing(Triple::getMiddle, command.metadata().comparator);
+
+            TopKSelector<Triple<PartitionInfo, Row, ?>> topK = new TopKSelector<>(comparator, limit);
+            while (partitions.hasNext())
+            {
+                try (BaseRowIterator<?> partitionRowIterator = partitions.next())
+                {
+                    if (expression.operator() == Operator.ANN || expression.operator() == Operator.BM25)
+                    {
+                        PartitionResults pr = processScoredPartition(partitionRowIterator);
+                        topK.addAll(pr.rows);
+                    }
+                    else
+                    {
+                        while (partitionRowIterator.hasNext())
+                        {
+                            Row row = (Row) partitionRowIterator.next();
+                            ByteBuffer value = row.getCell(expression.column()).buffer();
+                            topK.add(Triple.of(PartitionInfo.create(partitionRowIterator), row, value));
+                        }
+                    }
+                }
+            }
+
+            // Convert the topK results to a PartitionIterator
+            List<Pair<PartitionInfo, Row>> sortedRows = new ArrayList<>(topK.size());
+            for (Triple<PartitionInfo, Row, ?> triple : topK.getShared())
+                sortedRows.add(Pair.create(triple.getLeft(), triple.getMiddle()));
+            return InMemoryPartitionIterator.create(command, sortedRows);
+        }
+    }
+
+    /**
+     * Sort the specified unfiltered rows according to the {@code ORDER BY} clause, keep the first {@link #limit} rows,
+     * and then order them again by primary key.
+     * </p>
+     * This is meant to be used on the replica-side, before reconciliation. We need to order the rows by primary key
+     * after the top-k selection to avoid confusing reconciliation later, on the coordinator. Note that due to sstable
+     * overlap and how the full data set of each node is queried for top-k queries we can have multiple versions of the
+     * same row in the coordinator even with CL=ONE. Reconciliation should remove those duplicates, but it needs the
+     * rows to be ordered by primary key to do so. See CNDB-12308 for details.
+     * </p>
+     * All tombstones will be kept. Caller must close the supplied iterator.
+     *
+     * @param partitions the partitions collected in the replica side of a query. It will be closed as a side-effect.
+     * @return the provided rows, sorted by the requested {@code ORDER BY} chriteria, trimmed to {@link #limit} rows,
+     * and the sorted again by primary key.
+     */
+    public UnfilteredPartitionIterator filter(ScoreOrderedResultRetriever partitions)
+    {
+        try (partitions)
+        {
+            TreeMap<PartitionInfo, TreeSet<Unfiltered>> unfilteredByPartition = new TreeMap<>(Comparator.comparing(pi -> pi.key));
+
+            int rowsMatched = 0;
+            // Because each “partition” from ScoreOrderedResultRetriever is actually a single row
+            // or tombstone, we can simply read them until we have enough.
+            while (rowsMatched < limit && partitions.hasNext())
+            {
+                try (BaseRowIterator<?> partitionRowIterator = partitions.next())
+                {
+                    rowsMatched += processSingleRowPartition(unfilteredByPartition, partitionRowIterator);
+                }
+            }
+
+            return new InMemoryUnfilteredPartitionIterator(command, unfilteredByPartition);
+        }
+    }
+
+    /**
+     * Constructs a comparator for triple (PartitionInfo, Row, X) used for top-K ranking.
+     * For ANN/BM25 we compare descending by X (float score). For ORDER_BY_ASC or DESC,
+     * we compare ascending/descending by the row’s relevant ByteBuffer data.
+     */
+    private Comparator<Triple<PartitionInfo, Row, ?>> comparator()
+    {
+        if (expression.operator() == Operator.ANN || expression.operator() == Operator.BM25)
+        {
+            // For similarity, higher is better, so reversed
+            return Comparator.comparing((Triple<PartitionInfo, Row, ?> t) -> (Float) t.getRight()).reversed();
+        }
+
+        Comparator<Triple<PartitionInfo, Row, ?>> comparator = Comparator.comparing(t -> (ByteBuffer) t.getRight(), indexContext.getValidator());
+        if (expression.operator() == Operator.ORDER_BY_DESC)
+            comparator = comparator.reversed();
+        return comparator;
+    }
+
+    /**
+     * Simple holder for partial results of a single partition (score-based path).
+     */
+    private class PartitionResults
+    {
+        final PartitionInfo partitionInfo;
+        final SortedSet<Unfiltered> tombstones = new TreeSet<>(command.metadata().comparator);
+        final List<Triple<PartitionInfo, Row, Float>> rows = new ArrayList<>();
+
+        PartitionResults(PartitionInfo partitionInfo)
+        {
+            this.partitionInfo = partitionInfo;
+        }
+
+        void addTombstone(Unfiltered uf)
+        {
+            tombstones.add(uf);
+        }
+
+        void addRow(Triple<PartitionInfo, Row, Float> triple)
+        {
+            rows.add(triple);
+        }
+    }
+
+    /**
+     * Processes all rows in a single partition to compute scores (for ANN or BM25)
+     */
+    private PartitionResults processScoredPartition(BaseRowIterator<?> partitionRowIterator)
+    {
+        // Compute key and static row score once per partition
+        DecoratedKey key = partitionRowIterator.partitionKey();
+        Row staticRow = partitionRowIterator.staticRow();
+        PartitionInfo partitionInfo = PartitionInfo.create(partitionRowIterator);
+        float keyAndStaticScore = getScoreForRow(key, staticRow);
+        var pr = new PartitionResults(partitionInfo);
+
+        while (partitionRowIterator.hasNext())
+        {
+            Unfiltered unfiltered = partitionRowIterator.next();
+            // Always include tombstones for coordinator. It relies on ReadCommand#withMetricsRecording to throw
+            // TombstoneOverwhelmingException to prevent OOM.
+            if (unfiltered.isRangeTombstoneMarker())
+            {
+                pr.addTombstone(unfiltered);
+                continue;
+            }
+
+            Row row = (Row) unfiltered;
+            float rowScore = getScoreForRow(null, row);
+            pr.addRow(Triple.of(partitionInfo, row, keyAndStaticScore + rowScore));
+        }
+
+        return pr;
+    }
+
+    /**
+     * Processes a single partition, without scoring it.
+     */
+    private int processSingleRowPartition(TreeMap<PartitionInfo, TreeSet<Unfiltered>> unfilteredByPartition,
+                                          BaseRowIterator<?> partitionRowIterator)
+    {
+        if (!partitionRowIterator.hasNext())
+            return 0;
+
+        Unfiltered unfiltered = partitionRowIterator.next();
+        assert !partitionRowIterator.hasNext() : "Only one row should be returned";
+        // Always include tombstones for coordinator. It relies on ReadCommand#withMetricsRecording to throw
+        // TombstoneOverwhelmingException to prevent OOM.
+        PartitionInfo partitionInfo = PartitionInfo.create(partitionRowIterator);
+        addUnfiltered(unfilteredByPartition, partitionInfo, unfiltered);
+        return unfiltered.isRangeTombstoneMarker() ? 0 : 1;
+    }
+
+    private void addUnfiltered(SortedMap<PartitionInfo, TreeSet<Unfiltered>> unfilteredByPartition,
+                               PartitionInfo partitionInfo,
+                               Unfiltered unfiltered)
+    {
+        var map = unfilteredByPartition.computeIfAbsent(partitionInfo, k -> new TreeSet<>(command.metadata().comparator));
+        map.add(unfiltered);
+    }
+
+    private float getScoreForRow(DecoratedKey key, Row row)
+    {
+        ColumnMetadata column = indexContext.getDefinition();
+
+        if (column.isPrimaryKeyColumn() && key == null)
+            return 0;
+
+        if (column.isStatic() && !row.isStatic())
+            return 0;
+
+        if ((column.isClusteringColumn() || column.isRegular()) && row.isStatic())
+            return 0;
+
+        // If we have a synthetic score column, use it
+        var scoreData = row.getColumnData(scoreColumn);
+        if (scoreData != null)
+        {
+            var cell = (Cell<?>) scoreData;
+            return FloatType.instance.compose(cell.buffer());
+        }
+
+        // TODO remove this once we enable ANN_USE_SYNTHETIC_SCORE
+        ByteBuffer value = indexContext.getValueOf(key, row, FBUtilities.nowInSeconds());
+        if (value != null)
+        {
+            var vector = vts.createFloatVector(TypeUtil.decomposeVector(indexContext, value));
+            return indexContext.getIndexWriterConfig().getSimilarityFunction().compare(vector, queryVector);
+        }
+        return 0;
+    }
+
+    private Pair<IndexContext, RowFilter.Expression> findTopKIndexContext()
+    {
+        ColumnFamilyStore cfs = Keyspace.openAndGetStore(command.metadata());
+
+        for (RowFilter.Expression expression : command.rowFilter().expressions())
+        {
+            StorageAttachedIndex sai = findOrderingIndexFor(cfs.indexManager, expression);
+            if (sai != null)
+                return Pair.create(sai.getIndexContext(), expression);
+        }
+
+        return null;
+    }
+
+    @Nullable
+    private StorageAttachedIndex findOrderingIndexFor(SecondaryIndexManager sim, RowFilter.Expression e)
+    {
+        if (e.operator() != Operator.ANN
+            && e.operator() != Operator.BM25
+            && e.operator() != Operator.ORDER_BY_ASC
+            && e.operator() != Operator.ORDER_BY_DESC)
+        {
+            return null;
+        }
+
+        Optional<Index> index = sim.getBestIndexFor(e);
+        return (StorageAttachedIndex) index.filter(i -> i instanceof StorageAttachedIndex).orElse(null);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/VectorTopKProcessor.java b/src/java/org/apache/cassandra/index/sai/plan/VectorTopKProcessor.java
deleted file mode 100644
index b3a34356f204..000000000000
--- a/src/java/org/apache/cassandra/index/sai/plan/VectorTopKProcessor.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.plan;
-
-import java.nio.ByteBuffer;
-import java.util.Comparator;
-import java.util.PriorityQueue;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import javax.annotation.Nullable;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.tuple.Triple;
-
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.partitions.BasePartitionIterator;
-import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.rows.BaseRowIterator;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.Unfiltered;
-import org.apache.cassandra.index.SecondaryIndexManager;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.utils.InMemoryPartitionIterator;
-import org.apache.cassandra.index.sai.utils.InMemoryUnfilteredPartitionIterator;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.index.sai.utils.PartitionInfo;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-
-/**
- * Processor that scans all rows from given partitions and selects rows with top-k scores based on vector indexes.
- * <p>
- * This processor performs the following steps:
- * - collect rows with score into {@link PriorityQueue} that sorts rows based on score. If there are multiple vector indexes,
- *   the final score is the sum of all vector index scores.
- * - remove rows with the lowest scores from PQ if PQ size exceeds limit
- * - return rows from PQ in primary key order to client
- */
-public class VectorTopKProcessor
-{
-    private final ReadCommand command;
-    private final StorageAttachedIndex index;
-    private final IndexTermType indexTermType;
-    private final float[] queryVector;
-
-    private final int limit;
-
-    public VectorTopKProcessor(ReadCommand command)
-    {
-        this.command = command;
-
-        Pair<StorageAttachedIndex, float[]> annIndexAndExpression = findTopKIndex();
-        Preconditions.checkNotNull(annIndexAndExpression);
-
-        this.index = annIndexAndExpression.left;
-        this.indexTermType = annIndexAndExpression.left().termType();
-        this.queryVector = annIndexAndExpression.right;
-        this.limit = command.limits().count();
-    }
-
-    /**
-     * Filter given partitions and keep the rows with the highest scores. In case of {@link UnfilteredPartitionIterator},
-     * all tombstones will be kept.
-     */
-    public <U extends Unfiltered, R extends BaseRowIterator<U>, P extends BasePartitionIterator<R>> BasePartitionIterator<?> filter(P partitions)
-    {
-        // priority queue ordered by score in ascending order
-        PriorityQueue<Triple<PartitionInfo, Row, Float>> topK = new PriorityQueue<>(limit + 1, Comparator.comparing(Triple::getRight));
-        // to store top-k results in primary key order
-        TreeMap<PartitionInfo, TreeSet<Unfiltered>> unfilteredByPartition = new TreeMap<>(Comparator.comparing(p -> p.key));
-
-        while (partitions.hasNext())
-        {
-            try (R partition = partitions.next())
-            {
-                DecoratedKey key = partition.partitionKey();
-                Row staticRow = partition.staticRow();
-                PartitionInfo partitionInfo = PartitionInfo.create(partition);
-                // compute key and static row score once per partition
-                float keyAndStaticScore = getScoreForRow(key, staticRow);
-
-                while (partition.hasNext())
-                {
-                    Unfiltered unfiltered = partition.next();
-                    // Always include tombstones for coordinator. It relies on ReadCommand#withMetricsRecording to throw
-                    // TombstoneOverwhelmingException to prevent OOM.
-                    if (!unfiltered.isRow())
-                    {
-                        unfilteredByPartition.computeIfAbsent(partitionInfo, k -> new TreeSet<>(command.metadata().comparator))
-                                             .add(unfiltered);
-                        continue;
-                    }
-
-                    Row row = (Row) unfiltered;
-                    float rowScore = getScoreForRow(null, row);
-                    topK.add(Triple.of(partitionInfo, row, keyAndStaticScore + rowScore));
-
-                    // when exceeding limit, remove row with low score
-                    while (topK.size() > limit)
-                        topK.poll();
-                }
-            }
-        }
-        partitions.close();
-
-        // reorder rows in partition/clustering order
-        for (Triple<PartitionInfo, Row, Float> triple : topK)
-            unfilteredByPartition.computeIfAbsent(triple.getLeft(), k -> new TreeSet<>(command.metadata().comparator))
-                                 .add(triple.getMiddle());
-
-        if (partitions instanceof PartitionIterator)
-            return new InMemoryPartitionIterator(command, unfilteredByPartition);
-        return new InMemoryUnfilteredPartitionIterator(command, unfilteredByPartition);
-    }
-
-    /**
-     * Sum the scores from different vector indexes for the row
-     */
-    private float getScoreForRow(DecoratedKey key, Row row)
-    {
-        ColumnMetadata column = indexTermType.columnMetadata();
-
-        if (column.isPrimaryKeyColumn() && key == null)
-            return 0;
-
-        if (column.isStatic() && !row.isStatic())
-            return 0;
-
-        if ((column.isClusteringColumn() || column.isRegular()) && row.isStatic())
-            return 0;
-
-        ByteBuffer value = indexTermType.valueOf(key, row, FBUtilities.nowInSeconds());
-        if (value != null)
-        {
-            float[] vector = indexTermType.decomposeVector(value);
-            return index.indexWriterConfig().getSimilarityFunction().compare(vector, queryVector);
-        }
-        return 0;
-    }
-
-
-    private Pair<StorageAttachedIndex, float[]> findTopKIndex()
-    {
-        ColumnFamilyStore cfs = Keyspace.openAndGetStore(command.metadata());
-
-        for (RowFilter.Expression expression : command.rowFilter().getExpressions())
-        {
-            StorageAttachedIndex sai = findVectorIndexFor(cfs.indexManager, expression);
-            if (sai != null)
-            {
-                float[] qv = sai.termType().decomposeVector(expression.getIndexValue().duplicate());
-                return Pair.create(sai, qv);
-            }
-        }
-
-        return null;
-    }
-
-    @Nullable
-    private StorageAttachedIndex findVectorIndexFor(SecondaryIndexManager sim, RowFilter.Expression e)
-    {
-        if (e.operator() != Operator.ANN)
-            return null;
-
-        return sim.getBestIndexFor(e, StorageAttachedIndex.class).orElse(null);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/postings/IntArrayPostingList.java b/src/java/org/apache/cassandra/index/sai/postings/IntArrayPostingList.java
index 24ef9fa737a1..7dd20c502c55 100644
--- a/src/java/org/apache/cassandra/index/sai/postings/IntArrayPostingList.java
+++ b/src/java/org/apache/cassandra/index/sai/postings/IntArrayPostingList.java
@@ -17,8 +17,12 @@
  */
 package org.apache.cassandra.index.sai.postings;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.v1.postings.OrdinalPostingList;
+
 public class IntArrayPostingList implements OrdinalPostingList
 {
     private final int[] postings;
@@ -30,13 +34,13 @@ public IntArrayPostingList(int[] postings)
     }
 
     @Override
-    public long getOrdinal()
+    public int getOrdinal()
     {
         return idx;
     }
 
     @Override
-    public long nextPosting()
+    public int nextPosting()
     {
         if (idx >= postings.length)
         {
@@ -46,13 +50,13 @@ public long nextPosting()
     }
 
     @Override
-    public long size()
+    public int size()
     {
         return postings.length;
     }
 
     @Override
-    public long advance(long targetRowID)
+    public int advance(int targetRowID)
     {
         for (int i = idx; i < postings.length; ++i)
         {
@@ -77,6 +81,12 @@ public String toString()
                           .toString();
     }
 
+    @VisibleForTesting
+    public void reset()
+    {
+        idx = 0;
+    }
+
     public int getPostingAt(int i)
     {
         return postings[i];
diff --git a/src/java/org/apache/cassandra/index/sai/postings/PeekablePostingList.java b/src/java/org/apache/cassandra/index/sai/postings/PeekablePostingList.java
deleted file mode 100644
index 02a3ae49cb43..000000000000
--- a/src/java/org/apache/cassandra/index/sai/postings/PeekablePostingList.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.postings;
-
-import java.io.IOException;
-
-import javax.annotation.concurrent.NotThreadSafe;
-
-import org.apache.cassandra.utils.Throwables;
-
-/**
- * A peekable wrapper around a {@link PostingList} that allows the next value to be
- * looked at without advancing the state of the {@link PostingList}
- */
-@NotThreadSafe
-public class PeekablePostingList implements PostingList
-{
-    private final PostingList wrapped;
-
-    private boolean peeked = false;
-    private long next;
-
-    public static PeekablePostingList makePeekable(PostingList postingList)
-    {
-        return postingList instanceof PeekablePostingList ? (PeekablePostingList) postingList
-                                                          : new PeekablePostingList(postingList);
-    }
-
-    private PeekablePostingList(PostingList wrapped)
-    {
-        this.wrapped = wrapped;
-    }
-
-    public long peek()
-    {
-        if (peeked)
-            return next;
-
-        try
-        {
-            peeked = true;
-            return next = wrapped.nextPosting();
-        }
-        catch (IOException e)
-        {
-            throw Throwables.cleaned(e);
-        }
-    }
-
-    public void advanceWithoutConsuming(long targetRowID) throws IOException
-    {
-        if (peek() == END_OF_STREAM)
-            return;
-
-        if (peek() >= targetRowID)
-        {
-            peek();
-            return;
-        }
-
-        peeked = true;
-        next = wrapped.advance(targetRowID);
-    }
-
-    @Override
-    public long minimum()
-    {
-        return wrapped.maximum();
-    }
-
-    @Override
-    public long maximum()
-    {
-        return wrapped.maximum();
-    }
-
-    @Override
-    public long nextPosting() throws IOException
-    {
-        if (peeked)
-        {
-            peeked = false;
-            return next;
-        }
-        return wrapped.nextPosting();
-    }
-
-    @Override
-    public long size()
-    {
-        return wrapped.size();
-    }
-
-    @Override
-    public long advance(long targetRowID) throws IOException
-    {
-        if (peeked && next >= targetRowID)
-        {
-            peeked = false;
-            return next;
-        }
-
-        peeked = false;
-        return wrapped.advance(targetRowID);
-    }
-
-    @Override
-    public void close()
-    {
-        wrapped.close();
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/postings/PostingList.java b/src/java/org/apache/cassandra/index/sai/postings/PostingList.java
deleted file mode 100644
index 9c6485db903a..000000000000
--- a/src/java/org/apache/cassandra/index/sai/postings/PostingList.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.postings;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * Interface for advancing on and consuming a posting list.
- */
-public interface PostingList extends Closeable
-{
-    PostingList EMPTY = new EmptyPostingList();
-
-    long OFFSET_NOT_FOUND = -1;
-    long END_OF_STREAM = Long.MAX_VALUE;
-
-    @Override
-    default void close() {}
-
-    default long minimum()
-    {
-        return Long.MIN_VALUE;
-    }
-
-    default long maximum()
-    {
-        return Long.MAX_VALUE;
-    }
-
-    /**
-     * Retrieves the next segment row ID, not including row IDs that have been returned by {@link #advance(long)}.
-     *
-     * @return next segment row ID
-     */
-    long nextPosting() throws IOException;
-
-    /**
-     * Returns the upper bound of postings in the list. During a merge individual postings may be
-     * de-duplicated, so we can't return the exact size only the upper bound of the size.
-     */
-    long size();
-
-    /**
-     * Advances to the first row ID beyond the current that is greater than or equal to the
-     * target, and returns that row ID. Exhausts the iterator and returns {@link #END_OF_STREAM} if
-     * the target is greater than the highest row ID.
-     * <p>
-     * Note: Callers must use the return value of this method before calling {@link #nextPosting()}, as calling
-     * that method will return the next posting, not the one to which we have just advanced.
-     *
-     * @param targetRowID target row ID to advance to
-     *
-     * @return first segment row ID which is >= the target row ID or {@link PostingList#END_OF_STREAM} if one does not exist
-     */
-    long advance(long targetRowID) throws IOException;
-
-    class EmptyPostingList implements PostingList
-    {
-        @Override
-        public long nextPosting() throws IOException
-        {
-            return END_OF_STREAM;
-        }
-
-        @Override
-        public long size()
-        {
-            return 0;
-        }
-
-        @Override
-        public long advance(long targetRowID) throws IOException
-        {
-            return END_OF_STREAM;
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/AbortedOperationException.java b/src/java/org/apache/cassandra/index/sai/utils/AbortedOperationException.java
new file mode 100644
index 000000000000..072110808721
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/AbortedOperationException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+
+/**
+ * This exception indicates that a request was aborted, normally because it was taking too much time.
+ *
+ * It is handled in a special way by the verb handlers and the request execute method: it is simply
+ * passed to the onAborted callback without logging any message. Therefore if any logging is required,
+ * it is up to the code raising this exception to log anything.
+ */
+// TODO OSS doesn't support onAbort and timeout response
+public class AbortedOperationException extends RuntimeException
+{
+    public AbortedOperationException()
+    {
+        super();
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/utils/AtomicRatio.java b/src/java/org/apache/cassandra/index/sai/utils/AtomicRatio.java
index d7348ea91132..dab74ba74d5a 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/AtomicRatio.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/AtomicRatio.java
@@ -21,45 +21,33 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-/**
- * AtomicRatio provides thread safe operations to maintain a {@link Ratio} of a numerator and denominator.
- * The ratio can be updated atomically by multiple threads calling the {@link #update(long, long)} method.
- * The current ratio value can be retrieved via the {@link #get} method.
- * <p>
- * The class also provides a thread safe {@link #updateCount} that maintains the number of times the {@link Ratio}
- * has been updated. This can be used to determine whether the {@link Ratio} is useful based on the number of updates.
- */
 public class AtomicRatio
 {
     private final AtomicReference<Ratio> ratio = new AtomicReference<>(new Ratio(0, 0));
     private final AtomicInteger updateCount = new AtomicInteger();
 
-    private static class Ratio
-    {
+    private static class Ratio {
         public final long numerator;
         public final long denominator;
 
-        public Ratio(long numerator, long denominator)
-        {
+        public Ratio(long numerator, long denominator) {
             this.numerator = numerator;
             this.denominator = denominator;
         }
     }
 
-    public void update(long numerator, long denominator)
-    {
-        ratio.updateAndGet((current) -> new Ratio(current.numerator + numerator, current.denominator + denominator));
+    public double updateAndGet(long numerator, long denominator) {
+        var updated = ratio.updateAndGet((current) -> new Ratio(current.numerator + numerator, current.denominator + denominator));
         updateCount.incrementAndGet();
+        return (double) updated.numerator / updated.denominator;
     }
 
-    public double get()
-    {
+    public double get() {
         var current = ratio.get();
         return (double) current.numerator / current.denominator;
     }
 
-    public int getUpdateCount()
-    {
+    public int getUpdateCount() {
         return updateCount.get();
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/utils/BM25Utils.java b/src/java/org/apache/cassandra/index/sai/utils/BM25Utils.java
new file mode 100644
index 000000000000..ba8bfbcbd0ad
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/BM25Utils.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import io.github.jbellis.jvector.graph.NodeQueue;
+import io.github.jbellis.jvector.util.BoundedLongHeap;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CloseableIterator;
+
+public class BM25Utils
+{
+    private static final float K1 = 1.2f;  // BM25 term frequency saturation parameter
+    private static final float B = 0.75f;  // BM25 length normalization parameter
+
+    /**
+     * Term frequencies across all documents.  Each document is only counted once.
+     */
+    public static class DocStats
+    {
+        // Map of term -> count of docs containing that term
+        private final Map<ByteBuffer, Long> frequencies;
+        // total number of docs in the index
+        private final long docCount;
+
+        public DocStats(Map<ByteBuffer, Long> frequencies, long docCount)
+        {
+            this.frequencies = frequencies;
+            this.docCount = docCount;
+        }
+    }
+
+    /**
+     * Term frequencies within a single document.  All instances of a term are counted. Allows us to optimize for
+     * the sstable use case, which is able to skip some reads from disk as well as some memory allocations.
+     */
+    public interface DocTF
+    {
+        int getTermFrequency(ByteBuffer term);
+        int termCount();
+        PrimaryKeyWithSortKey primaryKey(IndexContext context, Memtable source, float score);
+        PrimaryKeyWithSortKey primaryKey(IndexContext context, SSTableId<?> source, float score);
+    }
+
+    /**
+     * Term frequencies within a single document.  All instances of a term are counted. It is eager in that the
+     * PrimaryKey is already created.
+     */
+    public static class EagerDocTF implements DocTF
+    {
+        private final PrimaryKey pk;
+        private final Map<ByteBuffer, Integer> frequencies;
+        private final int termCount;
+
+        public EagerDocTF(PrimaryKey pk, int termCount, Map<ByteBuffer, Integer> frequencies)
+        {
+            this.pk = pk;
+            this.frequencies = frequencies;
+            this.termCount = termCount;
+        }
+
+        public int getTermFrequency(ByteBuffer term)
+        {
+            return frequencies.getOrDefault(term, 0);
+        }
+
+        public int termCount()
+        {
+            return termCount;
+        }
+
+        public PrimaryKeyWithSortKey primaryKey(IndexContext context, Memtable source, float score)
+        {
+            return new PrimaryKeyWithScore(context, source, pk, score);
+        }
+
+        public PrimaryKeyWithSortKey primaryKey(IndexContext context, SSTableId<?> source, float score)
+        {
+            return new PrimaryKeyWithScore(context, source, pk, score);
+        }
+
+        @Nullable
+        public static DocTF createFromDocument(PrimaryKey pk,
+                                               Cell<?> cell,
+                                               AbstractAnalyzer docAnalyzer,
+                                               Collection<ByteBuffer> queryTerms)
+        {
+            if (cell == null)
+                return null;
+
+            int count = 0;
+            Map<ByteBuffer, Integer> frequencies = new HashMap<>();
+            docAnalyzer.reset(cell.buffer());
+            try
+            {
+                while (docAnalyzer.hasNext())
+                {
+                    ByteBuffer term = docAnalyzer.next();
+                    count++;
+                    if (queryTerms.contains(term))
+                        frequencies.merge(term, 1, Integer::sum);
+                }
+            }
+            finally
+            {
+                docAnalyzer.end();
+            }
+
+            // Every query term must be present in the document
+            if (queryTerms.size() > frequencies.size())
+                return null;
+
+            return new EagerDocTF(pk, count, frequencies);
+        }
+    }
+
+    public static CloseableIterator<PrimaryKeyWithSortKey> computeScores(CloseableIterator<DocTF> docIterator,
+                                                                         List<ByteBuffer> queryTerms,
+                                                                         DocStats docStats,
+                                                                         IndexContext indexContext,
+                                                                         Object source)
+    {
+        assert source instanceof Memtable || source instanceof SSTableId : "Invalid source " + source.getClass();
+
+        // data structures for document stats and frequencies
+        ArrayList<DocTF> documents = new ArrayList<>();
+        double totalTermCount = 0;
+
+        // Compute TF within each document
+        while (docIterator.hasNext())
+        {
+            var tf = docIterator.next();
+            documents.add(tf);
+            totalTermCount += tf.termCount();
+        }
+
+        if (documents.isEmpty())
+            return CloseableIterator.emptyIterator();
+
+        // Calculate average document length
+        double avgDocLength = totalTermCount / documents.size();
+
+        // Calculate BM25 scores. Uses a nodequeue that avoids additional allocations and has heap time complexity
+        var nodeQueue = new NodeQueue(new BoundedLongHeap(documents.size()), NodeQueue.Order.MAX_HEAP);
+        for (int i = 0; i < documents.size(); i++)
+        {
+            var doc = documents.get(i);
+            double score = 0.0;
+            for (var queryTerm : queryTerms)
+            {
+                int tf = doc.getTermFrequency(queryTerm);
+                Long df = docStats.frequencies.get(queryTerm);
+                // we shouldn't have more hits for a term than we counted total documents
+                assert df <= docStats.docCount : String.format("df=%d, totalDocs=%d", df, docStats.docCount);
+
+                double normalizedTf = tf / (tf + K1 * (1 - B + B * doc.termCount() / avgDocLength));
+                double idf = Math.log(1 + (docStats.docCount - df + 0.5) / (df + 0.5));
+                double deltaScore = normalizedTf * idf;
+                assert deltaScore >= 0 : String.format("BM25 score for tf=%d, df=%d, tc=%d, totalDocs=%d is %f",
+                                                       tf, df, doc.termCount(), docStats.docCount, deltaScore);
+                score += deltaScore;
+            }
+            nodeQueue.push(i, (float) score);
+        }
+
+        return new NodeQueueDocTFIterator(nodeQueue, documents, indexContext, source, docIterator);
+    }
+
+    private static class NodeQueueDocTFIterator extends AbstractIterator<PrimaryKeyWithSortKey>
+    {
+        private final NodeQueue nodeQueue;
+        private final List<DocTF> documents;
+        private final IndexContext indexContext;
+        private final Object source;
+        private final CloseableIterator<DocTF> docIterator;
+
+        NodeQueueDocTFIterator(NodeQueue nodeQueue, List<DocTF> documents, IndexContext indexContext, Object source, CloseableIterator<DocTF> docIterator)
+        {
+            this.nodeQueue = nodeQueue;
+            this.documents = documents;
+            this.indexContext = indexContext;
+            this.source = source;
+            this.docIterator = docIterator;
+        }
+
+        @Override
+        protected PrimaryKeyWithSortKey computeNext()
+        {
+            if (nodeQueue.size() == 0)
+                return endOfData();
+
+            var score = nodeQueue.topScore();
+            var node = nodeQueue.pop();
+            var doc = documents.get(node);
+            if (source instanceof Memtable)
+                return doc.primaryKey(indexContext, (Memtable) source, score);
+            else
+                return doc.primaryKey(indexContext, (SSTableId<?>) source, score);
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(docIterator);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/CellWithSourceTable.java b/src/java/org/apache/cassandra/index/sai/utils/CellWithSourceTable.java
new file mode 100644
index 000000000000..1cc6689dbe07
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/CellWithSourceTable.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.DeletionPurger;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.memory.ByteBufferCloner;
+
+/**
+ * A wrapped {@link Cell} that includes a reference to the cell's source table.
+ * @param <T> the type of the cell's value
+ */
+public class CellWithSourceTable<T> extends Cell<T>
+{
+    private final Cell<T> cell;
+    private final Object sourceTable;
+
+    public CellWithSourceTable(Cell<T> cell, Object sourceTable)
+    {
+        super(cell.column());
+        this.cell = cell;
+        this.sourceTable = sourceTable;
+    }
+
+    public Object sourceTable()
+    {
+        return sourceTable;
+    }
+
+    @Override
+    public boolean isCounterCell()
+    {
+        return cell.isCounterCell();
+    }
+
+    @Override
+    public T value()
+    {
+        return cell.value();
+    }
+
+    @Override
+    public ValueAccessor<T> accessor()
+    {
+        return cell.accessor();
+    }
+
+    @Override
+    public long timestamp()
+    {
+        return cell.timestamp();
+    }
+
+    @Override
+    public int ttl()
+    {
+        return cell.ttl();
+    }
+
+    @Override
+    public long localDeletionTime()
+    {
+        return cell.localDeletionTime();
+    }
+
+    @Override
+    public boolean isTombstone()
+    {
+        return cell.isTombstone();
+    }
+
+    @Override
+    public boolean isExpiring()
+    {
+        return cell.isExpiring();
+    }
+
+    @Override
+    public boolean isLive(long nowInSec)
+    {
+        return cell.isLive(nowInSec);
+    }
+
+    @Override
+    public CellPath path()
+    {
+        return cell.path();
+    }
+
+    @Override
+    public Cell<?> withUpdatedColumn(ColumnMetadata newColumn)
+    {
+        return wrapIfNew(cell.withUpdatedColumn(newColumn));
+    }
+
+    @Override
+    public Cell<?> withUpdatedValue(ByteBuffer newValue)
+    {
+        return wrapIfNew(cell.withUpdatedValue(newValue));
+    }
+
+    @Override
+    public Cell<?> withUpdatedTimestampAndLocalDeletionTime(long newTimestamp, long newLocalDeletionTime)
+    {
+        return wrapIfNew(cell.withUpdatedTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime));
+    }
+
+    @Override
+    public Cell<?> withSkippedValue()
+    {
+        return wrapIfNew(cell.withSkippedValue());
+    }
+
+    @Override
+    public Cell<?> clone(ByteBufferCloner cloner)
+    {
+        return wrapIfNew(cell.clone(cloner));
+    }
+
+    @Override
+    public int dataSize()
+    {
+        return cell.dataSize();
+    }
+
+    @Override
+    public long unsharedHeapSizeExcludingData()
+    {
+        return cell.unsharedHeapSizeExcludingData();
+    }
+
+    @Override
+    public long unsharedHeapSize()
+    {
+        return cell.unsharedHeapSize();
+    }
+
+    @Override
+    public void validate()
+    {
+        cell.validate();
+    }
+
+    @Override
+    public boolean hasInvalidDeletions()
+    {
+        return cell.hasInvalidDeletions();
+    }
+
+    @Override
+    public void digest(Digest digest)
+    {
+        cell.digest(digest);
+    }
+
+    @Override
+    public ColumnData updateAllTimestamp(long newTimestamp)
+    {
+        var maybeNewCell = cell.updateAllTimestamp(newTimestamp);
+        if (maybeNewCell instanceof Cell)
+            return wrapIfNew((Cell<?>) maybeNewCell);
+        if (maybeNewCell instanceof ComplexColumnData)
+            return ((ComplexColumnData) maybeNewCell).transform(this::wrapIfNew);
+        // It's not clear when we would hit this code path, but it seems we should not
+        // hit this from SAI.
+        throw new IllegalStateException("Expected a Cell instance, but got " + maybeNewCell);
+    }
+
+    @Override
+    public Cell<?> markCounterLocalToBeCleared()
+    {
+        return wrapIfNew(cell.markCounterLocalToBeCleared());
+    }
+
+    @Override
+    public Cell<?> purge(DeletionPurger purger, long nowInSec)
+    {
+        return wrapIfNew(cell.purge(purger, nowInSec));
+    }
+
+    @Override
+    public Cell<?> purgeDataOlderThan(long timestamp)
+    {
+        return wrapIfNew(cell.purgeDataOlderThan(timestamp));
+    }
+
+    @Override
+    public int localDeletionTimeAsUnsignedInt()
+    {
+        return cell.localDeletionTimeAsUnsignedInt();
+    }
+
+    @Override
+    public long maxTimestamp()
+    {
+        return cell.maxTimestamp();
+    }
+
+    @Override
+    public long minTimestamp()
+    {
+        return cell.minTimestamp();
+    }
+
+    private Cell<?> wrapIfNew(Cell<?> maybeNewCell)
+    {
+        if (maybeNewCell == null)
+            return null;
+        // If the cell's method returned a reference to the same cell, then
+        // we can skip creating a new wrapper.
+        if (maybeNewCell == this.cell)
+            return this;
+        return new CellWithSourceTable<>(maybeNewCell, sourceTable);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/GeoUtil.java b/src/java/org/apache/cassandra/index/sai/utils/GeoUtil.java
new file mode 100644
index 000000000000..2510df17f3c1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/GeoUtil.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+public class GeoUtil
+{
+    // The distance in meters between two lines of longitude at the equator. We round down slightly to be more conservative
+    // and therefore include more results.
+    private static final double DISTANCE_PER_DEGREE_LONGITUDE_AT_EQUATOR = 110_000;
+
+    /**
+     * Determines the worst ratio for meters to degrees for a given latitude. The worst ratio will be the distance in
+     * meters of 1 degree longitude.
+     * @param lat the search latitude
+     * @return
+     */
+    private static double metersToDegreesRatioForLatitude(float lat)
+    {
+        // Got this formula from https://sciencing.com/what-parallels-maps-4689046.html. It seems
+        // to produce accurate results, but it'd be good to find additional support for its correctness.
+        return Math.cos(Math.toRadians(lat)) * DISTANCE_PER_DEGREE_LONGITUDE_AT_EQUATOR;
+    }
+
+    /**
+     * Calculate the maximum bound for a squared distance between lat/long points on the earth. The result is
+     * increased proportionally to the latitude of the search vector because the distance between two lines of
+     * longitude decreases as you move away from the equator.
+     * @param vector search vector
+     * @param distanceInMeters the search radius
+     * @return the threshold to use for the given geo point and distance
+     */
+    public static float amplifiedEuclideanSimilarityThreshold(float[] vector, float distanceInMeters)
+    {
+        // Get the conversion ratio for meters to degrees at the given latitude.
+        double distanceBetweenDegreeLatitude = metersToDegreesRatioForLatitude(vector[0]);
+
+        // Calculate the number of degrees that the search radius represents because we're finding the distance between
+        // two points that are also using degrees as their units.
+        double degrees = distanceInMeters / distanceBetweenDegreeLatitude;
+
+        return (float) (1.0 / (1 + Math.pow((float) degrees, 2)));
+    }
+
+    /**
+     * Determine if the lat/lon intersects with the antimeridian for the given distance.
+     * @param lat the latitude
+     * @param lon the longitude
+     * @param distanceInMeters the search radius
+     * @return true if the search radius crosses the antimeridian
+     */
+    public static boolean crossesAntimeridian(float lat, float lon, float distanceInMeters)
+    {
+        // Get the conversion ratio for meters to degrees at the given latitude.
+        // Result is always non-negative.
+        double distanceBetweenDegreeLatitude = metersToDegreesRatioForLatitude(lat);
+
+        // Calculate the number of degrees that the search radius represents because we're finding the distance between
+        // two points that are also using degrees as their units.
+        double degrees = distanceInMeters / distanceBetweenDegreeLatitude;
+
+        return Math.abs(lon) + degrees > 180;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/InMemoryPartitionIterator.java b/src/java/org/apache/cassandra/index/sai/utils/InMemoryPartitionIterator.java
index 651959dbc177..bb6e464900a0 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/InMemoryPartitionIterator.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/InMemoryPartitionIterator.java
@@ -18,10 +18,10 @@
 
 package org.apache.cassandra.index.sai.utils;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.TreeSet;
+import java.util.List;
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.ReadCommand;
@@ -29,18 +29,48 @@
 import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Pair;
 
 public class InMemoryPartitionIterator implements PartitionIterator
 {
-    private final ReadCommand command;
-    private final Iterator<Map.Entry<PartitionInfo, TreeSet<Unfiltered>>> partitions;
+    private final Iterator<InMemoryRowIterator> partitions;
 
-    public InMemoryPartitionIterator(ReadCommand command, TreeMap<PartitionInfo, TreeSet<Unfiltered>> rowsByPartitions)
+    private InMemoryPartitionIterator(List<InMemoryRowIterator> partitions)
     {
-        this.command = command;
-        this.partitions = rowsByPartitions.entrySet().iterator();
+        this.partitions = partitions.iterator();
+    }
+
+    public static InMemoryPartitionIterator create(ReadCommand command, List<Pair<PartitionInfo, Row>> sortedRows)
+    {
+        if (sortedRows.isEmpty())
+            return new InMemoryPartitionIterator(Collections.emptyList());
+
+        List<InMemoryRowIterator> partitions = new ArrayList<>();
+
+        PartitionInfo currentPartitionInfo = null;
+        List<Row> currentRows = null;
+
+        for (Pair<PartitionInfo, Row> pair : sortedRows)
+        {
+            PartitionInfo partitionInfo = pair.left;
+            Row row = pair.right;
+
+            if (currentPartitionInfo == null || !currentPartitionInfo.key.equals(partitionInfo.key))
+            {
+                if (currentPartitionInfo != null)
+                    partitions.add(new InMemoryRowIterator(command, currentPartitionInfo, currentRows));
+
+                currentPartitionInfo = partitionInfo;
+                currentRows = new ArrayList<>(1);
+            }
+
+            currentRows.add(row);
+        }
+
+        partitions.add(new InMemoryRowIterator(command, currentPartitionInfo, currentRows));
+
+        return new InMemoryPartitionIterator(partitions);
     }
 
     @Override
@@ -57,19 +87,21 @@ public boolean hasNext()
     @Override
     public RowIterator next()
     {
-        return new InMemoryRowIterator(partitions.next());
+        return partitions.next();
     }
 
 
-    private class InMemoryRowIterator implements RowIterator
+    private static class InMemoryRowIterator implements RowIterator
     {
+        private final ReadCommand command;
         private final PartitionInfo partitionInfo;
-        private final Iterator<Unfiltered> rows;
+        private final Iterator<Row> rows;
 
-        public InMemoryRowIterator(Map.Entry<PartitionInfo, TreeSet<Unfiltered>> rows)
+        public InMemoryRowIterator(ReadCommand command, PartitionInfo partitionInfo, List<Row> rows)
         {
-            this.partitionInfo = rows.getKey();
-            this.rows = rows.getValue().iterator();
+            this.command = command;
+            this.partitionInfo = partitionInfo;
+            this.rows = rows.iterator();
         }
 
         @Override
@@ -86,7 +118,7 @@ public boolean hasNext()
         @Override
         public Row next()
         {
-            return (Row) rows.next();
+            return rows.next();
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/index/sai/utils/InMemoryUnfilteredPartitionIterator.java b/src/java/org/apache/cassandra/index/sai/utils/InMemoryUnfilteredPartitionIterator.java
index 6aab722e2a85..848ecae24c52 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/InMemoryUnfilteredPartitionIterator.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/InMemoryUnfilteredPartitionIterator.java
@@ -20,7 +20,7 @@
 
 import java.util.Iterator;
 import java.util.Map;
-import java.util.TreeMap;
+import java.util.SortedMap;
 import java.util.TreeSet;
 
 import org.apache.cassandra.db.DecoratedKey;
@@ -39,7 +39,7 @@ public class InMemoryUnfilteredPartitionIterator implements UnfilteredPartitionI
     private final ReadCommand command;
     private final Iterator<Map.Entry<PartitionInfo, TreeSet<Unfiltered>>> partitions;
 
-    public InMemoryUnfilteredPartitionIterator(ReadCommand command, TreeMap<PartitionInfo, TreeSet<Unfiltered>> rowsByPartitions)
+    public InMemoryUnfilteredPartitionIterator(ReadCommand command, SortedMap<PartitionInfo, TreeSet<Unfiltered>> rowsByPartitions)
     {
         this.command = command;
         this.partitions = rowsByPartitions.entrySet().iterator();
diff --git a/src/java/org/apache/cassandra/index/sai/utils/IndexEntry.java b/src/java/org/apache/cassandra/index/sai/utils/IndexEntry.java
deleted file mode 100644
index a47d7a8ef43d..000000000000
--- a/src/java/org/apache/cassandra/index/sai/utils/IndexEntry.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.utils;
-
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-
-public class IndexEntry
-{
-    public final ByteComparable term;
-    public final PostingList postingList;
-
-    private IndexEntry(ByteComparable term, PostingList postingList)
-    {
-        this.term = term;
-        this.postingList = postingList;
-    }
-
-    public static IndexEntry create(ByteComparable term, PostingList postingList)
-    {
-        return new IndexEntry(term, postingList);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/IndexIdentifier.java b/src/java/org/apache/cassandra/index/sai/utils/IndexIdentifier.java
deleted file mode 100644
index f324207d6b58..000000000000
--- a/src/java/org/apache/cassandra/index/sai/utils/IndexIdentifier.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.utils;
-
-import com.google.common.base.Objects;
-
-/**
- * This is a simple wrapper around the index identity. Its primary purpose is to isolate classes that only need
- * access to the identity from the main index classes. This is useful in testing but also makes it easier to pass
- * the log message wrapper {@link #logMessage(String)} to classes that don't need any other information about the index.
- */
-public class IndexIdentifier
-{
-    public final String keyspaceName;
-    public final String tableName;
-    public final String indexName;
-
-    public IndexIdentifier(String keyspaceName, String tableName, String indexName)
-    {
-        this.keyspaceName = keyspaceName;
-        this.tableName = tableName;
-        this.indexName = indexName;
-    }
-
-    /**
-     * A helper method for constructing consistent log messages for specific column indexes.
-     * <p>
-     * Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message
-     * "Flushing new index segment..." will produce...
-     * <p>
-     * "[ks.tb.idx] Flushing new index segment..."
-     *
-     * @param message The raw content of a logging message, without information identifying it with an index.
-     *
-     * @return A log message with the proper keyspace, table and index name prepended to it.
-     */
-    public String logMessage(String message)
-    {
-        // Index names are unique only within a keyspace.
-        return String.format("[%s.%s.%s] %s", keyspaceName, tableName, indexName, message);
-    }
-
-    @Override
-    public String toString()
-    {
-        return String.format("%s.%s", keyspaceName, indexName);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hashCode(keyspaceName, tableName, indexName);
-    }
-
-    @Override
-    public boolean equals(Object obj)
-    {
-        if (this == obj) return true;
-        if (obj == null || getClass() != obj.getClass()) return false;
-        IndexIdentifier other = (IndexIdentifier) obj;
-        return Objects.equal(keyspaceName, other.keyspaceName) &&
-               Objects.equal(tableName, other.tableName) &&
-               Objects.equal(indexName, other.indexName);
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java b/src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java
deleted file mode 100644
index a558d5ee82aa..000000000000
--- a/src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java
+++ /dev/null
@@ -1,872 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.utils;
-
-import java.math.BigInteger;
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-
-import com.google.common.base.MoreObjects;
-import com.google.common.collect.ImmutableSet;
-
-import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
-import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.statements.schema.IndexTarget;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.marshal.BooleanType;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.db.marshal.CollectionType;
-import org.apache.cassandra.db.marshal.CompositeType;
-import org.apache.cassandra.db.marshal.DecimalType;
-import org.apache.cassandra.db.marshal.InetAddressType;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.marshal.StringType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.db.marshal.VectorType;
-import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.db.rows.ComplexColumnData;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FastByteOperations;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
-
-/**
- * This class is a representation of an {@link AbstractType} as an indexable type. It is responsible for determining the
- * capabilities of the type and provides helper methods for handling term values associated with the type.
- */
-public class IndexTermType
-{
-    private static final Set<AbstractType<?>> EQ_ONLY_TYPES = ImmutableSet.of(UTF8Type.instance,
-                                                                              AsciiType.instance,
-                                                                              BooleanType.instance,
-                                                                              UUIDType.instance);
-
-    private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 };
-
-    /**
-     * DecimalType / BigDecimal values are indexed by truncating their asComparableBytes representation to this size,
-     * padding on the right with zero-value-bytes until this size is reached (if necessary).  This causes
-     * false-positives that must be filtered in a separate step after hitting the index and reading the associated
-     * (full) values.
-     */
-    private static final int DECIMAL_APPROXIMATION_BYTES = 24;
-    private static final int BIG_INTEGER_APPROXIMATION_BYTES = 20;
-    private static final int INET_ADDRESS_SIZE = 16;
-    private static final int DEFAULT_FIXED_LENGTH = 16;
-
-    private enum Capability
-    {
-        STRING,
-        VECTOR,
-        INET_ADDRESS,
-        BIG_INTEGER,
-        BIG_DECIMAL,
-        LONG,
-        BOOLEAN,
-        LITERAL,
-        REVERSED,
-        FROZEN,
-        COLLECTION,
-        NON_FROZEN_COLLECTION,
-        COMPOSITE,
-        COMPOSITE_PARTITION
-    }
-
-    private final ColumnMetadata columnMetadata;
-    private final IndexTarget.Type indexTargetType;
-    private final AbstractType<?> indexType;
-    private final List<IndexTermType> subTypes;
-    private final AbstractType<?> vectorElementType;
-    private final int vectorDimension;
-    private final EnumSet<Capability> capabilities;
-
-    /**
-     * Create an {@link IndexTermType} from a {@link ColumnMetadata} and {@link IndexTarget.Type}.
-     *
-     * @param columnMetadata the {@link ColumnMetadata} for the column being indexed
-     * @param partitionColumns the partition columns for the table this column belongs to. This is used for identifying
-     *                         if the {@code columnMetadata} is a partition column and if it belongs to a composite
-     *                         partition
-     * @param indexTargetType the {@link IndexTarget.Type} for the index
-     *
-     * @return the {@link IndexTermType}
-     */
-    public static IndexTermType create(ColumnMetadata columnMetadata, List<ColumnMetadata> partitionColumns, IndexTarget.Type indexTargetType)
-    {
-        return new IndexTermType(columnMetadata, partitionColumns, indexTargetType);
-    }
-
-    private IndexTermType(ColumnMetadata columnMetadata, List<ColumnMetadata> partitionColumns, IndexTarget.Type indexTargetType)
-    {
-        this.columnMetadata = columnMetadata;
-        this.indexTargetType = indexTargetType;
-        this.capabilities = calculateCapabilities(columnMetadata, partitionColumns, indexTargetType);
-        this.indexType = calculateIndexType(columnMetadata.type, capabilities, indexTargetType);
-        if (indexType.subTypes().isEmpty())
-        {
-            this.subTypes = Collections.emptyList();
-        }
-        else
-        {
-            List<IndexTermType> subTypes = new ArrayList<>(indexType.subTypes().size());
-            for (AbstractType<?> subType : indexType.subTypes())
-                subTypes.add(new IndexTermType(columnMetadata.withNewType(subType), partitionColumns, indexTargetType));
-            this.subTypes = Collections.unmodifiableList(subTypes);
-        }
-        if (isVector())
-        {
-            VectorType<?> vectorType = (VectorType<?>) indexType;
-            vectorElementType = vectorType.elementType;
-            vectorDimension = vectorType.dimension;
-        }
-        else
-        {
-            vectorElementType = null;
-            vectorDimension = -1;
-        }
-    }
-
-    /**
-     * Returns {@code true} if the index type is a literal type and will use a literal index. This applies to
-     * string types, frozen types, composite types and boolean type.
-     */
-    public boolean isLiteral()
-    {
-        return capabilities.contains(Capability.LITERAL);
-    }
-
-    /**
-     * Returns {@code true} if the index type is a string type. This is used to determine if the type supports
-     * analysis.
-     */
-    public boolean isString()
-    {
-        return capabilities.contains(Capability.STRING);
-    }
-
-    /**
-     * Returns {@code true} if the index type is a vector type. Note: being a vector type does not mean that the type
-     * is valid for indexing in that we don't check the element type and dimension constraints here.
-     */
-    public boolean isVector()
-    {
-        return capabilities.contains(Capability.VECTOR);
-    }
-
-    /**
-     * Returns {@code true} if the index type is reversed. This is only the case (currently) for clustering keys with
-     * descending ordering.
-     */
-    public boolean isReversed()
-    {
-        return capabilities.contains(Capability.REVERSED);
-    }
-
-    /**
-     * Returns {@code true} if the index type is frozen, e.g. the type is wrapped with {@code frozen<type>}.
-     */
-    public boolean isFrozen()
-    {
-        return capabilities.contains(Capability.FROZEN);
-    }
-
-    /**
-     * Returns {@code true} if the index type is a non-frozen collection
-     */
-    public boolean isNonFrozenCollection()
-    {
-        return capabilities.contains(Capability.NON_FROZEN_COLLECTION);
-    }
-
-    /**
-     * Returns {@code true} if the index type is a frozen collection. This is the inverse of a non-frozen collection
-     * but this method is here for clarity.
-     */
-    public boolean isFrozenCollection()
-    {
-        return capabilities.contains(Capability.COLLECTION) && capabilities.contains(Capability.FROZEN);
-    }
-
-    /**
-     * Returns {@code true} if the index type is a composite type, e.g. it has the form {@code Composite<typea, typeb>}
-     */
-    public boolean isComposite()
-    {
-        return capabilities.contains(Capability.COMPOSITE);
-    }
-
-    /**
-     * Returns {@code true} if the {@link RowFilter.Expression} passed is backed by a non-frozen collection and the
-     * {@code Operator} is one that cannot be merged together.
-     */
-    public boolean isMultiExpression(RowFilter.Expression expression)
-    {
-        boolean multiExpression = false;
-        switch (expression.operator())
-        {
-            case EQ:
-                multiExpression = isNonFrozenCollection();
-                break;
-            case CONTAINS:
-            case CONTAINS_KEY:
-                multiExpression = true;
-                break;
-        }
-        return multiExpression;
-    }
-
-    /**
-     * Returns <code>true</code> if given buffer would pass the {@link AbstractType#validate(ByteBuffer)}
-     * check. False otherwise.
-     */
-    public boolean isValid(ByteBuffer term)
-    {
-        try
-        {
-            indexType.validate(term);
-            return true;
-        }
-        catch (MarshalException e)
-        {
-            return false;
-        }
-    }
-
-    public AbstractType<?> indexType()
-    {
-        return indexType;
-    }
-
-    public Collection<IndexTermType> subTypes()
-    {
-        return subTypes;
-    }
-
-    public CQL3Type asCQL3Type()
-    {
-        return indexType.asCQL3Type();
-    }
-
-    public ColumnMetadata columnMetadata()
-    {
-        return columnMetadata;
-    }
-
-    public String columnName()
-    {
-        return columnMetadata.name.toString();
-    }
-
-    public AbstractType<?> vectorElementType()
-    {
-        assert isVector();
-
-        return vectorElementType;
-    }
-
-    public int vectorDimension()
-    {
-        assert isVector();
-
-        return vectorDimension;
-    }
-
-    public boolean dependsOn(ColumnMetadata columnMetadata)
-    {
-        return this.columnMetadata.compareTo(columnMetadata) == 0;
-    }
-
-    /**
-     * Indicates if the type encoding supports rounding of the raw value.
-     * <p>
-     * This is significant in range searches where we have to make all range
-     * queries inclusive when searching the indexes in order to avoid excluding
-     * rounded values. Excluded values are removed by post-filtering.
-     */
-    public boolean supportsRounding()
-    {
-        return isBigInteger() || isBigDecimal();
-    }
-
-    /**
-     * Returns the value length for the given {@link AbstractType}, selecting 16 for types
-     * that officially use VARIABLE_LENGTH but are, in fact, of a fixed length.
-     */
-    public int fixedSizeOf()
-    {
-        if (indexType.isValueLengthFixed())
-            return indexType.valueLengthIfFixed();
-        else if (isInetAddress())
-            return INET_ADDRESS_SIZE;
-        else if (isBigInteger())
-            return BIG_INTEGER_APPROXIMATION_BYTES;
-        else if (isBigDecimal())
-            return DECIMAL_APPROXIMATION_BYTES;
-        return DEFAULT_FIXED_LENGTH;
-    }
-
-    /**
-     * Allows overriding the default getString method for {@link CompositeType}. It is
-     * a requirement of the {@link ConcurrentRadixTree} that the keys are strings but
-     * the getString method of {@link CompositeType} does not return a string that compares
-     * in the same order as the underlying {@link ByteBuffer}. To get round this we convert
-     * the {@link CompositeType} bytes to a hex string.
-     */
-    public String asString(ByteBuffer value)
-    {
-        if (isComposite())
-            return ByteBufferUtil.bytesToHex(value);
-        return indexType.getString(value);
-    }
-
-    /**
-     * The inverse of the above method. Overrides the fromString method on {@link CompositeType}
-     * in order to convert the hex string to bytes.
-     */
-    public ByteBuffer fromString(String value)
-    {
-        if (isComposite())
-            return ByteBufferUtil.hexToBytes(value);
-        return indexType.fromString(value);
-    }
-
-    /**
-     * Returns the cell value from the {@link DecoratedKey} or {@link Row} for the {@link IndexTermType} based on the
-     * kind of column this {@link IndexTermType} is based on.
-     *
-     * @param key the {@link DecoratedKey} of the row
-     * @param row the {@link Row} containing the non-partition column data
-     * @param nowInSecs the time that the index write operation started
-     *
-     * @return a {@link ByteBuffer} containing the cell value
-     */
-    public ByteBuffer valueOf(DecoratedKey key, Row row, long nowInSecs)
-    {
-        if (row == null)
-            return null;
-
-        switch (columnMetadata.kind)
-        {
-            case PARTITION_KEY:
-                return isCompositePartition() ? CompositeType.extractComponent(key.getKey(), columnMetadata.position())
-                                              : key.getKey();
-            case CLUSTERING:
-                // skip indexing of static clustering when regular column is indexed
-                return row.isStatic() ? null : row.clustering().bufferAt(columnMetadata.position());
-
-            // treat static cell retrieval the same was as regular
-            // only if row kind is STATIC otherwise return null
-            case STATIC:
-                if (!row.isStatic())
-                    return null;
-            case REGULAR:
-                Cell<?> cell = row.getCell(columnMetadata);
-                return cell == null || !cell.isLive(nowInSecs) ? null : cell.buffer();
-
-            default:
-                return null;
-        }
-    }
-
-    /**
-     * Returns a value iterator for collection type {@link IndexTermType}s.
-     *
-     * @param row the {@link Row} containing the column data
-     * @param nowInSecs the time that the index write operation started
-     *
-     * @return an {@link Iterator} of the collection values
-     */
-    public Iterator<ByteBuffer> valuesOf(Row row, long nowInSecs)
-    {
-        if (row == null)
-            return null;
-
-        switch (columnMetadata.kind)
-        {
-            // treat static cell retrieval the same was as regular
-            // only if row kind is STATIC otherwise return null
-            case STATIC:
-                if (!row.isStatic())
-                    return null;
-            case REGULAR:
-                return collectionIterator(row.getComplexColumnData(columnMetadata), nowInSecs);
-
-            default:
-                return null;
-        }
-    }
-
-    public Comparator<ByteBuffer> comparator()
-    {
-        // Override the comparator for BigInteger, frozen collections and composite types
-        if (isBigInteger() || isBigDecimal() || isComposite() || isFrozen())
-            return FastByteOperations::compareUnsigned;
-
-        return indexType;
-    }
-
-    /**
-     * Compare two terms based on their type. This is used in place of {@link AbstractType#compare(ByteBuffer, ByteBuffer)}
-     * so that the default comparison can be overridden for specific types.
-     * <p>
-     * Note: This should be used for all term comparison
-     */
-    public int compare(ByteBuffer b1, ByteBuffer b2)
-    {
-        if (isInetAddress())
-            return compareInet(b1, b2);
-            // BigInteger values, frozen types and composite types (map entries) use compareUnsigned to maintain
-            // a consistent order between the in-memory index and the on-disk index.
-        else if (isBigInteger() || isBigDecimal() || isComposite() || isFrozen())
-            return FastByteOperations.compareUnsigned(b1, b2);
-
-        return indexType.compare(b1, b2 );
-    }
-
-    /**
-     * Returns the smaller of two {@code ByteBuffer} values, based on the result of {@link
-     * #compare(ByteBuffer, ByteBuffer)} comparision.
-     */
-    public ByteBuffer min(ByteBuffer a, ByteBuffer b)
-    {
-        return a == null ? b : (b == null || compare(b, a) > 0) ? a : b;
-    }
-
-    /**
-     * Returns the greater of two {@code ByteBuffer} values, based on the result of {@link
-     * #compare(ByteBuffer, ByteBuffer)} comparision.
-     */
-    public ByteBuffer max(ByteBuffer a, ByteBuffer b)
-    {
-        return a == null ? b : (b == null || compare(b, a) < 0) ? a : b;
-    }
-
-    /**
-     * This is used for value comparison in post-filtering - {@link Expression#isSatisfiedBy(ByteBuffer)}.
-     * <p>
-     * This allows types to decide whether they should be compared based on their encoded value or their
-     * raw value. At present only {@link InetAddressType} values are compared by their encoded values to
-     * allow for ipv4 -> ipv6 equivalency in searches.
-     */
-    public int comparePostFilter(Expression.Value requestedValue, Expression.Value columnValue)
-    {
-        if (isInetAddress())
-            return compareInet(requestedValue.encoded, columnValue.encoded);
-            // Override comparisons for frozen collections and composite types (map entries)
-        else if (isComposite() || isFrozen())
-            return FastByteOperations.compareUnsigned(requestedValue.raw, columnValue.raw);
-
-        return indexType.compare(requestedValue.raw, columnValue.raw);
-    }
-
-    /**
-     * Fills a byte array with the comparable bytes for a type.
-     * <p>
-     * This method expects a {@code value} parameter generated by calling {@link #asIndexBytes(ByteBuffer)}.
-     * It is not generally safe to pass the output of other serialization methods to this method.  For instance, it is
-     * not generally safe to pass the output of {@link AbstractType#decompose(Object)} as the {@code value} parameter
-     * (there are certain types for which this is technically OK, but that doesn't hold for all types).
-     *
-     * @param value a value buffer returned by {@link #asIndexBytes(ByteBuffer)}
-     * @param bytes this method's output
-     */
-    public void toComparableBytes(ByteBuffer value, byte[] bytes)
-    {
-        if (isInetAddress())
-            ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, INET_ADDRESS_SIZE);
-        else if (isBigInteger())
-            ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, BIG_INTEGER_APPROXIMATION_BYTES);
-        else if (isBigDecimal())
-            ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, DECIMAL_APPROXIMATION_BYTES);
-        else
-            ByteSourceInverse.copyBytes(asComparableBytes(value, ByteComparable.Version.OSS50), bytes);
-    }
-
-    public ByteSource asComparableBytes(ByteBuffer value, ByteComparable.Version version)
-    {
-        if (isInetAddress() || isBigInteger() || isBigDecimal())
-            return ByteSource.optionalFixedLength(ByteBufferAccessor.instance, value);
-        else if (isLong())
-            // The LongType.asComparableBytes uses variableLengthInteger which doesn't play well with
-            // the balanced tree because it is expecting fixed length data. So for SAI we use a optionalSignedFixedLengthNumber
-            // to keep all comparable values the same length
-            return ByteSource.optionalSignedFixedLengthNumber(ByteBufferAccessor.instance, value);
-        else if (isFrozen())
-            // We need to override the default frozen implementation here because it will defer to the underlying
-            // type's implementation which will be incorrect, for us, for the case of multi-cell types.
-            return ByteSource.of(value, version);
-        return indexType.asComparableBytes(value, version);
-    }
-
-    /**
-     * Translates the external value of specific types into a format used by the index.
-     */
-    public ByteBuffer asIndexBytes(ByteBuffer value)
-    {
-        if (value == null)
-            return null;
-
-        if (isInetAddress())
-            return encodeInetAddress(value);
-        else if (isBigInteger())
-            return encodeBigInteger(value);
-        else if (isBigDecimal())
-            return encodeDecimal(value);
-        return value;
-    }
-
-    public float[] decomposeVector(ByteBuffer byteBuffer)
-    {
-        assert isVector();
-        return ((VectorType<?>) indexType).composeAsFloat(byteBuffer);
-    }
-
-    public boolean supports(Operator operator)
-    {
-        if (operator == Operator.LIKE ||
-            operator == Operator.LIKE_CONTAINS ||
-            operator == Operator.LIKE_PREFIX ||
-            operator == Operator.LIKE_MATCHES ||
-            operator == Operator.LIKE_SUFFIX) return false;
-
-        // ANN is only supported against vectors, and vector indexes only support ANN
-        if (operator == Operator.ANN)
-            return isVector();
-
-        Expression.IndexOperator indexOperator = Expression.IndexOperator.valueOf(operator);
-
-        if (isNonFrozenCollection())
-        {
-            if (indexTargetType == IndexTarget.Type.KEYS) return indexOperator == Expression.IndexOperator.CONTAINS_KEY;
-            if (indexTargetType == IndexTarget.Type.VALUES) return indexOperator == Expression.IndexOperator.CONTAINS_VALUE;
-            return indexTargetType == IndexTarget.Type.KEYS_AND_VALUES && indexOperator == Expression.IndexOperator.EQ;
-        }
-
-        if (indexTargetType == IndexTarget.Type.FULL)
-            return indexOperator == Expression.IndexOperator.EQ;
-
-        if (indexOperator != Expression.IndexOperator.EQ && EQ_ONLY_TYPES.contains(indexType)) return false;
-
-        // RANGE only applicable to non-literal indexes
-        return (indexOperator != null) && !(isLiteral() && indexOperator == Expression.IndexOperator.RANGE);
-    }
-
-    @Override
-    public String toString()
-    {
-        return MoreObjects.toStringHelper(this)
-                          .add("column", columnMetadata)
-                          .add("type", indexType)
-                          .add("indexType", indexTargetType)
-                          .toString();
-    }
-
-    @Override
-    public boolean equals(Object obj)
-    {
-        if (obj == this)
-            return true;
-
-        if (!(obj instanceof IndexTermType))
-            return false;
-
-        IndexTermType other = (IndexTermType) obj;
-
-        return Objects.equals(columnMetadata, other.columnMetadata) && (indexTargetType == other.indexTargetType);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(columnMetadata, indexTargetType);
-    }
-
-    private EnumSet<Capability> calculateCapabilities(ColumnMetadata columnMetadata, List<ColumnMetadata> partitionKeyColumns, IndexTarget.Type indexTargetType)
-    {
-        EnumSet<Capability> capabilities = EnumSet.noneOf(Capability.class);
-
-        if (partitionKeyColumns.contains(columnMetadata) && partitionKeyColumns.size() > 1)
-            capabilities.add(Capability.COMPOSITE_PARTITION);
-
-        AbstractType<?> type = columnMetadata.type;
-
-        if (type.isReversed())
-            capabilities.add(Capability.REVERSED);
-
-        AbstractType<?> baseType = type.unwrap();
-
-        if (baseType.isCollection())
-            capabilities.add(Capability.COLLECTION);
-
-        if (baseType.isCollection() && baseType.isMultiCell())
-            capabilities.add(Capability.NON_FROZEN_COLLECTION);
-
-        if (!baseType.subTypes().isEmpty() && !baseType.isMultiCell())
-            capabilities.add(Capability.FROZEN);
-
-        AbstractType<?> indexType = calculateIndexType(baseType, capabilities, indexTargetType);
-
-        if (indexType instanceof CompositeType)
-            capabilities.add(Capability.COMPOSITE);
-        else if (!indexType.subTypes().isEmpty() && !indexType.isMultiCell())
-            capabilities.add(Capability.FROZEN);
-
-        if (indexType instanceof StringType)
-            capabilities.add(Capability.STRING);
-
-        if (indexType instanceof BooleanType)
-            capabilities.add(Capability.BOOLEAN);
-
-        if (capabilities.contains(Capability.STRING) ||
-            capabilities.contains(Capability.BOOLEAN) ||
-            capabilities.contains(Capability.FROZEN) ||
-            capabilities.contains(Capability.COMPOSITE))
-            capabilities.add(Capability.LITERAL);
-
-        if (indexType instanceof VectorType<?>)
-            capabilities.add(Capability.VECTOR);
-
-        if (indexType instanceof InetAddressType)
-            capabilities.add(Capability.INET_ADDRESS);
-
-        if (indexType instanceof IntegerType)
-            capabilities.add(Capability.BIG_INTEGER);
-
-        if (indexType instanceof DecimalType)
-            capabilities.add(Capability.BIG_DECIMAL);
-
-        if (indexType instanceof LongType)
-            capabilities.add(Capability.LONG);
-
-        return capabilities;
-    }
-
-    private AbstractType<?> calculateIndexType(AbstractType<?> baseType, EnumSet<Capability> capabilities, IndexTarget.Type indexTargetType)
-    {
-        return capabilities.contains(Capability.NON_FROZEN_COLLECTION) ? collectionCellValueType(baseType, indexTargetType) : baseType;
-    }
-
-    private Iterator<ByteBuffer> collectionIterator(ComplexColumnData cellData, long nowInSecs)
-    {
-        if (cellData == null)
-            return null;
-
-        Stream<ByteBuffer> stream = StreamSupport.stream(cellData.spliterator(), false)
-                                                 .filter(cell -> cell != null && cell.isLive(nowInSecs))
-                                                 .map(this::cellValue);
-
-        if (isInetAddress())
-            stream = stream.sorted((c1, c2) -> compareInet(encodeInetAddress(c1), encodeInetAddress(c2)));
-
-        return stream.iterator();
-    }
-
-    private ByteBuffer cellValue(Cell<?> cell)
-    {
-        if (isNonFrozenCollection())
-        {
-            switch (((CollectionType<?>) columnMetadata.type).kind)
-            {
-                case LIST:
-                    return cell.buffer();
-                case SET:
-                    return cell.path().get(0);
-                case MAP:
-                    switch (indexTargetType)
-                    {
-                        case KEYS:
-                            return cell.path().get(0);
-                        case VALUES:
-                            return cell.buffer();
-                        case KEYS_AND_VALUES:
-                            return CompositeType.build(ByteBufferAccessor.instance, cell.path().get(0), cell.buffer());
-                    }
-            }
-        }
-        return cell.buffer();
-    }
-
-    private AbstractType<?> collectionCellValueType(AbstractType<?> type, IndexTarget.Type indexType)
-    {
-        CollectionType<?> collection = ((CollectionType<?>) type);
-        switch (collection.kind)
-        {
-            case LIST:
-                return collection.valueComparator();
-            case SET:
-                return collection.nameComparator();
-            case MAP:
-                switch (indexType)
-                {
-                    case KEYS:
-                        return collection.nameComparator();
-                    case VALUES:
-                        return collection.valueComparator();
-                    case KEYS_AND_VALUES:
-                        return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator());
-                }
-            default:
-                throw new IllegalArgumentException("Unsupported collection type: " + collection.kind);
-        }
-    }
-
-    private boolean isCompositePartition()
-    {
-        return capabilities.contains(Capability.COMPOSITE_PARTITION);
-    }
-
-    /**
-     * Returns <code>true</code> if given {@link AbstractType} is {@link InetAddressType}
-     */
-    private boolean isInetAddress()
-    {
-        return capabilities.contains(Capability.INET_ADDRESS);
-    }
-
-    /**
-     * Returns <code>true</code> if given {@link AbstractType} is {@link IntegerType}
-     */
-    private boolean isBigInteger()
-    {
-        return capabilities.contains(Capability.BIG_INTEGER);
-    }
-
-    /**
-     * Returns <code>true</code> if given {@link AbstractType} is {@link DecimalType}
-     */
-    private boolean isBigDecimal()
-    {
-        return capabilities.contains(Capability.BIG_DECIMAL);
-    }
-
-    private boolean isLong()
-    {
-        return capabilities.contains(Capability.LONG);
-    }
-
-    /**
-     * Compares 2 InetAddress terms by ensuring that both addresses are represented as
-     * ipv6 addresses.
-     */
-    private static int compareInet(ByteBuffer b1, ByteBuffer b2)
-    {
-        assert isIPv6(b1) && isIPv6(b2);
-
-        return FastByteOperations.compareUnsigned(b1, b2);
-    }
-
-    private static boolean isIPv6(ByteBuffer address)
-    {
-        return address.remaining() == INET_ADDRESS_SIZE;
-    }
-
-    /**
-     * Encode a {@link InetAddress} into a fixed width 16 byte encoded value.
-     * <p>
-     * The encoded value is byte comparable and prefix compressible.
-     * <p>
-     * The encoding is done by converting ipv4 addresses to their ipv6 equivalent.
-     */
-    private static ByteBuffer encodeInetAddress(ByteBuffer value)
-    {
-        if (value.remaining() == 4)
-        {
-            int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
-            ByteBuffer mapped = ByteBuffer.allocate(INET_ADDRESS_SIZE);
-            System.arraycopy(IPV4_PREFIX, 0, mapped.array(), 0, IPV4_PREFIX.length);
-            ByteBufferUtil.copyBytes(value, position, mapped, IPV4_PREFIX.length, value.remaining());
-            return mapped;
-        }
-        return value;
-    }
-
-    /**
-     * Encode a {@link BigInteger} into a fixed width 20 byte encoded value. The encoded value is byte comparable
-     * and prefix compressible.
-     * <p>
-     * The format of the encoding is:
-     * <p>
-     *  The first 4 bytes contain the integer length of the {@link BigInteger} byte array
-     *  with the top bit flipped for positive values.
-     * <p>
-     *  The remaining 16 bytes contain the 16 most significant bytes of the
-     *  {@link BigInteger} byte array.
-     * <p>
-     *  For {@link BigInteger} values whose underlying byte array is less than
-     *  16 bytes, the encoded value is sign extended.
-     */
-    public static ByteBuffer encodeBigInteger(ByteBuffer value)
-    {
-        int size = value.remaining();
-        int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
-        byte[] bytes = new byte[BIG_INTEGER_APPROXIMATION_BYTES];
-        if (size < BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES)
-        {
-            ByteBufferUtil.copyBytes(value, position, bytes, bytes.length - size, size);
-            if ((bytes[bytes.length - size] & 0x80) != 0)
-                Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0xff);
-            else
-                Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0x00);
-        }
-        else
-        {
-            ByteBufferUtil.copyBytes(value, position, bytes, Integer.BYTES, BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES);
-        }
-        if ((bytes[4] & 0x80) != 0)
-        {
-            size = -size;
-        }
-        bytes[0] = (byte)(size >> 24 & 0xff);
-        bytes[1] = (byte)(size >> 16 & 0xff);
-        bytes[2] = (byte)(size >> 8 & 0xff);
-        bytes[3] = (byte)(size & 0xff);
-        bytes[0] ^= 0x80;
-        return ByteBuffer.wrap(bytes);
-    }
-
-    public static ByteBuffer encodeDecimal(ByteBuffer value)
-    {
-        ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS50);
-        bs = ByteSource.cutOrRightPad(bs, DECIMAL_APPROXIMATION_BYTES, 0);
-        return ByteBuffer.wrap(ByteSourceInverse.readBytes(bs, DECIMAL_APPROXIMATION_BYTES));
-    }
-}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/LowPriorityThreadFactory.java b/src/java/org/apache/cassandra/index/sai/utils/LowPriorityThreadFactory.java
new file mode 100644
index 000000000000..41cfdd41a6a2
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/LowPriorityThreadFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinWorkerThread;
+
+public class LowPriorityThreadFactory implements ForkJoinPool.ForkJoinWorkerThreadFactory
+{
+    @Override
+    public ForkJoinWorkerThread newThread(ForkJoinPool pool) {
+        ForkJoinWorkerThread worker = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool);
+        worker.setPriority(Thread.MIN_PRIORITY);
+        return worker;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/MemtableOrdering.java b/src/java/org/apache/cassandra/index/sai/utils/MemtableOrdering.java
new file mode 100644
index 000000000000..3989192e337b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/MemtableOrdering.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.List;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.utils.CloseableIterator;
+
+/***
+ * Analogue of SegmentOrdering, but for memtables.
+ */
+public interface MemtableOrdering
+{
+
+    /**
+     * Order the index based on the given expression.
+     *
+     * @param queryContext - the query context
+     * @param orderer      - the expression to order by
+     * @param slice    - the expression to restrict index search by
+     * @param keyRange     - the key range to search
+     * @param limit        - can be used to inform the search, but should not be used to prematurely limit the iterator
+     * @return an iterator over the results in score order.
+     */
+    List<CloseableIterator<PrimaryKeyWithSortKey>> orderBy(QueryContext queryContext,
+                                                           Orderer orderer,
+                                                           Expression slice,
+                                                           AbstractBounds<PartitionPosition> keyRange,
+                                                           int limit);
+
+    /**
+     * Order the given list of {@link PrimaryKey} results corresponding to the given expression.
+     * Returns an iterator over the results in score order.
+     *
+     * Assumes that the given  spans the same rows as the implementing index's segment.
+     */
+    CloseableIterator<PrimaryKeyWithSortKey> orderResultsBy(QueryContext context, List<PrimaryKey> keys, Orderer orderer, int limit);
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/NamedMemoryLimiter.java b/src/java/org/apache/cassandra/index/sai/utils/NamedMemoryLimiter.java
index 0a5fdf69c2b9..988fb1be44ca 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/NamedMemoryLimiter.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/NamedMemoryLimiter.java
@@ -20,7 +20,6 @@
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.concurrent.ThreadSafe;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,17 +33,16 @@ public final class NamedMemoryLimiter
 {
     private static final Logger logger = LoggerFactory.getLogger(NamedMemoryLimiter.class);
     
+    private final long limitBytes;
     private final AtomicLong bytesUsed = new AtomicLong(0);
     private final String scope;
 
-    private long limitBytes;
-
     public NamedMemoryLimiter(long limitBytes, String scope)
     {
         this.limitBytes = limitBytes;
         this.scope = scope;
 
-        logger.info("[{}]: Memory limiter using limit of {}...", scope, FBUtilities.prettyPrintMemory(limitBytes));
+        logger.debug("[{}]: Memory limiter using limit of {}...", scope, FBUtilities.prettyPrintMemory(limitBytes));
     }
 
     /**
@@ -59,29 +57,23 @@ public long increment(long bytes)
     {
         if (logger.isTraceEnabled())
             logger.trace("[{}]: Incrementing tracked memory usage by {} bytes from current usage of {}...", scope, bytes, currentBytesUsed());
-        return bytesUsed.addAndGet(bytes);
+        return this.bytesUsed.addAndGet(bytes);
     }
 
     public long decrement(long bytes)
     {
         if (logger.isTraceEnabled())
             logger.trace("[{}]: Decrementing tracked memory usage by {} bytes from current usage of {}...", scope, bytes, currentBytesUsed());
-        return bytesUsed.addAndGet(-bytes);
+        return this.bytesUsed.addAndGet(-bytes);
     }
 
     public long currentBytesUsed()
     {
-        return bytesUsed.get();
+        return this.bytesUsed.get();
     }
     
     public long limitBytes()
     {
-        return limitBytes;
-    }
-
-    @VisibleForTesting
-    public void setLimitBytes(long bytes)
-    {
-        limitBytes = bytes;
+        return this.limitBytes;
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/utils/OrderingFilterRangeIterator.java b/src/java/org/apache/cassandra/index/sai/utils/OrderingFilterRangeIterator.java
new file mode 100644
index 000000000000..40a47358e595
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/OrderingFilterRangeIterator.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * An iterator that consumes a chunk of {@link PrimaryKey}s from the {@link KeyRangeIterator}, passes them to the
+ * {@link Function} to filter the chunk of {@link PrimaryKey}s and then pass the results to next consumer.
+ * The PKs are currently returned in score order.
+ */
+@NotThreadSafe
+public class OrderingFilterRangeIterator<T> implements Iterator<T>, AutoCloseable
+{
+    private final KeyRangeIterator input;
+    private final QueryContext context;
+    private final int chunkSize;
+    private final Function<List<PrimaryKey>, T> nextRangeFunction;
+
+    public OrderingFilterRangeIterator(KeyRangeIterator input,
+                                       int chunkSize,
+                                       QueryContext context,
+                                       Function<List<PrimaryKey>, T> nextRangeFunction)
+    {
+        this.input = input;
+        this.chunkSize = chunkSize;
+        this.context = context;
+        this.nextRangeFunction = nextRangeFunction;
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        return input.hasNext();
+    }
+
+    @Override
+    public T next()
+    {
+        List<PrimaryKey> nextKeys = new ArrayList<>(chunkSize);
+        do
+        {
+            nextKeys.add(input.next());
+        }
+        while (nextKeys.size() < chunkSize && input.hasNext());
+        context.addRowsFiltered(nextKeys.size());
+        return nextRangeFunction.apply(nextKeys);
+    }
+
+    public void close() {
+        FileUtils.closeQuietly(input);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PartitionInfo.java b/src/java/org/apache/cassandra/index/sai/utils/PartitionInfo.java
index c8e1c62b6543..cc322e4c077c 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/PartitionInfo.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/PartitionInfo.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.index.sai.utils;
 
 import java.util.Objects;
+
 import javax.annotation.Nullable;
 
 import org.apache.cassandra.db.DecoratedKey;
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKey.java b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKey.java
index c079a66c4c48..cb99b7c6da1b 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKey.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKey.java
@@ -17,476 +17,166 @@
  */
 package org.apache.cassandra.index.sai.utils;
 
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Objects;
-import java.util.stream.Collectors;
+import java.util.function.Supplier;
 
-import org.apache.cassandra.db.BufferDecoratedKey;
+import io.github.jbellis.jvector.util.Accountable;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.index.sai.disk.format.IndexFeatureSet;
+import org.apache.cassandra.index.sai.disk.v1.PartitionAwarePrimaryKeyFactory;
+import org.apache.cassandra.index.sai.disk.v2.RowAwarePrimaryKeyFactory;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
 /**
  * Representation of the primary key for a row consisting of the {@link DecoratedKey} and
  * {@link Clustering} associated with a {@link org.apache.cassandra.db.rows.Row}.
- * The {@link Factory.TokenOnlyPrimaryKey} is used by the {@link org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher} to
- * position the search within the query range.
+ *
+ * For legacy V1 support only the {@link DecoratedKey} will ever be supported for a row.
+ *
+ * For the V2 on-disk format the {@link DecoratedKey} and {@link Clustering} are supported.
+ *
  */
-public interface PrimaryKey extends Comparable<PrimaryKey>, ByteComparable
+public interface PrimaryKey extends Comparable<PrimaryKey>, Accountable
 {
     /**
-     * See the javadoc for {@link #kind()} for how this enum is used.
-      */
-    enum Kind
-    {
-        TOKEN(false),
-        SKINNY(false),
-        WIDE(true),
-        STATIC(true);
-
-        public final boolean hasClustering;
-
-        Kind(boolean hasClustering)
-        {
-            this.hasClustering = hasClustering;
-        }
-
-        public boolean isIntersectable(Kind other)
-        {
-            if (this == TOKEN)
-                return other == TOKEN;
-            else if (this == SKINNY)
-                return other == SKINNY;
-            else if (this == WIDE || this == STATIC)
-                return other == WIDE || other == STATIC;
-            
-            throw new AssertionError("Unknown Kind: " + other);
-        }
-    }
-
-    class Factory
+     * A factory for creating {@link PrimaryKey} instances
+     */
+    interface Factory
     {
-        private final IPartitioner partitioner;
-        private final ClusteringComparator clusteringComparator;
-
-        public Factory(IPartitioner partitioner, ClusteringComparator clusteringComparator)
-        {
-            this.partitioner = partitioner;
-            this.clusteringComparator = clusteringComparator;
-        }
-
         /**
          * Creates a {@link PrimaryKey} that is represented by a {@link Token}.
-         * <p>
+         *
          * {@link Token} only primary keys are used for defining the partition range
          * of a query.
+         *
+         * @param token the {@link Token}
+         * @return a {@link PrimaryKey} represented by a token only
          */
-        public PrimaryKey create(Token token)
-        {
-            assert token != null : "Cannot create a primary key with a null token";
-
-            return new TokenOnlyPrimaryKey(token);
-        }
-
-        /**
-         * Create a {@link PrimaryKey} for tables without clustering columns
-         */
-        public PrimaryKey create(DecoratedKey partitionKey)
-        {
-            assert clusteringComparator.size() == 0 : "Cannot create a skinny primary key for a table with clustering columns";
-            assert partitionKey != null : "Cannot create a primary key with a null partition key";
-
-            return new SkinnyPrimaryKey(partitionKey);
-        }
-
-        /**
-         * Creates a {@link PrimaryKey} that is fully represented by partition key
-         * and clustering.
-         */
-        public PrimaryKey create(DecoratedKey partitionKey, Clustering<?> clustering)
-        {
-            assert clusteringComparator.size() > 0 : "Cannot create a wide primary key for a table without clustering columns";
-            assert partitionKey != null : "Cannot create a primary key with a null partition key";
-            assert clustering != null : "Cannot create a primary key with a null clustering";
-
-            return clustering == Clustering.STATIC_CLUSTERING ? new StaticPrimaryKey(partitionKey) : new WidePrimaryKey(partitionKey, clustering);
-        }
+        PrimaryKey createTokenOnly(Token token);
 
         /**
-         * Create a {@link PrimaryKey} from a {@link ByteSource}. This should only be used with {@link ByteSource} instances
-         * created by calls to {@link PrimaryKey#asComparableBytes(Version)}.
+         * Creates a {@link PrimaryKey} that is represented by a {@link DecoratedKey}.
+         *
+         * {@link DecoratedKey} only primary keys are used to define the minumum and
+         * maximum coverage of an index.
+         *
+         * @param partitionKey the {@link DecoratedKey}
+         * @return a {@link PrimaryKey} represented by a partition key only
          */
-        public PrimaryKey fromComparableBytes(ByteSource byteSource)
+        default PrimaryKey createPartitionKeyOnly(DecoratedKey partitionKey)
         {
-            if (clusteringComparator.size() > 0)
-            {
-                ByteSource.Peekable peekable = ByteSource.peekable(byteSource);
-                DecoratedKey partitionKey = partitionKeyFromComparableBytes(ByteSourceInverse.nextComponentSource(peekable));
-                Clustering<?> clustering = clusteringFromByteComparable(ByteSourceInverse.nextComponentSource(peekable));
-                return create(partitionKey, clustering);
-            }
-            else
-            {
-                return create(partitionKeyFromComparableBytes(byteSource));
-            }
+            return create(partitionKey, Clustering.EMPTY);
         }
 
         /**
-         * Create a {@link DecoratedKey} from a {@link ByteSource}. This is a separate method because of it's use by
-         * the {@link org.apache.cassandra.index.sai.disk.PrimaryKeyMap} implementations to create partition keys.
+         * Creates a {@link PrimaryKey} with deferred loading. Deferred loading means
+         * that the key will only be fully loaded when the full representation of the
+         * key is needed for comparison. Before the key is loaded it will be represented
+         * by a token only, so it will only need loading if the token is matched in a
+         * comparison or the byte comparable representation of the key is required.
+         *
+         * @param token the {@link Token}
+         * @param primaryKeySupplier the supplier of the full key
+         * @return a {@link PrimaryKey} the token and a primary key supplier
          */
-        public DecoratedKey partitionKeyFromComparableBytes(ByteSource byteSource)
-        {
-            ByteBuffer decoratedKey = ByteBuffer.wrap(ByteSourceInverse.getUnescapedBytes(ByteSource.peekable(byteSource)));
-            return new BufferDecoratedKey(partitioner.getToken(decoratedKey), decoratedKey);
-        }
+        PrimaryKey createDeferred(Token token, Supplier<PrimaryKey> primaryKeySupplier);
 
         /**
-         * Create a {@link Clustering} from a {@link ByteSource}. This is a separate method because of its use by
-         * the {@link org.apache.cassandra.index.sai.disk.v1.WidePrimaryKeyMap} to create its clustering keys.
+         * Creates a {@link PrimaryKey} that is fully represented by partition key
+         * and clustering.
+         *
+         * @param partitionKey the {@link DecoratedKey}
+         * @param clustering the {@link Clustering}
+         * @return a {@link PrimaryKey} contain the partition key and clustering
          */
-        public Clustering<?> clusteringFromByteComparable(ByteSource byteSource)
-        {
-            Clustering<?> clustering = clusteringComparator.clusteringFromByteComparable(ByteBufferAccessor.instance, v -> byteSource);
-
-            // Clustering is null for static rows
-            return (clustering == null) ? Clustering.STATIC_CLUSTERING : clustering;
-        }
-
-        class TokenOnlyPrimaryKey implements PrimaryKey
-        {
-            protected final Token token;
-
-            TokenOnlyPrimaryKey(Token token)
-            {
-                this.token = token;
-            }
-
-            @Override
-            public Kind kind()
-            {
-                return Kind.TOKEN;
-            }
-
-            @Override
-            public Token token()
-            {
-                return token;
-            }
-
-            @Override
-            public DecoratedKey partitionKey()
-            {
-                throw new UnsupportedOperationException();
-            }
-
-            @Override
-            public Clustering<?> clustering()
-            {
-                throw new UnsupportedOperationException();
-            }
-
-            @Override
-            public ByteSource asComparableBytes(Version version)
-            {
-                throw new UnsupportedOperationException();
-            }
-
-            @Override
-            public int compareTo(PrimaryKey o)
-            {
-                return token().compareTo(o.token());
-            }
-
-            @Override
-            public int hashCode()
-            {
-                return Objects.hash(token(), clusteringComparator);
-            }
-
-            @Override
-            public boolean equals(Object obj)
-            {
-                if (obj instanceof PrimaryKey)
-                    return compareTo((PrimaryKey) obj) == 0;
-                return false;
-            }
-
-            @Override
-            public String toString()
-            {
-                return String.format("PrimaryKey: { token: %s }", token());
-            }
-        }
-
-        class SkinnyPrimaryKey extends TokenOnlyPrimaryKey
-        {
-            protected final DecoratedKey partitionKey;
-
-            SkinnyPrimaryKey(DecoratedKey partitionKey)
-            {
-                super(partitionKey.getToken());
-                this.partitionKey = partitionKey;
-            }
-
-            @Override
-            public Kind kind()
-            {
-                return Kind.SKINNY;
-            }
-
-            @Override
-            public DecoratedKey partitionKey()
-            {
-                return partitionKey;
-            }
-
-            @Override
-            public ByteSource asComparableBytes(Version version)
-            {
-                return ByteSource.of(partitionKey().getKey(), version);
-            }
-
-            @Override
-            public int compareTo(PrimaryKey o)
-            {
-                int cmp = super.compareTo(o);
-
-                // If the tokens don't match then we don't need to compare any more of the key.
-                // Otherwise, if the other key is token only we can only compare tokens
-                // This is used by the ResultRetriever to skip to the current key range start position
-                // during result retrieval.
-                if ((cmp != 0) || o.kind() == Kind.TOKEN)
-                    return cmp;
-
-                // Finally compare the partition keys
-                return partitionKey().compareTo(o.partitionKey());
-            }
-
-            @Override
-            public int hashCode()
-            {
-                return Objects.hash(token(), partitionKey(), Clustering.EMPTY, clusteringComparator);
-            }
-
-            @Override
-            public String toString()
-            {
-                return String.format("PrimaryKey: { token: %s, partition: %s }", token(), partitionKey());
-            }
-        }
-
-        class StaticPrimaryKey extends SkinnyPrimaryKey
-        {
-            StaticPrimaryKey(DecoratedKey partitionKey)
-            {
-                super(partitionKey);
-            }
-
-            @Override
-            public Kind kind()
-            {
-                return Kind.STATIC;
-            }
-
-            @Override
-            public Clustering<?> clustering()
-            {
-                return Clustering.STATIC_CLUSTERING;
-            }
-
-            @Override
-            public ByteSource asComparableBytes(ByteComparable.Version version)
-            {
-                ByteSource keyComparable = ByteSource.of(partitionKey().getKey(), version);
-                // Static clustering cannot be serialized or made to a byte comparable, so we use null as the component.
-                return ByteSource.withTerminator(version == ByteComparable.Version.LEGACY ? ByteSource.END_OF_STREAM
-                                                                                          : ByteSource.TERMINATOR,
-                                                 keyComparable,
-                                                 null);
-            }
-
-            @Override
-            public int compareTo(PrimaryKey o)
-            {
-                int cmp = super.compareTo(o);
-                if (cmp != 0 || o.kind() == Kind.TOKEN || o.kind() == Kind.SKINNY)
-                    return cmp;
-                // At this point the other key is in the same partition as this static key so is equal to it. This
-                // has to be the case because otherwise, intersections between static column indexes and ordinary
-                // indexes will fail.
-                return 0;
-            }
-
-            @Override
-            public int compareToStrict(PrimaryKey o)
-            {
-                int cmp = compareTo(o);
-                // Always order this STATIC key before a WIDE key in the same partition, as this corresponds to the
-                // order of the corresponding row IDs in an on-disk postings list.
-                return o.kind() == Kind.WIDE && cmp == 0 ? -1 : cmp;
-            }
-
-            @Override
-            public int hashCode()
-            {
-                return Objects.hash(token(), partitionKey(), Clustering.STATIC_CLUSTERING, clusteringComparator);
-            }
-
-            @Override
-            public String toString()
-            {
-                return String.format("PrimaryKey: { token: %s, partition: %s, clustering: STATIC } ", token(), partitionKey());
-            }
-
-            @Override
-            public PrimaryKey toStatic()
-            {
-                return this;
-            }
-        }
-
-        class WidePrimaryKey extends SkinnyPrimaryKey
-        {
-            private final Clustering<?> clustering;
-
-            WidePrimaryKey(DecoratedKey partitionKey, Clustering<?> clustering)
-            {
-                super(partitionKey);
-                this.clustering = clustering;
-            }
-
-            @Override
-            public Kind kind()
-            {
-                return Kind.WIDE;
-            }
-
-            @Override
-            public Clustering<?> clustering()
-            {
-                return clustering;
-            }
-
-            @Override
-            public ByteSource asComparableBytes(ByteComparable.Version version)
-            {
-                ByteSource keyComparable = ByteSource.of(partitionKey().getKey(), version);
-                // It is important that the ClusteringComparator.asBytesComparable method is used
-                // to maintain the correct clustering sort order.
-                ByteSource clusteringComparable = clusteringComparator.asByteComparable(clustering()).asComparableBytes(version);
-                return ByteSource.withTerminator(version == ByteComparable.Version.LEGACY ? ByteSource.END_OF_STREAM
-                                                                                          : ByteSource.TERMINATOR,
-                                                 keyComparable,
-                                                 clusteringComparable);
-            }
-
-            @Override
-            public int compareTo(PrimaryKey o)
-            {
-                int cmp = super.compareTo(o);
-                if (cmp != 0 || o.kind() == Kind.TOKEN || o.kind() == Kind.SKINNY)
-                    return cmp;
-                // At this point this key is in the same partition as the other key so if the other key is a static
-                // key then it must be equal to it. See comment in the compareTo for static keys above.
-                if (o.kind() == Kind.STATIC)
-                    return 0;
-                return clusteringComparator.compare(clustering(), o.clustering());
-            }
-
-            @Override
-            public int compareToStrict(PrimaryKey o)
-            {
-                int cmp = compareTo(o);
-                // Always order this WIDE key before a STATIC key in the same partition, as this corresponds to the
-                // order of the corresponding row IDs in an on-disk postings list.
-                return o.kind() == Kind.STATIC && cmp == 0 ? 1 : cmp;
-            }
-
-            @Override
-            public int hashCode()
-            {
-                return Objects.hash(token(), partitionKey(), clustering(), clusteringComparator);
-            }
-
-            @Override
-            public String toString()
-            {
-                return String.format("PrimaryKey: { token: %s, partition: %s, clustering: %s:%s } ",
-                                     token(),
-                                     partitionKey(),
-                                     clustering().kind(),
-                                     Arrays.stream(clustering().getBufferArray())
-                                           .map(ByteBufferUtil::bytesToHex)
-                                           .collect(Collectors.joining(", ")));
-            }
-
-            @Override
-            public PrimaryKey toStatic()
-            {
-                return new StaticPrimaryKey(partitionKey);
-            }
-        }
+        PrimaryKey create(DecoratedKey partitionKey, Clustering clustering);
     }
 
     /**
-     * Returns the {@link Kind} of the {@link PrimaryKey}. The {@link Kind} is used locally in the {@link #compareTo(Object)}
-     * methods to determine how far the comparision needs to go between keys.
-     * <p>
-     * The {@link Kind} values have a categorization of {@code isClustering}. This indicates whether the key belongs to
-     * a table with clustering tables or not.
+     * Returns a {@link Factory} for creating {@link PrimaryKey} instances. The factory
+     * returned is based on the capabilities of the {@link IndexFeatureSet}.
+     *
+     * @param clusteringComparator the {@link ClusteringComparator} used by the
+     *                             {@link RowAwarePrimaryKeyFactory} for clustering comparisons
+     * @param indexFeatureSet the {@link IndexFeatureSet} used to decide the type of
+     *                        factory to use
+     * @return a {@link Factory} for {@link PrimaryKey} creation
      */
-    Kind kind();
+    static Factory factory(ClusteringComparator clusteringComparator, IndexFeatureSet indexFeatureSet)
+    {
+        return indexFeatureSet.isRowAware() ? new RowAwarePrimaryKeyFactory(clusteringComparator)
+                                            : new PartitionAwarePrimaryKeyFactory();
+    }
 
     /**
-     * Returns the {@link Token} component of the {@link PrimaryKey}
+     * Returns the {@link Token} associated with this primary key.
+     *
+     * @return the {@link Token}
      */
     Token token();
 
     /**
-     * Returns the {@link DecoratedKey} representing the partition key of the {@link PrimaryKey}.
-     * <p>
-     * Note: This cannot be null but some {@link PrimaryKey} implementations can throw {@link UnsupportedOperationException}
-     * if they do not support partition keys.
+     * Returns the {@link DecoratedKey} associated with this primary key.
+     *
+     * @return the {@link DecoratedKey}
      */
     DecoratedKey partitionKey();
 
     /**
-     * Returns the {@link Clustering} representing the clustering component of the {@link PrimaryKey}.
-     * <p>
-     * Note: This cannot be null but some {@link PrimaryKey} implementations can throw {@link UnsupportedOperationException}
-     * if they do not support clustering columns.
+     * Returns the {@link Clustering} associated with this primary key
+     *
+     * @return the {@link Clustering}
      */
-    Clustering<?> clustering();
+    Clustering clustering();
+
+    /**
+     * Return whether the primary key has an empty clustering or not.
+     * By default the clustering is empty if the internal clustering
+     * is null or is empty.
+     *
+     * @return {@code true} if the clustering is empty, otherwise {@code false}
+     */
+    default boolean hasEmptyClustering()
+    {
+        return clustering() == null || clustering().isEmpty();
+    }
+
+    /**
+     * Load the primary key from the {@link Supplier (PrimaryKey)} (if one
+     * is available) and fully populate the primary key.
+     *
+     * @return the fully populated {@link PrimaryKey}
+     */
+    PrimaryKey loadDeferred();
 
     /**
      * Returns the {@link PrimaryKey} as a {@link ByteSource} byte comparable representation.
-     * <p>
+     *
      * It is important that these representations are only ever used with byte comparables using
      * the same elements. This means that {@code asComparableBytes} responses can only be used
      * together from the same {@link PrimaryKey} implementation.
      *
      * @param version the {@link ByteComparable.Version} to use for the implementation
      * @return the {@code ByteSource} byte comparable.
-     * @throws UnsupportedOperationException for {@link PrimaryKey} implementations that are not byte-comparable
      */
     ByteSource asComparableBytes(ByteComparable.Version version);
 
-    default PrimaryKey toStatic()
-    {
-        throw new UnsupportedOperationException("Only STATIC and WIDE keys can be converted to STATIC");
-    }
+    /**
+     * Returns the {@link PrimaryKey} as a {@link ByteSource} min prefix byte comparable representation.
+     *
+     * @param version the {@link ByteComparable.Version} to use for the implementation
+     * @return the {@code ByteSource} min prefix byte comparable.
+     */
+    ByteSource asComparableBytesMinPrefix(ByteComparable.Version version);
 
-    default int compareToStrict(PrimaryKey o)
-    {
-        return compareTo(o);
-    }
+    /**
+     * Returns the {@link PrimaryKey} as a {@link ByteSource} max prefix byte comparable representation.
+     *
+     * @param version the {@link ByteComparable.Version} to use for the implementation
+     * @return the {@code ByteSource} max prefix byte comparable.
+     */
+    ByteSource asComparableBytesMaxPrefix(ByteComparable.Version version);
 }
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithByteComparable.java b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithByteComparable.java
new file mode 100644
index 000000000000..c2a3c9bf69eb
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithByteComparable.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+/**
+ * A {@link PrimaryKey} that includes a {@link ByteComparable} value from a source index.
+ * Note: this class has a natural ordering that is inconsistent with equals.
+ */
+public class PrimaryKeyWithByteComparable extends PrimaryKeyWithSortKey
+{
+    private final ByteComparable byteComparable;
+
+    public PrimaryKeyWithByteComparable(IndexContext context, Memtable sourceTable, PrimaryKey primaryKey, ByteComparable byteComparable)
+    {
+        super(context, sourceTable, primaryKey);
+        this.byteComparable = byteComparable;
+    }
+
+    public PrimaryKeyWithByteComparable(IndexContext context, SSTableId sourceTable, PrimaryKey primaryKey, ByteComparable byteComparable)
+    {
+        super(context, sourceTable, primaryKey);
+        this.byteComparable = byteComparable;
+    }
+
+    @Override
+    protected boolean isIndexDataEqualToLiveData(ByteBuffer value)
+    {
+        if (context.isLiteral())
+        {
+            ByteSource byteSource = byteComparable.asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION);
+            byte[] indexedValue = ByteSourceInverse.readBytes(byteSource);
+            byte[] liveValue = ByteBufferUtil.getArray(value);
+            return Arrays.compare(indexedValue, liveValue) == 0;
+        }
+        else
+        {
+            var peekableBytes = ByteSource.peekable(byteComparable.asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION));
+            var bytes = context.getValidator().fromComparableBytes(peekableBytes, TypeUtil.BYTE_COMPARABLE_VERSION);
+            return value.compareTo(bytes) == 0;
+        }
+    }
+
+    @Override
+    public int compareTo(PrimaryKey o)
+    {
+        if (!(o instanceof PrimaryKeyWithByteComparable))
+            throw new IllegalArgumentException("Cannot compare PrimaryKeyWithByteComparable with " + o.getClass().getSimpleName());
+
+        return ByteComparable.compare(byteComparable, ((PrimaryKeyWithByteComparable) o).byteComparable, TypeUtil.BYTE_COMPARABLE_VERSION);
+    }
+
+    @Override
+    public long ramBytesUsed()
+    {
+        return super.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithScore.java b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithScore.java
new file mode 100644
index 000000000000..b88c210d65f2
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithScore.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.io.sstable.SSTableId;
+
+/**
+ * A {@link PrimaryKey} that includes a score from a source index.
+ * Note: this class has a natural ordering that is inconsistent with equals.
+ */
+public class PrimaryKeyWithScore extends PrimaryKeyWithSortKey
+{
+    public final float indexScore;
+
+    public PrimaryKeyWithScore(IndexContext context, Memtable source, PrimaryKey primaryKey, float indexScore)
+    {
+        super(context, source, primaryKey);
+        this.indexScore = indexScore;
+    }
+
+    public PrimaryKeyWithScore(IndexContext context, SSTableId source, PrimaryKey primaryKey, float indexScore)
+    {
+        super(context, source, primaryKey);
+        this.indexScore = indexScore;
+    }
+
+    @Override
+    protected boolean isIndexDataEqualToLiveData(ByteBuffer value)
+    {
+        // Vector indexes handle updated rows properly and not allow a row to have more than one value in the same
+        // index segment. Therefore, there is no need to validate the index data against the live data.
+        return true;
+    }
+
+    @Override
+    public int compareTo(PrimaryKey o)
+    {
+        if (!(o instanceof PrimaryKeyWithScore))
+            throw new IllegalArgumentException("Cannot compare PrimaryKeyWithScore with " + o.getClass().getSimpleName());
+
+        // Descending order
+        return Float.compare(((PrimaryKeyWithScore) o).indexScore, indexScore);
+    }
+
+    @Override
+    public long ramBytesUsed()
+    {
+        // Include super class fields plus float value
+        return super.ramBytesUsed() + Float.BYTES;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithSortKey.java b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithSortKey.java
new file mode 100644
index 000000000000..4da4a8403030
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyWithSortKey.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.nio.ByteBuffer;
+
+import io.github.jbellis.jvector.util.RamUsageEstimator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * A PrimaryKey with one piece of metadata. Subclasses define the metadata, and to prevent unnecessary boxing, the
+ * metadata is not referenced in this calss. The metadata is not used to determine equality or hash code, but it is used
+ * to compare the PrimaryKey objects.
+ * Note: this class has a natural ordering that is inconsistent with equals.
+ */
+public abstract class PrimaryKeyWithSortKey implements PrimaryKey
+{
+    protected final IndexContext context;
+    private final PrimaryKey primaryKey;
+    // Either a Memtable reference or an SSTableId reference
+    private final Object sourceTable;
+
+    protected PrimaryKeyWithSortKey(IndexContext context, Memtable sourceTable, PrimaryKey primaryKey)
+    {
+        this.context = context;
+        this.sourceTable = sourceTable;
+        this.primaryKey = primaryKey;
+    }
+
+    protected PrimaryKeyWithSortKey(IndexContext context, SSTableId sourceTable, PrimaryKey primaryKey)
+    {
+        this.context = context;
+        this.sourceTable = sourceTable;
+        this.primaryKey = primaryKey;
+    }
+
+    public PrimaryKey primaryKey()
+    {
+        return primaryKey;
+    }
+
+    public boolean isIndexDataValid(Row row, long nowInSecs)
+    {
+        assert context.getDefinition().isRegular() : "Only regular columns are supported, got " + context.getDefinition();
+        var cell = row.getCell(context.getDefinition());
+        if (!cell.isLive(nowInSecs))
+            return false;
+        assert cell instanceof CellWithSourceTable : "Expected CellWithSource, got " + cell.getClass();
+        return sourceTable.equals(((CellWithSourceTable<?>) cell).sourceTable())
+               && isIndexDataEqualToLiveData(cell.buffer());
+    }
+
+    /**
+     * Compares the index data to the live data to ensure that the index data is still valid. This is only
+     * necessary when an index allows one row to have multiple values associated with it.
+     */
+    abstract protected boolean isIndexDataEqualToLiveData(ByteBuffer value);
+
+    @Override
+    public final int hashCode()
+    {
+        // The sort key must not affect the hash code because
+        // the same Primary Key could have different scores depending
+        // on the source sstable/index, and we store this object
+        // in a HashMap to prevent loading the same row multiple times.
+        return primaryKey.hashCode();
+    }
+
+    @Override
+    public final boolean equals(Object obj)
+    {
+        if (!(obj instanceof PrimaryKeyWithSortKey))
+            return false;
+
+        // The sort key must not affect the equality because
+        // the same Primary Key could have different scores depending
+        // on the source sstable/index, and we store this object
+        // in a HashMap to prevent loading the same row multiple times.
+        return primaryKey.equals(((PrimaryKeyWithSortKey) obj).primaryKey());
+    }
+
+
+    // Generic primary key wrapper methods:
+    @Override
+    public Token token()
+    {
+        return primaryKey.token();
+    }
+
+    @Override
+    public DecoratedKey partitionKey()
+    {
+        return primaryKey.partitionKey();
+    }
+
+    @Override
+    public Clustering<?> clustering()
+    {
+        return primaryKey.clustering();
+    }
+
+    @Override
+    public PrimaryKey loadDeferred()
+    {
+        return primaryKey.loadDeferred();
+    }
+
+    @Override
+    public ByteSource asComparableBytes(ByteComparable.Version version)
+    {
+        return primaryKey.asComparableBytes(version);
+    }
+
+    @Override
+    public ByteSource asComparableBytesMinPrefix(ByteComparable.Version version)
+    {
+        return primaryKey.asComparableBytesMinPrefix(version);
+    }
+
+    @Override
+    public ByteSource asComparableBytesMaxPrefix(ByteComparable.Version version)
+    {
+        return primaryKey.asComparableBytesMaxPrefix(version);
+    }
+
+    @Override
+    public long ramBytesUsed()
+    {
+        // Object header + 3 references (context, primaryKey, sourceTable)
+        return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER +
+               3L * RamUsageEstimator.NUM_BYTES_OBJECT_REF +
+               primaryKey.ramBytesUsed();
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeys.java b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeys.java
index 3ba7c08af07c..15431b7d3a14 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeys.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeys.java
@@ -19,10 +19,11 @@
 
 import java.util.Iterator;
 import java.util.SortedSet;
-import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ConcurrentSkipListMap;
 
-import javax.annotation.concurrent.ThreadSafe;
+import com.google.common.collect.Iterators;
 
+import org.apache.cassandra.index.sai.memory.MemoryIndex;
 import org.apache.cassandra.utils.ObjectSizes;
 
 /**
@@ -30,26 +31,52 @@
  *
  * The primary keys are sorted first by token, then by partition key value, and then by clustering.
  */
-@ThreadSafe
-public class PrimaryKeys implements Iterable<PrimaryKey>
+public class PrimaryKeys implements Iterable<MemoryIndex.PkWithFrequency>
 {
     private static final long EMPTY_SIZE = ObjectSizes.measure(new PrimaryKeys());
+
     // from https://github.com/gaul/java-collection-overhead
-    private static final long SET_ENTRY_OVERHEAD = 36;
+    private static final long MAP_ENTRY_OVERHEAD = 40 + Integer.BYTES;
+
+    private final ConcurrentSkipListMap<PrimaryKey, Integer> keys = new ConcurrentSkipListMap<>();
 
-    private final ConcurrentSkipListSet<PrimaryKey> keys = new ConcurrentSkipListSet<>();
+    /**
+     * Adds the specified {@link PrimaryKey} incrementing its frequency.
+     *
+     * @param key a primary key
+     * @return the bytes allocated for the key (0 if it already existed in the set)
+     */
+    public long addAndIncrementFrequency(PrimaryKey key)
+    {
+        return keys.compute(key, (k, v) -> v == null ? 1 : v + 1) == 1 ? MAP_ENTRY_OVERHEAD : 0;
+    }
+
+    /**
+     * Adds the specified {@link PrimaryKey} resetting its frequency to 1.
+     *
+     * @param key a primary key
+     * @return the bytes allocated for the key (0 if it already existed in the set)
+     */
+    public long addAndResetFrequency(PrimaryKey key)
+    {
+        Object prev = keys.put(key, 1);
+        return prev == null ? MAP_ENTRY_OVERHEAD : 0;
+    }
 
     /**
-     * Adds a {@link PrimaryKey} and returns the on-heap memory used if the key was added
+     * Removes the specified {@link PrimaryKey}.
+     *
+     * @param key the key to remove
+     * @return
      */
-    public long add(PrimaryKey key)
+    public long remove(PrimaryKey key)
     {
-        return keys.add(key) ? SET_ENTRY_OVERHEAD : 0;
+        return keys.remove(key) != null ? -MAP_ENTRY_OVERHEAD : 0;
     }
 
     public SortedSet<PrimaryKey> keys()
     {
-        return keys;
+        return keys.keySet();
     }
 
     public int size()
@@ -62,14 +89,15 @@ public boolean isEmpty()
         return keys.isEmpty();
     }
 
-    public long unsharedHeapSize()
+    public static long unsharedHeapSize()
     {
         return EMPTY_SIZE;
     }
 
     @Override
-    public Iterator<PrimaryKey> iterator()
+    public Iterator<MemoryIndex.PkWithFrequency> iterator()
     {
-        return keys.iterator();
+        return Iterators.transform(keys.entrySet().iterator(),
+                                   entry -> new MemoryIndex.PkWithFrequency(entry.getKey(), entry.getValue()));
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/utils/RangeUtil.java b/src/java/org/apache/cassandra/index/sai/utils/RangeUtil.java
index 8a46a2d42935..b23acc0823da 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/RangeUtil.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/RangeUtil.java
@@ -21,7 +21,9 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 
 public class RangeUtil
 {
@@ -31,4 +33,38 @@ public static boolean coversFullRing(AbstractBounds<PartitionPosition> keyRange)
     {
         return keyRange.left.equals(MIN_KEY_BOUND) && keyRange.right.equals(MIN_KEY_BOUND);
     }
+
+    /**
+     * Check if the provided {@link SSTableReader} intersects with the provided key range.
+     * @param reader SSTableReader
+     * @param keyRange key range
+     * @return true the key range intersects with the min/max key bounds
+     */
+    public static boolean intersects(SSTableReader reader, AbstractBounds<PartitionPosition> keyRange)
+    {
+        return intersects(reader.first.getToken().minKeyBound(), reader.last.getToken().maxKeyBound(), keyRange);
+    }
+
+    /**
+     * Check if the min/max key bounds intersects with the keyRange
+     * @param minKeyBound min key bound
+     * @param maxKeyBound max key bound
+     * @param keyRange key range
+     * @return true the key range intersects with the min/max key bounds
+     */
+    public static boolean intersects(Token.KeyBound minKeyBound, Token.KeyBound maxKeyBound, AbstractBounds<PartitionPosition> keyRange)
+    {
+        if (keyRange instanceof Range && ((Range<?>)keyRange).isWrapAround())
+            return keyRange.contains(minKeyBound) || keyRange.contains(maxKeyBound);
+
+        int cmp = keyRange.right.compareTo(minKeyBound);
+        // if right is minimum, it means right is the max token and bigger than maxKey.
+        // if right bound is less than minKeyBound, no intersection
+        if (!keyRange.right.isMinimum() && (!keyRange.inclusiveRight() && cmp == 0 || cmp < 0))
+            return false;
+
+        cmp = keyRange.left.compareTo(maxKeyBound);
+        // if left bound is bigger than maxKeyBound, no intersection
+        return (keyRange.isStartInclusive() || cmp != 0) && cmp <= 0;
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/utils/RowIdWithByteComparable.java b/src/java/org/apache/cassandra/index/sai/utils/RowIdWithByteComparable.java
new file mode 100644
index 000000000000..d14611ce7de1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/RowIdWithByteComparable.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+public class RowIdWithByteComparable extends RowIdWithMeta
+{
+    private final ByteComparable byteComparable;
+
+    public RowIdWithByteComparable(int segmentRowId, ByteComparable byteComparable)
+    {
+        super(segmentRowId);
+        this.byteComparable = byteComparable;
+    }
+
+    @Override
+    protected PrimaryKeyWithSortKey wrapPrimaryKey(IndexContext context, SSTableId<?> sstableId, PrimaryKey primaryKey)
+    {
+        return new PrimaryKeyWithByteComparable(context, sstableId, primaryKey, byteComparable);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/index/sai/utils/RowIdWithMeta.java b/src/java/org/apache/cassandra/index/sai/utils/RowIdWithMeta.java
new file mode 100644
index 000000000000..88f0d3bb26b1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/RowIdWithMeta.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.io.sstable.SSTableId;
+
+/**
+ * Represents a row id with additional metadata. The metadata is not a type parameter to prevent unnecessary boxing.
+ */
+public abstract class RowIdWithMeta
+{
+    private final int segmentRowId;
+
+    protected RowIdWithMeta(int segmentRowId)
+    {
+        this.segmentRowId = segmentRowId;
+    }
+
+    public final int getSegmentRowId()
+    {
+        return segmentRowId;
+    }
+
+    public PrimaryKeyWithSortKey buildPrimaryKeyWithSortKey(IndexContext indexContext,
+                                                            SSTableId<?> sstableId,
+                                                            PrimaryKeyMap primaryKeyMap,
+                                                            long segmentRowIdOffset)
+    {
+        var pk = primaryKeyMap.primaryKeyFromRowId(segmentRowIdOffset + segmentRowId);
+        return wrapPrimaryKey(indexContext, sstableId, pk);
+    }
+
+    /**
+     * Wrap the provided primary key with the stored metadata.
+     * @param indexContext the index context
+     * @param sstableId the sstable id
+     * @param primaryKey the primary key
+     * @return the wrapped primary key with its associated metadata
+     */
+    protected abstract PrimaryKeyWithSortKey wrapPrimaryKey(IndexContext indexContext, SSTableId<?> sstableId, PrimaryKey primaryKey);
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/RowIdWithScore.java b/src/java/org/apache/cassandra/index/sai/utils/RowIdWithScore.java
new file mode 100644
index 000000000000..c6ed4708e0c8
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/RowIdWithScore.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.io.sstable.SSTableId;
+
+/**
+ * Represents a row id with a score.
+ */
+public class RowIdWithScore extends RowIdWithMeta
+{
+    public final float score;
+
+    public RowIdWithScore(int segmentRowId, float score)
+    {
+        super(segmentRowId);
+        this.score = score;
+    }
+
+    public static int compare(RowIdWithScore l, RowIdWithScore r)
+    {
+        // Inverted comparison to sort in descending order
+        return Float.compare(r.score, l.score);
+    }
+
+    @Override
+    protected PrimaryKeyWithSortKey wrapPrimaryKey(IndexContext indexContext, SSTableId<?> sstableId, PrimaryKey primaryKey)
+    {
+        return new PrimaryKeyWithScore(indexContext, sstableId, primaryKey, score);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/RowWithSourceTable.java b/src/java/org/apache/cassandra/index/sai/utils/RowWithSourceTable.java
new file mode 100644
index 000000000000..93ff20600c94
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/RowWithSourceTable.java
@@ -0,0 +1,399 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DeletionPurger;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.BiLongAccumulator;
+import org.apache.cassandra.utils.LongAccumulator;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.SearchIterator;
+import org.apache.cassandra.utils.memory.Cloner;
+
+/**
+ * A Row wrapper that has a source object that gets added to cell as part of the getCell call. This can only be used
+ * validly when all the cells share a common source object.
+ */
+public class RowWithSourceTable implements Row
+{
+    private static final long EMPTY_SIZE = ObjectSizes.measure(new RowWithSourceTable(null, null));
+
+    private final Row row;
+    private final Object source;
+
+    public RowWithSourceTable(Row row, Object source)
+    {
+        this.row = row;
+        this.source = source;
+    }
+
+    @Override
+    public Kind kind()
+    {
+        return row.kind();
+    }
+
+    @Override
+    public Clustering<?> clustering()
+    {
+        return row.clustering();
+    }
+
+    @Override
+    public void digest(Digest digest)
+    {
+        row.digest(digest);
+    }
+
+    @Override
+    public void validateData(TableMetadata metadata)
+    {
+        row.validateData(metadata);
+    }
+
+    @Override
+    public boolean hasInvalidDeletions()
+    {
+        return row.hasInvalidDeletions();
+    }
+
+    @Override
+    public Collection<ColumnMetadata> columns()
+    {
+        return row.columns();
+    }
+
+    @Override
+    public int columnCount()
+    {
+        return row.columnCount();
+    }
+
+    @Override
+    public Deletion deletion()
+    {
+        return row.deletion();
+    }
+
+    @Override
+    public LivenessInfo primaryKeyLivenessInfo()
+    {
+        return row.primaryKeyLivenessInfo();
+    }
+
+    @Override
+    public boolean isStatic()
+    {
+        return row.isStatic();
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return row.isEmpty();
+    }
+
+    @Override
+    public String toString(TableMetadata metadata)
+    {
+        return row.toString(metadata);
+    }
+
+    @Override
+    public boolean hasLiveData(long nowInSec, boolean enforceStrictLiveness)
+    {
+        return row.hasLiveData(nowInSec, enforceStrictLiveness);
+    }
+
+    @Override
+    public Cell<?> getCell(ColumnMetadata c)
+    {
+        var cell = row.getCell(c);
+        if (cell == null)
+            return null;
+        return new CellWithSourceTable<>(cell, source);
+    }
+
+    @Override
+    public Cell<?> getCell(ColumnMetadata c, CellPath path)
+    {
+        return wrapCell(row.getCell(c, path));
+    }
+
+    @Override
+    public ComplexColumnData getComplexColumnData(ColumnMetadata c)
+    {
+        return (ComplexColumnData) wrapColumnData(row.getComplexColumnData(c));
+    }
+
+    @Override
+    public ColumnData getColumnData(ColumnMetadata c)
+    {
+        return wrapColumnData(row.getColumnData(c));
+    }
+
+    @Override
+    public Iterable<Cell<?>> cells()
+    {
+        return Iterables.transform(row.cells(), this::wrapCell);
+    }
+
+    @Override
+    public Collection<ColumnData> columnData()
+    {
+        return Collections2.transform(row.columnData(), this::wrapColumnData);
+    }
+
+    @Override
+    public Iterable<Cell<?>> cellsInLegacyOrder(TableMetadata metadata, boolean reversed)
+    {
+        return Iterables.transform(row.cellsInLegacyOrder(metadata, reversed), this::wrapCell);
+    }
+
+    @Override
+    public boolean hasComplexDeletion()
+    {
+        return row.hasComplexDeletion();
+    }
+
+    @Override
+    public boolean hasComplex()
+    {
+        return row.hasComplex();
+    }
+
+    @Override
+    public boolean hasDeletion(long nowInSec)
+    {
+        return row.hasDeletion(nowInSec);
+    }
+
+    @Override
+    public SearchIterator<ColumnMetadata, ColumnData> searchIterator()
+    {
+        var iterator = row.searchIterator();
+        return key -> wrapColumnData(iterator.next(key));
+    }
+
+    @Override
+    public Row filter(ColumnFilter filter, TableMetadata metadata)
+    {
+        return maybeWrapRow(row.filter(filter, metadata));
+    }
+
+    @Override
+    public Row filter(ColumnFilter filter, DeletionTime activeDeletion, boolean setActiveDeletionToRow, TableMetadata metadata)
+    {
+        return maybeWrapRow(row.filter(filter, activeDeletion, setActiveDeletionToRow, metadata));
+    }
+
+    @Override
+    public Row transformAndFilter(LivenessInfo info, Deletion deletion, Function<ColumnData, ColumnData> function)
+    {
+        return maybeWrapRow(row.transformAndFilter(info, deletion, function));
+    }
+
+    @Override
+    public Row transformAndFilter(Function<ColumnData, ColumnData> function)
+    {
+        return maybeWrapRow(row.transformAndFilter(function));
+    }
+
+    @Override
+    public Row clone(Cloner cloner)
+    {
+        return maybeWrapRow(row.clone(cloner));
+    }
+
+    @Override
+    public Row purge(DeletionPurger purger, long nowInSec, boolean enforceStrictLiveness)
+    {
+        return maybeWrapRow(row.purge(purger, nowInSec, enforceStrictLiveness));
+    }
+
+    @Override
+    public Row withOnlyQueriedData(ColumnFilter filter)
+    {
+        return maybeWrapRow(row.withOnlyQueriedData(filter));
+    }
+
+    @Override
+    public Row purgeDataOlderThan(long timestamp, boolean enforceStrictLiveness)
+    {
+        return maybeWrapRow(row.purgeDataOlderThan(timestamp, enforceStrictLiveness));
+    }
+
+    @Override
+    public Row markCounterLocalToBeCleared()
+    {
+        return maybeWrapRow(row.markCounterLocalToBeCleared());
+    }
+
+    @Override
+    public Row updateAllTimestamp(long newTimestamp)
+    {
+        return maybeWrapRow(row.updateAllTimestamp(newTimestamp));
+    }
+
+    @Override
+    public Row withRowDeletion(DeletionTime deletion)
+    {
+        return maybeWrapRow(row.withRowDeletion(deletion));
+    }
+
+    @Override
+    public int dataSize()
+    {
+        return row.dataSize();
+    }
+
+    @Override
+    public long unsharedHeapSizeExcludingData()
+    {
+        return row.unsharedHeapSizeExcludingData() + EMPTY_SIZE;
+    }
+
+    @Override
+    public String toString(TableMetadata metadata, boolean fullDetails)
+    {
+        return row.toString(metadata, fullDetails);
+    }
+
+    @Override
+    public long unsharedHeapSize()
+    {
+        return row.unsharedHeapSize();
+    }
+
+    @Override
+    public String toString(TableMetadata metadata, boolean includeClusterKeys, boolean fullDetails)
+    {
+        return row.toString(metadata, includeClusterKeys, fullDetails);
+    }
+
+    @Override
+    public long minTimestamp()
+    {
+        return row.minTimestamp();
+    }
+
+    @Override
+    public long maxTimestamp()
+    {
+        return row.maxTimestamp();
+    }
+
+    @Override
+    public void apply(Consumer<ColumnData> function)
+    {
+        row.apply(function);
+    }
+
+    @Override
+    public <A> void apply(BiConsumer<A, ColumnData> function, A arg)
+    {
+        row.apply(function, arg);
+    }
+
+    @Override
+    public long accumulate(LongAccumulator<ColumnData> accumulator, long initialValue)
+    {
+        return row.accumulate(accumulator, initialValue);
+    }
+
+    @Override
+    public long accumulate(LongAccumulator<ColumnData> accumulator, Comparator<ColumnData> comparator, ColumnData from, long initialValue)
+    {
+        return row.accumulate(accumulator, comparator, from, initialValue);
+    }
+
+    @Override
+    public <A> long accumulate(BiLongAccumulator<A, ColumnData> accumulator, A arg, long initialValue)
+    {
+        return row.accumulate(accumulator, arg, initialValue);
+    }
+
+    @Override
+    public <A> long accumulate(BiLongAccumulator<A, ColumnData> accumulator, A arg, Comparator<ColumnData> comparator, ColumnData from, long initialValue)
+    {
+        return row.accumulate(accumulator, arg, comparator, from, initialValue);
+    }
+
+    @Override
+    public Iterator<ColumnData> iterator()
+    {
+        return Iterators.transform(row.iterator(), this::wrapColumnData);
+    }
+
+    private ColumnData wrapColumnData(ColumnData c)
+    {
+        if (c == null)
+            return null;
+        if (c instanceof Cell<?>)
+            return new CellWithSourceTable<>((Cell<?>) c, source);
+        if (c instanceof ComplexColumnData)
+            return ((ComplexColumnData) c).transform(c1 -> new CellWithSourceTable<>(c1, source));
+        throw new IllegalStateException("Unexpected ColumnData type: " + c.getClass().getName());
+    }
+
+    private Cell<?> wrapCell(Cell<?> c)
+    {
+        return c != null ? new CellWithSourceTable<>(c, source) : null;
+    }
+
+    private Row maybeWrapRow(Row r)
+    {
+        if (r == null)
+            return null;
+        if (r == this.row)
+            return this;
+        return new RowWithSourceTable(r, source);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "RowWithSourceTable{" +
+               row +
+               ", source=" + source +
+               '}';
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/SAICodecUtils.java b/src/java/org/apache/cassandra/index/sai/utils/SAICodecUtils.java
new file mode 100644
index 000000000000..cc944cc54735
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/SAICodecUtils.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import io.github.jbellis.jvector.disk.RandomAccessWriter;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.io.compress.CorruptBlockException;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.OutputStreamDataOutput;
+
+import static org.apache.lucene.codecs.CodecUtil.CODEC_MAGIC;
+import static org.apache.lucene.codecs.CodecUtil.FOOTER_MAGIC;
+import static org.apache.lucene.codecs.CodecUtil.footerLength;
+import static org.apache.lucene.codecs.CodecUtil.readBEInt;
+import static org.apache.lucene.codecs.CodecUtil.readBELong;
+import static org.apache.lucene.codecs.CodecUtil.writeBEInt;
+import static org.apache.lucene.codecs.CodecUtil.writeBELong;
+
+public class SAICodecUtils
+{
+    public static final String FOOTER_POINTER = "footerPointer";
+
+    public static DataOutput toLuceneOutput(java.io.DataOutput out) {
+        var os = new OutputStream()
+        {
+            @Override
+            public void write(int b) throws IOException
+            {
+                out.write(b);
+            }
+        };
+        return new OutputStreamDataOutput(os);
+    }
+
+    public static void writeHeader(DataOutput out) throws IOException
+    {
+        writeBEInt(out, CODEC_MAGIC);
+        out.writeString(Version.latest().toString());
+    }
+
+    public static int headerSize() {
+        // Lucene's string-writing code is complex, but this is what it works out to
+        // until version length exceeds 127 characters or we add non-ascii characters
+        return 7;
+    }
+
+    public static void writeFooter(IndexOutput out) throws IOException
+    {
+        writeBEInt(out, FOOTER_MAGIC);
+        writeBEInt(out, 0);
+        writeChecksum(out);
+    }
+
+    public static void writeFooter(RandomAccessWriter braw, long checksum) throws IOException
+    {
+        var out = toLuceneOutput(braw);
+        writeBEInt(out, FOOTER_MAGIC);
+        writeBEInt(out, 0);
+        writeBELong(out, checksum);
+    }
+
+    public static Version checkHeader(DataInput in) throws IOException
+    {
+        return checkHeader(in, Version.EARLIEST);
+    }
+
+    public static Version checkHeader(DataInput in, Version earliest) throws IOException
+    {
+        try
+        {
+            final int actualMagic = readBEInt(in);
+            if (actualMagic != CODEC_MAGIC)
+            {
+                throw new CorruptIndexException("codec header mismatch: actual header=" + actualMagic + " vs expected header=" + CODEC_MAGIC, in);
+            }
+            final Version actualVersion = Version.parse(in.readString());
+            if (!actualVersion.onOrAfter(earliest))
+            {
+                throw new IOException("Unsupported version: " + actualVersion);
+            }
+            return actualVersion;
+        }
+        catch (Throwable th)
+        {
+            if (th.getCause() instanceof CorruptBlockException)
+            {
+                throw new CorruptIndexException("corrupted", in, th.getCause());
+            }
+            else
+            {
+                throw th;
+            }
+        }
+    }
+
+    public static long checkFooter(ChecksumIndexInput in) throws IOException
+    {
+        validateFooter(in, false);
+        long actualChecksum = in.getChecksum();
+        long expectedChecksum = readChecksum(in);
+        if (expectedChecksum != actualChecksum)
+        {
+            throw new CorruptIndexException("checksum failed (hardware problem?) : expected=" + Long.toHexString(expectedChecksum) +
+                                            " actual=" + Long.toHexString(actualChecksum), in);
+        }
+        return actualChecksum;
+    }
+
+    public static void validate(IndexInput input) throws IOException
+    {
+        validate(input, Version.EARLIEST);
+    }
+
+    public static void validate(IndexInput input, Version earliest) throws IOException
+    {
+        checkHeader(input, earliest);
+        validateFooterAndResetPosition(input);
+    }
+
+    public static void validate(IndexInput input, long footerPointer) throws IOException
+    {
+        checkHeader(input);
+
+        long current = input.getFilePointer();
+        input.seek(footerPointer);
+        validateFooter(input, true);
+
+        input.seek(current);
+    }
+
+    public static void validateFooterAndResetPosition(IndexInput in) throws IOException
+    {
+        long position = in.getFilePointer();
+        long fileLength = in.length();
+        long footerLength = footerLength();
+        long footerPosition = fileLength - footerLength;
+
+        if (footerPosition < 0)
+        {
+            throw new CorruptIndexException("invalid codec footer (file truncated?): file length=" + fileLength + ", footer length=" + footerLength, in);
+        }
+
+        in.seek(footerPosition);
+        validateFooter(in, false);
+        in.seek(position);
+    }
+
+    /**
+     * See {@link org.apache.lucene.codecs.CodecUtil#checksumEntireFile(org.apache.lucene.store.IndexInput)}.
+     *
+     * @param input   IndexInput to validate.
+     * @param version Index version
+     * @throws IOException if a corruption is detected.
+     */
+    public static void validateChecksum(IndexInput input, Version version) throws IOException
+    {
+        IndexInput clone = input.clone();
+        clone.seek(0L);
+        ChecksumIndexInput in = IndexFileUtils.getBufferedChecksumIndexInput(clone, version);
+
+        assert in.getFilePointer() == 0L : in.getFilePointer() + " bytes already read from this input!";
+
+        if (in.length() < (long) footerLength())
+            throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), input);
+        else
+        {
+            in.seek(in.length() - (long) footerLength());
+            checkFooter(in);
+        }
+    }
+
+    /**
+     * Copied from org.apache.lucene.codecs.CodecUtil.validateFooter(IndexInput)
+     */
+    public static void validateFooter(IndexInput in, boolean padded) throws IOException
+    {
+        long remaining = in.length() - in.getFilePointer();
+        long expected = footerLength();
+
+        if (remaining >= 4)
+        {
+            final int magic = readBEInt(in);
+
+            if (magic != FOOTER_MAGIC)
+            {
+                String additionalDetails = "";
+                if (remaining != expected)
+                    additionalDetails = " (and invalid number of bytes: remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer() + ')';
+                throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC + additionalDetails, in);
+            }
+        }
+
+        if (!padded)
+        {
+            if (remaining < expected)
+            {
+                throw new CorruptIndexException("misplaced codec footer (file truncated?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
+            }
+            else if (remaining > expected)
+            {
+                throw new CorruptIndexException("misplaced codec footer (file extended?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
+            }
+        }
+
+
+        final int algorithmID = readBEInt(in);
+
+        if (algorithmID != 0)
+        {
+            throw new CorruptIndexException("codec footer mismatch: unknown algorithmID: " + algorithmID, in);
+        }
+    }
+
+
+    // Copied from Lucene CodecUtil as they are not public
+
+    /**
+     * Writes checksum value as a 64-bit long to the output.
+     * @throws IllegalStateException if CRC is formatted incorrectly (wrong bits set)
+     * @throws IOException if an i/o error occurs
+     */
+    static void writeChecksum(IndexOutput output) throws IOException {
+        long value = output.getChecksum();
+        if ((value & 0xFFFFFFFF00000000L) != 0) {
+            throw new IllegalStateException("Illegal checksum: " + value + " (resource=" + output + ")");
+        }
+        writeBELong(output, value);
+    }
+
+    /**
+     * Reads checksum value as a 64-bit long from the input.
+     * @throws CorruptIndexException if CRC is formatted incorrectly (wrong bits set)
+     * @throws IOException if an i/o error occurs
+     */
+    static long readChecksum(IndexInput input) throws IOException {
+        long value = readBELong(input);
+        if ((value & 0xFFFFFFFF00000000L) != 0) {
+            throw new CorruptIndexException("Illegal checksum: " + value, input);
+        }
+        return value;
+    }
+
+    // Copied from Lucene PackedInts as they are not public
+
+    public static int checkBlockSize(int blockSize, int minBlockSize, int maxBlockSize) {
+        if (blockSize >= minBlockSize && blockSize <= maxBlockSize) {
+            if ((blockSize & blockSize - 1) != 0) {
+                throw new IllegalArgumentException("blockSize must be a power of two, got " + blockSize);
+            } else {
+                return Integer.numberOfTrailingZeros(blockSize);
+            }
+        } else {
+            throw new IllegalArgumentException("blockSize must be >= " + minBlockSize + " and <= " + maxBlockSize + ", got " + blockSize);
+        }
+    }
+
+    public static int numBlocks(long size, int blockSize) {
+        int numBlocks = (int)(size / (long)blockSize) + (size % (long)blockSize == 0L ? 0 : 1);
+        if ((long)numBlocks * (long)blockSize < size) {
+            throw new IllegalArgumentException("size is too large for this block size");
+        } else {
+            return numBlocks;
+        }
+    }
+
+    // Copied from Lucene BlockPackedReaderIterator as they are not public
+
+    /**
+     * Same as DataInput.readVLong but supports negative values
+     */
+    public static long readVLong(DataInput in) throws IOException
+    {
+        byte b = in.readByte();
+        if (b >= 0) return b;
+        long i = b & 0x7FL;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 7;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 14;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 21;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 28;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 35;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 42;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 49;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0xFFL) << 56;
+        return i;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/SeekingRandomAccessInput.java b/src/java/org/apache/cassandra/index/sai/utils/SeekingRandomAccessInput.java
new file mode 100644
index 000000000000..988a6909ce7a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/SeekingRandomAccessInput.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+
+/**
+ * {@link IndexInput} adapter that exposes it as a {@link RandomAccessInput} type.
+ */
+public class SeekingRandomAccessInput implements RandomAccessInput
+{
+    private final IndexInput in;
+    private final ByteOrder order;
+
+    public SeekingRandomAccessInput(org.apache.cassandra.index.sai.disk.io.IndexInput in)
+    {
+        this.in = in;
+        this.order = in.order();
+    }
+
+    @VisibleForTesting
+    public SeekingRandomAccessInput(IndexInput in, ByteOrder order)
+    {
+        this.in = in;
+        this.order = order;
+    }
+
+    public ByteOrder order()
+    {
+        return order;
+    }
+
+    @Override
+    public byte readByte(long pos) throws IOException
+    {
+        in.seek(pos);
+        return in.readByte();
+    }
+
+    @Override
+    public short readShort(long pos) throws IOException
+    {
+        in.seek(pos);
+        return in.readShort();
+    }
+
+    @Override
+    public int readInt(long pos) throws IOException
+    {
+        in.seek(pos);
+        return in.readInt();
+    }
+
+    @Override
+    public long readLong(long pos) throws IOException
+    {
+        in.seek(pos);
+        return in.readLong();
+    }
+
+    @Override
+    public String toString()
+    {
+        return "SeekingRandomAccessInput(" + in + ")";
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/SegmentOrdering.java b/src/java/org/apache/cassandra/index/sai/utils/SegmentOrdering.java
new file mode 100644
index 000000000000..3fe522d679c3
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/SegmentOrdering.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.v1.IndexSearcher;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.CloseableIterator;
+
+/**
+ * A {@link SegmentOrdering} orders an index and produces a stream of {@link PrimaryKeyWithSortKey}s.
+ *
+ * The limit can be used to lazily order the {@link PrimaryKey}s. Due to the possiblity for
+ * shadowed or updated keys, a {@link SegmentOrdering} should be able to order the whole index
+ * until exhausted.
+ *
+ * When using {@link SegmentOrdering} there are several steps to
+ * build the list of Primary Keys to be ordered:
+ *
+ * 1. Find all primary keys that match each non-ordering query predicate.
+ * 2. Union and intersect the results of step 1 to build a single {@link KeyRangeIterator}
+ *    ordered by {@link PrimaryKey}.
+ * 3. Fan the primary keys from step 2 out to each sstable segment to order the list of primary keys.
+ *
+ * SegmentOrdering handles the third step.
+ *
+ * Note: a segment ordering is only used when a query has both ordering and non-ordering predicates.
+ * Where a query has only ordering predicates, the ordering is handled by the
+ * {@link IndexSearcher#orderBy(Orderer, org.apache.cassandra.index.sai.plan.Expression, AbstractBounds, QueryContext, int)}.
+ */
+public interface SegmentOrdering
+{
+    /**
+     * Order a list of primary keys to the top results. The limit is a hint indicating the minimum number of
+     * results the query requested. The keys passed to the method will already be limited to keys in the segment's
+     * Primary Key range.
+     */
+    CloseableIterator<PrimaryKeyWithSortKey> orderResultsBy(SSTableReader reader, QueryContext context, List<PrimaryKey> keys, Orderer orderer, int limit) throws IOException;
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/SegmentRowIdOrdinalPairs.java b/src/java/org/apache/cassandra/index/sai/utils/SegmentRowIdOrdinalPairs.java
new file mode 100644
index 000000000000..0f87fc279185
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/SegmentRowIdOrdinalPairs.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.function.IntConsumer;
+
+import org.agrona.collections.IntIntConsumer;
+
+/**
+ * A specialized data structure that stores segment row id to ordinal pairs efficiently. Implemented as an array of int
+ * pairs that avoids boxing.
+ */
+public class SegmentRowIdOrdinalPairs
+{
+    private final int capacity;
+    private int size;
+    private final int[] array;
+
+    /**
+     * Create a new IntIntPairArray with the given capacity.
+     * @param capacity the capacity
+     */
+    public SegmentRowIdOrdinalPairs(int capacity)
+    {
+        assert capacity < Integer.MAX_VALUE / 2 : "capacity is too large " + capacity;
+        this.capacity = capacity;
+        this.size = 0;
+        this.array = new int[capacity * 2];
+    }
+
+    /**
+     * Add a pair to the array.
+     * @param segmentRowId the first value
+     * @param ordinal the second value
+     */
+    public void add(int segmentRowId, int ordinal)
+    {
+        if (size == capacity)
+            throw new ArrayIndexOutOfBoundsException(size);
+        array[size * 2] = segmentRowId;
+        array[size * 2 + 1] = ordinal;
+        size++;
+    }
+
+    /**
+     * Get the row id at the given index.
+     * @param index the index
+     * @return the row id
+     */
+    public int getSegmentRowId(int index)
+    {
+        if ( index < 0 || index >= size)
+            throw new ArrayIndexOutOfBoundsException(index);
+        return array[index * 2];
+    }
+
+    /**
+     * Get the ordinal at the given index.
+     * @param index the index
+     * @return the ordinal
+     */
+    public int getOrdinal(int index)
+    {
+        if ( index < 0 || index >= size)
+            throw new ArrayIndexOutOfBoundsException(index);
+        return array[index * 2 + 1];
+    }
+
+    /**
+     * The number of pairs in the array.
+     * @return the number of pairs in the array
+     */
+    public int size()
+    {
+        return size;
+    }
+
+    /**
+     * Iterate over the pairs in the array, calling the consumer for each pair passing (index, x, y).
+     * @param consumer the consumer to call for each pair
+     */
+    public void forEachSegmentRowIdOrdinalPair(IntIntConsumer consumer)
+    {
+        for (int i = 0; i < size; i++)
+            consumer.accept(array[i * 2], array[i * 2 + 1]);
+    }
+
+    /**
+     * Iterate over the pairs in the array, calling the consumer for each pair passing (index, x, y).
+     * @param consumer the consumer to call for each pair
+     */
+    public void forEachIndexOrdinalPair(IntIntConsumer consumer)
+    {
+        for (int i = 0; i < size; i++)
+            consumer.accept(i, array[i * 2 + 1]);
+    }
+
+
+    /**
+     * Calls the consumer for each right value in each pair of the array.
+     * @param consumer the consumer to call for each right value
+     */
+    public void forEachOrdinal(IntConsumer consumer)
+    {
+        for (int i = 0; i < size; i++)
+            consumer.accept(array[i * 2 + 1]);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/SingletonIntIterator.java b/src/java/org/apache/cassandra/index/sai/utils/SingletonIntIterator.java
new file mode 100644
index 000000000000..45418949dd3c
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/SingletonIntIterator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.NoSuchElementException;
+import java.util.PrimitiveIterator;
+
+/**
+ * Singleton int iterator used to prevent unnecessary object creation
+ */
+public class SingletonIntIterator implements PrimitiveIterator.OfInt
+{
+    private final int value;
+    private boolean hasNext = true;
+
+    public SingletonIntIterator(int value)
+    {
+        this.value = value;
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        return hasNext;
+    }
+
+    @Override
+    public int nextInt()
+    {
+        if (!hasNext)
+            throw new NoSuchElementException();
+        hasNext = false;
+        return value;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/SoftLimitUtil.java b/src/java/org/apache/cassandra/index/sai/utils/SoftLimitUtil.java
new file mode 100644
index 000000000000..081ef8365fb8
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/SoftLimitUtil.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import org.apache.commons.math3.distribution.PascalDistribution;
+
+public class SoftLimitUtil
+{
+    /**
+     * Computes the number of items (e.g. keys, rows) that should be requested from a lower-layer of the system
+     * (e.g. storage) so that we obtain at least targetLimit number of items with given probability.
+     * It assumes that each item may randomly fail, in which case it is not delivered, thus the number of items
+     * delivered may be smaller than the number of items requested. Items are assumed to fail independently.
+     * <p>
+     * For example, if we want to deliver 100 rows to the user, but we know 20% of rows are tombstoned and would
+     * be rejected, then we should request `softLimit(100, 0.95, 0.8)` rows from the storage, and that would deliver
+     * in 95% of cases a sufficient number of rows, without having to query again for more.
+     *
+     * @param targetLimit the number of items that should be delivered to the user or upper layer in the system
+     * @param confidenceLevel the desired probability we obtain enough items in range, given in range [0.0, 1.0),
+     *                        typically you want to set it close to 1.0.
+     * @param perItemSuccessRate the probability of obtaining an item, given in range [0.0, 1.0]
+     * @return the number of items that should be requested from the lower layer of the system; >= targetLimit;
+     *    if the true result is greater than Integer.MAX_VALUE it is clamped to Integer.MAX_VALUE
+     */
+    public static int softLimit(int targetLimit, double confidenceLevel, double perItemSuccessRate)
+    {
+        if (Double.isNaN(confidenceLevel))
+            throw new IllegalArgumentException("confidenceLevel must not be NaN");
+        if (confidenceLevel < 0.0 || confidenceLevel >= 1.0)
+            throw new IllegalArgumentException("confidenceLevel out of range [0.0, 1.0): " + confidenceLevel);
+        if (Double.isNaN(perItemSuccessRate))
+            throw new IllegalArgumentException("perItemSuccessRate must not be NaN");
+        if (perItemSuccessRate < 0.0 || perItemSuccessRate > 1.0)
+            throw new IllegalArgumentException("perItemSuccessRate out of range [0.0, 1.0]: " + perItemSuccessRate);
+        if (targetLimit < 0)
+            throw new IllegalArgumentException("targetLimit must not be < 0: " + targetLimit);
+
+        // PascalDistribution (see further) cannot handle this case properly
+        if (targetLimit == 0)
+            return 0;
+
+        // Consider we perform attempts until we get R successes (=targetLimit), where the probability of success is
+        // P (=perItemSuccessRate). In this case the number of failures is described by a negative binomial
+        // distribution NB(R, P). We use PascalDistribution, which is an optimized special case
+        // of NB for dealing with integers.
+        final var failureDistrib = new PascalDistribution(targetLimit, perItemSuccessRate);
+        long maxExpectedFailures = failureDistrib.inverseCumulativeProbability(confidenceLevel);
+
+        long softLimit = (long) targetLimit + maxExpectedFailures;
+        return (int) Math.min(softLimit, Integer.MAX_VALUE);
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/TermsIterator.java b/src/java/org/apache/cassandra/index/sai/utils/TermsIterator.java
deleted file mode 100644
index 25649c71b518..000000000000
--- a/src/java/org/apache/cassandra/index/sai/utils/TermsIterator.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.utils;
-
-import java.io.Closeable;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import javax.annotation.concurrent.NotThreadSafe;
-
-/**
- * An iterator over the contents of an index that extends {@link Iterator}&lt;{@link IndexEntry}&gt; that provides the min and max
- * terms in the index. Each {@link IndexEntry} contains a term and the postings associated with that term.
- */
-@NotThreadSafe
-public interface TermsIterator extends Iterator<IndexEntry>, Closeable
-{
-    ByteBuffer getMinTerm();
-
-    ByteBuffer getMaxTerm();
-}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/TreeFormatter.java b/src/java/org/apache/cassandra/index/sai/utils/TreeFormatter.java
new file mode 100644
index 000000000000..de5940cd3129
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/TreeFormatter.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.function.Function;
+
+/**
+ * Pretty prints heterogenous tree structures like this:
+ * <pre>
+ * root
+ *   ├─ child 1
+ *   │   ├─ child 1a
+ *   │   └─ child 1b
+ *   └─ child 2
+ *       ├─ child 2a
+ *       └─ child 2b
+ * </pre>
+ * @param <T> type of the node of the tree
+ */
+public class TreeFormatter<T>
+{
+    private final Function<T, Iterable<? extends T>> children;
+    private final Function<T, String> toString;
+
+    /**
+     * Constructs a formatter that knows how to format trees of given type.
+     *
+     * @param toString a function that returns the text describing each tree node
+     * @param children a function that returns a list of children nodes
+     */
+    public TreeFormatter(Function<T, String> toString, Function<T, Iterable<? extends T>> children)
+    {
+        this.children = children;
+        this.toString = toString;
+    }
+
+    /**
+     * Returns a multiline String with a formatted tree
+     * @param root root node of the tree
+     */
+    public String format(T root)
+    {
+        StringBuilder sb = new StringBuilder();
+        append(root, sb, new StringBuilder(), true, false);
+        return sb.toString();
+    }
+
+    /**
+     * Traverses the tree depth first and prints the tree.
+     * Called once per each node.
+     */
+    private void append(T node, StringBuilder sb, StringBuilder padding, boolean isRoot, boolean hasRightSibling)
+    {
+        int origPaddingLength = padding.length();
+        if (!isRoot)
+        {
+            sb.append(padding);
+            sb.append(hasRightSibling ? " ├─ " : " └─ ");
+            padding.append(hasRightSibling ? " │  " : "    ");
+        }
+
+        String[] nodeStr = toString.apply(node).split("\n");
+        sb.append(nodeStr[0]);
+        sb.append('\n');
+        for (int i = 1; i < nodeStr.length; i++)
+        {
+            sb.append(padding);
+            sb.append(nodeStr[i]);
+            sb.append('\n');
+        }
+
+        var iter = children.apply(node).iterator();
+        while (iter.hasNext())
+        {
+            T child = iter.next();
+            append(child, sb, padding, false, iter.hasNext());
+        }
+        padding.setLength(origPaddingLength);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java
new file mode 100644
index 000000000000..d92032c5f5b3
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java
@@ -0,0 +1,670 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FastByteOperations;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+public class TypeUtil
+{
+    private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 };
+
+    /**
+     * DecimalType / BigDecimal values are indexed by truncating their asComparableBytes representation to this size,
+     * padding on the right with zero-value-bytes until this size is reached (if necessary).  This causes
+     * false-positives that must be filtered in a separate step after hitting the index and reading the associated
+     * (full) values.
+     */
+    public static final int DECIMAL_APPROXIMATION_BYTES = 24;
+
+    public static final int BIG_INTEGER_APPROXIMATION_BYTES = 20;
+
+    public static final int INET_ADDRESS_SIZE = 16;
+
+    public static final int DEFAULT_FIXED_LENGTH = 16;
+    /**
+     * Byte comparable version currently used for all SAI files and structures, with the exception of terms data in
+     * the early AA on-disk format.
+     */
+    public static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS41;
+
+    private TypeUtil() {}
+
+    /**
+     * Returns <code>true</code> if given buffer would pass the {@link AbstractType#validate(ByteBuffer)}
+     * check. False otherwise.
+     */
+    public static boolean isValid(ByteBuffer term, AbstractType<?> validator)
+    {
+        try
+        {
+            validator.validate(term);
+            return true;
+        }
+        catch (MarshalException e)
+        {
+            return false;
+        }
+    }
+
+    /**
+     * Indicates if the type encoding supports rounding of the raw value.
+     *
+     * This is significant in range searches where we have to make all range
+     * queries inclusive when searching the indexes in order to avoid excluding
+     * rounded values. Excluded values are removed by post-filtering.
+     */
+    public static boolean supportsRounding(AbstractType<?> type)
+    {
+        return isBigInteger(type) || isBigDecimal(type);
+    }
+
+    /**
+     * Returns the smaller of two {@code ByteBuffer} values, based on the result of {@link
+     * #compare(ByteBuffer, ByteBuffer, AbstractType, Version)} comparision.
+     */
+    public static ByteBuffer min(ByteBuffer a, ByteBuffer b, AbstractType<?> type, Version version)
+    {
+        return a == null ?  b : (b == null || compare(b, a, type, version) > 0) ? a : b;
+    }
+
+    /**
+     * Returns the greater of two {@code ByteBuffer} values, based on the result of {@link
+     * #compare(ByteBuffer, ByteBuffer, AbstractType, Version)} comparision.
+     */
+    public static ByteBuffer max(ByteBuffer a, ByteBuffer b, AbstractType<?> type, Version version)
+    {
+        return a == null ?  b : (b == null || compare(b, a, type, version) < 0) ? a : b;
+    }
+
+    /**
+     * Returns the value length for the given {@link AbstractType}, selecting 16 for types
+     * that officially use VARIABLE_LENGTH but are, in fact, of a fixed length.
+     */
+    public static int fixedSizeOf(AbstractType<?> type)
+    {
+        if (type.isValueLengthFixed())
+            return type.valueLengthIfFixed();
+        else if (isInetAddress(type))
+            return INET_ADDRESS_SIZE;
+        else if (isBigInteger(type))
+            return BIG_INTEGER_APPROXIMATION_BYTES;
+        else if (isBigDecimal(type))
+            return DECIMAL_APPROXIMATION_BYTES;
+        return DEFAULT_FIXED_LENGTH;
+    }
+
+    public static AbstractType<?> cellValueType(ColumnMetadata columnMetadata, IndexTarget.Type indexType)
+    {
+        AbstractType<?> type = columnMetadata.type;
+        if (isNonFrozenCollection(type))
+        {
+            CollectionType<?> collection = ((CollectionType<?>) type);
+            switch (collection.kind)
+            {
+                case LIST:
+                    return collection.valueComparator();
+                case SET:
+                    return collection.nameComparator();
+                case MAP:
+                    switch (indexType)
+                    {
+                        case KEYS:
+                            return collection.nameComparator();
+                        case VALUES:
+                            return collection.valueComparator();
+                        case KEYS_AND_VALUES:
+                            return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator());
+                    }
+            }
+        }
+        return type;
+    }
+
+    /**
+     * Allows overriding the default getString method for {@link CompositeType}. It is
+     * a requirement of the {@link ConcurrentRadixTree} that the keys are strings but
+     * the getString method of {@link CompositeType} does not return a string that compares
+     * in the same order as the underlying {@link ByteBuffer}. To get round this we convert
+     * the {@link CompositeType} bytes to a hex string.
+     */
+    public static String getString(ByteBuffer value, AbstractType<?> type)
+    {
+        if (isComposite(type))
+            return ByteBufferUtil.bytesToHex(value);
+        return type.getString(value);
+    }
+
+    /**
+     * The inverse of the above method. Overrides the fromString method on {@link CompositeType}
+     * in order to convert the hex string to bytes.
+     */
+    public static ByteBuffer fromString(String value, AbstractType<?> type)
+    {
+        if (isComposite(type))
+            return ByteBufferUtil.hexToBytes(value);
+        return type.fromString(value);
+    }
+
+    public static ByteBuffer fromComparableBytes(ByteComparable value, AbstractType<?> type, ByteComparable.Version version)
+    {
+        if (type instanceof InetAddressType || type instanceof IntegerType || type instanceof DecimalType)
+            return ByteBuffer.wrap(ByteSourceInverse.readBytes(value.asComparableBytes(version)));
+
+        return type.fromComparableBytes(ByteSource.peekable(value.asComparableBytes(version)), version);
+    }
+
+    public static ByteComparable asComparableBytes(ByteBuffer value, AbstractType<?> type)
+    {
+        return version -> asComparableBytes(value, type, version);
+    }
+
+    public static ByteSource asComparableBytes(ByteBuffer value, AbstractType<?> type, ByteComparable.Version version)
+    {
+        if (type instanceof InetAddressType || type instanceof IntegerType || type instanceof DecimalType)
+            return ByteSource.optionalFixedLength(ByteBufferAccessor.instance, value);
+        // The LongType.asComparableBytes uses variableLengthInteger which doesn't play well with
+        // the balanced tree because it is expecting fixed length data. So for SAI we use a optionalSignedFixedLengthNumber
+        // to keep all comparable values the same length
+        else if (type instanceof LongType)
+            return ByteSource.optionalSignedFixedLengthNumber(ByteBufferAccessor.instance, value);
+        return type.asComparableBytes(value, version);
+    }
+
+    /**
+     * Convenience method to create a {@link ByteComparable} from a {@link ByteBuffer} value for a given {@link CompositeType}
+     * with a terminator. This method is in this class to keep references to the {@link ByteBufferAccessor#instance} here.
+     */
+    public static ByteComparable asComparableBytes(ByteBuffer value, int terminator, CompositeType type)
+    {
+        return v -> type.asComparableBytes(ByteBufferAccessor.instance, value, v, terminator);
+    }
+
+
+    /**
+     * Fills a byte array with the comparable bytes for a type.
+     * <p>
+     * This method expects a {@code value} parameter generated by calling {@link #asIndexBytes(ByteBuffer, AbstractType)}.
+     * It is not generally safe to pass the output of other serialization methods to this method.  For instance, it is
+     * not generally safe to pass the output of {@link AbstractType#decompose(Object)} as the {@code value} parameter
+     * (there are certain types for which this is technically OK, but that doesn't hold for all types).
+     *
+     * @param value a value buffer returned by {@link #asIndexBytes(ByteBuffer, AbstractType)}
+     * @param type the type associated with the encoded {@code value} parameter
+     * @param bytes this method's output
+     */
+    public static void toComparableBytes(ByteBuffer value, AbstractType<?> type, byte[] bytes)
+    {
+        if (isInetAddress(type))
+            ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, INET_ADDRESS_SIZE);
+        else if (isBigInteger(type))
+            ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, BIG_INTEGER_APPROXIMATION_BYTES);
+        else if (isBigDecimal(type))
+            ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, DECIMAL_APPROXIMATION_BYTES);
+        else
+            ByteSourceInverse.readBytesMustFit(type.asComparableBytes(value, BYTE_COMPARABLE_VERSION), bytes);
+    }
+
+    /**
+     * Translates the external value of specific types into a format used by the index.
+     */
+    public static ByteBuffer asIndexBytes(ByteBuffer value, AbstractType<?> type)
+    {
+        if (value == null || value.remaining() == 0)
+            return value;
+
+        if (isInetAddress(type))
+            return encodeInetAddress(value);
+        else if (isBigInteger(type))
+            return encodeBigInteger(value);
+        else if (type instanceof DecimalType)
+            return encodeDecimal(value);
+        return value;
+    }
+
+    /**
+     * Tries its best to return the inverse of {@link #encode}.
+     * For most of the types it returns the exact inverse.
+     * For big integers and decimals, which could be truncated by encode, some precision loss is possible.
+     */
+    public static ByteBuffer decode(ByteBuffer value, AbstractType<?> type)
+    {
+        if (value == null)
+            return null;
+
+        if (isInetAddress(type))
+            return decodeInetAddress(value);
+        else if (isBigInteger(type))
+            return decodeBigInteger(value);
+        else if (type instanceof DecimalType)
+            return decodeDecimal(value);
+        return value;
+    }
+
+    /**
+     * Compare two terms based on their type. This is used in place of {@link AbstractType#compare(ByteBuffer, ByteBuffer)}
+     * so that the default comparison can be overridden for specific types.
+     *
+     * Note: This should be used for all term comparison
+     */
+    public static int compare(ByteBuffer b1, ByteBuffer b2, AbstractType<?> type, Version version)
+    {
+        if (isInetAddress(type))
+            return compareInet(b1, b2);
+        else if (useFastByteOperations(type, version))
+            return FastByteOperations.compareUnsigned(b1, b2);
+
+        return type.compare(b1, b2);
+    }
+
+    /**
+     * This is used for value comparison in post-filtering - {@link Expression#isSatisfiedBy(ByteBuffer)}.
+     *
+     * This allows types to decide whether they should be compared based on their encoded value or their
+     * raw value. At present only {@link InetAddressType} values are compared by their encoded values to
+     * allow for ipv4 -> ipv6 equivalency in searches.
+     */
+    public static int comparePostFilter(Expression.Value requestedValue, Expression.Value columnValue, AbstractType<?> type)
+    {
+        if (isInetAddress(type))
+            return compareInet(requestedValue.encoded, columnValue.encoded);
+        // Override comparisons for frozen collections
+        else if (isFrozen(type))
+            return FastByteOperations.compareUnsigned(requestedValue.raw, columnValue.raw);
+
+        return type.compare(requestedValue.raw, columnValue.raw);
+    }
+
+    public static Iterator<ByteBuffer> collectionIterator(AbstractType<?> validator,
+                                                          ComplexColumnData cellData,
+                                                          ColumnMetadata columnMetadata,
+                                                          IndexTarget.Type indexType,
+                                                          long nowInSecs)
+    {
+        if (cellData == null)
+            return null;
+
+        Stream<ByteBuffer> stream = StreamSupport.stream(cellData.spliterator(), false).filter(cell -> cell != null && cell.isLive(nowInSecs))
+                                                 .map(cell -> cellValue(columnMetadata, indexType, cell));
+
+        if (isInetAddress(validator))
+            stream = stream.sorted((c1, c2) -> compareInet(encodeInetAddress(c1), encodeInetAddress(c2)));
+
+        return stream.iterator();
+    }
+
+    public static Comparator<ByteBuffer> comparator(AbstractType<?> type, Version version)
+    {
+        // Override the comparator for BigInteger, frozen collections (not including composite types) and
+        // composite types before DB version to maintain a consistent order between the in-memory index and the on-disk index.
+        if (useFastByteOperations(type, version))
+            return FastByteOperations::compareUnsigned;
+
+        return type;
+    }
+
+    private static boolean useFastByteOperations(AbstractType<?> type, Version version)
+    {
+        // BigInteger types, BigDecimal types, frozen types and composite types (map entries) use compareUnsigned to
+        // maintain a consistent order between the in-memory index and the on-disk index. Starting with Version.DB,
+        // composite types are compared using their AbstractType.
+        return isBigInteger(type)
+               || isBigDecimal(type)
+               || (!isComposite(type) && isFrozen(type))
+               || (isComposite(type) && !version.onOrAfter(Version.DB));
+    }
+
+    public static float[] decomposeVector(AbstractType<?> type, ByteBuffer byteBuffer)
+    {
+        return ((VectorType<?>.VectorSerializer)type.getSerializer()).deserializeFloatArray(byteBuffer);
+    }
+
+    public static float[] decomposeVector(IndexContext indexContext, ByteBuffer byteBuffer)
+    {
+        return decomposeVector(indexContext.getValidator(), byteBuffer);
+    }
+
+    private static ByteBuffer cellValue(ColumnMetadata columnMetadata, IndexTarget.Type indexType, Cell<?> cell)
+    {
+        if (columnMetadata.type.isCollection() && columnMetadata.type.isMultiCell())
+        {
+            switch (((CollectionType<?>) columnMetadata.type).kind)
+            {
+                case LIST:
+                    //TODO Is there any optimisation can be done here with cell values?
+                    return cell.buffer();
+                case SET:
+                    return cell.path().get(0);
+                case MAP:
+                    switch (indexType)
+                    {
+                        case KEYS:
+                            return cell.path().get(0);
+                        case VALUES:
+                            return cell.buffer();
+                        case KEYS_AND_VALUES:
+                            return CompositeType.build(ByteBufferAccessor.instance, cell.path().get(0), cell.buffer());
+                    }
+            }
+        }
+        return cell.buffer();
+    }
+
+    /**
+     * Compares 2 InetAddress terms by ensuring that both addresses are represented as
+     * ipv6 addresses.
+     */
+    private static int compareInet(ByteBuffer b1, ByteBuffer b2)
+    {
+        assert isIPv6(b1) && isIPv6(b2);
+
+        return FastByteOperations.compareUnsigned(b1, b2);
+    }
+
+    private static boolean isIPv6(ByteBuffer address)
+    {
+        return address.remaining() == INET_ADDRESS_SIZE;
+    }
+
+    /**
+     * Encode a {@link InetAddress} into a fixed width 16 byte encoded value.
+     *
+     * The encoded value is byte comparable and prefix compressible.
+     *
+     * The encoding is done by converting ipv4 addresses to their ipv6 equivalent.
+     */
+    private static ByteBuffer encodeInetAddress(ByteBuffer value)
+    {
+        if (value.remaining() == 4)
+        {
+            int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
+            ByteBuffer mapped = ByteBuffer.allocate(INET_ADDRESS_SIZE);
+            System.arraycopy(IPV4_PREFIX, 0, mapped.array(), 0, IPV4_PREFIX.length);
+            ByteBufferUtil.copyBytes(value, position, mapped, IPV4_PREFIX.length, value.remaining());
+            return mapped;
+        }
+        return value;
+    }
+
+    private static ByteBuffer decodeInetAddress(ByteBuffer value)
+    {
+        throw new UnsupportedOperationException("Decoding InetAddress not implemented yet");
+    }
+
+
+    /**
+     * Encode a {@link BigInteger} into a fixed width 20 byte encoded value.
+     *
+     * The encoded value is byte comparable and prefix compressible.
+     *
+     * The format of the encoding is:
+     *
+     *  The first 4 bytes contain the integer length of the {@link BigInteger} byte array
+     *  with the top bit flipped for positive values.
+     *
+     *  The remaining 16 bytes contain the 16 most significant bytes of the
+     *  {@link BigInteger} byte array.
+     *
+     *  For {@link BigInteger} values whose underlying byte array is less than
+     *  16 bytes, the encoded value is sign extended.
+     */
+    public static ByteBuffer encodeBigInteger(ByteBuffer value)
+    {
+        int size = value.remaining();
+        int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
+        byte[] bytes = new byte[BIG_INTEGER_APPROXIMATION_BYTES];
+        if (size < BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES)
+        {
+            ByteBufferUtil.copyBytes(value, position, bytes, bytes.length - size, size);
+            if ((bytes[bytes.length - size] & 0x80) != 0)
+                Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0xff);
+            else
+                Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0x00);
+        }
+        else
+        {
+            ByteBufferUtil.copyBytes(value, position, bytes, Integer.BYTES, BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES);
+        }
+        if ((bytes[4] & 0x80) != 0)
+        {
+            size = -size;
+        }
+        bytes[0] = (byte)(size >> 24 & 0xff);
+        bytes[1] = (byte)(size >> 16 & 0xff);
+        bytes[2] = (byte)(size >> 8 & 0xff);
+        bytes[3] = (byte)(size & 0xff);
+        bytes[0] ^= 0x80;
+        return ByteBuffer.wrap(bytes);
+    }
+
+
+    public static ByteBuffer decodeBigInteger(ByteBuffer encoded)
+    {
+        byte[] bytes = new byte[20];
+        encoded.get(bytes);
+        encoded.rewind();
+
+        // Undo the XOR operation on the first byte
+        bytes[0] ^= 0x80;
+
+        // Extract the size (the first 4 bytes)
+        int size = ((bytes[0] & 0xff) << 24) | ((bytes[1] & 0xff) << 16) | ((bytes[2] & 0xff) << 8) | (bytes[3] & 0xff);
+
+        boolean isNegative = size < 0;
+        if (isNegative)
+            size = -size;
+
+        ByteBuffer result;
+        if (size < 16)
+        {
+            int offset = 20 - size;
+            result = ByteBuffer.wrap(Arrays.copyOfRange(bytes, offset, 20));
+        }
+        else
+        {
+            // Size >= 16 means we extract 16 bytes starting from index 4
+            var resultBytes = new byte[size];
+            System.arraycopy(bytes, 4, resultBytes, 0, 16);
+            result = ByteBuffer.wrap(resultBytes);
+        }
+
+        return result;
+    }
+
+
+    /* Type comparison to get rid of ReversedType */
+
+    /**
+     * Returns <code>true</code> if values of the given {@link AbstractType} should be indexed as literals.
+     */
+    public static boolean isLiteral(AbstractType<?> type)
+    {
+        return isUTF8OrAscii(type) || isCompositeOrFrozen(type) || baseType(type) instanceof BooleanType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is UTF8 or Ascii
+     */
+    public static boolean isUTF8OrAscii(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof UTF8Type || type instanceof AsciiType;
+    }
+
+//    /**
+//     * Returns <code>true</code> if given {@link AbstractType} is based on a string, e.g. UTF8 or Ascii
+//     */
+//    public static boolean isString(AbstractType<?> type)
+//    {
+//        type = baseType(type);
+//        return type instanceof StringType;
+//    }
+//
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is a Composite(map entry) or frozen.
+     */
+    public static boolean isCompositeOrFrozen(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof CompositeType || isFrozen(type);
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is frozen.
+     */
+    public static boolean isFrozen(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return !type.subTypes().isEmpty() && !type.isMultiCell();
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is a frozen collection.
+     */
+    public static boolean isFrozenCollection(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type.isCollection() && !type.isMultiCell();
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is a non-frozen collection.
+     */
+    public static boolean isNonFrozenCollection(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type.isCollection() && type.isMultiCell();
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is included in the types.
+     */
+    public static boolean isIn(AbstractType<?> type, Set<AbstractType<?>> types)
+    {
+        type = baseType(type);
+        return types.contains(type);
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link InetAddressType}
+     */
+    private static boolean isInetAddress(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof InetAddressType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link IntegerType}
+     */
+    private static boolean isBigInteger(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof IntegerType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link DecimalType}
+     */
+    private static boolean isBigDecimal(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof DecimalType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link CompositeType}
+     */
+    public static boolean isComposite(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof CompositeType;
+    }
+
+    /**
+     * @return {@code true} if the empty values of the given type should be excluded from indexing, {@code false} otherwise.
+     */
+    public static boolean skipsEmptyValue(AbstractType<?> type)
+    {
+        return !type.allowsEmpty() || !isLiteral(type);
+    }
+
+    /**
+     * @return base type if given type is reversed, otherwise return itself
+     */
+    private static AbstractType<?> baseType(AbstractType<?> type)
+    {
+        return type.unwrap();
+    }
+
+    public static ByteBuffer encodeDecimal(ByteBuffer value)
+    {
+        ByteSource bs = DecimalType.instance.asComparableBytes(value, BYTE_COMPARABLE_VERSION);
+        byte[] data = new byte[DECIMAL_APPROXIMATION_BYTES];    // initialized with 0s
+        bs.nextBytes(data); // reads up to the number of bytes in the array, leaving 0s in the remaining bytes
+        return ByteBuffer.wrap(data);
+    }
+
+    public static ByteBuffer decodeDecimal(ByteBuffer value)
+    {
+        var peekableValue = ByteSource.peekable(ByteSource.preencoded(value));
+        return DecimalType.instance.fromComparableBytes(peekableValue, BYTE_COMPARABLE_VERSION);
+    }
+
+    public static ByteComparable.Version byteComparableVersionForTermsData()
+    {
+        return Version.latest().byteComparableVersionFor(IndexComponentType.TERMS_DATA, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion());
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java b/src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java
index 5bbaccd69180..3546bf7cbae0 100644
--- a/src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java
+++ b/src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java
@@ -21,24 +21,28 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.index.sai.IndexValidation;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SSTableContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.SSTableIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.utils.Pair;
 
 /**
- * Maintain an atomic view for read requests, so that requests can read all data during concurrent compactions.
- * <p>
+ * Maintain a atomic view for read requests, so that requests can read all data during concurrent compactions.
+ *
  * All per-column {@link SSTableIndex} updates should be proxied by {@link StorageAttachedIndexGroup} to make
  * sure per-sstable {@link SSTableContext} are in-sync.
  */
@@ -46,16 +50,23 @@ public class IndexViewManager
 {
     private static final Logger logger = LoggerFactory.getLogger(IndexViewManager.class);
     
-    private final StorageAttachedIndex index;
+    private final IndexContext context;
     private final AtomicReference<View> view = new AtomicReference<>();
 
-    public IndexViewManager(StorageAttachedIndex index)
+
+    public IndexViewManager(IndexContext context)
+    {
+        this(context, Collections.emptySet(), Collections.emptySet());
+    }
+
+    @VisibleForTesting
+    IndexViewManager(IndexContext context, Collection<Descriptor> sstables, Collection<SSTableIndex> indices)
     {
-        this.index = index;
-        this.view.set(new View(index.termType(), Collections.emptySet()));
+        this.context = context;
+        this.view.set(new View(context, sstables, indices));
     }
 
-    public View view()
+    public View getView()
     {
         return view.get();
     }
@@ -64,19 +75,24 @@ public View view()
      * Replaces old SSTables with new by creating new immutable view.
      *
      * @param oldSSTables A set of SSTables to remove.
+     * @param newSSTables A set of SSTables added in Cassandra.
      * @param newSSTableContexts A set of SSTableContexts to add to tracker.
-     * @param validation Controls how indexes should be validated
+     * @param validate if true, per-column index files' header and footer will be validated.
      *
      * @return A set of SSTables which have attached to them invalid index components.
      */
-    public Collection<SSTableContext> update(Collection<SSTableReader> oldSSTables, Collection<SSTableContext> newSSTableContexts, IndexValidation validation)
+    public Set<SSTableContext> update(Collection<SSTableReader> oldSSTables,
+                                      Collection<SSTableReader> newSSTables,
+                                      Collection<SSTableContext> newSSTableContexts,
+                                      boolean validate)
     {
         // Valid indexes on the left and invalid SSTable contexts on the right...
-        Pair<Collection<SSTableIndex>, Collection<SSTableContext>> indexes = getBuiltIndexes(newSSTableContexts, validation);
+        Pair<Set<SSTableIndex>, Set<SSTableContext>> indexes = context.getBuiltIndexes(newSSTableContexts, validate);
 
         View currentView, newView;
-        Collection<SSTableIndex> newViewIndexes = new HashSet<>();
+        Map<Descriptor, SSTableIndex> newViewIndexes = new HashMap<>();
         Collection<SSTableIndex> releasableIndexes = new ArrayList<>();
+        Collection<SSTableReader> toRemove = new HashSet<>(oldSSTables);
 
         do
         {
@@ -84,125 +100,106 @@ public Collection<SSTableContext> update(Collection<SSTableReader> oldSSTables,
             newViewIndexes.clear();
             releasableIndexes.clear();
 
+            Set<Descriptor> sstables = new HashSet<>(currentView.getSSTables());
+            for (SSTableReader sstable : oldSSTables)
+                sstables.remove(sstable.descriptor);
+            for (SSTableReader sstable : newSSTables)
+                sstables.add(sstable.descriptor);
+
             for (SSTableIndex sstableIndex : currentView)
             {
                 // When aborting early open transaction, toRemove may have the same sstable files as newSSTableContexts,
                 // but different SSTableReader java objects with different start positions. So we need to release them
-                // from existing view.
+                // from existing view.  see DSP-19677
                 SSTableReader sstable = sstableIndex.getSSTable();
-                if (oldSSTables.contains(sstable) || newViewIndexes.contains(sstableIndex))
+                if (toRemove.contains(sstable))
                     releasableIndexes.add(sstableIndex);
                 else
-                    newViewIndexes.add(sstableIndex);
+                    addOrUpdateSSTableIndex(sstableIndex, newViewIndexes, releasableIndexes);
             }
 
             for (SSTableIndex sstableIndex : indexes.left)
             {
-                if (newViewIndexes.contains(sstableIndex))
-                    releasableIndexes.add(sstableIndex);
-                else
-                    newViewIndexes.add(sstableIndex);
+                addOrUpdateSSTableIndex(sstableIndex, newViewIndexes, releasableIndexes);
             }
 
-            newView = new View(index.termType(), newViewIndexes);
+            newView = new View(context, sstables, newViewIndexes.values());
         }
         while (!view.compareAndSet(currentView, newView));
 
         releasableIndexes.forEach(SSTableIndex::release);
 
         if (logger.isTraceEnabled())
-            logger.trace(index.identifier().logMessage("There are now {} active SSTable indexes."), view.get().getIndexes().size());
+            logger.trace(context.logMessage("There are now {} active SSTable indexes."), view.get().getIndexes().size());
 
         return indexes.right;
     }
 
-    public void drop(Collection<SSTableReader> sstablesToRebuild)
+    private static void addOrUpdateSSTableIndex(SSTableIndex ssTableIndex, Map<Descriptor, SSTableIndex> addTo, Collection<SSTableIndex> toRelease)
     {
-        View currentView = view.get();
-
-        Set<SSTableReader> toRemove = new HashSet<>(sstablesToRebuild);
-        for (SSTableIndex index : currentView)
+        var descriptor = ssTableIndex.getSSTable().descriptor;
+        SSTableIndex previous = addTo.get(descriptor);
+        if (previous != null)
         {
-            SSTableReader sstable = index.getSSTable();
-            if (!toRemove.contains(sstable))
-                continue;
+            // If the new index use the same files that the exiting one (and the previous one is still complete, meaning
+            // that the files weren't corrupted), then keep the old one (no point in changing for the same thing).
+            if (previous.usedPerIndexComponents().isComplete() && ssTableIndex.usedPerIndexComponents().buildId().equals(previous.usedPerIndexComponents().buildId()))
+            {
+                toRelease.add(ssTableIndex);
+                return;
+            }
 
-            index.markObsolete();
+            // Otherwise, release the old, and we'll replace by the new one below.
+            toRelease.add(previous);
         }
-
-        update(toRemove, Collections.emptyList(), IndexValidation.NONE);
+        addTo.put(descriptor, ssTableIndex);
     }
 
-    /**
-     * Called when index is dropped. Mark all {@link SSTableIndex} as released and per-column index files
-     * will be removed when in-flight queries are completed.
-     */
-    public void invalidate()
+    public void prepareSSTablesForRebuild(Collection<SSTableReader> sstablesToRebuild)
     {
-        View previousView = view.getAndSet(new View(index.termType(), Collections.emptyList()));
-
-        for (SSTableIndex index : previousView)
+        Set<SSTableReader> toRemove = new HashSet<>(sstablesToRebuild);
+        View oldView, newView;
+        Set<SSTableIndex> indexesToRemove;
+        do
         {
-            index.markObsolete();
+            oldView = view.get();
+            indexesToRemove = oldView.getIndexes()
+                                     .stream()
+                                     .filter(index -> toRemove.contains(index.getSSTable()))
+                                     .collect(Collectors.toSet());
+            var newIndexes = new HashSet<>(oldView.getIndexes());
+            newIndexes.removeAll(indexesToRemove);
+            newView = new View(context, oldView.getSSTables(), newIndexes);
         }
+        while (!view.compareAndSet(oldView, newView));
+
+        for (SSTableIndex index : indexesToRemove)
+            index.release();
     }
 
     /**
-     * @return the indexes that are built on the given SSTables on the left and corrupted indexes'
-     * corresponding contexts on the right
+     * Called when index is dropped. Mark all {@link SSTableIndex} as released and per-column index files
+     * will be removed when in-flight queries completed and {@code obsolete} is true.
+     *
+     * @param indexWasDropped true if the index is invalidated because it was dropped; false if the index is simply
+     *                        being unloaded.
      */
-    private Pair<Collection<SSTableIndex>, Collection<SSTableContext>> getBuiltIndexes(Collection<SSTableContext> sstableContexts, IndexValidation validation)
+    public void invalidate(boolean indexWasDropped)
     {
-        Set<SSTableIndex> valid = new HashSet<>(sstableContexts.size());
-        Set<SSTableContext> invalid = new HashSet<>();
-
-        for (SSTableContext sstableContext : sstableContexts)
+        View oldView, newView;
+        do
         {
-            if (sstableContext.sstable.isMarkedCompacted())
-                continue;
+            oldView = view.get();
+            newView = new View(context, oldView.getSSTables(), Collections.emptySet());
 
-            if (!sstableContext.indexDescriptor.isPerColumnIndexBuildComplete(index.identifier()))
-            {
-                logger.debug(index.identifier().logMessage("An on-disk index build for SSTable {} has not completed."), sstableContext.descriptor());
-                continue;
-            }
+        } while (!view.compareAndSet(oldView, newView));
 
-            if (sstableContext.indexDescriptor.isIndexEmpty(index.termType(), index.identifier()))
-            {
-                logger.debug(index.identifier().logMessage("No on-disk index was built for SSTable {} because the SSTable " +
-                                                           "had no indexable rows for the index."), sstableContext.descriptor());
-                continue;
-            }
-
-            try
-            {
-                if (validation != IndexValidation.NONE)
-                {
-                    if (!sstableContext.indexDescriptor.validatePerIndexComponents(index.termType(), index.identifier(), validation, true, false))
-                    {
-                        invalid.add(sstableContext);
-                        continue;
-                    }
-                }
-
-                SSTableIndex ssTableIndex = sstableContext.newSSTableIndex(index);
-                logger.debug(index.identifier().logMessage("Successfully created index for SSTable {}."), sstableContext.descriptor());
-
-                // Try to add new index to the set, if set already has such index, we'll simply release and move on.
-                // This covers situation when SSTable collection has the same SSTable multiple
-                // times because we don't know what kind of collection it actually is.
-                if (!valid.add(ssTableIndex))
-                {
-                    ssTableIndex.release();
-                }
-            }
-            catch (Throwable e)
-            {
-                logger.warn(index.identifier().logMessage("Failed to update per-column components for SSTable {}"), sstableContext.descriptor(), e);
-                invalid.add(sstableContext);
-            }
+        for (SSTableIndex index : oldView)
+        {
+            if (indexWasDropped)
+                index.markIndexDropped();
+            else
+                index.release();
         }
-
-        return Pair.create(valid, invalid);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java b/src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java
index 2da7acfb1906..52629ff7e395 100644
--- a/src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java
+++ b/src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java
@@ -18,83 +18,96 @@
 
 package org.apache.cassandra.index.sai.view;
 
+import java.lang.invoke.MethodHandles;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
-import com.google.common.base.MoreObjects;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.utils.Interval;
 import org.apache.cassandra.utils.IntervalTree;
 
-public class RangeTermTree
+public class RangeTermTree implements TermTree
 {
-    private static final Logger logger = LoggerFactory.getLogger(RangeTermTree.class);
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-    protected final ByteBuffer min, max;
-    protected final IndexTermType indexTermType;
-    
-    private final IntervalTree<Term, SSTableIndex, Interval<Term, SSTableIndex>> rangeTree;
+    protected final AbstractType<?> comparator;
+    // Because each version can have different encodings, we group indexes by version.
+    private final Map<Version, IntervalTree<Term, SSTableIndex, Interval<Term, SSTableIndex>>> rangeTrees;
 
-    private RangeTermTree(ByteBuffer min, ByteBuffer max, IntervalTree<Term, SSTableIndex, Interval<Term, SSTableIndex>> rangeTree, IndexTermType indexTermType)
+    private RangeTermTree(Map<Version, IntervalTree<Term, SSTableIndex, Interval<Term, SSTableIndex>>> rangeTrees, AbstractType<?> comparator)
     {
-        this.min = min;
-        this.max = max;
-        this.rangeTree = rangeTree;
-        this.indexTermType = indexTermType;
+        this.rangeTrees = rangeTrees;
+        this.comparator = comparator;
     }
 
-    public List<SSTableIndex> search(Expression e)
+    public Set<SSTableIndex> search(Expression e)
     {
-        ByteBuffer minTerm = e.lower() == null ? min : e.lower().value.encoded;
-        ByteBuffer maxTerm = e.upper() == null ? max : e.upper().value.encoded;
-
-        return rangeTree.search(Interval.create(new Term(minTerm, indexTermType),
-                                                new Term(maxTerm, indexTermType),
-                                                null));
+        Set<SSTableIndex> result = new HashSet<>();
+        rangeTrees.forEach((version, rangeTree) -> {
+            // Get the bounds given the version. Notice that we use the partially-encoded representation for bounds
+            // because that is how we store them in the range tree. The comparator is used to compare the bounds to
+            // each tree's min/max to see if the sstable index is in the query range.
+            Term minTerm = e.lower == null ? rangeTree.min() : new Term(e.getPartiallyEncodedLowerBound(version), comparator, version);
+            Term maxTerm = e.upper == null ? rangeTree.max() : new Term(e.getPartiallyEncodedUpperBound(version), comparator, version);
+            result.addAll(rangeTree.search(Interval.create(minTerm, maxTerm, null)));
+        });
+        return result;
     }
 
-    static class Builder
+    static class Builder extends TermTree.Builder
     {
-        private final IndexTermType indexTermType;
-        private ByteBuffer min, max;
-
-        final List<Interval<Term, SSTableIndex>> intervals = new ArrayList<>();
+        // Because different indexes can have different encodings, we must track the versions of the indexes
+        final Map<Version, List<Interval<Term, SSTableIndex>>> intervalsByVersion = new HashMap<>();
 
-        protected Builder(IndexTermType indexTermType)
+        protected Builder(AbstractType<?> comparator)
         {
-            this.indexTermType = indexTermType;
+            super(comparator);
         }
 
-        public final void add(SSTableIndex index)
+        public void addIndex(SSTableIndex index)
         {
-            assert !indexTermType.isVector();
-
             Interval<Term, SSTableIndex> interval =
-                    Interval.create(new Term(index.minTerm(), indexTermType), new Term(index.maxTerm(), indexTermType), index);
+                    Interval.create(new Term(index.minTerm(), comparator, index.getVersion()),
+                                    new Term(index.maxTerm(), comparator, index.getVersion()),
+                                    index);
 
             if (logger.isTraceEnabled())
             {
-                logger.trace(index.getIndexIdentifier().logMessage("Adding index for SSTable {} with minTerm={} and maxTerm={}..."),
-                                                                   index.getSSTable().descriptor,
-                                                                   index.minTerm() != null ? indexTermType.indexType().compose(index.minTerm()) : null,
-                                                                   index.maxTerm() != null ? indexTermType.indexType().compose(index.maxTerm()) : null);
+                IndexContext context = index.getIndexContext();
+                logger.trace(context.logMessage("Adding index for SSTable {} with minTerm={} and maxTerm={} and version={}..."),
+                                                index.getSSTable().descriptor, 
+                                                index.minTerm() != null ? comparator.compose(index.minTerm()) : null,
+                                                index.maxTerm() != null ? comparator.compose(index.maxTerm()) : null,
+                                                index.getVersion());
             }
 
-            intervals.add(interval);
-
-            min = min == null || index.getIndexTermType().compare(min, index.minTerm()) > 0 ? index.minTerm() : min;
-            max = max == null || index.getIndexTermType().compare(max, index.maxTerm()) < 0 ? index.maxTerm() : max;
+            intervalsByVersion.compute(index.getVersion(), (__, list) ->
+            {
+                if (list == null)
+                    list = new ArrayList<>();
+                list.add(interval);
+                return list;
+            });
         }
 
-        public RangeTermTree build()
+        public TermTree build()
         {
-            return new RangeTermTree(min, max, IntervalTree.build(intervals), indexTermType);
+            Map<Version, IntervalTree<Term, SSTableIndex, Interval<Term, SSTableIndex>>> trees = new HashMap<>();
+            intervalsByVersion.forEach((version, intervals) -> trees.put(version, IntervalTree.build(intervals)));
+            return new RangeTermTree(trees, comparator);
         }
     }
 
@@ -105,24 +118,26 @@ public RangeTermTree build()
     protected static class Term implements Comparable<Term>
     {
         private final ByteBuffer term;
-        private final IndexTermType indexTermType;
+        private final AbstractType<?> comparator;
+        private final Version version;
 
-        Term(ByteBuffer term, IndexTermType indexTermType)
+        Term(ByteBuffer term, AbstractType<?> comparator, Version version)
         {
             this.term = term;
-            this.indexTermType = indexTermType;
+            this.comparator = comparator;
+            this.version = version;
         }
 
-        @Override
         public int compareTo(Term o)
         {
-            return indexTermType.compare(term, o.term);
-        }
-
-        @Override
-        public String toString()
-        {
-            return MoreObjects.toStringHelper(this).add("term", indexTermType.asString(term)).toString();
+            assert version == o.version : "Cannot compare terms from different versions, but found " + version + " and " + o.version;
+            if (term == null && o.term == null)
+                return 0;
+            if (term == null)
+                return -1;
+            if (o.term == null)
+                return 1;
+            return TypeUtil.compare(term, o.term, comparator, version);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/view/TermTree.java b/src/java/org/apache/cassandra/index/sai/view/TermTree.java
new file mode 100644
index 000000000000..72eece0c2e1b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/view/TermTree.java
@@ -0,0 +1,55 @@
+/*
+ * All changes to the original code are Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.view;
+
+import java.util.Set;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.plan.Expression;
+
+public interface TermTree
+{
+    Set<SSTableIndex> search(Expression e);
+
+    abstract class Builder
+    {
+        protected final AbstractType<?> comparator;
+
+        protected Builder(AbstractType<?> comparator)
+        {
+            this.comparator = comparator;
+        }
+
+        public final void add(SSTableIndex index)
+        {
+            addIndex(index);
+        }
+
+        protected abstract void addIndex(SSTableIndex index);
+
+        public abstract TermTree build();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/view/View.java b/src/java/org/apache/cassandra/index/sai/view/View.java
index 2e30d61422d9..dd04dde81089 100644
--- a/src/java/org/apache/cassandra/index/sai/view/View.java
+++ b/src/java/org/apache/cassandra/index/sai/view/View.java
@@ -18,73 +18,93 @@
 
 package org.apache.cassandra.index.sai.view;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
 import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
 import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-
-/**
- * The View is an immutable, point in time, view of the avalailable {@link SSTableIndex}es for an index.
- * <p>
- * The view maintains a {@link RangeTermTree} for querying the view by value range. This is used by the
- * {@link org.apache.cassandra.index.sai.plan.QueryViewBuilder} to select the set of {@link SSTableIndex}es
- * to perform a query without needing to query indexes that are known not to contain to the requested
- * expression value range.
- */
+import org.apache.cassandra.utils.Interval;
+import org.apache.cassandra.utils.IntervalTree;
+
 public class View implements Iterable<SSTableIndex>
 {
+    private final Set<Descriptor> sstables;
     private final Map<Descriptor, SSTableIndex> view;
 
-    private final RangeTermTree rangeTermTree;
+    private final TermTree termTree;
+    private final AbstractType<?> keyValidator;
+    private final IntervalTree<Key, SSTableIndex, Interval<Key, SSTableIndex>> keyIntervalTree;
 
-    public View(IndexTermType indexTermType, Collection<SSTableIndex> indexes)
+    public View(IndexContext context, Collection<Descriptor> sstables, Collection<SSTableIndex> indexes)
     {
         this.view = new HashMap<>();
+        this.sstables = new HashSet<>(sstables);
+        this.keyValidator = context.keyValidator();
 
-        RangeTermTree.Builder rangeTermTreeBuilder = new RangeTermTree.Builder(indexTermType);
+        AbstractType<?> validator = context.getValidator();
 
+        TermTree.Builder termTreeBuilder = new RangeTermTree.Builder(validator);
+
+        List<Interval<Key, SSTableIndex>> keyIntervals = new ArrayList<>();
         for (SSTableIndex sstableIndex : indexes)
         {
             this.view.put(sstableIndex.getSSTable().descriptor, sstableIndex);
-            if (!indexTermType.isVector())
-                rangeTermTreeBuilder.add(sstableIndex);
+            if (!sstableIndex.getIndexContext().isVector())
+                termTreeBuilder.add(sstableIndex);
+
+            keyIntervals.add(Interval.create(new Key(sstableIndex.minKey()),
+                                             new Key(sstableIndex.maxKey()),
+                                             sstableIndex));
         }
 
-        this.rangeTermTree = rangeTermTreeBuilder.build();
+        this.termTree = termTreeBuilder.build();
+        this.keyIntervalTree = IntervalTree.build(keyIntervals);
     }
 
     /**
      * Search for a list of {@link SSTableIndex}es that contain values within
-     * the value range requested in the {@link Expression}
+     * the value range requested in the {@link Expression}. Expressions associated with ORDER BY are not
+     * expected, and will throw an exception.
      */
-    public Collection<SSTableIndex> match(Expression expression)
+    public Set<SSTableIndex> match(Expression expression)
     {
-        if (expression.getIndexOperator() == Expression.IndexOperator.ANN)
-            return getIndexes();
+        if (expression.getOp() == Expression.Op.ORDER_BY)
+            throw new IllegalArgumentException("ORDER BY expression is not supported");
+        if (expression.getOp() == Expression.Op.BOUNDED_ANN || expression.getOp().isNonEquality())
+            return new HashSet<>(getIndexes());
+        return termTree.search(expression);
+    }
 
-        return rangeTermTree.search(expression);
+    public List<SSTableIndex> match(DecoratedKey minKey, DecoratedKey maxKey)
+    {
+        return keyIntervalTree.search(Interval.create(new Key(minKey), new Key(maxKey), null));
     }
 
-    @Override
     public Iterator<SSTableIndex> iterator()
     {
         return view.values().iterator();
     }
 
-    public Collection<SSTableIndex> getIndexes()
+    public Collection<Descriptor> getSSTables()
     {
-        return view.values();
+        return sstables;
     }
 
-    public boolean containsSSTable(SSTableReader sstable)
+    public Collection<SSTableIndex> getIndexes()
     {
-        return view.containsKey(sstable.descriptor);
+        return view.values();
     }
 
     public int size()
@@ -92,9 +112,64 @@ public int size()
         return view.size();
     }
 
+    public @Nullable SSTableIndex getSSTableIndex(Descriptor descriptor)
+    {
+        return view.get(descriptor);
+    }
+    
+    /**
+     * Tells if an index for the given sstable exists.
+     * It's equivalent to {@code getSSTableIndex(descriptor) != null }.
+     * @param descriptor identifies the sstable
+     */
+    public boolean containsSSTableIndex(Descriptor descriptor)
+    {
+        return view.containsKey(descriptor);
+    }
+
+    /**
+     * Returns true if this view has been based on the Cassandra view containing given sstable.
+     * In other words, it tells if SAI was given a chance to load the index for the given sstable.
+     * It does not determine if the index exists and was actually loaded.
+     * To check the existence of the index, use {@link #containsSSTableIndex(Descriptor)}.
+     * <p>
+     * This method allows to distinguish a situation when the sstable has no index, the index is
+     * invalid, or was not loaded for whatever reason,
+     * from a situation where the view hasn't been updated yet to reflect the newly added sstable.
+     */
+    public boolean isAwareOfSSTable(Descriptor descriptor)
+    {
+        return sstables.contains(descriptor);
+    }
+
+    /**
+     * This is required since IntervalTree doesn't support custom Comparator
+     * implementations and relied on items to be comparable which "raw" keys are not.
+     */
+    private static class Key implements Comparable<Key>
+    {
+        private final DecoratedKey key;
+
+        public Key(DecoratedKey key)
+        {
+            this.key = key;
+        }
+
+        public int compareTo(Key o)
+        {
+            if (key == null && o.key == null)
+                return 0;
+            if (key == null)
+                return -1;
+            if (o.key == null)
+                return 1;
+            return key.compareTo(o.key);
+        }
+    }
+
     @Override
     public String toString()
     {
-        return String.format("View{view=%s}", view);
+        return String.format("View{view=%s, keyValidator=%s, keyIntervalTree=%s}", view, keyValidator, keyIntervalTree);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java
index 2d2e5e2634c4..7430ce341280 100644
--- a/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java
+++ b/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java
@@ -27,7 +27,9 @@
 import org.apache.cassandra.db.virtual.VirtualTable;
 import org.apache.cassandra.dht.LocalPartitioner;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
 import org.apache.cassandra.schema.Schema;
@@ -90,18 +92,19 @@ public DataSet data()
 
                 if (group != null)
                 {
-                    group.getIndexes().forEach(i -> {
-                        StorageAttachedIndex index = (StorageAttachedIndex) i;
-                        String indexName = index.identifier().indexName;
+                    for (Index index : group.getIndexes())
+                    {
+                        IndexContext context = ((StorageAttachedIndex) index).getIndexContext();
+                        String indexName = context.getIndexName();
 
                         dataset.row(ks, indexName)
                                .column(TABLE_NAME, cfs.name)
-                               .column(COLUMN_NAME, index.termType().columnName())
+                               .column(COLUMN_NAME, context.getColumnName())
                                .column(IS_QUERYABLE, manager.isIndexQueryable(index))
                                .column(IS_BUILDING, manager.isIndexBuilding(indexName))
-                               .column(IS_STRING, index.termType().isLiteral())
-                               .column(ANALYZER, index.hasAnalyzer() ? index.analyzer().toString() : "NoOpAnalyzer");
-                    });
+                               .column(IS_STRING, context.isLiteral())
+                               .column(ANALYZER, context.getAnalyzerFactory().toString());
+                    }
                 }
             }
         }
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/IndexesSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/IndexesSystemView.java
new file mode 100644
index 000000000000..79fc350839cd
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/virtual/IndexesSystemView.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.virtual;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.virtual.AbstractVirtualTable;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.db.virtual.VirtualTable;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.index.sai.view.View;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * A {@link VirtualTable} providing a system view of per-column storage-attached index metadata.
+ */
+public class IndexesSystemView extends AbstractVirtualTable
+{
+    static final String NAME = "indexes";
+
+    static final String KEYSPACE_NAME = "keyspace_name";
+    static final String INDEX_NAME = "index_name";
+    static final String TABLE_NAME = "table_name";
+    static final String COLUMN_NAME = "column_name";
+    static final String IS_QUERYABLE = "is_queryable";
+    static final String IS_BUILDING = "is_building";
+    static final String IS_STRING = "is_string";
+    static final String ANALYZER = "analyzer";
+    static final String INDEXED_SSTABLE_COUNT = "indexed_sstable_count";
+    static final String SSTABLE_COUNT = "sstable_count";
+    static final String CELL_COUNT = "cell_count";
+    static final String PER_TABLE_DISK_SIZE = "per_table_disk_size";
+    static final String PER_COLUMN_DISK_SIZE = "per_column_disk_size";
+
+    public IndexesSystemView(String keyspace)
+    {
+        super(TableMetadata.builder(keyspace, NAME)
+                           .partitioner(new LocalPartitioner(UTF8Type.instance))
+                           .comment("Storage-attached column index metadata")
+                           .kind(TableMetadata.Kind.VIRTUAL)
+                           .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+                           .addClusteringColumn(INDEX_NAME, UTF8Type.instance)
+                           .addRegularColumn(TABLE_NAME, UTF8Type.instance)
+                           .addRegularColumn(COLUMN_NAME, UTF8Type.instance)
+                           .addRegularColumn(IS_QUERYABLE, BooleanType.instance)
+                           .addRegularColumn(IS_BUILDING, BooleanType.instance)
+                           .addRegularColumn(IS_STRING, BooleanType.instance)
+                           .addRegularColumn(ANALYZER, UTF8Type.instance)
+                           .addRegularColumn(INDEXED_SSTABLE_COUNT, Int32Type.instance)
+                           .addRegularColumn(SSTABLE_COUNT, Int32Type.instance)
+                           .addRegularColumn(CELL_COUNT, LongType.instance)
+                           .addRegularColumn(PER_TABLE_DISK_SIZE, LongType.instance)
+                           .addRegularColumn(PER_COLUMN_DISK_SIZE, LongType.instance)
+                           .build());
+    }
+
+    @Override
+    public void apply(PartitionUpdate update)
+    {
+        // TODO port DataSet. Now we can't change index queryability via system view
+        throw new InvalidRequestException("Modification is not supported by table " + metadata);
+    }
+
+    @Override
+    public DataSet data()
+    {
+        SimpleDataSet dataset = new SimpleDataSet(metadata());
+
+        for (String ks : Schema.instance.getUserKeyspaces())
+        {
+            Keyspace keyspace = Schema.instance.getKeyspaceInstance(ks);
+            if (keyspace == null)
+                throw new IllegalArgumentException("Unknown keyspace " + ks);
+
+            for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+            {
+                SecondaryIndexManager manager = cfs.indexManager;
+                StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+
+                int sstables = cfs.getLiveSSTables().size();
+                if (group != null)
+                {
+                    for (Index index : group.getIndexes())
+                    {
+                        IndexContext context = ((StorageAttachedIndex)index).getIndexContext();
+                        String indexName = context.getIndexName();
+                        View view = context.getView();
+
+                        dataset.row(ks, indexName)
+                               .column(TABLE_NAME, cfs.name)
+                               .column(COLUMN_NAME, context.getColumnName())
+                               .column(IS_QUERYABLE, manager.isIndexQueryable(index))
+                               .column(IS_BUILDING, manager.isIndexBuilding(indexName))
+                               .column(IS_STRING, context.isLiteral())
+                               .column(ANALYZER, context.getAnalyzerFactory().toString())
+                               .column(INDEXED_SSTABLE_COUNT, view.size())
+                               .column(SSTABLE_COUNT, sstables)
+                               .column(CELL_COUNT, context.getCellCount())
+                               .column(PER_TABLE_DISK_SIZE, group.diskUsage())
+                               .column(PER_COLUMN_DISK_SIZE, context.diskUsage());
+                    }
+                }
+            }
+        }
+
+        return dataset;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java
index 7bc54261956b..a68c0fa38585 100644
--- a/src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java
+++ b/src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java
@@ -27,9 +27,11 @@
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.LocalPartitioner;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.Schema;
@@ -97,28 +99,29 @@ public DataSet data()
                 {
                     Token.TokenFactory tokenFactory = cfs.metadata().partitioner.getTokenFactory();
 
-                    group.getIndexes().forEach(i -> {
-                        StorageAttachedIndex index = (StorageAttachedIndex)i;
+                    for (Index index : group.getIndexes())
+                    {
+                        IndexContext indexContext = ((StorageAttachedIndex)index).getIndexContext();
 
-                        for (SSTableIndex sstableIndex : index.view())
+                        for (SSTableIndex sstableIndex : indexContext.getView())
                         {
                             SSTableReader sstable = sstableIndex.getSSTable();
                             Descriptor descriptor = sstable.descriptor;
                             AbstractBounds<Token> bounds = sstable.getBounds();
 
-                            dataset.row(ks, index.identifier().indexName, sstable.getFilename())
+                            dataset.row(ks, indexContext.getIndexName(), sstable.getFilename())
                                    .column(TABLE_NAME, descriptor.cfname)
-                                   .column(COLUMN_NAME, index.termType().columnName())
+                                   .column(COLUMN_NAME, indexContext.getColumnName())
                                    .column(FORMAT_VERSION, sstableIndex.getVersion().toString())
                                    .column(CELL_COUNT, sstableIndex.getRowCount())
                                    .column(MIN_ROW_ID, sstableIndex.minSSTableRowId())
                                    .column(MAX_ROW_ID, sstableIndex.maxSSTableRowId())
                                    .column(START_TOKEN, tokenFactory.toString(bounds.left))
                                    .column(END_TOKEN, tokenFactory.toString(bounds.right))
-                                   .column(PER_TABLE_DISK_SIZE, sstableIndex.getSSTableContext().diskUsage())
+                                   .column(PER_TABLE_DISK_SIZE, sstableIndex.sizeOfPerSSTableComponents())
                                    .column(PER_COLUMN_DISK_SIZE, sstableIndex.sizeOfPerColumnComponents());
                         }
-                    });
+                    }
                 }
             }
         }
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/SSTablesSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/SSTablesSystemView.java
new file mode 100644
index 000000000000..68a258520072
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/virtual/SSTablesSystemView.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.virtual;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.virtual.AbstractVirtualTable;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.db.virtual.VirtualTable;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * A {@link VirtualTable} providing a system view of SSTable index metadata.
+ */
+public class SSTablesSystemView extends AbstractVirtualTable
+{
+    static final String NAME = "sstable_indexes";
+
+    static final String KEYSPACE_NAME = "keyspace_name";
+    static final String INDEX_NAME = "index_name";
+    static final String SSTABLE_NAME = "sstable_name";
+    static final String TABLE_NAME = "table_name";
+    static final String COLUMN_NAME = "column_name";
+    static final String FORMAT_VERSION = "format_version";
+    static final String CELL_COUNT = "cell_count";
+    static final String MIN_ROW_ID = "min_row_id";
+    static final String MAX_ROW_ID = "max_row_id";
+    static final String START_TOKEN = "start_token";
+    static final String END_TOKEN = "end_token";
+    static final String PER_TABLE_DISK_SIZE = "per_table_disk_size";
+    static final String PER_COLUMN_DISK_SIZE = "per_column_disk_size";
+
+    public SSTablesSystemView(String keyspace)
+    {
+        super(TableMetadata.builder(keyspace, NAME)
+                           .partitioner(new LocalPartitioner(UTF8Type.instance))
+                           .comment("SSTable index metadata")
+                           .kind(TableMetadata.Kind.VIRTUAL)
+                           .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+                           .addClusteringColumn(INDEX_NAME, UTF8Type.instance)
+                           .addClusteringColumn(SSTABLE_NAME, UTF8Type.instance)
+                           .addRegularColumn(TABLE_NAME, UTF8Type.instance)
+                           .addRegularColumn(COLUMN_NAME, UTF8Type.instance)
+                           .addRegularColumn(FORMAT_VERSION, UTF8Type.instance)
+                           .addRegularColumn(CELL_COUNT, LongType.instance)
+                           .addRegularColumn(MIN_ROW_ID, LongType.instance)
+                           .addRegularColumn(MAX_ROW_ID, LongType.instance)
+                           .addRegularColumn(START_TOKEN, UTF8Type.instance)
+                           .addRegularColumn(END_TOKEN, UTF8Type.instance)
+                           .addRegularColumn(PER_TABLE_DISK_SIZE, LongType.instance)
+                           .addRegularColumn(PER_COLUMN_DISK_SIZE, LongType.instance)
+                           .build());
+    }
+
+    @Override
+    public DataSet data()
+    {
+        SimpleDataSet dataset = new SimpleDataSet(metadata());
+
+        for (String ks : Schema.instance.getUserKeyspaces())
+        {
+            Keyspace keyspace = Schema.instance.getKeyspaceInstance(ks);
+            if (keyspace == null)
+                throw new IllegalArgumentException("Unknown keyspace " + ks);
+
+            for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+            {
+                StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+
+                if (group != null)
+                {
+                    Token.TokenFactory tokenFactory = cfs.metadata().partitioner.getTokenFactory();
+
+                    for (Index index : group.getIndexes())
+                    {
+                        IndexContext indexContext = ((StorageAttachedIndex)index).getIndexContext();
+
+                        for (SSTableIndex sstableIndex : indexContext.getView())
+                        {
+                            // Empty indexes were introduced to make negative searches
+                            // (NOT_EQ, NOT_CONTAINS, NOT_CONTAINS_KEY) work, but they don't have any representation
+                            // on disk, so for backwards compatibility we're not reporting them.
+                            if (sstableIndex.isEmpty())
+                                continue;
+
+                            SSTableReader sstable = sstableIndex.getSSTable();
+                            Descriptor descriptor = sstable.descriptor;
+                            AbstractBounds<Token> bounds = sstable.getBounds();
+
+                            dataset.row(ks, indexContext.getIndexName(), sstable.getFilename())
+                                   .column(TABLE_NAME, descriptor.cfname)
+                                   .column(COLUMN_NAME, indexContext.getColumnName())
+                                   .column(FORMAT_VERSION, sstableIndex.getVersion().toString())
+                                   .column(CELL_COUNT, sstableIndex.getRowCount())
+                                   .column(MIN_ROW_ID, sstableIndex.minSSTableRowId())
+                                   .column(MAX_ROW_ID, sstableIndex.maxSSTableRowId())
+                                   .column(START_TOKEN, tokenFactory.toString(bounds.left))
+                                   .column(END_TOKEN, tokenFactory.toString(bounds.right))
+                                   .column(PER_TABLE_DISK_SIZE, sstableIndex.sizeOfPerSSTableComponents())
+                                   .column(PER_COLUMN_DISK_SIZE, sstableIndex.sizeOfPerColumnComponents());
+                        }
+                    }
+                }
+            }
+        }
+
+        return dataset;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java
index d3206f1766d8..e4f4dd1d1978 100644
--- a/src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java
+++ b/src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java
@@ -28,9 +28,11 @@
 import org.apache.cassandra.db.virtual.SimpleDataSet;
 import org.apache.cassandra.db.virtual.VirtualTable;
 import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 
@@ -39,7 +41,7 @@
  */
 public class SegmentsSystemView extends AbstractVirtualTable
 {
-    public static final String NAME = "sai_sstable_index_segments";
+    public static final String NAME = "sstable_index_segments";
 
     public static final String KEYSPACE_NAME = "keyspace_name";
     public static final String INDEX_NAME = "index_name";
@@ -87,8 +89,8 @@ public DataSet data()
     {
         SimpleDataSet dataset = new SimpleDataSet(metadata());
 
-        forEachIndex(index -> {
-            for (SSTableIndex sstableIndex : index.view())
+        forEachIndex(indexContext -> {
+            for (SSTableIndex sstableIndex : indexContext.getView())
             {
                 sstableIndex.populateSegmentView(dataset);
             }
@@ -97,20 +99,25 @@ public DataSet data()
         return dataset;
     }
 
-    private void forEachIndex(Consumer<StorageAttachedIndex> process)
+    private void forEachIndex(Consumer<IndexContext> process)
     {
         for (String ks : Schema.instance.getUserKeyspaces())
         {
             Keyspace keyspace = Schema.instance.getKeyspaceInstance(ks);
             if (keyspace == null)
-                throw new IllegalStateException("Unknown keyspace " + ks + ". This can occur if the keyspace is being dropped.");
+                throw new IllegalArgumentException("Unknown keyspace " + ks);
 
             for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
             {
                 StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
 
                 if (group != null)
-                    group.getIndexes().stream().map(index -> (StorageAttachedIndex) index).forEach(process);
+                {
+                    for (Index index : group.getIndexes())
+                    {
+                        process.accept(((StorageAttachedIndex)index).getIndexContext());
+                    }
+                }
             }
         }
     }
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
index ccfed7f5c9c9..6761570ff82c 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
@@ -112,7 +112,7 @@ public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs,
                                         });
                    });
 
-            return new SASIIndexBuilder(cfs, sstables);
+            return new SASIIndexBuilder(cfs, sstables, indexes);
         }
     }
 
@@ -135,7 +135,7 @@ public SASIIndex(ColumnFamilyStore baseCfs, IndexMetadata config)
 
         SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> toRebuild = new TreeMap<>(SSTableReader.idComparator);
 
-        for (SSTableReader sstable : index.init(tracker.getView().liveSSTables()))
+        for (SSTableReader sstable : index.init(tracker.getLiveSSTables()))
         {
             Map<ColumnMetadata, ColumnIndex> perSSTable = toRebuild.get(sstable);
             if (perSSTable == null)
@@ -144,7 +144,7 @@ public SASIIndex(ColumnFamilyStore baseCfs, IndexMetadata config)
             perSSTable.put(index.getDefinition(), index);
         }
 
-        CompactionManager.instance.submitIndexBuild(new SASIIndexBuilder(baseCfs, toRebuild));
+        CompactionManager.instance.submitIndexBuild(new SASIIndexBuilder(baseCfs, toRebuild, Collections.singleton(this)));
     }
 
     /**
@@ -187,7 +187,7 @@ public static Map<String, String> validateOptions(Map<String, String> options, T
     @Override
     public void register(IndexRegistry registry)
     {
-        registry.registerIndex(this, new Group.Key(this), () -> new SASIIndexGroup(this));
+        registry.registerIndex(this, new Index.Group.Key(index), () -> new SASIIndexGroup());
     }
 
     public IndexMetadata getIndexMetadata()
@@ -256,7 +256,9 @@ public AbstractType<?> customExpressionValueType()
 
     public RowFilter getPostIndexQueryFilter(RowFilter filter)
     {
-        return filter.withoutExpressions();
+        // This index doesn't support disjunctions, so if the query has any, we simply apply the entire filter.
+        // Otherwise, the index searcher should be able to handle the entire filter without postfiltering.
+        return filter.containsDisjunctions() ? filter : filter.withoutExpressions();
     }
 
     public long getEstimatedResultRows()
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
index 555bce1b9add..c6ac2fdf6fd0 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
@@ -24,20 +24,21 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 import java.util.SortedMap;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.SecondaryIndexBuilder;
 import org.apache.cassandra.index.sasi.conf.ColumnIndex;
 import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.sstable.KeyReader;
 import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
+import org.apache.cassandra.io.sstable.SSTableWatcher;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.RandomAccessReader;
@@ -55,12 +56,13 @@ class SASIIndexBuilder extends SecondaryIndexBuilder
     // Keep targetDirectory for compactions, needed for `nodetool compactionstats`
     private String targetDirectory;
 
+    private final Set<Index> builtIndexes;
     private final SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables;
 
     private long bytesProcessed = 0;
     private final long totalBytesToProcess;
 
-    public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables)
+    public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables, Set<Index> indexes)
     {
         long totalBytesToProcess = 0;
         for (SSTableReader sstable : sstables.keySet())
@@ -68,6 +70,7 @@ public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap<SSTableReader, Map<Colu
 
         this.cfs = cfs;
         this.sstables = sstables;
+        this.builtIndexes = indexes;
         this.totalBytesToProcess = totalBytesToProcess;
     }
 
@@ -80,6 +83,7 @@ public void build()
             SSTableReader sstable = e.getKey();
             Map<ColumnMetadata, ColumnIndex> indexes = e.getValue();
 
+            SSTableWatcher.instance.onIndexBuild(sstable, builtIndexes);
             try (RandomAccessReader dataFile = sstable.openDataReader())
             {
                 PerSSTableIndexWriter indexWriter = SASIIndex.newWriter(keyValidator, sstable.descriptor, indexes, OperationType.COMPACTION);
@@ -89,8 +93,7 @@ public void build()
                 {
                     while (!keys.isExhausted())
                     {
-                        if (isStopRequested())
-                            throw new CompactionInterruptedException(getCompactionInfo());
+                        throwIfStopRequested();
 
                         final DecoratedKey key = sstable.decorateKey(keys.key());
                         final long keyPosition = keys.keyPositionForSecondaryIndex();
@@ -128,20 +131,21 @@ public void build()
         }
     }
 
-    public CompactionInfo getCompactionInfo()
+    @Override
+    public OperationProgress getProgress()
     {
-        return new CompactionInfo(cfs.metadata(),
-                                  OperationType.INDEX_BUILD,
-                                  bytesProcessed,
-                                  totalBytesToProcess,
-                                  compactionId,
-                                  sstables.keySet(),
-                                  targetDirectory);
+        return new OperationProgress(cfs.metadata(),
+                                     OperationType.INDEX_BUILD,
+                                     bytesProcessed,
+                                     totalBytesToProcess,
+                                     compactionId,
+                                     sstables.keySet(),
+                                     targetDirectory);
     }
 
     private void completeSSTable(PerSSTableIndexWriter indexWriter, SSTableReader sstable, Collection<ColumnIndex> indexes)
     {
-        indexWriter.complete();
+        indexWriter.complete(sstable);
 
         for (ColumnIndex index : indexes)
         {
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndexGroup.java b/src/java/org/apache/cassandra/index/sasi/SASIIndexGroup.java
index 8b4c6fa0cbcf..f9aa3d526bd0 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndexGroup.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndexGroup.java
@@ -25,9 +25,9 @@
 
 public class SASIIndexGroup extends SingletonIndexGroup
 {
-    public SASIIndexGroup(SASIIndex index)
+    public SASIIndexGroup()
     {
-        super(index);
+        super();
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java b/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java
index de9c0c2b4ee6..0d5be7ec787e 100644
--- a/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java
@@ -17,15 +17,12 @@
  */
 package org.apache.cassandra.index.sasi;
 
-import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import com.google.common.base.Function;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
-import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.index.sasi.conf.ColumnIndex;
 import org.apache.cassandra.index.sasi.disk.OnDiskIndex;
@@ -33,7 +30,6 @@
 import org.apache.cassandra.index.sasi.disk.Token;
 import org.apache.cassandra.index.sasi.plan.Expression;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
-import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
@@ -42,7 +38,7 @@
 public class SSTableIndex
 {
     private final ColumnIndex columnIndex;
-    private final Ref<SSTableReader> sstableRef;
+    private final Ref<? extends SSTableReader> sstableRef;
     private final SSTableReader sstable;
     private final OnDiskIndex index;
     private final AtomicInteger references = new AtomicInteger(1);
@@ -64,7 +60,7 @@ public SSTableIndex(ColumnIndex index, File indexFile, SSTableReader referent)
                 sstable.getFilename(),
                 columnIndex.getIndexName());
 
-        this.index = new OnDiskIndex(indexFile, validator, new DecoratedKeyFetcher(sstable));
+        this.index = new OnDiskIndex(indexFile, validator, sstable.openKeyFetcher(true));
     }
 
     public OnDiskIndexBuilder.Mode mode()
@@ -162,36 +158,4 @@ public String toString()
         return String.format("SSTableIndex(column: %s, SSTable: %s)", columnIndex.getColumnName(), sstable.descriptor);
     }
 
-    private static class DecoratedKeyFetcher implements Function<Long, DecoratedKey>
-    {
-        private final SSTableReader sstable;
-
-        DecoratedKeyFetcher(SSTableReader reader)
-        {
-            sstable = reader;
-        }
-
-        public DecoratedKey apply(Long offset)
-        {
-            try
-            {
-                return sstable.keyAtPositionFromSecondaryIndex(offset);
-            }
-            catch (IOException e)
-            {
-                throw new FSReadError(new IOException("Failed to read key from " + sstable.descriptor, e), sstable.getFilename());
-            }
-        }
-
-        public int hashCode()
-        {
-            return sstable.descriptor.hashCode();
-        }
-
-        public boolean equals(Object other)
-        {
-            return other instanceof DecoratedKeyFetcher
-                    && sstable.descriptor.equals(((DecoratedKeyFetcher) other).sstable.descriptor);
-        }
-    }
 }
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java b/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
index 9786a86ae28f..77d7233859b9 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
@@ -25,11 +25,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.tartarus.snowball.SnowballStemmer;
+
 import com.github.benmanes.caffeine.cache.CacheLoader;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
-import org.tartarus.snowball.SnowballStemmer;
 import org.tartarus.snowball.ext.DanishStemmer;
 import org.tartarus.snowball.ext.DutchStemmer;
 import org.tartarus.snowball.ext.EnglishStemmer;
diff --git a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
index afbeaaaf88d3..5828441d2a3e 100644
--- a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
@@ -222,6 +222,9 @@ public boolean supports(Operator op)
         if (op == Operator.LIKE)
             return isLiteral();
 
+        if (op == Operator.ANALYZER_MATCHES)
+            return false;
+
         Op operator = Op.valueOf(op);
         return !(isTokenized && operator == Op.EQ) // EQ is only applicable to non-tokenized indexes
                && operator != Op.IN // IN operator is not supported
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
index 0eab229556ed..1ffec07d4f12 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
@@ -30,20 +30,19 @@
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 
-import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
 
-import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.index.sasi.Term;
 import org.apache.cassandra.index.sasi.plan.Expression;
 import org.apache.cassandra.index.sasi.plan.Expression.Op;
 import org.apache.cassandra.index.sasi.utils.MappedBuffer;
-import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
+import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
 import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
 import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
@@ -116,7 +115,7 @@ public int startAt(SearchResult<DataTerm> found, boolean inclusive)
     protected final long indexSize;
     protected final boolean hasMarkedPartials;
 
-    protected final Function<Long, DecoratedKey> keyFetcher;
+    protected final IKeyFetcher keyFetcher;
 
     protected final String indexPath;
 
@@ -125,9 +124,9 @@ public int startAt(SearchResult<DataTerm> found, boolean inclusive)
 
     protected final ByteBuffer minTerm, maxTerm, minKey, maxKey;
 
-    public OnDiskIndex(File index, AbstractType<?> cmp, Function<Long, DecoratedKey> keyReader)
+    public OnDiskIndex(File index, AbstractType<?> cmp, IKeyFetcher keyFetcher)
     {
-        keyFetcher = keyReader;
+        this.keyFetcher = keyFetcher;
 
         comparator = cmp;
         indexPath = index.absolutePath();
@@ -441,7 +440,7 @@ public Iterator<DataTerm> iterator()
 
     public void close() throws IOException
     {
-        FileUtils.closeQuietly(indexFile);
+        FileUtils.closeQuietly(keyFetcher, indexFile);
     }
 
     private PointerTerm findPointer(ByteBuffer query)
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndexBuilder.java b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndexBuilder.java
index 9071e1088419..086e0b5c53fc 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndexBuilder.java
@@ -129,9 +129,9 @@ public static TermSize sizeOf(AbstractType<?> comparator)
     public static final int SUPER_BLOCK_SIZE = 64;
     public static final int IS_PARTIAL_BIT = 15;
 
-    private static final SequentialWriterOption WRITER_OPTION = SequentialWriterOption.newBuilder()
-                                                                                      .bufferSize(BLOCK_SIZE)
-                                                                                      .build();
+    public static final SequentialWriterOption WRITER_OPTION = SequentialWriterOption.newBuilder()
+                                                                                     .bufferSize(BLOCK_SIZE)
+                                                                                     .build();
 
     private final List<MutableLevel<InMemoryPointerTerm>> levels = new ArrayList<>();
     private MutableLevel<InMemoryDataTerm> dataLevel;
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
index 5b01cad12402..cb7d25a260b6 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
@@ -44,6 +44,7 @@
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.ColumnMetadata;
@@ -105,10 +106,10 @@ public void begin()
     {}
 
     @Override
-    public void startPartition(DecoratedKey key, long keyPosition, long KeyPositionForSASI)
+    public void startPartition(DecoratedKey key, long keyPosition, long keyPositionForSASI)
     {
         currentKey = key;
-        currentKeyPosition = KeyPositionForSASI;
+        currentKeyPosition = keyPositionForSASI;
     }
 
     @Override
@@ -138,7 +139,7 @@ public void nextUnfilteredCluster(Unfiltered unfiltered)
     }
 
     @Override
-    public void complete()
+    public void complete(SSTable sstable)
     {
         if (isComplete)
             return;
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/StaticTokenTreeBuilder.java b/src/java/org/apache/cassandra/index/sasi/disk/StaticTokenTreeBuilder.java
index 7a41b38d7a80..42efcf3660d5 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/StaticTokenTreeBuilder.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/StaticTokenTreeBuilder.java
@@ -36,7 +36,7 @@
  * when multiple index segments produced by {@link PerSSTableIndexWriter} are stitched together
  * by {@link PerSSTableIndexWriter#complete()}.
  *
- * This class uses the RangeIterator, now provided by
+ * This class uses the KeyRangeIterator, now provided by
  * {@link CombinedTerm#getTokenIterator()}, to iterate the data twice.
  * The first iteration builds the tree with leaves that contain only enough
  * information to build the upper layers -- these leaves do not store more
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java b/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
index 7f8f3a0f36bd..44495d6a2c6e 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
@@ -18,21 +18,27 @@
 package org.apache.cassandra.index.sasi.disk;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.LongFunction;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import com.carrotsearch.hppc.LongHashSet;
+import com.carrotsearch.hppc.LongSet;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.index.sasi.utils.CombinedValue;
 import org.apache.cassandra.index.sasi.utils.MappedBuffer;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
 import org.apache.cassandra.utils.AbstractGuavaIterator;
 import org.apache.cassandra.utils.MergeIterator;
-
-import com.carrotsearch.hppc.LongHashSet;
-import com.carrotsearch.hppc.LongSet;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.cassandra.utils.Reducer;
 
 import static org.apache.cassandra.index.sasi.disk.TokenTreeBuilder.EntryType;
 
@@ -79,12 +85,12 @@ public long getCount()
         return tokenCount;
     }
 
-    public RangeIterator<Long, Token> iterator(Function<Long, DecoratedKey> keyFetcher)
+    public RangeIterator<Long, Token> iterator(LongFunction<DecoratedKey> keyFetcher)
     {
         return new TokenTreeIterator(file.duplicate(), keyFetcher);
     }
 
-    public OnDiskToken get(final long searchToken, Function<Long, DecoratedKey> keyFetcher)
+    public OnDiskToken get(final long searchToken, LongFunction<DecoratedKey> keyFetcher)
     {
         seekToLeaf(searchToken, file);
         long leafStart = file.position();
@@ -211,7 +217,7 @@ private short searchLeaf(long searchToken, short tokenCount)
 
     public class TokenTreeIterator extends RangeIterator<Long, Token>
     {
-        private final Function<Long, DecoratedKey> keyFetcher;
+        private final LongFunction<DecoratedKey> keyFetcher;
         private final MappedBuffer file;
 
         private long currentLeafStart;
@@ -224,7 +230,7 @@ public class TokenTreeIterator extends RangeIterator<Long, Token>
         protected boolean firstIteration = true;
         private boolean lastLeaf;
 
-        TokenTreeIterator(MappedBuffer file, Function<Long, DecoratedKey> keyFetcher)
+        TokenTreeIterator(MappedBuffer file, LongFunction<DecoratedKey> keyFetcher)
         {
             super(treeMinToken, treeMaxToken, tokenCount);
 
@@ -352,7 +358,7 @@ public static class OnDiskToken extends Token
         private final Set<TokenInfo> info = new HashSet<>(2);
         private final Set<DecoratedKey> loadedKeys = new TreeSet<>(DecoratedKey.comparator);
 
-        public OnDiskToken(MappedBuffer buffer, long position, short leafSize, Function<Long, DecoratedKey> keyFetcher)
+        public OnDiskToken(MappedBuffer buffer, long position, short leafSize, LongFunction<DecoratedKey> keyFetcher)
         {
             super(buffer.getLong(position + (2 * SHORT_BYTES)));
             info.add(new TokenInfo(buffer, position, leafSize, keyFetcher));
@@ -387,11 +393,11 @@ public Iterator<DecoratedKey> iterator()
             if (!loadedKeys.isEmpty())
                 keys.add(loadedKeys.iterator());
 
-            return MergeIterator.get(keys, DecoratedKey.comparator, new MergeIterator.Reducer<DecoratedKey, DecoratedKey>()
+            return MergeIterator.get(keys, DecoratedKey.comparator, new Reducer<DecoratedKey, DecoratedKey>()
             {
                 DecoratedKey reduced = null;
 
-                public boolean trivialReduceIsTrivial()
+                public boolean singleSourceReduceIsTrivial()
                 {
                     return true;
                 }
@@ -401,7 +407,7 @@ public void reduce(int idx, DecoratedKey current)
                     reduced = current;
                 }
 
-                protected DecoratedKey getReduced()
+                public DecoratedKey getReduced()
                 {
                     return reduced;
                 }
@@ -420,7 +426,7 @@ public LongSet getOffsets()
             return offsets;
         }
 
-        public static OnDiskToken getTokenAt(MappedBuffer buffer, int idx, short leafSize, Function<Long, DecoratedKey> keyFetcher)
+        public static OnDiskToken getTokenAt(MappedBuffer buffer, int idx, short leafSize, LongFunction<DecoratedKey> keyFetcher)
         {
             return new OnDiskToken(buffer, getEntryPosition(idx, buffer), leafSize, keyFetcher);
         }
@@ -435,12 +441,12 @@ private static long getEntryPosition(int idx, MappedBuffer file)
     private static class TokenInfo
     {
         private final MappedBuffer buffer;
-        private final Function<Long, DecoratedKey> keyFetcher;
+        private final LongFunction<DecoratedKey> keyFetcher;
 
         private final long position;
         private final short leafSize;
 
-        public TokenInfo(MappedBuffer buffer, long position, short leafSize, Function<Long, DecoratedKey> keyFetcher)
+        public TokenInfo(MappedBuffer buffer, long position, short leafSize, LongFunction<DecoratedKey> keyFetcher)
         {
             this.keyFetcher = keyFetcher;
             this.buffer = buffer;
@@ -505,11 +511,11 @@ private long[] fetchOffsets()
 
     private static class KeyIterator extends AbstractGuavaIterator<DecoratedKey>
     {
-        private final Function<Long, DecoratedKey> keyFetcher;
+        private final LongFunction<DecoratedKey> keyFetcher;
         private final long[] offsets;
         private int index = 0;
 
-        public KeyIterator(Function<Long, DecoratedKey> keyFetcher, long[] offsets)
+        public KeyIterator(LongFunction<DecoratedKey> keyFetcher, long[] offsets)
         {
             this.keyFetcher = keyFetcher;
             this.offsets = offsets;
@@ -517,7 +523,9 @@ public KeyIterator(Function<Long, DecoratedKey> keyFetcher, long[] offsets)
 
         public DecoratedKey computeNext()
         {
-            return index < offsets.length ? keyFetcher.apply(offsets[index++]) : endOfData();
+            return index < offsets.length
+                   ? keyFetcher.apply(offsets[index++])
+                   : endOfData();
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java b/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java
index 432dd8067705..834adfd5ec5f 100644
--- a/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java
+++ b/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java
@@ -78,7 +78,8 @@ public TableMetadata metadata()
 
     public Collection<RowFilter.Expression> getExpressions()
     {
-        return command.rowFilter().getExpressions();
+        // This index doesn't support disjunctions, so we only consider the top-level AND expressions.
+        return command.rowFilter().withoutDisjunctions().expressions();
     }
 
     public DataRange dataRange()
diff --git a/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexQueryPlan.java b/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexQueryPlan.java
index 06664a18343b..1a61468f48cd 100644
--- a/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexQueryPlan.java
+++ b/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexQueryPlan.java
@@ -34,7 +34,8 @@ private SASIIndexQueryPlan(SASIIndex index, RowFilter postIndexFilter)
     @Nullable
     public static SASIIndexQueryPlan create(SASIIndex index, RowFilter rowFilter)
     {
-        for (RowFilter.Expression e : rowFilter.getExpressions())
+        // SASI doesn't support disjunctions, so we only consider the top-level AND expressions for index selection.
+        for (RowFilter.Expression e : rowFilter.withoutDisjunctions().expressions())
         {
             if (index.supportsExpression(e.column(), e.operator()))
                 return new SASIIndexQueryPlan(index, index.getPostIndexQueryFilter(rowFilter));
diff --git a/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java b/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java
index 331f4edff349..78292b4f778a 100644
--- a/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java
+++ b/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java
@@ -23,10 +23,10 @@
 import java.util.List;
 import java.util.PriorityQueue;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterators;
-import org.apache.cassandra.io.util.FileUtils;
 
-import com.google.common.annotations.VisibleForTesting;
+import org.apache.cassandra.io.util.FileUtils;
 
 public class RangeIntersectionIterator
 {
diff --git a/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java b/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java
index f2295699704f..c186e8fb1228 100644
--- a/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java
+++ b/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java
@@ -24,7 +24,7 @@
 import org.apache.cassandra.io.util.FileUtils;
 
 /**
- * Range Union Iterator is used to return sorted stream of elements from multiple RangeIterator instances.
+ * Range Union Iterator is used to return sorted stream of elements from multiple KeyRangeIterator instances.
  *
  * PriorityQueue is used as a sorting mechanism for the ranges, where each computeNext() operation would poll
  * from the queue (and push when done), which returns range that contains the smallest element, because
diff --git a/src/java/org/apache/cassandra/index/sasi/utils/trie/AbstractPatriciaTrie.java b/src/java/org/apache/cassandra/index/sasi/utils/trie/AbstractPatriciaTrie.java
index 8067ccc30cdc..09d32699bd8b 100644
--- a/src/java/org/apache/cassandra/index/sasi/utils/trie/AbstractPatriciaTrie.java
+++ b/src/java/org/apache/cassandra/index/sasi/utils/trie/AbstractPatriciaTrie.java
@@ -790,7 +790,8 @@ TrieEntry<K, V> nextEntryImpl(TrieEntry<K, V> start, TrieEntry<K, V> previous, T
      * This is implemented by going always to the left until
      * we encounter a valid uplink. That uplink is the first key.
      */
-    TrieEntry<K, V> firstEntry()
+    // @Override needed in JDK 21+.
+    public TrieEntry<K, V> firstEntry()
     {
         // if Trie is empty, no first node.
         return isEmpty() ? null : followLeft(root);
diff --git a/src/java/org/apache/cassandra/index/sasi/utils/trie/PatriciaTrie.java b/src/java/org/apache/cassandra/index/sasi/utils/trie/PatriciaTrie.java
index a36af9828c8b..22c65e1d944c 100644
--- a/src/java/org/apache/cassandra/index/sasi/utils/trie/PatriciaTrie.java
+++ b/src/java/org/apache/cassandra/index/sasi/utils/trie/PatriciaTrie.java
@@ -414,7 +414,8 @@ private TrieEntry<K, V> subtree(K prefix)
      * <p>This is implemented by going always to the right until
      * we encounter a valid uplink. That uplink is the last key.
      */
-    private TrieEntry<K, V> lastEntry()
+    // @Override needed in JDK 21+.
+    public TrieEntry<K, V> lastEntry()
     {
         return followRight(root.left);
     }
diff --git a/src/java/org/apache/cassandra/index/transactions/UpdateTransaction.java b/src/java/org/apache/cassandra/index/transactions/UpdateTransaction.java
index 51533c22a9a7..b97de9c86598 100644
--- a/src/java/org/apache/cassandra/index/transactions/UpdateTransaction.java
+++ b/src/java/org/apache/cassandra/index/transactions/UpdateTransaction.java
@@ -63,6 +63,11 @@ public interface UpdateTransaction extends IndexTransaction
     void onPartitionDeletion(DeletionTime deletionTime);
     void onRangeTombstone(RangeTombstone rangeTombstone);
     void onInserted(Row row);
+
+    /**
+     * @param existing the existing row from the Memtable
+     * @param updated the updated version, which includes the update merged with the existing version
+     */
     void onUpdated(Row existing, Row updated);
 
     UpdateTransaction NO_OP = new UpdateTransaction()
diff --git a/src/java/org/apache/cassandra/io/FSError.java b/src/java/org/apache/cassandra/io/FSError.java
index 4c06d9c61f1a..afe1fc635dce 100644
--- a/src/java/org/apache/cassandra/io/FSError.java
+++ b/src/java/org/apache/cassandra/io/FSError.java
@@ -25,11 +25,11 @@
 public abstract class FSError extends IOError
 {
     final String message;
-    public final String path;
+    public final File file;
 
-    public FSError(Throwable cause, File path)
+    public FSError(Throwable cause, File file)
     {
-        this(null, cause, path);
+        this(null, cause, file);
     }
 
     public FSError(Throwable cause, Path path)
@@ -37,18 +37,24 @@ public FSError(Throwable cause, Path path)
         this(null, cause, path);
     }
 
-    public FSError(String message, Throwable cause, File path)
+    public FSError(String message, Throwable cause, File file)
     {
         super(cause);
         this.message = message;
-        this.path = path.toString();
+        this.file = file;
     }
 
     public FSError(String message, Throwable cause, Path path)
     {
         super(cause);
         this.message = message;
-        this.path = path.toString();
+        this.file = new File(path);
+    }
+
+    @Override
+    public String getMessage()
+    {
+        return super.getMessage() + " on file " + String.valueOf(file);
     }
 
     /**
@@ -70,6 +76,6 @@ public static FSError findNested(Throwable top)
     @Override
     public String toString()
     {
-        return getClass().getSimpleName() + (message != null ? ' ' + message : "") + (path != null ? " in " + path : "");
+        return getClass().getSimpleName() + (message != null ? ' ' + message : "") + (file != null ? " in " + file : "");
     }
 }
diff --git a/src/java/org/apache/cassandra/io/FSReadError.java b/src/java/org/apache/cassandra/io/FSReadError.java
index ac1553477032..b785a28efa7c 100644
--- a/src/java/org/apache/cassandra/io/FSReadError.java
+++ b/src/java/org/apache/cassandra/io/FSReadError.java
@@ -29,9 +29,9 @@ public FSReadError(Throwable cause, Path path)
         super(cause, path);
     }
 
-    public FSReadError(Throwable cause, File path)
+    public FSReadError(Throwable cause, File file)
     {
-        super(cause, path);
+        super(cause, file);
     }
 
     public FSReadError(Throwable cause, String path)
diff --git a/src/java/org/apache/cassandra/io/compress/AdaptiveCompressor.java b/src/java/org/apache/cassandra/io/compress/AdaptiveCompressor.java
new file mode 100644
index 000000000000..20703b431388
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/compress/AdaptiveCompressor.java
@@ -0,0 +1,519 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.compress;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.github.luben.zstd.Zstd;
+import com.github.luben.zstd.ZstdCompressCtx;
+import com.github.luben.zstd.ZstdDecompressCtx;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.memtable.AbstractAllocatorMemtable;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+import org.apache.cassandra.metrics.MetricNameFactory;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.ExpMovingAverage;
+
+/**
+ * A compressor that dynamically adapts the compression level to the load.
+ * If the system is not heavily loaded by writes, data are compressed using high compression level.
+ * If the number of compactions or flushes queue up, it decreases the compression level to speed up
+ * flushing / compacting.
+ * <p>
+ * Underneath, the ZStandard compressor is used. The compression level can be changed between the frames,
+ * even when compressing the same sstable file. ZStandard was chosen because at the fast end it
+ * can reach compression speed of LZ4, but at moderate compression levels it usually offers much better
+ * compression ratio (typically files are smaller by 20-40%) than LZ4 without compromising compression speed by too much.
+ * <p>
+ * This compressor can be used for either of Uses: FAST_COMPRESSION and GENERAL.
+ * Each use can have different minimum and maximum compression level limits.
+ * For FAST_COMPRESSION, the number of pending flushes is used as the indicator of write load.
+ * For GENERAL compression, the number of pending compactions is used as the indicator of write load.
+ * <p>
+ * Valid compression levels are in range 0..15 (inclusive), where 0 means fastest compression and 15 means slowest/best.
+ * Usually levels around 7-11 strike the best balance between performance and compresion ratio.
+ * Going above level 12 usually only results in slower compression but not much compression ratio improvement.
+ * <p>
+ * Caution: This compressor decompresses about 2x-4x slower than LZ4Compressor, regardless of the compression level.
+ * Therefore, it may negatively affect read speed from very read-heavy tables, especially when the chunk-cache
+ * hit ratio is low. In synthetic tests with chunk cache disabled, read throughput turned out to be up to 10%
+ * lower than when using LZ4 on some workloads.
+ */
+public class AdaptiveCompressor implements ICompressor
+{
+    @VisibleForTesting
+    static final Map<Uses, AdaptiveCompressor.Metrics> metrics = new EnumMap<>(Map.of(
+        Uses.FAST_COMPRESSION, new Metrics(Uses.FAST_COMPRESSION),
+        Uses.GENERAL, new Metrics(Uses.GENERAL)
+    ));
+
+    protected static final String MIN_COMPRESSION_LEVEL_OPTION_NAME = "min_compression_level";
+    protected static final String MAX_COMPRESSION_LEVEL_OPTION_NAME = "max_compression_level";
+    protected static final String MAX_COMPACTION_QUEUE_LENGTH_OPTION_NAME = "max_compaction_queue_length";
+
+
+    /**
+     * Maps AdaptiveCompressor compression level to underlying ZStandard compression levels.
+     * This mapping is needed because ZStandard levels are not continuous, zstd level 0 is special and means level 3.
+     * Hence, we just use our own continuous scale starting at 0.
+     */
+    private static final int[] zstdCompressionLevels = {
+        -7, // 0  (very fast but compresses poorly)
+        -6, // 1
+        -5, // 2  (LZ4 level is somewhere here)
+        -4, // 3
+        -3, // 4
+        -2, // 5
+        -1, // 6
+        1,  // 7  (sweet spot area usually here)
+        2,  // 8  (sweet spot area usually here)
+        3,  // 9  (sweet spot area usually here, ~50% slower than LZ4)
+        4,  // 10 (sweet spot area usually here)
+        5,  // 11 (sweet spot area usually here)
+        6,  // 12
+        7,  // 13
+        8,  // 14
+        9,  // 15 (very slow, usually over 10x slower than LZ4)
+    };
+
+    public static final int MIN_COMPRESSION_LEVEL = 0;
+    public static final int MAX_COMPRESSION_LEVEL = 15;
+
+    public static final int DEFAULT_MIN_FAST_COMPRESSION_LEVEL = 2;     // zstd level -5
+    public static final int DEFAULT_MAX_FAST_COMPRESSION_LEVEL = 9;     // zstd level 5
+    public static final int DEFAULT_MIN_GENERAL_COMPRESSION_LEVEL = 7;  // zstd level 1
+    public static final int DEFAULT_MAX_GENERAL_COMPRESSION_LEVEL = 12; // zstd level 6
+    public static final int DEFAULT_MAX_COMPACTION_QUEUE_LENGTH = 16;
+
+    private static final ConcurrentHashMap<Params, AdaptiveCompressor> instances = new ConcurrentHashMap<>();
+
+    public static AdaptiveCompressor create(Map<String, String> options)
+    {
+        int minCompressionLevel = getMinCompressionLevel(Uses.GENERAL, options);
+        int maxCompressionLevel = getMaxCompressionLevel(Uses.GENERAL, options);
+        int maxCompactionQueueLength = getMaxCompactionQueueLength(options);
+        return createForCompaction(minCompressionLevel, maxCompressionLevel, maxCompactionQueueLength);
+    }
+
+    private static AdaptiveCompressor createForCompaction(int minCompressionLevel, int maxCompressionLevel, int maxCompactionQueueLength)
+    {
+        Params params = new Params(Uses.GENERAL, minCompressionLevel, maxCompressionLevel, maxCompactionQueueLength);
+        Supplier<Double> compactionPressureSupplier = () -> getCompactionPressure(maxCompactionQueueLength);
+        return instances.computeIfAbsent(params, p -> new AdaptiveCompressor(p, compactionPressureSupplier));
+    }
+
+    /**
+     * Creates a compressor that doesn't refer to any other C* components like compaction manager or memory pools.
+     */
+    @VisibleForTesting
+    public static ICompressor createForUnitTesting()
+    {
+        Params params = new Params(Uses.GENERAL, 9, 9, 0);
+        return new AdaptiveCompressor(params, () -> 0.0);
+    }
+
+    public static AdaptiveCompressor createForFlush(Map<String, String> options)
+    {
+        int minCompressionLevel = getMinCompressionLevel(Uses.FAST_COMPRESSION, options);
+        int maxCompressionLevel = getMaxCompressionLevel(Uses.FAST_COMPRESSION, options);
+        return createForFlush(minCompressionLevel, maxCompressionLevel);
+    }
+
+    private static AdaptiveCompressor createForFlush(int minCompressionLevel, int maxCompressionLevel)
+    {
+        Params params = new Params(Uses.FAST_COMPRESSION, minCompressionLevel, maxCompressionLevel, 0);
+        return instances.computeIfAbsent(params, p -> new AdaptiveCompressor(p, AdaptiveCompressor::getFlushPressure));
+    }
+
+    private final Params params;
+    private final ThreadLocal<State> state;
+    private final Supplier<Double> writePressureSupplier;
+
+    static class Params
+    {
+        final Uses use;
+        final int minCompressionLevel;
+        final int maxCompressionLevel;
+        final int maxCompactionQueueLength;
+
+        Params(Uses use, int minCompressionLevel, int maxCompressionLevel, int maxCompactionQueueLength)
+        {
+            if (minCompressionLevel < MIN_COMPRESSION_LEVEL || minCompressionLevel > MAX_COMPRESSION_LEVEL)
+                throw new IllegalArgumentException("Min compression level " + minCompressionLevel + "out of range" +
+                                                   " [" + MIN_COMPRESSION_LEVEL + ", " + MAX_COMPRESSION_LEVEL + ']');
+            if (maxCompressionLevel < MIN_COMPRESSION_LEVEL || maxCompressionLevel > MAX_COMPRESSION_LEVEL)
+                throw new IllegalArgumentException("Max compression level " + maxCompressionLevel + "out of range" +
+                                                   " [" + MIN_COMPRESSION_LEVEL + ", " + MAX_COMPRESSION_LEVEL + ']');
+            if (maxCompactionQueueLength < 0)
+                throw new IllegalArgumentException("Negative max compaction queue length: " + maxCompactionQueueLength);
+
+            this.use = use;
+            this.minCompressionLevel = minCompressionLevel;
+            this.maxCompressionLevel = maxCompressionLevel;
+            this.maxCompactionQueueLength = maxCompactionQueueLength;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (o == null || getClass() != o.getClass()) return false;
+            Params params = (Params) o;
+            return minCompressionLevel == params.minCompressionLevel && maxCompressionLevel == params.maxCompressionLevel && use == params.use;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(use, minCompressionLevel, maxCompressionLevel);
+        }
+    }
+
+    /**
+     * Keeps thread local state.
+     * We need this because we want to not only monitor pending flushes/compactions but also how much
+     * time we spend in compression relative to time spent by the thread in non-compression tasks like preparation
+     * of data or writing. Because ICompressor can be shared by multiple threads, we need to keep
+     * track of each thread separately.
+     */
+    class State
+    {
+        final ZstdCompressCtx compressCtx = new ZstdCompressCtx().setChecksum(true);
+        final ZstdDecompressCtx decompressCtx = new ZstdDecompressCtx();
+
+        /**
+         * ZStandard compression level that was used when compressing the previous chunk.
+         * Can be adjusted up or down by at most 1 with every next block.
+         */
+        int currentCompressionLevel;
+
+        /**
+         * How much time is spent by the thread in compression relative to the time spent in non-compression code.
+         * Valid range is [0.0, 1.0].
+         * 1.0 means we're doing only compression and nothing else.
+         * 0.0 means we're not spending any time doing compression.
+         * This indicator allows us to detect whether we're bottlenecked by something else than compression,
+         * e.g. by disk I/O or by preparation of data to compress (e.g. iterating the memtable trie).
+         * If this value is low, then there is not much gain in decreasing the compression
+         * level.
+         */
+        ExpMovingAverage relativeTimeSpentCompressing = ExpMovingAverage.decayBy10();
+
+        long lastCompressionStartTime;
+        long lastCompressionDuration;
+
+        /**
+         * Computes the new compression level to use for the next chunk, based on the load.
+         */
+        public void adjustCompressionLevel(long currentTime)
+        {
+            // The more write "pressure", the faster we want to go, so the lower the desired compression level.
+            double pressure = getWritePressure();
+            assert pressure >= 0.0 && pressure <= 1.0 : "pressure (" + pressure + ") out of valid range [0.0, 1.0]";
+
+            // Use minCompressionLevel when pressure = 1.0, maxCompressionLevel when pressure = 0.0
+            int pressurePoints = (int) (pressure * (params.maxCompressionLevel - params.minCompressionLevel));
+            int compressionLevelTarget = params.maxCompressionLevel - pressurePoints;
+
+            // We use wall clock time and not CPU time, because we also want to include time spent by I/O.
+            // If we're bottlenecked by writing the data to disk, this indicator should be low.
+            double relativeTimeSpentCompressing = (double) (1 + lastCompressionDuration) / (1 + currentTime - lastCompressionStartTime);
+
+            // Some smoothing is needed to avoid changing level too fast due to performance hiccups
+            this.relativeTimeSpentCompressing.update(relativeTimeSpentCompressing);
+
+            // If we're under pressure to write data fast, we need to decrease compression level.
+            // But we do that only if we're really spending significant amount of time doing compression.
+            if (compressionLevelTarget < currentCompressionLevel && this.relativeTimeSpentCompressing.get() > 0.1)
+                currentCompressionLevel--;
+            // If we're not under heavy write pressure, or we're spending very little time compressing data,
+            // we can increase the compression level and get some space savings at a low performance overhead:
+            else if (compressionLevelTarget > currentCompressionLevel || this.relativeTimeSpentCompressing.get() < 0.02)
+                currentCompressionLevel++;
+
+            currentCompressionLevel = clampCompressionLevel(currentCompressionLevel);
+            compressCtx.setLevel(zstdCompressionLevels[currentCompressionLevel]);
+        }
+
+        /**
+         * Must be called after compressing a chunk,
+         * so we can measure how much time we spend in compressing vs time spent not-compressing.
+         */
+        public void recordCompressionDuration(long startTime, long endTime)
+        {
+            this.lastCompressionDuration = endTime - startTime;
+            this.lastCompressionStartTime = startTime;
+        }
+
+        @VisibleForTesting
+        double getRelativeTimeSpentCompressing()
+        {
+            return this.relativeTimeSpentCompressing.get();
+        }
+    }
+
+    /**
+     * @param params user-provided configuration such as min/max compression level range
+     * @param writePressureSupplier returns a non-negative score determining the write load on the system which
+     *                               is used to control the desired compression level. Influences the compression
+     *                               level linearly: an inceease of pressure by 1 point causes the target
+     *                               compression level to be decreased by 1 point. Zero will select the
+     *                               maximum allowed compression level.
+     */
+    @VisibleForTesting
+    AdaptiveCompressor(Params params, Supplier<Double> writePressureSupplier)
+    {
+        this.params = params;
+        this.state = new ThreadLocal<>();
+        this.writePressureSupplier = writePressureSupplier;
+    }
+    
+    @Override
+    public int initialCompressedBufferLength(int chunkLength)
+    {
+        return (int) Zstd.compressBound(chunkLength);
+    }
+
+
+    @Override
+    public void compress(ByteBuffer input, ByteBuffer output) throws IOException
+    {
+        try
+        {
+            State state = getThreadLocalState();
+            long startTime = Clock.Global.nanoTime();
+            state.adjustCompressionLevel(startTime);
+            long inputSize = input.remaining();
+            state.compressCtx.compress(output, input);
+            long endTime = Clock.Global.nanoTime();
+            state.recordCompressionDuration(startTime, endTime);
+
+            Metrics m = metrics.get(params.use);
+            m.updateFrom(state);
+            m.compressionRate.mark(inputSize);
+        }
+        catch (Exception e)
+        {
+            throw new IOException("Compression failed", e);
+        }
+
+    }
+
+    @Override
+    public int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset) throws IOException
+    {
+        State state = getThreadLocalState();
+        long dsz = state.decompressCtx.decompressByteArray(output, outputOffset, output.length - outputOffset,
+                                                           input, inputOffset, inputLength);
+
+        if (Zstd.isError(dsz))
+            throw new IOException(String.format("Decompression failed due to %s", Zstd.getErrorName(dsz)));
+
+        metrics.get(params.use).decompressionRate.mark(dsz);
+        return (int) dsz;
+    }
+
+    @Override
+    public void uncompress(ByteBuffer input, ByteBuffer output) throws IOException
+    {
+        try
+        {
+            State state = getThreadLocalState();
+            long dsz = state.decompressCtx.decompress(output, input);
+            metrics.get(params.use).decompressionRate.mark(dsz);
+        } catch (Exception e)
+        {
+            throw new IOException("Decompression failed", e);
+        }
+    }
+
+    @Override
+    public BufferType preferredBufferType()
+    {
+        return BufferType.OFF_HEAP;
+    }
+
+    @Override
+    public Set<Uses> recommendedUses()
+    {
+        return params.minCompressionLevel <= DEFAULT_MIN_FAST_COMPRESSION_LEVEL
+               ? EnumSet.of(Uses.GENERAL, Uses.FAST_COMPRESSION)
+               : EnumSet.of(params.use);
+    }
+
+    @Override
+    public ICompressor forUse(Uses use)
+    {
+        if (use == params.use)
+            return this;
+
+        switch (use)
+        {
+            case GENERAL:
+                return createForCompaction(params.minCompressionLevel, params.maxCompressionLevel, params.maxCompactionQueueLength);
+            case FAST_COMPRESSION:
+                return createForFlush(params.minCompressionLevel, params.maxCompressionLevel);
+        }
+
+        return null;
+    }
+
+    @Override
+    public boolean supports(BufferType bufferType)
+    {
+        return bufferType == BufferType.OFF_HEAP;
+    }
+
+    @Override
+    public Set<String> supportedOptions()
+    {
+        return Set.of("max_compression_level", "min_compression_level");
+    }
+
+    private static int getMinCompressionLevel(Uses mode, Map<String, String> options)
+    {
+        int defaultValue = mode == Uses.FAST_COMPRESSION ? DEFAULT_MIN_FAST_COMPRESSION_LEVEL : DEFAULT_MIN_GENERAL_COMPRESSION_LEVEL;
+        return getIntOption(options, MIN_COMPRESSION_LEVEL_OPTION_NAME, defaultValue);
+    }
+
+    private static int getMaxCompressionLevel(Uses mode, Map<String, String> options)
+    {
+        var defaultValue = mode == Uses.FAST_COMPRESSION ? DEFAULT_MAX_FAST_COMPRESSION_LEVEL : DEFAULT_MAX_GENERAL_COMPRESSION_LEVEL;
+        return getIntOption(options, MAX_COMPRESSION_LEVEL_OPTION_NAME, defaultValue);
+    }
+
+    private static int getMaxCompactionQueueLength(Map<String, String> options)
+    {
+        return getIntOption(options, MAX_COMPACTION_QUEUE_LENGTH_OPTION_NAME, DEFAULT_MAX_COMPACTION_QUEUE_LENGTH);
+    }
+
+    private static int getIntOption(Map<String, String> options, String key, int defaultValue)
+    {
+        if (options == null)
+            return defaultValue;
+
+        String val = options.get(key);
+        if (val == null)
+            return defaultValue;
+
+        return Integer.parseInt(val);
+    }
+
+    private double getWritePressure()
+    {
+        return writePressureSupplier.get();
+    }
+
+    private static double getFlushPressure()
+    {
+        var memoryPool = AbstractAllocatorMemtable.MEMORY_POOL;
+        var usedRatio = Math.max(memoryPool.onHeap.usedRatio(), memoryPool.offHeap.usedRatio());
+        var cleanupThreshold = DatabaseDescriptor.getMemtableCleanupThreshold();
+        // we max out the pressure when we're halfway between the cleanupThreshold and max memory
+        // so we still have some memory left while compression already working at max speed;
+        // setting the compressor to maximum speed when we exhausted all memory would be too late
+        return Math.min(1.0, Math.max(0.0, 2 * (usedRatio - cleanupThreshold)) / (1.0 - cleanupThreshold));
+    }
+
+    private static double getCompactionPressure(int maxCompactionQueueLength)
+    {
+        CompactionManager compactionManager = CompactionManager.instance;
+        long rateLimit = DatabaseDescriptor.getCompactionThroughputMebibytesPerSecAsInt() * FileUtils.ONE_MIB;
+        if (rateLimit == 0)
+            rateLimit = Long.MAX_VALUE;
+        double actualRate = compactionManager.getMetrics().bytesCompactedThroughput.getOneMinuteRate();
+        // We don't want to speed up compression if we can keep up with the configured compression rate limit
+        // 0.0 if actualRate >= rateLimit
+        // 1.0 if actualRate <= 0.8 * rateLimit;
+        double rateLimitFactor = Math.min(1.0, Math.max(0.0, (rateLimit - actualRate) / (0.2 * rateLimit)));
+
+        long pendingCompactions = compactionManager.getPendingTasks();
+        long activeCompactions = compactionManager.getActiveCompactions();
+        long queuedCompactions = pendingCompactions - activeCompactions;
+        double compactionQueuePressure = Math.min(1.0, (double) queuedCompactions / (maxCompactionQueueLength * DatabaseDescriptor.getConcurrentCompactors()));
+        return compactionQueuePressure * rateLimitFactor;
+    }
+
+    private int clampCompressionLevel(long compressionLevel)
+    {
+        return (int) Math.min(params.maxCompressionLevel, Math.max(params.minCompressionLevel, compressionLevel));
+    }
+
+    @VisibleForTesting
+    State getThreadLocalState()
+    {
+        State state = this.state.get();
+        if (state == null)
+        {
+            state = new State();
+            state.currentCompressionLevel = params.maxCompressionLevel;
+            state.lastCompressionDuration = 0;
+            this.state.set(state);
+        }
+        return state;
+    }
+
+    static class Metrics
+    {
+        private final Counter[] compressionLevelHistogram;  // separate counters for each compression level
+        private final Histogram relativeTimeSpentCompressing; // in % (i.e. multiplied by 100 becaue Histogram can only keep integers)
+        private final Meter compressionRate;
+        private final Meter decompressionRate;
+
+
+        Metrics(Uses use)
+        {
+            MetricNameFactory factory = new DefaultNameFactory("AdaptiveCompression");
+
+            // cannot use Metrics.histogram for compression levels, because histograms do not handle negative numbers;
+            // also this histogram is small enough that storing all buckets is not a problem, but it gives
+            // much more information
+            compressionLevelHistogram = new Counter[MAX_COMPRESSION_LEVEL + 1];
+            for (int i = 0; i < compressionLevelHistogram.length; i++)
+            {
+                CassandraMetricsRegistry.MetricName metricName = factory.createMetricName(String.format("CompressionLevel_%s_%02d", use.name(), i));
+                compressionLevelHistogram[i] = CassandraMetricsRegistry.Metrics.counter(metricName);
+            }
+
+            relativeTimeSpentCompressing = CassandraMetricsRegistry.Metrics.histogram(factory.createMetricName("RelativeTimeSpentCompressing_" + use.name()), true);
+
+            compressionRate = CassandraMetricsRegistry.Metrics.meter(factory.createMetricName("CompressionRate_" + use.name()));
+            decompressionRate = CassandraMetricsRegistry.Metrics.meter(factory.createMetricName("DecompressionRate_" + use.name()));
+        }
+
+        void updateFrom(State state)
+        {
+            compressionLevelHistogram[state.currentCompressionLevel].inc();
+            relativeTimeSpentCompressing.update((int)(state.getRelativeTimeSpentCompressing() * 100.0));
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
index 4a7f8161cc0d..7d8e859093c7 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
@@ -20,10 +20,16 @@
 import java.io.DataOutputStream;
 import java.io.EOFException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
 import java.util.Optional;
 import java.util.zip.CRC32;
+import java.util.zip.CheckedInputStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.FSWriteError;
@@ -31,6 +37,7 @@
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.util.ChecksumWriter;
 import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.SequentialWriter;
@@ -42,6 +49,8 @@
 
 public class CompressedSequentialWriter extends SequentialWriter
 {
+    protected static final Logger logger = LoggerFactory.getLogger(CompressedSequentialWriter.class);
+
     private final ChecksumWriter crcMetadata;
 
     // holds offset in the file where current chunk should be written
@@ -58,8 +67,6 @@ public class CompressedSequentialWriter extends SequentialWriter
     // holds a number of already written chunks
     private int chunkCount = 0;
 
-    private long uncompressedSize = 0, compressedSize = 0;
-
     private final MetadataCollector sstableMetadataCollector;
 
     private final ByteBuffer crcCheckBuffer = ByteBuffer.allocate(4);
@@ -67,6 +74,12 @@ public class CompressedSequentialWriter extends SequentialWriter
 
     private final int maxCompressedLength;
 
+    /**
+     * When corruption is found, the file writer is reset to previous data point but we can't reset the CRC checksum.
+     * So we have to recompute digest value.
+     */
+    private boolean recomputeChecksum = false;
+
     /**
      * Create CompressedSequentialWriter without digest file.
      *
@@ -85,12 +98,12 @@ public CompressedSequentialWriter(File file,
                                       MetadataCollector sstableMetadataCollector)
     {
         super(file, SequentialWriterOption.newBuilder()
-                            .bufferSize(option.bufferSize())
-                            .bufferType(option.bufferType())
-                            .bufferSize(parameters.chunkLength())
-                            .bufferType(parameters.getSstableCompressor().preferredBufferType())
-                            .finishOnClose(option.finishOnClose())
-                            .build());
+                                          .bufferSize(option.bufferSize())
+                                          .bufferType(option.bufferType())
+                                          .bufferSize(parameters.chunkLength())
+                                          .bufferType(parameters.getSstableCompressor().preferredBufferType())
+                                          .finishOnClose(option.finishOnClose())
+                                          .build());
         this.compressor = parameters.getSstableCompressor();
         this.digestFile = Optional.ofNullable(digestFile);
 
@@ -115,7 +128,7 @@ public long getOnDiskFilePointer()
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, getPath());
+            throw new FSReadError(e, getFile());
         }
     }
 
@@ -141,6 +154,15 @@ protected void flushData()
     {
         seekToChunkStart(); // why is this necessary? seems like it should always be at chunk start in normal operation
 
+        if (buffer.limit() == 0)
+        {
+            // nothing to compress
+            if (runPostFlush != null)
+                runPostFlush.accept(0);
+
+            return;
+        }
+
         try
         {
             // compressing data with buffer re-use
@@ -155,7 +177,6 @@ protected void flushData()
 
         int uncompressedLength = buffer.position();
         int compressedLength = compressed.position();
-        uncompressedSize += uncompressedLength;
         ByteBuffer toWrite = compressed;
         if (compressedLength >= maxCompressedLength)
         {
@@ -175,7 +196,6 @@ protected void flushData()
                 compressedLength = maxCompressedLength;
             }
         }
-        compressedSize += compressedLength;
 
         try
         {
@@ -190,11 +210,11 @@ protected void flushData()
             // write corresponding checksum
             toWrite.rewind();
             crcMetadata.appendDirect(toWrite, true);
-            lastFlushOffset = uncompressedSize;
+            lastFlushOffset += uncompressedLength;
         }
         catch (IOException e)
         {
-            throw new FSWriteError(e, getPath());
+            throw new FSWriteError(e, getFile());
         }
         if (toWrite == buffer)
             buffer.position(uncompressedLength);
@@ -208,7 +228,7 @@ protected void flushData()
     public CompressionMetadata open(long overrideLength)
     {
         if (overrideLength <= 0)
-            overrideLength = uncompressedSize;
+            overrideLength = lastFlushOffset;
         return metadataWriter.open(overrideLength, chunkOffset);
     }
 
@@ -222,6 +242,21 @@ public DataPosition mark()
 
     @Override
     public synchronized void resetAndTruncate(DataPosition mark)
+    {
+        try
+        {
+            doResetAndTruncate(mark);
+        }
+        catch (Throwable t)
+        {
+            CompressedFileWriterMark realMark = mark instanceof CompressedFileWriterMark ? (CompressedFileWriterMark) mark : null;
+            logger.error("Failed to reset and truncate {} at chunk offset {} because of {}", file.name(),
+                         realMark == null ? -1 : realMark.chunkOffset, t.getMessage());
+            throw t;
+        }
+    }
+
+    private synchronized void doResetAndTruncate(DataPosition mark)
     {
         assert mark instanceof CompressedFileWriterMark;
 
@@ -250,12 +285,12 @@ public synchronized void resetAndTruncate(DataPosition mark)
             compressed = compressor.preferredBufferType().allocate(chunkSize);
         }
 
-        try
+        try(FileChannel readChannel = StorageProvider.instance.writeTimeReadFileChannelFor(getFile()))
         {
             compressed.clear();
             compressed.limit(chunkSize);
-            fchannel.position(chunkOffset);
-            fchannel.read(compressed);
+            readChannel.position(chunkOffset);
+            readChannel.read(compressed);
 
             try
             {
@@ -269,7 +304,7 @@ public synchronized void resetAndTruncate(DataPosition mark)
             }
             catch (IOException e)
             {
-                throw new CorruptBlockException(getPath(), chunkOffset, chunkSize, e);
+                throw new CorruptBlockException(getFile().toString(), chunkOffset, chunkSize, e);
             }
 
             CRC32 checksum = new CRC32();
@@ -277,22 +312,24 @@ public synchronized void resetAndTruncate(DataPosition mark)
             checksum.update(compressed);
 
             crcCheckBuffer.clear();
-            fchannel.read(crcCheckBuffer);
+            readChannel.read(crcCheckBuffer);
             crcCheckBuffer.flip();
-            if (crcCheckBuffer.getInt() != (int) checksum.getValue())
-                throw new CorruptBlockException(getPath(), chunkOffset, chunkSize);
+            int storedChecksum = crcCheckBuffer.getInt();
+            int computedChecksum = (int) checksum.getValue();
+            if (storedChecksum != computedChecksum)
+                throw new CorruptBlockException(getFile().toString(), chunkOffset, chunkSize, storedChecksum, computedChecksum);
         }
         catch (CorruptBlockException e)
         {
-            throw new CorruptSSTableException(e, getPath());
+            throw new CorruptSSTableException(e, getFile());
         }
         catch (EOFException e)
         {
-            throw new CorruptSSTableException(new CorruptBlockException(getPath(), chunkOffset, chunkSize), getPath());
+            throw new CorruptSSTableException(new CorruptBlockException(getFile().toString(), chunkOffset, chunkSize), getFile());
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, getPath());
+            throw new FSReadError(e, getFile());
         }
 
         // Mark as dirty so we can guarantee the newly buffered bytes won't be lost on a rebuffer
@@ -301,9 +338,13 @@ public synchronized void resetAndTruncate(DataPosition mark)
         bufferOffset = truncateTarget - buffer.position();
         chunkCount = realMark.nextChunkIndex - 1;
 
-        // truncate data and index file
+        // truncate data and index file. Unfortunately we can't reset and truncate CRC value, we have to recompute
+        // the CRC value otherwise it won't match the actual file checksum
+        recomputeChecksum = true;
         truncate(chunkOffset, bufferOffset);
         metadataWriter.resetAndTruncate(realMark.nextChunkIndex - 1);
+
+        logger.info("reset and truncated {} to {}", file, chunkOffset);
     }
 
     private void truncate(long toFileSize, long toBufferOffset)
@@ -315,7 +356,7 @@ private void truncate(long toFileSize, long toBufferOffset)
         }
         catch (IOException e)
         {
-            throw new FSWriteError(e, getPath());
+            throw new FSWriteError(e, getFile());
         }
     }
 
@@ -332,7 +373,7 @@ private void seekToChunkStart()
             }
             catch (IOException e)
             {
-                throw new FSReadError(e, getPath());
+                throw new FSReadError(e, getFile());
             }
         }
     }
@@ -390,8 +431,9 @@ protected Throwable doAbort(Throwable accumulate)
         protected void doPrepare()
         {
             syncInternal();
-            digestFile.ifPresent(crcMetadata::writeFullChecksum);
-            sstableMetadataCollector.addCompressionRatio(compressedSize, uncompressedSize);
+            maybeWriteChecksum();
+
+            sstableMetadataCollector.addCompressionRatio(chunkOffset, lastFlushOffset);
             metadataWriter.finalizeLength(current(), chunkCount).prepareToCommit();
         }
 
@@ -410,6 +452,40 @@ protected Throwable doPreCleanup(Throwable accumulate)
         }
     }
 
+    private void maybeWriteChecksum()
+    {
+        if (digestFile.isEmpty())
+            return;
+
+        File digest = digestFile.get();
+        if (recomputeChecksum)
+        {
+            logger.info("Rescanning data file to populate digest into {} because file writer has been reset and truncated", digest);
+            try (FileChannel fileChannel = StorageProvider.instance.writeTimeReadFileChannelFor(file);
+                 InputStream stream = Channels.newInputStream(fileChannel))
+            {
+                CRC32 checksum = new CRC32();
+                try (CheckedInputStream checkedInputStream = new CheckedInputStream(stream, checksum))
+                {
+                    byte[] chunk = new byte[64 * 1024];
+                    while (checkedInputStream.read(chunk) >= 0) {}
+
+                    long digestValue = checkedInputStream.getChecksum().getValue();
+                    ChecksumWriter.writeFullChecksum(digest, digestValue);
+                }
+            }
+            catch (IOException e)
+            {
+                throw new FSWriteError(e, digest);
+            }
+            logger.info("Successfully recomputed checksum for {}", digest);
+        }
+        else
+        {
+            crcMetadata.writeFullChecksum(digest);
+        }
+    }
+
     @Override
     protected SequentialWriter.TransactionalProxy txnProxy()
     {
diff --git a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
index 96b4ce841825..b1e6552f3fea 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
@@ -27,8 +27,10 @@
 import java.util.Map;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.primitives.Longs;
 
 import org.apache.cassandra.db.TypeSizes;
@@ -45,7 +47,9 @@
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
 import org.apache.cassandra.io.util.Memory;
 import org.apache.cassandra.io.util.SafeMemory;
+import org.apache.cassandra.io.util.SliceDescriptor;
 import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.concurrent.Transactional;
 import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable;
@@ -56,22 +60,79 @@
  */
 public class CompressionMetadata extends WrappedSharedCloseable
 {
-    // dataLength can represent either the true length of the file
-    // or some shorter value, in the case we want to impose a shorter limit on readers
-    // (when early opening, we want to ensure readers cannot read past fully written sections)
+    /**
+     * This class extends Memory.LongArray in order to record the memory usage.
+     */
+    public static class ChunkOffsetMemory extends Memory.LongArray
+    {
+
+        public ChunkOffsetMemory(long size)
+        {
+            super(size);
+        }
+
+        public ChunkOffsetMemory(SafeMemory memory, long cnt)
+        {
+            super(memory, cnt);
+        }
+
+        @Override
+        public void close()
+        {
+            NATIVE_MEMORY_USAGE.addAndGet(-memoryUsed());
+            super.close();
+        }
+    }
+
+    private static final AtomicLong NATIVE_MEMORY_USAGE = new AtomicLong(0);
+    /**
+     * DataLength can represent either the true length of the file
+     * or some shorter value, in the case we want to impose a shorter limit on readers
+     * (when early opening, we want to ensure readers cannot read past fully written sections).
+     * If zero copy metadata is present, this is the uncompressed length of the partial data file.
+     */
     public final long dataLength;
+
+    /**
+     * Length of the compressed file in bytes. This refers to the partial file length if zero copy metadata is present.
+     */
     public final long compressedFileLength;
-    private final Memory chunkOffsets;
-    private final long chunkOffsetsSize;
+
+    /**
+     * Offsets of consecutive chunks in the (compressed) data file. The length of this array is equal to the number of
+     * chunks. Each item is of Long type, thus 8 bytes long. Note that even if we deal with a partial data file (zero
+     * copy metadata is present), we store offsets of all chunks for the original (compressed) data file.
+     */
+    private final ChunkOffsetMemory chunkOffsets;
     public final File chunksIndexFile;
     public final CompressionParams parameters;
 
-    @VisibleForTesting
+    /**
+     * The length of the chunk in bits. The chunk length must be a power of 2, so this is the number of trailing zeros
+     * in the chunk length.
+     */
+    private final int chunkLengthBits;
+
+    /**
+     * If we don't want to load the all offsets into memory, for example when we deal with a slice, this is the index of
+     * the first offset we loaded.
+     */
+    private final int startChunkIndex;
+
     public static CompressionMetadata open(File chunksIndexFile, long compressedLength, boolean hasMaxCompressedSize)
     {
+        return open(chunksIndexFile, compressedLength, hasMaxCompressedSize, SliceDescriptor.NONE);
+    }
+
+    @VisibleForTesting
+    public static CompressionMetadata open(File chunksIndexFile, long compressedLength, boolean hasMaxCompressedSize, SliceDescriptor sliceDescriptor)
+    {
+        long uncompressedOffset = sliceDescriptor.exists() ? sliceDescriptor.sliceStart : 0;
+        long uncompressedLength = sliceDescriptor.exists() ? sliceDescriptor.dataEnd - sliceDescriptor.sliceStart : -1;
+
         CompressionParams parameters;
         long dataLength;
-        Memory chunkOffsets;
+        ChunkOffsetMemory chunkOffsets;
 
         try (FileInputStreamPlus stream = chunksIndexFile.newInputStream())
         {
@@ -97,8 +158,25 @@ public static CompressionMetadata open(File chunksIndexFile, long compressedLeng
                 throw new RuntimeException("Cannot create CompressionParams for stored parameters", e);
             }
 
-            dataLength = stream.readLong();
-            chunkOffsets = readChunkOffsets(stream);
+            assert Integer.bitCount(chunkLength) == 1;
+            int chunkLengthBits = Integer.numberOfTrailingZeros(chunkLength);
+            long readDataLength = stream.readLong();
+            dataLength = uncompressedLength >= 0 ? uncompressedLength : readDataLength;
+
+            int startChunkIndex = Math.toIntExact(uncompressedOffset >> chunkLengthBits);
+            assert uncompressedOffset == (long) startChunkIndex << chunkLengthBits;
+
+            int endChunkIndex = Math.toIntExact((uncompressedOffset + dataLength - 1) >> chunkLengthBits) + 1;
+
+            Pair<ChunkOffsetMemory, Long> offsetsAndLimit = readChunkOffsets(stream, startChunkIndex, endChunkIndex, compressedLength);
+            chunkOffsets = offsetsAndLimit.left;
+            // We adjust the compressed file length to store the position after the last chunk just to be able to
+            // calculate the offset of the chunk next to the last one (in order to calculate the length of the last chunk).
+            // Obvously, we could use the compressed file length for that purpose but unfortunately, sometimes there is
+            // an empty chunk added to the end of the file thus we cannot rely on the file length.
+            long compressedFileLength = offsetsAndLimit.right;
+
+            return new CompressionMetadata(chunksIndexFile, parameters, chunkOffsets, dataLength, compressedFileLength, chunkLengthBits, startChunkIndex);
         }
         catch (FileNotFoundException | NoSuchFileException e)
         {
@@ -108,18 +186,17 @@ public static CompressionMetadata open(File chunksIndexFile, long compressedLeng
         {
             throw new CorruptSSTableException(e, chunksIndexFile);
         }
-
-        return new CompressionMetadata(chunksIndexFile, parameters, chunkOffsets, chunkOffsets.size(), dataLength, compressedLength);
     }
 
     // do not call this constructor directly, unless used in testing
     @VisibleForTesting
     public CompressionMetadata(File chunksIndexFile,
                                CompressionParams parameters,
-                               Memory chunkOffsets,
-                               long chunkOffsetsSize,
+                               ChunkOffsetMemory chunkOffsets,
                                long dataLength,
-                               long compressedFileLength)
+                               long compressedFileLength,
+                               int chunkLengthBits,
+                               int startChunkIndex)
     {
         super(chunkOffsets);
         this.chunksIndexFile = chunksIndexFile;
@@ -127,7 +204,8 @@ public CompressionMetadata(File chunksIndexFile,
         this.dataLength = dataLength;
         this.compressedFileLength = compressedFileLength;
         this.chunkOffsets = chunkOffsets;
-        this.chunkOffsetsSize = chunkOffsetsSize;
+        this.chunkLengthBits = chunkLengthBits;
+        this.startChunkIndex = startChunkIndex;
     }
 
     private CompressionMetadata(CompressionMetadata copy)
@@ -138,7 +216,13 @@ private CompressionMetadata(CompressionMetadata copy)
         this.dataLength = copy.dataLength;
         this.compressedFileLength = copy.compressedFileLength;
         this.chunkOffsets = copy.chunkOffsets;
-        this.chunkOffsetsSize = copy.chunkOffsetsSize;
+        this.chunkLengthBits = copy.chunkLengthBits;
+        this.startChunkIndex = copy.startChunkIndex;
+    }
+
+    public static long nativeMemoryAllocated()
+    {
+        return NATIVE_MEMORY_USAGE.get();
     }
 
     public ICompressor compressor()
@@ -162,14 +246,14 @@ public int maxCompressedLength()
      */
     public long offHeapSize()
     {
-        return chunkOffsets.size();
+        return chunkOffsets.memory.size();
     }
 
     @Override
     public void addTo(Ref.IdentityCollection identities)
     {
         super.addTo(identities);
-        identities.add(chunkOffsets);
+        identities.add(chunkOffsets.memory);
     }
 
     @Override
@@ -179,14 +263,19 @@ public CompressionMetadata sharedCopy()
     }
 
     /**
-     * Read offsets of the individual chunks from the given input.
+     * Reads offsets of the individual chunks from the given input, filtering out non-relevant offsets (outside the
+     * specified range).
      *
-     * @param input Source of the data.
+     * @param input Source of the data
+     * @param startIndex Index of the first chunk to read, inclusive
+     * @param endIndex Index of the last chunk to read, exclusive
+     * @param compressedFileLength compressed file length
      *
-     * @return collection of the chunk offsets.
+     * @return A pair of chunk offsets array and the offset next to the last read chunk
      */
-    private static Memory readChunkOffsets(FileInputStreamPlus input)
+    private static Pair<ChunkOffsetMemory, Long> readChunkOffsets(FileInputStreamPlus input, int startIndex, int endIndex, long compressedFileLength)
     {
+        final ChunkOffsetMemory offsets;
         final int chunkCount;
         try
         {
@@ -199,29 +288,41 @@ private static Memory readChunkOffsets(FileInputStreamPlus input)
             throw new FSReadError(e, input.file);
         }
 
-        Memory offsets = Memory.allocate(chunkCount * 8L);
-        int i = 0;
+        Preconditions.checkState(startIndex < chunkCount, "The start index %s has to be < chunk count %s", startIndex, chunkCount);
+        Preconditions.checkState(endIndex <= chunkCount, "The end index %s has to be <= chunk count %s", endIndex, chunkCount);
+        Preconditions.checkState(startIndex <= endIndex, "The start index %s has to be < end index %s", startIndex, endIndex);
+
+        int chunksToRead = endIndex - startIndex;
+
+        if (chunksToRead == 0)
+            return Pair.create(new ChunkOffsetMemory(0), 0L);
+
+        offsets = new ChunkOffsetMemory(chunksToRead);
+        long i = 0;
         try
         {
-
-            for (i = 0; i < chunkCount; i++)
+            input.skipBytes(startIndex * 8);
+            long lastOffset;
+            for (i = 0; i < chunksToRead; i++)
             {
-                offsets.setLong(i * 8L, input.readLong());
+                lastOffset = input.readLong();
+                offsets.set(i, lastOffset);
             }
 
-            return offsets;
+            lastOffset = endIndex < chunkCount ? input.readLong() - offsets.get(0) : compressedFileLength;
+            NATIVE_MEMORY_USAGE.addAndGet(offsets.memoryUsed());
+            return Pair.create(offsets, lastOffset);
+        }
+        catch (EOFException e)
+        {
+            offsets.close();
+            String msg = String.format("Corrupted Index File %s: read %d but expected at least %d chunks.",
+                                       input, i, chunksToRead);
+            throw new CorruptSSTableException(new IOException(msg, e), input.file);
         }
         catch (IOException e)
         {
-            if (offsets != null)
-                offsets.close();
-
-            if (e instanceof EOFException)
-            {
-                String msg = String.format("Corrupted Index File %s: read %d but expected %d chunks.",
-                                           input.file.path(), i, chunkCount);
-                throw new CorruptSSTableException(new IOException(msg, e), input.file);
-            }
+            offsets.close();
             throw new FSReadError(e, input.file);
         }
     }
@@ -229,46 +330,74 @@ private static Memory readChunkOffsets(FileInputStreamPlus input)
     /**
      * Get a chunk of compressed data (offset, length) corresponding to given position
      *
-     * @param position Position in the file.
-     * @return pair of chunk offset and length.
+     * @param uncompressedDataPosition Position in the uncompressed data. If we deal with a slice, this is the position
+     *                                 in the original uncompressed data.
+     * @return A pair of chunk offset and length. If we deal with a slice, the chunk offset refers to the position in
+     * the compressed slice.
      */
-    public Chunk chunkFor(long position)
+    public Chunk chunkFor(long uncompressedDataPosition)
     {
-        // position of the chunk
-        long idx = 8 * (position / parameters.chunkLength());
+        int chunkIdx = chunkIndex(uncompressedDataPosition);
+        return chunk(chunkIdx);
+    }
 
-        if (idx >= chunkOffsetsSize)
-            throw new CorruptSSTableException(new EOFException(), chunksIndexFile);
+    private Chunk chunk(long chunkOffset, long nextChunkOffset)
+    {
+        return new Chunk(chunkOffset, Math.toIntExact(nextChunkOffset - chunkOffset - 4)); // "4" bytes reserved for checksum
+    }
 
-        if (idx < 0)
-            throw new CorruptSSTableException(new IllegalArgumentException(String.format("Invalid negative chunk index %d with position %d", idx, position)),
-                                              chunksIndexFile);
+    private Chunk chunk(int chunkIdx)
+    {
+        long chunkOffset = chunkOffset(chunkIdx);
+        long nextChunkOffset = nextChunkOffset(chunkIdx);
+        return chunk(chunkOffset, nextChunkOffset);
+    }
 
-        long chunkOffset = chunkOffsets.getLong(idx);
-        long nextChunkOffset = (idx + 8 == chunkOffsetsSize)
-                                ? compressedFileLength
-                                : chunkOffsets.getLong(idx + 8);
+    private long nextChunkOffset(int chunkIdx)
+    {
+        if (chunkIdx == chunkOffsets.size() - 1)
+            return compressedFileLength + chunkOffsets.get(0);
+        return chunkOffset(chunkIdx + 1);
+    }
+
+    private long chunkOffset(int chunkIdx)
+    {
+        if (chunkIdx >= chunkOffsets.size())
+            throw new CorruptSSTableException(new EOFException(String.format("Chunk %d out of bounds: %d", chunkIdx, chunkOffsets.size())), chunksIndexFile);
+
+        return chunkOffsets.get(chunkIdx);
+    }
 
-        return new Chunk(chunkOffset, (int) (nextChunkOffset - chunkOffset - 4)); // "4" bytes reserved for checksum
+    private int chunkIndex(long uncompressedDataPosition)
+    {
+        return Math.toIntExact(uncompressedDataPosition >> chunkLengthBits) - startChunkIndex;
     }
 
+    /**
+     * Searches for the chunk with the given offset and returns the offset of uncompressed data for the found chunk.
+     * @param chunkOffset exact chunk offset to search for; if we deal with a slice this is a chunk offset
+     *                    in the original compressed file
+     * @return offset of uncompressed data for the found chunk; if we deal with a slice this is the offset
+     * in the original uncompressed data
+     * @throws IllegalArgumentException if no chunk with the given offset is found
+     */
     public long getDataOffsetForChunkOffset(long chunkOffset)
     {
         long l = 0;
-        long h = (chunkOffsetsSize >> 3) - 1;
+        long h = chunkOffsets.size() - 1;
         long idx, offset;
 
         while (l <= h)
         {
             idx = (l + h) >>> 1;
-            offset = chunkOffsets.getLong(idx << 3);
+            offset = chunkOffsets.get(idx);
 
             if (offset < chunkOffset)
                 l = idx + 1;
             else if (offset > chunkOffset)
                 h = idx - 1;
             else
-                return idx * parameters.chunkLength();
+                return (idx + startChunkIndex) << chunkLengthBits;
         }
 
         throw new IllegalArgumentException("No chunk with offset " + chunkOffset);
@@ -281,35 +410,28 @@ else if (offset > chunkOffset)
     public long getTotalSizeForSections(Collection<SSTableReader.PartitionPositionBounds> sections)
     {
         long size = 0;
-        long lastOffset = -1;
+        int lastIncludedChunkIdx = -1;
         for (SSTableReader.PartitionPositionBounds section : sections)
         {
-            int startIndex = (int) (section.lowerPosition / parameters.chunkLength());
+            int sectionStartIdx = Math.max(chunkIndex(section.lowerPosition), lastIncludedChunkIdx + 1);
+            int sectionEndIdx = chunkIndex(section.upperPosition - 1); // we need to include the last byte of the seciont but not the upper position (which is excludded)
 
-            int endIndex = (int) (section.upperPosition / parameters.chunkLength());
-            if (section.upperPosition % parameters.chunkLength() == 0)
-                endIndex--;
-
-            for (int i = startIndex; i <= endIndex; i++)
+            for (int idx = sectionStartIdx; idx <= sectionEndIdx; idx++)
             {
-                long offset = i * 8L;
-                long chunkOffset = chunkOffsets.getLong(offset);
-                if (chunkOffset > lastOffset)
-                {
-                    lastOffset = chunkOffset;
-                    long nextChunkOffset = offset + 8 == chunkOffsetsSize
-                                                   ? compressedFileLength
-                                                   : chunkOffsets.getLong(offset + 8);
-                    size += (nextChunkOffset - chunkOffset);
-                }
+                long chunkOffset = chunkOffset(idx);
+                long nextChunkOffset = nextChunkOffset(idx);
+                size += nextChunkOffset - chunkOffset;
             }
+            lastIncludedChunkIdx = sectionEndIdx;
         }
         return size;
     }
 
     /**
-     * @param sections Collection of sections in uncompressed file
-     * @return Array of chunks which corresponds to given sections of uncompressed file, sorted by chunk offset
+     * @param sections Collection of sections in uncompressed data. If we deal with a slice, the sections refer to the
+     *                 positions in the original uncompressed data.
+     * @return Array of chunks which corresponds to given sections of uncompressed file, sorted by chunk offset.
+     * Note that if we deal with a slice, the chunk offsets refer to the positions in the compressed slice.
      */
     public Chunk[] getChunksForSections(Collection<SSTableReader.PartitionPositionBounds> sections)
     {
@@ -318,21 +440,11 @@ public Chunk[] getChunksForSections(Collection<SSTableReader.PartitionPositionBo
 
         for (SSTableReader.PartitionPositionBounds section : sections)
         {
-            int startIndex = (int) (section.lowerPosition / parameters.chunkLength());
-
-            int endIndex = (int) (section.upperPosition / parameters.chunkLength());
-            if (section.upperPosition % parameters.chunkLength() == 0)
-                endIndex--;
+            int sectionStartIdx = chunkIndex(section.lowerPosition);
+            int sectionEndIdx = chunkIndex(section.upperPosition - 1); // we need to include the last byte of the seciont but not the upper position (which is excludded)
 
-            for (int i = startIndex; i <= endIndex; i++)
-            {
-                long offset = i * 8L;
-                long chunkOffset = chunkOffsets.getLong(offset);
-                long nextChunkOffset = offset + 8 == chunkOffsetsSize
-                                     ? compressedFileLength
-                                     : chunkOffsets.getLong(offset + 8);
-                offsets.add(new Chunk(chunkOffset, (int) (nextChunkOffset - chunkOffset - 4))); // "4" bytes reserved for checksum
-            }
+            for (int idx = sectionStartIdx; idx <= sectionEndIdx; idx++)
+                offsets.add(chunk(idx));
         }
 
         return offsets.toArray(new Chunk[offsets.size()]);
@@ -344,7 +456,7 @@ public static class Writer extends Transactional.AbstractTransactional implement
         private final CompressionParams parameters;
         private final File file;
         private int maxCount = 100;
-        private SafeMemory offsets = new SafeMemory(maxCount * 8L);
+        private SafeMemory offsets;
         private int count = 0;
 
         // provided by user when setDescriptor
@@ -354,6 +466,8 @@ private Writer(CompressionParams parameters, File file)
         {
             this.parameters = parameters;
             this.file = file;
+            offsets = new SafeMemory(maxCount * 8L);
+            NATIVE_MEMORY_USAGE.addAndGet(offsets.size());
         }
 
         public static Writer open(CompressionParams parameters, File file)
@@ -366,6 +480,7 @@ public void addOffset(long offset)
             if (count == maxCount)
             {
                 SafeMemory newOffsets = offsets.copy((maxCount *= 2L) * 8L);
+                NATIVE_MEMORY_USAGE.addAndGet(newOffsets.size() - offsets.size());
                 offsets.close();
                 offsets = newOffsets;
             }
@@ -416,6 +531,7 @@ public void doPrepare()
             {
                 SafeMemory tmp = offsets;
                 offsets = offsets.copy(count * 8L);
+                NATIVE_MEMORY_USAGE.addAndGet(offsets.size() - tmp.size());
                 tmp.free();
             }
 
@@ -441,6 +557,11 @@ public void doPrepare()
 
         public CompressionMetadata open(long dataLength, long compressedLength)
         {
+            // we are overcounting for a while, but it's not worth the effort to fix
+            // the problem is that the offsets object is allocated by the Writer and `closed` when
+            // the Writer is closed, but the offsets object is used by the CompressionMetadata object
+            // that will survive for all the time the SSTableReader is open
+            NATIVE_MEMORY_USAGE.addAndGet(offsets.size());
             SafeMemory tOffsets = this.offsets.sharedCopy();
 
             // calculate how many entries we need, if our dataLength is truncated
@@ -453,11 +574,11 @@ public CompressionMetadata open(long dataLength, long compressedLength)
             if (tCount < this.count)
                 compressedLength = tOffsets.getLong(tCount * 8L);
 
-            return new CompressionMetadata(file, parameters, tOffsets, tCount * 8L, dataLength, compressedLength);
+            return new CompressionMetadata(file, parameters, new ChunkOffsetMemory(tOffsets, tCount), dataLength, compressedLength, Integer.numberOfTrailingZeros(parameters.chunkLength()), 0);
         }
 
         /**
-         * Get a chunk offset by it's index.
+         * Get a chunk offset by its index.
          *
          * @param chunkIndex Index of the chunk.
          *
@@ -482,6 +603,7 @@ public void resetAndTruncate(int chunkIndex)
         @Override
         protected Throwable doPostCleanup(Throwable failed)
         {
+            NATIVE_MEMORY_USAGE.addAndGet(-offsets.size());
             return offsets.close(failed);
         }
 
@@ -510,7 +632,7 @@ public static class Chunk
 
         public Chunk(long offset, int length)
         {
-            assert(length > 0);
+            assert(length >= 0);
 
             this.offset = offset;
             this.length = length;
@@ -539,6 +661,14 @@ public String toString()
         {
             return String.format("Chunk<offset: %d, length: %d>", offset, length);
         }
+
+        /**
+         * @return the end of the chunk in the file, including the checksum
+         */
+        public long chunkEnd()
+        {
+            return offset + length + 4;
+        }
     }
 
     static class ChunkSerializer implements IVersionedSerializer<Chunk>
diff --git a/src/java/org/apache/cassandra/io/compress/CorruptBlockException.java b/src/java/org/apache/cassandra/io/compress/CorruptBlockException.java
index bcce6b9ca060..a41d8dfc05b7 100644
--- a/src/java/org/apache/cassandra/io/compress/CorruptBlockException.java
+++ b/src/java/org/apache/cassandra/io/compress/CorruptBlockException.java
@@ -40,4 +40,16 @@ public CorruptBlockException(String filePath, long offset, int length, Throwable
     {
         super(String.format("(%s): corruption detected, chunk at %d of length %d.", filePath, offset, length), cause);
     }
+
+    public CorruptBlockException(String filePath, CompressionMetadata.Chunk chunk, int storedChecksum, int calculatedChecksum)
+    {
+        this(filePath, chunk.offset, chunk.length, storedChecksum, calculatedChecksum);
+    }
+
+    public CorruptBlockException(String filePath, long offset, int length, int storedChecksum, int calculatedChecksum)
+    {
+        super(String.format("(%s): corruption detected, chunk at %d of length %d has mismatched checksums. Expected %d, but calculated %d",
+                            filePath, offset, length, storedChecksum, calculatedChecksum));
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/io/compress/ICompressor.java b/src/java/org/apache/cassandra/io/compress/ICompressor.java
index fd6a104431b3..bdc8fa79cf7b 100644
--- a/src/java/org/apache/cassandra/io/compress/ICompressor.java
+++ b/src/java/org/apache/cassandra/io/compress/ICompressor.java
@@ -22,6 +22,8 @@
 import java.util.EnumSet;
 import java.util.Set;
 
+import javax.annotation.Nullable;
+
 import com.google.common.collect.ImmutableSet;
 
 public interface ICompressor
@@ -83,4 +85,18 @@ default Set<Uses> recommendedUses()
     {
         return ImmutableSet.copyOf(EnumSet.allOf(Uses.class));
     }
+
+    /**
+     * Returns the compressor configured for a particular use.
+     * Allows creating a compressor implementation that can handle multiple uses but requires different configurations
+     * adapted to a particular use.
+     * <p>
+     * May return this object.
+     * May not modify this object.
+     * Should return null if the request cannot be satisfied.
+     */
+    default @Nullable ICompressor forUse(Uses use)
+    {
+        return recommendedUses().contains(use) ? this : null;
+    }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java
index e6fbad882e12..4124a03cc0b5 100644
--- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java
@@ -44,6 +44,7 @@
 import org.apache.cassandra.db.rows.UnfilteredSerializer;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.schema.TableMetadata;
@@ -140,7 +141,7 @@ protected AbstractSSTableIterator(SSTableReader sstable,
             catch (IOException e)
             {
                 sstable.markSuspect();
-                String filePath = file.getPath();
+                File filePath = file.getFile();
                 if (shouldCloseFile)
                 {
                     try
@@ -174,10 +175,10 @@ private Slice nextSlice()
      */
     protected abstract boolean hasMoreSlices();
 
-    private static Row readStaticRow(SSTableReader sstable,
-                                     FileDataInput file,
-                                     DeserializationHelper helper,
-                                     Columns statics) throws IOException
+    public static Row readStaticRow(SSTableReader sstable,
+                                    FileDataInput file,
+                                    DeserializationHelper helper,
+                                    Columns statics) throws IOException
     {
         if (!sstable.header.hasStatic())
             return Rows.EMPTY_STATIC_ROW;
@@ -316,13 +317,13 @@ public interface Reader extends Iterator<Unfiltered>, Closeable {
 
     public abstract class AbstractReader implements Reader
     {
-        private final boolean shouldCloseFile;
         public FileDataInput file;
 
         public UnfilteredDeserializer deserializer;
 
         // Records the currently open range tombstone (if any)
         public DeletionTime openMarker;
+        protected final boolean shouldCloseFile;
 
         protected AbstractReader(FileDataInput file, boolean shouldCloseFile)
         {
@@ -415,6 +416,7 @@ public Unfiltered next()
         public abstract void setForSlice(Slice slice) throws IOException;
 
         protected abstract boolean hasNextInternal() throws IOException;
+
         protected abstract Unfiltered nextInternal() throws IOException;
 
         @Override
diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
index 7e503829aacc..8e817fb80417 100644
--- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
@@ -355,6 +355,11 @@ public UserType getUDType(String dataType)
         return (UserType) JavaDriverUtils.driverType(userType);
     }
 
+    public TableMetadataRef getMetadata()
+    {
+        return writer.metadata;
+    }
+
     /**
      * Close this writer.
      * <p>
diff --git a/src/java/org/apache/cassandra/io/sstable/Component.java b/src/java/org/apache/cassandra/io/sstable/Component.java
index 0d89cf0b927d..ee6c6af9ec63 100644
--- a/src/java/org/apache/cassandra/io/sstable/Component.java
+++ b/src/java/org/apache/cassandra/io/sstable/Component.java
@@ -30,6 +30,10 @@
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components.Types;
 
+import org.apache.cassandra.io.storage.StorageProvider;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PathUtils;
+
 /**
  * SSTables are made up of multiple components in separate files. Components are
  * identified by a type and an id, but required unique components (such as the Data
@@ -37,7 +41,7 @@
  */
 public class Component
 {
-    public static final char separator = '-';
+    public static final char SEPARATOR = '-';
 
     /**
      * WARNING: Be careful while changing the names or string representation of the enum
@@ -163,10 +167,10 @@ public Component getSingleton()
             return Objects.requireNonNull(singleton);
         }
 
-        public Component createComponent(String repr)
+        public Component createComponent(String componentFileName)
         {
             Preconditions.checkArgument(singleton == null);
-            return new Component(this, repr);
+            return new Component(this, componentFileName);
         }
     }
 
@@ -179,7 +183,7 @@ private Component(Type type)
         this(type, type.repr);
     }
 
-    private Component(Type type, String name)
+    public Component(Type type, String name)
     {
         assert name != null : "Component name cannot be null";
 
@@ -223,10 +227,24 @@ public boolean isValidFor(Descriptor descriptor)
         return type.formatClass.isAssignableFrom(descriptor.version.format.getClass());
     }
 
+    public File getFile(String absolutePath)
+    {
+        File ret;
+        if (absolutePath.lastIndexOf(SEPARATOR) != (absolutePath.length() - 1))
+            ret = new File(PathUtils.getPath(absolutePath + SEPARATOR + name));
+        else
+            ret = new File(PathUtils.getPath(absolutePath + name));
+
+        return StorageProvider.instance.withOpenOptions(ret, this);
+    }
+
     @Override
     public String toString()
     {
-        return this.name();
+        return "Component{" +
+               "name='" + name + '\'' +
+               ", type=" + type +
+               '}';
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/io/sstable/CorruptSSTableException.java b/src/java/org/apache/cassandra/io/sstable/CorruptSSTableException.java
index 991a91d904f8..c2a02664ecba 100644
--- a/src/java/org/apache/cassandra/io/sstable/CorruptSSTableException.java
+++ b/src/java/org/apache/cassandra/io/sstable/CorruptSSTableException.java
@@ -17,21 +17,36 @@
  */
 package org.apache.cassandra.io.sstable;
 
+import java.nio.file.Path;
 
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.DseLegacy;
 
 public class CorruptSSTableException extends RuntimeException
 {
-    public final File path;
+    public final File file;
 
-    public CorruptSSTableException(Throwable cause, File path)
+    public CorruptSSTableException(Throwable cause, File file)
     {
-        super("Corrupted: " + path, cause);
-        this.path = path;
+        super("Corrupted: " + file, cause);
+        this.file = file;
     }
 
     public CorruptSSTableException(Throwable cause, String path)
     {
         this(cause, new File(path));
     }
+
+    protected CorruptSSTableException(String msg, Throwable cause, File file)
+    {
+        super(msg, cause);
+        this.file = file;
+    }
+
+    @DseLegacy
+    public CorruptSSTableException(Throwable cause, Path path)
+    {
+        this(cause, new File(path));
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/DefaultStorageHandler.java b/src/java/org/apache/cassandra/io/sstable/DefaultStorageHandler.java
new file mode 100644
index 000000000000..71c446d800e5
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/DefaultStorageHandler.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.service.StorageService;
+
+/**
+ * The default storage handler, used when sstables are stored on the local file system.
+ */
+public class DefaultStorageHandler extends StorageHandler
+{
+    public DefaultStorageHandler(SSTable.Owner owner, TableMetadataRef metadata, Directories directories, Tracker dataTracker)
+    {
+        super(owner, metadata, directories, dataTracker);
+    }
+
+    @Override
+    public boolean isReady()
+    {
+        return !StorageService.instance.isBootstrapMode();
+    }
+
+    @Override
+    public Collection<SSTableReader> loadInitialSSTables()
+    {
+        Directories.SSTableLister sstableFiles = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
+        Collection<SSTableReader> sstables = SSTableReader.openAll(owner, sstableFiles.list().entrySet(), metadata);
+        dataTracker.addInitialSSTablesWithoutUpdatingSize(sstables);
+        return sstables;
+    }
+
+    @Override
+    public Collection<SSTableReader> reloadSSTables(ReloadReason reason)
+    {
+        // no op for local storage
+        return Collections.emptySet();
+    }
+
+    @Override
+    public void unload()
+    {
+        // no op for local storage
+    }
+
+    @Override
+    public boolean enableAutoCompaction()
+    {
+        return true;
+    }
+
+    @Override
+    public void runWithReloadingDisabled(Runnable runnable)
+    {
+        // by default no sstables are loaded, so we just need to execute the runnable
+        runnable.run();
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
index bf58fff71761..b249d89d7b62 100644
--- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java
+++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
@@ -17,10 +17,13 @@
  */
 package org.apache.cassandra.io.sstable;
 
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 import java.util.regex.Matcher;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.regex.Pattern;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -37,11 +40,13 @@
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer;
 import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.DseLegacy;
 import org.apache.cassandra.utils.Pair;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.cassandra.io.sstable.Component.separator;
+import static org.apache.cassandra.io.sstable.Component.SEPARATOR;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
 /**
@@ -96,6 +101,8 @@ public class Descriptor
     private final int hashCode;
     private final String prefix;
     private final File baseFile;
+    private final String baseFileURI;
+    private final ConcurrentMap<Component, File> componentFileMap;
 
     /**
      * A descriptor that assumes CURRENT_VERSION.
@@ -140,23 +147,26 @@ public Descriptor(Version version, File directory, String ksname, String cfname,
 
         // directory is unnecessary for hashCode, and for simulator consistency we do not include it
         hashCode = Objects.hashCode(version, id, ksname, cfname);
-    }
 
-    private String tmpFilenameFor(Component component)
-    {
-        return fileFor(component) + TMP_EXT;
+        String locationURI = directory.toUri().toString();
+        if (!locationURI.endsWith(java.io.File.separator))
+            locationURI = locationURI + java.io.File.separatorChar;
+        baseFileURI = locationURI + prefix;
+
+        componentFileMap = new ConcurrentHashMap<>();
     }
 
     public File tmpFileFor(Component component)
     {
-        return new File(directory.toPath().resolve(tmpFilenameFor(component)));
+        File file = StorageProvider.instance.getLocalPath(fileFor(component));
+        return file.resolveSibling(file.name() + TMP_EXT);
     }
 
     private String tmpFilenameForStreaming(Component component)
     {
         // Use UUID to handle concurrent streamings on the same sstable.
         // TMP_EXT allows temp file to be removed by {@link ColumnFamilyStore#scrubDataDirectories}
-        return String.format("%s.%s%s", filenameFor(component), nextTimeUUID(), TMP_EXT);
+        return String.format("%s.%s%s", fileFor(component), nextTimeUUID(), TMP_EXT);
     }
 
     /**
@@ -167,14 +177,33 @@ public File tmpFileForStreaming(Component component)
         return new File(directory.toPath().resolve(tmpFilenameForStreaming(component)));
     }
 
-    private String filenameFor(Component component)
+    public String filenameFor(Component component)
     {
-        return prefix + separator + component.name();
+        return prefix + SEPARATOR + component.name();
     }
 
     public File fileFor(Component component)
     {
-        return new File(directory.toPath().resolve(filenameFor(component)));
+        return componentFileMap.computeIfAbsent(component, c -> {
+            // STAR-1892 - CNDB depdends on using Component.getFile here to be able to create a RemotePath if the
+            // URI matches. However, tests that extend CQLTester.InMemory (using the jimfs file system) will fail
+            // with an UnsupportedOperationException. One such test is CQLVectorTest.
+            //
+            // Because Component.getFile uses the file URI to create a new Path object, it will not be wrapped by 
+            // ListenablePath. That leads to an UnsupportedOperationException because ListenablePath implements certain 
+            // methods that are not implemented by jimfs.
+            //
+            // Using the resolve method on the directory to create a new Path object keeps it wrapped by ListenablePath 
+            // and will prevent an UnsupportedOperationException.
+            if (baseFileUri().startsWith("jimfs:"))
+            {
+                return new File(directory.toPath().resolve(filenameFor(component)));
+            }
+            else {
+                return component.getFile(baseFileUri());
+            }
+
+        });
     }
 
     public File baseFile()
@@ -184,9 +213,19 @@ public File baseFile()
 
     private void appendFileName(StringBuilder buff)
     {
-        buff.append(version).append(separator);
+        buff.append(version).append(SEPARATOR);
         buff.append(id.toString());
-        buff.append(separator).append(version.format.name());
+        buff.append(SEPARATOR).append(version.format.name());
+    }
+
+    public String baseFileUri()
+    {
+        return baseFileURI;
+    }
+
+    public String filenamePart()
+    {
+        return prefix;
     }
 
     public String relativeFilenameFor(Component component)
@@ -198,7 +237,7 @@ public String relativeFilenameFor(Component component)
         }
 
         appendFileName(buff);
-        buff.append(separator).append(component.name());
+        buff.append(SEPARATOR).append(component.name());
         return buff.toString();
     }
 
@@ -264,12 +303,43 @@ public static Descriptor fromFile(File file)
         return fromFileWithComponent(file).left;
     }
 
+    public static Descriptor fromFilename(String filename)
+    {
+        return fromFile(new File(filename));
+    }
+
+    public static Descriptor fromFilename(File file)
+    {
+        return fromFileWithComponent(file).left;
+    }
+
+    public static Pair<Descriptor, Component> fromFilenameWithComponent(File file)
+    {
+        return fromFileWithComponent(file);
+    }
+
     public static Component componentFromFile(File file)
     {
-        String name = file.name();
-        List<String> tokens = filenameTokens(name);
+        return validFilenameWithComponent(file.name());
+    }
+    
+    public static Component validFilenameWithComponent(String name)
+    {
+        try
+        {
+            List<String> tokens = filenameTokens(name);
 
-        return Component.parse(tokens.get(3), formatFromName(name, tokens));
+            return Component.parse(tokens.get(3), formatFromName(name, tokens));
+        }
+        catch (Exception e)
+        {
+            return null;
+        }
+    }
+    
+    public static boolean validFilename(String name)
+    {
+        return validFilenameWithComponent(name) != null;
     }
 
     private static SSTableFormat<?, ?> formatFromName(String fileName, List<String> tokens)
@@ -463,7 +533,7 @@ public Set<Component> discoverComponents()
     @Override
     public String toString()
     {
-        return baseFile().absolutePath();
+        return baseFileUri();
     }
 
     @Override
@@ -488,4 +558,22 @@ public int hashCode()
     {
         return hashCode;
     }
+
+    @DseLegacy
+    public Path getDirectory()
+    {
+        return directory.toPath();
+    }
+
+    @DseLegacy
+    public Path pathFor(Component component)
+    {
+        return fileFor(component).toPath();
+    }
+
+    @DseLegacy
+    public String baseFileURI()
+    {
+        return baseFileUri();
+    }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java
index 8976ed413072..b17fa13475c2 100644
--- a/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java
@@ -56,6 +56,11 @@ public Set<SSTableReader> getBackingSSTables()
         return ImmutableSet.of(sstable);
     }
 
+    @Override
+    public int level() {
+        return 0;
+    }
+
     public long getCurrentPosition()
     {
         return 0;
diff --git a/src/java/org/apache/cassandra/io/sstable/IKeyFetcher.java b/src/java/org/apache/cassandra/io/sstable/IKeyFetcher.java
new file mode 100644
index 000000000000..3485d03cbf44
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/IKeyFetcher.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.function.LongFunction;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.utils.Closeable;
+
+public interface IKeyFetcher extends LongFunction<DecoratedKey>, Closeable
+{
+    /**
+     * @param keyOffset the offset of the key
+     * @return the key at the given offset, or null if the key is not present or the offset is out of range
+     */
+    DecoratedKey apply(long keyOffset);
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
index 671bccb824b5..c24c2b23d7e4 100644
--- a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
@@ -34,13 +34,14 @@
  */
 public interface ISSTableScanner extends UnfilteredPartitionIterator
 {
-    public long getLengthInBytes();
-    public long getCompressedLengthInBytes();
-    public long getCurrentPosition();
-    public long getBytesScanned();
-    public Set<SSTableReader> getBackingSSTables();
+    long getLengthInBytes();
+    long getCompressedLengthInBytes();
+    long getCurrentPosition();
+    long getBytesScanned();
+    Set<SSTableReader> getBackingSSTables();
+    int level();
 
-    public static void closeAllAndPropagate(Collection<ISSTableScanner> scanners, Throwable throwable)
+    static Throwable closeAllAndPropagate(Collection<ISSTableScanner> scanners, Throwable throwable)
     {
         for (ISSTableScanner scanner: scanners)
         {
@@ -67,6 +68,6 @@ public static void closeAllAndPropagate(Collection<ISSTableScanner> scanners, Th
             Throwables.throwIfUnchecked(throwable);
             throw new RuntimeException(throwable);
         }
-
+        return null;
     }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/IScrubber.java b/src/java/org/apache/cassandra/io/sstable/IScrubber.java
index 50c6eb35fa16..e280ee3dac95 100644
--- a/src/java/org/apache/cassandra/io/sstable/IScrubber.java
+++ b/src/java/org/apache/cassandra/io/sstable/IScrubber.java
@@ -18,20 +18,22 @@
 
 package org.apache.cassandra.io.sstable;
 
+import java.util.List;
 import java.util.StringJoiner;
 
 import com.google.common.annotations.VisibleForTesting;
 
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.TableOperation;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.utils.Closeable;
 
 public interface IScrubber extends Closeable
 {
-    void scrub();
+    List<SSTableReader> scrub();
 
     void close();
 
-    CompactionInfo.Holder getScrubInfo();
+    TableOperation getScrubInfo();
 
     @VisibleForTesting
     ScrubResult scrubWithResult();
@@ -46,12 +48,14 @@ final class ScrubResult
         public final int goodPartitions;
         public final int badPartitions;
         public final int emptyPartitions;
+        public final List<SSTableReader> scrubbed;
 
-        public ScrubResult(int goodPartitions, int badPartitions, int emptyPartitions)
+        public ScrubResult(int goodPartitions, int badPartitions, int emptyPartitions, List<SSTableReader> scrubbed)
         {
             this.goodPartitions = goodPartitions;
             this.badPartitions = badPartitions;
             this.emptyPartitions = emptyPartitions;
+            this.scrubbed = scrubbed;
         }
     }
 
@@ -60,12 +64,14 @@ class Options
         public final boolean checkData;
         public final boolean reinsertOverflowedTTLRows;
         public final boolean skipCorrupted;
+        public final boolean overrideTxnIsOffline;
 
-        private Options(boolean checkData, boolean reinsertOverflowedTTLRows, boolean skipCorrupted)
+        private Options(boolean checkData, boolean reinsertOverflowedTTLRows, boolean skipCorrupted, boolean overrideTxnIsOffline)
         {
             this.checkData = checkData;
             this.reinsertOverflowedTTLRows = reinsertOverflowedTTLRows;
             this.skipCorrupted = skipCorrupted;
+            this.overrideTxnIsOffline = overrideTxnIsOffline;
         }
 
         @Override
@@ -83,6 +89,7 @@ public static class Builder
             private boolean checkData = false;
             private boolean reinsertOverflowedTTLRows = false;
             private boolean skipCorrupted = false;
+            private boolean overrideTxnIsOffline = false;
 
             public Builder checkData()
             {
@@ -120,9 +127,21 @@ public Builder skipCorrupted(boolean skipCorrupted)
                 return this;
             }
 
+            public Builder overrideTxnIsOffline()
+            {
+                this.overrideTxnIsOffline = true;
+                return this;
+            }
+
+            public Builder overrideTxnIsOffline(boolean overrideTxnIsOffline)
+            {
+                this.overrideTxnIsOffline = overrideTxnIsOffline;
+                return this;
+            }
+
             public Options build()
             {
-                return new Options(checkData, reinsertOverflowedTTLRows, skipCorrupted);
+                return new Options(checkData, reinsertOverflowedTTLRows, skipCorrupted, overrideTxnIsOffline);
             }
         }
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/IVerifier.java b/src/java/org/apache/cassandra/io/sstable/IVerifier.java
index 62ec0659af61..a23fe686a745 100644
--- a/src/java/org/apache/cassandra/io/sstable/IVerifier.java
+++ b/src/java/org/apache/cassandra/io/sstable/IVerifier.java
@@ -22,7 +22,7 @@
 import java.util.Collection;
 import java.util.function.Function;
 
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.service.StorageService;
@@ -39,7 +39,7 @@ static Options.Builder options()
     @Override
     void close();
 
-    CompactionInfo.Holder getVerifyInfo();
+    AbstractTableOperation getVerifyInfo();
 
     class Options
     {
@@ -50,6 +50,7 @@ class Options
          * if there is no digest present. Setting it along with quick makes no sense.
          */
         public final boolean extendedVerification;
+        public final boolean validateAllRows;
 
         public final boolean checkVersion;
         public final boolean mutateRepairStatus;
@@ -64,6 +65,7 @@ class Options
 
         private Options(boolean invokeDiskFailurePolicy,
                         boolean extendedVerification,
+                        boolean validateAllRows,
                         boolean checkVersion,
                         boolean mutateRepairStatus,
                         boolean checkOwnsTokens,
@@ -72,11 +74,15 @@ private Options(boolean invokeDiskFailurePolicy,
         {
             this.invokeDiskFailurePolicy = invokeDiskFailurePolicy;
             this.extendedVerification = extendedVerification;
+            this.validateAllRows = validateAllRows;
             this.checkVersion = checkVersion;
             this.mutateRepairStatus = mutateRepairStatus;
             this.checkOwnsTokens = checkOwnsTokens;
             this.quick = quick;
             this.tokenLookup = tokenLookup;
+
+            if (validateAllRows && !extendedVerification)
+                throw new IllegalArgumentException("validateAllRows must be enabled with extended verification");
         }
 
         @Override
@@ -85,6 +91,7 @@ public String toString()
             return "Options{" +
                    "invokeDiskFailurePolicy=" + invokeDiskFailurePolicy +
                    ", extendedVerification=" + extendedVerification +
+                   ", validateAllRows=" + validateAllRows +
                    ", checkVersion=" + checkVersion +
                    ", mutateRepairStatus=" + mutateRepairStatus +
                    ", checkOwnsTokens=" + checkOwnsTokens +
@@ -96,6 +103,7 @@ public static class Builder
         {
             private boolean invokeDiskFailurePolicy = false; // invoking disk failure policy can stop the node if we find a corrupt stable
             private boolean extendedVerification = false;
+            private boolean validateAllRows = false; // whether to validate all rows in each partition in extended verification mode
             private boolean checkVersion = false;
             private boolean mutateRepairStatus = false; // mutating repair status can be dangerous
             private boolean checkOwnsTokens = false;
@@ -114,6 +122,12 @@ public Builder extendedVerification(boolean param)
                 return this;
             }
 
+            public Builder validateAllRows(boolean param)
+            {
+                this.validateAllRows = param;
+                return this;
+            }
+
             public Builder checkVersion(boolean param)
             {
                 this.checkVersion = param;
@@ -146,7 +160,7 @@ public Builder tokenLookup(Function<String, ? extends Collection<Range<Token>>>
 
             public Options build()
             {
-                return new Options(invokeDiskFailurePolicy, extendedVerification, checkVersion, mutateRepairStatus, checkOwnsTokens, quick, tokenLookup);
+                return new Options(invokeDiskFailurePolicy, extendedVerification, validateAllRows, checkVersion, mutateRepairStatus, checkOwnsTokens, quick, tokenLookup);
             }
         }
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
index dbe501f36e7e..b6dd9934861a 100644
--- a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
@@ -25,6 +25,7 @@
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.CloseableIterator;
 
+// TODO STAR-247: Implement a unit test
 public class KeyIterator extends AbstractIterator<DecoratedKey> implements CloseableIterator<DecoratedKey>
 {
     private final IPartitioner partitioner;
diff --git a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java
index 422c6eaa6eb7..534fd7c42ef2 100644
--- a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java
@@ -26,21 +26,24 @@
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.DiskBoundaries;
-import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 
 public class RangeAwareSSTableWriter implements SSTableMultiWriter
 {
-    private final List<PartitionPosition> boundaries;
+    private final List<Token> boundaries;
     private final List<Directories.DataDirectory> directories;
     private final int sstableLevel;
+    private final IntervalSet<CommitLogPosition> commitLogIntervals;
     private final long estimatedKeys;
     private final long repairedAt;
     private final TimeUUID pendingRepair;
@@ -54,7 +57,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
     private final List<SSTableReader> finishedReaders = new ArrayList<>();
     private SSTableMultiWriter currentWriter = null;
 
-    public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SSTableFormat<?, ?> format, int sstableLevel, long totalSize, LifecycleNewTracker lifecycleNewTracker, SerializationHeader header) throws IOException
+    public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SSTableFormat<?, ?> format, IntervalSet<CommitLogPosition> commitLogIntervals, int sstableLevel, long totalSize, LifecycleNewTracker lifecycleNewTracker, SerializationHeader header) throws IOException
     {
         DiskBoundaries db = cfs.getDiskBoundaries();
         directories = db.directories;
@@ -67,7 +70,8 @@ public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long r
         this.format = format;
         this.lifecycleNewTracker = lifecycleNewTracker;
         this.header = header;
-        boundaries = db.positions;
+        this.commitLogIntervals = commitLogIntervals;
+        boundaries = db.getPositions();
         if (boundaries == null)
         {
             Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
@@ -75,7 +79,7 @@ public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long r
                 throw new IOException(String.format("Insufficient disk space to store %s",
                                                     FBUtilities.prettyPrintMemory(totalSize)));
             Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format);
-            currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, null, sstableLevel, header, lifecycleNewTracker);
+            currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, commitLogIntervals, sstableLevel, header, lifecycleNewTracker);
         }
     }
 
@@ -85,7 +89,7 @@ private void maybeSwitchWriter(DecoratedKey key)
             return;
 
         boolean switched = false;
-        while (currentIndex < 0 || key.compareTo(boundaries.get(currentIndex)) > 0)
+        while (currentIndex < 0 || key.getToken().compareTo(boundaries.get(currentIndex)) > 0)
         {
             switched = true;
             currentIndex++;
@@ -97,7 +101,7 @@ private void maybeSwitchWriter(DecoratedKey key)
                 finishedWriters.add(currentWriter);
 
             Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format);
-            currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, null, sstableLevel, header, lifecycleNewTracker);
+            currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, commitLogIntervals, sstableLevel, header, lifecycleNewTracker);
         }
     }
 
@@ -130,11 +134,10 @@ public Collection<SSTableReader> finished()
     }
 
     @Override
-    public SSTableMultiWriter setOpenResult(boolean openResult)
+    public void openResult()
     {
-        finishedWriters.forEach((w) -> w.setOpenResult(openResult));
-        currentWriter.setOpenResult(openResult);
-        return this;
+        finishedWriters.forEach(SSTableMultiWriter::openResult);
+        currentWriter.openResult();
     }
 
     public String getFilename()
@@ -145,13 +148,25 @@ public String getFilename()
     @Override
     public long getBytesWritten()
     {
-       return currentWriter != null ? currentWriter.getBytesWritten() : 0L;
+        long bytesWritten = currentWriter != null ? currentWriter.getBytesWritten() : 0L;
+        for (SSTableMultiWriter writer : finishedWriters)
+            bytesWritten += writer.getBytesWritten();
+        return bytesWritten;
     }
 
     @Override
     public long getOnDiskBytesWritten()
     {
-        return currentWriter != null ? currentWriter.getOnDiskBytesWritten() : 0L;
+        long bytesWritten = currentWriter != null ? currentWriter.getOnDiskBytesWritten() : 0L;
+        for (SSTableMultiWriter writer : finishedWriters)
+            bytesWritten += writer.getOnDiskBytesWritten();
+        return bytesWritten;
+    }
+
+    @Override
+    public int getSegmentCount()
+    {
+        return finishedWriters.size() + 1;
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
index 1cd780e0769d..201d4384f7e7 100644
--- a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
@@ -20,14 +20,13 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Iterator;
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.CloseableIterator;
-import org.apache.cassandra.utils.IMergeIterator;
 import org.apache.cassandra.utils.MergeIterator;
+import org.apache.cassandra.utils.Reducer;
 import org.apache.cassandra.utils.Throwables;
 
 /**
@@ -36,23 +35,28 @@
 public class ReducingKeyIterator implements CloseableIterator<DecoratedKey>
 {
     private final ArrayList<KeyIterator> iters;
-    private volatile IMergeIterator<DecoratedKey, DecoratedKey> mi;
+    private volatile CloseableIterator<DecoratedKey> mi;
+    private final long totalLength;
 
     public ReducingKeyIterator(Collection<SSTableReader> sstables)
     {
         iters = new ArrayList<>(sstables.size());
+        long len = 0;
         for (SSTableReader sstable : sstables)
         {
             try
             {
-                iters.add(sstable.keyIterator());
+                KeyIterator iter = sstable.keyIterator();
+                iters.add(iter);
+                len += iter.getTotalBytes();
             }
-            catch (IOException ex)
+            catch (IOException | RuntimeException ex)
             {
                 iters.forEach(FileUtils::closeQuietly);
-                throw new RuntimeException("Failed to create a key iterator for sstable " + sstable.getFilename());
+                throw new RuntimeException("Failed to create a key iterator for sstable " + sstable.getFilename(), ex);
             }
         }
+        this.totalLength = len;
     }
 
     private void maybeInit()
@@ -64,12 +68,12 @@ private void maybeInit()
         {
             if (mi == null)
             {
-                mi = MergeIterator.get(iters, DecoratedKey.comparator, new MergeIterator.Reducer<DecoratedKey, DecoratedKey>()
+                mi = MergeIterator.getCloseable(iters, DecoratedKey.comparator, new Reducer<>()
                 {
                     DecoratedKey reduced = null;
 
                     @Override
-                    public boolean trivialReduceIsTrivial()
+                    public boolean singleSourceReduceIsTrivial()
                     {
                         return true;
                     }
@@ -79,7 +83,7 @@ public void reduce(int idx, DecoratedKey current)
                         reduced = current;
                     }
 
-                    protected DecoratedKey getReduced()
+                    public DecoratedKey getReduced()
                     {
                         return reduced;
                     }
@@ -106,14 +110,7 @@ public void close()
 
     public long getTotalBytes()
     {
-        maybeInit();
-
-        long m = 0;
-        for (Iterator<DecoratedKey> iter : mi.iterators())
-        {
-            m += ((KeyIterator) iter).getTotalBytes();
-        }
-        return m;
+        return totalLength;
     }
 
     public long getBytesRead()
@@ -121,9 +118,9 @@ public long getBytesRead()
         maybeInit();
 
         long m = 0;
-        for (Iterator<DecoratedKey> iter : mi.iterators())
+        for (KeyIterator iter : iters)
         {
-            m += ((KeyIterator) iter).getBytesRead();
+            m += iter.getBytesRead();
         }
         return m;
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java
index 475f92beeb49..3cf2d0150779 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTable.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java
@@ -17,32 +17,33 @@
  */
 package org.apache.cassandra.io.sstable;
 
+import java.io.IOException;
 import java.lang.ref.WeakReference;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.cache.ChunkCache;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.lifecycle.Tracker;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.filter.BloomFilterTracker;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.format.TOCComponent;
@@ -66,12 +67,14 @@
  */
 public abstract class SSTable
 {
+    static final Logger logger = LoggerFactory.getLogger(SSTable.class);
+
     public static final int TOMBSTONE_HISTOGRAM_BIN_SIZE = 100;
     public static final int TOMBSTONE_HISTOGRAM_SPOOL_SIZE = 100000;
     public static final int TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS = CassandraRelevantProperties.STREAMING_HISTOGRAM_ROUND_SECONDS.getInt();
 
     public final Descriptor descriptor;
-    protected final Set<Component> components;
+    private volatile ImmutableSet<Component> components;
     public final boolean compression;
 
     protected final TableMetadataRef metadata;
@@ -90,7 +93,7 @@ public SSTable(Builder<?, ?> builder, Owner owner)
 
         this.descriptor = builder.descriptor;
         this.ioOptions = builder.getIOOptions();
-        this.components = new CopyOnWriteArraySet<>(builder.getComponents());
+        this.components = ImmutableSet.copyOf(builder.getComponents());
         this.compression = components.contains(Components.COMPRESSION_INFO);
         this.metadata = builder.getTableMetadataRef();
         this.chunkCache = builder.getChunkCache();
@@ -151,16 +154,15 @@ public static void hardlink(Descriptor tmpdesc, Descriptor newdesc, Set<Componen
                   .forEach(c -> FileUtils.createHardLinkWithoutConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c)));
     }
 
-    public abstract DecoratedKey getFirst();
+    public abstract PartitionPosition getFirst();
 
-    public abstract DecoratedKey getLast();
+    public abstract PartitionPosition getLast();
 
     public abstract AbstractBounds<Token> getBounds();
 
-    @VisibleForTesting
-    public Set<Component> getComponents()
+    public ImmutableSet<Component> components()
     {
-        return ImmutableSet.copyOf(components);
+        return components;
     }
 
     /**
@@ -178,6 +180,11 @@ public TableMetadata metadata()
         return metadata.get();
     }
 
+    public TableMetadataRef metadataRef()
+    {
+        return metadata;
+    }
+
     public IPartitioner getPartitioner()
     {
         return metadata().partitioner;
@@ -190,7 +197,12 @@ public DecoratedKey decorateKey(ByteBuffer key)
 
     public String getFilename()
     {
-        return descriptor.fileFor(Components.DATA).absolutePath();
+        return getDataFile().path();
+    }
+
+    public File getDataFile()
+    {
+        return descriptor.fileFor(Components.DATA);
     }
 
     public String getColumnFamilyName()
@@ -203,12 +215,14 @@ public String getKeyspaceName()
         return descriptor.ksname;
     }
 
-    public List<String> getAllFilePaths()
+    public SSTableId getId()
     {
-        List<String> ret = new ArrayList<>(components.size());
-        for (Component component : components)
-            ret.add(descriptor.fileFor(component).absolutePath());
-        return ret;
+        return descriptor.id;
+    }
+
+    public int getComponentSize()
+    {
+        return components.size();
     }
 
     /**
@@ -319,9 +333,7 @@ public static void validateRepairedMetadata(long repairedAt, TimeUUID pendingRep
      */
     public synchronized void addComponents(Collection<Component> newComponents)
     {
-        Collection<Component> componentsToAdd = Collections2.filter(newComponents, Predicates.not(Predicates.in(components)));
-        TOCComponent.appendTOC(descriptor, componentsToAdd);
-        components.addAll(componentsToAdd);
+        registerComponents(newComponents, null);
     }
 
     /**
@@ -332,15 +344,13 @@ public synchronized void addComponents(Collection<Component> newComponents)
     public synchronized void registerComponents(Collection<Component> newComponents, Tracker tracker)
     {
         Collection<Component> componentsToAdd = new HashSet<>(Collections2.filter(newComponents, x -> !components.contains(x)));
+        if (componentsToAdd.isEmpty())
+            return;
+
         TOCComponent.appendTOC(descriptor, componentsToAdd);
-        components.addAll(componentsToAdd);
+        components = ImmutableSet.<Component>builder().addAll(components).addAll(componentsToAdd).build();
 
-        for (Component component : componentsToAdd)
-        {
-            File file = descriptor.fileFor(component);
-            if (file.exists())
-                tracker.updateLiveDiskSpaceUsed(file.length());
-        }
+        updateComponentsTracking(componentsToAdd, tracker, 1);
     }
 
     /**
@@ -350,22 +360,69 @@ public synchronized void registerComponents(Collection<Component> newComponents,
      */
     public synchronized void unregisterComponents(Collection<Component> removeComponents, Tracker tracker)
     {
-        Collection<Component> componentsToRemove = new HashSet<>(Collections2.filter(removeComponents, components::contains));
-        components.removeAll(componentsToRemove);
+        Set<Component> componentsToRemove = new HashSet<>(Collections2.filter(removeComponents, components::contains));
+        components = Sets.difference(components, componentsToRemove).immutableCopy();
         TOCComponent.rewriteTOC(descriptor, components);
 
-        for (Component component : componentsToRemove)
+        updateComponentsTracking(componentsToRemove, tracker, -1);
+    }
+
+    private void updateComponentsTracking(Collection<Component> toUpdate, Tracker tracker, long multiplier)
+    {
+        if (tracker == null)
+            return;
+
+        for (Component component : toUpdate)
         {
             File file = descriptor.fileFor(component);
             if (file.exists())
-                tracker.updateLiveDiskSpaceUsed(-file.length());
+                tracker.updateSizeTracking(multiplier * file.length());
         }
     }
 
+    /**
+     * Reads components from the TOC file and update the `components` set of this object accordindly.
+     * <p>
+     * Usually, components are added/removed through {@link #addComponents}, {@link #registerComponents} or
+     * {@link #unregisterComponents}, which both update this object component and update the TOC file accordingly, and
+     * this method should not be used. But some implementation of tiered storage may add components/rewrite the TOC
+     * "externally" (one reason can be offloading index rebuild) and need those change to be reflected to this object
+     * and this is where this method comes in.
+     * <p>
+     * If the TOC file does not exist, cannot be read, or does not at least contains the minimal components that all
+     * sstables should have when this is called, this method is a no-op.
+     */
+    public synchronized void reloadComponentsFromTOC(Tracker tracker)
+    {
+        try
+        {
+            Set<Component> tocComponents = TOCComponent.loadTOC(descriptor);
+            Set<Component> requiredComponents = descriptor.getFormat().primaryComponents();
+            if (!tocComponents.containsAll(requiredComponents))
+            {
+                logger.error("Cannot reload components from read TOC file for {}; the TOC does not contain all the required components for the sstable type and is like corrupted (components in TOC: {}, required by sstable format: {})",
+                             descriptor, tocComponents, requiredComponents);
+                return;
+            }
+
+            Set<Component> toAdd = Sets.difference(tocComponents, components);
+            Set<Component> toRemove = Sets.difference(components, tocComponents);
+            components = ImmutableSet.copyOf(tocComponents);
+
+            updateComponentsTracking(toAdd, tracker, 1);
+            updateComponentsTracking(toRemove, tracker, -1);
+
+        }
+        catch (IOException e)
+        {
+            logger.error("Failed to read TOC file for {}; ignoring component reload", descriptor, e);
+        }
+    }
+    
     public interface Owner
     {
         Double getCrcCheckChance();
-
+        BloomFilterTracker getBloomFilterTracker();
         OpOrder.Barrier newReadOrderingBarrier();
 
         TableMetrics getMetrics();
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java b/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java
index 159e0d43bdf2..908a54e8d59e 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java
@@ -20,10 +20,9 @@
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Unfiltered;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
 
 /**
- * Observer for events in the lifecycle of writing out an sstable.
+ * Observer for events in the lifecycle of writing out an sstable -- for compaction as well as for flush!
  */
 public interface SSTableFlushObserver
 {
@@ -41,7 +40,7 @@ public interface SSTableFlushObserver
      * @param keyPositionForSASI SSTable format specific key position for storage attached indexes, it can be
      *                           in data file or in some index file. It is the same position as returned by
      *                           {@link KeyReader#keyPositionForSecondaryIndex()} for the same format, and the same
-     *                           position as expected by {@link SSTableReader#keyAtPositionFromSecondaryIndex(long)}.
+     *                           position as expected by {@link IKeyFetcher#apply(long)} when created for SASI.
      */
     void startPartition(DecoratedKey key, long keyPosition, long keyPositionForSASI);
 
@@ -66,7 +65,7 @@ public interface SSTableFlushObserver
     /**
      * Called when all data is written to the file and it's ready to be finished up.
      */
-    void complete();
+    void complete(SSTable ssTable);
 
     /**
      * Clean up resources on error. There should be no side effects if called multiple times.
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java
new file mode 100644
index 000000000000..c927fabc5e2c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java
@@ -0,0 +1,920 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.AbstractCompositeType;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DynamicCompositeType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_SKIP_AUTOMATIC_UDT_FIX;
+
+/**
+ * Validates and fixes type issues in the serialization-header of sstables.
+ */
+public abstract class SSTableHeaderFix
+{
+    // C* 3.0 upgrade code
+
+    private static final boolean SKIP_AUTOMATIC_FIX_ON_UPGRADE = CASSANDRA_SKIP_AUTOMATIC_UDT_FIX.getBoolean();
+
+    public static void fixNonFrozenUDTIfUpgradeFrom30()
+    {
+        String previousVersionString = FBUtilities.getPreviousReleaseVersionString();
+        if (previousVersionString == null)
+            return;
+        CassandraVersion previousVersion = new CassandraVersion(previousVersionString);
+        if (previousVersion.major != 3 || previousVersion.minor > 0)
+        {
+            // Not an upgrade from 3.0 to 3.x, nothing to do here
+            return;
+        }
+
+        if (SKIP_AUTOMATIC_FIX_ON_UPGRADE)
+        {
+            logger.warn("Detected upgrade from {} to {}, but -D{}=true, NOT fixing UDT type references in " +
+                        "sstable metadata serialization-headers",
+                        previousVersionString,
+                        FBUtilities.getReleaseVersionString(),
+                        CASSANDRA_SKIP_AUTOMATIC_UDT_FIX.getKey());
+            return;
+        }
+
+        logger.info("Detected upgrade from {} to {}, fixing UDT type references in sstable metadata serialization-headers",
+                    previousVersionString,
+                    FBUtilities.getReleaseVersionString());
+
+        SSTableHeaderFix instance = SSTableHeaderFix.builder()
+                                                    .schemaCallback(() -> Schema.instance::getTableMetadata)
+                                                    .build();
+        instance.execute();
+    }
+
+    // "regular" SSTableHeaderFix code, also used by StandaloneScrubber.
+
+    private static final Logger logger = LoggerFactory.getLogger(SSTableHeaderFix.class);
+
+    protected final Consumer<String> info;
+    protected final Consumer<String> warn;
+    protected final Consumer<String> error;
+    protected final boolean dryRun;
+    protected final Function<Descriptor, TableMetadata> schemaCallback;
+
+    private final List<Descriptor> descriptors;
+
+    private final List<Pair<Descriptor, Map<MetadataType, MetadataComponent>>> updates = new ArrayList<>();
+    private boolean hasErrors;
+
+    SSTableHeaderFix(Builder builder)
+    {
+        this.info = builder.info;
+        this.warn = builder.warn;
+        this.error = builder.error;
+        this.dryRun = builder.dryRun;
+        this.schemaCallback = builder.schemaCallback.get();
+        this.descriptors = new ArrayList<>(builder.descriptors);
+        Objects.requireNonNull(this.info, "info is null");
+        Objects.requireNonNull(this.warn, "warn is null");
+        Objects.requireNonNull(this.error, "error is null");
+        Objects.requireNonNull(this.schemaCallback, "schemaCallback is null");
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    /**
+     * Builder to configure and construct an instance of {@link SSTableHeaderFix}.
+     * Default settings:
+     * <ul>
+     *     <li>log via the slf4j logger of {@link SSTableHeaderFix}</li>
+     *     <li>no dry-run (i.e. validate and fix, if no serious errors are detected)</li>
+     *     <li>no schema callback</li>
+     * </ul>
+     * If neither {@link #withDescriptor(Descriptor)} nor {@link #withPath(Path)} are used,
+     * all "live" sstables in all data directories will be scanned.
+     */
+    public static class Builder
+    {
+        private final List<Path> paths = new ArrayList<>();
+        private final List<Descriptor> descriptors = new ArrayList<>();
+        private Consumer<String> info = (ln) -> logger.info("{}", ln);
+        private Consumer<String> warn = (ln) -> logger.warn("{}", ln);
+        private Consumer<String> error = (ln) -> logger.error("{}", ln);
+        private boolean dryRun;
+        private Supplier<Function<Descriptor, TableMetadata>> schemaCallback = () -> null;
+
+        private Builder()
+        {}
+
+        /**
+         * Only validate and prepare fix, but do not write updated (fixed) sstable serialization-headers.
+         */
+        public Builder dryRun()
+        {
+            dryRun = true;
+            return this;
+        }
+
+        public Builder info(Consumer<String> output)
+        {
+            this.info = output;
+            return this;
+        }
+
+        public Builder warn(Consumer<String> warn)
+        {
+            this.warn = warn;
+            return this;
+        }
+
+        public Builder error(Consumer<String> error)
+        {
+            this.error = error;
+            return this;
+        }
+
+        /**
+         * Manually provide an individual sstable or directory containing sstables.
+         *
+         * Implementation note: procesing "live" sstables in their data directories as well as sstables
+         * in snapshots and backups in the data directories works.
+         *
+         * But processing sstables that reside somewhere else (i.e. verifying sstables before import)
+         * requires the use of {@link #withDescriptor(Descriptor)}.
+         */
+        public Builder withPath(Path path)
+        {
+            this.paths.add(path);
+            return this;
+        }
+
+        public Builder withDescriptor(Descriptor descriptor)
+        {
+            this.descriptors.add(descriptor);
+            return this;
+        }
+
+        /**
+         * Schema callback to retrieve the schema of a table. Production code always delegates to the
+         * live schema ({@code Schema.instance}). Unit tests use this method to feed a custom schema.
+         */
+        public Builder schemaCallback(Supplier<Function<Descriptor, TableMetadata>> schemaCallback)
+        {
+            this.schemaCallback = schemaCallback;
+            return this;
+        }
+
+        public SSTableHeaderFix build()
+        {
+            if (paths.isEmpty() && descriptors.isEmpty())
+                return new AutomaticHeaderFix(this);
+
+            return new ManualHeaderFix(this);
+        }
+
+        public Builder logToList(List<String> output)
+        {
+            return info(ln -> output.add("INFO  " + ln))
+                   .warn(ln -> output.add("WARN  " + ln))
+                   .error(ln -> output.add("ERROR " + ln));
+        }
+    }
+
+    public final void execute()
+    {
+        prepare();
+
+        logger.debug("Processing {} sstables:{}",
+                     descriptors.size(),
+                     descriptors.stream().map(Descriptor::toString).collect(Collectors.joining("\n    ", "\n    ", "")));
+
+        descriptors.forEach(this::processSSTable);
+
+        if (updates.isEmpty())
+            return;
+
+        if (hasErrors)
+        {
+            info.accept("Stopping due to previous errors. Either fix the errors or specify the ignore-errors option.");
+            return;
+        }
+
+        if (dryRun)
+        {
+            info.accept("Not fixing identified and fixable serialization-header issues.");
+            return;
+        }
+
+        info.accept("Writing new metadata files");
+        updates.forEach(descAndMeta -> writeNewMetadata(descAndMeta.left, descAndMeta.right));
+        info.accept("Finished writing new metadata files");
+    }
+
+    /**
+     * Whether {@link #execute()} encountered an error.
+     */
+    public boolean hasError()
+    {
+        return hasErrors;
+    }
+
+    /**
+     * Whether {@link #execute()} found mismatches.
+     */
+    public boolean hasChanges()
+    {
+        return !updates.isEmpty();
+    }
+
+    abstract void prepare();
+
+    private void error(String format, Object... args)
+    {
+        hasErrors = true;
+        error.accept(String.format(format, args));
+    }
+
+    void processFileOrDirectory(Path path)
+    {
+        Stream.of(path)
+              .flatMap(SSTableHeaderFix::maybeExpandDirectory)
+              .filter(p -> Descriptor.fromFileWithComponent(new File(p)).right.type == SSTableFormat.Components.DATA.type)
+              .map(Path::toString)
+              .map((String file) -> Descriptor.fromFile(new File(file)))
+              .forEach(descriptors::add);
+    }
+
+    private static Stream<Path> maybeExpandDirectory(Path path)
+    {
+        if (Files.isRegularFile(path))
+            return Stream.of(path);
+        return LifecycleTransaction.getFiles(path, (file, fileType) -> fileType == Directories.FileType.FINAL, Directories.OnTxnErr.IGNORE)
+                                   .stream()
+                                   .map(File::toPath);
+    }
+
+    private void processSSTable(Descriptor desc)
+    {
+        if (desc.cfname.indexOf('.') != -1)
+        {
+            // secondary index not checked
+
+            // partition-key is the indexed column type
+            // clustering-key is org.apache.cassandra.db.marshal.PartitionerDefinedOrder
+            // no static columns, no regular columns
+            return;
+        }
+
+        TableMetadata tableMetadata = schemaCallback.apply(desc);
+        if (tableMetadata == null)
+        {
+            error("Table %s.%s not found in the schema - NOT checking sstable %s", desc.ksname, desc.cfname, desc);
+            return;
+        }
+
+        Set<Component> components = desc.discoverComponents();
+        if (components.stream().noneMatch(c -> c.type == SSTableFormat.Components.STATS.type))
+        {
+            error("sstable %s has no -Statistics.db component.", desc);
+            return;
+        }
+
+        Map<MetadataType, MetadataComponent> metadata = readSSTableMetadata(desc);
+        if (metadata == null)
+            return;
+
+        MetadataComponent component = metadata.get(MetadataType.HEADER);
+        if (!(component instanceof SerializationHeader.Component))
+        {
+            error("sstable %s: Expected %s, but got %s from metadata.get(MetadataType.HEADER)",
+                  desc,
+                  SerializationHeader.Component.class.getName(),
+                  component != null ? component.getClass().getName() : "'null'");
+            return;
+        }
+        SerializationHeader.Component header = (SerializationHeader.Component) component;
+
+        // check partition key type
+        AbstractType<?> keyType = validatePartitionKey(desc, tableMetadata, header);
+
+        // check clustering columns
+        List<AbstractType<?>> clusteringTypes = validateClusteringColumns(desc, tableMetadata, header);
+
+        // check static and regular columns
+        LinkedHashMap<ByteBuffer, AbstractType<?>> staticColumns = validateColumns(desc, tableMetadata, header.getStaticColumns(), ColumnMetadata.Kind.STATIC);
+        LinkedHashMap<ByteBuffer, AbstractType<?>> regularColumns = validateColumns(desc, tableMetadata, header.getRegularColumns(), ColumnMetadata.Kind.REGULAR);
+
+        SerializationHeader.Component newHeader = SerializationHeader.Component.buildComponentForTools(keyType,
+                                                                                                       clusteringTypes,
+                                                                                                       staticColumns,
+                                                                                                       regularColumns,
+                                                                                                       header.getEncodingStats());
+
+        // SerializationHeader.Component has no equals(), but a "good" toString()
+        if (header.toString().equals(newHeader.toString()))
+            return;
+
+        Map<MetadataType, MetadataComponent> newMetadata = new LinkedHashMap<>(metadata);
+        newMetadata.put(MetadataType.HEADER, newHeader);
+
+        updates.add(Pair.create(desc, newMetadata));
+    }
+
+    private AbstractType<?> validatePartitionKey(Descriptor desc, TableMetadata tableMetadata, SerializationHeader.Component header)
+    {
+        boolean keyMismatch = false;
+        AbstractType<?> headerKeyType = header.getKeyType();
+        AbstractType<?> schemaKeyType = tableMetadata.partitionKeyType;
+        boolean headerKeyComposite = headerKeyType instanceof CompositeType;
+        boolean schemaKeyComposite = schemaKeyType instanceof CompositeType;
+        if (headerKeyComposite != schemaKeyComposite)
+        {
+            // one is a composite partition key, the other is not - very suspicious
+            keyMismatch = true;
+        }
+        else if (headerKeyComposite) // && schemaKeyComposite
+        {
+            // Note, the logic is similar as just calling 'fixType()' using the composite partition key,
+            // but the log messages should use the composite partition key column names.
+            List<AbstractType<?>> headerKeyComponents = ((CompositeType) headerKeyType).subTypes();
+            List<AbstractType<?>> schemaKeyComponents = ((CompositeType) schemaKeyType).subTypes();
+            if (headerKeyComponents.size() != schemaKeyComponents.size())
+            {
+                // different number of components in composite partition keys - very suspicious
+                keyMismatch = true;
+                // Just use the original type from the header. Since the number of partition key components
+                // don't match, there's nothing to meaningfully validate against.
+            }
+            else
+            {
+                // fix components in composite partition key, if necessary
+                List<AbstractType<?>> newComponents = new ArrayList<>(schemaKeyComponents.size());
+                for (int i = 0; i < schemaKeyComponents.size(); i++)
+                {
+                    AbstractType<?> headerKeyComponent = headerKeyComponents.get(i);
+                    AbstractType<?> schemaKeyComponent = schemaKeyComponents.get(i);
+                    AbstractType<?> fixedType = fixType(desc,
+                                                        tableMetadata.partitionKeyColumns().get(i).name.bytes,
+                                                        headerKeyComponent,
+                                                        schemaKeyComponent,
+                                                        false);
+                    if (fixedType == null)
+                        keyMismatch = true;
+                    else
+                        headerKeyComponent = fixedType;
+                    newComponents.add(fixType(desc,
+                                              tableMetadata.partitionKeyColumns().get(i).name.bytes,
+                                              headerKeyComponent,
+                                              schemaKeyComponent,
+                                              false));
+                }
+                headerKeyType = CompositeType.getInstance(newComponents);
+            }
+        }
+        else
+        {
+            // fix non-composite partition key, if necessary
+            AbstractType<?> fixedType = fixType(desc, tableMetadata.partitionKeyColumns().get(0).name.bytes, headerKeyType, schemaKeyType, false);
+            if (fixedType == null)
+                // non-composite partition key doesn't match and cannot be fixed
+                keyMismatch = true;
+            else
+                headerKeyType = fixedType;
+        }
+        if (keyMismatch)
+            error("sstable %s: Mismatch in partition key type between sstable serialization-header and schema (%s vs %s)",
+                  desc,
+                  headerKeyType.asCQL3Type(),
+                  schemaKeyType.asCQL3Type());
+        return headerKeyType;
+    }
+
+    private List<AbstractType<?>> validateClusteringColumns(Descriptor desc, TableMetadata tableMetadata, SerializationHeader.Component header)
+    {
+        List<AbstractType<?>> headerClusteringTypes = header.getClusteringTypes();
+        List<AbstractType<?>> clusteringTypes = new ArrayList<>();
+        boolean clusteringMismatch = false;
+        List<ColumnMetadata> schemaClustering = tableMetadata.clusteringColumns();
+        if (schemaClustering.size() != headerClusteringTypes.size())
+        {
+            clusteringMismatch = true;
+            // Just use the original types. Since the number of clustering columns don't match, there's nothing to
+            // meaningfully validate against.
+            clusteringTypes.addAll(headerClusteringTypes);
+        }
+        else
+        {
+            for (int i = 0; i < headerClusteringTypes.size(); i++)
+            {
+                AbstractType<?> headerType = headerClusteringTypes.get(i);
+                ColumnMetadata column = schemaClustering.get(i);
+                AbstractType<?> schemaType = column.type;
+                AbstractType<?> fixedType = fixType(desc, column.name.bytes, headerType, schemaType, false);
+                if (fixedType == null)
+                    clusteringMismatch = true;
+                else
+                    headerType = fixedType;
+                clusteringTypes.add(headerType);
+            }
+        }
+        if (clusteringMismatch)
+            error("sstable %s: mismatch in clustering columns between sstable serialization-header and schema (%s vs %s)",
+                  desc,
+                  headerClusteringTypes.stream().map(AbstractType::asCQL3Type).map(CQL3Type::toString).collect(Collectors.joining(",")),
+                  schemaClustering.stream().map(cd -> cd.type.asCQL3Type().toString()).collect(Collectors.joining(",")));
+        return clusteringTypes;
+    }
+
+    private LinkedHashMap<ByteBuffer, AbstractType<?>> validateColumns(Descriptor desc, TableMetadata tableMetadata, Map<ByteBuffer, AbstractType<?>> columns, ColumnMetadata.Kind kind)
+    {
+        LinkedHashMap<ByteBuffer, AbstractType<?>> target = new LinkedHashMap<>();
+        for (Map.Entry<ByteBuffer, AbstractType<?>> nameAndType : columns.entrySet())
+        {
+            ByteBuffer name = nameAndType.getKey();
+            AbstractType<?> type = nameAndType.getValue();
+
+            AbstractType<?> fixedType = validateColumn(desc, tableMetadata, kind, name, type);
+            if (fixedType == null)
+            {
+                error("sstable %s: contains column '%s' of type '%s', which could not be validated",
+                      desc,
+                      type,
+                      logColumnName(name));
+                // don't use a "null" type instance
+                fixedType = type;
+            }
+
+            target.put(name, fixedType);
+        }
+        return target;
+    }
+
+    private AbstractType<?> validateColumn(Descriptor desc, TableMetadata tableMetadata, ColumnMetadata.Kind kind, ByteBuffer name, AbstractType<?> type)
+    {
+        ColumnMetadata cd = tableMetadata.getColumn(name);
+        if (cd == null)
+        {
+            // In case the column was dropped, there is not much that we can actually validate.
+            // The column could have been recreated using the same or a different kind or the same or
+            // a different type. Lottery...
+
+            cd = tableMetadata.getDroppedColumn(name, kind == ColumnMetadata.Kind.STATIC);
+            if (cd == null)
+            {
+                for (IndexMetadata indexMetadata : tableMetadata.indexes)
+                {
+                    String target = indexMetadata.options.get(IndexTarget.TARGET_OPTION_NAME);
+                    if (target != null && ByteBufferUtil.bytes(target).equals(name))
+                    {
+                        warn.accept(String.format("sstable %s: contains column '%s', which is not a column in the table '%s.%s', but a target for that table's index '%s'",
+                                desc,
+                                logColumnName(name),
+                                tableMetadata.keyspace,
+                                tableMetadata.name,
+                                indexMetadata.name));
+                        return type;
+                    }
+                }
+
+                warn.accept(String.format("sstable %s: contains column '%s', which is not present in the schema",
+                                          desc,
+                                          logColumnName(name)));
+            }
+            else
+            {
+                // This is a best-effort approach to handle the case of a UDT column created *AND* dropped in
+                // C* 3.0.
+                if (type instanceof UserType && cd.type instanceof TupleType)
+                {
+                    // At this point, we know that the type belongs to a dropped column, recorded with the
+                    // dropped column type "TupleType" and using "UserType" in the sstable. So it is very
+                    // likely, that this belongs to a dropped UDT. Fix that information to tuple-type.
+                    return fixType(desc, name, type, cd.type, true);
+                }
+            }
+
+            return type;
+        }
+
+        // At this point, the column name is known to be a "non-dropped" column in the table.
+        if (cd.kind != kind)
+            error("sstable %s: contains column '%s' as a %s column, but is of kind %s in the schema",
+                  desc,
+                  logColumnName(name),
+                  kind.name().toLowerCase(),
+                  cd.kind.name().toLowerCase());
+        else
+            type = fixType(desc, name, type, cd.type, false);
+        return type;
+    }
+
+    private AbstractType<?> fixType(Descriptor desc, ByteBuffer name, AbstractType<?> typeInHeader, AbstractType<?> typeInSchema, boolean droppedColumnMode)
+    {
+        AbstractType<?> fixedType = fixTypeInner(typeInHeader, typeInSchema, droppedColumnMode);
+        if (fixedType != null)
+        {
+            if (fixedType != typeInHeader)
+                info.accept(String.format("sstable %s: Column '%s' needs to be updated from type '%s' to '%s'",
+                                          desc,
+                                          logColumnName(name),
+                                          typeInHeader.asCQL3Type(),
+                                          fixedType.asCQL3Type()));
+            return fixedType;
+        }
+
+        error("sstable %s: contains column '%s' as type '%s', but schema mentions '%s'",
+              desc,
+              logColumnName(name),
+              typeInHeader.asCQL3Type(),
+              typeInSchema.asCQL3Type());
+
+        return typeInHeader;
+    }
+
+    private AbstractType<?> fixTypeInner(AbstractType<?> typeInHeader, AbstractType<?> typeInSchema, boolean droppedColumnMode)
+    {
+        if (typeEquals(typeInHeader, typeInSchema))
+            return typeInHeader;
+
+        if (typeInHeader instanceof CollectionType)
+            return fixTypeInnerCollection(typeInHeader, typeInSchema, droppedColumnMode);
+
+        if (typeInHeader instanceof AbstractCompositeType)
+            return fixTypeInnerAbstractComposite(typeInHeader, typeInSchema, droppedColumnMode);
+
+        if (typeInHeader instanceof TupleType)
+            return fixTypeInnerAbstractTuple(typeInHeader, typeInSchema, droppedColumnMode);
+
+        // all types, beside CollectionType + AbstractCompositeType + TupleType, should be ok (no nested types) - just check for compatibility
+        if (typeInHeader.isCompatibleWith(typeInSchema))
+            return typeInHeader;
+
+        return null;
+    }
+
+    private AbstractType<?> fixTypeInnerAbstractTuple(AbstractType<?> typeInHeader, AbstractType<?> typeInSchema, boolean droppedColumnMode)
+    {
+        // This first 'if' handles the case when a UDT has been dropped, as a dropped UDT is recorded as a tuple
+        // in dropped_columns. If a UDT is to be replaced with a tuple, then also do that for the inner UDTs.
+        if (droppedColumnMode && typeInHeader.getClass() == UserType.class && typeInSchema instanceof TupleType)
+            return fixTypeInnerUserTypeDropped((UserType) typeInHeader, (TupleType) typeInSchema);
+
+        if (typeInHeader.getClass() != typeInSchema.getClass())
+            return null;
+
+        if (typeInHeader.getClass() == UserType.class)
+            return fixTypeInnerUserType((UserType) typeInHeader, (UserType) typeInSchema);
+
+        if (typeInHeader.getClass() == TupleType.class)
+            return fixTypeInnerTuple((TupleType) typeInHeader, (TupleType) typeInSchema, droppedColumnMode);
+
+        throw new IllegalArgumentException("Unknown tuple type class " + typeInHeader.getClass().getName());
+    }
+
+    private AbstractType<?> fixTypeInnerCollection(AbstractType<?> typeInHeader, AbstractType<?> typeInSchema, boolean droppedColumnMode)
+    {
+        if (typeInHeader.getClass() != typeInSchema.getClass())
+            return null;
+
+        if (typeInHeader.getClass() == ListType.class)
+            return fixTypeInnerList((ListType<?>) typeInHeader, (ListType<?>) typeInSchema, droppedColumnMode);
+
+        if (typeInHeader.getClass() == SetType.class)
+            return fixTypeInnerSet((SetType<?>) typeInHeader, (SetType<?>) typeInSchema, droppedColumnMode);
+
+        if (typeInHeader.getClass() == MapType.class)
+            return fixTypeInnerMap((MapType<?, ?>) typeInHeader, (MapType<?, ?>) typeInSchema, droppedColumnMode);
+
+        throw new IllegalArgumentException("Unknown collection type class " + typeInHeader.getClass().getName());
+    }
+
+    private AbstractType<?> fixTypeInnerAbstractComposite(AbstractType<?> typeInHeader, AbstractType<?> typeInSchema, boolean droppedColumnMode)
+    {
+        if (typeInHeader.getClass() != typeInSchema.getClass())
+            return null;
+
+        if (typeInHeader.getClass() == CompositeType.class)
+            return fixTypeInnerComposite((CompositeType) typeInHeader, (CompositeType) typeInSchema, droppedColumnMode);
+
+        if (typeInHeader.getClass() == DynamicCompositeType.class)
+        {
+            // Not sure if we should care about UDTs in DynamicCompositeType at all...
+            if (!typeInHeader.isCompatibleWith(typeInSchema))
+                return null;
+
+            return typeInHeader;
+        }
+
+        throw new IllegalArgumentException("Unknown composite type class " + typeInHeader.getClass().getName());
+    }
+
+    private AbstractType<?> fixTypeInnerUserType(UserType cHeader, UserType cSchema)
+    {
+        if (!cHeader.keyspace.equals(cSchema.keyspace) || !cHeader.name.equals(cSchema.name))
+            // different UDT - bummer...
+            return null;
+
+        if (cHeader.isMultiCell() != cSchema.isMultiCell())
+        {
+            if (cHeader.isMultiCell() && !cSchema.isMultiCell())
+            {
+                // C* 3.0 writes broken SerializationHeader.Component instances - i.e. broken UDT type
+                // definitions into the sstable -Stats.db file, because 3.0 does not enclose frozen UDTs
+                // (and all UDTs in 3.0 were frozen) with an '' bracket. Since CASSANDRA-7423 (support
+                // for non-frozen UDTs, committed to C* 3.6), that frozen-bracket is quite important.
+                // Non-frozen (= multi-cell) UDTs are serialized in a fundamentally different way than
+                // frozen UDTs in sstables - most importantly, the order of serialized columns depends on
+                // the type: fixed-width types first, then variable length types (like frozen types),
+                // multi-cell types last. If C* >= 3.6 reads an sstable with a UDT that's written by
+                // C* < 3.6, a variety of CorruptSSTableExceptions get logged and clients will encounter
+                // read errors.
+                // At this point, we know that the type belongs to a "live" (non-dropped) column, so it
+                // is safe to correct the information from the header.
+                return cSchema;
+            }
+
+            // In all other cases, there's not much we can do.
+            return null;
+        }
+
+        return cHeader;
+    }
+
+    private AbstractType<?> fixTypeInnerUserTypeDropped(UserType cHeader, TupleType cSchema)
+    {
+        // Do not mess around with the UserType in the serialization header, if the column has been dropped.
+        // Only fix the multi-cell status when the header contains it as a multicell (non-frozen) UserType,
+        // but the schema says "frozen".
+        if (cHeader.isMultiCell() && !cSchema.isMultiCell())
+        {
+            return new UserType(cHeader.keyspace, cHeader.name, cHeader.fieldNames(), cHeader.fieldTypes(), cSchema.isMultiCell());
+        }
+
+        return cHeader;
+    }
+
+    private AbstractType<?> fixTypeInnerTuple(TupleType cHeader, TupleType cSchema, boolean droppedColumnMode)
+    {
+        if (cHeader.size() != cSchema.size())
+            // different number of components - bummer...
+            return null;
+        List<AbstractType<?>> cHeaderFixed = new ArrayList<>(cHeader.size());
+        boolean anyChanged = false;
+        for (int i = 0; i < cHeader.size(); i++)
+        {
+            AbstractType<?> cHeaderComp = cHeader.type(i);
+            AbstractType<?> cHeaderCompFixed = fixTypeInner(cHeaderComp, cSchema.type(i), droppedColumnMode);
+            if (cHeaderCompFixed == null)
+                // incompatible, bummer...
+                return null;
+            cHeaderFixed.add(cHeaderCompFixed);
+            anyChanged |= cHeaderComp != cHeaderCompFixed;
+        }
+        if (anyChanged || cSchema.isMultiCell() != cHeader.isMultiCell())
+            // TODO this should create a non-frozen tuple type for the sake of handling a dropped, non-frozen UDT
+            return new TupleType(cHeaderFixed);
+        return cHeader;
+    }
+
+    private AbstractType<?> fixTypeInnerComposite(CompositeType cHeader, CompositeType cSchema, boolean droppedColumnMode)
+    {
+        if (cHeader.subTypes().size() != cSchema.subTypes().size())
+            // different number of components - bummer...
+            return null;
+        List<AbstractType<?>> cHeaderFixed = new ArrayList<>(cHeader.subTypes().size());
+        boolean anyChanged = false;
+        for (int i = 0; i < cHeader.subTypes().size(); i++)
+        {
+            AbstractType<?> cHeaderComp = cHeader.subTypes().get(i);
+            AbstractType<?> cHeaderCompFixed = fixTypeInner(cHeaderComp, cSchema.subTypes().get(i), droppedColumnMode);
+            if (cHeaderCompFixed == null)
+                // incompatible, bummer...
+                return null;
+            cHeaderFixed.add(cHeaderCompFixed);
+            anyChanged |= cHeaderComp != cHeaderCompFixed;
+        }
+        if (anyChanged)
+            return CompositeType.getInstance(cHeaderFixed);
+        return cHeader;
+    }
+
+    private AbstractType<?> fixTypeInnerList(ListType<?> cHeader, ListType<?> cSchema, boolean droppedColumnMode)
+    {
+        AbstractType<?> cHeaderElem = cHeader.getElementsType();
+        AbstractType<?> cHeaderElemFixed = fixTypeInner(cHeaderElem, cSchema.getElementsType(), droppedColumnMode);
+        if (cHeaderElemFixed == null)
+            // bummer...
+            return null;
+        if (cHeaderElem != cHeaderElemFixed)
+            // element type changed
+            return ListType.getInstance(cHeaderElemFixed, cHeader.isMultiCell());
+        return cHeader;
+    }
+
+    private AbstractType<?> fixTypeInnerSet(SetType<?> cHeader, SetType<?> cSchema, boolean droppedColumnMode)
+    {
+        AbstractType<?> cHeaderElem = cHeader.getElementsType();
+        AbstractType<?> cHeaderElemFixed = fixTypeInner(cHeaderElem, cSchema.getElementsType(), droppedColumnMode);
+        if (cHeaderElemFixed == null)
+            // bummer...
+            return null;
+        if (cHeaderElem != cHeaderElemFixed)
+            // element type changed
+            return SetType.getInstance(cHeaderElemFixed, cHeader.isMultiCell());
+        return cHeader;
+    }
+
+    private AbstractType<?> fixTypeInnerMap(MapType<?, ?> cHeader, MapType<?, ?> cSchema, boolean droppedColumnMode)
+    {
+        AbstractType<?> cHeaderKey = cHeader.getKeysType();
+        AbstractType<?> cHeaderVal = cHeader.getValuesType();
+        AbstractType<?> cHeaderKeyFixed = fixTypeInner(cHeaderKey, cSchema.getKeysType(), droppedColumnMode);
+        AbstractType<?> cHeaderValFixed = fixTypeInner(cHeaderVal, cSchema.getValuesType(), droppedColumnMode);
+        if (cHeaderKeyFixed == null || cHeaderValFixed == null)
+            // bummer...
+            return null;
+        if (cHeaderKey != cHeaderKeyFixed || cHeaderVal != cHeaderValFixed)
+            // element type changed
+            return MapType.getInstance(cHeaderKeyFixed, cHeaderValFixed, cHeader.isMultiCell());
+        return cHeader;
+    }
+
+    private boolean typeEquals(AbstractType<?> typeInHeader, AbstractType<?> typeInSchema)
+    {
+        // Quite annoying, but the implementations of equals() on some implementation of AbstractType seems to be
+        // wrong, but toString() seems to work in such cases.
+        return typeInHeader.equals(typeInSchema) || typeInHeader.toString().equals(typeInSchema.toString());
+    }
+
+    private static String logColumnName(ByteBuffer columnName)
+    {
+        try
+        {
+            return ByteBufferUtil.string(columnName);
+        }
+        catch (CharacterCodingException e)
+        {
+            return "?? " + e;
+        }
+    }
+
+    private Map<MetadataType, MetadataComponent> readSSTableMetadata(Descriptor desc)
+    {
+        Map<MetadataType, MetadataComponent> metadata;
+        try
+        {
+            metadata = desc.getMetadataSerializer().deserialize(desc, EnumSet.allOf(MetadataType.class));
+        }
+        catch (IOException e)
+        {
+            error("Failed to deserialize metadata for sstable %s: %s", desc, e.toString());
+            return null;
+        }
+        return metadata;
+    }
+
+    private void writeNewMetadata(Descriptor desc, Map<MetadataType, MetadataComponent> newMetadata)
+    {
+        File file = desc.fileFor(SSTableFormat.Components.STATS);
+        info.accept(String.format("  Writing new metadata file %s", file));
+        try
+        {
+            desc.getMetadataSerializer().rewriteSSTableMetadata(desc, newMetadata);
+        }
+        catch (IOException e)
+        {
+            error("Failed to write metadata component for %s: %s", file, e.toString());
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Fix individually provided sstables or directories containing sstables.
+     */
+    static class ManualHeaderFix extends SSTableHeaderFix
+    {
+        private final List<Path> paths;
+
+        ManualHeaderFix(Builder builder)
+        {
+            super(builder);
+            this.paths = builder.paths;
+        }
+
+        public void prepare()
+        {
+            paths.forEach(this::processFileOrDirectory);
+        }
+    }
+
+    /**
+     * Fix all sstables in the configured data-directories.
+     */
+    static class AutomaticHeaderFix extends SSTableHeaderFix
+    {
+        AutomaticHeaderFix(Builder builder)
+        {
+            super(builder);
+        }
+
+        public void prepare()
+        {
+            info.accept("Scanning all data directories...");
+            for (Directories.DataDirectory dataDirectory : Directories.dataDirectories)
+                scanDataDirectory(dataDirectory);
+            info.accept("Finished scanning all data directories...");
+        }
+
+        private void scanDataDirectory(Directories.DataDirectory dataDirectory)
+        {
+            info.accept(String.format("Scanning data directory %s", dataDirectory.location));
+            File[] ksDirs = dataDirectory.location.tryList();
+            if (ksDirs == null)
+                return;
+            for (File ksDir : ksDirs)
+            {
+                if (!ksDir.isDirectory() || !ksDir.isReadable())
+                    continue;
+
+                String name = ksDir.name();
+
+                // silently ignore all system keyspaces
+                if (SchemaConstants.isLocalSystemKeyspace(name) || SchemaConstants.isReplicatedSystemKeyspace(name))
+                    continue;
+
+                File[] tabDirs = ksDir.tryList();
+                if (tabDirs == null)
+                    continue;
+                for (File tabDir : tabDirs)
+                {
+                    if (!tabDir.isDirectory() || !tabDir.isReadable())
+                        continue;
+
+                    processFileOrDirectory(tabDir.toPath());
+                }
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableId.java b/src/java/org/apache/cassandra/io/sstable/SSTableId.java
index 7a2235b8f995..6be11ff355bd 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableId.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableId.java
@@ -37,7 +37,7 @@
  * - must be case-insensitive because the sstables can be stored on case-insensitive file system
  * <p>
  */
-public interface SSTableId
+public interface SSTableId<T> extends Comparable<T>
 {
     /**
      * Creates a byte format of the identifier that can be parsed by
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableIdFactory.java b/src/java/org/apache/cassandra/io/sstable/SSTableIdFactory.java
index d5b3276ed523..66a41fb030f2 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableIdFactory.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableIdFactory.java
@@ -22,12 +22,34 @@
 import java.util.Comparator;
 import java.util.stream.Stream;
 
+import org.apache.commons.lang3.tuple.Pair;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.TimeUUID;
 
 public class SSTableIdFactory
 {
     public static final SSTableIdFactory instance = new SSTableIdFactory();
 
+    private static boolean isULIDImpl()
+    {
+        String impl = CassandraRelevantProperties.SSTABLE_UUID_IMPL.getString().toLowerCase();
+        if ("uuid".equals(impl))
+            return false;
+        else if ("ulid".equals(impl))
+            return true;
+        else
+            throw new IllegalArgumentException("Unsupported value for property " + CassandraRelevantProperties.SSTABLE_UUID_IMPL.getKey() + ": " + impl);
+    }
+
+    private Stream<SSTableId.Builder<?>> makeIdBuildersStream()
+    {
+        return isULIDImpl()
+               ? Stream.of(ULIDBasedSSTableId.Builder.instance, UUIDBasedSSTableId.Builder.instance, SequenceBasedSSTableId.Builder.instance)
+               : Stream.of(UUIDBasedSSTableId.Builder.instance, ULIDBasedSSTableId.Builder.instance, SequenceBasedSSTableId.Builder.instance);
+    }
+
     /**
      * Constructs the instance of {@link SSTableId} from the given string representation.
      * It finds the right builder by verifying whether the given string is the representation of the related identifier
@@ -37,18 +59,17 @@ public class SSTableIdFactory
      */
     public SSTableId fromString(String str) throws IllegalArgumentException
     {
-        return Stream.of(UUIDBasedSSTableId.Builder.instance, SequenceBasedSSTableId.Builder.instance)
-                     .filter(b -> b.isUniqueIdentifier(str))
-                     .findFirst()
-                     .map(b -> b.fromString(str))
-                     .orElseThrow(() -> new IllegalArgumentException("String '" + str + "' does not match any SSTable identifier format"));
+        return makeIdBuildersStream().filter(b -> b.isUniqueIdentifier(str))
+                                     .findFirst()
+                                     .map(b -> b.fromString(str))
+                                     .orElseThrow(() -> new IllegalArgumentException("String '" + str + "' does not match any SSTable identifier format"));
     }
 
     /**
      * Constructs the instance of {@link SSTableId} from the given bytes.
      * It finds the right builder by verifying whether the given buffer is the representation of the related identifier
      * type using {@link SSTableId.Builder#isUniqueIdentifier(ByteBuffer)} method.
-     *
+     * <p>
      * The method expects the identifier is encoded in all remaining bytes of the buffer. The method does not move the
      * pointer of the buffer.
      *
@@ -56,42 +77,39 @@ public SSTableId fromString(String str) throws IllegalArgumentException
      */
     public SSTableId fromBytes(ByteBuffer bytes)
     {
-        return Stream.of(UUIDBasedSSTableId.Builder.instance, SequenceBasedSSTableId.Builder.instance)
-                     .filter(b -> b.isUniqueIdentifier(bytes))
-                     .findFirst()
-                     .map(b -> b.fromBytes(bytes))
-                     .orElseThrow(() -> new IllegalArgumentException("Byte buffer of length " + bytes.remaining() + " does not match any SSTable identifier format"));
+        return makeIdBuildersStream().filter(b -> b.isUniqueIdentifier(bytes))
+                                     .findFirst()
+                                     .map(b -> b.fromBytes(bytes))
+                                     .orElseThrow(() -> new IllegalArgumentException("Byte buffer of length " + bytes.remaining() + " does not match any SSTable identifier format"));
     }
 
     /**
      * Returns default identifiers builder.
      */
-    @SuppressWarnings("unchecked")
-    public SSTableId.Builder<SSTableId> defaultBuilder()
+    public SSTableId.Builder<? extends SSTableId> defaultBuilder()
     {
-        SSTableId.Builder<? extends SSTableId> builder = DatabaseDescriptor.isUUIDSSTableIdentifiersEnabled()
-                                                         ? UUIDBasedSSTableId.Builder.instance
-                                                         : SequenceBasedSSTableId.Builder.instance;
-        return (SSTableId.Builder<SSTableId>) builder;
+        if (DatabaseDescriptor.isUUIDSSTableIdentifiersEnabled())
+            return isULIDImpl()
+                   ? ULIDBasedSSTableId.Builder.instance
+                   : UUIDBasedSSTableId.Builder.instance;
+        else
+            return SequenceBasedSSTableId.Builder.instance;
     }
 
     /**
      * Compare sstable identifiers so that UUID based identifier is always greater than sequence based identifier
      */
-    public final static Comparator<SSTableId> COMPARATOR = Comparator.nullsFirst((id1, id2) -> {
-        if (id1 instanceof UUIDBasedSSTableId)
-        {
-            UUIDBasedSSTableId uuidId1 = (UUIDBasedSSTableId) id1;
-            return (id2 instanceof UUIDBasedSSTableId) ? uuidId1.compareTo((UUIDBasedSSTableId) id2) : 1;
-        }
-        else if (id1 instanceof SequenceBasedSSTableId)
-        {
-            SequenceBasedSSTableId seqId1 = (SequenceBasedSSTableId) id1;
-            return (id2 instanceof SequenceBasedSSTableId) ? seqId1.compareTo((SequenceBasedSSTableId) id2) : -1;
-        }
+    public static final Comparator<SSTableId> COMPARATOR = Comparator.nullsFirst(Comparator.comparing(SSTableIdFactory::asTimeUUID));
+
+    private static Pair<TimeUUID, Integer> asTimeUUID(SSTableId id)
+    {
+        if (id instanceof UUIDBasedSSTableId)
+            return Pair.of(((UUIDBasedSSTableId) id).uuid, null);
+        else if (id instanceof ULIDBasedSSTableId)
+            return Pair.of(((ULIDBasedSSTableId) id).approximateTimeUUID, null);
+        else if (id instanceof SequenceBasedSSTableId)
+            return Pair.of(null, ((SequenceBasedSSTableId) id).generation);
         else
-        {
-            throw new AssertionError("Unsupported comparison between " + id1.getClass().getName() + " and  " + id2.getClass().getName());
-        }
-    });
+            throw new AssertionError("Unsupported sstable identifier type " + id.getClass().getName());
+    }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
index cc201b4125c5..df8a2253f3ff 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
@@ -30,6 +30,7 @@
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.schema.TableMetadata;
@@ -42,19 +43,20 @@ public class SSTableIdentityIterator implements Comparable<SSTableIdentityIterat
     private final SSTableReader sstable;
     private final DecoratedKey key;
     private final DeletionTime partitionLevelDeletion;
-    private final String filename;
+    private final File file;
 
     protected final SSTableSimpleIterator iterator;
     private final Row staticRow;
+    boolean isClosed = false;
 
     public SSTableIdentityIterator(SSTableReader sstable, DecoratedKey key, DeletionTime partitionLevelDeletion,
-            String filename, SSTableSimpleIterator iterator) throws IOException
+                                   File file, SSTableSimpleIterator iterator) throws IOException
     {
         super();
         this.sstable = sstable;
         this.key = key;
         this.partitionLevelDeletion = partitionLevelDeletion;
-        this.filename = filename;
+        this.file = file;
         this.iterator = iterator;
         this.staticRow = iterator.readStaticRow();
     }
@@ -68,12 +70,12 @@ public static SSTableIdentityIterator create(SSTableReader sstable, RandomAccess
                 UnfilteredValidation.handleInvalid(sstable.metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
             DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
             SSTableSimpleIterator iterator = SSTableSimpleIterator.create(sstable.metadata(), file, sstable.header, helper, partitionLevelDeletion);
-            return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, file.getPath(), iterator);
+            return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, file.getFile(), iterator);
         }
         catch (IOException e)
         {
             sstable.markSuspect();
-            throw new CorruptSSTableException(e, file.getPath());
+            throw new CorruptSSTableException(e, file.getFile());
         }
     }
 
@@ -91,12 +93,35 @@ public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInpu
             SSTableSimpleIterator iterator = tombstoneOnly
                     ? SSTableSimpleIterator.createTombstoneOnly(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion)
                     : SSTableSimpleIterator.create(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion);
-            return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, dfile.getPath(), iterator);
+            return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, dfile.getFile(), iterator);
         }
         catch (IOException e)
         {
             sstable.markSuspect();
-            throw new CorruptSSTableException(e, dfile.getPath());
+            throw new CorruptSSTableException(e, dfile.getFile());
+        }
+    }
+
+    public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInput dfile, boolean tombstoneOnly)
+    {
+        try
+        {
+            DecoratedKey key = sstable.decorateKey(ByteBufferUtil.readWithShortLength(dfile));
+//            DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(dfile);
+            DeletionTime partitionLevelDeletion = DeletionTime.getSerializer(sstable.descriptor.version).deserialize(dfile);
+            if (!partitionLevelDeletion.validate())
+                UnfilteredValidation.handleInvalid(sstable.metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
+
+            DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
+            SSTableSimpleIterator iterator = tombstoneOnly
+                                             ? SSTableSimpleIterator.createTombstoneOnly(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion)
+                                             : SSTableSimpleIterator.create(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion);
+            return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, dfile.getFile(), iterator);
+        }
+        catch (IOException e)
+        {
+            sstable.markSuspect();
+            throw new CorruptSSTableException(e, dfile.getFile());
         }
     }
 
@@ -139,14 +164,14 @@ public boolean hasNext()
         catch (IndexOutOfBoundsException | VIntOutOfRangeException | AssertionError e)
         {
             sstable.markSuspect();
-            throw new CorruptSSTableException(e, filename);
+            throw new CorruptSSTableException(e, file);
         }
         catch (IOError e)
         {
             if (e.getCause() instanceof IOException)
             {
                 sstable.markSuspect();
-                throw new CorruptSSTableException((Exception)e.getCause(), filename);
+                throw new CorruptSSTableException(e.getCause(), file);
             }
             else
             {
@@ -159,19 +184,22 @@ public Unfiltered next()
     {
         try
         {
+            if (isClosed)
+                throw new IllegalStateException("Iterator used after closing.");
+
             return doCompute();
         }
         catch (IndexOutOfBoundsException | VIntOutOfRangeException | AssertionError e)
         {
             sstable.markSuspect();
-            throw new CorruptSSTableException(e, filename);
+            throw new CorruptSSTableException(e, file);
         }
         catch (IOError e)
         {
             if (e.getCause() instanceof IOException)
             {
                 sstable.markSuspect();
-                throw new CorruptSSTableException((Exception)e.getCause(), filename);
+                throw new CorruptSSTableException((Exception)e.getCause(), file);
             }
             else
             {
@@ -190,11 +218,47 @@ protected Unfiltered doCompute()
     public void close()
     {
         // creator is responsible for closing file when finished
+        isClosed = true;
+    }
+
+    public boolean isClosed()
+    {
+        return isClosed;
+    }
+
+    /**
+     * Called to advance to the next partition and make sure that we process all outstanding rows if user did not
+     * do so. Unlike next() and hasNext(), this can and will be called after the iterator is closed.
+     */
+    public void exhaust()
+    {
+        try
+        {
+            while (iterator.hasNext())
+                iterator.next();
+        }
+        catch (IndexOutOfBoundsException | VIntOutOfRangeException | AssertionError e)
+        {
+            sstable.markSuspect();
+            throw new CorruptSSTableException(e, file);
+        }
+        catch (IOError e)
+        {
+            if (e.getCause() instanceof IOException)
+            {
+                sstable.markSuspect();
+                throw new CorruptSSTableException((Exception)e.getCause(), file);
+            }
+            else
+            {
+                throw e;
+            }
+        }
     }
 
-    public String getPath()
+    public File getFile()
     {
-        return filename;
+        return file;
     }
 
     public EncodingStats stats()
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java
index 9a7968071b25..bc1195b1cfb5 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java
@@ -39,11 +39,18 @@ public interface SSTableMultiWriter extends Transactional
     Collection<SSTableReader> finish(boolean openResult);
     Collection<SSTableReader> finished();
 
-    SSTableMultiWriter setOpenResult(boolean openResult);
+    /**
+     * Opens the resulting sstables after writing has finished. If those readers need to be accessed, then this must
+     * be called after `prepareToCommit` (so the writing is complete) but before `commit` (because committing closes
+     * some of the resources used to create the underlying readers). When used, the readers can then be accessed by
+     * calling `finished()`.
+     */
+    void openResult();
 
     String getFilename();
     long getBytesWritten();
     long getOnDiskBytesWritten();
+    int getSegmentCount();
     TableId getTableId();
 
     static void abortOrDie(SSTableMultiWriter writer)
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java b/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java
index 6b494d6ab0ff..59200b168cbf 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java
@@ -84,11 +84,18 @@ default void onSSTableSkipped(SSTableReader sstable, SkippingReason reason)
     {
     }
 
+    /**
+     * Handles notification that the specified SSTable has been selected for searching its partition index.
+     * @param sstable the sstable reader
+     */
+    default void onSSTablePartitionIndexAccessed(SSTableReader sstable)
+    {
+    }
+
     /**
      * Handles notification that the specified SSTable has been selected during a single partition query.
      *
      * @param sstable the SSTable reader
-     * @param indexEntry the index entry
      * @param reason the reason for which the SSTable has been selected
      */
     default void onSSTableSelected(SSTableReader sstable, SelectionReason reason)
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
index 31df8b112005..b50942520167 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.io.sstable;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
@@ -24,9 +25,12 @@
 import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.db.compaction.writers.SSTableDataSink;
 import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
@@ -47,7 +51,7 @@
  * but leave any hard-links in place for the readers we opened, and clean-up when the readers finish as we would do
  * if we had finished successfully.
  */
-public class SSTableRewriter extends Transactional.AbstractTransactional implements Transactional
+public class SSTableRewriter extends Transactional.AbstractTransactional implements Transactional, SSTableDataSink
 {
     @VisibleForTesting
     public static boolean disableEarlyOpeningForTests = false;
@@ -97,9 +101,14 @@ public static SSTableRewriter constructWithoutEarlyOpening(ILifecycleTransaction
         return new SSTableRewriter(transaction, maxAge, calculateOpenInterval(false), keepOriginals, true);
     }
 
-    public static SSTableRewriter construct(ColumnFamilyStore cfs, ILifecycleTransaction transaction, boolean keepOriginals, long maxAge)
+    public static SSTableRewriter construct(CompactionRealm realm, ILifecycleTransaction transaction, boolean keepOriginals, long maxAge)
     {
-        return new SSTableRewriter(transaction, maxAge, calculateOpenInterval(cfs.supportsEarlyOpen()), keepOriginals, true);
+        return new SSTableRewriter(transaction, maxAge, calculateOpenInterval(realm.supportsEarlyOpen()), keepOriginals, true);
+    }
+
+    public static SSTableRewriter construct(CompactionRealm realm, ILifecycleTransaction transaction, boolean keepOriginals, long maxAge, boolean earlyOpenAllowed)
+    {
+        return new SSTableRewriter(transaction, maxAge, calculateOpenInterval(earlyOpenAllowed && realm.supportsEarlyOpen()), keepOriginals, true);
     }
 
     private static long calculateOpenInterval(boolean shouldOpenEarly)
@@ -128,6 +137,7 @@ public void forEachWriter(Consumer<SSTableWriter> op)
             op.accept(writer);
     }
 
+    @Override
     public AbstractRowIndexEntry append(UnfilteredRowIterator partition)
     {
         // we do this before appending to ensure we can resetAndTruncate() safely if appending fails
@@ -151,6 +161,25 @@ public AbstractRowIndexEntry tryAppend(UnfilteredRowIterator partition)
         }
     }
 
+    @Override
+    public boolean startPartition(DecoratedKey key, DeletionTime deletionTime) throws IOException
+    {
+        maybeReopenEarly(key);
+        return writer.startPartition(key, deletionTime);
+    }
+
+    @Override
+    public void addUnfiltered(Unfiltered unfiltered) throws IOException
+    {
+        writer.addUnfiltered(unfiltered);
+    }
+
+    @Override
+    public AbstractRowIndexEntry endPartition() throws IOException
+    {
+        return writer.endPartition();
+    }
+
     private void maybeReopenEarly(DecoratedKey key)
     {
         if (writer.getFilePointer() - currentlyOpenedEarlyAt > preemptiveOpenInterval)
@@ -328,12 +357,13 @@ protected void doPrepare()
         {
             assert writer.getFilePointer() > 0;
             writer.setRepairedAt(repairedAt);
-            writer.setOpenResult(true);
             writer.prepareToCommit();
+            writer.openResult();
             SSTableReader reader = writer.finished();
             transaction.update(reader, false);
             preparedForCommit.add(reader);
         }
+        // staged sstables will be made visible in Tracker
         transaction.checkpoint();
 
         if (throwLate)
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
index ce927d994767..a8d047a83e3f 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
@@ -27,6 +27,8 @@
 
 import io.netty.util.concurrent.FastThreadLocalThread;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RangeTombstone;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -35,6 +37,7 @@
 import org.apache.cassandra.db.rows.SerializationHelper;
 import org.apache.cassandra.db.rows.UnfilteredSerializer;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
@@ -118,15 +121,59 @@ private void maybeSync() throws SyncException
 
     private PartitionUpdate.Builder createPartitionUpdateBuilder(DecoratedKey key)
     {
-        return new PartitionUpdate.Builder(metadata.get(), key, columns, 4)
+        PartitionUpdate.Builder wrapped = PartitionUpdate.builder(metadata.get(), key, columns, 4);
+
+        return new PartitionUpdate.Builder()
         {
             @Override
             public void add(Row row)
             {
-                super.add(row);
+                wrapped.add(row);
                 countRow(row);
                 maybeSync();
             }
+
+            @Override
+            public void addPartitionDeletion(DeletionTime deletionTime)
+            {
+                wrapped.addPartitionDeletion(deletionTime);
+            }
+
+            @Override
+            public void add(RangeTombstone range)
+            {
+                wrapped.add(range);
+            }
+
+            @Override
+            public DecoratedKey partitionKey()
+            {
+                return wrapped.partitionKey();
+            }
+
+            @Override
+            public TableMetadata metadata()
+            {
+                return wrapped.metadata();
+            }
+
+            @Override
+            public PartitionUpdate build()
+            {
+                return wrapped.build();
+            }
+
+            @Override
+            public RegularAndStaticColumns columns()
+            {
+                return wrapped.columns();
+            }
+
+            @Override
+            public DeletionTime partitionLevelDeletion()
+            {
+                return wrapped.partitionLevelDeletion();
+            }
         };
     }
 
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
index ce423f2ec5b0..657d4d44cc47 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
@@ -81,7 +81,7 @@ PartitionUpdate.Builder getUpdateFor(DecoratedKey key) throws IOException
                 writePartition(update.build()); // might switch to a new sstable writer and reset currentSize
 
             currentKey = key;
-            update = new PartitionUpdate.Builder(metadata.get(), currentKey, columns, 4);
+            update = PartitionUpdate.builder(metadata.get(), currentKey, columns, 4);
         }
 
         Preconditions.checkState(update != null, "Partition update to write cannot be null");
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
index 3b43dcfdf4fb..2ab53bbf9565 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
@@ -24,6 +24,7 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -102,15 +103,17 @@ protected Throwable doPostCleanup(Throwable accumulate)
 
     public Collection<SSTableReader> finish(boolean openResult)
     {
-        writer.setOpenResult(openResult);
-        finish();
+        prepareToCommit();
+        if (openResult)
+            writer.openResult();
+        commit();
         return writer.finished();
     }
 
     @SuppressWarnings({"resource", "RedundantSuppression"}) // log and writer closed during doPostCleanup
     public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header)
     {
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE, cfs.metadata);
         SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, header, txn);
         return new SSTableTxnWriter(txn, writer);
     }
@@ -125,11 +128,11 @@ public static SSTableTxnWriter createRangeAware(TableMetadataRef metadata,
     {
 
         ColumnFamilyStore cfs = Keyspace.open(metadata.keyspace).getColumnFamilyStore(metadata.name);
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE, cfs.metadata);
         SSTableMultiWriter writer;
         try
         {
-            writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, type, 0, 0, txn, header);
+            writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, type, IntervalSet.empty(), 0, 0, txn, header);
         }
         catch (IOException e)
         {
@@ -153,8 +156,8 @@ public static SSTableTxnWriter create(TableMetadataRef metadata,
                                           SSTable.Owner owner)
     {
         // if the column family store does not exist, we create a new default SSTableMultiWriter to use:
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
-        SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, null, 0, header, indexGroups, txn, owner);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE, metadata);
+        SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, IntervalSet.empty(), 0, header, indexGroups, txn, owner);
         return new SSTableTxnWriter(txn, writer);
     }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWatcher.java b/src/java/org/apache/cassandra/io/sstable/SSTableWatcher.java
new file mode 100644
index 000000000000..41f58a49f828
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableWatcher.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.Set;
+
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_SSTABLE_WATCHER;
+
+/**
+ * Watcher used when opening sstables to discover extra components, eg. archive component
+ */
+public interface SSTableWatcher
+{
+    SSTableWatcher instance = !CUSTOM_SSTABLE_WATCHER.isPresent()
+                               ? new SSTableWatcher() {}
+                               : FBUtilities.construct(CUSTOM_SSTABLE_WATCHER.getString(), "sstable watcher");
+
+    /**
+     * Discover extra components before reading TOC file
+     *
+     * @param descriptor sstable descriptor for current sstable
+     */
+    default void discoverComponents(Descriptor descriptor)
+    {
+    }
+
+    /**
+     * Discover extra components before opening sstable
+     *
+     * @param descriptor sstable descriptor for current sstable
+     * @param existing existing sstable components
+     * @return all discovered sstable components
+     */
+    default Set<Component> discoverComponents(Descriptor descriptor, Set<Component> existing)
+    {
+        return existing;
+    }
+
+    /**
+     * Called before executing index build on existing sstable
+     */
+    default void onIndexBuild(SSTableReader sstable, Set<? extends Index> indexes)
+    {
+    }
+
+    /**
+     * Called when an index is dropped on index components affected by that drop.
+     * <p>
+     * By default, this method simply deletes the components locally, but it can overriden if different/additional
+     * behavior is needed.
+     *
+     * @param metadata table metadata of the table the index was dropped from.
+     * @param components index components that are no longer in used due to an index drop. Note that this can
+     *                   be either per-index components (for the components of the exact index being dropped),
+     *                   or per-sstable components if the index dropped was the only index for the table and the
+     *                   per-sstable components are no longer needed. More precisely, if the last index of a table
+     *                   is dropped, then this method will usually be called twice per sstable, once for the index
+     *                   components, and once for the per-sstable components.
+     */
+    default void onIndexDropped(TableMetadata metadata, IndexComponents.ForWrite components)
+    {
+        components.forceDeleteAllComponents();
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java
index 3bf21f1155ec..1e15c949ebca 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java
@@ -37,6 +37,7 @@
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.TOCComponent;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.SequentialWriter;
 import org.apache.cassandra.net.AsyncStreamingInputPlus;
@@ -51,23 +52,25 @@ public class SSTableZeroCopyWriter extends SSTable implements SSTableMultiWriter
 
     private volatile SSTableReader finalReader;
     private final Map<String, SequentialWriter> componentWriters; // indexed by component name
+    private final LifecycleNewTracker lifecycleNewTracker;
 
     public SSTableZeroCopyWriter(Builder<?, ?> builder,
                                  LifecycleNewTracker lifecycleNewTracker,
                                  SSTable.Owner owner)
     {
         super(builder, owner);
+        this.lifecycleNewTracker = lifecycleNewTracker;
 
         lifecycleNewTracker.trackNew(this);
         this.componentWriters = new HashMap<>();
 
-        Set<Component> unsupported = components.stream()
+        Set<Component> unsupported = components().stream()
                                                .filter(c -> !c.type.streamable)
                                                .collect(Collectors.toSet());
         if (!unsupported.isEmpty())
             throw new AssertionError(format("Unsupported streaming components detected: %s", unsupported));
 
-        for (Component c : components)
+        for (Component c : components())
             componentWriters.put(c.name, makeWriter(descriptor, c));
     }
 
@@ -126,11 +129,12 @@ public void append(UnfilteredRowIterator partition)
     @Override
     public Collection<SSTableReader> finish(boolean openResult)
     {
-        setOpenResult(openResult);
-
         for (SequentialWriter writer : componentWriters.values())
             writer.finish();
 
+        TOCComponent.appendTOC(descriptor, components());
+
+        lifecycleNewTracker.trackNewWritten(this);
         return finished();
     }
 
@@ -138,20 +142,20 @@ public Collection<SSTableReader> finish(boolean openResult)
     public Collection<SSTableReader> finished()
     {
         if (finalReader == null)
-            finalReader = SSTableReader.open(owner().orElse(null), descriptor, components, metadata);
+            finalReader = SSTableReader.open(owner().orElse(null), descriptor, components(), metadata);
 
         return ImmutableList.of(finalReader);
     }
 
     @Override
-    public SSTableMultiWriter setOpenResult(boolean openResult)
+    public void openResult()
     {
-        return null;
     }
 
     @Override
     public long getBytesWritten()
     {
+        // TODO: these two may need fixing.
         return 0;
     }
 
@@ -161,6 +165,12 @@ public long getOnDiskBytesWritten()
         return 0;
     }
 
+    @Override
+    public int getSegmentCount()
+    {
+        return 1;
+    }
+
     @Override
     public TableId getTableId()
     {
@@ -188,6 +198,8 @@ public void prepareToCommit()
     {
         for (SequentialWriter writer : componentWriters.values())
             writer.prepareToCommit();
+
+        lifecycleNewTracker.trackNewWritten(this);
     }
 
     @Override
@@ -200,7 +212,7 @@ public void close()
     public void writeComponent(Component component, DataInputPlus in, long size) throws ClosedChannelException
     {
         SequentialWriter writer = componentWriters.get(component.name);
-        logger.info("Writing component {} to {} length {}", component, writer.getPath(), prettyPrintMemory(size));
+        logger.info("Writing component {} to {} length {}", component, writer.getFile(), prettyPrintMemory(size));
 
         if (in instanceof AsyncStreamingInputPlus)
             write((AsyncStreamingInputPlus) in, size, writer);
@@ -210,7 +222,7 @@ public void writeComponent(Component component, DataInputPlus in, long size) thr
 
     private void write(AsyncStreamingInputPlus in, long size, SequentialWriter writer) throws ClosedChannelException
     {
-        logger.info("Block Writing component to {} length {}", writer.getPath(), prettyPrintMemory(size));
+        logger.info("Block Writing component to {} length {}", writer.getFile(), prettyPrintMemory(size));
 
         try
         {
@@ -230,7 +242,7 @@ private void write(AsyncStreamingInputPlus in, long size, SequentialWriter write
         }
         catch (IOException e)
         {
-            throw new FSWriteError(e, writer.getPath());
+            throw new FSWriteError(e, writer.getFile());
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/ScannerList.java b/src/java/org/apache/cassandra/io/sstable/ScannerList.java
new file mode 100644
index 000000000000..00683382e917
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/ScannerList.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.base.Throwables;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+
+public class ScannerList implements AutoCloseable
+{
+    public final List<ISSTableScanner> scanners;
+    public ScannerList(List<ISSTableScanner> scanners)
+    {
+        this.scanners = scanners;
+    }
+
+    public long getTotalBytesScanned()
+    {
+        long bytesScanned = 0L;
+        for (ISSTableScanner scanner : scanners)
+            bytesScanned += scanner.getBytesScanned();
+
+        return bytesScanned;
+    }
+
+    public long getTotalCompressedSize()
+    {
+        long compressedSize = 0;
+        for (int i=0, isize=scanners.size(); i<isize; i++)
+            compressedSize += scanners.get(i).getCompressedLengthInBytes();
+
+        return compressedSize;
+    }
+
+    public double getCompressionRatio()
+    {
+        double compressed = 0.0;
+        double uncompressed = 0.0;
+
+        for (int i=0, isize=scanners.size(); i<isize; i++)
+        {
+            @SuppressWarnings("resource")
+            ISSTableScanner scanner = scanners.get(i);
+            compressed += scanner.getCompressedLengthInBytes();
+            uncompressed += scanner.getLengthInBytes();
+        }
+
+        if (compressed == uncompressed || uncompressed == 0)
+            return MetadataCollector.NO_COMPRESSION_RATIO;
+
+        return compressed / uncompressed;
+    }
+
+    public void close()
+    {
+        ISSTableScanner.closeAllAndPropagate(scanners, null);
+    }
+
+    public static ScannerList of(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+    {
+        ArrayList<ISSTableScanner> scanners = new ArrayList<>();
+        try
+        {
+            for (SSTableReader sstable : sstables)
+                scanners.add(sstable.getScanner(ranges));
+            return new ScannerList(scanners);
+        }
+        catch (Throwable t)
+        {
+            throw Throwables.propagate(ISSTableScanner.closeAllAndPropagate(scanners, t));
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/SequenceBasedSSTableId.java b/src/java/org/apache/cassandra/io/sstable/SequenceBasedSSTableId.java
index acb91f8d7546..6007f62707f9 100644
--- a/src/java/org/apache/cassandra/io/sstable/SequenceBasedSSTableId.java
+++ b/src/java/org/apache/cassandra/io/sstable/SequenceBasedSSTableId.java
@@ -31,7 +31,7 @@
  * Generation identifier based on sequence of integers.
  * This has been the standard implementation in C* since inception.
  */
-public class SequenceBasedSSTableId implements SSTableId, Comparable<SequenceBasedSSTableId>
+public class SequenceBasedSSTableId implements SSTableId<SequenceBasedSSTableId>
 {
     public final int generation;
 
diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
index 99406dba31cf..a1ebb1d1d372 100644
--- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
@@ -20,9 +20,9 @@
 import java.util.Collection;
 import java.util.Collections;
 
-import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.index.Index;
@@ -59,10 +59,9 @@ public Collection<SSTableReader> finished()
         return Collections.singleton(writer.finished());
     }
 
-    public SSTableMultiWriter setOpenResult(boolean openResult)
+    public void openResult()
     {
-        writer.setOpenResult(openResult);
-        return this;
+        writer.openResult();
     }
 
     public String getFilename()
@@ -80,6 +79,11 @@ public long getOnDiskBytesWritten()
         return writer.getEstimatedOnDiskBytesWritten();
     }
 
+    public int getSegmentCount()
+    {
+        return 1;
+    }
+
     public TableId getTableId()
     {
         return writer.metadata().id;
diff --git a/src/java/org/apache/cassandra/io/sstable/StorageHandler.java b/src/java/org/apache/cassandra/io/sstable/StorageHandler.java
new file mode 100644
index 000000000000..3719f217abce
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/StorageHandler.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collection;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.REMOTE_STORAGE_HANDLER_FACTORY;
+
+/**
+ * The handler of the storage of sstables, and possibly other files such as txn logs.
+ * <p/>
+ * If sstables are stored on the local disk, then this handler is a thin wrapper of {@link Directories.SSTableLister},
+ * but for sstables stored remotely, for example on S3, then the handler may need to perform more
+ * work, such as selecting only part of the remote sstables available, or adding new ones when offline compaction
+ * has run. This behaviour can be implemented in a sub-class created from factory that can be set with {@link #remoteStorageHandlerFactory}.
+ * <p/>
+ */
+public abstract class StorageHandler
+{
+    private final static String remoteStorageHandlerFactory = REMOTE_STORAGE_HANDLER_FACTORY.getString();
+
+    private static class InstanceHolder
+    {
+        private static final StorageHandlerFactory FACTORY = maybeInitializeFactory(remoteStorageHandlerFactory);
+    }
+
+    public enum ReloadReason
+    {
+        /** New nodes joined or left */
+        TOPOLOGY_CHANGED(true),
+        /** Data was truncated */
+        TRUNCATION(false),
+        /** SSTables might have been added or removed, regardless of a specific reason
+         * e.g. it could be compaction or flushing or regions being updated which caused
+         * new sstables to arrive */
+        SSTABLES_CHANGED(false),
+        /** Data was replayed either from the commit log or a batch log */
+        DATA_REPLAYED(true),
+        /** When repair task started */
+        REPAIR(true),
+        /** A request over forced by users to reload. */
+        USER_REQUESTED(true),
+        /** When region status changed */
+        REGION_CHANGED(false),
+        /** When index is built */
+        INDEX_BUILT(false),
+        /** New node restarted with existing on disk data */
+        REPLACE(true),
+        /** Retry in case of failure, i.e. if a timeout occurred **/
+        RETRY(false);
+
+        /** When this is true, a reload operation will reload all sstables even those that could
+         * have been flushed by other nodes. */
+        public final boolean loadFlushedSSTables;
+
+        ReloadReason(boolean loadFlushedSSTables)
+        {
+            this.loadFlushedSSTables = loadFlushedSSTables;
+        }
+    }
+
+    protected final SSTable.Owner owner;
+    protected final TableMetadataRef metadata;
+    protected final Directories directories;
+    protected final Tracker dataTracker;
+
+    public StorageHandler(SSTable.Owner owner, TableMetadataRef metadata, Directories directories, Tracker dataTracker)
+    {
+        Preconditions.checkNotNull(directories, "Directories should not be null");
+
+        this.owner = owner;
+        this.metadata = metadata;
+        this.directories = directories;
+        this.dataTracker = dataTracker;
+    }
+
+    /**
+     * @return true if the node is ready to serve data for this table. This means that the
+     *         node is not bootstrapping and that no data may be missing, e.g. if sstables are
+     *         being downloaded from remote storage or streamed from other nodes then isReady()
+     *         would return false. Generally, user read queries should not succeed if this method
+     *         returns false.
+     */
+    public abstract boolean isReady();
+
+    /**
+     * Load the initial sstables into the tracker that was passed in to the constructor.
+     *
+     * @return the sstables that were loaded
+     */
+    public abstract Collection<SSTableReader> loadInitialSSTables();
+
+    /**
+     * Reload any sstables that may have been created and not yet loaded. This is normally
+     * a no-op for the default local storage, but for remote storage implementations it
+     * signals that sstables need to be refreshed.
+     *
+     * @return the sstables that were loaded
+     */
+    public abstract Collection<SSTableReader> reloadSSTables(ReloadReason reason);
+
+    /**
+     * This method determines if the backing storage handler allows auto compaction
+     * <p/>
+     * @return true if auto compaction should be enabled
+     */
+    public abstract boolean enableAutoCompaction();
+
+    /**
+     * This method will run the operation specified by the {@link Runnable} passed it
+     * whilst guaranteeing the guarantees that no sstable will be loaded or unloaded
+     * whilst this operation is running, by waiting for in-progress operation to complete.
+     * In other words, the storage handler must not change the status of the tracker,
+     * or try to load any sstable as long as this operation is executing.
+     *
+     * @param runnable the operation to execute.
+     */
+    public abstract void runWithReloadingDisabled(Runnable runnable);
+
+    /**
+     * Called when the CFS is unloaded, this needs to perform any cleanup.
+     */
+    public abstract void unload();
+
+    public static StorageHandler create(SSTable.Owner owner, TableMetadataRef metadata, Directories directories, Tracker dataTracker)
+    {
+        return InstanceHolder.FACTORY.create(owner, metadata, directories, dataTracker);
+    }
+
+    private static StorageHandlerFactory maybeInitializeFactory(String factory)
+    {
+        if (factory == null)
+            return StorageHandlerFactory.DEFAULT;
+
+        Class<StorageHandlerFactory> factoryClass =  FBUtilities.classForName(factory, "Remote storage handler factory");
+
+        try
+        {
+            return factoryClass.getConstructor().newInstance();
+        }
+        catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e)
+        {
+            throw new ConfigurationException("Unable to find correct constructor for " + factory, e);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/StorageHandlerFactory.java b/src/java/org/apache/cassandra/io/sstable/StorageHandlerFactory.java
new file mode 100644
index 000000000000..02e9c4ae4630
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/StorageHandlerFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.schema.TableMetadataRef;
+
+public interface StorageHandlerFactory
+{
+    StorageHandlerFactory DEFAULT = new StorageHandlerFactory() {};
+
+    default StorageHandler create(SSTable.Owner owner, TableMetadataRef metadata, Directories directories, Tracker dataTracker)
+    {
+        return new DefaultStorageHandler(owner, metadata, directories, dataTracker);
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/ULIDBasedSSTableId.java b/src/java/org/apache/cassandra/io/sstable/ULIDBasedSSTableId.java
new file mode 100644
index 000000000000..15579f669669
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/ULIDBasedSSTableId.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.nio.ByteBuffer;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+
+import com.google.common.base.Preconditions;
+
+import de.huxhorn.sulky.ulid.ULID;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * SSTable generation identifiers that can be stored across nodes in one directory/bucket
+ * Uses the ULID based identifiers
+ */
+public final class ULIDBasedSSTableId implements SSTableId<ULIDBasedSSTableId>
+{
+    public static final int STRING_LEN = 26;
+    public static final int BYTES_LEN = 16;
+
+    final ULID.Value ulid;
+    final TimeUUID approximateTimeUUID;
+
+    public ULIDBasedSSTableId(ULID.Value ulid)
+    {
+        this.ulid = ulid;
+        this.approximateTimeUUID = approximateFromULID(ulid);
+    }
+
+    public static TimeUUID approximateFromULID(ULID.Value ulid)
+    {
+        long rawTimestamp = TimeUUID.unixMillisToRawTimestamp(ulid.timestamp(), (10_000L * (ulid.getMostSignificantBits() & 0xFFFF)) >> 16);
+        return new TimeUUID(rawTimestamp, ulid.getLeastSignificantBits());
+    }
+
+    @Override
+    public ByteBuffer asBytes()
+    {
+        return ByteBuffer.wrap(ulid.toBytes());
+    }
+
+    @Override
+    public String toString()
+    {
+        return ulid.toString();
+    }
+
+    @Override
+    public int compareTo(ULIDBasedSSTableId o)
+    {
+        if (o == null)
+            return 1;
+        else if (o == this)
+            return 0;
+
+        return ulid.compareTo(o.ulid);
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        ULIDBasedSSTableId that = (ULIDBasedSSTableId) o;
+        return ulid.equals(that.ulid);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(ulid);
+    }
+
+    public static class Builder implements SSTableId.Builder<ULIDBasedSSTableId>
+    {
+        private static final Pattern PATTERN = Pattern.compile("[0-9a-z]{26}", Pattern.CASE_INSENSITIVE);
+
+        public static final Builder instance = new Builder();
+
+        private static final ULID ulid = new ULID();
+        private static final AtomicReference<ULID.Value> prevRef = new AtomicReference<>();
+
+        /**
+         * Creates a new ULID based identifiers generator.
+         *
+         * @param existingIdentifiers not used by UUID based generator
+         */
+        @Override
+        public Supplier<ULIDBasedSSTableId> generator(Stream<SSTableId> existingIdentifiers)
+        {
+            return () -> {
+                ULID.Value prevVal;
+                ULID.Value newVal = null;
+                do
+                {
+                    prevVal = prevRef.get();
+                    if (prevVal != null)
+                    {
+                        Optional<ULID.Value> newValOpt = ulid.nextStrictlyMonotonicValue(prevVal);
+                        if (!newValOpt.isPresent())
+                            continue;
+                        newVal = newValOpt.get();
+                    }
+                    else
+                    {
+                        newVal = ulid.nextValue();
+                    }
+                } while (newVal != null && !prevRef.compareAndSet(prevVal, newVal));
+                return new ULIDBasedSSTableId(newVal);
+            };
+        }
+
+        @Override
+        public boolean isUniqueIdentifier(String str)
+        {
+            return str != null && str.length() == STRING_LEN && PATTERN.matcher(str).matches();
+        }
+
+        @Override
+        public boolean isUniqueIdentifier(ByteBuffer bytes)
+        {
+            return bytes != null && bytes.remaining() == BYTES_LEN;
+        }
+
+        @Override
+        public ULIDBasedSSTableId fromString(@Nonnull String s) throws IllegalArgumentException
+        {
+            Matcher m = PATTERN.matcher(s);
+            if (!m.matches())
+                throw new IllegalArgumentException("String '" + s + "' is not a valid ULID based sstable identifier");
+
+            return new ULIDBasedSSTableId(ULID.parseULID(s));
+        }
+
+        @Override
+        public ULIDBasedSSTableId fromBytes(@Nonnull ByteBuffer bytes) throws IllegalArgumentException
+        {
+            Preconditions.checkArgument(bytes.remaining() == ULIDBasedSSTableId.BYTES_LEN,
+                                        "Buffer does not have a valid number of bytes remaining. Expecting: %s but was: %s",
+                                        ULIDBasedSSTableId.BYTES_LEN, bytes.remaining());
+
+            return new ULIDBasedSSTableId(ULID.fromBytes(ByteBufferUtil.getArray(bytes)));
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/UUIDBasedSSTableId.java b/src/java/org/apache/cassandra/io/sstable/UUIDBasedSSTableId.java
index 9cec5879f62d..bab5b7868cce 100644
--- a/src/java/org/apache/cassandra/io/sstable/UUIDBasedSSTableId.java
+++ b/src/java/org/apache/cassandra/io/sstable/UUIDBasedSSTableId.java
@@ -36,12 +36,12 @@
  * <p>
  * Uses the UUID v1 identifiers
  */
-public final class UUIDBasedSSTableId implements SSTableId, Comparable<UUIDBasedSSTableId>
+public final class UUIDBasedSSTableId implements SSTableId<UUIDBasedSSTableId>
 {
     public final static int STRING_LEN = 28;
     public final static int BYTES_LEN = 16;
 
-    private final TimeUUID uuid;
+    final TimeUUID uuid;
     private final String repr;
 
     public UUIDBasedSSTableId(TimeUUID uuid)
diff --git a/src/java/org/apache/cassandra/io/sstable/UnsupportedSSTableException.java b/src/java/org/apache/cassandra/io/sstable/UnsupportedSSTableException.java
new file mode 100644
index 000000000000..f4e7f58d683e
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/UnsupportedSSTableException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import org.apache.cassandra.io.util.File;
+
+public class UnsupportedSSTableException extends CorruptSSTableException
+{
+    public UnsupportedSSTableException(String msg, Throwable cause, File path)
+    {
+        super(msg, cause, path);
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/compaction/IteratorFromCursor.java b/src/java/org/apache/cassandra/io/sstable/compaction/IteratorFromCursor.java
new file mode 100644
index 000000000000..c86aac660fb8
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/compaction/IteratorFromCursor.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.compaction;
+
+import java.util.NoSuchElementException;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Wrapper that converts a cursor into an UnfilteredPartitionIterator for testing.
+ */
+public class IteratorFromCursor implements UnfilteredPartitionIterator
+{
+    final TableMetadata metadata;
+    final SSTableCursor cursor;
+    final Row.Builder rowBuilder;
+
+    public IteratorFromCursor(TableMetadata metadata, SSTableCursor cursor)
+    {
+        this.metadata = metadata;
+        this.cursor = cursor;
+        this.rowBuilder = BTreeRow.sortedBuilder();
+    }
+
+    public TableMetadata metadata()
+    {
+        return metadata;
+    }
+
+    public void close()
+    {
+        cursor.close();
+    }
+
+    public boolean hasNext()
+    {
+        return (advanceToNextPartition() == SSTableCursor.Type.PARTITION);
+    }
+
+    private SSTableCursor.Type advanceToNextPartition()
+    {
+        SSTableCursor.Type type = cursor.type();
+        while (true)
+        {
+            switch (type)
+            {
+                case PARTITION:
+                case EXHAUSTED:
+                    return type;
+                default:
+                    type = cursor.advance();
+            }
+        }
+    }
+
+    public UnfilteredRowIterator next()
+    {
+        SSTableCursor.Type type = advanceToNextPartition();
+        if (type == SSTableCursor.Type.EXHAUSTED)
+            throw new NoSuchElementException();
+        assert type == SSTableCursor.Type.PARTITION;
+        switch (cursor.advance())
+        {
+            case PARTITION:
+            case EXHAUSTED:
+                return EmptyIterators.unfilteredRow(metadata,
+                                                    cursor.partitionKey(),
+                                                    false,
+                                                    Rows.EMPTY_STATIC_ROW,
+                                                    cursor.partitionLevelDeletion());
+            case ROW:
+            case RANGE_TOMBSTONE:
+                return new RowIterator();
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    class RowIterator implements UnfilteredRowIterator
+    {
+        final DecoratedKey partitionKey;
+        final Row staticRow;
+        final DeletionTime partitionLevelDeletion;
+
+        protected RowIterator()
+        {
+            this.partitionKey = cursor.partitionKey();
+            this.partitionLevelDeletion = cursor.partitionLevelDeletion();
+            if (Clustering.STATIC_CLUSTERING.equals(cursor.clusteringKey()))
+            {
+                staticRow = collectRow(cursor, rowBuilder);
+            }
+            else
+            {
+                staticRow = Rows.EMPTY_STATIC_ROW;
+            }
+        }
+
+        public boolean hasNext()
+        {
+            return cursor.type().level == SSTableCursor.Type.ROW.level;
+        }
+
+        public Unfiltered next()
+        {
+            switch (cursor.type())
+            {
+                case ROW:
+                    return collectRow(cursor, rowBuilder);
+                case RANGE_TOMBSTONE:
+                    return collectRangeTombstoneMarker(cursor);
+                default:
+                    throw new AssertionError();
+            }
+        }
+
+        public TableMetadata metadata()
+        {
+            return metadata;
+        }
+
+        public boolean isReverseOrder()
+        {
+            return false;
+        }
+
+        public RegularAndStaticColumns columns()
+        {
+            return metadata.regularAndStaticColumns();
+        }
+
+        public DecoratedKey partitionKey()
+        {
+            return partitionKey;
+        }
+
+        public Row staticRow()
+        {
+            return staticRow;
+        }
+
+        public DeletionTime partitionLevelDeletion()
+        {
+            return partitionLevelDeletion;
+        }
+
+        public EncodingStats stats()
+        {
+            return EncodingStats.NO_STATS;
+        }
+
+        public void close()
+        {
+            // Nothing to do on row close
+        }
+    }
+
+    public static RangeTombstoneMarker collectRangeTombstoneMarker(SSTableCursor cursor)
+    {
+        ClusteringPrefix key = cursor.clusteringKey();
+        DeletionTime previous = cursor.activeRangeDeletion();
+        DeletionTime next = cursor.rowLevelDeletion();
+        cursor.advance();
+        switch (key.kind())
+        {
+            case INCL_START_BOUND:
+            case EXCL_START_BOUND:
+                return new RangeTombstoneBoundMarker((ClusteringBound) key, next);
+            case INCL_END_BOUND:
+            case EXCL_END_BOUND:
+                return new RangeTombstoneBoundMarker((ClusteringBound) key, previous);
+            case EXCL_END_INCL_START_BOUNDARY:
+            case INCL_END_EXCL_START_BOUNDARY:
+                return new RangeTombstoneBoundaryMarker((ClusteringBoundary) key, previous, next);
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    public static Row collectRow(SSTableCursor cursor, Row.Builder builder)
+    {
+        builder.newRow((Clustering) cursor.clusteringKey());
+        builder.addPrimaryKeyLivenessInfo(cursor.clusteringKeyLivenessInfo());
+        builder.addRowDeletion(Row.Deletion.regular(cursor.rowLevelDeletion()));
+        while (true)
+        {
+            switch (cursor.advance())
+            {
+                case COMPLEX_COLUMN_CELL:
+                case SIMPLE_COLUMN:
+                    builder.addCell(cursor.cell());
+                    break;
+                case COMPLEX_COLUMN:
+                    // Note: we want to create complex deletion cell even if there is no deletion because this passes
+                    // the correct version of the column metadata to the builder.
+                    builder.addComplexDeletion(cursor.column(), cursor.complexColumnDeletion());
+                    break;
+                default:
+                    return builder.build();
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/compaction/PurgeCursor.java b/src/java/org/apache/cassandra/io/sstable/compaction/PurgeCursor.java
new file mode 100644
index 000000000000..e50f492bd097
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/compaction/PurgeCursor.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.compaction;
+
+import java.util.function.LongPredicate;
+
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionPurger;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+/**
+ * A wrapping cursor that applies tombstone purging, counterpart to
+ * {@link org.apache.cassandra.db.compaction.CompactionIterator.Purger}. Purging is the process of removing tombstones
+ * that do not need to be preserved, defined at minimum as:
+ * - there is no data in sstables not taking part in this compaction that may be covered by the tombstone, and
+ * - the gc_grace period, during which we protect tombstones to ensure that they are propagated to other replicas, has
+ *   expired, and
+ * - we are compacting repaired sstables or the CFS does not request that only repaired tombstones are purged.
+ * Additionally, the purger converts expiring cells that have gone beyond their time-to-live to tombstones (deleting
+ * their data), and collects said tombstones if they are purgeable.
+ *
+ * Note that this may end up creating empty rows (i.e. headers with no deletion/timestamp and no cells) -- typically
+ * a SkipEmptyDataCursor would be required to apply on top of the result.
+ */
+public class PurgeCursor implements SSTableCursor, DeletionPurger
+{
+    private final SSTableCursor wrapped;
+    private final CompactionController controller;
+    private final long nowInSec;
+    private final long gcBefore;
+    private final boolean purgeTombstones;
+    private LongPredicate purgeEvaluator;
+
+    private DeletionTime partitionLevelDeletion;
+    private DeletionTime activeRangeDeletion = DeletionTime.LIVE;
+    private DeletionTime rowLevelDeletion;
+    private DeletionTime complexColumnDeletion;
+    private LivenessInfo clusteringKeyLivenessInfo;
+    private ClusteringPrefix<?> clusteringKey;
+    private Cell<?> cell;
+
+    public PurgeCursor(SSTableCursor wrapped, CompactionController controller, long nowInSec)
+    {
+        this.gcBefore = controller.gcBefore;
+        this.purgeTombstones = controller.compactingRepaired(); // this is also true if !cfs.onlyPurgeRepairedTombstones
+        this.wrapped = wrapped;
+        this.controller = controller;
+        this.nowInSec = nowInSec;
+    }
+
+    @Override
+    public boolean shouldPurge(long timestamp, long localDeletionTime)
+    {
+        return purgeTombstones
+               && (localDeletionTime < gcBefore || controller.realm.shouldIgnoreGcGraceForKey(partitionKey()))
+               && getPurgeEvaluator().test(timestamp);
+    }
+
+    /*
+     * Evaluates whether a tombstone with the given deletion timestamp can be purged. This is the minimum
+     * timestamp for any sstable containing `currentKey` outside of the set of sstables involved in this compaction.
+     * This is computed lazily on demand as we only need this if there is tombstones and this a bit expensive
+     * (see #8914).
+     */
+    protected LongPredicate getPurgeEvaluator()
+    {
+        if (purgeEvaluator == null)
+            purgeEvaluator = controller.getPurgeEvaluator(partitionKey());
+
+        return purgeEvaluator;
+    }
+
+    public Type advance()
+    {
+        if (wrapped.type() == Type.RANGE_TOMBSTONE)
+            activeRangeDeletion = rowLevelDeletion;
+
+        while (true)
+        {
+            Type type = wrapped.advance();
+            switch (type)
+            {
+                case EXHAUSTED:
+                    return type;
+                case PARTITION:
+                    purgeEvaluator = null;
+                    partitionLevelDeletion = maybePurge(wrapped.partitionLevelDeletion());
+                    assert activeRangeDeletion == DeletionTime.LIVE;
+                    return type;
+                case RANGE_TOMBSTONE:
+                    rowLevelDeletion = maybePurge(wrapped.rowLevelDeletion());
+                    clusteringKey = maybePurge(wrapped.clusteringKey(), activeRangeDeletion, rowLevelDeletion);
+                    if (clusteringKey != null)
+                        return type;
+                    else
+                        break;  // no bound remained, move on to next item
+                case ROW:
+                    clusteringKey = wrapped.clusteringKey();
+                    rowLevelDeletion = maybePurge(wrapped.rowLevelDeletion());
+                    clusteringKeyLivenessInfo = maybePurge(wrapped.clusteringKeyLivenessInfo(), nowInSec);
+                    return type;
+                case COMPLEX_COLUMN:
+                    this.complexColumnDeletion = maybePurge(wrapped.complexColumnDeletion());
+                    return type;
+                case SIMPLE_COLUMN:
+                case COMPLEX_COLUMN_CELL:
+                    // This also applies cells' time-to-live, converting expired cells to tombstones.
+                    cell = wrapped.cell().purge(this, nowInSec);
+                    if (cell != null)
+                        return type;
+                    break;  // otherwise, skip this cell
+                default:
+                    throw new AssertionError();
+            }
+        }
+    }
+
+    private DeletionTime maybePurge(DeletionTime deletionTime)
+    {
+        return shouldPurge(deletionTime) ? DeletionTime.LIVE : deletionTime;
+    }
+
+    private LivenessInfo maybePurge(LivenessInfo liveness, long nowInSec)
+    {
+        return shouldPurge(liveness, nowInSec) ? LivenessInfo.EMPTY : liveness;
+    }
+
+    private ClusteringPrefix<?> maybePurge(ClusteringPrefix<?> clusteringKey, DeletionTime deletionBefore, DeletionTime deletionAfter)
+    {
+        // We pass only the current deletion to the purger. This may mean close bounds' deletion time is
+        // already purged.
+        if (deletionBefore.isLive() && clusteringKey.kind().isEnd())
+        {
+            // we need to strip the closing part of the tombstone
+            // if only a close bound, or the new deletion is also purged, do not return
+            if (clusteringKey.kind().isBound() || deletionAfter.isLive())
+                return null;
+
+            return ClusteringBound.create(clusteringKey.kind().openBoundOfBoundary(false), clusteringKey);
+        }
+        else if (clusteringKey.kind().isStart() && deletionAfter.isLive())
+        {
+            // we need to strip the opening part of the tombstone
+            if (clusteringKey.kind().isBound())
+                return null;  // only an open bound whose time is now purged. Do not return.
+            assert !deletionBefore.isLive();  // If ending was also deleted, we would have gone through the path above.
+            return ClusteringBound.create(clusteringKey.kind().closeBoundOfBoundary(false), clusteringKey);
+        }
+        else    // Nothing is dropped
+            return clusteringKey;
+    }
+
+    public Type type()
+    {
+        return wrapped.type();
+    }
+
+    public DecoratedKey partitionKey()
+    {
+        return wrapped.partitionKey();
+    }
+
+    public DeletionTime partitionLevelDeletion()
+    {
+        return partitionLevelDeletion;
+    }
+
+    public ClusteringPrefix clusteringKey()
+    {
+        return clusteringKey;
+    }
+
+    public LivenessInfo clusteringKeyLivenessInfo()
+    {
+        return clusteringKeyLivenessInfo;
+    }
+
+    public DeletionTime rowLevelDeletion()
+    {
+        return rowLevelDeletion;
+    }
+
+    public DeletionTime activeRangeDeletion()
+    {
+        return activeRangeDeletion;
+    }
+
+    public DeletionTime complexColumnDeletion()
+    {
+        return complexColumnDeletion;
+    }
+
+    public ColumnMetadata column()
+    {
+        return wrapped.column();
+    }
+
+    public Cell cell()
+    {
+        return cell;
+    }
+
+    public long bytesProcessed()
+    {
+        return wrapped.bytesProcessed();
+    }
+
+    public long bytesTotal()
+    {
+        return wrapped.bytesTotal();
+    }
+
+    public void close()
+    {
+        wrapped.close();
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/compaction/SSTableCursor.java b/src/java/org/apache/cassandra/io/sstable/compaction/SSTableCursor.java
new file mode 100644
index 000000000000..1785e4ec542e
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/compaction/SSTableCursor.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.compaction;
+
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+/**
+ * An sstable cursor is an iterator-like object that is used to enumerate and merge the content of sstables.
+ * It produces a stream of cells, broken up with row and partition boundaries -- doing this allows the merging to be
+ * done using a single container and merger instead of the hierarchy used in UnfilteredPartitionIterator-
+ * UnfilteredRowIterator-Row-ComplexColumn-Cell.
+ *
+ * There are two other important differences done to improve merging performance:
+ *   - static rows are not special and are specified as normal rows with STATIC_CLUSTERING
+ *   - cursors track the currently active range deletion
+ *
+ * More details about the design, functionality and performance of cursors can be found in the included cursors.md file.
+ */
+public interface SSTableCursor extends AutoCloseable
+{
+    /**
+     * Enumeration of the type of item at the current position: either a cell, a range tombstone marker, a header of
+     * some upper level of the hierarchy, or an end-of-stream.
+     * This combines information about the object seen with information about its level in the logical hierarchy
+     * (specified in the "level" field). The latter is key for comparing the position of two cursors which are iterated
+     * together in a merge: if a cursor is positioned on a lower level in the hierarchy than another, it is listing
+     * content in a group that is either exhausted or not opened in the other cursor, and in both cases the other
+     * cursor's position must have a bigger higher-level key. That is, a cursor with a smaller level is always before
+     * a cursor with a higher one (see {@link SSTableCursorMerger#mergeComparator}).
+     */
+    enum Type
+    {
+        COMPLEX_COLUMN_CELL(0),
+        COMPLEX_COLUMN(1),
+        SIMPLE_COLUMN(1),
+        ROW(2),
+        RANGE_TOMBSTONE(2),
+        PARTITION(3),
+        EXHAUSTED(4),
+        UNINITIALIZED(-1);
+
+        /** The actual level, used for comparisons (some types e.g. ROW and RANGE_TOMBSTONE share a level). */
+        final int level;
+
+        Type(int level)
+        {
+            this.level = level;
+        }
+    }
+
+    /**
+     * Advance the cursor and return the level of the next element. The returned level is the same as what level()
+     * returns next.
+     * Any errors during read should be converted to CorruptSSTableException.
+     */
+    Type advance();
+
+    /**
+     * The current level. UNINITIALIZED if iteration has not started, otherwise what the last advance() returned.
+     */
+    Type type();
+
+    /**
+     * Current partition key. Only valid if a partition is in effect, i.e. if level() <= PARTITION.
+     */
+    DecoratedKey partitionKey();
+
+    /**
+     * Partition level deletion. Only valid in a partition.
+     */
+    DeletionTime partitionLevelDeletion();
+
+    /**
+     * Current clustering key. Only valid if positioned within/on a row/unfiltered, i.e. if level() <= ROW.
+     * For rows, this will be Clustering, and range tombstone markers will use prefix.
+     */
+    ClusteringPrefix<?> clusteringKey();
+
+    /**
+     * Liveness info for the current row's clustering key. Only valid within a row.
+     */
+    LivenessInfo clusteringKeyLivenessInfo();
+
+    /**
+     * Row level deletion. Only valid within a row or on a range deletion. In the latter case, reports the new
+     * deletion being set.
+     */
+    DeletionTime rowLevelDeletion();
+
+    /**
+     * Currently open range deletion. This tracks the last set range deletion, LIVE if none has been seen.
+     * If positioned on a range tombstone marker, this will report the _previous_ deletion.
+     */
+    DeletionTime activeRangeDeletion();
+
+    /**
+     * Metadata for the current column.  Only valid within/on a column, i.e. if
+     * level() <= SIMPLE/COMPLEX_COLUMN.
+     * In a merged complex column this may be different from cell().column() because it contains the most up-to-date
+     * version while individual cell sources will report their own.
+     */
+    ColumnMetadata column();
+
+    /**
+     * Deletion of the current complex column. Only valid within/on a complex column, i.e. if
+     * level() == COMPLEX_COLUMN[_CELL].
+     */
+    DeletionTime complexColumnDeletion();
+
+    /**
+     * Current cell. This may be a column or a cell within a complex column. Only valid if positioned on a cell,
+     * which may be a simple column or a cell in a complex one, i.e. level() == SIMPLE_COLUMN or COMPLEX_COLUMN_CELL.
+     */
+    Cell<?> cell();
+
+    /**
+     * @return number of bytes processed. This should be used as progress indication together with bytesTotal.
+     */
+    long bytesProcessed();
+    /**
+     * @return number of bytes total. This should be used as progress indication together with bytesProcessed.
+     */
+    long bytesTotal();
+
+    void close();
+
+    static SSTableCursor empty()
+    {
+        return new SSTableCursor()
+        {
+            boolean initialized = false;
+
+            public Type advance()
+            {
+                initialized = true;
+                return Type.EXHAUSTED;
+            }
+
+            public Type type()
+            {
+                return initialized ? Type.EXHAUSTED : Type.UNINITIALIZED;
+            }
+
+            public DecoratedKey partitionKey()
+            {
+                return null;
+            }
+
+            public DeletionTime partitionLevelDeletion()
+            {
+                return null;
+            }
+
+            public ClusteringPrefix clusteringKey()
+            {
+                return null;
+            }
+
+            public LivenessInfo clusteringKeyLivenessInfo()
+            {
+                return null;
+            }
+
+            public DeletionTime rowLevelDeletion()
+            {
+                return null;
+            }
+
+            public DeletionTime activeRangeDeletion()
+            {
+                return null;
+            }
+
+            public DeletionTime complexColumnDeletion()
+            {
+                return null;
+            }
+
+            public ColumnMetadata column()
+            {
+                return null;
+            }
+
+            public Cell cell()
+            {
+                return null;
+            }
+
+            public long bytesProcessed()
+            {
+                return 0;
+            }
+
+            public long bytesTotal()
+            {
+                return 0;
+            }
+
+            public void close()
+            {
+                // nothing
+            }
+        };
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/compaction/SSTableCursorMerger.java b/src/java/org/apache/cassandra/io/sstable/compaction/SSTableCursorMerger.java
new file mode 100644
index 000000000000..89ff5d521003
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/compaction/SSTableCursorMerger.java
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.compaction;
+
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Cells;
+import org.apache.cassandra.db.rows.ColumnMetadataVersionComparator;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Merger;
+import org.apache.cassandra.utils.Reducer;
+
+/**
+ * Cursor merger, which employs the Merger object to combine multiple cursors into a single stream.
+ *
+ * Most of the complexity of this class is in applying hierarchical deletions, of which there are four kinds:
+ *  - partition-level deletion
+ *  - range tombstones
+ *  - row-level deletion
+ *  - complex column deletion
+ * In addition to the values for each level (which we must report to the consumer), we also track combined
+ *  - activeRangeDeletion (newest of partition-level and active range tombstone)
+ *  - mergedRowDeletion (newest of active and row-level deletion)
+ *  - mergedCellDeletion (newest of merged row and complex column deletion)
+ *  and use the mergedCellDeletion to remove no-longer active cells.
+ */
+public class SSTableCursorMerger extends Reducer<SSTableCursor, SSTableCursor> implements SSTableCursor
+{
+    private final Merger<SSTableCursor, SSTableCursor, SSTableCursor> merger;
+    private final MergeListener mergeListener;
+
+    private Type currentType;
+    private DecoratedKey currentPartitionKey;
+    private DeletionTime partitionLevelDeletion;
+    private ClusteringPrefix<?> currentClusteringKey;
+    private DeletionTime rowLevelDeletion;
+    private LivenessInfo currentLivenessInfo;
+    private DeletionTime activeRangeDeletion; // uses partitionLevelDeletion as base instead of LIVE (which makes the logic a little simpler)
+    private DeletionTime mergedRowDeletion; // the deletion that applies to this row, merge(activeRangeDeletion, rowLevelDeletion)
+    private DeletionTime mergedCellDeletion; // the deletion that applies to this cell, merge(mergedRowDeletion, complexColumnDeletion)
+
+    private ColumnMetadata columnMetadata;
+    private DeletionTime complexColumnDeletion;
+
+    private Cell<?> currentCell;
+    private int currentIndex;
+    private int numMergedVersions = 0;
+
+    public SSTableCursorMerger(List<SSTableCursor> cursors, TableMetadata metadata)
+    {
+        this(cursors, metadata, NO_MERGE_LISTENER);
+    }
+
+    public SSTableCursorMerger(List<SSTableCursor> cursors, TableMetadata metadata, MergeListener mergeListener)
+    {
+        assert !cursors.isEmpty();
+        this.mergeListener = mergeListener;
+        this.merger = new Merger<>(cursors,
+                                   x -> {
+                                       x.advance();
+                                       return x;
+                                   },
+                                   SSTableCursor::close,
+                                   mergeComparator(metadata),
+                                   this);
+        this.currentType = Type.UNINITIALIZED;
+    }
+
+    public Type type()
+    {
+        return currentType;
+    }
+
+    public DecoratedKey partitionKey()
+    {
+        return currentPartitionKey;
+    }
+
+    public DeletionTime partitionLevelDeletion()
+    {
+        return partitionLevelDeletion;
+    }
+
+    public ClusteringPrefix clusteringKey()
+    {
+        return currentClusteringKey;
+    }
+
+    public LivenessInfo clusteringKeyLivenessInfo()
+    {
+        return currentLivenessInfo;
+    }
+
+    public DeletionTime rowLevelDeletion()
+    {
+        // Note: this is used for both range tombstone markers and rows. For the former we default to the
+        // partition-level deletion if no (newer) range tombstone is in effect, but we must report this case
+        // as LIVE to any consumer.
+        return rowLevelDeletion == partitionLevelDeletion ? DeletionTime.LIVE : rowLevelDeletion;
+    }
+
+    public DeletionTime activeRangeDeletion()
+    {
+        return activeRangeDeletion == partitionLevelDeletion ? DeletionTime.LIVE : activeRangeDeletion;
+    }
+
+    public DeletionTime complexColumnDeletion()
+    {
+        return complexColumnDeletion;
+    }
+
+    public ColumnMetadata column()
+    {
+        return columnMetadata;
+    }
+
+    public Cell cell()
+    {
+        return currentCell;
+    }
+
+    public long bytesProcessed()
+    {
+        long bytesProcessed = 0;
+        for (SSTableCursor cursor : merger.allSources())
+            bytesProcessed += cursor.bytesProcessed();
+        return bytesProcessed;
+    }
+
+    public long bytesTotal()
+    {
+        long bytesTotal = 0;
+        for (SSTableCursor cursor : merger.allSources())
+            bytesTotal += cursor.bytesTotal();
+        return bytesTotal;
+    }
+
+    public void close()
+    {
+        merger.close(); // this will also close the inputs via the supplied onClose consumer
+    }
+
+    public Type advance()
+    {
+        if (currentType == Type.RANGE_TOMBSTONE)
+            activeRangeDeletion = rowLevelDeletion;
+
+        // We don't need to use hasNext because the streams finish on Level.EXHAUSTED.
+        // If the reducer returns null, get next entry.
+        while (merger.next() == null) {}
+
+        return currentType;
+    }
+
+    public void onKeyChange()
+    {
+        currentIndex = -1;
+        numMergedVersions = 0;
+    }
+
+    public void reduce(int idx, SSTableCursor current)
+    {
+        ++numMergedVersions;
+        if (currentIndex == -1)
+        {
+            currentIndex = idx;
+            currentType = current.type();
+            switch (currentType)
+            {
+                case COMPLEX_COLUMN:
+                    columnMetadata = current.column();
+                    complexColumnDeletion = current.complexColumnDeletion();
+                    return;
+                case SIMPLE_COLUMN:
+                    mergedCellDeletion = mergedRowDeletion;
+                case COMPLEX_COLUMN_CELL:
+                    Cell cell = current.cell();
+                    if (!mergedCellDeletion.deletes(cell))
+                        currentCell = cell;
+                    else
+                        currentCell = null;
+                    return;
+                case ROW:
+                    currentClusteringKey = current.clusteringKey();
+                    currentLivenessInfo = current.clusteringKeyLivenessInfo();
+                    rowLevelDeletion = current.rowLevelDeletion();
+                    return;
+                case RANGE_TOMBSTONE:
+                    currentClusteringKey = current.clusteringKey();
+                    rowLevelDeletion = current.rowLevelDeletion();
+                    return;
+                case PARTITION:
+                    currentPartitionKey = current.partitionKey();
+                    partitionLevelDeletion = current.partitionLevelDeletion();
+                    return;
+                case EXHAUSTED:
+                default:
+                    return;
+            }
+        }
+        else
+        {
+            switch (currentType)
+            {
+                case COMPLEX_COLUMN:
+                    if ((ColumnMetadataVersionComparator.INSTANCE.compare(columnMetadata, current.column()) < 0))
+                        columnMetadata = current.column();
+                    if (current.complexColumnDeletion().supersedes(complexColumnDeletion))
+                        complexColumnDeletion = current.complexColumnDeletion();
+                    return;
+                case SIMPLE_COLUMN:
+                case COMPLEX_COLUMN_CELL:
+                    Cell cell = current.cell();
+                    if (!mergedCellDeletion.deletes(cell))
+                        currentCell = currentCell != null
+                                      ? Cells.reconcile(currentCell, cell)
+                                      : cell;
+                    return;
+                case ROW:
+                    currentLivenessInfo = LivenessInfo.merge(currentLivenessInfo, current.clusteringKeyLivenessInfo());
+                    rowLevelDeletion = DeletionTime.merge(rowLevelDeletion, current.rowLevelDeletion());
+                    return;
+                case RANGE_TOMBSTONE:
+                    rowLevelDeletion = DeletionTime.merge(rowLevelDeletion, current.rowLevelDeletion());
+                    return;
+                case PARTITION:
+                    partitionLevelDeletion = DeletionTime.merge(partitionLevelDeletion, current.partitionLevelDeletion());
+                    return;
+                case EXHAUSTED:
+                default:
+                    return;
+            }
+        }
+    }
+
+    public SSTableCursor getReduced()
+    {
+        mergeListener.onItem(this, numMergedVersions);
+
+        switch (currentType)
+        {
+            case COMPLEX_COLUMN_CELL:
+                if (currentCell == null)
+                    return null;
+                break;
+            case COMPLEX_COLUMN:
+                if (complexColumnDeletion.supersedes(mergedRowDeletion))
+                    mergedCellDeletion = complexColumnDeletion;
+                else
+                {
+                    complexColumnDeletion = DeletionTime.LIVE;
+                    mergedCellDeletion = mergedRowDeletion;
+                }
+                break;
+            case SIMPLE_COLUMN:
+                if (currentCell == null)
+                    return null;
+                columnMetadata = currentCell.column();
+                break;
+            case ROW:
+                if (rowLevelDeletion.supersedes(activeRangeDeletion))
+                    mergedRowDeletion = rowLevelDeletion;
+                else
+                {
+                    rowLevelDeletion = DeletionTime.LIVE;
+                    mergedRowDeletion = activeRangeDeletion;
+                }
+
+                if (mergedRowDeletion.deletes(currentLivenessInfo))
+                    currentLivenessInfo = LivenessInfo.EMPTY;
+                break;
+            case RANGE_TOMBSTONE:
+                if (!rowLevelDeletion.supersedes(activeRangeDeletion))
+                {
+                    // The new deletion is older than some of the active. We need to check if this is the end of the
+                    // deletion that is currently active, or something else (some previous start or end that got itself
+                    // deleted). To do this, check all active deletions for the sources that did not take part in this
+                    // tombstone -- if something newer is still active, we should be using that deletion time instead.
+
+                    // For example, consider the merge of deletions over 1-6 with time 3, and 3-9 with time 2:
+                    //       at 1 we have active=LIVE row=3 other=LIVE and switch from LIVE to 3, i.e. return row=3
+                    //       at 3 we have active=3 row=2 other=3 and switch from 3 to 3, i.e. issue nothing
+                    //       at 6 we have active=3 row=LIVE other=2 and switch from 3 to 2, i.e. return row=2
+                    //       at 9 we have active=2 row=LIVE other=LIVE and switch from 2 to LIVE, i.e. return row=LIVE
+                    // (where active stands for activeRangeDeletion, row - rowLevelDeletion and other - otherActive)
+
+                    if (activeRangeDeletion.equals(rowLevelDeletion))
+                        return null;    // nothing to report, old and new are the same
+                    DeletionTime otherActive = gatherDeletions(partitionLevelDeletion, merger.allGreaterValues());
+                    if (!rowLevelDeletion.supersedes(otherActive))
+                    {
+                        if (activeRangeDeletion.equals(otherActive))
+                            return null;   // this deletion is fully covered by other sources, nothing has changed
+                        else
+                            rowLevelDeletion = otherActive; // a newer deletion was closed, use the still valid from others
+                    }
+                } // otherwise this is introducing a deletion that beats all and should be reported
+
+                currentClusteringKey = adjustClusteringKeyForMarker(currentClusteringKey,
+                                                                    activeRangeDeletion != partitionLevelDeletion,
+                                                                    rowLevelDeletion != partitionLevelDeletion);
+                break;
+            case PARTITION:
+                activeRangeDeletion = partitionLevelDeletion;
+                break;
+        }
+
+        return this;
+    }
+
+    private DeletionTime gatherDeletions(DeletionTime initialValue, Iterable<SSTableCursor> sources)
+    {
+        DeletionTime collected = initialValue;
+        for (SSTableCursor cursor : sources)
+        {
+            if (cursor.type() == Type.ROW || cursor.type() == Type.RANGE_TOMBSTONE)
+                collected = DeletionTime.merge(collected, cursor.activeRangeDeletion());
+        }
+        return collected;
+    }
+
+    /**
+     * Adjust the clustering key for the type of range tombstone marker needed. For the different marker types we have
+     * equal but separate clustering kinds. As a the new marker may be the result of combining multiple different ones,
+     * the type of marker we have to issue is not guaranteed to be the type of marker we got as input (for example,
+     * if one deletion ends at 2 exclusive but another starts at 2 inclusive, these two markers have equal clustering
+     * keys, but their merge is not the same as either, but a boundary of the exclusive-end-inclusive-start type).
+     */
+    private static ClusteringPrefix adjustClusteringKeyForMarker(ClusteringPrefix clusteringKey, boolean activeBefore, boolean activeAfter)
+    {
+        if (activeBefore & activeAfter)
+        {
+            // We may need to upgrade from a pair of bounds to a boundary.
+            switch (clusteringKey.kind())
+            {
+                case EXCL_END_INCL_START_BOUNDARY:
+                case INCL_END_EXCL_START_BOUNDARY:
+                    return clusteringKey;  // already a boundary, good
+                case INCL_START_BOUND:
+                case EXCL_END_BOUND:
+                    return ClusteringBoundary.create(ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY,
+                                                     clusteringKey);
+                case EXCL_START_BOUND:
+                case INCL_END_BOUND:
+                    return ClusteringBoundary.create(ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY,
+                                                     clusteringKey);
+                default:
+                    throw new AssertionError();
+            }
+        }
+        else if (activeBefore)
+        {
+            // Partition-level deletion can cause one side of a boundary to be dropped.
+            // Note that because we can have many deletions clashing on the same position, we may have even picked up
+            // the clustering key from an overwritten open marker.
+            switch (clusteringKey.kind())
+            {
+                case EXCL_END_BOUND:
+                case INCL_END_BOUND:
+                    return clusteringKey;  // already a close bound, good
+                case EXCL_END_INCL_START_BOUNDARY:
+                case INCL_START_BOUND:
+                    return ClusteringBound.create(ClusteringPrefix.Kind.EXCL_END_BOUND,
+                                                  clusteringKey);
+                case INCL_END_EXCL_START_BOUNDARY:
+                case EXCL_START_BOUND:
+                    return ClusteringBound.create(ClusteringPrefix.Kind.INCL_END_BOUND,
+                                                  clusteringKey);
+                default:
+                    throw new AssertionError();
+            }
+        }
+        else if (activeAfter)
+        {
+            switch (clusteringKey.kind())
+            {
+                case EXCL_START_BOUND:
+                case INCL_START_BOUND:
+                    return clusteringKey;  // already an open bound, good
+                case EXCL_END_INCL_START_BOUNDARY:
+                case EXCL_END_BOUND:
+                    return ClusteringBound.create(ClusteringPrefix.Kind.INCL_START_BOUND,
+                                                  clusteringKey);
+                case INCL_END_EXCL_START_BOUNDARY:
+                case INCL_END_BOUND:
+                    return ClusteringBound.create(ClusteringPrefix.Kind.EXCL_START_BOUND,
+                                                  clusteringKey);
+                default:
+                    throw new AssertionError();
+            }
+        }
+        else
+            throw new AssertionError();
+    }
+
+    public interface MergeListener
+    {
+        void onItem(SSTableCursor cursor, int numVersions);
+    }
+
+    static MergeListener NO_MERGE_LISTENER = (cursor, numVersions) -> {};
+
+    public static Comparator<SSTableCursor> mergeComparator(TableMetadata metadata)
+    {
+        ClusteringComparator clusteringComparator = metadata.comparator;
+        return (a, b) ->
+        {
+            // Since we are advancing the sources together, a difference in levels means that either:
+            // - we compared partition/clustering/column keys before, they were different, and we did not advance one of
+            //   the sources into the partition/row's content
+            // - one of the sources exhausted the partition/row/column's content and is now producing the next
+            // In either case the other source is still producing content for a partition/row/column that should be
+            // exhausted before we have to look at that key again.
+            if (a.type().level != b.type().level)
+                return Integer.compare(a.type().level, b.type().level);
+
+            // If the sources are at the same level, we are guaranteed by the order and comparison above that all
+            // keys above this level match and thus we only need to compare the current.
+            switch (a.type())
+            {
+                case COMPLEX_COLUMN_CELL:
+                    return a.cell().column().cellPathComparator().compare(a.cell().path(), b.cell().path());
+                case SIMPLE_COLUMN:
+                case COMPLEX_COLUMN:
+                    return a.column().compareTo(b.column());
+                case ROW:
+                case RANGE_TOMBSTONE:
+                    return clusteringComparator.compare(a.clusteringKey(), b.clusteringKey());
+                case PARTITION:
+                    return a.partitionKey().compareTo(b.partitionKey());
+                case EXHAUSTED:
+                default:
+                    return 0;
+            }
+        };
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/compaction/SkipEmptyDataCursor.java b/src/java/org/apache/cassandra/io/sstable/compaction/SkipEmptyDataCursor.java
new file mode 100644
index 000000000000..001cdf5bbcf9
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/compaction/SkipEmptyDataCursor.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.compaction;
+
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+/**
+ * Wrapper that skips empty data. This is done by only reporting a header (complex column/row/partition) with no
+ * deletion or timestamp after valid content (cell, range tombstone, header with deletion/timestamp) is found; if no
+ * such content is found, the source is advanced to the next header without reporting the empty block to the consumer.
+ *
+ * In other words, in response to a single advance() call, this will take multiple steps in the source cursor until it
+ * reaches content, and when it does it reports the highest-level header it has had to go through to reach that content.
+ * On the next advance() call it will descend one level in the logical hierarchy and report the next header. This
+ * repeats until it reaches the level of the content. After reporting the level of the non-empty content, the next
+ * advance() call will repeat the procedure, starting with searching the input for non-empty content.
+ *
+ * Here's a sample evolution of input and output:
+ *                  [input]               [output]
+ *                  UNINITIALIZED         UNINITIALIZED
+ *                  PARTITION Pa
+ *                  ROW Raa
+ *                  SIMPLE_COLUMN Caa     PARTITION Pa
+ *                                        ROW Raa
+ *                                        SIMPLE_COLUMN Caa
+ *                  ROW Rab
+ *                  SIMPLE_COLUMN Cab     ROW Rab
+ *                                        SIMPLE_COLUMN Cab
+ *                  PARTITION Pb
+ *                  ROW Rba
+ *                  PARTITION Pc
+ *                  RANGE_TOMBSTONE Tca   PARTITION Pc
+ *                                        RANGE_TOMBSTONE Tca
+ *                  RANGE_TOMBSTONE Tcb   RANGE_TOMBSTONE Tcb
+ *                  EXHAUSTED             EXHAUSTED
+ *
+ * To report a header it is sufficient to just issue its level and pass on the data from the wrapped cursor, because
+ * cursors always make the upper-level data (e.g. partition key) available while they advance within that level.
+ */
+public class SkipEmptyDataCursor implements SSTableCursor
+{
+    private final SSTableCursor wrapped;
+    private Type type = Type.UNINITIALIZED;
+
+    public SkipEmptyDataCursor(SSTableCursor wrapped)
+    {
+        this.wrapped = wrapped;
+    }
+
+    public Type advance()
+    {
+        Type current = wrapped.type();
+        if (current != type)
+            return type = advanceOurLevel(current);
+
+        type = wrapped.advance();
+        while (true)
+        {
+            switch (type)
+            {
+                case EXHAUSTED:
+                    return type;
+                case SIMPLE_COLUMN:
+                case COMPLEX_COLUMN_CELL:
+                case RANGE_TOMBSTONE:
+                    // we are good, we have content
+                    return type;
+                case COMPLEX_COLUMN:
+                    if (!complexColumnDeletion().isLive())
+                        return type;   // we have to report this column even without any cells
+                    if (advanceToComplexCell())
+                        return type; // we found a cell and should now report the column
+                    // There is no cell of this complex column. We may have advance to another column, row or partition.
+                    break;
+                case ROW:
+                    if (!rowLevelDeletion().isLive() || !clusteringKeyLivenessInfo().isEmpty())
+                        return type;   // we have to report this row even without any columns
+                    if (advanceToColumn())
+                        return type;   // we have reached a cell, but we must still report the row
+                    // There is no cell. We may have advanced to a new row or new partition.
+                    break;
+                case PARTITION:
+                    if (!partitionLevelDeletion().isLive())
+                        return type;   // we have to report this partition even without any content
+                    if (advanceToNonEmptyRow())
+                        return type;   // We have reached a cell (or RT). We must report the partition, then the row.
+                                       // The wrapped cursor still returns their information (pkey, ckey etc.)
+                    // No rows or all empty. We must have advanced to new partition or exhausted.
+                    break;
+                default:
+                    throw new AssertionError();
+            }
+            type = wrapped.type();
+        }
+    }
+
+    /**
+     * Called to report content that has been advanced to, but not yet reported. This will descend one level in the
+     * logical hierarchy towards the target and return the resulting position.
+     * For example, on seeing a row header we first advance to a non-empty cell and report the header, and on the
+     * following advance() call report the cell we have advanced to. This method takes care of the latter part.
+     */
+    private Type advanceOurLevel(Type target)
+    {
+        switch (type)
+        {
+            case PARTITION:
+                switch (target)
+                {
+                    case COMPLEX_COLUMN_CELL:
+                    case SIMPLE_COLUMN:
+                    case COMPLEX_COLUMN:
+                        return Type.ROW;
+                    case ROW:
+                    case RANGE_TOMBSTONE:
+                        return target;
+                    default:
+                        throw new AssertionError();
+                }
+            case ROW:
+                switch (target)
+                {
+                    case COMPLEX_COLUMN_CELL:
+                        return Type.COMPLEX_COLUMN;
+                    case SIMPLE_COLUMN:
+                    case COMPLEX_COLUMN:
+                        return target;
+                    default:
+                        throw new AssertionError();
+                }
+            case COMPLEX_COLUMN:
+                switch (target)
+                {
+                    case COMPLEX_COLUMN_CELL:
+                        return target;
+                    default:
+                        throw new AssertionError();
+                }
+
+            default:
+                // can't have any differences in any other case
+                throw new AssertionError();
+        }
+    }
+
+    private boolean advanceToComplexCell()
+    {
+        Type current = wrapped.advance();
+        switch (current)
+        {
+            case COMPLEX_COLUMN_CELL:
+                return true;
+            case SIMPLE_COLUMN:
+            case COMPLEX_COLUMN:
+            case ROW:
+            case RANGE_TOMBSTONE:
+            case PARTITION:
+            case EXHAUSTED:
+                return false;
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    private boolean advanceToColumn()
+    {
+        Type current = wrapped.advance();
+        while (true)
+        {
+            switch (current)
+            {
+                case SIMPLE_COLUMN:
+                    return true;
+                case COMPLEX_COLUMN:
+                    if (!complexColumnDeletion().isLive())
+                        return true;
+                    if (advanceToComplexCell())
+                        return true;
+                    // There is no cell, skip this complex column. We may have a new column, or a new partition or row.
+                    break;
+                case ROW:
+                case RANGE_TOMBSTONE:
+                case PARTITION:
+                case EXHAUSTED:
+                    return false;
+                case COMPLEX_COLUMN_CELL:
+                    // can't jump directly to cell without going through COMPLEX_COLUMN
+                default:
+                    throw new AssertionError();
+            }
+            current = wrapped.type();
+        }
+    }
+
+    private boolean advanceToNonEmptyRow()
+    {
+        Type current = wrapped.advance();
+        while (true)
+        {
+            switch (current)
+            {
+                case RANGE_TOMBSTONE:
+                    // we have content
+                    return true;
+                case ROW:
+                    if (!rowLevelDeletion().isLive() || !clusteringKeyLivenessInfo().isEmpty())
+                        return true;   // we have to report this row even without any cells
+                    if (advanceToColumn())
+                        return true;
+                    // There is no column. We may have advanced to a new row or new partition.
+                    break;
+                case PARTITION:
+                case EXHAUSTED:
+                    return false;
+                case SIMPLE_COLUMN:
+                case COMPLEX_COLUMN:
+                case COMPLEX_COLUMN_CELL:
+                    // Can't jump directly from partition to cell.
+                default:
+                    throw new AssertionError();
+            }
+            current = wrapped.type();
+        }
+    }
+
+    public Type type()
+    {
+        return type;
+    }
+
+    public DecoratedKey partitionKey()
+    {
+        return wrapped.partitionKey();
+    }
+
+    public DeletionTime partitionLevelDeletion()
+    {
+        return wrapped.partitionLevelDeletion();
+    }
+
+    public ClusteringPrefix clusteringKey()
+    {
+        return wrapped.clusteringKey();
+    }
+
+    public LivenessInfo clusteringKeyLivenessInfo()
+    {
+        return wrapped.clusteringKeyLivenessInfo();
+    }
+
+    public DeletionTime rowLevelDeletion()
+    {
+        return wrapped.rowLevelDeletion();
+    }
+
+    public DeletionTime activeRangeDeletion()
+    {
+        return wrapped.activeRangeDeletion();
+    }
+
+    public DeletionTime complexColumnDeletion()
+    {
+        return wrapped.complexColumnDeletion();
+    }
+
+    public ColumnMetadata column()
+    {
+        return wrapped.column();
+    }
+
+    public Cell cell()
+    {
+        return wrapped.cell();
+    }
+
+    public long bytesProcessed()
+    {
+        return wrapped.bytesProcessed();
+    }
+
+    public long bytesTotal()
+    {
+        return wrapped.bytesTotal();
+    }
+
+    public void close()
+    {
+        wrapped.close();
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/compaction/SortedStringTableCursor.java b/src/java/org/apache/cassandra/io/sstable/compaction/SortedStringTableCursor.java
new file mode 100644
index 000000000000..988787bf2e71
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/compaction/SortedStringTableCursor.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.compaction;
+
+import java.io.IOException;
+
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBoundOrBoundary;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.UnfilteredValidation;
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Cursor over sstable data files.
+ * Supports both BIG and BTI formats (which differ only in index and whose data file formats are identical).
+ */
+public class SortedStringTableCursor implements SSTableCursor
+{
+    private final RandomAccessReader dataFile;
+    private final SSTableReader sstable;
+    private final DeserializationHelper helper;
+    private final SerializationHeader header;
+
+    private DecoratedKey partitionKey;
+    private ClusteringPrefix<?> clusteringKey;
+
+    private int rowFlags;
+    private int currentColumnIndex;
+    private int columnsToRead;
+    private ColumnMetadata[] columns;
+    private final ColumnMetadata[] columnsReusableArray;
+    private final ColumnMetadata[] regularColumns;
+    private final ColumnMetadata[] staticColumns;
+    private ColumnMetadata columnMetadata;
+    private int cellsLeftInColumn;
+    private Cell<byte[]> currentCell;
+
+    private DeletionTime partitionLevelDeletion;
+    private DeletionTime activeRangeDeletion;
+    private DeletionTime rowLevelDeletion;
+    private LivenessInfo rowLivenessInfo;
+    private DeletionTime complexColumnDeletion;
+
+    private final long endPosition;
+    private final long startPosition;
+
+    private Type currentType = Type.UNINITIALIZED;
+
+    public SortedStringTableCursor(SSTableReader sstable)
+    {
+        this(sstable, sstable.openDataReader(), null);
+    }
+
+    public SortedStringTableCursor(SSTableReader sstable, Range<Token> range)
+    {
+        this(sstable, sstable.openDataReader(), range);
+    }
+
+    public SortedStringTableCursor(SSTableReader sstable, Range<Token> tokenRange, RateLimiter limiter)
+    {
+        this(sstable, sstable.openDataReader(limiter), tokenRange);
+    }
+
+    public SortedStringTableCursor(SSTableReader sstable, RandomAccessReader dataFile, Range<Token> tokenRange)
+    {
+        try
+        {
+            this.dataFile = dataFile;
+            this.header = sstable.header;
+            this.helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
+            this.sstable = sstable;
+            this.activeRangeDeletion = DeletionTime.LIVE;
+            this.regularColumns = toArray(header.columns(false));
+            this.staticColumns = toArray(header.columns(true));
+            this.columnsReusableArray = new ColumnMetadata[Math.max(regularColumns.length, staticColumns.length)];
+
+            SSTableReader.PartitionPositionBounds bounds = tokenRange == null ? sstable.getPositionsForFullRange()
+                                                                              : sstable.getPositionsForBounds(Range.makeRowRange(tokenRange));
+            if (bounds != null)
+            {
+                this.startPosition = bounds.lowerPosition;
+                this.endPosition = bounds.upperPosition;
+            }
+            else
+            {
+                // The range is empty. Rather than fail, use 0/0 as bounds which will not return any data.
+                this.endPosition = this.startPosition = 0;
+            }
+        }
+        catch (Throwable t)
+        {
+            dataFile.close();
+            throw t;
+        }
+    }
+
+    private static ColumnMetadata[] toArray(Columns columns)
+    {
+        ColumnMetadata[] array = new ColumnMetadata[columns.size()];
+        int index = 0;
+        for (ColumnMetadata cm : columns)
+            array[index++] = cm;
+        assert index == array.length;
+        return array;
+    }
+
+    private boolean consumePartitionHeader() throws IOException
+    {
+        if (dataFile.getFilePointer() == endPosition)
+        {
+            currentType = Type.EXHAUSTED;
+            return false;
+        }
+        else if (dataFile.getFilePointer() > endPosition)
+        {
+            throw new IOException(String.format("Consuming partition header at %s, but end position is %s",
+                                                dataFile.getFilePointer(), endPosition));
+        }
+
+        currentType = Type.PARTITION;
+        partitionKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(dataFile));
+        partitionLevelDeletion = DeletionTime.getSerializer(sstable.descriptor.version).deserialize(dataFile);
+        if (!partitionLevelDeletion.validate())
+            UnfilteredValidation.handleInvalid(sstable.metadata(), partitionKey, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
+        if (!activeRangeDeletion.isLive())
+            throw new IOException(String.format("Invalid active range tombstone at the beginning of partition %s: %s",
+                                                partitionKey.toString(),
+                                                activeRangeDeletion));
+        rowLevelDeletion = null;
+        rowLivenessInfo = null;
+        complexColumnDeletion = null;
+        currentCell = null;
+        columnMetadata = null;
+        clusteringKey = null;
+        return true;
+    }
+
+    private boolean consumeUnfilteredHeader() throws IOException
+    {
+        boolean haveData;
+        do
+        {
+            rowFlags = dataFile.readUnsignedByte();
+            if (UnfilteredSerializer.isEndOfPartition(rowFlags))
+                return false;
+
+            int rowExtendedFlags = UnfilteredSerializer.readExtendedFlags(dataFile, rowFlags);
+
+            switch (UnfilteredSerializer.kind(rowFlags))
+            {
+                case ROW:
+                    haveData = consumeRowHeader(rowExtendedFlags);
+                    currentType = Type.ROW;
+                    break;
+                case RANGE_TOMBSTONE_MARKER:
+                    haveData = consumeRangeTombstoneMarker();
+                    currentType = Type.RANGE_TOMBSTONE;
+                    break;
+                default:
+                    throw new AssertionError();
+            }
+        }
+        while (!haveData);
+        complexColumnDeletion = null;
+        currentCell = null;
+        columnMetadata = null;
+        return true;
+    }
+
+    private boolean consumeRangeTombstoneMarker() throws IOException
+    {
+        ClusteringBoundOrBoundary<?> bound = ClusteringBoundOrBoundary.serializer.deserialize(dataFile, helper.version, header.clusteringTypes());
+
+        if (header.isForSSTable())
+        {
+            dataFile.readUnsignedVInt(); // marker size
+            dataFile.readUnsignedVInt(); // previous unfiltered size
+        }
+
+        if (bound.kind().isEnd())
+        {
+            DeletionTime endDeletion = header.readDeletionTime(dataFile);
+            if (!endDeletion.equals(activeRangeDeletion))
+                throw new IOException(String.format("Invalid tombstone end boundary in partition %s, expected %s was %s",
+                                                    partitionKey.toString(),
+                                                    activeRangeDeletion,
+                                                    endDeletion));
+        }
+
+        if (bound.kind().isStart())
+            rowLevelDeletion = header.readDeletionTime(dataFile);
+        else
+            rowLevelDeletion = DeletionTime.LIVE;
+
+        if (!rowLevelDeletion.validate())
+            UnfilteredValidation.handleInvalid(sstable.metadata(), partitionKey, sstable, "rowLevelDeletion="+rowLevelDeletion.toString());
+
+        clusteringKey = bound;
+        return true;
+    }
+
+    /**
+     * @return false if empty
+     * @throws IOException
+     */
+    private boolean consumeRowHeader(int rowExtendedFlags) throws IOException
+    {
+        boolean isStatic = UnfilteredSerializer.isStatic(rowExtendedFlags);
+
+        if (isStatic)
+        {
+            if (!header.hasStatic())
+                throw new IOException(String.format("Static row encountered in partition %s on table without static columns",
+                                                    partitionKey.toString()));
+
+            clusteringKey = Clustering.STATIC_CLUSTERING;
+        }
+        else
+            clusteringKey = Clustering.serializer.deserialize(dataFile, helper.version, header.clusteringTypes());
+
+        if (header.isForSSTable())
+        {
+            dataFile.readUnsignedVInt(); // Skip row size
+            dataFile.readUnsignedVInt(); // previous unfiltered size
+        }
+
+        boolean hasTimestamp = (rowFlags & UnfilteredSerializer.HAS_TIMESTAMP) != 0;
+        boolean hasTTL = (rowFlags & UnfilteredSerializer.HAS_TTL) != 0;
+        boolean hasDeletion = (rowFlags & UnfilteredSerializer.HAS_DELETION) != 0;
+        // shadowable deletions are obsolete
+        boolean hasAllColumns = (rowFlags & UnfilteredSerializer.HAS_ALL_COLUMNS) != 0;
+        ColumnMetadata[] headerColumns = isStatic ? staticColumns : regularColumns;
+
+        if (hasTimestamp)
+        {
+            long timestamp = header.readTimestamp(dataFile);
+            int ttl = hasTTL ? header.readTTL(dataFile) : LivenessInfo.NO_TTL;
+            long localDeletionTime = hasTTL ? header.readLocalDeletionTime(dataFile) : LivenessInfo.NO_EXPIRATION_TIME;
+            localDeletionTime = Cell.decodeLocalDeletionTime(localDeletionTime, ttl, helper);
+            rowLivenessInfo = LivenessInfo.withExpirationTime(timestamp, ttl, localDeletionTime);
+            if (rowLivenessInfo.isExpiring() && (rowLivenessInfo.ttl() < 0 || rowLivenessInfo.localExpirationTime() < 0))
+                UnfilteredValidation.handleInvalid(sstable.metadata(), partitionKey, sstable, "rowLivenessInfo="+rowLivenessInfo.toString());
+        }
+        else
+            rowLivenessInfo = LivenessInfo.EMPTY;
+
+        if (hasDeletion)
+        {
+            rowLevelDeletion = header.readDeletionTime(dataFile);
+            if (!rowLevelDeletion.validate())
+                UnfilteredValidation.handleInvalid(sstable.metadata(), partitionKey, sstable, "rowLevelDeletion="+rowLevelDeletion.toString());
+        }
+        else
+            rowLevelDeletion = DeletionTime.LIVE;
+
+        if (hasAllColumns)
+        {
+            columns = headerColumns;
+            columnsToRead = headerColumns.length;
+        }
+        else
+        {
+            columns = columnsReusableArray;
+            columnsToRead = Columns.serializer.deserializeSubset(headerColumns, dataFile, columns);
+        }
+
+        if (!hasTimestamp && !hasDeletion && columnsToRead == 0)
+            return false;
+
+        this.cellsLeftInColumn = 0;
+        this.currentColumnIndex = 0;
+        return true;
+    }
+
+    public boolean consumeColumn() throws IOException
+    {
+        while (true)
+        {
+            if (cellsLeftInColumn == 0)
+            {
+                if (currentColumnIndex == columnsToRead)
+                    return false;
+
+                columnMetadata = columns[currentColumnIndex++];
+                assert helper.includes(columnMetadata);  // we are fetching all columns
+                if (columnMetadata.isComplex())
+                {
+                    helper.startOfComplexColumn(columnMetadata);
+                    DeletionTime complexDeletion = DeletionTime.LIVE;
+                    if ((rowFlags & UnfilteredSerializer.HAS_COMPLEX_DELETION) != 0)
+                    {
+                        complexDeletion = header.readDeletionTime(dataFile);
+                        if (!complexDeletion.validate())
+                            UnfilteredValidation.handleInvalid(sstable.metadata(), partitionKey, sstable,
+                                                               "complexColumnDeletion="+complexDeletion.toString());
+                        if (helper.isDroppedComplexDeletion(complexDeletion))
+                            complexDeletion = DeletionTime.LIVE;
+                    }
+
+                    cellsLeftInColumn = (int) dataFile.readUnsignedVInt();
+
+                    currentType = Type.COMPLEX_COLUMN;
+                    complexColumnDeletion = complexDeletion;
+                    return true;
+                    // not issuing helper.endOfComplexColumn, but that should be okay
+                }
+                else
+                {
+                    currentType = Type.SIMPLE_COLUMN;
+                    Cell<byte[]> cell = Cell.serializer.deserialize(dataFile, rowLivenessInfo, columnMetadata, header, helper, ByteArrayAccessor.instance);
+                    if (cell.hasInvalidDeletions())
+                        UnfilteredValidation.handleInvalid(sstable.metadata(), partitionKey, sstable, cell.toString());
+                    if (!helper.isDropped(cell, false))
+                    {
+                        currentCell = cell;
+                        return true;
+                    }
+                }
+            }
+        }
+    }
+
+    public boolean consumeComplexCell() throws IOException
+    {
+        while (cellsLeftInColumn > 0)
+        {
+            --cellsLeftInColumn;
+            Cell<byte[]> cell = Cell.serializer.deserialize(dataFile, rowLivenessInfo, columnMetadata, header, helper, ByteArrayAccessor.instance);
+            if (cell.hasInvalidDeletions())
+                UnfilteredValidation.handleInvalid(sstable.metadata(), partitionKey, sstable, cell.toString());
+            if (!helper.isDropped(cell, true))
+            {
+                currentType = Type.COMPLEX_COLUMN_CELL;
+                currentCell = cell;
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public Type advance()
+    {
+        if (currentType == Type.RANGE_TOMBSTONE)
+            activeRangeDeletion = rowLevelDeletion;
+
+        try
+        {
+            switch (currentType)
+            {
+                case EXHAUSTED:
+                    throw new IllegalStateException("Cursor advanced after exhaustion.");
+                case COMPLEX_COLUMN_CELL:
+                case COMPLEX_COLUMN:
+                    if (consumeComplexCell())
+                        return currentType;
+                    // else fall through
+                case SIMPLE_COLUMN:
+                case ROW:
+                    if (consumeColumn())
+                        return currentType;
+                    // else fall through
+                case RANGE_TOMBSTONE:
+                case PARTITION:
+                    if (consumeUnfilteredHeader())
+                        return currentType;
+
+                    consumePartitionHeader();
+                    return currentType;
+                case UNINITIALIZED:
+                    dataFile.seek(this.startPosition);
+                    consumePartitionHeader();
+                    return currentType;
+                default:
+                    throw new AssertionError();
+            }
+        }
+        catch (CorruptSSTableException e)
+        {
+            sstable.markSuspect();
+            throw e;
+        }
+        catch (IOException | IndexOutOfBoundsException | AssertionError e)
+        {
+            sstable.markSuspect();
+            throw new CorruptSSTableException(e, dataFile.getFile().path());
+        }
+    }
+
+    public Type type()
+    {
+        return currentType;
+    }
+
+    public DecoratedKey partitionKey()
+    {
+        return partitionKey;
+    }
+
+    public DeletionTime partitionLevelDeletion()
+    {
+        return partitionLevelDeletion;
+    }
+
+    public ClusteringPrefix<?> clusteringKey()
+    {
+        return clusteringKey;
+    }
+
+    public LivenessInfo clusteringKeyLivenessInfo()
+    {
+        return rowLivenessInfo;
+    }
+
+    public DeletionTime rowLevelDeletion()
+    {
+        return rowLevelDeletion;
+    }
+
+    public DeletionTime activeRangeDeletion()
+    {
+        return activeRangeDeletion;
+    }
+
+    public DeletionTime complexColumnDeletion()
+    {
+        return complexColumnDeletion;
+    }
+
+    public ColumnMetadata column()
+    {
+        return columnMetadata;
+    }
+
+    public Cell<byte[]> cell()
+    {
+        return currentCell;
+    }
+
+    public long bytesProcessed()
+    {
+        return dataFile.getFilePointer() - startPosition;
+    }
+
+    public long bytesTotal()
+    {
+        return endPosition - startPosition;
+    }
+
+    public void close()
+    {
+        FileUtils.closeQuietly(dataFile);
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/compaction/cursors.md b/src/java/org/apache/cassandra/io/sstable/compaction/cursors.md
new file mode 100644
index 000000000000..d97c8be26646
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/compaction/cursors.md
@@ -0,0 +1,273 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Compaction process optimization
+
+This is a refactoring of the sstable content iteration mechanisms to strip much of the avoidable overhead. The main idea
+is the restructuring of the iteration into one stream of cells mixed with column, row and partition headers, which
+avoids having to recreate the `Row`s and `UnfilteredRowIterator`s together with all the infrastructure needed to merge
+them.
+
+The data are exposed via a stateful `SSTableCursor` object, which starts uninitialized and can be `advance()`-d to
+obtain items of the stream. The cursor exposes the state of iteration, including:
+
+- the current partition key
+- the current clustering key
+- the current column
+- the current cell (which may include a cell path)
+
+together with an indication of the "level" in the sstable hierarchy that the current position is at. On a valid
+position, the latter can be `PARTITION`, `ROW`/`RANGE_TOMBSTONE`, `SIMPLE/COMPLEX_COLUMN` or `COMPLEX_COLUMN_CELL`,
+indicating that advancing has ended up, respectively, at the start of a new partition, new row, new range tombstone
+marker, new simple column cell, the start of a complex column, or a cell in the complex column. If on an upper level of
+the hierarchy, the details about features of the lower levels (e.g. clustering key on `PARTITION`, cell on `ROW`
+or `PARTITION`) are invalid, but the information for all higher levels (e.g. partition key on `ROW`) is present.
+
+Cursors' state (i.e. position in the stream) can be compared by comparing the keys in order. More importantly, when we
+iterate several cursors in order while merging, we can observe that a cursor positioned on a higher level of the
+hierarchy must have a position later than the cursors that are still producing items at a lower level and thus
+comparisons can be done by only comparing the level and then (if levels match), the key at the common level. This is
+crucial for the efficiency of merging.
+
+Note: we also considered using a "level change" flag instead of stopping on headers (i.e. instead of advancing through
+`PARTITION(pk1), ROW(ck1), SIMPLE_COLUMN(c1), ROW(ck2), SIMPLE_COLUMN(c2)` to
+list `PARTITION(pk1, ck1, c1), ROW(ck2, c2)`). While it does look like with this we can still efficiently compare via
+the level, we need to somehow consume the level advance on matching comparison, which is non-trivial and error-prone.
+For example, consider merging:
+
+- `PARTITION(pk1, ck1, c1)`, `ROW(ck3, c2)`
+- `PARTITION(pk1, ck2, c3)`
+
+Cell, row and partition deletions are directly reported by methods that return the relevant deletion times. Range
+tombstone markers are reported on the rows level with the deletion time they switch to by the `rowLevelDeletion()`
+method (i.e. the open time if it's an open bound or a boundary, or LIVE if it's a close bound). The currently active
+deletion time is also tracked and reported through the `activeRangeDeletion()` method; note that if the stream is
+positioned or a range tombstone marker, it reports the deletion active _before_ it, so that both deletions are
+available (useful both for reconstructing range tombstone markers on write, and for merging, where we need to know the
+active range deletion before the position on the sources that are positioned later in the stream). The merge cursor
+takes care of applying the active deletion (the newest of complex-column, range, row- and partition-level deletion) to
+the data it processes to remove any deleted data and tombstones.
+
+There are a couple of further differences with iterators:
+
+- Static rows are listed as the first row in a partition, only if they are not empty &mdash; separating them is only
+  useful for reverse iteration which cursors don't aim to support.
+- Cursors only iterate on data files, which avoids walking the partition index. This means less resilience to error, but
+  in compactions this is not a problem as we abort on error.
+- "Shadowable row deletions" (a deprecated feature which is no longer in use) are not reported as such.
+
+Cursors don't currently support all of the functionality of merging over sstable iterators. For details of the
+limitations, see the TODO list below.
+
+Beyond the above, the implementation is straight-forward:
+
+- `SSTableCursor` is the main abstraction, a cursor over sstables.
+- `SortedStringTableCursor` is the main implementation of `SSTableCursor` which walks over an sstable data file and
+  extracts its data stream. To do this it reimplements the functionality of the deserializers for parsing partition, row
+  and column headers and relies on an instance of the deserializer to read cells. Supports both BIG and BTI formats
+  (which differ only in index and whose data file formats are identical).
+- `SSTableCursorMerger` implements merging several `SSTableCursor`s into one. This is implemented via an extracted merge
+  core from `MergeIterator` configured to work on cursors.
+- `PurgeCursor` implements removal of collectable tombstones.
+- `SkipEmptyDataCursor` delays the reporting of headers until content is found, in order to avoid creating empty complex
+  columns, rows or partitions in the compacted view.
+- `CompactionCursor` sets up a merger over multiple sstable cursors for compaction and implements writing a cursor into
+  a new sstable. Note: we currently still create an in-memory row to be able to send it to the serializer for writing.
+- `CompactionTask.CompactionOperationCursor` is a cursor counterpart of `CompactionTask.CompactionOperationIterator`.
+  The former is chosen if cursors can support the compaction, i.e. if it is known that a secondary index is not in use,
+  that the compaction strategy supports cursors (initially we only intend to release this
+  for `UnifiedCompactionStrategy`; even afterwards, `TieredCompactionStrategy` would need special support), and that a
+  garbage collection compaction is not requested.
+
+Additionally,
+
+- `IteratorFromCursor` converts a cursor into an unfiltered partition iterator for testing and can also be used as a
+  reference of the differences.
+
+### Further work
+
+- Writing without going through `Row`, i.e. sending individual cells to the writer instead of the in-memory `Row`
+  objects, using a refactoring similar to what is currently done to write rows instead of partitions, should improve
+  performance further.
+
+- Secondary indexes currently can't use cursors, because we do not construct the right input for their listeners. Doing
+  this may require reconstructing rows for the merge, which is something we definitely do not want to do in the normal
+  case. It would probably be better to find the specific needs of SAI and support only them, and leave legacy / custom
+  indexes to use iterators (note: since TPC is not going to be developed further, we no longer plan to fully replace the
+  iterators with this).
+
+- Garbage collection compaction, i.e. compaction using tombstones from newer non-participating sstables to delete as
+  much as possible from the compacted sstables, could be implemented for cursors too.
+
+- If we are going to support all compaction strategies, it may be beneficial to restore levelled compaction's sstable
+  concatenation scanner. However, this will only save one comparison per partition, so I doubt it's really worth doing.
+
+## Benchmark results collected during development (most recent results first)
+
+Perhaps most relevant at this time are the differences between `iterateThroughCompactionCursor` vs
+`iterateThroughCompactionIterator` (sending the compaction of two sstables to a null writer). Other meaninful
+comparisons are `iterateThroughCursor` vs `iterateThroughTableScanner`
+(iterating the content of a single sstable without merging) and `iterateThroughMergeCursor` vs
+`iterateThroughMergeIterator` (iterating the merge of two sstables, similar to `iterateThroughCompactionCursor`
+but without constructing in-memory rows).
+
+```
+Benchmark                                                               (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score    Error  Units
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   818.816 ± 46.403  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursorWithLimiter                          0             1          false        DEFAULT             0.3      10               2  avgt   10   841.232 ± 41.720  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                                   0             1          false        DEFAULT             0.3      10               2  avgt   10  1731.936 ± 72.224  ms/op
+CompactionBreakdownBenchmark.iterateThroughCursor                                               0             1          false        DEFAULT             0.3      10               2  avgt   10   480.876 ± 45.553  ms/op
+CompactionBreakdownBenchmark.iterateThroughCursorToIterator                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   602.826 ± 42.933  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeCursor                                          0             1          false        DEFAULT             0.3      10               2  avgt   10   719.327 ± 41.238  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeIterator                                        0             1          false        DEFAULT             0.3      10               2  avgt   10  1406.228 ± 74.427  ms/op
+CompactionBreakdownBenchmark.iterateThroughPartitionIndexIterator                               0             1          false        DEFAULT             0.3      10               2  avgt   10   340.876 ± 27.913  ms/op
+CompactionBreakdownBenchmark.iterateThroughTableScanner                                         0             1          false        DEFAULT             0.3      10               2  avgt   10  1039.944 ± 89.224  ms/op
+```
+
+```
+Benchmark                                                      (compactionMbSecThrottle)  (compactors)  (compression)               (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score     Error  Units
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false                     DEFAULT             0.3      10               2  avgt   10   812.300 ±  35.196  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false                     DEFAULT             0.3      10               2  avgt   10  1799.127 ± 100.290  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false            BLOB_CLUSTER_KEY             0.3      10               2  avgt   10   874.638 ±  46.639  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false            BLOB_CLUSTER_KEY             0.3      10               2  avgt   10  1813.990 ±  89.474  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false                  BLOB_VALUE             0.3      10               2  avgt   10   850.173 ±  37.608  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false                  BLOB_VALUE             0.3      10               2  avgt   10  1773.747 ±  82.984  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false           MANY_CLUSTER_KEYS             0.3      10               2  avgt   10  1501.582 ±  91.084  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false           MANY_CLUSTER_KEYS             0.3      10               2  avgt   10  2470.640 ±  79.072  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false                 MANY_FIELDS             0.3      10               2  avgt   10  2643.602 ±  85.875  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false                 MANY_FIELDS             0.3      10               2  avgt   10  3095.176 ±  71.408  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false             WIDE_PARTITIONS             0.3      10               2  avgt   10   524.839 ±  19.716  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false             WIDE_PARTITIONS             0.3      10               2  avgt   10   564.349 ±  20.299  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false      COMPLEX_COLUMNS_INSERT             0.3      10               2  avgt   10  1735.086 ±  92.367  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false      COMPLEX_COLUMNS_INSERT             0.3      10               2  avgt   10  2730.369 ±  93.184  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false  COMPLEX_COLUMNS_UPDATE_SET             0.3      10               2  avgt   10  1691.803 ±  83.824  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false  COMPLEX_COLUMNS_UPDATE_SET             0.3      10               2  avgt   10  2671.245 ±  90.731  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false  COMPLEX_COLUMNS_UPDATE_ADD             0.3      10               2  avgt   10  1541.798 ±  96.077  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false  COMPLEX_COLUMNS_UPDATE_ADD             0.3      10               2  avgt   10  2649.346 ± 101.025  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false                  TOMBSTONES             0.3      10               2  avgt   10   971.576 ±  76.193  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false                  TOMBSTONES             0.3      10               2  avgt   10  1908.025 ±  80.601  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false             TOMBSTONES_WIDE             0.3      10               2  avgt   10   594.306 ±  11.245  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false             TOMBSTONES_WIDE             0.3      10               2  avgt   10   701.378 ±  15.859  ms/op
+```
+
+```
+Benchmark                                            (compaction)  (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score     Error  Units
+CompactionBenchmark.compactSSTables  SizeTieredCompactionStrategy                          0             1          false        DEFAULT             0.3      10               2  avgt   10  4151.567 ± 393.045  ms/op
+CompactionBenchmark.compactSSTables     UnifiedCompactionStrategy                          0             1          false        DEFAULT             0.3      10               2  avgt   10  3097.753 ±  90.189  ms/op
+```
+
+With tombstone and purging support, no row reconstructing in `iterateThroughCompactionCursor`.
+
+```
+Benchmark                                                               (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score    Error  Units
+CompactionBenchmark.compactSSTables                                                             0             1          false        DEFAULT             0.3      10               2  avgt   10  3275.471 ± 99.962  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   726.345 ± 53.326  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursorWithLimiter                          0             1          false        DEFAULT             0.3      10               2  avgt   10   705.488 ± 40.847  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                                   0             1          false        DEFAULT             0.3      10               2  avgt   10  1820.006 ± 92.980  ms/op
+CompactionBreakdownBenchmark.iterateThroughCursor                                               0             1          false        DEFAULT             0.3      10               2  avgt   10   463.956 ± 32.461  ms/op
+CompactionBreakdownBenchmark.iterateThroughCursorToIterator                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   595.363 ± 42.723  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeCursor                                          0             1          false        DEFAULT             0.3      10               2  avgt   10   699.307 ± 44.041  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeIterator                                        0             1          false        DEFAULT             0.3      10               2  avgt   10  1434.870 ± 84.047  ms/op
+CompactionBreakdownBenchmark.iterateThroughPartitionIndexIterator                               0             1          false        DEFAULT             0.3      10               2  avgt   10   347.405 ± 25.364  ms/op
+CompactionBreakdownBenchmark.iterateThroughTableScanner                                         0             1          false        DEFAULT             0.3      10               2  avgt   10  1014.317 ± 90.495  ms/op
+CompactionBreakdownBenchmark.scannerToCompactionWriter                                          0             1          false        DEFAULT             0.3      10               2  avgt   10  2000.747 ± 76.360  ms/op
+```
+
+With tombstone and purging support
+
+```
+Benchmark                                                               (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score    Error  Units
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   701.806 ± 42.179  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursorWithLimiter                          0             1          false        DEFAULT             0.3      10               2  avgt   10   710.938 ± 41.999  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                                   0             1          false        DEFAULT             0.3      10               2  avgt   10  1777.159 ± 87.460  ms/op
+CompactionBreakdownBenchmark.iterateThroughCursor                                               0             1          false        DEFAULT             0.3      10               2  avgt   10   457.567 ± 27.544  ms/op
+CompactionBreakdownBenchmark.iterateThroughCursorToIterator                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   592.213 ± 36.001  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeCursor                                          0             1          false        DEFAULT             0.3      10               2  avgt   10   747.161 ± 52.323  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeIterator                                        0             1          false        DEFAULT             0.3      10               2  avgt   10  1500.166 ± 87.268  ms/op
+CompactionBreakdownBenchmark.iterateThroughPartitionIndexIterator                               0             1          false        DEFAULT             0.3      10               2  avgt   10   345.971 ± 24.798  ms/op
+CompactionBreakdownBenchmark.iterateThroughTableScanner                                         0             1          false        DEFAULT             0.3      10               2  avgt   10  1040.594 ± 82.862  ms/op
+CompactionBreakdownBenchmark.scannerToCompactionWriter                                          0             1          false        DEFAULT             0.3      10               2  avgt   10  1968.898 ± 98.314  ms/op
+CompactionBenchmark.compactSSTables                                                             0             1          false        DEFAULT             0.3      10               2  avgt   10  3072.202 ± 104.127  ms/op
+```
+
+Cell-level stream, deserialized cells, recombined rows on write
+
+```
+Benchmark                                                               (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score     Error  Units
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   799.025 ±  26.943  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursorWithLimiter                          0             1          false        DEFAULT             0.3      10               2  avgt   10   795.218 ±  17.373  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                                   0             1          false        DEFAULT             0.3      10               2  avgt   10  1996.695 ±  50.663  ms/op
+CompactionBreakdownBenchmark.iterateThroughCursor                                               0             1          false        DEFAULT             0.3      10               2  avgt   10   488.837 ±   9.936  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeCursor                                          0             1          false        DEFAULT             0.3      10               2  avgt   10   788.899 ±  20.713  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeIterator                                        0             1          false        DEFAULT             0.3      10               2  avgt   10  1698.324 ± 199.704  ms/op
+CompactionBreakdownBenchmark.iterateThroughPartitionIndexIterator                               0             1          false        DEFAULT             0.3      10               2  avgt   10   394.046 ±  11.057  ms/op
+CompactionBreakdownBenchmark.iterateThroughTableScanner                                         0             1          false        DEFAULT             0.3      10               2  avgt   10  1250.365 ±  45.029  ms/op
+```
+
+With direct write, row stream
+
+```
+Benchmark                                                               (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score    Error  Units
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   960.035 ± 21.594  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursorWithLimiter                          0             1          false        DEFAULT             0.3      10               2  avgt   10   966.950 ± 43.067  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                                   0             1          false        DEFAULT             0.3      10               2  avgt   10  2062.935 ± 30.653  ms/op
+CompactionBenchmark.compactSSTables                                                             0             1          false        DEFAULT             0.3      10               2  avgt   10  3294.247 ± 101.186  ms/op
+```
+
+With progress indication and rate limiting, row stream, converted to iterator
+
+```
+Benchmark                                                               (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score    Error  Units
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   964.176 ± 19.588  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionCursorWithLimiter                          0             1          false        DEFAULT             0.3      10               2  avgt   10  1000.995 ± 28.157  ms/op       Note: has synchronization, uncontended in this test
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                                   0             1          false        DEFAULT             0.3      10               2  avgt   10  2043.687 ± 35.059  ms/op
+CompactionBenchmark.compactSSTables                          									0             1          false        DEFAULT             0.3      10               2  avgt   10  3575.346 ± 94.917  ms/op
+```
+
+With CompactionCursor, merge through cursor, deserialized rows, no progress/deletions/indexes
+
+```
+Benchmark                                                      (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score    Error  Units
+CompactionBreakdownBenchmark.iterateThroughCompactionCursor                            0             1          false        DEFAULT             0.3      10               2  avgt   10   944.843 ± 13.718  ms/op
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                          0             1          false        DEFAULT             0.3      10               2  avgt   10  2070.773 ± 43.277  ms/op
+CompactionBenchmark.compactSSTables               							           0             1          false        DEFAULT             0.3      10               2  avgt    9  3329.419 ± 81.107  ms/op
+```
+
+Initial implementation, row stream, converted to iterator
+
+```
+Benchmark                                                          (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt     Score     Error  Units
+CompactionBreakdownBenchmark.iterateThroughCompactionIterator                              0             1          false        DEFAULT             0.3      10               2  avgt   10  2064.699 ±  27.296  ms/op
+CompactionBreakdownBenchmark.iterateThroughCursor                                          0             1          false        DEFAULT             0.3      10               2  avgt   10   597.398 ±  20.261  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeCursor                                     0             1          false        DEFAULT             0.3      10               2  avgt   10   949.216 ±  13.484  ms/op
+CompactionBreakdownBenchmark.iterateThroughMergeIterator                                   0             1          false        DEFAULT             0.3      10               2  avgt   10  1619.175 ±  39.231  ms/op
+CompactionBreakdownBenchmark.iterateThroughPartitionIndexIterator                          0             1          false        DEFAULT             0.3      10               2  avgt   10   413.512 ± 100.120  ms/op
+CompactionBreakdownBenchmark.iterateThroughTableScanner                                    0             1          false        DEFAULT             0.3      10               2  avgt   10  1167.989 ±  37.205  ms/op
+CompactionBreakdownBenchmark.scannerToCompctionWriter                                      0             1          false        DEFAULT             0.3      10               2  avgt    9  2516.367 ±  82.761  ms/op
+CompactionBenchmark.compactSSTables                                                        0             1          false        DEFAULT             0.3      10               2  avgt    9  4622.173 ± 157.001  ms/op
+```
+
+For information only -- skipping row body
+
+```
+Benchmark                                                          (compactionMbSecThrottle)  (compactors)  (compression)  (dataBuilder)  (overlapRatio)  (size)  (sstableCount)  Mode  Cnt    Score    Error  Units
+CompactionBreakdownBenchmark.iterateThroughCursor                                          0             1          false        DEFAULT             0.3      10               2  avgt   10  402.963 ± 11.514  ms/op   - 200
+CompactionBreakdownBenchmark.iterateThroughMergeCursor                                     0             1          false        DEFAULT             0.3      10               2  avgt   10  668.527 ± 13.703  ms/op   - 300
+CompactionBreakdownBenchmark.iterateThroughTableScanner                                    0             1          false        DEFAULT             0.3      10               2  avgt   10  942.351 ± 24.458  ms/op   - 200
+```
+
diff --git a/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java
index 362926338f53..f7d68b966ba5 100644
--- a/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java
+++ b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java
@@ -17,66 +17,192 @@
  */
 package org.apache.cassandra.io.sstable.filter;
 
+import com.codahale.metrics.Meter;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.LongAdder;
 
-public class BloomFilterTracker
+public abstract class BloomFilterTracker
 {
-    private final LongAdder falsePositiveCount = new LongAdder();
-    private final LongAdder truePositiveCount = new LongAdder();
-    private final LongAdder trueNegativeCount = new LongAdder();
-    private final AtomicLong lastFalsePositiveCount = new AtomicLong();
-    private final AtomicLong lastTruePositiveCount = new AtomicLong();
-    private final AtomicLong lastTrueNegativeCount = new AtomicLong();
-
-    public void addFalsePositive()
-    {
-        falsePositiveCount.increment();
-    }
+    public abstract void addFalsePositive();
+    public abstract void addTruePositive();
+    public abstract void addTrueNegative();
+    public abstract long getFalsePositiveCount();
+    public abstract long getRecentFalsePositiveCount();
+    public abstract double getRecentFalsePositiveRate();
+    public abstract long getTruePositiveCount();
+    public abstract long getRecentTruePositiveCount();
+    public abstract double getRecentTruePositiveRate();
+    public abstract long getTrueNegativeCount();
+    public abstract long getRecentTrueNegativeCount();
+    public abstract double getRecentTrueNegativeRate();
 
-    public void addTruePositive()
+    public static BloomFilterTracker createNoopTracker()
     {
-        truePositiveCount.increment();
+        return NoopBloomFilterTracker.instance;
     }
 
-    public void addTrueNegative()
+    public static BloomFilterTracker createMeterTracker()
     {
-        trueNegativeCount.increment();
+        return new MeterBloomFilterTracker();
     }
 
-    public long getFalsePositiveCount()
+    private static class MeterBloomFilterTracker extends BloomFilterTracker
     {
-        return falsePositiveCount.sum();
-    }
+        private final Meter falsePositiveCount = new Meter();
+        private final Meter truePositiveCount = new Meter();
+        private final Meter trueNegativeCount = new Meter();
+        private final AtomicLong lastFalsePositiveCount = new AtomicLong();
+        private final AtomicLong lastTruePositiveCount = new AtomicLong();
+        private final AtomicLong lastTrueNegativeCount = new AtomicLong();
 
-    public long getRecentFalsePositiveCount()
-    {
-        long fpc = getFalsePositiveCount();
-        long last = lastFalsePositiveCount.getAndSet(fpc);
-        return fpc - last;
-    }
+        @Override
+        public void addFalsePositive()
+        {
+            falsePositiveCount.mark();
+        }
 
-    public long getTruePositiveCount()
-    {
-        return truePositiveCount.sum();
-    }
+        @Override
+        public void addTruePositive()
+        {
+            truePositiveCount.mark();
+        }
 
-    public long getRecentTruePositiveCount()
-    {
-        long tpc = getTruePositiveCount();
-        long last = lastTruePositiveCount.getAndSet(tpc);
-        return tpc - last;
-    }
+        @Override
+        public void addTrueNegative()
+        {
+            trueNegativeCount.mark();
+        }
 
-    public long getTrueNegativeCount()
-    {
-        return trueNegativeCount.sum();
+        @Override
+        public long getFalsePositiveCount()
+        {
+            return falsePositiveCount.getCount();
+        }
+
+        public long getRecentFalsePositiveCount()
+        {
+            long fpc = getFalsePositiveCount();
+            long last = lastFalsePositiveCount.getAndSet(fpc);
+            return fpc - last;
+        }
+
+        @Override
+        public double getRecentFalsePositiveRate()
+        {
+            return falsePositiveCount.getFifteenMinuteRate();
+        }
+
+        @Override
+        public long getTruePositiveCount()
+        {
+            return truePositiveCount.getCount();
+        }
+
+        public long getRecentTruePositiveCount()
+        {
+            long tpc = getTruePositiveCount();
+            long last = lastTruePositiveCount.getAndSet(tpc);
+            return tpc - last;
+        }
+
+        @Override
+        public double getRecentTruePositiveRate()
+        {
+            return truePositiveCount.getFifteenMinuteRate();
+        }
+
+        @Override
+        public long getTrueNegativeCount()
+        {
+            return trueNegativeCount.getCount();
+        }
+
+        public long getRecentTrueNegativeCount()
+        {
+            long tnc = getTrueNegativeCount();
+            long last = lastTrueNegativeCount.getAndSet(tnc);
+            return tnc - last;
+        }
+        @Override
+        public double getRecentTrueNegativeRate()
+        {
+            return trueNegativeCount.getFifteenMinuteRate();
+        }
     }
 
-    public long getRecentTrueNegativeCount()
+    /**
+     * Bloom filter tracker that does nothing and always returns 0 for all counters.
+     *
+     * Bloom Filter tracking is managed on the CFS level, so there is no reason to count anything if an SSTable does not
+     * belong (yet) to a CFS. This tracker is used initially on SSTableReaders and is overwritten during setup
+     * in {@link SSTableReader#setupOnline()} or {@link SSTableReader#setupOnline(ColumnFamilyStore)}}.
+     */
+    private static class NoopBloomFilterTracker extends BloomFilterTracker
     {
-        long tnc = getTrueNegativeCount();
-        long last = lastTrueNegativeCount.getAndSet(tnc);
-        return tnc - last;
+        static final NoopBloomFilterTracker instance = new NoopBloomFilterTracker();
+
+        @Override
+        public void addFalsePositive() {}
+
+        @Override
+        public void addTruePositive() {}
+
+        @Override
+        public void addTrueNegative() {}
+
+        @Override
+        public long getFalsePositiveCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public long getRecentFalsePositiveCount()
+        {
+            return 0;
+        }
+        
+        @Override
+        public double getRecentFalsePositiveRate()
+        {
+            return 0;
+        }
+
+        @Override
+        public long getTruePositiveCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public long getRecentTruePositiveCount()
+        {
+            return 0;
+        }
+        @Override
+        public double getRecentTruePositiveRate()
+        {
+            return 0;
+        }
+
+        @Override
+        public long getTrueNegativeCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public long getRecentTrueNegativeCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public double getRecentTrueNegativeRate()
+        {
+            return 0;
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/AbstractKeyFetcher.java b/src/java/org/apache/cassandra/io/sstable/format/AbstractKeyFetcher.java
new file mode 100644
index 000000000000..3a65a2f70c2b
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/AbstractKeyFetcher.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+public abstract class AbstractKeyFetcher implements IKeyFetcher
+{
+    private final RandomAccessReader reader;
+
+    protected AbstractKeyFetcher(RandomAccessReader reader)
+    {
+        this.reader = reader;
+    }
+
+    @Override
+    public DecoratedKey apply(long keyOffset)
+    {
+        if (keyOffset < 0)
+            return null;
+
+        try
+        {
+            reader.seek(keyOffset);
+            if (reader.isEOF())
+                return null;
+
+            return readKey(reader);
+        }
+        catch (IOException e)
+        {
+            throw new FSReadError(new IOException("Failed to read key from " + reader.getChannel().file(), e), reader.getChannel().file());
+        }
+    }
+
+    public abstract DecoratedKey readKey(RandomAccessReader reader) throws IOException;
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        AbstractKeyFetcher that = (AbstractKeyFetcher) o;
+        return Objects.equals(reader.getChannel().file(), that.reader.getChannel().file());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(reader.getChannel().file());
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("KeyFetcher{file=%s}", reader.getChannel().file());
+    }
+
+    @Override
+    public void close()
+    {
+        reader.close();
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java b/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java
index 0e24fa991d72..cfadf555d91e 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java
@@ -29,30 +29,37 @@
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.SliceDescriptor;
 
 public class CompressionInfoComponent
 {
-    public static CompressionMetadata maybeLoad(Descriptor descriptor, Set<Component> components)
+    public static CompressionMetadata maybeLoad(Descriptor descriptor, Set<Component> components, SliceDescriptor sliceDescriptor)
     {
         if (components.contains(Components.COMPRESSION_INFO))
-            return load(descriptor);
+            return load(descriptor, sliceDescriptor);
 
         return null;
     }
 
-    public static CompressionMetadata loadIfExists(Descriptor descriptor)
+    public static CompressionMetadata loadIfExists(Descriptor descriptor, SliceDescriptor sliceDescriptor)
     {
         if (descriptor.fileFor(Components.COMPRESSION_INFO).exists())
-            return load(descriptor);
+            return load(descriptor, sliceDescriptor);
 
         return null;
     }
 
     public static CompressionMetadata load(Descriptor descriptor)
+    {
+        return load(descriptor, SliceDescriptor.NONE);
+    }
+
+    public static CompressionMetadata load(Descriptor descriptor, SliceDescriptor sliceDescriptor)
     {
         return CompressionMetadata.open(descriptor.fileFor(Components.COMPRESSION_INFO),
                                         descriptor.fileFor(Components.DATA).length(),
-                                        descriptor.version.hasMaxCompressedLength());
+                                        descriptor.version.hasMaxCompressedLength(),
+                                        sliceDescriptor);
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java b/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java
index 9367cb444d80..04584c153c0e 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java
@@ -18,6 +18,8 @@
 
 package org.apache.cassandra.io.sstable.format;
 
+import java.util.Optional;
+
 import org.apache.cassandra.config.Config.FlushCompression;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.compress.CompressedSequentialWriter;
@@ -88,13 +90,21 @@ private static CompressionParams buildCompressionParams(TableMetadata metadata,
                 case fast:
                     if (!compressor.recommendedUses().contains(ICompressor.Uses.FAST_COMPRESSION))
                     {
-                        // The default compressor is generally fast (LZ4 with 16KiB block size)
-                        compressionParams = CompressionParams.DEFAULT;
+                        compressionParams = CompressionParams.FAST;
+                        break;
+                    }
+                    // else fall through
+                case adaptive:
+                    if (!compressor.recommendedUses().contains(ICompressor.Uses.FAST_COMPRESSION))
+                    {
+                        compressionParams = CompressionParams.FAST_ADAPTIVE;
                         break;
                     }
                     // else fall through
                 case table:
                 default:
+                    compressionParams = Optional.ofNullable(compressionParams.forUse(ICompressor.Uses.FAST_COMPRESSION))
+                                                .orElse(compressionParams);
                     break;
             }
         }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java b/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java
index 9f99d7dac0cd..eab8f4ce825f 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java
@@ -31,18 +31,20 @@
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.BloomFilter;
 import org.apache.cassandra.utils.BloomFilterSerializer;
 import org.apache.cassandra.utils.FilterFactory;
 import org.apache.cassandra.utils.IFilter;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.BF_FP_CHANCE_TOLERANCE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BF_RECREATE_ON_FP_CHANCE_CHANGE;
+
 public class FilterComponent
 {
     private static final Logger logger = LoggerFactory.getLogger(FilterComponent.class);
 
-    final static boolean rebuildFilterOnFPChanceChange = false;
-    final static double filterFPChanceTolerance = 0d;
-
     private FilterComponent()
     {
     }
@@ -116,13 +118,18 @@ else if (!components.contains(Components.FILTER) || Double.isNaN(currentFPChance
 
             return null;
         }
-        else if (!isFPChanceDiffNegligible(desiredFPChance, currentFPChance) && rebuildFilterOnFPChanceChange)
+        else if (!isFPChanceDiffNegligible(desiredFPChance, currentFPChance) && BF_RECREATE_ON_FP_CHANCE_CHANGE.getBoolean())
         {
             if (logger.isTraceEnabled())
                 logger.trace("Bloom filter for {} will not be loaded because fpChance has changed from {} to {} and the filter should be recreated", descriptor, currentFPChance, desiredFPChance);
 
             return null;
         }
+        else if (BloomFilter.lazyLoading() && !SchemaConstants.isLocalSystemKeyspace(metadata.keyspace))
+        {
+            logger.debug("postponing bloom filter deserialization for {}", descriptor.fileFor(Components.FILTER));
+            return FilterFactory.AlwaysPresentForLazyLoading;
+        }
 
         try
         {
@@ -140,11 +147,11 @@ else if (!isFPChanceDiffNegligible(desiredFPChance, currentFPChance) && rebuildF
 
     static boolean shouldUseBloomFilter(double fpChance)
     {
-        return !(Math.abs(1 - fpChance) <= filterFPChanceTolerance);
+        return !(Math.abs(1 - fpChance) <= BF_FP_CHANCE_TOLERANCE.getDouble());
     }
 
     static boolean isFPChanceDiffNegligible(double fpChance1, double fpChance2)
     {
-        return Math.abs(fpChance1 - fpChance2) <= filterFPChanceTolerance;
+        return Math.abs(fpChance1 - fpChance2) <= BF_FP_CHANCE_TOLERANCE.getDouble();
     }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java b/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java
index 45dfc62b2c8b..fbf39bd626f6 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java
@@ -19,27 +19,28 @@
 package org.apache.cassandra.io.sstable.format;
 
 import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.IOOptions;
 import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.FileHandle;
 
 public class IndexComponent
 {
-    public static FileHandle.Builder fileBuilder(File file, IOOptions ioOptions, ChunkCache chunkCache)
+    public static FileHandle.Builder fileBuilder(Descriptor descriptor, Component component, IOOptions ioOptions, ChunkCache chunkCache)
     {
-        return new FileHandle.Builder(file).withChunkCache(chunkCache)
-                                           .mmapped(ioOptions.indexDiskAccessMode);
+        return StorageProvider.instance.primaryIndexWriteTimeFileHandleBuilderFor(descriptor, component, ioOptions.indexDiskAccessMode, chunkCache, OperationType.UNKNOWN);
     }
 
     public static FileHandle.Builder fileBuilder(Component component, SSTable ssTable)
     {
-        return fileBuilder(ssTable.descriptor.fileFor(component), ssTable.ioOptions, ssTable.chunkCache);
+        return fileBuilder(ssTable.descriptor, component, ssTable.ioOptions, ssTable.chunkCache);
     }
 
-    public static FileHandle.Builder fileBuilder(Component component, SSTable.Builder<?, ?> builder)
+    public static FileHandle.Builder fileBuilder(Component component, SSTable.Builder<?, ?> builder, OperationType operationType)
     {
-        return fileBuilder(builder.descriptor.fileFor(component), builder.getIOOptions(), builder.getChunkCache());
+        return StorageProvider.instance.primaryIndexWriteTimeFileHandleBuilderFor(builder.descriptor, component, builder.getIOOptions().indexDiskAccessMode, builder.getChunkCache(), operationType);
     }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index 4f622be5a51e..083a87703d4f 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.lang.ref.WeakReference;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -28,6 +29,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -37,12 +39,11 @@
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Longs;
 import com.google.common.util.concurrent.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -59,6 +60,8 @@
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.db.lifecycle.AbstractLogTransaction;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -74,17 +77,19 @@
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.IVerifier;
 import org.apache.cassandra.io.sstable.KeyIterator;
 import org.apache.cassandra.io.sstable.KeyReader;
 import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.SSTableIdFactory;
 import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.metadata.CompactionMetadata;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.CheckedFunction;
 import org.apache.cassandra.io.util.DataIntegrityMetadata;
@@ -93,20 +98,23 @@
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SliceDescriptor;
 import org.apache.cassandra.metrics.RestorableMeter;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.BloomFilter;
 import org.apache.cassandra.utils.EstimatedHistogram;
 import org.apache.cassandra.utils.ExecutorUtils;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.INativeLibrary;
 import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.NativeLibrary;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.RefCounted;
 import org.apache.cassandra.utils.concurrent.SelfRefCounted;
 import org.apache.cassandra.utils.concurrent.SharedCloseable;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
@@ -149,9 +157,9 @@
  * <p>
  * TODO: fill in details about Tracker and lifecycle interactions for tools, and for compaction strategies
  */
-public abstract class SSTableReader extends SSTable implements UnfilteredSource, SelfRefCounted<SSTableReader>
+public abstract class SSTableReader extends SSTable implements UnfilteredSource, SelfRefCounted<SSTableReader>, CompactionSSTable
 {
-    private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
+    protected static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
 
     private static final boolean TRACK_ACTIVITY = CassandraRelevantProperties.DISABLE_SSTABLE_ACTIVITY_TRACKING.getBoolean();
 
@@ -183,15 +191,6 @@ public static final class UniqueIdentifier
     }
     public final UniqueIdentifier instanceId = new UniqueIdentifier();
 
-    public static final Comparator<SSTableReader> firstKeyComparator = (o1, o2) -> o1.getFirst().compareTo(o2.getFirst());
-    public static final Ordering<SSTableReader> firstKeyOrdering = Ordering.from(firstKeyComparator);
-    public static final Comparator<SSTableReader> lastKeyComparator = (o1, o2) -> o1.getLast().compareTo(o2.getLast());
-
-    public static final Comparator<SSTableReader> idComparator = Comparator.comparing(t -> t.descriptor.id, SSTableIdFactory.COMPARATOR);
-    public static final Comparator<SSTableReader> idReverseComparator = idComparator.reversed();
-
-    public static final Comparator<SSTableReader> sizeComparator = (o1, o2) -> Longs.compare(o1.onDiskLength(), o2.onDiskLength());
-
     /**
      * maxDataAge is a timestamp in local server time (e.g. Global.currentTimeMilli) which represents an upper bound
      * to the newest piece of data stored in the sstable. In other words, this sstable does not contain items created
@@ -252,6 +251,19 @@ public enum OpenReason
 
     protected final FileHandle dfile;
 
+    // Unlike readMeter, which is global and tracks access to data files, this meter
+    // is incremented as soon as the partition index is accessed with SinglePartitionReadCommand EQ. This includes
+    // the case where the sstable does not contain the partition the query was looking for.
+    // we use a restorable meter to gain access to the moving averages, we don't
+    // really restore it from disk
+    protected final Optional<RestorableMeter> partitionIndexReadMeter = BloomFilter.lazyLoading()
+                                                                      ? Optional.of(BloomFilter.lazyLoadingWindow() > 0
+                                                                                    // when window > 0, use rate at given window
+                                                                                    ? RestorableMeter.builder().withWindow(BloomFilter.lazyLoadingWindow()).build()
+                                                                                    // when window <= 0, it only cares about absolute count
+                                                                                    : RestorableMeter.builder().build())
+                                                                      : Optional.empty();
+
     // technically isCompacted is not necessary since it should never be unreferenced unless it is also compacted,
     // but it seems like a good extra layer of protection against reference counting bugs to not delete data based on that alone
     public final AtomicBoolean isSuspect = new AtomicBoolean(false);
@@ -261,15 +273,15 @@ public enum OpenReason
 
     public final SerializationHeader header;
 
-    private final InstanceTidier tidy;
+    protected final InstanceTidier tidy;
     private final Ref<SSTableReader> selfRef;
 
     private RestorableMeter readMeter;
 
     private volatile double crcCheckChance;
 
-    protected final DecoratedKey first;
-    protected final DecoratedKey last;
+    public final DecoratedKey first;
+    public final DecoratedKey last;
     public final AbstractBounds<Token> bounds;
 
     /**
@@ -281,12 +293,12 @@ public enum OpenReason
      * @param sstables SSTables to calculate key count
      * @return estimated key count
      */
-    public static long getApproximateKeyCount(Iterable<SSTableReader> sstables)
+    public static long getApproximateKeyCount(Iterable<? extends SSTableReader> sstables)
     {
         long count = -1;
 
         if (Iterables.isEmpty(sstables))
-            return count;
+            return 0;
 
         boolean failed = false;
         ICardinality cardinality = null;
@@ -338,6 +350,31 @@ public static long getApproximateKeyCount(Iterable<SSTableReader> sstables)
         return count;
     }
 
+    /**
+     * The key cardinality estimator for the sstable, if it can be loaded.
+     *
+     * @return the sstable key cardinality estimator created during flush/compaction, or {@code null} if that estimator
+     * cannot be loaded for any reason.
+     */
+    @VisibleForTesting
+    public ICardinality keyCardinalityEstimator()
+    {
+        if (openReason == OpenReason.EARLY)
+            return null;
+
+        try
+        {
+            CompactionMetadata metadata = (CompactionMetadata) descriptor.getMetadataSerializer()
+                                                                         .deserialize(descriptor, MetadataType.COMPACTION);
+            return metadata == null ? null : metadata.cardinalityEstimator;
+        }
+        catch (IOException e)
+        {
+            logger.warn("Reading cardinality from Statistics.db failed for {}.", this, e);
+            return null;
+        }
+    }
+
     public static SSTableReader open(SSTable.Owner owner, Descriptor descriptor)
     {
         return open(owner, descriptor, null);
@@ -447,6 +484,7 @@ public static Collection<SSTableReader> openAll(SSTable.Owner owner, Set<Map.Ent
         return sstables;
     }
 
+
     protected SSTableReader(Builder<?, ?> builder, Owner owner)
     {
         super(builder, owner);
@@ -528,14 +566,32 @@ public int hashCode()
         return this.descriptor.hashCode();
     }
 
+    @Override
     public String getFilename()
     {
         return dfile.path();
     }
 
+    @Override
+    public Descriptor getDescriptor()
+    {
+        return descriptor;
+    }
+
+    @Override
+    public Path getFile()
+    {
+        return descriptor.pathFor(Components.DATA);
+    }
+
+    public SliceDescriptor getDataFileSliceDescriptor()
+    {
+        return dfile.sliceDescriptor;
+    }
+
     public void setupOnline()
     {
-         owner().ifPresent(o -> setCrcCheckChance(o.getCrcCheckChance()));
+        owner().ifPresent(o -> setCrcCheckChance(o.getCrcCheckChance()));
     }
 
     /**
@@ -625,6 +681,29 @@ public RestorableMeter getReadMeter()
         return readMeter;
     }
 
+    @VisibleForTesting
+    @Nullable
+    public RestorableMeter getPartitionIndexReadMeter()
+    {
+        return partitionIndexReadMeter.orElse(null);
+    }
+
+    /**
+     * Called by {@link org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy} and other compaction
+     * strategies to determine the read hotness of this sstables, this method returna a "read hotness" which is
+     * calculated by looking at the last two hours read rate and dividing this number by the estimated number of keys.
+     * <p/>
+     * Note that some system tables do not have read meters, in which case this method will return zero.
+     *
+     * @return the last two hours read rate per estimated key
+     */
+    @Override
+    public double hotness()
+    {
+        // system tables don't have read meters, just use 0.0 for the hotness
+        return readMeter == null ? 0.0 : readMeter.twoHourRate() / estimatedKeys();
+    }
+
     /**
      * All the resources which should be released upon closing this sstable reader are registered with in
      * {@link GlobalTidy}. This method lets close a provided resource explicitly any time and unregister it from
@@ -699,6 +778,7 @@ public long getCompressionMetadataOffHeapSize()
     /**
      * Calculates an estimate of the number of keys in the sstable represented by this reader.
      */
+    @Override
     public abstract long estimatedKeys();
 
     /**
@@ -720,7 +800,7 @@ public long getCompressionMetadataOffHeapSize()
     /**
      * Determine the minimal set of sections that can be extracted from this SSTable to cover the given ranges.
      *
-     * @return A sorted list of (offset,end) pairs that cover the given ranges in the datafile for this SSTable.
+     * @return A sorted list of [offset,end) pairs that cover the given ranges in the datafile for this SSTable.
      */
     public List<PartitionPositionBounds> getPositionsForRanges(Collection<Range<Token>> ranges)
     {
@@ -729,27 +809,122 @@ public List<PartitionPositionBounds> getPositionsForRanges(Collection<Range<Toke
         for (Range<Token> range : Range.normalize(ranges))
         {
             assert !range.isWrapAround() || range.right.isMinimum();
-            // truncate the range so it at most covers the sstable
             AbstractBounds<PartitionPosition> bounds = Range.makeRowRange(range);
-            PartitionPosition leftBound = bounds.left.compareTo(first) > 0 ? bounds.left : first.getToken().minKeyBound();
-            PartitionPosition rightBound = bounds.right.isMinimum() ? last.getToken().maxKeyBound() : bounds.right;
+            PartitionPositionBounds pb = getPositionsForBounds(bounds);
+            if (pb != null)
+                positions.add(pb);
+        }
+        return positions;
+    }
 
-            if (leftBound.compareTo(last) > 0 || rightBound.compareTo(first) < 0)
-                continue;
+    /**
+     * Get a list of data positions in this SSTable that correspond to the given list of bounds. This method will remove
+     * non-covered intervals, but will not correct order or overlap in the supplied list, e.g. if bounds overlap, the
+     * result will be sections of the data file that repeat the same positions.
+     *
+     * @return A sorted list of [offset,end) pairs corresponding to the given boundsList in the datafile for this
+     *         SSTable.
+     */
+    public List<PartitionPositionBounds> getPositionsForBoundsIterator(Iterator<AbstractBounds<PartitionPosition>> boundsList)
+    {
+        // use the index to determine a minimal section for each range
+        List<PartitionPositionBounds> positions = new ArrayList<>();
+        while (boundsList.hasNext())
+        {
+            AbstractBounds<PartitionPosition> bounds = boundsList.next();
+            PartitionPositionBounds pb = getPositionsForBounds(bounds);
+            if (pb != null)
+                positions.add(pb);
+        }
+        return positions;
+    }
+
+    /**
+     * Determine the data positions in this SSTable that cover the given bounds.
+     *
+     * @return An [offset,end) pair that cover the given bounds in the datafile for this SSTable, or null if the range
+     *         is not covered by the sstable or is empty.
+     */
+    public PartitionPositionBounds getPositionsForBounds(AbstractBounds<PartitionPosition> bounds)
+    {
+        long rieLeft = getPosition(bounds.left, bounds.inclusiveLeft() ? Operator.GE : Operator.GT);
+        // Note: getPosition will apply a moved start if the sstable is in MOVED_START state.
+        if (rieLeft == -1) // empty range
+            return null;
+        long left = rieLeft;
+
+        long rieRight = bounds.right.isMinimum() ? -1
+                                                 : getPosition(bounds.right, bounds.inclusiveRight() ? Operator.GT
+                                                                                                     : Operator.GE);
+        long right;
+        if (rieRight != -1)
+            right = rieRight;
+        else // right is beyond end
+            right = uncompressedLength();   // this should also be correct for EARLY readers
+
+        if (left >= right)
+        {
+            // empty range
+            return null;
+        }
 
-            long left = getPosition(leftBound, Operator.GT);
-            long right = (rightBound.compareTo(last) > 0)
-                         ? uncompressedLength()
-                         : getPosition(rightBound, Operator.GT);
+        return new PartitionPositionBounds(left, right);
+    }
 
-            if (left == right)
-                // empty range
-                continue;
+    /**
+     * Return a [offset, end) pair that covers the whole file. This could be null if the sstable's moved start has
+     * made the sstable effectively empty.
+     */
+    public PartitionPositionBounds getPositionsForFullRange()
+    {
+        if (openReason != OpenReason.MOVED_START)
+            return new PartitionPositionBounds(0, uncompressedLength());
+        else
+        {
+            // query a full range, so that the required adjustments can be applied
+            PartitionPosition minToken = getPartitioner().getMinimumToken().minKeyBound();
+            return getPositionsForBounds(new Range<>(minToken, minToken));
+        }
+    }
 
-            assert left < right : String.format("Range=%s openReason=%s first=%s last=%s left=%d right=%d", range, openReason, first, last, left, right);
-            positions.add(new PartitionPositionBounds(left, right));
+    /**
+     * Calculate a total on-disk (compressed) size for the given partition positions. For uncompressed files this is
+     * equal to the sum of the size of the covered ranges. For compressed files this is the sum of the size of the
+     * chunks that contain the requested ranges and may be significantly bigger than the size of the requested ranges.
+     *
+     * @param positionBounds a list of [offset,end) pairs that specify the relevant sections of the data file; this must
+     *                       be non-overlapping and in ascending order.
+     */
+    public long onDiskSizeForPartitionPositions(Collection<PartitionPositionBounds> positionBounds)
+    {
+        long total = 0;
+        if (!compression)
+        {
+            for (PartitionPositionBounds position : positionBounds)
+                total += position.upperPosition - position.lowerPosition;
         }
-        return positions;
+        else
+        {
+            final CompressionMetadata compressionMetadata = getCompressionMetadata();
+            long lastEnd = 0;
+            for (PartitionPositionBounds position : positionBounds)
+            {
+                assert position.lowerPosition >= 0 : "the partition lower cannot be negative";
+                if (position.upperPosition == position.lowerPosition)
+                {
+                    continue;
+                }
+                assert position.upperPosition >= position.lowerPosition : "the partition upper position cannot be lower than lower position";
+
+                long upperChunkEnd = compressionMetadata.chunkFor(position.upperPosition - 1).chunkEnd();
+                long lowerChunkStart = compressionMetadata.chunkFor(position.lowerPosition).offset;
+                if (lowerChunkStart < lastEnd)  // if regions include the same chunk, count it only once
+                    lowerChunkStart = lastEnd;
+                total += upperChunkEnd - lowerChunkStart;
+                lastEnd = upperChunkEnd;
+            }
+        }
+        return total;
     }
 
     /**
@@ -784,7 +959,7 @@ public final long getPosition(PartitionPosition key,
      * @param op          The Operator defining matching keys: the nearest key to the target matching the operator wins.
      * @param updateStats true if updating stats and cache
      * @param listener    a listener used to handle internal events
-     * @return The index entry corresponding to the key, or null if the key is not present
+     * @return The index entry corresponding to the key, or -1 if the key is not present
      */
     protected long getPosition(PartitionPosition key,
                                Operator op,
@@ -838,6 +1013,7 @@ public KeyIterator keyIterator() throws IOException
      * Returns the length in bytes of the (uncompressed) data for this SSTable. For compressed files, this is not
      * the same thing as the on disk size (see {@link #onDiskLength()}).
      */
+    @Override
     public long uncompressedLength()
     {
         return dfile.dataLength();
@@ -860,11 +1036,23 @@ public double tokenSpaceCoverage()
      * The length in bytes of the on disk size for this SSTable. For compressed files, this is not the same thing
      * as the data length (see {@link #uncompressedLength()}).
      */
+    @Override
     public long onDiskLength()
     {
         return dfile.onDiskLength;
     }
 
+    public long onDiskComponentsSize()
+    {
+        long total = 0;
+        for (Component component : components())
+        {
+            total += FileUtils.size(descriptor.pathFor(component));
+        }
+
+        return total;
+    }
+
     @VisibleForTesting
     public double getCrcCheckChance()
     {
@@ -888,7 +1076,7 @@ public void setCrcCheckChance(double crcCheckChance)
      * <p>
      * Calling it multiple times is usually buggy.
      */
-    public void markObsolete(Runnable tidier)
+    public void markObsolete(AbstractLogTransaction.ReaderTidier tidier)
     {
         if (logger.isTraceEnabled())
             logger.trace("Marking {} compacted", getFilename());
@@ -903,11 +1091,24 @@ public void markObsolete(Runnable tidier)
         }
     }
 
+    @Override
     public boolean isMarkedCompacted()
     {
         return tidy.global.obsoletion != null;
     }
 
+    /**
+     * Used by CNDB to detect sstables marked as obsolete (compacted). Without obtaining the actual
+     * {@link SSTableReader} instance. See {@link #isMarkedCompacted()} that performs the same check for an
+     * exisiting reader instance.
+     * @see #isMarkedCompacted()
+     * @see #markObsolete(AbstractLogTransaction.ReaderTidier)
+     */
+    public static boolean isMarkedCompacted(Descriptor descriptor)
+    {
+        return GlobalTidy.hasTidier(descriptor);
+    }
+
     public void markSuspect()
     {
         if (logger.isTraceEnabled())
@@ -922,11 +1123,18 @@ public void unmarkSuspect()
         isSuspect.getAndSet(false);
     }
 
+    @Override
     public boolean isMarkedSuspect()
     {
         return isSuspect.get();
     }
 
+    @Override
+    public boolean isSuitableForCompaction()
+    {
+        return !isMarkedSuspect() && openReason != SSTableReader.OpenReason.EARLY;
+    }
+
     /**
      * Direct I/O SSTableScanner over a defined range of tokens.
      *
@@ -937,15 +1145,19 @@ public ISSTableScanner getScanner(Range<Token> range)
     {
         if (range == null)
             return getScanner();
-        return getScanner(Collections.singletonList(range));
+        else
+            return getScanner(Collections.singletonList(range));
     }
 
     /**
-     * Direct I/O SSTableScanner over the entirety of the sstable..
+     * Direct I/O SSTableScanner over the entirety of the sstable.
      *
      * @return A Scanner over the full content of the SSTable.
      */
-    public abstract ISSTableScanner getScanner();
+    public ISSTableScanner getScanner()
+    {
+        return new SSTableSimpleScanner(this, Collections.singletonList(getPositionsForFullRange()));
+    }
 
     /**
      * Direct I/O SSTableScanner over a defined collection of ranges of tokens.
@@ -953,15 +1165,24 @@ public ISSTableScanner getScanner(Range<Token> range)
      * @param ranges the range of keys to cover
      * @return A Scanner for seeking over the rows of the SSTable.
      */
-    public abstract ISSTableScanner getScanner(Collection<Range<Token>> ranges);
+    public ISSTableScanner getScanner(Collection<Range<Token>> ranges)
+    {
+        if (ranges != null)
+            return new SSTableSimpleScanner(this, getPositionsForRanges(ranges));
+        else
+            return getScanner();
+    }
 
     /**
      * Direct I/O SSTableScanner over an iterator of bounds.
      *
-     * @param rangeIterator the keys to cover
+     * @param boundsIterator the keys to cover
      * @return A Scanner for seeking over the rows of the SSTable.
      */
-    public abstract ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> rangeIterator);
+    public ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> boundsIterator)
+    {
+        return new SSTableSimpleScanner(this, getPositionsForBoundsIterator(boundsIterator));
+    }
 
     /**
      * Create a {@link FileDataInput} for the data file of the sstable represented by this reader. This method returns
@@ -993,7 +1214,7 @@ public void createLinks(String snapshotDirectoryPath)
 
     public void createLinks(String snapshotDirectoryPath, RateLimiter rateLimiter)
     {
-        createLinks(descriptor, components, snapshotDirectoryPath, rateLimiter);
+        createLinks(descriptor, components(), snapshotDirectoryPath, rateLimiter);
     }
 
     public static void createLinks(Descriptor descriptor, Set<Component> components, String snapshotDirectoryPath)
@@ -1005,7 +1226,8 @@ public static void createLinks(Descriptor descriptor, Set<Component> components,
     {
         for (Component component : components)
         {
-            File sourceFile = descriptor.fileFor(component);
+            // Convert a potential RemotePath to a local one since RemotePaths don't support hard links.
+            File sourceFile = new File(descriptor.fileFor(component).absolutePath());
             if (!sourceFile.exists())
                 continue;
             if (null != limiter)
@@ -1015,38 +1237,40 @@ public static void createLinks(Descriptor descriptor, Set<Component> components,
         }
     }
 
+    @Override
     public boolean isRepaired()
     {
         return sstableMetadata.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE;
     }
 
     /**
-     * Reads the key stored at the position saved in SASI.
-     * <p>
-     * When SASI is created, it uses key locations retrieved from {@link KeyReader#keyPositionForSecondaryIndex()}.
-     * This method is to read the key stored at such position. It is up to the concrete SSTable format implementation
-     * what that position means and which file it refers. The only requirement is that it is consistent with what
-     * {@link KeyReader#keyPositionForSecondaryIndex()} returns.
+     * Returns an instance of {@link IKeyFetcher} that can be used to fetch keys from this SSTable.
      *
-     * @return key if found, {@code null} otherwise
+     * @param isForSASI whether the key fetcher is for SASI index - SASI indexes may use a different source of keys
+     *                  depending on the SSTableFormat (for example index file vs data file). If false, the keys are
+     *                  fetched from the data file.
      */
-    public abstract DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException;
+    public abstract IKeyFetcher openKeyFetcher(boolean isForSASI);
 
+    @Override
     public boolean isPendingRepair()
     {
         return sstableMetadata.pendingRepair != ActiveRepairService.NO_PENDING_REPAIR;
     }
 
+    @Override
     public TimeUUID getPendingRepair()
     {
         return sstableMetadata.pendingRepair;
     }
 
+    @Override
     public long getRepairedAt()
     {
         return sstableMetadata.repairedAt;
     }
 
+    @Override
     public boolean isTransient()
     {
         return sstableMetadata.isTransient;
@@ -1075,6 +1299,7 @@ public abstract static class Operator
 
         final static class Equals extends Operator
         {
+            @Override
             public int apply(int comparison)
             {
                 return -comparison;
@@ -1083,6 +1308,7 @@ public int apply(int comparison)
 
         final static class GreaterThanOrEqualTo extends Operator
         {
+            @Override
             public int apply(int comparison)
             {
                 return comparison >= 0 ? 0 : 1;
@@ -1091,6 +1317,7 @@ public int apply(int comparison)
 
         final static class GreaterThan extends Operator
         {
+            @Override
             public int apply(int comparison)
             {
                 return comparison > 0 ? 0 : 1;
@@ -1108,6 +1335,7 @@ public EstimatedHistogram getEstimatedCellPerPartitionCount()
         return sstableMetadata.estimatedCellPerPartitionCount;
     }
 
+    @Override
     public double getEstimatedDroppableTombstoneRatio(long gcBefore)
     {
         return sstableMetadata.getEstimatedDroppableTombstoneRatio(gcBefore);
@@ -1123,21 +1351,25 @@ public double getCompressionRatio()
         return sstableMetadata.compressionRatio;
     }
 
+    @Override
     public long getMinTimestamp()
     {
         return sstableMetadata.minTimestamp;
     }
 
+    @Override
     public long getMaxTimestamp()
     {
         return sstableMetadata.maxTimestamp;
     }
 
+    @Override
     public long getMinLocalDeletionTime()
     {
         return sstableMetadata.minLocalDeletionTime;
     }
 
+    @Override
     public long getMaxLocalDeletionTime()
     {
         return sstableMetadata.maxLocalDeletionTime;
@@ -1150,6 +1382,7 @@ public long getMaxLocalDeletionTime()
      * cell tombstone, no range tombstone maker and no expiring columns), but having it return {@code true} doesn't
      * guarantee it contains any as it may simply have non-expired cells.
      */
+    @Override
     public boolean mayHaveTombstones()
     {
         // A sstable is guaranteed to have no tombstones if minLocalDeletionTime is still set to its default,
@@ -1184,6 +1417,7 @@ public int getAvgColumnSetPerRow()
                : (sstableMetadata.totalRows == 0 ? 0 : (int) (sstableMetadata.totalColumnsSet / sstableMetadata.totalRows));
     }
 
+    @Override
     public int getSSTableLevel()
     {
         return sstableMetadata.sstableLevel;
@@ -1192,12 +1426,21 @@ public int getSSTableLevel()
     /**
      * Mutate sstable level with a lock to avoid racing with entire-sstable-streaming and then reload sstable metadata
      */
+    @Override
     public void mutateLevelAndReload(int newLevel) throws IOException
     {
-        synchronized (tidy.global)
+        try
         {
-            descriptor.getMetadataSerializer().mutateLevel(descriptor, newLevel);
-            reloadSSTableMetadata();
+            synchronized (tidy.global)
+            {
+                descriptor.getMetadataSerializer().mutateLevel(descriptor, newLevel);
+                reloadSSTableMetadata();
+            }
+        }
+        catch (IOException e)
+        {
+            markSuspect();
+            throw e;
         }
     }
 
@@ -1246,7 +1489,7 @@ public RandomAccessReader openDataReader()
     public void trySkipFileCacheBefore(DecoratedKey key)
     {
         long position = getPosition(key, SSTableReader.Operator.GE);
-        NativeLibrary.trySkipCache(descriptor.fileFor(Components.DATA).absolutePath(), 0, position < 0 ? 0 : position);
+        INativeLibrary.instance.trySkipCache(getDataFile(), 0, position < 0 ? 0 : position);
     }
 
     public ChannelProxy getDataChannel()
@@ -1272,6 +1515,14 @@ public void incrementReadCount()
             readMeter.mark();
     }
 
+    /**
+     * Increment the total read count and read rate for accessing partition index.
+     */
+    public void incrementIndexReadCount()
+    {
+        partitionIndexReadMeter.ifPresent(RestorableMeter::mark);
+    }
+
     public EncodingStats stats()
     {
         // We could return sstable.header.stats(), but this may not be as accurate than the actual sstable stats (see
@@ -1279,16 +1530,19 @@ public EncodingStats stats()
         return sstableMetadata.encodingStats;
     }
 
+    @Override
     public Ref<SSTableReader> tryRef()
     {
         return selfRef.tryRef();
     }
 
+    @Override
     public Ref<SSTableReader> selfRef()
     {
         return selfRef;
     }
 
+    @Override
     public Ref<SSTableReader> ref()
     {
         return selfRef.ref();
@@ -1324,10 +1578,13 @@ public void addTo(Ref.IdentityCollection identities)
     }
 
     /**
-     * The method verifies whether the sstable may contain the provided key. The method does approximation using
-     * Bloom filter if it is present and if it is not, performs accurate check in the index.
+     * @return true if global reference exists for the physical sstable corresponding to the provided descriptor.
      */
-    public abstract boolean mayContainAssumingKeyIsInRange(DecoratedKey key);
+    @VisibleForTesting
+    public static boolean hasGlobalReference(Descriptor descriptor)
+    {
+        return GlobalTidy.exists(descriptor);
+    }
 
     /**
      * One instance per SSTableReader we create.
@@ -1343,7 +1600,7 @@ protected static final class InstanceTidier implements Tidy
         private final Descriptor descriptor;
         private final WeakReference<Owner> owner;
 
-        private List<? extends AutoCloseable> closeables;
+        private List<AutoCloseable> closeables;
         private Runnable runOnClose;
 
         private boolean isReplaced = false;
@@ -1358,7 +1615,7 @@ protected static final class InstanceTidier implements Tidy
         public void setup(SSTableReader reader, boolean trackHotness, Collection<? extends AutoCloseable> closeables)
         {
             // get a new reference to the shared descriptor-type tidy
-            this.globalRef = GlobalTidy.get(reader);
+            this.globalRef = GlobalTidy.get(reader.getDescriptor());
             this.global = globalRef.get();
             if (trackHotness)
                 global.ensureReadMeter();
@@ -1373,6 +1630,12 @@ private InstanceTidier(Descriptor descriptor, Owner owner)
             this.owner = new WeakReference<>(owner);
         }
 
+        public void addCloseable(AutoCloseable closeable)
+        {
+            if (closeable != null)
+                closeables.add(closeable); // Last added is first to be closed.
+        }
+
         @Override
         public void tidy()
         {
@@ -1395,66 +1658,75 @@ public void tidy()
                 barrier = null;
             }
 
-            ScheduledExecutors.nonPeriodicTasks.execute(new Runnable()
-            {
-                public void run()
-                {
-                    if (logger.isTraceEnabled())
-                        logger.trace("Async instance tidier for {}, before barrier", descriptor);
+            Runnable cleanup = new CleanupTask(barrier);
+            ScheduledExecutors.nonPeriodicTasks.execute(cleanup);
+        }
 
-                    if (barrier != null)
-                        barrier.await();
+        public String name()
+        {
+            return descriptor.toString();
+        }
 
-                    if (logger.isTraceEnabled())
-                        logger.trace("Async instance tidier for {}, after barrier", descriptor);
+        private class CleanupTask implements Runnable
+        {
+            private final OpOrder.Barrier barrier;
 
-                    Throwable exceptions = null;
-                    if (runOnClose != null) try
-                    {
-                        runOnClose.run();
-                    }
-                    catch (RuntimeException | Error ex)
-                    {
-                        logger.error("Failed to run on-close listeners for sstable " + descriptor.baseFile(), ex);
-                        exceptions = ex;
-                    }
+            public CleanupTask(OpOrder.Barrier barrier) {
+                this.barrier = barrier;
+            }
 
-                    Throwable closeExceptions = Throwables.close(null, Iterables.filter(closeables, Objects::nonNull));
-                    if (closeExceptions != null)
-                    {
-                        logger.error("Failed to close some sstable components of " + descriptor.baseFile(), closeExceptions);
-                        exceptions = Throwables.merge(exceptions, closeExceptions);
-                    }
+            @Override
+            public void run()
+            {
+                if (logger.isTraceEnabled())
+                    logger.trace("Async instance tidier for {}, before barrier", descriptor);
 
-                    try
-                    {
-                        globalRef.release();
-                    }
-                    catch (RuntimeException | Error ex)
-                    {
-                        logger.error("Failed to release the global ref of " + descriptor.baseFile(), ex);
-                        exceptions = Throwables.merge(exceptions, ex);
-                    }
+                if (barrier != null)
+                    barrier.await();
 
-                    if (exceptions != null)
-                        JVMStabilityInspector.inspectThrowable(exceptions);
+                if (logger.isTraceEnabled())
+                    logger.trace("Async instance tidier for {}, after barrier", descriptor);
 
-                    if (logger.isTraceEnabled())
-                        logger.trace("Async instance tidier for {}, completed", descriptor);
+                Throwable exceptions = null;
+                if (runOnClose != null) try
+                {
+                    runOnClose.run();
+                }
+                catch (RuntimeException | Error ex)
+                {
+                    logger.error("Failed to run on-close listeners for sstable " + descriptor.baseFile(), ex);
+                    exceptions = ex;
                 }
 
-                @Override
-                public String toString()
+                Throwable closeExceptions = Throwables.close(null, Iterables.filter(closeables, Objects::nonNull));
+                if (closeExceptions != null)
                 {
-                    return "Tidy " + descriptor.ksname + '.' + descriptor.cfname + '-' + descriptor.id;
+                    logger.error("Failed to close some sstable components of " + descriptor.baseFile(), closeExceptions);
+                    exceptions = Throwables.merge(exceptions, closeExceptions);
                 }
-            });
-        }
 
-        @Override
-        public String name()
-        {
-            return descriptor.toString();
+                try
+                {
+                    globalRef.release();
+                }
+                catch (RuntimeException | Error ex)
+                {
+                    logger.error("Failed to release the global ref of " + descriptor.baseFile(), ex);
+                    exceptions = Throwables.merge(exceptions, ex);
+                }
+
+                if (exceptions != null)
+                    JVMStabilityInspector.inspectThrowable(exceptions);
+
+                if (logger.isTraceEnabled())
+                    logger.trace("Async instance tidier for {}, completed", descriptor);
+            }
+
+            @Override
+            public String toString()
+            {
+                return "Tidy " + descriptor.ksname + '.' + descriptor.cfname + '-' + descriptor.id;
+            }
         }
     }
 
@@ -1466,7 +1738,7 @@ public String name()
      * and stash a reference to it to be released when they are. Once all such references are
      * released, this shared tidy will be performed.
      */
-    static final class GlobalTidy implements Tidy
+    public static final class GlobalTidy implements RefCounted.Tidy
     {
         static final WeakReference<ScheduledFuture<?>> NULL = new WeakReference<>(null);
         // keyed by descriptor, mapping to the shared GlobalTidy for that descriptor
@@ -1480,11 +1752,11 @@ static final class GlobalTidy implements Tidy
         // sstable have been released
         private WeakReference<ScheduledFuture<?>> readMeterSyncFuture = NULL;
         // shared state managing if the logical sstable has been compacted; this is used in cleanup
-        private volatile Runnable obsoletion;
+        private volatile AbstractLogTransaction.ReaderTidier obsoletion;
 
-        GlobalTidy(final SSTableReader reader)
+        GlobalTidy(Descriptor descriptor)
         {
-            this.desc = reader.descriptor;
+            this.desc = descriptor;
         }
 
         void ensureReadMeter()
@@ -1502,6 +1774,13 @@ void ensureReadMeter()
                 return;
             }
 
+            if (!DatabaseDescriptor.supportsSSTableReadMeter())
+            {
+                readMeter = RestorableMeter.createWithDefaultRates();
+                readMeterSyncFuture = NULL;
+                return;
+            }
+
             readMeter = SystemKeyspace.getSSTableReadMeter(desc.ksname, desc.cfname, desc.id);
             // sync the average read rate to system.sstable_activity every five minutes, starting one minute from now
             readMeterSyncFuture = new WeakReference<>(syncExecutor.scheduleAtFixedRate(this::maybePersistSSTableReadMeter, 1, 5, TimeUnit.MINUTES));
@@ -1526,48 +1805,142 @@ private void stopReadMeterPersistence()
             }
         }
 
+        /**
+         * Used by CNDB RepairRemoteStorageHandler to abort existing tidier before reloading sstable with orphan reference
+         *
+         * @return sstable reader tidier if exists
+         */
+        @Nullable
+        public AbstractLogTransaction.ReaderTidier getTidier()
+        {
+            return obsoletion;
+        }
+
+        /**
+         * Used by CNDB RepairRemoteStorageHandler to reset reader tidier before reloading sstable with orphan reference
+         * @param tidier new reader tidier for the global tidy. could be null
+         */
+        public void setTidier(@Nullable AbstractLogTransaction.ReaderTidier tidier)
+        {
+            this.obsoletion = tidier;
+        }
+
         public void tidy()
         {
-            lookup.remove(desc);
+            // Before proceeding with lookup.remove(desc) and with the tidier,
+            // make sure this instance is actually the one stored in the lookup.
+            // If there is no instance stored, or if the referent is not this
+            // instance, then this GlobalTidy instance was created in GlobalTidy.get()
+            // because of a race, and should not remove the real tidy from the lookup,
+            // or perform any cleanup
+            Ref<GlobalTidy> existing = lookup.get(desc);
+            if (existing == null || !existing.refers(this))
+            {
+                return;
+            }
 
-            if (obsoletion != null)
-                obsoletion.run();
+            try
+            {
+                // don't ideally want to dropPageCache for the file until all instances have been released
+                StorageProvider.instance.invalidateFileSystemCache(desc, obsoletion != null);
 
-            // don't ideally want to dropPageCache for the file until all instances have been released
-            for (Component c : desc.discoverComponents())
-                NativeLibrary.trySkipCache(desc.fileFor(c).absolutePath(), 0, 0);
+                if (obsoletion != null)
+                    obsoletion.commit();
+            }
+            finally
+            {
+                // remove reference after deleting local files, to avoid racing with {@link GlobalTidy#exists}
+                boolean removed = lookup.remove(desc, existing);
+                if (!removed)
+                {
+                    throw new IllegalStateException("the reference changed behind our back? existing: " + existing + ", in lookup: " + lookup.get(desc));
+                }
+            }
         }
 
+        @Override
         public String name()
         {
             return desc.toString();
         }
 
         // get a new reference to the shared GlobalTidy for this sstable
-        public static Ref<GlobalTidy> get(SSTableReader sstable)
+        public static Ref<GlobalTidy> get(Descriptor descriptor)
         {
-            Descriptor descriptor = sstable.descriptor;
-
-            while (true)
+            for (Ref<GlobalTidy> globallySharedTidy = null;;)
             {
-                Ref<GlobalTidy> ref = lookup.get(descriptor);
-                if (ref == null)
+                if (globallySharedTidy == null)
+                {
+                    globallySharedTidy = lookup.get(descriptor);
+                }
+                if (globallySharedTidy != null)
                 {
-                    final GlobalTidy tidy = new GlobalTidy(sstable);
-                    ref = new Ref<>(tidy, tidy);
-                    Ref<GlobalTidy> ex = lookup.putIfAbsent(descriptor, ref);
-                    if (ex == null)
-                        return ref;
-                    ref = ex;
+                    // there's a potentialy alive ref in our lookup table;
+                    // try to bump the counter
+                    Ref<GlobalTidy> newRef = globallySharedTidy.tryRef();
+                    if (newRef != null)
+                    {
+                        // the Ref was alive, bumping ref count succeeded.
+                        // we're ok to return the newRef
+                        return newRef;
+                    }
+                    else
+                    {
+                        // bumping ref count failed => ref count dropped to zero; tidy is in progress
+                        // globallySharedTidy is a dead reference
+                        // active waiting for tidy to complete and remove
+                        // the old entry from the lookup table;
+                        globallySharedTidy = null;
+                        Thread.yield();
+                    }
+                }
+                else
+                {
+                    // there is no entry in the lookup table for this sstable
+                    // let's create one and memoize it (if we're lucky)
+
+                    final GlobalTidy tidy = new GlobalTidy(descriptor);
+                    Ref<GlobalTidy> newRef = new Ref<>(tidy, tidy);
+                    globallySharedTidy = lookup.putIfAbsent(descriptor, newRef);
+                    if (globallySharedTidy != null)
+                    {
+                        // we raced with another put; tough luck, lets try again
+                        // we've got to clean up the just-created Ref
+                        // it's OK to close this Ref because GlobalTidy.tidy() is a no-op if
+                        // the ref in lookup is different
+                        newRef.close();
+                    }
+                    else
+                    {
+                        // put succeeded; returning reference
+                        return newRef;
+                    }
                 }
+            }
+        }
 
-                Ref<GlobalTidy> newRef = ref.tryRef();
-                if (newRef != null)
-                    return newRef;
+        public static boolean exists(Descriptor descriptor)
+        {
+            return lookup.containsKey(descriptor);
+        }
 
-                // raced with tidy
-                lookup.remove(descriptor, ref);
+        private static boolean hasTidier(Descriptor descriptor)
+        {
+            Ref<GlobalTidy> globalTidyRef = lookup.get(descriptor);
+            if (globalTidyRef != null)
+            {
+                try
+                {
+                    GlobalTidy globalTidy = globalTidyRef.get();
+                    if (globalTidy != null)
+                        return globalTidy.obsoletion != null;
+                }
+                catch (AssertionError e)
+                {
+                    // ignore, we're just checking if the tidier exists
+                }
             }
+            return false;
         }
     }
 
@@ -1717,7 +2090,7 @@ public long logicalBytesOnDisk()
     private long bytesOnDisk(boolean logical)
     {
         long bytes = 0;
-        for (Component component : components)
+        for (Component component : components())
         {
             // Only the data file is compressable.
             bytes += logical && component == Components.DATA && compression
@@ -1734,7 +2107,7 @@ public void maybePersistSSTableReadMeter()
     }
 
     /**
-     * Returns a new verifier for this sstable. Note that the reader must match the provided cfs.
+     * Returns a new verifier for this sstable. Note that the reader must match the provided cfs unless cfs is null.
      */
     public abstract IVerifier getVerifier(ColumnFamilyStore cfs,
                                           OutputHandler outputHandler,
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java
index aedd860922e1..9cff1c286408 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java
@@ -33,6 +33,7 @@
 import org.apache.cassandra.io.sstable.IOOptions;
 import org.apache.cassandra.io.sstable.KeyReader;
 import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableWatcher;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
 import org.apache.cassandra.metrics.TableMetrics;
@@ -60,7 +61,14 @@ public abstract class SSTableReaderLoadingBuilder<R extends SSTableReader, B ext
     public SSTableReaderLoadingBuilder(SSTable.Builder<?, ?> builder)
     {
         this.descriptor = builder.descriptor;
-        this.components = builder.getComponents() != null ? ImmutableSet.copyOf(builder.getComponents()) : TOCComponent.loadOrCreate(this.descriptor);
+
+        Set<Component> ssTableComponents;
+        if (builder.getComponents() != null)
+            ssTableComponents = ImmutableSet.copyOf(builder.getComponents());
+        else
+            ssTableComponents = TOCComponent.loadOrCreate(this.descriptor);
+        this.components = SSTableWatcher.instance.discoverComponents(descriptor, ssTableComponents);
+
         this.tableMetadataRef = builder.getTableMetadataRef() != null ? builder.getTableMetadataRef() : resolveTableMetadataRef();
         this.ioOptions = builder.getIOOptions() != null ? builder.getIOOptions() : IOOptions.fromDatabaseDescriptor();
         this.chunkCache = builder.getChunkCache() != null ? builder.getChunkCache() : ChunkCache.instance;
@@ -71,9 +79,10 @@ public SSTableReaderLoadingBuilder(SSTable.Builder<?, ?> builder)
 
     public R build(SSTable.Owner owner, boolean validate, boolean online)
     {
+
         checkArgument(components.contains(Components.DATA), "Data component is missing for sstable %s", descriptor);
         if (validate)
-            checkArgument(this.components.containsAll(descriptor.getFormat().primaryComponents()), "Some required components (%s) are missing for sstable %s", Sets.difference(descriptor.getFormat().primaryComponents(), this.components), descriptor);
+            checkArgument(components.containsAll(descriptor.getFormat().primaryComponents()), "Some required components (%s) are missing for sstable %s", Sets.difference(descriptor.getFormat().primaryComponents(), this.components), descriptor);
 
         B builder = (B) descriptor.getFormat().getReaderFactory().builder(descriptor);
         builder.setOpenReason(NORMAL);
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java
index 5aac1d622c30..f46d5a2807b2 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java
@@ -18,31 +18,45 @@
 
 package org.apache.cassandra.io.sstable.format;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
+import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
 import org.apache.cassandra.io.sstable.filter.BloomFilterTracker;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.FilterFactory;
 import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.concurrent.Ref;
 
 import static org.apache.cassandra.utils.concurrent.SharedCloseable.sharedCopyOrNull;
 
 public abstract class SSTableReaderWithFilter extends SSTableReader
 {
-    private final IFilter filter;
+    protected volatile IFilter filter;
+    private final AtomicBoolean bfDeserializationStarted = new AtomicBoolean(false);
+    private final boolean bloomFilterLazyLoading = BloomFilter.lazyLoading();
+    private final int bloomFilterLazyLoadingWindow = BloomFilter.lazyLoadingWindow();
+    private final long bloomFilterLazyLoadingThreshold = BloomFilter.lazyLoadingThreshold();
+
     private final BloomFilterTracker filterTracker;
 
     protected SSTableReaderWithFilter(Builder<?, ?> builder, Owner owner)
     {
         super(builder, owner);
         this.filter = Objects.requireNonNull(builder.getFilter());
-        this.filterTracker = new BloomFilterTracker();
+        this.filterTracker = owner().map(Owner::getBloomFilterTracker)
+                                    .orElseGet(BloomFilterTracker::createNoopTracker);
     }
 
     @Override
@@ -69,11 +83,95 @@ protected boolean isPresentInFilter(IFilter.FilterKey key)
     @Override
     public boolean mayContainAssumingKeyIsInRange(DecoratedKey key)
     {
+        maybeDeserializeLazyBloomFilter();
         // if we don't have bloom filter(bf_fp_chance=1.0 or filter file is missing),
         // we check index file instead.
         return !filter.isInformative() && getPosition(key, Operator.EQ, false) >= 0 || filter.isPresent(key);
     }
 
+    protected boolean inBloomFilter(DecoratedKey dk)
+    {
+        maybeDeserializeLazyBloomFilter();
+        return filter.isPresent(dk);
+    }
+
+    /**
+     * Defer BF deserialization when enabled to reduce memory pressure in use case where many sstables are not accessed frequently
+     *
+     * @return true if BF deserialization is attempted; false otherwise.
+     */
+    @VisibleForTesting
+    boolean maybeDeserializeLazyBloomFilter()
+    {
+        if (!bloomFilterLazyLoading || filter != FilterFactory.AlwaysPresentForLazyLoading)
+            return false;
+
+        Preconditions.checkState(partitionIndexReadMeter.isPresent(), "Read index meter should have been available");
+
+        boolean loadBloomFilter = false;
+
+        // If the threshold was set to zero we always want to deserialize
+        if (bloomFilterLazyLoadingThreshold == 0)
+            loadBloomFilter = true;
+            // otherwise, if window is <= 0 we use the threshold as an absolute count
+        else if (bloomFilterLazyLoadingWindow <= 0 && partitionIndexReadMeter.get().count() >= bloomFilterLazyLoadingThreshold)
+            loadBloomFilter = true;
+            // otherwise we look at the count in the specified window
+        else if (bloomFilterLazyLoadingWindow > 0 && partitionIndexReadMeter.get().rate(bloomFilterLazyLoadingWindow) >= bloomFilterLazyLoadingThreshold)
+            loadBloomFilter = true;
+
+        if (!loadBloomFilter)
+            return false;
+
+        // concurrent reads should only trigger async bloom filter deserialization once
+        if (!bfDeserializationStarted.compareAndSet(false, true))
+            return false;
+
+        Stage.IO.execute(() ->
+                         {
+                             logger.debug("Deserializing lazy bloom filter for {}", descriptor.baseFileURI());
+
+                             // hold sstable reference to prevent sstable being released before bloom filter deserialization completes
+                             Ref<SSTableReader> ref = tryRef();
+                             if (ref == null)
+                             {
+                                 logger.error("Unable to reference sstable {}, will use pass-through bloom filter", descriptor.baseFileUri());
+                                 filter = FilterFactory.AlwaysPresent;
+                             }
+                             else
+                             {
+                                 try
+                                 {
+                                     // the only recoverable BF deserialization error is remote storage timeout; but it should be
+                                     // fine to continue with pass-through filter and wait for compaction to replace current sstable.
+                                     IFilter loaded = FilterComponent.load(descriptor);
+                                     if (loaded == null)
+                                     {
+                                         filter = FilterFactory.AlwaysPresent;
+                                         logger.error("Failed to deserialize lazy bloom filter, will use pass-through bloom filter");
+                                     }
+                                     else
+                                     {
+                                         logger.debug("Successfuly loaded lazy bloom filter for {}", descriptor.baseFileURI());
+
+                                         filter = loaded;
+                                         tidy.addCloseable(loaded); // close newly created bloom filter on sstable close
+                                     }
+                                 }
+                                 catch (IOException e)
+                                 {
+                                     logger.info("Bloom filter for " + descriptor + " could not be deserialized", e);
+                                 }
+                                 finally
+                                 {
+                                     ref.release();
+                                 }
+                             }
+                         });
+
+        return true;
+    }
+
     @Override
     protected void notifySelected(SSTableReadsListener.SelectionReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats, AbstractRowIndexEntry entry)
     {
@@ -122,6 +220,12 @@ public long getFilterOffHeapSize()
     {
         return filter.offHeapSize();
     }
+    
+    @VisibleForTesting
+    public IFilter getFilter()
+    {
+        return filter;
+    }
 
     public abstract SSTableReaderWithFilter cloneAndReplace(IFilter filter);
 
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
index 217c17720639..b0a65ff96002 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
@@ -19,7 +19,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
@@ -35,13 +34,12 @@
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.AbstractBounds.Boundary;
-import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.AbstractIterator;
@@ -78,20 +76,22 @@ protected SSTableScanner(S sstable,
     {
         assert sstable != null;
 
-        this.dfile = sstable.openDataReader();
-        this.sstable = sstable;
-        this.columns = columns;
-        this.dataRange = dataRange;
-        this.rangeIterator = rangeIterator;
-        this.listener = listener;
-    }
-
-    protected static List<AbstractBounds<PartitionPosition>> makeBounds(SSTableReader sstable, Collection<Range<Token>> tokenRanges)
-    {
-        List<AbstractBounds<PartitionPosition>> boundsList = new ArrayList<>(tokenRanges.size());
-        for (Range<Token> range : Range.normalize(tokenRanges))
-            addRange(sstable, Range.makeRowRange(range), boundsList);
-        return boundsList;
+        RandomAccessReader dfile = null;
+        try
+        {
+            dfile = sstable.openDataReader();
+            this.sstable = sstable;
+            this.columns = columns;
+            this.dataRange = dataRange;
+            this.rangeIterator = rangeIterator;
+            this.listener = listener;
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(dfile);
+            throw t;
+        }
+        this.dfile = dfile;
     }
 
     protected static List<AbstractBounds<PartitionPosition>> makeBounds(SSTableReader sstable, DataRange dataRange)
@@ -101,11 +101,6 @@ protected static List<AbstractBounds<PartitionPosition>> makeBounds(SSTableReade
         return boundsList;
     }
 
-    protected static AbstractBounds<PartitionPosition> fullRange(SSTableReader sstable)
-    {
-        return new Bounds<>(sstable.getFirst(), sstable.getLast());
-    }
-
     private static void addRange(SSTableReader sstable, AbstractBounds<PartitionPosition> requested, List<AbstractBounds<PartitionPosition>> boundsList)
     {
         if (requested instanceof Range && ((Range<?>) requested).isWrapAround())
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java
new file mode 100644
index 000000000000..8f5bcf316d57
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.io.sstable.format.SSTableReader.PartitionPositionBounds;
+
+/// Simple SSTable scanner that reads sequentially through an SSTable without using the index.
+///
+/// This is a significant improvement for the performance of compaction over using the full-blown DataRange-capable
+/// SSTable scanners and enables correct calculation of data sizes to process.
+public class SSTableSimpleScanner
+implements ISSTableScanner
+{
+    private final AtomicBoolean isClosed = new AtomicBoolean(false);
+    private final RandomAccessReader dfile;
+    private final SSTableReader sstable;
+
+    private final Iterator<PartitionPositionBounds> rangeIterator;
+
+    private long bytesScannedInPreviousRanges;
+
+    private final long sizeInBytes;
+    private final long compressedSizeInBytes;
+
+    private long currentEndPosition;
+    private long currentStartPosition;
+
+    private SSTableIdentityIterator currentIterator;
+    private DecoratedKey lastKey;
+
+    /// Create a new simple scanner over the given sstables and the given ranges of uncompressed positions.
+    /// Each range must start and end on a partition boundary, and, to satisfy the contract of [ISSTableScanner], the
+    /// ranges must be non-overlapping and in ascending order. This scanner will throw an [IllegalArgumentException] if
+    /// the latter is not true.
+    ///
+    /// The ranges can be constructed by [SSTableReader#getPositionsForRanges] and similar methods as done by the
+    /// various [SSTableReader#getScanner] variations.
+    public SSTableSimpleScanner(SSTableReader sstable,
+                                Collection<PartitionPositionBounds> boundsList)
+    {
+        assert sstable != null;
+
+        this.dfile = sstable.openDataReader();
+        this.sstable = sstable;
+        this.sizeInBytes = boundsList.stream().mapToLong(ppb -> ppb.upperPosition - ppb.lowerPosition).sum();
+        this.compressedSizeInBytes = sstable.compression ? sstable.onDiskSizeForPartitionPositions(boundsList) : sizeInBytes;
+        this.rangeIterator = boundsList.iterator();
+        this.currentEndPosition = 0;
+        this.currentStartPosition = 0;
+        this.bytesScannedInPreviousRanges = 0;
+        this.currentIterator = null;
+        this.lastKey = null;
+    }
+
+    public void close()
+    {
+        if (isClosed.compareAndSet(false, true))
+        {
+            // ensure we report what we have actually processed
+            bytesScannedInPreviousRanges += dfile.getFilePointer() - currentStartPosition;
+            dfile.close();
+            // close() may change the file pointer, update so that the difference is 0 when reported by getBytesScanned()
+            currentStartPosition = dfile.getFilePointer();
+        }
+    }
+
+    @Override
+    public long getLengthInBytes()
+    {
+        return sizeInBytes;
+    }
+
+
+    public long getCompressedLengthInBytes()
+    {
+        return compressedSizeInBytes;
+    }
+
+    @Override
+    public long getCurrentPosition()
+    {
+        return dfile.getFilePointer();
+    }
+
+    public long getBytesScanned()
+    {
+        return bytesScannedInPreviousRanges + dfile.getFilePointer() - currentStartPosition;
+    }
+
+    @Override
+    public Set<SSTableReader> getBackingSSTables()
+    {
+        return ImmutableSet.of(sstable);
+    }
+
+    @Override
+    public int level()
+    {
+        return sstable.getSSTableLevel();
+    }
+
+    public TableMetadata metadata()
+    {
+        return sstable.metadata();
+    }
+
+    public boolean hasNext()
+    {
+        if (currentIterator != null)
+        {
+            currentIterator.close(); // Ensure that the iterator cannot be used further. No op if already closed.
+
+            // Row iterator must be exhausted to advance to next partition
+            currentIterator.exhaust();
+            currentIterator = null;
+        }
+
+        if (dfile.getFilePointer() < currentEndPosition)
+            return true;
+
+        return advanceRange();
+    }
+
+    boolean advanceRange()
+    {
+        if (!rangeIterator.hasNext())
+            return false;
+
+        bytesScannedInPreviousRanges += currentEndPosition - currentStartPosition;
+
+        PartitionPositionBounds nextRange = rangeIterator.next();
+        if (currentEndPosition > nextRange.lowerPosition)
+            throw new IllegalArgumentException("Ranges supplied to SSTableSimpleScanner must be non-overlapping and in ascending order.");
+
+        currentEndPosition = nextRange.upperPosition;
+        currentStartPosition = nextRange.lowerPosition;
+        dfile.seek(currentStartPosition);
+        return true;
+    }
+
+    public UnfilteredRowIterator next()
+    {
+        if (!hasNext())
+            throw new NoSuchElementException();
+
+        currentIterator = SSTableIdentityIterator.create(sstable, dfile, false);
+        DecoratedKey currentKey = currentIterator.partitionKey();
+        if (lastKey != null && lastKey.compareTo(currentKey) >= 0)
+        {
+            sstable.markSuspect();
+            throw new CorruptSSTableException(new IllegalStateException(String.format("Invalid key order: current %s <= previous %s",
+                                                                                      currentKey,
+                                                                                      lastKey)),
+                                              sstable.getFilename());
+        }
+        lastKey = currentKey;
+        return currentIterator;
+    }
+
+    public void remove()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("%s(sstable=%s)", getClass().getSimpleName(), sstable);
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
index 38efd1955e34..5c2ef16a9d10 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -37,6 +37,7 @@
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.writers.SSTableDataSink;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
@@ -66,7 +67,7 @@
  * {@link Builder}, a {@link LifecycleNewTracker} and {@link SSTable.Owner} instances. Implementing classes should
  * not extend that list and all the additional properties should be included in the builder.
  */
-public abstract class SSTableWriter extends SSTable implements Transactional
+public abstract class SSTableWriter extends SSTable implements Transactional, SSTableDataSink
 {
     private final static Logger logger = LoggerFactory.getLogger(SSTableWriter.class);
 
@@ -111,7 +112,7 @@ protected SSTableWriter(Builder<?, ?> builder, LifecycleNewTracker lifecycleNewT
         // sstable files were created before the sstable is registered in the lifecycle transaction, which may lead
         // to a race such that the sstable is listed as completed due to the lack of the transaction file before
         // anything is actually written to it.
-        Set<Component> existingComponents = Sets.filter(components, c -> descriptor.fileFor(c).exists());
+        Set<Component> existingComponents = Sets.filter(components(), c -> descriptor.fileFor(c).exists());
         assert existingComponents.isEmpty() : String.format("Cannot create a new SSTable in directory %s as component files %s already exist there",
                                                             descriptor.directory,
                                                             existingComponents);
@@ -124,7 +125,7 @@ protected SSTableWriter(Builder<?, ?> builder, LifecycleNewTracker lifecycleNewT
             this.observers = Collections.unmodifiableList(observers);
             for (Index.Group group : builder.getIndexGroups())
             {
-                SSTableFlushObserver observer = group.getFlushObserver(descriptor, lifecycleNewTracker, metadata.getLocal());
+                SSTableFlushObserver observer = group.getFlushObserver(descriptor, lifecycleNewTracker, metadata.getLocal(), keyCount);
                 if (observer != null)
                 {
                     observer.begin();
@@ -155,7 +156,7 @@ protected void handleConstructionFailure(Throwable ex)
         logger.warn("Failed to open " + descriptor + " for writing", ex);
         for (int i = observers.size()-1; i >= 0; i--)
             observers.get(i).abort(ex);
-        descriptor.getFormat().deleteOrphanedComponents(descriptor, components);
+        descriptor.getFormat().deleteOrphanedComponents(descriptor, components());
         lifecycleNewTracker.untrackNew(this);
     }
 
@@ -233,9 +234,9 @@ public SSTableWriter setTokenSpaceCoverage(double rangeSpanned)
         return this;
     }
 
-    public void setOpenResult(boolean openResult)
+    public void openResult()
     {
-        txnProxy.openResult = openResult;
+        txnProxy.openResult();
     }
 
     /**
@@ -260,9 +261,10 @@ public void setOpenResult(boolean openResult)
 
     public SSTableReader finish(boolean openResult)
     {
-        this.setOpenResult(openResult);
-        observers.forEach(SSTableFlushObserver::complete);
-        txnProxy.finish();
+        prepareToCommit();
+        if (openResult)
+            openResult();
+        txnProxy.commit();
         return finished();
     }
 
@@ -278,22 +280,23 @@ public SSTableReader finished()
 
     // finalise our state on disk, including renaming
     public final void prepareToCommit()
-    {
-        txnProxy.prepareToCommit();
-    }
-
-    public final Throwable commit(Throwable accumulate)
     {
         try
         {
-            observers.forEach(SSTableFlushObserver::complete);
+            txnProxy.prepareToCommit();
         }
-        catch (Throwable t)
+        finally
         {
-            // Return without advancing to COMMITTED, which will trigger abort() when the Transactional closes...
-            return Throwables.merge(accumulate, t);
-        }
+            // need to generate all index files before commit, so they will be included in txn log
+            observers.forEach(obs -> obs.complete(this));
 
+            // track newly written sstable after index files are written
+            lifecycleNewTracker.trackNewWritten(this);
+         }
+    }
+
+    public final Throwable commit(Throwable accumulate)
+    {
         return txnProxy.commit(accumulate);
     }
 
@@ -366,7 +369,6 @@ protected class TransactionalProxy extends AbstractTransactional
         private final Supplier<ImmutableList<Transactional>> transactionals;
 
         private SSTableReader finalReader;
-        private boolean openResult;
         private boolean finalReaderAccessed;
 
         public TransactionalProxy(Supplier<ImmutableList<Transactional>> transactionals)
@@ -378,13 +380,15 @@ public TransactionalProxy(Supplier<ImmutableList<Transactional>> transactionals)
         protected void doPrepare()
         {
             transactionals.get().forEach(Transactional::prepareToCommit);
-            new StatsComponent(finalizeMetadata()).save(descriptor);
+            new StatsComponent(descriptor, finalizeMetadata()).save(descriptor);
 
             // save the table of components
-            TOCComponent.appendTOC(descriptor, components);
+            TOCComponent.appendTOC(descriptor, components());
+        }
 
-            if (openResult)
-                finalReader = openFinal(SSTableReader.OpenReason.NORMAL);
+        private void openResult()
+        {
+            finalReader = openFinal(SSTableReader.OpenReason.NORMAL);
         }
 
         protected Throwable doCommit(Throwable accumulate)
@@ -502,7 +506,7 @@ private static Set<Component> indexComponents(Collection<Index.Group> indexGroup
             Set<Component> components = new HashSet<>();
             for (Index.Group group : indexGroups)
             {
-                components.addAll(group.getComponents());
+                components.addAll(group.componentsForNewSSTable());
             }
 
             return components;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java
index 46b65140c54e..4e0d9e573117 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java
@@ -26,6 +26,7 @@
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.rows.RangeTombstoneMarker;
 import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
 import org.apache.cassandra.db.rows.SerializationHelper;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredSerializer;
@@ -56,6 +57,9 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
     protected DeletionTime openMarker = DeletionTime.LIVE;
     protected DeletionTime startOpenMarker = DeletionTime.LIVE;
 
+    private DecoratedKey lastKey;
+    private DeletionTime lastPartitionLevelDeletion;
+
     // Sequence control, also used to add empty static row if `addStaticRow` is not called.
     private enum State
     {
@@ -104,6 +108,9 @@ public void start(DecoratedKey key, DeletionTime partitionLevelDeletion) throws
         ByteBufferUtil.writeWithShortLength(key.getKey(), writer);
         DeletionTime.getSerializer(version).serialize(partitionLevelDeletion, writer);
 
+        lastKey = key;
+        lastPartitionLevelDeletion = partitionLevelDeletion;
+
         if (!header.hasStatic())
         {
             this.headerLength = writer.position() - initialPosition;
@@ -127,6 +134,17 @@ public void addStaticRow(Row staticRow) throws IOException
 
     public void addUnfiltered(Unfiltered unfiltered) throws IOException
     {
+        if (state == State.AWAITING_STATIC_ROW)
+        {
+            if (unfiltered.isRow() && ((Row) unfiltered).isStatic())
+            {
+                addStaticRow((Row) unfiltered);
+                return;
+            }
+
+            addStaticRow(Rows.EMPTY_STATIC_ROW);
+        }
+
         checkState(state == State.AWAITING_ROWS);
 
         long pos = currentPosition();
@@ -155,6 +173,8 @@ public void addUnfiltered(Unfiltered unfiltered) throws IOException
 
     protected long finish() throws IOException
     {
+        if (state == State.AWAITING_STATIC_ROW)
+            addStaticRow(Rows.EMPTY_STATIC_ROW);
         checkState(state == State.AWAITING_ROWS);
 
         state = State.COMPLETED;
@@ -162,6 +182,9 @@ protected long finish() throws IOException
         long endPosition = currentPosition();
         unfilteredSerializer.writeEndOfPartition(writer);
 
+        lastKey = null;
+        lastPartitionLevelDeletion = null;
+
         return endPosition;
     }
 
@@ -174,4 +197,14 @@ public long getInitialPosition()
     {
         return initialPosition;
     }
+
+    public DecoratedKey getLastKey()
+    {
+        return lastKey;
+    }
+
+    public DeletionTime getLastPartitionLevelDeletion()
+    {
+        return lastPartitionLevelDeletion;
+    }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableReaderLoadingBuilder.java
index 4b647549cdd9..7190a3ee725a 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableReaderLoadingBuilder.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableReaderLoadingBuilder.java
@@ -25,6 +25,7 @@
 import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.IFilter;
@@ -58,11 +59,12 @@ protected FileHandle.Builder dataFileBuilder(StatsMetadata statsMetadata)
         int bufferSize = ioOptions.diskOptimizationStrategy.bufferSize(recordSize);
 
         if (dataFileBuilder == null)
-            dataFileBuilder = new FileHandle.Builder(descriptor.fileFor(BtiFormat.Components.DATA));
+            dataFileBuilder = StorageProvider.instance.fileHandleBuilderFor(descriptor, BtiFormat.Components.DATA);
 
         dataFileBuilder.bufferSize(bufferSize);
         dataFileBuilder.withChunkCache(chunkCache);
         dataFileBuilder.mmapped(ioOptions.defaultDiskAccessMode);
+        dataFileBuilder.slice(statsMetadata.zeroCopyMetadata);
 
         return dataFileBuilder;
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java
index e8fbea22d279..ecb08b38f7a2 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java
@@ -39,12 +39,13 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.ClusteringComparator;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.LivenessInfo;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
 import org.apache.cassandra.db.partitions.Partition;
@@ -84,7 +85,7 @@ public abstract class SortedTableScrubber<R extends SSTableReaderWithFilter> imp
 {
     private final static Logger logger = LoggerFactory.getLogger(SortedTableScrubber.class);
 
-    protected final ColumnFamilyStore cfs;
+    protected final CompactionRealm realm;
     protected final LifecycleTransaction transaction;
     protected final File destination;
     protected final IScrubber.Options options;
@@ -107,33 +108,33 @@ public abstract class SortedTableScrubber<R extends SSTableReaderWithFilter> imp
     protected int emptyPartitions;
 
 
-    protected SortedTableScrubber(ColumnFamilyStore cfs,
+    protected SortedTableScrubber(CompactionRealm realm,
                                   LifecycleTransaction transaction,
                                   OutputHandler outputHandler,
                                   Options options)
     {
         this.sstable = (R) transaction.onlyOne();
         Preconditions.checkNotNull(sstable.metadata());
-        assert sstable.metadata().keyspace.equals(cfs.getKeyspaceName());
-        if (!sstable.descriptor.cfname.equals(cfs.metadata().name))
+        assert sstable.metadata().keyspace.equals(realm.getKeyspaceName());
+        if (!sstable.descriptor.cfname.equals(realm.metadata().name))
         {
-            logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, cfs.metadata().name);
+            logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, realm.metadata().name);
         }
         try
         {
-            sstable.metadata().validateCompatibility(cfs.metadata());
+            sstable.metadata().validateCompatibility(realm.metadata());
         }
         catch (ConfigurationException ex)
         {
-            logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, cfs.metadata().name);
+            logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, realm.metadata().name);
         }
 
-        this.cfs = cfs;
+        this.realm = realm;
         this.transaction = transaction;
         this.outputHandler = outputHandler;
         this.options = options;
-        this.destination = cfs.getDirectories().getLocationForDisk(cfs.getDiskBoundaries().getCorrectDiskForSSTable(sstable));
-        this.isCommutative = cfs.metadata().isCounter();
+        this.destination = realm.getDirectories().getLocationForDisk(realm.getDiskBoundaries().getCorrectDiskForSSTable(sstable));
+        this.isCommutative = realm.metadata().isCounter();
 
         List<SSTableReader> toScrub = Collections.singletonList(sstable);
 
@@ -146,13 +147,14 @@ protected SortedTableScrubber(ColumnFamilyStore cfs,
         {
             approximateKeyCount = 0;
         }
-        this.expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, approximateKeyCount);
+        this.expectedBloomFilterSize = Math.max(realm.metadata().params.minIndexInterval, approximateKeyCount);
 
         // loop through each partition, deserializing to check for damage.
         // We'll also loop through the index at the same time, using the position from the index to recover if the
         // partition header (key or data size) is corrupt. (This means our position in the index file will be one
         // partition "ahead" of the data file.)
-        this.dataFile = transaction.isOffline()
+        boolean isOffline = options.overrideTxnIsOffline ? false : transaction.isOffline();
+        this.dataFile = isOffline
                         ? sstable.openDataReader()
                         : sstable.openDataReader(CompactionManager.instance.getRateLimiter());
 
@@ -180,15 +182,15 @@ public static void deleteOrphanedComponents(Descriptor descriptor, Set<Component
     }
 
     @Override
-    public void scrub()
+    public List<SSTableReader>  scrub()
     {
         List<SSTableReader> finished = new ArrayList<>();
         outputHandler.output("Scrubbing %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length()));
-        try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge);
+        try (SSTableRewriter writer = SSTableRewriter.construct(realm, transaction, false, sstable.maxDataAge);
              Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
         {
             StatsMetadata metadata = sstable.getSSTableMetadata();
-            writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction));
+            writer.switchWriter(CompactionManager.createWriter(realm, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction));
 
             scrubInternal(writer);
 
@@ -205,11 +207,14 @@ public void scrub()
         }
         finally
         {
-            if (transaction.isOffline())
+            boolean isOffline = options.overrideTxnIsOffline ? false : transaction.isOffline();
+            if (isOffline)
                 finished.forEach(sstable -> sstable.selfRef().release());
         }
 
         outputSummary(finished);
+
+        return finished; // already released
     }
 
     protected abstract void scrubInternal(SSTableRewriter writer) throws IOException;
@@ -238,7 +243,7 @@ private SSTableReader writeOutOfOrderPartitions(StatsMetadata metadata)
         // out of order partitions/rows, but no bad partition found - we can keep our repairedAt time
         long repairedAt = badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt;
         SSTableReader newInOrderSstable;
-        try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction))
+        try (SSTableWriter inOrderWriter = CompactionManager.createWriter(realm, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction))
         {
             for (Partition partition : outOfOrder)
                 inOrderWriter.append(partition.unfilteredIterator());
@@ -252,18 +257,18 @@ private SSTableReader writeOutOfOrderPartitions(StatsMetadata metadata)
         return newInOrderSstable;
     }
 
-    protected abstract UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename);
+    protected abstract UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, File file);
 
     @Override
     @VisibleForTesting
     public ScrubResult scrubWithResult()
     {
-        scrub();
-        return new ScrubResult(goodPartitions, badPartitions, emptyPartitions);
+        List<SSTableReader> scrubbed = scrub();
+        return new ScrubResult(goodPartitions, badPartitions, emptyPartitions, scrubbed);
     }
 
     @Override
-    public CompactionInfo.Holder getScrubInfo()
+    public TableOperation getScrubInfo()
     {
         return scrubInfo;
     }
@@ -275,7 +280,7 @@ protected String keyString(DecoratedKey key)
 
         try
         {
-            return cfs.metadata().partitionKeyType.getString(key.getKey());
+            return realm.metadata().partitionKeyType.getString(key.getKey());
         }
         catch (Exception e)
         {
@@ -288,8 +293,8 @@ protected boolean tryAppend(DecoratedKey prevKey, DecoratedKey key, SSTableRewri
         // OrderCheckerIterator will check, at iteration time, that the rows are in the proper order. If it detects
         // that one row is out of order, it will stop returning them. The remaining rows will be sorted and added
         // to the outOfOrder set that will be later written to a new SSTable.
-        try (OrderCheckerIterator sstableIterator = new OrderCheckerIterator(getIterator(key), cfs.metadata().comparator);
-             UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getPath()))
+        try (OrderCheckerIterator sstableIterator = new OrderCheckerIterator(getIterator(key), realm.metadata().comparator);
+             UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getFile()))
         {
             if (prevKey != null && prevKey.compareTo(key) > 0)
             {
@@ -358,7 +363,7 @@ protected void throwIfCannotContinue(DecoratedKey key, Throwable th)
     }
 
 
-    public static class ScrubInfo extends CompactionInfo.Holder
+    public static class ScrubInfo extends AbstractTableOperation
     {
         private final RandomAccessReader dataFile;
         private final SSTableReader sstable;
@@ -373,18 +378,18 @@ public ScrubInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileRe
             scrubCompactionId = nextTimeUUID();
         }
 
-        public CompactionInfo getCompactionInfo()
+        public OperationProgress getProgress()
         {
             fileReadLock.lock();
             try
             {
-                return new CompactionInfo(sstable.metadata(),
-                                          OperationType.SCRUB,
-                                          dataFile.getFilePointer(),
-                                          dataFile.length(),
-                                          scrubCompactionId,
-                                          ImmutableSet.of(sstable),
-                                          File.getPath(sstable.getFilename()).getParent().toString());
+                return new OperationProgress(sstable.metadata(), 
+                                             OperationType.SCRUB,
+                                             dataFile.getFilePointer(),
+                                             dataFile.length(),
+                                             scrubCompactionId,
+                                             ImmutableSet.of(sstable),
+                                             File.getPath(sstable.getFilename()).getParent().toString());
             }
             catch (Exception e)
             {
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
index f68e1c968455..41384e12c56e 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
@@ -28,22 +28,23 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Function;
-import java.util.function.LongPredicate;
+
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableSet;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.compaction.CompactionController;
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.dht.LocalPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -68,7 +69,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
 {
     private final static Logger logger = LoggerFactory.getLogger(SortedTableVerifier.class);
 
-    protected final ColumnFamilyStore cfs;
+    protected final @Nullable CompactionRealm realm;
     protected final R sstable;
 
     protected final ReadWriteLock fileAccessLock;
@@ -87,9 +88,11 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
 
     protected final OutputHandler outputHandler;
 
-    public SortedTableVerifier(ColumnFamilyStore cfs, R sstable, OutputHandler outputHandler, boolean isOffline, Options options)
+    public SortedTableVerifier(CompactionRealm realm, R sstable, OutputHandler outputHandler, boolean isOffline, Options options)
     {
-        this.cfs = cfs;
+        Preconditions.checkArgument(realm != null || !options.mutateRepairStatus);
+
+        this.realm = realm;
         this.sstable = sstable;
         this.outputHandler = outputHandler;
 
@@ -111,7 +114,7 @@ protected void deserializeBloomFilter(SSTableReader sstable) throws IOException
         }
     }
 
-    public CompactionInfo.Holder getVerifyInfo()
+    public AbstractTableOperation getVerifyInfo()
     {
         return verifyInfo;
     }
@@ -127,8 +130,8 @@ protected void markAndThrow(Throwable cause, boolean mutateRepaired)
         {
             try
             {
-                sstable.mutateRepairedAndReload(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient());
-                cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
+                // note that it additionally uses a lock and verification
+                realm.mutateRepairedWithLock(Collections.singleton(sstable), ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient());
             }
             catch (IOException ioe)
             {
@@ -152,7 +155,7 @@ public void verify()
 
         verifyBloomFilter();
 
-        if (options.checkOwnsTokens && !isOffline && !(cfs.getPartitioner() instanceof LocalPartitioner))
+        if (options.checkOwnsTokens && !isOffline && !(sstable.getPartitioner() instanceof LocalPartitioner))
         {
             if (verifyOwnedRanges() == 0)
                 return;
@@ -218,7 +221,7 @@ protected int verifyOwnedRanges()
         outputHandler.debug("Checking that all tokens are owned by the current node");
         try (KeyIterator iter = sstable.keyIterator())
         {
-            ownedRanges = Range.normalize(tokenLookup.apply(cfs.metadata.keyspace));
+            ownedRanges = Range.normalize(tokenLookup.apply(sstable.getKeyspaceName()));
             if (ownedRanges.isEmpty())
                 return 0;
             RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges);
@@ -268,21 +271,19 @@ protected void verifySSTable()
     {
         outputHandler.output("Extended Verify requested, proceeding to inspect values");
 
-        try (VerifyController verifyController = new VerifyController(cfs);
-             KeyReader indexIterator = sstable.keyReader())
+        try (KeyReader indexIterator = sstable.keyReader())
         {
-            if (indexIterator.dataPosition() != 0)
+            if (indexIterator.dataPosition() != sstable.getDataFileSliceDescriptor().dataStart)
                 markAndThrow(new RuntimeException("First row position from index != 0: " + indexIterator.dataPosition()));
 
-            List<Range<Token>> ownedRanges = isOffline ? Collections.emptyList() : Range.normalize(tokenLookup.apply(cfs.metadata().keyspace));
+            List<Range<Token>> ownedRanges = isOffline ? Collections.emptyList() : Range.normalize(tokenLookup.apply(sstable.getKeyspaceName()));
             RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges);
             DecoratedKey prevKey = null;
 
             while (!dataFile.isEOF())
             {
 
-                if (verifyInfo.isStopRequested())
-                    throw new CompactionInterruptedException(verifyInfo.getCompactionInfo());
+                verifyInfo.throwIfStopRequested();
 
                 long rowStart = dataFile.getFilePointer();
                 outputHandler.debug("Reading row at %d", rowStart);
@@ -297,7 +298,7 @@ protected void verifySSTable()
                     markAndThrow(th);
                 }
 
-                if (options.checkOwnsTokens && ownedRanges.size() > 0 && !(cfs.getPartitioner() instanceof LocalPartitioner))
+                if (options.checkOwnsTokens && !ownedRanges.isEmpty() && !(sstable.getPartitioner() instanceof LocalPartitioner))
                 {
                     try
                     {
@@ -338,7 +339,9 @@ protected void verifySSTable()
                     if (key == null || dataSize > dataFile.length())
                         markAndThrow(new RuntimeException(String.format("key = %s, dataSize=%d, dataFile.length() = %d", key, dataSize, dataFile.length())));
 
-                    try (UnfilteredRowIterator iterator = SSTableIdentityIterator.create(sstable, dataFile, key))
+                    //mimic the scrub read path
+                    try (UnfilteredRowIterator identity = SSTableIdentityIterator.create(sstable, dataFile, key);
+                         UnfilteredRowIterator iterator = UnfilteredRowIterators.withValidation(identity, dataFile.getFile()))
                     {
                         verifyPartition(key, iterator);
                     }
@@ -469,7 +472,7 @@ public boolean check(DecoratedKey key)
         }
     }
 
-    protected static class VerifyInfo extends CompactionInfo.Holder
+    protected static class VerifyInfo extends AbstractTableOperation
     {
         private final RandomAccessReader dataFile;
         private final SSTableReader sstable;
@@ -484,17 +487,17 @@ public VerifyInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileR
             verificationCompactionId = TimeUUID.Generator.nextTimeUUID();
         }
 
-        public CompactionInfo getCompactionInfo()
+        public OperationProgress getProgress()
         {
             fileReadLock.lock();
             try
             {
-                return new CompactionInfo(sstable.metadata(),
-                                          OperationType.VERIFY,
-                                          dataFile.getFilePointer(),
-                                          dataFile.length(),
-                                          verificationCompactionId,
-                                          ImmutableSet.of(sstable));
+                return new OperationProgress(sstable.metadata(),
+                                             OperationType.VERIFY,
+                                             dataFile.getFilePointer(),
+                                             dataFile.length(),
+                                             verificationCompactionId,
+                                             ImmutableSet.of(sstable));
             }
             catch (Exception e)
             {
@@ -511,18 +514,4 @@ public boolean isGlobal()
             return false;
         }
     }
-
-    protected static class VerifyController extends CompactionController
-    {
-        public VerifyController(ColumnFamilyStore cfs)
-        {
-            super(cfs, Integer.MAX_VALUE);
-        }
-
-        @Override
-        public LongPredicate getPurgeEvaluator(DecoratedKey key)
-        {
-            return time -> false;
-        }
-    }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
index 473176be5bb7..3b87de4cbad8 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
@@ -24,6 +24,7 @@
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
+import javax.annotation.concurrent.NotThreadSafe;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -33,6 +34,7 @@
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionPurger;
 import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.compaction.writers.SSTableDataSink;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.guardrails.Threshold;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
@@ -56,6 +58,7 @@
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.DataPosition;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.SequentialWriter;
@@ -69,25 +72,30 @@
 import org.apache.cassandra.utils.concurrent.Transactional;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
 
 /**
  * A generic implementation of a writer which assumes the existence of some partition index and bloom filter.
  */
-public abstract class SortedTableWriter<P extends SortedTablePartitionWriter, I extends SortedTableWriter.AbstractIndexWriter> extends SSTableWriter
+@NotThreadSafe
+public abstract class SortedTableWriter<P extends SortedTablePartitionWriter, I extends SortedTableWriter.AbstractIndexWriter> extends SSTableWriter implements SSTableDataSink
 {
     private final static Logger logger = LoggerFactory.getLogger(SortedTableWriter.class);
+    private final boolean isInternalKeyspace;
 
     // TODO dataWriter is not needed to be directly accessible - we can access everything we need for the dataWriter
     //   from a partition writer
     protected final SequentialWriter dataWriter;
     protected final I indexWriter;
     protected final P partitionWriter;
-    private final FileHandle.Builder dataFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.DATA));
+    private final FileHandle.Builder dataFileBuilder = StorageProvider.instance.fileHandleBuilderFor(descriptor, Components.DATA);
     private DecoratedKey lastWrittenKey;
     private DataPosition dataMark;
     private long lastEarlyOpenLength;
     private final Supplier<Double> crcCheckChanceSupplier;
 
+    private Throwable failure = null;
+
     public SortedTableWriter(Builder<P, I, ?, ?> builder, LifecycleNewTracker lifecycleNewTracker, SSTable.Owner owner)
     {
         super(builder, lifecycleNewTracker, owner);
@@ -111,15 +119,23 @@ public SortedTableWriter(Builder<P, I, ?, ?> builder, LifecycleNewTracker lifecy
             this.dataWriter = dataWriter;
             this.indexWriter = indexWriter;
             this.partitionWriter = partitionWriter;
+            this.isInternalKeyspace = SchemaConstants.isInternalKeyspace(metadata.keyspace);
         }
         catch (RuntimeException | Error ex)
         {
             Throwables.closeNonNullAndAddSuppressed(ex, partitionWriter, indexWriter, dataWriter);
             handleConstructionFailure(ex);
+            failure = ex;
             throw ex;
         }
     }
 
+    private void assertNotBroken()
+    {
+        if (failure != null)
+            throw new AssertionError("Cannot use a broken writer", failure);
+    }
+
     /**
      * Appends partition data to this writer.
      *
@@ -131,16 +147,16 @@ public SortedTableWriter(Builder<P, I, ?, ?> builder, LifecycleNewTracker lifecy
     @Override
     public final AbstractRowIndexEntry append(UnfilteredRowIterator partition)
     {
+        assertNotBroken();
+
         if (partition.isEmpty())
             return null;
 
         try
         {
-            if (!verifyPartition(partition.partitionKey()))
+            if (!startPartition(partition.partitionKey(), partition.partitionLevelDeletion()))
                 return null;
 
-            startPartition(partition.partitionKey(), partition.partitionLevelDeletion());
-
             AbstractRowIndexEntry indexEntry;
             if (header.hasStatic())
                 addStaticRow(partition.partitionKey(), partition.staticRow());
@@ -154,14 +170,59 @@ public final AbstractRowIndexEntry append(UnfilteredRowIterator partition)
         }
         catch (BufferOverflowException boe)
         {
+            failure = boe;
             throw new PartitionSerializationException(partition, boe);
         }
         catch (IOException e)
         {
+            failure = e;
             throw new FSWriteError(e, getFilename());
         }
     }
 
+    @Override
+    public final void addUnfiltered(Unfiltered unfiltered)
+    {
+        assertNotBroken();
+
+        try
+        {
+            if (unfiltered.isRow())
+            {
+                Row row = (Row) unfiltered;
+                if (row.isStatic())
+                    addStaticRow(requireNonNull(partitionWriter.getLastKey()), row);
+                else
+                    addRow(requireNonNull(partitionWriter.getLastKey()), row);
+            }
+            else
+            {
+                addRangeTomstoneMarker((RangeTombstoneMarker) unfiltered);
+            }
+        }
+        catch (IOException | RuntimeException ex)
+        {
+            failure = ex;
+            throw new FSWriteError(ex, getFilename());
+        }
+    }
+
+    @Override
+    public final AbstractRowIndexEntry endPartition()
+    {
+        assertNotBroken();
+
+        try
+        {
+            return endPartition(requireNonNull(partitionWriter.getLastKey()), partitionWriter.getLastPartitionLevelDeletion());
+        }
+        catch (IOException | RuntimeException ex)
+        {
+            failure = ex;
+            throw new FSWriteError(ex, getFilename());
+        }
+    }
+
     private boolean verifyPartition(DecoratedKey key)
     {
         assert key != null : "Keys must not be null"; // empty keys ARE allowed b/c of indexed column values
@@ -173,17 +234,33 @@ private boolean verifyPartition(DecoratedKey key)
         }
 
         if (lastWrittenKey != null && lastWrittenKey.compareTo(key) >= 0)
-            throw new RuntimeException(String.format("Last written key %s >= current key %s, writing into %s", lastWrittenKey, key, getFilename()));
+            throw new AssertionError("Last written key " + lastWrittenKey + " >= current key " + key + " writing into " + getDataFile());
 
         return true;
     }
 
-    private void startPartition(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException
+    @Override
+    public boolean startPartition(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException
     {
-        partitionWriter.start(key, partitionLevelDeletion);
-        metadataCollector.updatePartitionDeletion(partitionLevelDeletion);
+        assertNotBroken();
+
+        if (!verifyPartition(key))
+            return false;
+
+        try
+        {
+            partitionWriter.start(key, partitionLevelDeletion);
+            metadataCollector.updatePartitionDeletion(partitionLevelDeletion);
+
+            onStartPartition(key);
+        }
+        catch (IOException | RuntimeException ex)
+        {
+            failure = ex;
+            throw ex;
+        }
 
-        onStartPartition(key);
+        return true;
     }
 
     private void addStaticRow(DecoratedKey key, Row row) throws IOException
@@ -364,7 +441,7 @@ protected FileHandle openDataFile(long lengthOverride, StatsMetadata statsMetada
             if (chunkCache != null)
             {
                 if (lastEarlyOpenLength != 0 && dataFile.dataLength() > lastEarlyOpenLength)
-                    chunkCache.invalidatePosition(dataFile, lastEarlyOpenLength);
+                    dataFile.rebuffererFactory().invalidateIfCached(lastEarlyOpenLength);
             }
             lastEarlyOpenLength = dataFile.dataLength();
         }
@@ -379,6 +456,9 @@ protected FileHandle openDataFile(long lengthOverride, StatsMetadata statsMetada
 
     private void guardPartitionThreshold(Threshold guardrail, DecoratedKey key, long size)
     {
+        if (isInternalKeyspace)
+            return;
+
         if (guardrail.triggersOn(size, null))
         {
             String message = String.format("%s.%s:%s on sstable %s",
@@ -392,6 +472,9 @@ private void guardPartitionThreshold(Threshold guardrail, DecoratedKey key, long
 
     private void guardCollectionSize(DecoratedKey partitionKey, Row row)
     {
+        if (isInternalKeyspace)
+            return;
+        
         if (!Guardrails.collectionSize.enabled() && !Guardrails.itemsPerCollection.enabled())
             return;
 
@@ -418,16 +501,22 @@ private void guardCollectionSize(DecoratedKey partitionKey, Row row)
                 !Guardrails.itemsPerCollection.triggersOn(cellsCount, null))
                 continue;
 
-            String keyString = metadata.getLocal().primaryKeyAsCQLLiteral(partitionKey.getKey(), row.clustering());
-            String msg = String.format("%s in row %s in table %s",
+            String msg = String.format("%s in table %s",
                                        column.name.toString(),
-                                       keyString,
                                        metadata);
-            Guardrails.collectionSize.guard(cellsSize, msg, true, null);
-            Guardrails.itemsPerCollection.guard(cellsCount, msg, true, null);
+            Guardrails.collectionSize.guard(cellsSize, msg, false, null);
+            Guardrails.itemsPerCollection.guard(cellsCount, msg, false, null);
         }
     }
 
+    protected void invalidateCacheAtPreviousBoundary(FileHandle dfile, long newBoundary)
+    {
+        if (lastEarlyOpenLength != 0 && newBoundary > lastEarlyOpenLength)
+            dfile.rebuffererFactory().invalidateIfCached(lastEarlyOpenLength);
+
+        lastEarlyOpenLength = newBoundary;
+    }
+
     protected static abstract class AbstractIndexWriter extends AbstractTransactional implements Transactional
     {
         protected final Descriptor descriptor;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java b/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java
index 25042e331fac..c1dda7277e33 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.EnumMap;
 import java.util.EnumSet;
 import java.util.Map;
 
@@ -36,17 +37,16 @@
 import org.apache.cassandra.io.sstable.metadata.MetadataType;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.SequentialWriter;
-import org.apache.cassandra.io.util.SequentialWriterOption;
 import org.apache.cassandra.schema.TableMetadata;
 
 public class StatsComponent
 {
+    public final Descriptor descriptor;
     public final Map<MetadataType, MetadataComponent> metadata;
 
-    public StatsComponent(Map<MetadataType, MetadataComponent> metadata)
+    public StatsComponent(Descriptor descriptor, Map<MetadataType, MetadataComponent> metadata)
     {
+        this.descriptor = descriptor;
         this.metadata = ImmutableMap.copyOf(metadata);
     }
 
@@ -67,7 +67,7 @@ public static StatsComponent load(Descriptor descriptor, MetadataType... types)
             throw new CorruptSSTableException(e, descriptor.fileFor(Components.STATS));
         }
 
-        return new StatsComponent(metadata);
+        return new StatsComponent(descriptor, metadata);
     }
 
     public SerializationHeader.Component serializationHeader()
@@ -75,14 +75,14 @@ public SerializationHeader.Component serializationHeader()
         return (SerializationHeader.Component) metadata.get(MetadataType.HEADER);
     }
 
-    public SerializationHeader serializationHeader(TableMetadata metadata)
+    public SerializationHeader serializationHeader(Descriptor descriptor, TableMetadata metadata)
     {
         SerializationHeader.Component header = serializationHeader();
         if (header != null)
         {
             try
             {
-                return header.toHeader(metadata);
+                return header.toHeader(descriptor, metadata);
             }
             catch (UnknownColumnException ex)
             {
@@ -108,18 +108,22 @@ public StatsMetadata statsMetadata()
         return (StatsMetadata) metadata.get(MetadataType.STATS);
     }
 
+    public StatsComponent with(ValidationMetadata validationMetadata)
+    {
+        Map<MetadataType, MetadataComponent> newMetadata = new EnumMap<>(metadata);
+        newMetadata.put(MetadataType.VALIDATION, validationMetadata);
+        return new StatsComponent(descriptor, newMetadata);
+    }
+
     public void save(Descriptor desc)
     {
-        File file = desc.fileFor(Components.STATS);
-        try (SequentialWriter out = new SequentialWriter(file, SequentialWriterOption.DEFAULT))
+        try
         {
-            desc.getMetadataSerializer().serialize(metadata, out, desc.version);
-            out.finish();
+            desc.getMetadataSerializer().rewriteSSTableMetadata(desc, metadata);
         }
         catch (IOException e)
         {
-            throw new FSWriteError(e, file.path());
+            throw new FSWriteError(e);
         }
     }
-
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java b/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java
index 15d81d0ba1a9..92f57ce94e75 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java
@@ -35,11 +35,13 @@
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableWatcher;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
 
 import static org.apache.cassandra.io.util.File.WriteMode.APPEND;
+import static org.apache.cassandra.io.util.File.WriteMode.OVERWRITE;
 
 public class TOCComponent
 {
@@ -78,12 +80,11 @@ public static Set<Component> loadTOC(Descriptor descriptor, boolean skipMissing)
     }
 
     /**
-     * Appends new component names to the TOC component.
+     * Write TOC file with given components and write mode
      */
-    public static void appendTOC(Descriptor descriptor, Collection<Component> components)
+    public static void writeTOC(File tocFile, Collection<Component> components, File.WriteMode writeMode)
     {
-        File tocFile = descriptor.fileFor(Components.TOC);
-        try (FileOutputStreamPlus out = tocFile.newOutputStream(APPEND);
+        try (FileOutputStreamPlus out = tocFile.newOutputStream(writeMode);
              PrintWriter w = new PrintWriter(out))
         {
             for (Component component : components)
@@ -97,12 +98,23 @@ public static void appendTOC(Descriptor descriptor, Collection<Component> compon
         }
     }
 
+    /**
+     * Appends new component names to the TOC component.
+     */
+    @SuppressWarnings("resource")
+    public static void appendTOC(Descriptor descriptor, Collection<Component> components)
+    {
+        File tocFile = descriptor.fileFor(Components.TOC);
+        writeTOC(tocFile, components, APPEND);
+    }
+
     public static Set<Component> loadOrCreate(Descriptor descriptor)
     {
         try
         {
             try
             {
+                SSTableWatcher.instance.discoverComponents(descriptor);
                 return TOCComponent.loadTOC(descriptor);
             }
             catch (FileNotFoundException | NoSuchFileException e)
@@ -128,8 +140,25 @@ public static Set<Component> loadOrCreate(Descriptor descriptor)
     public static void rewriteTOC(Descriptor descriptor, Collection<Component> components)
     {
         File tocFile = descriptor.fileFor(Components.TOC);
-        if (!tocFile.tryDelete())
-            logger.error("Failed to delete TOC component for " + descriptor);
-        appendTOC(descriptor, components);
+        // As this method *re*-write the TOC (and is currently only called by "unregisterComponents"), it should only
+        // be called in contexts where the TOC is expected to exist. If it doesn't, there is probably something
+        // unexpected happening, so we log relevant information to help diagnose a potential earlier problem.
+        // But in principle, this isn't a big deal for this method, and we still end up with the TOC in the state we
+        // expect.
+        if (!tocFile.exists())
+        {
+            // Note: we pass a dummy runtime exception as a simple way to get a stack-trace. Knowing from where this
+            // is called in this case is likely useful information.
+            logger.warn("Was asked to 'rewrite' TOC file {} for sstable {}, but it does not exists. The file will be created but this is unexpected. The components to 'overwrite' are: {}", tocFile, descriptor, components, new RuntimeException());
+        }
+
+        writeTOC(tocFile, components, OVERWRITE);
+    }
+
+    public static void maybeAdd(Descriptor descriptor, Component component) throws IOException
+    {
+        Set<Component> toc = TOCComponent.loadOrCreate(descriptor);
+        if (!toc.isEmpty() && toc.add(component))
+            TOCComponent.rewriteTOC(descriptor, toc);
     }
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index 2b214ab56d5b..4f02118cc23b 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -20,6 +20,8 @@
 import java.util.Objects;
 import java.util.regex.Pattern;
 
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
 
 /**
  * A set of feature flags associated with a SSTable format
@@ -59,7 +61,7 @@ protected Version(SSTableFormat format, String version)
     public abstract boolean hasIsTransient();
 
     public abstract boolean hasMetadataChecksum();
-    
+
     /**
      * This format raises the legacy int year 2038 limit to 2106 by using an uint instead
      */
@@ -101,6 +103,12 @@ protected Version(SSTableFormat format, String version)
      */
     public abstract boolean hasPartitionLevelDeletionsPresenceMarker();
 
+    /**
+     * Records in th stats if the sstable has any partition deletions. Note that this is DSE specific as we had this
+     * field in some BTI versions in a different place than in the OSS.
+     */
+    public abstract boolean hasMisplacedPartitionLevelDeletionsPresenceMarker();
+
     public abstract boolean hasKeyRange();
 
     /**
@@ -143,4 +151,24 @@ public int hashCode()
     {
         return Objects.hash(version, format.name());
     }
+
+    // the fields below are present only in DSE but we do not use them here; though in order to be able to read
+    // DSE sstables we need to at least skip that data
+    public abstract boolean hasZeroCopyMetadata();
+
+    public abstract boolean hasIncrementalNodeSyncMetadata();
+
+    // TODO TBD
+    public abstract boolean hasMaxColumnValueLengths();
+
+    public abstract ByteComparable.Version getByteComparableVersion();
+
+    /**
+     * Whether we expect that sstable has explicitly frozen tuples in its {@link org.apache.cassandra.db.SerializationHeader}.
+     * If {@code false}, we don't try to fix non-frozen tuples that are not types of dropped columns and fail loading
+     * the sstable. If {@code true}, we try to fix non-frozen tuples and load the sstable.
+     *
+     * See <a href="https://github.com/riptano/cndb/issues/8696">this</a> for reference.
+     */
+    public abstract boolean hasImplicitlyFrozenTuples();
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 4abfc88b5a4f..7e5ce502b6e8 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -29,7 +29,6 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,11 +39,11 @@
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.GaugeProvider;
 import org.apache.cassandra.io.sstable.IScrubber;
 import org.apache.cassandra.io.sstable.MetricsProviders;
+import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics;
 import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
@@ -62,6 +61,7 @@
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 import static org.apache.cassandra.io.sstable.format.SSTableFormat.Components.DATA;
 
@@ -228,7 +228,7 @@ public void deleteOrphanedComponents(Descriptor descriptor, Set<Component> compo
 
     private void delete(Descriptor desc, List<Component> components)
     {
-        logger.info("Deleting sstable: {}", desc);
+        logger.debug("Deleting sstable: {}", desc);
 
         if (components.remove(DATA))
             components.add(0, DATA); // DATA component should be first
@@ -384,27 +384,31 @@ static class BigVersion extends Version
         {
             super(format, version);
 
+            boolean oOrLater = version.compareTo("o") >= 0;
+            boolean nOrLater = oOrLater || version.compareTo("n") >= 0;
+            boolean mOrLater = nOrLater || version.compareTo("m") >= 0;
+
             isLatestVersion = version.compareTo(current_version) == 0;
 
             // Note that, we probably forgot to change that to 40 for N version, and therefore we cannot do it now.
             correspondingMessagingVersion = version.compareTo("oa") >= 0 ? MessagingService.VERSION_50 : MessagingService.VERSION_30;
 
-            hasCommitLogLowerBound = version.compareTo("mb") >= 0;
-            hasCommitLogIntervals = version.compareTo("mc") >= 0;
-            hasAccurateMinMax = version.matches("(m[d-z])|(n[a-z])"); // deprecated in 'oa' and to be removed after 'oa'
-            hasLegacyMinMax = version.matches("(m[a-z])|(n[a-z])"); // deprecated in 'oa' and to be removed after 'oa'
+            hasCommitLogLowerBound = mOrLater && version.compareTo("mb") >= 0;
+            hasCommitLogIntervals = mOrLater && version.compareTo("mc") >= 0;
+            hasAccurateMinMax = mOrLater && version.compareTo("md") >= 0 && !oOrLater; // deprecated in 'nc' and to be removed in 'oa'
+            hasLegacyMinMax = mOrLater && !oOrLater; // deprecated in 'nc' and to be removed in 'oa'
             // When adding a new version you might need to add it here
-            hasOriginatingHostId = version.compareTo("nb") >= 0 || version.matches("(m[e-z])");
-            hasMaxCompressedLength = version.compareTo("na") >= 0;
-            hasPendingRepair = version.compareTo("na") >= 0;
-            hasIsTransient = version.compareTo("na") >= 0;
-            hasMetadataChecksum = version.compareTo("na") >= 0;
-            hasOldBfFormat = version.compareTo("na") < 0;
-            hasImprovedMinMax = version.compareTo("oa") >= 0;
-            hasPartitionLevelDeletionPresenceMarker = version.compareTo("oa") >= 0;
-            hasKeyRange = version.compareTo("oa") >= 0;
-            hasUintDeletionTime = version.compareTo("oa") >= 0;
-            hasTokenSpaceCoverage = version.compareTo("oa") >= 0;
+            hasOriginatingHostId = nOrLater && version.compareTo("nb") >= 0 || mOrLater && !nOrLater && version.compareTo("me") >= 0;
+            hasMaxCompressedLength = nOrLater;
+            hasPendingRepair = nOrLater;
+            hasIsTransient = nOrLater;
+            hasMetadataChecksum = nOrLater;
+            hasOldBfFormat = !nOrLater;
+            hasImprovedMinMax = oOrLater;
+            hasPartitionLevelDeletionPresenceMarker = oOrLater;
+            hasKeyRange = oOrLater;
+            hasUintDeletionTime = oOrLater;
+            hasTokenSpaceCoverage = oOrLater;
         }
 
         @Override
@@ -497,6 +501,12 @@ public boolean hasPartitionLevelDeletionsPresenceMarker()
             return hasPartitionLevelDeletionPresenceMarker;
         }
 
+        @Override
+        public boolean hasMisplacedPartitionLevelDeletionsPresenceMarker()
+        {
+            return false;
+        }
+
         @Override
         public boolean hasUIntDeletionTime()
         {
@@ -520,6 +530,36 @@ public boolean isCompatibleForStreaming()
         {
             return isCompatible() && version.charAt(0) == current_version.charAt(0);
         }
+
+        @Override
+        public boolean hasZeroCopyMetadata()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean hasIncrementalNodeSyncMetadata()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean hasMaxColumnValueLengths()
+        {
+            return false;
+        }
+
+        @Override
+        public ByteComparable.Version getByteComparableVersion()
+        {
+            return ByteComparable.Version.OSS41;
+        }
+
+        @Override
+        public boolean hasImplicitlyFrozenTuples()
+        {
+            return false;
+        }
     }
 
     private static class BigTableSpecificMetricsProviders implements MetricsProviders
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java
index 801982d5ec59..a34a12becf61 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java
@@ -32,7 +32,6 @@
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.io.ISerializer;
-import org.apache.cassandra.io.sstable.IndexInfo;
 import org.apache.cassandra.io.sstable.format.SortedTablePartitionWriter;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.DataOutputBuffer;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java
index 84e02217d565..72addbe86042 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java
@@ -35,6 +35,7 @@
 import org.apache.cassandra.io.sstable.format.IndexComponent;
 import org.apache.cassandra.io.sstable.format.SortedTableReaderLoadingBuilder;
 import org.apache.cassandra.io.sstable.format.StatsComponent;
+import org.apache.cassandra.io.sstable.format.TOCComponent;
 import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
 import org.apache.cassandra.io.sstable.indexsummary.IndexSummary;
 import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder;
@@ -75,7 +76,7 @@ protected void openComponents(BigTableReader.Builder builder, SSTable.Owner owne
                 builder.setKeyCache(new KeyCache(CacheService.instance.keyCache));
 
             StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.HEADER, MetadataType.VALIDATION);
-            builder.setSerializationHeader(statsComponent.serializationHeader(builder.getTableMetadataRef().getLocal()));
+            builder.setSerializationHeader(statsComponent.serializationHeader(descriptor, builder.getTableMetadataRef().getLocal()));
             checkArgument(!online || builder.getSerializationHeader() != null);
 
             builder.setStatsMetadata(statsComponent.statsMetadata());
@@ -137,7 +138,7 @@ protected void openComponents(BigTableReader.Builder builder, SSTable.Owner owne
                 }
             }
 
-            try (CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components))
+            try (CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components, statsComponent.statsMetadata().zeroCopyMetadata))
             {
                 builder.setDataFile(dataFileBuilder(builder.getStatsMetadata())
                                     .withCompressionMetadata(compressionMetadata)
@@ -146,7 +147,25 @@ protected void openComponents(BigTableReader.Builder builder, SSTable.Owner owne
             }
 
             if (builder.getFilter() == null)
+            {
                 builder.setFilter(FilterFactory.AlwaysPresent);
+                logger.warn("Could not recreate or deserialize existing bloom filter, continuing with a pass-through " +
+                            "bloom filter but this will significantly impact reads performance");
+            }
+            else if (rebuildFilter)
+            {
+                if (validationMetadata.bloomFilterFPChance != tableMetadataRef.getLocal().params.bloomFilterFpChance)
+                {
+                    StatsComponent.load(descriptor, MetadataType.values())
+                                  .with(validationMetadata.withBloomFilterFPChance(tableMetadataRef.getLocal().params.bloomFilterFpChance))
+                                  .save(descriptor);
+                }
+                if (descriptor.fileFor(Components.FILTER).exists())
+                    TOCComponent.maybeAdd(descriptor, Components.FILTER);
+            }
+
+            if (rebuildSummary && descriptor.fileFor(Components.SUMMARY).exists())
+                TOCComponent.maybeAdd(descriptor, Components.SUMMARY);
 
             if (builder.getComponents().contains(Components.PRIMARY_INDEX))
                 builder.setIndexFile(indexFileBuilder(builder.getIndexSummary()).complete());
@@ -162,7 +181,7 @@ protected void openComponents(BigTableReader.Builder builder, SSTable.Owner owne
     public KeyReader buildKeyReader(TableMetrics tableMetrics) throws IOException
     {
         StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.HEADER, MetadataType.VALIDATION);
-        SerializationHeader header = statsComponent.serializationHeader(tableMetadataRef.getLocal());
+        SerializationHeader header = statsComponent.serializationHeader(descriptor, tableMetadataRef.getLocal());
         try (FileHandle indexFile = indexFileBuilder(null).complete())
         {
             return createKeyReader(indexFile, header, tableMetrics);
@@ -300,7 +319,7 @@ private FileHandle.Builder indexFileBuilder(IndexSummary indexSummary)
                                                            : OptionalInt.empty();
 
         if (indexFileBuilder == null)
-            indexFileBuilder = IndexComponent.fileBuilder(descriptor.fileFor(Components.PRIMARY_INDEX), ioOptions, chunkCache)
+            indexFileBuilder = IndexComponent.fileBuilder(descriptor, Components.PRIMARY_INDEX, ioOptions, chunkCache)
                                              .bufferSize(indexBufferSize.orElse(DiskOptimizationStrategy.MAX_BUFFER_SIZE));
 
         indexBufferSize.ifPresent(indexFileBuilder::bufferSize);
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
index b58dbc532eaf..192870e90e10 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
@@ -21,7 +21,6 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -46,21 +45,21 @@
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound;
 import org.apache.cassandra.db.rows.UnfilteredRowIterators;
-import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.Downsampling;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.IVerifier;
-import org.apache.cassandra.io.sstable.IndexInfo;
 import org.apache.cassandra.io.sstable.KeyReader;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
 import org.apache.cassandra.io.sstable.SSTableReadsListener.SelectionReason;
 import org.apache.cassandra.io.sstable.SSTableReadsListener.SkippingReason;
+import org.apache.cassandra.io.sstable.format.AbstractKeyFetcher;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
 import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
@@ -70,6 +69,7 @@
 import org.apache.cassandra.io.sstable.keycache.KeyCache;
 import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
 import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.FileUtils;
@@ -157,41 +157,6 @@ public KeyReader keyReader() throws IOException
         return BigTableKeyReader.create(ifile, rowIndexEntrySerializer);
     }
 
-    /**
-     * Direct I/O SSTableScanner over an iterator of bounds.
-     *
-     * @param boundsIterator the keys to cover
-     * @return A Scanner for seeking over the rows of the SSTable.
-     */
-    public ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> boundsIterator)
-    {
-        return BigTableScanner.getScanner(this, boundsIterator);
-    }
-
-    /**
-     * Direct I/O SSTableScanner over the full sstable.
-     *
-     * @return A Scanner for reading the full SSTable.
-     */
-    public ISSTableScanner getScanner()
-    {
-        return BigTableScanner.getScanner(this);
-    }
-
-    /**
-     * Direct I/O SSTableScanner over a defined collection of ranges of tokens.
-     *
-     * @param ranges the range of keys to cover
-     * @return A Scanner for seeking over the rows of the SSTable.
-     */
-    public ISSTableScanner getScanner(Collection<Range<Token>> ranges)
-    {
-        if (ranges != null)
-            return BigTableScanner.getScanner(this, ranges);
-        else
-            return getScanner();
-    }
-
     /**
      * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists.
      */
@@ -206,10 +171,10 @@ public DecoratedKey firstKeyBeyond(PartitionPosition token)
         if (ifile == null)
             return null;
 
-        String path = null;
+        File path = null;
         try (FileDataInput in = ifile.createReader(sampledPosition))
         {
-            path = in.getPath();
+            path = in.getFile();
             while (!in.isEOF())
             {
                 ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
@@ -292,7 +257,7 @@ public RowIndexEntry getRowIndexEntry(PartitionPosition key,
         if (searchOp == Operator.EQ)
         {
             assert key instanceof DecoratedKey; // EQ only make sense if the key is a valid row key
-            if (!isPresentInFilter((IFilter.FilterKey) key))
+            if (!inBloomFilter((DecoratedKey)key))
             {
                 notifySkipped(SkippingReason.BLOOM_FILTER, listener, operator, updateStats);
                 return null;
@@ -323,11 +288,13 @@ public RowIndexEntry getRowIndexEntry(PartitionPosition key,
         // next index position because the searched key can be greater the last key of the index interval checked if it
         // is lesser than the first key of next interval (and in that case we must return the position of the first key
         // of the next interval).
+        listener.onSSTablePartitionIndexAccessed(this);
+
         int i = 0;
-        String path = null;
+        File path = null;
         try (FileDataInput in = ifile.createReader(sampledPosition))
         {
-            path = in.getPath();
+            path = in.getFile();
             while (!in.isEOF())
             {
                 i++;
@@ -372,7 +339,7 @@ public RowIndexEntry getRowIndexEntry(PartitionPosition key,
                             {
                                 DecoratedKey keyInDisk = decorateKey(ByteBufferUtil.readWithShortLength(fdi));
                                 if (!keyInDisk.equals(key))
-                                    throw new AssertionError(String.format("%s != %s in %s", keyInDisk, key, fdi.getPath()));
+                                    throw new AssertionError(String.format("%s != %s in %s", keyInDisk, key, fdi.getFile()));
                             }
                         }
 
@@ -414,23 +381,16 @@ protected long getPosition(PartitionPosition key,
     }
 
     @Override
-    public DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException
+    public IKeyFetcher openKeyFetcher(boolean isForSASI)
     {
-        DecoratedKey key;
-        try (FileDataInput in = ifile.createReader(keyPositionFromSecondaryIndex))
+        return new AbstractKeyFetcher(isForSASI ? openIndexReader() : openDataReader())
         {
-            if (in.isEOF())
-                return null;
-
-            key = decorateKey(ByteBufferUtil.readWithShortLength(in));
-
-            // hint read path about key location if caching is enabled
-            // this saves index summary lookup and index file iteration which whould be pretty costly
-            // especially in presence of promoted column indexes
-            cacheKey(key, rowIndexEntrySerializer.deserialize(in));
-        }
-
-        return key;
+            @Override
+            public DecoratedKey readKey(RandomAccessReader reader) throws IOException
+            {
+                return decorateKey(ByteBufferUtil.readWithShortLength(reader));
+            }
+        };
     }
 
     @Override
@@ -519,7 +479,7 @@ public FileHandle getIndexFile()
     @Override
     public IVerifier getVerifier(ColumnFamilyStore cfs, OutputHandler outputHandler, boolean isOffline, IVerifier.Options options)
     {
-        Preconditions.checkArgument(cfs.metadata().equals(metadata()));
+        Preconditions.checkArgument(cfs == null || cfs.metadata().equals(metadata()));
         return new BigTableVerifier(cfs, this, outputHandler, isOffline, options);
     }
 
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
index 887d99784665..0fae19322395 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
@@ -18,11 +18,8 @@
 package org.apache.cassandra.io.sstable.format.big;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Iterator;
 
-import com.google.common.collect.Iterators;
-
 import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.PartitionPosition;
@@ -30,8 +27,6 @@
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTable;
@@ -50,12 +45,6 @@ public class BigTableScanner extends SSTableScanner<BigTableReader, RowIndexEntr
 
     private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
 
-    // Full scan of the sstables
-    public static ISSTableScanner getScanner(BigTableReader sstable)
-    {
-        return getScanner(sstable, Iterators.singletonIterator(fullRange(sstable)));
-    }
-
     public static ISSTableScanner getScanner(BigTableReader sstable,
                                              ColumnFilter columns,
                                              DataRange dataRange,
@@ -64,16 +53,6 @@ public static ISSTableScanner getScanner(BigTableReader sstable,
         return new BigTableScanner(sstable, columns, dataRange, makeBounds(sstable, dataRange).iterator(), listener);
     }
 
-    public static ISSTableScanner getScanner(BigTableReader sstable, Collection<Range<Token>> tokenRanges)
-    {
-        return getScanner(sstable, makeBounds(sstable, tokenRanges).iterator());
-    }
-
-    public static ISSTableScanner getScanner(BigTableReader sstable, Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
-    {
-        return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata()), null, rangeIterator, SSTableReadsListener.NOOP_LISTENER);
-    }
-
     private BigTableScanner(BigTableReader sstable,
                             ColumnFilter columns,
                             DataRange dataRange,
@@ -81,8 +60,19 @@ private BigTableScanner(BigTableReader sstable,
                             SSTableReadsListener listener)
     {
         super(sstable, columns, dataRange, rangeIterator, listener);
-        this.ifile = sstable.openIndexReader();
-        this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(sstable.descriptor.version, sstable.header, sstable.owner().map(SSTable.Owner::getMetrics).orElse(null));
+
+        RandomAccessReader ifile = null;
+        try
+        {
+            ifile = sstable.openIndexReader();
+            this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(sstable.descriptor.version, sstable.header, sstable.owner().map(SSTable.Owner::getMetrics).orElse(null));
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(ifile);
+            throw t;
+        }
+        this.ifile = ifile;
     }
 
     private void seekToCurrentRangeStart()
@@ -127,6 +117,11 @@ protected BigScanningIterator doCreateIterator()
         return new BigScanningIterator();
     }
 
+    @Override
+    public int level() {
+        return sstable.getSSTableLevel();
+    }
+
     protected class BigScanningIterator extends SSTableScanner<BigTableReader, RowIndexEntry, BigTableScanner.BigScanningIterator>.BaseKeyScanningIterator
     {
         private DecoratedKey nextKey;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java
index dc991f491fb1..dc832caa78c4 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java
@@ -23,7 +23,6 @@
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.rows.UnfilteredRowIterators;
@@ -31,6 +30,7 @@
 import org.apache.cassandra.io.sstable.SSTableRewriter;
 import org.apache.cassandra.io.sstable.format.SortedTableScrubber;
 import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -76,9 +76,9 @@ public BigTableScrubber(ColumnFamilyStore cfs,
     }
 
     @Override
-    protected UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+    protected UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, File file)
     {
-        return options.checkData && !isIndex ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+        return options.checkData && !isIndex ? UnfilteredRowIterators.withValidation(iter, file) : iter;
     }
 
     @Override
@@ -107,8 +107,7 @@ protected void scrubInternal(SSTableRewriter writer) throws IOException
 
         while (!dataFile.isEOF())
         {
-            if (scrubInfo.isStopRequested())
-                throw new CompactionInterruptedException(scrubInfo.getCompactionInfo());
+            scrubInfo.throwIfStopRequested();
 
             long partitionStart = dataFile.getFilePointer();
             outputHandler.debug("Reading row at %d", partitionStart);
@@ -117,8 +116,8 @@ protected void scrubInternal(SSTableRewriter writer) throws IOException
             try
             {
                 ByteBuffer raw = ByteBufferUtil.readWithShortLength(dataFile);
-                if (!cfs.metadata.getLocal().isIndex())
-                    cfs.metadata.getLocal().partitionKeyType.validate(raw);
+                if (!realm.metadataRef().getLocal().isIndex())
+                    realm.metadataRef().getLocal().partitionKeyType.validate(raw);
                 key = sstable.decorateKey(raw);
             }
             catch (Throwable th)
@@ -181,8 +180,8 @@ protected void scrubInternal(SSTableRewriter writer) throws IOException
                     key = sstable.decorateKey(currentIndexKey);
                     try
                     {
-                        if (!cfs.metadata.getLocal().isIndex())
-                            cfs.metadata.getLocal().partitionKeyType.validate(key.getKey());
+                        if (!realm.metadataRef().getLocal().isIndex())
+                            realm.metadataRef().getLocal().partitionKeyType.validate(key.getKey());
                         dataFile.seek(dataStartFromIndex);
 
                         if (tryAppend(prevKey, key, writer))
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java
index 70df3e1c0a1f..86d242866655 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java
@@ -126,7 +126,7 @@ private String dateString(long time)
 
     private void deserializeIndexSummary(SSTableReader sstable) throws IOException
     {
-        IndexSummaryComponent summaryComponent = IndexSummaryComponent.load(sstable.descriptor.fileFor(Components.SUMMARY), cfs.metadata());
+        IndexSummaryComponent summaryComponent = IndexSummaryComponent.load(sstable.descriptor.fileFor(Components.SUMMARY), sstable.metadata());
         if (summaryComponent == null)
             throw new NoSuchFileException("Index summary component of sstable " + sstable.descriptor + " is missing");
         FileUtils.closeQuietly(summaryComponent.indexSummary);
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index bf8bb79ce276..0ad85138076c 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -251,7 +251,7 @@ protected IndexWriter(Builder b, SequentialWriter dataWriter)
             super(b);
             this.rowIndexEntrySerializer = b.getRowIndexEntrySerializer();
             writer = new SequentialWriter(b.descriptor.fileFor(Components.PRIMARY_INDEX), b.getIOOptions().writerOptions);
-            builder = IndexComponent.fileBuilder(Components.PRIMARY_INDEX, b).withMmappedRegionsCache(b.getMmappedRegionsCache());
+            builder = IndexComponent.fileBuilder(Components.PRIMARY_INDEX, b, b.operationType).withMmappedRegionsCache(b.getMmappedRegionsCache());
             summary = new IndexSummaryBuilder(b.getKeyCount(), b.getTableMetadataRef().getLocal().params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL);
             // register listeners to be alerted when the data files are flushed
             writer.setPostFlushListener(summary::markIndexSynced);
@@ -279,7 +279,7 @@ public void append(DecoratedKey key, RowIndexEntry indexEntry, long dataEnd, Byt
             }
             catch (IOException e)
             {
-                throw new FSWriteError(e, writer.getPath());
+                throw new FSWriteError(e, writer.getFile());
             }
             long indexEnd = writer.position();
 
@@ -314,7 +314,7 @@ protected void doPrepare()
             // truncate index file
             long position = writer.position();
             writer.prepareToCommit();
-            FileUtils.truncate(writer.getPath(), position);
+            FileUtils.truncate(writer.getFile(), position);
 
             // save summary
             summary.prepareToCommit();
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java b/src/java/org/apache/cassandra/io/sstable/format/big/IndexInfo.java
similarity index 98%
rename from src/java/org/apache/cassandra/io/sstable/IndexInfo.java
rename to src/java/org/apache/cassandra/io/sstable/format/big/IndexInfo.java
index 350a98eb9eaf..6399e9e60c13 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/IndexInfo.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.io.sstable;
+package org.apache.cassandra.io.sstable.format.big;
 
 import java.io.IOException;
 import java.util.List;
@@ -28,7 +28,6 @@
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.io.ISerializer;
 import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.ObjectSizes;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java b/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java
index 754b34b06571..50326970aa30 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java
@@ -25,7 +25,6 @@
 import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.ClusteringPrefix;
 import org.apache.cassandra.io.sstable.AbstractSSTableIterator;
-import org.apache.cassandra.io.sstable.IndexInfo;
 import org.apache.cassandra.io.util.DataPosition;
 import org.apache.cassandra.io.util.FileHandle;
 
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java b/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java
index 9d2ab6e93a9a..5165c48db563 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java
@@ -33,7 +33,6 @@
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.ISerializer;
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
-import org.apache.cassandra.io.sstable.IndexInfo;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
 import org.apache.cassandra.io.util.DataInputPlus;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java
index e7703c6a0612..11944f978475 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import com.google.common.base.Preconditions;
@@ -29,6 +30,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
@@ -53,6 +55,7 @@
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 /**
  * Bigtable format with trie indices. See BTIFormat.md for the format documentation.
@@ -121,6 +124,11 @@ public static boolean is(SSTableFormat<?, ?> format)
         return format.name().equals(NAME);
     }
 
+    public static BtiFormat getInstance()
+    {
+        return (BtiFormat) Objects.requireNonNull(DatabaseDescriptor.getSSTableFormats().get(NAME), "Unknown SSTable format: " + NAME);
+    }
+
     public static boolean isSelected()
     {
         return is(DatabaseDescriptor.getSelectedSSTableFormat());
@@ -213,7 +221,7 @@ public void deleteOrphanedComponents(Descriptor descriptor, Set<Component> compo
 
     private void delete(Descriptor desc, List<Component> components)
     {
-        logger.info("Deleting sstable: {}", desc);
+        logger.debug("Deleting sstable: {}", desc);
 
         if (components.remove(SSTableFormat.Components.DATA))
             components.add(0, SSTableFormat.Components.DATA); // DATA component should be first
@@ -256,7 +264,7 @@ public SSTableReaderLoadingBuilder<BtiTableReader, BtiTableReader.Builder> loadi
         @Override
         public Pair<DecoratedKey, DecoratedKey> readKeyRange(Descriptor descriptor, IPartitioner partitioner) throws IOException
         {
-            return PartitionIndex.readFirstAndLastKey(descriptor.fileFor(Components.PARTITION_INDEX), partitioner);
+            return PartitionIndex.readFirstAndLastKey(descriptor, Components.PARTITION_INDEX, partitioner, descriptor.version.getByteComparableVersion());
         }
 
         @Override
@@ -286,8 +294,32 @@ public long estimateSize(SSTableWriter.SSTableSizeParameters parameters)
 
     static class BtiVersion extends Version
     {
-        public static final String current_version = "da";
-        public static final String earliest_supported_version = "da";
+        private static final Logger logger = LoggerFactory.getLogger(BtiVersion.class);
+
+        public static final String current_version = CassandraRelevantProperties.TRIE_INDEX_FORMAT_VERSION.getString();
+
+        static
+        {
+            logger.info("Trie index format current version: {}", current_version);
+        }
+
+        public static final String earliest_supported_version = "aa";
+
+        // aa (DSE 6.0): trie index format
+        // ab (DSE pre-6.8): ILLEGAL - handled as 'b' (predates 'ba'). Pre-GA "LABS" releases of DSE 6.8 used this
+        //                   sstable version.
+        // ac (DSE 6.0.11, 6.7.6): corrected sstable min/max clustering (DB-3691/CASSANDRA-14861)
+        // ad (DSE 6.0.14, 6.7.11): added hostId of the node from which the sstable originated (DB-4629)
+        // b  (DSE early 6.8 "LABS") has some of 6.8 features but not all
+        // ba (DSE 6.8): encrypted indices and metadata
+        //               new BloomFilter serialization format
+        //               add incremental NodeSync information to metadata
+        //               improved min/max clustering representation
+        //               presence marker for partition level deletions
+        // bb (DSE 6.8.5): added hostId of the node from which the sstable originated (DB-4629)
+        // ca (DSE-DB aka Stargazer based on OSS 4.0): bb fields without maxColumnValueLengths + all OSS fields
+        // cb (OSS 5.0): token space coverage
+        // cc : added explicitly frozen tuples in header, non-frozen UDT columns dropping support
 
         // versions aa-cz are not supported in OSS
         // da (5.0): initial version of the BIT format
@@ -295,14 +327,62 @@ static class BtiVersion extends Version
 
         private final boolean isLatestVersion;
 
+        /**
+         * DB-2648/CASSANDRA-9067: DSE 6.8/OSS 4.0 bloom filter representation changed (bitset data is no longer stored
+         * as BIG_ENDIAN longs, which avoids some redundant bit twiddling).
+         */
+        private final boolean hasOldBfFormat;
+        private final boolean hasAccurateLegacyMinMax;
+        private final boolean hasOriginatingHostId;
+        private final boolean hasMaxColumnValueLengths;
+        private final boolean hasImprovedMinMax;
+        private final boolean hasLegacyMinMax;
+        private final boolean hasZeroCopyMetadata;
+        private final boolean hasIncrementalNodeSyncMetadata;
+        private final boolean hasIsTransient;
+        private final boolean hasTokenSpaceCoverage;
+        private final boolean hasMisplacedPartitionLevelDeletionsPresenceMarker;
+
+
         private final int correspondingMessagingVersion;
+        private final ByteComparable.Version byteComparableVersion;
+        private final boolean hasPartitionLevelDeletionsPresenceMarker;
+        private final boolean hasKeyRange;
+        private final boolean hasUIntDeletionTime;
+        private final boolean hasImplicitlyFrozenTuples;
 
         BtiVersion(BtiFormat format, String version)
         {
             super(format, version);
 
+            boolean dOrLater = version.compareTo("d") >= 0;
+            boolean cOrLater = dOrLater || version.startsWith("c");
+            boolean bOrLater = cOrLater || version.startsWith("b");
+            boolean aOrLater = bOrLater || version.startsWith("a");
+
             isLatestVersion = version.compareTo(current_version) == 0;
             correspondingMessagingVersion = MessagingService.VERSION_50;
+            byteComparableVersion = version.compareTo("da") >= 0 ? ByteComparable.Version.OSS50
+                                                                 : version.compareTo("ca") >= 0 ? ByteComparable.Version.OSS41
+                                                                                                : ByteComparable.Version.LEGACY;
+            hasOldBfFormat = aOrLater && !bOrLater;
+            hasImprovedMinMax = bOrLater;
+            hasLegacyMinMax = aOrLater && !bOrLater;
+            hasAccurateLegacyMinMax = !bOrLater && version.compareTo("ac") >= 0;
+            hasOriginatingHostId = bOrLater && version.compareTo("bb") >= 0 || !bOrLater && version.compareTo("ad") >= 0;
+            hasIsTransient = cOrLater;
+            hasTokenSpaceCoverage = version.compareTo("cb") >= 0;
+            hasMisplacedPartitionLevelDeletionsPresenceMarker = bOrLater && !dOrLater;
+            hasPartitionLevelDeletionsPresenceMarker = dOrLater;
+            hasKeyRange = dOrLater;
+            hasUIntDeletionTime = dOrLater;
+
+            hasMaxColumnValueLengths = bOrLater && !cOrLater; // DSE only field
+            hasZeroCopyMetadata = bOrLater && !cOrLater; // DSE only field
+            hasIncrementalNodeSyncMetadata = bOrLater && !cOrLater; // DSE only field
+
+            // TODO figure out which versions support that
+            hasImplicitlyFrozenTuples = version.compareTo("cc") < 0; // we don't know if what DA is going to be eventually, but it is almost certain it will not include explicitly frozen tuples
         }
 
         @Override
@@ -341,10 +421,17 @@ public boolean hasPendingRepair()
             return true;
         }
 
+        // this field is not present in DSE
         @Override
         public boolean hasIsTransient()
         {
-            return true;
+            return hasIsTransient;
+        }
+
+        @Override
+        public ByteComparable.Version getByteComparableVersion()
+        {
+            return byteComparableVersion;
         }
 
         @Override
@@ -356,47 +443,53 @@ public boolean hasMetadataChecksum()
         @Override
         public boolean hasOldBfFormat()
         {
-            return false;
+            return hasOldBfFormat;
         }
 
         @Override
         public boolean hasAccurateMinMax()
         {
-            return true;
+            return hasAccurateLegacyMinMax;
         }
 
         public boolean hasLegacyMinMax()
         {
-            return false;
+            return hasLegacyMinMax;
         }
 
         @Override
         public boolean hasOriginatingHostId()
         {
-            return true;
+            return hasOriginatingHostId;
         }
 
         @Override
         public boolean hasImprovedMinMax() {
-            return true;
+            return hasImprovedMinMax;
         }
 
         @Override
         public boolean hasTokenSpaceCoverage()
         {
-            return true;
+            return hasTokenSpaceCoverage;
         }
 
         @Override
         public boolean hasPartitionLevelDeletionsPresenceMarker()
         {
-            return true;
+            return hasPartitionLevelDeletionsPresenceMarker;
+        }
+
+        @Override
+        public boolean hasMisplacedPartitionLevelDeletionsPresenceMarker()
+        {
+            return hasMisplacedPartitionLevelDeletionsPresenceMarker;
         }
 
         @Override
         public boolean hasKeyRange()
         {
-            return true;
+            return hasKeyRange;
         }
 
         @Override
@@ -414,9 +507,32 @@ public boolean isCompatibleForStreaming()
         @Override
         public boolean hasUIntDeletionTime()
         {
-            return true;
+            return hasUIntDeletionTime;
         }
 
+        @Override
+        public boolean hasZeroCopyMetadata()
+        {
+            return hasZeroCopyMetadata;
+        }
+
+        @Override
+        public boolean hasIncrementalNodeSyncMetadata()
+        {
+            return hasIncrementalNodeSyncMetadata;
+        }
+
+        @Override
+        public boolean hasMaxColumnValueLengths()
+        {
+            return hasMaxColumnValueLengths;
+        }
+
+        @Override
+        public boolean hasImplicitlyFrozenTuples()
+        {
+            return hasImplicitlyFrozenTuples;
+        }
     }
 
     private static class BtiTableSpecificMetricsProviders implements MetricsProviders
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
index c5571e7fbbe3..80c352f7b52a 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
@@ -22,7 +22,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -45,12 +44,13 @@
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
 import org.apache.cassandra.io.sstable.IVerifier;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
 import org.apache.cassandra.io.sstable.SSTableReadsListener.SelectionReason;
 import org.apache.cassandra.io.sstable.SSTableReadsListener.SkippingReason;
+import org.apache.cassandra.io.sstable.format.AbstractKeyFetcher;
 import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
 import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.FileHandle;
@@ -108,7 +108,7 @@ protected List<AutoCloseable> setupInstance(boolean trackHotness)
      */
     protected boolean filterFirst()
     {
-        return openReason == OpenReason.MOVED_START;
+        return openReason == OpenReason.MOVED_START || sstableMetadata.zeroCopyMetadata.exists();
     }
 
     /**
@@ -118,7 +118,7 @@ protected boolean filterFirst()
      */
     protected boolean filterLast()
     {
-        return openReason == OpenReason.EARLY && partitionIndex instanceof PartitionIndexEarly;
+        return openReason == OpenReason.EARLY && partitionIndex instanceof PartitionIndexEarly || sstableMetadata.zeroCopyMetadata.exists();
     }
 
     public long estimatedKeys()
@@ -211,15 +211,16 @@ private TrieIndexEntry retrieveEntryIfAcceptable(Operator searchOp, PartitionPos
     }
 
     @Override
-    public DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException
+    public IKeyFetcher openKeyFetcher(boolean isForSASI)
     {
-        try (RandomAccessReader reader = openDataReader())
+        return new AbstractKeyFetcher(openDataReader())
         {
-            reader.seek(keyPositionFromSecondaryIndex);
-            if (reader.isEOF())
-                return null;
-            return decorateKey(ByteBufferUtil.readWithShortLength(reader));
-        }
+            @Override
+            public DecoratedKey readKey(RandomAccessReader reader) throws IOException
+            {
+                return decorateKey(ByteBufferUtil.readWithShortLength(reader));
+            }
+        };
     }
 
     TrieIndexEntry getExactPosition(DecoratedKey dk,
@@ -232,12 +233,14 @@ TrieIndexEntry getExactPosition(DecoratedKey dk,
             return null;
         }
 
-        if (!isPresentInFilter(dk))
+        if (!inBloomFilter(dk))
         {
             notifySkipped(SkippingReason.BLOOM_FILTER, listener, EQ, updateStats);
             return null;
         }
 
+        listener.onSSTablePartitionIndexAccessed(this);
+
         try (PartitionIndex.Reader reader = partitionIndex.openReader())
         {
             long indexPos = reader.exactCandidate(dk);
@@ -382,27 +385,6 @@ public UnfilteredRowIterator rowIterator(FileDataInput dataFileInput,
             return new SSTableIterator(this, dataFileInput, key, indexEntry, slices, selectedColumns, rowIndexFile);
     }
 
-    @Override
-    public ISSTableScanner getScanner()
-    {
-        return BtiTableScanner.getScanner(this);
-    }
-
-    @Override
-    public ISSTableScanner getScanner(Collection<Range<Token>> ranges)
-    {
-        if (ranges != null)
-            return BtiTableScanner.getScanner(this, ranges);
-        else
-            return getScanner();
-    }
-
-    @Override
-    public ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
-    {
-        return BtiTableScanner.getScanner(this, rangeIterator);
-    }
-
     @VisibleForTesting
     @Override
     public BtiTableReader cloneAndReplace(IFilter filter)
@@ -490,7 +472,7 @@ public UnfilteredPartitionIterator partitionIterator(ColumnFilter columnFilter,
     @Override
     public IVerifier getVerifier(ColumnFamilyStore cfs, OutputHandler outputHandler, boolean isOffline, IVerifier.Options options)
     {
-        Preconditions.checkArgument(cfs.metadata().equals(metadata()));
+        Preconditions.checkArgument(cfs == null || cfs.metadata().equals(metadata()));
         return new BtiTableVerifier(cfs, this, outputHandler, isOffline, options);
     }
 
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java
index fa408adc5d0e..00f601c5da35 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java
@@ -32,10 +32,14 @@
 import org.apache.cassandra.io.sstable.format.FilterComponent;
 import org.apache.cassandra.io.sstable.format.SortedTableReaderLoadingBuilder;
 import org.apache.cassandra.io.sstable.format.StatsComponent;
+import org.apache.cassandra.io.sstable.format.TOCComponent;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.sstable.format.bti.BtiFormat.Components;
 import org.apache.cassandra.io.sstable.metadata.MetadataType;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.io.sstable.metadata.ZeroCopyMetadata;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.utils.FilterFactory;
@@ -68,8 +72,8 @@ private KeyReader createKeyReader(StatsMetadata statsMetadata) throws IOExceptio
     {
         checkNotNull(statsMetadata);
 
-        try (PartitionIndex index = PartitionIndex.load(partitionIndexFileBuilder(), tableMetadataRef.getLocal().partitioner, false);
-             CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components);
+        try (PartitionIndex index = PartitionIndex.load(partitionIndexFileBuilder(), tableMetadataRef.getLocal().partitioner, false, descriptor.version.getByteComparableVersion());
+             CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components, statsMetadata.zeroCopyMetadata);
              FileHandle dFile = dataFileBuilder(statsMetadata).withCompressionMetadata(compressionMetadata)
                                                               .withCrcCheckChance(() -> tableMetadataRef.getLocal().params.crcCheckChance)
                                                               .complete();
@@ -89,7 +93,7 @@ protected void openComponents(BtiTableReader.Builder builder, SSTable.Owner owne
         try
         {
             StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.VALIDATION, MetadataType.HEADER);
-            builder.setSerializationHeader(statsComponent.serializationHeader(builder.getTableMetadataRef().getLocal()));
+            builder.setSerializationHeader(statsComponent.serializationHeader(descriptor, builder.getTableMetadataRef().getLocal()));
             checkArgument(!online || builder.getSerializationHeader() != null);
 
             builder.setStatsMetadata(statsComponent.statsMetadata());
@@ -106,8 +110,17 @@ protected void openComponents(BtiTableReader.Builder builder, SSTable.Owner owne
                 IFilter filter = buildBloomFilter(statsComponent.statsMetadata());
                 builder.setFilter(filter);
                 FilterComponent.save(filter, descriptor, false);
+                if (validationMetadata.bloomFilterFPChance != tableMetadataRef.getLocal().params.bloomFilterFpChance)
+                {
+                    StatsComponent.load(descriptor, MetadataType.values())
+                                  .with(validationMetadata.withBloomFilterFPChance(tableMetadataRef.getLocal().params.bloomFilterFpChance))
+                                  .save(descriptor);
+                }
+                if (descriptor.fileFor(Components.FILTER).exists())
+                    TOCComponent.maybeAdd(descriptor, BigFormat.Components.FILTER);
             }
 
+
             if (builder.getFilter() == null)
                 builder.setFilter(FilterFactory.AlwaysPresent);
 
@@ -123,7 +136,7 @@ protected void openComponents(BtiTableReader.Builder builder, SSTable.Owner owne
 
             if (builder.getComponents().contains(Components.PARTITION_INDEX))
             {
-                builder.setPartitionIndex(openPartitionIndex(!builder.getFilter().isInformative()));
+                builder.setPartitionIndex(openPartitionIndex(!builder.getFilter().isInformative(), statsComponent.statsMetadata().zeroCopyMetadata));
                 if (builder.getFirst() == null || builder.getLast() == null)
                 {
                     builder.setFirst(builder.getPartitionIndex().firstKey());
@@ -131,7 +144,7 @@ protected void openComponents(BtiTableReader.Builder builder, SSTable.Owner owne
                 }
             }
 
-            try (CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components))
+            try (CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components, statsComponent.statsMetadata().zeroCopyMetadata))
             {
                 builder.setDataFile(dataFileBuilder(builder.getStatsMetadata())
                                     .withCompressionMetadata(compressionMetadata)
@@ -142,7 +155,7 @@ protected void openComponents(BtiTableReader.Builder builder, SSTable.Owner owne
         catch (IOException | RuntimeException | Error ex)
         {
             // in case of failure, close only those components which have been opened in this try-catch block
-            Throwables.closeAndAddSuppressed(ex, builder.getPartitionIndex(), builder.getRowIndexFile(), builder.getDataFile(), builder.getFilter());
+            Throwables.closeNonNullAndAddSuppressed(ex, builder.getPartitionIndex(), builder.getRowIndexFile(), builder.getDataFile(), builder.getFilter());
             throw ex;
         }
     }
@@ -165,18 +178,18 @@ private IFilter buildBloomFilter(StatsMetadata statsMetadata) throws IOException
         }
         catch (IOException | RuntimeException | Error ex)
         {
-            Throwables.closeAndAddSuppressed(ex, bf);
+            Throwables.closeNonNullAndAddSuppressed(ex, bf);
             throw ex;
         }
 
         return bf;
     }
 
-    private PartitionIndex openPartitionIndex(boolean preload) throws IOException
+    private PartitionIndex openPartitionIndex(boolean preload, ZeroCopyMetadata zeroCopyMetadata) throws IOException
     {
         try (FileHandle indexFile = partitionIndexFileBuilder().complete())
         {
-            return PartitionIndex.load(indexFile, tableMetadataRef.getLocal().partitioner, preload);
+            return PartitionIndex.load(indexFile, tableMetadataRef.getLocal().partitioner, preload, zeroCopyMetadata, descriptor.version.getByteComparableVersion());
         }
         catch (IOException ex)
         {
@@ -190,7 +203,7 @@ private FileHandle.Builder rowIndexFileBuilder()
         assert rowIndexFileBuilder == null || rowIndexFileBuilder.file.equals(descriptor.fileFor(Components.ROW_INDEX));
 
         if (rowIndexFileBuilder == null)
-            rowIndexFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.ROW_INDEX));
+            rowIndexFileBuilder = StorageProvider.instance.fileHandleBuilderFor(descriptor, Components.ROW_INDEX);
 
         rowIndexFileBuilder.withChunkCache(chunkCache);
         rowIndexFileBuilder.mmapped(ioOptions.indexDiskAccessMode);
@@ -203,7 +216,7 @@ private FileHandle.Builder partitionIndexFileBuilder()
         assert partitionIndexFileBuilder == null || partitionIndexFileBuilder.file.equals(descriptor.fileFor(Components.PARTITION_INDEX));
 
         if (partitionIndexFileBuilder == null)
-            partitionIndexFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.PARTITION_INDEX));
+            partitionIndexFileBuilder = StorageProvider.instance.fileHandleBuilderFor(descriptor, Components.PARTITION_INDEX);
 
         partitionIndexFileBuilder.withChunkCache(chunkCache);
         partitionIndexFileBuilder.mmapped(ioOptions.indexDiskAccessMode);
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScanner.java
index a9f862c68b50..1460bfbb2272 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScanner.java
@@ -19,11 +19,8 @@
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Iterator;
 
-import com.google.common.collect.Iterators;
-
 import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.PartitionPosition;
@@ -31,20 +28,12 @@
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
 import org.apache.cassandra.io.sstable.format.SSTableScanner;
 import org.apache.cassandra.io.util.FileUtils;
 
 public class BtiTableScanner extends SSTableScanner<BtiTableReader, TrieIndexEntry, BtiTableScanner.BtiScanningIterator>
 {
-    // Full scan of the sstables
-    public static BtiTableScanner getScanner(BtiTableReader sstable)
-    {
-        return getScanner(sstable, Iterators.singletonIterator(fullRange(sstable)));
-    }
-
     public static BtiTableScanner getScanner(BtiTableReader sstable,
                                              ColumnFilter columns,
                                              DataRange dataRange,
@@ -53,16 +42,6 @@ public static BtiTableScanner getScanner(BtiTableReader sstable,
         return new BtiTableScanner(sstable, columns, dataRange, makeBounds(sstable, dataRange).iterator(), listener);
     }
 
-    public static BtiTableScanner getScanner(BtiTableReader sstable, Collection<Range<Token>> tokenRanges)
-    {
-        return getScanner(sstable, makeBounds(sstable, tokenRanges).iterator());
-    }
-
-    public static BtiTableScanner getScanner(BtiTableReader sstable, Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
-    {
-        return new BtiTableScanner(sstable, ColumnFilter.all(sstable.metadata()), null, rangeIterator, SSTableReadsListener.NOOP_LISTENER);
-    }
-
     private BtiTableScanner(BtiTableReader sstable,
                             ColumnFilter columns,
                             DataRange dataRange,
@@ -83,6 +62,11 @@ protected BtiScanningIterator doCreateIterator()
         return new BtiScanningIterator();
     }
 
+    @Override
+    public int level() {
+        return sstable.getSSTableLevel();
+    }
+
     protected class BtiScanningIterator extends SSTableScanner<BtiTableReader, TrieIndexEntry, BtiTableScanner.BtiScanningIterator>.BaseKeyScanningIterator implements Closeable
     {
         private PartitionIterator iterator;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScrubber.java
index 238ed7e7de58..1f6cbc8ec475 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScrubber.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScrubber.java
@@ -24,7 +24,6 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -33,6 +32,7 @@
 import org.apache.cassandra.io.sstable.SSTableRewriter;
 import org.apache.cassandra.io.sstable.format.SortedTableScrubber;
 import org.apache.cassandra.io.sstable.format.bti.BtiFormat.Components;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
@@ -52,7 +52,7 @@ public BtiTableScrubber(ColumnFamilyStore cfs,
     {
         super(cfs, transaction, outputHandler, options);
 
-        boolean hasIndexFile = sstable.getComponents().contains(Components.PARTITION_INDEX);
+        boolean hasIndexFile = sstable.components().contains(Components.PARTITION_INDEX);
         this.isIndex = cfs.isIndex();
         this.partitionKeyType = cfs.metadata.get().partitionKeyType;
         if (!hasIndexFile)
@@ -87,19 +87,18 @@ private ScrubPartitionIterator openIndexIterator()
     }
 
     @Override
-    protected UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+    protected UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, File file)
     {
-        return options.checkData && !isIndex ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+        return options.checkData && !isIndex ? UnfilteredRowIterators.withValidation(iter, file) : iter;
     }
 
     @Override
     public void scrubInternal(SSTableRewriter writer)
     {
-        if (indexAvailable() && indexIterator.dataPosition() != 0)
+        if (indexAvailable() && indexIterator.dataPosition() != sstable.getDataFileSliceDescriptor().dataStart)
         {
-            outputHandler.warn("First position reported by index should be 0, was " +
-                               indexIterator.dataPosition() +
-                               ", continuing without index.");
+            outputHandler.warn("First position reported by index should be {}, was {}, continuing without index.",
+                               sstable.getDataFileSliceDescriptor().dataStart, indexIterator.dataPosition());
             indexIterator.close();
             indexIterator = null;
         }
@@ -108,8 +107,7 @@ public void scrubInternal(SSTableRewriter writer)
 
         while (!dataFile.isEOF())
         {
-            if (scrubInfo.isStopRequested())
-                throw new CompactionInterruptedException(scrubInfo.getCompactionInfo());
+            scrubInfo.throwIfStopRequested();
 
             // position in a data file where the partition starts
             long dataStart = dataFile.getFilePointer();
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableVerifier.java
index 6125af805b03..d6f76eea8c90 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableVerifier.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableVerifier.java
@@ -33,6 +33,12 @@ public BtiTableVerifier(ColumnFamilyStore cfs, BtiTableReader sstable, OutputHan
 
     protected void verifyPartition(DecoratedKey key, UnfilteredRowIterator iterator)
     {
+        if (options.validateAllRows)
+        {
+            // validate all rows and cells
+            while (iterator.hasNext())
+                iterator.next();
+        }
         // The trie writers abort if supplied with badly ordered or duplicate row keys. Verification is not necessary.
         // no-op, just open and close partition.
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
index 7aad38511f16..53c6bcfe90bd 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
@@ -172,10 +172,10 @@ protected static class IndexWriter extends SortedTableWriter.AbstractIndexWriter
         {
             super(b);
             rowIndexWriter = new SequentialWriter(descriptor.fileFor(Components.ROW_INDEX), b.getIOOptions().writerOptions);
-            rowIndexFHBuilder = IndexComponent.fileBuilder(Components.ROW_INDEX, b).withMmappedRegionsCache(b.getMmappedRegionsCache());
+            rowIndexFHBuilder = IndexComponent.fileBuilder(Components.ROW_INDEX, b, b.operationType).withMmappedRegionsCache(b.getMmappedRegionsCache());
             partitionIndexWriter = new SequentialWriter(descriptor.fileFor(Components.PARTITION_INDEX), b.getIOOptions().writerOptions);
-            partitionIndexFHBuilder = IndexComponent.fileBuilder(Components.PARTITION_INDEX, b).withMmappedRegionsCache(b.getMmappedRegionsCache());
-            partitionIndex = new PartitionIndexBuilder(partitionIndexWriter, partitionIndexFHBuilder);
+            partitionIndexFHBuilder = IndexComponent.fileBuilder(Components.PARTITION_INDEX, b, b.operationType).withMmappedRegionsCache(b.getMmappedRegionsCache());
+            partitionIndex = new PartitionIndexBuilder(partitionIndexWriter, partitionIndexFHBuilder, descriptor.version.getByteComparableVersion());
             // register listeners to be alerted when the data files are flushed
             partitionIndexWriter.setPostFlushListener(partitionIndex::markPartitionIndexSynced);
             rowIndexWriter.setPostFlushListener(partitionIndex::markRowIndexSynced);
@@ -266,7 +266,7 @@ PartitionIndex completedPartitionIndex()
             partitionIndexFHBuilder.withLengthOverride(0);
             try
             {
-                return PartitionIndex.load(partitionIndexFHBuilder, metadata.getLocal().partitioner, false);
+                return PartitionIndex.load(partitionIndexFHBuilder, metadata.getLocal().partitioner, false, descriptor.version.getByteComparableVersion());
             }
             catch (IOException e)
             {
@@ -382,7 +382,7 @@ protected BtiTableWriter buildInternal(LifecycleNewTracker lifecycleNewTracker,
             }
             catch (RuntimeException | Error ex)
             {
-                Throwables.closeAndAddSuppressed(ex, mmappedRegionsCache);
+                Throwables.closeNonNullAndAddSuppressed(ex, mmappedRegionsCache);
                 throw ex;
             }
             finally
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
index 12d35d7cd6bd..742937df5a5b 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
@@ -28,13 +28,16 @@
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.metadata.ZeroCopyMetadata;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.io.tries.SerializationNode;
 import org.apache.cassandra.io.tries.TrieNode;
 import org.apache.cassandra.io.tries.TrieSerializer;
 import org.apache.cassandra.io.tries.ValueIterator;
 import org.apache.cassandra.io.tries.Walker;
 import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.PageAware;
@@ -42,6 +45,7 @@
 import org.apache.cassandra.io.util.SizedInts;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.concurrent.SharedCloseable;
@@ -73,24 +77,37 @@ public class PartitionIndex implements SharedCloseable
     private final DecoratedKey first;
     private final DecoratedKey last;
     private final long root;
+    /** Key to apply when a caller asks for a full index. Normally null, but set to first for zero-copied indexes. */
+    private final DecoratedKey filterFirst;
+    /** Key to apply when a caller asks for a full index. Normally null, but set to last for zero-copied indexes. */
+    private final DecoratedKey filterLast;
+
+    public final ByteComparable.Version version;
 
     public static final long NOT_FOUND = Long.MIN_VALUE;
     public static final int FOOTER_LENGTH = 3 * 8;
     private static final int FLAG_HAS_HASH_BYTE = 8;
 
-    @VisibleForTesting
-    public PartitionIndex(FileHandle fh, long trieRoot, long keyCount, DecoratedKey first, DecoratedKey last)
+    public PartitionIndex(FileHandle fh, long trieRoot, long keyCount, DecoratedKey first, DecoratedKey last, ByteComparable.Version version)
+    {
+        this(fh, trieRoot, keyCount, first, last, null, null, version);
+    }
+
+    public PartitionIndex(FileHandle fh, long trieRoot, long keyCount, DecoratedKey first, DecoratedKey last, DecoratedKey filterFirst, DecoratedKey filterLast, ByteComparable.Version version)
     {
         this.keyCount = keyCount;
         this.fh = fh.sharedCopy();
         this.first = first;
         this.last = last;
         this.root = trieRoot;
+        this.filterFirst = filterFirst;
+        this.filterLast = filterLast;
+        this.version = version;
     }
 
     private PartitionIndex(PartitionIndex src)
     {
-        this(src.fh, src.root, src.keyCount, src.first, src.last);
+        this(src.fh, src.root, src.keyCount, src.first, src.last, src.filterFirst, src.filterLast, src.version);
     }
 
     static class Payload
@@ -169,23 +186,37 @@ public void addTo(Ref.IdentityCollection identities)
 
     public static PartitionIndex load(FileHandle.Builder fhBuilder,
                                       IPartitioner partitioner,
-                                      boolean preload) throws IOException
+                                      boolean preload,
+                                      ByteComparable.Version version) throws IOException
+    {
+        return load(fhBuilder, partitioner, preload, null, version);
+    }
+
+    public static PartitionIndex load(FileHandle.Builder fhBuilder,
+                                      IPartitioner partitioner,
+                                      boolean preload,
+                                      ZeroCopyMetadata zeroCopyMetadata,
+                                      ByteComparable.Version version) throws IOException
     {
         try (FileHandle fh = fhBuilder.complete())
         {
-            return load(fh, partitioner, preload);
+            return load(fh, partitioner, preload, zeroCopyMetadata, version);
         }
     }
 
-    public static Pair<DecoratedKey, DecoratedKey> readFirstAndLastKey(File file, IPartitioner partitioner) throws IOException
+    public static Pair<DecoratedKey, DecoratedKey> readFirstAndLastKey(Descriptor descriptor, Component component, IPartitioner partitioner, ByteComparable.Version version) throws IOException
     {
-        try (PartitionIndex index = load(new FileHandle.Builder(file), partitioner, false))
+        try (PartitionIndex index = load(StorageProvider.instance.fileHandleBuilderFor(descriptor, component), partitioner, false, version))
         {
             return Pair.create(index.firstKey(), index.lastKey());
         }
     }
 
-    public static PartitionIndex load(FileHandle fh, IPartitioner partitioner, boolean preload) throws IOException
+    public static PartitionIndex load(FileHandle fh,
+                                      IPartitioner partitioner,
+                                      boolean preload,
+                                      ZeroCopyMetadata zeroCopyMetadata,
+                                      ByteComparable.Version version) throws IOException
     {
         try (FileDataInput rdr = fh.createReader(fh.dataLength() - FOOTER_LENGTH))
         {
@@ -207,7 +238,26 @@ public static PartitionIndex load(FileHandle fh, IPartitioner partitioner, boole
                 logger.trace("Checksum {}", csum);      // Note: trace is required so that reads aren't optimized away.
             }
 
-            return new PartitionIndex(fh, root, keyCount, first, last);
+            DecoratedKey filterFirst = null;
+            DecoratedKey filterLast = null;
+
+            // Adjust keys estimate plus bounds if ZeroCopy, otherwise we would see un-owned data from the index:
+            if (zeroCopyMetadata != null && zeroCopyMetadata.exists() && partitioner != null)
+            {
+                DecoratedKey newFirst = partitioner.decorateKey(zeroCopyMetadata.firstKey());
+                DecoratedKey newLast = partitioner.decorateKey(zeroCopyMetadata.lastKey());
+                if (!newFirst.equals(first))
+                {
+                    filterFirst = first = newFirst;
+                }
+                if (!newLast.equals(last))
+                {
+                    filterLast = last = newLast;
+                }
+                keyCount = zeroCopyMetadata.estimatedKeys();
+            }
+
+            return new PartitionIndex(fh, root, keyCount, first, last, filterFirst, filterLast, version);
         }
     }
 
@@ -225,7 +275,7 @@ public Throwable close(Throwable accumulate)
 
     public Reader openReader()
     {
-        return new Reader(this);
+        return new Reader(this, version);
     }
 
     protected IndexPosIterator allKeysIterator()
@@ -270,9 +320,9 @@ public interface Acceptor<ArgType, ResultType>
      */
     public static class Reader extends Walker<Reader>
     {
-        protected Reader(PartitionIndex index)
+        protected Reader(PartitionIndex index, ByteComparable.Version version)
         {
-            super(index.instantiateRebufferer(), index.root);
+            super(index.instantiateRebufferer(), index.root, version);
         }
 
         /**
@@ -388,9 +438,6 @@ protected int payloadSize()
      */
     public static class IndexPosIterator extends ValueIterator<IndexPosIterator>
     {
-        static final long INVALID = -1;
-        long pos = INVALID;
-
         /**
          * @param index PartitionIndex to use for the iteration.
          * <p>
@@ -399,12 +446,12 @@ public static class IndexPosIterator extends ValueIterator<IndexPosIterator>
          */
         public IndexPosIterator(PartitionIndex index)
         {
-            super(index.instantiateRebufferer(), index.root);
+            super(index.instantiateRebufferer(), index.root, index.filterFirst, index.filterLast, LeftBoundTreatment.ADMIT_PREFIXES, index.version);
         }
 
         IndexPosIterator(PartitionIndex index, PartitionPosition start, PartitionPosition end)
         {
-            super(index.instantiateRebufferer(), index.root, start, end, true);
+            super(index.instantiateRebufferer(), index.root, start, end, LeftBoundTreatment.ADMIT_PREFIXES, index.version);
         }
 
         /**
@@ -412,18 +459,12 @@ public IndexPosIterator(PartitionIndex index)
          */
         protected long nextIndexPos()
         {
-            // without missing positions, we save and reuse the unreturned position.
-            if (pos == INVALID)
-            {
-                pos = nextPayloadedNode();
-                if (pos == INVALID)
-                    return NOT_FOUND;
-            }
-
-            go(pos);
+            return nextValueAsLong(this::getCurrentIndexPos, NOT_FOUND);
+        }
 
-            pos = INVALID; // make sure next time we call nextPayloadedNode() again
-            return getIndexPos(buf, payloadPosition(), payloadFlags()); // this should not throw
+        private long getCurrentIndexPos()
+        {
+            return getIndexPos(buf, payloadPosition(), payloadFlags());
         }
     }
 
@@ -439,7 +480,7 @@ public void dumpTrie(String fileName)
         }
         catch (Throwable t)
         {
-            logger.warn("Failed to dump trie to {} due to exception {}", fileName, t);
+            logger.warn("Failed to dump trie to {} due to exception", fileName, t);
         }
     }
 
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexBuilder.java
index b096b9d05ffb..f9c29fa9e392 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexBuilder.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexBuilder.java
@@ -22,7 +22,6 @@
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.io.tries.IncrementalTrieWriter;
-import org.apache.cassandra.io.tries.Walker;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.SequentialWriter;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -33,11 +32,13 @@
  * <p>
  * The files created by this builder are read by {@link PartitionIndex}.
  */
-class PartitionIndexBuilder implements AutoCloseable
+// Used by CNDB
+public class PartitionIndexBuilder implements AutoCloseable
 {
     private final SequentialWriter writer;
     private final IncrementalTrieWriter<PartitionIndex.Payload> trieWriter;
     private final FileHandle.Builder fhBuilder;
+    private final ByteComparable.Version version;
 
     // the last synced data file position
     private long dataSyncPosition;
@@ -60,10 +61,11 @@ class PartitionIndexBuilder implements AutoCloseable
     private DecoratedKey lastWrittenKey;
     private PartitionIndex.Payload lastPayload;
 
-    public PartitionIndexBuilder(SequentialWriter writer, FileHandle.Builder fhBuilder)
+    public PartitionIndexBuilder(SequentialWriter writer, FileHandle.Builder fhBuilder, ByteComparable.Version version)
     {
+        this.version = version;
         this.writer = writer;
-        this.trieWriter = IncrementalTrieWriter.open(PartitionIndex.TRIE_SERIALIZER, writer);
+        this.trieWriter = IncrementalTrieWriter.open(PartitionIndex.TRIE_SERIALIZER, writer, version);
         this.fhBuilder = fhBuilder;
     }
 
@@ -110,7 +112,14 @@ private void refreshReadableBoundary()
 
         try (FileHandle fh = fhBuilder.withLengthOverride(writer.getLastFlushOffset()).complete())
         {
-            PartitionIndex pi = new PartitionIndexEarly(fh, partialIndexTail.root(), partialIndexTail.count(), firstKey, partialIndexLastKey, partialIndexTail.cutoff(), partialIndexTail.tail());
+            PartitionIndex pi = new PartitionIndexEarly(fh,
+                                                        partialIndexTail.root(),
+                                                        partialIndexTail.count(),
+                                                        firstKey.retainable(),
+                                                        partialIndexLastKey.retainable(),
+                                                        partialIndexTail.cutoff(),
+                                                        partialIndexTail.tail(),
+                                                        version);
             partialIndexConsumer.accept(pi);
             partialIndexConsumer = null;
         }
@@ -136,7 +145,7 @@ public void addEntry(DecoratedKey decoratedKey, long position) throws IOExceptio
         }
         else
         {
-            int diffPoint = ByteComparable.diffPoint(lastKey, decoratedKey, Walker.BYTE_COMPARABLE_VERSION);
+            int diffPoint = ByteComparable.diffPoint(lastKey, decoratedKey, version);
             ByteComparable prevPrefix = ByteComparable.cut(lastKey, Math.max(diffPoint, lastDiffPoint));
             trieWriter.add(prevPrefix, lastPayload);
             lastWrittenKey = lastKey;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexEarly.java b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexEarly.java
index 3486e87dd6f5..7598ef696aad 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexEarly.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexEarly.java
@@ -23,6 +23,7 @@
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.Rebufferer;
 import org.apache.cassandra.io.util.TailOverridingRebufferer;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 /**
  * Early-opened partition index. Part of the data is already written to file, but some nodes, including the ones in the
@@ -35,9 +36,9 @@ class PartitionIndexEarly extends PartitionIndex
     final ByteBuffer tail;
 
     public PartitionIndexEarly(FileHandle fh, long trieRoot, long keyCount, DecoratedKey first, DecoratedKey last,
-                               long cutoff, ByteBuffer tail)
+                               long cutoff, ByteBuffer tail, ByteComparable.Version version)
     {
-        super(fh, trieRoot, keyCount, first, last);
+        super(fh, trieRoot, keyCount, first, last, null, last, version);
         this.cutoff = cutoff;
         this.tail = tail;
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReader.java b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReader.java
index 3bfd2903fc89..674c4fad8ccb 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReader.java
@@ -65,7 +65,7 @@ public static class IndexInfo
 
     public RowIndexReader(FileHandle file, long root, Version version)
     {
-        super(file.instantiateRebufferer(null), root);
+        super(file.instantiateRebufferer(null), root, version.getByteComparableVersion());
         this.version = version;
     }
 
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReverseIterator.java b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReverseIterator.java
index 0d7878973b43..f61fbb456b91 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReverseIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReverseIterator.java
@@ -23,6 +23,7 @@
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
 import org.apache.cassandra.io.tries.ReverseValueIterator;
+import org.apache.cassandra.io.tries.ValueIterator;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
@@ -36,7 +37,7 @@ class RowIndexReverseIterator extends ReverseValueIterator<RowIndexReverseIterat
 
     public RowIndexReverseIterator(FileHandle file, long root, ByteComparable start, ByteComparable end, Version version)
     {
-        super(file.instantiateRebufferer(null), root, start, end, true);
+        super(file.instantiateRebufferer(null), root, start, end, ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, version.getByteComparableVersion());
         this.version = version;
     }
 
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexWriter.java b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexWriter.java
index 321238396ffd..d290b8346f78 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexWriter.java
@@ -25,7 +25,6 @@
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
 import org.apache.cassandra.io.tries.IncrementalTrieWriter;
-import org.apache.cassandra.io.tries.Walker;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
@@ -41,13 +40,15 @@ class RowIndexWriter implements AutoCloseable
 {
     private final ClusteringComparator comparator;
     private final IncrementalTrieWriter<IndexInfo> trie;
+    private final ByteComparable.Version byeComparableVersion;
     private ByteComparable prevMax = null;
     private ByteComparable prevSep = null;
 
     RowIndexWriter(ClusteringComparator comparator, DataOutputPlus out, Version version)
     {
         this.comparator = comparator;
-        this.trie = IncrementalTrieWriter.open(RowIndexReader.getSerializer(version), out);
+        this.byeComparableVersion = version != null ? version.getByteComparableVersion() : null;
+        this.trie = IncrementalTrieWriter.open(RowIndexReader.getSerializer(version), out, byeComparableVersion);
     }
 
     void reset()
@@ -79,8 +80,8 @@ public long complete(long endPos) throws IOException
         // Add a separator after the last section, so that greater inputs can be quickly rejected.
         // To maximize its efficiency we add it with the length of the last added separator.
         int i = 0;
-        ByteSource max = prevMax.asComparableBytes(Walker.BYTE_COMPARABLE_VERSION);
-        ByteSource sep = prevSep.asComparableBytes(Walker.BYTE_COMPARABLE_VERSION);
+        ByteSource max = prevMax.asComparableBytes(byeComparableVersion);
+        ByteSource sep = prevSep.asComparableBytes(byeComparableVersion);
         int c;
         while ((c = max.next()) == sep.next() && c != ByteSource.END_OF_STREAM)
             ++i;
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java
index 7de33ba2f117..e64397d71a27 100644
--- a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java
@@ -102,8 +102,8 @@ private IndexSummaryManager(Supplier<List<T>> indexSummariesProvider)
 
         long indexSummarySizeInMB = DatabaseDescriptor.getIndexSummaryCapacityInMiB();
         int interval = DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes();
-        logger.info("Initializing index summary manager with a memory pool size of {} MB and a resize interval of {} minutes",
-                    indexSummarySizeInMB, interval);
+        logger.debug("Initializing index summary manager with a memory pool size of {} MB and a resize interval of {} minutes",
+                     indexSummarySizeInMB, interval);
 
         setMemoryPoolCapacityInMB(DatabaseDescriptor.getIndexSummaryCapacityInMiB());
         setResizeIntervalInMinutes(DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
@@ -219,7 +219,7 @@ private Pair<Long, Map<TableId, LifecycleTransaction>> getRestributionTransactio
                 {
                     View view = cfStore.getTracker().getView();
                     allSSTables = ImmutableSet.copyOf(view.select(SSTableSet.CANONICAL));
-                    nonCompacting = ImmutableSet.copyOf(view.getUncompacting(allSSTables));
+                    nonCompacting = ImmutableSet.copyOf(view.getNoncompacting(allSSTables));
                 }
                 while (null == (txn = cfStore.getTracker().tryModify(nonCompacting, OperationType.INDEX_SUMMARY)));
 
@@ -283,9 +283,9 @@ public void redistributeSummaries() throws IOException
      * @return a list of new SSTableReader instances
      */
     @VisibleForTesting
-    public static <T extends SSTableReader & IndexSummarySupport> List<T> redistributeSummaries(IndexSummaryRedistribution redistribution) throws IOException
+    public static <T extends SSTableReader & IndexSummarySupport<T>> List<T> redistributeSummaries(IndexSummaryRedistribution redistribution) throws IOException
     {
-        return (List<T>) CompactionManager.instance.runAsActiveCompaction(redistribution, redistribution::redistributeSummaries);
+        return CompactionManager.instance.runIndexSummaryRedistribution(redistribution);
     }
 
     @VisibleForTesting
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java
index d27969719ab0..e2ace80e537a 100644
--- a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java
@@ -33,9 +33,7 @@
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.io.sstable.Downsampling;
@@ -51,7 +49,7 @@
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
-public class IndexSummaryRedistribution extends CompactionInfo.Holder
+public class IndexSummaryRedistribution extends AbstractTableOperation
 {
     private static final Logger logger = LoggerFactory.getLogger(IndexSummaryRedistribution.class);
 
@@ -119,8 +117,7 @@ public <T extends SSTableReader & IndexSummarySupport<T>> List<T> redistributeSu
         double totalReadsPerSec = 0.0;
         for (T sstable : redistribute)
         {
-            if (isStopRequested())
-                throw new CompactionInterruptedException(getCompactionInfo());
+            throwIfStopRequested();
 
             if (sstable.getReadMeter() != null)
             {
@@ -173,8 +170,7 @@ private <T extends SSTableReader & IndexSummarySupport<T>> List<T> adjustSamplin
         remainingSpace = memoryPoolCapacity;
         for (T sstable : sstables)
         {
-            if (isStopRequested())
-                throw new CompactionInterruptedException(getCompactionInfo());
+            throwIfStopRequested();
 
             int minIndexInterval = sstable.metadata().params.minIndexInterval;
             int maxIndexInterval = sstable.metadata().params.maxIndexInterval;
@@ -271,8 +267,7 @@ else if (targetNumEntries < currentNumEntries * DOWNSAMPLE_THESHOLD && newSampli
         toDownsample.addAll(forceUpsample);
         for (ResampleEntry<T> entry : toDownsample)
         {
-            if (isStopRequested())
-                throw new CompactionInterruptedException(getCompactionInfo());
+            throwIfStopRequested();
 
             T sstable = entry.sstable;
             logger.trace("Re-sampling index summary for {} from {}/{} to {}/{} of the original number of entries",
@@ -355,9 +350,14 @@ static <T extends SSTableReader & IndexSummarySupport<T>> Pair<List<T>, List<Res
         return Pair.create(willNotDownsample, toDownsample.subList(noDownsampleCutoff, toDownsample.size()));
     }
 
-    public CompactionInfo getCompactionInfo()
+    public OperationProgress getProgress()
     {
-        return CompactionInfo.withoutSSTables(null, OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, Unit.BYTES, compactionId);
+        return OperationProgress.withoutSSTables(null,
+                                                 OperationType.INDEX_SUMMARY,
+                                                 (memoryPoolBytes - remainingSpace),
+                                                 memoryPoolBytes,
+                                                 Unit.BYTES,
+                                                 compactionId);
     }
 
     public boolean isGlobal()
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
index ff1e604d04c5..946e1f470b79 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
@@ -100,4 +100,11 @@ public interface IMetadataSerializer
      * Replace the sstable metadata file ({@code -Statistics.db}) with the given components.
      */
     void rewriteSSTableMetadata(Descriptor descriptor, Map<MetadataType, MetadataComponent> currentComponents) throws IOException;
+
+    /**
+     * Updates the sstable metadata components (works similarly to {@link #rewriteSSTableMetadata(Descriptor, Map)} but
+     * only updates the provided components rather than replacing the whole metadata map).
+     */
+    void updateSSTableMetadata(Descriptor descriptor, Map<MetadataType, MetadataComponent> updatedComponents) throws IOException;
+
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 7b841c7cd8de..0022cca0ba8a 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -101,7 +101,9 @@ public static StatsMetadata defaultStatsMetadata()
                                  false,
                                  true,
                                  ByteBufferUtil.EMPTY_BYTE_BUFFER,
-                                 ByteBufferUtil.EMPTY_BYTE_BUFFER);
+                                 ByteBufferUtil.EMPTY_BYTE_BUFFER,
+                                 Collections.emptyMap(),
+                                 ZeroCopyMetadata.EMPTY);
     }
 
     protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram();
@@ -177,6 +179,12 @@ public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator
         commitLogIntervals(intervals.build());
     }
 
+    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
+    {
+        this(sstables, comparator);
+        sstableLevel(level);
+    }
+
     public MetadataCollector addKey(ByteBuffer key)
     {
         long hashed = MurmurHash.hash2_64(key, key.position(), key.remaining(), 0);
@@ -363,7 +371,7 @@ public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner,
         components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
         components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
                                                              estimatedCellPerPartitionCount,
-                                                             commitLogIntervals,
+                                                             commitLogIntervals != null ? commitLogIntervals : IntervalSet.empty(),
                                                              timestampTracker.min(),
                                                              timestampTracker.max(),
                                                              localDeletionTimeTracker.min(),
@@ -385,7 +393,9 @@ public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner,
                                                              isTransient,
                                                              hasPartitionLevelDeletions,
                                                              firstKey,
-                                                             lastKey));
+                                                             lastKey,
+                                                             Collections.emptyMap(),
+                                                             ZeroCopyMetadata.EMPTY));
         components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality));
         components.put(MetadataType.HEADER, header.toComponent());
         return components;
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
index 5ecb582b04e6..2655aa6f143f 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
@@ -278,4 +278,12 @@ public void rewriteSSTableMetadata(Descriptor descriptor, Map<MetadataType, Meta
         }
         file.move(descriptor.fileFor(Components.STATS));
     }
+
+    public void updateSSTableMetadata(Descriptor descriptor, Map<MetadataType, MetadataComponent> updatedComponents) throws IOException
+    {
+        Map<MetadataType, MetadataComponent> currentComponents = deserialize(descriptor, EnumSet.allOf(MetadataType.class));
+        currentComponents.putAll(updatedComponents);
+        rewriteSSTableMetadata(descriptor, currentComponents);
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index b509e3f3ae6d..286c7c979813 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -19,13 +19,16 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 
+import com.google.common.collect.ImmutableMap;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,6 +48,7 @@
 import org.apache.cassandra.serializers.AbstractTypeSerializer;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.UUIDSerializer;
 import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
@@ -91,6 +95,7 @@ public class StatsMetadata extends MetadataComponent
      * deletions in this sstable. Obviously, this is pretty imprecise: a single partition deletion in the sstable
      * means we have to assume _any_ key may have a partition deletion. This is still likely useful as workloads that
      * does not use partition level deletions, or only very rarely, are probably not that rare.
+     *
      * TODO we could replace this by a small bloom-filter instead; the only downside being that we'd have to care about
      *  the size of this bloom filters not getting out of hands, and it's a tiny bit unclear if it's worth the added
      *  complexity.
@@ -99,6 +104,9 @@ public class StatsMetadata extends MetadataComponent
 
     public final ByteBuffer firstKey;
     public final ByteBuffer lastKey;
+    private final ImmutableMap<ByteBuffer, Integer> maxColumnValueLengths;
+
+    public final ZeroCopyMetadata zeroCopyMetadata;
 
     public StatsMetadata(EstimatedHistogram estimatedPartitionSize,
                          EstimatedHistogram estimatedCellPerPartitionCount,
@@ -124,7 +132,9 @@ public StatsMetadata(EstimatedHistogram estimatedPartitionSize,
                          boolean isTransient,
                          boolean hasPartitionLevelDeletions,
                          ByteBuffer firstKey,
-                         ByteBuffer lastKey)
+                         ByteBuffer lastKey,
+                         Map<ByteBuffer, Integer> maxColumnValueLengths,
+                         ZeroCopyMetadata zeroCopyMetadata)
     {
         this.estimatedPartitionSize = estimatedPartitionSize;
         this.estimatedCellPerPartitionCount = estimatedCellPerPartitionCount;
@@ -152,6 +162,8 @@ public StatsMetadata(EstimatedHistogram estimatedPartitionSize,
         this.hasPartitionLevelDeletions = hasPartitionLevelDeletions;
         this.firstKey = firstKey;
         this.lastKey = lastKey;
+        this.maxColumnValueLengths = ImmutableMap.copyOf(maxColumnValueLengths);
+        this.zeroCopyMetadata = zeroCopyMetadata;
     }
 
     public MetadataType getType()
@@ -209,7 +221,9 @@ public StatsMetadata mutateLevel(int newLevel)
                                  isTransient,
                                  hasPartitionLevelDeletions,
                                  firstKey,
-                                 lastKey);
+                                 lastKey,
+                                 maxColumnValueLengths,
+                                 zeroCopyMetadata);
     }
 
     public StatsMetadata mutateRepairedMetadata(long newRepairedAt, TimeUUID newPendingRepair, boolean newIsTransient)
@@ -238,7 +252,9 @@ public StatsMetadata mutateRepairedMetadata(long newRepairedAt, TimeUUID newPend
                                  newIsTransient,
                                  hasPartitionLevelDeletions,
                                  firstKey,
-                                 lastKey);
+                                 lastKey,
+                                 maxColumnValueLengths,
+                                 zeroCopyMetadata);
     }
 
     @Override
@@ -272,6 +288,8 @@ public boolean equals(Object o)
                        .append(hasPartitionLevelDeletions, that.hasPartitionLevelDeletions)
                        .append(firstKey, that.firstKey)
                        .append(lastKey, that.lastKey)
+                       .append(maxColumnValueLengths, that.maxColumnValueLengths)
+                       .append(zeroCopyMetadata, that.zeroCopyMetadata)
                        .build();
     }
 
@@ -302,18 +320,21 @@ public int hashCode()
                        .append(hasPartitionLevelDeletions)
                        .append(firstKey)
                        .append(lastKey)
+                       .append(maxColumnValueLengths)
+                       .append(zeroCopyMetadata)
                        .build();
     }
 
     public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
     {
         private static final Logger logger = LoggerFactory.getLogger(StatsMetadataSerializer.class);
+        private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, TimeUnit.MINUTES);
 
         private final AbstractTypeSerializer typeSerializer = new AbstractTypeSerializer();
 
         public int serializedSize(Version version, StatsMetadata component) throws IOException
         {
-            int size = 0;
+            long size = 0;
             size += EstimatedHistogram.serializer.serializedSize(component.estimatedPartitionSize);
             size += EstimatedHistogram.serializer.serializedSize(component.estimatedCellPerPartitionCount);
             size += CommitLogPosition.serializer.serializedSize(component.commitLogIntervals.upperBound().orElse(CommitLogPosition.NONE));
@@ -329,11 +350,11 @@ public int serializedSize(Version version, StatsMetadata component) throws IOExc
                 // min column names
                 size += 4;
                 ClusteringBound<?> minClusteringValues = component.coveredClustering.start();
-                size += minClusteringValues.size() * 2 /* short length */ + minClusteringValues.dataSize();
+                size += countUntilNull(minClusteringValues.getBufferArray()) * 2L /* short length */ + minClusteringValues.dataSize();
                 // max column names
                 size += 4;
                 ClusteringBound<?> maxClusteringValues = component.coveredClustering.end();
-                size += maxClusteringValues.size() * 2 /* short length */ + maxClusteringValues.dataSize();
+                size += countUntilNull(maxClusteringValues.getBufferArray()) * 2L /* short length */ + maxClusteringValues.dataSize();
             }
             else if (version.hasImprovedMinMax())
             {
@@ -354,16 +375,42 @@ else if (version.hasImprovedMinMax())
                     size += TimeUUID.sizeInBytes();
             }
 
+            // we do not have zero copy metadata, but we need to support loading such sstables
+            if (version.hasZeroCopyMetadata())
+            {
+                size += 1;
+                if (component.zeroCopyMetadata.exists())
+                    size += ZeroCopyMetadata.serializer.serializedSize(component.zeroCopyMetadata);
+            }
+
+            // we do not have node sync metadata
+            if (version.hasIncrementalNodeSyncMetadata())
+            {
+                size += Long.BYTES;
+            }
+
+            if (version.hasMaxColumnValueLengths())
+            {
+                size += 4; // num columns
+                for (Map.Entry<ByteBuffer, Integer> entry : component.maxColumnValueLengths.entrySet())
+                    size += ByteBufferUtil.serializedSizeWithVIntLength(entry.getKey()) + 4; // column name, max value length
+            }
+
             if (version.hasIsTransient())
             {
                 size += TypeSizes.sizeof(component.isTransient);
             }
 
+            if (version.hasMisplacedPartitionLevelDeletionsPresenceMarker())
+            {
+                size += TypeSizes.sizeof(component.hasPartitionLevelDeletions);
+            }
+
             if (version.hasOriginatingHostId())
             {
                 size += 1; // boolean: is originatingHostId present
                 if (component.originatingHostId != null)
-                    size += UUIDSerializer.serializer.serializedSize(component.originatingHostId, version.correspondingMessagingVersion());
+                    size += (int) UUIDSerializer.serializer.serializedSize(component.originatingHostId, version.correspondingMessagingVersion());
             }
 
             if (version.hasPartitionLevelDeletionsPresenceMarker())
@@ -371,11 +418,6 @@ else if (version.hasImprovedMinMax())
                 size += TypeSizes.sizeof(component.hasPartitionLevelDeletions);
             }
 
-            if (version.hasImprovedMinMax() && version.hasLegacyMinMax())
-            {
-                size = improvedMinMaxSize(version, component, size);
-            }
-
             if (version.hasKeyRange())
             {
                 size += ByteBufferUtil.serializedSizeWithVIntLength(component.firstKey);
@@ -387,15 +429,15 @@ else if (version.hasImprovedMinMax())
                 size += Double.BYTES;
             }
 
-            return size;
+            return Math.toIntExact(size);
         }
 
-        private int improvedMinMaxSize(Version version, StatsMetadata component, int size)
+        private long improvedMinMaxSize(Version version, StatsMetadata component, long size)
         {
             size += typeSerializer.serializedListSize(component.clusteringTypes);
             size += Slice.serializer.serializedSize(component.coveredClustering,
-                                                    version.correspondingMessagingVersion(),
-                                                    component.clusteringTypes);
+                                                          version.correspondingMessagingVersion(),
+                                                          component.clusteringTypes);
             return size;
         }
 
@@ -472,11 +514,47 @@ else if (version.hasImprovedMinMax())
                 }
             }
 
+            // we do not produce such sstables, but we need to be able to rewrite the metadata
+            if (version.hasZeroCopyMetadata())
+            {
+                if (component.zeroCopyMetadata != null && component.zeroCopyMetadata.exists())
+                {
+                    out.writeByte(1);
+                    ZeroCopyMetadata.serializer.serialize(component.zeroCopyMetadata, out);
+                }
+                else
+                {
+                    out.writeByte(0);
+                }
+            }
+
+            // we do not have node sync metadata
+            if (version.hasIncrementalNodeSyncMetadata())
+            {
+                out.writeLong(Long.MAX_VALUE);
+            }
+
+            // left for being able to import DSE sstables, not used
+            if (version.hasMaxColumnValueLengths())
+            {
+                out.writeInt(component.maxColumnValueLengths.size());
+                for (Map.Entry<ByteBuffer, Integer> entry : component.maxColumnValueLengths.entrySet())
+                {
+                    ByteBufferUtil.writeWithVIntLength(entry.getKey(), out);
+                    out.writeInt(entry.getValue());
+                }
+            }
+
             if (version.hasIsTransient())
             {
                 out.writeBoolean(component.isTransient);
             }
 
+            if (version.hasMisplacedPartitionLevelDeletionsPresenceMarker())
+            {
+                out.writeBoolean(component.hasPartitionLevelDeletions);
+            }
+
             if (version.hasOriginatingHostId())
             {
                 if (component.originatingHostId != null)
@@ -495,11 +573,6 @@ else if (version.hasImprovedMinMax())
                 out.writeBoolean(component.hasPartitionLevelDeletions);
             }
 
-            if (version.hasImprovedMinMax() && version.hasLegacyMinMax())
-            {
-                serializeImprovedMinMax(version, component, out);
-            }
-
             if (version.hasKeyRange())
             {
                 ByteBufferUtil.writeWithVIntLength(component.firstKey, out);
@@ -620,25 +693,51 @@ else if (version.hasImprovedMinMax())
                 pendingRepair = TimeUUID.deserialize(in);
             }
 
-            boolean isTransient = version.hasIsTransient() && in.readBoolean();
+            ZeroCopyMetadata zeroCopyMetadata = ZeroCopyMetadata.EMPTY;
+            if (version.hasZeroCopyMetadata() && in.readByte() != 0)
+            {
+                zeroCopyMetadata = ZeroCopyMetadata.serializer.deserialize(in);
+            }
 
-            UUID originatingHostId = null;
-            if (version.hasOriginatingHostId() && in.readByte() != 0)
-                originatingHostId = UUIDSerializer.serializer.deserialize(in, 0);
+            if (version.hasIncrementalNodeSyncMetadata())
+            {
+                noSpamLogger.warn("Ignoring incremental node sync metadata from {} as it is not supported", in);
+                in.readLong();
+            }
+
+            // left for being able to import DSE sstables, not used
+            final Map<ByteBuffer, Integer> maxColumnValueLengths;
+            if (version.hasMaxColumnValueLengths())
+            {
+                int colCount = in.readInt();
+                ImmutableMap.Builder<ByteBuffer, Integer> builder = ImmutableMap.builderWithExpectedSize(colCount);
+
+                for (int i = 0; i < colCount; i++)
+                    builder.put(ByteBufferUtil.readWithVIntLength(in), in.readInt());
+                maxColumnValueLengths = builder.build();
+            }
+            else
+            {
+                maxColumnValueLengths = Collections.emptyMap();
+            }
+
+            boolean isTransient = version.hasIsTransient() && in.readBoolean();
 
             // If not recorded, the only time we can guarantee there is no partition level deletion is if there is no
             // deletion at all. Otherwise, we have to assume there may be some.
             boolean hasPartitionLevelDeletions = minLocalDeletionTime != Cell.NO_DELETION_TIME;
-            if (version.hasPartitionLevelDeletionsPresenceMarker())
+            if (version.hasMisplacedPartitionLevelDeletionsPresenceMarker())
             {
                 hasPartitionLevelDeletions = in.readBoolean();
             }
 
-            if (version.hasImprovedMinMax() && version.hasLegacyMinMax())
+            UUID originatingHostId = null;
+            if (version.hasOriginatingHostId() && in.readByte() != 0)
+                originatingHostId = UUIDSerializer.serializer.deserialize(in, 0);
+
+            if (version.hasPartitionLevelDeletionsPresenceMarker())
             {
-                // improvedMinMax will be in this place until legacyMinMax is removed
-                clusteringTypes = typeSerializer.deserializeList(in);
-                coveredClustering = Slice.serializer.deserialize(in, version.correspondingMessagingVersion(), clusteringTypes);
+                hasPartitionLevelDeletions = in.readBoolean();
             }
 
             ByteBuffer firstKey = null;
@@ -679,7 +778,9 @@ else if (version.hasImprovedMinMax())
                                      isTransient,
                                      hasPartitionLevelDeletions,
                                      firstKey,
-                                     lastKey);
+                                     lastKey,
+                                     maxColumnValueLengths,
+                                     zeroCopyMetadata);
         }
 
         private int countUntilNull(ByteBuffer[] bufferArray)
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
index 0eda8eb7753e..c980705c94b0 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
@@ -69,6 +69,11 @@ public int hashCode()
         return result;
     }
 
+    public ValidationMetadata withBloomFilterFPChance(double bloomFilterFpChance)
+    {
+        return new ValidationMetadata(partitioner, bloomFilterFpChance);
+    }
+
     public static class ValidationMetadataSerializer implements IMetadataComponentSerializer<ValidationMetadata>
     {
         public int serializedSize(Version version, ValidationMetadata component) throws IOException
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/ZeroCopyMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/ZeroCopyMetadata.java
new file mode 100644
index 000000000000..ad64dad52b16
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/ZeroCopyMetadata.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.metadata;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+import com.google.common.primitives.Longs;
+
+import org.apache.cassandra.io.ISerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.SliceDescriptor;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Metadata related to sstables copied via zero copy, stored in the {@link StatsMetadata}. ZCS allows for streaming
+ * part of the data file while streaming the original index and stats files (thus data file positions included by the
+ * indexes refer to the orignal data file rather than to the streamed slice). This metadata contains information about
+ * slice of the data file that was streamed this way.
+ * <ul>
+ * <li>{@link #dataStart}:     Disk offset in the original data file where the first streamed partition begins (inclusive)</li>
+ * <li>{@link #dataEnd}:       Disk offset in the original data file where the last streamed row ends (exclusive)</li>
+ * <li>{@link #firstKey}:      Byte representation of the first key in the streamed sstable slice</li>
+ * <li>{@link #lastKey}:       Byte representation of the last key in the streamed sstable slice</li>
+ * <li>{@link #chunkSize}:     If the file is written/read in chunks, this is the size in bytes of the chunk, otherwise 0</li>
+ * <li>{@link #estimatedKeys}: Estimated keys contained in the sstable</li>
+ * </ul>
+ * When the sstable is written in chunks, the {@link #dataStart} might not match the actual offset as it needs to be
+ * aligned by the {@link #chunkSize}. In such case, if you need to access the actual start offset, please use
+ * {@link #sliceStart}. For example:
+ * <pre>
+ * 0                16               32               48               64               80               96
+ * |----------------|----------------|----------------|----------------|----------------|----------------|
+ * | chunk 1        | chunk 2        | chunk 3        | chunk 4        | chunk 5        | chunk 6        |
+ * |----------------|----------------|----------------|----------------|----------------|----------------|
+ *  #key1      #key2                    #key3      #key4                    #key5     #key6     #key7
+ * </pre>
+ * Say the slice ZCS sends is from chunk 3 to chunk 5. The first key is 3 and the last key is 5. The start offset
+ * is 34 (the exact position) and the start offset aligned is 32 (the position aligned to the chunk size). The end
+ * offset is 78 (the exact position).
+ * The transferred data file looks as follows:
+ * <pre>
+ * 0                16               32               48
+ * |----------------|----------------|----------------|
+ * | chunk 3        | chunk 4        | chunk 5        |
+ * |----------------|----------------|----------------|
+ *    #key3      #key4                    #key5
+ * </pre>
+ * So to get the actual start offset of the first key, that is 2, you need to calculate is as follows:
+ * {@code dataStart - sliceStart}. When you get a position of say key 4, it is 45, so you need to calculate
+ * the actual position in slice as follows: {@code 45 - sliceStart}, which gives you 12 - the key 4 position in
+ * the local slice.
+ */
+public class ZeroCopyMetadata extends SliceDescriptor
+{
+    public static final Serializer serializer = new Serializer();
+    public static final ZeroCopyMetadata EMPTY = new ZeroCopyMetadata(0, 0, 0, 0, null, null);
+
+    private final long estimatedKeys;
+    private final ByteBuffer firstKey;
+    private final ByteBuffer lastKey;
+
+    public ZeroCopyMetadata(long dataStart, long dataEnd, int chunkSize, long estimatedKeys, ByteBuffer firstKey, ByteBuffer lastKey)
+    {
+        super(dataStart, dataEnd, chunkSize);
+        this.estimatedKeys = estimatedKeys;
+        this.firstKey = firstKey;
+        this.lastKey = lastKey;
+    }
+
+    public ByteBuffer firstKey()
+    {
+        return this.firstKey.duplicate();
+    }
+
+    public ByteBuffer lastKey()
+    {
+        return this.lastKey.duplicate();
+    }
+
+    public long estimatedKeys()
+    {
+        return this.estimatedKeys;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        ZeroCopyMetadata that = (ZeroCopyMetadata) o;
+        return estimatedKeys == that.estimatedKeys
+               && Objects.equals(firstKey, that.firstKey)
+               && Objects.equals(lastKey, that.lastKey);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(super.hashCode(), estimatedKeys, firstKey, lastKey);
+    }
+
+    public static class Serializer implements ISerializer<ZeroCopyMetadata>
+    {
+        @Override
+        public long serializedSize(ZeroCopyMetadata metadata)
+        {
+            if (!metadata.exists())
+                return 0;
+
+            return 3 * Longs.BYTES + Integer.BYTES + ByteBufferUtil.serializedSizeWithShortLength(metadata.firstKey) + ByteBufferUtil.serializedSizeWithShortLength(metadata.lastKey);
+        }
+
+        @Override
+        public void serialize(ZeroCopyMetadata component, DataOutputPlus out) throws IOException
+        {
+            out.writeLong(component.dataStart);
+            out.writeLong(component.dataEnd);
+            out.writeInt(component.chunkSize);
+            out.writeLong(component.estimatedKeys);
+            ByteBufferUtil.writeWithShortLength(component.firstKey.duplicate(), out);
+            ByteBufferUtil.writeWithShortLength(component.lastKey.duplicate(), out);
+        }
+
+        @Override
+        public ZeroCopyMetadata deserialize(DataInputPlus in) throws IOException
+        {
+            return new ZeroCopyMetadata(
+                in.readLong(), 
+                in.readLong(), 
+                in.readInt(), 
+                in.readLong(), 
+                ByteBufferUtil.readWithShortLength(in), 
+                ByteBufferUtil.readWithShortLength(in));
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/storage/StorageProvider.java b/src/java/org/apache/cassandra/io/storage/StorageProvider.java
new file mode 100644
index 000000000000..c1c2380c9faa
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/storage/StorageProvider.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.storage;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.INativeLibrary;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_STORAGE_PROVIDER;
+
+/**
+ * The storage provider is used to support directory creation and remote/local conversion for remote storage.
+ * The default implementation {@link DefaultProvider} is based on local file system.
+ */
+public interface StorageProvider
+{
+    Logger logger = LoggerFactory.getLogger(StorageProvider.class);
+
+    StorageProvider instance = !CUSTOM_STORAGE_PROVIDER.isPresent()
+                               ? new DefaultProvider()
+                               : FBUtilities.construct(CUSTOM_STORAGE_PROVIDER.getString(), "storage provider");
+
+    enum DirectoryType
+    {
+        DATA("data_file_directories"),
+        LOCAL_SYSTEM_DATA("local_system_data_file_directories"),
+        METADATA("metadata_directory"),
+        COMMITLOG("commit_log_directory"),
+        HINTS("hints_directory"),
+        SAVED_CACHES("saved_caches_directory"),
+        CDC("cdc_raw_directory"),
+        SNAPSHOT("snapshot_directory"),
+        NODES("nodes_local_directory"),
+        LOG_TRANSACTION("log_transaction_directory"),
+        LOGS("logs_directory"),
+        TEMP("temp_directory"),
+        OTHERS("other_directory");
+
+        final String name;
+
+        final boolean readable;
+        final boolean writable;
+
+        DirectoryType(String name)
+        {
+            this.name = name;
+            this.readable = true;
+            this.writable = true;
+        }
+    }
+
+    /**
+     * @return local path if given path is remote path, otherwise returns itself
+     */
+    File getLocalPath(File path);
+
+    /**
+     * @return local path if given path is remote path, otherwise returns itself
+     */
+    Path getLocalPath(Path path);
+
+    /**
+     * update the given path with open options for the sstable components
+     */
+    File withOpenOptions(File ret, Component component);
+
+    /**
+     * Create data directories for given table
+     *
+     * @param ksMetadata   The keyspace metadata, can be null. This is used when schema metadata is
+     *                     not available in {@link Schema}, eg. CNDB backup & restore
+     * @param tableMetadata the metadata of the table
+     * @param dirs         current local data directories
+     * @return data directories that are created
+     */
+    Directories.DataDirectory[] createDataDirectories(@Nullable KeyspaceMetadata ksMetadata, TableMetadata tableMetadata, Directories.DataDirectory[] dirs);
+
+    /**
+     * Create directory for the given path and type, either locally or remotely if any remote storage parameters are passed in.
+     *
+     * @param dir  the directory absolute path to create
+     * @param type the type of directory to create
+     * @return the actual directory path, which can be either local or remote; or null if directory can't be created
+     */
+    File createDirectory(String dir, DirectoryType type);
+
+    /**
+     * Remove the give file from any local cache, for example the OS page cache, or at least it tries to.
+     * @param file the file that is no longer required in the file system caches
+     */
+    void invalidateFileSystemCache(File file);
+
+    /**
+     * Remove the give sstable from any local cache, for example the OS page cache, or at least it tries to.
+     *
+     * @param descriptor the descriptor for the sstable that is no longer required in the file system caches
+     * @param tidied whether ReaderTidier has been run, aka. deleting sstable files.
+     */
+    void invalidateFileSystemCache(Descriptor descriptor, boolean tidied);
+
+    /**
+     * Creates a new {@link FileHandle.Builder} for the given sstable component.
+     * <p>
+     * The returned builder will be configured with the appropriate "access mode" (mmap or not), and the "chunk cache"
+     * will have been set if appropriate.
+     *
+     * @param descriptor descriptor for the sstable whose handler is built.
+     * @param component sstable component for which to build the handler.
+     * @return a new {@link FileHandle.Builder} for the provided sstable component with access mode and chunk cache
+     *   configured as appropriate.
+     */
+    FileHandle.Builder fileHandleBuilderFor(Descriptor descriptor, Component component);
+
+    /**
+     * Creates a new {@link FileHandle.Builder} for the given primary index component during primary index writing time.
+     * <p>
+     * The returned builder will be configured with the appropriate "access mode" (mmap or not), and the "chunk cache"
+     * will have been set if appropriate.
+     *
+     * @param descriptor    descriptor for the sstable whose handler is built.
+     * @param component     sstable component for which to build the handler.
+     * @param operationType the operation for current primary index writer
+     * @return a new {@link FileHandle.Builder} for the provided primary index component with access mode and chunk cache
+     * configured as appropriate.
+     */
+    FileHandle.Builder primaryIndexWriteTimeFileHandleBuilderFor(Descriptor descriptor, Component component, Config.DiskAccessMode diskAccessMode, ChunkCache chunkCache, OperationType operationType);
+
+    /**
+     * Creates a new {@link FileHandle.Builder} for the given SAI component.
+     * <p>
+     * The returned builder will be configured with the appropriate "access mode" (mmap or not), and the "chunk cache"
+     * will have been set if appropriate.
+     *
+     * @param component index component for which to build the handler.
+     * @return a new {@link FileHandle.Builder} for the provided SAI component with access mode and chunk cache
+     *   configured as appropriate.
+     */
+    FileHandle.Builder fileHandleBuilderFor(IndexComponent.ForRead component);
+
+    /**
+     * Creates a new {@link FileChannel} to read the given file, that is suitable for reading the file "at write time",
+     * that is typcally for when we need to access the partially written file to complete checksum.
+     *
+     * @param file the file to be read
+     * @return a new {@link FileChannel} for the provided file
+     */
+    default FileChannel writeTimeReadFileChannelFor(File file) throws IOException
+    {
+        return FileChannel.open(file.toPath(), StandardOpenOption.READ);
+    }
+
+    /**
+     * Creates a new {@link FileHandle.Builder} for the given SAI component and context (for index with per-index files),
+     * that is suitable for reading the component during index build, that is typcally for when we need to access the
+     * component to complete the writing of another related component.
+     * <p>
+     * Other the fact that this method will be called a different time, it's requirements are the same than for
+     * {@link #fileHandleBuilderFor(IndexComponent.ForRead)}.
+     *
+     * @param component index component for which to build the handler.
+     * @return a new {@link FileHandle.Builder} for the provided SAI component with access mode and chunk cache
+     *   configured as appropriate.
+     */
+    FileHandle.Builder indexBuildTimeFileHandleBuilderFor(IndexComponent.ForRead component);
+
+    class DefaultProvider implements StorageProvider
+    {
+        @Override
+        public File getLocalPath(File path)
+        {
+            return path;
+        }
+
+        @Override
+        public Path getLocalPath(Path path)
+        {
+            return path;
+        }
+
+        @Override
+        public File withOpenOptions(File ret, Component component)
+        {
+            return ret;
+        }
+
+        @Override
+        public Directories.DataDirectory[] createDataDirectories(@Nullable KeyspaceMetadata ksMetadata, TableMetadata tableMetadata, Directories.DataDirectory[] dirs)
+        {
+            // data directories are already created in DatabadeDescriptor#createAllDirectories
+            return dirs;
+        }
+
+        @Override
+        public File createDirectory(String dir, DirectoryType type)
+        {
+            File ret = new File(dir);
+            PathUtils.createDirectoriesIfNotExists(ret.toPath());
+            return ret;
+        }
+
+        @Override
+        public void invalidateFileSystemCache(File file)
+        {
+            INativeLibrary.instance.trySkipCache(file, 0, 0);
+            if (ChunkCache.instance != null)
+                ChunkCache.instance.invalidateFile(file);
+        }
+
+        @Override
+        public void invalidateFileSystemCache(Descriptor desc, boolean tidied)
+        {
+            for (Component c : desc.discoverComponents())
+                StorageProvider.instance.invalidateFileSystemCache(desc.fileFor(c));
+        }
+
+        @Override
+        @SuppressWarnings("resource")
+        public FileHandle.Builder fileHandleBuilderFor(Descriptor descriptor, Component component)
+        {
+            return new FileHandle.Builder(descriptor.fileFor(component));
+        }
+
+        @Override
+        public  FileHandle.Builder primaryIndexWriteTimeFileHandleBuilderFor(Descriptor descriptor, Component component, Config.DiskAccessMode diskAccessMode, ChunkCache chunkCache, OperationType operationType)
+        {
+            // By default, no difference between accesses during sstable writing and "at query time", but subclasses may need
+            // to differenciate both.
+            return fileHandleBuilderFor(descriptor, component)
+                   .mmapped(diskAccessMode)
+                   .withChunkCache(chunkCache);
+        }
+
+        @Override
+        @SuppressWarnings("resource")
+        public FileHandle.Builder fileHandleBuilderFor(IndexComponent.ForRead component)
+        {
+            File file = component.file();
+            if (logger.isTraceEnabled())
+            {
+                logger.trace(component.parent().logMessage("Opening {} file handle for {} ({})"),
+                             file, FBUtilities.prettyPrintMemory(file.length()));
+            }
+            var builder = new FileHandle.Builder(file);
+            // Comments on why we don't use adviseRandom for some components where you might expect it:
+            //
+            // KD_TREE: no adviseRandom because we do a large bulk read on startup, queries later may
+            //     benefit from adviseRandom but there's no way to split those apart
+            // POSTINGS_LISTS: for common terms with 1000s of rows, adviseRandom seems likely to
+            //     make it slower; no way to get cardinality at this point in the code
+            //     (and we already have shortcut code for the common 1:1 vector case)
+            //     so we leave it alone here
+            if (component.componentType() == IndexComponentType.TERMS_DATA
+                || component.componentType() == IndexComponentType.VECTOR
+                || component.componentType() == IndexComponentType.PRIMARY_KEY_TRIE)
+            {
+                builder = builder.adviseRandom();
+            }
+            return builder.mmapped(true);
+        }
+
+        @Override
+        public FileHandle.Builder indexBuildTimeFileHandleBuilderFor(IndexComponent.ForRead component)
+        {
+            // By default, no difference between accesses "at flush time" and "at query time", but subclasses may need
+            // to differenciate both.
+            return fileHandleBuilderFor(component);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/tries/BaseValueIterator.java b/src/java/org/apache/cassandra/io/tries/BaseValueIterator.java
new file mode 100644
index 000000000000..3bafd538c678
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/BaseValueIterator.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.tries;
+
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
+
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+public abstract class BaseValueIterator<CONCRETE extends BaseValueIterator<CONCRETE>> extends Walker<CONCRETE>
+{
+    protected static final long NOT_PREPARED = -2;
+    protected final ByteSource limit;
+    protected final TransitionBytesCollector collector;
+    protected IterationPosition stack;
+    protected long next;
+
+    public BaseValueIterator(Rebufferer source, long root, ByteSource limit, boolean collecting, ByteComparable.Version version)
+    {
+        super(source, root, version);
+        this.limit = limit;
+        collector = collecting ? new TransitionBytesCollector(byteComparableVersion) : null;
+    }
+
+    /**
+     * Returns the payload node position.
+     * <p>
+     * This method must be async-read-safe, see {@link #advanceNode()}.
+     */
+    protected long nextPayloadedNode()
+    {
+        if (next != NOT_PREPARED)
+        {
+            long toReturn = next;
+            next = NOT_PREPARED;
+            return toReturn;
+        }
+        else
+            return advanceNode();
+    }
+
+    protected boolean hasNext()
+    {
+        if (next == NOT_PREPARED)
+            next = advanceNode();
+        return next != NONE;
+    }
+
+    protected <VALUE> VALUE nextValue(Supplier<VALUE> supplier)
+    {
+        long node = nextPayloadedNode();
+        if (node == NONE)
+            return null;
+        go(node);
+        return supplier.get();
+    }
+
+    protected long nextValueAsLong(LongSupplier supplier, long valueIfNone)
+    {
+        long node = nextPayloadedNode();
+        if (node == NONE)
+            return valueIfNone;
+        go(node);
+        return supplier.getAsLong();
+    }
+
+    protected ByteComparable collectedKey()
+    {
+        assert collector != null : "Cannot get a collected value from a non-collecting iterator";
+        return collector.toByteComparable();
+    }
+
+    protected abstract long advanceNode();
+
+    protected enum LeftBoundTreatment
+    {
+        ADMIT_PREFIXES,
+        ADMIT_EXACT,
+        GREATER
+    }
+
+    protected static class IterationPosition
+    {
+        final long node;
+        final int limit;
+        final IterationPosition prev;
+        int childIndex;
+
+        IterationPosition(long node, int childIndex, int limit, IterationPosition prev)
+        {
+            super();
+            this.node = node;
+            this.childIndex = childIndex;
+            this.limit = limit;
+            this.prev = prev;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("[Node %d, child %d, limit %d]", node, childIndex, limit);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalDeepTrieWriterPageAware.java b/src/java/org/apache/cassandra/io/tries/IncrementalDeepTrieWriterPageAware.java
index c4b550c7dd08..ab32ad1b2a2e 100644
--- a/src/java/org/apache/cassandra/io/tries/IncrementalDeepTrieWriterPageAware.java
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalDeepTrieWriterPageAware.java
@@ -26,6 +26,7 @@
 import javax.annotation.concurrent.NotThreadSafe;
 
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 /**
  * This class is a variant of {@link IncrementalTrieWriterPageAware} which is able to build even very deep
@@ -38,19 +39,19 @@
  * and thus stack overflow failures.
  */
 @NotThreadSafe
-public class IncrementalDeepTrieWriterPageAware<VALUE> extends IncrementalTrieWriterPageAware<VALUE>
+class IncrementalDeepTrieWriterPageAware<VALUE> extends IncrementalTrieWriterPageAware<VALUE>
 {
     private final int maxRecursionDepth;
 
-    public IncrementalDeepTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest, int maxRecursionDepth)
+    IncrementalDeepTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest, int maxRecursionDepth, ByteComparable.Version version)
     {
-        super(trieSerializer, dest);
+        super(trieSerializer, dest, version);
         this.maxRecursionDepth = maxRecursionDepth;
     }
 
-    public IncrementalDeepTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest)
+    IncrementalDeepTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest, ByteComparable.Version version)
     {
-        this(trieSerializer, dest, 64);
+        this(trieSerializer, dest, 64, version);
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriter.java b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriter.java
index e2c1e4c845e5..5d1cb2d78216 100644
--- a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriter.java
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriter.java
@@ -75,8 +75,8 @@ interface PartialTail
     /**
      * Construct a suitable trie writer.
      */
-    static <VALUE> IncrementalTrieWriter<VALUE> open(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest)
+    static <VALUE> IncrementalTrieWriter<VALUE> open(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest, ByteComparable.Version version)
     {
-        return new IncrementalDeepTrieWriterPageAware<>(trieSerializer, dest);
+        return new IncrementalDeepTrieWriterPageAware<>(trieSerializer, dest, version);
     }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterBase.java b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterBase.java
index c46099a5d658..fe3ae6ffe655 100644
--- a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterBase.java
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterBase.java
@@ -39,9 +39,11 @@ public abstract class IncrementalTrieWriterBase<VALUE, DEST, NODE extends Increm
     protected final DEST dest;
     protected ByteComparable prev = null;
     long count = 0;
+    protected final ByteComparable.Version version;
 
-    protected IncrementalTrieWriterBase(TrieSerializer<VALUE, ? super DEST> serializer, DEST dest, NODE root)
+    protected IncrementalTrieWriterBase(TrieSerializer<VALUE, ? super DEST> serializer, DEST dest, NODE root, ByteComparable.Version version)
     {
+        this.version = version;
         this.serializer = serializer;
         this.dest = dest;
         this.stack.addLast(root);
@@ -69,20 +71,20 @@ public void add(ByteComparable next, VALUE value) throws IOException
     {
         ++count;
         int stackpos = 0;
-        ByteSource sn = next.asComparableBytes(Walker.BYTE_COMPARABLE_VERSION);
+        ByteSource sn = next.asComparableBytes(version);
         int n = sn.next();
 
         if (prev != null)
         {
-            ByteSource sp = prev.asComparableBytes(Walker.BYTE_COMPARABLE_VERSION);
+            ByteSource sp = prev.asComparableBytes(version);
             int p = sp.next();
             while ( n == p )
             {
                 assert n != ByteSource.END_OF_STREAM : String.format("Incremental trie requires unique sorted keys, got equal %s(%s) after %s(%s).",
                                                                      next,
-                                                                     next.byteComparableAsString(Walker.BYTE_COMPARABLE_VERSION),
+                                                                     next.byteComparableAsString(version),
                                                                      prev,
-                                                                     prev.byteComparableAsString(Walker.BYTE_COMPARABLE_VERSION));
+                                                                     prev.byteComparableAsString(version));
 
                 ++stackpos;
                 n = sn.next();
@@ -90,9 +92,9 @@ public void add(ByteComparable next, VALUE value) throws IOException
             }
             assert p < n : String.format("Incremental trie requires sorted keys, got %s(%s) after %s(%s).",
                                          next,
-                                         next.byteComparableAsString(Walker.BYTE_COMPARABLE_VERSION),
+                                         next.byteComparableAsString(version),
                                          prev,
-                                         prev.byteComparableAsString(Walker.BYTE_COMPARABLE_VERSION));
+                                         prev.byteComparableAsString(version));
         }
         prev = next;
 
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterPageAware.java b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterPageAware.java
index 2274975e48e0..f4b5899132af 100644
--- a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterPageAware.java
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterPageAware.java
@@ -28,6 +28,7 @@
 
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 /**
  * Incremental builders of on-disk tries which packs trie stages into disk cache pages.
@@ -103,9 +104,9 @@ public class IncrementalTrieWriterPageAware<VALUE>
         return c;
     };
 
-    IncrementalTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest)
+    IncrementalTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest, ByteComparable.Version version)
     {
-        super(trieSerializer, dest, new Node<>((byte) 0));
+        super(trieSerializer, dest, new Node<>((byte) 0), version);
         this.maxBytesPerPage = dest.maxBytesInPage();
     }
 
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterSimple.java b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterSimple.java
index 6620b2e1c844..7f34010f8e66 100644
--- a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterSimple.java
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterSimple.java
@@ -23,6 +23,7 @@
 
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 /**
  * Incremental builder of on-disk tries. Takes sorted input.
@@ -44,9 +45,9 @@ public class IncrementalTrieWriterSimple<VALUE>
 {
     private long position = 0;
 
-    public IncrementalTrieWriterSimple(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest)
+    public IncrementalTrieWriterSimple(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest, ByteComparable.Version version)
     {
-        super(trieSerializer, dest, new Node<>((byte) 0));
+        super(trieSerializer, dest, new Node<>((byte) 0), version);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/io/tries/ReverseValueIterator.java b/src/java/org/apache/cassandra/io/tries/ReverseValueIterator.java
index 27c199a68523..7b11ba79c759 100644
--- a/src/java/org/apache/cassandra/io/tries/ReverseValueIterator.java
+++ b/src/java/org/apache/cassandra/io/tries/ReverseValueIterator.java
@@ -28,40 +28,30 @@
  * <p>
  * The main utility of this class is the {@link #nextPayloadedNode()} method, which lists all nodes that contain a
  * payload within the requested bounds. The treatment of the bounds is non-standard (see
- * {@link #ReverseValueIterator(Rebufferer, long, ByteComparable, ByteComparable, boolean)}), necessary to properly walk
- * tries of prefixes and separators.
+ * {@link #ReverseValueIterator(Rebufferer, long, ByteComparable, ByteComparable, LeftBoundTreatment, ByteComparable.Version)}), necessary to
+ * properly walk tries of prefixes and separators.
  */
 @NotThreadSafe
-public class ReverseValueIterator<Concrete extends ReverseValueIterator<Concrete>> extends Walker<Concrete>
+public class ReverseValueIterator<Concrete extends ReverseValueIterator<Concrete>> extends BaseValueIterator<Concrete>
 {
     static final int NOT_AT_LIMIT = Integer.MIN_VALUE;
-    private final ByteSource limit;
-    private IterationPosition stack;
-    private long next;
-    private boolean reportingPrefixes;
+    private LeftBoundTreatment reportingPrefixes;
+    private boolean popOnAdvance = true;
 
-    static class IterationPosition
+    protected ReverseValueIterator(Rebufferer source, long root, ByteComparable.Version version)
     {
-        final long node;
-        final int limit;
-        final IterationPosition prev;
-        int childIndex;
+        this(source, root, false, version);
+    }
 
-        public IterationPosition(long node, int childIndex, int limit, IterationPosition prev)
-        {
-            super();
-            this.node = node;
-            this.childIndex = childIndex;
-            this.limit = limit;
-            this.prev = prev;
-        }
+    protected ReverseValueIterator(Rebufferer source, long root, boolean collecting, ByteComparable.Version version)
+    {
+        super(source, root, null, collecting, version);
+        initializeNoRightBound(root, NOT_AT_LIMIT, LeftBoundTreatment.GREATER);
     }
 
-    protected ReverseValueIterator(Rebufferer source, long root)
+    protected ReverseValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, LeftBoundTreatment admitPrefix, ByteComparable.Version version)
     {
-        super(source, root);
-        limit = null;
-        initializeNoRightBound(root, NOT_AT_LIMIT, false);
+        this(source, root, start, end, admitPrefix, false, version);
     }
 
     /**
@@ -75,18 +65,17 @@ protected ReverseValueIterator(Rebufferer source, long root)
      * </ul>
      * This behaviour is shared with the forward counterpart {@link ValueIterator}.
      */
-    protected ReverseValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, boolean admitPrefix)
+    protected ReverseValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, LeftBoundTreatment admitPrefix, boolean collecting, ByteComparable.Version version)
     {
-        super(source, root);
-        limit = start != null ? start.asComparableBytes(BYTE_COMPARABLE_VERSION) : null;
+        super(source, root, start != null ? start.asComparableBytes(version) : null, collecting, version);
 
         if (end != null)
-            initializeWithRightBound(root, end.asComparableBytes(BYTE_COMPARABLE_VERSION), admitPrefix, limit != null);
+            initializeWithRightBound(root, end.asComparableBytes(version), admitPrefix, limit != null);
         else
             initializeNoRightBound(root, limit != null ? limit.next() : NOT_AT_LIMIT, admitPrefix);
     }
 
-    void initializeWithRightBound(long root, ByteSource endStream, boolean admitPrefix, boolean hasLimit)
+    void initializeWithRightBound(long root, ByteSource endStream, LeftBoundTreatment admitPrefix, boolean hasLimit)
     {
         IterationPosition prev = null;
         boolean atLimit = hasLimit;
@@ -112,37 +101,31 @@ void initializeWithRightBound(long root, ByteSource endStream, boolean admitPref
                 break;
 
             prev = new IterationPosition(position, childIndex, limitByte, prev);
+            if (collector != null)
+                collector.add(s);
             go(transition(childIndex)); // childIndex is positive, this transition must exist
         }
 
         // Advancing now gives us first match.
         childIndex = -1 - childIndex;
         stack = new IterationPosition(position, childIndex, limitByte, prev);
-        next = advanceNode();
+        next = NOT_PREPARED;
+        popOnAdvance = false;
     }
 
-    private void initializeNoRightBound(long root, int limitByte, boolean admitPrefix)
+    private void initializeNoRightBound(long root, int limitByte, LeftBoundTreatment admitPrefix)
     {
         go(root);
         stack = new IterationPosition(root, -1 - search(256), limitByte, null);
-        next = advanceNode();
+        if (hasPayload())
+            next = root;
+        else
+            next = NOT_PREPARED;
+        popOnAdvance = false;
         reportingPrefixes = admitPrefix;
     }
 
-
-
-    /**
-     * Returns the position of the next node with payload contained in the iterated span.
-     */
-    protected long nextPayloadedNode()
-    {
-        long toReturn = next;
-        if (next != -1)
-            next = advanceNode();
-        return toReturn;
-    }
-
-    long advanceNode()
+    protected long advanceNode()
     {
         if (stack == null)
             return -1;
@@ -150,6 +133,15 @@ long advanceNode()
         long child;
         int transitionByte;
 
+        if (collector != null)
+        {
+            // We need to pop the last character unless we have not yet advanced to an entry.
+            if (popOnAdvance)
+                collector.pop();
+            else
+                popOnAdvance = true;
+        }
+
         go(stack.node);
         while (true)
         {
@@ -163,7 +155,7 @@ long advanceNode()
                 if (beyondLimit)
                 {
                     assert stack.limit >= 0;    // we are at a limit position (not in a node that's completely within the span)
-                    reportingPrefixes = false;  // there exists a smaller child than limit, no longer should report prefixes
+                    reportingPrefixes = null;  // there exists a smaller child than limit, no longer should report prefixes
                 }
             }
             else
@@ -182,16 +174,19 @@ long advanceNode()
                     // If we are fully inside the covered space, report.
                     // Note that on the exact match of the limit, stackTop.limit would be END_OF_STREAM.
                     // This comparison rejects the exact match; if we wanted to include it, we could test < 0 instead.
-                    if (stackTop.limit == NOT_AT_LIMIT)
+                    if (stackTop.limit == NOT_AT_LIMIT || stackTop.limit == ByteSource.END_OF_STREAM && reportingPrefixes == LeftBoundTreatment.ADMIT_EXACT)
                         return stackTop.node;
-                    else if (reportingPrefixes)
+                    else if (reportingPrefixes == LeftBoundTreatment.ADMIT_PREFIXES)
                     {
-                        reportingPrefixes = false; // if we are at limit position only report one prefix, the closest
+                        reportingPrefixes = null; // if we are at limit position only report one prefix, the closest
                         return stackTop.node;
                     }
                     // else skip this payload
                 }
 
+                if (collector != null)
+                    collector.pop();
+
                 if (stack == null)        // exhausted whole trie
                     return NONE;
                 go(stack.node);
@@ -211,6 +206,8 @@ else if (reportingPrefixes)
                     l = limit.next();
 
                 stack = new IterationPosition(child, transitionRange(), l, stack);
+                if (collector != null)
+                    collector.add(transitionByte);
             }
             else
             {
diff --git a/src/java/org/apache/cassandra/io/tries/ValueIterator.java b/src/java/org/apache/cassandra/io/tries/ValueIterator.java
index 6ddbebad2a2d..16f3fcde94a3 100644
--- a/src/java/org/apache/cassandra/io/tries/ValueIterator.java
+++ b/src/java/org/apache/cassandra/io/tries/ValueIterator.java
@@ -28,133 +28,56 @@
  * <p>
  * The main utility of this class is the {@link #nextPayloadedNode()} method, which lists all nodes that contain a
  * payload within the requested bounds. The treatment of the bounds is non-standard (see
- * {@link #ValueIterator(Rebufferer, long, ByteComparable, ByteComparable, boolean)}), necessary to properly walk
- * tries of prefixes and separators.
+ * {@link #ValueIterator(Rebufferer, long, ByteComparable, ByteComparable, LeftBoundTreatment, boolean, ByteComparable.Version)}), necessary to
+ * properly walk tries of prefixes and separators.
  */
 @NotThreadSafe
-public class ValueIterator<CONCRETE extends ValueIterator<CONCRETE>> extends Walker<CONCRETE>
+public class ValueIterator<CONCRETE extends ValueIterator<CONCRETE>> extends BaseValueIterator<CONCRETE>
 {
-    private final ByteSource limit;
-    private final TransitionBytesCollector collector;
-    protected IterationPosition stack;
-    private long next;
 
-    public static class IterationPosition
+    protected ValueIterator(Rebufferer source, long root, ByteComparable.Version version)
     {
-        final long node;
-        final int limit;
-        final IterationPosition prev;
-        int childIndex;
-
-        public IterationPosition(long node, int childIndex, int limit, IterationPosition prev)
-        {
-            super();
-            this.node = node;
-            this.childIndex = childIndex;
-            this.limit = limit;
-            this.prev = prev;
-        }
-
-        @Override
-        public String toString()
-        {
-            return String.format("[Node %d, child %d, limit %d]", node, childIndex, limit);
-        }
+        this(source, root, false, version);
     }
 
-    protected ValueIterator(Rebufferer source, long root)
+    protected ValueIterator(Rebufferer source, long root, boolean collecting, ByteComparable.Version version)
     {
-        this(source, root, false);
-    }
-    
-    protected ValueIterator(Rebufferer source, long root, boolean collecting)
-    {
-        super(source, root);
-        limit = null;
-        collector = collecting ? new TransitionBytesCollector() : null;
+        super(source, root, null, true, version);
         initializeNoLeftBound(root, 256);
     }
 
-    protected ValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, boolean admitPrefix)
+    protected ValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, LeftBoundTreatment admitPrefix, ByteComparable.Version version)
     {
-        this(source, root, start, end, admitPrefix, false);
+        this(source, root, start, end, admitPrefix, false, version);
     }
 
     /**
-     * Constrained iterator. The end position is always treated as inclusive, and we have two possible treatments for
-     * the start:
+     * Constrained iterator. The end position is always treated as inclusive, and we have three possible treatments for
+     * the start, specified in admitPrefix:
      * <ul>
-     *   <li> When {@code admitPrefix=false}, exact matches and any prefixes of the start are excluded.
-     *   <li> When {@code admitPrefix=true}, the longest prefix of the start present in the trie is also included,
+     *   <li> When {@code GREATER}, exact matches and any prefixes of the start are excluded.
+     *   <li> When {@code ADMIT_EXACT}, exact matches are included.
+     *   <li> When {@code ADMIT_PREFIXES}, the longest prefix of the start present in the trie is also included,
      *        provided that there is no entry in the trie between that prefix and the start. An exact match also
      *        satisfies this and is included.
      * </ul>
      * This behaviour is shared with the reverse counterpart {@link ReverseValueIterator}.
      */
-    protected ValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, boolean admitPrefix, boolean collecting)
+    protected ValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, LeftBoundTreatment admitPrefix, boolean collecting, ByteComparable.Version version)
     {
-        super(source, root);
-        limit = end != null ? end.asComparableBytes(BYTE_COMPARABLE_VERSION) : null;
-        collector = collecting ? new TransitionBytesCollector() : null;
+        super(source, root, end != null ? end.asComparableBytes(version) : null, collecting, version);
 
         if (start != null)
-            initializeWithLeftBound(root, start.asComparableBytes(BYTE_COMPARABLE_VERSION), admitPrefix, limit != null);
+            initializeWithLeftBound(root, start.asComparableBytes(byteComparableVersion), admitPrefix, limit != null);
         else
             initializeNoLeftBound(root, limit != null ? limit.next() : 256);
     }
 
-    private void initializeWithLeftBound(long root, ByteSource startStream, boolean admitPrefix, boolean atLimit)
+    private void initializeWithLeftBound(long root, ByteSource start, LeftBoundTreatment admitPrefix, boolean atLimit)
     {
-        IterationPosition prev = null;
-        int childIndex;
-        int limitByte;
-        long payloadedNode = -1;
-
         try
         {
-            // Follow start position while we still have a prefix, stacking path and saving prefixes.
-            go(root);
-            while (true)
-            {
-                int s = startStream.next();
-                childIndex = search(s);
-
-                // For a separator trie the latest payload met along the prefix is a potential match for start
-                if (admitPrefix)
-                {
-                    if (childIndex == 0 || childIndex == -1)
-                    {
-                        if (hasPayload())
-                            payloadedNode = position;
-                    }
-                    else
-                    {
-                        payloadedNode = -1;
-                    }
-                }
-
-                limitByte = 256;
-                if (atLimit)
-                {
-                    limitByte = limit.next();
-                    if (s < limitByte)
-                        atLimit = false;
-                }
-                if (childIndex < 0)
-                    break;
-
-                prev = new IterationPosition(position, childIndex, limitByte, prev);
-                go(transition(childIndex)); // child index is positive, transition must exist
-            }
-
-            childIndex = -1 - childIndex - 1;
-            stack = new IterationPosition(position, childIndex, limitByte, prev);
-
-            // Advancing now gives us first match if we didn't find one already.
-            if (payloadedNode != -1)
-                next = payloadedNode;
-            else
-                next = advanceNode();
+            descendWith(start.next(), start, atLimit ? limit.next() : 256, null, root, admitPrefix);
         }
         catch (Throwable t)
         {
@@ -173,7 +96,7 @@ private void initializeNoLeftBound(long root, int limitByte)
             if (hasPayload())
                 next = root;
             else
-                next = advanceNode();
+                next = NOT_PREPARED;
         }
         catch (Throwable t)
         {
@@ -183,28 +106,120 @@ private void initializeNoLeftBound(long root, int limitByte)
     }
 
     /**
-     * Returns the payload node position without advancing.
+     * Skip to the given key or the closest after it in iteration order. Inclusive when admitPrefix = ADMIT_EXACT,
+     * exclusive when GREATER (ADMIT_PREFIXES is not supported).
+     * Requires that the iterator is collecting bytes.
+     * To get the next entry, use getNextPayloadedNode as normal.
      */
-    protected long peekNode()
+    protected void skipTo(ByteComparable skipTo, LeftBoundTreatment admitPrefix)
     {
-        return next;
+        assert skipTo != null;
+        assert collector != null : "Cannot skip without collecting bytes";
+        // TODO: Figure out what you need to know to say if an earlier prefix would still be acceptable
+        // to support skipping with ADMIT_PREFIXES.
+        assert admitPrefix != LeftBoundTreatment.ADMIT_PREFIXES : "Skipping with ADMIT_PREFIXES is not supported";
+        if (stack == null)
+            return; // exhausted whole trie
+        ByteSource skipToBytes = skipTo.asComparableBytes(byteComparableVersion);
+        int pos;
+        int nextByte = skipToBytes.next();
+        final int collectedLength = collector.pos;
+        final byte[] collectedBytes = collector.bytes;
+        for (pos = 0; pos < collectedLength; ++pos)
+        {
+            if (nextByte != collectedBytes[pos])
+            {
+                if (nextByte < collectedBytes[pos])
+                    return;    // the position we are already advanced to is beyond skipTo
+                else
+                    break;     // matched a prefix of skipTo, now we need to advance through the rest of it
+            }
+            nextByte = skipToBytes.next();
+        }
+        int upLevels = collectedLength - pos;
+        IterationPosition stack = this.stack;
+        for (int i = 0; i < upLevels; ++i)
+            stack = stack.prev;
+        collector.pos = pos;
+
+        descendWith(nextByte, skipToBytes, stack.limit, stack.prev, stack.node, admitPrefix);
     }
 
-    /**
-     * Returns the position of the next node with payload contained in the iterated span.
-     */
-    protected long nextPayloadedNode()
+    private void descendWith(int skipToFirstByte, ByteSource skipToRest, int limitByte, IterationPosition stackPrev, long startNode, LeftBoundTreatment admitPrefix)
     {
-        long toReturn = next;
-        if (next != -1)
-            next = advanceNode();
-        return toReturn;
+        int childIndex;
+        long payloadedNode = NOT_PREPARED;
+        // Follow start position while we still have a prefix, stacking path and saving prefixes.
+        go(startNode);
+        while (true)
+        {
+            childIndex = search(skipToFirstByte);
+
+            // For a separator trie the latest payload met along the prefix is a potential match for start
+            payloadedNode = maybeCollectPayloadedNode(admitPrefix, childIndex, payloadedNode);
+
+            if (childIndex < 0)
+                break;
+
+            stackPrev = new IterationPosition(position, childIndex, limitByte, stackPrev);
+            if (collector != null)
+                collector.add(skipToFirstByte);
+            go(transition(childIndex)); // child index is positive, transition must exist
+
+            if (limitByte < 256)
+            {
+                if (skipToFirstByte == limitByte)
+                    limitByte = limit.next();
+                else if (skipToFirstByte < limitByte)
+                    limitByte = 256;
+                else // went beyond the limit
+                {
+                    stack = null;
+                    next = NONE;
+                    return;
+                }
+            }
+            skipToFirstByte = skipToRest.next();
+        }
+
+        childIndex = -1 - childIndex - 1;
+        stack = new IterationPosition(position, childIndex, limitByte, stackPrev);
+
+        // Advancing now gives us first match if we didn't find one already.
+        next = maybeAcceptPayloadedNode(admitPrefix, skipToFirstByte, payloadedNode);
+    }
+
+    private long maybeAcceptPayloadedNode(LeftBoundTreatment admitPrefix, int trailingByte, long payloadedNode)
+    {
+        switch (admitPrefix)
+        {
+            case ADMIT_PREFIXES:
+                return payloadedNode;
+            case ADMIT_EXACT:
+                if (trailingByte == ByteSource.END_OF_STREAM && hasPayload())
+                    return position;
+                // else fall through
+            case GREATER:
+            default:
+                return NOT_PREPARED;
+        }
     }
 
-    protected ByteComparable nextCollectedValue()
+    private long maybeCollectPayloadedNode(LeftBoundTreatment admitPrefix, int childIndex, long payloadedNode)
     {
-        assert collector != null : "Cannot get a collected value from a non-collecting iterator";
-        return collector.toByteComparable();
+        if (admitPrefix == LeftBoundTreatment.ADMIT_PREFIXES)
+        {
+            if (childIndex == 0 || childIndex == -1)
+            {
+                if (hasPayload())
+                    payloadedNode = position;
+            }
+            else
+            {
+                payloadedNode = NOT_PREPARED;
+            }
+        }
+        return payloadedNode;
     }
 
     protected long advanceNode()
@@ -225,7 +240,7 @@ protected long advanceNode()
                 if (collector != null)
                     collector.pop();
                 if (stack == null)        // exhausted whole trie
-                    return -1;
+                    return NONE;
                 go(stack.node);
                 continue;
             }
diff --git a/src/java/org/apache/cassandra/io/tries/Walker.java b/src/java/org/apache/cassandra/io/tries/Walker.java
index c2b28a6db715..20db06c344cd 100644
--- a/src/java/org/apache/cassandra/io/tries/Walker.java
+++ b/src/java/org/apache/cassandra/io/tries/Walker.java
@@ -20,13 +20,13 @@
 import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.ByteBuffer;
-import java.util.Arrays;
 import javax.annotation.concurrent.NotThreadSafe;
 
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.PageAware;
 import org.apache.cassandra.io.util.Rebufferer;
 import org.apache.cassandra.io.util.Rebufferer.BufferHolder;
+import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.lucene.util.ArrayUtil;
@@ -44,9 +44,9 @@
 public class Walker<CONCRETE extends Walker<CONCRETE>> implements AutoCloseable
 {
     /** Value used to indicate a branch (e.g. lesser/greaterBranch) does not exist. */
-    public static int NONE = TrieNode.NONE;
+    public static int NONE = -1;
 
-    private final Rebufferer source;
+    protected final Rebufferer source;
     protected final long root;
 
     // State relating to current node.
@@ -60,16 +60,17 @@ public class Walker<CONCRETE extends Walker<CONCRETE>> implements AutoCloseable
     protected long greaterBranch;
     protected long lesserBranch;
 
-    // Version of the byte comparable conversion to use
-    public static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS50;
+    // Version of the byte comparable conversion to use -- trie-based indices use the 6.0 conversion
+    public final ByteComparable.Version byteComparableVersion;
 
     /**
      * Creates a walker. Rebufferer must be aligned and with a buffer size that is at least 4k.
      */
-    public Walker(Rebufferer source, long root)
+    public Walker(Rebufferer source, long root, ByteComparable.Version version)
     {
         this.source = source;
         this.root = root;
+        this.byteComparableVersion = version;
         try
         {
             bh = source.rebuffer(root);
@@ -201,7 +202,7 @@ public interface Extractor<RESULT, VALUE>
      */
     public int follow(ByteComparable key)
     {
-        ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+        ByteSource stream = key.asComparableBytes(byteComparableVersion);
         go(root);
         while (true)
         {
@@ -226,7 +227,7 @@ public int followWithGreater(ByteComparable key)
     {
         greaterBranch = NONE;
 
-        ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+        ByteSource stream = key.asComparableBytes(byteComparableVersion);
         go(root);
         while (true)
         {
@@ -252,7 +253,7 @@ public int followWithLesser(ByteComparable key)
     {
         lesserBranch = NONE;
 
-        ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+        ByteSource stream = key.asComparableBytes(byteComparableVersion);
         go(root);
         while (true)
         {
@@ -282,7 +283,7 @@ public <RESULT> RESULT prefix(ByteComparable key, Extractor<RESULT, CONCRETE> ex
     {
         RESULT payload = null;
 
-        ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+        ByteSource stream = key.asComparableBytes(byteComparableVersion);
         go(root);
         while (true)
         {
@@ -321,7 +322,7 @@ public <RESULT> RESULT prefixAndNeighbours(ByteComparable key, Extractor<RESULT,
         greaterBranch = NONE;
         lesserBranch = NONE;
 
-        ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+        ByteSource stream = key.asComparableBytes(byteComparableVersion);
         go(root);
         while (true)
         {
@@ -351,7 +352,7 @@ public <RESULT> RESULT prefixAndNeighbours(ByteComparable key, Extractor<RESULT,
 
     public ByteComparable getMaxTerm()
     {
-        TransitionBytesCollector collector = new TransitionBytesCollector();
+        TransitionBytesCollector collector = new TransitionBytesCollector(byteComparableVersion);
         go(root);
         while (true)
         {
@@ -368,7 +369,7 @@ public ByteComparable getMaxTerm()
 
     public ByteComparable getMinTerm()
     {
-        TransitionBytesCollector collector = new TransitionBytesCollector();
+        TransitionBytesCollector collector = new TransitionBytesCollector(byteComparableVersion);
         go(root);
         while (true)
         {
@@ -434,9 +435,15 @@ public String toString()
 
     public static class TransitionBytesCollector
     {
+        private final ByteComparable.Version byteComparableVersion;
         protected byte[] bytes = new byte[32];
         protected int pos = 0;
 
+        public TransitionBytesCollector(ByteComparable.Version byteComparableVersion)
+        {
+            this.byteComparableVersion = byteComparableVersion;
+        }
+
         public void add(int b)
         {
             if (pos == bytes.length)
@@ -454,17 +461,17 @@ public void pop()
 
         public ByteComparable toByteComparable()
         {
-            if (pos <= 0)
+            if (pos < 0)
                 return null;
             byte[] value = new byte[pos];
             System.arraycopy(bytes, 0, value, 0, pos);
-            return v -> ByteSource.fixedLength(value, 0, value.length);
+            return ByteComparable.preencoded(byteComparableVersion, value, 0, value.length);
         }
 
         @Override
         public String toString()
         {
-            return String.format("[Bytes %s, pos %d]", Arrays.toString(bytes), pos);
+            return ByteBufferUtil.bytesToHex(ByteBuffer.wrap(bytes, 0, pos));
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/io/util/BufferManagingRebufferer.java b/src/java/org/apache/cassandra/io/util/BufferManagingRebufferer.java
index 3a297ee0e260..4a978f42e6e2 100644
--- a/src/java/org/apache/cassandra/io/util/BufferManagingRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/BufferManagingRebufferer.java
@@ -25,12 +25,14 @@
 
 import org.apache.cassandra.utils.memory.BufferPools;
 
-/**
- * Buffer manager used for reading from a ChunkReader when cache is not in use. Instances of this class are
- * reader-specific and thus do not need to be thread-safe since the reader itself isn't.
- *
- * The instances reuse themselves as the BufferHolder to avoid having to return a new object for each rebuffer call.
- */
+/// Buffer manager used for reading from a [ChunkReader] when cache is not in use. They use a buffer produced by the
+/// "networking" buffer pool, which is the one to be used for buffers that are not to be retained for a long time
+/// (the lifetime of this object is contained by the lifetime of a [RandomAccessReader] which is contained in a read
+/// operation's lifetime).
+///
+/// Instances of this class are reader-specific and thus do not need to be thread-safe since the reader itself isn't.
+///
+/// The instances reuse themselves as the BufferHolder to avoid having to return a new object for each rebuffer call.
 public abstract class BufferManagingRebufferer implements Rebufferer, Rebufferer.BufferHolder
 {
     protected final ChunkReader source;
@@ -42,14 +44,20 @@ public abstract class BufferManagingRebufferer implements Rebufferer, Rebufferer
     protected BufferManagingRebufferer(ChunkReader wrapped)
     {
         this.source = wrapped;
-        buffer = BufferPools.forChunkCache().get(wrapped.chunkSize(), wrapped.preferredBufferType()).order(ByteOrder.BIG_ENDIAN);
+        // Note: This class uses the networking buffer pool which makes better sense for short-lifetime buffers.
+        // Because this is meant to be used when the chunk cache is disabled, it also makes sense to use any memory
+        // that may have been allocated for in-flight data by using the chunk-cache pool.
+        // However, if some new functionality decides to use this class in the presence of the chunk cache (e.g.
+        // cache-bypassing compaction), using the chunk-cache pool here will certainly cause hard-to-diagnose issues
+        // that we would prefer to avoid.
+        buffer = BufferPools.forNetworking().get(wrapped.chunkSize(), wrapped.preferredBufferType()).order(ByteOrder.BIG_ENDIAN);
         buffer.limit(0);
     }
 
     @Override
     public void closeReader()
     {
-        BufferPools.forChunkCache().put(buffer);
+        BufferPools.forNetworking().put(buffer);
         offset = -1;
     }
 
@@ -96,7 +104,7 @@ public String toString()
 
     public ByteBuffer buffer()
     {
-        return buffer;
+        return buffer.duplicate();
     }
 
     public long offset()
diff --git a/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java b/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
index a712ba6e4c18..d6c1b2d98fc7 100644
--- a/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
+++ b/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
@@ -162,22 +162,6 @@ public void writeByte(int v) throws IOException
         write(v);
     }
 
-    @Override
-    public void writeMostSignificantBytes(long register, int bytes) throws IOException
-    {
-        assert buffer != null : "Attempt to use a closed data output";
-        if (buffer.remaining() < Long.BYTES)
-        {
-            super.writeMostSignificantBytes(register, bytes);
-        }
-        else
-        {
-            int pos = buffer.position();
-            buffer.putLong(pos, register);
-            buffer.position(pos + bytes);
-        }
-    }
-
     @Override
     public void writeShort(int v) throws IOException
     {
@@ -227,7 +211,7 @@ public void writeDouble(double v) throws IOException
     }
 
     @DontInline
-    private void writeSlow(long bytes, int count) throws IOException
+    protected void writeSlow(long bytes, int count) throws IOException
     {
         assert buffer != null : "Attempt to use a closed data output";
         int origCount = count;
diff --git a/src/java/org/apache/cassandra/io/util/ChannelProxy.java b/src/java/org/apache/cassandra/io/util/ChannelProxy.java
index 81665beecdb9..99e10dd07762 100644
--- a/src/java/org/apache/cassandra/io/util/ChannelProxy.java
+++ b/src/java/org/apache/cassandra/io/util/ChannelProxy.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.io.util;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
@@ -25,7 +26,7 @@
 import java.nio.file.StandardOpenOption;
 
 import org.apache.cassandra.io.FSReadError;
-import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.INativeLibrary;
 import org.apache.cassandra.utils.concurrent.RefCounted;
 import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
 
@@ -52,15 +53,10 @@ public static FileChannel openChannel(File file)
         }
         catch (IOException e)
         {
-            throw new RuntimeException(e);
+            throw new UncheckedIOException(e);
         }
     }
 
-    public ChannelProxy(String path)
-    {
-        this (new File(path));
-    }
-
     public ChannelProxy(File file)
     {
         this(file, openChannel(file));
@@ -120,7 +116,7 @@ public void tidy()
      */
     public final ChannelProxy newChannel()
     {
-        return new ChannelProxy(filePath);
+        return new ChannelProxy(file);
     }
 
     public ChannelProxy sharedCopy()
@@ -130,7 +126,12 @@ public ChannelProxy sharedCopy()
 
     public String filePath()
     {
-        return filePath;
+        return file.path();
+    }
+
+    public File getFile()
+    {
+        return file;
     }
 
     public File file()
@@ -147,7 +148,7 @@ public int read(ByteBuffer buffer, long position)
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, filePath);
+            throw new FSReadError(e, filePath());
         }
     }
 
@@ -159,7 +160,7 @@ public long transferTo(long position, long count, WritableByteChannel target)
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, filePath);
+            throw new FSReadError(e, filePath());
         }
     }
 
@@ -171,7 +172,7 @@ public MappedByteBuffer map(FileChannel.MapMode mode, long position, long size)
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, filePath);
+            throw new FSReadError(e, filePath());
         }
     }
 
@@ -183,13 +184,17 @@ public long size()
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, filePath);
+            throw new FSReadError(e, filePath());
         }
     }
 
-    public int getFileDescriptor()
+    /**
+     * Apply FADV_DONTNEED to the file region.
+     */
+    public void trySkipCache(long offset, long length)
     {
-        return NativeLibrary.getfd(channel);
+        int fd = INativeLibrary.instance.getfd(channel);
+        INativeLibrary.instance.trySkipCache(fd, offset, length, file.absolutePath());
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/io/util/ChecksumWriter.java b/src/java/org/apache/cassandra/io/util/ChecksumWriter.java
index 194602c550ea..d69a40b5f52c 100644
--- a/src/java/org/apache/cassandra/io/util/ChecksumWriter.java
+++ b/src/java/org/apache/cassandra/io/util/ChecksumWriter.java
@@ -28,6 +28,7 @@
 import javax.annotation.Nonnull;
 
 import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.compress.CompressedSequentialWriter;
 
 public class ChecksumWriter
 {
@@ -89,10 +90,19 @@ public void appendDirect(ByteBuffer bb, boolean checksumIncrementalResult)
     }
 
     public void writeFullChecksum(@Nonnull File digestFile)
+    {
+        writeFullChecksum(digestFile, fullChecksum.getValue());
+    }
+
+    /**
+     * Write given checksum into the digest file. This is used when {@link CompressedSequentialWriter} is reset and truncated,
+     * and we need to recompute digest for the whole file.
+     */
+    public static void writeFullChecksum(@Nonnull File digestFile, long checksum)
     {
         try (FileOutputStreamPlus fos = new FileOutputStreamPlus(digestFile))
         {
-            fos.write(String.valueOf(fullChecksum.getValue()).getBytes(StandardCharsets.UTF_8));
+            fos.write(String.valueOf(checksum).getBytes(StandardCharsets.UTF_8));
             fos.flush();
             fos.getChannel().force(true);
         }
diff --git a/src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java b/src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java
index 8f0206e2a014..4780f02f9591 100644
--- a/src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java
+++ b/src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java
@@ -23,16 +23,21 @@
 
 public final class ChecksummedRandomAccessReader
 {
-    @SuppressWarnings({ "resource", "RedundantSuppression" }) // The Rebufferer owns both the channel and the validator and handles closing both.
     public static RandomAccessReader open(File file, File crcFile) throws IOException
+    {
+        return open(file, crcFile, 0);
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"}) // The Rebufferer owns both the channel and the validator and handles closing both.
+    public static RandomAccessReader open(File file, File crcFile, long startOffset) throws IOException
     {
         ChannelProxy channel = new ChannelProxy(file);
         try
         {
             DataIntegrityMetadata.ChecksumValidator validator = new DataIntegrityMetadata.ChecksumValidator(ChecksumType.CRC32,
                                                                                                             RandomAccessReader.open(crcFile),
-                                                                                                            file.path());
-            Rebufferer rebufferer = new ChecksummedRebufferer(channel, validator);
+                                                                                                            file);
+            Rebufferer rebufferer = new ChecksummedRebufferer(channel, validator, startOffset);
             return new RandomAccessReader.RandomAccessReaderWithOwnChannel(rebufferer);
         }
         catch (Throwable t)
diff --git a/src/java/org/apache/cassandra/io/util/ChecksummedRebufferer.java b/src/java/org/apache/cassandra/io/util/ChecksummedRebufferer.java
index bc8695fd24d6..af2e805c89cd 100644
--- a/src/java/org/apache/cassandra/io/util/ChecksummedRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/ChecksummedRebufferer.java
@@ -27,9 +27,9 @@ class ChecksummedRebufferer extends BufferManagingRebufferer
 {
     private final DataIntegrityMetadata.ChecksumValidator validator;
 
-    ChecksummedRebufferer(ChannelProxy channel, DataIntegrityMetadata.ChecksumValidator validator)
+    ChecksummedRebufferer(ChannelProxy channel, DataIntegrityMetadata.ChecksumValidator validator, long startOffset)
     {
-        super(new SimpleChunkReader(channel, channel.size(), BufferType.ON_HEAP, validator.chunkSize));
+        super(new SimpleChunkReader(channel, channel.size(), BufferType.ON_HEAP, validator.chunkSize, startOffset));
         this.validator = validator;
     }
 
diff --git a/src/java/org/apache/cassandra/io/util/ChunkReader.java b/src/java/org/apache/cassandra/io/util/ChunkReader.java
index 33bf7921edd6..437c65603bdf 100644
--- a/src/java/org/apache/cassandra/io/util/ChunkReader.java
+++ b/src/java/org/apache/cassandra/io/util/ChunkReader.java
@@ -34,7 +34,11 @@ public interface ChunkReader extends RebuffererFactory
      * Read the chunk at the given position, attempting to fill the capacity of the given buffer.
      * The filled buffer must be positioned at 0, with limit set at the size of the available data.
      * The source may have requirements for the positioning and/or size of the buffer (e.g. chunk-aligned and
-     * chunk-sized). These must be satisfied by the caller. 
+     * chunk-sized). These must be satisfied by the caller.
+     * <p/>
+     * If the reader is created for a partial file described by {@link SliceDescriptor}, the provided position refers
+     * to the original file, not the slice, that is, the caller can provide only the position from the range of the
+     * slice (i.e. {@link SliceDescriptor#sliceStart} (incl) ... {@link SliceDescriptor#sliceEnd} (excl)).
      */
     void readChunk(long position, ByteBuffer buffer);
 
@@ -48,4 +52,17 @@ public interface ChunkReader extends RebuffererFactory
      * This is not guaranteed to be fulfilled.
      */
     BufferType preferredBufferType();
+
+    /**
+     * In some cases we may end up with both compressed and uncompressed data for the same file in
+     * the cache. This type is used to distinguish between them.
+     */
+    enum ReaderType
+    {
+        SIMPLE,
+        COMPRESSED;
+        /** The number of types. Declared as a constant to avoid allocating on values(). */
+        public static final int COUNT = ReaderType.values().length;
+    }
+    ReaderType type();
 }
diff --git a/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java b/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java
index b0aa24bd8f6f..164886752c04 100644
--- a/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java
+++ b/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java
@@ -30,21 +30,27 @@
 import org.apache.cassandra.io.compress.CompressionMetadata;
 import org.apache.cassandra.io.compress.CorruptBlockException;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.utils.ChecksumType;
+import org.apache.cassandra.utils.memory.BufferPools;
 
 public abstract class CompressedChunkReader extends AbstractReaderFileProxy implements ChunkReader
 {
     final CompressionMetadata metadata;
     final int maxCompressedLength;
     final Supplier<Double> crcCheckChanceSupplier;
+    protected final long startOffset;
+    protected final long onDiskStartOffset;
 
-    protected CompressedChunkReader(ChannelProxy channel, CompressionMetadata metadata, Supplier<Double> crcCheckChanceSupplier)
+    protected CompressedChunkReader(ChannelProxy channel, CompressionMetadata metadata, Supplier<Double> crcCheckChanceSupplier, long startOffset)
     {
-        super(channel, metadata.dataLength);
+        super(channel, metadata.dataLength + startOffset);
         this.metadata = metadata;
         this.maxCompressedLength = metadata.maxCompressedLength();
         this.crcCheckChanceSupplier = crcCheckChanceSupplier;
+        this.startOffset = startOffset;
         assert Integer.bitCount(metadata.chunkLength()) == 1; //must be a power of two
+        this.onDiskStartOffset = startOffset == 0 ? 0 : metadata.chunkFor(startOffset).offset;
     }
 
     @VisibleForTesting
@@ -62,12 +68,15 @@ boolean shouldCheckCrc()
     @Override
     public String toString()
     {
-        return String.format("CompressedChunkReader.%s(%s - %s, chunk length %d, data length %d)",
+        return String.format(startOffset > 0
+                             ? "CompressedChunkReader.%s(%s - %s, chunk length %d, data length %d, slice offset %s)"
+                             : "CompressedChunkReader.%s(%s - %s, chunk length %d, data length %d)",
                              getClass().getSimpleName(),
                              channel.filePath(),
                              metadata.compressor().getClass().getSimpleName(),
                              metadata.chunkLength(),
-                             metadata.dataLength);
+                             metadata.dataLength,
+                             startOffset);
     }
 
     @Override
@@ -88,15 +97,22 @@ public Rebufferer instantiateRebufferer()
         return new BufferManagingRebufferer.Aligned(this);
     }
 
+    public ReaderType type()
+    {
+        return ReaderType.COMPRESSED;
+    }
+
     public static class Standard extends CompressedChunkReader
     {
         // we read the raw compressed bytes into this buffer, then uncompressed them into the provided one.
-        private final ThreadLocalByteBufferHolder bufferHolder;
-
         public Standard(ChannelProxy channel, CompressionMetadata metadata, Supplier<Double> crcCheckChanceSupplier)
         {
-            super(channel, metadata, crcCheckChanceSupplier);
-            bufferHolder = new ThreadLocalByteBufferHolder(metadata.compressor().preferredBufferType());
+            this(channel, metadata, crcCheckChanceSupplier, 0);
+        }
+
+        public Standard(ChannelProxy channel, CompressionMetadata metadata, Supplier<Double> crcCheckChanceSupplier, long startOffset)
+        {
+            super(channel, metadata, crcCheckChanceSupplier, startOffset);
         }
 
         @Override
@@ -113,64 +129,72 @@ public void readChunk(long position, ByteBuffer uncompressed)
                 int length = shouldCheckCrc ? chunk.length + Integer.BYTES // compressed length + checksum length
                                             : chunk.length;
 
-                if (chunk.length < maxCompressedLength)
+                long chunkOffset = chunk.offset - onDiskStartOffset;
+                boolean shouldDecompress = chunk.length < maxCompressedLength;
+                if (shouldDecompress || shouldCheckCrc) // when we need to read the CRC too, follow the decompression path to avoid a second channel read call
                 {
-                    ByteBuffer compressed = bufferHolder.getBuffer(length);
-
-                    if (channel.read(compressed, chunk.offset) != length)
-                        throw new CorruptBlockException(channel.filePath(), chunk);
-
-                    compressed.flip();
-                    compressed.limit(chunk.length);
-                    uncompressed.clear();
+                    ByteBuffer compressed = BufferPools.forNetworking().getAtLeast(length, metadata.compressor().preferredBufferType());
 
-                    if (shouldCheckCrc)
+                    try
                     {
-                        int checksum = (int) ChecksumType.CRC32.of(compressed);
-
                         compressed.limit(length);
-                        if (compressed.getInt() != checksum)
+                        if (channel.read(compressed, chunkOffset) != length)
                             throw new CorruptBlockException(channel.filePath(), chunk);
 
-                        compressed.position(0).limit(chunk.length);
-                    }
+                        if (shouldCheckCrc)
+                        {
+                            // compute checksum of the compressed data
+                            compressed.position(0).limit(chunk.length);
+                            int checksum = (int) ChecksumType.CRC32.of(compressed);
+                            // the remaining bytes are the checksum
+                            compressed.limit(length);
+                            int storedChecksum = compressed.getInt();
+                            if (storedChecksum != checksum)
+                                throw new CorruptBlockException(channel.filePath(), chunk, storedChecksum, checksum);
+                        }
 
-                    try
-                    {
-                        metadata.compressor().uncompress(compressed, uncompressed);
+                        compressed.position(0).limit(chunk.length);
+                        uncompressed.clear();
+
+                        try
+                        {
+                            if (shouldDecompress)
+                                metadata.compressor().uncompress(compressed, uncompressed);
+                            else
+                                uncompressed.put(compressed);
+                        }
+                        catch (IOException e)
+                        {
+                            throw new CorruptBlockException(channel.filePath(), chunk, e);
+                        }
                     }
-                    catch (IOException e)
+                    finally
                     {
-                        throw new CorruptBlockException(channel.filePath(), chunk, e);
+                        BufferPools.forNetworking().put(compressed);
                     }
                 }
                 else
                 {
                     uncompressed.position(0).limit(chunk.length);
-                    if (channel.read(uncompressed, chunk.offset) != chunk.length)
+                    if (channel.read(uncompressed, chunkOffset) != chunk.length)
                         throw new CorruptBlockException(channel.filePath(), chunk);
-
-                    if (shouldCheckCrc)
-                    {
-                        uncompressed.flip();
-                        int checksum = (int) ChecksumType.CRC32.of(uncompressed);
-
-                        ByteBuffer scratch = bufferHolder.getBuffer(Integer.BYTES);
-
-                        if (channel.read(scratch, chunk.offset + chunk.length) != Integer.BYTES
-                                || scratch.getInt(0) != checksum)
-                            throw new CorruptBlockException(channel.filePath(), chunk);
-                    }
                 }
                 uncompressed.flip();
             }
             catch (CorruptBlockException e)
             {
+                StorageProvider.instance.invalidateFileSystemCache(channel.getFile());
+
                 // Make sure reader does not see stale data.
                 uncompressed.position(0).limit(0);
                 throw new CorruptSSTableException(e, channel.filePath());
             }
         }
+
+        @Override
+        public void invalidateIfCached(long position)
+        {
+        }
     }
 
     public static class Mmap extends CompressedChunkReader
@@ -179,7 +203,12 @@ public static class Mmap extends CompressedChunkReader
 
         public Mmap(ChannelProxy channel, CompressionMetadata metadata, MmappedRegions regions, Supplier<Double> crcCheckChanceSupplier)
         {
-            super(channel, metadata, crcCheckChanceSupplier);
+            this(channel, metadata, regions, crcCheckChanceSupplier, 0);
+        }
+
+        public Mmap(ChannelProxy channel, CompressionMetadata metadata, MmappedRegions regions, Supplier<Double> crcCheckChanceSupplier, long startOffset)
+        {
+            super(channel, metadata, crcCheckChanceSupplier, startOffset);
             this.regions = regions;
         }
 
@@ -196,26 +225,25 @@ public void readChunk(long position, ByteBuffer uncompressed)
 
                 MmappedRegions.Region region = regions.floor(chunk.offset);
                 long segmentOffset = region.offset();
-                int chunkOffset = Ints.checkedCast(chunk.offset - segmentOffset);
+                int chunkOffsetInSegment = Ints.checkedCast(chunk.offset - segmentOffset);
                 ByteBuffer compressedChunk = region.buffer();
 
-                compressedChunk.position(chunkOffset).limit(chunkOffset + chunk.length);
-
-                uncompressed.clear();
-
                 try
                 {
                     if (shouldCheckCrc())
                     {
+                        compressedChunk.position(chunkOffsetInSegment).limit(chunkOffsetInSegment + chunk.length);
                         int checksum = (int) ChecksumType.CRC32.of(compressedChunk);
 
                         compressedChunk.limit(compressedChunk.capacity());
-                        if (compressedChunk.getInt() != checksum)
-                            throw new CorruptBlockException(channel.filePath(), chunk);
-
-                        compressedChunk.position(chunkOffset).limit(chunkOffset + chunk.length);
+                        int storedChecksum = compressedChunk.getInt();
+                        if (storedChecksum != checksum)
+                            throw new CorruptBlockException(channel.filePath(), chunk, storedChecksum, checksum);
                     }
 
+                    compressedChunk.position(chunkOffsetInSegment).limit(chunkOffsetInSegment + chunk.length);
+                    uncompressed.clear();
+
                     if (chunk.length < maxCompressedLength)
                         metadata.compressor().uncompress(compressedChunk, uncompressed);
                     else
@@ -241,5 +269,10 @@ public void close()
             regions.closeQuietly();
             super.close();
         }
+
+        @Override
+        public void invalidateIfCached(long position)
+        {
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
index aef3614da62b..080e80ee914b 100644
--- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
+++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
@@ -23,6 +23,7 @@
 import java.util.zip.CheckedInputStream;
 import java.util.zip.Checksum;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.cassandra.utils.ChecksumType;
 
 public class DataIntegrityMetadata
@@ -32,21 +33,32 @@ public static class ChecksumValidator implements Closeable
         private final ChecksumType checksumType;
         private final RandomAccessReader reader;
         public final int chunkSize;
+        private final File dataFile;
 
         public ChecksumValidator(File dataFile, File crcFile) throws IOException
         {
             this(ChecksumType.CRC32,
                  RandomAccessReader.open(crcFile),
-                 dataFile.absolutePath());
+                 dataFile);
         }
 
-        public ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, String dataFilename) throws IOException
+        public ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, File dataFile) throws IOException
         {
             this.checksumType = checksumType;
             this.reader = reader;
+            this.dataFile = dataFile;
             chunkSize = reader.readInt();
         }
 
+        @VisibleForTesting
+        protected ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, int chunkSize)
+        {
+            this.checksumType = checksumType;
+            this.reader = reader;
+            this.dataFile = null;
+            this.chunkSize = chunkSize;
+        }
+
         public void seek(long offset)
         {
             long start = chunkStart(offset);
@@ -64,7 +76,7 @@ public void validate(byte[] bytes, int start, int end) throws IOException
             int calculatedValue = (int) checksumType.of(bytes, start, end);
             int storedValue = reader.readInt();
             if (calculatedValue != storedValue)
-                throw new IOException(String.format("Corrupted file: integrity check (%s) failed for %s: %d != %d", checksumType.name(), reader.getPath(), storedValue, calculatedValue));
+                throw new IOException(String.format("Corrupted file: integrity check (%s) failed for %s: %d != %d", checksumType.name(), dataFile, storedValue, calculatedValue));
         }
 
         /**
@@ -78,7 +90,7 @@ public void validate(ByteBuffer buffer) throws IOException
             int calculatedValue = (int) checksumType.of(buffer);
             int storedValue = reader.readInt();
             if (calculatedValue != storedValue)
-                throw new IOException(String.format("Corrupted file: integrity check (%s) failed for %s: %d != %d", checksumType.name(), reader.getPath(), storedValue, calculatedValue));
+                throw new IOException(String.format("Corrupted file: integrity check (%s) failed for %s: %d != %d", checksumType.name(), dataFile, storedValue, calculatedValue));
         }
 
         public void close()
diff --git a/src/java/org/apache/cassandra/io/util/DataOutputPlus.java b/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
index f8bc95953164..ed75c1b19791 100644
--- a/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
+++ b/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
@@ -170,6 +170,7 @@ default int maxBytesInPage()
     /**
      * Pad this with zeroes until the next page boundary. If the destination position
      * is already at a page boundary, do not do anything.
+     * @throws IOException
      */
     default void padToPageBoundary() throws IOException
     {
@@ -194,4 +195,4 @@ default long paddedPosition()
     {
         throw new UnsupportedOperationException();
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java b/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java
index aa8e7e046f39..3883b53d204e 100644
--- a/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java
@@ -68,4 +68,9 @@ public Rebufferer instantiateRebufferer()
     {
         return this;
     }
+
+    @Override
+    public void invalidateIfCached(long position)
+    {
+    }
 }
diff --git a/src/java/org/apache/cassandra/io/util/File.java b/src/java/org/apache/cassandra/io/util/File.java
index de415388ed9e..fc801f00435b 100644
--- a/src/java/org/apache/cassandra/io/util/File.java
+++ b/src/java/org/apache/cassandra/io/util/File.java
@@ -23,12 +23,12 @@
 import java.io.UncheckedIOException;
 import java.net.URI;
 import java.nio.channels.FileChannel;
+import java.nio.file.*;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
-import java.nio.file.Paths; // checkstyle: permit this import
 import java.util.Objects;
 import java.util.function.BiPredicate;
 import java.util.function.Consumer;
@@ -143,9 +143,6 @@ public File(FileSystem fs, String first, String... more)
      */
     public File(Path path)
     {
-        if (path != null && path.getFileSystem() != filesystem)
-            throw new IllegalArgumentException("Incompatible file system; path FileSystem (" + path.getFileSystem() + ") is not the same reference (" + filesystem + ")");
-
         this.path = path;
     }
 
@@ -261,6 +258,47 @@ public void move(File to)
         PathUtils.rename(toPathForRead(), to.toPathForWrite());
     }
 
+    public void copy(File target, StandardCopyOption options)
+    {
+        PathUtils.copy(toPathForRead(), target.toPathForWrite(), options);
+    }
+
+    /**
+     * Constructs a relative path between this path and a given path.
+     */
+    public File relativize(File other)
+    {
+        Path relative = toPathForRead().relativize(other.toPathForRead());
+        return new File(relative);
+    }
+
+    /**
+     * Resolves give path against this path's parent path
+     */
+    public File resolveSibling(String path)
+    {
+        Path sibling = toPathForRead().resolveSibling(path);
+        return new File(sibling);
+    }
+
+    /**
+     * Resolves give path against this path
+     */
+    public File resolve(String path)
+    {
+        Path sibling = toPathForRead().resolve(path);
+        return new File(sibling);
+    }
+
+    /**
+     * Resolves give path against this path
+     */
+    public File resolve(File path)
+    {
+        Path sibling = toPathForRead().resolve(path.toPathForRead());
+        return new File(sibling);
+    }
+
     /**
      * @return the length of the file if it exists and if we can read it; 0 otherwise.
      */
@@ -718,6 +756,11 @@ public int compareTo(File that)
         return this.path.compareTo(that.path);
     }
 
+    public URI toUri()
+    {
+        return Objects.requireNonNull(path).toUri();
+    }
+
     public java.io.File toJavaIOFile()
     {
         return path == null ? new java.io.File("") // checkstyle: permit this instantiation
diff --git a/src/java/org/apache/cassandra/io/util/FileDataInput.java b/src/java/org/apache/cassandra/io/util/FileDataInput.java
index 1059b0111cab..fdaacf3a22d4 100644
--- a/src/java/org/apache/cassandra/io/util/FileDataInput.java
+++ b/src/java/org/apache/cassandra/io/util/FileDataInput.java
@@ -22,7 +22,7 @@
 
 public interface FileDataInput extends RewindableDataInput, Closeable
 {
-    String getPath();
+    File getFile();
 
     boolean isEOF() throws IOException;
 
diff --git a/src/java/org/apache/cassandra/io/util/FileHandle.java b/src/java/org/apache/cassandra/io/util/FileHandle.java
index 943355d01dd7..11572e1262ab 100644
--- a/src/java/org/apache/cassandra/io/util/FileHandle.java
+++ b/src/java/org/apache/cassandra/io/util/FileHandle.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.io.util;
 
+import java.nio.ByteOrder;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -24,11 +25,13 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.RateLimiter;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.cache.ChunkCache;
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.utils.NativeLibrary;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.concurrent.RefCounted;
@@ -48,9 +51,14 @@
  */
 public class FileHandle extends SharedCloseableImpl
 {
+    private static final Logger logger = LoggerFactory.getLogger(FileHandle.class);
+
     public final ChannelProxy channel;
 
     public final long onDiskLength;
+    private final ByteOrder order;
+
+    public final SliceDescriptor sliceDescriptor;
 
     /*
      * Rebufferer factory to use when constructing RandomAccessReaders
@@ -66,13 +74,17 @@ private FileHandle(Cleanup cleanup,
                        ChannelProxy channel,
                        RebuffererFactory rebuffererFactory,
                        CompressionMetadata compressionMetadata,
-                       long onDiskLength)
+                       ByteOrder order,
+                       long onDiskLength,
+                       SliceDescriptor sliceDescriptor)
     {
         super(cleanup);
         this.rebuffererFactory = rebuffererFactory;
         this.channel = channel;
         this.compressionMetadata = Optional.ofNullable(compressionMetadata);
+        this.order = order;
         this.onDiskLength = onDiskLength;
+        this.sliceDescriptor = sliceDescriptor;
     }
 
     private FileHandle(FileHandle copy)
@@ -81,7 +93,9 @@ private FileHandle(FileHandle copy)
         channel = copy.channel;
         rebuffererFactory = copy.rebuffererFactory;
         compressionMetadata = copy.compressionMetadata;
+        order = copy.order;
         onDiskLength = copy.onDiskLength;
+        sliceDescriptor = copy.sliceDescriptor;
     }
 
     /**
@@ -99,7 +113,7 @@ public String path()
 
     public long dataLength()
     {
-        return compressionMetadata.map(c -> c.dataLength).orElseGet(rebuffererFactory::fileLength);
+        return rebuffererFactory.fileLength();
     }
 
     public RebuffererFactory rebuffererFactory()
@@ -125,7 +139,8 @@ public FileHandle sharedCopy()
     }
 
     /**
-     * Create {@link RandomAccessReader} with configured method of reading content of the file.
+     * Create {@link RandomAccessReader} with configured method of reading content of the file. Positions the reader
+     * at the start of the file or the start of the data if the handle is created for a slice (see {@link SliceDescriptor}).
      *
      * @return RandomAccessReader for the file
      */
@@ -134,38 +149,31 @@ public RandomAccessReader createReader()
         return createReader(null);
     }
 
+    public RandomAccessReader createReader(RateLimiter limiter)
+    {
+        return createReader(limiter, sliceDescriptor.dataStart);
+    }
+
+    public RandomAccessReader createReader(long position)
+    {
+        return createReader(null, position);
+    }
+
     /**
      * Create {@link RandomAccessReader} with configured method of reading content of the file.
      * Reading from file will be rate limited by given {@link RateLimiter}.
      *
      * @param limiter RateLimiter to use for rate limiting read
+     * @param position Position in the file to start reading from
      * @return RandomAccessReader for the file
      */
-    public RandomAccessReader createReader(RateLimiter limiter)
+    public RandomAccessReader createReader(RateLimiter limiter, long position)
     {
-        return new RandomAccessReader(instantiateRebufferer(limiter));
-    }
-
-    public FileDataInput createReader(long position)
-    {
-        RandomAccessReader reader = createReader();
-        try
-        {
-            reader.seek(position);
-            return reader;
-        }
-        catch (Throwable t)
-        {
-            try
-            {
-                reader.close();
-            }
-            catch (Throwable t2)
-            {
-                t.addSuppressed(t2);
-            }
-            throw t;
-        }
+        assert position >= 0 : "Position must be non-negative - file: " + channel.filePath() + ", position: " + position;
+        Rebufferer.BufferHolder bufferHolder = position > 0
+                                               ? Rebufferer.emptyBufferHolderAt(position)
+                                               : Rebufferer.EMPTY;
+        return new RandomAccessReader(instantiateRebufferer(limiter), order, bufferHolder);
     }
 
     /**
@@ -179,9 +187,13 @@ public void dropPageCache(long before)
             if (before >= metadata.dataLength)
                 return 0L;
             else
-                return metadata.chunkFor(before).offset;
-        }).orElse(before);
-        NativeLibrary.trySkipCache(channel.getFileDescriptor(), 0, position, file().absolutePath());
+                return metadata.chunkFor(before).offset - metadata.chunkFor(sliceDescriptor.sliceStart).offset;
+        }).orElse(before - sliceDescriptor.sliceStart);
+
+        if (position > 0)
+            channel.trySkipCache(0, position);
+        else
+            channel.trySkipCache(0, onDiskLength);
     }
 
     public Rebufferer instantiateRebufferer(RateLimiter limiter)
@@ -201,17 +213,14 @@ private static class Cleanup implements RefCounted.Tidy
         final ChannelProxy channel;
         final RebuffererFactory rebufferer;
         final CompressionMetadata compressionMetadata;
-        final Optional<ChunkCache> chunkCache;
 
         private Cleanup(ChannelProxy channel,
                         RebuffererFactory rebufferer,
-                        CompressionMetadata compressionMetadata,
-                        ChunkCache chunkCache)
+                        CompressionMetadata compressionMetadata)
         {
             this.channel = channel;
             this.rebufferer = rebufferer;
             this.compressionMetadata = compressionMetadata;
-            this.chunkCache = Optional.ofNullable(chunkCache);
         }
 
         public String name()
@@ -221,7 +230,8 @@ public String name()
 
         public void tidy()
         {
-            chunkCache.ifPresent(cache -> cache.invalidateFile(name()));
+            // Note: we cannot release data held by the chunk cache at this point, because this would release data that
+            // is pre-cached by early open. Release is done during SSTableReader cleanup. See EarlyOpenCachingTest.
             try
             {
                 if (compressionMetadata != null)
@@ -257,9 +267,13 @@ public static class Builder
         private ChunkCache chunkCache;
         private int bufferSize = RandomAccessReader.DEFAULT_BUFFER_SIZE;
         private BufferType bufferType = BufferType.OFF_HEAP;
+        private ByteOrder order = ByteOrder.BIG_ENDIAN;
+
         private boolean mmapped = false;
         private long lengthOverride = -1;
         private MmappedRegionsCache mmappedRegionsCache;
+        private SliceDescriptor sliceDescriptor = SliceDescriptor.NONE;
+        private boolean adviseRandom = false;
 
         public Builder(File file)
         {
@@ -347,6 +361,17 @@ public Builder bufferType(BufferType bufferType)
             return this;
         }
 
+        /**
+         * Set the byte order to apply to each buffer.
+         * @param order
+         * @return
+         */
+        public Builder order(ByteOrder order)
+        {
+            this.order = order;
+            return this;
+        }
+
         /**
          * Override the file length.
          *
@@ -360,6 +385,18 @@ public Builder withLengthOverride(long lengthOverride)
             return this;
         }
 
+        public Builder slice(SliceDescriptor sliceDescriptor)
+        {
+            this.sliceDescriptor = sliceDescriptor;
+            return this;
+        }
+
+        public Builder adviseRandom()
+        {
+            adviseRandom = true;
+            return this;
+        }
+
         /**
          * Complete building {@link FileHandle}.
          */
@@ -391,32 +428,39 @@ else if (mmapped)
                 {
                     if (compressionMetadata != null)
                     {
-                        regions = mmappedRegionsCache != null ? mmappedRegionsCache.getOrCreate(channel, compressionMetadata)
-                                                              : MmappedRegions.map(channel, compressionMetadata);
-                        rebuffererFactory = maybeCached(new CompressedChunkReader.Mmap(channel, compressionMetadata, regions, crcCheckChanceSupplier));
+                        regions = mmappedRegionsCache != null ? mmappedRegionsCache.getOrCreate(channel, compressionMetadata, bufferSize, sliceDescriptor.sliceStart)
+                                                              : MmappedRegions.map(channel, compressionMetadata, sliceDescriptor.sliceStart, adviseRandom);
+                        rebuffererFactory = maybeCached(new CompressedChunkReader.Mmap(channel, compressionMetadata, regions, crcCheckChanceSupplier, sliceDescriptor.sliceStart));
                     }
                     else
                     {
-                        regions = mmappedRegionsCache != null ? mmappedRegionsCache.getOrCreate(channel, length)
-                                                              : MmappedRegions.map(channel, length);
-                        rebuffererFactory = new MmapRebufferer(channel, length, regions);
+                        regions = mmappedRegionsCache != null ? mmappedRegionsCache.getOrCreate(channel, sliceDescriptor.dataEndOr(length), bufferSize, sliceDescriptor.sliceStart)
+                                                              : MmappedRegions.map(channel, sliceDescriptor.dataEndOr(length), bufferSize, sliceDescriptor.sliceStart, adviseRandom);
+                        rebuffererFactory = new MmapRebufferer(channel, sliceDescriptor.dataEndOr(length), regions);
                     }
                 }
                 else
                 {
+                    if (adviseRandom)
+                        logger.warn("adviseRandom ignored for non-mmapped FileHandle {}", file);
+
                     if (compressionMetadata != null)
                     {
-                        rebuffererFactory = maybeCached(new CompressedChunkReader.Standard(channel, compressionMetadata, crcCheckChanceSupplier));
+                        rebuffererFactory = maybeCached(new CompressedChunkReader.Standard(channel, compressionMetadata, crcCheckChanceSupplier, sliceDescriptor.sliceStart));
                     }
                     else
                     {
                         int chunkSize = DiskOptimizationStrategy.roundForCaching(bufferSize, ChunkCache.roundUp);
-                        rebuffererFactory = maybeCached(new SimpleChunkReader(channel, length, bufferType, chunkSize));
+                        if (sliceDescriptor.chunkSize > 0 && sliceDescriptor.chunkSize < chunkSize)
+                            // if the chunk size in the slice was smaller than the one we used in the rebufferer,
+                            // we could end up aligning the file position to the value lower than the slice start
+                            chunkSize = sliceDescriptor.chunkSize;
+                        rebuffererFactory = maybeCached(new SimpleChunkReader(channel, sliceDescriptor.dataEndOr(length), bufferType, chunkSize, sliceDescriptor.sliceStart));
                     }
                 }
-                Cleanup cleanup = new Cleanup(channel, rebuffererFactory, compressionMetadata, chunkCache);
+                Cleanup cleanup = new Cleanup(channel, rebuffererFactory, compressionMetadata);
 
-                FileHandle fileHandle = new FileHandle(cleanup, channel, rebuffererFactory, compressionMetadata, length);
+                FileHandle fileHandle = new FileHandle(cleanup, channel, rebuffererFactory, compressionMetadata, order, length, sliceDescriptor);
                 return fileHandle;
             }
             catch (Throwable t)
@@ -429,7 +473,7 @@ else if (mmapped)
         private RebuffererFactory maybeCached(ChunkReader reader)
         {
             if (chunkCache != null && chunkCache.capacity() > 0)
-                return chunkCache.wrap(reader);
+                return chunkCache.maybeWrap(reader);
             return reader;
         }
     }
diff --git a/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java b/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java
index 2bd57a99d2a7..bfb694ca8f7c 100644
--- a/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java
+++ b/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java
@@ -84,4 +84,10 @@ public void close() throws IOException
             }
         }
     }
+
+    @Override
+    public String toString()
+    {
+        return file.toString();
+    }
 }
diff --git a/src/java/org/apache/cassandra/io/util/FileSegmentInputStream.java b/src/java/org/apache/cassandra/io/util/FileSegmentInputStream.java
index a58521527ddc..255a46a16338 100644
--- a/src/java/org/apache/cassandra/io/util/FileSegmentInputStream.java
+++ b/src/java/org/apache/cassandra/io/util/FileSegmentInputStream.java
@@ -26,19 +26,20 @@
  */
 public class FileSegmentInputStream extends DataInputBuffer implements FileDataInput
 {
-    private final String filePath;
+    private final File file;
     private final long offset;
 
-    public FileSegmentInputStream(ByteBuffer buffer, String filePath, long offset)
+    public FileSegmentInputStream(ByteBuffer buffer, File file, long offset)
     {
         super(buffer, false);
-        this.filePath = filePath;
+        this.file = file;
         this.offset = offset;
     }
 
-    public String getPath()
+    @Override
+    public File getFile()
     {
-        return filePath;
+        return file;
     }
 
     private long size()
@@ -61,7 +62,7 @@ public void seek(long pos)
         if (pos < 0 || pos > size())
             throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in partial mode",
                                                              pos,
-                                                             getPath(),
+                                                             getFile(),
                                                              size()));
 
 
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index 7027d6e114e2..eeef01ddeb70 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -19,7 +19,9 @@
 
 import java.io.BufferedWriter;
 import java.io.Closeable;
+import java.io.DataInput;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
@@ -48,9 +50,10 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.RateLimiter;
 import org.slf4j.Logger;
@@ -60,6 +63,7 @@
 import org.apache.cassandra.io.FSErrorHandler;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.utils.DseLegacy;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.SyncUtil;
 
@@ -82,6 +86,7 @@ public final class FileUtils
 
     private static final Class clsDirectBuffer;
     private static final MethodHandle mhDirectBufferCleaner;
+    private static final MethodHandle mhDirectBufferAttachment;
     private static final MethodHandle mhCleanerClean;
 
     static
@@ -91,6 +96,8 @@ public final class FileUtils
             clsDirectBuffer = Class.forName("sun.nio.ch.DirectBuffer");
             Method mDirectBufferCleaner = clsDirectBuffer.getMethod("cleaner");
             mhDirectBufferCleaner = MethodHandles.lookup().unreflect(mDirectBufferCleaner);
+            Method mDirectBufferAttachment = clsDirectBuffer.getMethod("attachment");
+            mhDirectBufferAttachment = MethodHandles.lookup().unreflect(mDirectBufferAttachment);
             Method mCleanerClean = mDirectBufferCleaner.getReturnType().getMethod("clean");
             mhCleanerClean = MethodHandles.lookup().unreflect(mCleanerClean);
 
@@ -233,12 +240,13 @@ public static void copyWithOutConfirm(File from, File to)
     {
         try
         {
-            Files.copy(from.toPath(), to.toPath());
+            if (from.exists())
+                Files.copy(from.toPath(), to.toPath());
         }
         catch (IOException e)
         {
             if (logger.isTraceEnabled())
-                logger.trace("Could not copy file" + from + " to " + to, e);
+                logger.trace("Could not copy file " + from + " to " + to, e);
         }
     }
 
@@ -265,7 +273,11 @@ public static void copyWithConfirm(File from, File to)
 
     public static void truncate(String path, long size)
     {
-        File file = new File(path);
+        truncate(new File(path), size);
+    }
+
+    public static void truncate(File file, long size)
+    {
         try (FileChannel channel = file.newReadWriteChannel())
         {
             channel.truncate(size);
@@ -302,15 +314,15 @@ public static void closeQuietly(AutoCloseable c)
         }
     }
 
-    public static void close(Closeable... cs) throws IOException
+    public static void close(AutoCloseable... cs) throws IOException
     {
         close(Arrays.asList(cs));
     }
 
-    public static void close(Iterable<? extends Closeable> cs) throws IOException
+    public static void close(Iterable<? extends AutoCloseable> cs) throws IOException
     {
         Throwable e = null;
-        for (Closeable c : cs)
+        for (AutoCloseable c : cs)
         {
             try
             {
@@ -327,6 +339,17 @@ public static void close(Iterable<? extends Closeable> cs) throws IOException
         maybeFail(e, IOException.class);
     }
 
+    public static void closeQuietly(Closeable... cs)
+    {
+        closeQuietly(Arrays.asList(cs));
+    }
+
+    public static void closeQuietly(AutoCloseable... cs)
+    {
+        for (AutoCloseable c : cs)
+            closeQuietly(c);
+    }
+
     public static void closeQuietly(Iterable<? extends AutoCloseable> cs)
     {
         for (AutoCloseable c : cs)
@@ -359,7 +382,7 @@ public static boolean isContained(File folder, File file)
         return folder.isAncestorOf(file);
     }
 
-    public static void clean(ByteBuffer buffer)
+    public static void clean(ByteBuffer buffer, boolean withAttachment)
     {
         if (buffer == null || !buffer.isDirect())
             return;
@@ -370,10 +393,18 @@ public static void clean(ByteBuffer buffer)
 
         try
         {
-            Object cleaner = mhDirectBufferCleaner.bindTo(buffer).invoke();
+            Object buf = buffer;
+            if (withAttachment)
+            {
+                while (mhDirectBufferCleaner.bindTo(buf).invoke() == null && mhDirectBufferAttachment.bindTo(buf).invoke() != null && mhDirectBufferAttachment.bindTo(buf).invoke().getClass().isInstance(clsDirectBuffer))
+                {
+                    buf = mhDirectBufferAttachment.bindTo(buf).invoke();
+                }
+            }
+
+            Object cleaner = mhDirectBufferCleaner.bindTo(buf).invoke();
             if (cleaner != null)
             {
-                // ((DirectBuffer) buf).cleaner().clean();
                 mhCleanerClean.bindTo(cleaner).invoke();
             }
         }
@@ -387,6 +418,16 @@ public static void clean(ByteBuffer buffer)
         }
     }
 
+    public static void clean(ByteBuffer buffer)
+    {
+        clean(buffer, false);
+    }
+
+    public static void cleanWithAttachment(ByteBuffer buffer)
+    {
+        clean(buffer, true);
+    }
+
     public static long parseFileSize(String value)
     {
         long result;
@@ -528,6 +569,26 @@ public FileVisitResult visitFileFailed(Path path, IOException e) throws IOExcept
         return sizeArr[0];
     }
 
+    public static void copyTo(DataInput in, OutputStream out, int length) throws IOException
+    {
+        byte[] buffer = new byte[64 * 1024];
+        int copiedBytes = 0;
+
+        while (copiedBytes + buffer.length < length)
+        {
+            in.readFully(buffer);
+            out.write(buffer);
+            copiedBytes += buffer.length;
+        }
+
+        if (copiedBytes < length)
+        {
+            int left = length - copiedBytes;
+            in.readFully(buffer, 0, left);
+            out.write(buffer, 0, left);
+        }
+    }
+
     public static void append(File file, String ... lines)
     {
         if (file.exists())
@@ -696,6 +757,16 @@ public static boolean isSubDirectory(File parent, File child)
     {
         return parent.isAncestorOf(child);
     }
+    
+    /**
+     * Handle large file system by returning {@code Long.MAX_VALUE} when the size overflows.
+     * @param size returned by the Java's FileStore methods
+     * @return the size or {@code Long.MAX_VALUE} if the size was bigger than {@code Long.MAX_VALUE}
+     */
+    public static long handleLargeFileSystem(long size)
+    {
+        return size < 0 ? Long.MAX_VALUE : size;
+    }
 
     /** @deprecated See CASSANDRA-16926 */
     @Deprecated(since = "4.1")
@@ -762,15 +833,15 @@ private FileUtils()
      * @param source the directory containing the files to move
      * @param target the directory where the files must be moved
      */
-    public static void moveRecursively(Path source, Path target) throws IOException
+    public static void moveRecursively(File source, File target) throws IOException
     {
         logger.info("Moving {} to {}" , source, target);
 
-        if (Files.isDirectory(source))
+        if (source.isDirectory())
         {
-            Files.createDirectories(target);
+            target.tryCreateDirectories();
 
-            for (File f : new File(source).tryList())
+            for (File f : source.tryList())
             {
                 String fileName = f.name();
                 moveRecursively(source.resolve(fileName), target.resolve(fileName));
@@ -780,43 +851,62 @@ public static void moveRecursively(Path source, Path target) throws IOException
         }
         else
         {
-            if (Files.exists(target))
+            if (target.exists())
             {
                 logger.warn("Cannot move the file {} to {} as the target file already exists." , source, target);
             }
             else
             {
-                Files.copy(source, target, StandardCopyOption.COPY_ATTRIBUTES);
-                Files.delete(source);
+                source.copy(target, StandardCopyOption.COPY_ATTRIBUTES);
+                source.delete();
             }
         }
     }
 
+    @VisibleForTesting
+    /** @deprecated See CNDB-1707 */
+    @Deprecated(since = "5.0")
+    public static void moveRecursively(Path source, Path target) throws IOException
+    {
+        moveRecursively(new File(source), new File(target));
+    }
+
     /**
      * Deletes the specified directory if it is empty
      *
-     * @param path the path to the directory
+     * @param file the path to the directory
      */
-    public static void deleteDirectoryIfEmpty(Path path) throws IOException
+    public static void deleteDirectoryIfEmpty(File file) throws IOException
     {
-        Preconditions.checkArgument(Files.isDirectory(path), String.format("%s is not a directory", path));
+        Preconditions.checkArgument(file.isDirectory(), String.format("%s is not a directory", file));
 
         try
         {
-            logger.info("Deleting directory {}", path);
-            Files.delete(path);
+            logger.info("Deleting directory {}", file);
+            Files.delete(file.toPath());
         }
         catch (DirectoryNotEmptyException e)
         {
-            try (Stream<Path> paths = Files.list(path))
-            {
-                String content = paths.map(p -> p.getFileName().toString()).collect(Collectors.joining(", "));
-
-                logger.warn("Cannot delete the directory {} as it is not empty. (Content: {})", path, content);
-            }
+            String content = Arrays.stream(file.tryList()).map(File::name).collect(Collectors.joining(", "));
+            logger.warn("Cannot delete the directory {} as it is not empty. (Content: {})", file, content);
         }
     }
 
+    @VisibleForTesting
+    /** @deprecated See CNDB-1707 */
+    @Deprecated(since = "5.0")
+    public static void deleteDirectoryIfEmpty(Path path) throws IOException
+    {
+        deleteDirectoryIfEmpty(new File(path));
+    }
+
+    /** @deprecated See CNDB-1707 */
+    @Deprecated(since = "5.0")
+    public static long size(Path path)
+    {
+        return PathUtils.size(path);
+    }
+
     public static int getBlockSize(File directory)
     {
         File f = FileUtils.createTempFile("block-size-test", ".tmp", directory);
@@ -835,4 +925,114 @@ public static int getBlockSize(File directory)
             f.tryDelete();
         }
     }
-}
\ No newline at end of file
+
+    @DseLegacy
+    public static void deleteRecursive(Path dir)
+    {
+        deleteRecursive(dir, false);
+    }
+
+    @DseLegacy
+    public static void deleteRecursive(Path dir, boolean failOnError)
+    {
+        if (failOnError)
+        {
+            PathUtils.deleteRecursive(dir);
+        }
+        else
+        {
+            PathUtils.deleteQuietly(dir);
+        }
+    }
+
+    @DseLegacy
+    public static Path getPath(String pathOrURI)
+    {
+        return PathUtils.getPath(pathOrURI);
+    }
+
+    @DseLegacy
+    public static void createDirectory(Path directory)
+    {
+        // yes, use create*Directories*, because that's the semantics of createDirectory in DSE
+        PathUtils.createDirectoriesIfNotExists(directory);
+    }
+
+    @DseLegacy
+    public static void appendAndSync(Path file, String... lines)
+    {
+        appendAndSync(new File(file), lines);
+    }
+
+    @DseLegacy
+    public static void delete(Path path)
+    {
+        PathUtils.delete(path);
+    }
+
+    @DseLegacy
+    public static void deleteContent(Path path)
+    {
+        PathUtils.deleteContent(path);
+    }
+
+    @DseLegacy
+    public static List<Path> listPaths(Path dir)
+    {
+        return PathUtils.listPaths(dir);
+    }
+
+    @DseLegacy
+    public static List<Path> listPaths(Path dir, Predicate<Path> filter)
+    {
+        return PathUtils.listPaths(dir, filter);
+    }
+
+    /**
+     * List paths that match this absolute path.
+     *
+     * This method is more efficient than {@link #listPaths(Path, Predicate)} if underlying file system can apply
+     * prefix filter earlier.
+     */
+    public static List<Path> listPathsWithAbsolutePath(String absolutePath)
+    {
+        return listPathsWithAbsolutePath(FileUtils.getPath(absolutePath));
+    }
+
+    public static List<Path> listPathsWithAbsolutePath(Path absolutePath)
+    {
+        Path parent = absolutePath.getParent();
+        String prefix = absolutePath.getFileName().toString();
+
+        return FileUtils.listPaths(parent, p -> FileUtils.fileNameMatchesPrefix(p.toString(), prefix));
+    }
+
+    /**
+     * Memory optimized, zero-copy version of the common {@code FileUtils.getFileName(path).startsW
+     * ith(prefix)} idiom.
+     *
+     * @param pathStr The path whose filename portion we want to match against.
+     * @param prefix The prefix to match.
+     *
+     * @return True if matching, false otherwise.
+     */
+    public static boolean fileNameMatchesPrefix(String pathStr, String prefix)
+    {
+        int pathLen = pathStr.length();
+        int prefixLen = prefix.length();
+        if (pathLen == 0)
+            return false;
+
+        int sepIndex = pathLen - 2; // Skip the separator if the strings ends with it
+        for (; sepIndex >= 0; sepIndex--)
+            if (pathStr.charAt(sepIndex) == '/')
+                break;
+
+        return pathStr.regionMatches(false, // Linux is case-sensitive, so let's optimize for that
+                                     sepIndex >= 0 ? sepIndex + 1 : 0,
+                                     prefix,
+                                     0,
+                                     prefixLen);
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java b/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
index bcbf2ef58a7e..5d5702ee58cf 100644
--- a/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
@@ -69,4 +69,4 @@ public String toString()
     {
         return "LimitingRebufferer[" + limiter + "]:" + wrapped;
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/io/util/Memory.java b/src/java/org/apache/cassandra/io/util/Memory.java
index 1d1fca2edf96..fd7063264fbf 100644
--- a/src/java/org/apache/cassandra/io/util/Memory.java
+++ b/src/java/org/apache/cassandra/io/util/Memory.java
@@ -21,7 +21,6 @@
 import java.nio.ByteBuffer;
 
 import net.nicoulaj.compilecommand.annotations.Inline;
-
 import org.apache.cassandra.utils.Architecture;
 import org.apache.cassandra.utils.FastByteOperations;
 import org.apache.cassandra.utils.concurrent.Ref;
@@ -339,7 +338,7 @@ public Memory copy(long newSize)
 
     public void free()
     {
-        if (peer != 0) MemoryUtil.free(peer);
+        if (peer != 0) MemoryUtil.free(peer, size);
         else assert size == 0;
         peer = 0;
     }
@@ -408,4 +407,59 @@ protected static String toString(long peer, long size)
     {
         return String.format("Memory@[%x..%x)", peer, peer + size);
     }
+
+    public static class LongArray implements AutoCloseable
+    {
+        public final Memory memory;
+        private final long size;
+
+        public LongArray(long size)
+        {
+            assert size >= 0;
+            this.memory = size > 0 ? Memory.allocate(size << 3) : null;
+            this.size = size;
+        }
+
+        public LongArray(SafeMemory memory, long cnt)
+        {
+            assert cnt <= memory.size >> 3;
+            this.memory = memory;
+            this.size = cnt;
+        }
+
+        public void set(long offset, long value)
+        {
+            checkBounds(offset);
+            memory.setLong(offset << 3, value);
+        }
+
+        public long get(long offset)
+        {
+            checkBounds(offset);
+            return memory.getLong(offset << 3);
+        }
+
+        public long size()
+        {
+            return size;
+        }
+
+        public long memoryUsed()
+        {
+            return memory != null ? memory.size() : 0;
+        }
+
+        @Override
+        public void close()
+        {
+            if (memory != null)
+                memory.close();
+        }
+
+        private void checkBounds(long offset)
+        {
+            if (memory == null || offset < 0 || offset >= size)
+                throw new IndexOutOfBoundsException();
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/io/util/MmapRebufferer.java b/src/java/org/apache/cassandra/io/util/MmapRebufferer.java
index 8df6370c5e69..d3758b3cd648 100644
--- a/src/java/org/apache/cassandra/io/util/MmapRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/MmapRebufferer.java
@@ -46,6 +46,11 @@ public Rebufferer instantiateRebufferer()
         return this;
     }
 
+    @Override
+    public void invalidateIfCached(long position)
+    {
+    }
+
     @Override
     public void close()
     {
diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegions.java b/src/java/org/apache/cassandra/io/util/MmappedRegions.java
index 0ab07b8d0f74..348a3b06d683 100644
--- a/src/java/org/apache/cassandra/io/util/MmappedRegions.java
+++ b/src/java/org/apache/cassandra/io/util/MmappedRegions.java
@@ -19,8 +19,6 @@
 package org.apache.cassandra.io.util;
 
 import java.nio.ByteBuffer;
-import java.nio.FloatBuffer;
-import java.nio.IntBuffer;
 import java.nio.channels.FileChannel;
 import java.util.Arrays;
 
@@ -28,20 +26,20 @@
 
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.utils.INativeLibrary;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.RefCounted;
 import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
 
 import static java.util.stream.Stream.of;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MMAPPED_MAX_SEGMENT_SIZE_IN_MB;
 import static org.apache.cassandra.utils.Throwables.perform;
 
 public class MmappedRegions extends SharedCloseableImpl
 {
-    /**
-     * In a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size
-     */
-    public static int MAX_SEGMENT_SIZE = Integer.MAX_VALUE;
+    /** In a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size */
+    public static int MAX_SEGMENT_SIZE = MMAPPED_MAX_SEGMENT_SIZE_IN_MB.getInt(Integer.MAX_VALUE);
 
     /**
      * When we need to grow the arrays, we add this number of region slots
@@ -65,26 +63,21 @@ public class MmappedRegions extends SharedCloseableImpl
      */
     private volatile State copy;
 
-    private MmappedRegions(ChannelProxy channel, CompressionMetadata metadata, long length)
+    private MmappedRegions(State state, CompressionMetadata metadata, long uncompressedSliceOffset)
     {
-        this(new State(channel), metadata, length);
+        super(new Tidier(state));
+        this.state = state;
+        updateState(metadata, uncompressedSliceOffset);
+        this.copy = new State(state);
     }
 
-    private MmappedRegions(State state, CompressionMetadata metadata, long length)
+    private MmappedRegions(State state, long length, int chunkSize)
     {
         super(new Tidier(state));
 
         this.state = state;
-
-        if (metadata != null)
-        {
-            assert length == 0 : "expected no length with metadata";
-            updateState(metadata);
-        }
-        else if (length > 0)
-        {
-            updateState(length);
-        }
+        if (length > 0)
+            updateState(length, chunkSize);
 
         this.copy = new State(state);
     }
@@ -97,28 +90,35 @@ private MmappedRegions(MmappedRegions original)
 
     public static MmappedRegions empty(ChannelProxy channel)
     {
-        return new MmappedRegions(channel, null, 0);
+        return new MmappedRegions(new State(channel, 0, false), 0, 0);
     }
 
     /**
-     * @param channel  file to map. the MmappedRegions instance will hold shared copy of given channel.
-     * @param metadata
+     * Create memory mapped regions for the given compressed file.
+     *
+     * @param channel     file to map. The {@link MmappedRegions} instance will hold shared copy of given channel.
+     * @param metadata    compression metadata for the mapped file, cannot be null. A shared copy of the metadata is not
+     *                    created, so it needs to me managed by the caller.
+     * @param uncompressedSliceOffset if the file represents a slice of the origial file, this is the offset of the slice in
+     *                    the original file (in uncompressed data), namely the value of {@link SliceDescriptor#sliceStart}.
+     * @param adviseRandom whether to apply MADV_RANDOM to mapped regions
      * @return new instance
      */
-    public static MmappedRegions map(ChannelProxy channel, CompressionMetadata metadata)
+    public static MmappedRegions map(ChannelProxy channel, CompressionMetadata metadata, long uncompressedSliceOffset, boolean adviseRandom)
     {
         if (metadata == null)
             throw new IllegalArgumentException("metadata cannot be null");
-
-        return new MmappedRegions(channel, metadata, 0);
+        State state = new State(channel, metadata.chunkFor(uncompressedSliceOffset).offset, adviseRandom);
+        return new MmappedRegions(state, metadata, uncompressedSliceOffset);
     }
 
-    public static MmappedRegions map(ChannelProxy channel, long length)
+    public static MmappedRegions map(ChannelProxy channel, long length, int chunkSize, long uncompressedSliceOffset, boolean adviseRandom)
     {
         if (length <= 0)
             throw new IllegalArgumentException("Length must be positive");
 
-        return new MmappedRegions(channel, null, length);
+        State state = new State(channel, uncompressedSliceOffset, adviseRandom);
+        return new MmappedRegions(state, length, chunkSize);
     }
 
     /**
@@ -137,23 +137,21 @@ private boolean isCopy()
 
     /**
      * Extends this collection of mmapped regions up to the provided total length.
-     *
-     * @return {@code true} if new regions have been created
      */
-    public boolean extend(long length)
+    public void extend(long length, int chunkSize)
     {
+        // We cannot enforce length to be a multiple of chunkSize (at the very least the last extend on a file
+        // will not satisfy this), so we hope the caller knows what they are doing.
         if (length < 0)
             throw new IllegalArgumentException("Length must not be negative");
 
         assert !isCopy() : "Copies cannot be extended";
 
         if (length <= state.length)
-            return false;
+            return;
 
-        int initialRegions = state.last;
-        updateState(length);
+        updateState(length, chunkSize);
         copy = new State(state);
-        return state.last > initialRegions;
     }
 
     /**
@@ -162,7 +160,7 @@ public boolean extend(long length)
      *
      * @return {@code true} if new regions have been created
      */
-    public boolean extend(CompressionMetadata compressionMetadata)
+    public boolean extend(CompressionMetadata compressionMetadata, int chunkSize, long uncompressedSliceOffset)
     {
         assert !isCopy() : "Copies cannot be extended";
 
@@ -171,9 +169,9 @@ public boolean extend(CompressionMetadata compressionMetadata)
 
         int initialRegions = state.last;
         if (compressionMetadata.compressedFileLength - state.length <= MAX_SEGMENT_SIZE)
-            updateState(compressionMetadata.compressedFileLength);
+            updateState(compressionMetadata.compressedFileLength, chunkSize);
         else
-            updateState(compressionMetadata);
+            updateState(compressionMetadata, uncompressedSliceOffset);
 
         copy = new State(state);
         return state.last > initialRegions;
@@ -183,47 +181,52 @@ public boolean extend(CompressionMetadata compressionMetadata)
      * Updates state by adding the remaining segments. It starts with the current state last segment end position and
      * subsequently add new segments until all data up to the provided length are mapped.
      */
-    private void updateState(long length)
+    private void updateState(long length, int chunkSize)
     {
+        // make sure the regions span whole chunks
+        long maxSize = (long) (MAX_SEGMENT_SIZE / chunkSize) * chunkSize;
         state.length = length;
         long pos = state.getPosition();
         while (pos < length)
         {
-            long size = Math.min(MAX_SEGMENT_SIZE, length - pos);
+            long size = Math.min(maxSize, length - pos);
             state.add(pos, size);
             pos += size;
         }
     }
 
-    private void updateState(CompressionMetadata metadata)
+    private void updateState(CompressionMetadata metadata, long uncompressedSliceOffset)
     {
-        long lastSegmentOffset = state.getPosition();
-        long offset = metadata.getDataOffsetForChunkOffset(lastSegmentOffset);
+        long uncompressedPosition = metadata.getDataOffsetForChunkOffset(state.getPosition()); // uncompressed position of the current compressed chunk in the original (compressed) file
+        long compressedPosition = state.getPosition();   // position on disk of the current compressed chunk in the original (compressed) file
         long segmentSize = 0;
 
-        while (offset < metadata.dataLength)
+        assert metadata.chunkFor(uncompressedPosition).offset == compressedPosition;
+
+        while (uncompressedPosition - uncompressedSliceOffset < metadata.dataLength)
         {
-            CompressionMetadata.Chunk chunk = metadata.chunkFor(offset);
+            // chunk contains the position on disk in the original file
+            CompressionMetadata.Chunk chunk = metadata.chunkFor(uncompressedPosition);
 
             //Reached a new mmap boundary
             if (segmentSize + chunk.length + 4 > MAX_SEGMENT_SIZE)
             {
                 if (segmentSize > 0)
                 {
-                    state.add(lastSegmentOffset, segmentSize);
-                    lastSegmentOffset += segmentSize;
+                    state.add(compressedPosition, segmentSize);
+                    compressedPosition += segmentSize;
                     segmentSize = 0;
                 }
             }
 
-            segmentSize += chunk.length + 4; //checksum
-            offset += metadata.chunkLength();
+            segmentSize += chunk.length + 4; // compressed size of the chunk including 4 bytes of checksum
+            uncompressedPosition += metadata.chunkLength(); // uncompressed size of the chunk
         }
 
         if (segmentSize > 0)
-            state.add(lastSegmentOffset, segmentSize);
+            state.add(compressedPosition, segmentSize);
 
-        state.length = lastSegmentOffset + segmentSize;
+        state.length = compressedPosition + segmentSize;
     }
 
     public boolean isValid(ChannelProxy channel)
@@ -236,6 +239,11 @@ public boolean isEmpty()
         return state.isEmpty();
     }
 
+    /**
+     * Get the region containing the given position
+     *
+     * @param position the position on disk (not in the uncompressed data) in the original file (not in the slice)
+     */
     public Region floor(long position)
     {
         assert !isCleanedUp() : "Attempted to use closed region";
@@ -265,27 +273,11 @@ public Region(long offset, ByteBuffer buffer)
             this.buffer = buffer;
         }
 
-        @Override
         public ByteBuffer buffer()
         {
             return buffer.duplicate();
         }
 
-        @Override
-        public FloatBuffer floatBuffer()
-        {
-            // this does an implicit duplicate(), so we need to expose it directly to avoid doing it twice unnecessarily
-            return buffer.asFloatBuffer();
-        }
-
-        @Override
-        public IntBuffer intBuffer()
-        {
-            // this does an implicit duplicate(), so we need to expose it directly to avoid doing it twice unnecessarily
-            return buffer.asIntBuffer();
-        }
-
-        @Override
         public long offset()
         {
             return offset;
@@ -296,7 +288,6 @@ public long end()
             return offset + buffer.capacity();
         }
 
-        @Override
         public void release()
         {
             // only released after no readers are present
@@ -330,22 +321,33 @@ private static final class State
          */
         private int last;
 
-        private State(ChannelProxy channel)
+        /** The position of the first region of the slice in the original file (if the file is compressed, the offset
+          * refers to position on disk, not the uncompressed data) */
+        private final long onDiskSliceOffset;
+
+        /** whether to apply fadv_random to mapped regions */
+        private final boolean adviseRandom;
+
+        private State(ChannelProxy channel, long onDiskSliceOffset, boolean adviseRandom)
         {
             this.channel = channel.sharedCopy();
+            this.adviseRandom = adviseRandom;
             this.buffers = new ByteBuffer[REGION_ALLOC_SIZE];
             this.offsets = new long[REGION_ALLOC_SIZE];
             this.length = 0;
             this.last = -1;
+            this.onDiskSliceOffset = onDiskSliceOffset;
         }
 
         private State(State original)
         {
             this.channel = original.channel;
+            this.adviseRandom = original.adviseRandom;
             this.buffers = original.buffers;
             this.offsets = original.offsets;
             this.length = original.length;
             this.last = original.last;
+            this.onDiskSliceOffset = original.onDiskSliceOffset;
         }
 
         private boolean isEmpty()
@@ -355,12 +357,13 @@ private boolean isEmpty()
 
         private boolean isValid(ChannelProxy channel)
         {
+            // todo maybe extend validation to verify slice offset?
             return this.channel.filePath().equals(channel.filePath());
         }
 
         private Region floor(long position)
         {
-            assert 0 <= position && position <= length : String.format("%d > %d", position, length);
+            assert onDiskSliceOffset <= position && position <= length : String.format("%d > %d", position, length);
 
             int idx = Arrays.binarySearch(offsets, 0, last + 1, position);
             assert idx != -1 : String.format("Bad position %d for regions %s, last %d in %s", position, Arrays.toString(offsets), last, channel);
@@ -372,12 +375,19 @@ private Region floor(long position)
 
         private long getPosition()
         {
-            return last < 0 ? 0 : offsets[last] + buffers[last].capacity();
+            return last < 0 ? onDiskSliceOffset : offsets[last] + buffers[last].capacity();
         }
 
+        /**
+         * Add a new region to the state
+         * @param pos the position on disk (not in the uncompressed data) in the original file (not the slice)
+         * @param size the size of the region
+         */
         private void add(long pos, long size)
         {
-            ByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, pos, size);
+            var buffer = channel.map(FileChannel.MapMode.READ_ONLY, pos - onDiskSliceOffset, size);
+            if (adviseRandom)
+                INativeLibrary.instance.adviseRandom(buffer, size, channel.filePath());
 
             ++last;
 
diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java b/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java
index dff9561f4f7d..3fd5e421a64a 100644
--- a/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java
+++ b/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java
@@ -45,12 +45,12 @@ public class MmappedRegionsCache implements AutoCloseable
      * @param length  length of the file
      * @return a shared copy of the cached mmapped regions
      */
-    public MmappedRegions getOrCreate(ChannelProxy channel, long length)
+    public MmappedRegions getOrCreate(ChannelProxy channel, long length, int bufferSize, long uncompressedSliceOffset)
     {
         Preconditions.checkState(!closed);
-        MmappedRegions regions = cache.computeIfAbsent(channel.file(), ignored -> MmappedRegions.map(channel, length));
+        MmappedRegions regions = cache.computeIfAbsent(channel.file(), ignored -> MmappedRegions.map(channel, length, bufferSize, uncompressedSliceOffset, false));
         Preconditions.checkArgument(regions.isValid(channel));
-        regions.extend(length);
+        regions.extend(length, bufferSize);
         return regions.sharedCopy();
     }
 
@@ -62,12 +62,12 @@ public MmappedRegions getOrCreate(ChannelProxy channel, long length)
      * @param metadata compression metadata of the file
      * @return a shared copy of the cached mmapped regions
      */
-    public MmappedRegions getOrCreate(ChannelProxy channel, CompressionMetadata metadata)
+    public MmappedRegions getOrCreate(ChannelProxy channel, CompressionMetadata metadata, int bufferSize, long uncompressedSliceOffset)
     {
         Preconditions.checkState(!closed);
-        MmappedRegions regions = cache.computeIfAbsent(channel.file(), ignored -> MmappedRegions.map(channel, metadata));
+        MmappedRegions regions = cache.computeIfAbsent(channel.file(), ignored -> MmappedRegions.map(channel, metadata, uncompressedSliceOffset, false));
         Preconditions.checkArgument(regions.isValid(channel));
-        regions.extend(metadata);
+        regions.extend(metadata, bufferSize, uncompressedSliceOffset);
         return regions.sharedCopy();
     }
 
diff --git a/src/java/org/apache/cassandra/io/util/PathUtils.java b/src/java/org/apache/cassandra/io/util/PathUtils.java
index 8ddd939b4c09..98cc0a41210c 100644
--- a/src/java/org/apache/cassandra/io/util/PathUtils.java
+++ b/src/java/org/apache/cassandra/io/util/PathUtils.java
@@ -23,12 +23,15 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
+import java.net.URI;
 import java.nio.channels.FileChannel;
+import java.nio.file.*;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.FileStore;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
+import java.nio.file.NotDirectoryException;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
@@ -47,12 +50,14 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.IntFunction;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,6 +67,7 @@
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.storage.StorageProvider;
 import org.apache.cassandra.utils.NoSpamLogger;
 
 import static java.nio.file.StandardOpenOption.APPEND;
@@ -70,7 +76,6 @@
 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
 import static java.nio.file.StandardOpenOption.WRITE;
 import static java.util.Collections.unmodifiableSet;
-import static org.apache.cassandra.config.CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE;
 import static org.apache.cassandra.utils.Throwables.merge;
 
 /**
@@ -78,7 +83,7 @@
  *
  * This class tries to apply uniform IOException handling, and does not propagate IOException except for NoSuchFileException.
  * Any harmless/application error exceptions are propagated as UncheckedIOException, and anything else as an FSReadError or FSWriteError.
- * Semantically this is a little incoherent throughout the codebase, as we intercept IOException haphazardly and treaat
+ * Semantically this is a little incoherent throughout the codebase, as we intercept IOException haphazardly and treat
  * it inconsistently - we should ideally migrate to using {@link #propagate(IOException, Path, boolean)} et al globally.
  */
 public final class PathUtils
@@ -95,6 +100,7 @@ public final class PathUtils
     private static final NoSpamLogger nospam1m = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
 
     private static Consumer<Path> onDeletion = path -> {};
+    private static final boolean USE_NIX_RECURSIVE_DELETE = CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE.getBoolean();
 
     public static FileChannel newReadChannel(Path path) throws NoSuchFileException
     {
@@ -388,7 +394,7 @@ private static void deleteRecursiveUsingNixCommand(Path path, boolean quietly)
      */
     public static void deleteRecursive(Path path)
     {
-        if (USE_NIX_RECURSIVE_DELETE.getBoolean() && path.getFileSystem() == java.nio.file.FileSystems.getDefault())
+        if (USE_NIX_RECURSIVE_DELETE && path.getFileSystem() == FileSystems.getDefault())
         {
             deleteRecursiveUsingNixCommand(path, false);
             return;
@@ -401,6 +407,26 @@ public static void deleteRecursive(Path path)
         delete(path);
     }
 
+    /**
+     * Deletes all files and subdirectories under "path",
+     * ignoring IOExceptions along the way.
+     * @param path file to be deleted
+     */
+    public static void deleteQuietly(Path path)
+    {
+        if (USE_NIX_RECURSIVE_DELETE && path.getFileSystem() == FileSystems.getDefault())
+        {
+            deleteRecursiveUsingNixCommand(path, true);
+            return;
+        }
+
+        if (isDirectory(path))
+            forEach(path, PathUtils::deleteQuietly);
+
+        // The directory is now empty so now it can be smoked
+        tryDelete(path);
+    }
+
     /**
      * Deletes all files and subdirectories under "path".
      * @param path file to be deleted
@@ -408,7 +434,7 @@ public static void deleteRecursive(Path path)
      */
     public static void deleteRecursive(Path path, RateLimiter rateLimiter)
     {
-        if (USE_NIX_RECURSIVE_DELETE.getBoolean() && path.getFileSystem() == java.nio.file.FileSystems.getDefault())
+        if (USE_NIX_RECURSIVE_DELETE && path.getFileSystem() == java.nio.file.FileSystems.getDefault())
         {
             deleteRecursiveUsingNixCommand(path, false);
             return;
@@ -431,6 +457,49 @@ private static void deleteRecursive(Path path, RateLimiter rateLimiter, Consumer
         delete(path, rateLimiter);
     }
 
+    /**
+     * Recursively delete the content of the directory, but not the directory itself.
+     * @param dirPath directory for which content should be deleted
+     */
+    public static void deleteContent(Path dirPath)
+    {
+        if (isDirectory(dirPath))
+            forEach(dirPath, PathUtils::deleteRecursive);
+    }
+
+    /**
+     * List all paths in this directory
+     * @param dirPath directory for which to list all paths
+     * @return list of all paths contained in the given directory
+     */
+    public static List<Path> listPaths(Path dirPath)
+    {
+        return listPaths(dirPath, p -> true);
+    }
+
+    /**
+     * List paths in this directory that match the filter
+     * @param dirPath directory for which to list all paths matching the given filter
+     * @param filter predicate used to filter paths
+     * @return filtered list of paths contained in the given directory
+     */
+    public static List<Path> listPaths(Path dirPath, Predicate<Path> filter)
+    {
+        try (Stream<Path> stream = Files.list(dirPath))
+        {
+            return (consistentDirectoryListings ? stream.sorted() : stream).filter(filter).collect(Collectors.toList());
+        }
+        catch(NotDirectoryException | NoSuchFileException ex)
+        {
+            // Don't throw if the file does not exist or is not a directory
+            return ImmutableList.of();
+        }
+        catch(IOException ex)
+        {
+            throw new FSReadError(ex, dirPath);
+        }
+    }
+
     /**
      * Schedules deletion of all file and subdirectories under "dir" on JVM shutdown.
      * @param dir Directory to be deleted
@@ -454,7 +523,7 @@ public static boolean tryRename(Path from, Path to)
         logger.trace("Renaming {} to {}", from, to);
         try
         {
-            atomicMoveWithFallback(from, to);
+            atomicMoveWithFallback(StorageProvider.instance.getLocalPath(from), StorageProvider.instance.getLocalPath(to));
             return true;
         }
         catch (IOException e)
@@ -469,7 +538,7 @@ public static void rename(Path from, Path to)
         logger.trace("Renaming {} to {}", from, to);
         try
         {
-            atomicMoveWithFallback(from, to);
+            atomicMoveWithFallback(StorageProvider.instance.getLocalPath(from), StorageProvider.instance.getLocalPath(to));
         }
         catch (IOException e)
         {
@@ -496,6 +565,22 @@ private static void atomicMoveWithFallback(Path from, Path to) throws IOExceptio
         }
     }
 
+    /**
+     * Copy a file to a target file
+     */
+    public static void copy(Path from, Path to, StandardCopyOption option)
+    {
+        logger.trace("Copying {} to {}", from, to);
+        try
+        {
+            Files.copy(from, to, option);
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(String.format("Failed to copy %s to %s", from, to), e);
+        }
+    }
+
     // true if can determine exists, false if any exception occurs
     public static boolean exists(Path path)
     {
@@ -599,6 +684,41 @@ public static Path toCanonicalPath(Path file)
         return toRealPath(parent).resolve(parent.relativize(file));
     }
 
+    /**
+     * @param path to check file szie
+     * @return file size or 0 if failed to get file size
+     */
+    public static long size(Path path)
+    {
+        try
+        {
+            return Files.size(path);
+        }
+        catch (IOException e)
+        {
+            // it's possible that between the time that the caller has checked if the file exists and the time it retrieves the creation time,
+            // the file is actually deleted. File.length() returns a positive value only if the file is valid, otherwise it returns 0L, here
+            // we do the same
+            return 0;
+        }
+    }
+
+    /**
+     * @param pathOrURI path or uri in string
+     * @return nio Path
+     */
+    public static Path getPath(String pathOrURI)
+    {
+        try
+        {
+            return Paths.get(URI.create(pathOrURI));
+        }
+        catch (IllegalArgumentException ex)
+        {
+            return Paths.get(pathOrURI);
+        }
+    }
+
     private static Path toRealPath(Path path)
     {
         try
diff --git a/src/java/org/apache/cassandra/io/util/RandomAccessReader.java b/src/java/org/apache/cassandra/io/util/RandomAccessReader.java
index 3ce1a2eb0862..ec4984297269 100644
--- a/src/java/org/apache/cassandra/io/util/RandomAccessReader.java
+++ b/src/java/org/apache/cassandra/io/util/RandomAccessReader.java
@@ -18,8 +18,11 @@
 package org.apache.cassandra.io.util;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
-
+import java.nio.FloatBuffer;
+import java.nio.IntBuffer;
+import java.nio.LongBuffer;
 import javax.annotation.concurrent.NotThreadSafe;
 
 import com.google.common.primitives.Ints;
@@ -28,7 +31,7 @@
 import org.apache.cassandra.io.util.Rebufferer.BufferHolder;
 
 @NotThreadSafe
-public class RandomAccessReader extends RebufferingInputStream implements FileDataInput
+public class RandomAccessReader extends RebufferingInputStream implements FileDataInput, io.github.jbellis.jvector.disk.RandomAccessReader
 {
     // The default buffer size when the client doesn't specify it
     public static final int DEFAULT_BUFFER_SIZE = 4096;
@@ -37,17 +40,20 @@ public class RandomAccessReader extends RebufferingInputStream implements FileDa
     private long markedPointer;
 
     final Rebufferer rebufferer;
-    protected BufferHolder bufferHolder = Rebufferer.EMPTY;
+    private BufferHolder bufferHolder;
+    private final ByteOrder order;
 
     /**
      * Only created through Builder
      *
      * @param rebufferer Rebufferer to use
      */
-    protected RandomAccessReader(Rebufferer rebufferer)
+    RandomAccessReader(Rebufferer rebufferer, ByteOrder order, BufferHolder bufferHolder)
     {
-        super(Rebufferer.EMPTY.buffer());
+        super(bufferHolder.buffer(), false);
+        this.bufferHolder = bufferHolder;
         this.rebufferer = rebufferer;
+        this.order = order;
     }
 
     /**
@@ -64,11 +70,141 @@ public void reBuffer()
     private void reBufferAt(long position)
     {
         bufferHolder.release();
-        bufferHolder = rebufferer.rebuffer(position);
-        buffer = bufferHolder.buffer();
-        buffer.position(Ints.checkedCast(position - bufferHolder.offset()));
+        if (position == length())
+        {
+            bufferHolder = Rebufferer.emptyBufferHolderAt(position);
+            buffer = bufferHolder.buffer();
+        }
+        else
+        {
+            bufferHolder = Rebufferer.EMPTY; // prevents double release if the call below fails
+            bufferHolder = rebufferer.rebuffer(position);
+            buffer = bufferHolder.buffer();
+            buffer.position(Ints.checkedCast(position - bufferHolder.offset()));
+        }
+        buffer.order(order);
+    }
 
-        assert buffer.order() == ByteOrder.BIG_ENDIAN : "Buffer must have BIG ENDIAN byte ordering";
+    public ByteOrder order()
+    {
+        return order;
+    }
+
+    @Override
+    public void read(float[] dest, int offset, int count) throws IOException
+    {
+        for (int inBuffer = buffer.remaining() / Float.BYTES;
+             inBuffer < count;
+             inBuffer = buffer.remaining() / Float.BYTES)
+        {
+            if (inBuffer >= 1)
+            {
+                // read as much as we can from the buffer
+                readFloats(buffer, order, dest, offset, inBuffer);
+                offset += inBuffer;
+                count -= inBuffer;
+            }
+
+            if (buffer.remaining() > 0)
+            {
+                // read the buffer-spanning value using the slow path
+                dest[offset++] = readFloat();
+                --count;
+            }
+            else
+                reBuffer();
+        }
+
+        readFloats(buffer, order, dest, offset, count);
+    }
+
+    @Override
+    public void readFully(long[] dest) throws IOException
+    {
+        read(dest, 0, dest.length);
+    }
+
+    public void read(long[] dest, int offset, int count) throws IOException
+    {
+        for (int inBuffer = buffer.remaining() / Long.BYTES;
+             inBuffer < count;
+             inBuffer = buffer.remaining() / Long.BYTES)
+        {
+            if (inBuffer >= 1)
+            {
+                // read as much as we can from the buffer
+                readLongs(buffer, order, dest, offset, inBuffer);
+                offset += inBuffer;
+                count -= inBuffer;
+            }
+
+            if (buffer.remaining() > 0)
+            {
+                // read the buffer-spanning value using the slow path
+                dest[offset++] = readLong();
+                --count;
+            }
+            else
+                reBuffer();
+        }
+
+        readLongs(buffer, order, dest, offset, count);
+    }
+
+    @Override
+    public void read(int[] dest, int offset, int count) throws IOException
+    {
+        for (int inBuffer = buffer.remaining() / Integer.BYTES;
+             inBuffer < count;
+             inBuffer = buffer.remaining() / Integer.BYTES)
+        {
+            if (inBuffer >= 1)
+            {
+                // read as much as we can from the buffer
+                readInts(buffer, order, dest, offset, inBuffer);
+                offset += inBuffer;
+                count -= inBuffer;
+            }
+
+            if (buffer.remaining() > 0)
+            {
+                // read the buffer-spanning value using the slow path
+                dest[offset++] = readInt();
+                --count;
+            }
+            else
+                reBuffer();
+        }
+
+        readInts(buffer, order, dest, offset, count);
+    }
+
+    private static void readFloats(ByteBuffer buffer, ByteOrder order, float[] dest, int offset, int count)
+    {
+        FloatBuffer floatBuffer = updateBufferByteOrderIfNeeded(buffer, order).asFloatBuffer();
+        floatBuffer.get(dest, offset, count);
+        buffer.position(buffer.position() + count * Float.BYTES);
+    }
+
+    private static void readLongs(ByteBuffer buffer, ByteOrder order, long[] dest, int offset, int count)
+    {
+        LongBuffer longBuffer = updateBufferByteOrderIfNeeded(buffer, order).asLongBuffer();
+        longBuffer.get(dest, offset, count);
+        buffer.position(buffer.position() + count * Long.BYTES);
+    }
+
+    private static void readInts(ByteBuffer buffer, ByteOrder order, int[] dest, int offset, int count)
+    {
+        IntBuffer intBuffer = updateBufferByteOrderIfNeeded(buffer, order).asIntBuffer();
+        intBuffer.get(dest, offset, count);
+        buffer.position(buffer.position() + count * Integer.BYTES);
+    }
+
+    private static ByteBuffer updateBufferByteOrderIfNeeded(ByteBuffer buffer, ByteOrder order)
+    {
+        return buffer.order() != order
+               ? buffer.duplicate().order(order)
+               : buffer;    // Note: ?: rather than if to hit one-liner inlining path
     }
 
     @Override
@@ -84,9 +220,10 @@ protected long current()
         return bufferHolder.offset() + buffer.position();
     }
 
-    public String getPath()
+    @Override
+    public File getFile()
     {
-        return getChannel().filePath();
+        return getChannel().getFile();
     }
 
     public ChannelProxy getChannel()
@@ -158,7 +295,6 @@ public void close()
         // close needs to be idempotent.
         if (buffer == null)
             return;
-
         bufferHolder.release();
         rebufferer.closeReader();
         buffer = null;
@@ -205,7 +341,7 @@ public void seek(long newPosition)
 
         if (newPosition > length())
             throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in read-only mode",
-                                                         newPosition, getPath(), length()));
+                                                             newPosition, getFile(), length()));
         reBufferAt(newPosition);
     }
 
@@ -295,7 +431,7 @@ static class RandomAccessReaderWithOwnChannel extends RandomAccessReader
     {
         RandomAccessReaderWithOwnChannel(Rebufferer rebufferer)
         {
-            super(rebufferer);
+            super(rebufferer, ByteOrder.BIG_ENDIAN, Rebufferer.EMPTY);
         }
 
         @Override
@@ -341,4 +477,5 @@ public static RandomAccessReader open(File file)
             throw t;
         }
     }
+
 }
diff --git a/src/java/org/apache/cassandra/io/util/Rebufferer.java b/src/java/org/apache/cassandra/io/util/Rebufferer.java
index a7fbc7149d51..0c0bee8d86f8 100644
--- a/src/java/org/apache/cassandra/io/util/Rebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/Rebufferer.java
@@ -19,8 +19,6 @@
 package org.apache.cassandra.io.util;
 
 import java.nio.ByteBuffer;
-import java.nio.FloatBuffer;
-import java.nio.IntBuffer;
 
 /**
  * Rebufferer for reading data by a RandomAccessReader.
@@ -49,16 +47,6 @@ interface BufferHolder
          */
         ByteBuffer buffer();
 
-        default FloatBuffer floatBuffer()
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        default IntBuffer intBuffer()
-        {
-            throw new UnsupportedOperationException();
-        }
-
         /**
          * Position in the file of the start of the buffer.
          */
@@ -93,4 +81,28 @@ public void release()
             // nothing to do
         }
     };
+
+    static BufferHolder emptyBufferHolderAt(long offset)
+    {
+        return new BufferHolder()
+        {
+            @Override
+            public ByteBuffer buffer()
+            {
+                return EMPTY.buffer();
+            }
+
+            @Override
+            public long offset()
+            {
+                return offset;
+            }
+
+            @Override
+            public void release()
+            {
+                // nothing to do
+            }
+        };
+    }
 }
diff --git a/src/java/org/apache/cassandra/io/util/RebuffererFactory.java b/src/java/org/apache/cassandra/io/util/RebuffererFactory.java
index ec35f0ba530b..d8fb15604336 100644
--- a/src/java/org/apache/cassandra/io/util/RebuffererFactory.java
+++ b/src/java/org/apache/cassandra/io/util/RebuffererFactory.java
@@ -29,4 +29,6 @@
 public interface RebuffererFactory extends ReaderFileProxy
 {
     Rebufferer instantiateRebufferer();
+
+    void invalidateIfCached(long position);
 }
diff --git a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
index b7ae205e9661..32cb2b69c3d8 100644
--- a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
+++ b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
@@ -45,7 +45,14 @@ public abstract class RebufferingInputStream extends DataInputStreamPlus impleme
 
     protected RebufferingInputStream(ByteBuffer buffer)
     {
-        Preconditions.checkArgument(buffer == null || buffer.order() == ByteOrder.BIG_ENDIAN, "Buffer must have BIG ENDIAN byte ordering");
+        this(buffer, true);
+    }
+
+    protected RebufferingInputStream(ByteBuffer buffer, boolean validateByteOrder)
+    {
+        if (validateByteOrder)
+            Preconditions.checkArgument(buffer == null || buffer.order() == ByteOrder.BIG_ENDIAN,
+                                        "Buffer must have BIG ENDIAN byte ordering");
         this.buffer = buffer;
     }
 
@@ -135,7 +142,7 @@ public void readFully(ByteBuffer dst) throws IOException
     }
 
     @DontInline
-    protected long readPrimitiveSlowly(int bytes) throws IOException
+    protected long readBigEndianPrimitiveSlowly(int bytes) throws IOException
     {
         long result = 0;
         for (int i = 0; i < bytes; i++)
@@ -194,8 +201,10 @@ public short readShort() throws IOException
     {
         if (buffer.remaining() >= 2)
             return buffer.getShort();
-        else
-            return (short) readPrimitiveSlowly(2);
+        var result = (short) readBigEndianPrimitiveSlowly(2);
+        if (buffer.order() == ByteOrder.LITTLE_ENDIAN)
+            return Short.reverseBytes(result);
+        return result;
     }
 
     @Override
@@ -209,8 +218,10 @@ public char readChar() throws IOException
     {
         if (buffer.remaining() >= 2)
             return buffer.getChar();
-        else
-            return (char) readPrimitiveSlowly(2);
+        var result = (char) readBigEndianPrimitiveSlowly(2);
+        if (buffer.order() == ByteOrder.LITTLE_ENDIAN)
+            return Character.reverseBytes(result);
+        return result;
     }
 
     @Override
@@ -218,8 +229,10 @@ public int readInt() throws IOException
     {
         if (buffer.remaining() >= 4)
             return buffer.getInt();
-        else
-            return (int) readPrimitiveSlowly(4);
+        var result = (int) readBigEndianPrimitiveSlowly(4);
+        if (buffer.order() == ByteOrder.LITTLE_ENDIAN)
+            return Integer.reverseBytes(result);
+        return result;
     }
 
     @Override
@@ -227,8 +240,10 @@ public long readLong() throws IOException
     {
         if (buffer.remaining() >= 8)
             return buffer.getLong();
-        else
-            return readPrimitiveSlowly(8);
+        var result = readBigEndianPrimitiveSlowly(8);
+        if (buffer.order() == ByteOrder.LITTLE_ENDIAN)
+            return Long.reverseBytes(result);
+        return result;
     }
 
     @Override
@@ -286,8 +301,10 @@ public float readFloat() throws IOException
     {
         if (buffer.remaining() >= 4)
             return buffer.getFloat();
-        else
-            return Float.intBitsToFloat((int)readPrimitiveSlowly(4));
+        var intBits = (int) readBigEndianPrimitiveSlowly(4);
+        if (buffer.order() == ByteOrder.LITTLE_ENDIAN)
+            intBits = Integer.reverseBytes(intBits);
+        return Float.intBitsToFloat(intBits);
     }
 
     @Override
@@ -295,8 +312,10 @@ public double readDouble() throws IOException
     {
         if (buffer.remaining() >= 8)
             return buffer.getDouble();
-        else
-            return Double.longBitsToDouble(readPrimitiveSlowly(8));
+        var longBits = readBigEndianPrimitiveSlowly(8);
+        if (buffer.order() == ByteOrder.LITTLE_ENDIAN)
+            longBits = Long.reverseBytes(longBits);
+        return Double.longBitsToDouble(longBits);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/io/util/SafeMemory.java b/src/java/org/apache/cassandra/io/util/SafeMemory.java
index 4482d96019bd..cca8df90e128 100644
--- a/src/java/org/apache/cassandra/io/util/SafeMemory.java
+++ b/src/java/org/apache/cassandra/io/util/SafeMemory.java
@@ -88,7 +88,7 @@ public void tidy()
         {
             /** see {@link Memory#Memory(long)} re: null pointers*/
             if (peer != 0)
-                MemoryUtil.free(peer);
+                MemoryUtil.free(peer, size);
         }
 
         public String name()
diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
index 69643be98730..e0c00ee2fa24 100644
--- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
@@ -37,8 +37,7 @@
 public class SequentialWriter extends BufferedDataOutputStreamPlus implements Transactional
 {
     // absolute path to the given file
-    private final String filePath;
-    private final File file;
+    protected final File file;
 
     // Offset for start of buffer relative to underlying file
     protected long bufferOffset;
@@ -111,11 +110,11 @@ private static FileChannel openChannel(File file)
         {
             if (file.exists())
             {
-                return FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE);
+                return FileChannel.open(file.toPath(), StandardOpenOption.WRITE);
             }
             else
             {
-                FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+                FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
                 try
                 {
                     SyncUtil.trySyncDir(file.parent());
@@ -141,7 +140,7 @@ private static FileChannel openChannel(File file)
      */
     public SequentialWriter(File file)
     {
-       this(file, SequentialWriterOption.DEFAULT);
+        this(file, SequentialWriterOption.DEFAULT);
     }
 
     /**
@@ -168,7 +167,6 @@ public SequentialWriter(File file, SequentialWriterOption option, boolean strict
         this.fchannel = (FileChannel)channel;
 
         this.file = file;
-        this.filePath = file.absolutePath();
 
         this.option = option;
     }
@@ -196,7 +194,7 @@ protected void syncDataOnlyInternal()
         }
         catch (IOException e)
         {
-            throw new FSWriteError(e, getPath());
+            throw new FSWriteError(e, getFile());
         }
     }
 
@@ -250,7 +248,7 @@ protected void flushData()
         }
         catch (IOException e)
         {
-            throw new FSWriteError(e, getPath());
+            throw new FSWriteError(e, getFile());
         }
         if (runPostFlush != null)
             runPostFlush.accept(getLastFlushOffset());
@@ -268,6 +266,8 @@ public long position()
         return current();
     }
 
+    // Page management using on-disk pages
+
     @Override
     public int maxBytesInPage()
     {
@@ -319,15 +319,10 @@ public long length()
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, getPath());
+            throw new FSReadError(e, getFile());
         }
     }
 
-    public String getPath()
-    {
-        return filePath;
-    }
-
     public File getFile()
     {
         return file;
@@ -380,7 +375,7 @@ public void resetAndTruncate(DataPosition mark)
         }
         catch (IOException e)
         {
-            throw new FSReadError(e, getPath());
+            throw new FSReadError(e, getFile());
         }
 
         bufferOffset = truncateTarget;
@@ -401,7 +396,7 @@ public void truncate(long toSize)
         }
         catch (IOException e)
         {
-            throw new FSWriteError(e, getPath());
+            throw new FSWriteError(e, getFile());
         }
     }
 
@@ -422,6 +417,13 @@ public final Throwable commit(Throwable accumulate)
         return txnProxy.commit(accumulate);
     }
 
+    /**
+     * Stop the operation after errors, i.e. close and release all held resources.
+     *
+     * Do not use this to interrupt a write operation running in another thread.
+     * This is thread-unsafe, releasing and cleaning the buffer while it is being written can have disastrous
+     * consequences (e.g. SIGSEGV).
+     */
     @Override
     public final Throwable abort(Throwable accumulate)
     {
@@ -429,7 +431,7 @@ public final Throwable abort(Throwable accumulate)
     }
 
     @Override
-    public final void close()
+    public void close()
     {
         if (option.finishOnClose())
             txnProxy.finish();
diff --git a/src/java/org/apache/cassandra/io/util/SimpleChunkReader.java b/src/java/org/apache/cassandra/io/util/SimpleChunkReader.java
index 8d00ce5d4000..b112a2ccd073 100644
--- a/src/java/org/apache/cassandra/io/util/SimpleChunkReader.java
+++ b/src/java/org/apache/cassandra/io/util/SimpleChunkReader.java
@@ -26,19 +26,26 @@ class SimpleChunkReader extends AbstractReaderFileProxy implements ChunkReader
 {
     private final int bufferSize;
     private final BufferType bufferType;
+    private final long startOffset;
 
     SimpleChunkReader(ChannelProxy channel, long fileLength, BufferType bufferType, int bufferSize)
+    {
+        this(channel, fileLength, bufferType, bufferSize, 0);
+    }
+
+    SimpleChunkReader(ChannelProxy channel, long fileLength, BufferType bufferType, int bufferSize, long startOffset)
     {
         super(channel, fileLength);
         this.bufferSize = bufferSize;
         this.bufferType = bufferType;
+        this.startOffset = startOffset;
     }
 
     @Override
     public void readChunk(long position, ByteBuffer buffer)
     {
         buffer.clear();
-        channel.read(buffer, position);
+        channel.read(buffer, position - startOffset);
         buffer.flip();
     }
 
@@ -58,11 +65,24 @@ public BufferType preferredBufferType()
     public Rebufferer instantiateRebufferer()
     {
         if (Integer.bitCount(bufferSize) == 1)
+        {
+            assert startOffset == (startOffset & -bufferSize) : "startOffset must be aligned to buffer size";
             return new BufferManagingRebufferer.Aligned(this);
+        }
         else
             return new BufferManagingRebufferer.Unaligned(this);
     }
 
+    @Override
+    public void invalidateIfCached(long position)
+    {
+    }
+
+    public ReaderType type()
+    {
+        return ReaderType.SIMPLE;
+    }
+
     @Override
     public String toString()
     {
@@ -72,4 +92,4 @@ public String toString()
                              bufferSize,
                              fileLength());
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/io/util/SliceDescriptor.java b/src/java/org/apache/cassandra/io/util/SliceDescriptor.java
new file mode 100644
index 000000000000..27a997ab481b
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/SliceDescriptor.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.util.Objects;
+import java.util.StringJoiner;
+
+public class SliceDescriptor
+{
+    public static final SliceDescriptor NONE = new SliceDescriptor(0, 0, 0);
+
+    /**
+     * The position of the beginning of the data in the original file (inclusive).
+     */
+    public final long dataStart;
+
+    /**
+     * The position of the end of the data in the original file (exclusive).
+     */
+    public final long dataEnd;
+
+    /**
+     * The size of the chunk to which the slice is aligned.
+     */
+    public final int chunkSize;
+
+    /**
+     * The position of beginning of this slice in the original file (inclusive). It is the {@link #dataStart}
+     * aligned to the chunk size.
+     */
+    public final long sliceStart;
+
+    /**
+     * The position of the end of this slice in the original file (exclusive). It is the {@link #dataEnd}
+     * aligned to the chunk size. {@code sliceEnd - sliceStart} is equals to the actual size of the partial file.
+     */
+    public final long sliceEnd;
+
+    public SliceDescriptor(long dataStart, long dataEnd, int chunkSize)
+    {
+        this.dataStart = dataStart;
+        this.dataEnd = dataEnd;
+        this.chunkSize = chunkSize;
+
+        this.sliceStart = chunkSize == 0 ? dataStart : dataStart & -chunkSize;
+        this.sliceEnd = chunkSize == 0 ? dataEnd : (chunkSize + dataEnd - 1) & -chunkSize;
+    }
+
+    public boolean exists()
+    {
+        return dataStart > 0 || dataEnd > 0;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SliceDescriptor that = (SliceDescriptor) o;
+        return dataStart == that.dataStart && dataEnd == that.dataEnd && chunkSize == that.chunkSize;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(dataStart, dataEnd, chunkSize);
+    }
+
+    @Override
+    public String toString()
+    {
+        return new StringJoiner(", ", SliceDescriptor.class.getSimpleName() + "[", "]")
+               .add("dataStart=" + dataStart)
+               .add("dataEnd=" + dataEnd)
+               .add("chunkSize=" + chunkSize)
+               .add("sliceStart=" + sliceStart)
+               .add("sliceEnd=" + sliceEnd)
+               .toString();
+    }
+
+    public long dataEndOr(long dataEndIfNotExists)
+    {
+        return exists() ? dataEnd : dataEndIfNotExists;
+    }
+}
diff --git a/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java b/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java
index 5fbe5eaa0040..0a05d4206636 100644
--- a/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java
@@ -118,4 +118,5 @@ public void release()
         }
         buffer = null;
     }
-}
\ No newline at end of file
+
+}
diff --git a/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceSnitch.java b/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceSnitch.java
index 345e75918e1d..1f3508100968 100644
--- a/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceSnitch.java
+++ b/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceSnitch.java
@@ -20,13 +20,11 @@
 
 import java.util.Map;
 
+import org.apache.cassandra.nodes.Nodes;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.gms.ApplicationState;
-import org.apache.cassandra.gms.EndpointState;
-import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 
@@ -73,8 +71,8 @@ public final String getRack(InetAddressAndPort endpoint)
     {
         if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return getLocalRack();
-        EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
-        if (state == null || state.getApplicationState(ApplicationState.RACK) == null)
+        String rack = Nodes.getRack(endpoint, null);
+        if (rack == null)
         {
             if (savedEndpoints == null)
                 savedEndpoints = SystemKeyspace.loadDcRackInfo();
@@ -82,7 +80,7 @@ public final String getRack(InetAddressAndPort endpoint)
                 return savedEndpoints.get(endpoint).get("rack");
             return DEFAULT_RACK;
         }
-        return state.getApplicationState(ApplicationState.RACK).value;
+        return rack;
     }
 
     @Override
@@ -90,8 +88,8 @@ public final String getDatacenter(InetAddressAndPort endpoint)
     {
         if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return getLocalDatacenter();
-        EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
-        if (state == null || state.getApplicationState(ApplicationState.DC) == null)
+        String dc = Nodes.getDataCenter(endpoint, null);
+        if (dc == null)
         {
             if (savedEndpoints == null)
                 savedEndpoints = SystemKeyspace.loadDcRackInfo();
@@ -99,6 +97,6 @@ public final String getDatacenter(InetAddressAndPort endpoint)
                 return savedEndpoints.get(endpoint).get("data_center");
             return DEFAULT_DC;
         }
-        return state.getApplicationState(ApplicationState.DC).value;
+        return dc;
     }
 }
diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
index 31e60e611dd8..70e48ae9f991 100644
--- a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
+++ b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
@@ -20,18 +20,21 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 
 import com.google.common.base.Preconditions;
-
+import org.cliffc.high_scale_lib.NonBlockingHashMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Mutation;
@@ -48,7 +51,6 @@
 import org.apache.cassandra.service.WriteResponseHandler;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
-import org.cliffc.high_scale_lib.NonBlockingHashMap;
 
 /**
  * A abstract parent for all replication strategies.
@@ -73,6 +75,11 @@ protected AbstractReplicationStrategy(String keyspaceName, TokenMetadata tokenMe
         this.keyspaceName = keyspaceName;
     }
 
+    public TokenMetadata getTokenMetadata()
+    {
+        return tokenMetadata;
+    }
+
     public EndpointsForRange getCachedReplicas(long ringVersion, Token t)
     {
         return replicas.get(ringVersion, t);
@@ -92,9 +99,13 @@ public EndpointsForToken getNaturalReplicasForToken(RingPosition<?> searchPositi
 
     public EndpointsForRange getNaturalReplicas(RingPosition<?> searchPosition)
     {
+        ArrayList<Token> sortedTokens = tokenMetadata.sortedTokens();
+        if (sortedTokens.isEmpty())
+            return EndpointsForRange.empty(new Range<>(tokenMetadata.partitioner.getMinimumToken(), tokenMetadata.partitioner.getMinimumToken()));
+
         Token searchToken = searchPosition.getToken();
         long currentRingVersion = tokenMetadata.getRingVersion();
-        Token keyToken = TokenMetadata.firstToken(tokenMetadata.sortedTokens(), searchToken);
+        Token keyToken = TokenMetadata.firstToken(sortedTokens, searchToken);
         EndpointsForRange endpoints = getCachedReplicas(currentRingVersion, keyToken);
         if (endpoints == null)
         {
@@ -273,6 +284,23 @@ public EndpointsByRange getRangeAddresses(TokenMetadata metadata)
         return map.build();
     }
 
+    public Set<InetAddressAndPort> getAllEndpoints()
+    {
+        return tokenMetadata.cloneOnlyTokenMap().getAllEndpoints();
+    }
+
+    public EndpointsForRange getEndpointsForFullRange()
+    {
+        Range<Token> replicaRange = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMinimumToken());
+        Set<InetAddressAndPort> allEndpoints = tokenMetadata.cloneOnlyTokenMap().getAllEndpoints();
+        EndpointsForRange.Builder replicas = new EndpointsForRange.Builder(replicaRange, allEndpoints.size());
+
+        for (InetAddressAndPort ep : allEndpoints)
+            replicas.add(new Replica(ep, replicaRange, true));
+
+        return replicas.build();
+    }
+
     public RangesByEndpoint getAddressReplicas()
     {
         return getAddressReplicas(tokenMetadata.cloneOnlyTokenMap());
@@ -283,6 +311,14 @@ public RangesAtEndpoint getAddressReplicas(InetAddressAndPort endpoint)
         return getAddressReplicas(tokenMetadata.cloneOnlyTokenMap(), endpoint);
     }
 
+    /**
+     * Returns the number of token-owning nodes.
+     */
+    protected int getSizeOfRingMemebers()
+    {
+        return tokenMetadata.getAllRingMembers().size();
+    }
+
     public RangesAtEndpoint getPendingAddressRanges(TokenMetadata metadata, Token pendingToken, InetAddressAndPort pendingAddress)
     {
         return getPendingAddressRanges(metadata, Collections.singleton(pendingToken), pendingAddress);
@@ -370,6 +406,14 @@ public static AbstractReplicationStrategy createReplicationStrategy(String keysp
         return strategy;
     }
 
+    /**
+     * Whether this strategy partitions data across the ring
+     */
+    public boolean isPartitioned()
+    {
+        return true;
+    }
+
     /**
      * Before constructing the ARS we first give it a chance to prepare the options map in any way it
      * would like to. For example datacenter auto-expansion or other templating to make the user interface
@@ -467,7 +511,18 @@ protected void validateExpectedOptions() throws ConfigurationException
         for (String key : configOptions.keySet())
         {
             if (!expectedOptions.contains(key))
-                throw new ConfigurationException(String.format("Unrecognized strategy option {%s} passed to %s for keyspace %s", key, getClass().getSimpleName(), keyspaceName));
+            {
+                String message = String.format("Unrecognized strategy option {%s} passed to %s for keyspace %s", key, getClass().getSimpleName(), keyspaceName);
+
+                if (CassandraRelevantProperties.DATACENTER_SKIP_NAME_VALIDATION.getBoolean())
+                {
+                    logger.warn("{}=true. Ignoring: {}", CassandraRelevantProperties.DATACENTER_SKIP_NAME_VALIDATION.getKey(), message);
+                }
+                else
+                {
+                    throw new ConfigurationException(message);
+                }
+            }
         }
     }
 
diff --git a/src/java/org/apache/cassandra/locator/DefaultTokenMetadataProvider.java b/src/java/org/apache/cassandra/locator/DefaultTokenMetadataProvider.java
new file mode 100644
index 000000000000..0b0be05c08c6
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/DefaultTokenMetadataProvider.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+public class DefaultTokenMetadataProvider implements TokenMetadataProvider
+{
+    private volatile TokenMetadata tokenMetadata;
+
+    public DefaultTokenMetadataProvider()
+    {
+        this.tokenMetadata = new TokenMetadata();
+    }
+
+    @Override
+    public TokenMetadata getTokenMetadata()
+    {
+        return tokenMetadata;
+    }
+
+    @Override
+    public TokenMetadata getTokenMetadataForKeyspace(String keyspace)
+    {
+        return tokenMetadata;
+    }
+
+    /** @deprecated See STAR-1032 */
+    @Deprecated(forRemoval = true, since = "CC 4.0") // since we can select TMDP implementation via config, this method is no longer needed
+    public void replaceTokenMetadata(TokenMetadata newTokenMetadata)
+    {
+        this.tokenMetadata = newTokenMetadata;
+    }
+}
diff --git a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
index 77e04e6887ab..765cd341eaa2 100644
--- a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
+++ b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
@@ -20,26 +20,28 @@
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import com.codahale.metrics.ExponentiallyDecayingReservoir;
-
 import com.codahale.metrics.Snapshot;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.gms.ApplicationState;
-import org.apache.cassandra.gms.EndpointState;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.net.LatencySubscribers;
 import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MBeanWrapper;
 
@@ -55,6 +57,10 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat
     private static final double ALPHA = 0.75; // set to 0.75 to make EDS more biased to towards the newer values
     private static final int WINDOW_SIZE = 100;
 
+    // these need not be volatile; eventually the snitch will see the update and that's good enough
+    private double replicaLatencyQuantile = CassandraRelevantProperties.DYNAMIC_ENDPOINT_SNITCH_QUANTILE.getDouble();
+    private boolean quantizeToMillis = CassandraRelevantProperties.DYNAMIC_ENDPOINT_SNITCH_QUANTIZE_TO_MILLIS.getBoolean();
+
     private volatile int dynamicUpdateInterval = DatabaseDescriptor.getDynamicUpdateInterval();
     private volatile int dynamicResetInterval = DatabaseDescriptor.getDynamicResetInterval();
     private volatile double dynamicBadnessThreshold = DatabaseDescriptor.getDynamicBadnessThreshold();
@@ -267,21 +273,17 @@ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2)
 
     public void receiveTiming(InetAddressAndPort host, long latency, TimeUnit unit) // this is cheap
     {
-        ExponentiallyDecayingReservoir sample = samples.get(host);
-        if (sample == null)
-        {
-            ExponentiallyDecayingReservoir maybeNewSample = new ExponentiallyDecayingReservoir(WINDOW_SIZE, ALPHA);
-            sample = samples.putIfAbsent(host, maybeNewSample);
-            if (sample == null)
-                sample = maybeNewSample;
-        }
-        sample.update(unit.toMillis(latency));
+        ExponentiallyDecayingReservoir sample = samples.computeIfAbsent(host, k -> new ExponentiallyDecayingReservoir(WINDOW_SIZE, ALPHA));
+        if (quantizeToMillis)
+            sample.update(unit.toMillis(latency));
+        else
+            sample.update(unit.toNanos(latency));
     }
 
     @VisibleForTesting
     public void updateScores() // this is expensive
     {
-        if (!StorageService.instance.isInitialized())
+        if (!DynamicSnitchSeverityProvider.instance.isReady())
             return;
         if (!registered)
         {
@@ -302,17 +304,17 @@ public void updateScores() // this is expensive
 
         // We're going to weight the latency for each host against the worst one we see, to
         // arrive at sort of a 'badness percentage' for them. First, find the worst for each:
-        HashMap<InetAddressAndPort, Double> newScores = new HashMap<>();
+        HashMap<InetAddressAndPort, Double> newScores = new HashMap<>(samples.size());
         for (Map.Entry<InetAddressAndPort, Snapshot> entry : snapshots.entrySet())
         {
-            double mean = entry.getValue().getMedian();
-            if (mean > maxLatency)
-                maxLatency = mean;
+            double replicaLatency = entry.getValue().getValue(replicaLatencyQuantile);
+            if (replicaLatency > maxLatency)
+                maxLatency = replicaLatency;
         }
         // now make another pass to do the weighting based on the maximums we found before
         for (Map.Entry<InetAddressAndPort, Snapshot> entry : snapshots.entrySet())
         {
-            double score = entry.getValue().getMedian() / maxLatency;
+            double score = entry.getValue().getValue(replicaLatencyQuantile) / maxLatency;
             // finally, add the severity without any weighting, since hosts scale this relative to their own load and the size of the task causing the severity.
             // "Severity" is basically a measure of compaction activity (CASSANDRA-3722).
             if (USE_SEVERITY)
@@ -377,28 +379,49 @@ public void setSeverity(double severity)
 
     public static void addSeverity(double severity)
     {
-        Gossiper.instance.addLocalApplicationState(ApplicationState.SEVERITY, StorageService.instance.valueFactory.severity(severity));
+        DynamicSnitchSeverityProvider.instance.setSeverity(FBUtilities.getBroadcastAddressAndPort(), severity);
     }
 
     @VisibleForTesting
     public static double getSeverity(InetAddressAndPort endpoint)
     {
-        EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
-        if (state == null)
-            return 0.0;
-
-        VersionedValue event = state.getApplicationState(ApplicationState.SEVERITY);
-        if (event == null)
-            return 0.0;
-
-        return Double.parseDouble(event.value);
+        return DynamicSnitchSeverityProvider.instance.getSeverity(endpoint);
     }
 
+    @Override
     public double getSeverity()
     {
         return getSeverity(FBUtilities.getBroadcastAddressAndPort());
     }
 
+    @Override
+    public void setQuantile(double quantile)
+    {
+        if (quantile < 0.0 || quantile > 1.0 || Double.isNaN(quantile))
+        {
+            throw new IllegalArgumentException(quantile + " is not in [0..1]");
+        }
+        replicaLatencyQuantile = quantile;
+    }
+
+    @Override
+    public double getQuantile()
+    {
+        return replicaLatencyQuantile;
+    }
+
+    @Override
+    public void setQuantizationToMillis(boolean enabled)
+    {
+        quantizeToMillis = enabled;
+    }
+
+    @Override
+    public boolean getQuantizationToMillis()
+    {
+        return quantizeToMillis;
+    }
+
     public boolean isWorthMergingForRangeQuery(ReplicaCollection<?> merged, ReplicaCollection<?> l1, ReplicaCollection<?> l2)
     {
         if (!subsnitch.isWorthMergingForRangeQuery(merged, l1, l2))
@@ -438,4 +461,22 @@ public boolean validate(Set<String> datacenters, Set<String> racks)
     {
         return subsnitch.validate(datacenters, racks);
     }
+
+    @Override
+    public InetAddressAndPort getPreferredAddress(InetAddressAndPort remoteEndpoint)
+    {
+        return subsnitch.getPreferredAddress(remoteEndpoint);
+    }
+
+    @Override
+    public boolean acceptsNodesFromSameRack(int rf, int rackCount)
+    {
+        return subsnitch.acceptsNodesFromSameRack(rf, rackCount);
+    }
+
+    @Override
+    public Predicate<Replica> filterByAffinity(String keyspace)
+    {
+        return subsnitch.filterByAffinity(keyspace);
+    }
 }
diff --git a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java
index dd07b80e6649..6117f5736bb6 100644
--- a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java
+++ b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java
@@ -59,4 +59,23 @@ public interface DynamicEndpointSnitchMBean
      * @return the current manually injected Severity.
      */
     public double getSeverity();
+
+    /**
+     * set replica latency quantile used for replica score computation
+     * @param quantile (0.0 - 1.0); for default see
+     *        {@link org.apache.cassandra.config.CassandraRelevantProperties#DYNAMIC_ENDPOINT_SNITCH_QUANTILE}
+     */
+    public void setQuantile(double quantile);
+
+    /**
+     * @return the replica latency quantile currently used for replica score computation
+     */
+    public double getQuantile();
+
+    /**
+     * set replica latency quantization to 1ms
+     */
+    public void setQuantizationToMillis(boolean enabled);
+
+    public boolean getQuantizationToMillis();
 }
diff --git a/src/java/org/apache/cassandra/locator/DynamicSnitchSeverityProvider.java b/src/java/org/apache/cassandra/locator/DynamicSnitchSeverityProvider.java
new file mode 100644
index 000000000000..f022e318fa27
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/DynamicSnitchSeverityProvider.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.DYNAMIC_SNITCH_SEVERITY_PROVIDER;
+
+
+/**
+ * Class to abstract gossiper out of dynamic snitch
+ */
+public interface DynamicSnitchSeverityProvider
+{
+    DynamicSnitchSeverityProvider instance = DYNAMIC_SNITCH_SEVERITY_PROVIDER.isPresent()
+                                             ? FBUtilities.construct(DYNAMIC_SNITCH_SEVERITY_PROVIDER.getString(),
+                                                                     "Dynamic Snitch Severity Provider")
+                                             : new DefaultProvider();
+
+    /**
+     * @return true if initialization is completed and ready to update dynamic snitch scores
+     */
+    boolean isReady();
+
+    /**
+     * update the severity for given endpoint
+     *
+     * @param endpoint endpoint to be updated
+     * @param severity severity for the endpoint
+     */
+    void setSeverity(InetAddressAndPort endpoint, double severity);
+
+    /**
+     * @return severity for the endpoint or 0.0 if not found
+     */
+    double getSeverity(InetAddressAndPort endpoint);
+
+    class DefaultProvider implements DynamicSnitchSeverityProvider
+    {
+        @Override
+        public boolean isReady()
+        {
+            return StorageService.instance.isInitialized();
+        }
+
+        @Override
+        public void setSeverity(InetAddressAndPort endpoint, double severity)
+        {
+            if (!endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
+                throw new UnsupportedOperationException("Default severity provider only supports setting local severity, but got " + endpoint);
+
+            Gossiper.instance.addLocalApplicationState(ApplicationState.SEVERITY, StorageService.instance.valueFactory.severity(severity));
+        }
+
+        @Override
+        public double getSeverity(InetAddressAndPort endpoint)
+        {
+            EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
+            if (state == null)
+                return 0.0;
+
+            VersionedValue event = state.getApplicationState(ApplicationState.SEVERITY);
+            if (event == null)
+                return 0.0;
+
+            return Double.parseDouble(event.value);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java b/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java
index d836cd18062b..4ae60a8e740f 100644
--- a/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java
+++ b/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java
@@ -19,16 +19,24 @@
 
 import java.net.UnknownHostException;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.utils.MBeanWrapper;
 
 public class EndpointSnitchInfo implements EndpointSnitchInfoMBean
 {
-    public static void create()
+    public static void registerMBean()
     {
         MBeanWrapper.instance.registerMBean(new EndpointSnitchInfo(), "org.apache.cassandra.db:type=EndpointSnitchInfo");
     }
 
+    @VisibleForTesting
+    public static void unregisterMBean()
+    {
+        MBeanWrapper.instance.unregisterMBean("org.apache.cassandra.db:type=EndpointSnitchInfo");
+    }
+
     public String getDatacenter(String host) throws UnknownHostException
     {
         return DatabaseDescriptor.getEndpointSnitch().getDatacenter(InetAddressAndPort.getByName(host));
diff --git a/src/java/org/apache/cassandra/locator/EverywhereStrategy.java b/src/java/org/apache/cassandra/locator/EverywhereStrategy.java
new file mode 100644
index 000000000000..b6ac8b9d35bc
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/EverywhereStrategy.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+/**
+ * Strategy that replicate data on every {@code live} node.
+ *
+ * <p>This strategy is a {@code MultiDatacentersStrategy}. By consequence, it will handle properly local consistency levels.
+ * Nevertheless, as the data is replicated on every node, consistency levels such as QUORUM should not be used
+ * on clusters having more than 5 nodes.<p>
+ *
+ * <p>During bootstrap the time at which the data will be available is unknown and if the bootstrap is performed with
+ * autobootstrap=false on a seed node, there will be no data locally until rebuild is run.</p>
+ *
+ */
+public class EverywhereStrategy extends AbstractReplicationStrategy
+{
+    public EverywhereStrategy(String keyspaceName,
+                              TokenMetadata tokenMetadata,
+                              IEndpointSnitch snitch,
+                              Map<String, String> configOptions) throws ConfigurationException
+    {
+        super(keyspaceName, tokenMetadata, snitch, configOptions);
+    }
+
+    @Override
+    public EndpointsForRange calculateNaturalReplicas(Token searchToken, TokenMetadata tokenMetadata)
+    {
+        // Even if primary range repairs do not make a lot of sense for this strategy we want the behavior to be
+        // correct if somebody use it.
+        // Primary range repair expect the first endpoint of the list to be the primary range owner.
+        Set<Replica> replicas = new LinkedHashSet<>();
+        Iterator<Token> iter = TokenMetadata.ringIterator(tokenMetadata.sortedTokens(), searchToken, false);
+
+        if (iter.hasNext())
+        {
+            Token end = iter.next();
+            Token start = tokenMetadata.getPredecessor(end);
+            Range<Token> range = new Range<>(start, end);
+
+            InetAddressAndPort endpoint = tokenMetadata.getEndpoint(end);
+            replicas.add(Replica.fullReplica(endpoint, range));
+
+            while (iter.hasNext())
+            {
+                endpoint = tokenMetadata.getEndpoint(iter.next());
+                replicas.add(Replica.fullReplica(endpoint, range));
+            }
+        }
+
+        return EndpointsForRange.copyOf(replicas);
+    }
+
+    @Override
+    public ReplicationFactor getReplicationFactor()
+    {
+        return ReplicationFactor.fullOnly(getSizeOfRingMemebers());
+    }
+
+    @Override
+    public void validateOptions() throws ConfigurationException
+    {
+        // noop
+    }
+
+    @Override
+    public void maybeWarnOnOptions()
+    {
+        // noop
+    }
+
+    @Override
+    public Collection<String> recognizedOptions()
+    {
+        return Collections.emptyList();
+    }
+
+    /**
+     * CASSANDRA-12510 added a check that forbids decommission when the number of
+     * nodes will drop below the RF for a given keyspace. This check is breaking on
+     * EverywhereStrategy because all nodes replicate the keyspace, so this check does
+     * not make sense for partitioned keyspaces such as LocalStrategy and EverywhereStrategy.
+     *
+     * @return <code>false</code> because the data is not partitioned across the ring.
+     */
+    @Override
+    public boolean isPartitioned()
+    {
+        return false;
+    }
+}
diff --git a/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java b/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java
index 5aa7791e633b..510a90363d59 100644
--- a/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java
+++ b/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java
@@ -18,8 +18,8 @@
 
 package org.apache.cassandra.locator;
 
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -27,8 +27,8 @@
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.gms.ApplicationState;
-import org.apache.cassandra.gms.EndpointState;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -88,8 +88,8 @@ public String getDatacenter(InetAddressAndPort endpoint)
         if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return myDC;
 
-        EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
-        if (epState == null || epState.getApplicationState(ApplicationState.DC) == null)
+        String dc = Nodes.getDataCenter(endpoint, null);
+        if (dc == null)
         {
             if (psnitch == null)
             {
@@ -102,7 +102,7 @@ public String getDatacenter(InetAddressAndPort endpoint)
             else
                 return psnitch.getDatacenter(endpoint);
         }
-        return epState.getApplicationState(ApplicationState.DC).value;
+        return dc;
     }
 
     /**
@@ -116,8 +116,8 @@ public String getRack(InetAddressAndPort endpoint)
         if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return myRack;
 
-        EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
-        if (epState == null || epState.getApplicationState(ApplicationState.RACK) == null)
+        String rack = Nodes.getRack(endpoint, null);
+        if (rack == null)
         {
             if (psnitch == null)
             {
@@ -130,7 +130,7 @@ public String getRack(InetAddressAndPort endpoint)
             else
                 return psnitch.getRack(endpoint);
         }
-        return epState.getApplicationState(ApplicationState.RACK).value;
+        return rack;
     }
 
     public void gossiperStarting()
diff --git a/src/java/org/apache/cassandra/locator/IEndpointSnitch.java b/src/java/org/apache/cassandra/locator/IEndpointSnitch.java
index 0120391265d1..b2905f1af5d1 100644
--- a/src/java/org/apache/cassandra/locator/IEndpointSnitch.java
+++ b/src/java/org/apache/cassandra/locator/IEndpointSnitch.java
@@ -19,7 +19,9 @@
 
 import java.net.InetSocketAddress;
 import java.util.Set;
+import java.util.function.Predicate;
 
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.utils.FBUtilities;
 
 /**
@@ -94,4 +96,34 @@ default boolean validate(Set<String> datacenters, Set<String> racks)
     {
         return true;
     }
+
+    /**
+     * Get this endpoint address to advertise for connections to provided remote endpoint.
+     */
+    default InetAddressAndPort getPreferredAddress(InetAddressAndPort remoteEndpoint)
+    {
+        return FBUtilities.getBroadcastAddressAndPort();
+    }
+
+    /**
+     * Given the following {@code rf} and {@code rackCount}, returns true if nodes from the same rack should be accepted
+     * according to {@link AbstractReplicationStrategy#calculateNaturalReplicas(Token, TokenMetadata)}
+     * implementations, false otherwise.
+     * <br/><br/>
+     * Always returns true by default.
+     */
+    default boolean acceptsNodesFromSameRack(int rf, int rackCount)
+    {
+        return true;
+    }
+
+    /**
+     * Filters the given {@code addresses} by affinity to the given keyspace.
+     * <br/><br/>
+     * Always returns true by default.
+     */
+    default Predicate<Replica> filterByAffinity(String keyspace)
+    {
+        return replica -> true;
+    }
 }
diff --git a/src/java/org/apache/cassandra/locator/LocalStrategy.java b/src/java/org/apache/cassandra/locator/LocalStrategy.java
index 0e3a9185feda..64ab89c272e1 100644
--- a/src/java/org/apache/cassandra/locator/LocalStrategy.java
+++ b/src/java/org/apache/cassandra/locator/LocalStrategy.java
@@ -80,4 +80,10 @@ public Collection<String> recognizedOptions()
         // LocalStrategy doesn't expect any options.
         return Collections.emptySet();
     }
+
+    @Override
+    public boolean isPartitioned()
+    {
+        return false;
+    }
 }
diff --git a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
index 1615d8e5f57f..5eb9716903af 100644
--- a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
+++ b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
@@ -17,19 +17,30 @@
  */
 package org.apache.cassandra.locator;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.dht.Datacenters;
 import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
 import org.apache.cassandra.locator.TokenMetadata.Topology;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.ClientState;
@@ -38,11 +49,6 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-import com.google.common.collect.Sets;
-
 /**
  * <p>
  * This Replication Strategy takes a property file that gives the intended
@@ -116,7 +122,8 @@ private static final class DatacenterEndpoints
                             int rackCount,
                             int nodeCount,
                             EndpointsForRange.Builder replicas,
-                            Set<Pair<String, String>> racks)
+                            Set<Pair<String, String>> racks,
+                            IEndpointSnitch snitch)
         {
             this.replicas = replicas;
             this.racks = racks;
@@ -124,7 +131,7 @@ private static final class DatacenterEndpoints
             this.rfLeft = Math.min(rf.allReplicas, nodeCount);
             // If there aren't enough racks in this DC to fill the RF, we'll still use at least one node from each rack,
             // and the difference is to be filled by the first encountered nodes.
-            acceptableRackRepeats = rf.allReplicas - rackCount;
+            acceptableRackRepeats = snitch.acceptsNodesFromSameRack(rf.allReplicas, rackCount) ? rf.allReplicas - rackCount : 0;
 
             // if we have fewer replicas than rf calls for, reduce transients accordingly
             int reduceTransients = rf.allReplicas - this.rfLeft;
@@ -178,8 +185,12 @@ boolean done()
     @Override
     public EndpointsForRange calculateNaturalReplicas(Token searchToken, TokenMetadata tokenMetadata)
     {
-        // we want to preserve insertion order so that the first added endpoint becomes primary
         ArrayList<Token> sortedTokens = tokenMetadata.sortedTokens();
+        // handle the case of an empty ring and return an empty EndpointsForRange
+        if (sortedTokens.isEmpty())
+            return EndpointsForRange.empty(new Range<>(tokenMetadata.partitioner.getMinimumToken(), tokenMetadata.partitioner.getMinimumToken()));
+
+        // we want to preserve insertion order so that the first added endpoint becomes primary
         Token replicaEnd = TokenMetadata.firstToken(sortedTokens, searchToken);
         Token replicaStart = tokenMetadata.getPredecessor(replicaEnd);
         Range<Token> replicatedRange = new Range<>(replicaStart, replicaEnd);
@@ -207,7 +218,7 @@ public EndpointsForRange calculateNaturalReplicas(Token searchToken, TokenMetada
             if (rf.allReplicas <= 0 || nodeCount <= 0)
                 continue;
 
-            DatacenterEndpoints dcEndpoints = new DatacenterEndpoints(rf, sizeOrZero(racks.get(dc)), nodeCount, builder, seenRacks);
+            DatacenterEndpoints dcEndpoints = new DatacenterEndpoints(rf, sizeOrZero(racks.get(dc)), nodeCount, builder, seenRacks, snitch);
             dcs.put(dc, dcEndpoints);
             ++dcsToFill;
         }
@@ -343,7 +354,7 @@ public void maybeWarnOnOptions(ClientState state)
     {
         if (!SchemaConstants.isSystemKeyspace(keyspaceName))
         {
-            ImmutableMultimap<String, InetAddressAndPort> dcsNodes = Multimaps.index(StorageService.instance.getTokenMetadata().getAllMembers(), snitch::getDatacenter);
+            ImmutableMultimap<String, InetAddressAndPort> dcsNodes = Multimaps.index(StorageService.instance.getTokenMetadataForKeyspace(keyspaceName).getAllMembers(), snitch::getDatacenter);
             for (Entry<String, String> e : this.configOptions.entrySet())
             {
 
diff --git a/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java b/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java
index 3a9b161356f7..bf38f4eb46b9 100644
--- a/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java
+++ b/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java
@@ -24,8 +24,10 @@
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.concurrent.Callable;
 
 import com.google.common.collect.Sets;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,13 +35,10 @@
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.ResourceWatcher;
-import org.apache.cassandra.utils.WrappedRunnable;
-import org.apache.commons.lang3.StringUtils;
 
 /**
- * <p>
  * Used to determine if two IP's are in the same datacenter or on the same rack.
- * </p>
+ *
  * Based on a properties file in the following format:
  *
  * 10.0.0.13=DC1:RAC2
@@ -71,13 +70,7 @@ public PropertyFileSnitch(int refreshPeriodInSeconds) throws ConfigurationExcept
         try
         {
             FBUtilities.resourceToFile(SNITCH_PROPERTIES_FILENAME);
-            Runnable runnable = new WrappedRunnable()
-            {
-                protected void runMayThrow() throws ConfigurationException
-                {
-                    reloadConfiguration(true);
-                }
-            };
+            Callable<Boolean> runnable = () -> reloadConfiguration(true);
             ResourceWatcher.watch(SNITCH_PROPERTIES_FILENAME, runnable, refreshPeriodInSeconds * 1000);
         }
         catch (ConfigurationException ex)
@@ -137,7 +130,7 @@ public String getRack(InetAddressAndPort endpoint)
         return info[1];
     }
 
-    public void reloadConfiguration(boolean isUpdate) throws ConfigurationException
+    public boolean reloadConfiguration(boolean isUpdate) throws ConfigurationException
     {
         HashMap<InetAddressAndPort, String[]> reloadedMap = new HashMap<>();
         String[] reloadedDefaultDCRack = null;
@@ -161,9 +154,9 @@ public void reloadConfiguration(boolean isUpdate) throws ConfigurationException
             {
                 String[] newDefault = value.split(":");
                 if (newDefault.length < 2)
-                    reloadedDefaultDCRack = new String[] { "default", "default" };
+                    reloadedDefaultDCRack = new String[]{ "default", "default" };
                 else
-                    reloadedDefaultDCRack = new String[] { newDefault[0].trim(), newDefault[1].trim() };
+                    reloadedDefaultDCRack = new String[]{ newDefault[0].trim(), newDefault[1].trim() };
             }
             else
             {
@@ -179,9 +172,9 @@ public void reloadConfiguration(boolean isUpdate) throws ConfigurationException
                 }
                 String[] token = value.split(":");
                 if (token.length < 2)
-                    token = new String[] { "default", "default" };
+                    token = new String[]{ "default", "default" };
                 else
-                    token = new String[] { token[0].trim(), token[1].trim() };
+                    token = new String[]{ token[0].trim(), token[1].trim() };
                 reloadedMap.put(host, token);
             }
         }
@@ -198,19 +191,20 @@ public void reloadConfiguration(boolean isUpdate) throws ConfigurationException
             reloadedMap.put(localAddress, localInfo);
 
         if (isUpdate && !livenessCheck(reloadedMap, reloadedDefaultDCRack))
-            return;
+            return false;
 
-        if (logger.isTraceEnabled())
+        if (logger.isDebugEnabled())
         {
             StringBuilder sb = new StringBuilder();
             for (Map.Entry<InetAddressAndPort, String[]> entry : reloadedMap.entrySet())
                 sb.append(entry.getKey()).append(':').append(Arrays.toString(entry.getValue())).append(", ");
-            logger.trace("Loaded network topology from property file: {}", StringUtils.removeEnd(sb.toString(), ", "));
+            logger.debug("Loaded network topology from property file: {}", StringUtils.removeEnd(sb.toString(), ", "));
         }
 
-
         defaultDCRack = reloadedDefaultDCRack;
         endpointMap = reloadedMap;
+
+        //noinspection ConstantConditions
         if (StorageService.instance != null) // null check tolerates circular dependency; see CASSANDRA-4145
         {
             if (isUpdate)
@@ -220,13 +214,18 @@ public void reloadConfiguration(boolean isUpdate) throws ConfigurationException
         }
 
         if (gossipStarted)
+        {
             StorageService.instance.gossipSnitchInfo();
+            return true;
+        }
+
+        return false;
     }
 
     /**
      * We cannot update rack or data-center for a live node, see CASSANDRA-10243.
      *
-     * @param reloadedMap - the new map of hosts to dc:rack properties
+     * @param reloadedMap           - the new map of hosts to dc:rack properties
      * @param reloadedDefaultDCRack - the default dc:rack or null if no default
      * @return true if we can continue updating (no live host had dc or rack updated)
      */
@@ -236,14 +235,14 @@ private static boolean livenessCheck(HashMap<InetAddressAndPort, String[]> reloa
         // host quickly and interrupt the loop. Otherwise we only check the live hosts that were either
         // in the old set or in the new set
         Set<InetAddressAndPort> hosts = Arrays.equals(defaultDCRack, reloadedDefaultDCRack)
-                                 ? Sets.intersection(StorageService.instance.getLiveRingMembers(), // same default
-                                                     Sets.union(endpointMap.keySet(), reloadedMap.keySet()))
-                                 : StorageService.instance.getLiveRingMembers(); // default updated
+                                        ? Sets.intersection(StorageService.instance.getLiveRingMembers(), // same default
+                                                            Sets.union(endpointMap.keySet(), reloadedMap.keySet()))
+                                        : StorageService.instance.getLiveRingMembers(); // default updated
 
         for (InetAddressAndPort host : hosts)
         {
-            String[] origValue = endpointMap.containsKey(host) ? endpointMap.get(host) : defaultDCRack;
-            String[] updateValue = reloadedMap.containsKey(host) ? reloadedMap.get(host) : reloadedDefaultDCRack;
+            String[] origValue = endpointMap.getOrDefault(host, defaultDCRack);
+            String[] updateValue = reloadedMap.getOrDefault(host, reloadedDefaultDCRack);
 
             if (!Arrays.equals(origValue, updateValue))
             {
@@ -251,7 +250,7 @@ private static boolean livenessCheck(HashMap<InetAddressAndPort, String[]> reloa
                              origValue,
                              updateValue,
                              host);
-                 return false;
+                return false;
             }
         }
 
diff --git a/src/java/org/apache/cassandra/locator/ReplicaLayout.java b/src/java/org/apache/cassandra/locator/ReplicaLayout.java
index 1e939b2fc42b..62fb3236042c 100644
--- a/src/java/org/apache/cassandra/locator/ReplicaLayout.java
+++ b/src/java/org/apache/cassandra/locator/ReplicaLayout.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.locator;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
@@ -26,6 +27,7 @@
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.utils.FBUtilities;
 
 import java.util.Set;
@@ -330,7 +332,7 @@ public static ReplicaLayout.ForTokenRead forTokenReadLiveSorted(AbstractReplicat
     {
         EndpointsForToken replicas = replicationStrategy.getNaturalReplicasForToken(token);
         replicas = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas);
-        replicas = replicas.filter(FailureDetector.isReplicaAlive);
+        replicas = replicas.filter(IFailureDetector.isReplicaAlive);
         return new ReplicaLayout.ForTokenRead(replicationStrategy, replicas);
     }
 
@@ -343,6 +345,18 @@ static ReplicaLayout.ForRangeRead forRangeReadLiveSorted(AbstractReplicationStra
     {
         EndpointsForRange replicas = replicationStrategy.getNaturalReplicas(range.right);
         replicas = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas);
+        replicas = replicas.filter(IFailureDetector.isReplicaAlive);
+        return new ReplicaLayout.ForRangeRead(replicationStrategy, range, replicas);
+    }
+
+    // note that: range may span multiple vnodes
+    public static ReplicaLayout.ForRangeRead forFullRangeReadLiveSorted(AbstractReplicationStrategy replicationStrategy, AbstractBounds<PartitionPosition> range)
+    {
+        Preconditions.checkState(range.left.equals(DatabaseDescriptor.getPartitioner().getMinimumToken().minKeyBound()));
+        Preconditions.checkState(range.right.equals(DatabaseDescriptor.getPartitioner().getMinimumToken().minKeyBound()));
+
+        EndpointsForRange replicas = replicationStrategy.getEndpointsForFullRange();
+        replicas = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas);
         replicas = replicas.filter(FailureDetector.isReplicaAlive);
         return new ReplicaLayout.ForRangeRead(replicationStrategy, range, replicas);
     }
diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlan.java b/src/java/org/apache/cassandra/locator/ReplicaPlan.java
index 31dc2491fcf2..09ecc492e0c2 100644
--- a/src/java/org/apache/cassandra/locator/ReplicaPlan.java
+++ b/src/java/org/apache/cassandra/locator/ReplicaPlan.java
@@ -18,15 +18,16 @@
 
 package org.apache.cassandra.locator;
 
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
 import com.google.common.collect.Iterables;
+
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
 
-import java.util.function.Predicate;
-import java.util.function.Supplier;
-
 public interface ReplicaPlan<E extends Endpoints<E>, P extends ReplicaPlan<E, P>>
 {
     Keyspace keyspace();
diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java
index 0b370533ab04..5018b0165f16 100644
--- a/src/java/org/apache/cassandra/locator/ReplicaPlans.java
+++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java
@@ -18,16 +18,36 @@
 
 package org.apache.cassandra.locator;
 
-import com.carrotsearch.hppc.ObjectIntHashMap;
-import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
 
+import com.carrotsearch.hppc.ObjectIntHashMap;
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.DecoratedKey;
@@ -37,30 +57,14 @@
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.UnavailableException;
-import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexStatusManager;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.reads.AlwaysSpeculativeRetryPolicy;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
-
 import org.apache.cassandra.utils.FBUtilities;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import java.util.function.Predicate;
-
-import javax.annotation.Nullable;
 
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.filter;
@@ -79,6 +83,15 @@ public class ReplicaPlans
 
     private static final Range<Token> FULL_TOKEN_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMinimumToken());
 
+    private static final int REQUIRED_BATCHLOG_REPLICA_COUNT = Math.max(1, Math.min(2, CassandraRelevantProperties.REQUIRED_BATCHLOG_REPLICA_COUNT.getInt()));
+
+    static
+    {
+        int batchlogReplicaCount = CassandraRelevantProperties.REQUIRED_BATCHLOG_REPLICA_COUNT.getInt();
+        if (batchlogReplicaCount < 1 || 2 < batchlogReplicaCount)
+            logger.warn("System property {} was set to {} but must be 1 or 2. Running with {}", CassandraRelevantProperties.REQUIRED_BATCHLOG_REPLICA_COUNT.getKey(), batchlogReplicaCount, REQUIRED_BATCHLOG_REPLICA_COUNT);
+    }
+
     public static boolean isSufficientLiveReplicasForRead(AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints<?> liveReplicas)
     {
         switch (consistencyLevel)
@@ -214,17 +227,18 @@ public static ReplicaPlan.ForWrite forLocalBatchlogWrite()
     }
 
     /**
-     * Requires that the provided endpoints are alive.  Converts them to their relevant system replicas.
-     * Note that the liveAndDown collection and live are equal to the provided endpoints.
+     * Returns endpoint for a batchlog write.
      *
      * @param isAny if batch consistency level is ANY, in which case a local node will be picked
+     * @param preferLocalRack if true, a random endpoint from the local rack will be preferred for batch storage
+     * @param keyspaceName the name of the keyspace used to compute batch storage endpoints
      */
-    public static ReplicaPlan.ForWrite forBatchlogWrite(boolean isAny) throws UnavailableException
+    public static ReplicaPlan.ForWrite forBatchlogWrite(boolean isAny, boolean preferLocalRack, String keyspaceName) throws UnavailableException
     {
         // A single case we write not for range or token, but multiple mutations to many tokens
         Token token = DatabaseDescriptor.getPartitioner().getMinimumToken();
 
-        TokenMetadata.Topology topology = StorageService.instance.getTokenMetadata().cachedOnlyTokenMap().getTopology();
+        TokenMetadata.Topology topology = StorageService.instance.getTokenMetadataForKeyspace(keyspaceName).cachedOnlyTokenMap().getTopology();
         IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
         Multimap<String, InetAddressAndPort> localEndpoints = HashMultimap.create(topology.getDatacenterRacks()
                                                                                           .get(snitch.getLocalDatacenter()));
@@ -233,10 +247,29 @@ public static ReplicaPlan.ForWrite forBatchlogWrite(boolean isAny) throws Unavai
         //  - replicas should be in the local datacenter
         //  - choose min(2, number of qualifying candiates above)
         //  - allow the local node to be the only replica only if it's a single-node DC
-        Collection<InetAddressAndPort> chosenEndpoints = filterBatchlogEndpoints(snitch.getLocalRack(), localEndpoints);
+        Collection<InetAddressAndPort> chosenEndpoints = filterBatchlogEndpoints(preferLocalRack, snitch.getLocalRack(), localEndpoints);
 
-        if (chosenEndpoints.isEmpty() && isAny)
-            chosenEndpoints = Collections.singleton(FBUtilities.getBroadcastAddressAndPort());
+        // Batchlog is hosted by either one node or two nodes from different racks.
+        ConsistencyLevel consistencyLevel = chosenEndpoints.size() == 1 ? ConsistencyLevel.ONE : ConsistencyLevel.TWO;
+
+        if (chosenEndpoints.isEmpty())
+        {
+            if (isAny)
+            {
+                chosenEndpoints = Collections.singleton(FBUtilities.getBroadcastAddressAndPort());
+            }
+            else
+            {
+                // New/changed since DSP-23003: we immediately throw an UnavailableException here instead
+                // of letting the batchlog write unnecessarily timeout.
+                throw new UnavailableException("Cannot achieve consistency level " + consistencyLevel
+                        + " for batchlog in local DC, required:" + REQUIRED_BATCHLOG_REPLICA_COUNT
+                        + ", available:" + 0,
+                        consistencyLevel,
+                        REQUIRED_BATCHLOG_REPLICA_COUNT,
+                        0);
+            }
+        }
 
         Keyspace systemKeypsace = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME);
         ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWrite(
@@ -244,36 +277,36 @@ public static ReplicaPlan.ForWrite forBatchlogWrite(boolean isAny) throws Unavai
                 SystemReplicas.getSystemReplicas(chosenEndpoints).forToken(token),
                 EndpointsForToken.empty(token)
         );
-        // Batchlog is hosted by either one node or two nodes from different racks.
-        ConsistencyLevel consistencyLevel = liveAndDown.all().size() == 1 ? ConsistencyLevel.ONE : ConsistencyLevel.TWO;
         // assume that we have already been given live endpoints, and skip applying the failure detector
         return forWrite(systemKeypsace, consistencyLevel, liveAndDown, liveAndDown, writeAll);
     }
 
-    private static Collection<InetAddressAndPort> filterBatchlogEndpoints(String localRack,
+    @VisibleForTesting
+    public static Collection<InetAddressAndPort> filterBatchlogEndpoints(boolean preferLocalRack,
+                                                                          String localRack,
                                                                           Multimap<String, InetAddressAndPort> endpoints)
     {
-        return filterBatchlogEndpoints(localRack,
-                                       endpoints,
-                                       Collections::shuffle,
-                                       FailureDetector.isEndpointAlive,
-                                       ThreadLocalRandom.current()::nextInt);
+        return DatabaseDescriptor.getBatchlogEndpointStrategy().useDynamicSnitchScores && DatabaseDescriptor.isDynamicEndpointSnitch()
+               ? filterBatchlogEndpointsDynamic(preferLocalRack,localRack, endpoints, IFailureDetector.isEndpointAlive)
+               : filterBatchlogEndpointsRandom(preferLocalRack, localRack, endpoints,
+                                               Collections::shuffle,
+                                               IFailureDetector.isEndpointAlive,
+                                               ThreadLocalRandom.current()::nextInt);
     }
 
-    // Collect a list of candidates for batchlog hosting. If possible these will be two nodes from different racks.
-    @VisibleForTesting
-    public static Collection<InetAddressAndPort> filterBatchlogEndpoints(String localRack,
-                                                                         Multimap<String, InetAddressAndPort> endpoints,
-                                                                         Consumer<List<?>> shuffle,
-                                                                         Predicate<InetAddressAndPort> isAlive,
-                                                                         Function<Integer, Integer> indexPicker)
+
+    private static ListMultimap<String, InetAddressAndPort> validate(boolean preferLocalRack, String localRack,
+                                                                     Multimap<String, InetAddressAndPort> endpoints,
+                                                                     Predicate<InetAddressAndPort> isAlive)
     {
+        int endpointCount = endpoints.values().size();
         // special case for single-node data centers
-        if (endpoints.values().size() == 1)
-            return endpoints.values();
+        if (endpointCount <= REQUIRED_BATCHLOG_REPLICA_COUNT)
+            return ArrayListMultimap.create(endpoints);
 
         // strip out dead endpoints and localhost
-        ListMultimap<String, InetAddressAndPort> validated = ArrayListMultimap.create();
+        int rackCount = endpoints.keySet().size();
+        ListMultimap<String, InetAddressAndPort> validated = ArrayListMultimap.create(rackCount, endpointCount / rackCount);
         for (Map.Entry<String, InetAddressAndPort> entry : endpoints.entries())
         {
             InetAddressAndPort addr = entry.getValue();
@@ -281,42 +314,81 @@ public static Collection<InetAddressAndPort> filterBatchlogEndpoints(String loca
                 validated.put(entry.getKey(), entry.getValue());
         }
 
-        if (validated.size() <= 2)
-            return validated.values();
+        // return early if no more than 2 nodes:
+        if (validated.size() <= REQUIRED_BATCHLOG_REPLICA_COUNT)
+            return validated;
 
-        if (validated.size() - validated.get(localRack).size() >= 2)
+        // if the local rack is not preferred and there are enough nodes in other racks, remove it:
+        if (!(DatabaseDescriptor.getBatchlogEndpointStrategy().preferLocalRack || preferLocalRack)
+            && validated.size() - validated.get(localRack).size() >= REQUIRED_BATCHLOG_REPLICA_COUNT)
         {
-            // we have enough endpoints in other racks
+            // if the local rack should not be preferred and there are enough nodes in other racks, remove it:
             validated.removeAll(localRack);
         }
 
+        return validated;
+    }
+
+    // Collect a list of candidates for batchlog hosting. If possible these will be two nodes from different racks.
+    // Replicas are picked manually:
+    //  - replicas should be alive according to the failure detector
+    //  - replicas should be in the local datacenter
+    //  - choose min(2, number of qualifying candiates above)
+    //  - allow the local node to be the only replica only if it's a single-node DC
+    @VisibleForTesting
+    public static Collection<InetAddressAndPort> filterBatchlogEndpointsRandom(boolean preferLocalRack, String localRack,
+                                                                               Multimap<String, InetAddressAndPort> endpoints,
+                                                                               Consumer<List<?>> shuffle,
+                                                                               Predicate<InetAddressAndPort> isAlive,
+                                                                               Function<Integer, Integer> indexPicker)
+    {
+        ListMultimap<String, InetAddressAndPort> validated = validate(preferLocalRack, localRack, endpoints, isAlive);
+
+        // return early if no more than 2 nodes:
+        if (validated.size() <= REQUIRED_BATCHLOG_REPLICA_COUNT)
+            return validated.values();
+
+        /*
+         * if we have only 1 `other` rack to select replicas from (whether it be the local rack or a single non-local rack),
+         * pick two random nodes from there and return early;
+         * we are guaranteed to have at least two nodes in the single remaining rack because of the above if block.
+         */
         if (validated.keySet().size() == 1)
         {
             /*
-             * we have only 1 `other` rack to select replicas from (whether it be the local rack or a single non-local rack)
-             * pick two random nodes from there; we are guaranteed to have at least two nodes in the single remaining rack
-             * because of the preceding if block.
+             * if we have only 1 `other` rack to select replicas from (whether it be the local rack or a single non-local rack),
+             * pick two random nodes from there and return early;
+             * we are guaranteed to have at least two nodes in the single remaining rack because of the above if block.
              */
             List<InetAddressAndPort> otherRack = Lists.newArrayList(validated.values());
             shuffle.accept(otherRack);
-            return otherRack.subList(0, 2);
+            return otherRack.subList(0, REQUIRED_BATCHLOG_REPLICA_COUNT);
         }
 
         // randomize which racks we pick from if more than 2 remaining
+
         Collection<String> racks;
-        if (validated.keySet().size() == 2)
+        if (validated.keySet().size() == REQUIRED_BATCHLOG_REPLICA_COUNT)
         {
             racks = validated.keySet();
         }
+        else if (preferLocalRack || DatabaseDescriptor.getBatchlogEndpointStrategy().preferLocalRack)
+        {
+            List<String> nonLocalRacks = Lists.newArrayList(Sets.difference(validated.keySet(), ImmutableSet.of(localRack)));
+            racks = new LinkedHashSet<>();
+            racks.add(localRack);
+            racks.add(nonLocalRacks.get(indexPicker.apply(nonLocalRacks.size())));
+        }
         else
         {
             racks = Lists.newArrayList(validated.keySet());
             shuffle.accept((List<?>) racks);
         }
 
-        // grab a random member of up to two racks
-        List<InetAddressAndPort> result = new ArrayList<>(2);
-        for (String rack : Iterables.limit(racks, 2))
+        // grab two random nodes from two different racks
+
+        List<InetAddressAndPort> result = new ArrayList<>(REQUIRED_BATCHLOG_REPLICA_COUNT);
+        for (String rack : Iterables.limit(racks, REQUIRED_BATCHLOG_REPLICA_COUNT))
         {
             List<InetAddressAndPort> rackMembers = validated.get(rack);
             result.add(rackMembers.get(indexPicker.apply(rackMembers.size())));
@@ -325,6 +397,56 @@ public static Collection<InetAddressAndPort> filterBatchlogEndpoints(String loca
         return result;
     }
 
+    @VisibleForTesting
+    public static Collection<InetAddressAndPort> filterBatchlogEndpointsDynamic(boolean preferLocalRack, String localRack,
+                                                                                Multimap<String, InetAddressAndPort> endpoints,
+                                                                                Predicate<InetAddressAndPort> isAlive)
+    {
+        ListMultimap<String, InetAddressAndPort> validated = validate(preferLocalRack, localRack, endpoints, isAlive);
+
+        // return early if no more than 2 nodes:
+        if (validated.size() <= REQUIRED_BATCHLOG_REPLICA_COUNT)
+            return validated.values();
+
+        // sort _all_ nodes to pick the best racks
+        List<InetAddressAndPort> sorted = sortByProximity(validated.values());
+
+        List<InetAddressAndPort> result = new ArrayList<>(REQUIRED_BATCHLOG_REPLICA_COUNT);
+        Set<String> racks = new HashSet<>();
+
+        while (result.size() < REQUIRED_BATCHLOG_REPLICA_COUNT)
+        {
+            for (InetAddressAndPort endpoint : sorted)
+            {
+                if (result.size() == REQUIRED_BATCHLOG_REPLICA_COUNT)
+                    break;
+
+                if (racks.isEmpty())
+                    racks.addAll(validated.keySet());
+
+                String rack = DatabaseDescriptor.getEndpointSnitch().getRack(endpoint);
+                if (!racks.remove(rack))
+                    continue;
+                if (result.contains(endpoint))
+                    continue;
+
+                result.add(endpoint);
+            }
+        }
+
+        return result;
+    }
+
+    @VisibleForTesting
+    public static List<InetAddressAndPort> sortByProximity(Collection<InetAddressAndPort> endpoints)
+    {
+        EndpointsForRange endpointsForRange = SystemReplicas.getSystemReplicas(endpoints);
+        return DatabaseDescriptor.getEndpointSnitch()
+                .sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), endpointsForRange)
+                .endpointList();
+    }
+
+
     public static ReplicaPlan.ForWrite forReadRepair(Token token, ReplicaPlan<?, ?> readPlan) throws UnavailableException
     {
         return forWrite(readPlan.keyspace(), readPlan.consistencyLevel(), token, writeReadRepair(readPlan));
@@ -343,7 +465,7 @@ public static ReplicaPlan.ForWrite forWrite(Keyspace keyspace, ConsistencyLevel
 
     public static ReplicaPlan.ForWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, Selector selector) throws UnavailableException
     {
-        return forWrite(keyspace, consistencyLevel, liveAndDown, FailureDetector.isReplicaAlive, selector);
+        return forWrite(keyspace, consistencyLevel, liveAndDown, IFailureDetector.isReplicaAlive, selector);
     }
 
     private static ReplicaPlan.ForWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, Predicate<Replica> isAlive, Selector selector) throws UnavailableException
@@ -510,7 +632,7 @@ public static ReplicaPlan.ForPaxosWrite forPaxos(Keyspace keyspace, DecoratedKey
             liveAndDown = liveAndDown.filter(InOurDc.replicas());
         }
 
-        ReplicaLayout.ForTokenWrite live = liveAndDown.filter(FailureDetector.isReplicaAlive);
+        ReplicaLayout.ForTokenWrite live = liveAndDown.filter(IFailureDetector.isReplicaAlive);
 
         // TODO: this should use assureSufficientReplicas
         int participants = liveAndDown.all().size();
@@ -606,8 +728,11 @@ public static ReplicaPlan.ForTokenRead forRead(Keyspace keyspace,
                                                    SpeculativeRetryPolicy retry)
     {
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
-        EndpointsForToken candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, ReplicaLayout.forTokenReadLiveSorted(replicationStrategy, token).natural());
-        EndpointsForToken contacts = contactForRead(replicationStrategy, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates);
+        IEndpointSnitch endpointSnitch = DatabaseDescriptor.getEndpointSnitch();
+        EndpointsForToken candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, ReplicaLayout.forTokenReadLiveSorted(replicationStrategy, token).natural())
+                                       .filter(endpointSnitch.filterByAffinity(keyspace.getName()));
+        EndpointsForToken contacts = contactForRead(replicationStrategy, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates)
+                                     .filter(endpointSnitch.filterByAffinity(keyspace.getName()));
 
         assureSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, contacts);
         return new ReplicaPlan.ForTokenRead(keyspace, replicationStrategy, consistencyLevel, candidates, contacts);
@@ -627,8 +752,11 @@ public static ReplicaPlan.ForRangeRead forRangeRead(Keyspace keyspace,
                                                         int vnodeCount)
     {
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
-        EndpointsForRange candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, ReplicaLayout.forRangeReadLiveSorted(replicationStrategy, range).natural());
-        EndpointsForRange contacts = contactForRead(replicationStrategy, consistencyLevel, false, candidates);
+        IEndpointSnitch endpointSnitch = DatabaseDescriptor.getEndpointSnitch();
+        EndpointsForRange candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, ReplicaLayout.forRangeReadLiveSorted(replicationStrategy, range).natural())
+                                       .filter(endpointSnitch.filterByAffinity(keyspace.getName()));
+        EndpointsForRange contacts = contactForRead(replicationStrategy, consistencyLevel, false, candidates)
+                                     .filter(endpointSnitch.filterByAffinity(keyspace.getName()));
 
         assureSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, contacts);
         return new ReplicaPlan.ForRangeRead(keyspace, replicationStrategy, consistencyLevel, range, candidates, contacts, vnodeCount);
diff --git a/src/java/org/apache/cassandra/locator/ReplicationFactor.java b/src/java/org/apache/cassandra/locator/ReplicationFactor.java
index ee971d900dcf..35f2d16184a6 100644
--- a/src/java/org/apache/cassandra/locator/ReplicationFactor.java
+++ b/src/java/org/apache/cassandra/locator/ReplicationFactor.java
@@ -28,6 +28,8 @@
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.nodes.IPeerInfo;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.utils.FBUtilities;
 
 public class ReplicationFactor
@@ -73,7 +75,10 @@ static void validate(int totalRF, int transientRF)
                                         "Transient nodes are not allowed with multiple tokens");
             Stream<InetAddressAndPort> endpoints = Stream.concat(Gossiper.instance.getLiveMembers().stream(), Gossiper.instance.getUnreachableMembers().stream());
             List<InetAddressAndPort> badVersionEndpoints = endpoints.filter(Predicates.not(FBUtilities.getBroadcastAddressAndPort()::equals))
-                                                                    .filter(endpoint -> Gossiper.instance.getReleaseVersion(endpoint) != null && Gossiper.instance.getReleaseVersion(endpoint).major < 4)
+                                                                    .map(endpoint -> Nodes.peers().get(endpoint))
+                                                                    .filter(Objects::nonNull)
+                                                                    .filter(info -> info.getReleaseVersion() != null && info.getReleaseVersion().major < 4)
+                                                                    .map(IPeerInfo::getPeerAddressAndPort)
                                                                     .collect(Collectors.toList());
             if (!badVersionEndpoints.isEmpty())
                 throw new IllegalArgumentException("Transient replication is not supported in mixed version clusters with nodes < 4.0. Bad nodes: " + badVersionEndpoints);
diff --git a/src/java/org/apache/cassandra/locator/SystemReplicas.java b/src/java/org/apache/cassandra/locator/SystemReplicas.java
index 456bae5a5272..421c5c680a61 100644
--- a/src/java/org/apache/cassandra/locator/SystemReplicas.java
+++ b/src/java/org/apache/cassandra/locator/SystemReplicas.java
@@ -19,20 +19,25 @@
 package org.apache.cassandra.locator;
 
 import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.collect.Collections2;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 
 public class SystemReplicas
 {
-    private static final Map<InetAddressAndPort, Replica> systemReplicas = new ConcurrentHashMap<>();
     public static final Range<Token> FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(),
                                                               DatabaseDescriptor.getPartitioner().getMinimumToken());
 
+    // System replicas cache: entries expire after 1 day of being unused to avoid growing indefinitely
+    private static final Cache<InetAddressAndPort, Replica> systemReplicas = Caffeine.newBuilder().expireAfterAccess(1, TimeUnit.DAYS).build();
+
+
     private static Replica createSystemReplica(InetAddressAndPort endpoint)
     {
         return new Replica(endpoint, FULL_RANGE, true);
@@ -44,7 +49,7 @@ private static Replica createSystemReplica(InetAddressAndPort endpoint)
      */
     public static Replica getSystemReplica(InetAddressAndPort endpoint)
     {
-        return systemReplicas.computeIfAbsent(endpoint, SystemReplicas::createSystemReplica);
+        return systemReplicas.get(endpoint, SystemReplicas::createSystemReplica);
     }
 
     public static EndpointsForRange getSystemReplicas(Collection<InetAddressAndPort> endpoints)
diff --git a/src/java/org/apache/cassandra/locator/TokenMetadata.java b/src/java/org/apache/cassandra/locator/TokenMetadata.java
index 7cb3a449948f..e635bed22965 100644
--- a/src/java/org/apache/cassandra/locator/TokenMetadata.java
+++ b/src/java/org/apache/cassandra/locator/TokenMetadata.java
@@ -42,7 +42,7 @@
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.BiMultiValMap;
@@ -152,7 +152,32 @@ private TokenMetadata(BiMultiValMap<Token, InetAddressAndPort> tokenToEndpointMa
     @VisibleForTesting
     public TokenMetadata cloneWithNewPartitioner(IPartitioner newPartitioner)
     {
-        return new TokenMetadata(tokenToEndpointMap, endpointToHostIdMap, topology, newPartitioner);
+        lock.readLock().lock();
+        try
+        {
+            return new TokenMetadata(tokenToEndpointMap, endpointToHostIdMap, topology, newPartitioner);
+        }
+        finally
+        {
+            lock.readLock().unlock();
+        }
+    }
+
+    /**
+     * To be used by tests only (via {@link org.apache.cassandra.service.StorageService#setPartitionerUnsafe}).
+     */
+    public TokenMetadata cloneWithNewSnitch(IEndpointSnitch snitch)
+    {
+        lock.readLock().lock();
+        try
+        {
+            var clonedTopology = topology.unbuild().withSnitchSupplier(() -> snitch).build();
+            return new TokenMetadata(tokenToEndpointMap, endpointToHostIdMap, clonedTopology, partitioner);
+        }
+        finally
+        {
+            lock.readLock().unlock();
+        }
     }
 
     private ArrayList<Token> sortTokens()
@@ -289,7 +314,7 @@ private void updateEndpointToHostIdMap(UUID hostId, InetAddressAndPort endpoint)
         InetAddressAndPort storedEp = endpointToHostIdMap.inverse().get(hostId);
         if (storedEp != null)
         {
-            if (!storedEp.equals(endpoint) && (FailureDetector.instance.isAlive(storedEp)))
+            if (!storedEp.equals(endpoint) && (IFailureDetector.instance.isAlive(storedEp)))
             {
                 throw new RuntimeException(String.format("Host ID collision between active endpoint %s and %s (id=%s)",
                                                          storedEp,
@@ -1164,6 +1189,19 @@ public int getSizeOfMovingEndpoints()
         }
     }
 
+    public Set<InetAddressAndPort> getAllRingMembers()
+    {
+        lock.readLock().lock();
+        try
+        {
+            return ImmutableSet.copyOf(tokenToEndpointMap.valueSet());
+        }
+        finally
+        {
+            lock.readLock().unlock();
+        }
+    }
+
     public static int firstTokenIndex(final ArrayList<Token> ring, Token start, boolean insertMin)
     {
         assert ring.size() > 0;
@@ -1492,7 +1530,7 @@ static Builder builder(Supplier<IEndpointSnitch> snitchSupplier)
 
         static Topology empty()
         {
-            return builder(() -> DatabaseDescriptor.getEndpointSnitch()).build();
+            return builder(DatabaseDescriptor::getEndpointSnitch).build();
         }
 
         private static class Builder
@@ -1503,7 +1541,7 @@ private static class Builder
             private final Map<String, Multimap<String, InetAddressAndPort>> dcRacks;
             /** reverse-lookup map for endpoint to current known dc/rack assignment */
             private final Map<InetAddressAndPort, Pair<String, String>> currentLocations;
-            private final Supplier<IEndpointSnitch> snitchSupplier;
+            private Supplier<IEndpointSnitch> snitchSupplier;
 
             Builder(Supplier<IEndpointSnitch> snitchSupplier)
             {
@@ -1575,7 +1613,7 @@ private void doRemoveEndpoint(InetAddressAndPort ep, Pair<String, String> curren
 
             Builder updateEndpoint(InetAddressAndPort ep)
             {
-                IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+                IEndpointSnitch snitch = snitchSupplier.get();
                 if (snitch == null || !currentLocations.containsKey(ep))
                     return this;
 
@@ -1585,7 +1623,7 @@ Builder updateEndpoint(InetAddressAndPort ep)
 
             Builder updateEndpoints()
             {
-                IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+                IEndpointSnitch snitch = snitchSupplier.get();
                 if (snitch == null)
                     return this;
 
@@ -1607,6 +1645,12 @@ private void updateEndpoint(InetAddressAndPort ep, IEndpointSnitch snitch)
                 doAddEndpoint(ep, dc, rack);
             }
 
+            Builder withSnitchSupplier(Supplier<IEndpointSnitch> snitchSupplier)
+            {
+                this.snitchSupplier = snitchSupplier;
+                return this;
+            }
+
             Topology build()
             {
                 return new Topology(this);
diff --git a/src/java/org/apache/cassandra/locator/TokenMetadataProvider.java b/src/java/org/apache/cassandra/locator/TokenMetadataProvider.java
new file mode 100644
index 000000000000..085ec6b47abd
--- /dev/null
+++ b/src/java/org/apache/cassandra/locator/TokenMetadataProvider.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_TMD_PROVIDER_PROPERTY;
+
+/**
+ * Provides access to the {@link TokenMetadata} instance used by this node.
+ */
+public interface TokenMetadataProvider
+{
+    TokenMetadataProvider instance = CUSTOM_TMD_PROVIDER_PROPERTY.isPresent()
+                                     ? FBUtilities.construct(CUSTOM_TMD_PROVIDER_PROPERTY.getString(),
+                                                             "Token Metadata Provider")
+                                     : new DefaultTokenMetadataProvider();
+
+    /**
+     * Returns the default TokenMetadata instance.
+     */
+    TokenMetadata getTokenMetadata();
+
+    /**
+     * Returns the per-keyspace TokenMetadata instance.
+     */
+    TokenMetadata getTokenMetadataForKeyspace(String keyspace);
+
+    @VisibleForTesting
+    /** @deprecated See STAR-1032 */
+    @Deprecated(forRemoval = true, since = "CC 4.0") // since we can select TMDP implementation via config, this method is no longer needed
+    void replaceTokenMetadata(TokenMetadata newTokenMetadata);
+}
diff --git a/src/java/org/apache/cassandra/metrics/BatchMetrics.java b/src/java/org/apache/cassandra/metrics/BatchMetrics.java
index 9bea16211694..db69cf7e126b 100644
--- a/src/java/org/apache/cassandra/metrics/BatchMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/BatchMetrics.java
@@ -17,7 +17,9 @@
  */
 package org.apache.cassandra.metrics;
 
+import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
+import org.apache.cassandra.cql3.statements.BatchStatement;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
@@ -25,14 +27,54 @@ public class BatchMetrics
 {
     private static final MetricNameFactory factory = new DefaultNameFactory("Batch");
 
+    public final Counter numLoggedBatches;
+    public final Counter numUnloggedBatches;
+    public final Counter numCounterBatches;
+
     public final Histogram partitionsPerLoggedBatch;
     public final Histogram partitionsPerUnloggedBatch;
     public final Histogram partitionsPerCounterBatch;
 
+    public final Histogram columnsPerLoggedBatch;
+    public final Histogram columnsPerUnloggedBatch;
+    public final Histogram columnsPerCounterBatch;
+
     public BatchMetrics()
     {
+        numLoggedBatches = Metrics.counter(factory.createMetricName("NumLoggedBatches"));
+        numUnloggedBatches = Metrics.counter(factory.createMetricName("NumUnloggedBatches"));
+        numCounterBatches = Metrics.counter(factory.createMetricName("NumCounterBatches"));
+
         partitionsPerLoggedBatch = Metrics.histogram(factory.createMetricName("PartitionsPerLoggedBatch"), false);
         partitionsPerUnloggedBatch = Metrics.histogram(factory.createMetricName("PartitionsPerUnloggedBatch"), false);
         partitionsPerCounterBatch = Metrics.histogram(factory.createMetricName("PartitionsPerCounterBatch"), false);
+
+        columnsPerLoggedBatch = Metrics.histogram(factory.createMetricName("ColumnsPerLoggedBatch"), false);
+        columnsPerUnloggedBatch = Metrics.histogram(factory.createMetricName("ColumnsPerUnloggedBatch"), false);
+        columnsPerCounterBatch = Metrics.histogram(factory.createMetricName("ColumnsPerCounterBatch"), false);
+    }
+
+    public void update(BatchStatement.Type batchType, int updatedPartitions, int updatedColumns)
+    {
+        switch (batchType)
+        {
+            case LOGGED:
+                numLoggedBatches.inc();
+                partitionsPerLoggedBatch.update(updatedPartitions);
+                columnsPerLoggedBatch.update(updatedColumns);
+                break;
+            case COUNTER:
+                numCounterBatches.inc();
+                partitionsPerCounterBatch.update(updatedPartitions);
+                columnsPerCounterBatch.update(updatedColumns);
+                break;
+            case UNLOGGED:
+                numUnloggedBatches.inc();
+                partitionsPerUnloggedBatch.update(updatedPartitions);
+                columnsPerUnloggedBatch.update(updatedColumns);
+                break;
+            default:
+                throw new IllegalStateException("Unexpected batch type: " + batchType);
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/metrics/BufferPoolMetrics.java b/src/java/org/apache/cassandra/metrics/BufferPoolMetrics.java
index 71373b35e886..49fd8ff33ee0 100644
--- a/src/java/org/apache/cassandra/metrics/BufferPoolMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/BufferPoolMetrics.java
@@ -17,51 +17,34 @@
  */
 package org.apache.cassandra.metrics;
 
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Meter;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.utils.memory.BufferPool;
 
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-
-public class BufferPoolMetrics
+public interface BufferPoolMetrics
 {
-    /** Total number of hits */
-    public final Meter hits;
-
-    /** Total number of misses */
-    public final Meter misses;
-
-    /** Total threshold for a certain type of buffer pool*/
-    public final Gauge<Long> capacity;
-
-    /** Total size of buffer pools, in bytes, including overflow allocation */
-    public final Gauge<Long> size;
-
-    /** Total size, in bytes, of active buffered being used from the pool currently + overflow */
-    public final Gauge<Long> usedSize;
-
-    /**
-     * Total size, in bytes, of direct or heap buffers allocated by the pool but not part of the pool
-     * either because they are too large to fit or because the pool has exceeded its maximum limit or because it's
-     * on-heap allocation.
-     */
-    public final Gauge<Long> overflowSize;
-
-    public BufferPoolMetrics(String scope, BufferPool bufferPool)
+    static BufferPoolMetrics create(String name, BufferPool bufferPool)
     {
-        MetricNameFactory factory = new DefaultNameFactory("BufferPool", scope);
+        return CassandraRelevantProperties.USE_MICROMETER.getBoolean()
+               ? new MicrometerBufferPoolMetrics(name, bufferPool)
+               : new CodahaleBufferPoolMetrics(name, bufferPool);
+    }
 
-        hits = Metrics.meter(factory.createMetricName("Hits"));
+    void markHit();
 
-        misses = Metrics.meter(factory.createMetricName("Misses"));
+    long hits();
 
-        capacity = Metrics.register(factory.createMetricName("Capacity"), bufferPool::memoryUsageThreshold);
+    void markMissed();
 
-        overflowSize = Metrics.register(factory.createMetricName("OverflowSize"), bufferPool::overflowMemoryInBytes);
+    long misses();
 
-        usedSize = Metrics.register(factory.createMetricName("UsedSize"), bufferPool::usedSizeInBytes);
+    long overflowSize();
 
-        size = Metrics.register(factory.createMetricName("Size"), bufferPool::sizeInBytes);
-    }
+    long usedSize();
 
+    long size();
+
+    /**
+     * used to register alias for 3.0/3.11 compatibility
+     */
+    void register3xAlias();
 }
diff --git a/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java b/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
index 654bb059d16e..2a069d7e88e3 100644
--- a/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
@@ -30,19 +30,50 @@ public class CASClientRequestMetrics extends ClientRequestMetrics
     public final Counter unfinishedCommit;
     public final Meter unknownResult;
 
-    public CASClientRequestMetrics(String scope)
+    // latencies for 4 paxos phases
+    public final LatencyMetrics prepareLatency;
+    public final LatencyMetrics createProposalLatency;
+    public final LatencyMetrics proposeLatency;
+    public final LatencyMetrics commitLatency;
+
+    // latency for backoff when there is contention
+    public final LatencyMetrics contentionBackoffLatency;
+
+    // num of replicas that are missing MRC
+    public final Counter missingMostRecentCommit;
+
+    public CASClientRequestMetrics(String scope, String namePrefix)
     {
-        super(scope);
-        contention = Metrics.histogram(factory.createMetricName("ContentionHistogram"), false);
-        unfinishedCommit = Metrics.counter(factory.createMetricName("UnfinishedCommit"));
-        unknownResult = Metrics.meter(factory.createMetricName("UnknownResult"));
+        super(scope, namePrefix);
+
+        contention = Metrics.histogram(factory.createMetricName(namePrefix + "ContentionHistogram"), false);
+        unfinishedCommit = Metrics.counter(factory.createMetricName(namePrefix + "UnfinishedCommit"));
+        unknownResult = Metrics.meter(factory.createMetricName(namePrefix + "UnknownResult"));
+
+        prepareLatency = new LatencyMetrics(factory, namePrefix + "Prepare");
+        createProposalLatency = new LatencyMetrics(factory, namePrefix + "CreateProposal");
+        proposeLatency = new LatencyMetrics(factory, namePrefix + "Propose");
+        commitLatency = new LatencyMetrics(factory, namePrefix + "Commit");
+
+        contentionBackoffLatency = new LatencyMetrics(factory, namePrefix + "ContentionBackoff");
+
+        missingMostRecentCommit = Metrics.counter(factory.createMetricName(namePrefix + "MissingMostRecentCommit"));
     }
 
     public void release()
     {
         super.release();
-        Metrics.remove(factory.createMetricName("ContentionHistogram"));
-        Metrics.remove(factory.createMetricName("UnfinishedCommit"));
-        Metrics.remove(factory.createMetricName("UnknownResult"));
+        Metrics.remove(factory.createMetricName(namePrefix + "ContentionHistogram"));
+        Metrics.remove(factory.createMetricName(namePrefix + "UnfinishedCommit"));
+        Metrics.remove(factory.createMetricName(namePrefix + "UnknownResult"));
+
+        prepareLatency.release();
+        createProposalLatency.release();
+        proposeLatency.release();
+        commitLatency.release();
+
+        contentionBackoffLatency.release();
+
+        Metrics.remove(factory.createMetricName(namePrefix + "MissingMostRecentCommit"));
     }
 }
diff --git a/src/java/org/apache/cassandra/metrics/CASClientWriteRequestMetrics.java b/src/java/org/apache/cassandra/metrics/CASClientWriteRequestMetrics.java
index 87c0d5354132..5789f27e8407 100644
--- a/src/java/org/apache/cassandra/metrics/CASClientWriteRequestMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CASClientWriteRequestMetrics.java
@@ -36,20 +36,20 @@ public class CASClientWriteRequestMetrics extends CASClientRequestMetrics
 
     public final Counter conditionNotMet;
 
-    public CASClientWriteRequestMetrics(String scope)
+    public CASClientWriteRequestMetrics(String scope, String namePrefix)
     {
-        super(scope);
-        mutationSize = Metrics.histogram(factory.createMetricName("MutationSizeHistogram"), false);
+        super(scope, namePrefix);
+        mutationSize = Metrics.histogram(factory.createMetricName(namePrefix + "MutationSizeHistogram"), false);
         // scope for this metric was changed in 4.0; adding backward compatibility
-        conditionNotMet = Metrics.counter(factory.createMetricName("ConditionNotMet"),
-                                          DefaultNameFactory.createMetricName("ClientRequest", "ConditionNotMet", "CASRead"));
+        conditionNotMet = Metrics.counter(factory.createMetricName(namePrefix + "ConditionNotMet"),
+                                          DefaultNameFactory.createMetricName("ClientRequest", namePrefix + "ConditionNotMet", "CASRead"));
     }
 
     public void release()
     {
         super.release();
-        Metrics.remove(factory.createMetricName("ConditionNotMet"),
-                       DefaultNameFactory.createMetricName("ClientRequest", "ConditionNotMet", "CASRead"));
-        Metrics.remove(factory.createMetricName("MutationSizeHistogram"));
+        Metrics.remove(factory.createMetricName(namePrefix + "ConditionNotMet"),
+                       DefaultNameFactory.createMetricName("ClientRequest", namePrefix + "ConditionNotMet", "CASRead"));
+        Metrics.remove(factory.createMetricName(namePrefix + "MutationSizeHistogram"));
     }
 }
diff --git a/src/java/org/apache/cassandra/metrics/CacheMetrics.java b/src/java/org/apache/cassandra/metrics/CacheMetrics.java
index 34746bcee94b..7402386c6c27 100644
--- a/src/java/org/apache/cassandra/metrics/CacheMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CacheMetrics.java
@@ -17,98 +17,44 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.util.function.DoubleSupplier;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import com.codahale.metrics.*;
 import org.apache.cassandra.cache.CacheSize;
+import org.apache.cassandra.service.CacheService;
 
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+import static org.apache.cassandra.config.CassandraRelevantProperties.USE_MICROMETER;
 
 /**
  * Metrics for {@code ICache}.
  */
-public class CacheMetrics
+public interface CacheMetrics
 {
-    /** Cache capacity in bytes */
-    public final Gauge<Long> capacity;
-    /** Total size of cache, in bytes */
-    public final Gauge<Long> size;
-    /** Total number of cache entries */
-    public final Gauge<Integer> entries;
+    static CacheMetrics create(CacheService.CacheType cacheType, CacheSize cache)
+    {
+        return USE_MICROMETER.getBoolean()
+               ? new MicrometerCacheMetrics(cacheType.micrometerMetricsPrefix(), cache)
+               : new CodahaleCacheMetrics(cacheType.toString(), cache);
+    }
 
-    /** Total number of cache hits */
-    public final Meter hits;
-    /** Total number of cache misses */
-    public final Meter misses;
-    /** Total number of cache requests */
-    public final Meter requests;
+    long requests();
 
-    /** all time cache hit rate */
-    public final Gauge<Double> hitRate;
-    /** 1m hit rate */
-    public final Gauge<Double> oneMinuteHitRate;
-    /** 5m hit rate */
-    public final Gauge<Double> fiveMinuteHitRate;
-    /** 15m hit rate */
-    public final Gauge<Double> fifteenMinuteHitRate;
+    long capacity();
 
-    protected final MetricNameFactory factory;
+    long size();
 
-    /**
-     * Create metrics for given cache.
-     *
-     * @param type Type of Cache to identify metrics.
-     * @param cache Cache to measure metrics
-     */
-    public CacheMetrics(String type, CacheSize cache)
-    {
-        factory = new DefaultNameFactory("Cache", type);
+    long entries();
 
-        capacity = Metrics.register(factory.createMetricName("Capacity"), cache::capacity);
-        size = Metrics.register(factory.createMetricName("Size"), cache::weightedSize);
-        entries = Metrics.register(factory.createMetricName("Entries"), cache::size);
+    long hits();
 
-        hits = Metrics.meter(factory.createMetricName("Hits"));
-        misses = Metrics.meter(factory.createMetricName("Misses"));
-        requests = Metrics.meter(factory.createMetricName("Requests"));
+    long misses();
 
-        hitRate =
-            Metrics.register(factory.createMetricName("HitRate"),
-                             ratioGauge(hits::getCount, requests::getCount));
-        oneMinuteHitRate =
-            Metrics.register(factory.createMetricName("OneMinuteHitRate"),
-                             ratioGauge(hits::getOneMinuteRate, requests::getOneMinuteRate));
-        fiveMinuteHitRate =
-            Metrics.register(factory.createMetricName("FiveMinuteHitRate"),
-                             ratioGauge(hits::getFiveMinuteRate, requests::getFiveMinuteRate));
-        fifteenMinuteHitRate =
-            Metrics.register(factory.createMetricName("FifteenMinuteHitRate"),
-                             ratioGauge(hits::getFifteenMinuteRate, requests::getFifteenMinuteRate));
-    }
+    double hitRate();
 
-    @VisibleForTesting
-    public void reset()
-    {
-        // No actual reset happens. The Meter counter is put to zero but will not reset the moving averages
-        // It rather injects a weird value into them.
-        // This method is being only used by CacheMetricsTest and CachingBench so fixing this issue was acknowledged
-        // but not considered mandatory to be fixed now (CASSANDRA-16228)
-        hits.mark(-hits.getCount());
-        misses.mark(-misses.getCount());
-        requests.mark(-requests.getCount());
-    }
+    double hitOneMinuteRate();
+    double hitFiveMinuteRate();
+    double hitFifteenMinuteRate();
 
-    private static RatioGauge ratioGauge(DoubleSupplier numeratorSupplier, DoubleSupplier denominatorSupplier)
-    {
-        return new RatioGauge()
-        {
-            @Override
-            public Ratio getRatio()
-            {
-                return Ratio.of(numeratorSupplier.getAsDouble(), denominatorSupplier.getAsDouble());
-            }
-        };
-    }
+    double requestsFifteenMinuteRate();
+
+    void recordHits(int count);
+
+    void recordMisses(int count);
 }
diff --git a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
index 598e484f6bdf..ce22c35053ef 100644
--- a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
+++ b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
@@ -334,6 +334,8 @@ public interface JmxHistogramMBean extends MetricMBean
         long[] values();
 
         long[] getRecentValues();
+
+        default void clear() {}
     }
 
     private static class JmxHistogram extends AbstractBean implements JmxHistogramMBean
@@ -435,6 +437,13 @@ public synchronized long[] getRecentValues()
             last = now;
             return delta;
         }
+
+        @Override
+        public void clear()
+        {
+            if (metric instanceof ClearableHistogram)
+                ((ClearableHistogram) metric).clear();
+        }
     }
 
     public interface JmxCounterMBean extends MetricMBean
diff --git a/src/java/org/apache/cassandra/metrics/ChunkCacheMetrics.java b/src/java/org/apache/cassandra/metrics/ChunkCacheMetrics.java
index 8195aafbf33c..bf623379e183 100644
--- a/src/java/org/apache/cassandra/metrics/ChunkCacheMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ChunkCacheMetrics.java
@@ -1,4 +1,5 @@
 /*
+ *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -9,78 +10,67 @@
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
  */
 package org.apache.cassandra.metrics;
 
-import java.util.concurrent.TimeUnit;
 import javax.annotation.Nonnull;
 
-import com.codahale.metrics.Timer;
 import com.github.benmanes.caffeine.cache.RemovalCause;
 import com.github.benmanes.caffeine.cache.stats.CacheStats;
 import com.github.benmanes.caffeine.cache.stats.StatsCounter;
+
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.cache.ChunkCache;
 
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+import static org.apache.cassandra.config.CassandraRelevantProperties.USE_MICROMETER;
 
-/**
- * Metrics for {@code ICache}.
- */
-public class ChunkCacheMetrics extends CacheMetrics implements StatsCounter
+public interface ChunkCacheMetrics extends StatsCounter, CacheMetrics
 {
-    /** Latency of misses */
-    public final Timer missLatency;
-
-    /**
-     * Create metrics for the provided chunk cache.
-     *
-     * @param cache Chunk cache to measure metrics
-     */
-    public ChunkCacheMetrics(ChunkCache cache)
+    static ChunkCacheMetrics create(ChunkCache cache)
     {
-        super("ChunkCache", cache);
-        missLatency = Metrics.timer(factory.createMetricName("MissLatency"));
+        return create(cache, "chunk_cache");
     }
 
-    @Override
-    public void recordHits(int count)
+    static ChunkCacheMetrics create(ChunkCache cache, String prefix)
     {
-        requests.mark(count);
-        hits.mark(count);
+        return USE_MICROMETER.getBoolean()
+               ? new MicrometerChunkCacheMetrics(cache, prefix)
+               : new CodahaleChunkCacheMetrics(cache);
     }
 
     @Override
-    public void recordMisses(int count)
-    {
-        requests.mark(count);
-        misses.mark(count);
-    }
+    void recordHits(int count);
 
     @Override
-    public void recordLoadSuccess(long loadTime)
-    {
-        missLatency.update(loadTime, TimeUnit.NANOSECONDS);
-    }
+    void recordMisses(int count);
 
     @Override
-    public void recordLoadFailure(long loadTime)
-    {
-    }
+    void recordLoadSuccess(long loadTime);
 
     @Override
-    public void recordEviction(int weight, RemovalCause cause)
-    {
-    }
+    void recordLoadFailure(long loadTime);
+
+    @Override
+    void recordEviction(int weight, RemovalCause cause);
+
+    void recordEviction();
+
+    double missLatency();
+
+    long entries();
 
     @Nonnull
     @Override
-    public CacheStats snapshot()
-    {
-        return CacheStats.of(hits.getCount(), misses.getCount(), missLatency.getCount(), 0L, missLatency.getCount(), 0L, 0L);
-    }
+    CacheStats snapshot();
+
+    @VisibleForTesting
+    void reset();
 }
diff --git a/src/java/org/apache/cassandra/metrics/ClientMetrics.java b/src/java/org/apache/cassandra/metrics/ClientMetrics.java
index 5616571b9a2a..8643f49e7a48 100644
--- a/src/java/org/apache/cassandra/metrics/ClientMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ClientMetrics.java
@@ -30,6 +30,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import com.codahale.metrics.Counter;
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
@@ -55,13 +56,19 @@ public final class ClientMetrics
     @SuppressWarnings({ "unused", "FieldCanBeLocal" })
     private Gauge<Integer> pausedConnectionsGauge;
     private Meter connectionPaused;
+
     private Meter requestDiscarded;
     private Meter requestDispatched;
 
-    private Meter timedOutBeforeProcessing;
+    public Meter timedOutBeforeProcessing;
+    public Meter timedOutBeforeAsyncProcessing;
+    public Timer queueTime; // time between Message creation and execution on NTR
+    public Counter totalQueueTime; // total queue time (in nanoseconds) for use in histogram timer
+    public Timer asyncQueueTime; // time between Message creation and execution on an async stage. This includes the time recorded in queueTime metric.
+    public Counter totalAsyncQueueTime; // total async queue time (in nanoseconds) for use in histogram timer
+
     private Meter protocolException;
     private Meter unknownException;
-    private Timer queueTime;
 
     private ClientMetrics()
     {
@@ -88,6 +95,7 @@ public void pauseConnection()
         connectionPaused.mark();
         pausedConnections.incrementAndGet();
     }
+
     public void unpauseConnection() { pausedConnections.decrementAndGet(); }
 
     public void markRequestDiscarded() { requestDiscarded.mark(); }
@@ -104,6 +112,35 @@ public List<ConnectedClient> allConnectedClients()
         return clients;
     }
 
+    public void markTimedOutBeforeAsyncProcessing()
+    {
+        timedOutBeforeAsyncProcessing.mark();
+    }
+
+    /**
+     * Record time between Message creation and execution on NTR.
+     * @param value time elapsed
+     * @param unit time unit
+     */
+    public void recordQueueTime(long value, TimeUnit unit)
+    {
+        queueTime.update(value, unit);
+        totalQueueTime.inc(TimeUnit.NANOSECONDS.convert(value, unit));
+    }
+
+    /**
+     * Record time between Message creation and execution on an async stage, if present.
+     * Note that this includes the queue time previously recorded before execution on the NTR stage,
+     * so for a given request, asyncQueueTime >= queueTime.
+     * @param value time elapsed
+     * @param unit time unit
+     */
+    public void recordAsyncQueueTime(long value, TimeUnit unit)
+    {
+        asyncQueueTime.update(value, unit);
+        totalAsyncQueueTime.inc(TimeUnit.NANOSECONDS.convert(value, unit));
+    }
+
     public void markProtocolException()
     {
         protocolException.mark();
@@ -142,12 +179,18 @@ public long getCount()
         authFailure = registerMeter("AuthFailure");
 
         pausedConnections = new AtomicInteger();
+        connectionPaused = registerMeter("ConnectionPaused");
         pausedConnectionsGauge = registerGauge("PausedConnections", pausedConnections::get);
         connectionPaused = registerMeter("ConnectionPaused");
         requestDiscarded = registerMeter("RequestDiscarded");
         requestDispatched = registerMeter("RequestDispatched");
 
         timedOutBeforeProcessing = registerMeter("TimedOutBeforeProcessing");
+        timedOutBeforeAsyncProcessing = registerMeter("TimedOutBeforeAsyncProcessing");
+        totalQueueTime = registerCounter("TotalQueueTime");
+        asyncQueueTime = registerTimer("AsyncQueueTime");
+        totalAsyncQueueTime = registerCounter("TotalAsyncQueueTime");
+
         protocolException = registerMeter("ProtocolException");
         unknownException = registerMeter("UnknownException");
 
@@ -227,4 +270,9 @@ public void queueTime(long value, TimeUnit unit)
     {
         queueTime.update(value, unit);
     }
+
+    private Counter registerCounter(String name)
+    {
+        return Metrics.counter(factory.createMetricName(name));
+    }
 }
diff --git a/src/java/org/apache/cassandra/metrics/ClientRangeRequestMetrics.java b/src/java/org/apache/cassandra/metrics/ClientRangeRequestMetrics.java
index c974651381b5..a44bc61a1dbd 100644
--- a/src/java/org/apache/cassandra/metrics/ClientRangeRequestMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ClientRangeRequestMetrics.java
@@ -33,15 +33,15 @@ public class ClientRangeRequestMetrics extends ClientRequestMetrics
      */
     public final Histogram roundTrips;
 
-    public ClientRangeRequestMetrics(String scope)
+    public ClientRangeRequestMetrics(String scope, String namePrefix)
     {
-        super(scope);
-        roundTrips = Metrics.histogram(factory.createMetricName("RoundTripsPerReadHistogram"), false);
+        super(scope, namePrefix);
+        roundTrips = Metrics.histogram(factory.createMetricName(namePrefix + "RoundTripsPerReadHistogram"), false);
     }
 
     public void release()
     {
         super.release();
-        Metrics.remove(factory.createMetricName("RoundTripsPerReadHistogram"));
+        Metrics.remove(factory.createMetricName(namePrefix + "RoundTripsPerReadHistogram"));
     }
 }
diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestMetrics.java b/src/java/org/apache/cassandra/metrics/ClientRequestMetrics.java
index 408087051e2a..a56538bf47b5 100644
--- a/src/java/org/apache/cassandra/metrics/ClientRequestMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ClientRequestMetrics.java
@@ -29,7 +29,7 @@
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
 
-public class ClientRequestMetrics extends LatencyMetrics
+public class ClientRequestMetrics
 {
     public final Meter timeouts;
     public final Meter unavailables;
@@ -40,18 +40,35 @@ public class ClientRequestMetrics extends LatencyMetrics
     public final Meter localRequests;
     public final Meter remoteRequests;
 
-    public ClientRequestMetrics(String scope)
-    {
-        super("ClientRequest", scope);
+    /**
+     * this is the metric that measures the actual execution time of a certain request;
+     * for example, the duration of StorageProxy::readRegular method for regular reads
+     */
+    public final LatencyMetrics executionTimeMetrics;
+
+    /**
+     * this is the metric that measures the time a request spent in the system;
+     * for example, the duration between requestTime and the end of StorageProxy::readRegular method for regular reads
+     */
+    public final LatencyMetrics serviceTimeMetrics;
 
-        timeouts = Metrics.meter(factory.createMetricName("Timeouts"));
-        unavailables = Metrics.meter(factory.createMetricName("Unavailables"));
-        failures = Metrics.meter(factory.createMetricName("Failures"));
-        aborts = Metrics.meter(factory.createMetricName("Aborts"));
-        tombstoneAborts = Metrics.meter(factory.createMetricName("TombstoneAborts"));
-        readSizeAborts = Metrics.meter(factory.createMetricName("ReadSizeAborts"));
-        localRequests = Metrics.meter(factory.createMetricName("LocalRequests"));
-        remoteRequests = Metrics.meter(factory.createMetricName("RemoteRequests"));
+    protected final String namePrefix;
+    protected final MetricNameFactory factory;
+
+    public ClientRequestMetrics(String scope, String prefix)
+    {
+        namePrefix = prefix;
+        factory = new DefaultNameFactory("ClientRequest", scope);
+        timeouts = Metrics.meter(factory.createMetricName(namePrefix + "Timeouts"));
+        unavailables = Metrics.meter(factory.createMetricName(namePrefix + "Unavailables"));
+        failures = Metrics.meter(factory.createMetricName(namePrefix + "Failures"));
+        aborts = Metrics.meter(factory.createMetricName(namePrefix + "Aborts"));
+        tombstoneAborts = Metrics.meter(factory.createMetricName(namePrefix + "TombstoneAborts"));
+        readSizeAborts = Metrics.meter(factory.createMetricName(namePrefix + "ReadSizeAborts"));
+        localRequests = Metrics.meter(factory.createMetricName(namePrefix + "LocalRequests"));
+        remoteRequests = Metrics.meter(factory.createMetricName(namePrefix + "RemoteRequests"));
+        executionTimeMetrics = new LatencyMetrics(factory, namePrefix);
+        serviceTimeMetrics = new LatencyMetrics(factory, namePrefix + "ServiceTime");
     }
 
     public void markAbort(Throwable cause)
@@ -71,14 +88,15 @@ else if (cause instanceof ReadSizeAbortException)
 
     public void release()
     {
-        super.release();
-        Metrics.remove(factory.createMetricName("Timeouts"));
-        Metrics.remove(factory.createMetricName("Unavailables"));
-        Metrics.remove(factory.createMetricName("Failures"));
-        Metrics.remove(factory.createMetricName("Aborts"));
-        Metrics.remove(factory.createMetricName("TombstoneAborts"));
-        Metrics.remove(factory.createMetricName("ReadSizeAborts"));
-        Metrics.remove(factory.createMetricName("LocalRequests"));
-        Metrics.remove(factory.createMetricName("RemoteRequests"));
+        Metrics.remove(factory.createMetricName(namePrefix + "Timeouts"));
+        Metrics.remove(factory.createMetricName(namePrefix + "Unavailables"));
+        Metrics.remove(factory.createMetricName(namePrefix + "Failures"));
+        Metrics.remove(factory.createMetricName(namePrefix + "Aborts"));
+        Metrics.remove(factory.createMetricName(namePrefix + "TombstoneAborts"));
+        Metrics.remove(factory.createMetricName(namePrefix + "ReadSizeAborts"));
+        Metrics.remove(factory.createMetricName(namePrefix + "LocalRequests"));
+        Metrics.remove(factory.createMetricName(namePrefix + "RemoteRequests"));
+        executionTimeMetrics.release();
+        serviceTimeMetrics.release();
     }
 }
diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestsMetrics.java b/src/java/org/apache/cassandra/metrics/ClientRequestsMetrics.java
new file mode 100644
index 000000000000..757384b61612
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/ClientRequestsMetrics.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.EnumMap;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+
+public class ClientRequestsMetrics
+{
+    public final ClientRequestMetrics readMetrics;
+    public final ClientRangeRequestMetrics rangeMetrics;
+    public final ClientWriteRequestMetrics writeMetrics;
+    public final CASClientWriteRequestMetrics casWriteMetrics;
+    public final CASClientRequestMetrics casReadMetrics;
+    public final ViewWriteMetrics viewWriteMetrics;
+    private final Map<ConsistencyLevel, ClientRequestMetrics> readMetricsMap;
+    private final Map<ConsistencyLevel, ClientWriteRequestMetrics> writeMetricsMap;
+
+    public ClientRequestsMetrics()
+    {
+        this("");
+    }
+
+    /**
+     * CassandraMetricsRegistry requires unique metrics name, otherwise it returns previous metrics.
+     * CNDB will create coordinator metrics with unique name prefix for each tenant
+     */
+    public ClientRequestsMetrics(String namePrefix)
+    {
+        readMetrics = new ClientRequestMetrics("Read", namePrefix);
+        rangeMetrics = new ClientRangeRequestMetrics("RangeSlice", namePrefix);
+        writeMetrics = new ClientWriteRequestMetrics("Write", namePrefix);
+        casWriteMetrics = new CASClientWriteRequestMetrics("CASWrite", namePrefix);
+        casReadMetrics = new CASClientRequestMetrics("CASRead", namePrefix);
+        viewWriteMetrics = new ViewWriteMetrics("ViewWrite", namePrefix);
+        readMetricsMap = new EnumMap<>(ConsistencyLevel.class);
+        writeMetricsMap = new EnumMap<>(ConsistencyLevel.class);
+        for (ConsistencyLevel level : ConsistencyLevel.values())
+        {
+            readMetricsMap.put(level, new ClientRequestMetrics("Read-" + level.name(), namePrefix));
+            writeMetricsMap.put(level, new ClientWriteRequestMetrics("Write-" + level.name(), namePrefix));
+        }
+    }
+
+    public ClientRequestMetrics readMetricsForLevel(ConsistencyLevel level)
+    {
+        return readMetricsMap.get(level);
+    }
+
+    public ClientWriteRequestMetrics writeMetricsForLevel(ConsistencyLevel level)
+    {
+        return writeMetricsMap.get(level);
+    }
+
+    /**
+     * When we want to reset metrics, say in a test env, it is not enough to create a new {@link ClientRequestsMetrics}
+     * object because the internal histograms would be initialized with the already registered, existing instances.
+     * In order to unregister and make the constructor really create new metrics histograms, we need to call this method
+     * on the old instance first.
+     */
+    @VisibleForTesting
+    public void release()
+    {
+        readMetrics.release();
+        rangeMetrics.release();
+        writeMetrics.release();
+        casWriteMetrics.release();
+        casReadMetrics.release();
+        readMetricsMap.values().forEach(ClientRequestMetrics::release);
+        writeMetricsMap.values().forEach(ClientRequestMetrics::release);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java
deleted file mode 100644
index 26f2913263e6..000000000000
--- a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.metrics;
-
-import java.util.EnumMap;
-import java.util.Map;
-
-import org.apache.cassandra.db.ConsistencyLevel;
-
-public final class ClientRequestsMetricsHolder
-{
-    public static final ClientRequestMetrics readMetrics = new ClientRequestMetrics("Read");
-    public static final ClientWriteRequestMetrics writeMetrics = new ClientWriteRequestMetrics("Write");
-    public static final CASClientWriteRequestMetrics casWriteMetrics = new CASClientWriteRequestMetrics("CASWrite");
-    public static final CASClientRequestMetrics casReadMetrics = new CASClientRequestMetrics("CASRead");
-    public static final ViewWriteMetrics viewWriteMetrics = new ViewWriteMetrics("ViewWrite");
-
-    public static final Map<ConsistencyLevel, ClientRequestMetrics> readMetricsMap = new EnumMap<>(ConsistencyLevel.class);
-    public static final Map<ConsistencyLevel, ClientWriteRequestMetrics> writeMetricsMap = new EnumMap<>(ConsistencyLevel.class);
-
-    static
-    {
-        for (ConsistencyLevel level : ConsistencyLevel.values())
-        {
-             readMetricsMap.put(level, new ClientRequestMetrics("Read-" + level.name()));
-            writeMetricsMap.put(level, new ClientWriteRequestMetrics("Write-" + level.name()));
-        }
-    }
-
-    public static ClientRequestMetrics readMetricsForLevel(ConsistencyLevel level)
-    {
-        return readMetricsMap.get(level);
-    }
-
-    public static ClientWriteRequestMetrics writeMetricsForLevel(ConsistencyLevel level)
-    {
-        return writeMetricsMap.get(level);
-    }
-}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsProvider.java b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsProvider.java
new file mode 100644
index 000000000000..04e1b94d9286
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsProvider.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY;
+
+/**
+ * Provides access to the {@link ClientRequestsMetrics} instance used by this node
+ * and provides per-tenant metrics in CNDB.
+ */
+public interface ClientRequestsMetricsProvider
+{
+    ClientRequestsMetricsProvider instance = CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY.isPresent()
+                                             ? FBUtilities.construct(CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY.getString(),
+                                                                     "Client Request Metrics Provider")
+                                             : new DefaultClientRequestsMetricsProvider();
+
+    ClientRequestsMetrics metrics(String keyspace);
+
+    class DefaultClientRequestsMetricsProvider implements ClientRequestsMetricsProvider
+    {
+        private final ClientRequestsMetrics metrics = new ClientRequestsMetrics("");
+
+        @Override
+        public ClientRequestsMetrics metrics(String keyspace)
+        {
+            return metrics;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/ClientWriteRequestMetrics.java b/src/java/org/apache/cassandra/metrics/ClientWriteRequestMetrics.java
index 50427af0735f..69d3e6edaa0d 100644
--- a/src/java/org/apache/cassandra/metrics/ClientWriteRequestMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ClientWriteRequestMetrics.java
@@ -33,15 +33,15 @@ public class ClientWriteRequestMetrics extends ClientRequestMetrics
      */
     public final Histogram mutationSize;
 
-    public ClientWriteRequestMetrics(String scope)
+    public ClientWriteRequestMetrics(String scope, String namePrefix)
     {
-        super(scope);
-        mutationSize = Metrics.histogram(factory.createMetricName("MutationSizeHistogram"), false);
+        super(scope, namePrefix);
+        mutationSize = Metrics.histogram(factory.createMetricName(namePrefix + "MutationSizeHistogram"), false);
     }
 
     public void release()
     {
         super.release();
-        Metrics.remove(factory.createMetricName("MutationSizeHistogram"));
+        Metrics.remove(factory.createMetricName(namePrefix + "MutationSizeHistogram"));
     }
 }
diff --git a/src/java/org/apache/cassandra/metrics/CodahaleBufferPoolMetrics.java b/src/java/org/apache/cassandra/metrics/CodahaleBufferPoolMetrics.java
new file mode 100644
index 000000000000..efb4b9b2b9e5
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/CodahaleBufferPoolMetrics.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Gauge;
+
+import com.codahale.metrics.Meter;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+class CodahaleBufferPoolMetrics implements BufferPoolMetrics
+{
+    /** Total number of hits */
+    private final Meter hits;
+
+    /** Total number of misses */
+    private final Meter misses;
+
+    /** Total size of buffer pools, in bytes, including overflow allocation */
+    private final Gauge<Long> size;
+
+    /** Total size, in bytes, of active buffered being used from the pool currently + overflow */
+    private final Gauge<Long> usedSize;
+
+    /**
+     * Total size, in bytes, of direct or heap buffers allocated by the pool but not part of the pool
+     * either because they are too large to fit or because the pool has exceeded its maximum limit or because it's
+     * on-heap allocation.
+     */
+    private final Gauge<Long> overflowSize;
+
+    public CodahaleBufferPoolMetrics(String scope, BufferPool bufferPool)
+    {
+        MetricNameFactory factory = new DefaultNameFactory("BufferPool", scope);
+
+        hits = Metrics.meter(factory.createMetricName("Hits"));
+
+        misses = Metrics.meter(factory.createMetricName("Misses"));
+
+        overflowSize = Metrics.register(factory.createMetricName("OverflowSize"), bufferPool::overflowMemoryInBytes);
+
+        usedSize = Metrics.register(factory.createMetricName("UsedSize"), bufferPool::usedSizeInBytes);
+
+        size = Metrics.register(factory.createMetricName("Size"), bufferPool::sizeInBytes);
+    }
+
+    @Override
+    public void markHit()
+    {
+        hits.mark();
+    }
+
+    @Override
+    public long hits()
+    {
+        return hits.getCount();
+    }
+
+    @Override
+    public void markMissed()
+    {
+        misses.mark();
+    }
+
+    @Override
+    public long misses()
+    {
+        return misses.getCount();
+    }
+
+    @Override
+    public long overflowSize()
+    {
+        return overflowSize.getValue();
+    }
+
+    @Override
+    public long usedSize()
+    {
+        return usedSize.getValue();
+    }
+
+    @Override
+    public long size()
+    {
+        return size.getValue();
+    }
+
+    @Override
+    public void register3xAlias()
+    {
+        MetricNameFactory legacyFactory = new DefaultNameFactory("BufferPool");
+        Metrics.registerMBean(misses, legacyFactory.createMetricName("Misses").getMBeanName());
+        Metrics.registerMBean(size, legacyFactory.createMetricName("Size").getMBeanName());
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/CodahaleCacheMetrics.java b/src/java/org/apache/cassandra/metrics/CodahaleCacheMetrics.java
new file mode 100644
index 000000000000..daceeea9f3c0
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/CodahaleCacheMetrics.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+
+import java.util.function.DoubleSupplier;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.cache.CacheSize;
+
+import static java.lang.Double.isInfinite;
+import static java.lang.Double.isNaN;
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+/**
+ * Metrics for {@code ICache}.
+ */
+public class CodahaleCacheMetrics implements CacheMetrics
+{
+    /** Cache capacity in bytes */
+    public final Gauge<Long> capacity;
+    /** Total size of cache, in bytes */
+    public final Gauge<Long> size;
+    /** Total number of cache entries */
+    public final Gauge<Integer> entries;
+
+    /** Total number of cache hits */
+    public final Meter hits;
+    /** Total number of cache misses */
+    public final Meter misses;
+    /** Total number of cache requests */
+    public final Meter requests;
+
+    /** all time cache hit rate */
+    public final Gauge<Double> hitRate;
+    /** 1m hit rate */
+    public final Gauge<Double> oneMinuteHitRate;
+    /** 5m hit rate */
+    public final Gauge<Double> fiveMinuteHitRate;
+    /** 15m hit rate */
+    public final Gauge<Double> fifteenMinuteHitRate;
+
+    public final String cacheType;
+
+    private final MetricNameFactory factory;
+
+    /**
+     * Create metrics for given cache.
+     *
+     * @param type Type of Cache to identify metrics.
+     * @param cache Cache to measure metrics
+     */
+    public CodahaleCacheMetrics(String type, final CacheSize cache)
+    {
+        cacheType = type;
+        factory = new DefaultNameFactory("Cache", type);
+
+        capacity = registerGauge("Capacity", cache::capacity);
+        size = registerGauge("Size", cache::weightedSize);
+        entries = registerGauge("Entries", cache::size);
+
+        requests = registerMeter("Requests");
+
+        hits = registerMeter("Hits");
+        misses = registerMeter("Misses");
+
+        hitRate = registerGauge("HitRate", ratioGauge(hits::getCount, requests::getCount));
+        oneMinuteHitRate = registerGauge("OneMinuteHitRate", ratioGauge(hits::getOneMinuteRate, requests::getOneMinuteRate));
+        fiveMinuteHitRate = registerGauge("FiveMinuteHitRate", ratioGauge(hits::getFiveMinuteRate, requests::getFiveMinuteRate));
+        fifteenMinuteHitRate = registerGauge("FifteenMinuteHitRate", ratioGauge(hits::getFifteenMinuteRate, requests::getFifteenMinuteRate));
+    }
+
+    @Override
+    public long requests()
+    {
+        return requests.getCount();
+    }
+
+    @Override
+    public long capacity()
+    {
+        return capacity.getValue();
+    }
+
+    @Override
+    public long size()
+    {
+        return size.getValue();
+    }
+
+    @Override
+    public long entries()
+    {
+        return entries.getValue();
+    }
+
+    @Override
+    public long hits()
+    {
+        return hits.getCount();
+    }
+
+    @Override
+    public long misses()
+    {
+        return misses.getCount();
+    }
+
+    @Override
+    public double hitRate()
+    {
+        return hitRate.getValue();
+    }
+
+    @Override
+    public double hitOneMinuteRate()
+    {
+        return oneMinuteHitRate.getValue();
+    }
+
+    @Override
+    public double hitFiveMinuteRate()
+    {
+        return fiveMinuteHitRate.getValue();
+    }
+
+    @Override
+    public double hitFifteenMinuteRate()
+    {
+        return fifteenMinuteHitRate.getValue();
+    }
+
+    @Override
+    public double requestsFifteenMinuteRate()
+    {
+        return requests.getFifteenMinuteRate();
+    }
+
+    @Override
+    public void recordHits(int count)
+    {
+        requests.mark();
+        hits.mark(count);
+    }
+
+    @Override
+    public void recordMisses(int count)
+    {
+        requests.mark();
+        misses.mark(count);
+    }
+
+    /**
+     * Computes the ratio between the specified numerator and denominator
+     *
+     * @param numerator the numerator
+     * @param denominator the denominator
+     * @return the ratio between the numerator and the denominator
+     */
+    private static double ratio(double numerator, double denominator)
+    {
+        if (isNaN(denominator) || isInfinite(denominator) || denominator == 0)
+            return Double.NaN;
+
+        return numerator / denominator;
+    }
+
+    protected final <T> Gauge<T> registerGauge(String name, Gauge<T> gauge)
+    {
+        return Metrics.register(factory.createMetricName(name), gauge);
+    }
+
+    protected final Meter registerMeter(String name)
+    {
+        return Metrics.meter(factory.createMetricName(name));
+    }
+
+    protected final Timer registerTimer(String name)
+    {
+        return Metrics.timer(factory.createMetricName(name));
+    }
+
+    @VisibleForTesting
+    public void reset()
+    {
+        // No actual reset happens. The Meter counter is put to zero but will not reset the moving averages
+        // It rather injects a weird value into them.
+        // This method is being only used by CacheMetricsTest and CachingBench so fixing this issue was acknowledged
+        // but not considered mandatory to be fixed now (CASSANDRA-16228)
+        hits.mark(-hits.getCount());
+        misses.mark(-misses.getCount());
+        requests.mark(-requests.getCount());
+    }
+
+    /**
+     * Returns a {@code Gauge} that will compute the ratio between the number supplied by the suppliers.
+     *
+     * <p>{@code RatioGauge} create {@code Ratio} objects for each call which is a bit inefficcient. That method
+     * computes the ratio using a simple method call.</p>
+     *
+     * @param numeratorSupplier the supplier for the numerator
+     * @param denominatorSupplier the supplier for the denominator
+     * @return a {@code Gauge} that will compute the ratio between the number supplied by the suppliers
+     */
+    public static Gauge<Double> ratioGauge(DoubleSupplier numeratorSupplier, DoubleSupplier denominatorSupplier)
+    {
+        return () -> ratio(numeratorSupplier.getAsDouble(), denominatorSupplier.getAsDouble());
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/metrics/CodahaleChunkCacheMetrics.java b/src/java/org/apache/cassandra/metrics/CodahaleChunkCacheMetrics.java
new file mode 100644
index 000000000000..52393414d13d
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/CodahaleChunkCacheMetrics.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.codahale.metrics.Timer;
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import com.github.benmanes.caffeine.cache.stats.CacheStats;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Codahale implementation for the chunk cache metrics.
+ */
+public class CodahaleChunkCacheMetrics implements ChunkCacheMetrics
+{
+    /** Metrics in common with ICache implementations */
+    private final CodahaleCacheMetrics metrics;
+
+    /** Latency of misses */
+    public final Timer missLatency;
+
+    /**
+     * Create metrics for the provided chunk cache.
+     *
+     * @param cache Chunk cache to measure metrics
+     */
+    CodahaleChunkCacheMetrics(ChunkCache cache)
+    {
+        metrics = new CodahaleCacheMetrics("ChunkCache", cache);
+        missLatency = metrics.registerTimer("MissLatency");
+    }
+
+    @Override
+    public void recordHits(int count)
+    {
+        metrics.requests.mark(count);
+        metrics.hits.mark(count);
+    }
+
+    @Override
+    public void recordMisses(int count)
+    {
+        metrics.requests.mark(count);
+        metrics.misses.mark(count);
+    }
+
+    @Override
+    public void recordLoadSuccess(long loadTime)
+    {
+        missLatency.update(loadTime, TimeUnit.NANOSECONDS);
+    }
+
+    @Override
+    public void recordLoadFailure(long loadTime)
+    {
+    }
+
+    @Override
+    public void recordEviction(int weight, RemovalCause cause)
+    {
+    }
+
+    @Override
+    public void recordEviction()
+    {
+    }
+
+    @Override
+    public double hitRate()
+    {
+        return metrics.hitRate.getValue();
+    }
+
+    @Override
+    public double hitOneMinuteRate()
+    {
+        return metrics.hitOneMinuteRate();
+    }
+
+    @Override
+    public double hitFiveMinuteRate()
+    {
+        return metrics.hitFiveMinuteRate();
+    }
+
+    @Override
+    public double hitFifteenMinuteRate()
+    {
+        return metrics.hitFifteenMinuteRate();
+    }
+
+    @Override
+    public double requestsFifteenMinuteRate()
+    {
+        return metrics.requestsFifteenMinuteRate();
+    }
+
+    @Override
+    public long requests()
+    {
+        return metrics.requests.getCount();
+    }
+
+    @Override
+    public long misses()
+    {
+        return metrics.misses.getCount();
+    }
+
+    @Override
+    public long hits()
+    {
+        return metrics.hits.getCount();
+    }
+
+    @Override
+    public double missLatency()
+    {
+        return missLatency.getOneMinuteRate();
+    }
+
+    @Override
+    public long capacity()
+    {
+        return metrics.capacity.getValue();
+    }
+
+    @Override
+    public long size()
+    {
+        return metrics.size.getValue();
+    }
+
+    @Override
+    public long entries()
+    {
+        return metrics.entries();
+    }
+
+    @Nonnull
+    @Override
+    public CacheStats snapshot()
+    {
+        return CacheStats.of(metrics.hits.getCount(), metrics.misses.getCount(), missLatency.getCount(), 0L, missLatency.getCount(), 0L, 0L);
+    }
+
+    @Override
+    @VisibleForTesting
+    public void reset()
+    {
+        metrics.reset();
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Chunk cache metrics: " + System.lineSeparator() +
+               "Miss latency in seconds: " + missLatency() + System.lineSeparator() +
+               "Misses count: " + misses() + System.lineSeparator() +
+               "Hits count: " + hits() + System.lineSeparator() +
+               "Cache requests count: " + requests() + System.lineSeparator() +
+               "Moving hit rate: " + hitRate() + System.lineSeparator() +
+               "Num entries: " + entries() + System.lineSeparator() +
+               "Size in memory: " + FBUtilities.prettyPrintMemory(size()) + System.lineSeparator() +
+               "Capacity: " + FBUtilities.prettyPrintMemory(capacity());
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/CodehaleNativeMemoryMetrics.java b/src/java/org/apache/cassandra/metrics/CodehaleNativeMemoryMetrics.java
new file mode 100644
index 000000000000..c7ffe9b05e99
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/CodehaleNativeMemoryMetrics.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class CodehaleNativeMemoryMetrics implements NativeMemoryMetrics
+{
+    private static final Logger logger = LoggerFactory.getLogger(CodehaleNativeMemoryMetrics.class);
+
+    private final MetricNameFactory factory;
+
+    /** Total size of memory allocated directly by calling Native.malloc via {@link MemoryUtil}, bypassing the JVM.
+     * This is in addition to nio direct memory, for example off-heap memtables will use this type of memory. */
+    private final Gauge<Long> rawNativeMemory;
+
+    /**
+     * Total size of memory used by bloom filters, part of {@link this#rawNativeMemory}.
+     */
+    private final Gauge<Long> bloomFilterMemory;
+
+    /** The memory allocated for direct byte buffers, aligned or otherwise, without counting any padding due to alignment.
+     *  If {@code -Dio.netty.directMemory} is not set to {@code 0}, the direct memory used by Netty is <em>not</em> included in this value. */
+    private final Gauge<Long> usedNioDirectMemory;
+
+    /** The total memory allocated for direct byte buffers, aligned or otherwise, including any padding due to alignment.
+     * If -Dsun.nio.PageAlignDirectMemory=true is not set then this will be identical to usedNioDirectMemory.
+     * If {@code -Dio.netty.directMemory} is not set to {@code 0}, the direct memory used by Netty is <em>not</em> included in this value. */
+    private final Gauge<Long> totalNioDirectMemory;
+
+    /** The memory used by direct byte buffers allocated via the Netty library. These buffers are used for network communications.
+     * A limit can be set with "-Dio.netty.maxDirectMemory". When this property is zero (the default in jvm.options), then
+     * Netty will use the JVM NIO direct memory. Therefore, this value will be included in {@link #usedNioDirectMemory}
+     * and {@link #totalNioDirectMemory} only when the property is set to zero, otherwise this value is extra. */
+    private final Gauge<Long> networkDirectMemory;
+
+    /** The total number of direct byte buffers allocated, aligned or otherwise. */
+    private final Gauge<Long> nioDirectBufferCount;
+
+    /** The total memory allocated, including direct byte buffers, network direct memory, and raw malloc memory */
+    private final Gauge<Long> totalMemory;
+
+    public CodehaleNativeMemoryMetrics()
+    {
+        factory = new DefaultNameFactory("NativeMemory");
+
+        if (directBufferPool == null)
+            logger.error("Direct memory buffer pool MBean not present, native memory metrics will be missing for nio buffers");
+
+        rawNativeMemory = Metrics.register(factory.createMetricName("RawNativeMemory"), this::rawNativeMemory);
+        bloomFilterMemory = Metrics.register(factory.createMetricName("BloomFilterMemory"), this::bloomFilterMemory);
+        usedNioDirectMemory = Metrics.register(factory.createMetricName("UsedNioDirectMemory"), this::usedNioDirectMemory);
+        totalNioDirectMemory = Metrics.register(factory.createMetricName("TotalNioDirectMemory"), this::totalNioDirectMemory);
+        networkDirectMemory = Metrics.register(factory.createMetricName("NetworkDirectMemory"), this::networkDirectMemory);
+        nioDirectBufferCount = Metrics.register(factory.createMetricName("NioDirectBufferCount"), this::nioDirectBufferCount);
+        totalMemory = Metrics.register(factory.createMetricName("TotalMemory"), this::totalMemory);
+    }
+
+    @Override
+    public long usedNioDirectMemoryValue()
+    {
+        return usedNioDirectMemory.getValue();
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
index 0fe1ec7418ce..6860da7d20c6 100644
--- a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
@@ -18,8 +18,12 @@
 package org.apache.cassandra.metrics;
 
 import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
+import com.codahale.metrics.CachedGauge;
 import com.codahale.metrics.Counter;
+import com.codahale.metrics.DerivativeGauge;
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Meter;
 
@@ -27,15 +31,21 @@
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
+import org.apache.cassandra.db.compaction.CompactionAggregateStatistics;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionStrategyStatistics;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
 /**
- * Metrics for compaction.
+ * Metrics for the compaction executor. Note that several different operations execute on the compaction
+ * executor, for example index or view building. These operations are abstracted by {@link AbstractTableOperation}
+ * but previously we would refer to these operations as "compactions", so this incorrect name may still be
+ * found in the metrics that are exported to the users.
  */
 public class CompactionMetrics
 {
@@ -46,14 +56,41 @@ public class CompactionMetrics
     /** Estimated number of compactions remaining to perform, group by keyspace and then table name */
     public final Gauge<Map<String, Map<String, Integer>>> pendingTasksByTableName;
 
-    /** Number of completed compactions since server [re]start */
+    /** Write amplification of compactions (bytes compacted / bytes flushed), group by keyspace and then table name */
+    public final Gauge<Map<String, Map<String, Double>>> writeAmplificationByTableName;
+
+    /** Number of completed operations since server [re]start */
     public final Gauge<Long> completedTasks;
-    /** Total number of compactions since server [re]start */
+    /** Total number of operations since server [re]start */
     public final Meter totalCompactionsCompleted;
-    /** Total number of bytes compacted since server [re]start */
+    /** Total number of failed compactions since server [re]start */
+    public final Counter totalCompactionsFailed;
+    /** Total number of bytes processed by operations since server [re]start */
     public final Counter bytesCompacted;
     /** Time spent redistributing index summaries */
     public final Timer indexSummaryRedistributionTime;
+    /** Recent/current throughput of compactions take */
+    public final Meter bytesCompactedThroughput;
+
+    /**
+     * The compaction strategy information for each table. Cached, because its computation might be fairly expensive.
+     */
+    public final CachedGauge<List<CompactionStrategyStatistics>> aggregateCompactions;
+
+    /*
+     * The compaction metrics below are derivatives of the complex compaction statistics metric aggregateCompactions.
+     */
+
+    /** Number of currently running compactions for all tables */
+    public final DerivativeGauge<List<CompactionStrategyStatistics>, Integer> runningCompactions;
+    /** Mean read throughput of currently running compactions in bytes per second */
+    public final DerivativeGauge<List<CompactionStrategyStatistics>, Double> meanCompactionReadThroughput;
+    /** Mean write throughput of currently running compactions in bytes per second */
+    public final DerivativeGauge<List<CompactionStrategyStatistics>, Double> meanCompactionWriteThroughput;
+    /** Total bytes to compact from currently running compactions */
+    public final DerivativeGauge<List<CompactionStrategyStatistics>, Long> runningCompactionsTotalBytes;
+    /** Remaining bytes to compact from currently running compactions */
+    public final DerivativeGauge<List<CompactionStrategyStatistics>, Long> runningCompactionsRemainingBytes;
 
     /** Total number of compactions that have had sstables drop out of them */
     public final Counter compactionsReduced;
@@ -64,74 +101,87 @@ public class CompactionMetrics
     /** Total number of compactions which have outright failed due to lack of disk space */
     public final Counter compactionsAborted;
 
+    /** Total number of deleted expired SSTables */
+    public final Meter removedExpiredSSTables;
+    /** Total number compactions that consisted of only expired SSTables */
+    public final Meter deleteOnlyCompactions;
+
+    public final Gauge<Map<String, Map<String, Map<String, String>>>> overlapsMap;
+
     public CompactionMetrics(final ExecutorPlus... collectors)
     {
-        pendingTasks = Metrics.register(factory.createMetricName("PendingTasks"), new Gauge<Integer>()
-        {
-            public Integer getValue()
+        pendingTasks = Metrics.register(factory.createMetricName("PendingTasks"), () -> {
+            int n = 0;
+            // add estimate number of compactions need to be done
+            for (String keyspaceName : Schema.instance.getKeyspaces())
             {
-                int n = 0;
-                // add estimate number of compactions need to be done
-                for (String keyspaceName : Schema.instance.getKeyspaces())
-                {
-                    for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
-                        n += cfs.getCompactionStrategyManager().getEstimatedRemainingTasks();
-                }
-                // add number of currently running compactions
-                return n + CompactionManager.instance.active.getCompactions().size();
+                for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
+                    n += cfs.getCompactionStrategy().getEstimatedRemainingTasks();
             }
+            // add number of currently running compactions
+            return n + CompactionManager.instance.active.getTableOperations().size();
         });
 
-        pendingTasksByTableName = Metrics.register(factory.createMetricName("PendingTasksByTableName"),
-            new Gauge<Map<String, Map<String, Integer>>>()
-        {
-            @Override
-            public Map<String, Map<String, Integer>> getValue() 
+        pendingTasksByTableName = Metrics.register(factory.createMetricName("PendingTasksByTableName"), () -> {
+            Map<String, Map<String, Integer>> resultMap = new HashMap<>();
+            // estimation of compactions need to be done
+            for (String keyspaceName : Schema.instance.getKeyspaces())
             {
-                Map<String, Map<String, Integer>> resultMap = new HashMap<>();
-                // estimation of compactions need to be done
-                for (String keyspaceName : Schema.instance.getKeyspaces())
+                for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
                 {
-                    for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
+                    int taskNumber = cfs.getCompactionStrategy().getEstimatedRemainingTasks();
+                    if (taskNumber > 0)
                     {
-                        int taskNumber = cfs.getCompactionStrategyManager().getEstimatedRemainingTasks();
-                        if (taskNumber > 0)
+                        if (!resultMap.containsKey(keyspaceName))
                         {
-                            if (!resultMap.containsKey(keyspaceName))
-                            {
-                                resultMap.put(keyspaceName, new HashMap<>());
-                            }
-                            resultMap.get(keyspaceName).put(cfs.getTableName(), taskNumber);
+                            resultMap.put(keyspaceName, new HashMap<>());
                         }
+                        resultMap.get(keyspaceName).put(cfs.getTableName(), taskNumber);
                     }
                 }
+            }
 
-                // currently running compactions
-                for (CompactionInfo.Holder compaction : CompactionManager.instance.active.getCompactions())
+            // currently running compactions
+            // TODO DB-2701 - this includes all operations (previous behaviour), if we wanted only real
+            // compactions we could remove this block of code and call getTotalCompactions() from the strategy managers
+            for (TableOperation op : CompactionManager.instance.active.getTableOperations())
+            {
+                TableMetadata metaData = op.getProgress().metadata();
+                if (metaData == null)
                 {
-                    TableMetadata metaData = compaction.getCompactionInfo().getTableMetadata();
-                    if (metaData == null)
-                    {
-                        continue;
-                    }
-                    if (!resultMap.containsKey(metaData.keyspace))
-                    {
-                        resultMap.put(metaData.keyspace, new HashMap<>());
-                    }
+                    continue;
+                }
+                if (!resultMap.containsKey(metaData.keyspace))
+                {
+                    resultMap.put(metaData.keyspace, new HashMap<>());
+                }
 
-                    Map<String, Integer> tableNameToCountMap = resultMap.get(metaData.keyspace);
-                    if (tableNameToCountMap.containsKey(metaData.name))
-                    {
-                        tableNameToCountMap.put(metaData.name,
-                                                tableNameToCountMap.get(metaData.name) + 1);
-                    }
-                    else
-                    {
-                        tableNameToCountMap.put(metaData.name, 1);
-                    }
+                Map<String, Integer> tableNameToCountMap = resultMap.get(metaData.keyspace);
+                if (tableNameToCountMap.containsKey(metaData.name))
+                {
+                    tableNameToCountMap.put(metaData.name,
+                                            tableNameToCountMap.get(metaData.name) + 1);
                 }
-                return resultMap;
+                else
+                {
+                    tableNameToCountMap.put(metaData.name, 1);
+                }
+            }
+            return resultMap;
+        });
+
+        writeAmplificationByTableName = Metrics.register(factory.createMetricName("WriteAmplificationByTableName"), () -> {
+            Map<String, Map<String, Double>> resultMap = new HashMap<>();
+
+            for (String keyspaceName : Schema.instance.getKeyspaces())
+            {
+                Map<String, Double> ksMap = new HashMap<>();
+                resultMap.put(keyspaceName, ksMap);
+                for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
+                    ksMap.put(cfs.getTableName(), cfs.getWA());
             }
+
+            return resultMap;
         });
 
         completedTasks = Metrics.register(factory.createMetricName("CompletedTasks"), new Gauge<Long>()
@@ -145,12 +195,136 @@ public Long getValue()
             }
         });
         totalCompactionsCompleted = Metrics.meter(factory.createMetricName("TotalCompactionsCompleted"));
+        totalCompactionsFailed = Metrics.counter(factory.createMetricName("FailedCompactions"));
         bytesCompacted = Metrics.counter(factory.createMetricName("BytesCompacted"));
+        bytesCompactedThroughput = Metrics.meter(factory.createMetricName("BytesCompactedThroughput"));
 
         // compaction failure metrics
         compactionsReduced = Metrics.counter(factory.createMetricName("CompactionsReduced"));
         sstablesDropppedFromCompactions = Metrics.counter(factory.createMetricName("SSTablesDroppedFromCompaction"));
         compactionsAborted = Metrics.counter(factory.createMetricName("CompactionsAborted"));
         indexSummaryRedistributionTime = Metrics.timer(factory.createMetricName("IndexSummaryRedistributionTime"));
+
+        removedExpiredSSTables = Metrics.meter(factory.createMetricName("ExpiredSSTablesDropped"));
+        deleteOnlyCompactions = Metrics.meter(factory.createMetricName("DeleteOnlyCompactions"));
+
+        aggregateCompactions = Metrics.register(factory.createMetricName("AggregateCompactions"),
+                                                // TODO 50 ms is 100x less than the default report interval of our distributed test harness (Fallout) at
+                                                //  the moment of writing this. This implies that even a bigger timeout might be OK.
+                                                new CachedGauge<List<CompactionStrategyStatistics>>(50, TimeUnit.MILLISECONDS)
+                                                {
+                                                    @Override
+                                                    protected List<CompactionStrategyStatistics> loadValue()
+                                                    {
+                                                        List<CompactionStrategyStatistics> ret = new ArrayList<>();
+                                                        for (String keyspaceName : Schema.instance.getKeyspaces())
+                                                        {
+                                                            // Scan all the compactions strategies of all tables and find those that have compactions in progress.
+                                                            for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
+                                                                // For those return the statistics.
+                                                                ret.addAll(cfs.getCompactionStrategy().getStatistics());
+                                                        }
+
+                                                        return ret;
+                                                    }
+                                                });
+
+        overlapsMap = Metrics.register(factory.createMetricName("MaxOverlapsMap"),
+                                    new CachedGauge<Map<String, Map<String, Map<String, String>>>>(50, TimeUnit.MILLISECONDS)
+                                    {
+                                        public Map<String, Map<String, Map<String, String>>> loadValue()
+                                        {
+                                            Map<String, Map<String, Map<String, String>>> ret = new HashMap<>();
+                                            for (String keyspaceName : Schema.instance.getKeyspaces())
+                                            {
+                                                Map<String, Map<String, String>> ksMap = new HashMap<>();
+                                                ret.put(keyspaceName, ksMap);
+                                                for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores())
+                                                {
+                                                    Map<String, String> overlaps = cfs.getCompactionStrategy().getMaxOverlapsMap();
+                                                    ksMap.put(cfs.getTableName(), overlaps);
+                                                }
+                                            }
+                                            return ret;
+                                        }
+                                    });
+
+        runningCompactions = Metrics.register(factory.createMetricName("RunningCompactions"),
+                                              new DerivativeGauge<List<CompactionStrategyStatistics>, Integer>(aggregateCompactions)
+        {
+            @Override
+            protected Integer transform(List<CompactionStrategyStatistics> value)
+            {
+                return deriveSafeAggregateStatisticsStream(value)
+                       .mapToInt(CompactionAggregateStatistics::numCompactionsInProgress)
+                       .sum();
+            }
+        });
+        meanCompactionReadThroughput = Metrics.register(factory.createMetricName("MeanCompactionReadThroughput"),
+                                                        new DerivativeGauge<List<CompactionStrategyStatistics>, Double>(aggregateCompactions)
+        {
+            @Override
+            protected Double transform(List<CompactionStrategyStatistics> value)
+            {
+                return deriveSafeAggregateStatisticsStream(value)
+                       // Don't take into account aggregates for which there are no running compactions
+                       .filter(s -> s.numCompactionsInProgress() > 0)
+                       .mapToDouble(CompactionAggregateStatistics::readThroughput)
+                       .average()
+                       .orElse(0.0);
+            }
+        });
+        meanCompactionWriteThroughput = Metrics.register(factory.createMetricName("MeanCompactionWriteThroughput"),
+                                                         new DerivativeGauge<List<CompactionStrategyStatistics>, Double>(aggregateCompactions)
+        {
+            @Override
+            protected Double transform(List<CompactionStrategyStatistics> value)
+            {
+                return deriveSafeAggregateStatisticsStream(value)
+                       // Don't take into account aggregates for which there are no running compactions
+                       .filter(s -> s.numCompactionsInProgress() > 0)
+                       .mapToDouble(CompactionAggregateStatistics::writeThroughput)
+                       .average()
+                       .orElse(0.0);
+            }
+        });
+        runningCompactionsTotalBytes = Metrics.register(factory.createMetricName("RunningCompactionsTotalBytes"),
+                                                        new DerivativeGauge<List<CompactionStrategyStatistics>, Long>(aggregateCompactions)
+        {
+            @Override
+            protected Long transform(List<CompactionStrategyStatistics> value)
+            {
+                return deriveSafeAggregateStatisticsStream(value)
+                       .mapToLong(CompactionAggregateStatistics::tot)
+                       .sum();
+            }
+        });
+        runningCompactionsRemainingBytes = Metrics.register(factory.createMetricName("RunningCompactionsRemainingBytes"),
+                                                            new DerivativeGauge<List<CompactionStrategyStatistics>, Long>(aggregateCompactions)
+        {
+            @Override
+            protected Long transform(List<CompactionStrategyStatistics> value)
+            {
+                return deriveSafeAggregateStatisticsStream(value)
+                       .mapToLong(s -> s.tot() - s.read())
+                       .sum();
+            }
+        });
+    }
+
+    /**
+     * Needed because deriving from a CachedGauge might hit a NullPointerException until we move to a version of
+     * dropwizard's metrics-core where https://github.com/dropwizard/metrics/pull/711 /
+     * https://github.com/dropwizard/metrics/pull/1566 are fixed (currently targeting metrics-core 4.1.7).
+     *
+     * @param aggregateCompactions The cached compaction strategy statistics to derive from.
+     *
+     * @return A stream (potentially empty) of the aggregate statistics corresponding to the given strategy statistics.
+     */
+    private static Stream<CompactionAggregateStatistics> deriveSafeAggregateStatisticsStream(List<CompactionStrategyStatistics> aggregateCompactions)
+    {
+        if (aggregateCompactions == null)
+            return Stream.empty();
+        return aggregateCompactions.stream().flatMap(s -> s.aggregates().stream());
     }
 }
diff --git a/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java b/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
index cfca6460421b..b0c2f85cbcc4 100644
--- a/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
+++ b/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
@@ -23,6 +23,7 @@
 import java.io.PrintWriter;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLongArray;
@@ -43,6 +44,7 @@
 import static java.lang.Math.max;
 import static java.lang.Math.min;
 import static org.apache.cassandra.config.CassandraRelevantProperties.DECAYING_ESTIMATED_HISTOGRAM_RESERVOIR_STRIPE_COUNT;
+import static org.apache.cassandra.utils.EstimatedHistogram.USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES;
 
 /**
  * A decaying histogram reservoir where values collected during each minute will be twice as significant as the values
@@ -100,16 +102,33 @@ public class DecayingEstimatedHistogramReservoir implements SnapshottingReservoi
     private static final int[] DISTRIBUTION_PRIMES = new int[] { 17, 19, 23, 29 };
 
     // The offsets used with a default sized bucket array without a separate bucket for zero values.
-    public static final long[] DEFAULT_WITHOUT_ZERO_BUCKET_OFFSETS = EstimatedHistogram.newOffsets(DEFAULT_BUCKET_COUNT, false);
+    private static final long[] DEFAULT_WITHOUT_ZERO_BUCKET_OFFSETS = EstimatedHistogram.newCassandraOffsets(DEFAULT_BUCKET_COUNT, false);
 
     // The offsets used with a default sized bucket array with a separate bucket for zero values.
-    public static final long[] DEFAULT_WITH_ZERO_BUCKET_OFFSETS = EstimatedHistogram.newOffsets(DEFAULT_BUCKET_COUNT, true);
+    private static final long[] DEFAULT_WITH_ZERO_BUCKET_OFFSETS = EstimatedHistogram.newCassandraOffsets(DEFAULT_BUCKET_COUNT, true);
 
     private static final int TABLE_BITS = 4;
     private static final int TABLE_MASK = -1 >>> (32 - TABLE_BITS);
     private static final float[] LOG2_TABLE = computeTable(TABLE_BITS);
     private static final float log2_12_recp = (float) (1d / slowLog2(1.2d));
 
+    // DSE COMPATIBILITY CHANGES START
+    // The DSE-compatible offsets used with a default sized bucket array without a separate bucket for zero values.
+    private static final long[] DEFAULT_DSE_WITHOUT_ZERO_BUCKET_OFFSETS = newDseOffsets(DEFAULT_BUCKET_COUNT, false);
+
+    // The DSE-compatibleoffsets used with a default sized bucket array with a separate bucket for zero values.
+    private static final long[] DEFAULT_DSE_WITH_ZERO_BUCKET_OFFSETS = newDseOffsets(DEFAULT_BUCKET_COUNT, true);
+
+    /** Values for calculating buckets and indexes */
+    final static int subBucketCount = 8;  // number of sub-buckets in each bucket
+    final static int subBucketHalfCount = subBucketCount / 2;
+    final static int unitMagnitude = 0; // power of two of the unit in bucket zero (2^0 = 1)
+    final static int subBucketCountMagnitude = 3; // power of two of the number of sub buckets
+    final static int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1; // power of two of half the number of sub-buckets
+    final static long subBucketMask = (long)(subBucketCount - 1) << unitMagnitude;
+    final static int leadingZeroCountBase = 64 - unitMagnitude - subBucketHalfCountMagnitude - 1;
+    // DSE COMPATIBILITY CHANGES END
+
     private static float[] computeTable(int bits)
     {
         float[] table = new float[1 << bits];
@@ -219,14 +238,10 @@ public DecayingEstimatedHistogramReservoir(boolean considerZeroes,
 
         if (bucketCount == DEFAULT_BUCKET_COUNT)
         {
-            if (considerZeroes == true)
-            {
-                bucketOffsets = DEFAULT_WITH_ZERO_BUCKET_OFFSETS;
-            }
+            if (USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES)
+                bucketOffsets = considerZeroes ? DEFAULT_DSE_WITH_ZERO_BUCKET_OFFSETS : DEFAULT_DSE_WITHOUT_ZERO_BUCKET_OFFSETS;
             else
-            {
-                bucketOffsets = DEFAULT_WITHOUT_ZERO_BUCKET_OFFSETS;
-            }
+                bucketOffsets = considerZeroes ? DEFAULT_WITH_ZERO_BUCKET_OFFSETS : DEFAULT_WITHOUT_ZERO_BUCKET_OFFSETS;
         }
         else
         {
@@ -280,6 +295,9 @@ public int stripedIndex(int offsetIndex, int stripe)
     @VisibleForTesting
     public static int findIndex(long[] bucketOffsets, long value)
     {
+        if (USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES)
+            return findIndexDse(bucketOffsets, value);
+
         // values below zero are nonsense, but we have never failed when presented them
         value = max(value, 0);
 
@@ -300,6 +318,66 @@ public static int findIndex(long[] bucketOffsets, long value)
         return value <= bucketOffsets[firstCandidate] ? firstCandidate : firstCandidate + 1;
     }
 
+    /**
+     * this is almost a copy-paste from DSE DecayingEstimatedHistogram::BucketProperties::getIndex
+     * Almost, because:
+     * 1. C* and DSE differently implement the "considerZeroes" flag.
+     * The zeroesCorrection variable is used to adjust the index in the C* case.
+     * <p/>
+     * 2. C* and DSE differently implement the histogram overflow.
+     * In DSE, there is a separate flag isOverflowed which is set when a value doesn't fit in buckets; the getIndex
+     * function is supposed to always return index for an actual bucket.
+     * In C* there is a special bucket for overflowed values, and the findIndex function is supposed to return
+     * the index of this additional bucket if the value doesn't fit in the regular buckets.
+     * This is the origin of the min() function in the return statement.
+     *
+     * @param bucketOffsets the offsets of the histogram buckets (upper inclusive bounds)
+     * @param value the value with which we want to update the histogram
+     * @return index of the bucket that keeps track of the value OR the index of the last bucket which is used for
+     * overflowed values
+     */
+    private static int findIndexDse(long[] bucketOffsets, long value)
+    {
+        if (value < 0) {
+            throw new ArrayIndexOutOfBoundsException("Histogram recorded value cannot be negative.");
+        }
+
+        // Calculates the number of powers of two by which the value is greater than the biggest value that fits in
+        // bucket 0. This is the bucket index since each successive bucket can hold a value 2x greater.
+        // The mask maps small values to bucket 0.
+        final int bucketIndex = leadingZeroCountBase - Long.numberOfLeadingZeros(value | subBucketMask);
+
+        // For bucketIndex 0, this is just value, so it may be anywhere in 0 to subBucketCount.
+        // For other bucketIndex, this will always end up in the top half of subBucketCount: assume that for some bucket
+        // k > 0, this calculation will yield a value in the bottom half of 0 to subBucketCount. Then, because of how
+        // buckets overlap, it would have also been in the top half of bucket k-1, and therefore would have
+        // returned k-1 in getBucketIndex(). Since we would then shift it one fewer bits here, it would be twice as big,
+        // and therefore in the top half of subBucketCount.
+        final int subBucketIndex = (int)(value >>> (bucketIndex + unitMagnitude));
+
+        //assert(subBucketIndex < subBucketCount);
+        //assert(bucketIndex == 0 || (subBucketIndex >= subBucketHalfCount));
+        // Calculate the index for the first entry that will be used in the bucket (halfway through subBucketCount).
+        // For bucketIndex 0, all subBucketCount entries may be used, but bucketBaseIndex is still set in the middle.
+        final int bucketBaseIndex = (bucketIndex + 1) << subBucketHalfCountMagnitude;
+
+        // Calculate the offset in the bucket. This subtraction will result in a positive value in all buckets except
+        // the 0th bucket (since a value in that bucket may be less than half the bucket's 0 to subBucketCount range).
+        // However, this works out since we give bucket 0 twice as much space.
+        final int offsetInBucket = subBucketIndex - subBucketHalfCount;
+
+        // The following is the equivalent of ((subBucketIndex  - subBucketHalfCount) + bucketBaseIndex,
+        final int dseBucket = bucketBaseIndex + offsetInBucket;
+
+        // DSE bucket for zero values always exists, and during snapshot creation it is added to the second bucket
+        // if the histogram should not "considerZeroes".
+        // Cassandra does that differently. We either have or have not a separate bucket for zeroes.
+        // Thus, we should subtract 1 from the DSE index if we don't consider zeroes AND the value > 0.
+        final int zeroesCorrection = bucketOffsets[0] > 0 && value > 0 ? 1 : 0;
+
+        return min(bucketOffsets.length, bucketBaseIndex + offsetInBucket - zeroesCorrection);
+    }
+
     /**
      * Returns the logical number of buckets where recorded values are stored. The actual number of physical buckets
      * is size() * stripeCount()
@@ -657,7 +735,7 @@ public void dump(OutputStream output)
      * The decaying buckets will be used for quantile calculations and mean values, but the non decaying buckets will be
      * exposed for calls to {@link Snapshot#getValues()}.
      */
-    static class EstimatedHistogramReservoirSnapshot extends AbstractSnapshot
+    public static class EstimatedHistogramReservoirSnapshot extends AbstractSnapshot
     {
         private final long[] values;
         private long count;
@@ -699,6 +777,11 @@ public long[] getValues()
             return values;
         }
 
+        public long[] getOffsets()
+        {
+            return bucketOffsets;
+        }
+        
         @Override
         public int size()
         {
@@ -855,4 +938,49 @@ public String toString()
             return "[" + min + ',' + max + ']';
         }
     }
+
+    /**
+     * this is almost a copy-paste from DSE DecayingEstimatedHistogram::makeOffsets, except that it's been adjusted
+     * to the C*-specific ability of specifying the number of buckets.
+     * Please note, that DSE bucket offsets are inclusive lower bounds and C* bucket offsets are inclusive upper bounds.
+     * For simplicity, we use the same bucket offsets in both cases, but this means there might be a slight
+     * difference for any samples that are exactly on the bucket boundary. I think we can safely ignore that.
+     *
+     * @param size the number of regular buckets to create; the special bucket for overflow values is not included
+     *             in this count
+     * @param considerZeroes whether to include a separate bucket for zero values
+     * @return the offsets for the buckets; in that context offsets mean the upper inclusive bounds of each bucket
+     *         the name "offset" stays for historic reasons.
+     *
+     */
+    public static long[] newDseOffsets(int size, boolean considerZeroes)
+    {
+        ArrayList<Long> ret = new ArrayList<>();
+        if (considerZeroes)
+            ret.add(0L);
+
+        for (int i = 1; i <= subBucketCount && ret.size() < size; i++)
+        {
+            ret.add((long) i);
+        }
+
+        long last = subBucketCount;
+        long unit = 1 << (unitMagnitude + 1);
+
+        while (ret.size() < size)
+        {
+            for (int i = 0; i < subBucketHalfCount; i++)
+            {
+                assert last + unit > last : "Overflow in DSE histogram bucket calculation; too big size requested: " + size;
+                last += unit;
+                ret.add(last);
+                if (ret.size() >= size)
+                    break;
+            }
+            unit *= 2;
+        }
+
+        assert ret.size() == size : "DSE histogram bucket count mismatch: " + ret.size() + " != " + size;
+        return ret.stream().mapToLong(i->i).toArray();
+    }
 }
diff --git a/src/java/org/apache/cassandra/metrics/DroppedMessageMetrics.java b/src/java/org/apache/cassandra/metrics/DroppedMessageMetrics.java
index 91a680e50d99..6e288734e0ba 100644
--- a/src/java/org/apache/cassandra/metrics/DroppedMessageMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/DroppedMessageMetrics.java
@@ -17,10 +17,10 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.Map;
 
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
 
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Timer;
@@ -42,7 +42,7 @@ public class DroppedMessageMetrics
 
     static
     {
-        EnumMap<Verb, String> aliases = new EnumMap<>(Verb.class);
+        Map<Verb, String> aliases = new HashMap<>();
         aliases.put(Verb.BATCH_REMOVE_REQ, "BATCH_REMOVE");
         aliases.put(Verb.BATCH_STORE_REQ, "BATCH_STORE");
         aliases.put(Verb.COUNTER_MUTATION_REQ, "COUNTER_MUTATION");
@@ -53,7 +53,7 @@ public class DroppedMessageMetrics
         aliases.put(Verb.READ_REPAIR_REQ, "READ_REPAIR");
         aliases.put(Verb.REQUEST_RSP, "REQUEST_RESPONSE");
 
-        REQUEST_VERB_ALIAS = Maps.immutableEnumMap(aliases);
+        REQUEST_VERB_ALIAS = ImmutableMap.copyOf(aliases);
     }
 
     /** Number of dropped messages */
diff --git a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
index bcff38957715..f12ce71b20bd 100644
--- a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
@@ -20,6 +20,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.github.benmanes.caffeine.cache.Caffeine;
@@ -50,6 +51,10 @@ public final class HintsServiceMetrics
                                                                                                .executor(ImmediateExecutor.INSTANCE)
                                                                                                .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.toString().replace(':', '.')), false));
 
+
+    public static final Counter hintsOnDisk = Metrics.counter(factory.createMetricName("HintsOnDisk"));
+    public static final Counter corruptedHintsOnDisk = Metrics.counter(factory.createMetricName("CorruptedHintsOnDisk"));
+
     public static void updateDelayMetrics(InetAddressAndPort endpoint, long delay)
     {
         if (delay <= 0)
diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
index ae15bbf95a04..1595b1e8d33f 100644
--- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
@@ -86,6 +86,22 @@ public class KeyspaceMetrics
     public final Histogram sstablesPerRangeReadHistogram;
     /** Tombstones scanned in queries on this Keyspace */
     public final Histogram tombstoneScannedHistogram;
+    /** Time spent flushing memtables */
+    public final Counter flushTime;
+    public final Counter storageAttachedIndexBuildTime;
+
+    /** Time spent writing SAI */
+    public final Counter storageAttachedIndexWritingTimeForIndexBuild;
+    public final Counter storageAttachedIndexWritingTimeForCompaction;
+    public final Counter storageAttachedIndexWritingTimeForFlush;
+    public final Counter storageAttachedIndexWritingTimeForOther;
+
+    /** Time spent writing  memtables during compaction */
+    public final Counter compactionTime;
+
+    /** Shadowed keys scan metrics **/
+    public final Histogram shadowedKeysScannedHistogram;
+    public final Histogram shadowedKeysLoopsHistogram;
     /** Live cells scanned in queries on this Keyspace */
     public final Histogram liveScannedHistogram;
     /** Column update time delta on this Keyspace */
@@ -136,6 +152,14 @@ public class KeyspaceMetrics
     public final Counter outOfRangeTokenWrites;
     /** Lifetime count of paxos requests for keys outside the node's owned token ranges for this keyspace **/
     public final Counter outOfRangeTokenPaxosRequests;
+    /** Coordinator read metrics */
+    public final Timer coordinatorReadLatency;
+    /** Coordinator range metrics */
+    public final Timer coordinatorScanLatency;
+    /** Coordinator write metrics */
+    public final Timer coordinatorWriteLatency;
+    /** Time spent waiting for free memtable space, either on- or off-heap */
+    public final Histogram waitingOnFreeMemtableSpace;
 
     /*
      * Metrics for inconsistencies detected between repaired data sets across replicas. These
@@ -238,6 +262,15 @@ public KeyspaceMetrics(final Keyspace ks)
         sstablesPerReadHistogram = createKeyspaceHistogram("SSTablesPerReadHistogram", true);
         sstablesPerRangeReadHistogram = createKeyspaceHistogram("SSTablesPerRangeReadHistogram", true);
         tombstoneScannedHistogram = createKeyspaceHistogram("TombstoneScannedHistogram", false);
+        flushTime = createKeyspaceCounter("FlushTime", v -> v.flushTime.getCount());
+        storageAttachedIndexBuildTime = createKeyspaceCounter("StorageAttachedIndexBuildTime", v -> v.storageAttachedIndexBuildTime.getCount());
+        storageAttachedIndexWritingTimeForIndexBuild = createKeyspaceCounter("StorageAttachedIndexWritingTimeForIndexBuild", v -> v.storageAttachedIndexWritingTimeForIndexBuild.getCount());
+        storageAttachedIndexWritingTimeForCompaction = createKeyspaceCounter("StorageAttachedIndexWritingTimeForCompaction", v -> v.storageAttachedIndexWritingTimeForCompaction.getCount());
+        storageAttachedIndexWritingTimeForFlush = createKeyspaceCounter("StorageAttachedIndexWritingTimeForFlush", v -> v.storageAttachedIndexWritingTimeForFlush.getCount());
+        storageAttachedIndexWritingTimeForOther = createKeyspaceCounter("StorageAttachedIndexWritingTimeForOther", v -> v.storageAttachedIndexWritingTimeForOther.getCount());
+        compactionTime = createKeyspaceCounter("CompactionTime", v -> v.compactionTime.getCount());
+        shadowedKeysScannedHistogram = createKeyspaceHistogram("ShadowedKeysScannedHistogram", false);
+        shadowedKeysLoopsHistogram = createKeyspaceHistogram("ShadowedKeysLoopsHistogram", false);
         liveScannedHistogram = createKeyspaceHistogram("LiveScannedHistogram", false);
         colUpdateTimeDeltaHistogram = createKeyspaceHistogram("ColUpdateTimeDeltaHistogram", false);
         viewLockAcquireTime = createKeyspaceTimer("ViewLockAcquireTime");
@@ -263,6 +296,11 @@ public KeyspaceMetrics(final Keyspace ks)
         partitionsValidated = createKeyspaceHistogram("PartitionsValidated", false);
         bytesValidated = createKeyspaceHistogram("BytesValidated", false);
 
+        coordinatorReadLatency = createKeyspaceTimer("CoordinatorReadLatency");
+        coordinatorScanLatency = createKeyspaceTimer("CoordinatorScanLatency");
+        coordinatorWriteLatency = createKeyspaceTimer("CoordinatorWriteLatency");
+        waitingOnFreeMemtableSpace = createKeyspaceHistogram("WaitingOnFreeMemtableSpace", false);
+
         confirmedRepairedInconsistencies = createKeyspaceMeter("RepairedDataInconsistenciesConfirmed");
         unconfirmedRepairedInconsistencies = createKeyspaceMeter("RepairedDataInconsistenciesUnconfirmed");
 
diff --git a/src/java/org/apache/cassandra/metrics/LinearFit.java b/src/java/org/apache/cassandra/metrics/LinearFit.java
new file mode 100644
index 000000000000..869cac4f581a
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/LinearFit.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.apache.cassandra.utils.Pair;
+
+public class LinearFit
+{
+    /**
+     * Computes the intercept and slope for the best linear fit to the given values.
+     */
+    public static Pair<Double, Double> interceptSlopeFor(PairedSlidingWindowReservoir.IntIntPair[] values)
+    {
+        double xSum = 0;
+        double ySum = 0;
+        for (var pair : values)
+        {
+            xSum += pair.x;
+            ySum += pair.y;
+        }
+        double xMean = xSum / values.length;
+        double yMean = ySum / values.length;
+
+        double covariance = 0;
+        double variance = 0;
+        for (var pair : values)
+        {
+            double dX = pair.x - xMean;
+            double dY = pair.y - yMean;
+            covariance += dX * dY;
+            variance += dX * dX;
+        }
+
+        // if all points have the same X value, return the Y:X ratio.  this does the right thing
+        // for `estimateCost`
+        if (variance == 0)
+            return Pair.create(0.0, yMean / xMean);
+
+        double slope = covariance / variance;
+        double intercept = yMean - slope * xMean;
+        return Pair.create(intercept, slope);
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/MessagingMetrics.java b/src/java/org/apache/cassandra/metrics/MessagingMetrics.java
index bef6d087373f..682ee421cbaf 100644
--- a/src/java/org/apache/cassandra/metrics/MessagingMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/MessagingMetrics.java
@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -58,13 +57,14 @@ public static class DCLatencyRecorder implements LatencyConsumer
         public final Timer dcLatency;
         public final Timer allLatency;
 
-        DCLatencyRecorder(Timer dcLatency, Timer allLatency)
+        public DCLatencyRecorder(Timer dcLatency, Timer allLatency)
         {
             this.dcLatency = dcLatency;
             this.allLatency = allLatency;
         }
 
-        public void accept(long timeTaken, TimeUnit units)
+        @Override
+        public void accept(Verb verb, long timeTaken, TimeUnit units)
         {
             if (timeTaken > 0)
             {
@@ -94,41 +94,46 @@ private static final class DroppedForVerb
     }
 
     private final Timer allLatency;
-    public final ConcurrentHashMap<String, DCLatencyRecorder> dcLatency;
-    public final EnumMap<Verb, Timer> internalLatency;
+    public final Map<String, DCLatencyRecorder> dcLatency = new ConcurrentHashMap<>();
+    public final Map<Verb, Timer> internalLatency = new ConcurrentHashMap<>();
 
     // total dropped message counts for server lifetime
-    private final Map<Verb, DroppedForVerb> droppedMessages = new EnumMap<>(Verb.class);
+    private final Map<Verb, DroppedForVerb> droppedMessages = new ConcurrentHashMap<>();
 
     public MessagingMetrics()
     {
         allLatency = Metrics.timer(factory.createMetricName("CrossNodeLatency"));
-        dcLatency = new ConcurrentHashMap<>();
-        internalLatency = new EnumMap<>(Verb.class);
-        for (Verb verb : Verb.VERBS)
+        for (Verb verb : Verb.getValues())
+        {
             internalLatency.put(verb, Metrics.timer(factory.createMetricName(verb + "-WaitLatency")));
-        for (Verb verb : Verb.values())
             droppedMessages.put(verb, new DroppedForVerb(verb));
+        }
     }
 
-    public DCLatencyRecorder internodeLatencyRecorder(InetAddressAndPort from)
+    public LatencyConsumer internodeLatencyRecorder(InetAddressAndPort from)
     {
         String dcName = DatabaseDescriptor.getEndpointSnitch().getDatacenter(from);
-        DCLatencyRecorder dcUpdater = dcLatency.get(dcName);
-        if (dcUpdater == null)
-            dcUpdater = dcLatency.computeIfAbsent(dcName, k -> new DCLatencyRecorder(Metrics.timer(factory.createMetricName(dcName + "-Latency")), allLatency));
+        DCLatencyRecorder dcUpdater = dcLatency.computeIfAbsent(dcName,
+                                                                k -> new DCLatencyRecorder(Metrics.timer(factory.createMetricName(dcName + "-Latency")),
+                                                                                           allLatency));
         return dcUpdater;
     }
 
-    public void recordInternalLatency(Verb verb, long timeTaken, TimeUnit units)
+    public void recordInternalLatency(Verb verb, InetAddressAndPort from, long timeTaken, TimeUnit units)
     {
         if (timeTaken > 0)
-            internalLatency.get(verb).update(timeTaken, units);
+        {
+            // We need to potentially compute absent entries if this is a custom verb
+            // that is not present in the Verb.getValues() list because it was
+            // instantiated after the Messaging metrics was created.
+            Timer latency = internalLatency.computeIfAbsent(verb, v -> Metrics.timer(factory.createMetricName(v + "-WaitLatency")));
+            latency.update(timeTaken, units);
+        }
     }
 
     public void recordSelfDroppedMessage(Verb verb)
     {
-        recordDroppedMessage(droppedMessages.get(verb), false);
+        recordDroppedMessage(droppedMessages.computeIfAbsent(verb, v -> new DroppedForVerb(v)), false);
     }
 
     public void recordSelfDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit)
@@ -141,6 +146,18 @@ public void recordInternodeDroppedMessage(Verb verb, long timeElapsed, TimeUnit
         recordDroppedMessage(verb, timeElapsed, timeUnit, true);
     }
 
+    @Override
+    public void recordMessageStageProcessingTime(Verb verb, InetAddressAndPort from, long timeElapsed, TimeUnit unit)
+    {
+        // NOOP
+    }
+
+    @Override
+    public void recordTotalMessageProcessingTime(Verb verb, InetAddressAndPort from, long timeElapsed, TimeUnit unit)
+    {
+        // NOOP
+    }
+
     public void recordDroppedMessage(Message<?> message, long timeElapsed, TimeUnit timeUnit)
     {
         recordDroppedMessage(message.verb(), timeElapsed, timeUnit, message.isCrossNode());
@@ -148,7 +165,7 @@ public void recordDroppedMessage(Message<?> message, long timeElapsed, TimeUnit
 
     public void recordDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit, boolean isCrossNode)
     {
-        recordDroppedMessage(droppedMessages.get(verb), timeElapsed, timeUnit, isCrossNode);
+        recordDroppedMessage(droppedMessages.computeIfAbsent(verb, v -> new DroppedForVerb(v)), timeElapsed, timeUnit, isCrossNode);
     }
 
     private static void recordDroppedMessage(DroppedForVerb droppedMessages, long timeTaken, TimeUnit units, boolean isCrossNode)
@@ -223,5 +240,4 @@ public void resetDroppedMessages()
     {
         droppedMessages.replaceAll((u, v) -> new DroppedForVerb(new DroppedMessageMetrics(u)));
     }
-
 }
diff --git a/src/java/org/apache/cassandra/metrics/MicrometerBufferPoolMetrics.java b/src/java/org/apache/cassandra/metrics/MicrometerBufferPoolMetrics.java
new file mode 100644
index 000000000000..e5368eb05640
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/MicrometerBufferPoolMetrics.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Meter;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tags;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+public class MicrometerBufferPoolMetrics extends MicrometerMetrics implements BufferPoolMetrics
+{
+    private static final String METRICS_PREFIX = "buffer_pool";
+
+    public static final String TOTAL_SIZE_BYTES = METRICS_PREFIX + "_total_size_bytes";
+    public static final String USED_SIZE_BYTES = METRICS_PREFIX + "_used_size_bytes";
+    public static final String OVERFLOW_SIZE_BYTES = METRICS_PREFIX + "_overflow_size_bytes";
+    public static final String OVERFLOW_ALLOCATIONS = METRICS_PREFIX + "_overflow_allocations";
+    public static final String POOL_ALLOCATIONS = METRICS_PREFIX + "_pool_allocations";
+    public static final String NAME_TAG = "pool_name";
+
+    private final String scope;
+    private final BufferPool bufferPool;
+
+    /** Total number of hits */
+    private final Meter hits;
+
+    /** Total number of misses */
+    private final Meter misses;
+
+    public MicrometerBufferPoolMetrics(String scope, BufferPool bufferPool)
+    {
+        super();
+
+        this.scope = scope;
+        this.bufferPool = bufferPool;
+        this.hits = new Meter();
+        this.misses = new Meter();
+    }
+
+    @Override
+    public synchronized void register(MeterRegistry newRegistry, Tags newTags)
+    {
+        super.register(newRegistry, newTags.and(NAME_TAG, scope));
+
+        gauge(TOTAL_SIZE_BYTES, bufferPool, BufferPool::sizeInBytes);
+        gauge(USED_SIZE_BYTES, bufferPool, BufferPool::usedSizeInBytes);
+        gauge(OVERFLOW_SIZE_BYTES, bufferPool, BufferPool::overflowMemoryInBytes);
+        gauge(OVERFLOW_ALLOCATIONS, misses, Meter::getMeanRate);
+        gauge(POOL_ALLOCATIONS, hits, Meter::getMeanRate);
+    }
+
+    @Override
+    public void markHit()
+    {
+        hits.mark();
+    }
+
+    @Override
+    public long hits()
+    {
+        return hits.getCount();
+    }
+
+    public void markMissed()
+    {
+        misses.mark();
+    }
+
+    @Override
+    public long misses()
+    {
+        return misses.getCount();
+    }
+
+    @Override
+    public long overflowSize()
+    {
+        return bufferPool.overflowMemoryInBytes();
+    }
+
+    @Override
+    public long usedSize()
+    {
+        return bufferPool.usedSizeInBytes();
+    }
+
+    @Override
+    public long size()
+    {
+        return bufferPool.sizeInBytes();
+    }
+
+    @Override
+    public void register3xAlias()
+    {
+        // Not implemented
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/MicrometerCacheMetrics.java b/src/java/org/apache/cassandra/metrics/MicrometerCacheMetrics.java
new file mode 100644
index 000000000000..663702147822
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/MicrometerCacheMetrics.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tags;
+import org.apache.cassandra.cache.CacheSize;
+import org.apache.cassandra.utils.ExpMovingAverage;
+import org.apache.cassandra.utils.MovingAverage;
+
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+public class MicrometerCacheMetrics extends MicrometerMetrics implements CacheMetrics
+{
+    @VisibleForTesting
+    final static long hitRateUpdateIntervalNanos = TimeUnit.MILLISECONDS.toNanos(100);
+
+    private final String metricsPrefix;
+    private final CacheSize cache;
+    private final MovingAverage hitRate;
+    private final AtomicLong hitRateUpdateTime;
+
+    private volatile Counter misses;
+    private volatile Counter hits;
+    private volatile Counter requests;
+    private volatile double totalRequestsLastUpdate;
+
+    public MicrometerCacheMetrics(String metricsPrefix, CacheSize cache)
+    {
+        this.metricsPrefix = metricsPrefix;
+        this.cache = cache;
+        this.hitRate = ExpMovingAverage.decayBy1000();
+        this.hitRateUpdateTime = new AtomicLong(nanoTime());
+
+        this.misses = counter(metricsPrefix + "_misses");
+        this.hits = counter(metricsPrefix + "_hits");
+        this.requests = counter(metricsPrefix + "_requests");
+    }
+
+    @Override
+    public synchronized void register(MeterRegistry newRegistry, Tags newTags)
+    {
+        super.register(newRegistry, newTags);
+
+        gauge(metricsPrefix + "_capacity", cache, CacheSize::capacity);
+        gauge(metricsPrefix + "_size", cache, CacheSize::weightedSize);
+        gauge(metricsPrefix + "_num_entries", cache, CacheSize::size);
+        gauge(metricsPrefix + "_hit_rate", hitRate, MovingAverage::get);
+
+        this.misses = counter(metricsPrefix + "_misses");
+        this.hits = counter(metricsPrefix + "_hits");
+        this.requests = counter(metricsPrefix + "_requests");
+    }
+
+    @Override
+    public long requests()
+    {
+        return (long) (misses.count() + hits.count());
+    }
+
+    @Override
+    public long capacity()
+    {
+        return cache.capacity();
+    }
+
+    @Override
+    public long size()
+    {
+        return cache.weightedSize();
+    }
+
+    @Override
+    public long entries()
+    {
+        return cache.size();
+    }
+
+    @Override
+    public long hits()
+    {
+        return (long) hits.count();
+    }
+
+    @Override
+    public long misses()
+    {
+        return (long) misses.count();
+    }
+
+    @Override
+    public double hitRate()
+    {
+        return hitRate.get();
+    }
+
+    @Override
+    public double hitOneMinuteRate()
+    {
+        return Double.NaN;
+    }
+
+    @Override
+    public double hitFiveMinuteRate()
+    {
+        return Double.NaN;
+    }
+
+    @Override
+    public double hitFifteenMinuteRate()
+    {
+        return Double.NaN;
+    }
+
+    @Override
+    public double requestsFifteenMinuteRate()
+    {
+        return Double.NaN;
+    }
+
+    @Override
+    public void recordHits(int count)
+    {
+        hits.increment(count);
+        updateHitRate();
+    }
+
+    @Override
+    public void recordMisses(int count)
+    {
+        misses.increment(count);
+        updateHitRate();
+    }
+
+    private void updateHitRate()
+    {
+        long lastUpdate = hitRateUpdateTime.get();
+        long now = nanoTime();
+        if (now - lastUpdate > hitRateUpdateIntervalNanos)
+        {
+            if (hitRateUpdateTime.compareAndSet(lastUpdate, now))
+            {
+                double hitCount = hits.count();
+                double numRequests = hitCount + misses.count();
+                double delta = numRequests - totalRequestsLastUpdate;
+                requests.increment(delta);
+                totalRequestsLastUpdate = numRequests;
+                if (numRequests > 0)
+                    hitRate.update(hitCount / numRequests);
+                else
+                    hitRate.update(0);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/metrics/MicrometerChunkCacheMetrics.java b/src/java/org/apache/cassandra/metrics/MicrometerChunkCacheMetrics.java
new file mode 100644
index 000000000000..840a9d19382f
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/MicrometerChunkCacheMetrics.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import com.github.benmanes.caffeine.cache.stats.CacheStats;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.Timer;
+import org.apache.cassandra.cache.CacheSize;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Micrometer implementation for the chunk cache metrics.
+ */
+public class MicrometerChunkCacheMetrics extends MicrometerMetrics implements ChunkCacheMetrics
+{
+    private final CacheSize cache;
+    private final String metricsPrefix;
+
+    private volatile MicrometerCacheMetrics metrics;
+    private volatile Timer missLatency;
+    private volatile Counter evictions;
+    private final ConcurrentHashMap<RemovalCause, Counter> evitictionByRemovalCause = new ConcurrentHashMap<>();
+
+    public MicrometerChunkCacheMetrics(CacheSize cache, String metricsPrefix)
+    {
+        this.cache = cache;
+        this.metricsPrefix = metricsPrefix;
+
+        registerMetrics(registryWithTags().left, registryWithTags().right);
+    }
+
+    private void registerMetrics(MeterRegistry registry, Tags tags)
+    {
+        this.metrics = new MicrometerCacheMetrics(metricsPrefix, cache);
+        this.metrics.register(registry, tags);
+
+        this.missLatency = timer(metricsPrefix + "_miss_latency_seconds");
+        this.evictions = counter(metricsPrefix + "_evictions");
+
+        for (RemovalCause cause : RemovalCause.values())
+        {
+            evitictionByRemovalCause.put(cause, counter(metricsPrefix + "_evictions_" + cause.toString().toLowerCase()));
+        }
+    }
+
+    @Override
+    public synchronized void register(MeterRegistry newRegistry, Tags newTags)
+    {
+        super.register(newRegistry, newTags);
+        registerMetrics(newRegistry, newTags);
+    }
+
+    @Override
+    public void recordMisses(int count)
+    {
+        metrics.recordMisses(count);
+    }
+
+    @Override
+    public void recordLoadSuccess(long val)
+    {
+        missLatency.record(val, TimeUnit.NANOSECONDS);
+    }
+
+    @Override
+    public void recordLoadFailure(long val)
+    {
+    }
+
+    @Override
+    public void recordEviction(int weight, RemovalCause removalCause) {
+        if (removalCause.wasEvicted())
+        {
+            evictions.increment(1);
+        }
+        Counter counter = evitictionByRemovalCause.get(removalCause);
+        if (counter != null) {
+            counter.increment(1);
+        }
+    }
+
+    @Override
+    public void recordEviction()
+    {
+        evictions.increment();
+    }
+
+    @Override
+    public void recordHits(int count)
+    {
+        metrics.recordHits(count);
+    }
+
+    @Override
+    public double hitRate()
+    {
+        return metrics.hitRate();
+    }
+
+    @Override
+    public double hitOneMinuteRate()
+    {
+        return metrics.hitOneMinuteRate();
+    }
+
+    @Override
+    public double hitFiveMinuteRate()
+    {
+        return metrics.hitFiveMinuteRate();
+    }
+
+    @Override
+    public double hitFifteenMinuteRate()
+    {
+        return metrics.hitFifteenMinuteRate();
+    }
+
+    @Override
+    public double requestsFifteenMinuteRate()
+    {
+        return metrics.requestsFifteenMinuteRate();
+    }
+
+    @Override
+    public long requests()
+    {
+        return metrics.requests();
+    }
+
+    @Override
+    public long misses()
+    {
+        return metrics.misses();
+    }
+
+    @Override
+    public long hits()
+    {
+        return metrics.hits();
+    }
+
+    @Override
+    public double missLatency()
+    {
+        return missLatency.mean(TimeUnit.NANOSECONDS);
+    }
+
+    @Override
+    public long capacity()
+    {
+        return metrics.capacity();
+    }
+
+    @Override
+    public long size()
+    {
+        return metrics.size();
+    }
+
+    public long entries()
+    {
+        return metrics.entries();
+    }
+
+    @Override
+    @VisibleForTesting
+    public void reset()
+    {
+        // This method is only used for unit tests, and unit tests only use the codahale implementation
+        throw new UnsupportedOperationException("This was not expected to be called and should be implemented if required");
+    }
+
+    @Nonnull
+    @Override
+    public CacheStats snapshot()
+    {
+        return CacheStats.of(metrics.hits(), metrics.misses(), missLatency.count(),
+                0L, (long) missLatency.totalTime(TimeUnit.NANOSECONDS), (long) evictions.count(), 0L);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Chunk cache metrics: " + System.lineSeparator() +
+               "Miss latency in seconds: " + missLatency() + System.lineSeparator() +
+               "Misses count: " + misses() + System.lineSeparator() +
+               "Hits count: " + hits() + System.lineSeparator() +
+               "Cache requests count: " + requests() + System.lineSeparator() +
+               "Moving hit rate: " + hitRate() + System.lineSeparator() +
+               "Num entries: " + entries() + System.lineSeparator() +
+               "Size in memory: " + FBUtilities.prettyPrintMemory(size()) + System.lineSeparator() +
+               "Capacity: " + FBUtilities.prettyPrintMemory(capacity());
+    }
+
+    public Map<RemovalCause, Double> getEvictionCountByRemovalCause()
+    {
+        Map<RemovalCause, Double> result = new HashMap<>();
+        for (Map.Entry<RemovalCause, Counter> entry : evitictionByRemovalCause.entrySet())
+        {
+            result.put(entry.getKey(), entry.getValue().count());
+        }
+        return result;
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/MicrometerMetrics.java b/src/java/org/apache/cassandra/metrics/MicrometerMetrics.java
new file mode 100644
index 000000000000..c8116df2d676
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/MicrometerMetrics.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.function.ToDoubleFunction;
+
+import com.google.common.base.Preconditions;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.Timer;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import org.apache.cassandra.utils.Pair;
+
+public abstract class MicrometerMetrics
+{
+    private volatile Pair<MeterRegistry, Tags> registryWithTags;
+
+    protected MicrometerMetrics()
+    {
+        this.registryWithTags = Pair.create(new SimpleMeterRegistry(), Tags.empty());
+    }
+
+    public Counter counter(String name)
+    {
+        Pair<MeterRegistry, Tags> current = registryWithTags;
+        return current.left.counter(name, current.right);
+    }
+
+    public Counter counter(String name, Tags tags)
+    {
+        Pair<MeterRegistry, Tags> current = registryWithTags;
+        return current.left.counter(name, current.right.and(tags));
+    }
+
+    public Timer timer(String name)
+    {
+        return timer(name, false);
+    }
+
+    public Timer timer(String name, boolean publishHistogram)
+    {
+        return timer(name, publishHistogram, Tags.empty());
+    }
+
+    public Timer timer(String name, boolean publishHistogram, Tags tags)
+    {
+        Pair<MeterRegistry, Tags> current = registryWithTags;
+        Timer.Builder builder = Timer.builder(name).tags(current.right.and(tags));
+        if (publishHistogram)
+            builder = builder.publishPercentileHistogram();
+
+        return builder.register(current.left);
+    }
+
+    public <T> T gauge(String name, T obj, ToDoubleFunction<T> fcn)
+    {
+        Pair<MeterRegistry, Tags> current = registryWithTags;
+        return current.left.gauge(name, current.right, obj, fcn);
+    }
+
+    public synchronized void register(MeterRegistry newRegistry, Tags newTags)
+    {
+        Preconditions.checkArgument(!this.registryWithTags.left.equals(newRegistry), "Cannot set the same registry twice!");
+        this.registryWithTags = Pair.create(newRegistry, newTags);
+    }
+
+    public Pair<MeterRegistry, Tags> registryWithTags()
+    {
+        return this.registryWithTags;
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/MicrometerNativeMemoryMetrics.java b/src/java/org/apache/cassandra/metrics/MicrometerNativeMemoryMetrics.java
new file mode 100644
index 000000000000..5e6e660fd07b
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/MicrometerNativeMemoryMetrics.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tags;
+
+public class MicrometerNativeMemoryMetrics extends MicrometerMetrics implements NativeMemoryMetrics
+{
+    private static final Logger logger = LoggerFactory.getLogger(MicrometerNativeMemoryMetrics.class);
+
+    private static final String METRICS_PREFIX = "jvm_native_memory";
+
+    public static final String RAW_NATIVE_MEMORY = METRICS_PREFIX + "_raw_native_memory";
+    public static final String BLOOM_FILTER_MEMORY = METRICS_PREFIX + "_bloom_filter_memory";
+    public static final String COMPRESSION_METADATA_MEMORY = METRICS_PREFIX + "_compression_metadata_memory";
+    public static final String NETWORK_DIRECT_MEMORY = METRICS_PREFIX + "_network_direct_memory";
+    public static final String USED_NIO_DIRECT_MEMORY = METRICS_PREFIX + "_used_nio_direct_memory";
+    public static final String TOTAL_NIO_MEMORY = METRICS_PREFIX + "_total_nio_direct_memory";
+    public static final String NIO_DIRECT_BUFFER_COUNT = METRICS_PREFIX + "_nio_direct_buffer_count";
+    public static final String TOTAL_MEMORY = METRICS_PREFIX + "_total_memory";
+
+    public MicrometerNativeMemoryMetrics()
+    {
+        if (directBufferPool == null)
+            logger.error("Direct memory buffer pool MBean not present, native memory metrics will be missing for nio buffers");
+    }
+
+    @Override
+    public synchronized void register(MeterRegistry newRegistry, Tags newTags)
+    {
+        super.register(newRegistry, newTags);
+
+        gauge(RAW_NATIVE_MEMORY, this, NativeMemoryMetrics::rawNativeMemory);
+        gauge(BLOOM_FILTER_MEMORY, this, NativeMemoryMetrics::bloomFilterMemory);
+        gauge(COMPRESSION_METADATA_MEMORY, this, NativeMemoryMetrics::compressionMetadataMemory);
+        gauge(NETWORK_DIRECT_MEMORY, this, NativeMemoryMetrics::networkDirectMemory);
+        gauge(USED_NIO_DIRECT_MEMORY, this, NativeMemoryMetrics::usedNioDirectMemory);
+        gauge(TOTAL_NIO_MEMORY, this, NativeMemoryMetrics::totalNioDirectMemory);
+        gauge(NIO_DIRECT_BUFFER_COUNT, this, NativeMemoryMetrics::nioDirectBufferCount);
+        gauge(TOTAL_MEMORY, this, NativeMemoryMetrics::totalMemory);
+    }
+
+    @Override
+    public long usedNioDirectMemoryValue()
+    {
+        return usedNioDirectMemory();
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/MinMaxAvgMetric.java b/src/java/org/apache/cassandra/metrics/MinMaxAvgMetric.java
index b65f52f486b7..6e985536bc15 100644
--- a/src/java/org/apache/cassandra/metrics/MinMaxAvgMetric.java
+++ b/src/java/org/apache/cassandra/metrics/MinMaxAvgMetric.java
@@ -44,20 +44,20 @@ public MinMaxAvgMetric(MetricNameFactory factory, String namePrefix)
         this.factory = factory;
         this.namePrefix = namePrefix;
 
-        minGauge = Metrics.register(factory.createMetricName(namePrefix + "Min"), () -> min);
-        maxGauge = Metrics.register(factory.createMetricName(namePrefix + "Max"), () -> max);
-        avgGauge = Metrics.register(factory.createMetricName(namePrefix + "Avg"), () -> numSamples > 0 ? ((double) sum) / numSamples : 0);
-        stddevGauge = Metrics.register(factory.createMetricName(namePrefix + "StdDev"), () -> stddev());
-        numSamplesGauge = Metrics.register(factory.createMetricName(namePrefix + "NumSamples"), () -> numSamples);
+        minGauge = Metrics.register(factory.createMetricName(namePrefix + " Min"), () -> min);
+        maxGauge = Metrics.register(factory.createMetricName(namePrefix + " Max"), () -> max);
+        avgGauge = Metrics.register(factory.createMetricName(namePrefix + " Avg"), () -> numSamples > 0 ? ((double) sum) / numSamples : 0);
+        stddevGauge = Metrics.register(factory.createMetricName(namePrefix + " StdDev"), () -> stddev());
+        numSamplesGauge = Metrics.register(factory.createMetricName(namePrefix + " NumSamples"), () -> numSamples);
     }
 
     public void release()
     {
-        Metrics.remove(factory.createMetricName(namePrefix + "Min"));
-        Metrics.remove(factory.createMetricName(namePrefix + "Max"));
-        Metrics.remove(factory.createMetricName(namePrefix + "Avg"));
-        Metrics.remove(factory.createMetricName(namePrefix + "StdDev"));
-        Metrics.remove(factory.createMetricName(namePrefix + "NumSamples"));
+        Metrics.remove(factory.createMetricName(namePrefix + " Min"));
+        Metrics.remove(factory.createMetricName(namePrefix + " Max"));
+        Metrics.remove(factory.createMetricName(namePrefix + " Avg"));
+        Metrics.remove(factory.createMetricName(namePrefix + " StdDev"));
+        Metrics.remove(factory.createMetricName(namePrefix + " NumSamples"));
     }
 
     public void reset()
diff --git a/src/java/org/apache/cassandra/metrics/NativeMemoryMetrics.java b/src/java/org/apache/cassandra/metrics/NativeMemoryMetrics.java
new file mode 100644
index 000000000000..1909733680ae
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/NativeMemoryMetrics.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.lang.management.BufferPoolMXBean;
+import java.lang.management.ManagementFactory;
+
+import io.netty.util.internal.PlatformDependent;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+public interface NativeMemoryMetrics
+{
+    BufferPoolMXBean directBufferPool = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)
+                                                         .stream()
+                                                         .filter(bpMBean -> bpMBean.getName().equals("direct"))
+                                                         .findFirst()
+                                                         .orElse(null);
+
+    NativeMemoryMetrics instance = CassandraRelevantProperties.USE_MICROMETER.getBoolean()
+                                   ? new MicrometerNativeMemoryMetrics()
+                                   : new CodehaleNativeMemoryMetrics();
+
+    long usedNioDirectMemoryValue();
+
+    default long rawNativeMemory()
+    {
+        return MemoryUtil.allocated();
+    }
+
+    default long bloomFilterMemory()
+    {
+        return BloomFilter.memoryLimiter.memoryAllocated();
+    }
+
+    default long compressionMetadataMemory()
+    {
+        return CompressionMetadata.nativeMemoryAllocated();
+    }
+
+    default long usedNioDirectMemory()
+    {
+        return directBufferPool == null ? 0 : directBufferPool.getMemoryUsed();
+    }
+
+    default long totalNioDirectMemory()
+    {
+        return directBufferPool == null ? 0 : directBufferPool.getTotalCapacity();
+    }
+
+    default long nioDirectBufferCount()
+    {
+        return directBufferPool == null ? 0 : directBufferPool.getCount();
+    }
+
+    default long networkDirectMemory()
+    {
+        return PlatformDependent.usedDirectMemory();
+    }
+
+    default boolean usingNioMemoryForNetwork()
+    {
+        return !PlatformDependent.useDirectBufferNoCleaner();
+    }
+
+    default long totalMemory()
+    {
+        // Use totalNioDirectMemory() instead of usedNioDirectMemory() because without
+        // -Dsun.nio.PageAlignDirectMemory=true the two are identical. If someone adds
+        // this flag again, we would prefer to include the JVM padding in the total memory.
+        // Also only add the network memory if it's not allocated as NIO direct memory
+        return rawNativeMemory() + totalNioDirectMemory() + (usingNioMemoryForNetwork() ? 0 : networkDirectMemory());
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/PairedSlidingWindowReservoir.java b/src/java/org/apache/cassandra/metrics/PairedSlidingWindowReservoir.java
new file mode 100644
index 000000000000..0d512d801c84
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/PairedSlidingWindowReservoir.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+/**
+ * A Reservoir-like class that tracks the last N paired measurements.
+ */
+public class PairedSlidingWindowReservoir
+{
+    private final IntIntPair[] measurements;
+    private long count;
+
+    public PairedSlidingWindowReservoir(int size)
+    {
+        this.measurements = new IntIntPair[size];
+        this.count = 0L;
+    }
+
+    public int size()
+    {
+        if (this.count >= this.measurements.length)
+            return this.measurements.length;
+
+        synchronized (this)
+        {
+            return (int) Math.min(this.count, this.measurements.length);
+        }
+    }
+
+    public synchronized void update(int value1, int value2)
+    {
+        this.measurements[(int) (this.count++ % this.measurements.length)] = new IntIntPair(value1, value2);
+    }
+
+    public PairedSnapshot getSnapshot()
+    {
+        var values = new IntIntPair[this.size()];
+        System.arraycopy(this.measurements, 0, values, 0, values.length);
+        return new PairedSnapshot(values);
+    }
+
+    /**
+     * A pair of ints.  "y" and "x" are used to imply that the first value is the
+     * dependent one for a LinearFit computation.
+     */
+    public static class IntIntPair
+    {
+        public final int x;
+        public final int y;
+
+        IntIntPair(int x, int y)
+        {
+            this.y = y;
+            this.x = x;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("(%d,%d)", x, y);
+        }
+    }
+
+    public static class PairedSnapshot
+    {
+        public final IntIntPair[] values;
+
+        public PairedSnapshot(IntIntPair[] values)
+        {
+            this.values = values;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/QuickSlidingWindowReservoir.java b/src/java/org/apache/cassandra/metrics/QuickSlidingWindowReservoir.java
new file mode 100644
index 000000000000..1a897ef3877a
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/QuickSlidingWindowReservoir.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Reservoir;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.UniformSnapshot;
+
+/**
+ * A reservoir that stores the last N measurements, following the same design
+ * as com.codahale.metrics.SlidingWindowReservoir but adding a snapshot-free getMean().
+ */
+public class QuickSlidingWindowReservoir implements Reservoir
+{
+    private final long[] measurements;
+    private long count;
+
+    public QuickSlidingWindowReservoir(int size)
+    {
+        this.measurements = new long[size];
+        this.count = 0L;
+    }
+
+    public int size()
+    {
+        if (this.count >= this.measurements.length)
+            return this.measurements.length;
+
+        synchronized (this)
+        {
+            return (int) Math.min(this.count, this.measurements.length);
+        }
+    }
+
+    public synchronized void update(long value)
+    {
+        this.measurements[(int) (this.count++ % this.measurements.length)] = value;
+    }
+
+    /**
+     * Returns the mean of the values in the reservoir, without synchronization.  (Generally,
+     * new values will be just as valid as the old ones.)  For a strictly consistent view,
+     * use {@link #getSnapshot()}.
+     */
+    public double getMean()
+    {
+        final int sz = size();
+
+        if (sz == 0)
+            return 0.0;
+
+        double sum = 0.0;
+        for (int i = 0; i < sz; ++i)
+            sum += this.measurements[i];
+
+        return sum / sz;
+    }
+
+    public Snapshot getSnapshot()
+    {
+        long[] values = new long[this.size()];
+        synchronized (this)
+        {
+            System.arraycopy(this.measurements, 0, values, 0, values.length);
+        }
+        return new UniformSnapshot(values);
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/ReadCoordinationMetrics.java b/src/java/org/apache/cassandra/metrics/ReadCoordinationMetrics.java
new file mode 100644
index 000000000000..2f7aedad49b4
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/ReadCoordinationMetrics.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+/**
+ * Metrics for read coordination behaviors.
+ */
+public final class ReadCoordinationMetrics
+{
+    private static final MetricNameFactory factory = new DefaultNameFactory("ReadCoordination");
+
+    public static final Counter nonreplicaRequests = Metrics.counter(factory.createMetricName("LocalNodeNonreplicaRequests"));
+    public static final Counter preferredOtherReplicas = Metrics.counter(factory.createMetricName("PreferredOtherReplicas"));
+
+    private static final ConcurrentMap<InetAddressAndPort, Histogram> replicaLatencies = new ConcurrentHashMap<>();
+
+    public static void updateReplicaLatency(InetAddressAndPort address, long latency, TimeUnit timeUnit)
+    {
+        if (latency >= DatabaseDescriptor.getReadRpcTimeout(timeUnit))
+            return; // don't track timeouts
+
+        Histogram histogram = replicaLatencies.get(address);
+
+        // avoid computeIfAbsent() call on the common path
+        if (null == histogram)
+            histogram = replicaLatencies.computeIfAbsent(address, ReadCoordinationMetrics::createHistogram);
+
+        histogram.update(latency);
+    }
+
+    private static Histogram createHistogram(InetAddressAndPort h)
+    {
+        CassandraMetricsRegistry.MetricName metricName = DefaultNameFactory.createMetricName("ReadCoordination", "ReplicaLatency", h.getHostAddressAndPort().replace(':', '.'));
+        return Metrics.histogram(metricName, false);
+    }
+    
+    @VisibleForTesting
+    static Histogram getReplicaLatencyHistogram(InetAddressAndPort address)
+    {
+        return replicaLatencies.get(address);
+    }
+}
diff --git a/src/java/org/apache/cassandra/metrics/RepairMetrics.java b/src/java/org/apache/cassandra/metrics/RepairMetrics.java
index 27dbbd31181c..364e4ae80e83 100644
--- a/src/java/org/apache/cassandra/metrics/RepairMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/RepairMetrics.java
@@ -19,7 +19,7 @@
 package org.apache.cassandra.metrics;
 
 import java.util.Collections;
-import java.util.EnumMap;
+import java.util.HashMap;
 import java.util.Map;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -44,9 +44,9 @@ public class RepairMetrics
 
     static
     {
-        Map<Verb, Histogram> retries = new EnumMap<>(Verb.class);
-        Map<Verb, Counter> timeout = new EnumMap<>(Verb.class);
-        Map<Verb, Counter> failure = new EnumMap<>(Verb.class);
+        Map<Verb, Histogram> retries = new HashMap<>();
+        Map<Verb, Counter> timeout = new HashMap<>();
+        Map<Verb, Counter> failure = new HashMap<>();
         for (Verb verb : RepairMessage.ALLOWS_RETRY)
         {
             retries.put(verb, Metrics.histogram(DefaultNameFactory.createMetricName(TYPE_NAME, "Retries-" + verb.name(), null), false));
diff --git a/src/java/org/apache/cassandra/metrics/RestorableMeter.java b/src/java/org/apache/cassandra/metrics/RestorableMeter.java
index ea3fddeb2be6..166cef2b0e2a 100644
--- a/src/java/org/apache/cassandra/metrics/RestorableMeter.java
+++ b/src/java/org/apache/cassandra/metrics/RestorableMeter.java
@@ -19,14 +19,20 @@
 package org.apache.cassandra.metrics;
 
 
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
 import static java.lang.Math.exp;
 import com.codahale.metrics.Clock;
 
 /**
- * A meter metric which measures mean throughput as well as fifteen-minute and two-hour
+ * A meter metric which measures mean throughput as well as one-minute, five-minute, fifteen-minute and two-hour
  * exponentially-weighted moving average throughputs.
  *
  * This is based heavily on the Meter and EWMA classes from codahale/yammer metrics.
@@ -35,10 +41,21 @@
  */
 public class RestorableMeter
 {
-    private static final long TICK_INTERVAL = TimeUnit.SECONDS.toNanos(5);
+    public static final Set<Integer> AVAILABLE_WINDOWS = Set.of(1, 5, 15, 120);
+
+    public static final long TICK_INTERVAL = TimeUnit.SECONDS.toNanos(5);
     private static final double NANOS_PER_SECOND = TimeUnit.SECONDS.toNanos(1);
 
+    @Nullable
+    private final RestorableEWMA m1Rate;
+
+    @Nullable
+    private final RestorableEWMA m5Rate;
+
+    @Nullable
     private final RestorableEWMA m15Rate;
+
+    @Nullable
     private final RestorableEWMA m120Rate;
 
     private final AtomicLong count = new AtomicLong();
@@ -47,29 +64,125 @@ public class RestorableMeter
     private final Clock clock = Clock.defaultClock();
 
     /**
-     * Creates a new, uninitialized RestorableMeter.
+     * Creates a new RestorableMeter with given ewmas.
      */
-    public RestorableMeter()
+    private RestorableMeter(@Nullable RestorableEWMA m1Rate, @Nullable RestorableEWMA m5Rate, @Nullable RestorableEWMA m15Rate, @Nullable RestorableEWMA m120Rate)
     {
-        this.m15Rate = new RestorableEWMA(TimeUnit.MINUTES.toSeconds(15));
-        this.m120Rate = new RestorableEWMA(TimeUnit.MINUTES.toSeconds(120));
+        this.m1Rate = m1Rate;
+        this.m5Rate = m5Rate;
+        this.m15Rate = m15Rate;
+        this.m120Rate = m120Rate;
         this.startTime = this.clock.getTick();
         this.lastTick = new AtomicLong(startTime);
     }
 
     /**
-     * Restores a RestorableMeter from the last seen 15m and 2h rates.
+     * Restores a RestorableMeter from the last seen 15m and 2h rates. 1m and 5m rates are not initialized.
+     *
      * @param lastM15Rate the last-seen 15m rate, in terms of events per second
      * @param lastM120Rate the last seen 2h rate, in terms of events per second
      */
+    @VisibleForTesting
     public RestorableMeter(double lastM15Rate, double lastM120Rate)
     {
-        this.m15Rate = new RestorableEWMA(lastM15Rate, TimeUnit.MINUTES.toSeconds(15));
-        this.m120Rate = new RestorableEWMA(lastM120Rate, TimeUnit.MINUTES.toSeconds(120));
+        this.m1Rate = null;
+        this.m5Rate = null;
+        this.m15Rate = ewma(15, lastM15Rate);
+        this.m120Rate = ewma(120, lastM120Rate);
         this.startTime = this.clock.getTick();
         this.lastTick = new AtomicLong(startTime);
     }
 
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    /**
+     * Create a restorable meter with default rates (15m and 120m)
+     */
+    public static RestorableMeter createWithDefaultRates()
+    {
+        return new Builder().withM15Rate().withM120Rate().build();
+    }
+
+    public static class Builder
+    {
+        private RestorableEWMA m1Rate;
+        private RestorableEWMA m5Rate;
+        private RestorableEWMA m15Rate;
+        private RestorableEWMA m120Rate;
+
+        public Builder withWindow(int window)
+        {
+            switch (window)
+            {
+                case 1   : return withM1Rate();
+                case 5   : return withM5Rate();
+                case 15  : return withM15Rate();
+                case 120 : return withM120Rate();
+                default  : throw new IllegalArgumentException(String.format("Found invalid window=%s, available windows: %s", window, AVAILABLE_WINDOWS));
+            }
+        }
+
+        public Builder withM1Rate()
+        {
+            Preconditions.checkState(m1Rate == null);
+            this.m1Rate = ewma(1);
+            return this;
+        }
+
+        public Builder withM5Rate()
+        {
+            Preconditions.checkState(m5Rate == null);
+            this.m5Rate = ewma(5);
+            return this;
+        }
+
+        public Builder withM15Rate()
+        {
+            Preconditions.checkState(m15Rate == null);
+            this.m15Rate = ewma(15);
+            return this;
+        }
+
+        public Builder withM15Rate(double lastM15Rate)
+        {
+            Preconditions.checkState(m15Rate == null);
+            this.m15Rate = ewma(15, lastM15Rate);
+            return this;
+        }
+
+        public Builder withM120Rate()
+        {
+            Preconditions.checkState(m120Rate == null);
+            this.m120Rate = ewma(120);
+            return this;
+        }
+
+        public Builder withM120Rate(double lastM120Rate)
+        {
+            Preconditions.checkState(m120Rate == null);
+            this.m120Rate = ewma(120, lastM120Rate);
+            return this;
+        }
+
+        public RestorableMeter build()
+        {
+            return new RestorableMeter(m1Rate, m5Rate, m15Rate, m120Rate);
+        }
+    }
+
+    private static RestorableEWMA ewma(int minute, double lastRate)
+    {
+        return new RestorableEWMA(lastRate, TimeUnit.MINUTES.toSeconds(minute));
+    }
+
+    private static RestorableEWMA ewma(int minute)
+    {
+        return new RestorableEWMA(TimeUnit.MINUTES.toSeconds(minute));
+    }
+
     /**
      * Updates the moving averages as needed.
      */
@@ -86,8 +199,10 @@ private void tickIfNecessary()
                 final long requiredTicks = age / TICK_INTERVAL;
                 for (long i = 0; i < requiredTicks; i++)
                 {
-                    m15Rate.tick();
-                    m120Rate.tick();
+                    if (m1Rate != null) m1Rate.tick();
+                    if (m5Rate != null) m5Rate.tick();
+                    if (m15Rate != null) m15Rate.tick();
+                    if (m120Rate != null) m120Rate.tick();
                 }
             }
         }
@@ -110,8 +225,30 @@ public void mark(long n)
     {
         tickIfNecessary();
         count.addAndGet(n);
-        m15Rate.update(n);
-        m120Rate.update(n);
+        if (m1Rate != null) m1Rate.update(n);
+        if (m5Rate != null) m5Rate.update(n);
+        if (m15Rate != null) m15Rate.update(n);
+        if (m120Rate != null) m120Rate.update(n);
+    }
+
+    /**
+     * Returns the 1-minute rate in terms of events per second.  This DOES NOT carry the previous rate when restored.
+     */
+    public double oneMinuteRate()
+    {
+        Preconditions.checkNotNull(m1Rate);
+        tickIfNecessary();
+        return m1Rate.rate();
+    }
+
+    /**
+     * Returns the 5-minute rate in terms of events per second.  This DOES NOT carry the previous rate when restored.
+     */
+    public double fiveMinuteRate()
+    {
+        Preconditions.checkNotNull(m5Rate);
+        tickIfNecessary();
+        return m5Rate.rate();
     }
 
     /**
@@ -119,6 +256,7 @@ public void mark(long n)
      */
     public double fifteenMinuteRate()
     {
+        Preconditions.checkNotNull(m15Rate);
         tickIfNecessary();
         return m15Rate.rate();
     }
@@ -128,10 +266,27 @@ public double fifteenMinuteRate()
      */
     public double twoHourRate()
     {
+        Preconditions.checkNotNull(m120Rate);
         tickIfNecessary();
         return m120Rate.rate();
     }
 
+    /**
+     * @param window window of time in minutes
+     * @return rate in terms of events per second with given window.
+     */
+    public double rate(int window)
+    {
+        switch (window)
+        {
+            case 1   : return oneMinuteRate();
+            case 5   : return fiveMinuteRate();
+            case 15  : return fifteenMinuteRate();
+            case 120 : return twoHourRate();
+            default  : throw new IllegalArgumentException(String.format("Found invalid window=%s, available windows: %s", window, AVAILABLE_WINDOWS));
+        }
+    }
+
     /**
      * The total number of events that have occurred since this object was created.  Note that the previous count
      * is *not* carried over when a RestorableMeter is restored.
diff --git a/src/java/org/apache/cassandra/metrics/Sampler.java b/src/java/org/apache/cassandra/metrics/Sampler.java
index 4c4739b32984..3e48bf22efbc 100644
--- a/src/java/org/apache/cassandra/metrics/Sampler.java
+++ b/src/java/org/apache/cassandra/metrics/Sampler.java
@@ -106,7 +106,7 @@ void format(SamplingManager.ResultBuilder resultBuilder, PrintStream ps)
 
     private long endTimeNanos = -1;
 
-    public void addSample(final T item, final int value)
+    public void addSample(final T item, final long value)
     {
         if (isEnabled())
             samplerExecutor.submit(() -> insert(item, value));
diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java
index b2a8e61fe4ca..67708a99a64a 100644
--- a/src/java/org/apache/cassandra/metrics/TableMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java
@@ -17,34 +17,48 @@
  */
 package org.apache.cassandra.metrics;
 
+import java.lang.ref.WeakReference;
+import java.nio.BufferUnderflowException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongSupplier;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.apache.commons.lang3.ArrayUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.CachedGauge;
 import com.codahale.metrics.Counter;
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Metric;
 import com.codahale.metrics.Timer;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.memtable.Memtable;
@@ -57,10 +71,14 @@
 import org.apache.cassandra.metrics.Sampler.SamplerType;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.EstimatedHistogram;
 import org.apache.cassandra.utils.ExpMovingAverage;
+import org.apache.cassandra.utils.Hex;
 import org.apache.cassandra.utils.MovingAverage;
 import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.concurrent.Refs;
 
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -71,6 +89,63 @@
  */
 public class TableMetrics
 {
+    // CNDB will set this to MetricsAggregation.AGGREGATED in order to support large number of tenants and dedicated
+    // tenants with a large number of tables
+    public static final String TABLE_METRICS_DEFAULT_HISTOGRAMS_AGGREGATION =
+    CassandraRelevantProperties.TABLE_METRICS_DEFAULT_HISTOGRAMS_AGGREGATION.getString();
+
+    // CNDB will set this to false as it does not need global metrics since the aggregation is done in Prometheus
+    public static final boolean EXPORT_GLOBAL_METRICS = CassandraRelevantProperties.TABLE_METRICS_EXPORT_GLOBALS.getBoolean();
+
+    private static final Logger logger = LoggerFactory.getLogger(TableMetrics.class);
+
+    public static final String TABLE_EXTENSIONS_HISTOGRAMS_METRICS_KEY = "HISTOGRAM_METRICS";
+
+    public enum MetricsAggregation
+    {
+        AGGREGATED((byte) 0x00),
+        INDIVIDUAL((byte) 0x01);
+
+        public final byte val;
+
+        MetricsAggregation(byte val)
+        {
+            this.val = val;
+        }
+
+        public static MetricsAggregation fromMetadata(TableMetadata metadata)
+        {
+            MetricsAggregation defaultValue = MetricsAggregation.valueOf(TABLE_METRICS_DEFAULT_HISTOGRAMS_AGGREGATION);
+            ByteBuffer bb = null;
+            try
+            {
+                bb = metadata.params.extensions.get(TABLE_EXTENSIONS_HISTOGRAMS_METRICS_KEY);
+                return bb == null ? defaultValue : MetricsAggregation.fromByte(bb.get(bb.position())); // do not change the position of the ByteBuffer!
+            }
+            catch (BufferUnderflowException | IllegalStateException ex)
+            {
+                logger.error("Failed to decode metadata extensions for metrics aggregation ({}), using default value {}", bb, defaultValue);
+                return defaultValue;
+            }
+        }
+
+        public static MetricsAggregation fromByte(byte val) throws IllegalStateException
+        {
+            for (MetricsAggregation aggr : values())
+            {
+                if (aggr.val == val)
+                    return aggr;
+            }
+
+            throw new IllegalStateException("Invalid byte: " + val);
+        }
+
+        public String asCQLString()
+        {
+            return "0x" + Hex.bytesToHex(val);
+        }
+    }
+
     /**
      * stores metrics that will be rolled into a single global metric
      */
@@ -79,9 +154,9 @@ public class TableMetrics
     private static final MetricNameFactory GLOBAL_FACTORY = new AllTableMetricNameFactory("Table");
     private static final MetricNameFactory GLOBAL_ALIAS_FACTORY = new AllTableMetricNameFactory("ColumnFamily");
 
-    public final static LatencyMetrics GLOBAL_READ_LATENCY = new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Read");
-    public final static LatencyMetrics GLOBAL_WRITE_LATENCY = new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Write");
-    public final static LatencyMetrics GLOBAL_RANGE_LATENCY = new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Range");
+    public final static Optional<LatencyMetrics> GLOBAL_READ_LATENCY = EXPORT_GLOBAL_METRICS ? Optional.of(new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Read")) : Optional.empty();
+    public final static Optional<LatencyMetrics> GLOBAL_WRITE_LATENCY = EXPORT_GLOBAL_METRICS ? Optional.of(new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Write")) : Optional.empty();
+    public final static Optional<LatencyMetrics> GLOBAL_RANGE_LATENCY = EXPORT_GLOBAL_METRICS ? Optional.of(new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Range")) : Optional.empty();
 
     /** Total amount of data stored in the memtable that resides on-heap, including column related overhead and partitions overwritten. */
     public final Gauge<Long> memtableOnHeapDataSize;
@@ -105,27 +180,66 @@ public class TableMetrics
     public final Gauge<long[]> estimatedPartitionSizeHistogram;
     /** Approximate number of keys in table. */
     public final Gauge<Long> estimatedPartitionCount;
+    /** This function is used to calculate estimated partition count in sstables and store the calculated value for the
+     *  current set of sstables. */
+    public final LongSupplier estimatedPartitionCountInSSTables;
+    /** A cached version of the estimated partition count in sstables, used by compaction. This value will be more
+     *  precise when the table has a small number of partitions that keep getting written to. */
+    public final Gauge<Long> estimatedPartitionCountInSSTablesCached;
     /** Histogram of estimated number of columns. */
     public final Gauge<long[]> estimatedColumnCountHistogram;
+    /** Approximate number of rows in SSTable*/
+    public final Gauge<Long> estimatedRowCount;
     /** Histogram of the number of sstable data files accessed per single partition read */
     public final TableHistogram sstablesPerReadHistogram;
     /** Histogram of the number of sstable data files accessed per partition range read */
     public final TableHistogram sstablesPerRangeReadHistogram;
+    /** An approximate measure of how long it takes to read a partition from an sstable, in nanoseconds. This is
+     * a moving average of a very rough approximation: the total latency for a single partition
+     * read command divided by the number of sstables that were accessed for that command.
+     * Therefore it currently includes other costs, which is not ideal but it does give a rough estimate.
+     * since disk costs would dominate computing costs. */
+    public final MovingAverage sstablePartitionReadLatency;
     /** (Local) read metrics */
-    public final LatencyMetrics readLatency;
+    public final TableLatencyMetrics readLatency;
     /** (Local) range slice metrics */
-    public final LatencyMetrics rangeLatency;
+    public final TableLatencyMetrics rangeLatency;
     /** (Local) write metrics */
-    public final LatencyMetrics writeLatency;
+    public final TableLatencyMetrics writeLatency;
+    /** The number of single partition read requests, including those dropped due to timeouts */
+    public final Counter readRequests;
+    /** The number of range read requests, including those dropped due to timeouts */
+    public final Counter rangeRequests;
     /** Estimated number of tasks pending for this table */
     public final Counter pendingFlushes;
     /** Total number of bytes flushed since server [re]start */
     public final Counter bytesFlushed;
+    /** The average flushed size for sstables, which is derived from {@link this#bytesFlushed}. */
+    public final MovingAverage flushSize;
     /** The average on-disk flushed size for sstables. */
-    public final MovingAverage flushSizeOnDisk;
+    private final MovingAverage flushSizeOnDisk;
+    /** The average number of sstables created on flush. */
+    public final MovingAverage flushSegmentCount;
+    /** The average duration per 1Kb of data flushed, in nanoseconds. */
+    public final MovingAverage flushTimePerKb;
+    /** Time spent in flushing memtables */
+    public final Counter flushTime;
+    public final Counter storageAttachedIndexBuildTime;
+    public final Counter storageAttachedIndexWritingTimeForIndexBuild;
+    public final Counter storageAttachedIndexWritingTimeForCompaction;
+    public final Counter storageAttachedIndexWritingTimeForFlush;
+    public final Counter storageAttachedIndexWritingTimeForOther;
+    /** Total number of bytes inserted into memtables since server [re]start. */
+    public final Counter bytesInserted;
     /** Total number of bytes written by compaction since server [re]start */
     public final Counter compactionBytesWritten;
-    /** Estimate of number of pending compactios for this table */
+    /** Total number of bytes read by compaction since server [re]start */
+    public final Counter compactionBytesRead;
+    /** The average duration per 1Kb of data compacted, in nanoseconds. */
+    public final MovingAverage compactionTimePerKb;
+    /** Time spent in writing sstables during compaction  */
+    public final Counter compactionTime;
+    /** Estimate of number of pending compactions for this table */
     public final Gauge<Integer> pendingCompactions;
     /** Number of SSTables on disk for this CF */
     public final Gauge<Integer> liveSSTableCount;
@@ -147,10 +261,18 @@ public class TableMetrics
     public final Gauge<Long> maxPartitionSize;
     /** Size of the smallest compacted partition */
     public final Gauge<Long> meanPartitionSize;
+    /** False positive ratio of bloom filter */
+    public final Gauge<Double> bloomFilterFalseRatio;
     /** Off heap memory used by compression meta data*/
     public final Gauge<Long> compressionMetadataOffHeapMemoryUsed;
+
+    /** Shadowed keys scan metrics **/
+    public final TableHistogram shadowedKeysScannedHistogram;
+    public final TableHistogram shadowedKeysLoopsHistogram;
+
     /** Tombstones scanned in queries on this CF */
     public final TableHistogram tombstoneScannedHistogram;
+    public final Counter tombstoneScannedCounter;
     /** Live rows scanned in queries on this CF */
     public final TableHistogram liveScannedHistogram;
     /** Column update time delta on this CF */
@@ -176,11 +298,11 @@ public class TableMetrics
      */
     public final Counter tombstoneWarnings;
     /** CAS Prepare metrics */
-    public final LatencyMetrics casPrepare;
+    public final TableLatencyMetrics casPrepare;
     /** CAS Propose metrics */
-    public final LatencyMetrics casPropose;
+    public final TableLatencyMetrics casPropose;
     /** CAS Commit metrics */
-    public final LatencyMetrics casCommit;
+    public final TableLatencyMetrics casCommit;
     /** percent of the data that is repaired */
     public final Gauge<Double> percentRepaired;
     /** Reports the size of sstables in repaired, unrepaired, and any ongoing repair buckets */
@@ -208,9 +330,9 @@ public class TableMetrics
     /** ratio of how much we anticompact vs how much we could mutate the repair status*/
     public final Gauge<Double> mutatedAnticompactionGauge;
 
-    public final SnapshottingTimer coordinatorReadLatency;
-    public final Timer coordinatorScanLatency;
-    public final SnapshottingTimer coordinatorWriteLatency;
+    public final TableTimer<SnapshottingTimer> coordinatorReadLatency;
+    public final TableTimer coordinatorScanLatency;
+    public final TableTimer<SnapshottingTimer> coordinatorWriteLatency;
 
     private final MetricNameFactory factory;
     private final MetricNameFactory aliasFactory;
@@ -282,6 +404,12 @@ public class TableMetrics
 
     public final ImmutableMap<SSTableFormat<?, ?>, ImmutableMap<String, Gauge<? extends Number>>> formatSpecificGauges;
 
+    /**
+     * This property determines if new metrics dedicated to this table are created, or if keyspace metrics are
+     * used instead.
+     * */
+    public final MetricsAggregation metricsAggregation;
+
     private static Pair<Long, Long> totalNonSystemTablesSize(Predicate<SSTableReader> predicate)
     {
         long total = 0;
@@ -401,6 +529,9 @@ public static long[] addHistogram(long[] sums, long[] buckets)
      */
     public TableMetrics(final ColumnFamilyStore cfs, ReleasableMetric memtableMetrics)
     {
+        metricsAggregation = MetricsAggregation.fromMetadata(cfs.metadata());
+        logger.trace("Using {} histograms for table={}", metricsAggregation, cfs.metadata());
+
         factory = new TableMetricNameFactory(cfs, "Table");
         aliasFactory = new TableMetricNameFactory(cfs, "ColumnFamily");
 
@@ -527,26 +658,95 @@ public Long getValue()
         estimatedPartitionSizeHistogram = createTableGauge("EstimatedPartitionSizeHistogram", "EstimatedRowSizeHistogram",
                                                            () -> combineHistograms(cfs.getSSTables(SSTableSet.CANONICAL),
                                                                                    SSTableReader::getEstimatedPartitionSize), null);
-        
+
+        estimatedPartitionCountInSSTables = new LongSupplier()
+        {
+            // Since the sstables only change when the tracker view changes, we can cache the value.
+            AtomicReference<Pair<WeakReference<View>, Long>> collected = new AtomicReference<>(Pair.create(new WeakReference<>(null), 0L));
+
+            public long getAsLong()
+            {
+                final View currentView = cfs.getTracker().getView();
+                final Pair<WeakReference<View>, Long> currentCollected = collected.get();
+                if (currentView != currentCollected.left.get())
+                {
+                    Refs<SSTableReader> refs = Refs.tryRef(currentView.select(SSTableSet.CANONICAL));
+                    if (refs != null)
+                    {
+                        try (refs)
+                        {
+                            long collectedValue = SSTableReader.getApproximateKeyCount(refs);
+                            final Pair<WeakReference<View>, Long> newCollected = Pair.create(new WeakReference<>(currentView), collectedValue);
+                            collected.compareAndSet(currentCollected, newCollected); // okay if failed, a different thread did it
+                            return collectedValue;
+                        }
+                    }
+                    // If we can't reference, simply return the previous collected value; it can only result in a delay
+                    // in reporting the correct key count.
+                }
+                return currentCollected.right;
+            }
+        };
         estimatedPartitionCount = createTableGauge("EstimatedPartitionCount", "EstimatedRowCount", new Gauge<Long>()
         {
             public Long getValue()
             {
-                long memtablePartitions = 0;
+                long estimatedPartitions = estimatedPartitionCountInSSTables.getAsLong();
                 for (Memtable memtable : cfs.getTracker().getView().getAllMemtables())
-                   memtablePartitions += memtable.partitionCount();
-                try(ColumnFamilyStore.RefViewFragment refViewFragment = cfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL)))
-                {
-                    return SSTableReader.getApproximateKeyCount(refViewFragment.sstables) + memtablePartitions;
-                }
+                    estimatedPartitions += memtable.partitionCount();
+                return estimatedPartitions;
             }
         }, null);
+        estimatedPartitionCountInSSTablesCached = new CachedGauge<Long>(1, TimeUnit.SECONDS)
+        {
+            public Long loadValue()
+            {
+                return estimatedPartitionCountInSSTables.getAsLong();
+            }
+        };
+
         estimatedColumnCountHistogram = createTableGauge("EstimatedColumnCountHistogram", "EstimatedColumnCountHistogram",
                                                          () -> combineHistograms(cfs.getSSTables(SSTableSet.CANONICAL), 
                                                                                  SSTableReader::getEstimatedCellPerPartitionCount), null);
+
+        estimatedRowCount = createTableGauge("EstimatedRowCount", "EstimatedRowCount", new CachedGauge<>(1, TimeUnit.SECONDS)
+        {
+            public Long loadValue()
+            {
+                long memtableRows = 0;
+                OpOrder.Group readGroup = null;
+                try
+                {
+                    for (Memtable memtable : cfs.getTracker().getView().getAllMemtables())
+                    {
+                        if (readGroup == null)
+                        {
+                            readGroup = memtable.readOrdering().start();
+                        }
+                        memtableRows += Memtable.estimateRowCount(memtable);
+                    }
+                }
+                finally
+                {
+                    if (readGroup != null)
+                        readGroup.close();
+                }
+
+                long sstableRows = 0;
+                try(ColumnFamilyStore.RefViewFragment refViewFragment = cfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL)))
+                {
+                    for (SSTableReader reader: refViewFragment.sstables)
+                    {
+                        sstableRows += reader.getTotalRows();
+                    }
+                }
+                return sstableRows + memtableRows;
+            }
+        }, null);
         
-        sstablesPerReadHistogram = createTableHistogram("SSTablesPerReadHistogram", cfs.keyspace.metric.sstablesPerReadHistogram, true);
-        sstablesPerRangeReadHistogram = createTableHistogram("SSTablesPerRangeReadHistogram", cfs.keyspace.metric.sstablesPerRangeReadHistogram, true);
+        sstablesPerReadHistogram = createTableHistogram("SSTablesPerReadHistogram", cfs.getKeyspaceMetrics().sstablesPerReadHistogram, true);
+        sstablesPerRangeReadHistogram = createTableHistogram("SSTablesPerRangeReadHistogram", cfs.getKeyspaceMetrics().sstablesPerRangeReadHistogram, true);
+        sstablePartitionReadLatency = ExpMovingAverage.decayBy100();
         compressionRatio = createTableGauge("CompressionRatio", new Gauge<Double>()
         {
             public Double getValue()
@@ -619,16 +819,33 @@ public Long getValue()
             }
         });
 
-        readLatency = createLatencyMetrics("Read", cfs.keyspace.metric.readLatency, GLOBAL_READ_LATENCY);
-        writeLatency = createLatencyMetrics("Write", cfs.keyspace.metric.writeLatency, GLOBAL_WRITE_LATENCY);
-        rangeLatency = createLatencyMetrics("Range", cfs.keyspace.metric.rangeLatency, GLOBAL_RANGE_LATENCY);
+        readLatency = createLatencyMetrics("Read", cfs.getKeyspaceMetrics().readLatency, GLOBAL_READ_LATENCY);
+        writeLatency = createLatencyMetrics("Write", cfs.getKeyspaceMetrics().writeLatency, GLOBAL_WRITE_LATENCY);
+        rangeLatency = createLatencyMetrics("Range", cfs.getKeyspaceMetrics().rangeLatency, GLOBAL_RANGE_LATENCY);
+
+        readRequests = createTableCounter("ReadRequests");
+        rangeRequests = createTableCounter("RangeRequests");
+
         pendingFlushes = createTableCounter("PendingFlushes");
         bytesFlushed = createTableCounter("BytesFlushed");
+        flushSize = ExpMovingAverage.decayBy100();
         flushSizeOnDisk = ExpMovingAverage.decayBy1000();
+        flushSegmentCount = ExpMovingAverage.decayBy1000();
+        flushTimePerKb = ExpMovingAverage.decayBy100();
+        flushTime = createTableCounter("FlushTime");
+        storageAttachedIndexBuildTime = createTableCounter("StorageAttachedIndexBuildTime");
+        storageAttachedIndexWritingTimeForIndexBuild = createTableCounter("StorageAttachedIndexWritingTimeForIndexBuild");
+        storageAttachedIndexWritingTimeForCompaction = createTableCounter("StorageAttachedIndexWritingTimeForCompaction");
+        storageAttachedIndexWritingTimeForFlush = createTableCounter("StorageAttachedIndexWritingTimeForFlush");
+        storageAttachedIndexWritingTimeForOther= createTableCounter("StorageAttachedIndexWritingTimeForOther");
+        bytesInserted = createTableCounter("BytesInserted");
 
         compactionBytesWritten = createTableCounter("CompactionBytesWritten");
-        pendingCompactions = createTableGauge("PendingCompactions", () -> cfs.getCompactionStrategyManager().getEstimatedRemainingTasks());
-        liveSSTableCount = createTableGauge("LiveSSTableCount", () -> cfs.getTracker().getView().liveSSTables().size());
+        compactionBytesRead = createTableCounter("CompactionBytesRead");
+        compactionTimePerKb = ExpMovingAverage.decayBy100();
+        compactionTime = createTableCounter("CompactionTime");
+        pendingCompactions = createTableGauge("PendingCompactions", () -> cfs.getCompactionStrategy().getEstimatedRemainingTasks());
+        liveSSTableCount = createTableGauge("LiveSSTableCount", () -> cfs.getLiveSSTables().size());
         oldVersionSSTableCount = createTableGauge("OldVersionSSTableCount", new Gauge<Integer>()
         {
             public Integer getValue()
@@ -752,6 +969,39 @@ public Long getValue()
                 return count > 0 ? sum / count : 0;
             }
         });
+        bloomFilterFalseRatio = createTableGauge("BloomFilterFalseRatio", new Gauge<Double>()
+        {
+            public Double getValue()
+            {
+                long falsePositiveCount = cfs.getBloomFilterFalsePositiveCount();
+                long truePositiveCount = cfs.getBloomFilterTruePositiveCount();
+                long trueNegativeCount = cfs.getBloomFilterTrueNegativeCount();
+
+                if (falsePositiveCount == 0L && truePositiveCount == 0L)
+                    return 0d;
+                return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount);
+            }
+        }, new Gauge<Double>() // global gauge
+        {
+            public Double getValue()
+            {
+                long falsePositiveCount = 0L;
+                long truePositiveCount = 0L;
+                long trueNegativeCount = 0L;
+                for (Keyspace keyspace : Keyspace.all())
+                {
+                    for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+                    {
+                        falsePositiveCount += cfs.getBloomFilterFalsePositiveCount();
+                        truePositiveCount += cfs.getBloomFilterTruePositiveCount();
+                        trueNegativeCount += cfs.getBloomFilterTrueNegativeCount();
+                    }
+                }
+                if (falsePositiveCount == 0L && truePositiveCount == 0L)
+                    return 0d;
+                return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount);
+            }
+        });
         compressionMetadataOffHeapMemoryUsed = createTableGauge("CompressionMetadataOffHeapMemoryUsed", new Gauge<Long>()
         {
             public Long getValue()
@@ -770,12 +1020,15 @@ public Long getValue()
         additionalWrites = createTableCounter("AdditionalWrites");
         additionalWriteLatencyNanos = createTableGauge("AdditionalWriteLatencyNanos", () -> MICROSECONDS.toNanos(cfs.additionalWriteLatencyMicros));
 
-        tombstoneScannedHistogram = createTableHistogram("TombstoneScannedHistogram", cfs.keyspace.metric.tombstoneScannedHistogram, false);
-        liveScannedHistogram = createTableHistogram("LiveScannedHistogram", cfs.keyspace.metric.liveScannedHistogram, false);
-        colUpdateTimeDeltaHistogram = createTableHistogram("ColUpdateTimeDeltaHistogram", cfs.keyspace.metric.colUpdateTimeDeltaHistogram, false);
-        coordinatorReadLatency = createTableTimer("CoordinatorReadLatency");
-        coordinatorScanLatency = createTableTimer("CoordinatorScanLatency");
-        coordinatorWriteLatency = createTableTimer("CoordinatorWriteLatency");
+        tombstoneScannedHistogram = createTableHistogram("TombstoneScannedHistogram", cfs.getKeyspaceMetrics().tombstoneScannedHistogram, false);
+        shadowedKeysScannedHistogram = createTableHistogram("ShadowedKeysScannedHistogram", cfs.getKeyspaceMetrics().shadowedKeysScannedHistogram, false);
+        shadowedKeysLoopsHistogram = createTableHistogram("ShadowedKeysLoopsHistogram", cfs.getKeyspaceMetrics().shadowedKeysLoopsHistogram, false);
+        tombstoneScannedCounter = createTableCounter("TombstoneScannedCounter");
+        liveScannedHistogram = createTableHistogram("LiveScannedHistogram", cfs.getKeyspaceMetrics().liveScannedHistogram, false);
+        colUpdateTimeDeltaHistogram = createTableHistogram("ColUpdateTimeDeltaHistogram", cfs.getKeyspaceMetrics().colUpdateTimeDeltaHistogram, false);
+        coordinatorReadLatency = createTableTimer("CoordinatorReadLatency", cfs.getKeyspaceMetrics().coordinatorReadLatency);
+        coordinatorScanLatency = createTableTimer("CoordinatorScanLatency", cfs.getKeyspaceMetrics().coordinatorScanLatency);
+        coordinatorWriteLatency = createTableTimer("CoordinatorWriteLatency", cfs.getKeyspaceMetrics().coordinatorWriteLatency);
 
         // We do not want to capture view mutation specific metrics for a view
         // They only makes sense to capture on the base table
@@ -786,8 +1039,8 @@ public Long getValue()
         }
         else
         {
-            viewLockAcquireTime = createTableTimer("ViewLockAcquireTime", cfs.keyspace.metric.viewLockAcquireTime);
-            viewReadTime = createTableTimer("ViewReadTime", cfs.keyspace.metric.viewReadTime);
+            viewLockAcquireTime = createTableTimer("ViewLockAcquireTime", cfs.getKeyspaceMetrics().viewLockAcquireTime);
+            viewReadTime = createTableTimer("ViewReadTime", cfs.getKeyspaceMetrics().viewReadTime);
         }
 
         trueSnapshotsSize = createTableGauge("SnapshotsSize", cfs::trueSnapshotsSize);
@@ -798,19 +1051,19 @@ public Long getValue()
         tombstoneFailures = createTableCounter("TombstoneFailures");
         tombstoneWarnings = createTableCounter("TombstoneWarnings");
 
-        casPrepare = createLatencyMetrics("CasPrepare", cfs.keyspace.metric.casPrepare);
-        casPropose = createLatencyMetrics("CasPropose", cfs.keyspace.metric.casPropose);
-        casCommit = createLatencyMetrics("CasCommit", cfs.keyspace.metric.casCommit);
+        casPrepare = createLatencyMetrics("CasPrepare", cfs.getKeyspaceMetrics().casPrepare, Optional.empty());
+        casPropose = createLatencyMetrics("CasPropose", cfs.getKeyspaceMetrics().casPropose, Optional.empty());
+        casCommit = createLatencyMetrics("CasCommit", cfs.getKeyspaceMetrics().casCommit, Optional.empty());
 
         repairsStarted = createTableCounter("RepairJobsStarted");
         repairsCompleted = createTableCounter("RepairJobsCompleted");
 
-        anticompactionTime = createTableTimer("AnticompactionTime", cfs.keyspace.metric.anticompactionTime);
-        validationTime = createTableTimer("ValidationTime", cfs.keyspace.metric.validationTime);
-        repairSyncTime = createTableTimer("RepairSyncTime", cfs.keyspace.metric.repairSyncTime);
+        anticompactionTime = createTableTimer("AnticompactionTime", cfs.getKeyspaceMetrics().anticompactionTime);
+        validationTime = createTableTimer("ValidationTime", cfs.getKeyspaceMetrics().validationTime);
+        repairSyncTime = createTableTimer("RepairSyncTime", cfs.getKeyspaceMetrics().repairSyncTime);
 
-        bytesValidated = createTableHistogram("BytesValidated", cfs.keyspace.metric.bytesValidated, false);
-        partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false);
+        bytesValidated = createTableHistogram("BytesValidated", cfs.getKeyspaceMetrics().bytesValidated, false);
+        partitionsValidated = createTableHistogram("PartitionsValidated", cfs.getKeyspaceMetrics().partitionsValidated, false);
         bytesAnticompacted = createTableCounter("BytesAnticompacted");
         bytesMutatedAnticompaction = createTableCounter("BytesMutatedAnticompaction");
         mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () ->
@@ -827,11 +1080,11 @@ public Long getValue()
         replicaFilteringProtectionRequests = createTableMeter("ReplicaFilteringProtectionRequests");
         rfpRowsCachedPerQuery = createHistogram("ReplicaFilteringProtectionRowsCachedPerQuery", true);
 
-        confirmedRepairedInconsistencies = createTableMeter("RepairedDataInconsistenciesConfirmed", cfs.keyspace.metric.confirmedRepairedInconsistencies);
-        unconfirmedRepairedInconsistencies = createTableMeter("RepairedDataInconsistenciesUnconfirmed", cfs.keyspace.metric.unconfirmedRepairedInconsistencies);
+        confirmedRepairedInconsistencies = createTableMeter("RepairedDataInconsistenciesConfirmed", cfs.getKeyspaceMetrics().confirmedRepairedInconsistencies);
+        unconfirmedRepairedInconsistencies = createTableMeter("RepairedDataInconsistenciesUnconfirmed", cfs.getKeyspaceMetrics().unconfirmedRepairedInconsistencies);
 
-        repairedDataTrackingOverreadRows = createTableHistogram("RepairedDataTrackingOverreadRows", cfs.keyspace.metric.repairedDataTrackingOverreadRows, false);
-        repairedDataTrackingOverreadTime = createTableTimer("RepairedDataTrackingOverreadTime", cfs.keyspace.metric.repairedDataTrackingOverreadTime);
+        repairedDataTrackingOverreadRows = createTableHistogram("RepairedDataTrackingOverreadRows", cfs.getKeyspaceMetrics().repairedDataTrackingOverreadRows, false);
+        repairedDataTrackingOverreadTime = createTableTimer("RepairedDataTrackingOverreadTime", cfs.getKeyspaceMetrics().repairedDataTrackingOverreadTime);
 
         unleveledSSTables = createTableGauge("UnleveledSSTables", cfs::getUnleveledSSTables, () -> {
             // global gauge
@@ -843,27 +1096,32 @@ public Long getValue()
             return cnt;
         });
 
-        clientTombstoneWarnings = createTableMeter("ClientTombstoneWarnings", cfs.keyspace.metric.clientTombstoneWarnings);
-        clientTombstoneAborts = createTableMeter("ClientTombstoneAborts", cfs.keyspace.metric.clientTombstoneAborts);
+        clientTombstoneWarnings = createTableMeter("ClientTombstoneWarnings", cfs.getKeyspaceMetrics().clientTombstoneWarnings);
+        clientTombstoneAborts = createTableMeter("ClientTombstoneAborts", cfs.getKeyspaceMetrics().clientTombstoneAborts);
 
-        coordinatorReadSizeWarnings = createTableMeter("CoordinatorReadSizeWarnings", cfs.keyspace.metric.coordinatorReadSizeWarnings);
-        coordinatorReadSizeAborts = createTableMeter("CoordinatorReadSizeAborts", cfs.keyspace.metric.coordinatorReadSizeAborts);
-        coordinatorReadSize = createTableHistogram("CoordinatorReadSize", cfs.keyspace.metric.coordinatorReadSize, false);
+        coordinatorReadSizeWarnings = createTableMeter("CoordinatorReadSizeWarnings", cfs.getKeyspaceMetrics().coordinatorReadSizeWarnings);
+        coordinatorReadSizeAborts = createTableMeter("CoordinatorReadSizeAborts", cfs.getKeyspaceMetrics().coordinatorReadSizeAborts);
+        coordinatorReadSize = createTableHistogram("CoordinatorReadSize", cfs.getKeyspaceMetrics().coordinatorReadSize, false);
 
-        localReadSizeWarnings = createTableMeter("LocalReadSizeWarnings", cfs.keyspace.metric.localReadSizeWarnings);
-        localReadSizeAborts = createTableMeter("LocalReadSizeAborts", cfs.keyspace.metric.localReadSizeAborts);
-        localReadSize = createTableHistogram("LocalReadSize", cfs.keyspace.metric.localReadSize, false);
+        localReadSizeWarnings = createTableMeter("LocalReadSizeWarnings", cfs.getKeyspaceMetrics().localReadSizeWarnings);
+        localReadSizeAborts = createTableMeter("LocalReadSizeAborts", cfs.getKeyspaceMetrics().localReadSizeAborts);
+        localReadSize = createTableHistogram("LocalReadSize", cfs.getKeyspaceMetrics().localReadSize, false);
 
-        rowIndexSizeWarnings = createTableMeter("RowIndexSizeWarnings", cfs.keyspace.metric.rowIndexSizeWarnings);
-        rowIndexSizeAborts = createTableMeter("RowIndexSizeAborts", cfs.keyspace.metric.rowIndexSizeAborts);
-        rowIndexSize = createTableHistogram("RowIndexSize", cfs.keyspace.metric.rowIndexSize, false);
+        rowIndexSizeWarnings = createTableMeter("RowIndexSizeWarnings", cfs.getKeyspaceMetrics().rowIndexSizeWarnings);
+        rowIndexSizeAborts = createTableMeter("RowIndexSizeAborts", cfs.getKeyspaceMetrics().rowIndexSizeAborts);
+        rowIndexSize = createTableHistogram("RowIndexSize", cfs.getKeyspaceMetrics().rowIndexSize, false);
 
-        tooManySSTableIndexesReadWarnings = createTableMeter("TooManySSTableIndexesReadWarnings", cfs.keyspace.metric.tooManySSTableIndexesReadWarnings);
-        tooManySSTableIndexesReadAborts = createTableMeter("TooManySSTableIndexesReadAborts", cfs.keyspace.metric.tooManySSTableIndexesReadAborts);
+        tooManySSTableIndexesReadWarnings = createTableMeter("TooManySSTableIndexesReadWarnings", cfs.getKeyspaceMetrics().tooManySSTableIndexesReadWarnings);
+        tooManySSTableIndexesReadAborts = createTableMeter("TooManySSTableIndexesReadAborts", cfs.getKeyspaceMetrics().tooManySSTableIndexesReadAborts);
 
         formatSpecificGauges = createFormatSpecificGauges(cfs);
     }
 
+    public MovingAverage flushSizeOnDisk()
+    {
+        return flushSizeOnDisk;
+    }
+
     private Memtable.MemoryUsage getMemoryUsageWithIndexes(ColumnFamilyStore cfs)
     {
         Memtable.MemoryUsage usage = Memtable.newMemoryUsage();
@@ -873,9 +1131,78 @@ private Memtable.MemoryUsage getMemoryUsageWithIndexes(ColumnFamilyStore cfs)
         return usage;
     }
 
-    public void updateSSTableIterated(int count)
+    public void incLiveRows(long liveRows)
+    {
+        liveScannedHistogram.update(liveRows);
+    }
+
+    public void incShadowedKeys(long numLoops, long numShadowedKeys)
+    {
+        shadowedKeysLoopsHistogram.update(numLoops);
+        shadowedKeysScannedHistogram.update(numShadowedKeys);
+    }
+
+    public void incTombstones(long tombstones, boolean triggerWarning)
+    {
+        tombstoneScannedHistogram.update(tombstones);
+        tombstoneScannedCounter.inc(tombstones);
+
+        if (triggerWarning)
+            tombstoneWarnings.inc();
+    }
+
+    public void incBytesFlushed(long inputSize, long outputSize, long elapsedNanos)
+    {
+        bytesFlushed.inc(outputSize);
+        flushSize.update(outputSize);
+        // this assumes that at least 1 Kb was flushed, which should always be the case, then rounds down
+        flushTimePerKb.update(elapsedNanos / (double) Math.max(1, inputSize / 1024L));
+    }
+
+    public void updateStorageAttachedIndexBuildTime(long totalTimeSpentNanos)
+    {
+        storageAttachedIndexBuildTime.inc(TimeUnit.NANOSECONDS.toMicros(totalTimeSpentNanos));
+    }
+
+    public void updateStorageAttachedIndexWritingTime(long totalTimeSpentNanos, OperationType opType)
+    {
+        long totalTimeSpentMicros = TimeUnit.NANOSECONDS.toMicros(totalTimeSpentNanos);
+        switch (opType)
+        {
+            case INDEX_BUILD:
+                storageAttachedIndexWritingTimeForIndexBuild.inc(totalTimeSpentMicros);
+                break;
+            case COMPACTION:
+                storageAttachedIndexWritingTimeForCompaction.inc(totalTimeSpentMicros);
+                break;
+            case FLUSH:
+                storageAttachedIndexWritingTimeForFlush.inc(totalTimeSpentMicros);
+                break;
+            default:
+                storageAttachedIndexWritingTimeForOther.inc(totalTimeSpentMicros);
+        }
+    }
+
+    public void memTableFlushCompleted(long totalTimeSpentNanos) {
+        flushTime.inc(TimeUnit.NANOSECONDS.toMicros(totalTimeSpentNanos));
+    }
+
+    public void incBytesCompacted(long inputDiskSize, long outputDiskSize, long elapsedMillis)
+    {
+        compactionBytesRead.inc(inputDiskSize);
+        compactionBytesWritten.inc(outputDiskSize);
+        compactionTime.inc(TimeUnit.MILLISECONDS.toMicros(elapsedMillis));
+        // only update compactionTimePerKb when there are non-expired sstables (inputDiskSize > 0)
+        if (inputDiskSize > 0)
+            compactionTimePerKb.update(1024.0 * elapsedMillis / inputDiskSize);
+    }
+
+    public void updateSSTableIterated(int count, int intersectingCount, long elapsedNanos)
     {
         sstablesPerReadHistogram.update(count);
+
+        if (intersectingCount > 0)
+            sstablePartitionReadLatency.update(elapsedNanos / (double) intersectingCount);
     }
 
     public void updateSSTableIteratedInRangeRead(int count)
@@ -984,17 +1311,17 @@ protected Counter createTableCounter(final String name, final String alias)
             Metrics.register(GLOBAL_FACTORY.createMetricName(name),
                              GLOBAL_ALIAS_FACTORY.createMetricName(alias),
                              new Gauge<Long>()
-            {
-                public Long getValue()
-                {
-                    long total = 0;
-                    for (Metric cfGauge : ALL_TABLE_METRICS.get(name))
-                    {
-                        total += ((Counter) cfGauge).getCount();
-                    }
-                    return total;
-                }
-            });
+                             {
+                                 public Long getValue()
+                                 {
+                                     long total = 0;
+                                     for (Metric cfGauge : ALL_TABLE_METRICS.get(name))
+                                     {
+                                         total += ((Counter) cfGauge).getCount();
+                                     }
+                                     return total;
+                                 }
+                             });
         }
         return cfCounter;
     }
@@ -1055,13 +1382,22 @@ protected TableHistogram createTableHistogram(String name, Histogram keyspaceHis
 
     protected TableHistogram createTableHistogram(String name, String alias, Histogram keyspaceHistogram, boolean considerZeroes)
     {
-        Histogram cfHistogram = Metrics.histogram(factory.createMetricName(name), aliasFactory.createMetricName(alias), considerZeroes);
-        register(name, alias, cfHistogram);
-        return new TableHistogram(cfHistogram,
-                                  keyspaceHistogram,
-                                  Metrics.histogram(GLOBAL_FACTORY.createMetricName(name),
-                                                    GLOBAL_ALIAS_FACTORY.createMetricName(alias),
-                                                    considerZeroes));
+        Histogram globalHistogram = null;
+        if (EXPORT_GLOBAL_METRICS)
+        {
+            globalHistogram = Metrics.histogram(GLOBAL_FACTORY.createMetricName(name),
+                                                GLOBAL_ALIAS_FACTORY.createMetricName(alias),
+                                                considerZeroes);
+        }
+
+        Histogram tableHistogram = null;
+        if (metricsAggregation == MetricsAggregation.INDIVIDUAL)
+        {
+            tableHistogram = Metrics.histogram(factory.createMetricName(name), aliasFactory.createMetricName(alias), considerZeroes);
+            register(name, alias, tableHistogram);
+        }
+
+        return new TableHistogram(tableHistogram, keyspaceHistogram, globalHistogram);
     }
 
     protected Histogram createTableHistogram(String name, boolean considerZeroes)
@@ -1078,11 +1414,20 @@ protected Histogram createTableHistogram(String name, String alias, boolean cons
 
     protected TableTimer createTableTimer(String name, Timer keyspaceTimer)
     {
-        Timer cfTimer = Metrics.timer(factory.createMetricName(name), aliasFactory.createMetricName(name));
-        register(name, name, keyspaceTimer);
-        Timer global = Metrics.timer(GLOBAL_FACTORY.createMetricName(name), GLOBAL_ALIAS_FACTORY.createMetricName(name));
+        Timer globalTimer = null;
+        if (EXPORT_GLOBAL_METRICS)
+        {
+            globalTimer = Metrics.timer(GLOBAL_FACTORY.createMetricName(name), GLOBAL_ALIAS_FACTORY.createMetricName(name));
+        }
+
+        Timer tableTimer = null;
+        if (metricsAggregation == MetricsAggregation.INDIVIDUAL)
+        {
+            tableTimer = Metrics.timer(factory.createMetricName(name), aliasFactory.createMetricName(name));
+            register(name, name, keyspaceTimer);
+        }
 
-        return new TableTimer(cfTimer, keyspaceTimer, global);
+        return new TableTimer(tableTimer, keyspaceTimer, globalTimer);
     }
 
     protected SnapshottingTimer createTableTimer(String name)
@@ -1099,18 +1444,38 @@ protected TableMeter createTableMeter(String name, Meter keyspaceMeter)
 
     protected TableMeter createTableMeter(String name, String alias, Meter keyspaceMeter)
     {
-        Meter meter = Metrics.meter(factory.createMetricName(name), aliasFactory.createMetricName(alias));
-        register(name, alias, meter);
-        return new TableMeter(meter,
-                              keyspaceMeter,
-                              Metrics.meter(GLOBAL_FACTORY.createMetricName(name),
-                                            GLOBAL_ALIAS_FACTORY.createMetricName(alias)));
+        Meter globalMeter = null;
+        if (EXPORT_GLOBAL_METRICS)
+        {
+            globalMeter = Metrics.meter(GLOBAL_FACTORY.createMetricName(name),
+                                        GLOBAL_ALIAS_FACTORY.createMetricName(alias));
+        }
+
+        Meter tableMeter = null;
+        if (metricsAggregation == MetricsAggregation.INDIVIDUAL)
+        {
+            tableMeter = Metrics.meter(factory.createMetricName(name), aliasFactory.createMetricName(alias));
+            register(name, alias, tableMeter);
+        }
+
+        return new TableMeter(tableMeter, keyspaceMeter, globalMeter);
     }
 
-    private LatencyMetrics createLatencyMetrics(String namePrefix, LatencyMetrics ... parents)
+    private TableLatencyMetrics createLatencyMetrics(String namePrefix, LatencyMetrics keyspace, Optional<LatencyMetrics> global)
     {
-        LatencyMetrics metric = new LatencyMetrics(factory, namePrefix, parents);
-        all.add(metric::release);
+        TableLatencyMetrics metric;
+        if (metricsAggregation == MetricsAggregation.INDIVIDUAL)
+        {
+            LatencyMetrics[] parents = Stream.of(Optional.of(keyspace), global).filter(Optional::isPresent)
+                                             .map(Optional::get).toArray(LatencyMetrics[]::new);
+            LatencyMetrics innerMetrics = new LatencyMetrics(factory, namePrefix, parents);
+            metric = new TableLatencyMetrics.IndividualTableLatencyMetrics(innerMetrics);
+        }
+        else
+        {
+            metric = new TableLatencyMetrics.AggregatingTableLatencyMetrics(keyspace, global);
+        }
+        all.add(metric);
         return metric;
     }
 
@@ -1162,17 +1527,105 @@ private void releaseMetric(String tableMetricName, String cfMetricName, String t
         }
     }
 
+    public interface TableLatencyMetrics extends ReleasableMetric
+    {
+        void addNano(long latencyNanos);
+
+        LatencyMetrics tableOrKeyspaceMetric();
+
+        /**
+         * Used when {@link MetricsAggregation#AGGREGATED} is set for this table.
+         * <br/>
+         * Table latency metrics that forwards all calls to the first parent metric (keyspace metric by convention).
+         * Thanks to the forwarding, the table doesn't have to maintain its own metrics. The metrics for this table
+         * are aggregated with metrics comming from other tables that use {@link MetricsAggregation#AGGREGATED}.
+         */
+        class AggregatingTableLatencyMetrics implements TableLatencyMetrics
+        {
+            private final LatencyMetrics keyspace;
+            private final Optional<LatencyMetrics> global;
+
+            public AggregatingTableLatencyMetrics(LatencyMetrics keyspace, Optional<LatencyMetrics> global)
+            {
+                this.keyspace = keyspace;
+                this.global = global;
+                Preconditions.checkState(keyspace != null, "Keyspace metrics should not be null");
+            }
+
+            @Override
+            public void addNano(long latencyNanos)
+            {
+                keyspace.addNano(latencyNanos);
+                global.ifPresent(g -> g.addNano(latencyNanos));
+            }
+
+            @Override
+            public LatencyMetrics tableOrKeyspaceMetric()
+            {
+                return keyspace;
+            }
+
+            @Override
+            public void release()
+            {
+                // noop
+            }
+        }
+
+        /**
+         * Used when {@link MetricsAggregation#INDIVIDUAL} is set for this table.
+         * <br/>
+         * Table latency metrics that don't aggreagte, i.e. the given table maintains its own latency metrics.
+         */
+        class IndividualTableLatencyMetrics implements TableLatencyMetrics
+        {
+            private final LatencyMetrics latencyMetrics;
+
+            public IndividualTableLatencyMetrics(LatencyMetrics latencyMetrics)
+            {
+                this.latencyMetrics = latencyMetrics;
+            }
+
+            @Override
+            public void addNano(long latencyNanos)
+            {
+                latencyMetrics.addNano(latencyNanos);
+            }
+
+            @Override
+            public LatencyMetrics tableOrKeyspaceMetric()
+            {
+                return latencyMetrics;
+            }
+
+            @Override
+            public void release()
+            {
+                latencyMetrics.release();
+            }
+        }
+    }
+
     public static class TableMeter
     {
         public final Meter[] all;
-        public final Meter table;
-        public final Meter global;
-
-        private TableMeter(Meter table, Meter keyspace, Meter global)
+        @Nullable
+        private final Meter table;
+        private final Meter keyspace;
+
+        /**
+         * Table meter wrapper that forwards updates to all provided non-null meters.
+         *
+         * @param table meter that is {@code null} if the metrics are not collected indidually for each table, see {@link TableMetrics#metricsAggregation}.
+         * @param keyspace meter
+         * @param global meter that is {@code null} if global metrics are not collected, see {@link TableMetrics#EXPORT_GLOBAL_METRICS}
+         */
+        private TableMeter(@Nullable Meter table, Meter keyspace, @Nullable Meter global)
         {
+            Preconditions.checkState(keyspace != null, "Keyspace meter can't be null");
             this.table = table;
-            this.global = global;
-            this.all = new Meter[]{table, keyspace, global};
+            this.keyspace = keyspace;
+            this.all = Stream.of(table, keyspace, global).filter(Objects::nonNull).toArray(Meter[]::new);
         }
 
         public void mark()
@@ -1182,46 +1635,74 @@ public void mark()
                 meter.mark();
             }
         }
+
+        public Meter tableOrKeyspaceMeter()
+        {
+            return table == null ? keyspace : table;
+        }
     }
 
     public static class TableHistogram
     {
-        public final Histogram[] all;
-        public final Histogram cf;
-        public final Histogram global;
-
-        private TableHistogram(Histogram cf, Histogram keyspace, Histogram global)
+        private final Histogram[] all;
+        @Nullable
+        private final Histogram table;
+        private final Histogram keyspace;
+
+        /**
+         * Table histogram wrapper that forwards updates to all provided non-null histograms.
+         *
+         * @param table histogram that is {@code null} if the metrics are not collected indidually for each table, see {@link TableMetrics#metricsAggregation}.
+         * @param keyspace histogram
+         * @param global histogram that is {@code null} if global metrics are not collected, see {@link TableMetrics#EXPORT_GLOBAL_METRICS}
+         */
+        private TableHistogram(@Nullable Histogram table, Histogram keyspace, @Nullable Histogram global)
         {
-            this.cf = cf;
-            this.global = global;
-            this.all = new Histogram[]{cf, keyspace, global};
+            Preconditions.checkState(keyspace != null, "Keyspace histogram can't be null");
+            this.table = table;
+            this.keyspace = keyspace;
+            this.all = Stream.of(table, keyspace, global).filter(Objects::nonNull).toArray(Histogram[]::new);
         }
 
         public void update(long i)
         {
-            for(Histogram histo : all)
+            for (Histogram histo : all)
             {
                 histo.update(i);
             }
         }
+
+        public Histogram tableOrKeyspaceHistogram()
+        {
+            return table == null ? keyspace : table;
+        }
     }
 
-    public static class TableTimer
+    public static class TableTimer<T extends Timer>
     {
-        public final Timer[] all;
-        public final Timer cf;
-        public final Timer global;
-
-        private TableTimer(Timer cf, Timer keyspace, Timer global)
+        private final Timer[] all;
+        @Nullable
+        private final T cf;
+        private final T keyspace;
+
+        /**
+         * Table timer wrapper that forwards updates to all provided non-null timers.
+         *
+         * @param cf timer that is {@code null} if the metrics are not collected indidually for each table, see {@link TableMetrics#metricsAggregation}.
+         * @param keyspace timer
+         * @param global timer that is {@code null} if global metrics are not collected, see {@link TableMetrics#EXPORT_GLOBAL_METRICS}
+         */
+        private TableTimer(@Nullable T cf, T keyspace, @Nullable T global)
         {
+            Preconditions.checkState(keyspace != null, "Keyspace timer can't be null");
             this.cf = cf;
-            this.global = global;
-            this.all = new Timer[]{cf, keyspace, global};
+            this.keyspace = keyspace;
+            this.all = Stream.of(cf, keyspace, global).filter(Objects::nonNull).map(t -> (Timer) t).toArray(Timer[]::new);
         }
 
         public void update(long i, TimeUnit unit)
         {
-            for(Timer timer : all)
+            for (Timer timer : all)
             {
                 timer.update(i, unit);
             }
@@ -1232,6 +1713,11 @@ public Context time()
             return new Context(all);
         }
 
+        public T tableOrKeyspaceTimer()
+        {
+            return cf == null ? keyspace : cf;
+        }
+
         public static class Context implements AutoCloseable
         {
             private final long start;
@@ -1262,7 +1748,7 @@ static class TableMetricNameFactory implements MetricNameFactory
         TableMetricNameFactory(ColumnFamilyStore cfs, String type)
         {
             this.keyspaceName = cfs.getKeyspaceName();
-            this.tableName = cfs.name;
+            this.tableName = cfs.getTableName();
             this.isIndex = cfs.isIndex();
             this.type = type;
         }
diff --git a/src/java/org/apache/cassandra/metrics/TrieMemtableMetricsView.java b/src/java/org/apache/cassandra/metrics/TrieMemtableMetricsView.java
index 934350399945..d0e49ff5eeea 100644
--- a/src/java/org/apache/cassandra/metrics/TrieMemtableMetricsView.java
+++ b/src/java/org/apache/cassandra/metrics/TrieMemtableMetricsView.java
@@ -18,6 +18,9 @@
 
 package org.apache.cassandra.metrics;
 
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
 import com.codahale.metrics.Counter;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -29,6 +32,8 @@ public class TrieMemtableMetricsView
     private static final String CONTENTION_TIME = "Contention time";
     private static final String LAST_FLUSH_SHARD_SIZES = "Shard sizes during last flush";
 
+    private static final Map<String, TrieMemtableMetricsView> perTableMetrics = new ConcurrentHashMap<>();
+
     // the number of memtable puts that did not need to wait on write lock
     public final Counter uncontendedPuts;
 
@@ -42,9 +47,18 @@ public class TrieMemtableMetricsView
     public final MinMaxAvgMetric lastFlushShardDataSizes;
 
     private final TrieMemtableMetricNameFactory factory;
+    private final String keyspace;
+    private final String table;
+
+    public static TrieMemtableMetricsView getOrCreate(String keyspace, String table)
+    {
+        return perTableMetrics.computeIfAbsent(getKey(keyspace, table), k -> new TrieMemtableMetricsView(keyspace, table));
+    }
 
-    public TrieMemtableMetricsView(String keyspace, String table)
+    private TrieMemtableMetricsView(String keyspace, String table)
     {
+        this.keyspace = keyspace;
+        this.table = table;
         factory = new TrieMemtableMetricNameFactory(keyspace, table);
         
         uncontendedPuts = Metrics.counter(factory.createMetricName(UNCONTENDED_PUTS));
@@ -55,6 +69,8 @@ public TrieMemtableMetricsView(String keyspace, String table)
 
     public void release()
     {
+        perTableMetrics.remove(getKey(keyspace, table));
+
         Metrics.remove(factory.createMetricName(UNCONTENDED_PUTS));
         Metrics.remove(factory.createMetricName(CONTENDED_PUTS));
         contentionTime.release();
@@ -87,4 +103,9 @@ public CassandraMetricsRegistry.MetricName createMetricName(String metricName)
             return new CassandraMetricsRegistry.MetricName(groupName, type, metricName, keyspace + "." + table, mbeanName.toString());
         }
     }
+
+    private static String getKey(String keyspace, String table)
+    {
+        return keyspace + "." + table;
+    }
 }
diff --git a/src/java/org/apache/cassandra/metrics/ViewWriteMetrics.java b/src/java/org/apache/cassandra/metrics/ViewWriteMetrics.java
index 98363d413db4..360c359bf4e0 100644
--- a/src/java/org/apache/cassandra/metrics/ViewWriteMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ViewWriteMetrics.java
@@ -30,14 +30,15 @@ public class ViewWriteMetrics extends ClientRequestMetrics
     public final Counter viewReplicasSuccess;
     // time between when mutation is applied to local memtable to when CL.ONE is achieved on MV
     public final Timer viewWriteLatency;
+    public final Gauge<Long> viewPendingMutations;
 
-    public ViewWriteMetrics(String scope)
+    public ViewWriteMetrics(String scope, String namePrefix)
     {
-        super(scope);
-        viewReplicasAttempted = Metrics.counter(factory.createMetricName("ViewReplicasAttempted"));
-        viewReplicasSuccess = Metrics.counter(factory.createMetricName("ViewReplicasSuccess"));
-        viewWriteLatency = Metrics.timer(factory.createMetricName("ViewWriteLatency"));
-        Metrics.register(factory.createMetricName("ViewPendingMutations"), new Gauge<Long>()
+        super(scope, namePrefix);
+        viewReplicasAttempted = Metrics.counter(factory.createMetricName(namePrefix + "ViewReplicasAttempted"));
+        viewReplicasSuccess = Metrics.counter(factory.createMetricName(namePrefix + "ViewReplicasSuccess"));
+        viewWriteLatency = Metrics.timer(factory.createMetricName(namePrefix + "ViewWriteLatency"));
+        viewPendingMutations = Metrics.register(factory.createMetricName(namePrefix + "ViewPendingMutations"), new Gauge<Long>()
                 {
                     public Long getValue()
                     {
@@ -49,9 +50,9 @@ public Long getValue()
     public void release()
     {
         super.release();
-        Metrics.remove(factory.createMetricName("ViewReplicasAttempted"));
-        Metrics.remove(factory.createMetricName("ViewReplicasSuccess"));
-        Metrics.remove(factory.createMetricName("ViewWriteLatency"));
-        Metrics.remove(factory.createMetricName("ViewPendingMutations"));
+        Metrics.remove(factory.createMetricName(namePrefix + "ViewReplicasAttempted"));
+        Metrics.remove(factory.createMetricName(namePrefix + "ViewReplicasSuccess"));
+        Metrics.remove(factory.createMetricName(namePrefix + "ViewWriteLatency"));
+        Metrics.remove(factory.createMetricName(namePrefix + "ViewPendingMutations"));
     }
 }
diff --git a/src/java/org/apache/cassandra/net/AbstractMessageHandler.java b/src/java/org/apache/cassandra/net/AbstractMessageHandler.java
index e2cf68d6d1ee..bf00172a0e6e 100644
--- a/src/java/org/apache/cassandra/net/AbstractMessageHandler.java
+++ b/src/java/org/apache/cassandra/net/AbstractMessageHandler.java
@@ -33,7 +33,6 @@
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
 import io.netty.channel.EventLoop;
-import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.net.FrameDecoder.CorruptFrame;
 import org.apache.cassandra.net.FrameDecoder.Frame;
 import org.apache.cassandra.net.FrameDecoder.FrameProcessor;
@@ -219,6 +218,11 @@ public boolean process(Frame frame) throws IOException
         return true;
     }
 
+    /**
+     * React to the decoder being reactivated
+     */
+    protected abstract void onDecoderReactivated();
+
     private boolean processIntactFrame(IntactFrame frame, Limit endpointReserve, Limit globalReserve) throws IOException
     {
         if (frame.isSelfContained)
@@ -311,7 +315,7 @@ private void onReserveCapacityRegained(Limit endpointReserve, Limit globalReserv
                 decoder.reactivate();
 
                 if (decoder.isActive())
-                    ClientMetrics.instance.unpauseConnection();
+                    onDecoderReactivated();
             }
         }
         catch (Throwable t)
diff --git a/src/java/org/apache/cassandra/net/AsyncStreamingOutputPlus.java b/src/java/org/apache/cassandra/net/AsyncStreamingOutputPlus.java
index 915e8a31b604..fd7ba305ea01 100644
--- a/src/java/org/apache/cassandra/net/AsyncStreamingOutputPlus.java
+++ b/src/java/org/apache/cassandra/net/AsyncStreamingOutputPlus.java
@@ -33,6 +33,7 @@
 import io.netty.channel.FileRegion;
 import io.netty.channel.WriteBufferWaterMark;
 import io.netty.handler.ssl.SslHandler;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.net.SharedDefaultFileRegion.SharedFileChannel;
@@ -156,8 +157,8 @@ class Holder
      */
     public long writeFileToChannel(FileChannel file, RateLimiter limiter) throws IOException
     {
-        if (channel.pipeline().get(SslHandler.class) != null)
-            // each batch is loaded into ByteBuffer, 64KiB is more BufferPool friendly.
+        if (channel.pipeline().get(SslHandler.class) != null || !DatabaseDescriptor.nettyZerocopyEnabled())
+            // each batch is loaded into ByteBuffer, 64kb is more BufferPool friendly.
             return writeFileToChannel(file, limiter, 1 << 16);
         else
             // write files in 1MiB chunks, since there may be blocking work performed to fetch it from disk,
@@ -170,17 +171,17 @@ long writeFileToChannel(FileChannel fc, RateLimiter limiter, int batchSize) thro
     {
         final long length = fc.size();
         long bytesTransferred = 0;
+        assert fc.position() == 0;
 
         try
         {
             while (bytesTransferred < length)
             {
                 int toWrite = (int) min(batchSize, length - bytesTransferred);
-                final long position = bytesTransferred;
 
                 writeToChannel(bufferSupplier -> {
                     ByteBuffer outBuffer = bufferSupplier.get(toWrite);
-                    long read = fc.read(outBuffer, position);
+                    long read = fc.read(outBuffer);
                     if (read != toWrite)
                         throw new IOException(String.format("could not read required number of bytes from " +
                                                             "file to be streamed: read %d bytes, wanted %d bytes",
diff --git a/src/java/org/apache/cassandra/net/CustomResponseVerbHandlerProvider.java b/src/java/org/apache/cassandra/net/CustomResponseVerbHandlerProvider.java
new file mode 100644
index 000000000000..9f016b1a600c
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/CustomResponseVerbHandlerProvider.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.function.Supplier;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Priovides a response handler for response messages ({@link org.apache.cassandra.net.Verb#REQUEST_RSP} and
+ * {@link org.apache.cassandra.net.Verb#FAILURE_RSP}).
+ * Defaults to {@link ResponseVerbHandler#instance}.
+ */
+public interface CustomResponseVerbHandlerProvider extends Supplier<IVerbHandler<?>>
+{
+    CustomResponseVerbHandlerProvider instance = CassandraRelevantProperties.CUSTOM_RESPONSE_VERB_HANDLER_PROVIDER.getString() == null ?
+                                                 () -> ResponseVerbHandler.instance :
+                                                 FBUtilities.construct(CassandraRelevantProperties.CUSTOM_RESPONSE_VERB_HANDLER_PROVIDER.getString(), "custom response verb handler");
+
+    IVerbHandler<?> get();
+
+}
diff --git a/src/java/org/apache/cassandra/net/EndpointMessagingVersions.java b/src/java/org/apache/cassandra/net/EndpointMessagingVersions.java
index dceffc7cc1e3..b5d217086d2b 100644
--- a/src/java/org/apache/cassandra/net/EndpointMessagingVersions.java
+++ b/src/java/org/apache/cassandra/net/EndpointMessagingVersions.java
@@ -39,6 +39,15 @@ public class EndpointMessagingVersions
     // protocol versions of the other nodes in the cluster
     private final ConcurrentMap<InetAddressAndPort, Integer> versions = new NonBlockingHashMap<>();
 
+    public EndpointMessagingVersions()
+    {
+    }
+
+    private EndpointMessagingVersions(EndpointMessagingVersions versions)
+    {
+        this.versions.putAll(versions.versions);
+    }
+
     /**
      * @return the last version associated with address, or @param version if this is the first such version
      */
@@ -69,7 +78,7 @@ public int get(InetAddressAndPort endpoint)
         if (v == null)
         {
             // we don't know the version. assume current. we'll know soon enough if that was incorrect.
-            logger.trace("Assuming current protocol version for {}", endpoint);
+            logger.debug("Assuming current protocol version for {}", endpoint);
             return MessagingService.current_version;
         }
         else
@@ -96,4 +105,9 @@ public boolean knows(InetAddressAndPort endpoint)
     {
         return versions.containsKey(endpoint);
     }
+
+    public EndpointMessagingVersions copy()
+    {
+        return new EndpointMessagingVersions(this);
+    }
 }
diff --git a/src/java/org/apache/cassandra/net/InboundMessageCallbacks.java b/src/java/org/apache/cassandra/net/InboundMessageCallbacks.java
index ffa4243b9d10..9984fe27c78f 100644
--- a/src/java/org/apache/cassandra/net/InboundMessageCallbacks.java
+++ b/src/java/org/apache/cassandra/net/InboundMessageCallbacks.java
@@ -94,6 +94,13 @@ interface InboundMessageCallbacks
 
     /**
      * Invoked at the very end of execution of the message-processing task, no matter the outcome of processing.
+     * timeElapsed is the duration on message processing in the relevant stage
      */
     void onExecuted(int messageSize, Header header, long timeElapsed, TimeUnit unit);
+
+    /**
+     * Invoked at the very end of execution of the message-processing task, no matter the outcome of processing.
+     * timeElapsed is the duration of the whole messaging processing, including deserialization, stage queue wait time
+     */
+    void onMessageHandlingCompleted(Header header, long timeElapsed, TimeUnit unit);
 }
diff --git a/src/java/org/apache/cassandra/net/InboundMessageHandler.java b/src/java/org/apache/cassandra/net/InboundMessageHandler.java
index 50a42e7b718b..307a084014ba 100644
--- a/src/java/org/apache/cassandra/net/InboundMessageHandler.java
+++ b/src/java/org/apache/cassandra/net/InboundMessageHandler.java
@@ -75,7 +75,7 @@ public class InboundMessageHandler extends AbstractMessageHandler
     private final ConnectionType type;
     private final InetAddressAndPort self;
     private final InetAddressAndPort peer;
-    private final int version;
+    final int version;
 
     private final InboundMessageCallbacks callbacks;
     private final Consumer<Message<?>> consumer;
@@ -118,6 +118,12 @@ public class InboundMessageHandler extends AbstractMessageHandler
         this.consumer = consumer;
     }
 
+    @Override
+    protected void onDecoderReactivated()
+    {
+        // No-op for this implementation, as the InboundMessageHandler should not use ClientMetrics
+    }
+
     protected boolean processOneContainedMessage(ShareableBytes bytes, Limit endpointReserve, Limit globalReserve) throws IOException
     {
         ByteBuffer buf = bytes.get();
@@ -146,14 +152,14 @@ protected boolean processOneContainedMessage(ShareableBytes bytes, Limit endpoin
         receivedBytes += size;
 
         if (size <= largeThreshold)
-            processSmallMessage(bytes, size, header);
+            processSmallMessage(bytes, size, header, currentTimeNanos);
         else
-            processLargeMessage(bytes, size, header);
+            processLargeMessage(bytes, size, header, currentTimeNanos);
 
         return true;
     }
 
-    private void processSmallMessage(ShareableBytes bytes, int size, Header header)
+    private void processSmallMessage(ShareableBytes bytes, int size, Header header, long messageProcessingStartTimeNanos)
     {
         ByteBuffer buf = bytes.get();
         final int begin = buf.position();
@@ -191,13 +197,13 @@ private void processSmallMessage(ShareableBytes bytes, int size, Header header)
         }
 
         if (null != message)
-            dispatch(new ProcessSmallMessage(message, size));
+            dispatch(new ProcessSmallMessage(message, size, messageProcessingStartTimeNanos));
     }
 
     // for various reasons, it's possible for a large message to be contained in a single frame
-    private void processLargeMessage(ShareableBytes bytes, int size, Header header)
+    private void processLargeMessage(ShareableBytes bytes, int size, Header header, long handlingStartNanos)
     {
-        new LargeMessage(size, header, bytes.sliceAndConsume(size).share()).schedule();
+        new LargeMessage(size, header, bytes.sliceAndConsume(size).share(), handlingStartNanos).schedule();
     }
 
     /*
@@ -219,7 +225,7 @@ protected boolean processFirstFrameOfLargeMessage(IntactFrame frame, Limit endpo
 
         callbacks.onHeaderArrived(size, header, currentTimeNanos - header.createdAtNanos, NANOSECONDS);
         receivedBytes += buf.remaining();
-        largeMessage = new LargeMessage(size, header, expired);
+        largeMessage = new LargeMessage(size, header, expired, currentTimeNanos);
         largeMessage.supply(frame);
         return true;
     }
@@ -314,19 +320,23 @@ protected void fatalExceptionCaught(Throwable cause)
      */
     private class LargeMessage extends AbstractMessageHandler.LargeMessage<Header>
     {
-        private LargeMessage(int size, Header header, boolean isExpired)
+        private long handlingStartNanos;
+
+        private LargeMessage(int size, Header header, boolean isExpired, long handlingStartNanos)
         {
             super(size, header, header.expiresAtNanos, isExpired);
+            this.handlingStartNanos = handlingStartNanos;
         }
 
-        private LargeMessage(int size, Header header, ShareableBytes bytes)
+        private LargeMessage(int size, Header header, ShareableBytes bytes, long handlingStartNanos)
         {
             super(size, header, header.expiresAtNanos, bytes);
+            this.handlingStartNanos = handlingStartNanos;
         }
 
         private void schedule()
         {
-            dispatch(new ProcessLargeMessage(this));
+            dispatch(new ProcessLargeMessage(this, handlingStartNanos));
         }
 
         protected void onComplete()
@@ -396,7 +406,7 @@ private void dispatch(ProcessMessage task)
         if (state != null) state.trace("{} message received from {}", header.verb, header.from);
 
         callbacks.onDispatched(task.size(), header);
-        header.verb.stage.execute(ExecutorLocals.create(state), task);
+        header.verb.stage.execute(task, ExecutorLocals.create(state));
     }
 
     private abstract class ProcessMessage implements Runnable
@@ -441,10 +451,13 @@ public void run()
 
                 releaseResources();
 
-                callbacks.onExecuted(size(), header, approxTime.now() - approxStartTimeNanos, NANOSECONDS);
+                long now = approxTime.now();
+                callbacks.onExecuted(size(), header, now - approxStartTimeNanos, NANOSECONDS);
+                callbacks.onMessageHandlingCompleted(header, now - handlingStartNanos(), NANOSECONDS);
             }
         }
 
+        abstract long handlingStartNanos();
         abstract int size();
         abstract Header header();
         abstract Message provideMessage();
@@ -455,11 +468,19 @@ private class ProcessSmallMessage extends ProcessMessage
     {
         private final int size;
         private final Message message;
+        private final long handlingStartNanos;
 
-        ProcessSmallMessage(Message message, int size)
+        ProcessSmallMessage(Message message, int size, long handlingStartNanos)
         {
             this.size = size;
             this.message = message;
+            this.handlingStartNanos = handlingStartNanos;
+        }
+
+        @Override
+        long handlingStartNanos()
+        {
+            return handlingStartNanos;
         }
 
         int size()
@@ -481,10 +502,18 @@ Message provideMessage()
     private class ProcessLargeMessage extends ProcessMessage
     {
         private final LargeMessage message;
+        private final long handlingStartNanos;
 
-        ProcessLargeMessage(LargeMessage message)
+        ProcessLargeMessage(LargeMessage message, long handlingStartNanos)
         {
             this.message = message;
+            this.handlingStartNanos = handlingStartNanos;
+        }
+
+        @Override
+        long handlingStartNanos()
+        {
+            return handlingStartNanos;
         }
 
         int size()
diff --git a/src/java/org/apache/cassandra/net/InboundMessageHandlers.java b/src/java/org/apache/cassandra/net/InboundMessageHandlers.java
index c7b946350d09..2b176a113054 100644
--- a/src/java/org/apache/cassandra/net/InboundMessageHandlers.java
+++ b/src/java/org/apache/cassandra/net/InboundMessageHandlers.java
@@ -31,9 +31,6 @@
 import org.apache.cassandra.metrics.InternodeInboundMetrics;
 import org.apache.cassandra.net.Message.Header;
 
-import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
-
 /**
  * An aggregation of {@link InboundMessageHandler}s for all connections from a peer.
  *
@@ -88,8 +85,10 @@ public interface MessageConsumer extends Consumer<Message<?>>
     public interface GlobalMetricCallbacks
     {
         LatencyConsumer internodeLatencyRecorder(InetAddressAndPort to);
-        void recordInternalLatency(Verb verb, long timeElapsed, TimeUnit timeUnit);
+        void recordInternalLatency(Verb verb, InetAddressAndPort from, long timeElapsed, TimeUnit timeUnit);
         void recordInternodeDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit);
+        void recordMessageStageProcessingTime(Verb verb, InetAddressAndPort from, long timeElapsed, TimeUnit unit);
+        void recordTotalMessageProcessingTime(Verb verb, InetAddressAndPort from, long timeElapsed, TimeUnit unit);
     }
 
     public InboundMessageHandlers(InetAddressAndPort self,
@@ -201,9 +200,9 @@ private static InboundMessageCallbacks makeMessageCallbacks(InetAddressAndPort p
             @Override
             public void onHeaderArrived(int messageSize, Header header, long timeElapsed, TimeUnit unit)
             {
-                // do not log latency if we are within error bars of zero
-                if (timeElapsed > unit.convert(approxTime.error(), NANOSECONDS))
-                    internodeLatency.accept(timeElapsed, unit);
+                // log latency even if we are within error bars of zero
+                // log negative numbers too; we are interested in the distribution, not precise values
+                internodeLatency.accept(header.verb, timeElapsed, unit);
             }
 
             @Override
@@ -264,13 +263,20 @@ public void onDispatched(int messageSize, Header header)
             @Override
             public void onExecuting(int messageSize, Header header, long timeElapsed, TimeUnit unit)
             {
-                globalMetrics.recordInternalLatency(header.verb, timeElapsed, unit);
+                globalMetrics.recordInternalLatency(header.verb, header.from, timeElapsed, unit);
             }
 
             @Override
             public void onExecuted(int messageSize, Header header, long timeElapsed, TimeUnit unit)
             {
                 counters.removePending(messageSize);
+                globalMetrics.recordMessageStageProcessingTime(header.verb, header.from, timeElapsed, unit);
+            }
+
+            @Override
+            public void onMessageHandlingCompleted(Header header, long timeElapsed, TimeUnit unit)
+            {
+                globalMetrics.recordTotalMessageProcessingTime(header.verb, header.from, timeElapsed, unit);
             }
 
             @Override
@@ -431,6 +437,13 @@ private long sumCounters(ToLongFunction<InboundCounters> mapping)
              + mapping.applyAsLong(legacyCounters);
     }
 
+    @VisibleForTesting
+    public void assertHandlersMessagingVersion(int expectedVersion)
+    {
+        for (InboundMessageHandler handler : handlers)
+            assert handler.version == expectedVersion : "Expected all handlers to be at version " + expectedVersion + " but found " + handler.version;
+    }
+
     interface HandlerProvider
     {
         InboundMessageHandler provide(FrameDecoder decoder,
diff --git a/src/java/org/apache/cassandra/net/InboundSink.java b/src/java/org/apache/cassandra/net/InboundSink.java
index 9d68ba1aa078..78e549ffcc62 100644
--- a/src/java/org/apache/cassandra/net/InboundSink.java
+++ b/src/java/org/apache/cassandra/net/InboundSink.java
@@ -22,6 +22,8 @@
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import java.util.function.Predicate;
 
+import org.apache.cassandra.index.IndexBuildInProgressException;
+import org.apache.cassandra.index.sai.utils.AbortedOperationException;
 import org.slf4j.LoggerFactory;
 
 import net.openhft.chronicle.core.util.ThrowingConsumer;
@@ -100,12 +102,24 @@ public void accept(Message<?> message)
         {
             fail(message.header, t);
 
-            if (t instanceof TombstoneOverwhelmingException || t instanceof IndexNotAvailableException)
+            // The site throwing AbortedOperationException is responsible for logging it.
+            if (t instanceof AbortedOperationException)
+                return;
+
+            if (t instanceof TombstoneOverwhelmingException || 
+                t instanceof IndexNotAvailableException || 
+                t instanceof IndexBuildInProgressException)
+            {
                 noSpamLogger.error(t.getMessage());
+            }
             else if (t instanceof RuntimeException)
+            {
                 throw (RuntimeException) t;
+            }
             else
+            {
                 throw new RuntimeException(t);
+            }
         }
     }
 
diff --git a/src/java/org/apache/cassandra/net/LatencyConsumer.java b/src/java/org/apache/cassandra/net/LatencyConsumer.java
index 3f10d4146a13..cc466d78af95 100644
--- a/src/java/org/apache/cassandra/net/LatencyConsumer.java
+++ b/src/java/org/apache/cassandra/net/LatencyConsumer.java
@@ -21,5 +21,5 @@
 
 public interface LatencyConsumer
 {
-    void accept(long timeElapsed, TimeUnit unit);
+    void accept(Verb verb, long timeElapsed, TimeUnit unit);
 }
diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java
index 705061562a27..7a0766cda51c 100644
--- a/src/java/org/apache/cassandra/net/Message.java
+++ b/src/java/org/apache/cassandra/net/Message.java
@@ -26,13 +26,13 @@
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.primitives.Ints;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
@@ -54,9 +54,16 @@
 import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
 import static org.apache.cassandra.net.MessagingService.VERSION_40;
 import static org.apache.cassandra.net.MessagingService.VERSION_50;
+import static org.apache.cassandra.net.MessagingService.VERSION_DSE_68;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_10;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_11;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_20;
 import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
 import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
-import static org.apache.cassandra.utils.vint.VIntCoding.*;
+import static org.apache.cassandra.utils.vint.VIntCoding.computeUnsignedVIntSize;
+import static org.apache.cassandra.utils.vint.VIntCoding.getUnsignedVInt;
+import static org.apache.cassandra.utils.vint.VIntCoding.getUnsignedVInt32;
+import static org.apache.cassandra.utils.vint.VIntCoding.skipUnsignedVInt;
 
 /**
  * Immutable main unit of internode communication - what used to be {@code MessageIn} and {@code MessageOut} fused
@@ -204,19 +211,19 @@ public static <T> Message<T> synthetic(InetAddressAndPort from, Verb verb, T pay
 
     public static <T> Message<T> out(Verb verb, T payload, long expiresAtNanos)
     {
-        return outWithParam(nextId(), verb, expiresAtNanos, payload, 0, null, null);
+        return outWithParam(nextId(), verb, expiresAtNanos, payload, 0, null, null).build();
     }
 
     public static <T> Message<T> outWithFlag(Verb verb, T payload, MessageFlag flag)
     {
         assert !verb.isResponse();
-        return outWithParam(nextId(), verb, 0, payload, flag.addTo(0), null, null);
+        return outWithParam(nextId(), verb, 0, payload, flag.addTo(0), null, null).build();
     }
 
     public static <T> Message<T> outWithFlags(Verb verb, T payload, MessageFlag flag1, MessageFlag flag2)
     {
         assert !verb.isResponse();
-        return outWithParam(nextId(), verb, 0, payload, flag2.addTo(flag1.addTo(0)), null, null);
+        return outWithParam(nextId(), verb, 0, payload, flag2.addTo(flag1.addTo(0)), null, null).build();
     }
 
     public static <T> Message<T> outWithFlags(Verb verb, T payload, Dispatcher.RequestTime requestTime, List<MessageFlag> flags)
@@ -239,20 +246,20 @@ public static <T> Message<T> outWithFlags(Verb verb, T payload, Dispatcher.Reque
     @VisibleForTesting
     static <T> Message<T> outWithParam(long id, Verb verb, T payload, ParamType paramType, Object paramValue)
     {
-        return outWithParam(id, verb, 0, payload, paramType, paramValue);
+        return outWithParam(id, verb, 0, payload, paramType, paramValue).build();
     }
 
-    private static <T> Message<T> outWithParam(long id, Verb verb, long expiresAtNanos, T payload, ParamType paramType, Object paramValue)
+    private static <T> Builder<T> outWithParam(long id, Verb verb, long expiresAtNanos, T payload, ParamType paramType, Object paramValue)
     {
         return outWithParam(id, verb, expiresAtNanos, payload, 0, paramType, paramValue);
     }
 
-    private static <T> Message<T> outWithParam(long id, Verb verb, long expiresAtNanos, T payload, int flags, ParamType paramType, Object paramValue)
+    private static <T> Builder<T> outWithParam(long id, Verb verb, long expiresAtNanos, T payload, int flags, ParamType paramType, Object paramValue)
     {
         return withParam(getBroadcastAddressAndPort(), id, verb, expiresAtNanos, payload, flags, paramType, paramValue);
     }
 
-    private static <T> Message<T> withParam(InetAddressAndPort from, long id, Verb verb, long expiresAtNanos, T payload, int flags, ParamType paramType, Object paramValue)
+    private static <T> Builder<T> withParam(InetAddressAndPort from, long id, Verb verb, long expiresAtNanos, T payload, int flags, ParamType paramType, Object paramValue)
     {
         if (payload == null)
             throw new IllegalArgumentException();
@@ -260,8 +267,14 @@ private static <T> Message<T> withParam(InetAddressAndPort from, long id, Verb v
         long createdAtNanos = approxTime.now();
         if (expiresAtNanos == 0)
             expiresAtNanos = verb.expiresAtNanos(createdAtNanos);
-
-        return new Message<>(new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, buildParams(paramType, paramValue)), payload);
+        return new Builder<T>().ofVerb(verb)
+                               .withPayload(payload)
+                               .from(from)
+                               .withId(id)
+                               .withExpiresAt(expiresAtNanos)
+                               .withCreatedAt(createdAtNanos)
+                               .withFlags(flags)
+                               .withParams(buildParams(paramType, paramValue));
     }
 
     public static <T> Message<T> internalResponse(Verb verb, T payload)
@@ -274,16 +287,22 @@ public static <T> Message<T> internalResponse(Verb verb, T payload)
      * Used by the {@code MultiRangeReadCommand} to split multi-range responses from a replica
      * into single-range responses.
      */
-    public static <T> Message<T> remoteResponse(InetAddressAndPort from, Verb verb, T payload)
+    public static <T> Message<T> remoteResponse(InetAddressAndPort from, Verb verb, Map<ParamType, Object> params, T payload)
     {
         assert verb.isResponse();
         long createdAtNanos = approxTime.now();
         long expiresAtNanos = verb.expiresAtNanos(createdAtNanos);
-        return new Message<>(new Header(0, verb, from, createdAtNanos, expiresAtNanos, 0, NO_PARAMS), payload);
+        return new Message<>(new Header(0, verb, from, createdAtNanos, expiresAtNanos, 0, params), payload);
     }
 
     /** Builds a response Message with provided payload, and all the right fields inferred from request Message */
     public <T> Message<T> responseWith(T payload)
+    {
+        return outWithParam(id(), verb().responseVerb, expiresAtNanos(), payload, null, null).build();
+    }
+
+    /** Builds a response Message builder with provided payload, and all the right fields inferred from request Message */
+    public <T> Builder<T> responseWithBuilder(T payload)
     {
         return outWithParam(id(), verb().responseVerb, expiresAtNanos(), payload, null, null);
     }
@@ -294,6 +313,12 @@ public Message<NoPayload> emptyResponse()
         return responseWith(NoPayload.noPayload);
     }
 
+    /** Builds a response Builder with no payload, to allow for adding custom params if needed */
+    public Builder<NoPayload> emptyResponseBuilder()
+    {
+        return responseWithBuilder(NoPayload.noPayload);
+    }
+
     /** Builds a failure response Message with an explicit reason, and fields inferred from request Message */
     public Message<RequestFailureReason> failureResponse(RequestFailureReason reason)
     {
@@ -302,7 +327,7 @@ public Message<RequestFailureReason> failureResponse(RequestFailureReason reason
 
     static Message<RequestFailureReason> failureResponse(long id, long expiresAtNanos, RequestFailureReason reason)
     {
-        return outWithParam(id, Verb.FAILURE_RSP, expiresAtNanos, reason, null, null);
+        return outWithParam(id, Verb.FAILURE_RSP, expiresAtNanos, reason, null, null).build();
     }
 
     public <V> Message<V> withPayload(V newPayload)
@@ -342,7 +367,7 @@ public Message<T> withParams(Map<ParamType, Object> values)
         return new Message<>(header.withParams(values), payload);
     }
 
-    private static final EnumMap<ParamType, Object> NO_PARAMS = new EnumMap<>(ParamType.class);
+    public static final EnumMap<ParamType, Object> NO_PARAMS = new EnumMap<>(ParamType.class);
 
     private static Map<ParamType, Object> buildParams(ParamType type, Object value)
     {
@@ -388,7 +413,7 @@ private static Map<ParamType, Object> addParams(Map<ParamType, Object> params, M
 
     private static final AtomicInteger nextId = new AtomicInteger(0);
 
-    private static long nextId()
+    public static long nextId()
     {
         long id;
         do
@@ -527,7 +552,21 @@ public Map<ParamType, Object> params()
         @Nullable
         public Map<String,byte[]> customParams()
         {
-            return (Map<String,byte[]>) params.get(ParamType.CUSTOM_MAP);
+            return (Map<String, byte[]>) params.get(ParamType.CUSTOM_MAP);
+        }
+
+        public int flags()
+        {
+            return flags;
+        }
+
+        /**
+         * Keyspace that is beeing traced by the trace session attached to this message (if any).
+         */
+        @Nullable
+        public String traceKeyspace()
+        {
+            return (String) params.get(ParamType.TRACE_KEYSPACE);
         }
     }
 
@@ -545,6 +584,8 @@ public static class Builder<T>
 
         private boolean hasId;
 
+        private Message cachedMessage;
+
         private Builder()
         {
         }
@@ -652,7 +693,31 @@ public Message<T> build()
             if (payload == null)
                 throw new IllegalArgumentException();
 
-            return new Message<>(new Header(hasId ? id : nextId(), verb, from, createdAtNanos, expiresAtNanos, flags, params), payload);
+            return doBuild(hasId ? id : nextId());
+        }
+
+        public int currentPayloadSize(int version)
+        {
+            // use dummy id just for the sake of computing the serialized size
+            Message<T> tmp = doBuild(0);
+            cachedMessage = tmp;
+            return tmp.payloadSize(version);
+        }
+
+        private Message<T> doBuild(long id)
+        {
+            if (verb == null)
+                throw new IllegalArgumentException();
+            if (from == null)
+                throw new IllegalArgumentException();
+            if (payload == null)
+                throw new IllegalArgumentException();
+
+            Message<T> tmp = new Message<>(new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, params), payload);
+            if (cachedMessage != null)
+                tmp.maybeCachePayloadSize(cachedMessage);
+
+            return tmp;
         }
     }
 
@@ -1098,6 +1163,10 @@ private <T> int payloadSize(Message<T> message, int version)
 
     private int serializedSize40;
     private int serializedSize50;
+    private int serializedSizeDS10;
+    private int serializedSizeDS11;
+    private int serializedSizeDS20;
+    private int serializedSizeDSE68;
 
     /**
      * Serialized size of the entire message, for the provided messaging version. Caches the calculated value.
@@ -1114,15 +1183,35 @@ public int serializedSize(int version)
                 if (serializedSize50 == 0)
                     serializedSize50 = serializer.serializedSize(this, VERSION_50);
                 return serializedSize50;
+            case VERSION_DS_10:
+                if (serializedSizeDS10 == 0)
+                    serializedSizeDS10 = serializer.serializedSize(this, VERSION_DS_10);
+                return serializedSizeDS10;
+            case VERSION_DS_11:
+                if (serializedSizeDS11 == 0)
+                    serializedSizeDS11 = serializer.serializedSize(this, VERSION_DS_11);
+                return serializedSizeDS11;
+            case VERSION_DS_20:
+                if (serializedSizeDS20 == 0)
+                    serializedSizeDS20 = (int) serializer.serializedSize(this, VERSION_DS_20);
+                return serializedSizeDS20;
+            case VERSION_DSE_68:
+                if (serializedSizeDSE68 == 0)
+                    serializedSizeDSE68 = serializer.serializedSize(this, VERSION_DSE_68);
+                return serializedSizeDSE68;
             default:
                 throw new IllegalStateException("Unkown serialization version " + version);
         }
     }
 
-    private int payloadSize40   = -1;
-    private int payloadSize50   = -1;
+    private int payloadSize40    = -1;
+    private int payloadSize50    = -1;
+    private int payloadSizeDS10 = -1;
+    private int payloadSizeDS11 = -1;
+    private int payloadSizeDS20 = -1;
+    private int payloadSizeDSE68 = -1;
 
-    private int payloadSize(int version)
+    public int payloadSize(int version)
     {
         switch (version)
         {
@@ -1134,12 +1223,44 @@ private int payloadSize(int version)
                 if (payloadSize50 < 0)
                     payloadSize50 = serializer.payloadSize(this, VERSION_50);
                 return payloadSize50;
-
+            case VERSION_DS_10:
+                if (payloadSizeDS10 < 0)
+                    payloadSizeDS10 = serializer.payloadSize(this, VERSION_DS_10);
+                return payloadSizeDS10;
+            case VERSION_DS_11:
+                if (payloadSizeDS11 < 0)
+                    payloadSizeDS11 = serializer.payloadSize(this, VERSION_DS_11);
+                return payloadSizeDS11;
+            case VERSION_DS_20:
+                if (payloadSizeDS20 < 0)
+                    payloadSizeDS20 = serializer.payloadSize(this, VERSION_DS_20);
+                return payloadSizeDS20;
+            case VERSION_DSE_68:
+                if (payloadSizeDSE68 < 0)
+                    payloadSizeDSE68 = serializer.payloadSize(this, VERSION_DSE_68);
+                return payloadSizeDSE68;
             default:
                 throw new IllegalStateException("Unkown serialization version " + version);
         }
     }
 
+    protected void maybeCachePayloadSize(Message other)
+    {
+        if (payload == other.payload)
+        {
+            if (other.payloadSize40 > 0)
+                payloadSize40 = other.payloadSize40;
+            if (other.payloadSize50 > 0)
+                payloadSize50 = other.payloadSize50;
+            if (other.payloadSizeDS10 > 0)
+                payloadSizeDS10 = other.payloadSizeDS10;
+            if (other.payloadSizeDS20 > 0)
+                payloadSizeDS20 = other.payloadSizeDS20;
+            if (other.payloadSizeDSE68 > 0)
+                payloadSizeDSE68 = other.payloadSizeDSE68;
+        }
+    }
+
     static class OversizedMessageException extends RuntimeException
     {
         OversizedMessageException(int size)
diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java
index 94586b41c850..cd499ce48c6a 100644
--- a/src/java/org/apache/cassandra/net/MessagingService.java
+++ b/src/java/org/apache/cassandra/net/MessagingService.java
@@ -22,8 +22,10 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -37,6 +39,7 @@
 import io.netty.util.concurrent.Future; //checkstyle: permit this import
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.exceptions.RequestFailureReason;
@@ -44,6 +47,7 @@
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.metrics.MessagingMetrics;
 import org.apache.cassandra.service.AbstractWriteResponseHandler;
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.ExecutorUtils;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
@@ -52,7 +56,7 @@
 import static java.util.Collections.synchronizedList;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static org.apache.cassandra.concurrent.Stage.MUTATION;
-import static org.apache.cassandra.config.CassandraRelevantProperties.NON_GRACEFUL_SHUTDOWN;
+import static org.apache.cassandra.config.CassandraRelevantProperties.*;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.Throwables.maybeFail;
 
@@ -213,19 +217,28 @@ public enum Version
     {
         /** @deprecated See CASSANDRA-18314 */
         @Deprecated(since = "5.0")
-        VERSION_30(10),
+        VERSION_30(MessagingService.VERSION_30, false),
         /** @deprecated See CASSANDRA-18314 */
         @Deprecated(since = "5.0")
-        VERSION_3014(11),
-        VERSION_40(12),
+        VERSION_3014(MessagingService.VERSION_3014, false),
+        VERSION_40(MessagingService.VERSION_40, false),
         // c14227 TTL overflow, 'uint' timestamps
-        VERSION_50(13);
+        VERSION_50(MessagingService.VERSION_50, true),
+        VERSION_DS_10(MessagingService.VERSION_DS_10, false), // DS Converged Cassandra 4.0
+        VERSION_DS_11(MessagingService.VERSION_DS_11, false),
+        VERSION_DS_20(MessagingService.VERSION_DS_20, true), // DS Converged Cassandra 5.0
+        VERSION_DSE_68(MessagingService.VERSION_DSE_68, false), // DSE 6.8
+        ;
+
+        public static final Version CURRENT = VERSION_DS_20; // TODO - we should consider what should be there - also there is CASSANDRA-19126 which changes the logic here
 
         public final int value;
+        public final boolean supportsExtendedDeletionTime;
 
-        Version(int value)
+        Version(int value, boolean extendedDeletionTime)
         {
             this.value = value;
+            this.supportsExtendedDeletionTime = extendedDeletionTime;
         }
 
         public static List<Version> supportedVersions()
@@ -237,6 +250,11 @@ public static List<Version> supportedVersions()
 
             return Collections.unmodifiableList(versions);
         }
+
+        public static boolean supportsExtendedDeletionTime(int value)
+        {
+            return  Version.values()[versionOrdinalMap.get(value)].supportsExtendedDeletionTime;
+        }
     }
     // Maintance Note:
     // Try to keep Version enum in-sync for testing.  By having the versions in the enum tests can get access without forcing this class
@@ -249,12 +267,19 @@ public static List<Version> supportedVersions()
     public static final int VERSION_3014 = 11;
     public static final int VERSION_40 = 12;
     public static final int VERSION_50 = 13; // c14227 TTL overflow, 'uint' timestamps
+    public static final int VERSION_DS_10 = 100; // DS Converged Cassandra 4.0
+    // Current DataStax version while we have serialization differences.
+    // If differences get merged upstream then we can revert to OS versioning.
+    public static final int VERSION_DS_11 = 101; // adds ann_options (CNDB-12456)
+    public static final int VERSION_DS_20 = 110; // DS Converged Cassandra 5.0
     public static final int minimum_version = VERSION_40;
-    public static final int maximum_version = VERSION_50;
+    public static final int maximum_version = VERSION_DS_20;
     // we want to use a modified behavior for the tools and clients - that is, since they are not running a server, they
     // should not need to run in a compatibility mode. They should be able to connect to the server regardless whether
     // it uses messaving version 4 or 5
-    public static final int current_version = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50;
+    public static final int current_version = currentVersion();
+    // DSE 6.8 version for backward compatibility
+    public static final int VERSION_DSE_68 = 168;
     static AcceptVersions accept_messaging;
     static AcceptVersions accept_streaming;
     static
@@ -267,10 +292,21 @@ public static List<Version> supportedVersions()
         else
         {
             accept_messaging = new AcceptVersions(minimum_version, current_version);
-            accept_streaming = new AcceptVersions(current_version, current_version);
+            accept_streaming = new AcceptVersions(minimum_version, current_version);
         }
     }
-    static Map<Integer, Integer> versionOrdinalMap = Arrays.stream(Version.values()).collect(Collectors.toMap(v -> v.value, v -> v.ordinal()));
+    static Map<Integer, Integer> versionOrdinalMap = Arrays.stream(Version.values()).collect(Collectors.toMap(v -> v.value, Enum::ordinal));
+
+    private static int currentVersion()
+    {
+        int version = CassandraRelevantProperties.DS_CURRENT_MESSAGING_VERSION.getInt();
+        for (Version v : Version.values())
+        {
+            if (v.value == version)
+                return version;
+        }
+        throw new IllegalArgumentException("Unsupported current messaging version: " + version);
+    }
 
     /**
      * This is an optimisation to speed up the translation of the serialization
@@ -288,6 +324,9 @@ public static int getVersionOrdinal(int version)
         return ordinal;
     }
 
+    public final static boolean GRACEFUL_CLOSE = !NON_GRACEFUL_CLOSE.getBoolean();
+    public final static boolean UNUSED_CONNECTION_MONITORING = !DISABLE_UNUSED_CONNECTION_MONITORING.getBoolean();
+
     private static class MSHandle
     {
         public static final MessagingService instance = new MessagingService(false);
@@ -325,14 +364,18 @@ public static MessagingService instance()
     @VisibleForTesting
     MessagingService(boolean testOnly)
     {
-        this(testOnly, new EndpointMessagingVersions(), new MessagingMetrics());
+        this(testOnly, new EndpointMessagingVersions(),
+             CUSTOM_MESSAGING_METRICS_PROVIDER_PROPERTY.isPresent() ?
+                FBUtilities.construct(CUSTOM_MESSAGING_METRICS_PROVIDER_PROPERTY.getString(), "Messaging Metrics Provider") :
+                new MessagingMetrics());
     }
 
     @VisibleForTesting
     MessagingService(boolean testOnly, EndpointMessagingVersions versions, MessagingMetrics metrics)
     {
         super(testOnly, versions, metrics);
-        OutboundConnections.scheduleUnusedConnectionMonitoring(this, ScheduledExecutors.scheduledTasks, 1L, TimeUnit.HOURS);
+        if (UNUSED_CONNECTION_MONITORING)
+            OutboundConnections.scheduleUnusedConnectionMonitoring(this, ScheduledExecutors.scheduledTasks, 1L, TimeUnit.HOURS);
     }
 
     @Override
@@ -467,7 +510,7 @@ private void doSend(Message message, InetAddressAndPort to, ConnectionType speci
         // expire the callback if the message failed to enqueue (failed to establish a connection or exceeded queue capacity)
         while (true)
         {
-            OutboundConnections connections = getOutbound(to);
+            OutboundConnections connections = getOutbound(to, true);
             try
             {
                 connections.enqueue(message, specifyConnection);
@@ -509,11 +552,19 @@ public void closeOutbound(InetAddressAndPort to)
      */
     void closeOutboundNow(OutboundConnections connections)
     {
-        connections.close(true).addListener(
+        connections.close(GRACEFUL_CLOSE).addListener(
             future -> channelManagers.remove(connections.template().to, connections)
         );
     }
 
+    // Used by CNDB
+    public void closeOutboundNow(InetAddressAndPort to)
+    {
+        OutboundConnections pool = channelManagers.get(to);
+        if (pool != null)
+            closeOutboundNow(pool);
+    }
+
     /**
      * Only to be invoked once we believe the connections will never be used again.
      */
@@ -583,14 +634,14 @@ public void shutdown(long timeout, TimeUnit units, boolean shutdownGracefully, b
         isShuttingDown = true;
         logger.info("Waiting for messaging service to quiesce");
         // We may need to schedule hints on the mutation stage, so it's erroneous to shut down the mutation stage first
-        assert !MUTATION.executor().isShutdown();
+        assert !MUTATION.isShutdown();
 
         if (shutdownGracefully)
         {
             callbacks.shutdownGracefully();
             List<Future<Void>> closing = new ArrayList<>();
             for (OutboundConnections pool : channelManagers.values())
-                closing.add(pool.close(true));
+                closing.add(pool.close(GRACEFUL_CLOSE));
 
             long deadline = nanoTime() + units.toNanos(timeout);
             maybeFail(() -> FutureCombiner.nettySuccessListener(closing).get(timeout, units),
@@ -617,15 +668,25 @@ public void shutdown(long timeout, TimeUnit units, boolean shutdownGracefully, b
                 closing.add(pool.close(false));
 
             long deadline = nanoTime() + units.toNanos(timeout);
-            maybeFail(() -> FutureCombiner.nettySuccessListener(closing).get(timeout, units),
-                      () -> {
-                          if (shutdownExecutors)
-                              shutdownExecutors(deadline);
-                      },
-                      () -> ExecutorUtils.awaitTermination(timeout, units, inboundExecutors),
-                      () -> callbacks.awaitTerminationUntil(deadline),
-                      inboundSink::clear,
-                      outboundSink::clear);
+            try
+            {
+                maybeFail(() -> FutureCombiner.nettySuccessListener(closing).get(timeout, units),
+                          () -> {
+                              if (shutdownExecutors)
+                                  shutdownExecutors(deadline);
+                          },
+                          () -> ExecutorUtils.awaitTermination(timeout, units, inboundExecutors),
+                          () -> callbacks.awaitTerminationUntil(deadline),
+                          inboundSink::clear,
+                          outboundSink::clear);
+            }
+            catch (Throwable t)
+            {
+                if (NON_GRACEFUL_SHUTDOWN.getBoolean())
+                    logger.info("Timeout when waiting for messaging service shutdown", t);
+                else
+                    throw t;
+            }
         }
     }
 
@@ -641,7 +702,7 @@ public void shutdownAbrubtly()
         isShuttingDown = true;
         logger.info("Waiting for messaging service to quiesce");
         // We may need to schedule hints on the mutation stage, so it's erroneous to shut down the mutation stage first
-        assert !MUTATION.executor().isShutdown();
+        assert !MUTATION.isShutdown();
 
         callbacks.shutdownNow(false);
         inboundSockets.close();
@@ -659,10 +720,10 @@ private void shutdownExecutors(long deadlineNanos) throws TimeoutException, Inte
         socketFactory.awaitTerminationUntil(deadlineNanos);
     }
 
-    private OutboundConnections getOutbound(InetAddressAndPort to)
+    private OutboundConnections getOutbound(InetAddressAndPort to, boolean tryRegister)
     {
         OutboundConnections connections = channelManagers.get(to);
-        if (connections == null)
+        if (connections == null && tryRegister)
             connections = OutboundConnections.tryRegister(channelManagers, to, new OutboundConnectionSettings(to).withDefaults(ConnectionCategory.MESSAGING));
         return connections;
     }
@@ -700,4 +761,47 @@ public void waitUntilListening() throws InterruptedException
     {
         inboundSockets.open().await();
     }
+
+    /**
+     * Returns the endpoints for the given keyspace that are known to be alive and are using a messaging version older
+     * than the given version.
+     *
+     * @param keyspace a keyspace
+     * @param version a messaging version
+     * @return a set of alive endpoints in the given keyspace with messaging version below the given version
+     */
+    public Set<InetAddressAndPort> endpointsWithVersionBelow(String keyspace, int version)
+    {
+        Set<InetAddressAndPort> nodes = new HashSet<>();
+        for (InetAddressAndPort node : StorageService.instance.getTokenMetadataForKeyspace(keyspace).getAllEndpoints())
+        {
+            if (versions.knows(node) && versions.getRaw(node) < version)
+                nodes.add(node);
+        }
+        return nodes;
+    }
+
+    /**
+     * Returns the endpoints for the given keyspace that are known to be alive and have a connection whose
+     * messaging version is older than the given version. To be used for example when we want to be sure a message
+     * can be serialized to all endpoints, according to their negotiated version at connection time.
+     *
+     * @param keyspace a keyspace
+     * @param version a messaging version
+     * @return a set of alive endpoints in the given keyspace with messaging version below the given version
+     */
+    public Set<InetAddressAndPort> endpointsWithConnectionsOnVersionBelow(String keyspace, int version)
+    {
+        Set<InetAddressAndPort> nodes = new HashSet<>();
+        for (InetAddressAndPort node : StorageService.instance.getTokenMetadataForKeyspace(keyspace).getAllEndpoints())
+        {
+            ConnectionType.MESSAGING_TYPES.forEach(type -> {
+                OutboundConnections connections = getOutbound(node, false);
+                OutboundConnection connection = connections != null ? connections.connectionFor(type) : null;
+                if (connection != null && connection.messagingVersion() < version)
+                    nodes.add(node);
+            });
+        }
+        return nodes;
+    }
 }
diff --git a/src/java/org/apache/cassandra/net/OutboundConnection.java b/src/java/org/apache/cassandra/net/OutboundConnection.java
index cfb9f1ffc03e..16d2026eed8d 100644
--- a/src/java/org/apache/cassandra/net/OutboundConnection.java
+++ b/src/java/org/apache/cassandra/net/OutboundConnection.java
@@ -1120,6 +1120,8 @@ void onCompletedHandshake(Result<MessagingSuccess> result)
                         assert !state.isClosed();
 
                         MessagingSuccess success = result.success();
+                        messagingVersion = success.messagingVersion;
+                        settings.endpointToVersion.set(settings.to, messagingVersion);
                         debug.onConnect(success.messagingVersion, settings);
                         state.disconnected().maintenance.cancel(false);
 
@@ -1461,6 +1463,8 @@ public Future<Void> close(boolean flushQueue)
 
                 try
                 {
+                    logger.debug("Closing connection {}", id());
+
                     // note that we never clear the queue, to ensure that an enqueue has the opportunity to remove itself
                     // if it raced with close, to potentially requeue the message on a replacement connection
 
diff --git a/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java b/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java
index ccc74f0aba4f..62da4ebb2336 100644
--- a/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java
+++ b/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java
@@ -18,6 +18,8 @@
 
 package org.apache.cassandra.net;
 
+import java.util.Objects;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
@@ -348,6 +350,13 @@ public EndpointMessagingVersions endpointToVersion()
 
     public InetAddressAndPort from()
     {
+        InetAddressAndPort from = this.from;
+        InetAddressAndPort preferredLocalAddress = DatabaseDescriptor.getEndpointSnitch() != null
+                                                   ? DatabaseDescriptor.getEndpointSnitch().getPreferredAddress(connectTo())
+                                                   : null;
+        if (!Objects.equals(preferredLocalAddress, from))
+            from = preferredLocalAddress;
+
         return from != null ? from : FBUtilities.getBroadcastAddressAndPort();
     }
 
@@ -476,7 +485,10 @@ public OutboundConnectionSettings withDefaults(ConnectionCategory category)
                                               applicationSendQueueReserveGlobalCapacityInBytes(),
                                               tcpNoDelay(), flushLowWaterMark, flushHighWaterMark,
                                               tcpConnectTimeoutInMS(), tcpUserTimeoutInMS(category), acceptVersions(category),
-                                              from(), socketFactory(), callbacks(), debug(), endpointToVersion());
+                                              from(), socketFactory(), callbacks(), debug(),
+                                              // If a set of versions is passed, make sure we do a copy of it, as the version might be later updated
+                                              // depending on the handshake result (i.e. nodes might handshake a different version)
+                                              endpointToVersion().copy());
     }
 
     private static boolean isInLocalDC(IEndpointSnitch snitch, InetAddressAndPort localHost, InetAddressAndPort remoteHost)
diff --git a/src/java/org/apache/cassandra/net/OutboundConnections.java b/src/java/org/apache/cassandra/net/OutboundConnections.java
index aacc2b44736b..88ec13b726a1 100644
--- a/src/java/org/apache/cassandra/net/OutboundConnections.java
+++ b/src/java/org/apache/cassandra/net/OutboundConnections.java
@@ -36,12 +36,12 @@
 import io.netty.util.concurrent.Future; //checkstyle: permit this import
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.InternodeOutboundMetrics;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static java.lang.Math.max;
 import static org.apache.cassandra.config.CassandraRelevantProperties.OTCP_LARGE_MESSAGE_THRESHOLD;
-import static org.apache.cassandra.gms.Gossiper.instance;
 import static org.apache.cassandra.net.FrameEncoderCrc.HEADER_AND_TRAILER_LENGTH;
 import static org.apache.cassandra.net.MessagingService.current_version;
 import static org.apache.cassandra.net.ConnectionType.URGENT_MESSAGES;
@@ -311,9 +311,9 @@ private void closeUnusedSinceLastRun()
                     continue;
 
                 if (cur.small == prev.small && cur.large == prev.large && cur.urgent == prev.urgent
-                    && !instance.isKnownEndpoint(connections.template.to))
+                    && !Nodes.isKnownEndpoint(connections.template.to))
                 {
-                    logger.info("Closing outbound connections to {}, as inactive and not known by Gossiper",
+                    logger.info("Closing outbound connections to {}, as inactive and not known",
                                 connections.template.to);
                     // close entirely if no traffic and the endpoint is unknown
                     messagingService.closeOutboundNow(connections);
diff --git a/src/java/org/apache/cassandra/net/OutboundSink.java b/src/java/org/apache/cassandra/net/OutboundSink.java
index 34c72dbc3a11..16fea0890312 100644
--- a/src/java/org/apache/cassandra/net/OutboundSink.java
+++ b/src/java/org/apache/cassandra/net/OutboundSink.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.net;
 
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.BiConsumer;
 import java.util.function.BiPredicate;
 
 import org.apache.cassandra.locator.InetAddressAndPort;
@@ -27,7 +28,7 @@
  *
  * Default sink {@link Sink} used by {@link MessagingService} is {@link MessagingService#doSend(Message, InetAddressAndPort, ConnectionType)}, which proceeds to
  * send messages over the network, but it can be overridden to filter out certain messages, record the fact
- * of attempted delivery, or delay they delivery.
+ * of attempted delivery, delay the delivery, or perform some action after delivery occurs.
  *
  * This facility is most useful for test code.
  */
@@ -56,6 +57,24 @@ public void accept(Message<?> message, InetAddressAndPort to, ConnectionType con
         }
     }
 
+    private static class PostSink implements Sink
+    {
+        final BiConsumer<Message<?>, InetAddressAndPort> postSink;
+        final Sink sink;
+
+        private PostSink(BiConsumer<Message<?>, InetAddressAndPort> postSink, Sink sink)
+        {
+            this.postSink = postSink;
+            this.sink = sink;
+        }
+
+        public void accept(Message<?> message, InetAddressAndPort to, ConnectionType connectionType)
+        {
+            sink.accept(message, to, connectionType);
+            postSink.accept(message, to);
+        }
+    }
+
     private volatile Sink sink;
     private static final AtomicReferenceFieldUpdater<OutboundSink, Sink> sinkUpdater
         = AtomicReferenceFieldUpdater.newUpdater(OutboundSink.class, Sink.class, "sink");
@@ -75,6 +94,18 @@ public void add(BiPredicate<Message<?>, InetAddressAndPort> allow)
         sinkUpdater.updateAndGet(this, sink -> new Filtered(allow, sink));
     }
 
+    /**
+     * Add a method that gets called after {@link OutboundSink#accept(Message, InetAddressAndPort, ConnectionType)}.
+     *
+     * <p>This is useful if you want to perform additional work after a message has been sent to the sink.</p>
+     *
+     * @param post the method to call after {@link OutboundSink#accept}.
+     */
+    public void addPost(BiConsumer<Message<?>, InetAddressAndPort> post)
+    {
+        sinkUpdater.updateAndGet(this, sink -> new PostSink(post, sink));
+    }
+
     public void remove(BiPredicate<Message<?>, InetAddressAndPort> allow)
     {
         sinkUpdater.updateAndGet(this, sink -> without(sink, allow));
diff --git a/src/java/org/apache/cassandra/net/ParamType.java b/src/java/org/apache/cassandra/net/ParamType.java
index 77c0f32771ff..356779c5a871 100644
--- a/src/java/org/apache/cassandra/net/ParamType.java
+++ b/src/java/org/apache/cassandra/net/ParamType.java
@@ -25,6 +25,7 @@
 import org.apache.cassandra.utils.Int64Serializer;
 import org.apache.cassandra.utils.RangesSerializer;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.StringSerializer;
 
 import static java.lang.Math.max;
 
@@ -57,7 +58,11 @@ public enum ParamType
     CUSTOM_MAP                       (14, CustomParamsSerializer.serializer),
     SNAPSHOT_RANGES                  (15, RangesSerializer.serializer),
     TOO_MANY_REFERENCED_INDEXES_WARN (16, Int32Serializer.serializer),
-    TOO_MANY_REFERENCED_INDEXES_FAIL (17, Int32Serializer.serializer);
+    TOO_MANY_REFERENCED_INDEXES_FAIL (17, Int32Serializer.serializer),
+    /**
+     * Messages with tracing sessions are decorated with the traced keyspace.
+     */
+    TRACE_KEYSPACE                   (18, StringSerializer.serializer);
 
     final int id;
     final IVersionedSerializer serializer;
diff --git a/src/java/org/apache/cassandra/net/RequestCallback.java b/src/java/org/apache/cassandra/net/RequestCallback.java
index 14e0169b858a..65b99ad18379 100644
--- a/src/java/org/apache/cassandra/net/RequestCallback.java
+++ b/src/java/org/apache/cassandra/net/RequestCallback.java
@@ -18,9 +18,11 @@
 package org.apache.cassandra.net;
 
 import java.util.Map;
+import javax.annotation.Nullable;
 
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.sensors.RequestSensors;
 
 /**
  * implementors of {@link RequestCallback} need to make sure that any public methods
@@ -80,4 +82,13 @@ static boolean isTimeout(Map<InetAddressAndPort, RequestFailureReason> failureRe
         return failureReasonByEndpoint.values().stream().allMatch(RequestFailureReason.TIMEOUT::equals);
     }
 
+    /**
+     * @return the {@link RequestSensors} associated with the request to track sensors as reported by response replicas.
+     * If null, sensor tracking will be disabled for this request.
+     */
+    @Nullable
+    default RequestSensors getRequestSensors()
+    {
+        return null;
+    }
 }
diff --git a/src/java/org/apache/cassandra/net/RequestCallbacks.java b/src/java/org/apache/cassandra/net/RequestCallbacks.java
index ee63c5a3e652..ae9f5d1840c8 100644
--- a/src/java/org/apache/cassandra/net/RequestCallbacks.java
+++ b/src/java/org/apache/cassandra/net/RequestCallbacks.java
@@ -31,11 +31,13 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.IMutation;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.metrics.InternodeOutboundMetrics;
 import org.apache.cassandra.service.AbstractWriteResponseHandler;
+import org.apache.cassandra.service.paxos.Commit;
 
 import static java.lang.String.format;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -63,6 +65,8 @@ public class RequestCallbacks implements OutboundMessageCallbacks
     private final ScheduledExecutorPlus executor = executorFactory().scheduled("Callback-Map-Reaper", DISCARD);
     private final ConcurrentMap<CallbackKey, CallbackInfo> callbacks = new ConcurrentHashMap<>();
 
+    private volatile boolean shutdown;
+
     RequestCallbacks(MessagingService messagingService)
     {
         this.messagingService = messagingService;
@@ -75,7 +79,7 @@ public class RequestCallbacks implements OutboundMessageCallbacks
      * @return the registered {@link CallbackInfo} for this id and peer, or {@code null} if unset or expired.
      */
     @Nullable
-    CallbackInfo get(long id, InetAddressAndPort peer)
+    public CallbackInfo get(long id, InetAddressAndPort peer)
     {
         return callbacks.get(key(id, peer));
     }
@@ -93,10 +97,17 @@ public CallbackInfo remove(long id, InetAddressAndPort peer)
     /**
      * Register the provided {@link RequestCallback}, inferring expiry and id from the provided {@link Message}.
      */
+    @VisibleForTesting
     public void addWithExpiration(RequestCallback<?> cb, Message<?> message, InetAddressAndPort to)
     {
         // mutations need to call the overload
         assert message.verb() != Verb.MUTATION_REQ && message.verb() != Verb.COUNTER_MUTATION_REQ;
+        if (shutdown)
+        {
+            if (logger.isTraceEnabled())
+                logger.trace("Received request after messaging service shutdown so ignoring it");
+            return;
+        }
         CallbackInfo previous = callbacks.put(key(message.id(), to), new CallbackInfo(message, to, cb));
         assert previous == null : format("Callback already exists for id %d/%s! (%s)", message.id(), to, previous);
     }
@@ -104,7 +115,13 @@ public void addWithExpiration(RequestCallback<?> cb, Message<?> message, InetAdd
     public void addWithExpiration(AbstractWriteResponseHandler<?> cb, Message<?> message, Replica to)
     {
         assert message.verb() == Verb.MUTATION_REQ || message.verb() == Verb.COUNTER_MUTATION_REQ || message.verb() == Verb.PAXOS_COMMIT_REQ;
-        CallbackInfo previous = callbacks.put(key(message.id(), to.endpoint()), new CallbackInfo(message, to.endpoint(), cb));
+        if (shutdown)
+        {
+            if (logger.isTraceEnabled())
+                logger.trace("Received request after messaging service shutdown so ignoring it");
+            return;
+        }
+        CallbackInfo previous = callbacks.put(key(message.id(), to.endpoint()), new WriteCallbackInfo(message, to.endpoint(), cb));
         assert previous == null : format("Callback already exists for id %d/%s! (%s)", message.id(), to.endpoint(), previous);
     }
 
@@ -159,6 +176,7 @@ private void onExpired(CallbackInfo info)
 
     void shutdownNow(boolean expireCallbacks)
     {
+        shutdown = true;
         executor.shutdownNow();
         if (expireCallbacks)
             forceExpire();
@@ -166,6 +184,7 @@ void shutdownNow(boolean expireCallbacks)
 
     void shutdownGracefully()
     {
+        shutdown = true;
         expire();
         if (!callbacks.isEmpty())
             executor.schedule(this::shutdownGracefully, 100L, MILLISECONDS);
@@ -236,7 +255,7 @@ public static class CallbackInfo
         final InetAddressAndPort peer;
         public final RequestCallback callback;
 
-        private CallbackInfo(Message message, InetAddressAndPort peer, RequestCallback callback)
+        public CallbackInfo(Message message, InetAddressAndPort peer, RequestCallback callback)
         {
             this.createdAtNanos = message.createdAtNanos();
             this.expiresAtNanos = message.expiresAtNanos();
@@ -265,6 +284,34 @@ public String toString()
         }
     }
 
+    static class WriteCallbackInfo extends CallbackInfo
+    {
+        // either a Mutation, or a Paxos Commit (MessageOut)
+        private final Object mutation;
+
+        @VisibleForTesting
+        WriteCallbackInfo(Message message, InetAddressAndPort peer, RequestCallback<?> callback)
+        {
+            super(message, peer, callback);
+            this.mutation = message.payload;
+        }
+
+        /**
+         * Used for sensors tracking.
+         */
+        public IMutation iMutation()
+        {
+            return iMutation(mutation);
+        }
+
+        private static IMutation iMutation(Object object)
+        {
+            assert object instanceof Commit || object instanceof IMutation : object;
+            return object instanceof Commit ? ((Commit) object).makeMutation()
+                                            : (IMutation) object;
+        }
+    }
+
     @Override
     public void onOverloaded(Message<?> message, InetAddressAndPort peer)
     {
diff --git a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java
index 1cee468cd3d7..80f2bbfe0563 100644
--- a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java
+++ b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java
@@ -17,16 +17,27 @@
  */
 package org.apache.cassandra.net;
 
+import java.util.Optional;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.Sensor;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+import org.apache.cassandra.sensors.Type;
+import org.apache.cassandra.service.paxos.v1.AbstractPaxosCallback;
+import org.apache.cassandra.service.reads.ReadCallback;
 import org.apache.cassandra.tracing.Tracing;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 
-class ResponseVerbHandler implements IVerbHandler
+public class ResponseVerbHandler implements IVerbHandler
 {
     public static final ResponseVerbHandler instance = new ResponseVerbHandler();
 
@@ -47,7 +58,7 @@ public void doVerb(Message message)
         long latencyNanos = approxTime.now() - callbackInfo.createdAtNanos;
         Tracing.trace("Processing response from {}", message.from());
 
-        RequestCallback cb = callbackInfo.callback;
+        RequestCallback<?> cb = callbackInfo.callback;
         if (message.isFailureResponse())
         {
             cb.onFailure(message.from(), (RequestFailureReason) message.payload);
@@ -55,7 +66,61 @@ public void doVerb(Message message)
         else
         {
             MessagingService.instance().latencySubscribers.maybeAdd(cb, message.from(), latencyNanos, NANOSECONDS);
+            trackReplicaSensors(callbackInfo, message);
             cb.onResponse(message);
         }
     }
+
+    private void trackReplicaSensors(RequestCallbacks.CallbackInfo callbackInfo, Message<?> message)
+    {
+        RequestSensors sensors = callbackInfo.callback.getRequestSensors();
+        if (sensors == null)
+            return;
+
+        if (callbackInfo instanceof RequestCallbacks.WriteCallbackInfo)
+        {
+            RequestCallbacks.WriteCallbackInfo writerInfo = (RequestCallbacks.WriteCallbackInfo) callbackInfo;
+            IMutation mutation = writerInfo.iMutation();
+            if (mutation == null)
+                return;
+
+            for (PartitionUpdate pu : mutation.getPartitionUpdates())
+            {
+                Context context = Context.from(pu.metadata());
+                if (pu.metadata().isIndex()) continue;
+                incrementSensor(sensors, context, Type.WRITE_BYTES, message);
+            }
+        }
+        else if (callbackInfo.callback instanceof ReadCallback)
+        {
+            ReadCallback<?, ?> readCallback = (ReadCallback<?, ?>) callbackInfo.callback;
+            Context context = Context.from(readCallback.command());
+            incrementSensor(sensors, context, Type.READ_BYTES, message);
+        }
+        // Covers Paxos Prepare and Propose callbacks. Paxos Commit callback is a regular WriteCallbackInfo
+        else if (callbackInfo.callback instanceof AbstractPaxosCallback)
+        {
+            AbstractPaxosCallback<?> paxosCallback = (AbstractPaxosCallback<?>) callbackInfo.callback;
+            Context context = Context.from(paxosCallback.getMetadata());
+            incrementSensor(sensors, context, Type.READ_BYTES, message);
+            incrementSensor(sensors, context, Type.WRITE_BYTES, message);
+        }
+    }
+
+    /**
+     * Increments the sensor for the given context and type based on the value encoded in the replica response message.
+     */
+    private void incrementSensor(RequestSensors sensors, Context context, Type type, Message<?> message)
+    {
+        Optional<Sensor> sensor = sensors.getSensor(context, type);
+        if (sensor.isEmpty())
+            return;
+
+        Optional<String> customParam = SensorsCustomParams.paramForRequestSensor(sensor.get());
+        if (customParam.isEmpty())
+            return;
+
+        double sensorValue = SensorsCustomParams.sensorValueFromInternodeResponse(message, customParam.get());
+        sensors.incrementSensor(context, type, sensorValue);
+    }
 }
diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java
index c85f0ddeca44..404f5a14fa84 100644
--- a/src/java/org/apache/cassandra/net/Verb.java
+++ b/src/java/org/apache/cassandra/net/Verb.java
@@ -19,10 +19,14 @@
 
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.function.ToLongFunction;
+import java.util.stream.Collectors;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
@@ -34,6 +38,8 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.CounterMutation;
 import org.apache.cassandra.db.CounterMutationVerbHandler;
+import org.apache.cassandra.db.MultiRangeReadCommand;
+import org.apache.cassandra.db.MultiRangeReadResponse;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.MutationVerbHandler;
 import org.apache.cassandra.db.ReadCommand;
@@ -110,125 +116,129 @@
 /**
  * Note that priorities except P0 are presently unused.  P0 corresponds to urgent, i.e. what used to be the "Gossip" connection.
  */
-public enum Verb
+public class Verb
 {
-    MUTATION_RSP           (60,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    MUTATION_REQ           (0,   P3, writeTimeout,    MUTATION,          () -> Mutation.serializer,                  () -> MutationVerbHandler.instance,        MUTATION_RSP        ),
-    HINT_RSP               (61,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    HINT_REQ               (1,   P4, writeTimeout,    MUTATION,          () -> HintMessage.serializer,               () -> HintVerbHandler.instance,            HINT_RSP            ),
-    READ_REPAIR_RSP        (62,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    READ_REPAIR_REQ        (2,   P1, writeTimeout,    MUTATION,          () -> Mutation.serializer,                  () -> ReadRepairVerbHandler.instance,      READ_REPAIR_RSP     ),
-    BATCH_STORE_RSP        (65,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    BATCH_STORE_REQ        (5,   P3, writeTimeout,    MUTATION,          () -> Batch.serializer,                     () -> BatchStoreVerbHandler.instance,      BATCH_STORE_RSP     ),
-    BATCH_REMOVE_RSP       (66,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    BATCH_REMOVE_REQ       (6,   P3, writeTimeout,    MUTATION,          () -> TimeUUID.Serializer.instance,         () -> BatchRemoveVerbHandler.instance,     BATCH_REMOVE_RSP    ),
-
-    PAXOS_PREPARE_RSP      (93,  P2, writeTimeout,    REQUEST_RESPONSE,  () -> PrepareResponse.serializer,           () -> ResponseVerbHandler.instance                             ),
-    PAXOS_PREPARE_REQ      (33,  P2, writeTimeout,    MUTATION,          () -> Commit.serializer,                    () -> PrepareVerbHandler.instance,         PAXOS_PREPARE_RSP   ),
-    PAXOS_PROPOSE_RSP      (94,  P2, writeTimeout,    REQUEST_RESPONSE,  () -> BooleanSerializer.serializer,         () -> ResponseVerbHandler.instance                             ),
-    PAXOS_PROPOSE_REQ      (34,  P2, writeTimeout,    MUTATION,          () -> Commit.serializer,                    () -> ProposeVerbHandler.instance,         PAXOS_PROPOSE_RSP   ),
-    PAXOS_COMMIT_RSP       (95,  P2, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    PAXOS_COMMIT_REQ       (35,  P2, writeTimeout,    MUTATION,          () -> Agreed.serializer,                    () -> PaxosCommit.requestHandler,          PAXOS_COMMIT_RSP    ),
-
-    TRUNCATE_RSP           (79,  P0, truncateTimeout, REQUEST_RESPONSE,  () -> TruncateResponse.serializer,          () -> ResponseVerbHandler.instance                             ),
-    TRUNCATE_REQ           (19,  P0, truncateTimeout, MUTATION,          () -> TruncateRequest.serializer,           () -> TruncateVerbHandler.instance,        TRUNCATE_RSP        ),
-
-    COUNTER_MUTATION_RSP   (84,  P1, counterTimeout,  REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    COUNTER_MUTATION_REQ   (24,  P2, counterTimeout,  COUNTER_MUTATION,  () -> CounterMutation.serializer,           () -> CounterMutationVerbHandler.instance, COUNTER_MUTATION_RSP),
-
-    READ_RSP               (63,  P2, readTimeout,     REQUEST_RESPONSE,  () -> ReadResponse.serializer,              () -> ResponseVerbHandler.instance                             ),
-    READ_REQ               (3,   P3, readTimeout,     READ,              () -> ReadCommand.serializer,               () -> ReadCommandVerbHandler.instance,     READ_RSP            ),
-    RANGE_RSP              (69,  P2, rangeTimeout,    REQUEST_RESPONSE,  () -> ReadResponse.serializer,              () -> ResponseVerbHandler.instance                             ),
-    RANGE_REQ              (9,   P3, rangeTimeout,    READ,              () -> ReadCommand.serializer,               () -> ReadCommandVerbHandler.instance,     RANGE_RSP           ),
-
-    GOSSIP_DIGEST_SYN      (14,  P0, longTimeout,     GOSSIP,            () -> GossipDigestSyn.serializer,           () -> GossipDigestSynVerbHandler.instance                      ),
-    GOSSIP_DIGEST_ACK      (15,  P0, longTimeout,     GOSSIP,            () -> GossipDigestAck.serializer,           () -> GossipDigestAckVerbHandler.instance                      ),
-    GOSSIP_DIGEST_ACK2     (16,  P0, longTimeout,     GOSSIP,            () -> GossipDigestAck2.serializer,          () -> GossipDigestAck2VerbHandler.instance                     ),
-    GOSSIP_SHUTDOWN        (29,  P0, rpcTimeout,      GOSSIP,            () -> GossipShutdown.serializer,            () -> GossipShutdownVerbHandler.instance                       ),
-
-    ECHO_RSP               (91,  P0, rpcTimeout,      GOSSIP,            () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    ECHO_REQ               (31,  P0, rpcTimeout,      GOSSIP,            () -> NoPayload.serializer,                 () -> EchoVerbHandler.instance,            ECHO_RSP            ),
-    PING_RSP               (97,  P1, pingTimeout,     GOSSIP,            () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    PING_REQ               (37,  P1, pingTimeout,     GOSSIP,            () -> PingRequest.serializer,               () -> PingVerbHandler.instance,            PING_RSP            ),
-
-    // P1 because messages can be arbitrarily large or aren't crucial
-    SCHEMA_PUSH_RSP        (98,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    SCHEMA_PUSH_REQ        (18,  P1, rpcTimeout,      MIGRATION,         () -> SchemaMutationsSerializer.instance,   () -> SchemaPushVerbHandler.instance,      SCHEMA_PUSH_RSP     ),
-    SCHEMA_PULL_RSP        (88,  P1, rpcTimeout,      MIGRATION,         () -> SchemaMutationsSerializer.instance,   () -> ResponseVerbHandler.instance                             ),
-    SCHEMA_PULL_REQ        (28,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> SchemaPullVerbHandler.instance,      SCHEMA_PULL_RSP     ),
-    SCHEMA_VERSION_RSP     (80,  P1, rpcTimeout,      MIGRATION,         () -> UUIDSerializer.serializer,            () -> ResponseVerbHandler.instance                             ),
-    SCHEMA_VERSION_REQ     (20,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> SchemaVersionVerbHandler.instance,   SCHEMA_VERSION_RSP  ),
+    private static final List<Verb> verbs = new ArrayList<>();
+
+    public static List<Verb> getValues()
+    {
+        return ImmutableList.copyOf(verbs);
+    }
+
+    public static Verb MUTATION_RSP           = new Verb("MUTATION_RSP",           60,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance);
+    public static Verb MUTATION_REQ           = new Verb("MUTATION_REQ",           0,   P3, writeTimeout,    MUTATION,          () -> Mutation.serializer,                  () -> MutationVerbHandler.instance,        MUTATION_RSP);
+    public static Verb HINT_RSP               = new Verb("HINT_RSP",               61,  P1, hintTimeout,     REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb HINT_REQ               = new Verb("HINT_REQ",               1,   P4, hintTimeout,     MUTATION,          () -> HintMessage.serializer,               () -> HintVerbHandler.instance,            HINT_RSP            );
+    public static Verb READ_REPAIR_RSP        = new Verb("READ_REPAIR_RSP",        62,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb READ_REPAIR_REQ        = new Verb("READ_REPAIR_REQ",        2,   P1, writeTimeout,    MUTATION,          () -> Mutation.serializer,                  () -> ReadRepairVerbHandler.instance,      READ_REPAIR_RSP     );
+    public static Verb BATCH_STORE_RSP        = new Verb("BATCH_STORE_RSP",        65,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb BATCH_STORE_REQ        = new Verb("BATCH_STORE_REQ",        5,   P3, writeTimeout,    MUTATION,          () -> Batch.serializer,                     () -> BatchStoreVerbHandler.instance,      BATCH_STORE_RSP     );
+    public static Verb BATCH_REMOVE_RSP       = new Verb("BATCH_REMOVE_RSP",       66,  P1, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb BATCH_REMOVE_REQ       = new Verb("BATCH_REMOVE_REQ",       6,   P3, writeTimeout,    MUTATION,          () -> TimeUUID.Serializer.instance,         () -> BatchRemoveVerbHandler.instance,     BATCH_REMOVE_RSP    );
+
+    public static Verb PAXOS_PREPARE_RSP      = new Verb("PAXOS_PREPARE_RSP",      93,  P2, writeTimeout,    REQUEST_RESPONSE,  () -> PrepareResponse.serializer,           () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS_PREPARE_REQ      = new Verb("PAXOS_PREPARE_REQ",      33,  P2, writeTimeout,    MUTATION,          () -> Commit.serializer,                    () -> PrepareVerbHandler.instance,         PAXOS_PREPARE_RSP   );
+    public static Verb PAXOS_PROPOSE_RSP      = new Verb("PAXOS_PROPOSE_RSP",      94,  P2, writeTimeout,    REQUEST_RESPONSE,  () -> BooleanSerializer.serializer,         () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS_PROPOSE_REQ      = new Verb("PAXOS_PROPOSE_REQ",      34,  P2, writeTimeout,    MUTATION,          () -> Commit.serializer,                    () -> ProposeVerbHandler.instance,         PAXOS_PROPOSE_RSP   );
+    public static Verb PAXOS_COMMIT_RSP       = new Verb("PAXOS_COMMIT_RSP",       95,  P2, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS_COMMIT_REQ       = new Verb("PAXOS_COMMIT_REQ",       35,  P2, writeTimeout,    MUTATION,          () -> Agreed.serializer,                    () -> PaxosCommit.requestHandler,          PAXOS_COMMIT_RSP    );
+
+    public static Verb TRUNCATE_RSP           = new Verb("TRUNCATE_RSP",           79,  P0, truncateTimeout, REQUEST_RESPONSE,  () -> TruncateResponse.serializer,          () -> ResponseVerbHandler.instance                             );
+    public static Verb TRUNCATE_REQ           = new Verb("TRUNCATE_REQ",           19,  P0, truncateTimeout, MUTATION,          () -> TruncateRequest.serializer,           () -> TruncateVerbHandler.instance,        TRUNCATE_RSP        );
+
+    public static Verb GOSSIP_DIGEST_SYN      = new Verb("GOSSIP_DIGEST_SYN",      14,  P0, longTimeout,     GOSSIP,            () -> GossipDigestSyn.serializer,           () -> GossipDigestSynVerbHandler.instance                      );
+    public static Verb GOSSIP_DIGEST_ACK      = new Verb("GOSSIP_DIGEST_ACK",      15,  P0, longTimeout,     GOSSIP,            () -> GossipDigestAck.serializer,           () -> GossipDigestAckVerbHandler.instance                      );
+    public static Verb GOSSIP_DIGEST_ACK2     = new Verb("GOSSIP_DIGEST_ACK2",     16,  P0, longTimeout,     GOSSIP,            () -> GossipDigestAck2.serializer,          () -> GossipDigestAck2VerbHandler.instance                     );
+    public static Verb GOSSIP_SHUTDOWN        = new Verb("GOSSIP_SHUTDOWN",        29,  P0, rpcTimeout,      GOSSIP,            () -> GossipShutdown.serializer,            () -> GossipShutdownVerbHandler.instance                       );
+    public static Verb COUNTER_MUTATION_RSP   = new Verb("COUNTER_MUTATION_RSP",   84,  P1, counterTimeout,  REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb COUNTER_MUTATION_REQ   = new Verb("COUNTER_MUTATION_REQ",   24,  P2, counterTimeout,  COUNTER_MUTATION,  () -> CounterMutation.serializer,           () -> CounterMutationVerbHandler.instance, COUNTER_MUTATION_RSP);
+
+    public static Verb READ_RSP               = new Verb("READ_RSP",               63,  P2, readTimeout,     REQUEST_RESPONSE,  () -> ReadResponse.serializer,              () -> ResponseVerbHandler.instance                             );
+    public static Verb READ_REQ               = new Verb("READ_REQ",               3,   P3, readTimeout,     READ,              () -> ReadCommand.serializer,               () -> ReadCommandVerbHandler.instance,     READ_RSP            );
+    public static Verb RANGE_RSP              = new Verb("RANGE_RSP",              69,  P2, rangeTimeout,    REQUEST_RESPONSE,  () -> ReadResponse.serializer,              () -> ResponseVerbHandler.instance                             );
+    public static Verb RANGE_REQ              = new Verb("RANGE_REQ",              9,   P3, rangeTimeout,    READ,              () -> ReadCommand.serializer,               () -> ReadCommandVerbHandler.instance,     RANGE_RSP           );
+    public static Verb MULTI_RANGE_RSP        = new Verb("MULTI_RANGE_RSP",        67,  P2, rangeTimeout,    REQUEST_RESPONSE,  () -> MultiRangeReadResponse.serializer,    () -> ResponseVerbHandler.instance                             );
+    public static Verb MULTI_RANGE_REQ        = new Verb("MULTI_RANGE_REQ",        7,   P3, rangeTimeout,    READ,              () -> MultiRangeReadCommand.serializer,     () -> ReadCommandVerbHandler.instance,     MULTI_RANGE_RSP     );
+
+    public static Verb ECHO_RSP               = new Verb("ECHO_RSP",               91,  P0, rpcTimeout,      GOSSIP,            () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb ECHO_REQ               = new Verb("ECHO_REQ",               31,  P0, rpcTimeout,      GOSSIP,            () -> NoPayload.serializer,                 () -> EchoVerbHandler.instance,            ECHO_RSP            );
+    public static Verb PING_RSP               = new Verb("PING_RSP",               97,  P1, pingTimeout,     GOSSIP,            () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb PING_REQ               = new Verb("PING_REQ",               37,  P1, pingTimeout,     GOSSIP,            () -> PingRequest.serializer,               () -> PingVerbHandler.instance,            PING_RSP            );
+
+    // public static Verb P1 because messages can be arbitrarily large or aren't crucial
+    public static Verb SCHEMA_PUSH_RSP        = new Verb("SCHEMA_PUSH_RSP",        98,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb SCHEMA_PUSH_REQ        = new Verb("SCHEMA_PUSH_REQ",        18,  P1, rpcTimeout,      MIGRATION,         () -> SchemaMutationsSerializer.instance,   () -> SchemaPushVerbHandler.instance,      SCHEMA_PUSH_RSP     );
+    public static Verb SCHEMA_PULL_RSP        = new Verb("SCHEMA_PULL_RSP",        88,  P1, rpcTimeout,      MIGRATION,         () -> SchemaMutationsSerializer.instance,   () -> ResponseVerbHandler.instance                             );
+    public static Verb SCHEMA_PULL_REQ        = new Verb("SCHEMA_PULL_REQ",        28,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> SchemaPullVerbHandler.instance,      SCHEMA_PULL_RSP     );
+    public static Verb SCHEMA_VERSION_RSP     = new Verb("SCHEMA_VERSION_RSP",     80,  P1, rpcTimeout,      MIGRATION,         () -> UUIDSerializer.serializer,            () -> ResponseVerbHandler.instance                             );
+    public static Verb SCHEMA_VERSION_REQ     = new Verb("SCHEMA_VERSION_REQ",     20,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> SchemaVersionVerbHandler.instance,   SCHEMA_VERSION_RSP  );
 
     // repair; mostly doesn't use callbacks and sends responses as their own request messages, with matching sessions by uuid; should eventually harmonize and make idiomatic
     // for the repair messages that implement retry logic, use rpcTimeout so the single request fails faster, then retries can be used to recover
-    REPAIR_RSP             (100, P1, repairTimeout,   REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    VALIDATION_RSP         (102, P1, repairValidationRspTimeout,    ANTI_ENTROPY,      () -> ValidationResponse.serializer,        () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    VALIDATION_REQ         (101, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> ValidationRequest.serializer,         () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    SYNC_RSP               (104, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> SyncResponse.serializer,              () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    SYNC_REQ               (103, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> SyncRequest.serializer,               () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    PREPARE_MSG            (105, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> PrepareMessage.serializer,            () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    SNAPSHOT_MSG           (106, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> SnapshotMessage.serializer,           () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    CLEANUP_MSG            (107, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> CleanupMessage.serializer,            () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    PREPARE_CONSISTENT_RSP (109, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    PREPARE_CONSISTENT_REQ (108, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> PrepareConsistentRequest.serializer,  () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    FINALIZE_PROPOSE_MSG   (110, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> FinalizePropose.serializer,           () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    FINALIZE_PROMISE_MSG   (111, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> FinalizePromise.serializer,           () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    FINALIZE_COMMIT_MSG    (112, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> FinalizeCommit.serializer,            () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    FAILED_SESSION_MSG     (113, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> FailSession.serializer,               () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    STATUS_RSP             (115, P1, repairTimeout,   ANTI_ENTROPY,      () -> StatusResponse.serializer,            () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-    STATUS_REQ             (114, P1, repairTimeout,   ANTI_ENTROPY,      () -> StatusRequest.serializer,             () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
-
-    REPLICATION_DONE_RSP   (82,  P0, rpcTimeout,      MISC,              () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    REPLICATION_DONE_REQ   (22,  P0, rpcTimeout,      MISC,              () -> NoPayload.serializer,                 () -> ReplicationDoneVerbHandler.instance, REPLICATION_DONE_RSP),
-    SNAPSHOT_RSP           (87,  P0, rpcTimeout,      MISC,              () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    SNAPSHOT_REQ           (27,  P0, rpcTimeout,      MISC,              () -> SnapshotCommand.serializer,           () -> SnapshotVerbHandler.instance,        SNAPSHOT_RSP        ),
-
-    PAXOS2_COMMIT_REMOTE_REQ         (38, P2, writeTimeout,  MUTATION,          () -> Mutation.serializer,                     () -> MutationVerbHandler.instance,                          MUTATION_RSP                     ),
-    PAXOS2_COMMIT_REMOTE_RSP         (39, P2, writeTimeout,  REQUEST_RESPONSE,  () -> NoPayload.serializer,                    () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_PREPARE_RSP               (50, P2, writeTimeout,  REQUEST_RESPONSE,  () -> PaxosPrepare.responseSerializer,         () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_PREPARE_REQ               (40, P2, writeTimeout,  MUTATION,          () -> PaxosPrepare.requestSerializer,          () -> PaxosPrepare.requestHandler,                           PAXOS2_PREPARE_RSP               ),
-    PAXOS2_PREPARE_REFRESH_RSP       (51, P2, writeTimeout,  REQUEST_RESPONSE,  () -> PaxosPrepareRefresh.responseSerializer,  () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_PREPARE_REFRESH_REQ       (41, P2, writeTimeout,  MUTATION,          () -> PaxosPrepareRefresh.requestSerializer,   () -> PaxosPrepareRefresh.requestHandler,                    PAXOS2_PREPARE_REFRESH_RSP       ),
-    PAXOS2_PROPOSE_RSP               (52, P2, writeTimeout,  REQUEST_RESPONSE,  () -> PaxosPropose.responseSerializer,         () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_PROPOSE_REQ               (42, P2, writeTimeout,  MUTATION,          () -> PaxosPropose.requestSerializer,          () -> PaxosPropose.requestHandler,                           PAXOS2_PROPOSE_RSP               ),
-    PAXOS2_COMMIT_AND_PREPARE_RSP    (53, P2, writeTimeout,  REQUEST_RESPONSE,  () -> PaxosPrepare.responseSerializer,         () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_COMMIT_AND_PREPARE_REQ    (43, P2, writeTimeout,  MUTATION,          () -> PaxosCommitAndPrepare.requestSerializer, () -> PaxosCommitAndPrepare.requestHandler,                  PAXOS2_COMMIT_AND_PREPARE_RSP    ),
-    PAXOS2_REPAIR_RSP                (54, P2, writeTimeout,  PAXOS_REPAIR,      () -> PaxosRepair.responseSerializer,          () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_REPAIR_REQ                (44, P2, writeTimeout,  PAXOS_REPAIR,      () -> PaxosRepair.requestSerializer,           () -> PaxosRepair.requestHandler,                            PAXOS2_REPAIR_RSP                ),
-    PAXOS2_CLEANUP_START_PREPARE_RSP (55, P2, repairTimeout, PAXOS_REPAIR,      () -> PaxosCleanupHistory.serializer,          () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_CLEANUP_START_PREPARE_REQ (45, P2, repairTimeout, PAXOS_REPAIR,      () -> PaxosStartPrepareCleanup.serializer,     () -> PaxosStartPrepareCleanup.verbHandler,                  PAXOS2_CLEANUP_START_PREPARE_RSP ),
-    PAXOS2_CLEANUP_RSP               (56, P2, repairTimeout, PAXOS_REPAIR,      () -> NoPayload.serializer,                    () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_CLEANUP_REQ               (46, P2, repairTimeout, PAXOS_REPAIR,      () -> PaxosCleanupRequest.serializer,          () -> PaxosCleanupRequest.verbHandler,                       PAXOS2_CLEANUP_RSP               ),
-    PAXOS2_CLEANUP_RSP2              (57, P2, repairTimeout, PAXOS_REPAIR,      () -> PaxosCleanupResponse.serializer,         () -> PaxosCleanupResponse.verbHandler                                                        ),
-    PAXOS2_CLEANUP_FINISH_PREPARE_RSP(58, P2, repairTimeout, PAXOS_REPAIR,      () -> NoPayload.serializer,                    () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_CLEANUP_FINISH_PREPARE_REQ(47, P2, repairTimeout, IMMEDIATE,         () -> PaxosCleanupHistory.serializer,          () -> PaxosFinishPrepareCleanup.verbHandler,                 PAXOS2_CLEANUP_FINISH_PREPARE_RSP),
-    PAXOS2_CLEANUP_COMPLETE_RSP      (59, P2, repairTimeout, PAXOS_REPAIR,      () -> NoPayload.serializer,                    () -> ResponseVerbHandler.instance                                                            ),
-    PAXOS2_CLEANUP_COMPLETE_REQ      (48, P2, repairTimeout, PAXOS_REPAIR,      () -> PaxosCleanupComplete.serializer,         () -> PaxosCleanupComplete.verbHandler,                      PAXOS2_CLEANUP_COMPLETE_RSP      ),
+    public static Verb REPAIR_RSP             = new Verb("REPAIR_RSP",             100, P1, repairTimeout,   REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb VALIDATION_RSP         = new Verb("VALIDATION_RSP",         102, P1, repairValidationRspTimeout, ANTI_ENTROPY, () -> ValidationResponse.serializer,  () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb VALIDATION_REQ         = new Verb("VALIDATION_REQ",         101, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> ValidationRequest.serializer,     () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb SYNC_RSP               = new Verb("SYNC_RSP",               104, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SyncResponse.serializer,          () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb SYNC_REQ               = new Verb("SYNC_REQ",               103, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SyncRequest.serializer,           () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb PREPARE_MSG            = new Verb("PREPARE_MSG",            105, P1, prepareWithBackoffTimeout, ANTI_ENTROPY, () -> PrepareMessage.serializer,                  () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb SNAPSHOT_MSG           = new Verb("SNAPSHOT_MSG",           106, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SnapshotMessage.serializer,       () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb CLEANUP_MSG            = new Verb("CLEANUP_MSG",            107, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> CleanupMessage.serializer,        () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb PREPARE_CONSISTENT_RSP = new Verb("PREPARE_CONSISTENT_RSP", 109, P1, repairWithBackoffTimeout, ANTI_ENTROPY,      () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb PREPARE_CONSISTENT_REQ = new Verb("PREPARE_CONSISTENT_REQ", 108, P1, repairWithBackoffTimeout, ANTI_ENTROPY,      () -> PrepareConsistentRequest.serializer,  () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb FINALIZE_PROPOSE_MSG   = new Verb("FINALIZE_PROPOSE_MSG",   110, P1, repairWithBackoffTimeout, ANTI_ENTROPY,      () -> FinalizePropose.serializer,           () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb FINALIZE_PROMISE_MSG   = new Verb("FINALIZE_PROMISE_MSG",   111, P1, repairWithBackoffTimeout, ANTI_ENTROPY,      () -> FinalizePromise.serializer,           () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb FINALIZE_COMMIT_MSG    = new Verb("FINALIZE_COMMIT_MSG",    112, P1, repairWithBackoffTimeout, ANTI_ENTROPY,      () -> FinalizeCommit.serializer,            () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb FAILED_SESSION_MSG     = new Verb("FAILED_SESSION_MSG",     113, P1, repairWithBackoffTimeout, ANTI_ENTROPY,      () -> FailSession.serializer,               () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb STATUS_RSP             = new Verb("STATUS_RSP",             115, P1, repairTimeout,   ANTI_ENTROPY,      () -> StatusResponse.serializer,            () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+    public static Verb STATUS_REQ             = new Verb("STATUS_REQ",             114, P1, repairTimeout,   ANTI_ENTROPY,      () -> StatusRequest.serializer,             () -> RepairMessageVerbHandler.instance(), REPAIR_RSP          );
+
+    public static Verb REPLICATION_DONE_RSP   = new Verb("REPLICATION_DONE_RSP",   82,  P0, rpcTimeout,      MISC,              () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb REPLICATION_DONE_REQ   = new Verb("REPLICATION_DONE_REQ",   22,  P0, rpcTimeout,      MISC,              () -> NoPayload.serializer,                 () -> ReplicationDoneVerbHandler.instance, REPLICATION_DONE_RSP);
+    public static Verb SNAPSHOT_RSP           = new Verb("SNAPSHOT_RSP",           87,  P0, rpcTimeout,      MISC,              () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb SNAPSHOT_REQ           = new Verb("SNAPSHOT_REQ",           27,  P0, rpcTimeout,      MISC,              () -> SnapshotCommand.serializer,           () -> SnapshotVerbHandler.instance,        SNAPSHOT_RSP        );
+
+    public static Verb PAXOS2_COMMIT_REMOTE_REQ=new Verb("PAXOS2_COMMIT_REMOTE_REQ",38, P2, writeTimeout,    MUTATION,          () -> Mutation.serializer,                  () -> MutationVerbHandler.instance,        MUTATION_RSP        );
+    public static Verb PAXOS2_COMMIT_REMOTE_RSP=new Verb("PAXOS2_COMMIT_REMOTE_RSP",39, P2, writeTimeout,    REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS2_PREPARE_RSP     = new Verb("PAXOS2_PREPARE_RSP",     50,  P2, writeTimeout,    REQUEST_RESPONSE,  () -> PaxosPrepare.responseSerializer,      () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS2_PREPARE_REQ     = new Verb("PAXOS2_PREPARE_REQ",     40,  P2, writeTimeout,    MUTATION,          () -> PaxosPrepare.requestSerializer,       () -> PaxosPrepare.requestHandler,         PAXOS2_PREPARE_RSP  );
+    public static Verb PAXOS2_PREPARE_REFRESH_RSP= new Verb("PAXOS2_PREPARE_REFRESH_RSP", 51, P2, writeTimeout, REQUEST_RESPONSE,  () -> PaxosPrepareRefresh.responseSerializer,  () -> ResponseVerbHandler.instance                       );
+    public static Verb PAXOS2_PREPARE_REFRESH_REQ= new Verb("PAXOS2_PREPARE_REFRESH_REQ", 41, P2, writeTimeout, MUTATION,       () -> PaxosPrepareRefresh.requestSerializer,() -> PaxosPrepareRefresh.requestHandler,  PAXOS2_PREPARE_REFRESH_RSP);
+    public static Verb PAXOS2_PROPOSE_RSP     = new Verb("PAXOS2_PROPOSE_RSP",     52,  P2, writeTimeout,    REQUEST_RESPONSE,  () -> PaxosPropose.responseSerializer,      () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS2_PROPOSE_REQ     = new Verb("PAXOS2_PROPOSE_REQ",     42,  P2, writeTimeout,    MUTATION,          () -> PaxosPropose.requestSerializer,       () -> PaxosPropose.requestHandler,         PAXOS2_PROPOSE_RSP  );
+    public static Verb PAXOS2_COMMIT_AND_PREPARE_RSP= new Verb("PAXOS2_COMMIT_AND_PREPARE_RSP", 53, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPrepare.responseSerializer, () -> ResponseVerbHandler.instance                          );
+    public static Verb PAXOS2_COMMIT_AND_PREPARE_REQ= new Verb("PAXOS2_COMMIT_AND_PREPARE_REQ", 43, P2, writeTimeout, MUTATION, () -> PaxosCommitAndPrepare.requestSerializer, () -> PaxosCommitAndPrepare.requestHandler, PAXOS2_COMMIT_AND_PREPARE_RSP    );
+    public static Verb PAXOS2_REPAIR_RSP                = new Verb("PAXOS2_REPAIR_RSP", 54, P2, writeTimeout,  PAXOS_REPAIR,    () -> PaxosRepair.responseSerializer,       () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS2_REPAIR_REQ                = new Verb("PAXOS2_REPAIR_REQ", 44, P2, writeTimeout,  PAXOS_REPAIR,    () -> PaxosRepair.requestSerializer,        () -> PaxosRepair.requestHandler,          PAXOS2_REPAIR_RSP   );
+    public static Verb PAXOS2_CLEANUP_START_PREPARE_RSP = new Verb("PAXOS2_CLEANUP_START_PREPARE_RSP", 55, P2, repairTimeout, PAXOS_REPAIR, () -> PaxosCleanupHistory.serializer, () -> ResponseVerbHandler.instance                       );
+    public static Verb PAXOS2_CLEANUP_START_PREPARE_REQ = new Verb("PAXOS2_CLEANUP_START_PREPARE_REQ", 45, P2, repairTimeout, PAXOS_REPAIR, () -> PaxosStartPrepareCleanup.serializer, () -> PaxosStartPrepareCleanup.verbHandler, PAXOS2_CLEANUP_START_PREPARE_RSP );
+    public static Verb PAXOS2_CLEANUP_RSP               = new Verb("PAXOS2_CLEANUP_RSP", 56, P2, repairTimeout, PAXOS_REPAIR,   () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS2_CLEANUP_REQ               = new Verb("PAXOS2_CLEANUP_REQ", 46, P2, repairTimeout, PAXOS_REPAIR,   () -> PaxosCleanupRequest.serializer,       () -> PaxosCleanupRequest.verbHandler,     PAXOS2_CLEANUP_RSP  );
+    public static Verb PAXOS2_CLEANUP_RSP2              = new Verb("PAXOS2_CLEANUP_RSP2", 57, P2, repairTimeout, PAXOS_REPAIR,  () -> PaxosCleanupResponse.serializer,      () -> PaxosCleanupResponse.verbHandler                         );
+    public static Verb PAXOS2_CLEANUP_FINISH_PREPARE_RSP= new Verb("PAXOS2_CLEANUP_FINISH_PREPARE_RSP", 58, P2, repairTimeout, PAXOS_REPAIR, () -> NoPayload.serializer,    () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS2_CLEANUP_FINISH_PREPARE_REQ= new Verb("PAXOS2_CLEANUP_FINISH_PREPARE_REQ", 47, P2, repairTimeout, IMMEDIATE, () -> PaxosCleanupHistory.serializer, () -> PaxosFinishPrepareCleanup.verbHandler, PAXOS2_CLEANUP_FINISH_PREPARE_RSP);
+    public static Verb PAXOS2_CLEANUP_COMPLETE_RSP      = new Verb("PAXOS2_CLEANUP_COMPLETE_RSP", 59, P2, repairTimeout, PAXOS_REPAIR, () -> NoPayload.serializer,          () -> ResponseVerbHandler.instance                             );
+    public static Verb PAXOS2_CLEANUP_COMPLETE_REQ      = new Verb("PAXOS2_CLEANUP_COMPLETE_REQ", 48, P2, repairTimeout, PAXOS_REPAIR, () -> PaxosCleanupComplete.serializer, () -> PaxosCleanupComplete.verbHandler,  PAXOS2_CLEANUP_COMPLETE_RSP);
 
     // generic failure response
-    FAILURE_RSP            (99,  P0, noTimeout,       REQUEST_RESPONSE,  () -> RequestFailureReason.serializer,      () -> ResponseVerbHandler.instance                             ),
+    public static Verb FAILURE_RSP            = new Verb("FAILURE_RSP",            99,  P0, noTimeout,       REQUEST_RESPONSE,  () -> RequestFailureReason.serializer,      CustomResponseVerbHandlerProvider.instance                     );
 
     // dummy verbs
-    _TRACE                 (30,  P1, rpcTimeout,      TRACING,           () -> NoPayload.serializer,                 () -> null                                                     ),
-    _SAMPLE                (49,  P1, rpcTimeout,      INTERNAL_RESPONSE, () -> NoPayload.serializer,                 () -> null                                                     ),
-    _TEST_1                (10,  P0, writeTimeout,    IMMEDIATE,         () -> NoPayload.serializer,                 () -> null                                                     ),
-    _TEST_2                (11,  P1, rpcTimeout,      IMMEDIATE,         () -> NoPayload.serializer,                 () -> null                                                     ),
+    public static Verb _TRACE                 = new Verb("_TRACE",                 30,  P1, rpcTimeout,      TRACING,           () -> NoPayload.serializer,                 () -> null                                                     );
+    public static Verb _SAMPLE                = new Verb("_SAMPLE",                49,  P1, rpcTimeout,      INTERNAL_RESPONSE, () -> NoPayload.serializer,                 () -> null                                                     );
+    public static Verb _TEST_1                = new Verb("_TEST_1",                10,  P0, writeTimeout,    IMMEDIATE,         () -> NoPayload.serializer,                 () -> null                                                     );
+    public static Verb _TEST_2                = new Verb("_TEST_2",                11,  P1, rpcTimeout,      IMMEDIATE,         () -> NoPayload.serializer,                 () -> null                                                     );
 
     /** @deprecated See CASSANDRA-15066 */
     @Deprecated(since = "4.0")
-    REQUEST_RSP            (4,   P1, rpcTimeout,      REQUEST_RESPONSE,  () -> null,                                 () -> ResponseVerbHandler.instance                             ),
+    public static Verb REQUEST_RSP            = new Verb("REQUEST_RSP",            4,   P1, rpcTimeout,      REQUEST_RESPONSE,  () -> null,                                 CustomResponseVerbHandlerProvider.instance                     );
     /** @deprecated See CASSANDRA-15066 */
     @Deprecated(since = "4.0")
-    INTERNAL_RSP           (23,  P1, rpcTimeout,      INTERNAL_RESPONSE, () -> null,                                 () -> ResponseVerbHandler.instance                             ),
+    public static Verb INTERNAL_RSP           = new Verb("INTERNAL_RSP",           23,  P1, rpcTimeout,      INTERNAL_RESPONSE, () -> null,                                 () -> ResponseVerbHandler.instance                             );
 
-    // largest used ID: 116
+    // largest used public static Verb ID: 116
 
     // CUSTOM VERBS
-    UNUSED_CUSTOM_VERB     (CUSTOM,
-                            0,   P1, rpcTimeout,      INTERNAL_RESPONSE, () -> null,                                 () -> null                                                     ),
-    ;
-
-    public static final List<Verb> VERBS = ImmutableList.copyOf(Verb.values());
+    public static Verb UNUSED_CUSTOM_VERB     = new Verb("UNUSED_CUSTOM_VERB", CUSTOM,0,P1, rpcTimeout,      INTERNAL_RESPONSE, () -> null,                                 () -> null                                                     );
 
     public enum Priority
     {
@@ -245,6 +255,7 @@ public enum Kind
         CUSTOM
     }
 
+    private final String name;
     public final int id;
     public final Priority priority;
     public final Stage stage;
@@ -262,38 +273,37 @@ public enum Kind
      * NOTE: we use a Supplier to avoid loading the dependent classes until necessary.
      */
     private final Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer;
-    private final Supplier<? extends IVerbHandler<?>> handler;
+    private Supplier<? extends IVerbHandler<?>> handler;
 
     public final Verb responseVerb;
 
     private final ToLongFunction<TimeUnit> expiration;
 
-
     /**
      * Verbs it's okay to drop if the request has been queued longer than the request timeout.  These
      * all correspond to client requests or something triggered by them; we don't want to
      * drop internal messages like bootstrap or repair notifications.
      */
-    Verb(int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler)
+    Verb(String name, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler)
     {
-        this(id, priority, expiration, stage, serializer, handler, null);
+        this(name, id, priority, expiration, stage, serializer, handler, null);
     }
 
-    Verb(int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler, Verb responseVerb)
+    Verb(String name, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler, Verb responseVerb)
     {
-        this(NORMAL, id, priority, expiration, stage, serializer, handler, responseVerb);
+        this(name, NORMAL, id, priority, expiration, stage, serializer, handler, responseVerb);
     }
 
-    Verb(Kind kind, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler)
+    Verb(String name, Kind kind, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler)
     {
-        this(kind, id, priority, expiration, stage, serializer, handler, null);
+        this(name, kind, id, priority, expiration, stage, serializer, handler, null);
     }
 
-    Verb(Kind kind, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler, Verb responseVerb)
+    Verb(String name, Kind kind, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler, Verb responseVerb)
     {
         this.stage = stage;
         if (id < 0)
-            throw new IllegalArgumentException("Verb id must be non-negative, got " + id + " for verb " + name());
+            throw new IllegalArgumentException("Verb id must be non-negative, got " + id + " for verb " + name);
 
         if (kind == CUSTOM)
         {
@@ -307,12 +317,15 @@ public enum Kind
                 throw new AssertionError("Invalid verb id " + id + " - we only allow ids between 0 and " + (CUSTOM_VERB_START - MAX_CUSTOM_VERB_ID));
             this.id = id;
         }
+        this.name = name;
         this.priority = priority;
         this.serializer = serializer;
         this.handler = handler;
         this.responseVerb = responseVerb;
         this.expiration = expiration;
         this.kind = kind;
+
+         verbs.add(this);
     }
 
     public <In, Out> IVersionedAsymmetricSerializer<In, Out> serializer()
@@ -380,6 +393,17 @@ ToLongFunction<TimeUnit> unsafeSetExpiration(ToLongFunction<TimeUnit> expiration
         return original;
     }
 
+    @Override
+    public String toString()
+    {
+        return name();
+    }
+
+    public String name()
+    {
+        return name;
+    }
+
     // This is the largest number we can store in 2 bytes using VIntCoding (1 bit per byte is used to indicate if there is more data coming).
     // When generating ids we count *down* from this number
     private static final int CUSTOM_VERB_START = (1 << (7 * 2)) - 1;
@@ -389,12 +413,12 @@ ToLongFunction<TimeUnit> unsafeSetExpiration(ToLongFunction<TimeUnit> expiration
     private static final int MAX_CUSTOM_VERB_ID = 1000;
 
     private static final Verb[] idToVerbMap;
-    private static final Verb[] idToCustomVerbMap;
-    private static final int minCustomId;
+    private static volatile Verb[] idToCustomVerbMap;
+    private static volatile int minCustomId;
 
     static
     {
-        Verb[] verbs = values();
+        List<Verb> verbs = getValues();
         int max = -1;
         int minCustom = Integer.MAX_VALUE;
         for (Verb v : verbs)
@@ -430,8 +454,7 @@ ToLongFunction<TimeUnit> unsafeSetExpiration(ToLongFunction<TimeUnit> expiration
                     break;
                 case CUSTOM:
                     int relativeId = idForCustomVerb(v.id);
-                    if (customIdMap[relativeId] != null)
-                        throw new IllegalArgumentException("cannot have two custom verbs that map to the same id: " + v + " and " + customIdMap[relativeId]);
+                    assertCustomIdIsUnused(customIdMap, relativeId, v.name);
                     customIdMap[relativeId] = v;
                     break;
                 default:
@@ -443,6 +466,12 @@ ToLongFunction<TimeUnit> unsafeSetExpiration(ToLongFunction<TimeUnit> expiration
         idToCustomVerbMap = customIdMap;
     }
 
+    private static void assertCustomIdIsUnused(Verb[] customIdMap, int id, String name)
+    {
+        if (id < customIdMap.length && customIdMap[id] != null)
+            throw new IllegalArgumentException("cannot have two custom verbs that map to the same id: " + name + " and " + customIdMap[id]);
+    }
+
     public static Verb fromId(int id)
     {
         Verb[] verbs = idToVerbMap;
@@ -458,12 +487,82 @@ public static Verb fromId(int id)
     }
 
     /**
-     * calculate an id for a custom verb
+     * Convert to/from relative and absolute id for a custom verb.
+     *
+     * <pre>{@code
+     *          relId = idForCustomVerb(absId)
+     *          absId = idForCustomVerb(relId).
+     * }</pre>
+     *
+     * <p>Relative ids can be used for indexing idToCustomVerbMap. Absolute ids exist to distinguish
+     * regular verbs from custom verbs in the id space.</p>
+     *
+     * @param id the relative or absolute id.
+     * @return a relative id if {@code id} is absolute, or absolute id if {@code id} is relative.
      */
     private static int idForCustomVerb(int id)
     {
         return CUSTOM_VERB_START - id;
     }
+
+    /**
+     * Add a new custom verb to the list of verbs.
+     *
+     * <p>While we could dynamically generate an {@code id} for callers, it's safer to have users
+     * explicitly control the id space since it prevents nodes with different versions disagreeing on which
+     * verb has which id, e.g. during upgrade.</p>
+     *
+     * @param name the name of the new verb.
+     * @param id the identifier for this custom verb (must be relative id and >= 0 && <= MAX_CUSTOM_VERB_ID).
+     * @param priority the priority of the new verb.
+     * @param expiration an optional timeout for this verb. @see VerbTimeouts.
+     * @param stage The stage this verb should execute in.
+     * @param serializer A method to serialize this verb
+     * @param handler A method to handle this verb when received by the network
+     * @param responseVerb The verb to respond with (optional)
+     * @return A Verb for the newly added verb.
+     */
+    public static synchronized Verb addCustomVerb(String name, int id, Priority priority, ToLongFunction<TimeUnit> expiration, Stage stage, Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> serializer, Supplier<? extends IVerbHandler<?>> handler, Verb responseVerb)
+    {
+        assertNameIsUnused(name);
+        assertCustomIdIsUnused(idToCustomVerbMap, id, name);
+        Verb verb = new Verb(name, CUSTOM, id, priority, expiration, stage, serializer, handler, responseVerb);
+
+        int absoluteId = idForCustomVerb(id);
+        minCustomId = Math.min(absoluteId, minCustomId);
+
+        Verb[] newMap = Arrays.copyOf(idToCustomVerbMap, CUSTOM_VERB_START - minCustomId + 1);
+        System.arraycopy(idToCustomVerbMap, 0, newMap, 0, idToCustomVerbMap.length);
+        newMap[id] = verb;
+        idToCustomVerbMap = newMap;
+
+        return verb;
+    }
+
+    // Callers must take care of synchronizing to protect against concurrent updates to verbs.
+    private static void assertNameIsUnused(String name)
+    {
+        if (verbs.stream().map(v -> v.name).collect(Collectors.toList()).contains(name))
+            throw new IllegalArgumentException("Verb name '" + name + "' already exists");
+    }
+
+    /**
+     * Decorates the specified verb handler with the provided method.
+     *
+     * <p>An example use case is to run a custom method after every write request.</p>
+     *
+     * @param verbs the list of verbs whose handlers should be wrapped by {@code decoratorFn}.
+     * @param decoratorFn the method that decorates the handlers in verbs.
+     */
+    public static synchronized void decorateHandler(List<Verb> verbs, Function<IVerbHandler<?>, IVerbHandler<?>> decoratorFn)
+    {
+        for (Verb v : verbs)
+        {
+            IVerbHandler<?> handler = v.handler();
+            final IVerbHandler<?> decoratedHandler = decoratorFn.apply(handler);
+            v.handler = () -> decoratedHandler;
+        }
+    }
 }
 
 @SuppressWarnings("unused")
@@ -471,6 +570,7 @@ class VerbTimeouts
 {
     static final ToLongFunction<TimeUnit> rpcTimeout      = DatabaseDescriptor::getRpcTimeout;
     static final ToLongFunction<TimeUnit> writeTimeout    = DatabaseDescriptor::getWriteRpcTimeout;
+    static final ToLongFunction<TimeUnit> hintTimeout     = DatabaseDescriptor::getHintsRpcTimeout;
     static final ToLongFunction<TimeUnit> readTimeout     = DatabaseDescriptor::getReadRpcTimeout;
     static final ToLongFunction<TimeUnit> rangeTimeout    = DatabaseDescriptor::getRangeRpcTimeout;
     static final ToLongFunction<TimeUnit> counterTimeout  = DatabaseDescriptor::getCounterWriteRpcTimeout;
@@ -491,4 +591,11 @@ class VerbTimeouts
             return longTimeout.applyAsLong(units);
         return rpcTimeout.applyAsLong(units);
     };
+    static final ToLongFunction<TimeUnit> prepareTimeout  = DatabaseDescriptor::getRepairPrepareMessageTimeout;
+    static final ToLongFunction<TimeUnit> prepareWithBackoffTimeout    = units -> {
+        if (!DatabaseDescriptor.getRepairRetrySpec().isEnabled())
+            return prepareTimeout.applyAsLong(units);
+        return rpcTimeout.applyAsLong(units);
+    };
+    static final ToLongFunction<TimeUnit> repairMsgTimeout= DatabaseDescriptor::getRepairRpcTimeout;
 }
diff --git a/src/java/org/apache/cassandra/nodes/ILocalInfo.java b/src/java/org/apache/cassandra/nodes/ILocalInfo.java
new file mode 100644
index 000000000000..1e4d5c2ab116
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/ILocalInfo.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.CassandraVersion;
+
+public interface ILocalInfo extends INodeInfo<LocalInfo>
+{
+    InetAddressAndPort getBroadcastAddressAndPort();
+
+    SystemKeyspace.BootstrapState getBootstrapState();
+
+    String getClusterName();
+
+    CassandraVersion getCqlVersion();
+
+    InetAddressAndPort getListenAddressAndPort();
+
+    ProtocolVersion getNativeProtocolVersion();
+
+    Class<? extends IPartitioner> getPartitionerClass();
+
+    ImmutableMap<UUID, TruncationRecord> getTruncationRecords();
+
+    @Override
+    LocalInfo duplicate();
+}
diff --git a/src/java/org/apache/cassandra/nodes/INodeInfo.java b/src/java/org/apache/cassandra/nodes/INodeInfo.java
new file mode 100644
index 000000000000..7984694690d3
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/INodeInfo.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import java.util.Collection;
+import java.util.UUID;
+import javax.annotation.Nonnull;
+
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.CassandraVersion;
+
+public interface INodeInfo<T extends NodeInfo<T>> extends Cloneable
+{
+    UUID getHostId();
+
+    String getDataCenter();
+
+    String getRack();
+
+    CassandraVersion getReleaseVersion();
+
+    UUID getSchemaVersion();
+
+    @Nonnull
+    Collection<Token> getTokens();
+
+    InetAddressAndPort getNativeTransportAddressAndPort();
+
+    T duplicate();
+}
diff --git a/src/java/org/apache/cassandra/nodes/INodesPersistence.java b/src/java/org/apache/cassandra/nodes/INodesPersistence.java
new file mode 100644
index 000000000000..23fb8a20aaee
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/INodesPersistence.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import java.util.stream.Stream;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public interface INodesPersistence
+{
+    public static final INodesPersistence NO_NODES_PERSISTENCE = new INodesPersistence()
+    {
+        @Override
+        public LocalInfo loadLocal()
+        {
+            return null;
+        }
+
+        @Override
+        public void saveLocal(LocalInfo info)
+        {
+            // no-op
+        }
+
+        @Override
+        public void syncLocal()
+        {
+            // no-op
+        }
+
+        @Override
+        public Stream<PeerInfo> loadPeers()
+        {
+            return Stream.empty();
+        }
+
+        @Override
+        public void savePeer(PeerInfo info)
+        {
+            // no-op
+        }
+
+        @Override
+        public void deletePeer(InetAddressAndPort endpoint)
+        {
+            // no-op
+        }
+
+        @Override
+        public void syncPeers()
+        {
+            // no-op
+        }
+    };
+
+    LocalInfo loadLocal();
+
+    void saveLocal(LocalInfo info);
+
+    void syncLocal();
+
+    Stream<PeerInfo> loadPeers();
+
+    void savePeer(PeerInfo info);
+
+    void deletePeer(InetAddressAndPort endpoint);
+
+    void syncPeers();
+}
diff --git a/src/java/org/apache/cassandra/nodes/IPeerInfo.java b/src/java/org/apache/cassandra/nodes/IPeerInfo.java
new file mode 100644
index 000000000000..693109581097
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/IPeerInfo.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public interface IPeerInfo extends INodeInfo<PeerInfo>
+{
+    InetAddressAndPort getPeerAddressAndPort();
+
+    InetAddressAndPort getPreferredAddressAndPort();
+
+    boolean isRemoved();
+
+    boolean isExisting();
+
+    @Override
+    PeerInfo duplicate();
+}
diff --git a/src/java/org/apache/cassandra/nodes/LocalInfo.java b/src/java/org/apache/cassandra/nodes/LocalInfo.java
new file mode 100644
index 000000000000..3839044c8bf8
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/LocalInfo.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.nodes;
+
+import java.net.InetAddress;
+import java.util.Map;
+import java.util.Objects;
+import java.util.UUID;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import org.apache.cassandra.db.SystemKeyspace.BootstrapState;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.ImmutableUtils;
+
+@NotThreadSafe
+public final class LocalInfo extends NodeInfo<LocalInfo> implements ILocalInfo
+{
+    private volatile InetAddressAndPort broadcastAddressAndPort;
+    private volatile BootstrapState bootstrapState;
+    private volatile String clusterName;
+    private volatile CassandraVersion cqlVersion;
+    private volatile InetAddressAndPort listenAddressAndPort;
+    private volatile ProtocolVersion nativeProtocolVersion;
+    private volatile Class<? extends IPartitioner> partitionerClass;
+    private volatile ImmutableMap<UUID, TruncationRecord> truncationRecords = ImmutableMap.of();
+
+    @Override
+    public InetAddressAndPort getBroadcastAddressAndPort()
+    {
+        return broadcastAddressAndPort;
+    }
+
+    public LocalInfo setBroadcastAddressAndPort(InetAddressAndPort broadcastAddressAndPort)
+    {
+        this.broadcastAddressAndPort = broadcastAddressAndPort;
+        return this;
+    }
+
+    @Override
+    public BootstrapState getBootstrapState()
+    {
+        return bootstrapState;
+    }
+
+    public LocalInfo setBootstrapState(BootstrapState bootstrapState)
+    {
+        this.bootstrapState = bootstrapState;
+        return this;
+    }
+
+    @Override
+    public String getClusterName()
+    {
+        return clusterName;
+    }
+
+    public LocalInfo setClusterName(String clusterName)
+    {
+        this.clusterName = clusterName;
+        return this;
+    }
+
+    @Override
+    public CassandraVersion getCqlVersion()
+    {
+        return cqlVersion;
+    }
+
+    public LocalInfo setCqlVersion(CassandraVersion cqlVersion)
+    {
+        this.cqlVersion = cqlVersion;
+        return this;
+    }
+
+    @Override
+    public InetAddressAndPort getListenAddressAndPort()
+    {
+        return listenAddressAndPort;
+    }
+
+    public LocalInfo setListenAddressAndPort(InetAddressAndPort listenAddressAndPort)
+    {
+        this.listenAddressAndPort = listenAddressAndPort;
+        return this;
+    }
+
+    public LocalInfo setListenAddressOnly(InetAddress address, int defaultPort)
+    {
+        this.listenAddressAndPort = getAddressAndPort(getListenAddressAndPort(), address, defaultPort);
+        return this;
+    }
+
+    @Override
+    public ProtocolVersion getNativeProtocolVersion()
+    {
+        return nativeProtocolVersion;
+    }
+
+    public LocalInfo setNativeProtocolVersion(ProtocolVersion nativeProtocolVersion)
+    {
+        this.nativeProtocolVersion = nativeProtocolVersion;
+        return this;
+    }
+
+    @Override
+    public Class<? extends IPartitioner> getPartitionerClass()
+    {
+        return partitionerClass;
+    }
+
+    public LocalInfo setPartitionerClass(Class<? extends IPartitioner> partitionerClass)
+    {
+        this.partitionerClass = partitionerClass;
+        return this;
+    }
+
+    @Override
+    public ImmutableMap<UUID, TruncationRecord> getTruncationRecords()
+    {
+        return truncationRecords;
+    }
+
+    public LocalInfo setTruncationRecords(Map<UUID, TruncationRecord> truncationRecords)
+    {
+        this.truncationRecords = ImmutableMap.copyOf(truncationRecords);
+        return this;
+    }
+
+    public LocalInfo removeTruncationRecord(UUID tableId)
+    {
+        return setTruncationRecords(ImmutableUtils.without(getTruncationRecords(), tableId));
+    }
+
+    public LocalInfo addTruncationRecord(UUID tableId, TruncationRecord truncationRecord)
+    {
+        return setTruncationRecords(ImmutableUtils.withAddedOrUpdated(getTruncationRecords(), tableId, truncationRecord));
+    }
+
+    @Override
+    public LocalInfo duplicate()
+    {
+        try
+        {
+            return (LocalInfo) clone();
+        }
+        catch (CloneNotSupportedException e)
+        {
+            throw new AssertionError(e);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (!(o instanceof LocalInfo)) return false;
+        if (!super.equals(o)) return false;
+        LocalInfo localInfo = (LocalInfo) o;
+        return Objects.equals(getBroadcastAddressAndPort(), localInfo.getBroadcastAddressAndPort())
+               && getBootstrapState() == localInfo.getBootstrapState()
+               && Objects.equals(getClusterName(), localInfo.getClusterName())
+               && Objects.equals(getCqlVersion(), localInfo.getCqlVersion())
+               && Objects.equals(getListenAddressAndPort(), localInfo.getListenAddressAndPort())
+               && Objects.equals(getNativeProtocolVersion(), localInfo.getNativeProtocolVersion())
+               && Objects.equals(getPartitionerClass(), localInfo.getPartitionerClass())
+               && Objects.equals(getTruncationRecords(), localInfo.getTruncationRecords());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(super.hashCode(),
+                            getBroadcastAddressAndPort(),
+                            getBootstrapState(),
+                            getClusterName(),
+                            getCqlVersion(),
+                            getListenAddressAndPort(),
+                            getNativeProtocolVersion(),
+                            getPartitionerClass(),
+                            getTruncationRecords());
+    }
+
+    @Override
+    public String toString()
+    {
+        return new ToStringBuilder(this)
+        .appendSuper(super.toString())
+        .append("broadcastAddress", getBroadcastAddressAndPort())
+        .append("bootstrapState", getBootstrapState())
+        .append("clusterName", getClusterName())
+        .append("cqlVersion", getCqlVersion())
+        .append("listenAddress", getListenAddressAndPort())
+        .append("nativeProtocolVersion", getNativeProtocolVersion())
+        .append("partitioner", getPartitionerClass())
+        .append("truncationRecords", getTruncationRecords())
+        .toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/nodes/NodeInfo.java b/src/java/org/apache/cassandra/nodes/NodeInfo.java
new file mode 100644
index 000000000000..6bbe44d8aa92
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/NodeInfo.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.nodes;
+
+import java.net.InetAddress;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.UUID;
+import javax.annotation.Nonnull;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.CassandraVersion;
+
+@SuppressWarnings("unchecked")
+public abstract class NodeInfo<T extends NodeInfo<T>> implements INodeInfo<T>
+{
+    private volatile UUID hostId;
+    private volatile String dataCenter;
+    private volatile String rack;
+    private volatile CassandraVersion releaseVersion;
+    private volatile UUID schemaVersion;
+
+    @Nonnull
+    private volatile ImmutableSet<Token> tokens = ImmutableSet.of();
+
+    private volatile InetAddressAndPort nativeTransportAddressAndPort;
+
+    @Override
+    public UUID getHostId()
+    {
+        return hostId;
+    }
+
+    public T setHostId(UUID hostId)
+    {
+        this.hostId = hostId;
+        return (T) this;
+    }
+
+    @Override
+    public String getDataCenter()
+    {
+        return dataCenter;
+    }
+
+    public T setDataCenter(String dataCenter)
+    {
+        this.dataCenter = dataCenter;
+        return (T) this;
+    }
+
+    @Override
+    public String getRack()
+    {
+        return rack;
+    }
+
+    public T setRack(String rack)
+    {
+        this.rack = rack;
+        return (T) this;
+    }
+
+    @Override
+    public CassandraVersion getReleaseVersion()
+    {
+        return releaseVersion;
+    }
+
+    public T setReleaseVersion(CassandraVersion releaseVersion)
+    {
+        this.releaseVersion = releaseVersion;
+        return (T) this;
+    }
+
+    @Override
+    public UUID getSchemaVersion()
+    {
+        return schemaVersion;
+    }
+
+    public T setSchemaVersion(UUID schemaVersion)
+    {
+        this.schemaVersion = schemaVersion;
+        return (T) this;
+    }
+
+    @Override
+    public @Nonnull
+    Collection<Token> getTokens()
+    {
+        return tokens;
+    }
+
+    public T setTokens(@Nonnull Iterable<Token> tokens)
+    {
+        Preconditions.checkNotNull(tokens);
+        this.tokens = ImmutableSet.copyOf(tokens);
+        return (T) this;
+    }
+
+    @Override
+    public InetAddressAndPort getNativeTransportAddressAndPort()
+    {
+        return nativeTransportAddressAndPort;
+    }
+
+    public T setNativeTransportAddressAndPort(InetAddressAndPort nativeTransportAddressAndPort)
+    {
+        this.nativeTransportAddressAndPort = nativeTransportAddressAndPort;
+        return (T) this;
+    }
+
+    public T setNativeTransportAddressOnly(InetAddress address, int defaultPort)
+    {
+        this.nativeTransportAddressAndPort = getAddressAndPort(getNativeTransportAddressAndPort(), address, defaultPort);
+        return (T) this;
+    }
+
+    InetAddressAndPort getAddressAndPort(InetAddressAndPort current, InetAddress newAddress, int defaultPort)
+    {
+        if (newAddress == null)
+        {
+            return null;
+        }
+        else
+        {
+            int port = current != null && current.getPort() > 0 ? current.getPort() : defaultPort;
+            return InetAddressAndPort.getByAddressOverrideDefaults(newAddress, port);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (!(o instanceof NodeInfo<?>)) return false;
+        NodeInfo<?> nodeInfo = (NodeInfo<?>) o;
+        return Objects.equals(getHostId(), nodeInfo.getHostId())
+               && Objects.equals(getDataCenter(), nodeInfo.getDataCenter())
+               && Objects.equals(getRack(), nodeInfo.getRack())
+               && Objects.equals(getReleaseVersion(), nodeInfo.getReleaseVersion())
+               && Objects.equals(getSchemaVersion(), nodeInfo.getSchemaVersion())
+               && Objects.equals(getTokens(), nodeInfo.getTokens())
+               && Objects.equals(getNativeTransportAddressAndPort(), nodeInfo.getNativeTransportAddressAndPort());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(getHostId(),
+                            getDataCenter(),
+                            getRack(),
+                            getReleaseVersion(),
+                            getSchemaVersion(),
+                            getTokens(),
+                            getNativeTransportAddressAndPort());
+    }
+
+    @Override
+    public String toString()
+    {
+        return new ToStringBuilder(this)
+        .append("hostId", getHostId())
+        .append("dataCenter", getDataCenter())
+        .append("rack", getRack())
+        .append("releaseVersion", getReleaseVersion())
+        .append("schemaVersion", getSchemaVersion())
+        .append("tokens", getTokens())
+        .append("nativeTransportAddress", getNativeTransportAddressAndPort())
+        .toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/nodes/Nodes.java b/src/java/org/apache/cassandra/nodes/Nodes.java
new file mode 100644
index 000000000000..d11d73e3944c
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/Nodes.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import java.util.Collection;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.cliffc.high_scale_lib.NonBlockingHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODES_PERSISTENCE_CLASS;
+
+/**
+ * Provides access and updates the locally stored information about this and other nodes. The information is cached in
+ * memory in a thread-safe way and additionally stored using the provided implementation of {@link INodesPersistence},
+ * which is {@link NodesPersistence} by default and stores everything in system keyspace.
+ */
+public class Nodes
+{
+    private static final Logger logger = LoggerFactory.getLogger(Nodes.class);
+
+    @VisibleForTesting
+    private final ExecutorService updateExecutor;
+
+    private final INodesPersistence nodesPersistence;
+
+    private final Peers peers;
+    private final Local local;
+
+    private static class InstanceHolder
+    {
+        // put into subclass for lazy initialization
+        private static final Nodes instance = new Nodes();
+    }
+
+    /**
+     * Returns the singleton instance
+     */
+    public static Nodes getInstance()
+    {
+        return InstanceHolder.instance;
+    }
+
+    /**
+     * Returns the singleton instance of {@link Peers}
+     */
+    public static Peers peers()
+    {
+        return getInstance().getPeers();
+    }
+
+    /**
+     * Returns the singleton instance of {@link Local}
+     */
+    public static Local local()
+    {
+        return getInstance().getLocal();
+    }
+
+    /**
+     * Returns information about the node with the given address - if the node address matches the local (broadcast)
+     * address, the returned object is a {@link LocalInfo}. Otherwise, it is {@link PeerInfo} (or {@code null} if no
+     * informatino is available).
+     */
+    @Nullable
+    public static INodeInfo<?> localOrPeerInfo(InetAddressAndPort endpoint)
+    {
+        return Objects.equals(endpoint, FBUtilities.getBroadcastAddressAndPort()) ? local().get() : peers().get(endpoint);
+    }
+
+    public static Optional<INodeInfo<?>> localOrPeerInfoOpt(InetAddressAndPort endpoint)
+    {
+        return Optional.ofNullable(localOrPeerInfo(endpoint));
+    }
+
+    /**
+     * @see #updateLocalOrPeer(InetAddressAndPort, UnaryOperator, boolean, boolean)
+     */
+    public static INodeInfo<?> updateLocalOrPeer(InetAddressAndPort endpoint, UnaryOperator<NodeInfo<?>> update)
+    {
+        return updateLocalOrPeer(endpoint, update, false);
+    }
+
+    /**
+     * @see #updateLocalOrPeer(InetAddressAndPort, UnaryOperator, boolean, boolean)
+     */
+    public static INodeInfo<?> updateLocalOrPeer(InetAddressAndPort endpoint, UnaryOperator<NodeInfo<?>> update, boolean blocking)
+    {
+        return updateLocalOrPeer(endpoint, update, blocking, false);
+    }
+
+    /**
+     * Updates either local or peer information in a thread-safe way, depeending on whether the provided address matches
+     * the local (broadcast) address.
+     *
+     * @see Local#updateLocalOrPeer(InetAddressAndPort, UnaryOperator, boolean, boolean)
+     * @see Peers#updateLocalOrPeer(InetAddressAndPort, UnaryOperator, boolean, boolean)
+     */
+    public static INodeInfo<?> updateLocalOrPeer(InetAddressAndPort endpoint, UnaryOperator<NodeInfo<?>> update, boolean blocking, boolean force)
+    {
+        if (Objects.equals(endpoint, FBUtilities.getBroadcastAddressAndPort()))
+            return local().update(info -> (LocalInfo) update.apply(info), blocking, force);
+        else
+            return peers().update(endpoint, info -> (PeerInfo) update.apply(info), blocking, force);
+    }
+
+    public void forcePersist()
+    {
+        local().forcePersist();
+        peers().forcePersist();
+    }
+
+    /**
+     * Checks whether the provided address is local or known peer address.
+     */
+    public static boolean isKnownEndpoint(InetAddressAndPort endpoint)
+    {
+        return localOrPeerInfo(endpoint) != null;
+    }
+
+    public static UUID getHostId(InetAddressAndPort endpoint, UUID defaultValue)
+    {
+        INodeInfo<?> info = localOrPeerInfo(endpoint);
+        return info != null ? info.getHostId() : defaultValue;
+    }
+
+    public static String getDataCenter(InetAddressAndPort endpoint, String defaultValue)
+    {
+        INodeInfo<?> info = localOrPeerInfo(endpoint);
+        return info != null ? info.getDataCenter() : defaultValue;
+    }
+
+    public static String getRack(InetAddressAndPort endpoint, String defaultValue)
+    {
+        INodeInfo<?> info = localOrPeerInfo(endpoint);
+        return info != null ? info.getRack() : defaultValue;
+    }
+
+    public static CassandraVersion getReleaseVersion(InetAddressAndPort endpoint, CassandraVersion defaultValue)
+    {
+        INodeInfo<?> info = localOrPeerInfo(endpoint);
+        return info != null ? info.getReleaseVersion() : defaultValue;
+    }
+
+    public static UUID getSchemaVersion(InetAddressAndPort endpoint, UUID defaultValue)
+    {
+        INodeInfo<?> info = localOrPeerInfo(endpoint);
+        return info != null ? info.getSchemaVersion() : defaultValue;
+    }
+
+    public static Collection<Token> getTokens(InetAddressAndPort endpoint, Collection<Token> defaultValue)
+    {
+        INodeInfo<?> info = localOrPeerInfo(endpoint);
+        return info != null ? info.getTokens() : defaultValue;
+    }
+
+    public static InetAddressAndPort getNativeTransportAddressAndPort(InetAddressAndPort endpoint, InetAddressAndPort defaultValue)
+    {
+        INodeInfo<?> info = localOrPeerInfo(endpoint);
+        return info != null ? info.getNativeTransportAddressAndPort() : defaultValue;
+    }
+
+    /**
+     * Initializes singleton instance of {@link Nodes}. If it is not a Cassandra server process or
+     * {@code cassandra.nodes.disablePersitingToSystemKeyspace} is set to {@code true}, the instance does not persist
+     * stored information anywhere.
+     */
+    private Nodes()
+    {
+        this(createNodesPersistence(), ExecutorFactory.Global.executorFactory().sequential("nodes-info-persistence"));
+    }
+
+    private static INodesPersistence createNodesPersistence()
+    {
+        String nodesPersistenceClassName =  NODES_PERSISTENCE_CLASS.getString();
+        if (nodesPersistenceClassName != null)
+        {
+            try
+            {
+                return FBUtilities.instanceOrConstruct(nodesPersistenceClassName, "INodesPersistence implementation (" + NODES_PERSISTENCE_CLASS.getKey() + ")");
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException("Failed to instantiate " + nodesPersistenceClassName, e);
+            }
+        }
+        return !DatabaseDescriptor.isDaemonInitialized() || NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE.getBoolean()
+               ? INodesPersistence.NO_NODES_PERSISTENCE
+               : new NodesPersistence();
+    }
+
+    @VisibleForTesting
+    public Nodes(INodesPersistence nodesPersistence, ExecutorService updateExecutor)
+    {
+        this.updateExecutor = updateExecutor;
+        this.nodesPersistence = nodesPersistence;
+        this.local = new Local().load();
+        this.peers = new Peers().load();
+    }
+
+    public void reload()
+    {
+        this.local.load();
+        this.peers.load();
+    }
+
+    public Peers getPeers()
+    {
+        return peers;
+    }
+
+    public Local getLocal()
+    {
+        return local;
+    }
+
+    private Runnable wrapPersistenceTask(String name, Runnable task)
+    {
+        return () -> {
+            try
+            {
+                task.run();
+            }
+            catch (RuntimeException ex)
+            {
+                logger.error("Unexpected exception - " + name, ex);
+                throw ex;
+            }
+        };
+    }
+
+    public class Peers
+    {
+        private final NonBlockingHashMap<InetAddressAndPort, PeerInfo> internalMap = new NonBlockingHashMap<>();
+
+        public void forcePersist()
+        {
+            for (PeerInfo info : internalMap.values())
+            {
+                internalMap.computeIfPresent(info.getPeerAddressAndPort(), (key, existingPeerInfo) -> {
+                    if (existingPeerInfo.isExisting())
+                        save(existingPeerInfo, existingPeerInfo, true, true);
+                    else
+                        delete(info.getPeerAddressAndPort(), true);
+                    return existingPeerInfo;
+                });
+            }
+        }
+
+        public IPeerInfo update(InetAddressAndPort peer, UnaryOperator<PeerInfo> update)
+        {
+            return update(peer, update, false);
+        }
+
+        public IPeerInfo update(InetAddressAndPort peer, UnaryOperator<PeerInfo> update, boolean blocking)
+        {
+            return update(peer, update, blocking, false);
+        }
+
+        /**
+         * Updates peer information in a thread-safe way.
+         *
+         * @param peer     address of a peer to be updated
+         * @param update   update function, which receives a copy of the current {@link PeerInfo} and is expected to
+         *                 return the updated {@link PeerInfo}; the function may apply updates directly on the received
+         *                 copy and return it
+         * @param blocking if set, the method will block until the changes are persisted
+         * @param force    the update will be persisted even if no changes are made
+         * @return the updated object
+         */
+        public IPeerInfo update(InetAddressAndPort peer, UnaryOperator<PeerInfo> update, boolean blocking, boolean force)
+        {
+            return internalMap.compute(peer, (key, existingPeerInfo) -> {
+                PeerInfo updated = existingPeerInfo == null
+                                   ? update.apply(new PeerInfo().setPeerAddressAndPort(peer))
+                                   : update.apply(existingPeerInfo.duplicate()); // since we operate on mutable objects, we don't want to let the update function to operate on the live object
+
+                if (updated.getPeerAddressAndPort() == null)
+                    updated.setPeerAddressAndPort(peer);
+                else
+                    Preconditions.checkArgument(Objects.equals(updated.getPeerAddressAndPort(), peer));
+
+                updated.setRemoved(false);
+                save(existingPeerInfo, updated, blocking, force);
+                return updated;
+            });
+        }
+
+        /**
+         * @param peer peer to remove
+         * @param blocking block until the removal is persisted and synced
+         * @param hard remove also the transient state instead of just setting {@link PeerInfo#isRemoved()} state
+         * @return the remove
+         */
+        public IPeerInfo remove(InetAddressAndPort peer, boolean blocking, boolean hard)
+        {
+            AtomicReference<PeerInfo> removed = new AtomicReference<>();
+            internalMap.computeIfPresent(peer, (key, existingPeerInfo) -> {
+                delete(peer, blocking);
+                existingPeerInfo.setRemoved(true);
+                removed.set(existingPeerInfo);
+                return hard ? null : existingPeerInfo;
+            });
+            return removed.get();
+        }
+
+        /**
+         * Returns a peer information for a given address if the peer is known. Otherwise, returns {@code null}.
+         * Note that you should never try to manually cast the returned object to a mutable instnace and modify it.
+         */
+        @Nullable
+        public IPeerInfo get(InetAddressAndPort peer)
+        {
+            return internalMap.get(peer);
+        }
+
+        /**
+         * Returns optional of a peer information for a given address.
+         * Note that you should never try to manually cast the returned object to a mutable instnace and modify it.
+         */
+        public Optional<IPeerInfo> getOpt(InetAddressAndPort peer)
+        {
+            return Optional.ofNullable(get(peer));
+        }
+
+        /**
+         * Returns a stream of all known peers.
+         * Note that you should never try to manually cast the returned objects to a mutable instnaces and modify it.
+         */
+        public Stream<IPeerInfo> get()
+        {
+            return internalMap.values().stream().map(IPeerInfo.class::cast);
+        }
+
+        private void save(PeerInfo previousInfo, PeerInfo newInfo, boolean blocking, boolean force)
+        {
+            if (!force && Objects.equals(previousInfo, newInfo))
+            {
+                logger.trace("Saving peer skipped: {}", previousInfo);
+                return;
+            }
+
+            logger.trace("Saving peer: {}, blocking = {}, force = {}", newInfo, blocking, force);
+            Future<?> f = updateExecutor.submit(wrapPersistenceTask("saving peer information: " + newInfo, () -> {
+                nodesPersistence.savePeer(newInfo);
+                logger.trace("Saved peer: {}", newInfo);
+                if (blocking)
+                    nodesPersistence.syncPeers();
+            }));
+            if (blocking)
+                FBUtilities.waitOnFuture(f);
+        }
+
+        private Peers load()
+        {
+            logger.trace("Loading peers...");
+            nodesPersistence.loadPeers().forEach(info -> internalMap.compute(info.getPeerAddressAndPort(), (key, existingPeerInfo) -> info));
+            if (logger.isTraceEnabled())
+                logger.trace("Loaded peers: {}", internalMap.values().stream().collect(Collectors.toList()));
+            return this;
+        }
+
+        private void delete(InetAddressAndPort peer, boolean blocking)
+        {
+            if (logger.isTraceEnabled())
+                logger.trace("Deleting peer " + peer + ", blocking = " + blocking, new Throwable());
+            Future<?> f = updateExecutor.submit(wrapPersistenceTask("deleting peer information: " + peer, () -> {
+                nodesPersistence.deletePeer(peer);
+                logger.trace("Deleted peer {}", peer);
+                if (blocking)
+                    nodesPersistence.syncPeers();
+            }));
+
+            if (blocking)
+                FBUtilities.waitOnFuture(f);
+        }
+    }
+
+    public class Local
+    {
+        private final NonBlockingHashMap<InetAddressAndPort, LocalInfo> internalMap = new NonBlockingHashMap<>();
+        private final InetAddressAndPort localInfoKey = InetAddressAndPort.getLoopbackAddress();
+
+        public void forcePersist()
+        {
+            internalMap.computeIfPresent(localInfoKey, (key, existingLocalInfo) -> {
+                save(existingLocalInfo, existingLocalInfo, true, true);
+                return existingLocalInfo;
+            });
+        }
+
+        /**
+         * @see #update(UnaryOperator, boolean, boolean)
+         */
+        public ILocalInfo update(UnaryOperator<LocalInfo> update)
+        {
+            return update(update, false);
+        }
+
+        /**
+         * @see #update(UnaryOperator, boolean, boolean)
+         */
+        public ILocalInfo update(UnaryOperator<LocalInfo> update, boolean blocking)
+        {
+            return update(update, blocking, false);
+        }
+
+        /**
+         * Updates local node information in a thread-safe way.
+         *
+         * @param update   update function, which receives a copy of the current {@link LocalInfo} and is expected to
+         *                 return the updated {@link LocalInfo}; the function may apply updates directly on the received
+         *                 copy and return it
+         * @param blocking if set, the method will block until the changes are persisted
+         * @param force    the update will be persisted even if no changes are made
+         * @return a copy of updated object
+         */
+        public ILocalInfo update(UnaryOperator<LocalInfo> update, boolean blocking, boolean force)
+        {
+            return internalMap.compute(localInfoKey, (key, existingLocalInfo) -> {
+                LocalInfo updated = existingLocalInfo == null
+                                    ? update.apply(new LocalInfo())
+                                    : update.apply(existingLocalInfo.duplicate()); // since we operate on mutable objects, we don't want to let the update function to operate on the live object
+                save(existingLocalInfo, updated, blocking, force);
+                return updated;
+            });
+        }
+
+        /**
+         * Returns information about the local node (if present).
+         * Note that you should never try to manually cast the returned object to a mutable instnace and modify it.
+         */
+        public ILocalInfo get()
+        {
+            return internalMap.get(localInfoKey);
+        }
+
+        private void save(LocalInfo previousInfo, LocalInfo newInfo, boolean blocking, boolean force)
+        {
+            if (!force && Objects.equals(previousInfo, newInfo))
+            {
+                logger.trace("Saving local skipped: {}", previousInfo);
+                return;
+            }
+
+            Future<?> f = updateExecutor.submit(wrapPersistenceTask("saving local node information: " + newInfo, () -> {
+                nodesPersistence.saveLocal(newInfo);
+                logger.trace("Saving local: {}, blocking = {}, force = {}", newInfo, blocking, force);
+                if (blocking)
+                    nodesPersistence.syncLocal();
+            }));
+
+            if (blocking)
+                FBUtilities.waitOnFuture(f);
+        }
+
+        private Local load()
+        {
+            logger.trace("Loading local...");
+            internalMap.compute(localInfoKey, (key, existingLocalInfo) -> {
+                LocalInfo info = nodesPersistence.loadLocal();
+                return info != null ? info : new LocalInfo();
+            });
+            if (logger.isTraceEnabled())
+                logger.trace("Loaded local: {}", internalMap.get(localInfoKey));
+            return this;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/nodes/NodesPersistence.java b/src/java/org/apache/cassandra/nodes/NodesPersistence.java
new file mode 100644
index 000000000000..4853a22de927
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/NodesPersistence.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.UntypedResultSet.Row;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.Throwables;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.db.SystemKeyspace.LEGACY_PEERS;
+import static org.apache.cassandra.db.SystemKeyspace.LOCAL;
+import static org.apache.cassandra.db.SystemKeyspace.PEERS_V2;
+import static org.apache.cassandra.db.SystemKeyspace.forceBlockingFlush;
+
+public class NodesPersistence implements INodesPersistence
+{
+    private static final Logger logger = LoggerFactory.getLogger(NodesPersistence.class);
+
+    private static final List<String> COMMON_COLUMNS = ImmutableList.of("data_center",
+                                                                        "rack",
+                                                                        "host_id",
+                                                                        "release_version",
+                                                                        "schema_version",
+                                                                        "tokens");
+
+    private static final String INSERT_LOCAL_STMT = format("INSERT INTO system.%s (" +
+                                                           "     key, " +
+                                                           "     bootstrapped, " +
+                                                           "     broadcast_address, " +
+                                                           "     broadcast_port, " +
+                                                           "     cluster_name, " +
+                                                           "     cql_version, " +
+                                                           "     listen_address, " +
+                                                           "     listen_port, " +
+                                                           "     native_protocol_version, " +
+                                                           "     partitioner, " +
+                                                           "     rpc_address, " +
+                                                           "     rpc_port, " +
+                                                           "     truncated_at, " +
+                                                           "     %s) " +
+                                                           "VALUES (%s)", LOCAL,
+                                                           StringUtils.join(COMMON_COLUMNS, ", "),
+                                                           StringUtils.repeat("?", ", ", 13 + COMMON_COLUMNS.size()));
+
+    private static final String INSERT_PEER_STMT = format("INSERT INTO system.%s (" +
+                                                          "     peer, " +
+                                                          "     peer_port, " +
+                                                          "     preferred_ip, " +
+                                                          "     preferred_port, " +
+                                                          "     native_address, " +
+                                                          "     native_port, " +
+                                                          "     %s) " +
+                                                          "VALUES (%s)", PEERS_V2,
+                                                          StringUtils.join(COMMON_COLUMNS, ", "),
+                                                          StringUtils.repeat("?", ", ", 6 + COMMON_COLUMNS.size()));
+
+    private static final String INSERT_LEGACY_PEER_STMT = format("INSERT INTO system.%s (" +
+                                                                 "     peer, " +
+                                                                 "     preferred_ip, " +
+                                                                 "     rpc_address, " +
+                                                                 "     %s) " +
+                                                                 "VALUES (%s)", LEGACY_PEERS,
+                                                                 StringUtils.join(COMMON_COLUMNS, ", "),
+                                                                 StringUtils.repeat("?", ", ", 3 + COMMON_COLUMNS.size()));
+
+    private static final String DELETE_PEER_STMT = format("DELETE FROM system.%s " +
+                                                          "WHERE peer = ? AND peer_port = ?", PEERS_V2);
+
+    private static final String DELETE_LEGACY_PEER_STMT = format("DELETE FROM system.%s " +
+                                                                 "WHERE peer = ?", LEGACY_PEERS);
+
+
+    @Override
+    public LocalInfo loadLocal()
+    {
+        UntypedResultSet results = QueryProcessor.executeInternal("SELECT * FROM system." + LOCAL + " WHERE key = ?", LOCAL);
+        if (results == null || results.isEmpty())
+            return null;
+        Row row = results.one();
+        LocalInfo info = readCommonInfo(new LocalInfo(), row);
+        info.setBroadcastAddressAndPort(readInetAddressAndPort(row, "broadcast_address", "broadcast_port", DatabaseDescriptor.getStoragePort()))
+            .setListenAddressAndPort(readInetAddressAndPort(row, "listen_address", "listen_port", DatabaseDescriptor.getStoragePort()))
+            .setNativeTransportAddressAndPort(readInetAddressAndPort(row, "rpc_address", "rpc_port", DatabaseDescriptor.getNativeTransportPort()))
+            .setBootstrapState(readBootstrapState(row, "bootstrapped"))
+            .setClusterName(row.has("cluster_name") ? row.getString("cluster_name") : null)
+            .setCqlVersion(readCassandraVersion(row, "cql_version"))
+            .setNativeProtocolVersion(readNativeProtocol(row, "native_protocol_version"))
+            .setPartitionerClass(readPartitionerClass(row, "partitioner"))
+            .setTruncationRecords(readTruncationRecords(row, "truncated_at"));
+        return info;
+    }
+
+    @Override
+    public void saveLocal(LocalInfo info)
+    {
+        Object[] values = ArrayUtils.addAll(new Object[]{ "local",
+                                                          info.getBootstrapState() != null ? info.getBootstrapState().name() : null,
+                                                          serializeAddress(info.getBroadcastAddressAndPort()),
+                                                          serializePort(info.getBroadcastAddressAndPort()),
+                                                          info.getClusterName(),
+                                                          serializeCassandraVersion(info.getCqlVersion()),
+                                                          serializeAddress(info.getListenAddressAndPort()),
+                                                          serializePort(info.getListenAddressAndPort()),
+                                                          serializeProtocolVersion(info.getNativeProtocolVersion()),
+                                                          info.getPartitionerClass() != null ? info.getPartitionerClass().getName() : null,
+                                                          serializeAddress(info.getNativeTransportAddressAndPort()),
+                                                          serializePort(info.getNativeTransportAddressAndPort()),
+                                                          serializeTruncationRecords(info.getTruncationRecords()) }, serializeCommonInfo(info));
+
+        QueryProcessor.executeInternal(INSERT_LOCAL_STMT, values);
+    }
+
+    private String serializeProtocolVersion(ProtocolVersion protocolVersion)
+    {
+        return protocolVersion == null ? null : String.valueOf(protocolVersion.asInt());
+    }
+
+    @Override
+    public void syncLocal()
+    {
+        try
+        {
+            forceBlockingFlush(LOCAL);
+        }
+        catch (RejectedExecutionException ex)
+        {
+            logger.warn("Could not flush peers table because the thread pool has shut down", ex);
+        }
+    }
+
+    @Override
+    public Stream<PeerInfo> loadPeers()
+    {
+        UntypedResultSet results = QueryProcessor.executeInternal("SELECT * FROM system." + PEERS_V2);
+        return StreamSupport.stream(results.spliterator(), false).map(row -> {
+            PeerInfo info = readCommonInfo(new PeerInfo(), row);
+            info.setPeerAddressAndPort(readInetAddressAndPort(row, "peer", "peer_port", DatabaseDescriptor.getStoragePort()))
+                .setPreferredAddressAndPort(readInetAddressAndPort(row, "preferred_ip", "preferred_port", DatabaseDescriptor.getStoragePort()))
+                .setNativeTransportAddressAndPort(readInetAddressAndPort(row, "native_address", "native_port", DatabaseDescriptor.getNativeTransportPort()));
+            return info;
+        });
+    }
+
+    @Override
+    public void savePeer(PeerInfo info)
+    {
+        Object[] peersValues = ArrayUtils.addAll(new Object[]{ serializeAddress(info.getPeerAddressAndPort()),
+                                                               serializePort(info.getPeerAddressAndPort()),
+                                                               serializeAddress(info.getPreferredAddressAndPort()),
+                                                               serializePort(info.getPreferredAddressAndPort()),
+                                                               serializeAddress(info.getNativeTransportAddressAndPort()),
+                                                               serializePort(info.getNativeTransportAddressAndPort()),
+                                                               }, serializeCommonInfo(info));
+        QueryProcessor.executeInternal(INSERT_PEER_STMT, peersValues);
+
+        Object[] legacyPeersValues = ArrayUtils.addAll(new Object[]{ serializeAddress(info.getPeerAddressAndPort()),
+                                                                     serializeAddress(info.getPreferredAddressAndPort()),
+                                                                     serializeAddress(info.getNativeTransportAddressAndPort()),
+                                                                     }, serializeCommonInfo(info));
+
+        QueryProcessor.executeInternal(INSERT_LEGACY_PEER_STMT, legacyPeersValues);
+    }
+
+    @Override
+    public void deletePeer(InetAddressAndPort endpoint)
+    {
+        QueryProcessor.executeInternal(DELETE_PEER_STMT, serializeAddress(endpoint), serializePort(endpoint));
+        QueryProcessor.executeInternal(DELETE_LEGACY_PEER_STMT, serializeAddress(endpoint));
+    }
+
+    @Override
+    public void syncPeers()
+    {
+        try
+        {
+            forceBlockingFlush(LEGACY_PEERS, PEERS_V2);
+        }
+        catch (RejectedExecutionException ex)
+        {
+            logger.warn("Could not flush peers table because the thread pool has shut down", ex);
+        }
+    }
+
+    private <T extends NodeInfo<T>> T readCommonInfo(T info, Row row)
+    {
+        info.setDataCenter(row.has("data_center") ? row.getString("data_center") : null)
+            .setRack(row.has("rack") ? row.getString("rack") : null)
+            .setHostId(row.has("host_id") ? row.getUUID("host_id") : null)
+            .setReleaseVersion(readCassandraVersion(row, "release_version"))
+            .setSchemaVersion(row.has("schema_version") ? row.getUUID("schema_version") : null)
+            .setTokens(readTokens(row, "tokens"));
+        return info;
+    }
+
+    private InetAddressAndPort readInetAddressAndPort(Row row, String addressCol, String portCol, int defaultPort)
+    {
+        InetAddress address = row.has(addressCol) ? row.getInetAddress(addressCol) : null;
+        if (address == null)
+            return null;
+        int port = row.has(portCol) ? row.getInt(portCol) : defaultPort;
+        return InetAddressAndPort.getByAddressOverrideDefaults(address, port);
+    }
+
+    private CassandraVersion readCassandraVersion(Row row, String col)
+    {
+        String v = row.has(col) ? row.getString(col) : null;
+        if (v == null)
+            return null;
+        return new CassandraVersion(v);
+    }
+
+    private Collection<Token> readTokens(Row row, String col)
+    {
+        Set<String> tokensStrings = row.has(col) ? row.getSet(col, UTF8Type.instance) : new HashSet<>();
+        Token.TokenFactory factory = DatabaseDescriptor.getPartitioner().getTokenFactory();
+        List<Token> tokens = new ArrayList<>(tokensStrings.size());
+        for (String tk : tokensStrings)
+            tokens.add(factory.fromString(tk));
+        return tokens;
+    }
+
+    private SystemKeyspace.BootstrapState readBootstrapState(Row row, String col)
+    {
+        String s = row.has(col) ? row.getString(col) : null;
+        if (s == null)
+            return null;
+
+        return SystemKeyspace.BootstrapState.valueOf(s);
+    }
+
+    @SuppressWarnings("unchecked")
+    private Class<? extends IPartitioner> readPartitionerClass(Row row, String col)
+    {
+        String s = row.has(col) ? row.getString(col) : null;
+        if (s == null)
+            return null;
+
+        try
+        {
+            return (Class<? extends IPartitioner>) Class.forName(s);
+        }
+        catch (ClassNotFoundException e)
+        {
+            throw Throwables.unchecked(e);
+        }
+    }
+
+    private ProtocolVersion readNativeProtocol(Row row, String col)
+    {
+        String s = row.has(col) ? row.getString(col) : null;
+        if (s == null)
+            return null;
+
+        return ProtocolVersion.decode(Integer.parseInt(s), true);
+    }
+
+    private Map<UUID, TruncationRecord> readTruncationRecords(Row row, String col)
+    {
+        Map<UUID, ByteBuffer> raw = row.has(col) ? row.getMap(col, UUIDType.instance, BytesType.instance) : ImmutableMap.of();
+        if (raw == null)
+            return null;
+
+        return raw.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> truncationRecordFromBlob(e.getValue())));
+    }
+
+    private TruncationRecord truncationRecordFromBlob(ByteBuffer bytes)
+    {
+        try (RebufferingInputStream in = new DataInputBuffer(bytes, true))
+        {
+            return new TruncationRecord(CommitLogPosition.serializer.deserialize(in), in.available() > 0 ? in.readLong() : Long.MIN_VALUE);
+        }
+        catch (IOException e)
+        {
+            throw Throwables.unchecked(e);
+        }
+    }
+
+    public static int serializePort(InetAddressAndPort addressAndPort)
+    {
+        return addressAndPort != null ? addressAndPort.getPort() : -1;
+    }
+
+    public static InetAddress serializeAddress(InetAddressAndPort addressAndPort)
+    {
+        return addressAndPort != null ? addressAndPort.getAddress() : null;
+    }
+
+    public static String serializeCassandraVersion(CassandraVersion version)
+    {
+        return version != null ? version.toString() : null;
+    }
+
+    public static Set<String> serializeTokens(Collection<Token> tokens)
+    {
+        if (tokens.isEmpty())
+            return Collections.emptySet();
+        Set<String> s = new HashSet<>(tokens.size());
+        for (Token tk : tokens)
+            s.add(tk.getPartitioner().getTokenFactory().toString(tk));
+        return s;
+    }
+
+    private Object[] serializeCommonInfo(NodeInfo<?> info)
+    {
+        return new Object[]{ info.getDataCenter(),
+                             info.getRack(),
+                             info.getHostId(),
+                             serializeCassandraVersion(info.getReleaseVersion()),
+                             info.getSchemaVersion(),
+                             serializeTokens(info.getTokens()) };
+    }
+
+    public static ByteBuffer serializeTruncationRecord(TruncationRecord truncationRecord)
+    {
+        try (DataOutputBuffer out = DataOutputBuffer.scratchBuffer.get())
+        {
+            CommitLogPosition.serializer.serialize(truncationRecord.position, out);
+            out.writeLong(truncationRecord.truncatedAt);
+            return out.asNewBuffer();
+        }
+        catch (IOException e)
+        {
+            throw Throwables.unchecked(e);
+        }
+    }
+
+    public static Map<UUID, ByteBuffer> serializeTruncationRecords(Map<UUID, TruncationRecord> truncationRecords)
+    {
+        return truncationRecords.entrySet()
+                                .stream()
+                                .collect(Collectors.toMap(Map.Entry::getKey, e -> serializeTruncationRecord(e.getValue())));
+    }
+}
diff --git a/src/java/org/apache/cassandra/nodes/PeerInfo.java b/src/java/org/apache/cassandra/nodes/PeerInfo.java
new file mode 100644
index 000000000000..576246c5df37
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/PeerInfo.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.nodes;
+
+import java.util.Objects;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public final class PeerInfo extends NodeInfo<PeerInfo> implements IPeerInfo
+{
+    private volatile InetAddressAndPort peerAddressAndPort;
+    private volatile InetAddressAndPort preferredAddressAndPort;
+    private volatile boolean removed;
+
+    @Override
+    public InetAddressAndPort getPeerAddressAndPort()
+    {
+        return peerAddressAndPort;
+    }
+
+    public PeerInfo setPeerAddressAndPort(InetAddressAndPort peerAddressAndPort)
+    {
+        this.peerAddressAndPort = peerAddressAndPort;
+        return this;
+    }
+
+    @Override
+    public InetAddressAndPort getPreferredAddressAndPort()
+    {
+        return preferredAddressAndPort;
+    }
+
+    public PeerInfo setPreferredAddressAndPort(InetAddressAndPort preferredAddressAndPort)
+    {
+        this.preferredAddressAndPort = preferredAddressAndPort;
+        return this;
+    }
+
+    @Override
+    public boolean isRemoved()
+    {
+        return removed;
+    }
+
+    @Override
+    public boolean isExisting()
+    {
+        return !isRemoved();
+    }
+
+    PeerInfo setRemoved(boolean removed)
+    {
+        this.removed = removed;
+        return this;
+    }
+
+    @Override
+    public PeerInfo duplicate()
+    {
+        try
+        {
+            return (PeerInfo) clone();
+        }
+        catch (CloneNotSupportedException e)
+        {
+            throw new AssertionError(e);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (!(o instanceof PeerInfo)) return false;
+        if (!super.equals(o)) return false;
+        PeerInfo peerInfo = (PeerInfo) o;
+        return isRemoved() == peerInfo.isRemoved()
+               && Objects.equals(getPeerAddressAndPort(), peerInfo.getPeerAddressAndPort())
+               && Objects.equals(getPreferredAddressAndPort(), peerInfo.getPreferredAddressAndPort());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(super.hashCode(),
+                            getPeerAddressAndPort(),
+                            getPreferredAddressAndPort(),
+                            isRemoved());
+    }
+
+    @Override
+    public String toString()
+    {
+        return new ToStringBuilder(this)
+        .appendSuper(super.toString())
+        .append("peer", getPeerAddressAndPort())
+        .append("preferredIp", getPreferredAddressAndPort())
+        .append("isRemoved", isRemoved())
+        .toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/nodes/TruncationRecord.java b/src/java/org/apache/cassandra/nodes/TruncationRecord.java
new file mode 100644
index 000000000000..eb98159027d7
--- /dev/null
+++ b/src/java/org/apache/cassandra/nodes/TruncationRecord.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import java.util.Objects;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+
+public final class TruncationRecord
+{
+    public final CommitLogPosition position;
+    public final long truncatedAt;
+
+    public TruncationRecord(CommitLogPosition position, long truncatedAt)
+    {
+        this.position = position;
+        this.truncatedAt = truncatedAt;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TruncationRecord that = (TruncationRecord) o;
+        return truncatedAt == that.truncatedAt &&
+               Objects.equals(position, that.position);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(position, truncatedAt);
+    }
+
+
+    @Override
+    public String toString()
+    {
+        return new ToStringBuilder(this)
+        .append("position", position)
+        .append("truncatedAt", truncatedAt)
+        .toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java b/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
index 857af698473d..833bc51cfde5 100644
--- a/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
+++ b/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
@@ -21,8 +21,10 @@
 
 import javax.annotation.Nullable;
 
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
 
 /**
  * Notification sent after SSTables are added to their {@link org.apache.cassandra.db.ColumnFamilyStore}.
@@ -36,17 +38,39 @@ public class SSTableAddedNotification implements INotification
     @Nullable
     private final Memtable memtable;
 
+    /** The type of operation that created the sstables */
+    public final OperationType operationType;
+
+    /** The id of the operation that created the sstables, if available */
+    public final Optional<TimeUUID> operationId;
+
     /**
-     * Creates a new {@code SSTableAddedNotification} for the specified SSTables and optional memtable.
+     * Creates a new {@code SSTableAddedNotification} for the specified SSTables and optional memtable using
+     * an unknown operation type.
      *
      * @param added    the added SSTables
      * @param memtable the memtable from which the tables come when they have been added due to a memtable flush,
      *                 or {@code null} if they don't come from a flush
      */
     public SSTableAddedNotification(Iterable<SSTableReader> added, @Nullable Memtable memtable)
+    {
+        this(added, memtable, OperationType.UNKNOWN, Optional.empty());
+    }
+
+    /**
+     * Creates a new {@code SSTableAddedNotification} for the specified SSTables and optional memtable.
+     *
+     * @param added    the added SSTables
+     * @param memtable the memtable from which the tables come when they have been added due to a memtable flush,
+     *                 or {@code null} if they don't come from a flush
+     * @param operationType the type of operation that created the sstables
+     */
+    public SSTableAddedNotification(Iterable<SSTableReader> added, @Nullable Memtable memtable, OperationType operationType, Optional<TimeUUID> operationId)
     {
         this.added = added;
         this.memtable = memtable;
+        this.operationType = operationType;
+        this.operationId = operationId;
     }
 
     /**
@@ -59,4 +83,14 @@ public Optional<Memtable> memtable()
     {
         return Optional.ofNullable(memtable);
     }
+
+    /**
+     * @return true if curent notification is due to streaming sstables
+     */
+    public boolean fromStreaming()
+    {
+        return operationType == OperationType.STREAM
+               || operationType == OperationType.REGION_DECOMMISSION
+               || operationType == OperationType.REGION_REPAIR;
+    }
 }
diff --git a/src/java/org/apache/cassandra/notifications/SSTableAddingNotification.java b/src/java/org/apache/cassandra/notifications/SSTableAddingNotification.java
new file mode 100644
index 000000000000..fe56ff57dfe3
--- /dev/null
+++ b/src/java/org/apache/cassandra/notifications/SSTableAddingNotification.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.notifications;
+
+import java.util.Optional;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * Notification sent before SSTables are added to their {@link org.apache.cassandra.db.ColumnFamilyStore}.
+ */
+public class SSTableAddingNotification implements INotification
+{
+    /** The SSTables to be added*/
+    public final Iterable<SSTableReader> adding;
+
+    /** The memtable from which the sstables come when they need to be added due to a flush, {@code null} otherwise. */
+    @Nullable
+    private final Memtable memtable;
+
+    /** The type of operation that created the sstables */
+    public final OperationType operationType;
+
+    /** The id of the operation that created the sstables, if available */
+    public final Optional<TimeUUID> operationId;
+
+    /**
+     * Creates a new {@code SSTableAddingNotification} for the specified SSTables and optional memtable.
+     *
+     * @param adding    the SSTables to be added
+     * @param memtable the memtable from which the sstables come when they need to be added due to a memtable flush,
+     *                 or {@code null} if they don't come from a flush
+     * @param operationType the type of operation that created the sstables
+     * @param operationId the id of the operation (transaction) that created the sstables, or empty if no id is available
+     */
+    public SSTableAddingNotification(Iterable<SSTableReader> adding, @Nullable Memtable memtable, OperationType operationType, Optional<TimeUUID> operationId)
+    {
+        this.adding = adding;
+        this.memtable = memtable;
+        this.operationType = operationType;
+        this.operationId = operationId;
+    }
+
+    /**
+     * Returns the memtable from which the sstables come when they need to be addeddue to a memtable flush. If not, an
+     * empty Optional should be returned.
+     *
+     * @return the origin memtable in case of a flush, {@link Optional#empty()} otherwise
+     */
+    public Optional<Memtable> memtable()
+    {
+        return Optional.ofNullable(memtable);
+    }
+}
diff --git a/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java b/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
index 7ca574bf16f4..b6d0ed878da5 100644
--- a/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
+++ b/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
@@ -18,20 +18,24 @@
 package org.apache.cassandra.notifications;
 
 import java.util.Collection;
+import java.util.Optional;
 
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
 
 public class SSTableListChangedNotification implements INotification
 {
     public final Collection<SSTableReader> removed;
     public final Collection<SSTableReader> added;
-    public final OperationType compactionType;
+    public final OperationType operationType;
+    public final Optional<TimeUUID> operationId;
 
-    public SSTableListChangedNotification(Collection<SSTableReader> added, Collection<SSTableReader> removed, OperationType compactionType)
+    public SSTableListChangedNotification(Collection<SSTableReader> added, Collection<SSTableReader> removed, OperationType operationType, Optional<TimeUUID> operationId)
     {
         this.removed = removed;
         this.added = added;
-        this.compactionType = compactionType;
+        this.operationType = operationType;
+        this.operationId = operationId;
     }
 }
diff --git a/src/java/org/apache/cassandra/notifications/SSTableMetadataChanged.java b/src/java/org/apache/cassandra/notifications/SSTableMetadataChanged.java
deleted file mode 100644
index 83cfe60bc73c..000000000000
--- a/src/java/org/apache/cassandra/notifications/SSTableMetadataChanged.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.notifications;
-
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-
-public class SSTableMetadataChanged implements INotification
-{
-    public final SSTableReader sstable;
-    public final StatsMetadata oldMetadata;
-
-    public SSTableMetadataChanged(SSTableReader levelChanged, StatsMetadata oldMetadata)
-    {
-        this.sstable = levelChanged;
-        this.oldMetadata = oldMetadata;
-    }
-}
diff --git a/src/java/org/apache/cassandra/notifications/TruncationNotification.java b/src/java/org/apache/cassandra/notifications/TruncationNotification.java
index 345dd17e290c..271ae0d00bf5 100644
--- a/src/java/org/apache/cassandra/notifications/TruncationNotification.java
+++ b/src/java/org/apache/cassandra/notifications/TruncationNotification.java
@@ -17,16 +17,20 @@
  */
 package org.apache.cassandra.notifications;
 
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+
 /**
  * Fired during truncate, after the memtable has been flushed but before any
  * snapshot is taken and SSTables are discarded
  */
 public class TruncationNotification implements INotification
 {
+    public final CommitLogPosition replayAfter;
     public final long truncatedAt;
 
-    public TruncationNotification(long truncatedAt)
+    public TruncationNotification(CommitLogPosition replayAfter, long truncatedAt)
     {
+        this.replayAfter = replayAfter;
         this.truncatedAt = truncatedAt;
     }
 }
diff --git a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java
index 94cc3545c210..577c9eab1ce5 100644
--- a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java
@@ -71,6 +71,7 @@ private List<RepairSession> submitRepairSessions(TimeUUID parentSession,
                                                                                  keyspace,
                                                                                  options.getParallelism(),
                                                                                  isIncremental,
+                                                                                 options.isPushRepair(), 
                                                                                  options.isPullRepair(),
                                                                                  options.getPreviewKind(),
                                                                                  options.optimiseStreams(),
diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
index 379ba4b2a1b6..241232a19e69 100644
--- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
@@ -119,6 +119,7 @@ protected void startSync()
             Tracing.traceRepair(message);
 
             StreamPlan plan = createStreamPlan();
+            logger.info("{} {} {}", previewKind.logPrefix(desc.sessionId), "Starting streaming plan with id", plan.getPlanId());
             ctx.streamExecutor().execute(plan);
             planPromise.setSuccess(plan);
         }
diff --git a/src/java/org/apache/cassandra/repair/ParentRepairSessionListener.java b/src/java/org/apache/cassandra/repair/ParentRepairSessionListener.java
new file mode 100644
index 000000000000..72b3f02d2c5d
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/ParentRepairSessionListener.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair;
+
+import java.util.concurrent.Future;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.FBUtilities;
+
+public interface ParentRepairSessionListener
+{
+    ParentRepairSessionListener instance = CassandraRelevantProperties.REPAIR_PARENT_SESSION_LISTENER.isPresent()
+                                      ? FBUtilities.construct(CassandraRelevantProperties.REPAIR_PARENT_SESSION_LISTENER.getString(),
+                                                              "Parent Repair Session Listener")
+                                      : new NoopParentRepairSessionListener();
+
+    /**
+     * Call when parent repair session is registered
+     */
+    void onRegistered(TimeUUID sessionId, ActiveRepairService.ParentRepairSession session);
+
+    /**
+     * Call when parent repair session is removed
+     */
+    void onRemoved(TimeUUID sessionId, ActiveRepairService.ParentRepairSession session);
+
+    /**
+     * Call when validation task started for given repair session
+     */
+    void onValidation(RepairJobDesc desc, Future validationTask);
+
+    /**
+     * Call when sync task started for given repair session
+     */
+    void onSync(RepairJobDesc desc, Future syncTask);
+
+    static class NoopParentRepairSessionListener implements ParentRepairSessionListener
+    {
+        @Override
+        public void onRegistered(TimeUUID sessionId, ActiveRepairService.ParentRepairSession session)
+        {
+        }
+
+        @Override
+        public void onRemoved(TimeUUID sessionId, ActiveRepairService.ParentRepairSession session)
+        {
+        }
+
+        @Override
+        public void onValidation(RepairJobDesc desc, Future validationTask)
+        {
+        }
+
+        @Override
+        public void onSync(RepairJobDesc desc, Future syncTask)
+        {
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/repair/RepairCoordinator.java b/src/java/org/apache/cassandra/repair/RepairCoordinator.java
index 091819c65d7b..47ab33067fbf 100644
--- a/src/java/org/apache/cassandra/repair/RepairCoordinator.java
+++ b/src/java/org/apache/cassandra/repair/RepairCoordinator.java
@@ -17,10 +17,12 @@
  */
 package org.apache.cassandra.repair;
 
+import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -32,6 +34,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -39,6 +42,7 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.SettableFuture;
 
 import org.apache.cassandra.locator.RangesAtEndpoint;
 import org.apache.cassandra.net.Verb;
@@ -72,7 +76,6 @@
 import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.repair.state.CoordinatorState;
 import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.schema.SystemDistributedKeyspace;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus;
 import org.apache.cassandra.service.ClientState;
@@ -94,6 +97,7 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai
 {
     private static final Logger logger = LoggerFactory.getLogger(RepairCoordinator.class);
 
+    private final SettableFuture<?> result = SettableFuture.create();
     private static final AtomicInteger THREAD_COUNTER = new AtomicInteger(1);
 
     public final CoordinatorState state;
@@ -201,6 +205,8 @@ private void skip(String msg)
 
     private void success(String msg)
     {
+        result.set(null);
+
         state.phase.success(msg);
         fireProgressEvent(jmxEvent(ProgressEventType.SUCCESS, msg));
         ctx.repair().recordRepairStatus(state.cmd, ActiveRepairService.ParentRepairStatus.COMPLETED,
@@ -215,6 +221,8 @@ private void fail(String reason)
             Throwable error = firstError.get();
             reason = error != null ? error.toString() : "Some repair failed";
         }
+        result.setException(new RuntimeException(reason));
+
         state.phase.fail(reason);
         ParticipateState p = ctx.repair().participate(state.id);
         if (p != null)
@@ -268,6 +276,11 @@ private void complete(String msg)
         Keyspace.open(state.keyspace).metric.repairTime.update(durationMillis, TimeUnit.MILLISECONDS);
     }
 
+    public java.util.concurrent.Future<?> getResult()
+    {
+        return result;
+    }
+
     public void run()
     {
         try
@@ -348,7 +361,7 @@ private TraceState maybeCreateTraceState(Iterable<ColumnFamilyStore> columnFamil
         for (ColumnFamilyStore cfs : columnFamilyStores)
             cfsb.append(", ").append(cfs.getKeyspaceName()).append(".").append(cfs.name);
 
-        TimeUUID sessionId = Tracing.instance.newSession(Tracing.TraceType.REPAIR);
+        TimeUUID sessionId = Tracing.instance.newSession(ClientState.forInternalCalls(), Tracing.TraceType.REPAIR);
         TraceState traceState = Tracing.instance.begin("repair", ImmutableMap.of("keyspace", state.keyspace, "columnFamilies",
                                                                                  cfsb.substring(2)));
         traceState.enableActivityNotification(tag);
@@ -370,6 +383,10 @@ private void notifyStarting()
 
     private NeighborsAndRanges getNeighborsAndRanges() throws RepairException
     {
+        // if it's offline service, don't check token metadata and storage service.
+        if (state.options.isOfflineService())
+            return createNeighbordAndRangesForOfflineService(state.options);
+
         Set<InetAddressAndPort> allNeighbors = new HashSet<>();
         List<CommonRange> commonRanges = new ArrayList<>();
 
@@ -382,6 +399,7 @@ private NeighborsAndRanges getNeighborsAndRanges() throws RepairException
             EndpointsForRange neighbors = ctx.repair().getNeighbors(state.keyspace, keyspaceLocalRanges, range,
                                                                            state.options.getDataCenters(),
                                                                            state.options.getHosts());
+            // local RF = 1 or given range is not part of local range, neighbors would be empty.
             if (neighbors.isEmpty())
             {
                 if (state.options.ignoreUnreplicatedKeyspaces())
@@ -416,11 +434,41 @@ private NeighborsAndRanges getNeighborsAndRanges() throws RepairException
         return new NeighborsAndRanges(shouldExcludeDeadParticipants, allNeighbors, commonRanges);
     }
 
+    @VisibleForTesting
+    public static NeighborsAndRanges createNeighbordAndRangesForOfflineService(RepairOption options)
+    {
+        Preconditions.checkArgument(!options.getHosts().isEmpty(), "There should be at least 1 host when repairing via offline service");
+        Preconditions.checkArgument(!options.getRanges().isEmpty(), "Token ranges must be specified when repairing via offline service. " +
+                                                                    "Please specify at least one token range which all hosts have in common.");
+
+        Set<InetAddressAndPort> allNeighbors = new HashSet<>();
+        List<CommonRange> commonRanges = new ArrayList<>();
+
+        for (String host : options.getHosts())
+        {
+            try
+            {
+                InetAddressAndPort endpoint = InetAddressAndPort.getByName(host.trim());
+                if (!endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
+                    allNeighbors.add(endpoint);
+            }
+            catch (UnknownHostException e)
+            {
+                throw new IllegalArgumentException("Unknown host specified " + host, e);
+            }
+        }
+
+        Preconditions.checkArgument(!allNeighbors.isEmpty(), "There should be at least 1 neighbor when repairing via offline service");
+
+        commonRanges.add(new CommonRange(allNeighbors, Collections.emptySet(), options.getRanges()));
+        return new NeighborsAndRanges(false, allNeighbors, commonRanges);
+    }
+
     private void maybeStoreParentRepairStart(String[] cfnames)
     {
         if (!state.options.isPreview())
         {
-            SystemDistributedKeyspace.startParentRepair(state.id, state.keyspace, cfnames, state.options);
+            RepairProgressReporter.instance.onParentRepairStarted(state.id, state.keyspace, cfnames, state.options);
         }
     }
 
@@ -428,7 +476,7 @@ private void maybeStoreParentRepairSuccess(Collection<Range<Token>> successfulRa
     {
         if (!state.options.isPreview())
         {
-            SystemDistributedKeyspace.successfulParentRepair(state.id, successfulRanges);
+            RepairProgressReporter.instance.onParentRepairSucceeded(state.id, successfulRanges);
         }
     }
 
@@ -436,7 +484,7 @@ private void maybeStoreParentRepairFailure(Throwable error)
     {
         if (!state.options.isPreview())
         {
-            SystemDistributedKeyspace.failParentRepair(state.id, error);
+            RepairProgressReporter.instance.onParentRepairFailed(state.id, error);
         }
     }
 
diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java
index c54336a6b39d..2424a5982869 100644
--- a/src/java/org/apache/cassandra/repair/RepairJob.java
+++ b/src/java/org/apache/cassandra/repair/RepairJob.java
@@ -47,9 +47,8 @@
 import org.apache.cassandra.repair.asymmetric.HostDifferences;
 import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter;
 import org.apache.cassandra.repair.asymmetric.ReduceHelper;
-import org.apache.cassandra.schema.SystemDistributedKeyspace;
-import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup;
 import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MerkleTrees;
@@ -207,7 +206,7 @@ public void onSuccess(List<SyncStat> stats)
                 if (!session.previewKind.isPreview())
                 {
                     logger.info("{} {}.{} is fully synced", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily);
-                    SystemDistributedKeyspace.successfulRepairJob(session.getId(), desc.keyspace, desc.columnFamily);
+                    RepairProgressReporter.instance.onRepairSucceeded(session.getId(), desc.keyspace, desc.columnFamily);
                 }
                 cfs.metric.repairsCompleted.inc();
                 trySuccess(new RepairResult(desc, stats));
@@ -225,7 +224,7 @@ public void onFailure(Throwable t)
                 if (!session.previewKind.isPreview())
                 {
                     logger.warn("{} {}.{} sync failed", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily);
-                    SystemDistributedKeyspace.failedRepairJob(session.getId(), desc.keyspace, desc.columnFamily, t);
+                    RepairProgressReporter.instance.onRepairFailed(session.getId(), desc.keyspace, desc.columnFamily, t);
                 }
                 cfs.metric.repairsCompleted.inc();
                 tryFailure(t instanceof NoSuchRepairSessionExceptionWrapper
@@ -287,6 +286,7 @@ private List<SyncTask> createStandardSyncTasks(List<TreeResponse> trees)
                                        ctx.broadcastAddressAndPort(),
                                        this::isTransient,
                                        session.isIncremental,
+                                       session.pushRepair,
                                        session.pullRepair,
                                        session.previewKind);
     }
@@ -298,6 +298,7 @@ static List<SyncTask> createStandardSyncTasks(SharedContext ctx,
                                                   InetAddressAndPort local,
                                                   Predicate<InetAddressAndPort> isTransient,
                                                   boolean isIncremental,
+                                                  boolean pushRepair,
                                                   boolean pullRepair,
                                                   PreviewKind previewKind)
     {
@@ -321,14 +322,14 @@ static List<SyncTask> createStandardSyncTasks(SharedContext ctx,
                 if (differences.isEmpty())
                     continue;
 
-                SyncTask task;
+                SyncTask task = null;
                 if (r1.endpoint.equals(local) || r2.endpoint.equals(local))
                 {
                     TreeResponse self = r1.endpoint.equals(local) ? r1 : r2;
                     TreeResponse remote = r2.endpoint.equals(local) ? r1 : r2;
 
-                    // pull only if local is full
-                    boolean requestRanges = !isTransient.test(self.endpoint);
+                    // pull only if local is full; additionally check for push repair
+                    boolean requestRanges = !isTransient.test(self.endpoint) && !pushRepair;
                     // push only if remote is full; additionally check for pull repair
                     boolean transferRanges = !isTransient.test(remote.endpoint) && !pullRepair;
 
@@ -341,16 +342,19 @@ static List<SyncTask> createStandardSyncTasks(SharedContext ctx,
                 }
                 else if (isTransient.test(r1.endpoint) || isTransient.test(r2.endpoint))
                 {
+                    Preconditions.checkArgument(!pushRepair, "Push Repair doesn't support transient replica");
+
                     // Stream only from transient replica
                     TreeResponse streamFrom = isTransient.test(r1.endpoint) ? r1 : r2;
                     TreeResponse streamTo = isTransient.test(r1.endpoint) ? r2 : r1;
                     task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind);
                 }
-                else
+                else if (!pushRepair)
                 {
                     task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind);
                 }
-                syncTasks.add(task);
+                if (task != null)
+                    syncTasks.add(task);
             }
             trees.get(i).trees.release();
         }
@@ -403,6 +407,7 @@ private NoSuchRepairSessionExceptionWrapper(NoSuchRepairSessionException wrapped
 
     private List<SyncTask> createOptimisedSyncingSyncTasks(List<TreeResponse> trees)
     {
+        Preconditions.checkArgument(!session.pushRepair, "Push Repair doesn't support optimized sync");
         return createOptimisedSyncingSyncTasks(ctx,
                                                desc,
                                                trees,
diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
index 621aacc378da..39c693426885 100644
--- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
+++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.function.BiFunction;
 import java.util.function.Function;
+import java.util.concurrent.Future;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
@@ -52,6 +53,20 @@
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.TimeUUID;
 
+import static org.apache.cassandra.net.Verb.CLEANUP_MSG;
+import static org.apache.cassandra.net.Verb.FAILED_SESSION_MSG;
+import static org.apache.cassandra.net.Verb.FINALIZE_COMMIT_MSG;
+import static org.apache.cassandra.net.Verb.FINALIZE_PROMISE_MSG;
+import static org.apache.cassandra.net.Verb.FINALIZE_PROPOSE_MSG;
+import static org.apache.cassandra.net.Verb.PREPARE_CONSISTENT_REQ;
+import static org.apache.cassandra.net.Verb.PREPARE_CONSISTENT_RSP;
+import static org.apache.cassandra.net.Verb.PREPARE_MSG;
+import static org.apache.cassandra.net.Verb.SNAPSHOT_MSG;
+import static org.apache.cassandra.net.Verb.STATUS_REQ;
+import static org.apache.cassandra.net.Verb.STATUS_RSP;
+import static org.apache.cassandra.net.Verb.SYNC_REQ;
+import static org.apache.cassandra.net.Verb.VALIDATION_REQ;
+
 /**
  * Handles all repair related message.
  *
@@ -102,255 +117,245 @@ public void doVerb(final Message<RepairMessage> message)
         RepairJobDesc desc = message.payload.desc;
         try
         {
-            switch (message.verb())
+            if (message.verb() == PREPARE_MSG)
             {
-                case PREPARE_MSG:
+                PrepareMessage prepareMessage = (PrepareMessage) message.payload;
+                logger.debug("Preparing, {}", prepareMessage);
+                ParticipateState state = new ParticipateState(ctx.clock(), message.from(), prepareMessage);
+                if (!ctx.repair().register(state))
                 {
-                    PrepareMessage prepareMessage = (PrepareMessage) message.payload;
-                    logger.debug("Preparing, {}", prepareMessage);
-                    ParticipateState state = new ParticipateState(ctx.clock(), message.from(), prepareMessage);
-                    if (!ctx.repair().register(state))
-                    {
-                        replyDedup(ctx.repair().participate(state.id), message);
-                        return;
-                    }
-                    if (!ctx.repair().verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
-                    {
-                        // error is logged in verifyCompactionsPendingThreshold
-                        state.phase.fail("Too many pending compactions");
-
-                        sendFailureResponse(message);
-                        return;
-                    }
+                    replyDedup(ctx.repair().participate(state.id), message);
+                    return;
+                }
+                if (!ctx.repair().verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
+                {
+                    // error is logged in verifyCompactionsPendingThreshold
+                    state.phase.fail("Too many pending compactions");
 
-                    List<ColumnFamilyStore> columnFamilyStores = new ArrayList<>(prepareMessage.tableIds.size());
-                    for (TableId tableId : prepareMessage.tableIds)
-                    {
-                        ColumnFamilyStore columnFamilyStore = ColumnFamilyStore.getIfExists(tableId);
-                        if (columnFamilyStore == null)
-                        {
-                            String reason = String.format("Table with id %s was dropped during prepare phase of repair",
-                                                          tableId);
-                            state.phase.fail(reason);
-                            logErrorAndSendFailureResponse(reason, message);
-                            return;
-                        }
-                        columnFamilyStores.add(columnFamilyStore);
-                    }
-                    state.phase.accept();
-                    ctx.repair().registerParentRepairSession(prepareMessage.parentRepairSession,
-                                                                    message.from(),
-                                                                    columnFamilyStores,
-                                                                    prepareMessage.ranges,
-                                                                    prepareMessage.isIncremental,
-                                                                    prepareMessage.repairedAt,
-                                                                    prepareMessage.isGlobal,
-                                                                    prepareMessage.previewKind);
-                    sendAck(message);
+                    sendFailureResponse(message);
+                    return;
                 }
-                    break;
 
-                case SNAPSHOT_MSG:
+                List<ColumnFamilyStore> columnFamilyStores = new ArrayList<>(prepareMessage.tableIds.size());
+                for (TableId tableId : prepareMessage.tableIds)
                 {
-                    logger.debug("Snapshotting {}", desc);
-                    ParticipateState state = ctx.repair().participate(desc.parentSessionId);
-                    if (state == null)
-                    {
-                        logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
-                        return;
-                    }
-                    final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
-                    if (cfs == null)
+                    ColumnFamilyStore columnFamilyStore = ColumnFamilyStore.getIfExists(tableId);
+                    if (columnFamilyStore == null)
                     {
-                        String reason = String.format("Table %s.%s was dropped during snapshot phase of repair %s",
-                                                      desc.keyspace, desc.columnFamily, desc.parentSessionId);
+                        String reason = String.format("Table with id %s was dropped during prepare phase of repair",
+                                                      tableId);
                         state.phase.fail(reason);
                         logErrorAndSendFailureResponse(reason, message);
                         return;
                     }
+                    columnFamilyStores.add(columnFamilyStore);
+                }
+                state.phase.accept();
+                ctx.repair().registerParentRepairSession(prepareMessage.parentRepairSession,
+                                                                message.from(),
+                                                                columnFamilyStores,
+                                                                prepareMessage.ranges,
+                                                                prepareMessage.isIncremental,
+                                                                prepareMessage.repairedAt,
+                                                                prepareMessage.isGlobal,
+                                                                prepareMessage.previewKind);
+                sendAck(message);
+            }
+            else if (message.verb() == SNAPSHOT_MSG)
+            {
+                logger.debug("Snapshotting {}", desc);
+                ParticipateState state = ctx.repair().participate(desc.parentSessionId);
+                if (state == null)
+                {
+                    logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
+                    return;
+                }
+                final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
+                if (cfs == null)
+                {
+                    String reason = String.format("Table %s.%s was dropped during snapshot phase of repair %s",
+                                                  desc.keyspace, desc.columnFamily, desc.parentSessionId);
+                    state.phase.fail(reason);
+                    logErrorAndSendFailureResponse(reason, message);
+                    return;
+                }
 
-                    ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(desc.parentSessionId);
-                    if (prs.setHasSnapshots())
+                ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(desc.parentSessionId);
+                if (prs.setHasSnapshots())
+                {
+                    state.getOrCreateJob(desc).snapshot();
+                    TableRepairManager repairManager = cfs.getRepairManager();
+                    if (prs.isGlobal)
                     {
-                        state.getOrCreateJob(desc).snapshot();
-                        TableRepairManager repairManager = cfs.getRepairManager();
-                        if (prs.isGlobal)
-                        {
-                            repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false);
-                        }
-                        else
-                        {
-                            repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true);
-                        }
-                        logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from());
+                        repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false);
                     }
-                    sendAck(message);
+                    else
+                    {
+                        repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true);
+                    }
+                    logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from());
                 }
-                    break;
+                sendAck(message);
+            }
+            else if (message.verb() == VALIDATION_REQ)
+            {
+                ValidationRequest validationRequest = (ValidationRequest) message.payload;
+                logger.debug("Validating {}", validationRequest);
 
-                case VALIDATION_REQ:
+                ParticipateState participate = ctx.repair().participate(desc.parentSessionId);
+                if (participate == null)
                 {
-                    ValidationRequest validationRequest = (ValidationRequest) message.payload;
-                    logger.debug("Validating {}", validationRequest);
+                    logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
+                    return;
+                }
 
-                    ParticipateState participate = ctx.repair().participate(desc.parentSessionId);
-                    if (participate == null)
+                ValidationState vState = new ValidationState(ctx.clock(), desc, message.from());
+                if (!register(message, participate, vState,
+                              participate::register,
+                              (d, i) -> participate.validation(d)))
+                    return;
+                try
+                {
+                    // trigger read-only compaction
+                    ColumnFamilyStore store = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
+                    if (store == null)
                     {
-                        logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
+                        String msg = String.format("Table %s.%s was dropped during validation phase of repair %s", desc.keyspace, desc.columnFamily, desc.parentSessionId);
+                        vState.phase.fail(msg);
+                        logErrorAndSendFailureResponse(msg, message);
                         return;
                     }
 
-                    ValidationState vState = new ValidationState(ctx.clock(), desc, message.from());
-                    if (!register(message, participate, vState,
-                                  participate::register,
-                                  (d, i) -> participate.validation(d)))
-                        return;
                     try
                     {
-                        // trigger read-only compaction
-                        ColumnFamilyStore store = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
-                        if (store == null)
-                        {
-                            String msg = String.format("Table %s.%s was dropped during validation phase of repair %s", desc.keyspace, desc.columnFamily, desc.parentSessionId);
-                            vState.phase.fail(msg);
-                            logErrorAndSendFailureResponse(msg, message);
-                            return;
-                        }
-
-                        try
-                        {
-                            ctx.repair().consistent.local.maybeSetRepairing(desc.parentSessionId);
-                        }
-                        catch (Throwable t)
-                        {
-                            JVMStabilityInspector.inspectThrowable(t);
-                            vState.phase.fail(t.toString());
-                            logErrorAndSendFailureResponse(t.toString(), message);
-                            return;
-                        }
-                        PreviewKind previewKind;
-                        try
-                        {
-                            previewKind = previewKind(desc.parentSessionId);
-                        }
-                        catch (NoSuchRepairSessionException e)
-                        {
-                            logger.warn("Parent repair session {} has been removed, failing repair", desc.parentSessionId);
-                            vState.phase.fail(e);
-                            sendFailureResponse(message);
-                            return;
-                        }
-                        vState.phase.accept();
-                        sendAck(message);
-
-                        Validator validator = new Validator(ctx, vState, validationRequest.nowInSec,
-                                                            isIncremental(desc.parentSessionId), previewKind);
-                        if (acceptMessage(ctx, validationRequest, message.from()))
-                        {
-                            ctx.validationManager().submitValidation(store, validator);
-                        }
-                        else
-                        {
-                            validator.fail(new RepairOutOfTokenRangeException(validationRequest.desc.ranges));
-                        }
+                        ctx.repair().consistent.local.maybeSetRepairing(desc.parentSessionId);
                     }
                     catch (Throwable t)
                     {
-                        vState.phase.fail(t);
-                        throw t;
+                        JVMStabilityInspector.inspectThrowable(t);
+                        vState.phase.fail(t.toString());
+                        logErrorAndSendFailureResponse(t.toString(), message);
+                        return;
                     }
-                }
-                    break;
-
-                case SYNC_REQ:
-                {
-                    // forwarded sync request
-                    SyncRequest request = (SyncRequest) message.payload;
-                    logger.debug("Syncing {}", request);
-
-                    ParticipateState participate = ctx.repair().participate(desc.parentSessionId);
-                    if (participate == null)
+                    PreviewKind previewKind;
+                    try
                     {
-                        logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
-                        return;
+                        previewKind = previewKind(desc.parentSessionId);
                     }
-                    SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst);
-                    if (!register(message, participate, state,
-                                  participate::register,
-                                  participate::sync))
+                    catch (NoSuchRepairSessionException e)
+                    {
+                        logger.warn("Parent repair session {} has been removed, failing repair", desc.parentSessionId);
+                        vState.phase.fail(e);
+                        sendFailureResponse(message);
                         return;
-                    state.phase.accept();
-                    StreamingRepairTask task = new StreamingRepairTask(ctx, state, desc,
-                                                                       request.initiator,
-                                                                       request.src,
-                                                                       request.dst,
-                                                                       request.ranges,
-                                                                       isIncremental(desc.parentSessionId) ? desc.parentSessionId : null,
-                                                                       request.previewKind,
-                                                                       request.asymmetric);
-                    task.run();
+                    }
+                    vState.phase.accept();
                     sendAck(message);
-                }
-                    break;
 
-                case CLEANUP_MSG:
+                    Validator validator = new Validator(ctx, vState, validationRequest.nowInSec,
+                                                        isIncremental(desc.parentSessionId), previewKind);
+                    if (acceptMessage(ctx, validationRequest, message.from()))
+                    {
+                        Future<?> validationFuture = ctx.validationManager().submitValidation(store, validator);
+                        ParentRepairSessionListener.instance.onValidation(desc, validationFuture);
+                    }
+                    else
+                    {
+                        validator.fail(new RepairOutOfTokenRangeException(validationRequest.desc.ranges));
+                    }
+                }
+                catch (Throwable t)
                 {
-                    logger.debug("cleaning up repair");
-                    CleanupMessage cleanup = (CleanupMessage) message.payload;
-                    ParticipateState state = ctx.repair().participate(cleanup.parentRepairSession);
-                    if (state != null)
-                        state.phase.success("Cleanup message recieved");
-                    ctx.repair().removeParentRepairSession(cleanup.parentRepairSession);
-                    sendAck(message);
+                    vState.phase.fail(t);
+                    throw t;
                 }
-                    break;
-
-                case PREPARE_CONSISTENT_REQ:
-                    ctx.repair().consistent.local.handlePrepareMessage(message);
-                    break;
-
-                case PREPARE_CONSISTENT_RSP:
-                    ctx.repair().consistent.coordinated.handlePrepareResponse(message);
-                    break;
-
-                case FINALIZE_PROPOSE_MSG:
-                    ctx.repair().consistent.local.handleFinalizeProposeMessage(message);
-                    break;
-
-                case FINALIZE_PROMISE_MSG:
-                    ctx.repair().consistent.coordinated.handleFinalizePromiseMessage(message);
-                    break;
-
-                case FINALIZE_COMMIT_MSG:
-                    ctx.repair().consistent.local.handleFinalizeCommitMessage(message);
-                    break;
-
-                case FAILED_SESSION_MSG:
-                    FailSession failure = (FailSession) message.payload;
-                    sendAck(message);
-                    ParticipateState p = ctx.repair().participate(failure.sessionID);
-                    if (p != null)
-                        p.phase.fail("Failure message from " + message.from());
-                    ctx.repair().consistent.coordinated.handleFailSessionMessage(failure);
-                    ctx.repair().consistent.local.handleFailSessionMessage(message.from(), failure);
-                    break;
-
-                case STATUS_REQ:
-                    ctx.repair().consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload);
-                    break;
-
-                case STATUS_RSP:
-                    ctx.repair().consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload);
-                    break;
+            }
+            else if (message.verb() == SYNC_REQ)
+            {
+                // forwarded sync request
+                SyncRequest request = (SyncRequest) message.payload;
+                logger.debug("Syncing {}", request);
 
-                default:
+                ParticipateState participate = ctx.repair().participate(desc.parentSessionId);
+                if (participate == null)
+                {
+                    logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
+                    return;
+                }
+                SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst);
+                if (!register(message, participate, state,
+                              participate::register,
+                              participate::sync))
+                    return;
+                state.phase.accept();
+                StreamingRepairTask task = new StreamingRepairTask(ctx, state, desc,
+                                                                   request.initiator,
+                                                                   request.src,
+                                                                   request.dst,
+                                                                   request.ranges,
+                                                                   isIncremental(desc.parentSessionId) ? desc.parentSessionId : null,
+                                                                   request.previewKind,
+                                                                   request.asymmetric);
+                Future<?> syncFuture = task.execute();
+                ParentRepairSessionListener.instance.onSync(desc, syncFuture);
+                sendAck(message);
+            }
+            else if (message.verb() == CLEANUP_MSG)
+            {
+                logger.debug("cleaning up repair");
+                CleanupMessage cleanup = (CleanupMessage) message.payload;
+                ParticipateState state = ctx.repair().participate(cleanup.parentRepairSession);
+                if (state != null)
+                    state.phase.success("Cleanup message recieved");
+                ctx.repair().removeParentRepairSession(cleanup.parentRepairSession);
+                sendAck(message);
+            }
+            else if (message.verb() == PREPARE_CONSISTENT_REQ)
+            {
+                ctx.repair().consistent.local.handlePrepareMessage(message);
+            }
+                else if (message.verb() == PREPARE_CONSISTENT_RSP)
+            {
+                ctx.repair().consistent.coordinated.handlePrepareResponse(message);
+            }
+            else if (message.verb() == FINALIZE_PROPOSE_MSG)
+            {
+                ctx.repair().consistent.local.handleFinalizeProposeMessage(message);
+            }
+            else if (message.verb() == FINALIZE_PROMISE_MSG)
+            {
+                ctx.repair().consistent.coordinated.handleFinalizePromiseMessage(message);
+            }
+            else if (message.verb() == FINALIZE_COMMIT_MSG)
+            {
+                ctx.repair().consistent.local.handleFinalizeCommitMessage(message);
+            }
+            else if (message.verb() == FAILED_SESSION_MSG)
+            {
+                FailSession failure = (FailSession) message.payload;
+                sendAck(message);
+                ParticipateState p = ctx.repair().participate(failure.sessionID);
+                if (p != null)
+                    p.phase.fail("Failure message from " + message.from());
+                ctx.repair().consistent.coordinated.handleFailSessionMessage(failure);
+                ctx.repair().consistent.local.handleFailSessionMessage(message.from(), failure);
+            }
+            else if (message.verb() == STATUS_REQ)
+            {
+                ctx.repair().consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload);
+            }
+            else if (message.verb() == STATUS_RSP)
+            {
+                ctx.repair().consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload);
+            }
+            else
+            {
                     ctx.repair().handleMessage(message);
-                    break;
             }
         }
         catch (Exception e)
         {
-            logger.error("Got error, removing parent repair session");
+            logger.error("Got error processing {}, removing parent repair session", message.verb());
             if (desc != null && desc.parentSessionId != null)
             {
                 ParticipateState parcipate = ctx.repair().participate(desc.parentSessionId);
diff --git a/src/java/org/apache/cassandra/repair/RepairProgressReporter.java b/src/java/org/apache/cassandra/repair/RepairProgressReporter.java
new file mode 100644
index 000000000000..b78797107abe
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/RepairProgressReporter.java
@@ -0,0 +1,98 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.repair;
+
+import java.util.Collection;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.schema.SystemDistributedKeyspace;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.FBUtilities;
+
+public interface RepairProgressReporter
+{
+    RepairProgressReporter instance = CassandraRelevantProperties.REPAIR_PROGRESS_REPORTER.isPresent()
+                                      ? FBUtilities.construct(CassandraRelevantProperties.REPAIR_PROGRESS_REPORTER.getString(),
+                                                              "Repair Progress Reporter")
+                                      : new DefaultRepairProgressReporter();
+
+    void onParentRepairStarted(TimeUUID parentSession, String keyspaceName, String[] cfnames, RepairOption options);
+
+    void onParentRepairSucceeded(TimeUUID parentSession, Collection<Range<Token>> successfulRanges);
+
+    void onParentRepairFailed(TimeUUID parentSession, Throwable t);
+
+    void onRepairsStarted(TimeUUID id, TimeUUID parentRepairSession, String keyspaceName, String[] cfnames, CommonRange commonRange);
+
+    void onRepairsFailed(TimeUUID id, String keyspaceName, String[] cfnames, Throwable t);
+
+    void onRepairFailed(TimeUUID id, String keyspaceName, String cfname, Throwable t);
+
+    void onRepairSucceeded(TimeUUID id, String keyspaceName, String cfname);
+
+    class DefaultRepairProgressReporter implements RepairProgressReporter
+    {
+        @Override
+        public void onParentRepairStarted(TimeUUID parentSession, String keyspaceName, String[] cfnames, RepairOption options)
+        {
+            SystemDistributedKeyspace.startParentRepair(parentSession, keyspaceName, cfnames, options);
+        }
+
+        @Override
+        public void onParentRepairSucceeded(TimeUUID parentSession, Collection<Range<Token>> successfulRanges)
+        {
+            SystemDistributedKeyspace.successfulParentRepair(parentSession, successfulRanges);
+        }
+
+        @Override
+        public void onParentRepairFailed(TimeUUID parentSession, Throwable t)
+        {
+            SystemDistributedKeyspace.failParentRepair(parentSession, t);
+        }
+
+        @Override
+        public void onRepairsStarted(TimeUUID id, TimeUUID parentRepairSession, String keyspaceName, String[] cfnames, CommonRange commonRange)
+        {
+            SystemDistributedKeyspace.startRepairs(id, parentRepairSession, keyspaceName, cfnames, commonRange);
+        }
+
+        @Override
+        public void onRepairsFailed(TimeUUID id, String keyspaceName, String[] cfnames, Throwable t)
+        {
+            SystemDistributedKeyspace.failRepairs(id, keyspaceName, cfnames, t);
+        }
+
+        @Override
+        public void onRepairFailed(TimeUUID id, String keyspaceName, String cfname, Throwable t)
+        {
+            SystemDistributedKeyspace.failedRepairJob(id, keyspaceName, cfname, t);
+        }
+
+        @Override
+        public void onRepairSucceeded(TimeUUID id, String keyspaceName, String cfname)
+        {
+            SystemDistributedKeyspace.successfulRepairJob(id, keyspaceName, cfname);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java
index 7ec64502eb2f..e921e540ea83 100644
--- a/src/java/org/apache/cassandra/repair/RepairSession.java
+++ b/src/java/org/apache/cassandra/repair/RepairSession.java
@@ -114,6 +114,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
 
     public final SessionState state;
     public final RepairParallelism parallelismDegree;
+    public final boolean pushRepair;
     public final boolean pullRepair;
 
     /** Range to repair */
@@ -144,6 +145,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
      * @param commonRange ranges to repair
      * @param keyspace name of keyspace
      * @param parallelismDegree specifies the degree of parallelism when calculating the merkle trees
+     * @param pushRepair true if the repair should be one way pushing differences to remote host
      * @param pullRepair true if the repair should be one way (from remote host to this host and only applicable between two hosts--see RepairOption)
      * @param repairPaxos true if incomplete paxos operations should be completed as part of repair
      * @param paxosOnly true if we should only complete paxos operations, not run a normal repair
@@ -156,6 +158,7 @@ public RepairSession(SharedContext ctx,
                          String keyspace,
                          RepairParallelism parallelismDegree,
                          boolean isIncremental,
+                         boolean pushRepair,
                          boolean pullRepair,
                          PreviewKind previewKind,
                          boolean optimiseStreams,
@@ -170,6 +173,7 @@ public RepairSession(SharedContext ctx,
         assert cfnames.length > 0 : "Repairing no column families seems pointless, doesn't it";
         this.state = new SessionState(ctx.clock(), parentRepairSession, keyspace, cfnames, commonRange);
         this.parallelismDegree = parallelismDegree;
+        this.pushRepair = pushRepair;
         this.isIncremental = isIncremental;
         this.previewKind = previewKind;
         this.pullRepair = pullRepair;
@@ -300,6 +304,7 @@ public void start(ExecutorPlus executor)
         if (!previewKind.isPreview() && !paxosOnly)
         {
             SystemDistributedKeyspace.startRepairs(getId(), state.parentRepairSession, state.keyspace, state.cfnames, state.commonRange);
+            RepairProgressReporter.instance.onRepairsStarted(getId(), state.parentRepairSession, state.keyspace, state.cfnames, state.commonRange);
         }
 
         if (state.commonRange.endpoints.isEmpty())
@@ -311,6 +316,7 @@ public void start(ExecutorPlus executor)
             if (!previewKind.isPreview())
             {
                 SystemDistributedKeyspace.failRepairs(getId(), state.keyspace, state.cfnames, new RuntimeException(message));
+                RepairProgressReporter.instance.onRepairsFailed(getId(), state.keyspace, state.cfnames, new RuntimeException(message));
             }
             return;
         }
@@ -328,6 +334,7 @@ public void start(ExecutorPlus executor)
                 if (!previewKind.isPreview())
                 {
                     SystemDistributedKeyspace.failRepairs(getId(), state.keyspace, state.cfnames, e);
+                    RepairProgressReporter.instance.onRepairsFailed(getId(), state.keyspace, state.cfnames, e);
                 }
                 return;
             }
diff --git a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
index 0f84d66893ef..a6af3e0481b8 100644
--- a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
@@ -36,6 +36,7 @@
 import org.apache.cassandra.streaming.StreamEvent;
 import org.apache.cassandra.streaming.StreamEventHandler;
 import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.StreamResultFuture;
 import org.apache.cassandra.streaming.StreamState;
 import org.apache.cassandra.streaming.StreamOperation;
 import org.apache.cassandra.utils.TimeUUID;
@@ -50,7 +51,7 @@
  * StreamingRepairTask performs data streaming between two remote replicas, neither of which is repair coordinator.
  * Task will send {@link SyncResponse} message back to coordinator upon streaming completion.
  */
-public class StreamingRepairTask implements Runnable, StreamEventHandler
+public class StreamingRepairTask implements StreamEventHandler
 {
     private static final Logger logger = LoggerFactory.getLogger(StreamingRepairTask.class);
 
@@ -79,14 +80,14 @@ public StreamingRepairTask(SharedContext ctx, SyncState state, RepairJobDesc des
         this.previewKind = previewKind;
     }
 
-    public void run()
+    public StreamResultFuture execute()
     {
         logger.info("[streaming task #{}] Performing {}streaming repair of {} ranges with {}", desc.sessionId, asymmetric ? "asymmetric " : "", ranges.size(), dst);
         long start = approxTime.now();
         StreamPlan streamPlan = createStreamPlan(dst);
         logger.info("[streaming task #{}] Stream plan created in {}ms", desc.sessionId, MILLISECONDS.convert(approxTime.now() - start, NANOSECONDS));
         state.phase.start();
-        ctx.streamExecutor().execute(streamPlan);
+        return ctx.streamExecutor().execute(streamPlan);
     }
 
     @VisibleForTesting
diff --git a/src/java/org/apache/cassandra/repair/SyncTask.java b/src/java/org/apache/cassandra/repair/SyncTask.java
index a3b1a574937d..4f4da3dbf895 100644
--- a/src/java/org/apache/cassandra/repair/SyncTask.java
+++ b/src/java/org/apache/cassandra/repair/SyncTask.java
@@ -77,6 +77,9 @@ public SyncNodePair nodePair()
      */
     public final void run()
     {
+        if (logger.isTraceEnabled())
+            logger.trace("{} Starting sync {} <-> {}", previewKind.logPrefix(desc.sessionId), nodePair.coordinator, nodePair.peer);
+
         startTime = ctx.clock().currentTimeMillis();
 
         // choose a repair method based on the significance of the difference
diff --git a/src/java/org/apache/cassandra/repair/ValidationManager.java b/src/java/org/apache/cassandra/repair/ValidationManager.java
index e3598cd38f87..16d09b23788a 100644
--- a/src/java/org/apache/cassandra/repair/ValidationManager.java
+++ b/src/java/org/apache/cassandra/repair/ValidationManager.java
@@ -31,6 +31,8 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -41,6 +43,7 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MerkleTree;
 import org.apache.cassandra.utils.MerkleTrees;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 
 public class ValidationManager implements IValidationManager
 {
@@ -124,20 +127,24 @@ public static void doValidation(ColumnFamilyStore cfs, Validator validator) thro
         {
             state.phase.start(vi.estimatedPartitions(), vi.getEstimatedBytes());
             MerkleTrees trees = createMerkleTrees(vi, validator.desc.ranges, cfs);
-            // validate the CF as we iterate over it
-            validator.prepare(cfs, trees, topPartitionCollector);
-            while (vi.hasNext())
+            TableOperation op = vi.getCompactionIterator().getOperation();
+            try (NonThrowingCloseable cls = CompactionManager.instance.active.onOperationStart(op))
             {
-                try (UnfilteredRowIterator partition = vi.next())
+                // validate the CF as we iterate over it
+                validator.prepare(cfs, trees, topPartitionCollector);
+                while (vi.hasNext())
                 {
-                    validator.add(partition);
-                    state.partitionsProcessed++;
-                    state.bytesRead = vi.getBytesRead();
-                    if (state.partitionsProcessed % 1024 == 0) // update every so often
-                        state.updated();
+                    try (UnfilteredRowIterator partition = vi.next())
+                    {
+                        validator.add(partition);
+                        state.partitionsProcessed++;
+                        state.bytesRead = vi.getBytesRead();
+                        if (state.partitionsProcessed % 1024 == 0) // update every so often
+                            state.updated();
+                    }
                 }
+                validator.complete();
             }
-            validator.complete();
         }
         finally
         {
diff --git a/src/java/org/apache/cassandra/repair/ValidationPartitionIterator.java b/src/java/org/apache/cassandra/repair/ValidationPartitionIterator.java
index a8f457d782fc..88fa9d847f09 100644
--- a/src/java/org/apache/cassandra/repair/ValidationPartitionIterator.java
+++ b/src/java/org/apache/cassandra/repair/ValidationPartitionIterator.java
@@ -20,6 +20,7 @@
 
 import java.util.Map;
 
+import org.apache.cassandra.db.compaction.CompactionIterator;
 import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -30,4 +31,5 @@ public abstract class ValidationPartitionIterator extends AbstractUnfilteredPart
     public abstract long estimatedPartitions();
     public abstract long getBytesRead();
     public abstract Map<Range<Token>, Long> getRangePartitionCounts();
+    public abstract CompactionIterator getCompactionIterator();
 }
diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
index a2c916ce9a96..a02f6ed5167b 100644
--- a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
+++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
@@ -35,13 +35,14 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BooleanSupplier;
-import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
 import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -51,7 +52,16 @@
 import com.google.common.primitives.Ints;
 import com.google.common.util.concurrent.FutureCallback;
 
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.db.compaction.AbstractCompactionTask;
+import org.apache.cassandra.db.compaction.CleanupTask;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.RepairFinishedCompactionTask;
+import org.apache.cassandra.db.compaction.TableOperation;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.locator.RangesAtEndpoint;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -71,6 +81,7 @@
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.dht.IPartitioner;
@@ -79,6 +90,8 @@
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.repair.NoSuchRepairSessionException;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.repair.messages.FailSession;
 import org.apache.cassandra.repair.messages.FinalizeCommit;
 import org.apache.cassandra.repair.messages.FinalizePromise;
@@ -88,14 +101,14 @@
 import org.apache.cassandra.repair.messages.RepairMessage;
 import org.apache.cassandra.repair.messages.StatusRequest;
 import org.apache.cassandra.repair.messages.StatusResponse;
-import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.Pair;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_CLEANUP_INTERVAL_SECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_DELETE_TIMEOUT_SECONDS;
@@ -312,6 +325,12 @@ public PendingStats getPendingStats(TableId tid, Collection<Range<Token>> ranges
         return new PendingStats(cfs.getKeyspaceName(), cfs.name, pending.build(), finalized.build(), failed.build());
     }
 
+    /**
+     * promotes (or demotes) data attached to an incremental repair session that has either completed successfully,
+     * or failed
+     *
+     * @return session ids whose data could not be released
+     */
     public CleanupSummary cleanup(TableId tid, Collection<Range<Token>> ranges, boolean force)
     {
         Iterable<LocalSession> candidates = Iterables.filter(sessions.values(),
@@ -320,10 +339,86 @@ public CleanupSummary cleanup(TableId tid, Collection<Range<Token>> ranges, bool
                                                                    && Range.intersects(ls.ranges, ranges));
 
         ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tid);
+        Preconditions.checkNotNull(cfs);
+
         Set<TimeUUID> sessionIds = Sets.newHashSet(Iterables.transform(candidates, s -> s.sessionID));
+        return releaseRepairData(cfs, sessionIds, force);
+    }
+
+    private CleanupSummary releaseRepairData(ColumnFamilyStore cfs, Collection<TimeUUID> sessions, boolean force)
+    {
+        if (force)
+        {
+            Predicate<SSTableReader> predicate = sst -> {
+                TimeUUID session = sst.getPendingRepair();
+                return session != null && sessions.contains(session);
+            };
+            return cfs.runWithCompactionsDisabled(() -> doReleaseRepairData(cfs, sessions),
+                                                  predicate, OperationType.STREAM, false, true, true, TableOperation.StopTrigger.CLEANUP);
+        }
+        else
+        {
+            return doReleaseRepairData(cfs, sessions);
+        }
+    }
+
+    private CleanupSummary doReleaseRepairData(ColumnFamilyStore cfs, Collection<TimeUUID> sessions)
+    {
+        List<Pair<TimeUUID, RepairFinishedCompactionTask>> tasks = new ArrayList<>(sessions.size());
+        for (TimeUUID session : sessions)
+        {
+            if (canCleanup(session))
+                tasks.add(Pair.create(session, getRepairFinishedCompactionTask(cfs, session)));
+        }
+
+        return new CleanupTask(cfs, tasks).cleanup();
+    }
+
+    private RepairFinishedCompactionTask getRepairFinishedCompactionTask(ColumnFamilyStore cfs, TimeUUID session)
+    {
+        Set<SSTableReader> sstables = cfs.getPendingRepairSSTables(session);
+        if (sstables.isEmpty())
+            return null;
+
+        return getRepairFinishedCompactionTask(cfs, session, sstables);
+    }
+
+    private RepairFinishedCompactionTask getRepairFinishedCompactionTask(CompactionRealm realm,
+                                                                         TimeUUID session,
+                                                                         Collection<? extends CompactionSSTable> sstables)
+    {
+        long repairedAt = getFinalSessionRepairedAt(session);
+        boolean isTransient = sstables.iterator().next().isTransient();
+        LifecycleTransaction txn = realm.tryModify(sstables, OperationType.COMPACTION);
+        return txn == null ? null : new RepairFinishedCompactionTask(realm, txn, session, repairedAt, isTransient);
+    }
 
+    /**
+     * Some finalized repairs leave sstables behind that need cleaning. This generates the tasks to clean them up.
+     */
+    public Collection<AbstractCompactionTask> getZombieRepairFinalizationTasks(CompactionRealm realm, Collection<? extends CompactionSSTable> sstables)
+    {
+        Map<TimeUUID, Collection<CompactionSSTable>> finalizations = new HashMap<>();
+        for (CompactionSSTable sstable : sstables)
+        {
+            TableMetadata tableMetadata = Schema.instance.getTableMetadata(sstable.getKeyspaceName(), sstable.getColumnFamilyName());
+            if (tableMetadata != null && sstable.isPendingRepair() && canCleanup(sstable.getPendingRepair()))
+            {
+                logger.debug("Going to cleanup sstable {} for already finalized repair {}", sstable.getPendingRepair(), tableMetadata.toDebugString());
+                finalizations.computeIfAbsent(sstable.getPendingRepair(), pr -> new ArrayList<>()).add(sstable);
+            }
+        }
 
-        return cfs.releaseRepairData(sessionIds, force);
+        return finalizations.entrySet()
+                            .stream()
+                            .map(entry -> getRepairFinishedCompactionTask(realm, entry.getKey(), entry.getValue()))
+                            .filter(Predicates.notNull())
+                            .collect(Collectors.toList());
+    }
+
+    public boolean canCleanup(TimeUUID sessionID)
+    {
+        return !isSessionInProgress(sessionID);
     }
 
     /**
@@ -357,7 +452,7 @@ public synchronized void start()
         int loadedSessionsCount = 0;
         Preconditions.checkArgument(!started, "LocalSessions.start can only be called once");
         Preconditions.checkArgument(sessions.isEmpty(), "No sessions should be added before start");
-        UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(String.format("SELECT * FROM %s.%s", keyspace, table), 1000);
+        UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(String.format("SELECT * FROM %s.%s", keyspace, table), PageSize.inRows(1000));
         Map<TimeUUID, LocalSession> loadedSessions = new HashMap<>();
         Map<TableId, List<RepairedState.Level>> initialLevels = new HashMap<>();
         for (UntypedResultSet.Row row : rows)
@@ -479,6 +574,8 @@ else if (!sessionHasData(session))
                     }
                     else
                     {
+                        // If this happens too often or for a long time check sstables pending repair are not being
+                        // left behind
                         logger.warn("Skipping delete of LocalSession {} because it still contains sstables", session.sessionID);
                     }
                 }
@@ -1072,6 +1169,7 @@ public void handleStatusResponse(InetAddressAndPort from, StatusResponse respons
         }
         else
         {
+            session.setLastUpdate();
             logger.debug("Received StatusResponse for repair session {} with state {}, which is not actionable. Doing nothing.", sessionID, response.state);
         }
     }
@@ -1107,7 +1205,7 @@ protected boolean sessionHasData(LocalSession session)
     {
         Predicate<TableId> predicate = tid -> {
             ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tid);
-            return cfs != null && cfs.getCompactionStrategyManager().hasDataForPendingRepair(session.sessionID);
+            return cfs != null && cfs.hasPendingRepairSSTables(session.sessionID);
 
         };
         return Iterables.any(session.tableIds, predicate::test);
diff --git a/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java b/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java
index f715cc98f5b6..52fdaa2a3c9f 100644
--- a/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java
+++ b/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java
@@ -22,7 +22,6 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-
 import javax.management.openmbean.ArrayType;
 import javax.management.openmbean.CompositeData;
 import javax.management.openmbean.CompositeDataSupport;
@@ -34,7 +33,7 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Sets;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.CompactionRealm;
 import org.apache.cassandra.utils.TimeUUID;
 
 public class CleanupSummary
@@ -74,9 +73,9 @@ public CleanupSummary(String keyspace, String table, Set<TimeUUID> successful, S
         this.unsuccessful = unsuccessful;
     }
 
-    public CleanupSummary(ColumnFamilyStore cfs, Set<TimeUUID> successful, Set<TimeUUID> unsuccessful)
+    public CleanupSummary(CompactionRealm cfs, Set<TimeUUID> successful, Set<TimeUUID> unsuccessful)
     {
-        this(cfs.getKeyspaceName(), cfs.name, successful, unsuccessful);
+        this(cfs.getKeyspaceName(), cfs.getTableName(), successful, unsuccessful);
     }
 
     public static CleanupSummary add(CleanupSummary l, CleanupSummary r)
diff --git a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
index f0cbf78f38ce..9b1425d4b027 100644
--- a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
+++ b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
@@ -18,16 +18,17 @@
 package org.apache.cassandra.repair.messages;
 
 import java.util.Collections;
-import java.util.EnumMap;
-import java.util.EnumSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.*;
 import java.util.function.Supplier;
 
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.nodes.INodeInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,6 +43,7 @@
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.repair.RepairJobDesc;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.utils.Backoff;
@@ -64,7 +66,7 @@ private enum ErrorHandling { NONE, TIMEOUT, RETRY }
     static final CassandraVersion SUPPORTS_RETRY = new CassandraVersion("5.0.0-alpha2.SNAPSHOT");
     private static final Map<Verb, CassandraVersion> VERB_TIMEOUT_VERSIONS;
     public static final Set<Verb> ALLOWS_RETRY;
-    private static final Set<Verb> SUPPORTS_RETRY_WITHOUT_VERSION_CHECK = Collections.unmodifiableSet(EnumSet.of(Verb.CLEANUP_MSG));
+    private static final Set<Verb> SUPPORTS_RETRY_WITHOUT_VERSION_CHECK = Collections.unmodifiableSet(ImmutableSet.of(Verb.CLEANUP_MSG));
     public static final RequestCallback<Object> NOOP_CALLBACK = new RequestCallback<>()
     {
         @Override
@@ -81,7 +83,7 @@ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReaso
     static
     {
         CassandraVersion timeoutVersion = new CassandraVersion("4.0.7-SNAPSHOT");
-        EnumMap<Verb, CassandraVersion> map = new EnumMap<>(Verb.class);
+        HashMap<Verb, CassandraVersion> map = new HashMap<>();
         map.put(Verb.VALIDATION_REQ, timeoutVersion);
         map.put(Verb.SYNC_REQ, timeoutVersion);
         map.put(Verb.VALIDATION_RSP, SUPPORTS_RETRY);
@@ -95,7 +97,7 @@ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReaso
         map.put(Verb.FAILED_SESSION_MSG, SUPPORTS_RETRY);
         VERB_TIMEOUT_VERSIONS = Collections.unmodifiableMap(map);
 
-        EnumSet<Verb> allowsRetry = EnumSet.noneOf(Verb.class);
+        Set<Verb> allowsRetry = new HashSet<>();
         allowsRetry.add(Verb.PREPARE_MSG);
         allowsRetry.add(Verb.VALIDATION_REQ);
         allowsRetry.add(Verb.VALIDATION_RSP);
@@ -281,7 +283,7 @@ private static ErrorHandling errorHandlingSupported(SharedContext ctx, InetAddre
             return ErrorHandling.RETRY;
         // Repair in mixed mode isn't fully supported, but also not activally blocked... so in the common case all participants
         // will be on the same version as this instance, so can avoid the lookup from gossip
-        CassandraVersion remoteVersion = ctx.gossiper().getReleaseVersion(from);
+        CassandraVersion remoteVersion = Nodes.localOrPeerInfoOpt(from).map(INodeInfo::getReleaseVersion).orElse(null);
         if (remoteVersion == null)
         {
             if (VERB_TIMEOUT_VERSIONS.containsKey(verb))
@@ -294,7 +296,7 @@ private static ErrorHandling errorHandlingSupported(SharedContext ctx, InetAddre
         if (remoteVersion.compareTo(SUPPORTS_RETRY) >= 0)
             return ErrorHandling.RETRY;
         CassandraVersion timeoutVersion = VERB_TIMEOUT_VERSIONS.get(verb);
-        if (timeoutVersion == null || remoteVersion.compareTo(timeoutVersion) >= 0)
+        if (timeoutVersion == null || remoteVersion.compareTo(timeoutVersion, true) >= 0)
             return ErrorHandling.TIMEOUT;
         return ErrorHandling.NONE;
     }
diff --git a/src/java/org/apache/cassandra/repair/messages/RepairOption.java b/src/java/org/apache/cassandra/repair/messages/RepairOption.java
index f0508a3e4226..471fb4aa3221 100644
--- a/src/java/org/apache/cassandra/repair/messages/RepairOption.java
+++ b/src/java/org/apache/cassandra/repair/messages/RepairOption.java
@@ -46,6 +46,7 @@ public class RepairOption
     public static final String HOSTS_KEY = "hosts";
     public static final String TRACE_KEY = "trace";
     public static final String SUB_RANGE_REPAIR_KEY = "sub_range_repair";
+    public static final String PUSH_REPAIR_KEY = "pushRepair";
     public static final String PULL_REPAIR_KEY = "pullRepair";
     public static final String FORCE_REPAIR_KEY = "forceRepair";
     public static final String PREVIEW = "previewKind";
@@ -53,6 +54,7 @@ public class RepairOption
     public static final String IGNORE_UNREPLICATED_KS = "ignoreUnreplicatedKeyspaces";
     public static final String REPAIR_PAXOS_KEY = "repairPaxos";
     public static final String PAXOS_ONLY_KEY = "paxosOnly";
+    public static final String OFFLINE_SERVICE = "offlineService";
 
     // we don't want to push nodes too much for repair
     public static final int MAX_JOB_THREADS = 4;
@@ -148,6 +150,11 @@ public static Set<Range<Token>> parseRanges(String rangesStr, IPartitioner parti
      *             <td></td>
      *         </tr>
      *         <tr>
+     *             <td>pushRepair</td>
+     *             <td>"true" if the repair should only stream data one way from local host to remote host.</td>
+     *             <td>false</td>
+     *         </tr>
+     *         <tr>
      *             <td>pullRepair</td>
      *             <td>"true" if the repair should only stream data one way from a remote host to this host.
      *             This is only allowed if exactly 2 hosts are specified along with a token range that they share.</td>
@@ -164,6 +171,12 @@ public static Set<Range<Token>> parseRanges(String rangesStr, IPartitioner parti
      *             ranges to the same host multiple times</td>
      *             <td>false</td>
      *         </tr>
+     *         <tr>
+     *             <td>offlineService</td>
+     *             <td>"true" if current repair task is executed by an offline service which has no token metadata and
+     *              it's not part of the ring. Repair should use tokens and hosts directly from repair options.</td>
+     *             <td>false</td>
+     *         </tr>
      *     </tbody>
      * </table>
      *
@@ -180,6 +193,7 @@ public static RepairOption parse(Map<String, String> options, IPartitioner parti
         PreviewKind previewKind = PreviewKind.valueOf(options.getOrDefault(PREVIEW, PreviewKind.NONE.toString()));
         boolean trace = Boolean.parseBoolean(options.get(TRACE_KEY));
         boolean force = Boolean.parseBoolean(options.get(FORCE_REPAIR_KEY));
+        boolean pushRepair = Boolean.parseBoolean(options.get(PUSH_REPAIR_KEY));
         boolean pullRepair = Boolean.parseBoolean(options.get(PULL_REPAIR_KEY));
         boolean ignoreUnreplicatedKeyspaces = Boolean.parseBoolean(options.get(IGNORE_UNREPLICATED_KS));
         boolean repairPaxos = Boolean.parseBoolean(options.get(REPAIR_PAXOS_KEY));
@@ -190,6 +204,9 @@ public static RepairOption parse(Map<String, String> options, IPartitioner parti
             Preconditions.checkArgument(!repairPaxos, "repairPaxos must be set to false for preview repairs");
             Preconditions.checkArgument(!paxosOnly, "paxosOnly must be set to false for preview repairs");
         }
+        boolean offlineService = Boolean.parseBoolean(options.get(OFFLINE_SERVICE));
+
+        Preconditions.checkArgument(!pullRepair || !pushRepair, "Cannot use pushRepair and pullRepair as the same time");
 
         int jobThreads = 1;
         if (options.containsKey(JOB_THREADS_KEY))
@@ -206,7 +223,9 @@ public static RepairOption parse(Map<String, String> options, IPartitioner parti
 
         boolean asymmetricSyncing = Boolean.parseBoolean(options.get(OPTIMISE_STREAMS_KEY));
 
-        RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges, !ranges.isEmpty(), pullRepair, force, previewKind, asymmetricSyncing, ignoreUnreplicatedKeyspaces, repairPaxos, paxosOnly);
+        RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges,
+                                               !ranges.isEmpty(), pushRepair, pullRepair, force, previewKind, asymmetricSyncing,
+                                               ignoreUnreplicatedKeyspaces, repairPaxos, paxosOnly, offlineService);
 
         // data centers
         String dataCentersStr = options.get(DATACENTERS_KEY);
@@ -281,6 +300,7 @@ else if (ranges.isEmpty())
     private final boolean trace;
     private final int jobThreads;
     private final boolean isSubrangeRepair;
+    private final boolean pushRepair;
     private final boolean pullRepair;
     private final boolean forceRepair;
     private final PreviewKind previewKind;
@@ -288,13 +308,17 @@ else if (ranges.isEmpty())
     private final boolean ignoreUnreplicatedKeyspaces;
     private final boolean repairPaxos;
     private final boolean paxosOnly;
+    private final boolean offlineService;
 
     private final Collection<String> columnFamilies = new HashSet<>();
     private final Collection<String> dataCenters = new HashSet<>();
     private final Collection<String> hosts = new HashSet<>();
     private final Collection<Range<Token>> ranges = new HashSet<>();
 
-    public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean incremental, boolean trace, int jobThreads, Collection<Range<Token>> ranges, boolean isSubrangeRepair, boolean pullRepair, boolean forceRepair, PreviewKind previewKind, boolean optimiseStreams, boolean ignoreUnreplicatedKeyspaces, boolean repairPaxos, boolean paxosOnly)
+    public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean incremental, boolean trace,
+                        int jobThreads, Collection<Range<Token>> ranges, boolean isSubrangeRepair, boolean pushRepair,
+                        boolean pullRepair, boolean forceRepair, PreviewKind previewKind, boolean optimiseStreams,
+                        boolean ignoreUnreplicatedKeyspaces, boolean repairPaxos, boolean paxosOnly, boolean offlineService)
     {
 
         this.parallelism = parallelism;
@@ -303,6 +327,7 @@ public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean
         this.trace = trace;
         this.jobThreads = jobThreads;
         this.ranges.addAll(ranges);
+        this.pushRepair = pushRepair;
         this.isSubrangeRepair = isSubrangeRepair;
         this.pullRepair = pullRepair;
         this.forceRepair = forceRepair;
@@ -311,6 +336,7 @@ public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean
         this.ignoreUnreplicatedKeyspaces = ignoreUnreplicatedKeyspaces;
         this.repairPaxos = repairPaxos;
         this.paxosOnly = paxosOnly;
+        this.offlineService = offlineService;
     }
 
     public RepairParallelism getParallelism()
@@ -333,6 +359,16 @@ public boolean isTraced()
         return trace;
     }
 
+    public boolean isOfflineService()
+    {
+        return offlineService;
+    }
+
+    public boolean isPushRepair()
+    {
+        return pushRepair;
+    }
+
     public boolean isPullRepair()
     {
         return pullRepair;
@@ -439,12 +475,14 @@ public String toString()
                ", hosts: " + hosts +
                ", previewKind: " + previewKind +
                ", # of ranges: " + ranges.size() +
+               ", push repair: " + pushRepair +
                ", pull repair: " + pullRepair +
                ", force repair: " + forceRepair +
                ", optimise streams: "+ optimiseStreams() +
                ", ignore unreplicated keyspaces: "+ ignoreUnreplicatedKeyspaces +
                ", repairPaxos: " + repairPaxos +
                ", paxosOnly: " + paxosOnly +
+               ", offline service: " + offlineService +
                ')';
     }
 
@@ -461,12 +499,14 @@ public Map<String, String> asMap()
         options.put(SUB_RANGE_REPAIR_KEY, Boolean.toString(isSubrangeRepair));
         options.put(TRACE_KEY, Boolean.toString(trace));
         options.put(RANGES_KEY, Joiner.on(",").join(ranges));
+        options.put(PUSH_REPAIR_KEY, Boolean.toString(pushRepair));
         options.put(PULL_REPAIR_KEY, Boolean.toString(pullRepair));
         options.put(FORCE_REPAIR_KEY, Boolean.toString(forceRepair));
         options.put(PREVIEW, previewKind.toString());
         options.put(OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams));
         options.put(REPAIR_PAXOS_KEY, Boolean.toString(repairPaxos));
         options.put(PAXOS_ONLY_KEY, Boolean.toString(paxosOnly));
+        options.put(OFFLINE_SERVICE, Boolean.toString(offlineService));
         return options;
     }
 }
diff --git a/src/java/org/apache/cassandra/repair/messages/ValidationRequest.java b/src/java/org/apache/cassandra/repair/messages/ValidationRequest.java
index 1e651a96d2d7..70b505223bbf 100644
--- a/src/java/org/apache/cassandra/repair/messages/ValidationRequest.java
+++ b/src/java/org/apache/cassandra/repair/messages/ValidationRequest.java
@@ -71,13 +71,13 @@ public int hashCode()
         public void serialize(ValidationRequest message, DataOutputPlus out, int version) throws IOException
         {
             RepairJobDesc.serializer.serialize(message.desc, out, version);
-            out.writeInt(version >= MessagingService.VERSION_50 ? CassandraUInt.fromLong(message.nowInSec) : (int) message.nowInSec);
+            out.writeInt(MessagingService.Version.supportsExtendedDeletionTime(version) ? CassandraUInt.fromLong(message.nowInSec) : (int) message.nowInSec);
         }
 
         public ValidationRequest deserialize(DataInputPlus dis, int version) throws IOException
         {
             RepairJobDesc desc = RepairJobDesc.serializer.deserialize(dis, version);
-            long nowInsec = version >= MessagingService.VERSION_50 ? CassandraUInt.toLong(dis.readInt()) : dis.readInt();
+            long nowInsec = MessagingService.Version.supportsExtendedDeletionTime(version) ? CassandraUInt.toLong(dis.readInt()) : dis.readInt();
             return new ValidationRequest(desc, nowInsec);
         }
 
diff --git a/src/java/org/apache/cassandra/schema/CQLTypeParser.java b/src/java/org/apache/cassandra/schema/CQLTypeParser.java
index c79de881550e..392872c87c2a 100644
--- a/src/java/org/apache/cassandra/schema/CQLTypeParser.java
+++ b/src/java/org/apache/cassandra/schema/CQLTypeParser.java
@@ -19,7 +19,9 @@
 
 import com.google.common.collect.ImmutableSet;
 
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLFragmentParser;
+import org.apache.cassandra.cql3.CqlParser;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UserType;
 
@@ -50,7 +52,36 @@ public static AbstractType<?> parse(String keyspace, String unparsed, Types user
         if (udt != null)
             return udt;
 
-        return parseRaw(unparsed).prepareInternal(keyspace, userTypes).getType();
+        return parseRaw(unparsed).prepare(keyspace, userTypes).getType();
+    }
+
+    /**
+     * Parse the type for a dropped column in the schema.
+     * </p>
+     * The reason we need a specific method for this is that when we record dropped column types, we "expand" user
+     * types into tuples ({@link AbstractType#expandUserTypes()}) and this in order to save us from having to preserve
+     * dropped user types definitions. But a consequence of that expansion is that we have to have some support for
+     * non-frozen tuples, since the dropped type could be a non-frozen UDT, and that differs from normal CQL where
+     * tuples are frozen by default and {@code tuple<X, Y>} is indistinguishable from {@code frozen<tuple<X, Y>>}. So,
+     * to handle this, we rely on the fact that types for dropped columns will have been recorded using
+     * {@link CQL3Type#toSchemaString()}, which explicitly handles the frozen/non-frozen difference for tuples, which
+     * this method makes use of.
+     * </p>
+     * Concretely, while {@link #parse(String, String, Types)} will return a frozen type for {@code tuple<...>}
+     * (since again, tuple are frozen by default in CQL), this method will return a non-frozen type.
+     */
+    public static AbstractType<?> parseDroppedType(String keyspace, String unparsed)
+    {
+
+        // fast path for the common case of a primitive type
+        if (PRIMITIVE_TYPES.contains(unparsed.toLowerCase()))
+            return CQL3Type.Native.valueOf(unparsed.toUpperCase()).getType();
+
+        // We can't have UDT in dropped types...
+        CQL3Type.Raw rawType = CQLFragmentParser.parseAny(CqlParser::comparatorTypeWithMultiCellTuple,
+                                                          unparsed,
+                                                          "CQL dropped type");
+        return rawType.prepare(keyspace, Types.none()).getType();
     }
 
     static CQL3Type.Raw parseRaw(String type)
diff --git a/src/java/org/apache/cassandra/schema/ColumnMetadata.java b/src/java/org/apache/cassandra/schema/ColumnMetadata.java
index f68a7b5ff364..5f793ad9d0d6 100644
--- a/src/java/org/apache/cassandra/schema/ColumnMetadata.java
+++ b/src/java/org/apache/cassandra/schema/ColumnMetadata.java
@@ -18,31 +18,52 @@
 package org.apache.cassandra.schema;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Predicate;
-
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.Collections2;
 
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.VariableSpecifications;
 import org.apache.cassandra.cql3.functions.masking.ColumnMask;
 import org.apache.cassandra.cql3.selection.Selectable;
 import org.apache.cassandra.cql3.selection.Selector;
 import org.apache.cassandra.cql3.selection.SimpleSelector;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.MultiCellCapableType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.ColumnData;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.serializers.MarshalException;
 import org.github.jamm.Unmetered;
 
+import static java.lang.String.format;
+
 @Unmetered
 public final class ColumnMetadata extends ColumnSpecification implements Selectable, Comparable<ColumnMetadata>
 {
-    public static final Comparator<Object> asymmetricColumnDataComparator =
-        (a, b) -> ((ColumnData) a).column().compareTo((ColumnMetadata) b);
+    public static final Comparator<Object> asymmetricColumnDataComparator = new Comparator<Object>()
+    {
+        @Override
+        public int compare(Object a, Object b)
+        {
+            return ((ColumnData) a).column().compareTo((ColumnMetadata) b);
+        }
+    };
 
     public static final int NO_POSITION = -1;
 
@@ -53,9 +74,9 @@ public enum ClusteringOrder
 
     /**
      * The type of CQL3 column this definition represents.
-     * There is 4 main type of CQL3 columns: those parts of the partition key,
-     * those parts of the clustering columns and amongst the others, regular and
-     * static ones.
+     * There are 5 types of columns: those parts of the partition key,
+     * those parts of the clustering columns and amongst the others, regular,
+     * static, and synthetic ones.
      *
      * IMPORTANT: this enum is serialized as toString() and deserialized by calling
      * Kind.valueOf(), so do not override toString() or rename existing values.
@@ -63,18 +84,27 @@ public enum ClusteringOrder
     public enum Kind
     {
         // NOTE: if adding a new type, must modify comparisonOrder
+        SYNTHETIC,
         PARTITION_KEY,
         CLUSTERING,
         REGULAR,
         STATIC;
+        // it is not possible to add new Kinds after Synthetic without invasive changes to BTreeRow, which
+        // assumes that complex regulr/static columns are the last ones
 
         public boolean isPrimaryKeyKind()
         {
             return this == PARTITION_KEY || this == CLUSTERING;
         }
-
     }
 
+    public static final ColumnIdentifier SYNTHETIC_SCORE_ID = ColumnIdentifier.getInterned("+:!score", true);
+
+    /**
+     * Whether this is a dropped column.
+     */
+    private final boolean isDropped;
+
     public final Kind kind;
 
     /*
@@ -90,6 +120,9 @@ public boolean isPrimaryKeyKind()
     private final Comparator<Object> asymmetricCellPathComparator;
     private final Comparator<? super Cell<?>> cellComparator;
 
+    // When the kind is SYNTHETIC, this is the column from which the synthetic column is derived
+    public final ColumnIdentifier sythenticSourceColumn;
+
     private int hash;
 
     /**
@@ -104,10 +137,18 @@ public boolean isPrimaryKeyKind()
     @Nullable
     private final ColumnMask mask;
 
+    /**
+     * The type of CQL3 column this definition represents.
+     * Bit layout (from most to least significant):
+     * - Bits 61-63: Kind ordinal (3 bits, supporting up to 8 Kind values)
+     * - Bit 60: isComplex flag
+     * - Bits 48-59: position (12 bits, see assert)
+     * - Bits 0-47: name.prefixComparison (shifted right by 16)
+     */
     private static long comparisonOrder(Kind kind, boolean isComplex, long position, ColumnIdentifier name)
     {
         assert position >= 0 && position < 1 << 12;
-        return   (((long) kind.ordinal()) << 61)
+        return (((long) kind.ordinal()) << 61)
                | (isComplex ? 1L << 60 : 0)
                | (position << 48)
                | (name.prefixComparison >>> 16);
@@ -153,6 +194,34 @@ public static ColumnMetadata staticColumn(String keyspace, String table, String
         return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.STATIC, null);
     }
 
+    /**
+     * Creates a new synthetic column metadata instance.
+     */
+    public static ColumnMetadata syntheticScoreColumn(ColumnMetadata sourceColumn, AbstractType<?> type)
+    {
+        return new ColumnMetadata(sourceColumn.ksName, sourceColumn.cfName, SYNTHETIC_SCORE_ID, type, NO_POSITION, Kind.SYNTHETIC, null, false, sourceColumn.name);
+    }
+
+    /**
+     * Rebuild the metadata for a dropped column from its recorded data.
+     *
+     * <p>Please note that this method expect that the provided arguments are those of a dropped column, and in
+     * particular that the type uses no UDT (any should have been expanded). If a column is being dropped, prefer
+     * {@link #asDropped()} to transform the existing column to a dropped one as this deal with type expansion directly.
+     */
+    public static ColumnMetadata droppedColumn(String keyspace,
+                                               String table,
+                                               ColumnIdentifier name,
+                                               AbstractType<?> type,
+                                               Kind kind,
+                                               @Nullable ColumnMask mask)
+    {
+        assert !kind.isPrimaryKeyKind();
+        assert !type.referencesUserTypes()
+        : format("In %s.%s, dropped column %s type should not contain UDT; got %s" , keyspace, table, name, type);
+        return new ColumnMetadata(keyspace, table, name, type, NO_POSITION, kind, mask, true);
+    }
+
     public ColumnMetadata(TableMetadata table,
                           ByteBuffer name,
                           AbstractType<?> type,
@@ -177,6 +246,31 @@ public ColumnMetadata(String ksName,
                           int position,
                           Kind kind,
                           @Nullable ColumnMask mask)
+    {
+        this(ksName, cfName, name, type, position, kind, mask, false);
+    }
+
+    public ColumnMetadata(String ksName,
+                          String cfName,
+                          ColumnIdentifier name,
+                          AbstractType<?> type,
+                          int position,
+                          Kind kind,
+                          @Nullable ColumnMask mask,
+                          boolean isDropped)
+    {
+        this(ksName, cfName, name, type, position, kind, mask, isDropped, null);
+    }
+
+    public ColumnMetadata(String ksName,
+                          String cfName,
+                          ColumnIdentifier name,
+                          AbstractType<?> type,
+                          int position,
+                          Kind kind,
+                          @Nullable ColumnMask mask,
+                          boolean isDropped,
+                          ColumnIdentifier sythenticSourceColumnName)
     {
         super(ksName, cfName, name, type);
         assert name != null && type != null && kind != null;
@@ -191,57 +285,116 @@ public ColumnMetadata(String ksName,
         this.kind = kind;
         this.position = position;
         this.cellPathComparator = makeCellPathComparator(kind, type);
-        this.cellComparator = cellPathComparator == null ? ColumnData.comparator : (a, b) -> cellPathComparator.compare(a.path(), b.path());
-        this.asymmetricCellPathComparator = cellPathComparator == null ? null : (a, b) -> cellPathComparator.compare(((Cell<?>)a).path(), (CellPath) b);
+        assert kind != Kind.SYNTHETIC || cellPathComparator == null;
+        this.cellComparator = cellPathComparator == null ? ColumnData.comparator : new Comparator<Cell<?>>()
+        {
+            @Override
+            public int compare(Cell<?> a, Cell<?> b)
+            {
+                return cellPathComparator.compare(a.path(), b.path());
+            }
+        };
+        this.asymmetricCellPathComparator = cellPathComparator == null ? null : new Comparator<Object>()
+        {
+            @Override
+            public int compare(Object a, Object b)
+            {
+                return cellPathComparator.compare(((Cell<?>) a).path(), (CellPath) b);
+            }
+        };
         this.comparisonOrder = comparisonOrder(kind, isComplex(), Math.max(0, position), name);
         this.mask = mask;
+        this.isDropped = isDropped;
+
+        // Synthetic columns are the only ones that can have a source column
+        assert kind == Kind.SYNTHETIC || sythenticSourceColumnName == null;
+        this.sythenticSourceColumn = sythenticSourceColumnName;
     }
 
     private static Comparator<CellPath> makeCellPathComparator(Kind kind, AbstractType<?> type)
     {
         if (kind.isPrimaryKeyKind() || !type.isMultiCell())
             return null;
+        assert !type.isReversed() : "This should not happen because reversed types can be only constructed for " +
+                                    "clustering columns which are part of primary keys and should be excluded by the above condition";
 
-        AbstractType<?> nameComparator = type.isCollection()
-                                       ? ((CollectionType) type).nameComparator()
-                                       : ((UserType) type).nameComparator();
+        AbstractType<?> nameComparator = ((MultiCellCapableType<?>) type).nameComparator();
 
 
-        return (path1, path2) ->
+        return new Comparator<CellPath>()
         {
-            if (path1.size() == 0 || path2.size() == 0)
+            @Override
+            public int compare(CellPath path1, CellPath path2)
             {
-                if (path1 == CellPath.BOTTOM)
-                    return path2 == CellPath.BOTTOM ? 0 : -1;
-                if (path1 == CellPath.TOP)
-                    return path2 == CellPath.TOP ? 0 : 1;
-                return path2 == CellPath.BOTTOM ? 1 : -1;
+                if (path1.size() == 0 || path2.size() == 0)
+                {
+                    if (path1 == CellPath.BOTTOM)
+                        return path2 == CellPath.BOTTOM ? 0 : -1;
+                    if (path1 == CellPath.TOP)
+                        return path2 == CellPath.TOP ? 0 : 1;
+                    return path2 == CellPath.BOTTOM ? 1 : -1;
+                }
+
+                // This will get more complicated once we have non-frozen UDT and nested collections
+                assert path1.size() == 1 && path2.size() == 1;
+                return nameComparator.compare(path1.get(0), path2.get(0));
             }
-
-            // This will get more complicated once we have non-frozen UDT and nested collections
-            assert path1.size() == 1 && path2.size() == 1;
-            return nameComparator.compare(path1.get(0), path2.get(0));
         };
     }
 
+    /**
+     * Whether that is the column metadata of a dropped column.
+     */
+    public boolean isDropped()
+    {
+        return isDropped;
+    }
+
     public ColumnMetadata copy()
     {
-        return new ColumnMetadata(ksName, cfName, name, type, position, kind, mask);
+        return new ColumnMetadata(ksName, cfName, name, type, position, kind, mask, isDropped);
+    }
+
+    public ColumnMetadata withNewKeyspace(String newKeyspace, Types udts)
+    {
+        return new ColumnMetadata(newKeyspace, cfName, name, type.withUpdatedUserTypes(udts), position, kind, mask, isDropped);
     }
 
     public ColumnMetadata withNewName(ColumnIdentifier newName)
     {
-        return new ColumnMetadata(ksName, cfName, newName, type, position, kind, mask);
+        return new ColumnMetadata(ksName, cfName, newName, type, position, kind, mask, isDropped);
     }
 
     public ColumnMetadata withNewType(AbstractType<?> newType)
     {
-        return new ColumnMetadata(ksName, cfName, name, newType, position, kind, mask);
+        return new ColumnMetadata(ksName, cfName, name, newType, position, kind, mask, isDropped);
     }
 
     public ColumnMetadata withNewMask(@Nullable ColumnMask newMask)
     {
-        return new ColumnMetadata(ksName, cfName, name, type, position, kind, newMask);
+        return new ColumnMetadata(ksName, cfName, name, type, position, kind, newMask, isDropped);
+    }
+
+    /**
+     * Transforms this (non-dropped) column metadata into one suitable when the column is dropped.
+     *
+     * <p>This should be used when a column is dropped to create the relevant {@link DroppedColumn} record.
+     *
+     * @return the transformed metadata. It will be equivalent to {@code this} except that 1) its {@link #isDropped}
+     * method will return {@code true} and 2) any UDT within the column type will have been expanded to tuples (see
+     * {@link AbstractType#expandUserTypes()}).
+     */
+    ColumnMetadata asDropped()
+    {
+        assert !isDropped : this + " was already dropped";
+        return new ColumnMetadata(ksName,
+                                  cfName,
+                                  name,
+                                  type.expandUserTypes(),
+                                  position,
+                                  kind,
+                                  mask,
+                                  true);
     }
 
     public boolean isPartitionKey()
@@ -395,7 +548,7 @@ public int compareTo(ColumnMetadata other)
             return 0;
 
         if (comparisonOrder != other.comparisonOrder)
-            return Long.compare(comparisonOrder, other.comparisonOrder);
+            return Long.compareUnsigned(comparisonOrder, other.comparisonOrder);
 
         return this.name.compareTo(other.name);
     }
@@ -530,6 +683,11 @@ public boolean isCounterColumn()
         return type.isCounter();
     }
 
+    public boolean isSynthetic()
+    {
+        return kind == Kind.SYNTHETIC;
+    }
+
     public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames) throws InvalidRequestException
     {
         return SimpleSelector.newFactory(this, addAndGetIndex(this, defs), false);
@@ -539,4 +697,14 @@ public AbstractType<?> getExactTypeIfKnown(String keyspace)
     {
         return type;
     }
+
+    /**
+     * Validate whether the column definition is valid (mostly, that the type is valid for the type of column this is).
+     *
+     * @param isCounterTable whether the table the column is part of is a counter table.
+     */
+    public void validate(boolean isCounterTable)
+    {
+        type.validateForColumn(name.bytes, isPrimaryKeyColumn(), isCounterTable, isDropped, false);
+    }
 }
diff --git a/src/java/org/apache/cassandra/schema/CompactionParams.java b/src/java/org/apache/cassandra/schema/CompactionParams.java
index 7da6b50280eb..55936d4e311a 100644
--- a/src/java/org/apache/cassandra/schema/CompactionParams.java
+++ b/src/java/org/apache/cassandra/schema/CompactionParams.java
@@ -17,36 +17,33 @@
  */
 package org.apache.cassandra.schema;
 
-import java.lang.reflect.InvocationTargetException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
-import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
+
+import org.apache.cassandra.db.compaction.CompactionStrategy;
+import org.apache.cassandra.db.compaction.CompactionStrategyOptions;
 import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
 import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
 import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
 import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.commons.lang3.StringUtils;
 
 import static java.lang.String.format;
 import static org.apache.cassandra.config.CassandraRelevantProperties.DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES;
 
 public final class CompactionParams
 {
-    private static final Logger logger = LoggerFactory.getLogger(CompactionParams.class);
-
     public enum Option
     {
         CLASS,
@@ -76,16 +73,13 @@ public static Optional<TombstoneOption> forName(String name)
         }
     }
 
-    public static final int DEFAULT_MIN_THRESHOLD = 4;
-    public static final int DEFAULT_MAX_THRESHOLD = 32;
-
     public static final boolean DEFAULT_ENABLED = true;
     public static final TombstoneOption DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES_PROPERTY_VALUE =
         DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES.getEnum(TombstoneOption.NONE);
 
     public static final Map<String, String> DEFAULT_THRESHOLDS =
-        ImmutableMap.of(Option.MIN_THRESHOLD.toString(), Integer.toString(DEFAULT_MIN_THRESHOLD),
-                        Option.MAX_THRESHOLD.toString(), Integer.toString(DEFAULT_MAX_THRESHOLD));
+        ImmutableMap.of(Option.MIN_THRESHOLD.toString(), Integer.toString(CompactionStrategyOptions.DEFAULT_MIN_THRESHOLD),
+                        Option.MAX_THRESHOLD.toString(), Integer.toString(CompactionStrategyOptions.DEFAULT_MAX_THRESHOLD));
 
     public static final CompactionParams DEFAULT;
     static
@@ -93,8 +87,8 @@ public static Optional<TombstoneOption> forName(String name)
         ParameterizedClass defaultCompaction = DatabaseDescriptor.getDefaultCompaction();
         if (defaultCompaction == null)
         {
-            DEFAULT = new CompactionParams(SizeTieredCompactionStrategy.class,
-                                           DEFAULT_THRESHOLDS,
+            DEFAULT = new CompactionParams(UnifiedCompactionStrategy.class,
+                                           Collections.emptyMap(),
                                            DEFAULT_ENABLED,
                                            DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES_PROPERTY_VALUE);
         }
@@ -105,20 +99,18 @@ public static Optional<TombstoneOption> forName(String name)
         }
     }
 
-    private final Class<? extends AbstractCompactionStrategy> klass;
-    private final ImmutableMap<String, String> options;
+    private final CompactionStrategyOptions strategyOptions;
     private final boolean isEnabled;
     private final TombstoneOption tombstoneOption;
 
-    private CompactionParams(Class<? extends AbstractCompactionStrategy> klass, Map<String, String> options, boolean isEnabled, TombstoneOption tombstoneOption)
+    private CompactionParams(Class<? extends CompactionStrategy> klass, Map<String, String> options, boolean isEnabled, TombstoneOption tombstoneOption)
     {
-        this.klass = klass;
-        this.options = ImmutableMap.copyOf(options);
+        this.strategyOptions = new CompactionStrategyOptions(klass, options, true);
         this.isEnabled = isEnabled;
         this.tombstoneOption = tombstoneOption;
     }
 
-    public static CompactionParams create(Class<? extends AbstractCompactionStrategy> klass, Map<String, String> options)
+    public static CompactionParams create(Class<? extends CompactionStrategy> klass, Map<String, String> options)
     {
         boolean isEnabled = options.containsKey(Option.ENABLED.toString())
                           ? Boolean.parseBoolean(options.get(Option.ENABLED.toString()))
@@ -134,14 +126,7 @@ public static CompactionParams create(Class<? extends AbstractCompactionStrategy
         }
         TombstoneOption tombstoneOption = tombstoneOptional.get();
 
-        Map<String, String> allOptions = new HashMap<>(options);
-        if (supportsThresholdParams(klass))
-        {
-            allOptions.putIfAbsent(Option.MIN_THRESHOLD.toString(), Integer.toString(DEFAULT_MIN_THRESHOLD));
-            allOptions.putIfAbsent(Option.MAX_THRESHOLD.toString(), Integer.toString(DEFAULT_MAX_THRESHOLD));
-        }
-
-        return new CompactionParams(klass, allOptions, isEnabled, tombstoneOption);
+        return new CompactionParams(klass, new HashMap<>(options), isEnabled, tombstoneOption);
     }
 
     public static CompactionParams stcs(Map<String, String> options)
@@ -166,18 +151,12 @@ public static CompactionParams twcs(Map<String, String> options)
 
     public int minCompactionThreshold()
     {
-        String threshold = options.get(Option.MIN_THRESHOLD.toString());
-        return threshold == null
-             ? DEFAULT_MIN_THRESHOLD
-             : Integer.parseInt(threshold);
+        return strategyOptions.minCompactionThreshold();
     }
 
     public int maxCompactionThreshold()
     {
-        String threshold = options.get(Option.MAX_THRESHOLD.toString());
-        return threshold == null
-             ? DEFAULT_MAX_THRESHOLD
-             : Integer.parseInt(threshold);
+        return strategyOptions.maxCompactionThreshold();
     }
 
     public TombstoneOption tombstoneOption()
@@ -185,87 +164,14 @@ public TombstoneOption tombstoneOption()
         return tombstoneOption;
     }
 
-    public void validate()
-    {
-        try
-        {
-            Map<?, ?> unknownOptions = (Map) klass.getMethod("validateOptions", Map.class).invoke(null, options);
-            if (!unknownOptions.isEmpty())
-            {
-                throw new ConfigurationException(format("Properties specified %s are not understood by %s",
-                                                        unknownOptions.keySet(),
-                                                        klass.getSimpleName()));
-            }
-        }
-        catch (NoSuchMethodException e)
-        {
-            logger.warn("Compaction strategy {} does not have a static validateOptions method. Validation ignored",
-                        klass.getName());
-        }
-        catch (InvocationTargetException e)
-        {
-            if (e.getTargetException() instanceof ConfigurationException)
-                throw (ConfigurationException) e.getTargetException();
-
-            Throwable cause = e.getCause() == null
-                            ? e
-                            : e.getCause();
-
-            throw new ConfigurationException(format("%s.validateOptions() threw an error: %s %s",
-                                                    klass.getName(),
-                                                    cause.getClass().getName(),
-                                                    cause.getMessage()),
-                                             e);
-        }
-        catch (IllegalAccessException e)
-        {
-            throw new ConfigurationException("Cannot access method validateOptions in " + klass.getName(), e);
-        }
-
-        String minThreshold = options.get(Option.MIN_THRESHOLD.toString());
-        if (minThreshold != null && !StringUtils.isNumeric(minThreshold))
-        {
-            throw new ConfigurationException(format("Invalid value %s for '%s' compaction sub-option - must be an integer",
-                                                    minThreshold,
-                                                    Option.MIN_THRESHOLD));
-        }
-
-        String maxThreshold = options.get(Option.MAX_THRESHOLD.toString());
-        if (maxThreshold != null && !StringUtils.isNumeric(maxThreshold))
-        {
-            throw new ConfigurationException(format("Invalid value %s for '%s' compaction sub-option - must be an integer",
-                                                    maxThreshold,
-                                                    Option.MAX_THRESHOLD));
-        }
-
-        if (minCompactionThreshold() <= 0 || maxCompactionThreshold() <= 0)
-        {
-            throw new ConfigurationException("Disabling compaction by setting compaction thresholds to 0 has been removed,"
-                                             + " set the compaction option 'enabled' to false instead.");
-        }
-
-        if (minCompactionThreshold() <= 1)
-        {
-            throw new ConfigurationException(format("Min compaction threshold cannot be less than 2 (got %d)",
-                                                    minCompactionThreshold()));
-        }
-
-        if (minCompactionThreshold() > maxCompactionThreshold())
-        {
-            throw new ConfigurationException(format("Min compaction threshold (got %d) cannot be greater than max compaction threshold (got %d)",
-                                                    minCompactionThreshold(),
-                                                    maxCompactionThreshold()));
-        }
-    }
-
     double defaultBloomFilterFbChance()
     {
-        return klass.equals(LeveledCompactionStrategy.class) ? 0.1 : 0.01;
+        return klass().equals(LeveledCompactionStrategy.class) ? 0.1 : 0.01;
     }
 
-    public Class<? extends AbstractCompactionStrategy> klass()
+    public Class<? extends CompactionStrategy> klass()
     {
-        return klass;
+        return strategyOptions.klass();
     }
 
     /**
@@ -273,7 +179,7 @@ public Class<? extends AbstractCompactionStrategy> klass()
      */
     public Map<String, String> options()
     {
-        return options;
+        return strategyOptions.getOptions();
     }
 
     public boolean isEnabled()
@@ -296,14 +202,14 @@ public static CompactionParams fromMap(Map<String, String> map)
         return create(classFromName(className), options);
     }
 
-    public static Class<? extends AbstractCompactionStrategy> classFromName(String name)
+    public static Class<? extends CompactionStrategy> classFromName(String name)
     {
         String className = name.contains(".")
                          ? name
                          : "org.apache.cassandra.db.compaction." + name;
-        Class<AbstractCompactionStrategy> strategyClass = FBUtilities.classForName(className, "compaction strategy");
+        Class<CompactionStrategy> strategyClass = FBUtilities.classForName(className, "compaction strategy");
 
-        if (!AbstractCompactionStrategy.class.isAssignableFrom(strategyClass))
+        if (!CompactionStrategy.class.isAssignableFrom(strategyClass))
         {
             throw new ConfigurationException(format("Compaction strategy class %s is not derived from AbstractReplicationStrategy",
                                                     className));
@@ -312,40 +218,17 @@ public static Class<? extends AbstractCompactionStrategy> classFromName(String n
         return strategyClass;
     }
 
-    /*
-     * LCS doesn't, STCS and DTCS do
-     */
-    @SuppressWarnings("unchecked")
-    public static boolean supportsThresholdParams(Class<? extends AbstractCompactionStrategy> klass)
-    {
-        try
-        {
-            Map<String, String> unrecognizedOptions =
-                (Map<String, String>) klass.getMethod("validateOptions", Map.class)
-                                           .invoke(null, DEFAULT_THRESHOLDS);
-
-            return unrecognizedOptions.isEmpty();
-        }
-        catch (Exception e)
-        {
-            throw new RuntimeException(e);
-        }
-    }
-
     public Map<String, String> asMap()
     {
         Map<String, String> map = new HashMap<>(options());
-        map.put(Option.CLASS.toString(), klass.getName());
+        map.put(Option.CLASS.toString(), klass().getName());
         return map;
     }
 
     @Override
     public String toString()
     {
-        return MoreObjects.toStringHelper(this)
-                          .add("class", klass.getName())
-                          .add("options", options)
-                          .toString();
+        return strategyOptions.toString();
     }
 
     @Override
@@ -359,12 +242,12 @@ public boolean equals(Object o)
 
         CompactionParams cp = (CompactionParams) o;
 
-        return klass.equals(cp.klass) && options.equals(cp.options);
+        return strategyOptions.equals(cp.strategyOptions);
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hash(klass, options);
+        return Objects.hash(strategyOptions);
     }
 }
diff --git a/src/java/org/apache/cassandra/schema/CompressionParams.java b/src/java/org/apache/cassandra/schema/CompressionParams.java
index 0e7c3da13ab0..aa4eaae1cd75 100644
--- a/src/java/org/apache/cassandra/schema/CompressionParams.java
+++ b/src/java/org/apache/cassandra/schema/CompressionParams.java
@@ -30,7 +30,12 @@
 
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cache.ChunkCache;
 import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.exceptions.ConfigurationException;
@@ -38,12 +43,15 @@
 import org.apache.cassandra.io.compress.*;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.PageAware;
 import org.apache.cassandra.net.MessagingService;
 
 import static java.lang.String.format;
 
 public final class CompressionParams
 {
+    private static final Logger logger = LoggerFactory.getLogger(CompressionParams.class);
+
     public static final int DEFAULT_CHUNK_LENGTH = 1024 * 16;
     public static final double DEFAULT_MIN_COMPRESS_RATIO = 0.0;        // Since pre-4.0 versions do not understand the
                                                                         // new compression parameter we can't use a
@@ -55,13 +63,27 @@ public final class CompressionParams
     public static final String ENABLED = "enabled";
     public static final String MIN_COMPRESS_RATIO = "min_compress_ratio";
 
+    public static final CompressionParams FAST = new CompressionParams(LZ4Compressor.create(Collections.emptyMap()),
+                                                                          DEFAULT_CHUNK_LENGTH,
+                                                                          calcMaxCompressedLength(DEFAULT_CHUNK_LENGTH, DEFAULT_MIN_COMPRESS_RATIO),
+                                                                          DEFAULT_MIN_COMPRESS_RATIO,
+                                                                          Collections.emptyMap());
+
+    public static final CompressionParams ADAPTIVE = new CompressionParams(AdaptiveCompressor.create(Collections.emptyMap()),
+                                                                           DEFAULT_CHUNK_LENGTH,
+                                                                           calcMaxCompressedLength(DEFAULT_CHUNK_LENGTH, DEFAULT_MIN_COMPRESS_RATIO),
+                                                                           DEFAULT_MIN_COMPRESS_RATIO,
+                                                                           Collections.emptyMap());
+
+    public static final CompressionParams FAST_ADAPTIVE = new CompressionParams(AdaptiveCompressor.createForFlush(Collections.emptyMap()),
+                                                                       DEFAULT_CHUNK_LENGTH,
+                                                                       calcMaxCompressedLength(DEFAULT_CHUNK_LENGTH, DEFAULT_MIN_COMPRESS_RATIO),
+                                                                       DEFAULT_MIN_COMPRESS_RATIO,
+                                                                       Collections.emptyMap());
+
     public static final CompressionParams DEFAULT = !CassandraRelevantProperties.DETERMINISM_SSTABLE_COMPRESSION_DEFAULT.getBoolean()
                                                     ? noCompression()
-                                                    : new CompressionParams(LZ4Compressor.create(Collections.emptyMap()),
-                                                                            DEFAULT_CHUNK_LENGTH,
-                                                                            calcMaxCompressedLength(DEFAULT_CHUNK_LENGTH, DEFAULT_MIN_COMPRESS_RATIO),
-                                                                            DEFAULT_MIN_COMPRESS_RATIO,
-                                                                            Collections.emptyMap());
+                                                    : DatabaseDescriptor.shouldUseAdaptiveCompressionByDefault() ? ADAPTIVE : FAST;
 
     public static final CompressionParams NOOP = new CompressionParams(NoopCompressor.create(Collections.emptyMap()),
                                                                        // 4 KiB is often the underlying disk block size
@@ -223,6 +245,24 @@ public boolean isEnabled()
         return sstableCompressor != null;
     }
 
+    /**
+     * Specializes the compressor for given use.
+     * May cause reconfiguration of parameters on some compressors.
+     * Returns null if params are not compatible with the given use.
+     */
+    public CompressionParams forUse(ICompressor.Uses use)
+    {
+        ICompressor specializedCompressor = this.sstableCompressor.forUse(use);
+        if (specializedCompressor == null)
+            return null;
+
+        assert specializedCompressor.recommendedUses().contains(use);
+        if (specializedCompressor == sstableCompressor)
+            return this;
+
+        return new CompressionParams(specializedCompressor, chunkLength, maxCompressedLength, minCompressRatio, otherOptions);
+    }
+
     /**
      * Returns the SSTable compressor.
      * @return the SSTable compressor or {@code null} if compression is disabled.
@@ -348,6 +388,10 @@ private static Integer parseChunkLength(String chLengthKB) throws ConfigurationE
             int parsed = Integer.parseInt(chLengthKB);
             if (parsed > Integer.MAX_VALUE / 1024)
                 throw new ConfigurationException(format("Value of %s is too large (%s)", CHUNK_LENGTH_IN_KB,parsed));
+            if (parsed * 1024 < PageAware.PAGE_SIZE && ChunkCache.instance != null && ChunkCache.instance.isEnabled())
+                logger.warn("Chunk length {} KiB is smaller than the page size {} KiB. " +
+                            "This is not recommended as it will cause wasted chunk cache space.",
+                            parsed, PageAware.PAGE_SIZE / 1024);
             return 1024 * parsed;
         }
         catch (NumberFormatException e)
diff --git a/src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java b/src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java
index a81affc26e34..21718ce1c0b0 100644
--- a/src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java
+++ b/src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java
@@ -68,7 +68,7 @@ public class DefaultSchemaUpdateHandler implements SchemaUpdateHandler, IEndpoin
     private MigrationCoordinator createMigrationCoordinator(MessagingService messagingService)
     {
         return new MigrationCoordinator(messagingService,
-                                        Stage.MIGRATION.executor(),
+                                        Stage.MIGRATION,
                                         ScheduledExecutors.scheduledTasks,
                                         MAX_OUTSTANDING_VERSION_REQUESTS,
                                         Gossiper.instance,
@@ -212,7 +212,7 @@ synchronized SchemaTransformationResult applyMutations(Collection<Mutation> sche
         // apply the schema mutations and fetch the new versions of the altered keyspaces
         Keyspaces updatedKeyspaces = SchemaKeyspace.fetchKeyspaces(affectedKeyspaces);
         Set<String> removedKeyspaces = affectedKeyspaces.stream().filter(ks -> !updatedKeyspaces.containsKeyspace(ks)).collect(Collectors.toSet());
-        Keyspaces afterKeyspaces = before.getKeyspaces().withAddedOrReplaced(updatedKeyspaces).without(removedKeyspaces);
+        Keyspaces afterKeyspaces = before.getKeyspaces().withAddedOrUpdated(updatedKeyspaces).without(removedKeyspaces);
 
         Keyspaces.KeyspacesDiff diff = Keyspaces.diff(before.getKeyspaces(), afterKeyspaces);
         UUID version = SchemaKeyspace.calculateSchemaDigest();
diff --git a/src/java/org/apache/cassandra/schema/DroppedColumn.java b/src/java/org/apache/cassandra/schema/DroppedColumn.java
index 90dfe651f7e0..2ba9dabf4154 100644
--- a/src/java/org/apache/cassandra/schema/DroppedColumn.java
+++ b/src/java/org/apache/cassandra/schema/DroppedColumn.java
@@ -20,17 +20,36 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
 
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.marshal.AbstractType;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
 public final class DroppedColumn
 {
     public final ColumnMetadata column;
     public final long droppedTime; // drop timestamp, in microseconds, yet with millisecond granularity
 
+    /**
+     * Creates a new dropped column record.
+     *
+     * @param column the metadata for the dropped column. This <b>must</b> be a dropped metadata, that is we should
+     * have {@code column.isDropped() == true}.
+     * @param droppedTime the time at which the column was dropped, in microseconds.
+     */
     public DroppedColumn(ColumnMetadata column, long droppedTime)
     {
+        assert column.isDropped() : column.debugString() + " should be dropped";
         this.column = column;
         this.droppedTime = droppedTime;
     }
 
+    public DroppedColumn withNewKeyspace(String newKeyspace, Types udts)
+    {
+        return new DroppedColumn(column.withNewKeyspace(newKeyspace, udts), droppedTime);
+    }
+
     @Override
     public boolean equals(Object o)
     {
@@ -51,9 +70,54 @@ public int hashCode()
         return Objects.hashCode(column, droppedTime);
     }
 
+    public String toCQLString()
+    {
+        return String.format("DROPPED COLUMN RECORD %s %s%s USING TIMESTAMP %d",
+                             column.name.toCQLString(),
+                             column.type.asCQL3Type().toSchemaString(),
+                             column.isStatic() ? " static" : "",
+                             droppedTime);
+    }
+
     @Override
     public String toString()
     {
         return MoreObjects.toStringHelper(this).add("column", column).add("droppedTime", droppedTime).toString();
     }
+
+    /**
+     * A parsed dropped column record (from CREATE TABLE ... WITH DROPPED COLUMN RECORD ...).
+     */
+    public static final class Raw
+    {
+        private final ColumnIdentifier name;
+        private final CQL3Type.Raw type;
+        private final boolean isStatic;
+        private final long timestamp;
+
+        public Raw(ColumnIdentifier name, CQL3Type.Raw type, boolean isStatic, long timestamp)
+        {
+            this.name = name;
+            this.type = type;
+            this.isStatic = isStatic;
+            this.timestamp = timestamp;
+        }
+
+        public DroppedColumn prepare(String keyspace, String table, Types types)
+        {
+            ColumnMetadata.Kind kind = isStatic ? ColumnMetadata.Kind.STATIC : ColumnMetadata.Kind.REGULAR;
+            AbstractType<?> parsedType = type.prepare(keyspace, types).getType();
+            if (parsedType.referencesUserTypes())
+                throw invalidRequest("Invalid type %s for DROPPED COLUMN RECORD on %s: dropped column types should "
+                                     + "not have user types", type, name);
+
+            ColumnMetadata droppedColumn = ColumnMetadata.droppedColumn(keyspace,
+                                                                        table,
+                                                                        name,
+                                                                        parsedType,
+                                                                        kind, 
+                                                                        null);
+            return new DroppedColumn(droppedColumn, timestamp);
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/schema/IndexMetadata.java b/src/java/org/apache/cassandra/schema/IndexMetadata.java
index 795abad984b4..469f50732fca 100644
--- a/src/java/org/apache/cassandra/schema/IndexMetadata.java
+++ b/src/java/org/apache/cassandra/schema/IndexMetadata.java
@@ -20,9 +20,11 @@
 
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import com.google.common.base.Objects;
@@ -36,17 +38,23 @@
 import org.apache.cassandra.cql3.CqlBuilder;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.exceptions.UnknownIndexException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.sasi.SASIIndex;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.UUIDSerializer;
 
+import javax.annotation.Nullable;
+
+import static org.apache.cassandra.schema.SchemaConstants.PATTERN_NON_WORD_CHAR;
+import static org.apache.cassandra.schema.SchemaConstants.isValidName;
+
 /**
  * An immutable representation of secondary index metadata.
  */
@@ -54,12 +62,9 @@ public final class IndexMetadata
 {
     private static final Logger logger = LoggerFactory.getLogger(IndexMetadata.class);
 
-    private static final Pattern PATTERN_NON_WORD_CHAR = Pattern.compile("\\W");
-    private static final Pattern PATTERN_WORD_CHARS = Pattern.compile("\\w+");
-
-
     public static final Serializer serializer = new Serializer();
 
+    static final String INDEX_POSTFIX = "_idx";
     /**
      * A mapping of user-friendly index names to their fully qualified index class names.
      */
@@ -106,30 +111,60 @@ public static IndexMetadata fromIndexTargets(List<IndexTarget> targets,
     {
         Map<String, String> newOptions = new HashMap<>(options);
         newOptions.put(IndexTarget.TARGET_OPTION_NAME, targets.stream()
-                                                              .map(target -> target.asCqlString())
+                                                              .map(IndexTarget::asCqlString)
                                                               .collect(Collectors.joining(", ")));
         return new IndexMetadata(name, newOptions, kind);
     }
 
-    public static boolean isNameValid(String name)
+    /**
+     * Generates a default index name from the table and column names.
+     * Characters other than alphanumeric and underscore are removed.
+     * Long index names are truncated to fit the length allowing constructing filenames.
+     *
+     * @param table  the table name
+     * @param column the column identifier. Can be null if the index is not column specific.
+     * @return the generated index name
+     */
+    public static String generateDefaultIndexName(String table, @Nullable ColumnIdentifier column)
     {
-        return name != null && !name.isEmpty() && PATTERN_WORD_CHARS.matcher(name).matches();
+        String indexNameUncleaned = table;
+        if (column != null)
+            indexNameUncleaned += '_' + column.toString();
+        String indexNameUntrimmed = PATTERN_NON_WORD_CHAR.matcher(indexNameUncleaned).replaceAll("");
+        String indexNameTrimmed = indexNameUntrimmed
+                                  .substring(0,
+                                             Math.min(calculateGeneratedIndexNameMaxLength(),
+                                                      indexNameUntrimmed.length()));
+        return indexNameTrimmed + INDEX_POSTFIX;
     }
 
-    public static String generateDefaultIndexName(String table, ColumnIdentifier column)
+    /**
+     * Calculates the maximum length of the generated index name to fit file names.
+     * It includes the generated suffixes in account.
+     * The calculation depends on how index implements file names construciton from index names.
+     * This needs to be addressed, see CNDB-13240.
+     *
+     * @return the allowed length of the generated index name
+     */
+    private static int calculateGeneratedIndexNameMaxLength()
     {
-        return PATTERN_NON_WORD_CHAR.matcher(table + "_" + column.toString() + "_idx").replaceAll("");
-    }
+        // Speculative assumption that uniqueness breaker will fit into 999.
+        // The value is used for trimming the index name if needed.
+        // Introducing validation of index name length is TODO for CNDB-13198.
+        int uniquenessSuffixLength = 4;
+        int indexNameAddition = uniquenessSuffixLength + INDEX_POSTFIX.length();
+        int allowedIndexNameLength = Version.calculateIndexNameAllowedLength();
 
-    public static String generateDefaultIndexName(String table)
-    {
-        return PATTERN_NON_WORD_CHAR.matcher(table + "_" + "idx").replaceAll("");
+        assert allowedIndexNameLength >= indexNameAddition : "cannot happen with current implementation as allowedIndexNameLength is approximately 255 - ~76. However, allowedIndexNameLength was " + allowedIndexNameLength + " and  indexNameAddition was " + indexNameAddition;
+
+        return allowedIndexNameLength - indexNameAddition;
     }
 
     public void validate(TableMetadata table)
     {
-        if (!isNameValid(name))
-            throw new ConfigurationException("Illegal index name " + name);
+        if (!isValidName(name, true))
+            throw new ConfigurationException(String.format("Index name must not be empty, or contain non-alphanumeric-underscore characters (got \"%s\")",
+                                                           name));
 
         if (kind == null)
             throw new ConfigurationException("Index kind is null for index " + name);
@@ -139,9 +174,8 @@ public void validate(TableMetadata table)
             if (options == null || !options.containsKey(IndexTarget.CUSTOM_INDEX_OPTION_NAME))
                 throw new ConfigurationException(String.format("Required option missing for index %s : %s",
                                                                name, IndexTarget.CUSTOM_INDEX_OPTION_NAME));
-
-            // Get the fully qualified class name:
-            String className = getIndexClassName();
+            // Find any aliases to the fully qualified index class name:
+            String className = expandAliases(options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
 
             Class<Index> indexerClass = FBUtilities.classForName(className, "custom indexer");
             if (!Index.class.isAssignableFrom(indexerClass))
@@ -153,13 +187,15 @@ public void validate(TableMetadata table)
     public String getIndexClassName()
     {
         if (isCustom())
-        {
-            String className = options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
-            return indexNameAliases.getOrDefault(className.toLowerCase(), className);
-        }
+            return expandAliases(options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
         return CassandraIndex.class.getName();
     }
 
+    public static String expandAliases(String className)
+    {
+        return indexNameAliases.getOrDefault(className.toLowerCase(), className);
+    }
+
     private void validateCustomIndexOptions(TableMetadata table, Class<? extends Index> indexerClass, Map<String, String> options)
     {
         try
@@ -189,11 +225,11 @@ private void validateCustomIndexOptions(TableMetadata table, Class<? extends Ind
         }
         catch (InvocationTargetException e)
         {
-            if (e.getTargetException() instanceof InvalidRequestException)
-                throw (InvalidRequestException) e.getTargetException();
+            if (e.getTargetException() instanceof RequestValidationException)
+                throw (RequestValidationException) e.getTargetException();
             if (e.getTargetException() instanceof ConfigurationException)
                 throw (ConfigurationException) e.getTargetException();
-            throw new ConfigurationException("Failed to validate custom indexer options: " + options);
+            throw new ConfigurationException("Failed to validate custom indexer options: " + options, e);
         }
         catch (ConfigurationException e)
         {
diff --git a/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java b/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
index 5d62cf8b43f6..4462f94b22a3 100644
--- a/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
+++ b/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
@@ -21,11 +21,11 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Stream;
-
 import javax.annotation.Nullable;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -37,15 +37,17 @@
 import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
-import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
+import org.apache.cassandra.locator.TokenMetadataProvider;
 import org.apache.cassandra.schema.Tables.TablesDiff;
 import org.apache.cassandra.schema.Types.TypesDiff;
+import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
 import org.apache.cassandra.schema.Views.ViewsDiff;
-import org.apache.cassandra.service.StorageService;
-
-import static java.lang.String.format;
+import org.apache.cassandra.service.ClientState;
 
 import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.transform;
+import static com.google.common.collect.Maps.transformValues;
+import static java.lang.String.format;
 
 /**
  * An immutable representation of keyspace metadata (name, params, tables, types, and functions).
@@ -96,6 +98,30 @@ public static KeyspaceMetadata virtual(String name, Tables tables)
         return new KeyspaceMetadata(name, Kind.VIRTUAL, KeyspaceParams.local(), tables, Views.none(), Types.none(), UserFunctions.none());
     }
 
+    public KeyspaceMetadata rename(String newName)
+    {
+        Types newTypes = types.withNewKeyspace(newName);
+        UserFunctions newFunctions = userFunctions.withNewKeyspace(newName, newTypes);
+        Tables newTables = tables.withNewKeyspace(newName, newTypes);
+
+        Views.Builder viewsBuilder = Views.builder();
+
+        for (ViewMetadata view : views)
+        {
+            TableMetadata newMetadata = newTables.getNullable(view.baseTableName);
+            TableMetadata.Builder tableBuilder = TableMetadata.builder(newName, view.metadata.name)
+                                                              .partitioner(view.metadata.partitioner)
+                                                              .kind(view.metadata.kind)
+                                                              .params(view.metadata.params)
+                                                              .flags(view.metadata.flags)
+                                                              .addColumns(transform(view.metadata.columns(), c -> c.withNewKeyspace(newName, newTypes)))
+                                                              .droppedColumns(transformValues(view.metadata.droppedColumns, c -> c.withNewKeyspace(newName, newTypes)));
+            viewsBuilder.put(new ViewMetadata(newMetadata.id, newMetadata.name, view.includeAllColumns, view.whereClause, tableBuilder.build()));
+        }
+
+        return new KeyspaceMetadata(newName, kind, params, newTables, viewsBuilder.build(), newTypes, newFunctions);
+    }
+
     public KeyspaceMetadata withSwapped(KeyspaceParams params)
     {
         return new KeyspaceMetadata(name, kind, params, tables, views, types, userFunctions);
@@ -121,6 +147,38 @@ public KeyspaceMetadata withSwapped(UserFunctions functions)
         return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
     }
 
+    /**
+     * Returns a new instance of this {@link KeyspaceMetadata} which is obtained by applying the provided
+     * <code>transformFunction</code> to the {@link TableParams} of all the tables and views contained in
+     * this keyspace.
+     *
+     * @param transformFunction the function used to transform the table parameters
+     * @return a copy of this keyspace with table params transformed in all tables and views
+     */
+    public KeyspaceMetadata withTransformedTableParams(java.util.function.Function<TableParams, TableParams> transformFunction)
+    {
+        // Transform the params for all the tables
+        Tables newTables = tables.withTransformedParams(transformFunction);
+        Views.Builder newViews = Views.builder();
+
+        // Then transform the params for all the views
+        for (ViewMetadata view : views)
+        {
+            String baseTableName = view.baseTableName;
+            TableMetadata newBaseTable = newTables.getNullable(baseTableName);
+            Preconditions.checkNotNull(newBaseTable, "Table " + baseTableName + " is the base table of the view "
+                                        + view.name() + " but has not been found among the updated tables.");
+
+            newViews.put(new ViewMetadata(view.baseTableId,
+                                          view.baseTableName,
+                                          view.includeAllColumns,
+                                          view.whereClause,
+                                          newBaseTable));
+        }
+
+        return new KeyspaceMetadata(name, kind, params, newTables, newViews.build(), types, userFunctions);
+    }
+
     public KeyspaceMetadata empty()
     {
         return new KeyspaceMetadata(this.name, this.kind, this.params, Tables.none(), Views.none(), Types.none(), UserFunctions.none());
@@ -315,7 +373,7 @@ public String toCqlString(boolean withInternals, boolean ifNotExists)
         return builder.toString();
     }
 
-    public void validate()
+    public void validate(ClientState clientState)
     {
         if (!SchemaConstants.isValidName(name))
         {
@@ -325,7 +383,7 @@ public void validate()
                                                     name));
         }
 
-        params.validate(name, null);
+        params.validate(name, clientState);
 
         tablesAndViews().forEach(TableMetadata::validate);
 
@@ -346,7 +404,7 @@ public AbstractReplicationStrategy createReplicationStrategy()
     {
         return AbstractReplicationStrategy.createReplicationStrategy(name,
                                                                      params.replication.klass,
-                                                                     StorageService.instance.getTokenMetadata(),
+                                                                     TokenMetadataProvider.instance.getTokenMetadataForKeyspace(name),
                                                                      DatabaseDescriptor.getEndpointSnitch(),
                                                                      params.replication.options);
     }
diff --git a/src/java/org/apache/cassandra/schema/KeyspaceParams.java b/src/java/org/apache/cassandra/schema/KeyspaceParams.java
index 539993e2b32a..644e6b832808 100644
--- a/src/java/org/apache/cassandra/schema/KeyspaceParams.java
+++ b/src/java/org/apache/cassandra/schema/KeyspaceParams.java
@@ -17,11 +17,22 @@
  */
 package org.apache.cassandra.schema;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.locator.NetworkTopologyStrategy;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_NTS_DC_OVERRIDE_PROPERTY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_NTS_RF_OVERRIDE_PROPERTY;
 
 import org.apache.cassandra.service.ClientState;
 
@@ -30,6 +41,8 @@
  */
 public final class KeyspaceParams
 {
+    private static final Logger logger = LoggerFactory.getLogger(KeyspaceParams.class);
+
     public static final boolean DEFAULT_DURABLE_WRITES = true;
 
     /**
@@ -55,6 +68,8 @@ public String toString()
     public final boolean durableWrites;
     public final ReplicationParams replication;
 
+    private static final Map<String, String> SYSTEM_DISTRIBUTED_NTS_OVERRIDE = getSystemDistributedNtsOverride();
+
     public KeyspaceParams(boolean durableWrites, ReplicationParams replication)
     {
         this.durableWrites = durableWrites;
@@ -86,6 +101,11 @@ public static KeyspaceParams simpleTransient(int replicationFactor)
         return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor));
     }
 
+    public static KeyspaceParams everywhere()
+    {
+        return new KeyspaceParams(true, ReplicationParams.everywhere());
+    }
+
     public static KeyspaceParams nts(Object... args)
     {
         return new KeyspaceParams(true, ReplicationParams.nts(args));
@@ -96,6 +116,25 @@ public void validate(String name, ClientState state)
         replication.validate(name, state);
     }
 
+    /**
+     * Used to pick the default replication strategy for all distributed system keyspaces.
+     * The default will be SimpleStrategy and a hard coded RF factor.
+     * <p>
+     * One can change this default to NTS by passing in system properties:
+     * -Dcassandra.system_distributed_replication_per_dc=3
+     * -Dcassandra.system_distributed_replication_dc_names=cloud-east,cloud-west
+     */
+    public static KeyspaceParams systemDistributed(int rf)
+    {
+        if (!SYSTEM_DISTRIBUTED_NTS_OVERRIDE.isEmpty())
+        {
+            logger.info("Using override for distributed system keyspaces: {}", SYSTEM_DISTRIBUTED_NTS_OVERRIDE);
+            return create(true, SYSTEM_DISTRIBUTED_NTS_OVERRIDE);
+        }
+
+        return simple(rf);
+    }
+
     @Override
     public boolean equals(Object o)
     {
@@ -124,4 +163,41 @@ public String toString()
                           .add(Option.REPLICATION.toString(), replication)
                           .toString();
     }
+
+    @VisibleForTesting
+    static Map<String, String> getSystemDistributedNtsOverride()
+    {
+        int rfOverride = -1;
+        List<String> dcOverride = Collections.emptyList();
+        ImmutableMap.Builder<String, String> ntsOverride = ImmutableMap.builder();
+
+        try
+        {
+            rfOverride = SYSTEM_DISTRIBUTED_NTS_RF_OVERRIDE_PROPERTY.getInt(-1);
+            dcOverride = Splitter.on(',').trimResults().omitEmptyStrings().splitToList(SYSTEM_DISTRIBUTED_NTS_DC_OVERRIDE_PROPERTY.getString(","));
+        }
+        catch (RuntimeException ex)
+        {
+            logger.error("Error parsing system distributed replication override properties", ex);
+        }
+
+        if (rfOverride != -1 && !dcOverride.isEmpty())
+        {
+            // Validate reasonable defaults
+            if (rfOverride <= 0 || rfOverride > 5)
+            {
+                logger.error("Invalid value for {}", SYSTEM_DISTRIBUTED_NTS_RF_OVERRIDE_PROPERTY.getKey());
+            }
+            else
+            {
+                for (String dc : dcOverride)
+                    ntsOverride.put(dc, String.valueOf(rfOverride));
+
+                ntsOverride.put(ReplicationParams.CLASS, NetworkTopologyStrategy.class.getCanonicalName());
+                return ntsOverride.build();
+            }
+        }
+
+        return Collections.emptyMap();
+    }
 }
diff --git a/src/java/org/apache/cassandra/schema/Keyspaces.java b/src/java/org/apache/cassandra/schema/Keyspaces.java
index e9bd92c7c5d7..24f8e44dfbcd 100644
--- a/src/java/org/apache/cassandra/schema/Keyspaces.java
+++ b/src/java/org/apache/cassandra/schema/Keyspaces.java
@@ -129,25 +129,17 @@ public Keyspaces without(Collection<String> names)
         return filter(k -> !names.contains(k.name));
     }
 
-    public Keyspaces withAddedOrUpdated(KeyspaceMetadata keyspace)
-    {
-        return builder().add(Iterables.filter(this, k -> !k.name.equals(keyspace.name)))
-                        .add(keyspace)
-                        .build();
-    }
-
     /**
      * Returns a new {@link Keyspaces} equivalent to this one, but with the provided keyspace metadata either added (if
      * this {@link Keyspaces} does not have that keyspace), or replaced by the provided definition.
      *
      * <p>Note that if this contains the provided keyspace, its pre-existing definition is discarded and completely
-     * replaced with the newly provided one. See {@link #withAddedOrUpdated(KeyspaceMetadata)} if you wish the provided
-     * definition to be "merged" with the existing one instead.
+     * replaced with the newly provided one.
      *
      * @param keyspace the keyspace metadata to add, or replace the existing definition with.
      * @return the newly created object.
      */
-    public Keyspaces withAddedOrReplaced(KeyspaceMetadata keyspace)
+    public Keyspaces withAddedOrUpdated(KeyspaceMetadata keyspace)
     {
         return builder().add(Iterables.filter(this, k -> !k.name.equals(keyspace.name)))
                         .add(keyspace)
@@ -155,23 +147,18 @@ public Keyspaces withAddedOrReplaced(KeyspaceMetadata keyspace)
     }
 
     /**
-     * Calls {@link #withAddedOrReplaced(KeyspaceMetadata)} on all the keyspaces of the provided {@link Keyspaces}.
+     * Calls {@link #withAddedOrUpdated(Keyspaces)} on all the keyspaces of the provided {@link Keyspaces}.
      *
      * @param keyspaces the keyspaces to add, or replace if existing.
      * @return the newly created object.
      */
-    public Keyspaces withAddedOrReplaced(Keyspaces keyspaces)
+    public Keyspaces withAddedOrUpdated(Keyspaces keyspaces)
     {
         return builder().add(Iterables.filter(this, k -> !keyspaces.containsKeyspace(k.name)))
                         .add(keyspaces)
                         .build();
     }
 
-    public void validate()
-    {
-        keyspaces.values().forEach(KeyspaceMetadata::validate);
-    }
-
     @Override
     public boolean equals(Object o)
     {
diff --git a/src/java/org/apache/cassandra/schema/MemtableParams.java b/src/java/org/apache/cassandra/schema/MemtableParams.java
index 7d88f6518ee5..27bee6dcc729 100644
--- a/src/java/org/apache/cassandra/schema/MemtableParams.java
+++ b/src/java/org/apache/cassandra/schema/MemtableParams.java
@@ -36,7 +36,7 @@
 import org.apache.cassandra.config.InheritingClass;
 import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.db.memtable.SkipListMemtableFactory;
+import org.apache.cassandra.db.memtable.TrieMemtableFactory;
 import org.apache.cassandra.exceptions.ConfigurationException;
 
 /**
@@ -50,7 +50,7 @@
   */
 public final class MemtableParams
 {
-    private final Memtable.Factory factory;
+    public final Memtable.Factory factory;
     private final String configurationKey;
 
     private MemtableParams(Memtable.Factory factory, String configurationKey)
@@ -96,8 +96,8 @@ public int hashCode()
     }
 
     private static final String DEFAULT_CONFIGURATION_KEY = "default";
-    private static final Memtable.Factory DEFAULT_MEMTABLE_FACTORY = SkipListMemtableFactory.INSTANCE;
-    private static final ParameterizedClass DEFAULT_CONFIGURATION = SkipListMemtableFactory.CONFIGURATION;
+    private static final Memtable.Factory DEFAULT_MEMTABLE_FACTORY = TrieMemtableFactory.INSTANCE;
+    private static final ParameterizedClass DEFAULT_CONFIGURATION = TrieMemtableFactory.CONFIGURATION;
     private static final Map<String, ParameterizedClass>
         CONFIGURATION_DEFINITIONS = expandDefinitions(DatabaseDescriptor.getMemtableConfigurations());
     private static final Map<String, MemtableParams> CONFIGURATIONS = new HashMap<>();
@@ -135,6 +135,17 @@ public static MemtableParams getWithFallback(String key)
         }
     }
 
+    /**
+     * Useful for testing where we can provide a factory that produces spied instances of memtable so that we can
+     * modify behaviour of certains methods.
+     */
+    // Used by CNDB
+    @VisibleForTesting
+    public static MemtableParams forTesting(Memtable.Factory factory, String configurationKey)
+    {
+        return new MemtableParams(factory, configurationKey);
+    }
+
     @VisibleForTesting
     static Map<String, ParameterizedClass> expandDefinitions(Map<String, InheritingClass> memtableConfigurations)
     {
diff --git a/src/java/org/apache/cassandra/schema/MigrationCoordinator.java b/src/java/org/apache/cassandra/schema/MigrationCoordinator.java
index 980f3c217056..e1a499d262d9 100644
--- a/src/java/org/apache/cassandra/schema/MigrationCoordinator.java
+++ b/src/java/org/apache/cassandra/schema/MigrationCoordinator.java
@@ -49,9 +49,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.FutureTask;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.gms.ApplicationState;
@@ -229,7 +229,7 @@ public String toString()
     private final BiConsumer<InetAddressAndPort, Collection<Mutation>> schemaUpdateCallback;
     private final Set<InetAddressAndPort> lastPullFailures = new HashSet<>();
 
-    final ExecutorPlus executor;
+    final Stage executor;
 
     /**
      * Creates but does not start migration coordinator instance.
@@ -238,7 +238,7 @@ public String toString()
      * @param periodicCheckExecutor executor on which the periodic checks are scheduled
      */
     MigrationCoordinator(MessagingService messagingService,
-                         ExecutorPlus executor,
+                         Stage executor,
                          ScheduledExecutorService periodicCheckExecutor,
                          int maxOutstandingVersionRequests,
                          Gossiper gossiper,
diff --git a/src/java/org/apache/cassandra/schema/ReplicationParams.java b/src/java/org/apache/cassandra/schema/ReplicationParams.java
index 2998aa57ada8..2a75d1da2c91 100644
--- a/src/java/org/apache/cassandra/schema/ReplicationParams.java
+++ b/src/java/org/apache/cassandra/schema/ReplicationParams.java
@@ -58,6 +58,11 @@ static ReplicationParams simple(String replicationFactor)
         return new ReplicationParams(SimpleStrategy.class, ImmutableMap.of("replication_factor", replicationFactor));
     }
 
+    static ReplicationParams everywhere()
+    {
+        return new ReplicationParams(EverywhereStrategy.class, ImmutableMap.of());
+    }
+
     static ReplicationParams nts(Object... args)
     {
         assert args.length % 2 == 0;
@@ -74,7 +79,7 @@ static ReplicationParams nts(Object... args)
     public void validate(String name, ClientState state)
     {
         // Attempt to instantiate the ARS, which will throw a ConfigurationException if the options aren't valid.
-        TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+        TokenMetadata tmd = StorageService.instance.getTokenMetadataForKeyspace(name);
         IEndpointSnitch eps = DatabaseDescriptor.getEndpointSnitch();
         AbstractReplicationStrategy.validateReplicationStrategy(name, klass, tmd, eps, options, state);
     }
diff --git a/src/java/org/apache/cassandra/schema/Schema.java b/src/java/org/apache/cassandra/schema/Schema.java
index b704fd299700..e9f2b34cce36 100644
--- a/src/java/org/apache/cassandra/schema/Schema.java
+++ b/src/java/org/apache/cassandra/schema/Schema.java
@@ -52,6 +52,7 @@
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.exceptions.UnknownKeyspaceException;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.LocalStrategy;
@@ -93,7 +94,7 @@ public class Schema implements SchemaProvider
 
     private final Keyspaces localKeyspaces;
 
-    private volatile TableMetadataRefCache tableMetadataRefCache = TableMetadataRefCache.EMPTY;
+    private TableMetadataRefCache tableMetadataRefCache = TableMetadataRefCache.EMPTY;
 
     // Keyspace objects, one per keyspace. Only one instance should ever exist for any given keyspace.
     // We operate on loading map because we need to achieve atomic initialization with at-most-once semantics for
@@ -161,7 +162,7 @@ public void loadFromDisk()
      *
      * @param ksm The metadata about keyspace
      */
-    private synchronized void load(KeyspaceMetadata ksm)
+    public synchronized void load(KeyspaceMetadata ksm)
     {
         Preconditions.checkArgument(!SchemaConstants.isLocalSystemKeyspace(ksm.name));
         KeyspaceMetadata previous = distributedKeyspaces.getNullable(ksm.name);
@@ -212,8 +213,7 @@ public void unregisterListener(SchemaChangeListener listener)
      * Get keyspace instance by name
      *
      * @param keyspaceName The name of the keyspace
-     *
-     * @return Keyspace object or null if keyspace was not found
+     * @return Keyspace object or null if keyspace was not found, or if the keyspace has not completed construction yet
      */
     @Override
     public Keyspace getKeyspaceInstance(String keyspaceName)
@@ -243,12 +243,13 @@ public ColumnFamilyStore getColumnFamilyStoreInstance(TableId id)
     }
 
     @Override
-    public Keyspace maybeAddKeyspaceInstance(String keyspaceName, Supplier<Keyspace> loadFunction)
+    public Keyspace maybeAddKeyspaceInstance(String keyspaceName, Supplier<Keyspace> loadFunction) throws UnknownKeyspaceException
     {
         return keyspaceInstances.blockingLoadIfAbsent(keyspaceName, loadFunction);
     }
 
-    private Keyspace maybeRemoveKeyspaceInstance(String keyspaceName, Consumer<Keyspace> unloadFunction)
+    // Used by CNDB
+    public Keyspace maybeRemoveKeyspaceInstance(String keyspaceName, Consumer<Keyspace> unloadFunction)
     {
         try
         {
@@ -323,6 +324,14 @@ public KeyspaceMetadata getKeyspaceMetadata(String keyspaceName)
         return null != ksm ? ksm : VirtualKeyspaceRegistry.instance.getKeyspaceMetadataNullable(keyspaceName);
     }
 
+    /**
+     * Returns keyspaces that partition data across the ring.
+     */
+    public Keyspaces getPartitionedKeyspaces()
+    {
+        return distributedKeyspaces.filter(keyspace -> Keyspace.open(keyspace.name, this, true).getReplicationStrategy().isPartitioned());
+    }
+
     /**
      * Returns user keyspaces, that is all but {@link SchemaConstants#LOCAL_SYSTEM_KEYSPACE_NAMES},
      * {@link SchemaConstants#REPLICATED_SYSTEM_KEYSPACE_NAMES} or virtual keyspaces.
@@ -595,7 +604,8 @@ public void reloadSchemaAndAnnounceVersion()
     public synchronized void mergeAndUpdateVersion(SchemaTransformationResult result, boolean dropData)
     {
         result = localDiff(result);
-        assert result.after.getKeyspaces().stream().noneMatch(ksm -> ksm.params.replication.klass == LocalStrategy.class) : "LocalStrategy should not be used";
+        assert CassandraRelevantProperties.TEST_ALLOW_LOCAL_STRATEGY.getBoolean()
+               || result.after.getKeyspaces().stream().noneMatch(ksm -> ksm.params.replication.klass == LocalStrategy.class) : "LocalStrategy should not be used";
         schemaChangeNotifier.notifyPreChanges(result);
         merge(result.diff, dropData);
         updateVersion(result.after.getVersion());
@@ -689,9 +699,10 @@ private void createKeyspace(KeyspaceMetadata keyspace)
     {
         SchemaDiagnostics.keyspaceCreating(this, keyspace);
         load(keyspace);
+        Keyspace instance = null;
         if (Keyspace.isInitialized())
         {
-            Keyspace.open(keyspace.name, this, true);
+            instance = Keyspace.open(keyspace.name, this, true);
         }
 
         schemaChangeNotifier.notifyKeyspaceCreated(keyspace);
@@ -699,9 +710,9 @@ private void createKeyspace(KeyspaceMetadata keyspace)
 
         // If keyspace has been added, we need to recalculate pending ranges to make sure
         // we send mutations to the correct set of bootstrapping nodes. Refer CASSANDRA-15433.
-        if (keyspace.params.replication.klass != LocalStrategy.class && Keyspace.isInitialized())
+        if (keyspace.params.replication.klass != LocalStrategy.class && instance != null)
         {
-            PendingRangeCalculatorService.calculatePendingRanges(Keyspace.open(keyspace.name, this, true).getReplicationStrategy(), keyspace.name);
+            PendingRangeCalculatorService.instance.calculatePendingRanges(instance.getReplicationStrategy(), keyspace.name);
         }
     }
 
@@ -716,6 +727,7 @@ private void dropKeyspace(KeyspaceMetadata keyspaceMetadata, boolean dropData)
             if (keyspace == null)
                 return;
 
+            logger.debug("Dropping keyspace {}", keyspaceMetadata.name);
             keyspaceMetadata.views.forEach(v -> dropView(keyspace, v, dropData));
             keyspaceMetadata.tables.forEach(t -> dropTable(keyspace, t, dropData));
 
@@ -723,9 +735,11 @@ private void dropKeyspace(KeyspaceMetadata keyspaceMetadata, boolean dropData)
             Keyspace unloadedKeyspace = maybeRemoveKeyspaceInstance(keyspaceMetadata.name, ks -> {
                 ks.unload(dropData);
                 unload(keyspaceMetadata);
+                logger.debug("Instance removed for keyspace {}", ks.getName());
             });
             assert unloadedKeyspace == keyspace;
 
+            logger.debug("Awaiting on write barrier before dropping keyspace {}", keyspaceMetadata.name);
             Keyspace.writeOrder.awaitNewBarrier();
         }
         else
@@ -785,4 +799,26 @@ public Map<UUID, Set<InetAddressAndPort>> getOutstandingSchemaVersions()
                : Collections.emptyMap();
     }
 
-}
\ No newline at end of file
+    /**
+     * @return whether or not the keyspace is a really system one (w/ LocalStrategy, unmodifiable, hardcoded)
+     * or it's having {@link LocalStrategy}
+     */
+    public static boolean isKeyspaceWithLocalStrategy(String keyspaceName)
+    {
+        KeyspaceMetadata ksm = instance.getKeyspaceMetadata(keyspaceName);
+        return SchemaConstants.isLocalSystemKeyspace(keyspaceName) ||
+               (ksm != null && ksm.params.replication.klass.equals(LocalStrategy.class));
+    }
+
+    /**
+     * Equivalent to {@link #isKeyspaceWithLocalStrategy(String)} but uses the provided keyspace metadata instead
+     * of getting the metadata from the schema manager
+     *
+     * @param keyspace the keyspace metadata to check
+     * @return if the provided keyspace uses local replication strategy
+     */
+    public static boolean isKeyspaceWithLocalStrategy(KeyspaceMetadata keyspace)
+    {
+        return SchemaConstants.isLocalSystemKeyspace(keyspace.name) || keyspace.params.replication.klass.equals(LocalStrategy.class);
+    }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaConstants.java b/src/java/org/apache/cassandra/schema/SchemaConstants.java
index 2c36d3e35b6b..f9fb583f58d2 100644
--- a/src/java/org/apache/cassandra/schema/SchemaConstants.java
+++ b/src/java/org/apache/cassandra/schema/SchemaConstants.java
@@ -38,6 +38,8 @@
 public final class SchemaConstants
 {
     public static final Pattern PATTERN_WORD_CHARS = Pattern.compile("\\w+");
+    public static final Pattern PATTERN_NON_WORD_CHAR = Pattern.compile("\\W");
+
 
     public static final String SYSTEM_KEYSPACE_NAME = "system";
     public static final String SCHEMA_KEYSPACE_NAME = "system_schema";
@@ -49,36 +51,95 @@ public final class SchemaConstants
     public static final String VIRTUAL_SCHEMA = "system_virtual_schema";
 
     public static final String VIRTUAL_VIEWS = "system_views";
+    public static final String SCHEMA_VIRTUAL_KEYSPACE_NAME = "system_virtual_schema";
+    public static final String SYSTEM_VIEWS_KEYSPACE_NAME = "system_views";
 
     public static final String DUMMY_KEYSPACE_OR_TABLE_NAME = "--dummy--";
 
     /* system keyspace names (the ones with LocalStrategy replication strategy) */
-    public static final Set<String> LOCAL_SYSTEM_KEYSPACE_NAMES =
-        ImmutableSet.of(SYSTEM_KEYSPACE_NAME, SCHEMA_KEYSPACE_NAME);
+    public static final Set<String> LOCAL_SYSTEM_KEYSPACE_NAMES = ImmutableSet.of(SYSTEM_KEYSPACE_NAME, SCHEMA_KEYSPACE_NAME);
 
     /* virtual table system keyspace names */
     public static final Set<String> VIRTUAL_SYSTEM_KEYSPACE_NAMES =
         ImmutableSet.of(VIRTUAL_VIEWS, VIRTUAL_SCHEMA);
 
     /* replicate system keyspace names (the ones with a "true" replication strategy) */
-    public static final Set<String> REPLICATED_SYSTEM_KEYSPACE_NAMES =
-        ImmutableSet.of(TRACE_KEYSPACE_NAME, AUTH_KEYSPACE_NAME, DISTRIBUTED_KEYSPACE_NAME);
+    public static final Set<String> REPLICATED_SYSTEM_KEYSPACE_NAMES = ImmutableSet.of(TRACE_KEYSPACE_NAME, AUTH_KEYSPACE_NAME, DISTRIBUTED_KEYSPACE_NAME);
+
+    /* virtual keyspace names */
+    public static final Set<String> VIRTUAL_KEYSPACE_NAMES = ImmutableSet.of(SCHEMA_VIRTUAL_KEYSPACE_NAME, SYSTEM_VIEWS_KEYSPACE_NAME);
+
     /**
+     * Longest acceptable file name. Longer names lead to file write or read errors.
+     */
+    public static final int FILENAME_LENGTH = 255;
+
+     /**
      * The longest permissible KS or CF name.
      *
      * Before CASSANDRA-16956, we used to care about not having the entire path longer than 255 characters because of
      * Windows support but this limit is by implementing CASSANDRA-16956 not in effect anymore.
+     *
+     * Note: This extended to 222 for CNDB tenant specific keyspaces.
+     * 222 is maximum filename length of 255 chars minus a separator char and
+     * 32 chars for table UUID.
      */
-    public static final int NAME_LENGTH = 48;
+    public static final int NAME_LENGTH = FILENAME_LENGTH - 32 - 1;
+
+    /**
+     * Longest permissible index name, so no index can fail on file name error.
+     * It is based on the most restrictive requirement coming from SAI and calculated by
+     * {@link org.apache.cassandra.index.sai.disk.format.Version#calculateIndexNameAllowedLength}.
+     * The exact number is used here, since it will be in user's documentation.
+     */
+    public static final int INDEX_NAME_LENGTH = 182;
 
     // 59adb24e-f3cd-3e02-97f0-5b395827453f
     public static final UUID emptyVersion;
 
     public static final List<String> LEGACY_AUTH_TABLES = Arrays.asList("credentials", "users", "permissions");
 
+    /**
+     * Checks if the length of the given name will be suitable to be used
+     * in constructed file names.
+     *
+     * @param name the name to check
+     * @return true if the name is short enough to be safe to use, otherwise false
+     */
+    public static boolean isSafeLengthForFilename(String name)
+    {
+        return name.length() <= NAME_LENGTH;
+    }
+
+    /**
+     * Names such as keyspace, table, index names are used in file paths and file names,
+     * so, they need to be safe for the use there, i.e., short enough and
+     * containing only alphanumeric characters and underscores.
+     *
+     * @param name the name to check
+     * @return whether the name is safe for use in file paths and file names
+     */
     public static boolean isValidName(String name)
     {
-        return name != null && !name.isEmpty() && name.length() <= NAME_LENGTH && PATTERN_WORD_CHARS.matcher(name).matches();
+        return isValidName(name, false);
+    }
+
+    /**
+     * Names such as keyspace, table, index names are used in file paths and file names,
+     * so, they need to be safe for the use there, i.e., short enough and
+     * containing only alphanumeric characters and underscores.
+     * However, historically not all names were checked for their length.
+     * Such legacy behaviour is supported through passing true for doNotCheckLength.
+     *
+     * @param name             the name to check
+     * @param doNotCheckLength specifies if no check on the name length should be done
+     *                         to support legacy behaviour
+     * @return true if the name is valid, false otherwise
+     */
+    public static boolean isValidName(String name, boolean doNotCheckLength)
+    {
+        return name != null && !name.isEmpty() && PATTERN_WORD_CHARS.matcher(name).matches() &&
+               (doNotCheckLength || isSafeLengthForFilename(name));
     }
 
     static
@@ -153,4 +214,27 @@ public static Set<String> getLocalAndReplicatedSystemTableNames()
                            .addAll(SystemDistributedKeyspace.TABLE_NAMES)
                            .build();
     }
+    
+    /**
+     * @return whether or not the keyspace is a virtual keyspace (system_virtual_schema, system_views)
+     */
+    public static boolean isVirtualKeyspace(String keyspaceName)
+    {
+        return VIRTUAL_KEYSPACE_NAMES.contains(keyspaceName.toLowerCase());
+    }
+
+    public static boolean isInternalKeyspace(String keyspaceName)
+    {
+        return isLocalSystemKeyspace(keyspaceName)
+               || isReplicatedSystemKeyspace(keyspaceName)
+               || isVirtualKeyspace(keyspaceName);
+    }
+    
+    /**
+     * @return whether or not the keyspace is a user keyspace
+     */
+    public static boolean isUserKeyspace(String keyspaceName)
+    {
+        return !isInternalKeyspace(keyspaceName);
+    }
 }
diff --git a/src/java/org/apache/cassandra/schema/SchemaDiagnostics.java b/src/java/org/apache/cassandra/schema/SchemaDiagnostics.java
index 29243039dae1..79bc5bca1bd2 100644
--- a/src/java/org/apache/cassandra/schema/SchemaDiagnostics.java
+++ b/src/java/org/apache/cassandra/schema/SchemaDiagnostics.java
@@ -20,11 +20,15 @@
 
 import com.google.common.collect.MapDifference;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.diag.DiagnosticEventService;
 import org.apache.cassandra.schema.SchemaEvent.SchemaEventType;
 
 final class SchemaDiagnostics
 {
+    private static final Logger logger = LoggerFactory.getLogger(SchemaDiagnostics.class);
     private static final DiagnosticEventService service = DiagnosticEventService.instance();
 
     private SchemaDiagnostics()
@@ -74,6 +78,8 @@ static void keyspaceCreated(Schema schema, KeyspaceMetadata keyspace)
 
     static void keyspaceAltering(Schema schema, KeyspaceMetadata.KeyspaceDiff delta)
     {
+        if (logger.isTraceEnabled())
+            logger.trace("Altering keyspace {}", delta.before.name);
         if (isEnabled(SchemaEventType.KS_ALTERING))
             service.publish(new SchemaEvent(SchemaEventType.KS_ALTERING, schema, delta.after,
                                             delta.before, delta, null, null, null, null));
@@ -81,6 +87,8 @@ static void keyspaceAltering(Schema schema, KeyspaceMetadata.KeyspaceDiff delta)
 
     static void keyspaceAltered(Schema schema, KeyspaceMetadata.KeyspaceDiff delta)
     {
+        if (logger.isTraceEnabled())
+            logger.trace("Keyspace {} altered", delta.before.name);
         if (isEnabled(SchemaEventType.KS_ALTERED))
             service.publish(new SchemaEvent(SchemaEventType.KS_ALTERED, schema, delta.after,
                                             delta.before, delta, null, null, null, null));
@@ -95,6 +103,8 @@ static void keyspaceDropping(Schema schema, KeyspaceMetadata keyspace)
 
     static void keyspaceDropped(Schema schema, KeyspaceMetadata keyspace)
     {
+        if (logger.isTraceEnabled())
+            logger.trace("Keyspace {} dropped", keyspace.name);
         if (isEnabled(SchemaEventType.KS_DROPPED))
             service.publish(new SchemaEvent(SchemaEventType.KS_DROPPED, schema, keyspace,
                                             null, null, null, null, null, null));
@@ -158,6 +168,8 @@ static void tableAltered(Schema schema, TableMetadata table)
 
     static void tableDropping(Schema schema, TableMetadata table)
     {
+        if (logger.isTraceEnabled())
+            logger.trace("Dropping table {}", table);
         if (isEnabled(SchemaEventType.TABLE_DROPPING))
             service.publish(new SchemaEvent(SchemaEventType.TABLE_DROPPING, schema, null,
                                             null, null, table, null, null, null));
diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
index f7044b55f7fa..53405abd8a08 100644
--- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
@@ -32,9 +32,20 @@
 import org.slf4j.LoggerFactory;
 
 import org.antlr.runtime.RecognitionException;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.Terms;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.WhereClause;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionResolver;
+import org.apache.cassandra.cql3.functions.ScalarFunction;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.cql3.functions.UserFunction;
 import org.apache.cassandra.cql3.functions.masking.ColumnMask;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.*;
@@ -55,12 +66,22 @@
 
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toSet;
-
 import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORE_CORRUPTED_SCHEMA_TABLES;
-import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_FLUSH_LOCAL_SCHEMA_CHANGES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UNSAFE_SYSTEM;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
 import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
-import static org.apache.cassandra.schema.SchemaKeyspaceTables.*;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.AGGREGATES;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.ALL;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.COLUMNS;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.COLUMN_MASKS;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.DROPPED_COLUMNS;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.FUNCTIONS;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.INDEXES;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.KEYSPACES;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.TABLES;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.TRIGGERS;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.TYPES;
+import static org.apache.cassandra.schema.SchemaKeyspaceTables.VIEWS;
 import static org.apache.cassandra.utils.Simulate.With.GLOBAL_CLOCK;
 
 /**
@@ -78,7 +99,6 @@ private SchemaKeyspace()
 
     private static final Logger logger = LoggerFactory.getLogger(SchemaKeyspace.class);
 
-    private static final boolean FLUSH_SCHEMA_TABLES = TEST_FLUSH_LOCAL_SCHEMA_CHANGES.getBoolean();
     private static final boolean IGNORE_CORRUPTED_SCHEMA_TABLES_PROPERTY_VALUE = IGNORE_CORRUPTED_SCHEMA_TABLES.getBoolean();
 
     /**
@@ -94,6 +114,7 @@ private SchemaKeyspace()
               + "keyspace_name text,"
               + "durable_writes boolean,"
               + "replication frozen<map<text, text>>,"
+              + "graph_engine text,"
               + "PRIMARY KEY ((keyspace_name)))");
 
     private static final TableMetadata Tables =
@@ -120,6 +141,7 @@ private SchemaKeyspace()
               + "max_index_interval int,"
               + "memtable_flush_period_in_ms int,"
               + "min_index_interval int,"
+              + "nodesync frozen<map<text, text>>,"
               + "read_repair_chance double," // no longer used, left for drivers' sake
               + "speculative_retry text,"
               + "additional_write_policy text,"
@@ -139,6 +161,7 @@ private SchemaKeyspace()
               + "kind text,"
               + "position int,"
               + "type text,"
+              + "required_for_liveness boolean,"
               + "PRIMARY KEY ((keyspace_name), table_name, column_name))");
 
     private static final TableMetadata ColumnMasks =
@@ -204,10 +227,12 @@ private SchemaKeyspace()
               + "max_index_interval int,"
               + "memtable_flush_period_in_ms int,"
               + "min_index_interval int,"
+              + "nodesync frozen<map<text, text>>,"
               + "read_repair_chance double," // no longer used, left for drivers' sake
               + "speculative_retry text,"
               + "additional_write_policy text,"
               + "cdc boolean,"
+              + "version int,"
               + "read_repair text,"
               + "PRIMARY KEY ((keyspace_name), view_name))");
 
@@ -244,6 +269,9 @@ private SchemaKeyspace()
               + "language text,"
               + "return_type text,"
               + "called_on_null_input boolean,"
+              + "deterministic boolean,"
+              + "monotonic boolean,"
+              + "monotonic_on frozen<list<text>>,"
               + "PRIMARY KEY ((keyspace_name), function_name, argument_types))");
 
     private static final TableMetadata Aggregates =
@@ -258,6 +286,7 @@ private SchemaKeyspace()
               + "return_type text,"
               + "state_func text,"
               + "state_type text,"
+              + "deterministic boolean,"
               + "PRIMARY KEY ((keyspace_name), aggregate_name, argument_types))");
 
     private static final List<TableMetadata> ALL_TABLE_METADATA = ImmutableList.of(Keyspaces,
@@ -357,7 +386,7 @@ static void truncate()
 
     private static void flush()
     {
-        if (!DatabaseDescriptor.isUnsafeSystem())
+        if (!UNSAFE_SYSTEM.getBoolean())
             ALL.forEach(table -> FBUtilities.waitOnFuture(getSchemaCFS(table).forceFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED)));
     }
 
@@ -520,7 +549,7 @@ private static void addTypeToSchemaMutation(UserType type, Mutation.SimpleBuilde
         mutation.update(Types)
                 .row(type.getNameAsString())
                 .add("field_names", type.fieldNames().stream().map(FieldIdentifier::toString).collect(toList()))
-                .add("field_types", type.fieldTypes().stream().map(AbstractType::asCQL3Type).map(CQL3Type::toString).collect(toList()));
+                .add("field_types", type.fieldTypes().stream().map(AbstractType::asCQL3Type).map(CQL3Type::toSchemaString).collect(toList()));
     }
 
     private static void addDropTypeToSchemaMutation(UserType type, Mutation.SimpleBuilder builder)
@@ -711,9 +740,7 @@ private static void addDropTableToSchemaMutation(TableMetadata table, Mutation.S
 
     private static void addColumnToSchemaMutation(TableMetadata table, ColumnMetadata column, Mutation.SimpleBuilder builder)
     {
-        AbstractType<?> type = column.type;
-        if (type instanceof ReversedType)
-            type = ((ReversedType<?>) type).baseType;
+        AbstractType<?> type = column.type.unwrap();
 
         builder.update(Columns)
                .row(table.name, column.name.toString())
@@ -721,7 +748,7 @@ private static void addColumnToSchemaMutation(TableMetadata table, ColumnMetadat
                .add("kind", column.kind.toString().toLowerCase())
                .add("position", column.position())
                .add("clustering_order", column.clusteringOrder().toString().toLowerCase())
-               .add("type", type.asCQL3Type().toString());
+               .add("type", type.asCQL3Type().toSchemaString());
 
         ColumnMask mask = column.getMask();
         if (SchemaConstants.isReplicatedSystemKeyspace(table.keyspace))
@@ -753,7 +780,7 @@ private static void addColumnToSchemaMutation(TableMetadata table, ColumnMetadat
                 for (int i = 0; i < numArgs; i++)
                 {
                     AbstractType<?> argType = partialTypes.get(i);
-                    types.add(argType.asCQL3Type().toString());
+                    types.add(argType.asCQL3Type().toSchemaString());
 
                     ByteBuffer argValue = partialValues.get(i);
                     boolean isNull = argValue == null;
@@ -781,7 +808,7 @@ private static void addDroppedColumnToSchemaMutation(TableMetadata table, Droppe
         builder.update(DroppedColumns)
                .row(table.name, column.column.name.toString())
                .add("dropped_time", new Date(TimeUnit.MICROSECONDS.toMillis(column.droppedTime)))
-               .add("type", column.column.type.asCQL3Type().toString())
+               .add("type", column.column.type.asCQL3Type().toSchemaString())
                .add("kind", column.column.kind.toString().toLowerCase());
     }
 
@@ -880,9 +907,12 @@ private static void addFunctionToSchemaMutation(UDFunction function, Mutation.Si
                .row(function.name().name, function.argumentsList())
                .add("body", function.body())
                .add("language", function.language())
-               .add("return_type", function.returnType().asCQL3Type().toString())
+               .add("return_type", function.returnType().asCQL3Type().toSchemaString())
                .add("called_on_null_input", function.isCalledOnNullInput())
-               .add("argument_names", function.argNames().stream().map((c) -> bbToString(c.bytes)).collect(toList()));
+               .add("argument_names", function.argNames().stream().map((c) -> bbToString(c.bytes)).collect(toList()))
+               .add("deterministic", function.isDeterministic())
+               .add("monotonic", function.isMonotonic())
+               .add("monotonic_on", function.monotonicOn().stream().map((c) -> bbToString(c.bytes)).collect(toList()));
     }
 
     private static String bbToString(ByteBuffer bb)
@@ -906,10 +936,11 @@ private static void addAggregateToSchemaMutation(UDAggregate aggregate, Mutation
     {
         builder.update(Aggregates)
                .row(aggregate.name().name, aggregate.argumentsList())
-               .add("return_type", aggregate.returnType().asCQL3Type().toString())
+               .add("return_type", aggregate.returnType().asCQL3Type().toSchemaString())
                .add("state_func", aggregate.stateFunction().name().name)
-               .add("state_type", aggregate.stateType().asCQL3Type().toString())
+               .add("state_type", aggregate.stateType().asCQL3Type().toSchemaString())
                .add("final_func", aggregate.finalFunction() != null ? aggregate.finalFunction().name().name : null)
+               .add("deterministic", aggregate.isDeterministic())
                .add("initcond", aggregate.initialCondition() != null
                                 // must use the frozen state type here, as 'null' for unfrozen collections may mean 'empty'
                                 ? aggregate.stateType().freeze().asCQL3Type().toCQLLiteral(aggregate.initialCondition())
@@ -1026,11 +1057,12 @@ private static TableMetadata fetchTable(String keyspaceName, String tableName, T
         UntypedResultSet.Row row = rows.one();
 
         Set<TableMetadata.Flag> flags = TableMetadata.Flag.fromStringSet(row.getFrozenSet("flags", UTF8Type.instance));
+        boolean isCounter = flags.contains(TableMetadata.Flag.COUNTER);
         return TableMetadata.builder(keyspaceName, tableName, TableId.fromUUID(row.getUUID("id")))
                             .flags(flags)
                             .params(createTableParamsFromRow(row))
-                            .addColumns(fetchColumns(keyspaceName, tableName, types, functions))
-                            .droppedColumns(fetchDroppedColumns(keyspaceName, tableName))
+                            .addColumns(fetchColumns(keyspaceName, tableName, types, functions, isCounter))
+                            .droppedColumns(fetchDroppedColumns(keyspaceName, tableName, flags.contains(TableMetadata.Flag.COUNTER)))
                             .indexes(fetchIndexes(keyspaceName, tableName))
                             .triggers(fetchTriggers(keyspaceName, tableName))
                             .build();
@@ -1073,7 +1105,7 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row)
         return builder.build();
     }
 
-    private static List<ColumnMetadata> fetchColumns(String keyspace, String table, Types types, UserFunctions functions)
+    private static List<ColumnMetadata> fetchColumns(String keyspace, String table, Types types, UserFunctions functions, boolean isCounterTable)
     {
         String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, COLUMNS);
         UntypedResultSet columnRows = query(query, keyspace, table);
@@ -1081,7 +1113,7 @@ private static List<ColumnMetadata> fetchColumns(String keyspace, String table,
             throw new MissingColumns("Columns not found in schema table for " + keyspace + '.' + table);
 
         List<ColumnMetadata> columns = new ArrayList<>();
-        columnRows.forEach(row -> columns.add(createColumnFromRow(row, types, functions)));
+        columnRows.forEach(row -> columns.add(createColumnFromRow(row, types, functions, isCounterTable)));
 
         if (columns.stream().noneMatch(ColumnMetadata::isPartitionKey))
             throw new MissingColumns("No partition key columns found in schema table for " + keyspace + "." + table);
@@ -1090,7 +1122,7 @@ private static List<ColumnMetadata> fetchColumns(String keyspace, String table,
     }
 
     @VisibleForTesting
-    public static ColumnMetadata createColumnFromRow(UntypedResultSet.Row row, Types types, UserFunctions functions)
+    public static ColumnMetadata createColumnFromRow(UntypedResultSet.Row row, Types types, UserFunctions functions, boolean isCounterTable)
     {
         String keyspace = row.getString("keyspace_name");
         String table = row.getString("table_name");
@@ -1104,7 +1136,10 @@ public static ColumnMetadata createColumnFromRow(UntypedResultSet.Row row, Types
         if (order == ClusteringOrder.DESC)
             type = ReversedType.getInstance(type);
 
-        ColumnIdentifier name = new ColumnIdentifier(row.getBytes("column_name_bytes"), row.getString("column_name"));
+        ByteBuffer columnNameBytes = row.getBytes("column_name_bytes");
+        type.validateForColumn(columnNameBytes, kind.isPrimaryKeyKind(), isCounterTable, false, false);
+
+        ColumnIdentifier name = new ColumnIdentifier(columnNameBytes, row.getString("column_name"));
 
         ColumnMask mask = null;
         String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ? AND column_name = ?",
@@ -1155,19 +1190,19 @@ else if (!(function instanceof ScalarFunction))
         return new ColumnMetadata(keyspace, table, name, type, position, kind, mask);
     }
 
-    private static Map<ByteBuffer, DroppedColumn> fetchDroppedColumns(String keyspace, String table)
+    private static Map<ByteBuffer, DroppedColumn> fetchDroppedColumns(String keyspace, String table, boolean isCounterTable)
     {
         String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, DROPPED_COLUMNS);
         Map<ByteBuffer, DroppedColumn> columns = new HashMap<>();
         for (UntypedResultSet.Row row : query(query, keyspace, table))
         {
-            DroppedColumn column = createDroppedColumnFromRow(row);
+            DroppedColumn column = createDroppedColumnFromRow(row, isCounterTable);
             columns.put(column.column.name.bytes, column);
         }
         return columns;
     }
 
-    private static DroppedColumn createDroppedColumnFromRow(UntypedResultSet.Row row)
+    private static DroppedColumn createDroppedColumnFromRow(UntypedResultSet.Row row, boolean isCounterTable)
     {
         String keyspace = row.getString("keyspace_name");
         String table = row.getString("table_name");
@@ -1177,14 +1212,15 @@ private static DroppedColumn createDroppedColumnFromRow(UntypedResultSet.Row row
          * them anymore), so before storing dropped columns in schema we expand UDTs to tuples. See expandUserTypes method.
          * Because of that, we can safely pass Types.none() to parse()
          */
-        AbstractType<?> type = CQLTypeParser.parse(keyspace, row.getString("type"), org.apache.cassandra.schema.Types.none());
+        AbstractType<?> type = CQLTypeParser.parseDroppedType(keyspace, row.getString("type"));
         ColumnMetadata.Kind kind = row.has("kind")
                                  ? ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase())
                                  : ColumnMetadata.Kind.REGULAR;
         assert kind == ColumnMetadata.Kind.REGULAR || kind == ColumnMetadata.Kind.STATIC
             : "Unexpected dropped column kind: " + kind;
 
-        ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind, null);
+        type.validateForColumn(UTF8Type.instance.decompose(name), false, isCounterTable, true, false);
+        ColumnMetadata column = ColumnMetadata.droppedColumn(keyspace, table, ColumnIdentifier.getInterned(name, true), type, kind, null);
         long droppedTime = TimeUnit.MILLISECONDS.toMicros(row.getLong("dropped_time"));
         return new DroppedColumn(column, droppedTime);
     }
@@ -1243,13 +1279,13 @@ private static ViewMetadata fetchView(String keyspaceName, String viewName, Type
         boolean includeAll = row.getBoolean("include_all_columns");
         String whereClauseString = row.getString("where_clause");
 
-        List<ColumnMetadata> columns = fetchColumns(keyspaceName, viewName, types, functions);
+        List<ColumnMetadata> columns = fetchColumns(keyspaceName, viewName, types, functions, false);
 
         TableMetadata metadata =
             TableMetadata.builder(keyspaceName, viewName, TableId.fromUUID(row.getUUID("id")))
                          .kind(TableMetadata.Kind.VIEW)
                          .addColumns(columns)
-                         .droppedColumns(fetchDroppedColumns(keyspaceName, viewName))
+                         .droppedColumns(fetchDroppedColumns(keyspaceName, viewName, false))
                          .params(createTableParamsFromRow(row))
                          .build();
 
@@ -1304,6 +1340,15 @@ private static UDFunction createUDFFromRow(UntypedResultSet.Row row, Types types
         String language = row.getString("language");
         String body = row.getString("body");
         boolean calledOnNullInput = row.getBoolean("called_on_null_input");
+        boolean deterministic = row.has("deterministic") && row.getBoolean("deterministic");
+        boolean monotonic = row.has("monotonic") && row.getBoolean("monotonic");
+
+        List<ColumnIdentifier> monotonicOn = row.has("monotonic_on")
+                                             ? row.getFrozenList("monotonic_on", UTF8Type.instance)
+                                                  .stream()
+                                                  .map(arg -> new ColumnIdentifier(arg, true))
+                                                  .collect(toList())
+                                             : Collections.emptyList();
 
         /*
          * TODO: find a way to get rid of Schema.instance dependency; evaluate if the opimisation below makes a difference
@@ -1332,12 +1377,12 @@ private static UDFunction createUDFFromRow(UntypedResultSet.Row row, Types types
 
         try
         {
-            return UDFunction.create(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+            return UDFunction.create(name, argNames, argTypes, returnType, calledOnNullInput, language, body, deterministic, monotonic, monotonicOn);
         }
         catch (InvalidRequestException e)
         {
             logger.error(String.format("Cannot load function '%s' from schema: this function won't be available (on this node)", name), e);
-            return UDFunction.createBrokenFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body, e);
+            return UDFunction.createBrokenFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body, deterministic, monotonic, monotonicOn, e);
         }
     }
 
@@ -1369,8 +1414,9 @@ private static UDAggregate createUDAFromRow(UntypedResultSet.Row row, Collection
         FunctionName finalFunc = row.has("final_func") ? new FunctionName(ksName, row.getString("final_func")) : null;
         AbstractType<?> stateType = row.has("state_type") ? CQLTypeParser.parse(ksName, row.getString("state_type"), types) : null;
         ByteBuffer initcond = row.has("initcond") ? Terms.asBytes(ksName, row.getString("initcond"), stateType) : null;
+        boolean deterministic = row.has("deterministic") && row.getBoolean("deterministic");
 
-        return UDAggregate.create(functions, name, argTypes, returnType, stateFunc, finalFunc, stateType, initcond);
+        return UDAggregate.create(functions, name, argTypes, returnType, stateFunc, finalFunc, stateType, initcond, deterministic);
     }
 
     private static UntypedResultSet query(String query, Object... variables)
@@ -1396,8 +1442,7 @@ static Set<String> affectedKeyspaces(Collection<Mutation> mutations)
     static void applyChanges(Collection<Mutation> mutations)
     {
         mutations.forEach(Mutation::apply);
-        if (SchemaKeyspace.FLUSH_SCHEMA_TABLES)
-            SchemaKeyspace.flush();
+        SchemaKeyspace.flush();
     }
 
     static Keyspaces fetchKeyspaces(Set<String> toFetch)
diff --git a/src/java/org/apache/cassandra/schema/SchemaProvider.java b/src/java/org/apache/cassandra/schema/SchemaProvider.java
index cbad42e530b7..05e0d09ee8a1 100644
--- a/src/java/org/apache/cassandra/schema/SchemaProvider.java
+++ b/src/java/org/apache/cassandra/schema/SchemaProvider.java
@@ -19,9 +19,11 @@
 package org.apache.cassandra.schema;
 
 import java.util.function.Supplier;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.exceptions.UnknownKeyspaceException;
 import org.apache.cassandra.exceptions.UnknownTableException;
 import org.apache.cassandra.io.sstable.Descriptor;
 
@@ -30,7 +32,7 @@ public interface SchemaProvider
     @Nullable
     Keyspace getKeyspaceInstance(String keyspaceName);
 
-    Keyspace maybeAddKeyspaceInstance(String keyspaceName, Supplier<Keyspace> loadFunction);
+    Keyspace maybeAddKeyspaceInstance(String keyspaceName, Supplier<Keyspace> loadFunction) throws UnknownKeyspaceException;
 
     @Nullable
     KeyspaceMetadata getKeyspaceMetadata(String keyspaceName);
@@ -41,6 +43,7 @@ public interface SchemaProvider
     @Nullable
     TableMetadata getTableMetadata(String keyspace, String table);
 
+    @Nonnull
     default TableMetadata getExistingTableMetadata(TableId id) throws UnknownTableException
     {
         TableMetadata metadata = getTableMetadata(id);
diff --git a/src/java/org/apache/cassandra/schema/SchemaTransformations.java b/src/java/org/apache/cassandra/schema/SchemaTransformations.java
index 124f9a6ef645..ad2d1838935f 100644
--- a/src/java/org/apache/cassandra/schema/SchemaTransformations.java
+++ b/src/java/org/apache/cassandra/schema/SchemaTransformations.java
@@ -113,7 +113,33 @@ public static SchemaTransformation addTypes(Types toAdd, boolean ignoreIfExists)
 
                 types = types.with(type);
             }
-            return schema.withAddedOrReplaced(keyspace.withSwapped(types));
+            return schema.withAddedOrUpdated(keyspace.withSwapped(types));
+        };
+    }
+
+    /**
+     * Creates a schema transformation that either add the provided type, or "update" (replace really) it to be the
+     * provided type.
+     *
+     * <p>Please note that this usually <b>unsafe</b>: if the type exists, this replace it without any particular check
+     * and so could replace it with an incompatible version. This is used internally however for hard-coded tables
+     * (System ones, including DSE ones) to force the "last version".
+     *
+     * @param type the type to add/update.
+     * @return the created transformation.
+     */
+    public static SchemaTransformation addOrUpdateType(UserType type)
+    {
+        return schema ->
+        {
+            KeyspaceMetadata keyspace = schema.getNullable(type.keyspace);
+            if (null == keyspace)
+                throw invalidRequest("Keyspace '%s' doesn't exist", type.keyspace);
+
+            Types newTypes = keyspace.types.get(type.name).isPresent()
+                             ? keyspace.types.withUpdatedUserType(type)
+                             : keyspace.types.with(type);
+            return schema.withAddedOrUpdated(keyspace.withSwapped(newTypes));
         };
     }
 
@@ -202,7 +228,7 @@ public Keyspaces apply(Keyspaces schema)
                         }
                     }
                 }
-                return schema.withAddedOrReplaced(updatedKeyspace);
+                return schema.withAddedOrUpdated(updatedKeyspace);
             }
         };
     }
diff --git a/src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactory.java b/src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactory.java
index f324a5d6e682..39545a0254d2 100644
--- a/src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactory.java
+++ b/src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactory.java
@@ -29,7 +29,8 @@ public interface SchemaUpdateHandlerFactory
      * different run modes (client, tool, daemon).
      *
      * @param online               whether schema update handler should work online and be aware of the other nodes (when in daemon mode)
-     * @param updateSchemaCallback callback which will be called right after the shared schema is updated
+     * @param updateSchemaCallback callback which will be called right after the shared schema is updated, the args represent
+     *                             the schema transformation result and a flag whether the data should be actually removed for the dropped tables
      */
     SchemaUpdateHandler getSchemaUpdateHandler(boolean online, BiConsumer<SchemaTransformationResult, Boolean> updateSchemaCallback);
 }
diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
index b36ec64874f3..cb9128d147a9 100644
--- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
@@ -58,6 +58,7 @@
 
 import static java.lang.String.format;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.UNSAFE_SYSTEM;
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
 
 public final class SystemDistributedKeyspace
@@ -170,7 +171,9 @@ private static TableMetadata.Builder parse(String table, String description, Str
 
     public static KeyspaceMetadata metadata()
     {
-        return KeyspaceMetadata.create(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, KeyspaceParams.simple(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())), Tables.of(RepairHistory, ParentRepairHistory, ViewBuildStatus, PartitionDenylistTable));
+        return KeyspaceMetadata.create(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, 
+                                       KeyspaceParams.systemDistributed(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())), 
+                                       Tables.of(RepairHistory, ParentRepairHistory, ViewBuildStatus, PartitionDenylistTable));
     }
 
     public static void startParentRepair(TimeUUID parent_id, String keyspaceName, String[] cfnames, RepairOption options)
@@ -394,7 +397,7 @@ private static void processSilent(String fmtQry, String... values)
 
     public static void forceBlockingFlush(String table, ColumnFamilyStore.FlushReason reason)
     {
-        if (!DatabaseDescriptor.isUnsafeSystem())
+        if (!UNSAFE_SYSTEM.getBoolean())
             FBUtilities.waitOnFuture(Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME)
                                              .getColumnFamilyStore(table)
                                              .forceFlush(reason));
diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java
index 93cdc3548225..f393cea4495a 100644
--- a/src/java/org/apache/cassandra/schema/TableMetadata.java
+++ b/src/java/org/apache/cassandra/schema/TableMetadata.java
@@ -26,10 +26,10 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
 import com.google.common.base.MoreObjects;
@@ -61,6 +61,7 @@
 import org.apache.cassandra.db.marshal.EmptyType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -70,10 +71,11 @@
 
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.transform;
+import static com.google.common.collect.Maps.transformValues;
 import static java.lang.String.format;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toSet;
-import static org.apache.cassandra.schema.IndexMetadata.isNameValid;
+import static org.apache.cassandra.schema.SchemaConstants.isValidName;
 
 @Unmetered
 public class TableMetadata implements SchemaElement
@@ -204,6 +206,12 @@ protected TableMetadata(Builder builder)
         regularAndStaticColumns = RegularAndStaticColumns.builder().addAll(builder.regularAndStaticColumns).build();
         columns = ImmutableMap.copyOf(builder.columns);
 
+        assert columns.values().stream().noneMatch(ColumnMetadata::isDropped) :
+                "Invalid columns (contains dropped): " + columns.values()
+                                                                .stream()
+                                                                .map(ColumnMetadata::debugString)
+                                                                .collect(Collectors.joining(", "));
+
         indexes = builder.indexes;
         triggers = builder.triggers;
 
@@ -288,7 +296,7 @@ public boolean isCompactTable()
     {
         return false;
     }
-    
+
     public boolean isIncrementalBackupsEnabled()
     {
         return params.incrementalBackups;
@@ -444,6 +452,16 @@ public boolean hasStaticColumns()
         return !staticColumns().isEmpty();
     }
 
+    public boolean hasVectorType()
+    {
+        for (ColumnMetadata column : columns.values())
+        {
+            if (column.type.isVector())
+                return true;
+        }
+        return false;
+    }
+
     /**
      * @return {@code true} if the table has any masked column, {@code false} otherwise.
      */
@@ -475,30 +493,15 @@ public boolean dependsOn(Function function)
 
     public void validate()
     {
-        if (!isNameValid(keyspace))
-            except("Keyspace name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got \"%s\")", SchemaConstants.NAME_LENGTH, keyspace);
+        if (!isValidName(keyspace, true))
+            except("Keyspace name must not be empty or contain non-alphanumeric-underscore characters (got \"%s\")", keyspace);
 
-        if (!isNameValid(name))
-            except("Table name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got \"%s\")", SchemaConstants.NAME_LENGTH, name);
+        if (!isValidName(name, true))
+            except("Table name must not be empty or contain non-alphanumeric-underscore characters (got \"%s\")", name);
 
         params.validate();
 
-        if (partitionKeyColumns.stream().anyMatch(c -> c.type.isCounter()))
-            except("PRIMARY KEY columns cannot contain counters");
-
-        // Mixing counter with non counter columns is not supported (#2614)
-        if (isCounter())
-        {
-            for (ColumnMetadata column : regularAndStaticColumns)
-                if (!(column.type.isCounter()) && !isSuperColumnMapColumnName(column.name))
-                    except("Cannot have a non counter column (\"%s\") in a counter table", column.name);
-        }
-        else
-        {
-            for (ColumnMetadata column : regularAndStaticColumns)
-                if (column.type.isCounter())
-                    except("Cannot have a counter column (\"%s\") in a non counter table", column.name);
-        }
+        columns().forEach(c -> c.validate(isCounter()));
 
         // All tables should have a partition key
         if (partitionKeyColumns.isEmpty())
@@ -523,9 +526,42 @@ public void validate()
      *   table with counters to rename that weirdly name map to something more meaningful (it's not possible today
      *   as after renaming the validation in {@link #validate} would trigger).
      */
-    private static boolean isSuperColumnMapColumnName(ColumnIdentifier columnName)
+    public static boolean isSuperColumnMapColumnName(ByteBuffer columnName)
+    {
+        return !columnName.hasRemaining();
+    }
+
+    /**
+     * Method that compares two TableMetadata objects. This is a modified version of {@link #validateCompatibility} that is used
+     * when checking the compatibility between the schema metadata from an SSTable against the schema metadata from a
+     * CQL table.
+     * <p>
+     * The serialization header of the SSTable does not contain exactly the same information available in the schema of
+     * a CQL table, so the comparison needs to be adapted to the information available.
+     * <p>
+     * For example, the serialization header does not contain the partition key columns: it only contains the composite
+     * type of the whole partition key. For this reason, the comparison must be between the partition key types contained
+     * in the two metadata objects, rather than comparing the individual column as {@link #validateCompatibility} does.
+     * This comparison is sufficient anyway because the composite types are the same only if their components are of the
+     * same type and in the same order.
+     * <p>
+     * Another difference worth pointing out is that this method compares the table name, but not the keyspace name or table id.
+     * This is to allow the comparison between externally generated SSTables and a CQL schema, in which case the keyspace name
+     * and table id may be different.
+     *
+     * @param other TableMetadata instance to compare against
+     */
+    public void validateTableNameAndStructureCompatibility(TableMetadata other)
     {
-        return !columnName.bytes.hasRemaining();
+        if (isIndex())
+            return;
+
+        validateTableName(other);
+        validateTableType(other);
+        // comparing the types of the partition keys rather than the individual columns, as explained in the comment above
+        validatePartitionKeyTypes(other);
+        validateClusteringColumns(other);
+        validateRegularAndStaticColumns(other);
     }
 
     public void validateCompatibility(TableMetadata previous)
@@ -533,18 +569,51 @@ public void validateCompatibility(TableMetadata previous)
         if (isIndex())
             return;
 
+        validateKeyspaceName(previous);
+        validateTableName(previous);
+        validateTableId(previous);
+        validateTableType(previous);
+        validatePartitionKeyColumns(previous);
+        validateClusteringColumns(previous);
+        validateRegularAndStaticColumns(previous);
+    }
+
+    private void validateKeyspaceName(TableMetadata previous)
+    {
         if (!previous.keyspace.equals(keyspace))
             except("Keyspace mismatch (found %s; expected %s)", keyspace, previous.keyspace);
+    }
 
+    private void validateTableName(TableMetadata previous)
+    {
         if (!previous.name.equals(name))
             except("Table mismatch (found %s; expected %s)", name, previous.name);
+    }
 
+    private void validateTableId(TableMetadata previous)
+    {
         if (!previous.id.equals(id))
             except("Table ID mismatch (found %s; expected %s)", id, previous.id);
+    }
 
+    private void validateTableType(TableMetadata previous)
+    {
         if (!previous.flags.equals(flags) && (!Flag.isCQLTable(flags) || Flag.isCQLTable(previous.flags)))
             except("Table type mismatch (found %s; expected %s)", flags, previous.flags);
+    }
+
+    private void validatePartitionKeyTypes(TableMetadata previous)
+    {
+        if (!partitionKeyType.isCompatibleWith(previous.partitionKeyType))
+        {
+            except("Partition keys of different types (found %s; expected %s)",
+                   partitionKeyType,
+                   previous.partitionKeyType);
+        }
+    }
 
+    private void validatePartitionKeyColumns(TableMetadata previous)
+    {
         if (previous.partitionKeyColumns.size() != partitionKeyColumns.size())
         {
             except("Partition keys of different length (found %s; expected %s)",
@@ -561,7 +630,10 @@ public void validateCompatibility(TableMetadata previous)
                        previous.partitionKeyColumns.get(i).type);
             }
         }
+    }
 
+    private void validateClusteringColumns(TableMetadata previous)
+    {
         if (previous.clusteringColumns.size() != clusteringColumns.size())
         {
             except("Clustering columns of different length (found %s; expected %s)",
@@ -578,7 +650,10 @@ public void validateCompatibility(TableMetadata previous)
                        previous.clusteringColumns.get(i).type);
             }
         }
+    }
 
+    private void validateRegularAndStaticColumns(TableMetadata previous)
+    {
         for (ColumnMetadata previousColumn : previous.regularAndStaticColumns)
         {
             ColumnMetadata column = getColumn(previousColumn.name);
@@ -618,6 +693,7 @@ boolean changeAffectsPreparedStatements(TableMetadata updated)
             || !regularAndStaticColumns.equals(updated.regularAndStaticColumns)
             || !indexes.equals(updated.indexes)
             || params.defaultTimeToLive != updated.params.defaultTimeToLive
+            || params.cdc != updated.params.cdc
             || params.gcGraceSeconds != updated.params.gcGraceSeconds
             || ( !Flag.isCQLTable(flags) && Flag.isCQLTable(updated.flags) );
     }
@@ -650,6 +726,49 @@ boolean referencesUserType(ByteBuffer name)
         return any(columns(), c -> c.type.referencesUserType(name));
     }
 
+    /**
+     * Create a copy of this {@code TableMetadata} for a new keyspace.
+     * Note that a new table id will be generated for the returned {@link TableMetadata}.
+     *
+     * @param newKeyspace the name of the new keyspace
+     * @param udts the user defined types of the new keyspace
+     * @return a copy of this {@code TableMetadata} for a new keyspace
+     */
+    TableMetadata withNewKeyspace(String newKeyspace,
+                                  Types udts)
+    {
+        return builder(newKeyspace, name).partitioner(partitioner)
+                                         .kind(kind)
+                                         .params(params)
+                                         .flags(flags)
+                                         .addColumns(transform(columns(), c -> c.withNewKeyspace(newKeyspace, udts)))
+                                         .droppedColumns(transformValues(droppedColumns, c -> c.withNewKeyspace(newKeyspace, udts)))
+                                         .indexes(indexes)
+                                         .triggers(triggers)
+                                         .build();
+    }
+
+    /**
+     * Create a copy of this {@code TableMetadata} with new params computed by applying the <code>transformFunction</code>.
+     * Note that the table id will be maintained.
+     *
+     * @param transformFunction The function used to transform the params.
+     * @return a copy of this {@code TableMetadata} containing the transformed params.
+     */
+    TableMetadata withTransformedParams(java.util.function.Function<TableParams, TableParams> transformFunction)
+    {
+        return builder(keyspace, name, id)
+                .partitioner(partitioner)
+                .kind(kind)
+                .params(transformFunction.apply(params))
+                .flags(flags)
+                .addColumns(columns())
+                .droppedColumns(droppedColumns)
+                .indexes(indexes)
+                .triggers(triggers)
+                .build();
+    }
+
     public TableMetadata withUpdatedUserType(UserType udt)
     {
         if (!referencesUserType(udt.name))
@@ -663,7 +782,12 @@ public TableMetadata withUpdatedUserType(UserType udt)
 
     protected void except(String format, Object... args)
     {
-        throw new ConfigurationException(keyspace + "." + name + ": " + format(format, args));
+        throw new ConfigurationException(keyspace + '.' + name + ": " + format(format, args));
+    }
+
+    public PartitionUpdate.Factory partitionUpdateFactory()
+    {
+        return params.memtable.factory.partitionUpdateFactory();
     }
 
     @Override
@@ -1044,8 +1168,7 @@ public Builder addStaticColumn(ColumnIdentifier name, AbstractType<?> type, @Nul
 
         public Builder addColumn(ColumnMetadata column)
         {
-            if (columns.containsKey(column.name.bytes))
-                throw new IllegalArgumentException();
+            assert !columns.containsKey(column.name.bytes) : column.name + " is already present";
 
             switch (column.kind)
             {
@@ -1093,7 +1216,19 @@ public Builder recordDeprecatedSystemColumn(String name, AbstractType<?> type)
 
         public Builder recordColumnDrop(ColumnMetadata column, long timeMicros)
         {
-            droppedColumns.put(column.name.bytes, new DroppedColumn(column.withNewType(column.type.expandUserTypes()), timeMicros));
+            return recordColumnDrop(new DroppedColumn(column.asDropped(), timeMicros));
+        }
+
+        public Builder recordColumnDrop(DroppedColumn dropped)
+        {
+            DroppedColumn previous = droppedColumns.get(dropped.column.name.bytes);
+            if (previous != null && previous.droppedTime > dropped.droppedTime)
+                throw new ConfigurationException(String.format("Invalid dropped column record for column %s in %s at "
+                                                               + "%d: pre-existing record at %d is newer",
+                                                               dropped.column.name, this.name, previous.droppedTime,
+                                                               dropped.droppedTime));
+
+            droppedColumns.put(dropped.column.name.bytes, dropped);
             return this;
         }
 
@@ -1331,7 +1466,7 @@ public void appendCqlTo(CqlBuilder builder,
         builder.append(" WITH ")
                .increaseIndent();
 
-        appendTableOptions(builder, withInternals);
+        appendTableOptions(builder, withInternals, includeDroppedColumns);
 
         builder.decreaseIndent();
 
@@ -1340,9 +1475,6 @@ public void appendCqlTo(CqlBuilder builder,
             builder.newLine()
                    .append("*/");
         }
-
-        if (includeDroppedColumns)
-            appendDropColumns(builder);
     }
 
     private void appendColumnDefinitions(CqlBuilder builder,
@@ -1353,37 +1485,16 @@ private void appendColumnDefinitions(CqlBuilder builder,
         while (iter.hasNext())
         {
             ColumnMetadata column = iter.next();
-            // If the column has been re-added after a drop, we don't include it right away. Instead, we'll add the
-            // dropped one first below, then we'll issue the DROP and then the actual ADD for this column, thus
-            // simulating the proper sequence of events.
-            if (includeDroppedColumns && droppedColumns.containsKey(column.name.bytes))
-                continue;
-
             column.appendCqlTo(builder);
 
             if (hasSingleColumnPrimaryKey && column.isPartitionKey())
                 builder.append(" PRIMARY KEY");
 
-            if (!hasSingleColumnPrimaryKey || (includeDroppedColumns && !droppedColumns.isEmpty()) || iter.hasNext())
+            if (!hasSingleColumnPrimaryKey || iter.hasNext())
                 builder.append(',');
 
             builder.newLine();
         }
-
-        if (includeDroppedColumns)
-        {
-            Iterator<DroppedColumn> iterDropped = droppedColumns.values().iterator();
-            while (iterDropped.hasNext())
-            {
-                DroppedColumn dropped = iterDropped.next();
-                dropped.column.appendCqlTo(builder);
-
-                if (!hasSingleColumnPrimaryKey || iterDropped.hasNext())
-                    builder.append(',');
-
-                builder.newLine();
-            }
-        }
     }
 
     void appendPrimaryKey(CqlBuilder builder)
@@ -1414,7 +1525,7 @@ void appendPrimaryKey(CqlBuilder builder)
                .newLine();
     }
 
-    void appendTableOptions(CqlBuilder builder, boolean withInternals)
+    void appendTableOptions(CqlBuilder builder, boolean withInternals, boolean includeDroppedColumns)
     {
         if (withInternals)
             builder.append("ID = ")
@@ -1438,6 +1549,8 @@ void appendTableOptions(CqlBuilder builder, boolean withInternals)
         }
         else
         {
+            if (includeDroppedColumns)
+                appendDropColumns(builder);
             params.appendCqlTo(builder, isView());
         }
         builder.append(";");
@@ -1445,31 +1558,11 @@ void appendTableOptions(CqlBuilder builder, boolean withInternals)
 
     private void appendDropColumns(CqlBuilder builder)
     {
-        for (Entry<ByteBuffer, DroppedColumn> entry : droppedColumns.entrySet())
+        for (DroppedColumn dropped : droppedColumns.values())
         {
-            DroppedColumn dropped = entry.getValue();
-
-            builder.newLine()
-                   .append("ALTER TABLE ")
-                   .append(toString())
-                   .append(" DROP ")
-                   .append(dropped.column.name)
-                   .append(" USING TIMESTAMP ")
-                   .append(dropped.droppedTime)
-                   .append(';');
-
-            ColumnMetadata column = getColumn(entry.getKey());
-            if (column != null)
-            {
-                builder.newLine()
-                       .append("ALTER TABLE ")
-                       .append(toString())
-                       .append(" ADD ");
-
-                column.appendCqlTo(builder);
-
-                builder.append(';');
-            }
+            builder.append(dropped.toCQLString())
+                   .newLine()
+                   .append("AND ");
         }
     }
 
@@ -1505,7 +1598,7 @@ public String primaryKeyAsCQLLiteral(ByteBuffer partitionKey, Clustering<?> clus
 
         if (partitionKeyType instanceof CompositeType)
         {
-            List<AbstractType<?>> components = partitionKeyType.getComponents();
+            List<AbstractType<?>> components = partitionKeyType.subTypes();
             int size = components.size();
             literals = new String[size + clusteringSize];
             ByteBuffer[] values = ((CompositeType) partitionKeyType).split(partitionKey);
@@ -1702,13 +1795,13 @@ public void appendCqlTo(CqlBuilder builder,
                    .append("*/");
         }
 
-        void appendTableOptions(CqlBuilder builder, boolean internals)
+        void appendTableOptions(CqlBuilder builder, boolean internals, boolean includeDroppedColumns)
         {
             builder.append("COMPACT STORAGE")
                    .newLine()
                    .append("AND ");
 
-            super.appendTableOptions(builder, internals);
+            super.appendTableOptions(builder, internals, includeDroppedColumns);
         }
 
         public static ColumnMetadata getCompactValueColumn(RegularAndStaticColumns columns)
diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java
index 8f883f8f4783..e79f215b3643 100644
--- a/src/java/org/apache/cassandra/schema/TableParams.java
+++ b/src/java/org/apache/cassandra/schema/TableParams.java
@@ -149,7 +149,7 @@ public Builder unbuild()
 
     public void validate()
     {
-        compaction.validate();
+        // compaction parameters are validated during CompactionParams construction
         compression.validate();
 
         double minBloomFilterFpChanceValue = BloomCalculations.minSupportedBloomFilterFpChance();
diff --git a/src/java/org/apache/cassandra/schema/Tables.java b/src/java/org/apache/cassandra/schema/Tables.java
index 0f8f6b31908a..c2236f478ef7 100644
--- a/src/java/org/apache/cassandra/schema/Tables.java
+++ b/src/java/org/apache/cassandra/schema/Tables.java
@@ -22,13 +22,18 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
-
 import javax.annotation.Nullable;
 
-import com.google.common.collect.*;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
 
 import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.index.internal.CassandraIndex;
@@ -179,6 +184,31 @@ public Tables withUpdatedUserType(UserType udt)
              : this;
     }
 
+    public Tables withNewKeyspace(String newName, Types udts)
+    {
+        Map<String, TableMetadata> updated = new HashMap<>();
+        for (TableMetadata table : this)
+        {
+            updated.put(table.name, table.withNewKeyspace(newName, udts));
+        }
+
+        return builder().add(updated.values()).build();
+    }
+
+    public Tables withTransformedParams(Function<TableParams, TableParams> transformFunction)
+    {
+        Map<String, TableMetadata> updated = new HashMap<>();
+
+        // We order the tables by dependencies so that vertices tables are
+        // processed before edges tables, in case graph constructs are used
+        for (TableMetadata table : this)
+        {
+            updated.put(table.name, table.withTransformedParams(transformFunction));
+        }
+
+        return builder().add(updated.values()).build();
+    }
+
     MapDifference<String, TableMetadata> indexesDiff(Tables other)
     {
         Map<String, TableMetadata> thisIndexTables = new HashMap<>();
diff --git a/src/java/org/apache/cassandra/schema/Types.java b/src/java/org/apache/cassandra/schema/Types.java
index 0d264c4f492c..bf9430cbe50d 100644
--- a/src/java/org/apache/cassandra/schema/Types.java
+++ b/src/java/org/apache/cassandra/schema/Types.java
@@ -36,6 +36,7 @@
 import static java.lang.String.format;
 import static java.util.stream.Collectors.toList;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.transform;
 
@@ -109,6 +110,32 @@ public Iterable<UserType> referencingUserType(ByteBuffer name)
         return Iterables.filter(types.values(), t -> t.referencesUserType(name) && !t.name.equals(name));
     }
 
+    /**
+     * Returns the types ordered by dependencies.
+     *
+     * @return the types ordered by dependencies.
+     */
+    private Set<UserType> getTypesOrderedByDependencies()
+    {
+        Set<UserType> orderedTypesByDependencies = new LinkedHashSet<>();
+        for (UserType type : this)
+        {
+            recordNestedTypes(type, orderedTypesByDependencies);
+        }
+        return orderedTypesByDependencies;
+    }
+
+    private void recordNestedTypes(AbstractType<?> userType, Set<UserType> userTypes)
+    {
+        for (AbstractType<?> subType : userType.subTypes())
+        {
+            recordNestedTypes(subType, userTypes);
+        }
+
+        if (userType.isUDT())
+            userTypes.add((UserType) userType);
+    }
+
     public boolean isEmpty()
     {
         return types.isEmpty();
@@ -254,6 +281,33 @@ private static void addUserTypes(AbstractType<?> type, Set<ByteBuffer> types)
             types.add(((UserType) type).name);
     }
 
+    /**
+     * Changes the keyspace of all the types.
+     *
+     * @param newKeyspace the name of the new keyspace
+     * @return the new types
+     */
+    public Types withNewKeyspace(String newKeyspace)
+    {
+        Map<ByteBuffer, UserType> updatedTypes = new HashMap<>();
+
+        for (UserType originalType : getTypesOrderedByDependencies())
+        {
+            UserType type = new UserType(newKeyspace,
+                                         originalType.name,
+                                         originalType.fieldNames(),
+                                         originalType.fieldTypes()
+                                                     .stream()
+                                                     .map(t -> t.withUpdatedUserTypes(updatedTypes.values()))
+                                                     .collect(ImmutableList.toImmutableList()),
+                                         true);
+
+            updatedTypes.put(type.name, type);
+        }
+
+        return new Types(ImmutableSortedMap.copyOf(updatedTypes));
+    }
+
     public static final class Builder
     {
         final ImmutableSortedMap.Builder<ByteBuffer, UserType> types = ImmutableSortedMap.naturalOrder();
@@ -384,15 +438,15 @@ boolean referencesUserType(RawUDT other)
 
             UserType prepare(String keyspace, Types types)
             {
-                List<FieldIdentifier> preparedFieldNames =
-                    fieldNames.stream()
-                              .map(FieldIdentifier::forInternalString)
-                              .collect(toList());
-
-                List<AbstractType<?>> preparedFieldTypes =
-                    fieldTypes.stream()
-                              .map(t -> t.prepareInternal(keyspace, types).getType())
-                              .collect(toList());
+                ImmutableList<FieldIdentifier> preparedFieldNames =
+                fieldNames.stream()
+                          .map(FieldIdentifier::forInternalString)
+                          .collect(toImmutableList());
+
+                ImmutableList<AbstractType<?>> preparedFieldTypes =
+                fieldTypes.stream()
+                          .map(t -> t.prepare(keyspace, types).getType())
+                          .collect(toImmutableList());
 
                 return new UserType(keyspace, bytes(name), preparedFieldNames, preparedFieldTypes, true);
             }
diff --git a/src/java/org/apache/cassandra/schema/UserFunctions.java b/src/java/org/apache/cassandra/schema/UserFunctions.java
index b40c704d0b98..101687a59ff7 100644
--- a/src/java/org/apache/cassandra/schema/UserFunctions.java
+++ b/src/java/org/apache/cassandra/schema/UserFunctions.java
@@ -18,20 +18,30 @@
 package org.apache.cassandra.schema;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.google.common.collect.*;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Iterables;
 
-import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.cql3.functions.UserFunction;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UserType;
 
-import static java.util.stream.Collectors.toList;
-
 import static com.google.common.collect.Iterables.any;
+import static java.util.stream.Collectors.toList;
 
 /**
  * An immutable container for a keyspace's UDAs and UDFs.
@@ -117,6 +127,14 @@ public UserFunctions withUpdatedUserType(UserType udt)
         return builder().add(udfs).add(udas).build();
     }
 
+    public UserFunctions withNewKeyspace(String newKeyspace, Types udts)
+    {
+        Collection<UDFunction> udfs = udfs().map(f -> f.withNewKeyspace(newKeyspace, udts)).collect(toList());
+        Collection<UDAggregate> udas = udas().map(f -> f.withNewKeyspace(newKeyspace, udfs, udts)).collect(toList());
+
+        return builder().add(udfs).add(udas).build();
+    }
+
     /**
      * @return a stream of aggregates that use the provided function as either a state or a final function
      * @param function the referree function
diff --git a/src/java/org/apache/cassandra/schema/ViewMetadata.java b/src/java/org/apache/cassandra/schema/ViewMetadata.java
index df26a134e24e..71635c443c3f 100644
--- a/src/java/org/apache/cassandra/schema/ViewMetadata.java
+++ b/src/java/org/apache/cassandra/schema/ViewMetadata.java
@@ -210,7 +210,7 @@ public void appendCqlTo(CqlBuilder builder,
                .append(" WITH ")
                .increaseIndent();
 
-        metadata.appendTableOptions(builder, internals);
+        metadata.appendTableOptions(builder, internals, false);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/sensors/ActiveRequestSensors.java b/src/java/org/apache/cassandra/sensors/ActiveRequestSensors.java
new file mode 100644
index 000000000000..9c4c418eaddc
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/ActiveRequestSensors.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Groups {@link Sensor}s associated to a given request/response and related {@link Context}: this is the main entry
+ * point to create and modify sensors. More specifically:
+ * <ul>
+ *     <li>Create a new sensor associated to the request/response via {@link #registerSensor(Context, Type)}.</li>
+ *     <li>Increment the sensor value for the request/response via {@link #incrementSensor(Context, Type, double)}.</li>
+ *     <li>Sync this request/response sensor value to the {@link SensorsRegistry} via {@link #syncAllSensors()}.</li>
+ * </ul>
+ * Sensor values related to a given request/response are isolated from other sensors, and the "same" sensor
+ * (for a given context and type) registered to different requests/responses will have a different value: in other words,
+ * there is no automatic synchronization or coordination across sensor values belonging to different
+ * {@link RequestSensors} objects, hence {@link #syncAllSensors()} MUST be invoked to propagate the sensors values
+ * at a global level to the {@link SensorsRegistry}.
+ * <br/>
+ * Please note instances of this class should be created via the configured {@link SensorsFactory}.
+ */
+public class ActiveRequestSensors implements RequestSensors
+{
+    private final Supplier<SensorsRegistry> sensorsRegistry;
+
+    // Using Map of array values for performance reasons to avoid wrapping key into another Object (.eg. Pair(context,type)).
+    // Note that array values can contain NULL so be careful to filter NULLs when iterating over array
+    private final HashMap<Context, Sensor[]> sensors = new LinkedHashMap<>();
+
+    private final Map<Sensor, Double> latestSyncedValuePerSensor = new HashMap<>();
+
+    @VisibleForTesting
+    public ActiveRequestSensors()
+    {
+        this(() -> SensorsRegistry.instance);
+    }
+
+    @VisibleForTesting
+    public ActiveRequestSensors(Supplier<SensorsRegistry> sensorsRegistry)
+    {
+        this.sensorsRegistry = sensorsRegistry;
+    }
+
+    public synchronized void registerSensor(Context context, Type type)
+    {
+        Sensor[] typeSensors = sensors.computeIfAbsent(context, key ->
+        {
+            Sensor[] newTypeSensors = new Sensor[Type.values().length];
+            newTypeSensors[type.ordinal()] = new Sensor(context, type);
+            return newTypeSensors;
+        });
+        if (typeSensors[type.ordinal()] == null)
+            typeSensors[type.ordinal()] = new Sensor(context, type);
+    }
+
+    public synchronized Optional<Sensor> getSensor(Context context, Type type)
+    {
+        return Optional.ofNullable(getSensorFast(context, type));
+    }
+
+    public synchronized Set<Sensor> getSensors(Predicate<Sensor> filter)
+    {
+        return sensors.values().stream().flatMap(Arrays::stream).filter(Objects::nonNull).filter(filter).collect(Collectors.toSet());
+    }
+
+    public synchronized void incrementSensor(Context context, Type type, double value)
+    {
+        Sensor sensor = getSensorFast(context, type);
+        if (sensor != null)
+            sensor.increment(value);
+    }
+
+    public synchronized void syncAllSensors()
+    {
+        sensors.values().forEach(types -> {
+            for (int i = 0; i < types.length; i++)
+            {
+                if (types[i] != null)
+                {
+                    Sensor sensor = types[i];
+                    double current = latestSyncedValuePerSensor.getOrDefault(sensor, 0d);
+                    double update = sensor.getValue() - current;
+                    if (update == 0d)
+                        continue;
+
+                    latestSyncedValuePerSensor.put(sensor, sensor.getValue());
+                    sensorsRegistry.get().incrementSensor(sensor.getContext(), sensor.getType(), update);
+                }
+            }
+        });
+    }
+
+    /**
+     * To get best perfromance we are not returning Optional here
+     */
+    @Nullable
+    private Sensor getSensorFast(Context context, Type type)
+    {
+        Sensor[] typeSensors = sensors.get(context);
+        if (typeSensors != null)
+            return typeSensors[type.ordinal()];
+
+        return null;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ActiveRequestSensors other = (ActiveRequestSensors) o;
+        return Objects.equals(sensors, other.sensors);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(sensors);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "ActiveRequestSensors{" +
+               "sensors=" + sensors +
+               '}';
+    }
+}
diff --git a/src/java/org/apache/cassandra/sensors/ActiveSensorsFactory.java b/src/java/org/apache/cassandra/sensors/ActiveSensorsFactory.java
new file mode 100644
index 000000000000..7ae1ea058b7e
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/ActiveSensorsFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Optional;
+
+/**
+ * Implementation of the {@link SensorsFactory} that creates:
+ * <ul>
+ *   <li> a new {@link ActiveRequestSensors} instance for all keyspaces.</li>
+ *   <li> a singleton {@link SensorEncoder} implementation that encodes the sensor name as {@literal "<SENSOR_TYPE>_REQUEST.<TABLE_NAME>"} for request sensors and {@literal "<SENSOR_TYPE>_GLOBAL.<TABLE_NAME>"} for global sensors.</li>
+ * </ul>
+ */
+public class ActiveSensorsFactory implements SensorsFactory
+{
+    private static final SensorEncoder SENSOR_ENCODER = new SensorEncoder()
+    {
+        @Override
+        public Optional<String> encodeRequestSensorName(Sensor sensor)
+        {
+            return Optional.of(sensor.getType() + "_REQUEST." + sensor.getContext().getKeyspace() + '.' + sensor.getContext().getTable());
+        }
+
+        @Override
+        public Optional<String> encodeGlobalSensorName(Sensor sensor)
+        {
+            return Optional.of(sensor.getType() + "_GLOBAL." + sensor.getContext().getKeyspace() + '.' + sensor.getContext().getTable());
+        }
+    };
+
+    @Override
+    public RequestSensors createRequestSensors(String... keyspaces)
+    {
+        return new ActiveRequestSensors();
+    }
+
+    @Override
+    public SensorEncoder createSensorEncoder()
+    {
+        return SENSOR_ENCODER;
+    }
+}
diff --git a/src/java/org/apache/cassandra/sensors/Context.java b/src/java/org/apache/cassandra/sensors/Context.java
new file mode 100644
index 000000000000..a82e534b1461
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/Context.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Objects;
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Represents the context for a (group of) {@link Sensor}(s), made up of:
+ * <ul>
+ *     <li>The keyspace the sensor refers to.</li>
+ *     <li>The table the sensor refers to.</li>
+ *     <li>The related table id.</li>
+ * </ul>
+ */
+public class Context
+{
+    private final String keyspace;
+    private final String table;
+    private final String tableId;
+
+    private final int hashCode;
+
+    public Context(String keyspace, String table, String tableId)
+    {
+        this.keyspace = keyspace;
+        this.table = table;
+        this.tableId = tableId;
+        this.hashCode = Objects.hash(keyspace, table, tableId);
+    }
+
+    public String getKeyspace()
+    {
+        return keyspace;
+    }
+
+    public String getTable()
+    {
+        return table;
+    }
+
+    public String getTableId()
+    {
+        return tableId;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Context context = (Context) o;
+        return Objects.equals(keyspace, context.keyspace) && Objects.equals(table, context.table) && Objects.equals(tableId, context.tableId);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return hashCode;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Context{" +
+               "keyspace='" + keyspace + '\'' +
+               ", table='" + table + '\'' +
+               ", tableId='" + tableId + '\'' +
+               '}';
+    }
+
+    public static Context from(ReadCommand command)
+    {
+        return from(command.metadata());
+    }
+
+    public static Context from(TableMetadata table)
+    {
+        return new Context(table.keyspace, table.name, table.id.toString());
+    }
+
+    public static Context from(IndexContext indexContext)
+    {
+        return new Context(indexContext.getKeyspace(), indexContext.getTable(), indexContext.getTableId().toString());
+    }
+}
diff --git a/src/java/org/apache/cassandra/sensors/NoOpRequestSensors.java b/src/java/org/apache/cassandra/sensors/NoOpRequestSensors.java
new file mode 100644
index 000000000000..dae952bd0d83
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/NoOpRequestSensors.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * No-op implementation of {@link RequestSensors}. This is used when sensors are disabled.
+ */
+public class NoOpRequestSensors implements RequestSensors
+{
+    public static final NoOpRequestSensors instance = new NoOpRequestSensors();
+
+    @Override
+    public void registerSensor(Context context, Type type)
+    {
+
+    }
+
+    @Override
+    public Optional<Sensor> getSensor(Context context, Type type)
+    {
+        return Optional.empty();
+    }
+
+    @Override
+    public Set<Sensor> getSensors(Predicate<Sensor> filter)
+    {
+        return ImmutableSet.of();
+    }
+
+    @Override
+    public void incrementSensor(Context context, Type type, double value)
+    {
+
+    }
+
+    @Override
+    public void syncAllSensors()
+    {
+
+    }
+}
diff --git a/src/java/org/apache/cassandra/sensors/RequestSensors.java b/src/java/org/apache/cassandra/sensors/RequestSensors.java
new file mode 100644
index 000000000000..9ce2641bb418
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/RequestSensors.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Collection;
+import java.util.Optional;
+import java.util.function.Predicate;
+
+/**
+ * Groups {@link Sensor}s associated to a given request/response and related {@link Context}: this is the main entry
+ * point to create and modify sensors. Actual implementations can be created via {@link SensorsFactory}.
+ */
+public interface RequestSensors
+{
+    /**
+     * Register a new sensor associated to the given context and type. It is up to the implementation to decide the
+     * idempotency of this operation.
+     *
+     * @param context the sensor context associated with the request/response
+     * @param type    the type of the sensor
+     */
+    void registerSensor(Context context, Type type);
+
+    /**
+     * Returns the sensor associated to the given context and type, if any.
+     *
+     * @param context the sensor context associated with the request/response
+     * @param type    the type of the sensor
+     * @return the sensor associated to the given context and type, if any
+     */
+    Optional<Sensor> getSensor(Context context, Type type);
+
+    /**
+     * Returns all the sensors that match the given filter
+     *
+     * @param filter a predicate applied to each sensor to decide if it should be included in the returned collection
+     * @return a collection of sensors matching the given predicate
+     */
+    Collection<Sensor> getSensors(Predicate<Sensor> filter);
+
+    /**
+     * Increment the sensor value associated to the given context and type by the given value.
+     *
+     * @param context the sensor context associated with the request/response
+     * @param type    the type of the sensor
+     * @param value   the value to increment the sensor by
+     */
+    void incrementSensor(Context context, Type type, double value);
+
+    /**
+     * Sync all the sensors values tracked for this request to the global {@link SensorsRegistry}. This method
+     * will be called at least once per request/response so it is recommended to make the implementation idempotent.
+     */
+    void syncAllSensors();
+}
diff --git a/src/java/org/apache/cassandra/sensors/RequestTracker.java b/src/java/org/apache/cassandra/sensors/RequestTracker.java
new file mode 100644
index 000000000000..42a799779820
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/RequestTracker.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import org.apache.cassandra.concurrent.ExecutorLocals;
+
+/**
+ * Extends {@link ExecutorLocals} implementation to track and propagate {@link RequestSensors} associated to a given request/response.
+ */
+public class RequestTracker extends ExecutorLocals.Impl
+{
+    public static final RequestTracker instance = new RequestTracker();
+
+    private RequestTracker()
+    {}
+
+    public RequestSensors get()
+    {
+        return ExecutorLocals.current().sensors;
+    }
+
+    public void set(RequestSensors sensors)
+    {
+        ExecutorLocals current = ExecutorLocals.current();
+        ExecutorLocals.Impl.set(current.traceState, current.clientWarnState, sensors, current.operationContext);
+    }
+}
diff --git a/src/java/org/apache/cassandra/sensors/Sensor.java b/src/java/org/apache/cassandra/sensors/Sensor.java
new file mode 100644
index 000000000000..75519bf50bf4
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/Sensor.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Objects;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.AtomicDouble;
+
+/**
+ * Tracks the {@link #value} for a given measurement of a given {@link Type} and {@link Context}, during any
+ * request/response cycle.
+ * <br/><br/>
+ * Sensors can be read (via {@link #getValue()}) but cannot be directly created or incremented, because their lifecycle
+ * and values are managed by the {@link RequestSensors} and {@link SensorsRegistry} classes, more specifically:
+ * <ul>
+ *     <li>In order to track a given measurement for a given request/response, register a sensor of the related type via
+ *     {@link RequestSensors#registerSensor(Type)}.</li>
+ *     <li>Once registered, the sensor lifecycle spans across multiple request/response cycles, and its "global"
+ *     value can be accessed via {@link SensorsRegistry}.</li>
+ * </ul>
+ */
+public class Sensor
+{
+    private final Context context;
+    private final Type type;
+    private final AtomicDouble value;
+
+    private final int hashCode;
+
+    protected Sensor(Context context, Type type)
+    {
+        this.context = context;
+        this.type = type;
+        this.value = new AtomicDouble();
+        this.hashCode = Objects.hash(context, type);
+    }
+
+    @VisibleForTesting
+    public void increment(double value)
+    {
+        this.value.addAndGet(value);
+    }
+
+    public Context getContext()
+    {
+        return context;
+    }
+
+    public Type getType()
+    {
+        return type;
+    }
+
+    public double getValue()
+    {
+        return value.doubleValue();
+    }
+
+    @VisibleForTesting
+    public void reset()
+    {
+        value.set(0);
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Sensor sensor = (Sensor) o;
+        return Objects.equals(context, sensor.context) && type == sensor.type;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return hashCode;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Sensor{" +
+               "context=" + context +
+               ", type=" + type +
+               ", value=" + value +
+               '}';
+    }
+}
diff --git a/src/java/org/apache/cassandra/sensors/SensorEncoder.java b/src/java/org/apache/cassandra/sensors/SensorEncoder.java
new file mode 100644
index 000000000000..8894026b17f5
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/SensorEncoder.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Optional;
+
+/**
+ * Encodes sensor name as string to be used on the wire (let it be in internode messages as custom params or in native protocol
+ * messages as custom payloads). Note that the sensor value itself will always be encoded as bytes in the big endian order
+ * (see {@link SensorsCustomParams#sensorValueAsBytes(double)} and {@link SensorsCustomParams#sensorValueAsByteBuffer(double)}).
+ * Implementations should be very efficient as sensor names are potentially encoded with each request. They should also encode
+ * enough information to differentiate between sensors of the same type that belong to the same request but different
+ * keyspaces and/or tables.
+ */
+public interface SensorEncoder
+{
+    /**
+     * Encodes request sensor name as a string to be used on the wire. A request sensor tracks usage per request. See {@link RequestSensors}.
+     *
+     * @param sensor the sensor to encode
+     * @return the encoded sensor as a string. If the optional is empty, the sensor will not be encoded.
+     */
+    Optional<String> encodeRequestSensorName(Sensor sensor);
+
+    /**
+     * Encodes global sensor name as a string to be used on the wire. A global sensor tracks usage globally across different requests. See {@link SensorsRegistry}.
+     *
+     * @param sensor the sensor to encode
+     * @return the encoded sensor as a string. If the optional is empty, the sensor will not be encoded.
+     */
+    Optional<String> encodeGlobalSensorName(Sensor sensor);
+}
diff --git a/src/java/org/apache/cassandra/sensors/SensorsCustomParams.java b/src/java/org/apache/cassandra/sensors/SensorsCustomParams.java
new file mode 100644
index 000000000000..9303ef71d3bb
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/SensorsCustomParams.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * A utility class that groups methods to facilitate encoding sensors in native or internode protocol messages:
+ * <ul>
+ *   <li>Sensors in internode messages: used to communicate sensors values from replicas to coordinators in the internode
+ *   message response {@link Message.Header#customParams()} bytes map.
+ *   See {@link SensorsCustomParams#addSensorsToInternodeResponse(RequestSensors, Message.Builder)} and
+ *   {@link SensorsCustomParams#sensorValueFromInternodeResponse(Message, String)}.</li>
+ *   <li>Sensors in native protocol messages: used to communicate sensors values from coordinator to upstream callers via the native protocol
+ *   response {@link org.apache.cassandra.transport.Message#getCustomPayload()} bytes map.
+ *   See {@link SensorsCustomParams#addSensorToCQLResponse(org.apache.cassandra.transport.Message.Response, ProtocolVersion, RequestSensors, Context, Type)}.</li>
+ * </ul>
+ */
+public final class SensorsCustomParams
+{
+    private static final SensorEncoder SENSOR_ENCODER = SensorsFactory.instance.createSensorEncoder();
+
+    private SensorsCustomParams()
+    {
+    }
+
+    /**
+     * Utility method to encode sensor value as byte[] in the big endian order.
+     */
+    public static byte[] sensorValueAsBytes(double value)
+    {
+        ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES);
+        buffer.putDouble(value);
+
+        return buffer.array();
+    }
+
+    /**
+     * Utility method to encode sensor value as ByteBuffer in the big endian order.
+     */
+    public static ByteBuffer sensorValueAsByteBuffer(double value)
+    {
+        ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES);
+        buffer.putDouble(value);
+        buffer.flip();
+        return buffer;
+    }
+
+    public static double sensorValueFromBytes(byte[] bytes)
+    {
+        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+        return buffer.getDouble();
+    }
+
+    /**
+     * Iterate over all sensors in the {@link RequestSensors} and encodes each sensor value by applying the given
+     * {@param valueFunction} in the internode response message as custom parameters.
+     *
+     * @param sensors the collection of sensors to encode in the response
+     * @param valueFunction the function to get the sensor value
+     * @param response the response message builder to add the sensors to
+     * @param <T> the response message builder type
+     */
+    public static <T> void addSensorsToInternodeResponse(RequestSensors sensors, Function<Sensor, Double> valueFunction, Message.Builder<T> response)
+    {
+        Preconditions.checkNotNull(sensors);
+        Preconditions.checkNotNull(response);
+
+        for (Sensor sensor : sensors.getSensors(ignored -> true))
+            addSensorToInternodeResponse(response, sensor, valueFunction);
+    }
+
+    /**
+     * Iterate over all sensors in the {@link RequestSensors} and encodes each sensor values in the internode response
+     * message as custom parameters.
+     *
+     * @param sensors the collection of sensors to encode in the response
+     * @param response the response message builder to add the sensors to
+     * @param <T> the response message builder type
+     */
+    public static <T> void addSensorsToInternodeResponse(RequestSensors sensors, Message.Builder<T> response)
+    {
+        addSensorsToInternodeResponse(sensors, Sensor::getValue, response);
+    }
+
+    /**
+     * Reads the sensor value encoded in the response message header as {@link Message.Header#customParams()} bytes map.
+     *
+     * @param message the message to read the sensor value from
+     * @param customParam the name of the header in custom params to read the sensor value from
+     * @param <T> the message type
+     * @return the sensor value
+     */
+    public static <T> double sensorValueFromInternodeResponse(Message<T> message, String customParam)
+    {
+        if (customParam == null)
+            return 0.0;
+
+        Map<String, byte[]> customParams = message.header.customParams();
+        if (customParams == null)
+            return 0.0;
+
+        byte[] readBytes = message.header.customParams().get(customParam);
+        if (readBytes == null)
+            return 0.0;
+
+        return sensorValueFromBytes(readBytes);
+    }
+
+    /**
+     * Adds a sensor of a given type and context to the native protocol response message encoded in the custom payload bytes map.
+     * If the sensor is already present in the custom payload, it will be overwritten.
+     *
+     * @param response the response message to add the sensors to
+     * @param protocolVersion the protocol version specified in query options to determine if custom payload is supported (should be V4 or later).
+     * @param sensors the requests sensors associated with the request to get the sensor values from.
+     * @param context the context of the sensor to add to the response
+     * @param type the type of the sensor to add to the response
+     */
+    public static void addSensorToCQLResponse(org.apache.cassandra.transport.Message.Response response,
+                                              ProtocolVersion protocolVersion,
+                                              RequestSensors sensors,
+                                              Context context,
+                                              Type type)
+    {
+        if (!CassandraRelevantProperties.SENSORS_VIA_NATIVE_PROTOCOL.getBoolean())
+            return;
+
+        // Custom payload is not supported for protocol versions < 4
+        if (protocolVersion.isSmallerThan(ProtocolVersion.V4))
+            return;
+
+        if (response == null || sensors == null)
+            return;
+
+        Optional<Sensor> requestSensor = sensors.getSensor(context, type);
+        if (requestSensor.isEmpty())
+            return;
+
+        Optional<String> headerName = SENSOR_ENCODER.encodeRequestSensorName(requestSensor.get());
+        if (headerName.isEmpty())
+            return;
+
+        Map<String, ByteBuffer> customPayload = response.getCustomPayload() == null ? new HashMap<>() : response.getCustomPayload();
+        ByteBuffer bytes = SensorsCustomParams.sensorValueAsByteBuffer(requestSensor.get().getValue());
+        customPayload.put(headerName.get(), bytes);
+        response.setCustomPayload(customPayload);
+    }
+
+    private static <T> void addSensorToInternodeResponse(Message.Builder<T> response, Sensor requestSensor, Function<Sensor, Double> valueFunction)
+    {
+        Optional<String> requestParam = paramForRequestSensor(requestSensor);
+        if (requestParam.isEmpty())
+            return;
+
+        byte[] requestBytes = SensorsCustomParams.sensorValueAsBytes(valueFunction.apply(requestSensor));
+        response.withCustomParam(requestParam.get(), requestBytes);
+
+        Optional<Sensor> globalSensor = SensorsRegistry.instance.getSensor(requestSensor.getContext(), requestSensor.getType());
+        if (globalSensor.isEmpty())
+            return;
+
+        Optional<String> globalParam = paramForGlobalSensor(globalSensor.get());
+        if (globalParam.isEmpty())
+            return;
+
+        byte[] globalBytes = SensorsCustomParams.sensorValueAsBytes(valueFunction.apply(globalSensor.get()));
+        response.withCustomParam(globalParam.get(), globalBytes);
+    }
+
+    public static Optional<String> paramForRequestSensor(Sensor sensor)
+    {
+        return SENSOR_ENCODER.encodeRequestSensorName(sensor);
+    }
+
+    public static Optional<String> paramForGlobalSensor(Sensor sensor)
+    {
+        return SENSOR_ENCODER.encodeGlobalSensorName(sensor);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/sensors/SensorsFactory.java b/src/java/org/apache/cassandra/sensors/SensorsFactory.java
new file mode 100644
index 000000000000..6bc49896955f
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/SensorsFactory.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Optional;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SENSORS_FACTORY;
+
+/**
+ * Provides a factory to customize the behaviour of sensors tracking in CNDB by providing two factory methods:
+ * <ul>
+ *   <li>{@link SensorsFactory#createRequestSensors} provides a {@link RequestSensors} implementation to track sensors per keyspace.</li>
+ *   <li>{@link SensorsFactory#createSensorEncoder} provides a {@link SensorEncoder} implementation to control how sensors are encoded as string on the wire.</li>
+ * </ul>
+ * The concrete implementation of this factory is configured by the {@link CassandraRelevantProperties#SENSORS_FACTORY} system property.
+ */
+public interface SensorsFactory
+{
+    SensorsFactory instance = SENSORS_FACTORY.getString() == null ?
+                              new SensorsFactory() {} :
+                              FBUtilities.construct(CassandraRelevantProperties.SENSORS_FACTORY.getString(), "sensors factory");
+
+    SensorEncoder NOOP_SENSOR_ENCODER = new SensorEncoder()
+    {
+        @Override
+        public Optional<String> encodeRequestSensorName(Sensor sensor)
+        {
+            return Optional.empty();
+        }
+
+        @Override
+        public Optional<String> encodeGlobalSensorName(Sensor sensor)
+        {
+            return Optional.empty();
+        }
+    };
+
+    /**
+     * Creates {@link RequestSensors} for the given keyspaces. This method is invoked by coordinators and replicas when
+     * handling requests at various stages/thread pools (e.g. when processing CQL queries or when applying verbs).
+     * Consequently, implementations should be very efficient.
+     *
+     * @param keyspaces the keyspaces associated with the request.
+     * @return a {@link RequestSensors} instance. The default implementation returns a singleton no-op instance.
+     */
+    default RequestSensors createRequestSensors(String... keyspaces)
+    {
+        return NoOpRequestSensors.instance;
+    }
+
+    /**
+     * Create a {@link SensorEncoder} that will be invoked when encoding the sensor on the wire. The default implementation returns a noop encoder.
+     */
+    default SensorEncoder createSensorEncoder()
+    {
+        return NOOP_SENSOR_ENCODER;
+    }
+}
diff --git a/src/java/org/apache/cassandra/sensors/SensorsRegistry.java b/src/java/org/apache/cassandra/sensors/SensorsRegistry.java
new file mode 100644
index 000000000000..fa30338ca506
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/SensorsRegistry.java
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Striped;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaChangeListener;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.concurrent.Timer;
+
+/**
+ * This class tracks {@link Sensor}s at a "global" level, allowing to:
+ * <ul>
+ *     <li>Getting or creating (if not existing) sensors of a given {@link Context} and {@link Type}.</li>
+ *     <li>Accessing sensors by keyspace, table id or type.</li>
+ * </ul>
+ * The returned sensors are global, meaning that their value spans across requests/responses, but cannot be modified either
+ * directly or indirectly via this class (whose update methods are package protected). In order to modify a sensor value,
+ * it must be registered to a request/response via {@link RequestSensors#registerSensor(Context, Type)} and incremented via
+ * {@link RequestSensors#incrementSensor(Context, Type, double)}, then synced via {@link RequestSensors#syncAllSensors()}, which
+ * will update the related global sensors.
+ * <br/><br/>
+ * Given sensors are tied to a context, that is to a given keyspace and table, their global instance will be deleted
+ * if the related keyspace/table is dropped.
+ * <br/><br/>
+ * It's also possible to:
+ * <ul>
+ *     <li>
+ *         Register listeners via the {@link #registerListener(SensorsRegistryListener)} method.
+ *         Such listeners will get notified on creation and removal of sensors.
+ *     </li>
+ *     <li>
+ *         Unregister listeners via the {@link #unregisterListener(SensorsRegistryListener)} method.
+ *         Such listeners will not be notified anymore about creation or removal of sensors.
+ *     </li>
+ * </ul>
+ */
+public class SensorsRegistry implements SchemaChangeListener
+{
+    private static final int LOCK_SRIPES = 1024;
+
+    public static final SensorsRegistry instance = new SensorsRegistry();
+    private static final Logger logger = LoggerFactory.getLogger(SensorsRegistry.class);
+
+    private final Timer asyncUpdater = Timer.INSTANCE;
+
+    private final Striped<ReadWriteLock> stripedUpdateLock = Striped.readWriteLock(LOCK_SRIPES); // we stripe per keyspace
+
+    private final Set<String> keyspaces = Sets.newConcurrentHashSet();
+    private final Set<String> tableIds = Sets.newConcurrentHashSet();
+
+    // Using Map of array values for performance reasons to avoid wrapping key into another Object, e.g. Pair(context,type)).
+    // Note that array values can contain NULL so be careful to filter NULLs when iterating over array
+    private final ConcurrentMap<Context, Sensor[]> identity = new ConcurrentHashMap<>();
+
+    private final ConcurrentMap<String, Set<Sensor>> byKeyspace = new ConcurrentHashMap<>();
+    private final ConcurrentMap<String, Set<Sensor>> byTableId = new ConcurrentHashMap<>();
+    private final ConcurrentMap<String, Set<Sensor>> byType = new ConcurrentHashMap<>();
+
+    private final CopyOnWriteArrayList<SensorsRegistryListener> listeners = new CopyOnWriteArrayList<>();
+
+    private SensorsRegistry()
+    {
+        Schema.instance.registerListener(this);
+    }
+
+    public void registerListener(SensorsRegistryListener listener)
+    {
+        listeners.add(listener);
+        logger.debug("Listener {} registered", listener);
+    }
+
+    public void unregisterListener(SensorsRegistryListener listener)
+    {
+        listeners.remove(listener);
+        logger.debug("Listener {} unregistered", listener);
+    }
+
+    public Optional<Sensor> getSensor(Context context, Type type)
+    {
+        return Optional.ofNullable(getSensorFast(context, type));
+    }
+
+    public Optional<Sensor> getOrCreateSensor(Context context, Type type)
+    {
+        return Optional.ofNullable(getOrCreateSensorFast(context, type));
+    }
+
+    protected void incrementSensor(Context context, Type type, double value)
+    {
+        Sensor sensor = getOrCreateSensorFast(context, type);
+        if (sensor != null)
+            sensor.increment(value);
+    }
+
+    protected Future<Void> incrementSensorAsync(Context context, Type type, double value, long delay, TimeUnit unit)
+    {
+        return asyncUpdater.onTimeout(() ->
+                                      getOrCreateSensor(context, type).ifPresent(s -> s.increment(value)),
+                                      delay, unit);
+    }
+
+    public Set<Sensor> getSensorsByKeyspace(String keyspace)
+    {
+        return Optional.ofNullable(byKeyspace.get(keyspace)).orElseGet(() -> ImmutableSet.of());
+    }
+
+    public Set<Sensor> getSensorsByTableId(String tableId)
+    {
+        return Optional.ofNullable(byTableId.get(tableId)).orElseGet(() -> ImmutableSet.of());
+    }
+
+    public Set<Sensor> getSensorsByType(Type type)
+    {
+        return Optional.ofNullable(byType.get(type.name())).orElseGet(() -> ImmutableSet.of());
+    }
+
+    public void removeSensorsByKeyspace(String keyspaceName)
+    {
+        stripedUpdateLock.getAt(getLockStripe(keyspaceName.hashCode())).writeLock().lock();
+        try
+        {
+            byKeyspace.remove(keyspaceName);
+
+            Set<Sensor> removed = removeSensorArrays(ImmutableSet.of(identity.values()), s -> s.getContext().getKeyspace().equals(keyspaceName));
+            removed.forEach(this::notifyOnSensorRemoved);
+
+            removeSensor(byTableId.values(), s -> s.getContext().getKeyspace().equals(keyspaceName));
+            removeSensor(byType.values(), s -> s.getContext().getKeyspace().equals(keyspaceName));
+        }
+        finally
+        {
+            stripedUpdateLock.getAt(getLockStripe(keyspaceName.hashCode())).writeLock().unlock();
+        }
+    }
+
+    public void removeSensorsByTableId(String keyspaceName, String tableId)
+    {
+        stripedUpdateLock.getAt(getLockStripe(keyspaceName.hashCode())).writeLock().lock();
+        try
+        {
+            Set<Sensor> removed = removeSensorArrays(ImmutableSet.of(identity.values()), s -> s.getContext().getTableId().equals(tableId));
+            removed.forEach(this::notifyOnSensorRemoved);
+
+            byTableId.remove(tableId);
+            removeSensor(byType.values(), s -> s.getContext().getTableId().equals(tableId));
+        }
+        finally
+        {
+            stripedUpdateLock.getAt(getLockStripe(keyspaceName.hashCode())).writeLock().unlock();
+        }
+    }
+
+    @Override
+    public void onCreateKeyspace(KeyspaceMetadata keyspace)
+    {
+        keyspaces.add(keyspace.name);
+    }
+
+    @Override
+    public void onCreateTable(TableMetadata table)
+    {
+        tableIds.add(table.id.toString());
+    }
+
+    @Override
+    public void onDropKeyspace(KeyspaceMetadata keyspace, boolean dropData)
+    {
+        stripedUpdateLock.getAt(getLockStripe(keyspace.name.hashCode())).writeLock().lock();
+        try
+        {
+            keyspaces.remove(keyspace.name);
+            byKeyspace.remove(keyspace.name);
+
+            Set<Sensor> removed = removeSensorArrays(ImmutableSet.of(identity.values()), s -> s.getContext().getKeyspace().equals(keyspace.name));
+            removed.forEach(this::notifyOnSensorRemoved);
+
+            removeSensor(byTableId.values(), s -> s.getContext().getKeyspace().equals(keyspace.name));
+            removeSensor(byType.values(), s -> s.getContext().getKeyspace().equals(keyspace.name));
+        }
+        finally
+        {
+            stripedUpdateLock.getAt(getLockStripe(keyspace.name.hashCode())).writeLock().unlock();
+        }
+    }
+
+    @Override
+    public void onDropTable(TableMetadata table, boolean dropData)
+    {
+        stripedUpdateLock.getAt(getLockStripe(table.keyspace.hashCode())).writeLock().lock();
+        try
+        {
+            String tableId = table.id.toString();
+            tableIds.remove(tableId);
+            byTableId.remove(tableId);
+
+            Set<Sensor> removed = removeSensorArrays(ImmutableSet.of(identity.values()), s -> s.getContext().getTableId().equals(tableId));
+            removed.forEach(this::notifyOnSensorRemoved);
+
+            removeSensor(byKeyspace.values(), s -> s.getContext().getTableId().equals(tableId));
+            removeSensor(byType.values(), s -> s.getContext().getTableId().equals(tableId));
+        }
+        finally
+        {
+            stripedUpdateLock.getAt(getLockStripe(table.keyspace.hashCode())).writeLock().unlock();
+        }
+    }
+
+    private static int getLockStripe(int hashCode)
+    {
+        return Math.abs(hashCode) % LOCK_SRIPES;
+    }
+
+    /**
+     * Remove sensors from a collection of candidates based on the given predicate
+     *
+     * @param candidates the candidates to remove from
+     * @param accept the predicate used to select the sensors to remove
+     * @return the set of removed sensors
+     */
+    private Set<Sensor> removeSensor(Collection<? extends Collection<Sensor>> candidates, Predicate<Sensor> accept)
+    {
+        Set<Sensor> removed = new HashSet<>();
+
+        for (Collection<Sensor> sensors : candidates)
+        {
+            Iterator<Sensor> sensorIt = sensors.iterator();
+            while (sensorIt.hasNext())
+            {
+                Sensor sensor = sensorIt.next();
+                if (!accept.test(sensor))
+                    continue;
+
+                sensorIt.remove();
+                removed.add(sensor);
+            }
+        }
+
+        return removed;
+    }
+
+    /**
+     * To get best perfromance we are not returning Optional here
+     */
+    @Nullable
+    private Sensor getSensorFast(Context context, Type type)
+    {
+        Sensor[] typeSensors = identity.get(context);
+        return  typeSensors != null ? typeSensors[type.ordinal()] : null;
+    }
+
+    /**
+     * To get best perfromance we are not returning Optional here
+     */
+    @Nullable
+    private Sensor getOrCreateSensorFast(Context context, Type type)
+    {
+        Sensor sensor = getSensorFast(context, type);
+        if (sensor != null)
+            return sensor;
+
+        stripedUpdateLock.getAt(getLockStripe(context.getKeyspace().hashCode())).readLock().lock();
+        try
+        {
+            if (!keyspaces.contains(context.getKeyspace()) || !tableIds.contains(context.getTableId()))
+                return null;
+
+            Sensor[] typeSensors = identity.compute(context, (key, types) -> {
+                Sensor[] computed = types != null ? types : new Sensor[Type.values().length];
+                if (computed[type.ordinal()] == null)
+                {
+                    computed[type.ordinal()] = new Sensor(context, type);
+                    notifyOnSensorCreated(computed[type.ordinal()]);
+                }
+                return computed;
+            });
+            sensor = typeSensors[type.ordinal()];
+
+            Set<Sensor> keyspaceSet = byKeyspace.get(sensor.getContext().getKeyspace());
+            keyspaceSet = keyspaceSet != null ? keyspaceSet : byKeyspace.computeIfAbsent(sensor.getContext().getKeyspace(), (ignored) -> Sets.newConcurrentHashSet());
+            keyspaceSet.add(sensor);
+
+            Set<Sensor> tableSet = byTableId.get(sensor.getContext().getTableId());
+            tableSet = tableSet != null ? tableSet : byTableId.computeIfAbsent(sensor.getContext().getTableId(), (ignored) -> Sets.newConcurrentHashSet());
+            tableSet.add(sensor);
+
+            Set<Sensor> opSet = byType.get(sensor.getType().name());
+            opSet = opSet != null ? opSet : byType.computeIfAbsent(sensor.getType().name(), (ignored) -> Sets.newConcurrentHashSet());
+            opSet.add(sensor);
+
+            return sensor;
+        }
+        finally
+        {
+            stripedUpdateLock.getAt(getLockStripe(context.getKeyspace().hashCode())).readLock().unlock();
+        }
+    }
+
+    /**
+     * Removes array of sensors if any sensor in the array matches the predicate.
+     * This function is used by `identity` map that holds an array of Sensors (each item in the array maps to Type)
+     */
+    private Set<Sensor> removeSensorArrays(Collection<? extends Collection<Sensor[]>> candidates, Predicate<Sensor> accept)
+    {
+        Set<Sensor> removed = new HashSet<>();
+
+        for (Collection<Sensor[]> sensors : candidates)
+        {
+            Iterator<Sensor[]> sensorIt = sensors.iterator();
+            while (sensorIt.hasNext())
+            {
+                List<Sensor> typeSensors = Arrays.stream(sensorIt.next()).filter(Objects::nonNull).collect(Collectors.toList());
+                if (typeSensors.size() > 0 && accept.test(typeSensors.get(0)))
+                {
+                    removed.addAll(typeSensors);
+                    sensorIt.remove();
+                }
+            }
+        }
+
+        return removed;
+    }
+
+    @VisibleForTesting
+    public void clear()
+    {
+        keyspaces.clear();
+        tableIds.clear();
+        identity.clear();
+        byKeyspace.clear();
+        byTableId.clear();
+        byType.clear();
+    }
+
+    private void notifyOnSensorCreated(Sensor sensor)
+    {
+        tryNotifyListeners(sensor, SensorsRegistryListener::onSensorCreated, "created");
+    }
+
+    private void notifyOnSensorRemoved(Sensor sensor)
+    {
+        tryNotifyListeners(sensor, SensorsRegistryListener::onSensorRemoved, "removed");
+    }
+
+    private void tryNotifyListeners(Sensor sensor, BiConsumer<SensorsRegistryListener, Sensor> notification, String action)
+    {
+        for (SensorsRegistryListener l: listeners)
+        {
+            try
+            {
+                notification.accept(l, sensor);
+                logger.trace("Listener {} correctly notified on sensor {} being {}", l, sensor, action);
+            }
+            catch (Throwable t)
+            {
+                logger.error("Failed to notify listener {} on sensor {} being {}", l, sensor, action);
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/sensors/SensorsRegistryListener.java b/src/java/org/apache/cassandra/sensors/SensorsRegistryListener.java
new file mode 100644
index 000000000000..9fc52f1a4b7f
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/SensorsRegistryListener.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+/**
+ * Listener that gets notified by the {@link SensorsRegistry}
+ * about the creation and removal of {@link Sensor}s.
+ */
+public interface SensorsRegistryListener
+{
+    /**
+     * React to the creation of a new sensor
+     * @param sensor the sensor just created
+     */
+    void onSensorCreated(Sensor sensor);
+
+    /**
+     * React to the removal of a sensor
+     * @param sensor the sensor just removed
+     */
+    void onSensorRemoved(Sensor sensor);
+}
diff --git a/src/java/org/apache/cassandra/sensors/Type.java b/src/java/org/apache/cassandra/sensors/Type.java
new file mode 100644
index 000000000000..25fad4e2e2bd
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/Type.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+/**
+ * The type of the measurement a {@link Sensor} refers to.
+ */
+public enum Type
+{
+    INTERNODE_BYTES,
+
+    READ_BYTES,
+
+    WRITE_BYTES,
+    INDEX_WRITE_BYTES
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/sensors/read/TrackingRowIterator.java b/src/java/org/apache/cassandra/sensors/read/TrackingRowIterator.java
new file mode 100644
index 000000000000..c6b64bc34d60
--- /dev/null
+++ b/src/java/org/apache/cassandra/sensors/read/TrackingRowIterator.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors.read;
+
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.Sensor;
+import org.apache.cassandra.sensors.Type;
+
+/**
+ * Increment {@link Type#READ_BYTES} {@link Sensor}s for a given {@link Context } by the data size of each iterated row and sync the sensor values
+ * when the iterator is closed.
+ */
+public class TrackingRowIterator extends Transformation<UnfilteredRowIterator>
+{
+    private final RequestTracker requestTracker;
+    private final Context context;
+
+    public TrackingRowIterator(Context context)
+    {
+        this.requestTracker = RequestTracker.instance;
+        this.context = context;
+    }
+
+    @Override
+    public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator iter)
+    {
+        return Transformation.apply(iter, this);
+    }
+
+    @Override
+    public Row applyToStatic(Row row)
+    {
+        // TODO: Not worth tracking the static row?
+        return row;
+    }
+
+    @Override
+    public Row applyToRow(Row row)
+    {
+        RequestSensors sensors = requestTracker.get();
+        if (sensors != null && row.isRow())
+            sensors.incrementSensor(context, Type.READ_BYTES, row.dataSize());
+
+        return row;
+    }
+
+    @Override
+    protected void onClose()
+    {
+        super.onClose();
+
+        RequestSensors sensors = requestTracker.get();
+        if (sensors != null)
+            sensors.syncAllSensors();
+    }
+}
diff --git a/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java b/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java
index 1be4d61f68ec..8561f2ca02f2 100644
--- a/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java
@@ -45,6 +45,8 @@ public void serializeList(List<AbstractType<?>> types, DataOutputPlus out) throw
             serialize(type, out);
     }
 
+    // Used only in serialization header, when deserializing a type from the sstable header,
+    // not used in commit log or internode transport.
     public AbstractType<?> deserialize(DataInputPlus in) throws IOException
     {
         ByteBuffer raw = ByteBufferUtil.readWithVIntLength(in);
@@ -72,4 +74,4 @@ public long serializedListSize(List<AbstractType<?>> types)
             size += serializedSize(type);
         return size;
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/serializers/CollectionSerializer.java b/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
index b514d2490b0a..ca3371857863 100644
--- a/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
@@ -229,4 +229,45 @@ public void forEach(ByteBuffer input, Consumer<ByteBuffer> action)
             throw new MarshalException("Not enough bytes to read a set");
         }
     }
+
+    /**
+     * Checks if the specified serialized collection contains the specified serialized collection element.
+     *
+     * @param elementType the type of the collection elements
+     * @param collection a serialized collection
+     * @param element a serialized collection element
+     * @param hasKeys whether the collection has keys, that is, it's a map
+     * @param getKeys whether to check keys or values
+     * @return {@code true} if the collection contains the element, {@code false} otherwise
+     */
+    public static boolean contains(AbstractType<?> elementType,
+                                   ByteBuffer collection,
+                                   ByteBuffer element,
+                                   boolean hasKeys,
+                                   boolean getKeys)
+    {
+        assert hasKeys || !getKeys;
+        int size = readCollectionSize(collection, ByteBufferAccessor.instance);
+        int offset = sizeOfCollectionSize();
+
+        for (int i = 0; i < size; i++)
+        {
+            // read the key (if the collection has keys)
+            if (hasKeys)
+            {
+                ByteBuffer key = readValue(collection, ByteBufferAccessor.instance, offset);
+                if (getKeys && elementType.compare(key, element) == 0)
+                    return true;
+                offset += sizeOfValue(key, ByteBufferAccessor.instance);
+            }
+
+            // read the value
+            ByteBuffer value = readValue(collection, ByteBufferAccessor.instance, offset);
+            if (!getKeys && elementType.compare(value, element) == 0)
+                return true;
+            offset += sizeOfValue(value, ByteBufferAccessor.instance);
+        }
+
+        return false;
+    }
 }
diff --git a/src/java/org/apache/cassandra/serializers/DateRangeSerializer.java b/src/java/org/apache/cassandra/serializers/DateRangeSerializer.java
new file mode 100644
index 000000000000..c7cd84094f7b
--- /dev/null
+++ b/src/java/org/apache/cassandra/serializers/DateRangeSerializer.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.serializers;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.time.Instant;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.db.marshal.datetime.DateRange;
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBound.Precision;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+///**
+// * Responsible for {@link DateRange} serialization/deserialization with respect to the following format:
+// * -------------------------
+// * <type>[<time0><precision0>[<time1><precision1>]]
+// *
+// * Where:
+// *
+// * <type> is a [byte] encoding of
+// * - 0x00 - single value as in "2001-01-01"
+// * - 0x01 - closed range as in "[2001-01-01 TO 2001-01-31]"
+// * - 0x02 - open range high as in "[2001-01-01 TO *]"
+// * - 0x03 - open range low as in "[* TO 2001-01-01]"
+// * - 0x04 - both ranges open as in "[* TO *]"
+// * - 0x05 - single value open as in "*"
+// *
+// * <time0> is an optional [long] millisecond offset from epoch. Absent for <type> in [4,5], present otherwise.
+// * Represents a single date value for <type> = 0, the range start for <type> in [1,2], or range end for <type> = 3.
+// *
+// * <precision0> is an optional [byte]s and represents the precision of field <time0>. Absent for <type> in [4,5], present otherwise.
+// * Possible values are:
+// * - 0x00 - year
+// * - 0x01 - month
+// * - 0x02 - day
+// * - 0x03 - hour
+// * - 0x04 - minute
+// * - 0x05 - second
+// * - 0x06 - millisecond
+// *
+// * <time1> is an optional [long] millisecond offset from epoch. Represents the range end for <type> = 1. Not present
+// * otherwise.
+// *
+// * <precision1> is an optional [byte] and represents the precision of field <time1>. Only present if <type> = 1. Values
+// * are the same as for <precision0>.
+// */
+public final class DateRangeSerializer extends TypeSerializer<DateRange>
+{
+    public static final DateRangeSerializer instance = new DateRangeSerializer();
+
+    // e.g. [2001-01-01]
+    private final static byte DATE_RANGE_TYPE_SINGLE_DATE = 0x00;
+    // e.g. [2001-01-01 TO 2001-01-31]
+    private final static byte DATE_RANGE_TYPE_CLOSED_RANGE = 0x01;
+    // e.g. [2001-01-01 TO *]
+    private final static byte DATE_RANGE_TYPE_OPEN_RANGE_HIGH = 0x02;
+    // e.g. [* TO 2001-01-01]
+    private final static byte DATE_RANGE_TYPE_OPEN_RANGE_LOW = 0x03;
+    // [* TO *]
+    private final static byte DATE_RANGE_TYPE_BOTH_OPEN_RANGE = 0x04;
+    // *
+    private final static byte DATE_RANGE_TYPE_SINGLE_DATE_OPEN = 0x05;
+
+    /**
+     * Size of the single serialized DateRange boundary. As specified in @{@link DateRangeSerializer}.
+     *
+     * Tightly coupled with {@link #deserializeDateRangeLowerBound(int, Object, ValueAccessor)} and
+     * {@link #deserializeDateRangeUpperBound(int, Object, ValueAccessor)}.
+     */
+    private final static int SERIALIZED_DATE_RANGE_BOUND_SIZE = TypeSizes.LONG_SIZE + TypeSizes.BYTE_SIZE;
+
+    private static final List<Integer> VALID_SERIALIZED_LENGTHS = ImmutableList.of(
+            // types: 0x04, 0x05
+            Byte.BYTES,
+            // types: 0x00, 0x02, 0x03
+            Byte.BYTES + Long.BYTES + Byte.BYTES,
+            // types: 0x01
+            Byte.BYTES + Long.BYTES + Byte.BYTES + Long.BYTES + Byte.BYTES
+    );
+
+
+    @Override
+    public ByteBuffer serialize(DateRange dateRange)
+    {
+        if (dateRange == null)
+        {
+            return ByteBufferUtil.EMPTY_BYTE_BUFFER;
+        }
+
+        byte rangeType = encodeType(dateRange);
+
+        int bufferSize = 1;
+        if (!dateRange.getLowerBound().isUnbounded())
+        {
+            bufferSize += 9;
+        }
+        if (dateRange.isUpperBoundDefined() && !dateRange.getUpperBound().isUnbounded())
+        {
+            bufferSize += 9;
+        }
+
+        try (DataOutputBuffer output = new DataOutputBuffer(bufferSize))
+        {
+            output.writeByte(rangeType);
+            DateRange.DateRangeBound lowerBound = dateRange.getLowerBound();
+            if (!lowerBound.isUnbounded())
+            {
+                output.writeLong(lowerBound.getTimestamp().toEpochMilli());
+                output.writeByte(lowerBound.getPrecision().toEncoded());
+            }
+
+            if (dateRange.isUpperBoundDefined())
+            {
+                DateRange.DateRangeBound upperBound = dateRange.getUpperBound();
+                if (!upperBound.isUnbounded())
+                {
+                    output.writeLong(upperBound.getTimestamp().toEpochMilli());
+                    output.writeByte(upperBound.getPrecision().toEncoded());
+                }
+            }
+            return output.buffer();
+        }
+        catch (IOException e)
+        {
+            throw new AssertionError("Unexpected error", e);
+        }
+    }
+
+    @Override
+    public <V> DateRange deserialize(V value, ValueAccessor<V> accessor)
+    {
+        if (accessor.isEmpty(value))
+        {
+            return null;
+        }
+
+        try
+        {
+            byte type = accessor.toByte(value);
+            int offset = TypeSizes.BYTE_SIZE;
+            switch (type)
+            {
+                case DATE_RANGE_TYPE_SINGLE_DATE:
+                    return new DateRange(deserializeDateRangeLowerBound(offset, value, accessor));
+                case DATE_RANGE_TYPE_CLOSED_RANGE:
+                    DateRange.DateRangeBound lowerBound = deserializeDateRangeLowerBound(offset, value, accessor);
+                    offset += SERIALIZED_DATE_RANGE_BOUND_SIZE;
+                    DateRange.DateRangeBound upperBound = deserializeDateRangeUpperBound(offset, value, accessor);
+                    return new DateRange(lowerBound, upperBound);
+                case DATE_RANGE_TYPE_OPEN_RANGE_HIGH:
+                    return new DateRange(deserializeDateRangeLowerBound(offset, value, accessor), DateRange.DateRangeBound.UNBOUNDED);
+                case DATE_RANGE_TYPE_OPEN_RANGE_LOW:
+                    return new DateRange(DateRange.DateRangeBound.UNBOUNDED, deserializeDateRangeUpperBound(offset, value, accessor));
+                case DATE_RANGE_TYPE_BOTH_OPEN_RANGE:
+                    return new DateRange(DateRange.DateRangeBound.UNBOUNDED, DateRange.DateRangeBound.UNBOUNDED);
+                case DATE_RANGE_TYPE_SINGLE_DATE_OPEN:
+                    return new DateRange(DateRange.DateRangeBound.UNBOUNDED);
+                default:
+                    throw new IllegalArgumentException("Unknown date range type: " + type);
+            }
+        }
+        catch (IOException e)
+        {
+            throw new AssertionError("Unexpected error", e);
+        }
+    }
+
+    @Override
+    public <V> void validate(V value, ValueAccessor<V> accessor) throws MarshalException
+    {
+        if (!VALID_SERIALIZED_LENGTHS.contains(accessor.size(value)))
+        {
+            throw new MarshalException(String.format("Date range should be have %s bytes, got %d instead.", VALID_SERIALIZED_LENGTHS, accessor.size(value)));
+        }
+        DateRange dateRange = deserialize(value, accessor);
+        validateDateRange(dateRange);
+    }
+
+    @Override
+    public String toString(DateRange dateRange)
+    {
+        return dateRange == null ? "" : dateRange.formatToSolrString();
+    }
+
+    @Override
+    public Class<DateRange> getType()
+    {
+        return DateRange.class;
+    }
+
+    private byte encodeType(DateRange dateRange)
+    {
+        if (dateRange.isUpperBoundDefined())
+        {
+            if (dateRange.getLowerBound().isUnbounded())
+            {
+                return dateRange.getUpperBound().isUnbounded() ? DATE_RANGE_TYPE_BOTH_OPEN_RANGE : DATE_RANGE_TYPE_OPEN_RANGE_LOW;
+            }
+            else
+            {
+                return dateRange.getUpperBound().isUnbounded() ? DATE_RANGE_TYPE_OPEN_RANGE_HIGH : DATE_RANGE_TYPE_CLOSED_RANGE;
+            }
+        }
+        else
+        {
+            return dateRange.getLowerBound().isUnbounded() ? DATE_RANGE_TYPE_SINGLE_DATE_OPEN : DATE_RANGE_TYPE_SINGLE_DATE;
+        }
+    }
+
+    private <V> DateRange.DateRangeBound deserializeDateRangeLowerBound(int offset, V value, ValueAccessor<V> accessor) throws IOException
+    {
+        long epochMillis = accessor.getLong(value, offset);
+        offset += TypeSizes.LONG_SIZE;
+        Precision precision = Precision.fromEncoded(accessor.getByte(value, offset));
+        return DateRange.DateRangeBound.lowerBound(Instant.ofEpochMilli(epochMillis), precision);
+    }
+
+    private <V> DateRange.DateRangeBound deserializeDateRangeUpperBound(int offset, V value, ValueAccessor<V> accessor) throws IOException
+    {
+        long epochMillis = accessor.getLong(value, offset);
+        offset += TypeSizes.LONG_SIZE;
+        Precision precision = Precision.fromEncoded(accessor.getByte(value, offset));
+        return DateRange.DateRangeBound.upperBound(Instant.ofEpochMilli(epochMillis), precision);
+    }
+
+    private void validateDateRange(DateRange dateRange)
+    {
+        if (dateRange != null && !dateRange.getLowerBound().isUnbounded() && dateRange.isUpperBoundDefined() && !dateRange.getUpperBound().isUnbounded())
+        {
+            if (dateRange.getLowerBound().getTimestamp().isAfter(dateRange.getUpperBound().getTimestamp()))
+            {
+                throw new MarshalException(String.format("Lower bound of a date range should be before upper bound, got: %s",
+                        dateRange.formatToSolrString()));
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java b/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java
index 764565c384eb..b5da62e214bd 100644
--- a/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java
@@ -70,14 +70,7 @@ public static int dateStringToDays(String source) throws MarshalException
         {
             LocalDate parsed = formatter.parse(source, LocalDate::from);
             long millis = parsed.atStartOfDay(UTC).toInstant().toEpochMilli();
-            if (millis < minSupportedDateMillis)
-                throw new MarshalException(String.format("Input date %s is less than min supported date %s", source,
-                        ZonedDateTime.ofInstant(Instant.ofEpochMilli(minSupportedDateMillis), UTC).toString()));
-            if (millis > maxSupportedDateMillis)
-                throw new MarshalException(String.format("Input date %s is greater than max supported date %s", source,
-                        ZonedDateTime.ofInstant(Instant.ofEpochMilli(maxSupportedDateMillis), UTC).toString()));
-
-            return timeInMillisToDay(millis);
+            return timeInMillisToDay(source, millis);
         }
         catch (DateTimeParseException| ArithmeticException e1)
         {
@@ -107,6 +100,23 @@ private static int parseRaw(String source) {
 
     public static int timeInMillisToDay(long millis)
     {
+        return timeInMillisToDay(null, millis);
+    }
+
+    private static int timeInMillisToDay(String source, long millis)
+    {
+        if (millis < minSupportedDateMillis)
+        {
+            throw new MarshalException(String.format("Input date %s is less than min supported date %s",
+                                                     null == source ? ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC).toLocalDate() : source,
+                                                     ZonedDateTime.ofInstant(Instant.ofEpochMilli(minSupportedDateMillis), UTC).toLocalDate()));
+        }
+        if (millis > maxSupportedDateMillis)
+        {
+            throw new MarshalException(String.format("Input date %s is greater than max supported date %s",
+                                                     null == source ? ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC).toLocalDate() : source,
+                                                     ZonedDateTime.ofInstant(Instant.ofEpochMilli(maxSupportedDateMillis), UTC).toLocalDate()));
+        }
         return (int) (Duration.ofMillis(millis).toDays() - Integer.MIN_VALUE);
     }
 
diff --git a/src/java/org/apache/cassandra/serializers/TupleSerializer.java b/src/java/org/apache/cassandra/serializers/TupleSerializer.java
index afdf2484db12..0813a322ec72 100644
--- a/src/java/org/apache/cassandra/serializers/TupleSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/TupleSerializer.java
@@ -19,16 +19,18 @@
 
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
+
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.marshal.ValueAccessor;
 
 public class TupleSerializer extends BytesSerializer
 {
-    public final List<TypeSerializer<?>> fields;
+    public final ImmutableList<TypeSerializer<?>> fields;
 
     public TupleSerializer(List<TypeSerializer<?>> fields)
     {
-        this.fields = fields;
+        this.fields = ImmutableList.copyOf(fields);
     }
 
     public <V> void validate(V input, ValueAccessor<V> accessor) throws MarshalException
diff --git a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
index 50c51b168fdf..83ee37cb31db 100644
--- a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.service;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -27,6 +28,7 @@
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,6 +46,8 @@
 import org.apache.cassandra.locator.ReplicaPlan.ForWrite;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.concurrent.Condition;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
@@ -73,13 +77,16 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
 
     protected final Runnable callback;
     protected final WriteType writeType;
-    private static final AtomicIntegerFieldUpdater<AbstractWriteResponseHandler> failuresUpdater =
+    protected static final AtomicIntegerFieldUpdater<AbstractWriteResponseHandler> failuresUpdater =
         AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "failures");
     private volatile int failures = 0;
-    private final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
+    // Used by CNDB
+    protected final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
     private final Dispatcher.RequestTime requestTime;
     private @Nullable final Supplier<Mutation> hintOnFailure;
 
+    private final RequestSensors requestSensors;
+
     /**
       * Delegate to another WriteResponseHandler or possibly this one to track if the ideal consistency level was reached.
       * Will be set to null if ideal CL was not configured
@@ -107,21 +114,22 @@ protected AbstractWriteResponseHandler(ForWrite replicaPlan, Runnable callback,
         this.hintOnFailure = hintOnFailure;
         this.failureReasonByEndpoint = new ConcurrentHashMap<>();
         this.requestTime = requestTime;
+        this.requestSensors = RequestTracker.instance.get();
     }
 
-    public void get() throws WriteTimeoutException, WriteFailureException
+    public int failures()
     {
-        long timeoutNanos = currentTimeoutNanos();
+        return failures;
+    }
 
-        boolean signaled;
-        try
-        {
-            signaled = condition.await(timeoutNanos, NANOSECONDS);
-        }
-        catch (InterruptedException e)
-        {
-            throw new UncheckedInterruptedException(e);
-        }
+    public Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint()
+    {
+        return Collections.unmodifiableMap(failureReasonByEndpoint);
+    }
+
+    public void get() throws WriteTimeoutException, WriteFailureException
+    {
+        boolean signaled = await();
 
         if (!signaled)
             throwTimeout();
@@ -137,6 +145,19 @@ public void get() throws WriteTimeoutException, WriteFailureException
         }
     }
 
+    public boolean await() throws UncheckedInterruptedException
+    {
+        long timeoutNanos = currentTimeoutNanos();
+        try
+        {
+            return condition.await(timeoutNanos, NANOSECONDS);
+        }
+        catch (InterruptedException e)
+        {
+            throw new UncheckedInterruptedException(e);
+        }
+    }
+
     private void throwTimeout()
     {
         int blockedFor = blockFor();
@@ -149,7 +170,7 @@ private void throwTimeout()
         throw new WriteTimeoutException(writeType, replicaPlan.consistencyLevel(), acks, blockedFor);
     }
 
-    public final long currentTimeoutNanos()
+    public long currentTimeoutNanos()
     {
         long now = nanoTime();
         long requestTimeout = writeType == COUNTER
@@ -158,6 +179,27 @@ public final long currentTimeoutNanos()
         return requestTime.computeTimeout(now, requestTimeout);
     }
 
+    public ReplicaPlan.ForWrite replicaPlan()
+    {
+        return replicaPlan;
+    }
+
+    public WriteType writeType()
+    {
+        return writeType;
+    }
+
+    public Dispatcher.RequestTime requestTime()
+    {
+        return requestTime;
+    }
+    
+    // Used by CNDB
+    public Supplier<Mutation> hintOnFailure()
+    {
+        return hintOnFailure;
+    }
+
     /**
      * Set a delegate ideal CL write response handler. Note that this could be the same as this
      * if the ideal CL and requested CL are the same.
@@ -224,7 +266,7 @@ public final void expired()
     /**
      * @return the minimum number of endpoints that must respond.
      */
-    protected int blockFor()
+    public int blockFor()
     {
         // During bootstrap, we have to include the pending endpoints or we may fail the consistency level
         // guarantees (see #833)
@@ -236,7 +278,7 @@ protected int blockFor()
      *       this needs to be aware of which nodes are live/down
      * @return the total number of endpoints the request can send to.
      */
-    protected int candidateReplicaCount()
+    public int candidateReplicaCount()
     {
         if (replicaPlan.consistencyLevel().isDatacenterLocal())
             return countInOurDc(replicaPlan.liveAndDown()).allReplicas();
@@ -252,7 +294,7 @@ public ConsistencyLevel consistencyLevel()
     /**
      * @return true if the message counts towards the blockFor() threshold
      */
-    protected boolean waitingFor(InetAddressAndPort from)
+    public boolean waitingFor(InetAddressAndPort from)
     {
         return true;
     }
@@ -260,14 +302,14 @@ protected boolean waitingFor(InetAddressAndPort from)
     /**
      * @return number of responses received
      */
-    protected abstract int ackCount();
+    public abstract int ackCount();
 
     /**
      * null message means "response from local write"
      */
     public abstract void onResponse(Message<T> msg);
 
-    protected void signal()
+    public void signal()
     {
         //The ideal CL should only count as a strike if the requested CL was achieved.
         //If the requested CL is not achieved it's fine for the ideal CL to also not be achieved.
@@ -285,6 +327,23 @@ protected void signal()
             callback.run();
     }
 
+    /**
+     * @return true if condition is signaled either for success or failure
+     */
+    @VisibleForTesting
+    public boolean isCompleted()
+    {
+        return condition.isSignalled();
+    }
+
+    /**
+     * @return true if condition is signaled for failure
+     */
+    public boolean isCompletedExceptionally()
+    {
+        return isCompleted() && blockFor() + failures > candidateReplicaCount();
+    }
+
     @Override
     public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
     {
@@ -311,6 +370,12 @@ public boolean invokeOnFailure()
         return true;
     }
 
+    @Override
+    public RequestSensors getRequestSensors()
+    {
+        return requestSensors;
+    }
+
     /**
      * Decrement the counter for all responses/expirations and if the counter
      * hits 0 check to see if the ideal consistency level (this write response handler)
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index e120122c0844..7ad2c64d4ad4 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -86,6 +86,7 @@
 import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.service.paxos.PaxosRepair;
 import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup;
+import org.apache.cassandra.repair.ParentRepairSessionListener;
 import org.apache.cassandra.repair.RepairJobDesc;
 import org.apache.cassandra.repair.RepairParallelism;
 import org.apache.cassandra.repair.RepairSession;
@@ -124,6 +125,8 @@
 import static org.apache.cassandra.net.Verb.PREPARE_MSG;
 import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
 import static org.apache.cassandra.utils.Simulate.With.MONITORS;
+import static org.apache.cassandra.net.Verb.SYNC_RSP;
+import static org.apache.cassandra.net.Verb.VALIDATION_RSP;
 
 /**
  * ActiveRepairService is the starting point for manual "active" repairs.
@@ -440,6 +443,7 @@ public RepairSession submitRepairSession(TimeUUID parentRepairSession,
                                              String keyspace,
                                              RepairParallelism parallelismDegree,
                                              boolean isIncremental,
+                                             boolean pushRepair,
                                              boolean pullRepair,
                                              PreviewKind previewKind,
                                              boolean optimiseStreams,
@@ -459,7 +463,7 @@ public RepairSession submitRepairSession(TimeUUID parentRepairSession,
             return null;
 
         final RepairSession session = new RepairSession(ctx, validationScheduler, parentRepairSession, range, keyspace,
-                                                        parallelismDegree, isIncremental, pullRepair,
+                                                        parallelismDegree, isIncremental, pushRepair, pullRepair,
                                                         previewKind, optimiseStreams, repairPaxos, paxosOnly, cfnames);
         repairs.getIfPresent(parentRepairSession).register(session.state);
 
@@ -517,7 +521,11 @@ public synchronized void terminateSessions()
         {
             session.forceShutdown(cause);
         }
+        Collection<Map.Entry<TimeUUID, ParentRepairSession>> sessions = new ArrayList<>(parentRepairSessions.entrySet());
         parentRepairSessions.clear();
+
+        for (Map.Entry<TimeUUID, ParentRepairSession> e : sessions)
+            ParentRepairSessionListener.instance.onRemoved(e.getKey(), e.getValue());
     }
 
     public void recordRepairStatus(int cmd, ParentRepairStatus parentRepairStatus, List<String> messages)
@@ -684,7 +692,7 @@ public Future<?> prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPo
             }
         }
         // implement timeout to bound the runtime of the future
-        long timeoutMillis = getRepairRetrySpec().isEnabled() ? getRepairRpcTimeout(MILLISECONDS)
+        long timeoutMillis = getRepairRetrySpec().isEnabled() ? getRepairPrepareMessageTimeout(MILLISECONDS)
                                                               : getRpcTimeout(MILLISECONDS);
         ctx.optionalTasks().schedule(() -> {
             if (promise.isDone())
@@ -820,7 +828,9 @@ public synchronized void registerParentRepairSession(TimeUUID parentRepairSessio
 
         if (!parentRepairSessions.containsKey(parentRepairSession))
         {
-            parentRepairSessions.put(parentRepairSession, new ParentRepairSession(coordinator, columnFamilyStores, ranges, isIncremental, repairedAt, isGlobal, previewKind));
+            ParentRepairSession session = new ParentRepairSession(coordinator, columnFamilyStores, ranges, isIncremental, repairedAt, isGlobal, previewKind);
+            parentRepairSessions.put(parentRepairSession, session);
+            ParentRepairSessionListener.instance.onRegistered(parentRepairSession, session);
         }
     }
 
@@ -858,6 +868,8 @@ public synchronized ParentRepairSession removeParentRepairSession(TimeUUID paren
             return null;
 
         String snapshotName = parentSessionId.toString();
+        ParentRepairSessionListener.instance.onRemoved(parentSessionId, session);
+
         if (session.hasSnapshots.get())
         {
             snapshotExecutor.submit(() -> {
@@ -885,12 +897,9 @@ public void handleMessage(Message<? extends RepairMessage> message)
 
         if (session == null)
         {
-            switch (message.verb())
+            if (message.verb() == VALIDATION_RSP || message.verb() == SYNC_RSP)
             {
-                case VALIDATION_RSP:
-                case SYNC_RSP:
                     ctx.messaging().send(message.emptyResponse(), message.from());
-                    break;
             }
             if (payload instanceof ValidationResponse)
             {
@@ -906,16 +915,13 @@ public void handleMessage(Message<? extends RepairMessage> message)
             return;
         }
 
-        switch (message.verb())
+        if (message.verb() == VALIDATION_RSP)
         {
-            case VALIDATION_RSP:
-                session.validationComplete(desc, (Message<ValidationResponse>) message);
-                break;
-            case SYNC_RSP:
-                session.syncComplete(desc, (Message<SyncResponse>) message);
-                break;
-            default:
-                break;
+            session.validationComplete(desc, (Message<ValidationResponse>) message);
+        }
+        else if (message.verb() == SYNC_RSP)
+        {
+            session.syncComplete(desc, (Message<SyncResponse>) message);
         }
     }
 
diff --git a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java
index 0fa284770080..2982f2ebeaca 100644
--- a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java
@@ -43,7 +43,7 @@ public BatchlogResponseHandler(AbstractWriteResponseHandler<T> wrapped, int requ
         this.cleanup = cleanup;
     }
 
-    protected int ackCount()
+    public int ackCount()
     {
         return wrapped.ackCount();
     }
@@ -70,22 +70,22 @@ public void get() throws WriteTimeoutException, WriteFailureException
         wrapped.get();
     }
 
-    protected int blockFor()
+    public int blockFor()
     {
         return wrapped.blockFor();
     }
 
-    protected int candidateReplicaCount()
+    public int candidateReplicaCount()
     {
         return wrapped.candidateReplicaCount();
     }
 
-    protected boolean waitingFor(InetAddressAndPort from)
+    public boolean waitingFor(InetAddressAndPort from)
     {
         return wrapped.waitingFor(from);
     }
 
-    protected void signal()
+    public void signal()
     {
         wrapped.signal();
     }
diff --git a/src/java/org/apache/cassandra/service/CacheService.java b/src/java/org/apache/cassandra/service/CacheService.java
index 8240c2880f78..6c2a35dd5494 100644
--- a/src/java/org/apache/cassandra/service/CacheService.java
+++ b/src/java/org/apache/cassandra/service/CacheService.java
@@ -28,6 +28,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.tuple.ImmutableTriple;
 import org.slf4j.Logger;
@@ -47,6 +48,7 @@
 import org.apache.cassandra.db.ClockAndCount;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.context.CounterContext;
@@ -81,15 +83,22 @@ public class CacheService implements CacheServiceMBean
 
     public enum CacheType
     {
-        KEY_CACHE("KeyCache"),
-        ROW_CACHE("RowCache"),
-        COUNTER_CACHE("CounterCache");
+        KEY_CACHE("KeyCache", "key_cache"),
+        ROW_CACHE("RowCache", "row_cache"),
+        COUNTER_CACHE("CounterCache", "counter_cache");
 
         private final String name;
+        private final String micrometerMetricsPrefix;
 
-        CacheType(String typeName)
+        CacheType(String typeName, String micrometerMetricsPrefix)
         {
-            name = typeName;
+            this.name = typeName;
+            this.micrometerMetricsPrefix = micrometerMetricsPrefix;
+        }
+
+        public String micrometerMetricsPrefix()
+        {
+            return micrometerMetricsPrefix;
         }
 
         public String toString()
@@ -118,7 +127,7 @@ private CacheService()
      */
     private AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> initKeyCache()
     {
-        logger.info("Initializing key cache with capacity of {} MiBs.", DatabaseDescriptor.getKeyCacheSizeInMiB());
+        logger.debug("Initializing key cache with capacity of {} MiBs.", DatabaseDescriptor.getKeyCacheSizeInMiB());
 
         long keyCacheInMemoryCapacity = DatabaseDescriptor.getKeyCacheSizeInMiB() * 1024 * 1024;
 
@@ -126,7 +135,15 @@ private AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> initKeyCache()
         // where 48 = 40 bytes (average size of the key) + 8 bytes (size of value)
         ICache<KeyCacheKey, AbstractRowIndexEntry> kc;
         kc = CaffeineCache.create(keyCacheInMemoryCapacity);
-        AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache = new AutoSavingCache<>(kc, CacheType.KEY_CACHE, new KeyCacheSerializer());
+
+        AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache = new AutoSavingCache<>(kc, CacheType.KEY_CACHE, new KeyCacheSerializer(), () -> {
+            Set<Descriptor> liveDescriptors = Keyspace.allExisting()
+                                                      .flatMap(keyspace -> keyspace.getColumnFamilyStores().stream()
+                                                                                   .flatMap(cfs -> cfs.getLiveSSTables().stream()
+                                                                                                      .map(SSTableReader::getDescriptor)))
+                                                      .collect(Collectors.toSet());
+            return key -> liveDescriptors.contains(key.desc);
+        });
 
         int keyCacheKeysToSave = DatabaseDescriptor.getKeyCacheKeysToSave();
 
@@ -140,7 +157,7 @@ private AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> initKeyCache()
      */
     private AutoSavingCache<RowCacheKey, IRowCacheEntry> initRowCache()
     {
-        logger.info("Initializing row cache with capacity of {} MiBs", DatabaseDescriptor.getRowCacheSizeInMiB());
+        logger.debug("Initializing row cache with capacity of {} MiBs", DatabaseDescriptor.getRowCacheSizeInMiB());
 
         CacheProvider<RowCacheKey, IRowCacheEntry> cacheProvider;
         String cacheProviderClassName = DatabaseDescriptor.getRowCacheSizeInMiB() > 0
@@ -158,7 +175,7 @@ private AutoSavingCache<RowCacheKey, IRowCacheEntry> initRowCache()
 
         // cache object
         ICache<RowCacheKey, IRowCacheEntry> rc = cacheProvider.create();
-        AutoSavingCache<RowCacheKey, IRowCacheEntry> rowCache = new AutoSavingCache<>(rc, CacheType.ROW_CACHE, new RowCacheSerializer());
+        AutoSavingCache<RowCacheKey, IRowCacheEntry> rowCache = new AutoSavingCache<>(rc, CacheType.ROW_CACHE, new RowCacheSerializer(), null);
 
         int rowCacheKeysToSave = DatabaseDescriptor.getRowCacheKeysToSave();
 
@@ -169,27 +186,27 @@ private AutoSavingCache<RowCacheKey, IRowCacheEntry> initRowCache()
 
     private AutoSavingCache<CounterCacheKey, ClockAndCount> initCounterCache()
     {
-        logger.info("Initializing counter cache with capacity of {} MiBs", DatabaseDescriptor.getCounterCacheSizeInMiB());
+        logger.debug("Initializing counter cache with capacity of {} MiBs", DatabaseDescriptor.getCounterCacheSizeInMiB());
 
         long capacity = DatabaseDescriptor.getCounterCacheSizeInMiB() * 1024 * 1024;
 
         AutoSavingCache<CounterCacheKey, ClockAndCount> cache =
             new AutoSavingCache<>(CaffeineCache.create(capacity),
                                   CacheType.COUNTER_CACHE,
-                                  new CounterCacheSerializer());
+                                  new CounterCacheSerializer(),
+                                  null);
 
         int keysToSave = DatabaseDescriptor.getCounterCacheKeysToSave();
 
-        logger.info("Scheduling counter cache save to every {} seconds (going to save {} keys).",
-                    DatabaseDescriptor.getCounterCacheSavePeriod(),
-                    keysToSave == Integer.MAX_VALUE ? "all" : keysToSave);
+        logger.debug("Scheduling counter cache save to every {} seconds (going to save {} keys).",
+                     DatabaseDescriptor.getCounterCacheSavePeriod(),
+                     keysToSave == Integer.MAX_VALUE ? "all" : keysToSave);
 
         cache.scheduleSaving(DatabaseDescriptor.getCounterCacheSavePeriod(), keysToSave);
 
         return cache;
     }
 
-
     public int getRowCacheSavePeriodInSeconds()
     {
         return DatabaseDescriptor.getRowCacheSavePeriod();
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 4fc4010e85b4..a6c0d4102a62 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -22,12 +22,10 @@
 import java.lang.management.MemoryPoolMXBean;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.nio.file.Files;
-import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
 import javax.management.ObjectName;
 import javax.management.StandardMBean;
 import javax.management.remote.JMXConnectorServer;
@@ -63,6 +61,7 @@
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.metrics.DefaultNameFactory;
 import org.apache.cassandra.net.StartupClusterConnectivityChecker;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
@@ -74,11 +73,11 @@
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MBeanWrapper;
 import org.apache.cassandra.utils.Mx4jTool;
-import org.apache.cassandra.utils.NativeLibrary;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.FutureCombiner;
 import org.apache.cassandra.utils.logging.LoggingSupportFactory;
 import org.apache.cassandra.utils.logging.VirtualTableAppender;
+import org.apache.cassandra.utils.INativeLibrary;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_FOREGROUND;
@@ -251,7 +250,7 @@ protected void setup()
 
         logSystemInfo(logger);
 
-        NativeLibrary.tryMlockall();
+        INativeLibrary.instance.tryMlockall();
 
         CommitLog.instance.start();
 
@@ -342,13 +341,15 @@ protected void setup()
         // replay the log if necessary
         try
         {
-            CommitLog.instance.recoverSegmentsOnDisk();
+            CommitLog.instance.recoverSegmentsOnDisk(ColumnFamilyStore.FlushReason.STARTUP);
         }
         catch (IOException e)
         {
             throw new RuntimeException(e);
         }
 
+        Nodes.getInstance().reload();
+
         // Re-populate token metadata after commit log recover (new peers might be loaded onto system keyspace #10293)
         StorageService.instance.populateTokenMetadata();
 
@@ -413,7 +414,7 @@ protected void setup()
                 for (final ColumnFamilyStore store : cfs.concatWithIndexes())
                 {
                     store.reload(); //reload CFs in case there was a change of disk boundaries
-                    if (store.getCompactionStrategyManager().shouldBeEnabled())
+                    if (store.compactionShouldBeEnabled())
                     {
                         if (DatabaseDescriptor.getAutocompactionOnStartupEnabled())
                         {
@@ -487,45 +488,41 @@ public void migrateSystemDataIfNeeded() throws IOException
         //     the system keyspace location configured by the user (upgrade to 4.0)
         //  3) The system data are stored in the first data location and need to be moved to
         //     the system keyspace location configured by the user (system_data_file_directory has been configured)
-        Path target = File.getPath(DatabaseDescriptor.getLocalSystemKeyspacesDataFileLocations()[0]);
+        File target = DatabaseDescriptor.getLocalSystemKeyspacesDataFileLocations()[0];
 
-        String[] nonLocalSystemKeyspacesFileLocations = DatabaseDescriptor.getNonLocalSystemKeyspacesDataFileLocations();
-        String[] sources = DatabaseDescriptor.useSpecificLocationForLocalSystemData() ? nonLocalSystemKeyspacesFileLocations
+        File[] nonLocalSystemKeyspacesFileLocations = DatabaseDescriptor.getNonLocalSystemKeyspacesDataFileLocations();
+        File[] sources = DatabaseDescriptor.useSpecificLocationForLocalSystemData() ? nonLocalSystemKeyspacesFileLocations
                                                                                       : Arrays.copyOfRange(nonLocalSystemKeyspacesFileLocations,
                                                                                                            1,
                                                                                                            nonLocalSystemKeyspacesFileLocations.length);
 
-        for (String source : sources)
+        for (File dataFileLocation : sources)
         {
-            Path dataFileLocation = File.getPath(source);
-
-            if (!Files.exists(dataFileLocation))
+            if (!dataFileLocation.exists())
                 continue;
 
-            try (Stream<Path> locationChildren = Files.list(dataFileLocation))
+            List<File> keyspaceDirectories = new ArrayList<>();
+            dataFileLocation.forEach(f -> {
+                if (SchemaConstants.isLocalSystemKeyspace(f.name()))
+                    keyspaceDirectories.add(f);
+            });
+
+            for (File keyspaceDirectory : keyspaceDirectories)
             {
-                Path[] keyspaceDirectories = locationChildren.filter(p -> SchemaConstants.isLocalSystemKeyspace(p.getFileName().toString()))
-                                                             .toArray(Path[]::new);
+                List<File> tableDirectories = new ArrayList<>();
+                keyspaceDirectory.forEach(f -> {
+                    if (f.isDirectory() && SystemKeyspace.TABLES_SPLIT_ACROSS_MULTIPLE_DISKS.stream().noneMatch(t -> f.name().startsWith(t + '-')))
+                        tableDirectories.add(f);
+                });
 
-                for (Path keyspaceDirectory : keyspaceDirectories)
+                for (File tableDirectory : tableDirectories)
                 {
-                    try (Stream<Path> keyspaceChildren = Files.list(keyspaceDirectory))
-                    {
-                        Path[] tableDirectories = keyspaceChildren.filter(Files::isDirectory)
-                                                                  .filter(p -> SystemKeyspace.TABLES_SPLIT_ACROSS_MULTIPLE_DISKS.stream().noneMatch(t -> p.getFileName().toString().startsWith(t + '-')))
-                                                                  .toArray(Path[]::new);
-
-                        for (Path tableDirectory : tableDirectories)
-                        {
-                            FileUtils.moveRecursively(tableDirectory,
-                                                      target.resolve(dataFileLocation.relativize(tableDirectory)));
-                        }
+                    FileUtils.moveRecursively(tableDirectory, target.resolve(dataFileLocation.relativize(tableDirectory)));
+                }
 
-                        if (!SchemaConstants.SYSTEM_KEYSPACE_NAME.equals(keyspaceDirectory.getFileName().toString()))
-                        {
-                            FileUtils.deleteDirectoryIfEmpty(keyspaceDirectory);
-                        }
-                    }
+                if (!SchemaConstants.SYSTEM_KEYSPACE_NAME.equals(keyspaceDirectory.name()))
+                {
+                    FileUtils.deleteDirectoryIfEmpty(keyspaceDirectory);
                 }
             }
         }
@@ -612,11 +609,11 @@ public static void logSystemInfo(Logger logger)
                         FBUtilities.prettyPrintMemory(Runtime.getRuntime().maxMemory()));
 
 	        for(MemoryPoolMXBean pool: ManagementFactory.getMemoryPoolMXBeans())
-	            logger.info("{} {}: {}", pool.getName(), pool.getType(), pool.getPeakUsage());
+	            logger.debug("{} {}: {}", pool.getName(), pool.getType(), pool.getPeakUsage());
 
-	        logger.info("Classpath: {}", JAVA_CLASS_PATH.getString());
+	        logger.debug("Classpath: {}", JAVA_CLASS_PATH.getString());
 
-            logger.info("JVM Arguments: {}", ManagementFactory.getRuntimeMXBean().getInputArguments());
+            logger.debug("JVM Arguments: {}", ManagementFactory.getRuntimeMXBean().getInputArguments());
     	}
     }
 
@@ -893,12 +890,12 @@ static class NativeAccess implements NativeAccessMBean
     {
         public boolean isAvailable()
         {
-            return NativeLibrary.isAvailable();
+            return INativeLibrary.instance.isAvailable();
         }
 
         public boolean isMemoryLockable()
         {
-            return NativeLibrary.jnaMemoryLockable();
+            return INativeLibrary.instance.jnaMemoryLockable();
         }
     }
 
diff --git a/src/java/org/apache/cassandra/service/ClientState.java b/src/java/org/apache/cassandra/service/ClientState.java
index d9a0dbbc9e6b..d7ab77f5f1e2 100644
--- a/src/java/org/apache/cassandra/service/ClientState.java
+++ b/src/java/org/apache/cassandra/service/ClientState.java
@@ -58,7 +58,7 @@
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MD5Digest;
 
-import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_QUERY_HANDLER_CLASS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.*;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 
 /**
@@ -176,6 +176,13 @@ protected ClientState(ClientState source)
         this.clientOptions = source.clientOptions;
     }
 
+    private ClientState(AuthenticatedUser user)
+    {
+        this.isInternal = false;
+        this.remoteAddress = null;
+        this.user = user;
+    }
+
     /**
      * @return a ClientState object for internal C* calls (not limited by any kind of auth).
      */
@@ -199,6 +206,14 @@ public static ClientState forExternalCalls(SocketAddress remoteAddress)
         return new ClientState((InetSocketAddress)remoteAddress);
     }
 
+    /**
+     * @return a ClientState object for internal calls with the given user logged in (not limited by any kind of auth).
+     */
+    public static ClientState forExternalCalls(AuthenticatedUser user)
+    {
+        return new ClientState(user);
+    }
+
     /**
      * Clone this ClientState object, but use the provided keyspace instead of the
      * keyspace in this ClientState object.
@@ -555,17 +570,28 @@ public void ensureNotAnonymous()
      */
     public boolean isOrdinaryUser()
     {
+        if (ENABLE_GUARDRAILS_FOR_ANONYMOUS_USER.getBoolean())
+            return !isSuperIgnoreAnonymousUser() && !isSystem();
         return !isSuper() && !isSystem();
     }
 
     /**
-     * Checks if this user is a super user.
+     * Checks if this user is a super user. When authentication is disabled the anonymous user is considered
+     * a super user.
      */
     public boolean isSuper()
     {
         return !DatabaseDescriptor.getAuthenticator().requireAuthentication() || (user != null && user.isSuper());
     }
 
+    /**
+     * Checks if this user is a super user. An anonymous user is never considered a super user.
+     */
+    public boolean isSuperIgnoreAnonymousUser()
+    {
+        return user != null && user.isSuper();
+    }
+
     /**
      * Checks if the user is the system user.
      *
diff --git a/src/java/org/apache/cassandra/service/ClientWarn.java b/src/java/org/apache/cassandra/service/ClientWarn.java
index 7f67a1168a30..84e4f37de96d 100644
--- a/src/java/org/apache/cassandra/service/ClientWarn.java
+++ b/src/java/org/apache/cassandra/service/ClientWarn.java
@@ -18,7 +18,9 @@
 package org.apache.cassandra.service;
 
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.cassandra.concurrent.ExecutorLocals;
 import org.apache.cassandra.utils.FBUtilities;
@@ -40,14 +42,22 @@ public State get()
     public void set(State value)
     {
         ExecutorLocals current = ExecutorLocals.current();
-        ExecutorLocals.Impl.set(current.traceState, value);
+        ExecutorLocals.Impl.set(current.traceState, value, current.sensors, current.operationContext);
     }
 
     public void warn(String text)
+    {
+        warn(text, null);
+    }
+
+    /**
+     * Issue the given warning if this is the first time `key` is seen.
+     */
+    public void warn(String text, Object key)
     {
         State state = get();
         if (state != null)
-            state.add(text);
+            state.add(text, key);
     }
 
     public void captureWarnings()
@@ -73,11 +83,16 @@ public static class State
         // This must be a thread-safe list. Even though it's wrapped in a ThreadLocal, it's propagated to each thread
         // from shared state, so multiple threads can reference the same State.
         private final List<String> warnings = new CopyOnWriteArrayList<>();
+        private final Set<Object> keysAdded = new HashSet<>();
 
-        private void add(String warning)
+        private void add(String warning, Object key)
         {
             if (warnings.size() < FBUtilities.MAX_UNSIGNED_SHORT)
+            {
+                if (key != null && !keysAdded.add(key))
+                    return;
                 warnings.add(maybeTruncate(warning));
+            }
         }
 
         private static String maybeTruncate(String warning)
diff --git a/src/java/org/apache/cassandra/service/DataResurrectionCheck.java b/src/java/org/apache/cassandra/service/DataResurrectionCheck.java
index 4cf32781100d..db85d75bc1e2 100644
--- a/src/java/org/apache/cassandra/service/DataResurrectionCheck.java
+++ b/src/java/org/apache/cassandra/service/DataResurrectionCheck.java
@@ -134,7 +134,7 @@ static File getHeartbeatFile(Map<String, Object> config)
         }
         else
         {
-            String[] dataFileLocations = DatabaseDescriptor.getLocalSystemKeyspacesDataFileLocations();
+            File[] dataFileLocations = DatabaseDescriptor.getLocalSystemKeyspacesDataFileLocations();
             assert dataFileLocations.length != 0;
             heartbeatFile = new File(dataFileLocations[0], DEFAULT_HEARTBEAT_FILE);
         }
diff --git a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java
index e4b208b582fb..7a513a5af7cb 100644
--- a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java
@@ -23,14 +23,14 @@
 import java.util.function.Supplier;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.WriteType;
 import org.apache.cassandra.locator.IEndpointSnitch;
 import org.apache.cassandra.locator.NetworkTopologyStrategy;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.WriteType;
 import org.apache.cassandra.transport.Dispatcher;
 
 /**
@@ -102,7 +102,7 @@ public void onResponse(Message<T> message)
         }
     }
 
-    protected int ackCount()
+    public int ackCount()
     {
         return acks.get();
     }
diff --git a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java
index f33d6607e1c2..29b170052597 100644
--- a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java
@@ -17,6 +17,9 @@
  */
 package org.apache.cassandra.service;
 
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.WriteType;
 import org.apache.cassandra.locator.InOurDc;
@@ -25,9 +28,6 @@
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.transport.Dispatcher;
 
-import java.util.function.Predicate;
-import java.util.function.Supplier;
-
 /**
  * This class blocks for a quorum of responses _in the local datacenter only_ (CL.LOCAL_QUORUM).
  */
@@ -61,7 +61,7 @@ public void onResponse(Message<T> message)
     }
 
     @Override
-    protected boolean waitingFor(InetAddressAndPort from)
+    public boolean waitingFor(InetAddressAndPort from)
     {
         return waitingFor.test(from);
     }
diff --git a/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java b/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java
index 8b182942b23e..34b90326f0e4 100644
--- a/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java
+++ b/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java
@@ -54,6 +54,7 @@ public void handleCorruptSSTable(CorruptSSTableException e)
                 logger.error("Stopping transports as disk_failure_policy is " + DatabaseDescriptor.getDiskFailurePolicy());
                 StorageService.instance.stopTransports();
                 break;
+
         }
     }
 
@@ -86,10 +87,10 @@ public void handleFSError(FSError e)
                 }
 
                 // for both read and write errors mark the path as unwritable.
-                DisallowedDirectories.maybeMarkUnwritable(new File(e.path));
+                DisallowedDirectories.maybeMarkUnwritable(e.file);
                 if (e instanceof FSReadError && shouldMaybeRemoveData(e))
                 {
-                    File directory = DisallowedDirectories.maybeMarkUnreadable(new File(e.path));
+                    File directory = DisallowedDirectories.maybeMarkUnreadable(e.file);
                     if (directory != null)
                         Keyspace.removeUnreadableSSTables(directory);
                 }
diff --git a/src/java/org/apache/cassandra/service/FileSystemOwnershipCheck.java b/src/java/org/apache/cassandra/service/FileSystemOwnershipCheck.java
index 3d69c9e7631c..fa58ee4a289c 100644
--- a/src/java/org/apache/cassandra/service/FileSystemOwnershipCheck.java
+++ b/src/java/org/apache/cassandra/service/FileSystemOwnershipCheck.java
@@ -95,18 +95,18 @@ public class FileSystemOwnershipCheck implements StartupCheck
     static final String INVALID_PROPERTY_VALUE                  = "invalid or missing value for property '%s'";
     static final String READ_EXCEPTION                          = "error when checking for fs ownership file";
 
-    private final Supplier<Iterable<String>> dirs;
+    private final Supplier<Iterable<File>> dirs;
 
     FileSystemOwnershipCheck()
     {
         this(() -> Iterables.concat(Arrays.asList(DatabaseDescriptor.getAllDataFileLocations()),
                                     Arrays.asList(DatabaseDescriptor.getCommitLogLocation(),
                                                   DatabaseDescriptor.getSavedCachesLocation(),
-                                                  DatabaseDescriptor.getHintsDirectory().absolutePath())));
+                                                  DatabaseDescriptor.getHintsDirectory())));
     }
 
     @VisibleForTesting
-    FileSystemOwnershipCheck(Supplier<Iterable<String>> dirs)
+    FileSystemOwnershipCheck(Supplier<Iterable<File>> dirs)
     {
         this.dirs = dirs;
     }
@@ -134,11 +134,11 @@ public void execute(StartupChecksOptions options) throws StartupException
         Map<Path, Properties> foundProperties = new HashMap<>();
 
         // Step 1: Traverse the filesystem from each target dir upward, looking for marker files
-        for (String dataDir : dirs.get())
+        for (File dataDir : dirs.get())
         {
             logger.info("Checking for fs ownership details in file hierarchy for {}", dataDir);
             int foundFiles = 0;
-            Path dir = File.getPath(dataDir).normalize();
+            Path dir = dataDir.toPath().normalize();
             do
             {
                 File tokenFile = resolve(dir, tokenFilename);
@@ -163,7 +163,7 @@ public void execute(StartupChecksOptions options) throws StartupException
                 dir = dir.getParent();
             } while (dir != null);
 
-            foundPerTargetDir.put(dataDir, foundFiles);
+            foundPerTargetDir.put(dataDir.toString(), foundFiles);
         }
 
         // If a marker file couldn't be found for every target directory, error.
diff --git a/src/java/org/apache/cassandra/service/Mutator.java b/src/java/org/apache/cassandra/service/Mutator.java
new file mode 100644
index 000000000000..2cad36402bdd
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/Mutator.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.util.Collection;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.exceptions.OverloadedException;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * Facilitates mutations for counters, simple inserts, unlogged batches and LWTs.
+ * Used on the coordinator.
+ * <br/>
+ * The implementations may choose how and where to send the mutations.
+ * <br/>
+ * An instance of this interface implementation must be obtained via {@link MutatorProvider#instance}.
+ */
+public interface Mutator
+{
+
+    /**
+     * Used for handling the given {@code mutations} as a logged batch.
+     */
+    void mutateAtomically(Collection<Mutation> mutations,
+                          ConsistencyLevel consistencyLevel,
+                          boolean requireQuorumForRemove,
+                          Dispatcher.RequestTime requestTime,
+                          ClientRequestsMetrics metrics,
+                          ClientState clientState)
+    throws UnavailableException, OverloadedException, WriteTimeoutException;
+
+    /**
+     * Used for handling counter mutations on the coordinator level:
+     * - if coordinator is a replica, it will apply the counter mutation locally and forward the applied mutation to other counter replica
+     * - if coordinator is not a replica, it will forward the counter mutation to a counter leader which is a replica
+     */
+    AbstractWriteResponseHandler<IMutation> mutateCounter(CounterMutation cm, String localDataCenter, Dispatcher.RequestTime requestTime);
+
+    /**
+     * Used for handling counter mutations on the counter leader level
+     */
+    AbstractWriteResponseHandler<IMutation> mutateCounterOnLeader(CounterMutation mutation,
+                                                                  String localDataCenter,
+                                                                  StorageProxy.WritePerformer performer,
+                                                                  Runnable callback,
+                                                                  Dispatcher.RequestTime requestTime);
+
+    /**
+     * Used for standard inserts and unlogged batchs.
+     */
+    AbstractWriteResponseHandler<IMutation> mutateStandard(Mutation mutation,
+                                                           ConsistencyLevel consistencyLevel,
+                                                           String localDataCenter,
+                                                           StorageProxy.WritePerformer writePerformer,
+                                                           Runnable callback,
+                                                           WriteType writeType,
+                                                           Dispatcher.RequestTime requestTime);
+
+    /**
+     * Used for LWT mutation at the last (COMMIT) phase of Paxos.
+     */
+    @Nullable
+    AbstractWriteResponseHandler<Commit> mutatePaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, Dispatcher.RequestTime requestTime);
+
+    /**
+     * Used to persist the given batch of mutations. Usually invoked as part of
+     * {@link #mutateAtomically(Collection, ConsistencyLevel, boolean, long, ClientRequestsMetrics, ClientState)}.
+     */
+    void persistBatchlog(Collection<Mutation> mutations, Dispatcher.RequestTime requestTime, ReplicaPlan.ForWrite replicaPlan, TimeUUID batchUUID);
+
+    /**
+     * Used to clear the given batch id. Usually invoked as part of
+     * {@link #mutateAtomically(Collection, ConsistencyLevel, boolean, long, ClientRequestsMetrics, ClientState)}.
+     */
+    void clearBatchlog(String keyspace, Dispatcher.RequestTime requestTime, ReplicaPlan.ForWrite replicaPlan, TimeUUID batchUUID);
+
+    /**
+     * Callback invoked when the given {@code mutation} is localy applied.
+     */
+    default void onAppliedMutation(IMutation mutation)
+    {
+        // no-op
+    }
+
+    /**
+     * Callback invoked when the given {@code counter} is localy applied.
+     */
+    default void onAppliedCounter(IMutation counter, AbstractWriteResponseHandler<IMutation> handler)
+    {
+        // no-op
+    }
+
+    /**
+     * Callback invoked when the given {@code proposal} is localy committed.
+     */
+    default void onAppliedProposal(Commit proposal)
+    {
+        // no-op
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/service/MutatorProvider.java b/src/java/org/apache/cassandra/service/MutatorProvider.java
new file mode 100644
index 000000000000..c592915832a7
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/MutatorProvider.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_MUTATOR_CLASS;
+
+/**
+ * Provides an instance of {@link Mutator} that facilitates mutation writes for standard mutations, unlogged batches,
+ * counters and paxos commits (LWT)s.
+ * <br/>
+ * An implementation may choose to fallback to the default implementation ({@link StorageProxy.DefaultMutator})
+ * obtained via {@link #getDefaultMutator()}.
+ */
+public abstract class MutatorProvider
+{
+    static final Mutator instance = getCustomOrDefault();
+
+    public static Mutator getCustomOrDefault()
+    {
+        if (CUSTOM_MUTATOR_CLASS.isPresent())
+        {
+            return FBUtilities.construct(CUSTOM_MUTATOR_CLASS.getString(),
+                                         "custom mutator class (set with " + CUSTOM_MUTATOR_CLASS.getKey() + ")");
+        }
+        else
+        {
+            return getDefaultMutator();
+        }
+    }
+
+    public static Mutator getDefaultMutator()
+    {
+        return new StorageProxy.DefaultMutator();
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/NativeTransportService.java b/src/java/org/apache/cassandra/service/NativeTransportService.java
index c51b29e80485..b24be8041dd4 100644
--- a/src/java/org/apache/cassandra/service/NativeTransportService.java
+++ b/src/java/org/apache/cassandra/service/NativeTransportService.java
@@ -38,7 +38,7 @@
 import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Server;
-import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.INativeLibrary;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.NATIVE_EPOLL_ENABLED;
 
@@ -129,7 +129,7 @@ synchronized void initialize()
      */
     public void start()
     {
-        logger.info("Using Netty Version: {}", Version.identify().entrySet());
+        logger.debug("Using Netty Version: {}", Version.identify().entrySet());
         initialize();
         servers.forEach(Server::start);
     }
@@ -169,7 +169,7 @@ public static boolean useEpoll()
     {
         final boolean enableEpoll = NATIVE_EPOLL_ENABLED.getBoolean();
 
-        if (enableEpoll && !Epoll.isAvailable() && NativeLibrary.osType == NativeLibrary.OSType.LINUX)
+        if (enableEpoll && !Epoll.isAvailable() && INativeLibrary.instance.isOS(INativeLibrary.OSType.LINUX))
             logger.warn("epoll not available", Epoll.unavailabilityCause());
 
         return enableEpoll && Epoll.isAvailable();
diff --git a/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java b/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java
index cd096a888e9c..80586b7c50fc 100644
--- a/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java
+++ b/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java
@@ -19,8 +19,14 @@
 package org.apache.cassandra.service;
 
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
@@ -30,9 +36,11 @@
 import org.apache.cassandra.concurrent.SequentialExecutorPlus.AtLeastOnceTrigger;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.TokenMetadataProvider;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.utils.ExecutorUtils;
 
+import static java.util.Objects.requireNonNull;
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 
@@ -44,29 +52,89 @@ public class PendingRangeCalculatorService
 
     // the executor will only run a single range calculation at a time while keeping at most one task queued in order
     // to trigger an update only after the most recent state change and not for each update individually
-    private final SequentialExecutorPlus executor = executorFactory()
-            .withJmxInternal()
-            .configureSequential("PendingRangeCalculator")
-            .withRejectedExecutionHandler((r, e) -> {})  // silently handle rejected tasks, this::update takes care of bookkeeping
-            .build();
-
-    private final AtLeastOnceTrigger update = executor.atLeastOnceTrigger(() -> {
-        PendingRangeCalculatorServiceDiagnostics.taskStarted(1);
-        long start = currentTimeMillis();
-        Collection<String> keyspaces = Schema.instance.distributedKeyspaces().names();
-        for (String keyspaceName : keyspaces)
-            calculatePendingRanges(Keyspace.open(keyspaceName).getReplicationStrategy(), keyspaceName);
-        if (logger.isTraceEnabled())
-            logger.trace("Finished PendingRangeTask for {} keyspaces in {}ms", keyspaces.size(), currentTimeMillis() - start);
-        PendingRangeCalculatorServiceDiagnostics.taskFinished();
-    });
+    private final SequentialExecutorPlus executor;
+
+    private final Schema schema;
+
+    private final AtLeastOnceTrigger update;
+
+    private final Set<String> keyspacesWithPendingRanges = new CopyOnWriteArraySet<>();
+
+    private final TokenMetadataProvider tokenMetadataProvider;
+
+    private void doUpdate()
+    {
+        // repeat until all keyspaced are consumed
+        while (!keyspacesWithPendingRanges.isEmpty())
+        {
+            long start = currentTimeMillis();
+
+            int updated = 0;
+            int total = 0;
+            PendingRangeCalculatorServiceDiagnostics.taskStarted(1);
+            try
+            {
+                Set<String> keyspaces = new HashSet<>(keyspacesWithPendingRanges);
+                total = keyspaces.size();
+                keyspacesWithPendingRanges.removeAll(keyspaces); // only remove those which were consumed
+
+                Iterator<String> it = keyspaces.iterator();
+                while (it.hasNext())
+                {
+                    String keyspaceName = it.next();
+                    try
+                    {
+                        calculatePendingRanges(keyspaceName);
+                        it.remove();
+                        updated++;
+                    }
+                    catch (RuntimeException | Error ex)
+                    {
+                        logger.error("Error calculating pending ranges for keyspace {}", keyspaceName, ex);
+                    }
+                }
+            }
+            finally
+            {
+                PendingRangeCalculatorServiceDiagnostics.taskFinished();
+                if (logger.isTraceEnabled())
+                    logger.trace("Finished PendingRangeTask for {} keyspaces out of {} in {}ms", updated, total, currentTimeMillis() - start);
+            }
+        }
+    }
 
     public PendingRangeCalculatorService()
     {
+        this("PendingRangeCalculator", Schema.instance);
+    }
+
+    public PendingRangeCalculatorService(String executorName, Schema schema)
+    {
+        this(executorFactory().withJmxInternal()
+                              .configureSequential(executorName)
+                              .withRejectedExecutionHandler((r, e) -> {})  // silently handle rejected tasks, this::update takes care of bookkeeping
+                              .build(),
+             TokenMetadataProvider.instance,
+             schema);
+    }
+
+    public PendingRangeCalculatorService(SequentialExecutorPlus executor, TokenMetadataProvider tokenMetadataProvider, Schema schema)
+    {
+        this.executor = requireNonNull(executor);
+        this.tokenMetadataProvider = requireNonNull(tokenMetadataProvider);
+        this.schema = requireNonNull(schema);
+        this.update = executor.atLeastOnceTrigger(this::doUpdate);
     }
 
     public void update()
     {
+        update(keyspaceName -> true);
+    }
+
+    public void update(Predicate<String> keyspaceNamePredicate)
+    {
+        Collection<String> affectedKeyspaces = schema.distributedKeyspaces().names().stream().filter(keyspaceNamePredicate).collect(Collectors.toList());
+        keyspacesWithPendingRanges.addAll(affectedKeyspaces);
         boolean success = update.trigger();
         if (!success) PendingRangeCalculatorServiceDiagnostics.taskRejected(1);
         else PendingRangeCalculatorServiceDiagnostics.taskCountChanged(1);
@@ -77,16 +145,23 @@ public void blockUntilFinished()
         update.sync();
     }
 
-
     public void executeWhenFinished(Runnable runnable)
     {
         update.runAfter(runnable);
     }
 
-    // public & static for testing purposes
-    public static void calculatePendingRanges(AbstractReplicationStrategy strategy, String keyspaceName)
+    @VisibleForTesting
+    protected void calculatePendingRanges(String keyspaceName)
+    {
+        Keyspace keyspace = Keyspace.open(keyspaceName);
+        AbstractReplicationStrategy strategy = keyspace.getReplicationStrategy();
+        calculatePendingRanges(strategy, keyspaceName);
+    }
+
+    @VisibleForTesting
+    public void calculatePendingRanges(AbstractReplicationStrategy strategy, String keyspaceName)
     {
-        StorageService.instance.getTokenMetadata().calculatePendingRanges(strategy, keyspaceName);
+        tokenMetadataProvider.getTokenMetadataForKeyspace(keyspaceName).calculatePendingRanges(strategy, keyspaceName);
     }
 
     @VisibleForTesting
diff --git a/src/java/org/apache/cassandra/service/QueryInfoTracker.java b/src/java/org/apache/cassandra/service/QueryInfoTracker.java
new file mode 100644
index 000000000000..85ccf457f632
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/QueryInfoTracker.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * A tracker notified about executed queries.
+ *
+ * <p>The goal of this interface is to provide enough information to accurately estimate how
+ * much "work" a query has performed. So while for writes this mostly just means passing the generated mutations, for
+ * reads this means passing the unfiltered result of the query.
+ *
+ * <p>The tracker methods are called from {@link StorageProxy} and are thus "coordinator level". All
+ * user queries and internal distributed system table queries trigger a call to one of these methods.
+ * Internal local system table queries don't.
+ *
+ * <p>For writes, the {@link #onWrite} method is only called for the "user write", but if that write triggers either
+ * secondary index or materialized views updates, those additional updates do not trigger additional calls.
+ *
+ * <p>The tracker methods are called on write and read hot paths, so they should be as lightweight as possible.
+ */
+public interface QueryInfoTracker
+{
+    /**
+     * Called before every (non-LWT) write coordinated on the local node.
+     *
+     * @param state            the state of the client that performed the write
+     * @param isLogged         whether this is a logged batch write.
+     * @param mutations        the mutations written by the write.
+     * @param consistencyLevel the consistency level of the write.
+     * @return a tracker that should be notified when either the read error out or completes successfully.
+     */
+    WriteTracker onWrite(ClientState state,
+                         boolean isLogged,
+                         Collection<? extends IMutation> mutations,
+                         ConsistencyLevel consistencyLevel);
+
+    /**
+     * Called before every non-range read coordinated on the local node.
+     *
+     * @param state            the state of the client that performed the read
+     * @param table            the metadata for the table read.
+     * @param commands         the commands for the read performed.
+     * @param consistencyLevel the consistency level of the read.
+     * @return a tracker that should be notified when either the read error out or completes successfully.
+     */
+    ReadTracker onRead(ClientState state,
+                       TableMetadata table,
+                       List<SinglePartitionReadCommand> commands,
+                       ConsistencyLevel consistencyLevel);
+
+    /**
+     * Called before every range read coordinated on the local node.
+     *
+     * @param state            the state of the client that performed the range read
+     * @param table            the metadata for the table read.
+     * @param command          the command for the read performed.
+     * @param consistencyLevel the consistency level of the read.
+     * @return a tracker that should  be notified when either the read error out or completes successfully.
+     */
+    ReadTracker onRangeRead(ClientState state,
+                            TableMetadata table,
+                            PartitionRangeReadCommand command,
+                            ConsistencyLevel consistencyLevel);
+
+    /**
+     * Called before every LWT coordinated by the local node.
+     *
+     * @param state             the state of the client that performed the LWT
+     * @param table             the metadata of the table on which the LWT applies.
+     * @param key               the partition key on which the LWT operates.
+     * @param serialConsistency the serial consistency of the LWT.
+     * @param commitConsistency the commit consistency of the LWT.
+     * @return a {@link LWTWriteTracker} objects whose methods are called as part of the LWT execution.
+     */
+    LWTWriteTracker onLWTWrite(ClientState state,
+                               TableMetadata table,
+                               DecoratedKey key,
+                               ConsistencyLevel serialConsistency,
+                               ConsistencyLevel commitConsistency);
+
+    /**
+     * A tracker that does nothing.
+     */
+    QueryInfoTracker NOOP = new QueryInfoTracker()
+    {
+        @Override
+        public WriteTracker onWrite(ClientState state,
+                                    boolean isLogged,
+                                    Collection<? extends IMutation> mutations,
+                                    ConsistencyLevel consistencyLevel)
+        {
+            return WriteTracker.NOOP;
+        }
+
+        @Override
+        public ReadTracker onRead(ClientState state,
+                                  TableMetadata table,
+                                  List<SinglePartitionReadCommand> commands,
+                                  ConsistencyLevel consistencyLevel)
+        {
+            return ReadTracker.NOOP;
+        }
+
+        @Override
+        public ReadTracker onRangeRead(ClientState state,
+                                       TableMetadata table,
+                                       PartitionRangeReadCommand command,
+                                       ConsistencyLevel consistencyLevel)
+        {
+            return ReadTracker.NOOP;
+        }
+
+        @Override
+        public LWTWriteTracker onLWTWrite(ClientState state,
+                                          TableMetadata table,
+                                          DecoratedKey key,
+                                          ConsistencyLevel serialConsistency,
+                                          ConsistencyLevel commitConsistency)
+        {
+            return LWTWriteTracker.NOOP;
+        }
+    };
+
+    /**
+     * A tracker for a specific query.
+     *
+     * <p>For the tracked query, exactly one of its method should be called.
+     */
+    interface Tracker
+    {
+        /**
+         * Called when the tracked query completes successfully.
+         */
+        void onDone();
+
+        /**
+         * Called when the tracked query completes with an error.
+         */
+        void onError(Throwable exception);
+    }
+
+    /**
+     * Tracker for a write query.
+     */
+    interface WriteTracker extends Tracker
+    {
+        WriteTracker NOOP = new WriteTracker()
+        {
+            @Override
+            public void onDone()
+            {
+            }
+
+            @Override
+            public void onError(Throwable exception)
+            {
+            }
+        };
+    }
+
+    /**
+     * Tracker for a read query.
+     */
+    interface ReadTracker extends Tracker
+    {
+        /**
+         * Called just before the queries are sent to the replica plan contacts.
+         * Note that this callback method may be invoked more than once for a given read,
+         * e.g. range quries spanning multiple partitions are internally issued as a
+         * number of subrange requests to different replicas (with different
+         * ReplicaPlans). This callback is called at least once for a given read.
+         *
+         * @param replicaPlan the queried nodes.
+         */
+        void onReplicaPlan(ReplicaPlan.ForRead<?, ?> replicaPlan);
+
+        /**
+         * Called on every new reconciled partition.
+         *
+         * @param partitionKey the partition key.
+         */
+        void onPartition(DecoratedKey partitionKey);
+
+        /**
+         * Called on every row read.
+         *
+         * @param row          the merged row.
+         */
+        void onRow(Row row);
+
+        /**
+         * Called on every partition after filtering and post-processing
+         *
+         * @param partitionKey
+         */
+        void onFilteredPartition(DecoratedKey partitionKey);
+
+        /**
+         * Called on every row after filtering and post-processing
+         *
+         * @param row          the merged row.
+         */
+        void onFilteredRow(Row row);
+
+        ReadTracker NOOP = new ReadTracker()
+        {
+            @Override
+            public void onReplicaPlan(ReplicaPlan.ForRead<?, ?> replicaPlan)
+            {
+            }
+
+            @Override
+            public void onPartition(DecoratedKey partitionKey)
+            {
+            }
+
+            @Override
+            public void onRow(Row row)
+            {
+            }
+
+            @Override
+            public void onFilteredPartition(DecoratedKey partitionKey)
+            {
+            }
+
+            @Override
+            public void onFilteredRow(Row row)
+            {
+            }
+
+            @Override
+            public void onDone()
+            {
+            }
+
+            @Override
+            public void onError(Throwable exception)
+            {
+            }
+        };
+    }
+
+    /**
+     * Tracker for LWTs, used to get information on the actual work done by the LWT.
+     *
+     * <p>For a given LWT, the tracker created by {@link #onLWTWrite} will first have its read
+     * methods called. Then, based on that read result and the LWT conditions, either the {@link #onNotApplied()} or
+     * the {@link #onApplied} method will be called.
+     */
+    interface LWTWriteTracker extends ReadTracker
+    {
+        /**
+         * Called if the LWT this is tracking does not apply (it's condition evaluates to {@code false}).
+         */
+        void onNotApplied();
+
+        /**
+         * Called if the LWT this is tracking does apply.
+         *
+         * @param update the update that is committed by the LWT.
+         */
+        void onApplied(PartitionUpdate update);
+
+        /**
+         * A tracker that does nothing.
+         */
+        LWTWriteTracker NOOP = new LWTWriteTracker()
+        {
+            @Override
+            public void onReplicaPlan(ReplicaPlan.ForRead<?, ?> replicaPlan)
+            {
+            }
+
+            @Override
+            public void onPartition(DecoratedKey partitionKey)
+            {
+            }
+
+            @Override
+            public void onRow(Row row)
+            {
+            }
+
+            @Override
+            public void onFilteredPartition(DecoratedKey partitionKey)
+            {
+            }
+
+            @Override
+            public void onFilteredRow(Row row)
+            {
+            }
+
+            @Override
+            public void onNotApplied()
+            {
+            }
+
+            @Override
+            public void onApplied(PartitionUpdate update)
+            {
+            }
+
+            @Override
+            public void onDone()
+            {
+            }
+
+            @Override
+            public void onError(Throwable exception)
+            {
+            }
+        };
+
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/QueryState.java b/src/java/org/apache/cassandra/service/QueryState.java
index d4d4d73717f5..8aa6056468e1 100644
--- a/src/java/org/apache/cassandra/service/QueryState.java
+++ b/src/java/org/apache/cassandra/service/QueryState.java
@@ -19,6 +19,7 @@
 
 import java.net.InetAddress;
 
+import org.apache.cassandra.auth.AuthenticatedUser;
 import org.apache.cassandra.utils.FBUtilities;
 
 /**
@@ -114,4 +115,15 @@ public InetAddress getClientAddress()
     {
         return clientState.getClientAddress();
     }
+
+    /**
+     * Checks if this user is an ordinary user (not a super or system user).
+     *
+     * @return {@code true} if this user is an ordinary user, {@code false} otherwise.
+     */
+    public boolean isOrdinaryUser()
+    {
+        AuthenticatedUser user = getClientState().getUser();
+        return !getClientState().isInternal && null != user && !user.isSystem() && !user.isSuper();
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/RangeRelocator.java b/src/java/org/apache/cassandra/service/RangeRelocator.java
index b63c105bd2f5..bb9daa8280bd 100644
--- a/src/java/org/apache/cassandra/service/RangeRelocator.java
+++ b/src/java/org/apache/cassandra/service/RangeRelocator.java
@@ -36,7 +36,7 @@
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.RangeStreamer;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.EndpointsByReplica;
 import org.apache.cassandra.locator.EndpointsForRange;
@@ -100,7 +100,7 @@ private static Multimap<InetAddressAndPort, RangeStreamer.FetchReplica> calculat
                                                                    tmdBefore,
                                                                    tmdAfter,
                                                                    keyspace,
-                                                                   Arrays.asList(new RangeStreamer.FailureDetectorSourceFilter(FailureDetector.instance),
+                                                                   Arrays.asList(new RangeStreamer.FailureDetectorSourceFilter(IFailureDetector.instance),
                                                                                  new RangeStreamer.ExcludeLocalNodeFilter()));
         return RangeStreamer.convertPreferredEndpointsToWorkMap(preferredEndpoints);
     }
diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java
index 934a17b06494..2eeb1f1aab33 100644
--- a/src/java/org/apache/cassandra/service/StartupChecks.java
+++ b/src/java/org/apache/cassandra/service/StartupChecks.java
@@ -72,7 +72,7 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JavaUtils;
-import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.INativeLibrary;
 import org.apache.cassandra.utils.SigarLibrary;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT;
@@ -205,7 +205,7 @@ public void execute(StartupChecksOptions startupChecksOptions) throws StartupExc
 
             Set<Path> directIOWritePaths = new HashSet<>();
             if (DatabaseDescriptor.getCommitLogWriteDiskAccessMode() == Config.DiskAccessMode.direct)
-                directIOWritePaths.add(new File(DatabaseDescriptor.getCommitLogLocation()).toPath());
+                directIOWritePaths.add(DatabaseDescriptor.getCommitLogLocation().toPath());
             // TODO: add data directories when direct IO is supported for flushing and compaction
 
             if (!directIOWritePaths.isEmpty() && IGNORE_KERNEL_BUG_1057843_CHECK.getBoolean())
@@ -414,7 +414,7 @@ public void execute(StartupChecksOptions options) throws StartupException
             if (options.isDisabled(getStartupCheckType()))
                 return;
             // Fail-fast if the native library could not be linked.
-            if (!NativeLibrary.isAvailable())
+            if (!INativeLibrary.instance.isAvailable())
                 throw new StartupException(StartupException.ERR_WRONG_MACHINE_STATE, "The native library could not be initialized properly. ");
         }
     };
@@ -566,29 +566,26 @@ public void execute(StartupChecksOptions options) throws StartupException
             if (options.isDisabled(getStartupCheckType()))
                 return;
             // check all directories(data, commitlog, saved cache) for existence and permission
-            Iterable<String> dirs = Iterables.concat(Arrays.asList(DatabaseDescriptor.getAllDataFileLocations()),
+            Iterable<File> dirs = Iterables.concat(Arrays.asList(DatabaseDescriptor.getAllDataFileLocations()),
                                                      Arrays.asList(DatabaseDescriptor.getCommitLogLocation(),
                                                                    DatabaseDescriptor.getSavedCachesLocation(),
-                                                                   DatabaseDescriptor.getHintsDirectory().absolutePath()));
-            for (String dataDir : dirs)
-            {
-                logger.debug("Checking directory {}", dataDir);
-                File dir = new File(dataDir);
-
+                                                                   DatabaseDescriptor.getHintsDirectory()));
+            for (File dir : dirs) {
+                logger.debug("Checking directory {}", dir);
+    
                 // check that directories exist.
-                if (!dir.exists())
-                {
-                    logger.warn("Directory {} doesn't exist", dataDir);
+                if (!dir.exists()) {
+                    logger.warn("Directory {} doesn't exist", dir);
                     // if they don't, failing their creation, stop cassandra.
                     if (!dir.tryCreateDirectories())
                         throw new StartupException(StartupException.ERR_WRONG_DISK_STATE,
-                                                   "Has no permission to create directory "+ dataDir);
+                                "Has no permission to create directory " + dir);
                 }
-
+    
                 // if directories exist verify their permissions
-                if (!Directories.verifyFullPermissions(dir, dataDir))
+                if (!Directories.verifyFullPermissions(dir))
                     throw new StartupException(StartupException.ERR_WRONG_DISK_STATE,
-                                               "Insufficient permissions on directory " + dataDir);
+                            "Insufficient permissions on directory " + dir);
             }
         }
     };
@@ -676,11 +673,11 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
                 }
             };
 
-            for (String dataDir : DatabaseDescriptor.getAllDataFileLocations())
+            for (File dataDir : DatabaseDescriptor.getAllDataFileLocations())
             {
                 try
                 {
-                    Files.walkFileTree(new File(dataDir).toPath(), sstableVisitor);
+                    Files.walkFileTree(dataDir.toPath(), sstableVisitor);
                 }
                 catch (IOException e)
                 {
diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java
index 43261f8b4c6e..fcb081dd19e5 100644
--- a/src/java/org/apache/cassandra/service/StorageProxy.java
+++ b/src/java/org/apache/cassandra/service/StorageProxy.java
@@ -39,12 +39,11 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 
 import com.google.common.base.Preconditions;
 import com.google.common.cache.CacheLoader;
 import com.google.common.collect.Iterables;
-import com.google.common.util.concurrent.Uninterruptibles;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -58,6 +57,7 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.CounterMutationCallback;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
 import org.apache.cassandra.db.Keyspace;
@@ -70,6 +70,7 @@
 import org.apache.cassandra.db.RejectException;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.TruncateRequest;
+import org.apache.cassandra.db.WriteOptions;
 import org.apache.cassandra.db.WriteType;
 import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
 import org.apache.cassandra.db.partitions.FilteredPartition;
@@ -92,11 +93,13 @@
 import org.apache.cassandra.exceptions.RequestFailureException;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.exceptions.RequestTimeoutException;
+import org.apache.cassandra.exceptions.TruncateException;
 import org.apache.cassandra.exceptions.UnavailableException;
 import org.apache.cassandra.exceptions.WriteFailureException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.hints.Hint;
 import org.apache.cassandra.hints.HintsService;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
@@ -111,6 +114,8 @@
 import org.apache.cassandra.locator.Replicas;
 import org.apache.cassandra.metrics.CASClientRequestMetrics;
 import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
 import org.apache.cassandra.metrics.DenylistMetrics;
 import org.apache.cassandra.metrics.ReadRepairMetrics;
 import org.apache.cassandra.metrics.StorageMetrics;
@@ -125,6 +130,11 @@
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.SensorsFactory;
+import org.apache.cassandra.sensors.Type;
 import org.apache.cassandra.service.paxos.Ballot;
 import org.apache.cassandra.service.paxos.Commit;
 import org.apache.cassandra.service.paxos.ContentionStrategy;
@@ -132,6 +142,7 @@
 import org.apache.cassandra.service.paxos.PaxosState;
 import org.apache.cassandra.service.paxos.v1.PrepareCallback;
 import org.apache.cassandra.service.paxos.v1.ProposeCallback;
+import org.apache.cassandra.service.paxos.PaxosUtils;
 import org.apache.cassandra.service.reads.AbstractReadExecutor;
 import org.apache.cassandra.service.reads.ReadCallback;
 import org.apache.cassandra.service.reads.range.RangeCommands;
@@ -149,20 +160,10 @@
 import org.apache.cassandra.utils.concurrent.CountDownLatch;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
+import static com.google.common.collect.Iterables.concat;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
-
-import static com.google.common.collect.Iterables.concat;
-import static org.apache.commons.lang3.StringUtils.join;
-
 import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsForLevel;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.viewWriteMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetricsForLevel;
 import static org.apache.cassandra.net.Message.out;
 import static org.apache.cassandra.net.NoPayload.noPayload;
 import static org.apache.cassandra.net.Verb.BATCH_STORE_REQ;
@@ -182,6 +183,7 @@
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
+import static org.apache.commons.lang3.StringUtils.join;
 
 public class StorageProxy implements StorageProxyMBean
 {
@@ -198,6 +200,183 @@ public class StorageProxy implements StorageProxyMBean
 
     public static final StorageProxy instance = new StorageProxy();
 
+    private static final Mutator mutator = MutatorProvider.instance;
+
+    public static class DefaultMutator implements Mutator
+    {
+        @Override
+        public AbstractWriteResponseHandler<IMutation> mutateCounter(CounterMutation cm, String localDataCenter, Dispatcher.RequestTime requestTime)
+        {
+            return defaultMutateCounter(cm, localDataCenter, requestTime);
+        }
+
+        @Override
+        public AbstractWriteResponseHandler<IMutation> mutateCounterOnLeader(CounterMutation mutation,
+                                                                             String localDataCenter,
+                                                                             StorageProxy.WritePerformer performer,
+                                                                             Runnable callback,
+                                                                             Dispatcher.RequestTime requestTime)
+        {
+            return performWrite(mutation, mutation.consistency(), localDataCenter, performer, callback, WriteType.COUNTER, requestTime);
+        }
+
+        @Override
+        public AbstractWriteResponseHandler<IMutation> mutateStandard(Mutation mutation, ConsistencyLevel consistencyLevel, String localDataCenter, WritePerformer standardWritePerformer, Runnable callback, WriteType writeType, Dispatcher.RequestTime requestTime)
+        {
+            return performWrite(mutation, consistencyLevel, localDataCenter, standardWritePerformer, callback, writeType, requestTime);
+        }
+
+        @Override
+        public AbstractWriteResponseHandler<Commit> mutatePaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, Dispatcher.RequestTime requestTime)
+        {
+            return defaultCommitPaxos(proposal, consistencyLevel, allowHints, requestTime);
+        }
+
+        @Override
+        public void mutateAtomically(Collection<Mutation> mutations, ConsistencyLevel consistencyLevel, boolean requireQuorumForRemove, Dispatcher.RequestTime requestTime, ClientRequestsMetrics metrics, ClientState clientState) throws UnavailableException, OverloadedException, WriteTimeoutException
+        {
+            Tracing.trace("Determining replicas for atomic batch");
+            long startTime = nanoTime();
+
+            QueryInfoTracker.WriteTracker writeTracker = queryTracker().onWrite(clientState, true, mutations, consistencyLevel);
+
+            // Request sensors are utilized to track usages from replicas serving atomic batch request
+            RequestSensors sensors = SensorsFactory.instance.createRequestSensors(mutations.stream().map(IMutation::getKeyspaceName).toArray(String[]::new));
+            RequestTracker.instance.set(sensors);
+
+            if (mutations.stream().anyMatch(mutation -> Keyspace.open(mutation.getKeyspaceName()).getReplicationStrategy().hasTransientReplicas()))
+                throw new AssertionError("Logged batches are unsupported with transient replication");
+
+            try
+            {
+
+                // If we are requiring quorum nodes for removal, we upgrade consistency level to QUORUM unless we already
+                // require ALL, or EACH_QUORUM. This is so that *at least* QUORUM nodes see the update.
+                ConsistencyLevel batchConsistencyLevel = requireQuorumForRemove
+                                                         ? ConsistencyLevel.QUORUM
+                                                         : consistencyLevel;
+
+                switch (consistencyLevel)
+                {
+                    case ALL:
+                    case EACH_QUORUM:
+                        batchConsistencyLevel = consistencyLevel;
+                }
+
+                ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forBatchlogWrite(batchConsistencyLevel == ConsistencyLevel.ANY,
+                                                                                 false,
+                                                                                 mutations.iterator().next().getKeyspaceName());
+
+                final TimeUUID batchUUID = nextTimeUUID();
+                BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(),
+                                                              () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID, requestTime));
+
+                // add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
+                List<WriteResponseHandlerWrapper> wrappers = wrapBatchResponseHandlers(mutations, consistencyLevel, batchConsistencyLevel, cleanup, requestTime, sensors);
+
+                // write to the batchlog
+                syncWriteToBatchlog(mutations, replicaPlan, batchUUID, requestTime);
+
+                // now actually perform the writes and wait for them to complete
+                // note this is the actual change between CC and OSS - OSS uses syncWriteBatchedMutations and does not
+                // include waiting for the batched mutations to complete
+                asyncWriteBatchedMutations(wrappers, requestTime);
+
+                // wait for batched mutations to complete
+                for (StorageProxy.WriteResponseHandlerWrapper wrapper : wrappers)
+                    wrapper.handler.get();
+
+                writeTracker.onDone();
+            }
+            catch (UnavailableException e)
+            {
+                metrics.writeMetrics.unavailables.mark();
+                metrics.writeMetricsForLevel(consistencyLevel).unavailables.mark();
+                Tracing.trace("Unavailable");
+                writeTracker.onError(e);
+                throw e;
+            }
+            catch (WriteTimeoutException e)
+            {
+                metrics.writeMetrics.timeouts.mark();
+                metrics.writeMetricsForLevel(consistencyLevel).timeouts.mark();
+                Tracing.trace("Write timeout; received {} of {} required replies", e.received, e.blockFor);
+                writeTracker.onError(e);
+                throw e;
+            }
+            catch (WriteFailureException e)
+            {
+                metrics.writeMetrics.failures.mark();
+                metrics.writeMetricsForLevel(consistencyLevel).failures.mark();
+                Tracing.trace("Write failure; received {} of {} required replies", e.received, e.blockFor);
+                writeTracker.onError(e);
+                throw e;
+            }
+            finally
+            {
+                long endTime = nanoTime();
+                long latency = endTime - startTime;
+                long serviceLatency = endTime - requestTime.startedAtNanos();
+                metrics.writeMetrics.executionTimeMetrics.addNano(latency);
+                metrics.writeMetrics.serviceTimeMetrics.addNano(serviceLatency);
+                metrics.writeMetricsForLevel(consistencyLevel).executionTimeMetrics.addNano(latency);
+                metrics.writeMetricsForLevel(consistencyLevel).serviceTimeMetrics.addNano(serviceLatency);
+                StorageProxy.updateCoordinatorWriteLatencyTableMetric(mutations, latency);
+            }
+        }
+
+        @Override
+        public void clearBatchlog(String keyspace, Dispatcher.RequestTime requestTime, ReplicaPlan.ForWrite replicaPlan, TimeUUID batchUUID)
+        {
+            StorageProxy.asyncRemoveFromBatchlog(replicaPlan, batchUUID, requestTime);
+        }
+
+        @Override
+        public void persistBatchlog(Collection<Mutation> mutations, Dispatcher.RequestTime requestTime, ReplicaPlan.ForWrite replicaPlan, TimeUUID batchUUID)
+        {
+            // write to the batchlog
+            StorageProxy.syncWriteToBatchlog(mutations, replicaPlan, batchUUID, requestTime);
+        }
+
+        private List<WriteResponseHandlerWrapper> wrapBatchResponseHandlers(Collection<Mutation> mutations,
+                                                                              ConsistencyLevel consistencyLevel,
+                                                                              ConsistencyLevel batchConsistencyLevel,
+                                                                              BatchlogResponseHandler.BatchlogCleanup cleanup,
+                                                                              Dispatcher.RequestTime requestTime,
+                                                                              RequestSensors sensors)
+        {
+            List<StorageProxy.WriteResponseHandlerWrapper> wrappers = new ArrayList<>(mutations.size());
+
+            // add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
+            for (Mutation mutation : mutations)
+            {
+                // register the sensors for the mutation before the actual write is performed
+                for (PartitionUpdate pu: mutation.getPartitionUpdates())
+                {
+                    if (pu.metadata().isIndex()) continue;
+                    sensors.registerSensor(Context.from(pu.metadata()), Type.WRITE_BYTES);
+                }
+                StorageProxy.WriteResponseHandlerWrapper wrapper = StorageProxy.wrapBatchResponseHandler(mutation,
+                                                                                                         consistencyLevel,
+                                                                                                         batchConsistencyLevel,
+                                                                                                         WriteType.BATCH,
+                                                                                                         cleanup,
+                                                                                                         requestTime);
+                // exit early if we can't fulfill the CL at this time.
+                wrappers.add(wrapper);
+            }
+
+            return wrappers;
+        }
+
+        private void asyncWriteBatchedMutations(List<StorageProxy.WriteResponseHandlerWrapper> wrappers, Dispatcher.RequestTime requestTime)
+        throws WriteTimeoutException, OverloadedException
+        {
+            String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+            StorageProxy.asyncWriteBatchedMutations(wrappers, localDataCenter, Stage.MUTATION, requestTime);
+        }
+    }
+
     private static volatile int maxHintsInProgress = 128 * FBUtilities.getAvailableProcessors();
     private static final CacheLoader<InetAddressAndPort, AtomicInteger> hintsInProgress = new CacheLoader<InetAddressAndPort, AtomicInteger>()
     {
@@ -212,6 +391,7 @@ public AtomicInteger load(InetAddressAndPort inetAddress)
     private static final PartitionDenylist partitionDenylist = new PartitionDenylist();
 
     private volatile long logBlockingReadRepairAttemptsUntilNanos = Long.MIN_VALUE;
+    private static volatile QueryInfoTracker queryInfoTracker = QueryInfoTracker.NOOP;
 
     private StorageProxy()
     {
@@ -221,11 +401,13 @@ private StorageProxy()
     {
         MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME);
         HintsService.instance.registerMBean();
+        PaxosUtils.instance.registerMBean();
 
         standardWritePerformer = (mutation, targets, responseHandler, localDataCenter, requestTime) ->
         {
             assert mutation instanceof Mutation;
             sendToHintedReplicas((Mutation) mutation, targets, responseHandler, localDataCenter, Stage.MUTATION, requestTime);
+            mutator.onAppliedMutation(mutation);
         };
 
         /*
@@ -245,11 +427,9 @@ private StorageProxy()
         {
             EndpointsForToken selected = targets.contacts().withoutSelf();
             Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548
-            Stage.COUNTER_MUTATION.executor()
-                                  .execute(counterWriteTask(mutation, targets.withContacts(selected), responseHandler, localDataCenter, requestTime));
+            Stage.COUNTER_MUTATION.execute(counterWriteTask(mutation, targets.withContacts(selected), responseHandler, localDataCenter, requestTime));
         };
 
-
         ReadRepairMetrics.init();
 
         if (!Paxos.isLinearizable())
@@ -263,6 +443,23 @@ private StorageProxy()
         }
     }
 
+    /**
+     * Registers the provided query info tracker
+     *
+     * <p>Note that only 1 query tracker can be registered at a time, so the provided tracker will unconditionally
+     * replace the currently registered tracker.
+     *
+     * @param tracker the tracker to register.
+     */
+    public void registerQueryTracker(QueryInfoTracker tracker) {
+        Objects.requireNonNull(tracker);
+        queryInfoTracker = tracker;
+    }
+
+    public static QueryInfoTracker queryTracker() {
+        return queryInfoTracker;
+    }
+
     /**
      * Apply @param updates if and only if the current values in the row for @param key
      * match the provided @param conditions.  The algorithm is "raw" Paxos: that is, Paxos
@@ -338,50 +535,77 @@ public static RowIterator legacyCas(String keyspaceName,
                                         Dispatcher.RequestTime requestTime)
     throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException
     {
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(keyspaceName);
+        TableMetadata metadata = Schema.instance.validateTable(keyspaceName, cfName);
+        QueryInfoTracker.LWTWriteTracker lwtTracker = queryTracker().onLWTWrite(clientState,
+                                                                                metadata,
+                                                                                key,
+                                                                                consistencyForPaxos,
+                                                                                consistencyForCommit);
+        // Request sensors are utilized to track usages from replicas serving a cas request
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors(keyspaceName);
+        Context context = Context.from(metadata);
+        sensors.registerSensor(context, Type.WRITE_BYTES); // track user table + paxos table write bytes
+        sensors.registerSensor(context, Type.READ_BYTES); // track user table + paxos table read bytes
+        RequestTracker.instance.set(sensors);
         try
         {
-            TableMetadata metadata = Schema.instance.validateTable(keyspaceName, cfName);
+            consistencyForPaxos.validateForCas(keyspaceName, clientState);
+            consistencyForCommit.validateForCasCommit(Keyspace.open(keyspaceName).getReplicationStrategy(), keyspaceName, clientState);
 
             Function<Ballot, Pair<PartitionUpdate, RowIterator>> updateProposer = ballot ->
             {
-                // read the current values and check they validate the conditions
-                Tracing.trace("Reading existing values for CAS precondition");
-                SinglePartitionReadCommand readCommand = request.readCommand(nowInSeconds);
-                ConsistencyLevel readConsistency = consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM;
-
-                FilteredPartition current;
-                try (RowIterator rowIter = readOne(readCommand, readConsistency, requestTime))
+                long startTimeNanos = Clock.Global.nanoTime();
+                try
                 {
-                    current = FilteredPartition.create(rowIter);
-                }
+                    // read the current values and check they validate the conditions
+                    Tracing.trace("Reading existing values for CAS precondition");
+                    SinglePartitionReadCommand readCommand = (SinglePartitionReadCommand) request.readCommand(nowInSeconds);
+                    ConsistencyLevel readConsistency = consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM;
 
-                if (!request.appliesTo(current))
-                {
-                    Tracing.trace("CAS precondition does not match current values {}", current);
-                    casWriteMetrics.conditionNotMet.inc();
-                    return Pair.create(PartitionUpdate.emptyUpdate(metadata, key), current.rowIterator());
-                }
+                    FilteredPartition current;
 
-                // Create the desired updates
-                PartitionUpdate updates = request.makeUpdates(current, clientState, ballot);
+                    try (RowIterator rowIter = readOne(readCommand, readConsistency, clientState, requestTime, lwtTracker))
+                    {
+                        current = FilteredPartition.create(rowIter);
+                    }
 
-                // Update the metrics before triggers potentially add mutations.
-                ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(updates);
+                    if (!request.appliesTo(current))
+                    {
+                        Tracing.trace("CAS precondition does not match current values {}", current);
+                        lwtTracker.onNotApplied();
+                        lwtTracker.onDone();
+                        metrics.casWriteMetrics.conditionNotMet.inc();
+                        return Pair.create(PartitionUpdate.emptyUpdate(metadata, key), current.rowIterator());
+                    }
 
-                long size = updates.dataSize();
-                casWriteMetrics.mutationSize.update(size);
-                writeMetricsForLevel(consistencyForPaxos).mutationSize.update(size);
+                    // Create the desired updates
+                    PartitionUpdate updates = request.makeUpdates(current, clientState, ballot);
 
-                // Apply triggers to cas updates. A consideration here is that
-                // triggers emit Mutations, and so a given trigger implementation
-                // may generate mutations for partitions other than the one this
-                // paxos round is scoped for. In this case, TriggerExecutor will
-                // validate that the generated mutations are targetted at the same
-                // partition as the initial updates and reject (via an
-                // InvalidRequestException) any which aren't.
-                updates = TriggerExecutor.instance.execute(updates);
+                    // Update the metrics before triggers potentially add mutations.
+                    ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(updates);
+                    lwtTracker.onApplied(updates);
+                    lwtTracker.onDone();
 
-                return Pair.create(updates, null);
+                    long size = updates.dataSize();
+                    metrics.casWriteMetrics.mutationSize.update(size);
+                    metrics.writeMetricsForLevel(consistencyForPaxos).mutationSize.update(size);
+
+                    // Apply triggers to cas updates. A consideration here is that
+                    // triggers emit Mutations, and so a given trigger implementation
+                    // may generate mutations for partitions other than the one this
+                    // paxos round is scoped for. In this case, TriggerExecutor will
+                    // validate that the generated mutations are targetted at the same
+                    // partition as the initial updates and reject (via an
+                    // InvalidRequestException) any which aren't.
+                    updates = TriggerExecutor.instance.execute(updates);
+
+                    return Pair.create(updates, null);
+                }
+                finally
+                {
+                    metrics.casWriteMetrics.createProposalLatency.addNano(Clock.Global.nanoTime() - startTimeNanos);
+                }
             };
 
             return doPaxos(metadata,
@@ -389,44 +613,51 @@ public static RowIterator legacyCas(String keyspaceName,
                            consistencyForPaxos,
                            consistencyForCommit,
                            consistencyForCommit,
+                           clientState,
                            requestTime,
-                           casWriteMetrics,
-                           updateProposer);
+                           metrics.casWriteMetrics,
+                           updateProposer,
+                           false);
 
         }
         catch (CasWriteUnknownResultException e)
         {
-            casWriteMetrics.unknownResult.mark();
+            metrics.casWriteMetrics.unknownResult.mark();
+            lwtTracker.onError(e);
             throw e;
         }
         catch (CasWriteTimeoutException wte)
         {
-            casWriteMetrics.timeouts.mark();
-            writeMetricsForLevel(consistencyForPaxos).timeouts.mark();
+            metrics.casWriteMetrics.timeouts.mark();
+            metrics.writeMetricsForLevel(consistencyForPaxos).timeouts.mark();
+            lwtTracker.onError(wte);
             throw new CasWriteTimeoutException(wte.writeType, wte.consistency, wte.received, wte.blockFor, wte.contentions);
         }
         catch (ReadTimeoutException e)
         {
-            casWriteMetrics.timeouts.mark();
-            writeMetricsForLevel(consistencyForPaxos).timeouts.mark();
+            metrics.casWriteMetrics.timeouts.mark();
+            metrics.writeMetricsForLevel(consistencyForPaxos).timeouts.mark();
+            lwtTracker.onError(e);
             throw e;
         }
         catch (ReadAbortException e)
         {
-            casWriteMetrics.markAbort(e);
-            writeMetricsForLevel(consistencyForPaxos).markAbort(e);
+            metrics.casWriteMetrics.markAbort(e);
+            metrics.writeMetricsForLevel(consistencyForPaxos).markAbort(e);
             throw e;
         }
         catch (WriteFailureException | ReadFailureException e)
         {
-            casWriteMetrics.failures.mark();
-            writeMetricsForLevel(consistencyForPaxos).failures.mark();
+            metrics.casWriteMetrics.failures.mark();
+            metrics.writeMetricsForLevel(consistencyForPaxos).failures.mark();
+            lwtTracker.onError(e);
             throw e;
         }
         catch (UnavailableException e)
         {
-            casWriteMetrics.unavailables.mark();
-            writeMetricsForLevel(consistencyForPaxos).unavailables.mark();
+            metrics.casWriteMetrics.unavailables.mark();
+            metrics.writeMetricsForLevel(consistencyForPaxos).unavailables.mark();
+            lwtTracker.onError(e);
             throw e;
         }
         finally
@@ -434,8 +665,10 @@ public static RowIterator legacyCas(String keyspaceName,
             // We track latency based on request processing time, since the amount of time that request spends in the queue
             // is not a representative metric of replica performance.
             long latency = nanoTime() - requestTime.startedAtNanos();
-            casWriteMetrics.addNano(latency);
-            writeMetricsForLevel(consistencyForPaxos).addNano(latency);
+            metrics.casWriteMetrics.executionTimeMetrics.addNano(latency);
+            metrics.casWriteMetrics.serviceTimeMetrics.addNano(latency);
+            metrics.writeMetricsForLevel(consistencyForPaxos).executionTimeMetrics.addNano(latency);
+            metrics.writeMetricsForLevel(consistencyForPaxos).serviceTimeMetrics.addNano(latency);
         }
     }
 
@@ -469,6 +702,7 @@ private static void recordCasContention(TableMetadata table,
      *     {@link ConsistencyLevel#LOCAL_SERIAL}).
      * @param consistencyForReplayCommits the consistency for the commit phase of "replayed" in-progress operations.
      * @param consistencyForCommit the consistency for the commit phase of _this_ operation update.
+     * @param clientState the client state.
      * @param requestTime the nano time for the start of the query this is part of. This is the base time for
      *     timeouts.
      * @param casMetrics the metrics to update for this operation.
@@ -476,6 +710,7 @@ private static void recordCasContention(TableMetadata table,
      *     this operation and 2) the result that the whole method should return. This can return {@code null} in the
      *     special where, after having "prepared" (and thus potentially replayed in-progress upgdates), we don't want
      *     to propose anything (the whole method then return {@code null}).
+     * @param skipCommitConsistencyValidation whether to skip {@link ConsistencyLevel#validateForCasCommit} for commit consistency
      * @return the second element of the pair returned by {@code createUpdateProposal} (for the last call of that method
      *     if that method is called multiple times due to retries).
      */
@@ -484,9 +719,11 @@ private static RowIterator doPaxos(TableMetadata metadata,
                                        ConsistencyLevel consistencyForPaxos,
                                        ConsistencyLevel consistencyForReplayCommits,
                                        ConsistencyLevel consistencyForCommit,
+                                       ClientState clientState,
                                        Dispatcher.RequestTime requestTime,
                                        CASClientRequestMetrics casMetrics,
-                                       Function<Ballot, Pair<PartitionUpdate, RowIterator>> createUpdateProposal)
+                                       Function<Ballot, Pair<PartitionUpdate, RowIterator>> createUpdateProposal,
+                                       boolean skipCommitConsistencyValidation)
     throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException
     {
         int contentions = 0;
@@ -494,9 +731,10 @@ private static RowIterator doPaxos(TableMetadata metadata,
         AbstractReplicationStrategy latestRs = keyspace.getReplicationStrategy();
         try
         {
-            consistencyForPaxos.validateForCas();
-            consistencyForReplayCommits.validateForCasCommit(latestRs);
-            consistencyForCommit.validateForCasCommit(latestRs);
+            consistencyForPaxos.validateForCas(metadata.keyspace, clientState);
+            consistencyForReplayCommits.validateForCasCommit(latestRs, metadata.keyspace, clientState);
+            if (!skipCommitConsistencyValidation)
+                consistencyForCommit.validateForCasCommit(latestRs, metadata.keyspace, clientState);
 
             long timeoutNanos = DatabaseDescriptor.getCasContentionTimeout(NANOSECONDS);
             long deadline = requestTime.computeDeadline(timeoutNanos);
@@ -523,14 +761,14 @@ private static RowIterator doPaxos(TableMetadata metadata,
 
                 Commit proposal = Commit.newProposal(ballot, proposalPair.left);
                 Tracing.trace("CAS precondition is met; proposing client-requested updates for {}", ballot);
-                if (proposePaxos(proposal, replicaPlan, true, requestTime))
+                if (proposePaxos(proposal, replicaPlan, true, requestTime, casMetrics))
                 {
                     // We skip committing accepted updates when they are empty. This is an optimization which works
                     // because we also skip replaying those same empty update in beginAndRepairPaxos (see the longer
                     // comment there). As empty update are somewhat common (serial reads and non-applying CAS propose
                     // them), this is worth bothering.
                     if (!proposal.update.isEmpty())
-                        commitPaxos(proposal, consistencyForCommit, true, requestTime);
+                        commitPaxos(proposal, consistencyForCommit, true, requestTime, casMetrics);
                     RowIterator result = proposalPair.right;
                     if (result != null)
                         Tracing.trace("CAS did not apply");
@@ -541,8 +779,7 @@ private static RowIterator doPaxos(TableMetadata metadata,
 
                 Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)");
                 contentions++;
-
-                Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), TimeUnit.MILLISECONDS);
+                PaxosUtils.applyPaxosContentionBackoff(casMetrics);
                 // continue to retry
             }
         }
@@ -603,13 +840,13 @@ private static PaxosBallotAndContention beginAndRepairPaxos(Dispatcher.RequestTi
             {
                 Tracing.trace("Preparing {}", ballot);
                 Commit toPrepare = Commit.newPrepare(key, metadata, ballot);
-                summary = preparePaxos(toPrepare, paxosPlan, requestTime);
+                summary = preparePaxos(toPrepare, paxosPlan, requestTime, casMetrics);
                 if (!summary.promised)
                 {
                     Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting");
                     contentions++;
                     // sleep a random amount to give the other proposer a chance to finish
-                    Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), MILLISECONDS);
+                    PaxosUtils.applyPaxosContentionBackoff(casMetrics);
                     continue;
                 }
 
@@ -639,16 +876,16 @@ private static PaxosBallotAndContention beginAndRepairPaxos(Dispatcher.RequestTi
                     Tracing.trace("Finishing incomplete paxos round {}", inProgress);
                     casMetrics.unfinishedCommit.inc();
                     Commit refreshedInProgress = Commit.newProposal(ballot, inProgress.update);
-                    if (proposePaxos(refreshedInProgress, paxosPlan, false, requestTime))
+                    if (proposePaxos(refreshedInProgress, paxosPlan, false, requestTime, casMetrics))
                     {
-                        commitPaxos(refreshedInProgress, consistencyForCommit, false, requestTime);
+                        commitPaxos(refreshedInProgress, consistencyForCommit, false, requestTime, casMetrics);
                     }
                     else
                     {
                         Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting");
                         // sleep a random amount to give the other proposer a chance to finish
                         contentions++;
-                        Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), MILLISECONDS);
+                        PaxosUtils.applyPaxosContentionBackoff(casMetrics);
                     }
                     continue;
                 }
@@ -658,9 +895,11 @@ private static PaxosBallotAndContention beginAndRepairPaxos(Dispatcher.RequestTi
                 // Since we waited for quorum nodes, if some of them haven't seen the last commit (which may just be a timing issue, but may also
                 // mean we lost messages), we pro-actively "repair" those nodes, and retry.
                 Iterable<InetAddressAndPort> missingMRC = summary.replicasMissingMostRecentCommit();
-                if (Iterables.size(missingMRC) > 0)
+                int missingMRCSize = Iterables.size(missingMRC);
+                if (missingMRCSize > 0)
                 {
                     Tracing.trace("Repairing replicas that missed the most recent commit");
+                    casMetrics.missingMostRecentCommit.inc(missingMRCSize);
                     sendCommit(mostRecent, missingMRC);
                     // TODO: provided commits don't invalid the prepare we just did above (which they don't), we could just wait
                     // for all the missingMRC to acknowledge this commit and then move on with proposing our value. But that means
@@ -691,43 +930,53 @@ private static void sendCommit(Commit commit, Iterable<InetAddressAndPort> repli
             MessagingService.instance().send(message, target);
     }
 
-    private static PrepareCallback preparePaxos(Commit toPrepare, ReplicaPlan.ForPaxosWrite replicaPlan, Dispatcher.RequestTime requestTime)
+    private static PrepareCallback preparePaxos(Commit toPrepare, ReplicaPlan.ForPaxosWrite replicaPlan, Dispatcher.RequestTime requestTime,
+                                                CASClientRequestMetrics casMetrics) 
     throws WriteTimeoutException
     {
-        PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), replicaPlan.requiredParticipants(), replicaPlan.consistencyLevel(), requestTime);
-        Message<Commit> message = Message.out(PAXOS_PREPARE_REQ, toPrepare);
+        long startTimeNanos = Clock.Global.nanoTime();
+        try
+        {
+            PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), replicaPlan.requiredParticipants(), replicaPlan.consistencyLevel(), requestTime);
+            Message<Commit> message = Message.out(PAXOS_PREPARE_REQ, toPrepare);
 
-        boolean hasLocalRequest = false;
+            boolean hasLocalRequest = false;
 
-        for (Replica replica: replicaPlan.contacts())
-        {
-            if (replica.isSelf())
-            {
-                hasLocalRequest = true;
-                PAXOS_PREPARE_REQ.stage.execute(() -> {
-                    try
-                    {
-                        callback.onResponse(message.responseWith(doPrepare(toPrepare)));
-                    }
-                    catch (Exception ex)
-                    {
-                        logger.error("Failed paxos prepare locally", ex);
-                    }
-                });
-            }
-            else
+            for (Replica replica : replicaPlan.contacts())
             {
-                MessagingService.instance().sendWithCallback(message, replica.endpoint(), callback);
+                if (replica.isSelf())
+                {
+                    hasLocalRequest = true;
+                    PAXOS_PREPARE_REQ.stage.execute(() -> {
+                        try
+                        {
+                            callback.onResponse(message.responseWith(doPrepare(toPrepare)));
+                        }
+                        catch (Exception ex)
+                        {
+                            logger.error("Failed paxos prepare locally", ex);
+                        }
+                    });
+                }
+                else
+                {
+                    MessagingService.instance().sendWithCallback(message, replica.endpoint(), callback);
+                }
             }
-        }
 
-        if (hasLocalRequest)
-            writeMetrics.localRequests.mark();
-        else
-            writeMetrics.remoteRequests.mark();
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(toPrepare.update.metadata().keyspace);
+            if (hasLocalRequest)
+                metrics.writeMetrics.localRequests.mark();
+            else
+                metrics.writeMetrics.remoteRequests.mark();
 
-        callback.await();
-        return callback;
+            callback.await();
+            return callback;
+        }
+        finally
+        {
+            casMetrics.prepareLatency.addNano(Clock.Global.nanoTime() - startTimeNanos);
+        }
     }
 
     /**
@@ -735,44 +984,74 @@ private static PrepareCallback preparePaxos(Commit toPrepare, ReplicaPlan.ForPax
      * When {@param backoffIfPartial} is true, the proposer backs off when seeing the proposal being accepted by some but not a quorum.
      * The result of the cooresponding CAS in uncertain as the accepted proposal may or may not be spread to other nodes in later rounds.
      */
-    private static boolean proposePaxos(Commit proposal, ReplicaPlan.ForPaxosWrite replicaPlan, boolean backoffIfPartial, Dispatcher.RequestTime requestTime)
+    private static boolean proposePaxos(Commit proposal, ReplicaPlan.ForPaxosWrite replicaPlan, boolean backoffIfPartial,
+                                        Dispatcher.RequestTime requestTime, CASClientRequestMetrics casMetrics)
     throws WriteTimeoutException, CasWriteUnknownResultException
     {
-        ProposeCallback callback = new ProposeCallback(replicaPlan.contacts().size(), replicaPlan.requiredParticipants(), !backoffIfPartial, replicaPlan.consistencyLevel(), requestTime);
-        Message<Commit> message = Message.out(PAXOS_PROPOSE_REQ, proposal);
-        for (Replica replica : replicaPlan.contacts())
+        long startTimeNanos = Clock.Global.nanoTime();
+        try
         {
-            if (replica.isSelf())
+            ProposeCallback callback = new ProposeCallback(proposal.update.metadata(), replicaPlan.contacts().size(), replicaPlan.requiredParticipants(), !backoffIfPartial, replicaPlan.consistencyLevel(), requestTime);
+            Message<Commit> message = Message.out(PAXOS_PROPOSE_REQ, proposal);
+            for (Replica replica : replicaPlan.contacts())
             {
-                PAXOS_PROPOSE_REQ.stage.execute(() -> {
-                    try
-                    {
-                        Message<Boolean> response = message.responseWith(doPropose(proposal));
-                        callback.onResponse(response);
-                    }
-                    catch (Exception ex)
-                    {
-                        logger.error("Failed paxos propose locally", ex);
-                    }
-                });
-            }
-            else
-            {
-                MessagingService.instance().sendWithCallback(message, replica.endpoint(), callback);
+                if (replica.isSelf())
+                {
+                    PAXOS_PROPOSE_REQ.stage.execute(() -> {
+                        try
+                        {
+                            Message<Boolean> response = message.responseWith(doPropose(proposal));
+                            callback.onResponse(response);
+                        }
+                        catch (Exception ex)
+                        {
+                            logger.error("Failed paxos propose locally", ex);
+                        }
+                    });
+                }
+                else
+                {
+                    MessagingService.instance().sendWithCallback(message, replica.endpoint(), callback);
+                }
             }
-        }
-        callback.await();
+            callback.await();
 
-        if (callback.isSuccessful())
-            return true;
+            if (callback.isSuccessful())
+                return true;
 
-        if (backoffIfPartial && !callback.isFullyRefused())
-            throw new CasWriteUnknownResultException(replicaPlan.consistencyLevel(), callback.getAcceptCount(), replicaPlan.requiredParticipants());
+            if (backoffIfPartial && !callback.isFullyRefused())
+                throw new CasWriteUnknownResultException(replicaPlan.consistencyLevel(), callback.getAcceptCount(), replicaPlan.requiredParticipants());
+        }
+        finally
+        {
+            casMetrics.proposeLatency.addNano(Clock.Global.nanoTime() - startTimeNanos);
+        }
 
         return false;
     }
 
-    private static void commitPaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, Dispatcher.RequestTime requestTime) throws WriteTimeoutException
+    @Nullable
+    private static void commitPaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, Dispatcher.RequestTime requestTime,
+                                    CASClientRequestMetrics casMetrics) throws WriteTimeoutException
+    {
+        long startTimeNanos = Clock.Global.nanoTime();
+        boolean shouldBlock = consistencyLevel != ConsistencyLevel.ANY;
+        AbstractWriteResponseHandler<Commit> responseHandler = mutator.mutatePaxos(proposal, consistencyLevel, allowHints, requestTime);
+        if (shouldBlock && responseHandler != null)
+        {
+            try
+            {
+                responseHandler.get();
+            }
+            finally
+            {
+                casMetrics.commitLatency.addNano(Clock.Global.nanoTime() - startTimeNanos);
+            }
+        }
+    }
+
+    @Nullable
+    private static AbstractWriteResponseHandler<Commit> defaultCommitPaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, Dispatcher.RequestTime requestTime) throws WriteTimeoutException
     {
         boolean shouldBlock = consistencyLevel != ConsistencyLevel.ANY;
         Keyspace keyspace = Keyspace.open(proposal.update.metadata().keyspace);
@@ -821,8 +1100,7 @@ private static void commitPaxos(Commit proposal, ConsistencyLevel consistencyLev
             }
         }
 
-        if (shouldBlock)
-            responseHandler.get();
+        return responseHandler;
     }
 
     /**
@@ -838,7 +1116,7 @@ public void runMayThrow()
             {
                 try
                 {
-                    PaxosState.commitDirect(message.payload);
+                    PaxosState.commitDirect(message.payload, p -> mutator.onAppliedProposal(p));
                     if (responseHandler != null)
                         responseHandler.onResponse(null);
                 }
@@ -874,12 +1152,22 @@ protected Verb verb()
      * @param consistencyLevel the consistency level for the operation
      * @param requestTime object holding times when request got enqueued and started execution
      */
-    public static void mutate(List<? extends IMutation> mutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    public static void mutate(List<? extends IMutation> mutations,
+                              ConsistencyLevel consistencyLevel,
+                              Dispatcher.RequestTime requestTime,
+                              ClientRequestsMetrics metrics,
+                              ClientState state)
     throws UnavailableException, OverloadedException, WriteTimeoutException, WriteFailureException
     {
         Tracing.trace("Determining replicas for mutation");
         final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
 
+        QueryInfoTracker.WriteTracker writeTracker = queryTracker().onWrite(state, false, mutations, consistencyLevel);
+
+        // Request sensors are utilized to track usages from replicas serving a write request
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors(mutations.stream().map(IMutation::getKeyspaceName).toArray(String[]::new));
+        RequestTracker.instance.set(sensors);
+
         List<AbstractWriteResponseHandler<IMutation>> responseHandlers = new ArrayList<>(mutations.size());
         WriteType plainWriteType = mutations.size() <= 1 ? WriteType.SIMPLE : WriteType.UNLOGGED_BATCH;
 
@@ -887,22 +1175,31 @@ public static void mutate(List<? extends IMutation> mutations, ConsistencyLevel
         {
             for (IMutation mutation : mutations)
             {
+                // register the sensors for the mutation before the actual write is performed
+                for (PartitionUpdate pu: mutation.getPartitionUpdates())
+                {
+                    if (pu.metadata().isIndex()) continue;
+                    sensors.registerSensor(Context.from(pu.metadata()), Type.WRITE_BYTES);
+                }
+
                 if (mutation instanceof CounterMutation)
                     responseHandlers.add(mutateCounter((CounterMutation)mutation, localDataCenter, requestTime));
                 else
-                    responseHandlers.add(performWrite(mutation, consistencyLevel, localDataCenter, standardWritePerformer, null, plainWriteType, requestTime));
+                    responseHandlers.add(mutator.mutateStandard((Mutation)mutation, consistencyLevel, localDataCenter, standardWritePerformer, null, plainWriteType, requestTime));
             }
 
             // upgrade to full quorum any failed cheap quorums
             for (int i = 0 ; i < mutations.size() ; ++i)
             {
-                if (!(mutations.get(i) instanceof CounterMutation)) // at the moment, only non-counter writes support cheap quorums
+                if (!(mutations.get(i) instanceof CounterMutation) && mutator instanceof DefaultMutator) // at the moment, only non-counter writes support cheap quorums
                     responseHandlers.get(i).maybeTryAdditionalReplicas(mutations.get(i), standardWritePerformer, localDataCenter);
             }
 
             // wait for writes.  throws TimeoutException if necessary
             for (AbstractWriteResponseHandler<IMutation> responseHandler : responseHandlers)
                 responseHandler.get();
+
+            writeTracker.onDone();
         }
         catch (WriteTimeoutException|WriteFailureException ex)
         {
@@ -914,34 +1211,37 @@ public static void mutate(List<? extends IMutation> mutations, ConsistencyLevel
             {
                 if (ex instanceof WriteFailureException)
                 {
-                    writeMetrics.failures.mark();
-                    writeMetricsForLevel(consistencyLevel).failures.mark();
+                    metrics.writeMetrics.failures.mark();
+                    metrics.writeMetricsForLevel(consistencyLevel).failures.mark();
                     WriteFailureException fe = (WriteFailureException)ex;
                     Tracing.trace("Write failure; received {} of {} required replies, failed {} requests",
                                   fe.received, fe.blockFor, fe.failureReasonByEndpoint.size());
                 }
                 else
                 {
-                    writeMetrics.timeouts.mark();
-                    writeMetricsForLevel(consistencyLevel).timeouts.mark();
+                    metrics.writeMetrics.timeouts.mark();
+                    metrics.writeMetricsForLevel(consistencyLevel).timeouts.mark();
                     WriteTimeoutException te = (WriteTimeoutException)ex;
                     Tracing.trace("Write timeout; received {} of {} required replies", te.received, te.blockFor);
                 }
+                writeTracker.onError(ex);
                 throw ex;
             }
         }
         catch (UnavailableException e)
         {
-            writeMetrics.unavailables.mark();
-            writeMetricsForLevel(consistencyLevel).unavailables.mark();
+            metrics.writeMetrics.unavailables.mark();
+            metrics.writeMetricsForLevel(consistencyLevel).unavailables.mark();
             Tracing.trace("Unavailable");
+            writeTracker.onError(e);
             throw e;
         }
         catch (OverloadedException e)
         {
-            writeMetrics.unavailables.mark();
-            writeMetricsForLevel(consistencyLevel).unavailables.mark();
+            metrics.writeMetrics.unavailables.mark();
+            metrics.writeMetricsForLevel(consistencyLevel).unavailables.mark();
             Tracing.trace("Overloaded");
+            writeTracker.onError(e);
             throw e;
         }
         finally
@@ -949,8 +1249,10 @@ public static void mutate(List<? extends IMutation> mutations, ConsistencyLevel
             // We track latency based on request processing time, since the amount of time that request spends in the queue
             // is not a representative metric of replica performance.
             long latency = nanoTime() - requestTime.startedAtNanos();
-            writeMetrics.addNano(latency);
-            writeMetricsForLevel(consistencyLevel).addNano(latency);
+            metrics.writeMetrics.executionTimeMetrics.addNano(latency);
+            metrics.writeMetrics.serviceTimeMetrics.addNano(latency);
+            metrics.writeMetricsForLevel(consistencyLevel).executionTimeMetrics.addNano(latency);
+            metrics.writeMetricsForLevel(consistencyLevel).serviceTimeMetrics.addNano(latency);
             updateCoordinatorWriteLatencyTableMetric(mutations, latency);
         }
     }
@@ -1002,18 +1304,19 @@ public boolean appliesLocally(Mutation mutation)
      * across all replicas.
      *
      * @param mutations the mutations to be applied across the replicas
-     * @param writeCommitLog if commitlog should be written
+     * @param writeOptions describes desired write properties
      * @param baseComplete time from epoch in ms that the local base mutation was(or will be) completed
      * @param requestTime object holding times when request got enqueued and started execution
      */
-    public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations, boolean writeCommitLog, AtomicLong baseComplete, Dispatcher.RequestTime requestTime)
-    throws UnavailableException, OverloadedException, WriteTimeoutException
+    public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations, WriteOptions writeOptions, AtomicLong baseComplete, Dispatcher.RequestTime requestTime)
+            throws UnavailableException, OverloadedException, WriteTimeoutException
     {
         Tracing.trace("Determining replicas for mutation");
         final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
 
         long startTime = nanoTime();
-
+        String ks = mutations.iterator().next().getKeyspaceName();
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(ks);
 
         try
         {
@@ -1022,15 +1325,14 @@ public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations,
 
             if (StorageService.instance.isStarting() || StorageService.instance.isJoining() || StorageService.instance.isMoving())
             {
-                BatchlogManager.store(Batch.createLocal(batchUUID, FBUtilities.timestampMicros(),
-                                                        mutations), writeCommitLog);
+                BatchlogManager.store(Batch.createLocal(batchUUID, FBUtilities.timestampMicros(), mutations), writeOptions);
             }
             else
             {
                 List<WriteResponseHandlerWrapper> wrappers = new ArrayList<>(mutations.size());
                 //non-local mutations rely on the base mutation commit-log entry for eventual consistency
                 Set<Mutation> nonLocalMutations = new HashSet<>(mutations);
-                Token baseToken = StorageService.instance.getTokenMetadata().partitioner.getToken(dataKey);
+                Token baseToken = StorageService.instance.getTokenMetadataForKeyspace(ks).partitioner.getToken(dataKey);
 
                 ConsistencyLevel consistencyLevel = ConsistencyLevel.ONE;
 
@@ -1046,7 +1348,7 @@ public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations,
                     Token tk = mutation.key().getToken();
                     AbstractReplicationStrategy replicationStrategy = Keyspace.open(keyspaceName).getReplicationStrategy();
                     Optional<Replica> pairedEndpoint = ViewUtils.getViewNaturalEndpoint(replicationStrategy, baseToken, tk);
-                    EndpointsForToken pendingReplicas = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(tk, keyspaceName);
+                    EndpointsForToken pendingReplicas = StorageService.instance.getTokenMetadataForKeyspace(keyspaceName).pendingEndpointsForToken(tk, keyspaceName);
 
                     // if there are no paired endpoints there are probably range movements going on, so we write to the local batchlog to replay later
                     if (!pairedEndpoint.isPresent())
@@ -1068,7 +1370,7 @@ public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations,
                     {
                         try
                         {
-                            mutation.apply(writeCommitLog);
+                            mutation.apply(writeOptions);
                             nonLocalMutations.remove(mutation);
                             // won't trigger cleanup
                             cleanup.decrement();
@@ -1092,13 +1394,14 @@ public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations,
                                                                   baseComplete,
                                                                   WriteType.BATCH,
                                                                   cleanup,
-                                                                  requestTime));
+                                                                  requestTime,
+                                                                  metrics));
                     }
                 }
 
                 // Apply to local batchlog memtable in this thread
                 if (!nonLocalMutations.isEmpty())
-                    BatchlogManager.store(Batch.createLocal(batchUUID, FBUtilities.timestampMicros(), nonLocalMutations), writeCommitLog);
+                    BatchlogManager.store(Batch.createLocal(batchUUID, FBUtilities.timestampMicros(), nonLocalMutations), writeOptions);
 
                 // Perform remote writes
                 if (!wrappers.isEmpty())
@@ -1107,7 +1410,9 @@ public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations,
         }
         finally
         {
-            viewWriteMetrics.addNano(nanoTime() - startTime);
+            final long endTime = nanoTime();
+            metrics.viewWriteMetrics.executionTimeMetrics.addNano(endTime - startTime);
+            metrics.viewWriteMetrics.serviceTimeMetrics.addNano(endTime - requestTime.startedAtNanos());
         }
     }
 
@@ -1115,7 +1420,8 @@ public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations,
     public static void mutateWithTriggers(List<? extends IMutation> mutations,
                                           ConsistencyLevel consistencyLevel,
                                           boolean mutateAtomically,
-                                          Dispatcher.RequestTime requestTime)
+                                          Dispatcher.RequestTime requestTime,
+                                          ClientState state)
     throws WriteTimeoutException, WriteFailureException, UnavailableException, OverloadedException, InvalidRequestException
     {
         if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistWritesEnabled())
@@ -1138,23 +1444,25 @@ public static void mutateWithTriggers(List<? extends IMutation> mutations,
         }
 
         Collection<Mutation> augmented = TriggerExecutor.instance.execute(mutations);
+        String keyspaceName = mutations.iterator().next().getKeyspaceName();
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(keyspaceName);
 
         boolean updatesView = Keyspace.open(mutations.iterator().next().getKeyspaceName())
                               .viewManager
                               .updatesAffectView(mutations, true);
 
         long size = IMutation.dataSize(mutations);
-        writeMetrics.mutationSize.update(size);
-        writeMetricsForLevel(consistencyLevel).mutationSize.update(size);
+        metrics.writeMetrics.mutationSize.update(size);
+        metrics.writeMetricsForLevel(consistencyLevel).mutationSize.update(size);
 
         if (augmented != null)
-            mutateAtomically(augmented, consistencyLevel, updatesView, requestTime);
+            mutateAtomically(augmented, consistencyLevel, updatesView, requestTime, metrics, state);
         else
         {
             if (mutateAtomically || updatesView)
-                mutateAtomically((Collection<Mutation>) mutations, consistencyLevel, updatesView, requestTime);
+                mutateAtomically((Collection<Mutation>) mutations, consistencyLevel, updatesView, requestTime, metrics, state);
             else
-                mutate(mutations, consistencyLevel, requestTime);
+                mutate(mutations, consistencyLevel, requestTime, metrics, state);
         }
     }
 
@@ -1165,96 +1473,22 @@ public static void mutateWithTriggers(List<? extends IMutation> mutations,
      * After: remove the batchlog entry (after writing hints for the batch rows, if necessary).
      *
      * @param mutations the Mutations to be applied across the replicas
-     * @param consistency_level the consistency level for the operation
+     * @param consistencyLevel the consistency level for the operation
      * @param requireQuorumForRemove at least a quorum of nodes will see update before deleting batchlog
      * @param requestTime object holding times when request got enqueued and started execution
      */
     public static void mutateAtomically(Collection<Mutation> mutations,
-                                        ConsistencyLevel consistency_level,
+                                        ConsistencyLevel consistencyLevel,
                                         boolean requireQuorumForRemove,
-                                        Dispatcher.RequestTime requestTime)
+                                        Dispatcher.RequestTime requestTime,
+                                        ClientRequestsMetrics metrics,
+                                        ClientState clientState)
     throws UnavailableException, OverloadedException, WriteTimeoutException
     {
-        Tracing.trace("Determining replicas for atomic batch");
-        long startTime = nanoTime();
-
-        List<WriteResponseHandlerWrapper> wrappers = new ArrayList<>(mutations.size());
-
-        if (mutations.stream().anyMatch(mutation -> Keyspace.open(mutation.getKeyspaceName()).getReplicationStrategy().hasTransientReplicas()))
-            throw new AssertionError("Logged batches are unsupported with transient replication");
-
-        try
-        {
-
-            // If we are requiring quorum nodes for removal, we upgrade consistency level to QUORUM unless we already
-            // require ALL, or EACH_QUORUM. This is so that *at least* QUORUM nodes see the update.
-            ConsistencyLevel batchConsistencyLevel = requireQuorumForRemove
-                                                     ? ConsistencyLevel.QUORUM
-                                                     : consistency_level;
-
-            switch (consistency_level)
-            {
-                case ALL:
-                case EACH_QUORUM:
-                    batchConsistencyLevel = consistency_level;
-            }
-
-            ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forBatchlogWrite(batchConsistencyLevel == ConsistencyLevel.ANY);
-
-            final TimeUUID batchUUID = nextTimeUUID();
-            BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(),
-                                                          () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID, requestTime));
-
-            // add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
-            for (Mutation mutation : mutations)
-            {
-                WriteResponseHandlerWrapper wrapper = wrapBatchResponseHandler(mutation,
-                                                                               consistency_level,
-                                                                               batchConsistencyLevel,
-                                                                               WriteType.BATCH,
-                                                                               cleanup,
-                                                                               requestTime);
-                // exit early if we can't fulfill the CL at this time.
-                wrappers.add(wrapper);
-            }
-
-            // write to the batchlog
-            syncWriteToBatchlog(mutations, replicaPlan, batchUUID, requestTime);
-
-            // now actually perform the writes and wait for them to complete
-            syncWriteBatchedMutations(wrappers, Stage.MUTATION, requestTime);
-        }
-        catch (UnavailableException e)
-        {
-            writeMetrics.unavailables.mark();
-            writeMetricsForLevel(consistency_level).unavailables.mark();
-            Tracing.trace("Unavailable");
-            throw e;
-        }
-        catch (WriteTimeoutException e)
-        {
-            writeMetrics.timeouts.mark();
-            writeMetricsForLevel(consistency_level).timeouts.mark();
-            Tracing.trace("Write timeout; received {} of {} required replies", e.received, e.blockFor);
-            throw e;
-        }
-        catch (WriteFailureException e)
-        {
-            writeMetrics.failures.mark();
-            writeMetricsForLevel(consistency_level).failures.mark();
-            Tracing.trace("Write failure; received {} of {} required replies", e.received, e.blockFor);
-            throw e;
-        }
-        finally
-        {
-            long latency = nanoTime() - startTime;
-            writeMetrics.addNano(latency);
-            writeMetricsForLevel(consistency_level).addNano(latency);
-            updateCoordinatorWriteLatencyTableMetric(mutations, latency);
-        }
+        mutator.mutateAtomically(mutations, consistencyLevel, requireQuorumForRemove, requestTime, metrics, clientState);
     }
 
-    private static void updateCoordinatorWriteLatencyTableMetric(Collection<? extends IMutation> mutations, long latency)
+    public static void updateCoordinatorWriteLatencyTableMetric(Collection<? extends IMutation> mutations, long latency)
     {
         if (null == mutations)
         {
@@ -1304,7 +1538,7 @@ private static void syncWriteToBatchlog(Collection<Mutation> mutations, ReplicaP
         handler.get();
     }
 
-    private static void asyncRemoveFromBatchlog(ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid, Dispatcher.RequestTime requestTime)
+    protected static void asyncRemoveFromBatchlog(ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid, Dispatcher.RequestTime requestTime)
     {
         Message<TimeUUID> message = Message.out(Verb.BATCH_REMOVE_REQ, uuid);
         for (Replica target : replicaPlan.contacts())
@@ -1337,20 +1571,30 @@ private static void asyncWriteBatchedMutations(List<WriteResponseHandlerWrapper>
         }
     }
 
-    private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, Stage stage, Dispatcher.RequestTime requestTime)
-    throws WriteTimeoutException, OverloadedException
+    public static AbstractWriteResponseHandler<IMutation> getWriteResponseHandler(IMutation mutation,
+                                                                                  ConsistencyLevel consistencyLevel,
+                                                                                  @Nullable Runnable callback,
+                                                                                  WriteType writeType,
+                                                                                  Dispatcher.RequestTime requestTime)
     {
-        String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+        Keyspace keyspace = mutation.getKeyspace();
+        Token tk = mutation.key().getToken();
 
-        for (WriteResponseHandlerWrapper wrapper : wrappers)
+        ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(keyspace.getName());
+        if (replicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null)
+            metrics.writeMetrics.localRequests.mark();
+        else
+            metrics.writeMetrics.remoteRequests.mark();
+
+        AbstractReplicationStrategy rs = replicaPlan.replicationStrategy();
+
+        AbstractWriteResponseHandler<IMutation> responseHandler = rs.getWriteResponseHandler(replicaPlan, callback, writeType, mutation.hintOnFailure(), requestTime);
+        if (callback instanceof CounterMutationCallback)
         {
-            EndpointsForToken sendTo = wrapper.handler.replicaPlan.liveAndDown();
-            Replicas.temporaryAssertFull(sendTo); // TODO: CASSANDRA-14549
-            sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaPlan.withContacts(sendTo), wrapper.handler, localDataCenter, stage, requestTime);
+            ((CounterMutationCallback) callback).setReplicaCount(replicaPlan.contacts().size());
         }
-
-        for (WriteResponseHandlerWrapper wrapper : wrappers)
-            wrapper.handler.get();
+        return responseHandler;
     }
 
     /**
@@ -1375,45 +1619,22 @@ public static AbstractWriteResponseHandler<IMutation> performWrite(IMutation mut
                                                                        WriteType writeType,
                                                                        Dispatcher.RequestTime requestTime)
     {
-        String keyspaceName = mutation.getKeyspaceName();
-        Keyspace keyspace = Keyspace.open(keyspaceName);
-        Token tk = mutation.key().getToken();
-
-        ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
-
-        if (replicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null)
-            writeMetrics.localRequests.mark();
-        else
-            writeMetrics.remoteRequests.mark();
-
-        AbstractReplicationStrategy rs = replicaPlan.replicationStrategy();
-        AbstractWriteResponseHandler<IMutation> responseHandler = rs.getWriteResponseHandler(replicaPlan, callback, writeType, mutation.hintOnFailure(), requestTime);
-
-        performer.apply(mutation, replicaPlan, responseHandler, localDataCenter, requestTime);
+        AbstractWriteResponseHandler<IMutation> responseHandler = getWriteResponseHandler(mutation, consistencyLevel, callback, writeType, requestTime);
+        performer.apply(mutation, responseHandler.replicaPlan, responseHandler, localDataCenter, requestTime);
         return responseHandler;
     }
 
     // same as performWrites except does not initiate writes (but does perform availability checks).
-    private static WriteResponseHandlerWrapper wrapBatchResponseHandler(Mutation mutation,
-                                                                        ConsistencyLevel consistencyLevel,
-                                                                        ConsistencyLevel batchConsistencyLevel,
-                                                                        WriteType writeType,
-                                                                        BatchlogResponseHandler.BatchlogCleanup cleanup,
-                                                                        Dispatcher.RequestTime requestTime)
+    public static WriteResponseHandlerWrapper wrapBatchResponseHandler(Mutation mutation,
+                                                                       ConsistencyLevel consistencyLevel,
+                                                                       ConsistencyLevel batchConsistencyLevel,
+                                                                       WriteType writeType,
+                                                                       BatchlogResponseHandler.BatchlogCleanup cleanup,
+                                                                       Dispatcher.RequestTime requestTime)
     {
-        Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
-        Token tk = mutation.key().getToken();
-
-        ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
-
-        if (replicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null)
-            writeMetrics.localRequests.mark();
-        else
-            writeMetrics.remoteRequests.mark();
-
-        AbstractReplicationStrategy rs = replicaPlan.replicationStrategy();
-        AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(replicaPlan, null, writeType, mutation, requestTime);
-        BatchlogResponseHandler<IMutation> batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(rs), cleanup, requestTime);
+        AbstractWriteResponseHandler<IMutation> writeHandler = getWriteResponseHandler(mutation, consistencyLevel, null, writeType, requestTime);
+        int batchlogBlockFor = batchConsistencyLevel.blockFor(writeHandler.replicaPlan().replicationStrategy());
+        BatchlogResponseHandler<IMutation> batchHandler = new BatchlogResponseHandler<>(writeHandler, batchlogBlockFor, cleanup, requestTime);
         return new WriteResponseHandlerWrapper(batchHandler, mutation);
     }
 
@@ -1428,26 +1649,27 @@ private static WriteResponseHandlerWrapper wrapViewBatchResponseHandler(Mutation
                                                                             AtomicLong baseComplete,
                                                                             WriteType writeType,
                                                                             BatchlogResponseHandler.BatchlogCleanup cleanup,
-                                                                            Dispatcher.RequestTime requestTime)
+                                                                            Dispatcher.RequestTime requestTime,
+                                                                            ClientRequestsMetrics metrics)
     {
         Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
         ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, liveAndDown, ReplicaPlans.writeAll);
         AbstractReplicationStrategy replicationStrategy = replicaPlan.replicationStrategy();
         AbstractWriteResponseHandler<IMutation> writeHandler = replicationStrategy.getWriteResponseHandler(replicaPlan, () -> {
             long delay = Math.max(0, currentTimeMillis() - baseComplete.get());
-            viewWriteMetrics.viewWriteLatency.update(delay, MILLISECONDS);
+            metrics.viewWriteMetrics.viewWriteLatency.update(delay, MILLISECONDS);
         }, writeType, mutation, requestTime);
-        BatchlogResponseHandler<IMutation> batchHandler = new ViewWriteMetricsWrapped(writeHandler, batchConsistencyLevel.blockFor(replicationStrategy), cleanup, requestTime);
+        BatchlogResponseHandler<IMutation> batchHandler = new ViewWriteMetricsWrapped(writeHandler, batchConsistencyLevel.blockFor(replicationStrategy), cleanup, requestTime, metrics);
         return new WriteResponseHandlerWrapper(batchHandler, mutation);
     }
 
     // used by atomic_batch_mutate to decouple availability check from the write itself, caches consistency level and endpoints.
-    private static class WriteResponseHandlerWrapper
+    public static class WriteResponseHandlerWrapper
     {
-        final BatchlogResponseHandler<IMutation> handler;
-        final Mutation mutation;
+        public final BatchlogResponseHandler<IMutation> handler;
+        public final Mutation mutation;
 
-        WriteResponseHandlerWrapper(BatchlogResponseHandler<IMutation> handler, Mutation mutation)
+        public WriteResponseHandlerWrapper(BatchlogResponseHandler<IMutation> handler, Mutation mutation)
         {
             this.handler = handler;
             this.mutation = mutation;
@@ -1720,6 +1942,11 @@ protected Verb verb()
      * the write latencies at the coordinator node to make gathering point similar to the case of standard writes.
      */
     public static AbstractWriteResponseHandler<IMutation> mutateCounter(CounterMutation cm, String localDataCenter, Dispatcher.RequestTime requestTime) throws UnavailableException, OverloadedException
+    {
+        return mutator.mutateCounter(cm, localDataCenter, requestTime);
+    }
+
+    private static AbstractWriteResponseHandler<IMutation> defaultMutateCounter(CounterMutation cm, String localDataCenter, Dispatcher.RequestTime requestTime) throws UnavailableException, OverloadedException
     {
         Replica replica = findSuitableReplica(cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency());
 
@@ -1740,7 +1967,8 @@ public static AbstractWriteResponseHandler<IMutation> mutateCounter(CounterMutat
             // This host isn't a replica, so mark the request as being remote. If this host is a
             // replica, applyCounterMutationOnCoordinator() in the branch above will call performWrite(), and
             // there we'll mark a local request against the metrics.
-            writeMetrics.remoteRequests.mark();
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(keyspaceName);
+            metrics.writeMetrics.remoteRequests.mark();
 
             // Forward the actual update to the chosen leader replica
             AbstractWriteResponseHandler<IMutation> responseHandler = new WriteResponseHandler<>(ReplicaPlans.forForwardingCounterWrite(keyspace, tk, replica),
@@ -1771,7 +1999,9 @@ private static Replica findSuitableReplica(String keyspaceName, DecoratedKey key
         EndpointsForToken replicas = replicationStrategy.getNaturalReplicasForToken(key);
 
         // CASSANDRA-13043: filter out those endpoints not accepting clients yet, maybe because still bootstrapping
-        replicas = replicas.filter(replica -> StorageService.instance.isRpcReady(replica.endpoint()));
+        // We have a keyspace, so filter by affinity too
+        replicas = replicas.filter(IFailureDetector.isReplicaAlive)
+                           .filter(snitch.filterByAffinity(keyspace.getName()));
 
         // CASSANDRA-17411: filter out endpoints that are not alive
         replicas = replicas.filter(replica -> FailureDetector.instance.isAlive(replica.endpoint()));
@@ -1805,7 +2035,7 @@ private static Replica findSuitableReplica(String keyspaceName, DecoratedKey key
     public static AbstractWriteResponseHandler<IMutation> applyCounterMutationOnLeader(CounterMutation cm, String localDataCenter, Runnable callback, Dispatcher.RequestTime requestTime)
     throws UnavailableException, OverloadedException
     {
-        return performWrite(cm, cm.consistency(), localDataCenter, counterWritePerformer, callback, WriteType.COUNTER, requestTime);
+        return mutator.mutateCounterOnLeader(cm, localDataCenter, counterWritePerformer, callback, requestTime);
     }
 
     // Same as applyCounterMutationOnLeader but must with the difference that it use the MUTATION stage to execute the write (while
@@ -1813,7 +2043,7 @@ public static AbstractWriteResponseHandler<IMutation> applyCounterMutationOnLead
     public static AbstractWriteResponseHandler<IMutation> applyCounterMutationOnCoordinator(CounterMutation cm, String localDataCenter, Dispatcher.RequestTime requestTime)
     throws UnavailableException, OverloadedException
     {
-        return performWrite(cm, cm.consistency(), localDataCenter, counterWriteOnCoordinatorPerformer, null, WriteType.COUNTER, requestTime);
+        return mutator.mutateCounterOnLeader(cm, localDataCenter, counterWriteOnCoordinatorPerformer, null, requestTime);
     }
 
     private static Runnable counterWriteTask(final IMutation mutation,
@@ -1831,6 +2061,7 @@ public void runMayThrow() throws OverloadedException, WriteTimeoutException
 
                 Mutation result = ((CounterMutation) mutation).applyCounterMutation();
                 responseHandler.onResponse(null);
+                mutator.onAppliedCounter(result, responseHandler);
                 sendToHintedReplicas(result, replicaPlan, responseHandler, localDataCenter, Stage.COUNTER_MUTATION, requestTime);
             }
         };
@@ -1844,23 +2075,58 @@ private static boolean systemKeyspaceQuery(List<? extends ReadCommand> cmds)
         return true;
     }
 
-    public static RowIterator readOne(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    public static RowIterator readOne(SinglePartitionReadCommand command,
+                                      ConsistencyLevel consistencyLevel,
+                                      ClientState clientState,
+                                      Dispatcher.RequestTime requestTime,
+                                      QueryInfoTracker.ReadTracker readTracker)
     throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException
     {
-        return PartitionIterators.getOnlyElement(read(SinglePartitionReadCommand.Group.one(command), consistencyLevel, requestTime), command);
+        return PartitionIterators.getOnlyElement(read(SinglePartitionReadCommand.Group.one(command),
+                                                      consistencyLevel,
+                                                      clientState,
+                                                      requestTime,
+                                                      readTracker),
+                                                 command);
+    }
+
+    public static PartitionIterator read(SinglePartitionReadCommand.Group group,
+                                         ConsistencyLevel consistencyLevel,
+                                         ClientState clientState,
+                                         Dispatcher.RequestTime requestTime)
+    {
+        QueryInfoTracker.ReadTracker readTracker = StorageProxy.queryTracker().onRead(clientState,
+                                                                                      group.metadata(),
+                                                                                      group.queries,
+                                                                                      consistencyLevel);
+        // Request sensors are utilized to track usages from replicas serving a read request
+        RequestSensors requestSensors = SensorsFactory.instance.createRequestSensors(group.metadata().keyspace);
+        Context context = Context.from(group.metadata());
+        requestSensors.registerSensor(context, Type.READ_BYTES);
+        RequestTracker.instance.set(requestSensors);
+        PartitionIterator partitions = read(group, consistencyLevel, clientState, requestTime, readTracker);
+        partitions = PartitionIterators.filteredRowTrackingIterator(partitions, readTracker::onFilteredPartition, readTracker::onFilteredRow, readTracker::onFilteredRow);
+        return PartitionIterators.doOnClose(partitions, readTracker::onDone);
     }
 
     /**
      * Performs the actual reading of a row out of the StorageService, fetching
      * a specific set of column names from a given column family.
      */
-    public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    public static PartitionIterator read(SinglePartitionReadCommand.Group group,
+                                         ConsistencyLevel consistencyLevel,
+                                         ClientState clientState,
+                                         Dispatcher.RequestTime requestTime,
+                                         QueryInfoTracker.ReadTracker readTracker)
     throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException
     {
-        if (!isSafeToPerformRead(group.queries))
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(group.metadata().keyspace);
+        ColumnFamilyStore cfs = Keyspace.openAndGetStore(group.metadata());
+
+        if (!cfs.isReadyToServeData() && !systemKeyspaceQuery(group.queries))
         {
-            readMetrics.unavailables.mark();
-            readMetricsForLevel(consistencyLevel).unavailables.mark();
+            metrics.readMetrics.unavailables.mark();
+            metrics.readMetricsForLevel(consistencyLevel).unavailables.mark();
             IsBootstrappingException exception = new IsBootstrappingException();
             logRequestException(exception, group.queries);
             throw exception;
@@ -1880,8 +2146,8 @@ public static PartitionIterator read(SinglePartitionReadCommand.Group group, Con
         }
 
         return consistencyLevel.isSerialConsistency()
-             ? readWithPaxos(group, consistencyLevel, requestTime)
-             : readRegular(group, consistencyLevel, requestTime);
+             ? readWithPaxos(group, consistencyLevel, clientState, requestTime, readTracker)
+             : readRegular(group, consistencyLevel, requestTime, readTracker);
     }
 
     public static boolean isSafeToPerformRead(List<SinglePartitionReadCommand> queries)
@@ -1894,21 +2160,29 @@ public static boolean isSafeToPerformRead()
         return !StorageService.instance.isBootstrapMode();
     }
 
-    private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    /**
+     * Performs a read for paxos reads and paxos writes (cas method)
+     */
+    private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group,
+                                                   ConsistencyLevel consistencyLevel,
+                                                   ClientState clientState,
+                                                   Dispatcher.RequestTime requestTime,
+                                                   QueryInfoTracker.ReadTracker readTracker)
     throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
     {
         return Paxos.useV2()
                 ? Paxos.read(group, consistencyLevel, requestTime)
-                : legacyReadWithPaxos(group, consistencyLevel, requestTime);
+                : legacyReadWithPaxos(group, consistencyLevel, clientState, requestTime, readTracker);
     }
 
-    private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ClientState clientState, Dispatcher.RequestTime requestTime, QueryInfoTracker.ReadTracker readTracker)
     throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
     {
         long start = nanoTime();
         if (group.queries.size() > 1)
             throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time");
 
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(group.metadata().keyspace);
         SinglePartitionReadCommand command = group.queries.get(0);
         TableMetadata metadata = command.metadata();
         DecoratedKey key = command.partitionKey();
@@ -1939,9 +2213,11 @@ private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.
                         consistencyLevel,
                         consistencyForReplayCommitsOrFetch,
                         ConsistencyLevel.ANY,
+                        clientState,
                         requestTime,
-                        casReadMetrics,
-                        updateProposer);
+                        metrics.casReadMetrics,
+                        updateProposer,
+                        true); // skip guardrail for ANY which is blocked by CNDB
             }
             catch (WriteTimeoutException e)
             {
@@ -1952,36 +2228,46 @@ private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.
                 throw new ReadFailureException(consistencyLevel, e.received, e.blockFor, false, e.failureReasonByEndpoint);
             }
 
-            result = fetchRows(group.queries, consistencyForReplayCommitsOrFetch, requestTime);
+            result = fetchRows(group.queries, consistencyForReplayCommitsOrFetch, requestTime, readTracker);
+        }
+        catch (CasWriteUnknownResultException e)
+        {
+            metrics.casReadMetrics.unknownResult.mark();
+            readTracker.onError(e);
+            throw e;
         }
         catch (UnavailableException e)
         {
-            readMetrics.unavailables.mark();
-            casReadMetrics.unavailables.mark();
-            readMetricsForLevel(consistencyLevel).unavailables.mark();
+            metrics.readMetrics.unavailables.mark();
+            metrics.casReadMetrics.unavailables.mark();
+            metrics.readMetricsForLevel(consistencyLevel).unavailables.mark();
             logRequestException(e, group.queries);
+            readTracker.onError(e);
             throw e;
         }
         catch (ReadTimeoutException e)
         {
-            readMetrics.timeouts.mark();
-            casReadMetrics.timeouts.mark();
-            readMetricsForLevel(consistencyLevel).timeouts.mark();
+            metrics.readMetrics.timeouts.mark();
+            metrics.casReadMetrics.timeouts.mark();
+            metrics.readMetricsForLevel(consistencyLevel).timeouts.mark();
             logRequestException(e, group.queries);
+            readTracker.onError(e);
             throw e;
         }
         catch (ReadAbortException e)
         {
-            readMetrics.markAbort(e);
-            casReadMetrics.markAbort(e);
-            readMetricsForLevel(consistencyLevel).markAbort(e);
+            metrics.readMetrics.markAbort(e);
+            metrics.casReadMetrics.markAbort(e);
+            metrics.readMetricsForLevel(consistencyLevel).markAbort(e);
+            readTracker.onError(e);
             throw e;
         }
         catch (ReadFailureException e)
         {
-            readMetrics.failures.mark();
-            casReadMetrics.failures.mark();
-            readMetricsForLevel(consistencyLevel).failures.mark();
+            metrics.readMetrics.failures.mark();
+            metrics.casReadMetrics.failures.mark();
+            metrics.readMetricsForLevel(consistencyLevel).failures.mark();
+            readTracker.onError(e);
             throw e;
         }
         finally
@@ -1990,24 +2276,31 @@ private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.
             // internal paging may be composed of multiple distinct reads, whereas RequestTime relates to the single
             // client request. This is a measure of how long this specific individual read took, not total time since
             // processing of the client began.
-            long latency = nanoTime() - start;
-            readMetrics.addNano(latency);
-            casReadMetrics.addNano(latency);
-            readMetricsForLevel(consistencyLevel).addNano(latency);
+            long endTime = nanoTime();
+            long latency = endTime - start;
+            long serviceLatency = endTime - requestTime.startedAtNanos();
+            metrics.readMetrics.executionTimeMetrics.addNano(latency);
+            metrics.readMetrics.serviceTimeMetrics.addNano(serviceLatency);
+            metrics.casReadMetrics.executionTimeMetrics.addNano(latency);
+            metrics.casReadMetrics.serviceTimeMetrics.addNano(serviceLatency);
+            metrics.readMetricsForLevel(consistencyLevel).executionTimeMetrics.addNano(latency);
+            metrics.readMetricsForLevel(consistencyLevel).serviceTimeMetrics.addNano(serviceLatency);
             Keyspace.open(metadata.keyspace).getColumnFamilyStore(metadata.name).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS);
         }
-
         return result;
     }
 
-    @SuppressWarnings("resource")
-    private static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    private static PartitionIterator readRegular(SinglePartitionReadCommand.Group group,
+                                                 ConsistencyLevel consistencyLevel,
+                                                 Dispatcher.RequestTime requestTime,
+                                                 QueryInfoTracker.ReadTracker readTracker)
     throws UnavailableException, ReadFailureException, ReadTimeoutException
     {
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(group.metadata().keyspace);
         long start = nanoTime();
         try
         {
-            PartitionIterator result = fetchRows(group.queries, consistencyLevel, requestTime);
+            PartitionIterator result = fetchRows(group.queries, consistencyLevel, requestTime, readTracker);
             // Note that the only difference between the command in a group must be the partition key on which
             // they applied.
             boolean enforceStrictLiveness = group.queries.get(0).metadata().enforceStrictLiveness();
@@ -2019,27 +2312,31 @@ private static PartitionIterator readRegular(SinglePartitionReadCommand.Group gr
         }
         catch (UnavailableException e)
         {
-            readMetrics.unavailables.mark();
-            readMetricsForLevel(consistencyLevel).unavailables.mark();
+            metrics.readMetrics.unavailables.mark();
+            metrics.readMetricsForLevel(consistencyLevel).unavailables.mark();
             logRequestException(e, group.queries);
+            readTracker.onError(e);
             throw e;
         }
         catch (ReadTimeoutException e)
         {
-            readMetrics.timeouts.mark();
-            readMetricsForLevel(consistencyLevel).timeouts.mark();
+            metrics.readMetrics.timeouts.mark();
+            metrics.readMetricsForLevel(consistencyLevel).timeouts.mark();
             logRequestException(e, group.queries);
+            readTracker.onError(e);
             throw e;
         }
         catch (ReadAbortException e)
         {
-            recordReadRegularAbort(consistencyLevel, e);
+            recordReadRegularAbort(consistencyLevel, e, metrics);
+            readTracker.onError(e);
             throw e;
         }
         catch (ReadFailureException e)
         {
-            readMetrics.failures.mark();
-            readMetricsForLevel(consistencyLevel).failures.mark();
+            metrics.readMetrics.failures.mark();
+            metrics.readMetricsForLevel(consistencyLevel).failures.mark();
+            readTracker.onError(e);
             throw e;
         }
         finally
@@ -2048,19 +2345,23 @@ private static PartitionIterator readRegular(SinglePartitionReadCommand.Group gr
             // internal paging may be composed of multiple distinct reads, whereas RequestTime relates to the single
             // client request. This is a measure of how long this specific individual read took, not total time since
             // processing of the client began.
-            long latency = nanoTime() - start;
-            readMetrics.addNano(latency);
-            readMetricsForLevel(consistencyLevel).addNano(latency);
+            long endTime = nanoTime();
+            long latency = endTime - start;
+            long serviceLatency = endTime - requestTime.startedAtNanos();
+            metrics.readMetrics.executionTimeMetrics.addNano(latency);
+            metrics.readMetrics.serviceTimeMetrics.addNano(serviceLatency);
+            metrics.readMetricsForLevel(consistencyLevel).executionTimeMetrics.addNano(latency);
+            metrics.readMetricsForLevel(consistencyLevel).serviceTimeMetrics.addNano(serviceLatency);
             // TODO avoid giving every command the same latency number.  Can fix this in CASSADRA-5329
             for (ReadCommand command : group.queries)
                 Keyspace.openAndGetStore(command.metadata()).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS);
         }
     }
 
-    public static void recordReadRegularAbort(ConsistencyLevel consistencyLevel, Throwable cause)
+    public static void recordReadRegularAbort(ConsistencyLevel consistencyLevel, Throwable cause, ClientRequestsMetrics metrics)
     {
-        readMetrics.markAbort(cause);
-        readMetricsForLevel(consistencyLevel).markAbort(cause);
+        metrics.readMetrics.markAbort(cause);
+        metrics.readMetricsForLevel(consistencyLevel).markAbort(cause);
     }
 
     public static PartitionIterator concatAndBlockOnRepair(List<PartitionIterator> iterators, List<ReadRepair<?, ?>> repairs)
@@ -2104,7 +2405,8 @@ public RowIterator next()
      */
     private static PartitionIterator fetchRows(List<SinglePartitionReadCommand> commands,
                                                ConsistencyLevel consistencyLevel,
-                                               Dispatcher.RequestTime requestTime)
+                                               Dispatcher.RequestTime requestTime,
+                                               QueryInfoTracker.ReadTracker readTracker)
     throws UnavailableException, ReadFailureException, ReadTimeoutException
     {
         int cmdCount = commands.size();
@@ -2115,24 +2417,24 @@ private static PartitionIterator fetchRows(List<SinglePartitionReadCommand> comm
         // for type of speculation we'll use in this read
         for (int i=0; i<cmdCount; i++)
         {
-            reads[i] = AbstractReadExecutor.getReadExecutor(commands.get(i),
-                                                            consistencyLevel,
-                                                            requestTime);
+            SinglePartitionReadCommand command = commands.get(i);
+            reads[i] = AbstractReadExecutor.getReadExecutor(command, consistencyLevel, requestTime, readTracker);
 
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(command.metadata().keyspace);
             if (reads[i].hasLocalRead())
-                readMetrics.localRequests.mark();
+                metrics.readMetrics.localRequests.mark();
             else
-                readMetrics.remoteRequests.mark();
+                metrics.readMetrics.remoteRequests.mark();
         }
 
         // sends a data request to the closest replica, and a digest request to the others. If we have a speculating
-        // read executoe, we'll only send read requests to enough replicas to satisfy the consistency level
+        // read executor, we'll only send read requests to enough replicas to satisfy the consistency level
         for (int i=0; i<cmdCount; i++)
         {
             reads[i].executeAsync();
         }
 
-        // if we have a speculating read executor and it looks like we may not receive a response from the initial
+        // if we have a speculating read executor, and it looks like we may not receive a response from the initial
         // set of replicas we sent messages to, speculatively send an additional messages to an un-contacted replica
         for (int i=0; i<cmdCount; i++)
         {
@@ -2213,7 +2515,7 @@ protected void runMayThrow()
                 {
                     if (!command.isTrackingWarnings())
                         throw e;
-                    
+
                     response = command.createEmptyResponse();
                     readRejected = true;
                 }
@@ -2275,7 +2577,8 @@ public String description()
 
     public static PartitionIterator getRangeSlice(PartitionRangeReadCommand command,
                                                   ConsistencyLevel consistencyLevel,
-                                                  Dispatcher.RequestTime requestTime)
+                                                  Dispatcher.RequestTime requestTime,
+                                                  ClientState clientState)
     {
         if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistRangeReadsEnabled())
         {
@@ -2289,7 +2592,20 @@ public static PartitionIterator getRangeSlice(PartitionRangeReadCommand command,
                                                                 tokens));
             }
         }
-        return RangeCommands.partitions(command, consistencyLevel, requestTime);
+        QueryInfoTracker.ReadTracker readTracker = queryTracker().onRangeRead(clientState,
+                                                                              command.metadata(),
+                                                                              command,
+                                                                              consistencyLevel);
+        // Request sensors are utilized to track usages from replicas serving a range request
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors(command.metadata().keyspace);
+        Context context = Context.from(command);
+        sensors.registerSensor(context, Type.READ_BYTES);
+        RequestTracker.instance.set(sensors);
+
+        PartitionIterator partitions = RangeCommands.partitions(command, consistencyLevel, requestTime, readTracker);
+        partitions = PartitionIterators.filteredRowTrackingIterator(partitions, readTracker::onFilteredPartition, readTracker::onFilteredRow, readTracker::onFilteredRow);
+
+        return PartitionIterators.doOnClose(partitions, readTracker::onDone);
     }
 
     public Map<String, List<String>> getSchemaVersions()
@@ -2505,9 +2821,11 @@ public static boolean shouldHint(Replica replica, boolean tryEnablePersistentWin
      * @param keyspace
      * @param cfname
      * @throws UnavailableException If some of the hosts in the ring are down.
-     * @throws TimeoutException
+     * @throws TimeoutException If the truncate operation doesn't complete within the truncation timeout limit.
+     * @throws TruncateException If the truncate operation fails on some replica.
      */
-    public static void truncateBlocking(String keyspace, String cfname) throws UnavailableException, TimeoutException
+    public void truncateBlocking(String keyspace, String cfname)
+    throws UnavailableException, TimeoutException, TruncateException
     {
         logger.debug("Starting a blocking truncate operation on keyspace {}, CF {}", keyspace, cfname);
         if (isAnyStorageHostDown())
@@ -2521,7 +2839,27 @@ public static void truncateBlocking(String keyspace, String cfname) throws Unava
         }
 
         Set<InetAddressAndPort> allEndpoints = StorageService.instance.getLiveRingMembers(true);
+        truncateBlocking(allEndpoints, keyspace, cfname);
+    }
 
+    /**
+     * Performs the truncate operatoin, which effectively deletes all data from
+     * the column family cfname.
+     * This method sends truncate requests and waits for the answers. It assumes taht all endpoints
+     * are live. This is either enforced by {@link StorageProxy#truncateBlocking(String, String)} or by the CNDB
+     * override.
+     *
+     * @param allEndpoints All endpoints where to send truncate requests.
+     * @param keyspace
+     * @param cfname
+     * @throws UnavailableException If some of the hosts in the ring are down (all nodes need to be up to perform
+     *                              a truncate operation).
+     * @throws TimeoutException If the truncate operation doesn't complete within the truncation timeout limit.
+     * @throws TruncateException If the truncate operation fails on some replica.
+     */
+    public void truncateBlocking(Set<InetAddressAndPort> allEndpoints, String keyspace, String cfname)
+    throws UnavailableException, TimeoutException, TruncateException
+    {
         int blockFor = allEndpoints.size();
         final TruncateResponseHandler responseHandler = new TruncateResponseHandler(blockFor);
 
@@ -2566,16 +2904,19 @@ public void apply(IMutation mutation,
      */
     private static class ViewWriteMetricsWrapped extends BatchlogResponseHandler<IMutation>
     {
-        public ViewWriteMetricsWrapped(AbstractWriteResponseHandler<IMutation> writeHandler, int i, BatchlogCleanup cleanup, Dispatcher.RequestTime requestTime)
+        ClientRequestsMetrics metrics;
+
+        public ViewWriteMetricsWrapped(AbstractWriteResponseHandler<IMutation> writeHandler, int i, BatchlogCleanup cleanup, Dispatcher.RequestTime requestTime, ClientRequestsMetrics metrics)
         {
             super(writeHandler, i, cleanup, requestTime);
-            viewWriteMetrics.viewReplicasAttempted.inc(candidateReplicaCount());
+            this.metrics = metrics;
+            metrics.viewWriteMetrics.viewReplicasAttempted.inc(candidateReplicaCount());
         }
 
         public void onResponse(Message<IMutation> msg)
         {
             super.onResponse(msg);
-            viewWriteMetrics.viewReplicasSuccess.inc();
+            metrics.viewWriteMetrics.viewReplicasSuccess.inc();
         }
     }
 
@@ -3262,4 +3603,5 @@ public void setClientRequestSizeMetricsEnabled(boolean enabled)
     {
         DatabaseDescriptor.setClientRequestSizeMetricsEnabled(enabled);
     }
+
 }
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 63ab298463e0..03da4af4f632 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -17,8 +17,6 @@
  */
 package org.apache.cassandra.service;
 
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
 import java.io.IOError;
 import java.io.IOException;
 import java.net.InetAddress;
@@ -35,11 +33,11 @@
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Scanner;
 import java.util.Set;
@@ -92,6 +90,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.Meter;
 import org.apache.cassandra.audit.AuditLogManager;
 import org.apache.cassandra.audit.AuditLogOptions;
 import org.apache.cassandra.auth.AuthCacheService;
@@ -107,7 +106,6 @@
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.Config.PaxosStatePurging;
-import org.apache.cassandra.config.Converters;
 import org.apache.cassandra.config.DataStorageSpec;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.DurationSpec;
@@ -123,6 +121,8 @@
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.guardrails.GuardrailsConfig;
+import org.apache.cassandra.db.guardrails.GuardrailsConfigProvider;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
 import org.apache.cassandra.dht.BootStrapper;
@@ -142,11 +142,9 @@
 import org.apache.cassandra.fql.FullQueryLoggerOptionsCompositeData;
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.EndpointState;
-import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
 import org.apache.cassandra.gms.IFailureDetector;
-import org.apache.cassandra.gms.TokenSerializer;
 import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.hints.HintsService;
 import org.apache.cassandra.index.IndexStatusManager;
@@ -174,6 +172,7 @@
 import org.apache.cassandra.locator.Replicas;
 import org.apache.cassandra.locator.SystemReplicas;
 import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.locator.TokenMetadataProvider;
 import org.apache.cassandra.metrics.Sampler;
 import org.apache.cassandra.metrics.SamplingManager;
 import org.apache.cassandra.metrics.StorageMetrics;
@@ -182,6 +181,7 @@
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.repair.RepairCoordinator;
 import org.apache.cassandra.repair.SharedContext;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
 import org.apache.cassandra.schema.KeyspaceMetadata;
@@ -225,6 +225,7 @@
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.FutureCombiner;
 import org.apache.cassandra.utils.concurrent.ImmediateFuture;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 import org.apache.cassandra.utils.logging.LoggingSupportFactory;
 import org.apache.cassandra.utils.progress.ProgressEvent;
@@ -264,6 +265,7 @@
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_WRITE_SURVEY;
 import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName;
 import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily;
+import static org.apache.cassandra.io.util.FileUtils.ONE_MIB;
 import static org.apache.cassandra.net.NoPayload.noPayload;
 import static org.apache.cassandra.net.Verb.REPLICATION_DONE_REQ;
 import static org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus;
@@ -296,7 +298,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         PathUtils.setDeletionListener(path -> {
             if (isDaemonSetupCompleted())
                 PathUtils.setDeletionListener(ignore -> {});
-            else
+            else if (logger.isTraceEnabled())
                 logger.trace("Deleting file during startup: {}", path);
         });
     }
@@ -305,16 +307,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     private static int getRingDelay()
     {
-        String newdelay = CassandraRelevantProperties.RING_DELAY.getString();
-        if (newdelay != null)
-        {
-            logger.info("Overriding RING_DELAY to {}ms", newdelay);
-            return Integer.parseInt(newdelay);
-        }
-        else
-        {
-            return 30 * 1000;
-        }
+        int defaultDelay = 30 * 1000;
+        int newDelay = CassandraRelevantProperties.RING_DELAY.getInt(defaultDelay);
+        Preconditions.checkArgument(newDelay >= 0, "%s must be >= 0", CassandraRelevantProperties.RING_DELAY.getKey());
+        if (newDelay != defaultDelay)
+            logger.info("Overriding {} to {}ms", CassandraRelevantProperties.RING_DELAY.getKey(), newDelay);
+        return newDelay;
     }
 
     private static int getSchemaDelay()
@@ -331,12 +329,8 @@ private static int getSchemaDelay()
         }
     }
 
-    /* This abstraction maintains the token/endpoint metadata information */
-    private TokenMetadata tokenMetadata = new TokenMetadata();
-
-    public volatile VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(tokenMetadata.partitioner);
+    public volatile VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(TokenMetadataProvider.instance.getTokenMetadata().partitioner);
 
-    private Thread drainOnShutdown = null;
     private volatile boolean isShutdown = false;
     private final List<Runnable> preShutdownHooks = new ArrayList<>();
     private final List<Runnable> postShutdownHooks = new ArrayList<>();
@@ -347,6 +341,8 @@ private static int getSchemaDelay()
 
     private final SamplingManager samplingManager = new SamplingManager();
 
+    // Newer versions of mockito contain mockito-inline which creates an issue in our test environment. Without this
+    // change, mocking of static methods is a problem with our DTest framework
     @VisibleForTesting // this is used for dtests only, see CASSANDRA-18152
     public volatile boolean skipNotificationListeners = false;
 
@@ -427,7 +423,7 @@ public List<Range<Token>> getLocalAndPendingRanges(String ks)
         List<Range<Token>> ranges = new ArrayList<>();
         for (Replica r : keyspace.getReplicationStrategy().getAddressReplicas(broadcastAddress))
             ranges.add(r.range());
-        for (Replica r : getTokenMetadata().getPendingRanges(ks, broadcastAddress))
+        for (Replica r : getTokenMetadataForKeyspace(ks).getPendingRanges(ks, broadcastAddress))
             ranges.add(r.range());
         return ranges;
     }
@@ -520,7 +516,7 @@ public void setTokens(Collection<Token> tokens)
         SystemKeyspace.updateTokens(tokens);
         Collection<Token> localTokens = getLocalTokens();
         setGossipTokens(localTokens);
-        tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
+        getTokenMetadata().updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
         setMode(Mode.NORMAL, false);
         invalidateLocalRanges();
     }
@@ -749,68 +745,65 @@ private synchronized UUID prepareForReplacement() throws ConfigurationException
         if (state == null)
             throw new RuntimeException(String.format("Cannot replace_address %s because it doesn't exist in gossip", replaceAddress));
 
-        validateEndpointSnitch(epStates.values().iterator());
-
-        try
-        {
-            VersionedValue tokensVersionedValue = state.getApplicationState(ApplicationState.TOKENS);
-            if (tokensVersionedValue == null)
-                throw new RuntimeException(String.format("Could not find tokens for %s to replace", replaceAddress));
-
-            Collection<Token> tokens = TokenSerializer.deserialize(tokenMetadata.partitioner, new DataInputStream(new ByteArrayInputStream(tokensVersionedValue.toBytes())));
-            bootstrapTokens = validateReplacementBootstrapTokens(tokenMetadata, replaceAddress, tokens);
+        validateEndpointSnitch(epStates.keySet());
+        return replaceNodeAndOwnTokens(replaceAddress, epStates, state);
+    }
 
-            if (state.isEmptyWithoutStatus() && REPLACEMENT_ALLOW_EMPTY.getBoolean())
-            {
-                logger.warn("Gossip state not present for replacing node {}. Adding temporary entry to continue.", replaceAddress);
-
-                // When replacing a node, we take ownership of all its tokens.
-                // If that node is currently down and not present in the gossip info
-                // of any other live peers, then we will not be able to take ownership
-                // of its tokens during bootstrap as they have no way of being propagated
-                // to this node's TokenMetadata. TM is loaded at startup (in which case
-                // it will be/ empty for a new replacement node) and only updated with
-                // tokens for an endpoint during normal state propagation (which will not
-                // occur if no peers have gossip state for it).
-                // However, the presence of host id and tokens in the system tables implies
-                // that the node managed to complete bootstrap at some point in the past.
-                // Peers may include this information loaded directly from system tables
-                // in a GossipDigestAck *only if* the GossipDigestSyn was sent as part of a
-                // shadow round (otherwise, a GossipDigestAck contains only state about peers
-                // learned via gossip).
-                // It is safe to do this here as since we completed a shadow round we know
-                // that :
-                // * replaceAddress successfully bootstrapped at some point and owned these
-                //   tokens
-                // * we know that no other node currently owns these tokens
-                // * we are going to completely take over replaceAddress's ownership of
-                //   these tokens.
-                tokenMetadata.updateNormalTokens(bootstrapTokens, replaceAddress);
-                UUID hostId = Gossiper.instance.getHostId(replaceAddress, epStates);
-                if (hostId != null)
-                    tokenMetadata.updateHostId(hostId, replaceAddress);
-
-                // If we were only able to learn about the node being replaced through the
-                // shadow gossip round (i.e. there is no state in gossip across the cluster
-                // about it, perhaps because the entire cluster has been bounced since it went
-                // down), then we're safe to proceed with the replacement. In this case, there
-                // will be no local endpoint state as we discard the results of the shadow
-                // round after preparing replacement info. We inject a minimal EndpointState
-                // to keep FailureDetector::isAlive and Gossiper::compareEndpointStartup from
-                // failing later in the replacement, as they both expect the replaced node to
-                // be fully present in gossip.
-                // Otherwise, if the replaced node is present in gossip, we need check that
-                // it is not in fact live.
-                // We choose to not include the EndpointState provided during the shadow round
-                // as its possible to include more state than is desired, so by creating a
-                // new empty endpoint without that information we can control what is in our
-                // local gossip state
-                Gossiper.instance.initializeUnreachableNodeUnsafe(replaceAddress);
-            }
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException(e);
+    @VisibleForTesting
+    UUID replaceNodeAndOwnTokens(InetAddressAndPort replaceAddress, Map<InetAddressAndPort, EndpointState> epStates, EndpointState state)
+    {
+        Collection<Token> tokens = state.getTokens(getTokenMetadata().partitioner);
+        if (tokens == null)
+            throw new RuntimeException(String.format("Could not find tokens for %s to replace", replaceAddress));
+
+        bootstrapTokens = validateReplacementBootstrapTokens(getTokenMetadata(), replaceAddress, tokens);
+
+        if (state.isEmptyWithoutStatus() && REPLACEMENT_ALLOW_EMPTY.getBoolean())
+        {
+            logger.warn("Gossip state not present for replacing node {}. Adding temporary entry to continue.", replaceAddress);
+
+            // When replacing a node, we take ownership of all its tokens.
+            // If that node is currently down and not present in the gossip info
+            // of any other live peers, then we will not be able to take ownership
+            // of its tokens during bootstrap as they have no way of being propagated
+            // to this node's TokenMetadata. TM is loaded at startup (in which case
+            // it will be/ empty for a new replacement node) and only updated with
+            // tokens for an endpoint during normal state propagation (which will not
+            // occur if no peers have gossip state for it).
+            // However, the presence of host id and tokens in the system tables implies
+            // that the node managed to complete bootstrap at some point in the past.
+            // Peers may include this information loaded directly from system tables
+            // in a GossipDigestAck *only if* the GossipDigestSyn was sent as part of a
+            // shadow round (otherwise, a GossipDigestAck contains only state about peers
+            // learned via gossip).
+            // It is safe to do this here as since we completed a shadow round we know
+            // that :
+            // * replaceAddress successfully bootstrapped at some point and owned these
+            //   tokens
+            // * we know that no other node currently owns these tokens
+            // * we are going to completely take over replaceAddress's ownership of
+            //   these tokens.
+            getTokenMetadata().updateNormalTokens(bootstrapTokens, replaceAddress);
+            UUID hostId = Gossiper.instance.getHostId(replaceAddress, epStates);
+            if (hostId != null)
+                getTokenMetadata().updateHostId(hostId, replaceAddress);
+
+            // If we were only able to learn about the node being replaced through the
+            // shadow gossip round (i.e. there is no state in gossip across the cluster
+            // about it, perhaps because the entire cluster has been bounced since it went
+            // down), then we're safe to proceed with the replacement. In this case, there
+            // will be no local endpoint state as we discard the results of the shadow
+            // round after preparing replacement info. We inject a minimal EndpointState
+            // to keep IFailureDetector::isAlive and Gossiper::compareEndpointStartup from
+            // failing later in the replacement, as they both expect the replaced node to
+            // be fully present in gossip.
+            // Otherwise, if the replaced node is present in gossip, we need check that
+            // it is not in fact live.
+            // We choose to not include the EndpointState provided during the shadow round
+            // as its possible to include more state than is desired, so by creating a
+            // new empty endpoint without that information we can control what is in our
+            // local gossip state
+            Gossiper.instance.initializeUnreachableNodeUnsafe(replaceAddress);
         }
 
         UUID localHostId = SystemKeyspace.getOrInitializeLocalHostId();
@@ -865,7 +858,7 @@ public synchronized void checkForEndpointCollision(UUID localHostId, Set<InetAdd
         Map<InetAddressAndPort, EndpointState> epStates = Gossiper.instance.doShadowRound(peers);
 
         if (epStates.isEmpty() && DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddressAndPort()))
-            logger.info("Unable to gossip with any peers but continuing anyway since node is in its own seed list");
+            logger.info("Unable to gossip with any peers but continuing anyway since node is in its own seed list. Broadcast address: {}, seeds: {}", FBUtilities.getBroadcastAddressAndPort(), DatabaseDescriptor.getSeeds());
 
         // If bootstrapping, check whether any previously known status for the endpoint makes it unsafe to do so.
         // If not bootstrapping, compare the host id for this endpoint learned from gossip (if any) with the local
@@ -878,7 +871,7 @@ public synchronized void checkForEndpointCollision(UUID localHostId, Set<InetAdd
                                                      FBUtilities.getBroadcastAddressAndPort(), REPLACE_ADDRESS.getKey()));
         }
 
-        validateEndpointSnitch(epStates.values().iterator());
+        validateEndpointSnitch(epStates.keySet());
 
         if (shouldBootstrap() && useStrictConsistency && !allowSimultaneousMoves())
         {
@@ -903,20 +896,16 @@ public synchronized void checkForEndpointCollision(UUID localHostId, Set<InetAdd
         }
     }
 
-    private static void validateEndpointSnitch(Iterator<EndpointState> endpointStates)
+    private static void validateEndpointSnitch(Collection<InetAddressAndPort> endpoints)
     {
         Set<String> datacenters = new HashSet<>();
         Set<String> racks = new HashSet<>();
-        while (endpointStates.hasNext())
-        {
-            EndpointState state = endpointStates.next();
-            VersionedValue val = state.getApplicationState(ApplicationState.DC);
-            if (val != null)
-                datacenters.add(val.value);
-            val = state.getApplicationState(ApplicationState.RACK);
-            if (val != null)
-                racks.add(val.value);
-        }
+        endpoints.stream().map(Nodes::localOrPeerInfo).filter(Objects::nonNull).forEach(nodeInfo -> {
+            if (nodeInfo.getDataCenter() != null)
+                datacenters.add(nodeInfo.getDataCenter());
+            if (nodeInfo.getRack() != null)
+                racks.add(nodeInfo.getRack());
+        });
 
         IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
         if (!snitch.validate(datacenters, racks))
@@ -973,14 +962,14 @@ public synchronized void initServer(int schemaTimeoutMillis, int ringTimeoutMill
 
         if (LOAD_RING_STATE.getBoolean())
         {
-            logger.info("Loading persisted ring state");
+            logger.debug("Loading persisted ring state");
             populatePeerTokenMetadata();
-            for (InetAddressAndPort endpoint : tokenMetadata.getAllEndpoints())
+            for (InetAddressAndPort endpoint : getTokenMetadata().getAllEndpoints())
                 Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.addSavedEndpoint(endpoint));
         }
 
         // daemon threads, like our executors', continue to run while shutdown hooks are invoked
-        drainOnShutdown = NamedThreadFactory.createThread(new WrappedRunnable()
+        Thread drainOnShutdown = NamedThreadFactory.createThread(new WrappedRunnable()
         {
             @Override
             public void runMayThrow() throws InterruptedException, ExecutionException, IOException
@@ -1001,7 +990,7 @@ public void runMayThrow() throws InterruptedException, ExecutionException, IOExc
                 }
             }
         }, "StorageServiceShutdownHook");
-        Runtime.getRuntime().addShutdownHook(drainOnShutdown);
+        JVMStabilityInspector.registerShutdownHook(drainOnShutdown, this::onShutdownHookRemoved);
 
         replacing = isReplacing();
 
@@ -1034,7 +1023,7 @@ public void runMayThrow() throws InterruptedException, ExecutionException, IOExc
             Collection<Token> tokens = SystemKeyspace.getSavedTokens();
             if (!tokens.isEmpty())
             {
-                tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
+                getTokenMetadata().updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
                 // order is important here, the gossiper can fire in between adding these two states.  It's ok to send TOKENS without STATUS, but *not* vice versa.
                 List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<Pair<ApplicationState, VersionedValue>>();
                 states.add(Pair.create(ApplicationState.TOKENS, valueFactory.tokens(tokens)));
@@ -1064,15 +1053,15 @@ public void populateTokenMetadata()
             populatePeerTokenMetadata();
             // if we have not completed bootstrapping, we should not add ourselves as a normal token
             if (!shouldBootstrap())
-                tokenMetadata.updateNormalTokens(SystemKeyspace.getSavedTokens(), FBUtilities.getBroadcastAddressAndPort());
+                getTokenMetadata().updateNormalTokens(SystemKeyspace.getSavedTokens(), FBUtilities.getBroadcastAddressAndPort());
 
-            logger.info("Token metadata: {}", tokenMetadata);
+            logger.info("Token metadata: {}", getTokenMetadata());
         }
     }
 
     private void populatePeerTokenMetadata()
     {
-        logger.info("Populating token metadata from system tables");
+        logger.debug("Populating token metadata from system tables");
         Multimap<InetAddressAndPort, Token> loadedTokens = SystemKeyspace.loadTokens();
 
         // entry has been mistakenly added, delete it
@@ -1087,8 +1076,8 @@ private void populatePeerTokenMetadata()
             if (hostId != null)
                 hostIdToEndpointMap.put(hostId, ep);
         }
-        tokenMetadata.updateNormalTokens(loadedTokens);
-        tokenMetadata.updateHostIds(hostIdToEndpointMap);
+        getTokenMetadata().updateNormalTokens(loadedTokens);
+        getTokenMetadata().updateHostIds(hostIdToEndpointMap);
     }
 
     public boolean isReplacing()
@@ -1108,12 +1097,9 @@ public boolean isReplacing()
     /**
      * In the event of forceful termination we need to remove the shutdown hook to prevent hanging (OOM for instance)
      */
-    public void removeShutdownHook()
+    public void onShutdownHookRemoved()
     {
         PathUtils.clearOnExitThreads();
-
-        if (drainOnShutdown != null)
-            Runtime.getRuntime().removeShutdownHook(drainOnShutdown);
     }
 
     private boolean shouldBootstrap()
@@ -1202,7 +1188,7 @@ else if (isReplacingSameAddress())
             appStates.put(ApplicationState.RELEASE_VERSION, valueFactory.releaseVersion());
             appStates.put(ApplicationState.SSTABLE_VERSIONS, valueFactory.sstableVersions(sstablesTracker.versionsInUse()));
 
-            logger.info("Starting up server gossip");
+            logger.debug("Starting up server gossip");
             Gossiper.instance.register(this);
             Gossiper.instance.start(SystemKeyspace.incrementAndGetGeneration(), appStates); // needed for node-ring gathering.
             gossipActive = true;
@@ -1295,14 +1281,14 @@ public void joinTokenRing(boolean finishJoiningRing,
             bootstrapTokens = SystemKeyspace.getSavedTokens();
             if (bootstrapTokens.isEmpty())
             {
-                bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddressAndPort(), schemaTimeoutMillis, ringTimeoutMillis);
+                bootstrapTokens = BootStrapper.getBootstrapTokens(getTokenMetadata(), FBUtilities.getBroadcastAddressAndPort(), schemaTimeoutMillis, ringTimeoutMillis);
             }
             else
             {
                 if (bootstrapTokens.size() != DatabaseDescriptor.getNumTokens())
                     throw new ConfigurationException("Cannot change the number of tokens from " + bootstrapTokens.size() + " to " + DatabaseDescriptor.getNumTokens());
                 else
-                    logger.info("Using saved tokens {}", bootstrapTokens);
+                    logger.debug("Using saved tokens {}", bootstrapTokens);
             }
         }
 
@@ -1414,7 +1400,7 @@ public void finishJoiningRing(boolean didBootstrap, Collection<Token> tokens)
         executePreJoinTasks(didBootstrap);
         setTokens(tokens);
 
-        assert tokenMetadata.sortedTokens().size() > 0;
+        assert getTokenMetadata().sortedTokens().size() > 0;
     }
 
     @VisibleForTesting
@@ -1460,7 +1446,7 @@ public void setUpDistributedSystemKeyspaces()
 
     public boolean isJoined()
     {
-        return tokenMetadata.isMember(FBUtilities.getBroadcastAddressAndPort()) && !isSurveyMode;
+        return getTokenMetadata().isMember(FBUtilities.getBroadcastAddressAndPort()) && !isSurveyMode;
     }
 
     public void rebuild(String sourceDc)
@@ -1511,7 +1497,7 @@ public void rebuild(String sourceDc, String keyspace, String tokens, String spec
 
             repairPaxosForTopologyChange("rebuild");
 
-            RangeStreamer streamer = new RangeStreamer(tokenMetadata,
+            RangeStreamer streamer = new RangeStreamer(getTokenMetadata(),
                                                        null,
                                                        FBUtilities.getBroadcastAddressAndPort(),
                                                        StreamOperation.REBUILD,
@@ -1892,6 +1878,21 @@ public void setCompactionThroughputMbPerSec(int value)
                     value, oldValue);
     }
 
+    /**
+     * Get the Current Compaction Throughput
+     * key is 1/5/15minute time dimension for statistics
+     * value is the metric double string (unit is:mib/s)
+     */
+    public Map<String, String> getCurrentCompactionThroughputMebibytesPerSec()
+    {
+        HashMap<String, String> result = new LinkedHashMap<>();
+        Meter rate = CompactionManager.instance.getCompactionThroughput();
+        result.put("1minute", String.format("%.3f", rate.getOneMinuteRate() / ONE_MIB));
+        result.put("5minute", String.format("%.3f", rate.getFiveMinuteRate() / ONE_MIB));
+        result.put("15minute", String.format("%.3f", rate.getFifteenMinuteRate() / ONE_MIB));
+        return result;
+    }
+
     public int getBatchlogReplayThrottleInKB()
     {
         return DatabaseDescriptor.getBatchlogReplayThrottleInKiB();
@@ -2048,14 +2049,14 @@ public Collection<InetAddressAndPort> prepareForBootstrap(long schemaTimeoutMill
 
         if (useStrictConsistency && !allowSimultaneousMoves() &&
             (
-            tokenMetadata.getBootstrapTokens().valueSet().size() > 0 ||
-            tokenMetadata.getSizeOfLeavingEndpoints() > 0 ||
-            tokenMetadata.getSizeOfMovingEndpoints() > 0
+            getTokenMetadata().getBootstrapTokens().valueSet().size() > 0 ||
+            getTokenMetadata().getSizeOfLeavingEndpoints() > 0 ||
+            getTokenMetadata().getSizeOfMovingEndpoints() > 0
             ))
         {
-            String bootstrapTokens = StringUtils.join(tokenMetadata.getBootstrapTokens().valueSet(), ',');
-            String leavingTokens = StringUtils.join(tokenMetadata.getLeavingEndpoints(), ',');
-            String movingTokens = StringUtils.join(tokenMetadata.getMovingEndpoints().stream().map(e -> e.right).toArray(), ',');
+            String bootstrapTokens = StringUtils.join(getTokenMetadata().getBootstrapTokens().valueSet(), ',');
+            String leavingTokens = StringUtils.join(getTokenMetadata().getLeavingEndpoints(), ',');
+            String movingTokens = StringUtils.join(getTokenMetadata().getMovingEndpoints().stream().map(e -> e.right).toArray(), ',');
             throw new UnsupportedOperationException(String.format("Other bootstrapping/leaving/moving nodes detected, cannot bootstrap while %s is true. Nodes detected, bootstrapping: %s; leaving: %s; moving: %s;",
                                                                   CONSISTENT_RANGE_MOVEMENT.getKey(), bootstrapTokens, leavingTokens, movingTokens));
         }
@@ -2063,13 +2064,13 @@ public Collection<InetAddressAndPort> prepareForBootstrap(long schemaTimeoutMill
         // get bootstrap tokens
         if (!replacing)
         {
-            if (tokenMetadata.isMember(FBUtilities.getBroadcastAddressAndPort()))
+            if (getTokenMetadata().isMember(FBUtilities.getBroadcastAddressAndPort()))
             {
                 String s = "This node is already a member of the token ring; bootstrap aborted. (If replacing a dead node, remove the old one from the ring first.)";
                 throw new UnsupportedOperationException(s);
             }
             setMode(Mode.JOINING, "getting bootstrap token", true);
-            bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddressAndPort(), schemaTimeoutMillis, ringTimeoutMillis);
+            bootstrapTokens = BootStrapper.getBootstrapTokens(getTokenMetadata(), FBUtilities.getBroadcastAddressAndPort(), schemaTimeoutMillis, ringTimeoutMillis);
         }
         else
         {
@@ -2097,7 +2098,7 @@ public Collection<InetAddressAndPort> prepareForBootstrap(long schemaTimeoutMill
                 long nanoDelay = MILLISECONDS.toNanos(ringTimeoutMillis);
                 for (Token token : bootstrapTokens)
                 {
-                    InetAddressAndPort existing = tokenMetadata.getEndpoint(token);
+                    InetAddressAndPort existing = getTokenMetadata().getEndpoint(token);
                     if (existing != null)
                     {
                         EndpointState endpointStateForExisting = Gossiper.instance.getEndpointStateForEndpoint(existing);
@@ -2169,11 +2170,15 @@ public boolean bootstrap(final Collection<Token> tokens, long bootstrapTimeoutMi
         else
         {
             // Dont set any state for the node which is bootstrapping the existing token...
-            tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
+            getTokenMetadata().updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
             SystemKeyspace.removeEndpoint(DatabaseDescriptor.getReplaceAddress());
         }
         if (!Gossiper.instance.seenAnySeed())
+        {
+            logger.info("Announcing shutdown to get out of the hibernation deadlock");
+            Gossiper.instance.announceShutdown();
             throw new IllegalStateException("Unable to contact any seeds: " + Gossiper.instance.getSeeds());
+        }
 
         if (RESET_BOOTSTRAP_PROGRESS.getBoolean())
         {
@@ -2212,7 +2217,7 @@ public Future<StreamState> startBootstrap(Collection<Token> tokens)
     public Future<StreamState> startBootstrap(Collection<Token> tokens, boolean replacing)
     {
         setMode(Mode.JOINING, "Starting to bootstrap...", true);
-        BootStrapper bootstrapper = new BootStrapper(FBUtilities.getBroadcastAddressAndPort(), tokens, tokenMetadata);
+        BootStrapper bootstrapper = new BootStrapper(FBUtilities.getBroadcastAddressAndPort(), tokens, getTokenMetadata());
         bootstrapper.addProgressListener(progressSupport);
         return bootstrapper.bootstrap(streamStateStore, useStrictConsistency && !replacing); // handles token update
     }
@@ -2225,7 +2230,7 @@ private void invalidateLocalRanges()
             {
                 for (final ColumnFamilyStore store : cfs.concatWithIndexes())
                 {
-                    store.invalidateLocalRanges();
+                    store.invalidateLocalRangesAndDiskBoundaries();
                 }
             }
         }
@@ -2267,7 +2272,7 @@ public boolean resumeBootstrap()
             // get bootstrap tokens saved in system keyspace
             final Collection<Token> tokens = SystemKeyspace.getSavedTokens();
             // already bootstrapped ranges are filtered during bootstrap
-            BootStrapper bootstrapper = new BootStrapper(FBUtilities.getBroadcastAddressAndPort(), tokens, tokenMetadata);
+            BootStrapper bootstrapper = new BootStrapper(FBUtilities.getBroadcastAddressAndPort(), tokens, getTokenMetadata());
             bootstrapper.addProgressListener(progressSupport);
             Future<StreamState> bootstrapStream = bootstrapper.bootstrap(streamStateStore, useStrictConsistency && !replacing); // handles token update
             bootstrapStream.addCallback(new FutureCallback<StreamState>()
@@ -2350,7 +2355,12 @@ public boolean isBootstrapMode()
 
     public TokenMetadata getTokenMetadata()
     {
-        return tokenMetadata;
+        return TokenMetadataProvider.instance.getTokenMetadata();
+    }
+
+    public TokenMetadata getTokenMetadataForKeyspace(String keyspaceName)
+    {
+        return TokenMetadataProvider.instance.getTokenMetadataForKeyspace(keyspaceName);
     }
 
     public Map<List<String>, List<String>> getRangeToEndpointMap(String keyspace)
@@ -2379,21 +2389,20 @@ public Map<List<String>, List<String>> getRangeToEndpointMap(String keyspace, bo
         return map;
     }
 
-    /**
-     * Return the native address associated with an endpoint as a string.
-     * @param endpoint The endpoint to get rpc address for
-     * @return the native address
-     */
-    public String getNativeaddress(InetAddressAndPort endpoint, boolean withPort)
+    public InetAddressAndPort getNativeAddressAndPort(InetAddressAndPort endpoint)
     {
+        InetAddressAndPort addr = Nodes.getNativeTransportAddressAndPort(endpoint, null);
+        if (addr != null)
+            return addr;
+
         if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
-            return FBUtilities.getBroadcastNativeAddressAndPort().getHostAddress(withPort);
+            return FBUtilities.getBroadcastNativeAddressAndPort();
         else if (Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT) != null)
         {
             try
             {
                 InetAddressAndPort address = InetAddressAndPort.getByName(Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT).value);
-                return address.getHostAddress(withPort);
+                return address;
             }
             catch (UnknownHostException e)
             {
@@ -2402,29 +2411,39 @@ else if (Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationS
         }
         else
         {
-             final String ipAddress;
-             // If RPC_ADDRESS present in gossip for this endpoint use it.  This is expected for 3.x nodes.
-             if (Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS) != null)
-             {
-                 ipAddress = Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS).value;
-             }
-             else
-             {
-                 // otherwise just use the IP of the endpoint itself.
-                 ipAddress = endpoint.getHostAddress(false);
-             }
-
-             // include the configured native_transport_port.
-             try
-             {
-                 InetAddressAndPort address = InetAddressAndPort.getByNameOverrideDefaults(ipAddress, DatabaseDescriptor.getNativeTransportPort());
-                 return address.getHostAddress(withPort);
-             }
-             catch (UnknownHostException e)
-             {
-                 throw new RuntimeException(e);
-             }
-         }
+            final String ipAddress;
+            // If RPC_ADDRESS present in gossip for this endpoint use it.  This is expected for 3.x nodes.
+            if (Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS) != null)
+            {
+                ipAddress = Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS).value;
+            }
+            else
+            {
+                // otherwise just use the IP of the endpoint itself.
+                ipAddress = endpoint.getHostAddress(false);
+            }
+
+            // include the configured native_transport_port.
+            try
+            {
+                InetAddressAndPort address = InetAddressAndPort.getByNameOverrideDefaults(ipAddress, DatabaseDescriptor.getNativeTransportPort());
+                return address;
+            }
+            catch (UnknownHostException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    /**
+     * Return the native address associated with an endpoint as a string.
+     * @param endpoint The endpoint to get rpc address for
+     * @return the native address
+     */
+    public String getNativeAddress(InetAddressAndPort endpoint, boolean withPort)
+    {
+        return getNativeAddressAndPort(endpoint).getHostAddress(withPort);
     }
 
     public Map<List<String>, List<String>> getRangeToRpcaddressMap(String keyspace)
@@ -2451,7 +2470,7 @@ private Map<List<String>, List<String>> getRangeToNativeaddressMap(String keyspa
             List<String> rpcaddrs = new ArrayList<>(entry.getValue().size());
             for (Replica replicas: entry.getValue())
             {
-                rpcaddrs.add(getNativeaddress(replicas.endpoint(), withPort));
+                rpcaddrs.add(getNativeAddress(replicas.endpoint(), withPort));
             }
             map.put(entry.getKey().asList(), rpcaddrs);
         }
@@ -2476,7 +2495,7 @@ private Map<List<String>, List<String>> getPendingRangeToEndpointMap(String keys
             keyspace = Schema.instance.distributedKeyspaces().iterator().next().name;
 
         Map<List<String>, List<String>> map = new HashMap<>();
-        for (Map.Entry<Range<Token>, EndpointsForRange> entry : tokenMetadata.getPendingRangesMM(keyspace).asMap().entrySet())
+        for (Map.Entry<Range<Token>, EndpointsForRange> entry : getTokenMetadata().getPendingRangesMM(keyspace).asMap().entrySet())
         {
             map.put(entry.getKey().asList(), Replicas.stringify(entry.getValue(), withPort));
         }
@@ -2485,7 +2504,7 @@ private Map<List<String>, List<String>> getPendingRangeToEndpointMap(String keys
 
     public EndpointsByRange getRangeToAddressMap(String keyspace)
     {
-        return getRangeToAddressMap(keyspace, tokenMetadata.sortedTokens());
+        return getRangeToAddressMap(keyspace, getTokenMetadataForKeyspace(keyspace).sortedTokens());
     }
 
     public EndpointsByRange getRangeToAddressMapInLocalDC(String keyspace)
@@ -2506,9 +2525,9 @@ public EndpointsByRange getRangeToAddressMapInLocalDC(String keyspace)
     private List<Token> getTokensInLocalDC()
     {
         List<Token> filteredTokens = Lists.newArrayList();
-        for (Token token : tokenMetadata.sortedTokens())
+        for (Token token : getTokenMetadata().sortedTokens())
         {
-            InetAddressAndPort endpoint = tokenMetadata.getEndpoint(token);
+            InetAddressAndPort endpoint = getTokenMetadata().getEndpoint(token);
             if (isLocalDC(endpoint))
                 filteredTokens.add(token);
         }
@@ -2626,7 +2645,7 @@ public Map<String, String> getTokenToEndpointWithPortMap()
 
     private Map<String, String> getTokenToEndpointMap(boolean withPort)
     {
-        Map<Token, InetAddressAndPort> mapInetAddress = tokenMetadata.getNormalAndBootstrappingTokenToEndpointMap();
+        Map<Token, InetAddressAndPort> mapInetAddress = getTokenMetadata().getNormalAndBootstrappingTokenToEndpointMap();
         // in order to preserve tokens in ascending order, we use LinkedHashMap here
         Map<String, String> mapString = new LinkedHashMap<>(mapInetAddress.size());
         List<Token> tokens = new ArrayList<>(mapInetAddress.keySet());
@@ -2808,43 +2827,9 @@ public void onChange(InetAddressAndPort endpoint, ApplicationState state, Versio
             {
                 switch (state)
                 {
-                    case RELEASE_VERSION:
-                        SystemKeyspace.updatePeerInfo(endpoint, "release_version", value.value);
-                        break;
                     case DC:
-                        updateTopology(endpoint);
-                        SystemKeyspace.updatePeerInfo(endpoint, "data_center", value.value);
-                        break;
                     case RACK:
                         updateTopology(endpoint);
-                        SystemKeyspace.updatePeerInfo(endpoint, "rack", value.value);
-                        break;
-                    case RPC_ADDRESS:
-                        try
-                        {
-                            SystemKeyspace.updatePeerInfo(endpoint, "rpc_address", InetAddress.getByName(value.value));
-                        }
-                        catch (UnknownHostException e)
-                        {
-                            throw new RuntimeException(e);
-                        }
-                        break;
-                    case NATIVE_ADDRESS_AND_PORT:
-                        try
-                        {
-                            InetAddressAndPort address = InetAddressAndPort.getByName(value.value);
-                            SystemKeyspace.updatePeerNativeAddress(endpoint, address);
-                        }
-                        catch (UnknownHostException e)
-                        {
-                            throw new RuntimeException(e);
-                        }
-                        break;
-                    case SCHEMA:
-                        SystemKeyspace.updatePeerInfo(endpoint, "schema_version", UUID.fromString(value.value));
-                        break;
-                    case HOST_ID:
-                        SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(value.value));
                         break;
                     case RPC_READY:
                         notifyRpcChange(endpoint, epState.isRpcReady());
@@ -2897,71 +2882,6 @@ public void updateTopology()
         getTokenMetadata().updateTopology();
     }
 
-    private void updatePeerInfo(InetAddressAndPort endpoint)
-    {
-        EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
-        InetAddress native_address = null;
-        int native_port = DatabaseDescriptor.getNativeTransportPort();
-
-        for (Map.Entry<ApplicationState, VersionedValue> entry : epState.states())
-        {
-            switch (entry.getKey())
-            {
-                case RELEASE_VERSION:
-                    SystemKeyspace.updatePeerInfo(endpoint, "release_version", entry.getValue().value);
-                    break;
-                case DC:
-                    SystemKeyspace.updatePeerInfo(endpoint, "data_center", entry.getValue().value);
-                    break;
-                case RACK:
-                    SystemKeyspace.updatePeerInfo(endpoint, "rack", entry.getValue().value);
-                    break;
-                case RPC_ADDRESS:
-                    try
-                    {
-                        native_address = InetAddress.getByName(entry.getValue().value);
-                    }
-                    catch (UnknownHostException e)
-                    {
-                        throw new RuntimeException(e);
-                    }
-                    break;
-                case NATIVE_ADDRESS_AND_PORT:
-                    try
-                    {
-                        InetAddressAndPort address = InetAddressAndPort.getByName(entry.getValue().value);
-                        native_address = address.getAddress();
-                        native_port = address.getPort();
-                    }
-                    catch (UnknownHostException e)
-                    {
-                        throw new RuntimeException(e);
-                    }
-                    break;
-                case SCHEMA:
-                    SystemKeyspace.updatePeerInfo(endpoint, "schema_version", UUID.fromString(entry.getValue().value));
-                    break;
-                case HOST_ID:
-                    SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(entry.getValue().value));
-                    break;
-                case INDEX_STATUS:
-                    // Need to set the peer index status in SIM here
-                    // to ensure the status is correct before the node
-                    // fully joins the ring
-                    updateIndexStatus(endpoint, entry.getValue());
-                    break;
-            }
-        }
-
-        //Some tests won't set all the states
-        if (native_address != null)
-        {
-            SystemKeyspace.updatePeerNativeAddress(endpoint,
-                                                   InetAddressAndPort.getByAddressOverrideDefaults(native_address,
-                                                                                                   native_port));
-        }
-    }
-
     private void notifyRpcChange(InetAddressAndPort endpoint, boolean ready)
     {
         if (ready)
@@ -3038,22 +2958,7 @@ public void setRpcReady(boolean value)
 
     private Collection<Token> getTokensFor(InetAddressAndPort endpoint)
     {
-        try
-        {
-            EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
-            if (state == null)
-                return Collections.emptyList();
-
-            VersionedValue versionedValue = state.getApplicationState(ApplicationState.TOKENS);
-            if (versionedValue == null)
-                return Collections.emptyList();
-
-            return TokenSerializer.deserialize(tokenMetadata.partitioner, new DataInputStream(new ByteArrayInputStream(versionedValue.toBytes())));
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException(e);
-        }
+        return Gossiper.instance.getTokensFor(endpoint, getTokenMetadata().partitioner);
     }
 
     /**
@@ -3073,22 +2978,22 @@ private void handleStateBootstrap(InetAddressAndPort endpoint)
         // if this node is present in token metadata, either we have missed intermediate states
         // or the node had crashed. Print warning if needed, clear obsolete stuff and
         // continue.
-        if (tokenMetadata.isMember(endpoint))
+        if (getTokenMetadata().isMember(endpoint))
         {
             // If isLeaving is false, we have missed both LEAVING and LEFT. However, if
             // isLeaving is true, we have only missed LEFT. Waiting time between completing
             // leave operation and rebootstrapping is relatively short, so the latter is quite
             // common (not enough time for gossip to spread). Therefore we report only the
             // former in the log.
-            if (!tokenMetadata.isLeaving(endpoint))
+            if (!getTokenMetadata().isLeaving(endpoint))
                 logger.info("Node {} state jump to bootstrap", endpoint);
-            tokenMetadata.removeEndpoint(endpoint);
+            getTokenMetadata().removeEndpoint(endpoint);
         }
 
-        tokenMetadata.addBootstrapTokens(tokens, endpoint);
+        getTokenMetadata().addBootstrapTokens(tokens, endpoint);
         PendingRangeCalculatorService.instance.update();
 
-        tokenMetadata.updateHostId(Gossiper.instance.getHostId(endpoint), endpoint);
+        getTokenMetadata().updateHostId(Objects.requireNonNull(Nodes.localOrPeerInfo(endpoint)).getHostId(), endpoint);
     }
 
     private void handleStateBootreplacing(InetAddressAndPort newNode, String[] pieces)
@@ -3104,12 +3009,12 @@ private void handleStateBootreplacing(InetAddressAndPort newNode, String[] piece
             return;
         }
 
-        if (FailureDetector.instance.isAlive(oldNode))
+        if (IFailureDetector.instance.isAlive(oldNode))
         {
             throw new RuntimeException(String.format("Node %s is trying to replace alive node %s.", newNode, oldNode));
         }
 
-        Optional<InetAddressAndPort> replacingNode = tokenMetadata.getReplacingNode(newNode);
+        Optional<InetAddressAndPort> replacingNode = getTokenMetadata().getReplacingNode(newNode);
         if (replacingNode.isPresent() && !replacingNode.get().equals(oldNode))
         {
             throw new RuntimeException(String.format("Node %s is already replacing %s but is trying to replace %s.",
@@ -3121,10 +3026,10 @@ private void handleStateBootreplacing(InetAddressAndPort newNode, String[] piece
         if (logger.isDebugEnabled())
             logger.debug("Node {} is replacing {}, tokens {}", newNode, oldNode, tokens);
 
-        tokenMetadata.addReplaceTokens(tokens, newNode, oldNode);
+        getTokenMetadata().addReplaceTokens(tokens, newNode, oldNode);
         PendingRangeCalculatorService.instance.update();
 
-        tokenMetadata.updateHostId(Gossiper.instance.getHostId(newNode), newNode);
+        getTokenMetadata().updateHostId(Objects.requireNonNull(Nodes.localOrPeerInfo(newNode)).getHostId(), newNode);
     }
 
     private void ensureUpToDateTokenMetadata(String status, InetAddressAndPort endpoint)
@@ -3137,12 +3042,12 @@ private void ensureUpToDateTokenMetadata(String status, InetAddressAndPort endpo
         // If the node is previously unknown or tokens do not match, update tokenmetadata to
         // have this node as 'normal' (it must have been using this token before the
         // leave). This way we'll get pending ranges right.
-        if (!tokenMetadata.isMember(endpoint))
+        if (!getTokenMetadata().isMember(endpoint))
         {
             logger.info("Node {} state jump to {}", endpoint, status);
             updateTokenMetadata(endpoint, tokens);
         }
-        else if (!tokens.equals(new TreeSet<>(tokenMetadata.getTokens(endpoint))))
+        else if (!tokens.equals(new TreeSet<>(getTokenMetadata().getTokens(endpoint))))
         {
             logger.warn("Node {} '{}' token mismatch. Long network partition?", endpoint, status);
             updateTokenMetadata(endpoint, tokens);
@@ -3162,7 +3067,7 @@ private void updateTokenMetadata(InetAddressAndPort endpoint, Iterable<Token> to
         for (final Token token : tokens)
         {
             // we don't want to update if this node is responsible for the token and it has a later startup time than endpoint.
-            InetAddressAndPort currentOwner = tokenMetadata.getEndpoint(token);
+            InetAddressAndPort currentOwner = getTokenMetadata().getEndpoint(token);
             if (currentOwner == null)
             {
                 logger.debug("New node {} at token {}", endpoint, token);
@@ -3177,7 +3082,7 @@ else if (endpoint.equals(currentOwner))
             }
             // Note: in test scenarios, there may not be any delta between the heartbeat generations of the old
             // and new nodes, so we first check whether the new endpoint is marked as a replacement for the old.
-            else if (endpoint.equals(tokenMetadata.getReplacementNode(currentOwner).orElse(null)) || Gossiper.instance.compareEndpointStartup(endpoint, currentOwner) > 0)
+            else if (endpoint.equals(getTokenMetadata().getReplacementNode(currentOwner).orElse(null)) || Gossiper.instance.compareEndpointStartup(endpoint, currentOwner) > 0)
             {
                 tokensToUpdateInMetadata.add(token);
                 tokensToUpdateInSystemKeyspace.add(token);
@@ -3197,7 +3102,7 @@ else if (endpoint.equals(tokenMetadata.getReplacementNode(currentOwner).orElse(n
             }
         }
 
-        tokenMetadata.updateNormalTokens(tokensToUpdateInMetadata, endpoint);
+        getTokenMetadata().updateNormalTokens(tokensToUpdateInMetadata, endpoint);
         for (InetAddressAndPort ep : endpointsToRemove)
         {
             removeEndpoint(ep);
@@ -3218,7 +3123,7 @@ public boolean isReplacingSameHostAddressAndHostId(UUID hostId)
         {
             return isReplacingSameAddress() &&
                     Gossiper.instance.getEndpointStateForEndpoint(DatabaseDescriptor.getReplaceAddress()) != null
-                    && hostId.equals(Gossiper.instance.getHostId(DatabaseDescriptor.getReplaceAddress()));
+                    && Objects.equals(hostId, Nodes.getHostId(DatabaseDescriptor.getReplaceAddress(), null));
         }
         catch (RuntimeException ex)
         {
@@ -3257,7 +3162,7 @@ private void handleStateNormal(final InetAddressAndPort endpoint, final String s
         if (logger.isDebugEnabled())
             logger.debug("Node {} state {}, token {}", endpoint, status, tokens);
 
-        if (tokenMetadata.isMember(endpoint))
+        if (getTokenMetadata().isMember(endpoint))
             logger.info("Node {} state jump to {}", endpoint, status);
 
         if (tokens.isEmpty() && status.equals(VersionedValue.STATUS_NORMAL))
@@ -3265,12 +3170,12 @@ private void handleStateNormal(final InetAddressAndPort endpoint, final String s
                          endpoint,
                          Gossiper.instance.getEndpointStateForEndpoint(endpoint));
 
-        Optional<InetAddressAndPort> replacingNode = tokenMetadata.getReplacingNode(endpoint);
+        Optional<InetAddressAndPort> replacingNode = getTokenMetadata().getReplacingNode(endpoint);
         if (replacingNode.isPresent())
         {
             assert !endpoint.equals(replacingNode.get()) : "Pending replacement endpoint with same address is not supported";
             logger.info("Node {} will complete replacement of {} for tokens {}", endpoint, replacingNode.get(), tokens);
-            if (FailureDetector.instance.isAlive(replacingNode.get()))
+            if (IFailureDetector.instance.isAlive(replacingNode.get()))
             {
                 logger.error("Node {} cannot complete replacement of alive node {}.", endpoint, replacingNode.get());
                 return;
@@ -3278,16 +3183,15 @@ private void handleStateNormal(final InetAddressAndPort endpoint, final String s
             endpointsToRemove.add(replacingNode.get());
         }
 
-        Optional<InetAddressAndPort> replacementNode = tokenMetadata.getReplacementNode(endpoint);
+        Optional<InetAddressAndPort> replacementNode = getTokenMetadata().getReplacementNode(endpoint);
         if (replacementNode.isPresent())
         {
             logger.warn("Node {} is currently being replaced by node {}.", endpoint, replacementNode.get());
         }
 
-        updatePeerInfo(endpoint);
         // Order Matters, TM.updateHostID() should be called before TM.updateNormalToken(), (see CASSANDRA-4300).
-        UUID hostId = Gossiper.instance.getHostId(endpoint);
-        InetAddressAndPort existing = tokenMetadata.getEndpointForHostId(hostId);
+        UUID hostId = Nodes.getHostId(endpoint, null);
+        InetAddressAndPort existing = getTokenMetadata().getEndpointForHostId(hostId);
         if (replacing && isReplacingSameHostAddressAndHostId(hostId))
         {
             logger.warn("Not updating token metadata for {} because I am replacing it", endpoint);
@@ -3299,36 +3203,36 @@ private void handleStateNormal(final InetAddressAndPort endpoint, final String s
                 if (existing.equals(FBUtilities.getBroadcastAddressAndPort()))
                 {
                     logger.warn("Not updating host ID {} for {} because it's mine", hostId, endpoint);
-                    tokenMetadata.removeEndpoint(endpoint);
+                    getTokenMetadata().removeEndpoint(endpoint);
                     endpointsToRemove.add(endpoint);
                 }
                 else if (Gossiper.instance.compareEndpointStartup(endpoint, existing) > 0)
                 {
                     logger.warn("Host ID collision for {} between {} and {}; {} is the new owner", hostId, existing, endpoint, endpoint);
-                    tokenMetadata.removeEndpoint(existing);
+                    getTokenMetadata().removeEndpoint(existing);
                     endpointsToRemove.add(existing);
-                    tokenMetadata.updateHostId(hostId, endpoint);
+                    getTokenMetadata().updateHostId(hostId, endpoint);
                 }
                 else
                 {
                     logger.warn("Host ID collision for {} between {} and {}; ignored {}", hostId, existing, endpoint, endpoint);
-                    tokenMetadata.removeEndpoint(endpoint);
+                    getTokenMetadata().removeEndpoint(endpoint);
                     endpointsToRemove.add(endpoint);
                 }
             }
             else
-                tokenMetadata.updateHostId(hostId, endpoint);
+                getTokenMetadata().updateHostId(hostId, endpoint);
         }
 
         // capture because updateNormalTokens clears moving and member status
-        boolean isMember = tokenMetadata.isMember(endpoint);
-        boolean isMoving = tokenMetadata.isMoving(endpoint);
+        boolean isMember = getTokenMetadata().isMember(endpoint);
+        boolean isMoving = getTokenMetadata().isMoving(endpoint);
 
         updateTokenMetadata(endpoint, tokens, endpointsToRemove);
 
         if (isMoving || operationMode == Mode.MOVING)
         {
-            tokenMetadata.removeFromMoving(endpoint);
+            getTokenMetadata().removeFromMoving(endpoint);
             // The above may change the local ownership.
             invalidateLocalRanges();
             notifyMoved(endpoint);
@@ -3356,7 +3260,7 @@ private void handleStateLeaving(InetAddressAndPort endpoint)
 
         // at this point the endpoint is certainly a member with this token, so let's proceed
         // normally
-        tokenMetadata.addLeavingEndpoint(endpoint);
+        getTokenMetadata().addLeavingEndpoint(endpoint);
         PendingRangeCalculatorService.instance.update();
     }
 
@@ -3393,7 +3297,7 @@ private void handleStateMoving(InetAddressAndPort endpoint, String[] pieces)
         if (logger.isDebugEnabled())
             logger.debug("Node {} state moving, new token {}", endpoint, token);
 
-        tokenMetadata.addMovingEndpoint(token, endpoint);
+        getTokenMetadata().addMovingEndpoint(token, endpoint);
 
         PendingRangeCalculatorService.instance.update();
     }
@@ -3421,10 +3325,10 @@ private void handleStateRemoving(InetAddressAndPort endpoint, String[] pieces)
             }
             return;
         }
-        if (tokenMetadata.isMember(endpoint))
+        if (getTokenMetadata().isMember(endpoint))
         {
             String state = pieces[0];
-            Collection<Token> removeTokens = tokenMetadata.getTokens(endpoint);
+            Collection<Token> removeTokens = getTokenMetadata().getTokens(endpoint);
 
             if (VersionedValue.REMOVED_TOKEN.equals(state))
             {
@@ -3438,14 +3342,14 @@ else if (VersionedValue.REMOVING_TOKEN.equals(state))
                     logger.debug("Tokens {} removed manually (endpoint was {})", removeTokens, endpoint);
 
                 // Note that the endpoint is being removed
-                tokenMetadata.addLeavingEndpoint(endpoint);
+                getTokenMetadata().addLeavingEndpoint(endpoint);
                 PendingRangeCalculatorService.instance.update();
 
                 // find the endpoint coordinating this removal that we need to notify when we're done
                 String[] coordinator = splitValue(Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.REMOVAL_COORDINATOR));
                 UUID hostId = UUID.fromString(coordinator[1]);
                 // grab any data we are now responsible for and notify responsible node
-                restoreReplicaCount(endpoint, tokenMetadata.getEndpointForHostId(hostId));
+                restoreReplicaCount(endpoint, getTokenMetadata().getEndpointForHostId(hostId));
             }
         }
         else // now that the gossiper has told us about this nonexistent member, notify the gossiper to remove it
@@ -3460,8 +3364,8 @@ private void excise(Collection<Token> tokens, InetAddressAndPort endpoint)
     {
         logger.info("Removing tokens {} for {}", tokens, endpoint);
 
-        UUID hostId = tokenMetadata.getHostId(endpoint);
-        if (hostId != null && tokenMetadata.isMember(endpoint))
+        UUID hostId = getTokenMetadata().getHostId(endpoint);
+        if (hostId != null && getTokenMetadata().isMember(endpoint))
         {
             // enough time for writes to expire and MessagingService timeout reporter callback to fire, which is where
             // hints are mostly written from - using getMinRpcTimeout() / 2 for the interval.
@@ -3470,9 +3374,9 @@ private void excise(Collection<Token> tokens, InetAddressAndPort endpoint)
         }
 
         removeEndpoint(endpoint);
-        tokenMetadata.removeEndpoint(endpoint);
+        getTokenMetadata().removeEndpoint(endpoint);
         if (!tokens.isEmpty())
-            tokenMetadata.removeBootstrapTokens(tokens);
+            getTokenMetadata().removeBootstrapTokens(tokens);
         notifyLeft(endpoint);
         PendingRangeCalculatorService.instance.update();
     }
@@ -3487,7 +3391,7 @@ private void excise(Collection<Token> tokens, InetAddressAndPort endpoint, long
     private void removeEndpoint(InetAddressAndPort endpoint)
     {
         Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.removeEndpoint(endpoint));
-        SystemKeyspace.removeEndpoint(endpoint);
+        Nodes.peers().remove(endpoint, true, false);
     }
 
     protected void addExpireTimeIfFound(InetAddressAndPort endpoint, long expireTime)
@@ -3513,13 +3417,19 @@ protected long extractExpireTime(String[] pieces)
     private Multimap<InetAddressAndPort, FetchReplica> getNewSourceReplicas(String keyspaceName, Set<LeavingReplica> leavingReplicas)
     {
         InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort();
-        EndpointsByRange rangeReplicas = Keyspace.open(keyspaceName).getReplicationStrategy().getRangeAddresses(tokenMetadata.cloneOnlyTokenMap());
-        Multimap<InetAddressAndPort, FetchReplica> sourceRanges = HashMultimap.create();
-        IFailureDetector failureDetector = FailureDetector.instance;
+        EndpointsByRange rangeReplicas = Keyspace.open(keyspaceName).getReplicationStrategy().getRangeAddresses(getTokenMetadata().cloneOnlyTokenMap());
 
         logger.debug("Getting new source replicas for {}", leavingReplicas);
+        return findLiveReplicasForRanges(leavingReplicas, rangeReplicas, myAddress);
+    }
+
+    // find alive sources for ranges
+    @VisibleForTesting
+    public Multimap<InetAddressAndPort, FetchReplica> findLiveReplicasForRanges(Set<LeavingReplica> leavingReplicas, EndpointsByRange rangeReplicas, InetAddressAndPort myAddress)
+    {
+        Multimap<InetAddressAndPort, FetchReplica> sourceRanges = HashMultimap.create();
+        IFailureDetector failureDetector = IFailureDetector.instance;
 
-        // find alive sources for our new ranges
         for (LeavingReplica leaver : leavingReplicas)
         {
             //We need this to find the replicas from before leaving to supply the data
@@ -3571,7 +3481,7 @@ private void sendReplicationNotification(InetAddressAndPort remote)
     {
         // notify the remote token
         Message msg = Message.out(REPLICATION_DONE_REQ, noPayload);
-        IFailureDetector failureDetector = FailureDetector.instance;
+        IFailureDetector failureDetector = IFailureDetector.instance;
         if (logger.isDebugEnabled())
             logger.debug("Notifying {} of replication completion\n", remote);
         while (failureDetector.isAlive(remote))
@@ -3589,7 +3499,8 @@ private void sendReplicationNotification(InetAddressAndPort remote)
         }
     }
 
-    private static class LeavingReplica
+    @VisibleForTesting
+    public static class LeavingReplica
     {
         //The node that is leaving
         private final Replica leavingReplica;
@@ -3651,7 +3562,7 @@ private void restoreReplicaCount(InetAddressAndPort endpoint, final InetAddressA
         for (String keyspaceName : Schema.instance.distributedKeyspaces().names())
         {
             logger.debug("Restoring replica count for keyspace {}", keyspaceName);
-            EndpointsByReplica changedReplicas = getChangedReplicasForLeaving(keyspaceName, endpoint, tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
+            EndpointsByReplica changedReplicas = getChangedReplicasForLeaving(keyspaceName, endpoint, getTokenMetadata(), Keyspace.open(keyspaceName).getReplicationStrategy());
             Set<LeavingReplica> myNewReplicas = new HashSet<>();
             for (Map.Entry<Replica, Replica> entry : changedReplicas.flattenEntries())
             {
@@ -3784,7 +3695,6 @@ static EndpointsByReplica getChangedReplicasForLeaving(String keyspaceName, Inet
         return changedRanges.build();
     }
 
-
     public void onJoin(InetAddressAndPort endpoint, EndpointState epState)
     {
         // Explicitly process STATUS or STATUS_WITH_PORT before the other
@@ -3812,13 +3722,13 @@ public void onJoin(InetAddressAndPort endpoint, EndpointState epState)
 
     public void onAlive(InetAddressAndPort endpoint, EndpointState state)
     {
-        if (tokenMetadata.isMember(endpoint))
+        if (getTokenMetadata().isMember(endpoint))
             notifyUp(endpoint);
     }
 
     public void onRemove(InetAddressAndPort endpoint)
     {
-        tokenMetadata.removeEndpoint(endpoint);
+        getTokenMetadata().removeEndpoint(endpoint);
         PendingRangeCalculatorService.instance.update();
     }
 
@@ -3893,13 +3803,13 @@ public Collection<Token> getLocalTokens()
     @Nullable
     public InetAddressAndPort getEndpointForHostId(UUID hostId)
     {
-        return tokenMetadata.getEndpointForHostId(hostId);
+        return getTokenMetadata().getEndpointForHostId(hostId);
     }
 
     @Nullable
     public UUID getHostIdForEndpoint(InetAddressAndPort address)
     {
-        return tokenMetadata.getHostId(address);
+        return getTokenMetadata().getHostId(address);
     }
 
     /* These methods belong to the MBean interface */
@@ -3952,12 +3862,12 @@ public String getKeyspaceReplicationInfo(String keyspaceName)
     @Deprecated(since = "4.0")
     public List<String> getLeavingNodes()
     {
-        return stringify(tokenMetadata.getLeavingEndpoints(), false);
+        return stringify(getTokenMetadata().getLeavingEndpoints(), false);
     }
 
     public List<String> getLeavingNodesWithPort()
     {
-        return stringify(tokenMetadata.getLeavingEndpoints(), true);
+        return stringify(getTokenMetadata().getLeavingEndpoints(), true);
     }
 
     /** @deprecated See CASSANDRA-7544 */
@@ -3966,7 +3876,7 @@ public List<String> getMovingNodes()
     {
         List<String> endpoints = new ArrayList<>();
 
-        for (Pair<Token, InetAddressAndPort> node : tokenMetadata.getMovingEndpoints())
+        for (Pair<Token, InetAddressAndPort> node : getTokenMetadata().getMovingEndpoints())
         {
             endpoints.add(node.right.getAddress().getHostAddress());
         }
@@ -3978,7 +3888,7 @@ public List<String> getMovingNodesWithPort()
     {
         List<String> endpoints = new ArrayList<>();
 
-        for (Pair<Token, InetAddressAndPort> node : tokenMetadata.getMovingEndpoints())
+        for (Pair<Token, InetAddressAndPort> node : getTokenMetadata().getMovingEndpoints())
         {
             endpoints.add(node.right.getHostAddressAndPort());
         }
@@ -3990,12 +3900,12 @@ public List<String> getMovingNodesWithPort()
     @Deprecated(since = "4.0")
     public List<String> getJoiningNodes()
     {
-        return stringify(tokenMetadata.getBootstrapTokens().valueSet(), false);
+        return stringify(getTokenMetadata().getBootstrapTokens().valueSet(), false);
     }
 
     public List<String> getJoiningNodesWithPort()
     {
-        return stringify(tokenMetadata.getBootstrapTokens().valueSet(), true);
+        return stringify(getTokenMetadata().getBootstrapTokens().valueSet(), true);
     }
 
     /** @deprecated See CASSANDRA-7544 */
@@ -4017,18 +3927,24 @@ public Set<InetAddressAndPort> getLiveRingMembers()
 
     public Set<InetAddressAndPort> getLiveRingMembers(boolean excludeDeadStates)
     {
-        Set<InetAddressAndPort> ret = new HashSet<>();
-        for (InetAddressAndPort ep : Gossiper.instance.getLiveMembers())
+        Set<InetAddressAndPort> allRingMembers = getTokenMetadata().getAllRingMembers();
+        Set<InetAddressAndPort> ret = new HashSet<>(allRingMembers.size());
+        for (InetAddressAndPort ep : getTokenMetadata().getAllRingMembers())
         {
-            if (excludeDeadStates)
+            if (Gossiper.instance.isEnabled())
             {
                 EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep);
-                if (epState == null || Gossiper.instance.isDeadState(epState))
+                if (epState == null)
+                    continue;
+
+                if (excludeDeadStates && Gossiper.instance.isDeadState(epState))
                     continue;
             }
 
-            if (tokenMetadata.isMember(ep))
-                ret.add(ep);
+            if (!IFailureDetector.instance.isAlive(ep))
+                continue;
+
+            ret.add(ep);
         }
         return ret;
     }
@@ -4052,7 +3968,7 @@ public String[] getAllDataFileLocations()
         return getCanonicalPaths(DatabaseDescriptor.getAllDataFileLocations());
     }
 
-    private String[] getCanonicalPaths(String[] paths)
+    private String[] getCanonicalPaths(File[] paths)
     {
         String[] locations = new String[paths.length];
         for (int i = 0; i < paths.length; i++)
@@ -4107,7 +4023,7 @@ public int forceKeyspaceCleanup(int jobs, String keyspaceName, String... tableNa
         if (SchemaConstants.isLocalSystemKeyspace(keyspaceName))
             throw new RuntimeException("Cleanup of the system keyspace is neither necessary nor wise");
 
-        if (!tokenMetadata.getPendingRanges(keyspaceName, getBroadcastAddressAndPort()).isEmpty())
+        if (getTokenMetadata().getPendingRanges(keyspaceName, getBroadcastAddressAndPort()).size() > 0)
             throw new RuntimeException("Node is involved in cluster membership changes. Not safe to run cleanup.");
 
         CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
@@ -4153,16 +4069,24 @@ public int verify(boolean extendedVerify, String keyspaceName, String... tableNa
         return verify(extendedVerify, false, false, false, false, false, keyspaceName, tableNames);
     }
 
+    /** @deprecated See CNDB-10054 */
+    @Deprecated(since = "CC4.0")
     public int verify(boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+    {
+        return verify(extendedVerify, false, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspaceName, tableNames);
+    }
+
+    public int verify(boolean extendedVerify, boolean validateAllRows, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
     {
         CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
         IVerifier.Options options = IVerifier.options().invokeDiskFailurePolicy(diskFailurePolicy)
                                              .extendedVerification(extendedVerify)
+                                             .validateAllRows(validateAllRows)
                                              .checkVersion(checkVersion)
                                              .mutateRepairStatus(mutateRepairStatus)
                                              .checkOwnsTokens(checkOwnsTokens)
                                              .quick(quick).build();
-        logger.info("Staring {} on {}.{} with options = {}", OperationType.VERIFY, keyspaceName, Arrays.toString(tableNames), options);
+        logger.info("Starting {} on {}.{} with options = {}", OperationType.VERIFY, keyspaceName, Arrays.toString(tableNames), options);
         for (ColumnFamilyStore cfStore : getValidColumnFamilies(false, false, keyspaceName, tableNames))
         {
             CompactionManager.AllSSTableOpStatus oneStatus = cfStore.verify(options);
@@ -4218,7 +4142,7 @@ public List<Pair<String, String>> getPreparedStatements()
     {
         List<Pair<String, String>> statements = new ArrayList<>();
         for (Entry<MD5Digest, QueryHandler.Prepared> e : QueryProcessor.instance.getPreparedStatements().entrySet())
-            statements.add(Pair.create(e.getKey().toString(), e.getValue().rawCQLStatement));
+            statements.add(Pair.create(e.getKey().toString(), e.getValue().statement.getRawCQLStatement()));
         return statements;
     }
 
@@ -4236,6 +4160,14 @@ public void forceKeyspaceCompaction(boolean splitOutput, String keyspaceName, St
         }
     }
 
+    public void forceKeyspaceCompaction(boolean splitOutput, int parallelism, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+    {
+        for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, false, keyspaceName, tableNames))
+        {
+            cfStore.forceMajorCompaction(splitOutput, parallelism);
+        }
+    }
+
     public int relocateSSTables(String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
     {
         return relocateSSTables(0, keyspaceName, tableNames);
@@ -4539,9 +4471,9 @@ public void clearSnapshot(Map<String, Object> options, String tag, String... key
             options = Collections.emptyMap();
 
         Set<String> keyspaces = new HashSet<>();
-        for (String dataDir : DatabaseDescriptor.getAllDataFileLocations())
+        for (File dataDir : DatabaseDescriptor.getAllDataFileLocations())
         {
-            for (String keyspaceDir : new File(dataDir).tryListNames())
+            for (String keyspaceDir : dataDir.tryListNames())
             {
                 // Only add a ks if it has been specified as a param, assuming params were actually provided.
                 if (keyspaceNames.length > 0 && !Arrays.asList(keyspaceNames).contains(keyspaceDir))
@@ -4720,7 +4652,7 @@ public int repairAsync(String keyspace, Map<String, String> repairSpec)
 
     public Pair<Integer, Future<?>> repair(String keyspace, Map<String, String> repairSpec, List<ProgressListener> listeners)
     {
-        RepairOption option = RepairOption.parse(repairSpec, tokenMetadata.partitioner);
+        RepairOption option = RepairOption.parse(repairSpec, getTokenMetadata().partitioner);
         return repair(keyspace, option, listeners);
     }
 
@@ -4745,8 +4677,11 @@ else if (option.isInLocalDCOnly())
                 Iterables.addAll(option.getRanges(), getLocalReplicas(keyspace).onlyFull().ranges());
             }
         }
-        if (option.getRanges().isEmpty() || Keyspace.open(keyspace).getReplicationStrategy().getReplicationFactor().allReplicas < 2)
+        if (option.getRanges().isEmpty() || Keyspace.open(keyspace).getReplicationStrategy().getReplicationFactor().allReplicas < 2
+            || getTokenMetadata().getAllEndpoints().size() < 2)
+        {
             return Pair.create(0, ImmediateFuture.success(null));
+        }
 
         int cmd = nextRepairCommand.incrementAndGet();
         return Pair.create(cmd, repairCommandExecutor().submit(createRepairTask(cmd, keyspace, option, listeners)));
@@ -4768,7 +4703,7 @@ Collection<Range<Token>> createRepairRangeFrom(String beginToken, String endToke
         // Break up given range to match ring layout in TokenMetadata
         ArrayList<Range<Token>> repairingRange = new ArrayList<>();
 
-        ArrayList<Token> tokens = new ArrayList<>(tokenMetadata.sortedTokens());
+        ArrayList<Token> tokens = new ArrayList<>(getTokenMetadata().sortedTokens());
         if (!tokens.contains(parsedBeginToken))
         {
             tokens.add(parsedBeginToken);
@@ -4792,7 +4727,7 @@ Collection<Range<Token>> createRepairRangeFrom(String beginToken, String endToke
 
     public TokenFactory getTokenFactory()
     {
-        return tokenMetadata.partitioner.getTokenFactory();
+        return getTokenMetadata().partitioner.getTokenFactory();
     }
 
     private FutureTask<Object> createRepairTask(final int cmd, final String keyspace, final RepairOption options, List<ProgressListener> listeners)
@@ -4801,7 +4736,7 @@ private FutureTask<Object> createRepairTask(final int cmd, final String keyspace
         {
             throw new IllegalArgumentException("the local data center must be part of the repair; requested " + options.getDataCenters() + " but DC is " + DatabaseDescriptor.getLocalDataCenter());
         }
-        Set<String> existingDatacenters = tokenMetadata.cloneOnlyTokenMap().getTopology().getDatacenterEndpoints().keys().elementSet();
+        Set<String> existingDatacenters = getTokenMetadata().cloneOnlyTokenMap().getTopology().getDatacenterEndpoints().keys().elementSet();
         List<String> datacenters = new ArrayList<>(options.getDataCenters());
         if (!existingDatacenters.containsAll(datacenters))
         {
@@ -4979,7 +4914,7 @@ public Collection<Range<Token>> getPrimaryRangesForEndpoint(String keyspace, Ine
     {
         AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy();
         Collection<Range<Token>> primaryRanges = new HashSet<>();
-        TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap();
+        TokenMetadata metadata = strategy.getTokenMetadata().cloneOnlyTokenMap();
         for (Token token : metadata.sortedTokens())
         {
             EndpointsForRange replicas = strategy.calculateNaturalReplicas(token, metadata);
@@ -5002,10 +4937,10 @@ public Collection<Range<Token>> getPrimaryRangesForEndpoint(String keyspace, Ine
      */
     public Collection<Range<Token>> getPrimaryRangeForEndpointWithinDC(String keyspace, InetAddressAndPort referenceEndpoint)
     {
-        TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap();
+        AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy();
+        TokenMetadata metadata = strategy.getTokenMetadata().cloneOnlyTokenMap();
         String localDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(referenceEndpoint);
         Collection<InetAddressAndPort> localDcNodes = metadata.getTopology().getDatacenterEndpoints().get(localDC);
-        AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy();
 
         Collection<Range<Token>> localDCPrimaryRanges = new HashSet<>();
         for (Token token : metadata.sortedTokens())
@@ -5035,7 +4970,7 @@ public Collection<Range<Token>> getLocalPrimaryRange()
     public Collection<Range<Token>> getLocalPrimaryRangeForEndpoint(InetAddressAndPort referenceEndpoint)
     {
         IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
-        TokenMetadata tokenMetadata = this.tokenMetadata.cloneOnlyTokenMap();
+        TokenMetadata tokenMetadata = this.getTokenMetadata().cloneOnlyTokenMap();
         if (!tokenMetadata.isMember(referenceEndpoint))
             return Collections.emptySet();
         String dc = snitch.getDatacenter(referenceEndpoint);
@@ -5128,13 +5063,13 @@ public EndpointsForToken getNaturalReplicasForToken(String keyspaceName, String
 
     public EndpointsForToken getNaturalReplicasForToken(String keyspaceName, ByteBuffer key)
     {
-        Token token = tokenMetadata.partitioner.getToken(key);
+        Token token = getTokenMetadata().partitioner.getToken(key);
         return Keyspace.open(keyspaceName).getReplicationStrategy().getNaturalReplicasForToken(token);
     }
 
     public DecoratedKey getKeyFromPartition(String keyspaceName, String table, String partitionKey)
     {
-        return tokenMetadata.partitioner.decorateKey(partitionKeyToBytes(keyspaceName, table, partitionKey));
+        return getTokenMetadata().partitioner.decorateKey(partitionKeyToBytes(keyspaceName, table, partitionKey));
     }
 
     private static ByteBuffer partitionKeyToBytes(String keyspaceName, String cf, String key)
@@ -5153,7 +5088,7 @@ private static ByteBuffer partitionKeyToBytes(String keyspaceName, String cf, St
     @Override
     public String getToken(String keyspaceName, String table, String key)
     {
-        return tokenMetadata.partitioner.getToken(partitionKeyToBytes(keyspaceName, table, key)).toString();
+        return getTokenMetadata().partitioner.getToken(partitionKeyToBytes(keyspaceName, table, key)).toString();
     }
 
     public boolean isEndpointValidForWrite(String keyspace, Token token)
@@ -5234,14 +5169,14 @@ private List<DecoratedKey> keySamples(Iterable<ColumnFamilyStore> cfses, Range<T
     }
 
     /**
-     * Broadcast leaving status and update local tokenMetadata accordingly
+     * Broadcast leaving status and update local getTokenMetadata() accordingly
      */
     private void startLeaving()
     {
         DatabaseDescriptor.getSeverityDuringDecommission().ifPresent(DynamicEndpointSnitch::addSeverity);
         Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.leaving(getLocalTokens()));
         Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.leaving(getLocalTokens()));
-        tokenMetadata.addLeavingEndpoint(FBUtilities.getBroadcastAddressAndPort());
+        getTokenMetadata().addLeavingEndpoint(FBUtilities.getBroadcastAddressAndPort());
         PendingRangeCalculatorService.instance.update();
     }
 
@@ -5259,11 +5194,11 @@ public void decommission(boolean force) throws InterruptedException
             return;
         }
 
-        TokenMetadata metadata = tokenMetadata.cloneAfterAllLeft();
+        TokenMetadata metadata = getTokenMetadata().cloneAfterAllLeft();
         // there is no point to do this logic again once node was decommissioning but failed to do so
         if (operationMode != Mode.LEAVING)
         {
-            if (!tokenMetadata.isMember(FBUtilities.getBroadcastAddressAndPort()))
+            if (!getTokenMetadata().isMember(FBUtilities.getBroadcastAddressAndPort()))
                 throw new UnsupportedOperationException("local node is not a member of the token ring yet");
             if (metadata.getAllEndpoints().size() < 2 && metadata.getAllEndpoints().contains(FBUtilities.getBroadcastAddressAndPort()))
                     throw new UnsupportedOperationException("no other normal nodes in the ring; decommission would be pointless");
@@ -5287,7 +5222,7 @@ public void decommission(boolean force) throws InterruptedException
             if (operationMode != Mode.LEAVING)
             {
                 int rf, numNodes;
-                for (String keyspaceName : Schema.instance.distributedKeyspaces().names())
+                for (String keyspaceName : Schema.instance.getPartitionedKeyspaces().names())
                 {
                     if (!force)
                     {
@@ -5317,7 +5252,7 @@ public void decommission(boolean force) throws InterruptedException
                                                                     + " Perform a forceful decommission to ignore.");
                     }
                     // TODO: do we care about fixing transient/full self-movements here? probably
-                    if (tokenMetadata.getPendingRanges(keyspaceName, FBUtilities.getBroadcastAddressAndPort()).size() > 0)
+                    if (getTokenMetadata().getPendingRanges(keyspaceName, FBUtilities.getBroadcastAddressAndPort()).size() > 0)
                         throw new UnsupportedOperationException("data is currently moving to this node; unable to leave the ring");
                 }
             }
@@ -5374,7 +5309,7 @@ public void decommission(boolean force) throws InterruptedException
     private void leaveRing()
     {
         SystemKeyspace.setBootstrapState(SystemKeyspace.BootstrapState.NEEDS_BOOTSTRAP);
-        tokenMetadata.removeEndpoint(FBUtilities.getBroadcastAddressAndPort());
+        getTokenMetadata().removeEndpoint(FBUtilities.getBroadcastAddressAndPort());
         PendingRangeCalculatorService.instance.update();
 
         Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.left(getLocalTokens(),Gossiper.computeExpireTime()));
@@ -5390,7 +5325,7 @@ public Supplier<Future<StreamState>> prepareUnbootstrapStreaming()
 
         for (String keyspaceName : Schema.instance.distributedKeyspaces().names())
         {
-            EndpointsByReplica rangesMM = getChangedReplicasForLeaving(keyspaceName, FBUtilities.getBroadcastAddressAndPort(), tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
+            EndpointsByReplica rangesMM = getChangedReplicasForLeaving(keyspaceName, FBUtilities.getBroadcastAddressAndPort(), getTokenMetadata(), Keyspace.open(keyspaceName).getReplicationStrategy());
 
             if (logger.isDebugEnabled())
                 logger.debug("Ranges needing transfer are [{}]", StringUtils.join(rangesMM.keySet(), ","));
@@ -5444,10 +5379,11 @@ private Future streamHints()
         return HintsService.instance.transferHints(this::getPreferredHintsStreamTarget);
     }
 
-    private static EndpointsForRange getStreamCandidates(Collection<InetAddressAndPort> endpoints)
+    @VisibleForTesting
+    public static EndpointsForRange getStreamCandidates(Collection<InetAddressAndPort> endpoints)
     {
         endpoints = endpoints.stream()
-                             .filter(endpoint -> FailureDetector.instance.isAlive(endpoint) && !FBUtilities.getBroadcastAddressAndPort().equals(endpoint))
+                             .filter(endpoint -> IFailureDetector.instance.isAlive(endpoint) && !FBUtilities.getBroadcastAddressAndPort().equals(endpoint))
                              .collect(Collectors.toList());
 
         return SystemReplicas.getSystemReplicas(endpoints);
@@ -5470,7 +5406,7 @@ private UUID getPreferredHintsStreamTarget()
             // stream to the closest peer as chosen by the snitch
             candidates = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), candidates);
             InetAddressAndPort hintsDestinationHost = candidates.get(0).endpoint();
-            return tokenMetadata.getHostId(hintsDestinationHost);
+            return getTokenMetadata().getHostId(hintsDestinationHost);
         }
     }
 
@@ -5499,7 +5435,7 @@ private void move(Token newToken) throws IOException
         if (newToken == null)
             throw new IOException("Can't move to the undefined (null) token.");
 
-        if (tokenMetadata.sortedTokens().contains(newToken))
+        if (getTokenMetadata().sortedTokens().contains(newToken))
             throw new IOException("target token " + newToken + " is already owned by another node.");
 
         // address of the current node
@@ -5519,7 +5455,7 @@ private void move(Token newToken) throws IOException
         for (String keyspaceName : keyspacesToProcess)
         {
             // TODO: do we care about fixing transient/full self-movements here?
-            if (tokenMetadata.getPendingRanges(keyspaceName, localAddress).size() > 0)
+            if (getTokenMetadata().getPendingRanges(keyspaceName, localAddress).size() > 0)
                 throw new UnsupportedOperationException("data is currently moving to this node; unable to leave the ring");
         }
 
@@ -5530,7 +5466,7 @@ private void move(Token newToken) throws IOException
         setMode(Mode.MOVING, String.format("Sleeping %s ms before start streaming/fetching ranges", RING_DELAY_MILLIS), true);
         Uninterruptibles.sleepUninterruptibly(RING_DELAY_MILLIS, MILLISECONDS);
 
-        RangeRelocator relocator = new RangeRelocator(Collections.singleton(newToken), keyspacesToProcess, tokenMetadata);
+        RangeRelocator relocator = new RangeRelocator(Collections.singleton(newToken), keyspacesToProcess, getTokenMetadata());
         relocator.calculateToFromStreams();
 
         repairPaxosForTopologyChange("move");
@@ -5592,7 +5528,7 @@ private String getRemovalStatus(boolean withPort)
         }
 
         return String.format("Removing token (%s). Waiting for replication confirmation from [%s].",
-                             tokenMetadata.getTokens(removingNode).iterator().next(),
+                             getTokenMetadata().getTokens(removingNode).iterator().next(),
                              StringUtils.join(toFormat, ","));
     }
 
@@ -5603,14 +5539,14 @@ private String getRemovalStatus(boolean withPort)
      */
     public void forceRemoveCompletion()
     {
-        if (!replicatingNodes.isEmpty()  || tokenMetadata.getSizeOfLeavingEndpoints() > 0)
+        if (!replicatingNodes.isEmpty()  || getTokenMetadata().getSizeOfLeavingEndpoints() > 0)
         {
             logger.warn("Removal not confirmed for for {}", StringUtils.join(this.replicatingNodes, ","));
-            for (InetAddressAndPort endpoint : tokenMetadata.getLeavingEndpoints())
+            for (InetAddressAndPort endpoint : getTokenMetadata().getLeavingEndpoints())
             {
-                UUID hostId = tokenMetadata.getHostId(endpoint);
+                UUID hostId = getTokenMetadata().getHostId(endpoint);
                 Gossiper.instance.advertiseTokenRemoved(endpoint, hostId);
-                excise(tokenMetadata.getTokens(endpoint), endpoint);
+                excise(getTokenMetadata().getTokens(endpoint), endpoint);
             }
             replicatingNodes.clear();
             removingNode = null;
@@ -5633,14 +5569,14 @@ public void forceRemoveCompletion()
     public void removeNode(String hostIdString)
     {
         InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort();
-        UUID localHostId = tokenMetadata.getHostId(myAddress);
+        UUID localHostId = getTokenMetadata().getHostId(myAddress);
         UUID hostId = UUID.fromString(hostIdString);
-        InetAddressAndPort endpoint = tokenMetadata.getEndpointForHostId(hostId);
+        InetAddressAndPort endpoint = getTokenMetadata().getEndpointForHostId(hostId);
 
         if (endpoint == null)
             throw new UnsupportedOperationException("Host ID not found.");
 
-        if (!tokenMetadata.isMember(endpoint))
+        if (!getTokenMetadata().isMember(endpoint))
             throw new UnsupportedOperationException("Node to be removed is not a member of the token ring");
 
         if (endpoint.equals(myAddress))
@@ -5650,13 +5586,13 @@ public void removeNode(String hostIdString)
             throw new UnsupportedOperationException("Node " + endpoint + " is alive and owns this ID. Use decommission command to remove it from the ring");
 
         // A leaving endpoint that is dead is already being removed.
-        if (tokenMetadata.isLeaving(endpoint))
+        if (getTokenMetadata().isLeaving(endpoint))
             logger.warn("Node {} is already being removed, continuing removal anyway", endpoint);
 
         if (!replicatingNodes.isEmpty())
             throw new UnsupportedOperationException("This node is already processing a removal. Wait for it to complete, or use 'removenode force' if this has failed.");
 
-        Collection<Token> tokens = tokenMetadata.getTokens(endpoint);
+        Collection<Token> tokens = getTokenMetadata().getTokens(endpoint);
 
         // Find the endpoints that are going to become responsible for data
         for (String keyspaceName : Schema.instance.distributedKeyspaces().names())
@@ -5667,8 +5603,8 @@ public void removeNode(String hostIdString)
 
             // get all ranges that change ownership (that is, a node needs
             // to take responsibility for new range)
-            EndpointsByReplica changedRanges = getChangedReplicasForLeaving(keyspaceName, endpoint, tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
-            IFailureDetector failureDetector = FailureDetector.instance;
+            EndpointsByReplica changedRanges = getChangedReplicasForLeaving(keyspaceName, endpoint, getTokenMetadata(), Keyspace.open(keyspaceName).getReplicationStrategy());
+            IFailureDetector failureDetector = IFailureDetector.instance;
             for (InetAddressAndPort ep : transform(changedRanges.flattenValues(), Replica::endpoint))
             {
                 if (failureDetector.isAlive(ep))
@@ -5679,7 +5615,7 @@ public void removeNode(String hostIdString)
         }
         removingNode = endpoint;
 
-        tokenMetadata.addLeavingEndpoint(endpoint);
+        getTokenMetadata().addLeavingEndpoint(endpoint);
         PendingRangeCalculatorService.instance.update();
 
         // the gossiper will handle spoofing this node's state to REMOVING_TOKEN for us
@@ -5843,13 +5779,6 @@ protected synchronized void drain(boolean isFinalShutdown) throws IOException, I
                 logger.error("Messaging service timed out shutting down", t);
             }
 
-            if (!isFinalShutdown)
-                setMode(Mode.DRAINING, "clearing mutation stage", false);
-            Stage.shutdownAndAwaitMutatingExecutors(false,
-                                                    DRAIN_EXECUTOR_TIMEOUT_MS.getInt(), TimeUnit.MILLISECONDS);
-
-            StorageProxy.instance.verifyNoHintsInProgress();
-
             if (!isFinalShutdown)
                 setMode(Mode.DRAINING, "flushing column families", false);
 
@@ -5902,7 +5831,28 @@ protected synchronized void drain(boolean isFinalShutdown) throws IOException, I
             }
             FBUtilities.waitOnFutures(flushes);
 
+            // Now that client requests, messaging service and compactions are shutdown, there shouldn't be any more
+            // mutations so let's wait for any pending mutations and then clear the stages. Note that the compaction
+            // manager can generated mutations, for example because of the view builder or the sstable_activity updates
+            // in the SSTableReader.GlobalTidy, so we do this step quite late, but before shutting down the CL
+            if (!isFinalShutdown)
+                setMode(Mode.DRAINING, "stopping mutations", false);
+
+            List<OpOrder.Barrier> barriers = StreamSupport.stream(Keyspace.all().spliterator(), false)
+                                                          .map(Keyspace::stopMutations)
+                                                          .collect(Collectors.toList());
+            barriers.forEach(OpOrder.Barrier::await); // we could parallelize this...
+
+            if (!isFinalShutdown)
+                setMode(Mode.DRAINING, "clearing mutation stage", false);
+            Stage.shutdownAndAwaitMutatingExecutors(false,
+                                                    DRAIN_EXECUTOR_TIMEOUT_MS.getInt(), TimeUnit.MILLISECONDS);
+
+            StorageProxy.instance.verifyNoHintsInProgress();
+
+
             SnapshotManager.shutdownAndWait(1L, MINUTES);
+
             HintsService.instance.shutdownBlocking();
 
             // Interrupt ongoing compactions and shutdown CM to prevent further compactions.
@@ -6021,15 +5971,16 @@ synchronized void checkServiceAllowedToStart(String service)
     public IPartitioner setPartitionerUnsafe(IPartitioner newPartitioner)
     {
         IPartitioner oldPartitioner = DatabaseDescriptor.setPartitionerUnsafe(newPartitioner);
-        tokenMetadata = tokenMetadata.cloneWithNewPartitioner(newPartitioner);
+        setTokenMetadataUnsafe(StorageService.instance.getTokenMetadata().cloneWithNewPartitioner(newPartitioner));
         valueFactory = new VersionedValue.VersionedValueFactory(newPartitioner);
         return oldPartitioner;
     }
 
+    @VisibleForTesting
     TokenMetadata setTokenMetadataUnsafe(TokenMetadata tmd)
     {
-        TokenMetadata old = tokenMetadata;
-        tokenMetadata = tmd;
+        TokenMetadata old = getTokenMetadata();
+        TokenMetadataProvider.instance.replaceTokenMetadata(tmd);
         return old;
     }
 
@@ -6039,7 +5990,7 @@ public void truncate(String keyspace, String table) throws TimeoutException, IOE
 
         try
         {
-            StorageProxy.truncateBlocking(keyspace, table);
+            StorageProxy.instance.truncateBlocking(keyspace, table);
         }
         catch (UnavailableException e)
         {
@@ -6049,13 +6000,13 @@ public void truncate(String keyspace, String table) throws TimeoutException, IOE
 
     public Map<InetAddress, Float> getOwnership()
     {
-        List<Token> sortedTokens = tokenMetadata.sortedTokens();
+        List<Token> sortedTokens = getTokenMetadata().sortedTokens();
         // describeOwnership returns tokens in an unspecified order, let's re-order them
-        Map<Token, Float> tokenMap = new TreeMap<Token, Float>(tokenMetadata.partitioner.describeOwnership(sortedTokens));
+        Map<Token, Float> tokenMap = new TreeMap<Token, Float>(getTokenMetadata().partitioner.describeOwnership(sortedTokens));
         Map<InetAddress, Float> nodeMap = new LinkedHashMap<>();
         for (Map.Entry<Token, Float> entry : tokenMap.entrySet())
         {
-            InetAddressAndPort endpoint = tokenMetadata.getEndpoint(entry.getKey());
+            InetAddressAndPort endpoint = getTokenMetadata().getEndpoint(entry.getKey());
             Float tokenOwnership = entry.getValue();
             if (nodeMap.containsKey(endpoint.getAddress()))
                 nodeMap.put(endpoint.getAddress(), nodeMap.get(endpoint.getAddress()) + tokenOwnership);
@@ -6067,13 +6018,13 @@ public Map<InetAddress, Float> getOwnership()
 
     public Map<String, Float> getOwnershipWithPort()
     {
-        List<Token> sortedTokens = tokenMetadata.sortedTokens();
+        List<Token> sortedTokens = getTokenMetadata().sortedTokens();
         // describeOwnership returns tokens in an unspecified order, let's re-order them
-        Map<Token, Float> tokenMap = new TreeMap<Token, Float>(tokenMetadata.partitioner.describeOwnership(sortedTokens));
+        Map<Token, Float> tokenMap = new TreeMap<Token, Float>(getTokenMetadata().partitioner.describeOwnership(sortedTokens));
         Map<String, Float> nodeMap = new LinkedHashMap<>();
         for (Map.Entry<Token, Float> entry : tokenMap.entrySet())
         {
-            InetAddressAndPort endpoint = tokenMetadata.getEndpoint(entry.getKey());
+            InetAddressAndPort endpoint = getTokenMetadata().getEndpoint(entry.getKey());
             Float tokenOwnership = entry.getValue();
             if (nodeMap.containsKey(endpoint.toString()))
                 nodeMap.put(endpoint.toString(), nodeMap.get(endpoint.toString()) + tokenOwnership);
@@ -6130,7 +6081,7 @@ private LinkedHashMap<InetAddressAndPort, Float> getEffectiveOwnership(String ke
             strategy = keyspaceInstance.getReplicationStrategy();
         }
 
-        TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap();
+        TokenMetadata metadata = getTokenMetadata().cloneOnlyTokenMap();
 
         Collection<Collection<InetAddressAndPort>> endpointsGroupedByDc = new ArrayList<>();
         // mapping of dc's to nodes, use sorted map so that we get dcs sorted
@@ -6138,7 +6089,7 @@ private LinkedHashMap<InetAddressAndPort, Float> getEffectiveOwnership(String ke
         for (Collection<InetAddressAndPort> endpoints : sortedDcsToEndpoints.values())
             endpointsGroupedByDc.add(endpoints);
 
-        Map<Token, Float> tokenOwnership = tokenMetadata.partitioner.describeOwnership(tokenMetadata.sortedTokens());
+        Map<Token, Float> tokenOwnership = getTokenMetadata().partitioner.describeOwnership(getTokenMetadata().sortedTokens());
         LinkedHashMap<InetAddressAndPort, Float> finalOwnership = Maps.newLinkedHashMap();
 
         RangesByEndpoint endpointToRanges = strategy.getAddressReplicas();
@@ -6194,7 +6145,7 @@ public List<String> getNonLocalStrategyKeyspaces()
     public Map<String, String> getViewBuildStatuses(String keyspace, String view, boolean withPort)
     {
         Map<UUID, String> coreViewStatus = SystemDistributedKeyspace.viewStatus(keyspace, view);
-        Map<InetAddressAndPort, UUID> hostIdToEndpoint = tokenMetadata.getEndpointToHostIdMapForReading();
+        Map<InetAddressAndPort, UUID> hostIdToEndpoint = getTokenMetadata().getEndpointToHostIdMapForReading();
         Map<String, String> result = new HashMap<>();
 
         for (Map.Entry<InetAddressAndPort, UUID> entry : hostIdToEndpoint.entrySet())
@@ -6304,6 +6255,16 @@ public void updateSnitch(String epSnitchClassName, Boolean dynamic, Integer dyna
         updateTopology();
     }
 
+    public String getBatchlogEndpointStrategy()
+    {
+        return DatabaseDescriptor.getBatchlogEndpointStrategy().name();
+    }
+
+    public void setBatchlogEndpointStrategy(String batchlogEndpointStrategy)
+    {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.valueOf(batchlogEndpointStrategy));
+    }
+
     /**
      * Send data to the endpoints that will be responsible for it in the future
      *
@@ -6626,11 +6587,15 @@ public int getSSTablePreemptiveOpenIntervalInMB()
         return DatabaseDescriptor.getSSTablePreemptiveOpenIntervalInMiB();
     }
 
+    /** @deprecated CPU-intensive optimization that visibly slows down compaction but does not provide a clear benefit (see STAR-782) */
+    @Deprecated(since = "CC 4.0")
     public boolean getMigrateKeycacheOnCompaction()
     {
         return DatabaseDescriptor.shouldMigrateKeycacheOnCompaction();
     }
 
+    /** @deprecated CPU-intensive optimization that visibly slows down compaction but does not provide a clear benefit (see STAR-782) */
+    @Deprecated(since = "CC 4.0")
     public void setMigrateKeycacheOnCompaction(boolean invalidateKeyCacheOnCompaction)
     {
         DatabaseDescriptor.setMigrateKeycacheOnCompaction(invalidateKeyCacheOnCompaction);
@@ -6638,23 +6603,25 @@ public void setMigrateKeycacheOnCompaction(boolean invalidateKeyCacheOnCompactio
 
     public int getTombstoneWarnThreshold()
     {
-        return DatabaseDescriptor.getTombstoneWarnThreshold();
+        return GuardrailsConfigProvider.instance.getOrCreate(null).getTombstoneWarnThreshold();
     }
 
     public void setTombstoneWarnThreshold(int threshold)
     {
-        DatabaseDescriptor.setTombstoneWarnThreshold(threshold);
+        GuardrailsConfig guardrailsConfig = GuardrailsConfigProvider.instance.getOrCreate(null);
+        guardrailsConfig.setTombstonesThreshold(threshold, guardrailsConfig.getTombstoneFailThreshold());
         logger.info("updated tombstone_warn_threshold to {}", threshold);
     }
 
     public int getTombstoneFailureThreshold()
     {
-        return DatabaseDescriptor.getTombstoneFailureThreshold();
+        return GuardrailsConfigProvider.instance.getOrCreate(null).getTombstoneFailThreshold();
     }
 
     public void setTombstoneFailureThreshold(int threshold)
     {
-        DatabaseDescriptor.setTombstoneFailureThreshold(threshold);
+        GuardrailsConfig guardrailsConfig = GuardrailsConfigProvider.instance.getOrCreate(null);
+        guardrailsConfig.setTombstonesThreshold(guardrailsConfig.getTombstoneWarnThreshold(), threshold);
         logger.info("updated tombstone_failure_threshold to {}", threshold);
     }
 
@@ -6724,7 +6691,7 @@ public int getColumnIndexCacheSize()
     @Override
     public void setColumnIndexCacheSize(int cacheSizeInKB)
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(cacheSizeInKB);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(cacheSizeInKB);
         logger.info("Updated column_index_cache_size to {}", cacheSizeInKB);
     }
 
@@ -6744,7 +6711,7 @@ public void setColumnIndexCacheSizeInKiB(int cacheSizeInKiB)
     {
         try
         {
-            DatabaseDescriptor.setColumnIndexCacheSize(cacheSizeInKiB);
+            DatabaseDescriptor.setColumnIndexCacheSizeInKiB(cacheSizeInKiB);
         }
         catch (ConfigurationException e)
         {
@@ -7020,15 +6987,6 @@ public void setOutOfTokenRangeRequestRejectionEnabled(boolean enabled)
         DatabaseDescriptor.setRejectOutOfTokenRangeRequests(enabled);
     }
 
-    @VisibleForTesting
-    public void shutdownServer()
-    {
-        if (drainOnShutdown != null)
-        {
-            Runtime.getRuntime().removeShutdownHook(drainOnShutdown);
-        }
-    }
-
     @Override
     public void enableFullQueryLogger(String path, String rollCycle, Boolean blocking, int maxQueueWeight, long maxLogSize, String archiveCommand, int maxArchiveRetries)
     {
@@ -7117,42 +7075,6 @@ public void setAutoOptimisePreviewRepairStreams(boolean enabled)
         DatabaseDescriptor.setAutoOptimisePreviewRepairStreams(enabled);
     }
 
-    /** @deprecated See CASSANDRA-17195 */
-    @Deprecated(since = "4.1")
-    public int getTableCountWarnThreshold()
-    {
-        return (int) Converters.TABLE_COUNT_THRESHOLD_TO_GUARDRAIL.unconvert(Guardrails.instance.getTablesWarnThreshold());
-    }
-
-    /** @deprecated See CASSANDRA-17195 */
-    @Deprecated(since = "4.1")
-    public void setTableCountWarnThreshold(int value)
-    {
-        if (value < 0)
-            throw new IllegalStateException("Table count warn threshold should be positive, not "+value);
-        logger.info("Changing table count warn threshold from {} to {}", getTableCountWarnThreshold(), value);
-        Guardrails.instance.setTablesThreshold((int) Converters.TABLE_COUNT_THRESHOLD_TO_GUARDRAIL.convert(value),
-                                               Guardrails.instance.getTablesFailThreshold());
-    }
-
-    /** @deprecated See CASSANDRA-17195 */
-    @Deprecated(since = "4.1")
-    public int getKeyspaceCountWarnThreshold()
-    {
-        return (int) Converters.KEYSPACE_COUNT_THRESHOLD_TO_GUARDRAIL.unconvert(Guardrails.instance.getKeyspacesWarnThreshold());
-    }
-
-    /** @deprecated See CASSANDRA-17195 */
-    @Deprecated(since = "4.1")
-    public void setKeyspaceCountWarnThreshold(int value)
-    {
-        if (value < 0)
-            throw new IllegalStateException("Keyspace count warn threshold should be positive, not "+value);
-        logger.info("Changing keyspace count warn threshold from {} to {}", getKeyspaceCountWarnThreshold(), value);
-        Guardrails.instance.setKeyspacesThreshold((int) Converters.KEYSPACE_COUNT_THRESHOLD_TO_GUARDRAIL.convert(value),
-                                                  Guardrails.instance.getKeyspacesFailThreshold());
-    }
-
     @Override
     public void setCompactionTombstoneWarningThreshold(int count)
     {
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index fc224c6514d8..dc3f07f56e87 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -364,6 +364,11 @@ public interface StorageServiceMBean extends NotificationEmitter
      */
     public void forceKeyspaceCompaction(boolean splitOutput, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
 
+    /**
+     * Forces major compaction of a single keyspace with the given parallelism limit
+     */
+    public void forceKeyspaceCompaction(boolean splitOutput, int parallelism, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
+
     /** @deprecated See CASSANDRA-11179 */
     @Deprecated(since = "3.5")
     public int relocateSSTables(String keyspace, String ... cfnames) throws IOException, ExecutionException, InterruptedException;
@@ -426,9 +431,14 @@ default int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkD
      * If tableNames array is empty, all CFs are verified.
      *
      * The entire sstable will be read to ensure each cell validates if extendedVerify is true
+     * @deprecated See CNDB-10054
      */
+    @Deprecated(since = "CC4.0")
     public int verify(boolean extendedVerify, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
+    /** @deprecated See CNDB-10054 */
+    @Deprecated(since = "CC4.0")
     public int verify(boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
+    public int verify(boolean extendedVerify, boolean validateAllRows, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
 
     /**
      * Rewrite all sstables to the latest version.
@@ -650,6 +660,13 @@ default int upgradeSSTables(String keyspaceName, boolean excludeCurrentVersion,
      */
     public int getDynamicUpdateInterval();
 
+    public String getBatchlogEndpointStrategy();
+
+    /**
+     * See {@link org.apache.cassandra.config.Config.BatchlogEndpointStrategy} for valid values.
+     */
+    public void setBatchlogEndpointStrategy(String batchlogEndpointStrategy);
+
     // allows a user to forcibly 'kill' a sick node
     public void stopGossiping();
 
@@ -804,6 +821,7 @@ default int upgradeSSTables(String keyspaceName, boolean excludeCurrentVersion,
     @Deprecated(since = "4.1")
     public int getCompactionThroughputMbPerSec();
     public void setCompactionThroughputMbPerSec(int value);
+    Map<String, String> getCurrentCompactionThroughputMebibytesPerSec();
 
     public int getBatchlogReplayThrottleInKB();
     public void setBatchlogReplayThrottleInKB(int value);
@@ -824,7 +842,11 @@ default int upgradeSSTables(String keyspaceName, boolean excludeCurrentVersion,
     public int getSSTablePreemptiveOpenIntervalInMB();
     public void setSSTablePreemptiveOpenIntervalInMB(int intervalInMB);
 
+    /** @deprecated CPU-intensive optimization that visibly slows down compaction but does not provide a clear benefit (see STAR-782) */
+    @Deprecated(since = "CC 4.0")
     public boolean getMigrateKeycacheOnCompaction();
+    /** @deprecated CPU-intensive optimization that visibly slows down compaction but does not provide a clear benefit (see STAR-782) */
+    @Deprecated(since = "CC 4.0")
     public void setMigrateKeycacheOnCompaction(boolean invalidateKeyCacheOnCompaction);
 
     public int getConcurrentViewBuilders();
@@ -1170,19 +1192,6 @@ public void enableAuditLog(String loggerName, String includedKeyspaces, String e
     public void setAutoOptimisePreviewRepairStreams(boolean enabled);
 
     // warning thresholds will be replaced by equivalent guardrails
-    /** @deprecated See CASSANDRA-17195 */
-    @Deprecated(since = "4.1")
-    int getTableCountWarnThreshold();
-    /** @deprecated See CASSANDRA-17195 */
-    @Deprecated(since = "4.1")
-    void setTableCountWarnThreshold(int value);
-    /** @deprecated See CASSANDRA-17195 */
-    @Deprecated(since = "4.1")
-    int getKeyspaceCountWarnThreshold();
-    /** @deprecated See CASSANDRA-17195 */
-    @Deprecated(since = "4.1")
-    void setKeyspaceCountWarnThreshold(int value);
-
     /** @deprecated See CASSANDRA-17194 */
     @Deprecated(since = "5.0")
     void setCompactionTombstoneWarningThreshold(int count);
diff --git a/src/java/org/apache/cassandra/service/TokenRange.java b/src/java/org/apache/cassandra/service/TokenRange.java
index 37971f5e4f6c..416ce0cdad10 100644
--- a/src/java/org/apache/cassandra/service/TokenRange.java
+++ b/src/java/org/apache/cassandra/service/TokenRange.java
@@ -60,7 +60,7 @@ public static TokenRange create(Token.TokenFactory tokenFactory, Range<Token> ra
         IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
         for (InetAddressAndPort ep : endpoints)
             details.add(new EndpointDetails(ep,
-                                            StorageService.instance.getNativeaddress(ep, withPorts),
+                                            StorageService.instance.getNativeAddress(ep, withPorts),
                                             snitch.getDatacenter(ep),
                                             snitch.getRack(ep)));
         return new TokenRange(tokenFactory, range, details);
diff --git a/src/java/org/apache/cassandra/service/TracingClientState.java b/src/java/org/apache/cassandra/service/TracingClientState.java
new file mode 100644
index 000000000000..8039c6cbe477
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/TracingClientState.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service;
+
+import javax.annotation.Nullable;
+
+/**
+ * As tracing can happen at both coordinator and replicas, at replica side, CNDB needs to know the traced keyspace for billing
+ */
+public class TracingClientState extends ClientState
+{
+    private final @Nullable String tracedKeyspace;
+
+    protected TracingClientState(String tracedKeyspace, ClientState state)
+    {
+        super(state);
+        this.tracedKeyspace = tracedKeyspace;
+    }
+
+    @Override
+    public ClientState cloneWithKeyspaceIfSet(String keyspace)
+    {
+        if (keyspace == null)
+            return this;
+        return new TracingClientState(tracedKeyspace, super.cloneWithKeyspaceIfSet(keyspace));
+    }
+
+    /**
+     * @return the keyspace being traced
+     */
+    @Nullable
+    public String tracedKeyspace()
+    {
+        return tracedKeyspace;
+    }
+
+    /**
+     * @return a ClientState object for internal C* calls (not limited by any kind of auth) with traced keyspace
+     */
+    public static TracingClientState withTracedKeyspace(@Nullable String tracedKeyspace)
+    {
+        return new TracingClientState(tracedKeyspace, ClientState.forInternalCalls());
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/TruncateResponseHandler.java b/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
index 54b1241006d7..630330394d00 100644
--- a/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
@@ -60,7 +60,7 @@ public TruncateResponseHandler(int responseCount)
         start = nanoTime();
     }
 
-    public void get() throws TimeoutException
+    public void get() throws TimeoutException, TruncateException
     {
         long timeoutNanos = getTruncateRpcTimeout(NANOSECONDS) - (nanoTime() - start);
         boolean signaled;
diff --git a/src/java/org/apache/cassandra/service/WriteResponseHandler.java b/src/java/org/apache/cassandra/service/WriteResponseHandler.java
index ec18238f9932..9739443bed11 100644
--- a/src/java/org/apache/cassandra/service/WriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/WriteResponseHandler.java
@@ -20,14 +20,14 @@
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.function.Supplier;
 
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.locator.ReplicaPlan;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.net.Message;
+import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.WriteType;
 import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.Message;
 
 /**
  * Handles blocking writes for ONE, ANY, TWO, THREE, QUORUM, and ALL consistency levels.
@@ -65,7 +65,7 @@ public void onResponse(Message<T> m)
         logResponseToIdealCLDelegate(m);
     }
 
-    protected int ackCount()
+    public int ackCount()
     {
         return blockFor() - responses;
     }
diff --git a/src/java/org/apache/cassandra/service/context/DefaultOperationContext.java b/src/java/org/apache/cassandra/service/context/DefaultOperationContext.java
new file mode 100644
index 000000000000..f22d8e9dd6fc
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/context/DefaultOperationContext.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.context;
+
+import java.util.function.Supplier;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadCommand;
+
+/**
+ * Default implementation of {@link OperationContext}.
+ * <p>
+ * This default implementation is mostly only useful for debugging as the only concrete method is provices is a
+ * {@link #toString()} method giving details on the operation the context corresponds to (though the context object
+ * also identify the operation, so it could also theoretically be used from 2 separate place in the code to decide
+ * if they execute as part of the same operation).
+ */
+public class DefaultOperationContext implements OperationContext
+{
+    private final Supplier<String> toDebugString;
+
+    private DefaultOperationContext(Supplier<String> toDebugString)
+    {
+        this.toDebugString = toDebugString;
+    }
+
+    @Override
+    public void close()
+    {
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("[%d] %s", System.identityHashCode(this), toDebugString.get());
+    }
+
+    /**
+     * Simple default implementation of {@link OperationContext.Factory} that creates {@link DefaultOperationContext}.
+     */
+    static class Factory implements OperationContext.Factory
+    {
+        @Override
+        public OperationContext forRead(ReadCommand command, ColumnFamilyStore cfs)
+        {
+            return new DefaultOperationContext(command::toCQLString);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/context/OperationContext.java b/src/java/org/apache/cassandra/service/context/OperationContext.java
new file mode 100644
index 000000000000..926143990cc6
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/context/OperationContext.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.context;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.OPERATION_CONTEXT_FACTORY;
+
+/**
+ * Represents some context about a "top-level" operation.
+ * <p>
+ * This interface is fairly open on purpose, as implementations for different operations could look fairly different.
+ * But it is also open-ended as it is an extension point: the {@link #FACTORY} used to create the context instances
+ * is configurable, and meant to allow extensions to add whatever information they need to the context.
+ * <p>
+ * Also note that what consistute a "top-level" operation is not strictly defined. At the time of this writing, those
+ * context are not serialized across nodes, so "top-level" is understood as "for a node", and so correspond to
+ * operations like "a `ReadCommand` execution on a replica".
+ * <p>
+ * The context of executing operation is tracked by {@link OperationContextTracker} which use the {@link ExecutorLocal}
+ * concept to make that context available to any methods that execute as part of the operation. Basically, this is a way
+ * to make the context available everwhere along the path of execution of the operation, without needing to pass that
+ * context as argument of every single method that could be involved by the operation execution (which in most cases
+ * would be <b>a lot of methods</b>).
+*/
+public interface OperationContext extends AutoCloseable
+{
+    Factory FACTORY = OPERATION_CONTEXT_FACTORY.getString() == null
+                      ? new DefaultOperationContext.Factory()
+                      : FBUtilities.construct(OPERATION_CONTEXT_FACTORY.getString(), "operation context factory");
+
+
+    /**
+     * Called when the operation this is a context of terminates, and thus when the context will not be used/retrieved
+     * anymore.
+     */
+    @Override
+    void close();
+
+    /**
+     * Factory used to create {@link OperationContext} instances.
+     * <p>
+     * The intent is that every operation that wants to set a context should have its own method in this interface, but
+     * operations are added as needed (instead of trying to cover every possible operation upfront).
+     * <p>
+     * Do note however that there can only be one operation context "active" at any given time (meaning, any thread
+     * execute can only see at most one context), so the context should be set at the higher level that make sense
+     * (and if necessary, sub-operations can enrich the context of their parent, assuming the parent context make room
+     * for this).
+     */
+    interface Factory
+    {
+        OperationContext forRead(ReadCommand command, ColumnFamilyStore cfs);
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/context/OperationContextTracker.java b/src/java/org/apache/cassandra/service/context/OperationContextTracker.java
new file mode 100644
index 000000000000..d88145dc771c
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/context/OperationContextTracker.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.context;
+
+import org.apache.cassandra.concurrent.ExecutorLocals;
+
+public class OperationContextTracker extends ExecutorLocals.Impl
+{
+    public static final OperationContextTracker instance = new OperationContextTracker();
+
+    private OperationContextTracker()
+    {}
+
+    public OperationContext get()
+    {
+        return ExecutorLocals.current().operationContext;
+    }
+
+    public void set(OperationContext operationContext)
+    {
+        ExecutorLocals current = ExecutorLocals.current();
+        ExecutorLocals.Impl.set(current.traceState, current.clientWarnState, current.sensors, operationContext);
+    }
+
+    public static void start(OperationContext context)
+    {
+        instance.set(context);
+    }
+
+    public static void endCurrent()
+    {
+        OperationContext ctx = instance.get();
+        if (ctx != null)
+        {
+            ctx.close();
+            instance.set(null);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java b/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
index 292264984f8f..942f9a5da405 100644
--- a/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
+++ b/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
@@ -17,6 +17,14 @@
  */
 package org.apache.cassandra.service.pager;
 
+import java.util.StringJoiner;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.db.partitions.*;
@@ -27,23 +35,42 @@
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 
+@NotThreadSafe
 abstract class AbstractQueryPager<T extends ReadQuery> implements QueryPager
 {
+    private static final Logger logger = LoggerFactory.getLogger(AbstractQueryPager.class);
+
     protected final T query;
+
+    // the limits provided as a part of the query
     protected final DataLimits limits;
     protected final ProtocolVersion protocolVersion;
     private final boolean enforceStrictLiveness;
 
-    private int remaining;
+    // This is the counter which was used for the last page we fetched. It can be used to obtain the number of
+    // fetched rows or bytes.
+    private DataLimits.Counter lastCounter;
 
-    // This is the last key we've been reading from (or can still be reading within). This the key for
+    // This is the last key we've been reading from (or can still be reading within). This is the key for
     // which remainingInPartition makes sense: if we're starting another key, we should reset remainingInPartition
     // (and this is done in PagerIterator). This can be null (when we start).
     private DecoratedKey lastKey;
+
+    // The remaining and remainingInPartition are initially set to the user limits provided in the query (via the
+    // LIMIT and PER PARTITION LIMIT clauses). When a page is fetched, iterated and closed, those values are updated
+    // with the number of items counted on that recently fetched page.
+    private int remaining;
     private int remainingInPartition;
 
+    // Whether the pager is exhausted or not - the pager gets exhausted if the recently fetched, iterated and closed
+    // page has less items than the requested page size
     private boolean exhausted;
 
+    // The paging transformation which is used for the recently requested page. It is set when we request the new page
+    // and then cleaned when the page is closed. We use it to prevent fetching a new page until the previous one is
+    // closed.
+    private PagerTransformation<?> currentPagerTransformation;
+
     protected AbstractQueryPager(T query, ProtocolVersion protocolVersion)
     {
         this.query = query;
@@ -60,58 +87,77 @@ public ReadExecutionController executionController()
         return query.executionController();
     }
 
-    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime)
+    public PartitionIterator fetchPage(PageSize pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime)
     {
+        assert currentPagerTransformation == null;
+
         if (isExhausted())
             return EmptyIterators.partition();
 
-        pageSize = Math.min(pageSize, remaining);
-        Pager pager = new RowPager(limits.forPaging(pageSize), query.nowInSec());
-        ReadQuery readQuery = nextPageReadQuery(pageSize);
+        DataLimits updatedQueryLimits = nextPageLimits();
+        RowPagerTransformation pagerTransformation = new RowPagerTransformation(updatedQueryLimits.forPaging(pageSize), query.nowInSec());
+        ReadQuery readQuery = nextPageReadQuery(pageSize, updatedQueryLimits);
         if (readQuery == null)
         {
             exhausted = true;
             return EmptyIterators.partition();
         }
-        return Transformation.apply(readQuery.execute(consistency, clientState, requestTime), pager);
+        currentPagerTransformation = pagerTransformation;
+        return Transformation.apply(readQuery.execute(consistency, clientState, requestTime), pagerTransformation);
     }
 
-    public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController)
+    @Override
+    public PartitionIterator fetchPageInternal(PageSize pageSize, ReadExecutionController executionController)
     {
+        assert currentPagerTransformation == null;
+
         if (isExhausted())
             return EmptyIterators.partition();
 
-        pageSize = Math.min(pageSize, remaining);
-        RowPager pager = new RowPager(limits.forPaging(pageSize), query.nowInSec());
-        ReadQuery readQuery = nextPageReadQuery(pageSize);
+        DataLimits updatedQueryLimits = nextPageLimits();
+        RowPagerTransformation pagerTransformation = new RowPagerTransformation(updatedQueryLimits.forPaging(pageSize), query.nowInSec());
+        ReadQuery readQuery = nextPageReadQuery(pageSize, updatedQueryLimits);
         if (readQuery == null)
         {
             exhausted = true;
             return EmptyIterators.partition();
         }
-        return Transformation.apply(readQuery.executeInternal(executionController), pager);
+        currentPagerTransformation = pagerTransformation;
+        return Transformation.apply(readQuery.executeInternal(executionController), pagerTransformation);
     }
 
-    public UnfilteredPartitionIterator fetchPageUnfiltered(TableMetadata metadata, int pageSize, ReadExecutionController executionController)
+    public UnfilteredPartitionIterator fetchPageUnfiltered(TableMetadata metadata, PageSize pageSize, ReadExecutionController executionController)
     {
+        assert currentPagerTransformation == null;
+
         if (isExhausted())
             return EmptyIterators.unfilteredPartition(metadata);
 
-        pageSize = Math.min(pageSize, remaining);
-        UnfilteredPager pager = new UnfilteredPager(limits.forPaging(pageSize), query.nowInSec());
-        ReadQuery readQuery = nextPageReadQuery(pageSize);
+        DataLimits updatedQueryLimits = nextPageLimits();
+        UnfilteredPagerTransformation pagerTransformation = new UnfilteredPagerTransformation(updatedQueryLimits.forPaging(pageSize), query.nowInSec());
+        ReadQuery readQuery = nextPageReadQuery(pageSize, updatedQueryLimits);
         if (readQuery == null)
         {
             exhausted = true;
             return EmptyIterators.unfilteredPartition(metadata);
         }
-        return Transformation.apply(readQuery.executeLocally(executionController), pager);
+        currentPagerTransformation = pagerTransformation;
+        return Transformation.apply(readQuery.executeLocally(executionController), pagerTransformation);
     }
 
-    private class UnfilteredPager extends Pager<Unfiltered>
+    /**
+     * For subsequent pages we want to limit the number of rows to the minimum of the currently set limit in the query
+     * and the number of remaining rows in page. Note that paging itself will be applied separately.
+     */
+    protected DataLimits nextPageLimits()
     {
+        return limits.withCountedLimit(Math.min(limits.count(), remaining));
+    }
 
-        private UnfilteredPager(DataLimits pageLimits, long nowInSec)
+    private class UnfilteredPagerTransformation extends PagerTransformation<Unfiltered>
+    {
+
+        private UnfilteredPagerTransformation(DataLimits pageLimits, long nowInSec)
         {
             super(pageLimits, nowInSec);
         }
@@ -122,10 +168,10 @@ protected BaseRowIterator<Unfiltered> apply(BaseRowIterator<Unfiltered> partitio
         }
     }
 
-    private class RowPager extends Pager<Row>
+    private class RowPagerTransformation extends PagerTransformation<Row>
     {
 
-        private RowPager(DataLimits pageLimits, long nowInSec)
+        private RowPagerTransformation(DataLimits pageLimits, long nowInSec)
         {
             super(pageLimits, nowInSec);
         }
@@ -136,7 +182,7 @@ protected BaseRowIterator<Row> apply(BaseRowIterator<Row> partition)
         }
     }
 
-    private abstract class Pager<T extends Unfiltered> extends Transformation<BaseRowIterator<T>>
+    private abstract class PagerTransformation<T extends Unfiltered> extends Transformation<BaseRowIterator<T>>
     {
         private final DataLimits pageLimits;
         protected final DataLimits.Counter counter;
@@ -144,10 +190,20 @@ private abstract class Pager<T extends Unfiltered> extends Transformation<BaseRo
         private Row lastRow;
         private boolean isFirstPartition = true;
 
-        private Pager(DataLimits pageLimits, long nowInSec)
+        private PagerTransformation(DataLimits pageLimits, long nowInSec)
         {
             this.counter = pageLimits.newCounter(nowInSec, true, query.selectsFullPartition(), enforceStrictLiveness);
+            lastCounter = this.counter;
+
+            // Page limits are passed to the transformation for two reasons
+            // - to create a counter,
+            // - to determine if the query pager is exhausted when we reach the end of data (if the end of data
+            //   is reached before we reach the page limit, we conclude that the query pager is exhausted and there
+            //   are no more pages to fetch)
             this.pageLimits = pageLimits;
+
+            if (logger.isTraceEnabled())
+                logger.trace("Fetching new page - created {}", this);
         }
 
         @Override
@@ -178,6 +234,7 @@ public BaseRowIterator<T> applyToPartition(BaseRowIterator<T> partition)
         @Override
         public void onClose()
         {
+            assert lastCounter == counter;
             // In some case like GROUP BY a counter need to know when the processing is completed.
             counter.onClose();
 
@@ -197,7 +254,9 @@ public void onClose()
             {
                 remainingInPartition -= counter.countedInCurrentPartition();
             }
-            exhausted = pageLimits.isExhausted(counter);
+            // if the counter did not count up to the page limits, then the iteration must have reached the end
+            exhausted = pageLimits.isCounterBelowLimits(counter);
+            currentPagerTransformation = null;
         }
 
         public Row applyToStatic(Row row)
@@ -223,6 +282,18 @@ public Row applyToRow(Row row)
             lastRow = row;
             return row;
         }
+
+        @Override
+        public String toString()
+        {
+            return new StringJoiner(", ", PagerTransformation.class.getSimpleName() + "[", "]")
+                   .add("pageLimits=" + pageLimits)
+                   .add("counter=" + counter)
+                   .add("currentKey=" + currentKey)
+                   .add("lastRow=" + lastRow)
+                   .add("isFirstPartition=" + isFirstPartition)
+                   .toString();
+        }
     }
 
     protected void restoreState(DecoratedKey lastKey, int remaining, int remainingInPartition)
@@ -234,7 +305,7 @@ protected void restoreState(DecoratedKey lastKey, int remaining, int remainingIn
 
     public boolean isExhausted()
     {
-        return exhausted || remaining == 0 || ((this instanceof SinglePartitionPager) && remainingInPartition == 0);
+        return exhausted || remaining == 0;
     }
 
     public int maxRemaining()
@@ -247,7 +318,30 @@ protected int remainingInPartition()
         return remainingInPartition;
     }
 
-    protected abstract T nextPageReadQuery(int pageSize);
+    /**
+     * Returns the {@link DataLimits.Counter} for the page which was last fetched (the last page in the meaning
+     * the last returned and traversed row iterator, the iterator must be closed in order for this method to return
+     * proper counter)
+     */
+    public DataLimits.Counter getLastCounter()
+    {
+        return lastCounter;
+    }
+
+    protected abstract T nextPageReadQuery(PageSize pageSize, DataLimits limits);
     protected abstract void recordLast(DecoratedKey key, Row row);
     protected abstract boolean isPreviouslyReturnedPartition(DecoratedKey key);
+
+    @Override
+    public String toString()
+    {
+        return new StringJoiner(", ", AbstractQueryPager.class.getSimpleName() + "[", "]")
+               .add("limits=" + limits)
+               .add("remaining=" + remaining)
+               .add("lastCounter=" + lastCounter)
+               .add("lastKey=" + lastKey)
+               .add("remainingInPartition=" + remainingInPartition)
+               .add("exhausted=" + exhausted)
+               .toString();
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java b/src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java
index 95d910de6052..c385238a1df1 100644
--- a/src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java
+++ b/src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java
@@ -19,7 +19,14 @@
 
 import java.nio.ByteBuffer;
 import java.util.NoSuchElementException;
+import java.util.StringJoiner;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.OperationExecutionException;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.aggregation.GroupingState;
@@ -35,31 +42,54 @@
  * <p>
  * For aggregation/group by queries, the user page size is in number of groups. But each group could be composed of very
  * many rows so to avoid running into OOMs, this pager will page internal queries into sub-pages. So each call to
- * {@link fetchPage} may (transparently) yield multiple internal queries (sub-pages).
+ * {@link #fetchPage(PageSize, ConsistencyLevel, ClientState, Dispatcher.RequestTime)} may (transparently) yield multiple internal queries
+ * (sub-pages).
  */
 public final class AggregationQueryPager implements QueryPager
 {
+    private static final Logger logger = LoggerFactory.getLogger(AggregationQueryPager.class);
+
     private final DataLimits limits;
 
+    private final PageSize subPageSize;
+
     // The sub-pager, used to retrieve the next sub-page.
     private QueryPager subPager;
 
-    public AggregationQueryPager(QueryPager subPager, DataLimits limits)
+    public AggregationQueryPager(QueryPager subPager, PageSize subPageSize, DataLimits limits)
     {
         this.subPager = subPager;
         this.limits = limits;
+        this.subPageSize = subPageSize;
     }
 
+    /**
+     * This will return the iterator over the partitions. The iterator is limited by the provided page size and the user
+     * specified limit (in the query). Both the limit and the page size are applied to the number of groups covered by
+     * the returned data.
+     * <p/>
+     * In case of group-by queries the page size can be provided only in rows unit ({@link OperationExecutionException}
+     * is thrown otherwise). In case of 'aggregate everything' queries, the provided page size and the limits are
+     * ignored as we always return a single row.
+     *
+     * @param pageSize    the maximum number of elements to return in the next page (groups)
+     * @param consistency the consistency level to achieve for the query
+     * @param clientState  the {@code QueryState} for the query. In practice, this can be null unless
+     *                    {@code consistency} is a serial consistency
+     */
     @Override
-    public PartitionIterator fetchPage(int pageSize,
+    public PartitionIterator fetchPage(PageSize pageSize,
                                        ConsistencyLevel consistency,
                                        ClientState clientState,
                                        Dispatcher.RequestTime requestTime)
     {
+        if (pageSize.isDefined() && pageSize.getUnit() != PageSize.PageUnit.ROWS)
+            throw new InvalidRequestException("Paging in bytes is not supported for aggregation queries. Please specify the page size in rows.");
+
         if (limits.isGroupByLimit())
-            return new GroupByPartitionIterator(pageSize, consistency, clientState, requestTime);
+            return new GroupByPartitionIterator(pageSize, subPageSize, consistency, clientState, requestTime);
 
-        return new AggregationPartitionIterator(pageSize, consistency, clientState, requestTime);
+        return new AggregationPartitionIterator(subPageSize, consistency, clientState, requestTime);
     }
 
     @Override
@@ -68,13 +98,22 @@ public ReadExecutionController executionController()
         return subPager.executionController();
     }
 
+    /**
+     * {@see #fetchPage}
+     *
+     * @param pageSize the maximum number of elements to return in the next page
+     * @param executionController the {@code ReadExecutionController} protecting the read
+     */
     @Override
-    public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController)
+    public PartitionIterator fetchPageInternal(PageSize pageSize, ReadExecutionController executionController)
     {
+        if (pageSize.isDefined() && pageSize.getUnit() != PageSize.PageUnit.ROWS)
+            throw new InvalidRequestException("Paging in bytes is not supported for aggregation queries. Please specify the page size in rows.");
+
         if (limits.isGroupByLimit())
-            return new GroupByPartitionIterator(pageSize, executionController, Dispatcher.RequestTime.forImmediateExecution());
+            return new GroupByPartitionIterator(pageSize, subPageSize, executionController, Dispatcher.RequestTime.forImmediateExecution());
 
-        return new AggregationPartitionIterator(pageSize, executionController, Dispatcher.RequestTime.forImmediateExecution());
+        return new AggregationPartitionIterator(subPageSize, executionController, Dispatcher.RequestTime.forImmediateExecution());
     }
 
     @Override
@@ -116,7 +155,12 @@ public class GroupByPartitionIterator implements PartitionIterator
         /**
          * The top-level page size in number of groups.
          */
-        private final int pageSize;
+        private final PageSize groupsPageSize;
+
+        /**
+         * Page size for internal paging
+         */
+        private final PageSize subPageSize;
 
         // For "normal" queries
         private final ConsistencyLevel consistency;
@@ -158,43 +202,44 @@ public class GroupByPartitionIterator implements PartitionIterator
         /**
          * The initial amount of row remaining
          */
-        private int initialMaxRemaining;
+        protected int initialMaxRemaining;
 
         private Dispatcher.RequestTime requestTime;
 
-        public GroupByPartitionIterator(int pageSize,
+        public GroupByPartitionIterator(PageSize groupsPageSize,
+                                        PageSize subPageSize,
                                         ConsistencyLevel consistency,
                                         ClientState clientState,
                                         Dispatcher.RequestTime requestTime)
         {
-            this(pageSize, consistency, clientState, null, requestTime);
+            this(groupsPageSize, subPageSize, consistency, clientState, null, requestTime);
         }
 
-        public GroupByPartitionIterator(int pageSize,
+        public GroupByPartitionIterator(PageSize groupsPageSize,
+                                        PageSize subPageSize,
                                         ReadExecutionController executionController,
                                         Dispatcher.RequestTime requestTime)
        {
-           this(pageSize, null, null, executionController, requestTime);
+           this(groupsPageSize, subPageSize, null, null, executionController, requestTime);
        }
 
-        private GroupByPartitionIterator(int pageSize,
+        private GroupByPartitionIterator(PageSize groupsPageSize,
+                                         PageSize subPageSize,
                                          ConsistencyLevel consistency,
                                          ClientState clientState,
                                          ReadExecutionController executionController,
                                          Dispatcher.RequestTime requestTime)
         {
-            this.pageSize = handlePagingOff(pageSize);
+            this.groupsPageSize = groupsPageSize;
+            this.subPageSize = subPageSize;
             this.consistency = consistency;
             this.clientState = clientState;
             this.executionController = executionController;
             this.requestTime = requestTime;
-        }
+            subPager = subPager.withUpdatedLimit(limits.withCountedLimit(groupsPageSize.minRowsCount(maxRemaining())));
 
-        private int handlePagingOff(int pageSize)
-        {
-            // If the paging is off, the pageSize will be <= 0. So we need to replace
-            // it by DataLimits.NO_LIMIT
-            return pageSize <= 0 ? DataLimits.NO_LIMIT : pageSize;
+            if (logger.isTraceEnabled())
+                logger.trace("Fetching a new page - created {}", this);
         }
 
         public final void close()
@@ -220,39 +265,42 @@ public final boolean hasNext()
         }
 
         /**
-         * Loads the next <code>RowIterator</code> to be returned.
+         * Loads the next <code>RowIterator</code> to be returned. The iteration finishes when we reach either the
+         * user groups limit or the groups page size. The user provided limit is initially set in subPager.maxRemaining().
          */
         private void fetchNextRowIterator()
         {
+            // we haven't started yet, fetch the first sub page (partition iterator with sub-page limit)
             if (partitionIterator == null)
             {
                 initialMaxRemaining = subPager.maxRemaining();
-                partitionIterator = fetchSubPage(pageSize);
+                partitionIterator = fetchSubPage(subPageSize);
             }
 
             while (!partitionIterator.hasNext())
             {
                 partitionIterator.close();
 
-                int counted = initialMaxRemaining - subPager.maxRemaining();
-
-                if (isDone(pageSize, counted) || subPager.isExhausted())
+                int remaining = getRemaining();
+                assert remaining >= 0;
+                if (remaining == 0 || subPager.isExhausted())
                 {
                     endOfData = true;
                     closed = true;
                     return;
                 }
 
-                subPager = updatePagerLimit(subPager, limits, lastPartitionKey, lastClustering);
-                partitionIterator = fetchSubPage(computeSubPageSize(pageSize, counted));
+                subPager = updatePagerLimit(subPager, limits.withCountedLimit(remaining), lastPartitionKey, lastClustering);
+                partitionIterator = fetchSubPage(subPageSize);
             }
 
             next = partitionIterator.next();
         }
 
-        protected boolean isDone(int pageSize, int counted)
+        protected int getRemaining()
         {
-            return counted == pageSize;
+            int counted = initialMaxRemaining - subPager.maxRemaining();
+            return groupsPageSize.withDecreasedRows(counted).rows();
         }
 
         /**
@@ -274,25 +322,13 @@ protected QueryPager updatePagerLimit(QueryPager pager,
             return pager.withUpdatedLimit(newLimits);
         }
 
-        /**
-         * Computes the size of the next sub-page to retrieve.
-         *
-         * @param pageSize the top-level page size
-         * @param counted the number of result returned so far by the previous sub-pages
-         * @return the size of the next sub-page to retrieve
-         */
-        protected int computeSubPageSize(int pageSize, int counted)
-        {
-            return pageSize - counted;
-        }
-
         /**
          * Fetchs the next sub-page.
          *
          * @param subPageSize the sub-page size in number of groups
          * @return the next sub-page
          */
-        private final PartitionIterator fetchSubPage(int subPageSize)
+        private final PartitionIterator fetchSubPage(PageSize subPageSize)
         {
             return consistency != null ? subPager.fetchPage(subPageSize, consistency, clientState, requestTime)
                                        : subPager.fetchPageInternal(subPageSize, executionController);
@@ -391,11 +427,42 @@ public boolean hasNext()
 
             public Row next()
             {
+                // we need to check this because this.rowIterator may exhaust if the sub-page is done and in such a case
+                // #hasNext switches this.rowIterator to the new one, which is obtained for the next page
+                if (!hasNext())
+                    throw new NoSuchElementException();
+
                 Row row = this.rowIterator.next();
                 lastClustering = row.clustering();
                 return row;
             }
         }
+
+        @Override
+        public String toString()
+        {
+            return new StringJoiner(", ", GroupByPartitionIterator.class.getSimpleName() + "[", "]")
+                   .add("groupsPageSize=" + groupsPageSize)
+                   .add("subPageSize=" + subPageSize)
+                   .add("endOfData=" + endOfData)
+                   .add("closed=" + closed)
+                   .add("limits=" + limits)
+                   .add("lastPartitionKey=" + lastPartitionKey)
+                   .add("lastClustering=" + ((lastClustering != null && subPager.executionController() != null) ? lastClustering.toString(subPager.executionController().metadata()): String.valueOf(lastClustering)))
+                   .add("initialMaxRemaining=" + initialMaxRemaining)
+                   .add("sub-pager=" + subPager.toString())
+                   .toString();
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return new StringJoiner(", ", AggregationQueryPager.class.getSimpleName() + "[", "]")
+               .add("limits=" + limits)
+               .add("subPageSize=" + subPageSize)
+               .add("subPager=" + subPager)
+               .toString();
     }
 
     /**
@@ -405,19 +472,19 @@ public Row next()
      */
     public final class AggregationPartitionIterator extends GroupByPartitionIterator
     {
-        public AggregationPartitionIterator(int pageSize,
+        public AggregationPartitionIterator(PageSize subPageSize,
                                             ConsistencyLevel consistency,
                                             ClientState clientState,
                                             Dispatcher.RequestTime requestTime)
         {
-            super(pageSize, consistency, clientState, requestTime);
+            super(PageSize.NONE, subPageSize, consistency, clientState, requestTime);
         }
 
-        public AggregationPartitionIterator(int pageSize,
+        public AggregationPartitionIterator(PageSize subPageSize,
                                             ReadExecutionController executionController,
                                             Dispatcher.RequestTime requestTime)
         {
-            super(pageSize, executionController, requestTime);
+            super(PageSize.NONE, subPageSize, executionController, requestTime);
         }
 
         @Override
@@ -430,15 +497,9 @@ protected QueryPager updatePagerLimit(QueryPager pager,
         }
 
         @Override
-        protected boolean isDone(int pageSize, int counted)
-        {
-            return false;
-        }
-
-        @Override
-        protected int computeSubPageSize(int pageSize, int counted)
+        protected int getRemaining()
         {
-            return pageSize;
+            return initialMaxRemaining;
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
index 842eb35e8196..740c7de452fb 100644
--- a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
+++ b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
@@ -18,10 +18,17 @@
 package org.apache.cassandra.service.pager;
 
 import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.AbstractIterator;
 
 import java.util.Arrays;
+import java.util.StringJoiner;
+
+import javax.annotation.Nonnull;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.rows.*;
@@ -47,12 +54,24 @@
  */
 public class MultiPartitionPager<T extends SinglePartitionReadQuery> implements QueryPager
 {
+    private static final Logger logger = LoggerFactory.getLogger(MultiPartitionPager.class);
+
+    private static final SinglePartitionPager[] NO_PAGERS = new SinglePartitionPager[0];
+
+    // a pager per queried partition
+    @Nonnull
     private final SinglePartitionPager[] pagers;
+
+    // user limit
     private final DataLimits limit;
 
     private final long nowInSec;
 
+    // the number of rows left to be returned according to the user limits (those provided in query)
+    // when remaining reaches 0, the pager is considered exhausted
     private int remaining;
+
+    // the index of the current single partition pager
     private int current;
 
     public MultiPartitionPager(SinglePartitionReadQuery.Group<T> group, PagingState state, ProtocolVersion protocolVersion)
@@ -70,7 +89,7 @@ public MultiPartitionPager(SinglePartitionReadQuery.Group<T> group, PagingState
 
         if (i >= group.queries.size())
         {
-            pagers = null;
+            pagers = NO_PAGERS;
             return;
         }
 
@@ -123,7 +142,8 @@ public PagingState state()
 
     public boolean isExhausted()
     {
-        if (remaining <= 0 || pagers == null)
+        assert remaining >= 0;
+        if (remaining == 0)
             return true;
 
         while (current < pagers.length)
@@ -150,22 +170,26 @@ public ReadExecutionController executionController()
 
     @SuppressWarnings("resource") // iter closed via countingIter
     @Override
-    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException
+    public PartitionIterator fetchPage(PageSize pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException
     {
-        int toQuery = Math.min(remaining, pageSize);
-        return new PagersIterator(toQuery, consistency, clientState, null, requestTime);
+        return new PagersIterator(pageSize, consistency, clientState, null, requestTime);
     }
 
-    public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController) throws RequestValidationException, RequestExecutionException
+    public PartitionIterator fetchPageInternal(PageSize pageSize, ReadExecutionController executionController) throws RequestValidationException, RequestExecutionException
     {
-        int toQuery = Math.min(remaining, pageSize);
-        return new PagersIterator(toQuery, null, null, executionController, Dispatcher.RequestTime.forImmediateExecution());
+        return new PagersIterator(pageSize, null, null, executionController, Dispatcher.RequestTime.forImmediateExecution());
     }
 
+    /**
+     * This is an iterator over RowIterators (subsequent partitions). It starts from {@link #pagers} at {@link #current}
+     * and make sure that the overall amount of data does not exceed the provided {@link PagersIterator#pageSize}.
+     * This means that it can cut the row iteration in the first partition or return multiple partitions and cut the
+     * row iterator in n-th partition. It will update the {@link #current} index and {@link #remaining} as it goes.
+     */
     private class PagersIterator extends AbstractIterator<RowIterator> implements PartitionIterator
     {
-        private final int pageSize;
-        private PartitionIterator result;
+        private final PageSize pageSize;
+        private PartitionIterator partitionIterator;
         private boolean closed;
         private final Dispatcher.RequestTime requestTime;
 
@@ -176,32 +200,48 @@ private class PagersIterator extends AbstractIterator<RowIterator> implements Pa
         // For internal queries
         private final ReadExecutionController executionController;
 
-        private int pagerMaxRemaining;
+        private int countedRows;
+        private int countedBytes;
         private int counted;
 
-        public PagersIterator(int pageSize, ConsistencyLevel consistency, ClientState clientState, ReadExecutionController executionController, Dispatcher.RequestTime requestTime)
+        public PagersIterator(PageSize pageSize, ConsistencyLevel consistency, ClientState clientState, ReadExecutionController executionController, Dispatcher.RequestTime requestTime)
         {
             this.pageSize = pageSize;
             this.consistency = consistency;
             this.clientState = clientState;
             this.executionController = executionController;
             this.requestTime = requestTime;
+
+            if (logger.isTraceEnabled())
+                logger.trace("Fetching a new page - created {}", this);
         }
 
         protected RowIterator computeNext()
         {
-            while (result == null || !result.hasNext())
+            while (partitionIterator == null || !partitionIterator.hasNext())
             {
-                if (result != null)
+                DataLimits.Counter lastPageCounter = null;
+                if (partitionIterator != null)
                 {
-                    result.close();
-                    counted += pagerMaxRemaining - pagers[current].maxRemaining();
+                    // we've just reached the end of partition,
+                    // let's close the row iterator and update the global counters
+                    partitionIterator.close();
+
+                    lastPageCounter = pagers[current].getLastCounter();
+                    countedRows += lastPageCounter.rowsCounted();
+                    countedBytes += lastPageCounter.bytesCounted();
+                    counted += lastPageCounter.counted();
+                    remaining -= lastPageCounter.counted();
                 }
 
-                // We are done if we have reached the page size or in the case of GROUP BY if the current pager
-                // is not exhausted.
-                boolean isDone = counted >= pageSize
-                        || (result != null && limit.isGroupByLimit() && !pagers[current].isExhausted());
+                // We are done if:
+                // - we have reached the page size,
+                // - or in the case of GROUP BY if the current pager is not exhausted - which means that we read all the rows withing the limit before exhausting the pager
+                boolean isDone = pageSize.isCompleted(countedRows, PageSize.PageUnit.ROWS)
+                                 || pageSize.isCompleted(countedBytes, PageSize.PageUnit.BYTES)
+                                 || limit.count() <= counted
+                                 || limit.bytes() <= countedBytes
+                                 || (partitionIterator != null && limit.isGroupByLimit() && !pagers[current].isExhausted());
 
                 // isExhausted() will sets us on the first non-exhausted pager
                 if (isDone || isExhausted())
@@ -210,20 +250,44 @@ protected RowIterator computeNext()
                     return endOfData();
                 }
 
-                pagerMaxRemaining = pagers[current].maxRemaining();
-                int toQuery = pageSize - counted;
-                result = consistency == null
-                       ? pagers[current].fetchPageInternal(toQuery, executionController)
-                       : pagers[current].fetchPage(toQuery, consistency, clientState, requestTime);
+                // we will update the limits for the current pager before using it so that we can be sure we don't fetch
+                // more than remaining or more than what was left to be fetched according to the recently set limits
+                // (for example in case of groups paging) - that later limit is just the limit which was set minus what
+                // we counted so far
+                int newCountedLimit = Math.max(0, Math.min(remaining, limit.count() - counted));
+                // this works exactly the same way as above - it is required for the limits imposed by Guardrails,
+                // whihc are set on the query
+                int newBytesLimit = Math.max(0, limit.bytes() - countedBytes);
+
+                DataLimits updatedLimit = pagers[current].limits.withCountedLimit(newCountedLimit).withBytesLimit(newBytesLimit);
+                pagers[current] = pagers[current].withUpdatedLimit(updatedLimit);
+
+                PageSize remainingPagePart = pageSize.withDecreasedRows(countedRows)
+                                                     .withDecreasedBytes(countedBytes);
+
+                partitionIterator = consistency == null
+                                    ? pagers[current].fetchPageInternal(remainingPagePart, executionController)
+                                    : pagers[current].fetchPage(remainingPagePart, consistency, clientState, requestTime);
             }
-            return result.next();
+            return partitionIterator.next();
         }
 
         public void close()
         {
-            remaining -= counted;
-            if (result != null && !closed)
-                result.close();
+            if (partitionIterator != null && !closed)
+                partitionIterator.close();
+        }
+
+        @Override
+        public String toString()
+        {
+            return new StringJoiner(", ", PagersIterator.class.getSimpleName() + "[", "]")
+                   .add("pageSize=" + pageSize)
+                   .add("closed=" + closed)
+                   .add("countedRows=" + countedRows)
+                   .add("countedBytes=" + countedBytes)
+                   .add("counted=" + counted)
+                   .toString();
         }
     }
 
@@ -231,4 +295,15 @@ public int maxRemaining()
     {
         return remaining;
     }
+
+    @Override
+    public String toString()
+    {
+        return new StringJoiner(", ", MultiPartitionPager.class.getSimpleName() + "[", "]")
+               .add("current=" + current)
+               .add("pagers.length=" + pagers.length)
+               .add("limit=" + limit)
+               .add("remaining=" + remaining)
+               .toString();
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/pager/PagedPartitionIterator.java b/src/java/org/apache/cassandra/service/pager/PagedPartitionIterator.java
new file mode 100644
index 000000000000..862558c0ba49
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/pager/PagedPartitionIterator.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.pager;
+
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Dispatcher;
+
+/**
+ * A partition iterator that reads its rows from a provided {@link QueryPager}, consuming it until it's exhausted.
+ */
+abstract class PagedPartitionIterator implements PartitionIterator
+{
+    protected final QueryPager pager;
+    protected final PageSize pageSize;
+    protected PartitionIterator current;
+
+    protected PagedPartitionIterator(QueryPager pager, PageSize pageSize)
+    {
+        this.pager = pager;
+        this.pageSize = pageSize;
+    }
+
+    @Override
+    public void close()
+    {
+        if (current != null)
+        {
+            current.close();
+            current = null;
+        }
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        maybeFetch();
+        return current != null && current.hasNext();
+    }
+
+    @Override
+    public RowIterator next()
+    {
+        maybeFetch();
+        return current.next();
+    }
+
+    private void maybeFetch()
+    {
+        if (current == null || !current.hasNext())
+        {
+            if (current != null)
+            {
+                current.close();
+                current = null;
+            }
+
+            if (!pager.isExhausted())
+                current = fetch();
+        }
+    }
+
+    protected abstract PartitionIterator fetch();
+
+    /**
+     * {@link PagedPartitionIterator} that for local queries.
+     */
+    public static class Internal extends PagedPartitionIterator
+    {
+        private final ReadExecutionController controller;
+
+        public Internal(QueryPager pager, PageSize pageSize, ReadExecutionController controller)
+        {
+            super(pager, pageSize);
+            this.controller = controller;
+        }
+
+        @Override
+        protected PartitionIterator fetch()
+        {
+            return pager.fetchPageInternal(pageSize, controller);
+        }
+    }
+
+    /**
+     * {@link PagedPartitionIterator} that for distributed queries.
+     */
+    public static class Distributed extends PagedPartitionIterator
+    {
+        private final ConsistencyLevel consistency;
+        private final ClientState state;
+        private final Dispatcher.RequestTime requestTime;
+
+        public Distributed(QueryPager pager,
+                           PageSize pageSize,
+                           ConsistencyLevel consistency,
+                           ClientState state,
+                           Dispatcher.RequestTime requestTime)
+        {
+            super(pager, pageSize);
+            this.consistency = consistency;
+            this.state = state;
+            this.requestTime = requestTime;
+        }
+
+        @Override
+        protected PartitionIterator fetch()
+        {
+            return pager.fetchPage(pageSize, consistency, state, requestTime);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/pager/PagingState.java b/src/java/org/apache/cassandra/service/pager/PagingState.java
index 627f958ff3ec..ec43068f3830 100644
--- a/src/java/org/apache/cassandra/service/pager/PagingState.java
+++ b/src/java/org/apache/cassandra/service/pager/PagingState.java
@@ -23,6 +23,9 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
@@ -47,6 +50,8 @@
 @SuppressWarnings("WeakerAccess")
 public class PagingState
 {
+    private static final Logger logger = LoggerFactory.getLogger(PagingState.class);
+
     public final ByteBuffer partitionKey;  // Can be null for single partition queries.
     public final RowMark rowMark;          // Can be null if not needed.
     public final int remaining;
@@ -113,10 +118,14 @@ public static PagingState deserialize(ByteBuffer bytes, ProtocolVersion protocol
         }
         catch (IOException e)
         {
-            throw new ProtocolException("Invalid value for the paging state");
+            String msg =  "Failed to deserialize the paging state with protocol version: " + protocolVersion;
+            logger.trace(msg, e);
+            throw new ProtocolException(msg, protocolVersion);
         }
 
-        throw new ProtocolException("Invalid value for the paging state");
+        String msg =  "The serialized paging state does not match any serialization format for protocol version: " + protocolVersion;
+        logger.trace(msg);
+        throw new ProtocolException(msg, protocolVersion);
     }
 
     /*
diff --git a/src/java/org/apache/cassandra/service/pager/PartitionRangeQueryPager.java b/src/java/org/apache/cassandra/service/pager/PartitionRangeQueryPager.java
index 3ee90d707416..ef64f65ba19d 100644
--- a/src/java/org/apache/cassandra/service/pager/PartitionRangeQueryPager.java
+++ b/src/java/org/apache/cassandra/service/pager/PartitionRangeQueryPager.java
@@ -17,6 +17,9 @@
  */
 package org.apache.cassandra.service.pager;
 
+import java.util.StringJoiner;
+
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.rows.Row;
@@ -74,15 +77,14 @@ public PagingState state()
     }
 
     @Override
-    protected PartitionRangeReadQuery nextPageReadQuery(int pageSize)
+    protected PartitionRangeReadQuery nextPageReadQuery(PageSize pageSize, DataLimits limits)
     {
-        DataLimits limits;
         DataRange fullRange = query.dataRange();
         DataRange pageRange;
         if (lastReturnedKey == null)
         {
             pageRange = fullRange;
-            limits = query.limits().forPaging(pageSize);
+            limits = limits.forPaging(pageSize);
         }
         // if the last key was the one of the end of the range we know that we are done
         else if (lastReturnedKey.equals(fullRange.keyRange().right) && remainingInPartition() == 0 && lastReturnedRow == null)
@@ -97,12 +99,12 @@ else if (lastReturnedKey.equals(fullRange.keyRange().right) && remainingInPartit
             if (includeLastKey)
             {
                 pageRange = fullRange.forPaging(bounds, query.metadata().comparator, lastReturnedRow.clustering(query.metadata()), false);
-                limits = query.limits().forPaging(pageSize, lastReturnedKey.getKey(), remainingInPartition());
+                limits = limits.forPaging(pageSize, lastReturnedKey.getKey(), remainingInPartition());
             }
             else
             {
                 pageRange = fullRange.forSubRange(bounds);
-                limits = query.limits().forPaging(pageSize);
+                limits = limits.forPaging(pageSize);
             }
         }
 
@@ -145,4 +147,14 @@ public boolean isTopK()
     {
         return query.isTopK();
     }
+
+    @Override
+    public String toString()
+    {
+        return new StringJoiner(", ", PartitionRangeQueryPager.class.getSimpleName() + "[", "]")
+               .add("super=" + super.toString())
+               .add("lastReturnedKey=" + lastReturnedKey)
+               .add("lastReturnedRow=" + (lastReturnedRow != null ? lastReturnedRow.clustering(query.metadata()).toString(query.metadata()) : null))
+               .toString();
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/pager/QueryPager.java b/src/java/org/apache/cassandra/service/pager/QueryPager.java
index 1619af8a3898..cbd76e173216 100644
--- a/src/java/org/apache/cassandra/service/pager/QueryPager.java
+++ b/src/java/org/apache/cassandra/service/pager/QueryPager.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.service.pager;
 
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.filter.DataLimits;
@@ -55,12 +56,12 @@ public ReadExecutionController executionController()
             return ReadExecutionController.empty();
         }
 
-        public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException
+        public PartitionIterator fetchPage(PageSize pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException
         {
             return EmptyIterators.partition();
         }
 
-        public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController) throws RequestValidationException, RequestExecutionException
+        public PartitionIterator fetchPageInternal(PageSize pageSize, ReadExecutionController executionController) throws RequestValidationException, RequestExecutionException
         {
             return EmptyIterators.partition();
         }
@@ -95,13 +96,13 @@ public QueryPager withUpdatedLimit(DataLimits newLimits)
      * {@code consistency} is a serial consistency.
      * @return the page of result.
      */
-    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException;
+    public PartitionIterator fetchPage(PageSize pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException;
 
     /**
      * Starts a new read operation.
      * <p>
-     * This must be called before {@link fetchPageInternal} and passed to it to protect the read.
-     * The returned object <b>must</b> be closed on all path and it is thus strongly advised to
+     * This must be called before {@link #fetchPageInternal(PageSize, ReadExecutionController)} and passed to it
+     * to protect the read. The returned object <b>must</b> be closed on all path and it is thus strongly advised to
      * use it in a try-with-ressource construction.
      *
      * @return a newly started order group for this {@code QueryPager}.
@@ -115,7 +116,7 @@ public QueryPager withUpdatedLimit(DataLimits newLimits)
      * @param executionController the {@code ReadExecutionController} protecting the read.
      * @return the page of result.
      */
-    public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController) throws RequestValidationException, RequestExecutionException;
+    public PartitionIterator fetchPageInternal(PageSize pageSize, ReadExecutionController executionController) throws RequestValidationException, RequestExecutionException;
 
     /**
      * Whether or not this pager is exhausted, i.e. whether or not a call to
@@ -150,7 +151,6 @@ public QueryPager withUpdatedLimit(DataLimits newLimits)
      */
     public QueryPager withUpdatedLimit(DataLimits newLimits);
 
-
     /**
      * @return true given read query is a top-k request
      */
@@ -158,4 +158,36 @@ default boolean isTopK()
     {
         return false;
     }
+
+    /**
+     * Reads all the rows in this pager using paging internally.
+     * </p>
+     * Pages will be lazily fetched according to the provided page size as the returned {@link PartitionIterator} is
+     * consumed.
+     *
+     * @param pageSize the maximum number of elements to be fetched on each internal page.
+     * @param consistency the consistency level to achieve for the query.
+     * @param clientState the {@code ClientState} for the query. In practice, this can be null unless {@code consistency}
+     * is a serial consistency.
+     * @return all the rows in this pager.
+     */
+    default PartitionIterator readAll(PageSize pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime)
+    {
+        return new PagedPartitionIterator.Distributed(this, pageSize, consistency, clientState, requestTime);
+    }
+
+    /**
+     * Reads all the rows in this pager using paging internally, using local queries.
+     * </p>
+     * Pages will be lazily fetched according to the provided page size as the returned {@link PartitionIterator} is
+     * consumed.
+     *
+     * @param pageSize the maximum number of elements to be fetched on each internal page.
+     * @param executionController the {@code ReadExecutionController} protecting the read.
+     * @return all the rows in this pager.
+     */
+    default PartitionIterator readAllInternal(PageSize pageSize, ReadExecutionController executionController)
+    {
+        return new PagedPartitionIterator.Internal(this, pageSize, executionController);
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java b/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java
index 832526e5ce6c..68d355571490 100644
--- a/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java
+++ b/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java
@@ -18,7 +18,9 @@
 package org.apache.cassandra.service.pager;
 
 import java.nio.ByteBuffer;
+import java.util.StringJoiner;
 
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.db.filter.*;
@@ -26,7 +28,7 @@
 
 /**
  * Common interface to single partition queries (by slice and by name).
- *
+ * <p>
  * For use by MultiPartitionPager.
  */
 public class SinglePartitionPager extends AbstractQueryPager<SinglePartitionReadQuery>
@@ -78,21 +80,27 @@ public DataLimits limits()
     public PagingState state()
     {
         return lastReturned == null
-             ? null
-             : new PagingState(null, lastReturned, maxRemaining(), remainingInPartition());
+               ? null
+               : new PagingState(null, lastReturned, maxRemaining(), remainingInPartition());
     }
 
     @Override
-    protected SinglePartitionReadQuery nextPageReadQuery(int pageSize)
+    protected SinglePartitionReadQuery nextPageReadQuery(PageSize pageSize, DataLimits limits)
     {
         Clustering<?> clustering = lastReturned == null ? null : lastReturned.clustering(query.metadata());
-        DataLimits limits = lastReturned == null
-                          ? limits().forPaging(pageSize)
-                          : limits().forPaging(pageSize, key(), remainingInPartition());
+        limits = lastReturned == null
+                 ? limits.forPaging(pageSize)
+                 : limits.forPaging(pageSize, key(), remainingInPartition());
 
         return query.forPaging(clustering, limits);
     }
 
+    @Override
+    public boolean isExhausted()
+    {
+        return super.isExhausted() || remainingInPartition() == 0;
+    }
+
     protected void recordLast(DecoratedKey key, Row last)
     {
         if (last != null && last.clustering() != Clustering.STATIC_CLUSTERING)
@@ -103,4 +111,13 @@ protected boolean isPreviouslyReturnedPartition(DecoratedKey key)
     {
         return lastReturned != null;
     }
+
+    @Override
+    public String toString()
+    {
+        return new StringJoiner(", ", SinglePartitionPager.class.getSimpleName() + "[", "]")
+               .add("super=" + super.toString())
+               .add("lastReturned=" + (lastReturned != null ? lastReturned.clustering(query.metadata()).toString(query.metadata()) : null))
+               .toString();
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/paxos/Commit.java b/src/java/org/apache/cassandra/service/paxos/Commit.java
index 3aa8d65bcef0..c1d123ae060f 100644
--- a/src/java/org/apache/cassandra/service/paxos/Commit.java
+++ b/src/java/org/apache/cassandra/service/paxos/Commit.java
@@ -29,6 +29,7 @@
 import com.google.common.base.Objects;
 
 import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.io.IVersionedSerializer;
@@ -342,7 +343,7 @@ public String toString()
 
     public String toString(String kind)
     {
-        return String.format("%s(%d:%s, %d:%s)", kind, ballot.uuidTimestamp(), ballot, update.stats().minTimestamp, update.toString(false));
+        return String.format("%s(%d:%s, %d:%s)", kind, ballot.uuidTimestamp(), ballot, update.stats().minTimestamp, Partition.toString(update, false));
     }
 
     /**
@@ -472,7 +473,7 @@ public static boolean timestampsClash(@Nullable Ballot a, @Nullable Ballot b)
 
     private static PartitionUpdate withTimestamp(PartitionUpdate update, long timestamp)
     {
-        return new PartitionUpdate.Builder(update, 0).updateAllTimestamp(timestamp).build();
+        return update.withUpdatedTimestamps(timestamp);
     }
 
     public static class CommitSerializer<T extends Commit> implements IVersionedSerializer<T>
diff --git a/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java
new file mode 100644
index 000000000000..7781dbd78b9e
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java
@@ -0,0 +1,59 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.service.paxos;
+
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
+import org.apache.cassandra.sensors.SensorsFactory;
+import org.apache.cassandra.sensors.Type;
+import org.apache.cassandra.service.MutatorProvider;
+import org.apache.cassandra.tracing.Tracing;
+
+public class CommitVerbHandler implements IVerbHandler<Commit>
+{
+    public static final CommitVerbHandler instance = new CommitVerbHandler();
+
+    public void doVerb(Message<Commit> message)
+    {
+        // Initialize the sensor and set ExecutorLocals
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors(message.payload.update.metadata().keyspace);
+        Context context = Context.from(message.payload.update.metadata());
+        sensors.registerSensor(context, Type.WRITE_BYTES);
+        sensors.registerSensor(context, Type.INTERNODE_BYTES);
+        sensors.incrementSensor(context, Type.INTERNODE_BYTES, message.payloadSize(MessagingService.current_version));
+        RequestTracker.instance.set(sensors);
+
+        PaxosState.commitDirect(message.payload, p -> MutatorProvider.getCustomOrDefault().onAppliedProposal(p));
+
+        Tracing.trace("Enqueuing acknowledge to {}", message.from());
+        Message.Builder<NoPayload> reply = message.emptyResponseBuilder();
+
+        // no need to calculate outbound internode bytes because the response is NoPayload
+        SensorsCustomParams.addSensorsToInternodeResponse(sensors, reply);
+        MessagingService.instance().send(reply.build(), message.from());
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java b/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java
index 7f38567f6a15..a2b8970d7f6e 100644
--- a/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java
+++ b/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java
@@ -26,7 +26,10 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.NoSpamLogger;
@@ -48,8 +51,6 @@
 import static java.util.Arrays.stream;
 import static java.util.concurrent.TimeUnit.*;
 import static org.apache.cassandra.config.DatabaseDescriptor.*;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.Clock.waitUntil;
 
@@ -316,8 +317,9 @@ static class Bound
             this.onFailure = onFailure;
             this.modifier = modifier;
             this.selector = selector;
-            this.reads = new TimeLimitedLatencySupplier(casReadMetrics.latency::getSnapshot, 10L, SECONDS);
-            this.writes = new TimeLimitedLatencySupplier(casWriteMetrics.latency::getSnapshot, 10L, SECONDS);
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
+            this.reads = new TimeLimitedLatencySupplier(metrics.casReadMetrics.executionTimeMetrics.latency::getSnapshot, 10L, SECONDS);
+            this.writes = new TimeLimitedLatencySupplier(metrics.casWriteMetrics.executionTimeMetrics.latency::getSnapshot, 10L, SECONDS);
         }
 
         long get(int attempts)
@@ -377,7 +379,7 @@ long computeWaitUntilForContention(int attempts, TableMetadata table, DecoratedK
     {
         if (attempts >= traceAfterAttempts && !Tracing.isTracing())
         {
-            Tracing.instance.newSession(Tracing.TraceType.QUERY);
+            Tracing.instance.newSession(ClientState.forInternalCalls(), Tracing.TraceType.QUERY);
             Tracing.instance.begin(type.traceTitle,
                                    ImmutableMap.of(
                                        "keyspace", table.keyspace,
diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.java b/src/java/org/apache/cassandra/service/paxos/Paxos.java
index e38c9cfd0347..b4e0305955a8 100644
--- a/src/java/org/apache/cassandra/service/paxos/Paxos.java
+++ b/src/java/org/apache/cassandra/service/paxos/Paxos.java
@@ -35,6 +35,9 @@
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Maps;
 
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -112,11 +115,6 @@
 import static org.apache.cassandra.db.ConsistencyLevel.*;
 import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
 import static org.apache.cassandra.locator.ReplicaLayout.forTokenWriteLiveAndDown;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsMap;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetricsMap;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL;
 import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot;
@@ -422,7 +420,7 @@ void assureSufficientLiveNodes(boolean isWrite) throws UnavailableException
         {
             if (sizeOfConsensusQuorum > sizeOfPoll())
             {
-                mark(isWrite, m -> m.unavailables, consistencyForConsensus);
+                mark(isWrite, m -> m.unavailables, consistencyForConsensus, ClientRequestsMetricsProvider.instance.metrics(keyspace.getName()));
                 throw new UnavailableException("Cannot achieve consistency level " + consistencyForConsensus, consistencyForConsensus, sizeOfConsensusQuorum, sizeOfPoll());
             }
         }
@@ -545,11 +543,11 @@ private static int failureCount(Map<InetAddressAndPort, RequestFailureReason> fa
         /**
          * update relevant counters and throw the relevant exception
          */
-        RequestExecutionException markAndThrowAsTimeoutOrFailure(boolean isWrite, ConsistencyLevel consistency, int failedAttemptsDueToContention)
+        RequestExecutionException markAndThrowAsTimeoutOrFailure(boolean isWrite, ConsistencyLevel consistency, int failedAttemptsDueToContention, ClientRequestsMetrics metrics)
         {
             if (isFailure)
             {
-                mark(isWrite, m -> m.failures, consistency);
+                mark(isWrite, m -> m.failures, consistency, metrics);
                 throw serverError != null ? new RequestFailureException(ExceptionCode.SERVER_ERROR, serverError, consistency, successes, required, failures)
                                           : isWrite
                                             ? new WriteFailureException(consistency, successes, required, WriteType.CAS, failures)
@@ -557,7 +555,7 @@ RequestExecutionException markAndThrowAsTimeoutOrFailure(boolean isWrite, Consis
             }
             else
             {
-                mark(isWrite, m -> m.timeouts, consistency);
+                mark(isWrite, m -> m.timeouts, consistency, metrics);
                 throw isWrite
                         ? new CasWriteTimeoutException(WriteType.CAS, consistency, successes, required, failedAttemptsDueToContention)
                         : new ReadTimeoutException(consistency, successes, required, false);
@@ -658,11 +656,12 @@ private static RowIterator cas(DecoratedKey partitionKey,
         SinglePartitionReadCommand readCommand = request.readCommand(FBUtilities.nowInSeconds());
         TableMetadata metadata = readCommand.metadata();
 
-        consistencyForConsensus.validateForCas();
-        consistencyForCommit.validateForCasCommit(Keyspace.open(metadata.keyspace).getReplicationStrategy());
+        consistencyForConsensus.validateForCas(metadata.keyspace, clientState);
+        consistencyForCommit.validateForCasCommit(Keyspace.open(metadata.keyspace).getReplicationStrategy(), metadata.keyspace, clientState);
 
         Ballot minimumBallot = null;
         int failedAttemptsDueToContention = 0;
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(metadata.keyspace);
         try (PaxosOperationLock lock = PaxosState.lock(partitionKey, metadata, proposeDeadline, consistencyForConsensus, true))
         {
             Paxos.Async<PaxosCommit.Status> commit = null;
@@ -690,7 +689,7 @@ private static RowIterator cas(DecoratedKey partitionKey,
                     if (getPaxosVariant() == v2_without_linearizable_reads_or_rejected_writes)
                     {
                         Tracing.trace("CAS precondition rejected", current);
-                        casWriteMetrics.conditionNotMet.inc();
+                        metrics.casWriteMetrics.conditionNotMet.inc();
                         return current.rowIterator();
                     }
 
@@ -747,7 +746,7 @@ else if (begin.isPromised)
                     default: throw new IllegalStateException();
 
                     case MAYBE_FAILURE:
-                        throw propose.maybeFailure().markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention);
+                        throw propose.maybeFailure().markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention, metrics);
 
                     case SUCCESS:
                     {
@@ -774,14 +773,14 @@ else if (begin.isPromised)
                                 // our proposal.  We yield our uncertainty to the caller via timeout exception.
                                 // TODO: should return more useful result to client, and should also avoid this situation where possible
                                 throw new MaybeFailure(false, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, 0, emptyMap())
-                                        .markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention);
+                                        .markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention, metrics);
 
                             case NO:
                                 minimumBallot = propose.superseded().by;
                                 // We have been superseded without our proposal being accepted by anyone, so we can safely retry
                                 Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)");
                                 if (!waitForContention(proposeDeadline, ++failedAttemptsDueToContention, metadata, partitionKey, consistencyForConsensus, WRITE))
-                                    throw MaybeFailure.noResponses(participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention);
+                                    throw MaybeFailure.noResponses(participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention, metrics);
                         }
                     }
                 }
@@ -792,7 +791,7 @@ else if (begin.isPromised)
             {
                 PaxosCommit.Status result = commit.awaitUntil(commitDeadline);
                 if (!result.isSuccess())
-                    throw result.maybeFailure().markAndThrowAsTimeoutOrFailure(true, consistencyForCommit, failedAttemptsDueToContention);
+                    throw result.maybeFailure().markAndThrowAsTimeoutOrFailure(true, consistencyForCommit, failedAttemptsDueToContention, metrics);
             }
             Tracing.trace("CAS successful");
             return null;
@@ -804,20 +803,21 @@ else if (begin.isPromised)
 
             if (failedAttemptsDueToContention > 0)
             {
-                casWriteMetrics.contention.update(failedAttemptsDueToContention);
+                metrics.casWriteMetrics.contention.update(failedAttemptsDueToContention);
                 openAndGetStore(metadata).metric.topCasPartitionContention.addSample(partitionKey.getKey(), failedAttemptsDueToContention);
             }
 
 
-            casWriteMetrics.addNano(latency);
-            writeMetricsMap.get(consistencyForConsensus).addNano(latency);
+            metrics.casWriteMetrics.executionTimeMetrics.addNano(latency);
+            metrics.writeMetricsForLevel(consistencyForConsensus).executionTimeMetrics.addNano(latency);
         }
     }
 
     private static RowIterator conditionNotMet(FilteredPartition read)
     {
         Tracing.trace("CAS precondition rejected", read);
-        casWriteMetrics.conditionNotMet.inc();
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
+        metrics.casWriteMetrics.conditionNotMet.inc();
         return read.rowIterator();
     }
 
@@ -844,6 +844,7 @@ private static PartitionIterator read(SinglePartitionReadCommand.Group group, Co
         int failedAttemptsDueToContention = 0;
         Ballot minimumBallot = null;
         SinglePartitionReadCommand read = group.queries.get(0);
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(read.metadata().keyspace);
         try (PaxosOperationLock lock = PaxosState.lock(read.partitionKey(), read.metadata(), deadline, consistencyForConsensus, false))
         {
             while (true)
@@ -873,7 +874,7 @@ private static PartitionIterator read(SinglePartitionReadCommand.Group group, Co
                     default: throw new IllegalStateException();
 
                     case MAYBE_FAILURE:
-                        throw propose.maybeFailure().markAndThrowAsTimeoutOrFailure(false, consistencyForConsensus, failedAttemptsDueToContention);
+                        throw propose.maybeFailure().markAndThrowAsTimeoutOrFailure(false, consistencyForConsensus, failedAttemptsDueToContention, metrics);
 
                     case SUCCESS:
                         return begin.readResponse;
@@ -888,14 +889,14 @@ private static PartitionIterator read(SinglePartitionReadCommand.Group group, Co
                                 // our proposal.  We yield our uncertainty to the caller via timeout exception.
                                 // TODO: should return more useful result to client, and should also avoid this situation where possible
                                 throw new MaybeFailure(false, begin.participants.sizeOfPoll(), begin.participants.sizeOfConsensusQuorum, 0, emptyMap())
-                                      .markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention);
+                                      .markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention, metrics);
 
                             case NO:
                                 minimumBallot = propose.superseded().by;
                                 // We have been superseded without our proposal being accepted by anyone, so we can safely retry
                                 Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)");
                                 if (!waitForContention(deadline, ++failedAttemptsDueToContention, group.metadata(), group.queries.get(0).partitionKey(), consistencyForConsensus, READ))
-                                    throw MaybeFailure.noResponses(begin.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention);
+                                    throw MaybeFailure.noResponses(begin.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention, metrics);
                         }
                 }
             }
@@ -907,13 +908,13 @@ private static PartitionIterator read(SinglePartitionReadCommand.Group group, Co
             // client request. This is a measure of how long this specific individual read took, not total time since
             // processing of the client began.
             long latency = nanoTime() - start;
-            readMetrics.addNano(latency);
-            casReadMetrics.addNano(latency);
-            readMetricsMap.get(consistencyForConsensus).addNano(latency);
+            metrics.readMetrics.executionTimeMetrics.addNano(latency);
+            metrics.casReadMetrics.executionTimeMetrics.addNano(latency);
+            metrics.readMetricsForLevel(consistencyForConsensus).executionTimeMetrics.addNano(latency);
             TableMetadata table = read.metadata();
             Keyspace.open(table.keyspace).getColumnFamilyStore(table.name).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS);
             if (failedAttemptsDueToContention > 0)
-                casReadMetrics.contention.update(failedAttemptsDueToContention);
+                metrics.casReadMetrics.contention.update(failedAttemptsDueToContention);
         }
     }
 
@@ -974,6 +975,7 @@ private static BeginResult begin(long deadline,
         Participants initialParticipants = Participants.get(query.metadata(), query.partitionKey(), consistencyForConsensus);
         initialParticipants.assureSufficientLiveNodes(isWrite);
         PaxosPrepare preparing = prepare(minimumBallot, initialParticipants, query, isWrite, acceptEarlyReadPermission);
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(query.metadata().keyspace);
         while (true)
         {
             // prepare
@@ -996,9 +998,9 @@ private static BeginResult begin(long deadline,
                     FoundIncompleteAccepted inProgress = prepare.incompleteAccepted();
                     Tracing.trace("Finishing incomplete paxos round {}", inProgress.accepted);
                     if (isWrite)
-                        casWriteMetrics.unfinishedCommit.inc();
+                        metrics.casWriteMetrics.unfinishedCommit.inc();
                     else
-                        casReadMetrics.unfinishedCommit.inc();
+                        metrics.casReadMetrics.unfinishedCommit.inc();
 
                     // we DO NOT need to change the timestamp of this commit - either we or somebody else will finish it
                     // and the original timestamp is correctly linearised. By not updatinig the timestamp we leave enough
@@ -1013,7 +1015,7 @@ private static BeginResult begin(long deadline,
                         default: throw new IllegalStateException();
 
                         case MAYBE_FAILURE:
-                            throw proposeResult.maybeFailure().markAndThrowAsTimeoutOrFailure(isWrite, consistencyForConsensus, failedAttemptsDueToContention);
+                            throw proposeResult.maybeFailure().markAndThrowAsTimeoutOrFailure(isWrite, consistencyForConsensus, failedAttemptsDueToContention, metrics);
 
                         case SUCCESS:
                             retry = commitAndPrepare(repropose.agreed(), inProgress.participants, query, isWrite, acceptEarlyReadPermission);
@@ -1033,7 +1035,7 @@ private static BeginResult begin(long deadline,
                     Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting");
                     // sleep a random amount to give the other proposer a chance to finish
                     if (!waitForContention(deadline, ++failedAttemptsDueToContention, query.metadata(), query.partitionKey(), consistencyForConsensus, isWrite ? WRITE : READ))
-                        throw MaybeFailure.noResponses(prepare.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention);
+                        throw MaybeFailure.noResponses(prepare.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention, metrics);
                     retry = prepare(prepare.retryWithAtLeast(), prepare.participants, query, isWrite, acceptEarlyReadPermission);
                     break;
                 }
@@ -1044,7 +1046,7 @@ private static BeginResult begin(long deadline,
                     // round's proposal (if any).
                     PaxosPrepare.Success success = prepare.success();
 
-                    DataResolver<?, ?> resolver = new DataResolver(query, success.participants, NoopReadRepair.instance, new Dispatcher.RequestTime(query.creationTimeNanos()));
+                    DataResolver<?, ?> resolver = new DataResolver(query, success.participants, NoopReadRepair.instance, new Dispatcher.RequestTime(query.creationTimeNanos()), QueryInfoTracker.ReadTracker.NOOP);
                     for (int i = 0 ; i < success.responses.size() ; ++i)
                         resolver.preprocess(success.responses.get(i));
 
@@ -1066,7 +1068,7 @@ class WasRun implements Runnable { boolean v; public void run() { v = true; } }
                 }
 
                 case MAYBE_FAILURE:
-                    throw prepare.maybeFailure().markAndThrowAsTimeoutOrFailure(isWrite, consistencyForConsensus, failedAttemptsDueToContention);
+                    throw prepare.maybeFailure().markAndThrowAsTimeoutOrFailure(isWrite, consistencyForConsensus, failedAttemptsDueToContention, metrics);
 
                 case ELECTORATE_MISMATCH:
                     Participants participants = Participants.get(query.metadata(), query.partitionKey(), consistencyForConsensus);
@@ -1081,7 +1083,7 @@ class WasRun implements Runnable { boolean v; public void run() { v = true; } }
                 Tracing.trace("Some replicas have already promised a higher ballot than ours; retrying");
                 // sleep a random amount to give the other proposer a chance to finish
                 if (!waitForContention(deadline, ++failedAttemptsDueToContention, query.metadata(), query.partitionKey(), consistencyForConsensus, isWrite ? WRITE : READ))
-                    throw MaybeFailure.noResponses(prepare.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention);
+                    throw MaybeFailure.noResponses(prepare.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention, metrics);
                 retry = prepare(prepare.retryWithAtLeast(), prepare.participants, query, isWrite, acceptEarlyReadPermission);
             }
 
@@ -1107,17 +1109,17 @@ static ConsistencyLevel nonSerial(ConsistencyLevel serial)
         }
     }
 
-    private static void mark(boolean isWrite, Function<ClientRequestMetrics, Meter> toMark, ConsistencyLevel consistency)
+    private static void mark(boolean isWrite, Function<ClientRequestMetrics, Meter> toMark, ConsistencyLevel consistency, ClientRequestsMetrics metrics)
     {
         if (isWrite)
         {
-            toMark.apply(casWriteMetrics).mark();
-            toMark.apply(writeMetricsMap.get(consistency)).mark();
+            toMark.apply(metrics.casWriteMetrics).mark();
+            toMark.apply(metrics.writeMetricsForLevel(consistency)).mark();
         }
         else
         {
-            toMark.apply(casReadMetrics).mark();
-            toMark.apply(readMetricsMap.get(consistency)).mark();
+            toMark.apply(metrics.casReadMetrics).mark();
+            toMark.apply(metrics.readMetricsForLevel(consistency)).mark();
         }
     }
 
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosState.java b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
index a3f019e4bf25..b3b4dae5dae8 100644
--- a/src/java/org/apache/cassandra/service/paxos/PaxosState.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
@@ -1,5 +1,5 @@
 /*
- * 
+ *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -7,16 +7,16 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
- * 
+ *
  */
 package org.apache.cassandra.service.paxos;
 
@@ -26,9 +26,9 @@
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
-
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.function.Consumer;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
@@ -37,12 +37,19 @@
 import com.github.benmanes.caffeine.cache.Caffeine;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.metrics.PaxosMetrics;
-import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.WriteOptions;
+import org.apache.cassandra.db.WriteType;
 import org.apache.cassandra.exceptions.ReadTimeoutException;
 import org.apache.cassandra.exceptions.RequestTimeoutException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.metrics.PaxosMetrics;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosBallotTracker;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosStateTracker;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTracker;
@@ -51,15 +58,23 @@
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_DISABLE_COORDINATOR_LOCKING;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.config.Config.PaxosStatePurging.gc_grace;
 import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy;
 import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging;
-import static org.apache.cassandra.service.paxos.Commit.*;
-import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.*;
+import static org.apache.cassandra.service.paxos.Commit.Accepted;
 import static org.apache.cassandra.service.paxos.Commit.Accepted.latestAccepted;
+import static org.apache.cassandra.service.paxos.Commit.AcceptedWithTTL;
+import static org.apache.cassandra.service.paxos.Commit.Agreed;
+import static org.apache.cassandra.service.paxos.Commit.Committed;
 import static org.apache.cassandra.service.paxos.Commit.Committed.latestCommitted;
+import static org.apache.cassandra.service.paxos.Commit.CommittedWithTTL;
+import static org.apache.cassandra.service.paxos.Commit.Proposal;
 import static org.apache.cassandra.service.paxos.Commit.isAfter;
+import static org.apache.cassandra.service.paxos.Commit.latest;
+import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.PERMIT_READ;
+import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.PROMISE;
+import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.REJECT;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
  * We save to memory the result of each operation before persisting to disk, however each operation that performs
@@ -412,7 +427,7 @@ public static PaxosOperationLock lock(DecoratedKey partitionKey, TableMetadata m
             throw t;
         }
     }
-    
+
     private static RequestTimeoutException throwTimeout(TableMetadata metadata, ConsistencyLevel consistencyForConsensus, boolean isWrite)
     {
         int blockFor = consistencyForConsensus.blockFor(Keyspace.open(metadata.keyspace).getReplicationStrategy());
@@ -664,14 +679,19 @@ public Ballot acceptIfLatest(Proposal proposal)
 
     public void commit(Agreed commit)
     {
-        applyCommit(commit, this, (apply, to) ->
+        applyCommit(commit, this, c -> {}, (apply, to) ->
             currentUpdater.accumulateAndGet(to, new UnsafeSnapshot(apply), Snapshot::merge)
         );
     }
 
     public static void commitDirect(Commit commit)
     {
-        applyCommit(commit, null, (apply, ignore) -> {
+        commitDirect(commit, c -> {});
+    }
+    
+    public static void commitDirect(Commit commit, Consumer<Commit> callback)
+    {
+        applyCommit(commit, null, callback, (apply, ignore) -> {
             try (PaxosState state = tryGetUnsafe(apply.update.partitionKey(), apply.update.metadata()))
             {
                 if (state != null)
@@ -680,7 +700,7 @@ public static void commitDirect(Commit commit)
         });
     }
 
-    private static void applyCommit(Commit commit, PaxosState state, BiConsumer<Commit, PaxosState> postCommit)
+    private static void applyCommit(Commit commit, PaxosState state, Consumer<Commit> callback, BiConsumer<Commit, PaxosState> postCommit)
     {
         if (paxosStatePurging() == legacy && !(commit instanceof CommittedWithTTL))
             commit = CommittedWithTTL.withDefaultTTL(commit);
@@ -695,7 +715,8 @@ private static void applyCommit(Commit commit, PaxosState state, BiConsumer<Comm
             {
                 Tracing.trace("Committing proposal {}", commit);
                 Mutation mutation = commit.makeMutation();
-                Keyspace.open(mutation.getKeyspaceName()).apply(mutation, true);
+                Keyspace.open(mutation.getKeyspaceName()).apply(mutation, WriteOptions.FOR_PAXOS_COMMIT);
+                callback.accept(commit);
             }
             else
             {
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosUtils.java b/src/java/org/apache/cassandra/service/paxos/PaxosUtils.java
new file mode 100644
index 000000000000..83f904865f37
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosUtils.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.paxos;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.metrics.CASClientRequestMetrics;
+import org.apache.cassandra.utils.MBeanWrapper;
+
+public final class PaxosUtils implements PaxosUtilsMBean
+{
+    public static final PaxosUtils instance = new PaxosUtils();
+
+    private static final String MBEAN_NAME = "org.apache.cassandra.service:type=Paxos";
+    private static volatile Integer maxPaxosBackoffMillis = CassandraRelevantProperties.LWT_MAX_BACKOFF_MS.getInt();
+    private static volatile Integer minPaxosBackoffMillis = CassandraRelevantProperties.LWT_MIN_BACKOFF_MS.getInt();
+
+    private PaxosUtils()
+    {
+    }
+
+    public void registerMBean()
+    {
+        MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
+    }
+
+    /**
+     * Applies a random sleep time between minPaxosBackoffMillis (inclusive) and maxPaxosBackoffMillis (exclusive)
+     * and emits the contentionBackoffLatency metric.
+     */
+    public static void applyPaxosContentionBackoff(CASClientRequestMetrics casMetrics)
+    {
+        int sleepInMillis = ThreadLocalRandom.current().nextInt(minPaxosBackoffMillis, maxPaxosBackoffMillis);
+        Uninterruptibles.sleepUninterruptibly(sleepInMillis, TimeUnit.MILLISECONDS);
+        long sleepInNanos = TimeUnit.MILLISECONDS.toNanos(sleepInMillis);
+        casMetrics.contentionBackoffLatency.addNano(sleepInNanos);
+    }
+
+    @Override
+    public int getMaxPaxosBackoffMillis()
+    {
+        return maxPaxosBackoffMillis;
+    }
+
+    @Override
+    public void setMaxPaxosBackoffMillis(int maxPaxosBackoffMillis)
+    {
+        PaxosUtils.maxPaxosBackoffMillis = maxPaxosBackoffMillis;
+    }
+
+    @Override
+    public int getMinPaxosBackoffMillis()
+    {
+        return minPaxosBackoffMillis;
+    }
+
+    @Override
+    public void setMinPaxosBackoffMillis(int minPaxosBackoffMillis)
+    {
+        PaxosUtils.minPaxosBackoffMillis = minPaxosBackoffMillis;
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosUtilsMBean.java b/src/java/org/apache/cassandra/service/paxos/PaxosUtilsMBean.java
new file mode 100644
index 000000000000..6e0b5df41196
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosUtilsMBean.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.paxos;
+
+public interface PaxosUtilsMBean
+{
+    int getMaxPaxosBackoffMillis();
+
+    void setMaxPaxosBackoffMillis(int maxPaxosBackoffMillis);
+
+    int getMinPaxosBackoffMillis();
+
+    void setMinPaxosBackoffMillis(int minPaxosBackoffMillis);
+}
diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosKeyState.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosKeyState.java
index c55269afa112..45fe0b7a7ca8 100644
--- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosKeyState.java
+++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosKeyState.java
@@ -118,7 +118,7 @@ static PaxosKeyState merge(PaxosKeyState left, PaxosKeyState right)
             return cmp > 0 ? left : right;
     }
 
-    static class Reducer extends MergeIterator.Reducer<PaxosKeyState, PaxosKeyState>
+    static class Reducer extends org.apache.cassandra.utils.Reducer<PaxosKeyState, PaxosKeyState>
     {
         private PaxosKeyState mostRecent = null;
 
@@ -127,12 +127,12 @@ public void reduce(int idx, PaxosKeyState current)
             mostRecent = merge(mostRecent, current);
         }
 
-        protected PaxosKeyState getReduced()
+        public PaxosKeyState getReduced()
         {
             return mostRecent;
         }
 
-        protected void onKeyChange()
+        public void onKeyChange()
         {
             super.onKeyChange();
             mostRecent = null;
@@ -141,7 +141,7 @@ protected void onKeyChange()
 
     public static CloseableIterator<PaxosKeyState> mergeUncommitted(CloseableIterator<PaxosKeyState>... iterators)
     {
-        return MergeIterator.get(Lists.newArrayList(iterators), PaxosKeyState.KEY_COMPARATOR, new Reducer());
+        return MergeIterator.getCloseable(Lists.newArrayList(iterators), PaxosKeyState.KEY_COMPARATOR, new Reducer());
     }
 
     public static CloseableIterator<UncommittedPaxosKey> toUncommittedInfo(CloseableIterator<PaxosKeyState> iter)
diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java
index 744dd4d07dc3..1649641f746a 100644
--- a/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java
+++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java
@@ -225,7 +225,7 @@ void truncate()
         }
     }
 
-    private static class Reducer extends MergeIterator.Reducer<PaxosKeyState, PaxosKeyState>
+    private static class Reducer extends org.apache.cassandra.utils.Reducer<PaxosKeyState, PaxosKeyState>
     {
         PaxosKeyState merged = null;
 
@@ -234,12 +234,12 @@ public void reduce(int idx, PaxosKeyState current)
             merged = PaxosKeyState.merge(merged, current);
         }
 
-        protected PaxosKeyState getReduced()
+        public PaxosKeyState getReduced()
         {
             return merged;
         }
 
-        protected void onKeyChange()
+        public void onKeyChange()
         {
             merged = null;
         }
@@ -256,7 +256,7 @@ private static CloseableIterator<PaxosKeyState> merge(Collection<UncommittedData
                 if (iterator == null) continue;
                 iterators.add(iterator);
             }
-            return MergeIterator.get(iterators, PaxosKeyState.KEY_COMPARATOR, new Reducer());
+            return MergeIterator.getCloseable(iterators, PaxosKeyState.KEY_COMPARATOR, new Reducer());
         }
         catch (Throwable t)
         {
diff --git a/src/java/org/apache/cassandra/service/paxos/v1/AbstractPaxosCallback.java b/src/java/org/apache/cassandra/service/paxos/v1/AbstractPaxosCallback.java
index 79ef0a74b29b..ea31a7cc8c9e 100644
--- a/src/java/org/apache/cassandra/service/paxos/v1/AbstractPaxosCallback.java
+++ b/src/java/org/apache/cassandra/service/paxos/v1/AbstractPaxosCallback.java
@@ -27,6 +27,9 @@
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 import org.apache.cassandra.utils.Clock;
@@ -41,15 +44,26 @@ public abstract class AbstractPaxosCallback<T> implements RequestCallback<T>
 
     protected final CountDownLatch latch;
     protected final int targets;
+    private final TableMetadata metadata;
     private final ConsistencyLevel consistency;
     private final Dispatcher.RequestTime requestTime;
 
-    public AbstractPaxosCallback(int targets, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime)
+    private final RequestSensors requestSensors;
+
+    public AbstractPaxosCallback(TableMetadata metadata, int targets, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime)
     {
+        this.metadata = metadata;
         this.targets = targets;
         this.consistency = consistency;
         latch = newCountDownLatch(targets);
         this.requestTime = requestTime;
+        this.requestSensors = RequestTracker.instance.get();
+    }
+
+    @Override
+    public RequestSensors getRequestSensors()
+    {
+        return requestSensors;
     }
 
     public int getResponseCount()
@@ -57,6 +71,11 @@ public int getResponseCount()
         return targets - latch.count();
     }
 
+    public TableMetadata getMetadata()
+    {
+        return metadata;
+    }
+
     public void await() throws WriteTimeoutException
     {
         try
diff --git a/src/java/org/apache/cassandra/service/paxos/v1/PrepareCallback.java b/src/java/org/apache/cassandra/service/paxos/v1/PrepareCallback.java
index 4aedb6d63dee..de8990e249f2 100644
--- a/src/java/org/apache/cassandra/service/paxos/v1/PrepareCallback.java
+++ b/src/java/org/apache/cassandra/service/paxos/v1/PrepareCallback.java
@@ -48,7 +48,7 @@ public class PrepareCallback extends AbstractPaxosCallback<PrepareResponse>
 
     public PrepareCallback(DecoratedKey key, TableMetadata metadata, int targets, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime)
     {
-        super(targets, consistency, requestTime);
+        super(metadata, targets, consistency, requestTime);
         // need to inject the right key in the empty commit so comparing with empty commits in the response works as expected
         mostRecentCommit = Commit.emptyCommit(key, metadata);
         mostRecentInProgressCommit = Commit.emptyCommit(key, metadata);
diff --git a/src/java/org/apache/cassandra/service/paxos/v1/PrepareVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/v1/PrepareVerbHandler.java
index b31900ea40f5..df69abe89214 100644
--- a/src/java/org/apache/cassandra/service/paxos/v1/PrepareVerbHandler.java
+++ b/src/java/org/apache/cassandra/service/paxos/v1/PrepareVerbHandler.java
@@ -19,9 +19,15 @@
 package org.apache.cassandra.service.paxos.v1;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.sensors.RequestTracker;
 import org.apache.cassandra.service.paxos.Commit;
 import org.apache.cassandra.service.paxos.PaxosState;
 import org.apache.cassandra.service.paxos.PrepareResponse;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+import org.apache.cassandra.sensors.SensorsFactory;
+import org.apache.cassandra.sensors.Type;
 
 public class PrepareVerbHandler extends AbstractPaxosVerbHandler
 {
@@ -35,7 +41,24 @@ public static PrepareResponse doPrepare(Commit toPrepare)
     @Override
     public void processMessage(Message<Commit> message)
     {
-        Message<PrepareResponse> reply = message.responseWith(doPrepare(message.payload));
-        MessagingService.instance().send(reply, message.from());
+        // Initialize the sensor and set ExecutorLocals
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors(message.payload.update.metadata().keyspace);
+        Context context = Context.from(message.payload.update.metadata());
+
+        // Prepare phase incorporates a read to check the cas condition, so a read sensor is registered in addition to the write sensor
+        sensors.registerSensor(context, Type.READ_BYTES);
+        sensors.registerSensor(context, Type.WRITE_BYTES);
+        sensors.registerSensor(context, Type.INTERNODE_BYTES);
+        sensors.incrementSensor(context, Type.INTERNODE_BYTES, message.payloadSize(MessagingService.current_version));
+        RequestTracker.instance.set(sensors);
+
+        Message.Builder<PrepareResponse> reply = message.responseWithBuilder(doPrepare(message.payload));
+
+        // calculate outbound internode bytes before adding the sensor to the response
+        int size = reply.currentPayloadSize(MessagingService.current_version);
+        sensors.incrementSensor(context, Type.INTERNODE_BYTES, size);
+        sensors.syncAllSensors();
+        SensorsCustomParams.addSensorsToInternodeResponse(sensors, reply);
+        MessagingService.instance().send(reply.build(), message.from());
     }
 }
diff --git a/src/java/org/apache/cassandra/service/paxos/v1/ProposeCallback.java b/src/java/org/apache/cassandra/service/paxos/v1/ProposeCallback.java
index 2d83644e07b0..e1426f24806e 100644
--- a/src/java/org/apache/cassandra/service/paxos/v1/ProposeCallback.java
+++ b/src/java/org/apache/cassandra/service/paxos/v1/ProposeCallback.java
@@ -28,6 +28,7 @@
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.Nemesis;
+import org.apache.cassandra.schema.TableMetadata;
 
 /**
  * ProposeCallback has two modes of operation, controlled by the failFast parameter.
@@ -50,13 +51,14 @@ public class ProposeCallback extends AbstractPaxosCallback<Boolean>
     private final int requiredAccepts;
     private final boolean failFast;
 
-    public ProposeCallback(int totalTargets, int requiredTargets, boolean failFast, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime)
+    public ProposeCallback(TableMetadata metadata, int totalTargets, int requiredTargets, boolean failFast, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime)
     {
-        super(totalTargets, consistency, requestTime);
+        super(metadata, totalTargets, consistency, requestTime);
         this.requiredAccepts = requiredTargets;
         this.failFast = failFast;
     }
 
+    @Override
     public void onResponse(Message<Boolean> msg)
     {
         logger.trace("Propose response {} from {}", msg.payload, msg.from());
diff --git a/src/java/org/apache/cassandra/service/paxos/v1/ProposeVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/v1/ProposeVerbHandler.java
index d3069a290c37..ca3b98340278 100644
--- a/src/java/org/apache/cassandra/service/paxos/v1/ProposeVerbHandler.java
+++ b/src/java/org/apache/cassandra/service/paxos/v1/ProposeVerbHandler.java
@@ -20,8 +20,14 @@
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.sensors.RequestTracker;
 import org.apache.cassandra.service.paxos.Commit;
 import org.apache.cassandra.service.paxos.PaxosState;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+import org.apache.cassandra.sensors.SensorsFactory;
+import org.apache.cassandra.sensors.Type;
 
 public class ProposeVerbHandler extends AbstractPaxosVerbHandler implements IVerbHandler<Commit>
 {
@@ -35,8 +41,24 @@ public static Boolean doPropose(Commit proposal)
     @Override
     void processMessage(Message<Commit> message)
     {
-        Boolean response = doPropose(message.payload);
-        Message<Boolean> reply = message.responseWith(response);
-        MessagingService.instance().send(reply, message.from());
+        // Initialize the sensor and set ExecutorLocals
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors(message.payload.update.metadata().keyspace);
+        Context context = Context.from(message.payload.update.metadata());
+
+        // Propose phase consults the Paxos table for more recent promises, so a read sensor is registered in addition to the write sensor
+        sensors.registerSensor(context, Type.READ_BYTES);
+        sensors.registerSensor(context, Type.WRITE_BYTES);
+        sensors.registerSensor(context, Type.INTERNODE_BYTES);
+        sensors.incrementSensor(context, Type.INTERNODE_BYTES, message.payloadSize(MessagingService.current_version));
+        RequestTracker.instance.set(sensors);
+
+        Message.Builder<Boolean> reply = message.responseWithBuilder(doPropose(message.payload));
+
+        // calculate outbound internode bytes before adding the sensor to the response
+        int size = reply.currentPayloadSize(MessagingService.current_version);
+        sensors.incrementSensor(context, Type.INTERNODE_BYTES, size);
+        sensors.syncAllSensors();
+        SensorsCustomParams.addSensorsToInternodeResponse(sensors, reply);
+        MessagingService.instance().send(reply.build(), message.from());
     }
 }
diff --git a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
index 967673482437..f4b92ae199f5 100644
--- a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
+++ b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
@@ -16,7 +16,6 @@
  * limitations under the License.
  */
 package org.apache.cassandra.service.reads;
-
 import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -39,8 +38,10 @@
 import org.apache.cassandra.locator.ReplicaCollection;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlans;
+import org.apache.cassandra.metrics.ReadCoordinationMetrics;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.service.StorageProxy.LocalReadRunnable;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
 import org.apache.cassandra.tracing.TraceState;
@@ -76,20 +77,30 @@ public abstract class AbstractReadExecutor
 
     private   final int initialDataRequestCount;
     protected volatile PartitionIterator result = null;
-
-    AbstractReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, int initialDataRequestCount, Dispatcher.RequestTime requestTime)
+    protected final QueryInfoTracker.ReadTracker readTracker;
+    static
+    {
+        MessagingService.instance().latencySubscribers.subscribe(ReadCoordinationMetrics::updateReplicaLatency);
+    }
+    
+    AbstractReadExecutor(ColumnFamilyStore cfs,
+                         ReadCommand command,
+                         ReplicaPlan.ForTokenRead replicaPlan,
+                         int initialDataRequestCount,
+                         Dispatcher.RequestTime requestTime,
+                         QueryInfoTracker.ReadTracker readTracker)
     {
         this.command = command;
         this.replicaPlan = ReplicaPlan.shared(replicaPlan);
         this.initialDataRequestCount = initialDataRequestCount;
         // the ReadRepair and DigestResolver both need to see our updated
         this.readRepair = ReadRepair.create(command, this.replicaPlan, requestTime);
-        this.digestResolver = new DigestResolver<>(command, this.replicaPlan, requestTime);
+        this.digestResolver = new DigestResolver<>(command, this.replicaPlan, requestTime, readTracker);
         this.handler = new ReadCallback<>(digestResolver, command, this.replicaPlan, requestTime);
         this.cfs = cfs;
         this.traceState = Tracing.instance.get();
         this.requestTime = requestTime;
-
+        this.readTracker = readTracker;
 
         // Set the digest version (if we request some digests). This is the smallest version amongst all our target replicas since new nodes
         // knows how to produce older digest but the reverse is not true.
@@ -99,6 +110,8 @@ public abstract class AbstractReadExecutor
         for (Replica replica : replicaPlan.contacts())
             digestVersion = Math.min(digestVersion, MessagingService.instance().versions.get(replica.endpoint()));
         command.setDigestVersion(digestVersion);
+
+        readTracker.onReplicaPlan(replicaPlan);
     }
 
     public DecoratedKey getKey()
@@ -167,8 +180,9 @@ private void makeRequests(ReadCommand readCommand, Iterable<Replica> replicas)
     }
 
     /**
-     * Perform additional requests if it looks like the original will time out.  May block while it waits
-     * to see if the original requests are answered first.
+     * Perform additional requests if it looks like the original takes "too much time", as defined
+     * by the subclass.
+     * May block while it waits to see if the original requests are answered first.
      */
     public abstract void maybeTryAdditionalReplicas();
 
@@ -187,7 +201,10 @@ public void executeAsync()
     /**
      * @return an executor appropriate for the configured speculative read policy
      */
-    public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) throws UnavailableException
+    public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command,
+                                                       ConsistencyLevel consistencyLevel,
+                                                       Dispatcher.RequestTime requestTime,
+                                                       QueryInfoTracker.ReadTracker readTracker) throws UnavailableException
     {
         Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
@@ -199,23 +216,32 @@ public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand co
                                                                     consistencyLevel,
                                                                     retry);
 
+        if (replicaPlan.readCandidates().stream().noneMatch(replica -> replica.endpoint().equals(FBUtilities.getBroadcastAddressAndPort())))
+        {
+            ReadCoordinationMetrics.nonreplicaRequests.inc();
+        }
+        else if (replicaPlan.contacts().stream().noneMatch(replica -> replica.endpoint().equals(FBUtilities.getBroadcastAddressAndPort())))
+        {
+            ReadCoordinationMetrics.preferredOtherReplicas.inc();
+        }
+
         // Speculative retry is disabled *OR*
         // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses
         if (retry.equals(NeverSpeculativeRetryPolicy.INSTANCE) || consistencyLevel == ConsistencyLevel.EACH_QUORUM)
-            return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, false);
+            return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, false, readTracker);
 
         // There are simply no extra replicas to speculate.
         // Handle this separately so it can record failed attempts to speculate due to lack of replicas
         if (replicaPlan.contacts().size() == replicaPlan.readCandidates().size())
         {
             boolean recordFailedSpeculation = consistencyLevel != ConsistencyLevel.ALL;
-            return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, recordFailedSpeculation);
+            return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, recordFailedSpeculation, readTracker);
         }
 
         if (retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE))
-            return new AlwaysSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime);
+            return new AlwaysSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, readTracker);
         else // PERCENTILE or CUSTOM.
-            return new SpeculatingReadExecutor(cfs, command, replicaPlan, requestTime);
+            return new SpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, readTracker);
     }
 
     public boolean hasLocalRead()
@@ -268,13 +294,9 @@ public static class NeverSpeculatingReadExecutor extends AbstractReadExecutor
          */
         private final boolean logFailedSpeculation;
 
-        public NeverSpeculatingReadExecutor(ColumnFamilyStore cfs,
-                                            ReadCommand command,
-                                            ReplicaPlan.ForTokenRead replicaPlan,
-                                            Dispatcher.RequestTime requestTime,
-                                            boolean logFailedSpeculation)
+        public NeverSpeculatingReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, Dispatcher.RequestTime requestTime, boolean logFailedSpeculation, QueryInfoTracker.ReadTracker readTracker)
         {
-            super(cfs, command, replicaPlan, 1, requestTime);
+            super(cfs, command, replicaPlan, 1, requestTime, readTracker);
             this.logFailedSpeculation = logFailedSpeculation;
         }
 
@@ -294,12 +316,13 @@ static class SpeculatingReadExecutor extends AbstractReadExecutor
         public SpeculatingReadExecutor(ColumnFamilyStore cfs,
                                        ReadCommand command,
                                        ReplicaPlan.ForTokenRead replicaPlan,
-                                       Dispatcher.RequestTime requestTime)
+                                       Dispatcher.RequestTime requestTime,
+                                       QueryInfoTracker.ReadTracker readTracker)
         {
             // We're hitting additional targets for read repair (??).  Since our "extra" replica is the least-
             // preferred by the snitch, we do an extra data read to start with against a replica more
             // likely to respond; better to let RR fail than the entire query.
-            super(cfs, command, replicaPlan, replicaPlan.readQuorum() < replicaPlan.contacts().size() ? 2 : 1, requestTime);
+            super(cfs, command, replicaPlan, replicaPlan.readQuorum() < replicaPlan.contacts().size() ? 2 : 1, requestTime, readTracker);
         }
 
         public void maybeTryAdditionalReplicas()
@@ -365,11 +388,12 @@ private static class AlwaysSpeculatingReadExecutor extends AbstractReadExecutor
         public AlwaysSpeculatingReadExecutor(ColumnFamilyStore cfs,
                                              ReadCommand command,
                                              ReplicaPlan.ForTokenRead replicaPlan,
-                                             Dispatcher.RequestTime requestTime)
+                                             Dispatcher.RequestTime requestTime,
+                                             QueryInfoTracker.ReadTracker readTracker)
         {
             // presumably, we speculate an extra data request here in case it is our data request that fails to respond,
             // and there are no more nodes to consult
-            super(cfs, command, replicaPlan, replicaPlan.contacts().size() > 1 ? 2 : 1, requestTime);
+            super(cfs, command, replicaPlan, replicaPlan.contacts().size() > 1 ? 2 : 1, requestTime, readTracker);
         }
 
         public void maybeTryAdditionalReplicas()
diff --git a/src/java/org/apache/cassandra/service/reads/DataResolver.java b/src/java/org/apache/cassandra/service/reads/DataResolver.java
index 64e4c72b01dd..71bac811a7cf 100644
--- a/src/java/org/apache/cassandra/service/reads/DataResolver.java
+++ b/src/java/org/apache/cassandra/service/reads/DataResolver.java
@@ -50,8 +50,10 @@
 import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
 import org.apache.cassandra.service.reads.repair.RepairedDataTracker;
 import org.apache.cassandra.service.reads.repair.RepairedDataVerifier;
@@ -64,18 +66,24 @@ public class DataResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
     private final boolean enforceStrictLiveness;
     private final ReadRepair<E, P> readRepair;
     private final boolean trackRepairedStatus;
+    protected final QueryInfoTracker.ReadTracker readTracker;
 
-    public DataResolver(ReadCommand command, Supplier<? extends P> replicaPlan, ReadRepair<E, P> readRepair, Dispatcher.RequestTime requestTime)
+    public DataResolver(ReadCommand command,
+                        Supplier<? extends P> replicaPlan,
+                        ReadRepair<E, P> readRepair,
+                        Dispatcher.RequestTime requestTime,
+                        QueryInfoTracker.ReadTracker readTracker)
     {
-        this(command, replicaPlan, readRepair, requestTime, false);
+        this(command, replicaPlan, readRepair, requestTime, false, readTracker);
     }
 
-    public DataResolver(ReadCommand command, Supplier<? extends P> replicaPlan, ReadRepair<E, P> readRepair, Dispatcher.RequestTime requestTime, boolean trackRepairedStatus)
+    public DataResolver(ReadCommand command, Supplier<? extends P> replicaPlan, ReadRepair<E, P> readRepair, Dispatcher.RequestTime requestTime, boolean trackRepairedStatus, QueryInfoTracker.ReadTracker readTracker)
     {
         super(command, replicaPlan, requestTime);
         this.enforceStrictLiveness = command.metadata().enforceStrictLiveness();
         this.readRepair = readRepair;
         this.trackRepairedStatus = trackRepairedStatus;
+        this.readTracker = readTracker;
     }
 
     public PartitionIterator getData()
@@ -119,17 +127,19 @@ public PartitionIterator resolve(@Nullable Runnable runOnShortRead)
             });
         }
 
-        if (usesReplicaFilteringProtection())
-            return resolveWithReplicaFilteringProtection(replicas, repairedDataTracker);
+        if (!needsReplicaFilteringProtection())
+        {
+            ResolveContext context = new ResolveContext(replicas, true);
+            return resolveWithReadRepair(context,
+                                         i -> shortReadProtectedResponse(i, context, runOnShortRead),
+                                         UnaryOperator.identity(),
+                                         repairedDataTracker);
+        }
 
-        ResolveContext context = new ResolveContext(replicas, true);
-        return resolveWithReadRepair(context,
-                                     i -> shortReadProtectedResponse(i, context, runOnShortRead),
-                                     UnaryOperator.identity(),
-                                     repairedDataTracker);
+        return resolveWithReplicaFilteringProtection(replicas, repairedDataTracker);
     }
 
-    private boolean usesReplicaFilteringProtection()
+    private boolean needsReplicaFilteringProtection()
     {
         if (command.rowFilter().isEmpty())
             return false;
@@ -138,15 +148,19 @@ private boolean usesReplicaFilteringProtection()
             return false;
 
         Index.QueryPlan queryPlan = command.indexQueryPlan();
-        if (queryPlan == null )
+        IndexMetadata indexMetadata = queryPlan == null ? null : queryPlan.getFirst().getIndexMetadata();
+
+        if (indexMetadata == null || !indexMetadata.isCustom())
+        {
             return true;
+        }
 
         return queryPlan.supportsReplicaFilteringProtection(command.rowFilter());
     }
 
-    private class ResolveContext
+    protected class ResolveContext
     {
-        private final E replicas;
+        public final E replicas;
         private final DataLimits.Counter mergedResultCounter;
 
         /**
@@ -161,31 +175,32 @@ private ResolveContext(E replicas, boolean enforceLimits)
                                                                    command.selectsFullPartition(),
                                                                    enforceStrictLiveness);
 
-            // In case of top-k query, do not trim reconciled rows here because QueryPlan#postProcessor()
-            // needs to compare all rows. Also avoid enforcing the limit if explicitly requested.
+            // In case of top-k query, do not trim reconciled rows here because QueryPlan#postProcessor() needs to compare all rows
             if (command.isTopK() || !enforceLimits)
                 this.mergedResultCounter.onlyCount();
         }
 
         private boolean needsReadRepair()
         {
-            // Each replica may return different estimated top-K rows, it doesn't mean data is not replicated.
-            // Even though top-K queries are limited to CL ONE & LOCAL-ONE, they use the ScanAllRangesCommandIterator
-            // that combines the separate replica plans of each data range into a single replica plan. This is an
-            // optimisation but can result in the number of replicas being > 1.
+            // each replica may return different estimated top-K rows, it doesn't mean data is not replicated.
             if (command.isTopK())
                 return false;
 
             return replicas.size() > 1;
         }
 
-        private boolean needShortReadProtection()
+        public DataLimits.Counter mergedResultCounter()
+        {
+            return mergedResultCounter;
+        }
+
+        public boolean needShortReadProtection()
         {
             // SRP doesn't make sense for top-k which needs to re-query replica with larger limit instead of fetching more partitions
             if (command.isTopK())
                 return false;
 
-            // If we have only one result, there is no read repair to do, and we can't get short reads
+            // If we have only one result, there is no read repair to do and we can't get short reads
             // Also, so-called "short reads" stems from nodes returning only a subset of the results they have for a
             // partition due to the limit, but that subset not being enough post-reconciliation. So if we don't have limit,
             // don't bother protecting against short reads.
@@ -199,19 +214,24 @@ private interface ResponseProvider
         UnfilteredPartitionIterator getResponse(int i);
     }
 
-    private UnfilteredPartitionIterator shortReadProtectedResponse(int i, ResolveContext context, @Nullable Runnable onShortRead)
+    protected UnfilteredPartitionIterator shortReadProtectedResponse(int i, ResolveContext context, @Nullable Runnable onShortRead)
     {
         UnfilteredPartitionIterator originalResponse = responses.get(i).payload.makeIterator(command);
 
-        return context.needShortReadProtection()
-               ? ShortReadProtection.extend(context.replicas.get(i),
-                                            () -> { responses.clearUnsafe(i); if (onShortRead != null) onShortRead.run(); },
-                                            originalResponse,
-                                            command,
-                                            context.mergedResultCounter,
-                                            requestTime,
-                                            enforceStrictLiveness)
-               : originalResponse;
+        if (context.needShortReadProtection())
+        {
+            DataLimits.Counter singleResultCounter = command.createLimitedCounter(false);
+            return ShortReadProtection.extend(originalResponse,
+                                              command,
+                                              new ShortReadPartitionsProtection(command,
+                                                                                context.replicas.get(i),
+                                                                                () -> { responses.clearUnsafe(i); if (onShortRead != null) onShortRead.run(); },
+                                                                                singleResultCounter,
+                                                                                context.mergedResultCounter(),
+                                                                                requestTime),
+                                              singleResultCounter);
+        }
+        return originalResponse;
     }
 
     private PartitionIterator resolveWithReadRepair(ResolveContext context,
@@ -226,7 +246,7 @@ private PartitionIterator resolveWithReadRepair(ResolveContext context,
             listener = wrapMergeListener(readRepair.getMergeListener(sources), sources, repairedDataTracker);
         }
 
-        return resolveInternal(context, listener, responseProvider, preCountFilter);
+        return resolveInternal(context, listener, responseProvider, preCountFilter, readTracker);
     }
 
     private PartitionIterator resolveWithReplicaFilteringProtection(E replicas, RepairedDataTracker repairedDataTracker)
@@ -261,7 +281,8 @@ private PartitionIterator resolveWithReplicaFilteringProtection(E replicas, Repa
         PartitionIterator firstPhasePartitions = resolveInternal(firstPhaseContext,
                                                                  rfp.mergeController(),
                                                                  i -> shortReadProtectedResponse(i, firstPhaseContext, null),
-                                                                 null);
+                                                                 null,
+                                                                 QueryInfoTracker.ReadTracker.NOOP);
 
         ResolveContext secondPhaseContext = new ResolveContext(replicas, true);
         PartitionIterator completedPartitions = resolveWithReadRepair(secondPhaseContext,
@@ -275,19 +296,18 @@ private PartitionIterator resolveWithReplicaFilteringProtection(E replicas, Repa
 
     private  UnaryOperator<PartitionIterator> preCountFilterForReplicaFilteringProtection()
     {
-        return results -> {
-            Index.Searcher searcher = command.indexSearcher();
-            // in case of "ALLOW FILTERING" without index
-            if (searcher == null)
-                return command.rowFilter().filter(results, command.metadata(), command.nowInSec());
-            return searcher.filterReplicaFilteringProtection(results);
-        };
+        return results -> command.rowFilter().filter(results, command.metadata(), command.nowInSec());
     }
 
+    /**
+     * Uses the provided {@link org.apache.cassandra.service.QueryInfoTracker.ReadTracker} as internal calls
+     * may be not tracked, e.g. the first phase of RFP.
+     */
     private PartitionIterator resolveInternal(ResolveContext context,
                                               UnfilteredPartitionIterators.MergeListener mergeListener,
                                               ResponseProvider responseProvider,
-                                              @Nullable UnaryOperator<PartitionIterator> preCountFilter)
+                                              @Nullable UnaryOperator<PartitionIterator> preCountFilter,
+                                              QueryInfoTracker.ReadTracker resolveReadTracker)
     {
         int count = context.replicas.size();
         List<UnfilteredPartitionIterator> results = new ArrayList<>(count);
@@ -309,6 +329,10 @@ private PartitionIterator resolveInternal(ResolveContext context,
          */
 
         UnfilteredPartitionIterator merged = UnfilteredPartitionIterators.merge(results, mergeListener);
+        if (!QueryInfoTracker.ReadTracker.NOOP.equals(resolveReadTracker) && !QueryInfoTracker.LWTWriteTracker.NOOP.equals(resolveReadTracker))
+        {
+            merged = Transformation.apply(merged, new ReadTrackingTransformation(resolveReadTracker));
+        }
         Filter filter = new Filter(command.nowInSec(), command.metadata().enforceStrictLiveness());
         FilteredPartitions filtered = FilteredPartitions.filter(merged, filter);
 
diff --git a/src/java/org/apache/cassandra/service/reads/DigestResolver.java b/src/java/org/apache/cassandra/service/reads/DigestResolver.java
index cc248422c06c..df58894b47a6 100644
--- a/src/java/org/apache/cassandra/service/reads/DigestResolver.java
+++ b/src/java/org/apache/cassandra/service/reads/DigestResolver.java
@@ -28,12 +28,15 @@
 import org.apache.cassandra.db.ReadResponse;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.locator.Endpoints;
-import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -44,12 +47,17 @@
 public class DigestResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>> extends ResponseResolver<E, P>
 {
     private volatile Message<ReadResponse> dataResponse;
+    private final QueryInfoTracker.ReadTracker readTracker;
 
-    public DigestResolver(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
+    public DigestResolver(ReadCommand command,
+                          ReplicaPlan.Shared<E, P> replicaPlan,
+                          Dispatcher.RequestTime requestTime,
+                          QueryInfoTracker.ReadTracker readTracker)
     {
         super(command, replicaPlan, requestTime);
         Preconditions.checkArgument(command instanceof SinglePartitionReadCommand,
                                     "DigestResolver can only be used with SinglePartitionReadCommand commands");
+        this.readTracker = readTracker;
     }
 
     @Override
@@ -80,14 +88,20 @@ public PartitionIterator getData()
 
         if (!hasTransientResponse(responses))
         {
-            return UnfilteredPartitionIterators.filter(dataResponse.payload.makeIterator(command), command.nowInSec());
+            UnfilteredPartitionIterator unfilteredPartitionIterator = dataResponse.payload.makeIterator(command);
+            if (!QueryInfoTracker.ReadTracker.NOOP.equals(readTracker) && !QueryInfoTracker.LWTWriteTracker.NOOP.equals(readTracker))
+            {
+                unfilteredPartitionIterator = Transformation.apply(unfilteredPartitionIterator,
+                                                                  new ReadTrackingTransformation(readTracker));
+            }
+            return UnfilteredPartitionIterators.filter(unfilteredPartitionIterator, command.nowInSec());
         }
         else
         {
             // This path can be triggered only if we've got responses from full replicas and they match, but
             // transient replica response still contains data, which needs to be reconciled.
             DataResolver<E, P> dataResolver
-                    = new DataResolver<>(command, replicaPlan, NoopReadRepair.instance, requestTime);
+                    = new DataResolver<>(command, replicaPlan, NoopReadRepair.instance, requestTime, readTracker);
 
             dataResolver.preprocess(dataResponse);
             // Reconcile with transient replicas
@@ -151,6 +165,11 @@ public DigestResolverDebugResult[] getDigestsByEndpoint()
         return ret;
     }
 
+    public QueryInfoTracker.ReadTracker getReadTracker()
+    {
+        return readTracker;
+    }
+
     public static class DigestResolverDebugResult
     {
         public InetAddressAndPort from;
diff --git a/src/java/org/apache/cassandra/service/reads/ReadCallback.java b/src/java/org/apache/cassandra/service/reads/ReadCallback.java
index 899c55a8194e..4f91c57a0a3e 100644
--- a/src/java/org/apache/cassandra/service/reads/ReadCallback.java
+++ b/src/java/org/apache/cassandra/service/reads/ReadCallback.java
@@ -45,6 +45,8 @@
 import org.apache.cassandra.service.reads.thresholds.CoordinatorWarnings;
 import org.apache.cassandra.service.reads.thresholds.WarningContext;
 import org.apache.cassandra.service.reads.thresholds.WarningsSnapshot;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.RequestTracker;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.concurrent.Condition;
@@ -74,6 +76,8 @@ public class ReadCallback<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
     private volatile WarningContext warningContext;
     private static final AtomicReferenceFieldUpdater<ReadCallback, WarningContext> warningsUpdater
         = AtomicReferenceFieldUpdater.newUpdater(ReadCallback.class, WarningContext.class, "warningContext");
+    private final boolean couldSpeculate;
+    private final RequestSensors requestSensors;
 
     public ReadCallback(ResponseResolver<E, P> resolver, ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
     {
@@ -85,9 +89,16 @@ public ReadCallback(ResponseResolver<E, P> resolver, ReadCommand command, Replic
         this.failureReasonByEndpoint = new ConcurrentHashMap<>();
         // we don't support read repair (or rapid read protection) for range scans yet (CASSANDRA-6897)
         assert !(command instanceof PartitionRangeReadCommand) || blockFor >= replicaPlan().contacts().size();
+        SpeculativeRetryPolicy retry = replicaPlan()
+                .keyspace()
+                .getColumnFamilyStore(command.metadata().id)
+                .metadata()
+                .params.speculativeRetry;
+        this.couldSpeculate = !NeverSpeculativeRetryPolicy.INSTANCE.equals(retry);
 
         if (logger.isTraceEnabled())
             logger.trace("Blockfor is {}; setting up requests to {}", blockFor, this.replicaPlan);
+        this.requestSensors = RequestTracker.instance.get();
     }
 
     protected P replicaPlan()
@@ -95,6 +106,17 @@ protected P replicaPlan()
         return replicaPlan.get();
     }
 
+    public ReadCommand command()
+    {
+        return command;
+    }
+
+    @Override
+    public RequestSensors getRequestSensors()
+    {
+        return requestSensors;
+    }
+
     public boolean await(long commandTimeout, TimeUnit unit)
     {
         return awaitUntil(requestTime.computeDeadline(unit.toNanos(commandTimeout)));
@@ -245,7 +267,12 @@ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReaso
                 
         failureReasonByEndpoint.put(from, failureReason);
 
-        if (blockFor + failuresUpdater.incrementAndGet(this) > replicaPlan().contacts().size())
+        int numContacts = replicaPlan().contacts().size();
+        int numCandidates = replicaPlan().readCandidates().size();
+        // If potentially there is a replica which could be requested as part of the speculative read path
+        // then increase the number of nodes we wait for in case of failures.
+        int failFastPoint = (numContacts < numCandidates && couldSpeculate) ? numContacts + 1 : numContacts;
+        if (blockFor + failuresUpdater.incrementAndGet(this) > failFastPoint)
             condition.signalAll();
     }
 
diff --git a/src/java/org/apache/cassandra/service/reads/ReadTrackingTransformation.java b/src/java/org/apache/cassandra/service/reads/ReadTrackingTransformation.java
new file mode 100644
index 000000000000..bae14b91ac8e
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/ReadTrackingTransformation.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.service.QueryInfoTracker;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+/**
+ * {@code UnfilteredRowIterator} transformation that callbacks {@link QueryInfoTracker.ReadTracker} on
+ * each row and partition. The transformation may be extended with other callback methods (like
+ * {@code applyToStatic} or {@code applyToDeletion} if necessary.
+ *
+ * Do not move closing the tracker here (to @{code onClose} method). One read may include more than
+ * one row iterator, closing the tracker here may result in multiple close callbacks.
+ */
+class ReadTrackingTransformation extends Transformation<UnfilteredRowIterator>
+{
+    private final QueryInfoTracker.ReadTracker readTracker;
+    private static final Logger logger = LoggerFactory.getLogger(ReadTrackingTransformation.class);
+
+    public ReadTrackingTransformation(QueryInfoTracker.ReadTracker readTracker)
+    {
+        this.readTracker = readTracker;
+    }
+
+    @Override
+    protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+    {
+        return Transformation.apply(partition, this);
+    }
+
+    @Override
+    protected Row applyToRow(Row row)
+    {
+        try
+        {
+            readTracker.onRow(row);
+        }
+        catch (Exception exc)
+        {
+            NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 60, TimeUnit.SECONDS,
+                             "Tracking callback for read rows failed", exc);
+        }
+        return super.applyToRow(row);
+    }
+
+    @Override
+    protected Row applyToStatic(Row row)
+    {
+        try
+        {
+            if (!row.isEmpty())
+            {
+                readTracker.onRow(row);
+            }
+        }
+        catch (Exception exc)
+        {
+            NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 60, TimeUnit.SECONDS,
+                             "Tracking callback for read rows failed", exc);
+        }
+        return super.applyToRow(row);
+    }
+
+    @Override
+    protected DecoratedKey applyToPartitionKey(DecoratedKey key)
+    {
+        try
+        {
+            readTracker.onPartition(key);
+        }
+        catch (Exception exc)
+        {
+            NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 60, TimeUnit.SECONDS,
+                             "Tracking callback for read partitions failed", exc);
+        }
+        return super.applyToPartitionKey(key);
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
index 9ec02a5b2064..129f9bcb75b4 100644
--- a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
+++ b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
@@ -69,6 +69,7 @@
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
 import org.apache.cassandra.tracing.Tracing;
@@ -149,7 +150,11 @@ private UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, Replica
     {
         @SuppressWarnings("unchecked")
         DataResolver<EndpointsForToken, ReplicaPlan.ForTokenRead> resolver =
-            new DataResolver<>(cmd, replicaPlan, (NoopReadRepair<EndpointsForToken, ReplicaPlan.ForTokenRead>) NoopReadRepair.instance, requestTime);
+            new DataResolver<>(cmd,
+                               replicaPlan,
+                               (NoopReadRepair<EndpointsForToken, ReplicaPlan.ForTokenRead>) NoopReadRepair.instance,
+                               requestTime,
+                               QueryInfoTracker.ReadTracker.NOOP);
 
         ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime);
 
diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java
index e9870f1f1d7b..3db8f7456c5f 100644
--- a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java
+++ b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java
@@ -18,9 +18,8 @@
 
 package org.apache.cassandra.service.reads;
 
-import org.apache.cassandra.locator.Endpoints;
-import org.apache.cassandra.locator.ReplicaPlan;
-import org.apache.cassandra.locator.ReplicaPlans;
+import java.util.concurrent.TimeUnit;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,27 +40,35 @@
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.ExcludingBounds;
 import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
 import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.service.reads.repair.NoopReadRepair;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.reads.repair.NoopReadRepair;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.NoSpamLogger;
 
 public class ShortReadPartitionsProtection extends Transformation<UnfilteredRowIterator> implements MorePartitions<UnfilteredPartitionIterator>
 {
     private static final Logger logger = LoggerFactory.getLogger(ShortReadPartitionsProtection.class);
+    private static final NoSpamLogger oneMinuteLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+
     private final ReadCommand command;
     private final Replica source;
 
     private final Runnable preFetchCallback; // called immediately before fetching more contents
 
-    private final DataLimits.Counter singleResultCounter; // unmerged per-source counter
+    protected final DataLimits.Counter singleResultCounter; // unmerged per-source counter
     private final DataLimits.Counter mergedResultCounter; // merged end-result counter
 
     private DecoratedKey lastPartitionKey; // key of the last observed partition
 
     private boolean partitionsFetched; // whether we've seen any new partitions since iteration start or last moreContents() call
+    protected boolean rangeFetched = false; // fetched by original read request or SRP request
 
     private final Dispatcher.RequestTime requestTime;
 
@@ -84,6 +91,7 @@ public ShortReadPartitionsProtection(ReadCommand command,
     public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
     {
         partitionsFetched = true;
+        rangeFetched = true;
 
         lastPartitionKey = partition.partitionKey();
 
@@ -128,16 +136,17 @@ public UnfilteredPartitionIterator moreContents()
          * Can only take the short cut if there is no per partition limit set. Otherwise it's possible to hit false
          * positives due to some rows being uncounted for in certain scenarios (see CASSANDRA-13911).
          */
-        if (command.limits().isExhausted(singleResultCounter) && command.limits().perPartitionCount() == DataLimits.NO_LIMIT)
+        if (command.limits().isCounterBelowLimits(singleResultCounter) && command.limits().perPartitionCount() == DataLimits.NO_LIMIT)
             return null;
 
         /*
          * Either we had an empty iterator as the initial response, or our moreContents() call got us an empty iterator.
          * There is no point to ask the replica for more rows - it has no more in the requested range.
          */
-        if (!partitionsFetched)
+        if (rangeExhausted())
             return null;
         partitionsFetched = false;
+        rangeFetched = true;
 
         /*
          * We are going to fetch one partition at a time for thrift and potentially more for CQL.
@@ -151,7 +160,9 @@ public UnfilteredPartitionIterator moreContents()
 
         ColumnFamilyStore.metricsFor(command.metadata().id).shortReadProtectionRequests.mark();
         Tracing.trace("Requesting {} extra rows from {} for short read protection", toQuery, source);
-        logger.info("Requesting {} extra rows from {} for short read protection", toQuery, source);
+        // This is a NoSpamLogger because, in the event of unrepaired data or missing data on nodes in
+        // a cluster, we can end up spamming the logs with this message
+        oneMinuteLogger.info("Requesting {} extra rows from {} for short read protection", toQuery, source);
 
         // If we've arrived here, all responses have been consumed, and we're about to request more.
         preFetchCallback.run();
@@ -159,6 +170,11 @@ public UnfilteredPartitionIterator moreContents()
         return makeAndExecuteFetchAdditionalPartitionReadCommand(toQuery);
     }
 
+    public boolean rangeExhausted()
+    {
+        return !partitionsFetched;
+    }
+
     private UnfilteredPartitionIterator makeAndExecuteFetchAdditionalPartitionReadCommand(int toQuery)
     {
         PartitionRangeReadCommand cmd = (PartitionRangeReadCommand) command;
@@ -178,7 +194,11 @@ private UnfilteredPartitionIterator makeAndExecuteFetchAdditionalPartitionReadCo
     private <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
     UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, ReplicaPlan.Shared<E, P> replicaPlan)
     {
-        DataResolver<E, P> resolver = new DataResolver<>(cmd, replicaPlan, (NoopReadRepair<E, P>)NoopReadRepair.instance, requestTime);
+        DataResolver<E, P> resolver = new DataResolver<>(cmd,
+                                                         replicaPlan,
+                                                         (NoopReadRepair<E, P>)NoopReadRepair.instance,
+                                                         requestTime,
+                                                         QueryInfoTracker.ReadTracker.NOOP);
         ReadCallback<E, P> handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime);
 
         if (source.isSelf())
diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java
index 1eca190a7343..1d50b30c0224 100644
--- a/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java
+++ b/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java
@@ -18,14 +18,11 @@
 
 package org.apache.cassandra.service.reads;
 
-
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.transform.MorePartitions;
 import org.apache.cassandra.db.transform.Transformation;
-import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.transport.Dispatcher;
 
 /**
  * We have a potential short read if the result from a given node contains the requested number of rows
@@ -39,26 +36,11 @@
  */
 public class ShortReadProtection
 {
-    public static UnfilteredPartitionIterator extend(Replica source,
-                                                     Runnable preFetchCallback,
-                                                     UnfilteredPartitionIterator partitions,
+    public static UnfilteredPartitionIterator extend(UnfilteredPartitionIterator partitions,
                                                      ReadCommand command,
-                                                     DataLimits.Counter mergedResultCounter,
-                                                     Dispatcher.RequestTime requestTime,
-                                                     boolean enforceStrictLiveness)
+                                                     ShortReadPartitionsProtection protection,
+                                                     DataLimits.Counter singleResultCounter)
     {
-        DataLimits.Counter singleResultCounter = command.limits().newCounter(command.nowInSec(),
-                                                                             false,
-                                                                             command.selectsFullPartition(),
-                                                                             enforceStrictLiveness).onlyCount();
-
-        ShortReadPartitionsProtection protection = new ShortReadPartitionsProtection(command,
-                                                                                     source,
-                                                                                     preFetchCallback,
-                                                                                     singleResultCounter,
-                                                                                     mergedResultCounter,
-                                                                                     requestTime);
-
         /*
          * The order of extention and transformations is important here. Extending with more partitions has to happen
          * first due to the way BaseIterator.hasMoreContents() works: only transformations applied after extension will
diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java
index 4ed9e329563b..1c08e0ff293b 100644
--- a/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java
+++ b/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java
@@ -92,7 +92,7 @@ public UnfilteredRowIterator moreContents()
          * Can only take the short cut if there is no per partition limit set. Otherwise it's possible to hit false
          * positives due to some rows being uncounted for in certain scenarios (see CASSANDRA-13911).
          */
-        if (command.limits().isExhausted(singleResultCounter) && command.limits().perPartitionCount() == DataLimits.NO_LIMIT)
+        if (command.limits().isCounterBelowLimits(singleResultCounter) && command.limits().perPartitionCount() == DataLimits.NO_LIMIT)
             return null;
 
         /*
diff --git a/src/java/org/apache/cassandra/service/reads/range/EndpointGroupingCoordinator.java b/src/java/org/apache/cassandra/service/reads/range/EndpointGroupingCoordinator.java
new file mode 100644
index 000000000000..cf276c0e250e
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/range/EndpointGroupingCoordinator.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.range;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.MultiRangeReadCommand;
+import org.apache.cassandra.db.MultiRangeReadResponse;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.service.QueryInfoTracker;
+import org.apache.cassandra.service.reads.DataResolver;
+import org.apache.cassandra.service.reads.ReadCallback;
+import org.apache.cassandra.service.reads.ShortReadPartitionsProtection;
+import org.apache.cassandra.service.reads.ShortReadProtection;
+import org.apache.cassandra.service.reads.repair.NoopReadRepair;
+import org.apache.cassandra.service.reads.repair.ReadRepair;
+import org.apache.cassandra.transport.Dispatcher;
+
+import javax.annotation.Nullable;
+
+/**
+ * Coordinates the process of endpoint grouping queries for given vnode ranges based on concurrency factor:
+ * <ol>
+ *     <li> Collect token ranges required by concurrency factor in token order and group token ranges by endpoint
+ *     <li> Create single-range read callbacks corresponding to each vnode range, note that:
+ *        <ul>
+ *           <li> In order to maintain proper single result counting for short-read-protection, single-range read callback
+ *                cannot start resolving before previous one has finished resolving.
+ *        </ul>
+ *     <li> Execute {@link MultiRangeReadCommand} on each selected endpoint with all its replicated ranges at once.
+ *     <li> Upon receiving individual {@link MultiRangeReadResponse}:
+ *        <ol>
+ *           <li> It will split multi-range response into single-range responses by queried vnode ranges.
+ *           <li> It will pass single-range responses to their corresponding single-range read callback to allow progressive data merging.
+ *        </ol>
+ *     <li> Return single-range handlers' result in token order.
+ * </ol>
+ */
+public class EndpointGroupingCoordinator
+{
+    private final PartitionRangeReadCommand command;
+    private final DataLimits.Counter counter;
+    private final Map<InetAddressAndPort, EndpointQueryContext> endpointContexts;
+    private final List<ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead>> perRangeHandlers;
+    private final List<PartitionIterator> concurrentQueries;
+
+    private final Dispatcher.RequestTime requestTime;
+    private QueryInfoTracker.ReadTracker readTracker;
+    private final int vnodeRanges;
+
+    /**
+     * @param command current range read command
+     * @param counter the unlimited counter for the command
+     * @param replicaPlans to be queried
+     * @param concurrencyFactor number of vnode ranges to query at once
+     * @param requestTime the start time of the query
+     * @param readTracker
+     */
+    public EndpointGroupingCoordinator(PartitionRangeReadCommand command,
+                                       DataLimits.Counter counter,
+                                       Iterator<ReplicaPlan.ForRangeRead> replicaPlans,
+                                       int concurrencyFactor,
+                                       Dispatcher.RequestTime requestTime,
+                                       QueryInfoTracker.ReadTracker readTracker)
+    {
+        this.command = command;
+        this.counter = counter;
+        this.requestTime = requestTime;
+        this.readTracker = readTracker;
+        this.endpointContexts = new HashMap<>();
+
+        // Read callbacks in token order
+        perRangeHandlers = new ArrayList<>(concurrencyFactor);
+        // Range responses in token order
+        concurrentQueries = new ArrayList<>(concurrencyFactor);
+        int vnodeRanges = 0;
+
+        while (replicaPlans.hasNext() && vnodeRanges < concurrencyFactor)
+        {
+            ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next();
+            readTracker.onReplicaPlan(replicaPlan);
+
+            boolean isFirst = vnodeRanges == 0;
+            vnodeRanges += replicaPlan.vnodeCount();
+            concurrentQueries.add(createResponse(replicaPlan, isFirst));
+        }
+        this.vnodeRanges = vnodeRanges;
+    }
+
+    public int vnodeRanges()
+    {
+        return vnodeRanges;
+    }
+
+    public PartitionIterator execute()
+    {
+        for (EndpointQueryContext replica : replicas())
+            replica.queryReplica();
+
+        return counter.applyTo(PartitionIterators.concat(concurrentQueries));
+    }
+
+    @VisibleForTesting
+    Collection<EndpointQueryContext> endpointRanges()
+    {
+        return endpointContexts.values();
+    }
+
+    /**
+     * @return number of endpoints to be queried
+     */
+    int endpoints()
+    {
+        return endpointContexts.size();
+    }
+
+    private Collection<EndpointQueryContext> replicas()
+    {
+        return endpointContexts.values();
+    }
+
+    /**
+     * Create a {@link SingleRangeResponse} for a given vnode range. The responses are collected and concatenated by
+     * {@code execute}.
+     */
+    private SingleRangeResponse createResponse(ReplicaPlan.ForRangeRead replicaPlan, boolean isFirst)
+    {
+        PartitionRangeReadCommand subrangeCommand = command.forSubRange(replicaPlan.range(), isFirst);
+
+        ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
+
+        DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver = new EndpointDataResolver(subrangeCommand,
+                                                                                                      sharedReplicaPlan,
+                                                                                                      NoopReadRepair.instance,
+                                                                                                      requestTime,
+                                                                                                      readTracker);
+
+        // Create a handler for the range and add it, by replica, to the endpoint contexts.
+        ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler =
+                new ReadCallback<>(resolver, subrangeCommand, sharedReplicaPlan, requestTime);
+        
+        perRangeHandlers.add(handler);
+        for (Replica replica : replicaPlan.contacts())
+        {
+            endpointContexts.computeIfAbsent(replica.endpoint(),
+                                             k -> new EndpointQueryContext(replica.endpoint(),
+                                                                           command.createLimitedCounter(false),
+                                                                           requestTime)).add(handler);
+        }
+        return new SingleRangeResponse(resolver, handler, NoopReadRepair.instance);
+    }
+
+    /**
+     * Collect and query all involved ranges of a given endpoint
+     */
+    public static class EndpointQueryContext
+    {
+        private final InetAddressAndPort endpoint;
+        private final List<ReadCallback<?, ?>> handlers;
+        // used by SRP to track fetched data from each endpoint to determine if an endpoint is exhausted,
+        // aka. no more data can be fetched.
+        private final DataLimits.Counter singleResultCounter;
+        private final Dispatcher.RequestTime requestTime;
+
+        private MultiRangeReadCommand multiRangeCommand;
+
+        public EndpointQueryContext(InetAddressAndPort endpoint, DataLimits.Counter singleResultCounter, Dispatcher.RequestTime requestTime)
+        {
+            this.endpoint = endpoint;
+            this.handlers = new ArrayList<>();
+            this.singleResultCounter = singleResultCounter;
+            this.requestTime = requestTime;
+        }
+
+        /**
+         * @param handler read callback for a given vnode range on the current endpoint
+         */
+        public void add(ReadCallback<?, ?> handler)
+        {
+            assert multiRangeCommand == null : "Cannot add range to already queried context";
+            handlers.add(handler);
+        }
+
+        /**
+         * Query a single endpoint with multiple vnode ranges asynchronously
+         */
+        public void queryReplica()
+        {
+            assert multiRangeCommand == null : "Can only query given endpoint once";
+            this.multiRangeCommand = MultiRangeReadCommand.create(handlers);
+
+            SingleEndpointCallback proxy = new SingleEndpointCallback();
+            Message<ReadCommand> message = multiRangeCommand.createMessage(false, requestTime);
+            MessagingService.instance().sendWithCallback(message, endpoint, proxy);
+        }
+
+        @VisibleForTesting
+        public int rangesCount()
+        {
+            return handlers.size();
+        }
+
+        /**
+         * A proxy responsible for:
+         * 0. propagating failure/timeout to single-range handlers
+         * 1. receiving multi-range responses from a given endpoint
+         * 2. spliting the multi-range responses by vnode ranges
+         * 3. passing the split single-range response to a corresponding read callback which will
+         *    start resolving responses if it has got enough responses for the consistency level requirement.
+         */
+        private class SingleEndpointCallback implements RequestCallback<ReadResponse>
+        {
+            @Override
+            public void onResponse(Message<ReadResponse> response)
+            {
+                // split single-endpoint multi-range response into per-range handlers.
+                MultiRangeReadResponse multiRangeResponse = (MultiRangeReadResponse) response.payload;
+                for (ReadCallback<?, ?> handler : handlers)
+                {
+                    AbstractBounds<PartitionPosition> range = ((PartitionRangeReadCommand) handler.command()).dataRange().keyRange();
+
+                    // extract subrange response in token order
+                    ReadResponse subrangeResponse = multiRangeResponse.subrangeResponse(multiRangeCommand, range);
+                    handler.onResponse(Message.remoteResponse(response.header.from, Verb.RANGE_RSP, response.header.params(), subrangeResponse));
+                }
+            }
+
+            @Override
+            public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+            {
+                for (ReadCallback<?, ?> handler : handlers)
+                    handler.onFailure(from, failureReason);
+            }
+
+            @Override
+            public boolean invokeOnFailure()
+            {
+                return true;
+            }
+
+            @Override
+            public boolean trackLatencyForSnitch()
+            {
+                return true;
+            }
+        }
+    }
+
+    /**
+     * Short-read-protection needs to know if an endpoint has any more data or it has already reached the limit:
+     * If the endpoint has no more data, aka. the counter hasn't reached the limit, there is no point in doing SRP.
+     * If the endpoint might have more data, aka. the counter has reached the limit, SRP might be needed.
+     *
+     * With token ordered range query or single partition query, {@link DataResolver} uses a new single result counter
+     * per replica for a given range, as all replicas are queried with the same range.
+     *
+     * But with endpoint grouping, each source is queried with different token ranges. So we need a shared
+     * cross-range counter for each replica to know if given endpoint has more data.
+     */
+    private class EndpointDataResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>> extends DataResolver<E, P>
+    {
+        public EndpointDataResolver(ReadCommand command, ReplicaPlan.Shared replicaPlan, ReadRepair readRepair, Dispatcher.RequestTime requestTime, QueryInfoTracker.ReadTracker readTracker)
+        {
+            super(command, replicaPlan, readRepair, requestTime, readTracker);
+        }
+
+        @Override
+        protected UnfilteredPartitionIterator shortReadProtectedResponse(int i, DataResolver.ResolveContext context, @Nullable Runnable onShortRead)
+        {
+            UnfilteredPartitionIterator originalResponse = responses.get(i).payload.makeIterator(command);
+
+            if (context.needShortReadProtection())
+            {
+                DataLimits.Counter singleResultCounter = endpointContexts.get(context.replicas.get(i).endpoint()).singleResultCounter;
+                return ShortReadProtection.extend(originalResponse,
+                                                  command,
+                                                  new EndpointShortReadResponseProtection(command,
+                                                                                          context.replicas.get(i),
+                                                                                          () -> { responses.clearUnsafe(i); if (onShortRead != null) onShortRead.run(); },
+                                                                                          singleResultCounter,
+                                                                                          context.mergedResultCounter(),
+                                                                                          requestTime),
+                                                  singleResultCounter);
+            }
+            else
+                return originalResponse;
+        }
+
+        /**
+         * On replica, {@link MultiRangeReadCommand} stops fetching remaining ranges when it reaches limit.
+         *
+         * We should do short-read-protection if current range is not fetched due to limit.
+         */
+        public class EndpointShortReadResponseProtection extends ShortReadPartitionsProtection
+        {
+            public EndpointShortReadResponseProtection(ReadCommand command,
+                                                       Replica source,
+                                                       Runnable preFetchCallback,
+                                                       DataLimits.Counter singleResultCounter,
+                                                       DataLimits.Counter mergedResultCounter,
+                                                       Dispatcher.RequestTime requestTime)
+            {
+                super(command, source, preFetchCallback, singleResultCounter, mergedResultCounter, requestTime);
+            }
+
+            @Override
+            public boolean rangeExhausted()
+            {
+                // if the range is not fetched by original request or SRP, SRP is needed.
+                return super.rangeExhausted() && (rangeFetched || !singleResultCounter.isDone());
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/range/EndpointGroupingRangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/EndpointGroupingRangeCommandIterator.java
new file mode 100644
index 000000000000..383d847dd67c
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/range/EndpointGroupingRangeCommandIterator.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.range;
+
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.service.QueryInfoTracker;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.CloseableIterator;
+
+/**
+ * A range command iterator that executes requests by endpoints and then merges responses in token order. It's designed to
+ * reduce the number of range requests when scanning the whole token ring (eg. rows per range is low) for all range
+ * reads that don't use digests and also to reduce the amount of disk-access for storage-attached indexes, as they will
+ * be able to read index content for all required ranges at once.
+ *
+ * <ul>
+ *     <li> With the non-grouping range command iterator, scanning the entire ring requires "num_of_nodes * num_of_tokens * consistency"
+ *     range requests (assuming no ranges are merged by {@link ReplicaPlanMerger}) to their respective replicas.
+ *
+ *     <li> With the endpoint grouping range command iterator, scanning the entire ring only requires at most "num_of_nodes" multi-range
+ *     requests to their respective replicas. So coordinator will cache up to "num_of_nodes" responses.
+ * </ul>
+ */
+public class EndpointGroupingRangeCommandIterator extends RangeCommandIterator
+{
+    EndpointGroupingRangeCommandIterator(CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans,
+                                         PartitionRangeReadCommand command,
+                                         int concurrencyFactor,
+                                         int maxConcurrencyFactor,
+                                         int totalRangeCount,
+                                         Dispatcher.RequestTime requestTime,
+                                         QueryInfoTracker.ReadTracker readTracker)
+    {
+        super(replicaPlans, command, concurrencyFactor, maxConcurrencyFactor, totalRangeCount, requestTime, readTracker);
+    }
+
+    @Override
+    protected PartitionIterator sendNextRequests()
+    {
+        counter = command.createUnlimitedCounter(true);
+
+        EndpointGroupingCoordinator coordinator = new EndpointGroupingCoordinator(command,
+                                                                                  counter,
+                                                                                  replicaPlans,
+                                                                                  concurrencyFactor(),
+                                                                                  requestTime,
+                                                                                  readTracker);
+        PartitionIterator partitions = coordinator.execute();
+
+        rangesQueried += coordinator.vnodeRanges();
+        batchesRequested++;
+        Tracing.trace("Submitted concurrent grouped range read requests to {} endpoints", coordinator.endpoints());
+        return partitions;
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/range/NonGroupingRangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/NonGroupingRangeCommandIterator.java
new file mode 100644
index 000000000000..ed4548666298
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/range/NonGroupingRangeCommandIterator.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.range;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.QueryInfoTracker;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.reads.DataResolver;
+import org.apache.cassandra.service.reads.ReadCallback;
+import org.apache.cassandra.service.reads.repair.ReadRepair;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.CloseableIterator;
+
+public class NonGroupingRangeCommandIterator extends RangeCommandIterator
+{
+    NonGroupingRangeCommandIterator(CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans,
+                                    PartitionRangeReadCommand command,
+                                    int concurrencyFactor,
+                                    int maxConcurrencyFactor,
+                                    int totalRangeCount,
+                                    final Dispatcher.RequestTime requestTime,
+                                    QueryInfoTracker.ReadTracker readTracker)
+    {
+        super(replicaPlans, command, concurrencyFactor, maxConcurrencyFactor, totalRangeCount, requestTime, readTracker);
+    }
+
+    protected PartitionIterator sendNextRequests()
+    {
+        List<PartitionIterator> concurrentQueries = new ArrayList<>(concurrencyFactor);
+        List<ReadRepair<?, ?>> readRepairs = new ArrayList<>(concurrencyFactor);
+
+        try
+        {
+            for (int i = 0; i < concurrencyFactor() && replicaPlans.hasNext(); )
+            {
+                ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next();
+                readTracker.onReplicaPlan(replicaPlan);
+
+                @SuppressWarnings("resource") // response will be closed by concatAndBlockOnRepair, or in the catch block below
+                SingleRangeResponse response = query(replicaPlan, i == 0);
+                concurrentQueries.add(response);
+                readRepairs.add(response.getReadRepair());
+                // due to RangeMerger, coordinator may fetch more ranges than required by concurrency factor.
+                rangesQueried += replicaPlan.vnodeCount();
+                i += replicaPlan.vnodeCount();
+            }
+            batchesRequested++;
+        }
+        catch (Throwable t)
+        {
+            for (PartitionIterator response : concurrentQueries)
+                response.close();
+            throw t;
+        }
+
+        Tracing.trace("Submitted {} concurrent range requests", concurrentQueries.size());
+        // We want to count the results for the sake of updating the concurrency factor (see updateConcurrencyFactor)
+        // but we don't want to enforce any particular limit at this point (this could break code than rely on
+        // postReconciliationProcessing), hence the unlimited counter that uses DataLimits.NONE.
+        counter = command.createUnlimitedCounter(true);
+        return counter.applyTo(StorageProxy.concatAndBlockOnRepair(concurrentQueries, readRepairs));
+    }
+
+    /**
+     * Queries the provided sub-range.
+     *
+     * @param replicaPlan the subRange to query.
+     * @param isFirst in the case where multiple queries are sent in parallel, whether that's the first query on
+     * that batch or not. The reason it matters is that whe paging queries, the command (more specifically the
+     * {@code DataLimits}) may have "state" information and that state may only be valid for the first query (in
+     * that it's the query that "continues" whatever we're previously queried).
+     */
+    private SingleRangeResponse query(ReplicaPlan.ForRangeRead replicaPlan, boolean isFirst)
+    {
+        PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaPlan.range(), isFirst);
+
+        // If enabled, request repaired data tracking info from full replicas but
+        // only if there are multiple full replicas to compare results from
+        boolean trackRepairData = DatabaseDescriptor.getRepairedDataTrackingForRangeReadsEnabled()
+                                    && replicaPlan.contacts().filter(Replica::isFull).size() > 1;
+
+        ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
+        ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> readRepair =
+        ReadRepair.create(command, sharedReplicaPlan, requestTime);
+        DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver =
+        new DataResolver<>(rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairData, readTracker);
+        ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler =
+        new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, requestTime);
+
+        if (replicaPlan.contacts().size() == 1 && replicaPlan.contacts().get(0).isSelf())
+        {
+            Stage.READ.execute(new StorageProxy.LocalReadRunnable(rangeCommand, handler, requestTime));
+        }
+        else
+        {
+            for (Replica replica : replicaPlan.contacts())
+            {
+                Tracing.trace("Enqueuing request to {}", replica);
+                ReadCommand command = replica.isFull() ? rangeCommand : rangeCommand.copyAsTransientQuery(replica);
+                Message<ReadCommand> message = command.createMessage(trackRepairData && replica.isFull(), requestTime);
+                MessagingService.instance().sendWithCallback(message, replica.endpoint(), handler);
+            }
+        }
+
+        return new SingleRangeResponse(resolver, handler, readRepair);
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
index 3e9ac453c70c..b23a12511302 100644
--- a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
+++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
@@ -18,17 +18,13 @@
 
 package org.apache.cassandra.service.reads.range;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionRangeReadCommand;
 import org.apache.cassandra.db.ReadCommand;
@@ -39,62 +35,89 @@
 import org.apache.cassandra.exceptions.ReadFailureException;
 import org.apache.cassandra.exceptions.ReadTimeoutException;
 import org.apache.cassandra.exceptions.UnavailableException;
-import org.apache.cassandra.locator.EndpointsForRange;
-import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.metrics.ClientRangeRequestMetrics;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.service.reads.DataResolver;
-import org.apache.cassandra.service.reads.ReadCallback;
-import org.apache.cassandra.service.reads.repair.ReadRepair;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.CloseableIterator;
 
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RANGE_READ_ENDPOINT_GROUPING_ENABLED;
 
 @VisibleForTesting
-public class RangeCommandIterator extends AbstractIterator<RowIterator> implements PartitionIterator
+public abstract class RangeCommandIterator extends AbstractIterator<RowIterator> implements PartitionIterator
 {
     private static final Logger logger = LoggerFactory.getLogger(RangeCommandIterator.class);
+    private static final boolean ENDPOINT_GROUPING_ENABLED = RANGE_READ_ENDPOINT_GROUPING_ENABLED.getBoolean();
 
-    public static final ClientRangeRequestMetrics rangeMetrics = new ClientRangeRequestMetrics("RangeSlice");
+    @VisibleForTesting
+    public final ClientRangeRequestMetrics rangeMetrics;
 
+    final Dispatcher.RequestTime requestTime;
     final CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans;
     final int totalRangeCount;
     final PartitionRangeReadCommand command;
     final boolean enforceStrictLiveness;
-    final Dispatcher.RequestTime requestTime;
-
-    int rangesQueried;
-    int batchesRequested = 0;
 
-
-    private DataLimits.Counter counter;
+    protected DataLimits.Counter counter;
     private PartitionIterator sentQueryIterator;
+    protected QueryInfoTracker.ReadTracker readTracker;
 
     private final int maxConcurrencyFactor;
-    private int concurrencyFactor;
+    protected int concurrencyFactor;
     // The two following "metric" are maintained to improve the concurrencyFactor
     // when it was not good enough initially.
     private int liveReturned;
+    int rangesQueried;
+    int batchesRequested = 0;
+
+    @SuppressWarnings("resource")
+    public static RangeCommandIterator create(CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans,
+                                              PartitionRangeReadCommand command,
+                                              int concurrencyFactor,
+                                              int maxConcurrencyFactor,
+                                              int totalRangeCount,
+                                              Dispatcher.RequestTime requestTime,
+                                              QueryInfoTracker.ReadTracker readTracker)
+    {
+        return ENDPOINT_GROUPING_ENABLED && supportsEndpointGrouping(command) ? new EndpointGroupingRangeCommandIterator(replicaPlans,
+                                                                                            command,
+                                                                                            concurrencyFactor,
+                                                                                            maxConcurrencyFactor,
+                                                                                            totalRangeCount,
+                                                                                            requestTime,
+                                                                                            readTracker)
+                                                 : new NonGroupingRangeCommandIterator(replicaPlans,
+                                                                                       command,
+                                                                                       concurrencyFactor,
+                                                                                       maxConcurrencyFactor,
+                                                                                       totalRangeCount,
+                                                                                       requestTime,
+                                                                                       readTracker);
+    }
 
     RangeCommandIterator(CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans,
                          PartitionRangeReadCommand command,
                          int concurrencyFactor,
                          int maxConcurrencyFactor,
                          int totalRangeCount,
-                         Dispatcher.RequestTime requestTime)
+                         Dispatcher.RequestTime requestTime,
+                         QueryInfoTracker.ReadTracker readTracker)
     {
+        this.rangeMetrics = ClientRequestsMetricsProvider.instance.metrics(command.metadata().keyspace).rangeMetrics;
         this.replicaPlans = replicaPlans;
         this.command = command;
         this.concurrencyFactor = concurrencyFactor;
         this.maxConcurrencyFactor = maxConcurrencyFactor;
         this.totalRangeCount = totalRangeCount;
         this.requestTime = requestTime;
+        this.readTracker = readTracker;
+
         enforceStrictLiveness = command.metadata().enforceStrictLiveness();
     }
 
@@ -155,6 +178,17 @@ private void updateConcurrencyFactor()
         concurrencyFactor = computeConcurrencyFactor(totalRangeCount, rangesQueried, maxConcurrencyFactor, command.limits().count(), liveReturned);
     }
 
+    private static boolean supportsEndpointGrouping(ReadCommand command)
+    {
+        // With endpoint grouping, ranges executed on each endpoint are different, digest is unlikely to match.
+        if (command.isDigestQuery())
+            return false;
+
+        // Endpoint grouping is currently only supported by SAI
+        Index.QueryPlan queryPlan = command.indexQueryPlan();
+        return queryPlan != null && queryPlan.supportsMultiRangeReadCommand();
+    }
+
     @VisibleForTesting
     static int computeConcurrencyFactor(int totalRangeCount, int rangesQueried, int maxConcurrencyFactor, int limit, int liveReturned)
     {
@@ -176,84 +210,7 @@ static int computeConcurrencyFactor(int totalRangeCount, int rangesQueried, int
         return concurrencyFactor;
     }
 
-    /**
-     * Queries the provided sub-range.
-     *
-     * @param replicaPlan the subRange to query.
-     * @param isFirst in the case where multiple queries are sent in parallel, whether that's the first query on
-     * that batch or not. The reason it matters is that whe paging queries, the command (more specifically the
-     * {@code DataLimits}) may have "state" information and that state may only be valid for the first query (in
-     * that it's the query that "continues" whatever we're previously queried).
-     */
-    private SingleRangeResponse query(ReplicaPlan.ForRangeRead replicaPlan, boolean isFirst)
-    {
-        PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaPlan.range(), isFirst);
-        
-        // If enabled, request repaired data tracking info from full replicas, but
-        // only if there are multiple full replicas to compare results from.
-        boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForRangeReadsEnabled()
-                                      && replicaPlan.contacts().filter(Replica::isFull).size() > 1;
-
-        ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
-        ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> readRepair =
-                ReadRepair.create(command, sharedReplicaPlan, requestTime);
-        DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver =
-                new DataResolver<>(rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairedStatus);
-        ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler =
-                new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, requestTime);
-
-        if (replicaPlan.contacts().size() == 1 && replicaPlan.contacts().get(0).isSelf())
-        {
-            Stage.READ.execute(new StorageProxy.LocalReadRunnable(rangeCommand, handler, requestTime, trackRepairedStatus));
-        }
-        else
-        {
-            for (Replica replica : replicaPlan.contacts())
-            {
-                Tracing.trace("Enqueuing request to {}", replica);
-                ReadCommand command = replica.isFull() ? rangeCommand : rangeCommand.copyAsTransientQuery(replica);
-                Message<ReadCommand> message = command.createMessage(trackRepairedStatus && replica.isFull(), requestTime);
-                MessagingService.instance().sendWithCallback(message, replica.endpoint(), handler);
-            }
-        }
-
-        return new SingleRangeResponse(resolver, handler, readRepair);
-    }
-
-    PartitionIterator sendNextRequests()
-    {
-        List<PartitionIterator> concurrentQueries = new ArrayList<>(concurrencyFactor);
-        List<ReadRepair<?, ?>> readRepairs = new ArrayList<>(concurrencyFactor);
-
-        try
-        {
-            for (int i = 0; i < concurrencyFactor && replicaPlans.hasNext(); )
-            {
-                ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next();
-
-                SingleRangeResponse response = query(replicaPlan, i == 0);
-                concurrentQueries.add(response);
-                readRepairs.add(response.getReadRepair());
-                // due to RangeMerger, coordinator may fetch more ranges than required by concurrency factor.
-                rangesQueried += replicaPlan.vnodeCount();
-                i += replicaPlan.vnodeCount();
-            }
-            batchesRequested++;
-        }
-        catch (Throwable t)
-        {
-            for (PartitionIterator response : concurrentQueries)
-                response.close();
-            throw t;
-        }
-
-        Tracing.trace("Submitted {} concurrent range requests", concurrentQueries.size());
-        // We want to count the results for the sake of updating the concurrency factor (see updateConcurrencyFactor)
-        // but we don't want to enforce any particular limit at this point (this could break code than rely on
-        // postReconciliationProcessing), hence the DataLimits.NONE.
-        counter = DataLimits.NONE.newCounter(command.nowInSec(), true, command.selectsFullPartition(), enforceStrictLiveness);
-        return counter.applyTo(StorageProxy.concatAndBlockOnRepair(concurrentQueries, readRepairs));
-    }
+    protected abstract PartitionIterator sendNextRequests();
 
     @Override
     public void close()
@@ -267,11 +224,12 @@ public void close()
         }
         finally
         {
+            rangeMetrics.roundTrips.update(batchesRequested);
             // We track latency based on request processing time, since the amount of time that request spends in the queue
             // is not a representative metric of replica performance.
             long latency = nanoTime() - requestTime.startedAtNanos();
-            rangeMetrics.addNano(latency);
-            rangeMetrics.roundTrips.update(batchesRequested);
+            rangeMetrics.executionTimeMetrics.addNano(latency);
+            rangeMetrics.serviceTimeMetrics.addNano(latency);
             Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS);
         }
     }
diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
index ded4d4cdbea6..dce6f190242c 100644
--- a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
+++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
@@ -34,6 +34,7 @@
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.locator.ReplicaPlans;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
@@ -55,10 +56,11 @@ public class RangeCommands
 
     public static PartitionIterator partitions(PartitionRangeReadCommand command,
                                                ConsistencyLevel consistencyLevel,
-                                               Dispatcher.RequestTime requestTime)
+                                               Dispatcher.RequestTime requestTime,
+                                               QueryInfoTracker.ReadTracker readTracker)
     {
         // Note that in general, a RangeCommandIterator will honor the command limit for each range, but will not enforce it globally.
-        RangeCommandIterator rangeCommands = rangeCommandIterator(command, consistencyLevel, requestTime);
+        RangeCommandIterator rangeCommands = rangeCommandIterator(command, consistencyLevel, requestTime, readTracker);
         return command.limits().filter(command.postReconciliationProcessing(rangeCommands),
                                        command.nowInSec(),
                                        command.selectsFullPartition(),
@@ -68,7 +70,8 @@ public static PartitionIterator partitions(PartitionRangeReadCommand command,
     @VisibleForTesting
     static RangeCommandIterator rangeCommandIterator(PartitionRangeReadCommand command,
                                                      ConsistencyLevel consistencyLevel,
-                                                     Dispatcher.RequestTime requestTime)
+                                                     Dispatcher.RequestTime requestTime,
+                                                     QueryInfoTracker.ReadTracker readTracker)
     {
         Tracing.trace("Computing ranges to query");
 
@@ -77,9 +80,8 @@ static RangeCommandIterator rangeCommandIterator(PartitionRangeReadCommand comma
                                                                    command.indexQueryPlan(),
                                                                    keyspace,
                                                                    consistencyLevel);
-
         if (command.isTopK())
-            return new ScanAllRangesCommandIterator(keyspace, replicaPlans, command, replicaPlans.size(), requestTime);
+            return new ScanAllRangesCommandIterator(keyspace, replicaPlans, command, replicaPlans.size(), requestTime, readTracker);
 
         int maxConcurrencyFactor = Math.min(replicaPlans.size(), MAX_CONCURRENT_RANGE_REQUESTS);
         int concurrencyFactor = maxConcurrencyFactor;
@@ -107,12 +109,14 @@ static RangeCommandIterator rangeCommandIterator(PartitionRangeReadCommand comma
         }
 
         ReplicaPlanMerger mergedReplicaPlans = new ReplicaPlanMerger(replicaPlans, keyspace, consistencyLevel);
-        return new RangeCommandIterator(mergedReplicaPlans,
-                                        command,
-                                        concurrencyFactor,
-                                        maxConcurrencyFactor,
-                                        replicaPlans.size(),
-                                        requestTime);
+
+        return RangeCommandIterator.create(mergedReplicaPlans,
+                                           command,
+                                           concurrencyFactor,
+                                           maxConcurrencyFactor,
+                                           replicaPlans.size(),
+                                           requestTime,
+                                           readTracker);
     }
 
     /**
@@ -128,7 +132,7 @@ static float estimateResultsPerRange(PartitionRangeReadCommand command, Keyspace
         Index.QueryPlan index = command.indexQueryPlan();
         float maxExpectedResults = index == null
                                    ? command.limits().estimateTotalResults(cfs)
-                                   : index.getEstimatedResultRows();
+                                   : command.indexQueryPlan().getEstimatedResultRows();
 
         // adjust maxExpectedResults by the number of tokens this node has and the replication factor for this ks
         return (maxExpectedResults / DatabaseDescriptor.getNumTokens())
diff --git a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java b/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java
index c14dc3c4c850..70476be62fe2 100644
--- a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java
+++ b/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java
@@ -22,7 +22,6 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -38,7 +37,6 @@
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlans;
 import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.Pair;
 
@@ -60,9 +58,10 @@ class ReplicaPlanIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
         this.keyspace = keyspace;
         this.consistency = consistency;
 
+
         List<? extends AbstractBounds<PartitionPosition>> l = keyspace.getReplicationStrategy() instanceof LocalStrategy
                                                               ? keyRange.unwrap()
-                                                              : getRestrictedRanges(keyRange);
+                                                              : getRestrictedRanges(keyspace.getReplicationStrategy().getTokenMetadata(), keyRange);
         this.ranges = l.iterator();
         this.rangeCount = l.size();
     }
@@ -88,7 +87,7 @@ protected ReplicaPlan.ForRangeRead computeNext()
      * Compute all ranges we're going to query, in sorted order. Nodes can be replica destinations for many ranges,
      * so we need to restrict each scan to the specific range we want, or else we'd get duplicate results.
      */
-    private static List<AbstractBounds<PartitionPosition>> getRestrictedRanges(final AbstractBounds<PartitionPosition> queryRange)
+    private static List<AbstractBounds<PartitionPosition>> getRestrictedRanges(TokenMetadata tokenMetadata, final AbstractBounds<PartitionPosition> queryRange)
     {
         // special case for bounds containing exactly 1 (non-minimum) token
         if (queryRange instanceof Bounds && queryRange.left.equals(queryRange.right) && !queryRange.left.isMinimum())
@@ -96,8 +95,6 @@ private static List<AbstractBounds<PartitionPosition>> getRestrictedRanges(final
             return Collections.singletonList(queryRange);
         }
 
-        TokenMetadata tokenMetadata = StorageService.instance.getTokenMetadata();
-
         List<AbstractBounds<PartitionPosition>> ranges = new ArrayList<>();
         // divide the queryRange into pieces delimited by the ring and minimum tokens
         Iterator<Token> ringIter = TokenMetadata.ringIterator(tokenMetadata.sortedTokens(), queryRange.left.getToken(), true);
diff --git a/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java
index 53f55f8938ae..e4630ae9f638 100644
--- a/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java
+++ b/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java
@@ -36,6 +36,7 @@
 import org.apache.cassandra.locator.ReplicaPlans;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.service.reads.DataResolver;
 import org.apache.cassandra.service.reads.ReadCallback;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
@@ -46,15 +47,9 @@
 /**
  * A custom {@link RangeCommandIterator} that queries all replicas required by consistency level at once with data range
  * specify in {@link PartitionRangeReadCommand}.
- * <p>
+ *
  * This is to speed up {@link Index.QueryPlan#isTopK()} queries that needs to find global top-k rows in the cluster, because
  * existing {@link RangeCommandIterator} has to execute a top-k search per vnode range which is wasting resources.
- * <p>
- * The implementation combines the replica plans for each data range into a single shared replica plan. This results in
- * queries using reconciliation where it may not be expected. This is handled in the {@link DataResolver} for top-K queries
- * so any usage for queries other that top-K should bear this in mind.
- * <p>
- * It is important to note that this implementation can only be used with {@link ConsistencyLevel#ONE} and {@link ConsistencyLevel#LOCAL_ONE}
  */
 public class ScanAllRangesCommandIterator extends RangeCommandIterator
 {
@@ -63,9 +58,10 @@ public class ScanAllRangesCommandIterator extends RangeCommandIterator
     ScanAllRangesCommandIterator(Keyspace keyspace, CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans,
                                  PartitionRangeReadCommand command,
                                  int totalRangeCount,
-                                 Dispatcher.RequestTime requestTime)
+                                 Dispatcher.RequestTime requestTime,
+                                 QueryInfoTracker.ReadTracker readTracker)
     {
-        super(replicaPlans, command, totalRangeCount, totalRangeCount, totalRangeCount, requestTime);
+        super(replicaPlans, command, totalRangeCount, totalRangeCount, totalRangeCount, requestTime, readTracker);
         Preconditions.checkState(command.isTopK());
 
         this.keyspace = keyspace;
@@ -92,7 +88,7 @@ protected PartitionIterator sendNextRequests()
 
         ReplicaPlan.ForRangeRead plan = ReplicaPlans.forFullRangeRead(keyspace, consistencyLevel, command.dataRange().keyRange(), replicasToQuery, totalRangeCount);
         ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(plan);
-        DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver = new DataResolver<>(command, sharedReplicaPlan, NoopReadRepair.instance, requestTime, false);
+        DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver = new DataResolver<>(command, sharedReplicaPlan, NoopReadRepair.instance, requestTime, false, readTracker);
         ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler = new ReadCallback<>(resolver, command, sharedReplicaPlan, requestTime);
 
         int nodes = 0;
diff --git a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java
index 8343b83b071e..a43fb5b3d8e1 100644
--- a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java
+++ b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java
@@ -139,7 +139,7 @@ public void startRepair(DigestResolver<E, P> digestResolver, Consumer<PartitionI
         boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForPartitionReadsEnabled();
 
         // Do a full data read to resolve the correct response (and repair node that need be)
-        DataResolver<E, P> resolver = new DataResolver<>(command, replicaPlan, this, requestTime, trackRepairedStatus);
+        DataResolver<E, P> resolver = new DataResolver<>(command, replicaPlan, this, requestTime, trackRepairedStatus, digestResolver.getReadTracker());
         ReadCallback<E, P> readCallback = new ReadCallback<>(resolver, command, replicaPlan, requestTime);
 
         digestRepair = new DigestRepair<>(resolver, readCallback, resultConsumer);
diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java
index a63cc7f6bfca..557b5b840709 100644
--- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java
+++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java
@@ -70,7 +70,7 @@ ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPla
      * to additional replicas not contacted in the initial full data read. If the collection of nodes that
      * end up responding in time end up agreeing on the data, and we don't consider the response from the
      * disagreeing replica that triggered the read repair, that's ok, since the disagreeing data would not
-     * have been successfully written and won't be included in the response the the client, preserving the
+     * have been successfully written and won't be included in the response the client, preserving the
      * expectation of monotonic quorum reads
      */
     public void maybeSendAdditionalReads();
diff --git a/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java b/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java
index 353200088fe5..83093f74e32b 100644
--- a/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java
+++ b/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java
@@ -183,13 +183,13 @@ private void applyToPartition(int i, Consumer<PartitionUpdate.Builder> f)
         if (writeBackTo.get(i))
         {
             if (repairs[i] == null)
-                repairs[i] = new PartitionUpdate.Builder(command.metadata(), partitionKey, columns, 1);
+                repairs[i] = PartitionUpdate.builder(command.metadata(), partitionKey, columns, 1);
             f.accept(repairs[i]);
         }
         if (buildFullDiff)
         {
             if (repairs[repairs.length - 1] == null)
-                repairs[repairs.length - 1] = new PartitionUpdate.Builder(command.metadata(), partitionKey, columns, 1);
+                repairs[repairs.length - 1] = PartitionUpdate.builder(command.metadata(), partitionKey, columns, 1);
             f.accept(repairs[repairs.length - 1]);
         }
     }
diff --git a/src/java/org/apache/cassandra/service/reads/thresholds/WarningsSnapshot.java b/src/java/org/apache/cassandra/service/reads/thresholds/WarningsSnapshot.java
index 0a07c8360ea9..591b73cc9412 100644
--- a/src/java/org/apache/cassandra/service/reads/thresholds/WarningsSnapshot.java
+++ b/src/java/org/apache/cassandra/service/reads/thresholds/WarningsSnapshot.java
@@ -194,7 +194,10 @@ public int hashCode()
     @Override
     public String toString()
     {
-        return "(tombstones=" + tombstones + ", localReadSize=" + localReadSize + ", rowIndexTooLarge=" + rowIndexReadSize + ')';
+        return "(tombstones=" + tombstones +
+                ", localReadSize=" + localReadSize +
+                ", rowIndexTooLarge=" + rowIndexReadSize +
+                ", indexReadSSTablesCount=" + indexReadSSTablesCount + ')';
     }
 
     public static final class Warnings
diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
index d31df361e5f7..fd50d9df2734 100644
--- a/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
+++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
@@ -56,26 +56,21 @@ public class SnapshotLoader
 
     static final Pattern SNAPSHOT_DIR_PATTERN = Pattern.compile("(?<keyspace>\\w+)/(?<tableName>\\w+)-(?<tableId>[0-9a-f]{32})/snapshots/(?<tag>.+)$");
 
-    private final Collection<Path> dataDirectories;
+    private final Collection<File> dataDirectories;
 
     public SnapshotLoader()
     {
         this(DatabaseDescriptor.getAllDataFileLocations());
     }
 
-    public SnapshotLoader(String[] dataDirectories)
+    public SnapshotLoader(File[] dataDirectories)
     {
-        this(Arrays.stream(dataDirectories).map(File::getPath).collect(Collectors.toList()));
-    }
-
-    public SnapshotLoader(Collection<Path> dataDirs)
-    {
-        this.dataDirectories = dataDirs;
+        this.dataDirectories = Arrays.stream(dataDirectories).collect(Collectors.toList());
     }
 
     public SnapshotLoader(Directories directories)
     {
-        this(directories.getCFDirectories().stream().map(File::toPath).collect(Collectors.toList()));
+        this.dataDirectories = directories.getCFDirectories();
     }
 
     @VisibleForTesting
@@ -166,15 +161,15 @@ public Set<TableSnapshot> loadSnapshots(String keyspace)
         Map<String, TableSnapshot.Builder> snapshots = new HashMap<>();
         Visitor visitor = new Visitor(snapshots);
 
-        for (Path dataDir : dataDirectories)
+        for (File dataDir : dataDirectories)
         {
             if (keyspace != null)
                 dataDir = dataDir.resolve(keyspace);
 
             try
             {
-                if (new File(dataDir).exists())
-                    Files.walkFileTree(dataDir, Collections.emptySet(), maxDepth, visitor);
+                if (dataDir.exists())
+                    Files.walkFileTree(dataDir.toPath(), Collections.emptySet(), maxDepth, visitor);
                 else
                     logger.debug("Skipping non-existing data directory {}", dataDir);
             }
diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java
index 3925f3f9dc7b..b59e2dfc1c2c 100644
--- a/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java
+++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java
@@ -53,7 +53,18 @@ public class SnapshotManager {
 
     private final long initialDelaySeconds;
     private final long cleanupPeriodSeconds;
-    private final SnapshotLoader snapshotLoader;
+
+    private static class SnapshotLoaderHolder
+    {
+        // Use subclass for lazy initialization to avoid race with DatabaseDescriptor.createAllDirectories()
+        private static final SnapshotLoader snapshotLoader = new SnapshotLoader(DatabaseDescriptor.getAllDataFileLocations());;
+    }
+
+    private static SnapshotLoader getSnapshotLoader()
+    {
+        // Return the singleton SnapshotLoader instance
+        return SnapshotManager.SnapshotLoaderHolder.snapshotLoader;
+    }
 
     @VisibleForTesting
     protected volatile ScheduledFuture<?> cleanupTaskFuture;
@@ -75,7 +86,6 @@ protected SnapshotManager(long initialDelaySeconds, long cleanupPeriodSeconds)
     {
         this.initialDelaySeconds = initialDelaySeconds;
         this.cleanupPeriodSeconds = cleanupPeriodSeconds;
-        snapshotLoader = new SnapshotLoader(DatabaseDescriptor.getAllDataFileLocations());
     }
 
     public Collection<TableSnapshot> getExpiringSnapshots()
@@ -111,12 +121,12 @@ public synchronized void addSnapshot(TableSnapshot snapshot)
 
     public synchronized Set<TableSnapshot> loadSnapshots(String keyspace)
     {
-        return snapshotLoader.loadSnapshots(keyspace);
+        return getSnapshotLoader().loadSnapshots(keyspace);
     }
 
     public synchronized Set<TableSnapshot> loadSnapshots()
     {
-        return snapshotLoader.loadSnapshots();
+        return getSnapshotLoader().loadSnapshots();
     }
 
     @VisibleForTesting
diff --git a/src/java/org/apache/cassandra/streaming/ProgressInfo.java b/src/java/org/apache/cassandra/streaming/ProgressInfo.java
index 159775c324cc..683b1360aa98 100644
--- a/src/java/org/apache/cassandra/streaming/ProgressInfo.java
+++ b/src/java/org/apache/cassandra/streaming/ProgressInfo.java
@@ -60,7 +60,6 @@ public static Direction fromByte(byte direction)
     public ProgressInfo(InetAddressAndPort peer, int sessionIndex, String fileName, Direction direction,
                         long currentBytes,  long deltaBytes, long totalBytes)
     {
-
         this.peer = peer;
         this.sessionIndex = sessionIndex;
         this.fileName = fileName;
diff --git a/src/java/org/apache/cassandra/streaming/StreamOperation.java b/src/java/org/apache/cassandra/streaming/StreamOperation.java
index 98a4070d2b0c..c195b4a0cf6c 100644
--- a/src/java/org/apache/cassandra/streaming/StreamOperation.java
+++ b/src/java/org/apache/cassandra/streaming/StreamOperation.java
@@ -17,6 +17,8 @@
  */
 package org.apache.cassandra.streaming;
 
+import org.apache.cassandra.db.compaction.OperationType;
+
 public enum StreamOperation
 {
     OTHER("Other", true, false), // Fallback to avoid null types when deserializing from string
@@ -26,7 +28,9 @@ public enum StreamOperation
     BOOTSTRAP("Bootstrap", false, true),
     REBUILD("Rebuild", false, true),
     BULK_LOAD("Bulk Load", true, false),
-    REPAIR("Repair", true, false);
+    REPAIR("Repair", true, false),
+    REGION_DECOMMISSION("Region Decommission", false, true),
+    REGION_REPAIR("Region Repair", true, false);
 
     private final String description;
     private final boolean requiresViewBuild;
@@ -71,4 +75,17 @@ public boolean keepSSTableLevel()
     {
         return keepSSTableLevel;
     }
+
+    /**
+     * @return the corresponding compaction operation type
+     */
+    public OperationType opType()
+    {
+        switch (this)
+        {
+            case REGION_DECOMMISSION: return OperationType.REGION_DECOMMISSION;
+            case REGION_REPAIR:       return OperationType.REGION_REPAIR;
+            default:                  return OperationType.STREAM;
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/streaming/StreamPlan.java b/src/java/org/apache/cassandra/streaming/StreamPlan.java
index 47fa9e1463bf..46a0d98420f2 100644
--- a/src/java/org/apache/cassandra/streaming/StreamPlan.java
+++ b/src/java/org/apache/cassandra/streaming/StreamPlan.java
@@ -215,6 +215,11 @@ public TimeUUID getPendingRepair()
         return coordinator.getPendingRepair();
     }
 
+    public TimeUUID getPlanId()
+    {
+        return planId;
+    }
+
     public boolean getFlushBeforeTransfer()
     {
         return flushBeforeTransfer;
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
index 002e1827148a..f1d7d595bb48 100644
--- a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
@@ -25,6 +25,8 @@
 import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -126,7 +128,8 @@ public void run()
         {
             try
             {
-                if (ColumnFamilyStore.getIfExists(task.tableId) == null)
+                ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(task.tableId);
+                if (cfs == null)
                 {
                     // schema was dropped during streaming
                     task.receiver.abort();
@@ -134,6 +137,10 @@ public void run()
                     return;
                 }
 
+
+                if (!CassandraRelevantProperties.CDC_STREAMING_ENABLED.getBoolean() && cfs.metadata().params.cdc)
+                    throw new RuntimeException(String.format("Streaming CDC-enabled sstables is not supported, aborting table %s", cfs));
+
                 task.receiver.finished();
                 task.session.taskCompleted(task);
             }
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 33f02c3d0daf..c819e772c692 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -49,6 +49,7 @@
 import io.netty.channel.Channel;
 import io.netty.util.concurrent.Future; //checkstyle: permit this import
 
+import org.apache.cassandra.db.compaction.CompactionStrategyContainer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -58,7 +59,6 @@
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.CompactionStrategyManager;
 import org.apache.cassandra.db.lifecycle.TransactionAlreadyCompletedException;
 import org.apache.cassandra.dht.OwnedRanges;
 import org.apache.cassandra.dht.Range;
@@ -446,7 +446,7 @@ synchronized void addTransferRanges(String keyspace, RangesAtEndpoint replicas,
     {
         failIfFinished();
         Collection<ColumnFamilyStore> stores = getColumnFamilyStores(keyspace, columnFamilies);
-        if (flushTables)
+        if (flushTables && DatabaseDescriptor.supportsFlushBeforeStreaming())
             flushSSTables(stores);
 
         //Was it safe to remove this normalize, sorting seems not to matter, merging? Maybe we should have?
@@ -969,7 +969,7 @@ static boolean checkPendingCompactions(Map<TableId, Long> perTableIdIncomingByte
                                                                                 .collect(Collectors.toMap(ks::getColumnFamilyStore, Function.identity()));
             for (ColumnFamilyStore cfs : ks.getColumnFamilyStores())
             {
-                CompactionStrategyManager csm = cfs.getCompactionStrategyManager();
+                CompactionStrategyContainer csm = cfs.getCompactionStrategyContainer();
                 int tasksOther = csm.getEstimatedRemainingTasks();
                 int tasksStreamed = tasksOther;
                 if (cfStreamed.containsKey(cfs))
diff --git a/src/java/org/apache/cassandra/tools/AuditLogViewer.java b/src/java/org/apache/cassandra/tools/AuditLogViewer.java
index f226aa2e706d..586f2824d5bc 100644
--- a/src/java/org/apache/cassandra/tools/AuditLogViewer.java
+++ b/src/java/org/apache/cassandra/tools/AuditLogViewer.java
@@ -32,10 +32,10 @@
 import org.apache.commons.cli.ParseException;
 
 import net.openhft.chronicle.core.io.IORuntimeException;
-import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.queue.ExcerptTailer;
 import net.openhft.chronicle.queue.RollCycles;
 import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue;
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.threads.Pauser;
 import net.openhft.chronicle.wire.ReadMarshallable;
 import net.openhft.chronicle.wire.WireIn;
diff --git a/src/java/org/apache/cassandra/tools/CompactionLogAnalyzer.java b/src/java/org/apache/cassandra/tools/CompactionLogAnalyzer.java
new file mode 100644
index 000000000000..e754936240e6
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/CompactionLogAnalyzer.java
@@ -0,0 +1,561 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.BufferedReader;
+import java.io.File; //checkstyle: permit this import
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.Table;
+import com.google.common.io.ByteStreams;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.utils.FBUtilities;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+
+//
+// Analyzes a collection of CSV logs from the unified compaction strategy. Run with
+//
+//   tools/bin/analyzecompactionlog <path-to-directory-with-csvs>
+//
+// It will process the CSVs are create a compaction_report.html file in the target directory. The file is similar to our
+// performance reports.
+//
+public class CompactionLogAnalyzer
+{
+
+    private static final Options options = new Options();
+    private static CommandLine cmd;
+
+    public static final String OPTION_LIMIT = "l";
+    public static final String OPTION_RESOLUTION = "r";
+
+    static
+    {
+        DatabaseDescriptor.toolInitialization();
+
+        Option optLimit = new Option(OPTION_LIMIT, true, "If specified, will only read this number of events " +
+                                                         "from the first file, and up to that time from the others.");
+        optLimit.setArgs(1);
+        options.addOption(optLimit);
+
+        Option optResolution = new Option(OPTION_RESOLUTION, true, "The resolution of the produced" +
+                                                                   "report in milliseconds, 100 by default.");
+        optResolution.setArgs(1);
+        options.addOption(optResolution);
+    }
+
+    /**
+     * A data point represents both an input data point as well as aggregated data for a level or total.
+     */
+    static class DataPoint
+    {
+        String shardId;
+        long timestamp;
+        int bucket;
+        // number of sstables
+        int sstables;
+        // max number of overlapping sstables in bucket
+        int overlap;
+        // total size of the sstables
+        long size;
+        // number of running compactions
+        int compactionsInProgress;
+        // number of compactions to do
+        int compactionsPending;
+        // bytes read per second
+        long readBytesPerSecond;
+        // bytes written per second
+        long writeBytesPerSecond;
+        // total bytes to compact
+        long totalBytes;
+        // remaining bytes to compact
+        long remainingReadBytes;
+        // value of scaling parameter W
+        int scalingParameter;
+
+        /**
+         * Called to aggregate data in response to a new data point for a bucket.
+         * Unless the process is just starting, the new data point will be replacing the older state of the bucket,
+         * thus this will add the new data but also remove the older values.
+         */
+        private void updateTotals(DataPoint toAdd, DataPoint toRemove)
+        {
+            timestamp = toAdd.timestamp;
+            compactionsInProgress += toAdd.compactionsInProgress - toRemove.compactionsInProgress;
+            compactionsPending += toAdd.compactionsPending - toRemove.compactionsPending;
+            sstables += toAdd.sstables - toRemove.sstables;
+            size += toAdd.size - toRemove.size;
+            readBytesPerSecond += toAdd.readBytesPerSecond - toRemove.readBytesPerSecond;
+            writeBytesPerSecond += toAdd.writeBytesPerSecond - toRemove.writeBytesPerSecond;
+            totalBytes += toAdd.totalBytes - toRemove.totalBytes;
+            remainingReadBytes += toAdd.remainingReadBytes - toRemove.remainingReadBytes;
+            scalingParameter = toAdd.scalingParameter;
+            overlap = toAdd.overlap;
+        }
+    }
+
+
+    static final Pattern CSVNamePattern = Pattern.compile("compaction-(\\w+)-([^-]*)-([^-]*)(-([^.]*))?\\.csv");
+    private static final String fullDateFormatter = "yyyy-MM-dd' 'HH:mm:ss.SSS";
+
+    static int reportResolutionInMs;
+
+    // Indexes of the relevant columns in the source CSV, set by initializeIndexes below.
+    static int timestampIndex = -1;
+    static int eventIndex;
+    static int bucketIndex;
+    static int sstablesIndex;
+    static int overlapIndex;
+    static int compactingSstablesIndex;
+    static int sizeIndex;
+    static int compactionsIndex;
+    static int readPerSecIndex;
+    static int writePerSecIndex;
+    static int sizesIndex;
+    static int Windex;
+
+    private static void initializeIndexes(String header)
+    {
+        if (timestampIndex < 0)
+            synchronized (CompactionLogAnalyzer.class) {
+                if (timestampIndex < 0)
+                {
+                    Map<String, Integer> indexMap = new HashMap<>();
+                    String[] headers = header.split(",");
+                    for (int i = 0; i < headers.length; ++i)
+                        indexMap.put(headers[i], i);
+
+                    timestampIndex = indexMap.get("Timestamp");
+                    eventIndex = indexMap.get("Event");
+                    bucketIndex = indexMap.getOrDefault("Level", indexMap.get("Bucket"));
+                    sstablesIndex = indexMap.get("Tot. SSTables");
+                    overlapIndex = indexMap.get("Overlap");
+                    compactingSstablesIndex = indexMap.get("Comp. SSTables");
+                    sizeIndex = indexMap.getOrDefault("Size (bytes)", -1);
+                    sizeIndex = indexMap.get("Tot. size (bytes)");
+                    compactionsIndex = indexMap.get("Compactions");
+                    readPerSecIndex = indexMap.get("Read (bytes/sec)");
+                    writePerSecIndex = indexMap.get("Write (bytes/sec)");
+                    sizesIndex = indexMap.getOrDefault("Tot/Read/Written", -1);
+                    sizesIndex = indexMap.get("Tot. comp. size/Read/Written (bytes)");
+                    Windex = indexMap.get("W");
+                }
+            }
+    }
+
+    static DataPoint parse(String shardId, String dataLine) throws ParseException
+    {
+        String[] data = dataLine.split(",");
+
+        DataPoint dp = new DataPoint();
+        dp.shardId = shardId;
+        dp.timestamp = getTimestamp(data[timestampIndex]);
+        dp.bucket = Integer.parseInt(data[bucketIndex]);
+        dp.sstables = Integer.parseInt(data[sstablesIndex]);
+        dp.size = parseHumanReadableSize(data[sizeIndex]);
+        final String[] compactions = data[compactionsIndex].split("/");
+        dp.compactionsInProgress = Integer.parseInt(compactions[1]);
+        dp.compactionsPending = Integer.parseInt(compactions[0]);
+        dp.readBytesPerSecond = parseHumanReadableRate(data[readPerSecIndex]);
+        dp.writeBytesPerSecond = parseHumanReadableRate(data[writePerSecIndex]);
+        String[] sizes = data[sizesIndex].split("/");
+        dp.totalBytes = parseHumanReadableSize(sizes[0]);
+        dp.remainingReadBytes = dp.totalBytes - parseHumanReadableSize(sizes[1]);
+        dp.scalingParameter = UnifiedCompactionStrategy.parseScalingParameter(data[Windex]);
+        if (overlapIndex >= 0)
+        {
+            dp.overlap = Integer.parseInt(data[overlapIndex]);
+            // Note: This overlap does not include the sstables that are currently compacting. Having such a measure
+            // could be valuable, but it needs processing that the strategy does not do (to improve efficiency the
+            // overlap sets construction only uses non-compacting sstables).
+        }
+        else
+        {
+            // The number of non-compacting sstables in a bucket is the proxy the strategy used for overlapping sstables.
+            int compactingSSTables = Integer.parseInt(data[compactingSstablesIndex].split("/")[1]);
+            dp.overlap = dp.sstables - compactingSSTables;
+        }
+        return dp;
+    }
+
+    private static long getTimestamp(String datum) throws ParseException
+    {
+        Date date = new SimpleDateFormat(fullDateFormatter).parse(datum);
+        return date.getTime();
+    }
+
+    private static long parseHumanReadableSize(String datum)
+    {
+        return FBUtilities.parseHumanReadableBytes(datum);
+    }
+
+    private static long parseHumanReadableRate(String datum)
+    {
+        return (long) FBUtilities.parseHumanReadable(datum, null, "B/s");
+    }
+
+    public static void generateGraph(File htmlFile, JSONObject stats)
+    {
+        try (PrintWriter out = new PrintWriter(htmlFile))
+        {
+            String statsBlock = "/* stats start */\nstats = " + stats.toJSONString() + ";\n/* stats end */\n";
+            String html = getGraphHTML().replaceFirst("/\\* stats start \\*/\n\n/\\* stats end \\*/\n", statsBlock);
+            out.write(html);
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException("Couldn't write stats html.");
+        }
+    }
+
+    private static String getGraphHTML()
+    {
+        try (InputStream graphHTMLRes = CompactionLogAnalyzer.class.getClassLoader().getResourceAsStream("org/apache/cassandra/graph/graph.html"))
+        {
+            return new String(ByteStreams.toByteArray(graphHTMLRes));
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static void main(String[] args) throws Exception
+    {
+        CommandLineParser parser = new PosixParser();
+        try
+        {
+            cmd = parser.parse(options, args);
+        }
+        catch (org.apache.commons.cli.ParseException e1)
+        {
+            System.err.println(e1.getMessage());
+            printUsage();
+            System.exit(1);
+        }
+
+        if (cmd.getArgs().length != 1)
+        {
+            System.err.println("You must supply exactly one log csv path.");
+            printUsage();
+            System.exit(1);
+        }
+
+        File logPath = new File(cmd.getArgs()[0]); // checkstyle: permit this instantiation
+        File[] files = logPath.listFiles(f -> CSVNamePattern.matcher(f.getName()).matches());
+        Arrays.sort(files);
+
+        reportResolutionInMs = Integer.parseInt(cmd.getOptionValue(OPTION_RESOLUTION, "100"));
+
+        final String limitOption = cmd.getOptionValue(OPTION_LIMIT);
+        Integer lineCountLimit = limitOption == null ? null : Integer.parseInt(limitOption);
+
+        List<DataPoint> dataPoints = readDataPoints(files, lineCountLimit);
+        dataPoints.sort((a, b) -> Long.compare(a.timestamp, b.timestamp));
+
+        JSONArray marr = processData(dataPoints);
+        JSONObject main = new JSONObject();
+        main.put("title", "Compaction report");
+        main.put("stats", marr);
+
+        generateGraph(new File(logPath.getPath() + File.separator + "compaction_report.html"), main); // checkstyle: permit this instantiation
+
+        System.exit(0);
+    }
+
+    @VisibleForTesting
+    static List<DataPoint> readDataPoints(File[] files, @Nullable Integer lineCountLimit) throws IOException, ParseException
+    {
+        List<DataPoint> dataPoints;
+
+        if (lineCountLimit != null)
+        {
+            long timestampLimit = Long.MAX_VALUE;
+            dataPoints = new ArrayList<>();
+
+            for (File file : files)
+                timestampLimit = readDataPoints(dataPoints, lineCountLimit, timestampLimit, file);
+        }
+        else
+        {
+            // Reading the files can take a long time. Do it in parallel.
+            dataPoints = Arrays.stream(files)
+                               .parallel()
+                               .flatMap(file ->
+                                    {
+                                        List<DataPoint> pts = new ArrayList<>();
+                                        try
+                                        {
+                                            readDataPoints(pts, Integer.MAX_VALUE, Long.MAX_VALUE, file);
+                                            return pts.stream();
+                                        }
+                                        catch (Exception e)
+                                        {
+                                            throw Throwables.propagate(e);
+                                        }
+                                    })
+                               .collect(Collectors.toList());
+        }
+
+        return dataPoints;
+    }
+
+    private static long readDataPoints(List<DataPoint> dataPoints, int lineCountLimit, long timestampLimit, File file) throws IOException, ParseException
+    {
+        Matcher m = CSVNamePattern.matcher(file.getName());
+        if (!m.matches())
+            throw new AssertionError();
+
+        String shardId = m.group(5);
+        if (shardId == null)
+            shardId = "none";
+        try (BufferedReader rdr = Files.newBufferedReader(file.toPath(), StandardCharsets.UTF_8))
+        {
+            String header = rdr.readLine();
+            initializeIndexes(header);
+            DataPoint curr = null;
+
+            int lineCount = 0;
+
+            while (rdr.ready())
+            {
+                if (++lineCount > lineCountLimit && curr != null)
+                {
+                    timestampLimit = curr.timestamp;
+                    break;
+                }
+
+                String line = rdr.readLine();
+                if (line.isEmpty())
+                    continue;
+
+                try
+                {
+                    curr = parse(shardId, line);
+                }
+                catch (NumberFormatException | ParseException | ArrayIndexOutOfBoundsException e)
+                {
+                    System.out.format("%s parsing line %s, skipping.\n", e.getMessage(), line);
+                    continue;
+                }
+                if (curr.timestamp > timestampLimit)
+                    break;
+                dataPoints.add(curr);
+            }
+            System.out.format("%d data points processed for shard %s.\n", lineCount, shardId);
+        }
+        return timestampLimit;
+    }
+
+    @VisibleForTesting
+    static JSONArray processData(List<DataPoint> dataPoints)
+    {
+        int levels = dataPoints.stream().mapToInt(dp -> dp.bucket).max().getAsInt() + 1;
+
+        // Prepare the JSON objects representing the data in the report
+        JSONArray marr = new JSONArray();
+
+        JSONArray[] intervalsPerLevel = new JSONArray[levels + 1];
+        Table<String, Integer, DataPoint> progressMap = HashBasedTable.create();
+        DataPoint totals = new DataPoint();
+        DataPoint[] perLevel = new DataPoint[levels + 1];
+        perLevel[levels] = totals;
+        DataPoint zero = new DataPoint();
+        totals.shardId = "Total";
+        totals.bucket = levels;
+
+        JSONArray metricsHeader = makeMetricsHeader();
+        for (int i = 0; i < levels; ++i)
+        {
+            perLevel[i] = new DataPoint();
+            perLevel[i].shardId = "Level " + i;
+            perLevel[i].bucket = i;
+        }
+
+
+        for (int i = 0; i <= levels; ++i)
+        {
+            intervalsPerLevel[i] = new JSONArray();
+
+            JSONObject stats = new JSONObject();
+            stats.put("revision", perLevel[i].shardId);
+            stats.put("test", "Compaction");
+            stats.put("metrics", metricsHeader);
+            stats.put("intervals", intervalsPerLevel[i]);
+            marr.add(stats);
+        }
+
+        System.out.println("Totals");
+        System.out.format("%25s %8s %9s %15s %15s %15s %15s\n", "Timestamp", "SSTables", "Run/Pendg", "Read tput", "Write tput", "TotalCompBytes", "RemCompBytes");
+
+        // Process the data points to compile aggregate state and report it with the specified resolution.
+        long startTimestamp = -1;
+        int count = 0;
+        for (DataPoint dp : dataPoints)
+        {
+            // Data points replace previous data for the given bucket. This map is used to find what is replaced.
+            DataPoint prev = progressMap.get(dp.shardId, dp.bucket);
+            if (prev == null)
+                prev = zero;
+
+            if (startTimestamp == -1)
+                startTimestamp = dp.timestamp;
+            else if (dp.timestamp >= totals.timestamp + reportResolutionInMs)
+            {
+                report(intervalsPerLevel, progressMap, perLevel, startTimestamp);
+                ++count;
+            }
+
+            totals.updateTotals(dp, prev);
+            perLevel[dp.bucket].updateTotals(dp, prev);
+            progressMap.put(dp.shardId, dp.bucket, dp);
+        }
+        report(intervalsPerLevel, progressMap, perLevel, startTimestamp);
+        ++count;
+
+        System.out.format("Wrote %d datapoints, spanning %.1f seconds\n", count, (totals.timestamp - startTimestamp) / 1000.0);
+        return marr;
+    }
+
+    private static void report(JSONArray[] intervalsPerLevel,
+                               Table<String, Integer, DataPoint> progressMap,
+                               DataPoint[] perLevel,
+                               long startTimestamp)
+    {
+        // Collect a histogram of the number of sstables per bucket.
+        int levels = perLevel.length - 1;
+
+        int maxOverlap = -1;
+        for (DataPoint bucket : progressMap.values())
+        {
+            maxOverlap = Math.max(maxOverlap, bucket.overlap);
+        }
+        perLevel[levels].overlap = maxOverlap;
+
+        print(perLevel[levels]);    // print out the totals on the console
+        for (int i = 0; i <= levels; ++i)
+            addMetrics(perLevel[i], intervalsPerLevel[i], startTimestamp);
+    }
+
+    private static JSONArray makeMetricsHeader()
+    {
+        JSONArray metrics = new JSONArray();
+        metrics.add("SSTables");
+        metrics.add("Size MB");
+        metrics.add("Running compactions");
+        metrics.add("Pending compactions");
+        metrics.add("Read throughput MB/s");
+        metrics.add("Write throughput MB/s");
+        metrics.add("Read throughput per thread MB/s");
+        metrics.add("Write throughput per thread MB/s");
+        metrics.add("Total GB to compact");
+        metrics.add("Remaining GB to compact");
+        metrics.add("Max overlapping SSTables");
+        metrics.add("Scaling parameter W");
+
+        metrics.add("time");
+        return metrics;
+    }
+
+    private static void addMetrics(DataPoint totals, JSONArray intervals, long startTimestamp)
+    {
+        if (totals.timestamp < startTimestamp)
+            return; // nothing to add yet
+
+        JSONArray metrics = new JSONArray();
+        metrics.add(totals.sstables);
+        metrics.add(Math.scalb(totals.size, -20));
+        metrics.add(totals.compactionsInProgress);
+        metrics.add(totals.compactionsPending);
+        metrics.add(Math.scalb(totals.readBytesPerSecond, -20));
+        metrics.add(Math.scalb(totals.writeBytesPerSecond, -20));
+        if (totals.compactionsInProgress > 0)
+        {
+            long readThroughput = totals.readBytesPerSecond / totals.compactionsInProgress;
+            long writeThroughput = totals.writeBytesPerSecond / totals.compactionsInProgress;
+            metrics.add(Math.scalb(readThroughput, -20));
+            metrics.add(Math.scalb(writeThroughput, -20));
+        }
+        else
+        {
+            metrics.add(null);
+            metrics.add(null);
+        }
+        metrics.add(Math.scalb(totals.totalBytes, -30));
+        metrics.add(Math.scalb(totals.remainingReadBytes, -30));
+
+        metrics.add(totals.overlap);
+
+        metrics.add(totals.scalingParameter);
+
+        metrics.add((totals.timestamp - startTimestamp) / 1000.0);
+        intervals.add(metrics);
+    }
+
+    static void print(DataPoint dp)
+    {
+        System.out.format("%25s %8s %3d/%5d %13s/s %13s/s %15s %15s\n",
+                          new SimpleDateFormat(fullDateFormatter).format(new Date(dp.timestamp)),
+                          dp.sstables,
+                          dp.compactionsInProgress,
+                          dp.compactionsPending,
+                          FBUtilities.prettyPrintMemory(dp.readBytesPerSecond),
+                          FBUtilities.prettyPrintMemory(dp.writeBytesPerSecond),
+                          FBUtilities.prettyPrintMemory(dp.totalBytes),
+                          FBUtilities.prettyPrintMemory(dp.remainingReadBytes));
+    }
+
+    private static void printUsage()
+    {
+        String usage = String.format("analyzecompactionlog <options> <log csvs path>%n");
+        String header = "Perform an analysis of the UCS compaction log.\n\n" +
+                        "The input is a directory that contains the per-shard CSV files generated using the " +
+                        "'logAll: true' flag by the unified compaction strategy.\n" +
+                        "Constructs a compaction_report.html in the target directory with summarized metrics.";
+        new HelpFormatter().printHelp(usage, header, options, "");
+    }
+}
diff --git a/src/java/org/apache/cassandra/tools/JMXTool.java b/src/java/org/apache/cassandra/tools/JMXTool.java
index 8cf5748a2e0f..1096b1a4607e 100644
--- a/src/java/org/apache/cassandra/tools/JMXTool.java
+++ b/src/java/org/apache/cassandra/tools/JMXTool.java
@@ -71,6 +71,7 @@
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.utils.JsonUtils;
+import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.TypeDescription;
 import org.yaml.snakeyaml.Yaml;
 import org.yaml.snakeyaml.constructor.Constructor;
@@ -163,7 +164,7 @@ void dump(OutputStream output, Map<String, Info> map) throws IOException
             {
                 void dump(OutputStream output, Map<String, Info> map) throws IOException
                 {
-                    Representer representer = new Representer();
+                    Representer representer = new Representer(new DumperOptions());
                     representer.addClassTag(Info.class, Tag.MAP); // avoid the auto added tag
                     Yaml yaml = new Yaml(representer);
                     yaml.dump(map, new OutputStreamWriter(output));
@@ -366,6 +367,12 @@ private DiffResult(SetView<T> notInRight, SetView<T> notInLeft, SetView<T> share
             }
         }
 
+        private static final org.yaml.snakeyaml.LoaderOptions LOADER_CONFIG = new org.yaml.snakeyaml.LoaderOptions();
+        {
+            // Set the max yaml file size to 30 mb.
+            LOADER_CONFIG.setCodePointLimit(31_457_280);
+        }
+
         public enum Format
         {
             json
@@ -379,7 +386,8 @@ Map<String, Info> load(InputStream input) throws IOException
             {
                 Map<String, Info> load(InputStream input) throws IOException
                 {
-                    Yaml yaml = new Yaml(new CustomConstructor());
+                    DumperOptions dOpts = new DumperOptions();
+                    Yaml yaml = new Yaml(new CustomConstructor(), new Representer(dOpts), dOpts, LOADER_CONFIG);
                     return (Map<String, Info>) yaml.load(input);
                 }
             };
@@ -394,6 +402,8 @@ private static final class CustomConstructor extends Constructor
 
             public CustomConstructor()
             {
+                super(LOADER_CONFIG);
+
                 this.rootTag = new Tag(ROOT);
                 this.addTypeDescription(INFO_TYPE);
             }
diff --git a/src/java/org/apache/cassandra/tools/JsonTransformer.java b/src/java/org/apache/cassandra/tools/JsonTransformer.java
index 8debfd3b7594..6b4f50f14889 100644
--- a/src/java/org/apache/cassandra/tools/JsonTransformer.java
+++ b/src/java/org/apache/cassandra/tools/JsonTransformer.java
@@ -169,9 +169,9 @@ private void serializePartitionKey(DecoratedKey key)
                 }
 
                 int i = 0;
-                while (keyBytes.remaining() > 0 && i < compositeType.getComponents().size())
+                while (keyBytes.remaining() > 0 && i < compositeType.subTypes().size())
                 {
-                    AbstractType<?> colType = compositeType.getComponents().get(i);
+                    AbstractType<?> colType = compositeType.subTypes().get(i);
 
                     ByteBuffer value = ByteBufferUtil.readBytesWithShortLength(keyBytes);
                     String colValue = colType.getString(value);
diff --git a/src/java/org/apache/cassandra/tools/LoaderOptions.java b/src/java/org/apache/cassandra/tools/LoaderOptions.java
index 03c8ee60244d..2136cbe195ba 100644
--- a/src/java/org/apache/cassandra/tools/LoaderOptions.java
+++ b/src/java/org/apache/cassandra/tools/LoaderOptions.java
@@ -546,8 +546,8 @@ public Builder parseArgs(String cmdArgs[])
                 throttleBytes = config.stream_throughput_outbound.toBytesPerSecondAsInt();
 
                 if (cmd.hasOption(SSL_STORAGE_PORT_OPTION))
-                    logger.info("ssl storage port is deprecated and not used, all communication goes though storage port " +
-                                "which is able to handle encrypted communication too.");
+                    System.out.println("ssl storage port is deprecated and not used, all communication goes through storage port " +
+                            "which is able to handle encrypted communication too.");
 
                 // Copy the encryption options and apply the config so that argument parsing can accesss isEnabled.
                 clientEncOptions = config.client_encryption_options.applyConfig();
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index c60853643272..6b5e15831508 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -121,7 +121,7 @@
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Uninterruptibles;
 import org.apache.cassandra.tools.nodetool.GetTimeout;
-import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.INativeLibrary;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.SSL_ENABLE;
@@ -359,9 +359,9 @@ public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkDa
         return ssProxy.scrub(disableSnapshot, skipCorrupted, checkData, reinsertOverflowedTTL, jobs, keyspaceName, tables);
     }
 
-    public int verify(boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+    public int verify(boolean extendedVerify, boolean validateAllRows, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
     {
-        return ssProxy.verify(extendedVerify, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspaceName, tableNames);
+        return ssProxy.verify(extendedVerify, validateAllRows, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspaceName, tableNames);
     }
 
     public int upgradeSSTables(String keyspaceName, boolean excludeCurrentVersion, long maxSSTableTimestamp, int jobs, String... tableNames) throws IOException, ExecutionException, InterruptedException
@@ -402,10 +402,10 @@ public void scrub(PrintStream out, boolean disableSnapshot, boolean skipCorrupte
                 "scrubbing");
     }
 
-    public void verify(PrintStream out, boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+    public void verify(PrintStream out, boolean extendedVerify, boolean validateAllRows, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
     {
         perform(out, keyspaceName,
-                () -> verify(extendedVerify, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspaceName, tableNames),
+                () -> verify(extendedVerify, validateAllRows, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspaceName, tableNames),
                 "verifying");
     }
 
@@ -464,6 +464,11 @@ public void forceKeyspaceCompaction(boolean splitOutput, String keyspaceName, St
         ssProxy.forceKeyspaceCompaction(splitOutput, keyspaceName, tableNames);
     }
 
+    public void forceKeyspaceCompaction(boolean splitOutput, int parallelism, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+    {
+        ssProxy.forceKeyspaceCompaction(splitOutput, parallelism, keyspaceName, tableNames);
+    }
+
     public void relocateSSTables(int jobs, String keyspace, String[] cfnames) throws IOException, ExecutionException, InterruptedException
     {
         ssProxy.relocateSSTables(jobs, keyspace, cfnames);
@@ -1409,6 +1414,11 @@ public long getCompactionThroughputBytes()
         return ssProxy.getCompactionThroughtputBytesPerSec();
     }
 
+    public Map<String, String> getCurrentCompactionThroughputMiBPerSec()
+    {
+        return ssProxy.getCurrentCompactionThroughputMebibytesPerSec();
+    }
+
     public void setBatchlogReplayThrottle(int value)
     {
         ssProxy.setBatchlogReplayThrottleInKB(value);
@@ -2013,6 +2023,9 @@ public Object getCompactionMetric(String metricName)
                 case "CompletedTasks":
                 case "PendingTasks":
                 case "PendingTasksByTableName":
+                case "WriteAmplificationByTableName":
+                case "AggregateCompactions":
+                case "MaxOverlapsMap":
                     return JMX.newMBeanProxy(mbeanServerConn,
                             new ObjectName("org.apache.cassandra.metrics:type=Compaction,name=" + metricName),
                             CassandraMetricsRegistry.JmxGaugeMBean.class).getValue();
@@ -2177,7 +2190,7 @@ public Map<String, String> getLoggingLevels()
 
     public long getPid()
     {
-        return NativeLibrary.getProcessID();
+        return INativeLibrary.instance.getProcessID();
     }
 
     public void resumeBootstrap(PrintStream out) throws IOException
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index 256c80d26903..276c8d67fb41 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -322,7 +322,7 @@ private void printSStableMetadata(File file, boolean scan) throws IOException
         CompactionMetadata compaction = statsComponent.compactionMetadata();
         SerializationHeader.Component header = statsComponent.serializationHeader();
         Class<? extends ICompressor> compressorClass = null;
-        try (CompressionMetadata compression = CompressionInfoComponent.loadIfExists(descriptor))
+        try (CompressionMetadata compression = CompressionInfoComponent.loadIfExists(descriptor, stats.zeroCopyMetadata))
         {
             compressorClass = compression != null ? compression.compressor().getClass() : null;
         }
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 36fda102e976..b82aed6c158f 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -37,6 +37,7 @@
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.AbstractStrategyHolder;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
 import org.apache.cassandra.db.compaction.CompactionStrategyManager;
 import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
 import org.apache.cassandra.db.compaction.LeveledManifest;
@@ -190,7 +191,7 @@ public static void main(String args[])
             }
 
             // Check (and repair) manifests
-            checkManifest(cfs.getCompactionStrategyManager(), cfs, sstables);
+            checkManifest(cfs, sstables);
             CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
             LifecycleTransaction.waitForDeletions();
             System.exit(0); // We need that to stop non daemonized threads
@@ -204,17 +205,18 @@ public static void main(String args[])
         }
     }
 
-    private static void checkManifest(CompactionStrategyManager strategyManager, ColumnFamilyStore cfs, Collection<SSTableReader> sstables)
+    private static void checkManifest(ColumnFamilyStore cfs, Collection<? extends CompactionSSTable> sstables)
     {
-        if (strategyManager.getCompactionParams().klass().equals(LeveledCompactionStrategy.class))
+        if (cfs.getCompactionParams().klass().equals(LeveledCompactionStrategy.class))
         {
-            int maxSizeInMiB = (int)((cfs.getCompactionStrategyManager().getMaxSSTableBytes()) / (1024L * 1024L));
-            int fanOut = cfs.getCompactionStrategyManager().getLevelFanoutSize();
-            for (AbstractStrategyHolder.GroupedSSTableContainer sstableGroup : strategyManager.groupSSTables(sstables))
+            int maxSizeInMiB = (int)((cfs.getCompactionStrategy().getMaxSSTableBytes()) / (1024L * 1024L));
+            int fanOut = cfs.getCompactionStrategy().getLevelFanoutSize();
+            CompactionStrategyManager csm = (CompactionStrategyManager) cfs.getCompactionStrategyContainer();
+            for (AbstractStrategyHolder.GroupedSSTableContainer<?> sstableGroup : csm.groupSSTables(sstables))
             {
                 for (int i = 0; i < sstableGroup.numGroups(); i++)
                 {
-                    List<SSTableReader> groupSSTables = new ArrayList<>(sstableGroup.getGroup(i));
+                    List<CompactionSSTable> groupSSTables = new ArrayList<>(sstableGroup.getGroup(i));
                     // creating the manifest makes sure the leveling is sane:
                     LeveledManifest.create(cfs, maxSizeInMiB, fanOut, groupSSTables);
                 }
diff --git a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
index 547a1e05f2b2..e41971d3fc59 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
@@ -59,6 +59,7 @@ public class StandaloneVerifier
     private static final String TOOL_NAME = "sstableverify";
     private static final String VERBOSE_OPTION  = "verbose";
     private static final String EXTENDED_OPTION = "extended";
+    private static final String VALIDATE_ALL_ROWS = "validate_all_rows";
     private static final String DEBUG_OPTION  = "debug";
     private static final String HELP_OPTION  = "help";
     private static final String CHECK_VERSION = "check_version";
@@ -125,6 +126,7 @@ public static void main(String args[])
             }
             IVerifier.Options verifyOptions = IVerifier.options().invokeDiskFailurePolicy(false)
                                                        .extendedVerification(options.extended)
+                                                       .validateAllRows(options.validateAllRows)
                                                        .checkVersion(options.checkVersion)
                                                        .mutateRepairStatus(options.mutateRepairStatus)
                                                        .checkOwnsTokens(!options.tokens.isEmpty())
@@ -182,6 +184,7 @@ private static class Options
         public boolean debug;
         public boolean verbose;
         public boolean extended;
+        public boolean validateAllRows;
         public boolean checkVersion;
         public boolean mutateRepairStatus;
         public boolean quick;
@@ -225,6 +228,7 @@ public static Options parseArgs(String cmdArgs[])
                 opts.debug = cmd.hasOption(DEBUG_OPTION);
                 opts.verbose = cmd.hasOption(VERBOSE_OPTION);
                 opts.extended = cmd.hasOption(EXTENDED_OPTION);
+                opts.validateAllRows = cmd.hasOption(VALIDATE_ALL_ROWS);
                 opts.checkVersion = cmd.hasOption(CHECK_VERSION);
                 opts.mutateRepairStatus = cmd.hasOption(MUTATE_REPAIR_STATUS);
                 opts.quick = cmd.hasOption(QUICK);
diff --git a/src/java/org/apache/cassandra/tools/Util.java b/src/java/org/apache/cassandra/tools/Util.java
index d8ef121f89fa..78df1d2afa6f 100644
--- a/src/java/org/apache/cassandra/tools/Util.java
+++ b/src/java/org/apache/cassandra/tools/Util.java
@@ -37,6 +37,7 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.exceptions.ConfigurationException;
@@ -303,13 +304,26 @@ public static <T> Stream<T> iterToStream(Iterator<T> iter)
     }
 
     /**
-     * Construct table schema from info stored in SSTable's Stats.db
+     * Construct table schema from info stored in SSTable's Stats.db.
+     * Hardcodes the keyspace and table name to default values to preserve the existing behavior.
      *
      * @param desc SSTable's descriptor
      * @return Restored CFMetaData
      * @throws IOException when Stats.db cannot be read
      */
     public static TableMetadata metadataFromSSTable(Descriptor desc) throws IOException
+    {
+        return metadataFromSSTable(desc, "keyspace", "table");
+    }
+
+    /**
+     * Construct table schema from info stored in SSTable's Stats.db, using the specified keyspace and table names.
+     *
+     * @param desc SSTable's descriptor
+     * @return Restored CFMetaData
+     * @throws IOException when Stats.db cannot be read
+     */
+    public static TableMetadata metadataFromSSTable(Descriptor desc, String keyspaceName, String tableName) throws IOException
     {
         if (!desc.version.isCompatible())
             throw new IOException("Unsupported SSTable version " + desc.getFormat().name() + "/" + desc.version);
@@ -319,17 +333,17 @@ public static TableMetadata metadataFromSSTable(Descriptor desc) throws IOExcept
 
         IPartitioner partitioner = FBUtilities.newPartitioner(desc);
 
-        TableMetadata.Builder builder = TableMetadata.builder("keyspace", "table").partitioner(partitioner);
+        TableMetadata.Builder builder = TableMetadata.builder(keyspaceName, tableName).partitioner(partitioner);
         header.getStaticColumns().entrySet().stream()
-                .forEach(entry -> {
-                    ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
-                    builder.addStaticColumn(ident, entry.getValue());
-                });
+              .forEach(entry -> {
+                  ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
+                  builder.addStaticColumn(ident, entry.getValue());
+              });
         header.getRegularColumns().entrySet().stream()
-                .forEach(entry -> {
-                    ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
-                    builder.addRegularColumn(ident, entry.getValue());
-                });
+              .forEach(entry -> {
+                  ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
+                  builder.addRegularColumn(ident, entry.getValue());
+              });
         builder.addPartitionKeyColumn("PartitionKey", header.getKeyType());
         for (int i = 0; i < header.getClusteringTypes().size(); i++)
         {
@@ -344,6 +358,8 @@ public static TableMetadata metadataFromSSTable(Descriptor desc) throws IOExcept
             builder.indexes(indexes);
             builder.kind(TableMetadata.Kind.INDEX);
         }
+        boolean isCounter = header.getRegularColumns().values().stream().anyMatch(AbstractType::isCounter) || header.getStaticColumns().values().stream().anyMatch(AbstractType::isCounter);
+        builder.isCounter(isCounter);
         return builder.build();
     }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Compact.java b/src/java/org/apache/cassandra/tools/nodetool/Compact.java
index f5a83ed90475..1f8b4c4b46e6 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Compact.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Compact.java
@@ -50,6 +50,12 @@ public class Compact extends NodeToolCmd
     @Option(title = "partition_key", name = {"--partition"}, description = "String representation of the partition key")
     private String partitionKey = EMPTY;
 
+    @Option(title = "jobs",
+            name = {"-j", "--jobs"},
+            description = "Use -j to specify the maximum number of threads to use for parallel compaction. " +
+                          "If not set, up to half the compaction threads will be used. " +
+                          "If set to 0, the major compaction will use all threads and will not permit other compactions to run until it completes (use with caution).")
+    private Integer parallelism = null;
 
     @Override
     public void execute(NodeProbe probe)
@@ -95,7 +101,10 @@ else if (partitionKeyProvided)
                 }
                 else
                 {
-                    probe.forceKeyspaceCompaction(splitOutput, keyspace, tableNames);
+                    if (parallelism != null)
+                        probe.forceKeyspaceCompaction(splitOutput, parallelism, keyspace, tableNames);
+                    else // avoid referring to the new method to work with older versions
+                        probe.forceKeyspaceCompaction(splitOutput, keyspace, tableNames);
                 }
             } catch (Exception e)
             {
diff --git a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
index c80de91d97d1..7a67e2b5b721 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
@@ -20,15 +20,22 @@
 import java.io.PrintStream;
 import java.text.DecimalFormat;
 import java.text.NumberFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Meter;
+import io.airlift.airline.Arguments;
 import io.airlift.airline.Command;
 import io.airlift.airline.Option;
 
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
+import org.apache.cassandra.db.compaction.CompactionStrategyStatistics;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.tools.NodeProbe;
@@ -50,6 +57,22 @@ public class CompactionStats extends NodeToolCmd
             description = "Display fields matching vtable output")
     private boolean vtableOutput = false;
 
+    @Option(title = "aggregate",
+    name = {"-A", "--aggregate"},
+    description = "Show the compaction aggregates for the compactions in progress, e.g. the levels for LCS or the buckets for STCS and TWCS.")
+    private boolean aggregate = false;
+
+    @Option(title = "overlap",
+    name = {"-O", "--overlap"},
+    description = "Show a map of the maximum sstable overlap per compaction region.\n" +
+                  "Note: This map includes all sstables in the system, including ones that are currently being compacted, " +
+                  "and also takes into account early opened sstables. Overlaps per level may be greater than the values " +
+                  "the --aggregate option reports.")
+    private boolean overlap = false;
+
+    @Arguments(usage = "[<keyspace> <tables>...]", description = "With --aggregate or --overlap, optionally list only the data for the specified keyspace and tables.")
+    private List<String> args = new ArrayList<>();
+
     @Override
     public void execute(NodeProbe probe)
     {
@@ -58,56 +81,132 @@ public void execute(NodeProbe probe)
         pendingTasksAndConcurrentCompactorsStats(probe, tableBuilder);
         compactionsStats(probe, tableBuilder);
         reportCompactionTable(probe.getCompactionManagerProxy().getCompactions(), probe.getCompactionThroughputBytes(), humanReadable, vtableOutput, out, tableBuilder);
+
+        Set<String> keyspaces = new HashSet<>(parseOptionalKeyspace(args, probe));
+        Set<String> tableNames = new HashSet<>(Arrays.asList(parseOptionalTables(args)));
+
+        if (aggregate)
+        {
+            reportAggregateCompactions(probe, keyspaces, tableNames, out);
+        }
+
+        if (overlap)
+            reportOverlap((Map<String, Map<String, Map<String, String>>>) probe.getCompactionMetric("MaxOverlapsMap"), keyspaces, tableNames, out);
     }
 
     private void pendingTasksAndConcurrentCompactorsStats(NodeProbe probe, TableBuilder tableBuilder)
     {
         Map<String, Map<String, Integer>> pendingTaskNumberByTable =
-        (Map<String, Map<String, Integer>>) probe.getCompactionMetric("PendingTasksByTableName");
+            (Map<String, Map<String, Integer>>) probe.getCompactionMetric("PendingTasksByTableName");
+        Map<String, Map<String, Double>> writeAmplificationByTableName =
+            (Map<String, Map<String, Double>>) probe.getCompactionMetric("WriteAmplificationByTableName");
 
         tableBuilder.add("concurrent compactors", Integer.toString(probe.getConcurrentCompactors()));
-        tableBuilder.add("pending tasks", Integer.toString(numPendingTasks(pendingTaskNumberByTable)));
-
-        for (Entry<String, Map<String, Integer>> ksEntry : pendingTaskNumberByTable.entrySet())
-            for (Entry<String, Integer> tableEntry : ksEntry.getValue().entrySet())
-                tableBuilder.add(ksEntry.getKey(), tableEntry.getKey(), tableEntry.getValue().toString());
-    }
-
-    private int numPendingTasks(Map<String, Map<String, Integer>> pendingTaskNumberByTable)
-    {
         int numTotalPendingTasks = 0;
+        double totWriteAmplification = 0;
         for (Entry<String, Map<String, Integer>> ksEntry : pendingTaskNumberByTable.entrySet())
+        {
+            Map<String, Double> ksWriteAmplification = writeAmplificationByTableName.get(ksEntry.getKey());
             for (Entry<String, Integer> tableEntry : ksEntry.getValue().entrySet())
+            {
                 numTotalPendingTasks += tableEntry.getValue();
+                if (ksWriteAmplification != null)
+                    totWriteAmplification += ksWriteAmplification.get(tableEntry.getKey());
+            }
+        }
+        tableBuilder.add("pending tasks", Integer.toString(numTotalPendingTasks));
+        tableBuilder.add("write amplification", String.format("%.2f", totWriteAmplification));
 
-        return numTotalPendingTasks;
+        for (Entry<String, Map<String, Integer>> ksEntry : pendingTaskNumberByTable.entrySet())
+        {
+            Map<String, Double> ksWriteAmplification = writeAmplificationByTableName.get(ksEntry.getKey());
+            for (Entry<String, Integer> tableEntry : ksEntry.getValue().entrySet())
+            {
+                double wa = ksWriteAmplification == null ? 0 : ksWriteAmplification.get(tableEntry.getKey());
+                tableBuilder.add(ksEntry.getKey(), tableEntry.getKey(), tableEntry.getValue().toString());
+                tableBuilder.add(ksEntry.getKey(), String.format("%s write amplification", tableEntry.getKey()), String.format("%.2f", wa));
+            }
+        }
     }
 
     private void compactionsStats(NodeProbe probe, TableBuilder tableBuilder)
     {
-        CassandraMetricsRegistry.JmxMeterMBean totalCompactionsCompletedMetrics =
-        (CassandraMetricsRegistry.JmxMeterMBean) probe.getCompactionMetric("TotalCompactionsCompleted");
-        tableBuilder.add("compactions completed", String.valueOf(totalCompactionsCompletedMetrics.getCount()));
+        // FIXME this is a hack to get the compaction metrics for NodeToolTest using InternalNodeProbe without JMX
+        Object totalCompactionsCompleted = probe.getCompactionMetric("TotalCompactionsCompleted");
+        double totalCompactionsCompletedFifteenMinuteRate;
+        double totalCompactionsCompletedMeanRate;
+        if (totalCompactionsCompleted instanceof Meter)
+        {
+            Meter totalCompactionsCompletedMeter = (Meter) totalCompactionsCompleted;
+            tableBuilder.add("compactions completed", String.valueOf(totalCompactionsCompletedMeter.getCount()));
+            totalCompactionsCompletedFifteenMinuteRate = totalCompactionsCompletedMeter.getFifteenMinuteRate();
+            totalCompactionsCompletedMeanRate = totalCompactionsCompletedMeter.getMeanRate();
+        }
+        else
+        {
+            CassandraMetricsRegistry.JmxMeterMBean totalCompactionsCompletedJmxMeterMBean = (CassandraMetricsRegistry.JmxMeterMBean) totalCompactionsCompleted;
+            tableBuilder.add("compactions completed", String.valueOf(totalCompactionsCompletedJmxMeterMBean.getCount()));
+            totalCompactionsCompletedFifteenMinuteRate = totalCompactionsCompletedJmxMeterMBean.getFifteenMinuteRate();
+            totalCompactionsCompletedMeanRate = totalCompactionsCompletedJmxMeterMBean.getMeanRate();
+        }
 
-        CassandraMetricsRegistry.JmxCounterMBean bytesCompacted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("BytesCompacted");
+        Object bytesCompacted = probe.getCompactionMetric("BytesCompacted");
+        long bytesCompactedCount;
+        if (bytesCompacted instanceof Counter)
+        {
+            Counter bytesCompactedCounter = (Counter) bytesCompacted;
+            bytesCompactedCount = bytesCompactedCounter.getCount();
+        }
+        else
+        {
+            CassandraMetricsRegistry.JmxCounterMBean bytesCompactedJmxCounterMBean = (CassandraMetricsRegistry.JmxCounterMBean) bytesCompacted;
+            bytesCompactedCount = bytesCompactedJmxCounterMBean.getCount();
+        }
         if (humanReadable)
-            tableBuilder.add("data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(bytesCompacted.getCount()))));
+            tableBuilder.add("data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(bytesCompactedCount))));
         else
-            tableBuilder.add("data compacted", Long.toString(bytesCompacted.getCount()));
+            tableBuilder.add("data compacted", Long.toString(bytesCompactedCount));
 
-        CassandraMetricsRegistry.JmxCounterMBean compactionsAborted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompactionsAborted");
-        tableBuilder.add("compactions aborted", Long.toString(compactionsAborted.getCount()));
+        Object compactionsAborted = probe.getCompactionMetric("CompactionsAborted");
+        if (compactionsAborted instanceof Counter)
+        {
+            Counter compactionsAbortedCounter = (Counter) compactionsAborted;
+            tableBuilder.add("compactions aborted", Long.toString(compactionsAbortedCounter.getCount()));
+        }
+        else
+        {
+            CassandraMetricsRegistry.JmxCounterMBean compactionsAbortedJmxCounterMBean = (CassandraMetricsRegistry.JmxCounterMBean) compactionsAborted;
+            tableBuilder.add("compactions aborted", Long.toString(compactionsAbortedJmxCounterMBean.getCount()));
+        }
 
-        CassandraMetricsRegistry.JmxCounterMBean compactionsReduced = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompactionsReduced");
-        tableBuilder.add("compactions reduced", Long.toString(compactionsReduced.getCount()));
+        Object compactionsReduced = probe.getCompactionMetric("CompactionsReduced");
+        if (compactionsReduced instanceof Counter)
+        {
+            Counter compactionsReducedCounter = (Counter) compactionsReduced;
+            tableBuilder.add("compactions reduced", Long.toString(compactionsReducedCounter.getCount()));
+        }
+        else
+        {
+            CassandraMetricsRegistry.JmxCounterMBean compactionsReducedJmxCounterMBean = (CassandraMetricsRegistry.JmxCounterMBean) compactionsReduced;
+            tableBuilder.add("compactions reduced", Long.toString(compactionsReducedJmxCounterMBean.getCount()));
+        }
 
-        CassandraMetricsRegistry.JmxCounterMBean sstablesDroppedFromCompaction = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("SSTablesDroppedFromCompaction");
-        tableBuilder.add("sstables dropped from compaction", Long.toString(sstablesDroppedFromCompaction.getCount()));
+        Object sstablesDroppedFromCompaction = probe.getCompactionMetric("SSTablesDroppedFromCompaction");
+        if (sstablesDroppedFromCompaction instanceof Counter)
+        {
+            Counter sstablesDroppedFromCompactionCounter = (Counter) sstablesDroppedFromCompaction;
+            tableBuilder.add("sstables dropped from compaction", Long.toString(sstablesDroppedFromCompactionCounter.getCount()));
+        }
+        else
+        {
+            CassandraMetricsRegistry.JmxCounterMBean sstablesDroppedFromCompactionJmxCounterMBean = (CassandraMetricsRegistry.JmxCounterMBean) sstablesDroppedFromCompaction;
+            tableBuilder.add("sstables dropped from compaction", Long.toString(sstablesDroppedFromCompactionJmxCounterMBean.getCount()));
+        }
 
         NumberFormat formatter = new DecimalFormat("0.00");
 
-        tableBuilder.add("15 minute rate", String.format("%s/minute", formatter.format(totalCompactionsCompletedMetrics.getFifteenMinuteRate() * 60)));
-        tableBuilder.add("mean rate", String.format("%s/hour", formatter.format(totalCompactionsCompletedMetrics.getMeanRate() * 60 * 60)));
+        tableBuilder.add("15 minute rate", String.format("%s/minute", formatter.format(totalCompactionsCompletedFifteenMinuteRate * 60)));
+        tableBuilder.add("mean rate", String.format("%s/hour", formatter.format(totalCompactionsCompletedMeanRate * 60 * 60)));
 
         double configured = probe.getStorageService().getCompactionThroughtputMibPerSecAsDouble();
         tableBuilder.add("compaction throughput (MiB/s)", configured == 0 ? "throttling disabled (0)" : Double.toString(configured));
@@ -135,21 +234,21 @@ public static void reportCompactionTable(List<Map<String,String>> compactions, l
 
         for (Map<String, String> c : compactions)
         {
-            long total = Long.parseLong(c.get(CompactionInfo.TOTAL));
-            long completed = Long.parseLong(c.get(CompactionInfo.COMPLETED));
-            String taskType = c.get(CompactionInfo.TASK_TYPE);
-            String keyspace = c.get(CompactionInfo.KEYSPACE);
-            String columnFamily = c.get(CompactionInfo.COLUMNFAMILY);
-            String unit = c.get(CompactionInfo.UNIT);
-            boolean toFileSize = humanReadable && Unit.isFileSize(unit);
-            String[] tables = c.get(CompactionInfo.SSTABLES).split(",");
+            long total = Long.parseLong(c.get(TableOperation.Progress.TOTAL));
+            long completed = Long.parseLong(c.get(TableOperation.Progress.COMPLETED));
+            String taskType = c.get(TableOperation.Progress.OPERATION_TYPE);
+            String keyspace = c.get(TableOperation.Progress.KEYSPACE);
+            String columnFamily = c.get(TableOperation.Progress.COLUMNFAMILY);
+            String unit = c.get(TableOperation.Progress.UNIT);
+            boolean toFileSize = humanReadable && TableOperation.Unit.isFileSize(unit);
+            String[] tables = c.get(TableOperation.Progress.SSTABLES).split(",");
             String progressStr = toFileSize ? FileUtils.stringifyFileSize(completed) : Long.toString(completed);
             String totalStr = toFileSize ? FileUtils.stringifyFileSize(total) : Long.toString(total);
             String percentComplete = total == 0 ? "n/a" : new DecimalFormat("0.00").format((double) completed / total * 100) + '%';
-            String id = c.get(CompactionInfo.COMPACTION_ID);
+            String id = c.get(TableOperation.Progress.OPERATION_ID);
             if (vtableOutput)
             {
-                String targetDirectory = c.get(CompactionInfo.TARGET_DIRECTORY);
+                String targetDirectory = c.get(TableOperation.Progress.TARGET_DIRECTORY);
                 table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, unit, targetDirectory);
             }
             else
@@ -169,4 +268,47 @@ public static void reportCompactionTable(List<Map<String,String>> compactions, l
         table.printTo(out);
     }
 
+    private static void reportAggregateCompactions(NodeProbe probe, Set<String> keyspaces, Set<String> tableNames, PrintStream out)
+    {
+        List<CompactionStrategyStatistics> statistics = (List<CompactionStrategyStatistics>) probe.getCompactionMetric("AggregateCompactions");
+        if (statistics.isEmpty())
+            return;
+
+        out.println("Aggregated view:");
+        for (CompactionStrategyStatistics stat : statistics)
+        {
+            if (!keyspaces.contains(stat.keyspace()))
+                continue;
+            if (!tableNames.isEmpty() && !tableNames.contains(stat.table()))
+                continue;
+            out.println(stat.toString());
+        }
+    }
+
+    private static void reportOverlap(Map<String, Map<String, Map<String, String>>> maxOverlap, Set<String> keyspaces, Set<String> tableNames, PrintStream out)
+    {
+        if (maxOverlap == null)
+        {
+            out.println("Overlap map is not available.");
+            return;
+        }
+
+        for (Map.Entry<String, Map<String, Map<String, String>>> ksEntry : maxOverlap.entrySet())
+        {
+            String ksName = ksEntry.getKey();
+            if (!keyspaces.contains(ksName))
+                continue;
+            for (Map.Entry<String, Map<String, String>> tableEntry : ksEntry.getValue().entrySet())
+            {
+                String tableName = tableEntry.getKey();
+                if (!tableNames.isEmpty() && !tableNames.contains(tableName))
+                    continue;
+                out.println("Max overlap map for " + ksName + "." + tableName + ":");
+                for (Map.Entry<String, String> compactionEntry : tableEntry.getValue().entrySet())
+                {
+                    out.println("  " + compactionEntry.getKey() + ": " + compactionEntry.getValue());
+                }
+            }
+        }
+    }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThroughput.java b/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThroughput.java
index e71fe0adef3e..cc917c0c3117 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThroughput.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetCompactionThroughput.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.tools.nodetool;
 
 import com.google.common.math.DoubleMath;
+import java.util.Map;
 
 import io.airlift.airline.Command;
 
@@ -45,6 +46,11 @@ public void execute(NodeProbe probe)
                 throw new RuntimeException("Use the -d flag to quiet this error and get the exact throughput in MiB/s");
 
             probe.output().out.println("Current compaction throughput: " + probe.getCompactionThroughput() + " MB/s");
+
+            Map<String, String> currentCompactionThroughputMetricsMap = probe.getCurrentCompactionThroughputMiBPerSec();
+            probe.output().out.println("Current compaction throughput (1 minute): " + currentCompactionThroughputMetricsMap.get("1minute") + " MiB/s");
+            probe.output().out.println("Current compaction throughput (5 minute): " + currentCompactionThroughputMetricsMap.get("5minute") + " MiB/s");
+            probe.output().out.println("Current compaction throughput (15 minute): " + currentCompactionThroughputMetricsMap.get("15minute") + " MiB/s");
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Repair.java b/src/java/org/apache/cassandra/tools/nodetool/Repair.java
index 35832408301c..7e96fccfc365 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Repair.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Repair.java
@@ -93,6 +93,9 @@ public class Repair extends NodeToolCmd
     @Option(title = "trace_repair", name = {"-tr", "--trace"}, description = "Use -tr to trace the repair. Traces are logged to system_traces.events.")
     private boolean trace = false;
 
+    @Option(title = "push_repair", name = {"-ps", "--push"}, description = "Use --push to perform a one way repair where data is only streamed from local node to remote node.")
+    private boolean pushRepair = false;
+
     @Option(title = "pull_repair", name = {"-pl", "--pull"}, description = "Use --pull to perform a one way repair where data is only streamed from a remote node to this node.")
     private boolean pullRepair = false;
 
@@ -179,6 +182,7 @@ else if (dcParallel)
         options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(numJobThreads));
         options.put(RepairOption.TRACE_KEY, Boolean.toString(trace));
         options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(cfnames, ","));
+        options.put(RepairOption.PUSH_REPAIR_KEY, Boolean.toString(pushRepair));
         options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(pullRepair));
         options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force));
         options.put(RepairOption.PREVIEW, getPreviewKind().toString());
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Sjk.java b/src/java/org/apache/cassandra/tools/nodetool/Sjk.java
index d7f7a043f606..dab03c4cb385 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Sjk.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Sjk.java
@@ -25,8 +25,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Verify.java b/src/java/org/apache/cassandra/tools/nodetool/Verify.java
index 0a610b3266a0..7fa126b37c85 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Verify.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Verify.java
@@ -36,9 +36,14 @@ public class Verify extends NodeToolCmd
 
     @Option(title = "extended_verify",
             name = {"-e", "--extended-verify"},
-            description = "Verify each cell data, beyond simply checking sstable checksums")
+            description = "Verify each partition data, beyond simply checking sstable checksums")
     private boolean extendedVerify = false;
 
+    @Option(title = "validate_all_rows",
+    name = {"-v", "--validate-all-rows"},
+    description = "Verify each row and cell data in the partition, beyond checking partition key. Must be enabled with extended verification")
+    private boolean validateAllRows = false;
+
     @Option(title = "check_version",
             name = {"-c", "--check-version"},
             description = "Also check that all sstables are the latest version")
@@ -93,7 +98,7 @@ public void execute(NodeProbe probe)
         {
             try
             {
-                probe.verify(out, extendedVerify, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspace, tableNames);
+                probe.verify(out, extendedVerify, validateAllRows, checkVersion, diskFailurePolicy, mutateRepairStatus, checkOwnsTokens, quick, keyspace, tableNames);
             } catch (Exception e)
             {
                 throw new RuntimeException("Error occurred during verifying", e);
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
index af5190e647ce..015bfae75a89 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
@@ -234,7 +234,7 @@ private void initializeKeyspaces(NodeProbe probe, boolean ignore, List<String> t
                 statsTable.maxSSTableSize = sstableSize == null ? 0 : sstableSize;
 
                 int[] leveledSStables = table.getSSTableCountPerLevel();
-                if (leveledSStables != null)
+                if (leveledSStables.length > 0)
                 {
                     statsTable.isLeveledSstable = true;
 
diff --git a/src/java/org/apache/cassandra/tracing/ExpiredTraceState.java b/src/java/org/apache/cassandra/tracing/ExpiredTraceState.java
index bf9508020191..e7dfd3974015 100644
--- a/src/java/org/apache/cassandra/tracing/ExpiredTraceState.java
+++ b/src/java/org/apache/cassandra/tracing/ExpiredTraceState.java
@@ -29,7 +29,7 @@ class ExpiredTraceState extends TraceState
 
     ExpiredTraceState(TraceState delegate)
     {
-        super(FBUtilities.getBroadcastAddressAndPort(), delegate.sessionId, delegate.traceType);
+        super(delegate.clientState, FBUtilities.getBroadcastAddressAndPort(), delegate.sessionId, delegate.traceType);
         this.delegate = delegate;
     }
 
diff --git a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java
index fb92f4dac59a..f46f3c46f369 100644
--- a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java
+++ b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java
@@ -109,7 +109,9 @@ private static TableMetadata parse(String table, String description, String cql)
 
     public static KeyspaceMetadata metadata()
     {
-        return KeyspaceMetadata.create(SchemaConstants.TRACE_KEYSPACE_NAME, KeyspaceParams.simple(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())), Tables.of(Sessions, Events));
+        return KeyspaceMetadata.create(SchemaConstants.TRACE_KEYSPACE_NAME, 
+                                       KeyspaceParams.systemDistributed(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())), 
+                                       Tables.of(Sessions, Events));
     }
 
     static Mutation makeStartSessionMutation(ByteBuffer sessionId,
diff --git a/src/java/org/apache/cassandra/tracing/TraceState.java b/src/java/org/apache/cassandra/tracing/TraceState.java
index 17133698e830..5c4a6d54d125 100644
--- a/src/java/org/apache/cassandra/tracing/TraceState.java
+++ b/src/java/org/apache/cassandra/tracing/TraceState.java
@@ -22,6 +22,7 @@
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.Nullable;
 
 import com.google.common.base.Stopwatch;
 import org.slf4j.helpers.MessageFormatter;
@@ -29,6 +30,8 @@
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.TracingClientState;
 import org.apache.cassandra.utils.progress.ProgressEvent;
 import org.apache.cassandra.utils.progress.ProgressEventNotifier;
 import org.apache.cassandra.utils.progress.ProgressListener;
@@ -45,6 +48,10 @@ public abstract class TraceState implements ProgressEventNotifier
     public final ByteBuffer sessionIdBytes;
     public final Tracing.TraceType traceType;
     public final int ttl;
+    public final ClientState clientState;
+
+    private boolean rangeQuery;
+    private String tracedKeyspace;
 
     private boolean notify;
     private final List<ProgressListener> listeners = new CopyOnWriteArrayList<>();
@@ -63,11 +70,12 @@ public enum Status
     // See CASSANDRA-7626 for more details.
     private final AtomicInteger references = new AtomicInteger(1);
 
-    protected TraceState(InetAddressAndPort coordinator, TimeUUID sessionId, Tracing.TraceType traceType)
+    protected TraceState(ClientState clientState, InetAddressAndPort coordinator, TimeUUID sessionId, Tracing.TraceType traceType)
     {
         assert coordinator != null;
         assert sessionId != null;
 
+        this.clientState = clientState;
         this.coordinator = coordinator;
         this.sessionId = sessionId;
         sessionIdBytes = sessionId.toBytes();
@@ -103,6 +111,34 @@ public void removeProgressListener(ProgressListener listener)
         listeners.remove(listener);
     }
 
+    public boolean isRangeQuery()
+    {
+        return rangeQuery;
+    }
+
+    public void setRangeQuery(boolean rangeQuery)
+    {
+        this.rangeQuery = rangeQuery;
+    }
+
+    /**
+     * @return the keyspace being traced.
+     */
+    public @Nullable String tracedKeyspace()
+    {
+        if (clientState instanceof TracingClientState)
+            return ((TracingClientState) clientState).tracedKeyspace();
+        return tracedKeyspace;
+    }
+
+    /**
+     * @param tracedKeyspace the keyspace being traced.
+     */
+    public void tracedKeyspace(String tracedKeyspace)
+    {
+        this.tracedKeyspace = tracedKeyspace;
+    }
+
     public int elapsed()
     {
         long elapsed = watch.elapsed(TimeUnit.MICROSECONDS);
diff --git a/src/java/org/apache/cassandra/tracing/TraceStateImpl.java b/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
index edc2cb796a86..2232ec76e834 100644
--- a/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
+++ b/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
@@ -32,6 +32,9 @@
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.exceptions.OverloadedException;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -55,9 +58,9 @@ public class TraceStateImpl extends TraceState
 
     private final Set<Future<?>> pendingFutures = ConcurrentHashMap.newKeySet();
 
-    public TraceStateImpl(InetAddressAndPort coordinator, TimeUUID sessionId, Tracing.TraceType traceType)
+    public TraceStateImpl(ClientState state, InetAddressAndPort coordinator, TimeUUID sessionId, Tracing.TraceType traceType)
     {
-        super(coordinator, sessionId, traceType);
+        super(state, coordinator, sessionId, traceType);
     }
 
     protected void traceImpl(String message)
@@ -103,17 +106,18 @@ protected void waitForPendingEvents()
 
     void executeMutation(final Mutation mutation)
     {
-        Future<Void> fut = Stage.TRACING.executor().submit(() -> mutateWithCatch(mutation), null);
+        Future<Void> fut = Stage.TRACING.submit(() -> mutateWithCatch(clientState, mutation), null);
         boolean ret = pendingFutures.add(fut);
         if (!ret)
             logger.warn("Failed to insert pending future, tracing synchronization may not work");
     }
 
-    static void mutateWithCatch(Mutation mutation)
+    static void mutateWithCatch(ClientState state, Mutation mutation)
     {
         try
         {
-            StorageProxy.mutate(singletonList(mutation), ANY, Dispatcher.RequestTime.forImmediateExecution());
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(mutation.getKeyspaceName());
+            StorageProxy.mutate(singletonList(mutation), ANY, Dispatcher.RequestTime.forImmediateExecution(), metrics, state);
         }
         catch (OverloadedException e)
         {
diff --git a/src/java/org/apache/cassandra/tracing/Tracing.java b/src/java/org/apache/cassandra/tracing/Tracing.java
index f1c5b54b94f1..a06d837663c2 100644
--- a/src/java/org/apache/cassandra/tracing/Tracing.java
+++ b/src/java/org/apache/cassandra/tracing/Tracing.java
@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -32,12 +33,17 @@
 
 import org.apache.cassandra.concurrent.ExecutorLocals;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.ParamType;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.TracingClientState;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.TimeUUID;
@@ -69,6 +75,12 @@ public long serializedSize(TraceType traceType, int version)
         }
     };
 
+    public static void logAndTrace(Logger logger, String message, Object... args)
+    {
+        logger.trace(message, args);
+        trace(message, args);
+    }
+
     /* this enum is used in serialization; preserve order for compatibility */
     public enum TraceType
     {
@@ -102,8 +114,6 @@ public int getTTL()
 
     protected static final Logger logger = LoggerFactory.getLogger(Tracing.class);
 
-    private final InetAddressAndPort localAddress = FBUtilities.getLocalAddressAndPort();
-
     protected final ConcurrentMap<TimeUUID, TraceState> sessions = new ConcurrentHashMap<>();
 
     public static final Tracing instance;
@@ -126,7 +136,7 @@ public int getTTL()
                 logger.error(String.format("Cannot use class %s for tracing, ignoring by defaulting to normal tracing", customTracingClass), e);
             }
         }
-        instance = null != tracing ? tracing : new TracingImpl();
+        instance = tracing == null ? new TracingImpl() : tracing;
     }
 
     public TimeUUID getSessionId()
@@ -147,6 +157,41 @@ public int getTTL()
         return get().ttl;
     }
 
+    public static boolean traceSinglePartitions()
+    {
+        return instance.get() != null && !instance.get().isRangeQuery();
+    }
+
+    public void setRangeQuery(boolean rangeQuery)
+    {
+        assert isTracing();
+        get().setRangeQuery(rangeQuery);
+    }
+
+    /**
+     * set traced keyspace into trace state which is later used to for billing to track source tenant at replicas.
+     */
+    public static void setupTracedKeyspace(CQLStatement statement)
+    {
+        if (!Tracing.isTracing())
+            return;
+
+        String keyspace = null;
+        if (statement instanceof CQLStatement.SingleKeyspaceCqlStatement)
+        {
+            keyspace = ((CQLStatement.SingleKeyspaceCqlStatement) statement).keyspace();
+        }
+
+        if (keyspace == null && statement instanceof BatchStatement)
+        {
+            // for batch statement, just pick any keyspace, as it's only used to extract tenant id in BillingQueryInfoTracker
+            List<ModificationStatement> batches = ((BatchStatement) statement).getStatements();
+            if (batches.size() > 0)
+                keyspace = batches.get(0).keyspace();
+        }
+        Tracing.instance.get().tracedKeyspace(keyspace);
+    }
+
     /**
      * Indicates if the current thread's execution is being traced.
      */
@@ -155,33 +200,35 @@ public static boolean isTracing()
         return instance.get() != null;
     }
 
-    public TimeUUID newSession(Map<String,ByteBuffer> customPayload)
+    public TimeUUID newSession(ClientState state, Map<String,ByteBuffer> customPayload)
     {
         return newSession(
+                state,
                 nextTimeUUID(),
                 TraceType.QUERY,
                 customPayload);
     }
 
-    public TimeUUID newSession(TraceType traceType)
+    public TimeUUID newSession(ClientState state, TraceType traceType)
     {
         return newSession(
+                state,
                 nextTimeUUID(),
                 traceType,
                 Collections.EMPTY_MAP);
     }
 
-    public TimeUUID newSession(TimeUUID sessionId, Map<String,ByteBuffer> customPayload)
+    public TimeUUID newSession(ClientState state, TimeUUID sessionId, Map<String,ByteBuffer> customPayload)
     {
-        return newSession(sessionId, TraceType.QUERY, customPayload);
+        return newSession(state, sessionId, TraceType.QUERY, customPayload);
     }
 
     /** This method is intended to be overridden in tracing implementations that need access to the customPayload */
-    protected TimeUUID newSession(TimeUUID sessionId, TraceType traceType, Map<String,ByteBuffer> customPayload)
+    protected TimeUUID newSession(ClientState state, TimeUUID sessionId, TraceType traceType, Map<String,ByteBuffer> customPayload)
     {
         assert get() == null;
 
-        TraceState ts = newTraceState(localAddress, sessionId, traceType);
+        TraceState ts = newTraceState(state, FBUtilities.getLocalAddressAndPort(), sessionId, traceType);
         set(ts);
         sessions.put(sessionId, ts);
 
@@ -230,7 +277,7 @@ public TraceState get(TimeUUID sessionId)
     public void set(TraceState tls)
     {
         ExecutorLocals current = ExecutorLocals.current();
-        ExecutorLocals.Impl.set(tls, current.clientWarnState);
+        ExecutorLocals.Impl.set(tls, current.clientWarnState, current.sensors, current.operationContext);
     }
 
     public TraceState begin(final String request, final Map<String, String> parameters)
@@ -257,14 +304,16 @@ public TraceState initializeFromMessage(final Message.Header header)
 
         TraceType traceType = header.traceType();
 
+        ClientState clientState = TracingClientState.withTracedKeyspace(header.traceKeyspace());
+        ts = newTraceState(clientState, header.from, sessionId, traceType);
+
         if (header.verb.isResponse())
         {
             // received a message for a session we've already closed out.  see CASSANDRA-5668
-            return new ExpiredTraceState(newTraceState(header.from, sessionId, traceType));
+            return new ExpiredTraceState(ts);
         }
         else
         {
-            ts = newTraceState(header.from, sessionId, traceType);
             sessions.put(sessionId, ts);
             return ts;
         }
@@ -288,7 +337,9 @@ public void traceOutgoingMessage(Message<?> message, int serializedSize, InetAdd
             if (state == null) // session may have already finished; see CASSANDRA-5668
             {
                 TraceType traceType = message.traceType();
-                trace(sessionId.toBytes(), logMessage, traceType.getTTL());
+                String traceKeyspace = message.header.traceKeyspace();
+                ClientState clientState = TracingClientState.withTracedKeyspace(traceKeyspace);
+                trace(clientState, sessionId.toBytes(), logMessage, traceType.getTTL());
             }
             else
             {
@@ -309,10 +360,19 @@ public Map<ParamType, Object> addTraceHeaders(Map<ParamType, Object> addToMutabl
 
         addToMutable.put(ParamType.TRACE_SESSION, Tracing.instance.getSessionId());
         addToMutable.put(ParamType.TRACE_TYPE, Tracing.instance.getTraceType());
+        String keyspace = Tracing.instance.get().tracedKeyspace();
+        if (keyspace != null)
+        {
+            addToMutable.put(ParamType.TRACE_KEYSPACE, keyspace);
+        }
         return addToMutable;
     }
 
-    protected abstract TraceState newTraceState(InetAddressAndPort coordinator, TimeUUID sessionId, Tracing.TraceType traceType);
+    protected abstract TraceState newTraceState(
+        ClientState state,
+        InetAddressAndPort coordinator,
+        TimeUUID sessionId,
+        Tracing.TraceType traceType);
 
     // repair just gets a varargs method since it's so heavyweight anyway
     public static void traceRepair(String format, Object... args)
@@ -364,5 +424,5 @@ public static void trace(String format, Object... args)
     /**
      * Called for non-local traces (traces that are not initiated by local node == coordinator).
      */
-    public abstract void trace(ByteBuffer sessionId, String message, int ttl);
+    public abstract void trace(ClientState clientState, ByteBuffer sessionId, String message, int ttl);
 }
diff --git a/src/java/org/apache/cassandra/tracing/TracingImpl.java b/src/java/org/apache/cassandra/tracing/TracingImpl.java
index 1885146bee2b..5b828187fa68 100644
--- a/src/java/org/apache/cassandra/tracing/TracingImpl.java
+++ b/src/java/org/apache/cassandra/tracing/TracingImpl.java
@@ -24,8 +24,10 @@
 import java.util.Map;
 
 import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@@ -95,15 +97,15 @@ private TraceStateImpl getStateImpl()
     }
 
     @Override
-    protected TraceState newTraceState(InetAddressAndPort coordinator, TimeUUID sessionId, TraceType traceType)
+    protected TraceState newTraceState(ClientState state, InetAddressAndPort coordinator, TimeUUID sessionId, TraceType traceType)
     {
-        return new TraceStateImpl(coordinator, sessionId, traceType);
+        return new TraceStateImpl(state, coordinator, sessionId, traceType);
     }
 
     /**
      * Called for non-local traces (traces that are not initiated by local node == coordinator).
      */
-    public void trace(final ByteBuffer sessionId, final String message, final int ttl)
+    public void trace(ClientState clientState, final ByteBuffer sessionId, final String message, final int ttl)
     {
         final String threadName = Thread.currentThread().getName();
 
@@ -111,7 +113,8 @@ public void trace(final ByteBuffer sessionId, final String message, final int tt
         {
             public void runMayThrow()
             {
-                TraceStateImpl.mutateWithCatch(TraceKeyspace.makeEventMutation(sessionId, message, -1, threadName, ttl));
+                Mutation mutation = TraceKeyspace.makeEventMutation(sessionId, message, -1, threadName, ttl);
+                TraceStateImpl.mutateWithCatch(clientState, mutation);
             }
         });
     }
diff --git a/src/java/org/apache/cassandra/transport/CQLMessageHandler.java b/src/java/org/apache/cassandra/transport/CQLMessageHandler.java
index a2527a0cc823..3040fc7ca355 100644
--- a/src/java/org/apache/cassandra/transport/CQLMessageHandler.java
+++ b/src/java/org/apache/cassandra/transport/CQLMessageHandler.java
@@ -148,6 +148,12 @@ public boolean process(FrameDecoder.Frame frame) throws IOException
         return super.process(frame);
     }
 
+    @Override
+    protected void onDecoderReactivated()
+    {
+        ClientMetrics.instance.unpauseConnection();
+    }
+
     /**
      * Checks limits on bytes in flight and the request rate limiter (if enabled), then takes one of three actions:
      * 
diff --git a/src/java/org/apache/cassandra/transport/Client.java b/src/java/org/apache/cassandra/transport/Client.java
index 45f5e1f2fad4..c9ba4de6dd4c 100644
--- a/src/java/org/apache/cassandra/transport/Client.java
+++ b/src/java/org/apache/cassandra/transport/Client.java
@@ -29,6 +29,7 @@
 import org.apache.cassandra.auth.PasswordAuthenticator;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.marshal.Int32Type;
@@ -131,14 +132,14 @@ else if (msgType.equals("QUERY"))
             line = line.substring(6);
             // Ugly hack to allow setting a page size, but that's playground code anyway
             String query = line;
-            int pageSize = -1;
+            PageSize pageSize = PageSize.NONE;
             if (line.matches(".+ !\\d+$"))
             {
                 int idx = line.lastIndexOf('!');
                 query = line.substring(0, idx-1);
                 try
                 {
-                    pageSize = Integer.parseInt(line.substring(idx+1, line.length()));
+                    pageSize = PageSize.inRows(Integer.parseInt(line.substring(idx + 1, line.length())));
                 }
                 catch (NumberFormatException e)
                 {
diff --git a/src/java/org/apache/cassandra/transport/DataType.java b/src/java/org/apache/cassandra/transport/DataType.java
index 1d1a9130b646..e1862bebfb91 100644
--- a/src/java/org/apache/cassandra/transport/DataType.java
+++ b/src/java/org/apache/cassandra/transport/DataType.java
@@ -21,15 +21,41 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Map;
 import java.util.List;
+import java.util.Map;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import io.netty.buffer.ByteBuf;
-
 import org.apache.cassandra.cql3.FieldIdentifier;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.DateType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.DurationType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.SimpleDateType;
+import org.apache.cassandra.db.marshal.TimeType;
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.TypeParser;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.utils.Pair;
 
@@ -224,8 +250,7 @@ public static Pair<DataType, Object> fromType(AbstractType type, ProtocolVersion
     {
         // For CQL3 clients, ReversedType is an implementation detail and they
         // shouldn't have to care about it.
-        if (type instanceof ReversedType)
-            type = ((ReversedType)type).baseType;
+        type = type.unwrap();
 
         // For compatibility sake, we still return DateType as the timestamp type in resultSet metadata (#5723)
         if (type instanceof DateType)
diff --git a/src/java/org/apache/cassandra/transport/Dispatcher.java b/src/java/org/apache/cassandra/transport/Dispatcher.java
index d6cb5e822f9f..6d71d64892bc 100644
--- a/src/java/org/apache/cassandra/transport/Dispatcher.java
+++ b/src/java/org/apache/cassandra/transport/Dispatcher.java
@@ -32,7 +32,10 @@
 import io.netty.channel.EventLoop;
 import io.netty.util.AttributeKey;
 import org.apache.cassandra.concurrent.DebuggableTask;
+import org.apache.cassandra.concurrent.ExecutorLocals;
+import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.LocalAwareExecutorPlus;
+import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.exceptions.OverloadedException;
 import org.apache.cassandra.metrics.ClientMetrics;
@@ -40,13 +43,18 @@
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.reads.thresholds.CoordinatorWarnings;
+import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.ClientResourceLimits.Overload;
 import org.apache.cassandra.transport.Flusher.FlushItem;
 import org.apache.cassandra.transport.messages.ErrorMessage;
 import org.apache.cassandra.transport.messages.EventMessage;
+import org.apache.cassandra.transport.messages.StartupMessage;
+import org.apache.cassandra.utils.Closeable;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MonotonicClock;
 import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 import static org.apache.cassandra.concurrent.SharedExecutorPool.SHARED;
 
@@ -55,10 +63,7 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
     private static final Logger logger = LoggerFactory.getLogger(Dispatcher.class);
 
     @VisibleForTesting
-    static final LocalAwareExecutorPlus requestExecutor = SHARED.newExecutor(DatabaseDescriptor.getNativeTransportMaxThreads(),
-                                                                             DatabaseDescriptor::setNativeTransportMaxThreads,
-                                                                             "transport",
-                                                                             "Native-Transport-Requests");
+    static final ExecutorPlus requestExecutor = Stage.NATIVE_TRANSPORT_REQUESTS.executor();
 
     /** CASSANDRA-17812: Rate-limit new client connection setup to avoid overwhelming during bcrypt
      *
@@ -122,7 +127,7 @@ public void dispatch(Channel channel, Message.Request request, FlushItemConverte
                               (request.type == Message.Type.AUTH_RESPONSE || request.type == Message.Type.CREDENTIALS);
 
         // Importantly, the authExecutor will handle the AUTHENTICATE message which may be CPU intensive.
-        LocalAwareExecutorPlus executor = isAuthQuery ? authExecutor : requestExecutor;
+        ExecutorPlus executor = isAuthQuery ? authExecutor : requestExecutor;
 
         executor.submit(new RequestProcessor(channel, request, forFlusher, backpressure));
         ClientMetrics.instance.markRequestDispatched();
@@ -348,11 +353,42 @@ public boolean hasQueueCapacity()
         return requestExecutor.oldestTaskQueueTime() < (DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS) * threshold);
     }
 
-    /**
-     * Note: this method may be executed on the netty event loop, during initial protocol negotiation; the caller is
-     * responsible for cleaning up any global or thread-local state. (ex. tracing, client warnings, etc.).
-     */
-    private static Message.Response processRequest(ServerConnection connection, Message.Request request, Overload backpressure, RequestTime requestTime)
+    void processRequest(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure, RequestTime requestTime)
+    {
+        assert request.connection() instanceof ServerConnection;
+        ServerConnection connection = (ServerConnection) request.connection();
+        try
+        {
+            this.processRequest(connection, request, backpressure, requestTime)
+                .addCallback((response, ex) -> {
+                    FlushItem<?> toFlush;
+                    if (ex == null)
+                    {
+                        toFlush = forFlusher.toFlushItem(channel, request, response);
+                        Message.logger.trace("Responding: {}, v={}", response, connection.getVersion());
+                    }
+                    else
+                    {
+                        JVMStabilityInspector.inspectThrowable(ex);
+                        Predicate<Throwable> handler = ExceptionHandlers.getUnexpectedExceptionHandler(channel, true);
+                        ErrorMessage error = ErrorMessage.fromException(ex, handler);
+                        error.setStreamId(request.getStreamId());
+                        error.setWarnings(ClientWarn.instance.getWarnings());
+                        toFlush = forFlusher.toFlushItem(channel, request, error);
+                    }
+                    flush(toFlush);
+                });
+        }
+        finally
+        {
+            // As the warnings and trace state has been potentially propagated and "reset" in another stage, we do reset
+            // again here on the "originating" stage to make sure the next request starts from a clean slate:
+            ClientWarn.instance.resetWarnings();
+            Tracing.instance.set(null);
+        }
+    }
+
+    Future<Message.Response> processRequest(ServerConnection connection, Message.Request request, Overload backpressure, RequestTime requestTime)
     {
         long queueTime = requestTime.timeSpentInQueueNanos();
 
@@ -364,7 +400,7 @@ private static Message.Response processRequest(ServerConnection connection, Mess
         if (queueTime > DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS))
         {
             ClientMetrics.instance.markTimedOutBeforeProcessing();
-            return ErrorMessage.fromException(new OverloadedException("Query timed out before it could start"));
+            return ImmediateFuture.success(ErrorMessage.fromException(new OverloadedException("Query timed out before it could start")));
         }
 
         if (connection.getVersion().isGreaterOrEqualTo(ProtocolVersion.V4))
@@ -413,56 +449,60 @@ private static Message.Response processRequest(ServerConnection connection, Mess
 
         Message.logger.trace("Received: {}, v={}", request, connection.getVersion());
         connection.requests.inc();
-        Message.Response response = request.execute(qstate, requestTime);
-
-        if (request.isTrackable())
-            CoordinatorWarnings.done();
-
-        response.setStreamId(request.getStreamId());
-        response.setWarnings(ClientWarn.instance.getWarnings());
-        response.attach(connection);
-        connection.applyStateTransition(request.type, response.type);
-        return response;
-    }
-    
-    /**
-     * Note: this method may be executed on the netty event loop.
-     */
-    static Message.Response processRequest(Channel channel, Message.Request request, Overload backpressure, RequestTime requestTime)
-    {
+        ExecutorLocals executorLocals = ExecutorLocals.current();
         try
         {
-            return processRequest((ServerConnection) request.connection(), request, backpressure, requestTime);
+            return request.execute(qstate, requestTime).addCallback((result, ignored) -> {
+                // If the request was executed on a different Stage, we need to restore the ExecutorLocals
+                // on the current thread. See CNDB-13432 and CNDB-10759.
+                try (Closeable close = executorLocals.get())
+                {
+                    if (request.isTrackable())
+                        CoordinatorWarnings.done();
+
+                    result.setStreamId(request.getStreamId());
+                    result.setWarnings(ClientWarn.instance.getWarnings());
+                    result.attach(connection);
+                    connection.applyStateTransition(request.type, result.type);
+                }
+                finally
+                {
+                    CoordinatorWarnings.reset();
+                    ClientWarn.instance.resetWarnings();
+                }
+            });
         }
         catch (Throwable t)
-        {
-            JVMStabilityInspector.inspectThrowable(t);
-
-            if (request.isTrackable())
-                CoordinatorWarnings.done();
-
-            Predicate<Throwable> handler = ExceptionHandlers.getUnexpectedExceptionHandler(channel, true);
-            ErrorMessage error = ErrorMessage.fromException(t, handler);
-            error.setStreamId(request.getStreamId());
-            error.setWarnings(ClientWarn.instance.getWarnings());
-            return error;
-        }
-        finally
         {
             CoordinatorWarnings.reset();
             ClientWarn.instance.resetWarnings();
+            return ImmediateFuture.failure(t);
         }
     }
 
-    /**
-     * Note: this method is not expected to execute on the netty event loop.
-     */
-    void processRequest(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure, RequestTime requestTime)
+    static Future<Message.Response> processInit(ServerConnection connection, StartupMessage request)
     {
-        Message.Response response = processRequest(channel, request, backpressure, requestTime);
-        FlushItem<?> toFlush = forFlusher.toFlushItem(channel, request, response);
-        Message.logger.trace("Responding: {}, v={}", response, request.connection().getVersion());
-        flush(toFlush);
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        if (connection.getVersion().isGreaterOrEqualTo(ProtocolVersion.V4))
+            ClientWarn.instance.captureWarnings();
+
+        QueryState qstate = connection.validateNewMessage(request.type, connection.getVersion());
+
+        Message.logger.trace("Received: {}, v={}", request, connection.getVersion());
+        connection.requests.inc();
+        return request.execute(qstate, requestTime).addCallback((result, ignored) -> {
+            try
+            {
+                result.setStreamId(request.getStreamId());
+                result.setWarnings(ClientWarn.instance.getWarnings());
+                result.attach(connection);
+                connection.applyStateTransition(request.type, result.type);
+            }
+            finally
+            {
+                ClientWarn.instance.resetWarnings();
+            }
+        });
     }
 
     private void flush(FlushItem<?> item)
@@ -488,7 +528,6 @@ public boolean isDone()
 
     public static void shutdown()
     {
-        requestExecutor.shutdown();
         authExecutor.shutdown();
     }
 
diff --git a/src/java/org/apache/cassandra/transport/Event.java b/src/java/org/apache/cassandra/transport/Event.java
index 5e8e201d9a4d..c26fcc7d03c9 100644
--- a/src/java/org/apache/cassandra/transport/Event.java
+++ b/src/java/org/apache/cassandra/transport/Event.java
@@ -20,10 +20,12 @@
 import java.net.InetSocketAddress;
 import java.util.Iterator;
 import java.util.List;
+import java.util.function.UnaryOperator;
 
 import com.google.common.base.Objects;
 
 import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.functions.UDAggregate;
 import org.apache.cassandra.cql3.functions.UDFunction;
 import org.apache.cassandra.locator.InetAddressAndPort;
@@ -270,6 +272,15 @@ public SchemaChange(Change change, String keyspace)
             this(change, Target.KEYSPACE, keyspace, null);
         }
 
+        public SchemaChange withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+        {
+            if (keyspaceMapper == Constants.IDENTITY_STRING_MAPPER)
+                return this;
+
+            String newKeyspaceName = keyspaceMapper.apply(keyspace);
+            return keyspace.equals(newKeyspaceName) ? this : new SchemaChange(change, target, newKeyspaceName, name, argTypes);
+        }
+
         public static SchemaChange forFunction(Change change, UDFunction function)
         {
             return new SchemaChange(change, Target.FUNCTION, function.name().keyspace, function.name().name, function.argumentsList());
diff --git a/src/java/org/apache/cassandra/transport/InitialConnectionHandler.java b/src/java/org/apache/cassandra/transport/InitialConnectionHandler.java
index 576af3e6dcb3..410a7431b802 100644
--- a/src/java/org/apache/cassandra/transport/InitialConnectionHandler.java
+++ b/src/java/org/apache/cassandra/transport/InitialConnectionHandler.java
@@ -25,7 +25,6 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.cassandra.transport.ClientResourceLimits.Overload;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -148,11 +147,21 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> li
                         promise = new VoidChannelPromise(ctx.channel(), false);
                     }
 
-                    final Message.Response response = Dispatcher.processRequest(ctx.channel(), startup, Overload.NONE, Dispatcher.RequestTime.forImmediateExecution());
-
-                    outbound = response.encode(inbound.header.version);
-                    ctx.writeAndFlush(outbound, promise);
-                    logger.trace("Configured pipeline: {}", ctx.pipeline());
+                    ProtocolVersion version = inbound.header.version;
+                    Dispatcher.processInit((ServerConnection) connection, startup).addCallback((response, error) -> {
+                        if (error == null)
+                        {
+                            Envelope encoded = response.encode(version);
+                            ctx.writeAndFlush(encoded, promise);
+                            logger.debug("Configured pipeline: {}", ctx.pipeline());
+                        }
+                        else
+                        {
+                            ErrorMessage message = ErrorMessage.fromException(new ProtocolException(String.format("Unexpected error %s", error.getMessage())));
+                            Envelope encoded = message.encode(version);
+                            ctx.writeAndFlush(encoded);
+                        }
+                    });
                     break;
 
                 default:
diff --git a/src/java/org/apache/cassandra/transport/Message.java b/src/java/org/apache/cassandra/transport/Message.java
index ed853c0cbd7c..2480dce933c4 100644
--- a/src/java/org/apache/cassandra/transport/Message.java
+++ b/src/java/org/apache/cassandra/transport/Message.java
@@ -23,22 +23,29 @@
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.*;
 
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.OverloadedException;
+import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.messages.*;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.MonotonicClock;
 import org.apache.cassandra.utils.ReflectionUtils;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.Future;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
@@ -231,10 +238,32 @@ protected boolean isTrackable()
             return false;
         }
 
-        protected abstract Response execute(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest);
+        protected abstract Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest);
 
-        public final Response execute(QueryState queryState, Dispatcher.RequestTime requestTime)
+        /**
+         * Returns the time elapsed since this request was created. Note that this is the total lifetime of the request
+         * in the system, so we expect increasing returned values across multiple calls to elapsedTimeSinceCreation.
+         *
+         * @param timeUnit the time unit in which to return the elapsed time
+         * @return the time elapsed since this request was created
+         */
+        protected long elapsedTimeSinceCreation(TimeUnit timeUnit)
+        {
+            return timeUnit.convert(MonotonicClock.Global.approxTime.now() - createdAtNanos, TimeUnit.NANOSECONDS);
+        }
+
+
+        public final Future<Response> execute(QueryState queryState, Dispatcher.RequestTime requestTime)
         {
+            // at the time of the check, this is approximately the time spent in the NTR stage's queue
+            long elapsedTimeSinceCreation = elapsedTimeSinceCreation(TimeUnit.NANOSECONDS);
+            ClientMetrics.instance.recordQueueTime(elapsedTimeSinceCreation, TimeUnit.NANOSECONDS);
+            if (elapsedTimeSinceCreation > DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS))
+            {
+                ClientMetrics.instance.markTimedOutBeforeProcessing();
+                return ImmediateFuture.success(ErrorMessage.fromException(new OverloadedException("Query timed out before it could start")));
+            }
+
             boolean shouldTrace = false;
             TimeUUID tracingSessionId = null;
 
@@ -244,33 +273,29 @@ public final Response execute(QueryState queryState, Dispatcher.RequestTime requ
                 {
                     shouldTrace = true;
                     tracingSessionId = nextTimeUUID();
-                    Tracing.instance.newSession(tracingSessionId, getCustomPayload());
+                    Tracing.instance.newSession(queryState.getClientState(), tracingSessionId, getCustomPayload());
                 }
                 else if (StorageService.instance.shouldTraceProbablistically())
                 {
                     shouldTrace = true;
-                    Tracing.instance.newSession(getCustomPayload());
+                    Tracing.instance.newSession(queryState.getClientState(), getCustomPayload());
                 }
             }
 
-            Response response;
-            try
-            {
-                response = execute(queryState, requestTime, shouldTrace);
-            }
-            finally
-            {
-                if (shouldTrace)
-                    Tracing.instance.stopSession();
-            }
-
-            if (isTraceable() && isTracingRequested())
-                response.setTracingId(tracingSessionId);
-
-            return response;
+            Tracing.trace("Initialized tracing in execute. Already elapsed {} ns", (Clock.Global.nanoTime() - requestTime.startedAtNanos()));
+            boolean finalShouldTrace = shouldTrace;
+            TimeUUID finalTracingSessionId = tracingSessionId;
+            return maybeExecuteAsync(queryState, requestTime, shouldTrace)
+                   .addCallback((result, ignored) -> {
+                       if (finalShouldTrace)
+                           Tracing.instance.stopSession();
+
+                       if (isTraceable() && isTracingRequested())
+                           result.setTracingId(finalTracingSessionId);
+                   });
         }
 
-        void setTracingRequested()
+        public void setTracingRequested()
         {
             tracingRequested = true;
         }
@@ -303,13 +328,13 @@ protected Response(Type type)
                 throw new IllegalArgumentException();
         }
 
-        Message setTracingId(TimeUUID tracingId)
+        public Message setTracingId(TimeUUID tracingId)
         {
             this.tracingId = tracingId;
             return this;
         }
 
-        TimeUUID getTracingId()
+        public TimeUUID getTracingId()
         {
             return tracingId;
         }
diff --git a/src/java/org/apache/cassandra/transport/PipelineConfigurator.java b/src/java/org/apache/cassandra/transport/PipelineConfigurator.java
index 864241a663b3..0827df940671 100644
--- a/src/java/org/apache/cassandra/transport/PipelineConfigurator.java
+++ b/src/java/org/apache/cassandra/transport/PipelineConfigurator.java
@@ -144,7 +144,7 @@ public ChannelFuture initializeChannel(final EventLoopGroup workerGroup,
         bootstrap.childHandler(initializer);
 
         // Bind and start to accept incoming connections.
-        logger.info("Using Netty Version: {}", Version.identify().entrySet());
+        logger.debug("Using Netty Version: {}", Version.identify().entrySet());
         logger.info("Starting listening for CQL clients on {} ({})...", socket, tlsEncryptionPolicy.description());
         return bootstrap.bind(socket);
     }
diff --git a/src/java/org/apache/cassandra/transport/Server.java b/src/java/org/apache/cassandra/transport/Server.java
index 6abaf72515e6..b6ed49428511 100644
--- a/src/java/org/apache/cassandra/transport/Server.java
+++ b/src/java/org/apache/cassandra/transport/Server.java
@@ -19,7 +19,6 @@
 
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
@@ -409,19 +408,9 @@ private void registerConnectionTracker(ConnectionTracker connectionTracker)
             this.connectionTracker = connectionTracker;
         }
 
-        private InetAddressAndPort getNativeAddress(InetAddressAndPort endpoint)
+        protected InetAddressAndPort getNativeAddress(InetAddressAndPort endpoint)
         {
-            try
-            {
-                return InetAddressAndPort.getByName(StorageService.instance.getNativeaddress(endpoint, true));
-            }
-            catch (UnknownHostException e)
-            {
-                // That should not happen, so log an error, but return the
-                // endpoint address since there's a good change this is right
-                logger.error("Problem retrieving RPC address for {}", endpoint, e);
-                return InetAddressAndPort.getByAddressOverrideDefaults(endpoint.getAddress(), DatabaseDescriptor.getNativeTransportPort());
-            }
+            return StorageService.instance.getNativeAddressAndPort(endpoint);
         }
 
         private void send(InetAddressAndPort endpoint, Event.NodeEvent event)
diff --git a/src/java/org/apache/cassandra/transport/messages/AuthResponse.java b/src/java/org/apache/cassandra/transport/messages/AuthResponse.java
index 16247ed6c866..f4f4896caf75 100644
--- a/src/java/org/apache/cassandra/transport/messages/AuthResponse.java
+++ b/src/java/org/apache/cassandra/transport/messages/AuthResponse.java
@@ -27,6 +27,8 @@
 import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.*;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 /**
  * A SASL token message sent from client to server. Some SASL
@@ -69,7 +71,12 @@ public AuthResponse(byte[] token)
     }
 
     @Override
-    protected Response execute(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    protected Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    {
+        return ImmediateFuture.success(executeSync(queryState, requestTime, traceRequest));
+    }
+
+    private Response executeSync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         try
         {
diff --git a/src/java/org/apache/cassandra/transport/messages/BatchMessage.java b/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
index 8a87efc781fa..3ceb178a61a0 100644
--- a/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
@@ -20,10 +20,14 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.collect.ImmutableMap;
 
 import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Attributes;
 import org.apache.cassandra.cql3.BatchQueryOptions;
 import org.apache.cassandra.cql3.CQLStatement;
@@ -35,19 +39,23 @@
 import org.apache.cassandra.cql3.statements.BatchStatement;
 import org.apache.cassandra.cql3.statements.ModificationStatement;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.OverloadedException;
 import org.apache.cassandra.exceptions.PreparedQueryNotFoundException;
+import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.reads.thresholds.CoordinatorWarnings;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.CBUtil;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Message;
 import org.apache.cassandra.transport.ProtocolException;
 import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MD5Digest;
-
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 public class BatchMessage extends Message.Request
 {
@@ -170,7 +178,7 @@ protected boolean isTrackable()
     }
 
     @Override
-    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    public Future<Response> maybeExecuteAsync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         List<QueryHandler.Prepared> prepared = null;
         try
@@ -192,9 +200,9 @@ protected Message.Response execute(QueryState state, Dispatcher.RequestTime requ
                 }
                 else
                 {
-                    p = handler.getPrepared((MD5Digest)query);
+                    p = handler.getPrepared((MD5Digest) query);
                     if (null == p)
-                        throw new PreparedQueryNotFoundException((MD5Digest)query);
+                        throw new PreparedQueryNotFoundException((MD5Digest) query);
                 }
 
                 List<ByteBuffer> queryValues = values.get(i);
@@ -213,7 +221,7 @@ protected Message.Response execute(QueryState state, Dispatcher.RequestTime requ
             {
                 CQLStatement statement = prepared.get(i).statement;
                 if (queries != null)
-                    queries.add(prepared.get(i).rawCQLStatement);
+                    queries.add(statement.getRawCQLStatement());
                 batchOptions.prepareStatement(i, statement.getBindVariables());
 
                 if (!(statement instanceof ModificationStatement))
@@ -224,20 +232,78 @@ protected Message.Response execute(QueryState state, Dispatcher.RequestTime requ
 
             // Note: It's ok at this point to pass a bogus value for the number of bound terms in the BatchState ctor
             // (and no value would be really correct, so we prefer passing a clearly wrong one).
-            BatchStatement batch = new BatchStatement(batchType, VariableSpecifications.empty(), statements, Attributes.none());
+            BatchStatement batch = new BatchStatement(null, batchType,
+                                                      VariableSpecifications.empty(), statements, Attributes.none());
+
+            Tracing.trace("Processing batch start");
+            long requestStartMillisTime = Clock.Global.currentTimeMillis();
+            Optional<Stage> asyncStage = Stage.fromStatement(batch);
+            if (asyncStage.isPresent())
+            {
+                // Execution will continue on a new thread, but Dispatcher already called CoordinatorWarnings.init
+                // on the current thread; the Dispatcher.processRequest request.execute() callback must call
+                // CoordinatorWarnings.done() on the same thread that called init(). Reset CoordinatorWarnings on the
+                // current thread, and init on the new thread. See CNDB-13432 and CNDB-10759.
+                CoordinatorWarnings.reset();
+                List<QueryHandler.Prepared> finalPrepared = prepared;
+                return asyncStage.get().submit(() ->
+                                               {
+                                                   try
+                                                   {
+                                                       if (isTrackable())
+                                                           CoordinatorWarnings.init();
+
+                                                       // at the time of the check, this includes the time spent in the NTR queue, basic query parsing/set up,
+                                                       // and any time spent in the queue for the async stage
+                                                       long elapsedTime = elapsedTimeSinceCreation(TimeUnit.NANOSECONDS);
+                                                       ClientMetrics.instance.recordAsyncQueueTime(elapsedTime, TimeUnit.NANOSECONDS);
+                                                       if (elapsedTime > DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS))
+                                                       {
+                                                           ClientMetrics.instance.markTimedOutBeforeAsyncProcessing();
+                                                           throw new OverloadedException("Query timed out before it could start");
+                                                       }
+                                                   }
+                                                   catch (Exception e)
+                                                   {
+                                                       return handleException(state, finalPrepared, e);
+                                                   }
+                                                   return handleRequest(state, requestTime, handler, batch, batchOptions, queries, statements, finalPrepared, requestStartMillisTime);
+                                               });
+            }
+            else
+                return ImmediateFuture.success(handleRequest(state, requestTime, handler, batch, batchOptions, queries, statements, prepared, requestStartMillisTime));
+        }
+        catch (Exception e)
+        {
+            return ImmediateFuture.success(handleException(state, prepared, e));
+        }
+    }
 
-            long queryTime = currentTimeMillis();
-            Message.Response response = handler.processBatch(batch, state, batchOptions, getCustomPayload(), requestTime);
+    private Response handleRequest(QueryState queryState, Dispatcher.RequestTime requestTime, QueryHandler queryHandler, BatchStatement batch, BatchQueryOptions batchOptions, List<String> queries, List<ModificationStatement> statements, List<QueryHandler.Prepared> preparedList, long requestStartMillisTime)
+    {
+        try
+        {
+            Response response = queryHandler.processBatch(batch, queryState, batchOptions, getCustomPayload(), requestTime);
             if (queries != null)
-                QueryEvents.instance.notifyBatchSuccess(batchType, statements, queries, values, options, state, queryTime, response);
+                QueryEvents.instance.notifyBatchSuccess(batchType, statements, queries, values, options, queryState, requestStartMillisTime, response);
+
             return response;
         }
-        catch (Exception e)
+        catch (Exception exception)
         {
-            QueryEvents.instance.notifyBatchFailure(prepared, batchType, queryOrIdList, values, options, state, e);
-            JVMStabilityInspector.inspectThrowable(e);
-            return ErrorMessage.fromException(e);
+            return handleException(queryState, preparedList, exception);
         }
+        finally
+        {
+            Tracing.trace("Processing batch complete");
+        }
+    }
+
+    private ErrorMessage handleException(QueryState state, List<QueryHandler.Prepared> prepared, Exception e)
+    {
+        QueryEvents.instance.notifyBatchFailure(prepared, batchType, queryOrIdList, values, options, state, e);
+        JVMStabilityInspector.inspectThrowable(e);
+        return ErrorMessage.fromException(e);
     }
 
     private void traceQuery(QueryState state)
@@ -245,8 +311,8 @@ private void traceQuery(QueryState state)
         ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
         if (options.getConsistency() != null)
             builder.put("consistency_level", options.getConsistency().name());
-        if (options.getSerialConsistency() != null)
-            builder.put("serial_consistency_level", options.getSerialConsistency().name());
+        if (options.getSerialConsistency(state) != null)
+            builder.put("serial_consistency_level", options.getSerialConsistency(state).name());
 
         // TODO we don't have [typed] access to CQL bind variables here.  CASSANDRA-4560 is open to add support.
         Tracing.instance.begin("Execute batch of CQL3 queries", state.getClientAddress(), builder.build());
diff --git a/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java b/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
index 89b7e2a4a224..7e3bfe95b518 100644
--- a/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
@@ -19,11 +19,14 @@
 
 import java.nio.ByteBuffer;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.collect.ImmutableMap;
 
 import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.ColumnSpecification;
 import org.apache.cassandra.cql3.QueryEvents;
@@ -31,9 +34,12 @@
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.ResultSet;
 import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.exceptions.OverloadedException;
 import org.apache.cassandra.exceptions.PreparedQueryNotFoundException;
+import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.reads.thresholds.CoordinatorWarnings;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.CBUtil;
 import org.apache.cassandra.transport.Dispatcher;
@@ -41,11 +47,12 @@
 import org.apache.cassandra.transport.ProtocolException;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MD5Digest;
 import org.apache.cassandra.utils.NoSpamLogger;
-
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 public class ExecuteMessage extends Message.Request
 {
@@ -128,7 +135,7 @@ protected boolean isTrackable()
     }
 
     @Override
-    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    protected Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         QueryHandler.Prepared prepared = null;
         try
@@ -139,36 +146,82 @@ protected Message.Response execute(QueryState state, Dispatcher.RequestTime requ
                 throw new PreparedQueryNotFoundException(statementId);
 
             if (!prepared.fullyQualified
-                && !Objects.equals(state.getClientState().getRawKeyspace(), prepared.keyspace)
+                && !Objects.equals(queryState.getClientState().getRawKeyspace(), prepared.keyspace)
                 // We can not reliably detect inconsistencies for batches yet
-                && !(prepared.statement instanceof BatchStatement)
-            )
+                && !(prepared.statement instanceof BatchStatement))
             {
-                state.getClientState().warnAboutUseWithPreparedStatements(statementId, prepared.keyspace);
+                queryState.getClientState().warnAboutUseWithPreparedStatements(statementId, prepared.keyspace);
                 String msg = String.format("Tried to execute a prepared unqalified statement on a keyspace it was not prepared on. " +
-                                           " Executing the resulting prepared statement will return unexpected results: %s (on keyspace %s, previously prepared on %s)",
-                                           statementId, state.getClientState().getRawKeyspace(), prepared.keyspace);
+                                           "Executing the resulting prepared statement will return unexpected results: %s (on keyspace %s, previously prepared on %s)",
+                                           statementId, queryState.getClientState().getRawKeyspace(), prepared.keyspace);
                 nospam.error(msg);
             }
 
             CQLStatement statement = prepared.statement;
             options.prepare(statement.getBindVariables());
 
-            if (options.getPageSize() == 0)
+            if (options.getPageSize().getSize() == 0)
                 throw new ProtocolException("The page size cannot be 0");
 
             if (traceRequest)
-                traceQuery(state, prepared);
+                traceQuery(queryState, prepared);
 
-            // Some custom QueryHandlers are interested by the bound names. We provide them this information
+            // Some custom QueryHandlers are interested in the bound names. We provide them this information
             // by wrapping the QueryOptions.
             QueryOptions queryOptions = QueryOptions.addColumnSpecifications(options, prepared.statement.getBindVariables());
 
-            long requestStartTime = currentTimeMillis();
+            Tracing.trace("Executing prepared message started");
+            long requestStartMillisTime = Clock.Global.currentTimeMillis();
+            Optional<Stage> asyncStage = Stage.fromStatement(statement);
+            if (asyncStage.isPresent())
+            {
+                // Execution will continue on a new thread, but Dispatcher already called CoordinatorWarnings.init
+                // on the current thread; the Dispatcher.processRequest request.execute() callback must call
+                // CoordinatorWarnings.done() on the same thread that called init(). Reset CoordinatorWarnings on the
+                // current thread, and init on the new thread. See CNDB-13432 and CNDB-10759.
+                CoordinatorWarnings.reset();
+                QueryHandler.Prepared finalPrepared = prepared;
+                return asyncStage.get().submit(() ->
+                                               {
+                                                   try
+                                                   {
+                                                       if (isTrackable())
+                                                           CoordinatorWarnings.init();
+
+                                                       // at the time of the check, this includes the time spent in the NTR queue, basic query parsing/set up,
+                                                       // and any time spent in the queue for the async stage
+                                                       long elapsedTime = elapsedTimeSinceCreation(TimeUnit.NANOSECONDS);
+                                                       ClientMetrics.instance.recordAsyncQueueTime(elapsedTime, TimeUnit.NANOSECONDS);
+                                                       if (elapsedTime > DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS))
+                                                       {
+                                                           ClientMetrics.instance.markTimedOutBeforeAsyncProcessing();
+                                                           throw new OverloadedException("Query timed out before it could start");
+                                                       }
+                                                   }
+                                                   catch (Exception e)
+                                                   {
+                                                       return handleException(queryState, finalPrepared, e);
+                                                   }
+                                                   return handleRequest(queryState, requestTime, handler, queryOptions, statement, finalPrepared, requestStartMillisTime);
+                                               });
+            }
+            else
+                return ImmediateFuture.success(handleRequest(queryState, requestTime, handler, queryOptions, statement, prepared, requestStartMillisTime));
+        }
+        catch (Exception e)
+        {
+            return ImmediateFuture.success(handleException(queryState, prepared, e));
+        }
+    }
 
-            Message.Response response = handler.processPrepared(statement, state, queryOptions, getCustomPayload(), requestTime);
+    private Response handleRequest(QueryState queryState, Dispatcher.RequestTime requestTime, QueryHandler queryHandler, QueryOptions queryOptions, CQLStatement statement, QueryHandler.Prepared prepared, long requestStartMillisTime)
+    {
+        try
+        {
+            Response response = queryHandler.processPrepared(statement, queryState, queryOptions, getCustomPayload(), requestTime);
 
-            QueryEvents.instance.notifyExecuteSuccess(prepared.statement, prepared.rawCQLStatement, options, state, requestStartTime, response);
+            QueryEvents.instance.notifyExecuteSuccess(prepared.statement, options, queryState,
+                                                      requestStartMillisTime, response);
 
             if (response instanceof ResultMessage.Rows)
             {
@@ -204,23 +257,35 @@ else if (options.skipMetadata())
         }
         catch (Exception e)
         {
-            QueryEvents.instance.notifyExecuteFailure(prepared, options, state, e);
-            JVMStabilityInspector.inspectThrowable(e);
-            return ErrorMessage.fromException(e);
+            return handleException(queryState, prepared, e);
+        }
+        finally
+        {
+            Tracing.trace("Executing prepared message completed");
         }
     }
 
+    private ErrorMessage handleException(QueryState queryState, QueryHandler.Prepared prepared, Exception e)
+    {
+        QueryEvents.instance.notifyExecuteFailure(prepared, options, queryState, e);
+        JVMStabilityInspector.inspectThrowable(e);
+        return ErrorMessage.fromException(e);
+    }
+
     private void traceQuery(QueryState state, QueryHandler.Prepared prepared)
     {
         ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
-        if (options.getPageSize() > 0)
-            builder.put("page_size", Integer.toString(options.getPageSize()));
+        if (options.getPageSize().isDefined())
+        {
+            builder.put("page_size", Integer.toString(options.getPageSize().getSize()));
+            builder.put("page_size_unit", options.getPageSize().getUnit().name());
+        }
         if (options.getConsistency() != null)
             builder.put("consistency_level", options.getConsistency().name());
-        if (options.getSerialConsistency() != null)
-            builder.put("serial_consistency_level", options.getSerialConsistency().name());
+        if (options.getSerialConsistency(state) != null)
+            builder.put("serial_consistency_level", options.getSerialConsistency(state).name());
 
-        builder.put("query", prepared.rawCQLStatement);
+        builder.put("query", prepared.statement.getRawCQLStatement());
 
         for (int i = 0; i < prepared.statement.getBindVariables().size(); i++)
         {
diff --git a/src/java/org/apache/cassandra/transport/messages/OptionsMessage.java b/src/java/org/apache/cassandra/transport/messages/OptionsMessage.java
index 1ed109db2eeb..f917563b5947 100644
--- a/src/java/org/apache/cassandra/transport/messages/OptionsMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/OptionsMessage.java
@@ -18,24 +18,35 @@
 package org.apache.cassandra.transport.messages;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import io.netty.buffer.ByteBuf;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.Compressor;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Message;
 import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.ProductType;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 /**
  * Message to indicate that the server is ready to receive requests.
  */
 public class OptionsMessage extends Message.Request
 {
+    private static final List<String> supportedPageUnits = Arrays.stream(PageSize.PageUnit.values()).map(PageSize.PageUnit::name).collect(Collectors.toList());
+
     public static final Message.Codec<OptionsMessage> codec = new Message.Codec<OptionsMessage>()
     {
         public OptionsMessage decode(ByteBuf body, ProtocolVersion version)
@@ -59,7 +70,12 @@ public OptionsMessage()
     }
 
     @Override
-    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    protected Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    {
+        return ImmediateFuture.success(executeSync(queryState, requestTime, traceRequest));
+    }
+
+    private Message.Response executeSync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         List<String> cqlVersions = new ArrayList<String>();
         cqlVersions.add(QueryProcessor.CQL_VERSION.toString());
@@ -74,6 +90,10 @@ protected Message.Response execute(QueryState state, Dispatcher.RequestTime requ
         supported.put(StartupMessage.CQL_VERSION, cqlVersions);
         supported.put(StartupMessage.COMPRESSION, compressions);
         supported.put(StartupMessage.PROTOCOL_VERSIONS, ProtocolVersion.supportedVersions());
+        supported.put(StartupMessage.EMULATE_DBAAS_DEFAULTS, Collections.singletonList(String.valueOf(DatabaseDescriptor.isEmulateDbaasDefaults())));
+        supported.put(StartupMessage.PAGE_UNIT, supportedPageUnits);
+        supported.put(StartupMessage.SERVER_VERSION, Collections.singletonList(FBUtilities.getReleaseVersionString()));
+        supported.put(StartupMessage.PRODUCT_TYPE, Collections.singletonList(ProductType.getProduct().toString()));
 
         return new SupportedMessage(supported);
     }
diff --git a/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java b/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
index bebea3cc5b2d..a9e36364bab2 100644
--- a/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
@@ -36,6 +36,8 @@
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 
@@ -115,7 +117,12 @@ protected boolean isTraceable()
     }
 
     @Override
-    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    protected Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    {
+        return ImmediateFuture.success(executeSync(queryState, requestTime, traceRequest));
+    }
+
+    private Message.Response executeSync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         try
         {
diff --git a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
index 665d62a8cb08..463509b3e192 100644
--- a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
@@ -17,26 +17,35 @@
  */
 package org.apache.cassandra.transport.messages;
 
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
 import com.google.common.collect.ImmutableMap;
 
 import io.netty.buffer.ByteBuf;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.QueryEvents;
 import org.apache.cassandra.cql3.QueryHandler;
 import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.exceptions.OverloadedException;
 import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.reads.thresholds.CoordinatorWarnings;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.CBUtil;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Message;
 import org.apache.cassandra.transport.ProtocolException;
 import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.JVMStabilityInspector;
-
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 /**
  * A CQL query
@@ -99,49 +108,110 @@ protected boolean isTrackable()
     }
 
     @Override
-    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    protected Future<Response> maybeExecuteAsync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         CQLStatement statement = null;
         try
         {
-            if (options.getPageSize() == 0)
+            if (options.getPageSize().getSize() == 0)
                 throw new ProtocolException("The page size cannot be 0");
 
             if (traceRequest)
                 traceQuery(state);
 
-            long queryStartTime = currentTimeMillis();
+            long requestStartMillisTime = Clock.Global.currentTimeMillis();
+            Tracing.trace("Executing query started");
 
             QueryHandler queryHandler = ClientState.getCQLQueryHandler();
             statement = queryHandler.parse(query, state, options);
-            Message.Response response = queryHandler.process(statement, state, options, getCustomPayload(), requestTime);
-            QueryEvents.instance.notifyQuerySuccess(statement, query, options, state, queryStartTime, response);
+
+            Optional<Stage> asyncStage = Stage.fromStatement(statement);
+            if (asyncStage.isPresent())
+            {
+                // Execution will continue on a new exectutor, but Dispatcher already called CoordinatorWarnings.init
+                // on the current thread; the Dispatcher.processRequest request.execute() callback must call
+                // CoordinatorWarnings.done() on the same thread that called init(). Reset CoordinatorWarnings on the
+                // current thread, and init on the new thread. See CNDB-13432 and CNDB-10759.
+                CoordinatorWarnings.reset();
+                CQLStatement finalStatement = statement;
+                return asyncStage.get().submit(() ->
+                                               {
+                                                   try
+                                                   {
+                                                       if (isTrackable())
+                                                           CoordinatorWarnings.init();
+
+                                                       // at the time of the check, this includes the time spent in the NTR queue, basic query parsing/set up,
+                                                       // and any time spent in the queue for the async stage
+                                                       long elapsedTime = elapsedTimeSinceCreation(TimeUnit.NANOSECONDS);
+                                                       ClientMetrics.instance.recordAsyncQueueTime(elapsedTime, TimeUnit.NANOSECONDS);
+                                                       if (elapsedTime > DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS))
+                                                       {
+                                                           ClientMetrics.instance.markTimedOutBeforeAsyncProcessing();
+                                                           throw new OverloadedException("Query timed out before it could start");
+                                                       }
+                                                   }
+                                                   catch (Exception e)
+                                                   {
+                                                       return handleException(state, finalStatement, e);
+                                                   }
+                                                   return handleRequest(state, queryHandler, requestTime, finalStatement, requestStartMillisTime);
+                                               });
+            }
+            else
+                return ImmediateFuture.success(handleRequest(state, queryHandler, requestTime, statement, requestStartMillisTime));
+        }
+        catch (Exception exception)
+        {
+            return ImmediateFuture.success(handleException(state, statement, exception));
+        }
+    }
+
+    private Response handleRequest(QueryState queryState, QueryHandler queryHandler, Dispatcher.RequestTime requestTime, CQLStatement statement, long requestStartMillisTime)
+    {
+        try
+        {
+            Response response = queryHandler.process(statement, queryState, options, getCustomPayload(), requestTime);
+            QueryEvents.instance.notifyQuerySuccess(statement, query, options, queryState, requestStartMillisTime, response);
 
             if (options.skipMetadata() && response instanceof ResultMessage.Rows)
-                ((ResultMessage.Rows)response).result.metadata.setSkipMetadata();
+                ((ResultMessage.Rows) response).result.metadata.setSkipMetadata();
 
             return response;
         }
-        catch (Exception e)
+        catch (Exception ex)
         {
-            QueryEvents.instance.notifyQueryFailure(statement, query, options, state, e);
-            JVMStabilityInspector.inspectThrowable(e);
-            if (!((e instanceof RequestValidationException) || (e instanceof RequestExecutionException)))
-                logger.error("Unexpected error during query", e);
-            return ErrorMessage.fromException(e);
+            return handleException(queryState, statement, ex);
+        }
+        finally
+        {
+            Tracing.trace("Executing query completed");
         }
     }
 
+    private ErrorMessage handleException(QueryState queryState, CQLStatement statement, Exception exception)
+    {
+        QueryEvents.instance.notifyQueryFailure(statement, query, options, queryState, exception);
+        JVMStabilityInspector.inspectThrowable(exception);
+        if (!((exception instanceof RequestValidationException) || (exception instanceof RequestExecutionException)))
+            logger.error("Unexpected error during query", exception);
+
+        return ErrorMessage.fromException(exception);
+    }
+
     private void traceQuery(QueryState state)
     {
         ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
         builder.put("query", query);
-        if (options.getPageSize() > 0)
-            builder.put("page_size", Integer.toString(options.getPageSize()));
+        if (options.getPageSize().isDefined())
+        {
+            builder.put("page_size", Integer.toString(options.getPageSize().getSize()));
+            builder.put("page_size_unit", options.getPageSize().getUnit().name());
+        }
         if (options.getConsistency() != null)
             builder.put("consistency_level", options.getConsistency().name());
-        if (options.getSerialConsistency() != null)
-            builder.put("serial_consistency_level", options.getSerialConsistency().name());
+        if (options.getSerialConsistency(state) != null)
+            builder.put("serial_consistency_level", options.getSerialConsistency(state).name());
 
         Tracing.instance.begin("Execute CQL3 query", state.getClientAddress(), builder.build());
     }
@@ -149,7 +219,7 @@ private void traceQuery(QueryState state)
     @Override
     public String toString()
     {
-        return String.format("QUERY %s [pageSize = %d] at consistency %s", 
+        return String.format("QUERY %s [pageSize = %s] at consistency %s",
                              query, options.getPageSize(), options.getConsistency());
     }
 }
diff --git a/src/java/org/apache/cassandra/transport/messages/RegisterMessage.java b/src/java/org/apache/cassandra/transport/messages/RegisterMessage.java
index 83f9cac3160a..570344717818 100644
--- a/src/java/org/apache/cassandra/transport/messages/RegisterMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/RegisterMessage.java
@@ -24,6 +24,8 @@
 
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.*;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 public class RegisterMessage extends Message.Request
 {
@@ -63,7 +65,12 @@ public RegisterMessage(List<Event.Type> eventTypes)
     }
 
     @Override
-    protected Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    protected Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    {
+        return ImmediateFuture.success(executeSync(queryState, requestTime, traceRequest));
+    }
+
+    private Response executeSync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         assert connection instanceof ServerConnection;
         Connection.Tracker tracker = connection.getTracker();
diff --git a/src/java/org/apache/cassandra/transport/messages/ResultMessage.java b/src/java/org/apache/cassandra/transport/messages/ResultMessage.java
index a8d8daec28bf..813b023808ee 100644
--- a/src/java/org/apache/cassandra/transport/messages/ResultMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/ResultMessage.java
@@ -18,15 +18,21 @@
 package org.apache.cassandra.transport.messages;
 
 
+import java.util.function.UnaryOperator;
+
 import com.google.common.annotations.VisibleForTesting;
 
 import io.netty.buffer.ByteBuf;
-
+import org.apache.cassandra.cql3.Constants;
 import org.apache.cassandra.cql3.ResultSet;
-import org.apache.cassandra.transport.*;
+import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Event;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.MD5Digest;
 
-public abstract class ResultMessage extends Message.Response
+public abstract class ResultMessage<T extends ResultMessage<T>> extends Message.Response
 {
     public static final Message.Codec<ResultMessage> codec = new Message.Codec<ResultMessage>()
     {
@@ -97,7 +103,12 @@ protected ResultMessage(Kind kind)
         this.kind = kind;
     }
 
-    public static class Void extends ResultMessage
+    public T withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+    {
+        return (T) this;
+    }
+
+    public static class Void extends ResultMessage<Void>
     {
         // Even though we have no specific information here, don't make a
         // singleton since as each message it has in fact a streamid and connection.
@@ -131,7 +142,7 @@ public String toString()
         }
     }
 
-    public static class SetKeyspace extends ResultMessage
+    public static class SetKeyspace extends ResultMessage<SetKeyspace>
     {
         public final String keyspace;
 
@@ -167,9 +178,27 @@ public String toString()
         {
             return "RESULT set keyspace " + keyspace;
         }
+
+        @Override
+        public SetKeyspace withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+        {
+            if (keyspaceMapper == Constants.IDENTITY_STRING_MAPPER)
+                return this;
+
+            String newKeyspaceName = keyspaceMapper.apply(keyspace);
+            if (keyspace.equals(newKeyspaceName))
+                return this;
+
+            SetKeyspace r = new SetKeyspace(newKeyspaceName);
+            r.setWarnings(getWarnings());
+            r.setCustomPayload(getCustomPayload());
+            r.setSource(getSource());
+            r.setStreamId(getStreamId());
+            return r;
+        }
     }
 
-    public static class Rows extends ResultMessage
+    public static class Rows extends ResultMessage<Rows>
     {
         public static final Message.Codec<ResultMessage> subcodec = new Message.Codec<ResultMessage>()
         {
@@ -206,9 +235,31 @@ public String toString()
         {
             return "ROWS " + result;
         }
+
+        @Override
+        public Rows withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+        {
+            if (keyspaceMapper == Constants.IDENTITY_STRING_MAPPER)
+                return this;
+
+            return withResultSet(result.withOverriddenKeyspace(keyspaceMapper));
+        }
+
+        public Rows withResultSet(ResultSet newResultSet)
+        {
+            if (newResultSet == result)
+                return this;
+
+            Rows r = new Rows(newResultSet);
+            r.setWarnings(getWarnings());
+            r.setCustomPayload(getCustomPayload());
+            r.setSource(getSource());
+            r.setStreamId(getStreamId());
+            return r;
+        }
     }
 
-    public static class Prepared extends ResultMessage
+    public static class Prepared extends ResultMessage<Prepared>
     {
         public static final Message.Codec<ResultMessage> subcodec = new Message.Codec<ResultMessage>()
         {
@@ -283,6 +334,28 @@ public Prepared withResultMetadata(ResultSet.ResultMetadata resultMetadata)
             return new Prepared(statementId, resultMetadata.getResultMetadataId(), metadata, resultMetadata);
         }
 
+        @Override
+        public Prepared withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+        {
+            if (keyspaceMapper == Constants.IDENTITY_STRING_MAPPER)
+                return this;
+
+            ResultSet.PreparedMetadata newPreparedMetadata = metadata.withOverriddenKeyspace(keyspaceMapper);
+            ResultSet.ResultMetadata newResultSetMetadata = resultMetadata.withOverriddenKeyspace(keyspaceMapper);
+            if (newPreparedMetadata == metadata && newResultSetMetadata == resultMetadata)
+                return this;
+
+            Prepared r = new Prepared(statementId,
+                                      resultMetadataId,
+                                      newPreparedMetadata,
+                                      newResultSetMetadata);
+            r.setWarnings(getWarnings());
+            r.setCustomPayload(getCustomPayload());
+            r.setSource(getSource());
+            r.setStreamId(getStreamId());
+            return r;
+        }
+
         @Override
         public String toString()
         {
@@ -290,7 +363,7 @@ public String toString()
         }
     }
 
-    public static class SchemaChange extends ResultMessage
+    public static class SchemaChange extends ResultMessage<SchemaChange>
     {
         public final Event.SchemaChange change;
 
@@ -322,6 +395,24 @@ public int encodedSize(ResultMessage msg, ProtocolVersion version)
             }
         };
 
+        @Override
+        public SchemaChange withOverriddenKeyspace(UnaryOperator<String> keyspaceMapper)
+        {
+            if (keyspaceMapper == Constants.IDENTITY_STRING_MAPPER)
+                return this;
+
+            Event.SchemaChange newEvent = change.withOverriddenKeyspace(keyspaceMapper);
+            if (change == newEvent)
+                return this;
+
+            SchemaChange r = new SchemaChange(newEvent);
+            r.setWarnings(getWarnings());
+            r.setCustomPayload(getCustomPayload());
+            r.setSource(getSource());
+            r.setStreamId(getStreamId());
+            return r;
+        }
+
         @Override
         public String toString()
         {
diff --git a/src/java/org/apache/cassandra/transport/messages/StartupMessage.java b/src/java/org/apache/cassandra/transport/messages/StartupMessage.java
index 2969009f448b..4f0d93c82b9d 100644
--- a/src/java/org/apache/cassandra/transport/messages/StartupMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/StartupMessage.java
@@ -28,6 +28,8 @@
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.*;
 import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 /**
  * The initial message of the protocol.
@@ -41,6 +43,10 @@ public class StartupMessage extends Message.Request
     public static final String DRIVER_NAME = "DRIVER_NAME";
     public static final String DRIVER_VERSION = "DRIVER_VERSION";
     public static final String THROW_ON_OVERLOAD = "THROW_ON_OVERLOAD";
+    public static final String EMULATE_DBAAS_DEFAULTS = "EMULATE_DBAAS_DEFAULTS";
+    public static final String PAGE_UNIT = "PAGE_UNIT";
+    public static final String SERVER_VERSION = "SERVER_VERSION";
+    public static final String PRODUCT_TYPE = "PRODUCT_TYPE";
 
     public static final Message.Codec<StartupMessage> codec = new Message.Codec<StartupMessage>()
     {
@@ -69,7 +75,12 @@ public StartupMessage(Map<String, String> options)
     }
 
     @Override
-    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    protected Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+    {
+        return ImmediateFuture.success(executeSync(queryState, requestTime, traceRequest));
+    }
+
+    private Message.Response executeSync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         String cqlVersion = options.get(CQL_VERSION);
         if (cqlVersion == null)
diff --git a/src/java/org/apache/cassandra/triggers/TriggerExecutor.java b/src/java/org/apache/cassandra/triggers/TriggerExecutor.java
index c76c6bd4b271..8f6884aa8a4d 100644
--- a/src/java/org/apache/cassandra/triggers/TriggerExecutor.java
+++ b/src/java/org/apache/cassandra/triggers/TriggerExecutor.java
@@ -19,16 +19,23 @@
 package org.apache.cassandra.triggers;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
 import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.exceptions.CassandraException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
diff --git a/src/java/org/apache/cassandra/utils/AbstractGuavaIterator.java b/src/java/org/apache/cassandra/utils/AbstractGuavaIterator.java
index 00756df87779..5ae4d13a05fe 100644
--- a/src/java/org/apache/cassandra/utils/AbstractGuavaIterator.java
+++ b/src/java/org/apache/cassandra/utils/AbstractGuavaIterator.java
@@ -29,7 +29,7 @@
 import static com.google.common.base.Preconditions.checkState;
 
 /**
- * This is fork of the Guava AbstractIterator, the only difference
+ * This is fork of the Guava AbstractGuavaIterator, the only difference
  * is that the next variable is now protected so that the KeyRangeIterator.skipTo
  * method can avoid early state changed.
  */
@@ -151,7 +151,7 @@ public void remove()
      * Returns the next element in the iteration without advancing the iteration,
      * according to the contract of {@link PeekingIterator#peek()}.
      *
-     * <p>Implementations of {@code AbstractIterator} that wish to expose this
+     * <p>Implementations of {@code AbstractGuavaIterator} that wish to expose this
      * functionality should implement {@code PeekingIterator}.
      */
     public final T peek()
diff --git a/src/java/org/apache/cassandra/utils/BinaryHeap.java b/src/java/org/apache/cassandra/utils/BinaryHeap.java
new file mode 100644
index 000000000000..26e8f1e0e188
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/BinaryHeap.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Comparator;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A base binary heap implementation with fixed size, supporting only operations that push
+ * data down in the heap (i.e. after the initial initialization of the heap from a collection
+ * of items, only top/smallest items can be modified (removed or replaced)).
+ * <p>
+ * This class's purpose is to implement various sources of sorted entries, e.g.
+ * for merging iterators (see e.g. {@link TrieMemoryIndex.SortingSingletonOrSetIterator}, producing
+ * a sorted iterator from an unsorted list of items ({@link SortingIterator}) or selecting the
+ * top items from data of unbounded size ({@link TopKSelector}).
+ * <p>
+ * As it does not support adding elements after the initial construction, the class does not
+ * implement a priority queue, where items need to be repeatedly added and removed. If a priority
+ * queue is required, consider using {@link LucenePriorityQueue}.
+ * <p>
+ * By default, the implementation supports nulls among the source entries (by comparing them greater
+ * than all other elements and using null as a marker of completion) and achieves removal by
+ * replacing items with null. This adds slight overhead for simple comparators (e.g. ints), but
+ * significantly improves performance when the comparator is complex.
+ */
+public abstract class BinaryHeap
+{
+    // Note: This class is tested via its descendants by SortingIteratorTest and TopKSelectorTest.
+
+    protected final Object[] heap;
+
+    /**
+     * Create a binary heap with the given array. The data must be heapified before being used.
+     */
+    protected BinaryHeap(Object[] data)
+    {
+        Preconditions.checkArgument(data.length > 0, "Binary heap needs at least one item.");
+        this.heap = data;
+        // Note that we can't perform any preparation here because the subclass defining greaterThan may have not been
+        // initialized yet.
+    }
+
+    /**
+     * Compare two objects and return true iff the first is greater.
+     * The method must treat nulls as greater than non-null objects.
+     */
+    protected abstract boolean greaterThan(Object a, Object b);
+
+    /**
+     * Get the size. Usually just the heap length because we don't count removed elements, but some descendants may
+     * choose to control size differently.
+     */
+    protected int size()
+    {
+        return heap.length;
+    }
+
+    /**
+     * Advance an item. Return null if there are no further entries.
+     * The default implementations assumes entries are single items and always returns null.
+     * Override it to implement merging of sorted iterators.
+     * @param item The heap item to advance
+     */
+    protected Object advanceItem(Object item)
+    {
+        return null;
+    }
+
+    /**
+     * Advance an item to the closest entry greater than or equal to the target.
+     * Return null if no such entry exists.
+     * The default implementations assumes entries are single items and always returns null.
+     * Override it to implement merging of sorted seeking iterators.
+     * @param item The heap item to advance
+     * @param targetKey The comparison key
+     */
+    protected Object advanceItemTo(Object item, Object targetKey)
+    {
+        return null;
+    }
+
+    /**
+     * Turn the current list of items into a binary heap by using the initial heap construction
+     * of the heapsort algorithm with complexity O(size()). Done recursively to improve caching on
+     * larger heaps.
+     */
+    protected void heapify()
+    {
+        heapifyRecursively(0, size());
+    }
+
+    protected boolean isEmpty()
+    {
+        return heap[0] == null;
+    }
+
+    /**
+     * Return the next element in the heap without advancing.
+     */
+    protected Object top()
+    {
+        return heap[0];
+    }
+
+    /**
+     * Get and remove the next element in the heap.
+     * If the heap contains duplicates, they will be returned in an arbitrary order.
+     */
+    protected Object pop()
+    {
+        Object item = heap[0];
+        heapifyDown(advanceItem(item), 0);
+        return item;
+    }
+
+    /**
+     * Get and replace the top item with a new one.
+     */
+    protected Object replaceTop(Object newItem)
+    {
+        Object item = heap[0];
+        heapifyDown(newItem, 0);
+        return item;
+    }
+
+    /**
+     * Get the next element and skip over all items equal to it.
+     * Calling this instead of {@link #pop} results in deduplication of the list
+     * of entries.
+     */
+    protected Object popAndSkipEqual()
+    {
+        Object item = heap[0];
+        advanceBeyond(item, item);
+        return item;
+    }
+
+    protected void advanceBeyond(Object targetKey, Object topItem)
+    {
+        Object advanced = advanceItem(topItem);
+        // avoid recomparing top element
+        int size = size();
+        if (1 < size)
+        {
+            if (2 < size)
+                applyAdvance(targetKey, 2, ADVANCE_BEYOND, size);
+            applyAdvance(targetKey, 1, ADVANCE_BEYOND, size);
+        }
+        heapifyDown(advanced, 0);
+    }
+
+    /**
+     * Skip to the first element that is greater than or equal to the given key.
+     */
+    protected void advanceTo(Object targetKey)
+    {
+        applyAdvance(targetKey, 0, ADVANCE_TO, size());
+    }
+
+    /**
+     * Interface used to specify an advancing operation for {@link #applyAdvance}.
+     */
+    protected interface AdvanceOperation
+    {
+        /**
+         * Return true if the necessary condition is satisfied by this heap entry.
+         * The condition is assumed to also be satisfied for all descendants of the
+         * entry (as they are equal or greater).
+         */
+        boolean shouldStop(BinaryHeap self, Object heapEntry, Object targetKey);
+
+        /**
+         * Apply the relevant advancing operation and return the entry to use.
+         */
+        Object advanceItem(BinaryHeap self, Object heapEntry, Object targetKey);
+    }
+
+    static final AdvanceOperation ADVANCE_BEYOND = new AdvanceOperation()
+    {
+        @Override
+        public boolean shouldStop(BinaryHeap self, Object heapEntry, Object targetKey)
+        {
+            return self.greaterThan(heapEntry, targetKey);
+        }
+
+        @Override
+        public Object advanceItem(BinaryHeap self, Object heapEntry, Object targetKey)
+        {
+            return self.advanceItem(heapEntry);
+        }
+    };
+
+    static final AdvanceOperation ADVANCE_TO = new AdvanceOperation()
+    {
+        @Override
+        public boolean shouldStop(BinaryHeap self, Object heapEntry, Object targetKey)
+        {
+            return !self.greaterThan(targetKey, heapEntry);
+        }
+
+        @Override
+        public Object advanceItem(BinaryHeap self, Object heapEntry, Object targetKey)
+        {
+            return self.advanceItemTo(heapEntry, targetKey);
+        }
+    };
+
+    /**
+     * Recursively apply the advance operation to all elements in the subheap rooted at the given heapIndex
+     * that do not satisfy the shouldStop condition, and restore the heap ordering on the way back from the recursion.
+     */
+    private void applyAdvance(Object targetKey, int heapIndex, AdvanceOperation advanceOperation, int size)
+    {
+        if (advanceOperation.shouldStop(this, heap[heapIndex], targetKey))
+            return;
+
+        if (heapIndex * 2 + 1 < size)
+        {
+            if (heapIndex * 2 + 2 < size)
+                applyAdvance(targetKey, heapIndex * 2 + 2, advanceOperation, size);
+            applyAdvance(targetKey, heapIndex * 2 + 1, advanceOperation, size);
+
+            Object advanced = advanceOperation.advanceItem(this, heap[heapIndex], targetKey);
+            heapifyDown(advanced, heapIndex);
+        }
+        else
+        {
+            Object advanced = advanceOperation.advanceItem(this, heap[heapIndex], targetKey);
+            heap[heapIndex] = advanced;
+        }
+    }
+
+    /**
+     * Perform the initial heapification of the data. This could be achieved with the method above (with shouldStop
+     * always false and advanceItem returning the item unchanged), but a direct implementation is much simpler and
+     * performs better.
+     */
+
+    private void heapifyRecursively(int heapIndex, int size)
+    {
+        if (heapIndex * 2 + 1 < size)
+        {
+            if (heapIndex * 2 + 2 < size)
+                heapifyRecursively(heapIndex * 2 + 2, size);
+            heapifyRecursively(heapIndex * 2 + 1, size);
+
+            heapifyDown(heap[heapIndex], heapIndex);
+        }
+    }
+
+    /**
+     * Push the given state down in the heap from the given index until it finds its proper place among
+     * the subheap rooted at that position.
+     */
+    private void heapifyDown(Object item, int index)
+    {
+        heapifyDownUpTo(item, index, size());
+    }
+
+    /**
+     * Push the given state down in the heap from the given index until it finds its proper place among
+     * the subheap rooted at that position.
+     */
+    private void heapifyDownUpTo(Object item, int index, int size)
+    {
+        while (true)
+        {
+            int next = index * 2 + 1;
+            if (next >= size)
+                break;
+            // Select the smaller of the two children to push down to.
+            if (next + 1 < size && greaterThan(heap[next], heap[next + 1]))
+                ++next;
+            // If the child is greater or equal, the invariant has been restored.
+            if (!greaterThan(item, heap[next]))
+                break;
+            heap[index] = heap[next];
+            index = next;
+        }
+        heap[index] = item;
+    }
+
+    /**
+     * Sort the heap by repeatedly popping the top item and placing it at the end of the heap array.
+     * The result will contain the elements in the heap sorted in descending order.
+     * The heap must be heapified before calling this method.
+     */
+    protected void heapSort()
+    {
+        // Sorting the ones from 1 will also make put the right value in heap[0]
+        heapSortFrom(1);
+    }
+
+    /**
+     * Partially sort the heap by repeatedly popping the top item and placing it at the end of the heap array,
+     * until the given start position is reached. This results in a partial sorting where the smallest items
+     * (according to the comparator) are placed at positions of the heap between start and size in descending order,
+     * and the items before that are left heapified.
+     * The heap must be heapified up to the size before calling this method.
+     * Used to fetch items after a certain offset in a top-k selection.
+     */
+    protected void heapSortFrom(int start)
+    {
+        // Data must already be heapified up to that size, comparator must be reverse
+        for (int i = size() - 1; i >= start; --i)
+        {
+            Object top = heap[0];
+            heapifyDownUpTo(heap[i], 0, i);
+            heap[i] = top;
+        }
+    }
+
+    /**
+     * A binary heap that uses a comparator to determine the order of elements, implementing the necessary handling
+     * of nulls.
+     */
+    public static class WithComparator<T> extends BinaryHeap
+    {
+        final Comparator<? super T> comparator;
+
+        public WithComparator(Comparator<? super T> comparator, Object[] data)
+        {
+            super(data);
+            this.comparator = comparator;
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        protected boolean greaterThan(Object a, Object b)
+        {
+            // nulls are treated as greater than non-nulls to be placed at the end of the sequence
+            if (a == null || b == null)
+                return b != null;
+            return comparator.compare((T) a, (T) b) > 0;
+        }
+    }
+
+    /**
+     * Create a mermaid graph for the current state of the heap. Used to create visuals for documentation/slides.
+     */
+    String toMermaid()
+    {
+        StringBuilder builder = new StringBuilder();
+        builder.append("flowchart\n");
+        int size = size();
+        for (int i = 0; i < size; ++i)
+            builder.append("  s" + i + "(" + heap[i] + ")\n");
+        builder.append("\n");
+        for (int i = 0; i * 2 + 1 < size; ++i)
+        {
+            builder.append("  s" + i + " ---|<=| s" + (i * 2 + 1) + "\n");
+            if (i * 2 + 2 < size)
+                builder.append("  s" + i + " ---|<=| s" + (i * 2 + 2) + "\n");
+        }
+        return builder.toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/BloomFilter.java b/src/java/org/apache/cassandra/utils/BloomFilter.java
index a95d131a3913..e1ba5794138d 100644
--- a/src/java/org/apache/cassandra/utils/BloomFilter.java
+++ b/src/java/org/apache/cassandra/utils/BloomFilter.java
@@ -23,13 +23,22 @@
 
 import io.netty.util.concurrent.FastThreadLocal;
 import net.nicoulaj.compilecommand.annotations.Inline;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable;
 import org.apache.cassandra.utils.obs.IBitSet;
+import org.apache.cassandra.utils.obs.MemoryLimiter;
+
+import static org.apache.cassandra.metrics.RestorableMeter.AVAILABLE_WINDOWS;
 
 public class BloomFilter extends WrappedSharedCloseable implements IFilter
 {
+    private static final long maxMemory = CassandraRelevantProperties.BF_MAX_MEMORY_MB.getLong() << 20;
+
+    public static final MemoryLimiter memoryLimiter = new MemoryLimiter(maxMemory != 0 ? maxMemory : Long.MAX_VALUE,
+                                                                        "Allocating %s for Bloom filter would reach max of %s (current %s)");
+
     private final static FastThreadLocal<long[]> reusableIndexes = new FastThreadLocal<long[]>()
     {
         @Override
@@ -56,6 +65,41 @@ private BloomFilter(BloomFilter copy)
         this.bitset = copy.bitset;
     }
 
+    /**
+     * @return true if sstable's bloom filter should be deserialized on read instead of when opening sstable. This
+     *         doesn't affect flushed sstable because there is bloom filter deserialization
+     */
+    public static boolean lazyLoading()
+    {
+        return CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING.getBoolean();
+    }
+
+    /**
+     * @return sstable hits per second to determine if a sstable is hot. 0 means BF should be loaded immediately on read.
+     *
+     * Note that when WINDOW <= 0, this is used as absolute primary index access count.
+     */
+    public static long lazyLoadingThreshold()
+    {
+        return CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_THRESHOLD.getInt();
+    }
+
+    /**
+     * @return Window of time by minute, available: 1 (default), 5, 15, 120.
+     *
+     * Note that if <= 0 then we use threshold as the absolute count
+     */
+    public static int lazyLoadingWindow()
+    {
+        int window = CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_WINDOW.getInt();
+        if (window >= 1 && !AVAILABLE_WINDOWS.contains(window))
+            throw new IllegalArgumentException(String.format("Found invalid %s=%s, available windows: %s",
+                                                             CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_WINDOW.getKey(),
+                                                             window,
+                                                             AVAILABLE_WINDOWS));
+        return window;
+    }
+
     public long serializedSize(boolean old)
     {
         return BloomFilterSerializer.forVersion(old).serializedSize(this);
@@ -171,4 +215,5 @@ public void addTo(Ref.IdentityCollection identities)
         super.addTo(identities);
         bitset.addTo(identities);
     }
+
 }
diff --git a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
index 91ec13f53c52..f2a4fbb1dc4c 100644
--- a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
+++ b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
@@ -19,15 +19,20 @@
 
 import java.io.IOException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IGenericSerializer;
 import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.utils.obs.IBitSet;
+import org.apache.cassandra.utils.obs.MemoryLimiter;
 import org.apache.cassandra.utils.obs.OffHeapBitSet;
 
 public final class BloomFilterSerializer implements IGenericSerializer<BloomFilter, DataInputStreamPlus, DataOutputStreamPlus>
 {
+    private final static Logger logger = LoggerFactory.getLogger(BloomFilterSerializer.class);
     public final static BloomFilterSerializer newFormatInstance = new BloomFilterSerializer(false);
     public final static BloomFilterSerializer oldFormatInstance = new BloomFilterSerializer(true);
 
@@ -71,9 +76,22 @@ public long serializedSize(BloomFilter bf)
 
     @Override
     public BloomFilter deserialize(DataInputStreamPlus in) throws IOException
+    {
+        return deserialize(in, BloomFilter.memoryLimiter);
+    }
+
+    public BloomFilter deserialize(DataInputStreamPlus in, MemoryLimiter memoryLimiter) throws IOException
     {
         int hashes = in.readInt();
-        IBitSet bs = OffHeapBitSet.deserialize(in, oldFormat);
+        IBitSet bs;
+        try
+        {
+            bs = OffHeapBitSet.deserialize(in, oldFormat, memoryLimiter);
+        }
+        catch (MemoryLimiter.ReachedMemoryLimitException | OutOfMemoryError e)
+        {
+            throw new RuntimeException("Out of native memory occured, You can avoid it by increasing the system ram space or by increasing bloom_filter_fp_chance.");
+        }
 
         return new BloomFilter(hashes, bs);
     }
diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
index 156c1c4bc840..9a654d088db4 100644
--- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
@@ -41,6 +41,7 @@
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.FileUtils;
 
 /**
@@ -309,6 +310,26 @@ public static void copyBytes(ByteBuffer src, int srcPos, ByteBuffer dst, int dst
         FastByteOperations.copy(src, srcPos, dst, dstPos, length);
     }
 
+    /**
+     * Transfer bytes from one ByteBuffer to another.
+     * This function acts as System.arrayCopy() but for ByteBuffers.
+     *
+     * @param src the source ByteBuffer
+     * @param srcPos starting position in the source ByteBuffer
+     * @param dst the destination ByteBuffer
+     * @param dstPos starting position in the destination ByteBuffer
+     * @param length the number of bytes to copy
+     */
+    public static void arrayCopy(ByteBuffer src, int srcPos, ByteBuffer dst, int dstPos, int length)
+    {
+        FastByteOperations.copy(src, srcPos, dst, dstPos, length);
+    }
+
+    public static void arrayCopy(ByteBuffer src, int srcPos, byte[] dst, int dstPos, int length)
+    {
+        FastByteOperations.copy(src, srcPos, dst, dstPos, length);
+    }
+
     public static int put(ByteBuffer src, ByteBuffer trg)
     {
         int length = Math.min(src.remaining(), trg.remaining());
@@ -437,6 +458,28 @@ public static void skipShortLength(DataInputPlus in) throws IOException
         in.skipBytesFully(skip);
     }
 
+    /**
+     * Returns true if the buffer at the current position in the input matches given buffer.
+     * If true, the input is positioned at the end of the consumed buffer.
+     * If false, the position of the input is undefined.
+     * <p>
+     * The matched buffer is unchanged
+     *
+     * @throws IOException
+     */
+    public static boolean equalsWithShortLength(FileDataInput in, ByteBuffer toMatch) throws IOException
+    {
+        int length = readShortLength(in);
+        if (length != toMatch.remaining())
+            return false;
+        int limit = toMatch.limit();
+        for (int i = toMatch.position(); i < limit; ++i)
+            if (toMatch.get(i) != in.readByte())
+                return false;
+
+        return true;
+    }
+
     public static ByteBuffer read(DataInput in, int length) throws IOException
     {
         if (length == 0)
@@ -917,4 +960,23 @@ public static void readFully(FileChannel channel, ByteBuffer dst, long position)
             position += read;
         }
     }
-}
\ No newline at end of file
+
+    /**
+     * Essentially the same as {@link #bytesToHex(ByteBuffer)} (though it prepends "0x" for clarity) but takes care of
+     * not output a string too long if the value is too big. This is to be used for error/debug message where we don't
+     * want to blow things up.
+     *
+     * @param bytes the bytes to convert to hexadecimal string.
+     * @return a string representation of {@code bytes} that may be only partial if {@code bytes} is too big.
+     */
+    public static String toDebugHexString(ByteBuffer bytes)
+    {
+        int maxSize = 50; // kind of arbitrary tbh but that's not hugely important
+        if (bytes.remaining() > maxSize)
+        {
+            bytes = bytes.duplicate();
+            bytes.limit(bytes.position() + maxSize);
+        }
+        return "0x" + bytesToHex(bytes);
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/CloseableIterator.java b/src/java/org/apache/cassandra/utils/CloseableIterator.java
index 32de799ba93f..069095d6c8f2 100644
--- a/src/java/org/apache/cassandra/utils/CloseableIterator.java
+++ b/src/java/org/apache/cassandra/utils/CloseableIterator.java
@@ -17,54 +17,85 @@
  */
 package org.apache.cassandra.utils;
 
+import java.io.Closeable;
+import java.util.Collections;
 import java.util.Iterator;
-import java.util.NoSuchElementException;
+
+import org.apache.cassandra.io.util.FileUtils;
+
 
 // so we can instantiate anonymous classes implementing both interfaces
 public interface CloseableIterator<T> extends Iterator<T>, AutoCloseable
 {
     public void close();
 
-    public static <T> CloseableIterator<T> wrap(Iterator<T> iter)
+    CloseableIterator<Object> EMPTY = CloseableIterator.wrap(Collections.emptyIterator());
+
+    /**
+     * Returns an empty {@link CloseableIterator}.
+     */
+    @SuppressWarnings("unchecked")
+    static <T> CloseableIterator<T> emptyIterator()
     {
-        return new CloseableIterator<T>()
+        return (CloseableIterator<T>) EMPTY;
+    }
+
+    /**
+     * Wraps an {@link Iterator} making it a {@link CloseableIterator}.
+     */
+    static <T> CloseableIterator<T> wrap(Iterator<T> iterator)
+    {
+        return new CloseableIterator<>()
         {
-            public void close()
+            public boolean hasNext()
             {
-                // noop
+                return iterator.hasNext();
             }
 
-            public boolean hasNext()
+            public T next()
             {
-                return iter.hasNext();
+                return iterator.next();
             }
 
-            public T next()
+            public void remove()
+            {
+                iterator.remove();
+            }
+
+            public void close()
             {
-                return iter.next();
             }
         };
     }
 
-    public static <T> CloseableIterator<T> empty()
+    /**
+     * Pairs a {@link CloseableIterator} and an {@link AutoCloseable} so that the latter is closed when the former is
+     * closed.
+     */
+    static <T> CloseableIterator<T> withOnClose(CloseableIterator<T> iterator, Closeable onClose)
     {
-        return new CloseableIterator<T>()
+        return new CloseableIterator<>()
         {
-            public void close()
+            public boolean hasNext()
             {
-                // noop
+                return iterator.hasNext();
             }
 
-            public boolean hasNext()
+            public T next()
             {
-                return false;
+                return iterator.next();
             }
 
-            public T next()
+            public void remove()
+            {
+                iterator.remove();
+            }
+
+            public void close()
             {
-                throw new NoSuchElementException();
+                iterator.close();
+                FileUtils.closeQuietly(onClose);
             }
         };
     }
-
 }
diff --git a/src/java/org/apache/cassandra/utils/Collections3.java b/src/java/org/apache/cassandra/utils/Collections3.java
new file mode 100644
index 000000000000..3dd899b7f13d
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/Collections3.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Collection;
+
+import com.google.common.collect.ImmutableList;
+
+public class Collections3
+{
+    public static <T> ImmutableList<T> withAppended(Iterable<T> list, T... elements)
+    {
+        if (elements.length == 0)
+            return ImmutableList.copyOf(list);
+
+        ImmutableList.Builder<T> builder = list instanceof Collection
+                                           ? ImmutableList.builderWithExpectedSize(((Collection<T>) list).size() + elements.length)
+                                           : ImmutableList.builder();
+        builder.addAll(list);
+        for (T element : elements)
+            builder.add(element);
+        return builder.build();
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/DseLegacy.java b/src/java/org/apache/cassandra/utils/DseLegacy.java
new file mode 100644
index 000000000000..14c92b33d6c1
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/DseLegacy.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+/**
+ * used to mark DSE legacy interface
+ * which will be removed once we transition CNDB to CC core
+ */
+public @interface DseLegacy
+{
+}
diff --git a/src/java/org/apache/cassandra/utils/EstimatedHistogram.java b/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
index 198f92286f74..75730517851e 100644
--- a/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
+++ b/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
@@ -26,14 +26,17 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.ISerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.metrics.DecayingEstimatedHistogramReservoir;
 
 public class EstimatedHistogram implements DoubleToLongFunction
 {
     public static final EstimatedHistogramSerializer serializer = new EstimatedHistogramSerializer();
+    public static final boolean USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES = CassandraRelevantProperties.USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES.getBoolean();
 
     public static final int DEFAULT_BUCKET_COUNT = 90;
 
@@ -89,6 +92,14 @@ public EstimatedHistogram(long[] offsets, long[] bucketData)
     }
 
     public static long[] newOffsets(int size, boolean considerZeroes)
+    {
+        if (USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES)
+            return DecayingEstimatedHistogramReservoir.newDseOffsets(size, considerZeroes);
+        else
+            return newCassandraOffsets(size, considerZeroes);
+    }
+
+    public static long[] newCassandraOffsets(int size, boolean considerZeroes)
     {
         long[] result = new long[size + (considerZeroes ? 1 : 0)];
         int i = 0;
diff --git a/src/java/org/apache/cassandra/utils/ExpiringMemoizingSupplier.java b/src/java/org/apache/cassandra/utils/ExpiringMemoizingSupplier.java
index 7cc13782e3c1..d261bf52bd67 100644
--- a/src/java/org/apache/cassandra/utils/ExpiringMemoizingSupplier.java
+++ b/src/java/org/apache/cassandra/utils/ExpiringMemoizingSupplier.java
@@ -41,7 +41,7 @@ public class ExpiringMemoizingSupplier<T> implements Supplier<T>
     // The special value 0 means "not yet initialized".
     transient volatile long expirationNanos;
 
-    public static <T> Supplier<T> memoizeWithExpiration(Supplier<ReturnValue<T>> delegate, long duration, TimeUnit unit)
+    public static <T> ExpiringMemoizingSupplier<T> memoizeWithExpiration(Supplier<ReturnValue<T>> delegate, long duration, TimeUnit unit)
     {
         return new ExpiringMemoizingSupplier<>(delegate, duration, unit);
     }
diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java
index ca3444d4654a..ada5797eb14b 100644
--- a/src/java/org/apache/cassandra/utils/FBUtilities.java
+++ b/src/java/org/apache/cassandra/utils/FBUtilities.java
@@ -212,9 +212,9 @@ public static InetAddressAndPort getLocalAddressAndPort()
     {
         if (localInetAddressAndPort == null)
         {
-            if(DatabaseDescriptor.getRawConfig() == null)
+            if (DatabaseDescriptor.getRawConfig() == null)
             {
-                localInetAddressAndPort = InetAddressAndPort.getByAddress(getJustLocalAddress());
+                throw new AssertionError("Local address and port should never be accessed before initializing DatabaseDescriptor");
             }
             else
             {
@@ -249,7 +249,7 @@ public static InetAddressAndPort getBroadcastAddressAndPort()
         {
             if(DatabaseDescriptor.getRawConfig() == null)
             {
-                broadcastInetAddressAndPort = InetAddressAndPort.getByAddress(getJustBroadcastAddress());
+                throw new AssertionError("Broadcast address and port should never be accessed before initializing DatabaseDescriptor");
             }
             else
             {
@@ -560,6 +560,29 @@ public static <T> T waitOnFuture(Future<T> future)
         }
     }
 
+    // Used in CNDB
+    public static <T> T waitOnFuture(Future<T> future, Duration timeout)
+    {
+        Preconditions.checkArgument(!timeout.isNegative(), "Timeout must not be negative, provided %s", timeout);
+        try
+        {
+            return future.get(timeout.toNanos(), TimeUnit.NANOSECONDS);
+        }
+        catch (ExecutionException ee)
+        {
+            logger.info("Exception occurred in async code", ee);
+            throw Throwables.cleaned(ee);
+        }
+        catch (InterruptedException ie)
+        {
+            throw new AssertionError(ie);
+        }
+        catch (TimeoutException e)
+        {
+            throw new RuntimeException("Timeout - task did not finish in " + timeout, e);
+        }
+    }
+
     public static <T, F extends Future<? extends T>> F waitOnFirstFuture(Iterable<? extends F> futures)
     {
         return waitOnFirstFuture(futures, 100);
@@ -832,7 +855,8 @@ public static <T> CloseableIterator<T> closeableIterator(Iterator<T> iterator)
     /**
      * Convert the given size in bytes to a human-readable value using binary (i.e. 2^10-based) modifiers.
      * For example, 1.000KiB, 2.100GiB etc., up to 8.000 EiB.
-     * @param size      Number to convert.
+     *
+     * @param size Number to convert.
      */
     public static String prettyPrintMemory(long size)
     {
@@ -848,6 +872,7 @@ public static String prettyPrintMemory(long size)
     public static String prettyPrintMemory(long size, String separator)
     {
         int prefixIndex = (63 - Long.numberOfLeadingZeros(Math.abs(size))) / 10;
+        // Note: if size is 0 we get prefixIndex=0 because the division truncates towards 0 (i.e. -1/10 = 0).
         if (prefixIndex == 0)
             return String.format("%d%sB", size, separator);
         else
@@ -886,9 +911,9 @@ else if (prefixIndex > UNIT_PREFIXES_BASE || prefixIndex < -UNIT_PREFIXES_BASE)
 
     /**
      * Convert the given value to a human-readable string using decimal (i.e. 10^3-based) modifiers.
-     * If the number is outside the modifier range (i.e. < 1 qi or > 1 Qi), it will be printed as vEe where e is a
+     * If the number is outside the modifier range (i.e. < 1 q or > 1 Q), it will be printed as vEe where e is a
      * multiple of 3 with sign.
-     * For example, 1.000km, 2.100 ms, 10E+45, NaN.
+     * For example, 1.000km, 215.100 ms, 10.000E+45, NaN.
      * @param value     Number to convert.
      * @param separator Separator between the number and the (modified) unit.
      */
@@ -1020,6 +1045,17 @@ public static double parsePercent(String value)
             return Double.parseDouble(value);
     }
 
+    /**
+     * Parse an integer value, allowing the string "max" to mean Integer.MAX_VALUE.
+     */
+    public static int parseIntAllowingMax(String value)
+    {
+        if (value.equalsIgnoreCase("max"))
+            return Integer.MAX_VALUE;
+        else
+            return Integer.parseInt(value);
+    }
+
     /**
      * Starts and waits for the given @param pb to finish.
      * @throws java.io.IOException on non-zero exit code
@@ -1415,4 +1451,73 @@ static Semver parseKernelVersion(String versionString)
         }
         throw new IllegalArgumentException("Error while trying to parse kernel version - no version found");
     }
-}
\ No newline at end of file
+    
+    /**
+     * A class containing some debug methods to be added and removed manually when debugging problems
+     * like failing unit tests.
+     */
+    public static final class Debug
+    {
+        public static final class ThreadInfo
+        {
+            private final String name;
+            private final boolean isDaemon;
+            private final StackTraceElement[] stack;
+
+            public ThreadInfo()
+            {
+                this(Thread.currentThread());
+            }
+
+            public ThreadInfo(Thread thread)
+            {
+                this.name =  thread.getName();
+                this.isDaemon = thread.isDaemon();
+                this.stack = thread.getStackTrace();
+            }
+
+        }
+
+        public static String getStackTrace()
+        {
+            return getStackTrace(new ThreadInfo());
+        }
+
+        public static String getStackTrace(Thread thread)
+        {
+            return getStackTrace(new ThreadInfo(thread));
+        }
+
+        public static String getStackTrace(ThreadInfo threadInfo)
+        {
+            StringBuilder sb = new StringBuilder();
+            sb.append("Thread ")
+              .append(threadInfo.name)
+              .append(" (")
+              .append(threadInfo.isDaemon ? "daemon" : "non-daemon")
+              .append(")")
+              .append("\n");
+            for (StackTraceElement element : threadInfo.stack)
+            {
+                sb.append(element);
+                sb.append("\n");
+            }
+            return sb.toString();
+        }
+    }
+
+    public static void busyWaitWhile(Supplier<Boolean> condition)
+    {
+        while (condition.get())
+        {
+            try
+            {
+                Thread.sleep(1);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/FilterFactory.java b/src/java/org/apache/cassandra/utils/FilterFactory.java
index 50dbffb7d143..87db90bd8de9 100644
--- a/src/java/org/apache/cassandra/utils/FilterFactory.java
+++ b/src/java/org/apache/cassandra/utils/FilterFactory.java
@@ -24,12 +24,26 @@
 
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.utils.concurrent.Ref;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tags;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+import org.apache.cassandra.metrics.MetricNameFactory;
+import org.apache.cassandra.metrics.MicrometerMetrics;
 import org.apache.cassandra.utils.obs.IBitSet;
+import org.apache.cassandra.utils.obs.MemoryLimiter;
 import org.apache.cassandra.utils.obs.OffHeapBitSet;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.USE_MICROMETER;
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
 public class FilterFactory
 {
     public static final IFilter AlwaysPresent = AlwaysPresentFilter.instance;
+    // marker for lazy bloom filter
+    public static final IFilter AlwaysPresentForLazyLoading = new AlwaysPresentFilter();
+
+    public static final FilterFactoryMetrics metrics = FilterFactoryMetrics.create();
 
     private static final Logger logger = LoggerFactory.getLogger(FilterFactory.class);
     private static final long BITSET_EXCESS = 20;
@@ -39,6 +53,11 @@ public class FilterFactory
      *         probability for the given number of elements.
      */
     public static IFilter getFilter(long numElements, int targetBucketsPerElem)
+    {
+        return getFilter(numElements, targetBucketsPerElem, BloomFilter.memoryLimiter);
+    }
+
+    public static IFilter getFilter(long numElements, int targetBucketsPerElem, MemoryLimiter memoryLimiter)
     {
         int maxBucketsPerElement = Math.max(1, BloomCalculations.maxBucketsPerElement(numElements));
         int bucketsPerElement = Math.min(targetBucketsPerElem, maxBucketsPerElement);
@@ -47,31 +66,49 @@ public static IFilter getFilter(long numElements, int targetBucketsPerElem)
             logger.warn("Cannot provide an optimal BloomFilter for {} elements ({}/{} buckets per element).", numElements, bucketsPerElement, targetBucketsPerElem);
         }
         BloomCalculations.BloomSpecification spec = BloomCalculations.computeBloomSpec(bucketsPerElement);
-        return createFilter(spec.K, numElements, spec.bucketsPerElement);
+        return createFilter(spec.K, numElements, spec.bucketsPerElement, memoryLimiter);
     }
 
     /**
      * @return The smallest BloomFilter that can provide the given false
-     *         positive probability rate for the given number of elements.
-     *
-     *         Asserts that the given probability can be satisfied using this
-     *         filter.
+     * positive probability rate for the given number of elements.
+     * <p>
+     * Asserts that the given probability can be satisfied using this
+     * filter.
      */
     public static IFilter getFilter(long numElements, double maxFalsePosProbability)
+    {
+        return getFilter(numElements, maxFalsePosProbability, BloomFilter.memoryLimiter);
+    }
+
+    public static IFilter getFilter(long numElements, double maxFalsePosProbability, MemoryLimiter memoryLimiter)
     {
         assert maxFalsePosProbability <= 1.0 : "Invalid probability";
         if (maxFalsePosProbability == 1.0)
             return FilterFactory.AlwaysPresent;
         int bucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
         BloomCalculations.BloomSpecification spec = BloomCalculations.computeBloomSpec(bucketsPerElement, maxFalsePosProbability);
-        return createFilter(spec.K, numElements, spec.bucketsPerElement);
+        return createFilter(spec.K, numElements, spec.bucketsPerElement, memoryLimiter);
     }
 
-    private static IFilter createFilter(int hash, long numElements, int bucketsPer)
+    @SuppressWarnings("resource")
+    private static IFilter createFilter(int hash, long numElements, int bucketsPer, MemoryLimiter memoryLimiter)
     {
-        long numBits = (numElements * bucketsPer) + BITSET_EXCESS;
-        IBitSet bitset = new OffHeapBitSet(numBits);
-        return new BloomFilter(hash, bitset);
+        try
+        {
+            long numBits = (numElements * bucketsPer) + BITSET_EXCESS;
+            IBitSet bitset = new OffHeapBitSet(numBits, memoryLimiter);
+            return new BloomFilter(hash, bitset);
+        }
+        catch (MemoryLimiter.ReachedMemoryLimitException | OutOfMemoryError e)
+        {
+            logger.error("Failed to create new Bloom filter with {} elements: ({}) - " +
+                         "continuing but this will have severe performance implications. Consider increasing FP chance " +
+                         "(bloom_filter_fp_chance) or increasing system ram space or" +
+                         "lowering number of sstables through compaction", numElements, e.getMessage());
+            metrics.incrementOOMError();
+            return AlwaysPresent;
+        }
     }
 
     private static class AlwaysPresentFilter implements IFilter
@@ -125,4 +162,71 @@ public boolean isInformative()
             return false;
         }
     }
+
+    public interface FilterFactoryMetrics
+    {
+        static FilterFactoryMetrics create()
+        {
+            return USE_MICROMETER.getBoolean() ? new FilterFactoryMicormeterMetrics()
+                                                : new FilterFactoryCodahaleMetrics();
+        }
+
+        void incrementOOMError();
+
+        long oomErrors();
+    }
+
+    /**
+     * Metrics exposed in Prometheus friendly format
+     */
+    public static final class FilterFactoryMicormeterMetrics extends MicrometerMetrics implements FilterFactoryMetrics
+    {
+        public static final String METRICS_PREFIX = "bloom_filter";
+        public static final String OOM_ERRORS = METRICS_PREFIX + "_oom_errors";
+        private volatile Counter oomCounter;
+        public FilterFactoryMicormeterMetrics()
+        {
+            this.oomCounter = counter(OOM_ERRORS);
+        }
+
+        @Override
+        public synchronized void register(MeterRegistry newRegistry, Tags newTags)
+        {
+            super.register(newRegistry, newTags);
+            this.oomCounter = counter(OOM_ERRORS);
+        }
+
+        @Override
+        public void incrementOOMError()
+        {
+            oomCounter.increment();
+        }
+
+        @Override
+        public long oomErrors()
+        {
+            return (long) oomCounter.count();
+        }
+    }
+
+    /**
+     * Metrics exposed in Codahale format
+     */
+    public static final class FilterFactoryCodahaleMetrics implements FilterFactoryMetrics
+    {
+        private static final MetricNameFactory metricNameFactory = new DefaultNameFactory("BloomFilter");
+        private static final com.codahale.metrics.Counter oomCounter = Metrics.counter(metricNameFactory.createMetricName("OutOfMemory"));
+
+        @Override
+        public void incrementOOMError()
+        {
+            oomCounter.inc();
+        }
+
+        @Override
+        public long oomErrors()
+        {
+            return oomCounter.getCount();
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/utils/Flags.java b/src/java/org/apache/cassandra/utils/Flags.java
new file mode 100644
index 000000000000..0d2f22fcaa24
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/Flags.java
@@ -0,0 +1,57 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.utils;
+
+import net.nicoulaj.compilecommand.annotations.Inline;
+
+public interface Flags
+{
+    @Inline
+    static boolean isEmpty(int flags)
+    {
+        return flags == 0;
+    }
+
+    @Inline
+    static boolean containsAll(int flags, int testFlags)
+    {
+        return (flags & testFlags) == testFlags;
+    }
+
+    @Inline
+    static boolean contains(int flags, int testFlags)
+    {
+        return (flags & testFlags) != 0;
+    }
+
+    @Inline
+    static int add(int flags, int toAdd)
+    {
+        return flags | toAdd;
+    }
+
+    @Inline
+    static int remove(int flags, int toRemove)
+    {
+        return flags & ~toRemove;
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/GuidGenerator.java b/src/java/org/apache/cassandra/utils/GuidGenerator.java
index e06270fa1d94..01e91aae72b4 100644
--- a/src/java/org/apache/cassandra/utils/GuidGenerator.java
+++ b/src/java/org/apache/cassandra/utils/GuidGenerator.java
@@ -26,24 +26,29 @@
 
 public class GuidGenerator
 {
-    private static final Random myRand;
-    private static final SecureRandom mySecureRand;
-    private static final String s_id;
-
-    static
+    private static class Instance
     {
-        if (!JAVA_SECURITY_EGD.isPresent())
+        final static Instance instance = new Instance();
+
+        final Random myRand;
+        final SecureRandom mySecureRand;
+        final String s_id;
+
+        private Instance()
         {
-            JAVA_SECURITY_EGD.setString("file:/dev/urandom");
-        }
-        mySecureRand = new SecureRandom();
-        long secureInitializer = mySecureRand.nextLong();
-        myRand = new Random(secureInitializer);
-        try {
-            s_id = FBUtilities.getLocalAddressAndPort().toString();
-        }
-        catch (RuntimeException e) {
-            throw new AssertionError(e);
+            if (!JAVA_SECURITY_EGD.isPresent())
+                JAVA_SECURITY_EGD.setString("file:/dev/urandom");
+            mySecureRand = new SecureRandom();
+            long secureInitializer = mySecureRand.nextLong();
+            myRand = new Random(secureInitializer);
+            try
+            {
+                s_id = FBUtilities.getLocalAddressAndPort().toString();
+            }
+            catch (RuntimeException e)
+            {
+                throw new AssertionError(e);
+            }
         }
     }
 
@@ -59,7 +64,7 @@ public static String guid()
             sb.append(Integer.toHexString(b));
         }
 
-        return convertToStandardFormat( sb.toString() );
+        return convertToStandardFormat(sb.toString());
     }
 
     public static String guidToString(byte[] bytes)
@@ -72,7 +77,7 @@ public static String guidToString(byte[] bytes)
             sb.append(Integer.toHexString(b));
         }
 
-        return convertToStandardFormat( sb.toString() );
+        return convertToStandardFormat(sb.toString());
     }
 
     public static ByteBuffer guidAsBytes(Random random, String hostId, long time)
@@ -91,14 +96,13 @@ public static ByteBuffer guidAsBytes(Random random, String hostId, long time)
 
     public static ByteBuffer guidAsBytes()
     {
-        return guidAsBytes(myRand, s_id, currentTimeMillis());
+        return guidAsBytes(Instance.instance.myRand, Instance.instance.s_id, currentTimeMillis());
     }
 
     /*
-        * Convert to the standard format for GUID
-        * Example: C2FEEEAC-CFCD-11D1-8B05-00600806D9B6
-    */
-
+     * Convert to the standard format for GUID
+     * Example: C2FEEEAC-CFCD-11D1-8B05-00600806D9B6
+     */
     private static String convertToStandardFormat(String valueAfterMD5)
     {
         String raw = valueAfterMD5.toUpperCase();
diff --git a/src/java/org/apache/cassandra/utils/HeapUtils.java b/src/java/org/apache/cassandra/utils/HeapUtils.java
index 38a696990845..d67961c093e5 100644
--- a/src/java/org/apache/cassandra/utils/HeapUtils.java
+++ b/src/java/org/apache/cassandra/utils/HeapUtils.java
@@ -190,9 +190,9 @@ private static void logProcessOutput(Process p) throws IOException
      * Retrieves the process ID or <code>null</code> if the process ID cannot be retrieved.
      * @return the process ID or <code>null</code> if the process ID cannot be retrieved.
      */
-    private static Long getProcessId()
+    public static Long getProcessId()
     {
-        long pid = NativeLibrary.getProcessID();
+        long pid = INativeLibrary.instance.getProcessID();
         if (pid >= 0)
             return pid;
 
diff --git a/src/java/org/apache/cassandra/utils/IFilter.java b/src/java/org/apache/cassandra/utils/IFilter.java
index f06ae6e86491..e41ba741fe83 100644
--- a/src/java/org/apache/cassandra/utils/IFilter.java
+++ b/src/java/org/apache/cassandra/utils/IFilter.java
@@ -26,9 +26,7 @@ public interface IFilter extends SharedCloseable
 {
     interface FilterKey
     {
-        /**
-         * Places the murmur3 hash of the key in the given long array of size at least two.
-         */
+        /** Places the murmur3 hash of the key in the first two elements of the given long array. */
         void filterHash(long[] dest);
 
         default short filterHashLowerBits()
diff --git a/src/java/org/apache/cassandra/utils/IMergeIterator.java b/src/java/org/apache/cassandra/utils/IMergeIterator.java
deleted file mode 100644
index e45b8976e9fe..000000000000
--- a/src/java/org/apache/cassandra/utils/IMergeIterator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import java.util.Iterator;
-
-public interface IMergeIterator<In, Out> extends CloseableIterator<Out>
-{
-
-    Iterable<? extends Iterator<In>> iterators();
-}
diff --git a/src/java/org/apache/cassandra/utils/INativeLibrary.java b/src/java/org/apache/cassandra/utils/INativeLibrary.java
new file mode 100644
index 000000000000..5d9b8aae2e92
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/INativeLibrary.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+import java.io.FileDescriptor;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.channels.FileChannel;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.util.File;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_NATIVE_LIBRARY;
+
+public interface INativeLibrary
+{
+    static final Logger logger = LoggerFactory.getLogger(INativeLibrary.class);
+
+    INativeLibrary instance = !CUSTOM_NATIVE_LIBRARY.isPresent()
+                             ? new NativeLibrary()
+                             : FBUtilities.construct(CUSTOM_NATIVE_LIBRARY.getString(), "native library");
+
+    public enum OSType
+    {
+        LINUX,
+        MAC,
+        WINDOWS,
+        AIX,
+        OTHER;
+    }
+
+    /**
+     * @return true if current OS type is same the provided type
+     */
+    boolean isOS(INativeLibrary.OSType type);
+
+    /**
+     * Checks if the library has been successfully linked.
+     * @return {@code true} if the library has been successfully linked, {@code false} otherwise.
+     */
+    boolean isAvailable();
+
+    /**
+     * @return true if jna memory is lockable
+     */
+    boolean jnaMemoryLockable();
+
+    /**
+     * try to lock JVM memory to avoid memory being swapped out
+     */
+    void tryMlockall();
+
+    /**
+     * try to advice OS to to free cached pages associated with the specified region.
+     */
+    void trySkipCache(File f, long offset, long len);
+
+    /**
+     * try to advice OS to to free cached pages associated with the specified region.
+     */
+    void trySkipCache(int fd, long offset, long len, String fileName);
+
+    /**
+     * try to advice OS to to free cached pages associated with the specified region.
+     */
+    void trySkipCache(int fd, long offset, int len, String fileName);
+
+    /**
+     * advise the OS to expect random i/o performed against the mapped address
+     */
+    void adviseRandom(MappedByteBuffer buffer, long len, String s);
+
+    /**
+     * execute OS file control command
+     */
+    int tryFcntl(int fd, int command, int flags);
+
+    /**
+     * try to open given directory
+     */
+    int tryOpenDirectory(File path);
+
+    /**
+     * try to open given directory
+     */
+    int tryOpenDirectory(String path);
+
+    /**
+     * try fsync on given file
+     */
+    void trySync(int fd);
+
+    /**
+     * try to close given file
+     */
+    void tryCloseFD(int fd);
+
+    /**
+     * @return file descriptor for given async channel
+     */
+    int getfd(AsynchronousFileChannel channel);
+
+    /**
+     * @return file descriptor for given async channel
+     */
+    FileDescriptor getFileDescriptor(AsynchronousFileChannel channel);
+
+    /**
+     * @return file descriptor for given channel
+     */
+    int getfd(FileChannel channel);
+
+    /**
+     * @return file descriptor for given channel
+     */
+    @Nullable
+    FileDescriptor getFileDescriptor(FileChannel channel);
+
+    /**
+     * Get system file descriptor from FileDescriptor object.
+     * @param descriptor - FileDescriptor objec to get fd from
+     * @return file descriptor, -1 or error
+     */
+    int getfd(FileDescriptor descriptor);
+
+    /**
+     * @return the PID of the JVM or -1 if we failed to get the PID
+     */
+    long getProcessID();
+}
diff --git a/src/java/org/apache/cassandra/utils/ImmutableUtils.java b/src/java/org/apache/cassandra/utils/ImmutableUtils.java
new file mode 100644
index 000000000000..e0e20b1396b1
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/ImmutableUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Objects;
+
+import com.google.common.collect.ImmutableMap;
+
+public class ImmutableUtils
+{
+    public static <K, V> ImmutableMap<K, V> without(ImmutableMap<K, V> map, K keyToRemove)
+    {
+        if (map.containsKey(keyToRemove))
+        {
+            ImmutableMap.Builder<K, V> builder = ImmutableMap.builderWithExpectedSize(map.size() - 1);
+            map.forEach((k, v) -> {
+                if (!Objects.equals(k, keyToRemove))
+                    builder.put(k, v);
+            });
+            return builder.build();
+        }
+        return map;
+    }
+
+    public static <K, V> ImmutableMap<K, V> withAddedOrUpdated(ImmutableMap<K, V> map, K keyToAdd, V valueToAdd)
+    {
+        V currentValue = map.get(keyToAdd);
+        if (Objects.equals(currentValue, valueToAdd))
+            return map;
+
+        ImmutableMap.Builder<K, V> builder;
+        if (currentValue != null)
+        {
+            builder = ImmutableMap.builderWithExpectedSize(map.size());
+            map.forEach((k, v) -> {
+                if (Objects.equals(k, keyToAdd))
+                    builder.put(keyToAdd, valueToAdd);
+                else
+                    builder.put(k, v);
+            });
+        }
+        else
+        {
+            builder = ImmutableMap.builderWithExpectedSize(map.size() + 1);
+            builder.putAll(map);
+            builder.put(keyToAdd, valueToAdd);
+        }
+        return builder.build();
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/IntMerger.java b/src/java/org/apache/cassandra/utils/IntMerger.java
new file mode 100644
index 000000000000..4901d80ea320
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/IntMerger.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+
+import java.io.IOException;
+import java.lang.reflect.Array;
+import java.util.Collection;
+import java.util.function.Consumer;
+
+/**
+ * <p>
+ * Integer version of the {@link Merger} class, throwing away the {@code equalParent} optimization as it is
+ * not beneficial for the integer comparisons.
+ * </p><p>
+ * This class merges sorted integer streams by direct value comparison. For simplicity, it assumes the user has
+ * some external method of recognizing completion (e.g. using {@code Integer.MAX_VALUE} as sentinel). The class
+ * will not advance any of the source iterators until a request for data has been made. If a source's input has been
+ * processed and returned by the merger, the next value will only be requested when the merger is asked for the next.
+ * </p><p>
+ * The most straightforward way to implement merging of iterators is to use a {@code PriorityQueue},
+ * {@code poll} it to find the next item to consume, then {@code add} the iterator back after advancing.
+ * This is not very efficient as {@code poll} and {@code add} in all cases require at least
+ * {@code log(size)} comparisons and swaps (usually more than {@code 2*log(size)}) per consumed item, even
+ * if the input is suitable for fast iteration.
+ * </p><p>
+ * The implementation below makes use of the fact that replacing the top element in a binary heap can be
+ * done much more efficiently than separately removing it and placing it back, especially in the cases where
+ * the top iterator is to be used again very soon (e.g. when there are large sections of the output where
+ * only a limited number of input iterators overlap, which is normally the case in many practically useful
+ * situations, e.g. levelled compaction).
+ * </p><p>
+ * The implementation builds and maintains a binary heap of sources (stored in an array), where we do not
+ * add items after the initial construction. Instead we advance the smallest element (which is at the top
+ * of the heap) and push it down to find its place for its new position. Should this source be exhausted,
+ * we swap it with the last source in the heap and proceed by pushing that down in the heap.
+ * </p><p>
+ * Duplicate values in multiple sources are merged together, but duplicates in any individual source are not resolved.
+ * In the case where we have multiple sources with matching positions, {@link #advance} advances all equal sources and
+ * then restores the heap structure in one operation over the heap. The latter is done equivalently to the process of
+ * initial construction of a min-heap using back-to-front heapification as done in the classic heapsort algorithm. It
+ * only needs to heapify subheaps whose top item is advanced (i.e. one whose position matches the current), and we can
+ * do that recursively from bottom to top.
+ * </p><p>
+ * To make it easier to advance efficienty in single-sourced branches of tries, we extract the current smallest
+ * source (the head) separately, and start any advance with comparing that to the heap's first. When the smallest
+ * source remains the same (e.g. in branches coming from a single source) this makes it possible to advance with
+ * just one comparison instead of two at the expense of increasing the number by one in the general case.
+ * </p>
+ */
+public abstract class IntMerger<S>
+{
+    /**
+     * The current smallest item from the sources, tracked separately to improve performance in single-source
+     * sections of the input.
+     */
+    protected int headItem;
+    /**
+     * The source corresponding to the smallest item.
+     */
+    protected S headSource;
+
+    /**
+     * Binary heap of the current items from each source. The smallest element is at position 0.
+     * Every element i is smaller than or equal to its two children, i.e.<br/>
+     *     {@code item[i] <= item[i*2 + 1] && item[i] <= item[i*2 + 2]}
+     */
+    private final int[] items;
+    /**
+     * Binary heap of the sources.
+     * <p/>
+     * Sources are moved up and down the heap together with the items, i.e. the source index corresponds to the item
+     * index in these two arrays.
+     */
+    private final S[] sources;
+
+    boolean started;
+
+    /** Advance the given source by one item and return it. */
+    protected abstract int advanceSource(S s) throws IOException;
+    /** Skip the given source to the smallest item that is greater or equal to the given target, and return that item. */
+    protected abstract int skipSource(S s, int target) throws IOException;
+
+    protected IntMerger(Collection<? extends S> inputs, Class<S> sourceClass)
+    {
+        int count = inputs.size();
+
+        // Get sources for all inputs. Put one of them in head and the rest in the heap.
+        @SuppressWarnings("unchecked")
+        S[] s = (S[]) Array.newInstance(sourceClass, count - 1);
+        sources = s;
+        items = new int[count - 1];
+        int i = -1;
+        for (S source : inputs)
+        {
+            if (i >= 0)
+                sources[i] = source;
+            else
+                headSource = source;
+            ++i;
+        }
+        // Do not fetch items until requested.
+        started = false;
+    }
+
+    /**
+     * Advance the merged state and return the next item.
+     */
+    protected int advance() throws IOException
+    {
+        if (started)
+            advanceHeap(headItem, 0);
+        else
+            initializeHeap();
+
+        return headItem = maybeSwapHead(advanceSource(headSource));
+    }
+
+    /**
+     * Descend recursively in the subheap structure from the given index to all children that match the given position.
+     * On the way back from the recursion, advance each matching iterator and restore the heap invariants.
+     */
+    private void advanceHeap(int advancedItem, int index) throws IOException
+    {
+        if (index >= items.length)
+            return;
+
+        if (items[index] != advancedItem)
+            return;
+
+        // If any of the children are at the same position, they also need advancing and their subheap
+        // invariant to be restored.
+        advanceHeap(advancedItem, index * 2 + 1);
+        advanceHeap(advancedItem, index * 2 + 2);
+
+        // On the way back from the recursion, advance and form a heap from the (already advanced and well-formed)
+        // children and the current node.
+        advanceSourceAndHeapify(index);
+        // The heap rooted at index is now advanced and well-formed.
+    }
+
+
+    /**
+     * Advance the source at the given index and restore the heap invariant for its subheap, assuming its child subheaps
+     * are already well-formed.
+     */
+    private void advanceSourceAndHeapify(int index) throws IOException
+    {
+        // Advance the source.
+        S source = sources[index];
+        int next = advanceSource(source);
+
+        // Place current node in its proper position, pulling any smaller child up. This completes the construction
+        // of the subheap rooted at this index.
+        heapifyDown(source, next, index);
+    }
+
+    /**
+     * Push the given state down in the heap from the given index until it finds its proper place among
+     * the subheap rooted at that position.
+     */
+    private void heapifyDown(S source, int item, int index)
+    {
+        while (true)
+        {
+            int next = index * 2 + 1;
+            if (next >= items.length)
+                break;
+            // Select the smaller of the two children to push down to.
+            int nextItem = items[next];
+            if (next + 1 < items.length)
+            {
+                int nextP1Item = items[next + 1];
+                if (nextItem > nextP1Item)
+                {
+                    nextItem = nextP1Item;
+                    ++next;
+                }
+            }
+            // If the child is greater or equal, the invariant has been restored.
+            if (item <= nextItem)
+                break;
+            items[index] = nextItem;
+            sources[index] = sources[next];
+            index = next;
+        }
+        items[index] = item;
+        sources[index] = source;
+    }
+
+    /**
+     * Check if the head is greater than the top element in the heap, and if so, swap them and push down the new
+     * top until its proper place.
+     */
+    private int maybeSwapHead(int newHeadItem)
+    {
+        int heap0Item = items[0];
+        if (newHeadItem <= heap0Item)
+            return newHeadItem;   // head is still smallest
+
+        // otherwise we need to swap heap and heap[0]
+        S newHeap0 = headSource;
+        headSource = sources[0];
+        heapifyDown(newHeap0, newHeadItem, 0);
+        return heap0Item;
+    }
+
+    /**
+     * Initialize the heap for the retrieving the first item. We do this in a separate method because we don't yet have
+     * target items with which to compare in the methods above.
+     */
+    private void initializeHeap() throws IOException
+    {
+        for (int i = items.length - 1; i >= 0; --i)
+            advanceSourceAndHeapify(i);
+
+        started = true;
+    }
+
+    /**
+     * Skip the merged iterator to the smallest value equal to or greater than the target and return the next item.
+     */
+    protected int skipTo(int target) throws IOException
+    {
+        // We need to advance all sources that stand before the requested position.
+        // If a child source does not need to advance as it is at the skip position or greater, neither of the ones
+        // below it in the heap hierarchy do as they can't have an earlier position.
+        if (started)
+            skipHeap(target, 0);
+        else
+            initializeSkipping(target);
+
+        return headItem = maybeSwapHead(skipSource(headSource, target));
+    }
+
+
+    /**
+     * Descend recursively in the subheap structure from the given index to all children that are smaller than the
+     * requested position.
+     * On the way back from the recursion, skip each matching iterator and restore the heap invariants.
+     */
+    private void skipHeap(int target, int index) throws IOException
+    {
+        if (index >= items.length)
+            return;
+
+        if (items[index] >= target)
+            return;
+
+        // If any of the children are at a smaller position, they also need advancing and their subheap
+        // invariant to be restored.
+        skipHeap(target, index * 2 + 1);
+        skipHeap(target, index * 2 + 2);
+
+        // On the way back from the recursion, advance and form a heap from the (already advanced and well-formed)
+        // children and the current node.
+        skipSourceAndHeapify(index, target);
+
+        // The heap rooted at index is now advanced and well-formed.
+    }
+
+    /**
+     * Skip the source at the given index and restore the heap invariant for its subheap, assuming its child subheaps
+     * are already well-formed.
+     */
+    private void skipSourceAndHeapify(int index, int target) throws IOException
+    {
+        // Advance the source.
+        S source = sources[index];
+        int next = skipSource(source, target);
+
+        // Place current node in its proper position, pulling any smaller child up. This completes the construction
+        // of the subheap rooted at this index.
+        heapifyDown(source, next, index);
+    }
+
+    /**
+     * Initialize the heap by skipping to the given target. We do this in a separate method because we don't yet have
+     * items with which to compare in the methods above.
+     */
+    private void initializeSkipping(int target) throws IOException
+    {
+        for (int i = items.length - 1; i >= 0; --i)
+            skipSourceAndHeapify(i, target);
+
+        started = true;
+    }
+
+    /**
+     * Apply a method to all sources.
+     */
+    protected void applyToAllSources(Consumer<S> op)
+    {
+        for (int i = sources.length - 1; i >= 0; --i)
+            op.accept(sources[i]);
+        op.accept(headSource);
+    }
+
+    // currentItem(), forEachCurrentSource() methods can be easily implemented if required
+}
+
diff --git a/src/java/org/apache/cassandra/utils/IteratorWithLowerBound.java b/src/java/org/apache/cassandra/utils/IteratorWithLowerBound.java
deleted file mode 100644
index 85eeede2e7cc..000000000000
--- a/src/java/org/apache/cassandra/utils/IteratorWithLowerBound.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.utils;
-
-public interface IteratorWithLowerBound<In>
-{
-    In lowerBound();
-}
diff --git a/src/java/org/apache/cassandra/utils/JVMKiller.java b/src/java/org/apache/cassandra/utils/JVMKiller.java
new file mode 100644
index 000000000000..450a7cadbbb1
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/JVMKiller.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+/**
+ * An interface implemented to kill the JVM abnormally.
+ *
+ * It's currently implemented by {@link JVMStabilityInspector.Killer}
+ * or tests.
+ */
+public interface JVMKiller
+{
+    /**
+     * Kills the JVM in a verbose fashion.
+     *
+     * @param t - the error that has caused the JVM to be killed
+     */
+    default void killJVM(Throwable t)
+    {
+        killJVM(t, false);
+    }
+
+    /**
+     * Kills the JVM.
+     *
+     * @param t - the error that has caused the JVM to be killed
+     * @param quiet - whether the error should be logged verbosely
+     */
+    public void killJVM(Throwable t, boolean quiet);
+}
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
index a396ef994722..417d4ad5002e 100644
--- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -26,6 +26,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
@@ -54,14 +55,25 @@
 public final class JVMStabilityInspector
 {
     private static final Logger logger = LoggerFactory.getLogger(JVMStabilityInspector.class);
-    private static Killer killer = new Killer();
+    private static JVMKiller killer = new Killer();
 
     private static Object lock = new Object();
     private static boolean printingHeapHistogram;
+    private static volatile Consumer<Throwable> globalHandler;
+    private static volatile Consumer<Throwable> diskHandler;
+    private static volatile Function<String, Consumer<Throwable>> commitLogHandler;
+    private static final List<Pair<Thread, Runnable>> shutdownHooks = new ArrayList<>(1);
 
     // It is used for unit test
     public static OnKillHook killerHook;
 
+    static
+    {
+        setGlobalErrorHandler(JVMStabilityInspector::defaultGlobalErrorHandler);
+        setDiskErrorHandler(JVMStabilityInspector::inspectDiskError);
+        setCommitLogErrorHandler(JVMStabilityInspector::createDefaultCommitLogErrorHandler);
+    }
+
     private JVMStabilityInspector() {}
 
     public static void uncaughtException(Thread thread, Throwable t)
@@ -78,6 +90,27 @@ public static void uncaughtException(Thread thread, Throwable t)
         JVMStabilityInspector.inspectThrowable(t);
     }
 
+    public static void setGlobalErrorHandler(Consumer<Throwable> errorHandler)
+    {
+        globalHandler = errorHandler;
+    }
+
+    @VisibleForTesting
+    public static Consumer<Throwable> getGlobalErrorHandler()
+    {
+        return globalHandler;
+    }
+
+    public static void setDiskErrorHandler(Consumer<Throwable> errorHandler)
+    {
+        diskHandler = errorHandler;
+    }
+
+    public static void setCommitLogErrorHandler(Function<String, Consumer<Throwable>> errorHandler)
+    {
+        commitLogHandler = errorHandler;
+    }
+
     /**
      * Certain Throwables and Exceptions represent "Die" conditions for the server.
      * This recursively checks the input Throwable's cause hierarchy until null.
@@ -86,12 +119,12 @@ public static void uncaughtException(Thread thread, Throwable t)
      */
     public static void inspectThrowable(Throwable t) throws OutOfMemoryError
     {
-        inspectThrowable(t, JVMStabilityInspector::inspectDiskError);
+        inspectThrowable(t, diskHandler);
     }
 
-    public static void inspectCommitLogThrowable(Throwable t)
+    public static void inspectCommitLogThrowable(String message, Throwable t)
     {
-        inspectThrowable(t, JVMStabilityInspector::inspectCommitLogError);
+        inspectThrowable(t, commitLogHandler.apply(message));
     }
 
     private static void inspectDiskError(Throwable t)
@@ -102,7 +135,20 @@ else if (t instanceof FSError)
             FileUtils.handleFSError((FSError) t);
     }
 
-    public static void inspectThrowable(Throwable t, Consumer<Throwable> fn) throws OutOfMemoryError
+    public static void inspectThrowable(Throwable t, Consumer<Throwable> additionalHandler) throws OutOfMemoryError
+    {
+        if (t == null)
+            return;
+        globalHandler.accept(t);
+        additionalHandler.accept(t);
+
+        for (Throwable suppressed : t.getSuppressed())
+            inspectThrowable(suppressed, additionalHandler);
+
+        inspectThrowable(t.getCause(), additionalHandler);
+    }
+
+    private static void defaultGlobalErrorHandler(Throwable t)
     {
         boolean isUnstable = false;
         if (t instanceof OutOfMemoryError)
@@ -122,7 +168,7 @@ public static void inspectThrowable(Throwable t, Consumer<Throwable> fn) throws
 
             logger.error("OutOfMemory error letting the JVM handle the error:", t);
 
-            StorageService.instance.removeShutdownHook();
+            removeShutdownHooks();
 
             forceHeapSpaceOomMaybe((OutOfMemoryError) t);
 
@@ -154,20 +200,8 @@ else if (t instanceof UnrecoverableIllegalStateException)
         {
             if (!StorageService.instance.isDaemonSetupCompleted())
                 FileUtils.handleStartupFSError(t);
-            killer.killCurrentJVM(t);
+            killer.killJVM(t);
         }
-
-        try
-        {
-            fn.accept(t);
-        }
-        catch (Exception | Error e)
-        {
-            logger.warn("Unexpected error while handling unexpected error", e);
-        }
-
-        if (t.getCause() != null)
-            inspectThrowable(t.getCause(), fn);
     }
 
     private static final Set<String> FORCE_HEAP_OOM_IGNORE_SET = ImmutableSet.of("Java heap space", "GC Overhead limit exceeded");
@@ -194,20 +228,25 @@ private static void forceHeapSpaceOomMaybe(OutOfMemoryError oom)
         }
     }
 
+    private static Consumer<Throwable> createDefaultCommitLogErrorHandler(String message)
+    {
+        return JVMStabilityInspector::inspectCommitLogError;
+    }
+    
     private static void inspectCommitLogError(Throwable t)
     {
         if (!StorageService.instance.isDaemonSetupCompleted())
         {
             logger.error("Exiting due to error while processing commit log during initialization.", t);
-            killer.killCurrentJVM(t, true);
+            killer.killJVM(t, true);
         }
         else if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
-            killer.killCurrentJVM(t);
+            killer.killJVM(t);
     }
 
     public static void killCurrentJVM(Throwable t, boolean quiet)
     {
-        killer.killCurrentJVM(t, quiet);
+        killer.killJVM(t, quiet);
     }
 
     public static void userFunctionTimeout(Throwable t)
@@ -216,10 +255,10 @@ public static void userFunctionTimeout(Throwable t)
         {
             case die:
                 // policy to give 250ms grace time to
-                ScheduledExecutors.nonPeriodicTasks.schedule(() -> killer.killCurrentJVM(t), 250, TimeUnit.MILLISECONDS);
+                ScheduledExecutors.nonPeriodicTasks.schedule(() -> killer.killJVM(t), 250, TimeUnit.MILLISECONDS);
                 break;
             case die_immediate:
-                killer.killCurrentJVM(t);
+                killer.killJVM(t);
                 break;
             case ignore:
                 logger.error(t.getMessage());
@@ -227,31 +266,48 @@ public static void userFunctionTimeout(Throwable t)
         }
     }
 
+    public static void registerShutdownHook(Thread hook, Runnable runOnHookRemoved)
+    {
+        Runtime.getRuntime().addShutdownHook(hook);
+        shutdownHooks.add(Pair.create(hook, runOnHookRemoved));
+    }
+
+    public static void removeShutdownHooks()
+    {
+        Throwable err = null;
+        for (Pair<Thread, Runnable> hook : shutdownHooks)
+        {
+            err = Throwables.perform(err,
+                                     () -> Runtime.getRuntime().removeShutdownHook(hook.left),
+                                     hook.right::run);
+        }
+
+        if (err != null)
+            logger.error("Got error(s) when removing shutdown hook(s): {}", err.getMessage(), err);
+
+        shutdownHooks.clear();
+    }
+
     @VisibleForTesting
-    public static Killer replaceKiller(Killer newKiller)
+    public static JVMKiller replaceKiller(JVMKiller newKiller)
     {
-        Killer oldKiller = JVMStabilityInspector.killer;
+        JVMKiller oldKiller = JVMStabilityInspector.killer;
         JVMStabilityInspector.killer = newKiller;
         return oldKiller;
     }
 
+    public static JVMKiller killer()
+    {
+        return killer;
+    }
+
     @VisibleForTesting
-    public static class Killer
+    public static class Killer implements JVMKiller
     {
         private final AtomicBoolean killing = new AtomicBoolean();
 
-        /**
-        * Certain situations represent "Die" conditions for the server, and if so, the reason is logged and the current JVM is killed.
-        *
-        * @param t
-        *      The Throwable to log before killing the current JVM
-        */
-        protected void killCurrentJVM(Throwable t)
-        {
-            killCurrentJVM(t, false);
-        }
-
-        protected void killCurrentJVM(Throwable t, boolean quiet)
+        @Override
+        public void killJVM(Throwable t, boolean quiet)
         {
             if (!quiet)
             {
@@ -263,7 +319,7 @@ protected void killCurrentJVM(Throwable t, boolean quiet)
 
             if (doExit && killing.compareAndSet(false, true))
             {
-                StorageService.instance.removeShutdownHook();
+                removeShutdownHooks();
                 System.exit(100);
             }
         }
diff --git a/src/java/org/apache/cassandra/utils/JsonUtils.java b/src/java/org/apache/cassandra/utils/JsonUtils.java
index 1cdc13c55cba..f0bcb2c628f9 100644
--- a/src/java/org/apache/cassandra/utils/JsonUtils.java
+++ b/src/java/org/apache/cassandra/utils/JsonUtils.java
@@ -27,7 +27,7 @@
 import java.util.Map;
 
 import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.util.BufferRecyclers;
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -60,8 +60,7 @@ private JsonUtils()
      */
     public static String quoteAsJsonString(String s)
     {
-        // In future should update to directly use `JsonStringEncoder.getInstance()` but for now:
-        return new String(BufferRecyclers.getJsonStringEncoder().quoteAsString(s));
+        return new String(JsonStringEncoder.getInstance().quoteAsString(s));
     }
 
     public static Object decodeJson(byte[] json)
diff --git a/src/java/org/apache/cassandra/utils/LucenePriorityQueue.java b/src/java/org/apache/cassandra/utils/LucenePriorityQueue.java
new file mode 100644
index 000000000000..54319e40d37d
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/LucenePriorityQueue.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Comparator;
+
+import org.apache.lucene.util.PriorityQueue;
+
+/**
+ * Version of lucene's priority queue that accepts a comparator.
+ * <p>
+ * This priority queue has several performance advantages compared to java's:
+ * <ul>
+ * <li>it can efficiently order items added through {@code addAll} using the O(n) bottom-up heapification process</li>
+ * <li>it implements an {@code updateTop} method which is much more efficient than {@code poll} + {@code add} for e.g.
+ * advancing a source and keeping it in the queue</li>
+ * </ul>
+ * <p>
+ * Use this class when elements need to be added to the queue after the initial construction. In case all elements are
+ * predetermined, a {@link SortingIterator} is usually preferable as it also implements skipping and deduplication.
+ * When sorting multiple iterators into one, a {@link MergeIterator} or the underlying {@link Merger} may provide a
+ * simpler solution. Finally, when operating on integer iterators, we have a special-case {@link IntMerger}.
+ */
+public class LucenePriorityQueue<T> extends PriorityQueue<T>
+{
+    final Comparator<? super T> comparator;
+
+    public LucenePriorityQueue(int size, Comparator<? super T> comparator)
+    {
+        super(size);
+        this.comparator = comparator;
+    }
+
+    @Override
+    protected boolean lessThan(T t, T t1)
+    {
+        return comparator.compare(t, t1) < 0;
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/MergeIterator.java b/src/java/org/apache/cassandra/utils/MergeIterator.java
index 1dd1f7833bd1..3a8add3e0c47 100644
--- a/src/java/org/apache/cassandra/utils/MergeIterator.java
+++ b/src/java/org/apache/cassandra/utils/MergeIterator.java
@@ -17,475 +17,158 @@
  */
 package org.apache.cassandra.utils;
 
-import java.util.*;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
+
+import com.google.common.base.Preconditions;
 
 /** Merges sorted input iterators which individually contain unique items. */
-public abstract class MergeIterator<In,Out> extends AbstractIterator<Out> implements IMergeIterator<In, Out>
+public abstract class MergeIterator
 {
-    protected final Reducer<In,Out> reducer;
-    protected final List<? extends Iterator<In>> iterators;
-
-    protected MergeIterator(List<? extends Iterator<In>> iters, Reducer<In, Out> reducer)
+    public static <In, Out> CloseableIterator<Out> getCloseable(List<? extends CloseableIterator<In>> sources,
+                                                                Comparator<? super In> comparator,
+                                                                Reducer<In, Out> reducer)
     {
-        this.iterators = iters;
-        this.reducer = reducer;
+        if (sources.size() == 1)
+        {
+            return reducer.singleSourceReduceIsTrivial()
+                 ? (CloseableIterator<Out>) sources.get(0)
+                 : new OneToOneCloseable<>(sources.get(0), reducer);
+        }
+        return new ManyToOneCloseable<>(sources, comparator, reducer);
     }
 
-    public static <In, Out> MergeIterator<In, Out> get(List<? extends Iterator<In>> sources,
-                                                       Comparator<? super In> comparator,
-                                                       Reducer<In, Out> reducer)
+    public static <In, Out> Iterator<Out> get(List<? extends Iterator<In>> sources,
+                                              Comparator<? super In> comparator,
+                                              Reducer<In, Out> reducer)
     {
         if (sources.size() == 1)
         {
-            return reducer.trivialReduceIsTrivial()
-                 ? new TrivialOneToOne<>(sources, reducer)
-                 : new OneToOne<>(sources, reducer);
+            return reducer.singleSourceReduceIsTrivial()
+                   ? (Iterator<Out>) sources.get(0)
+                   : new OneToOne<>(sources.get(0), reducer);
         }
         return new ManyToOne<>(sources, comparator, reducer);
     }
 
-    public Iterable<? extends Iterator<In>> iterators()
+    public static <In> Iterator<In> getNonReducing(List<? extends Iterator<In>> sources,
+                                                   Comparator<? super In> comparator)
     {
-        return iterators;
+        if (sources.size() == 1)
+            return sources.get(0);
+        else
+            return new NonReducing<>(sources, comparator);
     }
 
-    public void close()
+    public static <In> CloseableIterator<In> getNonReducingCloseable(List<? extends CloseableIterator<In>> sources,
+                                                                     Comparator<? super In> comparator)
     {
-        for (int i=0, isize=iterators.size(); i<isize; i++)
-        {
-            Iterator<In> iterator = iterators.get(i);
-            try
-            {
-                if (iterator instanceof AutoCloseable)
-                    ((AutoCloseable)iterator).close();
-            }
-            catch (Exception e)
-            {
-                throw new RuntimeException(e);
-            }
-        }
-
-        reducer.close();
+        if (sources.size() == 1)
+            return sources.get(0);
+        else
+            return new NonReducingCloseable<>(sources, comparator);
     }
 
-    /**
-     * A MergeIterator that consumes multiple input values per output value.
-     *
-     * The most straightforward way to implement this is to use a {@code PriorityQueue} of iterators, {@code poll} it to
-     * find the next item to consume, then {@code add} the iterator back after advancing. This is not very efficient as
-     * {@code poll} and {@code add} in all cases require at least {@code log(size)} comparisons (usually more than
-     * {@code 2*log(size)}) per consumed item, even if the input is suitable for fast iteration.
-     *
-     * The implementation below makes use of the fact that replacing the top element in a binary heap can be done much
-     * more efficiently than separately removing it and placing it back, especially in the cases where the top iterator
-     * is to be used again very soon (e.g. when there are large sections of the output where only a limited number of
-     * input iterators overlap, which is normally the case in many practically useful situations, e.g. levelled
-     * compaction). To further improve this particular scenario, we also use a short sorted section at the start of the
-     * queue.
-     *
-     * The heap is laid out as this (for {@code SORTED_SECTION_SIZE == 2}):
-     *                 0
-     *                 |
-     *                 1
-     *                 |
-     *                 2
-     *               /   \
-     *              3     4
-     *             / \   / \
-     *             5 6   7 8
-     *            .. .. .. ..
-     * Where each line is a <= relationship.
-     *
-     * In the sorted section we can advance with a single comparison per level, while advancing a level within the heap
-     * requires two (so that we can find the lighter element to pop up).
-     * The sorted section adds a constant overhead when data is uniformly distributed among the iterators, but may up
-     * to halve the iteration time when one iterator is dominant over sections of the merged data (as is the case with
-     * non-overlapping iterators).
-     *
-     * The iterator is further complicated by the need to avoid advancing the input iterators until an output is
-     * actually requested. To achieve this {@code consume} walks the heap to find equal items without advancing the
-     * iterators, and {@code advance} moves them and restores the heap structure before any items can be consumed.
-     * 
-     * To avoid having to do additional comparisons in consume to identify the equal items, we keep track of equality
-     * between children and their parents in the heap. More precisely, the lines in the diagram above define the
-     * following relationship:
-     *   parent <= child && (parent == child) == child.equalParent
-     * We can track, make use of and update the equalParent field without any additional comparisons.
-     *
-     * For more formal definitions and proof of correctness, see CASSANDRA-8915.
-     */
-    static final class ManyToOne<In,Out> extends MergeIterator<In,Out>
+    private static class ManyToOne<In, V extends Iterator<In>, Out> extends Merger<In, V, Out> implements Iterator<Out>
     {
-        protected final Candidate<In>[] heap;
-
-        /** Number of non-exhausted iterators. */
-        int size;
-
-        /**
-         * Position of the deepest, right-most child that needs advancing before we can start consuming.
-         * Because advancing changes the values of the items of each iterator, the parent-chain from any position
-         * in this range that needs advancing is not in correct order. The trees rooted at any position that does
-         * not need advancing, however, retain their prior-held binary heap property.
-         */
-        int needingAdvance;
-
-        /**
-         * The number of elements to keep in order before the binary heap starts, exclusive of the top heap element.
-         */
-        static final int SORTED_SECTION_SIZE = 4;
-
-        public ManyToOne(List<? extends Iterator<In>> iters, Comparator<? super In> comp, Reducer<In, Out> reducer)
+        public ManyToOne(List<V> iters, Comparator<? super In> comp, Reducer<In, Out> reducer)
         {
-            super(iters, reducer);
-
-            @SuppressWarnings("unchecked")
-            Candidate<In>[] heap = new Candidate[iters.size()];
-            this.heap = heap;
-            size = 0;
-
-            for (int i = 0; i < iters.size(); i++)
-            {
-                Candidate<In> candidate = new Candidate<>(i, iters.get(i), comp);
-                heap[size++] = candidate;
-            }
-            needingAdvance = size;
+            this(iters, null, comp, reducer);
         }
 
-        protected final Out computeNext()
+        ManyToOne(List<V> iters, Consumer<V> onClose, Comparator<? super In> comp, Reducer<In, Out> reducer)
         {
-            advance();
-            return consume();
+            super(iters, it -> it.hasNext() ? Preconditions.checkNotNull(it.next()) : null, onClose, comp, reducer);
         }
+    }
 
-        /**
-         * Advance all iterators that need to be advanced and place them into suitable positions in the heap.
-         *
-         * By walking the iterators backwards we know that everything after the point being processed already forms
-         * correctly ordered subheaps, thus we can build a subheap rooted at the current position by only sinking down
-         * the newly advanced iterator. Because all parents of a consumed iterator are also consumed there is no way
-         * that we can process one consumed iterator but skip over its parent.
-         *
-         * The procedure is the same as the one used for the initial building of a heap in the heapsort algorithm and
-         * has a maximum number of comparisons {@code (2 * log(size) + SORTED_SECTION_SIZE / 2)} multiplied by the
-         * number of iterators whose items were consumed at the previous step, but is also at most linear in the size of
-         * the heap if the number of consumed elements is high (as it is in the initial heap construction). With non- or
-         * lightly-overlapping iterators the procedure finishes after just one (resp. a couple of) comparisons.
-         */
-        private void advance()
+    private static class ManyToOneCloseable<In, V extends CloseableIterator<In>, Out> extends ManyToOne<In, V, Out> implements CloseableIterator<Out>
+    {
+        public ManyToOneCloseable(List<V> iters, Comparator<? super In> comp, Reducer<In, Out> reducer)
         {
-            // Turn the set of candidates into a heap.
-            for (int i = needingAdvance - 1; i >= 0; --i)
-            {
-                Candidate<In> candidate = heap[i];
-                /**
-                 *  needingAdvance runs to the maximum index (and deepest-right node) that may need advancing;
-                 *  since the equal items that were consumed at-once may occur in sub-heap "veins" of equality,
-                 *  not all items above this deepest-right position may have been consumed; these already form
-                 *  valid sub-heaps and can be skipped-over entirely
-                 */
-                if (candidate.needsAdvance())
-                    replaceAndSink(candidate.advance(), i);
-            }
+            super(iters, CloseableIterator::close, comp, reducer);
         }
+    }
 
-        /**
-         * Consume all items that sort like the current top of the heap. As we cannot advance the iterators to let
-         * equivalent items pop up, we walk the heap to find them and mark them as needing advance.
-         *
-         * This relies on the equalParent flag to avoid doing any comparisons.
-         */
-        private Out consume()
+    private static class NonReducing<In, V extends Iterator<In>> extends Merger<In, V, In> implements Iterator<In>
+    {
+        public NonReducing(List<V> iters, Comparator<? super In> comp)
         {
-            if (size == 0)
-                return endOfData();
-
-            reducer.onKeyChange();
-            assert !heap[0].equalParent;
-            heap[0].consume(reducer);
-            final int size = this.size;
-            final int sortedSectionSize = Math.min(size, SORTED_SECTION_SIZE);
-            int i;
-            consume: {
-                for (i = 1; i < sortedSectionSize; ++i)
-                {
-                    if (!heap[i].equalParent)
-                        break consume;
-                    heap[i].consume(reducer);
-                }
-                i = Math.max(i, consumeHeap(i) + 1);
-            }
-            needingAdvance = i;
-            return reducer.getReduced();
+            this(iters, null, comp);
         }
 
-        /**
-         * Recursively consume all items equal to equalItem in the binary subheap rooted at position idx.
-         *
-         * @return the largest equal index found in this search.
-         */
-        private int consumeHeap(int idx)
+        NonReducing(List<V> iters, Consumer<V> onClose, Comparator<? super In> comp)
         {
-            if (idx >= size || !heap[idx].equalParent)
-                return -1;
-
-            heap[idx].consume(reducer);
-            int nextIdx = (idx << 1) - (SORTED_SECTION_SIZE - 1);
-            return Math.max(idx, Math.max(consumeHeap(nextIdx), consumeHeap(nextIdx + 1)));
+            super(iters, it -> it.hasNext() ? Preconditions.checkNotNull(it.next()) : null, onClose, comp, null);
         }
 
-        /**
-         * Replace an iterator in the heap with the given position and move it down the heap until it finds its proper
-         * position, pulling lighter elements up the heap.
-         *
-         * Whenever an equality is found between two elements that form a new parent-child relationship, the child's
-         * equalParent flag is set to true if the elements are equal.
-         */
-        private void replaceAndSink(Candidate<In> candidate, int currIdx)
+        @Override
+        public In next()
         {
-            if (candidate == null)
-            {
-                // Drop iterator by replacing it with the last one in the heap.
-                candidate = heap[--size];
-                heap[size] = null; // not necessary but helpful for debugging
-            }
-            // The new element will be top of its heap, at this point there is no parent to be equal to.
-            candidate.equalParent = false;
-
-            final int size = this.size;
-            final int sortedSectionSize = Math.min(size - 1, SORTED_SECTION_SIZE);
-
-            int nextIdx;
-
-            // Advance within the sorted section, pulling up items lighter than candidate.
-            while ((nextIdx = currIdx + 1) <= sortedSectionSize)
-            {
-                if (!heap[nextIdx].equalParent) // if we were greater then an (or were the) equal parent, we are >= the child
-                {
-                    int cmp = candidate.compareTo(heap[nextIdx]);
-                    if (cmp <= 0)
-                    {
-                        heap[nextIdx].equalParent = cmp == 0;
-                        heap[currIdx] = candidate;
-                        return;
-                    }
-                }
-
-                heap[currIdx] = heap[nextIdx];
-                currIdx = nextIdx;
-            }
-            // If size <= SORTED_SECTION_SIZE, nextIdx below will be no less than size,
-            // because currIdx == sortedSectionSize == size - 1 and nextIdx becomes
-            // (size - 1) * 2) - (size - 1 - 1) == size.
-
-            // Advance in the binary heap, pulling up the lighter element from the two at each level.
-            while ((nextIdx = (currIdx * 2) - (sortedSectionSize - 1)) + 1 < size)
-            {
-                if (!heap[nextIdx].equalParent)
-                {
-                    if (!heap[nextIdx + 1].equalParent)
-                    {
-                        // pick the smallest of the two children
-                        int siblingCmp = heap[nextIdx + 1].compareTo(heap[nextIdx]);
-                        if (siblingCmp < 0)
-                            ++nextIdx;
-
-                        // if we're smaller than this, we are done, and must only restore the heap and equalParent properties
-                        int cmp = candidate.compareTo(heap[nextIdx]);
-                        if (cmp <= 0)
-                        {
-                            if (cmp == 0)
-                            {
-                                heap[nextIdx].equalParent = true;
-                                if (siblingCmp == 0) // siblingCmp == 0 => nextIdx is the left child
-                                    heap[nextIdx + 1].equalParent = true;
-                            }
-
-                            heap[currIdx] = candidate;
-                            return;
-                        }
-
-                        if (siblingCmp == 0)
-                        {
-                            // siblingCmp == 0 => nextIdx is still the left child
-                            // if the two siblings were equal, and we are inserting something greater, we will
-                            // pull up the left one; this means the right gets an equalParent
-                            heap[nextIdx + 1].equalParent = true;
-                        }
-                    }
-                    else
-                        ++nextIdx;  // descend down the path where we found the equal child
-                }
-
-                heap[currIdx] = heap[nextIdx];
-                currIdx = nextIdx;
-            }
-
-            // our loop guard ensures there are always two siblings to process; typically when we exit the loop we will
-            // be well past the end of the heap and this next condition will match...
-            if (nextIdx >= size)
-            {
-                heap[currIdx] = candidate;
-                return;
-            }
-
-            // ... but sometimes we will have one last child to compare against, that has no siblings
-            if (!heap[nextIdx].equalParent)
-            {
-                int cmp = candidate.compareTo(heap[nextIdx]);
-                if (cmp <= 0)
-                {
-                    heap[nextIdx].equalParent = cmp == 0;
-                    heap[currIdx] = candidate;
-                    return;
-                }
-            }
-
-            heap[currIdx] = heap[nextIdx];
-            heap[nextIdx] = candidate;
+            return super.nonReducingNext();
         }
     }
 
-    // Holds and is comparable by the head item of an iterator it owns
-    protected static final class Candidate<In> implements Comparable<Candidate<In>>
+    private static class NonReducingCloseable<In, V extends CloseableIterator<In>> extends NonReducing<In, V> implements CloseableIterator<In>
     {
-        private final Iterator<? extends In> iter;
-        private final Comparator<? super In> comp;
-        private final int idx;
-        private In item;
-        private In lowerBound;
-        boolean equalParent;
-
-        public Candidate(int idx, Iterator<? extends In> iter, Comparator<? super In> comp)
+        public NonReducingCloseable(List<V> iters, Comparator<? super In> comp)
         {
-            this.iter = iter;
-            this.comp = comp;
-            this.idx = idx;
-            this.lowerBound = iter instanceof IteratorWithLowerBound ? ((IteratorWithLowerBound<In>)iter).lowerBound() : null;
-        }
-
-        /** @return this if our iterator had an item, and it is now available, otherwise null */
-        protected Candidate<In> advance()
-        {
-            if (lowerBound != null)
-            {
-                item = lowerBound;
-                return this;
-            }
-
-            if (!iter.hasNext())
-                return null;
-
-            item = iter.next();
-            return this;
+            super(iters, CloseableIterator::close, comp);
         }
+    }
 
-        public int compareTo(Candidate<In> that)
-        {
-            assert this.item != null && that.item != null;
-            int ret = comp.compare(this.item, that.item);
-            if (ret == 0 && (this.isLowerBound() ^ that.isLowerBound()))
-            {   // if the items are equal and one of them is a lower bound (but not the other one)
-                // then ensure the lower bound is less than the real item so we can safely
-                // skip lower bounds when consuming
-                return this.isLowerBound() ? -1 : 1;
-            }
-            return ret;
-        }
+    private static class OneToOne<In, Out> implements Iterator<Out>
+    {
+        private final Iterator<In> source;
+        private final Reducer<In, Out> reducer;
 
-        private boolean isLowerBound()
+        public OneToOne(Iterator<In> source, Reducer<In, Out> reducer)
         {
-            assert item != null;
-            return item == lowerBound;
+            this.reducer = reducer;
+            this.source = source;
         }
 
-        public <Out> void consume(Reducer<In, Out> reducer)
+        public boolean hasNext()
         {
-            if (isLowerBound())
-            {
-                item = null;
-                lowerBound = null;
-            }
-            else
-            {
-                reducer.reduce(idx, item);
-                item = null;
-            }
+            return source.hasNext();
         }
 
-        public boolean needsAdvance()
+        public Out next()
         {
-            return item == null;
+            reducer.onKeyChange();
+            reducer.reduce(0, source.next());
+            return reducer.getReduced();
         }
     }
 
-    /** Accumulator that collects values of type A, and outputs a value of type B. */
-    public static abstract class Reducer<In,Out>
+    private static class OneToOneCloseable<In, Out> implements CloseableIterator<Out>
     {
-        /**
-         * @return true if Out is the same as In for the case of a single source iterator
-         */
-        public boolean trivialReduceIsTrivial()
+        private final CloseableIterator<In> source;
+        private final Reducer<In, Out> reducer;
+
+        public OneToOneCloseable(CloseableIterator<In> source, Reducer<In, Out> reducer)
         {
-            return false;
+            this.reducer = reducer;
+            this.source = source;
         }
 
-        /**
-         * combine this object with the previous ones.
-         * intermediate state is up to your implementation.
-         */
-        public abstract void reduce(int idx, In current);
-
-        /** @return The last object computed by reduce */
-        protected abstract Out getReduced();
-
-        /**
-         * Called at the beginning of each new key, before any reduce is called.
-         * To be overridden by implementing classes.
-         */
-        protected void onKeyChange() {}
-
-        /**
-         * May be overridden by implementations that require cleaning up after use
-         */
-        public void close() {}
-    }
-
-    private static class OneToOne<In, Out> extends MergeIterator<In, Out>
-    {
-        private final Iterator<In> source;
-
-        public OneToOne(List<? extends Iterator<In>> sources, Reducer<In, Out> reducer)
+        public boolean hasNext()
         {
-            super(sources, reducer);
-            source = sources.get(0);
+            return source.hasNext();
         }
 
-        protected Out computeNext()
+        public Out next()
         {
-            if (!source.hasNext())
-                return endOfData();
             reducer.onKeyChange();
             reducer.reduce(0, source.next());
             return reducer.getReduced();
         }
-    }
-
-    private static class TrivialOneToOne<In, Out> extends MergeIterator<In, Out>
-    {
-        private final Iterator<In> source;
-
-        public TrivialOneToOne(List<? extends Iterator<In>> sources, Reducer<In, Out> reducer)
-        {
-            super(sources, reducer);
-            source = sources.get(0);
-        }
 
-        @SuppressWarnings("unchecked")
-        protected Out computeNext()
+        public void close()
         {
-            if (!source.hasNext())
-                return endOfData();
-            return (Out) source.next();
+            source.close();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/cassandra/utils/Merger.java b/src/java/org/apache/cassandra/utils/Merger.java
new file mode 100644
index 000000000000..ff3cddecb0bb
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/Merger.java
@@ -0,0 +1,461 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+/**
+ * A merger of input streams (e.g. iterators or cursors) that may consume multiple input values per output value.
+ *
+ * The most straightforward way to implement this is to use a {@code PriorityQueue} of iterators, {@code poll} it to
+ * find the next item to consume, then {@code add} the iterator back after advancing. This is not very efficient as
+ * {@code poll} and {@code add} in all cases require at least {@code log(size)} comparisons (usually more than
+ * {@code 2*log(size)}) per consumed item, even if the input is suitable for fast iteration.
+ *
+ * The implementation below makes use of the fact that replacing the top element in a binary heap can be done much
+ * more efficiently than separately removing it and placing it back, especially in the cases where the top iterator
+ * is to be used again very soon (e.g. when there are large sections of the output where only a limited number of
+ * input iterators overlap, which is normally the case in many practically useful situations, e.g. levelled
+ * compaction). To further improve this particular scenario, we also use a short sorted section at the start of the
+ * queue.
+ *
+ * The heap is laid out as this (for {@code SORTED_SECTION_SIZE == 2}):
+ *                 0
+ *                 |
+ *                 1
+ *                 |
+ *                 2
+ *               /   \
+ *              3     4
+ *             / \   / \
+ *             5 6   7 8
+ *            .. .. .. ..
+ * Where each line is a <= relationship.
+ *
+ * In the sorted section we can advance with a single comparison per level, while advancing a level within the heap
+ * requires two (so that we can find the lighter element to pop up).
+ * The sorted section adds a constant overhead when data is uniformly distributed among the iterators, but may up
+ * to halve the iteration time when one iterator is dominant over sections of the merged data (as is the case with
+ * non-overlapping iterators).
+ *
+ * The iterator is further complicated by the need to avoid advancing the input iterators until an output is
+ * actually requested. To achieve this {@code consume} walks the heap to find equal items without advancing the
+ * iterators, and {@code advance} moves them and restores the heap structure before any items can be consumed.
+ *
+ * To avoid having to do additional comparisons in consume to identify the equal items, we keep track of equality
+ * between children and their parents in the heap. More precisely, the lines in the diagram above define the
+ * following relationship:
+ *   parent <= child && (parent == child) == child.equalParent
+ * We can track, make use of and update the equalParent field without any additional comparisons.
+ *
+ * For more formal definitions and proof of correctness, see CASSANDRA-8915.
+ */
+public class Merger<In, Source, Out> implements AutoCloseable
+{
+    /** The heap of candidates, each containing their current item and the source from which it was obtained. */
+    protected final Candidate<In, Source>[] heap;
+
+    /** Reducer, called for each input item to combine them into the output. */
+    final Reducer<In, Out> reducer;
+
+    /** Function called on each source to get the next item. Should return null if the source is exhausted. */
+    final Function<Source, In> inputRetriever;
+
+    /** Method to call on each source on close, may be null. */
+    final Consumer<Source> onClose;
+
+    /** Number of non-exhausted iterators. */
+    int size;
+
+    /**
+     * Position of the deepest, right-most child that needs advancing before we can start consuming.
+     * Because advancing changes the values of the items of each source, the parent-chain from any position
+     * in this range that needs advancing is not in correct order. The trees rooted at any position that does
+     * not need advancing, however, retain their prior-held binary heap property.
+     */
+    int needingAdvance;
+
+    /**
+     * The number of elements to keep in order before the binary heap starts, exclusive of the top heap element.
+     */
+    static final int SORTED_SECTION_SIZE = 4;
+
+    /**
+     * @param sources        The input sources.
+     * @param inputRetriever Function called on each source to get the next item. Should return null if the source is
+     *                       exhausted.
+     * @param onClose        Method to call on each source on close, may be null.
+     * @param comparator     Comparator of input items.
+     * @param reducer        Reducer, called for each input item to combine them into the output.
+     */
+    public Merger(List<Source> sources,
+                  Function<Source, In> inputRetriever,
+                  Consumer<Source> onClose,
+                  Comparator<? super In> comparator,
+                  Reducer<In, Out> reducer)
+    {
+        this.inputRetriever = inputRetriever;
+        this.onClose = onClose;
+        this.reducer = reducer;
+
+        heap = new Candidate[sources.size()];
+        size = 0;
+
+        for (int i = 0; i < sources.size(); i++)
+        {
+            Candidate<In, Source> candidate = new Candidate<>(i, sources.get(i), comparator);
+            heap[size++] = candidate;
+        }
+        needingAdvance = size;
+    }
+
+    public boolean hasNext()
+    {
+        advance();  // no-op if already advanced
+        return size > 0;
+    }
+
+    public Out next()
+    {
+        advance();  // no-op if already advanced (e.g. hasNext() called)
+        assert size > 0;
+        return consume();
+    }
+
+    public In nonReducingNext()
+    {
+        advance();  // no-op if already advanced (e.g. hasNext() called)
+        assert size > 0;
+        return consumeOne();
+    }
+
+    public void close()
+    {
+        if (onClose == null)
+            return;
+
+        Throwable t = null;
+        for (Candidate<In, Source> c : heap)
+        {
+            try
+            {
+                onClose.accept(c.input);
+            }
+            catch (Throwable e)
+            {
+                t = Throwables.merge(t, e);
+            }
+        }
+        Throwables.maybeFail(t);
+    }
+
+
+    /**
+     * Advance all sources that need to be advanced and place them into suitable positions in the heap.
+     *
+     * By walking the sources backwards we know that everything after the point being processed already forms
+     * correctly ordered subheaps, thus we can build a subheap rooted at the current position by only sinking down
+     * the newly advanced source. Because all parents of a consumed source are also consumed there is no way
+     * that we can process one consumed source but skip over its parent.
+     *
+     * The procedure is the same as the one used for the initial building of a heap in the heapsort algorithm and
+     * has a maximum number of comparisons {@code (2 * log(size) + SORTED_SECTION_SIZE / 2)} multiplied by the
+     * number of sources whose items were consumed at the previous step, but is also at most linear in the size of
+     * the heap if the number of consumed elements is high (as it is in the initial heap construction). With non- or
+     * lightly-overlapping sources the procedure finishes after just one (resp. a couple of) comparisons.
+     */
+    private void advance()
+    {
+        // Turn the set of candidates into a heap.
+        for (int i = needingAdvance - 1; i >= 0; --i)
+        {
+            Candidate<In, Source> candidate = heap[i];
+            /**
+             *  needingAdvance runs to the maximum index (and deepest-right node) that may need advancing;
+             *  since the equal items that were consumed at-once may occur in sub-heap "veins" of equality,
+             *  not all items above this deepest-right position may have been consumed; these already form
+             *  valid sub-heaps and can be skipped-over entirely
+             */
+            if (candidate.needsAdvance())
+                replaceAndSink(candidate, !candidate.advance(inputRetriever), i);
+        }
+        needingAdvance = 0;
+    }
+
+    /**
+     * Consume all items that sort like the current top of the heap. As we cannot advance the sources to let
+     * equivalent items pop up, we walk the heap to find them and mark them as needing advance.
+     *
+     * This relies on the equalParent flag to avoid doing any comparisons.
+     */
+    private Out consume()
+    {
+        reducer.onKeyChange();
+        assert !heap[0].equalParent;
+        heap[0].consume(reducer);
+        final int size = this.size;
+        final int sortedSectionSize = Math.min(size, SORTED_SECTION_SIZE);
+        int i;
+        consume: {
+            for (i = 1; i < sortedSectionSize; ++i)
+            {
+                if (!heap[i].equalParent)
+                    break consume;
+                heap[i].consume(reducer);
+            }
+            i = Math.max(i, consumeHeap(i) + 1);
+        }
+        needingAdvance = i;
+        return reducer.getReduced();
+    }
+
+    /**
+     * Consume only the top item, regardless if there are others that sort like it on the heap.
+     * No reducer is required for this.
+     */
+    private In consumeOne()
+    {
+        needingAdvance = 1;
+        return heap[0].consumeItem();
+    }
+
+    /**
+     * Recursively consume all items equal to equalItem in the binary subheap rooted at position idx.
+     *
+     * @return the largest equal index found in this search.
+     */
+    private int consumeHeap(int idx)
+    {
+        if (idx >= size || !heap[idx].equalParent)
+            return -1;
+
+        heap[idx].consume(reducer);
+        int nextIdx = (idx << 1) - (SORTED_SECTION_SIZE - 1);
+        return Math.max(idx, Math.max(consumeHeap(nextIdx), consumeHeap(nextIdx + 1)));
+    }
+
+    /**
+     * Replace a source in the heap with the given position and move it down the heap until it finds its proper
+     * position, pulling lighter elements up the heap.
+     *
+     * Whenever an equality is found between two elements that form a new parent-child relationship, the child's
+     * equalParent flag is set to true if the elements are equal.
+     */
+    private void replaceAndSink(Candidate<In, Source> candidate, boolean candidateDone, int currIdx)
+    {
+        if (candidateDone)
+        {
+            // Drop source by swapping it with the last one in the heap.
+            Candidate<In, Source> replacement = heap[--size];
+            heap[size] = candidate;
+            candidate = replacement;
+        }
+        // The new element will be top of its heap, at this point there is no parent to be equal to.
+        candidate.equalParent = false;
+
+        final int size = this.size;
+        final int sortedSectionSize = Math.min(size - 1, SORTED_SECTION_SIZE);
+
+        int nextIdx;
+
+        // Advance within the sorted section, pulling up items lighter than candidate.
+        while ((nextIdx = currIdx + 1) <= sortedSectionSize)
+        {
+            if (!heap[nextIdx].equalParent) // if we were greater then an (or were the) equal parent, we are >= the child
+            {
+                int cmp = candidate.compareTo(heap[nextIdx]);
+                if (cmp <= 0)
+                {
+                    heap[nextIdx].equalParent = cmp == 0;
+                    heap[currIdx] = candidate;
+                    return;
+                }
+            }
+
+            heap[currIdx] = heap[nextIdx];
+            currIdx = nextIdx;
+        }
+        // If size <= SORTED_SECTION_SIZE, nextIdx below will be no less than size,
+        // because currIdx == sortedSectionSize == size - 1 and nextIdx becomes
+        // (size - 1) * 2) - (size - 1 - 1) == size.
+
+        // Advance in the binary heap, pulling up the lighter element from the two at each level.
+        while ((nextIdx = (currIdx * 2) - (sortedSectionSize - 1)) + 1 < size)
+        {
+            if (!heap[nextIdx].equalParent)
+            {
+                if (!heap[nextIdx + 1].equalParent)
+                {
+                    // pick the smallest of the two children
+                    int siblingCmp = heap[nextIdx + 1].compareTo(heap[nextIdx]);
+                    if (siblingCmp < 0)
+                        ++nextIdx;
+
+                    // if we're smaller than this, we are done, and must only restore the heap and equalParent properties
+                    int cmp = candidate.compareTo(heap[nextIdx]);
+                    if (cmp <= 0)
+                    {
+                        if (cmp == 0)
+                        {
+                            heap[nextIdx].equalParent = true;
+                            if (siblingCmp == 0) // siblingCmp == 0 => nextIdx is the left child
+                                heap[nextIdx + 1].equalParent = true;
+                        }
+
+                        heap[currIdx] = candidate;
+                        return;
+                    }
+
+                    if (siblingCmp == 0)
+                    {
+                        // siblingCmp == 0 => nextIdx is still the left child
+                        // if the two siblings were equal, and we are inserting something greater, we will
+                        // pull up the left one; this means the right gets an equalParent
+                        heap[nextIdx + 1].equalParent = true;
+                    }
+                }
+                else
+                    ++nextIdx;  // descend down the path where we found the equal child
+            }
+
+            heap[currIdx] = heap[nextIdx];
+            currIdx = nextIdx;
+        }
+
+        // our loop guard ensures there are always two siblings to process; typically when we exit the loop we will
+        // be well past the end of the heap and this next condition will match...
+        if (nextIdx >= size)
+        {
+            heap[currIdx] = candidate;
+            return;
+        }
+
+        // ... but sometimes we will have one last child to compare against, that has no siblings
+        if (!heap[nextIdx].equalParent)
+        {
+            int cmp = candidate.compareTo(heap[nextIdx]);
+            if (cmp <= 0)
+            {
+                heap[nextIdx].equalParent = cmp == 0;
+                heap[currIdx] = candidate;
+                return;
+            }
+        }
+
+        heap[currIdx] = heap[nextIdx];
+        heap[nextIdx] = candidate;
+    }
+
+    /**
+     * Returns an iterable listing all the sources of this merger.
+     */
+    public Iterable<Source> allSources()
+    {
+        return () -> new Iterator<Source>()
+        {
+            int index = 0;
+
+            public boolean hasNext()
+            {
+                return index < heap.length;
+            }
+
+            public Source next()
+            {
+                return heap[index++].input;
+            }
+        };
+    }
+
+    /**
+     * Returns an iterable that lists all inputs that are positioned after the current position,
+     * i.e. all inputs that are not equal to the current top.
+     * Meant to be called inside getReduced.
+     */
+    public Iterable<In> allGreaterValues()
+    {
+        return () -> new AbstractIterator<In>()
+        {
+            int index = 1;  // skip first item, it's always equal
+
+            protected In computeNext()
+            {
+                while (true)
+                {
+                    if (index >= size)
+                        return endOfData();
+                    Candidate<In, Source> candidate = heap[index++];
+                    if (!candidate.needsAdvance())
+                        return candidate.item;
+                }
+            }
+        };
+    }
+
+    // Holds and is comparable by the head item of a source it owns
+    protected static final class Candidate<In, Source> implements Comparable<Candidate<In, Source>>
+    {
+        private final Source input;
+        private final Comparator<? super In> comp;
+        private final int idx;
+        private In item;
+        boolean equalParent;
+
+        public Candidate(int idx, Source input, Comparator<? super In> comp)
+        {
+            this.input = input;
+            this.comp = comp;
+            this.idx = idx;
+        }
+
+        /** Advance this source and returns true if it had an item, i.e. was not exhausted. */
+        protected boolean advance(Function<Source, In> inputRetriever)
+        {
+            item = inputRetriever.apply(input);
+            return item != null;
+        }
+
+        public int compareTo(Candidate<In, Source> that)
+        {
+            assert this.item != null && that.item != null;
+            return comp.compare(this.item, that.item);
+        }
+
+        public void consume(Reducer reducer)
+        {
+            reducer.reduce(idx, consumeItem());
+        }
+
+        public In consumeItem()
+        {
+            In v = item;
+            item = null;
+            return v;
+        }
+
+        public boolean needsAdvance()
+        {
+            return item == null;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/MonotonicClock.java b/src/java/org/apache/cassandra/utils/MonotonicClock.java
index 7be54c008b7f..a7c15853cbd4 100644
--- a/src/java/org/apache/cassandra/utils/MonotonicClock.java
+++ b/src/java/org/apache/cassandra/utils/MonotonicClock.java
@@ -205,7 +205,7 @@ public synchronized void resumeEpochSampling()
             if (almostSameTimeUpdater != null)
                 throw new IllegalStateException("Already running");
             updateAlmostSameTime();
-            logger.info("Scheduling approximate time conversion task with an interval of {} milliseconds", UPDATE_INTERVAL_MS);
+            logger.debug("Scheduling approximate time conversion task with an interval of {} milliseconds", UPDATE_INTERVAL_MS);
             almostSameTimeUpdater = ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(this::updateAlmostSameTime, UPDATE_INTERVAL_MS, UPDATE_INTERVAL_MS, MILLISECONDS);
         }
 
@@ -268,13 +268,13 @@ public long error()
         @Override
         public boolean isAfter(long instant)
         {
-            return now() > instant;
+            return instant - now() < 0;
         }
 
         @Override
         public boolean isAfter(long now, long instant)
         {
-            return now > instant;
+            return instant - now < 0;
         }
     }
 
@@ -341,7 +341,7 @@ public synchronized void resumeNowSampling()
                 throw new IllegalStateException("Already running");
 
             almostNow = precise.now();
-            logger.info("Scheduling approximate time-check task with a precision of {} milliseconds", UPDATE_INTERVAL_MS);
+            logger.debug("Scheduling approximate time-check task with a precision of {} milliseconds", UPDATE_INTERVAL_MS);
             almostNowUpdater = ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(() -> almostNow = precise.now(), UPDATE_INTERVAL_MS, UPDATE_INTERVAL_MS, MILLISECONDS);
         }
 
diff --git a/src/java/org/apache/cassandra/utils/NativeLibrary.java b/src/java/org/apache/cassandra/utils/NativeLibrary.java
index 934843393975..538019fd1d5d 100644
--- a/src/java/org/apache/cassandra/utils/NativeLibrary.java
+++ b/src/java/org/apache/cassandra/utils/NativeLibrary.java
@@ -20,6 +20,8 @@
 import java.io.FileDescriptor;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.AsynchronousFileChannel;
 import java.nio.channels.FileChannel;
 import java.util.concurrent.TimeUnit;
 
@@ -28,30 +30,24 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.sun.jna.LastErrorException;
-
+import com.sun.jna.Native;
+import com.sun.jna.Pointer;
 import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.utils.NativeLibraryWrapper.NativeError;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORE_MISSING_NATIVE_FILE_HINTS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.OS_ARCH;
 import static org.apache.cassandra.config.CassandraRelevantProperties.OS_NAME;
-import static org.apache.cassandra.utils.NativeLibrary.OSType.LINUX;
-import static org.apache.cassandra.utils.NativeLibrary.OSType.MAC;
-import static org.apache.cassandra.utils.NativeLibrary.OSType.AIX;
+import static org.apache.cassandra.utils.INativeLibrary.OSType.AIX;
+import static org.apache.cassandra.utils.INativeLibrary.OSType.LINUX;
+import static org.apache.cassandra.utils.INativeLibrary.OSType.MAC;
 
-public final class NativeLibrary
+public class NativeLibrary implements INativeLibrary
 {
     private static final Logger logger = LoggerFactory.getLogger(NativeLibrary.class);
     private static final boolean REQUIRE = !IGNORE_MISSING_NATIVE_FILE_HINTS.getBoolean();
 
-    public enum OSType
-    {
-        LINUX,
-        MAC,
-        AIX,
-        OTHER;
-    }
-
     public static final OSType osType;
 
     private static final int MCL_CURRENT;
@@ -72,11 +68,18 @@ public enum OSType
     private static final int POSIX_FADV_DONTNEED   = 4; /* fadvise.h */
     private static final int POSIX_FADV_NOREUSE    = 5; /* fadvise.h */
 
+    private static final int MADV_NORMAL     = 0; /* mman.h */
+    private static final int MADV_RANDOM     = 1; /* mman.h */
+    private static final int MADV_SEQUENTIAL = 2; /* mman.h */
+    private static final int MADV_WILLNEED   = 3; /* mman.h */
+    private static final int MADV_DONTNEED   = 4; /* mman.h */
+
     private static final NativeLibraryWrapper wrappedLibrary;
     private static boolean jnaLockable = false;
 
     private static final Field FILE_DESCRIPTOR_FD_FIELD;
     private static final Field FILE_CHANNEL_FD_FIELD;
+    private static final Field FILE_ASYNC_CHANNEL_FD_FIELD;
 
     static
     {
@@ -84,6 +87,7 @@ public enum OSType
         try
         {
             FILE_CHANNEL_FD_FIELD = FBUtilities.getProtectedField(Class.forName("sun.nio.ch.FileChannelImpl"), "fd");
+            FILE_ASYNC_CHANNEL_FD_FIELD = FBUtilities.getProtectedField(Class.forName("sun.nio.ch.AsynchronousFileChannelImpl"), "fdObj");
         }
         catch (ClassNotFoundException e)
         {
@@ -127,7 +131,7 @@ else if (osType == AIX)
         }
     }
 
-    private NativeLibrary() {}
+    NativeLibrary() {}
 
     /**
      * @return the detected OSType of the Operating System running the JVM using crude string matching
@@ -148,36 +152,26 @@ else if (osName.contains("mac"))
             return LINUX;
     }
 
-    private static int errno(RuntimeException e)
+    @Override
+    public boolean isOS(INativeLibrary.OSType type)
     {
-        assert e instanceof LastErrorException;
-        try
-        {
-            return ((LastErrorException) e).getErrorCode();
-        }
-        catch (NoSuchMethodError x)
-        {
-            if (REQUIRE)
-                logger.warn("Obsolete version of JNA present; unable to read errno. Upgrade to JNA 3.2.7 or later");
-            return 0;
-        }
+        return osType == type;
     }
 
-    /**
-     * Checks if the library has been successfully linked.
-     * @return {@code true} if the library has been successfully linked, {@code false} otherwise.
-     */
-    public static boolean isAvailable()
+    @Override
+    public boolean isAvailable()
     {
         return wrappedLibrary.isAvailable();
     }
 
-    public static boolean jnaMemoryLockable()
+    @Override
+    public boolean jnaMemoryLockable()
     {
         return jnaLockable;
     }
 
-    public static void tryMlockall()
+    @Override
+    public void tryMlockall()
     {
         try
         {
@@ -189,12 +183,9 @@ public static void tryMlockall()
         {
             // this will have already been logged by CLibrary, no need to repeat it
         }
-        catch (RuntimeException e)
+        catch (NativeError e)
         {
-            if (!(e instanceof LastErrorException))
-                throw e;
-
-            if (errno(e) == ENOMEM && osType == LINUX)
+            if (e.getErrno() == ENOMEM && osType == LINUX)
             {
                 logger.warn("Unable to lock JVM memory (ENOMEM)."
                         + " This can result in part of the JVM being swapped out, especially with mmapped I/O enabled."
@@ -203,125 +194,153 @@ public static void tryMlockall()
             else if (osType != MAC)
             {
                 // OS X allows mlockall to be called, but always returns an error
-                logger.warn("Unknown mlockall error {}", errno(e));
+                logger.error("Unknown mlockall error", e);
             }
         }
     }
 
-    public static void trySkipCache(String path, long offset, long len)
+    @Override
+    public void trySkipCache(File f, long offset, long len)
     {
-        File f = new File(path);
         if (!f.exists())
             return;
 
         try (FileInputStreamPlus fis = new FileInputStreamPlus(f))
         {
-            trySkipCache(getfd(fis.getChannel()), offset, len, path);
+            trySkipCache(getfd(fis.getChannel()), offset, len, f.path());
         }
         catch (IOException e)
         {
-            logger.warn("Could not skip cache", e);
+            logger.error("Could not open file to skip cache", e);
         }
     }
 
-    public static void trySkipCache(int fd, long offset, long len, String path)
+    @Override
+    public void trySkipCache(int fd, long offset, long len, String fileName)
     {
         if (len == 0)
-            trySkipCache(fd, 0, 0, path);
+            trySkipCache(fd, 0, 0, fileName);
 
         while (len > 0)
         {
             int sublen = (int) Math.min(Integer.MAX_VALUE, len);
-            trySkipCache(fd, offset, sublen, path);
+            trySkipCache(fd, offset, sublen, fileName);
             len -= sublen;
             offset -= sublen;
         }
     }
 
-    public static void trySkipCache(int fd, long offset, int len, String path)
+    @Override
+    public void trySkipCache(int fd, long offset, int len, String fileName)
     {
         if (fd < 0)
             return;
 
         try
         {
-            if (osType == LINUX)
-            {
-                int result = wrappedLibrary.callPosixFadvise(fd, offset, len, POSIX_FADV_DONTNEED);
-                if (result != 0)
-                    NoSpamLogger.log(
-                            logger,
-                            NoSpamLogger.Level.WARN,
-                            10,
-                            TimeUnit.MINUTES,
-                            "Failed trySkipCache on file: {} Error: " + wrappedLibrary.callStrerror(result).getString(0),
-                            path);
-            }
+            wrappedLibrary.callPosixFadvise(fd, offset, len, POSIX_FADV_DONTNEED);
         }
         catch (UnsatisfiedLinkError e)
         {
             // if JNA is unavailable just skipping Direct I/O
             // instance of this class will act like normal RandomAccessFile
         }
-        catch (RuntimeException e)
+        catch (NativeError e)
         {
-            if (!(e instanceof LastErrorException))
-                throw e;
-
-            logger.warn("posix_fadvise({}, {}) failed, errno ({}).", fd, offset, errno(e));
+            NoSpamLogger.log(logger,
+                             NoSpamLogger.Level.ERROR,
+                             10,
+                             TimeUnit.MINUTES,
+                             "Failed trySkipCache on file: {} Error: " + e.getMessage(),
+                             fileName);
         }
     }
 
-    public static int tryFcntl(int fd, int command, int flags)
-    {
-        // fcntl return value may or may not be useful, depending on the command
-        int result = -1;
+    /**
+     * @param buffer
+     * @param length
+     * @param filename -- source file backing buffer; logged on error
+     * <p>
+     * adviseRandom works even on buffers that are not aligned to page boundaries (which is the
+     * common case for how MmappedRegions is used).
+     */
+    public void adviseRandom(MappedByteBuffer buffer, long length, String filename) {
+        assert buffer != null;
+
+        var rawAddress = Native.getDirectBufferPointer(buffer);
+        // align to the nearest lower page boundary
+        var alignedAddress = new Pointer(Pointer.nativeValue(rawAddress) & -PageAware.PAGE_SIZE);
+        // we want to advise the whole buffer, so if the aligned address is lower than the raw one,
+        // we need to pad the length accordingly.  (we do not need to align `length`, Linux
+        // takes care of rounding it up for us.)
+        length += Pointer.nativeValue(rawAddress) - Pointer.nativeValue(alignedAddress);
 
         try
         {
-            result = wrappedLibrary.callFcntl(fd, command, flags);
+            wrappedLibrary.callPosixMadvise(alignedAddress, length, MADV_RANDOM);
         }
         catch (UnsatisfiedLinkError e)
         {
-            // if JNA is unavailable just skipping
+            // if JNA is unavailable just skipping Direct I/O
+            // instance of this class will act like normal RandomAccessFile
         }
-        catch (RuntimeException e)
+        catch (NativeError e)
         {
-            if (!(e instanceof LastErrorException))
-                throw e;
+            NoSpamLogger.log(logger,
+                             NoSpamLogger.Level.ERROR,
+                             10,
+                             TimeUnit.MINUTES,
+                             "Failed madvise on file: {}. Error: " + e.getMessage(),
+                             filename);
+        }
+    }
 
+    @Override
+    public int tryFcntl(int fd, int command, int flags)
+    {
+        try
+        {
+            return wrappedLibrary.callFcntl(fd, command, flags);
+        }
+        catch (UnsatisfiedLinkError e)
+        {
+            // Unsupported on this platform
+        }
+        catch (NativeError e)
+        {
             if (REQUIRE)
-                logger.warn("fcntl({}, {}, {}) failed, errno ({}).", fd, command, flags, errno(e));
+                logger.error("fcntl({}, {}, {}) failed, error {}", fd, command, flags, e.getMessage());
         }
-
-        return result;
+        return -1;
     }
 
-    public static int tryOpenDirectory(String path)
+    @Override
+    public int tryOpenDirectory(File file)
     {
-        int fd = -1;
+        return tryOpenDirectory(file.path());
+    }
 
+    @Override
+    public int tryOpenDirectory(String path)
+    {
         try
         {
             return wrappedLibrary.callOpen(path, O_RDONLY);
         }
         catch (UnsatisfiedLinkError e)
         {
-            // JNA is unavailable just skipping Direct I/O
+            // Unsupported on this platform
         }
-        catch (RuntimeException e)
+        catch (NativeError e)
         {
-            if (!(e instanceof LastErrorException))
-                throw e;
-
             if (REQUIRE)
-                logger.warn("open({}, O_RDONLY) failed, errno ({}).", path, errno(e));
+                logger.error("open({}, O_RDONLY) failed, error {}", path, e.getMessage());
         }
-
-        return fd;
+        return -1;
     }
 
-    public static void trySync(int fd)
+    @Override
+    public void trySync(int fd)
     {
         if (fd == -1)
             return;
@@ -334,21 +353,19 @@ public static void trySync(int fd)
         {
             // JNA is unavailable just skipping Direct I/O
         }
-        catch (RuntimeException e)
+        catch (NativeError e)
         {
-            if (!(e instanceof LastErrorException))
-                throw e;
-
             if (REQUIRE)
             {
-                String errMsg = String.format("fsync(%s) failed, errno (%s) %s", fd, errno(e), e.getMessage());
+                String errMsg = String.format("fsync(%s) failed, error %s", fd, e.getMessage());
                 logger.warn(errMsg);
                 throw new FSWriteError(e, errMsg);
             }
         }
     }
 
-    public static void tryCloseFD(int fd)
+    @Override
+    public void tryCloseFD(int fd)
     {
         if (fd == -1)
             return;
@@ -361,21 +378,46 @@ public static void tryCloseFD(int fd)
         {
             // JNA is unavailable just skipping Direct I/O
         }
-        catch (RuntimeException e)
+        catch (NativeError e)
         {
-            if (!(e instanceof LastErrorException))
-                throw e;
-
             if (REQUIRE)
             {
-                String errMsg = String.format("close(%d) failed, errno (%d).", fd, errno(e));
+                String errMsg = String.format("close(%d) failed, error %s", fd, e.getMessage());
                 logger.warn(errMsg);
                 throw new FSWriteError(e, errMsg);
             }
         }
     }
 
-    public static int getfd(FileChannel channel)
+    @Override
+    public int getfd(AsynchronousFileChannel channel)
+    {
+        try
+        {
+            return getfd((FileDescriptor) FILE_ASYNC_CHANNEL_FD_FIELD.get(channel));
+        }
+        catch (IllegalArgumentException|IllegalAccessException e)
+        {
+            logger.error("Unable to read fd field from FileChannel");
+        }
+        return -1;
+    }
+
+    @Override
+    public FileDescriptor getFileDescriptor(AsynchronousFileChannel channel)
+    {
+        try
+        {
+            return (FileDescriptor) FILE_ASYNC_CHANNEL_FD_FIELD.get(channel);
+        }
+        catch (IllegalArgumentException | IllegalAccessException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public int getfd(FileChannel channel)
     {
         try
         {
@@ -384,7 +426,7 @@ public static int getfd(FileChannel channel)
         catch (IllegalArgumentException|IllegalAccessException e)
         {
             if (REQUIRE)
-                logger.warn("Unable to read fd field from FileChannel", e);
+                logger.error("Unable to read fd field from FileChannel");
         }
         return -1;
     }
@@ -394,7 +436,8 @@ public static int getfd(FileChannel channel)
      * @param descriptor - FileDescriptor objec to get fd from
      * @return file descriptor, -1 or error
      */
-    public static int getfd(FileDescriptor descriptor)
+    @Override
+    public int getfd(FileDescriptor descriptor)
     {
         try
         {
@@ -405,7 +448,7 @@ public static int getfd(FileDescriptor descriptor)
             if (REQUIRE)
             {
                 JVMStabilityInspector.inspectThrowable(e);
-                logger.warn("Unable to read fd field from FileDescriptor", e);
+                logger.error("Unable to read fd field from FileDescriptor");
             }
         }
 
@@ -415,22 +458,32 @@ public static int getfd(FileDescriptor descriptor)
     /**
      * @return the PID of the JVM or -1 if we failed to get the PID
      */
-    public static long getProcessID()
+    @Override
+    public long getProcessID()
     {
         try
         {
             return wrappedLibrary.callGetpid();
         }
-        catch (UnsatisfiedLinkError e)
-        {
-            // if JNA is unavailable just skipping
-        }
-        catch (Exception e)
+        catch (NativeError e)
         {
             if (REQUIRE)
-                logger.info("Failed to get PID from JNA", e);
+                logger.error("Failed to get PID from JNA", e);
         }
 
         return -1;
     }
+
+    @Override
+    public FileDescriptor getFileDescriptor(FileChannel channel)
+    {
+        try
+        {
+            return (FileDescriptor)FILE_CHANNEL_FD_FIELD.get(channel);
+        }
+        catch (IllegalArgumentException | IllegalAccessException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/utils/NativeLibraryDarwin.java b/src/java/org/apache/cassandra/utils/NativeLibraryDarwin.java
index c1193113700f..a6cf95911568 100644
--- a/src/java/org/apache/cassandra/utils/NativeLibraryDarwin.java
+++ b/src/java/org/apache/cassandra/utils/NativeLibraryDarwin.java
@@ -23,7 +23,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.sun.jna.LastErrorException;
 import com.sun.jna.Native;
 import com.sun.jna.Pointer;
 
@@ -40,7 +39,7 @@
  * unavailable simply because of one native defined method not supported
  * on the runtime operating system.
  * @see org.apache.cassandra.utils.NativeLibraryWrapper
- * @see NativeLibrary
+ * @see INativeLibrary
  */
 @Shared
 public class NativeLibraryDarwin implements NativeLibraryWrapper
@@ -70,61 +69,93 @@ public class NativeLibraryDarwin implements NativeLibraryWrapper
         }
     }
 
-    private static native int mlockall(int flags) throws LastErrorException;
-    private static native int munlockall() throws LastErrorException;
-    private static native int fcntl(int fd, int command, long flags) throws LastErrorException;
-    private static native int open(String path, int flags) throws LastErrorException;
-    private static native int fsync(int fd) throws LastErrorException;
-    private static native int close(int fd) throws LastErrorException;
-    private static native Pointer strerror(int errnum) throws LastErrorException;
-    private static native long getpid() throws LastErrorException;
+    private static native int mlockall(int flags);
+    private static native int munlockall();
+    private static native int fcntl(int fd, int command, long flags);
+    private static native int open(String path, int flags);
+    private static native int fsync(int fd);
+    private static native int close(int fd);
+    private static native Pointer strerror(int errnum);
+    private static native long getpid();
 
-    public int callMlockall(int flags) throws UnsatisfiedLinkError, RuntimeException
+    private void throwNativeError() throws NativeError
     {
-        return mlockall(flags);
+        var errno = Native.getLastError();
+        throw new NativeError(strerror(errno).getString(0), errno);
     }
 
-    public int callMunlockall() throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callMlockall(int flags) throws NativeError
     {
-        return munlockall();
+        if (0 != mlockall(flags))
+            throwNativeError();
+        return 0;
     }
 
-    public int callFcntl(int fd, int command, long flags) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callMunlockall() throws NativeError
     {
-        return fcntl(fd, command, flags);
+        if (0 != munlockall())
+            throwNativeError();
+        return 0;
     }
 
-    public int callPosixFadvise(int fd, long offset, int len, int flag) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callFcntl(int fd, int command, long flags) throws NativeError
+    {
+        int r = fcntl(fd, command, flags);
+        if (r < 0)
+            throwNativeError();
+        return r;
+    }
+
+    @Override
+    public int callPosixFadvise(int fd, long offset, int len, int flag)
     {
-        // posix_fadvise is not available on Darwin/Mac
         throw new UnsatisfiedLinkError();
     }
 
-    public int callOpen(String path, int flags) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callPosixMadvise(Pointer addr, long length, int advice)
     {
-        return open(path, flags);
+        throw new UnsatisfiedLinkError();
     }
 
-    public int callFsync(int fd) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callOpen(String path, int flags) throws NativeError
     {
-        return fsync(fd);
+        int r = open(path, flags);
+        if (r < 0)
+            throwNativeError();
+        return r;
     }
 
-    public int callClose(int fd) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callFsync(int fd) throws NativeError
     {
-        return close(fd);
+        if (0 != fsync(fd))
+            throwNativeError();
+        return 0;
     }
 
-    public Pointer callStrerror(int errnum) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callClose(int fd) throws NativeError
     {
-        return strerror(errnum);
+        if (0 != close(fd))
+            throwNativeError();
+        return 0;
     }
 
-    public long callGetpid() throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public long callGetpid() throws NativeError
     {
-        return getpid();
+        long r = getpid();
+        if (r < 0)
+            throwNativeError();
+        return r;
     }
 
+    @Override
     public boolean isAvailable()
     {
         return available;
diff --git a/src/java/org/apache/cassandra/utils/NativeLibraryLinux.java b/src/java/org/apache/cassandra/utils/NativeLibraryLinux.java
index 9c7bb3b73b11..e0f43b160816 100644
--- a/src/java/org/apache/cassandra/utils/NativeLibraryLinux.java
+++ b/src/java/org/apache/cassandra/utils/NativeLibraryLinux.java
@@ -23,7 +23,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.sun.jna.LastErrorException;
 import com.sun.jna.Native;
 import com.sun.jna.Pointer;
 
@@ -40,7 +39,7 @@
  * unavailable simply because of one native defined method not supported
  * on the runtime operating system.
  * @see org.apache.cassandra.utils.NativeLibraryWrapper
- * @see NativeLibrary
+ * @see INativeLibrary
  */
 @Shared
 public class NativeLibraryLinux implements NativeLibraryWrapper
@@ -70,61 +69,99 @@ public class NativeLibraryLinux implements NativeLibraryWrapper
         }
     }
 
-    private static native int mlockall(int flags) throws LastErrorException;
-    private static native int munlockall() throws LastErrorException;
-    private static native int fcntl(int fd, int command, long flags) throws LastErrorException;
-    private static native int posix_fadvise(int fd, long offset, int len, int flag) throws LastErrorException;
-    private static native int open(String path, int flags) throws LastErrorException;
-    private static native int fsync(int fd) throws LastErrorException;
-    private static native int close(int fd) throws LastErrorException;
-    private static native Pointer strerror(int errnum) throws LastErrorException;
-    private static native long getpid() throws LastErrorException;
-
-    public int callMlockall(int flags) throws UnsatisfiedLinkError, RuntimeException
+    private static native int mlockall(int flags);
+    private static native int munlockall();
+    private static native int fcntl(int fd, int command, long flags);
+    private static native int posix_fadvise(int fd, long offset, int len, int flag);
+    private static native int posix_madvise(Pointer addr, long length, int advice);
+    private static native int open(String path, int flags);
+    private static native int fsync(int fd);
+    private static native int close(int fd);
+    private static native Pointer strerror(int errnum);
+    private static native long getpid();
+
+    private void throwNativeError() throws NativeError
     {
-        return mlockall(flags);
+        var errno = Native.getLastError();
+        throw new NativeError(strerror(errno).getString(0), errno);
     }
 
-    public int callMunlockall() throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callMlockall(int flags) throws NativeError
     {
-        return munlockall();
+        if (0 != mlockall(flags))
+            throwNativeError();
+        return 0;
     }
 
-    public int callFcntl(int fd, int command, long flags) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callMunlockall() throws NativeError
     {
-        return fcntl(fd, command, flags);
+        if (0 != munlockall())
+            throwNativeError();
+        return 0;
     }
 
-    public int callPosixFadvise(int fd, long offset, int len, int flag) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callFcntl(int fd, int command, long flags) throws NativeError
     {
-        return posix_fadvise(fd, offset, len, flag);
+        int r = fcntl(fd, command, flags);
+        if (r < 0)
+            throwNativeError();
+        return r;
     }
 
-    public int callOpen(String path, int flags) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callPosixFadvise(int fd, long offset, int len, int flag) throws NativeError
     {
-        return open(path, flags);
+        if (0 != posix_fadvise(fd, offset, len, flag))
+            throwNativeError();
+        return 0;
     }
 
-    public int callFsync(int fd) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callPosixMadvise(Pointer addr, long length, int advice) throws NativeError
     {
-        return fsync(fd);
+        if (0 != posix_madvise(addr, length, advice))
+            throwNativeError();
+        return 0;
     }
 
-    public int callClose(int fd) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callOpen(String path, int flags) throws NativeError
     {
-        return close(fd);
+        int r = open(path, flags);
+        if (r < 0)
+            throwNativeError();
+        return r;
     }
 
-    public Pointer callStrerror(int errnum) throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callFsync(int fd) throws NativeError
     {
-        return strerror(errnum);
+        if (0 != fsync(fd))
+            throwNativeError();
+        return 0;
     }
 
-    public long callGetpid() throws UnsatisfiedLinkError, RuntimeException
+    @Override
+    public int callClose(int fd) throws NativeError
     {
-        return getpid();
+        if (0 != close(fd))
+            throwNativeError();
+        return 0;
     }
 
+    @Override
+    public long callGetpid() throws NativeError
+    {
+        long r = getpid();
+        if (r < 0)
+            throwNativeError();
+        return r;
+    }
+
+    @Override
     public boolean isAvailable()
     {
         return available;
diff --git a/src/java/org/apache/cassandra/utils/NativeLibraryWindows.java b/src/java/org/apache/cassandra/utils/NativeLibraryWindows.java
new file mode 100644
index 000000000000..497dd074219a
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/NativeLibraryWindows.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Collections;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sun.jna.Native;
+import com.sun.jna.Pointer;
+
+/**
+ * A {@code NativeLibraryWrapper} implementation for Windows.
+ * <p> This implementation only offers support for the {@code callGetpid} method
+ * using the Windows/Kernel32 library.</p>
+ *
+ * @see org.apache.cassandra.utils.NativeLibraryWrapper
+ * @see INativeLibrary
+ */
+@Shared
+public class NativeLibraryWindows implements NativeLibraryWrapper
+{
+    private static final Logger logger = LoggerFactory.getLogger(NativeLibraryWindows.class);
+
+    private static boolean available;
+
+    static
+    {
+        try
+        {
+            Native.register(com.sun.jna.NativeLibrary.getInstance("kernel32", Collections.emptyMap()));
+            available = true;
+        }
+        catch (NoClassDefFoundError e)
+        {
+            logger.warn("JNA not found. Native methods will be disabled.");
+        }
+        catch (UnsatisfiedLinkError e)
+        {
+            logger.error("Failed to link the Windows/Kernel32 library against JNA. Native methods will be unavailable.", e);
+        }
+        catch (NoSuchMethodError e)
+        {
+            logger.warn("Obsolete version of JNA present; unable to register Windows/Kernel32 library. Upgrade to JNA 3.2.7 or later");
+        }
+    }
+
+    /**
+     * Retrieves the process identifier of the calling process (<a href='https://msdn.microsoft.com/en-us/library/windows/desktop/ms683180(v=vs.85).aspx'>GetCurrentProcessId function</a>).
+     *
+     * @return the process identifier of the calling process
+     */
+    private static native long GetCurrentProcessId();
+
+    private void throwNativeError() throws NativeError
+    {
+        var errno = Native.getLastError();
+        // TODO figure out how to get a human-readable error message on Windows
+        throw new NativeError(String.valueOf(errno), errno);
+    }
+
+    @Override
+    public int callMlockall(int flags)
+    {
+        throw new UnsatisfiedLinkError();
+    }
+
+    @Override
+    public int callMunlockall()
+    {
+        throw new UnsatisfiedLinkError();
+    }
+
+    @Override
+    public int callFcntl(int fd, int command, long flags)
+    {
+        throw new UnsatisfiedLinkError();
+    }
+
+    @Override
+    public int callPosixFadvise(int fd, long offset, int len, int flag)
+    {
+        throw new UnsatisfiedLinkError();
+    }
+
+    @Override
+    public int callPosixMadvise(Pointer addr, long length, int advice)
+    {
+        throw new UnsatisfiedLinkError();
+    }
+
+    @Override
+    public int callOpen(String path, int flags)
+    {
+        throw new UnsatisfiedLinkError();
+    }
+
+    @Override
+    public int callFsync(int fd)
+    {
+        throw new UnsatisfiedLinkError();
+    }
+
+    @Override
+    public int callClose(int fd)
+    {
+        throw new UnsatisfiedLinkError();
+    }
+
+    /**
+     * @return the PID of the JVM running
+     */
+    @Override
+    public long callGetpid() throws NativeError
+    {
+        long r = GetCurrentProcessId();
+        if (r < 0)
+            throwNativeError();
+        return r;
+    }
+
+    @Override
+    public boolean isAvailable()
+    {
+        return available;
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/NativeLibraryWrapper.java b/src/java/org/apache/cassandra/utils/NativeLibraryWrapper.java
index 2c3d47fa162c..796c8d940ff3 100644
--- a/src/java/org/apache/cassandra/utils/NativeLibraryWrapper.java
+++ b/src/java/org/apache/cassandra/utils/NativeLibraryWrapper.java
@@ -22,9 +22,15 @@
 
 /**
  * An interface to implement for using OS specific native methods.
- * @see NativeLibrary
+ * @see INativeLibrary
  */
 @Shared
+// Implementors are advised to NOT use JNA's convenient LastErrorException because it relies
+// on checking errno(), which is not reliable.  Linux man page explains,
+//       The value in errno is significant only when the return value of
+//       the call indicated an error (i.e., -1 from most system calls; -1
+//       or NULL from most library functions); ***a function that succeeds is
+//       allowed to change errno.***
 public interface NativeLibraryWrapper
 {
     /**
@@ -33,13 +39,33 @@ public interface NativeLibraryWrapper
      */
     boolean isAvailable();
 
-    int callMlockall(int flags) throws UnsatisfiedLinkError, RuntimeException;
-    int callMunlockall() throws UnsatisfiedLinkError, RuntimeException;
-    int callFcntl(int fd, int command, long flags) throws UnsatisfiedLinkError, RuntimeException;
-    int callPosixFadvise(int fd, long offset, int len, int flag) throws UnsatisfiedLinkError, RuntimeException;
-    int callOpen(String path, int flags) throws UnsatisfiedLinkError, RuntimeException;
-    int callFsync(int fd) throws UnsatisfiedLinkError, RuntimeException;
-    int callClose(int fd) throws UnsatisfiedLinkError, RuntimeException;
-    Pointer callStrerror(int errnum) throws UnsatisfiedLinkError, RuntimeException;
-    long callGetpid() throws UnsatisfiedLinkError, RuntimeException;
+    int callMlockall(int flags) throws UnsatisfiedLinkError, NativeError;
+    int callMunlockall() throws UnsatisfiedLinkError, NativeError;
+    int callFcntl(int fd, int command, long flags) throws UnsatisfiedLinkError, NativeError;
+    int callPosixFadvise(int fd, long offset, int len, int flag) throws UnsatisfiedLinkError, NativeError;
+    int callPosixMadvise(Pointer addr, long length, int advice) throws UnsatisfiedLinkError, NativeError;
+    int callOpen(String path, int flags) throws UnsatisfiedLinkError, NativeError;
+    int callFsync(int fd) throws UnsatisfiedLinkError, NativeError;
+    int callClose(int fd) throws UnsatisfiedLinkError, NativeError;
+    long callGetpid() throws NativeError;
+
+    /**
+     * This is a checked exception because the correct handling of the error is almost
+     * always to log it and move on, not to propagate it up the stack.
+     */
+    class NativeError extends Exception
+    {
+        private final int errno;
+
+        public NativeError(String nativeMessage, int errno)
+        {
+            super(nativeMessage);
+            this.errno = errno;
+        }
+
+        public int getErrno()
+        {
+            return errno;
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java b/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
index 19dcc23dcf3a..cd5454284ac3 100644
--- a/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
+++ b/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
@@ -17,23 +17,42 @@
  */
 package org.apache.cassandra.utils;
 
-import java.nio.ByteBuffer;
 import java.net.InetSocketAddress;
-import java.util.*;
-
-import com.datastax.driver.core.*;
-
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.datastax.driver.core.AuthProvider;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.Metadata;
+import com.datastax.driver.core.PlainTextAuthProvider;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.SSLOptions;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.TokenRange;
 import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.dht.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.dht.Token.TokenFactory;
 import org.apache.cassandra.io.sstable.SSTableLoader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.CQLTypeParser;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder;
+import org.apache.cassandra.schema.DroppedColumn;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaKeyspaceTables;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.Types;
 
 public class NativeSSTableLoaderClient extends SSTableLoader.Client
 {
@@ -220,7 +239,7 @@ private static DroppedColumn createDroppedColumnFromRow(Row row, String keyspace
         String name = row.getString("column_name");
         AbstractType<?> type = CQLTypeParser.parse(keyspace, row.getString("type"), Types.none());
         ColumnMetadata.Kind kind = ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase());
-        ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind, null);
+        ColumnMetadata column = ColumnMetadata.droppedColumn(keyspace, table, ColumnIdentifier.getInterned(name, true), type, kind, null);
         long droppedTime = row.getTimestamp("dropped_time").getTime();
         return new DroppedColumn(column, droppedTime);
     }
diff --git a/src/java/org/apache/cassandra/utils/NoSpamLogger.java b/src/java/org/apache/cassandra/utils/NoSpamLogger.java
index 0a13f6b2a5ae..c5686a816be4 100644
--- a/src/java/org/apache/cassandra/utils/NoSpamLogger.java
+++ b/src/java/org/apache/cassandra/utils/NoSpamLogger.java
@@ -47,7 +47,7 @@ public class NoSpamLogger
      */
     public enum Level
     {
-        INFO, WARN, ERROR
+        DEBUG, INFO, WARN, ERROR
     }
 
     @VisibleForTesting
@@ -100,6 +100,9 @@ private boolean logNoCheck(Level l, Object... objects)
         {
             switch (l)
             {
+                case DEBUG:
+                    wrapped.debug(statement, objects);
+                    break;
                 case INFO:
                     wrapped.info(statement, objects);
                     break;
@@ -115,6 +118,16 @@ private boolean logNoCheck(Level l, Object... objects)
             return true;
         }
 
+        public boolean debug(long nowNanos, Object... objects)
+        {
+            return NoSpamLogStatement.this.log(Level.DEBUG, nowNanos, objects);
+        }
+
+        public boolean debug(Object... objects)
+        {
+            return NoSpamLogStatement.this.debug(CLOCK.nanoTime(), objects);
+        }
+
         public boolean info(long nowNanos, Object... objects)
         {
             return NoSpamLogStatement.this.log(Level.INFO, nowNanos, objects);
@@ -217,6 +230,16 @@ private NoSpamLogger(Logger wrapped, long minInterval, TimeUnit timeUnit)
         minIntervalNanos = timeUnit.toNanos(minInterval);
     }
 
+    public boolean debug(long nowNanos, String s, Object... objects)
+    {
+        return NoSpamLogger.this.log( Level.DEBUG, s, nowNanos, objects);
+    }
+
+    public boolean debug(String s, Object... objects)
+    {
+        return NoSpamLogger.this.debug(CLOCK.nanoTime(), s, objects);
+    }
+
     public boolean info(long nowNanos, String s, Object... objects)
     {
         return NoSpamLogger.this.log( Level.INFO, s, nowNanos, objects);
diff --git a/src/java/org/apache/cassandra/utils/NonThrowingCloseable.java b/src/java/org/apache/cassandra/utils/NonThrowingCloseable.java
new file mode 100644
index 000000000000..684d6bd30346
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/NonThrowingCloseable.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.io.Closeable;
+
+/**
+ * A closeable that will not throw.
+ */
+public interface NonThrowingCloseable extends Closeable
+{
+    void close();
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/OutputHandler.java b/src/java/org/apache/cassandra/utils/OutputHandler.java
index 76eb34558ff2..95507763d685 100644
--- a/src/java/org/apache/cassandra/utils/OutputHandler.java
+++ b/src/java/org/apache/cassandra/utils/OutputHandler.java
@@ -57,7 +57,19 @@ default void warn(String msg, Object ... args)
 
     class LogOutput implements OutputHandler
     {
-        private static Logger logger = LoggerFactory.getLogger(LogOutput.class);
+        private static final Logger LOGGER_LOGOUTPUT = LoggerFactory.getLogger(LogOutput.class);
+
+        private final Logger logger;
+
+        public LogOutput(Logger logger)
+        {
+            this.logger = logger;
+        }
+
+        public LogOutput()
+        {
+            this(LOGGER_LOGOUTPUT);
+        }
 
         public void output(String msg)
         {
@@ -80,6 +92,15 @@ public void warn(Throwable th, String msg)
         }
     }
 
+    @DseLegacy
+    class CustomLogOutput extends LogOutput
+    {
+        public CustomLogOutput(Logger customLogger)
+        {
+            super(customLogger);
+        }
+    }
+
     class SystemOutput implements OutputHandler
     {
         public final boolean debug;
diff --git a/src/java/org/apache/cassandra/utils/Overlaps.java b/src/java/org/apache/cassandra/utils/Overlaps.java
index 6e7c2ef41636..9aae6072f06e 100644
--- a/src/java/org/apache/cassandra/utils/Overlaps.java
+++ b/src/java/org/apache/cassandra/utils/Overlaps.java
@@ -19,60 +19,126 @@
 package org.apache.cassandra.utils;
 
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.PriorityQueue;
 import java.util.Set;
+import java.util.function.BiFunction;
 import java.util.function.BiPredicate;
+import java.util.function.Consumer;
+import java.util.stream.IntStream;
 
 public class Overlaps
 {
-    /**
-     * Construct a minimal list of overlap sets, i.e. the sections of the range span when we have overlapping items,
-     * where we ensure:
-     * - non-overlapping items are never put in the same set
-     * - no item is present in non-consecutive sets
-     * - for any point where items overlap, the result includes a set listing all overlapping items
-     * <p>
-     * For example, for inputs A[0, 4), B[2, 8), C[6, 10), D[1, 9) the result would be the sets ABD and BCD. We are not
-     * interested in the spans where A, B, or C are present on their own or in combination with D, only that there
-     * exists a set in the list that is a superset of any such combination, and that the non-overlapping A and C are
-     * never together in a set.
-     * <p>
-     * Note that the full list of overlap sets A, AD, ABD, BD, BCD, CD, C is also an answer that satisfies the three
-     * conditions above, but it contains redundant sets (e.g. AD is already contained in ABD).
-     *
-     * @param items            A list of items to distribute in overlap sets. This is assumed to be a transient list and the method
-     *                         may modify or consume it. It is assumed that the start and end positions of an item are ordered,
-     *                         and the items are non-empty.
-     * @param startsAfter      Predicate determining if its left argument's start if fully after the right argument's end.
-     *                         This will only be used with arguments where left's start is known to be after right's start.
-     *                         It is up to the caller if this is a strict comparison -- strict (>) for end-inclusive spans
-     *                         and non-strict (>=) for end-exclusive.
-     * @param startsComparator Comparator of items' starting positions.
-     * @param endsComparator   Comparator of items' ending positions.
-     * @return List of overlap sets.
-     */
-    public static <E> List<Set<E>> constructOverlapSets(List<E> items,
+    /// Construct a minimal list of overlap sets, i.e. the sections of the range span when we have overlapping items,
+    /// where we ensure:
+    /// - non-overlapping items are never put in the same set
+    /// - no item is present in non-consecutive sets
+    /// - for any point where items overlap, the result includes a set listing all overlapping items
+    ///
+    /// For example, for inputs A[0, 4), B[2, 8), C[6, 10), D[1, 9) the result would be the sets ABD and BCD. We are not
+    /// interested in the spans where A, B, or C are present on their own or in combination with D, only that there
+    /// exists a set in the list that is a superset of any such combination, and that the non-overlapping A and C are
+    /// never together in a set.
+    ///
+    /// Note that the full list of overlap sets A, AD, ABD, BD, BCD, CD, C is also an answer that satisfies the three
+    /// conditions above, but it contains redundant sets (e.g. AD is already contained in ABD).
+    ///
+    /// @param items            A list of items to distribute in overlap sets. This is assumed to be a transient list and the method
+    ///                         may modify or consume it. It is assumed that the start and end positions of an item are ordered,
+    ///                         and the items are non-empty.
+    /// @param startsAfter      Predicate determining if its left argument's start if fully after the right argument's end.
+    ///                         This will only be used with arguments where left's start is known to be after right's start.
+    ///                         It is up to the caller if this is a strict comparison -- strict (>) for end-inclusive spans
+    ///                         and non-strict (>=) for end-exclusive.
+    /// @param startsComparator Comparator of items' starting positions.
+    /// @param endsComparator   Comparator of items' ending positions.
+    /// @return List of overlap sets.
+    public static <E> List<Set<E>> constructOverlapSets(Collection<E> items,
                                                         BiPredicate<E, E> startsAfter,
                                                         Comparator<E> startsComparator,
                                                         Comparator<E> endsComparator)
     {
-        List<Set<E>> overlaps = new ArrayList<>();
+        return constructOverlapSets(items, startsAfter, startsComparator, endsComparator,
+                                    (sets, active) -> {
+                                        sets.add(new HashSet<>(active));
+                                        return sets;
+                                    },
+                                    new ArrayList<>());
+    }
+
+    /// This is the same as the method above, but only returns the size of the biggest overlap set
+    ///
+    /// @param items            A list of items to distribute in overlap sets. This is assumed to be a transient list and the method
+    ///                         may modify or consume it. It is assumed that the start and end positions of an item are ordered,
+    ///                         and the items are non-empty.
+    /// @param startsAfter      Predicate determining if its left argument's start if fully after the right argument's end.
+    ///                         This will only be used with arguments where left's start is known to be after right's start.
+    ///                         It is up to the caller if this is a strict comparison -- strict (>) for end-inclusive spans
+    ///                         and non-strict (>=) for end-exclusive.
+    /// @param startsComparator Comparator of items' starting positions.
+    /// @param endsComparator   Comparator of items' ending positions.
+    /// @return The maximum overlap in the given set of items.
+    public static <E> int maxOverlap(Collection<E> items,
+                                     BiPredicate<? super E, ? super E> startsAfter,
+                                     Comparator<? super E> startsComparator,
+                                     Comparator<? super E> endsComparator)
+    {
+        return constructOverlapSets(items, startsAfter, startsComparator, endsComparator,
+                                    (max, active) -> Math.max(max, active.size()), 0);
+    }
+
+    /// Construct a minimal list of overlap sets, i.e. the sections of the range span when we have overlapping items,
+    /// where we ensure:
+    /// - non-overlapping items are never put in the same set
+    /// - no item is present in non-consecutive sets
+    /// - for any point where items overlap, the result includes a set listing all overlapping items
+    /// and process it with the given reducer function. Implements the methods above.
+    ///
+    /// For example, for inputs A[0, 4), B[2, 8), C[6, 10), D[1, 9) the result would be the sets ABD and BCD. We are not
+    /// interested in the spans where A, B, or C are present on their own or in combination with D, only that there
+    /// exists a set in the list that is a superset of any such combination, and that the non-overlapping A and C are
+    /// never together in a set.
+    ///
+    /// Note that the full list of overlap sets A, AD, ABD, BD, BCD, CD, C is also an answer that satisfies the three
+    /// conditions above, but it contains redundant sets (e.g. AD is already contained in ABD).
+    ///
+    /// @param items            A list of items to distribute in overlap sets. It is assumed that the start and end
+    ///                         positions of an item are ordered, and the items are non-empty.
+    /// @param startsAfter      Predicate determining if its left argument's start if fully after the right argument's end.
+    ///                         This will only be used with arguments where left's start is known to be after right's start.
+    ///                         It is up to the caller if this is a strict comparison -- strict (>) for end-inclusive spans
+    ///                         and non-strict (>=) for end-exclusive.
+    /// @param startsComparator Comparator of items' starting positions.
+    /// @param endsComparator   Comparator of items' ending positions.
+    /// @param reducer          Function to apply to each overlap set.
+    /// @param initialValue     Initial value for the reducer.
+    /// @return The result of processing the overlap sets.
+    public static <E, R> R constructOverlapSets(Collection<E> items,
+                                                BiPredicate<? super E, ? super E> startsAfter,
+                                                Comparator<? super E> startsComparator,
+                                                Comparator<? super E> endsComparator,
+                                                BiFunction<R, Collection<E>, R> reducer,
+                                                R initialValue)
+    {
+        R overlaps = initialValue;
         if (items.isEmpty())
             return overlaps;
 
         PriorityQueue<E> active = new PriorityQueue<>(endsComparator);
-        items.sort(startsComparator);
-        for (E item : items)
+        SortingIterator<E> itemsSorted = SortingIterator.create(startsComparator, items);
+        while (itemsSorted.hasNext())
         {
+            E item = itemsSorted.next();
             if (!active.isEmpty() && startsAfter.test(item, active.peek()))
             {
                 // New item starts after some active ends. It does not overlap with it, so:
                 // -- output the previous active set
-                overlaps.add(new HashSet<>(active));
+                overlaps = reducer.apply(overlaps, active);
                 // -- remove all items that also end before the current start
                 do
                 {
@@ -87,13 +153,71 @@ public static <E> List<Set<E>> constructOverlapSets(List<E> items,
         }
 
         assert !active.isEmpty();
-        overlaps.add(new HashSet<>(active));
+        overlaps = reducer.apply(overlaps, active);
 
         return overlaps;
     }
+
+    /// Transform a list to transitively combine adjacent sets that have a common element, resulting in disjoint sets.
+    public static <T> List<Set<T>> combineSetsWithCommonElement(List<? extends Set<T>> overlapSets)
+    {
+        Set<T> group = overlapSets.get(0);
+        List<Set<T>> groups = new ArrayList<>();
+        for (int i = 1; i < overlapSets.size(); ++i)
+        {
+            Set<T> current = overlapSets.get(i);
+            if (Collections.disjoint(current, group))
+            {
+                groups.add(group);
+                group = current;
+            }
+            else
+            {
+                group.addAll(current);
+            }
+        }
+        groups.add(group);
+        return groups;
+    }
+
+    /// Split a list of items into disjoint non-overlapping sets.
+    ///
+    /// @param items            A list of items to distribute in overlap sets. It is assumed that the start and end
+    ///                         positions of an item are ordered, and the items are non-empty.
+    /// @param startsAfter      Predicate determining if its left argument's start if fully after the right argument's end.
+    ///                         This will only be used with arguments where left's start is known to be after right's start.
+    ///                         It is up to the caller if this is a strict comparison -- strict (>) for end-inclusive spans
+    ///                         and non-strict (>=) for end-exclusive.
+    /// @param startsComparator Comparator of items' starting positions.
+    /// @param endsComparator   Comparator of items' ending positions.
+    /// @return list of non-overlapping sets of items
+    public static <T> List<Set<T>> splitInNonOverlappingSets(List<T> items,
+                                                             BiPredicate<T, T> startsAfter,
+                                                             Comparator<T> startsComparator,
+                                                             Comparator<T> endsComparator)
+    {
+        if (items.isEmpty())
+            return List.of();
+
+        List<Set<T>> overlapSets = Overlaps.constructOverlapSets(items, startsAfter, startsComparator, endsComparator);
+        return combineSetsWithCommonElement(overlapSets);
+    }
+
+
+    /// Overlap inclusion method to use when combining overlap sections into a bucket. For example, with
+    /// items A(0, 5), B(2, 9), C(6, 12), D(10, 12) whose overlap sections calculation returns \[AB, BC, CD\],
+    ///   - `NONE` means no sections are to be merged. AB, BC and CD will be separate buckets, selections AB, BC and CD
+    ///     will be added separately, thus some items will be partially used / single-source compacted, likely
+    ///     to be recompacted again with the next selected bucket.
+    ///   - `SINGLE` means only overlaps of the sstables in the selected bucket will be added. AB+BC will be one bucket,
+    ///     and CD will be another (as BC is already used). A middle ground of sorts, should reduce overcompaction but
+    ///     still has some.
+    ///   - `TRANSITIVE` means a transitive closure of overlapping sstables will be selected. AB+BC+CD will be in the
+    ///     same bucket, selected compactions will apply to all overlapping sstables and no overcompaction will be done,
+    ///     at the cost of reduced compaction parallelism and increased length of the operation.
     public enum InclusionMethod
     {
-        NONE, SINGLE, TRANSITIVE;
+        NONE, SINGLE, TRANSITIVE
     }
 
     public interface BucketMaker<E, B>
@@ -101,65 +225,166 @@ public interface BucketMaker<E, B>
         B makeBucket(List<Set<E>> sets, int startIndexInclusive, int endIndexExclusive);
     }
 
-    /**
-     * Assign overlap sections into buckets. Identify sections that have at least threshold-many overlapping
-     * items and apply the overlap inclusion method to combine with any neighbouring sections that contain
-     * selected sstables to make sure we make full use of any sstables selected for compaction (i.e. avoid
-     * recompacting, see {@link org.apache.cassandra.db.compaction.unified.Controller#overlapInclusionMethod()}).
-     *
-     * @param threshold       Threshold for selecting a bucket. Sets below this size will be ignored, unless they need
-     *                        to be grouped with a neighboring set due to overlap.
-     * @param inclusionMethod NONE to only form buckets of the overlapping sets, SINGLE to include all
-     *                        sets that share an sstable with a selected bucket, or TRANSITIVE to include
-     *                        all sets that have an overlap chain to a selected bucket.
-     * @param overlaps        An ordered list of overlap sets as returned by {@link #constructOverlapSets}.
-     * @param bucketer        Method used to create a bucket out of the supplied set indexes.
-     */
+    /// Assign overlap sections into buckets. Identify sections that have at least threshold-many overlapping
+    /// items and apply the overlap inclusion method to combine with any neighbouring sections that contain
+    /// selected sstables to make sure we make full use of any sstables selected for compaction (i.e. avoid
+    /// recompacting, see [InclusionMethod]).
+    ///
+    /// For non-transitive inclusion method the order in which we select the buckets matters because an sstables that
+    /// spans overlap sets could be chosen for only one of the candidate buckets containing it. To make the most
+    /// efficient selection we thus perform it by descending size, starting with the sets with most overlap.
+    ///
+    /// @param threshold       Threshold for selecting a bucket. Sets below this size will be ignored, unless they need
+    ///                        to be grouped with a neighboring set due to overlap.
+    /// @param inclusionMethod `NONE` to only form buckets of the overlapping sets, `SINGLE` to include all
+    ///                        sets that share an sstable with a selected bucket, or `TRANSITIVE` to include
+    ///                        all sets that have an overlap chain to a selected bucket.
+    /// @param overlaps        An ordered list of overlap sets as returned by [#constructOverlapSets].
+    /// @param bucketer        Method used to create a bucket out of the supplied set indexes.
+    /// @param unselectedHandler Action to take on sets that are below the threshold and not included in any bucket.
     public static <E, B> List<B> assignOverlapsIntoBuckets(int threshold,
                                                            InclusionMethod inclusionMethod,
                                                            List<Set<E>> overlaps,
-                                                           BucketMaker<E, B> bucketer)
+                                                           BucketMaker<E, B> bucketer,
+                                                           Consumer<Set<E>> unselectedHandler)
+    {
+        switch (inclusionMethod)
+        {
+            case TRANSITIVE:
+                return assignOverlapsTransitive(threshold, overlaps, bucketer, unselectedHandler);
+            case SINGLE:
+            case NONE:
+                return assignOverlapsSingleOrNone(threshold, inclusionMethod, overlaps, bucketer, unselectedHandler);
+            default:
+                throw new UnsupportedOperationException(inclusionMethod + " is not supported");
+        }
+    }
+
+    private static <E, B> List<B> assignOverlapsSingleOrNone(int threshold,
+                                                             InclusionMethod inclusionMethod,
+                                                             List<Set<E>> overlaps,
+                                                             BucketMaker<E, B> bucketer,
+                                                             Consumer<Set<E>> unselectedHandler)
     {
         List<B> buckets = new ArrayList<>();
         int regionCount = overlaps.size();
-        int lastEnd = -1;
-        for (int i = 0; i < regionCount; ++i)
+        SortingIterator<Integer> bySize = new SortingIterator<>((a, b) -> Integer.compare(overlaps.get(b).size(),
+                                                                                          overlaps.get(a).size()),
+                                                                overlaps.isEmpty() ? new Integer[1] : IntStream.range(0, overlaps.size()).boxed().toArray());
+
+        BitSet used = new BitSet(overlaps.size());
+        while (bySize.hasNext())
         {
-            Set<E> bucket = overlaps.get(i);
-            int maxOverlap = bucket.size();
-            if (maxOverlap < threshold)
+            final int i = bySize.next();
+            if (used.get(i))
                 continue;
+
+            Set<E> bucket = overlaps.get(i);
+            if (bucket.size() < threshold)
+                break;  // no more buckets will be above threshold
+            used.set(i);
+
+            Set<E> allOverlapping = bucket;
+            int j = i - 1;
+            int k = i + 1;
             int startIndex = i;
             int endIndex = i + 1;
-
-            if (inclusionMethod != InclusionMethod.NONE)
+            // expand to include neighbors that intersect with current bucket
+            if (inclusionMethod == InclusionMethod.SINGLE)
             {
-                Set<E> allOverlapping = new HashSet<>(bucket);
-                Set<E> overlapTarget = inclusionMethod == InclusionMethod.TRANSITIVE
-                                       ? allOverlapping
-                                       : bucket;
-                int j;
-                for (j = i - 1; j > lastEnd; --j)
+                // expand the bucket to include all overlapping sets
+                allOverlapping = new HashSet<>(bucket);
+                Set<E> overlapTarget = bucket;
+                for (; j >= 0 && !used.get(j); --j)
                 {
                     Set<E> next = overlaps.get(j);
                     if (!setsIntersect(next, overlapTarget))
                         break;
                     allOverlapping.addAll(next);
+                    used.set(j);
                 }
                 startIndex = j + 1;
-                for (j = i + 1; j < regionCount; ++j)
+                for (; k < regionCount && !used.get(k); ++k)
                 {
-                    Set<E> next = overlaps.get(j);
+                    Set<E> next = overlaps.get(k);
                     if (!setsIntersect(next, overlapTarget))
                         break;
                     allOverlapping.addAll(next);
+                    used.set(k);
                 }
-                i = j - 1;
-                endIndex = j;
+                endIndex = k;
+            }
+            // Now mark all overlapping with the extended as used
+            Set<E> overlapTarget = allOverlapping;
+            for (; j >= 0 && !used.get(j); --j)
+            {
+                Set<E> next = overlaps.get(j);
+                if (!setsIntersect(next, overlapTarget))
+                    break;
+                used.set(j);
+                unselectedHandler.accept(next);
+            }
+            for (; k < regionCount && !used.get(k); ++k)
+            {
+                Set<E> next = overlaps.get(k);
+                if (!setsIntersect(next, overlapTarget))
+                    break;
+                used.set(k);
+                unselectedHandler.accept(next);
+            }
+            buckets.add(bucketer.makeBucket(overlaps, startIndex, endIndex));
+        }
+
+        for (int i = used.nextClearBit(0); i < regionCount; i = used.nextClearBit(i + 1))
+            unselectedHandler.accept(overlaps.get(i));
+
+        return buckets;
+    }
+
+    private static <E, B> List<B> assignOverlapsTransitive(int threshold,
+                                                           List<Set<E>> overlaps,
+                                                           BucketMaker<E, B> bucketer,
+                                                           Consumer<Set<E>> unselectedHandler)
+    {
+        List<B> buckets = new ArrayList<>();
+        int regionCount = overlaps.size();
+        int lastEnd = 0;
+        for (int i = 0; i < regionCount; ++i)
+        {
+            Set<E> bucket = overlaps.get(i);
+            int maxOverlap = bucket.size();
+            if (maxOverlap < threshold)
+                continue;
+
+            // expand to include neighbors that intersect with expanded buckets
+            Set<E> allOverlapping = new HashSet<>(bucket);
+            Set<E> overlapTarget = allOverlapping;
+            int j;
+            for (j = i - 1; j >= lastEnd; --j)
+            {
+                Set<E> next = overlaps.get(j);
+                if (!setsIntersect(next, overlapTarget))
+                    break;
+                allOverlapping.addAll(next);
+            }
+            int startIndex = j + 1;
+            for (j = i + 1; j < regionCount; ++j)
+            {
+                Set<E> next = overlaps.get(j);
+                if (!setsIntersect(next, overlapTarget))
+                    break;
+                allOverlapping.addAll(next);
             }
+            i = j - 1;
+            int endIndex = j;
+
             buckets.add(bucketer.makeBucket(overlaps, startIndex, endIndex));
-            lastEnd = i;
+            for (int k = lastEnd; k < startIndex; ++k)
+                unselectedHandler.accept(overlaps.get(k));
+            lastEnd = endIndex;
         }
+        for (int k = lastEnd; k < regionCount; ++k)
+            unselectedHandler.accept(overlaps.get(k));
         return buckets;
     }
 
@@ -173,9 +398,7 @@ private static <E> boolean setsIntersect(Set<E> s1, Set<E> s2)
         return false;
     }
 
-    /**
-     * Pull the last elements from the given list, up to the given limit.
-     */
+    /// Pull the last elements from the given list, up to the given limit.
     public static <T> List<T> pullLast(List<T> source, int limit)
     {
         List<T> result = new ArrayList<>(limit);
@@ -184,11 +407,9 @@ public static <T> List<T> pullLast(List<T> source, int limit)
         return result;
     }
 
-    /**
-     * Select up to `limit` sstables from each overlapping set (more than `limit` in total) by taking the last entries
-     * from `allObjectsSorted`. To achieve this, keep selecting the last sstable until the next one we would add would
-     * bring the number selected in some overlap section over `limit`.
-     */
+    /// Select up to `limit` sstables from each overlapping set (more than `limit` in total) by taking the last entries
+    /// from `allObjectsSorted`. To achieve this, keep selecting the last sstable until the next one we would add would
+    /// bring the number selected in some overlap section over `limit`.
     public static <T> Collection<T> pullLastWithOverlapLimit(List<T> allObjectsSorted, List<Set<T>> overlapSets, int limit)
     {
         int setsCount = overlapSets.size();
diff --git a/src/java/org/apache/cassandra/utils/ProductType.java b/src/java/org/apache/cassandra/utils/ProductType.java
new file mode 100644
index 000000000000..35bfaa02f197
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/ProductType.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProductType
+{
+    private static final Logger logger = LoggerFactory.getLogger(ProductType.class);
+
+    public static Product product = getProduct();
+
+    public enum Product
+    {
+        /**
+         * On-Premises product
+         */
+        DATASTAX_CASSANDRA,
+
+        /**
+         * Datastax constellation database-as-a-service product (NOT referring to dse-db, aka. apollo)
+         */
+        DATASTAX_APOLLO
+    }
+
+    @VisibleForTesting
+    public static Product getProduct()
+    {
+        Product defaultType = Product.DATASTAX_CASSANDRA;
+        // checkstyle: suppress below 'blockSystemPropertyUsage'
+        String productType = System.getProperty("dse.product_type", defaultType.name());
+        try
+        {
+            return Product.valueOf(productType.toUpperCase());
+        }
+        catch (IllegalArgumentException e)
+        {
+            logger.info("Unknown product type '{}', will use default product type '{}'.", productType, defaultType.name());
+            return defaultType;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/ReadWriteLockedList.java b/src/java/org/apache/cassandra/utils/ReadWriteLockedList.java
new file mode 100644
index 000000000000..e6535003802e
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/ReadWriteLockedList.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.AbstractList;
+import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public class ReadWriteLockedList<T> extends AbstractList<T>
+{
+    private final List<T> list;
+    private final Lock readLock;
+    private final Lock writeLock;
+
+    public ReadWriteLockedList(List<T> list)
+    {
+        this.list = list;
+        ReadWriteLock rwLock = new ReentrantReadWriteLock();
+        readLock = rwLock.readLock();
+        writeLock = rwLock.writeLock();
+    }
+
+    @Override
+    public T set(int index, T element)
+    {
+        writeLock.lock();
+        try
+        {
+            return list.set(index, element);
+        }
+        finally
+        {
+            writeLock.unlock();
+        }
+    }
+
+    @Override
+    public boolean add(T item)
+    {
+        writeLock.lock();
+        try
+        {
+            return list.add(item);
+        }
+        finally
+        {
+            writeLock.unlock();
+        }
+    }
+
+    @Override
+    public T get(int index)
+    {
+        readLock.lock();
+        try
+        {
+            return list.get(index);
+        }
+        finally
+        {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public int size()
+    {
+        readLock.lock();
+        try
+        {
+            return list.size();
+        }
+        finally
+        {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        readLock.lock();
+        try
+        {
+            return list.isEmpty();
+        }
+        finally
+        {
+            readLock.unlock();
+        }
+    }
+
+    public static <T> ReadWriteLockedList<T> wrap(List<T> list)
+    {
+        return new ReadWriteLockedList<>(list);
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/Reducer.java b/src/java/org/apache/cassandra/utils/Reducer.java
new file mode 100644
index 000000000000..1915c460c1ff
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/Reducer.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+/** Accumulator that collects values of type A, and outputs a value of type B. */
+public abstract class Reducer<In,Out>
+{
+    /**
+     * @return true if Out is the same as In for the case of a single source iterator
+     */
+    public boolean singleSourceReduceIsTrivial()
+    {
+        return false;
+    }
+
+    /**
+     * combine this object with the previous ones.
+     * intermediate state is up to your implementation.
+     */
+    public abstract void reduce(int idx, In current);
+
+    Throwable errors = null;
+
+    public void error(Throwable error)
+    {
+        errors = Throwables.merge(errors, error);
+    }
+
+    public Throwable getErrors()
+    {
+        Throwable toReturn = errors;
+        errors = null;
+        return toReturn;
+    }
+
+    /** @return The last object computed by reduce */
+    public abstract Out getReduced();
+
+    /**
+     * Called at the beginning of each new key, before any reduce is called.
+     * To be overridden by implementing classes.
+     *
+     * Note: There's no need to clear error; merging completes once one is found.
+     */
+    public void onKeyChange() {}
+
+    public static <In> Reducer<In, In> getIdentity()
+    {
+        return new IdentityReducer<>();
+    }
+
+    private static class IdentityReducer<In> extends Reducer<In, In>
+    {
+        private In reduced;
+
+        @Override
+        public void reduce(int idx, In current)
+        {
+            this.reduced = current;
+        }
+
+        @Override
+        public In getReduced()
+        {
+            return reduced;
+        }
+
+        @Override
+        public void onKeyChange() {
+            this.reduced = null;
+        }
+
+        @Override
+        public boolean singleSourceReduceIsTrivial()
+        {
+            return true;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/ResourceWatcher.java b/src/java/org/apache/cassandra/utils/ResourceWatcher.java
index e8dcb8574372..71a2cace8f4e 100644
--- a/src/java/org/apache/cassandra/utils/ResourceWatcher.java
+++ b/src/java/org/apache/cassandra/utils/ResourceWatcher.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.utils;
 
+import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.cassandra.io.util.File;
@@ -27,7 +28,7 @@
 
 public class ResourceWatcher
 {
-    public static void watch(String resource, Runnable callback, int period)
+    public static void watch(String resource, Callable<Boolean> callback, int period)
     {
         ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(new WatchedResource(resource, callback), period, period, TimeUnit.MILLISECONDS);
     }
@@ -36,10 +37,10 @@ public static class WatchedResource implements Runnable
     {
         private static final Logger logger = LoggerFactory.getLogger(WatchedResource.class);
         private final String resource;
-        private final Runnable callback;
+        private final Callable<Boolean> callback;
         private long lastLoaded;
 
-        public WatchedResource(String resource, Runnable callback)
+        public WatchedResource(String resource, Callable<Boolean> callback)
         {
             this.resource = resource;
             this.callback = callback;
@@ -54,8 +55,8 @@ public void run()
                 long lastModified = new File(filename).lastModified();
                 if (lastModified > lastLoaded)
                 {
-                    callback.run();
-                    lastLoaded = lastModified;
+                    if (callback.call())
+                        lastLoaded = lastModified;
                 }
             }
             catch (Throwable t)
diff --git a/src/java/org/apache/cassandra/utils/SigarLibrary.java b/src/java/org/apache/cassandra/utils/SigarLibrary.java
index 830f7cab8eb7..578eba8a7089 100644
--- a/src/java/org/apache/cassandra/utils/SigarLibrary.java
+++ b/src/java/org/apache/cassandra/utils/SigarLibrary.java
@@ -44,7 +44,7 @@ public class SigarLibrary
 
     private SigarLibrary()
     {
-        logger.info("Initializing SIGAR library");
+        logger.debug("Initializing SIGAR library");
         try
         {
             sigar = new Sigar();
diff --git a/src/java/org/apache/cassandra/utils/SortingIterator.java b/src/java/org/apache/cassandra/utils/SortingIterator.java
new file mode 100644
index 000000000000..3bc53185f9b0
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/SortingIterator.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.function.Function;
+
+/**
+ * An iterator that lists a set of items in order.
+ * <p>
+ * This is intended for use where we would normally read only a small subset of the elements, or where we would skip
+ * over large sections of the sorted set. To implement this efficiently, we put the data in a binary heap and extract
+ * elements as the iterator is queried, effectively performing heapsort. We also implement a quicker skipTo operation
+ * where we remove all smaller elements and restore the heap for all of them in one step.
+ * <p>
+ * As in heapsort, the first stage of the process has complexity O(n), and every next item is extracted in O(log n)
+ * steps. skipTo works in O(m.log n) steps (where m is the number of skipped items), but is also limited to O(n) when m
+ * is large by the same argument as the initial heap construction.
+ * <p>
+ * The class accepts and stores nulls as non-present values, which turns out to be quite a bit more efficient for
+ * iterating these sets when the comparator is complex at the expense of a small slowdown for simple comparators. The
+ * reason for this is that we can remove entries by replacing them with nulls and letting these descend the heap, which
+ * avoids half the comparisons compared to using one of the largest live elements.
+ * <p>
+ * If the number of items necessary is small and known in advance, it may be preferable to use {@link TopKSelector}
+ * which keeps a smaller memory footprint.
+ */
+public class SortingIterator<T> extends BinaryHeap.WithComparator<T> implements Iterator<T>
+{
+    SortingIterator(Comparator<? super T> comparator, Object[] data)
+    {
+        super(comparator, data);
+        heapify();
+    }
+
+    /**
+     * Create a sorting iterator from a list of sources.
+     * Duplicates will be returned in arbitrary order.
+     */
+    public static <T> SortingIterator<T> create(Comparator<? super T> comparator, Collection<T> sources)
+    {
+        return new SortingIterator<>(comparator, sources.isEmpty() ? new Object[1] : sources.toArray());
+    }
+
+    /**
+     * Create a closeable sorting iterator from a list of sources, calling the given method on close.
+     * Duplicates will be returned in arbitrary order.
+     */
+    public static <T, V> CloseableIterator<T> createCloseable(Comparator<? super T> comparator, Collection<V> sources, Function<V, T> mapper, Runnable onClose)
+    {
+        return new Builder<>(sources, mapper).closeable(comparator, onClose);
+    }
+
+    /**
+     * Create a sorting and deduplicating iterator from a list of sources.
+     * Duplicate values will only be reported once, using an arbitrarily-chosen representative.
+     */
+    public static <T> SortingIterator<T> createDeduplicating(Comparator<? super T> comparator, Collection<T> sources)
+    {
+        return new Deduplicating<>(comparator, sources.isEmpty() ? new Object[1] : sources.toArray());
+    }
+
+    @Override
+    protected Object advanceItem(Object item)
+    {
+        return null;
+    }
+
+    @Override
+    protected Object advanceItemTo(Object item, Object targetKey)
+    {
+        return null;
+    }
+
+    @SuppressWarnings("unchecked")
+    public T peek()
+    {
+        return (T) super.top();
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        return !isEmpty();
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public T next()
+    {
+        Object item = pop();
+        if (item == null)
+            throw new NoSuchElementException();
+        return (T) item;
+    }
+
+    /**
+     * Skip to the first element that is greater than or equal to the given key.
+     */
+    public void skipTo(T targetKey)
+    {
+        advanceTo(targetKey);
+    }
+
+    public static class Closeable<T> extends SortingIterator<T> implements CloseableIterator<T>
+    {
+        final Runnable onClose;
+
+        public <V> Closeable(Comparator<? super T> comparator,
+                             Object[] data,
+                             Runnable onClose)
+        {
+            super(comparator, data);
+            this.onClose = onClose;
+        }
+
+        @Override
+        public void close()
+        {
+            onClose.run();
+        }
+    }
+
+    public static class Deduplicating<T> extends SortingIterator<T>
+    {
+        public Deduplicating(Comparator<? super T> comparator, Object[] data)
+        {
+            super(comparator, data);
+        }
+
+        @Override
+        public T next()
+        {
+            Object item = popAndSkipEqual();
+            if (item == null)
+                throw new NoSuchElementException();
+            return (T) item;
+        }
+    }
+
+    public static class Builder<T>
+    {
+        Object[] data;
+        int count;
+
+        public Builder()
+        {
+            this(16);
+        }
+
+        public Builder(int initialSize)
+        {
+            data = new Object[Math.max(initialSize, 1)]; // at least one element so that we don't need to special-case empty
+            count = 0;
+        }
+
+        public <V> Builder(Collection<V> collection, Function<V, T> mapper)
+        {
+            this(collection.size());
+            for (V item : collection)
+                data[count++] = mapper.apply(item); // this may be null, which the iterator will properly handle
+        }
+
+        public Builder<T> add(T element)
+        {
+            if (element != null)   // avoid growing if we don't need to
+            {
+                if (count == data.length)
+                    data = Arrays.copyOf(data, data.length * 2);
+                data[count++] = element;
+            }
+            return this;
+        }
+
+        public Builder<T> addAll(Collection<? extends T> collection)
+        {
+            if (count + collection.size() > data.length)
+                data = Arrays.copyOf(data, count + collection.size());
+            for (T item : collection)
+                data[count++] = item;
+            return this;
+        }
+
+        public <V> Builder<T> addAll(Collection<V> collection, Function<V, ? extends T> mapper)
+        {
+            if (count + collection.size() > data.length)
+                data = Arrays.copyOf(data, count + collection.size());
+            for (V item : collection)
+                data[count++] = mapper.apply(item); // this may be null, which the iterator will properly handle
+            return this;
+        }
+
+        public int size()
+        {
+            return count; // Note: may include null elements, depending on how data is added
+        }
+
+        /**
+         * Build a sorting iterator from the data added so far.
+         * The returned iterator will report duplicates in arbitrary order.
+         */
+        public SortingIterator<T> build(Comparator<? super T> comparator)
+        {
+            return new SortingIterator<>(comparator, data);    // this will have nulls at the end, which is okay
+        }
+
+        /**
+         * Build a closeable sorting iterator from the data added so far.
+         * The returned iterator will report duplicates in arbitrary order.
+         */
+        public Closeable<T> closeable(Comparator<? super T> comparator, Runnable onClose)
+        {
+            return new Closeable<>(comparator, data, onClose);
+        }
+
+        /**
+         * Build a sorting and deduplicating iterator from the data added so far.
+         * The returned iterator will only report equal items once, using an arbitrarily-chosen representative.
+         */
+        public SortingIterator<T> deduplicating(Comparator<? super T> comparator)
+        {
+            return new Deduplicating<>(comparator, data);
+        }
+
+        // This does not offer build methods that trim the array to count (i.e. Arrays.copyOf(data, count) instead of
+        // data), because it is only meant for short-lived operations where the iterator is not expected to live much
+        // longer than the builder and thus both the builder and iterator will almost always expire in the same GC cycle
+        // and thus the cost of trimming is not offset by any gains.
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/StringSerializer.java b/src/java/org/apache/cassandra/utils/StringSerializer.java
new file mode 100644
index 000000000000..1f8cb0dc11ce
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/StringSerializer.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public class StringSerializer implements IVersionedSerializer<String>
+{
+    public static StringSerializer serializer = new StringSerializer();
+
+    @Override
+    public void serialize(String value, DataOutputPlus out, int version) throws IOException
+    {
+        out.writeUTF(value);
+    }
+
+    @Override
+    public String deserialize(DataInputPlus in, int version) throws IOException
+    {
+        return in.readUTF();
+    }
+
+    @Override
+    public long serializedSize(String value, int version)
+    {
+        return TypeSizes.sizeof(value);
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/SyncUtil.java b/src/java/org/apache/cassandra/utils/SyncUtil.java
index 96985cef6398..84eca25d70d0 100644
--- a/src/java/org/apache/cassandra/utils/SyncUtil.java
+++ b/src/java/org/apache/cassandra/utils/SyncUtil.java
@@ -108,7 +108,7 @@ public static void trySync(int fd)
         if (SKIP_SYNC)
             return;
 
-        NativeLibrary.trySync(fd);
+        INativeLibrary.instance.trySync(fd);
     }
 
     public static void trySyncDir(File dir)
@@ -116,14 +116,14 @@ public static void trySyncDir(File dir)
         if (SKIP_SYNC)
             return;
 
-        int directoryFD = NativeLibrary.tryOpenDirectory(dir.path());
+        int directoryFD = INativeLibrary.instance.tryOpenDirectory(dir);
         try
         {
             trySync(directoryFD);
         }
         finally
         {
-            NativeLibrary.tryCloseFD(directoryFD);
+            INativeLibrary.instance.tryCloseFD(directoryFD);
         }
     }
 }
diff --git a/src/java/org/apache/cassandra/utils/ThreadsFactory.java b/src/java/org/apache/cassandra/utils/ThreadsFactory.java
new file mode 100644
index 000000000000..492a2cea936c
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/ThreadsFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.concurrent.ExecutorService;
+
+import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.apache.cassandra.concurrent.InlinedThreadLocalThread;
+
+public class ThreadsFactory
+{
+    /**
+     * @param name name of the thread for this executor
+     * @return a single threaded executor whose threads have names
+     */
+    public static ExecutorService newSingleThreadedExecutor(String name)
+    {
+        return ExecutorFactory.Global.executorFactory().sequential(name);
+    }
+
+    /**
+     * @param r runnable task for the thread
+     * @param name for the thread
+     * @return a new daemon thread which has the given name and task
+     */
+    public static Thread newDaemonThread(Runnable r, String name)
+    {
+        return newThread(r, name, true);
+    }
+
+    /**
+     * @param r runnable task for the thread
+     * @param name for the thread
+     * @param isDaemon
+     * @return a new thread which has the given name and task
+     */
+    public static Thread newThread(Runnable r, String name, boolean isDaemon)
+    {
+        Thread t = new InlinedThreadLocalThread(r, name);
+        t.setDaemon(isDaemon);
+        return t;
+    }
+
+    public static void addShutdownHook(Runnable r, String name)
+    {
+        // shutdown hook threads should not be daemon
+        Runtime.getRuntime().addShutdownHook(newThread(r, name, false));
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/Throwables.java b/src/java/org/apache/cassandra/utils/Throwables.java
index 3665dfca82d0..5d5ae5a01a77 100644
--- a/src/java/org/apache/cassandra/utils/Throwables.java
+++ b/src/java/org/apache/cassandra/utils/Throwables.java
@@ -48,11 +48,50 @@ public interface DiscreteAction<E extends Exception>
         void perform() throws E;
     }
 
+    /**
+     * Check if the provided throwable is of the provided class, or than any of the throwable in his clause chain is
+     * of the provided class.
+     *
+     * @param t the {@link Throwable} to check.
+     * @param causeClass the class to check if the exception is an instance of, or is caused by.
+     * @return {@code true} if {@code t} is of class {@code causeClass} or any of its cause is.
+     */
+    public static <T extends Throwable> boolean isCausedBy(Throwable t, Class<T> causeClass)
+    {
+        while (t != null)
+        {
+            if (causeClass.isInstance(t))
+                return true;
+            t = t.getCause();
+        }
+        return false;
+    }
+
     public static boolean isCausedBy(Throwable t, Predicate<Throwable> cause)
     {
         return cause.test(t) || (t.getCause() != null && cause.test(t.getCause()));
     }
 
+    /**
+     * Returns an Optional containing the provided throwable if it is of the provided class or the first throwable in the
+     * cause chain that is of the provided class.
+     *
+     * @param t the {@link Throwable} to check.
+     * @param causeClass the class to check if the Throwable is an instance of, or is caused by.
+     * @return Optional containing the provided throwable if it is of the provided class or the first throwable in the
+     * cause chain that is of the provided class, or an empty Optional if no such throwable is found.
+     */
+    public static <T extends Throwable> Optional<T> getCauseOfType(Throwable t, Class<T> causeClass)
+    {
+        while (t != null)
+        {
+            if (causeClass.isInstance(t))
+                return Optional.of(causeClass.cast(t));
+            t = t.getCause();
+        }
+        return Optional.empty();
+    }
+
     public static boolean anyCauseMatches(Throwable t, Predicate<Throwable> cause)
     {
         do
@@ -245,8 +284,13 @@ public static Throwable close(Throwable accumulate, AutoCloseable ... closeables
      */
     public static Throwable close(Throwable accumulate, Iterable<? extends AutoCloseable> closeables)
     {
+        if (closeables == null)
+            return accumulate;
+        
         for (AutoCloseable closeable : closeables)
         {
+            if (closeable != null)
+            {
             try
             {
                 closeable.close();
@@ -256,6 +300,7 @@ public static Throwable close(Throwable accumulate, Iterable<? extends AutoClose
                 accumulate = merge(accumulate, t);
             }
         }
+        }
         return accumulate;
     }
 
@@ -333,4 +378,11 @@ public static void assertAnyCause(Throwable err, Class<? extends Throwable> caus
         if (!anyCauseMatches(err, cause::isInstance))
             throw new AssertionError("The exception is not caused by " + cause.getName(), err);
     }
+
+    @VisibleForTesting
+    public static void assertAnyCause(Throwable err, Class<? extends Throwable>... causeClasses)
+    {
+        if (Arrays.stream(causeClasses).noneMatch(c -> anyCauseMatches(err, c::isInstance)))
+            throw new AssertionError("The exception is not caused by any of " + Arrays.toString(causeClasses), err);
+    }
 }
diff --git a/src/java/org/apache/cassandra/utils/TimeUUID.java b/src/java/org/apache/cassandra/utils/TimeUUID.java
index d993f171af32..8005f7b0e74a 100644
--- a/src/java/org/apache/cassandra/utils/TimeUUID.java
+++ b/src/java/org/apache/cassandra/utils/TimeUUID.java
@@ -203,6 +203,14 @@ public long unixMicros()
         return rawTimestampToUnixMicros(uuidTimestamp);
     }
 
+    /**
+     * The Cassandra internal millis-resolution timestamp of the TimeUUID, as of unix epoch
+     */
+    public long unixMillis()
+    {
+        return (uuidTimestamp / 10_000L) + UUID_EPOCH_UNIX_MILLIS;
+    }
+
     /**
      * The UUID-format timestamp, i.e. 10x micros-resolution, as of UUIDGen.UUID_EPOCH_UNIX_MILLIS
      * The tenths of a microsecond are used to store a flag value.
@@ -241,7 +249,7 @@ public static long msbToRawTimestamp(long msb)
     {
         assert (UUID_VERSION_BITS_IN_MSB & msb) == TIMESTAMP_UUID_VERSION_IN_MSB;
         msb &= ~TIMESTAMP_UUID_VERSION_IN_MSB;
-        return   (msb &     0xFFFFL) << 48
+        return (msb & 0xFFFFL) << 48
                | (msb & 0xFFFF0000L) << 16
                | (msb >>> 32);
     }
@@ -263,7 +271,12 @@ public int hashCode()
     @Override
     public boolean equals(Object that)
     {
-        return    (that instanceof UUID && equals((UUID) that))
+        if (this == that)
+            return true;
+        if (that == null)
+            return false;
+
+        return (that instanceof UUID && equals((UUID) that))
                || (that instanceof TimeUUID && equals((TimeUUID) that));
     }
 
@@ -406,6 +419,24 @@ public static byte[] nextTimeUUIDAsBytes()
             return toBytes(rawTimestampToMsb(unixMicrosToRawTimestamp(nextUnixMicros())), clockSeqAndNode);
         }
 
+        public static int sequence(TimeUUID timeUUID)
+        {
+            long lsb = timeUUID.asUUID().getLeastSignificantBits();
+            return (int) ((lsb >> 48) & 0x0000000000003FFFL);
+        }
+
+        /**
+         * Returns a new TimeUUID with the same timestamp as this one, but with the provided sequence value.
+         */
+        public static TimeUUID withSequence(TimeUUID timeUUID, long sequence)
+        {
+            long sequenceBits = 0x0000000000003FFFL;
+            long sequenceMask = ~(sequenceBits << 48);
+            final long bits = (sequence & sequenceBits) << 48;
+            UUID uuid = timeUUID.asUUID();
+            return TimeUUID.fromBytes(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits() & sequenceMask | bits);
+        }
+
         // needs to return two different values for the same when.
         // we can generate at most 10k UUIDs per ms.
         private static long nextUnixMicros()
@@ -494,7 +525,7 @@ private static byte[] hash(Collection<InetAddressAndPort> data)
             }
 
             // Identify the process on the load: we use both the PID and class loader hash.
-            long pid = NativeLibrary.getProcessID();
+            long pid = INativeLibrary.instance.getProcessID();
             if (pid < 0)
                 pid = new Random(currentTimeMillis()).nextLong();
             updateWithLong(hasher, pid);
diff --git a/src/java/org/apache/cassandra/utils/TopKSelector.java b/src/java/org/apache/cassandra/utils/TopKSelector.java
new file mode 100644
index 000000000000..7b32851c991e
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/TopKSelector.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * This class selects the smallest k items from a stream.
+ * <p>
+ * This is implemented as a binary heap with reversed comparator which keeps track of k items and keeps the largest of
+ * them on top of the heap. When a new item arrives, it is checked against the top: if it is larger or equal, it can
+ * be ignored as we already have k better items; if not, it replaces the top item and is pushed down to restore the
+ * properties of the heap.
+ * <p>
+ * This process has a time complexity of O(n log k) for n > k and uses O(k) space. Duplicates are not removed and are
+ * returned in arbitrary order.
+ * <p>
+ * If the number of items required is not known in advance, {@link SortingIterator} can be used instead to get an
+ * arbitrary number of ordered items at the expense of keeping track of all of them (using O(n + k log n) time and O(n)
+ * space).
+ */
+public class TopKSelector<T> extends BinaryHeap
+{
+    private final Comparator<? super T> comparator;
+    private int size;
+
+    public TopKSelector(Comparator<? super T> comparator, int limit)
+    {
+        super(new Object[limit]);
+        this.comparator = comparator;
+        size = 0;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected boolean greaterThan(Object a, Object b)
+    {
+        // Top-k uses an inverted comparator, so that the largest item, the one we should compare with and replace
+        // if something smaller is added, sits at the top. This is also the comparator suitable for doing the final
+        // heapsort steps required to arrange the end result in sort order.
+        return comparator.compare((T) a, (T) b) < 0;
+    }
+
+    public void add(T newItem)
+    {
+        if (newItem == null)
+            return;
+
+        if (size < heap.length)
+        {
+            heap[size] = newItem;
+            if (++size == heap.length)
+                heapify();
+        }
+        else
+        {
+            if (greaterThan(newItem, top()))
+                replaceTop(newItem);
+        }
+    }
+
+    public void addAll(Iterable<? extends T> items)
+    {
+        for (T item : items)
+            add(item);
+    }
+
+    @Override
+    public int size()
+    {
+        return size;
+    }
+
+    private void maybeHeapify()
+    {
+        if (size < heap.length)
+            heapify();
+    }
+
+    /**
+     * Get a copy of the top K elements.
+     * After this call the collector can be reused.
+     */
+    public List<T> get()
+    {
+        return new ArrayList<>(getShared());
+    }
+
+    /**
+     * Get a copy of the top K elements, applying the given transformation.
+     * After this call the collector can be reused.
+     */
+    public <R> List<R> getTransformed(Function<T, R> transformer)
+    {
+        return getTransformedSliced(transformer, 0);
+    }
+
+    /**
+     * Get a copy of the lowest size-startIndex elements.
+     * The top startIndex elements will remain in the selector.
+     */
+    public List<T> getSliced(int startIndex)
+    {
+        return getTransformedSliced(Function.identity(), startIndex);
+    }
+
+    /**
+     * Get a copy of the lowest size-startIndex elements, applying the given transformation.
+     * The top startIndex elements will remain in the selector.
+     */
+    public <R> List<R> getTransformedSliced(Function<T, R> transformer, int startIndex)
+    {
+        return new ArrayList<>(getTransformedSlicedShared(transformer, startIndex));
+    }
+
+    /**
+     * Get a shared list of the top K elements.
+     * If the selector is not used further, this is a quicker alternative to get().
+     */
+    public List<T> getShared()
+    {
+        maybeHeapify();
+        heapSort();
+        int completedSize = size;
+        size = 0;
+        return getUnsortedShared(completedSize);
+    }
+
+    /**
+     * Get a shared list of the top K elements in unsorted order.
+     * This avoids the final sort phase (and heapification if there are fewer than K elements).
+     */
+    public List<T> getUnsortedShared()
+    {
+        return getUnsortedShared(size);
+    }
+
+    private List<T> getUnsortedShared(int size)
+    {
+        return new AbstractList<T>()
+        {
+            @Override
+            public T get(int i)
+            {
+                return (T) heap[i];
+            }
+
+            @Override
+            public int size()
+            {
+                return size;
+            }
+        };
+    }
+
+    /**
+     * Get a shared list of the lowest size-startIndex elements, applying the given transformation.
+     * If the selector is not used further, this is a quicker alternative to getTransformedSliced().
+     */
+    public <R> List<R> getTransformedSlicedShared(Function<T, R> transformer, int startIndex)
+    {
+        int selectedSize = size() - startIndex;
+        if (selectedSize <= 0)
+            return List.of();
+        maybeHeapify();
+
+        heapSortFrom(startIndex);
+        size = startIndex; // the rest of the top items remain heapified and can be extracted later
+        return new AbstractList<R>()
+        {
+            @Override
+            public R get(int i)
+            {
+                return transformer.apply((T) heap[i + startIndex]);
+            }
+
+            @Override
+            public int size()
+            {
+                return selectedSize;
+            }
+        };
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/UniqueComparator.java b/src/java/org/apache/cassandra/utils/UniqueComparator.java
new file mode 100644
index 000000000000..9c6513f9647b
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/UniqueComparator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Comparator;
+
+/**
+ * Converts any comparator to a comparator that never treats distinct objects as equal,
+ * even if the original comparator considers them equal.
+ * For all other items, the order of the original comparator is preserved.
+ * Allows to store duplicate items in sorted sets.
+ */
+public class UniqueComparator<T> implements Comparator<T>
+{
+    private final Comparator<T> comparator;
+
+    public UniqueComparator(Comparator<T> comparator)
+    {
+        this.comparator = comparator;
+    }
+
+    @Override
+    public int compare(T o1, T o2)
+    {
+        int result = comparator.compare(o1, o2);
+        if (result == 0 && o1 != o2)
+        {
+            // If the wrapped comparator considers the items equal,
+            // but they are not actually the same object, distinguish them
+            return System.identityHashCode(o1) - System.identityHashCode(o2);
+        }
+        return result;
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/btree/BTree.java b/src/java/org/apache/cassandra/utils/btree/BTree.java
index 2ac80df48ea3..26c1893f88fd 100644
--- a/src/java/org/apache/cassandra/utils/btree/BTree.java
+++ b/src/java/org/apache/cassandra/utils/btree/BTree.java
@@ -349,8 +349,8 @@ public static <Compare, Existing extends Compare, Insert extends Compare> Object
 
         if (isEmpty(toUpdate))
         {
-            if (isSimple(updateF))
-                return insert; // if update is empty and updateF is trivial, return our new input
+//            if (isSimple(updateF))
+//                return insert; // if update is empty and updateF is trivial, return our new input
 
             // if update is empty and updateF is non-trivial, perform a simple fast transformation of the input tree
             insert = BTree.transform(insert, updateF::insert);
@@ -4217,4 +4217,38 @@ <Update> int copyKeysSmallerThan(Compare bound, Comparator<? super Compare> comp
             }
         }
     }
+
+    public interface ReduceFunction<ACC, I> extends BiFunction<ACC, I, ACC>
+    {
+        default public boolean stop(ACC res)
+        {
+            return false;
+        }
+    }
+
+    /**
+     * Walk the btree forwards and apply a reduce function. Return the reduced value.
+     */
+    public static <R, V> R reduce(Object[] btree, R seed, ReduceFunction<R, V> function)
+    {
+        boolean isLeaf = isLeaf(btree);
+        int childOffset = isLeaf ? Integer.MAX_VALUE : getChildStart(btree);
+        int limit = isLeaf ? getLeafKeyEnd(btree) : btree.length - 1;
+        for (int i = 0 ; i < limit ; i++)
+        {
+            // we want to visit in iteration order, so we visit our key nodes inbetween our children
+            int idx = isLeaf ? i : (i / 2) + (i % 2 == 0 ? childOffset : 0);
+            Object current = btree[idx];
+            if (idx < childOffset)
+                seed = function.apply(seed, (V)current);
+            else
+                seed = reduce((Object[])current, seed, function);
+
+            if (function.stop(seed))
+                break;
+        }
+
+        return seed;
+    }
+
 }
diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeSet.java b/src/java/org/apache/cassandra/utils/btree/BTreeSet.java
index 1bd324a7e178..08b0152113a6 100644
--- a/src/java/org/apache/cassandra/utils/btree/BTreeSet.java
+++ b/src/java/org/apache/cassandra/utils/btree/BTreeSet.java
@@ -323,6 +323,48 @@ public ListIterator<V> listIterator(int index)
         throw new UnsupportedOperationException();
     }
 
+    // @Override needed in JDK 21+.
+    public BTreeSet<V> reversed()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    // @Override needed in JDK 21+.
+    public V removeLast()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    // @Override needed in JDK 21+.
+    public V removeFirst()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    // @Override needed in JDK 21+.
+    public V getLast()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    // @Override needed in JDK 21+.
+    public V getFirst()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    // @Override needed in JDK 21+.
+    public void addLast(V v)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    // @Override needed in JDK 21+.
+    public void addFirst(V v)
+    {
+        throw new UnsupportedOperationException();
+    }
+
     public static class BTreeRange<V> extends BTreeSet<V>
     {
         // both inclusive
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java
index c79dffcd79d7..7b18805373cb 100644
--- a/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java
@@ -20,6 +20,9 @@
 
 import java.nio.ByteBuffer;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
 /**
  * Interface indicating a value can be represented/identified by a comparable {@link ByteSource}.
  *
@@ -36,8 +39,9 @@ public interface ByteComparable
 
     enum Version
     {
-        LEGACY, // Encoding used in legacy sstable format; forward (value to byte-comparable) translation only
-        OSS50,  // CASSANDRA 5.0 encoding
+        LEGACY,
+        OSS41,  // CASSANDRA 4.1 encoding, used in trie-based indices
+        OSS50,  // CASSANDRA 5.0 encoding, used by the trie memtable
     }
 
     ByteComparable EMPTY = (Version version) -> ByteSource.EMPTY;
@@ -56,10 +60,25 @@ default String byteComparableAsString(Version version)
         return builder.toString();
     }
 
+    /**
+     * Returns the full byte-comparable representation of the value as a byte array.
+     */
+    default byte[] asByteComparableArray(Version version)
+    {
+        return ByteSourceInverse.readBytes(asComparableBytes(version));
+    }
+
+    default Preencoded preencode(Version version)
+    {
+        return preencoded(version, asByteComparableArray(version));
+    }
+
     // Simple factories used for testing
 
+    @VisibleForTesting
     static ByteComparable of(String s)
     {
+        // Note: This is not prefix-free
         return v -> ByteSource.of(s, v);
     }
 
@@ -73,19 +92,54 @@ static ByteComparable of(int value)
         return v -> ByteSource.of(value);
     }
 
-    static ByteComparable fixedLength(ByteBuffer bytes)
+    interface Preencoded extends ByteComparable
     {
-        return v -> ByteSource.fixedLength(bytes);
+        Version encodingVersion();
+
+        ByteSource.Duplicatable getPreencodedBytes();
+
+        @Override
+        default ByteSource.Duplicatable asComparableBytes(Version version)
+        {
+            Preconditions.checkState(version == encodingVersion(),
+                                     "Preencoded byte-source at version %s queried at version %s",
+                                     encodingVersion(),
+                                     version);
+            return getPreencodedBytes();
+        }
+
+        @Override
+        default byte[] asByteComparableArray(Version version)
+        {
+            return asComparableBytes(version).remainingBytesToArray();
+        }
     }
 
-    static ByteComparable fixedLength(byte[] bytes)
+    /**
+     * A ByteComparable value that is already encoded for a specific version. Requesting the source with a different
+     * version will result in an exception.
+     */
+    static Preencoded preencoded(Version version, ByteBuffer bytes)
     {
-        return v -> ByteSource.fixedLength(bytes);
+        return new PreencodedByteComparable.Buffer(version, bytes);
     }
 
-    static ByteComparable fixedLength(byte[] bytes, int offset, int len)
+    /**
+     * A ByteComparable value that is already encoded for a specific version. Requesting the source with a different
+     * version will result in an exception.
+     */
+    static Preencoded preencoded(Version version, byte[] bytes)
     {
-        return v -> ByteSource.fixedLength(bytes, offset, len);
+        return new PreencodedByteComparable.Array(version, bytes);
+    }
+
+    /**
+     * A ByteComparable value that is already encoded for a specific version. Requesting the source with a different
+     * version will result in an exception.
+     */
+    static Preencoded preencoded(Version version, byte[] bytes, int offset, int len)
+    {
+        return new PreencodedByteComparable.Array(version, bytes, offset, len);
     }
 
     /**
@@ -127,29 +181,29 @@ static int length(ByteComparable src, Version version)
     }
 
     /**
-     * Compare two byte-comparable values by their byte-comparable representation. Used for tests.
+     * Compare two byte-comparable values by their byte-comparable representation.
      *
      * @return the result of the lexicographic unsigned byte comparison of the byte-comparable representations of the
      *         two arguments
      */
     static int compare(ByteComparable bytes1, ByteComparable bytes2, Version version)
     {
-        ByteSource s1 = bytes1.asComparableBytes(version);
-        ByteSource s2 = bytes2.asComparableBytes(version);
-
-        if (s1 == null || s2 == null)
-            return Boolean.compare(s1 != null, s2 != null);
+        return ByteSource.compare(bytes1.asComparableBytes(version), bytes2.asComparableBytes(version));
+    }
 
-        while (true)
-        {
-            int b1 = s1.next();
-            int b2 = s2.next();
-            int cmp = Integer.compare(b1, b2);
-            if (cmp != 0)
-                return cmp;
-            if (b1 == ByteSource.END_OF_STREAM)
-                return 0;
-        }
+    /**
+     * Compare two preencoded byte-comparable values, using their encoding versions.
+     *
+     * @return the result of the lexicographic unsigned byte comparison of the byte-comparable representations of the
+     *         two arguments
+     */
+    static int compare(Preencoded a, Preencoded b)
+    {
+        Preconditions.checkArgument(a.encodingVersion() == b.encodingVersion(),
+                                    "Cannot compare preencoded byte-comparables of different versions %s vs %s",
+                                    a.encodingVersion(),
+                                    b.encodingVersion());
+        return ByteSource.compare(a.getPreencodedBytes(), b.getPreencodedBytes());
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.md b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.md
index 8012e27b03e5..f26b256669ec 100644
--- a/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.md
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.md
@@ -334,7 +334,7 @@ This is the trivial case, as we can simply use the input bytes in big-endian ord
 and fixed length values are trivially prefix free, i.e. (1) and (2) are satisfied, and thus (3) and (4) follow from the
 observation above.
 
-## Fixed-length signed integers (byte, short, int, legacy bigint)
+## Fixed-length signed integers (byte, short, int, bigint for versions <= OSS41)
 
 As above, but we need to invert the sign bit of the number to put negative numbers before positives. This maps
 `MIN_VALUE` to `0x00`..., `-1` to `0x7F…`, `0` to `0x80…`, and `MAX_VALUE` to `0xFF…`; comparing the resulting number
@@ -457,15 +457,15 @@ end. The values we chose for the separator and terminator are `0x40` and `0x38`,
 Examples:
 
 
-| Types and values         | bytes                  | encodes as                     |
-| ------------------------ | ---------------------- | ------------------------------ |
+| Types and values         | bytes                  | encodes as                 |
+| ------------------------ | ---------------------- |----------------------------|
 | (short 1, float 1.0)     | 00 01, 3F 80 00 00     | 40·80 01·40·BF 80 00 00·38 |
-| (short -1, null)         | FF FF, —              | 40·7F FF·3E·38              |
+| (short -1, null)         | FF FF, —              | 40·7F FF·3E·38             |
 | ≥ (short 0, float -Inf) | 00 00, FF 80 00 00, >= | 40·80 00·40·00 7F FF FF·20 |
-| < (short MIN)            | 80 00, <=              | 40·00 00·20                  |
-| \> (null)                |                        | 3E·60                         |
-| BOTTOM                   |                        | 20                             |
-| TOP                      |                        | 60                             |
+| < (short MIN)            | 80 00, <=              | 40·00 00·20                |
+| \> (null)                |                        | 3E·60                      |
+| BOTTOM                   |                        | 20                         |
+| TOP                      |                        | 60                         |
 
 (The middle dot · doesn't exist in the encoding, it’s just a visualisation of the boundaries in the examples.)
 
@@ -501,21 +501,21 @@ The method we chose for this is the following:
 Examples:
 
 
-| bytes/sequence     | encodes as               |
-| ------------------ | ------------------------ |
-| 22 00              | 22 00 FE                 |
-| 22 00 00 33        | 22 00 FE FF 33 00        |
-| 22 00 11           | 22 00 FF 11 00           |
+| bytes/sequence     | encodes as           |
+| ------------------ |----------------------|
+| 22 00              | 22 00 FE             |
+| 22 00 00 33        | 22 00 FE FF 33 00    |
+| 22 00 11           | 22 00 FF 11 00       |
 | (blob 22, short 0) | 40·22 00·40·80 00·40 |
-| ≥ (blob 22 00)    | 40·22 00 FE·20         |
-| ≤ (blob 22 00 00) | 40·22 00 FE FE·60      |
+| ≥ (blob 22 00)    | 40·22 00 FE·20       |
+| ≤ (blob 22 00 00) | 40·22 00 FE FE·60    |
 
 Within the encoding, a `00` byte can only be followed by a `FE` or `FF` byte, and hence if an encoding is a prefix of
 another, the latter has to have a `FE` or `FF` as the next byte, which ensures both (4) (adding `10`-`EF` to the former
 makes it no longer a prefix of the latter) and (3) (adding `10`-`EF` to the former makes it smaller than the latter; in
 this case the original value of the former is a prefix of the original value of the latter).
 
-## Variable-length integers (varint, RandomPartitioner token), legacy encoding
+## Variable-length integers (varint, RandomPartitioner token), OSS41 and earlier
 
 If integers of unbounded length are guaranteed to start with a non-zero digit, to compare them we can first use a signed
 length, as numbers with longer representations have higher magnitudes. Only if the lengths match we need to compare the
@@ -544,8 +544,8 @@ as well.
 Examples:
 
 
-|   value | bytes            | encodes as              |
-| ------: | ---------------- | ----------------------- |
+|   value | bytes            | encodes as             |
+| ------: | ---------------- |------------------------|
 |       0 | 00               | 80·00                  |
 |       1 | 01               | 80·01                  |
 |      -1 | FF               | 7F·FF                  |
@@ -585,18 +585,18 @@ inverted length bytes), and bigger when positive.
 Examples:
 
 
-|   value | bytes                   | encodes as                      |
-| ------: | ----------------------- | ------------------------------- |
-|       0 | 00                      | 80                              |
-|       1 | 01                      | 81                              |
-|      -1 | FF                      | 7F                              |
-|     255 | 00 FF                   | C0 FF                           |
-|    -256 | FF 00                   | 3F 00                           |
-|     256 | 01 00                   | C1 00                           |
-|    2^16 | 01 00 00                | E1 00 00                        |
-|   -2^32 | FF 00 00 00 00          | 07 00 00 00 00                  |
-|  2^56-1 | 00 FF FF FF FF FF FF FF | FE FF FF FF FF FF FF FF         |
-|   -2^56 | FF 00 00 00 00 00 00 00 | 01 00 00 00 00 00 00 00         |
+|   value | bytes                   | encodes as                    |
+| ------: | ----------------------- |-------------------------------|
+|       0 | 00                      | 80                            |
+|       1 | 01                      | 81                            |
+|      -1 | FF                      | 7F                            |
+|     255 | 00 FF                   | C0 FF                         |
+|    -256 | FF 00                   | 3F 00                         |
+|     256 | 01 00                   | C1 00                         |
+|    2^16 | 01 00 00                | E1 00 00                      |
+|   -2^32 | FF 00 00 00 00          | 07 00 00 00 00                |
+|  2^56-1 | 00 FF FF FF FF FF FF FF | FE FF FF FF FF FF FF FF       |
+|   -2^56 | FF 00 00 00 00 00 00 00 | 01 00 00 00 00 00 00 00       |
 |    2^56 | 01 00 00 00 00 00 00 00 | FF·00·01 00 00 00 00 00 00 00 |
 | -2^56-1 | FE FF FF FF FF FF FF FF | 00·FF·FE FF FF FF FF FF FF FF |
 |  2^1024 | 01 00(128 times)        | FF·7A·01 00(128 times)        |
@@ -671,13 +671,13 @@ byte:
 Examples:
 
 
-|      value |  mexp | mantissa | mantissa in bytes | encodes as           |
-| ---------: | ----: | -------- | ----------------- | -------------------- |
+|      value |  mexp | mantissa | mantissa in bytes | encodes as        |
+| ---------: | ----: | -------- | ----------------- |-------------------|
 |        1.1 |     1 | 0.0110   | .  01 10          | C1·01·81 8A·00    |
 |          1 |     1 | 0.01     | .  01             | C1·01·81·00       |
-|       0.01 |     0 | 0.01     | .  01             | C0·81·00           |
-|          0 |       |          |                   | 80                   |
-|      -0.01 |     0 | -0.01    | . -01             | 40·81·00           |
+|       0.01 |     0 | 0.01     | .  01             | C0·81·00          |
+|          0 |       |          |                   | 80                |
+|      -0.01 |     0 | -0.01    | . -01             | 40·81·00          |
 |         -1 |    -1 | -0.01    | . -01             | 3F·FF·7F·00       |
 |       -1.1 |    -1 | -0.0110  | . -02 90          | 3F·FF·7E DA·00    |
 |      -98.9 |    -1 | -0.9890  | . -99 10          | 3F·FF·1D 8A·00    |
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java
index 83bb828f3096..9b73c6d5454b 100644
--- a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java
@@ -19,8 +19,12 @@
 
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
 import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FastByteOperations;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
 import org.apache.cassandra.utils.memory.MemoryUtil;
 
@@ -36,6 +40,26 @@ public interface ByteSource
     /** Consume the next byte, unsigned. Must be between 0 and 255, or END_OF_STREAM if there are no more bytes. */
     int next();
 
+    /**
+     * Consume the next bytes of the source and transfer them to the given array.
+     *
+     * @return The number of bytes transferred. If equal to the size of the destination, the source may have further
+     *         bytes to consume. Otherwise, the source has been fully consumed, and it would be an error to call this
+     *         method (or next()) again.
+     */
+    default int nextBytes(byte[] dest)
+    {
+        int i;
+        for (i = 0; i < dest.length; ++i)
+        {
+            int next = next();
+            if (next == END_OF_STREAM)
+                return i;
+            dest[i] = (byte) next;
+        }
+        return i;
+    }
+
     /** Value returned if at the end of the stream. */
     int END_OF_STREAM = -1;
 
@@ -57,11 +81,14 @@ public interface ByteSource
 
     // Next component marker.
     int NEXT_COMPONENT = 0x40;
-    // Marker used to present null values represented by empty buffers (e.g. by Int32Type)
-    int NEXT_COMPONENT_EMPTY = 0x3F;
-    int NEXT_COMPONENT_EMPTY_REVERSED = 0x41;
-    // Marker for null components in tuples, maps, sets and clustering keys.
-    int NEXT_COMPONENT_NULL = 0x3E;
+    // Marker used to present null values represented by empty buffers (e.g. by Int32Type), as well as nulls in
+    // collections.
+    int NEXT_COMPONENT_NULL = 0x3F;
+    int NEXT_COMPONENT_NULL_REVERSED = 0x41;
+    // Marker for null components in clustering keys. Null clusterings are normally encoded by empty buffers (which end
+    // up using NEXT_COMPONENT_EMPTY above), but in some cases (secondary indexes and compact storage) we may get null
+    // pointers that compare differently.
+    int NEXT_CLUSTERING_NULL = 0x3E;
 
     // Section for next component markers which is not allowed for use
     int MIN_NEXT_COMPONENT = 0x3C;
@@ -274,37 +301,27 @@ public int next()
         };
     }
 
-    /**
-     * Wrap a ByteSource in a length-fixing facade.
-     *
-     * If the length of {@code src} is less than {@code cutoff}, then pad it on the right with {@code padding} until
-     * the overall length equals {@code cutoff}.  If the length of {@code src} is greater than {@code cutoff}, then
-     * truncate {@code src} to that size.  Effectively a noop if {@code src} happens to have length {@code cutoff}.
-     *
-     * @param src the input source to wrap
-     * @param cutoff the size of the source returned
-     * @param padding a padding byte (an int subject to a 0xFF mask)
-     */
-    public static ByteSource cutOrRightPad(ByteSource src, int cutoff, int padding)
+    public static ByteSource append(ByteSource src, int lastByte)
     {
         return new ByteSource()
         {
-            int pos = 0;
+            boolean done = false;
 
             @Override
             public int next()
             {
-                if (pos++ >= cutoff)
-                {
+                if (done)
                     return END_OF_STREAM;
-                }
-                int next = src.next();
-                return next == END_OF_STREAM ? padding : next;
+                int n = src.next();
+                if (n != END_OF_STREAM)
+                    return n;
+
+                done = true;
+                return lastByte;
             }
         };
     }
 
-
     /**
      * Variable-length encoding. Escapes 0s as ESCAPE + zero or more ESCAPED_0_CONT + ESCAPED_0_DONE.
      * If the source ends in 0, we use ESCAPED_0_CONT to make sure that the encoding remains smaller than that source
@@ -666,7 +683,8 @@ public int next()
     }
 
     /**
-     * Combination of multiple byte sources. Adds {@link NEXT_COMPONENT} before sources, or {@link NEXT_COMPONENT_NULL} if next is null.
+     * Combination of multiple byte sources. Adds {@link #NEXT_COMPONENT} before sources, or {@link #NEXT_COMPONENT_NULL}
+     * if next is null.
      */
     static class Multi implements ByteSource
     {
@@ -737,29 +755,78 @@ public int next()
         }
     }
 
+    /**
+     * A byte source representing a value of fixed length than can be compared using unsigned byte comparison. Such
+     * value can be used unchanged because their fixed length ensures that the encoding is prefix-free.
+     * This method also permits the value to be empty and encodes this as null.
+     */
     static <V> ByteSource optionalFixedLength(ValueAccessor<V> accessor, V data)
     {
-        return !accessor.isEmpty(data) ? fixedLength(accessor, data) : null;
+        return !accessor.isEmpty(data) ? preencoded(accessor, data) : null;
     }
 
     /**
-     * A byte source of the given bytes without any encoding.
-     * The resulting source is only guaranteed to give correct comparison results and be prefix-free if the
-     * underlying type has a fixed length.
-     * In tests, this method is also used to generate non-escaped test cases.
+     * A byte source of the given bytes without any encoding. This has several uses:
+     * - to store a value that is already encoded for a given version (see ByteComparable.preencoded)
+     * - to store fixed-length values that can be used directly because their length ensures that the encoding is
+     *   prefix-free (see optionalFixedLength)
+     * - to implement ByteSource duplication
+     * - to store a value that has a custom encoding not handled by ByteSource and AbstractType implementations
+     *   (e.g. some SAI indexes)
+     * - to generate non-escaped test cases
      */
-    public static <V> ByteSource fixedLength(ValueAccessor<V> accessor, V data)
+    public static <V> ByteSource preencoded(ValueAccessor<V> accessor, V data)
     {
-        return new ByteSource()
+        return new PreencodedBytesByAccessor<>(accessor, data, 0, accessor.size(data));
+    }
+
+    class PreencodedBytesByAccessor<V> implements Duplicatable
+    {
+        int pos;
+        final int end;
+        final V data;
+        final ValueAccessor<V> accessor;
+
+        PreencodedBytesByAccessor(ValueAccessor<V> accessor, V data, int start, int end)
         {
-            int pos = -1;
+            this.data = data;
+            this.accessor = accessor;
+            this.pos = start;
+            this.end = end;
+        }
 
-            @Override
-            public int next()
-            {
-                return ++pos < accessor.size(data) ? accessor.getByte(data, pos) & 0xFF : END_OF_STREAM;
-            }
-        };
+        @Override
+        public int next()
+        {
+            return pos < end ? accessor.getByte(data, pos++) & 0xFF : END_OF_STREAM;
+        }
+
+        @Override
+        public int peek()
+        {
+            return pos < end ? accessor.getByte(data, pos) & 0xFF : END_OF_STREAM;
+        }
+
+        @Override
+        public int nextBytes(byte[] array)
+        {
+            int len = Math.min(end - pos, array.length);
+            accessor.copyTo(data, pos, array, ByteArrayAccessor.instance, 0, len);
+            pos += len;
+            return len;
+        }
+
+        @Override
+        public byte[] remainingBytesToArray()
+        {
+            return accessor.toArray(data, pos, end - pos);
+        }
+
+        @Override
+        public Duplicatable duplicate()
+        {
+            return new PreencodedBytesByAccessor(accessor, data, pos, end);
+        }
     }
 
     /**
@@ -768,18 +835,56 @@ public int next()
      * underlying type has a fixed length.
      * In tests, this method is also used to generate non-escaped test cases.
      */
-    public static ByteSource fixedLength(ByteBuffer b)
+    public static Duplicatable preencoded(ByteBuffer b)
     {
-        return new ByteSource()
+        return new PreencodedByteBuffer(b, b.position(), b.limit());
+    }
+
+    class PreencodedByteBuffer implements Duplicatable
+    {
+        int pos;
+        final int end;
+        final ByteBuffer b;
+
+        PreencodedByteBuffer(ByteBuffer b, int start, int end)
         {
-            int pos = b.position() - 1;
+            this.b = b;
+            this.pos = start;
+            this.end = end;
+        }
 
-            @Override
-            public int next()
-            {
-                return ++pos < b.limit() ? b.get(pos) & 0xFF : END_OF_STREAM;
-            }
-        };
+        @Override
+        public int next()
+        {
+            return pos < end ? b.get(pos++) & 0xFF : END_OF_STREAM;
+        }
+
+        @Override
+        public int peek()
+        {
+            return pos < end ? b.get(pos) & 0xFF : END_OF_STREAM;
+        }
+
+        @Override
+        public int nextBytes(byte[] array)
+        {
+            int len = Math.min(end - pos, array.length);
+            FastByteOperations.copy(b, pos, array, 0, len);
+            pos += len;
+            return len;
+        }
+
+        @Override
+        public byte[] remainingBytesToArray()
+        {
+            return ByteBufferUtil.getArray(b, pos, end - pos);
+        }
+
+        @Override
+        public Duplicatable duplicate()
+        {
+            return new PreencodedByteBuffer(b, pos, end);
+        }
     }
 
     /**
@@ -788,36 +893,79 @@ public int next()
      * underlying type has a fixed length.
      * In tests, this method is also used to generate non-escaped test cases.
      */
-    public static ByteSource fixedLength(byte[] b)
+    public static Duplicatable preencoded(byte[] b)
     {
-        return fixedLength(b, 0, b.length);
+        return preencoded(b, 0, b.length);
     }
 
-    public static ByteSource fixedLength(byte[] b, int offset, int length)
+    public static Duplicatable preencoded(byte[] b, int offset, int length)
     {
         checkArgument(offset >= 0 && offset <= b.length);
         checkArgument(length >= 0 && offset + length <= b.length);
 
-        return new ByteSource()
+        return new PreencodedBytes(b, offset, offset + length);
+    }
+
+    class PreencodedBytes implements Duplicatable
+    {
+        int pos;
+        final int end;
+        final byte[] b;
+
+        PreencodedBytes(byte[] b, int start, int end)
         {
-            int pos = offset - 1;
+            this.b = b;
+            this.pos = start;
+            this.end = end;
+        }
 
-            @Override
-            public int next()
-            {
-                return ++pos < offset + length ? b[pos] & 0xFF : END_OF_STREAM;
-            }
-        };
+        @Override
+        public int next()
+        {
+            return pos < end ? b[pos++] & 0xFF : END_OF_STREAM;
+        }
+
+        @Override
+        public int peek()
+        {
+            return pos < end ? b[pos] & 0xFF : END_OF_STREAM;
+        }
+
+        @Override
+        public int nextBytes(byte[] array)
+        {
+            int len = Math.min(end - pos, array.length);
+            FastByteOperations.copy(b, pos, array, 0, len);
+            pos += len;
+            return len;
+        }
+
+        @Override
+        public byte[] remainingBytesToArray()
+        {
+            return Arrays.copyOfRange(b, pos, end);
+        }
+
+        @Override
+        public Duplicatable duplicate()
+        {
+            return new PreencodedBytes(b, pos, end);
+        }
     }
 
-    public class Peekable implements ByteSource
+    interface Peekable extends ByteSource
+    {
+        int peek();
+    }
+
+    public class PeekableImpl implements Peekable
     {
         private static final int NONE = Integer.MIN_VALUE;
 
         private final ByteSource wrapped;
         private int peeked = NONE;
 
-        public Peekable(ByteSource wrapped)
+        public PeekableImpl(ByteSource wrapped)
         {
             this.wrapped = wrapped;
         }
@@ -852,6 +1000,41 @@ public static Peekable peekable(ByteSource p)
             return null;
         return (p instanceof Peekable)
                ? (Peekable) p
-               : new Peekable(p);
+               : new PeekableImpl(p);
+    }
+
+    interface ConvertableToArray extends Peekable
+    {
+        byte[] remainingBytesToArray();
+    }
+
+    interface Duplicatable extends ConvertableToArray
+    {
+        Duplicatable duplicate();
+    }
+
+    public static Duplicatable duplicatable(ByteSource src)
+    {
+        if (src instanceof Duplicatable)
+            return (Duplicatable) src;
+
+        return preencoded(ByteSourceInverse.readBytes(src));
+    }
+
+    static int compare(ByteSource s1, ByteSource s2)
+    {
+        if (s1 == null || s2 == null)
+            return Boolean.compare(s1 != null, s2 != null);
+
+        while (true)
+        {
+            int b1 = s1.next();
+            int b2 = s2.next();
+            int cmp = Integer.compare(b1, b2);
+            if (cmp != 0)
+                return cmp;
+            if (b1 == END_OF_STREAM)
+                return 0;
+        }
     }
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSourceInverse.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSourceInverse.java
index 4bf9d8c36522..a18bf2d2bd2e 100644
--- a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSourceInverse.java
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSourceInverse.java
@@ -18,11 +18,14 @@
 package org.apache.cassandra.utils.bytecomparable;
 
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 import com.google.common.base.Preconditions;
 
 import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.utils.ByteArrayUtil;
 
 /**
  * Contains inverse transformation utilities for {@link ByteSource}s.
@@ -31,7 +34,6 @@
  */
 public final class ByteSourceInverse
 {
-    private static final int INITIAL_BUFFER_CAPACITY = 32;
     private static final int BYTE_ALL_BITS = 0xFF;
     private static final int BYTE_NO_BITS = 0x00;
     private static final int BYTE_SIGN_BIT = 1 << 7;
@@ -353,53 +355,43 @@ public int next()
      * Reads the bytes of the given source into a byte array. Doesn't do any transformation on the bytes, just reads
      * them until it reads an {@link ByteSource#END_OF_STREAM} byte, after which it returns an array of all the read
      * bytes, <strong>excluding the {@link ByteSource#END_OF_STREAM}</strong>.
-     * <p>
-     * This method sizes a tentative internal buffer array at {@code initialBufferCapacity}.  However, if
-     * {@code byteSource} exceeds this size, the buffer array is recreated with doubled capacity as many times as
-     * necessary.  If, after {@code byteSource} is fully exhausted, the number of bytes read from it does not exactly
-     * match the current size of the tentative buffer array, then it is copied into another array sized to fit the
-     * number of bytes read; otherwise, it is returned without that final copy step.
      *
      * @param byteSource The source which bytes we're interested in.
-     * @param initialBufferCapacity The initial size of the internal buffer.
      * @return A byte array containing exactly all the read bytes. In case of a {@code null} source, the returned byte
      * array will be empty.
      */
-    public static byte[] readBytes(ByteSource byteSource, final int initialBufferCapacity)
+    public static byte[] readBytes(ByteSource byteSource)
     {
-        Preconditions.checkNotNull(byteSource);
+        if (byteSource instanceof ByteSource.ConvertableToArray)
+            return ((ByteSource.ConvertableToArray) byteSource).remainingBytesToArray();
 
-        int readBytes = 0;
-        byte[] buf = new byte[initialBufferCapacity];
-        int data;
-        while ((data = byteSource.next()) != ByteSource.END_OF_STREAM)
+        if (byteSource == null)
+            return ByteArrayUtil.EMPTY_BYTE_ARRAY;
+
+        int step = 232;    // size chosen so that new byte[step] fits into 256 bytes
+        byte[] last = new byte[step];
+        int copied = byteSource.nextBytes(last);
+        if (copied < step)
+            return Arrays.copyOf(last, copied);
+
+        List<byte[]> other = new ArrayList<>();
+        do
         {
-            buf = ensureCapacity(buf, readBytes);
-            buf[readBytes++] = (byte) data;
+            other.add(last);
+            last = new byte[step];
+            copied = byteSource.nextBytes(last);
         }
+        while (copied == step);
 
-        if (readBytes != buf.length)
+        byte[] dest = new byte[other.size() * step + copied];
+        int pos = 0;
+        for (byte[] b : other)
         {
-            buf = Arrays.copyOf(buf, readBytes);
+            System.arraycopy(b, 0, dest, pos, step);
+            pos += step;
         }
-        return buf;
-    }
-
-    /**
-     * Reads the bytes of the given source into a byte array. Doesn't do any transformation on the bytes, just reads
-     * them until it reads an {@link ByteSource#END_OF_STREAM} byte, after which it returns an array of all the read
-     * bytes, <strong>excluding the {@link ByteSource#END_OF_STREAM}</strong>.
-     * <p>
-     * This is equivalent to {@link #readBytes(ByteSource, int)} where the second actual parameter is
-     * {@linkplain #INITIAL_BUFFER_CAPACITY} ({@value INITIAL_BUFFER_CAPACITY}).
-     *
-     * @param byteSource The source which bytes we're interested in.
-     * @return A byte array containing exactly all the read bytes. In case of a {@code null} source, the returned byte
-     * array will be empty.
-     */
-    public static byte[] readBytes(ByteSource byteSource)
-    {
-        return readBytes(byteSource, INITIAL_BUFFER_CAPACITY);
+        System.arraycopy(last, 0, dest, pos, copied);
+        return dest;
     }
 
     public static void copyBytes(ByteSource byteSource, byte[] bytes)
@@ -416,19 +408,17 @@ public static void copyBytes(ByteSource byteSource, byte[] bytes)
     }
 
     /**
-     * Ensures the given buffer has capacity for taking data with the given length - if it doesn't, it returns a copy
-     * of the buffer, but with double the capacity.
+     * Reads the bytes of the given source into the given byte array and returns the number of bytes read. Doesn't do
+     * any transformation on the bytes, just reads them until it reads an {@code ByteSource.END_OF_STREAM} byte. If the
+     * target byte array does not have enough space to fit the whole source, a {@code RuntimeException} is thrown. See
+     * also {@link ByteSource#nextBytes(byte[])}.
      */
-    private static byte[] ensureCapacity(byte[] buf, int dataLengthInBytes)
+    public static int readBytesMustFit(ByteSource byteSource, byte[] dest)
     {
-        if (dataLengthInBytes == buf.length)
-            // We won't gain much with guarding against overflow. We'll overflow when dataLengthInBytes >= 1 << 30,
-            // and if we do guard, we'll be able to extend the capacity to Integer.MAX_VALUE (which is 1 << 31 - 1).
-            // Controlling the exception that will be thrown shouldn't matter that much, and  in practice, we almost
-            // surely won't be reading gigabytes of ByteSource data at once.
-            return Arrays.copyOf(buf, dataLengthInBytes * 2);
-        else
-            return buf;
+        int read = byteSource.nextBytes(dest);
+        if (read == dest.length && byteSource.next() != ByteSource.END_OF_STREAM)
+            throw new RuntimeException(String.format("Number of bytes available exceeds the buffer size of %d.", dest.length));
+        return read;
     }
 
     /**
@@ -478,7 +468,7 @@ public static ByteSource.Peekable nextComponentSource(ByteSource.Peekable source
 
     public static boolean nextComponentNull(int separator)
     {
-        return separator == ByteSource.NEXT_COMPONENT_NULL || separator == ByteSource.NEXT_COMPONENT_EMPTY
-               || separator == ByteSource.NEXT_COMPONENT_EMPTY_REVERSED;
+        return separator == ByteSource.NEXT_COMPONENT_NULL
+               || separator == ByteSource.NEXT_COMPONENT_NULL_REVERSED;
     }
 }
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/PreencodedByteComparable.java b/src/java/org/apache/cassandra/utils/bytecomparable/PreencodedByteComparable.java
new file mode 100644
index 000000000000..d14f466dcfce
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/PreencodedByteComparable.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.nio.ByteBuffer;
+
+abstract class PreencodedByteComparable implements ByteComparable.Preencoded
+{
+    private final Version version;
+
+    PreencodedByteComparable(Version version)
+    {
+        this.version = version;
+    }
+
+    @Override
+    public Version encodingVersion()
+    {
+        return version;
+    }
+
+    static class Array extends PreencodedByteComparable
+    {
+        private final byte[] bytes;
+        private final int offset;
+        private final int length;
+
+        Array(Version version, byte[] bytes)
+        {
+            this(version, bytes, 0, bytes.length);
+        }
+
+        Array(Version version, byte[] bytes, int offset, int length)
+        {
+            super(version);
+            this.bytes = bytes;
+            this.offset = offset;
+            this.length = length;
+        }
+
+        @Override
+        public ByteSource.Duplicatable getPreencodedBytes()
+        {
+            return ByteSource.preencoded(bytes, offset, length);
+        }
+    }
+
+    static class Buffer extends PreencodedByteComparable
+    {
+        private final ByteBuffer bytes;
+
+        Buffer(Version version, ByteBuffer bytes)
+        {
+            super(version);
+            this.bytes = bytes;
+        }
+
+        @Override
+        public ByteSource.Duplicatable getPreencodedBytes()
+        {
+            return ByteSource.preencoded(bytes);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/concurrent/LightweightRecycler.java b/src/java/org/apache/cassandra/utils/concurrent/LightweightRecycler.java
index 31fbf0c4794e..cdc6ea34c6e9 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/LightweightRecycler.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/LightweightRecycler.java
@@ -48,6 +48,7 @@ default T reuse()
     }
 
     /**
+     * @param supplier
      * @return a reusable instance, or allocate one via the provided supplier
      */
     default T reuseOrAllocate(Supplier<T> supplier)
diff --git a/src/java/org/apache/cassandra/utils/concurrent/OpOrder.java b/src/java/org/apache/cassandra/utils/concurrent/OpOrder.java
index 7f18a0ceaee4..096b102d7e52 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/OpOrder.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/OpOrder.java
@@ -425,6 +425,17 @@ public void await()
             current.await();
         }
 
+        /**
+         * @return true if all operations started prior to barrier.issue() have completed
+         */
+        public boolean allPriorOpsAreFinished()
+        {
+            Group current = orderOnOrBefore;
+            if (current == null)
+                throw new IllegalStateException("This barrier needs to have issue() called on it before prior operations can complete");
+            return current.isFinished();
+        }
+
         /**
          * returns the Group we are waiting on - any Group with {@code .compareTo(getSyncPoint()) <= 0}
          * must complete before await() returns
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Ref.java b/src/java/org/apache/cassandra/utils/concurrent/Ref.java
index e268f5fd73c2..1335aa5b5b80 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/Ref.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/Ref.java
@@ -53,7 +53,6 @@
 import org.apache.cassandra.utils.Shared;
 import sun.misc.Unsafe;
 import sun.nio.ch.DirectBuffer;
-
 import org.cliffc.high_scale_lib.NonBlockingHashMap;
 
 import static java.util.Collections.emptyList;
@@ -154,6 +153,11 @@ public T get()
         return referent;
     }
 
+    public boolean refers(T object)
+    {
+        return referent == object;
+    }
+
     public Ref<T> tryRef()
     {
         return state.globalState.ref() ? new Ref<>(referent, state.globalState) : null;
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Refs.java b/src/java/org/apache/cassandra/utils/concurrent/Refs.java
index fb6067e21d55..aa6d465ad19d 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/Refs.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/Refs.java
@@ -94,7 +94,7 @@ public Ref<T> get(T referenced)
      */
     public void release(T referenced)
     {
-        Ref ref = references.remove(referenced);
+        Ref<T> ref = references.remove(referenced);
         if (ref == null)
             throw new IllegalStateException("This Refs collection does not hold a reference to " + referenced);
         ref.release();
@@ -107,7 +107,7 @@ public void release(T referenced)
      */
     public boolean releaseIfHolds(T referenced)
     {
-        Ref ref = references.remove(referenced);
+        Ref<T> ref = references.remove(referenced);
         if (ref != null)
             ref.release();
         return ref != null;
@@ -119,9 +119,9 @@ public void relaseAllExcept(Collection<T> keep)
         release.retainAll(keep);
         release(release);
     }
+
     /**
      * Release a retained Ref to all of the provided objects; if any is not held, an exception will be thrown
-     * @param release
      */
     public void release(Collection<T> release)
     {
@@ -222,7 +222,7 @@ public static <T extends RefCounted<T>> Refs<T> tryRef(Iterable<? extends T> ref
             }
             refs.put(rc, ref);
         }
-        return new Refs<T>(refs);
+        return new Refs<>(refs);
     }
 
     public static <T extends RefCounted<T>> Refs<T> ref(Iterable<? extends T> reference)
@@ -237,9 +237,10 @@ public static void release(Iterable<? extends Ref<?>> refs)
     {
         maybeFail(release(refs, null));
     }
+
     public static Throwable release(Iterable<? extends Ref<?>> refs, Throwable accumulate)
     {
-        for (Ref ref : refs)
+        for (Ref<?> ref : refs)
         {
             try
             {
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Timer.java b/src/java/org/apache/cassandra/utils/concurrent/Timer.java
new file mode 100644
index 000000000000..dc586c0b68de
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/concurrent/Timer.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import io.netty.util.HashedWheelTimer;
+import io.netty.util.Timeout;
+import org.apache.cassandra.concurrent.ExecutorLocals;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.context.OperationContext;
+import org.apache.cassandra.tracing.TraceState;
+
+/**
+ * Timer implementation based on the hashed wheel algorithm with 100ms precision, using Netty's
+ * {@link HashedWheelTimer} under the hood.
+ *
+ * @see #onTimeout(Runnable, long, TimeUnit)
+ * @see #onTimeout(Runnable, long, TimeUnit, ExecutorLocals)
+ */
+public class Timer
+{
+    private static final String THREAD_NAME = "hashed-wheel-timer";
+    public static final Timer INSTANCE = new Timer();
+
+    private final HashedWheelTimer timer;
+
+    private Timer()
+    {
+        this.timer = new HashedWheelTimer(new ThreadFactoryBuilder().setDaemon(true).setNameFormat(THREAD_NAME).build(),
+                                          100, TimeUnit.MILLISECONDS);
+
+        this.timer.start();
+    }
+
+    /**
+     * @see #onTimeout(Runnable, long, TimeUnit, ExecutorLocals)
+     */
+    public Future<Void> onTimeout(Runnable task, long timeout, TimeUnit unit)
+    {
+        return onTimeout(task, timeout, unit, null);
+    }
+
+    /**
+     * Schedules the given {@code task} to be run after the given {@code timeout} with related {@code unit} expires,
+     * and returns a {@link Future} that can be used to check for expiration and cancel the timeout. Passed
+     * {@code executorLocals} are eventually propagated to the executed task.
+     */
+    public Future<Void> onTimeout(Runnable task, long timeout, TimeUnit unit, ExecutorLocals executorLocals)
+    {
+        ClientWarn.State clientWarnState = executorLocals == null ? null : executorLocals.clientWarnState;
+        TraceState traceState = executorLocals == null ? null : executorLocals.traceState;
+        RequestSensors sensors = executorLocals == null ? null : executorLocals.sensors;
+        OperationContext operationContext = executorLocals == null ? null : executorLocals.operationContext;
+        AsyncPromise<Void> result = new AsyncPromise<>();
+        Timeout handle = timer.newTimeout(ignored ->
+                                          {
+
+                                              ExecutorLocals.Impl.set(traceState, clientWarnState, sensors, operationContext);
+                                              try
+                                              {
+                                                  task.run();
+                                                  result.setSuccess(null);
+                                              }
+                                              catch (Throwable ex)
+                                              {
+                                                  result.setFailure(ex);
+                                              }
+                                          }, timeout, unit);
+
+        return new Future<Void>()
+        {
+            @Override
+            public boolean cancel(boolean mayInterruptIfRunning)
+            {
+                return handle.cancel();
+            }
+
+            @Override
+            public boolean isCancelled()
+            {
+                return handle.isCancelled();
+            }
+
+            @Override
+            public boolean isDone()
+            {
+                return handle.isExpired();
+            }
+
+            @Override
+            public Void get() throws InterruptedException, ExecutionException
+            {
+                return result.get();
+            }
+
+            @Override
+            public Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
+            {
+                return result.get(timeout, unit);
+            }
+        };
+    }
+
+    public void shutdown()
+    {
+        timer.stop();
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java b/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
index e710d44dd1dc..7b79998b1151 100644
--- a/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
+++ b/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
@@ -18,33 +18,35 @@
 
 package org.apache.cassandra.utils.logging;
 
-import java.lang.management.ManagementFactory;
 import java.security.AccessControlException;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 
-import javax.management.JMX;
-import javax.management.ObjectName;
-
-import org.apache.cassandra.security.ThreadAwareSecurityManager;
+import com.google.common.collect.Maps;
 import org.apache.commons.lang3.StringUtils;
+import org.slf4j.ILoggerFactory;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Maps;
-
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;
 import ch.qos.logback.classic.LoggerContext;
-import ch.qos.logback.classic.jmx.JMXConfiguratorMBean;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.classic.spi.TurboFilterList;
 import ch.qos.logback.classic.turbo.ReconfigureOnChangeFilter;
 import ch.qos.logback.classic.turbo.TurboFilter;
+import ch.qos.logback.classic.util.ContextInitializer;
 import ch.qos.logback.core.Appender;
-import ch.qos.logback.core.hook.DelayingShutdownHook;
+import ch.qos.logback.core.filter.Filter;
+import ch.qos.logback.core.hook.DefaultShutdownHook;
+import ch.qos.logback.core.spi.AppenderAttachable;
+import org.apache.cassandra.security.ThreadAwareSecurityManager;
 
 /**
  * Encapsulates all logback-specific implementations in a central place.
@@ -92,7 +94,7 @@ public void onStartup()
     @Override
     public void onShutdown()
     {
-        DelayingShutdownHook logbackHook = new DelayingShutdownHook();
+        DefaultShutdownHook logbackHook = new DefaultShutdownHook();
         logbackHook.setContext((LoggerContext) LoggerFactory.getILoggerFactory());
         logbackHook.run();
     }
@@ -105,10 +107,11 @@ public void setLoggingLevel(String classQualifier, String rawLevel) throws Excep
         // if both classQualifier and rawLevel are empty, reload from configuration
         if (StringUtils.isBlank(classQualifier) && StringUtils.isBlank(rawLevel))
         {
-            JMXConfiguratorMBean jmxConfiguratorMBean = JMX.newMBeanProxy(ManagementFactory.getPlatformMBeanServer(),
-                                                                          new ObjectName("ch.qos.logback.classic:Name=default,Type=ch.qos.logback.classic.jmx.JMXConfigurator"),
-                                                                          JMXConfiguratorMBean.class);
-            jmxConfiguratorMBean.reloadDefaultConfiguration();
+            LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
+            lc.reset();
+
+            ContextInitializer ci = new ContextInitializer(lc);
+            ci.autoConfig();
             return;
         }
         // classQualifier is set, but blank level given
@@ -177,6 +180,30 @@ private void checkOnlyOneVirtualTableAppender()
                                                           VirtualTableAppender.class.getName(), String.join(",", virtualAppenderNames)));
     }
 
+    private Set<Appender<?>> getAllLogbackAppenders()
+    {
+        ILoggerFactory factory = LoggerFactory.getILoggerFactory();
+        LoggerContext ctx = (LoggerContext) factory;
+
+        Set<Appender<?>> appenders = new HashSet<>();
+        ctx.getLoggerList().forEach(logger -> logger.iteratorForAppenders().forEachRemaining(a -> collectAppenders(a, appenders)));
+        return appenders;
+    }
+
+    private static void collectAppenders(Appender<?> appender, Collection<Appender<?>> collection)
+    {
+        collection.add(appender);
+        if (appender instanceof AppenderAttachable<?>)
+            ((AppenderAttachable<?>) appender).iteratorForAppenders().forEachRemaining(a -> collectAppenders(a, collection));
+    }
+
+    public Set<Filter<?>> getAllLogbackFilters()
+    {
+        return getAllLogbackAppenders().stream()
+                                       .flatMap(a -> a.getCopyOfAttachedFiltersList().stream())
+                                       .collect(Collectors.toSet());
+    }
+
     private boolean hasAppenders(Logger logBackLogger)
     {
         Iterator<Appender<ILoggingEvent>> it = logBackLogger.iteratorForAppenders();
diff --git a/src/java/org/apache/cassandra/utils/memory/BufferPool.java b/src/java/org/apache/cassandra/utils/memory/BufferPool.java
index cddfc8fe6122..1cedd59fa300 100644
--- a/src/java/org/apache/cassandra/utils/memory/BufferPool.java
+++ b/src/java/org/apache/cassandra/utils/memory/BufferPool.java
@@ -26,11 +26,14 @@
 import java.util.Collections;
 import java.util.Queue;
 import java.util.Set;
-import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicLongFieldUpdater;
-import java.util.concurrent.atomic.LongAdder;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.function.BiPredicate;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
@@ -39,12 +42,11 @@
 
 import jdk.internal.ref.Cleaner;
 import net.nicoulaj.compilecommand.annotations.Inline;
-import org.apache.cassandra.concurrent.Shutdownable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import io.netty.util.concurrent.FastThreadLocal;
-
+import org.apache.cassandra.concurrent.Shutdownable;
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.metrics.BufferPoolMetrics;
@@ -57,7 +59,8 @@
 import static com.google.common.collect.ImmutableList.of;
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.UNSAFE;
-import static org.apache.cassandra.utils.ExecutorUtils.*;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BUFFERPOOL_DISABLE_COMBINED_ALLOCATION;
+import static org.apache.cassandra.utils.ExecutorUtils.shutdownAndWait;
 import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
 import static org.apache.cassandra.utils.Shared.Scope.SIMULATION;
 import static org.apache.cassandra.utils.memory.MemoryUtil.isExactlyDirect;
@@ -134,6 +137,7 @@ public class BufferPool
     private static final Logger logger = LoggerFactory.getLogger(BufferPool.class);
     private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 15L, TimeUnit.MINUTES);
     private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocateDirect(0);
+    private static final boolean DISABLE_COMBINED_ALLOCATION = BUFFERPOOL_DISABLE_COMBINED_ALLOCATION.getBoolean();
 
     private volatile Debug debug = Debug.NO_OP;
     private volatile DebugLeaks debugLeaks = DebugLeaks.NO_OP;
@@ -190,7 +194,7 @@ public BufferPool(String name, long memoryUsageThreshold, boolean recyclePartial
         this.memoryUsageThreshold = memoryUsageThreshold;
         this.readableMemoryUsageThreshold = prettyPrintMemory(memoryUsageThreshold);
         this.globalPool = new GlobalPool();
-        this.metrics = new BufferPoolMetrics(name, this);
+        this.metrics = BufferPoolMetrics.create(name, this);
         this.recyclePartially = recyclePartially;
         this.localPoolCleaner = executorFactory().infiniteLoop("LocalPool-Cleaner-" + name, this::cleanupOneReference, UNSAFE);
     }
@@ -219,6 +223,44 @@ public ByteBuffer getAtLeast(int size, BufferType bufferType)
             return localPool.get().getAtLeast(size);
     }
 
+
+    /// Allocate the given amount of memory, where the caller can accept either of:
+    /// - a single buffer that can fit the whole region;
+    /// - multiple buffers of the given `chunkSize`.
+    ///
+    /// The total size must be a multiple of the chunk size.
+    ///
+    /// @param totalSize the total size to be allocated
+    /// @param chunkSize the size of each buffer returned, if the space cannot be allocated as one buffer
+    ///
+    /// @return an array of allocated buffers
+    public ByteBuffer[] getMultiple(int totalSize, int chunkSize, BufferType bufferType)
+    {
+        if (bufferType == BufferType.ON_HEAP)
+            return new ByteBuffer[] { allocate(totalSize, bufferType) };
+
+        // Try to find a buffer to fit the full request. Fragmentation can make this impossible even if we are below
+        // the limits.
+        LocalPool pool = localPool.get();
+        if (!DISABLE_COMBINED_ALLOCATION)
+        {
+            ByteBuffer full = pool.tryGet(totalSize, false);
+            if (full != null)
+                return new ByteBuffer[]{ full };
+        }
+
+        // If we don't get a whole buffer, allocate buffers of the requested chunk size.
+        int numBuffers = totalSize / chunkSize;
+        assert totalSize == chunkSize * numBuffers
+            : "Total size " + totalSize + " is not a multiple of chunk size " + chunkSize;
+        ByteBuffer[] buffers = new ByteBuffer[numBuffers];
+
+        for (int idx = 0; idx < numBuffers; ++idx)
+            buffers[idx] = pool.get(chunkSize);
+
+        return buffers;
+    }
+
     /** Unlike the get methods, this will return null if the pool is exhausted */
     public ByteBuffer tryGet(int size)
     {
@@ -246,6 +288,17 @@ public void put(ByteBuffer buffer)
             updateOverflowMemoryUsage(-buffer.capacity());
     }
 
+    /**
+     * Bulk release multiple buffers.
+     *
+     * @param buffers The buffers to be released.
+     */
+    public void putMultiple(ByteBuffer[] buffers)
+    {
+        for (ByteBuffer buffer : buffers)
+            put(buffer);
+    }
+
     public void putUnusedPortion(ByteBuffer buffer)
     {
         if (isExactlyDirect(buffer))
@@ -751,9 +804,9 @@ private void release()
             clearForEach(Chunk::release);
         }
 
-        private void unsafeRecycle()
+        private void unsafeRecycle(boolean forceEvicted)
         {
-            clearForEach(Chunk::unsafeRecycle);
+            clearForEach(chunk -> Chunk.unsafeRecycle(chunk, forceEvicted));
         }
     }
 
@@ -939,19 +992,19 @@ private ByteBuffer tryGet(int size, boolean sizeIsLowerBound)
             }
             else if (size > NORMAL_CHUNK_SIZE)
             {
-                metrics.misses.mark();
+                metrics.markMissed();
                 return null;
             }
 
             ByteBuffer ret = pool.tryGetInternal(size, sizeIsLowerBound);
             if (ret != null)
             {
-                metrics.hits.mark();
+                metrics.markHit();
                 memoryInUse.add(ret.capacity());
             }
             else
             {
-                metrics.misses.mark();
+                metrics.markMissed();
             }
             return ret;
         }
@@ -1046,9 +1099,9 @@ public void release()
         }
 
         @VisibleForTesting
-        void unsafeRecycle()
+        void unsafeRecycle(boolean forceEvicted)
         {
-            chunks.unsafeRecycle();
+            chunks.unsafeRecycle(forceEvicted);
         }
 
         @VisibleForTesting
@@ -1546,7 +1599,7 @@ void freeUnusedPortion(ByteBuffer buffer)
         @Override
         public String toString()
         {
-            return String.format("[slab %s, slots bitmap %s, capacity %d, free %d]", slab, Long.toBinaryString(freeSlots), capacity(), free());
+            return String.format("[slab %s, slots bitmap %s, capacity %d, free %d, owner %s, recycler %s]", slab, Long.toBinaryString(freeSlots), capacity(), free(), owner, recycler);
         }
 
         @VisibleForTesting
@@ -1562,15 +1615,17 @@ void unsafeFree()
             if (parent != null)
                 parent.free(slab);
             else
-                FileUtils.clean(slab);
+                FileUtils.cleanWithAttachment(slab);
         }
 
-        static void unsafeRecycle(Chunk chunk)
+        static void unsafeRecycle(Chunk chunk, boolean forceRecycle)
         {
             if (chunk != null)
             {
                 chunk.owner = null;
                 chunk.freeSlots = 0L;
+                if (forceRecycle && !chunk.recycler.canRecyclePartially())
+                    chunk.setEvicted();
                 chunk.recycleFully();
             }
         }
@@ -1626,11 +1681,16 @@ public BufferPoolMetrics metrics()
     /** This is not thread safe and should only be used for unit testing. */
     @VisibleForTesting
     public void unsafeReset()
+    {
+        unsafeReset(false);
+    }
+    @VisibleForTesting
+    public void unsafeReset(boolean forceEvicted)
     {
         overflowMemoryUsage.reset();
         memoryInUse.reset();
         memoryAllocated.set(0);
-        localPool.get().unsafeRecycle();
+        localPool.get().unsafeRecycle(forceEvicted);
         globalPool.unsafeFree();
     }
 
diff --git a/src/java/org/apache/cassandra/utils/memory/EnsureOnHeap.java b/src/java/org/apache/cassandra/utils/memory/EnsureOnHeap.java
index 34b9eaacd2f0..f03e29e64490 100644
--- a/src/java/org/apache/cassandra/utils/memory/EnsureOnHeap.java
+++ b/src/java/org/apache/cassandra/utils/memory/EnsureOnHeap.java
@@ -59,8 +59,9 @@ public DecoratedKey applyToPartitionKey(DecoratedKey key)
 
         public Row applyToRow(Row row)
         {
-            if (row == null)
-                return null;
+            // If current "row" is Rows.EMPTY_STATIC_ROW, don't copy it again, as "copied_empty_static_row" != EMPTY_STATIC_ROW
+            if (row == null || row == Rows.EMPTY_STATIC_ROW)
+                return row;
             return row.clone(HeapCloner.instance);
         }
 
diff --git a/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java b/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java
index 453f3eda1ba3..2828a95c6f1a 100644
--- a/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java
+++ b/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java
@@ -21,17 +21,18 @@
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.util.concurrent.atomic.AtomicLong;
 
 import com.sun.jna.Native;
-
+import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.Architecture;
-
 import sun.misc.Unsafe;
 
 public abstract class MemoryUtil
 {
     private static final long UNSAFE_COPY_THRESHOLD = 1024 * 1024L; // copied from java.nio.Bits
 
+    private static final AtomicLong memoryAllocated = new AtomicLong(0);
     private static final Unsafe unsafe;
     private static final Class<?> DIRECT_BYTE_BUFFER_CLASS, RO_DIRECT_BYTE_BUFFER_CLASS;
     private static final long DIRECT_BYTE_BUFFER_ADDRESS_OFFSET;
@@ -51,14 +52,19 @@ public abstract class MemoryUtil
             Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe");
             field.setAccessible(true);
             unsafe = (sun.misc.Unsafe) field.get(null);
-            Class<?> clazz = ByteBuffer.allocateDirect(0).getClass();
+            // OpenJDK for some reason allocates bytes when capacity == 0. When -Dsun.nio.PageAlignDirectMemory is false
+            // DirectByteBuffer allocates 1 byte, when true a whole page is allocated.
+            // This breaks our native memory metrics tests that don't expect Bits.RESERVED_MEMORY > Bits.TOTAL_CAPACITY.
+            // The buffer will be manually cleaned to mitigate the problem.
+            ByteBuffer byteBuffer = ByteBuffer.allocateDirect(0);
+            Class<?> clazz = byteBuffer.getClass();
             DIRECT_BYTE_BUFFER_ADDRESS_OFFSET = unsafe.objectFieldOffset(Buffer.class.getDeclaredField("address"));
             DIRECT_BYTE_BUFFER_CAPACITY_OFFSET = unsafe.objectFieldOffset(Buffer.class.getDeclaredField("capacity"));
             DIRECT_BYTE_BUFFER_LIMIT_OFFSET = unsafe.objectFieldOffset(Buffer.class.getDeclaredField("limit"));
             DIRECT_BYTE_BUFFER_POSITION_OFFSET = unsafe.objectFieldOffset(Buffer.class.getDeclaredField("position"));
             DIRECT_BYTE_BUFFER_ATTACHMENT_OFFSET = unsafe.objectFieldOffset(clazz.getDeclaredField("att"));
             DIRECT_BYTE_BUFFER_CLASS = clazz;
-            RO_DIRECT_BYTE_BUFFER_CLASS = ByteBuffer.allocateDirect(0).asReadOnlyBuffer().getClass();
+            RO_DIRECT_BYTE_BUFFER_CLASS = byteBuffer.asReadOnlyBuffer().getClass();
 
             clazz = ByteBuffer.allocate(0).getClass();
             BYTE_BUFFER_OFFSET_OFFSET = unsafe.objectFieldOffset(ByteBuffer.class.getDeclaredField("offset"));
@@ -66,6 +72,7 @@ public abstract class MemoryUtil
             BYTE_BUFFER_CLASS = clazz;
 
             BYTE_ARRAY_BASE_OFFSET = unsafe.arrayBaseOffset(byte[].class);
+            FileUtils.clean(byteBuffer);
         }
         catch (Exception e)
         {
@@ -86,14 +93,21 @@ public static long getAddress(ByteBuffer buffer)
 
     public static long allocate(long size)
     {
+        memoryAllocated.addAndGet(size);
         return Native.malloc(size);
     }
 
-    public static void free(long peer)
+    public static void free(long peer, long size)
     {
+        memoryAllocated.addAndGet(-size);
         Native.free(peer);
     }
 
+    public static long allocated()
+    {
+        return memoryAllocated.get();
+    }
+
     public static void setByte(long address, byte b)
     {
         unsafe.putByte(address, b);
@@ -154,6 +168,30 @@ public static long getLong(long address)
             return getLongByByte(address);
 	}
 
+    public static long getStaticFieldOffset(Field field)
+    {
+        return unsafe.staticFieldOffset(field);
+    }
+
+    /**
+     * @param address the memory address to use for the new buffer
+     * @param length in bytes of the new buffer
+     * @param capacity in bytes of the new buffer
+     * @param order byte order of the new buffer
+     * @param attachment byte buffer attachment
+     * @return a new DirectByteBuffer setup with the address, length and order required
+     */
+    public static ByteBuffer allocateByteBuffer(long address, int length, int capacity, ByteOrder order, Object attachment)
+    {
+        ByteBuffer instance = getHollowDirectByteBuffer(order);
+        setDirectByteBuffer(instance, address, length, capacity);
+
+        if (attachment != null)
+            MemoryUtil.setAttachment(instance, attachment);
+
+        return instance;
+    }
+
     public static ByteBuffer getByteBuffer(long address, int length)
     {
         return getByteBuffer(address, length, ByteOrder.nativeOrder());
@@ -213,7 +251,7 @@ public static Object getAttachment(ByteBuffer instance)
     }
 
     // Note: If encryption is used, the Object attached must implement sun.nio.ch.DirectBuffer
-    // @see CASSANDRA-18081
+    // @see CASSANDRA-18180
     public static void setAttachment(ByteBuffer instance, Object next)
     {
         assert instance.getClass() == DIRECT_BYTE_BUFFER_CLASS;
@@ -238,13 +276,23 @@ public static ByteBuffer sliceDirectByteBuffer(ByteBuffer source, ByteBuffer hol
     }
 
     public static void setDirectByteBuffer(ByteBuffer instance, long address, int length)
+    {
+        setDirectByteBuffer(instance, address, length, length);
+    }
+
+    public static void setDirectByteBuffer(ByteBuffer instance, long address, int length, int capacity)
     {
         unsafe.putLong(instance, DIRECT_BYTE_BUFFER_ADDRESS_OFFSET, address);
         unsafe.putInt(instance, DIRECT_BYTE_BUFFER_POSITION_OFFSET, 0);
-        unsafe.putInt(instance, DIRECT_BYTE_BUFFER_CAPACITY_OFFSET, length);
+        unsafe.putInt(instance, DIRECT_BYTE_BUFFER_CAPACITY_OFFSET, capacity);
         unsafe.putInt(instance, DIRECT_BYTE_BUFFER_LIMIT_OFFSET, length);
     }
 
+    public static void setObjectVolatile(Object o, long l, Object o1)
+    {
+        unsafe.putObjectVolatile(o, l, o1);
+    }
+
     public static void setByteBufferCapacity(ByteBuffer instance, int capacity)
     {
         unsafe.putInt(instance, DIRECT_BYTE_BUFFER_CAPACITY_OFFSET, capacity);
@@ -353,6 +401,12 @@ public static void putIntByByte(long address, int value)
         }
     }
 
+    /**
+     * Transfers the contents of a buffer to Memory
+     *
+     * @param address start offset in the memory
+     * @param buffer the data buffer
+     */
     public static void setBytes(long address, ByteBuffer buffer)
     {
         int start = buffer.position();
diff --git a/src/java/org/apache/cassandra/utils/memory/MemtableAllocator.java b/src/java/org/apache/cassandra/utils/memory/MemtableAllocator.java
index 139d4a06b20d..b298498ca7ec 100644
--- a/src/java/org/apache/cassandra/utils/memory/MemtableAllocator.java
+++ b/src/java/org/apache/cassandra/utils/memory/MemtableAllocator.java
@@ -27,6 +27,7 @@
 
 import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.utils.concurrent.WaitQueue;
+import org.github.jamm.Unmetered;
 
 public abstract class MemtableAllocator
 {
@@ -76,6 +77,11 @@ public SubAllocator offHeap()
         return offHeap;
     }
 
+    public long unusedReservedOnHeapMemory()
+    {
+        return 0; // only slabbed allocators would have non-zero here
+    }
+
     /**
      * Mark this allocator reclaiming; this will permit any outstanding allocations to temporarily
      * overshoot the maximum memory limit so that flushing can begin immediately
@@ -105,6 +111,7 @@ public boolean isLive()
     public static class SubAllocator
     {
         // the tracker we are owning memory from
+        @Unmetered  // total pool size should not be included in memtable's deep size
         private final MemtablePool.SubPool parent;
 
         // the state of the memtable
diff --git a/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java b/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java
index 0d1fdd488fce..5618fe1153ce 100644
--- a/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java
+++ b/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java
@@ -23,11 +23,17 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.NativeClustering;
+import org.apache.cassandra.db.NativeDecoratedKey;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.NativeCell;
+import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.utils.concurrent.OpOrder;
-import org.apache.cassandra.utils.concurrent.Semaphore;
 import org.apache.cassandra.utils.concurrent.OpOrder.Group;
+import org.apache.cassandra.utils.concurrent.Semaphore;
 
 import static org.apache.cassandra.utils.concurrent.Semaphore.newSemaphore;
 
@@ -79,7 +85,7 @@ private CloningBTreeRowBuilder(OpOrder.Group writeOp, NativeAllocator allocator)
         @Override
         public void newRow(Clustering<?> clustering)
         {
-            if (clustering != Clustering.STATIC_CLUSTERING)
+            if (clustering != Clustering.EMPTY && clustering != Clustering.STATIC_CLUSTERING)
                 clustering = new NativeClustering(allocator, writeOp, clustering);
             super.newRow(clustering);
         }
@@ -180,7 +186,7 @@ private void trySwapRegion(Region current, int minSize)
         if (currentRegion.compareAndSet(current, next))
             regions.add(next);
         else if (!raceAllocated.stash(next))
-            MemoryUtil.free(next.peer);
+            MemoryUtil.free(next.peer, next.capacity);
     }
 
     private long allocateOversize(int size)
@@ -200,7 +206,7 @@ private long allocateOversize(int size)
     public void setDiscarded()
     {
         for (Region region : regions)
-            MemoryUtil.free(region.peer);
+            MemoryUtil.free(region.peer, region.capacity);
 
         super.setDiscarded();
     }
diff --git a/src/java/org/apache/cassandra/utils/memory/SlabAllocator.java b/src/java/org/apache/cassandra/utils/memory/SlabAllocator.java
index 05f99275e467..682aef9877d7 100644
--- a/src/java/org/apache/cassandra/utils/memory/SlabAllocator.java
+++ b/src/java/org/apache/cassandra/utils/memory/SlabAllocator.java
@@ -23,6 +23,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -48,7 +49,8 @@ public class SlabAllocator extends MemtableBufferAllocator
 {
     private static final Logger logger = LoggerFactory.getLogger(SlabAllocator.class);
 
-    private final static int REGION_SIZE = 1024 * 1024;
+    @VisibleForTesting
+    public final static int REGION_SIZE = 1024 * 1024;
     private final static int MAX_CLONED_SIZE = 128 * 1024; // bigger than this don't go in the region
 
     // globally stash any Regions we allocate but are beaten to using, and use these up before allocating any more
@@ -75,6 +77,17 @@ public EnsureOnHeap ensureOnHeap()
         return ensureOnHeap;
     }
 
+    @Override
+    public long unusedReservedOnHeapMemory()
+    {
+        if (!allocateOnHeapOnly)
+            return 0;
+        Region current = currentRegion.get();
+        if (current == null)
+            return 0;
+        return current.unusedReservedMemory();
+    }
+
     public ByteBuffer allocate(int size)
     {
         return allocate(size, null);
@@ -152,9 +165,10 @@ private Region getRegion()
         }
     }
 
-    public Cloner cloner(OpOrder.Group writeOp)
+    @Override
+    public Cloner cloner(OpOrder.Group opGroup)
     {
-        return allocator(writeOp);
+        return allocator(opGroup);
     }
 
     /**
@@ -211,5 +225,10 @@ public String toString()
             return "Region@" + System.identityHashCode(this) +
                    "waste=" + Math.max(0, data.capacity() - nextFreeOffset.get());
         }
+
+        long unusedReservedMemory()
+        {
+            return data.capacity() - nextFreeOffset.get();
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/utils/obs/MemoryLimiter.java b/src/java/org/apache/cassandra/utils/obs/MemoryLimiter.java
new file mode 100644
index 000000000000..bb2eb28a341d
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/obs/MemoryLimiter.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.obs;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.cassandra.utils.FBUtilities;
+
+public class MemoryLimiter
+{
+    public final long maxMemory;
+    private final AtomicLong currentMemory;
+    private final String exceptionFormat;
+
+    public MemoryLimiter(long maxMemory, String exceptionFormat)
+    {
+        this.maxMemory = maxMemory;
+        this.currentMemory = new AtomicLong();
+        this.exceptionFormat = exceptionFormat;
+    }
+
+    public void increment(long bytesCount) throws ReachedMemoryLimitException
+    {
+        assert bytesCount >= 0;
+        long bytesCountAfterAllocation = this.currentMemory.addAndGet(bytesCount);
+        if (bytesCountAfterAllocation >= maxMemory)
+        {
+            this.currentMemory.addAndGet(-bytesCount);
+
+            throw new ReachedMemoryLimitException(String.format(exceptionFormat,
+                                                                FBUtilities.prettyPrintMemory(bytesCount),
+                                                                FBUtilities.prettyPrintMemory(maxMemory),
+                                                                FBUtilities.prettyPrintMemory(bytesCountAfterAllocation - bytesCount)));
+        }
+    }
+
+    public void decrement(long bytesCount)
+    {
+        assert bytesCount >= 0;
+        long result = this.currentMemory.addAndGet(-bytesCount);
+        assert result >= 0;
+    }
+
+    public long memoryAllocated()
+    {
+        return currentMemory.get();
+    }
+
+    public static class ReachedMemoryLimitException extends Exception
+    {
+        public ReachedMemoryLimitException(String message)
+        {
+            super(message);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
index be0ecf556f0b..cee21c3fb514 100644
--- a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
+++ b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.utils.obs;
 
 import java.io.*;
+import java.io.IOException;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -34,30 +35,53 @@
  */
 public class OffHeapBitSet implements IBitSet
 {
+    /**
+     * The maximum memory that can be used by bloom filters, in megabytes, overall.
+     * The default is unlimited, a limit should only be set as a last resort measure.
+     */
+    @VisibleForTesting
     private final Memory bytes;
+    private final MemoryLimiter memoryLimiter;
 
-    public OffHeapBitSet(long numBits)
+    public OffHeapBitSet(long numBits, MemoryLimiter memoryLimiter) throws MemoryLimiter.ReachedMemoryLimitException
     {
-        /** returns the number of 64 bit words it would take to hold numBits */
+        this.memoryLimiter = memoryLimiter;
+        // returns the number of 64 bit words it would take to hold numBits
         long wordCount = (((numBits - 1) >>> 6) + 1);
         if (wordCount > Integer.MAX_VALUE)
             throw new UnsupportedOperationException("Bloom filter size is > 16GB, reduce the bloom_filter_fp_chance");
+
+        long byteCount = wordCount * 8L;
+        bytes = allocate(byteCount, memoryLimiter); // Can possibly throw OOM, but we handle it in the caller
+        // flush/clear the existing memory.
+        clear();
+    }
+
+    private OffHeapBitSet(Memory bytes, MemoryLimiter memoryLimiter)
+    {
+        this.memoryLimiter = memoryLimiter;
+        this.bytes = bytes;
+    }
+
+    private static Memory allocate(long byteCount, MemoryLimiter memoryLimiter) throws MemoryLimiter.ReachedMemoryLimitException
+    {
+        memoryLimiter.increment(byteCount);
         try
         {
-            long byteCount = wordCount * 8L;
-            bytes = Memory.allocate(byteCount);
+            return Memory.allocate(byteCount);
         }
         catch (OutOfMemoryError e)
         {
-            throw new RuntimeException("Out of native memory occured, You can avoid it by increasing the system ram space or by increasing bloom_filter_fp_chance.");
+            memoryLimiter.decrement(byteCount);
+            throw e;
         }
-        // flush/clear the existing memory.
-        clear();
     }
 
-    private OffHeapBitSet(Memory bytes)
+    private static void release(Memory memory, MemoryLimiter memoryLimiter)
     {
-        this.bytes = bytes;
+        long size = memory.size();
+        memory.free();
+        memoryLimiter.decrement(size);
     }
 
     public long capacity()
@@ -141,10 +165,11 @@ public long serializedSize()
         return TypeSizes.sizeof((int) bytes.size()) + bytes.size();
     }
 
-    public static <I extends InputStream & DataInput> OffHeapBitSet deserialize(I in, boolean oldBfFormat) throws IOException
+    @SuppressWarnings("resource")
+    public static <I extends InputStream & DataInput> OffHeapBitSet deserialize(I in, boolean oldBfFormat, MemoryLimiter memoryLimiter) throws IOException, MemoryLimiter.ReachedMemoryLimitException
     {
         long byteCount = in.readInt() * 8L;
-        Memory memory = Memory.allocate(byteCount);
+        Memory memory = allocate(byteCount, memoryLimiter);
         if (oldBfFormat)
         {
             for (long i = 0; i < byteCount; )
@@ -164,12 +189,12 @@ public static <I extends InputStream & DataInput> OffHeapBitSet deserialize(I in
         {
             FBUtilities.copy(in, new MemoryOutputStream(memory), byteCount);
         }
-        return new OffHeapBitSet(memory);
+        return new OffHeapBitSet(memory, memoryLimiter);
     }
 
     public void close()
     {
-        bytes.free();
+        release(bytes, memoryLimiter);
     }
 
     @Override
@@ -188,7 +213,7 @@ public int hashCode()
     {
         // Similar to open bitset.
         long h = 0;
-        for (long i = bytes.size(); --i >= 0;)
+        for (long i = bytes.size(); --i >= 0; )
         {
             h ^= bytes.getByte(i);
             h = (h << 1) | (h >>> 63); // rotate left
@@ -198,6 +223,6 @@ public int hashCode()
 
     public String toString()
     {
-        return "[OffHeapBitSet]";
+        return String.format("[OffHeapBitSet %s]", FBUtilities.prettyPrintMemory(serializedSize()));
     }
 }
diff --git a/src/java/org/apache/cassandra/utils/units/RateUnit.java b/src/java/org/apache/cassandra/utils/units/RateUnit.java
new file mode 100644
index 000000000000..4dd1612e2eaf
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/units/RateUnit.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.utils.Comparables;
+
+/**
+ * Represents the unit of a rate of transfer/work in term of byte sizes dealt with in a given time. As such, a
+ * {@link RateUnit} unit is simply the combination of a {@link SizeUnit} and a {@link TimeUnit}.
+ * <p>
+ * Note that while the code is relatively in that it can manipulate any combination of size unit and time unit, we
+ * pre-declare only a handful of the most common rates (only in seconds in practice).
+ */
+public class RateUnit implements Comparable<RateUnit>
+{
+    /**
+     * Bytes per Seconds.
+     */
+    public static final RateUnit B_S = RateUnit.of(SizeUnit.BYTES, TimeUnit.SECONDS);
+    /**
+     * KiloBytes per Seconds.
+     */
+    public static final RateUnit KB_S = RateUnit.of(SizeUnit.KILOBYTES, TimeUnit.SECONDS);
+    /**
+     * MegaBytes per Seconds.
+     */
+    public static final RateUnit MB_S = RateUnit.of(SizeUnit.MEGABYTES, TimeUnit.SECONDS);
+    /**
+     * GigaBytes per Seconds.
+     */
+    public static final RateUnit GB_S = RateUnit.of(SizeUnit.GIGABYTES, TimeUnit.SECONDS);
+    /**
+     * TeraBytes per Seconds.
+     */
+    public static final RateUnit TB_S = RateUnit.of(SizeUnit.TERABYTES, TimeUnit.SECONDS);
+
+    public final SizeUnit sizeUnit;
+    public final TimeUnit timeUnit;
+
+    private RateUnit(SizeUnit sizeUnit, TimeUnit timeUnit)
+    {
+        this.sizeUnit = sizeUnit;
+        this.timeUnit = timeUnit;
+    }
+
+    public static RateUnit of(SizeUnit sizeUnit, TimeUnit timeUnit)
+    {
+        return new RateUnit(sizeUnit, timeUnit);
+    }
+
+    /**
+     * Convert the given rate in the given unit to this unit. Conversions from finer to coarser granularities truncate,
+     * so lose precision, conversions from coarser to finer granularities with arguments that would numerically overflow
+     * saturate to <tt>Long.MIN_VALUE</tt> if negative or <tt>Long.MAX_VALUE</tt> if positive.
+     * <p>
+     * For example, to convert 10 megabytes per seconds to bytes per seconds, use: {@code B_S.convert(10L, MB_S)}.
+     *
+     * @param sourceRate the rate to convert in the given {@code sourceUnit}.
+     * @param sourceUnit the unit of the {@code sourceSize} argument
+     * @return the converted size in this unit, or {@code Long.MIN_VALUE} if conversion would negatively overflow, or
+     * {@code Long.MAX_VALUE} if it would positively overflow.
+     */
+    public long convert(long sourceRate, RateUnit sourceUnit)
+    {
+        // We need to convert the size unit and the time unit. For the time unit, since it's a rate, we basically want
+        // to do the opposite of converting from the sourceUnit to the destinationUnit, so we convert from the
+        // destinationUnit to the sourceUnit, even though the value is obviously not in the destination unit in the
+        // first place.
+        // The order we apply the conversion matters however: say we convert '10 MB/s' to 'GB/days': if we were to apply
+        // the size conversion first, we'd get 0, since 10MB is 0GB. So we should apply the time conversion first
+        // ('10 MB/s' is '10 * 3600 * 24 MB/days') and then do the size conversion. Conversely, when converting
+        // '10 MB/s' to 'B/ms', we shouldn't convert by time first, as 10ms is 0s (we do the inverse conversion).
+        // In practice, if the source size unit is smaller than the destination one, we want to apply the time conversion
+        // first, otherwise, we can apply the size one first.
+        if (sourceUnit.sizeUnit.compareTo(sizeUnit) < 0)
+            return sizeUnit.convert(sourceUnit.timeUnit.convert(sourceRate, timeUnit), sourceUnit.sizeUnit);
+
+        return sourceUnit.timeUnit.convert(sizeUnit.convert(sourceRate, sourceUnit.sizeUnit), timeUnit);
+    }
+
+    /**
+     * Returns a Human Readable representation of the provided value in this unit.
+     * <p>
+     * Note that this method may discard precision for the sake of returning a more human readable value. In other
+     * words, if {@code value} is large, it will be converted to a bigger, more readable unit, even this imply
+     * truncating the value.
+     *
+     * @param value the value in this unit.
+     * @return a potentially truncated but human readable representation of {@code value}.
+     */
+    public String toHumanReadableString(long value)
+    {
+        return Units.toString(value, this);
+    }
+
+    public String toString(long value)
+    {
+        return Units.formatValue(value) + this;
+    }
+
+    static String toString(SizeUnit sizeUnit, TimeUnit timeUnit)
+    {
+        return String.format("%s/%s", sizeUnit.symbol, Units.TIME_UNIT_SYMBOL_FCT.apply(timeUnit));
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(sizeUnit, timeUnit);
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof RateUnit))
+            return false;
+
+        RateUnit that = (RateUnit) other;
+        return this.sizeUnit == that.sizeUnit && this.timeUnit == that.timeUnit;
+    }
+
+    @Override
+    public String toString()
+    {
+        return toString(sizeUnit, timeUnit);
+    }
+
+    /**
+     * Given a value in this unit, returns the smallest (most fine grained) unit in which that value can be represented
+     * without overflowing.
+     *
+     * @param value the value in this unit.
+     * @return the smallest unit, potentially this unit, at which the value can be represented without overflowing. If
+     * {@code value == Long.MAX_VALUE}, then this unit is returned.
+     */
+    RateUnit smallestRepresentableUnit(long value)
+    {
+        // This is kind of subtle because we get a smaller unit that this one by both decreasing the size unit
+        // and increasing the time unit, and both don't have the same effect, so we need to find the most optimal
+        // application of both operation that don't overflow our value.
+        // For instance, consider v1=(Long.MAX_VALUE-1 / 1000), then the smallest representable unit for
+        // v1 MB/ms is MB/s (kB/ms doesn't work), while for v2=(Long.MAX_VALUE-1 / 1024) MB/ms, the smallest
+        // representable unit is actually kB/ms (it's also representable as MB/s, but it's a bigger unit).
+        //
+        // So we proceed by applying each option (decreasing size or incrementing time), check if we overflow with each
+        // and if we don't apply recursively. We then compare the unit from both recursive call to find the smallest
+        // one.
+        if (value == Long.MAX_VALUE)
+            return this;
+
+        SizeUnit nextSizeUnit = next(sizeUnit);
+        TimeUnit nextTimeUnit = next(timeUnit);
+
+        long vSize = nextSizeUnit == null ? Long.MAX_VALUE : nextSizeUnit.convert(value, sizeUnit);
+        // Reminder that because the time divide the rate, the conversion should be applied in reverse
+        long vTime = nextTimeUnit == null ? Long.MAX_VALUE : timeUnit.convert(value, nextTimeUnit);
+
+        RateUnit smallestWithSize = vSize == Long.MAX_VALUE
+                                    ? this
+                                    : RateUnit.of(nextSizeUnit, timeUnit).smallestRepresentableUnit(vSize);
+        RateUnit smallestWithTime = vTime == Long.MAX_VALUE
+                                    ? this
+                                    : RateUnit.of(sizeUnit, nextTimeUnit).smallestRepresentableUnit(vTime);
+
+        return Comparables.min(smallestWithSize, smallestWithTime);
+    }
+
+    private static SizeUnit next(SizeUnit unit)
+    {
+        int ordinal = unit.ordinal();
+        return ordinal == 0 ? null : SizeUnit.values()[ordinal - 1];
+    }
+
+    private static TimeUnit next(TimeUnit unit)
+    {
+        int ordinal = unit.ordinal();
+        return ordinal == TimeUnit.values().length - 1 ? null : TimeUnit.values()[ordinal + 1];
+    }
+
+    public int compareTo(RateUnit that)
+    {
+        // Comparing rate units is a tad tricky. We're asking what is the biggest "transfer rate" between 1 of this unit
+        // versus 1 of 'that' unit. This is easier when one of the unit is the same in each unit however.
+        if (this.sizeUnit == that.sizeUnit)
+            return that.timeUnit.compareTo(this.timeUnit); // 1 MB/h is smaller/slower than 1 MB/s
+
+        if (this.timeUnit == that.timeUnit)
+            return this.sizeUnit.compareTo(that.sizeUnit); // 1 MB/s is smaller/slower than 1 TB/s
+
+        // Otherwise, we have to compute by how much it differs in size versus by how much it differs in time.
+        if (this.sizeUnit.compareTo(that.sizeUnit) < 0)
+        {
+            if (this.timeUnit.compareTo(that.timeUnit) < 0)
+            {
+                // this = 1 B/ms and that = 1 MB/s
+                // How much we'll multiply 'that' to get it into 'this' size unit
+                long thatScale = valueDiff(this.sizeUnit, that.sizeUnit);
+                // How much we'll multiply 'this' to get it into 'that' time unit
+                long thisScale = valueDiff(this.timeUnit, that.timeUnit);
+                // 'that' is bigger if it is bigger when put in the same unit than 'this', that is if we'll multiply it
+                // by a bigger value
+                return Long.compare(thisScale, thatScale);
+            }
+            else
+            {
+                // this = 1 B/s and that = 1 MB/ms
+                // That transfers more data in less time, it's definitively faster (bigger)
+                return -1;
+            }
+        }
+        else
+        {
+            if (this.timeUnit.compareTo(that.timeUnit) < 0)
+            {
+                // This transfers more data in less time, it's definitively faster (bigger)
+                return 1;
+            }
+            else
+            {
+                // this = 1 MB/s and that = 1 B/ms
+                // How much we'll multiply 'this' to get it into 'that' size unit
+                long thisScale = valueDiff(that.sizeUnit, this.sizeUnit);
+                // How much we'll multiply 'that' to get it into 'this' time unit
+                long thatScale = valueDiff(that.timeUnit, this.timeUnit);
+                // 'that' is bigger if it is bigger when put in the same unit than 'this', that is if we'll multiply it
+                // by a bigger value
+                return Long.compare(thisScale, thatScale);
+            }
+        }
+    }
+
+    /**
+     * The difference in value between 2 different size unit min and max, where min < max.
+     */
+    private static long valueDiff(SizeUnit min, SizeUnit max)
+    {
+        return 1024L * (max.ordinal() - min.ordinal());
+    }
+
+    /**
+     * The difference in value between 2 different time unit min and max, where min < max.
+     */
+    private static long valueDiff(TimeUnit min, TimeUnit max)
+    {
+        TimeUnit[] all = TimeUnit.values();
+        long val = 1;
+        for (int i = min.ordinal(); i < max.ordinal(); i++)
+            val *= Units.TIME_UNIT_SCALE_FCT.applyAsLong(all[i]);
+        return val;
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/units/RateValue.java b/src/java/org/apache/cassandra/utils/units/RateValue.java
new file mode 100644
index 000000000000..102dea76930f
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/units/RateValue.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import org.apache.cassandra.utils.Comparables;
+
+/**
+ * A {@code RateValue} represents a particular rate in a particular {@link RateUnit}.
+ * <p>
+ * Note that this can only represent positive sizes.
+ */
+public class RateValue implements Comparable<RateValue>
+{
+    public static final RateValue ZERO = new RateValue(0, RateUnit.B_S);
+
+    public final long value;
+    public final RateUnit unit;
+
+    private RateValue(long value, RateUnit unit)
+    {
+        assert value >= 0 && value != Long.MAX_VALUE;
+        this.value = value;
+        this.unit = unit;
+    }
+
+    /**
+     * Creates a new {@link RateValue} for the provided value in the provided unit.
+     *
+     * @param value the value in {@code unit}, which must be positive and strictly less than {@code Long.MAX_VALUE}
+     *              (the latter being used to represent overflows).
+     * @param unit  the unit of {@code value}.
+     * @return a newly created {@link RateValue} for {@code value} in {@code unit}.
+     * @throws IllegalArgumentException if {@code value} is negative or equal to {@code Long.MAX_VALUE}.
+     */
+    public static RateValue of(long value, RateUnit unit)
+    {
+        if (value < 0)
+            throw new IllegalArgumentException("Invalid negative value for a rate: " + value);
+        if (value == Long.MAX_VALUE)
+            throw new IllegalArgumentException("Invalid value for a rate, cannot be Long.MAX_VALUE");
+        return new RateValue(value, unit);
+    }
+
+    /**
+     * Computes the rate corresponding to "processing" {@code size} in {@code duration}.
+     *
+     * @param size     the size processed.
+     * @param duration the duration of the process.
+     * @return the rate corresponding to processing {@code size} in {@code duration}.
+     */
+    public static RateValue compute(SizeValue size, TimeValue duration)
+    {
+        SizeUnit bestSizeUnit = size.smallestRepresentableUnit();
+        return RateValue.of(size.in(bestSizeUnit) / duration.value, RateUnit.of(bestSizeUnit, duration.unit));
+    }
+
+    /**
+     * Returns the value this represents in the provided unit.
+     *
+     * @param destinationUnit the unit to return the value in.
+     * @return the value this represent in {@code unit}.
+     */
+    public long in(RateUnit destinationUnit)
+    {
+        return destinationUnit.convert(value, unit);
+    }
+
+    public RateValue convert(RateUnit destinationUnit)
+    {
+        return RateValue.of(in(destinationUnit), destinationUnit);
+    }
+
+    /**
+     * Returns the time required to "process" the provided size at this rate.
+     */
+    public TimeValue timeFor(SizeValue size)
+    {
+        // Convert both the rate and size in the smallest unit in which they don't overflow: this will ensure the most
+        // precise return value.
+        RateUnit smallestForRate = smallestRepresentableUnit();
+        SizeUnit smallestForSize = size.smallestRepresentableUnit();
+
+        SizeUnit toConvert = Comparables.max(smallestForSize, smallestForRate.sizeUnit);
+        return TimeValue.of(size.in(toConvert) / toConvert.convert(value, unit.sizeUnit), unit.timeUnit);
+    }
+
+    private RateUnit smallestRepresentableUnit()
+    {
+        return unit.smallestRepresentableUnit(value);
+    }
+
+    /**
+     * Returns a string representation of this value in the unit it was created with.
+     */
+    public String toRawString()
+    {
+        return unit.toString(value);
+    }
+
+    /**
+     * Returns a Human Readable representation of this value.
+     * <p>
+     * Note that this method may discard precision for the sake of returning a more human readable value. In other
+     * words, this will display the value is a bigger unit than the one it was created with if that improve readability
+     * and this even this imply truncating the value.
+     *
+     * @return a potentially truncated but human readable representation of this value.
+     */
+    @Override
+    public String toString()
+    {
+        return unit.toHumanReadableString(value);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        // Make sure that equals() => same hashCode()
+        return Long.hashCode(in(smallestRepresentableUnit()));
+    }
+
+    /**
+     * Checks the equality of this value with another value.
+     * <p>
+     * Two {@link RateValue} are equal if they represent exactly the same number of bytes in the same number of time.
+     *
+     * @param other the value to check equality with.
+     * @return whether this value and {@code other} represent the same rate.
+     */
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof RateValue))
+            return false;
+
+        RateValue that = (RateValue) other;
+
+        // Convert both value to the most precise unit in which they can both be represented without overflowing and
+        // check we get the same value. If both don't have the same smallest representable unit, they can't be
+        // representing the same number of bytes.
+        RateUnit smallest = this.smallestRepresentableUnit();
+        return smallest.equals(that.smallestRepresentableUnit()) && this.in(smallest) == that.in(smallest);
+    }
+
+    public int compareTo(RateValue that)
+    {
+        // To compare, we need to have the same unit.
+        RateUnit thisSmallest = this.smallestRepresentableUnit();
+        RateUnit thatSmallest = that.smallestRepresentableUnit();
+
+        if (thisSmallest.equals(thatSmallest))
+            return Long.compare(this.in(thisSmallest), that.in(thatSmallest));
+
+        // If one value overflow "before" (it has a bigger smallest representable unit) the other one, then that value
+        // is bigger. Note that rate units are not comparable in the absolute
+        return thisSmallest.compareTo(thatSmallest) > 0 ? 1 : -1;
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/units/SizeUnit.java b/src/java/org/apache/cassandra/utils/units/SizeUnit.java
new file mode 100644
index 000000000000..783097050a37
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/units/SizeUnit.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A {@code SizeUnit} represents byte sizes at a given unit of granularity and provide utility methods to convert across
+ * units. A {@code SizeUnit} does not maintain size information (see {@link SizeValue}), but only represent the unit
+ * itself. A kilobyte is defined a 1024 bytes, a megabyte as 1024 kilobytes, etc...
+ */
+public enum SizeUnit
+{
+    BYTES("B")
+    {
+        public long convert(long s, SizeUnit u)
+        {
+            return u.toBytes(s);
+        }
+
+        public long toBytes(long s)
+        {
+            return s;
+        }
+
+        public long toKiloBytes(long s)
+        {
+            return s / (C1 / C0);
+        }
+
+        public long toMegaBytes(long s)
+        {
+            return s / (C2 / C0);
+        }
+
+        public long toGigaBytes(long s)
+        {
+            return s / (C3 / C0);
+        }
+
+        public long toTeraBytes(long s)
+        {
+            return s / (C4 / C0);
+        }
+    },
+    KILOBYTES("kB")
+    {
+        public long convert(long s, SizeUnit u)
+        {
+            return u.toKiloBytes(s);
+        }
+
+        public long toBytes(long s)
+        {
+            return x(s, C1 / C0, MAX / (C1 / C0));
+        }
+
+        public long toKiloBytes(long s)
+        {
+            return s;
+        }
+
+        public long toMegaBytes(long s)
+        {
+            return s / (C2 / C1);
+        }
+
+        public long toGigaBytes(long s)
+        {
+            return s / (C3 / C1);
+        }
+
+        public long toTeraBytes(long s)
+        {
+            return s / (C4 / C1);
+        }
+    },
+    MEGABYTES("MB")
+    {
+        public long convert(long s, SizeUnit u)
+        {
+            return u.toMegaBytes(s);
+        }
+
+        public long toBytes(long s)
+        {
+            return x(s, C2 / C0, MAX / (C2 / C0));
+        }
+
+        public long toKiloBytes(long s)
+        {
+            return x(s, C2 / C1, MAX / (C2 / C1));
+        }
+
+        public long toMegaBytes(long s)
+        {
+            return s;
+        }
+
+        public long toGigaBytes(long s)
+        {
+            return s / (C3 / C2);
+        }
+
+        public long toTeraBytes(long s)
+        {
+            return s / (C4 / C2);
+        }
+    },
+    GIGABYTES("GB")
+    {
+        public long convert(long s, SizeUnit u)
+        {
+            return u.toGigaBytes(s);
+        }
+
+        public long toBytes(long s)
+        {
+            return x(s, C3 / C0, MAX / (C3 / C0));
+        }
+
+        public long toKiloBytes(long s)
+        {
+            return x(s, C3 / C1, MAX / (C3 / C1));
+        }
+
+        public long toMegaBytes(long s)
+        {
+            return x(s, C3 / C2, MAX / (C3 / C2));
+        }
+
+        public long toGigaBytes(long s)
+        {
+            return s;
+        }
+
+        public long toTeraBytes(long s)
+        {
+            return s / (C4 / C3);
+        }
+    },
+    TERABYTES("TB")
+    {
+        public long convert(long s, SizeUnit u)
+        {
+            return u.toTeraBytes(s);
+        }
+
+        public long toBytes(long s)
+        {
+            return x(s, C4 / C0, MAX / (C4 / C0));
+        }
+
+        public long toKiloBytes(long s)
+        {
+            return x(s, C4 / C1, MAX / (C4 / C1));
+        }
+
+        public long toMegaBytes(long s)
+        {
+            return x(s, C4 / C2, MAX / (C4 / C2));
+        }
+
+        public long toGigaBytes(long s)
+        {
+            return x(s, C4 / C3, MAX / (C4 / C3));
+        }
+
+        public long toTeraBytes(long s)
+        {
+            return s;
+        }
+    };
+
+    /**
+     * The string symbol for that unit
+     **/
+    public final String symbol;
+
+    SizeUnit(String symbol)
+    {
+        this.symbol = symbol;
+    }
+
+    // Handy constants for conversion methods (all are visible for testing)
+    static final long C0 = 1L;
+    static final long C1 = C0 * 1024L;
+    static final long C2 = C1 * 1024L;
+    static final long C3 = C2 * 1024L;
+    static final long C4 = C3 * 1024L;
+
+    private static final long MAX = Long.MAX_VALUE;
+
+    /**
+     * Scale d by m, checking for overflow.
+     * This has a short name to make above code more readable.
+     */
+    @VisibleForTesting
+    static long x(long d, long m, long over)
+    {
+        if (d > over) return Long.MAX_VALUE;
+        if (d < -over) return Long.MIN_VALUE;
+        return d * m;
+    }
+
+    /**
+     * Convert the given size in the given unit to this unit. Conversions from finer to coarser granularities truncate,
+     * so lose precision. For example converting {@code 1023} bytes to kilobytes results in {@code 0}. Conversions from
+     * coarser to finer granularities with arguments that would numerically overflow saturate to <tt>Long.MIN_VALUE</tt>
+     * if negative or <tt>Long.MAX_VALUE</tt> if positive.
+     * <p>
+     * For example, to convert 10 megabytes to bytes, use: {@code SizeUnit.BYTES.convert(10L, SizeUnit.MEGABYTES)}.
+     *
+     * @param sourceSize the size in the given {@code sourceUnit}.
+     * @param sourceUnit the unit of the {@code sourceSize} argument
+     * @return the converted size in this unit, or {@code Long.MIN_VALUE} if conversion would negatively overflow, or
+     * {@code Long.MAX_VALUE} if it would positively overflow.
+     */
+    public abstract long convert(long sourceSize, SizeUnit sourceUnit);
+
+    /**
+     * Equivalent to {@code BYTES.convert(size, this)}.
+     *
+     * @param size the size
+     * @return the converted size, or {@code Long.MIN_VALUE} if conversion would negatively overflow, or
+     * {@code Long.MAX_VALUE} if it would positively overflow.
+     * @see #convert
+     */
+    public abstract long toBytes(long size);
+
+    /**
+     * Equivalent to {@code KILOBYTES.convert(size, this)}.
+     *
+     * @param size the size
+     * @return the converted size, or {@code Long.MIN_VALUE} if conversion would negatively overflow, or
+     * {@code Long.MAX_VALUE} if it would positively overflow.
+     * @see #convert
+     */
+    public abstract long toKiloBytes(long size);
+
+    /**
+     * Equivalent to {@code MEGABYTES.convert(size, this)}.
+     *
+     * @param size the size
+     * @return the converted size, or {@code Long.MIN_VALUE} if conversion would negatively overflow, or
+     * {@code Long.MAX_VALUE} if it would positively overflow.
+     * @see #convert
+     */
+    public abstract long toMegaBytes(long size);
+
+    /**
+     * Equivalent to {@code GIGABYTES.convert(size, this)}.
+     *
+     * @param size the size
+     * @return the converted size, or {@code Long.MIN_VALUE} if conversion would negatively overflow, or
+     * {@code Long.MAX_VALUE} if it would positively overflow.
+     * @see #convert
+     */
+    public abstract long toGigaBytes(long size);
+
+    /**
+     * Equivalent to {@code TERABYTES.convert(size, this)}.
+     *
+     * @param size the size
+     * @return the converted size, or {@code Long.MIN_VALUE} if conversion would negatively overflow, or
+     * {@code Long.MAX_VALUE} if it would positively overflow.
+     * @see #convert
+     */
+    public abstract long toTeraBytes(long size);
+
+    /**
+     * Creates a {@link SizeValue} using the provided {@code value} and this unit.
+     *
+     * @param value the value.
+     * @return a new {@link SizeValue} for {@code value} at this unit.
+     */
+    public SizeValue value(long value)
+    {
+        return SizeValue.of(value, this);
+    }
+
+    /**
+     * Returns a Human Readable representation of the provided value in this unit.
+     * <p>
+     * Note that this method may discard precision for the sake of returning a more human readable value. In other
+     * words, if {@code value} is large, it will be converted to a bigger, more readable unit, even this imply
+     * truncating the value.
+     *
+     * @param value the value in this unit.
+     * @return a potentially truncated but human readable representation of {@code value}.
+     */
+    public String toHumanReadableString(long value)
+    {
+        return Units.toString(value, this);
+    }
+
+    /**
+     * Returns a string representation particularly suitable for logging a value. of this unit.
+     * <p>
+     * The returned representation combines the value displayed in bytes (for the sake of script parsing the log, so
+     * they don't have to bother with unit conversion), followed by the representation from {@link #toHumanReadableString} for
+     * humans.
+     *
+     * @param value the value in this unit.
+     * @return a string representation suitable for logging the value.
+     */
+    public String toLogString(long value)
+    {
+        return Units.toLogString(value, this);
+    }
+
+    /**
+     * Returns a string representation of a value in this unit.
+     *
+     * @param value the value in this unit.
+     * @return a string representation of {@code value} in this unit.
+     */
+    public String toString(long value)
+    {
+        return Units.formatValue(value) + symbol;
+    }
+
+    /**
+     * Given a value in this unit, returns the smallest (most fine grained) unit in which that value can be represented
+     * without overflowing.
+     *
+     * @param value the value in this unit.
+     * @return the smallest unit, potentially this unit, at which the value can be represented without overflowing. If
+     * {@code value == Long.MAX_VALUE}, then this unit is returned.
+     */
+    SizeUnit smallestRepresentableUnit(long value)
+    {
+        int i = ordinal();
+        while (i > 0 && value < Long.MAX_VALUE)
+        {
+            value = x(value, C1, MAX / C1);
+            i--;
+        }
+        return SizeUnit.values()[i];
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/units/SizeValue.java b/src/java/org/apache/cassandra/utils/units/SizeValue.java
new file mode 100644
index 000000000000..1ec494d7d096
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/units/SizeValue.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+/**
+ * A {@code SizeValue} represents a particular size in a particular {@link SizeUnit}.
+ * <p>
+ * Note that this can only represent positive sizes.
+ */
+public class SizeValue implements Comparable<SizeValue>
+{
+    public static final SizeValue ZERO = new SizeValue(0, SizeUnit.BYTES);
+
+    public final long value;
+    public final SizeUnit unit;
+
+    private SizeValue(long value, SizeUnit unit)
+    {
+        assert value >= 0 && value != Long.MAX_VALUE;
+        this.value = value;
+        this.unit = unit;
+    }
+
+    /**
+     * Creates a new {@link SizeValue} for the provided value in the provided unit.
+     *
+     * @param value the value in {@code unit}, which must be positive and strictly less than {@code Long.MAX_VALUE}
+     *              (the latter being used to represent overflows).
+     * @param unit  the unit of {@code value}.
+     * @return a newly created {@link SizeValue} for {@code value} in {@code unit}.
+     * @throws IllegalArgumentException if {@code value} is negative or equal to {@code Long.MAX_VALUE}.
+     */
+    public static SizeValue of(long value, SizeUnit unit)
+    {
+        if (value < 0)
+            throw new IllegalArgumentException("Invalid negative value for a size in bytes: " + value);
+        if (value == Long.MAX_VALUE)
+            throw new IllegalArgumentException("Invalid value for a size in bytes, cannot be Long.MAX_VALUE");
+        return new SizeValue(value, unit);
+    }
+
+    /**
+     * Returns the value this represents in the provided unit.
+     *
+     * @param destinationUnit the unit to return the value in.
+     * @return the value this represent in {@code unit}.
+     */
+    public long in(SizeUnit destinationUnit)
+    {
+        return destinationUnit.convert(value, unit);
+    }
+
+    SizeUnit smallestRepresentableUnit()
+    {
+        return unit.smallestRepresentableUnit(value);
+    }
+
+    /**
+     * Returns a string representation of this value in the unit it was created with.
+     */
+    public String toRawString()
+    {
+        return unit.toString(value);
+    }
+
+    /**
+     * Returns a string representation particularly suitable for logging the value.
+     * <p>
+     * The returned representation combines the value displayed in bytes (for the sake of script parsing the log, so
+     * they don't have to bother with unit conversion), followed by the representation from
+     * {@link SizeUnit#toHumanReadableString(long)} for humans.
+     *
+     * @return a string representation suitable for logging the value.
+     */
+    public String toLogString()
+    {
+        return unit.toLogString(value);
+    }
+
+    /**
+     * Returns a Human Readable representation of this value.
+     * <p>
+     * Note that this method may discard precision for the sake of returning a more human readable value. In other
+     * words, this will display the value is a bigger unit than the one it was created with if that improve readability
+     * and this even this imply truncating the value.
+     *
+     * @return a potentially truncated but human readable representation of this value.
+     */
+    @Override
+    public String toString()
+    {
+        return unit.toHumanReadableString(value);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        // Make sure that equals() => same hashCode()
+        return Long.hashCode(in(smallestRepresentableUnit()));
+    }
+
+    /**
+     * Checks the equality of this value with another value.
+     * <p>
+     * Two {@link SizeValue} are equal if they represent exactly the same number of bytes.
+     *
+     * @param other the value to check equality with.
+     * @return whether this value and {@code other} represent the same number of bytes.
+     */
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof SizeValue))
+            return false;
+
+        SizeValue that = (SizeValue) other;
+
+        // Convert both value to the most precise unit in which they can both be represented without overflowing and
+        // check we get the same value. If both don't have the same smallest representable unit, they can't be
+        // representing the same number of bytes.
+        SizeUnit smallest = this.smallestRepresentableUnit();
+        return smallest == that.smallestRepresentableUnit() && this.in(smallest) == that.in(smallest);
+    }
+
+    public int compareTo(SizeValue that)
+    {
+        // To compare, we need to have the same unit.
+        SizeUnit thisSmallest = this.smallestRepresentableUnit();
+        SizeUnit thatSmallest = that.smallestRepresentableUnit();
+
+        if (thisSmallest == thatSmallest)
+            return Long.compare(this.in(thisSmallest), that.in(thatSmallest));
+
+        // If one value overflow "before" (it has a bigger smallest representable unit) the other one, then that value
+        // is bigger.
+        return thisSmallest.compareTo(thatSmallest) > 0 ? 1 : -1;
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/units/TimeValue.java b/src/java/org/apache/cassandra/utils/units/TimeValue.java
new file mode 100644
index 000000000000..90885a73271b
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/units/TimeValue.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A {@code TimeValue} represents a particular duration in a particular {@link TimeUnit}.
+ */
+public class TimeValue implements Comparable<TimeValue>
+{
+    public static final TimeValue ZERO = new TimeValue(0, TimeUnit.NANOSECONDS);
+
+    final long value;
+    final TimeUnit unit;
+
+    private TimeValue(long value, TimeUnit unit)
+    {
+        this.value = value;
+        this.unit = unit;
+    }
+
+    /**
+     * Creates a new {@link TimeValue} for the provided value in the provided unit.
+     *
+     * @param value the value in {@code unit}.
+     * @param unit  the unit of {@code value}.
+     * @return a newly created {@link TimeValue} for {@code value} in {@code unit}.
+     */
+    public static TimeValue of(long value, TimeUnit unit)
+    {
+        return new TimeValue(value, unit);
+    }
+
+    /**
+     * Returns the value this represents in the provided unit.
+     *
+     * @param destinationUnit the unit to return the value in.
+     * @return the value this represent in {@code unit}.
+     */
+    public long in(TimeUnit destinationUnit)
+    {
+        return destinationUnit.convert(value, unit);
+    }
+
+    static TimeUnit smallestRepresentableUnit(long value, TimeUnit unit)
+    {
+        long v = value;
+        int i = unit.ordinal();
+        TimeUnit u = unit;
+        while (i > 0 && v < Long.MAX_VALUE)
+        {
+            TimeUnit current = u;
+            u = TimeUnit.values()[--i];
+            v = u.convert(v, current);
+        }
+        return u;
+    }
+
+    private TimeUnit smallestRepresentableUnit()
+    {
+        return smallestRepresentableUnit(value, unit);
+    }
+
+    /**
+     * Returns a string representation of this value in the unit it was created with.
+     */
+    public String toRawString()
+    {
+        return Units.formatValue(value) + Units.TIME_UNIT_SYMBOL_FCT.apply(unit);
+    }
+
+    /**
+     * Returns a Human Readable representation of this value.
+     * <p>
+     * Note that this method may discard precision for the sake of returning a more human readable value. In other
+     * words, this will display the value is a bigger unit than the one it was created with if that improve readability
+     * and this even this imply truncating the value.
+     *
+     * @return a potentially truncated but human readable representation of this value.
+     */
+    @Override
+    public String toString()
+    {
+        return Units.toString(value, unit);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        // Make sure that equals() => same hashCode()
+        return Long.hashCode(in(smallestRepresentableUnit()));
+    }
+
+    /**
+     * Checks the equality of this value with another value.
+     * <p>
+     * Two {@link TimeValue} are equal if they represent exactly the same number of nanoseconds.
+     *
+     * @param other the value to check equality with.
+     * @return whether this value and {@code other} represent the same number of nanoseconds.
+     */
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof TimeValue))
+            return false;
+
+        TimeValue that = (TimeValue) other;
+
+        // Convert both value to the most precise unit in which they can both be represented without overflowing and
+        // check we get the same value. If both don't have the same smallest representable unit, they can't be
+        // representing the same number of bytes.
+        TimeUnit smallest = this.smallestRepresentableUnit();
+        return smallest == that.smallestRepresentableUnit() && this.in(smallest) == that.in(smallest);
+    }
+
+    public int compareTo(TimeValue that)
+    {
+        // To compare, we need to have the same unit.
+        TimeUnit thisSmallest = this.smallestRepresentableUnit();
+        TimeUnit thatSmallest = that.smallestRepresentableUnit();
+
+        if (thisSmallest == thatSmallest)
+            return Long.compare(this.in(thisSmallest), that.in(thatSmallest));
+
+        // If one value overflow "before" (it has a bigger smallest representable unit) the other one, then that value
+        // is bigger.
+        return thisSmallest.compareTo(thatSmallest) > 0 ? 1 : -1;
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/units/Units.java b/src/java/org/apache/cassandra/utils/units/Units.java
new file mode 100644
index 000000000000..482e25555fc7
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/units/Units.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.ToLongFunction;
+
+/**
+ * Static methods used by work with units.
+ * <p>
+ * This is mostly useful for {@link TimeUnit}, as for other units the method provided  are more directly accessible in the
+ * unit class itself (we can't modify {@link TimeUnit}), but contains methods for all unit for symmetry.
+ */
+public class Units
+{
+    static final ToLongFunction<TimeUnit> TIME_UNIT_SCALE_FCT = u ->
+    {
+        switch (u)
+        {
+            case NANOSECONDS:
+            case MICROSECONDS:
+            case MILLISECONDS:
+                return 1000L;
+            case SECONDS:
+            case MINUTES:
+                return 60L;
+            case HOURS:
+                return 24L;
+            case DAYS:
+                return 365; // Never actually use but well...
+            default:
+                throw new AssertionError();
+        }
+    };
+    static final Function<TimeUnit, String> TIME_UNIT_SYMBOL_FCT = u ->
+    {
+        switch (u)
+        {
+            case NANOSECONDS:
+                return "ns";
+            case MICROSECONDS:
+                return "us";
+            case MILLISECONDS:
+                return "ms";
+            case SECONDS:
+                return "s";
+            case MINUTES:
+                return "m";
+            case HOURS:
+                return "h";
+            case DAYS:
+                return "d";
+            default:
+                throw new AssertionError();
+        }
+    };
+
+    private static final ToLongFunction<SizeUnit> SIZE_UNIT_SCALE_FCT = u -> SizeUnit.C1;
+    private static final Function<SizeUnit, String> SIZE_UNIT_SYMBOL_FCT = u -> u.symbol;
+
+
+    /**
+     * Returns a Human Readable representation of the provided duration given the unit of said duration.
+     * <p>
+     * This method strives to produce a short and human readable representation and may trade precision for that. In
+     * other words, if the value is large, this will display the value in a bigger unit than the one provided to improve
+     * readability and this even this imply truncating the value.
+     *
+     * @param value the value to build a string of.
+     * @param unit  the unit of {@code value}.
+     * @return a potentially truncated but human readable representation of {@code value}.
+     */
+    public static String toString(long value, TimeUnit unit)
+    {
+        return toString(value, unit, TimeUnit.class, TIME_UNIT_SCALE_FCT, TIME_UNIT_SYMBOL_FCT);
+    }
+
+    /**
+     * Returns a Human Readable representation of the provided size given the unit of said size.
+     * <p>
+     * This method strives to produce a short and human readable representation and may trade precision for that. In
+     * other words, if the value is large, this will display the value in a bigger unit than the one provided to improve
+     * readability and this even this imply truncating the value.
+     *
+     * @param value the value to build a string of.
+     * @param unit  the unit of {@code value}.
+     * @return a potentially truncated but human readable representation of {@code value}.
+     */
+    public static String toString(long value, SizeUnit unit)
+    {
+        return toString(value, unit, SizeUnit.class, SIZE_UNIT_SCALE_FCT, SIZE_UNIT_SYMBOL_FCT);
+    }
+
+    /**
+     * Returns a string representation for a size value (in a particular unit) that is suitable for logging the value.
+     * <p>
+     * The returned representation combines the value displayed in bytes (for the sake of script parsing the log, so
+     * they don't have to bother with unit conversion), followed by the representation from {@link #toString} for
+     * humans.
+     *
+     * @param value a size in {@code unit}.
+     * @param unit  the unit for {@code value}.
+     * @return a string representation suitable for logging the value.
+     */
+    public static String toLogString(long value, SizeUnit unit)
+    {
+        return String.format("%s (%s)", SizeUnit.BYTES.toString(unit.toBytes(value)), toString(value, unit));
+    }
+
+    /**
+     * Returns a Human Readable representation of the provided rate given the unit of said rate.
+     * <p>
+     * This method strives to produce a short and human readable representation and may trade precision for that. In
+     * other words, if the value is large, this will display the value in a bigger unit than the one provided to improve
+     * readability and this even this imply truncating the value.
+     *
+     * @param value the value to build a string of.
+     * @param unit  the unit of {@code value}.
+     * @return a potentially truncated but human readable representation of {@code value}.
+     */
+    public static String toString(long value, RateUnit unit)
+    {
+        // There is theoretically multiple options for any given (large) value since we can play on both the size
+        // and time unit. In practice though, it's much more common to reason with rate 'per second' so we force
+        // seconds as unit of time and play only on the size unit.
+        value = RateUnit.of(unit.sizeUnit, TimeUnit.SECONDS).convert(value, unit);
+        return toString(value, unit.sizeUnit, SizeUnit.class, SIZE_UNIT_SCALE_FCT, u -> RateUnit.toString(u, unit.timeUnit));
+    }
+
+    /**
+     * Format a value a in a human readable way, adding a comma (',') to separate every thousands.
+     * <p>
+     * For instance, {@code formatValue(4693234L) == "4,693,234"}
+     *
+     * @param value the value to format.
+     * @return a more human readable representation of {@code value}.
+     */
+    static String formatValue(long value)
+    {
+        return String.format("%,d", value);
+    }
+
+    /**
+     * The number of comma to use to format {@code digits} digit using ',' on every thousands.
+     */
+    private static int commaCount(int digits)
+    {
+        return (digits - 1) / 3;
+    }
+
+    /**
+     * Returns a Human Readable representation of the provided size/rate given the unit of said size/rate.
+     * <p>
+     * This method strives to produce a short and human readable representation and may trade precision for that. In
+     * other words, if the value is large, this will display the value in a bigger unit than the one provided to improve
+     * readability and this even this imply truncating the value.
+     *
+     * @param value     the value to build a string of.
+     * @param unit      the unit of {@code value}, which is currently either {@link SizeUnit} or {@link RateUnit}
+     * @param klass     Currently can be either a {@link SizeUnit} or {@link RateUnit} class
+     * @param scaleFct  A function that knows how to scale between units of the given {@code unit}
+     * @param symbolFct A function that knows how to scale between symbols of the given {@code unit}
+     * @param <E>       currently either {@link SizeUnit} or {@link RateUnit}
+     * @return a potentially truncated but human readable representation of {@code value}.
+     */
+    private static <E extends Enum<E>> String toString(long value,
+                                                       E unit,
+                                                       Class<E> klass,
+                                                       ToLongFunction<E> scaleFct,
+                                                       Function<E, String> symbolFct)
+    {
+        E[] enumVals = klass.getEnumConstants();
+
+        long v = value;
+        int i = unit.ordinal();
+        long remainder = 0;
+        // The scale is how much we need to go from unit to the next one
+        long scale = scaleFct.applyAsLong(unit);
+
+        while (i < enumVals.length - 1 && v >= scale)
+        {
+            remainder = v % scale;
+            v = v / scale;
+            unit = enumVals[++i];
+            scale = scaleFct.applyAsLong(unit);
+        }
+
+        // If the value is small (<10), include one decimal so the precision is not too truncated. Otherwise, don't
+        // bother, it's less relevant.
+        if (v >= 10 || remainder == 0)
+            return fmt(v, unit, symbolFct);
+
+        // Note that scale is the scale of the current unit, but remainder relates to the previous unit. Also not that
+        // can only get here is remainder != 0 so we know accessing the previous unit is legit
+        long prevScale = scaleFct.applyAsLong(enumVals[i - 1]);
+        int decimal = Math.round(((float) remainder / prevScale) * 10);
+        if (decimal == 0)
+            return fmt(v, unit, symbolFct);
+
+        // If the remainder amounts to more than 0.95 of C1, decimal will be 10. In that case, just bump the value by 1
+        if (decimal == 10)
+            return fmt(v + 1, unit, symbolFct);
+
+        return formatValue(v) + '.' + decimal + symbolFct.apply(unit);
+    }
+
+    private static <E extends Enum<E>> String fmt(long value, E unit, Function<E, String> symbolFct)
+    {
+        return formatValue(value) + symbolFct.apply(unit);
+    }
+}
diff --git a/src/resources/org/apache/cassandra/cql3/reserved_keywords.txt b/src/resources/org/apache/cassandra/cql3/reserved_keywords.txt
index 8a1d2987f9d2..ab0d6b06343f 100644
--- a/src/resources/org/apache/cassandra/cql3/reserved_keywords.txt
+++ b/src/resources/org/apache/cassandra/cql3/reserved_keywords.txt
@@ -54,4 +54,5 @@ USE
 USING
 VIEW
 WHERE
-WITH
\ No newline at end of file
+WITH
+GEO_DISTANCE
\ No newline at end of file
diff --git a/src/resources/org/apache/cassandra/graph/graph.html b/src/resources/org/apache/cassandra/graph/graph.html
new file mode 100644
index 000000000000..dd1f951fe748
--- /dev/null
+++ b/src/resources/org/apache/cassandra/graph/graph.html
@@ -0,0 +1,572 @@
+<!DOCTYPE html>
+<!--
+  Copyright DataStax, Inc.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License atYou may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!-- cstar_perf (https://github.com/datastax/cstar_perf) graphing
+utility adapted for use directly with command line cassandra-stress -->
+
+<head>
+  <meta charset="utf-8">
+  <script language="javascript" type="text/javascript">
+    <!--
+
+/* stats start */
+
+/* stats end */
+
+/*! jQuery v1.11.1 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */
+!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k={},l="1.11.1",m=function(a,b){return new m.fn.init(a,b)},n=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,o=/^-ms-/,p=/-([\da-z])/gi,q=function(a,b){return b.toUpperCase()};m.fn=m.prototype={jquery:l,constructor:m,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=m.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return m.each(this,a,b)},map:function(a){return this.pushStack(m.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},m.extend=m.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||m.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(m.isPlainObject(c)||(b=m.isArray(c)))?(b?(b=!1,f=a&&m.isArray(a)?a:[]):f=a&&m.isPlainObject(a)?a:{},g[d]=m.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},m.extend({expando:"jQuery"+(l+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===m.type(a)},isArray:Array.isArray||function(a){return"array"===m.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){return!m.isArray(a)&&a-parseFloat(a)>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==m.type(a)||a.nodeType||m.isWindow(a))return!1;try{if(a.constructor&&!j.call(a,"constructor")&&!j.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(k.ownLast)for(b in a)return j.call(a,b);for(b in a);return void 0===b||j.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(b){b&&m.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(o,"ms-").replace(p,q)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=r(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(n,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(r(Object(a))?m.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(g)return g.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=r(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(f=a[b],b=a,a=f),m.isFunction(a)?(c=d.call(arguments,2),e=function(){return a.apply(b||this,c.concat(d.call(arguments)))},e.guid=a.guid=a.guid||m.guid++,e):void 0},now:function(){return+new Date},support:k}),m.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function r(a){var b=a.length,c=m.type(a);return"function"===c||m.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var s=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+-new Date,v=a.document,w=0,x=0,y=gb(),z=gb(),A=gb(),B=function(a,b){return a===b&&(l=!0),0},C="undefined",D=1<<31,E={}.hasOwnProperty,F=[],G=F.pop,H=F.push,I=F.push,J=F.slice,K=F.indexOf||function(a){for(var b=0,c=this.length;c>b;b++)if(this[b]===a)return b;return-1},L="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",N="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",O=N.replace("w","w#"),P="\\["+M+"*("+N+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+O+"))|)"+M+"*\\]",Q=":("+N+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+P+")*)|.*)\\)|)",R=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),S=new RegExp("^"+M+"*,"+M+"*"),T=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp("="+M+"*([^\\]'\"]*?)"+M+"*\\]","g"),V=new RegExp(Q),W=new RegExp("^"+O+"$"),X={ID:new RegExp("^#("+N+")"),CLASS:new RegExp("^\\.("+N+")"),TAG:new RegExp("^("+N.replace("w","w*")+")"),ATTR:new RegExp("^"+P),PSEUDO:new RegExp("^"+Q),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+L+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,$=/^[^{]+\{\s*\[native \w/,_=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ab=/[+~]/,bb=/'|\\/g,cb=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),db=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)};try{I.apply(F=J.call(v.childNodes),v.childNodes),F[v.childNodes.length].nodeType}catch(eb){I={apply:F.length?function(a,b){H.apply(a,J.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function fb(a,b,d,e){var f,h,j,k,l,o,r,s,w,x;if((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,d=d||[],!a||"string"!=typeof a)return d;if(1!==(k=b.nodeType)&&9!==k)return[];if(p&&!e){if(f=_.exec(a))if(j=f[1]){if(9===k){if(h=b.getElementById(j),!h||!h.parentNode)return d;if(h.id===j)return d.push(h),d}else if(b.ownerDocument&&(h=b.ownerDocument.getElementById(j))&&t(b,h)&&h.id===j)return d.push(h),d}else{if(f[2])return I.apply(d,b.getElementsByTagName(a)),d;if((j=f[3])&&c.getElementsByClassName&&b.getElementsByClassName)return I.apply(d,b.getElementsByClassName(j)),d}if(c.qsa&&(!q||!q.test(a))){if(s=r=u,w=b,x=9===k&&a,1===k&&"object"!==b.nodeName.toLowerCase()){o=g(a),(r=b.getAttribute("id"))?s=r.replace(bb,"\\$&"):b.setAttribute("id",s),s="[id='"+s+"'] ",l=o.length;while(l--)o[l]=s+qb(o[l]);w=ab.test(a)&&ob(b.parentNode)||b,x=o.join(",")}if(x)try{return I.apply(d,w.querySelectorAll(x)),d}catch(y){}finally{r||b.removeAttribute("id")}}}return i(a.replace(R,"$1"),b,d,e)}function gb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function hb(a){return a[u]=!0,a}function ib(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function jb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function kb(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||D)-(~a.sourceIndex||D);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function lb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function mb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function nb(a){return hb(function(b){return b=+b,hb(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function ob(a){return a&&typeof a.getElementsByTagName!==C&&a}c=fb.support={},f=fb.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=fb.setDocument=function(a){var b,e=a?a.ownerDocument||a:v,g=e.defaultView;return e!==n&&9===e.nodeType&&e.documentElement?(n=e,o=e.documentElement,p=!f(e),g&&g!==g.top&&(g.addEventListener?g.addEventListener("unload",function(){m()},!1):g.attachEvent&&g.attachEvent("onunload",function(){m()})),c.attributes=ib(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=ib(function(a){return a.appendChild(e.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=$.test(e.getElementsByClassName)&&ib(function(a){return a.innerHTML="<div class='a'></div><div class='a i'></div>",a.firstChild.className="i",2===a.getElementsByClassName("i").length}),c.getById=ib(function(a){return o.appendChild(a).id=u,!e.getElementsByName||!e.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if(typeof b.getElementById!==C&&p){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){var c=typeof a.getAttributeNode!==C&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return typeof b.getElementsByTagName!==C?b.getElementsByTagName(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return typeof b.getElementsByClassName!==C&&p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=$.test(e.querySelectorAll))&&(ib(function(a){a.innerHTML="<select msallowclip=''><option selected=''></option></select>",a.querySelectorAll("[msallowclip^='']").length&&q.push("[*^$]="+M+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+M+"*(?:value|"+L+")"),a.querySelectorAll(":checked").length||q.push(":checked")}),ib(function(a){var b=e.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+M+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=$.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&ib(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",Q)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=$.test(o.compareDocumentPosition),t=b||$.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===e||a.ownerDocument===v&&t(v,a)?-1:b===e||b.ownerDocument===v&&t(v,b)?1:k?K.call(k,a)-K.call(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,f=a.parentNode,g=b.parentNode,h=[a],i=[b];if(!f||!g)return a===e?-1:b===e?1:f?-1:g?1:k?K.call(k,a)-K.call(k,b):0;if(f===g)return kb(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)i.unshift(c);while(h[d]===i[d])d++;return d?kb(h[d],i[d]):h[d]===v?-1:i[d]===v?1:0},e):n},fb.matches=function(a,b){return fb(a,null,null,b)},fb.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(U,"='$1']"),!(!c.matchesSelector||!p||r&&r.test(b)||q&&q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return fb(b,n,null,[a]).length>0},fb.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},fb.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&E.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},fb.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},fb.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=fb.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=fb.selectors={cacheLength:50,createPseudo:hb,match:X,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(cb,db),a[3]=(a[3]||a[4]||a[5]||"").replace(cb,db),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||fb.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&fb.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return X.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&V.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(cb,db).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+M+")"+a+"("+M+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||typeof a.getAttribute!==C&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=fb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){k=q[u]||(q[u]={}),j=k[a]||[],n=j[0]===w&&j[1],m=j[0]===w&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[w,n,m];break}}else if(s&&(j=(b[u]||(b[u]={}))[a])&&j[0]===w)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(s&&((l[u]||(l[u]={}))[a]=[w,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||fb.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?hb(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=K.call(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:hb(function(a){var b=[],c=[],d=h(a.replace(R,"$1"));return d[u]?hb(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),!c.pop()}}),has:hb(function(a){return function(b){return fb(a,b).length>0}}),contains:hb(function(a){return function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:hb(function(a){return W.test(a||"")||fb.error("unsupported lang: "+a),a=a.replace(cb,db).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Z.test(a.nodeName)},input:function(a){return Y.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:nb(function(){return[0]}),last:nb(function(a,b){return[b-1]}),eq:nb(function(a,b,c){return[0>c?c+b:c]}),even:nb(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:nb(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:nb(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:nb(function(a,b,c){for(var d=0>c?c+b:c;++d<b;)a.push(d);return a})}},d.pseudos.nth=d.pseudos.eq;for(b in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})d.pseudos[b]=lb(b);for(b in{submit:!0,reset:!0})d.pseudos[b]=mb(b);function pb(){}pb.prototype=d.filters=d.pseudos,d.setFilters=new pb,g=fb.tokenize=function(a,b){var c,e,f,g,h,i,j,k=z[a+" "];if(k)return b?0:k.slice(0);h=a,i=[],j=d.preFilter;while(h){(!c||(e=S.exec(h)))&&(e&&(h=h.slice(e[0].length)||h),i.push(f=[])),c=!1,(e=T.exec(h))&&(c=e.shift(),f.push({value:c,type:e[0].replace(R," ")}),h=h.slice(c.length));for(g in d.filter)!(e=X[g].exec(h))||j[g]&&!(e=j[g](e))||(c=e.shift(),f.push({value:c,type:g,matches:e}),h=h.slice(c.length));if(!c)break}return b?h.length:h?fb.error(a):z(a,i).slice(0)};function qb(a){for(var b=0,c=a.length,d="";c>b;b++)d+=a[b].value;return d}function rb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[u]||(b[u]={}),(h=i[d])&&h[0]===w&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function sb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function tb(a,b,c){for(var d=0,e=b.length;e>d;d++)fb(a,b[d],c);return c}function ub(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function vb(a,b,c,d,e,f){return d&&!d[u]&&(d=vb(d)),e&&!e[u]&&(e=vb(e,f)),hb(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||tb(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:ub(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=ub(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?K.call(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=ub(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):I.apply(g,r)})}function wb(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=rb(function(a){return a===b},h,!0),l=rb(function(a){return K.call(b,a)>-1},h,!0),m=[function(a,c,d){return!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d))}];f>i;i++)if(c=d.relative[a[i].type])m=[rb(sb(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return vb(i>1&&sb(m),i>1&&qb(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(R,"$1"),c,e>i&&wb(a.slice(i,e)),f>e&&wb(a=a.slice(e)),f>e&&qb(a))}m.push(c)}return sb(m)}function xb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,m,o,p=0,q="0",r=f&&[],s=[],t=j,u=f||e&&d.find.TAG("*",k),v=w+=null==t?1:Math.random()||.1,x=u.length;for(k&&(j=g!==n&&g);q!==x&&null!=(l=u[q]);q++){if(e&&l){m=0;while(o=a[m++])if(o(l,g,h)){i.push(l);break}k&&(w=v)}c&&((l=!o&&l)&&p--,f&&r.push(l))}if(p+=q,c&&q!==p){m=0;while(o=b[m++])o(r,s,g,h);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=G.call(i));s=ub(s)}I.apply(i,s),k&&!f&&s.length>0&&p+b.length>1&&fb.uniqueSort(i)}return k&&(w=v,j=t),r};return c?hb(f):f}return h=fb.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=wb(b[c]),f[u]?d.push(f):e.push(f);f=A(a,xb(e,d)),f.selector=a}return f},i=fb.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(cb,db),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=X.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(cb,db),ab.test(j[0].type)&&ob(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&qb(j),!a)return I.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,ab.test(a)&&ob(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=ib(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),ib(function(a){return a.innerHTML="<a href='#'></a>","#"===a.firstChild.getAttribute("href")})||jb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&ib(function(a){return a.innerHTML="<input/>",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||jb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),ib(function(a){return null==a.getAttribute("disabled")})||jb(L,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),fb}(a);m.find=s,m.expr=s.selectors,m.expr[":"]=m.expr.pseudos,m.unique=s.uniqueSort,m.text=s.getText,m.isXMLDoc=s.isXML,m.contains=s.contains;var t=m.expr.match.needsContext,u=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,v=/^.[^:#\[\.,]*$/;function w(a,b,c){if(m.isFunction(b))return m.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return m.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(v.test(b))return m.filter(b,a,c);b=m.filter(b,a)}return m.grep(a,function(a){return m.inArray(a,b)>=0!==c})}m.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?m.find.matchesSelector(d,a)?[d]:[]:m.find.matches(a,m.grep(b,function(a){return 1===a.nodeType}))},m.fn.extend({find:function(a){var b,c=[],d=this,e=d.length;if("string"!=typeof a)return this.pushStack(m(a).filter(function(){for(b=0;e>b;b++)if(m.contains(d[b],this))return!0}));for(b=0;e>b;b++)m.find(a,d[b],c);return c=this.pushStack(e>1?m.unique(c):c),c.selector=this.selector?this.selector+" "+a:a,c},filter:function(a){return this.pushStack(w(this,a||[],!1))},not:function(a){return this.pushStack(w(this,a||[],!0))},is:function(a){return!!w(this,"string"==typeof a&&t.test(a)?m(a):a||[],!1).length}});var x,y=a.document,z=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,A=m.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a.charAt(0)&&">"===a.charAt(a.length-1)&&a.length>=3?[null,a,null]:z.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||x).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof m?b[0]:b,m.merge(this,m.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:y,!0)),u.test(c[1])&&m.isPlainObject(b))for(c in b)m.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}if(d=y.getElementById(c[2]),d&&d.parentNode){if(d.id!==c[2])return x.find(a);this.length=1,this[0]=d}return this.context=y,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):m.isFunction(a)?"undefined"!=typeof x.ready?x.ready(a):a(m):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),m.makeArray(a,this))};A.prototype=m.fn,x=m(y);var B=/^(?:parents|prev(?:Until|All))/,C={children:!0,contents:!0,next:!0,prev:!0};m.extend({dir:function(a,b,c){var d=[],e=a[b];while(e&&9!==e.nodeType&&(void 0===c||1!==e.nodeType||!m(e).is(c)))1===e.nodeType&&d.push(e),e=e[b];return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),m.fn.extend({has:function(a){var b,c=m(a,this),d=c.length;return this.filter(function(){for(b=0;d>b;b++)if(m.contains(this,c[b]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=t.test(a)||"string"!=typeof a?m(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&m.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?m.unique(f):f)},index:function(a){return a?"string"==typeof a?m.inArray(this[0],m(a)):m.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(m.unique(m.merge(this.get(),m(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function D(a,b){do a=a[b];while(a&&1!==a.nodeType);return a}m.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return m.dir(a,"parentNode")},parentsUntil:function(a,b,c){return m.dir(a,"parentNode",c)},next:function(a){return D(a,"nextSibling")},prev:function(a){return D(a,"previousSibling")},nextAll:function(a){return m.dir(a,"nextSibling")},prevAll:function(a){return m.dir(a,"previousSibling")},nextUntil:function(a,b,c){return m.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return m.dir(a,"previousSibling",c)},siblings:function(a){return m.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return m.sibling(a.firstChild)},contents:function(a){return m.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:m.merge([],a.childNodes)}},function(a,b){m.fn[a]=function(c,d){var e=m.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=m.filter(d,e)),this.length>1&&(C[a]||(e=m.unique(e)),B.test(a)&&(e=e.reverse())),this.pushStack(e)}});var E=/\S+/g,F={};function G(a){var b=F[a]={};return m.each(a.match(E)||[],function(a,c){b[c]=!0}),b}m.Callbacks=function(a){a="string"==typeof a?F[a]||G(a):m.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(c=a.memory&&l,d=!0,f=g||0,g=0,e=h.length,b=!0;h&&e>f;f++)if(h[f].apply(l[0],l[1])===!1&&a.stopOnFalse){c=!1;break}b=!1,h&&(i?i.length&&j(i.shift()):c?h=[]:k.disable())},k={add:function(){if(h){var d=h.length;!function f(b){m.each(b,function(b,c){var d=m.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&f(c)})}(arguments),b?e=h.length:c&&(g=d,j(c))}return this},remove:function(){return h&&m.each(arguments,function(a,c){var d;while((d=m.inArray(c,h,d))>-1)h.splice(d,1),b&&(e>=d&&e--,f>=d&&f--)}),this},has:function(a){return a?m.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],e=0,this},disable:function(){return h=i=c=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,c||k.disable(),this},locked:function(){return!i},fireWith:function(a,c){return!h||d&&!i||(c=c||[],c=[a,c.slice?c.slice():c],b?i.push(c):j(c)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!d}};return k},m.extend({Deferred:function(a){var b=[["resolve","done",m.Callbacks("once memory"),"resolved"],["reject","fail",m.Callbacks("once memory"),"rejected"],["notify","progress",m.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return m.Deferred(function(c){m.each(b,function(b,f){var g=m.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&m.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?m.extend(a,d):d}},e={};return d.pipe=d.then,m.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&m.isFunction(a.promise)?e:0,g=1===f?a:m.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&m.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var H;m.fn.ready=function(a){return m.ready.promise().done(a),this},m.extend({isReady:!1,readyWait:1,holdReady:function(a){a?m.readyWait++:m.ready(!0)},ready:function(a){if(a===!0?!--m.readyWait:!m.isReady){if(!y.body)return setTimeout(m.ready);m.isReady=!0,a!==!0&&--m.readyWait>0||(H.resolveWith(y,[m]),m.fn.triggerHandler&&(m(y).triggerHandler("ready"),m(y).off("ready")))}}});function I(){y.addEventListener?(y.removeEventListener("DOMContentLoaded",J,!1),a.removeEventListener("load",J,!1)):(y.detachEvent("onreadystatechange",J),a.detachEvent("onload",J))}function J(){(y.addEventListener||"load"===event.type||"complete"===y.readyState)&&(I(),m.ready())}m.ready.promise=function(b){if(!H)if(H=m.Deferred(),"complete"===y.readyState)setTimeout(m.ready);else if(y.addEventListener)y.addEventListener("DOMContentLoaded",J,!1),a.addEventListener("load",J,!1);else{y.attachEvent("onreadystatechange",J),a.attachEvent("onload",J);var c=!1;try{c=null==a.frameElement&&y.documentElement}catch(d){}c&&c.doScroll&&!function e(){if(!m.isReady){try{c.doScroll("left")}catch(a){return setTimeout(e,50)}I(),m.ready()}}()}return H.promise(b)};var K="undefined",L;for(L in m(k))break;k.ownLast="0"!==L,k.inlineBlockNeedsLayout=!1,m(function(){var a,b,c,d;c=y.getElementsByTagName("body")[0],c&&c.style&&(b=y.createElement("div"),d=y.createElement("div"),d.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(d).appendChild(b),typeof b.style.zoom!==K&&(b.style.cssText="display:inline;margin:0;border:0;padding:1px;width:1px;zoom:1",k.inlineBlockNeedsLayout=a=3===b.offsetWidth,a&&(c.style.zoom=1)),c.removeChild(d))}),function(){var a=y.createElement("div");if(null==k.deleteExpando){k.deleteExpando=!0;try{delete a.test}catch(b){k.deleteExpando=!1}}a=null}(),m.acceptData=function(a){var b=m.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b};var M=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,N=/([A-Z])/g;function O(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(N,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:M.test(c)?m.parseJSON(c):c}catch(e){}m.data(a,b,c)}else c=void 0}return c}function P(a){var b;for(b in a)if(("data"!==b||!m.isEmptyObject(a[b]))&&"toJSON"!==b)return!1;return!0}function Q(a,b,d,e){if(m.acceptData(a)){var f,g,h=m.expando,i=a.nodeType,j=i?m.cache:a,k=i?a[h]:a[h]&&h;
+if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||m.guid++:h),j[k]||(j[k]=i?{}:{toJSON:m.noop}),("object"==typeof b||"function"==typeof b)&&(e?j[k]=m.extend(j[k],b):j[k].data=m.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[m.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[m.camelCase(b)])):f=g,f}}function R(a,b,c){if(m.acceptData(a)){var d,e,f=a.nodeType,g=f?m.cache:a,h=f?a[m.expando]:m.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){m.isArray(b)?b=b.concat(m.map(b,m.camelCase)):b in d?b=[b]:(b=m.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!P(d):!m.isEmptyObject(d))return}(c||(delete g[h].data,P(g[h])))&&(f?m.cleanData([a],!0):k.deleteExpando||g!=g.window?delete g[h]:g[h]=null)}}}m.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?m.cache[a[m.expando]]:a[m.expando],!!a&&!P(a)},data:function(a,b,c){return Q(a,b,c)},removeData:function(a,b){return R(a,b)},_data:function(a,b,c){return Q(a,b,c,!0)},_removeData:function(a,b){return R(a,b,!0)}}),m.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=m.data(f),1===f.nodeType&&!m._data(f,"parsedAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=m.camelCase(d.slice(5)),O(f,d,e[d])));m._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){m.data(this,a)}):arguments.length>1?this.each(function(){m.data(this,a,b)}):f?O(f,a,m.data(f,a)):void 0},removeData:function(a){return this.each(function(){m.removeData(this,a)})}}),m.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=m._data(a,b),c&&(!d||m.isArray(c)?d=m._data(a,b,m.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=m.queue(a,b),d=c.length,e=c.shift(),f=m._queueHooks(a,b),g=function(){m.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return m._data(a,c)||m._data(a,c,{empty:m.Callbacks("once memory").add(function(){m._removeData(a,b+"queue"),m._removeData(a,c)})})}}),m.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.length<c?m.queue(this[0],a):void 0===b?this:this.each(function(){var c=m.queue(this,a,b);m._queueHooks(this,a),"fx"===a&&"inprogress"!==c[0]&&m.dequeue(this,a)})},dequeue:function(a){return this.each(function(){m.dequeue(this,a)})},clearQueue:function(a){return this.queue(a||"fx",[])},promise:function(a,b){var c,d=1,e=m.Deferred(),f=this,g=this.length,h=function(){--d||e.resolveWith(f,[f])};"string"!=typeof a&&(b=a,a=void 0),a=a||"fx";while(g--)c=m._data(f[g],a+"queueHooks"),c&&c.empty&&(d++,c.empty.add(h));return h(),e.promise(b)}});var S=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,T=["Top","Right","Bottom","Left"],U=function(a,b){return a=b||a,"none"===m.css(a,"display")||!m.contains(a.ownerDocument,a)},V=m.access=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===m.type(c)){e=!0;for(h in c)m.access(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,m.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(m(a),c)})),b))for(;i>h;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},W=/^(?:checkbox|radio)$/i;!function(){var a=y.createElement("input"),b=y.createElement("div"),c=y.createDocumentFragment();if(b.innerHTML="  <link/><table></table><a href='/a'>a</a><input type='checkbox'/>",k.leadingWhitespace=3===b.firstChild.nodeType,k.tbody=!b.getElementsByTagName("tbody").length,k.htmlSerialize=!!b.getElementsByTagName("link").length,k.html5Clone="<:nav></:nav>"!==y.createElement("nav").cloneNode(!0).outerHTML,a.type="checkbox",a.checked=!0,c.appendChild(a),k.appendChecked=a.checked,b.innerHTML="<textarea>x</textarea>",k.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue,c.appendChild(b),b.innerHTML="<input type='radio' checked='checked' name='t'/>",k.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,k.noCloneEvent=!0,b.attachEvent&&(b.attachEvent("onclick",function(){k.noCloneEvent=!1}),b.cloneNode(!0).click()),null==k.deleteExpando){k.deleteExpando=!0;try{delete b.test}catch(d){k.deleteExpando=!1}}}(),function(){var b,c,d=y.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(k[b+"Bubbles"]=c in a)||(d.setAttribute(c,"t"),k[b+"Bubbles"]=d.attributes[c].expando===!1);d=null}();var X=/^(?:input|select|textarea)$/i,Y=/^key/,Z=/^(?:mouse|pointer|contextmenu)|click/,$=/^(?:focusinfocus|focusoutblur)$/,_=/^([^.]*)(?:\.(.+)|)$/;function ab(){return!0}function bb(){return!1}function cb(){try{return y.activeElement}catch(a){}}m.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=m.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return typeof m===K||a&&m.event.triggered===a.type?void 0:m.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(E)||[""],h=b.length;while(h--)f=_.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=m.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=m.event.special[o]||{},l=m.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&m.expr.match.needsContext.test(e),namespace:p.join(".")},i),(n=g[o])||(n=g[o]=[],n.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?n.splice(n.delegateCount++,0,l):n.push(l),m.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m.hasData(a)&&m._data(a);if(r&&(k=r.events)){b=(b||"").match(E)||[""],j=b.length;while(j--)if(h=_.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=m.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,n=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=n.length;while(f--)g=n[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(n.splice(f,1),g.selector&&n.delegateCount--,l.remove&&l.remove.call(a,g));i&&!n.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||m.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)m.event.remove(a,o+b[j],c,d,!0);m.isEmptyObject(k)&&(delete r.handle,m._removeData(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,l,n,o=[d||y],p=j.call(b,"type")?b.type:b,q=j.call(b,"namespace")?b.namespace.split("."):[];if(h=l=d=d||y,3!==d.nodeType&&8!==d.nodeType&&!$.test(p+m.event.triggered)&&(p.indexOf(".")>=0&&(q=p.split("."),p=q.shift(),q.sort()),g=p.indexOf(":")<0&&"on"+p,b=b[m.expando]?b:new m.Event(p,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=q.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:m.makeArray(c,[b]),k=m.event.special[p]||{},e||!k.trigger||k.trigger.apply(d,c)!==!1)){if(!e&&!k.noBubble&&!m.isWindow(d)){for(i=k.delegateType||p,$.test(i+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),l=h;l===(d.ownerDocument||y)&&o.push(l.defaultView||l.parentWindow||a)}n=0;while((h=o[n++])&&!b.isPropagationStopped())b.type=n>1?i:k.bindType||p,f=(m._data(h,"events")||{})[b.type]&&m._data(h,"handle"),f&&f.apply(h,c),f=g&&h[g],f&&f.apply&&m.acceptData(h)&&(b.result=f.apply(h,c),b.result===!1&&b.preventDefault());if(b.type=p,!e&&!b.isDefaultPrevented()&&(!k._default||k._default.apply(o.pop(),c)===!1)&&m.acceptData(d)&&g&&d[p]&&!m.isWindow(d)){l=d[g],l&&(d[g]=null),m.event.triggered=p;try{d[p]()}catch(r){}m.event.triggered=void 0,l&&(d[g]=l)}return b.result}},dispatch:function(a){a=m.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(m._data(this,"events")||{})[a.type]||[],k=m.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=m.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,g=0;while((e=f.handlers[g++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(e.namespace))&&(a.handleObj=e,a.data=e.data,c=((m.event.special[e.origType]||{}).handle||e.handler).apply(f.elem,i),void 0!==c&&(a.result=c)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(e=[],f=0;h>f;f++)d=b[f],c=d.selector+" ",void 0===e[c]&&(e[c]=d.needsContext?m(c,this).index(i)>=0:m.find(c,this,null,[i]).length),e[c]&&e.push(d);e.length&&g.push({elem:i,handlers:e})}return h<b.length&&g.push({elem:this,handlers:b.slice(h)}),g},fix:function(a){if(a[m.expando])return a;var b,c,d,e=a.type,f=a,g=this.fixHooks[e];g||(this.fixHooks[e]=g=Z.test(e)?this.mouseHooks:Y.test(e)?this.keyHooks:{}),d=g.props?this.props.concat(g.props):this.props,a=new m.Event(f),b=d.length;while(b--)c=d[b],a[c]=f[c];return a.target||(a.target=f.srcElement||y),3===a.target.nodeType&&(a.target=a.target.parentNode),a.metaKey=!!a.metaKey,g.filter?g.filter(a,f):a},props:"altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),fixHooks:{},keyHooks:{props:"char charCode key keyCode".split(" "),filter:function(a,b){return null==a.which&&(a.which=null!=b.charCode?b.charCode:b.keyCode),a}},mouseHooks:{props:"button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),filter:function(a,b){var c,d,e,f=b.button,g=b.fromElement;return null==a.pageX&&null!=b.clientX&&(d=a.target.ownerDocument||y,e=d.documentElement,c=d.body,a.pageX=b.clientX+(e&&e.scrollLeft||c&&c.scrollLeft||0)-(e&&e.clientLeft||c&&c.clientLeft||0),a.pageY=b.clientY+(e&&e.scrollTop||c&&c.scrollTop||0)-(e&&e.clientTop||c&&c.clientTop||0)),!a.relatedTarget&&g&&(a.relatedTarget=g===a.target?b.toElement:g),a.which||void 0===f||(a.which=1&f?1:2&f?3:4&f?2:0),a}},special:{load:{noBubble:!0},focus:{trigger:function(){if(this!==cb()&&this.focus)try{return this.focus(),!1}catch(a){}},delegateType:"focusin"},blur:{trigger:function(){return this===cb()&&this.blur?(this.blur(),!1):void 0},delegateType:"focusout"},click:{trigger:function(){return m.nodeName(this,"input")&&"checkbox"===this.type&&this.click?(this.click(),!1):void 0},_default:function(a){return m.nodeName(a.target,"a")}},beforeunload:{postDispatch:function(a){void 0!==a.result&&a.originalEvent&&(a.originalEvent.returnValue=a.result)}}},simulate:function(a,b,c,d){var e=m.extend(new m.Event,c,{type:a,isSimulated:!0,originalEvent:{}});d?m.event.trigger(e,null,b):m.event.dispatch.call(b,e),e.isDefaultPrevented()&&c.preventDefault()}},m.removeEvent=y.removeEventListener?function(a,b,c){a.removeEventListener&&a.removeEventListener(b,c,!1)}:function(a,b,c){var d="on"+b;a.detachEvent&&(typeof a[d]===K&&(a[d]=null),a.detachEvent(d,c))},m.Event=function(a,b){return this instanceof m.Event?(a&&a.type?(this.originalEvent=a,this.type=a.type,this.isDefaultPrevented=a.defaultPrevented||void 0===a.defaultPrevented&&a.returnValue===!1?ab:bb):this.type=a,b&&m.extend(this,b),this.timeStamp=a&&a.timeStamp||m.now(),void(this[m.expando]=!0)):new m.Event(a,b)},m.Event.prototype={isDefaultPrevented:bb,isPropagationStopped:bb,isImmediatePropagationStopped:bb,preventDefault:function(){var a=this.originalEvent;this.isDefaultPrevented=ab,a&&(a.preventDefault?a.preventDefault():a.returnValue=!1)},stopPropagation:function(){var a=this.originalEvent;this.isPropagationStopped=ab,a&&(a.stopPropagation&&a.stopPropagation(),a.cancelBubble=!0)},stopImmediatePropagation:function(){var a=this.originalEvent;this.isImmediatePropagationStopped=ab,a&&a.stopImmediatePropagation&&a.stopImmediatePropagation(),this.stopPropagation()}},m.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(a,b){m.event.special[a]={delegateType:b,bindType:b,handle:function(a){var c,d=this,e=a.relatedTarget,f=a.handleObj;return(!e||e!==d&&!m.contains(d,e))&&(a.type=f.origType,c=f.handler.apply(this,arguments),a.type=b),c}}}),k.submitBubbles||(m.event.special.submit={setup:function(){return m.nodeName(this,"form")?!1:void m.event.add(this,"click._submit keypress._submit",function(a){var b=a.target,c=m.nodeName(b,"input")||m.nodeName(b,"button")?b.form:void 0;c&&!m._data(c,"submitBubbles")&&(m.event.add(c,"submit._submit",function(a){a._submit_bubble=!0}),m._data(c,"submitBubbles",!0))})},postDispatch:function(a){a._submit_bubble&&(delete a._submit_bubble,this.parentNode&&!a.isTrigger&&m.event.simulate("submit",this.parentNode,a,!0))},teardown:function(){return m.nodeName(this,"form")?!1:void m.event.remove(this,"._submit")}}),k.changeBubbles||(m.event.special.change={setup:function(){return X.test(this.nodeName)?(("checkbox"===this.type||"radio"===this.type)&&(m.event.add(this,"propertychange._change",function(a){"checked"===a.originalEvent.propertyName&&(this._just_changed=!0)}),m.event.add(this,"click._change",function(a){this._just_changed&&!a.isTrigger&&(this._just_changed=!1),m.event.simulate("change",this,a,!0)})),!1):void m.event.add(this,"beforeactivate._change",function(a){var b=a.target;X.test(b.nodeName)&&!m._data(b,"changeBubbles")&&(m.event.add(b,"change._change",function(a){!this.parentNode||a.isSimulated||a.isTrigger||m.event.simulate("change",this.parentNode,a,!0)}),m._data(b,"changeBubbles",!0))})},handle:function(a){var b=a.target;return this!==b||a.isSimulated||a.isTrigger||"radio"!==b.type&&"checkbox"!==b.type?a.handleObj.handler.apply(this,arguments):void 0},teardown:function(){return m.event.remove(this,"._change"),!X.test(this.nodeName)}}),k.focusinBubbles||m.each({focus:"focusin",blur:"focusout"},function(a,b){var c=function(a){m.event.simulate(b,a.target,m.event.fix(a),!0)};m.event.special[b]={setup:function(){var d=this.ownerDocument||this,e=m._data(d,b);e||d.addEventListener(a,c,!0),m._data(d,b,(e||0)+1)},teardown:function(){var d=this.ownerDocument||this,e=m._data(d,b)-1;e?m._data(d,b,e):(d.removeEventListener(a,c,!0),m._removeData(d,b))}}}),m.fn.extend({on:function(a,b,c,d,e){var f,g;if("object"==typeof a){"string"!=typeof b&&(c=c||b,b=void 0);for(f in a)this.on(f,b,c,a[f],e);return this}if(null==c&&null==d?(d=b,c=b=void 0):null==d&&("string"==typeof b?(d=c,c=void 0):(d=c,c=b,b=void 0)),d===!1)d=bb;else if(!d)return this;return 1===e&&(g=d,d=function(a){return m().off(a),g.apply(this,arguments)},d.guid=g.guid||(g.guid=m.guid++)),this.each(function(){m.event.add(this,a,d,c,b)})},one:function(a,b,c,d){return this.on(a,b,c,d,1)},off:function(a,b,c){var d,e;if(a&&a.preventDefault&&a.handleObj)return d=a.handleObj,m(a.delegateTarget).off(d.namespace?d.origType+"."+d.namespace:d.origType,d.selector,d.handler),this;if("object"==typeof a){for(e in a)this.off(e,b,a[e]);return this}return(b===!1||"function"==typeof b)&&(c=b,b=void 0),c===!1&&(c=bb),this.each(function(){m.event.remove(this,a,c,b)})},trigger:function(a,b){return this.each(function(){m.event.trigger(a,b,this)})},triggerHandler:function(a,b){var c=this[0];return c?m.event.trigger(a,b,c,!0):void 0}});function db(a){var b=eb.split("|"),c=a.createDocumentFragment();if(c.createElement)while(b.length)c.createElement(b.pop());return c}var eb="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",fb=/ jQuery\d+="(?:null|\d+)"/g,gb=new RegExp("<(?:"+eb+")[\\s/>]","i"),hb=/^\s+/,ib=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,jb=/<([\w:]+)/,kb=/<tbody/i,lb=/<|&#?\w+;/,mb=/<(?:script|style|link)/i,nb=/checked\s*(?:[^=]|=\s*.checked.)/i,ob=/^$|\/(?:java|ecma)script/i,pb=/^true\/(.*)/,qb=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g,rb={option:[1,"<select multiple='multiple'>","</select>"],legend:[1,"<fieldset>","</fieldset>"],area:[1,"<map>","</map>"],param:[1,"<object>","</object>"],thead:[1,"<table>","</table>"],tr:[2,"<table><tbody>","</tbody></table>"],col:[2,"<table><tbody></tbody><colgroup>","</colgroup></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:k.htmlSerialize?[0,"",""]:[1,"X<div>","</div>"]},sb=db(y),tb=sb.appendChild(y.createElement("div"));rb.optgroup=rb.option,rb.tbody=rb.tfoot=rb.colgroup=rb.caption=rb.thead,rb.th=rb.td;function ub(a,b){var c,d,e=0,f=typeof a.getElementsByTagName!==K?a.getElementsByTagName(b||"*"):typeof a.querySelectorAll!==K?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||m.nodeName(d,b)?f.push(d):m.merge(f,ub(d,b));return void 0===b||b&&m.nodeName(a,b)?m.merge([a],f):f}function vb(a){W.test(a.type)&&(a.defaultChecked=a.checked)}function wb(a,b){return m.nodeName(a,"table")&&m.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function xb(a){return a.type=(null!==m.find.attr(a,"type"))+"/"+a.type,a}function yb(a){var b=pb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function zb(a,b){for(var c,d=0;null!=(c=a[d]);d++)m._data(c,"globalEval",!b||m._data(b[d],"globalEval"))}function Ab(a,b){if(1===b.nodeType&&m.hasData(a)){var c,d,e,f=m._data(a),g=m._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)m.event.add(b,c,h[c][d])}g.data&&(g.data=m.extend({},g.data))}}function Bb(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!k.noCloneEvent&&b[m.expando]){e=m._data(b);for(d in e.events)m.removeEvent(b,d,e.handle);b.removeAttribute(m.expando)}"script"===c&&b.text!==a.text?(xb(b).text=a.text,yb(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),k.html5Clone&&a.innerHTML&&!m.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&W.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}}m.extend({clone:function(a,b,c){var d,e,f,g,h,i=m.contains(a.ownerDocument,a);if(k.html5Clone||m.isXMLDoc(a)||!gb.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(tb.innerHTML=a.outerHTML,tb.removeChild(f=tb.firstChild)),!(k.noCloneEvent&&k.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||m.isXMLDoc(a)))for(d=ub(f),h=ub(a),g=0;null!=(e=h[g]);++g)d[g]&&Bb(e,d[g]);if(b)if(c)for(h=h||ub(a),d=d||ub(f),g=0;null!=(e=h[g]);g++)Ab(e,d[g]);else Ab(a,f);return d=ub(f,"script"),d.length>0&&zb(d,!i&&ub(a,"script")),d=h=e=null,f},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,l,n=a.length,o=db(b),p=[],q=0;n>q;q++)if(f=a[q],f||0===f)if("object"===m.type(f))m.merge(p,f.nodeType?[f]:f);else if(lb.test(f)){h=h||o.appendChild(b.createElement("div")),i=(jb.exec(f)||["",""])[1].toLowerCase(),l=rb[i]||rb._default,h.innerHTML=l[1]+f.replace(ib,"<$1></$2>")+l[2],e=l[0];while(e--)h=h.lastChild;if(!k.leadingWhitespace&&hb.test(f)&&p.push(b.createTextNode(hb.exec(f)[0])),!k.tbody){f="table"!==i||kb.test(f)?"<table>"!==l[1]||kb.test(f)?0:h:h.firstChild,e=f&&f.childNodes.length;while(e--)m.nodeName(j=f.childNodes[e],"tbody")&&!j.childNodes.length&&f.removeChild(j)}m.merge(p,h.childNodes),h.textContent="";while(h.firstChild)h.removeChild(h.firstChild);h=o.lastChild}else p.push(b.createTextNode(f));h&&o.removeChild(h),k.appendChecked||m.grep(ub(p,"input"),vb),q=0;while(f=p[q++])if((!d||-1===m.inArray(f,d))&&(g=m.contains(f.ownerDocument,f),h=ub(o.appendChild(f),"script"),g&&zb(h),c)){e=0;while(f=h[e++])ob.test(f.type||"")&&c.push(f)}return h=null,o},cleanData:function(a,b){for(var d,e,f,g,h=0,i=m.expando,j=m.cache,l=k.deleteExpando,n=m.event.special;null!=(d=a[h]);h++)if((b||m.acceptData(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)n[e]?m.event.remove(d,e):m.removeEvent(d,e,g.handle);j[f]&&(delete j[f],l?delete d[i]:typeof d.removeAttribute!==K?d.removeAttribute(i):d[i]=null,c.push(f))}}}),m.fn.extend({text:function(a){return V(this,function(a){return void 0===a?m.text(this):this.empty().append((this[0]&&this[0].ownerDocument||y).createTextNode(a))},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?m.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||m.cleanData(ub(c)),c.parentNode&&(b&&m.contains(c.ownerDocument,c)&&zb(ub(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&m.cleanData(ub(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&m.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return m.clone(this,a,b)})},html:function(a){return V(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(fb,""):void 0;if(!("string"!=typeof a||mb.test(a)||!k.htmlSerialize&&gb.test(a)||!k.leadingWhitespace&&hb.test(a)||rb[(jb.exec(a)||["",""])[1].toLowerCase()])){a=a.replace(ib,"<$1></$2>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(m.cleanData(ub(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,m.cleanData(ub(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,l=this.length,n=this,o=l-1,p=a[0],q=m.isFunction(p);if(q||l>1&&"string"==typeof p&&!k.checkClone&&nb.test(p))return this.each(function(c){var d=n.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(l&&(i=m.buildFragment(a,this[0].ownerDocument,!1,this),c=i.firstChild,1===i.childNodes.length&&(i=c),c)){for(g=m.map(ub(i,"script"),xb),f=g.length;l>j;j++)d=i,j!==o&&(d=m.clone(d,!0,!0),f&&m.merge(g,ub(d,"script"))),b.call(this[j],d,j);if(f)for(h=g[g.length-1].ownerDocument,m.map(g,yb),j=0;f>j;j++)d=g[j],ob.test(d.type||"")&&!m._data(d,"globalEval")&&m.contains(h,d)&&(d.src?m._evalUrl&&m._evalUrl(d.src):m.globalEval((d.text||d.textContent||d.innerHTML||"").replace(qb,"")));i=c=null}return this}}),m.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){m.fn[a]=function(a){for(var c,d=0,e=[],g=m(a),h=g.length-1;h>=d;d++)c=d===h?this:this.clone(!0),m(g[d])[b](c),f.apply(e,c.get());return this.pushStack(e)}});var Cb,Db={};function Eb(b,c){var d,e=m(c.createElement(b)).appendTo(c.body),f=a.getDefaultComputedStyle&&(d=a.getDefaultComputedStyle(e[0]))?d.display:m.css(e[0],"display");return e.detach(),f}function Fb(a){var b=y,c=Db[a];return c||(c=Eb(a,b),"none"!==c&&c||(Cb=(Cb||m("<iframe frameborder='0' width='0' height='0'/>")).appendTo(b.documentElement),b=(Cb[0].contentWindow||Cb[0].contentDocument).document,b.write(),b.close(),c=Eb(a,b),Cb.detach()),Db[a]=c),c}!function(){var a;k.shrinkWrapBlocks=function(){if(null!=a)return a;a=!1;var b,c,d;return c=y.getElementsByTagName("body")[0],c&&c.style?(b=y.createElement("div"),d=y.createElement("div"),d.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(d).appendChild(b),typeof b.style.zoom!==K&&(b.style.cssText="-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;display:block;margin:0;border:0;padding:1px;width:1px;zoom:1",b.appendChild(y.createElement("div")).style.width="5px",a=3!==b.offsetWidth),c.removeChild(d),a):void 0}}();var Gb=/^margin/,Hb=new RegExp("^("+S+")(?!px)[a-z%]+$","i"),Ib,Jb,Kb=/^(top|right|bottom|left)$/;a.getComputedStyle?(Ib=function(a){return a.ownerDocument.defaultView.getComputedStyle(a,null)},Jb=function(a,b,c){var d,e,f,g,h=a.style;return c=c||Ib(a),g=c?c.getPropertyValue(b)||c[b]:void 0,c&&(""!==g||m.contains(a.ownerDocument,a)||(g=m.style(a,b)),Hb.test(g)&&Gb.test(b)&&(d=h.width,e=h.minWidth,f=h.maxWidth,h.minWidth=h.maxWidth=h.width=g,g=c.width,h.width=d,h.minWidth=e,h.maxWidth=f)),void 0===g?g:g+""}):y.documentElement.currentStyle&&(Ib=function(a){return a.currentStyle},Jb=function(a,b,c){var d,e,f,g,h=a.style;return c=c||Ib(a),g=c?c[b]:void 0,null==g&&h&&h[b]&&(g=h[b]),Hb.test(g)&&!Kb.test(b)&&(d=h.left,e=a.runtimeStyle,f=e&&e.left,f&&(e.left=a.currentStyle.left),h.left="fontSize"===b?"1em":g,g=h.pixelLeft+"px",h.left=d,f&&(e.left=f)),void 0===g?g:g+""||"auto"});function Lb(a,b){return{get:function(){var c=a();if(null!=c)return c?void delete this.get:(this.get=b).apply(this,arguments)}}}!function(){var b,c,d,e,f,g,h;if(b=y.createElement("div"),b.innerHTML="  <link/><table></table><a href='/a'>a</a><input type='checkbox'/>",d=b.getElementsByTagName("a")[0],c=d&&d.style){c.cssText="float:left;opacity:.5",k.opacity="0.5"===c.opacity,k.cssFloat=!!c.cssFloat,b.style.backgroundClip="content-box",b.cloneNode(!0).style.backgroundClip="",k.clearCloneStyle="content-box"===b.style.backgroundClip,k.boxSizing=""===c.boxSizing||""===c.MozBoxSizing||""===c.WebkitBoxSizing,m.extend(k,{reliableHiddenOffsets:function(){return null==g&&i(),g},boxSizingReliable:function(){return null==f&&i(),f},pixelPosition:function(){return null==e&&i(),e},reliableMarginRight:function(){return null==h&&i(),h}});function i(){var b,c,d,i;c=y.getElementsByTagName("body")[0],c&&c.style&&(b=y.createElement("div"),d=y.createElement("div"),d.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(d).appendChild(b),b.style.cssText="-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;display:block;margin-top:1%;top:1%;border:1px;padding:1px;width:4px;position:absolute",e=f=!1,h=!0,a.getComputedStyle&&(e="1%"!==(a.getComputedStyle(b,null)||{}).top,f="4px"===(a.getComputedStyle(b,null)||{width:"4px"}).width,i=b.appendChild(y.createElement("div")),i.style.cssText=b.style.cssText="-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;display:block;margin:0;border:0;padding:0",i.style.marginRight=i.style.width="0",b.style.width="1px",h=!parseFloat((a.getComputedStyle(i,null)||{}).marginRight)),b.innerHTML="<table><tr><td></td><td>t</td></tr></table>",i=b.getElementsByTagName("td"),i[0].style.cssText="margin:0;border:0;padding:0;display:none",g=0===i[0].offsetHeight,g&&(i[0].style.display="",i[1].style.display="none",g=0===i[0].offsetHeight),c.removeChild(d))}}}(),m.swap=function(a,b,c,d){var e,f,g={};for(f in b)g[f]=a.style[f],a.style[f]=b[f];e=c.apply(a,d||[]);for(f in b)a.style[f]=g[f];return e};var Mb=/alpha\([^)]*\)/i,Nb=/opacity\s*=\s*([^)]*)/,Ob=/^(none|table(?!-c[ea]).+)/,Pb=new RegExp("^("+S+")(.*)$","i"),Qb=new RegExp("^([+-])=("+S+")","i"),Rb={position:"absolute",visibility:"hidden",display:"block"},Sb={letterSpacing:"0",fontWeight:"400"},Tb=["Webkit","O","Moz","ms"];function Ub(a,b){if(b in a)return b;var c=b.charAt(0).toUpperCase()+b.slice(1),d=b,e=Tb.length;while(e--)if(b=Tb[e]+c,b in a)return b;return d}function Vb(a,b){for(var c,d,e,f=[],g=0,h=a.length;h>g;g++)d=a[g],d.style&&(f[g]=m._data(d,"olddisplay"),c=d.style.display,b?(f[g]||"none"!==c||(d.style.display=""),""===d.style.display&&U(d)&&(f[g]=m._data(d,"olddisplay",Fb(d.nodeName)))):(e=U(d),(c&&"none"!==c||!e)&&m._data(d,"olddisplay",e?c:m.css(d,"display"))));for(g=0;h>g;g++)d=a[g],d.style&&(b&&"none"!==d.style.display&&""!==d.style.display||(d.style.display=b?f[g]||"":"none"));return a}function Wb(a,b,c){var d=Pb.exec(b);return d?Math.max(0,d[1]-(c||0))+(d[2]||"px"):b}function Xb(a,b,c,d,e){for(var f=c===(d?"border":"content")?4:"width"===b?1:0,g=0;4>f;f+=2)"margin"===c&&(g+=m.css(a,c+T[f],!0,e)),d?("content"===c&&(g-=m.css(a,"padding"+T[f],!0,e)),"margin"!==c&&(g-=m.css(a,"border"+T[f]+"Width",!0,e))):(g+=m.css(a,"padding"+T[f],!0,e),"padding"!==c&&(g+=m.css(a,"border"+T[f]+"Width",!0,e)));return g}function Yb(a,b,c){var d=!0,e="width"===b?a.offsetWidth:a.offsetHeight,f=Ib(a),g=k.boxSizing&&"border-box"===m.css(a,"boxSizing",!1,f);if(0>=e||null==e){if(e=Jb(a,b,f),(0>e||null==e)&&(e=a.style[b]),Hb.test(e))return e;d=g&&(k.boxSizingReliable()||e===a.style[b]),e=parseFloat(e)||0}return e+Xb(a,b,c||(g?"border":"content"),d,f)+"px"}m.extend({cssHooks:{opacity:{get:function(a,b){if(b){var c=Jb(a,"opacity");return""===c?"1":c}}}},cssNumber:{columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":k.cssFloat?"cssFloat":"styleFloat"},style:function(a,b,c,d){if(a&&3!==a.nodeType&&8!==a.nodeType&&a.style){var e,f,g,h=m.camelCase(b),i=a.style;if(b=m.cssProps[h]||(m.cssProps[h]=Ub(i,h)),g=m.cssHooks[b]||m.cssHooks[h],void 0===c)return g&&"get"in g&&void 0!==(e=g.get(a,!1,d))?e:i[b];if(f=typeof c,"string"===f&&(e=Qb.exec(c))&&(c=(e[1]+1)*e[2]+parseFloat(m.css(a,b)),f="number"),null!=c&&c===c&&("number"!==f||m.cssNumber[h]||(c+="px"),k.clearCloneStyle||""!==c||0!==b.indexOf("background")||(i[b]="inherit"),!(g&&"set"in g&&void 0===(c=g.set(a,c,d)))))try{i[b]=c}catch(j){}}},css:function(a,b,c,d){var e,f,g,h=m.camelCase(b);return b=m.cssProps[h]||(m.cssProps[h]=Ub(a.style,h)),g=m.cssHooks[b]||m.cssHooks[h],g&&"get"in g&&(f=g.get(a,!0,c)),void 0===f&&(f=Jb(a,b,d)),"normal"===f&&b in Sb&&(f=Sb[b]),""===c||c?(e=parseFloat(f),c===!0||m.isNumeric(e)?e||0:f):f}}),m.each(["height","width"],function(a,b){m.cssHooks[b]={get:function(a,c,d){return c?Ob.test(m.css(a,"display"))&&0===a.offsetWidth?m.swap(a,Rb,function(){return Yb(a,b,d)}):Yb(a,b,d):void 0},set:function(a,c,d){var e=d&&Ib(a);return Wb(a,c,d?Xb(a,b,d,k.boxSizing&&"border-box"===m.css(a,"boxSizing",!1,e),e):0)}}}),k.opacity||(m.cssHooks.opacity={get:function(a,b){return Nb.test((b&&a.currentStyle?a.currentStyle.filter:a.style.filter)||"")?.01*parseFloat(RegExp.$1)+"":b?"1":""},set:function(a,b){var c=a.style,d=a.currentStyle,e=m.isNumeric(b)?"alpha(opacity="+100*b+")":"",f=d&&d.filter||c.filter||"";c.zoom=1,(b>=1||""===b)&&""===m.trim(f.replace(Mb,""))&&c.removeAttribute&&(c.removeAttribute("filter"),""===b||d&&!d.filter)||(c.filter=Mb.test(f)?f.replace(Mb,e):f+" "+e)}}),m.cssHooks.marginRight=Lb(k.reliableMarginRight,function(a,b){return b?m.swap(a,{display:"inline-block"},Jb,[a,"marginRight"]):void 0}),m.each({margin:"",padding:"",border:"Width"},function(a,b){m.cssHooks[a+b]={expand:function(c){for(var d=0,e={},f="string"==typeof c?c.split(" "):[c];4>d;d++)e[a+T[d]+b]=f[d]||f[d-2]||f[0];return e}},Gb.test(a)||(m.cssHooks[a+b].set=Wb)}),m.fn.extend({css:function(a,b){return V(this,function(a,b,c){var d,e,f={},g=0;if(m.isArray(b)){for(d=Ib(a),e=b.length;e>g;g++)f[b[g]]=m.css(a,b[g],!1,d);return f}return void 0!==c?m.style(a,b,c):m.css(a,b)},a,b,arguments.length>1)},show:function(){return Vb(this,!0)},hide:function(){return Vb(this)},toggle:function(a){return"boolean"==typeof a?a?this.show():this.hide():this.each(function(){U(this)?m(this).show():m(this).hide()})}});function Zb(a,b,c,d,e){return new Zb.prototype.init(a,b,c,d,e)}m.Tween=Zb,Zb.prototype={constructor:Zb,init:function(a,b,c,d,e,f){this.elem=a,this.prop=c,this.easing=e||"swing",this.options=b,this.start=this.now=this.cur(),this.end=d,this.unit=f||(m.cssNumber[c]?"":"px")
+},cur:function(){var a=Zb.propHooks[this.prop];return a&&a.get?a.get(this):Zb.propHooks._default.get(this)},run:function(a){var b,c=Zb.propHooks[this.prop];return this.pos=b=this.options.duration?m.easing[this.easing](a,this.options.duration*a,0,1,this.options.duration):a,this.now=(this.end-this.start)*b+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),c&&c.set?c.set(this):Zb.propHooks._default.set(this),this}},Zb.prototype.init.prototype=Zb.prototype,Zb.propHooks={_default:{get:function(a){var b;return null==a.elem[a.prop]||a.elem.style&&null!=a.elem.style[a.prop]?(b=m.css(a.elem,a.prop,""),b&&"auto"!==b?b:0):a.elem[a.prop]},set:function(a){m.fx.step[a.prop]?m.fx.step[a.prop](a):a.elem.style&&(null!=a.elem.style[m.cssProps[a.prop]]||m.cssHooks[a.prop])?m.style(a.elem,a.prop,a.now+a.unit):a.elem[a.prop]=a.now}}},Zb.propHooks.scrollTop=Zb.propHooks.scrollLeft={set:function(a){a.elem.nodeType&&a.elem.parentNode&&(a.elem[a.prop]=a.now)}},m.easing={linear:function(a){return a},swing:function(a){return.5-Math.cos(a*Math.PI)/2}},m.fx=Zb.prototype.init,m.fx.step={};var $b,_b,ac=/^(?:toggle|show|hide)$/,bc=new RegExp("^(?:([+-])=|)("+S+")([a-z%]*)$","i"),cc=/queueHooks$/,dc=[ic],ec={"*":[function(a,b){var c=this.createTween(a,b),d=c.cur(),e=bc.exec(b),f=e&&e[3]||(m.cssNumber[a]?"":"px"),g=(m.cssNumber[a]||"px"!==f&&+d)&&bc.exec(m.css(c.elem,a)),h=1,i=20;if(g&&g[3]!==f){f=f||g[3],e=e||[],g=+d||1;do h=h||".5",g/=h,m.style(c.elem,a,g+f);while(h!==(h=c.cur()/d)&&1!==h&&--i)}return e&&(g=c.start=+g||+d||0,c.unit=f,c.end=e[1]?g+(e[1]+1)*e[2]:+e[2]),c}]};function fc(){return setTimeout(function(){$b=void 0}),$b=m.now()}function gc(a,b){var c,d={height:a},e=0;for(b=b?1:0;4>e;e+=2-b)c=T[e],d["margin"+c]=d["padding"+c]=a;return b&&(d.opacity=d.width=a),d}function hc(a,b,c){for(var d,e=(ec[b]||[]).concat(ec["*"]),f=0,g=e.length;g>f;f++)if(d=e[f].call(c,b,a))return d}function ic(a,b,c){var d,e,f,g,h,i,j,l,n=this,o={},p=a.style,q=a.nodeType&&U(a),r=m._data(a,"fxshow");c.queue||(h=m._queueHooks(a,"fx"),null==h.unqueued&&(h.unqueued=0,i=h.empty.fire,h.empty.fire=function(){h.unqueued||i()}),h.unqueued++,n.always(function(){n.always(function(){h.unqueued--,m.queue(a,"fx").length||h.empty.fire()})})),1===a.nodeType&&("height"in b||"width"in b)&&(c.overflow=[p.overflow,p.overflowX,p.overflowY],j=m.css(a,"display"),l="none"===j?m._data(a,"olddisplay")||Fb(a.nodeName):j,"inline"===l&&"none"===m.css(a,"float")&&(k.inlineBlockNeedsLayout&&"inline"!==Fb(a.nodeName)?p.zoom=1:p.display="inline-block")),c.overflow&&(p.overflow="hidden",k.shrinkWrapBlocks()||n.always(function(){p.overflow=c.overflow[0],p.overflowX=c.overflow[1],p.overflowY=c.overflow[2]}));for(d in b)if(e=b[d],ac.exec(e)){if(delete b[d],f=f||"toggle"===e,e===(q?"hide":"show")){if("show"!==e||!r||void 0===r[d])continue;q=!0}o[d]=r&&r[d]||m.style(a,d)}else j=void 0;if(m.isEmptyObject(o))"inline"===("none"===j?Fb(a.nodeName):j)&&(p.display=j);else{r?"hidden"in r&&(q=r.hidden):r=m._data(a,"fxshow",{}),f&&(r.hidden=!q),q?m(a).show():n.done(function(){m(a).hide()}),n.done(function(){var b;m._removeData(a,"fxshow");for(b in o)m.style(a,b,o[b])});for(d in o)g=hc(q?r[d]:0,d,n),d in r||(r[d]=g.start,q&&(g.end=g.start,g.start="width"===d||"height"===d?1:0))}}function jc(a,b){var c,d,e,f,g;for(c in a)if(d=m.camelCase(c),e=b[d],f=a[c],m.isArray(f)&&(e=f[1],f=a[c]=f[0]),c!==d&&(a[d]=f,delete a[c]),g=m.cssHooks[d],g&&"expand"in g){f=g.expand(f),delete a[d];for(c in f)c in a||(a[c]=f[c],b[c]=e)}else b[d]=e}function kc(a,b,c){var d,e,f=0,g=dc.length,h=m.Deferred().always(function(){delete i.elem}),i=function(){if(e)return!1;for(var b=$b||fc(),c=Math.max(0,j.startTime+j.duration-b),d=c/j.duration||0,f=1-d,g=0,i=j.tweens.length;i>g;g++)j.tweens[g].run(f);return h.notifyWith(a,[j,f,c]),1>f&&i?c:(h.resolveWith(a,[j]),!1)},j=h.promise({elem:a,props:m.extend({},b),opts:m.extend(!0,{specialEasing:{}},c),originalProperties:b,originalOptions:c,startTime:$b||fc(),duration:c.duration,tweens:[],createTween:function(b,c){var d=m.Tween(a,j.opts,b,c,j.opts.specialEasing[b]||j.opts.easing);return j.tweens.push(d),d},stop:function(b){var c=0,d=b?j.tweens.length:0;if(e)return this;for(e=!0;d>c;c++)j.tweens[c].run(1);return b?h.resolveWith(a,[j,b]):h.rejectWith(a,[j,b]),this}}),k=j.props;for(jc(k,j.opts.specialEasing);g>f;f++)if(d=dc[f].call(j,a,k,j.opts))return d;return m.map(k,hc,j),m.isFunction(j.opts.start)&&j.opts.start.call(a,j),m.fx.timer(m.extend(i,{elem:a,anim:j,queue:j.opts.queue})),j.progress(j.opts.progress).done(j.opts.done,j.opts.complete).fail(j.opts.fail).always(j.opts.always)}m.Animation=m.extend(kc,{tweener:function(a,b){m.isFunction(a)?(b=a,a=["*"]):a=a.split(" ");for(var c,d=0,e=a.length;e>d;d++)c=a[d],ec[c]=ec[c]||[],ec[c].unshift(b)},prefilter:function(a,b){b?dc.unshift(a):dc.push(a)}}),m.speed=function(a,b,c){var d=a&&"object"==typeof a?m.extend({},a):{complete:c||!c&&b||m.isFunction(a)&&a,duration:a,easing:c&&b||b&&!m.isFunction(b)&&b};return d.duration=m.fx.off?0:"number"==typeof d.duration?d.duration:d.duration in m.fx.speeds?m.fx.speeds[d.duration]:m.fx.speeds._default,(null==d.queue||d.queue===!0)&&(d.queue="fx"),d.old=d.complete,d.complete=function(){m.isFunction(d.old)&&d.old.call(this),d.queue&&m.dequeue(this,d.queue)},d},m.fn.extend({fadeTo:function(a,b,c,d){return this.filter(U).css("opacity",0).show().end().animate({opacity:b},a,c,d)},animate:function(a,b,c,d){var e=m.isEmptyObject(a),f=m.speed(b,c,d),g=function(){var b=kc(this,m.extend({},a),f);(e||m._data(this,"finish"))&&b.stop(!0)};return g.finish=g,e||f.queue===!1?this.each(g):this.queue(f.queue,g)},stop:function(a,b,c){var d=function(a){var b=a.stop;delete a.stop,b(c)};return"string"!=typeof a&&(c=b,b=a,a=void 0),b&&a!==!1&&this.queue(a||"fx",[]),this.each(function(){var b=!0,e=null!=a&&a+"queueHooks",f=m.timers,g=m._data(this);if(e)g[e]&&g[e].stop&&d(g[e]);else for(e in g)g[e]&&g[e].stop&&cc.test(e)&&d(g[e]);for(e=f.length;e--;)f[e].elem!==this||null!=a&&f[e].queue!==a||(f[e].anim.stop(c),b=!1,f.splice(e,1));(b||!c)&&m.dequeue(this,a)})},finish:function(a){return a!==!1&&(a=a||"fx"),this.each(function(){var b,c=m._data(this),d=c[a+"queue"],e=c[a+"queueHooks"],f=m.timers,g=d?d.length:0;for(c.finish=!0,m.queue(this,a,[]),e&&e.stop&&e.stop.call(this,!0),b=f.length;b--;)f[b].elem===this&&f[b].queue===a&&(f[b].anim.stop(!0),f.splice(b,1));for(b=0;g>b;b++)d[b]&&d[b].finish&&d[b].finish.call(this);delete c.finish})}}),m.each(["toggle","show","hide"],function(a,b){var c=m.fn[b];m.fn[b]=function(a,d,e){return null==a||"boolean"==typeof a?c.apply(this,arguments):this.animate(gc(b,!0),a,d,e)}}),m.each({slideDown:gc("show"),slideUp:gc("hide"),slideToggle:gc("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(a,b){m.fn[a]=function(a,c,d){return this.animate(b,a,c,d)}}),m.timers=[],m.fx.tick=function(){var a,b=m.timers,c=0;for($b=m.now();c<b.length;c++)a=b[c],a()||b[c]!==a||b.splice(c--,1);b.length||m.fx.stop(),$b=void 0},m.fx.timer=function(a){m.timers.push(a),a()?m.fx.start():m.timers.pop()},m.fx.interval=13,m.fx.start=function(){_b||(_b=setInterval(m.fx.tick,m.fx.interval))},m.fx.stop=function(){clearInterval(_b),_b=null},m.fx.speeds={slow:600,fast:200,_default:400},m.fn.delay=function(a,b){return a=m.fx?m.fx.speeds[a]||a:a,b=b||"fx",this.queue(b,function(b,c){var d=setTimeout(b,a);c.stop=function(){clearTimeout(d)}})},function(){var a,b,c,d,e;b=y.createElement("div"),b.setAttribute("className","t"),b.innerHTML="  <link/><table></table><a href='/a'>a</a><input type='checkbox'/>",d=b.getElementsByTagName("a")[0],c=y.createElement("select"),e=c.appendChild(y.createElement("option")),a=b.getElementsByTagName("input")[0],d.style.cssText="top:1px",k.getSetAttribute="t"!==b.className,k.style=/top/.test(d.getAttribute("style")),k.hrefNormalized="/a"===d.getAttribute("href"),k.checkOn=!!a.value,k.optSelected=e.selected,k.enctype=!!y.createElement("form").enctype,c.disabled=!0,k.optDisabled=!e.disabled,a=y.createElement("input"),a.setAttribute("value",""),k.input=""===a.getAttribute("value"),a.value="t",a.setAttribute("type","radio"),k.radioValue="t"===a.value}();var lc=/\r/g;m.fn.extend({val:function(a){var b,c,d,e=this[0];{if(arguments.length)return d=m.isFunction(a),this.each(function(c){var e;1===this.nodeType&&(e=d?a.call(this,c,m(this).val()):a,null==e?e="":"number"==typeof e?e+="":m.isArray(e)&&(e=m.map(e,function(a){return null==a?"":a+""})),b=m.valHooks[this.type]||m.valHooks[this.nodeName.toLowerCase()],b&&"set"in b&&void 0!==b.set(this,e,"value")||(this.value=e))});if(e)return b=m.valHooks[e.type]||m.valHooks[e.nodeName.toLowerCase()],b&&"get"in b&&void 0!==(c=b.get(e,"value"))?c:(c=e.value,"string"==typeof c?c.replace(lc,""):null==c?"":c)}}}),m.extend({valHooks:{option:{get:function(a){var b=m.find.attr(a,"value");return null!=b?b:m.trim(m.text(a))}},select:{get:function(a){for(var b,c,d=a.options,e=a.selectedIndex,f="select-one"===a.type||0>e,g=f?null:[],h=f?e+1:d.length,i=0>e?h:f?e:0;h>i;i++)if(c=d[i],!(!c.selected&&i!==e||(k.optDisabled?c.disabled:null!==c.getAttribute("disabled"))||c.parentNode.disabled&&m.nodeName(c.parentNode,"optgroup"))){if(b=m(c).val(),f)return b;g.push(b)}return g},set:function(a,b){var c,d,e=a.options,f=m.makeArray(b),g=e.length;while(g--)if(d=e[g],m.inArray(m.valHooks.option.get(d),f)>=0)try{d.selected=c=!0}catch(h){d.scrollHeight}else d.selected=!1;return c||(a.selectedIndex=-1),e}}}}),m.each(["radio","checkbox"],function(){m.valHooks[this]={set:function(a,b){return m.isArray(b)?a.checked=m.inArray(m(a).val(),b)>=0:void 0}},k.checkOn||(m.valHooks[this].get=function(a){return null===a.getAttribute("value")?"on":a.value})});var mc,nc,oc=m.expr.attrHandle,pc=/^(?:checked|selected)$/i,qc=k.getSetAttribute,rc=k.input;m.fn.extend({attr:function(a,b){return V(this,m.attr,a,b,arguments.length>1)},removeAttr:function(a){return this.each(function(){m.removeAttr(this,a)})}}),m.extend({attr:function(a,b,c){var d,e,f=a.nodeType;if(a&&3!==f&&8!==f&&2!==f)return typeof a.getAttribute===K?m.prop(a,b,c):(1===f&&m.isXMLDoc(a)||(b=b.toLowerCase(),d=m.attrHooks[b]||(m.expr.match.bool.test(b)?nc:mc)),void 0===c?d&&"get"in d&&null!==(e=d.get(a,b))?e:(e=m.find.attr(a,b),null==e?void 0:e):null!==c?d&&"set"in d&&void 0!==(e=d.set(a,c,b))?e:(a.setAttribute(b,c+""),c):void m.removeAttr(a,b))},removeAttr:function(a,b){var c,d,e=0,f=b&&b.match(E);if(f&&1===a.nodeType)while(c=f[e++])d=m.propFix[c]||c,m.expr.match.bool.test(c)?rc&&qc||!pc.test(c)?a[d]=!1:a[m.camelCase("default-"+c)]=a[d]=!1:m.attr(a,c,""),a.removeAttribute(qc?c:d)},attrHooks:{type:{set:function(a,b){if(!k.radioValue&&"radio"===b&&m.nodeName(a,"input")){var c=a.value;return a.setAttribute("type",b),c&&(a.value=c),b}}}}}),nc={set:function(a,b,c){return b===!1?m.removeAttr(a,c):rc&&qc||!pc.test(c)?a.setAttribute(!qc&&m.propFix[c]||c,c):a[m.camelCase("default-"+c)]=a[c]=!0,c}},m.each(m.expr.match.bool.source.match(/\w+/g),function(a,b){var c=oc[b]||m.find.attr;oc[b]=rc&&qc||!pc.test(b)?function(a,b,d){var e,f;return d||(f=oc[b],oc[b]=e,e=null!=c(a,b,d)?b.toLowerCase():null,oc[b]=f),e}:function(a,b,c){return c?void 0:a[m.camelCase("default-"+b)]?b.toLowerCase():null}}),rc&&qc||(m.attrHooks.value={set:function(a,b,c){return m.nodeName(a,"input")?void(a.defaultValue=b):mc&&mc.set(a,b,c)}}),qc||(mc={set:function(a,b,c){var d=a.getAttributeNode(c);return d||a.setAttributeNode(d=a.ownerDocument.createAttribute(c)),d.value=b+="","value"===c||b===a.getAttribute(c)?b:void 0}},oc.id=oc.name=oc.coords=function(a,b,c){var d;return c?void 0:(d=a.getAttributeNode(b))&&""!==d.value?d.value:null},m.valHooks.button={get:function(a,b){var c=a.getAttributeNode(b);return c&&c.specified?c.value:void 0},set:mc.set},m.attrHooks.contenteditable={set:function(a,b,c){mc.set(a,""===b?!1:b,c)}},m.each(["width","height"],function(a,b){m.attrHooks[b]={set:function(a,c){return""===c?(a.setAttribute(b,"auto"),c):void 0}}})),k.style||(m.attrHooks.style={get:function(a){return a.style.cssText||void 0},set:function(a,b){return a.style.cssText=b+""}});var sc=/^(?:input|select|textarea|button|object)$/i,tc=/^(?:a|area)$/i;m.fn.extend({prop:function(a,b){return V(this,m.prop,a,b,arguments.length>1)},removeProp:function(a){return a=m.propFix[a]||a,this.each(function(){try{this[a]=void 0,delete this[a]}catch(b){}})}}),m.extend({propFix:{"for":"htmlFor","class":"className"},prop:function(a,b,c){var d,e,f,g=a.nodeType;if(a&&3!==g&&8!==g&&2!==g)return f=1!==g||!m.isXMLDoc(a),f&&(b=m.propFix[b]||b,e=m.propHooks[b]),void 0!==c?e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:a[b]=c:e&&"get"in e&&null!==(d=e.get(a,b))?d:a[b]},propHooks:{tabIndex:{get:function(a){var b=m.find.attr(a,"tabindex");return b?parseInt(b,10):sc.test(a.nodeName)||tc.test(a.nodeName)&&a.href?0:-1}}}}),k.hrefNormalized||m.each(["href","src"],function(a,b){m.propHooks[b]={get:function(a){return a.getAttribute(b,4)}}}),k.optSelected||(m.propHooks.selected={get:function(a){var b=a.parentNode;return b&&(b.selectedIndex,b.parentNode&&b.parentNode.selectedIndex),null}}),m.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){m.propFix[this.toLowerCase()]=this}),k.enctype||(m.propFix.enctype="encoding");var uc=/[\t\r\n\f]/g;m.fn.extend({addClass:function(a){var b,c,d,e,f,g,h=0,i=this.length,j="string"==typeof a&&a;if(m.isFunction(a))return this.each(function(b){m(this).addClass(a.call(this,b,this.className))});if(j)for(b=(a||"").match(E)||[];i>h;h++)if(c=this[h],d=1===c.nodeType&&(c.className?(" "+c.className+" ").replace(uc," "):" ")){f=0;while(e=b[f++])d.indexOf(" "+e+" ")<0&&(d+=e+" ");g=m.trim(d),c.className!==g&&(c.className=g)}return this},removeClass:function(a){var b,c,d,e,f,g,h=0,i=this.length,j=0===arguments.length||"string"==typeof a&&a;if(m.isFunction(a))return this.each(function(b){m(this).removeClass(a.call(this,b,this.className))});if(j)for(b=(a||"").match(E)||[];i>h;h++)if(c=this[h],d=1===c.nodeType&&(c.className?(" "+c.className+" ").replace(uc," "):"")){f=0;while(e=b[f++])while(d.indexOf(" "+e+" ")>=0)d=d.replace(" "+e+" "," ");g=a?m.trim(d):"",c.className!==g&&(c.className=g)}return this},toggleClass:function(a,b){var c=typeof a;return"boolean"==typeof b&&"string"===c?b?this.addClass(a):this.removeClass(a):this.each(m.isFunction(a)?function(c){m(this).toggleClass(a.call(this,c,this.className,b),b)}:function(){if("string"===c){var b,d=0,e=m(this),f=a.match(E)||[];while(b=f[d++])e.hasClass(b)?e.removeClass(b):e.addClass(b)}else(c===K||"boolean"===c)&&(this.className&&m._data(this,"__className__",this.className),this.className=this.className||a===!1?"":m._data(this,"__className__")||"")})},hasClass:function(a){for(var b=" "+a+" ",c=0,d=this.length;d>c;c++)if(1===this[c].nodeType&&(" "+this[c].className+" ").replace(uc," ").indexOf(b)>=0)return!0;return!1}}),m.each("blur focus focusin focusout load resize scroll unload click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup error contextmenu".split(" "),function(a,b){m.fn[b]=function(a,c){return arguments.length>0?this.on(b,null,a,c):this.trigger(b)}}),m.fn.extend({hover:function(a,b){return this.mouseenter(a).mouseleave(b||a)},bind:function(a,b,c){return this.on(a,null,b,c)},unbind:function(a,b){return this.off(a,null,b)},delegate:function(a,b,c,d){return this.on(b,a,c,d)},undelegate:function(a,b,c){return 1===arguments.length?this.off(a,"**"):this.off(b,a||"**",c)}});var vc=m.now(),wc=/\?/,xc=/(,)|(\[|{)|(}|])|"(?:[^"\\\r\n]|\\["\\\/bfnrt]|\\u[\da-fA-F]{4})*"\s*:?|true|false|null|-?(?!0\d)\d+(?:\.\d+|)(?:[eE][+-]?\d+|)/g;m.parseJSON=function(b){if(a.JSON&&a.JSON.parse)return a.JSON.parse(b+"");var c,d=null,e=m.trim(b+"");return e&&!m.trim(e.replace(xc,function(a,b,e,f){return c&&b&&(d=0),0===d?a:(c=e||b,d+=!f-!e,"")}))?Function("return "+e)():m.error("Invalid JSON: "+b)},m.parseXML=function(b){var c,d;if(!b||"string"!=typeof b)return null;try{a.DOMParser?(d=new DOMParser,c=d.parseFromString(b,"text/xml")):(c=new ActiveXObject("Microsoft.XMLDOM"),c.async="false",c.loadXML(b))}catch(e){c=void 0}return c&&c.documentElement&&!c.getElementsByTagName("parsererror").length||m.error("Invalid XML: "+b),c};var yc,zc,Ac=/#.*$/,Bc=/([?&])_=[^&]*/,Cc=/^(.*?):[ \t]*([^\r\n]*)\r?$/gm,Dc=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Ec=/^(?:GET|HEAD)$/,Fc=/^\/\//,Gc=/^([\w.+-]+:)(?:\/\/(?:[^\/?#]*@|)([^\/?#:]*)(?::(\d+)|)|)/,Hc={},Ic={},Jc="*/".concat("*");try{zc=location.href}catch(Kc){zc=y.createElement("a"),zc.href="",zc=zc.href}yc=Gc.exec(zc.toLowerCase())||[];function Lc(a){return function(b,c){"string"!=typeof b&&(c=b,b="*");var d,e=0,f=b.toLowerCase().match(E)||[];if(m.isFunction(c))while(d=f[e++])"+"===d.charAt(0)?(d=d.slice(1)||"*",(a[d]=a[d]||[]).unshift(c)):(a[d]=a[d]||[]).push(c)}}function Mc(a,b,c,d){var e={},f=a===Ic;function g(h){var i;return e[h]=!0,m.each(a[h]||[],function(a,h){var j=h(b,c,d);return"string"!=typeof j||f||e[j]?f?!(i=j):void 0:(b.dataTypes.unshift(j),g(j),!1)}),i}return g(b.dataTypes[0])||!e["*"]&&g("*")}function Nc(a,b){var c,d,e=m.ajaxSettings.flatOptions||{};for(d in b)void 0!==b[d]&&((e[d]?a:c||(c={}))[d]=b[d]);return c&&m.extend(!0,a,c),a}function Oc(a,b,c){var d,e,f,g,h=a.contents,i=a.dataTypes;while("*"===i[0])i.shift(),void 0===e&&(e=a.mimeType||b.getResponseHeader("Content-Type"));if(e)for(g in h)if(h[g]&&h[g].test(e)){i.unshift(g);break}if(i[0]in c)f=i[0];else{for(g in c){if(!i[0]||a.converters[g+" "+i[0]]){f=g;break}d||(d=g)}f=f||d}return f?(f!==i[0]&&i.unshift(f),c[f]):void 0}function Pc(a,b,c,d){var e,f,g,h,i,j={},k=a.dataTypes.slice();if(k[1])for(g in a.converters)j[g.toLowerCase()]=a.converters[g];f=k.shift();while(f)if(a.responseFields[f]&&(c[a.responseFields[f]]=b),!i&&d&&a.dataFilter&&(b=a.dataFilter(b,a.dataType)),i=f,f=k.shift())if("*"===f)f=i;else if("*"!==i&&i!==f){if(g=j[i+" "+f]||j["* "+f],!g)for(e in j)if(h=e.split(" "),h[1]===f&&(g=j[i+" "+h[0]]||j["* "+h[0]])){g===!0?g=j[e]:j[e]!==!0&&(f=h[0],k.unshift(h[1]));break}if(g!==!0)if(g&&a["throws"])b=g(b);else try{b=g(b)}catch(l){return{state:"parsererror",error:g?l:"No conversion from "+i+" to "+f}}}return{state:"success",data:b}}m.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:zc,type:"GET",isLocal:Dc.test(yc[1]),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Jc,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/xml/,html:/html/,json:/json/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":m.parseJSON,"text xml":m.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(a,b){return b?Nc(Nc(a,m.ajaxSettings),b):Nc(m.ajaxSettings,a)},ajaxPrefilter:Lc(Hc),ajaxTransport:Lc(Ic),ajax:function(a,b){"object"==typeof a&&(b=a,a=void 0),b=b||{};var c,d,e,f,g,h,i,j,k=m.ajaxSetup({},b),l=k.context||k,n=k.context&&(l.nodeType||l.jquery)?m(l):m.event,o=m.Deferred(),p=m.Callbacks("once memory"),q=k.statusCode||{},r={},s={},t=0,u="canceled",v={readyState:0,getResponseHeader:function(a){var b;if(2===t){if(!j){j={};while(b=Cc.exec(f))j[b[1].toLowerCase()]=b[2]}b=j[a.toLowerCase()]}return null==b?null:b},getAllResponseHeaders:function(){return 2===t?f:null},setRequestHeader:function(a,b){var c=a.toLowerCase();return t||(a=s[c]=s[c]||a,r[a]=b),this},overrideMimeType:function(a){return t||(k.mimeType=a),this},statusCode:function(a){var b;if(a)if(2>t)for(b in a)q[b]=[q[b],a[b]];else v.always(a[v.status]);return this},abort:function(a){var b=a||u;return i&&i.abort(b),x(0,b),this}};if(o.promise(v).complete=p.add,v.success=v.done,v.error=v.fail,k.url=((a||k.url||zc)+"").replace(Ac,"").replace(Fc,yc[1]+"//"),k.type=b.method||b.type||k.method||k.type,k.dataTypes=m.trim(k.dataType||"*").toLowerCase().match(E)||[""],null==k.crossDomain&&(c=Gc.exec(k.url.toLowerCase()),k.crossDomain=!(!c||c[1]===yc[1]&&c[2]===yc[2]&&(c[3]||("http:"===c[1]?"80":"443"))===(yc[3]||("http:"===yc[1]?"80":"443")))),k.data&&k.processData&&"string"!=typeof k.data&&(k.data=m.param(k.data,k.traditional)),Mc(Hc,k,b,v),2===t)return v;h=k.global,h&&0===m.active++&&m.event.trigger("ajaxStart"),k.type=k.type.toUpperCase(),k.hasContent=!Ec.test(k.type),e=k.url,k.hasContent||(k.data&&(e=k.url+=(wc.test(e)?"&":"?")+k.data,delete k.data),k.cache===!1&&(k.url=Bc.test(e)?e.replace(Bc,"$1_="+vc++):e+(wc.test(e)?"&":"?")+"_="+vc++)),k.ifModified&&(m.lastModified[e]&&v.setRequestHeader("If-Modified-Since",m.lastModified[e]),m.etag[e]&&v.setRequestHeader("If-None-Match",m.etag[e])),(k.data&&k.hasContent&&k.contentType!==!1||b.contentType)&&v.setRequestHeader("Content-Type",k.contentType),v.setRequestHeader("Accept",k.dataTypes[0]&&k.accepts[k.dataTypes[0]]?k.accepts[k.dataTypes[0]]+("*"!==k.dataTypes[0]?", "+Jc+"; q=0.01":""):k.accepts["*"]);for(d in k.headers)v.setRequestHeader(d,k.headers[d]);if(k.beforeSend&&(k.beforeSend.call(l,v,k)===!1||2===t))return v.abort();u="abort";for(d in{success:1,error:1,complete:1})v[d](k[d]);if(i=Mc(Ic,k,b,v)){v.readyState=1,h&&n.trigger("ajaxSend",[v,k]),k.async&&k.timeout>0&&(g=setTimeout(function(){v.abort("timeout")},k.timeout));try{t=1,i.send(r,x)}catch(w){if(!(2>t))throw w;x(-1,w)}}else x(-1,"No Transport");function x(a,b,c,d){var j,r,s,u,w,x=b;2!==t&&(t=2,g&&clearTimeout(g),i=void 0,f=d||"",v.readyState=a>0?4:0,j=a>=200&&300>a||304===a,c&&(u=Oc(k,v,c)),u=Pc(k,u,v,j),j?(k.ifModified&&(w=v.getResponseHeader("Last-Modified"),w&&(m.lastModified[e]=w),w=v.getResponseHeader("etag"),w&&(m.etag[e]=w)),204===a||"HEAD"===k.type?x="nocontent":304===a?x="notmodified":(x=u.state,r=u.data,s=u.error,j=!s)):(s=x,(a||!x)&&(x="error",0>a&&(a=0))),v.status=a,v.statusText=(b||x)+"",j?o.resolveWith(l,[r,x,v]):o.rejectWith(l,[v,x,s]),v.statusCode(q),q=void 0,h&&n.trigger(j?"ajaxSuccess":"ajaxError",[v,k,j?r:s]),p.fireWith(l,[v,x]),h&&(n.trigger("ajaxComplete",[v,k]),--m.active||m.event.trigger("ajaxStop")))}return v},getJSON:function(a,b,c){return m.get(a,b,c,"json")},getScript:function(a,b){return m.get(a,void 0,b,"script")}}),m.each(["get","post"],function(a,b){m[b]=function(a,c,d,e){return m.isFunction(c)&&(e=e||d,d=c,c=void 0),m.ajax({url:a,type:b,dataType:e,data:c,success:d})}}),m.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(a,b){m.fn[b]=function(a){return this.on(b,a)}}),m._evalUrl=function(a){return m.ajax({url:a,type:"GET",dataType:"script",async:!1,global:!1,"throws":!0})},m.fn.extend({wrapAll:function(a){if(m.isFunction(a))return this.each(function(b){m(this).wrapAll(a.call(this,b))});if(this[0]){var b=m(a,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstChild&&1===a.firstChild.nodeType)a=a.firstChild;return a}).append(this)}return this},wrapInner:function(a){return this.each(m.isFunction(a)?function(b){m(this).wrapInner(a.call(this,b))}:function(){var b=m(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){var b=m.isFunction(a);return this.each(function(c){m(this).wrapAll(b?a.call(this,c):a)})},unwrap:function(){return this.parent().each(function(){m.nodeName(this,"body")||m(this).replaceWith(this.childNodes)}).end()}}),m.expr.filters.hidden=function(a){return a.offsetWidth<=0&&a.offsetHeight<=0||!k.reliableHiddenOffsets()&&"none"===(a.style&&a.style.display||m.css(a,"display"))},m.expr.filters.visible=function(a){return!m.expr.filters.hidden(a)};var Qc=/%20/g,Rc=/\[\]$/,Sc=/\r?\n/g,Tc=/^(?:submit|button|image|reset|file)$/i,Uc=/^(?:input|select|textarea|keygen)/i;function Vc(a,b,c,d){var e;if(m.isArray(b))m.each(b,function(b,e){c||Rc.test(a)?d(a,e):Vc(a+"["+("object"==typeof e?b:"")+"]",e,c,d)});else if(c||"object"!==m.type(b))d(a,b);else for(e in b)Vc(a+"["+e+"]",b[e],c,d)}m.param=function(a,b){var c,d=[],e=function(a,b){b=m.isFunction(b)?b():null==b?"":b,d[d.length]=encodeURIComponent(a)+"="+encodeURIComponent(b)};if(void 0===b&&(b=m.ajaxSettings&&m.ajaxSettings.traditional),m.isArray(a)||a.jquery&&!m.isPlainObject(a))m.each(a,function(){e(this.name,this.value)});else for(c in a)Vc(c,a[c],b,e);return d.join("&").replace(Qc,"+")},m.fn.extend({serialize:function(){return m.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var a=m.prop(this,"elements");return a?m.makeArray(a):this}).filter(function(){var a=this.type;return this.name&&!m(this).is(":disabled")&&Uc.test(this.nodeName)&&!Tc.test(a)&&(this.checked||!W.test(a))}).map(function(a,b){var c=m(this).val();return null==c?null:m.isArray(c)?m.map(c,function(a){return{name:b.name,value:a.replace(Sc,"\r\n")}}):{name:b.name,value:c.replace(Sc,"\r\n")}}).get()}}),m.ajaxSettings.xhr=void 0!==a.ActiveXObject?function(){return!this.isLocal&&/^(get|post|head|put|delete|options)$/i.test(this.type)&&Zc()||$c()}:Zc;var Wc=0,Xc={},Yc=m.ajaxSettings.xhr();a.ActiveXObject&&m(a).on("unload",function(){for(var a in Xc)Xc[a](void 0,!0)}),k.cors=!!Yc&&"withCredentials"in Yc,Yc=k.ajax=!!Yc,Yc&&m.ajaxTransport(function(a){if(!a.crossDomain||k.cors){var b;return{send:function(c,d){var e,f=a.xhr(),g=++Wc;if(f.open(a.type,a.url,a.async,a.username,a.password),a.xhrFields)for(e in a.xhrFields)f[e]=a.xhrFields[e];a.mimeType&&f.overrideMimeType&&f.overrideMimeType(a.mimeType),a.crossDomain||c["X-Requested-With"]||(c["X-Requested-With"]="XMLHttpRequest");for(e in c)void 0!==c[e]&&f.setRequestHeader(e,c[e]+"");f.send(a.hasContent&&a.data||null),b=function(c,e){var h,i,j;if(b&&(e||4===f.readyState))if(delete Xc[g],b=void 0,f.onreadystatechange=m.noop,e)4!==f.readyState&&f.abort();else{j={},h=f.status,"string"==typeof f.responseText&&(j.text=f.responseText);try{i=f.statusText}catch(k){i=""}h||!a.isLocal||a.crossDomain?1223===h&&(h=204):h=j.text?200:404}j&&d(h,i,j,f.getAllResponseHeaders())},a.async?4===f.readyState?setTimeout(b):f.onreadystatechange=Xc[g]=b:b()},abort:function(){b&&b(void 0,!0)}}}});function Zc(){try{return new a.XMLHttpRequest}catch(b){}}function $c(){try{return new a.ActiveXObject("Microsoft.XMLHTTP")}catch(b){}}m.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/(?:java|ecma)script/},converters:{"text script":function(a){return m.globalEval(a),a}}}),m.ajaxPrefilter("script",function(a){void 0===a.cache&&(a.cache=!1),a.crossDomain&&(a.type="GET",a.global=!1)}),m.ajaxTransport("script",function(a){if(a.crossDomain){var b,c=y.head||m("head")[0]||y.documentElement;return{send:function(d,e){b=y.createElement("script"),b.async=!0,a.scriptCharset&&(b.charset=a.scriptCharset),b.src=a.url,b.onload=b.onreadystatechange=function(a,c){(c||!b.readyState||/loaded|complete/.test(b.readyState))&&(b.onload=b.onreadystatechange=null,b.parentNode&&b.parentNode.removeChild(b),b=null,c||e(200,"success"))},c.insertBefore(b,c.firstChild)},abort:function(){b&&b.onload(void 0,!0)}}}});var _c=[],ad=/(=)\?(?=&|$)|\?\?/;m.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var a=_c.pop()||m.expando+"_"+vc++;return this[a]=!0,a}}),m.ajaxPrefilter("json jsonp",function(b,c,d){var e,f,g,h=b.jsonp!==!1&&(ad.test(b.url)?"url":"string"==typeof b.data&&!(b.contentType||"").indexOf("application/x-www-form-urlencoded")&&ad.test(b.data)&&"data");return h||"jsonp"===b.dataTypes[0]?(e=b.jsonpCallback=m.isFunction(b.jsonpCallback)?b.jsonpCallback():b.jsonpCallback,h?b[h]=b[h].replace(ad,"$1"+e):b.jsonp!==!1&&(b.url+=(wc.test(b.url)?"&":"?")+b.jsonp+"="+e),b.converters["script json"]=function(){return g||m.error(e+" was not called"),g[0]},b.dataTypes[0]="json",f=a[e],a[e]=function(){g=arguments},d.always(function(){a[e]=f,b[e]&&(b.jsonpCallback=c.jsonpCallback,_c.push(e)),g&&m.isFunction(f)&&f(g[0]),g=f=void 0}),"script"):void 0}),m.parseHTML=function(a,b,c){if(!a||"string"!=typeof a)return null;"boolean"==typeof b&&(c=b,b=!1),b=b||y;var d=u.exec(a),e=!c&&[];return d?[b.createElement(d[1])]:(d=m.buildFragment([a],b,e),e&&e.length&&m(e).remove(),m.merge([],d.childNodes))};var bd=m.fn.load;m.fn.load=function(a,b,c){if("string"!=typeof a&&bd)return bd.apply(this,arguments);var d,e,f,g=this,h=a.indexOf(" ");return h>=0&&(d=m.trim(a.slice(h,a.length)),a=a.slice(0,h)),m.isFunction(b)?(c=b,b=void 0):b&&"object"==typeof b&&(f="POST"),g.length>0&&m.ajax({url:a,type:f,dataType:"html",data:b}).done(function(a){e=arguments,g.html(d?m("<div>").append(m.parseHTML(a)).find(d):a)}).complete(c&&function(a,b){g.each(c,e||[a.responseText,b,a])}),this},m.expr.filters.animated=function(a){return m.grep(m.timers,function(b){return a===b.elem}).length};var cd=a.document.documentElement;function dd(a){return m.isWindow(a)?a:9===a.nodeType?a.defaultView||a.parentWindow:!1}m.offset={setOffset:function(a,b,c){var d,e,f,g,h,i,j,k=m.css(a,"position"),l=m(a),n={};"static"===k&&(a.style.position="relative"),h=l.offset(),f=m.css(a,"top"),i=m.css(a,"left"),j=("absolute"===k||"fixed"===k)&&m.inArray("auto",[f,i])>-1,j?(d=l.position(),g=d.top,e=d.left):(g=parseFloat(f)||0,e=parseFloat(i)||0),m.isFunction(b)&&(b=b.call(a,c,h)),null!=b.top&&(n.top=b.top-h.top+g),null!=b.left&&(n.left=b.left-h.left+e),"using"in b?b.using.call(a,n):l.css(n)}},m.fn.extend({offset:function(a){if(arguments.length)return void 0===a?this:this.each(function(b){m.offset.setOffset(this,a,b)});var b,c,d={top:0,left:0},e=this[0],f=e&&e.ownerDocument;if(f)return b=f.documentElement,m.contains(b,e)?(typeof e.getBoundingClientRect!==K&&(d=e.getBoundingClientRect()),c=dd(f),{top:d.top+(c.pageYOffset||b.scrollTop)-(b.clientTop||0),left:d.left+(c.pageXOffset||b.scrollLeft)-(b.clientLeft||0)}):d},position:function(){if(this[0]){var a,b,c={top:0,left:0},d=this[0];return"fixed"===m.css(d,"position")?b=d.getBoundingClientRect():(a=this.offsetParent(),b=this.offset(),m.nodeName(a[0],"html")||(c=a.offset()),c.top+=m.css(a[0],"borderTopWidth",!0),c.left+=m.css(a[0],"borderLeftWidth",!0)),{top:b.top-c.top-m.css(d,"marginTop",!0),left:b.left-c.left-m.css(d,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var a=this.offsetParent||cd;while(a&&!m.nodeName(a,"html")&&"static"===m.css(a,"position"))a=a.offsetParent;return a||cd})}}),m.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(a,b){var c=/Y/.test(b);m.fn[a]=function(d){return V(this,function(a,d,e){var f=dd(a);return void 0===e?f?b in f?f[b]:f.document.documentElement[d]:a[d]:void(f?f.scrollTo(c?m(f).scrollLeft():e,c?e:m(f).scrollTop()):a[d]=e)},a,d,arguments.length,null)}}),m.each(["top","left"],function(a,b){m.cssHooks[b]=Lb(k.pixelPosition,function(a,c){return c?(c=Jb(a,b),Hb.test(c)?m(a).position()[b]+"px":c):void 0})}),m.each({Height:"height",Width:"width"},function(a,b){m.each({padding:"inner"+a,content:b,"":"outer"+a},function(c,d){m.fn[d]=function(d,e){var f=arguments.length&&(c||"boolean"!=typeof d),g=c||(d===!0||e===!0?"margin":"border");return V(this,function(b,c,d){var e;return m.isWindow(b)?b.document.documentElement["client"+a]:9===b.nodeType?(e=b.documentElement,Math.max(b.body["scroll"+a],e["scroll"+a],b.body["offset"+a],e["offset"+a],e["client"+a])):void 0===d?m.css(b,c,g):m.style(b,c,d,g)},b,f?d:void 0,f,null)}})}),m.fn.size=function(){return this.length},m.fn.andSelf=m.fn.addBack,"function"==typeof define&&define.amd&&define("jquery",[],function(){return m});var ed=a.jQuery,fd=a.$;return m.noConflict=function(b){return a.$===m&&(a.$=fd),b&&a.jQuery===m&&(a.jQuery=ed),m},typeof b===K&&(a.jQuery=a.$=m),m});
+
+
+
+/** d3js **/
+!function(){function n(n,t){return t>n?-1:n>t?1:n>=t?0:0/0}function t(n){return null!=n&&!isNaN(n)}function e(n){return{left:function(t,e,r,u){for(arguments.length<3&&(r=0),arguments.length<4&&(u=t.length);u>r;){var i=r+u>>>1;n(t[i],e)<0?r=i+1:u=i}return r},right:function(t,e,r,u){for(arguments.length<3&&(r=0),arguments.length<4&&(u=t.length);u>r;){var i=r+u>>>1;n(t[i],e)>0?u=i:r=i+1}return r}}}function r(n){return n.length}function u(n){for(var t=1;n*t%1;)t*=10;return t}function i(n,t){try{for(var e in t)Object.defineProperty(n.prototype,e,{value:t[e],enumerable:!1})}catch(r){n.prototype=t}}function o(){}function a(n){return ia+n in this}function c(n){return n=ia+n,n in this&&delete this[n]}function s(){var n=[];return this.forEach(function(t){n.push(t)}),n}function l(){var n=0;for(var t in this)t.charCodeAt(0)===oa&&++n;return n}function f(){for(var n in this)if(n.charCodeAt(0)===oa)return!1;return!0}function h(){}function g(n,t,e){return function(){var r=e.apply(t,arguments);return r===t?n:r}}function p(n,t){if(t in n)return t;t=t.charAt(0).toUpperCase()+t.substring(1);for(var e=0,r=aa.length;r>e;++e){var u=aa[e]+t;if(u in n)return u}}function v(){}function d(){}function m(n){function t(){for(var t,r=e,u=-1,i=r.length;++u<i;)(t=r[u].on)&&t.apply(this,arguments);return n}var e=[],r=new o;return t.on=function(t,u){var i,o=r.get(t);return arguments.length<2?o&&o.on:(o&&(o.on=null,e=e.slice(0,i=e.indexOf(o)).concat(e.slice(i+1)),r.remove(t)),u&&e.push(r.set(t,{on:u})),n)},t}function y(){Zo.event.preventDefault()}function x(){for(var n,t=Zo.event;n=t.sourceEvent;)t=n;return t}function M(n){for(var t=new d,e=0,r=arguments.length;++e<r;)t[arguments[e]]=m(t);return t.of=function(e,r){return function(u){try{var i=u.sourceEvent=Zo.event;u.target=n,Zo.event=u,t[u.type].apply(e,r)}finally{Zo.event=i}}},t}function _(n){return sa(n,pa),n}function b(n){return"function"==typeof n?n:function(){return la(n,this)}}function w(n){return"function"==typeof n?n:function(){return fa(n,this)}}function S(n,t){function e(){this.removeAttribute(n)}function r(){this.removeAttributeNS(n.space,n.local)}function u(){this.setAttribute(n,t)}function i(){this.setAttributeNS(n.space,n.local,t)}function o(){var e=t.apply(this,arguments);null==e?this.removeAttribute(n):this.setAttribute(n,e)}function a(){var e=t.apply(this,arguments);null==e?this.removeAttributeNS(n.space,n.local):this.setAttributeNS(n.space,n.local,e)}return n=Zo.ns.qualify(n),null==t?n.local?r:e:"function"==typeof t?n.local?a:o:n.local?i:u}function k(n){return n.trim().replace(/\s+/g," ")}function E(n){return new RegExp("(?:^|\\s+)"+Zo.requote(n)+"(?:\\s+|$)","g")}function A(n){return(n+"").trim().split(/^|\s+/)}function C(n,t){function e(){for(var e=-1;++e<u;)n[e](this,t)}function r(){for(var e=-1,r=t.apply(this,arguments);++e<u;)n[e](this,r)}n=A(n).map(N);var u=n.length;return"function"==typeof t?r:e}function N(n){var t=E(n);return function(e,r){if(u=e.classList)return r?u.add(n):u.remove(n);var u=e.getAttribute("class")||"";r?(t.lastIndex=0,t.test(u)||e.setAttribute("class",k(u+" "+n))):e.setAttribute("class",k(u.replace(t," ")))}}function z(n,t,e){function r(){this.style.removeProperty(n)}function u(){this.style.setProperty(n,t,e)}function i(){var r=t.apply(this,arguments);null==r?this.style.removeProperty(n):this.style.setProperty(n,r,e)}return null==t?r:"function"==typeof t?i:u}function L(n,t){function e(){delete this[n]}function r(){this[n]=t}function u(){var e=t.apply(this,arguments);null==e?delete this[n]:this[n]=e}return null==t?e:"function"==typeof t?u:r}function T(n){return"function"==typeof n?n:(n=Zo.ns.qualify(n)).local?function(){return this.ownerDocument.createElementNS(n.space,n.local)}:function(){return this.ownerDocument.createElementNS(this.namespaceURI,n)}}function q(n){return{__data__:n}}function R(n){return function(){return ga(this,n)}}function D(t){return arguments.length||(t=n),function(n,e){return n&&e?t(n.__data__,e.__data__):!n-!e}}function P(n,t){for(var e=0,r=n.length;r>e;e++)for(var u,i=n[e],o=0,a=i.length;a>o;o++)(u=i[o])&&t(u,o,e);return n}function U(n){return sa(n,da),n}function j(n){var t,e;return function(r,u,i){var o,a=n[i].update,c=a.length;for(i!=e&&(e=i,t=0),u>=t&&(t=u+1);!(o=a[t])&&++t<c;);return o}}function H(){var n=this.__transition__;n&&++n.active}function F(n,t,e){function r(){var t=this[o];t&&(this.removeEventListener(n,t,t.$),delete this[o])}function u(){var u=c(t,Xo(arguments));r.call(this),this.addEventListener(n,this[o]=u,u.$=e),u._=t}function i(){var t,e=new RegExp("^__on([^.]+)"+Zo.requote(n)+"$");for(var r in this)if(t=r.match(e)){var u=this[r];this.removeEventListener(t[1],u,u.$),delete this[r]}}var o="__on"+n,a=n.indexOf("."),c=O;a>0&&(n=n.substring(0,a));var s=ya.get(n);return s&&(n=s,c=Y),a?t?u:r:t?v:i}function O(n,t){return function(e){var r=Zo.event;Zo.event=e,t[0]=this.__data__;try{n.apply(this,t)}finally{Zo.event=r}}}function Y(n,t){var e=O(n,t);return function(n){var t=this,r=n.relatedTarget;r&&(r===t||8&r.compareDocumentPosition(t))||e.call(t,n)}}function I(){var n=".dragsuppress-"+ ++Ma,t="click"+n,e=Zo.select(Wo).on("touchmove"+n,y).on("dragstart"+n,y).on("selectstart"+n,y);if(xa){var r=Bo.style,u=r[xa];r[xa]="none"}return function(i){function o(){e.on(t,null)}e.on(n,null),xa&&(r[xa]=u),i&&(e.on(t,function(){y(),o()},!0),setTimeout(o,0))}}function Z(n,t){t.changedTouches&&(t=t.changedTouches[0]);var e=n.ownerSVGElement||n;if(e.createSVGPoint){var r=e.createSVGPoint();if(0>_a&&(Wo.scrollX||Wo.scrollY)){e=Zo.select("body").append("svg").style({position:"absolute",top:0,left:0,margin:0,padding:0,border:"none"},"important");var u=e[0][0].getScreenCTM();_a=!(u.f||u.e),e.remove()}return _a?(r.x=t.pageX,r.y=t.pageY):(r.x=t.clientX,r.y=t.clientY),r=r.matrixTransform(n.getScreenCTM().inverse()),[r.x,r.y]}var i=n.getBoundingClientRect();return[t.clientX-i.left-n.clientLeft,t.clientY-i.top-n.clientTop]}function V(){return Zo.event.changedTouches[0].identifier}function X(){return Zo.event.target}function $(){return Wo}function B(n){return n>0?1:0>n?-1:0}function W(n,t,e){return(t[0]-n[0])*(e[1]-n[1])-(t[1]-n[1])*(e[0]-n[0])}function J(n){return n>1?0:-1>n?ba:Math.acos(n)}function G(n){return n>1?Sa:-1>n?-Sa:Math.asin(n)}function K(n){return((n=Math.exp(n))-1/n)/2}function Q(n){return((n=Math.exp(n))+1/n)/2}function nt(n){return((n=Math.exp(2*n))-1)/(n+1)}function tt(n){return(n=Math.sin(n/2))*n}function et(){}function rt(n,t,e){return this instanceof rt?(this.h=+n,this.s=+t,void(this.l=+e)):arguments.length<2?n instanceof rt?new rt(n.h,n.s,n.l):mt(""+n,yt,rt):new rt(n,t,e)}function ut(n,t,e){function r(n){return n>360?n-=360:0>n&&(n+=360),60>n?i+(o-i)*n/60:180>n?o:240>n?i+(o-i)*(240-n)/60:i}function u(n){return Math.round(255*r(n))}var i,o;return n=isNaN(n)?0:(n%=360)<0?n+360:n,t=isNaN(t)?0:0>t?0:t>1?1:t,e=0>e?0:e>1?1:e,o=.5>=e?e*(1+t):e+t-e*t,i=2*e-o,new gt(u(n+120),u(n),u(n-120))}function it(n,t,e){return this instanceof it?(this.h=+n,this.c=+t,void(this.l=+e)):arguments.length<2?n instanceof it?new it(n.h,n.c,n.l):n instanceof at?st(n.l,n.a,n.b):st((n=xt((n=Zo.rgb(n)).r,n.g,n.b)).l,n.a,n.b):new it(n,t,e)}function ot(n,t,e){return isNaN(n)&&(n=0),isNaN(t)&&(t=0),new at(e,Math.cos(n*=Aa)*t,Math.sin(n)*t)}function at(n,t,e){return this instanceof at?(this.l=+n,this.a=+t,void(this.b=+e)):arguments.length<2?n instanceof at?new at(n.l,n.a,n.b):n instanceof it?ot(n.l,n.c,n.h):xt((n=gt(n)).r,n.g,n.b):new at(n,t,e)}function ct(n,t,e){var r=(n+16)/116,u=r+t/500,i=r-e/200;return u=lt(u)*ja,r=lt(r)*Ha,i=lt(i)*Fa,new gt(ht(3.2404542*u-1.5371385*r-.4985314*i),ht(-.969266*u+1.8760108*r+.041556*i),ht(.0556434*u-.2040259*r+1.0572252*i))}function st(n,t,e){return n>0?new it(Math.atan2(e,t)*Ca,Math.sqrt(t*t+e*e),n):new it(0/0,0/0,n)}function lt(n){return n>.206893034?n*n*n:(n-4/29)/7.787037}function ft(n){return n>.008856?Math.pow(n,1/3):7.787037*n+4/29}function ht(n){return Math.round(255*(.00304>=n?12.92*n:1.055*Math.pow(n,1/2.4)-.055))}function gt(n,t,e){return this instanceof gt?(this.r=~~n,this.g=~~t,void(this.b=~~e)):arguments.length<2?n instanceof gt?new gt(n.r,n.g,n.b):mt(""+n,gt,ut):new gt(n,t,e)}function pt(n){return new gt(n>>16,255&n>>8,255&n)}function vt(n){return pt(n)+""}function dt(n){return 16>n?"0"+Math.max(0,n).toString(16):Math.min(255,n).toString(16)}function mt(n,t,e){var r,u,i,o=0,a=0,c=0;if(r=/([a-z]+)\((.*)\)/i.exec(n))switch(u=r[2].split(","),r[1]){case"hsl":return e(parseFloat(u[0]),parseFloat(u[1])/100,parseFloat(u[2])/100);case"rgb":return t(_t(u[0]),_t(u[1]),_t(u[2]))}return(i=Ia.get(n))?t(i.r,i.g,i.b):(null==n||"#"!==n.charAt(0)||isNaN(i=parseInt(n.substring(1),16))||(4===n.length?(o=(3840&i)>>4,o=o>>4|o,a=240&i,a=a>>4|a,c=15&i,c=c<<4|c):7===n.length&&(o=(16711680&i)>>16,a=(65280&i)>>8,c=255&i)),t(o,a,c))}function yt(n,t,e){var r,u,i=Math.min(n/=255,t/=255,e/=255),o=Math.max(n,t,e),a=o-i,c=(o+i)/2;return a?(u=.5>c?a/(o+i):a/(2-o-i),r=n==o?(t-e)/a+(e>t?6:0):t==o?(e-n)/a+2:(n-t)/a+4,r*=60):(r=0/0,u=c>0&&1>c?0:r),new rt(r,u,c)}function xt(n,t,e){n=Mt(n),t=Mt(t),e=Mt(e);var r=ft((.4124564*n+.3575761*t+.1804375*e)/ja),u=ft((.2126729*n+.7151522*t+.072175*e)/Ha),i=ft((.0193339*n+.119192*t+.9503041*e)/Fa);return at(116*u-16,500*(r-u),200*(u-i))}function Mt(n){return(n/=255)<=.04045?n/12.92:Math.pow((n+.055)/1.055,2.4)}function _t(n){var t=parseFloat(n);return"%"===n.charAt(n.length-1)?Math.round(2.55*t):t}function bt(n){return"function"==typeof n?n:function(){return n}}function wt(n){return n}function St(n){return function(t,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=null),kt(t,e,n,r)}}function kt(n,t,e,r){function u(){var n,t=c.status;if(!t&&c.responseText||t>=200&&300>t||304===t){try{n=e.call(i,c)}catch(r){return o.error.call(i,r),void 0}o.load.call(i,n)}else o.error.call(i,c)}var i={},o=Zo.dispatch("beforesend","progress","load","error"),a={},c=new XMLHttpRequest,s=null;return!Wo.XDomainRequest||"withCredentials"in c||!/^(http(s)?:)?\/\//.test(n)||(c=new XDomainRequest),"onload"in c?c.onload=c.onerror=u:c.onreadystatechange=function(){c.readyState>3&&u()},c.onprogress=function(n){var t=Zo.event;Zo.event=n;try{o.progress.call(i,c)}finally{Zo.event=t}},i.header=function(n,t){return n=(n+"").toLowerCase(),arguments.length<2?a[n]:(null==t?delete a[n]:a[n]=t+"",i)},i.mimeType=function(n){return arguments.length?(t=null==n?null:n+"",i):t},i.responseType=function(n){return arguments.length?(s=n,i):s},i.response=function(n){return e=n,i},["get","post"].forEach(function(n){i[n]=function(){return i.send.apply(i,[n].concat(Xo(arguments)))}}),i.send=function(e,r,u){if(2===arguments.length&&"function"==typeof r&&(u=r,r=null),c.open(e,n,!0),null==t||"accept"in a||(a.accept=t+",*/*"),c.setRequestHeader)for(var l in a)c.setRequestHeader(l,a[l]);return null!=t&&c.overrideMimeType&&c.overrideMimeType(t),null!=s&&(c.responseType=s),null!=u&&i.on("error",u).on("load",function(n){u(null,n)}),o.beforesend.call(i,c),c.send(null==r?null:r),i},i.abort=function(){return c.abort(),i},Zo.rebind(i,o,"on"),null==r?i:i.get(Et(r))}function Et(n){return 1===n.length?function(t,e){n(null==t?e:null)}:n}function At(){var n=Ct(),t=Nt()-n;t>24?(isFinite(t)&&(clearTimeout($a),$a=setTimeout(At,t)),Xa=0):(Xa=1,Wa(At))}function Ct(){var n=Date.now();for(Ba=Za;Ba;)n>=Ba.t&&(Ba.f=Ba.c(n-Ba.t)),Ba=Ba.n;return n}function Nt(){for(var n,t=Za,e=1/0;t;)t.f?t=n?n.n=t.n:Za=t.n:(t.t<e&&(e=t.t),t=(n=t).n);return Va=n,e}function zt(n,t){return t-(n?Math.ceil(Math.log(n)/Math.LN10):1)}function Lt(n,t){var e=Math.pow(10,3*ua(8-t));return{scale:t>8?function(n){return n/e}:function(n){return n*e},symbol:n}}function Tt(n){var t=n.decimal,e=n.thousands,r=n.grouping,u=n.currency,i=r?function(n){for(var t=n.length,u=[],i=0,o=r[0];t>0&&o>0;)u.push(n.substring(t-=o,t+o)),o=r[i=(i+1)%r.length];return u.reverse().join(e)}:wt;return function(n){var e=Ga.exec(n),r=e[1]||" ",o=e[2]||">",a=e[3]||"",c=e[4]||"",s=e[5],l=+e[6],f=e[7],h=e[8],g=e[9],p=1,v="",d="",m=!1;switch(h&&(h=+h.substring(1)),(s||"0"===r&&"="===o)&&(s=r="0",o="=",f&&(l-=Math.floor((l-1)/4))),g){case"n":f=!0,g="g";break;case"%":p=100,d="%",g="f";break;case"p":p=100,d="%",g="r";break;case"b":case"o":case"x":case"X":"#"===c&&(v="0"+g.toLowerCase());case"c":case"d":m=!0,h=0;break;case"s":p=-1,g="r"}"$"===c&&(v=u[0],d=u[1]),"r"!=g||h||(g="g"),null!=h&&("g"==g?h=Math.max(1,Math.min(21,h)):("e"==g||"f"==g)&&(h=Math.max(0,Math.min(20,h)))),g=Ka.get(g)||qt;var y=s&&f;return function(n){var e=d;if(m&&n%1)return"";var u=0>n||0===n&&0>1/n?(n=-n,"-"):a;if(0>p){var c=Zo.formatPrefix(n,h);n=c.scale(n),e=c.symbol+d}else n*=p;n=g(n,h);var x=n.lastIndexOf("."),M=0>x?n:n.substring(0,x),_=0>x?"":t+n.substring(x+1);!s&&f&&(M=i(M));var b=v.length+M.length+_.length+(y?0:u.length),w=l>b?new Array(b=l-b+1).join(r):"";return y&&(M=i(w+M)),u+=v,n=M+_,("<"===o?u+n+w:">"===o?w+u+n:"^"===o?w.substring(0,b>>=1)+u+n+w.substring(b):u+(y?n:w+n))+e}}}function qt(n){return n+""}function Rt(){this._=new Date(arguments.length>1?Date.UTC.apply(this,arguments):arguments[0])}function Dt(n,t,e){function r(t){var e=n(t),r=i(e,1);return r-t>t-e?e:r}function u(e){return t(e=n(new nc(e-1)),1),e}function i(n,e){return t(n=new nc(+n),e),n}function o(n,r,i){var o=u(n),a=[];if(i>1)for(;r>o;)e(o)%i||a.push(new Date(+o)),t(o,1);else for(;r>o;)a.push(new Date(+o)),t(o,1);return a}function a(n,t,e){try{nc=Rt;var r=new Rt;return r._=n,o(r,t,e)}finally{nc=Date}}n.floor=n,n.round=r,n.ceil=u,n.offset=i,n.range=o;var c=n.utc=Pt(n);return c.floor=c,c.round=Pt(r),c.ceil=Pt(u),c.offset=Pt(i),c.range=a,n}function Pt(n){return function(t,e){try{nc=Rt;var r=new Rt;return r._=t,n(r,e)._}finally{nc=Date}}}function Ut(n){function t(n){function t(t){for(var e,u,i,o=[],a=-1,c=0;++a<r;)37===n.charCodeAt(a)&&(o.push(n.substring(c,a)),null!=(u=ec[e=n.charAt(++a)])&&(e=n.charAt(++a)),(i=C[e])&&(e=i(t,null==u?"e"===e?" ":"0":u)),o.push(e),c=a+1);return o.push(n.substring(c,a)),o.join("")}var r=n.length;return t.parse=function(t){var r={y:1900,m:0,d:1,H:0,M:0,S:0,L:0,Z:null},u=e(r,n,t,0);if(u!=t.length)return null;"p"in r&&(r.H=r.H%12+12*r.p);var i=null!=r.Z&&nc!==Rt,o=new(i?Rt:nc);return"j"in r?o.setFullYear(r.y,0,r.j):"w"in r&&("W"in r||"U"in r)?(o.setFullYear(r.y,0,1),o.setFullYear(r.y,0,"W"in r?(r.w+6)%7+7*r.W-(o.getDay()+5)%7:r.w+7*r.U-(o.getDay()+6)%7)):o.setFullYear(r.y,r.m,r.d),o.setHours(r.H+Math.floor(r.Z/100),r.M+r.Z%100,r.S,r.L),i?o._:o},t.toString=function(){return n},t}function e(n,t,e,r){for(var u,i,o,a=0,c=t.length,s=e.length;c>a;){if(r>=s)return-1;if(u=t.charCodeAt(a++),37===u){if(o=t.charAt(a++),i=N[o in ec?t.charAt(a++):o],!i||(r=i(n,e,r))<0)return-1}else if(u!=e.charCodeAt(r++))return-1}return r}function r(n,t,e){b.lastIndex=0;var r=b.exec(t.substring(e));return r?(n.w=w.get(r[0].toLowerCase()),e+r[0].length):-1}function u(n,t,e){M.lastIndex=0;var r=M.exec(t.substring(e));return r?(n.w=_.get(r[0].toLowerCase()),e+r[0].length):-1}function i(n,t,e){E.lastIndex=0;var r=E.exec(t.substring(e));return r?(n.m=A.get(r[0].toLowerCase()),e+r[0].length):-1}function o(n,t,e){S.lastIndex=0;var r=S.exec(t.substring(e));return r?(n.m=k.get(r[0].toLowerCase()),e+r[0].length):-1}function a(n,t,r){return e(n,C.c.toString(),t,r)}function c(n,t,r){return e(n,C.x.toString(),t,r)}function s(n,t,r){return e(n,C.X.toString(),t,r)}function l(n,t,e){var r=x.get(t.substring(e,e+=2).toLowerCase());return null==r?-1:(n.p=r,e)}var f=n.dateTime,h=n.date,g=n.time,p=n.periods,v=n.days,d=n.shortDays,m=n.months,y=n.shortMonths;t.utc=function(n){function e(n){try{nc=Rt;var t=new nc;return t._=n,r(t)}finally{nc=Date}}var r=t(n);return e.parse=function(n){try{nc=Rt;var t=r.parse(n);return t&&t._}finally{nc=Date}},e.toString=r.toString,e},t.multi=t.utc.multi=re;var x=Zo.map(),M=Ht(v),_=Ft(v),b=Ht(d),w=Ft(d),S=Ht(m),k=Ft(m),E=Ht(y),A=Ft(y);p.forEach(function(n,t){x.set(n.toLowerCase(),t)});var C={a:function(n){return d[n.getDay()]},A:function(n){return v[n.getDay()]},b:function(n){return y[n.getMonth()]},B:function(n){return m[n.getMonth()]},c:t(f),d:function(n,t){return jt(n.getDate(),t,2)},e:function(n,t){return jt(n.getDate(),t,2)},H:function(n,t){return jt(n.getHours(),t,2)},I:function(n,t){return jt(n.getHours()%12||12,t,2)},j:function(n,t){return jt(1+Qa.dayOfYear(n),t,3)},L:function(n,t){return jt(n.getMilliseconds(),t,3)},m:function(n,t){return jt(n.getMonth()+1,t,2)},M:function(n,t){return jt(n.getMinutes(),t,2)},p:function(n){return p[+(n.getHours()>=12)]},S:function(n,t){return jt(n.getSeconds(),t,2)},U:function(n,t){return jt(Qa.sundayOfYear(n),t,2)},w:function(n){return n.getDay()},W:function(n,t){return jt(Qa.mondayOfYear(n),t,2)},x:t(h),X:t(g),y:function(n,t){return jt(n.getFullYear()%100,t,2)},Y:function(n,t){return jt(n.getFullYear()%1e4,t,4)},Z:te,"%":function(){return"%"}},N={a:r,A:u,b:i,B:o,c:a,d:Wt,e:Wt,H:Gt,I:Gt,j:Jt,L:ne,m:Bt,M:Kt,p:l,S:Qt,U:Yt,w:Ot,W:It,x:c,X:s,y:Vt,Y:Zt,Z:Xt,"%":ee};return t}function jt(n,t,e){var r=0>n?"-":"",u=(r?-n:n)+"",i=u.length;return r+(e>i?new Array(e-i+1).join(t)+u:u)}function Ht(n){return new RegExp("^(?:"+n.map(Zo.requote).join("|")+")","i")}function Ft(n){for(var t=new o,e=-1,r=n.length;++e<r;)t.set(n[e].toLowerCase(),e);return t}function Ot(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+1));return r?(n.w=+r[0],e+r[0].length):-1}function Yt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e));return r?(n.U=+r[0],e+r[0].length):-1}function It(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e));return r?(n.W=+r[0],e+r[0].length):-1}function Zt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+4));return r?(n.y=+r[0],e+r[0].length):-1}function Vt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+2));return r?(n.y=$t(+r[0]),e+r[0].length):-1}function Xt(n,t,e){return/^[+-]\d{4}$/.test(t=t.substring(e,e+5))?(n.Z=-t,e+5):-1}function $t(n){return n+(n>68?1900:2e3)}function Bt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+2));return r?(n.m=r[0]-1,e+r[0].length):-1}function Wt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+2));return r?(n.d=+r[0],e+r[0].length):-1}function Jt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+3));return r?(n.j=+r[0],e+r[0].length):-1}function Gt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+2));return r?(n.H=+r[0],e+r[0].length):-1}function Kt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+2));return r?(n.M=+r[0],e+r[0].length):-1}function Qt(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+2));return r?(n.S=+r[0],e+r[0].length):-1}function ne(n,t,e){rc.lastIndex=0;var r=rc.exec(t.substring(e,e+3));return r?(n.L=+r[0],e+r[0].length):-1}function te(n){var t=n.getTimezoneOffset(),e=t>0?"-":"+",r=~~(ua(t)/60),u=ua(t)%60;return e+jt(r,"0",2)+jt(u,"0",2)}function ee(n,t,e){uc.lastIndex=0;var r=uc.exec(t.substring(e,e+1));return r?e+r[0].length:-1}function re(n){for(var t=n.length,e=-1;++e<t;)n[e][0]=this(n[e][0]);return function(t){for(var e=0,r=n[e];!r[1](t);)r=n[++e];return r[0](t)}}function ue(){}function ie(n,t,e){var r=e.s=n+t,u=r-n,i=r-u;e.t=n-i+(t-u)}function oe(n,t){n&&cc.hasOwnProperty(n.type)&&cc[n.type](n,t)}function ae(n,t,e){var r,u=-1,i=n.length-e;for(t.lineStart();++u<i;)r=n[u],t.point(r[0],r[1],r[2]);t.lineEnd()}function ce(n,t){var e=-1,r=n.length;for(t.polygonStart();++e<r;)ae(n[e],t,1);t.polygonEnd()}function se(){function n(n,t){n*=Aa,t=t*Aa/2+ba/4;var e=n-r,o=e>=0?1:-1,a=o*e,c=Math.cos(t),s=Math.sin(t),l=i*s,f=u*c+l*Math.cos(a),h=l*o*Math.sin(a);lc.add(Math.atan2(h,f)),r=n,u=c,i=s}var t,e,r,u,i;fc.point=function(o,a){fc.point=n,r=(t=o)*Aa,u=Math.cos(a=(e=a)*Aa/2+ba/4),i=Math.sin(a)},fc.lineEnd=function(){n(t,e)}}function le(n){var t=n[0],e=n[1],r=Math.cos(e);return[r*Math.cos(t),r*Math.sin(t),Math.sin(e)]}function fe(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]}function he(n,t){return[n[1]*t[2]-n[2]*t[1],n[2]*t[0]-n[0]*t[2],n[0]*t[1]-n[1]*t[0]]}function ge(n,t){n[0]+=t[0],n[1]+=t[1],n[2]+=t[2]}function pe(n,t){return[n[0]*t,n[1]*t,n[2]*t]}function ve(n){var t=Math.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2]);n[0]/=t,n[1]/=t,n[2]/=t}function de(n){return[Math.atan2(n[1],n[0]),G(n[2])]}function me(n,t){return ua(n[0]-t[0])<ka&&ua(n[1]-t[1])<ka}function ye(n,t){n*=Aa;var e=Math.cos(t*=Aa);xe(e*Math.cos(n),e*Math.sin(n),Math.sin(t))}function xe(n,t,e){++hc,pc+=(n-pc)/hc,vc+=(t-vc)/hc,dc+=(e-dc)/hc}function Me(){function n(n,u){n*=Aa;var i=Math.cos(u*=Aa),o=i*Math.cos(n),a=i*Math.sin(n),c=Math.sin(u),s=Math.atan2(Math.sqrt((s=e*c-r*a)*s+(s=r*o-t*c)*s+(s=t*a-e*o)*s),t*o+e*a+r*c);gc+=s,mc+=s*(t+(t=o)),yc+=s*(e+(e=a)),xc+=s*(r+(r=c)),xe(t,e,r)}var t,e,r;wc.point=function(u,i){u*=Aa;var o=Math.cos(i*=Aa);t=o*Math.cos(u),e=o*Math.sin(u),r=Math.sin(i),wc.point=n,xe(t,e,r)}}function _e(){wc.point=ye}function be(){function n(n,t){n*=Aa;var e=Math.cos(t*=Aa),o=e*Math.cos(n),a=e*Math.sin(n),c=Math.sin(t),s=u*c-i*a,l=i*o-r*c,f=r*a-u*o,h=Math.sqrt(s*s+l*l+f*f),g=r*o+u*a+i*c,p=h&&-J(g)/h,v=Math.atan2(h,g);Mc+=p*s,_c+=p*l,bc+=p*f,gc+=v,mc+=v*(r+(r=o)),yc+=v*(u+(u=a)),xc+=v*(i+(i=c)),xe(r,u,i)}var t,e,r,u,i;wc.point=function(o,a){t=o,e=a,wc.point=n,o*=Aa;var c=Math.cos(a*=Aa);r=c*Math.cos(o),u=c*Math.sin(o),i=Math.sin(a),xe(r,u,i)},wc.lineEnd=function(){n(t,e),wc.lineEnd=_e,wc.point=ye}}function we(){return!0}function Se(n,t,e,r,u){var i=[],o=[];if(n.forEach(function(n){if(!((t=n.length-1)<=0)){var t,e=n[0],r=n[t];if(me(e,r)){u.lineStart();for(var a=0;t>a;++a)u.point((e=n[a])[0],e[1]);return u.lineEnd(),void 0}var c=new Ee(e,n,null,!0),s=new Ee(e,null,c,!1);c.o=s,i.push(c),o.push(s),c=new Ee(r,n,null,!1),s=new Ee(r,null,c,!0),c.o=s,i.push(c),o.push(s)}}),o.sort(t),ke(i),ke(o),i.length){for(var a=0,c=e,s=o.length;s>a;++a)o[a].e=c=!c;for(var l,f,h=i[0];;){for(var g=h,p=!0;g.v;)if((g=g.n)===h)return;l=g.z,u.lineStart();do{if(g.v=g.o.v=!0,g.e){if(p)for(var a=0,s=l.length;s>a;++a)u.point((f=l[a])[0],f[1]);else r(g.x,g.n.x,1,u);g=g.n}else{if(p){l=g.p.z;for(var a=l.length-1;a>=0;--a)u.point((f=l[a])[0],f[1])}else r(g.x,g.p.x,-1,u);g=g.p}g=g.o,l=g.z,p=!p}while(!g.v);u.lineEnd()}}}function ke(n){if(t=n.length){for(var t,e,r=0,u=n[0];++r<t;)u.n=e=n[r],e.p=u,u=e;u.n=e=n[0],e.p=u}}function Ee(n,t,e,r){this.x=n,this.z=t,this.o=e,this.e=r,this.v=!1,this.n=this.p=null}function Ae(n,t,e,r){return function(u,i){function o(t,e){var r=u(t,e);n(t=r[0],e=r[1])&&i.point(t,e)}function a(n,t){var e=u(n,t);d.point(e[0],e[1])}function c(){y.point=a,d.lineStart()}function s(){y.point=o,d.lineEnd()}function l(n,t){v.push([n,t]);var e=u(n,t);M.point(e[0],e[1])}function f(){M.lineStart(),v=[]}function h(){l(v[0][0],v[0][1]),M.lineEnd();var n,t=M.clean(),e=x.buffer(),r=e.length;if(v.pop(),p.push(v),v=null,r)if(1&t){n=e[0];var u,r=n.length-1,o=-1;if(r>0){for(_||(i.polygonStart(),_=!0),i.lineStart();++o<r;)i.point((u=n[o])[0],u[1]);i.lineEnd()}}else r>1&&2&t&&e.push(e.pop().concat(e.shift())),g.push(e.filter(Ce))}var g,p,v,d=t(i),m=u.invert(r[0],r[1]),y={point:o,lineStart:c,lineEnd:s,polygonStart:function(){y.point=l,y.lineStart=f,y.lineEnd=h,g=[],p=[]},polygonEnd:function(){y.point=o,y.lineStart=c,y.lineEnd=s,g=Zo.merge(g);var n=Le(m,p);g.length?(_||(i.polygonStart(),_=!0),Se(g,ze,n,e,i)):n&&(_||(i.polygonStart(),_=!0),i.lineStart(),e(null,null,1,i),i.lineEnd()),_&&(i.polygonEnd(),_=!1),g=p=null},sphere:function(){i.polygonStart(),i.lineStart(),e(null,null,1,i),i.lineEnd(),i.polygonEnd()}},x=Ne(),M=t(x),_=!1;return y}}function Ce(n){return n.length>1}function Ne(){var n,t=[];return{lineStart:function(){t.push(n=[])},point:function(t,e){n.push([t,e])},lineEnd:v,buffer:function(){var e=t;return t=[],n=null,e},rejoin:function(){t.length>1&&t.push(t.pop().concat(t.shift()))}}}function ze(n,t){return((n=n.x)[0]<0?n[1]-Sa-ka:Sa-n[1])-((t=t.x)[0]<0?t[1]-Sa-ka:Sa-t[1])}function Le(n,t){var e=n[0],r=n[1],u=[Math.sin(e),-Math.cos(e),0],i=0,o=0;lc.reset();for(var a=0,c=t.length;c>a;++a){var s=t[a],l=s.length;if(l)for(var f=s[0],h=f[0],g=f[1]/2+ba/4,p=Math.sin(g),v=Math.cos(g),d=1;;){d===l&&(d=0),n=s[d];var m=n[0],y=n[1]/2+ba/4,x=Math.sin(y),M=Math.cos(y),_=m-h,b=_>=0?1:-1,w=b*_,S=w>ba,k=p*x;if(lc.add(Math.atan2(k*b*Math.sin(w),v*M+k*Math.cos(w))),i+=S?_+b*wa:_,S^h>=e^m>=e){var E=he(le(f),le(n));ve(E);var A=he(u,E);ve(A);var C=(S^_>=0?-1:1)*G(A[2]);(r>C||r===C&&(E[0]||E[1]))&&(o+=S^_>=0?1:-1)}if(!d++)break;h=m,p=x,v=M,f=n}}return(-ka>i||ka>i&&0>lc)^1&o}function Te(n){var t,e=0/0,r=0/0,u=0/0;return{lineStart:function(){n.lineStart(),t=1},point:function(i,o){var a=i>0?ba:-ba,c=ua(i-e);ua(c-ba)<ka?(n.point(e,r=(r+o)/2>0?Sa:-Sa),n.point(u,r),n.lineEnd(),n.lineStart(),n.point(a,r),n.point(i,r),t=0):u!==a&&c>=ba&&(ua(e-u)<ka&&(e-=u*ka),ua(i-a)<ka&&(i-=a*ka),r=qe(e,r,i,o),n.point(u,r),n.lineEnd(),n.lineStart(),n.point(a,r),t=0),n.point(e=i,r=o),u=a},lineEnd:function(){n.lineEnd(),e=r=0/0},clean:function(){return 2-t}}}function qe(n,t,e,r){var u,i,o=Math.sin(n-e);return ua(o)>ka?Math.atan((Math.sin(t)*(i=Math.cos(r))*Math.sin(e)-Math.sin(r)*(u=Math.cos(t))*Math.sin(n))/(u*i*o)):(t+r)/2}function Re(n,t,e,r){var u;if(null==n)u=e*Sa,r.point(-ba,u),r.point(0,u),r.point(ba,u),r.point(ba,0),r.point(ba,-u),r.point(0,-u),r.point(-ba,-u),r.point(-ba,0),r.point(-ba,u);else if(ua(n[0]-t[0])>ka){var i=n[0]<t[0]?ba:-ba;u=e*i/2,r.point(-i,u),r.point(0,u),r.point(i,u)}else r.point(t[0],t[1])}function De(n){function t(n,t){return Math.cos(n)*Math.cos(t)>i}function e(n){var e,i,c,s,l;return{lineStart:function(){s=c=!1,l=1},point:function(f,h){var g,p=[f,h],v=t(f,h),d=o?v?0:u(f,h):v?u(f+(0>f?ba:-ba),h):0;if(!e&&(s=c=v)&&n.lineStart(),v!==c&&(g=r(e,p),(me(e,g)||me(p,g))&&(p[0]+=ka,p[1]+=ka,v=t(p[0],p[1]))),v!==c)l=0,v?(n.lineStart(),g=r(p,e),n.point(g[0],g[1])):(g=r(e,p),n.point(g[0],g[1]),n.lineEnd()),e=g;else if(a&&e&&o^v){var m;d&i||!(m=r(p,e,!0))||(l=0,o?(n.lineStart(),n.point(m[0][0],m[0][1]),n.point(m[1][0],m[1][1]),n.lineEnd()):(n.point(m[1][0],m[1][1]),n.lineEnd(),n.lineStart(),n.point(m[0][0],m[0][1])))}!v||e&&me(e,p)||n.point(p[0],p[1]),e=p,c=v,i=d},lineEnd:function(){c&&n.lineEnd(),e=null},clean:function(){return l|(s&&c)<<1}}}function r(n,t,e){var r=le(n),u=le(t),o=[1,0,0],a=he(r,u),c=fe(a,a),s=a[0],l=c-s*s;if(!l)return!e&&n;var f=i*c/l,h=-i*s/l,g=he(o,a),p=pe(o,f),v=pe(a,h);ge(p,v);var d=g,m=fe(p,d),y=fe(d,d),x=m*m-y*(fe(p,p)-1);if(!(0>x)){var M=Math.sqrt(x),_=pe(d,(-m-M)/y);if(ge(_,p),_=de(_),!e)return _;var b,w=n[0],S=t[0],k=n[1],E=t[1];w>S&&(b=w,w=S,S=b);var A=S-w,C=ua(A-ba)<ka,N=C||ka>A;if(!C&&k>E&&(b=k,k=E,E=b),N?C?k+E>0^_[1]<(ua(_[0]-w)<ka?k:E):k<=_[1]&&_[1]<=E:A>ba^(w<=_[0]&&_[0]<=S)){var z=pe(d,(-m+M)/y);return ge(z,p),[_,de(z)]}}}function u(t,e){var r=o?n:ba-n,u=0;return-r>t?u|=1:t>r&&(u|=2),-r>e?u|=4:e>r&&(u|=8),u}var i=Math.cos(n),o=i>0,a=ua(i)>ka,c=sr(n,6*Aa);return Ae(t,e,c,o?[0,-n]:[-ba,n-ba])}function Pe(n,t,e,r){return function(u){var i,o=u.a,a=u.b,c=o.x,s=o.y,l=a.x,f=a.y,h=0,g=1,p=l-c,v=f-s;if(i=n-c,p||!(i>0)){if(i/=p,0>p){if(h>i)return;g>i&&(g=i)}else if(p>0){if(i>g)return;i>h&&(h=i)}if(i=e-c,p||!(0>i)){if(i/=p,0>p){if(i>g)return;i>h&&(h=i)}else if(p>0){if(h>i)return;g>i&&(g=i)}if(i=t-s,v||!(i>0)){if(i/=v,0>v){if(h>i)return;g>i&&(g=i)}else if(v>0){if(i>g)return;i>h&&(h=i)}if(i=r-s,v||!(0>i)){if(i/=v,0>v){if(i>g)return;i>h&&(h=i)}else if(v>0){if(h>i)return;g>i&&(g=i)}return h>0&&(u.a={x:c+h*p,y:s+h*v}),1>g&&(u.b={x:c+g*p,y:s+g*v}),u}}}}}}function Ue(n,t,e,r){function u(r,u){return ua(r[0]-n)<ka?u>0?0:3:ua(r[0]-e)<ka?u>0?2:1:ua(r[1]-t)<ka?u>0?1:0:u>0?3:2}function i(n,t){return o(n.x,t.x)}function o(n,t){var e=u(n,1),r=u(t,1);return e!==r?e-r:0===e?t[1]-n[1]:1===e?n[0]-t[0]:2===e?n[1]-t[1]:t[0]-n[0]}return function(a){function c(n){for(var t=0,e=d.length,r=n[1],u=0;e>u;++u)for(var i,o=1,a=d[u],c=a.length,s=a[0];c>o;++o)i=a[o],s[1]<=r?i[1]>r&&W(s,i,n)>0&&++t:i[1]<=r&&W(s,i,n)<0&&--t,s=i;return 0!==t}function s(i,a,c,s){var l=0,f=0;if(null==i||(l=u(i,c))!==(f=u(a,c))||o(i,a)<0^c>0){do s.point(0===l||3===l?n:e,l>1?r:t);while((l=(l+c+4)%4)!==f)}else s.point(a[0],a[1])}function l(u,i){return u>=n&&e>=u&&i>=t&&r>=i}function f(n,t){l(n,t)&&a.point(n,t)}function h(){N.point=p,d&&d.push(m=[]),S=!0,w=!1,_=b=0/0}function g(){v&&(p(y,x),M&&w&&A.rejoin(),v.push(A.buffer())),N.point=f,w&&a.lineEnd()}function p(n,t){n=Math.max(-kc,Math.min(kc,n)),t=Math.max(-kc,Math.min(kc,t));var e=l(n,t);if(d&&m.push([n,t]),S)y=n,x=t,M=e,S=!1,e&&(a.lineStart(),a.point(n,t));else if(e&&w)a.point(n,t);else{var r={a:{x:_,y:b},b:{x:n,y:t}};C(r)?(w||(a.lineStart(),a.point(r.a.x,r.a.y)),a.point(r.b.x,r.b.y),e||a.lineEnd(),k=!1):e&&(a.lineStart(),a.point(n,t),k=!1)}_=n,b=t,w=e}var v,d,m,y,x,M,_,b,w,S,k,E=a,A=Ne(),C=Pe(n,t,e,r),N={point:f,lineStart:h,lineEnd:g,polygonStart:function(){a=A,v=[],d=[],k=!0},polygonEnd:function(){a=E,v=Zo.merge(v);var t=c([n,r]),e=k&&t,u=v.length;(e||u)&&(a.polygonStart(),e&&(a.lineStart(),s(null,null,1,a),a.lineEnd()),u&&Se(v,i,t,s,a),a.polygonEnd()),v=d=m=null}};return N}}function je(n,t){function e(e,r){return e=n(e,r),t(e[0],e[1])}return n.invert&&t.invert&&(e.invert=function(e,r){return e=t.invert(e,r),e&&n.invert(e[0],e[1])}),e}function He(n){var t=0,e=ba/3,r=tr(n),u=r(t,e);return u.parallels=function(n){return arguments.length?r(t=n[0]*ba/180,e=n[1]*ba/180):[180*(t/ba),180*(e/ba)]},u}function Fe(n,t){function e(n,t){var e=Math.sqrt(i-2*u*Math.sin(t))/u;return[e*Math.sin(n*=u),o-e*Math.cos(n)]}var r=Math.sin(n),u=(r+Math.sin(t))/2,i=1+r*(2*u-r),o=Math.sqrt(i)/u;return e.invert=function(n,t){var e=o-t;return[Math.atan2(n,e)/u,G((i-(n*n+e*e)*u*u)/(2*u))]},e}function Oe(){function n(n,t){Ac+=u*n-r*t,r=n,u=t}var t,e,r,u;Tc.point=function(i,o){Tc.point=n,t=r=i,e=u=o},Tc.lineEnd=function(){n(t,e)}}function Ye(n,t){Cc>n&&(Cc=n),n>zc&&(zc=n),Nc>t&&(Nc=t),t>Lc&&(Lc=t)}function Ie(){function n(n,t){o.push("M",n,",",t,i)}function t(n,t){o.push("M",n,",",t),a.point=e}function e(n,t){o.push("L",n,",",t)}function r(){a.point=n}function u(){o.push("Z")}var i=Ze(4.5),o=[],a={point:n,lineStart:function(){a.point=t},lineEnd:r,polygonStart:function(){a.lineEnd=u},polygonEnd:function(){a.lineEnd=r,a.point=n},pointRadius:function(n){return i=Ze(n),a},result:function(){if(o.length){var n=o.join("");return o=[],n}}};return a}function Ze(n){return"m0,"+n+"a"+n+","+n+" 0 1,1 0,"+-2*n+"a"+n+","+n+" 0 1,1 0,"+2*n+"z"}function Ve(n,t){pc+=n,vc+=t,++dc}function Xe(){function n(n,r){var u=n-t,i=r-e,o=Math.sqrt(u*u+i*i);mc+=o*(t+n)/2,yc+=o*(e+r)/2,xc+=o,Ve(t=n,e=r)}var t,e;Rc.point=function(r,u){Rc.point=n,Ve(t=r,e=u)}}function $e(){Rc.point=Ve}function Be(){function n(n,t){var e=n-r,i=t-u,o=Math.sqrt(e*e+i*i);mc+=o*(r+n)/2,yc+=o*(u+t)/2,xc+=o,o=u*n-r*t,Mc+=o*(r+n),_c+=o*(u+t),bc+=3*o,Ve(r=n,u=t)}var t,e,r,u;Rc.point=function(i,o){Rc.point=n,Ve(t=r=i,e=u=o)},Rc.lineEnd=function(){n(t,e)}}function We(n){function t(t,e){n.moveTo(t,e),n.arc(t,e,o,0,wa)}function e(t,e){n.moveTo(t,e),a.point=r}function r(t,e){n.lineTo(t,e)}function u(){a.point=t}function i(){n.closePath()}var o=4.5,a={point:t,lineStart:function(){a.point=e},lineEnd:u,polygonStart:function(){a.lineEnd=i},polygonEnd:function(){a.lineEnd=u,a.point=t},pointRadius:function(n){return o=n,a},result:v};return a}function Je(n){function t(n){return(a?r:e)(n)}function e(t){return Qe(t,function(e,r){e=n(e,r),t.point(e[0],e[1])})}function r(t){function e(e,r){e=n(e,r),t.point(e[0],e[1])}function r(){x=0/0,S.point=i,t.lineStart()}function i(e,r){var i=le([e,r]),o=n(e,r);u(x,M,y,_,b,w,x=o[0],M=o[1],y=e,_=i[0],b=i[1],w=i[2],a,t),t.point(x,M)}function o(){S.point=e,t.lineEnd()}function c(){r(),S.point=s,S.lineEnd=l}function s(n,t){i(f=n,h=t),g=x,p=M,v=_,d=b,m=w,S.point=i}function l(){u(x,M,y,_,b,w,g,p,f,v,d,m,a,t),S.lineEnd=o,o()}var f,h,g,p,v,d,m,y,x,M,_,b,w,S={point:e,lineStart:r,lineEnd:o,polygonStart:function(){t.polygonStart(),S.lineStart=c},polygonEnd:function(){t.polygonEnd(),S.lineStart=r}};return S}function u(t,e,r,a,c,s,l,f,h,g,p,v,d,m){var y=l-t,x=f-e,M=y*y+x*x;if(M>4*i&&d--){var _=a+g,b=c+p,w=s+v,S=Math.sqrt(_*_+b*b+w*w),k=Math.asin(w/=S),E=ua(ua(w)-1)<ka||ua(r-h)<ka?(r+h)/2:Math.atan2(b,_),A=n(E,k),C=A[0],N=A[1],z=C-t,L=N-e,T=x*z-y*L;(T*T/M>i||ua((y*z+x*L)/M-.5)>.3||o>a*g+c*p+s*v)&&(u(t,e,r,a,c,s,C,N,E,_/=S,b/=S,w,d,m),m.point(C,N),u(C,N,E,_,b,w,l,f,h,g,p,v,d,m))}}var i=.5,o=Math.cos(30*Aa),a=16;
+return t.precision=function(n){return arguments.length?(a=(i=n*n)>0&&16,t):Math.sqrt(i)},t}function Ge(n){var t=Je(function(t,e){return n([t*Ca,e*Ca])});return function(n){return er(t(n))}}function Ke(n){this.stream=n}function Qe(n,t){return{point:t,sphere:function(){n.sphere()},lineStart:function(){n.lineStart()},lineEnd:function(){n.lineEnd()},polygonStart:function(){n.polygonStart()},polygonEnd:function(){n.polygonEnd()}}}function nr(n){return tr(function(){return n})()}function tr(n){function t(n){return n=a(n[0]*Aa,n[1]*Aa),[n[0]*h+c,s-n[1]*h]}function e(n){return n=a.invert((n[0]-c)/h,(s-n[1])/h),n&&[n[0]*Ca,n[1]*Ca]}function r(){a=je(o=ir(m,y,x),i);var n=i(v,d);return c=g-n[0]*h,s=p+n[1]*h,u()}function u(){return l&&(l.valid=!1,l=null),t}var i,o,a,c,s,l,f=Je(function(n,t){return n=i(n,t),[n[0]*h+c,s-n[1]*h]}),h=150,g=480,p=250,v=0,d=0,m=0,y=0,x=0,M=Sc,_=wt,b=null,w=null;return t.stream=function(n){return l&&(l.valid=!1),l=er(M(o,f(_(n)))),l.valid=!0,l},t.clipAngle=function(n){return arguments.length?(M=null==n?(b=n,Sc):De((b=+n)*Aa),u()):b},t.clipExtent=function(n){return arguments.length?(w=n,_=n?Ue(n[0][0],n[0][1],n[1][0],n[1][1]):wt,u()):w},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return arguments.length?(g=+n[0],p=+n[1],r()):[g,p]},t.center=function(n){return arguments.length?(v=n[0]%360*Aa,d=n[1]%360*Aa,r()):[v*Ca,d*Ca]},t.rotate=function(n){return arguments.length?(m=n[0]%360*Aa,y=n[1]%360*Aa,x=n.length>2?n[2]%360*Aa:0,r()):[m*Ca,y*Ca,x*Ca]},Zo.rebind(t,f,"precision"),function(){return i=n.apply(this,arguments),t.invert=i.invert&&e,r()}}function er(n){return Qe(n,function(t,e){n.point(t*Aa,e*Aa)})}function rr(n,t){return[n,t]}function ur(n,t){return[n>ba?n-wa:-ba>n?n+wa:n,t]}function ir(n,t,e){return n?t||e?je(ar(n),cr(t,e)):ar(n):t||e?cr(t,e):ur}function or(n){return function(t,e){return t+=n,[t>ba?t-wa:-ba>t?t+wa:t,e]}}function ar(n){var t=or(n);return t.invert=or(-n),t}function cr(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,s=Math.sin(t),l=s*r+a*u;return[Math.atan2(c*i-l*o,a*r-s*u),G(l*i+c*o)]}var r=Math.cos(n),u=Math.sin(n),i=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,s=Math.sin(t),l=s*i-c*o;return[Math.atan2(c*i+s*o,a*r+l*u),G(l*r-a*u)]},e}function sr(n,t){var e=Math.cos(n),r=Math.sin(n);return function(u,i,o,a){var c=o*t;null!=u?(u=lr(e,u),i=lr(e,i),(o>0?i>u:u>i)&&(u+=o*wa)):(u=n+o*wa,i=n-.5*c);for(var s,l=u;o>0?l>i:i>l;l-=c)a.point((s=de([e,-r*Math.cos(l),-r*Math.sin(l)]))[0],s[1])}}function lr(n,t){var e=le(t);e[0]-=n,ve(e);var r=J(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-ka)%(2*Math.PI)}function fr(n,t,e){var r=Zo.range(n,t-ka,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function hr(n,t,e){var r=Zo.range(n,t-ka,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function gr(n){return n.source}function pr(n){return n.target}function vr(n,t,e,r){var u=Math.cos(t),i=Math.sin(t),o=Math.cos(r),a=Math.sin(r),c=u*Math.cos(n),s=u*Math.sin(n),l=o*Math.cos(e),f=o*Math.sin(e),h=2*Math.asin(Math.sqrt(tt(r-t)+u*o*tt(e-n))),g=1/Math.sin(h),p=h?function(n){var t=Math.sin(n*=h)*g,e=Math.sin(h-n)*g,r=e*c+t*l,u=e*s+t*f,o=e*i+t*a;return[Math.atan2(u,r)*Ca,Math.atan2(o,Math.sqrt(r*r+u*u))*Ca]}:function(){return[n*Ca,t*Ca]};return p.distance=h,p}function dr(){function n(n,u){var i=Math.sin(u*=Aa),o=Math.cos(u),a=ua((n*=Aa)-t),c=Math.cos(a);Dc+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*i-e*o*c)*a),e*i+r*o*c),t=n,e=i,r=o}var t,e,r;Pc.point=function(u,i){t=u*Aa,e=Math.sin(i*=Aa),r=Math.cos(i),Pc.point=n},Pc.lineEnd=function(){Pc.point=Pc.lineEnd=v}}function mr(n,t){function e(t,e){var r=Math.cos(t),u=Math.cos(e),i=n(r*u);return[i*u*Math.sin(t),i*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),u=t(r),i=Math.sin(u),o=Math.cos(u);return[Math.atan2(n*i,r*o),Math.asin(r&&e*i/r)]},e}function yr(n,t){function e(n,t){o>0?-Sa+ka>t&&(t=-Sa+ka):t>Sa-ka&&(t=Sa-ka);var e=o/Math.pow(u(t),i);return[e*Math.sin(i*n),o-e*Math.cos(i*n)]}var r=Math.cos(n),u=function(n){return Math.tan(ba/4+n/2)},i=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(u(t)/u(n)),o=r*Math.pow(u(n),i)/i;return i?(e.invert=function(n,t){var e=o-t,r=B(i)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/i,2*Math.atan(Math.pow(o/r,1/i))-Sa]},e):Mr}function xr(n,t){function e(n,t){var e=i-t;return[e*Math.sin(u*n),i-e*Math.cos(u*n)]}var r=Math.cos(n),u=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),i=r/u+n;return ua(u)<ka?rr:(e.invert=function(n,t){var e=i-t;return[Math.atan2(n,e)/u,i-B(u)*Math.sqrt(n*n+e*e)]},e)}function Mr(n,t){return[n,Math.log(Math.tan(ba/4+t/2))]}function _r(n){var t,e=nr(n),r=e.scale,u=e.translate,i=e.clipExtent;return e.scale=function(){var n=r.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.translate=function(){var n=u.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.clipExtent=function(n){var o=i.apply(e,arguments);if(o===e){if(t=null==n){var a=ba*r(),c=u();i([[c[0]-a,c[1]-a],[c[0]+a,c[1]+a]])}}else t&&(o=null);return o},e.clipExtent(null)}function br(n,t){return[Math.log(Math.tan(ba/4+t/2)),-n]}function wr(n){return n[0]}function Sr(n){return n[1]}function kr(n){for(var t=n.length,e=[0,1],r=2,u=2;t>u;u++){for(;r>1&&W(n[e[r-2]],n[e[r-1]],n[u])<=0;)--r;e[r++]=u}return e.slice(0,r)}function Er(n,t){return n[0]-t[0]||n[1]-t[1]}function Ar(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function Cr(n,t,e,r){var u=n[0],i=e[0],o=t[0]-u,a=r[0]-i,c=n[1],s=e[1],l=t[1]-c,f=r[1]-s,h=(a*(c-s)-f*(u-i))/(f*o-a*l);return[u+h*o,c+h*l]}function Nr(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function zr(){Gr(this),this.edge=this.site=this.circle=null}function Lr(n){var t=Bc.pop()||new zr;return t.site=n,t}function Tr(n){Yr(n),Vc.remove(n),Bc.push(n),Gr(n)}function qr(n){var t=n.circle,e=t.x,r=t.cy,u={x:e,y:r},i=n.P,o=n.N,a=[n];Tr(n);for(var c=i;c.circle&&ua(e-c.circle.x)<ka&&ua(r-c.circle.cy)<ka;)i=c.P,a.unshift(c),Tr(c),c=i;a.unshift(c),Yr(c);for(var s=o;s.circle&&ua(e-s.circle.x)<ka&&ua(r-s.circle.cy)<ka;)o=s.N,a.push(s),Tr(s),s=o;a.push(s),Yr(s);var l,f=a.length;for(l=1;f>l;++l)s=a[l],c=a[l-1],Br(s.edge,c.site,s.site,u);c=a[0],s=a[f-1],s.edge=Xr(c.site,s.site,null,u),Or(c),Or(s)}function Rr(n){for(var t,e,r,u,i=n.x,o=n.y,a=Vc._;a;)if(r=Dr(a,o)-i,r>ka)a=a.L;else{if(u=i-Pr(a,o),!(u>ka)){r>-ka?(t=a.P,e=a):u>-ka?(t=a,e=a.N):t=e=a;break}if(!a.R){t=a;break}a=a.R}var c=Lr(n);if(Vc.insert(t,c),t||e){if(t===e)return Yr(t),e=Lr(t.site),Vc.insert(c,e),c.edge=e.edge=Xr(t.site,c.site),Or(t),Or(e),void 0;if(!e)return c.edge=Xr(t.site,c.site),void 0;Yr(t),Yr(e);var s=t.site,l=s.x,f=s.y,h=n.x-l,g=n.y-f,p=e.site,v=p.x-l,d=p.y-f,m=2*(h*d-g*v),y=h*h+g*g,x=v*v+d*d,M={x:(d*y-g*x)/m+l,y:(h*x-v*y)/m+f};Br(e.edge,s,p,M),c.edge=Xr(s,n,null,M),e.edge=Xr(n,p,null,M),Or(t),Or(e)}}function Dr(n,t){var e=n.site,r=e.x,u=e.y,i=u-t;if(!i)return r;var o=n.P;if(!o)return-1/0;e=o.site;var a=e.x,c=e.y,s=c-t;if(!s)return a;var l=a-r,f=1/i-1/s,h=l/s;return f?(-h+Math.sqrt(h*h-2*f*(l*l/(-2*s)-c+s/2+u-i/2)))/f+r:(r+a)/2}function Pr(n,t){var e=n.N;if(e)return Dr(e,t);var r=n.site;return r.y===t?r.x:1/0}function Ur(n){this.site=n,this.edges=[]}function jr(n){for(var t,e,r,u,i,o,a,c,s,l,f=n[0][0],h=n[1][0],g=n[0][1],p=n[1][1],v=Zc,d=v.length;d--;)if(i=v[d],i&&i.prepare())for(a=i.edges,c=a.length,o=0;c>o;)l=a[o].end(),r=l.x,u=l.y,s=a[++o%c].start(),t=s.x,e=s.y,(ua(r-t)>ka||ua(u-e)>ka)&&(a.splice(o,0,new Wr($r(i.site,l,ua(r-f)<ka&&p-u>ka?{x:f,y:ua(t-f)<ka?e:p}:ua(u-p)<ka&&h-r>ka?{x:ua(e-p)<ka?t:h,y:p}:ua(r-h)<ka&&u-g>ka?{x:h,y:ua(t-h)<ka?e:g}:ua(u-g)<ka&&r-f>ka?{x:ua(e-g)<ka?t:f,y:g}:null),i.site,null)),++c)}function Hr(n,t){return t.angle-n.angle}function Fr(){Gr(this),this.x=this.y=this.arc=this.site=this.cy=null}function Or(n){var t=n.P,e=n.N;if(t&&e){var r=t.site,u=n.site,i=e.site;if(r!==i){var o=u.x,a=u.y,c=r.x-o,s=r.y-a,l=i.x-o,f=i.y-a,h=2*(c*f-s*l);if(!(h>=-Ea)){var g=c*c+s*s,p=l*l+f*f,v=(f*g-s*p)/h,d=(c*p-l*g)/h,f=d+a,m=Wc.pop()||new Fr;m.arc=n,m.site=u,m.x=v+o,m.y=f+Math.sqrt(v*v+d*d),m.cy=f,n.circle=m;for(var y=null,x=$c._;x;)if(m.y<x.y||m.y===x.y&&m.x<=x.x){if(!x.L){y=x.P;break}x=x.L}else{if(!x.R){y=x;break}x=x.R}$c.insert(y,m),y||(Xc=m)}}}}function Yr(n){var t=n.circle;t&&(t.P||(Xc=t.N),$c.remove(t),Wc.push(t),Gr(t),n.circle=null)}function Ir(n){for(var t,e=Ic,r=Pe(n[0][0],n[0][1],n[1][0],n[1][1]),u=e.length;u--;)t=e[u],(!Zr(t,n)||!r(t)||ua(t.a.x-t.b.x)<ka&&ua(t.a.y-t.b.y)<ka)&&(t.a=t.b=null,e.splice(u,1))}function Zr(n,t){var e=n.b;if(e)return!0;var r,u,i=n.a,o=t[0][0],a=t[1][0],c=t[0][1],s=t[1][1],l=n.l,f=n.r,h=l.x,g=l.y,p=f.x,v=f.y,d=(h+p)/2,m=(g+v)/2;if(v===g){if(o>d||d>=a)return;if(h>p){if(i){if(i.y>=s)return}else i={x:d,y:c};e={x:d,y:s}}else{if(i){if(i.y<c)return}else i={x:d,y:s};e={x:d,y:c}}}else if(r=(h-p)/(v-g),u=m-r*d,-1>r||r>1)if(h>p){if(i){if(i.y>=s)return}else i={x:(c-u)/r,y:c};e={x:(s-u)/r,y:s}}else{if(i){if(i.y<c)return}else i={x:(s-u)/r,y:s};e={x:(c-u)/r,y:c}}else if(v>g){if(i){if(i.x>=a)return}else i={x:o,y:r*o+u};e={x:a,y:r*a+u}}else{if(i){if(i.x<o)return}else i={x:a,y:r*a+u};e={x:o,y:r*o+u}}return n.a=i,n.b=e,!0}function Vr(n,t){this.l=n,this.r=t,this.a=this.b=null}function Xr(n,t,e,r){var u=new Vr(n,t);return Ic.push(u),e&&Br(u,n,t,e),r&&Br(u,t,n,r),Zc[n.i].edges.push(new Wr(u,n,t)),Zc[t.i].edges.push(new Wr(u,t,n)),u}function $r(n,t,e){var r=new Vr(n,null);return r.a=t,r.b=e,Ic.push(r),r}function Br(n,t,e,r){n.a||n.b?n.l===e?n.b=r:n.a=r:(n.a=r,n.l=t,n.r=e)}function Wr(n,t,e){var r=n.a,u=n.b;this.edge=n,this.site=t,this.angle=e?Math.atan2(e.y-t.y,e.x-t.x):n.l===t?Math.atan2(u.x-r.x,r.y-u.y):Math.atan2(r.x-u.x,u.y-r.y)}function Jr(){this._=null}function Gr(n){n.U=n.C=n.L=n.R=n.P=n.N=null}function Kr(n,t){var e=t,r=t.R,u=e.U;u?u.L===e?u.L=r:u.R=r:n._=r,r.U=u,e.U=r,e.R=r.L,e.R&&(e.R.U=e),r.L=e}function Qr(n,t){var e=t,r=t.L,u=e.U;u?u.L===e?u.L=r:u.R=r:n._=r,r.U=u,e.U=r,e.L=r.R,e.L&&(e.L.U=e),r.R=e}function nu(n){for(;n.L;)n=n.L;return n}function tu(n,t){var e,r,u,i=n.sort(eu).pop();for(Ic=[],Zc=new Array(n.length),Vc=new Jr,$c=new Jr;;)if(u=Xc,i&&(!u||i.y<u.y||i.y===u.y&&i.x<u.x))(i.x!==e||i.y!==r)&&(Zc[i.i]=new Ur(i),Rr(i),e=i.x,r=i.y),i=n.pop();else{if(!u)break;qr(u.arc)}t&&(Ir(t),jr(t));var o={cells:Zc,edges:Ic};return Vc=$c=Ic=Zc=null,o}function eu(n,t){return t.y-n.y||t.x-n.x}function ru(n,t,e){return(n.x-e.x)*(t.y-n.y)-(n.x-t.x)*(e.y-n.y)}function uu(n){return n.x}function iu(n){return n.y}function ou(){return{leaf:!0,nodes:[],point:null,x:null,y:null}}function au(n,t,e,r,u,i){if(!n(t,e,r,u,i)){var o=.5*(e+u),a=.5*(r+i),c=t.nodes;c[0]&&au(n,c[0],e,r,o,a),c[1]&&au(n,c[1],o,r,u,a),c[2]&&au(n,c[2],e,a,o,i),c[3]&&au(n,c[3],o,a,u,i)}}function cu(n,t){n=Zo.rgb(n),t=Zo.rgb(t);var e=n.r,r=n.g,u=n.b,i=t.r-e,o=t.g-r,a=t.b-u;return function(n){return"#"+dt(Math.round(e+i*n))+dt(Math.round(r+o*n))+dt(Math.round(u+a*n))}}function su(n,t){var e,r={},u={};for(e in n)e in t?r[e]=hu(n[e],t[e]):u[e]=n[e];for(e in t)e in n||(u[e]=t[e]);return function(n){for(e in r)u[e]=r[e](n);return u}}function lu(n,t){return t-=n=+n,function(e){return n+t*e}}function fu(n,t){var e,r,u,i=Gc.lastIndex=Kc.lastIndex=0,o=-1,a=[],c=[];for(n+="",t+="";(e=Gc.exec(n))&&(r=Kc.exec(t));)(u=r.index)>i&&(u=t.substring(i,u),a[o]?a[o]+=u:a[++o]=u),(e=e[0])===(r=r[0])?a[o]?a[o]+=r:a[++o]=r:(a[++o]=null,c.push({i:o,x:lu(e,r)})),i=Kc.lastIndex;return i<t.length&&(u=t.substring(i),a[o]?a[o]+=u:a[++o]=u),a.length<2?c[0]?(t=c[0].x,function(n){return t(n)+""}):function(){return t}:(t=c.length,function(n){for(var e,r=0;t>r;++r)a[(e=c[r]).i]=e.x(n);return a.join("")})}function hu(n,t){for(var e,r=Zo.interpolators.length;--r>=0&&!(e=Zo.interpolators[r](n,t)););return e}function gu(n,t){var e,r=[],u=[],i=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e;++e)r.push(hu(n[e],t[e]));for(;i>e;++e)u[e]=n[e];for(;o>e;++e)u[e]=t[e];return function(n){for(e=0;a>e;++e)u[e]=r[e](n);return u}}function pu(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function vu(n){return function(t){return 1-n(1-t)}}function du(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function mu(n){return n*n}function yu(n){return n*n*n}function xu(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function Mu(n){return function(t){return Math.pow(t,n)}}function _u(n){return 1-Math.cos(n*Sa)}function bu(n){return Math.pow(2,10*(n-1))}function wu(n){return 1-Math.sqrt(1-n*n)}function Su(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/wa*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*wa/t)}}function ku(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function Eu(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)*n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function Au(n,t){n=Zo.hcl(n),t=Zo.hcl(t);var e=n.h,r=n.c,u=n.l,i=t.h-e,o=t.c-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return ot(e+i*n,r+o*n,u+a*n)+""}}function Cu(n,t){n=Zo.hsl(n),t=Zo.hsl(t);var e=n.h,r=n.s,u=n.l,i=t.h-e,o=t.s-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return ut(e+i*n,r+o*n,u+a*n)+""}}function Nu(n,t){n=Zo.lab(n),t=Zo.lab(t);var e=n.l,r=n.a,u=n.b,i=t.l-e,o=t.a-r,a=t.b-u;return function(n){return ct(e+i*n,r+o*n,u+a*n)+""}}function zu(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function Lu(n){var t=[n.a,n.b],e=[n.c,n.d],r=qu(t),u=Tu(t,e),i=qu(Ru(e,t,-u))||0;t[0]*e[1]<e[0]*t[1]&&(t[0]*=-1,t[1]*=-1,r*=-1,u*=-1),this.rotate=(r?Math.atan2(t[1],t[0]):Math.atan2(-e[0],e[1]))*Ca,this.translate=[n.e,n.f],this.scale=[r,i],this.skew=i?Math.atan2(u,i)*Ca:0}function Tu(n,t){return n[0]*t[0]+n[1]*t[1]}function qu(n){var t=Math.sqrt(Tu(n,n));return t&&(n[0]/=t,n[1]/=t),t}function Ru(n,t,e){return n[0]+=e*t[0],n[1]+=e*t[1],n}function Du(n,t){var e,r=[],u=[],i=Zo.transform(n),o=Zo.transform(t),a=i.translate,c=o.translate,s=i.rotate,l=o.rotate,f=i.skew,h=o.skew,g=i.scale,p=o.scale;return a[0]!=c[0]||a[1]!=c[1]?(r.push("translate(",null,",",null,")"),u.push({i:1,x:lu(a[0],c[0])},{i:3,x:lu(a[1],c[1])})):c[0]||c[1]?r.push("translate("+c+")"):r.push(""),s!=l?(s-l>180?l+=360:l-s>180&&(s+=360),u.push({i:r.push(r.pop()+"rotate(",null,")")-2,x:lu(s,l)})):l&&r.push(r.pop()+"rotate("+l+")"),f!=h?u.push({i:r.push(r.pop()+"skewX(",null,")")-2,x:lu(f,h)}):h&&r.push(r.pop()+"skewX("+h+")"),g[0]!=p[0]||g[1]!=p[1]?(e=r.push(r.pop()+"scale(",null,",",null,")"),u.push({i:e-4,x:lu(g[0],p[0])},{i:e-2,x:lu(g[1],p[1])})):(1!=p[0]||1!=p[1])&&r.push(r.pop()+"scale("+p+")"),e=u.length,function(n){for(var t,i=-1;++i<e;)r[(t=u[i]).i]=t.x(n);return r.join("")}}function Pu(n,t){return t=t-(n=+n)?1/(t-n):0,function(e){return(e-n)*t}}function Uu(n,t){return t=t-(n=+n)?1/(t-n):0,function(e){return Math.max(0,Math.min(1,(e-n)*t))}}function ju(n){for(var t=n.source,e=n.target,r=Fu(t,e),u=[t];t!==r;)t=t.parent,u.push(t);for(var i=u.length;e!==r;)u.splice(i,0,e),e=e.parent;return u}function Hu(n){for(var t=[],e=n.parent;null!=e;)t.push(n),n=e,e=e.parent;return t.push(n),t}function Fu(n,t){if(n===t)return n;for(var e=Hu(n),r=Hu(t),u=e.pop(),i=r.pop(),o=null;u===i;)o=u,u=e.pop(),i=r.pop();return o}function Ou(n){n.fixed|=2}function Yu(n){n.fixed&=-7}function Iu(n){n.fixed|=4,n.px=n.x,n.py=n.y}function Zu(n){n.fixed&=-5}function Vu(n,t,e){var r=0,u=0;if(n.charge=0,!n.leaf)for(var i,o=n.nodes,a=o.length,c=-1;++c<a;)i=o[c],null!=i&&(Vu(i,t,e),n.charge+=i.charge,r+=i.charge*i.cx,u+=i.charge*i.cy);if(n.point){n.leaf||(n.point.x+=Math.random()-.5,n.point.y+=Math.random()-.5);var s=t*e[n.point.index];n.charge+=n.pointCharge=s,r+=s*n.point.x,u+=s*n.point.y}n.cx=r/n.charge,n.cy=u/n.charge}function Xu(n,t){return Zo.rebind(n,t,"sort","children","value"),n.nodes=n,n.links=Ku,n}function $u(n,t){for(var e=[n];null!=(n=e.pop());)if(t(n),(u=n.children)&&(r=u.length))for(var r,u;--r>=0;)e.push(u[r])}function Bu(n,t){for(var e=[n],r=[];null!=(n=e.pop());)if(r.push(n),(i=n.children)&&(u=i.length))for(var u,i,o=-1;++o<u;)e.push(i[o]);for(;null!=(n=r.pop());)t(n)}function Wu(n){return n.children}function Ju(n){return n.value}function Gu(n,t){return t.value-n.value}function Ku(n){return Zo.merge(n.map(function(n){return(n.children||[]).map(function(t){return{source:n,target:t}})}))}function Qu(n){return n.x}function ni(n){return n.y}function ti(n,t,e){n.y0=t,n.y=e}function ei(n){return Zo.range(n.length)}function ri(n){for(var t=-1,e=n[0].length,r=[];++t<e;)r[t]=0;return r}function ui(n){for(var t,e=1,r=0,u=n[0][1],i=n.length;i>e;++e)(t=n[e][1])>u&&(r=e,u=t);return r}function ii(n){return n.reduce(oi,0)}function oi(n,t){return n+t[1]}function ai(n,t){return ci(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function ci(n,t){for(var e=-1,r=+n[0],u=(n[1]-r)/t,i=[];++e<=t;)i[e]=u*e+r;return i}function si(n){return[Zo.min(n),Zo.max(n)]}function li(n,t){return n.value-t.value}function fi(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function hi(n,t){n._pack_next=t,t._pack_prev=n}function gi(n,t){var e=t.x-n.x,r=t.y-n.y,u=n.r+t.r;return.999*u*u>e*e+r*r}function pi(n){function t(n){l=Math.min(n.x-n.r,l),f=Math.max(n.x+n.r,f),h=Math.min(n.y-n.r,h),g=Math.max(n.y+n.r,g)}if((e=n.children)&&(s=e.length)){var e,r,u,i,o,a,c,s,l=1/0,f=-1/0,h=1/0,g=-1/0;if(e.forEach(vi),r=e[0],r.x=-r.r,r.y=0,t(r),s>1&&(u=e[1],u.x=u.r,u.y=0,t(u),s>2))for(i=e[2],yi(r,u,i),t(i),fi(r,i),r._pack_prev=i,fi(i,u),u=r._pack_next,o=3;s>o;o++){yi(r,u,i=e[o]);var p=0,v=1,d=1;for(a=u._pack_next;a!==u;a=a._pack_next,v++)if(gi(a,i)){p=1;break}if(1==p)for(c=r._pack_prev;c!==a._pack_prev&&!gi(c,i);c=c._pack_prev,d++);p?(d>v||v==d&&u.r<r.r?hi(r,u=a):hi(r=c,u),o--):(fi(r,i),u=i,t(i))}var m=(l+f)/2,y=(h+g)/2,x=0;for(o=0;s>o;o++)i=e[o],i.x-=m,i.y-=y,x=Math.max(x,i.r+Math.sqrt(i.x*i.x+i.y*i.y));n.r=x,e.forEach(di)}}function vi(n){n._pack_next=n._pack_prev=n}function di(n){delete n._pack_next,delete n._pack_prev}function mi(n,t,e,r){var u=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,u)for(var i=-1,o=u.length;++i<o;)mi(u[i],t,e,r)}function yi(n,t,e){var r=n.r+e.r,u=t.x-n.x,i=t.y-n.y;if(r&&(u||i)){var o=t.r+e.r,a=u*u+i*i;o*=o,r*=r;var c=.5+(r-o)/(2*a),s=Math.sqrt(Math.max(0,2*o*(r+a)-(r-=a)*r-o*o))/(2*a);e.x=n.x+c*u+s*i,e.y=n.y+c*i-s*u}else e.x=n.x+r,e.y=n.y}function xi(n,t){return n.parent==t.parent?1:2}function Mi(n){var t=n.children;return t.length?t[0]:n.t}function _i(n){var t,e=n.children;return(t=e.length)?e[t-1]:n.t}function bi(n,t,e){var r=e/(t.i-n.i);t.c-=r,t.s+=e,n.c+=r,t.z+=e,t.m+=e}function wi(n){for(var t,e=0,r=0,u=n.children,i=u.length;--i>=0;)t=u[i],t.z+=e,t.m+=e,e+=t.s+(r+=t.c)}function Si(n,t,e){return n.a.parent===t.parent?n.a:e}function ki(n){return 1+Zo.max(n,function(n){return n.y})}function Ei(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Ai(n){var t=n.children;return t&&t.length?Ai(t[0]):n}function Ci(n){var t,e=n.children;return e&&(t=e.length)?Ci(e[t-1]):n}function Ni(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function zi(n,t){var e=n.x+t[3],r=n.y+t[0],u=n.dx-t[1]-t[3],i=n.dy-t[0]-t[2];return 0>u&&(e+=u/2,u=0),0>i&&(r+=i/2,i=0),{x:e,y:r,dx:u,dy:i}}function Li(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function Ti(n){return n.rangeExtent?n.rangeExtent():Li(n.range())}function qi(n,t,e,r){var u=e(n[0],n[1]),i=r(t[0],t[1]);return function(n){return i(u(n))}}function Ri(n,t){var e,r=0,u=n.length-1,i=n[r],o=n[u];return i>o&&(e=r,r=u,u=e,e=i,i=o,o=e),n[r]=t.floor(i),n[u]=t.ceil(o),n}function Di(n){return n?{floor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:ss}function Pi(n,t,e,r){var u=[],i=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]<n[0]&&(n=n.slice().reverse(),t=t.slice().reverse());++o<=a;)u.push(e(n[o-1],n[o])),i.push(r(t[o-1],t[o]));return function(t){var e=Zo.bisect(n,t,1,a)-1;return i[e](u[e](t))}}function Ui(n,t,e,r){function u(){var u=Math.min(n.length,t.length)>2?Pi:qi,c=r?Uu:Pu;return o=u(n,t,c,e),a=u(t,n,c,hu),i}function i(n){return o(n)}var o,a;return i.invert=function(n){return a(n)},i.domain=function(t){return arguments.length?(n=t.map(Number),u()):n},i.range=function(n){return arguments.length?(t=n,u()):t},i.rangeRound=function(n){return i.range(n).interpolate(zu)},i.clamp=function(n){return arguments.length?(r=n,u()):r},i.interpolate=function(n){return arguments.length?(e=n,u()):e},i.ticks=function(t){return Oi(n,t)},i.tickFormat=function(t,e){return Yi(n,t,e)},i.nice=function(t){return Hi(n,t),u()},i.copy=function(){return Ui(n,t,e,r)},u()}function ji(n,t){return Zo.rebind(n,t,"range","rangeRound","interpolate","clamp")}function Hi(n,t){return Ri(n,Di(Fi(n,t)[2]))}function Fi(n,t){null==t&&(t=10);var e=Li(n),r=e[1]-e[0],u=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),i=t/r*u;return.15>=i?u*=10:.35>=i?u*=5:.75>=i&&(u*=2),e[0]=Math.ceil(e[0]/u)*u,e[1]=Math.floor(e[1]/u)*u+.5*u,e[2]=u,e}function Oi(n,t){return Zo.range.apply(Zo,Fi(n,t))}function Yi(n,t,e){var r=Fi(n,t);if(e){var u=Ga.exec(e);if(u.shift(),"s"===u[8]){var i=Zo.formatPrefix(Math.max(ua(r[0]),ua(r[1])));return u[7]||(u[7]="."+Ii(i.scale(r[2]))),u[8]="f",e=Zo.format(u.join("")),function(n){return e(i.scale(n))+i.symbol}}u[7]||(u[7]="."+Zi(u[8],r)),e=u.join("")}else e=",."+Ii(r[2])+"f";return Zo.format(e)}function Ii(n){return-Math.floor(Math.log(n)/Math.LN10+.01)}function Zi(n,t){var e=Ii(t[2]);return n in ls?Math.abs(e-Ii(Math.max(ua(t[0]),ua(t[1]))))+ +("e"!==n):e-2*("%"===n)}function Vi(n,t,e,r){function u(n){return(e?Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function i(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(u(t))}return o.invert=function(t){return i(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(u)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(u)),o):t},o.nice=function(){var t=Ri(r.map(u),e?Math:hs);return n.domain(t),r=t.map(i),o},o.ticks=function(){var n=Li(r),o=[],a=n[0],c=n[1],s=Math.floor(u(a)),l=Math.ceil(u(c)),f=t%1?2:t;if(isFinite(l-s)){if(e){for(;l>s;s++)for(var h=1;f>h;h++)o.push(i(s)*h);o.push(i(s))}else for(o.push(i(s));s++<l;)for(var h=f-1;h>0;h--)o.push(i(s)*h);for(s=0;o[s]<a;s++);for(l=o.length;o[l-1]>c;l--);o=o.slice(s,l)}return o},o.tickFormat=function(n,t){if(!arguments.length)return fs;arguments.length<2?t=fs:"function"!=typeof t&&(t=Zo.format(t));var r,a=Math.max(.1,n/o.ticks().length),c=e?(r=1e-12,Math.ceil):(r=-1e-12,Math.floor);return function(n){return n/i(c(u(n)+r))<=a?t(n):""}},o.copy=function(){return Vi(n.copy(),t,e,r)},ji(o,n)}function Xi(n,t,e){function r(t){return n(u(t))}var u=$i(t),i=$i(1/t);return r.invert=function(t){return i(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(u)),r):e},r.ticks=function(n){return Oi(e,n)},r.tickFormat=function(n,t){return Yi(e,n,t)},r.nice=function(n){return r.domain(Hi(e,n))},r.exponent=function(o){return arguments.length?(u=$i(t=o),i=$i(1/t),n.domain(e.map(u)),r):t},r.copy=function(){return Xi(n.copy(),t,e)},ji(r,n)}function $i(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function Bi(n,t){function e(e){return i[((u.get(e)||("range"===t.t?u.set(e,n.push(e)):0/0))-1)%i.length]}function r(t,e){return Zo.range(n.length).map(function(n){return t+e*n})}var u,i,a;return e.domain=function(r){if(!arguments.length)return n;n=[],u=new o;for(var i,a=-1,c=r.length;++a<c;)u.has(i=r[a])||u.set(i,n.push(i));return e[t.t].apply(e,t.a)},e.range=function(n){return arguments.length?(i=n,a=0,t={t:"range",a:arguments},e):i},e.rangePoints=function(u,o){arguments.length<2&&(o=0);var c=u[0],s=u[1],l=(s-c)/(Math.max(1,n.length-1)+o);return i=r(n.length<2?(c+s)/2:c+l*o/2,l),a=0,t={t:"rangePoints",a:arguments},e},e.rangeBands=function(u,o,c){arguments.length<2&&(o=0),arguments.length<3&&(c=o);var s=u[1]<u[0],l=u[s-0],f=u[1-s],h=(f-l)/(n.length-o+2*c);return i=r(l+h*c,h),s&&i.reverse(),a=h*(1-o),t={t:"rangeBands",a:arguments},e},e.rangeRoundBands=function(u,o,c){arguments.length<2&&(o=0),arguments.length<3&&(c=o);var s=u[1]<u[0],l=u[s-0],f=u[1-s],h=Math.floor((f-l)/(n.length-o+2*c)),g=f-l-(n.length-o)*h;return i=r(l+Math.round(g/2),h),s&&i.reverse(),a=Math.round(h*(1-o)),t={t:"rangeRoundBands",a:arguments},e},e.rangeBand=function(){return a},e.rangeExtent=function(){return Li(t.a[0])},e.copy=function(){return Bi(n,t)},e.domain(n)}function Wi(e,r){function u(){var n=0,t=r.length;for(o=[];++n<t;)o[n-1]=Zo.quantile(e,n/t);return i}function i(n){return isNaN(n=+n)?void 0:r[Zo.bisect(o,n)]}var o;return i.domain=function(r){return arguments.length?(e=r.filter(t).sort(n),u()):e},i.range=function(n){return arguments.length?(r=n,u()):r},i.quantiles=function(){return o},i.invertExtent=function(n){return n=r.indexOf(n),0>n?[0/0,0/0]:[n>0?o[n-1]:e[0],n<o.length?o[n]:e[e.length-1]]},i.copy=function(){return Wi(e,r)},u()}function Ji(n,t,e){function r(t){return e[Math.max(0,Math.min(o,Math.floor(i*(t-n))))]}function u(){return i=e.length/(t-n),o=e.length-1,r}var i,o;return r.domain=function(e){return arguments.length?(n=+e[0],t=+e[e.length-1],u()):[n,t]},r.range=function(n){return arguments.length?(e=n,u()):e},r.invertExtent=function(t){return t=e.indexOf(t),t=0>t?0/0:t/i+n,[t,t+1/i]},r.copy=function(){return Ji(n,t,e)},u()}function Gi(n,t){function e(e){return e>=e?t[Zo.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return Gi(n,t)},e}function Ki(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return Oi(n,t)},t.tickFormat=function(t,e){return Yi(n,t,e)},t.copy=function(){return Ki(n)},t}function Qi(n){return n.innerRadius}function no(n){return n.outerRadius}function to(n){return n.startAngle}function eo(n){return n.endAngle}function ro(n){function t(t){function o(){s.push("M",i(n(l),a))}for(var c,s=[],l=[],f=-1,h=t.length,g=bt(e),p=bt(r);++f<h;)u.call(this,c=t[f],f)?l.push([+g.call(this,c,f),+p.call(this,c,f)]):l.length&&(o(),l=[]);return l.length&&o(),s.length?s.join(""):null}var e=wr,r=Sr,u=we,i=uo,o=i.key,a=.7;return t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t.defined=function(n){return arguments.length?(u=n,t):u},t.interpolate=function(n){return arguments.length?(o="function"==typeof n?i=n:(i=xs.get(n)||uo).key,t):o},t.tension=function(n){return arguments.length?(a=n,t):a},t}function uo(n){return n.join("L")}function io(n){return uo(n)+"Z"}function oo(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("H",(r[0]+(r=n[t])[0])/2,"V",r[1]);return e>1&&u.push("H",r[0]),u.join("")}function ao(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("V",(r=n[t])[1],"H",r[0]);return u.join("")}function co(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("H",(r=n[t])[0],"V",r[1]);return u.join("")}function so(n,t){return n.length<4?uo(n):n[1]+ho(n.slice(1,n.length-1),go(n,t))}function lo(n,t){return n.length<3?uo(n):n[0]+ho((n.push(n[0]),n),go([n[n.length-2]].concat(n,[n[1]]),t))}function fo(n,t){return n.length<3?uo(n):n[0]+ho(n,go(n,t))}function ho(n,t){if(t.length<1||n.length!=t.length&&n.length!=t.length+2)return uo(n);var e=n.length!=t.length,r="",u=n[0],i=n[1],o=t[0],a=o,c=1;if(e&&(r+="Q"+(i[0]-2*o[0]/3)+","+(i[1]-2*o[1]/3)+","+i[0]+","+i[1],u=n[1],c=2),t.length>1){a=t[1],i=n[c],c++,r+="C"+(u[0]+o[0])+","+(u[1]+o[1])+","+(i[0]-a[0])+","+(i[1]-a[1])+","+i[0]+","+i[1];for(var s=2;s<t.length;s++,c++)i=n[c],a=t[s],r+="S"+(i[0]-a[0])+","+(i[1]-a[1])+","+i[0]+","+i[1]}if(e){var l=n[c];r+="Q"+(i[0]+2*a[0]/3)+","+(i[1]+2*a[1]/3)+","+l[0]+","+l[1]}return r}function go(n,t){for(var e,r=[],u=(1-t)/2,i=n[0],o=n[1],a=1,c=n.length;++a<c;)e=i,i=o,o=n[a],r.push([u*(o[0]-e[0]),u*(o[1]-e[1])]);return r}function po(n){if(n.length<3)return uo(n);var t=1,e=n.length,r=n[0],u=r[0],i=r[1],o=[u,u,u,(r=n[1])[0]],a=[i,i,i,r[1]],c=[u,",",i,"L",xo(bs,o),",",xo(bs,a)];for(n.push(n[e-1]);++t<=e;)r=n[t],o.shift(),o.push(r[0]),a.shift(),a.push(r[1]),Mo(c,o,a);return n.pop(),c.push("L",r),c.join("")}function vo(n){if(n.length<4)return uo(n);for(var t,e=[],r=-1,u=n.length,i=[0],o=[0];++r<3;)t=n[r],i.push(t[0]),o.push(t[1]);for(e.push(xo(bs,i)+","+xo(bs,o)),--r;++r<u;)t=n[r],i.shift(),i.push(t[0]),o.shift(),o.push(t[1]),Mo(e,i,o);return e.join("")}function mo(n){for(var t,e,r=-1,u=n.length,i=u+4,o=[],a=[];++r<4;)e=n[r%u],o.push(e[0]),a.push(e[1]);for(t=[xo(bs,o),",",xo(bs,a)],--r;++r<i;)e=n[r%u],o.shift(),o.push(e[0]),a.shift(),a.push(e[1]),Mo(t,o,a);return t.join("")}function yo(n,t){var e=n.length-1;if(e)for(var r,u,i=n[0][0],o=n[0][1],a=n[e][0]-i,c=n[e][1]-o,s=-1;++s<=e;)r=n[s],u=s/e,r[0]=t*r[0]+(1-t)*(i+u*a),r[1]=t*r[1]+(1-t)*(o+u*c);return po(n)}function xo(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]+n[3]*t[3]}function Mo(n,t,e){n.push("C",xo(Ms,t),",",xo(Ms,e),",",xo(_s,t),",",xo(_s,e),",",xo(bs,t),",",xo(bs,e))}function _o(n,t){return(t[1]-n[1])/(t[0]-n[0])}function bo(n){for(var t=0,e=n.length-1,r=[],u=n[0],i=n[1],o=r[0]=_o(u,i);++t<e;)r[t]=(o+(o=_o(u=i,i=n[t+1])))/2;return r[t]=o,r}function wo(n){for(var t,e,r,u,i=[],o=bo(n),a=-1,c=n.length-1;++a<c;)t=_o(n[a],n[a+1]),ua(t)<ka?o[a]=o[a+1]=0:(e=o[a]/t,r=o[a+1]/t,u=e*e+r*r,u>9&&(u=3*t/Math.sqrt(u),o[a]=u*e,o[a+1]=u*r));for(a=-1;++a<=c;)u=(n[Math.min(c,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),i.push([u||0,o[a]*u||0]);return i}function So(n){return n.length<3?uo(n):n[0]+ho(n,wo(n))}function ko(n){for(var t,e,r,u=-1,i=n.length;++u<i;)t=n[u],e=t[0],r=t[1]+ms,t[0]=e*Math.cos(r),t[1]=e*Math.sin(r);return n}function Eo(n){function t(t){function c(){v.push("M",a(n(m),f),l,s(n(d.reverse()),f),"Z")}for(var h,g,p,v=[],d=[],m=[],y=-1,x=t.length,M=bt(e),_=bt(u),b=e===r?function(){return g}:bt(r),w=u===i?function(){return p}:bt(i);++y<x;)o.call(this,h=t[y],y)?(d.push([g=+M.call(this,h,y),p=+_.call(this,h,y)]),m.push([+b.call(this,h,y),+w.call(this,h,y)])):d.length&&(c(),d=[],m=[]);return d.length&&c(),v.length?v.join(""):null}var e=wr,r=wr,u=0,i=Sr,o=we,a=uo,c=a.key,s=a,l="L",f=.7;return t.x=function(n){return arguments.length?(e=r=n,t):r},t.x0=function(n){return arguments.length?(e=n,t):e},t.x1=function(n){return arguments.length?(r=n,t):r},t.y=function(n){return arguments.length?(u=i=n,t):i},t.y0=function(n){return arguments.length?(u=n,t):u},t.y1=function(n){return arguments.length?(i=n,t):i},t.defined=function(n){return arguments.length?(o=n,t):o},t.interpolate=function(n){return arguments.length?(c="function"==typeof n?a=n:(a=xs.get(n)||uo).key,s=a.reverse||a,l=a.closed?"M":"L",t):c},t.tension=function(n){return arguments.length?(f=n,t):f},t}function Ao(n){return n.radius}function Co(n){return[n.x,n.y]}function No(n){return function(){var t=n.apply(this,arguments),e=t[0],r=t[1]+ms;return[e*Math.cos(r),e*Math.sin(r)]}}function zo(){return 64}function Lo(){return"circle"}function To(n){var t=Math.sqrt(n/ba);return"M0,"+t+"A"+t+","+t+" 0 1,1 0,"+-t+"A"+t+","+t+" 0 1,1 0,"+t+"Z"}function qo(n,t){return sa(n,Cs),n.id=t,n}function Ro(n,t,e,r){var u=n.id;return P(n,"function"==typeof e?function(n,i,o){n.__transition__[u].tween.set(t,r(e.call(n,n.__data__,i,o)))}:(e=r(e),function(n){n.__transition__[u].tween.set(t,e)}))}function Do(n){return null==n&&(n=""),function(){this.textContent=n}}function Po(n,t,e,r){var u=n.__transition__||(n.__transition__={active:0,count:0}),i=u[e];if(!i){var a=r.time;i=u[e]={tween:new o,time:a,ease:r.ease,delay:r.delay,duration:r.duration},++u.count,Zo.timer(function(r){function o(r){return u.active>e?s():(u.active=e,i.event&&i.event.start.call(n,l,t),i.tween.forEach(function(e,r){(r=r.call(n,l,t))&&v.push(r)}),Zo.timer(function(){return p.c=c(r||1)?we:c,1},0,a),void 0)}function c(r){if(u.active!==e)return s();for(var o=r/g,a=f(o),c=v.length;c>0;)v[--c].call(n,a);
+return o>=1?(i.event&&i.event.end.call(n,l,t),s()):void 0}function s(){return--u.count?delete u[e]:delete n.__transition__,1}var l=n.__data__,f=i.ease,h=i.delay,g=i.duration,p=Ba,v=[];return p.t=h+a,r>=h?o(r-h):(p.c=o,void 0)},0,a)}}function Uo(n,t){n.attr("transform",function(n){return"translate("+t(n)+",0)"})}function jo(n,t){n.attr("transform",function(n){return"translate(0,"+t(n)+")"})}function Ho(n){return n.toISOString()}function Fo(n,t,e){function r(t){return n(t)}function u(n,e){var r=n[1]-n[0],u=r/e,i=Zo.bisect(Us,u);return i==Us.length?[t.year,Fi(n.map(function(n){return n/31536e6}),e)[2]]:i?t[u/Us[i-1]<Us[i]/u?i-1:i]:[Fs,Fi(n,e)[2]]}return r.invert=function(t){return Oo(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain(t),r):n.domain().map(Oo)},r.nice=function(n,t){function e(e){return!isNaN(e)&&!n.range(e,Oo(+e+1),t).length}var i=r.domain(),o=Li(i),a=null==n?u(o,10):"number"==typeof n&&u(o,n);return a&&(n=a[0],t=a[1]),r.domain(Ri(i,t>1?{floor:function(t){for(;e(t=n.floor(t));)t=Oo(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=Oo(+t+1);return t}}:n))},r.ticks=function(n,t){var e=Li(r.domain()),i=null==n?u(e,10):"number"==typeof n?u(e,n):!n.range&&[{range:n},t];return i&&(n=i[0],t=i[1]),n.range(e[0],Oo(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return Fo(n.copy(),t,e)},ji(r,n)}function Oo(n){return new Date(n)}function Yo(n){return JSON.parse(n.responseText)}function Io(n){var t=$o.createRange();return t.selectNode($o.body),t.createContextualFragment(n.responseText)}var Zo={version:"3.4.11"};Date.now||(Date.now=function(){return+new Date});var Vo=[].slice,Xo=function(n){return Vo.call(n)},$o=document,Bo=$o.documentElement,Wo=window;try{Xo(Bo.childNodes)[0].nodeType}catch(Jo){Xo=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}try{$o.createElement("div").style.setProperty("opacity",0,"")}catch(Go){var Ko=Wo.Element.prototype,Qo=Ko.setAttribute,na=Ko.setAttributeNS,ta=Wo.CSSStyleDeclaration.prototype,ea=ta.setProperty;Ko.setAttribute=function(n,t){Qo.call(this,n,t+"")},Ko.setAttributeNS=function(n,t,e){na.call(this,n,t,e+"")},ta.setProperty=function(n,t,e){ea.call(this,n,t+"",e)}}Zo.ascending=n,Zo.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:0/0},Zo.min=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u<i&&!(null!=(e=n[u])&&e>=e);)e=void 0;for(;++u<i;)null!=(r=n[u])&&e>r&&(e=r)}else{for(;++u<i&&!(null!=(e=t.call(n,n[u],u))&&e>=e);)e=void 0;for(;++u<i;)null!=(r=t.call(n,n[u],u))&&e>r&&(e=r)}return e},Zo.max=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u<i&&!(null!=(e=n[u])&&e>=e);)e=void 0;for(;++u<i;)null!=(r=n[u])&&r>e&&(e=r)}else{for(;++u<i&&!(null!=(e=t.call(n,n[u],u))&&e>=e);)e=void 0;for(;++u<i;)null!=(r=t.call(n,n[u],u))&&r>e&&(e=r)}return e},Zo.extent=function(n,t){var e,r,u,i=-1,o=n.length;if(1===arguments.length){for(;++i<o&&!(null!=(e=u=n[i])&&e>=e);)e=u=void 0;for(;++i<o;)null!=(r=n[i])&&(e>r&&(e=r),r>u&&(u=r))}else{for(;++i<o&&!(null!=(e=u=t.call(n,n[i],i))&&e>=e);)e=void 0;for(;++i<o;)null!=(r=t.call(n,n[i],i))&&(e>r&&(e=r),r>u&&(u=r))}return[e,u]},Zo.sum=function(n,t){var e,r=0,u=n.length,i=-1;if(1===arguments.length)for(;++i<u;)isNaN(e=+n[i])||(r+=e);else for(;++i<u;)isNaN(e=+t.call(n,n[i],i))||(r+=e);return r},Zo.mean=function(n,e){var r,u=0,i=n.length,o=-1,a=i;if(1===arguments.length)for(;++o<i;)t(r=n[o])?u+=r:--a;else for(;++o<i;)t(r=e.call(n,n[o],o))?u+=r:--a;return a?u/a:void 0},Zo.quantile=function(n,t){var e=(n.length-1)*t+1,r=Math.floor(e),u=+n[r-1],i=e-r;return i?u+i*(n[r]-u):u},Zo.median=function(e,r){return arguments.length>1&&(e=e.map(r)),e=e.filter(t),e.length?Zo.quantile(e.sort(n),.5):void 0};var ra=e(n);Zo.bisectLeft=ra.left,Zo.bisect=Zo.bisectRight=ra.right,Zo.bisector=function(t){return e(1===t.length?function(e,r){return n(t(e),r)}:t)},Zo.shuffle=function(n){for(var t,e,r=n.length;r;)e=0|Math.random()*r--,t=n[r],n[r]=n[e],n[e]=t;return n},Zo.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},Zo.pairs=function(n){for(var t,e=0,r=n.length-1,u=n[0],i=new Array(0>r?0:r);r>e;)i[e]=[t=u,u=n[++e]];return i},Zo.zip=function(){if(!(u=arguments.length))return[];for(var n=-1,t=Zo.min(arguments,r),e=new Array(t);++n<t;)for(var u,i=-1,o=e[n]=new Array(u);++i<u;)o[i]=arguments[i][n];return e},Zo.transpose=function(n){return Zo.zip.apply(Zo,n)},Zo.keys=function(n){var t=[];for(var e in n)t.push(e);return t},Zo.values=function(n){var t=[];for(var e in n)t.push(n[e]);return t},Zo.entries=function(n){var t=[];for(var e in n)t.push({key:e,value:n[e]});return t},Zo.merge=function(n){for(var t,e,r,u=n.length,i=-1,o=0;++i<u;)o+=n[i].length;for(e=new Array(o);--u>=0;)for(r=n[u],t=r.length;--t>=0;)e[--o]=r[t];return e};var ua=Math.abs;Zo.range=function(n,t,e){if(arguments.length<3&&(e=1,arguments.length<2&&(t=n,n=0)),1/0===(t-n)/e)throw new Error("infinite range");var r,i=[],o=u(ua(e)),a=-1;if(n*=o,t*=o,e*=o,0>e)for(;(r=n+e*++a)>t;)i.push(r/o);else for(;(r=n+e*++a)<t;)i.push(r/o);return i},Zo.map=function(n){var t=new o;if(n instanceof o)n.forEach(function(n,e){t.set(n,e)});else for(var e in n)t.set(e,n[e]);return t},i(o,{has:a,get:function(n){return this[ia+n]},set:function(n,t){return this[ia+n]=t},remove:c,keys:s,values:function(){var n=[];return this.forEach(function(t,e){n.push(e)}),n},entries:function(){var n=[];return this.forEach(function(t,e){n.push({key:t,value:e})}),n},size:l,empty:f,forEach:function(n){for(var t in this)t.charCodeAt(0)===oa&&n.call(this,t.substring(1),this[t])}});var ia="\x00",oa=ia.charCodeAt(0);Zo.nest=function(){function n(t,a,c){if(c>=i.length)return r?r.call(u,a):e?a.sort(e):a;for(var s,l,f,h,g=-1,p=a.length,v=i[c++],d=new o;++g<p;)(h=d.get(s=v(l=a[g])))?h.push(l):d.set(s,[l]);return t?(l=t(),f=function(e,r){l.set(e,n(t,r,c))}):(l={},f=function(e,r){l[e]=n(t,r,c)}),d.forEach(f),l}function t(n,e){if(e>=i.length)return n;var r=[],u=a[e++];return n.forEach(function(n,u){r.push({key:n,values:t(u,e)})}),u?r.sort(function(n,t){return u(n.key,t.key)}):r}var e,r,u={},i=[],a=[];return u.map=function(t,e){return n(e,t,0)},u.entries=function(e){return t(n(Zo.map,e,0),0)},u.key=function(n){return i.push(n),u},u.sortKeys=function(n){return a[i.length-1]=n,u},u.sortValues=function(n){return e=n,u},u.rollup=function(n){return r=n,u},u},Zo.set=function(n){var t=new h;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},i(h,{has:a,add:function(n){return this[ia+n]=!0,n},remove:function(n){return n=ia+n,n in this&&delete this[n]},values:s,size:l,empty:f,forEach:function(n){for(var t in this)t.charCodeAt(0)===oa&&n.call(this,t.substring(1))}}),Zo.behavior={},Zo.rebind=function(n,t){for(var e,r=1,u=arguments.length;++r<u;)n[e=arguments[r]]=g(n,t,t[e]);return n};var aa=["webkit","ms","moz","Moz","o","O"];Zo.dispatch=function(){for(var n=new d,t=-1,e=arguments.length;++t<e;)n[arguments[t]]=m(n);return n},d.prototype.on=function(n,t){var e=n.indexOf("."),r="";if(e>=0&&(r=n.substring(e+1),n=n.substring(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},Zo.event=null,Zo.requote=function(n){return n.replace(ca,"\\$&")};var ca=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,sa={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},la=function(n,t){return t.querySelector(n)},fa=function(n,t){return t.querySelectorAll(n)},ha=Bo.matches||Bo[p(Bo,"matchesSelector")],ga=function(n,t){return ha.call(n,t)};"function"==typeof Sizzle&&(la=function(n,t){return Sizzle(n,t)[0]||null},fa=Sizzle,ga=Sizzle.matchesSelector),Zo.selection=function(){return ma};var pa=Zo.selection.prototype=[];pa.select=function(n){var t,e,r,u,i=[];n=b(n);for(var o=-1,a=this.length;++o<a;){i.push(t=[]),t.parentNode=(r=this[o]).parentNode;for(var c=-1,s=r.length;++c<s;)(u=r[c])?(t.push(e=n.call(u,u.__data__,c,o)),e&&"__data__"in u&&(e.__data__=u.__data__)):t.push(null)}return _(i)},pa.selectAll=function(n){var t,e,r=[];n=w(n);for(var u=-1,i=this.length;++u<i;)for(var o=this[u],a=-1,c=o.length;++a<c;)(e=o[a])&&(r.push(t=Xo(n.call(e,e.__data__,a,u))),t.parentNode=e);return _(r)};var va={svg:"http://www.w3.org/2000/svg",xhtml:"http://www.w3.org/1999/xhtml",xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};Zo.ns={prefix:va,qualify:function(n){var t=n.indexOf(":"),e=n;return t>=0&&(e=n.substring(0,t),n=n.substring(t+1)),va.hasOwnProperty(e)?{space:va[e],local:n}:n}},pa.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=Zo.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(S(t,n[t]));return this}return this.each(S(n,t))},pa.classed=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node(),r=(n=A(n)).length,u=-1;if(t=e.classList){for(;++u<r;)if(!t.contains(n[u]))return!1}else for(t=e.getAttribute("class");++u<r;)if(!E(n[u]).test(t))return!1;return!0}for(t in n)this.each(C(t,n[t]));return this}return this.each(C(n,t))},pa.style=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t="");for(e in n)this.each(z(e,n[e],t));return this}if(2>r)return Wo.getComputedStyle(this.node(),null).getPropertyValue(n);e=""}return this.each(z(n,t,e))},pa.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(L(t,n[t]));return this}return this.each(L(n,t))},pa.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},pa.html=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},pa.append=function(n){return n=T(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},pa.insert=function(n,t){return n=T(n),t=b(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments)||null)})},pa.remove=function(){return this.each(function(){var n=this.parentNode;n&&n.removeChild(this)})},pa.data=function(n,t){function e(n,e){var r,u,i,a=n.length,f=e.length,h=Math.min(a,f),g=new Array(f),p=new Array(f),v=new Array(a);if(t){var d,m=new o,y=new o,x=[];for(r=-1;++r<a;)d=t.call(u=n[r],u.__data__,r),m.has(d)?v[r]=u:m.set(d,u),x.push(d);for(r=-1;++r<f;)d=t.call(e,i=e[r],r),(u=m.get(d))?(g[r]=u,u.__data__=i):y.has(d)||(p[r]=q(i)),y.set(d,i),m.remove(d);for(r=-1;++r<a;)m.has(x[r])&&(v[r]=n[r])}else{for(r=-1;++r<h;)u=n[r],i=e[r],u?(u.__data__=i,g[r]=u):p[r]=q(i);for(;f>r;++r)p[r]=q(e[r]);for(;a>r;++r)v[r]=n[r]}p.update=g,p.parentNode=g.parentNode=v.parentNode=n.parentNode,c.push(p),s.push(g),l.push(v)}var r,u,i=-1,a=this.length;if(!arguments.length){for(n=new Array(a=(r=this[0]).length);++i<a;)(u=r[i])&&(n[i]=u.__data__);return n}var c=U([]),s=_([]),l=_([]);if("function"==typeof n)for(;++i<a;)e(r=this[i],n.call(r,r.parentNode.__data__,i));else for(;++i<a;)e(r=this[i],n);return s.enter=function(){return c},s.exit=function(){return l},s},pa.datum=function(n){return arguments.length?this.property("__data__",n):this.property("__data__")},pa.filter=function(n){var t,e,r,u=[];"function"!=typeof n&&(n=R(n));for(var i=0,o=this.length;o>i;i++){u.push(t=[]),t.parentNode=(e=this[i]).parentNode;for(var a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a,i)&&t.push(r)}return _(u)},pa.order=function(){for(var n=-1,t=this.length;++n<t;)for(var e,r=this[n],u=r.length-1,i=r[u];--u>=0;)(e=r[u])&&(i&&i!==e.nextSibling&&i.parentNode.insertBefore(e,i),i=e);return this},pa.sort=function(n){n=D.apply(this,arguments);for(var t=-1,e=this.length;++t<e;)this[t].sort(n);return this.order()},pa.each=function(n){return P(this,function(t,e,r){n.call(t,t.__data__,e,r)})},pa.call=function(n){var t=Xo(arguments);return n.apply(t[0]=this,t),this},pa.empty=function(){return!this.node()},pa.node=function(){for(var n=0,t=this.length;t>n;n++)for(var e=this[n],r=0,u=e.length;u>r;r++){var i=e[r];if(i)return i}return null},pa.size=function(){var n=0;return this.each(function(){++n}),n};var da=[];Zo.selection.enter=U,Zo.selection.enter.prototype=da,da.append=pa.append,da.empty=pa.empty,da.node=pa.node,da.call=pa.call,da.size=pa.size,da.select=function(n){for(var t,e,r,u,i,o=[],a=-1,c=this.length;++a<c;){r=(u=this[a]).update,o.push(t=[]),t.parentNode=u.parentNode;for(var s=-1,l=u.length;++s<l;)(i=u[s])?(t.push(r[s]=e=n.call(u.parentNode,i.__data__,s,a)),e.__data__=i.__data__):t.push(null)}return _(o)},da.insert=function(n,t){return arguments.length<2&&(t=j(this)),pa.insert.call(this,n,t)},pa.transition=function(){for(var n,t,e=Ss||++Ns,r=[],u=ks||{time:Date.now(),ease:xu,delay:0,duration:250},i=-1,o=this.length;++i<o;){r.push(n=[]);for(var a=this[i],c=-1,s=a.length;++c<s;)(t=a[c])&&Po(t,c,e,u),n.push(t)}return qo(r,e)},pa.interrupt=function(){return this.each(H)},Zo.select=function(n){var t=["string"==typeof n?la(n,$o):n];return t.parentNode=Bo,_([t])},Zo.selectAll=function(n){var t=Xo("string"==typeof n?fa(n,$o):n);return t.parentNode=Bo,_([t])};var ma=Zo.select(Bo);pa.on=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(F(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(F(n,t,e))};var ya=Zo.map({mouseenter:"mouseover",mouseleave:"mouseout"});ya.forEach(function(n){"on"+n in $o&&ya.remove(n)});var xa="onselectstart"in $o?null:p(Bo.style,"userSelect"),Ma=0;Zo.mouse=function(n){return Z(n,x())};var _a=/WebKit/.test(Wo.navigator.userAgent)?-1:0;Zo.touches=function(n,t){return arguments.length<2&&(t=x().touches),t?Xo(t).map(function(t){var e=Z(n,t);return e.identifier=t.identifier,e}):[]},Zo.behavior.drag=function(){function n(){this.on("mousedown.drag",u).on("touchstart.drag",i)}function t(n,t,u,i,o){return function(){function a(){var n,e,r=t(h,v);r&&(n=r[0]-x[0],e=r[1]-x[1],p|=n|e,x=r,g({type:"drag",x:r[0]+s[0],y:r[1]+s[1],dx:n,dy:e}))}function c(){t(h,v)&&(m.on(i+d,null).on(o+d,null),y(p&&Zo.event.target===f),g({type:"dragend"}))}var s,l=this,f=Zo.event.target,h=l.parentNode,g=e.of(l,arguments),p=0,v=n(),d=".drag"+(null==v?"":"-"+v),m=Zo.select(u()).on(i+d,a).on(o+d,c),y=I(),x=t(h,v);r?(s=r.apply(l,arguments),s=[s.x-x[0],s.y-x[1]]):s=[0,0],g({type:"dragstart"})}}var e=M(n,"drag","dragstart","dragend"),r=null,u=t(v,Zo.mouse,$,"mousemove","mouseup"),i=t(V,Zo.touch,X,"touchmove","touchend");return n.origin=function(t){return arguments.length?(r=t,n):r},Zo.rebind(n,e,"on")};var ba=Math.PI,wa=2*ba,Sa=ba/2,ka=1e-6,Ea=ka*ka,Aa=ba/180,Ca=180/ba,Na=Math.SQRT2,za=2,La=4;Zo.interpolateZoom=function(n,t){function e(n){var t=n*y;if(m){var e=Q(v),o=i/(za*h)*(e*nt(Na*t+v)-K(v));return[r+o*s,u+o*l,i*e/Q(Na*t+v)]}return[r+n*s,u+n*l,i*Math.exp(Na*t)]}var r=n[0],u=n[1],i=n[2],o=t[0],a=t[1],c=t[2],s=o-r,l=a-u,f=s*s+l*l,h=Math.sqrt(f),g=(c*c-i*i+La*f)/(2*i*za*h),p=(c*c-i*i-La*f)/(2*c*za*h),v=Math.log(Math.sqrt(g*g+1)-g),d=Math.log(Math.sqrt(p*p+1)-p),m=d-v,y=(m||Math.log(c/i))/Na;return e.duration=1e3*y,e},Zo.behavior.zoom=function(){function n(n){n.on(A,s).on(Ra+".zoom",f).on("dblclick.zoom",h).on(z,l)}function t(n){return[(n[0]-S.x)/S.k,(n[1]-S.y)/S.k]}function e(n){return[n[0]*S.k+S.x,n[1]*S.k+S.y]}function r(n){S.k=Math.max(E[0],Math.min(E[1],n))}function u(n,t){t=e(t),S.x+=n[0]-t[0],S.y+=n[1]-t[1]}function i(){_&&_.domain(x.range().map(function(n){return(n-S.x)/S.k}).map(x.invert)),w&&w.domain(b.range().map(function(n){return(n-S.y)/S.k}).map(b.invert))}function o(n){n({type:"zoomstart"})}function a(n){i(),n({type:"zoom",scale:S.k,translate:[S.x,S.y]})}function c(n){n({type:"zoomend"})}function s(){function n(){l=1,u(Zo.mouse(r),h),a(s)}function e(){f.on(C,null).on(N,null),g(l&&Zo.event.target===i),c(s)}var r=this,i=Zo.event.target,s=L.of(r,arguments),l=0,f=Zo.select(Wo).on(C,n).on(N,e),h=t(Zo.mouse(r)),g=I();H.call(r),o(s)}function l(){function n(){var n=Zo.touches(g);return h=S.k,n.forEach(function(n){n.identifier in v&&(v[n.identifier]=t(n))}),n}function e(){var t=Zo.event.target;Zo.select(t).on(M,i).on(_,f),b.push(t);for(var e=Zo.event.changedTouches,o=0,c=e.length;c>o;++o)v[e[o].identifier]=null;var s=n(),l=Date.now();if(1===s.length){if(500>l-m){var h=s[0],g=v[h.identifier];r(2*S.k),u(h,g),y(),a(p)}m=l}else if(s.length>1){var h=s[0],x=s[1],w=h[0]-x[0],k=h[1]-x[1];d=w*w+k*k}}function i(){for(var n,t,e,i,o=Zo.touches(g),c=0,s=o.length;s>c;++c,i=null)if(e=o[c],i=v[e.identifier]){if(t)break;n=e,t=i}if(i){var l=(l=e[0]-n[0])*l+(l=e[1]-n[1])*l,f=d&&Math.sqrt(l/d);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+i[0])/2,(t[1]+i[1])/2],r(f*h)}m=null,u(n,t),a(p)}function f(){if(Zo.event.touches.length){for(var t=Zo.event.changedTouches,e=0,r=t.length;r>e;++e)delete v[t[e].identifier];for(var u in v)return void n()}Zo.selectAll(b).on(x,null),w.on(A,s).on(z,l),k(),c(p)}var h,g=this,p=L.of(g,arguments),v={},d=0,x=".zoom-"+Zo.event.changedTouches[0].identifier,M="touchmove"+x,_="touchend"+x,b=[],w=Zo.select(g).on(A,null).on(z,e),k=I();H.call(g),e(),o(p)}function f(){var n=L.of(this,arguments);d?clearTimeout(d):(g=t(p=v||Zo.mouse(this)),H.call(this),o(n)),d=setTimeout(function(){d=null,c(n)},50),y(),r(Math.pow(2,.002*Ta())*S.k),u(p,g),a(n)}function h(){var n=L.of(this,arguments),e=Zo.mouse(this),i=t(e),s=Math.log(S.k)/Math.LN2;o(n),r(Math.pow(2,Zo.event.shiftKey?Math.ceil(s)-1:Math.floor(s)+1)),u(e,i),a(n),c(n)}var g,p,v,d,m,x,_,b,w,S={x:0,y:0,k:1},k=[960,500],E=qa,A="mousedown.zoom",C="mousemove.zoom",N="mouseup.zoom",z="touchstart.zoom",L=M(n,"zoomstart","zoom","zoomend");return n.event=function(n){n.each(function(){var n=L.of(this,arguments),t=S;Ss?Zo.select(this).transition().each("start.zoom",function(){S=this.__chart__||{x:0,y:0,k:1},o(n)}).tween("zoom:zoom",function(){var e=k[0],r=k[1],u=e/2,i=r/2,o=Zo.interpolateZoom([(u-S.x)/S.k,(i-S.y)/S.k,e/S.k],[(u-t.x)/t.k,(i-t.y)/t.k,e/t.k]);return function(t){var r=o(t),c=e/r[2];this.__chart__=S={x:u-r[0]*c,y:i-r[1]*c,k:c},a(n)}}).each("end.zoom",function(){c(n)}):(this.__chart__=S,o(n),a(n),c(n))})},n.translate=function(t){return arguments.length?(S={x:+t[0],y:+t[1],k:S.k},i(),n):[S.x,S.y]},n.scale=function(t){return arguments.length?(S={x:S.x,y:S.y,k:+t},i(),n):S.k},n.scaleExtent=function(t){return arguments.length?(E=null==t?qa:[+t[0],+t[1]],n):E},n.center=function(t){return arguments.length?(v=t&&[+t[0],+t[1]],n):v},n.size=function(t){return arguments.length?(k=t&&[+t[0],+t[1]],n):k},n.x=function(t){return arguments.length?(_=t,x=t.copy(),S={x:0,y:0,k:1},n):_},n.y=function(t){return arguments.length?(w=t,b=t.copy(),S={x:0,y:0,k:1},n):w},Zo.rebind(n,L,"on")};var Ta,qa=[0,1/0],Ra="onwheel"in $o?(Ta=function(){return-Zo.event.deltaY*(Zo.event.deltaMode?120:1)},"wheel"):"onmousewheel"in $o?(Ta=function(){return Zo.event.wheelDelta},"mousewheel"):(Ta=function(){return-Zo.event.detail},"MozMousePixelScroll");Zo.color=et,et.prototype.toString=function(){return this.rgb()+""},Zo.hsl=rt;var Da=rt.prototype=new et;Da.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),new rt(this.h,this.s,this.l/n)},Da.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new rt(this.h,this.s,n*this.l)},Da.rgb=function(){return ut(this.h,this.s,this.l)},Zo.hcl=it;var Pa=it.prototype=new et;Pa.brighter=function(n){return new it(this.h,this.c,Math.min(100,this.l+Ua*(arguments.length?n:1)))},Pa.darker=function(n){return new it(this.h,this.c,Math.max(0,this.l-Ua*(arguments.length?n:1)))},Pa.rgb=function(){return ot(this.h,this.c,this.l).rgb()},Zo.lab=at;var Ua=18,ja=.95047,Ha=1,Fa=1.08883,Oa=at.prototype=new et;Oa.brighter=function(n){return new at(Math.min(100,this.l+Ua*(arguments.length?n:1)),this.a,this.b)},Oa.darker=function(n){return new at(Math.max(0,this.l-Ua*(arguments.length?n:1)),this.a,this.b)},Oa.rgb=function(){return ct(this.l,this.a,this.b)},Zo.rgb=gt;var Ya=gt.prototype=new et;Ya.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,u=30;return t||e||r?(t&&u>t&&(t=u),e&&u>e&&(e=u),r&&u>r&&(r=u),new gt(Math.min(255,t/n),Math.min(255,e/n),Math.min(255,r/n))):new gt(u,u,u)},Ya.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new gt(n*this.r,n*this.g,n*this.b)},Ya.hsl=function(){return yt(this.r,this.g,this.b)},Ya.toString=function(){return"#"+dt(this.r)+dt(this.g)+dt(this.b)};var Ia=Zo.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074});Ia.forEach(function(n,t){Ia.set(n,pt(t))}),Zo.functor=bt,Zo.xhr=St(wt),Zo.dsv=function(n,t){function e(n,e,i){arguments.length<3&&(i=e,e=null);var o=kt(n,t,null==e?r:u(e),i);return o.row=function(n){return arguments.length?o.response(null==(e=n)?r:u(n)):e},o}function r(n){return e.parse(n.responseText)}function u(n){return function(t){return e.parse(t.responseText,n)}}function i(t){return t.map(o).join(n)}function o(n){return a.test(n)?'"'+n.replace(/\"/g,'""')+'"':n}var a=new RegExp('["'+n+"\n]"),c=n.charCodeAt(0);return e.parse=function(n,t){var r;return e.parseRows(n,function(n,e){if(r)return r(n,e-1);var u=new Function("d","return {"+n.map(function(n,t){return JSON.stringify(n)+": d["+t+"]"}).join(",")+"}");r=t?function(n,e){return t(u(n),e)}:u})},e.parseRows=function(n,t){function e(){if(l>=s)return o;if(u)return u=!1,i;var t=l;if(34===n.charCodeAt(t)){for(var e=t;e++<s;)if(34===n.charCodeAt(e)){if(34!==n.charCodeAt(e+1))break;++e}l=e+2;var r=n.charCodeAt(e+1);return 13===r?(u=!0,10===n.charCodeAt(e+2)&&++l):10===r&&(u=!0),n.substring(t+1,e).replace(/""/g,'"')}for(;s>l;){var r=n.charCodeAt(l++),a=1;if(10===r)u=!0;else if(13===r)u=!0,10===n.charCodeAt(l)&&(++l,++a);else if(r!==c)continue;return n.substring(t,l-a)}return n.substring(t)}for(var r,u,i={},o={},a=[],s=n.length,l=0,f=0;(r=e())!==o;){for(var h=[];r!==i&&r!==o;)h.push(r),r=e();(!t||(h=t(h,f++)))&&a.push(h)}return a},e.format=function(t){if(Array.isArray(t[0]))return e.formatRows(t);var r=new h,u=[];return t.forEach(function(n){for(var t in n)r.has(t)||u.push(r.add(t))}),[u.map(o).join(n)].concat(t.map(function(t){return u.map(function(n){return o(t[n])}).join(n)})).join("\n")},e.formatRows=function(n){return n.map(i).join("\n")},e},Zo.csv=Zo.dsv(",","text/csv"),Zo.tsv=Zo.dsv("	","text/tab-separated-values"),Zo.touch=function(n,t,e){if(arguments.length<3&&(e=t,t=x().changedTouches),t)for(var r,u=0,i=t.length;i>u;++u)if((r=t[u]).identifier===e)return Z(n,r)};var Za,Va,Xa,$a,Ba,Wa=Wo[p(Wo,"requestAnimationFrame")]||function(n){setTimeout(n,17)};Zo.timer=function(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var u=e+t,i={c:n,t:u,f:!1,n:null};Va?Va.n=i:Za=i,Va=i,Xa||($a=clearTimeout($a),Xa=1,Wa(At))},Zo.timer.flush=function(){Ct(),Nt()},Zo.round=function(n,t){return t?Math.round(n*(t=Math.pow(10,t)))/t:Math.round(n)};var Ja=["y","z","a","f","p","n","\xb5","m","","k","M","G","T","P","E","Z","Y"].map(Lt);Zo.formatPrefix=function(n,t){var e=0;return n&&(0>n&&(n*=-1),t&&(n=Zo.round(n,zt(n,t))),e=1+Math.floor(1e-12+Math.log(n)/Math.LN10),e=Math.max(-24,Math.min(24,3*Math.floor((e-1)/3)))),Ja[8+e/3]};var Ga=/(?:([^{])?([<>=^]))?([+\- ])?([$#])?(0)?(\d+)?(,)?(\.-?\d+)?([a-z%])?/i,Ka=Zo.map({b:function(n){return n.toString(2)},c:function(n){return String.fromCharCode(n)},o:function(n){return n.toString(8)},x:function(n){return n.toString(16)},X:function(n){return n.toString(16).toUpperCase()},g:function(n,t){return n.toPrecision(t)},e:function(n,t){return n.toExponential(t)},f:function(n,t){return n.toFixed(t)},r:function(n,t){return(n=Zo.round(n,zt(n,t))).toFixed(Math.max(0,Math.min(20,zt(n*(1+1e-15),t))))}}),Qa=Zo.time={},nc=Date;Rt.prototype={getDate:function(){return this._.getUTCDate()},getDay:function(){return this._.getUTCDay()},getFullYear:function(){return this._.getUTCFullYear()},getHours:function(){return this._.getUTCHours()},getMilliseconds:function(){return this._.getUTCMilliseconds()},getMinutes:function(){return this._.getUTCMinutes()},getMonth:function(){return this._.getUTCMonth()},getSeconds:function(){return this._.getUTCSeconds()},getTime:function(){return this._.getTime()},getTimezoneOffset:function(){return 0},valueOf:function(){return this._.valueOf()},setDate:function(){tc.setUTCDate.apply(this._,arguments)},setDay:function(){tc.setUTCDay.apply(this._,arguments)},setFullYear:function(){tc.setUTCFullYear.apply(this._,arguments)},setHours:function(){tc.setUTCHours.apply(this._,arguments)},setMilliseconds:function(){tc.setUTCMilliseconds.apply(this._,arguments)},setMinutes:function(){tc.setUTCMinutes.apply(this._,arguments)},setMonth:function(){tc.setUTCMonth.apply(this._,arguments)},setSeconds:function(){tc.setUTCSeconds.apply(this._,arguments)},setTime:function(){tc.setTime.apply(this._,arguments)}};var tc=Date.prototype;Qa.year=Dt(function(n){return n=Qa.day(n),n.setMonth(0,1),n},function(n,t){n.setFullYear(n.getFullYear()+t)},function(n){return n.getFullYear()}),Qa.years=Qa.year.range,Qa.years.utc=Qa.year.utc.range,Qa.day=Dt(function(n){var t=new nc(2e3,0);return t.setFullYear(n.getFullYear(),n.getMonth(),n.getDate()),t},function(n,t){n.setDate(n.getDate()+t)},function(n){return n.getDate()-1}),Qa.days=Qa.day.range,Qa.days.utc=Qa.day.utc.range,Qa.dayOfYear=function(n){var t=Qa.year(n);return Math.floor((n-t-6e4*(n.getTimezoneOffset()-t.getTimezoneOffset()))/864e5)},["sunday","monday","tuesday","wednesday","thursday","friday","saturday"].forEach(function(n,t){t=7-t;var e=Qa[n]=Dt(function(n){return(n=Qa.day(n)).setDate(n.getDate()-(n.getDay()+t)%7),n},function(n,t){n.setDate(n.getDate()+7*Math.floor(t))},function(n){var e=Qa.year(n).getDay();return Math.floor((Qa.dayOfYear(n)+(e+t)%7)/7)-(e!==t)});Qa[n+"s"]=e.range,Qa[n+"s"].utc=e.utc.range,Qa[n+"OfYear"]=function(n){var e=Qa.year(n).getDay();return Math.floor((Qa.dayOfYear(n)+(e+t)%7)/7)}}),Qa.week=Qa.sunday,Qa.weeks=Qa.sunday.range,Qa.weeks.utc=Qa.sunday.utc.range,Qa.weekOfYear=Qa.sundayOfYear;var ec={"-":"",_:" ",0:"0"},rc=/^\s*\d+/,uc=/^%/;Zo.locale=function(n){return{numberFormat:Tt(n),timeFormat:Ut(n)}};var ic=Zo.locale({decimal:".",thousands:",",grouping:[3],currency:["$",""],dateTime:"%a %b %e %X %Y",date:"%m/%d/%Y",time:"%H:%M:%S",periods:["AM","PM"],days:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],shortDays:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],months:["January","February","March","April","May","June","July","August","September","October","November","December"],shortMonths:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]});Zo.format=ic.numberFormat,Zo.geo={},ue.prototype={s:0,t:0,add:function(n){ie(n,this.t,oc),ie(oc.s,this.s,this),this.s?this.t+=oc.t:this.s=oc.t},reset:function(){this.s=this.t=0},valueOf:function(){return this.s}};var oc=new ue;Zo.geo.stream=function(n,t){n&&ac.hasOwnProperty(n.type)?ac[n.type](n,t):oe(n,t)};var ac={Feature:function(n,t){oe(n.geometry,t)},FeatureCollection:function(n,t){for(var e=n.features,r=-1,u=e.length;++r<u;)oe(e[r].geometry,t)}},cc={Sphere:function(n,t){t.sphere()},Point:function(n,t){n=n.coordinates,t.point(n[0],n[1],n[2])},MultiPoint:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)n=e[r],t.point(n[0],n[1],n[2])},LineString:function(n,t){ae(n.coordinates,t,0)},MultiLineString:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)ae(e[r],t,0)},Polygon:function(n,t){ce(n.coordinates,t)},MultiPolygon:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)ce(e[r],t)},GeometryCollection:function(n,t){for(var e=n.geometries,r=-1,u=e.length;++r<u;)oe(e[r],t)}};Zo.geo.area=function(n){return sc=0,Zo.geo.stream(n,fc),sc};var sc,lc=new ue,fc={sphere:function(){sc+=4*ba},point:v,lineStart:v,lineEnd:v,polygonStart:function(){lc.reset(),fc.lineStart=se},polygonEnd:function(){var n=2*lc;sc+=0>n?4*ba+n:n,fc.lineStart=fc.lineEnd=fc.point=v}};Zo.geo.bounds=function(){function n(n,t){x.push(M=[l=n,h=n]),f>t&&(f=t),t>g&&(g=t)}function t(t,e){var r=le([t*Aa,e*Aa]);if(m){var u=he(m,r),i=[u[1],-u[0],0],o=he(i,u);ve(o),o=de(o);var c=t-p,s=c>0?1:-1,v=o[0]*Ca*s,d=ua(c)>180;if(d^(v>s*p&&s*t>v)){var y=o[1]*Ca;y>g&&(g=y)}else if(v=(v+360)%360-180,d^(v>s*p&&s*t>v)){var y=-o[1]*Ca;f>y&&(f=y)}else f>e&&(f=e),e>g&&(g=e);d?p>t?a(l,t)>a(l,h)&&(h=t):a(t,h)>a(l,h)&&(l=t):h>=l?(l>t&&(l=t),t>h&&(h=t)):t>p?a(l,t)>a(l,h)&&(h=t):a(t,h)>a(l,h)&&(l=t)}else n(t,e);m=r,p=t}function e(){_.point=t}function r(){M[0]=l,M[1]=h,_.point=n,m=null}function u(n,e){if(m){var r=n-p;y+=ua(r)>180?r+(r>0?360:-360):r}else v=n,d=e;fc.point(n,e),t(n,e)}function i(){fc.lineStart()}function o(){u(v,d),fc.lineEnd(),ua(y)>ka&&(l=-(h=180)),M[0]=l,M[1]=h,m=null}function a(n,t){return(t-=n)<0?t+360:t}function c(n,t){return n[0]-t[0]}function s(n,t){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:n<t[0]||t[1]<n}var l,f,h,g,p,v,d,m,y,x,M,_={point:n,lineStart:e,lineEnd:r,polygonStart:function(){_.point=u,_.lineStart=i,_.lineEnd=o,y=0,fc.polygonStart()},polygonEnd:function(){fc.polygonEnd(),_.point=n,_.lineStart=e,_.lineEnd=r,0>lc?(l=-(h=180),f=-(g=90)):y>ka?g=90:-ka>y&&(f=-90),M[0]=l,M[1]=h}};return function(n){g=h=-(l=f=1/0),x=[],Zo.geo.stream(n,_);var t=x.length;if(t){x.sort(c);for(var e,r=1,u=x[0],i=[u];t>r;++r)e=x[r],s(e[0],u)||s(e[1],u)?(a(u[0],e[1])>a(u[0],u[1])&&(u[1]=e[1]),a(e[0],u[1])>a(u[0],u[1])&&(u[0]=e[0])):i.push(u=e);
+for(var o,e,p=-1/0,t=i.length-1,r=0,u=i[t];t>=r;u=e,++r)e=i[r],(o=a(u[1],e[0]))>p&&(p=o,l=e[0],h=u[1])}return x=M=null,1/0===l||1/0===f?[[0/0,0/0],[0/0,0/0]]:[[l,f],[h,g]]}}(),Zo.geo.centroid=function(n){hc=gc=pc=vc=dc=mc=yc=xc=Mc=_c=bc=0,Zo.geo.stream(n,wc);var t=Mc,e=_c,r=bc,u=t*t+e*e+r*r;return Ea>u&&(t=mc,e=yc,r=xc,ka>gc&&(t=pc,e=vc,r=dc),u=t*t+e*e+r*r,Ea>u)?[0/0,0/0]:[Math.atan2(e,t)*Ca,G(r/Math.sqrt(u))*Ca]};var hc,gc,pc,vc,dc,mc,yc,xc,Mc,_c,bc,wc={sphere:v,point:ye,lineStart:Me,lineEnd:_e,polygonStart:function(){wc.lineStart=be},polygonEnd:function(){wc.lineStart=Me}},Sc=Ae(we,Te,Re,[-ba,-ba/2]),kc=1e9;Zo.geo.clipExtent=function(){var n,t,e,r,u,i,o={stream:function(n){return u&&(u.valid=!1),u=i(n),u.valid=!0,u},extent:function(a){return arguments.length?(i=Ue(n=+a[0][0],t=+a[0][1],e=+a[1][0],r=+a[1][1]),u&&(u.valid=!1,u=null),o):[[n,t],[e,r]]}};return o.extent([[0,0],[960,500]])},(Zo.geo.conicEqualArea=function(){return He(Fe)}).raw=Fe,Zo.geo.albers=function(){return Zo.geo.conicEqualArea().rotate([96,0]).center([-.6,38.7]).parallels([29.5,45.5]).scale(1070)},Zo.geo.albersUsa=function(){function n(n){var i=n[0],o=n[1];return t=null,e(i,o),t||(r(i,o),t)||u(i,o),t}var t,e,r,u,i=Zo.geo.albers(),o=Zo.geo.conicEqualArea().rotate([154,0]).center([-2,58.5]).parallels([55,65]),a=Zo.geo.conicEqualArea().rotate([157,0]).center([-3,19.9]).parallels([8,18]),c={point:function(n,e){t=[n,e]}};return n.invert=function(n){var t=i.scale(),e=i.translate(),r=(n[0]-e[0])/t,u=(n[1]-e[1])/t;return(u>=.12&&.234>u&&r>=-.425&&-.214>r?o:u>=.166&&.234>u&&r>=-.214&&-.115>r?a:i).invert(n)},n.stream=function(n){var t=i.stream(n),e=o.stream(n),r=a.stream(n);return{point:function(n,u){t.point(n,u),e.point(n,u),r.point(n,u)},sphere:function(){t.sphere(),e.sphere(),r.sphere()},lineStart:function(){t.lineStart(),e.lineStart(),r.lineStart()},lineEnd:function(){t.lineEnd(),e.lineEnd(),r.lineEnd()},polygonStart:function(){t.polygonStart(),e.polygonStart(),r.polygonStart()},polygonEnd:function(){t.polygonEnd(),e.polygonEnd(),r.polygonEnd()}}},n.precision=function(t){return arguments.length?(i.precision(t),o.precision(t),a.precision(t),n):i.precision()},n.scale=function(t){return arguments.length?(i.scale(t),o.scale(.35*t),a.scale(t),n.translate(i.translate())):i.scale()},n.translate=function(t){if(!arguments.length)return i.translate();var s=i.scale(),l=+t[0],f=+t[1];return e=i.translate(t).clipExtent([[l-.455*s,f-.238*s],[l+.455*s,f+.238*s]]).stream(c).point,r=o.translate([l-.307*s,f+.201*s]).clipExtent([[l-.425*s+ka,f+.12*s+ka],[l-.214*s-ka,f+.234*s-ka]]).stream(c).point,u=a.translate([l-.205*s,f+.212*s]).clipExtent([[l-.214*s+ka,f+.166*s+ka],[l-.115*s-ka,f+.234*s-ka]]).stream(c).point,n},n.scale(1070)};var Ec,Ac,Cc,Nc,zc,Lc,Tc={point:v,lineStart:v,lineEnd:v,polygonStart:function(){Ac=0,Tc.lineStart=Oe},polygonEnd:function(){Tc.lineStart=Tc.lineEnd=Tc.point=v,Ec+=ua(Ac/2)}},qc={point:Ye,lineStart:v,lineEnd:v,polygonStart:v,polygonEnd:v},Rc={point:Ve,lineStart:Xe,lineEnd:$e,polygonStart:function(){Rc.lineStart=Be},polygonEnd:function(){Rc.point=Ve,Rc.lineStart=Xe,Rc.lineEnd=$e}};Zo.geo.path=function(){function n(n){return n&&("function"==typeof a&&i.pointRadius(+a.apply(this,arguments)),o&&o.valid||(o=u(i)),Zo.geo.stream(n,o)),i.result()}function t(){return o=null,n}var e,r,u,i,o,a=4.5;return n.area=function(n){return Ec=0,Zo.geo.stream(n,u(Tc)),Ec},n.centroid=function(n){return pc=vc=dc=mc=yc=xc=Mc=_c=bc=0,Zo.geo.stream(n,u(Rc)),bc?[Mc/bc,_c/bc]:xc?[mc/xc,yc/xc]:dc?[pc/dc,vc/dc]:[0/0,0/0]},n.bounds=function(n){return zc=Lc=-(Cc=Nc=1/0),Zo.geo.stream(n,u(qc)),[[Cc,Nc],[zc,Lc]]},n.projection=function(n){return arguments.length?(u=(e=n)?n.stream||Ge(n):wt,t()):e},n.context=function(n){return arguments.length?(i=null==(r=n)?new Ie:new We(n),"function"!=typeof a&&i.pointRadius(a),t()):r},n.pointRadius=function(t){return arguments.length?(a="function"==typeof t?t:(i.pointRadius(+t),+t),n):a},n.projection(Zo.geo.albersUsa()).context(null)},Zo.geo.transform=function(n){return{stream:function(t){var e=new Ke(t);for(var r in n)e[r]=n[r];return e}}},Ke.prototype={point:function(n,t){this.stream.point(n,t)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart()},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}},Zo.geo.projection=nr,Zo.geo.projectionMutator=tr,(Zo.geo.equirectangular=function(){return nr(rr)}).raw=rr.invert=rr,Zo.geo.rotation=function(n){function t(t){return t=n(t[0]*Aa,t[1]*Aa),t[0]*=Ca,t[1]*=Ca,t}return n=ir(n[0]%360*Aa,n[1]*Aa,n.length>2?n[2]*Aa:0),t.invert=function(t){return t=n.invert(t[0]*Aa,t[1]*Aa),t[0]*=Ca,t[1]*=Ca,t},t},ur.invert=rr,Zo.geo.circle=function(){function n(){var n="function"==typeof r?r.apply(this,arguments):r,t=ir(-n[0]*Aa,-n[1]*Aa,0).invert,u=[];return e(null,null,1,{point:function(n,e){u.push(n=t(n,e)),n[0]*=Ca,n[1]*=Ca}}),{type:"Polygon",coordinates:[u]}}var t,e,r=[0,0],u=6;return n.origin=function(t){return arguments.length?(r=t,n):r},n.angle=function(r){return arguments.length?(e=sr((t=+r)*Aa,u*Aa),n):t},n.precision=function(r){return arguments.length?(e=sr(t*Aa,(u=+r)*Aa),n):u},n.angle(90)},Zo.geo.distance=function(n,t){var e,r=(t[0]-n[0])*Aa,u=n[1]*Aa,i=t[1]*Aa,o=Math.sin(r),a=Math.cos(r),c=Math.sin(u),s=Math.cos(u),l=Math.sin(i),f=Math.cos(i);return Math.atan2(Math.sqrt((e=f*o)*e+(e=s*l-c*f*a)*e),c*l+s*f*a)},Zo.geo.graticule=function(){function n(){return{type:"MultiLineString",coordinates:t()}}function t(){return Zo.range(Math.ceil(i/d)*d,u,d).map(h).concat(Zo.range(Math.ceil(s/m)*m,c,m).map(g)).concat(Zo.range(Math.ceil(r/p)*p,e,p).filter(function(n){return ua(n%d)>ka}).map(l)).concat(Zo.range(Math.ceil(a/v)*v,o,v).filter(function(n){return ua(n%m)>ka}).map(f))}var e,r,u,i,o,a,c,s,l,f,h,g,p=10,v=p,d=90,m=360,y=2.5;return n.lines=function(){return t().map(function(n){return{type:"LineString",coordinates:n}})},n.outline=function(){return{type:"Polygon",coordinates:[h(i).concat(g(c).slice(1),h(u).reverse().slice(1),g(s).reverse().slice(1))]}},n.extent=function(t){return arguments.length?n.majorExtent(t).minorExtent(t):n.minorExtent()},n.majorExtent=function(t){return arguments.length?(i=+t[0][0],u=+t[1][0],s=+t[0][1],c=+t[1][1],i>u&&(t=i,i=u,u=t),s>c&&(t=s,s=c,c=t),n.precision(y)):[[i,s],[u,c]]},n.minorExtent=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],a=+t[0][1],o=+t[1][1],r>e&&(t=r,r=e,e=t),a>o&&(t=a,a=o,o=t),n.precision(y)):[[r,a],[e,o]]},n.step=function(t){return arguments.length?n.majorStep(t).minorStep(t):n.minorStep()},n.majorStep=function(t){return arguments.length?(d=+t[0],m=+t[1],n):[d,m]},n.minorStep=function(t){return arguments.length?(p=+t[0],v=+t[1],n):[p,v]},n.precision=function(t){return arguments.length?(y=+t,l=fr(a,o,90),f=hr(r,e,y),h=fr(s,c,90),g=hr(i,u,y),n):y},n.majorExtent([[-180,-90+ka],[180,90-ka]]).minorExtent([[-180,-80-ka],[180,80+ka]])},Zo.geo.greatArc=function(){function n(){return{type:"LineString",coordinates:[t||r.apply(this,arguments),e||u.apply(this,arguments)]}}var t,e,r=gr,u=pr;return n.distance=function(){return Zo.geo.distance(t||r.apply(this,arguments),e||u.apply(this,arguments))},n.source=function(e){return arguments.length?(r=e,t="function"==typeof e?null:e,n):r},n.target=function(t){return arguments.length?(u=t,e="function"==typeof t?null:t,n):u},n.precision=function(){return arguments.length?n:0},n},Zo.geo.interpolate=function(n,t){return vr(n[0]*Aa,n[1]*Aa,t[0]*Aa,t[1]*Aa)},Zo.geo.length=function(n){return Dc=0,Zo.geo.stream(n,Pc),Dc};var Dc,Pc={sphere:v,point:v,lineStart:dr,lineEnd:v,polygonStart:v,polygonEnd:v},Uc=mr(function(n){return Math.sqrt(2/(1+n))},function(n){return 2*Math.asin(n/2)});(Zo.geo.azimuthalEqualArea=function(){return nr(Uc)}).raw=Uc;var jc=mr(function(n){var t=Math.acos(n);return t&&t/Math.sin(t)},wt);(Zo.geo.azimuthalEquidistant=function(){return nr(jc)}).raw=jc,(Zo.geo.conicConformal=function(){return He(yr)}).raw=yr,(Zo.geo.conicEquidistant=function(){return He(xr)}).raw=xr;var Hc=mr(function(n){return 1/n},Math.atan);(Zo.geo.gnomonic=function(){return nr(Hc)}).raw=Hc,Mr.invert=function(n,t){return[n,2*Math.atan(Math.exp(t))-Sa]},(Zo.geo.mercator=function(){return _r(Mr)}).raw=Mr;var Fc=mr(function(){return 1},Math.asin);(Zo.geo.orthographic=function(){return nr(Fc)}).raw=Fc;var Oc=mr(function(n){return 1/(1+n)},function(n){return 2*Math.atan(n)});(Zo.geo.stereographic=function(){return nr(Oc)}).raw=Oc,br.invert=function(n,t){return[-t,2*Math.atan(Math.exp(n))-Sa]},(Zo.geo.transverseMercator=function(){var n=_r(br),t=n.center,e=n.rotate;return n.center=function(n){return n?t([-n[1],n[0]]):(n=t(),[n[1],-n[0]])},n.rotate=function(n){return n?e([n[0],n[1],n.length>2?n[2]+90:90]):(n=e(),[n[0],n[1],n[2]-90])},e([0,0,90])}).raw=br,Zo.geom={},Zo.geom.hull=function(n){function t(n){if(n.length<3)return[];var t,u=bt(e),i=bt(r),o=n.length,a=[],c=[];for(t=0;o>t;t++)a.push([+u.call(this,n[t],t),+i.call(this,n[t],t),t]);for(a.sort(Er),t=0;o>t;t++)c.push([a[t][0],-a[t][1]]);var s=kr(a),l=kr(c),f=l[0]===s[0],h=l[l.length-1]===s[s.length-1],g=[];for(t=s.length-1;t>=0;--t)g.push(n[a[s[t]][2]]);for(t=+f;t<l.length-h;++t)g.push(n[a[l[t]][2]]);return g}var e=wr,r=Sr;return arguments.length?t(n):(t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t)},Zo.geom.polygon=function(n){return sa(n,Yc),n};var Yc=Zo.geom.polygon.prototype=[];Yc.area=function(){for(var n,t=-1,e=this.length,r=this[e-1],u=0;++t<e;)n=r,r=this[t],u+=n[1]*r[0]-n[0]*r[1];return.5*u},Yc.centroid=function(n){var t,e,r=-1,u=this.length,i=0,o=0,a=this[u-1];for(arguments.length||(n=-1/(6*this.area()));++r<u;)t=a,a=this[r],e=t[0]*a[1]-a[0]*t[1],i+=(t[0]+a[0])*e,o+=(t[1]+a[1])*e;return[i*n,o*n]},Yc.clip=function(n){for(var t,e,r,u,i,o,a=Nr(n),c=-1,s=this.length-Nr(this),l=this[s-1];++c<s;){for(t=n.slice(),n.length=0,u=this[c],i=t[(r=t.length-a)-1],e=-1;++e<r;)o=t[e],Ar(o,l,u)?(Ar(i,l,u)||n.push(Cr(i,o,l,u)),n.push(o)):Ar(i,l,u)&&n.push(Cr(i,o,l,u)),i=o;a&&n.push(n[0]),l=u}return n};var Ic,Zc,Vc,Xc,$c,Bc=[],Wc=[];Ur.prototype.prepare=function(){for(var n,t=this.edges,e=t.length;e--;)n=t[e].edge,n.b&&n.a||t.splice(e,1);return t.sort(Hr),t.length},Wr.prototype={start:function(){return this.edge.l===this.site?this.edge.a:this.edge.b},end:function(){return this.edge.l===this.site?this.edge.b:this.edge.a}},Jr.prototype={insert:function(n,t){var e,r,u;if(n){if(t.P=n,t.N=n.N,n.N&&(n.N.P=t),n.N=t,n.R){for(n=n.R;n.L;)n=n.L;n.L=t}else n.R=t;e=n}else this._?(n=nu(this._),t.P=null,t.N=n,n.P=n.L=t,e=n):(t.P=t.N=null,this._=t,e=null);for(t.L=t.R=null,t.U=e,t.C=!0,n=t;e&&e.C;)r=e.U,e===r.L?(u=r.R,u&&u.C?(e.C=u.C=!1,r.C=!0,n=r):(n===e.R&&(Kr(this,e),n=e,e=n.U),e.C=!1,r.C=!0,Qr(this,r))):(u=r.L,u&&u.C?(e.C=u.C=!1,r.C=!0,n=r):(n===e.L&&(Qr(this,e),n=e,e=n.U),e.C=!1,r.C=!0,Kr(this,r))),e=n.U;this._.C=!1},remove:function(n){n.N&&(n.N.P=n.P),n.P&&(n.P.N=n.N),n.N=n.P=null;var t,e,r,u=n.U,i=n.L,o=n.R;if(e=i?o?nu(o):i:o,u?u.L===n?u.L=e:u.R=e:this._=e,i&&o?(r=e.C,e.C=n.C,e.L=i,i.U=e,e!==o?(u=e.U,e.U=n.U,n=e.R,u.L=n,e.R=o,o.U=e):(e.U=u,u=e,n=e.R)):(r=n.C,n=e),n&&(n.U=u),!r){if(n&&n.C)return n.C=!1,void 0;do{if(n===this._)break;if(n===u.L){if(t=u.R,t.C&&(t.C=!1,u.C=!0,Kr(this,u),t=u.R),t.L&&t.L.C||t.R&&t.R.C){t.R&&t.R.C||(t.L.C=!1,t.C=!0,Qr(this,t),t=u.R),t.C=u.C,u.C=t.R.C=!1,Kr(this,u),n=this._;break}}else if(t=u.L,t.C&&(t.C=!1,u.C=!0,Qr(this,u),t=u.L),t.L&&t.L.C||t.R&&t.R.C){t.L&&t.L.C||(t.R.C=!1,t.C=!0,Kr(this,t),t=u.L),t.C=u.C,u.C=t.L.C=!1,Qr(this,u),n=this._;break}t.C=!0,n=u,u=u.U}while(!n.C);n&&(n.C=!1)}}},Zo.geom.voronoi=function(n){function t(n){var t=new Array(n.length),r=a[0][0],u=a[0][1],i=a[1][0],o=a[1][1];return tu(e(n),a).cells.forEach(function(e,a){var c=e.edges,s=e.site,l=t[a]=c.length?c.map(function(n){var t=n.start();return[t.x,t.y]}):s.x>=r&&s.x<=i&&s.y>=u&&s.y<=o?[[r,o],[i,o],[i,u],[r,u]]:[];l.point=n[a]}),t}function e(n){return n.map(function(n,t){return{x:Math.round(i(n,t)/ka)*ka,y:Math.round(o(n,t)/ka)*ka,i:t}})}var r=wr,u=Sr,i=r,o=u,a=Jc;return n?t(n):(t.links=function(n){return tu(e(n)).edges.filter(function(n){return n.l&&n.r}).map(function(t){return{source:n[t.l.i],target:n[t.r.i]}})},t.triangles=function(n){var t=[];return tu(e(n)).cells.forEach(function(e,r){for(var u,i,o=e.site,a=e.edges.sort(Hr),c=-1,s=a.length,l=a[s-1].edge,f=l.l===o?l.r:l.l;++c<s;)u=l,i=f,l=a[c].edge,f=l.l===o?l.r:l.l,r<i.i&&r<f.i&&ru(o,i,f)<0&&t.push([n[r],n[i.i],n[f.i]])}),t},t.x=function(n){return arguments.length?(i=bt(r=n),t):r},t.y=function(n){return arguments.length?(o=bt(u=n),t):u},t.clipExtent=function(n){return arguments.length?(a=null==n?Jc:n,t):a===Jc?null:a},t.size=function(n){return arguments.length?t.clipExtent(n&&[[0,0],n]):a===Jc?null:a&&a[1]},t)};var Jc=[[-1e6,-1e6],[1e6,1e6]];Zo.geom.delaunay=function(n){return Zo.geom.voronoi().triangles(n)},Zo.geom.quadtree=function(n,t,e,r,u){function i(n){function i(n,t,e,r,u,i,o,a){if(!isNaN(e)&&!isNaN(r))if(n.leaf){var c=n.x,l=n.y;if(null!=c)if(ua(c-e)+ua(l-r)<.01)s(n,t,e,r,u,i,o,a);else{var f=n.point;n.x=n.y=n.point=null,s(n,f,c,l,u,i,o,a),s(n,t,e,r,u,i,o,a)}else n.x=e,n.y=r,n.point=t}else s(n,t,e,r,u,i,o,a)}function s(n,t,e,r,u,o,a,c){var s=.5*(u+a),l=.5*(o+c),f=e>=s,h=r>=l,g=(h<<1)+f;n.leaf=!1,n=n.nodes[g]||(n.nodes[g]=ou()),f?u=s:a=s,h?o=l:c=l,i(n,t,e,r,u,o,a,c)}var l,f,h,g,p,v,d,m,y,x=bt(a),M=bt(c);if(null!=t)v=t,d=e,m=r,y=u;else if(m=y=-(v=d=1/0),f=[],h=[],p=n.length,o)for(g=0;p>g;++g)l=n[g],l.x<v&&(v=l.x),l.y<d&&(d=l.y),l.x>m&&(m=l.x),l.y>y&&(y=l.y),f.push(l.x),h.push(l.y);else for(g=0;p>g;++g){var _=+x(l=n[g],g),b=+M(l,g);v>_&&(v=_),d>b&&(d=b),_>m&&(m=_),b>y&&(y=b),f.push(_),h.push(b)}var w=m-v,S=y-d;w>S?y=d+w:m=v+S;var k=ou();if(k.add=function(n){i(k,n,+x(n,++g),+M(n,g),v,d,m,y)},k.visit=function(n){au(n,k,v,d,m,y)},g=-1,null==t){for(;++g<p;)i(k,n[g],f[g],h[g],v,d,m,y);--g}else n.forEach(k.add);return f=h=n=l=null,k}var o,a=wr,c=Sr;return(o=arguments.length)?(a=uu,c=iu,3===o&&(u=e,r=t,e=t=0),i(n)):(i.x=function(n){return arguments.length?(a=n,i):a},i.y=function(n){return arguments.length?(c=n,i):c},i.extent=function(n){return arguments.length?(null==n?t=e=r=u=null:(t=+n[0][0],e=+n[0][1],r=+n[1][0],u=+n[1][1]),i):null==t?null:[[t,e],[r,u]]},i.size=function(n){return arguments.length?(null==n?t=e=r=u=null:(t=e=0,r=+n[0],u=+n[1]),i):null==t?null:[r-t,u-e]},i)},Zo.interpolateRgb=cu,Zo.interpolateObject=su,Zo.interpolateNumber=lu,Zo.interpolateString=fu;var Gc=/[-+]?(?:\d+\.?\d*|\.?\d+)(?:[eE][-+]?\d+)?/g,Kc=new RegExp(Gc.source,"g");Zo.interpolate=hu,Zo.interpolators=[function(n,t){var e=typeof t;return("string"===e?Ia.has(t)||/^(#|rgb\(|hsl\()/.test(t)?cu:fu:t instanceof et?cu:Array.isArray(t)?gu:"object"===e&&isNaN(t)?su:lu)(n,t)}],Zo.interpolateArray=gu;var Qc=function(){return wt},ns=Zo.map({linear:Qc,poly:Mu,quad:function(){return mu},cubic:function(){return yu},sin:function(){return _u},exp:function(){return bu},circle:function(){return wu},elastic:Su,back:ku,bounce:function(){return Eu}}),ts=Zo.map({"in":wt,out:vu,"in-out":du,"out-in":function(n){return du(vu(n))}});Zo.ease=function(n){var t=n.indexOf("-"),e=t>=0?n.substring(0,t):n,r=t>=0?n.substring(t+1):"in";return e=ns.get(e)||Qc,r=ts.get(r)||wt,pu(r(e.apply(null,Vo.call(arguments,1))))},Zo.interpolateHcl=Au,Zo.interpolateHsl=Cu,Zo.interpolateLab=Nu,Zo.interpolateRound=zu,Zo.transform=function(n){var t=$o.createElementNS(Zo.ns.prefix.svg,"g");return(Zo.transform=function(n){if(null!=n){t.setAttribute("transform",n);var e=t.transform.baseVal.consolidate()}return new Lu(e?e.matrix:es)})(n)},Lu.prototype.toString=function(){return"translate("+this.translate+")rotate("+this.rotate+")skewX("+this.skew+")scale("+this.scale+")"};var es={a:1,b:0,c:0,d:1,e:0,f:0};Zo.interpolateTransform=Du,Zo.layout={},Zo.layout.bundle=function(){return function(n){for(var t=[],e=-1,r=n.length;++e<r;)t.push(ju(n[e]));return t}},Zo.layout.chord=function(){function n(){var n,s,f,h,g,p={},v=[],d=Zo.range(i),m=[];for(e=[],r=[],n=0,h=-1;++h<i;){for(s=0,g=-1;++g<i;)s+=u[h][g];v.push(s),m.push(Zo.range(i)),n+=s}for(o&&d.sort(function(n,t){return o(v[n],v[t])}),a&&m.forEach(function(n,t){n.sort(function(n,e){return a(u[t][n],u[t][e])})}),n=(wa-l*i)/n,s=0,h=-1;++h<i;){for(f=s,g=-1;++g<i;){var y=d[h],x=m[y][g],M=u[y][x],_=s,b=s+=M*n;p[y+"-"+x]={index:y,subindex:x,startAngle:_,endAngle:b,value:M}}r[y]={index:y,startAngle:f,endAngle:s,value:(s-f)/n},s+=l}for(h=-1;++h<i;)for(g=h-1;++g<i;){var w=p[h+"-"+g],S=p[g+"-"+h];(w.value||S.value)&&e.push(w.value<S.value?{source:S,target:w}:{source:w,target:S})}c&&t()}function t(){e.sort(function(n,t){return c((n.source.value+n.target.value)/2,(t.source.value+t.target.value)/2)})}var e,r,u,i,o,a,c,s={},l=0;return s.matrix=function(n){return arguments.length?(i=(u=n)&&u.length,e=r=null,s):u},s.padding=function(n){return arguments.length?(l=n,e=r=null,s):l},s.sortGroups=function(n){return arguments.length?(o=n,e=r=null,s):o},s.sortSubgroups=function(n){return arguments.length?(a=n,e=null,s):a},s.sortChords=function(n){return arguments.length?(c=n,e&&t(),s):c},s.chords=function(){return e||n(),e},s.groups=function(){return r||n(),r},s},Zo.layout.force=function(){function n(n){return function(t,e,r,u){if(t.point!==n){var i=t.cx-n.x,o=t.cy-n.y,a=u-e,c=i*i+o*o;if(c>a*a/d){if(p>c){var s=t.charge/c;n.px-=i*s,n.py-=o*s}return!0}if(t.point&&c&&p>c){var s=t.pointCharge/c;n.px-=i*s,n.py-=o*s}}return!t.charge}}function t(n){n.px=Zo.event.x,n.py=Zo.event.y,a.resume()}var e,r,u,i,o,a={},c=Zo.dispatch("start","tick","end"),s=[1,1],l=.9,f=rs,h=us,g=-30,p=is,v=.1,d=.64,m=[],y=[];return a.tick=function(){if((r*=.99)<.005)return c.end({type:"end",alpha:r=0}),!0;var t,e,a,f,h,p,d,x,M,_=m.length,b=y.length;for(e=0;b>e;++e)a=y[e],f=a.source,h=a.target,x=h.x-f.x,M=h.y-f.y,(p=x*x+M*M)&&(p=r*i[e]*((p=Math.sqrt(p))-u[e])/p,x*=p,M*=p,h.x-=x*(d=f.weight/(h.weight+f.weight)),h.y-=M*d,f.x+=x*(d=1-d),f.y+=M*d);if((d=r*v)&&(x=s[0]/2,M=s[1]/2,e=-1,d))for(;++e<_;)a=m[e],a.x+=(x-a.x)*d,a.y+=(M-a.y)*d;if(g)for(Vu(t=Zo.geom.quadtree(m),r,o),e=-1;++e<_;)(a=m[e]).fixed||t.visit(n(a));for(e=-1;++e<_;)a=m[e],a.fixed?(a.x=a.px,a.y=a.py):(a.x-=(a.px-(a.px=a.x))*l,a.y-=(a.py-(a.py=a.y))*l);c.tick({type:"tick",alpha:r})},a.nodes=function(n){return arguments.length?(m=n,a):m},a.links=function(n){return arguments.length?(y=n,a):y},a.size=function(n){return arguments.length?(s=n,a):s},a.linkDistance=function(n){return arguments.length?(f="function"==typeof n?n:+n,a):f},a.distance=a.linkDistance,a.linkStrength=function(n){return arguments.length?(h="function"==typeof n?n:+n,a):h},a.friction=function(n){return arguments.length?(l=+n,a):l},a.charge=function(n){return arguments.length?(g="function"==typeof n?n:+n,a):g},a.chargeDistance=function(n){return arguments.length?(p=n*n,a):Math.sqrt(p)},a.gravity=function(n){return arguments.length?(v=+n,a):v},a.theta=function(n){return arguments.length?(d=n*n,a):Math.sqrt(d)},a.alpha=function(n){return arguments.length?(n=+n,r?r=n>0?n:0:n>0&&(c.start({type:"start",alpha:r=n}),Zo.timer(a.tick)),a):r},a.start=function(){function n(n,r){if(!e){for(e=new Array(c),a=0;c>a;++a)e[a]=[];for(a=0;s>a;++a){var u=y[a];e[u.source.index].push(u.target),e[u.target.index].push(u.source)}}for(var i,o=e[t],a=-1,s=o.length;++a<s;)if(!isNaN(i=o[a][n]))return i;return Math.random()*r}var t,e,r,c=m.length,l=y.length,p=s[0],v=s[1];for(t=0;c>t;++t)(r=m[t]).index=t,r.weight=0;for(t=0;l>t;++t)r=y[t],"number"==typeof r.source&&(r.source=m[r.source]),"number"==typeof r.target&&(r.target=m[r.target]),++r.source.weight,++r.target.weight;for(t=0;c>t;++t)r=m[t],isNaN(r.x)&&(r.x=n("x",p)),isNaN(r.y)&&(r.y=n("y",v)),isNaN(r.px)&&(r.px=r.x),isNaN(r.py)&&(r.py=r.y);if(u=[],"function"==typeof f)for(t=0;l>t;++t)u[t]=+f.call(this,y[t],t);else for(t=0;l>t;++t)u[t]=f;if(i=[],"function"==typeof h)for(t=0;l>t;++t)i[t]=+h.call(this,y[t],t);else for(t=0;l>t;++t)i[t]=h;if(o=[],"function"==typeof g)for(t=0;c>t;++t)o[t]=+g.call(this,m[t],t);else for(t=0;c>t;++t)o[t]=g;return a.resume()},a.resume=function(){return a.alpha(.1)},a.stop=function(){return a.alpha(0)},a.drag=function(){return e||(e=Zo.behavior.drag().origin(wt).on("dragstart.force",Ou).on("drag.force",t).on("dragend.force",Yu)),arguments.length?(this.on("mouseover.force",Iu).on("mouseout.force",Zu).call(e),void 0):e},Zo.rebind(a,c,"on")};var rs=20,us=1,is=1/0;Zo.layout.hierarchy=function(){function n(u){var i,o=[u],a=[];for(u.depth=0;null!=(i=o.pop());)if(a.push(i),(s=e.call(n,i,i.depth))&&(c=s.length)){for(var c,s,l;--c>=0;)o.push(l=s[c]),l.parent=i,l.depth=i.depth+1;r&&(i.value=0),i.children=s}else r&&(i.value=+r.call(n,i,i.depth)||0),delete i.children;return Bu(u,function(n){var e,u;t&&(e=n.children)&&e.sort(t),r&&(u=n.parent)&&(u.value+=n.value)}),a}var t=Gu,e=Wu,r=Ju;return n.sort=function(e){return arguments.length?(t=e,n):t},n.children=function(t){return arguments.length?(e=t,n):e},n.value=function(t){return arguments.length?(r=t,n):r},n.revalue=function(t){return r&&($u(t,function(n){n.children&&(n.value=0)}),Bu(t,function(t){var e;t.children||(t.value=+r.call(n,t,t.depth)||0),(e=t.parent)&&(e.value+=t.value)})),t},n},Zo.layout.partition=function(){function n(t,e,r,u){var i=t.children;if(t.x=e,t.y=t.depth*u,t.dx=r,t.dy=u,i&&(o=i.length)){var o,a,c,s=-1;for(r=t.value?r/t.value:0;++s<o;)n(a=i[s],e,c=a.value*r,u),e+=c}}function t(n){var e=n.children,r=0;if(e&&(u=e.length))for(var u,i=-1;++i<u;)r=Math.max(r,t(e[i]));return 1+r}function e(e,i){var o=r.call(this,e,i);return n(o[0],0,u[0],u[1]/t(o[0])),o}var r=Zo.layout.hierarchy(),u=[1,1];return e.size=function(n){return arguments.length?(u=n,e):u},Xu(e,r)},Zo.layout.pie=function(){function n(i){var o=i.map(function(e,r){return+t.call(n,e,r)}),a=+("function"==typeof r?r.apply(this,arguments):r),c=(("function"==typeof u?u.apply(this,arguments):u)-a)/Zo.sum(o),s=Zo.range(i.length);null!=e&&s.sort(e===os?function(n,t){return o[t]-o[n]}:function(n,t){return e(i[n],i[t])});var l=[];return s.forEach(function(n){var t;l[n]={data:i[n],value:t=o[n],startAngle:a,endAngle:a+=t*c}}),l}var t=Number,e=os,r=0,u=wa;return n.value=function(e){return arguments.length?(t=e,n):t},n.sort=function(t){return arguments.length?(e=t,n):e},n.startAngle=function(t){return arguments.length?(r=t,n):r},n.endAngle=function(t){return arguments.length?(u=t,n):u},n};var os={};Zo.layout.stack=function(){function n(a,c){var s=a.map(function(e,r){return t.call(n,e,r)}),l=s.map(function(t){return t.map(function(t,e){return[i.call(n,t,e),o.call(n,t,e)]})}),f=e.call(n,l,c);s=Zo.permute(s,f),l=Zo.permute(l,f);var h,g,p,v=r.call(n,l,c),d=s.length,m=s[0].length;for(g=0;m>g;++g)for(u.call(n,s[0][g],p=v[g],l[0][g][1]),h=1;d>h;++h)u.call(n,s[h][g],p+=l[h-1][g][1],l[h][g][1]);return a}var t=wt,e=ei,r=ri,u=ti,i=Qu,o=ni;return n.values=function(e){return arguments.length?(t=e,n):t},n.order=function(t){return arguments.length?(e="function"==typeof t?t:as.get(t)||ei,n):e},n.offset=function(t){return arguments.length?(r="function"==typeof t?t:cs.get(t)||ri,n):r},n.x=function(t){return arguments.length?(i=t,n):i},n.y=function(t){return arguments.length?(o=t,n):o},n.out=function(t){return arguments.length?(u=t,n):u},n};var as=Zo.map({"inside-out":function(n){var t,e,r=n.length,u=n.map(ui),i=n.map(ii),o=Zo.range(r).sort(function(n,t){return u[n]-u[t]}),a=0,c=0,s=[],l=[];for(t=0;r>t;++t)e=o[t],c>a?(a+=i[e],s.push(e)):(c+=i[e],l.push(e));return l.reverse().concat(s)},reverse:function(n){return Zo.range(n.length).reverse()},"default":ei}),cs=Zo.map({silhouette:function(n){var t,e,r,u=n.length,i=n[0].length,o=[],a=0,c=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];r>a&&(a=r),o.push(r)}for(e=0;i>e;++e)c[e]=(a-o[e])/2;return c},wiggle:function(n){var t,e,r,u,i,o,a,c,s,l=n.length,f=n[0],h=f.length,g=[];for(g[0]=c=s=0,e=1;h>e;++e){for(t=0,u=0;l>t;++t)u+=n[t][e][1];for(t=0,i=0,a=f[e][0]-f[e-1][0];l>t;++t){for(r=0,o=(n[t][e][1]-n[t][e-1][1])/(2*a);t>r;++r)o+=(n[r][e][1]-n[r][e-1][1])/a;i+=o*n[t][e][1]}g[e]=c-=u?i/u*a:0,s>c&&(s=c)}for(e=0;h>e;++e)g[e]-=s;return g},expand:function(n){var t,e,r,u=n.length,i=n[0].length,o=1/u,a=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];if(r)for(t=0;u>t;t++)n[t][e][1]/=r;else for(t=0;u>t;t++)n[t][e][1]=o}for(e=0;i>e;++e)a[e]=0;return a},zero:ri});Zo.layout.histogram=function(){function n(n,i){for(var o,a,c=[],s=n.map(e,this),l=r.call(this,s,i),f=u.call(this,l,s,i),i=-1,h=s.length,g=f.length-1,p=t?1:1/h;++i<g;)o=c[i]=[],o.dx=f[i+1]-(o.x=f[i]),o.y=0;if(g>0)for(i=-1;++i<h;)a=s[i],a>=l[0]&&a<=l[1]&&(o=c[Zo.bisect(f,a,1,g)-1],o.y+=p,o.push(n[i]));return c}var t=!0,e=Number,r=si,u=ai;return n.value=function(t){return arguments.length?(e=t,n):e},n.range=function(t){return arguments.length?(r=bt(t),n):r},n.bins=function(t){return arguments.length?(u="number"==typeof t?function(n){return ci(n,t)}:bt(t),n):u},n.frequency=function(e){return arguments.length?(t=!!e,n):t},n},Zo.layout.pack=function(){function n(n,i){var o=e.call(this,n,i),a=o[0],c=u[0],s=u[1],l=null==t?Math.sqrt:"function"==typeof t?t:function(){return t};if(a.x=a.y=0,Bu(a,function(n){n.r=+l(n.value)}),Bu(a,pi),r){var f=r*(t?1:Math.max(2*a.r/c,2*a.r/s))/2;Bu(a,function(n){n.r+=f}),Bu(a,pi),Bu(a,function(n){n.r-=f})}return mi(a,c/2,s/2,t?1:1/Math.max(2*a.r/c,2*a.r/s)),o}var t,e=Zo.layout.hierarchy().sort(li),r=0,u=[1,1];return n.size=function(t){return arguments.length?(u=t,n):u},n.radius=function(e){return arguments.length?(t=null==e||"function"==typeof e?e:+e,n):t},n.padding=function(t){return arguments.length?(r=+t,n):r},Xu(n,e)},Zo.layout.tree=function(){function n(n,u){var l=o.call(this,n,u),f=l[0],h=t(f);if(Bu(h,e),h.parent.m=-h.z,$u(h,r),s)$u(f,i);else{var g=f,p=f,v=f;$u(f,function(n){n.x<g.x&&(g=n),n.x>p.x&&(p=n),n.depth>v.depth&&(v=n)});var d=a(g,p)/2-g.x,m=c[0]/(p.x+a(p,g)/2+d),y=c[1]/(v.depth||1);$u(f,function(n){n.x=(n.x+d)*m,n.y=n.depth*y})}return l}function t(n){for(var t,e={A:null,children:[n]},r=[e];null!=(t=r.pop());)for(var u,i=t.children,o=0,a=i.length;a>o;++o)r.push((i[o]=u={_:i[o],parent:t,children:(u=i[o].children)&&u.slice()||[],A:null,a:null,z:0,m:0,c:0,s:0,t:null,i:o}).a=u);return e.children[0]}function e(n){var t=n.children,e=n.parent.children,r=n.i?e[n.i-1]:null;if(t.length){wi(n);var i=(t[0].z+t[t.length-1].z)/2;r?(n.z=r.z+a(n._,r._),n.m=n.z-i):n.z=i}else r&&(n.z=r.z+a(n._,r._));n.parent.A=u(n,r,n.parent.A||e[0])}function r(n){n._.x=n.z+n.parent.m,n.m+=n.parent.m}function u(n,t,e){if(t){for(var r,u=n,i=n,o=t,c=u.parent.children[0],s=u.m,l=i.m,f=o.m,h=c.m;o=_i(o),u=Mi(u),o&&u;)c=Mi(c),i=_i(i),i.a=n,r=o.z+f-u.z-s+a(o._,u._),r>0&&(bi(Si(o,n,e),n,r),s+=r,l+=r),f+=o.m,s+=u.m,h+=c.m,l+=i.m;o&&!_i(i)&&(i.t=o,i.m+=f-l),u&&!Mi(c)&&(c.t=u,c.m+=s-h,e=n)}return e}function i(n){n.x*=c[0],n.y=n.depth*c[1]}var o=Zo.layout.hierarchy().sort(null).value(null),a=xi,c=[1,1],s=null;return n.separation=function(t){return arguments.length?(a=t,n):a},n.size=function(t){return arguments.length?(s=null==(c=t)?i:null,n):s?null:c},n.nodeSize=function(t){return arguments.length?(s=null==(c=t)?null:i,n):s?c:null},Xu(n,o)},Zo.layout.cluster=function(){function n(n,i){var o,a=t.call(this,n,i),c=a[0],s=0;Bu(c,function(n){var t=n.children;t&&t.length?(n.x=Ei(t),n.y=ki(t)):(n.x=o?s+=e(n,o):0,n.y=0,o=n)});var l=Ai(c),f=Ci(c),h=l.x-e(l,f)/2,g=f.x+e(f,l)/2;return Bu(c,u?function(n){n.x=(n.x-c.x)*r[0],n.y=(c.y-n.y)*r[1]}:function(n){n.x=(n.x-h)/(g-h)*r[0],n.y=(1-(c.y?n.y/c.y:1))*r[1]}),a}var t=Zo.layout.hierarchy().sort(null).value(null),e=xi,r=[1,1],u=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(u=null==(r=t),n):u?null:r},n.nodeSize=function(t){return arguments.length?(u=null!=(r=t),n):u?r:null},Xu(n,t)},Zo.layout.treemap=function(){function n(n,t){for(var e,r,u=-1,i=n.length;++u<i;)r=(e=n[u]).value*(0>t?0:t),e.area=isNaN(r)||0>=r?0:r}function t(e){var i=e.children;if(i&&i.length){var o,a,c,s=f(e),l=[],h=i.slice(),p=1/0,v="slice"===g?s.dx:"dice"===g?s.dy:"slice-dice"===g?1&e.depth?s.dy:s.dx:Math.min(s.dx,s.dy);for(n(h,s.dx*s.dy/e.value),l.area=0;(c=h.length)>0;)l.push(o=h[c-1]),l.area+=o.area,"squarify"!==g||(a=r(l,v))<=p?(h.pop(),p=a):(l.area-=l.pop().area,u(l,v,s,!1),v=Math.min(s.dx,s.dy),l.length=l.area=0,p=1/0);l.length&&(u(l,v,s,!0),l.length=l.area=0),i.forEach(t)}}function e(t){var r=t.children;if(r&&r.length){var i,o=f(t),a=r.slice(),c=[];for(n(a,o.dx*o.dy/t.value),c.area=0;i=a.pop();)c.push(i),c.area+=i.area,null!=i.z&&(u(c,i.z?o.dx:o.dy,o,!a.length),c.length=c.area=0);r.forEach(e)}}function r(n,t){for(var e,r=n.area,u=0,i=1/0,o=-1,a=n.length;++o<a;)(e=n[o].area)&&(i>e&&(i=e),e>u&&(u=e));return r*=r,t*=t,r?Math.max(t*u*p/r,r/(t*i*p)):1/0}function u(n,t,e,r){var u,i=-1,o=n.length,a=e.x,s=e.y,l=t?c(n.area/t):0;if(t==e.dx){for((r||l>e.dy)&&(l=e.dy);++i<o;)u=n[i],u.x=a,u.y=s,u.dy=l,a+=u.dx=Math.min(e.x+e.dx-a,l?c(u.area/l):0);u.z=!0,u.dx+=e.x+e.dx-a,e.y+=l,e.dy-=l}else{for((r||l>e.dx)&&(l=e.dx);++i<o;)u=n[i],u.x=a,u.y=s,u.dx=l,s+=u.dy=Math.min(e.y+e.dy-s,l?c(u.area/l):0);u.z=!1,u.dy+=e.y+e.dy-s,e.x+=l,e.dx-=l}}function i(r){var u=o||a(r),i=u[0];return i.x=0,i.y=0,i.dx=s[0],i.dy=s[1],o&&a.revalue(i),n([i],i.dx*i.dy/i.value),(o?e:t)(i),h&&(o=u),u}var o,a=Zo.layout.hierarchy(),c=Math.round,s=[1,1],l=null,f=Ni,h=!1,g="squarify",p=.5*(1+Math.sqrt(5));return i.size=function(n){return arguments.length?(s=n,i):s},i.padding=function(n){function t(t){var e=n.call(i,t,t.depth);return null==e?Ni(t):zi(t,"number"==typeof e?[e,e,e,e]:e)}function e(t){return zi(t,n)}if(!arguments.length)return l;var r;return f=null==(l=n)?Ni:"function"==(r=typeof n)?t:"number"===r?(n=[n,n,n,n],e):e,i},i.round=function(n){return arguments.length?(c=n?Math.round:Number,i):c!=Number},i.sticky=function(n){return arguments.length?(h=n,o=null,i):h},i.ratio=function(n){return arguments.length?(p=n,i):p},i.mode=function(n){return arguments.length?(g=n+"",i):g},Xu(i,a)},Zo.random={normal:function(n,t){var e=arguments.length;return 2>e&&(t=1),1>e&&(n=0),function(){var e,r,u;do e=2*Math.random()-1,r=2*Math.random()-1,u=e*e+r*r;while(!u||u>1);return n+t*e*Math.sqrt(-2*Math.log(u)/u)}},logNormal:function(){var n=Zo.random.normal.apply(Zo,arguments);return function(){return Math.exp(n())}},bates:function(n){var t=Zo.random.irwinHall(n);return function(){return t()/n}},irwinHall:function(n){return function(){for(var t=0,e=0;n>e;e++)t+=Math.random();return t}}},Zo.scale={};var ss={floor:wt,ceil:wt};Zo.scale.linear=function(){return Ui([0,1],[0,1],hu,!1)};var ls={s:1,g:1,p:1,r:1,e:1};Zo.scale.log=function(){return Vi(Zo.scale.linear().domain([0,1]),10,!0,[1,10])};var fs=Zo.format(".0e"),hs={floor:function(n){return-Math.ceil(-n)},ceil:function(n){return-Math.floor(-n)}};Zo.scale.pow=function(){return Xi(Zo.scale.linear(),1,[0,1])},Zo.scale.sqrt=function(){return Zo.scale.pow().exponent(.5)},Zo.scale.ordinal=function(){return Bi([],{t:"range",a:[[]]})},Zo.scale.category10=function(){return Zo.scale.ordinal().range(gs)},Zo.scale.category20=function(){return Zo.scale.ordinal().range(ps)},Zo.scale.category20b=function(){return Zo.scale.ordinal().range(vs)},Zo.scale.category20c=function(){return Zo.scale.ordinal().range(ds)};var gs=[2062260,16744206,2924588,14034728,9725885,9197131,14907330,8355711,12369186,1556175].map(vt),ps=[2062260,11454440,16744206,16759672,2924588,10018698,14034728,16750742,9725885,12955861,9197131,12885140,14907330,16234194,8355711,13092807,12369186,14408589,1556175,10410725].map(vt),vs=[3750777,5395619,7040719,10264286,6519097,9216594,11915115,13556636,9202993,12426809,15186514,15190932,8666169,11356490,14049643,15177372,8077683,10834324,13528509,14589654].map(vt),ds=[3244733,7057110,10406625,13032431,15095053,16616764,16625259,16634018,3253076,7652470,10607003,13101504,7695281,10394312,12369372,14342891,6513507,9868950,12434877,14277081].map(vt);Zo.scale.quantile=function(){return Wi([],[])},Zo.scale.quantize=function(){return Ji(0,1,[0,1])},Zo.scale.threshold=function(){return Gi([.5],[0,1])},Zo.scale.identity=function(){return Ki([0,1])},Zo.svg={},Zo.svg.arc=function(){function n(){var n=t.apply(this,arguments),i=e.apply(this,arguments),o=r.apply(this,arguments)+ms,a=u.apply(this,arguments)+ms,c=(o>a&&(c=o,o=a,a=c),a-o),s=ba>c?"0":"1",l=Math.cos(o),f=Math.sin(o),h=Math.cos(a),g=Math.sin(a);
+return c>=ys?n?"M0,"+i+"A"+i+","+i+" 0 1,1 0,"+-i+"A"+i+","+i+" 0 1,1 0,"+i+"M0,"+n+"A"+n+","+n+" 0 1,0 0,"+-n+"A"+n+","+n+" 0 1,0 0,"+n+"Z":"M0,"+i+"A"+i+","+i+" 0 1,1 0,"+-i+"A"+i+","+i+" 0 1,1 0,"+i+"Z":n?"M"+i*l+","+i*f+"A"+i+","+i+" 0 "+s+",1 "+i*h+","+i*g+"L"+n*h+","+n*g+"A"+n+","+n+" 0 "+s+",0 "+n*l+","+n*f+"Z":"M"+i*l+","+i*f+"A"+i+","+i+" 0 "+s+",1 "+i*h+","+i*g+"L0,0"+"Z"}var t=Qi,e=no,r=to,u=eo;return n.innerRadius=function(e){return arguments.length?(t=bt(e),n):t},n.outerRadius=function(t){return arguments.length?(e=bt(t),n):e},n.startAngle=function(t){return arguments.length?(r=bt(t),n):r},n.endAngle=function(t){return arguments.length?(u=bt(t),n):u},n.centroid=function(){var n=(t.apply(this,arguments)+e.apply(this,arguments))/2,i=(r.apply(this,arguments)+u.apply(this,arguments))/2+ms;return[Math.cos(i)*n,Math.sin(i)*n]},n};var ms=-Sa,ys=wa-ka;Zo.svg.line=function(){return ro(wt)};var xs=Zo.map({linear:uo,"linear-closed":io,step:oo,"step-before":ao,"step-after":co,basis:po,"basis-open":vo,"basis-closed":mo,bundle:yo,cardinal:fo,"cardinal-open":so,"cardinal-closed":lo,monotone:So});xs.forEach(function(n,t){t.key=n,t.closed=/-closed$/.test(n)});var Ms=[0,2/3,1/3,0],_s=[0,1/3,2/3,0],bs=[0,1/6,2/3,1/6];Zo.svg.line.radial=function(){var n=ro(ko);return n.radius=n.x,delete n.x,n.angle=n.y,delete n.y,n},ao.reverse=co,co.reverse=ao,Zo.svg.area=function(){return Eo(wt)},Zo.svg.area.radial=function(){var n=Eo(ko);return n.radius=n.x,delete n.x,n.innerRadius=n.x0,delete n.x0,n.outerRadius=n.x1,delete n.x1,n.angle=n.y,delete n.y,n.startAngle=n.y0,delete n.y0,n.endAngle=n.y1,delete n.y1,n},Zo.svg.chord=function(){function n(n,a){var c=t(this,i,n,a),s=t(this,o,n,a);return"M"+c.p0+r(c.r,c.p1,c.a1-c.a0)+(e(c,s)?u(c.r,c.p1,c.r,c.p0):u(c.r,c.p1,s.r,s.p0)+r(s.r,s.p1,s.a1-s.a0)+u(s.r,s.p1,c.r,c.p0))+"Z"}function t(n,t,e,r){var u=t.call(n,e,r),i=a.call(n,u,r),o=c.call(n,u,r)+ms,l=s.call(n,u,r)+ms;return{r:i,a0:o,a1:l,p0:[i*Math.cos(o),i*Math.sin(o)],p1:[i*Math.cos(l),i*Math.sin(l)]}}function e(n,t){return n.a0==t.a0&&n.a1==t.a1}function r(n,t,e){return"A"+n+","+n+" 0 "+ +(e>ba)+",1 "+t}function u(n,t,e,r){return"Q 0,0 "+r}var i=gr,o=pr,a=Ao,c=to,s=eo;return n.radius=function(t){return arguments.length?(a=bt(t),n):a},n.source=function(t){return arguments.length?(i=bt(t),n):i},n.target=function(t){return arguments.length?(o=bt(t),n):o},n.startAngle=function(t){return arguments.length?(c=bt(t),n):c},n.endAngle=function(t){return arguments.length?(s=bt(t),n):s},n},Zo.svg.diagonal=function(){function n(n,u){var i=t.call(this,n,u),o=e.call(this,n,u),a=(i.y+o.y)/2,c=[i,{x:i.x,y:a},{x:o.x,y:a},o];return c=c.map(r),"M"+c[0]+"C"+c[1]+" "+c[2]+" "+c[3]}var t=gr,e=pr,r=Co;return n.source=function(e){return arguments.length?(t=bt(e),n):t},n.target=function(t){return arguments.length?(e=bt(t),n):e},n.projection=function(t){return arguments.length?(r=t,n):r},n},Zo.svg.diagonal.radial=function(){var n=Zo.svg.diagonal(),t=Co,e=n.projection;return n.projection=function(n){return arguments.length?e(No(t=n)):t},n},Zo.svg.symbol=function(){function n(n,r){return(ws.get(t.call(this,n,r))||To)(e.call(this,n,r))}var t=Lo,e=zo;return n.type=function(e){return arguments.length?(t=bt(e),n):t},n.size=function(t){return arguments.length?(e=bt(t),n):e},n};var ws=Zo.map({circle:To,cross:function(n){var t=Math.sqrt(n/5)/2;return"M"+-3*t+","+-t+"H"+-t+"V"+-3*t+"H"+t+"V"+-t+"H"+3*t+"V"+t+"H"+t+"V"+3*t+"H"+-t+"V"+t+"H"+-3*t+"Z"},diamond:function(n){var t=Math.sqrt(n/(2*As)),e=t*As;return"M0,"+-t+"L"+e+",0"+" 0,"+t+" "+-e+",0"+"Z"},square:function(n){var t=Math.sqrt(n)/2;return"M"+-t+","+-t+"L"+t+","+-t+" "+t+","+t+" "+-t+","+t+"Z"},"triangle-down":function(n){var t=Math.sqrt(n/Es),e=t*Es/2;return"M0,"+e+"L"+t+","+-e+" "+-t+","+-e+"Z"},"triangle-up":function(n){var t=Math.sqrt(n/Es),e=t*Es/2;return"M0,"+-e+"L"+t+","+e+" "+-t+","+e+"Z"}});Zo.svg.symbolTypes=ws.keys();var Ss,ks,Es=Math.sqrt(3),As=Math.tan(30*Aa),Cs=[],Ns=0;Cs.call=pa.call,Cs.empty=pa.empty,Cs.node=pa.node,Cs.size=pa.size,Zo.transition=function(n){return arguments.length?Ss?n.transition():n:ma.transition()},Zo.transition.prototype=Cs,Cs.select=function(n){var t,e,r,u=this.id,i=[];n=b(n);for(var o=-1,a=this.length;++o<a;){i.push(t=[]);for(var c=this[o],s=-1,l=c.length;++s<l;)(r=c[s])&&(e=n.call(r,r.__data__,s,o))?("__data__"in r&&(e.__data__=r.__data__),Po(e,s,u,r.__transition__[u]),t.push(e)):t.push(null)}return qo(i,u)},Cs.selectAll=function(n){var t,e,r,u,i,o=this.id,a=[];n=w(n);for(var c=-1,s=this.length;++c<s;)for(var l=this[c],f=-1,h=l.length;++f<h;)if(r=l[f]){i=r.__transition__[o],e=n.call(r,r.__data__,f,c),a.push(t=[]);for(var g=-1,p=e.length;++g<p;)(u=e[g])&&Po(u,g,o,i),t.push(u)}return qo(a,o)},Cs.filter=function(n){var t,e,r,u=[];"function"!=typeof n&&(n=R(n));for(var i=0,o=this.length;o>i;i++){u.push(t=[]);for(var e=this[i],a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a,i)&&t.push(r)}return qo(u,this.id)},Cs.tween=function(n,t){var e=this.id;return arguments.length<2?this.node().__transition__[e].tween.get(n):P(this,null==t?function(t){t.__transition__[e].tween.remove(n)}:function(r){r.__transition__[e].tween.set(n,t)})},Cs.attr=function(n,t){function e(){this.removeAttribute(a)}function r(){this.removeAttributeNS(a.space,a.local)}function u(n){return null==n?e:(n+="",function(){var t,e=this.getAttribute(a);return e!==n&&(t=o(e,n),function(n){this.setAttribute(a,t(n))})})}function i(n){return null==n?r:(n+="",function(){var t,e=this.getAttributeNS(a.space,a.local);return e!==n&&(t=o(e,n),function(n){this.setAttributeNS(a.space,a.local,t(n))})})}if(arguments.length<2){for(t in n)this.attr(t,n[t]);return this}var o="transform"==n?Du:hu,a=Zo.ns.qualify(n);return Ro(this,"attr."+n,t,a.local?i:u)},Cs.attrTween=function(n,t){function e(n,e){var r=t.call(this,n,e,this.getAttribute(u));return r&&function(n){this.setAttribute(u,r(n))}}function r(n,e){var r=t.call(this,n,e,this.getAttributeNS(u.space,u.local));return r&&function(n){this.setAttributeNS(u.space,u.local,r(n))}}var u=Zo.ns.qualify(n);return this.tween("attr."+n,u.local?r:e)},Cs.style=function(n,t,e){function r(){this.style.removeProperty(n)}function u(t){return null==t?r:(t+="",function(){var r,u=Wo.getComputedStyle(this,null).getPropertyValue(n);return u!==t&&(r=hu(u,t),function(t){this.style.setProperty(n,r(t),e)})})}var i=arguments.length;if(3>i){if("string"!=typeof n){2>i&&(t="");for(e in n)this.style(e,n[e],t);return this}e=""}return Ro(this,"style."+n,t,u)},Cs.styleTween=function(n,t,e){function r(r,u){var i=t.call(this,r,u,Wo.getComputedStyle(this,null).getPropertyValue(n));return i&&function(t){this.style.setProperty(n,i(t),e)}}return arguments.length<3&&(e=""),this.tween("style."+n,r)},Cs.text=function(n){return Ro(this,"text",n,Do)},Cs.remove=function(){return this.each("end.transition",function(){var n;this.__transition__.count<2&&(n=this.parentNode)&&n.removeChild(this)})},Cs.ease=function(n){var t=this.id;return arguments.length<1?this.node().__transition__[t].ease:("function"!=typeof n&&(n=Zo.ease.apply(Zo,arguments)),P(this,function(e){e.__transition__[t].ease=n}))},Cs.delay=function(n){var t=this.id;return arguments.length<1?this.node().__transition__[t].delay:P(this,"function"==typeof n?function(e,r,u){e.__transition__[t].delay=+n.call(e,e.__data__,r,u)}:(n=+n,function(e){e.__transition__[t].delay=n}))},Cs.duration=function(n){var t=this.id;return arguments.length<1?this.node().__transition__[t].duration:P(this,"function"==typeof n?function(e,r,u){e.__transition__[t].duration=Math.max(1,n.call(e,e.__data__,r,u))}:(n=Math.max(1,n),function(e){e.__transition__[t].duration=n}))},Cs.each=function(n,t){var e=this.id;if(arguments.length<2){var r=ks,u=Ss;Ss=e,P(this,function(t,r,u){ks=t.__transition__[e],n.call(t,t.__data__,r,u)}),ks=r,Ss=u}else P(this,function(r){var u=r.__transition__[e];(u.event||(u.event=Zo.dispatch("start","end"))).on(n,t)});return this},Cs.transition=function(){for(var n,t,e,r,u=this.id,i=++Ns,o=[],a=0,c=this.length;c>a;a++){o.push(n=[]);for(var t=this[a],s=0,l=t.length;l>s;s++)(e=t[s])&&(r=Object.create(e.__transition__[u]),r.delay+=r.duration,Po(e,s,i,r)),n.push(e)}return qo(o,i)},Zo.svg.axis=function(){function n(n){n.each(function(){var n,s=Zo.select(this),l=this.__chart__||e,f=this.__chart__=e.copy(),h=null==c?f.ticks?f.ticks.apply(f,a):f.domain():c,g=null==t?f.tickFormat?f.tickFormat.apply(f,a):wt:t,p=s.selectAll(".tick").data(h,f),v=p.enter().insert("g",".domain").attr("class","tick").style("opacity",ka),d=Zo.transition(p.exit()).style("opacity",ka).remove(),m=Zo.transition(p.order()).style("opacity",1),y=Ti(f),x=s.selectAll(".domain").data([0]),M=(x.enter().append("path").attr("class","domain"),Zo.transition(x));v.append("line"),v.append("text");var _=v.select("line"),b=m.select("line"),w=p.select("text").text(g),S=v.select("text"),k=m.select("text");switch(r){case"bottom":n=Uo,_.attr("y2",u),S.attr("y",Math.max(u,0)+o),b.attr("x2",0).attr("y2",u),k.attr("x",0).attr("y",Math.max(u,0)+o),w.attr("dy",".71em").style("text-anchor","middle"),M.attr("d","M"+y[0]+","+i+"V0H"+y[1]+"V"+i);break;case"top":n=Uo,_.attr("y2",-u),S.attr("y",-(Math.max(u,0)+o)),b.attr("x2",0).attr("y2",-u),k.attr("x",0).attr("y",-(Math.max(u,0)+o)),w.attr("dy","0em").style("text-anchor","middle"),M.attr("d","M"+y[0]+","+-i+"V0H"+y[1]+"V"+-i);break;case"left":n=jo,_.attr("x2",-u),S.attr("x",-(Math.max(u,0)+o)),b.attr("x2",-u).attr("y2",0),k.attr("x",-(Math.max(u,0)+o)).attr("y",0),w.attr("dy",".32em").style("text-anchor","end"),M.attr("d","M"+-i+","+y[0]+"H0V"+y[1]+"H"+-i);break;case"right":n=jo,_.attr("x2",u),S.attr("x",Math.max(u,0)+o),b.attr("x2",u).attr("y2",0),k.attr("x",Math.max(u,0)+o).attr("y",0),w.attr("dy",".32em").style("text-anchor","start"),M.attr("d","M"+i+","+y[0]+"H0V"+y[1]+"H"+i)}if(f.rangeBand){var E=f,A=E.rangeBand()/2;l=f=function(n){return E(n)+A}}else l.rangeBand?l=f:d.call(n,f);v.call(n,l),m.call(n,f)})}var t,e=Zo.scale.linear(),r=zs,u=6,i=6,o=3,a=[10],c=null;return n.scale=function(t){return arguments.length?(e=t,n):e},n.orient=function(t){return arguments.length?(r=t in Ls?t+"":zs,n):r},n.ticks=function(){return arguments.length?(a=arguments,n):a},n.tickValues=function(t){return arguments.length?(c=t,n):c},n.tickFormat=function(e){return arguments.length?(t=e,n):t},n.tickSize=function(t){var e=arguments.length;return e?(u=+t,i=+arguments[e-1],n):u},n.innerTickSize=function(t){return arguments.length?(u=+t,n):u},n.outerTickSize=function(t){return arguments.length?(i=+t,n):i},n.tickPadding=function(t){return arguments.length?(o=+t,n):o},n.tickSubdivide=function(){return arguments.length&&n},n};var zs="bottom",Ls={top:1,right:1,bottom:1,left:1};Zo.svg.brush=function(){function n(i){i.each(function(){var i=Zo.select(this).style("pointer-events","all").style("-webkit-tap-highlight-color","rgba(0,0,0,0)").on("mousedown.brush",u).on("touchstart.brush",u),o=i.selectAll(".background").data([0]);o.enter().append("rect").attr("class","background").style("visibility","hidden").style("cursor","crosshair"),i.selectAll(".extent").data([0]).enter().append("rect").attr("class","extent").style("cursor","move");var a=i.selectAll(".resize").data(p,wt);a.exit().remove(),a.enter().append("g").attr("class",function(n){return"resize "+n}).style("cursor",function(n){return Ts[n]}).append("rect").attr("x",function(n){return/[ew]$/.test(n)?-3:null}).attr("y",function(n){return/^[ns]/.test(n)?-3:null}).attr("width",6).attr("height",6).style("visibility","hidden"),a.style("display",n.empty()?"none":null);var l,f=Zo.transition(i),h=Zo.transition(o);c&&(l=Ti(c),h.attr("x",l[0]).attr("width",l[1]-l[0]),e(f)),s&&(l=Ti(s),h.attr("y",l[0]).attr("height",l[1]-l[0]),r(f)),t(f)})}function t(n){n.selectAll(".resize").attr("transform",function(n){return"translate("+l[+/e$/.test(n)]+","+f[+/^s/.test(n)]+")"})}function e(n){n.select(".extent").attr("x",l[0]),n.selectAll(".extent,.n>rect,.s>rect").attr("width",l[1]-l[0])}function r(n){n.select(".extent").attr("y",f[0]),n.selectAll(".extent,.e>rect,.w>rect").attr("height",f[1]-f[0])}function u(){function u(){32==Zo.event.keyCode&&(C||(x=null,z[0]-=l[1],z[1]-=f[1],C=2),y())}function p(){32==Zo.event.keyCode&&2==C&&(z[0]+=l[1],z[1]+=f[1],C=0,y())}function v(){var n=Zo.mouse(_),u=!1;M&&(n[0]+=M[0],n[1]+=M[1]),C||(Zo.event.altKey?(x||(x=[(l[0]+l[1])/2,(f[0]+f[1])/2]),z[0]=l[+(n[0]<x[0])],z[1]=f[+(n[1]<x[1])]):x=null),E&&d(n,c,0)&&(e(S),u=!0),A&&d(n,s,1)&&(r(S),u=!0),u&&(t(S),w({type:"brush",mode:C?"move":"resize"}))}function d(n,t,e){var r,u,a=Ti(t),c=a[0],s=a[1],p=z[e],v=e?f:l,d=v[1]-v[0];return C&&(c-=p,s-=d+p),r=(e?g:h)?Math.max(c,Math.min(s,n[e])):n[e],C?u=(r+=p)+d:(x&&(p=Math.max(c,Math.min(s,2*x[e]-r))),r>p?(u=r,r=p):u=p),v[0]!=r||v[1]!=u?(e?o=null:i=null,v[0]=r,v[1]=u,!0):void 0}function m(){v(),S.style("pointer-events","all").selectAll(".resize").style("display",n.empty()?"none":null),Zo.select("body").style("cursor",null),L.on("mousemove.brush",null).on("mouseup.brush",null).on("touchmove.brush",null).on("touchend.brush",null).on("keydown.brush",null).on("keyup.brush",null),N(),w({type:"brushend"})}var x,M,_=this,b=Zo.select(Zo.event.target),w=a.of(_,arguments),S=Zo.select(_),k=b.datum(),E=!/^(n|s)$/.test(k)&&c,A=!/^(e|w)$/.test(k)&&s,C=b.classed("extent"),N=I(),z=Zo.mouse(_),L=Zo.select(Wo).on("keydown.brush",u).on("keyup.brush",p);if(Zo.event.changedTouches?L.on("touchmove.brush",v).on("touchend.brush",m):L.on("mousemove.brush",v).on("mouseup.brush",m),S.interrupt().selectAll("*").interrupt(),C)z[0]=l[0]-z[0],z[1]=f[0]-z[1];else if(k){var T=+/w$/.test(k),q=+/^n/.test(k);M=[l[1-T]-z[0],f[1-q]-z[1]],z[0]=l[T],z[1]=f[q]}else Zo.event.altKey&&(x=z.slice());S.style("pointer-events","none").selectAll(".resize").style("display",null),Zo.select("body").style("cursor",b.style("cursor")),w({type:"brushstart"}),v()}var i,o,a=M(n,"brushstart","brush","brushend"),c=null,s=null,l=[0,0],f=[0,0],h=!0,g=!0,p=qs[0];return n.event=function(n){n.each(function(){var n=a.of(this,arguments),t={x:l,y:f,i:i,j:o},e=this.__chart__||t;this.__chart__=t,Ss?Zo.select(this).transition().each("start.brush",function(){i=e.i,o=e.j,l=e.x,f=e.y,n({type:"brushstart"})}).tween("brush:brush",function(){var e=gu(l,t.x),r=gu(f,t.y);return i=o=null,function(u){l=t.x=e(u),f=t.y=r(u),n({type:"brush",mode:"resize"})}}).each("end.brush",function(){i=t.i,o=t.j,n({type:"brush",mode:"resize"}),n({type:"brushend"})}):(n({type:"brushstart"}),n({type:"brush",mode:"resize"}),n({type:"brushend"}))})},n.x=function(t){return arguments.length?(c=t,p=qs[!c<<1|!s],n):c},n.y=function(t){return arguments.length?(s=t,p=qs[!c<<1|!s],n):s},n.clamp=function(t){return arguments.length?(c&&s?(h=!!t[0],g=!!t[1]):c?h=!!t:s&&(g=!!t),n):c&&s?[h,g]:c?h:s?g:null},n.extent=function(t){var e,r,u,a,h;return arguments.length?(c&&(e=t[0],r=t[1],s&&(e=e[0],r=r[0]),i=[e,r],c.invert&&(e=c(e),r=c(r)),e>r&&(h=e,e=r,r=h),(e!=l[0]||r!=l[1])&&(l=[e,r])),s&&(u=t[0],a=t[1],c&&(u=u[1],a=a[1]),o=[u,a],s.invert&&(u=s(u),a=s(a)),u>a&&(h=u,u=a,a=h),(u!=f[0]||a!=f[1])&&(f=[u,a])),n):(c&&(i?(e=i[0],r=i[1]):(e=l[0],r=l[1],c.invert&&(e=c.invert(e),r=c.invert(r)),e>r&&(h=e,e=r,r=h))),s&&(o?(u=o[0],a=o[1]):(u=f[0],a=f[1],s.invert&&(u=s.invert(u),a=s.invert(a)),u>a&&(h=u,u=a,a=h))),c&&s?[[e,u],[r,a]]:c?[e,r]:s&&[u,a])},n.clear=function(){return n.empty()||(l=[0,0],f=[0,0],i=o=null),n},n.empty=function(){return!!c&&l[0]==l[1]||!!s&&f[0]==f[1]},Zo.rebind(n,a,"on")};var Ts={n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},qs=[["n","e","s","w","nw","ne","se","sw"],["e","w"],["n","s"],[]],Rs=Qa.format=ic.timeFormat,Ds=Rs.utc,Ps=Ds("%Y-%m-%dT%H:%M:%S.%LZ");Rs.iso=Date.prototype.toISOString&&+new Date("2000-01-01T00:00:00.000Z")?Ho:Ps,Ho.parse=function(n){var t=new Date(n);return isNaN(t)?null:t},Ho.toString=Ps.toString,Qa.second=Dt(function(n){return new nc(1e3*Math.floor(n/1e3))},function(n,t){n.setTime(n.getTime()+1e3*Math.floor(t))},function(n){return n.getSeconds()}),Qa.seconds=Qa.second.range,Qa.seconds.utc=Qa.second.utc.range,Qa.minute=Dt(function(n){return new nc(6e4*Math.floor(n/6e4))},function(n,t){n.setTime(n.getTime()+6e4*Math.floor(t))},function(n){return n.getMinutes()}),Qa.minutes=Qa.minute.range,Qa.minutes.utc=Qa.minute.utc.range,Qa.hour=Dt(function(n){var t=n.getTimezoneOffset()/60;return new nc(36e5*(Math.floor(n/36e5-t)+t))},function(n,t){n.setTime(n.getTime()+36e5*Math.floor(t))},function(n){return n.getHours()}),Qa.hours=Qa.hour.range,Qa.hours.utc=Qa.hour.utc.range,Qa.month=Dt(function(n){return n=Qa.day(n),n.setDate(1),n},function(n,t){n.setMonth(n.getMonth()+t)},function(n){return n.getMonth()}),Qa.months=Qa.month.range,Qa.months.utc=Qa.month.utc.range;var Us=[1e3,5e3,15e3,3e4,6e4,3e5,9e5,18e5,36e5,108e5,216e5,432e5,864e5,1728e5,6048e5,2592e6,7776e6,31536e6],js=[[Qa.second,1],[Qa.second,5],[Qa.second,15],[Qa.second,30],[Qa.minute,1],[Qa.minute,5],[Qa.minute,15],[Qa.minute,30],[Qa.hour,1],[Qa.hour,3],[Qa.hour,6],[Qa.hour,12],[Qa.day,1],[Qa.day,2],[Qa.week,1],[Qa.month,1],[Qa.month,3],[Qa.year,1]],Hs=Rs.multi([[".%L",function(n){return n.getMilliseconds()}],[":%S",function(n){return n.getSeconds()}],["%I:%M",function(n){return n.getMinutes()}],["%I %p",function(n){return n.getHours()}],["%a %d",function(n){return n.getDay()&&1!=n.getDate()}],["%b %d",function(n){return 1!=n.getDate()}],["%B",function(n){return n.getMonth()}],["%Y",we]]),Fs={range:function(n,t,e){return Zo.range(Math.ceil(n/e)*e,+t,e).map(Oo)},floor:wt,ceil:wt};js.year=Qa.year,Qa.scale=function(){return Fo(Zo.scale.linear(),js,Hs)};var Os=js.map(function(n){return[n[0].utc,n[1]]}),Ys=Ds.multi([[".%L",function(n){return n.getUTCMilliseconds()}],[":%S",function(n){return n.getUTCSeconds()}],["%I:%M",function(n){return n.getUTCMinutes()}],["%I %p",function(n){return n.getUTCHours()}],["%a %d",function(n){return n.getUTCDay()&&1!=n.getUTCDate()}],["%b %d",function(n){return 1!=n.getUTCDate()}],["%B",function(n){return n.getUTCMonth()}],["%Y",we]]);Os.year=Qa.year.utc,Qa.scale.utc=function(){return Fo(Zo.scale.linear(),Os,Ys)},Zo.text=St(function(n){return n.responseText}),Zo.json=function(n,t){return kt(n,"application/json",Yo,t)},Zo.html=function(n,t){return kt(n,"text/html",Io,t)},Zo.xml=St(function(n){return n.responseXML}),"function"==typeof define&&define.amd?define(Zo):"object"==typeof module&&module.exports&&(module.exports=Zo),this.d3=Zo}();
+
+// parseUri 1.2.2
+// (c) Steven Levithan <stevenlevithan.com>
+// MIT License
+
+function parseUri (str) {
+	var	o   = parseUri.options,
+		m   = o.parser[o.strictMode ? "strict" : "loose"].exec(str),
+		uri = {},
+		i   = 14;
+
+	while (i--) uri[o.key[i]] = m[i] || "";
+
+	uri[o.q.name] = {};
+	uri[o.key[12]].replace(o.q.parser, function ($0, $1, $2) {
+		if ($1) uri[o.q.name][$1] = $2;
+	});
+
+	return uri;
+};
+
+parseUri.options = {
+	strictMode: false,
+	key: ["source","protocol","authority","userInfo","user","password","host","port","relative","path","directory","file","query","anchor"],
+	q:   {
+		name:   "queryKey",
+		parser: /(?:^|&)([^&=]*)=?([^&]*)/g
+	},
+	parser: {
+		strict: /^(?:([^:\/?#]+):)?(?:\/\/((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?))?((((?:[^?#\/]*\/)*)([^?#]*))(?:\?([^#]*))?(?:#(.*))?)/,
+		loose:  /^(?:(?![^:@]+:[^:@\/]*@)([^:\/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#\/]*\.[^?#\/.]+(?:[?#]|$)))*\/?)?([^?#\/]*))(?:\?([^#]*))?(?:#(.*))?)/
+	}
+};
+
+
+// graph.js
+var drawGraph = function() {
+
+    $("svg").remove();
+
+    //Dataset and metric to draw is passed via query option:
+    query = parseUri(location).queryKey;
+    query.stats = unescape(query.stats);
+    stats_db = '/tests/artifacts/' + query.stats + '/stats';
+    var metric = query.metric;
+    var operation = query.operation;
+    var smoothing = query.smoothing;
+
+    xmin = query.xmin;
+    xmax = query.xmax;
+    ymin = query.ymin;
+    ymax = query.ymax;
+
+    //Pull metrics from the stats json:
+    stress_metrics = $.extend([], stats['stats'][0]['metrics']);
+    $.each(stress_metrics, function(i,v) {
+        stress_metrics[i] = v.replace(/\W/g,"_");
+    });
+    stress_metric_names = {};
+    $.each(stress_metrics, function(i,v) {
+        stress_metric_names[v] = stats['stats'][0]['metrics'][i];
+    });
+
+    var updateURLBar = function() {
+        //Update the URL bar with the current parameters:
+        window.history.replaceState(null,null,parseUri(location).path + "?" + $.param(query));
+    };
+
+    //Check query parameters:
+    if (metric == undefined) {
+        metric = query.metric = 'SSTables';
+    }
+    if (operation == undefined) {
+        operation = query.operation = stats['stats'][0]['test'];
+    }
+    if (smoothing == undefined) {
+        smoothing = query.smoothing = 1;
+    }
+    updateURLBar();
+
+    var metric_index = stress_metrics.indexOf(metric);
+    var time_index = stress_metrics.indexOf('time');
+
+    /// Add dropdown controls to select chart criteria / options:
+    var chart_controls = $('<div id="chart_controls"/>');
+    var chart_controls_tbl = $('<table/>');
+    chart_controls.append(chart_controls_tbl);
+    $('body').append(chart_controls);
+    var metric_selector = $('<select id="metric_selector"/>');
+    $.each(stress_metric_names, function(k,v) {
+        if (k == 'time') {
+            return; //Elapsed time makes no sense to graph, skip it.
+        }
+        var option = $('<option/>').attr('value', k).text(v);
+        if (metric == k) {
+            option.attr('selected','selected');
+        }
+        metric_selector.append(option);
+
+    });
+    chart_controls_tbl.append('<tr><td><label for="metric_selector"/>Choose metric:</label></td><td id="metric_selector_td"></td></tr>')
+    $('#metric_selector_td').append(metric_selector);
+
+    var operation_selector = $('<select id="operation_selector"/>')
+    chart_controls_tbl.append('<tr><td><label for="operation_selector"/>Choose operation:</label></td><td id="operation_selector_td"></td></tr>')
+    $('#operation_selector_td').append(operation_selector);
+
+
+    var smoothing_selector = $('<select id="smoothing_selector"/>')
+    $.each([1,2,3,4,5,6,7,8], function(i, v) {
+        var option = $('<option/>').attr('value', v).text(v);
+        if (smoothing == v) {
+            option.attr('selected','selected');
+        }
+        smoothing_selector.append(option);
+    });
+    chart_controls_tbl.append('<tr><td style="width:150px"><label for="smoothing_selector"/>Data smoothing:</label></td><td id="smoothing_selector_td"></td></tr>')
+    $("#smoothing_selector_td").append(smoothing_selector);
+
+    chart_controls_tbl.append('<tr><td colspan="100%">Zoom: <a href="#" id="reset_zoom">reset</a><table id="zoom"><tr><td><label for="xmin"/>x min</label></td><td><input id="xmin"/></td><td><label for="xmax"/>x max</label></td><td><input id="xmax"/></td></tr><tr><td><label for="ymin"/>y min</label></td><td><input id="ymin"/></td><td><label for="ymax"/>y max</label></td><td><input id="ymax"/></td></tr></table></td></tr>');
+
+    chart_controls_tbl.append('<tr><td style="padding-top:10px" colspan="100%">To hide/show a dataset click on the associated colored box</td></tr>');
+
+    var raw_data;
+
+    //Callback to draw graph once we have json data.
+    var graph_callback = function() {
+        var data = [];
+        var trials = {};
+        var data_by_title = {};
+        //Keep track of what operations are availble from the test:
+        var operations = {};
+
+        raw_data.stats.forEach(function(d) {
+            // Make a copy of d so we never modify raw_data
+            d = $.extend({}, d);
+            operations[d.test] = true;
+            if (d.test!=operation) {
+                return;
+            }
+            d.title = d['label'] != undefined ? d['label'] : d['revision'];
+            data_by_title[d.title] = d;
+            data.push(d);
+            trials[d.title] = d;
+            //Clean up the intervals:
+            //Remove every other item, so as to smooth the line:
+            var new_intervals = [];
+            d.intervals.forEach(function(i, x) {
+                if (x % smoothing == 0) {
+                    new_intervals.push(i);
+                }
+            });
+            d.intervals = new_intervals;
+        });
+
+        //Fill operations available from test:
+        operation_selector.children().remove();
+        $.each(operations, function(k) {
+            var option = $('<option/>').attr('value', k).text(k);
+            if (operation == k) {
+                option.attr('selected','selected');
+            }
+            operation_selector.append(option);
+        });
+
+        var getMetricValue = function(d) {
+            if (metric_index >= 0) {
+                //This is one of the metrics directly reported by stress:
+                return d[metric_index];
+            } else {
+                //This metric is not reported by stress, so compute it ourselves:
+                if (metric == 'num_timeouts') {
+                    return d[stress_metrics.indexOf('interval_op_rate')] - d[stress_metrics.indexOf('interval_key_rate')];
+                }
+            }
+        };
+
+        //Parse the dates:
+        data.forEach(function(d) {
+            d.date = new Date(Date.parse(d.date));
+        });
+
+
+        $("svg").remove();
+        //Setup initial zoom level:
+        defaultZoom = function(initialize) {
+            if (!initialize) {
+                //Reset zoom query params:
+                query.xmin = xmin = undefined;
+                query.xmax = xmax = undefined;
+                query.ymin = ymin = undefined;
+                query.ymax = ymax = undefined;
+            }
+            query.xmin = xmin = query.xmin ? query.xmin : 0;
+            query.xmax = xmax = query.xmax ? query.xmax : Math.round(d3.max(data, function(d) {
+                if (d.intervals.length > 0) {
+                    return d.intervals[d.intervals.length-1][time_index];
+                }
+            }) * 1.1 * 100) / 100;
+            query.ymin = ymin = query.ymin ? query.ymin : Math.min(0, d3.min(data, function(d) {
+                return d3.min(d.intervals, function(i) {
+                    return getMetricValue(i);
+                });
+            }) * 1.1);
+            query.ymax = ymax = query.ymax ? query.ymax : Math.round(d3.max(data, function(d) {
+                return d3.max(d.intervals, function(i) {
+                    return getMetricValue(i);
+                });
+            }) * 1.1 * 100) / 100;
+            $("#xmin").val(xmin);
+            $("#xmax").val(xmax);
+            $("#ymin").val(ymin);
+            $("#ymax").val(ymax);
+            var updateX = function() {
+                query.xmin = xmin = $("#xmin").val();
+                query.xmax = xmax = $("#xmax").val();
+                x.domain([xmin,xmax]);
+                updateURLBar();
+            };
+            var updateY = function() {
+                query.ymin = ymin = $("#ymin").val();
+                query.ymax = ymax = $("#ymax").val();
+                y.domain([ymin, ymax]);
+                updateURLBar();
+            };
+            $("#xmin,#xmax").unbind().change(function(e) {
+                updateX();
+                redrawLines();
+            });
+            $("#ymin,#ymax").unbind().change(function(e) {
+                updateY();
+                redrawLines();
+            });
+            // The first time defaultZoom is called, we pass
+            // initialize=true, and we do not call the change() method
+            // yet. On subsequent calls, without initialize, we do.
+            if (!initialize) {
+                updateX();
+                updateY();
+                redrawLines();
+            }
+        }
+        defaultZoom(true);
+
+        $("#reset_zoom").click(function(e) {
+            defaultZoom();
+            e.preventDefault();
+        });
+
+        //Setup chart:
+        var margin = {top: 20, right: 1180, bottom: 4240, left: 60};
+        var width = 2060 - margin.left - margin.right;
+        var height = 4700 - margin.top - margin.bottom;
+
+        var x = d3.scale.linear()
+            .domain([xmin, xmax])
+            .range([0, width]);
+
+        var y = d3.scale.linear()
+            .domain([ymin, ymax])
+            .range([height, 0]);
+
+        var color = d3.scale.category10();
+        color.domain(data.map(function(d){return d.title}));
+
+        var xAxis = d3.svg.axis()
+            .scale(x)
+            .orient("bottom");
+
+        var yAxis = d3.svg.axis()
+            .scale(y)
+            .orient("left");
+
+        var line = d3.svg.line()
+            .interpolate("basis")
+            .x(function(d) {
+                return x(d[time_index]); //time in seconds
+            })
+            .y(function(d) {
+                return y(getMetricValue(d));
+            });
+
+        $("body").append("<div id='svg_container'>");
+
+        var redrawLines = function() {
+            svg.select(".x.axis").call(xAxis);
+            svg.select(".y.axis").call(yAxis);
+            svg.selectAll(".line")
+                .attr("class","line")
+                .attr("d", function(d) {
+                    return line(d.intervals);
+                })
+            $("#xmin").val(x.domain()[0]);
+            $("#xmax").val(x.domain()[1]);
+            $("#ymin").val(y.domain()[0]);
+            $("#ymax").val(y.domain()[1]);
+        }
+
+        var zoom = d3.behavior.zoom()
+            .x(x)
+            .y(y)
+            .on("zoom", redrawLines);
+
+        var svg = d3.select("div#svg_container").append("svg")
+            .attr("width", width + margin.left + margin.right + 250)
+            .attr("height", height + margin.top + margin.bottom)
+            .append("g")
+            .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
+
+        // Clip Path
+        svg.append("svg:clipPath")
+            .attr("id", "chart_clip")
+            .append("svg:rect")
+            .attr("width", width)
+            .attr("height", height);
+
+        // Chart title
+        svg.append("text")
+            .attr("x", width / 2 )
+            .attr("y", 0 )
+            .style('font-size', '2em')
+            .style("text-anchor", "middle")
+            .text(raw_data.title + ' - ' + operation);
+
+        // Chart subtitle
+        svg.append("text")
+            .attr("x", width / 2 )
+            .attr("y", 15 )
+            .style('font-size', '1.2em')
+            .style("text-anchor", "middle")
+            .text((raw_data.subtitle ? raw_data.subtitle : ''));
+
+        // x-axis - time
+        svg.append("g")
+            .attr("class", "x axis")
+            .attr("transform", "translate(0," + height + ")")
+            .call(xAxis);
+
+        // x-axis label
+        svg.append("text")
+            .attr("x", width / 2 )
+            .attr("y", height + 30 )
+            .style("text-anchor", "middle")
+            .style("font-size", "1.2em")
+            .text(stress_metric_names['time']);
+
+        // y-axis
+        svg.append("g")
+            .attr("class", "y axis")
+            .call(yAxis)
+            .append("text")
+            .attr("transform", "rotate(-90)")
+            .attr("y", -60)
+            .attr("dy", ".91em")
+            .style("font-size", "1.2em")
+            .style("text-anchor", "end")
+            .text(stress_metric_names[metric]);
+
+        var trial = svg.selectAll(".trial")
+            .data(data)
+            .enter().append("g")
+            .attr("class", "trial")
+            .attr("title", function(d) {
+                return d.title;
+            });
+
+        // Draw benchmarked data:
+        trial.append("path")
+            .attr("class", "line")
+            .attr("clip-path", "url(#chart_clip)")
+            .attr("d", function(d) {
+                return line(d.intervals);
+            })
+            .style("stroke", function(d) { return color(d.title); });
+
+        var legend = svg.selectAll(".legend")
+            .data(color.domain())
+            .enter().append("g")
+            .attr("class", "legend")
+            .attr("transform", function(d, i) {
+                var y_offset = 425 + (i*25) + 70;
+                var x_offset = -550;
+                return "translate(" + x_offset + "," + y_offset + ")";
+            });
+
+        var renderLegendText = function(linenum, getTextCallback) {
+            legend.append("text")
+                .attr("x", width - 24 - 250)
+                .attr("y", 12*linenum)
+                .attr("dy", ".35em")
+                .style("font-family", "monospace")
+                .style("font-size", "1.2em")
+                .style("text-anchor", "start")
+                .text(function(d) {
+                    return getTextCallback(d);
+                });
+        };
+
+        var padTextEnd = function(text, length) {
+            for(var x=text.length; x<length; x++) {
+                text = text + '\u00A0';
+            }
+            return text;
+        };
+        var padTextStart = function(text, length) {
+            for(var x=text.length; x<length; x++) {
+                text = '\u00A0' + text;
+            }
+            return text;
+        };
+
+        renderLegendText(1, function(title) {
+            return padTextStart(title, title.length + 5);
+        });
+
+        legend.append("rect")
+            .attr("x", width - 270)
+            .attr("width", 18)
+            .attr("height", 18)
+            .attr("class", "legend-rect")
+            .attr("title", function(title) {
+                return title;
+            })
+            .style("fill", color);
+
+        //Make trials hideable by double clicking on the colored legend box
+        $("rect.legend-rect").click(function() {
+            $("g.trial[title='" + $(this).attr('title') + "']").toggle();
+        });
+
+
+        // Chart control callbacks:
+        metric_selector.unbind().change(function(e) {
+            // change the metric in the url to reload the page:
+            metric = query.metric = this.value;
+            metric_index = stress_metrics.indexOf(metric);
+            graph_callback();
+            defaultZoom();
+        });
+        operation_selector.unbind().change(function(e) {
+            // change the metric in the url to reload the page:
+            operation = query.operation = this.value;
+            graph_callback();
+            defaultZoom();
+        });
+        smoothing_selector.unbind().change(function(e) {
+            // change the metric in the url to reload the page:
+            smoothing = query.smoothing = this.value;
+            graph_callback();
+            defaultZoom();
+        });
+        updateURLBar();
+
+        $("#dl-test-data").attr("href",stats_db);
+
+        // Chart zoom/drag surface
+        // This should always be last, so it's on top of everything else
+        svg.append("svg:rect")
+            .attr("id", "zoom_drag_surface")
+            .attr("width", width)
+            .attr("height", height);
+    }
+
+    raw_data = stats;
+    graph_callback();
+
+}
+
+$(document).ready(function(){
+
+    drawGraph();
+
+});
+      -->
+  </script>
+  <style type="text/css">
+div#chart_controls {
+    margin-left: 900px;
+    position: absolute;
+}
+
+#chart_controls > table {
+    width: 640px;
+}
+
+#chart_controls td {
+    padding: 2px;
+}
+
+#chart_controls #zoom input {
+    width: 50px;
+}
+
+#chart_controls table#zoom {
+    padding-left: 20px;
+}
+
+#chart_controls table#zoom td {
+    padding-left: 20px;
+}
+
+#zoom_drag_surface {
+    fill: rgba(250, 250, 255, 0.0);
+    z-index: 100;
+}
+
+svg {
+  font: 10px sans-serif;
+}
+
+.axis path,
+.axis line {
+  fill: none;
+  stroke: #000;
+  shape-rendering: crispEdges;
+}
+
+.x.axis path {
+  display: none;
+}
+
+.line {
+  fill: none;
+  stroke: steelblue;
+  stroke-width: 1.5px;
+}
+
+  </style>
+</head>
+<body>
+</body>
\ No newline at end of file
diff --git a/test/anttasks/org/apache/cassandra/anttasks/JdkProperties.java b/test/anttasks/org/apache/cassandra/anttasks/JdkProperties.java
index 2e5d202a20d9..59aba5e46420 100644
--- a/test/anttasks/org/apache/cassandra/anttasks/JdkProperties.java
+++ b/test/anttasks/org/apache/cassandra/anttasks/JdkProperties.java
@@ -27,7 +27,7 @@ public class JdkProperties extends Task
     public void execute()
     {
         Project project = getProject();
-        project.setNewProperty("java.version." + project.getProperty("ant.java.version").replace("1.", ""), "true");
-        project.setNewProperty("use-jdk" + project.getProperty("ant.java.version").replace("1.", ""), "true");
+        project.setNewProperty("java.version." + project.getProperty("ant.java.version"), "true");
+        project.setNewProperty("use-jdk" + project.getProperty("ant.java.version"), "true");
     }
 }
diff --git a/test/burn/org/apache/cassandra/index/sai/LongBM25Test.java b/test/burn/org/apache/cassandra/index/sai/LongBM25Test.java
new file mode 100644
index 000000000000..51d33a3c656f
--- /dev/null
+++ b/test/burn/org/apache/cassandra/index/sai/LongBM25Test.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+
+public class LongBM25Test extends SAITester
+{
+    private static final Logger logger = org.slf4j.LoggerFactory.getLogger(LongBM25Test.class);
+
+    private static final List<String> documentLines = new ArrayList<>();
+
+    static
+    {
+        try
+        {
+            var cl = LongBM25Test.class.getClassLoader();
+            var resourceDir = cl.getResource("bm25");
+            if (resourceDir == null)
+                throw new RuntimeException("Could not find resource directory test/resources/bm25/");
+
+            var dirPath = java.nio.file.Paths.get(resourceDir.toURI());
+            try (var files = java.nio.file.Files.list(dirPath))
+            {
+                files.forEach(file -> {
+                    try (var lines = java.nio.file.Files.lines(file))
+                    {
+                        lines.map(String::trim)
+                             .filter(line -> !line.isEmpty())
+                             .forEach(documentLines::add);
+                    }
+                    catch (IOException e)
+                    {
+                        throw new RuntimeException("Failed to read file: " + file, e);
+                    }
+                });
+            }
+            if (documentLines.isEmpty())
+            {
+                throw new RuntimeException("No document lines loaded from test/resources/bm25/");
+            }
+        }
+        catch (IOException | URISyntaxException e)
+        {
+            throw new RuntimeException("Failed to load test documents", e);
+        }
+    }
+
+    KeySet keysInserted = new KeySet();
+    private final int threadCount = 12;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        MEMTABLE_SHARD_COUNT.setInt(4 * threadCount);
+    }
+
+    @FunctionalInterface
+    private interface Op
+    {
+        void run(int i) throws Throwable;
+    }
+
+    public void testConcurrentOps(Op op) throws ExecutionException, InterruptedException
+    {
+        createTable("CREATE TABLE %s (key int primary key, value text)");
+        // Create analyzed index following BM25Test pattern
+        createIndex("CREATE CUSTOM INDEX ON %s(value) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = {" +
+                    "'index_analyzer': '{" +
+                    "\"tokenizer\" : {\"name\" : \"standard\"}, " +
+                    "\"filters\" : [{\"name\" : \"porterstem\"}]" +
+                    "}'}"
+        );
+
+        AtomicInteger counter = new AtomicInteger();
+        long start = System.currentTimeMillis();
+        var fjp = new ForkJoinPool(threadCount);
+        var keys = IntStream.range(0, 10_000_000).boxed().collect(Collectors.toList());
+        Collections.shuffle(keys);
+        var task = fjp.submit(() -> keys.stream().parallel().forEach(i ->
+                                                                     {
+                                                                         wrappedOp(op, i);
+                                                                         if (counter.incrementAndGet() % 10_000 == 0)
+                                                                         {
+                                                                             var elapsed = System.currentTimeMillis() - start;
+                                                                             logger.info("{} ops in {}ms = {} ops/s", counter.get(), elapsed, counter.get() * 1000.0 / elapsed);
+                                                                         }
+                                                                         if (ThreadLocalRandom.current().nextDouble() < 0.001)
+                                                                             flush();
+                                                                     }));
+        fjp.shutdown();
+        task.get(); // re-throw
+    }
+
+    private static void wrappedOp(Op op, Integer i)
+    {
+        try
+        {
+            op.run(i);
+        }
+        catch (Throwable e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static String randomDocument()
+    {
+        var R = ThreadLocalRandom.current();
+        int numLines = R.nextInt(5, 51); // 5 to 50 lines inclusive
+        var selectedLines = new ArrayList<String>();
+
+        for (int i = 0; i < numLines; i++)
+        {
+            selectedLines.add(randomQuery(R));
+        }
+
+        return String.join("\n", selectedLines);
+    }
+
+    private static String randomLine(ThreadLocalRandom R)
+    {
+        return documentLines.get(R.nextInt(documentLines.size()));
+    }
+
+    @Test
+    public void testConcurrentReadsWritesDeletes() throws ExecutionException, InterruptedException
+    {
+        testConcurrentOps(i -> {
+            var R = ThreadLocalRandom.current();
+            if (R.nextDouble() < 0.2 || keysInserted.isEmpty())
+            {
+                var doc = randomDocument();
+                execute("INSERT INTO %s (key, value) VALUES (?, ?)", i, doc);
+                keysInserted.add(i);
+            }
+            else if (R.nextDouble() < 0.1)
+            {
+                var key = keysInserted.getRandom();
+                execute("DELETE FROM %s WHERE key = ?", key);
+            }
+            else
+            {
+                var line = randomQuery(R);
+                execute("SELECT * FROM %s ORDER BY value BM25 OF ? LIMIT ?", line, R.nextInt(1, 100));
+            }
+        });
+    }
+
+    private static String randomQuery(ThreadLocalRandom R)
+    {
+        while (true)
+        {
+            var line = randomLine(R);
+            if (line.chars().anyMatch(Character::isAlphabetic))
+                return line;
+        }
+    }
+
+    @Test
+    public void testConcurrentReadsWrites() throws ExecutionException, InterruptedException
+    {
+        testConcurrentOps(i -> {
+            var R = ThreadLocalRandom.current();
+            if (R.nextDouble() < 0.1 || keysInserted.isEmpty())
+            {
+                var doc = randomDocument();
+                execute("INSERT INTO %s (key, value) VALUES (?, ?)", i, doc);
+                keysInserted.add(i);
+            }
+            else
+            {
+                var line = randomQuery(R);
+                execute("SELECT * FROM %s ORDER BY value BM25 OF ? LIMIT ?", line, R.nextInt(1, 100));
+            }
+        });
+    }
+
+    @Test
+    public void testConcurrentWrites() throws ExecutionException, InterruptedException
+    {
+        testConcurrentOps(i -> {
+            var doc = randomDocument();
+            execute("INSERT INTO %s (key, value) VALUES (?, ?)", i, doc);
+        });
+    }
+
+    private static class KeySet
+    {
+        private final Map<Integer, Integer> keys = new ConcurrentHashMap<>();
+        private final AtomicInteger ordinal = new AtomicInteger();
+
+        public void add(int key)
+        {
+            var i = ordinal.getAndIncrement();
+            keys.put(i, key);
+        }
+
+        public int getRandom()
+        {
+            if (isEmpty())
+                throw new IllegalStateException();
+            var i = ThreadLocalRandom.current().nextInt(ordinal.get());
+            // in case there is race with add(key), retry another random
+            return keys.containsKey(i) ? keys.get(i) : getRandom();
+        }
+
+        public boolean isEmpty()
+        {
+            return keys.isEmpty();
+        }
+    }
+}
diff --git a/test/burn/org/apache/cassandra/index/sai/LongVectorTest.java b/test/burn/org/apache/cassandra/index/sai/LongVectorTest.java
new file mode 100644
index 000000000000..99886a92eb5e
--- /dev/null
+++ b/test/burn/org/apache/cassandra/index/sai/LongVectorTest.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class LongVectorTest extends SAITester
+{
+    private static final Logger logger = org.slf4j.LoggerFactory.getLogger(LongVectorTest.class);
+
+    int dimension = 16; // getRandom().nextIntBetween(128, 768);
+
+    KeySet keysInserted = new KeySet();
+    private static final int threadCount = 12;
+
+    @BeforeClass
+    public static void setShardCount()
+    {
+        MEMTABLE_SHARD_COUNT.setInt(4 * threadCount);
+    }
+
+    @FunctionalInterface
+    private interface Op
+    {
+        public void run(int i) throws Throwable;
+    }
+
+    public void testConcurrentOps(Op op) throws ExecutionException, InterruptedException
+    {
+        createTable(String.format("CREATE TABLE %%s (key int primary key, value vector<float, %s>)", dimension));
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex' WITH OPTIONS = { 'similarity_function': 'dot_product' }");
+
+        AtomicInteger counter = new AtomicInteger();
+        long start = System.currentTimeMillis();
+        var fjp = new ForkJoinPool(threadCount);
+        var keys = IntStream.range(0, 10_000_000).boxed().collect(Collectors.toList());
+        Collections.shuffle(keys);
+        var task = fjp.submit(() -> keys.stream().parallel().forEach(i ->
+        {
+            wrappedOp(op, i);
+            if (counter.incrementAndGet() % 10_000 == 0)
+            {
+                var elapsed = System.currentTimeMillis() - start;
+                logger.info("{} ops in {}ms = {} ops/s", counter.get(), elapsed, counter.get() * 1000.0 / elapsed);
+            }
+            if (ThreadLocalRandom.current().nextDouble() < 0.001)
+                flush();
+        }));
+        fjp.shutdown();
+        task.get(); // re-throw
+    }
+
+    private static void wrappedOp(Op op, Integer i)
+    {
+        try
+        {
+            op.run(i);
+        }
+        catch (Throwable e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Test
+    public void testConcurrentReadsWritesDeletes() throws ExecutionException, InterruptedException
+    {
+        testConcurrentOps(i -> {
+            var R = ThreadLocalRandom.current();
+            var v = randomVectorBoxed(dimension);
+            if (R.nextDouble() < 0.2 || keysInserted.isEmpty())
+            {
+                execute("INSERT INTO %s (key, value) VALUES (?, ?)", i, v);
+                keysInserted.add(i);
+            } else if (R.nextDouble() < 0.1) {
+                var key = keysInserted.getRandom();
+                execute("DELETE FROM %s WHERE key = ?", key);
+            } else if (R.nextDouble() < 0.5) {
+                var key = keysInserted.getRandom();
+                execute("SELECT * FROM %s WHERE key = ? ORDER BY value ANN OF ? LIMIT ?", key, v, R.nextInt(1, 100));
+            } else {
+                execute("SELECT * FROM %s ORDER BY value ANN OF ? LIMIT ?", v, R.nextInt(1, 100));
+            }
+        });
+    }
+
+    // like testConcurrentReadsWritesDeletes, but generates multiple rows w/ the same vector, and
+    // the sub-op weights are biased more towards doing additional inserts
+    @Test
+    public void testMultiplePostings() throws ExecutionException, InterruptedException
+    {
+        testConcurrentOps(i -> {
+            var R = ThreadLocalRandom.current();
+            var v = sequentiallyDuplicateVector(i, dimension);
+            if (R.nextDouble() < 0.8 || keysInserted.isEmpty())
+            {
+                execute("INSERT INTO %s (key, value) VALUES (?, ?)", i, v);
+                keysInserted.add(i);
+            } else if (R.nextDouble() < 0.1) {
+                var key = keysInserted.getRandom();
+                execute("DELETE FROM %s WHERE key = ?", key);
+            } else if (R.nextDouble() < 0.5) {
+                var key = keysInserted.getRandom();
+                execute("SELECT * FROM %s WHERE key = ? ORDER BY value ANN OF ? LIMIT ?", key, v, R.nextInt(1, 100));
+            } else {
+                execute("SELECT * FROM %s ORDER BY value ANN OF ? LIMIT ?", v, R.nextInt(1, 100));
+            }
+        });
+    }
+
+    @Test
+    public void testConcurrentReadsWrites() throws ExecutionException, InterruptedException
+    {
+        testConcurrentOps(i -> {
+            var R = ThreadLocalRandom.current();
+            var v = randomVectorBoxed(dimension);
+            if (R.nextDouble() < 0.1 || keysInserted.isEmpty())
+            {
+                execute("INSERT INTO %s (key, value) VALUES (?, ?)", i, v);
+                keysInserted.add(i);
+            } else if (R.nextDouble() < 0.5) {
+                var key = keysInserted.getRandom();
+                var results = execute("SELECT * FROM %s WHERE key = ? ORDER BY value ANN OF ? LIMIT ?", key, v, R.nextInt(1, 100));
+                assertThat(results).hasSize(1);
+            } else {
+                var results = execute("SELECT * FROM %s ORDER BY value ANN OF ? LIMIT ?", v, R.nextInt(1, 100));
+                assertThat(results).hasSizeGreaterThan(0); // VSTODO can we make a stronger assertion?
+            }
+        });
+    }
+
+    @Test
+    public void testConcurrentWrites() throws ExecutionException, InterruptedException
+    {
+        testConcurrentOps(i -> {
+            var v = randomVectorBoxed(dimension);
+            execute("INSERT INTO %s (key, value) VALUES (?, ?)", i, v);
+        });
+    }
+
+    /**
+     * @return a normalized vector with the given dimension, where each vector from 0 .. N-1 is the same,
+     * N .. 2N-1 is the same, etc., where N is the number of cores.
+     */
+    private static Vector<Float> sequentiallyDuplicateVector(int i, int dimension)
+    {
+        int j = 1 + i / Runtime.getRuntime().availableProcessors();
+        var vector = new float[dimension];
+        outer:
+        while (true)
+        {
+            for (int k = 0; k < dimension; k++)
+            {
+                vector[k] += 1.0f;
+                if (j-- <= 0)
+                    break outer;
+            }
+        }
+        normalize(vector);
+        return vector(vector);
+    }
+
+    private static class KeySet
+    {
+        private final Map<Integer, Integer> keys = new ConcurrentHashMap<>();
+        private final AtomicInteger ordinal = new AtomicInteger();
+
+        public void add(int key)
+        {
+            var i = ordinal.getAndIncrement();
+            keys.put(i, key);
+        }
+
+        public int getRandom()
+        {
+            if (isEmpty())
+                throw new IllegalStateException();
+            var i = ThreadLocalRandom.current().nextInt(ordinal.get());
+            // in case there is race with add(key), retry another random
+            return keys.containsKey(i) ? keys.get(i) : getRandom();
+        }
+
+        public boolean isEmpty()
+        {
+            return keys.isEmpty();
+        }
+    }
+}
diff --git a/test/burn/org/apache/cassandra/net/Connection.java b/test/burn/org/apache/cassandra/net/Connection.java
index de5df6b65de5..52c0fcae164d 100644
--- a/test/burn/org/apache/cassandra/net/Connection.java
+++ b/test/burn/org/apache/cassandra/net/Connection.java
@@ -333,6 +333,11 @@ public void onExecuted(int messageSize, Message.Header header, long timeElapsed,
     {
     }
 
+    @Override
+    public void onMessageHandlingCompleted(Message.Header header, long timeElapsed, TimeUnit unit)
+    {
+    }
+
     InboundCounters inboundCounters()
     {
         return inbound.countersFor(outbound.type());
diff --git a/test/burn/org/apache/cassandra/net/ConnectionBurnTest.java b/test/burn/org/apache/cassandra/net/ConnectionBurnTest.java
index abacb6e034bd..941622ad4844 100644
--- a/test/burn/org/apache/cassandra/net/ConnectionBurnTest.java
+++ b/test/burn/org/apache/cassandra/net/ConnectionBurnTest.java
@@ -86,10 +86,21 @@ static class NoGlobalInboundMetrics implements InboundMessageHandlers.GlobalMetr
         static final NoGlobalInboundMetrics instance = new NoGlobalInboundMetrics();
         public LatencyConsumer internodeLatencyRecorder(InetAddressAndPort to)
         {
-            return (timeElapsed, timeUnit) -> {};
+            return (verb, timeElapsed, timeUnit) -> {};
         }
-        public void recordInternalLatency(Verb verb, long timeElapsed, TimeUnit timeUnit) {}
+        public void recordInternalLatency(Verb verb, InetAddressAndPort from, long timeElapsed, TimeUnit timeUnit) {}
+
         public void recordInternodeDroppedMessage(Verb verb, long timeElapsed, TimeUnit timeUnit) {}
+
+        @Override
+        public void recordMessageStageProcessingTime(Verb verb, InetAddressAndPort from, long timeElapsed, TimeUnit unit)
+        {
+        }
+
+        @Override
+        public void recordTotalMessageProcessingTime(Verb verb, InetAddressAndPort from, long timeElapsed, TimeUnit unit)
+        {
+        }
     }
 
     static class Inbound
@@ -564,6 +575,13 @@ public void onExecuted(int messageSize, Message.Header header, long timeElapsed,
                 forId(header.id).onExecuted(messageSize, header, timeElapsed, unit);
                 wrapped.onExecuted(messageSize, header, timeElapsed, unit);
             }
+
+            @Override
+            public void onMessageHandlingCompleted(Message.Header header, long timeElapsed, TimeUnit unit)
+            {
+                forId(header.id).onMessageHandlingCompleted(header, timeElapsed, unit);
+                wrapped.onMessageHandlingCompleted(header, timeElapsed, unit);
+            }
         }
 
         public void fail(Message.Header header, Throwable failure)
diff --git a/test/burn/org/apache/cassandra/transport/BurnTestUtil.java b/test/burn/org/apache/cassandra/transport/BurnTestUtil.java
index c8017d151f69..6a33cbd7a3bd 100644
--- a/test/burn/org/apache/cassandra/transport/BurnTestUtil.java
+++ b/test/burn/org/apache/cassandra/transport/BurnTestUtil.java
@@ -27,6 +27,7 @@
 import com.datastax.driver.core.SimpleStatement;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.ResultSet;
 import org.apache.cassandra.db.ConsistencyLevel;
@@ -78,7 +79,7 @@ public static QueryMessage generateQueryMessage(int idx, SizeCaps sizeCaps, Prot
         QueryOptions queryOptions = QueryOptions.create(ConsistencyLevel.ONE,
                                                         values,
                                                         true,
-                                                        10,
+                                                        PageSize.inRows(10),
                                                         null,
                                                         null,
                                                         version,
diff --git a/test/burn/org/apache/cassandra/transport/DriverBurnTest.java b/test/burn/org/apache/cassandra/transport/DriverBurnTest.java
index 42b7c6bbaadb..c37d43b4c1f7 100644
--- a/test/burn/org/apache/cassandra/transport/DriverBurnTest.java
+++ b/test/burn/org/apache/cassandra/transport/DriverBurnTest.java
@@ -19,8 +19,14 @@
 package org.apache.cassandra.transport;
 
 import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
@@ -31,7 +37,12 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import com.datastax.driver.core.*;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ProtocolOptions;
+import com.datastax.driver.core.ResultSetFuture;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
 import io.netty.buffer.ByteBuf;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.service.NativeTransportService;
@@ -39,6 +50,8 @@
 import org.apache.cassandra.transport.messages.QueryMessage;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.AssertUtil;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 import static org.apache.cassandra.config.EncryptionOptions.TlsEncryptionPolicy.UNENCRYPTED;
 import static org.apache.cassandra.transport.BurnTestUtil.SizeCaps;
@@ -78,18 +91,20 @@ public QueryMessage decode(ByteBuf body, ProtocolVersion version)
             {
                 QueryMessage queryMessage = QueryMessage.codec.decode(body, version);
                 return new QueryMessage(queryMessage.query, queryMessage.options) {
-                    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+
+                    @Override
+                    protected Future<Response> maybeExecuteAsync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
                     {
                         try
                         {
                             int idx = Integer.parseInt(queryMessage.query);
                             SizeCaps caps = idx % largeMessageFrequency == 0 ? largeMessageCap : smallMessageCap;
-                            return generateRows(idx, caps);
+                            return ImmediateFuture.success(generateRows(idx, caps));
                         }
                         catch (NumberFormatException e)
                         {
                             // for the requests driver issues under the hood
-                            return super.execute(state, requestTime, traceRequest);
+                            return super.maybeExecuteAsync(state, requestTime, traceRequest);
                         }
                     }
                 };
@@ -338,17 +353,19 @@ public QueryMessage decode(ByteBuf body, ProtocolVersion version)
             {
                 QueryMessage queryMessage = QueryMessage.codec.decode(body, version);
                 return new QueryMessage(queryMessage.query, queryMessage.options) {
-                    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+
+                    @Override
+                    protected Future<Response> maybeExecuteAsync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
                     {
                         try
                         {
                             int idx = Integer.parseInt(queryMessage.query); // unused
-                            return generateRows(idx, responseCaps);
+                            return ImmediateFuture.success(generateRows(idx, responseCaps));
                         }
                         catch (NumberFormatException e)
                         {
                             // for the requests driver issues under the hood
-                            return super.execute(state, requestTime, traceRequest);
+                            return super.maybeExecuteAsync(state, requestTime, traceRequest);
                         }
                     }
                 };
diff --git a/test/burn/org/apache/cassandra/transport/SimpleClientBurnTest.java b/test/burn/org/apache/cassandra/transport/SimpleClientBurnTest.java
index ef29146be2f8..7d7e4716c1fd 100644
--- a/test/burn/org/apache/cassandra/transport/SimpleClientBurnTest.java
+++ b/test/burn/org/apache/cassandra/transport/SimpleClientBurnTest.java
@@ -44,6 +44,8 @@
 import org.apache.cassandra.transport.messages.QueryMessage;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.AssertUtil;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 import static org.apache.cassandra.config.EncryptionOptions.TlsEncryptionPolicy.UNENCRYPTED;
 import static org.apache.cassandra.transport.BurnTestUtil.SizeCaps;
@@ -114,11 +116,11 @@ public QueryMessage decode(ByteBuf body, ProtocolVersion version)
                 return new QueryMessage(queryMessage.query, queryMessage.options)
                 {
                     @Override
-                    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+                    public Future<Response> maybeExecuteAsync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
                     {
                         int idx = Integer.parseInt(queryMessage.query);
                         SizeCaps caps = idx % largeMessageFrequency == 0 ? largeMessageCap : smallMessageCap;
-                        return generateRows(idx, caps);
+                        return ImmediateFuture.success(generateRows(idx, caps));
                     }
                 };
             }
diff --git a/test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java b/test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java
index 3f990cbca31d..3fd083c27e1e 100644
--- a/test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java
+++ b/test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java
@@ -50,6 +50,8 @@
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.AssertUtil;
 import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 import static org.apache.cassandra.transport.BurnTestUtil.SizeCaps;
 import static org.apache.cassandra.transport.BurnTestUtil.generateQueryMessage;
@@ -170,10 +172,10 @@ public QueryMessage decode(ByteBuf body, ProtocolVersion version)
                 return new QueryMessage(queryMessage.query, queryMessage.options)
                 {
                     @Override
-                    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
+                    public Future<Response> maybeExecuteAsync(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
                     {
                         int idx = Integer.parseInt(queryMessage.query); // unused
-                        return generateRows(idx, responseCaps);
+                        return ImmediateFuture.success(generateRows(idx, responseCaps));
                     }
                 };
             }
diff --git a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
index d8a4f81637e1..b49a63e5e8e2 100644
--- a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
+++ b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
@@ -653,7 +653,7 @@ private static RandomTree randomTreeByBuilder(long seed, Random random, int minS
     // return a value with the search position
     private static List<Integer> randomKeys(Random random, Iterable<Integer> canonical, boolean mixInNotPresentItems)
     {
-        boolean useFake = mixInNotPresentItems && random.nextBoolean();
+        final boolean useFake = mixInNotPresentItems && random.nextBoolean();
         final float fakeRatio = random.nextFloat();
         List<Integer> results = new ArrayList<>();
         Long fakeLb = (long) Integer.MIN_VALUE, fakeUb = null;
@@ -671,12 +671,12 @@ private static List<Integer> randomKeys(Random random, Iterable<Integer> canonic
             }
             else
             {
-                // otherwise we emit a fake value in the range immediately proceeding the last real value, and not
+                // otherwise we emit a fake value in the range immediately preceeding the last real value, and not
                 // exceeding the real value that would have proceeded (ignoring any other suppressed real values since)
                 if (fakeUb == null)
                     fakeUb = v.longValue() - 1;
                 long mid = (fakeLb + fakeUb) / 2;
-                assert mid < fakeUb;
+                assert mid < v.longValue();
                 results.add((int) mid);
                 fakeLb = mid;
             }
diff --git a/test/conf/bigtable.yaml b/test/conf/bigtable.yaml
new file mode 100644
index 000000000000..e1e454f5d38d
--- /dev/null
+++ b/test/conf/bigtable.yaml
@@ -0,0 +1,2 @@
+# Test legacy sstable format along with legacy sstable identifiers
+uuid_sstable_identifiers_enabled: false
diff --git a/test/conf/cassandra-mtls.yaml b/test/conf/cassandra-mtls.yaml
index d6f1b3e52c6b..c4b531f13c1b 100644
--- a/test/conf/cassandra-mtls.yaml
+++ b/test/conf/cassandra-mtls.yaml
@@ -28,6 +28,7 @@ commitlog_directory: build/test/cassandra/commitlog
 cdc_raw_directory: build/test/cassandra/cdc_raw
 cdc_enabled: false
 hints_directory: build/test/cassandra/hints
+metadata_directory: build/test/cassandra/metadata
 partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
 listen_address: 127.0.0.1
 storage_port: 7012
diff --git a/test/conf/cassandra-murmur.yaml b/test/conf/cassandra-murmur.yaml
index 2e5828fb56a0..46d6ed9cdd19 100644
--- a/test/conf/cassandra-murmur.yaml
+++ b/test/conf/cassandra-murmur.yaml
@@ -11,6 +11,7 @@ commitlog_directory: build/test/cassandra/commitlog
 cdc_raw_directory: build/test/cassandra/cdc_raw
 cdc_enabled: false
 hints_directory: build/test/cassandra/hints
+metadata_directory: build/test/cassandra/metadata
 partitioner: org.apache.cassandra.dht.Murmur3Partitioner
 listen_address: 127.0.0.1
 storage_port: 7012
@@ -42,3 +43,7 @@ user_defined_functions_enabled: true
 scripted_user_defined_functions_enabled: false
 sasi_indexes_enabled: true
 materialized_views_enabled: true
+default_compaction:
+    class_name: UnifiedCompactionStrategy
+    parameters:
+        base_shard_count: 1
diff --git a/test/conf/cassandra-old.yaml b/test/conf/cassandra-old.yaml
index b8c3b028c519..9b9bb51fcabf 100644
--- a/test/conf/cassandra-old.yaml
+++ b/test/conf/cassandra-old.yaml
@@ -14,6 +14,7 @@ commitlog_directory: build/test/cassandra/commitlog
 cdc_raw_directory: build/test/cassandra/cdc_raw
 cdc_enabled: false
 hints_directory: build/test/cassandra/hints
+metadata_directory: build/test/cassandra/metadata
 partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
 listen_address: 127.0.0.1
 storage_port: 7012
diff --git a/test/conf/cassandra-seeds.yaml b/test/conf/cassandra-seeds.yaml
index 53f82dd6ecd7..bdb15b8817ec 100644
--- a/test/conf/cassandra-seeds.yaml
+++ b/test/conf/cassandra-seeds.yaml
@@ -12,6 +12,7 @@ commitlog_directory: build/test/cassandra/commitlog
 cdc_raw_directory: build/test/cassandra/cdc_raw
 cdc_enabled: false
 hints_directory: build/test/cassandra/hints
+metadata_directory: build/test/cassandra/metadata
 partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
 listen_address: 127.0.0.1
 storage_port: 7012
diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml
index e9ba02c4415e..652dc9586487 100644
--- a/test/conf/cassandra.yaml
+++ b/test/conf/cassandra.yaml
@@ -13,8 +13,8 @@ commitlog_disk_access_mode: legacy
 # commitlog_compression:
 # - class_name: LZ4Compressor
 cdc_raw_directory: build/test/cassandra/cdc_raw
-cdc_enabled: false
 hints_directory: build/test/cassandra/hints
+metadata_directory: build/test/cassandra/metadata
 partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
 listen_address: 127.0.0.1
 storage_port: 7012
@@ -67,15 +67,29 @@ local_read_size_warn_threshold: 4096KiB
 local_read_size_fail_threshold: 8192KiB
 row_index_read_size_warn_threshold: 4096KiB
 row_index_read_size_fail_threshold: 8192KiB
+read_request_timeout_in_ms: 20000
+range_request_timeout_in_ms: 20000
+write_request_timeout_in_ms: 20000
+counter_write_request_timeout_in_ms: 20000
+cas_contention_timeout_in_ms: 20000
+request_timeout_in_ms: 20000
+default_compaction:
+    class_name: UnifiedCompactionStrategy
+    parameters:
+        base_shard_count: 1
 
 memtable:
     configurations:
         skiplist:
             class_name: SkipListMemtable
+        persistent_memory:
+            class_name: PersistentMemoryMemtable
         trie:
             class_name: TrieMemtable
             parameters:
                 shards: 4
+        trie_stage1:
+            class_name: TrieMemtableStage1
         skiplist_sharded:
             class_name: ShardedSkipListMemtable
             parameters:
diff --git a/test/conf/logback-burntest.xml b/test/conf/logback-burntest.xml
index e1e48a9d3fae..7f4d85fe30c0 100644
--- a/test/conf/logback-burntest.xml
+++ b/test/conf/logback-burntest.xml
@@ -20,7 +20,7 @@
   <define name="instance_id" class="org.apache.cassandra.distributed.impl.InstanceIDDefiner" />
 
   <!-- Shutdown hook ensures that async appender flushes -->
-  <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+  <shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook"/>
 
   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
 
@@ -59,7 +59,7 @@
   <logger name="org.apache.hadoop" level="WARN"/>
   <logger name="io.netty.handler.ssl.SslHandler" level="WARN"/>
 
-  <root level="DEBUG">
+  <root level="INFO">
     <appender-ref ref="ASYNCFILE" />
     <appender-ref ref="STDOUT" />
   </root>
diff --git a/test/conf/logback-dtest.xml b/test/conf/logback-dtest.xml
index 52eaf335ded3..f72899b4dbe0 100644
--- a/test/conf/logback-dtest.xml
+++ b/test/conf/logback-dtest.xml
@@ -22,7 +22,7 @@
   <define name="instance_id" class="org.apache.cassandra.distributed.impl.InstanceIDDefiner" />
 
   <!-- Shutdown hook ensures that async appender flushes -->
-  <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+  <shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook"/>
 
   <appender name="INSTANCEFILE" class="ch.qos.logback.core.FileAppender">
     <file>./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log</file>
@@ -34,7 +34,7 @@
 
   <appender name="INSTANCESTDERR" target="System.err" class="ch.qos.logback.core.ConsoleAppender">
     <encoder>
-      <pattern>%-5level %date{HH:mm:ss,SSS} %msg%n</pattern>
+      <pattern>%-5level %date{"HH:mm:ss,SSS"} %msg%n</pattern>
     </encoder>
     <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
       <level>WARN</level>
diff --git a/test/conf/logback-dtest_with_vtable_appender.xml b/test/conf/logback-dtest_with_vtable_appender.xml
index c9fd108c77d8..2eb8bba4eb26 100644
--- a/test/conf/logback-dtest_with_vtable_appender.xml
+++ b/test/conf/logback-dtest_with_vtable_appender.xml
@@ -21,7 +21,7 @@
     <define name="instance_id" class="org.apache.cassandra.distributed.impl.InstanceIDDefiner" />
 
     <!-- Shutdown hook ensures that async appender flushes -->
-    <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+    <shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook"/>
 
     <appender name="INSTANCEFILE" class="ch.qos.logback.core.FileAppender">
         <file>./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log</file>
@@ -33,7 +33,7 @@
 
     <appender name="INSTANCESTDERR" target="System.err" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%-5level %date{HH:mm:ss,SSS} %msg%n</pattern>
+            <pattern>%-5level %date{"HH:mm:ss,SSS"} %msg%n</pattern>
         </encoder>
         <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
             <level>WARN</level>
diff --git a/test/conf/logback-dtest_with_vtable_appender_invalid.xml b/test/conf/logback-dtest_with_vtable_appender_invalid.xml
index 1b30c141c2a7..13984d5fe265 100644
--- a/test/conf/logback-dtest_with_vtable_appender_invalid.xml
+++ b/test/conf/logback-dtest_with_vtable_appender_invalid.xml
@@ -21,7 +21,7 @@
     <define name="instance_id" class="org.apache.cassandra.distributed.impl.InstanceIDDefiner" />
 
     <!-- Shutdown hook ensures that async appender flushes -->
-    <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+    <shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook"/>
 
     <appender name="INSTANCEFILE" class="ch.qos.logback.core.FileAppender">
         <file>./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log</file>
@@ -33,7 +33,7 @@
 
     <appender name="INSTANCESTDERR" target="System.err" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%-5level %date{HH:mm:ss,SSS} %msg%n</pattern>
+            <pattern>%-5level %date{"HH:mm:ss,SSS"} %msg%n</pattern>
         </encoder>
         <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
             <level>WARN</level>
diff --git a/test/conf/logback-test-jenkins.xml b/test/conf/logback-test-jenkins.xml
new file mode 100644
index 000000000000..104b311046ce
--- /dev/null
+++ b/test/conf/logback-test-jenkins.xml
@@ -0,0 +1,64 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<configuration debug="false" scan="true" scanPeriod="60 seconds">
+  <!-- Shutdown hook ensures that async appender flushes -->
+  <shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook"/>
+
+  <!-- Status listener is used to wrap stdout/stderr and tee to log file -->
+  <statusListener class="org.apache.cassandra.LogbackStatusListener"/>
+
+  <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+
+    <file>./build/test/logs/${cassandra.testtag}/TEST-${suitename}.log</file>
+    <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+      <fileNamePattern>./build/test/logs/${cassandra.testtag}/TEST-${suitename}.log.%i.gz</fileNamePattern>
+      <minIndex>1</minIndex>
+      <maxIndex>20</maxIndex>
+    </rollingPolicy>
+
+    <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+      <maxFileSize>20MB</maxFileSize>
+    </triggeringPolicy>
+
+    <encoder>
+      <pattern>%-5level [%thread] %date{ISO8601} %msg%n</pattern>
+    </encoder>
+    <immediateFlush>false</immediateFlush>
+
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>DEBUG</level>
+    </filter>
+  </appender>
+
+  <!-- Do not change the name of this appender. LogbackStatusListener uses the thread name
+       tied to the appender name to know when to write to real stdout/stderr vs forwarding to logback -->
+  <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender">
+    <discardingThreshold>0</discardingThreshold>
+    <maxFlushTime>0</maxFlushTime>
+    <queueSize>1024</queueSize>
+    <appender-ref ref="FILE"/>
+    <includeCallerData>true</includeCallerData>
+    <filter class="org.apache.cassandra.ForbiddenLogEntriesFilter"/>
+  </appender>
+
+  <root level="TRACE">
+    <appender-ref ref="ASYNC"/>
+  </root>
+</configuration>
diff --git a/test/conf/logback-test.xml b/test/conf/logback-test.xml
index 3e3349fd82f0..47480e676d4a 100644
--- a/test/conf/logback-test.xml
+++ b/test/conf/logback-test.xml
@@ -19,10 +19,10 @@
 
 <configuration debug="false" scan="true" scanPeriod="60 seconds">
   <!-- Shutdown hook ensures that async appender flushes -->
-  <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+  <shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook"/>
 
   <!-- Status listener is used to wrap stdout/stderr and tee to log file -->
-  <statusListener class="org.apache.cassandra.LogbackStatusListener" />
+  <statusListener class="org.apache.cassandra.LogbackStatusListener"/>
 
   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
 
@@ -38,7 +38,7 @@
     </triggeringPolicy>
 
     <encoder>
-      <pattern>%-5level [%thread] %date{ISO8601} %msg%n</pattern>
+      <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
     </encoder>
     <immediateFlush>false</immediateFlush>
   </appender>
@@ -53,8 +53,8 @@
   </appender>
 
   <appender name="TEE" class="org.apache.cassandra.TeeingAppender">
-      <appender-ref ref="FILE"/>
-      <appender-ref ref="STDOUT"/>
+    <appender-ref ref="FILE"/>
+    <appender-ref ref="STDOUT"/>
   </appender>
 
   <logger name="org.apache.hadoop" level="WARN"/>
@@ -64,14 +64,17 @@
   <!-- Do not change the name of this appender. LogbackStatusListener uses the thread name
        tied to the appender name to know when to write to real stdout/stderr vs forwarding to logback -->
   <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender">
-      <discardingThreshold>0</discardingThreshold>
-      <maxFlushTime>0</maxFlushTime>
-      <queueSize>1024</queueSize>
-      <appender-ref ref="TEE"/>
-      <includeCallerData>true</includeCallerData>
+    <discardingThreshold>0</discardingThreshold>
+    <maxFlushTime>0</maxFlushTime>
+    <queueSize>1024</queueSize>
+    <appender-ref ref="TEE"/>
+    <includeCallerData>true</includeCallerData>
+    <filter class="org.apache.cassandra.ForbiddenLogEntriesFilter"/>
   </appender>
 
   <root level="DEBUG">
-    <appender-ref ref="ASYNC" />
+    <appender-ref ref="ASYNC"/>
   </root>
+
+  <logger name="io.netty.util.internal" level="WARN"/>
 </configuration>
diff --git a/test/conf/unit-test-conf/test-native-port.yaml b/test/conf/unit-test-conf/test-native-port.yaml
index 2d0b184f15c7..18a5e9fa66c1 100644
--- a/test/conf/unit-test-conf/test-native-port.yaml
+++ b/test/conf/unit-test-conf/test-native-port.yaml
@@ -43,7 +43,7 @@ compaction_throughput: 0MiB/s
 row_cache_class_name: org.apache.cassandra.cache.OHCProvider
 row_cache_size: 16MiB
 user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
 prepared_statements_cache_size: 1MiB
 corrupted_tombstone_strategy: exception
 stream_entire_sstables: true
diff --git a/test/data/config/version=5.0-alpha1.yml b/test/data/config/version=5.0-alpha1.yml
index 8dad0f60acc2..a700630ebb9e 100644
--- a/test/data/config/version=5.0-alpha1.yml
+++ b/test/data/config/version=5.0-alpha1.yml
@@ -143,8 +143,6 @@ networking_cache_size: "org.apache.cassandra.config.DataStorageSpec.IntMebibytes
 fields_per_udt_fail_threshold: "java.lang.Integer"
 key_cache_size: "org.apache.cassandra.config.DataStorageSpec.LongMebibytesBound"
 max_hint_window: "org.apache.cassandra.config.DurationSpec.IntMillisecondsBound"
-sai_options:
-  segment_write_buffer_size: "org.apache.cassandra.config.DataStorageSpec.IntMebibytesBound"
 vector_dimensions_fail_threshold: "java.lang.Integer"
 max_hints_size_per_host: "org.apache.cassandra.config.DataStorageSpec.LongBytesBound"
 partition_size_fail_threshold: "org.apache.cassandra.config.DataStorageSpec.LongBytesBound"
diff --git a/test/data/jmxdump/cassandra-4.0-jmx.yaml b/test/data/jmxdump/cassandra-4.0-jmx.yaml
index e0d01272c83a..9f1209ecb966 100644
--- a/test/data/jmxdump/cassandra-4.0-jmx.yaml
+++ b/test/data/jmxdump/cassandra-4.0-jmx.yaml
@@ -3359,7 +3359,6 @@ org.apache.cassandra.db:type=StorageService:
   - {access: read-only, name: Joined, type: boolean}
   - {access: read-only, name: JoiningNodes, type: java.util.List}
   - {access: read-only, name: JoiningNodesWithPort, type: java.util.List}
-  - {access: read/write, name: KeyspaceCountWarnThreshold, type: int}
   - {access: read-only, name: Keyspaces, type: java.util.List}
   - {access: read-only, name: LeavingNodes, type: java.util.List}
   - {access: read-only, name: LeavingNodesWithPort, type: java.util.List}
@@ -3401,7 +3400,6 @@ org.apache.cassandra.db:type=StorageService:
   - {access: read/write, name: SnapshotLinksPerSecond, type: long}
   - {access: read-only, name: Starting, type: boolean}
   - {access: read/write, name: StreamThroughputMbPerSec, type: int}
-  - {access: read/write, name: TableCountWarnThreshold, type: int}
   - {access: read-only, name: TokenToEndpointMap, type: java.util.Map}
   - {access: read-only, name: TokenToEndpointWithPortMap, type: java.util.Map}
   - {access: read-only, name: Tokens, type: java.util.List}
diff --git a/test/data/jmxdump/cassandra-4.1-jmx.yaml b/test/data/jmxdump/cassandra-4.1-jmx.yaml
index a5ea2a74a16b..b31f4366a5e1 100644
--- a/test/data/jmxdump/cassandra-4.1-jmx.yaml
+++ b/test/data/jmxdump/cassandra-4.1-jmx.yaml
@@ -3359,7 +3359,6 @@ org.apache.cassandra.db:type=StorageService:
   - {access: read-only, name: Joined, type: boolean}
   - {access: read-only, name: JoiningNodes, type: java.util.List}
   - {access: read-only, name: JoiningNodesWithPort, type: java.util.List}
-  - {access: read/write, name: KeyspaceCountWarnThreshold, type: int}
   - {access: read-only, name: Keyspaces, type: java.util.List}
   - {access: read-only, name: LeavingNodes, type: java.util.List}
   - {access: read-only, name: LeavingNodesWithPort, type: java.util.List}
@@ -3401,7 +3400,6 @@ org.apache.cassandra.db:type=StorageService:
   - {access: read/write, name: SnapshotLinksPerSecond, type: long}
   - {access: read-only, name: Starting, type: boolean}
   - {access: read/write, name: StreamThroughputMbPerSec, type: int}
-  - {access: read/write, name: TableCountWarnThreshold, type: int}
   - {access: read-only, name: TokenToEndpointMap, type: java.util.Map}
   - {access: read-only, name: TokenToEndpointWithPortMap, type: java.util.Map}
   - {access: read-only, name: Tokens, type: java.util.List}
diff --git a/test/data/legacy-sai/aa/bb-1-bti-CRC.db b/test/data/legacy-sai/aa/bb-1-bti-CRC.db
new file mode 100644
index 000000000000..d64361d7e03f
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-CRC.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-Data.db b/test/data/legacy-sai/aa/bb-1-bti-Data.db
new file mode 100644
index 000000000000..13734053a155
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-Data.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-Digest.crc32 b/test/data/legacy-sai/aa/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..ed002f710a8b
--- /dev/null
+++ b/test/data/legacy-sai/aa/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3882779689
\ No newline at end of file
diff --git a/test/data/legacy-sai/aa/bb-1-bti-Filter.db b/test/data/legacy-sai/aa/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ec742fc73dbd
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-Filter.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-Partitions.db b/test/data/legacy-sai/aa/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..43c824ad4fff
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-Rows.db b/test/data/legacy-sai/aa/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_GroupComplete.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_GroupComplete.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_GroupMeta.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_GroupMeta.db
new file mode 100644
index 000000000000..3a4def209bc8
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_GroupMeta.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_OffsetsValues.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_OffsetsValues.db
new file mode 100644
index 000000000000..839c13e73ec5
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_OffsetsValues.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_TokenValues.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_TokenValues.db
new file mode 100644
index 000000000000..b608f5d8efcb
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_TokenValues.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_ColumnComplete.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_ColumnComplete.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_KDTree.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_KDTree.db
new file mode 100644
index 000000000000..ca7105fcc8ba
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_KDTree.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_KDTreePostingLists.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_KDTreePostingLists.db
new file mode 100644
index 000000000000..1f6e4bd0f23e
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_KDTreePostingLists.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_Meta.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_Meta.db
new file mode 100644
index 000000000000..bb82c3fd88d0
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_int_index_Meta.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_ColumnComplete.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_ColumnComplete.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_Meta.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_Meta.db
new file mode 100644
index 000000000000..93637ad663b5
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_Meta.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_PostingLists.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_PostingLists.db
new file mode 100644
index 000000000000..aaa178868518
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_PostingLists.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_TermsData.db b/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_TermsData.db
new file mode 100644
index 000000000000..f5cda8197f06
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-SAI_text_index_TermsData.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-Statistics.db b/test/data/legacy-sai/aa/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..1e42b4385ab1
Binary files /dev/null and b/test/data/legacy-sai/aa/bb-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sai/aa/bb-1-bti-TOC.txt b/test/data/legacy-sai/aa/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..8fc8892e036e
--- /dev/null
+++ b/test/data/legacy-sai/aa/bb-1-bti-TOC.txt
@@ -0,0 +1,20 @@
+Digest.crc32
+SAI_table_0_text_value_idx_ColumnComplete.db
+SAI_table_0_int_value_idx_KDTreePostingLists.db
+SAI_table_0_text_value_idx_PostingLists.db
+SAI_table_0_int_value_idx_ColumnComplete.db
+SAI_table_0_int_value_idx_KDTree.db
+Data.db
+SAI_OffsetsValues.db
+Partitions.db
+SAI_table_0_text_value_idx_Meta.db
+SAI_table_0_int_value_idx_Meta.db
+SAI_table_0_text_value_idx_TermsData.db
+SAI_GroupComplete.db
+Statistics.db
+TOC.txt
+SAI_TokenValues.db
+Rows.db
+CRC.db
+Filter.db
+SAI_GroupMeta.db
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..dbc18f6cc256
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Data.db
new file mode 100644
index 000000000000..1f9357b1f068
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..3d9631973846
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3036065180
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b8cb5146f59d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..2cf64b034c96
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..3d552f20e788
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..db06c09bbb50
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Statistics.db
+TOC.txt
+Filter.db
+Digest.crc32
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..2e57b49d47a0
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Data.db
new file mode 100644
index 000000000000..b038dc27c272
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..4c817caeae51
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+112902994
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b8cb5146f59d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..50052c4fdc6f
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..e37caf6feba7
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..db06c09bbb50
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_compact/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Statistics.db
+TOC.txt
+Filter.db
+Digest.crc32
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..9f719378e34d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Data.db
new file mode 100644
index 000000000000..f4c625fb8992
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..5720255cb5e0
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+647001919
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b8cb5146f59d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..6b74492bf38e
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..02f7d47d3059
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..db06c09bbb50
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Statistics.db
+TOC.txt
+Filter.db
+Digest.crc32
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..1cdf45e437d7
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Data.db
new file mode 100644
index 000000000000..200f4d9e8256
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..672e8beb7691
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+400579342
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b8cb5146f59d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..6b74492bf38e
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..a8c896b1bfe5
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..db06c09bbb50
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_clust_counter_compact/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Statistics.db
+TOC.txt
+Filter.db
+Digest.crc32
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..b8449a126036
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Data.db
new file mode 100644
index 000000000000..801ff7c5dd85
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..a65a24d4b144
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2180385804
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b58e3946e230
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..c0f56d107fca
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..50bb4e272568
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..582d8fbce369
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_inaccurate_min_max/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Filter.db
+Data.db
+TOC.txt
+Statistics.db
+CompressionInfo.db
+Rows.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..fc38a25eea5d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Data.db
new file mode 100644
index 000000000000..9f7645e8e8e0
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..36c915b373f1
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+4174191692
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b8cb5146f59d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..e20b4e2f2700
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..5328b8858270
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..db06c09bbb50
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Statistics.db
+TOC.txt
+Filter.db
+Digest.crc32
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..1c738aa0288a
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Data.db
new file mode 100644
index 000000000000..95ea5e12f87a
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..314119cbea6e
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+230017823
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b8cb5146f59d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..657d5463fca8
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..c265700c200e
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..db06c09bbb50
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_compact/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Statistics.db
+TOC.txt
+Filter.db
+Digest.crc32
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..e2860e1eb16a
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Data.db
new file mode 100644
index 000000000000..be45380232b1
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..9d786a90ad7a
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3862701472
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b8cb5146f59d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..773d3c8891c3
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..036a76a00ade
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..db06c09bbb50
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Statistics.db
+TOC.txt
+Filter.db
+Digest.crc32
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-CompressionInfo.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..1237cc7f0057
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Data.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Data.db
new file mode 100644
index 000000000000..eccef889e67e
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Digest.crc32 b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Digest.crc32
new file mode 100644
index 000000000000..50f631b1a2c3
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3409102979
\ No newline at end of file
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Filter.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Filter.db
new file mode 100644
index 000000000000..b8cb5146f59d
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Partitions.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Partitions.db
new file mode 100644
index 000000000000..6c9a78cd043c
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Rows.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Statistics.db b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Statistics.db
new file mode 100644
index 000000000000..6ede946fc245
Binary files /dev/null and b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-TOC.txt b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-TOC.txt
new file mode 100644
index 000000000000..db06c09bbb50
--- /dev/null
+++ b/test/data/legacy-sstables/aa/legacy_tables/legacy_aa_simple_counter_compact/aa-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Statistics.db
+TOC.txt
+Filter.db
+Digest.crc32
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..c1a1a8ebd10d
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Data.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Data.db
new file mode 100644
index 000000000000..3d65c28a8e39
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Digest.crc32 b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Digest.crc32
new file mode 100644
index 000000000000..703e9a110763
--- /dev/null
+++ b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1685416100
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Filter.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Filter.db
new file mode 100644
index 000000000000..2e1d5d29ca06
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Partitions.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Rows.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Rows.db
new file mode 100644
index 000000000000..88f2a3b55db8
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Statistics.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Statistics.db
new file mode 100644
index 000000000000..bddde4d66453
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-TOC.txt b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-TOC.txt
new file mode 100644
index 000000000000..de43ad25cf42
--- /dev/null
+++ b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust/ac-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Statistics.db
+CompressionInfo.db
+TOC.txt
+Data.db
+Partitions.db
+Digest.crc32
+Rows.db
+Filter.db
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..f90cafcfb320
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Data.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Data.db
new file mode 100644
index 000000000000..d0438a81a233
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Digest.crc32 b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Digest.crc32
new file mode 100644
index 000000000000..dc2697987fc2
--- /dev/null
+++ b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3561445797
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Filter.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Filter.db
new file mode 100644
index 000000000000..2e1d5d29ca06
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Partitions.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Rows.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Rows.db
new file mode 100644
index 000000000000..1a324e57b52c
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Statistics.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Statistics.db
new file mode 100644
index 000000000000..838d351e6ce2
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-TOC.txt b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-TOC.txt
new file mode 100644
index 000000000000..de43ad25cf42
--- /dev/null
+++ b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_clust_counter/ac-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Statistics.db
+CompressionInfo.db
+TOC.txt
+Data.db
+Partitions.db
+Digest.crc32
+Rows.db
+Filter.db
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..fc38a25eea5d
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Data.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Data.db
new file mode 100644
index 000000000000..485ae9a782b5
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Digest.crc32 b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Digest.crc32
new file mode 100644
index 000000000000..773778b02738
--- /dev/null
+++ b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3030696842
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Filter.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Filter.db
new file mode 100644
index 000000000000..2e1d5d29ca06
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Partitions.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Partitions.db
new file mode 100644
index 000000000000..e20b4e2f2700
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Rows.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Statistics.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Statistics.db
new file mode 100644
index 000000000000..17cd637bc236
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-TOC.txt b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-TOC.txt
new file mode 100644
index 000000000000..de43ad25cf42
--- /dev/null
+++ b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple/ac-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Statistics.db
+CompressionInfo.db
+TOC.txt
+Data.db
+Partitions.db
+Digest.crc32
+Rows.db
+Filter.db
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..e2860e1eb16a
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Data.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Data.db
new file mode 100644
index 000000000000..c95bc74083d1
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Digest.crc32 b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Digest.crc32
new file mode 100644
index 000000000000..f8c73f51b6b7
--- /dev/null
+++ b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1495453984
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Filter.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Filter.db
new file mode 100644
index 000000000000..2e1d5d29ca06
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Partitions.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Partitions.db
new file mode 100644
index 000000000000..773d3c8891c3
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Rows.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Statistics.db b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Statistics.db
new file mode 100644
index 000000000000..4bc9a659aa90
Binary files /dev/null and b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-TOC.txt b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-TOC.txt
new file mode 100644
index 000000000000..de43ad25cf42
--- /dev/null
+++ b/test/data/legacy-sstables/ac/legacy_tables/legacy_ac_simple_counter/ac-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Statistics.db
+CompressionInfo.db
+TOC.txt
+Data.db
+Partitions.db
+Digest.crc32
+Rows.db
+Filter.db
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..a8a93a03c6e3
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Data.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Data.db
new file mode 100644
index 000000000000..63bb37e15c9d
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Digest.crc32 b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Digest.crc32
new file mode 100644
index 000000000000..b00168f4ba45
--- /dev/null
+++ b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1723118615
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Filter.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Filter.db
new file mode 100644
index 000000000000..2e1d5d29ca06
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Partitions.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Rows.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Rows.db
new file mode 100644
index 000000000000..88f2a3b55db8
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Statistics.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Statistics.db
new file mode 100644
index 000000000000..0662076e3897
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-TOC.txt b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-TOC.txt
new file mode 100644
index 000000000000..c1b10099fd70
--- /dev/null
+++ b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust/ad-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Rows.db
+Data.db
+TOC.txt
+CompressionInfo.db
+Digest.crc32
+Statistics.db
+Filter.db
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..aed58e0e5d50
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Data.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Data.db
new file mode 100644
index 000000000000..2fe9d5e2ed34
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Digest.crc32 b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Digest.crc32
new file mode 100644
index 000000000000..3c2551889938
--- /dev/null
+++ b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2961106595
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Filter.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Filter.db
new file mode 100644
index 000000000000..2e1d5d29ca06
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Partitions.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Rows.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Rows.db
new file mode 100644
index 000000000000..2f8e2aefce5f
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Statistics.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Statistics.db
new file mode 100644
index 000000000000..627ee8ecadb9
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-TOC.txt b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-TOC.txt
new file mode 100644
index 000000000000..c1b10099fd70
--- /dev/null
+++ b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_clust_counter/ad-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Rows.db
+Data.db
+TOC.txt
+CompressionInfo.db
+Digest.crc32
+Statistics.db
+Filter.db
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..fc38a25eea5d
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Data.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Data.db
new file mode 100644
index 000000000000..a2eff457f1c1
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Digest.crc32 b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Digest.crc32
new file mode 100644
index 000000000000..5dd842571884
--- /dev/null
+++ b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3089812609
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Filter.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Filter.db
new file mode 100644
index 000000000000..2e1d5d29ca06
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Partitions.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Partitions.db
new file mode 100644
index 000000000000..e20b4e2f2700
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Rows.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Statistics.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Statistics.db
new file mode 100644
index 000000000000..50687c4f16e9
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-TOC.txt b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-TOC.txt
new file mode 100644
index 000000000000..c1b10099fd70
--- /dev/null
+++ b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple/ad-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Rows.db
+Data.db
+TOC.txt
+CompressionInfo.db
+Digest.crc32
+Statistics.db
+Filter.db
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..e2860e1eb16a
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Data.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Data.db
new file mode 100644
index 000000000000..f2b7b5e0d297
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Digest.crc32 b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Digest.crc32
new file mode 100644
index 000000000000..a3dad7e92a66
--- /dev/null
+++ b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1039976897
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Filter.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Filter.db
new file mode 100644
index 000000000000..2e1d5d29ca06
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Partitions.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Partitions.db
new file mode 100644
index 000000000000..773d3c8891c3
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Rows.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Statistics.db b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Statistics.db
new file mode 100644
index 000000000000..474cedbc0b2f
Binary files /dev/null and b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-TOC.txt b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-TOC.txt
new file mode 100644
index 000000000000..c1b10099fd70
--- /dev/null
+++ b/test/data/legacy-sstables/ad/legacy_tables/legacy_ad_simple_counter/ad-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Rows.db
+Data.db
+TOC.txt
+CompressionInfo.db
+Digest.crc32
+Statistics.db
+Filter.db
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..7c74582095d6
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Data.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Data.db
new file mode 100644
index 000000000000..ae0e0497a3bb
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Digest.crc32 b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Digest.crc32
new file mode 100644
index 000000000000..dc10d1f6b92a
--- /dev/null
+++ b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Digest.crc32
@@ -0,0 +1 @@
+180495317
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Filter.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Partitions.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Rows.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Rows.db
new file mode 100644
index 000000000000..b500ebd8876d
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Statistics.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Statistics.db
new file mode 100644
index 000000000000..922484440442
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-TOC.txt b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-TOC.txt
new file mode 100644
index 000000000000..2b093990c9a5
--- /dev/null
+++ b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust/ba-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+CompressionInfo.db
+TOC.txt
+Digest.crc32
+Statistics.db
+Filter.db
+Rows.db
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..7b47ea847702
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Data.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Data.db
new file mode 100644
index 000000000000..593264dd21c1
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Digest.crc32 b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Digest.crc32
new file mode 100644
index 000000000000..46f41cdb9804
--- /dev/null
+++ b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3249746475
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Filter.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Partitions.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Rows.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Rows.db
new file mode 100644
index 000000000000..1a324e57b52c
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Statistics.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Statistics.db
new file mode 100644
index 000000000000..09913adde254
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-TOC.txt b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-TOC.txt
new file mode 100644
index 000000000000..2b093990c9a5
--- /dev/null
+++ b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_clust_counter/ba-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+CompressionInfo.db
+TOC.txt
+Digest.crc32
+Statistics.db
+Filter.db
+Rows.db
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..fc38a25eea5d
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Data.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Data.db
new file mode 100644
index 000000000000..0fac1ff244b6
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Digest.crc32 b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Digest.crc32
new file mode 100644
index 000000000000..40185ea83742
--- /dev/null
+++ b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1237147338
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Filter.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Partitions.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Partitions.db
new file mode 100644
index 000000000000..e20b4e2f2700
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Rows.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Statistics.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Statistics.db
new file mode 100644
index 000000000000..55d370a854f6
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-TOC.txt b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-TOC.txt
new file mode 100644
index 000000000000..2b093990c9a5
--- /dev/null
+++ b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple/ba-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+CompressionInfo.db
+TOC.txt
+Digest.crc32
+Statistics.db
+Filter.db
+Rows.db
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..e2860e1eb16a
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Data.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Data.db
new file mode 100644
index 000000000000..7b258def9d97
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Digest.crc32 b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Digest.crc32
new file mode 100644
index 000000000000..27b4432f967b
--- /dev/null
+++ b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Digest.crc32
@@ -0,0 +1 @@
+178975066
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Filter.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Partitions.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Partitions.db
new file mode 100644
index 000000000000..773d3c8891c3
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Rows.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Statistics.db b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Statistics.db
new file mode 100644
index 000000000000..fc9a7da7a404
Binary files /dev/null and b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-TOC.txt b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-TOC.txt
new file mode 100644
index 000000000000..2b093990c9a5
--- /dev/null
+++ b/test/data/legacy-sstables/ba/legacy_tables/legacy_ba_simple_counter/ba-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+CompressionInfo.db
+TOC.txt
+Digest.crc32
+Statistics.db
+Filter.db
+Rows.db
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-CompressionInfo.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..307774984c2e
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Data.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Data.db
new file mode 100644
index 000000000000..1a295113288a
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Digest.crc32 b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..a94bf9de2134
--- /dev/null
+++ b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3247259799
\ No newline at end of file
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Filter.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Partitions.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Rows.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..bf8566506058
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Statistics.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..8dcf1546d032
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-TOC.txt b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..5db770495d06
--- /dev/null
+++ b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Rows.db
+Data.db
+Partitions.db
+CompressionInfo.db
+Statistics.db
+Filter.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-CompressionInfo.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..22deb5490bef
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Data.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Data.db
new file mode 100644
index 000000000000..6045f6cda872
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Digest.crc32 b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..50cd0ea6af2d
--- /dev/null
+++ b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3625765685
\ No newline at end of file
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Filter.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Partitions.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Rows.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..6dfe025aa605
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Statistics.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..23e356f1700e
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-TOC.txt b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..5db770495d06
--- /dev/null
+++ b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_clust_counter/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Rows.db
+Data.db
+Partitions.db
+CompressionInfo.db
+Statistics.db
+Filter.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-CompressionInfo.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..dde7f70c2040
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Data.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Data.db
new file mode 100644
index 000000000000..bcb0684d4d06
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Digest.crc32 b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..48ec9c53b42f
--- /dev/null
+++ b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+82145779
\ No newline at end of file
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Filter.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Partitions.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..14cfb572e16f
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Rows.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Statistics.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..987b32a732d1
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-TOC.txt b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..5db770495d06
--- /dev/null
+++ b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Rows.db
+Data.db
+Partitions.db
+CompressionInfo.db
+Statistics.db
+Filter.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-CompressionInfo.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..fe5bd9a9faf3
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Data.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Data.db
new file mode 100644
index 000000000000..25a9967b157a
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Digest.crc32 b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..df0bd76bdd5d
--- /dev/null
+++ b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3351188643
\ No newline at end of file
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Filter.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Partitions.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..6a1e5a50993e
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Rows.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Statistics.db b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..25750b93928c
Binary files /dev/null and b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-TOC.txt b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..5db770495d06
--- /dev/null
+++ b/test/data/legacy-sstables/bb/legacy_tables/legacy_bb_simple_counter/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Rows.db
+Data.db
+Partitions.db
+CompressionInfo.db
+Statistics.db
+Filter.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..023476374d42
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Data.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Data.db
new file mode 100644
index 000000000000..dc64ae36adb7
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Digest.crc32 b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Digest.crc32
new file mode 100644
index 000000000000..2b2298994977
--- /dev/null
+++ b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2438029640
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Filter.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Partitions.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Rows.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Rows.db
new file mode 100644
index 000000000000..88f2a3b55db8
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Statistics.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Statistics.db
new file mode 100644
index 000000000000..a8ef980b891b
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-TOC.txt b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-TOC.txt
new file mode 100644
index 000000000000..8edd6bb68a27
--- /dev/null
+++ b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust/ca-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+TOC.txt
+Partitions.db
+CompressionInfo.db
+Rows.db
+Filter.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..8e305d7a1628
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Data.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Data.db
new file mode 100644
index 000000000000..666e28a7c040
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Digest.crc32 b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Digest.crc32
new file mode 100644
index 000000000000..88e7d42325b1
--- /dev/null
+++ b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Digest.crc32
@@ -0,0 +1 @@
+4022876624
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Filter.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Partitions.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Rows.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Rows.db
new file mode 100644
index 000000000000..2f8e2aefce5f
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Statistics.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Statistics.db
new file mode 100644
index 000000000000..21366868f89b
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-TOC.txt b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-TOC.txt
new file mode 100644
index 000000000000..8edd6bb68a27
--- /dev/null
+++ b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_clust_counter/ca-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+TOC.txt
+Partitions.db
+CompressionInfo.db
+Rows.db
+Filter.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..ef683177e8f6
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Data.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Data.db
new file mode 100644
index 000000000000..15a18fcb1b4b
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Digest.crc32 b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Digest.crc32
new file mode 100644
index 000000000000..43ff6368ec3a
--- /dev/null
+++ b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Digest.crc32
@@ -0,0 +1 @@
+786170984
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Filter.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Partitions.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Partitions.db
new file mode 100644
index 000000000000..e20b4e2f2700
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Rows.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Statistics.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Statistics.db
new file mode 100644
index 000000000000..b46a624817f9
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-TOC.txt b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-TOC.txt
new file mode 100644
index 000000000000..8edd6bb68a27
--- /dev/null
+++ b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple/ca-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+TOC.txt
+Partitions.db
+CompressionInfo.db
+Rows.db
+Filter.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-CompressionInfo.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..1db9aa06b311
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Data.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Data.db
new file mode 100644
index 000000000000..80aefbf1d126
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Data.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Digest.crc32 b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Digest.crc32
new file mode 100644
index 000000000000..08abf520f495
--- /dev/null
+++ b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1825820643
\ No newline at end of file
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Filter.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Partitions.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Partitions.db
new file mode 100644
index 000000000000..773d3c8891c3
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Rows.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Statistics.db b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Statistics.db
new file mode 100644
index 000000000000..23c2bd70eeb9
Binary files /dev/null and b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-TOC.txt b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-TOC.txt
new file mode 100644
index 000000000000..8edd6bb68a27
--- /dev/null
+++ b/test/data/legacy-sstables/ca/legacy_tables/legacy_ca_simple_counter/ca-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+TOC.txt
+Partitions.db
+CompressionInfo.db
+Rows.db
+Filter.db
+Digest.crc32
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-CompressionInfo.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-CompressionInfo.db
new file mode 100644
index 000000000000..8ae1d9d94938
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Data.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Data.db
new file mode 100644
index 000000000000..f439f3bfbcbc
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Data.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Digest.crc32 b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Digest.crc32
new file mode 100644
index 000000000000..716f62f5c161
--- /dev/null
+++ b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Digest.crc32
@@ -0,0 +1 @@
+4181397939
\ No newline at end of file
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Filter.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Partitions.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Rows.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Rows.db
new file mode 100644
index 000000000000..88f2a3b55db8
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Statistics.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Statistics.db
new file mode 100644
index 000000000000..8313895ee590
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-TOC.txt b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-TOC.txt
new file mode 100644
index 000000000000..b17b10a62aa3
--- /dev/null
+++ b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust/cb-3g3i_16ov_3sj4g2pdpcdz177rbj-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+TOC.txt
+Statistics.db
+CompressionInfo.db
+Digest.crc32
+Rows.db
+Data.db
+Filter.db
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-CompressionInfo.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-CompressionInfo.db
new file mode 100644
index 000000000000..be158b95993a
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Data.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Data.db
new file mode 100644
index 000000000000..f5b552593884
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Data.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Digest.crc32 b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Digest.crc32
new file mode 100644
index 000000000000..b787f6bce4be
--- /dev/null
+++ b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Digest.crc32
@@ -0,0 +1 @@
+4072062130
\ No newline at end of file
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Filter.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Partitions.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Rows.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Rows.db
new file mode 100644
index 000000000000..2f8e2aefce5f
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Statistics.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Statistics.db
new file mode 100644
index 000000000000..684fdc1717cd
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-TOC.txt b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-TOC.txt
new file mode 100644
index 000000000000..b17b10a62aa3
--- /dev/null
+++ b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_clust_counter/cb-3g3i_16ov_4w6lc2pdpcdz177rbj-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+TOC.txt
+Statistics.db
+CompressionInfo.db
+Digest.crc32
+Rows.db
+Data.db
+Filter.db
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-CompressionInfo.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-CompressionInfo.db
new file mode 100644
index 000000000000..ef683177e8f6
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Data.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Data.db
new file mode 100644
index 000000000000..b0a976e1effe
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Data.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Digest.crc32 b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Digest.crc32
new file mode 100644
index 000000000000..94439c64f860
--- /dev/null
+++ b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Digest.crc32
@@ -0,0 +1 @@
+3674319454
\ No newline at end of file
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Filter.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Partitions.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Partitions.db
new file mode 100644
index 000000000000..e20b4e2f2700
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Rows.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Statistics.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Statistics.db
new file mode 100644
index 000000000000..4cfbdd1711be
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-TOC.txt b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-TOC.txt
new file mode 100644
index 000000000000..b17b10a62aa3
--- /dev/null
+++ b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple/cb-3g3i_16ov_4qlz52pdpcdz177rbj-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+TOC.txt
+Statistics.db
+CompressionInfo.db
+Digest.crc32
+Rows.db
+Data.db
+Filter.db
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-CompressionInfo.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-CompressionInfo.db
new file mode 100644
index 000000000000..1db9aa06b311
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Data.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Data.db
new file mode 100644
index 000000000000..4575449203ad
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Data.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Digest.crc32 b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Digest.crc32
new file mode 100644
index 000000000000..5f789b2a7b9d
--- /dev/null
+++ b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Digest.crc32
@@ -0,0 +1 @@
+1753994572
\ No newline at end of file
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Filter.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Partitions.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Partitions.db
new file mode 100644
index 000000000000..773d3c8891c3
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Rows.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Statistics.db b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Statistics.db
new file mode 100644
index 000000000000..cac49df52695
Binary files /dev/null and b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-TOC.txt b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-TOC.txt
new file mode 100644
index 000000000000..b17b10a62aa3
--- /dev/null
+++ b/test/data/legacy-sstables/cb/legacy_tables/legacy_cb_simple_counter/cb-3g3i_16ov_4tea82pdpcdz177rbj-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+TOC.txt
+Statistics.db
+CompressionInfo.db
+Digest.crc32
+Rows.db
+Data.db
+Filter.db
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-CompressionInfo.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-CompressionInfo.db
new file mode 100644
index 000000000000..a8a164f96e5c
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Data.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Data.db
new file mode 100644
index 000000000000..bdf5cdf0de22
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Data.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Digest.crc32 b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Digest.crc32
new file mode 100644
index 000000000000..d45fa61b8b02
--- /dev/null
+++ b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Digest.crc32
@@ -0,0 +1 @@
+1987840382
\ No newline at end of file
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Filter.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Partitions.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Rows.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Rows.db
new file mode 100644
index 000000000000..bfca83b9cfba
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Statistics.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Statistics.db
new file mode 100644
index 000000000000..4755dddd51ac
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-TOC.txt b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-TOC.txt
new file mode 100644
index 000000000000..a480ea0c419b
--- /dev/null
+++ b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust/cc-3gdv_0vos_4v40g2l3kqtv0q1se8-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Rows.db
+Data.db
+Statistics.db
+Digest.crc32
+Filter.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-CompressionInfo.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-CompressionInfo.db
new file mode 100644
index 000000000000..599fbda0bfca
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Data.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Data.db
new file mode 100644
index 000000000000..a235e61086cb
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Data.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Digest.crc32 b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Digest.crc32
new file mode 100644
index 000000000000..dc601e76a6ce
--- /dev/null
+++ b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Digest.crc32
@@ -0,0 +1 @@
+964771694
\ No newline at end of file
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Filter.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Partitions.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Partitions.db
new file mode 100644
index 000000000000..daf1b01ec12e
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Rows.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Rows.db
new file mode 100644
index 000000000000..aab6a54a1f00
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Rows.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Statistics.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Statistics.db
new file mode 100644
index 000000000000..f866f199bf92
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-TOC.txt b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-TOC.txt
new file mode 100644
index 000000000000..a480ea0c419b
--- /dev/null
+++ b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_clust_counter/cc-3gdv_0vos_4jbmo2l3kqtv0q1se8-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Rows.db
+Data.db
+Statistics.db
+Digest.crc32
+Filter.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-CompressionInfo.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-CompressionInfo.db
new file mode 100644
index 000000000000..ef683177e8f6
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Data.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Data.db
new file mode 100644
index 000000000000..1154e2c4ea01
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Data.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Digest.crc32 b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Digest.crc32
new file mode 100644
index 000000000000..66d05061103b
--- /dev/null
+++ b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Digest.crc32
@@ -0,0 +1 @@
+169399782
\ No newline at end of file
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Filter.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Partitions.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Partitions.db
new file mode 100644
index 000000000000..e20b4e2f2700
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Rows.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Statistics.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Statistics.db
new file mode 100644
index 000000000000..4b9dadc6ad85
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-TOC.txt b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-TOC.txt
new file mode 100644
index 000000000000..a480ea0c419b
--- /dev/null
+++ b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple/cc-3gdv_0vos_48lts2l3kqtv0q1se8-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Rows.db
+Data.db
+Statistics.db
+Digest.crc32
+Filter.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-CompressionInfo.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-CompressionInfo.db
new file mode 100644
index 000000000000..1db9aa06b311
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Data.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Data.db
new file mode 100644
index 000000000000..96cf5c25ff6e
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Data.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Digest.crc32 b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Digest.crc32
new file mode 100644
index 000000000000..56280cb5e4d0
--- /dev/null
+++ b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Digest.crc32
@@ -0,0 +1 @@
+1463276668
\ No newline at end of file
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Filter.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Filter.db
new file mode 100644
index 000000000000..8868e5c18008
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Filter.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Partitions.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Partitions.db
new file mode 100644
index 000000000000..773d3c8891c3
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Partitions.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Rows.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Statistics.db b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Statistics.db
new file mode 100644
index 000000000000..3a8938eba0a4
Binary files /dev/null and b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-Statistics.db differ
diff --git a/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-TOC.txt b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-TOC.txt
new file mode 100644
index 000000000000..a480ea0c419b
--- /dev/null
+++ b/test/data/legacy-sstables/cc/legacy_tables/legacy_cc_simple_counter/cc-3gdv_0vos_3kdww2l3kqtv0q1se8-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Rows.db
+Data.db
+Statistics.db
+Digest.crc32
+Filter.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db
index 8fad34fe9e11..ab804238abcb 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db
index ae35335fbf9a..4cddfdb98d08 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
index 8a92f3c58325..83138ff5a6b7 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
@@ -1 +1 @@
-2977407251
\ No newline at end of file
+3857770523
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db
index d50fdeb4e209..aeeff9304559 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db
index 734186497e79..c9506ee78c2c 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
index b03b28372b5d..8a6a30b6db77 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
@@ -1,8 +1,8 @@
-Filter.db
-Digest.crc32
-Index.db
-TOC.txt
 Summary.db
-Statistics.db
-CompressionInfo.db
 Data.db
+TOC.txt
+CompressionInfo.db
+Statistics.db
+Digest.crc32
+Index.db
+Filter.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db
index f0a1cfb59e84..d9592e6c848b 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db
index b487fe88edf6..e21851ee0823 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
index ca286e0954b9..a85dfe8c5274 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
@@ -1 +1 @@
-2759187708
\ No newline at end of file
+2266872816
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db
index c981a226039e..0e8dc66e9f2d 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db
index 33fccc9c84bd..a14bd80fa4a1 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
index b03b28372b5d..8a6a30b6db77 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
@@ -1,8 +1,8 @@
-Filter.db
-Digest.crc32
-Index.db
-TOC.txt
 Summary.db
-Statistics.db
-CompressionInfo.db
 Data.db
+TOC.txt
+CompressionInfo.db
+Statistics.db
+Digest.crc32
+Index.db
+Filter.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db
index fc38a25eea5d..ef683177e8f6 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db
index 11219d037a8d..d0ce6c3af9bd 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
index 985d6dcf36d6..2e84cd9a71a0 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
@@ -1 +1 @@
-462858821
\ No newline at end of file
+739757235
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db
index 3c68ac568f56..daf1c16c63ee 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
index b03b28372b5d..8a6a30b6db77 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
@@ -1,8 +1,8 @@
-Filter.db
-Digest.crc32
-Index.db
-TOC.txt
 Summary.db
-Statistics.db
-CompressionInfo.db
 Data.db
+TOC.txt
+CompressionInfo.db
+Statistics.db
+Digest.crc32
+Index.db
+Filter.db
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db
index e2860e1eb16a..1db9aa06b311 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db
index 620cdf260e5a..11c0a684f10e 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
index bc5f671e0191..b905530e1d8c 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
@@ -1 +1 @@
-3987542254
\ No newline at end of file
+3918697890
\ No newline at end of file
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db
index 689bec8f1a85..c803970afe4c 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db differ
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
index b03b28372b5d..8a6a30b6db77 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
@@ -1,8 +1,8 @@
-Filter.db
-Digest.crc32
-Index.db
-TOC.txt
 Summary.db
-Statistics.db
-CompressionInfo.db
 Data.db
+TOC.txt
+CompressionInfo.db
+Statistics.db
+Digest.crc32
+Index.db
+Filter.db
diff --git a/test/data/serialization/DSE_68/batch.bin b/test/data/serialization/DSE_68/batch.bin
new file mode 100644
index 000000000000..1e82e37f1674
Binary files /dev/null and b/test/data/serialization/DSE_68/batch.bin differ
diff --git a/test/data/types-compatibility/cc-4.0.json.gz b/test/data/types-compatibility/cc-4.0.json.gz
new file mode 100644
index 000000000000..24778abf7576
Binary files /dev/null and b/test/data/types-compatibility/cc-4.0.json.gz differ
diff --git a/test/data/types-compatibility/cc-5.0.json.gz b/test/data/types-compatibility/cc-5.0.json.gz
new file mode 100644
index 000000000000..1c765e1fe0e7
Binary files /dev/null and b/test/data/types-compatibility/cc-5.0.json.gz differ
diff --git a/test/data/types-compatibility/dse-6.8-cndb.json.gz b/test/data/types-compatibility/dse-6.8-cndb.json.gz
new file mode 100644
index 000000000000..a48dae61a300
Binary files /dev/null and b/test/data/types-compatibility/dse-6.8-cndb.json.gz differ
diff --git a/test/data/types-compatibility/legacy-cc-4.0.json.gz b/test/data/types-compatibility/legacy-cc-4.0.json.gz
new file mode 100644
index 000000000000..f803e415cf72
Binary files /dev/null and b/test/data/types-compatibility/legacy-cc-4.0.json.gz differ
diff --git a/test/data/udt/cc40/commitlog/CommitLog-100-1717503185227.log b/test/data/udt/cc40/commitlog/CommitLog-100-1717503185227.log
new file mode 100644
index 000000000000..3d81721455e9
Binary files /dev/null and b/test/data/udt/cc40/commitlog/CommitLog-100-1717503185227.log differ
diff --git a/test/data/udt/cc40/data.json b/test/data/udt/cc40/data.json
new file mode 100644
index 000000000000..d2b25410672c
--- /dev/null
+++ b/test/data/udt/cc40/data.json
@@ -0,0 +1,2477 @@
+{
+  "tab5_tuple": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, 32],
+    [17, 17, 34],
+    [18, null, 36],
+    [19, 19, 38],
+    [20, null, 40],
+    [21, 21, 42],
+    [22, null, 44],
+    [23, 23, 46],
+    [24, null, 48],
+    [25, 25, 50],
+    [26, null, 52],
+    [27, 27, 54],
+    [28, null, 56],
+    [29, 29, 58],
+    [30, null, 60],
+    [31, 31, 62],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, 32],
+    [145, 17, 34],
+    [146, null, 32],
+    [147, 17, 34],
+    [148, null, 32],
+    [149, 17, 34],
+    [150, null, 32],
+    [151, 17, 34],
+    [152, null, 32],
+    [153, 17, 34],
+    [154, null, 32],
+    [155, 17, 34],
+    [156, null, 32],
+    [157, 17, 34],
+    [158, null, 32],
+    [159, 17, 34],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, 32],
+    [273, 17, 34],
+    [274, 1, 36],
+    [275, 19, 38],
+    [276, 1, 40],
+    [277, 21, 42],
+    [278, 1, 44],
+    [279, 23, 46],
+    [280, 1, 48],
+    [281, 25, 50],
+    [282, 1, 52],
+    [283, 27, 54],
+    [284, 1, 56],
+    [285, 29, 58],
+    [286, 1, 60],
+    [287, 31, 62],
+    [288, 1, null],
+    [289, null, 32],
+    [290, 17, 34],
+    [291, null, 32],
+    [292, 17, 34],
+    [293, null, 32],
+    [294, 17, 34],
+    [295, null, 32],
+    [296, 17, 34],
+    [297, null, 32],
+    [298, 17, 34],
+    [299, null, 32],
+    [300, 17, 34],
+    [301, null, 32],
+    [302, 17, 34],
+    [303, null, 32],
+    [304, 17, 34]
+  ],
+  "tab8_frozen_tuple_with_udt": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ],
+  "tab6_frozen_tuple": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, 32],
+    [17, 17, 34],
+    [18, null, 36],
+    [19, 19, 38],
+    [20, null, 40],
+    [21, 21, 42],
+    [22, null, 44],
+    [23, 23, 46],
+    [24, null, 48],
+    [25, 25, 50],
+    [26, null, 52],
+    [27, 27, 54],
+    [28, null, 56],
+    [29, 29, 58],
+    [30, null, 60],
+    [31, 31, 62],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, 32],
+    [145, 17, 34],
+    [146, null, 32],
+    [147, 17, 34],
+    [148, null, 32],
+    [149, 17, 34],
+    [150, null, 32],
+    [151, 17, 34],
+    [152, null, 32],
+    [153, 17, 34],
+    [154, null, 32],
+    [155, 17, 34],
+    [156, null, 32],
+    [157, 17, 34],
+    [158, null, 32],
+    [159, 17, 34],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, 32],
+    [273, 17, 34],
+    [274, 1, 36],
+    [275, 19, 38],
+    [276, 1, 40],
+    [277, 21, 42],
+    [278, 1, 44],
+    [279, 23, 46],
+    [280, 1, 48],
+    [281, 25, 50],
+    [282, 1, 52],
+    [283, 27, 54],
+    [284, 1, 56],
+    [285, 29, 58],
+    [286, 1, 60],
+    [287, 31, 62],
+    [288, 1, null],
+    [289, null, 32],
+    [290, 17, 34],
+    [291, null, 32],
+    [292, 17, 34],
+    [293, null, 32],
+    [294, 17, 34],
+    [295, null, 32],
+    [296, 17, 34],
+    [297, null, 32],
+    [298, 17, 34],
+    [299, null, 32],
+    [300, 17, 34],
+    [301, null, 32],
+    [302, 17, 34],
+    [303, null, 32],
+    [304, 17, 34]
+  ],
+  "tab2_frozen_udt1": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, 32],
+    [17, 17, 34],
+    [18, null, 36],
+    [19, 19, 38],
+    [20, null, 40],
+    [21, 21, 42],
+    [22, null, 44],
+    [23, 23, 46],
+    [24, null, 48],
+    [25, 25, 50],
+    [26, null, 52],
+    [27, 27, 54],
+    [28, null, 56],
+    [29, 29, 58],
+    [30, null, 60],
+    [31, 31, 62],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, 32],
+    [145, 17, 34],
+    [146, null, 32],
+    [147, 17, 34],
+    [148, null, 32],
+    [149, 17, 34],
+    [150, null, 32],
+    [151, 17, 34],
+    [152, null, 32],
+    [153, 17, 34],
+    [154, null, 32],
+    [155, 17, 34],
+    [156, null, 32],
+    [157, 17, 34],
+    [158, null, 32],
+    [159, 17, 34],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, 32],
+    [273, 17, 34],
+    [274, 1, 36],
+    [275, 19, 38],
+    [276, 1, 40],
+    [277, 21, 42],
+    [278, 1, 44],
+    [279, 23, 46],
+    [280, 1, 48],
+    [281, 25, 50],
+    [282, 1, 52],
+    [283, 27, 54],
+    [284, 1, 56],
+    [285, 29, 58],
+    [286, 1, 60],
+    [287, 31, 62],
+    [288, 1, null],
+    [289, null, 32],
+    [290, 17, 34],
+    [291, null, 32],
+    [292, 17, 34],
+    [293, null, 32],
+    [294, 17, 34],
+    [295, null, 32],
+    [296, 17, 34],
+    [297, null, 32],
+    [298, 17, 34],
+    [299, null, 32],
+    [300, 17, 34],
+    [301, null, 32],
+    [302, 17, 34],
+    [303, null, 32],
+    [304, 17, 34]
+  ],
+  "tab9_udt_with_tuple": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ],
+  "tab10_frozen_udt_with_tuple": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ],
+  "tab1_udt1": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, 32],
+    [17, 17, 34],
+    [18, null, 36],
+    [19, 19, 38],
+    [20, null, 40],
+    [21, 21, 42],
+    [22, null, 44],
+    [23, 23, 46],
+    [24, null, 48],
+    [25, 25, 50],
+    [26, null, 52],
+    [27, 27, 54],
+    [28, null, 56],
+    [29, 29, 58],
+    [30, null, 60],
+    [31, 31, 62],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, 32],
+    [145, 17, 34],
+    [146, null, 32],
+    [147, 17, 34],
+    [148, null, 32],
+    [149, 17, 34],
+    [150, null, 32],
+    [151, 17, 34],
+    [152, null, 32],
+    [153, 17, 34],
+    [154, null, 32],
+    [155, 17, 34],
+    [156, null, 32],
+    [157, 17, 34],
+    [158, null, 32],
+    [159, 17, 34],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, 32],
+    [273, 17, 34],
+    [274, 1, 36],
+    [275, 19, 38],
+    [276, 1, 40],
+    [277, 21, 42],
+    [278, 1, 44],
+    [279, 23, 46],
+    [280, 1, 48],
+    [281, 25, 50],
+    [282, 1, 52],
+    [283, 27, 54],
+    [284, 1, 56],
+    [285, 29, 58],
+    [286, 1, 60],
+    [287, 31, 62],
+    [288, 1, null],
+    [289, null, 32],
+    [290, 17, 34],
+    [291, null, 32],
+    [292, 17, 34],
+    [293, null, 32],
+    [294, 17, 34],
+    [295, null, 32],
+    [296, 17, 34],
+    [297, null, 32],
+    [298, 17, 34],
+    [299, null, 32],
+    [300, 17, 34],
+    [301, null, 32],
+    [302, 17, 34],
+    [303, null, 32],
+    [304, 17, 34]
+  ],
+  "tab4_frozen_udt2": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ],
+  "tab7_tuple_with_udt": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ]
+}
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-CompressionInfo.db
new file mode 100644
index 000000000000..00de5722ab5d
Binary files /dev/null and b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Data.db b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Data.db
new file mode 100644
index 000000000000..eaf77d15ab27
Binary files /dev/null and b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Digest.crc32
new file mode 100644
index 000000000000..b293110ce460
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Digest.crc32
@@ -0,0 +1 @@
+2990636004
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Filter.db b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Filter.db
new file mode 100644
index 000000000000..16483766eb9c
Binary files /dev/null and b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Partitions.db b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Partitions.db
new file mode 100644
index 000000000000..f7c3cae5f1a3
Binary files /dev/null and b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Rows.db b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Statistics.db b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Statistics.db
new file mode 100644
index 000000000000..a086b450d500
Binary files /dev/null and b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-Statistics.db differ
diff --git a/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-TOC.txt b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-3-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-CompressionInfo.db
new file mode 100644
index 000000000000..2e009a9881e5
Binary files /dev/null and b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Data.db b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Data.db
new file mode 100644
index 000000000000..5e695096dc67
Binary files /dev/null and b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Digest.crc32
new file mode 100644
index 000000000000..2ccf96d0dd85
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Digest.crc32
@@ -0,0 +1 @@
+3410272921
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Filter.db b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Filter.db
new file mode 100644
index 000000000000..87b3d6e6625c
Binary files /dev/null and b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Partitions.db b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Partitions.db
new file mode 100644
index 000000000000..ad334dfc1204
Binary files /dev/null and b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Rows.db b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Statistics.db b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Statistics.db
new file mode 100644
index 000000000000..823d9985a1f1
Binary files /dev/null and b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Statistics.db differ
diff --git a/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-TOC.txt b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..597c78abdcc4
Binary files /dev/null and b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Data.db b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Data.db
new file mode 100644
index 000000000000..4fc53721ed51
Binary files /dev/null and b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Digest.crc32
new file mode 100644
index 000000000000..2fd8e0693bad
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1869215403
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Filter.db b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Filter.db
new file mode 100644
index 000000000000..c32ee97affe3
Binary files /dev/null and b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Partitions.db b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Partitions.db
new file mode 100644
index 000000000000..8365fb958008
Binary files /dev/null and b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Rows.db b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Statistics.db b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Statistics.db
new file mode 100644
index 000000000000..89dfc540ad71
Binary files /dev/null and b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-Statistics.db differ
diff --git a/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-TOC.txt b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-CompressionInfo.db
new file mode 100644
index 000000000000..00de5722ab5d
Binary files /dev/null and b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Data.db b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Data.db
new file mode 100644
index 000000000000..bc5b8624a4e7
Binary files /dev/null and b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Digest.crc32
new file mode 100644
index 000000000000..ecdf21cc2496
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Digest.crc32
@@ -0,0 +1 @@
+1272423014
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Filter.db b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Filter.db
new file mode 100644
index 000000000000..16483766eb9c
Binary files /dev/null and b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Partitions.db b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Partitions.db
new file mode 100644
index 000000000000..f7c3cae5f1a3
Binary files /dev/null and b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Rows.db b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Statistics.db b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Statistics.db
new file mode 100644
index 000000000000..19014104b601
Binary files /dev/null and b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-Statistics.db differ
diff --git a/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-TOC.txt b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-3-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..597c78abdcc4
Binary files /dev/null and b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Data.db b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Data.db
new file mode 100644
index 000000000000..67591efa677b
Binary files /dev/null and b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Digest.crc32
new file mode 100644
index 000000000000..7fd54e8b8cbb
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3373806915
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Filter.db b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Filter.db
new file mode 100644
index 000000000000..c32ee97affe3
Binary files /dev/null and b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Partitions.db b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Partitions.db
new file mode 100644
index 000000000000..8365fb958008
Binary files /dev/null and b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Rows.db b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-TOC.txt b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..597c78abdcc4
Binary files /dev/null and b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Data.db b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Data.db
new file mode 100644
index 000000000000..7c8b70a68d6c
Binary files /dev/null and b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Digest.crc32
new file mode 100644
index 000000000000..605b01f079d6
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2313907068
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Filter.db b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Filter.db
new file mode 100644
index 000000000000..c32ee97affe3
Binary files /dev/null and b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Partitions.db b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Partitions.db
new file mode 100644
index 000000000000..8365fb958008
Binary files /dev/null and b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Rows.db b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-TOC.txt b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-CompressionInfo.db
new file mode 100644
index 000000000000..749bf370de1c
Binary files /dev/null and b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Data.db b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Data.db
new file mode 100644
index 000000000000..a3fb012e7071
Binary files /dev/null and b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Digest.crc32
new file mode 100644
index 000000000000..bd7eb3ea91ba
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Digest.crc32
@@ -0,0 +1 @@
+1571595259
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Filter.db b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Filter.db
new file mode 100644
index 000000000000..7348d46cacba
Binary files /dev/null and b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Partitions.db b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Partitions.db
new file mode 100644
index 000000000000..c9c85e8a2d07
Binary files /dev/null and b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Rows.db b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Statistics.db b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Statistics.db
new file mode 100644
index 000000000000..2d478d28db21
Binary files /dev/null and b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-Statistics.db differ
diff --git a/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-TOC.txt b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-2-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-CompressionInfo.db
new file mode 100644
index 000000000000..c26eb564641f
Binary files /dev/null and b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Data.db b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Data.db
new file mode 100644
index 000000000000..076f521a866a
Binary files /dev/null and b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Digest.crc32
new file mode 100644
index 000000000000..750b3ff2ed5e
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Digest.crc32
@@ -0,0 +1 @@
+836554447
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Filter.db b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Filter.db
new file mode 100644
index 000000000000..7348d46cacba
Binary files /dev/null and b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Partitions.db b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Partitions.db
new file mode 100644
index 000000000000..b6261320374e
Binary files /dev/null and b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Rows.db b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Statistics.db b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Statistics.db
new file mode 100644
index 000000000000..5d38bd35f8d5
Binary files /dev/null and b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-Statistics.db differ
diff --git a/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-TOC.txt b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-2-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-CompressionInfo.db b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..3740ade07054
Binary files /dev/null and b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Data.db b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Data.db
new file mode 100644
index 000000000000..c9ee5dbacf7c
Binary files /dev/null and b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Data.db differ
diff --git a/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Digest.crc32 b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Digest.crc32
new file mode 100644
index 000000000000..188b87180ae2
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2125162301
\ No newline at end of file
diff --git a/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Filter.db b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Filter.db
new file mode 100644
index 000000000000..f588219fa5f7
Binary files /dev/null and b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Filter.db differ
diff --git a/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Partitions.db b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Partitions.db
new file mode 100644
index 000000000000..2469f5e11569
Binary files /dev/null and b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Partitions.db differ
diff --git a/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Rows.db b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Statistics.db b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Statistics.db
new file mode 100644
index 000000000000..3ebfedbf44f9
Binary files /dev/null and b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-Statistics.db differ
diff --git a/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-TOC.txt b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-TOC.txt
new file mode 100644
index 000000000000..15654acb6450
--- /dev/null
+++ b/test/data/udt/cc40/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Rows.db
+Data.db
+Digest.crc32
+Filter.db
+Partitions.db
+Statistics.db
+TOC.txt
+CompressionInfo.db
diff --git a/test/data/udt/cc40/schema.txt b/test/data/udt/cc40/schema.txt
new file mode 100644
index 000000000000..d7f908f3ac94
--- /dev/null
+++ b/test/data/udt/cc40/schema.txt
@@ -0,0 +1,69 @@
+CREATE TYPE ks.udt1 (
+    foo int,
+    bar text,
+    baz int
+);
+CREATE TYPE ks.udt2 (
+    foo int,
+    bar udt1,
+    baz int
+);
+CREATE TYPE ks.udt3 (
+    foo int,
+    bar tuple<int, text, int>,
+    baz int
+);
+CREATE TABLE ks.tab10_frozen_udt_with_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 6a5cff4e-2f94-4c8b-9aa2-0fbd65292caa
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1717503192249000;
+CREATE TABLE ks.tab1_udt1 (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 513f2627-9356-41c4-a379-7ad42be97432
+    AND DROPPED COLUMN RECORD b_complex tuple<int, text, int> USING TIMESTAMP 1717503190617000;
+CREATE TABLE ks.tab2_frozen_udt1 (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 450f91fe-7c47-41c9-97bf-fdad854fa7e5
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1717503190798000;
+CREATE TABLE ks.tab4_frozen_udt2 (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 9c03c71c-6775-4357-9173-0f8808901afa
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1717503190972000;
+CREATE TABLE ks.tab5_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 90826dd3-8437-4585-9de4-15908236687f
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1717503191219000;
+CREATE TABLE ks.tab6_frozen_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 54185f9a-a6fd-487c-abc3-c01bd5835e48
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1717503191428000;
+CREATE TABLE ks.tab7_tuple_with_udt (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 4e78f403-7b63-4e0d-a231-42e42cba7cb5
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1717503191601000;
+CREATE TABLE ks.tab8_frozen_tuple_with_udt (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 8660f235-0816-4019-9cc9-1798fa7beb17
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1717503191868000;
+CREATE TABLE ks.tab9_udt_with_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = f670fd5a-8145-4669-aceb-75667c000ea6
+    AND DROPPED COLUMN RECORD b_complex tuple<int, frozen<tuple<int, text, int>>, int> USING TIMESTAMP 1717503192068000
\ No newline at end of file
diff --git a/test/data/udt/cc40/schema0.txt b/test/data/udt/cc40/schema0.txt
new file mode 100644
index 000000000000..e2eaba40a8fe
--- /dev/null
+++ b/test/data/udt/cc40/schema0.txt
@@ -0,0 +1,69 @@
+CREATE TYPE ks.udt1 (
+    foo int,
+    bar text,
+    baz int
+);
+CREATE TYPE ks.udt2 (
+    foo int,
+    bar udt1,
+    baz int
+);
+CREATE TYPE ks.udt3 (
+    foo int,
+    bar tuple<int, text, int>,
+    baz int
+);
+CREATE TABLE ks.tab10_frozen_udt_with_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex frozen<udt3>,
+    c_int int
+) WITH ID = 6a5cff4e-2f94-4c8b-9aa2-0fbd65292caa;
+CREATE TABLE ks.tab1_udt1 (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int,
+    b_complex udt1
+) WITH ID = 513f2627-9356-41c4-a379-7ad42be97432;
+CREATE TABLE ks.tab2_frozen_udt1 (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex frozen<udt1>,
+    c_int int
+) WITH ID = 450f91fe-7c47-41c9-97bf-fdad854fa7e5;
+CREATE TABLE ks.tab4_frozen_udt2 (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex frozen<udt2>,
+    c_int int
+) WITH ID = 9c03c71c-6775-4357-9173-0f8808901afa;
+CREATE TABLE ks.tab5_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex tuple<int, text, int>,
+    c_int int
+) WITH ID = 90826dd3-8437-4585-9de4-15908236687f;
+CREATE TABLE ks.tab6_frozen_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex tuple<int, text, int>,
+    c_int int
+) WITH ID = 54185f9a-a6fd-487c-abc3-c01bd5835e48;
+CREATE TABLE ks.tab7_tuple_with_udt (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex tuple<int, udt1, int>,
+    c_int int
+) WITH ID = 4e78f403-7b63-4e0d-a231-42e42cba7cb5;
+CREATE TABLE ks.tab8_frozen_tuple_with_udt (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex tuple<int, udt1, int>,
+    c_int int
+) WITH ID = 8660f235-0816-4019-9cc9-1798fa7beb17;
+CREATE TABLE ks.tab9_udt_with_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int,
+    b_complex udt3
+) WITH ID = f670fd5a-8145-4669-aceb-75667c000ea6
\ No newline at end of file
diff --git a/test/data/udt/cc50/commitlog/CommitLog-110-1722612901950.log b/test/data/udt/cc50/commitlog/CommitLog-110-1722612901950.log
new file mode 100644
index 000000000000..d345fc71ce7d
Binary files /dev/null and b/test/data/udt/cc50/commitlog/CommitLog-110-1722612901950.log differ
diff --git a/test/data/udt/cc50/data.json b/test/data/udt/cc50/data.json
new file mode 100644
index 000000000000..5b6cebdebb4e
--- /dev/null
+++ b/test/data/udt/cc50/data.json
@@ -0,0 +1 @@
+{"tab5_tuple":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,32],[17,17,34],[18,null,36],[19,19,38],[20,null,40],[21,21,42],[22,null,44],[23,23,46],[24,null,48],[25,25,50],[26,null,52],[27,27,54],[28,null,56],[29,29,58],[30,null,60],[31,31,62],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,32],[145,17,34],[146,null,32],[147,17,34],[148,null,32],[149,17,34],[150,null,32],[151,17,34],[152,null,32],[153,17,34],[154,null,32],[155,17,34],[156,null,32],[157,17,34],[158,null,32],[159,17,34],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,32],[273,17,34],[274,1,36],[275,19,38],[276,1,40],[277,21,42],[278,1,44],[279,23,46],[280,1,48],[281,25,50],[282,1,52],[283,27,54],[284,1,56],[285,29,58],[286,1,60],[287,31,62],[288,1,null],[289,null,32],[290,17,34],[291,null,32],[292,17,34],[293,null,32],[294,17,34],[295,null,32],[296,17,34],[297,null,32],[298,17,34],[299,null,32],[300,17,34],[301,null,32],[302,17,34],[303,null,32],[304,17,34]],"tab8_frozen_tuple_with_udt":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,null],[17,17,null],[18,null,null],[19,19,null],[20,null,null],[21,21,null],[22,null,null],[23,23,null],[24,null,null],[25,25,null],[26,null,null],[27,27,null],[28,null,null],[29,29,null],[30,null,null],[31,31,null],[32,null,null],[33,33,null],[34,null,null],[35,35,null],[36,null,null],[37,37,null],[38,null,null],[39,39,null],[40,null,null],[41,41,null],[42,null,null],[43,43,null],[44,null,null],[45,45,null],[46,null,null],[47,47,null],[48,null,null],[49,49,null],[50,null,null],[51,51,null],[52,null,null],[53,53,null],[54,null,null],[55,55,null],[56,null,null],[57,57,null],[58,null,null],[59,59,null],[60,null,null],[61,61,null],[62,null,null],[63,63,null],[64,null,128],[65,65,130],[66,null,132],[67,67,134],[68,null,136],[69,69,138],[70,null,140],[71,71,142],[72,null,144],[73,73,146],[74,null,148],[75,75,150],[76,null,152],[77,77,154],[78,null,156],[79,79,158],[80,null,160],[81,81,162],[82,null,164],[83,83,166],[84,null,168],[85,85,170],[86,null,172],[87,87,174],[88,null,176],[89,89,178],[90,null,180],[91,91,182],[92,null,184],[93,93,186],[94,null,188],[95,95,190],[96,null,192],[97,97,194],[98,null,196],[99,99,198],[100,null,200],[101,101,202],[102,null,204],[103,103,206],[104,null,208],[105,105,210],[106,null,212],[107,107,214],[108,null,216],[109,109,218],[110,null,220],[111,111,222],[112,null,224],[113,113,226],[114,null,228],[115,115,230],[116,null,232],[117,117,234],[118,null,236],[119,119,238],[120,null,240],[121,121,242],[122,null,244],[123,123,246],[124,null,248],[125,125,250],[126,null,252],[127,127,254],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,null],[145,1,null],[146,null,null],[147,1,null],[148,null,null],[149,1,null],[150,null,null],[151,1,null],[152,null,null],[153,1,null],[154,null,null],[155,1,null],[156,null,null],[157,1,null],[158,null,null],[159,1,null],[160,null,null],[161,1,null],[162,null,null],[163,1,null],[164,null,null],[165,1,null],[166,null,null],[167,1,null],[168,null,null],[169,1,null],[170,null,null],[171,1,null],[172,null,null],[173,1,null],[174,null,null],[175,1,null],[176,null,null],[177,1,null],[178,null,null],[179,1,null],[180,null,null],[181,1,null],[182,null,null],[183,1,null],[184,null,null],[185,1,null],[186,null,null],[187,1,null],[188,null,null],[189,1,null],[190,null,null],[191,1,null],[192,null,128],[193,65,130],[194,null,128],[195,65,130],[196,null,128],[197,65,130],[198,null,128],[199,65,130],[200,null,128],[201,65,130],[202,null,128],[203,65,130],[204,null,128],[205,65,130],[206,null,128],[207,65,130],[208,null,128],[209,65,130],[210,null,128],[211,65,130],[212,null,128],[213,65,130],[214,null,128],[215,65,130],[216,null,128],[217,65,130],[218,null,128],[219,65,130],[220,null,128],[221,65,130],[222,null,128],[223,65,130],[224,null,128],[225,65,130],[226,null,128],[227,65,130],[228,null,128],[229,65,130],[230,null,128],[231,65,130],[232,null,128],[233,65,130],[234,null,128],[235,65,130],[236,null,128],[237,65,130],[238,null,128],[239,65,130],[240,null,128],[241,65,130],[242,null,128],[243,65,130],[244,null,128],[245,65,130],[246,null,128],[247,65,130],[248,null,128],[249,65,130],[250,null,128],[251,65,130],[252,null,128],[253,65,130],[254,null,128],[255,65,130],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,null],[273,17,null],[274,1,null],[275,19,null],[276,1,null],[277,21,null],[278,1,null],[279,23,null],[280,1,null],[281,25,null],[282,1,null],[283,27,null],[284,1,null],[285,29,null],[286,1,null],[287,31,null],[288,1,null],[289,33,null],[290,1,null],[291,35,null],[292,1,null],[293,37,null],[294,1,null],[295,39,null],[296,1,null],[297,41,null],[298,1,null],[299,43,null],[300,1,null],[301,45,null],[302,1,null],[303,47,null],[304,1,null],[305,49,null],[306,1,null],[307,51,null],[308,1,null],[309,53,null],[310,1,null],[311,55,null],[312,1,null],[313,57,null],[314,1,null],[315,59,null],[316,1,null],[317,61,null],[318,1,null],[319,63,null],[320,1,128],[321,65,130],[322,1,132],[323,67,134],[324,1,136],[325,69,138],[326,1,140],[327,71,142],[328,1,144],[329,73,146],[330,1,148],[331,75,150],[332,1,152],[333,77,154],[334,1,156],[335,79,158],[336,1,160],[337,81,128],[338,65,130],[339,83,128],[340,65,130],[341,85,128],[342,65,130],[343,87,128],[344,65,130],[345,89,128],[346,65,130],[347,91,128],[348,65,130],[349,93,128],[350,65,130],[351,95,128],[352,65,130],[353,97,128],[354,65,130],[355,99,128],[356,65,130],[357,101,128],[358,65,130],[359,103,128],[360,65,130],[361,105,128],[362,65,130],[363,107,128],[364,65,130],[365,109,128],[366,65,130],[367,111,128],[368,65,130],[369,113,128],[370,65,130],[371,115,128],[372,65,130],[373,117,128],[374,65,130],[375,119,128],[376,65,130],[377,121,128],[378,65,130],[379,123,128],[380,65,130],[381,125,128],[382,65,130],[383,127,128],[384,65,130],[385,null,128],[386,65,130],[387,null,128],[388,65,130],[389,null,128],[390,65,130],[391,null,128],[392,65,130],[393,null,128],[394,65,130],[395,null,128],[396,65,130],[397,null,128],[398,65,130],[399,null,128],[400,65,130]],"tab6_frozen_tuple":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,32],[17,17,34],[18,null,36],[19,19,38],[20,null,40],[21,21,42],[22,null,44],[23,23,46],[24,null,48],[25,25,50],[26,null,52],[27,27,54],[28,null,56],[29,29,58],[30,null,60],[31,31,62],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,32],[145,17,34],[146,null,32],[147,17,34],[148,null,32],[149,17,34],[150,null,32],[151,17,34],[152,null,32],[153,17,34],[154,null,32],[155,17,34],[156,null,32],[157,17,34],[158,null,32],[159,17,34],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,32],[273,17,34],[274,1,36],[275,19,38],[276,1,40],[277,21,42],[278,1,44],[279,23,46],[280,1,48],[281,25,50],[282,1,52],[283,27,54],[284,1,56],[285,29,58],[286,1,60],[287,31,62],[288,1,null],[289,null,32],[290,17,34],[291,null,32],[292,17,34],[293,null,32],[294,17,34],[295,null,32],[296,17,34],[297,null,32],[298,17,34],[299,null,32],[300,17,34],[301,null,32],[302,17,34],[303,null,32],[304,17,34]],"tab2_frozen_udt1":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,32],[17,17,34],[18,null,36],[19,19,38],[20,null,40],[21,21,42],[22,null,44],[23,23,46],[24,null,48],[25,25,50],[26,null,52],[27,27,54],[28,null,56],[29,29,58],[30,null,60],[31,31,62],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,32],[145,17,34],[146,null,32],[147,17,34],[148,null,32],[149,17,34],[150,null,32],[151,17,34],[152,null,32],[153,17,34],[154,null,32],[155,17,34],[156,null,32],[157,17,34],[158,null,32],[159,17,34],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,32],[273,17,34],[274,1,36],[275,19,38],[276,1,40],[277,21,42],[278,1,44],[279,23,46],[280,1,48],[281,25,50],[282,1,52],[283,27,54],[284,1,56],[285,29,58],[286,1,60],[287,31,62],[288,1,null],[289,null,32],[290,17,34],[291,null,32],[292,17,34],[293,null,32],[294,17,34],[295,null,32],[296,17,34],[297,null,32],[298,17,34],[299,null,32],[300,17,34],[301,null,32],[302,17,34],[303,null,32],[304,17,34]],"tab9_udt_with_tuple":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,null],[17,17,null],[18,null,null],[19,19,null],[20,null,null],[21,21,null],[22,null,null],[23,23,null],[24,null,null],[25,25,null],[26,null,null],[27,27,null],[28,null,null],[29,29,null],[30,null,null],[31,31,null],[32,null,null],[33,33,null],[34,null,null],[35,35,null],[36,null,null],[37,37,null],[38,null,null],[39,39,null],[40,null,null],[41,41,null],[42,null,null],[43,43,null],[44,null,null],[45,45,null],[46,null,null],[47,47,null],[48,null,null],[49,49,null],[50,null,null],[51,51,null],[52,null,null],[53,53,null],[54,null,null],[55,55,null],[56,null,null],[57,57,null],[58,null,null],[59,59,null],[60,null,null],[61,61,null],[62,null,null],[63,63,null],[64,null,128],[65,65,130],[66,null,132],[67,67,134],[68,null,136],[69,69,138],[70,null,140],[71,71,142],[72,null,144],[73,73,146],[74,null,148],[75,75,150],[76,null,152],[77,77,154],[78,null,156],[79,79,158],[80,null,160],[81,81,162],[82,null,164],[83,83,166],[84,null,168],[85,85,170],[86,null,172],[87,87,174],[88,null,176],[89,89,178],[90,null,180],[91,91,182],[92,null,184],[93,93,186],[94,null,188],[95,95,190],[96,null,192],[97,97,194],[98,null,196],[99,99,198],[100,null,200],[101,101,202],[102,null,204],[103,103,206],[104,null,208],[105,105,210],[106,null,212],[107,107,214],[108,null,216],[109,109,218],[110,null,220],[111,111,222],[112,null,224],[113,113,226],[114,null,228],[115,115,230],[116,null,232],[117,117,234],[118,null,236],[119,119,238],[120,null,240],[121,121,242],[122,null,244],[123,123,246],[124,null,248],[125,125,250],[126,null,252],[127,127,254],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,null],[145,1,null],[146,null,null],[147,1,null],[148,null,null],[149,1,null],[150,null,null],[151,1,null],[152,null,null],[153,1,null],[154,null,null],[155,1,null],[156,null,null],[157,1,null],[158,null,null],[159,1,null],[160,null,null],[161,1,null],[162,null,null],[163,1,null],[164,null,null],[165,1,null],[166,null,null],[167,1,null],[168,null,null],[169,1,null],[170,null,null],[171,1,null],[172,null,null],[173,1,null],[174,null,null],[175,1,null],[176,null,null],[177,1,null],[178,null,null],[179,1,null],[180,null,null],[181,1,null],[182,null,null],[183,1,null],[184,null,null],[185,1,null],[186,null,null],[187,1,null],[188,null,null],[189,1,null],[190,null,null],[191,1,null],[192,null,128],[193,65,130],[194,null,128],[195,65,130],[196,null,128],[197,65,130],[198,null,128],[199,65,130],[200,null,128],[201,65,130],[202,null,128],[203,65,130],[204,null,128],[205,65,130],[206,null,128],[207,65,130],[208,null,128],[209,65,130],[210,null,128],[211,65,130],[212,null,128],[213,65,130],[214,null,128],[215,65,130],[216,null,128],[217,65,130],[218,null,128],[219,65,130],[220,null,128],[221,65,130],[222,null,128],[223,65,130],[224,null,128],[225,65,130],[226,null,128],[227,65,130],[228,null,128],[229,65,130],[230,null,128],[231,65,130],[232,null,128],[233,65,130],[234,null,128],[235,65,130],[236,null,128],[237,65,130],[238,null,128],[239,65,130],[240,null,128],[241,65,130],[242,null,128],[243,65,130],[244,null,128],[245,65,130],[246,null,128],[247,65,130],[248,null,128],[249,65,130],[250,null,128],[251,65,130],[252,null,128],[253,65,130],[254,null,128],[255,65,130],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,null],[273,17,null],[274,1,null],[275,19,null],[276,1,null],[277,21,null],[278,1,null],[279,23,null],[280,1,null],[281,25,null],[282,1,null],[283,27,null],[284,1,null],[285,29,null],[286,1,null],[287,31,null],[288,1,null],[289,33,null],[290,1,null],[291,35,null],[292,1,null],[293,37,null],[294,1,null],[295,39,null],[296,1,null],[297,41,null],[298,1,null],[299,43,null],[300,1,null],[301,45,null],[302,1,null],[303,47,null],[304,1,null],[305,49,null],[306,1,null],[307,51,null],[308,1,null],[309,53,null],[310,1,null],[311,55,null],[312,1,null],[313,57,null],[314,1,null],[315,59,null],[316,1,null],[317,61,null],[318,1,null],[319,63,null],[320,1,128],[321,65,130],[322,1,132],[323,67,134],[324,1,136],[325,69,138],[326,1,140],[327,71,142],[328,1,144],[329,73,146],[330,1,148],[331,75,150],[332,1,152],[333,77,154],[334,1,156],[335,79,158],[336,1,160],[337,81,128],[338,65,130],[339,83,128],[340,65,130],[341,85,128],[342,65,130],[343,87,128],[344,65,130],[345,89,128],[346,65,130],[347,91,128],[348,65,130],[349,93,128],[350,65,130],[351,95,128],[352,65,130],[353,97,128],[354,65,130],[355,99,128],[356,65,130],[357,101,128],[358,65,130],[359,103,128],[360,65,130],[361,105,128],[362,65,130],[363,107,128],[364,65,130],[365,109,128],[366,65,130],[367,111,128],[368,65,130],[369,113,128],[370,65,130],[371,115,128],[372,65,130],[373,117,128],[374,65,130],[375,119,128],[376,65,130],[377,121,128],[378,65,130],[379,123,128],[380,65,130],[381,125,128],[382,65,130],[383,127,128],[384,65,130],[385,null,128],[386,65,130],[387,null,128],[388,65,130],[389,null,128],[390,65,130],[391,null,128],[392,65,130],[393,null,128],[394,65,130],[395,null,128],[396,65,130],[397,null,128],[398,65,130],[399,null,128],[400,65,130]],"tab10_frozen_udt_with_tuple":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,null],[17,17,null],[18,null,null],[19,19,null],[20,null,null],[21,21,null],[22,null,null],[23,23,null],[24,null,null],[25,25,null],[26,null,null],[27,27,null],[28,null,null],[29,29,null],[30,null,null],[31,31,null],[32,null,null],[33,33,null],[34,null,null],[35,35,null],[36,null,null],[37,37,null],[38,null,null],[39,39,null],[40,null,null],[41,41,null],[42,null,null],[43,43,null],[44,null,null],[45,45,null],[46,null,null],[47,47,null],[48,null,null],[49,49,null],[50,null,null],[51,51,null],[52,null,null],[53,53,null],[54,null,null],[55,55,null],[56,null,null],[57,57,null],[58,null,null],[59,59,null],[60,null,null],[61,61,null],[62,null,null],[63,63,null],[64,null,128],[65,65,130],[66,null,132],[67,67,134],[68,null,136],[69,69,138],[70,null,140],[71,71,142],[72,null,144],[73,73,146],[74,null,148],[75,75,150],[76,null,152],[77,77,154],[78,null,156],[79,79,158],[80,null,160],[81,81,162],[82,null,164],[83,83,166],[84,null,168],[85,85,170],[86,null,172],[87,87,174],[88,null,176],[89,89,178],[90,null,180],[91,91,182],[92,null,184],[93,93,186],[94,null,188],[95,95,190],[96,null,192],[97,97,194],[98,null,196],[99,99,198],[100,null,200],[101,101,202],[102,null,204],[103,103,206],[104,null,208],[105,105,210],[106,null,212],[107,107,214],[108,null,216],[109,109,218],[110,null,220],[111,111,222],[112,null,224],[113,113,226],[114,null,228],[115,115,230],[116,null,232],[117,117,234],[118,null,236],[119,119,238],[120,null,240],[121,121,242],[122,null,244],[123,123,246],[124,null,248],[125,125,250],[126,null,252],[127,127,254],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,null],[145,1,null],[146,null,null],[147,1,null],[148,null,null],[149,1,null],[150,null,null],[151,1,null],[152,null,null],[153,1,null],[154,null,null],[155,1,null],[156,null,null],[157,1,null],[158,null,null],[159,1,null],[160,null,null],[161,1,null],[162,null,null],[163,1,null],[164,null,null],[165,1,null],[166,null,null],[167,1,null],[168,null,null],[169,1,null],[170,null,null],[171,1,null],[172,null,null],[173,1,null],[174,null,null],[175,1,null],[176,null,null],[177,1,null],[178,null,null],[179,1,null],[180,null,null],[181,1,null],[182,null,null],[183,1,null],[184,null,null],[185,1,null],[186,null,null],[187,1,null],[188,null,null],[189,1,null],[190,null,null],[191,1,null],[192,null,128],[193,65,130],[194,null,128],[195,65,130],[196,null,128],[197,65,130],[198,null,128],[199,65,130],[200,null,128],[201,65,130],[202,null,128],[203,65,130],[204,null,128],[205,65,130],[206,null,128],[207,65,130],[208,null,128],[209,65,130],[210,null,128],[211,65,130],[212,null,128],[213,65,130],[214,null,128],[215,65,130],[216,null,128],[217,65,130],[218,null,128],[219,65,130],[220,null,128],[221,65,130],[222,null,128],[223,65,130],[224,null,128],[225,65,130],[226,null,128],[227,65,130],[228,null,128],[229,65,130],[230,null,128],[231,65,130],[232,null,128],[233,65,130],[234,null,128],[235,65,130],[236,null,128],[237,65,130],[238,null,128],[239,65,130],[240,null,128],[241,65,130],[242,null,128],[243,65,130],[244,null,128],[245,65,130],[246,null,128],[247,65,130],[248,null,128],[249,65,130],[250,null,128],[251,65,130],[252,null,128],[253,65,130],[254,null,128],[255,65,130],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,null],[273,17,null],[274,1,null],[275,19,null],[276,1,null],[277,21,null],[278,1,null],[279,23,null],[280,1,null],[281,25,null],[282,1,null],[283,27,null],[284,1,null],[285,29,null],[286,1,null],[287,31,null],[288,1,null],[289,33,null],[290,1,null],[291,35,null],[292,1,null],[293,37,null],[294,1,null],[295,39,null],[296,1,null],[297,41,null],[298,1,null],[299,43,null],[300,1,null],[301,45,null],[302,1,null],[303,47,null],[304,1,null],[305,49,null],[306,1,null],[307,51,null],[308,1,null],[309,53,null],[310,1,null],[311,55,null],[312,1,null],[313,57,null],[314,1,null],[315,59,null],[316,1,null],[317,61,null],[318,1,null],[319,63,null],[320,1,128],[321,65,130],[322,1,132],[323,67,134],[324,1,136],[325,69,138],[326,1,140],[327,71,142],[328,1,144],[329,73,146],[330,1,148],[331,75,150],[332,1,152],[333,77,154],[334,1,156],[335,79,158],[336,1,160],[337,81,128],[338,65,130],[339,83,128],[340,65,130],[341,85,128],[342,65,130],[343,87,128],[344,65,130],[345,89,128],[346,65,130],[347,91,128],[348,65,130],[349,93,128],[350,65,130],[351,95,128],[352,65,130],[353,97,128],[354,65,130],[355,99,128],[356,65,130],[357,101,128],[358,65,130],[359,103,128],[360,65,130],[361,105,128],[362,65,130],[363,107,128],[364,65,130],[365,109,128],[366,65,130],[367,111,128],[368,65,130],[369,113,128],[370,65,130],[371,115,128],[372,65,130],[373,117,128],[374,65,130],[375,119,128],[376,65,130],[377,121,128],[378,65,130],[379,123,128],[380,65,130],[381,125,128],[382,65,130],[383,127,128],[384,65,130],[385,null,128],[386,65,130],[387,null,128],[388,65,130],[389,null,128],[390,65,130],[391,null,128],[392,65,130],[393,null,128],[394,65,130],[395,null,128],[396,65,130],[397,null,128],[398,65,130],[399,null,128],[400,65,130]],"tab1_udt1":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,32],[17,17,34],[18,null,36],[19,19,38],[20,null,40],[21,21,42],[22,null,44],[23,23,46],[24,null,48],[25,25,50],[26,null,52],[27,27,54],[28,null,56],[29,29,58],[30,null,60],[31,31,62],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,32],[145,17,34],[146,null,32],[147,17,34],[148,null,32],[149,17,34],[150,null,32],[151,17,34],[152,null,32],[153,17,34],[154,null,32],[155,17,34],[156,null,32],[157,17,34],[158,null,32],[159,17,34],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,32],[273,17,34],[274,1,36],[275,19,38],[276,1,40],[277,21,42],[278,1,44],[279,23,46],[280,1,48],[281,25,50],[282,1,52],[283,27,54],[284,1,56],[285,29,58],[286,1,60],[287,31,62],[288,1,null],[289,null,32],[290,17,34],[291,null,32],[292,17,34],[293,null,32],[294,17,34],[295,null,32],[296,17,34],[297,null,32],[298,17,34],[299,null,32],[300,17,34],[301,null,32],[302,17,34],[303,null,32],[304,17,34]],"tab4_frozen_udt2":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,null],[17,17,null],[18,null,null],[19,19,null],[20,null,null],[21,21,null],[22,null,null],[23,23,null],[24,null,null],[25,25,null],[26,null,null],[27,27,null],[28,null,null],[29,29,null],[30,null,null],[31,31,null],[32,null,null],[33,33,null],[34,null,null],[35,35,null],[36,null,null],[37,37,null],[38,null,null],[39,39,null],[40,null,null],[41,41,null],[42,null,null],[43,43,null],[44,null,null],[45,45,null],[46,null,null],[47,47,null],[48,null,null],[49,49,null],[50,null,null],[51,51,null],[52,null,null],[53,53,null],[54,null,null],[55,55,null],[56,null,null],[57,57,null],[58,null,null],[59,59,null],[60,null,null],[61,61,null],[62,null,null],[63,63,null],[64,null,128],[65,65,130],[66,null,132],[67,67,134],[68,null,136],[69,69,138],[70,null,140],[71,71,142],[72,null,144],[73,73,146],[74,null,148],[75,75,150],[76,null,152],[77,77,154],[78,null,156],[79,79,158],[80,null,160],[81,81,162],[82,null,164],[83,83,166],[84,null,168],[85,85,170],[86,null,172],[87,87,174],[88,null,176],[89,89,178],[90,null,180],[91,91,182],[92,null,184],[93,93,186],[94,null,188],[95,95,190],[96,null,192],[97,97,194],[98,null,196],[99,99,198],[100,null,200],[101,101,202],[102,null,204],[103,103,206],[104,null,208],[105,105,210],[106,null,212],[107,107,214],[108,null,216],[109,109,218],[110,null,220],[111,111,222],[112,null,224],[113,113,226],[114,null,228],[115,115,230],[116,null,232],[117,117,234],[118,null,236],[119,119,238],[120,null,240],[121,121,242],[122,null,244],[123,123,246],[124,null,248],[125,125,250],[126,null,252],[127,127,254],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,null],[145,1,null],[146,null,null],[147,1,null],[148,null,null],[149,1,null],[150,null,null],[151,1,null],[152,null,null],[153,1,null],[154,null,null],[155,1,null],[156,null,null],[157,1,null],[158,null,null],[159,1,null],[160,null,null],[161,1,null],[162,null,null],[163,1,null],[164,null,null],[165,1,null],[166,null,null],[167,1,null],[168,null,null],[169,1,null],[170,null,null],[171,1,null],[172,null,null],[173,1,null],[174,null,null],[175,1,null],[176,null,null],[177,1,null],[178,null,null],[179,1,null],[180,null,null],[181,1,null],[182,null,null],[183,1,null],[184,null,null],[185,1,null],[186,null,null],[187,1,null],[188,null,null],[189,1,null],[190,null,null],[191,1,null],[192,null,128],[193,65,130],[194,null,128],[195,65,130],[196,null,128],[197,65,130],[198,null,128],[199,65,130],[200,null,128],[201,65,130],[202,null,128],[203,65,130],[204,null,128],[205,65,130],[206,null,128],[207,65,130],[208,null,128],[209,65,130],[210,null,128],[211,65,130],[212,null,128],[213,65,130],[214,null,128],[215,65,130],[216,null,128],[217,65,130],[218,null,128],[219,65,130],[220,null,128],[221,65,130],[222,null,128],[223,65,130],[224,null,128],[225,65,130],[226,null,128],[227,65,130],[228,null,128],[229,65,130],[230,null,128],[231,65,130],[232,null,128],[233,65,130],[234,null,128],[235,65,130],[236,null,128],[237,65,130],[238,null,128],[239,65,130],[240,null,128],[241,65,130],[242,null,128],[243,65,130],[244,null,128],[245,65,130],[246,null,128],[247,65,130],[248,null,128],[249,65,130],[250,null,128],[251,65,130],[252,null,128],[253,65,130],[254,null,128],[255,65,130],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,null],[273,17,null],[274,1,null],[275,19,null],[276,1,null],[277,21,null],[278,1,null],[279,23,null],[280,1,null],[281,25,null],[282,1,null],[283,27,null],[284,1,null],[285,29,null],[286,1,null],[287,31,null],[288,1,null],[289,33,null],[290,1,null],[291,35,null],[292,1,null],[293,37,null],[294,1,null],[295,39,null],[296,1,null],[297,41,null],[298,1,null],[299,43,null],[300,1,null],[301,45,null],[302,1,null],[303,47,null],[304,1,null],[305,49,null],[306,1,null],[307,51,null],[308,1,null],[309,53,null],[310,1,null],[311,55,null],[312,1,null],[313,57,null],[314,1,null],[315,59,null],[316,1,null],[317,61,null],[318,1,null],[319,63,null],[320,1,128],[321,65,130],[322,1,132],[323,67,134],[324,1,136],[325,69,138],[326,1,140],[327,71,142],[328,1,144],[329,73,146],[330,1,148],[331,75,150],[332,1,152],[333,77,154],[334,1,156],[335,79,158],[336,1,160],[337,81,128],[338,65,130],[339,83,128],[340,65,130],[341,85,128],[342,65,130],[343,87,128],[344,65,130],[345,89,128],[346,65,130],[347,91,128],[348,65,130],[349,93,128],[350,65,130],[351,95,128],[352,65,130],[353,97,128],[354,65,130],[355,99,128],[356,65,130],[357,101,128],[358,65,130],[359,103,128],[360,65,130],[361,105,128],[362,65,130],[363,107,128],[364,65,130],[365,109,128],[366,65,130],[367,111,128],[368,65,130],[369,113,128],[370,65,130],[371,115,128],[372,65,130],[373,117,128],[374,65,130],[375,119,128],[376,65,130],[377,121,128],[378,65,130],[379,123,128],[380,65,130],[381,125,128],[382,65,130],[383,127,128],[384,65,130],[385,null,128],[386,65,130],[387,null,128],[388,65,130],[389,null,128],[390,65,130],[391,null,128],[392,65,130],[393,null,128],[394,65,130],[395,null,128],[396,65,130],[397,null,128],[398,65,130],[399,null,128],[400,65,130]],"tab7_tuple_with_udt":[[0,null,null],[1,1,null],[2,null,null],[3,3,null],[4,null,null],[5,5,null],[6,null,null],[7,7,null],[8,null,null],[9,9,null],[10,null,null],[11,11,null],[12,null,null],[13,13,null],[14,null,null],[15,15,null],[16,null,null],[17,17,null],[18,null,null],[19,19,null],[20,null,null],[21,21,null],[22,null,null],[23,23,null],[24,null,null],[25,25,null],[26,null,null],[27,27,null],[28,null,null],[29,29,null],[30,null,null],[31,31,null],[32,null,null],[33,33,null],[34,null,null],[35,35,null],[36,null,null],[37,37,null],[38,null,null],[39,39,null],[40,null,null],[41,41,null],[42,null,null],[43,43,null],[44,null,null],[45,45,null],[46,null,null],[47,47,null],[48,null,null],[49,49,null],[50,null,null],[51,51,null],[52,null,null],[53,53,null],[54,null,null],[55,55,null],[56,null,null],[57,57,null],[58,null,null],[59,59,null],[60,null,null],[61,61,null],[62,null,null],[63,63,null],[64,null,128],[65,65,130],[66,null,132],[67,67,134],[68,null,136],[69,69,138],[70,null,140],[71,71,142],[72,null,144],[73,73,146],[74,null,148],[75,75,150],[76,null,152],[77,77,154],[78,null,156],[79,79,158],[80,null,160],[81,81,162],[82,null,164],[83,83,166],[84,null,168],[85,85,170],[86,null,172],[87,87,174],[88,null,176],[89,89,178],[90,null,180],[91,91,182],[92,null,184],[93,93,186],[94,null,188],[95,95,190],[96,null,192],[97,97,194],[98,null,196],[99,99,198],[100,null,200],[101,101,202],[102,null,204],[103,103,206],[104,null,208],[105,105,210],[106,null,212],[107,107,214],[108,null,216],[109,109,218],[110,null,220],[111,111,222],[112,null,224],[113,113,226],[114,null,228],[115,115,230],[116,null,232],[117,117,234],[118,null,236],[119,119,238],[120,null,240],[121,121,242],[122,null,244],[123,123,246],[124,null,248],[125,125,250],[126,null,252],[127,127,254],[128,null,null],[129,1,null],[130,null,null],[131,1,null],[132,null,null],[133,1,null],[134,null,null],[135,1,null],[136,null,null],[137,1,null],[138,null,null],[139,1,null],[140,null,null],[141,1,null],[142,null,null],[143,1,null],[144,null,null],[145,1,null],[146,null,null],[147,1,null],[148,null,null],[149,1,null],[150,null,null],[151,1,null],[152,null,null],[153,1,null],[154,null,null],[155,1,null],[156,null,null],[157,1,null],[158,null,null],[159,1,null],[160,null,null],[161,1,null],[162,null,null],[163,1,null],[164,null,null],[165,1,null],[166,null,null],[167,1,null],[168,null,null],[169,1,null],[170,null,null],[171,1,null],[172,null,null],[173,1,null],[174,null,null],[175,1,null],[176,null,null],[177,1,null],[178,null,null],[179,1,null],[180,null,null],[181,1,null],[182,null,null],[183,1,null],[184,null,null],[185,1,null],[186,null,null],[187,1,null],[188,null,null],[189,1,null],[190,null,null],[191,1,null],[192,null,128],[193,65,130],[194,null,128],[195,65,130],[196,null,128],[197,65,130],[198,null,128],[199,65,130],[200,null,128],[201,65,130],[202,null,128],[203,65,130],[204,null,128],[205,65,130],[206,null,128],[207,65,130],[208,null,128],[209,65,130],[210,null,128],[211,65,130],[212,null,128],[213,65,130],[214,null,128],[215,65,130],[216,null,128],[217,65,130],[218,null,128],[219,65,130],[220,null,128],[221,65,130],[222,null,128],[223,65,130],[224,null,128],[225,65,130],[226,null,128],[227,65,130],[228,null,128],[229,65,130],[230,null,128],[231,65,130],[232,null,128],[233,65,130],[234,null,128],[235,65,130],[236,null,128],[237,65,130],[238,null,128],[239,65,130],[240,null,128],[241,65,130],[242,null,128],[243,65,130],[244,null,128],[245,65,130],[246,null,128],[247,65,130],[248,null,128],[249,65,130],[250,null,128],[251,65,130],[252,null,128],[253,65,130],[254,null,128],[255,65,130],[256,null,null],[257,1,null],[258,null,null],[259,3,null],[260,null,null],[261,5,null],[262,null,null],[263,7,null],[264,null,null],[265,9,null],[266,null,null],[267,11,null],[268,null,null],[269,13,null],[270,null,null],[271,15,null],[272,null,null],[273,17,null],[274,1,null],[275,19,null],[276,1,null],[277,21,null],[278,1,null],[279,23,null],[280,1,null],[281,25,null],[282,1,null],[283,27,null],[284,1,null],[285,29,null],[286,1,null],[287,31,null],[288,1,null],[289,33,null],[290,1,null],[291,35,null],[292,1,null],[293,37,null],[294,1,null],[295,39,null],[296,1,null],[297,41,null],[298,1,null],[299,43,null],[300,1,null],[301,45,null],[302,1,null],[303,47,null],[304,1,null],[305,49,null],[306,1,null],[307,51,null],[308,1,null],[309,53,null],[310,1,null],[311,55,null],[312,1,null],[313,57,null],[314,1,null],[315,59,null],[316,1,null],[317,61,null],[318,1,null],[319,63,null],[320,1,128],[321,65,130],[322,1,132],[323,67,134],[324,1,136],[325,69,138],[326,1,140],[327,71,142],[328,1,144],[329,73,146],[330,1,148],[331,75,150],[332,1,152],[333,77,154],[334,1,156],[335,79,158],[336,1,160],[337,81,128],[338,65,130],[339,83,128],[340,65,130],[341,85,128],[342,65,130],[343,87,128],[344,65,130],[345,89,128],[346,65,130],[347,91,128],[348,65,130],[349,93,128],[350,65,130],[351,95,128],[352,65,130],[353,97,128],[354,65,130],[355,99,128],[356,65,130],[357,101,128],[358,65,130],[359,103,128],[360,65,130],[361,105,128],[362,65,130],[363,107,128],[364,65,130],[365,109,128],[366,65,130],[367,111,128],[368,65,130],[369,113,128],[370,65,130],[371,115,128],[372,65,130],[373,117,128],[374,65,130],[375,119,128],[376,65,130],[377,121,128],[378,65,130],[379,123,128],[380,65,130],[381,125,128],[382,65,130],[383,127,128],[384,65,130],[385,null,128],[386,65,130],[387,null,128],[388,65,130],[389,null,128],[390,65,130],[391,null,128],[392,65,130],[393,null,128],[394,65,130],[395,null,128],[396,65,130],[397,null,128],[398,65,130],[399,null,128],[400,65,130]]}
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-CompressionInfo.db
new file mode 100644
index 000000000000..c26eb564641f
Binary files /dev/null and b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Data.db b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Data.db
new file mode 100644
index 000000000000..c3ed7f5a2f60
Binary files /dev/null and b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Digest.crc32
new file mode 100644
index 000000000000..5b21fa54fa43
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Digest.crc32
@@ -0,0 +1 @@
+726769704
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Filter.db b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Filter.db
new file mode 100644
index 000000000000..7348d46cacba
Binary files /dev/null and b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Partitions.db b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Partitions.db
new file mode 100644
index 000000000000..b6261320374e
Binary files /dev/null and b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Rows.db b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Statistics.db b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Statistics.db
new file mode 100644
index 000000000000..94551a18a660
Binary files /dev/null and b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-TOC.txt b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/cc-2-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-CompressionInfo.db
new file mode 100644
index 000000000000..2e009a9881e5
Binary files /dev/null and b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Data.db b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Data.db
new file mode 100644
index 000000000000..98eceb85b00d
Binary files /dev/null and b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Digest.crc32
new file mode 100644
index 000000000000..14862e4f6867
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Digest.crc32
@@ -0,0 +1 @@
+1896219668
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Filter.db b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Filter.db
new file mode 100644
index 000000000000..87b3d6e6625c
Binary files /dev/null and b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Partitions.db b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Partitions.db
new file mode 100644
index 000000000000..ad334dfc1204
Binary files /dev/null and b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Rows.db b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Statistics.db b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Statistics.db
new file mode 100644
index 000000000000..fec59a83ad78
Binary files /dev/null and b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-TOC.txt b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/cc-3-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-CompressionInfo.db
new file mode 100644
index 000000000000..6403723a2827
Binary files /dev/null and b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Data.db b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Data.db
new file mode 100644
index 000000000000..0c2964509a52
Binary files /dev/null and b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Digest.crc32
new file mode 100644
index 000000000000..4da29a2b9a89
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Digest.crc32
@@ -0,0 +1 @@
+938966910
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Filter.db b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Filter.db
new file mode 100644
index 000000000000..93024dfa9a8b
Binary files /dev/null and b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Partitions.db b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Partitions.db
new file mode 100644
index 000000000000..ee53643b87c7
Binary files /dev/null and b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Rows.db b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Statistics.db b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Statistics.db
new file mode 100644
index 000000000000..231b2917a5d4
Binary files /dev/null and b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-TOC.txt b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/cc-2-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..055f0aa9ef6b
Binary files /dev/null and b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Data.db b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Data.db
new file mode 100644
index 000000000000..3bb66fe8dd2b
Binary files /dev/null and b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Digest.crc32
new file mode 100644
index 000000000000..5a05d0a184eb
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2245855154
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Filter.db b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Filter.db
new file mode 100644
index 000000000000..f588219fa5f7
Binary files /dev/null and b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Partitions.db b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Partitions.db
new file mode 100644
index 000000000000..c24aab200ac6
Binary files /dev/null and b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Rows.db b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Statistics.db b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Statistics.db
new file mode 100644
index 000000000000..d82cee66cbaa
Binary files /dev/null and b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-TOC.txt b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/cc-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-CompressionInfo.db
new file mode 100644
index 000000000000..6403723a2827
Binary files /dev/null and b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Data.db b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Data.db
new file mode 100644
index 000000000000..5ea3f594fdfe
Binary files /dev/null and b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Digest.crc32
new file mode 100644
index 000000000000..6341074b0a82
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Digest.crc32
@@ -0,0 +1 @@
+1829688135
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Filter.db b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Filter.db
new file mode 100644
index 000000000000..93024dfa9a8b
Binary files /dev/null and b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Partitions.db b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Partitions.db
new file mode 100644
index 000000000000..ee53643b87c7
Binary files /dev/null and b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Rows.db b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Statistics.db b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Statistics.db
new file mode 100644
index 000000000000..d7f05a3844f4
Binary files /dev/null and b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-TOC.txt b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab5_tuple-90826dd3843745859de415908236687f/cc-2-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-CompressionInfo.db
new file mode 100644
index 000000000000..6403723a2827
Binary files /dev/null and b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Data.db b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Data.db
new file mode 100644
index 000000000000..834bfccb70a3
Binary files /dev/null and b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Digest.crc32
new file mode 100644
index 000000000000..81c8f0897cbc
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Digest.crc32
@@ -0,0 +1 @@
+776018960
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Filter.db b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Filter.db
new file mode 100644
index 000000000000..93024dfa9a8b
Binary files /dev/null and b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Partitions.db b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Partitions.db
new file mode 100644
index 000000000000..ee53643b87c7
Binary files /dev/null and b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Rows.db b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Statistics.db b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Statistics.db
new file mode 100644
index 000000000000..a241c72af654
Binary files /dev/null and b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-TOC.txt b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/cc-2-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-CompressionInfo.db
new file mode 100644
index 000000000000..9223a08faf38
Binary files /dev/null and b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Data.db b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Data.db
new file mode 100644
index 000000000000..0f6de6dc166e
Binary files /dev/null and b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Digest.crc32
new file mode 100644
index 000000000000..4533c527293c
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Digest.crc32
@@ -0,0 +1 @@
+407847420
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Filter.db b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Filter.db
new file mode 100644
index 000000000000..16483766eb9c
Binary files /dev/null and b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Partitions.db b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Partitions.db
new file mode 100644
index 000000000000..bdd62d5e0c1e
Binary files /dev/null and b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Rows.db b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Statistics.db b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Statistics.db
new file mode 100644
index 000000000000..33c9c6a46b7a
Binary files /dev/null and b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-TOC.txt b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/cc-3-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..055f0aa9ef6b
Binary files /dev/null and b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Data.db b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Data.db
new file mode 100644
index 000000000000..c8cc44862842
Binary files /dev/null and b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Digest.crc32
new file mode 100644
index 000000000000..02ebba7b9495
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3677019016
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Filter.db b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Filter.db
new file mode 100644
index 000000000000..f588219fa5f7
Binary files /dev/null and b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Partitions.db b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Partitions.db
new file mode 100644
index 000000000000..c24aab200ac6
Binary files /dev/null and b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Rows.db b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Statistics.db b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Statistics.db
new file mode 100644
index 000000000000..476cb1e4c0b7
Binary files /dev/null and b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-TOC.txt b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/cc-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-CompressionInfo.db b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-CompressionInfo.db
new file mode 100644
index 000000000000..381ad52138a9
Binary files /dev/null and b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-CompressionInfo.db differ
diff --git a/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Data.db b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Data.db
new file mode 100644
index 000000000000..4fa05548bcf6
Binary files /dev/null and b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Data.db differ
diff --git a/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Digest.crc32 b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Digest.crc32
new file mode 100644
index 000000000000..533408ba1a75
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Digest.crc32
@@ -0,0 +1 @@
+1347810627
\ No newline at end of file
diff --git a/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Filter.db b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Filter.db
new file mode 100644
index 000000000000..16483766eb9c
Binary files /dev/null and b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Filter.db differ
diff --git a/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Partitions.db b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Partitions.db
new file mode 100644
index 000000000000..6830e55082b4
Binary files /dev/null and b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Partitions.db differ
diff --git a/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Rows.db b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Statistics.db b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Statistics.db
new file mode 100644
index 000000000000..2ecaed412679
Binary files /dev/null and b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-Statistics.db differ
diff --git a/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-TOC.txt b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-TOC.txt
new file mode 100644
index 000000000000..298910cfdc58
--- /dev/null
+++ b/test/data/udt/cc50/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/cc-3-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/udt/cc50/schema.txt b/test/data/udt/cc50/schema.txt
new file mode 100644
index 000000000000..a58ee7fc781a
--- /dev/null
+++ b/test/data/udt/cc50/schema.txt
@@ -0,0 +1,69 @@
+CREATE TYPE ks.udt1 (
+    foo int,
+    bar text,
+    baz int
+);
+CREATE TYPE ks.udt2 (
+    foo int,
+    bar udt1,
+    baz int
+);
+CREATE TYPE ks.udt3 (
+    foo int,
+    bar tuple<int, text, int>,
+    baz int
+);
+CREATE TABLE ks.tab10_frozen_udt_with_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 6a5cff4e-2f94-4c8b-9aa2-0fbd65292caa
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1722612913166001;
+CREATE TABLE ks.tab1_udt1 (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 513f2627-9356-41c4-a379-7ad42be97432
+    AND DROPPED COLUMN RECORD b_complex tuple<int, text, int> USING TIMESTAMP 1722612910627000;
+CREATE TABLE ks.tab2_frozen_udt1 (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 450f91fe-7c47-41c9-97bf-fdad854fa7e5
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1722612910888000;
+CREATE TABLE ks.tab4_frozen_udt2 (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 9c03c71c-6775-4357-9173-0f8808901afa
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1722612911139001;
+CREATE TABLE ks.tab5_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 90826dd3-8437-4585-9de4-15908236687f
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1722612911562001;
+CREATE TABLE ks.tab6_frozen_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 54185f9a-a6fd-487c-abc3-c01bd5835e48
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1722612911881000;
+CREATE TABLE ks.tab7_tuple_with_udt (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 4e78f403-7b63-4e0d-a231-42e42cba7cb5
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1722612912136000;
+CREATE TABLE ks.tab8_frozen_tuple_with_udt (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = 8660f235-0816-4019-9cc9-1798fa7beb17
+    AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1722612912576001;
+CREATE TABLE ks.tab9_udt_with_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int
+) WITH ID = f670fd5a-8145-4669-aceb-75667c000ea6
+    AND DROPPED COLUMN RECORD b_complex tuple<int, frozen<tuple<int, text, int>>, int> USING TIMESTAMP 1722612912905001
\ No newline at end of file
diff --git a/test/data/udt/cc50/schema0.txt b/test/data/udt/cc50/schema0.txt
new file mode 100644
index 000000000000..e2eaba40a8fe
--- /dev/null
+++ b/test/data/udt/cc50/schema0.txt
@@ -0,0 +1,69 @@
+CREATE TYPE ks.udt1 (
+    foo int,
+    bar text,
+    baz int
+);
+CREATE TYPE ks.udt2 (
+    foo int,
+    bar udt1,
+    baz int
+);
+CREATE TYPE ks.udt3 (
+    foo int,
+    bar tuple<int, text, int>,
+    baz int
+);
+CREATE TABLE ks.tab10_frozen_udt_with_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex frozen<udt3>,
+    c_int int
+) WITH ID = 6a5cff4e-2f94-4c8b-9aa2-0fbd65292caa;
+CREATE TABLE ks.tab1_udt1 (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int,
+    b_complex udt1
+) WITH ID = 513f2627-9356-41c4-a379-7ad42be97432;
+CREATE TABLE ks.tab2_frozen_udt1 (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex frozen<udt1>,
+    c_int int
+) WITH ID = 450f91fe-7c47-41c9-97bf-fdad854fa7e5;
+CREATE TABLE ks.tab4_frozen_udt2 (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex frozen<udt2>,
+    c_int int
+) WITH ID = 9c03c71c-6775-4357-9173-0f8808901afa;
+CREATE TABLE ks.tab5_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex tuple<int, text, int>,
+    c_int int
+) WITH ID = 90826dd3-8437-4585-9de4-15908236687f;
+CREATE TABLE ks.tab6_frozen_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex tuple<int, text, int>,
+    c_int int
+) WITH ID = 54185f9a-a6fd-487c-abc3-c01bd5835e48;
+CREATE TABLE ks.tab7_tuple_with_udt (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex tuple<int, udt1, int>,
+    c_int int
+) WITH ID = 4e78f403-7b63-4e0d-a231-42e42cba7cb5;
+CREATE TABLE ks.tab8_frozen_tuple_with_udt (
+    pk int PRIMARY KEY,
+    a_int int,
+    b_complex tuple<int, udt1, int>,
+    c_int int
+) WITH ID = 8660f235-0816-4019-9cc9-1798fa7beb17;
+CREATE TABLE ks.tab9_udt_with_tuple (
+    pk int PRIMARY KEY,
+    a_int int,
+    c_int int,
+    b_complex udt3
+) WITH ID = f670fd5a-8145-4669-aceb-75667c000ea6
\ No newline at end of file
diff --git a/test/data/udt/dse/commitlog/CommitLog-680-1716886413137.log b/test/data/udt/dse/commitlog/CommitLog-680-1716886413137.log
new file mode 100644
index 000000000000..a01849ca11a9
Binary files /dev/null and b/test/data/udt/dse/commitlog/CommitLog-680-1716886413137.log differ
diff --git a/test/data/udt/dse/commitlog/CommitLog-680-1716886413138.log b/test/data/udt/dse/commitlog/CommitLog-680-1716886413138.log
new file mode 100644
index 000000000000..4c2f6e5864db
Binary files /dev/null and b/test/data/udt/dse/commitlog/CommitLog-680-1716886413138.log differ
diff --git a/test/data/udt/dse/data.json b/test/data/udt/dse/data.json
new file mode 100644
index 000000000000..d2b25410672c
--- /dev/null
+++ b/test/data/udt/dse/data.json
@@ -0,0 +1,2477 @@
+{
+  "tab5_tuple": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, 32],
+    [17, 17, 34],
+    [18, null, 36],
+    [19, 19, 38],
+    [20, null, 40],
+    [21, 21, 42],
+    [22, null, 44],
+    [23, 23, 46],
+    [24, null, 48],
+    [25, 25, 50],
+    [26, null, 52],
+    [27, 27, 54],
+    [28, null, 56],
+    [29, 29, 58],
+    [30, null, 60],
+    [31, 31, 62],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, 32],
+    [145, 17, 34],
+    [146, null, 32],
+    [147, 17, 34],
+    [148, null, 32],
+    [149, 17, 34],
+    [150, null, 32],
+    [151, 17, 34],
+    [152, null, 32],
+    [153, 17, 34],
+    [154, null, 32],
+    [155, 17, 34],
+    [156, null, 32],
+    [157, 17, 34],
+    [158, null, 32],
+    [159, 17, 34],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, 32],
+    [273, 17, 34],
+    [274, 1, 36],
+    [275, 19, 38],
+    [276, 1, 40],
+    [277, 21, 42],
+    [278, 1, 44],
+    [279, 23, 46],
+    [280, 1, 48],
+    [281, 25, 50],
+    [282, 1, 52],
+    [283, 27, 54],
+    [284, 1, 56],
+    [285, 29, 58],
+    [286, 1, 60],
+    [287, 31, 62],
+    [288, 1, null],
+    [289, null, 32],
+    [290, 17, 34],
+    [291, null, 32],
+    [292, 17, 34],
+    [293, null, 32],
+    [294, 17, 34],
+    [295, null, 32],
+    [296, 17, 34],
+    [297, null, 32],
+    [298, 17, 34],
+    [299, null, 32],
+    [300, 17, 34],
+    [301, null, 32],
+    [302, 17, 34],
+    [303, null, 32],
+    [304, 17, 34]
+  ],
+  "tab8_frozen_tuple_with_udt": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ],
+  "tab6_frozen_tuple": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, 32],
+    [17, 17, 34],
+    [18, null, 36],
+    [19, 19, 38],
+    [20, null, 40],
+    [21, 21, 42],
+    [22, null, 44],
+    [23, 23, 46],
+    [24, null, 48],
+    [25, 25, 50],
+    [26, null, 52],
+    [27, 27, 54],
+    [28, null, 56],
+    [29, 29, 58],
+    [30, null, 60],
+    [31, 31, 62],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, 32],
+    [145, 17, 34],
+    [146, null, 32],
+    [147, 17, 34],
+    [148, null, 32],
+    [149, 17, 34],
+    [150, null, 32],
+    [151, 17, 34],
+    [152, null, 32],
+    [153, 17, 34],
+    [154, null, 32],
+    [155, 17, 34],
+    [156, null, 32],
+    [157, 17, 34],
+    [158, null, 32],
+    [159, 17, 34],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, 32],
+    [273, 17, 34],
+    [274, 1, 36],
+    [275, 19, 38],
+    [276, 1, 40],
+    [277, 21, 42],
+    [278, 1, 44],
+    [279, 23, 46],
+    [280, 1, 48],
+    [281, 25, 50],
+    [282, 1, 52],
+    [283, 27, 54],
+    [284, 1, 56],
+    [285, 29, 58],
+    [286, 1, 60],
+    [287, 31, 62],
+    [288, 1, null],
+    [289, null, 32],
+    [290, 17, 34],
+    [291, null, 32],
+    [292, 17, 34],
+    [293, null, 32],
+    [294, 17, 34],
+    [295, null, 32],
+    [296, 17, 34],
+    [297, null, 32],
+    [298, 17, 34],
+    [299, null, 32],
+    [300, 17, 34],
+    [301, null, 32],
+    [302, 17, 34],
+    [303, null, 32],
+    [304, 17, 34]
+  ],
+  "tab2_frozen_udt1": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, 32],
+    [17, 17, 34],
+    [18, null, 36],
+    [19, 19, 38],
+    [20, null, 40],
+    [21, 21, 42],
+    [22, null, 44],
+    [23, 23, 46],
+    [24, null, 48],
+    [25, 25, 50],
+    [26, null, 52],
+    [27, 27, 54],
+    [28, null, 56],
+    [29, 29, 58],
+    [30, null, 60],
+    [31, 31, 62],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, 32],
+    [145, 17, 34],
+    [146, null, 32],
+    [147, 17, 34],
+    [148, null, 32],
+    [149, 17, 34],
+    [150, null, 32],
+    [151, 17, 34],
+    [152, null, 32],
+    [153, 17, 34],
+    [154, null, 32],
+    [155, 17, 34],
+    [156, null, 32],
+    [157, 17, 34],
+    [158, null, 32],
+    [159, 17, 34],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, 32],
+    [273, 17, 34],
+    [274, 1, 36],
+    [275, 19, 38],
+    [276, 1, 40],
+    [277, 21, 42],
+    [278, 1, 44],
+    [279, 23, 46],
+    [280, 1, 48],
+    [281, 25, 50],
+    [282, 1, 52],
+    [283, 27, 54],
+    [284, 1, 56],
+    [285, 29, 58],
+    [286, 1, 60],
+    [287, 31, 62],
+    [288, 1, null],
+    [289, null, 32],
+    [290, 17, 34],
+    [291, null, 32],
+    [292, 17, 34],
+    [293, null, 32],
+    [294, 17, 34],
+    [295, null, 32],
+    [296, 17, 34],
+    [297, null, 32],
+    [298, 17, 34],
+    [299, null, 32],
+    [300, 17, 34],
+    [301, null, 32],
+    [302, 17, 34],
+    [303, null, 32],
+    [304, 17, 34]
+  ],
+  "tab9_udt_with_tuple": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ],
+  "tab10_frozen_udt_with_tuple": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ],
+  "tab1_udt1": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, 32],
+    [17, 17, 34],
+    [18, null, 36],
+    [19, 19, 38],
+    [20, null, 40],
+    [21, 21, 42],
+    [22, null, 44],
+    [23, 23, 46],
+    [24, null, 48],
+    [25, 25, 50],
+    [26, null, 52],
+    [27, 27, 54],
+    [28, null, 56],
+    [29, 29, 58],
+    [30, null, 60],
+    [31, 31, 62],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, 32],
+    [145, 17, 34],
+    [146, null, 32],
+    [147, 17, 34],
+    [148, null, 32],
+    [149, 17, 34],
+    [150, null, 32],
+    [151, 17, 34],
+    [152, null, 32],
+    [153, 17, 34],
+    [154, null, 32],
+    [155, 17, 34],
+    [156, null, 32],
+    [157, 17, 34],
+    [158, null, 32],
+    [159, 17, 34],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, 32],
+    [273, 17, 34],
+    [274, 1, 36],
+    [275, 19, 38],
+    [276, 1, 40],
+    [277, 21, 42],
+    [278, 1, 44],
+    [279, 23, 46],
+    [280, 1, 48],
+    [281, 25, 50],
+    [282, 1, 52],
+    [283, 27, 54],
+    [284, 1, 56],
+    [285, 29, 58],
+    [286, 1, 60],
+    [287, 31, 62],
+    [288, 1, null],
+    [289, null, 32],
+    [290, 17, 34],
+    [291, null, 32],
+    [292, 17, 34],
+    [293, null, 32],
+    [294, 17, 34],
+    [295, null, 32],
+    [296, 17, 34],
+    [297, null, 32],
+    [298, 17, 34],
+    [299, null, 32],
+    [300, 17, 34],
+    [301, null, 32],
+    [302, 17, 34],
+    [303, null, 32],
+    [304, 17, 34]
+  ],
+  "tab4_frozen_udt2": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ],
+  "tab7_tuple_with_udt": [
+    [0, null, null],
+    [1, 1, null],
+    [2, null, null],
+    [3, 3, null],
+    [4, null, null],
+    [5, 5, null],
+    [6, null, null],
+    [7, 7, null],
+    [8, null, null],
+    [9, 9, null],
+    [10, null, null],
+    [11, 11, null],
+    [12, null, null],
+    [13, 13, null],
+    [14, null, null],
+    [15, 15, null],
+    [16, null, null],
+    [17, 17, null],
+    [18, null, null],
+    [19, 19, null],
+    [20, null, null],
+    [21, 21, null],
+    [22, null, null],
+    [23, 23, null],
+    [24, null, null],
+    [25, 25, null],
+    [26, null, null],
+    [27, 27, null],
+    [28, null, null],
+    [29, 29, null],
+    [30, null, null],
+    [31, 31, null],
+    [32, null, null],
+    [33, 33, null],
+    [34, null, null],
+    [35, 35, null],
+    [36, null, null],
+    [37, 37, null],
+    [38, null, null],
+    [39, 39, null],
+    [40, null, null],
+    [41, 41, null],
+    [42, null, null],
+    [43, 43, null],
+    [44, null, null],
+    [45, 45, null],
+    [46, null, null],
+    [47, 47, null],
+    [48, null, null],
+    [49, 49, null],
+    [50, null, null],
+    [51, 51, null],
+    [52, null, null],
+    [53, 53, null],
+    [54, null, null],
+    [55, 55, null],
+    [56, null, null],
+    [57, 57, null],
+    [58, null, null],
+    [59, 59, null],
+    [60, null, null],
+    [61, 61, null],
+    [62, null, null],
+    [63, 63, null],
+    [64, null, 128],
+    [65, 65, 130],
+    [66, null, 132],
+    [67, 67, 134],
+    [68, null, 136],
+    [69, 69, 138],
+    [70, null, 140],
+    [71, 71, 142],
+    [72, null, 144],
+    [73, 73, 146],
+    [74, null, 148],
+    [75, 75, 150],
+    [76, null, 152],
+    [77, 77, 154],
+    [78, null, 156],
+    [79, 79, 158],
+    [80, null, 160],
+    [81, 81, 162],
+    [82, null, 164],
+    [83, 83, 166],
+    [84, null, 168],
+    [85, 85, 170],
+    [86, null, 172],
+    [87, 87, 174],
+    [88, null, 176],
+    [89, 89, 178],
+    [90, null, 180],
+    [91, 91, 182],
+    [92, null, 184],
+    [93, 93, 186],
+    [94, null, 188],
+    [95, 95, 190],
+    [96, null, 192],
+    [97, 97, 194],
+    [98, null, 196],
+    [99, 99, 198],
+    [100, null, 200],
+    [101, 101, 202],
+    [102, null, 204],
+    [103, 103, 206],
+    [104, null, 208],
+    [105, 105, 210],
+    [106, null, 212],
+    [107, 107, 214],
+    [108, null, 216],
+    [109, 109, 218],
+    [110, null, 220],
+    [111, 111, 222],
+    [112, null, 224],
+    [113, 113, 226],
+    [114, null, 228],
+    [115, 115, 230],
+    [116, null, 232],
+    [117, 117, 234],
+    [118, null, 236],
+    [119, 119, 238],
+    [120, null, 240],
+    [121, 121, 242],
+    [122, null, 244],
+    [123, 123, 246],
+    [124, null, 248],
+    [125, 125, 250],
+    [126, null, 252],
+    [127, 127, 254],
+    [128, null, null],
+    [129, 1, null],
+    [130, null, null],
+    [131, 1, null],
+    [132, null, null],
+    [133, 1, null],
+    [134, null, null],
+    [135, 1, null],
+    [136, null, null],
+    [137, 1, null],
+    [138, null, null],
+    [139, 1, null],
+    [140, null, null],
+    [141, 1, null],
+    [142, null, null],
+    [143, 1, null],
+    [144, null, null],
+    [145, 1, null],
+    [146, null, null],
+    [147, 1, null],
+    [148, null, null],
+    [149, 1, null],
+    [150, null, null],
+    [151, 1, null],
+    [152, null, null],
+    [153, 1, null],
+    [154, null, null],
+    [155, 1, null],
+    [156, null, null],
+    [157, 1, null],
+    [158, null, null],
+    [159, 1, null],
+    [160, null, null],
+    [161, 1, null],
+    [162, null, null],
+    [163, 1, null],
+    [164, null, null],
+    [165, 1, null],
+    [166, null, null],
+    [167, 1, null],
+    [168, null, null],
+    [169, 1, null],
+    [170, null, null],
+    [171, 1, null],
+    [172, null, null],
+    [173, 1, null],
+    [174, null, null],
+    [175, 1, null],
+    [176, null, null],
+    [177, 1, null],
+    [178, null, null],
+    [179, 1, null],
+    [180, null, null],
+    [181, 1, null],
+    [182, null, null],
+    [183, 1, null],
+    [184, null, null],
+    [185, 1, null],
+    [186, null, null],
+    [187, 1, null],
+    [188, null, null],
+    [189, 1, null],
+    [190, null, null],
+    [191, 1, null],
+    [192, null, 128],
+    [193, 65, 130],
+    [194, null, 128],
+    [195, 65, 130],
+    [196, null, 128],
+    [197, 65, 130],
+    [198, null, 128],
+    [199, 65, 130],
+    [200, null, 128],
+    [201, 65, 130],
+    [202, null, 128],
+    [203, 65, 130],
+    [204, null, 128],
+    [205, 65, 130],
+    [206, null, 128],
+    [207, 65, 130],
+    [208, null, 128],
+    [209, 65, 130],
+    [210, null, 128],
+    [211, 65, 130],
+    [212, null, 128],
+    [213, 65, 130],
+    [214, null, 128],
+    [215, 65, 130],
+    [216, null, 128],
+    [217, 65, 130],
+    [218, null, 128],
+    [219, 65, 130],
+    [220, null, 128],
+    [221, 65, 130],
+    [222, null, 128],
+    [223, 65, 130],
+    [224, null, 128],
+    [225, 65, 130],
+    [226, null, 128],
+    [227, 65, 130],
+    [228, null, 128],
+    [229, 65, 130],
+    [230, null, 128],
+    [231, 65, 130],
+    [232, null, 128],
+    [233, 65, 130],
+    [234, null, 128],
+    [235, 65, 130],
+    [236, null, 128],
+    [237, 65, 130],
+    [238, null, 128],
+    [239, 65, 130],
+    [240, null, 128],
+    [241, 65, 130],
+    [242, null, 128],
+    [243, 65, 130],
+    [244, null, 128],
+    [245, 65, 130],
+    [246, null, 128],
+    [247, 65, 130],
+    [248, null, 128],
+    [249, 65, 130],
+    [250, null, 128],
+    [251, 65, 130],
+    [252, null, 128],
+    [253, 65, 130],
+    [254, null, 128],
+    [255, 65, 130],
+    [256, null, null],
+    [257, 1, null],
+    [258, null, null],
+    [259, 3, null],
+    [260, null, null],
+    [261, 5, null],
+    [262, null, null],
+    [263, 7, null],
+    [264, null, null],
+    [265, 9, null],
+    [266, null, null],
+    [267, 11, null],
+    [268, null, null],
+    [269, 13, null],
+    [270, null, null],
+    [271, 15, null],
+    [272, null, null],
+    [273, 17, null],
+    [274, 1, null],
+    [275, 19, null],
+    [276, 1, null],
+    [277, 21, null],
+    [278, 1, null],
+    [279, 23, null],
+    [280, 1, null],
+    [281, 25, null],
+    [282, 1, null],
+    [283, 27, null],
+    [284, 1, null],
+    [285, 29, null],
+    [286, 1, null],
+    [287, 31, null],
+    [288, 1, null],
+    [289, 33, null],
+    [290, 1, null],
+    [291, 35, null],
+    [292, 1, null],
+    [293, 37, null],
+    [294, 1, null],
+    [295, 39, null],
+    [296, 1, null],
+    [297, 41, null],
+    [298, 1, null],
+    [299, 43, null],
+    [300, 1, null],
+    [301, 45, null],
+    [302, 1, null],
+    [303, 47, null],
+    [304, 1, null],
+    [305, 49, null],
+    [306, 1, null],
+    [307, 51, null],
+    [308, 1, null],
+    [309, 53, null],
+    [310, 1, null],
+    [311, 55, null],
+    [312, 1, null],
+    [313, 57, null],
+    [314, 1, null],
+    [315, 59, null],
+    [316, 1, null],
+    [317, 61, null],
+    [318, 1, null],
+    [319, 63, null],
+    [320, 1, 128],
+    [321, 65, 130],
+    [322, 1, 132],
+    [323, 67, 134],
+    [324, 1, 136],
+    [325, 69, 138],
+    [326, 1, 140],
+    [327, 71, 142],
+    [328, 1, 144],
+    [329, 73, 146],
+    [330, 1, 148],
+    [331, 75, 150],
+    [332, 1, 152],
+    [333, 77, 154],
+    [334, 1, 156],
+    [335, 79, 158],
+    [336, 1, 160],
+    [337, 81, 128],
+    [338, 65, 130],
+    [339, 83, 128],
+    [340, 65, 130],
+    [341, 85, 128],
+    [342, 65, 130],
+    [343, 87, 128],
+    [344, 65, 130],
+    [345, 89, 128],
+    [346, 65, 130],
+    [347, 91, 128],
+    [348, 65, 130],
+    [349, 93, 128],
+    [350, 65, 130],
+    [351, 95, 128],
+    [352, 65, 130],
+    [353, 97, 128],
+    [354, 65, 130],
+    [355, 99, 128],
+    [356, 65, 130],
+    [357, 101, 128],
+    [358, 65, 130],
+    [359, 103, 128],
+    [360, 65, 130],
+    [361, 105, 128],
+    [362, 65, 130],
+    [363, 107, 128],
+    [364, 65, 130],
+    [365, 109, 128],
+    [366, 65, 130],
+    [367, 111, 128],
+    [368, 65, 130],
+    [369, 113, 128],
+    [370, 65, 130],
+    [371, 115, 128],
+    [372, 65, 130],
+    [373, 117, 128],
+    [374, 65, 130],
+    [375, 119, 128],
+    [376, 65, 130],
+    [377, 121, 128],
+    [378, 65, 130],
+    [379, 123, 128],
+    [380, 65, 130],
+    [381, 125, 128],
+    [382, 65, 130],
+    [383, 127, 128],
+    [384, 65, 130],
+    [385, null, 128],
+    [386, 65, 130],
+    [387, null, 128],
+    [388, 65, 130],
+    [389, null, 128],
+    [390, 65, 130],
+    [391, null, 128],
+    [392, 65, 130],
+    [393, null, 128],
+    [394, 65, 130],
+    [395, null, 128],
+    [396, 65, 130],
+    [397, null, 128],
+    [398, 65, 130],
+    [399, null, 128],
+    [400, 65, 130]
+  ]
+}
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..02d50ba350dc
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..7a2a2bdc96f5
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..bbd8e52fac83
--- /dev/null
+++ b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1585509291
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..1d148399d8c1
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..cf914532a760
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..02d50ba350dc
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Data.db
new file mode 100644
index 000000000000..7a2a2bdc96f5
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..bbd8e52fac83
--- /dev/null
+++ b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1585509291
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..1d148399d8c1
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..cf914532a760
Binary files /dev/null and b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab10_frozen_udt_with_tuple-6a5cff4e2f944c8b9aa20fbd65292caa/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..08a110019595
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..d73b49a2a733
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..0afdc44be807
--- /dev/null
+++ b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+267106807
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ddc10d84cfb5
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..f7e2506b30fe
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..d00caed6d2aa
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..08a110019595
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Data.db
new file mode 100644
index 000000000000..d73b49a2a733
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..0afdc44be807
--- /dev/null
+++ b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+267106807
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ddc10d84cfb5
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..f7e2506b30fe
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..d00caed6d2aa
Binary files /dev/null and b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab1_udt1-513f2627935641c4a3797ad42be97432/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..be07a5c39d2f
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..912133935ae0
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..6c42aa800a2f
--- /dev/null
+++ b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+4251699674
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ddc10d84cfb5
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..703ab1ae97a5
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..cf0c67e696f2
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..be07a5c39d2f
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Data.db
new file mode 100644
index 000000000000..912133935ae0
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..6c42aa800a2f
--- /dev/null
+++ b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+4251699674
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ddc10d84cfb5
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..703ab1ae97a5
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..cf0c67e696f2
Binary files /dev/null and b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab2_frozen_udt1-450f91fe7c4741c997bffdad854fa7e5/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..02d50ba350dc
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..e1a8763aa84f
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..d1d702e13ea8
--- /dev/null
+++ b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+81536759
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..1d148399d8c1
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..9653232e72a9
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..02d50ba350dc
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Data.db
new file mode 100644
index 000000000000..e1a8763aa84f
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..d1d702e13ea8
--- /dev/null
+++ b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+81536759
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..1d148399d8c1
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..9653232e72a9
Binary files /dev/null and b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab4_frozen_udt2-9c03c71c6775435791730f8808901afa/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..be07a5c39d2f
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..df71aecde743
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..4db23d2bb3a7
--- /dev/null
+++ b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1056628318
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ddc10d84cfb5
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..703ab1ae97a5
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..47a5ff770a3c
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..be07a5c39d2f
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Data.db
new file mode 100644
index 000000000000..df71aecde743
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..4db23d2bb3a7
--- /dev/null
+++ b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1056628318
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ddc10d84cfb5
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..703ab1ae97a5
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..47a5ff770a3c
Binary files /dev/null and b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab5_tuple-90826dd3843745859de415908236687f/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..be07a5c39d2f
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..374af52eea79
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..efa6aca2b123
--- /dev/null
+++ b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1446632359
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ddc10d84cfb5
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..703ab1ae97a5
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..478d9d98dfe1
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..be07a5c39d2f
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Data.db
new file mode 100644
index 000000000000..374af52eea79
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..efa6aca2b123
--- /dev/null
+++ b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1446632359
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..ddc10d84cfb5
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..703ab1ae97a5
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..478d9d98dfe1
Binary files /dev/null and b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab6_frozen_tuple-54185f9aa6fd487cabc3c01bd5835e48/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..02d50ba350dc
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..ea2d5e1de23c
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..5aa5ec180adf
--- /dev/null
+++ b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2998342438
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..1d148399d8c1
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..7909334265a8
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..02d50ba350dc
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Data.db
new file mode 100644
index 000000000000..ea2d5e1de23c
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..5aa5ec180adf
--- /dev/null
+++ b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2998342438
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..1d148399d8c1
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..7909334265a8
Binary files /dev/null and b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab7_tuple_with_udt-4e78f4037b634e0da23142e42cba7cb5/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..02d50ba350dc
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..c9a4926c9fce
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..f588ecdef934
--- /dev/null
+++ b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3453581958
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..1d148399d8c1
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..83a6ca6aaeb2
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..02d50ba350dc
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Data.db
new file mode 100644
index 000000000000..c9a4926c9fce
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..f588ecdef934
--- /dev/null
+++ b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3453581958
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..1d148399d8c1
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..83a6ca6aaeb2
Binary files /dev/null and b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab8_frozen_tuple_with_udt-8660f235081640199cc91798fa7beb17/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..3a678f05fb40
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Data.db
new file mode 100644
index 000000000000..f63bccf07d14
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..9c208ff22f01
--- /dev/null
+++ b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3940802110
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..bd3bc4ac61f6
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..61b723912e0f
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/backups/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-CompressionInfo.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..3a678f05fb40
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-CompressionInfo.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Data.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Data.db
new file mode 100644
index 000000000000..f63bccf07d14
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Data.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Digest.crc32 b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..9c208ff22f01
--- /dev/null
+++ b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3940802110
\ No newline at end of file
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Filter.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..8ec204b222ec
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Filter.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Partitions.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..bd3bc4ac61f6
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Partitions.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Rows.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Statistics.db b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..61b723912e0f
Binary files /dev/null and b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-Statistics.db differ
diff --git a/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-TOC.txt b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..96495ac1347d
--- /dev/null
+++ b/test/data/udt/dse/ks/tab9_udt_with_tuple-f670fd5a81454669aceb75667c000ea6/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Filter.db
+Data.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Rows.db
+CompressionInfo.db
+Partitions.db
diff --git a/test/data/udt/dse/schema.txt b/test/data/udt/dse/schema.txt
new file mode 100644
index 000000000000..c95278739f17
--- /dev/null
+++ b/test/data/udt/dse/schema.txt
@@ -0,0 +1,57 @@
+CREATE TYPE IF NOT EXISTS ks.udt1 (foo int, bar text, baz int);
+CREATE TYPE IF NOT EXISTS ks.udt2 (foo int, bar udt1, baz int);
+CREATE TYPE IF NOT EXISTS ks.udt3 (foo int, bar tuple<int, text, int>, baz int);
+CREATE TABLE IF NOT EXISTS ks.tab1_udt1 (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = 513f2627-9356-41c4-a379-7ad42be97432
+	AND DROPPED COLUMN RECORD b_complex tuple<int, text, int> USING TIMESTAMP 1716886419358000;
+CREATE TABLE IF NOT EXISTS ks.tab2_frozen_udt1 (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = 450f91fe-7c47-41c9-97bf-fdad854fa7e5
+	AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1716886419453000;
+CREATE TABLE IF NOT EXISTS ks.tab4_frozen_udt2 (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = 9c03c71c-6775-4357-9173-0f8808901afa
+	AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1716886419544000;
+CREATE TABLE IF NOT EXISTS ks.tab5_tuple (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = 90826dd3-8437-4585-9de4-15908236687f
+	AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1716886419641000;
+CREATE TABLE IF NOT EXISTS ks.tab6_frozen_tuple (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = 54185f9a-a6fd-487c-abc3-c01bd5835e48
+	AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, text, int>> USING TIMESTAMP 1716886419729000;
+CREATE TABLE IF NOT EXISTS ks.tab7_tuple_with_udt (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = 4e78f403-7b63-4e0d-a231-42e42cba7cb5
+	AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1716886419826000;
+CREATE TABLE IF NOT EXISTS ks.tab8_frozen_tuple_with_udt (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = 8660f235-0816-4019-9cc9-1798fa7beb17
+	AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1716886419931000;
+CREATE TABLE IF NOT EXISTS ks.tab9_udt_with_tuple (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = f670fd5a-8145-4669-aceb-75667c000ea6
+	AND DROPPED COLUMN RECORD b_complex tuple<int, frozen<tuple<int, text, int>>, int> USING TIMESTAMP 1716886420031000;
+CREATE TABLE IF NOT EXISTS ks.tab10_frozen_udt_with_tuple (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int)
+	WITH ID = 6a5cff4e-2f94-4c8b-9aa2-0fbd65292caa
+	AND DROPPED COLUMN RECORD b_complex frozen<tuple<int, frozen<tuple<int, text, int>>, int>> USING TIMESTAMP 1716886420131000
\ No newline at end of file
diff --git a/test/data/udt/dse/schema0.txt b/test/data/udt/dse/schema0.txt
new file mode 100644
index 000000000000..78d4c3562dee
--- /dev/null
+++ b/test/data/udt/dse/schema0.txt
@@ -0,0 +1,57 @@
+CREATE TYPE IF NOT EXISTS ks.udt1 (foo int, bar text, baz int);
+CREATE TYPE IF NOT EXISTS ks.udt2 (foo int, bar udt1, baz int);
+CREATE TYPE IF NOT EXISTS ks.udt3 (foo int, bar tuple<int, text, int>, baz int);
+CREATE TABLE IF NOT EXISTS ks.tab1_udt1 (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int,
+	b_complex udt1)
+	WITH ID = 513f2627-9356-41c4-a379-7ad42be97432;
+CREATE TABLE IF NOT EXISTS ks.tab2_frozen_udt1 (
+	pk int PRIMARY KEY,
+	a_int int,
+	b_complex frozen<udt1>,
+	c_int int)
+	WITH ID = 450f91fe-7c47-41c9-97bf-fdad854fa7e5;
+CREATE TABLE IF NOT EXISTS ks.tab4_frozen_udt2 (
+	pk int PRIMARY KEY,
+	a_int int,
+	b_complex frozen<udt2>,
+	c_int int)
+	WITH ID = 9c03c71c-6775-4357-9173-0f8808901afa;
+CREATE TABLE IF NOT EXISTS ks.tab5_tuple (
+	pk int PRIMARY KEY,
+	a_int int,
+	b_complex tuple<int, text, int>,
+	c_int int)
+	WITH ID = 90826dd3-8437-4585-9de4-15908236687f;
+CREATE TABLE IF NOT EXISTS ks.tab6_frozen_tuple (
+	pk int PRIMARY KEY,
+	a_int int,
+	b_complex tuple<int, text, int>,
+	c_int int)
+	WITH ID = 54185f9a-a6fd-487c-abc3-c01bd5835e48;
+CREATE TABLE IF NOT EXISTS ks.tab7_tuple_with_udt (
+	pk int PRIMARY KEY,
+	a_int int,
+	b_complex tuple<int, udt1, int>,
+	c_int int)
+	WITH ID = 4e78f403-7b63-4e0d-a231-42e42cba7cb5;
+CREATE TABLE IF NOT EXISTS ks.tab8_frozen_tuple_with_udt (
+	pk int PRIMARY KEY,
+	a_int int,
+	b_complex tuple<int, udt1, int>,
+	c_int int)
+	WITH ID = 8660f235-0816-4019-9cc9-1798fa7beb17;
+CREATE TABLE IF NOT EXISTS ks.tab9_udt_with_tuple (
+	pk int PRIMARY KEY,
+	a_int int,
+	c_int int,
+	b_complex udt3)
+	WITH ID = f670fd5a-8145-4669-aceb-75667c000ea6;
+CREATE TABLE IF NOT EXISTS ks.tab10_frozen_udt_with_tuple (
+	pk int PRIMARY KEY,
+	a_int int,
+	b_complex frozen<udt3>,
+	c_int int)
+	WITH ID = 6a5cff4e-2f94-4c8b-9aa2-0fbd65292caa
\ No newline at end of file
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-CompressionInfo.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-CompressionInfo.db
new file mode 100644
index 000000000000..7bd849dd2935
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-CompressionInfo.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Data.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Data.db
new file mode 100644
index 000000000000..b31194318fe6
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Data.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Filter.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Filter.db
new file mode 100644
index 000000000000..edad7a615522
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Filter.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Partitions.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Partitions.db
new file mode 100644
index 000000000000..0c14a5d171f0
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Partitions.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Rows.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Statistics.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Statistics.db
new file mode 100644
index 000000000000..dd754d5a9b1f
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Statistics.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-TOC.txt b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-TOC.txt
new file mode 100644
index 000000000000..d11c9ca66cdb
--- /dev/null
+++ b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-TOC.txt
@@ -0,0 +1,7 @@
+CompressionInfo.db
+Data.db
+Partitions.db
+TOC.txt
+Statistics.db
+Filter.db
+Rows.db
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-CompressionInfo.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-CompressionInfo.db
new file mode 100644
index 000000000000..7bd849dd2935
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-CompressionInfo.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Data.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Data.db
new file mode 100644
index 000000000000..e0ccfc590677
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Data.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Filter.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Filter.db
new file mode 100644
index 000000000000..e7e7bee15e1f
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Filter.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Partitions.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Partitions.db
new file mode 100644
index 000000000000..0c14a5d171f0
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Partitions.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Rows.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Statistics.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Statistics.db
new file mode 100644
index 000000000000..d3f62bdc50b9
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Statistics.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-TOC.txt b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-TOC.txt
new file mode 100644
index 000000000000..d11c9ca66cdb
--- /dev/null
+++ b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-TOC.txt
@@ -0,0 +1,7 @@
+CompressionInfo.db
+Data.db
+Partitions.db
+TOC.txt
+Statistics.db
+Filter.db
+Rows.db
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-CompressionInfo.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-CompressionInfo.db
new file mode 100644
index 000000000000..7bd849dd2935
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-CompressionInfo.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Data.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Data.db
new file mode 100644
index 000000000000..b31194318fe6
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Data.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Filter.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Filter.db
new file mode 100644
index 000000000000..edad7a615522
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Filter.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Partitions.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Partitions.db
new file mode 100644
index 000000000000..0c14a5d171f0
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Partitions.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Rows.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Statistics.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Statistics.db
new file mode 100644
index 000000000000..dd754d5a9b1f
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Statistics.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-TOC.txt b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-TOC.txt
new file mode 100644
index 000000000000..d11c9ca66cdb
--- /dev/null
+++ b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-TOC.txt
@@ -0,0 +1,7 @@
+CompressionInfo.db
+Data.db
+Partitions.db
+TOC.txt
+Statistics.db
+Filter.db
+Rows.db
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-CompressionInfo.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-CompressionInfo.db
new file mode 100644
index 000000000000..7bd849dd2935
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-CompressionInfo.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Data.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Data.db
new file mode 100644
index 000000000000..e0ccfc590677
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Data.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Filter.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Filter.db
new file mode 100644
index 000000000000..e7e7bee15e1f
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Filter.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Partitions.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Partitions.db
new file mode 100644
index 000000000000..0c14a5d171f0
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Partitions.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Rows.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Statistics.db b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Statistics.db
new file mode 100644
index 000000000000..d3f62bdc50b9
Binary files /dev/null and b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Statistics.db differ
diff --git a/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-TOC.txt b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-TOC.txt
new file mode 100644
index 000000000000..d11c9ca66cdb
--- /dev/null
+++ b/test/data/zcs/compressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-TOC.txt
@@ -0,0 +1,7 @@
+CompressionInfo.db
+Data.db
+Partitions.db
+TOC.txt
+Statistics.db
+Filter.db
+Rows.db
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-CRC.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-CRC.db
new file mode 100644
index 000000000000..1b96b2201276
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-CRC.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Data.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Data.db
new file mode 100644
index 000000000000..9d7da55dcec1
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Data.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Filter.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Filter.db
new file mode 100644
index 000000000000..edad7a615522
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Filter.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Partitions.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Partitions.db
new file mode 100644
index 000000000000..0c14a5d171f0
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Partitions.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Rows.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Statistics.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Statistics.db
new file mode 100644
index 000000000000..04650d480586
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-Statistics.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-TOC.txt b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-TOC.txt
new file mode 100644
index 000000000000..073da9f4a1b5
--- /dev/null
+++ b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-3-bti-TOC.txt
@@ -0,0 +1,7 @@
+Statistics.db
+Filter.db
+CRC.db
+Rows.db
+Data.db
+Partitions.db
+TOC.txt
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-CRC.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-CRC.db
new file mode 100644
index 000000000000..1b96b2201276
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-CRC.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Data.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Data.db
new file mode 100644
index 000000000000..02001ea2daac
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Data.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Filter.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Filter.db
new file mode 100644
index 000000000000..e7e7bee15e1f
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Filter.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Partitions.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Partitions.db
new file mode 100644
index 000000000000..0c14a5d171f0
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Partitions.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Rows.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Statistics.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Statistics.db
new file mode 100644
index 000000000000..8c4ec100ca7b
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-Statistics.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-TOC.txt b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-TOC.txt
new file mode 100644
index 000000000000..073da9f4a1b5
--- /dev/null
+++ b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-4-bti-TOC.txt
@@ -0,0 +1,7 @@
+Statistics.db
+Filter.db
+CRC.db
+Rows.db
+Data.db
+Partitions.db
+TOC.txt
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-CRC.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-CRC.db
new file mode 100644
index 000000000000..1b96b2201276
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-CRC.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Data.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Data.db
new file mode 100644
index 000000000000..9d7da55dcec1
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Data.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Filter.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Filter.db
new file mode 100644
index 000000000000..edad7a615522
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Filter.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Partitions.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Partitions.db
new file mode 100644
index 000000000000..0c14a5d171f0
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Partitions.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Rows.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Statistics.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Statistics.db
new file mode 100644
index 000000000000..04650d480586
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-Statistics.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-TOC.txt b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-TOC.txt
new file mode 100644
index 000000000000..073da9f4a1b5
--- /dev/null
+++ b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-6-bti-TOC.txt
@@ -0,0 +1,7 @@
+Statistics.db
+Filter.db
+CRC.db
+Rows.db
+Data.db
+Partitions.db
+TOC.txt
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-CRC.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-CRC.db
new file mode 100644
index 000000000000..1b96b2201276
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-CRC.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Data.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Data.db
new file mode 100644
index 000000000000..02001ea2daac
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Data.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Filter.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Filter.db
new file mode 100644
index 000000000000..e7e7bee15e1f
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Filter.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Partitions.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Partitions.db
new file mode 100644
index 000000000000..0c14a5d171f0
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Partitions.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Rows.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Statistics.db b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Statistics.db
new file mode 100644
index 000000000000..8c4ec100ca7b
Binary files /dev/null and b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-Statistics.db differ
diff --git a/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-TOC.txt b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-TOC.txt
new file mode 100644
index 000000000000..073da9f4a1b5
--- /dev/null
+++ b/test/data/zcs/uncompressed/ZeroCopyStreamingTest/Standard1/bb-7-bti-TOC.txt
@@ -0,0 +1,7 @@
+Statistics.db
+Filter.db
+CRC.db
+Rows.db
+Data.db
+Partitions.db
+TOC.txt
diff --git a/test/distributed/org/apache/cassandra/db/commitlog/MemoryMappedSegmentStartupTest.java b/test/distributed/org/apache/cassandra/db/commitlog/MemoryMappedSegmentStartupTest.java
new file mode 100644
index 000000000000..e6cdfd0b4d9c
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/db/commitlog/MemoryMappedSegmentStartupTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+import java.io.IOException;
+
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_SKIP_FILE_ADVICE;
+import static org.junit.Assert.assertEquals;
+
+public class MemoryMappedSegmentStartupTest
+{
+    @After
+    public void tearDown() throws Exception
+    {
+        COMMITLOG_SKIP_FILE_ADVICE.reset();
+    }
+
+    @Test
+    public void shouldSetSkipFileAdviceTrueWithParameterTrue() throws IOException
+    {
+        COMMITLOG_SKIP_FILE_ADVICE.setBoolean(true);
+        try (Cluster cluster = Cluster.build(1).start())
+        {
+            assertEquals(true, cluster.get(1).callOnInstance(() -> MemoryMappedSegment.skipFileAdviseToFreePageCache));
+        }
+    }
+
+    @Test
+    public void shouldSetSkipFileAdviceFalseWithParameterFalse() throws IOException
+    {
+        COMMITLOG_SKIP_FILE_ADVICE.setBoolean(false);
+        try (Cluster cluster = Cluster.build(1).start())
+        {
+            assertEquals(false, cluster.get(1).callOnInstance(() -> MemoryMappedSegment.skipFileAdviseToFreePageCache));
+        }
+    }
+
+    @Test
+    public void shouldSetSkipFileAdviceFalseWithParameterMissing() throws IOException
+    {
+        try (Cluster cluster = Cluster.build(1).start())
+        {
+            assertEquals(false, cluster.get(1).callOnInstance(() -> MemoryMappedSegment.skipFileAdviseToFreePageCache));
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java b/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java
index 54fe2303ad3a..83e6407f4deb 100644
--- a/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java
+++ b/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java
@@ -307,16 +307,16 @@ Mutation delete(long lts, long pd, Query query)
                                                  new AbstractMarker.Raw(values.size() - 1)));
         }
 
-        StatementRestrictions restrictions = new StatementRestrictions(null,
-                                                                       StatementType.DELETE,
-                                                                       metadata,
-                                                                       builder.build(),
-                                                                       new VariableSpecifications(variableNames),
-                                                                       Collections.emptyList(),
-                                                                       false,
-                                                                       false,
-                                                                       false,
-                                                                       false);
+        StatementRestrictions restrictions = StatementRestrictions.create(null,
+                                                                          StatementType.DELETE,
+                                                                          metadata,
+                                                                          builder.build(),
+                                                                          new VariableSpecifications(variableNames),
+                                                                          Collections.emptyList(),
+                                                                          false,
+                                                                          false,
+                                                                          false,
+                                                                          false);
 
         QueryOptions options = QueryOptions.forInternalCalls(ConsistencyLevel.QUORUM, values);
         SortedSet<ClusteringBound<?>> startBounds = restrictions.getClusteringColumnsBounds(Bound.START, options);
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
index c7e4c1052113..9a0796469eb2 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
@@ -191,7 +191,7 @@ public static abstract class AbstractBuilder<I extends IInstance, C extends IClu
             // Indicate that we are running in the in-jvm dtest environment
             CassandraRelevantProperties.DTEST_IS_IN_JVM_DTEST.setBoolean(true);
             // those properties may be set for unit-test optimizations; those should not be used when running dtests
-            CassandraRelevantProperties.TEST_FLUSH_LOCAL_SCHEMA_CHANGES.reset();
+            CassandraRelevantProperties.UNSAFE_SYSTEM.reset();
             CassandraRelevantProperties.NON_GRACEFUL_SHUTDOWN.reset();
             CassandraRelevantProperties.IO_NETTY_TRANSPORT_NONATIVE.setBoolean(false);
         }
@@ -591,6 +591,7 @@ InstanceConfig createInstanceConfig(int nodeNum)
         Collection<String> tokens = tokenSupplier.tokens(nodeNum);
         NetworkTopology topology = buildNetworkTopology(provisionStrategy, nodeIdTopology);
         InstanceConfig config = InstanceConfig.generate(nodeNum, provisionStrategy, topology, root, tokens, datadirCount);
+        logger.info("Instance {} config: {}", nodeNum, config);
         config.set(Constants.KEY_DTEST_API_CLUSTER_ID, clusterId.toString());
         // if a test sets num_tokens directly, then respect it and only run if vnode or no-vnode is defined
         int defaultTokenCount = config.getInt("num_tokens");
@@ -1012,7 +1013,13 @@ protected IListen.Cancel startPolling(IInstance instance)
 
         protected boolean isCompleted()
         {
-            return instances.stream().allMatch(i -> !i.config().has(Feature.GOSSIP) || i.liveMemberCount() == instances.size());
+            return instances.stream().allMatch(i -> {
+                if (!i.config().has(Feature.GOSSIP))
+                    return true;
+
+                logger.info("Instance {} reports {} live members count, required {}", i, i.liveMemberCount(), instances.size());
+                return i.liveMemberCount() == instances.size();
+            });
         }
 
         protected String getMonitorTimeoutMessage()
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java b/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java
index 71cf5aaa5a3e..d3e917659054 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java
@@ -29,6 +29,7 @@
 import com.google.common.collect.Iterators;
 
 import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.statements.SelectStatement;
@@ -61,12 +62,13 @@ public SimpleQueryResult executeWithResult(String query, ConsistencyLevel consis
         return instance().sync(() -> unsafeExecuteInternal(query, consistencyLevel, boundValues)).call();
     }
 
+    @Override
     public Future<SimpleQueryResult> asyncExecuteWithTracingWithResult(UUID sessionId, String query, ConsistencyLevel consistencyLevelOrigin, Object... boundValues)
     {
         return instance.async(() -> {
             try
             {
-                Tracing.instance.newSession(TimeUUID.fromUuid(sessionId), Collections.emptyMap());
+                Tracing.instance.newSession(ClientState.forInternalCalls(), TimeUUID.fromUuid(sessionId), Collections.emptyMap());
                 return unsafeExecuteInternal(query, consistencyLevelOrigin, boundValues);
             }
             finally
@@ -135,7 +137,7 @@ public QueryResult executeWithPagingWithResult(String query, ConsistencyLevel co
             QueryOptions initialOptions = QueryOptions.create(toCassandraCL(consistencyLevel),
                                                               boundBBValues,
                                                               false,
-                                                              pageSize,
+                                                              PageSize.inRows(pageSize),
                                                               null,
                                                               null,
                                                               ProtocolVersion.CURRENT,
@@ -158,7 +160,7 @@ public boolean hasNext()
                     QueryOptions nextOptions = QueryOptions.create(toCassandraCL(consistencyLevel),
                                                                    boundBBValues,
                                                                    true,
-                                                                   pageSize,
+                                                                   PageSize.inRows(pageSize),
                                                                    rows.result.metadata.getPagingState(),
                                                                    null,
                                                                    ProtocolVersion.CURRENT,
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/CoordinatorHelper.java b/test/distributed/org/apache/cassandra/distributed/impl/CoordinatorHelper.java
index 414b30e05c61..70bed51816a4 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/CoordinatorHelper.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/CoordinatorHelper.java
@@ -24,6 +24,7 @@
 import java.util.List;
 
 import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
@@ -68,7 +69,7 @@ public static SimpleQueryResult unsafeExecuteInternal(String query, ConsistencyL
                                                  QueryOptions.create(toCassandraCL(commitConsistencyLevel),
                                                                      boundBBValues,
                                                                      false,
-                                                                     Integer.MAX_VALUE,
+                                                                     PageSize.NONE,
                                                                      null,
                                                                      toCassandraSerialCL(serialConsistencyLevel),
                                                                      ProtocolVersion.CURRENT,
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java b/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java
index 6a892c416bb3..28bd058499b8 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java
@@ -31,6 +31,7 @@
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.locator.AbstractNetworkTopologySnitch;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaCollection;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -39,6 +40,20 @@ public class DistributedTestSnitch extends AbstractNetworkTopologySnitch
     private static NetworkTopology mapping = null;
     private static final Map<InetAddressAndPort, InetSocketAddress> cache = new ConcurrentHashMap<>();
     private static final Map<InetSocketAddress, InetAddressAndPort> cacheInverse = new ConcurrentHashMap<>();
+    public static volatile InetAddressAndPort sortByProximityAddressOverride = null;
+
+    public <C extends ReplicaCollection<? extends C>> C sortedByProximity(InetAddressAndPort address, C unsortedAddress)
+    {
+        C s;
+        if (sortByProximityAddressOverride != null)
+        {
+            return super.sortedByProximity(sortByProximityAddressOverride, unsortedAddress);
+        }
+        else
+        {
+            return super.sortedByProximity(address, unsortedAddress);
+        }
+    }
 
     public static InetAddressAndPort toCassandraInetAddressAndPort(InetSocketAddress addressAndPort)
     {
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
index 3077326c10d1..4250dae8b401 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
@@ -60,6 +60,7 @@
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.concurrent.SharedExecutorPool;
 import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.YamlConfigurationLoader;
@@ -75,6 +76,9 @@
 import org.apache.cassandra.db.compaction.CompactionLogger;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.memtable.AbstractAllocatorMemtable;
+import org.apache.cassandra.db.virtual.SystemViewsKeyspace;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualSchemaKeyspace;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.Constants;
 import org.apache.cassandra.distributed.action.GossipHelper;
@@ -112,9 +116,11 @@
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.NoPayload;
 import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.schema.MigrationCoordinator;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaKeyspace;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.CassandraDaemon;
 import org.apache.cassandra.service.ClientState;
@@ -337,7 +343,9 @@ private void registerMockMessaging(ICluster<?> cluster)
         MessagingService.instance().outboundSink.add((message, to) -> {
             if (!internodeMessagingStarted)
             {
-                inInstancelogger.debug("Dropping outbound message {} to {} as internode messaging has not been started yet",
+                // to avoid NPE in case this is called before inInstancelogger is created
+                if (inInstancelogger != null)
+                    inInstancelogger.debug("Dropping outbound message {} to {} as internode messaging has not been started yet",
                                        message, to);
                 return false;
             }
@@ -491,8 +499,10 @@ private SerializableConsumer<Boolean> receiveMessageRunnable(IMessage message)
         return runOnCaller -> {
             if (!internodeMessagingStarted)
             {
-                inInstancelogger.debug("Dropping inbound message {} to {} as internode messaging has not been started yet",
-                             message, config().broadcastAddress());
+                // to avoid NPE in case this is called before inInstancelogger is created
+                if (inInstancelogger != null)
+                    inInstancelogger.debug("Dropping inbound message {} to {} as internode messaging has not been started yet", 
+                                           message, config().broadcastAddress());
                 return;
             }
             if (message.version() > MessagingService.current_version)
@@ -589,6 +599,8 @@ public void startup(ICluster cluster)
             inInstancelogger = LoggerFactory.getLogger(Instance.class);
             try
             {
+                JVMStabilityInspector.replaceKiller(new InstanceKiller(Instance.this::shutdown));
+
                 // org.apache.cassandra.distributed.impl.AbstractCluster.startup sets the exception handler for the thread
                 // so extract it to populate ExecutorFactory.Global
                 ExecutorFactory.Global.tryUnsafeSet(new ExecutorFactory.Default(Thread.currentThread().getContextClassLoader(), null, Thread.getDefaultUncaughtExceptionHandler()));
@@ -626,6 +638,19 @@ public void startup(ICluster cluster)
                 CassandraDaemon.logSystemInfo(inInstancelogger);
                 CommitLog.instance.start();
 
+                // MessagingService setup needs to be configured before any interaction with Schema because Schema
+                // uses MessagingService under the hood (it does not need to listen yet, but we need to set filters
+                // and mocks
+                if (!config.has(NETWORK))
+                {
+                    // Even though we don't use MessagingService, access the static SocketFactory
+                    // instance here so that we start the static event loop state
+                    //  -- not sure what that means?  SocketFactory.instance.getClass();
+                    registerMockMessaging(cluster);
+                }
+                registerInboundFilter(cluster);
+                registerOutboundFilter(cluster);
+
                 CassandraDaemon.getInstanceForTesting().runStartupChecks();
 
                 // We need to persist this as soon as possible after startup checks.
@@ -633,6 +658,9 @@ public void startup(ICluster cluster)
                 SystemKeyspace.persistLocalMetadata(config::hostId);
                 SystemKeyspaceMigrator41.migrate();
 
+                VirtualKeyspaceRegistry.instance.register(VirtualSchemaKeyspace.instance);
+                VirtualKeyspaceRegistry.instance.register(SystemViewsKeyspace.instance);
+
                 // Same order to populate tokenMetadata for the first time,
                 // see org.apache.cassandra.service.CassandraDaemon.setup
                 StorageService.instance.populateTokenMetadata();
@@ -658,13 +686,15 @@ public void startup(ICluster cluster)
                 // Replay any CommitLogSegments found on disk
                 try
                 {
-                    CommitLog.instance.recoverSegmentsOnDisk();
+                    CommitLog.instance.recoverSegmentsOnDisk(ColumnFamilyStore.FlushReason.STARTUP);
                 }
                 catch (IOException e)
                 {
                     throw new RuntimeException(e);
                 }
 
+                Nodes.getInstance().reload();
+
                 // Re-populate token metadata after commit log recover (new peers might be loaded onto system keyspace #10293)
                 StorageService.instance.populateTokenMetadata();
 
@@ -680,25 +710,11 @@ public void startup(ICluster cluster)
                 Verb.HINT_REQ.unsafeSetSerializer(DTestSerializer::new);
 
                 if (config.has(NETWORK))
-                {
                     MessagingService.instance().listen();
-                }
                 else
-                {
-                    // Even though we don't use MessagingService, access the static SocketFactory
-                    // instance here so that we start the static event loop state
-//                    -- not sure what that means?  SocketFactory.instance.getClass();
-                    registerMockMessaging(cluster);
-                }
-                registerInboundFilter(cluster);
-                registerOutboundFilter(cluster);
-                if (!config.has(NETWORK))
-                {
                     propagateMessagingVersions(cluster); // fake messaging needs to know messaging version for filters
-                }
-                internodeMessagingStarted = true;
 
-                JVMStabilityInspector.replaceKiller(new InstanceKiller(Instance.this::shutdown));
+                internodeMessagingStarted = true;
 
                 // TODO: this is more than just gossip
                 StorageService.instance.registerDaemon(CassandraDaemon.getInstanceForTesting());
@@ -717,14 +733,14 @@ public void startup(ICluster cluster)
                             throw new RuntimeException("Unable to bind, run the following in a termanl and try again:\nfor subnet in $(seq 0 5); do for id in $(seq 0 5); do sudo ifconfig lo0 alias \"127.0.$subnet.$id\"; done; done;", e);
                         throw e;
                     }
-                    StorageService.instance.removeShutdownHook();
+                    JVMStabilityInspector.removeShutdownHooks();
 
                     Gossiper.waitToSettle();
                 }
                 else
                 {
                     Schema.instance.startSync();
-                    Stream peers = cluster.stream().filter(instance -> ((IInstance) instance).isValid());
+                    Stream<IInstance> peers = cluster.stream().filter(instance -> ((IInstance) instance).isValid());
                     SystemKeyspace.setLocalHostId(config.hostId());
                     if (config.has(BLANK_GOSSIP))
                         peers.forEach(peer -> GossipHelper.statusToBlank((IInvokableInstance) peer).accept(this));
@@ -765,6 +781,7 @@ else if (cluster instanceof Cluster)
             }
             catch (Throwable t)
             {
+                startedAt.set(0);
                 if (t instanceof RuntimeException)
                     throw (RuntimeException) t;
                 throw new RuntimeException(t);
@@ -851,6 +868,9 @@ public Future<Void> shutdown()
     public Future<Void> shutdown(boolean graceful)
     {
         inInstancelogger.info("Shutting down instance {} / {}", config.num(), config.broadcastAddress().getHostString());
+        if (!CassandraRelevantProperties.UNSAFE_SYSTEM.getBoolean())
+            flush(SchemaKeyspace.metadata().name);
+
         Future<?> future = async((ExecutorService executor) -> {
             Throwable error = null;
 
@@ -860,7 +880,7 @@ public Future<Void> shutdown(boolean graceful)
 
             if (config.has(GOSSIP) || config.has(NETWORK))
             {
-                StorageService.instance.shutdownServer();
+                JVMStabilityInspector.removeShutdownHooks();
             }
 
             error = parallelRun(error, executor, StorageService.instance::disableAutoCompaction);
@@ -965,6 +985,11 @@ public Future<Void> shutdown(boolean graceful)
             {
                 super.shutdown();
                 startedAt.set(0L);
+
+                // when the instance is eventually stopped, we need to release buffer pools manually
+                // they are assumed to gone along with JVM, but this is not the case in dtests
+                BufferPools.forNetworking().unsafeReset(true);
+                BufferPools.forChunkCache().unsafeReset(true);
             }
         });
     }
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
index ae891cf0d592..62dcb2691777 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
@@ -30,6 +30,12 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import com.google.common.base.Splitter;
+import com.google.common.net.HostAndPort;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.vdurmont.semver4j.Semver;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.distributed.api.Feature;
@@ -39,8 +45,12 @@
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.SimpleSeedProvider;
 
+import org.apache.commons.lang3.ObjectUtils;
+
 public class InstanceConfig implements IInstanceConfig
 {
+    private static final Logger logger = LoggerFactory.getLogger(InstanceConfig.class);
+
     public final int num;
     private final int jmxPort;
 
@@ -72,6 +82,7 @@ private InstanceConfig(int num,
                            String commitlog_directory,
                            String hints_directory,
                            String cdc_raw_directory,
+                           String metadata_directory,
                            Collection<String> initial_token,
                            int storage_port,
                            int native_transport_port,
@@ -92,6 +103,7 @@ private InstanceConfig(int num,
                 .set("commitlog_directory", commitlog_directory)
                 .set("hints_directory", hints_directory)
                 .set("cdc_raw_directory", cdc_raw_directory)
+                .set("metadata_directory", metadata_directory)
                 .set("partitioner", "org.apache.cassandra.dht.Murmur3Partitioner")
                 .set("start_native_transport", true)
                 .set("concurrent_writes", 2)
@@ -317,6 +329,7 @@ public static InstanceConfig generate(int nodeNum,
                                   String.format("%s/node%d/commitlog", root, nodeNum),
                                   String.format("%s/node%d/hints", root, nodeNum),
                                   String.format("%s/node%d/cdc", root, nodeNum),
+                                  String.format("%s/node%d/metadata", root, nodeNum),
                                   tokens,
                                   provisionStrategy.storagePort(nodeNum),
                                   provisionStrategy.nativeTransportPort(nodeNum),
@@ -334,13 +347,28 @@ private static String[] datadirs(int datadirCount, Path root, int nodeNum)
 
     public InstanceConfig forVersion(Semver version)
     {
+        ParameterizedClass seedProviderConfig = (ParameterizedClass) params.get("seed_provider");
+
         // Versions before 4.0 need to set 'seed_provider' without specifying the port
-        if (UpgradeTestBase.v40.compareTo(version) < 0)
+        // the extra comparison to strict version is due to a bug in Semver (see STAR-871)
+        if (version.isGreaterThanOrEqualTo(UpgradeTestBase.v40) || version.isGreaterThanOrEqualTo(UpgradeTestBase.v40.toStrict()) || seedProviderConfig == null)
             return this;
-        else
-            return new InstanceConfig(this)
-                            .set("seed_provider", new ParameterizedClass(SimpleSeedProvider.class.getName(),
-                                                                         Collections.singletonMap("seeds", "127.0.0.1")));
+
+        assert ObjectUtils.equals(seedProviderConfig.class_name, SimpleSeedProvider.class.getName());
+        String seedsStr = seedProviderConfig.parameters.get("seeds");
+        assert seedsStr != null;
+        seedsStr = Splitter.on(',')
+                           .omitEmptyStrings()
+                           .trimResults()
+                           .splitToList(seedsStr)
+                           .stream()
+                           .map(str -> HostAndPort.fromString(str).getHost())
+                           .collect(Collectors.joining(","));
+
+        seedProviderConfig = new ParameterizedClass(seedProviderConfig.class_name, Collections.singletonMap("seeds", seedsStr));
+
+        logger.warn("Stripping ports from seed addresses because the version {} is < {}, new seeds list is: {}", version, UpgradeTestBase.v40, seedsStr);
+        return new InstanceConfig(this).set("seed_provider", seedProviderConfig);
     }
 
     public String toString()
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/InstanceKiller.java b/test/distributed/org/apache/cassandra/distributed/impl/InstanceKiller.java
index 38b045b381dc..4e24e8b76bcd 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/InstanceKiller.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/InstanceKiller.java
@@ -45,7 +45,7 @@ public static void clear()
     }
 
     @Override
-    protected void killCurrentJVM(Throwable t, boolean quiet)
+    public void killJVM(Throwable t, boolean quiet)
     {
         KILL_ATTEMPTS.incrementAndGet();
         onKill.accept(quiet);
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java b/test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java
index dd53ba48295a..21f403895efe 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java
@@ -46,7 +46,6 @@
 import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
 import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_RMI_TRANSPORT_TCP_THREADKEEPALIVETIME;
 import static org.apache.cassandra.distributed.api.Feature.JMX;
-import static org.apache.cassandra.utils.ReflectionUtils.clearMapField;
 
 public class IsolatedJmx
 {
@@ -203,7 +202,7 @@ public void stopJmx()
         // make sure to remove the reference to them when the instance is shutting down.
         // Additionally, we must make sure to only clear endpoints created by this instance
         // As clearning the entire map can cause issues with starting and stopping nodes mid-test.
-        clearMapField(TCPEndpoint.class, null, "localEndpoints", this::endpointCreateByThisInstance);
+//        clearMapField(TCPEndpoint.class, null, "localEndpoints", this::endpointCreateByThisInstance);
         Uninterruptibles.sleepUninterruptibly(2 * RMI_KEEPALIVE_TIME, TimeUnit.MILLISECONDS); // Double the keep-alive time to give Distributed GC some time to clean up
     }
 
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Query.java b/test/distributed/org/apache/cassandra/distributed/impl/Query.java
index 57aefe3a816b..b284a93d6882 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Query.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Query.java
@@ -23,6 +23,7 @@
 import java.util.List;
 
 import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.db.ConsistencyLevel;
@@ -75,7 +76,7 @@ public Object[][] call()
                                              QueryOptions.create(commitConsistency,
                                                                  boundBBValues,
                                                                  false,
-                                                                 Integer.MAX_VALUE,
+                                                                 PageSize.NONE,
                                                                  null,
                                                                  serialConsistency,
                                                                  ProtocolVersion.V4,
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/TracingUtil.java b/test/distributed/org/apache/cassandra/distributed/impl/TracingUtil.java
index 9c347e07a276..f62247463bb4 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/TracingUtil.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/TracingUtil.java
@@ -27,6 +27,8 @@
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.utils.TimeUUID;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS;
+
 /**
  * Utilities for accessing the system_traces table from in-JVM dtests
  */
@@ -103,4 +105,12 @@ public static List<TraceEntry> getTraces(AbstractCluster cluster, ConsistencyLev
         }
         return traces;
     }
+
+    // Set up the wait for tracing time system property, returning the previous value.
+    // Handles being called again to reset with the original value, replacing the null
+    // with the default value.
+    public static String setWaitForTracingEventTimeoutSecs(String timeoutInSeconds)
+    {
+        return WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS.setString(timeoutInSeconds == null ? "0" : timeoutInSeconds);
+    }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
index 4de180636c89..bd562c8dd6b3 100644
--- a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
+++ b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
@@ -37,6 +37,7 @@
 import org.apache.cassandra.locator.EndpointSnitchInfo;
 import org.apache.cassandra.locator.EndpointSnitchInfoMBean;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
+import org.apache.cassandra.metrics.CompactionMetrics;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.CacheService;
@@ -160,7 +161,34 @@ public CassandraMetricsRegistry.JmxTimerMBean getMessagingQueueWaitMetrics(Strin
     @Override
     public Object getCompactionMetric(String metricName)
     {
-        throw new UnsupportedOperationException();
+        CompactionMetrics metrics = CompactionManager.instance.getMetrics();
+        switch(metricName)
+        {
+            case "BytesCompacted":
+                return metrics.bytesCompacted;
+            case "CompletedTasks":
+                return metrics.completedTasks.getValue();
+            case "CompactionsAborted":
+                return metrics.compactionsAborted;
+            case "CompactionsReduced":
+                return metrics.compactionsReduced;
+            case "PendingTasks":
+                return metrics.pendingTasks.getValue();
+            case "PendingTasksByTableName":
+                return metrics.pendingTasksByTableName.getValue();
+            case "WriteAmplificationByTableName":
+                return metrics.writeAmplificationByTableName.getValue();
+            case "AggregateCompactions":
+                return metrics.aggregateCompactions.getValue();
+            case "MaxOverlapsMap":
+                return metrics.overlapsMap.getValue();
+            case "SSTablesDroppedFromCompaction":
+                return metrics.sstablesDropppedFromCompactions;
+            case "TotalCompactionsCompleted":
+                return metrics.totalCompactionsCompleted;
+            default:
+                throw new RuntimeException("Unknown compaction metric: " + metricName);
+        }
     }
 
     @Override
diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
index 5a8da8c7cf52..e52dfaf90a5f 100644
--- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
+++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
@@ -40,14 +40,10 @@
 import java.util.stream.Collectors;
 
 import com.google.common.util.concurrent.Futures;
-
-import org.apache.cassandra.distributed.api.Feature;
-import org.apache.cassandra.gms.ApplicationState;
-import org.apache.cassandra.gms.VersionedValue;
-import org.apache.cassandra.io.util.File;
 import org.junit.Assert;
 
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.api.Feature;
 import org.apache.cassandra.distributed.api.ICluster;
 import org.apache.cassandra.distributed.api.IInstance;
 import org.apache.cassandra.distributed.api.IInstanceConfig;
@@ -56,6 +52,9 @@
 import org.apache.cassandra.distributed.api.NodeToolResult;
 import org.apache.cassandra.distributed.impl.AbstractCluster;
 import org.apache.cassandra.distributed.impl.InstanceConfig;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.tools.SystemExitException;
 import org.apache.cassandra.utils.FBUtilities;
@@ -742,6 +741,19 @@ public static List<File> getDataDirectories(IInstance instance)
         return files;
     }
 
+    /**
+     * Get the metadata directory where the information about nodes is stored.
+     *
+     * @param instance to get the metadata directory for
+     * @return metadata directory
+     */
+    public static File getMetadataDirectory(IInstance instance)
+    {
+        IInstanceConfig conf = instance.config();
+        String d = conf.getString("metadata_directory");
+        return new File(d);
+    }
+
     /**
      * Get the commit log directory for the given instance.
      *
@@ -800,6 +812,7 @@ public static List<File> getDirectories(IInstance instance)
         out.add(getCommitLogDirectory(instance));
         out.add(getHintsDirectory(instance));
         out.add(getSavedCachesDirectory(instance));
+        // out.add(getMetadataDirectory(instance));
         return out;
     }
 
diff --git a/test/distributed/org/apache/cassandra/distributed/shared/WithProperties.java b/test/distributed/org/apache/cassandra/distributed/shared/WithProperties.java
index d17d3e6f3af4..fc1864d8a07c 100644
--- a/test/distributed/org/apache/cassandra/distributed/shared/WithProperties.java
+++ b/test/distributed/org/apache/cassandra/distributed/shared/WithProperties.java
@@ -52,6 +52,15 @@ public WithProperties set(CassandraRelevantProperties prop, String value)
         return set(prop, () -> prop.setString(value));
     }
 
+    public WithProperties clear(CassandraRelevantProperties prop)
+    {
+        return set(prop, () -> {
+            String prev = prop.getString();
+            prop.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+            return prev;
+        });
+    }
+
     public WithProperties set(CassandraRelevantProperties prop, String... values)
     {
         return set(prop, Arrays.asList(values));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/AbstractEncryptionOptionsImpl.java b/test/distributed/org/apache/cassandra/distributed/test/AbstractEncryptionOptionsImpl.java
index b48886743308..02c01f909322 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/AbstractEncryptionOptionsImpl.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/AbstractEncryptionOptionsImpl.java
@@ -48,8 +48,6 @@
 import io.netty.handler.ssl.SslHandler;
 import io.netty.util.concurrent.FutureListener;
 import org.apache.cassandra.config.EncryptionOptions;
-import org.apache.cassandra.distributed.Cluster;
-import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.security.ISslContextFactory;
 import org.apache.cassandra.security.SSLFactory;
 
@@ -335,27 +333,4 @@ void assertReceivedHandshakeException()
                               lastThrowable.getCause() instanceof  SSLHandshakeException);
         }
     }
-
-    /* Provde the cluster cannot start with the configured options */
-    void assertCannotStartDueToConfigurationException(Cluster cluster)
-    {
-        Throwable tr = null;
-        try
-        {
-            cluster.startup();
-        }
-        catch (Throwable maybeConfigException)
-        {
-            tr = maybeConfigException;
-        }
-
-        if (tr == null)
-        {
-            Assert.fail("Expected a ConfigurationException");
-        }
-        else
-        {
-            Assert.assertEquals(ConfigurationException.class.getName(), tr.getClass().getName());
-        }
-    }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/AuthTest.java b/test/distributed/org/apache/cassandra/distributed/test/AuthTest.java
index 67cf4c79fb52..886dc8c271d9 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/AuthTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/AuthTest.java
@@ -29,6 +29,7 @@
 import com.datastax.driver.core.PlainTextAuthProvider;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.Session;
+import com.datastax.driver.core.exceptions.AuthenticationException;
 import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
@@ -36,17 +37,23 @@
 import org.apache.cassandra.distributed.api.IInstanceConfig;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.api.IMessageFilters.Filter;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
 import org.apache.cassandra.distributed.api.TokenSupplier;
-import org.apache.cassandra.distributed.util.Auth;
 import org.apache.cassandra.locator.SimpleSeedProvider;
 import org.apache.cassandra.service.StorageService;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_DEFAULT_ROLE_SETUP;
+import static org.apache.cassandra.distributed.action.GossipHelper.withProperty;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.util.Auth.waitForExistingRoles;
 import static org.awaitility.Awaitility.await;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
 public class AuthTest extends TestBaseImpl
@@ -82,7 +89,7 @@ public void testZeroTimestampForDefaultRoleCreation() throws Exception
                                                                     .set("authenticator", "PasswordAuthenticator"))
                                         .start())
         {
-            Auth.waitForExistingRoles(cluster.get(1));
+            waitForExistingRoles(cluster.get(1));
 
             long writeTime = getPasswordWritetime(cluster.coordinator(1));
             // TIMESTAMP 0 in action
@@ -101,7 +108,7 @@ public void testZeroTimestampForDefaultRoleCreation() throws Exception
             Filter from = cluster.filters().allVerbs().outbound().drop();
 
             secondNode.startup();
-            Auth.waitForExistingRoles(secondNode);
+            waitForExistingRoles(secondNode);
 
             long passwordWritetimeOnSecondNode = getPasswordWritetime(cluster.coordinator(2));
 
@@ -155,6 +162,42 @@ public void testZeroTimestampForDefaultRoleCreation() throws Exception
         }
     }
 
+    @Test
+    public void testSkipDefaultRoleCreation() throws Exception
+    {
+        try (Cluster cluster = builder().withDCs(1)
+                                        .withNodes(1)
+                                        .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(1, 1))
+                                        .withConfig(config -> config.with(NETWORK, GOSSIP, NATIVE_PROTOCOL)
+                                                                    .with()
+                                                                    .set("authenticator", "PasswordAuthenticator"))
+                                        .createWithoutStarting()) // don't start the cluster yet as we need to set the skip_default_role_setup property first
+        {
+            withProperty(SKIP_DEFAULT_ROLE_SETUP, true,
+                         cluster::startup);
+
+            waitForExistingRoles(cluster.get(1));
+
+            long writeTime = getPasswordWritetime(cluster.coordinator(1));
+            // TIMESTAMP 1 when skip_default_role_setup is true
+            assertEquals(1, writeTime);
+
+            String defaultRoleQuery = "select is_superuser, can_login, salted_hash from system_auth.roles where role = 'cassandra'";
+            SimpleQueryResult result = cluster.coordinator(1).executeWithResult(defaultRoleQuery, ONE);
+            assertTrue(result.hasNext());
+            org.apache.cassandra.distributed.api.Row row = result.next();
+            assertFalse(row.get("is_superuser"));
+            assertFalse(row.get("can_login"));
+            assertEquals("", row.get("salted_hash"));
+
+            // make sure SU cannot really login
+            assertThrows(AuthenticationException.class, () -> doWithSession("127.0.0.1",
+                                                                            "datacenter1",
+                                                                            "cassandra",
+                                                                            session -> session.execute(defaultRoleQuery)));
+        }
+    }
+
     private IInvokableInstance getSecondNode(Cluster cluster)
     {
         IInstanceConfig config = cluster.newInstanceConfig();
diff --git a/test/distributed/org/apache/cassandra/distributed/test/CasWriteTest.java b/test/distributed/org/apache/cassandra/distributed/test/CasWriteTest.java
index d6bc4a39b2be..854bcf2da57f 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/CasWriteTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/CasWriteTest.java
@@ -63,7 +63,6 @@
 import org.apache.cassandra.exceptions.CasWriteUnknownResultException;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.notifications.SSTableMetadataChanged;
 import org.apache.cassandra.service.paxos.Ballot;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
@@ -339,7 +338,7 @@ public void testStaleCommitInSystemPaxos() throws InterruptedException
                 {
                     StatsMetadata oldMetadata = s.getSSTableMetadata();
                     s.mutateLevelAndReload(3);
-                    cfs.getCompactionStrategyManager().handleNotification(new SSTableMetadataChanged(s, oldMetadata), null);
+//                    cfs.getCompactionStrategyContainer().handleNotification(new SSTableMetadataChanged(s, oldMetadata), null);
                 }
                 catch (Throwable t)
                 {
@@ -380,7 +379,7 @@ public void testStaleCommitInSystemPaxos() throws InterruptedException
         {
             ((IInvokableInstance)cluster.get(k)).runOnInstance(() -> {
                 ColumnFamilyStore cfs = Keyspace.open("system").getColumnFamilyStore("paxos");
-                while (cfs.getCompactionStrategyManager().getEstimatedRemainingTasks() > 0)
+                while (cfs.getCompactionStrategy().getEstimatedRemainingTasks() > 0)
                 {
                     try { Thread.sleep(1000); }
                     catch (InterruptedException e) { throw new RuntimeException(e); }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/CompactionControllerConfigTest.java b/test/distributed/org/apache/cassandra/distributed/test/CompactionControllerConfigTest.java
new file mode 100644
index 000000000000..c864e8e59e20
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/CompactionControllerConfigTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.UnifiedCompactionContainer;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.db.compaction.unified.AdaptiveController;
+import org.apache.cassandra.db.compaction.unified.Controller;
+import org.apache.cassandra.db.compaction.unified.StaticController;
+import org.apache.cassandra.distributed.Cluster;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES;
+import static org.apache.cassandra.distributed.shared.FutureUtils.waitOn;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class CompactionControllerConfigTest extends TestBaseImpl
+{
+    @Test
+    public void storedAdaptiveCompactionOptionsTest() throws Throwable
+    {
+        try(Cluster cluster = init(Cluster.build(1).start()))
+        {
+            cluster.schemaChange(withKeyspace("CREATE KEYSPACE ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};"));
+            cluster.schemaChange(withKeyspace("CREATE TABLE ks.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compaction = " +
+                                              "{'class': 'UnifiedCompactionStrategy', " +
+                                              "'adaptive': 'true'};"));
+            cluster.schemaChange(withKeyspace("CREATE TABLE ks.tbl2 (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compaction = " +
+                                              "{'class': 'UnifiedCompactionStrategy', " +
+                                              "'adaptive': 'true'};"));
+            cluster.get(1).runOnInstance(() ->
+                                             {
+                                                 ColumnFamilyStore cfs = Keyspace.open("ks").getColumnFamilyStore("tbl");
+                                                 UnifiedCompactionContainer container = (UnifiedCompactionContainer) cfs.getCompactionStrategy();
+                                                 UnifiedCompactionStrategy ucs = (UnifiedCompactionStrategy) container.getStrategies().get(0);
+                                                 Controller controller = ucs.getController();
+                                                 assertTrue(controller instanceof AdaptiveController);
+                                                 //scaling parameter on L0 should be 0 to start
+                                                 assertEquals(0, controller.getScalingParameter(0));
+
+                                                 //manually write new scaling parameters and flushSizeBytes to see if they are picked up on restart
+                                                 int[] scalingParameters = new int[32];
+                                                 Arrays.fill(scalingParameters, 5);
+                                                 AdaptiveController.storeOptions("ks", "tbl", scalingParameters, 10 << 20);
+
+
+                                                 //write different scaling parameters to second table to make sure each table keeps its own configuration
+                                                 Arrays.fill(scalingParameters, 8);
+                                                 AdaptiveController.storeOptions("ks", "tbl2", scalingParameters, 10 << 20);
+                                             });
+            waitOn(cluster.get(1).shutdown());
+            cluster.get(1).startup();
+
+            cluster.get(1).runOnInstance(() ->
+                                         {
+                                             ColumnFamilyStore cfs = Keyspace.open("ks").getColumnFamilyStore("tbl");
+                                             UnifiedCompactionContainer container = (UnifiedCompactionContainer) cfs.getCompactionStrategy();
+                                             UnifiedCompactionStrategy ucs = (UnifiedCompactionStrategy) container.getStrategies().get(0);
+                                             Controller controller = ucs.getController();
+                                             assertTrue(controller instanceof AdaptiveController);
+                                             //when the node is restarted, it should see the new configuration that was manually written
+                                             assertEquals(5, controller.getScalingParameter(0));
+                                             assertEquals(10 << 20, controller.getFlushSizeBytes());
+
+                                             ColumnFamilyStore cfs2 = Keyspace.open("ks").getColumnFamilyStore("tbl2");
+                                             UnifiedCompactionContainer container2 = (UnifiedCompactionContainer) cfs2.getCompactionStrategy();
+                                             UnifiedCompactionStrategy ucs2 = (UnifiedCompactionStrategy) container2.getStrategies().get(0);
+                                             Controller controller2 = ucs2.getController();
+                                             assertTrue(controller2 instanceof AdaptiveController);
+                                             //when the node is restarted, it should see the new configuration that was manually written
+                                             assertEquals(8, controller2.getScalingParameter(0));
+                                             assertEquals(10 << 20, controller2.getFlushSizeBytes());
+                                         });
+        }
+    }
+
+    @Test
+    public void storedStaticCompactionOptionsTest() throws Throwable
+    {
+        try(Cluster cluster = init(Cluster.build(1).start()))
+        {
+            cluster.schemaChange(withKeyspace("CREATE KEYSPACE ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};"));
+            cluster.schemaChange(withKeyspace("CREATE TABLE ks.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compaction = " +
+                                              "{'class': 'UnifiedCompactionStrategy', " +
+                                              "'adaptive': 'false', " +
+                                              "'scaling_parameters': '0'};"));
+            cluster.get(1).runOnInstance(() ->
+                                         {
+                                             ColumnFamilyStore cfs = Keyspace.open("ks").getColumnFamilyStore("tbl");
+                                             UnifiedCompactionContainer container = (UnifiedCompactionContainer) cfs.getCompactionStrategy();
+                                             UnifiedCompactionStrategy ucs = (UnifiedCompactionStrategy) container.getStrategies().get(0);
+                                             Controller controller = ucs.getController();
+                                             assertTrue(controller instanceof StaticController);
+                                             //scaling parameter on L0 should be 0 to start
+                                             assertEquals(0, controller.getScalingParameter(0));
+
+                                             //manually write new flushSizeBytes to see if it is picked up on restart
+                                             int[] scalingParameters = new int[32];
+                                             Arrays.fill(scalingParameters, 0);
+                                             AdaptiveController.storeOptions("ks", "tbl", scalingParameters, 10 << 20);
+                                         });
+            waitOn(cluster.get(1).shutdown());
+            cluster.get(1).startup();
+
+            cluster.get(1).runOnInstance(() ->
+                                         {
+                                             ColumnFamilyStore cfs = Keyspace.open("ks").getColumnFamilyStore("tbl");
+                                             UnifiedCompactionContainer container = (UnifiedCompactionContainer) cfs.getCompactionStrategy();
+                                             UnifiedCompactionStrategy ucs = (UnifiedCompactionStrategy) container.getStrategies().get(0);
+                                             Controller controller = ucs.getController();
+                                             assertTrue(controller instanceof StaticController);
+                                             //when the node is restarted, it should see the new configuration that was manually written
+                                             assertEquals(10 << 20, controller.getFlushSizeBytes());
+                                         });
+        }
+    }
+
+    @Test
+    public void testStoreAndCleanupControllerConfig() throws Throwable
+    {
+        try(Cluster cluster = init(Cluster.build(1).start()))
+        {
+            cluster.schemaChange(withKeyspace("CREATE KEYSPACE ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};"));
+            cluster.schemaChange(withKeyspace("CREATE TABLE ks.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compaction = " +
+                                              "{'class': 'UnifiedCompactionStrategy', " +
+                                              "'adaptive': 'false', " +
+                                              "'scaling_parameters': '0'};"));
+            cluster.schemaChange(withKeyspace("CREATE KEYSPACE ks2 WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};"));
+            cluster.schemaChange(withKeyspace("CREATE TABLE ks2.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compaction = " +
+                                              "{'class': 'UnifiedCompactionStrategy', " +
+                                              "'adaptive': 'false', " +
+                                              "'scaling_parameters': '0'};"));
+
+            cluster.get(1).runOnInstance(() ->
+                                         {
+                                             //logs should show that scaling parameters and flush size are written to a file for each table
+                                             CompactionManager.storeControllerConfig();
+
+                                             //store controller config for a table that does not exist to see if it is removed by the cleanup method
+                                             int[] scalingParameters = new int[32];
+                                             Arrays.fill(scalingParameters, 5);
+                                             AdaptiveController.storeOptions("does_not", "exist", scalingParameters, 10 << 20);
+
+                                             //verify that the file was created
+                                             assert Controller.getControllerConfigPath("does_not", "exist").exists();
+
+                                             //cleanup method should remove the file corresponding to the table "does_not.exist"
+                                             CompactionManager.cleanupControllerConfig();
+
+                                             //verify that the file was deleted
+                                             assert !Controller.getControllerConfigPath("does_not", "exist").exists();
+
+                                         });
+
+        }
+    }
+
+    @Test
+    public void testStoreLongTableName() throws Throwable
+    {
+        try (Cluster cluster = init(Cluster.build(1).start()))
+        {
+            cluster.get(1).runOnInstance(() ->
+                                         {
+                                             CompactionManager.storeControllerConfig();
+
+                                             // try to store controller config for a table with a long name
+                                             String keyspaceName = "g38373639353166362d356631322d343864652d393063362d653862616534343165333764_tpch";
+                                             String longTableName = "test_create_k8yq1r75bpzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
+                                             int[] scalingParameters = new int[32];
+                                             Arrays.fill(scalingParameters, 5);
+                                             AdaptiveController.storeOptions(keyspaceName, longTableName, scalingParameters, 10 << 20);
+
+                                             // verify that the file wasn't created
+                                             assert !Controller.getControllerConfigPath(keyspaceName, longTableName).exists();
+                                         });
+        }
+    }
+
+    @Test
+    public void testVectorControllerConfig() throws Throwable
+    {
+        vectorControllerConfig(true);
+        vectorControllerConfig(false);
+    }
+
+
+    public void vectorControllerConfig(boolean vectorOverride) throws Throwable
+    {
+        UCS_OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES.setBoolean(vectorOverride);
+        try(Cluster cluster = init(Cluster.build(1).start()))
+        {
+            cluster.schemaChange(withKeyspace("CREATE KEYSPACE ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};"));
+            cluster.schemaChange(withKeyspace("CREATE TABLE ks.tbl (pk int, ck int, val vector<float, 2>, PRIMARY KEY (pk, ck)) WITH compaction = " +
+                                              "{'class': 'UnifiedCompactionStrategy', " +
+                                              "'adaptive': 'false', " +
+                                              "'scaling_parameters': '0'};"));
+            cluster.schemaChange(withKeyspace("CREATE TABLE ks.tbl2 (pk int, ck int, PRIMARY KEY (pk, ck)) WITH compaction = " +
+                                              "{'class': 'UnifiedCompactionStrategy', " +
+                                              "'adaptive': 'false', " +
+                                              "'scaling_parameters': '0'};"));
+
+            cluster.get(1).runOnInstance(() ->
+                                         {
+                                             ColumnFamilyStore cfs = Keyspace.open("ks").getColumnFamilyStore("tbl");
+                                             UnifiedCompactionContainer container = (UnifiedCompactionContainer) cfs.getCompactionStrategy();
+                                             UnifiedCompactionStrategy ucs = (UnifiedCompactionStrategy) container.getStrategies().get(0);
+                                             Controller controller = ucs.getController();
+                                             // ucs config should be set to the vector config
+                                             assertEquals(vectorOverride ? Controller.DEFAULT_VECTOR_TARGET_SSTABLE_SIZE
+                                                                         : Controller.DEFAULT_TARGET_SSTABLE_SIZE,
+                                                          controller.getTargetSSTableSize());
+                                             // but any property set in the table compaction config should override the vector config
+                                             assertEquals(0, controller.getScalingParameter(0));
+
+                                             ColumnFamilyStore cfs2 = Keyspace.open("ks").getColumnFamilyStore("tbl2");
+                                             UnifiedCompactionContainer container2 = (UnifiedCompactionContainer) cfs2.getCompactionStrategy();
+                                             UnifiedCompactionStrategy ucs2 = (UnifiedCompactionStrategy) container2.getStrategies().get(0);
+                                             Controller controller2 = ucs2.getController();
+                                             // since tbl2 does not have a vectorType the ucs config should not be set to the vector config
+                                             assertEquals(Controller.DEFAULT_TARGET_SSTABLE_SIZE, controller2.getTargetSSTableSize());
+                                             assertEquals(0, controller2.getScalingParameter(0));
+                                         });
+            cluster.schemaChange(withKeyspace("ALTER TABLE ks.tbl2 ADD val vector<float, 2>;"));
+            cluster.get(1).runOnInstance(() ->
+                                         {
+                                             ColumnFamilyStore cfs2 = Keyspace.open("ks").getColumnFamilyStore("tbl2");
+                                             UnifiedCompactionContainer container2 = (UnifiedCompactionContainer) cfs2.getCompactionStrategy();
+                                             UnifiedCompactionStrategy ucs2 = (UnifiedCompactionStrategy) container2.getStrategies().get(0);
+                                             Controller controller2 = ucs2.getController();
+                                             // a vector was added to tbl2 so it should now have the vector config
+                                             assertEquals(vectorOverride ? Controller.DEFAULT_VECTOR_TARGET_SSTABLE_SIZE
+                                                                         : Controller.DEFAULT_TARGET_SSTABLE_SIZE,
+                                                          controller2.getTargetSSTableSize());
+                                             assertEquals(0, controller2.getScalingParameter(0));
+                                         });
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java
index 099f87dd40bd..fa999fa56d99 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java
@@ -26,6 +26,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.collect.ImmutableMap;
+
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileStoreUtils;
 import org.apache.cassandra.io.util.PathUtils;
@@ -37,7 +38,7 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.ActiveCompactions;
+import org.apache.cassandra.db.compaction.ActiveOperations;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 
@@ -112,7 +113,7 @@ public static class BB
         static File sstableDir;
         public static void install(ClassLoader cl, Integer node)
         {
-            new ByteBuddy().rebase(ActiveCompactions.class)
+            new ByteBuddy().rebase(ActiveOperations.class)
                            .method(named("estimatedRemainingWriteBytes"))
                            .intercept(MethodDelegation.to(BB.class))
                            .make()
diff --git a/test/distributed/org/apache/cassandra/distributed/test/DisableBinaryTest.java b/test/distributed/org/apache/cassandra/distributed/test/DisableBinaryTest.java
index a5c0b1a5c33b..eaff71153723 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/DisableBinaryTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/DisableBinaryTest.java
@@ -109,6 +109,16 @@ public void testFinishInProgressQueries() throws Throwable
         finally
         {
             executor.shutdown();
+            try
+            {
+                boolean shutdown = executor.awaitTermination(10, TimeUnit.SECONDS);
+                if (!shutdown)
+                    throw new AssertionError("Executor did not terminate");
+            }
+            catch (InterruptedException e)
+            {
+                throw new RuntimeException(e);
+            }
         }
     }
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/DistributedRepairUtils.java b/test/distributed/org/apache/cassandra/distributed/test/DistributedRepairUtils.java
index c71a611c012c..16e9016661bb 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/DistributedRepairUtils.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/DistributedRepairUtils.java
@@ -190,7 +190,8 @@ public static void assertNoSSTableLeak(ICluster<IInvokableInstance> cluster, Str
                     if (session != null && !session.isCompleted())
                         continue;
                     // The session is complete, yet the sstable is not updated... is this still pending in compaction?
-                    if (cfs.getCompactionStrategyManager().hasPendingRepairSSTable(pendingRepair, sstable))
+//                    if (cfs.getCompactionStrategyManager().hasPendingRepairSSTable(pendingRepair, sstable))
+                    if (cfs.hasPendingRepairSSTables(pendingRepair))
                         continue;
                     // compaction does not know about the pending repair... race condition since this check started?
                     if (sstable.getSSTableMetadata().pendingRepair == null)
diff --git a/test/distributed/org/apache/cassandra/distributed/test/DropUDTWithRestartTest.java b/test/distributed/org/apache/cassandra/distributed/test/DropUDTWithRestartTest.java
new file mode 100644
index 000000000000..d19abc2183f8
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/DropUDTWithRestartTest.java
@@ -0,0 +1,603 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.BiFunction;
+import java.util.function.IntFunction;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICoordinator;
+import org.apache.cassandra.distributed.api.IInstance;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.tools.SSTableExport;
+import org.apache.cassandra.tools.ToolRunner;
+import org.apache.cassandra.utils.Collectors3;
+import org.assertj.core.api.Assertions;
+import org.json.simple.JSONObject;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static java.util.Arrays.asList;
+import static org.apache.cassandra.config.DatabaseDescriptor.getCommitLogLocation;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class DropUDTWithRestartTest extends TestBaseImpl
+{
+    private final static Logger logger = LoggerFactory.getLogger(DropUDTWithRestartTest.class);
+
+    private final static Path TEST_DATA_UDT_PATH = Paths.get("test/data/udt");
+    private final static Path CC40_PRODUCT_PATH = TEST_DATA_UDT_PATH.resolve("cc40");
+    private final static Path CC50_PRODUCT_PATH = TEST_DATA_UDT_PATH.resolve("cc50");
+    private final static Path DSE_PRODUCT_PATH = TEST_DATA_UDT_PATH.resolve("dse");
+    private final static Path THIS_PRODUCT_PATH = CC50_PRODUCT_PATH;
+    private final static String COMMITLOG_DIR = "commitlog";
+    private final static String KS = "ks";
+    private final static String SCHEMA_TXT = "schema.txt";
+    private final static String SCHEMA0_TXT = "schema0.txt";
+    private final static String DATA_JSON = "data.json";
+
+    private Cluster startCluster() throws IOException
+    {
+        Cluster cluster = Cluster.build(1).withConfig(config -> config.set("auto_snapshot", "false")
+                                                                      .set("uuid_sstable_identifiers_enabled", "false")
+                                                                      .with(NATIVE_PROTOCOL)).start();
+        cluster.setUncaughtExceptionsFilter(t -> {
+            String cause = Optional.ofNullable(t.getCause()).map(c -> c.getClass().getName()).orElse("");
+            return t.getClass().getName().equals(FSWriteError.class.getName()) && cause.equals(AccessDeniedException.class.getName());
+        });
+        return cluster;
+    }
+
+    @Test
+    public void mergeDataFromSSTableAndCommitLogWithDroppedColumnTest() throws Throwable
+    {
+        try (Cluster cluster = startCluster())
+        {
+            // Create tables, populate them with the first dataset, drop complex column (which follows flushing),
+            // and then populate with the second dataset. This way we will have data on disk and in the memtable.
+            // Finally record query results.
+            IInvokableInstance node = cluster.get(1);
+            node.executeInternal("DROP KEYSPACE IF EXISTS " + KS);
+            node.executeInternal("CREATE KEYSPACE " + KS + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+            createTables(node);
+            cluster.disableAutoCompaction(KS);
+
+            // let's have some data in sstables
+            insertData(node, 0, true);
+            insertData(node, 256, true);
+
+            // then drop the complex column
+            dropComplexColumn(node);
+
+            // insert some other data
+            insertData(node, 128, false);
+            insertData(node, 256 + 17, false);
+
+            // and see what we have
+            Map<String, List<List<Object>>> data0 = selectData(node);
+            Map<String, List<List<Object>>> cqlData0 = selectCQLData(node);
+
+            assertThat(cqlData0).isEqualTo(data0);
+
+            // make sure we have the same after flushing and compacting
+            node.flush(KS);
+            assertThat(selectData(node)).isEqualTo(data0);
+            for (String table : data0.keySet())
+                node.forceCompact(KS, table);
+            assertThat(selectData(node)).isEqualTo(data0);
+
+            // Create tables, populate them with the first dataset, block data dir and drop complex column (prevent
+            // flushing so that the data stays in the commit log), restart the node to replay the commit log, populate
+            // the tables with the second data set, and finally record query results.
+            node.executeInternal("DROP KEYSPACE " + KS);
+            node.executeInternal("CREATE KEYSPACE " + KS + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+            createTables(node);
+            List<Path> dataDirs = getDataDirectories(node);
+
+            cluster.disableAutoCompaction(KS);
+            blockFlushing(dataDirs);
+            insertData(node, 0, true);
+            insertData(node, 256, true);
+            try
+            {
+                dropComplexColumn(node);
+            }
+            finally
+            {
+                unblockFlushing(dataDirs);
+            }
+            node.shutdown(true).get(10, TimeUnit.SECONDS);
+
+            node.startup();
+            insertData(node, 128, false);
+            insertData(node, 256 + 17, false);
+
+            // eventually we expect that the result sets from both runs are the same
+            assertThat(selectData(node)).isEqualTo(data0);
+
+            // make sure we have the same after flushing and compacting
+            node.flush(KS);
+            node.shutdown(true).get(10, TimeUnit.SECONDS);
+            node.startup();
+            assertThat(selectData(node)).isEqualTo(data0);
+
+            for (String table : data0.keySet())
+                node.forceCompact(KS, table);
+            assertThat(selectData(node)).isEqualTo(data0);
+        }
+    }
+
+    private static List<Path> getDataDirectories(IInvokableInstance node)
+    {
+        return node.callOnInstance(() -> Keyspace.open(KS).getColumnFamilyStores().stream().map(cfs -> cfs.getDirectories().getDirectoryForNewSSTables().toPath()).collect(Collectors.toList()));
+    }
+
+    /**
+     * This is actually not a test - it is used to generate data files to be used by {@code loadCommitLogAndSSTablesWithDroppedColumnTest*}.
+     * Those files should be populated across different products between which we want to verify the compatibility.
+     */
+    @Test
+    @Ignore
+    public void storeCommitLogAndSSTablesWithDroppedColumn() throws Throwable
+    {
+        Files.createDirectories(THIS_PRODUCT_PATH);
+        try (Cluster cluster = startCluster())
+        {
+            IInvokableInstance node = cluster.get(1);
+            node.executeInternal("DROP KEYSPACE IF EXISTS " + KS);
+            node.executeInternal("CREATE KEYSPACE " + KS + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+            createTables(node);
+            cluster.disableAutoCompaction(KS);
+
+            List<Path> dataDirs = getDataDirectories(node);
+            Path commitLogDir = node.callOnInstance(() -> getCommitLogLocation().toPath());
+
+            Map<String, String> schema0 = getSchemaDesc(node);
+            Files.writeString(THIS_PRODUCT_PATH.resolve(SCHEMA0_TXT),
+                              String.join(";\n", schema0.values()).replaceAll(";;", ";"),
+                              UTF_8,
+                              CREATE, TRUNCATE_EXISTING);
+
+            insertData(node, 0, true);
+            insertData(node, 256, true);
+            node.flush(KS);
+
+            blockFlushing(dataDirs);
+            try
+            {
+                dropComplexColumn(node);
+                insertData(node, 128, false);
+                insertData(node, 256 + 17, false);
+
+                Map<String, String> schema1 = getSchemaDesc(node);
+                Files.writeString(THIS_PRODUCT_PATH.resolve(SCHEMA_TXT),
+                                  String.join(";\n", schema1.values()).replaceAll(";;", ";"),
+                                  UTF_8,
+                                  CREATE, TRUNCATE_EXISTING);
+
+                node.shutdown(true).get(10, TimeUnit.SECONDS);
+
+                Path clTargetPath = THIS_PRODUCT_PATH.resolve(COMMITLOG_DIR);
+                Files.createDirectories(clTargetPath);
+                PathUtils.deleteContent(clTargetPath);
+                FileUtils.copyDirectory(commitLogDir.toFile(), clTargetPath.toFile());
+
+                Path ksTargetPath = THIS_PRODUCT_PATH.resolve(KS);
+                Files.createDirectories(ksTargetPath);
+                PathUtils.deleteContent(ksTargetPath);
+                for (Path dir : dataDirs)
+                {
+                    String name = dir.getFileName().toString();
+                    Path targetDir = ksTargetPath.resolve(name);
+                    Files.createDirectories(targetDir);
+                    FileUtils.copyDirectory(dir.toFile(), targetDir.toFile(), pathname -> !pathname.toString().endsWith(".log"));
+                }
+            }
+            finally
+            {
+                unblockFlushing(dataDirs);
+            }
+
+            node.startup();
+            node.flush(KS);
+            Map<String, List<List<Object>>> data = selectData(node);
+            Files.writeString(THIS_PRODUCT_PATH.resolve(DATA_JSON), JSONObject.toJSONString(data), UTF_8, CREATE, TRUNCATE_EXISTING);
+        }
+    }
+
+    @Test
+    public void loadCommitLogAndSSTablesWithDroppedColumnTestCC40() throws Exception
+    {
+        loadCommitLogAndSSTablesWithDroppedColumnTest(CC40_PRODUCT_PATH);
+    }
+
+    @Test
+    public void loadCommitLogAndSSTablesWithDroppedColumnTestCC50() throws Exception
+    {
+        loadCommitLogAndSSTablesWithDroppedColumnTest(CC50_PRODUCT_PATH);
+    }
+
+    @Test
+    public void loadCommitLogAndSSTablesWithDroppedColumnTestDSE() throws Exception
+    {
+        loadCommitLogAndSSTablesWithDroppedColumnTest(DSE_PRODUCT_PATH);
+    }
+
+    private void loadCommitLogAndSSTablesWithDroppedColumnTest(Path productPath) throws IOException, ExecutionException, InterruptedException, TimeoutException
+    {
+        try (Cluster cluster = startCluster())
+        {
+            IInvokableInstance node = cluster.get(1);
+            node.executeInternal("DROP KEYSPACE IF EXISTS " + KS);
+            node.executeInternal("CREATE KEYSPACE " + KS + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+
+            for (String stmt : Files.readString(productPath.resolve(SCHEMA_TXT), UTF_8).split(";"))
+            {
+                if (!stmt.isBlank())
+                {
+                    logger.info("Executing: {}", stmt);
+                    node.executeInternal(stmt);
+                }
+            }
+
+            cluster.disableAutoCompaction(KS);
+
+            List<Path> dataDirs = getDataDirectories(node);
+            Path commitLogDir = node.callOnInstance(() -> getCommitLogLocation().toPath());
+
+            node.shutdown(true).get(10, TimeUnit.SECONDS);
+
+            Path commitLogSourcePath = productPath.resolve(COMMITLOG_DIR);
+            FileUtils.copyDirectory(commitLogSourcePath.toFile(), commitLogDir.toFile());
+
+            Path ksSourcePath = productPath.resolve(KS);
+            for (Path dir : dataDirs)
+            {
+                String name = dir.getFileName().toString();
+                Path sourceDir = ksSourcePath.resolve(name);
+                FileUtils.copyDirectory(sourceDir.toFile(), dir.toFile());
+            }
+
+            logger.info("Restarting node");
+            node.startup();
+            Map<String, List<List<Object>>> data1 = selectData(node);
+
+            String jsonData0 = Files.readString(productPath.resolve(DATA_JSON), UTF_8);
+            String jsonData1 = JSONObject.toJSONString(data1);
+            assertThat(jsonData1).isEqualTo(jsonData0);
+
+            node.flush(KS);
+            node.shutdown(true).get(10, TimeUnit.SECONDS);
+            node.startup();
+
+            assertThat(selectData(node)).isEqualTo(data1);
+
+            for (String table : data1.keySet())
+                node.forceCompact(KS, table);
+
+            assertThat(selectData(node)).isEqualTo(data1);
+        }
+    }
+
+    private Map<String, String> getSchemaDesc(IInvokableInstance node)
+    {
+        return Arrays.stream(node.executeInternal("DESCRIBE " + KS + " WITH INTERNALS"))
+                     .filter(r -> r[1].equals("table") || r[1].equals("type"))
+                     .collect(Collectors3.toImmutableMap(r -> r[2].toString(),
+                                                         r -> Arrays.stream(r[3].toString().split("\\n"))
+                                                                    .filter(s -> !s.strip().startsWith("AND") || s.contains("DROPPED COLUMN RECORD"))
+                                                                    .collect(Collectors.joining("\n"))));
+    }
+
+    private static String udtValue(int i, List<Integer> bits, BiFunction<Integer, Integer, String> vals)
+    {
+        List<String> cols = asList("foo", "bar", "baz");
+        ArrayList<String> udtVals = new ArrayList<>();
+        for (int j = 0; j < bits.size(); j++)
+        {
+            if ((i & bits.get(j)) != 0)
+                udtVals.add(cols.get(j) + ": " + vals.apply(i, j));
+        }
+        return '{' + String.join(", ", udtVals) + '}';
+    }
+
+    private static String tupleValue(int i, List<Integer> bits, BiFunction<Integer, Integer, String> vals)
+    {
+        ArrayList<String> tupleVals = new ArrayList<>();
+        for (int j = 0; j < bits.size(); j++)
+        {
+            if ((i & bits.get(j)) != 0)
+                tupleVals.add(vals.apply(i, j));
+            else
+                tupleVals.add("null");
+        }
+        return '(' + String.join(", ", tupleVals) + ')';
+    }
+
+    private static String genInsert(int pk, int i, List<Integer> bits, BiFunction<Integer, Integer, String> vals)
+    {
+        List<String> cols = asList("a_int", "b_complex", "c_int");
+        ArrayList<String> c = new ArrayList<>();
+        ArrayList<String> v = new ArrayList<>();
+        for (int j = 0; j < bits.size(); j++)
+        {
+            if ((i & bits.get(j)) != 0)
+            {
+                c.add(cols.get(j));
+                v.add(vals.apply(i, j));
+            }
+        }
+        if (c.isEmpty())
+            return String.format("(pk) VALUES (%d)", pk);
+        else
+            return String.format("(pk, %s) VALUES (%d, %s)", String.join(", ", c), pk, String.join(", ", v));
+    }
+
+    private static BiFunction<Integer, Integer, String> valsFunction(IntFunction<String> nonIntFunction)
+    {
+        return (i, j) -> {
+            if (j == 0)
+                return Integer.toString(i);
+            if (j == 1)
+                return nonIntFunction.apply(i);
+            if (j == 2)
+                return Integer.toString(i * 2);
+            assert false;
+            return null;
+        };
+    }
+
+    private static BiFunction<Integer, Integer, String> valsFunction()
+    {
+        return (i, j) -> {
+            if (j == 0)
+                return Integer.toString(i);
+            if (j == 1)
+                return String.format("'bar%d'", i);
+            if (j == 2)
+                return Integer.toString(i * 2);
+            assert false;
+            return null;
+        };
+    }
+
+    private void insertData(IInstance node, int offset, boolean withComplex)
+    {
+        for (int pk = offset; pk < offset + (1 << 5); pk++)
+        {
+            int i = withComplex ? (pk - offset) : (pk - offset) & ~(2 + 4 + 8);
+            node.executeInternal("INSERT INTO " + KS + ".tab1_udt1 " + genInsert(pk, i, asList(1, 2 + 4 + 8, 16), valsFunction(j -> udtValue(j, asList(2, 4, 8), valsFunction()))));
+            node.executeInternal("INSERT INTO " + KS + ".tab2_frozen_udt1 " + genInsert(pk, i, asList(1, 2 + 4 + 8, 16), valsFunction(j -> udtValue(j, asList(2, 4, 8), valsFunction()))));
+            node.executeInternal("INSERT INTO " + KS + ".tab5_tuple " + genInsert(pk, i, asList(1, 2 + 4 + 8, 16), valsFunction(j -> tupleValue(j, asList(2, 4, 8), valsFunction()))));
+            node.executeInternal("INSERT INTO " + KS + ".tab6_frozen_tuple " + genInsert(pk, i, asList(1, 2 + 4 + 8, 16), valsFunction(j -> tupleValue(j, asList(2, 4, 8), valsFunction()))));
+        }
+
+        for (int pk = offset; pk < offset + (1 << 7); pk++)
+        {
+            int i = withComplex ? (pk - offset) : (pk - offset) & ~(2 + 4 + 8 + 16 + 32);
+            node.executeInternal("INSERT INTO " + KS + ".tab4_frozen_udt2 " + genInsert(pk, i, asList(1, 2 + 4 + 8 + 16 + 32, 64),
+                                                                                        valsFunction(j -> udtValue(j, asList(2, 4 + 8 + 16, 32), valsFunction(k -> udtValue(k, asList(4, 8, 16), valsFunction()))))));
+            node.executeInternal("INSERT INTO " + KS + ".tab7_tuple_with_udt " + genInsert(pk, i, asList(1, 2 + 4 + 8 + 16 + 32, 64),
+                                                                                           valsFunction(j -> tupleValue(j, asList(2, 4 + 8 + 16, 32), valsFunction(k -> udtValue(k, asList(4, 8, 16), valsFunction()))))));
+            node.executeInternal("INSERT INTO " + KS + ".tab8_frozen_tuple_with_udt " + genInsert(pk, i, asList(1, 2 + 4 + 8 + 16 + 32, 64),
+                                                                                                  valsFunction(j -> tupleValue(j, asList(2, 4 + 8 + 16, 32), valsFunction(k -> udtValue(k, asList(4, 8, 16), valsFunction()))))));
+            node.executeInternal("INSERT INTO " + KS + ".tab9_udt_with_tuple " + genInsert(pk, i, asList(1, 2 + 4 + 8 + 16 + 32, 64),
+                                                                                           valsFunction(j -> udtValue(j, asList(2, 4 + 8 + 16, 32), valsFunction(k -> tupleValue(k, asList(4, 8, 16), valsFunction()))))));
+            node.executeInternal("INSERT INTO " + KS + ".tab10_frozen_udt_with_tuple " + genInsert(pk, i, asList(1, 2 + 4 + 8 + 16 + 32, 64),
+                                                                                                   valsFunction(j -> udtValue(j, asList(2, 4 + 8 + 16, 32), valsFunction(k -> tupleValue(k, asList(4, 8, 16), valsFunction()))))));
+        }
+    }
+
+    private static void dropComplexColumn(IInvokableInstance node)
+    {
+        List<String> tables = node.callOnInstance(() -> Schema.instance.getKeyspaceMetadata(KS).tables.stream().map(t -> t.name).collect(Collectors.toList()));
+        for (String table : tables)
+            node.executeInternal("ALTER TABLE " + KS + "." + table + " DROP b_complex");
+    }
+
+    private Map<String, List<List<Object>>> selectData(IInvokableInstance node)
+    {
+        Map<String, List<List<Object>>> results = new HashMap<>();
+        List<String> tables = node.callOnInstance(() -> Schema.instance.getKeyspaceMetadata(KS).tables.stream().map(t -> t.name).collect(Collectors.toList()));
+        for (String table : tables)
+        {
+            Object[][] rows = node.executeInternal("SELECT * FROM " + KS + "." + table);
+            Arrays.sort(rows, Comparator.comparing(a -> ((Integer) a[0])));
+            results.put(table, Arrays.stream(rows).map(Arrays::asList).collect(Collectors.toList()));
+        }
+        return results;
+    }
+
+    private Map<String, List<List<Object>>> selectCQLData(IInvokableInstance node)
+    {
+        Map<String, List<List<Object>>> results = new HashMap<>();
+        List<String> tables = node.callOnInstance(() -> Schema.instance.getKeyspaceMetadata(KS).tables.stream().map(t -> t.name).collect(Collectors.toList()));
+        try (com.datastax.driver.core.Cluster cluster = com.datastax.driver.core.Cluster.builder().addContactPoint(node.broadcastAddress().getHostString()).build();
+             Session session = cluster.connect())
+        {
+            for (String table : tables)
+            {
+                ResultSet rs = session.execute("SELECT * FROM " + KS + "." + table);
+                assertThat(rs.getColumnDefinitions().contains("b_complex")).isFalse();
+                List<List<Object>> rows = rs.all().stream().map(r -> Arrays.<Object>asList(r.get("pk", Integer.class), r.get("a_int", Integer.class), r.get("c_int", Integer.class)))
+                                            .sorted(Comparator.comparing(a -> ((Integer) a.get(0))))
+                                            .collect(Collectors.toList());
+                results.put(table, rows);
+            }
+        }
+        return results;
+    }
+
+    private static void createTables(IInvokableInstance node)
+    {
+        node.executeInternal("CREATE TYPE " + KS + ".udt1(foo int, bar text, baz int)");
+        node.executeInternal("CREATE TYPE " + KS + ".udt2(foo int, bar udt1, baz int)");
+        node.executeInternal("CREATE TYPE " + KS + ".udt3(foo int, bar tuple<int, text, int>, baz int)");
+
+        node.executeInternal("CREATE TABLE " + KS + ".tab1_udt1                   (pk int PRIMARY KEY, a_int int, b_complex udt1, c_int int) WITH ID = 513f2627-9356-41c4-a379-7ad42be97432");
+        node.executeInternal("CREATE TABLE " + KS + ".tab2_frozen_udt1            (pk int PRIMARY KEY, a_int int, b_complex frozen<udt1>, c_int int) WITH ID = 450f91fe-7c47-41c9-97bf-fdad854fa7e5");
+        Assertions.assertThatExceptionOfType(RuntimeException.class).isThrownBy(
+          () -> node.executeInternal("CREATE TABLE " + KS + ".tab3_udt2             (pk int PRIMARY KEY, a_int int, b_complex udt2, c_int int) WITH ID = b613aee8-645c-4384-90d2-fc9e82fb1a59"));
+        node.executeInternal("CREATE TABLE " + KS + ".tab4_frozen_udt2            (pk int PRIMARY KEY, a_int int, b_complex frozen<udt2>, c_int int) WITH ID = 9c03c71c-6775-4357-9173-0f8808901afa");
+        node.executeInternal("CREATE TABLE " + KS + ".tab5_tuple                  (pk int PRIMARY KEY, a_int int, b_complex tuple<int, text, int>, c_int int) WITH ID = 90826dd3-8437-4585-9de4-15908236687f");
+        node.executeInternal("CREATE TABLE " + KS + ".tab6_frozen_tuple           (pk int PRIMARY KEY, a_int int, b_complex frozen<tuple<int, text, int>>, c_int int) WITH ID = 54185f9a-a6fd-487c-abc3-c01bd5835e48");
+        node.executeInternal("CREATE TABLE " + KS + ".tab7_tuple_with_udt         (pk int PRIMARY KEY, a_int int, b_complex tuple<int, udt1, int>, c_int int) WITH ID = 4e78f403-7b63-4e0d-a231-42e42cba7cb5");
+        node.executeInternal("CREATE TABLE " + KS + ".tab8_frozen_tuple_with_udt  (pk int PRIMARY KEY, a_int int, b_complex frozen<tuple<int, udt1, int>>, c_int int) WITH ID = 8660f235-0816-4019-9cc9-1798fa7beb17");
+        node.executeInternal("CREATE TABLE " + KS + ".tab9_udt_with_tuple         (pk int PRIMARY KEY, a_int int, b_complex udt3, c_int int) WITH ID = f670fd5a-8145-4669-aceb-75667c000ea6");
+        node.executeInternal("CREATE TABLE " + KS + ".tab10_frozen_udt_with_tuple (pk int PRIMARY KEY, a_int int, b_complex frozen<udt3>, c_int int) WITH ID = 6a5cff4e-2f94-4c8b-9aa2-0fbd65292caa");
+    }
+
+    private void blockFlushing(List<Path> dirs) throws IOException
+    {
+        for (Path dir : dirs)
+        {
+            Set<PosixFilePermission> permissions = Files.getPosixFilePermissions(dir);
+            permissions.remove(PosixFilePermission.OWNER_WRITE);
+            permissions.remove(PosixFilePermission.GROUP_WRITE);
+            permissions.remove(PosixFilePermission.OTHERS_WRITE);
+            Files.setPosixFilePermissions(dir, permissions);
+        }
+    }
+
+    private void unblockFlushing(List<Path> dirs) throws IOException
+    {
+        for (Path dir : dirs)
+        {
+            Set<PosixFilePermission> permissions = Files.getPosixFilePermissions(dir);
+            permissions.add(PosixFilePermission.OWNER_WRITE);
+            Files.setPosixFilePermissions(dir, permissions);
+        }
+    }
+
+    @Test
+    public void testReadingValuesOfDroppedColumns() throws Throwable
+    {
+        // given there is a table with a UDT column and some additional non-UDT columns, and there are rows with
+        // different combinations of values and nulls for all columns
+        try (Cluster cluster = Cluster.build(1).withConfig(c -> c.with(GOSSIP, NATIVE_PROTOCOL)).start())
+        {
+            IInvokableInstance node = cluster.get(1);
+            node.executeInternal("CREATE KEYSPACE " + KS + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+            node.executeInternal("CREATE TYPE " + KS + ".udt (foo text, bar text)");
+            node.executeInternal("CREATE TABLE " + KS + ".tab (pk int PRIMARY KEY, a_udt udt, b text, c text)");
+            node.executeInternal("INSERT INTO " + KS + ".tab (pk, c) VALUES (1, 'c_value')");
+            node.executeInternal("INSERT INTO " + KS + ".tab (pk, b) VALUES (2, 'b_value')");
+            node.executeInternal("INSERT INTO " + KS + ".tab (pk, a_udt) VALUES (3, {foo: 'a_foo', bar: 'a_bar'})");
+
+            File dataDir = new File(node.callOnInstance(() -> Keyspace.open(KS)
+                                                                      .getColumnFamilyStore("tab")
+                                                                      .getDirectories()
+                                                                      .getDirectoryForNewSSTables()
+                                                                      .absolutePath()));
+            checkData(cluster);
+
+            // when the UDT columns is dropped while the data cannot be flushed before drop and must remain in the commitlog
+
+            // prevent flushing the data
+            Set<PosixFilePermission> permissions = Files.getPosixFilePermissions(dataDir.toPath());
+            permissions.remove(PosixFilePermission.OWNER_WRITE);
+            permissions.remove(PosixFilePermission.GROUP_WRITE);
+            permissions.remove(PosixFilePermission.OTHERS_WRITE);
+            Files.setPosixFilePermissions(dataDir.toPath(), permissions);
+
+            node.executeInternal("ALTER TABLE " + KS + ".tab DROP a_udt");
+
+            // and the node is restarted
+            // restart is needed because this way we can simulate the situation where the commit log contains the data
+            // of the dropped cell, while the schema is already altered (the column moved to dropped columns and transformed)
+            node.shutdown(false).get(10, TimeUnit.SECONDS);
+
+            // unlock the ability to flush data
+            permissions = Files.getPosixFilePermissions(dataDir.toPath());
+            permissions.add(PosixFilePermission.OWNER_WRITE);
+            Files.setPosixFilePermissions(dataDir.toPath(), permissions);
+            node.startup();
+
+            // then, we should still be able to read the data of the remaining columns correctly
+            checkData(cluster);
+
+            // and even after flushing and restarting the node again
+            // the next restart is needed to make sure that the sstable header is read from disk
+            node.flush(KS);
+            node.shutdown(false).get(10, TimeUnit.SECONDS);
+            node.startup();
+
+            checkData(cluster);
+
+            // verify that the sstable can be read with sstabledump
+            String sstable = node.callOnInstance(() -> Keyspace.open(KS).getColumnFamilyStore("tab")
+                                                               .getDirectories().getCFDirectories()
+                                                               .get(0).tryList()[0].toString());
+            ToolRunner.ToolResult tool = ToolRunner.invokeClass(SSTableExport.class, sstable);
+            tool.assertCleanStdErr();
+            tool.assertOnExitCode();
+            assertThat(tool.getStdout())
+            .contains("\"key\" : [ \"1\" ],")
+            .contains("\"key\" : [ \"2\" ],")
+            .contains("{ \"name\" : \"c\", \"value\" : \"c_value\" }")
+            .contains("{ \"name\" : \"b\", \"value\" : \"b_value\" }");
+        }
+    }
+
+    private void checkData(Cluster cluster)
+    {
+        ICoordinator coordinator = cluster.coordinator(1);
+        String query = "SELECT b, c FROM " + KS + ".tab WHERE pk = ?";
+        assertRows(coordinator.execute(query, ConsistencyLevel.QUORUM, 1), row(null, "c_value"));
+        assertRows(coordinator.execute(query, ConsistencyLevel.QUORUM, 2), row("b_value", null));
+        assertRows(coordinator.execute(query, ConsistencyLevel.QUORUM, 3), row(null, null));
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
index 405279aae609..155bee54d10c 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
@@ -62,6 +62,7 @@
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnable;
 import org.apache.cassandra.distributed.impl.InstanceKiller;
+import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
@@ -81,12 +82,12 @@ public class FailingRepairTest extends TestBaseImpl implements Serializable
 {
     private static ICluster<IInvokableInstance> CLUSTER;
 
-    private final Verb messageType;
+    private final int messageType;
     private final RepairParallelism parallelism;
     private final boolean withTracing;
     private final SerializableRunnable setup;
 
-    public FailingRepairTest(Verb messageType, RepairParallelism parallelism, boolean withTracing, SerializableRunnable setup)
+    public FailingRepairTest(int messageType, RepairParallelism parallelism, boolean withTracing, SerializableRunnable setup)
     {
         this.messageType = messageType;
         this.parallelism = parallelism;
@@ -102,15 +103,16 @@ public static Collection<Object[]> messages()
         {
             for (Boolean withTracing : Arrays.asList(Boolean.TRUE, Boolean.FALSE))
             {
-                tests.add(new Object[]{ Verb.VALIDATION_REQ, parallelism, withTracing, failingReaders(Verb.VALIDATION_REQ, parallelism, withTracing) });
+                tests.add(new Object[]{ Verb.VALIDATION_REQ.id, parallelism, withTracing, failingReaders(Verb.VALIDATION_REQ.id, parallelism, withTracing) });
             }
         }
         return tests;
     }
 
-    private static SerializableRunnable failingReaders(Verb type, RepairParallelism parallelism, boolean withTracing)
+    private static SerializableRunnable failingReaders(int typeId, RepairParallelism parallelism, boolean withTracing)
     {
         return () -> {
+            Verb type = Verb.fromId(typeId);
             String cfName = getCfName(type, parallelism, withTracing);
             ColumnFamilyStore cf = Keyspace.open(KEYSPACE).getColumnFamilyStore(cfName);
             Util.flush(cf);
@@ -148,7 +150,11 @@ public static void setupCluster() throws IOException
                               .start());
         CLUSTER.setUncaughtExceptionsFilter((throwable) -> {
             if (throwable.getClass().toString().contains("InstanceShutdown") || // can't check instanceof as it is thrown by a different classloader
-                throwable.getMessage() != null && throwable.getMessage().contains("Parent repair session with id"))
+                (throwable.getMessage() != null && throwable.getMessage().contains("Parent repair session with id")) ||
+                (throwable.getClass().toString().contains("RepairException") &&
+                 throwable.getMessage() != null &&
+                 throwable.getMessage().contains("Validation failed"))
+                )
                 return true;
             return false;
         });
@@ -169,7 +175,14 @@ public void cleanupState()
             IInvokableInstance inst = CLUSTER.get(i);
             if (inst.isShutdown())
                 inst.startup();
-            inst.runOnInstance(InstanceKiller::clear);
+            inst.runOnInstance(() -> {
+                InstanceKiller.clear();
+                if (!StorageService.instance.isGossipActive())
+                {
+                    StorageService.instance.startGossiping();
+                    Gossiper.waitToSettle();
+                }
+            });
         }
     }
 
@@ -178,7 +191,7 @@ public void testFailingMessage() throws IOException
     {
         final int replica = 1;
         final int coordinator = 2;
-        String tableName = getCfName(messageType, parallelism, withTracing);
+        String tableName = getCfName(Verb.fromId(messageType), parallelism, withTracing);
         String fqtn = KEYSPACE + "." + tableName;
 
         CLUSTER.schemaChange("CREATE TABLE " + fqtn + " (k INT, PRIMARY KEY (k))");
@@ -336,6 +349,11 @@ public Set<SSTableReader> getBackingSSTables()
             return Collections.emptySet();
         }
 
+        public int level()
+        {
+            return 0;
+        }
+
         public TableMetadata metadata()
         {
             return null;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailureLoggingTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailureLoggingTest.java
index 58d44f558c19..421428d13a64 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FailureLoggingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FailureLoggingTest.java
@@ -39,8 +39,8 @@
 import org.apache.cassandra.exceptions.UnavailableException;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.reads.range.RangeCommandIterator;
 import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.service.reads.range.NonGroupingRangeCommandIterator;
 
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static org.junit.Assert.assertEquals;
@@ -142,7 +142,7 @@ static void install(ClassLoader cl, int nodeNumber)
               .make()
               .load(cl, ClassLoadingStrategy.Default.INJECTION);
 
-            bb.redefine(RangeCommandIterator.class)
+            bb.redefine(NonGroupingRangeCommandIterator.class)
               .method(named("sendNextRequests"))
               .intercept(MethodDelegation.to(BBRequestFailures.class))
               .make()
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FrozenUDTTest.java b/test/distributed/org/apache/cassandra/distributed/test/FrozenUDTTest.java
index 3314c2a6ba11..a9441d65ebe1 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FrozenUDTTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FrozenUDTTest.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
 
+import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.cassandra.distributed.Cluster;
@@ -124,6 +125,7 @@ public void testUpgradeSStables() throws IOException
         }
     }
 
+    /* See CASSANDRA-19764 */
     @Test
     public void testDivergentSchemas() throws Throwable
     {
@@ -133,11 +135,20 @@ public void testDivergentSchemas() throws Throwable
             cluster.schemaChange("create table " + KEYSPACE + ".x (id int, ck frozen<a>, i int, primary key (id, ck))");
 
             cluster.get(1).executeInternal("alter type " + KEYSPACE + ".a add bar text");
-            cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (?, " + json(1, 1) + ", ? )", ConsistencyLevel.ALL,
-                                           1, 1);
-            cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (?, " + json(1, 2) + ", ? )", ConsistencyLevel.ALL,
-                                           2, 2);
-            cluster.get(2).flush(KEYSPACE);
+            try
+            {
+                cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (?, " + json(1, 2) + ", ? )", ConsistencyLevel.ALL,
+                                               1, 2);
+                cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (?, " + json(1, 1) + ", ? )", ConsistencyLevel.ALL,
+                                               1, 1);
+                cluster.get(2).flush(KEYSPACE);
+                Assert.fail("Expected an exception to be thrown.");
+            }
+            catch (Exception e)
+            {
+                // correct path
+                System.out.println(e);
+            }
         }
     }
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/GossipSettlesTest.java b/test/distributed/org/apache/cassandra/distributed/test/GossipSettlesTest.java
index 341d85482d0c..2d1b2c2f3dc6 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/GossipSettlesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/GossipSettlesTest.java
@@ -32,6 +32,7 @@
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.SystemDistributedKeyspace;
 import org.apache.cassandra.schema.SchemaConstants;
@@ -64,7 +65,7 @@ public void testGossipSettles() throws Throwable
                 // First prove that the storage port is added
                 Assert.assertEquals("stuff 127.0.0.1:7012 morestuff 127.0.0.2:7012", addStoragePortToIP("stuff 127.0.0.1 morestuff 127.0.0.2"));
 
-                FailureDetector fd = ((FailureDetector) FailureDetector.instance);
+                FailureDetector fd = ((FailureDetector) IFailureDetector.instance);
                 Assert.assertEquals(addStoragePortToInstanceName(fd.getAllEndpointStates(false)),
                                     fd.getAllEndpointStates(true));
                 Assert.assertEquals(addPortToKeys(fd.getSimpleStates()), fd.getSimpleStatesWithPort());
diff --git a/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java b/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
index 39054118aa02..2d856ad14667 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.distributed.test;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.Collection;
 import java.util.concurrent.*;
@@ -34,6 +35,7 @@
 import net.bytebuddy.ByteBuddy;
 import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
 import net.bytebuddy.implementation.MethodDelegation;
+import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.*;
@@ -41,16 +43,19 @@
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.EndpointState;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IClusterVersionProvider;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.service.PendingRangeCalculatorService;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.streaming.StreamPlan;
 import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.FBUtilities;
 import org.assertj.core.api.Assertions;
 
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CLUSTER_VERSION_PROVIDER_CLASS_NAME;
 import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
 import static org.apache.cassandra.distributed.action.GossipHelper.withProperty;
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
@@ -59,12 +64,87 @@
 import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.runAndWaitForLogs;
 import static org.apache.cassandra.distributed.shared.NetworkTopology.singleDcNetworkTopology;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public class GossipTest extends TestBaseImpl
 {
+    public static class CustomClusterVersionProvider implements IClusterVersionProvider
+    {
+        public static volatile CassandraVersion version = CassandraVersion.NULL_VERSION;
+        public static volatile boolean initialized = false;
+        public static volatile long lastReset = 0;
+        public static volatile boolean upgradeInProgress = true;
+
+        public CustomClusterVersionProvider()
+        {
+            initialized = true;
+        }
+
+        @Override
+        public CassandraVersion getMinClusterVersion()
+        {
+            return version;
+        }
+
+        @Override
+        public void reset()
+        {
+            lastReset = System.currentTimeMillis();
+        }
+
+        @Override
+        public boolean isUpgradeInProgress()
+        {
+            return upgradeInProgress;
+        }
+    }
+
+    @Test
+    public void testCustomMinClusterVersionProvider() throws IOException
+    {
+        CLUSTER_VERSION_PROVIDER_CLASS_NAME.setString(CustomClusterVersionProvider.class.getName());
+
+        try (Cluster cluster = Cluster.build(1).withConfig(config -> config.with(GOSSIP)).start())
+        {
+            IInvokableInstance i = cluster.get(1);
+            assertThat(i.callOnInstance(() -> CustomClusterVersionProvider.initialized)).isTrue();
+
+            i.runOnInstance(() -> CustomClusterVersionProvider.version = CassandraVersion.NULL_VERSION);
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_3_4))).isTrue();
+
+            i.runOnInstance(() -> CustomClusterVersionProvider.version = CassandraVersion.CASSANDRA_3_4);
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_3_4))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0_RC2))).isTrue();
+
+            i.runOnInstance(() -> CustomClusterVersionProvider.version = CassandraVersion.CASSANDRA_4_0);
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_3_4))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0_RC2))).isTrue();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(SystemKeyspace.CURRENT_VERSION))).isTrue();
+
+            i.runOnInstance(() -> CustomClusterVersionProvider.version = CassandraVersion.CASSANDRA_4_0_RC2);
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_3_4))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0_RC2))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(SystemKeyspace.CURRENT_VERSION))).isTrue();
+
+            i.runOnInstance(() -> CustomClusterVersionProvider.version = SystemKeyspace.CURRENT_VERSION);
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_3_4))).isTrue();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0))).isTrue();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0_RC2))).isTrue();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(SystemKeyspace.CURRENT_VERSION))).isTrue();
+
+            i.runOnInstance(() -> CustomClusterVersionProvider.upgradeInProgress = false);
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_3_4))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0_RC2))).isFalse();
+            assertThat(i.callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(SystemKeyspace.CURRENT_VERSION))).isFalse();
+        }
+    }
+
     @Test
     public void nodeDownDuringMove() throws Throwable
     {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionOptionsTest.java b/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionOptionsTest.java
index 83bcaaad3c14..6ae233699082 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionOptionsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionOptionsTest.java
@@ -220,13 +220,6 @@ public void allInternodeEncryptionEstablishedTest() throws Throwable
 
     /**
      * Tests that the negotiated protocol is the highest common protocol between the client and server.
-     * <p> 
-     * Note: This test uses TLSV1.1, which is disabled by default in JDK 8 and higher. If the test fails with 
-     * FAILED_TO_NEGOTIATE, it may be necessary to check the java.security file in your JDK installation and remove 
-     * TLSv1.1 from the jdk.tls.disabledAlgorithms.
-     * @see <a href="https://issues.apache.org/jira/browse/CASSANDRA-18540">CASSANDRA-18540</a>
-     * @see <a href="https://senthilnayagan.medium.com/tlsv1-and-tlsv1-1-protocols-disabled-by-default-in-javas-latest-patch-released-on-april-20-2021-52c309f6b16d">
-     *     TLSv1 and TLSv1.1 Protocols are Disabled in Java!</a>
      */
     @Test
     public void negotiatedProtocolMustBeAcceptedProtocolTest() throws Throwable
@@ -236,7 +229,7 @@ public void negotiatedProtocolMustBeAcceptedProtocolTest() throws Throwable
             c.set("server_encryption_options",
                   ImmutableMap.builder().putAll(validKeystore)
                               .put("internode_encryption", "all")
-                              .put("accepted_protocols", ImmutableList.of("TLSv1.1", "TLSv1.2", "TLSv1.3"))
+                              .put("accepted_protocols", ImmutableList.of("TLSv1.2", "TLSv1.3"))
                               .build());
         }).start())
         {
@@ -250,9 +243,9 @@ public void negotiatedProtocolMustBeAcceptedProtocolTest() throws Throwable
             tls10Connection.assertReceivedHandshakeException();
 
             TlsConnection tls11Connection = new TlsConnection(address.getHostAddress(), port, Collections.singletonList("TLSv1.1"));
-            Assert.assertEquals("Should be possible to establish a TLSv1.1 connection",
-                                ConnectResult.NEGOTIATED, tls11Connection.connect());
-            Assert.assertEquals("TLSv1.1", tls11Connection.lastProtocol());
+            Assert.assertEquals("Should not be possible to establish a TLSv1.1 connection",
+                                ConnectResult.FAILED_TO_NEGOTIATE, tls11Connection.connect());
+            tls11Connection.assertReceivedHandshakeException();
 
             TlsConnection tls12Connection = new TlsConnection(address.getHostAddress(), port, Collections.singletonList("TLSv1.2"));
             Assert.assertEquals("Should be possible to establish a TLSv1.2 connection",
diff --git a/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorThrowableTest.java b/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorThrowableTest.java
index b63179b70e02..3112bc1d6c68 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorThrowableTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorThrowableTest.java
@@ -231,7 +231,7 @@ public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, Column
 
         private CorruptSSTableException throwCorrupted()
         {
-            throw new CorruptSSTableException(new IOException("failed to get position"), descriptor.baseFile());
+            throw new CorruptSSTableException(new IOException("failed to get position"), descriptor.baseFileUri());
         }
 
         private FSError throwFSError()
diff --git a/test/distributed/org/apache/cassandra/distributed/test/MetricsCountQueriesTest.java b/test/distributed/org/apache/cassandra/distributed/test/MetricsCountQueriesTest.java
index a742e483793c..16e9f361882c 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/MetricsCountQueriesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/MetricsCountQueriesTest.java
@@ -52,6 +52,6 @@ public void testMetricsCountQueries() throws Throwable
 
     private static long readCount(IInvokableInstance instance)
     {
-        return instance.callOnInstance(() -> Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl").metric.readLatency.latency.getCount());
+        return instance.callOnInstance(() -> Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl").metric.readLatency.tableOrKeyspaceMetric().latency.getCount());
     }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java b/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java
index 3e8c92648099..098aa7e236d6 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java
@@ -170,7 +170,7 @@ public void negotiatedProtocolMustBeAcceptedProtocolTest() throws Throwable
             c.set("client_encryption_options",
                   ImmutableMap.builder().putAll(validKeystore)
                               .put("enabled", true)
-                              .put("accepted_protocols", ImmutableList.of("TLSv1.1", "TLSv1.2", "TLSv1.3"))
+                              .put("accepted_protocols", ImmutableList.of("TLSv1.2", "TLSv1.3"))
                               .build());
         }).start())
         {
@@ -183,9 +183,9 @@ public void negotiatedProtocolMustBeAcceptedProtocolTest() throws Throwable
             tls10Connection.assertReceivedHandshakeException();
 
             TlsConnection tls11Connection = new TlsConnection(address.getHostAddress(), port, Collections.singletonList("TLSv1.1"));
-            Assert.assertEquals("Should be possible to establish a TLSv1.1 connection",
-                                ConnectResult.NEGOTIATED, tls11Connection.connect());
-            Assert.assertEquals("TLSv1.1", tls11Connection.lastProtocol());
+            Assert.assertEquals("Should not be possible to establish a TLSv1.1 connection",
+                                ConnectResult.FAILED_TO_NEGOTIATE, tls11Connection.connect());
+            tls11Connection.assertReceivedHandshakeException();
 
             TlsConnection tls12Connection = new TlsConnection(address.getHostAddress(), port, Collections.singletonList("TLSv1.2"));
             Assert.assertEquals("Should be possible to establish a TLSv1.2 connection",
diff --git a/test/distributed/org/apache/cassandra/distributed/test/NodeToolEnableDisableBinaryTest.java b/test/distributed/org/apache/cassandra/distributed/test/NodeToolEnableDisableBinaryTest.java
index 36803fbdfd9c..fed4df7447e6 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/NodeToolEnableDisableBinaryTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/NodeToolEnableDisableBinaryTest.java
@@ -96,7 +96,7 @@ public void testMaybeChangeDocs()
                         "            Remote jmx agent username\n" +
                         "\n" +
                         "\n";
-        Assertions.assertThat(tool.getStdout()).isEqualTo(help);
+        Assertions.assertThat(tool.getCleanedStdout()).isEqualTo(help);
 
         tool = ToolRunner.invokeNodetoolJvmDtest(cluster.get(1), "help", "enablebinary");
         help =  "NAME\n" +
@@ -128,7 +128,7 @@ public void testMaybeChangeDocs()
                 "            Remote jmx agent username\n" +
                 "\n" +
                 "\n";
-        Assertions.assertThat(tool.getStdout()).isEqualTo(help);
+        Assertions.assertThat(tool.getCleanedStdout()).isEqualTo(help);
     }
 
     @Test
diff --git a/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java b/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
index 24a65e3d4e55..51eed823300a 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
@@ -147,4 +147,18 @@ public void testVersionIncludesGitSHAWhenVerbose() throws Throwable
             .success()
             .stdoutContains("GitSHA:");
     }
+
+    @Test
+    public void testCompactionStats() throws Throwable
+    {
+        NodeToolResult result = NODE.nodetoolResult("compactionstats", "--aggregate", "--overlap");
+        result.asserts().success().stdoutContains("pending tasks");
+        result.asserts().success().stdoutContains("Aggregated view");
+        result.asserts().success().stdoutContains("Max overlap map");
+
+        result = NODE.nodetoolResult("compactionstats", "--aggregate", "--overlap", "--human-readable", "system_schema", "tables");
+        result.asserts().success().stdoutContains("system_schema.tables");
+        result.asserts().success().stdoutNotContains("system.peers");
+        result.asserts().success().stdoutNotContains("system_schema.keyspaces");
+    }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
index a150a33a95ae..7137d5d78e33 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
@@ -596,7 +596,7 @@ public SingleUpdateSupplier(TableMetadata cfm, DecoratedKey dk, Ballot ballot)
         public CloseableIterator<PaxosKeyState> repairIterator(TableId cfId, Collection<Range<Token>> ranges)
         {
             if (!cfId.equals(cfm.id))
-                return CloseableIterator.empty();
+                return CloseableIterator.emptyIterator();
             return CloseableIterator.wrap(Collections.singleton(new PaxosKeyState(cfId, dk, ballot, false)).iterator());
         }
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
index a0b643f0d309..39cd73a0f3ad 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
@@ -111,7 +111,7 @@ public void testWithMismatchingPending() throws Throwable
             // also disables autocompaction on the nodes
             cluster.forEach((node) -> node.runOnInstance(() -> {
                 ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
-                FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(cfs));
+                FBUtilities.waitOnFuture(CompactionManager.instance.submitBackground(cfs));
                 cfs.disableAutoCompaction();
             }));
             long[] marks = logMark(cluster);
@@ -120,7 +120,7 @@ public void testWithMismatchingPending() throws Throwable
             cluster.get(1).runOnInstance(() -> {
                 ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
                 cfs.enableAutoCompaction();
-                FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(cfs));
+                FBUtilities.waitOnFuture(CompactionManager.instance.submitBackground(cfs));
             });
 
             waitLogsRepairFullyFinished(cluster, marks);
@@ -210,6 +210,9 @@ public void testConcurrentIncRepairDuringPreview() throws IOException, Interrupt
                                                                 config.with(GOSSIP)
                                                                       .with(NETWORK)).start()))
         {
+            cluster.setUncaughtExceptionsFilter(t -> t.getClass().toString().contains("RepairException") &&
+                                                     t.getMessage() != null &&
+                                                     t.getMessage().contains("Validation failed"));
             cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
             insert(cluster.coordinator(1), 0, 100);
             cluster.forEach((node) -> node.flush(KEYSPACE));
@@ -419,7 +422,7 @@ private void unmarkRepaired(IInvokableInstance instance, String table)
             ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table);
             try
             {
-                cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), ActiveRepairService.UNREPAIRED_SSTABLE, null, false);
+                cfs.mutateRepaired(cfs.getLiveSSTables(), ActiveRepairService.UNREPAIRED_SSTABLE, null, false);
             }
             catch (IOException e)
             {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java b/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java
index b0c3902ad152..de6cd6e86672 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java
@@ -30,6 +30,8 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+
 
 import com.datastax.driver.core.Session;
 import org.apache.cassandra.db.Keyspace;
@@ -43,10 +45,15 @@
 import org.apache.cassandra.distributed.api.SimpleQueryResult;
 import org.apache.cassandra.utils.Throwables;
 
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
 import static org.junit.Assert.assertTrue;
 
+@RunWith(BMUnitRunner.class)
 public class QueriesTableTest extends TestBaseImpl
 {
     private static Cluster SHARED_CLUSTER;
@@ -76,6 +83,10 @@ public static void closeCluster()
     }
 
     @Test
+    @BMRule(name = "Make mutations slow",
+    targetClass = "Mutation",
+    targetMethod = "apply",
+    action = "Thread.sleep(100)")
     public void shouldExposeReadsAndWrites() throws Throwable
     {
         SHARED_CLUSTER.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (k int primary key, v int)");
diff --git a/test/distributed/org/apache/cassandra/distributed/test/QueryInfoTrackerDistributedTest.java b/test/distributed/org/apache/cassandra/distributed/test/QueryInfoTrackerDistributedTest.java
new file mode 100644
index 000000000000..eb5530dd2068
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/QueryInfoTrackerDistributedTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.distributed.test.sai.SAIUtil;
+import org.apache.cassandra.service.QueryInfoTrackerTest.TestQueryInfoTracker;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+public class QueryInfoTrackerDistributedTest extends TestBaseImpl
+{
+    private static Cluster cluster;
+    private final static String rfOneKs = "rfoneks";
+
+    @BeforeClass
+    public static void setupCluster() throws Throwable
+    {
+        cluster = init(Cluster.build().withNodes(3).withConfig(config -> config.with(NETWORK, GOSSIP)).start());
+        cluster.schemaChange("CREATE KEYSPACE " + rfOneKs +
+                             " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};");
+    }
+
+    @AfterClass
+    public static void close() throws Exception
+    {
+        cluster.close();
+        cluster = null;
+    }
+
+    @Test
+    @SuppressWarnings("rawtypes")
+    public void testTrackingInDataResolverResolve()
+    {
+        ReadRepairTester tester = new ReadRepairTester(cluster, ReadRepairStrategy.BLOCKING, 1, false, false, false)
+        {
+            @Override
+            ReadRepairTester self()
+            {
+                return this;
+            }
+        };
+
+        String keyspace = tester.qualifiedTableName.split("\\.")[0];
+
+        tester.createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+        cluster.coordinator(1).execute("INSERT INTO " + tester.qualifiedTableName + " (pk, ck, v) VALUES (1, 1, 1)",
+                                       ConsistencyLevel.QUORUM);
+
+        tester.mutate(2, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 2)");
+
+        setQueryTracker(tester.coordinator, keyspace);
+
+        tester.assertRowsDistributed("SELECT * FROM %s WHERE pk=1 AND ck=1",
+                                     2,
+                                     row(1, 1, 2));
+
+        assertQueryTracker(tester.coordinator, tracker -> {
+            Assert.assertEquals(1, tracker.reads.get());
+            Assert.assertEquals(1, tracker.readPartitions.get());
+            Assert.assertEquals(1, tracker.readRows.get());
+            Assert.assertEquals(1, tracker.replicaPlans.get());
+        });
+    }
+
+    @Test
+    public void testTrackingInDigestResolverGetData()
+    {
+        cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+        cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)",
+                                       ConsistencyLevel.QUORUM);
+
+        setQueryTracker(1, KEYSPACE);
+
+        assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+                                                  ConsistencyLevel.QUORUM),
+                   row(1, 1, 1));
+
+        assertQueryTracker(1, tracker -> {
+            Assert.assertEquals(1, tracker.reads.get());
+            Assert.assertEquals(1, tracker.readPartitions.get());
+            Assert.assertEquals(1, tracker.readRows.get());
+            Assert.assertEquals(1, tracker.replicaPlans.get());
+        });
+    }
+
+    @Test
+    public void testTrackingReadsWithEndpointGrouping()
+    {
+        String table = rfOneKs + ".saiTbl";
+        cluster.schemaChange("CREATE TABLE " + table + " (id1 TEXT PRIMARY KEY, v1 INT, v2 TEXT)");
+        cluster.schemaChange("CREATE CUSTOM INDEX IF NOT EXISTS test_idx ON " + table + " (v1) USING 'StorageAttachedIndex'");
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        int rowsCount = 1000;
+
+        for (int i = 0; i < rowsCount; ++i)
+        {
+            cluster.coordinator(1).execute("INSERT INTO " + table + " (id1, v1, v2) VALUES (?, ?, ?);",
+                                           ConsistencyLevel.QUORUM,
+                                           String.valueOf(i),
+                                           i,
+                                           String.valueOf(i));
+        }
+
+        setQueryTracker(1, rfOneKs);
+
+        cluster.coordinator(1).execute(String.format("SELECT id1 FROM %s WHERE v1>=0", table),
+                                       ConsistencyLevel.QUORUM);
+
+        assertQueryTracker(1, tracker -> {
+            Assert.assertEquals(1, tracker.rangeReads.get());
+            Assert.assertEquals(rowsCount, tracker.readPartitions.get());
+            Assert.assertEquals(rowsCount, tracker.readRows.get());
+            Assert.assertEquals(4, tracker.replicaPlans.get());
+        });
+    }
+
+    @Test
+    public void testANNQueryWithIndexRestrictionAndLIMIT()
+    {
+        String table = rfOneKs + ".ann_table";
+        cluster.schemaChange("CREATE TABLE " + table + " (p int PRIMARY KEY, v int, ni int, vec VECTOR<FLOAT, 2>)");
+        cluster.schemaChange("CREATE CUSTOM INDEX ON " + table + "(vec) USING 'StorageAttachedIndex'");
+        cluster.schemaChange("CREATE CUSTOM INDEX ON " + table + "(v) USING 'StorageAttachedIndex'");
+        SAIUtil.waitForIndexQueryable(cluster, rfOneKs);
+
+        for (int rowIdx = 0; rowIdx < 100; rowIdx++)
+        {
+            cluster.coordinator(1).execute("INSERT INTO " + table + "(p, v, ni, vec) VALUES (?, ?, ?, [0.5, 0.3])",
+                                           ConsistencyLevel.ALL, rowIdx, rowIdx, rowIdx);
+        }
+
+        setQueryTracker(1, rfOneKs);
+
+        cluster.coordinator(1).execute("SELECT * FROM " + table + " WHERE v > 50 ORDER BY vec ANN OF [0.1, 0.9] LIMIT 3",
+                                       ConsistencyLevel.ONE);
+
+        assertQueryTracker(1, tracker -> {
+            Assert.assertEquals(1, tracker.rangeReads.get());
+            Assert.assertEquals(3, tracker.readFilteredRows.get());
+        });
+    }
+
+    private void setQueryTracker(int node, String keyspace)
+    {
+        cluster.get(node).runOnInstance(() -> StorageProxy.instance.registerQueryTracker(new TestQueryInfoTracker(keyspace)));
+    }
+
+    private void assertQueryTracker(int node, IIsolatedExecutor.SerializableConsumer<TestQueryInfoTracker> tester)
+    {
+        cluster.get(node).runOnInstance(() -> {
+            TestQueryInfoTracker tracker = (TestQueryInfoTracker) StorageProxy.queryTracker();
+            tester.accept(tracker);
+        });
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadCoordinationMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadCoordinationMetricsTest.java
new file mode 100644
index 000000000000..0990af8a9e8e
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/ReadCoordinationMetricsTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.net.InetAddress;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.distributed.impl.DistributedTestSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.ReadCoordinationMetrics;
+import org.apache.cassandra.service.reads.AbstractReadExecutor;
+
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
+
+public class ReadCoordinationMetricsTest extends TestBaseImpl
+{
+    private static final int NUM_ROWS = 100;
+
+    private static long countNonreplicaRequests(IInvokableInstance node)
+    {
+        return node.callOnInstance(() -> ReadCoordinationMetrics.nonreplicaRequests.getCount());
+    }
+
+    private static long countPreferredOtherReplicas(IInvokableInstance node)
+    {
+        return node.callOnInstance(() -> ReadCoordinationMetrics.preferredOtherReplicas.getCount());
+    }
+
+    /**
+     * Two nodes with RF=1 so half the data will be owned by each node and the coordinator for queries is not
+     * always a replica in the list of candidates in {@link AbstractReadExecutor#getReadExecutor} where
+     * {@link ReadCoordinationMetrics#nonreplicaRequests} will be incremented.
+     *
+     * @throws Throwable
+     */
+    @Test
+    public void testNonReplicaRequests() throws Throwable
+    {
+        try (Cluster cluster = init(Cluster.create(2), 1))
+        {
+            cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))"));
+            for (int i = 0; i < NUM_ROWS; i++)
+                cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (pk, ck, v) VALUES (?,?,?)"), ALL, i, i, i);
+
+            long nonReplicaRequests1 = countNonreplicaRequests(cluster.get(1));
+            long nonReplicaRequests2 = countNonreplicaRequests(cluster.get(2));
+
+            for (int i = 0; i < NUM_ROWS; i++)
+            {
+                // When the coordinator is not a candidate replica, which will be half the time due to RF=1,
+                // the non-replica count metric will be incremented.
+                cluster.coordinator(1).execute(withKeyspace("SELECT * FROM %s.tbl WHERE pk = ? and ck = ?"), ALL, i, i);
+                cluster.coordinator(2).execute(withKeyspace("SELECT * FROM %s.tbl WHERE pk = ? and ck = ?"), ALL, i, i);
+            }
+
+            nonReplicaRequests1 = countNonreplicaRequests(cluster.get(1)) - nonReplicaRequests1;
+            nonReplicaRequests2 = countNonreplicaRequests(cluster.get(2)) - nonReplicaRequests2;
+            Assert.assertEquals(NUM_ROWS, nonReplicaRequests1 + nonReplicaRequests2);
+        }
+    }
+
+    /**
+     * Two nodes with RF=2 so that both nodes are replicas for all data; this ensures that the coordinator node for
+     * queries will be a replica in the list of candidates in {@link AbstractReadExecutor#getReadExecutor}.
+     * <p>
+     * When the candidates collection is created, the sort order is changed so that the coordinator node is last. Then,
+     * using with CL=1 in the query, the resulting contacts collection will not contain the coordinator node, causing
+     * {@link ReadCoordinationMetrics#preferredOtherReplicas} to be incremented.
+     *
+     * @throws Throwable
+     */
+    @Test
+    public void testPreferredOtherReplicas() throws Throwable
+    {
+        try (Cluster cluster =  init(builder()
+                                     .withNodes(2)
+                                     .withConfig(config -> config.set("dynamic_snitch", false)
+                                     ).start(), 2))
+        {
+            cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))"));
+            for (int i = 0; i < NUM_ROWS; i++)
+                cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (pk, ck, v) VALUES (?,?,?)"), ALL, i, i, i);
+
+            long preferredOtherReplicas1 = countPreferredOtherReplicas(cluster.get(1));
+
+            // Replica nodes are normally sorted by distance from the coordinator; override the test snitch to
+            // sort with respect to another node so that the coordinator node is last in the list of replicas.
+            // This will be used together with CL=1 to drop the coordinator node from the list of contacts, while
+            // remaining in the list of candidates.
+            InetAddress address2 = cluster.get(2).broadcastAddress().getAddress();
+            cluster.get(1).acceptsOnInstance((IIsolatedExecutor.SerializableConsumer<InetAddress>) (ks) -> {
+                DistributedTestSnitch.sortByProximityAddressOverride = InetAddressAndPort.getByAddress(ks);
+            }).accept(address2);
+
+            for (int i = 0; i < NUM_ROWS; i++)
+            {
+                // Query using CL=1 so that the subset of "candidate" replcas selected for the "contacts" collection
+                // will have just one node; since the "candidate" list was sorted with respect to the non-coordinator
+                // node, this will cause the preferredOtherReplicas count to be incremented.
+                cluster.coordinator(1).execute(withKeyspace("SELECT * FROM %s.tbl WHERE pk = ? and ck = ?"), ONE, i, i);
+            }
+
+            preferredOtherReplicas1 = countPreferredOtherReplicas(cluster.get(1)) - preferredOtherReplicas1;
+            Assert.assertEquals(NUM_ROWS, preferredOtherReplicas1);
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadFailureTest.java
deleted file mode 100644
index be8db6c7782f..000000000000
--- a/test/distributed/org/apache/cassandra/distributed/test/ReadFailureTest.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.test;
-
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.junit.Test;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.distributed.api.ICluster;
-import org.apache.cassandra.distributed.Cluster;
-import org.apache.cassandra.distributed.api.ConsistencyLevel;
-import org.apache.cassandra.exceptions.RequestFailureReason;
-
-public class ReadFailureTest extends TestBaseImpl
-{
-    static final int TOMBSTONE_FAIL_THRESHOLD = 20;
-    static final int TOMBSTONE_FAIL_KEY = 100001;
-    static final String TABLE = "t";
-
-    /**
-     * This test attempts to create a race condition with speculative executions that would previously cause an AssertionError.
-     * N=2, RF=2, read ONE
-     * The read will fail on the local node due to tombstone read threshold. At the same time, a spec exec is triggered
-     * reading from the other node.
-     * <p>
-     * See CASSANDRA-16097 for further details.
-     */
-    @Test
-    public void testSpecExecRace() throws Throwable
-    {
-        try (Cluster cluster = init(Cluster.build().withNodes(2).withConfig(config -> config.set("tombstone_failure_threshold", TOMBSTONE_FAIL_THRESHOLD)).start()))
-        {
-            // Create a table with the spec exec policy set to a low percentile so it's more likely to produce a spec exec racing with the local request.
-            // Not using 'Always' because that actually uses a different class/mechanism and doesn't exercise the bug
-            // we're trying to produce.
-            cluster.schemaChange(String.format("CREATE TABLE %s.%s (k int, c int, v int, PRIMARY KEY (k,c)) WITH speculative_retry = '5p';", KEYSPACE, TABLE));
-
-            // Create a partition with enough tombstones to create a read failure according to the configured threshold
-            for (int i = 0; i <= TOMBSTONE_FAIL_THRESHOLD; ++i)
-                cluster.coordinator(1).execute(String.format("DELETE FROM %s.t WHERE k=%d AND c=%d", KEYSPACE, TOMBSTONE_FAIL_KEY, i),
-                                               ConsistencyLevel.TWO);
-
-            // Create a bunch of latency samples for this failed operation.
-            loopFailStatement(cluster, 5000);
-            // Update the spec exec threshold based on the above samples.
-            // This would normally be done by the periodic task CassandraDaemon.SPECULATION_THRESHOLD_UPDATER.
-            cluster.get(1).runOnInstance(() ->
-                                         {
-                                             ColumnFamilyStore cfs = Keyspace.open(KEYSPACE)
-                                                                             .getColumnFamilyStore(TABLE);
-                                             cfs.updateSpeculationThreshold();
-                                         });
-
-            // Run the request a bunch of times under racy conditions.
-            loopFailStatement(cluster, 5000);
-        }
-    }
-
-    private void loopFailStatement(ICluster cluster, int iterations)
-    {
-        final String query = String.format("SELECT k FROM %s.t WHERE k=%d", KEYSPACE, TOMBSTONE_FAIL_KEY);
-        for (int i = 0; i < iterations; ++i)
-        {
-            try
-            {
-                cluster.coordinator(1).execute(query, ConsistencyLevel.ONE);
-                fail("Request did not throw a ReadFailureException as expected.");
-            }
-            catch (Throwable t) // Throwable because the raised ReadFailure is loaded from a different classloader and doesn't match "ours"
-            {
-                String onFail = String.format("Did not receive expected ReadFailureException. Instead caught %s\n%s",
-                                              t, ExceptionUtils.getStackTrace(t));
-                assertNotNull(onFail, t.getMessage());
-                assertTrue(onFail, t.getMessage().contains(RequestFailureReason.READ_TOO_MANY_TOMBSTONES.name()));
-            }
-        }
-    }
-}
-
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java
index 0fc2554b0139..321bc731d7fc 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java
@@ -33,6 +33,7 @@
 import org.apache.cassandra.distributed.api.NodeToolResult;
 import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
 import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+import org.apache.cassandra.io.util.FileUtils;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS;
 
@@ -88,8 +89,7 @@ public static void setupCluster() throws IOException
     @AfterClass
     public static void teardownCluster()
     {
-        if (CLUSTER != null)
-            CLUSTER.close();
+        FileUtils.closeQuietly(CLUSTER);
     }
 
     protected String tableName(String prefix) {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorNeighbourDown.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorNeighbourDown.java
index 590c65aa7282..da3762720485 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorNeighbourDown.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorNeighbourDown.java
@@ -33,7 +33,7 @@
 import org.apache.cassandra.distributed.api.NodeToolResult;
 import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
 import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
-import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.utils.FBUtilities;
@@ -92,7 +92,7 @@ public void neighbourDown()
                     {
                         throw new RuntimeException(e);
                     }
-                    while (FailureDetector.instance.isAlive(neighbor))
+                    while (IFailureDetector.instance.isAlive(neighbor))
                         Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
                 });
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorTimeout.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorTimeout.java
index b475e5515510..d2ebf75dd72a 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorTimeout.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorTimeout.java
@@ -24,6 +24,7 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.distributed.api.NodeToolResult;
 import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
 import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
@@ -46,6 +47,12 @@ public RepairCoordinatorTimeout(RepairType repairType, RepairParallelism paralle
     public void beforeTest()
     {
         CLUSTER.filters().reset();
+
+        CLUSTER.forEach(node -> node.runOnInstance(() -> {
+            // Set a larger PREPARE_MSG timeout for these tests to avoid faulure callbacks from being triggered, 
+            // causing IllegalStateException errors.
+            DatabaseDescriptor.setRepairPrepareMessageTimeout(120_000L);
+        }));
     }
 
     @Test
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
index b702855f680e..ae9890468cfa 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
@@ -610,7 +610,7 @@ private long getConfirmedInconsistencies(IInvokableInstance instance)
                                                      .getColumnFamilyStore(TABLE)
                                              .metric
                                              .confirmedRepairedInconsistencies
-                                             .table
+                                             .tableOrKeyspaceMeter()
                                              .getCount());
     }
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java
index 537599c29746..c4cd07434430 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java
@@ -27,6 +27,7 @@
 import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
 import net.bytebuddy.implementation.MethodDelegation;
 import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
 
@@ -262,7 +263,7 @@ public static void validateSSTableBoundsForAnticompaction(TimeUUID sessionID,
                                                                   Collection<SSTableReader> sstables,
                                                                   RangesAtEndpoint ranges)
         {
-            throw new CompactionInterruptedException(String.valueOf(sessionID));
+            throw new CompactionInterruptedException(String.valueOf(sessionID), TableOperation.StopTrigger.UNIT_TESTS);
         }
 
         @SuppressWarnings("unused")
diff --git a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java
index 62f6139b34ee..70155e7e9588 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java
@@ -35,7 +35,7 @@
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
+import org.apache.cassandra.db.compaction.CompactionStrategy;
 import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
 import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
 import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
@@ -160,7 +160,7 @@ public final void testCompactionStrategiesWithMixedSSTables() throws Exception
      * would get by merging data from the initial sstables.
      */
     @SafeVarargs
-    private final void testCompactionStrategiesWithMixedSSTables(final Class<? extends AbstractCompactionStrategy>... compactionStrategyClasses) throws Exception
+    private final void testCompactionStrategiesWithMixedSSTables(final Class<? extends CompactionStrategy>... compactionStrategyClasses) throws Exception
     {
         try (Cluster cluster = init(Cluster.build(1)
                                            .withDataDirCount(1)
@@ -168,7 +168,7 @@ private final void testCompactionStrategiesWithMixedSSTables(final Class<? exten
                                            .start()))
         {
             // create a table and two sstables with sequential id for each strategy, the sstables will contain overlapping partitions
-            for (Class<? extends AbstractCompactionStrategy> compactionStrategyClass : compactionStrategyClasses)
+            for (Class<? extends CompactionStrategy> compactionStrategyClass : compactionStrategyClasses)
             {
                 String tableName = "tbl_" + compactionStrategyClass.getSimpleName().toLowerCase();
                 cluster.schemaChange(createTableStmt(KEYSPACE, tableName, compactionStrategyClass));
@@ -181,7 +181,7 @@ private final void testCompactionStrategiesWithMixedSSTables(final Class<? exten
             restartNode(cluster, 1, true);
 
             // create another two sstables with uuid for each previously created table
-            for (Class<? extends AbstractCompactionStrategy> compactionStrategyClass : compactionStrategyClasses)
+            for (Class<? extends CompactionStrategy> compactionStrategyClass : compactionStrategyClasses)
             {
                 String tableName = "tbl_" + compactionStrategyClass.getSimpleName().toLowerCase();
 
@@ -407,7 +407,7 @@ private static Set<String> snapshot(IInvokableInstance instance, String ks, Stri
         return snapshotDirs;
     }
 
-    private static String createTableStmt(String ks, String name, Class<? extends AbstractCompactionStrategy> compactionStrategy)
+    private static String createTableStmt(String ks, String name, Class<? extends CompactionStrategy> compactionStrategy)
     {
         if (compactionStrategy == null)
             compactionStrategy = SizeTieredCompactionStrategy.class;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java b/test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java
index 94ea1d04416b..38f343094b79 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java
@@ -34,9 +34,9 @@
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.Feature;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.tools.BulkLoader;
 import org.apache.cassandra.tools.ToolRunner;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.NativeSSTableLoaderClient;
 
 import static org.junit.Assert.assertNotEquals;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java b/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java
index 9d168145c55b..674df194eee1 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java
@@ -26,7 +26,7 @@
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.distributed.Cluster;
@@ -34,6 +34,7 @@
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
@@ -58,15 +59,16 @@ public void test2iCompaction() throws IOException
                 i.getIndexCfs().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
                 Set<SSTableReader> idxSSTables = i.getIndexCfs().getLiveSSTables();
                 // emulate ongoing index compaction:
-                CompactionInfo.Holder h = new MockHolder(i.getIndexCfs().metadata(), idxSSTables);
-                CompactionManager.instance.active.beginCompaction(h);
-                CompactionManager.instance.active.estimatedRemainingWriteBytes();
-                CompactionManager.instance.active.finishCompaction(h);
+                AbstractTableOperation h = new MockHolder(i.getIndexCfs().metadata(), idxSSTables);
+                try (NonThrowingCloseable c = CompactionManager.instance.active.onOperationStart(h))
+                {
+                    CompactionManager.instance.active.estimatedRemainingWriteBytes();
+                }
             });
         }
     }
 
-    static class MockHolder extends CompactionInfo.Holder
+    static class MockHolder extends AbstractTableOperation
     {
         private final Set<SSTableReader> sstables;
         private final TableMetadata metadata;
@@ -77,9 +79,9 @@ public MockHolder(TableMetadata metadata, Set<SSTableReader> sstables)
             this.sstables = sstables;
         }
         @Override
-        public CompactionInfo getCompactionInfo()
+        public OperationProgress getProgress()
         {
-            return new CompactionInfo(metadata, OperationType.COMPACTION, 0, 1000, nextTimeUUID(), sstables);
+            return new OperationProgress(metadata, OperationType.COMPACTION, 0, 1000, nextTimeUUID(), sstables);
         }
 
         @Override
diff --git a/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java
index 5d72660fe188..89f287c63c80 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java
@@ -31,7 +31,7 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.ActiveCompactions;
+import org.apache.cassandra.db.compaction.ActiveOperations;
 import org.apache.cassandra.db.compaction.CompactionStrategyManager;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.streaming.StreamManager;
@@ -72,7 +72,7 @@ public void testAbortStreamsWhenOngoingCompactionsLeaveInsufficientSpace() throw
                                           .withConfig(config -> config.set("hinted_handoff_enabled", false)
                                                                       .with(GOSSIP)
                                                                       .with(NETWORK))
-                                          .withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveCompactions.class, "estimatedRemainingWriteBytes"))
+                                          .withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveOperations.class, "estimatedRemainingWriteBytes"))
                                           .start()))
         {
             cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int) with compaction={'class': 'SizeTieredCompactionStrategy'}");
diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
index 35c59046eae2..b6537dc4d9f6 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
@@ -18,34 +18,41 @@
 
 package org.apache.cassandra.distributed.test;
 
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.net.InetAddress;
+import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
+import java.nio.file.Files;
 import java.util.Arrays;
-import java.util.Date;
 import java.util.LinkedHashMap;
 import java.util.Map;
-import java.util.UUID;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.collect.ImmutableSet;
 
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 
-import org.apache.cassandra.cql3.Duration;
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
 import org.apache.cassandra.distributed.api.ICluster;
 import org.apache.cassandra.distributed.api.IInstanceConfig;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.shared.DistributedTestBase;
+import org.apache.cassandra.distributed.util.ColumnTypeUtil;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SCHEMA_DELAY_MS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
 import static org.apache.cassandra.distributed.action.GossipHelper.withProperty;
+import static org.awaitility.Awaitility.await;
 
 public class TestBaseImpl extends DistributedTestBase
 {
@@ -60,6 +67,7 @@ public void afterEach() {
     @BeforeClass
     public static void beforeClass() throws Throwable
     {
+        Files.createDirectories(FileUtils.getTempDir().toPath());
         ICluster.setup();
     }
 
@@ -103,7 +111,7 @@ public static ByteBuffer tuple(Object... values)
     {
         ByteBuffer[] bbs = new ByteBuffer[values.length];
         for (int i = 0; i < values.length; i++)
-            bbs[i] = makeByteBuffer(values[i]);
+            bbs[i] = ColumnTypeUtil.makeByteBuffer(values[i]);
         return TupleType.buildValue(bbs);
     }
 
@@ -119,74 +127,42 @@ public static String batch(String... queries)
 
     protected void bootstrapAndJoinNode(Cluster cluster)
     {
+        IInvokableInstance newInstance = bootstrapAndJoinNodeNoWait(cluster);
+
+        // Wait until all the other live nodes on the cluster see this node as NORMAL.
+        // The old nodes will update their tokens only after the new node announces its NORMAL state through gossip.
+        // This is to avoid disagreements about ring ownership between nodes and sudden ownership changes
+        // while running the tests.
+        InetAddressAndPort address = nodeAddress(newInstance.broadcastAddress());
+        await()
+            .atMost(90, TimeUnit.SECONDS)
+            .untilAsserted(() -> {
+                assert cluster.stream().allMatch(node -> node.isShutdown() || node.callOnInstance(() -> {
+                    EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(address);
+                    return state != null && state.isNormalState();
+                })) : "New node should be seen in NORMAL state by the other nodes in the cluster";
+        });
+    }
+
+    protected IInvokableInstance bootstrapAndJoinNodeNoWait(Cluster cluster)
+    {
+        cluster.stream().forEach(node -> {
+            assert node.config().has(Feature.NETWORK) : "Network feature must be enabled on the cluster";
+            assert node.config().has(Feature.GOSSIP) : "Gossip feature must be enabled on the cluster";
+        });
+
         IInstanceConfig config = cluster.newInstanceConfig();
         config.set("auto_bootstrap", true);
         IInvokableInstance newInstance = cluster.bootstrap(config);
         withProperty(BOOTSTRAP_SCHEMA_DELAY_MS, Integer.toString(90 * 1000),
                      () -> withProperty(JOIN_RING, false, () -> newInstance.startup(cluster)));
         newInstance.nodetoolResult("join").asserts().success();
+        return newInstance;
     }
 
-    @SuppressWarnings("unchecked")
-    private static ByteBuffer makeByteBuffer(Object value)
-    {
-        if (value == null)
-            return null;
-
-        if (value instanceof ByteBuffer)
-            return (ByteBuffer) value;
-
-        return typeFor(value).decompose(value);
-    }
-
-    private static AbstractType typeFor(Object value)
+    private static InetAddressAndPort nodeAddress(InetSocketAddress address)
     {
-        if (value instanceof ByteBuffer || value == null)
-            return BytesType.instance;
-
-        if (value instanceof Byte)
-            return ByteType.instance;
-
-        if (value instanceof Short)
-            return ShortType.instance;
-
-        if (value instanceof Integer)
-            return Int32Type.instance;
-
-        if (value instanceof Long)
-            return LongType.instance;
-
-        if (value instanceof Float)
-            return FloatType.instance;
-
-        if (value instanceof Duration)
-            return DurationType.instance;
-
-        if (value instanceof Double)
-            return DoubleType.instance;
-
-        if (value instanceof BigInteger)
-            return IntegerType.instance;
-
-        if (value instanceof BigDecimal)
-            return DecimalType.instance;
-
-        if (value instanceof String)
-            return UTF8Type.instance;
-
-        if (value instanceof Boolean)
-            return BooleanType.instance;
-
-        if (value instanceof InetAddress)
-            return InetAddressType.instance;
-
-        if (value instanceof Date)
-            return TimestampType.instance;
-
-        if (value instanceof UUID)
-            return UUIDType.instance;
-
-        throw new IllegalArgumentException("Unsupported value type (value is " + value + ')');
+        return InetAddressAndPort.getByAddressOverrideDefaults(address.getAddress(), address.getPort());
     }
 
     public static void fixDistributedSchemas(Cluster cluster)
@@ -212,4 +188,61 @@ public static void fixDistributedSchemas(Cluster cluster)
         // in real live repair is needed in this case, but in the test case it doesn't matter if the tables loose
         // anything, so ignoring repair to speed up the tests.
     }
+
+    /* Provide the cluster cannot start with the configured options */
+    void assertCannotStartDueToConfigurationException(Cluster cluster)
+    {
+        Throwable tr = null;
+        try
+        {
+            cluster.startup();
+        }
+        catch (Throwable maybeConfigException)
+        {
+            tr = maybeConfigException;
+        }
+
+        if (tr == null)
+        {
+            Assert.fail("Expected a ConfigurationException");
+        }
+        else
+        {
+            Assert.assertEquals(ConfigurationException.class.getName(), tr.getClass().getName());
+        }
+    }
+
+    /**
+     * Runs the given function before and after a flush of sstables.  This is useful for checking that behavior is
+     * the same whether data is in memtables or sstables.
+     *
+     * @param cluster the tested cluster
+     * @param keyspace the keyspace to flush
+     * @param runnable the test to run
+     */
+    public static void beforeAndAfterFlush(Cluster cluster, String keyspace, CQLTester.CheckedFunction runnable) throws Throwable
+    {
+        try
+        {
+            runnable.apply();
+        }
+        catch (Throwable t)
+        {
+            throw new AssertionError("Test failed before flush:\n" + t, t);
+        }
+
+        for (int i = 1; i <= cluster.size(); i++)
+        {
+            cluster.get(i).flush(keyspace);
+
+            try
+            {
+                runnable.apply();
+            }
+            catch (Throwable t)
+            {
+                throw new AssertionError("Test failed after flushing node " + i + ":\n" + t, t);
+            }
+        }
+    }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/UDFunctionTest.java b/test/distributed/org/apache/cassandra/distributed/test/UDFunctionTest.java
new file mode 100644
index 000000000000..56045422cd65
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/UDFunctionTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+
+public class UDFunctionTest extends TestBaseImpl
+{
+    @Test
+    public void nodeWillNotStartWithScriptedUDFsTest() throws Throwable
+    {
+        try (Cluster cluster = builder().withNodes(1)
+                                        .withConfig(c -> c.set("enable_scripted_user_defined_functions", true))
+                                        .createWithoutStarting())
+        {
+            assertCannotStartDueToConfigurationException(cluster);
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/UnableToParseClientMessageTest.java b/test/distributed/org/apache/cassandra/distributed/test/UnableToParseClientMessageTest.java
index 62969f03cf2d..fd2c93536612 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/UnableToParseClientMessageTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/UnableToParseClientMessageTest.java
@@ -46,6 +46,7 @@
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.transport.SimpleClient;
 import org.apache.cassandra.transport.messages.OptionsMessage;
+import org.apache.cassandra.utils.concurrent.Future;
 import org.assertj.core.api.Assertions;
 
 /**
@@ -263,7 +264,7 @@ public int encodedSize(Message message, ProtocolVersion version)
         }
 
         @Override
-        protected Response execute(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+        protected Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
         {
             throw new AssertionError("execute not supported");
         }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/UnifiedCompactionDensitiesTest.java b/test/distributed/org/apache/cassandra/distributed/test/UnifiedCompactionDensitiesTest.java
index 4ba721cdca43..d9ff6de3fd98 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/UnifiedCompactionDensitiesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/UnifiedCompactionDensitiesTest.java
@@ -33,6 +33,7 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.hamcrest.Matchers;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_L0_SHARDS_ENABLED;
 import static org.apache.cassandra.cql3.TombstonesWithIndexedSSTableTest.makeRandomString;
 import static org.junit.Assert.assertThat;
 
@@ -64,6 +65,7 @@ public void testTargetSSTableSize2Nodes3Dirs() throws IOException
 
     private void testTargetSSTableSize(int nodeCount, int dataDirs) throws IOException
     {
+        UCS_L0_SHARDS_ENABLED.setBoolean(true);
         try (Cluster cluster = init(builder().withNodes(nodeCount)
                                              .withDataDirCount(dataDirs)
                                              .withConfig(cfg -> cfg.set("memtable_heap_space", "100MiB"))
diff --git a/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java b/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java
index c2caac3e0e12..2b29c49b324c 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java
@@ -33,16 +33,17 @@
 import net.bytebuddy.implementation.bind.annotation.SuperCall;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.ActiveCompactions;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.ActiveOperations;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.ICluster;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.api.LogAction;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 import org.apache.cassandra.utils.concurrent.CountDownLatch;
 
 import static net.bytebuddy.matcher.ElementMatchers.named;
@@ -83,7 +84,7 @@ public void upgradeSSTablesInterruptsOngoingCompaction() throws Throwable
 
             Future<?> future = cluster.get(1).asyncAcceptsOnInstance((String ks) -> {
                 ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore("tbl");
-                CompactionManager.instance.submitMaximal(cfs, FBUtilities.nowInSeconds(), false, OperationType.COMPACTION);
+                CompactionManager.instance.submitMaximal(cfs, FBUtilities.nowInSeconds(), false, CompactionManager.instance.active, OperationType.COMPACTION);
             }).apply(KEYSPACE);
 
             Assert.assertTrue(cluster.get(1).callOnInstance(() -> CompactionLatchByteman.starting.awaitUninterruptibly(1, TimeUnit.MINUTES)));
@@ -129,7 +130,7 @@ public void compactionDoesNotCancelUpgradeSSTables() throws Throwable
 
             cluster.get(1).acceptsOnInstance((String ks) -> {
                 ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore("tbl");
-                FBUtilities.allOf(CompactionManager.instance.submitMaximal(cfs, FBUtilities.nowInSeconds(), false, OperationType.COMPACTION))
+                FBUtilities.allOf(CompactionManager.instance.submitMaximal(cfs, FBUtilities.nowInSeconds(), false, CompactionManager.instance.active, OperationType.COMPACTION))
                            .awaitUninterruptibly(1, TimeUnit.MINUTES);
 
             }).accept(KEYSPACE);
@@ -319,24 +320,25 @@ public static class UpgradeSStablesLatchByteman
 
         public static void install(ClassLoader classLoader, Integer num)
         {
-            new ByteBuddy().rebase(ActiveCompactions.class)
-                           .method(named("beginCompaction"))
+            new ByteBuddy().rebase(ActiveOperations.class)
+                           .method(named("onOperationStart"))
                            .intercept(MethodDelegation.to(UpgradeSStablesLatchByteman.class))
                            .make()
                            .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
         }
 
         @SuppressWarnings("unused")
-        public static void beginCompaction(CompactionInfo.Holder ci, @SuperCall Callable<Void> zuperCall)
+        public static NonThrowingCloseable onOperationStart(TableOperation op, @SuperCall Callable<NonThrowingCloseable> zuperCall)
         {
             try
             {
-                zuperCall.call();
-                if (ci.getCompactionInfo().getTaskType() == OperationType.UPGRADE_SSTABLES)
+                NonThrowingCloseable result = zuperCall.call();
+                if (op.getProgress().operationType() == OperationType.UPGRADE_SSTABLES)
                 {
                     starting.decrement();
                     Assert.assertTrue(start.awaitUninterruptibly(1, TimeUnit.MINUTES));
                 }
+                return result;
             }
             catch (Exception e)
             {
@@ -353,24 +355,25 @@ public static class CompactionLatchByteman
 
         public static void install(ClassLoader classLoader, Integer num)
         {
-            new ByteBuddy().rebase(ActiveCompactions.class)
-                           .method(named("beginCompaction"))
+            new ByteBuddy().rebase(ActiveOperations.class)
+                           .method(named("onOperationStart"))
                            .intercept(MethodDelegation.to(CompactionLatchByteman.class))
                            .make()
                            .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
         }
 
         @SuppressWarnings("unused")
-        public static void beginCompaction(CompactionInfo.Holder ci, @SuperCall Callable<Void> zuperCall)
+        public static NonThrowingCloseable onOperationStart(TableOperation op, @SuperCall Callable<NonThrowingCloseable> zuperCall)
         {
             try
             {
-                zuperCall.call();
-                if (ci.getCompactionInfo().getTaskType() == OperationType.COMPACTION)
+                NonThrowingCloseable result = zuperCall.call();
+                if (op.getProgress().operationType() == OperationType.COMPACTION)
                 {
                     starting.decrement();
                     Assert.assertTrue(start.awaitUninterruptibly(1, TimeUnit.MINUTES));
                 }
+                return result;
             }
             catch (Exception e)
             {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/cdc/ToggleCDCOnRepairEnabledTest.java b/test/distributed/org/apache/cassandra/distributed/test/cdc/ToggleCDCOnRepairEnabledTest.java
index 499cf076afa0..b932cab3cfc8 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/cdc/ToggleCDCOnRepairEnabledTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/cdc/ToggleCDCOnRepairEnabledTest.java
@@ -39,7 +39,7 @@ public void testCDCOnRepairIsEnabled() throws Exception
     {
         testCDCOnRepairEnabled(true, cluster -> {
             cluster.get(2).runOnInstance(() -> {
-                boolean containCDCInLog = CommitLog.instance.segmentManager
+                boolean containCDCInLog = CommitLog.instance.getSegmentManager()
                                               .getActiveSegments()
                                               .stream()
                                               .anyMatch(s -> s.getCDCState() == CommitLogSegment.CDCState.CONTAINS);
@@ -54,7 +54,7 @@ public void testCDCOnRepairIsDisabled() throws Exception
     {
         testCDCOnRepairEnabled(false, cluster -> {
             cluster.get(2).runOnInstance(() -> {
-                boolean containCDCInLog = CommitLog.instance.segmentManager
+                boolean containCDCInLog = CommitLog.instance.getSegmentManager()
                                               .getActiveSegments()
                                               .stream()
                                               .allMatch(s -> s.getCDCState() != CommitLogSegment.CDCState.CONTAINS);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCollectionSizeOnSSTableWriteTest.java b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCollectionSizeOnSSTableWriteTest.java
index 6b7daef26feb..178d4aaf0a5d 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCollectionSizeOnSSTableWriteTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCollectionSizeOnSSTableWriteTest.java
@@ -31,6 +31,8 @@
 import org.apache.cassandra.distributed.api.Feature;
 
 import static java.nio.ByteBuffer.allocate;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ENABLE_GUARDRAILS_FOR_ANONYMOUS_USER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISK_USAGE_NOTIFY_INTERVAL_MS;
 
 /**
  * Tests the guardrail for the size of collections, {@link Guardrails#collectionSize}.
@@ -52,6 +54,12 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester
     @BeforeClass
     public static void setupCluster() throws IOException
     {
+        // Disable guardrails for anonymous users (no client authentication) This allows the test to check guardrails 
+        // during sstable writes without being affected by the guardrail for anonymous users.
+        ENABLE_GUARDRAILS_FOR_ANONYMOUS_USER.setBoolean(false);
+        
+        // Ensure guardrail notifications are not suppressed
+        DISK_USAGE_NOTIFY_INTERVAL_MS.setLong(0L);
         cluster = init(Cluster.build(NUM_NODES)
                               .withConfig(c -> c.with(Feature.GOSSIP, Feature.NATIVE_PROTOCOL)
                                                 .set("collection_size_warn_threshold", WARN_THRESHOLD + "B")
@@ -65,6 +73,7 @@ public static void setupCluster() throws IOException
     @AfterClass
     public static void teardownCluster()
     {
+        DISK_USAGE_NOTIFY_INTERVAL_MS.reset();
         if (driverSession != null)
             driverSession.close();
 
@@ -278,7 +287,7 @@ public void testMapSize() throws Throwable
         execute("INSERT INTO %s (k, v) VALUES (5, ?)", map(allocate(WARN_THRESHOLD), allocate(1)));
         assertWarnedOnFlush(warnMessage("5"));
 
-        execute("INSERT INTO %s (k, v) VALUES (6, ?)", map(allocate(1), allocate(WARN_THRESHOLD)));
+        execute("INSERT INTO %s (k, v) VALUES (6, ?)", map(allocate(1), allocate(WARN_THRESHOLD + 1)));
         assertWarnedOnFlush(warnMessage("6"));
 
         execute("INSERT INTO %s (k, v) VALUES (7, ?)", map(allocate(WARN_THRESHOLD), allocate(WARN_THRESHOLD)));
@@ -287,7 +296,7 @@ public void testMapSize() throws Throwable
         execute("INSERT INTO %s (k, v) VALUES (8, ?)", map(allocate(FAIL_THRESHOLD), allocate(1)));
         assertFailedOnFlush(failMessage("8"));
 
-        execute("INSERT INTO %s (k, v) VALUES (9, ?)", map(allocate(1), allocate(FAIL_THRESHOLD)));
+        execute("INSERT INTO %s (k, v) VALUES (9, ?)", map(allocate(1), allocate(FAIL_THRESHOLD + 1)));
         assertFailedOnFlush(failMessage("9"));
 
         execute("INSERT INTO %s (k, v) VALUES (10, ?)", map(allocate(FAIL_THRESHOLD), allocate(FAIL_THRESHOLD)));
@@ -389,7 +398,7 @@ public void testMapSizeAfterCompaction()
 
         execute("INSERT INTO %s (k, v) VALUES (6, ?)", map(allocate(FAIL_THRESHOLD / 4), allocate(FAIL_THRESHOLD / 4)));
         assertWarnedOnFlush(failMessage("6"));
-        execute("UPDATE %s SET v = v + ? WHERE k = 6", map(allocate(FAIL_THRESHOLD / 4 + 1), allocate(FAIL_THRESHOLD / 4)));
+        execute("UPDATE %s SET v = v + ? WHERE k = 6", map(allocate(FAIL_THRESHOLD / 4 + 3), allocate(FAIL_THRESHOLD / 4 + 3)));
         assertWarnedOnFlush(warnMessage("6"));
         assertFailedOnCompact(failMessage("6"));
     }
@@ -427,11 +436,11 @@ private void execute(String query, Object... args)
 
     private String warnMessage(String key)
     {
-        return String.format("Detected collection v in row %s in table %s of size", key, qualifiedTableName);
+        return String.format("Detected collection v in table %s of size", qualifiedTableName);
     }
 
     private String failMessage(String key)
     {
-        return String.format("Detected collection v in row %s in table %s of size", key, qualifiedTableName);
+        return String.format("Detected collection v in table %s of size", qualifiedTableName);
     }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailDiskUsageTest.java b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailDiskUsageTest.java
index b2bb8ea098e6..92c2b7561af4 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailDiskUsageTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailDiskUsageTest.java
@@ -39,7 +39,6 @@
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.Feature;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
-import org.apache.cassandra.distributed.util.Auth;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster;
 import org.apache.cassandra.service.disk.usage.DiskUsageMonitor;
@@ -77,18 +76,7 @@ public static void setupCluster() throws IOException
                                                 .set("authenticator", "PasswordAuthenticator"))
                               .start(), 1);
 
-        Auth.waitForExistingRoles(cluster.get(1));
-
-        // create a regular user, since the default superuser is excluded from guardrails
-        com.datastax.driver.core.Cluster.Builder builder = com.datastax.driver.core.Cluster.builder().addContactPoint("127.0.0.1");
-        try (com.datastax.driver.core.Cluster c = builder.withCredentials("cassandra", "cassandra").build();
-             Session session = c.connect())
-        {
-            session.execute("CREATE USER test WITH PASSWORD 'test'");
-        }
-
-        // connect using that superuser, we use the driver to get access to the client warnings
-        driverCluster = builder.withCredentials("test", "test").build();
+        driverCluster = buildDriverCluster(cluster);
         driverSession = driverCluster.connect();
     }
 
@@ -111,8 +99,14 @@ protected Cluster getCluster()
         return cluster;
     }
 
+    @Override
+    protected Session getSession()
+    {
+        return driverSession;
+    }
+
     @Test
-    public void testDiskUsage() throws Throwable
+    public void testDiskUsage()
     {
         schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v int)");
         String insert = format("INSERT INTO %s(k, v) VALUES (?, 0)");
diff --git a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailItemsPerCollectionOnSSTableWriteTest.java b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailItemsPerCollectionOnSSTableWriteTest.java
index 260752e9d986..d79718f68ea4 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailItemsPerCollectionOnSSTableWriteTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailItemsPerCollectionOnSSTableWriteTest.java
@@ -29,6 +29,8 @@
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.ICoordinator;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISK_USAGE_NOTIFY_INTERVAL_MS;
+
 /**
  * Tests the guardrail for the number of items on a collection, {@link Guardrails#itemsPerCollection}.
  * <p>
@@ -48,6 +50,8 @@ public class GuardrailItemsPerCollectionOnSSTableWriteTest extends GuardrailTest
     @BeforeClass
     public static void setupCluster() throws IOException
     {
+        // Ensure guardrail notifications are not suppressed
+        DISK_USAGE_NOTIFY_INTERVAL_MS.setLong(0L);
         cluster = init(Cluster.build(NUM_NODES)
                               .withConfig(c -> c.set("items_per_collection_warn_threshold", WARN_THRESHOLD)
                                                 .set("items_per_collection_fail_threshold", FAIL_THRESHOLD))
@@ -59,6 +63,7 @@ public static void setupCluster() throws IOException
     @AfterClass
     public static void teardownCluster()
     {
+        DISK_USAGE_NOTIFY_INTERVAL_MS.reset();
         if (cluster != null)
             cluster.close();
     }
@@ -134,6 +139,12 @@ public void testSetSizeAfterCompaction() throws Throwable
         execute("UPDATE %s SET v = v + {2, 3} WHERE k = 1");
         assertNotWarnedOnFlush();
         assertWarnedOnCompact(warnMessage("1", 3));
+    }
+
+    @Test
+    public void testSetSizeAfterCompaction_failOnCompact() throws Throwable
+    {
+        schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v set<int>)");
 
         execute("INSERT INTO %s (k, v) VALUES (2, {1, 2})");
         assertNotWarnedOnFlush();
@@ -207,12 +218,24 @@ public void testListSizeAfterCompaction() throws Throwable
         execute("UPDATE %s SET v = v + [2, 3] WHERE k = 1");
         assertNotWarnedOnFlush();
         assertWarnedOnCompact(warnMessage("1", 3));
+    }
+
+    @Test
+    public void testListSizeAfterCompaction_failOnComapact() throws Throwable
+    {
+        schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v list<int>)");
 
         execute("INSERT INTO %s (k, v) VALUES (2, [1, 2])");
         assertNotWarnedOnFlush();
         execute("UPDATE %s SET v = v + [3, 4, 5] WHERE k = 2");
         assertWarnedOnFlush(warnMessage("2", 3));
         assertFailedOnCompact(failMessage("2", 5));
+    }
+
+    @Test
+    public void testListSizeAfterCompaction_nullNoWarn() throws Throwable
+    {
+        schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v list<int>)");
 
         execute("INSERT INTO %s (k, v) VALUES (3, [1, 2, 3])");
         assertWarnedOnFlush(warnMessage("3", 3));
@@ -280,12 +303,24 @@ public void testMapSizeAfterCompaction()
         execute("UPDATE %s SET v = v + {2:20} WHERE k = 0");
         assertNotWarnedOnFlush();
         assertNotWarnedOnCompact();
+    }
+
+    @Test
+    public void testMapSizeAfterCompaction_warnOnCompact()
+    {
+        schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v map<int, int>)");
 
         execute("INSERT INTO %s (k, v) VALUES (1, {1:10})");
         assertNotWarnedOnFlush();
         execute("UPDATE %s SET v = v + {2:20, 3:30} WHERE k = 1");
         assertNotWarnedOnFlush();
         assertWarnedOnCompact(warnMessage("1", 3));
+    }
+
+    @Test
+    public void testMapSizeAfterCompaction_failOnCompact()
+    {
+        schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v map<int, int>)");
 
         execute("INSERT INTO %s (k, v) VALUES (2, {1:10, 2:20})");
         assertNotWarnedOnFlush();
@@ -325,15 +360,15 @@ private void execute(String query)
 
     private String warnMessage(String key, int numItems)
     {
-        return String.format("Detected collection v in row %s in table %s with %d items, " +
+        return String.format("Detected collection v in table %s with %d items, " +
                              "this exceeds the warning threshold of %d.",
-                             key, qualifiedTableName, numItems, WARN_THRESHOLD);
+                             qualifiedTableName, numItems, WARN_THRESHOLD);
     }
 
     private String failMessage(String key, int numItems)
     {
-        return String.format("Detected collection v in row %s in table %s with %d items, " +
+        return String.format("Detected collection v in table %s with %d items, " +
                              "this exceeds the failure threshold of %d.",
-                             key, qualifiedTableName, numItems, FAIL_THRESHOLD);
+                             qualifiedTableName, numItems, FAIL_THRESHOLD);
     }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailNonPartitionRestrictedQueryTest.java b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailNonPartitionRestrictedQueryTest.java
index 24d13fd0252b..70249c2132ec 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailNonPartitionRestrictedQueryTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailNonPartitionRestrictedQueryTest.java
@@ -36,6 +36,7 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.marshal.LongType;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.Feature;
@@ -143,12 +144,13 @@ public void testSAIWarnThreshold()
 
         assertWarnAborts(0, 0);
 
-        // create 3 more SSTables on each node, this will trigger warn threshold (3 > 2 but < 5)
+        // create 3 more SSTables on each node, this will trigger warn threshold (4 > 2 but < 5)
         valueToQuery = createSSTables(3);
+        String valueToQueryString = LongType.instance.toCQLString(LongType.instance.decompose(valueToQuery), true);
         String expectedMessage = tooManyIndexesReadWarnMessage(cluster.size(),
-                                                               3,
+                                                               4,
                                                                String.format("SELECT * FROM %s.%s WHERE v1 = %s ALLOW FILTERING",
-                                                                             KEYSPACE, tableName, valueToQuery));
+                                                                             KEYSPACE, tableName, valueToQueryString));
         assertThat(getOnlyElement(executeSelect(valueToQuery, false))).contains(expectedMessage);
 
         assertWarnAborts(1, 0);
@@ -166,9 +168,9 @@ public void testSAIWarnThreshold()
 
         // notice we expect warnings from 2 nodes
         expectedMessage = tooManyIndexesReadWarnMessage(cluster.size() - 1,
-                                                        3,
+                                                        4,
                                                         String.format("SELECT * FROM %s.%s WHERE v1 = %s ALLOW FILTERING",
-                                                                      KEYSPACE, tableName, valueToQuery));
+                                                                      KEYSPACE, tableName, valueToQueryString));
 
         assertThat(getOnlyElement(executeSelect(valueToQuery, false))).contains(expectedMessage);
 
@@ -205,11 +207,10 @@ public void testSAIFailThreshold()
 
         // create 6 SSTables on each node, this will trigger fail threshold (6 > 5)
         long valueToQuery = createSSTables(6);
-
+        String valueToQueryString = LongType.instance.toCQLString(LongType.instance.decompose(valueToQuery), true);
         String expectedMessage = String.format("referenced %s SSTable indexes for a query without restrictions on partition key " +
                                                "and aborted the query SELECT * FROM %s.%s WHERE v1 = %s ALLOW FILTERING",
-                                               6, KEYSPACE, tableName, valueToQuery);
-
+                                               6, KEYSPACE, tableName, valueToQueryString);
         assertThat(getOnlyElement(executeSelect(valueToQuery, true))).contains(expectedMessage);
 
         assertWarnAborts(0, 1);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/FailedBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/FailedBootstrapTest.java
index 7a8e42186a0f..b8196fe90f37 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/FailedBootstrapTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/FailedBootstrapTest.java
@@ -27,6 +27,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -43,7 +45,6 @@
 import org.apache.cassandra.distributed.api.NodeToolResult;
 import org.apache.cassandra.distributed.api.TokenSupplier;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
-import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
 import org.apache.cassandra.streaming.StreamException;
 import org.apache.cassandra.streaming.StreamResultFuture;
 import org.assertj.core.api.Assertions;
@@ -96,8 +97,9 @@ public void roleSetupDoesNotProduceUnavailables() throws IOException
 
             // do we have any read metrics have unavailables?
             added.runOnInstance(() -> {
-                Assertions.assertThat(ClientRequestsMetricsHolder.readMetrics.unavailables.getCount()).describedAs("read unavailables").isEqualTo(0);
-                Assertions.assertThat(ClientRequestsMetricsHolder.casReadMetrics.unavailables.getCount()).describedAs("CAS read unavailables").isEqualTo(0);
+                ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
+                Assertions.assertThat(metrics.readMetrics.unavailables.getCount()).describedAs("read unavailables").isEqualTo(0);
+                Assertions.assertThat(metrics.casReadMetrics.unavailables.getCount()).describedAs("CAS read unavailables").isEqualTo(0);
             });
         }
     }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/HostReplacementTest.java b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/HostReplacementTest.java
index 8219d43ad1f0..4a3d7183858a 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/HostReplacementTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/HostReplacementTest.java
@@ -19,8 +19,10 @@
 package org.apache.cassandra.distributed.test.hostreplacement;
 
 import java.io.IOException;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.UUID;
 
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -32,22 +34,37 @@
 import org.apache.cassandra.distributed.api.Feature;
 import org.apache.cassandra.distributed.api.ICoordinator;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
 import org.apache.cassandra.distributed.api.SimpleQueryResult;
 import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.impl.InstanceConfig;
 import org.apache.cassandra.distributed.shared.AssertUtils;
 import org.apache.cassandra.distributed.shared.ClusterUtils;
+import org.apache.cassandra.distributed.shared.WithProperties;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.StorageService;
 import org.assertj.core.api.Assertions;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SKIP_SCHEMA_CHECK;
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIPER_QUARANTINE_DELAY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.assertInRing;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.assertRingIs;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingHealthy;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin;
+import static org.apache.cassandra.distributed.shared.ClusterUtils.getDirectories;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.getTokenMetadataTokens;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.replaceHostAndStart;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.junit.Assert.assertFalse;
 
 public class HostReplacementTest extends TestBaseImpl
 {
@@ -206,6 +223,91 @@ public void seedGoesDownBeforeDownHost() throws IOException
         }
     }
 
+    /**
+     * Make sure that a node stuck in hibernate state due to failed replacement can retry the replacement procedure and succeed.
+     */
+    @Test
+    public void retryingFailedReplaceWithNodeInHibernateState() throws IOException
+    {
+        try (WithProperties properties = new WithProperties())
+        {
+            properties.set(GOSSIP_DISABLE_THREAD_VALIDATION, "true");
+
+            // given a two node cluster with one seed
+            TokenSupplier even = TokenSupplier.evenlyDistributedTokens(2);
+            try (Cluster cluster = Cluster.build(2)
+                                          .withConfig(c -> c.with(Feature.GOSSIP, Feature.NATIVE_PROTOCOL)
+                                                            .set(Constants.KEY_DTEST_API_STARTUP_FAILURE_AS_SHUTDOWN, true))
+                                          .withTokenSupplier(node -> even.token(node == 3 ? 2 : node))
+                                          .start() )
+            {
+                IInvokableInstance seed = cluster.get(1);
+                IInvokableInstance nodeToReplace = cluster.get(2);
+
+                setupCluster(cluster);
+                SimpleQueryResult expectedState = nodeToReplace.coordinator().executeWithResult("SELECT * FROM " + KEYSPACE + ".tbl", ConsistencyLevel.ALL);
+
+                // when
+                // stop the node to replace
+                stopUnchecked(nodeToReplace);
+                // wipe the node to replace
+                getDirectories(nodeToReplace).forEach(FileUtils::deleteRecursive);
+
+                String toReplaceAddress = nodeToReplace.config().broadcastAddress().getAddress().getHostAddress();
+                // set hibernate status for the node to replace on seed
+                seed.runOnInstance(putInHibernation(toReplaceAddress));
+
+                // we need to fake a new host id
+                ((InstanceConfig) nodeToReplace.config()).setHostId(UUID.randomUUID());
+                // enable autoboostrap
+                nodeToReplace.config().set("auto_bootstrap", true);
+
+                // first replacement will fail as the node was announced as hibernated and no-one can contact it as startup
+                assertThatExceptionOfType(IllegalStateException.class).isThrownBy(() -> {
+                    ClusterUtils.start(nodeToReplace, props -> {
+                        // set the replacement address
+                        props.set(REPLACE_ADDRESS, toReplaceAddress);
+                    });
+                }).withMessageContaining("Unable to contact any seeds");
+
+                // then
+                // retrying replacement will succeed as the node announced itself as shutdown before killing itself
+                ClusterUtils.start(nodeToReplace, props -> {
+                    // set the replacement address
+                    props.set(REPLACE_ADDRESS, toReplaceAddress);
+                });
+                assertFalse("replaces node should be up", nodeToReplace.isShutdown());
+
+                // the data after replacement should be consistent
+                awaitRingJoin(seed, nodeToReplace);
+                awaitRingJoin(nodeToReplace, seed);
+
+                validateRows(seed.coordinator(), expectedState);
+                validateRows(nodeToReplace.coordinator(), expectedState);
+            }
+        }
+    }
+
+    private static IIsolatedExecutor.SerializableRunnable putInHibernation(String address)
+    {
+        return () -> {
+            InetAddressAndPort endpoint;
+            try
+            {
+                endpoint = InetAddressAndPort.getByName(address);
+            }
+            catch (UnknownHostException e)
+            {
+                throw new RuntimeException(e);
+            }
+            EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
+            VersionedValue newStatus = StorageService.instance.valueFactory.hibernate(true);
+            epState.addApplicationState(ApplicationState.STATUS, newStatus);
+            epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, newStatus);
+            Gossiper.instance.handleMajorStateChange(endpoint, epState);
+        };
+    }
+
     static void setupCluster(Cluster cluster)
     {
         fixDistributedSchemas(cluster);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/metrics/CoordinatorReadLatencyMetricTest.java b/test/distributed/org/apache/cassandra/distributed/test/metrics/CoordinatorReadLatencyMetricTest.java
index ab3de57cb66b..20a0a609fb03 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/metrics/CoordinatorReadLatencyMetricTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/metrics/CoordinatorReadLatencyMetricTest.java
@@ -22,13 +22,16 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;
 
 import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
-import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
 import org.apache.cassandra.service.paxos.Paxos;
 
 import static org.junit.Assert.assertTrue;
@@ -40,9 +43,14 @@ public void internalPagingWithAggregateTest() throws Throwable
     {
         try (Cluster cluster = init(builder().withNodes(1).start()))
         {
-            cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))"));
+            // CC only supports paging in bytes. Set the smallest page size allowed.
+            cluster.get(1).callOnInstance(() -> DatabaseDescriptor.getRawConfig().aggregation_subpage_size_in_kb = 1);
+
+            // Create rows greater than the 1kb subpage size
+            String payload = StringUtils.repeat('1', 1024 * 2);
+            cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck))"));
             for (int i = 0; i < 100; i++)
-                cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (pk, ck ,v) values (0, ?, 1)"), ConsistencyLevel.ALL, i);
+                cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (pk, ck ,v) values (0, ?, ?)"), ConsistencyLevel.ALL, i, payload);
 
             // Serial and non-serial reads have separates code paths, so exercise them both
             testAggregationQuery(cluster, ConsistencyLevel.ALL);
@@ -97,13 +105,25 @@ private void verifyLatencyMetricsWhenPaging(Cluster cluster,
                                                 String query,
                                                 ConsistencyLevel consistencyLevel)
     {
-        long countBefore = cluster.get(1).callOnInstance(() -> ClientRequestsMetricsHolder.readMetrics.latency.getCount());
-        long totalLatencyBefore = cluster.get(1).callOnInstance(() -> ClientRequestsMetricsHolder.readMetrics.totalLatency.getCount());
+        long countBefore = cluster.get(1).callOnInstance(() -> {
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
+            return metrics.readMetrics.executionTimeMetrics.latency.getCount();
+        });
+        long totalLatencyBefore = cluster.get(1).callOnInstance(() -> {
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
+            return metrics.readMetrics.executionTimeMetrics.totalLatency.getCount();
+        });
         long startTime = System.nanoTime();
         cluster.coordinator(1).executeWithPaging(query, consistencyLevel, pagesize);
         long elapsedTime = System.nanoTime() - startTime;
-        long countAfter = cluster.get(1).callOnInstance(() -> ClientRequestsMetricsHolder.readMetrics.latency.getCount());
-        long totalLatencyAfter = cluster.get(1).callOnInstance(() -> ClientRequestsMetricsHolder.readMetrics.totalLatency.getCount());
+        long countAfter = cluster.get(1).callOnInstance(() -> {
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
+            return metrics.readMetrics.executionTimeMetrics.latency.getCount();
+        });
+        long totalLatencyAfter = cluster.get(1).callOnInstance(() -> {
+            ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
+            return metrics.readMetrics.executionTimeMetrics.totalLatency.getCount();
+        });
 
         long latenciesRecorded = countAfter - countBefore;
         assertTrue("Expected to have recorded at least 1 latency measurement per-individual read", latenciesRecorded >= expectedQueries);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
index 5a042251bbee..b191952e4661 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
@@ -286,7 +286,7 @@ public void bootstrapJMXStatus() throws Throwable
                                         .withInstanceInitializer(BootstrapTest.BB::install)
                                         .start())
         {
-            bootstrapAndJoinNode(cluster);
+            bootstrapAndJoinNodeNoWait(cluster);
 
             IInvokableInstance joiningInstance = cluster.get(3);
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/ANNOptionsDistributedTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/ANNOptionsDistributedTest.java
new file mode 100644
index 000000000000..a73e8b05a3b4
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/ANNOptionsDistributedTest.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICoordinator;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.net.MessagingService;
+import org.assertj.core.api.Assertions;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+
+/**
+ * Distributed tests for ANN options.
+ */
+public class ANNOptionsDistributedTest extends TestBaseImpl
+{
+    private static final int NUM_REPLICAS = 2;
+    private static final int RF = 2;
+
+    /**
+     * Test that ANN options are accepted in clusters with all nodes in DS 11 (although SAI will reject them for now).
+     */
+    @Test
+    public void testANNOptionsWithAllDS11() throws Throwable
+    {
+        CassandraRelevantProperties.DS_CURRENT_MESSAGING_VERSION.setInt(MessagingService.VERSION_DS_11);
+
+        try (Cluster cluster = init(Cluster.build(NUM_REPLICAS)
+                                           .withConfig(config -> config.with(GOSSIP).with(NETWORK))
+                                           .start(), RF))
+        {
+            // null indicates that the query should succeed
+            testSelectWithAnnOptions(cluster, null);
+        }
+    }
+
+    /**
+     * Test that ANN options are rejected in clusters with all nodes below DS 11.
+     */
+    @Test
+    public void testANNOptionsWithAllDS10() throws Throwable
+    {
+        CassandraRelevantProperties.DS_CURRENT_MESSAGING_VERSION.setInt(MessagingService.VERSION_DS_10);
+
+        try (Cluster cluster = init(Cluster.build(NUM_REPLICAS)
+                                           .withConfig(config -> config.with(GOSSIP).with(NETWORK))
+                                           .start(), RF))
+        {
+            testSelectWithAnnOptions(cluster, "ANN options are not supported in clusters below DS 11.");
+        }
+    }
+
+    /**
+     * Test that ANN options are rejected in clusters with some nodes below DS 11.
+     */
+    @Test
+    public void testANNOptionsWithMixedDS10AndDS11() throws Throwable
+    {
+        assert CassandraRelevantProperties.DS_CURRENT_MESSAGING_VERSION.getInt() >= MessagingService.VERSION_DS_11;
+
+        try (Cluster cluster = init(Cluster.build(NUM_REPLICAS)
+                                           .withInstanceInitializer(BB::install)
+                                           .withConfig(config -> config.with(GOSSIP).with(NETWORK).with(NATIVE_PROTOCOL))
+                                           .start(), RF))
+        {
+            testSelectWithAnnOptions(cluster, "ANN options are not supported in clusters below DS 11.");
+        }
+    }
+
+    private static void testSelectWithAnnOptions(Cluster cluster, String expectedErrorMessage)
+    {
+        cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int PRIMARY KEY, n int, v vector<float, 2>)"));
+        cluster.schemaChange(withKeyspace("CREATE CUSTOM INDEX ON %s.t(v) USING 'StorageAttachedIndex'"));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        String select = withKeyspace("SELECT * FROM %s.t ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'rerank_k': 10}");
+
+        for (int i = 1; i <= cluster.size(); i++)
+        {
+            ICoordinator coordinator = cluster.coordinator(i);
+            if (expectedErrorMessage == null)
+                coordinator.execute(select, ConsistencyLevel.ONE);
+            else
+                Assertions.assertThatThrownBy(() -> coordinator.execute(select, ConsistencyLevel.ONE))
+                          .hasMessageContaining(expectedErrorMessage);
+        }
+    }
+
+    /**
+     * Injection to set the current version of the first cluster node to DS 10.
+     */
+    public static class BB
+    {
+        public static void install(ClassLoader classLoader, int node)
+        {
+            if (node == 1)
+            {
+                new ByteBuddy().rebase(MessagingService.class)
+                               .method(named("currentVersion"))
+                               .intercept(MethodDelegation.to(BB.class))
+                               .make()
+                               .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+            }
+        }
+
+        @SuppressWarnings("unused")
+        public static int currentVersion()
+        {
+            return MessagingService.VERSION_DS_10;
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/AnalyzerDistributedTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/AnalyzerDistributedTest.java
new file mode 100644
index 000000000000..3b2643ab93bf
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/AnalyzerDistributedTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+public class AnalyzerDistributedTest extends TestBaseImpl
+{
+    @Rule
+    public SAITester.FailureWatcher failureRule = new SAITester.FailureWatcher();
+
+    private static final String CREATE_KEYSPACE = "CREATE KEYSPACE IF NOT EXISTS %%s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d}";
+    private static final int NUM_REPLICAS = 3;
+    private static final int RF = 2;
+
+    private static final AtomicInteger seq = new AtomicInteger();
+    private static String table;
+
+    private static Cluster cluster;
+
+    @BeforeClass
+    public static void setupCluster() throws Exception
+    {
+        cluster = Cluster.build(NUM_REPLICAS)
+                         .withConfig(config -> config.with(GOSSIP).with(NETWORK))
+                         .start();
+
+        cluster.schemaChange(withKeyspace(String.format(CREATE_KEYSPACE, RF)));
+    }
+
+    @AfterClass
+    public static void closeCluster()
+    {
+        if (cluster != null)
+            cluster.close();
+    }
+
+    @Before
+    public void before()
+    {
+        table = "table_" + seq.getAndIncrement();
+    }
+
+    @After
+    public void after()
+    {
+        cluster.schemaChange(formatQuery("DROP TABLE IF EXISTS %s"));
+    }
+
+    @Test
+    public void testAnalyzerSearch()
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (pk int PRIMARY KEY, not_analyzed int, val text)"));
+        cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer': 'standard'}"));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        var iterations = 15000;
+        for (int i = 0; i < iterations; i++)
+        {
+            var x = i % 100;
+            if (i % 100 == 0)
+            {
+                execute(String.format(
+                "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')",
+                KEYSPACE + '.' + table, i, x, "this will be tokenized"));
+            }
+            else if (i % 2 == 0)
+            {
+                execute(String.format(
+                "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')",
+                KEYSPACE + '.' + table, i, x, "this is different"));
+            }
+            else
+            {
+                execute(String.format(
+                "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')",
+                KEYSPACE + '.' + table, i, x, "basic test"));
+            }
+        }
+        // We match the first inserted statement here, and that one is just written 1/100 times
+        var result = execute("SELECT * FROM %s WHERE val : 'tokenized'");
+        assertThat(result).hasNumberOfRows(iterations / 100);
+        // We match the first and second inserted statements here, and those account for 1/2 the inserts
+        result = execute("SELECT * FROM %s WHERE val : 'this'");
+        assertThat(result).hasNumberOfRows(iterations / 2);
+        // We match the last write here, and that accounts for the other 1/2 of the inserts
+        result = execute("SELECT * FROM %s WHERE val : 'test'");
+        assertThat(result).hasNumberOfRows(iterations / 2);
+    }
+
+    /**
+     * See CNDB-12739 for more details.
+     */
+    @Test
+    public void testIndexAndQueryAnalyzerSearch()
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (c1 int PRIMARY KEY , c2 text)"));
+        cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(c2) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                                         "'index_analyzer': '{" +
+                                         "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                                         "  \"filters\" : [ { \"name\" : \"lowercase\", \"args\": {} }, " +
+                                         "                  { \"name\" : \"edgengram\", \"args\": { \"minGramSize\":\"1\", \"maxGramSize\":\"30\" } }]," +
+                                         "  \"charFilters\" : []}', " +
+                                         "'query_analyzer': '{" +
+                                         "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                                         "  \"filters\" : [ {\"name\" : \"lowercase\",\"args\": {}} ]}'}"));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        execute("INSERT INTO %s(c1,c2) VALUES (1, 'astra quick fox')");
+        execute("INSERT INTO %s(c1,c2) VALUES (2, 'astra quick foxes')");
+        execute("INSERT INTO %s(c1,c2) VALUES (3, 'astra1')");
+        execute("INSERT INTO %s(c1,c2) VALUES (4, 'astra4 -1@a#')");
+
+        Object[][] result = execute("SELECT * FROM %s WHERE c2 :'ast' ");
+        assertThat(result).hasNumberOfRows(4);
+    }
+
+    @Test
+    public void testEdgeNgramFilterWithOR() throws Throwable
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (id text PRIMARY KEY, val text)"));
+        cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                                         "'index_analyzer': '{\n" +
+                                         "\t\"tokenizer\":{\"name\":\"standard\", \"args\":{}}," +
+                                         "\t\"filters\":[{\"name\":\"lowercase\", \"args\":{}}, " +
+                                         "{\"name\":\"edgengram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"30\"}}],\n" +
+                                         "\t\"charFilters\":[]" +
+                                         "}'};"));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 some other word')");
+        execute("INSERT INTO %s (id, val) VALUES ('4', 'WFS7093AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('5', 'WFS0565AU')");
+
+        beforeAndAfterFlush(cluster, KEYSPACE, () -> {
+
+            // match (:)
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL0133AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'WFS2684AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : ''").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val : 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : '' OR val : 'WFS2684AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : '' AND val : 'WFS2684AU'").length);
+
+            // equals (=)
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL0133AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'WFS2684AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = ''").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val = 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = '' OR val = 'WFS2684AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = '' AND val = 'WFS2684AU'").length);
+
+            // mixed match (:) and equals (=)
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val : 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = '' OR val : 'WFS2684AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = '' AND val : 'WFS2684AU'").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val = 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : '' OR val = 'WFS2684AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : '' AND val = 'WFS2684AU'").length);
+        });
+    }
+
+    @Test
+    public void testNgramFilterWithOR() throws Throwable
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (id text PRIMARY KEY, val text)"));
+        cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': '{\n" +
+                    "\t\"tokenizer\":{\"name\":\"standard\", \"args\":{}}," +
+                    "\t\"filters\":[{\"name\":\"lowercase\", \"args\":{}}, " +
+                    "{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"30\"}}],\n" +
+                    "\t\"charFilters\":[]" +
+                    "}'};"));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 some other words')");
+        execute("INSERT INTO %s (id, val) VALUES ('4', 'WFS7093AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('5', 'WFS0565AU')");
+
+        beforeAndAfterFlush(cluster, KEYSPACE, () -> {
+
+            // match (:)
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL0133AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : '268'").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val : 'WFS2684AU'").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : '133' OR val : 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL' AND val : 'AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : 'XYZ' AND val : 'AU'").length);
+
+            // equals (=)
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL0133AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = '268'").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val = 'WFS2684AU'").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = '133' OR val = 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL' AND val = 'AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = 'XYZ' AND val = 'AU'").length);
+
+            // mixed match (:) and equals (=)
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val = 'WFS2684AU'").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : '133' OR val = 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL' AND val = 'AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : 'XYZ' AND val = 'AU'").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val : 'WFS2684AU'").length);
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = '133' OR val : 'WFS2684AU'").length);
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL' AND val : 'AU'").length);
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = 'XYZ' AND val : 'AU'").length);
+        });
+    }
+
+    private static Object[][] execute(String query)
+    {
+        return cluster.coordinator(1).execute(formatQuery(query), ConsistencyLevel.QUORUM);
+    }
+
+    private static String formatQuery(String query)
+    {
+        return String.format(query, KEYSPACE + '.' + table);
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/BM25DistributedTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/BM25DistributedTest.java
new file mode 100644
index 000000000000..95c7587ea044
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/BM25DistributedTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class BM25DistributedTest extends TestBaseImpl
+{
+    private static final String CREATE_KEYSPACE = "CREATE KEYSPACE %%s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d}";
+    private static final String CREATE_TABLE = "CREATE TABLE %s (k int PRIMARY KEY, v text)";
+    private static final String CREATE_INDEX = "CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer': '{\"tokenizer\" : {\"name\" : \"standard\"}, \"filters\" : [{\"name\" : \"porterstem\"}]}'}";
+
+    // To get consistent results from BM25 we need to know which docs are evaluated, the easiest way
+    // to do that is to put all the docs on every replica
+    private static final int NUM_NODES = 3;
+    private static final int RF = 3;
+
+    private static Cluster cluster;
+    private static String table;
+
+    private static final AtomicInteger seq = new AtomicInteger();
+
+    @BeforeClass
+    public static void setupCluster() throws Exception
+    {
+        cluster = Cluster.build(NUM_NODES)
+                        .withTokenCount(1)
+                        .withDataDirCount(1)
+                        .withConfig(config -> config.with(GOSSIP).with(NETWORK))
+                        .start();
+
+        cluster.schemaChange(withKeyspace(String.format(CREATE_KEYSPACE, RF)));
+        cluster.forEach(i -> i.runOnInstance(() -> org.apache.cassandra.index.sai.SAIUtil.setLatestVersion(Version.EC)));
+    }
+
+    @AfterClass
+    public static void closeCluster()
+    {
+        if (cluster != null)
+            cluster.close();
+    }
+
+    @Before
+    public void before()
+    {
+        table = "table_" + seq.getAndIncrement();
+        cluster.schemaChange(formatQuery(CREATE_TABLE));
+        cluster.schemaChange(formatQuery(CREATE_INDEX));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+    }
+
+    @Test
+    public void testTermFrequencyOrdering()
+    {
+        // Insert documents with varying frequencies of the term "apple"
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'apple apple')");
+        execute("INSERT INTO %s (k, v) VALUES (3, 'apple apple apple')");
+
+        // Query memtable index
+        assertBM25Ordering();
+
+        // Flush and query on-disk index
+        cluster.forEach(n -> n.flush(KEYSPACE));
+        assertBM25Ordering();
+    }
+
+    private void assertBM25Ordering()
+    {
+        Object[][] result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3");
+        assertThat(result).hasNumberOfRows(3);
+        
+        // Results should be ordered by term frequency (highest to lowest)
+        assertThat((Integer) result[0][0]).isEqualTo(3); // 3 occurrences
+        assertThat((Integer) result[1][0]).isEqualTo(2); // 2 occurrences
+        assertThat((Integer) result[2][0]).isEqualTo(1); // 1 occurrence
+    }
+
+    private static Object[][] execute(String query)
+    {
+        return execute(query, ConsistencyLevel.QUORUM);
+    }
+
+    private static Object[][] execute(String query, ConsistencyLevel consistencyLevel)
+    {
+        return cluster.coordinator(1).execute(formatQuery(query), consistencyLevel);
+    }
+
+    private static String formatQuery(String query)
+    {
+        return String.format(query, KEYSPACE + '.' + table);
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/ConcurrencyFactorTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/ConcurrencyFactorTest.java
index 17921db942f0..7b811a3a3dd8 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/ConcurrencyFactorTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/ConcurrencyFactorTest.java
@@ -19,41 +19,65 @@
 package org.apache.cassandra.distributed.test.sai;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
 import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
 
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
-import org.apache.cassandra.distributed.api.TokenSupplier;
-import org.apache.cassandra.distributed.impl.TracingUtil;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
-import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
 
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
-import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
 
 public class ConcurrencyFactorTest extends TestBaseImpl
 {
     private static final String SAI_TABLE = "sai_simple_primary_key";
-    private static final int NODES = 3;
 
-    private Cluster cluster;
+    private static final int nodes = 3;
+
+    private org.apache.cassandra.distributed.Cluster cluster;
+
+    static
+    {
+        CassandraRelevantProperties.CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY.setString(OverridableClientRequestsMetricsProvider.class.getName());
+    }
+
+    public static class OverridableClientRequestsMetricsProvider implements ClientRequestsMetricsProvider
+    {
+        static volatile ClientRequestsMetrics metrics = new ClientRequestsMetrics("");
+
+        @Override
+        public ClientRequestsMetrics metrics(String keyspace)
+        {
+            return metrics;
+        }
+
+        public static void reset()
+        {
+            metrics.release();
+            metrics = new ClientRequestsMetrics("");
+        }
+    }
 
     @Before
     public void init() throws IOException
     {
-        cluster = init(Cluster.build(NODES).withTokenSupplier(generateTokenSupplier())
-                                           .withTokenCount(1)
-                                           .withConfig(config -> config.with(GOSSIP).with(NETWORK)).start());
+        cluster = init(Cluster.build(nodes).withTokenSupplier(node -> {
+            switch (node)
+            {
+                case 1: return -9223372036854775808L;
+                case 2: return -3074457345618258602L;
+                case 3: return 3074457345618258603L;
+                default: throw new IllegalArgumentException();
+            }
+        }).withConfig(config -> config.with(NETWORK).with(GOSSIP)).start());
     }
 
     @After
@@ -62,24 +86,29 @@ public void cleanup()
         cluster.close();
     }
 
-    @Test
-    public void testInitialConcurrencySelection()
+    private void insertRows(long startVal, long endVal, long increment)
     {
-        cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, state ascii, gdp bigint, PRIMARY KEY (pk)) WITH compaction = " +
-                                           " {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }", KEYSPACE, SAI_TABLE));
-        cluster.schemaChange(String.format("CREATE INDEX ON %s.%s (gdp) USING 'sai'", KEYSPACE, SAI_TABLE));
-        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
-
         String template = "INSERT INTO %s.%s (pk, state, gdp) VALUES (%s, %s)";
         Random rnd = new Random();
         String fakeState, rowData;
         int i = 0;
-        for (long val = 1_000_000_000L; val <= 16_000_000_000L; val += 1_000_000_000L)
+        for (long val = startVal; val <= endVal; val += increment)
         {
-            fakeState = String.format("%c%c", (char)(rnd.nextInt(26) + 'A'), (char)(rnd.nextInt(26) + 'A'));
+            fakeState = String.format("%c%c", (char) (rnd.nextInt(26) + 'A'), (char) (rnd.nextInt(26) + 'A'));
             rowData = String.format("'%s', %s", fakeState, val);
             cluster.coordinator(1).execute(String.format(template, KEYSPACE, SAI_TABLE, i++, rowData), ConsistencyLevel.LOCAL_ONE);
         }
+    }
+
+    @Test
+    public void testInitialConcurrencySelection()
+    {
+        cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, state ascii, gdp bigint, PRIMARY KEY (pk)) WITH compaction = " +
+                                           " {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }", KEYSPACE, SAI_TABLE));
+        cluster.schemaChange(String.format("CREATE CUSTOM INDEX ON %s.%s (gdp) USING 'StorageAttachedIndex'", KEYSPACE, SAI_TABLE));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        insertRows(1_000_000_000L, 16_000_000_000L, 1_000_000_000L);
 
         // flush all nodes, expected row distribution by partition key value
         // node0: 9, 14, 12, 3
@@ -87,52 +116,44 @@ public void testInitialConcurrencySelection()
         // node2: 4, 15, 7, 6
         cluster.forEach((node) -> node.flush(KEYSPACE));
 
-        // We are expecting any of 3 specific trace messages indicating how the query has been handled:
-        //
-        // Submitting range requests on <n> ranges with a concurrency of <n>
-        //   Initial concurrency wasn't estimated and max concurrency was used instead (and SAI index was involved)
-        // Submitting range requests on <n> ranges with a concurrency of <n> (<m> rows per range expected)
-        //   Initial concurrency was estimated based on estimated rows per range (non-SAI range query)
-        // Executing single-partition query on <table>
-        //   Non-range single-partition query
-
-        // SAI range query so should bypass initial concurrency estimation
+        // we expect to use StorageProxy#RangeCommandIterator and the hit count to increase
         String query = String.format("SELECT state FROM %s.%s WHERE gdp > ? AND gdp < ? LIMIT 20", KEYSPACE, SAI_TABLE);
-        runAndValidate("Submitting range requests on 3 ranges with a concurrency of 3", query, 3_000_000_000L, 7_000_000_000L);
+        runAndValidate(1, 1, query, 3_000_000_000L, 7_000_000_000L);
 
-        // Partition-restricted query so not a range query
+        // partition-restricted query
+        // we don't expect to use StorageProxy#RangeCommandIterator so previous hit count remains the same
         query = String.format("SELECT state FROM %s.%s WHERE pk = ?", KEYSPACE, SAI_TABLE);
-        runAndValidate("Executing single-partition query on sai_simple_primary_key", query, 0);
+        runAndValidate(0, 0, query, 0);
 
-        // Token-restricted range query not using SAI so should use initial concurrency estimation
+        // token-restricted query
+        // we expect StorageProxy#RangeCommandIterator to be used so reset previous hit count
         query = String.format("SELECT * FROM %s.%s WHERE token(pk) > 0", KEYSPACE, SAI_TABLE);
-        runAndValidate("Submitting range requests on 2 ranges with a concurrency of 2.*", query);
+        runAndValidate(1, 1, query);
 
-        // Token-restricted range query with SAI so should bypass initial concurrency estimation
+        // token-restricted query and index
+        // we expect StorageProxy#RangeCommandIterator to be used so reset previous hit count
         query = String.format("SELECT * FROM %s.%s WHERE token(pk) > 0 AND gdp > ?", KEYSPACE, SAI_TABLE);
-        runAndValidate("Submitting range requests on 2 ranges with a concurrency of 2", query, 3_000_000_000L);
+        runAndValidate(1, 1, query, 3_000_000_000L);
     }
 
-    /**
-     * Run the given query and check that the given trace message exists in the trace entries.
+    /*
+        Run the given query, check the hit count, check the max round trips.
      */
-    private void runAndValidate(String trace, String query, Object... bondValues)
+    private void runAndValidate(int expectedCount, int expectedMax, String query, Object... bondValues)
     {
-        UUID sessionId = TimeUUID.Generator.nextTimeAsUUID();
-
-        cluster.coordinator(1).executeWithTracingWithResult(sessionId, query, ConsistencyLevel.ALL, bondValues);
+        cluster.get(1).runOnInstance(OverridableClientRequestsMetricsProvider::reset);
+        cluster.coordinator(1).execute(query, ConsistencyLevel.ALL, bondValues);
+        assertEquals(expectedCount, getRangeReadCount());
+        assertEquals(expectedMax, getMaxRoundTrips());
+    }
 
-        await().atMost(5, TimeUnit.SECONDS).until(() -> {
-            List<TracingUtil.TraceEntry> traceEntries = TracingUtil.getTrace(cluster, sessionId, ConsistencyLevel.ONE);
-            return traceEntries.stream().anyMatch(entry -> entry.activity.matches(trace));
-        });
+    private int getRangeReadCount()
+    {
+        return cluster.get(1).callOnInstance(() -> Math.toIntExact(ClientRequestsMetricsProvider.instance.metrics(KEYSPACE).rangeMetrics.roundTrips.getCount()));
     }
 
-    private static TokenSupplier generateTokenSupplier()
+    private int getMaxRoundTrips()
     {
-        List<List<String>> tokens = Arrays.asList(Collections.singletonList("-9223372036854775808"),
-                                                  Collections.singletonList("-3074457345618258602"),
-                                                  Collections.singletonList("3074457345618258603"));
-        return nodeIdx -> tokens.get(nodeIdx - 1);
+        return cluster.get(1).callOnInstance(() -> Math.toIntExact(ClientRequestsMetricsProvider.instance.metrics(KEYSPACE).rangeMetrics.roundTrips.getSnapshot().getMax()));
     }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/GenericOrderByTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/GenericOrderByTest.java
new file mode 100644
index 000000000000..f6f2150a44f8
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/GenericOrderByTest.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICoordinator;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+/**
+ * Test for generic ORDER BY queries with SAI.
+ */
+public class GenericOrderByTest extends TestBaseImpl
+{
+    private static final int NUM_REPLICAS = 3;
+    private static final int RF = 2;
+
+    @Test
+    public void testOrderBy() throws Throwable
+    {
+        try (Cluster cluster = init(Cluster.build(NUM_REPLICAS)
+                                           .withConfig(config -> config.with(GOSSIP).with(NETWORK))
+                                           .start(), RF))
+        {
+            cluster.schemaChange(withKeyspace("CREATE TABLE %s.t(k int, c int, v int, PRIMARY KEY(k, c))"));
+            cluster.schemaChange(withKeyspace("CREATE CUSTOM INDEX ON %s.t(v) USING 'StorageAttachedIndex'"));
+            SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+            ICoordinator coordinator = cluster.coordinator(1);
+
+            String insertQuery = withKeyspace("INSERT INTO %s.t(k, c, v) VALUES (?, ?, ?)");
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 1, 1, 1);
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 1, 2, 8);
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 1, 3, 3);
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 2, 1, 6);
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 2, 2, 5);
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 2, 3, 4);
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 3, 1, 7);
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 3, 2, 2);
+            coordinator.execute(insertQuery, ConsistencyLevel.ALL, 3, 3, 9);
+
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t ORDER BY v ASC",
+                                row(1, 1, 1),
+                                row(3, 2, 2),
+                                row(1, 3, 3),
+                                row(2, 3, 4),
+                                row(2, 2, 5),
+                                row(2, 1, 6),
+                                row(3, 1, 7),
+                                row(1, 2, 8),
+                                row(3, 3, 9));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t ORDER BY v DESC",
+                                row(3, 3, 9),
+                                row(1, 2, 8),
+                                row(3, 1, 7),
+                                row(2, 1, 6),
+                                row(2, 2, 5),
+                                row(2, 3, 4),
+                                row(1, 3, 3),
+                                row(3, 2, 2),
+                                row(1, 1, 1));
+
+            // with partition key restriction
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE k=1 ORDER BY v ASC",
+                                row(1, 1, 1),
+                                row(1, 3, 3),
+                                row(1, 2, 8));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE k=1 ORDER BY v DESC",
+                                row(1, 2, 8),
+                                row(1, 3, 3),
+                                row(1, 1, 1));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE k=2 ORDER BY v ASC",
+                                row(2, 3, 4),
+                                row(2, 2, 5),
+                                row(2, 1, 6));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE k=2 ORDER BY v DESC",
+                                row(2, 1, 6),
+                                row(2, 2, 5),
+                                row(2, 3, 4));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE k=3 ORDER BY v ASC",
+                                row(3, 2, 2),
+                                row(3, 1, 7),
+                                row(3, 3, 9));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE k=3 ORDER BY v DESC",
+                                row(3, 3, 9),
+                                row(3, 1, 7),
+                                row(3, 2, 2));
+
+            // with indexed column filter
+            cluster.schemaChange(withKeyspace("CREATE CUSTOM INDEX ON %s.t(c) USING 'StorageAttachedIndex'"));
+            SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE c=1 ORDER BY v ASC",
+                                row(1, 1, 1),
+                                row(2, 1, 6),
+                                row(3, 1, 7));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE c=1 ORDER BY v DESC",
+                                row(3, 1, 7),
+                                row(2, 1, 6),
+                                row(1, 1, 1));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE c=2 ORDER BY v ASC",
+                                row(3, 2, 2),
+                                row(2, 2, 5),
+                                row(1, 2, 8));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE c=2 ORDER BY v DESC",
+                                row(1, 2, 8),
+                                row(2, 2, 5),
+                                row(3, 2, 2));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE c=3 ORDER BY v ASC",
+                                row(1, 3, 3),
+                                row(2, 3, 4),
+                                row(3, 3, 9));
+            assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE c=3 ORDER BY v DESC",
+                                row(3, 3, 9),
+                                row(2, 3, 4),
+                                row(1, 3, 3));
+        }
+    }
+
+    private void assertRowsWithLimit(Cluster cluster, String query, Object[]... expected)
+    {
+        for (int node = 1; node <= cluster.size(); node++)
+        {
+            assertRowsWithLimit(cluster.coordinator(node), query, expected);
+        }
+    }
+
+    private void assertRowsWithLimit(ICoordinator coordinator, String query, Object[]... expected)
+    {
+        for (int limit = 1; limit <= expected.length; limit++)
+        {
+            String queryWithLimit = withKeyspace(query) + " LIMIT " + limit;
+            Object[][] expectedWithLimit = Arrays.copyOfRange(expected, 0, limit);
+            assertRows(coordinator.execute(queryWithLimit, ConsistencyLevel.ONE), expectedWithLimit);
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/ImportIndexedSSTablesTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/ImportIndexedSSTablesTest.java
index 1b9997aeee6a..c979bd5f9dd8 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/ImportIndexedSSTablesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/ImportIndexedSSTablesTest.java
@@ -36,7 +36,7 @@
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.index.sai.StorageAttachedIndexBuilder;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.store.IndexInput;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java
index 3a9e111badfb..af9a41dff070 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java
@@ -25,6 +25,8 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.IntFunction;
 
 import com.google.common.base.Objects;
 import org.junit.Test;
@@ -36,6 +38,10 @@
 import org.apache.cassandra.index.IndexStatusManager;
 import org.apache.cassandra.index.SecondaryIndexManager;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
@@ -44,6 +50,7 @@
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
 import static org.apache.cassandra.distributed.test.sai.SAIUtil.waitForIndexQueryable;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
 import static org.awaitility.Awaitility.await;
 import static org.junit.Assert.assertEquals;
 
@@ -127,7 +134,7 @@ public void verifyIndexStatusPropagation() throws Exception
             assertIndexingStatus(cluster);
 
             // drop ks2 index2, there should be no ks2 index2 status on all node
-            cluster.schemaChange("DROP INDEX " + ks2 + "." + index2);
+            cluster.schemaChange("DROP INDEX " + ks2 + '.' + index2);
             expectedNodeIndexQueryability.keySet().forEach(k -> {
                 if (k.keyspace.equals(ks2) && k.index.equals(index2))
                     expectedNodeIndexQueryability.put(k, Index.Status.UNKNOWN);
@@ -135,7 +142,7 @@ public void verifyIndexStatusPropagation() throws Exception
             assertIndexingStatus(cluster);
 
             // drop ks3 cf1, there should be no ks3 index1/index2 status
-            cluster.schemaChange("DROP TABLE " + ks3 + "." + cf1);
+            cluster.schemaChange("DROP TABLE " + ks3 + '.' + cf1);
             expectedNodeIndexQueryability.keySet().forEach(k -> {
                 if (k.keyspace.equals(ks3))
                     expectedNodeIndexQueryability.put(k, Index.Status.UNKNOWN);
@@ -144,6 +151,198 @@ public void verifyIndexStatusPropagation() throws Exception
         }
     }
 
+    @Test
+    public void testNonQueryableNodeN2Rf2() throws Exception
+    {
+        shouldSkipNonQueryableNode(2, Collections.singletonList(1), Arrays.asList(1, 2));
+    }
+
+    @Test
+    public void testSkipNonQueryableNodeN3Rf3() throws Exception
+    {
+        shouldSkipNonQueryableNode(3, Collections.singletonList(1), Arrays.asList(1, 2), Arrays.asList(1, 2, 3));
+    }
+
+    @Test
+    public void testSkipNonQueryableNodeN1Rf1() throws Exception
+    {
+        shouldSkipNonQueryableNode(1, Collections.singletonList(1));
+    }
+
+    @Test
+    public void testIndexExceptionsTwoIndexesOn3NodeCluster() throws Exception
+    {
+        try (Cluster cluster = init(Cluster.build(3)
+                .withConfig(config -> config.with(GOSSIP)
+                        .with(NETWORK))
+                .start()))
+        {
+            String ks2 = "ks2";
+            String cf1 = "cf1";
+            String index1 = "cf1_idx1";
+            String index2 = "cf1_idx2";
+
+            // Create keyspace, table with correct column types
+            cluster.schemaChange(String.format(CREATE_KEYSPACE, ks2, 2));
+            cluster.schemaChange("CREATE TABLE " + ks2 + '.' + cf1 + " (pk int PRIMARY KEY, v1 int, v2 int)");
+            executeOnAllCoordinators(cluster,
+                                     "SELECT pk FROM " + ks2 + '.' + cf1 + " WHERE v1=0 AND v2=0 ALLOW FILTERING",
+                                     ConsistencyLevel.LOCAL_QUORUM,
+                                     0);
+            executeOnAllCoordinators(cluster,
+                                     "SELECT pk FROM " + ks2 + '.' + cf1 + " WHERE v2=0 ALLOW FILTERING",
+                                     ConsistencyLevel.LOCAL_QUORUM,
+                                     0);
+            executeOnAllCoordinators(cluster,
+                                     "SELECT pk FROM " + ks2 + '.' + cf1 + " WHERE v1=0 ALLOW FILTERING",
+                                     ConsistencyLevel.LOCAL_QUORUM,
+                                     0);
+
+            cluster.schemaChange(String.format(CREATE_INDEX, index1, ks2, cf1, "v1"));
+            cluster.schemaChange(String.format(CREATE_INDEX, index2, ks2, cf1, "v2"));
+            cluster.forEach(node -> expectedNodeIndexQueryability.put(NodeIndex.create(ks2, index1, node), Index.Status.BUILD_SUCCEEDED));
+            for (IInvokableInstance node : cluster.get(2, 1, 3))
+                for (IInvokableInstance replica : cluster.get(1, 2, 3))
+                    waitForIndexingStatus(node, ks2, index1, replica, Index.Status.BUILD_SUCCEEDED);
+
+            // Mark only index2 as building on node3, leave index1 in BUILD_SUCCEEDED state
+            markIndexBuilding(cluster.get(3), ks2, cf1, index2);
+            cluster.forEach(node -> expectedNodeIndexQueryability.put(NodeIndex.create(ks2, index2, node), Index.Status.FULL_REBUILD_STARTED));
+            waitForIndexingStatus(cluster.get(2), ks2, index2, cluster.get(3), Index.Status.FULL_REBUILD_STARTED);
+            waitForIndexingStatus(cluster.get(1), ks2, index2, cluster.get(3), Index.Status.FULL_REBUILD_STARTED);
+            waitForIndexingStatus(cluster.get(3), ks2, index2, cluster.get(3), Index.Status.FULL_REBUILD_STARTED);
+
+            assertThatThrownBy(() ->
+                    executeOnAllCoordinators(cluster,
+                            "SELECT pk FROM " + ks2 + '.' + cf1 + " WHERE v1=0 AND v2=0",
+                            ConsistencyLevel.LOCAL_QUORUM,
+                            0))
+                    .hasMessageContaining("Operation failed - received 1 responses and 1 failures: INDEX_BUILD_IN_PROGRESS");
+
+            // Mark only index2 as failing on node2, leave index1 in BUILD_SUCCEEDED state
+            markIndexBuilding(cluster.get(2), ks2, cf1, index2);
+            cluster.forEach(node -> expectedNodeIndexQueryability.put(NodeIndex.create(ks2, index2, node), Index.Status.BUILD_FAILED));
+
+            assertThatThrownBy(() ->
+                    executeOnAllCoordinators(cluster,
+                            "SELECT pk FROM " + ks2 + '.' + cf1 + " WHERE v1=0 AND v2=0",
+                            ConsistencyLevel.LOCAL_QUORUM,
+                            0))
+                    .hasMessageContaining("Operation failed - received 1 responses and 1 failures: INDEX_BUILD_IN_PROGRESS");
+
+            // Mark only index2 as failing on node1, leave index1 in BUILD_SUCCEEDED state
+            markIndexNonQueryable(cluster.get(1), ks2, cf1, index2);
+            cluster.forEach(node -> expectedNodeIndexQueryability.put(NodeIndex.create(ks2, index2, node), Index.Status.BUILD_FAILED));
+            waitForIndexingStatus(cluster.get(2), ks2, index2, cluster.get(1), Index.Status.BUILD_FAILED);
+            waitForIndexingStatus(cluster.get(1), ks2, index2, cluster.get(1), Index.Status.BUILD_FAILED);
+            waitForIndexingStatus(cluster.get(3), ks2, index2, cluster.get(1), Index.Status.BUILD_FAILED);
+
+            assertThatThrownBy(() ->
+                    executeOnAllCoordinators(cluster,
+                            "SELECT pk FROM " + ks2 + '.' + cf1 + " WHERE v1=0 AND v2=0",
+                            ConsistencyLevel.LOCAL_QUORUM,
+                            0))
+                    .hasMessageMatching("^Operation failed - received 0 responses and 2 failures: INDEX_NOT_AVAILABLE from .+, INDEX_BUILD_IN_PROGRESS from .+$");
+        }
+    }
+
+    private void shouldSkipNonQueryableNode(int nodes, List<Integer>... nonQueryableNodesList) throws Exception
+    {
+        try (Cluster cluster = init(Cluster.build(nodes)
+                                           .withConfig(config -> config.with(GOSSIP)
+                                                                       .with(NETWORK))
+                                           .start()))
+        {
+            String table = "non_queryable_node_test_" + System.currentTimeMillis();
+            cluster.schemaChange(String.format(CREATE_TABLE, KEYSPACE, table));
+            cluster.schemaChange(String.format(CREATE_INDEX, "", KEYSPACE, table, "v1"));
+            cluster.schemaChange(String.format(CREATE_INDEX, "", KEYSPACE, table, "v2"));
+            waitForIndexQueryable(cluster, KEYSPACE);
+
+            // create 100 rows in 1 sstable
+            int rows = 100;
+            for (int i = 0; i < rows; i++)
+                cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s(pk, v1, v2) VALUES ('%d', 0, '0');", KEYSPACE, table, i), ConsistencyLevel.QUORUM);
+            cluster.forEach(node -> node.flush(KEYSPACE));
+
+            String numericQuery = String.format("SELECT pk FROM %s.%s WHERE v1=0", KEYSPACE, table);
+            String stringQuery = String.format("SELECT pk FROM %s.%s WHERE v2='0'", KEYSPACE, table);
+            String multiIndexQuery = String.format("SELECT pk FROM %s.%s WHERE v1=0 AND v2='0'", KEYSPACE, table);
+
+            // get index name base on node id to have different non-queryable index on different nodes.
+            Function<Integer, String> nodeIdToColumn = nodeId -> "v" + (nodeId % 2 + 1);
+            IntFunction<String> nodeIdToIndex = nodeId -> IndexMetadata.generateDefaultIndexName(table, ColumnIdentifier.getInterned(nodeIdToColumn.apply(nodeId), false));
+
+            for (List<Integer> nonQueryableNodes : nonQueryableNodesList)
+            {
+                int numericLiveReplicas = (int) (nodes - nonQueryableNodes.stream().map(nodeIdToColumn).filter(c -> c.equals("v1")).count());
+                int stringLiveReplicas = (int) (nodes - nonQueryableNodes.stream().map(nodeIdToColumn).filter(c -> c.equals("v2")).count());
+                int liveReplicas = nodes - nonQueryableNodes.size();
+
+                // mark index non-queryable at once and wait for ack from remote peers
+                for (int local : nonQueryableNodes)
+                    markIndexNonQueryable(cluster.get(local), KEYSPACE, table, nodeIdToIndex.apply(local));
+
+                for (int local : nonQueryableNodes)
+                    for (int remote = 1; remote <= cluster.size(); remote++)
+                        waitForIndexingStatus(cluster.get(remote), KEYSPACE, nodeIdToIndex.apply(local), cluster.get(local), Index.Status.BUILD_FAILED);
+
+                // test different query types
+                executeOnAllCoordinatorsAllConsistencies(cluster, numericQuery, numericLiveReplicas, rows);
+                executeOnAllCoordinatorsAllConsistencies(cluster, stringQuery, stringLiveReplicas, rows);
+                executeOnAllCoordinatorsAllConsistencies(cluster, multiIndexQuery, liveReplicas, rows);
+
+                // rebuild local index at once and wait for remote ack
+                for (int local : nonQueryableNodes)
+                {
+                    String index = nodeIdToIndex.apply(local);
+                    cluster.get(local).runOnInstance(() -> ColumnFamilyStore.rebuildSecondaryIndex(KEYSPACE, table, index));
+                }
+
+                for (int local : nonQueryableNodes)
+                    for (int remote = 1; remote <= cluster.size(); remote++)
+                        waitForIndexingStatus(cluster.get(remote), KEYSPACE, nodeIdToIndex.apply(local), cluster.get(local), Index.Status.BUILD_SUCCEEDED);
+
+                // With cl=all, query should pass
+                executeOnAllCoordinators(cluster, numericQuery, ConsistencyLevel.ALL, rows);
+                executeOnAllCoordinators(cluster, stringQuery, ConsistencyLevel.ALL, rows);
+                executeOnAllCoordinators(cluster, multiIndexQuery, ConsistencyLevel.ALL, rows);
+            }
+        }
+    }
+
+    private void executeOnAllCoordinatorsAllConsistencies(Cluster cluster, String statement, int liveReplicas, int num)
+    {
+        int allReplicas = cluster.size();
+
+        // test different consistency levels
+        executeOnAllCoordinators(cluster, statement, ConsistencyLevel.ONE, liveReplicas >= 1 ? num : -1);
+        if (allReplicas >= 2)
+            executeOnAllCoordinators(cluster, statement, ConsistencyLevel.TWO, liveReplicas >= 2 ? num : -1);
+        executeOnAllCoordinators(cluster, statement, ConsistencyLevel.ALL, liveReplicas >= allReplicas ? num : -1);
+    }
+
+    private void executeOnAllCoordinators(Cluster cluster, String query, ConsistencyLevel level, int expected)
+    {
+        // test different coordinator
+        for (int nodeId = 1; nodeId <= cluster.size(); nodeId++)
+        {
+            if (expected >= 0)
+                assertEquals(expected, cluster.coordinator(nodeId).execute(query, level).length);
+            else
+            {
+                try
+                {
+                    cluster.coordinator(nodeId).execute(query, level);
+                }
+                catch (Throwable e)
+                {
+                    assertEquals("ReadFailureException", e.getClass().getSimpleName());
+                }
+            }
+        }
+    }
+
     private void markIndexNonQueryable(IInvokableInstance node, String keyspace, String table, String indexName)
     {
         expectedNodeIndexQueryability.put(NodeIndex.create(keyspace, indexName, node), Index.Status.BUILD_FAILED);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexConsistencyTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexConsistencyTest.java
new file mode 100644
index 000000000000..101d8f11d660
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexConsistencyTest.java
@@ -0,0 +1,798 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.shared.AssertUtils;
+import org.apache.cassandra.distributed.shared.Byteman;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.utils.Shared;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.shared.AssertUtils.fail;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests scenarios where two replicas have different versions of the same rows.
+ *
+ * If the coordinator detects rows that are present in only some of the replica responses, it should ask for them by
+ * primary key to those replicas where they were omitted to check if they have a more recent version that wasn't sent
+ * because of not satisfying the row filter.
+ *
+ * See CASSANDRA-8272, CASSANDRA-8273.
+ */
+public class IndexConsistencyTest extends TestBaseImpl
+{
+    private static final int NUM_REPLICAS = 2;
+
+    private static final String INJECTION_SCRIPT = "RULE fail indexer\n" +
+                                                   "CLASS org.apache.cassandra.index.sai.StorageAttachedIndexGroup\n" +
+                                                   "METHOD indexerFor\n" +
+                                                   "AT ENTRY\n" +
+                                                   "IF org.apache.cassandra.distributed.test.sai.IndexConsistencyTest$FailureEnabled.isEnabled(%d)\n" +
+                                                   "DO\n" +
+                                                   "   throw new java.lang.RuntimeException(\"Injected index failure\")\n" +
+                                                   "ENDRULE\n" +
+                                                   "RULE count indexer\n" +
+                                                   "CLASS org.apache.cassandra.index.sai.StorageAttachedIndexGroup\n" +
+                                                   "METHOD indexerFor\n" +
+                                                   "AT ENTRY\n" +
+                                                   "IF TRUE\n" +
+                                                   "DO\n" +
+                                                   "   org.apache.cassandra.distributed.test.sai.IndexConsistencyTest$Counter.increment(%d)\n" +
+                                                   "ENDRULE\n";
+
+
+
+
+    private static AtomicInteger seq = new AtomicInteger();
+    private static String table;
+
+    private static Cluster cluster;
+
+    @BeforeClass
+    public static void setupCluster() throws Exception
+    {
+        cluster = Cluster.build(NUM_REPLICAS)
+                         .withConfig(config -> config.with(NETWORK, GOSSIP)
+                                                     .set("hinted_handoff_enabled", false))
+                         .withInstanceInitializer((cl, nodeNumber) -> {
+                             Byteman.createFromText(String.format(INJECTION_SCRIPT, nodeNumber, nodeNumber)).install(cl);
+                         })
+                         .start();
+
+        cluster.setUncaughtExceptionsFilter((instance, ex) -> true);
+        
+        cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
+                                          "{'class': 'SimpleStrategy', 'replication_factor': " + NUM_REPLICAS + "};"));
+    }
+
+    @AfterClass
+    public static void closeCluster()
+    {
+        if (cluster != null)
+            cluster.close();
+    }
+
+    @Before
+    public void before()
+    {
+        table = "t_" + seq.getAndIncrement();
+    }
+
+    @After
+    public void after()
+    {
+        cluster.schemaChange(formatQuery("DROP TABLE IF EXISTS %s"));
+        FailureEnabled.clear();
+        Counter.clear();
+    }
+
+    @Test
+    public void testUpdateOnSkinnyTable() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v text)"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k, v) VALUES (0, 'old')");
+
+        executeIsolated(1, "UPDATE %s SET v = 'new' WHERE k = 0");
+
+        assertEmpty("SELECT * FROM %s WHERE v = 'old'");
+        assertRows("SELECT * FROM %s WHERE v = 'new'", row(0, "new"));
+    }
+
+    @Test
+    public void testUpdateOnWideTable() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v text, s int STATIC, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k, s) VALUES (0, 9)",
+                "INSERT INTO %s(k, c, v) VALUES (0, -1, 'old')",
+                "INSERT INTO %s(k, c, v) VALUES (0, 0, 'old')",
+                "INSERT INTO %s(k, c, v) VALUES (0, 1, 'old')");
+
+        executeIsolated(1, "UPDATE %s SET v = 'new' WHERE k = 0 AND c = 0");
+
+        assertRows("SELECT * FROM %s WHERE v = 'old'", row(0, -1, 9, "old"), row(0, 1, 9, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'new'", row(0, 0, 9, "new"));
+    }
+
+    @Test
+    public void testUpdateOnWideTableCaseInsensitive() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k1 int, k2 int, v text, v1 text, primary key(k1, k2)) with read_repair='NONE'"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v", false, false)));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k1, k2, v, v1) VALUES (0, 0, 'Old', '0')");
+
+        executeIsolated(1, "UPDATE %s SET v = 'New' WHERE k1=0 and k2=0");
+
+        assertEmpty("SELECT * FROM %s WHERE v : 'Old'");
+        assertEmpty("SELECT * FROM %s WHERE v : 'old'");
+
+        assertRows("SELECT * FROM %s WHERE v : 'NEW'", row(0, 0, "New", "0"));
+        assertRows("SELECT * FROM %s WHERE v : 'NEW' and v1 ='0' ALLOW FILTERING", row(0, 0, "New", "0"));
+        assertEmpty("SELECT * FROM %s WHERE v : 'NEW' and v1 ='1' ALLOW FILTERING");
+    }
+
+    @Test
+    public void testUpdateOnWideTableNormalized() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k1 int, k2 int, v text, v1 text, primary key(k1, k2)) with read_repair='NONE'"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v", true, true)));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k1, k2, v, v1) VALUES (0, 0, '\u00E1bc', '0')"); //
+
+        executeIsolated(1, "UPDATE %s SET v = '\u0061\u0301bc' WHERE k1=0 and k2=0");
+
+        assertEmpty("SELECT * FROM %s WHERE v : '\u0061\u0301bc' and v1 ='1' ALLOW FILTERING");
+        assertRows("SELECT * FROM %s WHERE v : '\u0061\u0301bc' and v1 ='0' ALLOW FILTERING",
+                   row(0, 0, "\u0061\u0301bc", "0"));
+        assertRows("SELECT * FROM %s WHERE v : '\u00E1bc'", row(0, 0, "\u0061\u0301bc", "0"));
+        assertRows("SELECT * FROM %s WHERE v : '\u0061\u0301bc'", row(0, 0, "\u0061\u0301bc", "0"));
+    }
+
+    @Test
+    public void testUpdateOnWideTableCaseInsensitiveNormalized() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k1 int, k2 int, v text, v1 text, primary key(k1, k2)) with read_repair='NONE'"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v", false, true)));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k1, k2, v, v1) VALUES (0, 0, '\u00E1Bc', '0')");
+
+        executeIsolated(1, "UPDATE %s SET v = '\u0061\u0301bC' WHERE k1=0 and k2=0");
+
+        assertEmpty("SELECT * FROM %s WHERE v : '\u0061\u0301bc' and v1 ='1' ALLOW FILTERING");
+        assertRows("SELECT * FROM %s WHERE v : '\u0061\u0301bc' and v1 ='0' ALLOW FILTERING",
+                   row(0, 0, "\u0061\u0301bC", "0"));
+        assertRows("SELECT * FROM %s WHERE v : '\u00E1Bc'", row(0, 0, "\u0061\u0301bC", "0"));
+        assertRows("SELECT * FROM %s WHERE v : '\u0061\u0301Bc'", row(0, 0, "\u0061\u0301bC", "0"));
+    }
+
+    @Test
+    public void testUpdateOnStaticColumnWithEmptyPartition() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v int, s text STATIC, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("s")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k, s) VALUES (0, 'old')",
+                "INSERT INTO %s(k, s) VALUES (1, 'old')");
+
+        executeIsolated(1, "UPDATE %s SET s = 'new' WHERE k = 0");
+
+        assertRows("SELECT * FROM %s WHERE s = 'old'", row(1, null, "old", null));
+        assertRows("SELECT * FROM %s WHERE s = 'new'", row(0, null, "new", null));
+    }
+
+    @Test
+    public void testUpdateOnStaticColumnWithNotEmptyPartition() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v int, s text STATIC, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("s")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k, s) VALUES (0, 'old')",
+                "INSERT INTO %s(k, s) VALUES (1, 'old')",
+                "INSERT INTO %s(k, c, v) VALUES (0, 10, 100)",
+                "INSERT INTO %s(k, c, v) VALUES (0, 20, 200)",
+                "INSERT INTO %s(k, c, v) VALUES (1, 30, 300)",
+                "INSERT INTO %s(k, c, v) VALUES (1, 40, 400)");
+
+        executeIsolated(1, "UPDATE %s SET s = 'new' WHERE k = 0");
+
+        assertRows("SELECT * FROM %s WHERE s = 'old'", row(1, 30, "old", 300), row(1, 40, "old", 400));
+        assertRows("SELECT * FROM %s WHERE s = 'new'", row(0, 10, "new", 100), row(0, 20, "new", 200));
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitOnPartitionKeyColumnWithEmptyPartitions() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k1 int, k2 int, c int, s int STATIC, PRIMARY KEY((k1, k2), c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("k1")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k1, k2, s) VALUES (0, 1, 10)",
+                "INSERT INTO %s (k1, k2, s) VALUES (0, 2, 20)");
+
+        executeIsolated(1, "DELETE FROM %s WHERE k1 = 0 AND k2 = 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k1 = 0 AND k2 = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE k1 = 0 LIMIT 1");
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitOnPartitionKeyColumnWithNotEmptyPartitions() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k1 int, k2 int, c int, s int STATIC, PRIMARY KEY((k1, k2), c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("k1")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k1, k2, c, s) VALUES (0, 1, 10, 100)",
+                "INSERT INTO %s (k1, k2, c, s) VALUES (0, 2, 20, 200)");
+
+        executeIsolated(1, "DELETE FROM %s WHERE k1 = 0 AND k2 = 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k1 = 0 AND k2 = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE k1 = 0 LIMIT 1");
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitOnClusteringKeyColumn() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("c")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c) VALUES (1, 0)",
+                "INSERT INTO %s (k, c) VALUES (2, 0)");
+
+        executeIsolated(1, "DELETE FROM %s WHERE k = 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE c = 0 LIMIT 1");
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitOnStaticColumnWithEmptyPartitions() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, s int STATIC, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("s")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, s) VALUES (1, 0)",
+                "INSERT INTO %s (k, s) VALUES (2, 0)");
+
+        executeIsolated(1, "DELETE FROM %s WHERE k = 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE s = 0 LIMIT 1");
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitOnStaticColumnWithEmptyPartitionsAndRowsAfter() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, s int STATIC, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("s")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, s) VALUES (1, 0)",
+                "INSERT INTO %s (k, s) VALUES (2, 0)",
+                "INSERT INTO %s (k, s) VALUES (3, 0)",
+                "INSERT INTO %s (k, c) VALUES (3, 1)",
+                "INSERT INTO %s (k, c) VALUES (3, 2)");
+
+        executeIsolated(1, "DELETE FROM %s WHERE k = 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 2");
+
+        assertRows("SELECT * FROM %s WHERE s = 0 LIMIT 1", row(3, 1, 0));
+        assertRows("SELECT * FROM %s WHERE s = 0 LIMIT 10", row(3, 1, 0), row(3, 2, 0));
+        assertRows("SELECT * FROM %s WHERE s = 0", row(3, 1, 0), row(3, 2, 0));
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitOnStaticColumnWithNotEmptyPartitions() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, s int STATIC, v int, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("s")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c, v, s) VALUES (1, 10, 100, 0)",
+                "INSERT INTO %s (k, c, v, s) VALUES (2, 20, 200, 0)");
+
+        executeIsolated(1, "DELETE FROM %s WHERE k = 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE s = 0 LIMIT 1");
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitOnStaticColumnWithNotEmptyPartitionsAndRowsAfter() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, s int STATIC, v int, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("s")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c, v, s) VALUES (1, 10, 100, 0)",
+                "INSERT INTO %s (k, c, v, s) VALUES (2, 20, 200, 0)",
+                "INSERT INTO %s (k, s) VALUES (3, 0)",
+                "INSERT INTO %s (k, c) VALUES (3, 1)",
+                "INSERT INTO %s (k, c) VALUES (3, 2)");
+
+        executeIsolated(1, "DELETE FROM %s WHERE k = 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 2");
+
+        assertRows("SELECT * FROM %s WHERE s = 0 LIMIT 1", row(3, 1, 0, null));
+        assertRows("SELECT * FROM %s WHERE s = 0 LIMIT 10", row(3, 1, 0, null), row(3, 2, 0, null));
+        assertRows("SELECT * FROM %s WHERE s = 0", row(3, 1, 0, null), row(3, 2, 0, null));
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitOnRegularColumn() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c)) WITH speculative_retry = 'NONE'"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 0)",
+                "INSERT INTO %s (k, c, v) VALUES (0, 2, 0)");
+
+        executeIsolated(1, "DELETE FROM %s WHERE k = 0 AND c = 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 0 AND c = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE v = 0 LIMIT 1");
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitAndRowsAfter() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 0)",
+                "INSERT INTO %s (k, c, v) VALUES (0, 2, 0)",
+                "INSERT INTO %s (k, c, v) VALUES (0, 3, 0)");
+
+        executeIsolated(1,
+                        "DELETE FROM %s WHERE k = 0 AND c = 1",
+                        "INSERT INTO %s (k, c, v) VALUES (0, 4, 0)");
+        executeIsolated(2,
+                        "INSERT INTO %s (k, c, v) VALUES (0, 5, 0)",
+                        "DELETE FROM %s WHERE k = 0 AND c = 2");
+
+        assertRows("SELECT * FROM %s WHERE v = 0 LIMIT 1", row(0, 3, 0));
+        assertRows("SELECT * FROM %s WHERE v = 0 LIMIT 2", row(0, 3, 0), row(0, 4, 0));
+        assertRows("SELECT * FROM %s WHERE v = 0 LIMIT 3", row(0, 3, 0), row(0, 4, 0), row(0, 5, 0));
+        assertRows("SELECT * FROM %s WHERE v = 0 LIMIT 4", row(0, 3, 0), row(0, 4, 0), row(0, 5, 0));
+    }
+
+    @Test
+    public void testComplementaryDeletionWithLimitAndRowsBetween() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 0)",
+                "INSERT INTO %s (k, c, v) VALUES (0, 4, 0)");
+
+        executeIsolated(1,
+                        "DELETE FROM %s WHERE k = 0 AND c = 1");
+        executeIsolated(2,
+                        "INSERT INTO %s (k, c, v) VALUES (0, 2, 0)",
+                        "INSERT INTO %s (k, c, v) VALUES (0, 3, 0)",
+                        "DELETE FROM %s WHERE k = 0 AND c = 4");
+
+        assertRows("SELECT * FROM %s WHERE v = 0 LIMIT 1", row(0, 2, 0));
+        assertRows("SELECT * FROM %s WHERE v = 0 LIMIT 2", row(0, 2, 0), row(0, 3, 0));
+        assertRows("SELECT * FROM %s WHERE v = 0 LIMIT 3", row(0, 2, 0), row(0, 3, 0));
+    }
+
+    @Test
+    public void testComplementaryUpdateWithLimitOnStaticColumnWithEmptyPartitions() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, s text STATIC, v int, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("s")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, s) VALUES (1, 'old')",
+                "INSERT INTO %s (k, s) VALUES (2, 'old')");
+
+        executeIsolated(1, "UPDATE %s SET s = 'new' WHERE k = 1");
+        executeIsolated(2, "UPDATE %s SET s = 'new' WHERE k = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE s = 'old' LIMIT 1");
+        assertRows("SELECT k, s FROM %s WHERE s = 'new' LIMIT 1", row(1, "new"));
+        assertRows("SELECT k, s FROM %s WHERE s = 'new'", row(1, "new"), row(2, "new"));
+    }
+
+    @Test
+    public void testComplementaryUpdateWithLimitOnStaticColumnWithNotEmptyPartitions() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, s text STATIC, v int, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("s")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c, v, s) VALUES (1, 10, 100, 'old')",
+                "INSERT INTO %s (k, c, v, s) VALUES (2, 20, 200, 'old')");
+
+        executeIsolated(1, "UPDATE %s SET s = 'new' WHERE k = 1");
+        executeIsolated(2, "UPDATE %s SET s = 'new' WHERE k = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE s = 'old' LIMIT 1");
+        assertRows("SELECT k, c, v, s FROM %s WHERE s = 'new' LIMIT 1", row(1, 10, 100, "new"));
+        assertRows("SELECT k, c, v, s FROM %s WHERE s = 'new'",
+                   row(1, 10, 100, "new"), row(2, 20, 200, "new"));
+    }
+
+    @Test
+    public void testComplementaryUpdateWithLimitOnRegularColumn() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 'old')",
+                "INSERT INTO %s (k, c, v) VALUES (0, 2, 'old')");
+
+        executeIsolated(1, "UPDATE %s SET v = 'new' WHERE k = 0 AND c = 1");
+        executeIsolated(2, "UPDATE %s SET v = 'new' WHERE k = 0 AND c = 2");
+
+        assertEmpty("SELECT * FROM %s WHERE v = 'old' LIMIT 1");
+        assertRows("SELECT * FROM %s WHERE v = 'new' LIMIT 1", row(0, 1, "new"));
+        assertRows("SELECT * FROM %s WHERE v = 'new'", row(0, 1, "new"), row(0, 2, "new"));
+    }
+
+    @Test
+    public void testComplementaryUpdateWithLimitAndRowsBetween() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 'old')",
+                "INSERT INTO %s (k, c, v) VALUES (0, 4, 'old')");
+
+        executeIsolated(1,
+                        "UPDATE %s SET v = 'new' WHERE k = 0 AND c = 1");
+        executeIsolated(2,
+                        "INSERT INTO %s (k, c, v) VALUES (0, 2, 'old')",
+                        "INSERT INTO %s (k, c, v) VALUES (0, 3, 'old')",
+                        "UPDATE %s SET v = 'new' WHERE k = 0 AND c = 4");
+
+        assertRows("SELECT * FROM %s WHERE v = 'old' LIMIT 1", row(0, 2, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'old' LIMIT 2", row(0, 2, "old"), row(0, 3, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'old' LIMIT 3", row(0, 2, "old"), row(0, 3, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'new' LIMIT 1", row(0, 1, "new"));
+        assertRows("SELECT * FROM %s WHERE v = 'new' ", row(0, 1, "new"), row(0, 4, "new"));
+    }
+
+    @Test
+    public void testPartitionDeletionOnSkinnyTable() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v text)"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        executeIsolated(1, "INSERT INTO %s (k, v) VALUES (0, 'old') USING TIMESTAMP 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 0");
+
+        assertEmpty("SELECT * FROM %s WHERE v = 'old' LIMIT 1");
+        assertEmpty("SELECT * FROM %s WHERE v = 'old'");
+    }
+
+    @Test
+    public void testPartitionDeletionOnWideTable() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        executeIsolated(1, "INSERT INTO %s (k, c, v) VALUES (0, 1, 'old') USING TIMESTAMP 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 0");
+
+        assertEmpty("SELECT * FROM %s WHERE v = 'old' LIMIT 1");
+        assertEmpty("SELECT * FROM %s WHERE v = 'old'");
+    }
+
+    @Test
+    public void testRowDeletionOnWideTable() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        executeIsolated(1, "INSERT INTO %s (k, c, v) VALUES (0, 1, 'old') USING TIMESTAMP 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 0 AND c = 1");
+
+        assertEmpty("SELECT * FROM %s WHERE v = 'old' LIMIT 1");
+        assertEmpty("SELECT * FROM %s WHERE v = 'old'");
+    }
+
+    @Test
+    public void testRangeDeletionOnWideTable() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        executeIsolated(1,
+                        "INSERT INTO %s (k, c, v) VALUES (0, 1, 'old') USING TIMESTAMP 1",
+                        "INSERT INTO %s (k, c, v) VALUES (0, 2, 'old') USING TIMESTAMP 1",
+                        "INSERT INTO %s (k, c, v) VALUES (0, 3, 'old') USING TIMESTAMP 1",
+                        "INSERT INTO %s (k, c, v) VALUES (0, 4, 'old') USING TIMESTAMP 1");
+        executeIsolated(2, "DELETE FROM %s WHERE k = 0 AND c > 1 AND c < 4");
+
+        assertRows("SELECT * FROM %s WHERE v = 'old' LIMIT 1", row(0, 1, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'old'", row(0, 1, "old"), row(0, 4, "old"));
+    }
+
+    @Test
+    public void testMismatchingInsertionsOnSkinnyTable() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v text)"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        executeIsolated(1, "INSERT INTO %s (k, v) VALUES (0, 'old') USING TIMESTAMP 1");
+        executeIsolated(2, "INSERT INTO %s (k, v) VALUES (0, 'new') USING TIMESTAMP 2");
+
+        assertEmpty("SELECT * FROM %s WHERE v = 'old' LIMIT 1");
+        assertEmpty("SELECT * FROM %s WHERE v = 'old'");
+        assertRows("SELECT * FROM %s WHERE v = 'new' ", row(0, "new"));
+    }
+
+    @Test
+    public void testMismatchingInsertionsOnWideTable() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY(k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        executeIsolated(1, "INSERT INTO %s (k, c, v) VALUES (0, 1, 'old') USING TIMESTAMP 1");
+        executeIsolated(2, "INSERT INTO %s (k, c, v) VALUES (0, 1, 'new') USING TIMESTAMP 2");
+
+        assertEmpty("SELECT * FROM %s WHERE v = 'old' LIMIT 1");
+        assertEmpty("SELECT * FROM %s WHERE v = 'old'");
+        assertRows("SELECT * FROM %s WHERE v = 'new' ", row(0, 1, "new"));
+    }
+
+    @Test
+    public void testConsistentSkinnyTable()
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v text)"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k, v) VALUES (1, 'old')", // updated to 'new'
+                "INSERT INTO %s(k, v) VALUES (2, 'old')",
+                "INSERT INTO %s(k, v) VALUES (3, 'old')", // updated to 'new'
+                "INSERT INTO %s(k, v) VALUES (4, 'old')",
+                "INSERT INTO %s(k, v) VALUES (5, 'old')", // deleted partition
+                "UPDATE %s SET v = 'new' WHERE k = 1",
+                "UPDATE %s SET v = 'new' WHERE k = 3",
+                "DELETE FROM %s WHERE k = 5");
+
+        assertRows("SELECT * FROM %s WHERE v = 'old' LIMIT 1", row(2, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'new' LIMIT 1", row(1, "new"));
+        assertRows("SELECT * FROM %s WHERE v = 'old'", row(2, "old"), row(4, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'new'", row(1, "new"), row(3, "new"));
+    }
+
+    @Test
+    public void testConsistentWideTable()
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY (k, c))"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k, c, v) VALUES (0, 1, 'old')", // updated to 'new'
+                "INSERT INTO %s(k, c, v) VALUES (0, 2, 'old')",
+                "INSERT INTO %s(k, c, v) VALUES (0, 3, 'old')", // updated to 'new'
+                "INSERT INTO %s(k, c, v) VALUES (0, 4, 'old')",
+                "INSERT INTO %s(k, c, v) VALUES (0, 5, 'old')", // deleted row
+                "INSERT INTO %s(k, c, v) VALUES (1, 1, 'old')", // deleted partition
+                "INSERT INTO %s(k, c, v) VALUES (1, 2, 'old')", // deleted partition
+                "UPDATE %s SET v = 'new' WHERE k = 0 AND c = 1",
+                "UPDATE %s SET v = 'new' WHERE k = 0 AND c = 3",
+                "DELETE FROM %s WHERE k = 0 AND c = 5",
+                "DELETE FROM %s WHERE k = 1");
+
+        assertRows("SELECT * FROM %s WHERE v = 'old' LIMIT 1", row(0, 2, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'new' LIMIT 1", row(0, 1, "new"));
+        assertRows("SELECT * FROM %s WHERE v = 'old'", row(0, 2, "old"), row(0, 4, "old"));
+        assertRows("SELECT * FROM %s WHERE v = 'new'", row(0, 1, "new"), row(0, 3, "new"));
+    }
+
+    @Test
+    public void testCount() throws Exception
+    {
+        cluster.schemaChange(formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v text)"));
+        cluster.schemaChange(formatQuery(createIndexQuery("v")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        execute("INSERT INTO %s(k, v) VALUES (1, 'old')",
+                "INSERT INTO %s(k, v) VALUES (2, 'old')",
+                "INSERT INTO %s(k, v) VALUES (3, 'old')",
+                "INSERT INTO %s(k, v) VALUES (4, 'old')",
+                "INSERT INTO %s(k, v) VALUES (5, 'old')");
+
+        executeIsolated(1,
+                        "UPDATE %s SET v = 'new' WHERE k = 2",
+                        "UPDATE %s SET v = 'new' WHERE k = 4");
+
+        assertRows("SELECT COUNT(*) FROM %s WHERE v = 'old' LIMIT 1", row(3L));
+        assertRows("SELECT COUNT(*) FROM %s WHERE v = 'old'", row(3L));
+        assertRows("SELECT COUNT(*) FROM %s WHERE v = 'new'", row(2L));
+    }
+
+    /**
+     * Executes the specified CQL query with CL=ALL, so all replicas get it.
+     *
+     * @param query the CQL queries to be executed in all replicas
+     *
+     * @return the query result
+     */
+    private static Object[][] execute(String query)
+    {
+        return cluster.coordinator(1).execute(formatQuery(query), ConsistencyLevel.ALL);
+    }
+
+    /**
+     * Executes the specified CQL queries with CL=ALL, so all replicas get them.
+     *
+     * @param queries the CQL queries to be executed in all replicas
+     */
+    private static void execute(String... queries)
+    {
+        for (String query : queries)
+        {
+            execute(query);
+        }
+    }
+
+    /**
+     * Executes the specified CQL queries only in the specified replica, with CL=ONE and the other replicas temporally
+     * rejecting mutations.
+     *
+     * @param targetNode the index of the replica that is going to receive the queries in isolation
+     * @param queries the CQL queries to be executed in a single replica
+     */
+    private static void executeIsolated(int targetNode, String... queries) throws Exception
+    {
+        try
+        {
+            // enable mutation failure and reset its verification counter in all the replicas of the target node
+            for (int node = 1; node <= NUM_REPLICAS; node++)
+            {
+                if (node != targetNode)
+                {
+                    FailureEnabled.enable(node);
+                    Counter.reset(node);
+                }
+            }
+
+            // execute queries in the target node with CL=ONE
+            for (String query : queries)
+            {
+                cluster.coordinator(targetNode).execute(formatQuery(query), ConsistencyLevel.ONE);
+            }
+
+            // verify that no mutation has been run in all the replicas of the target node
+            for (int node = 1; node <= NUM_REPLICAS; node++)
+            {
+                if (node != targetNode)
+                {
+                    assertEquals(0, Counter.get(node));
+                }
+            }
+        }
+        finally
+        {
+            // disable mutation failure in all the replicas of the target node
+            for (int node = 1; node <= NUM_REPLICAS; node++)
+            {
+                if (node != targetNode)
+                {
+                    FailureEnabled.disable(node);
+                }
+            }
+        }
+    }
+
+    private static void assertEmpty(String query)
+    {
+        Object[][] result = execute(query);
+        if (result != null && result.length > 0)
+            fail(String.format("Expected empty result but got %d rows", result.length));
+    }
+
+    private static void assertRows(String query, Object[]... expected)
+    {
+        AssertUtils.assertRows(execute(query), expected);
+    }
+
+    private static String formatQuery(String query)
+    {
+        return String.format(query, KEYSPACE + "." + table);
+    }
+
+    private static String createIndexQuery(String column, boolean caseSensitive, boolean normalize)
+    {
+        String options = String.format("WITH OPTIONS = { 'case_sensitive' : %s, 'normalize' : %s };", caseSensitive, normalize);
+        return String.format("CREATE CUSTOM INDEX ON %%s(%s) USING '%s' %s", column, StorageAttachedIndex.class.getName(), options);
+    }
+
+    private static String createIndexQuery(String column)
+    {
+        return String.format("CREATE CUSTOM INDEX ON %%s(%s) USING '%s'", column, StorageAttachedIndex.class.getName());
+    }
+
+    @Shared
+    private static final class FailureEnabled
+    {
+        private static volatile Map<Integer, Boolean> enabled = new HashMap<>();
+
+        public static boolean isEnabled(int node)
+        {
+            return enabled.containsKey(node) && enabled.get(node);
+        }
+
+        public static void enable(int node)
+        {
+            enabled.put(node, true);
+        }
+
+        public static void disable(int node)
+        {
+            enabled.put(node, false);
+        }
+
+        public static void clear()
+        {
+            enabled.clear();
+        }
+    }
+
+    @Shared
+    private static final class Counter
+    {
+        private static volatile ConcurrentMap<Integer, Integer> counters = new ConcurrentHashMap<>();
+
+        public static void increment(int node)
+        {
+            counters.put(node, counters.getOrDefault(node, 0) + 1);
+        }
+
+        public static int get(int node)
+        {
+            return counters.getOrDefault(node, 0);
+        }
+
+        public static void reset(int node)
+        {
+            counters.put(node, 0);
+        }
+
+        public static void clear()
+        {
+            counters.clear();
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java
index cb9795b3072c..9a9c659a0b24 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java
@@ -25,15 +25,15 @@
 import net.bytebuddy.ByteBuddy;
 import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
 import net.bytebuddy.implementation.MethodDelegation;
+
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.shared.ClusterUtils;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.store.IndexInput;
 
@@ -153,13 +153,14 @@ static void installValidateChecksumError(ClassLoader loader)
         }
 
         @SuppressWarnings("unused")
-        public static SegmentMetadata flush(IndexDescriptor indexDescriptor) throws IOException
+        public static SegmentMetadata flush() throws IOException
         {
             throw new IOException(TEST_ERROR_MESSAGE);
         }
 
+        // Object is added here for CC because simply importing another Version confuses AbstractCluster
         @SuppressWarnings("unused")
-        public static void validateChecksum(IndexInput input) throws IOException
+        public static void validateChecksum(IndexInput input, Object version) throws IOException
         {
             throw new CorruptIndexException(TEST_ERROR_MESSAGE, "Test resource");
         }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingTest.java
deleted file mode 100644
index c3f1f4207302..000000000000
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.distributed.test.sai;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.distributed.Cluster;
-import org.apache.cassandra.distributed.api.Feature;
-import org.apache.cassandra.distributed.api.IInvokableInstance;
-import org.apache.cassandra.distributed.api.Row;
-import org.apache.cassandra.distributed.api.SimpleQueryResult;
-import org.apache.cassandra.distributed.test.TestBaseImpl;
-import org.apache.cassandra.distributed.util.QueryResultUtil;
-import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
-import org.assertj.core.api.Assertions;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-@RunWith(Parameterized.class)
-public class IndexStreamingTest extends TestBaseImpl
-{
-    // streaming sends events every 65k, so need to make sure that the files are larger than this to hit
-    // all cases of the vtable - hence we add a big enough blob column
-    private static final ByteBuffer BLOB = ByteBuffer.wrap(new byte[1 << 16]);
-
-    static
-    {
-        DatabaseDescriptor.clientInitialization();
-    }
-
-    private static int sstableStreamingComponentsCount()
-    {
-        return (int) DatabaseDescriptor.getSelectedSSTableFormat()
-                                       .allComponents()
-                                       .stream()
-                                       .filter(c -> c.type.streamable)
-                                       .count() - 1;  // -1 because we don't include the compression component
-    }
-
-    @SuppressWarnings("DefaultAnnotationParam")
-    @Parameterized.Parameter(0)
-    public boolean isLiteral;
-    @Parameterized.Parameter(1)
-    public boolean isZeroCopyStreaming;
-    @Parameterized.Parameter(2)
-    public boolean isWide;
-
-    @Parameterized.Parameters(name = "isLiteral={0}, isZeroCopyStreaming={1}")
-    public static List<Object[]> data()
-    {
-        List<Object[]> result = new ArrayList<>();
-        for (boolean isLiteral : BOOLEANS)
-            for (boolean isZeroCopyStreaming : BOOLEANS)
-                for (boolean isWide : BOOLEANS)
-                    result.add(new Object[]{ isLiteral, isZeroCopyStreaming, isWide });
-        return result;
-    }
-
-    @Test
-    public void testIndexComponentStreaming() throws IOException
-    {
-        try (Cluster cluster = init(Cluster.build(2)
-                                           .withConfig(c -> c.with(Feature.values())
-                                                             .set("stream_entire_sstables", isZeroCopyStreaming)
-                                                             .set("streaming_slow_events_log_timeout", "0s"))
-                                           .start()))
-        {
-            cluster.schemaChange(withKeyspace(
-                isWide
-                ? "CREATE TABLE %s.test (pk int, ck int , literal text, numeric int, b blob, PRIMARY KEY(pk, ck)) WITH compression = { 'enabled' : false };"
-                : "CREATE TABLE %s.test (pk int PRIMARY KEY , literal text, numeric int, b blob) WITH compression = { 'enabled' : false };"
-            ));
-
-            int numSSTableComponents = isWide ? V1OnDiskFormat.WIDE_PER_SSTABLE_COMPONENTS.size() : V1OnDiskFormat.SKINNY_PER_SSTABLE_COMPONENTS.size();
-            int numIndexComponents = isLiteral ? V1OnDiskFormat.LITERAL_COMPONENTS.size() : V1OnDiskFormat.NUMERIC_COMPONENTS.size();
-            int numComponents = sstableStreamingComponentsCount() + numSSTableComponents + numIndexComponents + 1;
-
-            cluster.schemaChange(withKeyspace("CREATE INDEX ON %s.test(literal) USING 'sai';"));
-            cluster.schemaChange(withKeyspace("CREATE INDEX ON %s.test(numeric) USING 'sai';"));
-
-            cluster.stream().forEach(i ->
-                i.nodetoolResult("disableautocompaction", KEYSPACE).asserts().success()
-            );
-            IInvokableInstance first = cluster.get(1);
-            IInvokableInstance second = cluster.get(2);
-            long sstableCount = 10;
-            long expectedFiles = isZeroCopyStreaming ? sstableCount * numComponents : sstableCount;
-
-            for (int i = 0; i < sstableCount; i++)
-            {
-                if (isWide)
-                {
-                    String insertTemplate = "INSERT INTO %s.test(pk, ck, " + (isLiteral ? "literal" : "numeric") + ", b) VALUES (?, ?, ?, ?)";
-                    first.executeInternal(withKeyspace(insertTemplate), i, i, isLiteral ? "v" + i : Integer.valueOf(i), BLOB);
-                }
-                else
-                {
-                    String insertTemplate = "INSERT INTO %s.test(pk, " + (isLiteral ? "literal" : "numeric") + ", b) VALUES (?, ?, ?)";
-                    first.executeInternal(withKeyspace(insertTemplate), i, isLiteral ? "v" + i : Integer.valueOf(i), BLOB);
-                }
-                first.flush(KEYSPACE);
-            }
-
-            second.nodetoolResult("rebuild", "--keyspace", KEYSPACE).asserts().success();
-
-            SimpleQueryResult qr = first.executeInternalWithResult("SELECT * FROM system_views.streaming");
-            String txt = QueryResultUtil.expand(qr);
-            qr.reset();
-            assertThat(qr.toObjectArrays().length).describedAs("Found rows\n%s", txt).isEqualTo(1);
-            assertThat(qr.hasNext()).isTrue();
-            Row row = qr.next();
-            QueryResultUtil.assertThat(row)
-                           .isEqualTo("peers", Collections.singletonList(second.broadcastAddress().toString()))
-                           .isEqualTo("follower", true)
-                           .isEqualTo("operation", "Rebuild")
-                           .isEqualTo("status", "success")
-                           .isEqualTo("progress_percentage", 100.0F)
-                           .isEqualTo("success_message", null).isEqualTo("failure_cause", null)
-                           .isEqualTo("files_sent", expectedFiles)
-                           .columnsEqualTo("files_sent", "files_to_send")
-                           .columnsEqualTo("bytes_sent", "bytes_to_send")
-                           .isEqualTo("files_received", 0L)
-                           .columnsEqualTo("files_received", "files_to_receive", "bytes_received", "bytes_to_receive");
-            long totalBytes = row.getLong("bytes_sent");
-            assertThat(totalBytes).isGreaterThan(0);
-
-            qr = second.executeInternalWithResult("SELECT * FROM system_views.streaming");
-            txt = QueryResultUtil.expand(qr);
-            qr.reset();
-            assertThat(qr.toObjectArrays().length).describedAs("Found rows\n%s", txt).isEqualTo(1);
-            assertThat(qr.hasNext()).isTrue();
-
-            QueryResultUtil.assertThat(qr.next())
-                           .isEqualTo("peers", Collections.singletonList(first.broadcastAddress().toString()))
-                           .isEqualTo("follower", false)
-                           .isEqualTo("operation", "Rebuild")
-                           .isEqualTo("status", "success")
-                           .isEqualTo("progress_percentage", 100.0F)
-                           .isEqualTo("success_message", null).isEqualTo("failure_cause", null)
-                           .columnsEqualTo("files_to_receive", "files_received").isEqualTo("files_received", expectedFiles)
-                           .columnsEqualTo("bytes_to_receive", "bytes_received").isEqualTo("bytes_received", totalBytes)
-                           .columnsEqualTo("files_sent", "files_to_send", "bytes_sent", "bytes_to_send").isEqualTo("files_sent", 0L);
-
-            // did we trigger slow event log?
-            cluster.forEach(i -> Assertions.assertThat(i.logs().grep("Handling streaming events took longer than").getResult())
-                                           .describedAs("Unable to find slow log for node%d", i.config().num())
-                                           .isNotEmpty());
-
-            for (int i = 0; i < sstableCount; i++)
-            {
-                Object[][] rs = isLiteral ? second.executeInternal(withKeyspace("select pk from %s.test where literal = ?"), "v" + i)
-                                          : second.executeInternal(withKeyspace("select pk from %s.test where numeric = ?"), i);
-                assertThat(rs.length).isEqualTo(1);
-                assertThat(rs[0][0]).isEqualTo(i);
-            }
-        }
-    }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/MapEntryRangeQueryTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/MapEntryRangeQueryTest.java
new file mode 100644
index 000000000000..c6c19f3a591e
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/MapEntryRangeQueryTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.cql3.CQLTester.row;
+import static org.apache.cassandra.cql3.CQLTester.getRandom;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class MapEntryRangeQueryTest extends TestBaseImpl
+{
+    @Rule
+    public SAITester.FailureWatcher failureRule = new SAITester.FailureWatcher();
+
+    private static final String CREATE_KEYSPACE = "CREATE KEYSPACE %%s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d}";
+    private static final String CREATE_TABLE = "CREATE TABLE %%s (pk int PRIMARY KEY, inventory map<text,int>)";
+    private static final String CREATE_INDEX = "CREATE CUSTOM INDEX ON %%s(entries(%s)) USING 'StorageAttachedIndex'";
+    private static final int NUM_REPLICAS = 3;
+    private static final int RF = 2;
+
+    private static final AtomicInteger seq = new AtomicInteger();
+    private static String table;
+
+    private static Cluster cluster;
+
+    private static int dimensionCount;
+
+    @BeforeClass
+    public static void setupCluster() throws Exception
+    {
+        cluster = Cluster.build(NUM_REPLICAS)
+                         .withConfig(config -> config.with(GOSSIP).with(NETWORK))
+                         .start();
+
+        cluster.schemaChange(withKeyspace(String.format(CREATE_KEYSPACE, RF)));
+    }
+
+    @AfterClass
+    public static void closeCluster()
+    {
+        if (cluster != null)
+            cluster.close();
+    }
+
+    @Before
+    public void before()
+    {
+        table = "table_" + seq.getAndIncrement();
+        dimensionCount = getRandom().nextIntBetween(100, 2048);
+    }
+
+    @After
+    public void after()
+    {
+        cluster.schemaChange(formatQuery("DROP TABLE IF EXISTS %s"));
+    }
+
+    @Test
+    public void testRangeQueryOnMapEntries()
+    {
+        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount)));
+        cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "inventory")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        int entryCount = getRandom().nextIntBetween(500, 1000);
+        List<Pair<Integer, Integer>> values = IntStream.range(0, entryCount)
+                                                       .mapToObj(i -> Pair.create(i, getRandom().nextIntBetween(0, 100)))
+                                                       .collect(Collectors.toList());
+
+        for (var value : values)
+            execute(String.format("INSERT INTO %s (pk, inventory) VALUES (%s, {'apple':%s})",
+                KEYSPACE + '.' + table, value.left, value.right));
+
+        // Test each kind or range query
+        assertThat(Arrays.asList(execute("SELECT pk FROM %s WHERE inventory['apple'] >= 50")))
+        .hasSameElementsAs(getExpectedFilteredResults(values, i -> i >= 50));
+
+        assertThat(Arrays.asList(execute("SELECT pk FROM %s WHERE inventory['apple'] > 25")))
+        .hasSameElementsAs(getExpectedFilteredResults(values, i -> i > 25));
+
+        assertThat(Arrays.asList(execute("SELECT pk FROM %s WHERE inventory['apple'] <= 30")))
+        .hasSameElementsAs(getExpectedFilteredResults(values, i -> i <= 30));
+
+        assertThat(Arrays.asList(execute("SELECT pk FROM %s WHERE inventory['apple'] < 56")))
+        .hasSameElementsAs(getExpectedFilteredResults(values, i -> i < 56));
+    }
+
+    private List<Object[]> getExpectedFilteredResults(List<Pair<Integer, Integer>> values, Predicate<Integer> filter)
+    {
+        return values.stream()
+                     .filter(p -> filter.test(p.right))
+                     .map(p -> row(p.left))
+                     .collect(Collectors.toList());
+    }
+
+    private static Object[][] execute(String query)
+    {
+        return execute(query, ConsistencyLevel.QUORUM);
+    }
+
+    private static Object[][] execute(String query, ConsistencyLevel consistencyLevel)
+    {
+        return cluster.coordinator(1).execute(formatQuery(query), consistencyLevel);
+    }
+
+    private static String formatQuery(String query)
+    {
+        return String.format(query, KEYSPACE + '.' + table);
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/NativeIndexDDLTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/NativeIndexDDLTest.java
new file mode 100644
index 000000000000..da93d5a787e7
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/NativeIndexDDLTest.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.shared.Byteman;
+import org.apache.cassandra.distributed.shared.NetworkTopology;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class NativeIndexDDLTest extends TestBaseImpl
+{
+    private static final String FAILURE_SCRIPT = "RULE fail IndexGCTransaction\n" +
+                                                 "CLASS org.apache.cassandra.index.SecondaryIndexManager$IndexGCTransaction\n" +
+                                                 "METHOD <init>\n" +
+                                                 "AT ENTRY\n" +
+                                                 "IF TRUE\n" +
+                                                 "DO\n" +
+                                                 "   throw new java.lang.RuntimeException(\"Injected index failure\")\n" +
+                                                 "ENDRULE\n" +
+                                                 "RULE fail CleanupGCTransaction\n" +
+                                                 "CLASS org.apache.cassandra.index.SecondaryIndexManager$CleanupGCTransaction\n" +
+                                                 "METHOD <init>\n" +
+                                                 "AT ENTRY\n" +
+                                                 "IF TRUE\n" +
+                                                 "DO\n" +
+                                                 "   throw new java.lang.RuntimeException(\"Injected index failure\")\n" +
+                                                 "ENDRULE\n";
+
+    private static final String CREATE_TABLE_TEMPLATE = "CREATE TABLE %s.%s (id TEXT PRIMARY KEY, v1 INT, v2 TEXT) " +
+                                                        "WITH compaction = {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
+    protected static final String CREATE_INDEX_TEMPLATE = "CREATE CUSTOM INDEX ON %s.%s(%s) USING 'StorageAttachedIndex'";
+
+    private Cluster cluster;
+
+    @Before
+    public void setupClusterWithSingleNode() throws Throwable
+    {
+        cluster = builder().withNodes(1)
+                           .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(2))
+                           .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(2, "dc0", "rack0"))
+                           .withConfig(config -> config.with(GOSSIP, NETWORK))
+                           .withInstanceInitializer((cl, nodeNumber) -> {
+                               Byteman.createFromText(FAILURE_SCRIPT).install(cl);
+                           })
+                           .start();
+
+        cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}"));
+    }
+
+    @After
+    public void destroyCluster() throws Throwable
+    {
+        FileUtils.closeQuietly(cluster);
+    }
+
+    @Test
+    public void verifyIndexWithDecommission() throws Exception
+    {
+        // prepare schema ks rf=1 with 2 indexes
+        String table = "verify_ndi_during_decommission_test";
+        cluster.schemaChange(String.format(CREATE_TABLE_TEMPLATE, KEYSPACE, table));
+        cluster.schemaChange(String.format(CREATE_INDEX_TEMPLATE, KEYSPACE, table, "v1"));
+        cluster.schemaChange(String.format(CREATE_INDEX_TEMPLATE, KEYSPACE, table, "v2"));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        // create 100 rows in 1 sstable
+        int num = 100;
+        for (int i = 0; i < num; i++)
+        {
+            cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (id, v1, v2) VALUES ('%s', 0, '0')", KEYSPACE, table, i), ConsistencyLevel.ONE);
+        }
+
+        cluster.get(1).flush(KEYSPACE);
+
+        verifyIndexQuery(1, table, num, num);
+        assertEquals(num, getIndexedCellCount(1, table, "v1"));
+        assertEquals(num, getIndexedCellCount(1, table, "v2"));
+
+        // Start node2
+        bootstrapAndJoinNode(cluster);
+
+        // node1 still has all indexed data before cleanup
+        assertEquals(num, getIndexedCellCount(1, table, "v1"));
+        assertEquals(num, getIndexedCellCount(1, table, "v2"));
+
+        // compaction won't cleanup data
+        upgradeSSTables(1, KEYSPACE, table);
+        assertEquals(num, getIndexedCellCount(1, table, "v1"));
+        assertEquals(num, getIndexedCellCount(1, table, "v2"));
+
+        // repair streaming does not transfer entire storage-attached indexes
+        //TODO Is this assumption correct?
+        long indexRowsNode2 = getIndexedCellCount(2, table, "v1");
+        assertNotEquals(0, indexRowsNode2);
+        assertNotEquals(num, indexRowsNode2);
+        assertEquals(indexRowsNode2, getIndexedCellCount(2, table, "v2"));
+        verifyIndexQuery(2, table, num, num);
+
+        // rewrite storage-attached indexes on node2, SAI indexes should not contain rows belonging to node1
+        upgradeSSTables(2, KEYSPACE, table);
+        indexRowsNode2 = getIndexedCellCount(2, table, "v1");
+        assertNotEquals(0, indexRowsNode2);
+        assertNotEquals(num, indexRowsNode2);
+        assertEquals(indexRowsNode2, getIndexedCellCount(2, table, "v2"));
+
+        // verify data with concurrent nodetool cleanup
+        TestWithConcurrentVerification cleanupTest = new TestWithConcurrentVerification(() -> {
+            try
+            {
+                verifyIndexQuery(1, table, num, num);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }, () -> cluster.get(1).runOnInstance(() -> {
+            try
+            {
+                int status = StorageService.instance.forceKeyspaceCleanup(KEYSPACE, table);
+                assert status == CompactionManager.AllSSTableOpStatus.SUCCESSFUL.statusCode : "Cleanup failed";
+            }
+            catch (IOException | ExecutionException | InterruptedException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }));
+
+        cleanupTest.start();
+
+        // verify indexed rows on node1 and it should remove transferred data
+        long indexRowsNode1 = getIndexedCellCount(1, table, "v1");
+        assertNotEquals(0, indexRowsNode1);
+        assertEquals(indexRowsNode1, getIndexedCellCount(1, table, "v2"));
+        assertEquals(num, indexRowsNode1 + indexRowsNode2);
+
+        verifyIndexQuery(1, table, num, num);
+
+        // have to change system_distributed and system_traces to RF=1 for decommission to pass in 2-node setup
+        for (String ks : Arrays.asList("system_traces", "system_distributed"))
+        {
+            cluster.schemaChange("ALTER KEYSPACE " + ks + " WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+        }
+
+        // verify data with concurrent decommission
+        TestWithConcurrentVerification decommissionTest = new TestWithConcurrentVerification(() -> {
+            try
+            {
+                verifyIndexQuery(1, table, num, num);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }, () -> cluster.get(2).runOnInstance(() -> {
+            try
+            {
+                StorageService.instance.decommission(false);
+            }
+            catch (InterruptedException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }));
+
+        decommissionTest.start();
+        cluster.get(2).shutdown().get();
+
+        verifyIndexQuery(1, table, num, num);
+
+        // node1 has all indexed data after decommission
+        assertEquals(num, getIndexedCellCount(1, table, "v1"));
+        assertEquals(num, getIndexedCellCount(1, table, "v2"));
+    }
+
+    private void upgradeSSTables(int node, String keyspace, String table)
+    {
+        cluster.get(node).runOnInstance(() -> {
+            try
+            {
+                StorageService.instance.upgradeSSTables(keyspace, false, table);
+            }
+            catch (IOException | ExecutionException | InterruptedException e)
+            {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    private void verifyIndexQuery(int node, String table, int numericIndexRows, int stringIndexRows) throws Exception
+    {
+        verifyNumericIndexQuery(node, table, numericIndexRows);
+        verifyStringIndexQuery(node, table, stringIndexRows);
+    }
+
+    private void verifyNumericIndexQuery(int node, String table, int numericIndexRows) throws Exception
+    {
+        Object[][] result = cluster.coordinator(node).execute(String.format("SELECT id FROM %s.%s WHERE v1=0", KEYSPACE, table), ConsistencyLevel.ONE);
+        assertEquals(numericIndexRows, result.length);
+    }
+
+    private void verifyStringIndexQuery(int node, String table, int stringIndexRows) throws Exception
+    {
+        Object[][] result = cluster.coordinator(node).execute(String.format("SELECT id FROM %s.%s WHERE v2='0'", KEYSPACE, table), ConsistencyLevel.ONE);
+        assertEquals(stringIndexRows, result.length);
+    }
+
+    protected long getIndexedCellCount(int node, String table, String column) throws Exception
+    {
+        return cluster.get(node).callOnInstance(() -> {
+            try
+            {
+                ColumnIdentifier columnID = ColumnIdentifier.getInterned(column, true);
+                ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table);
+                String indexName = IndexMetadata.generateDefaultIndexName(table, columnID);
+                StorageAttachedIndex index = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName);
+                return index.getIndexContext().getCellCount();
+            }
+            catch (Throwable e)
+            {
+                return -1L;
+            }
+        });
+    }
+
+    protected static class TestWithConcurrentVerification
+    {
+        private final Runnable verificationTask;
+        private final CountDownLatch verificationStarted = new CountDownLatch(1);
+
+        private final Runnable targetTask;
+        private final CountDownLatch taskCompleted = new CountDownLatch(1);
+
+        private final int verificationIntervalInMs;
+        private final int verificationMaxInMs = 30000; // 30s
+
+        public TestWithConcurrentVerification(Runnable verificationTask, Runnable targetTask)
+        {
+            this(verificationTask, targetTask, 10);
+        }
+
+        /**
+         * @param verificationTask to be run concurrently with target task
+         * @param targetTask task to be performed once
+         * @param verificationIntervalInMs interval between each verification task, -1 to run verification task once
+         */
+        public TestWithConcurrentVerification(Runnable verificationTask, Runnable targetTask, int verificationIntervalInMs)
+        {
+            this.verificationTask = verificationTask;
+            this.targetTask = targetTask;
+            this.verificationIntervalInMs = verificationIntervalInMs;
+        }
+
+        public void start()
+        {
+            Thread verificationThread = new Thread(() -> {
+                verificationStarted.countDown();
+
+                while (true)
+                {
+                    try
+                    {
+                        verificationTask.run();
+
+                        if (verificationIntervalInMs < 0 || taskCompleted.await(verificationIntervalInMs, TimeUnit.MILLISECONDS))
+                            break;
+                    }
+                    catch (Throwable e)
+                    {
+                        throw Throwables.unchecked(e);
+                    }
+                }
+            });
+
+            try
+            {
+                verificationThread.start();
+                verificationStarted.await();
+
+                targetTask.run();
+                taskCompleted.countDown();
+
+                verificationThread.join(verificationMaxInMs);
+            }
+            catch (InterruptedException e)
+            {
+                throw Throwables.unchecked(e);
+            }
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/PartialUpdateHandlingTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/PartialUpdateHandlingTest.java
index b4825adbb079..5953ba148dd3 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/PartialUpdateHandlingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/PartialUpdateHandlingTest.java
@@ -32,6 +32,7 @@
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -42,13 +43,13 @@
 import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.index.sai.plan.Expression;
 
+import static org.apache.cassandra.index.sai.plan.Expression.Op.EQ;
+import static org.apache.cassandra.index.sai.plan.Expression.Op.RANGE;
 import static org.junit.Assert.assertEquals;
 import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
 import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
-import static org.apache.cassandra.index.sai.plan.Expression.IndexOperator.EQ;
-import static org.apache.cassandra.index.sai.plan.Expression.IndexOperator.RANGE;
 
 /**
  * SAI queries, like all filtering queries, must correctly resolve divergent views of row data across replicas. In
@@ -113,7 +114,7 @@ static class Specification
         final StatementType partialUpdateType;
         final int partitionKey;
         final boolean flushPartials;
-        final Expression.IndexOperator validationMode;
+        final Expression.Op validationMode;
 
         Specification(boolean restrictPartitionKey,
                       String[] columns,
@@ -121,7 +122,7 @@ static class Specification
                       StatementType partialUpdateType,
                       int partitionKey,
                       boolean flushPartials,
-                      Expression.IndexOperator validationMode)
+                      Expression.Op validationMode)
         {
             this.restrictPartitionKey = restrictPartitionKey;
             this.columns = columns;
@@ -450,6 +451,7 @@ public static List<Object[]> parameters()
         return parameters;
     }
 
+    @Ignore("CNDB-9331: Address PartialUpdateHandlingTest failures and possible CC rebase conflicts with CASSANDRA-19018")
     @Test
     public void testPartialUpdateResolution()
     {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/StrictFilteringTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/StrictFilteringTest.java
index 301336f8629d..250ce8b91ff7 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/StrictFilteringTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/StrictFilteringTest.java
@@ -24,6 +24,7 @@
 import org.assertj.core.api.Assertions;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.Operator;
@@ -55,6 +56,7 @@ public static void setUpCluster() throws IOException
     }
 
     @Test
+    @Ignore("CNDB-9331: Address CC rebase conflicts with CASSANDRA-19018")
     public void shouldRejectNonStrictIN()
     {
         CLUSTER.schemaChange(withKeyspace("CREATE TABLE %s.reject_in (k int PRIMARY KEY, a int, b int) WITH read_repair = 'NONE'"));
@@ -78,6 +80,7 @@ public void shouldRejectNonStrictIN()
     }
 
     @Test
+    @Ignore("CNDB-9331: Address CC rebase conflicts with CASSANDRA-19018")
     public void testPartialUpdates()
     {
         CLUSTER.schemaChange(withKeyspace("CREATE TABLE %s.partial_updates (k int PRIMARY KEY, a int, b int) WITH read_repair = 'NONE'"));
@@ -147,6 +150,7 @@ public void testPartialUpdatesStaticOnly()
     }
 
     @Test
+    @Ignore("CNDB-9331: Address CC rebase conflicts with CASSANDRA-19018")
     public void testShortReadWithRegularColumns()
     {
         CLUSTER.schemaChange(withKeyspace("CREATE TABLE %s.partial_updates_short_read (k int PRIMARY KEY, a int, b int) WITH read_repair = 'NONE'"));
@@ -167,6 +171,7 @@ public void testShortReadWithRegularColumns()
     }
 
     @Test
+    @Ignore("CNDB-9331: Address CC rebase conflicts with CASSANDRA-19018")
     public void testShortReadWithStaticColumn()
     {
         CLUSTER.schemaChange(withKeyspace("CREATE TABLE %s.partial_updates_short_read_static (k int, c int, a int, b int static, PRIMARY KEY(k, c)) WITH read_repair = 'NONE'"));
@@ -187,6 +192,7 @@ public void testShortReadWithStaticColumn()
     }
 
     @Test
+    @Ignore("CNDB-9331: Address CC rebase conflicts with CASSANDRA-19018")
     public void testTimestampCollision()
     {
         CLUSTER.schemaChange(withKeyspace("CREATE TABLE %s.timestamp_collision (k int PRIMARY KEY, a int, b int) WITH read_repair = 'NONE'"));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/TraceTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/TraceTest.java
new file mode 100644
index 000000000000..f51c063b5454
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/TraceTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.impl.TracingUtil;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+
+public class TraceTest extends TestBaseImpl
+{
+    private final static int ROWS = 100;
+    private final static int MATCHED_ROWS = 30;
+
+    private final static Pattern NUMBER_PATTERN = Pattern.compile("\\d+");
+
+    @Test
+    public void testMultiIndexTracing() throws Throwable
+    {
+        String originalTraceTimeout = TracingUtil.setWaitForTracingEventTimeoutSecs("1");
+
+        try (Cluster cluster = init(Cluster.build(3)
+                                           .withConfig(config -> config.with(GOSSIP, NETWORK))
+                                           .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3))
+                                           .start()))
+        {
+            cluster.schemaChange("CREATE KEYSPACE trace_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};");
+            cluster.schemaChange("CREATE TABLE trace_ks.tbl (pk int primary key, v1 int)");
+            cluster.schemaChange("CREATE CUSTOM INDEX tbl_v1_idx ON trace_ks.tbl(v1) USING 'StorageAttachedIndex'");
+
+            for (int row = 0; row < ROWS; row++)
+            {
+                cluster.coordinator(1).execute(String.format("INSERT INTO trace_ks.tbl (pk, v1) VALUES (%s, %s)", row, row), ConsistencyLevel.ONE);
+            }
+
+            cluster.forEach(c -> c.flush(KEYSPACE));
+
+            SAIUtil.waitForIndexQueryable(cluster, "trace_ks");
+
+            UUID sessionId = nextTimeUUID().asUUID();
+            cluster.coordinator(1).executeWithTracingWithResult(sessionId, "SELECT * from trace_ks.tbl WHERE v1 < " + MATCHED_ROWS, ConsistencyLevel.ONE);
+
+            await().atMost(5, TimeUnit.SECONDS).until(() -> {
+                List<TracingUtil.TraceEntry> traceEntries = TracingUtil.getTrace(cluster, sessionId, ConsistencyLevel.ONE);
+                return traceEntries.stream().map(traceEntry -> traceEntry.activity)
+                                   .filter(activity -> activity.contains("post-filtered"))
+                                   .mapToLong(this::fetchPartitionCount).sum() == MATCHED_ROWS;
+            });
+            
+            //TODO We can improve the asserts for this when we have improved tracing and multi-node support
+            assertEquals(MATCHED_ROWS, TracingUtil.getTrace(cluster, sessionId, ConsistencyLevel.ONE)
+                                        .stream()
+                                        .map(traceEntry -> traceEntry.activity)
+                                        .filter(activity -> activity.contains("post-filtered"))
+                                        .mapToLong(this::fetchPartitionCount).sum());
+        }
+        finally
+        {
+            TracingUtil.setWaitForTracingEventTimeoutSecs(originalTraceTimeout);
+        }
+    }
+    
+    private long fetchPartitionCount(String activity)
+    {
+        List<Long> values = new ArrayList<>();
+        Matcher matcher = NUMBER_PATTERN.matcher(activity);
+        while (matcher.find())
+            values.add(Long.parseLong(matcher.group()));
+        return values.get(3);
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/VectorDistributedTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/VectorDistributedTest.java
index 4c770860bcd1..3ff231d7c538 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/VectorDistributedTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/VectorDistributedTest.java
@@ -20,10 +20,12 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.IntToDoubleFunction;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -36,9 +38,12 @@
 import org.junit.Rule;
 import org.junit.Test;
 
-import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -46,26 +51,34 @@
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
-import org.apache.cassandra.index.sai.utils.Glove;
+import org.apache.cassandra.index.sai.cql.GeoDistanceAccuracyTest;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import org.apache.cassandra.index.sai.cql.VectorTester;
+import org.apache.cassandra.index.sai.disk.vector.VectorSourceModel;
 
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.index.sai.SAITester.getRandom;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 public class VectorDistributedTest extends TestBaseImpl
 {
+    private static final Logger logger = LoggerFactory.getLogger(VectorDistributedTest.class);
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
     @Rule
     public SAITester.FailureWatcher failureRule = new SAITester.FailureWatcher();
 
     private static final String CREATE_KEYSPACE = "CREATE KEYSPACE %%s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d}";
     private static final String CREATE_TABLE = "CREATE TABLE %%s (pk int primary key, val vector<float, %d>)";
-    private static final String CREATE_INDEX = "CREATE INDEX ON %%s(%s) USING 'sai' WITH OPTIONS={'optimize_for':'recall'}";
+    private static final String CREATE_INDEX = "CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'";
 
-    private static final VectorSimilarityFunction function = IndexWriterConfig.DEFAULT_SIMILARITY_FUNCTION;
+    private static final VectorSimilarityFunction function = VectorSourceModel.OTHER.defaultSimilarityFunction;
 
-    private static final double MIN_RECALL = 0.7;
+    private static final double MIN_RECALL_AVG = 0.8;
+    // Multiple runs of the geo search test shows the recall test results in between 89% and 97%
+    private static final double MIN_GEO_SEARCH_RECALL = 0.85;
 
     private static final int NUM_REPLICAS = 3;
     private static final int RF = 2;
@@ -75,20 +88,15 @@ public class VectorDistributedTest extends TestBaseImpl
 
     private static Cluster cluster;
 
-    protected static Glove.WordVector word2vec;
+    private static int dimensionCount;
 
     @BeforeClass
     public static void setupCluster() throws Exception
     {
-        word2vec = Glove.parse(VectorDistributedTest.class.getClassLoader().getResourceAsStream("glove.3K.50d.txt"));
-
         cluster = Cluster.build(NUM_REPLICAS)
-                         .withTokenCount(1)
-                         .withDataDirCount(1) // VSTODO Vector memtable flush doesn't support multiple directories yet
-                         .withConfig(config -> config.with(GOSSIP)
-                                                     .with(NETWORK)
-                                                     .set("memtable_allocation_type", Config.MemtableAllocationType.heap_buffers)
-                                                     .set("memtable_heap_space", "20MiB"))
+                         .withTokenCount(1) // VSTODO in-jvm-test in CC branch doesn't support multiple tokens
+                         .withDataDirCount(1) // VSTODO vector memtable flush doesn't support multiple directories yet
+                         .withConfig(config -> config.with(GOSSIP).with(NETWORK))
                          .start();
 
         cluster.schemaChange(withKeyspace(String.format(CREATE_KEYSPACE, RF)));
@@ -105,6 +113,7 @@ public static void closeCluster()
     public void before()
     {
         table = "table_" + seq.getAndIncrement();
+        dimensionCount = getRandom().nextIntBetween(100, 2048);
     }
 
     @After
@@ -116,11 +125,11 @@ public void after()
     @Test
     public void testVectorSearch()
     {
-        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, word2vec.dimension())));
+        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount)));
         cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "val")));
         SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
 
-        int vectorCount = SAITester.getRandom().nextIntBetween(500, 1000);
+        int vectorCount = getRandom().nextIntBetween(500, 1000);
         List<float[]> vectors = generateVectors(vectorCount);
 
         int pk = 0;
@@ -128,42 +137,47 @@ public void testVectorSearch()
             execute("INSERT INTO %s (pk, val) VALUES (" + (pk++) + ", " + vectorString(vector) + " )");
 
         // query memtable index
-        int limit = Math.min(SAITester.getRandom().nextIntBetween(10, 50), vectors.size());
-        float[] queryVector = randomVector();
-        Object[][] result = searchWithLimit(queryVector, limit);
-
-        List<float[]> resultVectors = getVectors(result);
-        assertDescendingScore(queryVector, resultVectors);
-
-        assertThatThrownBy(() -> searchWithoutLimit(randomVector(), vectorCount))
-        .hasMessageContaining(SelectStatement.TOPK_LIMIT_ERROR);
-
-        int pageSize = SAITester.getRandom().nextIntBetween(40, 70);
-        limit = SAITester.getRandom().nextIntBetween(20, 50);
-        result = searchWithPageAndLimit(queryVector, pageSize, limit);
-
-        resultVectors = getVectors(result);
-        assertDescendingScore(queryVector, resultVectors);
-        double memtableRecallWithPaging = getRecall(vectors, queryVector, resultVectors);
-        assertThat(memtableRecallWithPaging).isGreaterThanOrEqualTo(MIN_RECALL);
+        double memtableRecall = testMultiple((__) ->
+        {
+            float[] queryVector = randomVector();
+            int limit = Math.min(getRandom().nextIntBetween(10, 50), vectors.size());
+            Object[][] result = searchWithLimit(queryVector, limit);
+            return computeRecall(queryVector, vectors, getVectors(result));
+        });
+        assertThat(memtableRecall).isGreaterThanOrEqualTo(MIN_RECALL_AVG);
 
-        assertThatThrownBy(() -> searchWithPageWithoutLimit(randomVector()))
-        .hasMessageContaining(SelectStatement.TOPK_LIMIT_ERROR);
+        double memtableRecallWithPaging = testMultiple((__) -> {
+            float[] queryVector = randomVector();
+            int pageSize = getRandom().nextIntBetween(40, 70);
+            var limit = getRandom().nextIntBetween(20, 50);
+            var result = searchWithPageAndLimit(queryVector, pageSize, limit);
+            return computeRecall(queryVector, vectors, getVectors(result));
+        });
+        assertThat(memtableRecallWithPaging).isGreaterThanOrEqualTo(MIN_RECALL_AVG);
 
         // query on-disk index
         cluster.forEach(n -> n.flush(KEYSPACE));
 
-        limit = Math.min(SAITester.getRandom().nextIntBetween(10, 50), vectors.size());
-        queryVector = randomVector();
-        result = searchWithLimit(queryVector, limit);
-        double sstableRecall = getRecall(vectors, queryVector, getVectors(result));
-        assertThat(sstableRecall).isGreaterThanOrEqualTo(MIN_RECALL);
+        double sstableRecall = testMultiple((__) ->
+        {
+            float[] queryVector = randomVector();
+            var limit = Math.min(getRandom().nextIntBetween(20, 50), vectors.size());
+            var result = searchWithLimit(queryVector, limit);
+            return computeRecall(queryVector, vectors, getVectors(result));
+        });
+        assertThat(sstableRecall).isGreaterThanOrEqualTo(MIN_RECALL_AVG);
+    }
+
+    private double testMultiple(IntToDoubleFunction f)
+    {
+        int ITERS = 10;
+        return IntStream.range(0, ITERS).mapToDouble(f).sum() / ITERS;
     }
 
     @Test
     public void testMultiSSTablesVectorSearch()
     {
-        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, word2vec.dimension())));
+        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount)));
         cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "val")));
         SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
         // disable compaction
@@ -173,8 +187,8 @@ public void testMultiSSTablesVectorSearch()
             keyspace.getColumnFamilyStore(tableName).disableAutoCompaction();
         }));
 
-        int vectorCountPerSSTable = SAITester.getRandom().nextIntBetween(200, 500);
-        int sstableCount = SAITester.getRandom().nextIntBetween(3, 5);
+        int vectorCountPerSSTable = getRandom().nextIntBetween(200, 500);
+        int sstableCount = getRandom().nextIntBetween(3, 5);
         List<float[]> allVectors = new ArrayList<>(sstableCount * vectorCountPerSSTable);
 
         int pk = 0;
@@ -189,25 +203,71 @@ public void testMultiSSTablesVectorSearch()
         }
 
         // query multiple sstable indexes in multiple node
-        int limit = Math.min(SAITester.getRandom().nextIntBetween(50, 100), allVectors.size());
-        float[] queryVector = randomVector();
-        Object[][] result = searchWithLimit(queryVector, limit);
-
-        // expect recall to be at least 0.8
-        List<float[]> resultVectors = getVectors(result);
-        assertDescendingScore(queryVector, resultVectors);
-        double recall = getRecall(allVectors, queryVector, getVectors(result));
-        assertThat(recall).isGreaterThanOrEqualTo(MIN_RECALL);
+        double recall = testMultiple((__) ->
+        {
+            int limit = Math.min(getRandom().nextIntBetween(50, 100), allVectors.size());
+            float[] queryVector = randomVector();
+            Object[][] result = searchWithLimit(queryVector, limit);
+            return computeRecall(queryVector, allVectors, getVectors(result));
+        });
+        assertThat(recall).isGreaterThanOrEqualTo(MIN_RECALL_AVG);
+    }
+
+    @Test
+    public void testBasicGeoDistance()
+    {
+        dimensionCount = 2;
+        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount)));
+        // geo requries euclidean similarity function
+        cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "val") + " WITH OPTIONS = {'similarity_function' : 'euclidean'}"));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+        // disable compaction
+        String tableName = table;
+        cluster.forEach(n -> n.runOnInstance(() -> {
+            Keyspace keyspace = Keyspace.open(KEYSPACE);
+            keyspace.getColumnFamilyStore(tableName).disableAutoCompaction();
+        }));
+
+        int vectorCountPerSSTable = getRandom().nextIntBetween(3000, 5000);
+        int sstableCount = getRandom().nextIntBetween(7, 10);
+        List<float[]> allVectors = new ArrayList<>(sstableCount * vectorCountPerSSTable);
+
+        int pk = 0;
+        for (int i = 0; i < sstableCount; i++)
+        {
+            List<float[]> vectors = generateUSBoundedGeoVectors(vectorCountPerSSTable);
+            for (float[] vector : vectors)
+                execute("INSERT INTO %s (pk, val) VALUES (" + (pk++) + ", " + vectorString(vector) + " )");
+
+            allVectors.addAll(vectors);
+            cluster.forEach(n -> n.flush(KEYSPACE));
+        }
+
+        // Run the query 50 times to get an average of several queries
+        int queryCount = 50;
+        double recallSum = 0;
+        for (int i = 0; i < queryCount; i++)
+        {
+            // query multiple sstable indexes in multiple node
+            int searchRadiusMeters = getRandom().nextIntBetween(500, 20000);
+            float[] queryVector = randomUSVector();
+            Object[][] result = execute("SELECT val FROM %s WHERE GEO_DISTANCE(val, " + Arrays.toString(queryVector) + ") < " + searchRadiusMeters);
+
+            var recall = getGeoRecall(allVectors, queryVector, searchRadiusMeters, getVectors(result));
+            recallSum += recall;
+        }
+        logger.info("Observed recall rate: {}", recallSum / queryCount);
+        assertThat(recallSum / queryCount).isGreaterThanOrEqualTo(MIN_GEO_SEARCH_RECALL);
     }
 
     @Test
     public void testPartitionRestrictedVectorSearch()
     {
-        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, word2vec.dimension())));
+        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount)));
         cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "val")));
         SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
 
-        int vectorCount = SAITester.getRandom().nextIntBetween(500, 1000);
+        int vectorCount = getRandom().nextIntBetween(500, 1000);
         List<float[]> vectors = generateVectors(vectorCount);
 
         int pk = 0;
@@ -217,9 +277,9 @@ public void testPartitionRestrictedVectorSearch()
         // query memtable index
         for (int executionCount = 0; executionCount < 50; executionCount++)
         {
-            int key = SAITester.getRandom().nextIntBetween(0, vectorCount - 1);
+            int key = getRandom().nextIntBetween(0, vectorCount - 1);
             float[] queryVector = randomVector();
-            searchByKeyWithLimit(key, queryVector, vectors);
+            searchByKeyWithLimit(key, queryVector, 1, vectors);
         }
 
         cluster.forEach(n -> n.flush(KEYSPACE));
@@ -227,20 +287,20 @@ public void testPartitionRestrictedVectorSearch()
         // query on-disk index
         for (int executionCount = 0; executionCount < 50; executionCount++)
         {
-            int key = SAITester.getRandom().nextIntBetween(0, vectorCount - 1);
+            int key = getRandom().nextIntBetween(0, vectorCount - 1);
             float[] queryVector = randomVector();
-            searchByKeyWithLimit(key, queryVector, vectors);
+            searchByKeyWithLimit(key, queryVector, 1, vectors);
         }
     }
 
     @Test
     public void rangeRestrictedTest()
     {
-        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, word2vec.dimension())));
+        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount)));
         cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "val")));
         SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
 
-        int vectorCount = SAITester.getRandom().nextIntBetween(500, 1000);
+        int vectorCount = getRandom().nextIntBetween(500, 1000);
         List<float[]> vectors = IntStream.range(0, vectorCount).mapToObj(s -> randomVector()).collect(Collectors.toList());
 
         int pk = 0;
@@ -254,26 +314,27 @@ public void rangeRestrictedTest()
         // query memtable index
         for (int executionCount = 0; executionCount < 50; executionCount++)
         {
-            int key1 = SAITester.getRandom().nextIntBetween(1, vectorCount * 2);
+            int key1 = getRandom().nextIntBetween(1, vectorCount * 2);
             long token1 = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(key1)).getLongValue();
-            int key2 = SAITester.getRandom().nextIntBetween(1, vectorCount * 2);
+            int key2 = getRandom().nextIntBetween(1, vectorCount * 2);
             long token2 = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(key2)).getLongValue();
 
             long minToken = Math.min(token1, token2);
             long maxToken = Math.max(token1, token2);
+            float[] queryVector = randomVector();
             List<float[]> expected = vectorsByToken.entries().stream()
                                                    .filter(e -> e.getKey() >= minToken && e.getKey() <= maxToken)
                                                    .map(Map.Entry::getValue)
+                                                   .sorted(Comparator.comparingDouble(v -> function.compare(vts.createFloatVector(v), vts.createFloatVector(queryVector))).reversed())
                                                    .collect(Collectors.toList());
 
-            float[] queryVector = randomVector();
             List<float[]> resultVectors = searchWithRange(queryVector, minToken, maxToken, expected.size());
             if (expected.isEmpty())
                 assertThat(resultVectors).isEmpty();
             else
             {
-                double recall = getRecall(resultVectors, queryVector, expected);
-                assertThat(recall).isGreaterThanOrEqualTo(0.6);
+                double recall = computeRecall(queryVector, resultVectors, expected);
+                assertThat(recall).isGreaterThanOrEqualTo(0.8);
             }
         }
 
@@ -282,31 +343,65 @@ public void rangeRestrictedTest()
         // query on-disk index with existing key:
         for (int executionCount = 0; executionCount < 50; executionCount++)
         {
-            int key1 = SAITester.getRandom().nextIntBetween(1, vectorCount * 2);
+            int key1 = getRandom().nextIntBetween(1, vectorCount * 2);
             long token1 = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(key1)).getLongValue();
-            int key2 = SAITester.getRandom().nextIntBetween(1, vectorCount * 2);
+            int key2 = getRandom().nextIntBetween(1, vectorCount * 2);
             long token2 = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(key2)).getLongValue();
 
             long minToken = Math.min(token1, token2);
             long maxToken = Math.max(token1, token2);
+            float[] queryVector = randomVector();
             List<float[]> expected = vectorsByToken.entries().stream()
                                                    .filter(e -> e.getKey() >= minToken && e.getKey() <= maxToken)
                                                    .map(Map.Entry::getValue)
+                                                   .sorted(Comparator.comparingDouble(v -> function.compare(vts.createFloatVector(v), vts.createFloatVector(queryVector))).reversed())
                                                    .collect(Collectors.toList());
 
-            float[] queryVector = randomVector();
-
             List<float[]> resultVectors = searchWithRange(queryVector, minToken, maxToken, expected.size());
             if (expected.isEmpty())
                 assertThat(resultVectors).isEmpty();
             else
             {
-                double recall = getRecall(resultVectors, queryVector, expected);
+                double recall = computeRecall(queryVector, resultVectors, expected);
                 assertThat(recall).isGreaterThanOrEqualTo(0.8);
             }
         }
     }
 
+    @Test
+    public void testInvalidVectorQueriesWithCosineSimilarity()
+    {
+        dimensionCount = 2;
+        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount)));
+        cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "val") + " WITH OPTIONS = {'similarity_function' : 'cosine'}"));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        assertInvalidCosineOperations();
+    }
+
+    private static void assertInvalidCosineOperations()
+    {
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, val) VALUES (0, [0.0, 0.0])")).hasMessage("Zero and near-zero vectors cannot be indexed or queried with cosine similarity");
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, val) VALUES (0, [1, NaN])")).hasMessage("non-finite value at vector[1]=NaN");
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, val) VALUES (0, [1, Infinity])")).hasMessage("non-finite value at vector[1]=Infinity");
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, val) VALUES (0, [-Infinity, 1])")).hasMessage("non-finite value at vector[0]=-Infinity");
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY val ann of [0.0, 0.0] LIMIT 2")).hasMessage("Zero and near-zero vectors cannot be indexed or queried with cosine similarity");
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY val ann of [1, NaN] LIMIT 2")).hasMessage("non-finite value at vector[1]=NaN");
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY val ann of [1, Infinity] LIMIT 2")).hasMessage("non-finite value at vector[1]=Infinity");
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY val ann of [-Infinity, 1] LIMIT 2")).hasMessage("non-finite value at vector[0]=-Infinity");
+    }
+
+    @Test
+    public void testInvalidVectorQueriesWithDefaultSimilarity()
+    {
+        dimensionCount = 2;
+        cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount)));
+        cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "val")));
+        SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
+
+        assertInvalidCosineOperations();
+    }
+
     private List<float[]> searchWithRange(float[] queryVector, long minToken, long maxToken, int expectedSize)
     {
         Object[][] result = execute("SELECT val FROM %s WHERE token(pk) <= " + maxToken + " AND token(pk) >= " + minToken + " ORDER BY val ann of " + Arrays.toString(queryVector) + " LIMIT 1000");
@@ -321,38 +416,26 @@ private Object[][] searchWithLimit(float[] queryVector, int limit)
         return result;
     }
 
-    private void searchWithoutLimit(float[] queryVector, int results)
-    {
-        Object[][] result = execute("SELECT val FROM %s ORDER BY val ann of " + Arrays.toString(queryVector));
-        assertThat(result).hasNumberOfRows(results);
-    }
-
-
-    private void searchWithPageWithoutLimit(float[] queryVector)
-    {
-        executeWithPaging("SELECT val FROM %s ORDER BY val ann of " + Arrays.toString(queryVector), 10);
-    }
-
     private Object[][] searchWithPageAndLimit(float[] queryVector, int pageSize, int limit)
     {
         // we don't know how many will be returned in case of paging, because coordinator resumes from last-returned-row's partiton
         return executeWithPaging("SELECT val FROM %s ORDER BY val ann of " + Arrays.toString(queryVector) + " LIMIT " + limit, pageSize);
     }
 
-    private void searchByKeyWithLimit(int key, float[] queryVector, List<float[]> vectors)
+    private void searchByKeyWithLimit(int key, float[] queryVector, int limit, List<float[]> vectors)
     {
-        Object[][] result = execute("SELECT val FROM %s WHERE pk = " + key + " ORDER BY val ann of " + Arrays.toString(queryVector) + " LIMIT 1");
+        Object[][] result = execute("SELECT val FROM %s WHERE pk = " + key + " ORDER BY val ann of " + Arrays.toString(queryVector) + " LIMIT " + limit);
         assertThat(result).hasNumberOfRows(1);
         float[] output = getVectors(result).get(0);
         assertThat(output).isEqualTo(vectors.get(key));
     }
 
-    private void assertDescendingScore(float[] queryVector, List<float[]> resultVectors)
+    private static void assertDescendingScore(float[] queryVector, List<float[]> resultVectors)
     {
         float prevScore = -1;
         for (float[] current : resultVectors)
         {
-            float score = function.compare(current, queryVector);
+            float score = function.compare(vts.createFloatVector(current), vts.createFloatVector(queryVector));
             if (prevScore >= 0)
                 assertThat(score).isLessThanOrEqualTo(prevScore);
 
@@ -360,29 +443,10 @@ private void assertDescendingScore(float[] queryVector, List<float[]> resultVect
         }
     }
 
-    private double getRecall(List<float[]> vectors, float[] query, List<float[]> result)
+    private static double computeRecall(float[] query, List<float[]> vectors, List<float[]> results)
     {
-        List<float[]> sortedVectors = new ArrayList<>(vectors);
-        sortedVectors.sort((a, b) -> Double.compare(function.compare(b, query), function.compare(a, query)));
-
-        assertThat(sortedVectors).containsAll(result);
-
-        List<float[]> nearestNeighbors = sortedVectors.subList(0, result.size());
-
-        int matches = 0;
-        for (float[] in : nearestNeighbors)
-        {
-            for (float[] out : result)
-            {
-                if (Arrays.compare(in, out) ==0)
-                {
-                    matches++;
-                    break;
-                }
-            }
-        }
-
-        return matches * 1.0 / result.size();
+        assertDescendingScore(query, results);
+        return VectorTester.computeRecall(vectors, query, results, function);
     }
 
     private List<float[]> generateVectors(int vectorCount)
@@ -390,7 +454,6 @@ private List<float[]> generateVectors(int vectorCount)
         return IntStream.range(0, vectorCount).mapToObj(s -> randomVector()).collect(Collectors.toList());
     }
 
-    @SuppressWarnings("unchecked")
     private List<float[]> getVectors(Object[][] result)
     {
         List<float[]> vectors = new ArrayList<>();
@@ -415,17 +478,58 @@ private String vectorString(float[] vector)
 
     private float[] randomVector()
     {
-        return word2vec.vector(SAITester.getRandom().nextIntBetween(0, word2vec.size() - 1));
+        return CQLTester.randomVector(dimensionCount);
+    }
+
+    private List<float[]> generateUSBoundedGeoVectors(int vectorCount)
+    {
+        return IntStream.range(0, vectorCount).mapToObj(s -> randomUSVector()).collect(Collectors.toList());
+    }
+
+    private float[] randomUSVector()
+    {
+        // Approximate bounding box for contiguous US locations
+        var lat = getRandom().nextFloatBetween(24, 49);
+        var lon = getRandom().nextFloatBetween(-124, -67);
+        return new float[] {lat, lon};
     }
 
+    private double getGeoRecall(List<float[]> allVectors, float[] query, float distance, List<float[]> resultVectors)
+    {
+        assertThat(allVectors).containsAll(resultVectors);
+        var expectdVectors = allVectors.stream().filter(v -> GeoDistanceAccuracyTest.isWithinDistance(v, query, distance))
+                                 .collect(Collectors.toSet());
+        int matches = 0;
+        for (float[] expectedVector : expectdVectors)
+        {
+            for (float[] resultVector : resultVectors)
+            {
+                if (Arrays.compare(expectedVector, resultVector) == 0)
+                {
+                    matches++;
+                    break;
+                }
+            }
+        }
+        if (expectdVectors.isEmpty() && resultVectors.isEmpty())
+            return 1.0;
+        return matches * 1.0 / expectdVectors.size();
+    }
+
+
     private static Object[][] execute(String query)
     {
-        return cluster.coordinator(1).execute(formatQuery(query), ConsistencyLevel.ONE);
+        return execute(query, ConsistencyLevel.QUORUM);
+    }
+
+    private static Object[][] execute(String query, ConsistencyLevel consistencyLevel)
+    {
+        return cluster.coordinator(1).execute(formatQuery(query), consistencyLevel);
     }
 
     private static Object[][] executeWithPaging(String query, int pageSize)
     {
-        Iterator<Object[]> iterator = cluster.coordinator(1).executeWithPaging(formatQuery(query), ConsistencyLevel.ONE, pageSize);
+        Iterator<Object[]> iterator = cluster.coordinator(1).executeWithPaging(formatQuery(query), ConsistencyLevel.QUORUM, pageSize);
         List<Object[]> list = new ArrayList<>();
         iterator.forEachRemaining(list::add);
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/VectorValidationTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/VectorValidationTest.java
deleted file mode 100644
index df9cde79e437..000000000000
--- a/test/distributed/org/apache/cassandra/distributed/test/sai/VectorValidationTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.test.sai;
-
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.Cluster;
-import org.apache.cassandra.distributed.test.TestBaseImpl;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.utils.AssertionUtils;
-
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-public class VectorValidationTest extends TestBaseImpl
-{
-    @Test
-    public void vectorIndexNotAllowedWithMultipleDataDirectories() throws Throwable
-    {
-        try (Cluster cluster = Cluster.build(3)
-                                      .withTokenCount(1)
-                                      .withDataDirCount(3)
-                                      .start())
-        {
-            cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2}"));
-            cluster.schemaChange(withKeyspace("CREATE TABLE %s.data_dir_test (pk int primary key, val vector<float, 10>)"));
-            assertThatThrownBy(() -> cluster.schemaChange(withKeyspace("CREATE INDEX ON %s.data_dir_test(val) USING 'sai'")))
-            .is(AssertionUtils.is(InvalidRequestException.class))
-            .hasMessage(StorageAttachedIndex.VECTOR_MULTIPLE_DATA_DIRECTORY_ERROR);
-        }
-    }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/MultiNodeExecutor.java b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/MultiNodeExecutor.java
new file mode 100644
index 000000000000..e038b52ca53b
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/MultiNodeExecutor.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai.datamodels;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.sai.SAIUtil;
+import org.apache.cassandra.distributed.util.ColumnTypeUtil;
+import org.apache.cassandra.index.sai.cql.datamodels.DataModel;
+
+public class MultiNodeExecutor implements DataModel.Executor
+{
+    private final Cluster cluster;
+
+    public MultiNodeExecutor(Cluster cluster)
+    {
+        this.cluster = cluster;
+    }
+
+    @Override
+    public void createTable(String statement)
+    {
+        cluster.schemaChange(statement);
+    }
+
+    @Override
+    public void flush(String keyspace, String table)
+    {
+        cluster.forEach(node -> node.flush(keyspace));
+    }
+
+    @Override
+    public void compact(String keyspace, String table)
+    {
+        cluster.forEach(node -> node.forceCompact(keyspace, table));
+    }
+
+    @Override
+    public void disableCompaction(String keyspace, String table)
+    {
+        cluster.forEach((node) -> node.runOnInstance(() -> Keyspace.open(keyspace).getColumnFamilyStore(table).disableAutoCompaction()));
+    }
+
+    @Override
+    public void waitForTableIndexesQueryable(String keyspace, String table)
+    {
+        SAIUtil.waitForIndexQueryable(cluster, keyspace);
+    }
+
+    @Override
+    public void executeLocal(String query, Object... values)
+    {
+        Object[] buffers = ColumnTypeUtil.transformValues(values);
+        cluster.coordinator(1).execute(query, ConsistencyLevel.QUORUM, buffers);
+    }
+
+    @Override
+    public List<Object> executeRemote(String query, int fetchSize, Object... values)
+    {
+        Object[] buffers = ColumnTypeUtil.transformValues(values);
+        Iterator<Object> iterator = cluster.coordinator(1).executeWithPagingWithResult(query, ConsistencyLevel.QUORUM, fetchSize, buffers).map(row -> row.get(0));
+
+        List<Object> result = new ArrayList<>();
+        iterator.forEachRemaining(result::add);
+
+        return result;
+    }
+
+    @Override
+    public void counterReset()
+    {
+        MultiNodeQueryTester.Counter.reset();
+    }
+
+    @Override
+    public long getCounter()
+    {
+        return MultiNodeQueryTester.Counter.get();
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/MultiNodeQueryTester.java b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/MultiNodeQueryTester.java
new file mode 100644
index 000000000000..aa0d886cad21
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/MultiNodeQueryTester.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai.datamodels;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.shared.Byteman;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.index.sai.cql.datamodels.DataModel;
+import org.apache.cassandra.index.sai.cql.datamodels.IndexQuerySupport;
+import org.apache.cassandra.utils.Shared;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+
+@RunWith(Parameterized.class)
+abstract class MultiNodeQueryTester extends TestBaseImpl
+{
+    protected static final String INJECTION_SCRIPT = "RULE count searches\n" +
+                                                     "CLASS org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher\n" +
+                                                     "METHOD search\n" +
+                                                     "AT ENTRY\n" +
+                                                     "IF TRUE\n" +
+                                                     "DO\n" +
+                                                     "   org.apache.cassandra.distributed.test.sai.datamodels.MultiNodeQueryTester$Counter.increment()\n" +
+                                                     "ENDRULE\n";
+
+    @Parameterized.Parameter(0)
+    public String name;
+    @Parameterized.Parameter(1)
+    public Supplier<DataModel> dataModel;
+    @Parameterized.Parameter(2)
+    public List<IndexQuerySupport.BaseQuerySet> sets;
+
+    protected static DataModel.Executor executor;
+
+    protected static Cluster cluster;
+
+    @BeforeClass
+    public static void setupCluster() throws Exception
+    {
+        cluster = Cluster.build(3)
+                         .withConfig(config -> config.with(NETWORK, GOSSIP)
+                                                     .set("hinted_handoff_enabled", false))
+                         .withInstanceInitializer((cl, nodeNumber) -> {
+                             Byteman.createFromText(INJECTION_SCRIPT).install(cl);
+                         })
+                         .start();
+
+        cluster.schemaChange("CREATE KEYSPACE " + DataModel.KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2}");
+
+        executor = new MultiNodeExecutor(cluster);
+    }
+
+    @AfterClass
+    public static void closeCluster()
+    {
+        if (cluster != null)
+            cluster.close();
+    }
+
+    @SuppressWarnings("unused")
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params() throws Throwable
+    {
+        List<Object[]> scenarios = new LinkedList<>();
+
+        scenarios.add(new Object[]{ "BaseDataModel",
+                                    (Supplier<DataModel>) () -> new DataModel.BaseDataModel(DataModel.NORMAL_COLUMNS, DataModel.NORMAL_COLUMN_DATA),
+                                    IndexQuerySupport.BASE_QUERY_SETS });
+
+        scenarios.add(new Object[]{ "CompoundKeyDataModel",
+                                    (Supplier<DataModel>) () -> new DataModel.CompoundKeyDataModel(DataModel.NORMAL_COLUMNS, DataModel.NORMAL_COLUMN_DATA),
+                                    IndexQuerySupport.BASE_QUERY_SETS });
+
+        scenarios.add(new Object[]{ "CompoundKeyWithStaticsDataModel",
+                                    (Supplier<DataModel>) () -> new DataModel.CompoundKeyWithStaticsDataModel(DataModel.STATIC_COLUMNS, DataModel.STATIC_COLUMN_DATA),
+                                    IndexQuerySupport.STATIC_QUERY_SETS });
+
+        scenarios.add(new Object[]{ "CompositePartitionKeyDataModel",
+                                    (Supplier<DataModel>) () -> new DataModel.CompositePartitionKeyDataModel(DataModel.NORMAL_COLUMNS, DataModel.NORMAL_COLUMN_DATA),
+                                    ImmutableList.builder().addAll(IndexQuerySupport.BASE_QUERY_SETS).addAll(IndexQuerySupport.COMPOSITE_PARTITION_QUERY_SETS).build() });
+
+        return scenarios;
+    }
+
+    @Shared
+    protected static final class Counter
+    {
+        protected static AtomicLong counter = new AtomicLong(0);
+
+        public static void increment()
+        {
+            counter.incrementAndGet();
+        }
+
+        public static void reset()
+        {
+            counter.set(0);
+        }
+
+        public static long get()
+        {
+            return counter.get();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/QueryCellDeletionsTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryCellDeletionsTest.java
similarity index 76%
rename from test/unit/org/apache/cassandra/index/sai/cql/QueryCellDeletionsTest.java
rename to test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryCellDeletionsTest.java
index 6b423580e3bd..93abd062f061 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/QueryCellDeletionsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryCellDeletionsTest.java
@@ -15,15 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.index.sai.cql;
+
+package org.apache.cassandra.distributed.test.sai.datamodels;
 
 import org.junit.Test;
 
-public class QueryCellDeletionsTest extends AbstractQueryTester
+import org.apache.cassandra.index.sai.cql.datamodels.IndexQuerySupport;
+
+public class QueryCellDeletionsTest extends MultiNodeQueryTester
 {
     @Test
     public void testCellDeletions() throws Throwable
     {
-        IndexQuerySupport.cellDeletions(executor, dataModel, sets);
+        IndexQuerySupport.cellDeletions(executor, dataModel.get(), sets);
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/QueryRowDeletionsTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryRowDeletionsTest.java
similarity index 76%
rename from test/unit/org/apache/cassandra/index/sai/cql/QueryRowDeletionsTest.java
rename to test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryRowDeletionsTest.java
index a5a843d99e4d..f1f063f611ae 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/QueryRowDeletionsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryRowDeletionsTest.java
@@ -15,15 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.index.sai.cql;
+
+package org.apache.cassandra.distributed.test.sai.datamodels;
 
 import org.junit.Test;
 
-public class QueryRowDeletionsTest extends AbstractQueryTester
+import org.apache.cassandra.index.sai.cql.datamodels.IndexQuerySupport;
+
+public class QueryRowDeletionsTest extends MultiNodeQueryTester
 {
     @Test
     public void testRowDeletions() throws Throwable
     {
-        IndexQuerySupport.rowDeletions(executor, dataModel, sets);
+        IndexQuerySupport.rowDeletions(executor, dataModel.get(), sets);
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/QueryTimeToLiveTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryTimeToLiveTest.java
similarity index 77%
rename from test/unit/org/apache/cassandra/index/sai/cql/QueryTimeToLiveTest.java
rename to test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryTimeToLiveTest.java
index 39a77b496c48..9e223767c07e 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/QueryTimeToLiveTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryTimeToLiveTest.java
@@ -15,15 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.index.sai.cql;
+
+package org.apache.cassandra.distributed.test.sai.datamodels;
 
 import org.junit.Test;
 
-public class QueryTimeToLiveTest extends AbstractQueryTester
+import org.apache.cassandra.index.sai.cql.datamodels.IndexQuerySupport;
+
+public class QueryTimeToLiveTest extends MultiNodeQueryTester
 {
     @Test
     public void testTimeToLive() throws Throwable
     {
-        IndexQuerySupport.timeToLive(executor, dataModel, sets);
+        IndexQuerySupport.timeToLive(executor, dataModel.get(), sets);
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/QueryWriteLifecycleTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryWriteLifecycleTest.java
similarity index 76%
rename from test/unit/org/apache/cassandra/index/sai/cql/QueryWriteLifecycleTest.java
rename to test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryWriteLifecycleTest.java
index a41ff47f5f86..83525f703c4c 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/QueryWriteLifecycleTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/datamodels/QueryWriteLifecycleTest.java
@@ -15,15 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.index.sai.cql;
+
+package org.apache.cassandra.distributed.test.sai.datamodels;
 
 import org.junit.Test;
 
-public class QueryWriteLifecycleTest extends AbstractQueryTester
+import org.apache.cassandra.index.sai.cql.datamodels.IndexQuerySupport;
+
+public class QueryWriteLifecycleTest extends MultiNodeQueryTester
 {
     @Test
     public void testWriteLifecycle() throws Throwable
     {
-        IndexQuerySupport.writeLifecycle(executor, dataModel, sets);
+        IndexQuerySupport.writeLifecycle(executor, dataModel.get(), sets);
     }
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sensors/SensorsTest.java b/test/distributed/org/apache/cassandra/distributed/test/sensors/SensorsTest.java
new file mode 100644
index 000000000000..0cd3ef935388
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sensors/SensorsTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sensors;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.cql3.QueryHandler;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.sensors.ActiveSensorsFactory;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.assertj.core.api.Assertions;
+
+/**
+ * Test to verify that the sensors are propagated via the native protocol in the custom payload respecting
+ * the configuration set in {@link CassandraRelevantProperties#SENSORS_VIA_NATIVE_PROTOCOL}
+ */
+@RunWith(Parameterized.class)
+public class SensorsTest extends TestBaseImpl
+{
+    private static final String EXPECTED_WRITE_BYTES_HEADER = "WRITE_BYTES_REQUEST." + KEYSPACE + ".tbl";
+    private static final String EXPECTED_READ_BYTES_HEADER = "READ_BYTES_REQUEST." + KEYSPACE + ".tbl";
+    /**
+     * Using a combination of 2 nodes with ALL consistency level to ensure internode communication code paths are exercised in the test
+     */
+    private static final int NODES_COUNT = 2;
+    private static final ConsistencyLevel CONSISTENCY_LEVEL = ConsistencyLevel.ALL;
+
+    /**
+     * Schema to be used for the test
+     */
+    @Parameterized.Parameter(0)
+    public String schema;
+    /**
+     * Queries to be executed to prepare the table, for example insert some data before read to populate read sensors.
+     * Will be run before the {@link #testQuery}
+     */
+    @Parameterized.Parameter(1)
+    public String[] prepQueries;
+
+    /**
+     * Query to be executed to test the sensors, will be run after the {@link #prepQueries}
+     */
+    @Parameterized.Parameter(2)
+    public String testQuery;
+
+    /**
+     * Expected headers in the custom payload for the test queries
+     */
+    @Parameterized.Parameter(3)
+    public String[] expectedHeaders;
+
+    @BeforeClass
+    public static void setup()
+    {
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(ActiveSensorsFactory.class.getName());
+    }
+
+    @Parameterized.Parameters(name = "schema={0}, prepQueries={1}, testQuery={2}, expectedHeaders={3}")
+    public static Collection<Object[]> data()
+    {
+        String tableSchema = withKeyspace("CREATE TABLE %s.tbl (pk int PRIMARY KEY, v1 text)");
+        String counterTableSchema = withKeyspace("CREATE TABLE %s.tbl (pk int PRIMARY KEY, total counter)");
+
+        String write = withKeyspace("INSERT INTO %s.tbl(pk, v1) VALUES (1, 'read me')");
+        String counter = withKeyspace("UPDATE %s.tbl SET total = total + 1 WHERE pk = 1");
+        String read = withKeyspace("SELECT * FROM %s.tbl WHERE pk=1");
+        String cas = withKeyspace("UPDATE %s.tbl SET v1 = 'cas update' WHERE pk = 1 IF v1 = 'read me'");
+        String loggedBatch = String.format("BEGIN BATCH\n" +
+                                           "INSERT INTO %s.tbl(pk, v1) VALUES (2, 'read me 2');\n" +
+                                           "INSERT INTO %s.tbl(pk, v1) VALUES (3, 'read me 3');\n" +
+                                           "APPLY BATCH;", KEYSPACE, KEYSPACE);
+        String unloggedBatch = String.format("BEGIN UNLOGGED BATCH\n" +
+                                             "INSERT INTO %s.tbl(pk, v1) VALUES (4, 'read me 2');\n" +
+                                             "INSERT INTO %s.tbl(pk, v1) VALUES (4, 'read me 3');\n" +
+                                             "APPLY BATCH;", KEYSPACE, KEYSPACE);
+        String range = withKeyspace("SELECT * FROM %s.tbl");
+
+        List<Object[]> result = new ArrayList<>();
+        String[] noPrep = new String[0];
+        result.add(new Object[]{ tableSchema, noPrep, write, new String[]{ EXPECTED_WRITE_BYTES_HEADER } });
+        result.add(new Object[]{ counterTableSchema, noPrep, counter, new String[]{ EXPECTED_WRITE_BYTES_HEADER } });
+        result.add(new Object[]{ tableSchema, new String[]{ write }, read, new String[]{ EXPECTED_READ_BYTES_HEADER } });
+        // CAS requests incorporate read (and write) bytes from the paxos (and user) tables
+        result.add(new Object[]{ tableSchema, noPrep, cas, new String[]{ EXPECTED_WRITE_BYTES_HEADER } });
+        result.add(new Object[]{ tableSchema, noPrep, loggedBatch, new String[]{ EXPECTED_WRITE_BYTES_HEADER } });
+        result.add(new Object[]{ tableSchema, noPrep, unloggedBatch, new String[]{ EXPECTED_WRITE_BYTES_HEADER } });
+        result.add(new Object[]{ tableSchema, new String[]{ write }, range, new String[]{ EXPECTED_READ_BYTES_HEADER } });
+        return result;
+    }
+
+    @Test
+    public void testSensorsInCQLResponseEnabled() throws Throwable
+    {
+        Map<String, ByteBuffer> customPayload = executeTest(true);
+        for (String header : expectedHeaders)
+        {
+            double requestBytes = getBytesForHeader(customPayload, header);
+            Assertions.assertThat(requestBytes).isGreaterThan(0D);
+        }
+    }
+
+    @Test
+    public void testSensorsInCQLResponseDisabled() throws Throwable
+    {
+        Map<String, ByteBuffer> customPayload = executeTest(false);
+        // customPayload will be null if it has no headers. However, non-sensor headers could've been added. So here we check for nullability or non-existence of sensor headers
+        if (customPayload != null)
+        {
+            for (String header : expectedHeaders)
+            {
+                Assertions.assertThat(customPayload).doesNotContainKey(header);
+            }
+        } // else do nothing as null customPayload means no sensors were added
+    }
+
+    /**
+     * Execute the test with the given {@code propagateViaNativeProtocol} flag and return the custom payload
+     */
+    private Map<String, ByteBuffer> executeTest(boolean propagateViaNativeProtocol) throws Throwable
+    {
+        CassandraRelevantProperties.SENSORS_VIA_NATIVE_PROTOCOL.setBoolean(propagateViaNativeProtocol);
+        AtomicReference<Map<String, ByteBuffer>> customPayload = new AtomicReference<>();
+        try (Cluster cluster = init(Cluster.build(NODES_COUNT).start()))
+        {
+            cluster.schemaChange(schema);
+            for (String prepQuery : this.prepQueries)
+                cluster.coordinator(1).execute(prepQuery, ConsistencyLevel.ALL);
+            // work around serializability of @Parameterized.Parameter by providing a locally scoped variable
+            String query = this.testQuery;
+            // Any methods used inside the runOnInstance() block should be static, otherwise java.io.NotSerializableException will be thrown
+            cluster.get(1).acceptsOnInstance(
+                   (IIsolatedExecutor.SerializableConsumer<AtomicReference<Map<String, ByteBuffer>>>)
+                   (reference) -> reference.set(executeWithResult(query).getCustomPayload()))
+                   .accept(customPayload);
+        }
+
+        return customPayload.get();
+    }
+
+    private double getBytesForHeader(Map<String, ByteBuffer> customPayload, String expectedHeader)
+    {
+        Assertions.assertThat(customPayload).containsKey(expectedHeader);
+        return ByteBufferUtil.toDouble(customPayload.get(expectedHeader));
+    }
+
+    /**
+     * TODO: update SimpleQueryResult in the dtest-api project to expose custom payload and use Coordinator##executeWithResult instead
+     */
+    private static ResultMessage<?> executeWithResult(String query)
+    {
+        QueryHandler.Prepared prepared = QueryProcessor.prepareInternal(query);
+        ConsistencyLevel consistencyLevel = ConsistencyLevel.valueOf(CONSISTENCY_LEVEL.name());
+        org.apache.cassandra.db.ConsistencyLevel cl = org.apache.cassandra.db.ConsistencyLevel.fromCode(consistencyLevel.ordinal());
+        QueryOptions initialOptions = QueryOptions.create(cl,
+                                                          null,
+                                                          false,
+                                                          PageSize.inRows(512),
+                                                          null,
+                                                          null,
+                                                          ProtocolVersion.CURRENT,
+                                                          prepared.keyspace);
+        return prepared.statement.execute(QueryProcessor.internalQueryState(), initialOptions, Dispatcher.RequestTime.forImmediateExecution());
+    }
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java b/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java
index a0f7f602fdd2..3a785b863604 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java
@@ -46,7 +46,7 @@ public static void setupClass() throws IOException
             DatabaseDescriptor.setRowIndexReadSizeFailThreshold(new DataStorageSpec.LongBytesBound(2, KIBIBYTES));
 
             // hack to force multiple index entries
-            DatabaseDescriptor.setColumnIndexCacheSize(1 << 20);
+            DatabaseDescriptor.setColumnIndexCacheSizeInKiB(1 << 20);
             DatabaseDescriptor.setColumnIndexSizeInKiB(0);
         }));
     }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/thresholds/TombstoneCountWarningTest.java b/test/distributed/org/apache/cassandra/distributed/test/thresholds/TombstoneCountWarningTest.java
index 5e409cbc31e0..fb30e645942c 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/thresholds/TombstoneCountWarningTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/thresholds/TombstoneCountWarningTest.java
@@ -34,6 +34,7 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -71,6 +72,7 @@
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static org.assertj.core.api.Assertions.assertThat;
 
+@Ignore("STAR-543: scannedTombstones guardrail replaces tombstone client warnings from CASSANDRA-16896")
 public class TombstoneCountWarningTest extends TestBaseImpl
 {
     private static final Logger logger = LoggerFactory.getLogger(TombstoneCountWarningTest.class);
diff --git a/test/distributed/org/apache/cassandra/distributed/util/ColumnTypeUtil.java b/test/distributed/org/apache/cassandra/distributed/util/ColumnTypeUtil.java
new file mode 100644
index 000000000000..7e18d77fb193
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/util/ColumnTypeUtil.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.util;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.cassandra.cql3.Duration;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.DurationType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class ColumnTypeUtil
+{
+    public static Object[] transformValues(Object[] values)
+    {
+        // We could partly rely on QueryProcessor.executeOnceInternal doing type conversion for us, but
+        // it would complain with ClassCastException if we pass say a string where an int is excepted (since
+        // it bases conversion on what the value should be, not what it is). For testing, we sometimes
+        // want to pass value of the wrong type and assert that this properly raise an InvalidRequestException
+        // and executeOnceInternal goes into way. So instead, we pre-convert everything to bytes here based
+        // on the value.
+
+        Object[] buffers = new ByteBuffer[values.length];
+        for (int i = 0; i < values.length; i++)
+        {
+            Object value = values[i];
+            if (value == null)
+            {
+                buffers[i] = null;
+                continue;
+            }
+            else if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
+            {
+                buffers[i] = ByteBufferUtil.UNSET_BYTE_BUFFER;
+                continue;
+            }
+
+            try
+            {
+                buffers[i] = makeByteBuffer(value);
+            }
+            catch (Exception ex)
+            {
+                throw new IllegalArgumentException("Error serializing query parameter {}:" + value, ex);
+            }
+        }
+        return buffers;
+    }
+
+    @SuppressWarnings("unchecked")
+    public static ByteBuffer makeByteBuffer(Object value)
+    {
+        if (value == null)
+            return null;
+
+        if (value instanceof ByteBuffer)
+            return (ByteBuffer) value;
+
+        return typeFor(value).decompose(value);
+    }
+
+    private static AbstractType typeFor(Object value)
+    {
+        if (value instanceof ByteBuffer || value == null)
+            return BytesType.instance;
+
+        if (value instanceof Byte)
+            return ByteType.instance;
+
+        if (value instanceof Short)
+            return ShortType.instance;
+
+        if (value instanceof Integer)
+            return Int32Type.instance;
+
+        if (value instanceof Long)
+            return LongType.instance;
+
+        if (value instanceof Float)
+            return FloatType.instance;
+
+        if (value instanceof Duration)
+            return DurationType.instance;
+
+        if (value instanceof Double)
+            return DoubleType.instance;
+
+        if (value instanceof BigInteger)
+            return IntegerType.instance;
+
+        if (value instanceof BigDecimal)
+            return DecimalType.instance;
+
+        if (value instanceof String)
+            return UTF8Type.instance;
+
+        if (value instanceof Boolean)
+            return BooleanType.instance;
+
+        if (value instanceof InetAddress)
+            return InetAddressType.instance;
+
+        if (value instanceof Date)
+            return TimestampType.instance;
+
+        if (value instanceof UUID)
+            return UUIDType.instance;
+
+        if (value instanceof List)
+        {
+            List l = (List)value;
+            AbstractType elt = l.isEmpty() ? BytesType.instance : typeFor(l.get(0));
+            return ListType.getInstance(elt, true);
+        }
+
+        if (value instanceof Set)
+        {
+            Set s = (Set)value;
+            AbstractType elt = s.isEmpty() ? BytesType.instance : typeFor(s.iterator().next());
+            return SetType.getInstance(elt, true);
+        }
+
+        if (value instanceof Map)
+        {
+            Map m = (Map) value;
+            AbstractType keys, values;
+            if (m.isEmpty())
+            {
+                keys = BytesType.instance;
+                values = BytesType.instance;
+            }
+            else
+            {
+                Map.Entry entry = (Map.Entry) m.entrySet().iterator().next();
+                keys = typeFor(entry.getKey());
+                values = typeFor(entry.getValue());
+            }
+            return MapType.getInstance(keys, values, true);
+        }
+
+        throw new IllegalArgumentException("Unsupported value type (value is " + value + ')');
+    }
+
+
+
+}
diff --git a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
index 710a42664790..8a4663704307 100644
--- a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
+++ b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
@@ -23,8 +23,8 @@
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Set;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.util.concurrent.RateLimiter;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -34,6 +34,7 @@
 import org.apache.cassandra.db.Slices;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.lifecycle.AbstractLogTransaction;
 import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
@@ -44,6 +45,7 @@
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.IVerifier;
 import org.apache.cassandra.io.sstable.KeyIterator;
@@ -280,7 +282,7 @@ public void setCrcCheckChance(double crcCheckChance)
     }
 
     @Override
-    public void markObsolete(Runnable tidier)
+    public void markObsolete(AbstractLogTransaction.ReaderTidier tidier)
     {
         delegate.markObsolete(tidier);
     }
@@ -370,9 +372,9 @@ public boolean isRepaired()
     }
 
     @Override
-    public DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException
+    public IKeyFetcher openKeyFetcher(boolean isForSASI)
     {
-        return delegate.keyAtPositionFromSecondaryIndex(keyPositionFromSecondaryIndex);
+        return delegate.openKeyFetcher(isForSASI);
     }
 
     @Override
@@ -640,27 +642,27 @@ public String getKeyspaceName()
     }
 
     @Override
-    public List<String> getAllFilePaths()
+    public ImmutableSet<Component> components()
     {
-        return delegate.getAllFilePaths();
+        return delegate.components();
     }
 
     @Override
-    public long bytesOnDisk()
+    public int getComponentSize()
     {
-        return delegate.bytesOnDisk();
+        return delegate.getComponentSize();
     }
 
     @Override
-    public long logicalBytesOnDisk()
+    public long bytesOnDisk()
     {
-        return delegate.logicalBytesOnDisk();
+        return delegate.bytesOnDisk();
     }
 
     @Override
-    public Set<Component> getComponents()
+    public long logicalBytesOnDisk()
     {
-        return delegate.getComponents();
+        return delegate.logicalBytesOnDisk();
     }
 
     @Override
diff --git a/test/long/index/sai/cql/AnalyzerQueryLongTest.java b/test/long/index/sai/cql/AnalyzerQueryLongTest.java
new file mode 100644
index 000000000000..d00bf8b7d1ee
--- /dev/null
+++ b/test/long/index/sai/cql/AnalyzerQueryLongTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class AnalyzerQueryLongTest extends CQLTester
+{
+    @Test
+    public void manyWritesTest()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, not_analyzed int, val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+        var iterations = 15000;
+        for (int i = 0; i < iterations; i++)
+        {
+            var x = i % 100;
+            if (i % 100 == 0)
+            {
+                execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (?, ?, ?)", i, x, "this will be tokenized");
+            }
+            else if (i % 2 == 0)
+            {
+                execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (?, ?, ?)", i, x, "this is different");
+            }
+            else
+            {
+                execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (?, ?, ?)", i, x, "basic test");
+            }
+        }
+        // We match the first inserted statement here, and that one is just written 1/100 times
+        var result = execute("SELECT * FROM %s WHERE val : 'tokenized'");
+        assertThat(result).hasSize(iterations / 100);
+        // We match the first and second inserted statements here, and those account for 1/2 the inserts
+        result = execute("SELECT * FROM %s WHERE val : 'this'");
+        assertThat(result).hasSize(iterations / 2);
+        // We match the last write here, and that accounts for the other 1/2 of the inserts
+        result = execute("SELECT * FROM %s WHERE val : 'test'");
+        assertThat(result).hasSize(iterations / 2);
+    }
+
+    @Test
+    public void manyWritesAndUpsertsTest()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+        var iterations = 15000;
+        for (int i = 0; i < iterations; i++)
+        {
+            if (i % 999 == 0) {
+                // flush on irregular cadence so that final queries are executed based on paritially flushed data
+                flush();
+            }
+            if (i % 2 == 0)
+            {
+                // Upsert the same entry many times
+                execute("INSERT INTO %s (pk, val) VALUES (0, 'text to be analyzed')");
+            }
+            else
+            {
+                execute("INSERT INTO %s (pk, val) VALUES (?, 'different text to be analyzed')", i);
+            }
+        }
+        var result = execute("SELECT * FROM %s WHERE val : 'different'");
+        assertThat(result).hasSize(iterations / 2);
+        // We match both insert statements, but the first is continuously overwritting the same PK, so just 1 extra result
+        result = execute("SELECT * FROM %s WHERE val : 'text'");
+        assertThat(result).hasSize(iterations / 2 + 1);
+    }
+    @Test
+    public void manyWritesUpsertsAndDeletesForSamePKTest()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+        var iterations = 15000;
+        for (int i = 0; i < iterations; i++)
+        {
+            if (i % 999 == 0) {
+                // flush on irregular cadence so that final queries are executed based on paritially flushed data
+                flush();
+            }
+            // 3 works well because 14999 is the last value for i, and it is not divisible by 3
+            // Further, it by using 3, we first insert, then upsert, then delet, and repeat.
+            if (i % 3 == 0)
+            {
+                // Upsert the same entry many times
+                execute("DELETE FROM %s WHERE pk = 0");
+            }
+            else if (i % 2 == 0)
+            {
+                execute("INSERT INTO %s (pk, val) VALUES (0, 'text to be analyzed')");
+            }
+            else
+            {
+                execute("INSERT INTO %s (pk, val) VALUES (0, 'completely different value')");
+            }
+        }
+        // 'completely different value' wins
+        var result = execute("SELECT * FROM %s WHERE val : 'text'");
+        assertThat(result).hasSize(0);
+        result = execute("SELECT * FROM %s WHERE val : 'value'");
+        assertThat(result).hasSize(1);
+    }
+}
\ No newline at end of file
diff --git a/test/long/org/apache/cassandra/cql3/CorruptionTest.java b/test/long/org/apache/cassandra/cql3/CorruptionTest.java
index 0ef43a0991ed..5a8c5befa4d7 100644
--- a/test/long/org/apache/cassandra/cql3/CorruptionTest.java
+++ b/test/long/org/apache/cassandra/cql3/CorruptionTest.java
@@ -33,7 +33,12 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import com.datastax.driver.core.*;
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ConsistencyLevel;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
 import com.datastax.driver.core.policies.LoggingRetryPolicy;
 import com.datastax.driver.core.policies.Policies;
 import com.datastax.driver.core.utils.Bytes;
diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index 260ab835c93b..046442928297 100644
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@ -23,8 +23,18 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -41,19 +51,24 @@
 
 import io.netty.util.concurrent.FastThreadLocalThread;
 import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
 import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
 import org.apache.cassandra.config.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.io.compress.DeflateCompressor;
 import org.apache.cassandra.io.compress.LZ4Compressor;
 import org.apache.cassandra.io.compress.SnappyCompressor;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.security.EncryptionContext;
 import org.apache.cassandra.security.EncryptionContextGenerator;
 
@@ -75,7 +90,7 @@ public abstract class CommitLogStressTest
     public static int rateLimit = 0;
     public static int runTimeMs = 10000;
 
-    public static String location = DatabaseDescriptor.getCommitLogLocation() + "/stress";
+    public static File location = DatabaseDescriptor.getCommitLogLocation().resolve("stress");
 
     public static int hash(int hash, ByteBuffer bytes)
     {
@@ -126,10 +141,9 @@ static public void initialize() throws IOException
     @Before
     public void cleanDir() throws IOException
     {
-        File dir = new File(location);
-        if (dir.isDirectory())
+        if (location.isDirectory())
         {
-            File[] files = dir.tryList();
+            File[] files = location.tryList();
 
             for (File f : files)
                 if (!f.tryDelete())
@@ -137,7 +151,7 @@ public void cleanDir() throws IOException
         }
         else
         {
-            dir.tryCreateDirectory();
+            location.tryCreateDirectory();
         }
     }
 
@@ -180,11 +194,12 @@ public void testDiscardedRun() throws Exception
 
     private void testLog() throws IOException, InterruptedException
     {
-        String originalDir = DatabaseDescriptor.getCommitLogLocation();
+        File originalDir = DatabaseDescriptor.getCommitLogLocation();
         try
         {
             DatabaseDescriptor.setCommitLogLocation(location);
             DatabaseDescriptor.initializeCommitLogDiskAccessMode();
+            DatabaseDescriptor.getRawConfig().commitlog_directory = location.path();
             CommitLog commitLog = new CommitLog(CommitLogArchiver.disabled()).start();
             testLog(commitLog);
             assert !failed;
@@ -193,6 +208,7 @@ private void testLog() throws IOException, InterruptedException
         {
             DatabaseDescriptor.setCommitLogLocation(originalDir);
             DatabaseDescriptor.initializeCommitLogDiskAccessMode();
+            DatabaseDescriptor.getRawConfig().commitlog_directory = originalDir.path();
         }
     }
 
@@ -257,7 +273,7 @@ private void testLog(CommitLog commitLog) throws IOException, InterruptedExcepti
         System.out.println("Stopped. Replaying... ");
         System.out.flush();
         Reader reader = new Reader();
-        File[] files = new File(location).tryList();
+        File[] files = location.tryList();
 
         DummyHandler handler = new DummyHandler();
         reader.readAllFiles(handler, files);
@@ -292,10 +308,10 @@ private void verifySizes(CommitLog commitLog)
         // Complete anything that's still left to write.
         commitLog.executor.syncBlocking();
         // Wait for any concurrent segment allocations to complete.
-        commitLog.segmentManager.awaitManagementTasksCompletion();
+        commitLog.getSegmentManager().awaitManagementTasksCompletion();
 
         long combinedSize = 0;
-        for (File f : new File(commitLog.segmentManager.storageDirectory).tryList())
+        for (File f : commitLog.getSegmentManager().storageDirectory.tryList())
         {
             combinedSize += f.length();
         }
@@ -304,7 +320,7 @@ private void verifySizes(CommitLog commitLog)
 
         List<String> logFileNames = commitLog.getActiveSegmentNames();
         Map<String, Double> ratios = commitLog.getActiveSegmentCompressionRatios();
-        Collection<CommitLogSegment> segments = commitLog.segmentManager.getActiveSegments();
+        Collection<CommitLogSegment> segments = commitLog.getSegmentManager().getActiveSegments();
 
         for (CommitLogSegment segment : segments)
         {
@@ -479,7 +495,7 @@ else if (desc.id == discardedPos.segmentId && entryLocation <= discardedPos.posi
             for (PartitionUpdate cf : mutation.getPartitionUpdates())
             {
 
-                Iterator<Row> rowIterator = cf.iterator();
+                Iterator<Row> rowIterator = cf.rowIterator();
 
                 while (rowIterator.hasNext())
                 {
@@ -504,5 +520,7 @@ static class DummyHandler implements CommitLogReadHandler
         public void handleUnrecoverableError(CommitLogReadException exception) throws IOException { }
 
         public void handleMutation(Mutation m, int size, int entryLocation, CommitLogDescriptor desc) { }
+
+        public void handleInvalidMutation(TableId id) {}
     }
 }
diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
index ce72ae62a44f..94df5b6e88dd 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
@@ -18,26 +18,38 @@
 */
 package org.apache.cassandra.db.compaction;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
-import org.junit.BeforeClass;
+import com.google.common.collect.ImmutableSet;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
-import org.apache.cassandra.UpdateBuilder;
 import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.UpdateBuilder;
 import org.apache.cassandra.Util;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.SSTableUtils;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -45,11 +57,26 @@
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.junit.Assert.assertEquals;
 
+@RunWith(Parameterized.class)
 public class LongCompactionsTest
 {
     public static final String KEYSPACE1 = "Keyspace1";
     public static final String CF_STANDARD = "Standard1";
 
+    @Parameterized.Parameters(name = "useCursors={0}")
+    public static Iterable<Boolean> useCursorChoices()
+    {
+        return ImmutableSet.of(false, true);
+    }
+
+    private final boolean useCursors;
+
+    public LongCompactionsTest(boolean useCursors)
+    {
+        this.useCursors = useCursors;
+    }
+    private ColumnFamilyStore cfs;
+
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
     {
@@ -65,7 +92,7 @@ public static void defineSchema() throws ConfigurationException
     public void cleanupFiles()
     {
         Keyspace keyspace = Keyspace.open(KEYSPACE1);
-        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
+        cfs = keyspace.getColumnFamilyStore("Standard1");
         cfs.truncateBlocking();
     }
 
@@ -130,7 +157,7 @@ protected void testCompaction(int sstableCount, int partitionsPerSSTable, int ro
         try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.COMPACTION))
         {
             assert txn != null : "Cannot markCompacting all sstables";
-            new CompactionTask(store, txn, gcBefore).execute(ActiveCompactionsTracker.NOOP);
+            new CompactionTask(store, txn, gcBefore, false, CompactionTaskTest.mockStrategy(cfs, useCursors)).execute();
         }
         System.out.println(String.format("%s: sstables=%d rowsper=%d colsper=%d: %d ms",
                                          this.getClass().getName(),
@@ -192,7 +219,7 @@ private void forceCompactions(ColumnFamilyStore cfs)
         {
             ArrayList<Future<?>> compactions = new ArrayList<Future<?>>();
             for (int i = 0; i < 10; i++)
-                compactions.addAll(CompactionManager.instance.submitBackground(cfs));
+                compactions.add(CompactionManager.instance.submitBackground(cfs));
             // another compaction attempt will be launched in the background by
             // each completing compaction: not much we can do to control them here
             FBUtilities.waitOnFutures(compactions);
diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
index bbc1b577de37..e706b184eea1 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
@@ -18,24 +18,37 @@
 package org.apache.cassandra.db.compaction;
 
 import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import com.google.common.collect.Lists;
-
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
 import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.ScannerList;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.service.ActiveRepairService;
@@ -72,8 +85,10 @@ public void testParallelLeveledCompaction() throws Exception
         Keyspace keyspace = Keyspace.open(ksname);
         ColumnFamilyStore store = keyspace.getColumnFamilyStore(cfname);
         store.disableAutoCompaction();
-        CompactionStrategyManager mgr = store.getCompactionStrategyManager();
-        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) mgr.getStrategies().get(1).get(0);
+        CompactionStrategyContainer strategyContainer = store.getCompactionStrategyContainer();
+        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategyContainer
+                                                                    .getStrategies(false, null)
+                                                                    .get(0);
 
         ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KiB value, make it easy to have multiple files
 
@@ -88,17 +103,12 @@ public void testParallelLeveledCompaction() throws Exception
         {
             while (true)
             {
-                final AbstractCompactionTask nextTask = lcs.getNextBackgroundTask(Integer.MIN_VALUE);
-                if (nextTask == null)
+                final Collection<AbstractCompactionTask> nextTasks = lcs.getNextBackgroundTasks(Integer.MIN_VALUE);
+                if (nextTasks.isEmpty())
                     break;
-                tasks.add(new Runnable()
-                {
-                    public void run()
-                    {
-                        nextTask.execute(ActiveCompactionsTracker.NOOP);
-                    }
-                });
+                tasks.addAll(nextTasks.stream().map(t -> (Runnable) () -> t.execute()).collect(Collectors.toList()));
             }
+
             if (tasks.isEmpty())
                 break;
 
@@ -115,18 +125,18 @@ public void run()
         int levels = manifest.getLevelCount();
         for (int level = 0; level < levels; level++)
         {
-            Set<SSTableReader> sstables = manifest.getLevel(level);
+            Set<CompactionSSTable> sstables = manifest.getLevel(level);
             // score check
-            assert (double) SSTableReader.getTotalBytes(sstables) / manifest.maxBytesForLevel(level, 1 * 1024 * 1024) < 1.00;
+            assert (double) CompactionSSTable.getTotalDataBytes(sstables) / manifest.maxBytesForLevel(level, 1 * 1024 * 1024) < 1.00;
             // overlap check for levels greater than 0
-            for (SSTableReader sstable : sstables)
+            for (CompactionSSTable sstable : sstables)
             {
                 // level check
                 assert level == sstable.getSSTableLevel();
 
                 if (level > 0)
                 {// overlap check for levels greater than 0
-                    Set<SSTableReader> overlaps = LeveledManifest.overlapping(sstable.getFirst().getToken(), sstable.getLast().getToken(), sstables);
+                    Set<CompactionSSTable> overlaps = LeveledManifest.overlapping(sstable.getFirst().getToken(), sstable.getLast().getToken(), sstables);
                     assert overlaps.size() == 1 && overlaps.contains(sstable);
                 }
             }
@@ -144,8 +154,10 @@ public void testLeveledScanner() throws Exception
 
         LeveledCompactionStrategyTest.waitForLeveling(store);
         store.disableAutoCompaction();
-        CompactionStrategyManager mgr = store.getCompactionStrategyManager();
-        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) mgr.getStrategies().get(1).get(0);
+        CompactionStrategyContainer strategyContainer  = store.getCompactionStrategyContainer();
+        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategyContainer
+                                                                    .getStrategies(false, null)
+                                                                    .get(0);
 
         value = ByteBuffer.wrap(new byte[10 * 1024]); // 10 KiB value
 
@@ -179,7 +191,7 @@ public Void call() throws Exception
                     }
                 }
 
-                try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(Lists.newArrayList(allSSTables)))
+                try (ScannerList scannerList = lcs.getScanners(Lists.newArrayList(allSSTables)))
                 {
                     //Verify that leveled scanners will always iterate in ascending order (CASSANDRA-9935)
                     for (ISSTableScanner scanner : scannerList.scanners)
@@ -198,7 +210,7 @@ public Void call() throws Exception
                 }
                 return null;
             }
-        }, OperationType.COMPACTION, true, true);
+        }, OperationType.COMPACTION, true, true, TableOperation.StopTrigger.UNIT_TESTS);
     }
 
     @Test
@@ -210,15 +222,19 @@ public void testRepairStatusChanges() throws Exception
         ColumnFamilyStore store = keyspace.getColumnFamilyStore(cfname);
         store.disableAutoCompaction();
 
-        CompactionStrategyManager mgr = store.getCompactionStrategyManager();
-        LeveledCompactionStrategy repaired = (LeveledCompactionStrategy) mgr.getStrategies().get(0).get(0);
-        LeveledCompactionStrategy unrepaired = (LeveledCompactionStrategy) mgr.getStrategies().get(1).get(0);
+        CompactionStrategyContainer strategyContainer = store.getCompactionStrategyContainer();
+        LeveledCompactionStrategy repaired = (LeveledCompactionStrategy) strategyContainer
+                                                                         .getStrategies(true, null)
+                                                                         .get(0);
+        LeveledCompactionStrategy unrepaired = (LeveledCompactionStrategy) strategyContainer
+                                                                           .getStrategies(false, null)
+                                                                           .get(0);
 
         // populate repaired sstables
         populateSSTables(store);
         assertTrue(repaired.getSSTables().isEmpty());
         assertFalse(unrepaired.getSSTables().isEmpty());
-        mgr.mutateRepaired(store.getLiveSSTables(), FBUtilities.nowInSeconds(), null, false);
+        store.mutateRepaired(store.getLiveSSTables(), FBUtilities.nowInSeconds(), null, false);
         assertFalse(repaired.getSSTables().isEmpty());
         assertTrue(unrepaired.getSSTables().isEmpty());
 
@@ -233,7 +249,7 @@ public void testRepairStatusChanges() throws Exception
         assertFalse(unrepaired.getSSTables().isEmpty());
 
         // mark unrepair
-        mgr.mutateRepaired(store.getLiveSSTables().stream().filter(s -> s.isRepaired()).collect(Collectors.toList()),
+        store.mutateRepaired(store.getLiveSSTables().stream().filter(s -> s.isRepaired()).collect(Collectors.toList()),
                            ActiveRepairService.UNREPAIRED_SSTABLE,
                            null,
                            false);
diff --git a/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java b/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java
index 1b78c48b6eae..3f2762c609b7 100644
--- a/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java
+++ b/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java
@@ -28,7 +28,6 @@
 
 import com.google.common.collect.Iterables;
 
-import org.apache.cassandra.io.util.File;
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
@@ -36,6 +35,7 @@
 import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
@@ -124,12 +124,12 @@ private void verifyHints(File directory, HintsDescriptor descriptor)
                     Hint hint = hints.next();
 
                     long timestamp = baseTimestamp + index;
-                    Mutation mutation = hint.mutation;
+                    Mutation mutation = hint.mutation();
 
                     assertEquals(timestamp, hint.creationTime);
                     assertEquals(dk(bytes(index)), mutation.key());
 
-                    Row row = mutation.getPartitionUpdates().iterator().next().iterator().next();
+                    Row row = mutation.getPartitionUpdates().iterator().next().rowIterator().next();
                     assertEquals(1, Iterables.size(row.cells()));
                     assertEquals(bytes(index), toByteBuffer(row.clustering().get(0)));
                     Cell<?> cell = row.cells().iterator().next();
diff --git a/test/long/org/apache/cassandra/io/memtable/FlushFailingOnNotificationSubscriberTest.java b/test/long/org/apache/cassandra/io/memtable/FlushFailingOnNotificationSubscriberTest.java
new file mode 100644
index 000000000000..dcf4d9a20151
--- /dev/null
+++ b/test/long/org/apache/cassandra/io/memtable/FlushFailingOnNotificationSubscriberTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.memtable;
+
+import com.google.common.util.concurrent.AtomicDouble;
+
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.SSTableAddingNotification;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.db.memtable.AbstractAllocatorMemtable.MEMORY_POOL;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * this is a long-ish test that shows that writes do not block anymore
+ * after flush is failing in the notification subscriber
+ * <p>
+ * without the fix the test fails typically within a couple of seconds
+ * by default the test duration is set to 120s
+ */
+public class FlushFailingOnNotificationSubscriberTest extends CQLTester
+{
+    private static final int APPROXIMATE_TEST_DURATION_SECONDS = 120;
+    private static final double FLUSH_FAILURE_PROBABILITY = 0.25;
+    private final AtomicInteger numUserFlushes = new AtomicInteger();
+    private final AtomicInteger numFailedFlushes = new AtomicInteger();
+
+    volatile long maxTimeSinceCleanup = 0;
+    volatile long lastTimePoolNeededCleaning = System.nanoTime();
+
+    static AtomicDouble failFlushProbability = new AtomicDouble(FLUSH_FAILURE_PROBABILITY);
+
+    @BeforeClass
+    public static void setup()
+    {
+        Config conf = DatabaseDescriptor.getRawConfig();
+        // frequent flushes
+        conf.memtable_allocation_type = Config.MemtableAllocationType.offheap_objects;
+        conf.memtable_cleanup_threshold = 0.15f;
+        conf.memtable_heap_space = new DataStorageSpec.IntMebibytesBound(2);
+        conf.memtable_offheap_space = new DataStorageSpec.IntMebibytesBound(2);
+
+        CQLTester.setUpClass();
+    }
+
+    @Test
+    public void flushFailingOnSSTableAddingNotificationVSWritesTest() throws InterruptedException, ExecutionException, TimeoutException
+    {
+        try
+        {
+            ScheduledExecutorPlus scheduledExecutor = executorFactory().scheduled("forced flush");
+
+            createTable(KEYSPACE, "CREATE TABLE %s (pk int PRIMARY KEY, value int)", "failedflushtest");
+
+            ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+            cfs.getTracker().subscribeLateConsumer(this::maybeThrowWhenFlushAddsSSTables);
+
+            int flushPeriodSec = 15;
+
+            scheduledExecutor.scheduleAtFixedRate(this::successfulUserFlush, flushPeriodSec, flushPeriodSec, SECONDS);
+            scheduledExecutor.scheduleAtFixedRate(() -> {
+                updateMaxTimeSinceCleanup();
+                logState();
+            }, 1, 1, SECONDS);
+
+            int idx = 1;
+            while (numUserFlushes.get() < APPROXIMATE_TEST_DURATION_SECONDS / flushPeriodSec)
+            {
+                final int fidx = idx;
+                Future<UntypedResultSet> query = scheduledExecutor.submit(() -> execute("INSERT INTO %s (pk, value) VALUES (?, ?)", fidx, fidx));
+                try
+                {
+                    query.get(1, TimeUnit.SECONDS);
+                }
+                catch (TimeoutException e)
+                {
+                    logger.info("write timed out at iteration {}", idx);
+                    logState();
+                    throw new RuntimeException("Test failed because a write got stuck", e);
+                }
+
+                idx++;
+                if (MEMORY_POOL.needsCleaning())
+                {
+                    lastTimePoolNeededCleaning = System.nanoTime();
+                }
+                else
+                {
+                    updateMaxTimeSinceCleanup();
+                }
+
+                if (MEMORY_POOL.getNumPendingtasks() > 2)
+                {
+                    logger.info("Flushes seem to be backing up, sleeping at iteration {} to slow down writes", idx);
+                    logState();
+                    Thread.sleep(100);
+                }
+
+                assertEquals("write blocked on allocation", 0, MEMORY_POOL.blockedOnAllocating.getCount());
+            }
+            scheduledExecutor.shutdown();
+            assertTrue(scheduledExecutor.awaitTermination(1, TimeUnit.MINUTES));
+            successfulUserFlush();
+
+            // check the amount of memory used for the memtables is less than half of the limit
+            // it is expected that if this resource leaks then these assertions won't hold
+            assertTrue(MEMORY_POOL.onHeap.used() < 1_000_000);
+            assertTrue(MEMORY_POOL.offHeap.used() < 1_000_000);
+            // assert no memtables are stuck in a reclaiming state
+            assertEquals(0, MEMORY_POOL.onHeap.getReclaiming());
+            assertEquals(0, MEMORY_POOL.offHeap.getReclaiming());
+            // check that memtable cleanup is scheduled sufficiently often
+            assertTrue("memory pool did not clean for more than 10s: " + maxTimeSinceCleanup + " ms", maxTimeSinceCleanup < 10_000);
+            // and that there were no writes that actually got blocked due to memory pressure
+            assertEquals("write blocked on allocation", 0, MEMORY_POOL.blockedOnAllocating.getCount());
+        }
+        finally
+        {
+            logger.info("The ultimate system state:");
+            logState();
+            // If the test managed to reproduce the problem then writing may be blocked now.
+            // This means that @After in CQLTester will hang.
+            // To prevent this let's unblock writes by running a successful flush, which will
+            // free the current memtable.
+            successfulUserFlush();
+        }
+    }
+
+    private void updateMaxTimeSinceCleanup()
+    {
+        maxTimeSinceCleanup = Math.max(maxTimeSinceCleanup, (System.nanoTime() - lastTimePoolNeededCleaning) / TimeUnit.MILLISECONDS.toNanos(1));
+    }
+
+    private void logState()
+    {
+        logger.info(" --- STATE ---\n" +
+                    "Max time since pool needed cleaning: {} ms\n" +
+                    "Num failed flushes: {}\n" +
+                    "Memory pool: onHeap.used={} ({} %), onHeap.getReclaiming={} ({} %)\n" +
+                    "Memory pool: offHeap.used={} ({} %), offHeap.getReclaiming={} ({} %)\n" +
+                    "Total size of sstables: {} bytes\n" +
+                    "Num blocked allocations: {}",
+                    maxTimeSinceCleanup,
+                    numFailedFlushes,
+                    MEMORY_POOL.onHeap.used(), 100 * MEMORY_POOL.onHeap.usedRatio(), MEMORY_POOL.onHeap.getReclaiming(), 100 * MEMORY_POOL.onHeap.reclaimingRatio(),
+                    MEMORY_POOL.offHeap.used(), 100 * MEMORY_POOL.offHeap.usedRatio(), MEMORY_POOL.offHeap.getReclaiming(), 100 * MEMORY_POOL.offHeap.reclaimingRatio(),
+                    getCurrentColumnFamilyStore().getLiveSSTables().stream().mapToLong(SSTableReader::bytesOnDisk).sum(),
+                    MEMORY_POOL.blockedOnAllocating.getCount());
+    }
+
+    private void maybeThrowWhenFlushAddsSSTables(INotification notification, Object sender)
+    {
+        logger.info("Consuming notification {}", notification);
+        if (notification instanceof SSTableAddingNotification)
+        {
+            SSTableAddingNotification addingNotification = (SSTableAddingNotification) notification;
+            if (addingNotification.operationType == OperationType.FLUSH && addingNotification.memtable().get().metadata().name.equals("failedflushtest")
+                && failFlushProbability.get() > Math.random())
+            {
+                logger.info("Throwing exception for notification {}", notification);
+                numFailedFlushes.incrementAndGet();
+                throw new RuntimeException("hey I just broke your flush, haven't I?");
+            }
+        }
+    }
+
+    private void successfulUserFlush()
+    {
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.getAllMemtables().forEach(m -> logger.info("pre flush memtable: {}", m));
+        logState();
+        try
+        {
+            failFlushProbability.set(0.0);
+            getCurrentColumnFamilyStore().forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS).get();
+            failFlushProbability.set(FLUSH_FAILURE_PROBABILITY);
+        }
+        catch (InterruptedException | ExecutionException e)
+        {
+            throw new RuntimeException(e);
+        }
+        finally
+        {
+            cfs.getAllMemtables().forEach(m -> logger.info("post flush memtable: {}", m));
+            logState();
+            numUserFlushes.incrementAndGet();
+        }
+    }
+}
diff --git a/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java b/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java
index 2e27738efc18..2f8e88ff50db 100644
--- a/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java
+++ b/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java
@@ -31,11 +31,13 @@
 import org.apache.cassandra.utils.FBUtilities;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.*;
 
 public class DynamicEndpointSnitchLongTest
 {
     static
     {
+        NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
     }
 
diff --git a/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java b/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java
index e2a4edeca533..59161c0eef0e 100644
--- a/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java
+++ b/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java
@@ -249,10 +249,10 @@ default int executeReads()
         default void executeCompactions()
         {
             ColumnFamilyStore cfs = getCfs();
-            ActiveCompactions active = new ActiveCompactions();
+            ActiveOperations active = new ActiveOperations();
             Set<SSTableReader> sstables = cfs.getLiveSSTables();
 
-            CompactionTasks tasks = cfs.getCompactionStrategyManager()
+            CompactionTasks tasks = cfs.getCompactionStrategy()
                                        .getUserDefinedTasks(sstables, FBUtilities.nowInSeconds());
             
             Assert.assertFalse(tasks.isEmpty());
@@ -429,8 +429,15 @@ private static void measure(Workload workload) throws Throwable
             readSummaries.put(workload.name(), new ReadSummary(readSampler, readCount));
         }
 
-        String compactionSummary = "SKIPPED";
         ColumnFamilyStore cfs = workload.getCfs();
+        ActiveOperations active = new ActiveOperations();
+        Set<SSTableReader> sstables = cfs.getLiveSSTables();
+
+        CompactionTasks tasks = cfs.getCompactionStrategyContainer()
+                                   .getUserDefinedTasks(sstables, FBUtilities.nowInSeconds());
+        Assert.assertFalse(tasks.isEmpty());
+
+        String compactionSummary = "SKIPPED";
         if (!PROFILING_READS)
         {
             compactionSampler.start();
diff --git a/test/microbench/org/apache/cassandra/test/microbench/AbstractTypeByteSourceDecodingBench.java b/test/microbench/org/apache/cassandra/test/microbench/AbstractTypeByteSourceDecodingBench.java
index 9496172834a3..8f399fb91926 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/AbstractTypeByteSourceDecodingBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/AbstractTypeByteSourceDecodingBench.java
@@ -60,8 +60,19 @@
 @State(Scope.Benchmark)
 public class AbstractTypeByteSourceDecodingBench
 {
+    private Random prng = new Random();
+
+    @Param({"32", "128", "512"})
+    private int length;
 
-    private static final ByteComparable.Version LATEST = ByteComparable.Version.OSS50;
+    @Param({"UTF8Type", "BytesType", "IntegerType", "DecimalType"})
+    private String abstractTypeName;
+
+    @Param({"LEGACY", "OSS41", "OSS50"})
+    private static ByteComparable.Version version = ByteComparable.Version.OSS50;
+
+    private AbstractType abstractType;
+    private BiFunction<Random, Integer, ByteSource.Peekable> peekableGenerator;
 
     private static final Map<AbstractType, BiFunction<Random, Integer, ByteSource.Peekable>> PEEKABLE_GENERATOR_BY_TYPE = new HashMap<>();
     static
@@ -70,20 +81,20 @@ public class AbstractTypeByteSourceDecodingBench
         {
             byte[] randomBytes = new byte[length];
             prng.nextBytes(randomBytes);
-            return ByteSource.peekable(ByteSource.of(new String(randomBytes, StandardCharsets.UTF_8), LATEST));
+            return ByteSource.peekable(ByteSource.of(new String(randomBytes, StandardCharsets.UTF_8), version));
         });
         PEEKABLE_GENERATOR_BY_TYPE.put(BytesType.instance, (prng, length) ->
         {
             byte[] randomBytes = new byte[length];
             prng.nextBytes(randomBytes);
-            return ByteSource.peekable(ByteSource.of(randomBytes, LATEST));
+            return ByteSource.peekable(ByteSource.of(randomBytes, version));
         });
         PEEKABLE_GENERATOR_BY_TYPE.put(IntegerType.instance, (prng, length) ->
         {
             BigInteger randomVarint = BigInteger.valueOf(prng.nextLong());
             for (int i = 1; i < length / 8; ++i)
                 randomVarint = randomVarint.multiply(BigInteger.valueOf(prng.nextLong()));
-            return ByteSource.peekable(IntegerType.instance.asComparableBytes(IntegerType.instance.decompose(randomVarint), LATEST));
+            return ByteSource.peekable(IntegerType.instance.asComparableBytes(IntegerType.instance.decompose(randomVarint), version));
         });
         PEEKABLE_GENERATOR_BY_TYPE.put(DecimalType.instance, (prng, length) ->
         {
@@ -92,21 +103,10 @@ public class AbstractTypeByteSourceDecodingBench
                 randomMantissa = randomMantissa.multiply(BigInteger.valueOf(prng.nextLong()));
             int randomScale = prng.nextInt(Integer.MAX_VALUE >> 1) + Integer.MAX_VALUE >> 1;
             BigDecimal randomDecimal = new BigDecimal(randomMantissa, randomScale);
-            return ByteSource.peekable(DecimalType.instance.asComparableBytes(DecimalType.instance.decompose(randomDecimal), LATEST));
+            return ByteSource.peekable(DecimalType.instance.asComparableBytes(DecimalType.instance.decompose(randomDecimal), version));
         });
     }
 
-    private Random prng = new Random();
-
-    @Param({"32", "128", "512"})
-    private int length;
-
-    @Param({"UTF8Type", "BytesType", "IntegerType", "DecimalType"})
-    private String abstractTypeName;
-
-    private AbstractType abstractType;
-    private BiFunction<Random, Integer, ByteSource.Peekable> peekableGenerator;
-
     @Setup(Level.Trial)
     public void setup()
     {
@@ -135,6 +135,6 @@ public int baseline()
     public ByteBuffer fromComparableBytes()
     {
         ByteSource.Peekable peekableBytes = randomPeekableBytes();
-        return abstractType.fromComparableBytes(peekableBytes, ByteComparable.Version.OSS50);
+        return abstractType.fromComparableBytes(peekableBytes, version);
     }
 }
diff --git a/test/microbench/org/apache/cassandra/test/microbench/BaseCompactionBench.java b/test/microbench/org/apache/cassandra/test/microbench/BaseCompactionBench.java
new file mode 100644
index 000000000000..85d17911cac8
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/BaseCompactionBench.java
@@ -0,0 +1,461 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.ImmutableSet;
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSetFuture;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import static org.apache.cassandra.utils.JVMStabilityInspector.removeShutdownHooks;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 5, time = 1)
+@Measurement(iterations = 10, time = 1)
+@Fork(1)
+@Threads(1)
+@State(Scope.Benchmark)
+public abstract class BaseCompactionBench extends CQLTester
+{
+    @Param( {"DEFAULT"} )
+    DataBuilderHelper dataBuilder;
+
+    @Param( {"1", "2", "4"} )
+    int compactors;
+
+    @Param( {"1", "2", "4"} )
+    int size;
+
+    @Param( {"2"} )
+    int sstableCount;
+
+    @Param( {"0"} )
+    int compactionMbSecThrottle;
+
+    @Param("false")
+    boolean compression;
+
+    @Param("1.0")
+    double overlapRatio;
+
+    ColumnFamilyStore cfs;
+
+    protected long rowsPerSSTable;
+    protected long mergedRows;
+
+    @Setup(Level.Trial)
+    public final void setup() throws Throwable
+    {
+        if (sstableCount != 2)
+            throw new IllegalArgumentException("Not implemented yet");
+        if (compactors < 1)
+            throw new IllegalArgumentException();
+        if (size < 1)
+            throw new IllegalArgumentException();
+
+        prepareServer();
+
+        String keyspaceName = createKeyspace("CREATE KEYSPACE %s with replication = " +
+                "{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }" +
+                " and durable_writes = false");
+
+        String createTableStatement =
+                String.format("%s with compaction = { 'class' : '%s' } and compression = %s",
+                              dataBuilder.createTableStatement(),
+                              compactionClass(),
+                              (compression ? "{ 'sstable_compression' : 'LZ4Compressor' }" : "{ 'enabled' : false }")
+                              );
+        String tableName = createTable(keyspaceName, createTableStatement);
+
+        execute("use " + keyspaceName + ";");
+        String writeStatement = dataBuilder.writeStatement(tableName);
+
+        // we need this to use prepared statements
+        requireNetwork();
+        try (Session session = sessionNet())
+        {
+            session.execute("use " + keyspaceName + ";");
+
+            PreparedStatement write = session.prepare(writeStatement);
+            Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+            cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName);
+            cfs.disableAutoCompaction();
+
+            List<ResultSetFuture> futures = new ArrayList<>(256);
+            rowsPerSSTable = this.size * 100000;
+
+            System.err.print("\nWriting " + this.size + "00k");
+            cfs.unsafeRunWithoutFlushing(()-> writeSSTable(session, write, tableName, futures, rowsPerSSTable, 0));
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+            long start = (long) ((1.0 - overlapRatio) * rowsPerSSTable);
+
+            System.err.print("Writing " + this.size + "00k with overlapRatio=" + overlapRatio + " start=" + start);
+            cfs.unsafeRunWithoutFlushing(()-> writeSSTable(session, write, tableName, futures, rowsPerSSTable, start));
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+            mergedRows = start + rowsPerSSTable - dataBuilder.deletions(start + rowsPerSSTable);
+            if (cfs.getLiveSSTables().size() != sstableCount)
+                throw new IllegalStateException("Should have " + sstableCount + " SSTables");
+            StorageService.instance.setConcurrentCompactors(compactors);
+            DatabaseDescriptor.setCompactionThroughputMebibytesPerSec(compactionMbSecThrottle);
+            DatabaseDescriptor.setAutoSnapshot(false);
+        }
+    }
+
+    // overridden by CompactionBenchmark
+    public String compactionClass()
+    {
+        return SizeTieredCompactionStrategy.class.getSimpleName();
+    }
+
+    protected void writeSSTable(
+            Session session,
+            PreparedStatement write,
+            String tableName,
+            List<ResultSetFuture> futures,
+            long size,
+            long offset)
+    {
+        long startNs = System.nanoTime();
+        long percent = size / 100;
+        for (long i = offset; i < offset + size; i++)
+        {
+            futures.add(session.executeAsync(dataBuilder.bindWrite(session, tableName, write, i)));
+            if (futures.size() == 256)
+            {
+                FBUtilities.waitOnFutures(futures);
+                futures.clear();
+            }
+            if ((i - offset) % percent == 0)
+            {
+                System.err.print(".");
+            }
+
+        }
+        FBUtilities.waitOnFutures(futures);
+        futures.clear();
+        long doneWriting = System.nanoTime();
+        System.err.printf(" - done in %.3f seconds\n", (doneWriting - startNs) / 1_000_000_000.0);
+    }
+
+    @TearDown(Level.Invocation)
+    public final void teardownInvocation() throws IOException
+    {
+        // Tests are currently done without clearing cache to better show processing performance differences.
+        // ChunkCache.instance.enable(true);// drops the cache
+    }
+
+
+    @TearDown(Level.Trial)
+    public void teardown() throws Throwable
+    {
+        removeShutdownHooks();
+        tearDownClass();
+        cleanup();
+    }
+
+    // checkstyle: suppress below 'blockSystemPropertyUsage'
+    static final ByteBuffer BLOB = ByteBuffer.allocate(Integer.getInteger("benchmark.blobSize", 100)); // checkstyle: ignore
+    static final int FIELD_COUNT = Integer.getInteger("benchmark.fieldCount", 10);
+    static final int ROWS_PER_PARTITION = Integer.getInteger("benchmark.rowsPerPartition", 100);
+    static final int TOMBSTONE_FREQUENCY = Integer.getInteger("benchmark.fieldCount", 10);
+
+    public enum DataBuilderHelper
+    {
+        DEFAULT,
+        BLOB_CLUSTER_KEY {
+            @Override
+            protected String createTableStatement()
+            {
+                return "CREATE TABLE %s ( userid bigint, picid blob, commentid bigint, PRIMARY KEY(userid, picid))";
+            }
+
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                return write.bind(i, BLOB.duplicate(), i);
+            }
+        },
+        BLOB_VALUE{
+            @Override
+            protected String createTableStatement()
+            {
+                return "CREATE TABLE %s ( userid bigint, picid bigint, commentid blob, PRIMARY KEY(userid, picid))";
+            }
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                return write.bind(i, i, BLOB.duplicate());
+            }
+
+        },
+        MANY_CLUSTER_KEYS{
+            @Override
+            protected String createTableStatement()
+            {
+                String table = "CREATE TABLE %s ( userid bigint, ";
+                for (int i= 0; i < FIELD_COUNT; i++)
+                    table += "ck" + i +" bigint, ";
+                table += " commentid bigint, PRIMARY KEY(userid";
+                for (int i= 0; i < FIELD_COUNT; i++)
+                    table += ", ck" + i;
+                table += "))";
+                return table;
+            }
+
+            @Override
+            protected String writeStatement(String tableName)
+            {
+                String write = "INSERT INTO " + tableName + " (userid";
+                for (int i= 0; i < FIELD_COUNT; i++)
+                    write += ", ck" + i;
+
+                write += ", commentid) VALUES (?, ?";
+                for (int i= 0; i < FIELD_COUNT; i++)
+                    write += ", ?";
+                write += ")";
+                return write;
+            }
+
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                Long[] values = new Long[FIELD_COUNT +2];
+                Arrays.fill(values, i);
+                return write.bind((Object[])values);
+            }
+        },
+        MANY_FIELDS {
+            @Override
+            protected String createTableStatement()
+            {
+                String table = "CREATE TABLE %s ( userid bigint, picid bigint, ";
+                for (int i= 0; i < FIELD_COUNT; i++)
+                    table += "v" + i +" bigint, ";
+                table += " PRIMARY KEY(userid, picid))";
+                return table;
+            }
+
+            @Override
+            protected String writeStatement(String tableName)
+            {
+                String write = "INSERT INTO " + tableName + " (userid, picid";
+                for (int i= 0; i < FIELD_COUNT; i++)
+                    write += ", v" + i;
+
+                write += ") VALUES (?, ?";
+                for (int i= 0; i < FIELD_COUNT; i++)
+                    write += ", ?";
+                write += ")";
+                return write;
+            }
+
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                Long[] values = new Long[FIELD_COUNT +2];
+                Arrays.fill(values, i);
+                return write.bind((Object[]) values);
+            }
+        },
+        WIDE_PARTITIONS {
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                return write.bind(i / ROWS_PER_PARTITION, i % ROWS_PER_PARTITION, i);
+            }
+
+        },
+        COMPLEX_COLUMNS_INSERT {
+            Random rand = new Random(1);
+
+            @Override
+            protected String createTableStatement()
+            {
+                return "CREATE TABLE %s ( userid bigint, picid bigint, comments set<bigint>,  PRIMARY KEY(userid, picid))";
+            }
+
+            @Override
+            protected String writeStatement(String tableName)
+            {
+                return "INSERT INTO " + tableName + " (userid, picid, comments) VALUES (?, ?, ?)";
+            }
+
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                return write.bind(i, i, ImmutableSet.of(i + rand.nextInt(2), i + 2 + rand.nextInt(3)));
+            }
+        },
+        COMPLEX_COLUMNS_UPDATE_SET {
+            Random rand = new Random(1);
+
+            @Override
+            protected String createTableStatement()
+            {
+                return "CREATE TABLE %s ( userid bigint, picid bigint, comments set<bigint>,  PRIMARY KEY(userid, picid))";
+            }
+
+            @Override
+            protected String writeStatement(String tableName)
+            {
+                return "UPDATE " + tableName + " SET comments = ? WHERE userid = ? AND picid = ?";
+            }
+
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                return write.bind(ImmutableSet.of(i + rand.nextInt(2), i + 2 + rand.nextInt(3)), i, i);
+            }
+        },
+        COMPLEX_COLUMNS_UPDATE_ADD {
+            Random rand = new Random(1);
+
+            @Override
+            protected String createTableStatement()
+            {
+                return "CREATE TABLE %s ( userid bigint, picid bigint, comments set<bigint>,  PRIMARY KEY(userid, picid))";
+            }
+
+            @Override
+            protected String writeStatement(String tableName)
+            {
+                return "UPDATE " + tableName + " SET comments = comments + ? WHERE userid = ? AND picid = ?";
+            }
+
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                return write.bind(ImmutableSet.of(i + rand.nextInt(2), i + 2 + rand.nextInt(3)), i, i);
+            }
+        },
+        TOMBSTONES {
+            BitSet deletions = new BitSet();
+            Random rand = new Random(1);
+
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                switch (rand.nextInt(TOMBSTONE_FREQUENCY * 3))
+                {
+                    case 0: // partition tombstone
+                        deletions.set((int) i);
+                        return session.prepare("DELETE FROM " + tableName + " WHERE userid = ?").bind(i);
+                    case 1: // row tombstone
+                        deletions.clear((int) i);   // this still gets reported as a row (empty, with tombstone)
+                        return session.prepare("DELETE FROM " + tableName + " WHERE userid = ? AND picid = ?").bind(i, i);
+                    case 2: // range tombstone
+                        deletions.set((int) i);
+                        return session.prepare("DELETE FROM " + tableName + " WHERE userid = ? AND picid >= ? AND picid < ?").bind(i, i - 2, i + 2);
+                    default:
+                        deletions.clear((int) i);
+                        return super.bindWrite(session, tableName, write, i);
+                }
+            }
+
+            @Override
+            protected long deletions(long max)
+            {
+                return deletions.cardinality();
+            }
+        },
+        TOMBSTONES_WIDE {
+            BitSet deletions = new BitSet();
+            Random rand = new Random(1);
+
+            @Override
+            protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+            {
+                switch (rand.nextInt(TOMBSTONE_FREQUENCY * 2))
+                {
+                    case 0: // row tombstone
+                        deletions.clear((int) i);   // this still gets reported as a row (empty, with tombstone)
+                        return session.prepare("DELETE FROM " + tableName + " WHERE userid = ? AND picid = ?").bind(1L, i);
+                    case 1: // range tombstone
+                        deletions.set(Math.max(0, (int) i - 2), (int) i + 2);
+                        return session.prepare("DELETE FROM " + tableName + " WHERE userid = ? AND picid >= ? AND picid < ?").bind(1L, i - 2, i + 2);
+                    default:
+                        deletions.clear((int) i);
+                        return write.bind(1L, i, i);
+                }
+            }
+
+            @Override
+            protected long deletions(long max)
+            {
+                deletions.clear((int) max, (int) max + 2);
+                return deletions.cardinality();
+            }
+        }
+        ;
+
+        /** override the following to benchmark different schema or data distribution */
+        protected String writeStatement(String tableName)
+        {
+            return "INSERT INTO " + tableName + " (userid, picid, commentid) VALUES (?, ?, ?)";
+        }
+
+        protected BoundStatement bindWrite(Session session, String tableName, PreparedStatement write, long i)
+        {
+            return write.bind(i, i, i);
+        }
+
+        protected String createTableStatement()
+        {
+            return "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))";
+        }
+
+        protected long deletions(long max)
+        {
+            return 0;
+        }
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java b/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java
index 16dbbda72a0c..aa8b04afd2ad 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java
@@ -116,9 +116,9 @@ public void setup() throws Throwable
         {
             modifications.add((ModificationStatement) prepared.statement);
             parameters.add(Lists.newArrayList(bytes(uniquePartition ? i : 1), bytes(i), bytes(i)));
-            queryOrIdList.add(prepared.rawCQLStatement);
+            queryOrIdList.add(prepared.statement.getRawCQLStatement());
         }
-        bs = new BatchStatement(BatchStatement.Type.UNLOGGED, VariableSpecifications.empty(), modifications, Attributes.none());
+        bs = new BatchStatement(null, BatchStatement.Type.UNLOGGED, VariableSpecifications.empty(), modifications, Attributes.none());
         bqo = BatchQueryOptions.withPerStatementVariables(QueryOptions.DEFAULT, parameters, queryOrIdList);
     }
 
diff --git a/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java b/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java
index 3a0f09fc2a4d..9095de236e98 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java
@@ -42,6 +42,7 @@
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -236,23 +237,23 @@ public void testSetup(String compactionClass, String compressorClass, DiskAccess
         int startTombCount = countTombstoneMarkers(cfs);
         int startRowDeletions = countRowDeletions(cfs);
         int startTableCount = cfs.getLiveSSTables().size();
-        long startSize = SSTableReader.getTotalBytes(cfs.getLiveSSTables());
+        long startSize = CompactionSSTable.getTotalDataBytes(cfs.getLiveSSTables());
         System.out.println("\nCompession: " + cfs.getCompressionParameters().toString());
         System.out.println("Reader " + cfs.getLiveSSTables().iterator().next().getFileDataInput(0).toString());
         if (cacheEnabled)
             System.out.format("Cache size %s requests %,d hit ratio %f\n",
-                FileUtils.stringifyFileSize(ChunkCache.instance.metrics.size.getValue()),
-                ChunkCache.instance.metrics.requests.getCount(),
-                ChunkCache.instance.metrics.hitRate.getValue());
+                FileUtils.stringifyFileSize(ChunkCache.instance.metrics.size()),
+                ChunkCache.instance.metrics.requests(),
+                ChunkCache.instance.metrics.hitRate());
         else
         {
-            assertThat(ChunkCache.instance.metrics.requests.getCount()).as("Chunk cache had requests: %s",
-                                                                           ChunkCache.instance.metrics.requests.getCount())
+            assertThat(ChunkCache.instance.metrics.requests()).as("Chunk cache had requests: %s",
+                                                                           ChunkCache.instance.metrics.requests())
                                                                        .isLessThan(COUNT);
             System.out.println("Cache disabled");
         }
 
-        assertThat(ChunkCache.instance.metrics.missLatency.getCount()).isGreaterThan(0);
+        assertThat(ChunkCache.instance.metrics.missLatency()).isGreaterThan(0);
 
         System.out.println(String.format("Operations completed in %.3fs", (onEndTime - onStartTime) * 1e-3));
         if (!CONCURRENT_COMPACTIONS)
@@ -269,7 +270,7 @@ public void testSetup(String compactionClass, String compressorClass, DiskAccess
         int endTombCount = countTombstoneMarkers(cfs);
         int endRowDeletions = countRowDeletions(cfs);
         int endTableCount = cfs.getLiveSSTables().size();
-        long endSize = SSTableReader.getTotalBytes(cfs.getLiveSSTables());
+        long endSize = CompactionSSTable.getTotalDataBytes(cfs.getLiveSSTables());
 
         System.out.println(String.format("Major compaction completed in %.3fs",
                 (endTime - startTime) * 1e-3));
diff --git a/test/microbench/org/apache/cassandra/test/microbench/CollectionContainsBench.java b/test/microbench/org/apache/cassandra/test/microbench/CollectionContainsBench.java
new file mode 100644
index 000000000000..c242d5a15011
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/CollectionContainsBench.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.utils.AbstractTypeGenerators;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport;
+import static org.quicktheories.QuickTheory.qt;
+
+/**
+ * Benchmarks {@link org.apache.cassandra.cql3.Operator#CONTAINS} and {@link org.apache.cassandra.cql3.Operator#CONTAINS_KEY}
+ * comparing calls to {@link CollectionType#contains(ByteBuffer, ByteBuffer)} to the full collection deserialization
+ * followed by a call to {@link java.util.Collection#contains(Object)} that was done before CNDB-11760.
+ */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 1, time = 1) // seconds
+@Measurement(iterations = 3, time = 1) // seconds
+@Fork(value = 4)
+@Threads(4)
+@State(Scope.Benchmark)
+public class CollectionContainsBench
+{
+    @Param({ "INT", "TEXT" })
+    public String type;
+
+    @Param({ "1", "10", "100", "1000" })
+    public int collectionSize;
+
+    private ListType<?> listType;
+    private SetType<?> setType;
+    private MapType<?, ?> mapType;
+
+    private ByteBuffer list;
+    private ByteBuffer set;
+    private ByteBuffer map;
+
+    private final List<ByteBuffer> values = new ArrayList<>();
+
+    @Setup(Level.Trial)
+    public void setup() throws Throwable
+    {
+        AbstractType<?> elementsType = CQL3Type.Native.valueOf(type).getType();
+        setup(elementsType);
+    }
+
+    private <T> void setup(AbstractType<T> elementsType)
+    {
+        ListType<T> listType = ListType.getInstance(elementsType, false);
+        SetType<T> setType = SetType.getInstance(elementsType, false);
+        MapType<T, T> mapType = MapType.getInstance(elementsType, elementsType, false);
+
+        List<T> listValues = new ArrayList<>();
+        Set<T> setValues = new HashSet<>();
+        Map<T, T> mapValues = new HashMap<>();
+
+        AbstractTypeGenerators.TypeSupport<T> support = getTypeSupport(elementsType);
+        qt().withExamples(collectionSize).forAll(support.valueGen).checkAssert(value -> {
+            listValues.add(value);
+            setValues.add(value);
+            mapValues.put(value, value);
+        });
+
+        list = listType.decompose(listValues);
+        set = setType.decompose(setValues);
+        map = mapType.decompose(mapValues);
+
+        this.listType = listType;
+        this.setType = setType;
+        this.mapType = mapType;
+
+        qt().withExamples(100).forAll(support.bytesGen()).checkAssert(values::add);
+    }
+
+    @Benchmark
+    public Object listContainsNonDeserializing()
+    {
+        return test(v -> listType.contains(list, v));
+    }
+
+    @Benchmark
+    public Object listContainsDeserializing()
+    {
+        return test(v -> listType.compose(list).contains(listType.getElementsType().compose(v)));
+    }
+
+    @Benchmark
+    public Object setContainsNonDeserializing()
+    {
+        return test(v -> setType.contains(set, v));
+    }
+
+    @Benchmark
+    public Object setContainsDeserializing()
+    {
+        return test(v -> setType.compose(set).contains(setType.getElementsType().compose(v)));
+    }
+
+    @Benchmark
+    public Object mapContainsNonDeserializing()
+    {
+        return test(v -> mapType.contains(map, v));
+    }
+
+    @Benchmark
+    public Object mapContainsDeserializing()
+    {
+        return test(v -> mapType.compose(map).containsValue(mapType.getValuesType().compose(v)));
+    }
+
+    @Benchmark
+    public Object mapContainsKeyNonDeserializing()
+    {
+        return test(v -> mapType.containsKey(map, v));
+    }
+
+    @Benchmark
+    public Object mapContainsKeyDeserializing()
+    {
+        return test(v -> mapType.compose(map).containsKey(mapType.getKeysType().compose(v)));
+    }
+
+    private int test(Function<ByteBuffer, Boolean> containsFunction)
+    {
+        int contained = 0;
+        for (ByteBuffer v : values)
+        {
+            if (containsFunction.apply(v))
+                contained++;
+        }
+        return contained;
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
index 8d7e800755cc..798c37058bcb 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
@@ -19,114 +19,40 @@
 package org.apache.cassandra.test.microbench;
 
 
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.*;
+import java.util.Map;
 
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileUtils;
-import org.openjdk.jmh.annotations.*;
+import org.apache.cassandra.db.compaction.CompactionStrategyFactory;
+import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Param;
 
-@BenchmarkMode(Mode.AverageTime)
-@OutputTimeUnit(TimeUnit.MILLISECONDS)
-@Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
-@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
-@Fork(value = 1)
-@Threads(1)
-@State(Scope.Benchmark)
-public class CompactionBench extends CQLTester
+public class CompactionBench extends BaseCompactionBench
 {
-    static String keyspace;
-    String table;
-    String writeStatement;
-    String readStatement;
-    ColumnFamilyStore cfs;
-    List<File> snapshotFiles;
-    List<Descriptor> liveFiles;
+    @Param({"false", "true"})
+    boolean cursors;
 
-    @Setup(Level.Trial)
-    public void setup() throws Throwable
+    @Benchmark
+    public void compactSSTables() throws Throwable
     {
-        CQLTester.prepareServer();
-        keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
-        table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))");
-        execute("use "+keyspace+";");
-        writeStatement = "INSERT INTO "+table+"(userid,picid,commentid)VALUES(?,?,?)";
-        readStatement = "SELECT * from "+table+" limit 100";
-
-        Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
-
-        cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-        cfs.disableAutoCompaction();
-
-        //Warm up
-        System.err.println("Writing 50k");
-        for (long i = 0; i < 50000; i++)
-            execute(writeStatement, i, i, i );
-
-
-        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
-
-        System.err.println("Writing 50k again...");
-        for (long i = 0; i < 50000; i++)
-            execute(writeStatement, i, i, i );
-
-        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
-
-        cfs.snapshot("originals");
-
-        snapshotFiles = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots("originals").listFiles();
+        cfs.forceMajorCompaction();
     }
 
-    @TearDown(Level.Trial)
-    public void teardown() throws IOException, ExecutionException, InterruptedException
+    @Override
+    public String compactionClass()
     {
-        int active = Thread.currentThread().getThreadGroup().activeCount();
-        Thread[] threads = new Thread[active];
-        Thread.currentThread().getThreadGroup().enumerate(threads);
-        for (Thread t : threads)
-        {
-            if (!t.isDaemon())
-                System.err.println("Thread "+t.getName());
-        }
-
-        CQLTester.cleanup();
+        return cursors ? "SizeTieredCompactionStrategy" : "org.apache.cassandra.test.microbench.CompactionBenchmark$CursorDisabledStrategy";
     }
 
-
-    @TearDown(Level.Invocation)
-    public void resetSnapshot()
+    public static class CursorDisabledStrategy extends SizeTieredCompactionStrategy
     {
-        cfs.truncateBlocking();
-
-        List<File> directories = cfs.getDirectories().getCFDirectories();
-
-        for (File file : directories)
+        public CursorDisabledStrategy(CompactionStrategyFactory factory, Map<String, String> options)
         {
-            for (File f : file.tryList())
-            {
-                if (f.isDirectory())
-                    continue;
-
-                FileUtils.delete(f);
-            }
+            super(factory, options);
         }
 
-
-        for (File file : snapshotFiles)
-            FileUtils.createHardLink(file, new File(new File(file.toPath().getParent().getParent().getParent()), file.name()));
-
-        cfs.loadNewSSTables();
-    }
-
-    @Benchmark
-    public void compactTest() throws Throwable
-    {
-        cfs.forceMajorCompaction();
+        public boolean supportsCursorCompaction()
+        {
+            return false;
+        }
     }
 }
diff --git a/test/microbench/org/apache/cassandra/test/microbench/CompactionBreakdownBench.java b/test/microbench/org/apache/cassandra/test/microbench/CompactionBreakdownBench.java
new file mode 100644
index 000000000000..b1ed48733d9a
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/CompactionBreakdownBench.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.test.microbench;
+
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionCursor;
+import org.apache.cassandra.db.compaction.CompactionIterator;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionStrategy;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.compaction.writers.DefaultCompactionWriter;
+import org.apache.cassandra.db.compaction.writers.SSTableDataSink;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.ScannerList;
+import org.apache.cassandra.io.sstable.compaction.IteratorFromCursor;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursor;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursorMerger;
+import org.apache.cassandra.io.sstable.compaction.SortedStringTableCursor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+import org.mockito.Mockito;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.infra.Blackhole;
+
+public class CompactionBreakdownBench extends BaseCompactionBench
+{
+    @Benchmark
+    public void iterateThroughTableScanner(Blackhole bh) throws Throwable
+    {
+        long totalRows = 0;
+        Iterable<SSTableReader> ssTables = cfs.getSSTables(SSTableSet.LIVE);
+        for (SSTableReader reader : ssTables)
+        {
+            try (ISSTableScanner scanner = reader.getScanner())
+            {
+                totalRows = consumePartitionIterator(scanner, bh, totalRows);
+            }
+        }
+        if (totalRows != sstableCount * rowsPerSSTable)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + rowsPerSSTable * sstableCount);
+    }
+
+    @Benchmark
+    public void iterateThroughCursorToIterator(Blackhole bh) throws Throwable
+    {
+        long totalRows = 0;
+        Iterable<SSTableReader> ssTables = cfs.getSSTables(SSTableSet.LIVE);
+        for (SSTableReader reader : ssTables)
+        {
+            try (UnfilteredPartitionIterator iter = new IteratorFromCursor(reader.metadata(), new SortedStringTableCursor(reader)))
+            {
+                totalRows = consumePartitionIterator(iter, bh, totalRows);
+            }
+        }
+        if (totalRows != sstableCount * rowsPerSSTable)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + rowsPerSSTable * sstableCount);
+    }
+
+    @Benchmark
+    public void iterateThroughCursor(Blackhole bh) throws Throwable
+    {
+        long totalRows = 0;
+        Iterable<SSTableReader> ssTables = cfs.getSSTables(SSTableSet.LIVE);
+        for (SSTableReader reader : ssTables)
+        {
+            try (SSTableCursor cursor = new SortedStringTableCursor(reader))
+            {
+                totalRows = consumeCursor(cursor, bh, totalRows);
+            }
+        }
+        if (totalRows != sstableCount * rowsPerSSTable)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + rowsPerSSTable * sstableCount);
+    }
+
+    @Benchmark
+    public void iterateThroughMergeCursor(Blackhole bh) throws Throwable
+    {
+        long totalRows = 0;
+        Set<SSTableReader> ssTables = cfs.getLiveSSTables();
+        try (SSTableCursor cursor = new SSTableCursorMerger(ssTables.stream()
+                                                                    .map(SortedStringTableCursor::new)
+                                                                    .collect(Collectors.toList()),
+                                                            cfs.metadata());
+        )
+        {
+            totalRows = consumeCursor(cursor, bh, totalRows);
+        }
+
+        if (totalRows != mergedRows)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + rowsPerSSTable * sstableCount);
+    }
+
+    @Benchmark
+    public void iterateThroughPartitionIndexIterator() throws Throwable
+    {
+        long totalRows = 0;
+        Iterable<SSTableReader> ssTables = cfs.getSSTables(SSTableSet.LIVE);
+        for (SSTableReader reader : ssTables)
+        {
+            try (KeyIterator partitionIndexIterator = reader.keyIterator())
+            {
+
+                long start = partitionIndexIterator.getBytesRead();
+                long end = partitionIndexIterator.getBytesRead();
+                while (partitionIndexIterator.hasNext())
+                {
+                    totalRows++;
+                    start = end;
+                    partitionIndexIterator.next();
+                    end = partitionIndexIterator.getBytesRead();
+                }
+            }
+        }
+        if (totalRows != sstableCount * rowsPerSSTable)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + rowsPerSSTable * sstableCount);
+    }
+
+    @Benchmark
+    public void iterateThroughMergeIterator(Blackhole bh) throws Throwable
+    {
+        long totalRows = 0;
+        Set<SSTableReader> ssTables = cfs.getLiveSSTables();
+
+        try (UnfilteredPartitionIterator mergedScanner = UnfilteredPartitionIterators.merge(ssTables.stream()
+                                                                                                    .map(SSTableReader::getScanner)
+                                                                                                    .collect(Collectors.toList()),
+                                                                                            UnfilteredPartitionIterators.MergeListener.NOOP))
+        {
+            totalRows = consumePartitionIterator(mergedScanner, bh, totalRows);
+        }
+
+        // this is assuming
+        if (totalRows != mergedRows)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + mergedRows);
+    }
+
+    @Benchmark
+    public void iterateThroughCompactionIterator(Blackhole bh) throws Throwable
+    {
+        long totalRows = 0;
+        Iterable<SSTableReader> ssTables = cfs.getSSTables(SSTableSet.LIVE);
+        Set<SSTableReader> compacting = ImmutableSet.copyOf(ssTables);
+        CompactionStrategy strategy = cfs.getCompactionStrategy();
+        CompactionController controller = new CompactionController(cfs, compacting, CompactionManager.NO_GC);
+
+        try (ScannerList scanners = strategy.getScanners(compacting);
+             CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, FBUtilities.nowInSeconds(), TimeUUID.Generator.nextTimeUUID()))
+        {
+            totalRows = consumePartitionIterator(ci, bh, totalRows);
+        }
+
+        // this is assuming
+        if (totalRows != mergedRows)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + mergedRows);
+    }
+
+    @Benchmark
+    public void iterateThroughCompactionCursor(Blackhole bh) throws Throwable
+    {
+        long totalRows = 0;
+        Iterable<SSTableReader> ssTables = cfs.getLiveSSTables();
+        Set<SSTableReader> compacting = ImmutableSet.copyOf(ssTables);
+        CompactionController controller = new CompactionController(cfs, compacting, CompactionManager.NO_GC);
+
+        try (CompactionCursor ci = new CompactionCursor(OperationType.COMPACTION,
+                                                        compacting,
+                                                        null,
+                                                        controller,
+                                                        RateLimiter.create(Double.MAX_VALUE),
+                                                        FBUtilities.nowInSeconds()))
+        {
+            totalRows = consumeCompactionCursor(ci, bh, totalRows);
+        }
+
+        // this is assuming
+        if (totalRows != mergedRows)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + mergedRows);
+    }
+
+    @Benchmark
+    public void iterateThroughCompactionCursorWithLimiter(Blackhole bh) throws Throwable
+    {
+        long totalRows = 0;
+        Iterable<SSTableReader> ssTables = cfs.getLiveSSTables();
+        Set<SSTableReader> compacting = ImmutableSet.copyOf(ssTables);
+        CompactionController controller = new CompactionController(cfs, compacting, CompactionManager.NO_GC);
+
+        try (CompactionCursor ci = new CompactionCursor(OperationType.COMPACTION,
+                                                        compacting,
+                                                        null,
+                                                        controller,
+                                                        RateLimiter.create(Double.MAX_VALUE),
+                                                        FBUtilities.nowInSeconds()))
+        {
+            totalRows = consumeCompactionCursor(ci, bh, totalRows);
+        }
+
+        // this is assuming
+        if (totalRows != mergedRows)
+            throw new IllegalStateException("Total rows is: " + totalRows + " but should be: " + mergedRows);
+    }
+
+    public long consumePartitionIterator(Iterator<UnfilteredRowIterator> partitionIterator, Blackhole bh, long totalRows)
+    {
+        while (partitionIterator.hasNext())
+        {
+            UnfilteredRowIterator partition = partitionIterator.next();
+            while (partition.hasNext())
+            {
+                Unfiltered row = partition.next();
+                bh.consume(row);
+                if (row.isRow())
+                    totalRows++;
+            }
+        }
+        return totalRows;
+    }
+
+    private long consumeCursor(SSTableCursor cursor, Blackhole bh, long totalRows) throws IOException
+    {
+        while (true)
+        {
+            switch (cursor.advance())
+            {
+                case EXHAUSTED:
+                    return totalRows;
+                case ROW:
+                    ++totalRows;
+                default:
+                    // skip headers
+            }
+        }
+    }
+
+    private long consumeCompactionCursor(CompactionCursor cursor, Blackhole bh, long totalRows) throws IOException
+    {
+        class Sink implements SSTableDataSink {
+            long rows = 0;
+
+            public AbstractRowIndexEntry append(UnfilteredRowIterator partition)
+            {
+                return Mockito.mock(AbstractRowIndexEntry.class);
+            }
+
+            public boolean startPartition(DecoratedKey partitionKey, DeletionTime deletionTime) throws IOException
+            {
+                // nothing
+                return true;
+            }
+
+            public AbstractRowIndexEntry endPartition() throws IOException
+            {
+                // nothing
+                return null;
+            }
+
+            public void addUnfiltered(Unfiltered unfiltered) throws IOException
+            {
+                if (unfiltered.isRow())
+                    ++rows;
+            }
+        };
+
+        Sink sink = new Sink();
+        while (cursor.copyOne(sink) != SSTableCursor.Type.EXHAUSTED)
+        {}
+
+        return sink.rows + totalRows;
+    }
+
+    @Benchmark
+    public void scannerToCompactionWriter()
+    {
+
+        Iterable<SSTableReader> ssTables = cfs.getSSTables(SSTableSet.LIVE);
+        final SSTableReader ssTableReader = ssTables.iterator().next();
+        Set<SSTableReader> compacting = ImmutableSet.copyOf(cfs.getSSTables(SSTableSet.LIVE));
+
+        try (LifecycleTransaction transaction = cfs.getTracker().tryModify(compacting, OperationType.COMPACTION);
+             CompactionAwareWriter writer = new DefaultCompactionWriter(cfs, cfs.getDirectories(), transaction, compacting);)
+        {
+            try (final ISSTableScanner scanner = ssTableReader.getScanner())
+            {
+                while (scanner.hasNext())
+                {
+                    writer.append(scanner.next());
+                }
+            }
+            writer.finish();
+        }
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java b/test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java
index cd905e10d35a..cb8b38c5f074 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java
@@ -29,18 +29,18 @@
 import java.util.function.Predicate;
 
 import com.google.common.collect.Iterables;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.junit.Assert;
-
 import org.apache.cassandra.config.Config.CommitLogSync;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -56,6 +56,7 @@ public class GcCompactionBenchTest extends CQLTester
 {
     private static final String SIZE_TIERED_STRATEGY = "SizeTieredCompactionStrategy', 'min_sstable_size' : '0";
     private static final String LEVELED_STRATEGY = "LeveledCompactionStrategy', 'sstable_size_in_mb' : '16";
+    private static final String UNIFIED_STRATEGY = "UnifiedCompactionStrategy', 'min_sstable_size_in_mb' : '16";
 
     private static final int DEL_SECTIONS = 1000;
     private static final int FLUSH_FREQ = 10000;
@@ -236,7 +237,7 @@ public void testGcCompaction(TombstoneOption tombstoneOption, TombstoneOption ba
         int startRowDeletions = countRowDeletions(cfs);
         int startTableCount = cfs.getLiveSSTables().size();
         int startTableMaxLevel = cfs.getLiveSSTables().stream().mapToInt(SSTableReader::getSSTableLevel).max().orElseGet(() -> 0);
-        long startSize = SSTableReader.getTotalBytes(cfs.getLiveSSTables());
+        long startSize = CompactionSSTable.getTotalDataBytes(cfs.getLiveSSTables());
         System.out.println();
 
         String hashesBefore = getHashes();
@@ -250,7 +251,7 @@ public void testGcCompaction(TombstoneOption tombstoneOption, TombstoneOption ba
         int endRowDeletions = countRowDeletions(cfs);
         int endTableCount = cfs.getLiveSSTables().size();
         int endTableMaxLevel = cfs.getLiveSSTables().stream().mapToInt(SSTableReader::getSSTableLevel).max().orElseGet(() -> 0);
-        long endSize = SSTableReader.getTotalBytes(cfs.getLiveSSTables());
+        long endSize = CompactionSSTable.getTotalDataBytes(cfs.getLiveSSTables());
 
         System.out.println(cfs.getCompactionParametersJson());
         System.out.println(String.format("%s compactions completed in %.3fs",
@@ -307,6 +308,12 @@ public void testCopyCompaction() throws Throwable
         testGcCompaction(TombstoneOption.NONE, TombstoneOption.NONE, LEVELED_STRATEGY);
     }
 
+    @Test
+    public void testCopyCompactionUCS() throws Throwable
+    {
+        testGcCompaction(TombstoneOption.NONE, TombstoneOption.NONE, UNIFIED_STRATEGY);
+    }
+
     @Test
     public void testCellAtEndSizeTiered() throws Throwable
     {
diff --git a/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java b/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java
index 85cffafb9864..53359be29faa 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java
@@ -152,4 +152,4 @@ public ClusteringBoundary<?> nextClusteringBoundary()
     {
         return clusteringBoundaries[datumIndex++ & (clusteringBoundaries.length - 1)];
     }
-}
\ No newline at end of file
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/PartialSortingBench.java b/test/microbench/org/apache/cassandra/test/microbench/PartialSortingBench.java
new file mode 100644
index 000000000000..d169b38d4945
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/PartialSortingBench.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.Comparators;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.MinMaxPriorityQueue;
+import com.google.common.collect.Ordering;
+
+import org.apache.cassandra.utils.SortingIterator;
+import org.apache.cassandra.utils.TopKSelector;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(1)
+@Warmup(iterations = 5, time = 3)
+@Measurement(iterations = 10, time = 3)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@BenchmarkMode(Mode.AverageTime)
+@State(Scope.Thread)
+public class PartialSortingBench
+{
+
+    @Param({"100", "1000", "10000"})
+    public int size;
+
+    @Param({"0.1"})
+    double nullChance = 0.1;
+
+    @Param({"0", "0.1", "1"})
+    double consumeRatio = 0.1;
+
+    @Param({"0", "100"})
+    int comparatorSlowDown = 0;
+
+    public List<Integer> data;
+
+    @Setup(Level.Trial)
+    public void setUp() {
+        Random random = new Random();
+        data = new ArrayList<>(size * 100);
+        for (int i = 0; i < size * 100; i++) {
+            data.add(random.nextDouble() < nullChance ? null : random.nextInt());
+        }
+        comparator = comparatorSlowDown <= 0 ? Comparator.naturalOrder()
+                                             : (x, y) ->
+                                               {
+                                                   Blackhole.consumeCPU(comparatorSlowDown);
+                                                   return Integer.compare(x, y);
+                                               };
+        comparatorNulls = (a, b) -> {
+            if (a == null || b == null)
+                return b != null ? 1 : a != null ? -1 : 0;
+            return comparator.compare(a, b);
+        };
+    }
+
+    Comparator<Integer> comparator;
+    Comparator<Integer> comparatorNulls;
+
+    @Benchmark
+    public void testSortingIterator(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        Iterator<Integer> iterator = SortingIterator.create(comparator, integers);
+        int i = (int) Math.ceil(consumeRatio * size + 0.01);
+        while (iterator.hasNext() && i-- > 0) {
+            bh.consume(iterator.next());
+        }
+    }
+
+    @Benchmark
+    public void testArrayListSortIterator(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        var al = new ArrayList<Integer>(data.size());
+        for (Integer item : integers)
+            if (item != null)
+                al.add(item);
+        al.sort(comparator);
+        var iterator = al.iterator();
+        int i = (int) Math.ceil(consumeRatio * size + 0.01);
+        while (iterator.hasNext() && i-- > 0) {
+            bh.consume(iterator.next());
+        }
+    }
+
+    @Benchmark
+    public void testArraySortIterator(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        Integer[] al = new Integer[data.size()];
+        int sz = 0;
+        for (Integer item : integers)
+            if (item != null)
+                al[sz++] = item;
+        Arrays.sort(al, 0, sz, comparator);
+        var iterator = Iterators.forArray(al);
+
+        int i = (int) Math.ceil(consumeRatio * size + 0.01);
+        while (iterator.hasNext() && i-- > 0) {
+            bh.consume(iterator.next());
+        }
+    }
+
+    @Benchmark
+    public void testStreamIterator(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        int limit = (int) Math.ceil(consumeRatio * size + 0.01);
+        var iterator = integers.stream().filter(Predicates.notNull()).sorted(comparator).limit(limit).iterator();
+        while (iterator.hasNext()) {
+            bh.consume(iterator.next());
+        }
+    }
+
+    @Benchmark
+    public void testPriorityQueuePreLimit(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        int limit = (int) Math.ceil(consumeRatio * size + 0.01);
+        var pq = new PriorityQueue<Integer>(limit + 1, comparator.reversed());
+        for (Integer i : integers)
+        {
+            if (i == null)
+                continue;
+            pq.add(i);
+            if (pq.size() > limit)
+                pq.poll();
+        }
+        limit = pq.size(); // less if close to size with nulls
+        Integer[] values = new Integer[limit];
+        for (int i = limit - 1; i >= 0; --i)
+            values[i] = pq.poll();
+        var iterator = Iterators.forArray(values);
+        while (iterator.hasNext()) {
+            bh.consume(iterator.next());
+        }
+    }
+
+    @Benchmark
+    public void testLucenePriorityQueuePreLimit(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        int limit = (int) Math.ceil(consumeRatio * size + 0.01);
+        var pq = new org.apache.lucene.util.PriorityQueue<Integer>(limit)
+        {
+            @Override
+            protected boolean lessThan(Integer t, Integer t1)
+            {
+                return t > t1;
+            }
+        };
+        for (Integer i : integers)
+        {
+            if (i == null)
+                continue;
+            pq.insertWithOverflow(i);
+        }
+        limit = pq.size(); // less if close to size with nulls
+        Integer[] values = new Integer[limit];
+        for (int i = limit - 1; i >= 0; --i)
+            values[i] = pq.pop();
+        var iterator = Iterators.forArray(values);
+        while (iterator.hasNext()) {
+            bh.consume(iterator.next());
+        }
+    }
+
+    @Benchmark
+    public void testMinMaxPQPreLimit(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        int limit = (int) Math.ceil(consumeRatio * size + 0.01);
+        var pq = MinMaxPriorityQueue.orderedBy(comparatorNulls).maximumSize(limit).create();
+        for (Integer i : integers)
+        {
+            if (i == null)
+                continue;
+            pq.offer(i);
+        }
+        while (!pq.isEmpty()) {
+            bh.consume(pq.poll());
+        }
+    }
+
+    @Benchmark
+    public void testOrderingLeastOf(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        int limit = (int) Math.ceil(consumeRatio * size + 0.01);
+        var iterator = Ordering.from(comparatorNulls).leastOf(integers, limit).iterator();
+        while (iterator.hasNext()) {
+            bh.consume(iterator.next());
+        }
+    }
+
+    @Benchmark
+    public void testCollectLeast(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        int limit = (int) Math.ceil(consumeRatio * size + 0.01);
+        var iterator = integers.stream().collect(Comparators.least(limit, comparatorNulls)).iterator();
+        while (iterator.hasNext()) {
+            bh.consume(iterator.next());
+        }
+    }
+
+    @Benchmark
+    public void testTopKSelector(Blackhole bh) {
+        int startIndex = ThreadLocalRandom.current().nextInt(data.size() - size);
+        List<Integer> integers = data.subList(startIndex, startIndex + size);
+        int limit = (int) Math.ceil(consumeRatio * size + 0.01);
+        var selector = new TopKSelector<>(comparator, limit);
+        selector.addAll(integers);
+        var iterator = selector.getShared().iterator();
+        while (iterator.hasNext()) {
+            bh.consume(iterator.next());
+        }
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/ReadWriteBench.java b/test/microbench/org/apache/cassandra/test/microbench/ReadWriteBench.java
index 06d7c77e1a3b..b30d8be91157 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/ReadWriteBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/ReadWriteBench.java
@@ -25,6 +25,7 @@
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
 import org.openjdk.jmh.annotations.*;
 
 @BenchmarkMode(Mode.Throughput)
@@ -65,6 +66,7 @@ public void setup() throws Throwable
     @TearDown(Level.Trial)
     public void teardown() throws IOException, ExecutionException, InterruptedException
     {
+        CommitLog.instance.shutdownBlocking();
         CQLTester.cleanup();
     }
 
diff --git a/test/microbench/org/apache/cassandra/test/microbench/RequestSensorBench.java b/test/microbench/org/apache/cassandra/test/microbench/RequestSensorBench.java
new file mode 100644
index 000000000000..755f3730e7f3
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/RequestSensorBench.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.IntStream;
+
+import org.apache.commons.math3.distribution.ZipfDistribution;
+
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.ActiveRequestSensors;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.SensorsRegistry;
+import org.apache.cassandra.sensors.Type;
+import org.apache.cassandra.utils.Pair;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@Warmup(iterations = 1)
+@Fork(value = 1)
+@State(Scope.Benchmark)
+public class RequestSensorBench
+{
+    private static final int NUM_SENSORS = 1000;
+    private static final int THREADS = 100;
+    private static final int SENSORS_PER_THREAD = 10;
+    private static final int UPDATES_PER_THREAD = 100;
+    private static final ConcurrentMap<Integer, Pair<Context, Type>> contextFixtures = new ConcurrentHashMap();
+    private static final Fixture[][] fixtures = new Fixture[THREADS][SENSORS_PER_THREAD];
+    private static final RequestSensors[] requestSensorsPool = new RequestSensors[THREADS];
+    private static final Random randomGen = new Random(1234567890);
+    private static final AtomicInteger threadIdx = new AtomicInteger();
+
+    // Zipfian should more realisticly represent workload (few tenants generating most of the load)
+    private static final ZipfDistribution zipfDistributionContext = new ZipfDistribution(NUM_SENSORS - 1, 1);
+
+    private static class Fixture
+    {
+        Context context;
+        Type type;
+
+        Fixture(Context context, Type type)
+        {
+            this.context = context;
+            this.type = type;
+        }
+    }
+
+    @Setup
+    public void generateFixtures()
+    {
+        for (int i = 0; i < NUM_SENSORS; i++)
+        {
+            Context context = new Context("keyspace" + i, "table" + i, UUID.randomUUID().toString());
+            SensorsRegistry.instance.onCreateKeyspace(KeyspaceMetadata.create(context.getKeyspace(), null));
+            SensorsRegistry.instance.onCreateTable(TableMetadata.builder(context.getKeyspace(), context.getTable()).id(TableId.fromString(context.getTableId())).build());
+            contextFixtures.put(i, Pair.create(context, Type.values()[randomGen.nextInt(Type.values().length)]));
+        }
+
+        IntStream.range(0, THREADS).forEach(t -> {
+            requestSensorsPool[t] = new ActiveRequestSensors();
+            Pair<Context, Type> contextTypePair = contextFixtures.get(zipfDistributionContext.sample());
+            IntStream.range(0, SENSORS_PER_THREAD).forEach(s -> fixtures[t][s] = new Fixture(contextTypePair.left, contextTypePair.right));
+        });
+    }
+
+    @State(Scope.Thread)
+    public static class BenchState
+    {
+        int idx = threadIdx.getAndIncrement();
+        RequestSensors requestSensors = requestSensorsPool[idx];
+    }
+
+    @Benchmark
+    @Threads(THREADS)
+    public void syncAllSensors(BenchState benchState)
+    {
+        RequestSensors requestSensors = benchState.requestSensors;
+        for(int i = 0; i < SENSORS_PER_THREAD; i++)
+        {
+            Fixture f = fixtures[benchState.idx][i];
+            requestSensors.registerSensor(f.context, f.type);
+        }
+        for (int i = 0; i < UPDATES_PER_THREAD; i++)
+        {
+            Fixture f = fixtures[benchState.idx][i % SENSORS_PER_THREAD];
+            requestSensors.incrementSensor(f.context, f.type, 1);
+        }
+        requestSensors.syncAllSensors();
+    }
+
+    @Benchmark
+    @Threads(THREADS)
+    public void benchUsingSensorRegistryDirectly(BenchState benchState)
+    {
+        for (int i = 0; i < SENSORS_PER_THREAD; i++)
+        {
+            Fixture f = fixtures[benchState.idx][i];
+            SensorsRegistry.instance.getOrCreateSensor(f.context, f.type);
+        }
+        for (int i = 0; i < UPDATES_PER_THREAD; i++)
+        {
+            Fixture f = fixtures[benchState.idx][i % SENSORS_PER_THREAD];
+            SensorsRegistry.instance.getSensor(f.context, f.type).get().increment(1);
+        }
+    }
+
+    public static void main(String... args) throws Exception
+    {
+        Options options = new OptionsBuilder().include(RequestSensorBench.class.getSimpleName()).build();
+        new Runner(options).run();
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/SortedRowsBuilderBench.java b/test/microbench/org/apache/cassandra/test/microbench/SortedRowsBuilderBench.java
new file mode 100644
index 000000000000..f54435937a15
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/SortedRowsBuilderBench.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.selection.SortedRowsBuilder;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.openjdk.jmh.annotations.*;
+
+/**
+ * Benchmarks each implementation of {@link SortedRowsBuilder}.
+ */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 1, time = 2) // seconds
+@Measurement(iterations = 5, time = 2) // seconds
+@Fork(value = 4)
+@Threads(4)
+@State(Scope.Benchmark)
+public class SortedRowsBuilderBench extends CQLTester
+{
+    private static final int NUM_COLUMNS = 10;
+    private static final int SORTED_COLUMN_CARDINALITY = 1000;
+
+    private static final Comparator<List<ByteBuffer>> COMPARATOR = (o1, o2) -> Int32Type.instance.compare(o1.get(0), o2.get(0));
+    private static final Random RANDOM = new Random();
+
+    @Param({ "1", "2", "3", "4", "5", "6", "8", "16", "32" })
+    public int nodes;
+
+    @Param({ "10", "100", "1000", "10000" })
+    public int limit;
+
+    @Param({ "0", "0.1", "0.5", "1" })
+    public float offsetRatio;
+
+    private int offset;
+
+    private List<List<ByteBuffer>> rows;
+
+    @Setup(Level.Trial)
+    public void setup() throws Throwable
+    {
+        int rowsPerNode = limit + offset;
+        int rowsPerCoordinator = rowsPerNode * nodes;
+        offset = (int) (rowsPerNode * offsetRatio);
+        rows = new ArrayList<>(rowsPerCoordinator);
+        for (int r = 0; r < rowsPerCoordinator; r++)
+        {
+            rows.add(randomRow());
+        }
+    }
+
+    @Benchmark
+    public Object insertion()
+    {
+        return test(SortedRowsBuilder.create(limit, offset));
+    }
+
+    @Benchmark
+    public Object comparatorWithList()
+    {
+        return test(SortedRowsBuilder.WithListSort.create(limit, offset, COMPARATOR));
+    }
+
+    @Benchmark
+    public Object comparatorWithHeap()
+    {
+        return test(SortedRowsBuilder.WithHeapSort.create(limit, offset, COMPARATOR));
+    }
+
+    @Benchmark
+    public Object comparatorWithHybrid()
+    {
+        return test(SortedRowsBuilder.WithHybridSort.create(limit, offset, COMPARATOR));
+    }
+
+    private List<List<ByteBuffer>> test(SortedRowsBuilder builder)
+    {
+        rows.forEach(builder::add);
+        return builder.build();
+    }
+
+    private static List<ByteBuffer> randomRow()
+    {
+        List<ByteBuffer> row = new ArrayList<>(NUM_COLUMNS);
+        for (int c = 0; c < NUM_COLUMNS; c++)
+        {
+            row.add(Int32Type.instance.decompose(RANDOM.nextInt(SORTED_COLUMN_CARDINALITY)));
+        }
+        return row;
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java
index 7083832c012a..20d4897428a3 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java
@@ -52,6 +52,7 @@
 import org.apache.cassandra.db.marshal.MapType;
 import org.apache.cassandra.db.partitions.AtomicBTreePartition;
 import org.apache.cassandra.db.partitions.BTreePartitionData;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdate;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.BTreeRow;
 import org.apache.cassandra.db.rows.BufferCell;
@@ -240,7 +241,7 @@ PartitionUpdate next()
             try (BulkIterator<Row> iter = BulkIterator.of(insertBuffer))
             {
                 Object[] tree = BTree.build(iter, rowCount, UpdateFunction.noOp());
-                return PartitionUpdate.unsafeConstruct(metadata, decoratedKey, BTreePartitionData.unsafeConstruct(partitionColumns, tree, DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS), NO_DELETION_INFO, false);
+                return BTreePartitionUpdate.unsafeConstruct(metadata, decoratedKey, BTreePartitionData.unsafeConstruct(partitionColumns, tree, DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS), NO_DELETION_INFO, false);
             }
         }
 
@@ -316,7 +317,7 @@ Cell cell(ColumnMetadata column, CellPath path)
     private static class Batch
     {
         final AtomicBTreePartition update;
-        final PartitionUpdate[] insert;
+        final BTreePartitionUpdate[] insert;
         // low 20 bits contain the next insert we're performing this generation
         // next 20 bits are inserts we've performed this generation
         // next 24 bits are generation (i.e. number of times we've run this update)
@@ -352,7 +353,7 @@ public ByteBuffer allocate(int size)
             cloner = allocator.cloner(NO_ORDER.getCurrent());
 
             generator.reset();
-            insert = IntStream.range(0, rolloverAfterInserts).mapToObj(i -> generator.next()).toArray(PartitionUpdate[]::new);
+            insert = IntStream.range(0, rolloverAfterInserts).mapToObj(i -> generator.next()).toArray(BTreePartitionUpdate[]::new);
         }
 
         boolean performOne(int ifGeneration, Consumer<Batch> invokeBefore)
diff --git a/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/AbstractTrieMemoryIndexBench.java b/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/AbstractTrieMemoryIndexBench.java
new file mode 100644
index 000000000000..098d3cd08071
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/AbstractTrieMemoryIndexBench.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.index.sai.memory;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.TargetParser;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions;
+import org.apache.cassandra.index.sai.memory.MemoryIndex;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TimeUUID;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Setup;
+
+public abstract class AbstractTrieMemoryIndexBench
+{
+    private static final String KEYSPACE = "test_keyspace";
+    private static final String TABLE = "test_table";
+    private static final String PARTITION_KEY = "key";
+    private static final String STRING_COLUMN = "string";
+    private static final String STRING_INDEX = "string_index";
+    private static final String INTEGER_COLUMN = "integer";
+    private static final String INTEGER_INDEX = "integer_index";
+    private static final int RANDOM_STRING_SIZE = 64 * 1024 * 1024;
+
+    private char[] randomChars = new char[RANDOM_STRING_SIZE];
+
+    protected int randomSeed;
+
+    protected IndexContext stringContext;
+    protected IndexContext integerContext;
+
+    protected MemoryIndex stringIndex;
+    protected MemoryIndex integerIndex;
+
+    protected ByteBuffer[] stringTerms;
+    protected ByteBuffer[] integerTerms;
+    protected DecoratedKey[] partitionKeys;
+
+    @Setup(Level.Trial)
+    public void initialiseConfig()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        Random random = new Random();
+        randomSeed = random.nextInt();
+        for (int i = 0; i < RANDOM_STRING_SIZE; i++)
+        {
+            randomChars[i] = (char)('a' + random.nextInt(26));
+        }
+
+        ColumnMetadata string = ColumnMetadata.regularColumn(KEYSPACE, TABLE, STRING_COLUMN, UTF8Type.instance);
+        ColumnMetadata integer = ColumnMetadata.regularColumn(KEYSPACE, TABLE, INTEGER_COLUMN, Int32Type.instance);
+        TableMetadata table = TableMetadata.builder(KEYSPACE, TABLE)
+                .addPartitionKeyColumn(PARTITION_KEY, UTF8Type.instance)
+                .addRegularColumn(STRING_COLUMN, UTF8Type.instance)
+                .addRegularColumn(INTEGER_COLUMN, Int32Type.instance)
+                .partitioner(Murmur3Partitioner.instance)
+                .caching(CachingParams.CACHE_NOTHING)
+                .build();
+
+        Map<String, String> stringOptions = new HashMap<>();
+        stringOptions.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName());
+        stringOptions.put(NonTokenizingOptions.CASE_SENSITIVE, "true");
+        stringOptions.put(IndexTarget.TARGET_OPTION_NAME, STRING_COLUMN);
+
+        Map<String, String> integerOptions = new HashMap<>();
+        integerOptions.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName());
+        integerOptions.put(IndexTarget.TARGET_OPTION_NAME, INTEGER_COLUMN);
+
+        IndexMetadata stringMetadata = IndexMetadata.fromSchemaMetadata(STRING_INDEX, IndexMetadata.Kind.CUSTOM, stringOptions);
+        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(table, stringMetadata);
+        stringContext = new IndexContext(table.keyspace,
+                                         table.name,
+                                         table.id,
+                                         table.partitionKeyType,
+                                         table.comparator,
+                                         target.left,
+                                         target.right,
+                                         stringMetadata,
+                                         MockSchema.newCFS(table));
+
+        IndexMetadata integerMetadata = IndexMetadata.fromSchemaMetadata(INTEGER_INDEX, IndexMetadata.Kind.CUSTOM, integerOptions);
+        integerContext = new IndexContext(table.keyspace,
+                                          table.name,
+                                          table.id,
+                                          table.partitionKeyType,
+                                          table.comparator,
+                                          target.left,
+                                          target.right,
+                                          integerMetadata,
+                                          MockSchema.newCFS(table));
+    }
+
+
+    protected void initialiseColumnData(int numberOfTerms, int rowsPerPartition)
+    {
+        Random random = new Random(randomSeed);
+
+        int numberOfKeys = numberOfTerms / rowsPerPartition;
+        stringTerms = new ByteBuffer[numberOfTerms];
+        integerTerms = new ByteBuffer[numberOfTerms];
+        partitionKeys = new DecoratedKey[numberOfKeys];
+
+        int length = 64;
+
+        for (int i = 0; i < numberOfTerms; i++)
+        {
+            stringTerms[i] = UTF8Type.instance.decompose(generateRandomString(random, length));
+            integerTerms[i] = Int32Type.instance.decompose(i);
+        }
+
+        for (int i = 0; i < numberOfKeys; i++)
+        {
+            partitionKeys[i] = Murmur3Partitioner.instance.decorateKey(UUIDType.instance.decompose(TimeUUID.Generator.nextTimeUUID().asUUID()));
+        }
+    }
+
+    private String generateRandomString(Random random, int length)
+    {
+        return new String(randomChars, random.nextInt(RANDOM_STRING_SIZE - length), length);
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/ReadTrieMemoryIndexBench.java b/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/ReadTrieMemoryIndexBench.java
new file mode 100644
index 000000000000..0301069a9da4
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/ReadTrieMemoryIndexBench.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.index.sai.memory;
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.memory.TrieMemoryIndex;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@Fork(1)
+@Warmup(iterations = 5, time = 3)
+@Measurement(iterations = 10, time = 3)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@BenchmarkMode(Mode.AverageTime)
+@State(Scope.Thread)
+public class ReadTrieMemoryIndexBench extends AbstractTrieMemoryIndexBench
+{
+    private static final int NUMBER_OF_SEARCHES = 1000;
+    private static final AbstractBounds<PartitionPosition> ALL_DATA_RANGE = DataRange.allData(Murmur3Partitioner.instance).keyRange();
+
+    @Param({ "1000", "10000", "100000", "1000000" })
+    protected int numberOfTerms;
+
+    @Param({ "1", "10", "100"})
+    protected int rowsPerPartition;
+
+    private Random random;
+    private Expression[] stringEqualityExpressions;
+    private Expression[] integerEqualityExpressions;
+    private Expression[] integerRangeExpressions;
+
+    @Setup(Level.Iteration)
+    public void initialiseIndexes()
+    {
+        initialiseColumnData(numberOfTerms, rowsPerPartition);
+        stringIndex = new TrieMemoryIndex(stringContext);
+        integerIndex = new TrieMemoryIndex(integerContext);
+
+        int rowCount = 0;
+        int keyCount = 0;
+        for (int i = 0; i < numberOfTerms; i++)
+        {
+            stringIndex.add(partitionKeys[keyCount], Clustering.EMPTY, stringTerms[i], allocatedBytes -> {}, allocatesBytes -> {});
+            integerIndex.add(partitionKeys[keyCount], Clustering.EMPTY, integerTerms[i], allocatedBytes -> {}, allocatesBytes -> {});
+            if (++rowCount == rowsPerPartition)
+            {
+                rowCount = 0;
+                keyCount++;
+            }
+        }
+        random = new Random(randomSeed);
+
+        stringEqualityExpressions =  new Expression[NUMBER_OF_SEARCHES];
+        integerEqualityExpressions  =  new Expression[NUMBER_OF_SEARCHES];
+        integerRangeExpressions = new Expression[NUMBER_OF_SEARCHES];
+
+        for (int i = 0; i < NUMBER_OF_SEARCHES; i++)
+        {
+            stringEqualityExpressions[i] = new Expression(stringContext).add(Operator.EQ, stringTerms[random.nextInt(numberOfTerms)]);
+            integerEqualityExpressions[i] = new Expression(integerContext).add(Operator.EQ, integerTerms[random.nextInt(numberOfTerms)]);
+
+            int lowerValue = random.nextInt(numberOfTerms - 10);
+
+            integerRangeExpressions[i] = new Expression(integerContext)
+            {{
+                operation = Op.RANGE;
+                lower = new Bound(Int32Type.instance.decompose(lowerValue), Int32Type.instance, true);
+                upper = new Bound(Int32Type.instance.decompose(lowerValue + 10), Int32Type.instance, true);
+            }};
+        }
+    }
+
+    @Benchmark
+    public long stringEqualityBenchmark()
+    {
+        long size = 0;
+        for (int i = 0; i < NUMBER_OF_SEARCHES; i++)
+        {
+            stringIndex.search(stringEqualityExpressions[i], ALL_DATA_RANGE);
+        }
+        return size;
+    }
+
+    @Benchmark
+    public long integerEqualityBenchmark()
+    {
+        long size = 0;
+        for (int i = 0; i < NUMBER_OF_SEARCHES; i++)
+        {
+            integerIndex.search(integerEqualityExpressions[i], ALL_DATA_RANGE);
+        }
+        return size;
+    }
+
+    @Benchmark
+    public long integerRangeBenchmark()
+    {
+        long size = 0;
+        for (int i = 0; i < NUMBER_OF_SEARCHES; i++)
+        {
+            integerIndex.search(integerRangeExpressions[i], ALL_DATA_RANGE);
+        }
+        return size;
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/WriteTrieMemoryIndexBench.java b/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/WriteTrieMemoryIndexBench.java
new file mode 100644
index 000000000000..b42c5f450cc3
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/index/sai/memory/WriteTrieMemoryIndexBench.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.index.sai.memory;
+
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.index.sai.memory.TrieMemoryIndex;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@Fork(1)
+@Warmup(iterations = 5, time = 3)
+@Measurement(iterations = 10, time = 3)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@BenchmarkMode(Mode.AverageTime)
+@State(Scope.Thread)
+public class WriteTrieMemoryIndexBench extends AbstractTrieMemoryIndexBench
+{
+    @Param({ "1000", "10000", "100000", "1000000" })
+    protected int numberOfTerms;
+
+    @Param({ "1", "10", "100"})
+    protected int rowsPerPartition;
+
+    @Setup(Level.Iteration)
+    public void initialiseColumnData()
+    {
+        initialiseColumnData(numberOfTerms, rowsPerPartition);
+    }
+
+    @Setup(Level.Invocation)
+    public void initialiseIndexes()
+    {
+        stringIndex = new TrieMemoryIndex(stringContext);
+        integerIndex = new TrieMemoryIndex(integerContext);
+    }
+
+    @Benchmark
+    public long writeStringIndex()
+    {
+        long size = 0;
+        int rowCount = 0;
+        int keyCount = 0;
+        for (ByteBuffer term : stringTerms)
+        {
+            stringIndex.add(partitionKeys[keyCount], Clustering.EMPTY, term, allocatedBytes -> {}, allocatesBytes -> {});
+            if (++rowCount == rowsPerPartition)
+            {
+                rowCount = 0;
+                keyCount++;
+            }
+            size++;
+        }
+        return size;
+    }
+
+    @Benchmark
+    public long writeIntegerIndex()
+    {
+        long size = 0;
+        int rowCount = 0;
+        int keyCount = 0;
+        for (ByteBuffer term : integerTerms)
+        {
+            integerIndex.add(partitionKeys[keyCount], Clustering.EMPTY, term, allocatedBytes -> {}, allocatesBytes -> {});
+            if (++rowCount == rowsPerPartition)
+            {
+                rowCount = 0;
+                keyCount++;
+            }
+            size++;
+        }
+        return size;
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/AbstractOnDiskBench.java b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/AbstractOnDiskBench.java
new file mode 100644
index 000000000000..76bbedc8dffa
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/AbstractOnDiskBench.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.index.sai.v1;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Random;
+import java.util.stream.IntStream;
+
+import com.google.common.base.Stopwatch;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.apache.cassandra.index.sai.disk.v1.MetadataSource;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.v1.SSTableComponentsWriter;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.BlockPackedReader;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.TearDown;
+
+public abstract class AbstractOnDiskBench
+{
+    private static Random random = new Random();
+
+    protected TableMetadata metadata;
+    protected IndexDescriptor indexDescriptor;
+
+    private Descriptor descriptor;
+
+    String index;
+    IndexContext indexContext;
+    private FileHandle token;
+    private FileHandle postings;
+    private long summaryPosition;
+
+    /**
+     * @return num of rows to be stored in per-sstable components
+     */
+    public abstract int numRows();
+
+    /**
+     * @return num of postings to be written in posting file
+     */
+    public abstract int numPostings();
+
+    /**
+     * To be called before executing each @Benchmark method
+     */
+    public abstract void beforeInvocation() throws Throwable;
+
+    /**
+     * To be called after executing each @Benchmark method
+     */
+    public abstract void afterInvocation() throws Throwable;
+
+    protected int toPosting(int id)
+    {
+        return id;
+    }
+
+    protected long toToken(long id)
+    {
+        return id * 16_013L + random.nextInt(16_000);
+    }
+
+    protected long toOffset(long id)
+    {
+        return id * 16_013L + random.nextInt(16_000);
+    }
+
+    @Setup(Level.Trial)
+    public void perTrialSetup() throws IOException
+    {
+        DatabaseDescriptor.daemonInitialization(); // required to use ChunkCache
+        assert ChunkCache.instance != null;
+
+        String keyspaceName = "ks";
+        String tableName = this.getClass().getSimpleName();
+        metadata = TableMetadata
+                   .builder(keyspaceName, tableName)
+                   .partitioner(Murmur3Partitioner.instance)
+                   .addPartitionKeyColumn("pk", UTF8Type.instance)
+                   .addRegularColumn("col", IntegerType.instance)
+                   .build();
+
+        descriptor = new Descriptor(new File(Files.createTempDirectory("jmh").toFile()),
+                                    metadata.keyspace,
+                                    metadata.name,
+                                    Util.newUUIDGen().get());
+        indexDescriptor = IndexDescriptor.empty(descriptor);
+        index = "test";
+        indexContext = SAITester.createIndexContext(index, IntegerType.instance);
+
+        // write per-sstable components: token and offset
+        writeSSTableComponents(numRows());
+        token = indexDescriptor.perSSTableComponents().get(IndexComponentType.TOKEN_VALUES).createFileHandle();
+
+        // write postings
+        summaryPosition = writePostings(numPostings());
+        postings = indexDescriptor.perIndexComponents(indexContext).get(IndexComponentType.POSTING_LISTS).createFileHandle();
+    }
+
+    @TearDown(Level.Trial)
+    public void perTrialTearDown()
+    {
+        token.close();
+        postings.close();
+        FileUtils.deleteRecursive(descriptor.directory);
+    }
+
+    @Setup(Level.Invocation)
+    public void perInvocationSetup() throws Throwable
+    {
+        beforeInvocation();
+    }
+
+    @TearDown(Level.Invocation)
+    public void perInvocationTearDown() throws Throwable
+    {
+        afterInvocation();
+    }
+
+    private long writePostings(int rows) throws IOException
+    {
+        final int[] postings = IntStream.range(0, rows).map(this::toPosting).toArray();
+        final IntArrayPostingList postingList = new IntArrayPostingList(postings);
+
+        try (PostingsWriter writer = new PostingsWriter(indexDescriptor.newPerIndexComponentsForWrite(indexContext)))
+        {
+            long summaryPosition = writer.write(postingList);
+            writer.complete();
+
+            return summaryPosition;
+        }
+    }
+
+    protected final PostingsReader openPostingsReader() throws IOException
+    {
+        IndexInput input = IndexFileUtils.instance().openInput(postings);
+        IndexInput summaryInput = IndexFileUtils.instance().openInput(postings);
+
+        PostingsReader.BlocksSummary summary = new PostingsReader.BlocksSummary(summaryInput, summaryPosition);
+        return new PostingsReader(input, summary, QueryEventListener.PostingListEventListener.NO_OP);
+    }
+
+    private void writeSSTableComponents(int rows) throws IOException
+    {
+        SSTableComponentsWriter writer = new SSTableComponentsWriter(indexDescriptor.newPerSSTableComponentsForWrite());
+        for (int i = 0; i < rows; i++)
+            writer.recordCurrentTokenOffset(toToken(i), toOffset(i));
+
+        writer.complete(Stopwatch.createStarted());
+    }
+
+    protected final LongArray openRowIdToTokenReader() throws IOException
+    {
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        MetadataSource source = MetadataSource.loadMetadata(components);
+        NumericValuesMeta tokensMeta = new NumericValuesMeta(source.get(components.get(IndexComponentType.TOKEN_VALUES)));
+        return new BlockPackedReader(token, tokensMeta).open();
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/BlockPackedReaderBench.java b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/BlockPackedReaderBench.java
new file mode 100644
index 000000000000..e1c4ce009fd4
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/BlockPackedReaderBench.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.index.sai.v1;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(value = 1, jvmArgsAppend = {
+        //        "-XX:+UnlockCommercialFeatures", "-XX:+FlightRecorder","-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints",
+        //        "-XX:StartFlightRecording=duration=60s,filename=./BlockPackedReaderBenchmark.jfr,name=profile,settings=profile",
+        //                            "-XX:FlightRecorderOptions=settings=/home/jake/workspace/cassandra/profiling-advanced.jfc,samplethreads=true"
+})
+@Warmup(iterations = 3)
+@Measurement(iterations = 5, timeUnit = TimeUnit.NANOSECONDS)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+public class BlockPackedReaderBench extends AbstractOnDiskBench
+{
+    private static final int NUM_INVOCATIONS = 10_000;
+
+    @Param({ "1", "10", "100", "1000"})
+    public int skippingDistance;
+
+    protected LongArray rowIdToToken;
+    private int[] rowIds;
+    private long[] tokenValues;
+
+    @Override
+    public int numRows()
+    {
+        return 10_000_000;
+    }
+
+    @Override
+    public int numPostings()
+    {
+        return 10_000_000;
+    }
+
+    @Override
+    public void beforeInvocation() throws Throwable
+    {
+        // rowIdToToken.findTokenRowID keeps track of last position, so it must be per-benchmark-method-invocation.
+        rowIdToToken = openRowIdToTokenReader();
+
+        rowIds = new int[NUM_INVOCATIONS];
+        tokenValues = new long[NUM_INVOCATIONS];
+
+        for (int i = 0; i < rowIds.length; i++)
+        {
+            rowIds[i] = toPosting(i * skippingDistance);
+            tokenValues[i] = toToken(rowIds[i]);
+        }
+    }
+
+    @Override
+    public void afterInvocation() throws Throwable
+    {
+        rowIdToToken.close();
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput, Mode.AverageTime })
+    public void get(Blackhole bh)
+    {
+        for (int i = 0; i < rowIds.length;)
+        {
+            bh.consume(rowIdToToken.get(rowIds[i]));
+            i++;
+        }
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput, Mode.AverageTime })
+    public void findTokenRowID(Blackhole bh)
+    {
+        for (int i = 0; i < tokenValues.length;)
+        {
+            bh.consume(rowIdToToken.ceilingRowId(tokenValues[i]));
+            i++;
+        }
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/MergePostingListBench.java b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/MergePostingListBench.java
new file mode 100644
index 000000000000..760f1269956a
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/MergePostingListBench.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.index.sai.v1;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList;
+import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@State(Scope.Thread)
+public class MergePostingListBench
+{
+    @Param({"3", "17"})
+    int stepMax = 7;
+
+    @Param({"50", "5000"})
+    int sources = 50;
+
+    @Param({"1000000"})
+    int count = 1_000_000;
+
+    @Param({"UNIFORM", "NORMAL", "SEQUENTIAL", "ROUND_ROBIN"})
+    Distribution distribution = Distribution.NORMAL;
+
+    public enum Distribution
+    {
+        UNIFORM, NORMAL, SEQUENTIAL, ROUND_ROBIN
+    }
+
+    List<int[]> splitPostingLists = new ArrayList<>();
+    PostingList merge;
+
+    @Setup(Level.Trial)
+    public void generatePostings()
+    {
+        final AtomicInteger rowId = new AtomicInteger();
+        final Random rand = new Random(1);
+        final int[] postings = IntStream.generate(() -> rowId.addAndGet(rand.nextInt(stepMax)))
+                                        .limit(count)
+                                        .toArray();
+
+        // split postings into multiple lists
+        Function<Integer, Integer> grouping;
+        switch (distribution)
+        {
+            case UNIFORM:
+                grouping = x -> rand.nextInt(sources);
+                break;
+            case NORMAL:
+                grouping = x -> (int) Math.min(sources - 1, Math.abs(rand.nextGaussian()) * sources / 5);
+                break;
+            case SEQUENTIAL:
+            {
+                AtomicInteger index = new AtomicInteger();
+                int sizePerList = Math.max(count / sources, 1);
+                grouping = x -> index.getAndIncrement() / sizePerList;
+                break;
+            }
+            case ROUND_ROBIN:
+            {
+                AtomicInteger index = new AtomicInteger();
+                grouping = x -> index.getAndIncrement() % sources;
+                break;
+            }
+            default:
+                throw new AssertionError();
+        }
+        final Map<Integer, List<Integer>> splitPostings = Arrays.stream(postings)
+                                                                .boxed()
+                                                                .collect(Collectors.groupingBy(grouping));
+
+        for (List<Integer> split : splitPostings.values())
+        {
+            // Remove any duplicates in each individual set
+            int[] data = split.stream().distinct().mapToInt(Integer::intValue).toArray();
+            splitPostingLists.add(data);
+        }
+    }
+
+    @Setup(Level.Invocation)
+    public void mergePostings()
+    {
+        var lists = new ArrayList<PostingList>();
+        for (int[] postings : splitPostingLists)
+        {
+            lists.add(new IntArrayPostingList(postings));
+        }
+        merge = MergePostingList.merge(lists);
+    }
+
+    @Benchmark
+    @BenchmarkMode({ Mode.AverageTime })
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void nextPostingIteration(Blackhole bh) throws IOException
+    {
+        long id;
+        while ((id = merge.nextPosting()) != PostingList.END_OF_STREAM)
+        {
+            bh.consume(id);
+        }
+    }
+
+    @Benchmark
+    @BenchmarkMode({ Mode.AverageTime })
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void advanceIteration(Blackhole bh) throws IOException
+    {
+        int id = 0;
+        while ((id = merge.advance(id + stepMax)) != PostingList.END_OF_STREAM)
+        {
+            bh.consume(id);
+        }
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/PostingsReaderBench.java b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/PostingsReaderBench.java
new file mode 100644
index 000000000000..e831e9a09186
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v1/PostingsReaderBench.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.index.sai.v1;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5, timeUnit = TimeUnit.MILLISECONDS)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Thread)
+public class PostingsReaderBench extends AbstractOnDiskBench
+{
+    private static final int NUM_INVOCATIONS = 10_000;
+
+    @Param({ "1", "10", "100", "1000"})
+    public int skippingDistance;
+
+    protected LongArray rowIdToToken;
+    protected PostingsReader reader;
+    private int[] rowIds;
+    protected long[] tokenValues;
+
+    @Override
+    public int numRows()
+    {
+        return 10_000_000;
+    }
+
+    @Override
+    public int numPostings()
+    {
+        return 10_000_000;
+    }
+
+    @Override
+    public void beforeInvocation() throws Throwable
+    {
+        // rowIdToToken.findTokenRowID keeps track of last position, so it must be per-benchmark-method-invocation.
+        rowIdToToken = openRowIdToTokenReader();
+        reader = openPostingsReader();
+
+        tokenValues = new long[NUM_INVOCATIONS];
+        rowIds = new int[NUM_INVOCATIONS];
+        for (int i = 0; i < tokenValues.length; i++)
+        {
+            rowIds[i] = toPosting(i * skippingDistance);
+            tokenValues[i] = toToken(i * skippingDistance);
+        }
+    }
+
+    @Override
+    public void afterInvocation() throws Throwable
+    {
+        rowIdToToken.close();
+        reader.close();
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput, Mode.AverageTime })
+    public void skipAndRequestNext(Blackhole bh) throws Throwable
+    {
+        int rowId = -1;
+        for (int i = 0; i < tokenValues.length;)
+        {
+            long token = tokenValues[i];
+            if (rowId < 0)
+                rowId = (int) rowIdToToken.ceilingRowId(token);
+            bh.consume(reader.advance(rowId));
+            rowId = -1;
+
+            i++;
+        }
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput, Mode.AverageTime })
+    public void advance(Blackhole bh) throws Throwable
+    {
+        for (int i = 0; i < tokenValues.length;)
+        {
+            int rowId = rowIds[i];
+            bh.consume(reader.advance(rowId));
+
+            i++;
+        }
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/index/sai/v2/sortedbytes/SortedTermsBench.java b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v2/sortedbytes/SortedTermsBench.java
new file mode 100644
index 000000000000..946ac1ff2799
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/index/sai/v2/sortedbytes/SortedTermsBench.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.index.sai.v2.sortedbytes;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.apache.cassandra.index.sai.disk.v1.MetadataSource;
+import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
+import org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsMeta;
+import org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsReader;
+import org.apache.cassandra.index.sai.disk.v2.sortedterms.SortedTermsWriter;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.test.microbench.index.sai.v1.AbstractOnDiskBench;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(value = 1, jvmArgsAppend = {
+        //        "-XX:+UnlockCommercialFeatures", "-XX:+FlightRecorder","-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints",
+        //        "-XX:StartFlightRecording=duration=60s,filename=./BlockPackedReaderBenchmark.jfr,name=profile,settings=profile",
+        //                            "-XX:FlightRecorderOptions=settings=/home/jake/workspace/cassandra/profiling-advanced.jfc,samplethreads=true"
+})
+@Warmup(iterations = 1)
+@Measurement(iterations = 1, timeUnit = TimeUnit.MICROSECONDS)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Thread)
+public class SortedTermsBench extends AbstractOnDiskBench
+{
+    private static final int NUM_ROWS = 1_000_000;
+    private static final int NUM_INVOCATIONS = 1_000; // must be <= (NUM_ROWS / max(skippingDistance))
+
+    @Param({ "1", "10", "100", "1000"})
+    public int skippingDistance;
+
+    protected LongArray rowIdToToken;
+    private int[] rowIds;
+    private long[] tokenValues;
+    FileHandle trieFile;
+    FileHandle termsData;
+    FileHandle blockOffsets;
+    SortedTermsReader sortedTermsReader;
+    Path luceneDir;
+    Directory directory;
+    DirectoryReader luceneReader;
+    SortedDocValues columnASortedDocValues;
+
+    @Override
+    public int numRows()
+    {
+        return NUM_ROWS;
+    }
+
+    @Override
+    public int numPostings()
+    {
+        return NUM_ROWS;
+    }
+
+    byte[][] bcIntBytes = new byte[NUM_ROWS][];
+
+    @Setup(Level.Trial)
+    public void perTrialSetup2() throws IOException
+    {
+        IndexComponents.ForWrite components = indexDescriptor.newPerSSTableComponentsForWrite();
+        try (MetadataWriter metadataWriter = new MetadataWriter(components);
+             NumericValuesWriter blockFPWriter = new NumericValuesWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS),
+                                                                         metadataWriter, true);
+             SortedTermsWriter writer = new SortedTermsWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCKS),
+                                                              metadataWriter,
+                                                              blockFPWriter,
+                                                              components.addOrGet(IndexComponentType.PRIMARY_KEY_TRIE)))
+        {
+            for (int x = 0; x < NUM_ROWS; x++)
+            {
+                ByteBuffer buffer = Int32Type.instance.decompose(x);
+                ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, TypeUtil.BYTE_COMPARABLE_VERSION);
+                byte[] bytes = ByteSourceInverse.readBytes(byteSource);
+                bcIntBytes[x] = bytes;
+                writer.add(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, bytes));
+            }
+        }
+
+        // create the lucene index
+        luceneDir = Files.createTempDirectory("jmh_lucene_test");
+        directory = FSDirectory.open(luceneDir);
+        IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
+        IndexWriter indexWriter = new IndexWriter(directory, config);
+
+        Document document = new Document();
+
+        int i = 0;
+        for (int x = 0; x < NUM_ROWS; x++)
+        {
+            document.clear();
+            byte[] bytes = new byte[4];
+            NumericUtils.intToSortableBytes(x, bytes, 0);
+            document.add(new SortedDocValuesField("columnA", new BytesRef(bytes)));
+            indexWriter.addDocument(document);
+            luceneBytes[x] = bytes;
+        }
+        indexWriter.forceMerge(1);
+        indexWriter.close();
+    }
+
+    byte[][] luceneBytes = new byte[NUM_ROWS][];
+
+    @Override
+    public void beforeInvocation() throws Throwable
+    {
+        // rowIdToToken.findTokenRowID keeps track of last position, so it must be per-benchmark-method-invocation.
+        rowIdToToken = openRowIdToTokenReader();
+
+        rowIds = new int[NUM_ROWS];
+        tokenValues = new long[NUM_ROWS];
+
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        MetadataSource metadataSource = MetadataSource.loadMetadata(components);
+        IndexComponent.ForRead blocksComponent = components.get(IndexComponentType.PRIMARY_KEY_BLOCKS);
+        IndexComponent.ForRead blockOffsetsComponent = components.get(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS);
+        NumericValuesMeta blockOffsetMeta = new NumericValuesMeta(metadataSource.get(blockOffsetsComponent));
+        SortedTermsMeta sortedTermsMeta = new SortedTermsMeta(metadataSource.get(blocksComponent));
+        trieFile = components.get(IndexComponentType.PRIMARY_KEY_TRIE).createFileHandle();
+        termsData = blocksComponent.createFileHandle();
+        blockOffsets = blockOffsetsComponent.createFileHandle();
+
+        sortedTermsReader = new SortedTermsReader(termsData,blockOffsets, trieFile, sortedTermsMeta, blockOffsetMeta);
+
+        luceneReader = DirectoryReader.open(directory);
+        LeafReaderContext context = luceneReader.leaves().get(0);
+
+        columnASortedDocValues = context.reader().getSortedDocValues("columnA");
+    }
+
+    @Override
+    public void afterInvocation() throws Throwable
+    {
+        luceneReader.close();
+        termsData.close();
+        blockOffsets.close();
+        rowIdToToken.close();
+        trieFile.close();
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput})
+    public void luceneSeekToPointID(Blackhole bh) throws IOException
+    {
+        for (int i = 0; i < NUM_INVOCATIONS;)
+        {
+            bh.consume(columnASortedDocValues.lookupOrd(i));
+            i += skippingDistance;
+        }
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput})
+    public void luceneSeekToTerm(Blackhole bh) throws IOException
+    {
+        for (int i = 0; i < NUM_INVOCATIONS; i++)
+        {
+            bh.consume(columnASortedDocValues.lookupTerm(new BytesRef(luceneBytes[i * skippingDistance])));
+        }
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput})
+    public void advance(Blackhole bh) throws IOException
+    {
+        try (SortedTermsReader.Cursor cursor = sortedTermsReader.openCursor())
+        {
+            for (int i = 0; i < NUM_INVOCATIONS; i++)
+            {
+                cursor.advance();
+                bh.consume(cursor.term());
+            }
+        }
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput})
+    public void seekToPointID(Blackhole bh) throws IOException
+    {
+        try (SortedTermsReader.Cursor cursor = sortedTermsReader.openCursor())
+        {
+            for (int i = 0; i < NUM_INVOCATIONS; i++)
+            {
+                cursor.seekToPointId((long) i * skippingDistance);
+                bh.consume(cursor.term());
+            }
+        }
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput})
+    public void seekToTerm(Blackhole bh) throws IOException
+    {
+        try (SortedTermsReader.Cursor cursor = sortedTermsReader.openCursor())
+        {
+            for (int i = 0; i < NUM_INVOCATIONS; i++)
+            {
+                int iFinal = i;
+                bh.consume(cursor.ceiling(v -> ByteSource.preencoded(this.bcIntBytes[iFinal * skippingDistance])));
+            }
+        }
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput})
+    public void get(Blackhole bh)
+    {
+        for (int i = 0; i < NUM_INVOCATIONS; i++)
+        {
+            bh.consume(rowIdToToken.get(rowIds[i * skippingDistance]));
+        }
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_INVOCATIONS)
+    @BenchmarkMode({ Mode.Throughput})
+    public void longArrayFindTokenRowID(Blackhole bh)
+    {
+        for (int i = 0; i < NUM_INVOCATIONS; i++)
+        {
+            bh.consume(rowIdToToken.ceilingRowId(tokenValues[i * skippingDistance]));
+        }
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/ReadBenchBase.java b/test/microbench/org/apache/cassandra/test/microbench/instance/ReadBenchBase.java
index 3e2d64fcd4a3..c44e1675a500 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/instance/ReadBenchBase.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/instance/ReadBenchBase.java
@@ -22,15 +22,23 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 import com.google.common.base.Throwables;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.utils.FBUtilities;
-import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
 
 @State(Scope.Benchmark)
 public abstract class ReadBenchBase extends SimpleTableWriter
@@ -71,7 +79,9 @@ public void setup() throws Throwable
         switch (flush)
         {
         case YES:
+            long flushStart = System.currentTimeMillis();
             cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+            System.err.format("Flushed in %.3f s.\n", (System.currentTimeMillis() - flushStart) / 1000.0);
             break;
         case INMEM:
             if (!cfs.getLiveSSTables().isEmpty())
@@ -84,7 +94,7 @@ public void setup() throws Throwable
                           cfs.getLiveSSTables().size(),
                           FBUtilities.prettyPrintMemory(cfs.metric.liveDiskSpaceUsed.getCount()),
                           cfs.metric.estimatedPartitionCount.getValue(),
-                          cfs.metric.writeLatency.latency.getSnapshot().getMean());
+                          cfs.metric.writeLatency.tableOrKeyspaceMetric().latency.getSnapshot().getMean());
         // Needed to stabilize sstable count for off-cache sized tests (e.g. count = 100_000_000)
         while (cfs.getLiveSSTables().size() >= 15)
         {
@@ -94,10 +104,69 @@ public void setup() throws Throwable
                               cfs.getLiveSSTables().size(),
                               FBUtilities.prettyPrintMemory(cfs.metric.liveDiskSpaceUsed.getCount()),
                               cfs.metric.estimatedPartitionCount.getValue(),
-                              cfs.metric.writeLatency.latency.getSnapshot().getMean());
+                              cfs.metric.writeLatency.tableOrKeyspaceMetric().latency.getSnapshot().getMean());
         }
     }
 
+
+    public void performWrite(String writeStatement, long ofs, long count) throws Throwable
+    {
+        if (threadCount == 1)
+            performWriteSerial(writeStatement, ofs, count);
+        else
+            performWriteThreads(writeStatement, ofs, count);
+    }
+
+    public void performWriteSerial(String writeStatement, long ofs, long count) throws Throwable
+    {
+        for (long i = ofs; i < ofs + count; ++i)
+            execute(writeStatement, writeArguments(i));
+    }
+
+    public void performWriteThreads(String writeStatement, long ofs, long count) throws Throwable
+    {
+        List<Future<Integer>> futures = new ArrayList<>();
+        for (long i = 0; i < count; ++i)
+        {
+            long pos = ofs + i;
+            futures.add(executorService.submit(() ->
+                                               {
+                                                   try
+                                                   {
+                                                       execute(writeStatement, writeArguments(pos));
+                                                       return 1;
+                                                   }
+                                                   catch (Throwable throwable)
+                                                   {
+                                                       throw Throwables.propagate(throwable);
+                                                   }
+                                               }));
+        }
+        long done = 0;
+        for (Future<Integer> f : futures)
+            done += f.get();
+        assert count == done;
+    }
+
+    @TearDown(Level.Trial)
+    public void teardown() throws InterruptedException
+    {
+        if (flush == Flush.INMEM && !cfs.getLiveSSTables().isEmpty())
+            throw new AssertionError("SSTables created for INMEM test.");
+
+        executorService.shutdown();
+        executorService.awaitTermination(15, TimeUnit.SECONDS);
+
+        // do a flush to print sizes
+        long flushStart = System.currentTimeMillis();
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+        System.err.format("Flushed in %.3f s.\n", (System.currentTimeMillis() - flushStart) / 1000.0);
+
+        CommitLog.instance.shutdownBlocking();
+        CQLTester.tearDownClass();
+        CQLTester.cleanup();
+    }
+
     public Object performReadSerial(String readStatement, Supplier<Object[]> supplier) throws Throwable
     {
         long sum = 0;
diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java b/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java
index cc78e03e3a72..fdf515fa381f 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java
@@ -27,6 +27,8 @@
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Throwables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
@@ -44,6 +46,8 @@
 @State(Scope.Benchmark)
 public abstract class SimpleTableWriter extends CQLTester
 {
+    private final static Logger logger = LoggerFactory.getLogger(SimpleTableWriter.class);
+
     static String keyspace;
     String table;
     ColumnFamilyStore cfs;
@@ -66,14 +70,19 @@ public abstract class SimpleTableWriter extends CQLTester
     @Param({ "32" })
     int threadCount;
 
+    @Param({"1", "1000"})
+    int rowsPerPartition = 1;
+
+    int partitions;
+
     public void commonSetup() throws Throwable
     {
         rand = new Random(1);
         executorService = Executors.newFixedThreadPool(threadCount);
-        CQLTester.setUpClass();
-        CQLTester.prepareServer();
+        partitions = Math.max(1, count / rowsPerPartition);
         DatabaseDescriptor.setAutoSnapshot(false);
-        System.err.println("setupClass done.");
+        CQLTester.setUpClass();
+        logger.info("setupClass done.");
         String memtableSetup = "";
         if (!memtableClass.isEmpty())
             memtableSetup = String.format(" AND memtable = '%s'", memtableClass);
@@ -88,7 +97,8 @@ public void commonSetup() throws Throwable
             executeNet(getDefaultVersion(), "use " + keyspace + ";");
         }
         writeStatement = "INSERT INTO " + table + "(userid,picid,commentid)VALUES(?,?,?)";
-        System.err.println("Prepared, batch " + BATCH + " threads " + threadCount + extraInfo());
+        System.err.println(String.format("Prepared, batch %s threads %s flush %s", BATCH, threadCount, extraInfo()));
+        System.err.println(String.format("%s writes in %s partitions x %s rows", count, partitions, rowsPerPartition));
         System.err.println("Disk access mode " + DatabaseDescriptor.getDiskAccessMode() +
                            " index " + DatabaseDescriptor.getIndexAccessMode());
 
@@ -189,7 +199,7 @@ public void teardown() throws InterruptedException
 
         Memtable memtable = cfs.getTracker().getView().getCurrentMemtable();
         Memtable.MemoryUsage usage = Memtable.getMemoryUsage(memtable);
-        System.err.format("\n%s in %s mode: %d ops, %s serialized bytes, %s\n",
+        logger.info("\n{} in {} mode: {} ops, {} serialized bytes, {}\n",
                           memtable.getClass().getSimpleName(),
                           DatabaseDescriptor.getMemtableAllocationType(),
                           memtable.operationCount(),
diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/WriteBench.java b/test/microbench/org/apache/cassandra/test/microbench/instance/WriteBench.java
index 673895547583..2e964d4c7e75 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/instance/WriteBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/instance/WriteBench.java
@@ -59,24 +59,24 @@ public void writeTable() throws Throwable
 
         switch (flush)
         {
-        case FLUSH:
-            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
-            // if we flush we also must truncate to avoid accummulating sstables
-        case TRUNCATE:
-            execute("TRUNCATE TABLE " + table);
-            // note: we turn snapshotting and durable writes (which would have caused a flush) off for this benchmark
-            break;
-        case INMEM:
-            if (!cfs.getLiveSSTables().isEmpty())
-                throw new AssertionError("SSTables created for INMEM test.");
-            // leave unflushed, i.e. next iteration will overwrite data
-        default:
+            case FLUSH:
+                cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+                // if we flush we also must truncate to avoid accumulating sstables
+            case TRUNCATE:
+                execute("TRUNCATE TABLE " + table);
+                // note: we turn snapshotting and durable writes (which would have caused a flush) off for this benchmark
+                break;
+            case INMEM:
+                if (!cfs.getLiveSSTables().isEmpty())
+                    throw new AssertionError("SSTables created for INMEM test.");
+                // leave unflushed, i.e. next iteration will overwrite data
+            default:
         }
     }
 
     public Object[] writeArguments(long i)
     {
-        return new Object[] { i, i, i };
+        return new Object[] { i % partitions, i, i };
     }
 
     void doExtraChecks()
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sai/KeyLookupBench.java b/test/microbench/org/apache/cassandra/test/microbench/sai/KeyLookupBench.java
deleted file mode 100644
index 78f700d83fe4..000000000000
--- a/test/microbench/org/apache/cassandra/test/microbench/sai/KeyLookupBench.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.test.microbench.sai;
-
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.Arrays;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.marshal.CompositeType;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.SSTableComponentsWriter;
-import org.apache.cassandra.index.sai.disk.v1.WidePrimaryKeyMap;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.TableMetadata;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Level;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Warmup;
-
-import static org.apache.cassandra.index.sai.SAITester.getRandom;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-@BenchmarkMode({Mode.Throughput})
-@OutputTimeUnit(TimeUnit.MILLISECONDS)
-@Warmup(iterations = 3, time = 1)
-@Measurement(iterations = 3, time = 5)
-@Fork(value = 1, jvmArgsAppend = "-Xmx512M")
-@Threads(1)
-@State(Scope.Benchmark)
-public class KeyLookupBench
-{
-    private static final int rows = 1_000_000;
-
-    static
-    {
-        DatabaseDescriptor.toolInitialization();
-        // Partitioner is not set in client mode.
-        if (DatabaseDescriptor.getPartitioner() == null)
-            DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
-    }
-
-    protected TableMetadata metadata;
-    protected IndexDescriptor indexDescriptor;
-
-    private PrimaryKeyMap primaryKeyMap;
-
-    private PrimaryKey primaryKey;
-
-    @Param({"3", "4", "5"})
-    public int partitionBlockShift;
-
-    @Param({"3", "4", "5"})
-    public int clusteringBlockShift;
-
-    @Param({"10", "100", "1000", "10000"})
-    public int partitionSize;
-
-    @Param({"true", "false"})
-    public boolean randomClustering;
-
-    @Setup(Level.Trial)
-    public void trialSetup() throws Exception
-    {
-        String keyspaceName = "ks";
-        String tableName = this.getClass().getSimpleName();
-        metadata = TableMetadata
-                   .builder(keyspaceName, tableName)
-                   .partitioner(Murmur3Partitioner.instance)
-                   .addPartitionKeyColumn("pk1", LongType.instance)
-                   .addPartitionKeyColumn("pk2", LongType.instance)
-                   .addClusteringColumn("ck1", UTF8Type.instance)
-                   .addClusteringColumn("ck2", UTF8Type.instance)
-                   .build();
-
-        Descriptor descriptor = new Descriptor(new File(Files.createTempDirectory("jmh").toFile()),
-                                               metadata.keyspace,
-                                               metadata.name,
-                                               Util.newUUIDGen().get());
-
-        indexDescriptor = IndexDescriptor.create(descriptor, metadata.partitioner, metadata.comparator);
-
-        CassandraRelevantProperties.SAI_SORTED_TERMS_PARTITION_BLOCK_SHIFT.setInt(partitionBlockShift);
-        CassandraRelevantProperties.SAI_SORTED_TERMS_CLUSTERING_BLOCK_SHIFT.setInt(clusteringBlockShift);
-        SSTableComponentsWriter writer = new SSTableComponentsWriter(indexDescriptor);
-
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(metadata.partitioner, metadata.comparator);
-
-        PrimaryKey[] primaryKeys = new PrimaryKey[rows];
-        int partition = 0;
-        int partitionRowCounter = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            primaryKeys[index] = factory.create(makeKey(metadata, (long) partition, (long) partition), makeClustering(metadata));
-            partitionRowCounter++;
-            if (partitionRowCounter == partitionSize)
-            {
-                partition++;
-                partitionRowCounter = 0;
-            }
-        }
-
-        Arrays.sort(primaryKeys);
-
-        DecoratedKey lastKey = null;
-        for (PrimaryKey primaryKey : primaryKeys)
-        {
-            if (lastKey == null || lastKey.compareTo(primaryKey.partitionKey()) < 0)
-            {
-                lastKey = primaryKey.partitionKey();
-                writer.startPartition(lastKey);
-            }
-            writer.nextRow(primaryKey);
-        }
-
-        writer.complete();
-
-        SSTableReader sstableReader = mock(SSTableReader.class);
-        when(sstableReader.metadata()).thenReturn(metadata);
-
-        PrimaryKeyMap.Factory mapFactory = new WidePrimaryKeyMap.Factory(indexDescriptor, sstableReader);
-
-        primaryKeyMap = mapFactory.newPerSSTablePrimaryKeyMap();
-
-        primaryKey = primaryKeys[500000];
-    }
-
-    @Benchmark
-    public long advanceToKey()
-    {
-        return primaryKeyMap.rowIdFromPrimaryKey(primaryKey);
-    }
-
-    private static DecoratedKey makeKey(TableMetadata table, Object...partitionKeys)
-    {
-        ByteBuffer key;
-        if (table.partitionKeyType instanceof CompositeType)
-            key = ((CompositeType)table.partitionKeyType).decompose(partitionKeys);
-        else
-            key = table.partitionKeyType.fromString((String)partitionKeys[0]);
-        return table.partitioner.decorateKey(key);
-    }
-
-    private Clustering<?> makeClustering(TableMetadata table)
-    {
-        Clustering<?> clustering;
-        if (table.comparator.size() == 0)
-            clustering = Clustering.EMPTY;
-        else
-        {
-            ByteBuffer[] values = new ByteBuffer[table.comparator.size()];
-            for (int index = 0; index < table.comparator.size(); index++)
-                values[index] = table.comparator.subtype(index).fromString(makeClusteringString());
-            clustering = Clustering.make(values);
-        }
-        return clustering;
-    }
-
-    private String makeClusteringString()
-    {
-        if (randomClustering)
-            return getRandom().nextTextString(10, 100);
-        else
-            return String.format("%08d", getRandom().nextIntBetween(0, partitionSize));
-    }
-}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/AbstractSSTableBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/AbstractSSTableBench.java
new file mode 100644
index 000000000000..8f896a8f0ad4
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/AbstractSSTableBench.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+public abstract class AbstractSSTableBench
+{
+    public static final String KEYSPACE = "SSTableWriterBench";
+    public static final String TABLE = "table";
+    public static final String TABLE_WITH_CLUSTERING = "table_with_clustering";
+
+    public SSTableFormat<?, ?> getFormat(String formatName)
+    {
+        return DatabaseDescriptor.getSSTableFormats().get(formatName);
+    }
+
+    public Keyspace prepareMetadata()
+    {
+        ServerTestUtils.daemonInitialization();
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE, 0, BytesType.instance, BytesType.instance, BytesType.instance),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE_WITH_CLUSTERING, 0, BytesType.instance, BytesType.instance, BytesType.instance));
+
+        CommitLog.instance.stopUnsafe(true);
+        return Keyspace.open(KEYSPACE);
+    }
+
+    /**
+     * Create partition keys from numbers in range {@code [min; max)} of size in bytes as in {@code keySize}
+     */
+    public DecoratedKey[] prepareDecoratedKeys(int min, int max, int keySize)
+    {
+        int n = max - min;
+        DecoratedKey[] keys = new DecoratedKey[n];
+        for (int i = 0; i < n; i++)
+        {
+            ByteBuffer buf = ByteBuffer.allocate(keySize);
+            buf.putInt(0, i + min);
+            keys[i] = Murmur3Partitioner.instance.decorateKey(buf.duplicate());
+        }
+        Arrays.sort(keys);
+        return keys;
+    }
+
+    /**
+     * Create clustering keys from numbers in range {@code [min; max)} of size in bytes as in {@code keySize}
+     */
+    public ByteBuffer[] prepareBuffers(int min, int max, int KEY_SIZE)
+    {
+        int n = max - min;
+        ByteBuffer[] ckeys = new ByteBuffer[n];
+        for (int i = 0; i < n; i++)
+        {
+            ckeys[i] = ByteBuffer.allocate(KEY_SIZE);
+            ckeys[i].putInt(0, i + min);
+        }
+        return ckeys;
+    }
+
+    public SSTableWriter createWriter(ColumnFamilyStore table, SSTableFormat<?, ?> format, LifecycleTransaction txn) throws Exception
+    {
+        Path tableDir = Files.createTempDirectory(getClass().getSimpleName());
+        Descriptor desc = table.newSSTableDescriptor(new File(tableDir), format);
+
+        return format.getWriterFactory().builder(desc)
+                     .setTableMetadataRef(table.metadata)
+                     .setSerializationHeader(new SerializationHeader(true, table.metadata(), table.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))
+                     .setSecondaryIndexGroups(table.indexManager.listIndexGroups())
+                     .addDefaultComponents(table.indexManager.listIndexGroups())
+                     .setMetadataCollector(new MetadataCollector(table.metadata().comparator))
+                     .build(txn, table);
+    }
+
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReaderBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReaderBench.java
new file mode 100644
index 000000000000..1ebe11d2728c
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReaderBench.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.util.FileUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 1, time = 5)
+@Measurement(iterations = 3, time = 5)
+@Fork(value = 1, jvmArgsAppend = "-Xmx4G")
+@Threads(1)
+@State(Scope.Benchmark)
+public class SSTableReaderBench extends AbstractSSTableBench
+{
+    private final static Logger logger = LoggerFactory.getLogger(SSTableReaderBench.class);
+
+    public final static int KEY_SIZE = 8;
+    public final static int P_KEYS = 2 << 10;
+    public final static int C_KEYS = 2 << 10;
+    public final static int VAL_SIZE = 1;
+
+    public ByteBuffer[] ckeys;
+    public DecoratedKey[] pkeys;
+    public DecoratedKey[] nonpkeys;
+
+    private SSTableReader sstr;
+    private SSTableWriter sstw;
+
+    private int idx = 0;
+
+    private final int step = P_KEYS / 2 - 1;
+
+    @Param({ "table", "table_with_clustering" })
+    public String tableName;
+
+    @Param({ "big", "bti" })
+    public String formatName;
+    private ColumnFamilyStore table;
+    private LifecycleTransaction txn;
+
+    @Setup(Level.Trial)
+    public void setup() throws Exception
+    {
+        assert Integer.highestOneBit(P_KEYS) == Integer.lowestOneBit(P_KEYS);
+        assert Integer.highestOneBit(C_KEYS) == Integer.lowestOneBit(C_KEYS);
+        Keyspace ks = prepareMetadata();
+        table = ks.getColumnFamilyStore(tableName);
+        pkeys = prepareDecoratedKeys(0, P_KEYS, KEY_SIZE);
+        nonpkeys = prepareDecoratedKeys(P_KEYS, P_KEYS * 2, KEY_SIZE);
+        ckeys = prepareBuffers(0, C_KEYS, KEY_SIZE);
+
+        txn = LifecycleTransaction.offline(OperationType.WRITE, table.metadata);
+        sstw = prepareTable(getFormat(formatName), table, txn);
+    }
+
+    @Setup(Level.Iteration)
+    public void setupIteration()
+    {
+        sstr = prepareReader(sstw);
+    }
+
+    /**
+     * Generates a quasi random walk over keys but adding a little less than a half and wrapping around.
+     */
+    private int nextIdx() {
+        idx += step;
+        if (idx >= P_KEYS)
+            idx -= P_KEYS;
+        return idx;
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    public void getEQPosition()
+    {
+        sstr.getPosition(pkeys[nextIdx()], SSTableReader.Operator.EQ);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    public void getEQPositionNonExisting()
+    {
+        sstr.getPosition(nonpkeys[nextIdx()], SSTableReader.Operator.EQ);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    public void getGTPosition()
+    {
+        sstr.getPosition(pkeys[nextIdx()], SSTableReader.Operator.GT);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    public void getGTPositionNonExisting()
+    {
+        sstr.getPosition(nonpkeys[nextIdx()], SSTableReader.Operator.GT);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.AverageTime)
+    public void iterateOverAllKeys() throws Exception
+    {
+        try (KeyIterator it = sstr.keyIterator())
+        {
+            while (it.hasNext()) it.next();
+        }
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.AverageTime)
+    public void fullScanTest()
+    {
+        try (ISSTableScanner scanner = sstr.getScanner())
+        {
+            while (scanner.hasNext())
+            {
+                UnfilteredRowIterator rowIt = scanner.next();
+                while (rowIt.hasNext())
+                {
+                    rowIt.next();
+                }
+            }
+        }
+    }
+
+    @TearDown(Level.Iteration)
+    public void tearDownIteration()
+    {
+        sstr.selfRef().release();
+    }
+
+    @TearDown(Level.Trial)
+    public void tearDown()
+    {
+        txn.finish();
+        txn.close();
+        FileUtils.deleteRecursive(sstr.descriptor.directory);
+    }
+
+    private SSTableWriter prepareTable(SSTableFormat format, ColumnFamilyStore table, LifecycleTransaction txn) throws Exception
+    {
+        try (SSTableWriter tableWriter = createWriter(table, format, txn))
+        {
+            for (int i = 0; i < P_KEYS; i++)
+            {
+                UpdateBuilder builder = UpdateBuilder.create(table.metadata(), pkeys[i].getKey().duplicate()).withTimestamp(1);
+                if (table.metadata().clusteringColumns().isEmpty())
+                    builder.newRow().add("val", ByteBuffer.allocate(VAL_SIZE));
+                else
+                    for (int j = 0; j < C_KEYS; j++)
+                        builder.newRow(ckeys[j].duplicate()).add("val", ByteBuffer.allocate(VAL_SIZE));
+
+                tableWriter.append(builder.build().unfilteredIterator());
+            }
+
+            tableWriter.prepareToCommit();
+            Throwable t = tableWriter.commit(null);
+            if (t != null)
+                throw new Exception(t);
+
+            logger.info("Created the following files: \n{}", Arrays.stream(tableWriter.descriptor.directory.list())
+                                                                   .map(f -> f.name() + " - " + FileUtils.stringifyFileSize(f.length()))
+                                                                   .collect(Collectors.joining("\n")));
+
+            return tableWriter;
+        }
+    }
+
+    private SSTableReader prepareReader(SSTableWriter tableWriter)
+    {
+        return SSTableReader.openNoValidation(table, tableWriter.descriptor, table.metadata);
+    }
+}
+
+
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableWriterBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableWriterBench.java
new file mode 100644
index 000000000000..431a11630305
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableWriterBench.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.schema.TableMetadata;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 1, time = 5)
+@Measurement(iterations = 3, time = 5)
+@Fork(value = 1, jvmArgsAppend = "-Xmx4G")
+@Threads(1)
+@State(Scope.Benchmark)
+public class SSTableWriterBench extends AbstractSSTableBench
+{
+    @Param({ "table", "table_with_clustering" })
+    public String tableName;
+
+    @Param({ "big", "bti" })
+    public String formatName;
+
+    public final static int KEY_SIZE = 8;
+    public final static int P_KEYS = 1000;
+    public final static int C_KEYS = 1000;
+    public final static int VAL_SIZE = 1;
+
+    public ByteBuffer[] ckeys = new ByteBuffer[C_KEYS];
+    public DecoratedKey[] pkeys = new DecoratedKey[P_KEYS];
+
+    private SSTableWriter tableWriter;
+    private TableMetadata tableMetadata;
+    boolean hasClustering = false;
+    private ColumnFamilyStore table;
+    private LifecycleTransaction txn;
+
+    @Setup(Level.Trial)
+    public void setupTrial()
+    {
+        Keyspace ks = prepareMetadata();
+        pkeys = prepareDecoratedKeys(0, P_KEYS, KEY_SIZE);
+
+        table = ks.getColumnFamilyStore(tableName);
+        tableMetadata = table.metadata();
+        hasClustering = !tableMetadata.clusteringColumns().isEmpty();
+
+        if (hasClustering)
+        {
+            ckeys = new ByteBuffer[C_KEYS];
+            for (int i = 0; i < ckeys.length; i++)
+            {
+                ckeys[i] = ByteBuffer.allocate(KEY_SIZE);
+                ckeys[i].putInt(0, i);
+            }
+        }
+    }
+
+    @Setup(Level.Invocation)
+    public void setupInvocation() throws Exception
+    {
+        txn = LifecycleTransaction.offline(OperationType.WRITE, table.metadata);
+        tableWriter = createWriter(table, getFormat(formatName), txn);
+    }
+
+    @Benchmark
+    public void writeWithClusteringTest()
+    {
+        for (int i = 0; i < P_KEYS; i++)
+        {
+            UpdateBuilder builder = UpdateBuilder.create(tableMetadata, pkeys[i].getKey().duplicate()).withTimestamp(1);
+            if (hasClustering)
+                for (int j = 0; j < C_KEYS; j++)
+                    builder.newRow(ckeys[j].duplicate()).add("val", ByteBuffer.allocate(VAL_SIZE));
+            else
+                builder.newRow().add("val", ByteBuffer.allocate(VAL_SIZE));
+
+            tableWriter.append(builder.build().unfilteredIterator());
+        }
+    }
+
+    @TearDown(Level.Invocation)
+    public void tearDown()
+    {
+        tableWriter.abort();
+        tableWriter.close();
+        txn.close();
+
+        tableWriter.descriptor.directory.deleteRecursive();
+    }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java
index f52ab28d6b9a..d4116f473e40 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java
@@ -31,12 +31,12 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
-import com.google.common.base.Throwables;
 import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.db.marshal.DecimalType;
 import org.apache.cassandra.db.marshal.IntegerType;
 import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.utils.ByteArrayUtil;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@@ -85,6 +85,18 @@ public class ComparisonReadBench
     @Param({"LONG"})
     TypeOption type = TypeOption.LONG;
 
+    public enum TrieAllocation {
+        SHORT_LIVED,
+        LONG_LIVED_ON_HEAP,
+        LONG_LIVED_OFF_HEAP
+    }
+
+    @Param({"SHORT_LIVED"})
+    static TrieAllocation allocation = TrieAllocation.SHORT_LIVED;
+
+    @Param({"OSS50"})
+    static ByteComparable.Version byteComparableVersion = ByteComparable.Version.OSS50;
+
     final static InMemoryTrie.UpsertTransformer<Byte, Byte> resolver = (x, y) -> y;
 
     Access<?> access;
@@ -155,7 +167,7 @@ public Long fromLong(long l)
 
         public Long fromByteComparable(ByteComparable bc)
         {
-            return ByteSourceInverse.getSignedLong(bc.asComparableBytes(ByteComparable.Version.OSS50));
+            return ByteSourceInverse.getSignedLong(bc.asComparableBytes(byteComparableVersion));
         }
 
         public ByteComparable longToByteComparable(long l)
@@ -178,8 +190,8 @@ public BigInteger fromLong(long l)
 
         public BigInteger fromByteComparable(ByteComparable bc)
         {
-            return IntegerType.instance.compose(IntegerType.instance.fromComparableBytes(ByteSource.peekable(bc.asComparableBytes(ByteComparable.Version.OSS50)),
-                                                                                         ByteComparable.Version.OSS50));
+            return IntegerType.instance.compose(IntegerType.instance.fromComparableBytes(ByteSource.peekable(bc.asComparableBytes(byteComparableVersion)),
+                                                                                         byteComparableVersion));
         }
 
         public ByteComparable longToByteComparable(long l)
@@ -202,8 +214,8 @@ public BigDecimal fromLong(long l)
 
         public BigDecimal fromByteComparable(ByteComparable bc)
         {
-            return DecimalType.instance.compose(DecimalType.instance.fromComparableBytes(ByteSource.peekable(bc.asComparableBytes(ByteComparable.Version.OSS50)),
-                                                                                         ByteComparable.Version.OSS50));
+            return DecimalType.instance.compose(DecimalType.instance.fromComparableBytes(ByteSource.peekable(bc.asComparableBytes(byteComparableVersion)),
+                                                                                         byteComparableVersion));
         }
 
         public ByteComparable longToByteComparable(long l)
@@ -233,12 +245,12 @@ public String fromLong(long l)
 
         public String fromByteComparable(ByteComparable bc)
         {
-            return new String(ByteSourceInverse.readBytes(bc.asComparableBytes(ByteComparable.Version.OSS50)), StandardCharsets.UTF_8);
+            return new String(ByteSourceInverse.readBytes(bc.asComparableBytes(byteComparableVersion)), StandardCharsets.UTF_8);
         }
 
         public ByteComparable longToByteComparable(long l)
         {
-            return ByteComparable.fixedLength(fromLong(l).getBytes(StandardCharsets.UTF_8));
+            return ByteComparable.preencoded(byteComparableVersion, fromLong(l).getBytes(StandardCharsets.UTF_8));
         }
 
         public Comparator<String> comparator()
@@ -269,12 +281,12 @@ public byte[] fromLong(long l)
 
         public byte[] fromByteComparable(ByteComparable bc)
         {
-            return ByteSourceInverse.readBytes(bc.asComparableBytes(ByteComparable.Version.OSS50));
+            return ByteSourceInverse.readBytes(bc.asComparableBytes(byteComparableVersion));
         }
 
         public ByteComparable longToByteComparable(long l)
         {
-            return ByteComparable.fixedLength(fromLong(l));
+            return ByteComparable.preencoded(byteComparableVersion, fromLong(l));
         }
 
         public Comparator<byte[]> comparator()
@@ -288,7 +300,7 @@ interface Access<T>
         void put(long v, byte b);
         byte get(long v);
         Iterable<Byte> values();
-        Iterable<Byte> valuesSlice(long left, boolean includeLeft, long right, boolean includeRight);
+        Iterable<Byte> valuesSlice(long left, long right);
         Iterable<Map.Entry<T, Byte>> entrySet();
         void consumeValues(Consumer<Byte> consumer);
         void consumeEntries(BiConsumer<T, Byte> consumer);
@@ -310,7 +322,20 @@ class TrieAccess<T> implements Access<T>
         TrieAccess(Type<T> type)
         {
             this.type = type;
-            trie = new InMemoryTrie<>(bufferType);
+            switch (allocation)
+            {
+                case SHORT_LIVED:
+                    trie = InMemoryTrie.shortLived(byteComparableVersion);
+                    break;
+                case LONG_LIVED_ON_HEAP:
+                    trie = InMemoryTrie.longLived(byteComparableVersion, BufferType.ON_HEAP, null);
+                    break;
+                case LONG_LIVED_OFF_HEAP:
+                    trie = InMemoryTrie.longLived(byteComparableVersion, BufferType.OFF_HEAP, null);
+                    break;
+                default:
+                    throw new AssertionError();
+            };
         }
 
         public void put(long v, byte b)
@@ -319,9 +344,9 @@ public void put(long v, byte b)
             {
                 trie.putRecursive(type.longToByteComparable(v), b, resolver);
             }
-            catch (InMemoryTrie.SpaceExhaustedException e)
+            catch (TrieSpaceExhaustedException e)
             {
-                throw Throwables.propagate(e);
+                throw new AssertionError(e);
             }
         }
 
@@ -335,9 +360,9 @@ public Iterable<Byte> values()
             return trie.values();
         }
 
-        public Iterable<Byte> valuesSlice(long left, boolean includeLeft, long right, boolean includeRight)
+        public Iterable<Byte> valuesSlice(long left, long right)
         {
-            return trie.subtrie(type.longToByteComparable(left), includeLeft, type.longToByteComparable(right), includeRight)
+            return trie.subtrie(type.longToByteComparable(left), type.longToByteComparable(right))
                        .values();
         }
 
@@ -362,9 +387,9 @@ public void printSize()
         {
             long deepsize = meter.measureDeep(trie);
             System.out.format("Trie size on heap %,d off heap %,d deep size %,d\n",
-                              trie.sizeOnHeap(), trie.sizeOffHeap(), deepsize);
+                              trie.usedSizeOnHeap(), trie.usedSizeOffHeap(), deepsize);
             System.out.format("per entry on heap %.2f off heap %.2f deep size %.2f\n",
-                              trie.sizeOnHeap() * 1.0 / count, trie.sizeOffHeap() * 1.0 / count, deepsize * 1.0 / count);
+                              trie.usedSizeOnHeap() * 1.0 / count, trie.usedSizeOffHeap() * 1.0 / count, deepsize * 1.0 / count);
         }
     }
 
@@ -394,9 +419,9 @@ public Iterable<Byte> values()
             return navigableMap.values();
         }
 
-        public Iterable<Byte> valuesSlice(long left, boolean includeLeft, long right, boolean includeRight)
+        public Iterable<Byte> valuesSlice(long left, long right)
         {
-            return navigableMap.subMap(type.fromLong(left), includeLeft, type.fromLong(right), includeRight)
+            return navigableMap.subMap(type.fromLong(left), type.fromLong(right))
                                .values();
         }
 
@@ -500,7 +525,7 @@ public int getByIterateValueSlice()
         for (int i = 0; i < count; ++i)
         {
             long v = rand.nextLong();
-            Iterable<Byte> values = access.valuesSlice(v, true, v, true);
+            Iterable<Byte> values = access.valuesSlice(v, v);
             for (byte b : values)
                 sum += b;
         }
@@ -511,7 +536,7 @@ public int getByIterateValueSlice()
     public int iterateValuesLimited()
     {
         int sum = 0;
-        Iterable<Byte> values = access.valuesSlice(0L, false, Long.MAX_VALUE / 2, true); // 1/4
+        Iterable<Byte> values = access.valuesSlice(0L, Long.MAX_VALUE / 2); // 1/4
         for (byte b : values)
             sum += b;
         return sum;
diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java
index cff2e4a3ebed..8004d00554f4 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java
@@ -22,6 +22,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 
+import org.apache.cassandra.db.tries.Direction;
 import org.apache.cassandra.db.tries.InMemoryTrie;
 import org.apache.cassandra.db.tries.Trie;
 import org.apache.cassandra.db.tries.TrieEntriesWalker;
@@ -41,9 +42,15 @@ public class InMemoryTrieReadBench
     @Param({"ON_HEAP", "OFF_HEAP"})
     BufferType bufferType = BufferType.OFF_HEAP;
 
+    @Param({"OSS50"})
+    ByteComparable.Version byteComparableVersion = ByteComparable.Version.OSS50;
+
     @Param({"1000", "100000", "10000000"})
     int count = 1000;
 
+    @Param({"FORWARD"})
+    Direction direction = Direction.FORWARD;
+
     final static InMemoryTrie.UpsertTransformer<Byte, Byte> resolver = (x, y) -> y;
 
     InMemoryTrie<Byte> trie;
@@ -51,7 +58,7 @@ public class InMemoryTrieReadBench
     @Setup(Level.Trial)
     public void setup() throws Throwable
     {
-        trie = new InMemoryTrie<>(bufferType);
+        trie = InMemoryTrie.longLived(byteComparableVersion, bufferType, null);
         Random rand = new Random(1);
 
         System.out.format("Putting %,d\n", count);
@@ -61,9 +68,9 @@ public void setup() throws Throwable
             trie.putRecursive(ByteComparable.of(l), Byte.valueOf((byte) (l >> 56)), resolver);
         }
         System.out.format("Trie size on heap %,d off heap %,d\n",
-                          trie.sizeOnHeap(), trie.sizeOffHeap());
+                          trie.usedSizeOnHeap(), trie.usedSizeOffHeap());
         System.out.format("per entry on heap %.2f off heap %.2f\n",
-                          trie.sizeOnHeap() * 1.0 / count, trie.sizeOffHeap() * 1.0 / count);
+                          trie.usedSizeOnHeap() * 1.0 / count, trie.usedSizeOffHeap() * 1.0 / count);
     }
 
     @Benchmark
@@ -110,12 +117,12 @@ public void accept(Byte aByte)
     @Benchmark
     public int consumeEntries()
     {
-        class Counter implements BiConsumer<ByteComparable, Byte>
+        class Counter implements BiConsumer<ByteComparable.Preencoded, Byte>
         {
             int sum = 0;
 
             @Override
-            public void accept(ByteComparable byteComparable, Byte aByte)
+            public void accept(ByteComparable.Preencoded byteComparable, Byte aByte)
             {
                 sum += aByte;
             }
@@ -145,7 +152,7 @@ public Void complete()
             }
         }
         Counter counter = new Counter();
-        trie.process(counter);
+        trie.process(counter, direction);
         return counter.sum;
     }
 
@@ -162,7 +169,7 @@ public int iterateValuesUnordered()
     public int iterateEntries()
     {
         int sum = 0;
-        for (Map.Entry<ByteComparable, Byte> en : trie.entrySet())
+        for (Map.Entry<ByteComparable.Preencoded, Byte> en : trie.entrySet(direction))
             sum += en.getValue();
         return sum;
     }
@@ -171,9 +178,7 @@ public int iterateEntries()
     public int iterateValuesLimited()
     {
         Iterable<Byte> values = trie.subtrie(ByteComparable.of(0L),
-                                             true,
-                                             ByteComparable.of(Long.MAX_VALUE / 2),         // 1/4 of all
-                                             false)
+                                             ByteComparable.of(Long.MAX_VALUE / 2))         // 1/4 of all
                                     .values();
         int sum = 0;
         for (byte b : values)
diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java
index e32e20f7e6a3..643cc1a579b8 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java
@@ -43,6 +43,9 @@ public class InMemoryTrieUnionBench
     @Param({"ON_HEAP", "OFF_HEAP"})
     BufferType bufferType = BufferType.OFF_HEAP;
 
+    @Param({"OSS50"})
+    ByteComparable.Version byteComparableVersion = ByteComparable.Version.OSS50;
+
     @Param({"1000", "100000", "10000000"})
     int count = 1000;
 
@@ -66,7 +69,7 @@ public void setup() throws Throwable
         {
             long sz = 65536 / sources;
             for (int i = 0; i < sources; ++i)
-                tries.add(new InMemoryTrie<>(bufferType));
+                tries.add(InMemoryTrie.longLived(byteComparableVersion, bufferType, null));
 
             for (long current = 0; current < count; ++current)
             {
@@ -81,7 +84,7 @@ public void setup() throws Throwable
             long current = 0;
             for (int i = 0; i < sources; ++i)
             {
-                InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+                InMemoryTrie<Byte> trie = InMemoryTrie.longLived(byteComparableVersion, bufferType, null);
                 int currMax = this.count * (i + 1) / sources;
 
                 for (; current < currMax; ++current)
@@ -96,7 +99,7 @@ public void setup() throws Throwable
         for (InMemoryTrie<Byte> trie : tries)
         {
             System.out.format("Trie size on heap %,d off heap %,d\n",
-                              trie.sizeOnHeap(), trie.sizeOffHeap());
+                              trie.usedSizeOnHeap(), trie.usedSizeOffHeap());
         }
         trie = Trie.mergeDistinct(tries);
 
@@ -125,7 +128,7 @@ public int iterateValuesUnordered()
     public int iterateEntries()
     {
         int sum = 0;
-        for (Map.Entry<ByteComparable, Byte> en : trie.entrySet())
+        for (Map.Entry<ByteComparable. Preencoded, Byte> en : trie.entrySet())
             sum += en.getValue();
         return sum;
     }
@@ -134,9 +137,7 @@ public int iterateEntries()
     public int iterateValuesLimited()
     {
         Iterable<Byte> values = trie.subtrie(ByteComparable.of(0L),
-                                             true,
-                                             ByteComparable.of(Long.MAX_VALUE / 2),         // 1/4 of all
-                                             false)
+                                             ByteComparable.of(Long.MAX_VALUE / 2))         // 1/4 of all
                                     .values();
         int sum = 0;
         for (byte b : values)
diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieWriteBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieWriteBench.java
index f2be11dd8aff..2b91511fb33c 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieWriteBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieWriteBench.java
@@ -22,9 +22,22 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.TrieSpaceExhaustedException;
 import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.openjdk.jmh.annotations.*;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 
 @BenchmarkMode(Mode.AverageTime)
@@ -39,6 +52,9 @@ public class InMemoryTrieWriteBench
     @Param({"ON_HEAP", "OFF_HEAP"})
     BufferType bufferType = BufferType.OFF_HEAP;
 
+    @Param({"OSS50"})
+    ByteComparable.Version byteComparableVersion = ByteComparable.Version.OSS50;
+
     @Param({"1000", "100000", "10000000"})
     int count = 1000;
 
@@ -52,70 +68,86 @@ public class InMemoryTrieWriteBench
     final static boolean PRINT_SIZES = false;
 
     @Benchmark
-    public void putSequential(Blackhole bh) throws InMemoryTrie.SpaceExhaustedException
+    public void putSequential(Blackhole bh) throws TrieSpaceExhaustedException
     {
-        InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+        InMemoryTrie<Byte> trie = InMemoryTrie.longLived(byteComparableVersion, bufferType, null);
         ByteBuffer buf = ByteBuffer.allocate(keyLength);
 
         for (long current = 0; current < count; ++current)
         {
             long l = current;
             buf.putLong(keyLength - 8, l);
-            trie.putRecursive(ByteComparable.fixedLength(buf), Byte.valueOf((byte) (l >> 56)), resolver);
+            trie.putRecursive(v -> ByteSource.preencoded(buf), Byte.valueOf((byte) (l >> 56)), resolver);
         }
         if (PRINT_SIZES)
-            System.out.println(trie.valuesCount());
+        {
+            System.out.println(String.format("Size on heap %s off heap %s",
+                                             FBUtilities.prettyPrintMemory(trie.usedSizeOnHeap()),
+                                             FBUtilities.prettyPrintMemory(trie.usedSizeOffHeap())));
+        }
         bh.consume(trie);
     }
 
     @Benchmark
-    public void putRandom(Blackhole bh) throws InMemoryTrie.SpaceExhaustedException
+    public void putRandom(Blackhole bh) throws TrieSpaceExhaustedException
     {
-        InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+        InMemoryTrie<Byte> trie = InMemoryTrie.longLived(byteComparableVersion, bufferType, null);
         Random rand = new Random(1);
         byte[] buf = new byte[keyLength];
 
         for (long current = 0; current < count; ++current)
         {
             rand.nextBytes(buf);
-            trie.putRecursive(ByteComparable.fixedLength(buf), Byte.valueOf(buf[0]), resolver);
+            trie.putRecursive(v -> ByteSource.preencoded(buf), Byte.valueOf(buf[0]), resolver);
         }
         if (PRINT_SIZES)
-            System.out.println(trie.valuesCount());
+        {
+            System.out.println(String.format("Size on heap %s off heap %s",
+                                             FBUtilities.prettyPrintMemory(trie.usedSizeOnHeap()),
+                                             FBUtilities.prettyPrintMemory(trie.usedSizeOffHeap())));
+        }
         bh.consume(trie);
     }
 
     @Benchmark
-    public void applySequential(Blackhole bh) throws InMemoryTrie.SpaceExhaustedException
+    public void applySequential(Blackhole bh) throws TrieSpaceExhaustedException
     {
-        InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+        InMemoryTrie<Byte> trie = InMemoryTrie.longLived(byteComparableVersion, bufferType, null);
         ByteBuffer buf = ByteBuffer.allocate(keyLength);
 
         for (long current = 0; current < count; ++current)
         {
             long l = current;
             buf.putLong(keyLength - 8, l);
-            trie.putSingleton(ByteComparable.fixedLength(buf), Byte.valueOf((byte) (l >> 56)), resolver);
+            trie.putSingleton(v -> ByteSource.preencoded(buf), Byte.valueOf((byte) (l >> 56)), resolver);
         }
         if (PRINT_SIZES)
-            System.out.println(trie.valuesCount());
+        {
+            System.out.println(String.format("Size on heap %s off heap %s",
+                                             FBUtilities.prettyPrintMemory(trie.usedSizeOnHeap()),
+                                             FBUtilities.prettyPrintMemory(trie.usedSizeOffHeap())));
+        }
         bh.consume(trie);
     }
 
     @Benchmark
-    public void applyRandom(Blackhole bh) throws InMemoryTrie.SpaceExhaustedException
+    public void applyRandom(Blackhole bh) throws TrieSpaceExhaustedException
     {
-        InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+        InMemoryTrie<Byte> trie = InMemoryTrie.longLived(byteComparableVersion, bufferType, null);
         Random rand = new Random(1);
         byte[] buf = new byte[keyLength];
 
         for (long current = 0; current < count; ++current)
         {
             rand.nextBytes(buf);
-            trie.putSingleton(ByteComparable.fixedLength(buf), Byte.valueOf(buf[0]), resolver);
+            trie.putSingleton(v -> ByteSource.preencoded(buf), Byte.valueOf(buf[0]), resolver);
         }
         if (PRINT_SIZES)
-            System.out.println(trie.valuesCount());
+        {
+            System.out.println(String.format("Size on heap %s off heap %s",
+                                             FBUtilities.prettyPrintMemory(trie.usedSizeOnHeap()),
+                                             FBUtilities.prettyPrintMemory(trie.usedSizeOffHeap())));
+        }
         bh.consume(trie);
     }
 }
diff --git a/test/resources/org/apache/cassandra/tools/compaction_logs/compaction-UnifiedCompactionStrategy-density-blobs-2-repaired.csv b/test/resources/org/apache/cassandra/tools/compaction_logs/compaction-UnifiedCompactionStrategy-density-blobs-2-repaired.csv
new file mode 100644
index 000000000000..162ed261d0a2
--- /dev/null
+++ b/test/resources/org/apache/cassandra/tools/compaction_logs/compaction-UnifiedCompactionStrategy-density-blobs-2-repaired.csv
@@ -0,0 +1,1335 @@
+Timestamp,Event,Level,W,Min Density,Max Density,Overlap,Tot. SSTables,Tot. size (bytes),Compactions,Comp. SSTables,Read (bytes/sec),Write (bytes/sec),Tot. comp. size/Read/Written (bytes)
+2022-12-16 15:06:25.027,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.743GiB,3/1,12/4,0B/s,0B/s,2.349GiB/0B/0B
+2022-12-16 15:06:25.127,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.743GiB,4/2,16/8,0B/s,0B/s,3.133GiB/0B/0B
+2022-12-16 15:06:25.152,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.743GiB,3/3,12/12,0B/s,0B/s,2.349GiB/0B/0B
+2022-12-16 15:06:25.164,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.743GiB,4/4,16/16,0B/s,0B/s,3.133GiB/0B/0B
+2022-12-16 15:08:49.778,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,14.252MiB/s,14.292MiB/s,4.699GiB/2.013GiB/2.019GiB
+2022-12-16 15:08:49.795,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/6,32/24,14.251MiB/s,14.292MiB/s,6.265GiB/2.013GiB/2.019GiB
+2022-12-16 15:08:49.819,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,14.248MiB/s,14.292MiB/s,5.482GiB/2.013GiB/2.019GiB
+2022-12-16 15:08:49.839,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,14.246MiB/s,14.291MiB/s,6.265GiB/2.013GiB/2.019GiB
+2022-12-16 15:10:17.995,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,26.361MiB/s,26.411MiB/s,6.265GiB/4.200GiB/4.208GiB
+2022-12-16 15:10:19.607,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.701MiB/s,22.788MiB/s,5.481GiB/3.429GiB/3.439GiB
+2022-12-16 15:10:19.607,completed,1,T4,2.128 GiB,8.513 GiB,1,1,702.713MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:10:22.812,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,19.236MiB/s,19.257MiB/s,4.698GiB/2.699GiB/2.703GiB
+2022-12-16 15:10:22.813,completed,1,T4,2.128 GiB,8.513 GiB,1,2,1.372GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:10:23.390,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.785MiB/s,15.853MiB/s,3.915GiB/1.918GiB/1.924GiB
+2022-12-16 15:10:23.391,completed,1,T4,2.128 GiB,8.513 GiB,1,3,2.059GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:11:09.639,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,12.248MiB/s,12.287MiB/s,5.481GiB/1.672GiB/1.678GiB
+2022-12-16 15:11:09.639,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:11:09.641,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.248MiB/s,12.288MiB/s,5.481GiB/1.672GiB/1.678GiB
+2022-12-16 15:11:09.641,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:11:09.646,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.247MiB/s,12.289MiB/s,5.481GiB/1.672GiB/1.678GiB
+2022-12-16 15:11:09.647,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:11:09.658,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.246MiB/s,12.292MiB/s,6.264GiB/1.672GiB/1.679GiB
+2022-12-16 15:11:09.658,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:12:54.147,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,25.881MiB/s,26.021MiB/s,6.264GiB/4.383GiB/4.401GiB
+2022-12-16 15:12:54.147,completed,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:13:00.207,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.652MiB/s,22.741MiB/s,5.481GiB/3.741GiB/3.752GiB
+2022-12-16 15:13:00.208,completed,1,T4,2.128 GiB,8.513 GiB,2,5,3.431GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:13:03.930,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,19.472MiB/s,19.498MiB/s,4.698GiB/3.032GiB/3.036GiB
+2022-12-16 15:13:03.931,completed,1,T4,2.128 GiB,8.513 GiB,2,6,4.117GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:13:05.731,completed,0,T4,0.000 B,2.128 GiB,3,32,5.486GiB,5/5,20/20,16.248MiB/s,16.327MiB/s,3.916GiB/2.270GiB/2.279GiB
+2022-12-16 15:13:05.731,completed,1,T4,2.128 GiB,8.513 GiB,2,7,4.803GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:13:40.934,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,13.027MiB/s,13.102MiB/s,4.699GiB/1.925GiB/1.936GiB
+2022-12-16 15:13:40.937,submitted,1,T4,2.128 GiB,8.513 GiB,2,8,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:13:40.950,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,13.026MiB/s,13.102MiB/s,5.481GiB/1.925GiB/1.936GiB
+2022-12-16 15:13:40.954,submitted,1,T4,2.128 GiB,8.513 GiB,2,8,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:13:40.977,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,13.024MiB/s,13.102MiB/s,6.265GiB/1.925GiB/1.936GiB
+2022-12-16 15:13:40.981,submitted,1,T4,2.128 GiB,8.513 GiB,2,8,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:13:40.992,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,13.022MiB/s,13.103MiB/s,6.265GiB/1.925GiB/1.937GiB
+2022-12-16 15:13:40.994,submitted,1,T4,2.128 GiB,8.513 GiB,2,8,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:15:04.108,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,27.546MiB/s,27.636MiB/s,6.265GiB/4.214GiB/4.224GiB
+2022-12-16 15:15:04.113,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:15:08.115,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,23.862MiB/s,24.061MiB/s,5.482GiB/3.497GiB/3.519GiB
+2022-12-16 15:15:08.116,completed,1,T4,2.128 GiB,8.513 GiB,3,9,6.176GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:15:13.229,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,20.407MiB/s,20.557MiB/s,4.699GiB/2.808GiB/2.824GiB
+2022-12-16 15:15:13.232,completed,1,T4,2.128 GiB,8.513 GiB,3,10,6.862GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:15:14.452,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,17.215MiB/s,17.243MiB/s,3.915GiB/2.056GiB/2.058GiB
+2022-12-16 15:15:14.457,completed,1,T4,2.128 GiB,8.513 GiB,3,11,7.548GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:16:00.304,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,13.578MiB/s,13.609MiB/s,5.482GiB/1.847GiB/1.852GiB
+2022-12-16 15:16:00.313,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:16:00.332,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,13.575MiB/s,13.608MiB/s,5.482GiB/1.847GiB/1.852GiB
+2022-12-16 15:16:00.334,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:16:00.357,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,13.573MiB/s,13.608MiB/s,5.482GiB/1.847GiB/1.852GiB
+2022-12-16 15:16:00.359,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:16:00.378,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,13.571MiB/s,13.608MiB/s,6.264GiB/1.847GiB/1.852GiB
+2022-12-16 15:16:00.381,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:17:34.734,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,27.641MiB/s,27.766MiB/s,6.264GiB/4.393GiB/4.408GiB
+2022-12-16 15:17:34.734,completed,1,T4,2.128 GiB,8.513 GiB,3,12,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:17:34.738,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,24.210MiB/s,24.334MiB/s,5.481GiB/3.609GiB/3.625GiB
+2022-12-16 15:17:34.739,submitted,1,T4,2.128 GiB,8.513 GiB,4,13,8.921GiB,1/1,4/4,0B/s,0B/s,3.133GiB/0B/0B
+2022-12-16 15:17:37.988,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,24.114MiB/s,24.205MiB/s,5.481GiB/3.675GiB/3.685GiB
+2022-12-16 15:17:37.989,completed,1,T4,2.128 GiB,8.513 GiB,3,13,8.921GiB,1/1,4/4,2.729MiB/s,2.906MiB/s,3.133GiB/8.847MiB/9.420MiB
+2022-12-16 15:17:37.995,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,20.731MiB/s,20.822MiB/s,4.699GiB/2.892GiB/2.902GiB
+2022-12-16 15:17:37.995,submitted,1,T4,2.128 GiB,8.513 GiB,4,14,9.607GiB,2/2,8/8,2.724MiB/s,2.905MiB/s,6.266GiB/8.847MiB/9.438MiB
+2022-12-16 15:17:38.067,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,20.775MiB/s,20.823MiB/s,4.699GiB/2.897GiB/2.904GiB
+2022-12-16 15:17:38.068,completed,1,T4,2.128 GiB,8.513 GiB,3,14,9.607GiB,2/2,8/8,2.664MiB/s,2.883MiB/s,6.266GiB/8.847MiB/9.572MiB
+2022-12-16 15:17:38.076,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,17.390MiB/s,17.439MiB/s,3.915GiB/2.114GiB/2.120GiB
+2022-12-16 15:17:38.086,submitted,1,T4,2.128 GiB,8.513 GiB,4,15,10.293GiB,3/3,12/12,2.657MiB/s,2.881MiB/s,9.400GiB/8.847MiB/9.594MiB
+2022-12-16 15:17:40.310,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,17.496MiB/s,17.575MiB/s,3.915GiB/2.164GiB/2.172GiB
+2022-12-16 15:17:40.310,completed,1,T4,2.128 GiB,8.513 GiB,3,15,10.293GiB,3/3,12/12,9.960MiB/s,12.240MiB/s,9.400GiB/31.613MiB/39.393MiB
+2022-12-16 15:17:40.314,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.114GiB,4/4,16/16,14.146MiB/s,14.224MiB/s,3.132GiB/1.381GiB/1.388GiB
+2022-12-16 15:17:40.314,submitted,1,T4,2.128 GiB,8.513 GiB,4,16,10.980GiB,4/4,16/16,9.946MiB/s,12.224MiB/s,12.534GiB/31.613MiB/39.395MiB
+2022-12-16 15:18:30.708,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,13.749MiB/s,13.792MiB/s,5.481GiB/2.019GiB/2.025GiB
+2022-12-16 15:18:30.708,submitted,1,T4,2.128 GiB,8.513 GiB,4,16,10.980GiB,4/4,16/16,12.898MiB/s,13.142MiB/s,12.534GiB/682.291MiB/695.215MiB
+2022-12-16 15:18:30.722,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,13.748MiB/s,13.793MiB/s,5.481GiB/2.019GiB/2.025GiB
+2022-12-16 15:18:30.732,submitted,1,T4,2.128 GiB,8.513 GiB,4,16,10.980GiB,4/4,16/16,12.895MiB/s,13.142MiB/s,12.534GiB/682.291MiB/695.424MiB
+2022-12-16 15:18:30.747,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,13.746MiB/s,13.793MiB/s,5.481GiB/2.019GiB/2.026GiB
+2022-12-16 15:18:30.748,submitted,1,T4,2.128 GiB,8.513 GiB,4,16,10.980GiB,4/4,16/16,12.888MiB/s,13.143MiB/s,12.534GiB/682.291MiB/695.768MiB
+2022-12-16 15:18:30.754,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,13.745MiB/s,13.792MiB/s,6.264GiB/2.019GiB/2.026GiB
+2022-12-16 15:18:30.755,submitted,1,T4,2.128 GiB,8.513 GiB,4,16,10.980GiB,4/4,16/16,12.957MiB/s,13.142MiB/s,12.534GiB/685.993MiB/695.852MiB
+2022-12-16 15:19:52.575,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,26.489MiB/s,26.572MiB/s,6.264GiB/4.099GiB/4.109GiB
+2022-12-16 15:19:52.576,completed,1,T4,2.128 GiB,8.513 GiB,4,16,10.980GiB,4/4,16/16,12.774MiB/s,12.859MiB/s,12.534GiB/1.681GiB/1.692GiB
+2022-12-16 15:19:59.044,completed,0,T4,0.000 B,2.128 GiB,2,36,6.171GiB,7/7,28/28,22.821MiB/s,22.942MiB/s,5.481GiB/3.436GiB/3.449GiB
+2022-12-16 15:19:59.044,completed,1,T4,2.128 GiB,8.513 GiB,1,17,11.666GiB,4/4,16/16,12.652MiB/s,12.760MiB/s,12.534GiB/1.745GiB/1.760GiB
+2022-12-16 15:20:00.209,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,19.465MiB/s,19.513MiB/s,4.698GiB/2.675GiB/2.680GiB
+2022-12-16 15:20:00.209,completed,1,T4,2.128 GiB,8.513 GiB,1,18,12.352GiB,4/4,16/16,12.690MiB/s,12.733MiB/s,12.534GiB/1.765GiB/1.770GiB
+2022-12-16 15:20:04.738,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,16.028MiB/s,16.079MiB/s,3.915GiB/1.953GiB/1.957GiB
+2022-12-16 15:20:04.744,completed,1,T4,2.128 GiB,8.513 GiB,1,19,13.038GiB,4/4,16/16,12.613MiB/s,12.696MiB/s,12.534GiB/1.810GiB/1.821GiB
+2022-12-16 15:20:57.918,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,12.876MiB/s,12.946MiB/s,5.482GiB/1.851GiB/1.861GiB
+2022-12-16 15:20:57.926,submitted,1,T4,2.128 GiB,8.513 GiB,1,20,13.724GiB,4/4,16/16,12.621MiB/s,12.702MiB/s,12.534GiB/2.466GiB/2.482GiB
+2022-12-16 15:20:57.949,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,12.873MiB/s,12.945MiB/s,4.699GiB/1.851GiB/1.861GiB
+2022-12-16 15:20:57.950,submitted,1,T4,2.128 GiB,8.513 GiB,1,20,13.724GiB,4/4,16/16,12.639MiB/s,12.702MiB/s,12.534GiB/2.470GiB/2.482GiB
+2022-12-16 15:20:57.971,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,12.872MiB/s,12.945MiB/s,6.264GiB/1.851GiB/1.861GiB
+2022-12-16 15:20:57.971,submitted,1,T4,2.128 GiB,8.513 GiB,1,20,13.724GiB,4/4,16/16,12.638MiB/s,12.702MiB/s,12.534GiB/2.470GiB/2.483GiB
+2022-12-16 15:20:57.982,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.871MiB/s,12.945MiB/s,6.264GiB/1.851GiB/1.861GiB
+2022-12-16 15:20:57.983,submitted,1,T4,2.128 GiB,8.513 GiB,1,20,13.724GiB,4/4,16/16,12.637MiB/s,12.702MiB/s,12.534GiB/2.470GiB/2.483GiB
+2022-12-16 15:22:35.993,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,25.929MiB/s,25.969MiB/s,6.264GiB/4.342GiB/4.349GiB
+2022-12-16 15:22:36.001,completed,1,T4,2.128 GiB,8.513 GiB,1,20,13.724GiB,4/4,16/16,12.628MiB/s,12.666MiB/s,12.534GiB/3.677GiB/3.688GiB
+2022-12-16 15:22:38.828,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.502MiB/s,22.576MiB/s,5.481GiB/3.603GiB/3.613GiB
+2022-12-16 15:22:38.841,completed,1,T4,2.128 GiB,8.513 GiB,2,21,14.410GiB,4/4,16/16,12.573MiB/s,12.638MiB/s,12.534GiB/3.696GiB/3.715GiB
+2022-12-16 15:22:40.273,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,19.307MiB/s,19.349MiB/s,4.698GiB/2.853GiB/2.857GiB
+2022-12-16 15:22:40.274,completed,1,T4,2.128 GiB,8.513 GiB,2,22,15.096GiB,4/4,16/16,12.595MiB/s,12.646MiB/s,12.534GiB/3.720GiB/3.735GiB
+2022-12-16 15:22:40.695,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,16.033MiB/s,16.118MiB/s,3.915GiB/2.069GiB/2.078GiB
+2022-12-16 15:22:40.695,completed,1,T4,2.128 GiB,8.513 GiB,2,23,15.783GiB,4/4,16/16,12.577MiB/s,12.641MiB/s,12.534GiB/3.720GiB/3.739GiB
+2022-12-16 15:23:21.016,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,13.090MiB/s,13.152MiB/s,4.699GiB/1.829GiB/1.837GiB
+2022-12-16 15:23:21.023,submitted,1,T4,2.128 GiB,8.513 GiB,2,24,16.469GiB,4/4,16/16,12.672MiB/s,12.724MiB/s,12.534GiB/4.247GiB/4.264GiB
+2022-12-16 15:23:21.037,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,13.089MiB/s,13.152MiB/s,4.699GiB/1.829GiB/1.837GiB
+2022-12-16 15:23:21.038,submitted,1,T4,2.128 GiB,8.513 GiB,2,24,16.469GiB,4/4,16/16,12.671MiB/s,12.724MiB/s,12.534GiB/4.247GiB/4.265GiB
+2022-12-16 15:23:21.052,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,13.087MiB/s,13.153MiB/s,5.482GiB/1.829GiB/1.838GiB
+2022-12-16 15:23:21.066,submitted,1,T4,2.128 GiB,8.513 GiB,2,24,16.469GiB,4/4,16/16,12.671MiB/s,12.724MiB/s,12.534GiB/4.247GiB/4.265GiB
+2022-12-16 15:23:21.085,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,13.084MiB/s,13.152MiB/s,6.264GiB/1.829GiB/1.838GiB
+2022-12-16 15:23:21.086,submitted,1,T4,2.128 GiB,8.513 GiB,2,24,16.469GiB,4/4,16/16,12.669MiB/s,12.724MiB/s,12.534GiB/4.247GiB/4.265GiB
+2022-12-16 15:25:03.835,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,25.809MiB/s,25.914MiB/s,6.264GiB/4.402GiB/4.416GiB
+2022-12-16 15:25:03.835,completed,1,T4,2.128 GiB,8.513 GiB,2,24,16.469GiB,4/4,16/16,12.598MiB/s,12.655MiB/s,12.534GiB/5.487GiB/5.512GiB
+2022-12-16 15:25:05.241,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.583MiB/s,22.626MiB/s,5.481GiB/3.654GiB/3.660GiB
+2022-12-16 15:25:05.246,completed,1,T4,2.128 GiB,8.513 GiB,3,25,17.155GiB,4/4,16/16,12.602MiB/s,12.650MiB/s,12.534GiB/5.506GiB/5.527GiB
+2022-12-16 15:25:05.367,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,19.335MiB/s,19.377MiB/s,4.699GiB/2.874GiB/2.879GiB
+2022-12-16 15:25:05.367,completed,1,T4,2.128 GiB,8.513 GiB,3,26,17.841GiB,4/4,16/16,12.599MiB/s,12.649MiB/s,12.534GiB/5.506GiB/5.528GiB
+2022-12-16 15:25:06.826,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,16.020MiB/s,16.122MiB/s,3.915GiB/2.105GiB/2.116GiB
+2022-12-16 15:25:06.828,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.527GiB,4/4,16/16,12.584MiB/s,12.644MiB/s,12.534GiB/5.518GiB/5.544GiB
+2022-12-16 15:25:52.324,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,13.482MiB/s,13.520MiB/s,5.481GiB/1.992GiB/1.997GiB
+2022-12-16 15:25:52.332,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.214GiB,4/4,16/16,12.766MiB/s,12.816MiB/s,12.534GiB/6.165GiB/6.189GiB
+2022-12-16 15:25:52.352,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,13.480MiB/s,13.519MiB/s,4.698GiB/1.992GiB/1.997GiB
+2022-12-16 15:25:52.359,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.214GiB,4/4,16/16,12.765MiB/s,12.816MiB/s,12.534GiB/6.165GiB/6.189GiB
+2022-12-16 15:25:52.376,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,13.478MiB/s,13.519MiB/s,5.481GiB/1.992GiB/1.998GiB
+2022-12-16 15:25:52.377,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.214GiB,4/4,16/16,12.764MiB/s,12.816MiB/s,12.534GiB/6.165GiB/6.189GiB
+2022-12-16 15:25:52.392,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,13.476MiB/s,13.518MiB/s,6.264GiB/1.992GiB/1.998GiB
+2022-12-16 15:25:52.399,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.214GiB,4/4,16/16,12.764MiB/s,12.815MiB/s,12.534GiB/6.165GiB/6.190GiB
+2022-12-16 15:27:21.701,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,25.901MiB/s,26.021MiB/s,6.264GiB/4.203GiB/4.218GiB
+2022-12-16 15:27:21.705,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.214GiB,4/4,16/16,12.669MiB/s,12.713MiB/s,12.534GiB/7.224GiB/7.249GiB
+2022-12-16 15:27:21.711,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.568MiB/s,22.688MiB/s,5.481GiB/3.420GiB/3.435GiB
+2022-12-16 15:27:21.713,submitted,1,T4,2.128 GiB,8.513 GiB,4,29,19.900GiB,5/5,20/20,12.673MiB/s,12.713MiB/s,15.666GiB/7.226GiB/7.249GiB
+2022-12-16 15:27:23.373,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.514MiB/s,22.597MiB/s,5.481GiB/3.452GiB/3.461GiB
+2022-12-16 15:27:23.373,completed,1,T4,2.128 GiB,8.513 GiB,3,29,19.900GiB,5/5,20/20,13.935MiB/s,14.958MiB/s,15.666GiB/7.236GiB/7.266GiB
+2022-12-16 15:27:23.388,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.487GiB,6/6,24/24,19.205MiB/s,19.289MiB/s,4.699GiB/2.669GiB/2.678GiB
+2022-12-16 15:27:23.388,submitted,1,T4,2.128 GiB,8.513 GiB,4,30,20.585GiB,6/6,24/24,13.922MiB/s,14.959MiB/s,18.798GiB/7.236GiB/7.266GiB
+2022-12-16 15:27:26.710,completed,0,T4,0.000 B,2.128 GiB,2,32,5.487GiB,6/6,24/24,19.177MiB/s,19.266MiB/s,4.699GiB/2.728GiB/2.738GiB
+2022-12-16 15:27:26.712,completed,1,T4,2.128 GiB,8.513 GiB,3,30,20.585GiB,6/6,24/24,17.221MiB/s,18.219MiB/s,18.798GiB/7.292GiB/7.320GiB
+2022-12-16 15:27:26.719,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,15.911MiB/s,15.999MiB/s,3.915GiB/1.945GiB/1.954GiB
+2022-12-16 15:27:26.726,submitted,1,T4,2.128 GiB,8.513 GiB,4,31,21.272GiB,7/7,28/28,17.211MiB/s,18.227MiB/s,21.932GiB/7.292GiB/7.320GiB
+2022-12-16 15:27:28.533,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,15.850MiB/s,16.022MiB/s,3.915GiB/1.967GiB/1.984GiB
+2022-12-16 15:27:28.538,completed,1,T4,2.128 GiB,8.513 GiB,3,31,21.272GiB,7/7,28/28,19.043MiB/s,22.010MiB/s,21.932GiB/7.319GiB/7.363GiB
+2022-12-16 15:27:28.546,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.114GiB,4/4,16/16,12.607MiB/s,12.780MiB/s,3.132GiB/1.184GiB/1.200GiB
+2022-12-16 15:27:28.552,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.958GiB,8/8,32/32,19.023MiB/s,22.004MiB/s,25.066GiB/7.319GiB/7.363GiB
+2022-12-16 15:28:25.543,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,13.238MiB/s,13.270MiB/s,5.482GiB/1.980GiB/1.985GiB
+2022-12-16 15:28:25.550,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.958GiB,8/8,32/32,26.849MiB/s,27.032MiB/s,25.066GiB/8.902GiB/8.937GiB
+2022-12-16 15:28:25.566,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,13.236MiB/s,13.269MiB/s,4.699GiB/1.980GiB/1.985GiB
+2022-12-16 15:28:25.575,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.958GiB,8/8,32/32,26.843MiB/s,27.032MiB/s,25.066GiB/8.902GiB/8.938GiB
+2022-12-16 15:28:25.598,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,13.255MiB/s,13.268MiB/s,5.481GiB/1.983GiB/1.985GiB
+2022-12-16 15:28:25.599,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.958GiB,8/8,32/32,26.835MiB/s,27.033MiB/s,25.066GiB/8.902GiB/8.939GiB
+2022-12-16 15:28:25.617,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,13.255MiB/s,13.268MiB/s,6.264GiB/1.983GiB/1.985GiB
+2022-12-16 15:28:25.618,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.958GiB,8/8,32/32,26.833MiB/s,27.033MiB/s,25.066GiB/8.902GiB/8.939GiB
+2022-12-16 15:30:05.149,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,24.177MiB/s,24.269MiB/s,6.264GiB/4.239GiB/4.252GiB
+2022-12-16 15:30:05.155,completed,1,T4,2.128 GiB,8.513 GiB,4,32,21.958GiB,8/8,32/32,24.918MiB/s,25.043MiB/s,25.066GiB/11.093GiB/11.138GiB
+2022-12-16 15:30:05.752,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,21.045MiB/s,21.105MiB/s,5.481GiB/3.474GiB/3.481GiB
+2022-12-16 15:30:05.752,completed,1,T4,2.128 GiB,8.513 GiB,1,33,22.645GiB,8/8,32/32,24.929MiB/s,25.054MiB/s,25.066GiB/11.111GiB/11.155GiB
+2022-12-16 15:30:05.874,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.865MiB/s,17.940MiB/s,4.698GiB/2.691GiB/2.700GiB
+2022-12-16 15:30:05.874,completed,1,T4,2.128 GiB,8.513 GiB,1,34,23.330GiB,8/8,32/32,24.918MiB/s,25.054MiB/s,25.066GiB/11.111GiB/11.158GiB
+2022-12-16 15:30:07.915,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,14.816MiB/s,14.868MiB/s,3.915GiB/1.950GiB/1.955GiB
+2022-12-16 15:30:07.921,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.017GiB,8/8,32/32,24.959MiB/s,25.087MiB/s,25.066GiB/11.173GiB/11.219GiB
+2022-12-16 15:31:10.016,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,12.449MiB/s,12.471MiB/s,4.698GiB/1.999GiB/2.002GiB
+2022-12-16 15:31:10.020,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.703GiB,8/8,32/32,25.344MiB/s,25.442MiB/s,25.066GiB/12.810GiB/12.855GiB
+2022-12-16 15:31:10.029,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/6,32/24,12.448MiB/s,12.470MiB/s,6.264GiB/1.999GiB/2.002GiB
+2022-12-16 15:31:10.030,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.703GiB,8/8,32/32,25.343MiB/s,25.442MiB/s,25.066GiB/12.810GiB/12.855GiB
+2022-12-16 15:31:10.049,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,12.446MiB/s,12.470MiB/s,6.264GiB/1.999GiB/2.003GiB
+2022-12-16 15:31:10.056,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.703GiB,8/8,32/32,25.341MiB/s,25.442MiB/s,25.066GiB/12.810GiB/12.855GiB
+2022-12-16 15:31:10.067,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.445MiB/s,12.470MiB/s,6.264GiB/1.999GiB/2.003GiB
+2022-12-16 15:31:10.067,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.703GiB,8/8,32/32,25.340MiB/s,25.442MiB/s,25.066GiB/12.810GiB/12.855GiB
+2022-12-16 15:32:42.575,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,24.085MiB/s,24.191MiB/s,6.264GiB/4.151GiB/4.164GiB
+2022-12-16 15:32:42.575,completed,1,T4,2.128 GiB,8.513 GiB,1,36,24.703GiB,8/8,32/32,24.904MiB/s,25.015MiB/s,25.066GiB/14.895GiB/14.954GiB
+2022-12-16 15:32:46.197,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,21.097MiB/s,21.141MiB/s,5.481GiB/3.456GiB/3.462GiB
+2022-12-16 15:32:46.198,completed,1,T4,2.128 GiB,8.513 GiB,2,37,25.389GiB,8/8,32/32,24.941MiB/s,25.034MiB/s,25.066GiB/14.997GiB/15.049GiB
+2022-12-16 15:32:46.240,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,18.013MiB/s,18.061MiB/s,4.698GiB/2.673GiB/2.680GiB
+2022-12-16 15:32:46.240,completed,1,T4,2.128 GiB,8.513 GiB,2,38,26.075GiB,8/8,32/32,24.939MiB/s,25.033MiB/s,25.066GiB/14.997GiB/15.050GiB
+2022-12-16 15:32:48.748,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,14.934MiB/s,15.046MiB/s,3.915GiB/1.928GiB/1.939GiB
+2022-12-16 15:32:48.748,completed,1,T4,2.128 GiB,8.513 GiB,2,39,26.762GiB,8/8,32/32,24.927MiB/s,25.048MiB/s,25.066GiB/15.052GiB/15.117GiB
+2022-12-16 15:33:51.146,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,12.302MiB/s,12.328MiB/s,5.481GiB/1.935GiB/1.939GiB
+2022-12-16 15:33:51.150,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.447GiB,8/8,32/32,24.954MiB/s,25.052MiB/s,25.066GiB/16.580GiB/16.640GiB
+2022-12-16 15:33:51.165,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,12.300MiB/s,12.327MiB/s,4.698GiB/1.935GiB/1.939GiB
+2022-12-16 15:33:51.174,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.447GiB,8/8,32/32,24.953MiB/s,25.052MiB/s,25.066GiB/16.580GiB/16.640GiB
+2022-12-16 15:33:51.187,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.298MiB/s,12.327MiB/s,5.481GiB/1.935GiB/1.940GiB
+2022-12-16 15:33:51.201,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.447GiB,8/8,32/32,24.952MiB/s,25.052MiB/s,25.066GiB/16.580GiB/16.641GiB
+2022-12-16 15:33:51.213,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.296MiB/s,12.326MiB/s,6.264GiB/1.935GiB/1.940GiB
+2022-12-16 15:33:51.214,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.447GiB,8/8,32/32,24.951MiB/s,25.051MiB/s,25.066GiB/16.580GiB/16.641GiB
+2022-12-16 15:34:36.571,completed,0,T4,0.000 B,2.128 GiB,1,36,6.172GiB,8/8,32/32,24.325MiB/s,24.512MiB/s,6.264GiB/2.997GiB/3.013GiB
+2022-12-16 15:34:36.573,completed,1,T4,2.128 GiB,8.513 GiB,2,40,27.447GiB,8/8,32/32,24.829MiB/s,24.928MiB/s,25.066GiB/17.607GiB/17.671GiB
+2022-12-16 15:34:44.562,completed,0,T4,0.000 B,2.128 GiB,1,36,6.172GiB,8/8,32/32,24.437MiB/s,24.540MiB/s,6.264GiB/3.196GiB/3.207GiB
+2022-12-16 15:34:44.565,completed,1,T4,2.128 GiB,8.513 GiB,2,36,24.703GiB,7/7,28/28,21.695MiB/s,21.781MiB/s,21.933GiB/14.645GiB/14.698GiB
+2022-12-16 15:34:44.566,completed,2,T4,8.513 GiB,34.052 GiB,1,2,2.751GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:34:54.336,completed,0,T4,0.000 B,2.128 GiB,1,36,6.172GiB,8/8,32/32,24.549MiB/s,24.628MiB/s,6.264GiB/3.444GiB/3.452GiB
+2022-12-16 15:34:54.337,completed,1,T4,2.128 GiB,8.513 GiB,2,32,21.958GiB,6/6,24/24,18.586MiB/s,18.662MiB/s,18.799GiB/11.694GiB/11.739GiB
+2022-12-16 15:34:54.341,completed,2,T4,8.513 GiB,34.052 GiB,1,4,5.502GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:34:57.296,completed,0,T4,0.000 B,2.128 GiB,1,36,6.172GiB,8/8,32/32,24.662MiB/s,24.830MiB/s,6.264GiB/3.525GiB/3.540GiB
+2022-12-16 15:34:57.296,completed,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,15.490MiB/s,15.564MiB/s,15.665GiB/8.610GiB/8.647GiB
+2022-12-16 15:34:57.297,completed,2,T4,8.513 GiB,34.052 GiB,1,6,8.254GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:35:23.355,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,25.719MiB/s,25.792MiB/s,6.264GiB/4.293GiB/4.304GiB
+2022-12-16 15:35:23.355,completed,1,T4,2.128 GiB,8.513 GiB,2,24,16.468GiB,4/4,16/16,12.541MiB/s,12.592MiB/s,12.532GiB/5.857GiB/5.881GiB
+2022-12-16 15:35:23.355,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:35:23.807,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.502MiB/s,22.629MiB/s,5.482GiB/3.517GiB/3.531GiB
+2022-12-16 15:35:23.807,completed,1,T4,2.128 GiB,8.513 GiB,3,25,17.154GiB,4/4,16/16,12.529MiB/s,12.594MiB/s,12.532GiB/5.857GiB/5.887GiB
+2022-12-16 15:35:23.813,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:35:24.212,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,19.278MiB/s,19.483MiB/s,4.699GiB/2.734GiB/2.756GiB
+2022-12-16 15:35:24.218,completed,1,T4,2.128 GiB,8.513 GiB,3,26,17.840GiB,4/4,16/16,12.519MiB/s,12.596MiB/s,12.532GiB/5.857GiB/5.893GiB
+2022-12-16 15:35:24.218,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:35:27.761,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,16.304MiB/s,16.378MiB/s,3.916GiB/2.027GiB/2.035GiB
+2022-12-16 15:35:27.761,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.526GiB,4/4,16/16,12.550MiB/s,12.610MiB/s,12.532GiB/5.915GiB/5.943GiB
+2022-12-16 15:35:27.762,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:36:25.349,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,5/5,20/20,12.738MiB/s,12.777MiB/s,3.915GiB/1.918GiB/1.923GiB
+2022-12-16 15:36:25.359,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,12.481MiB/s,12.537MiB/s,12.532GiB/6.585GiB/6.614GiB
+2022-12-16 15:36:25.359,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:36:25.375,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.736MiB/s,12.776MiB/s,5.481GiB/1.918GiB/1.924GiB
+2022-12-16 15:36:25.389,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,12.481MiB/s,12.537MiB/s,12.532GiB/6.585GiB/6.614GiB
+2022-12-16 15:36:25.389,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:36:25.401,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.733MiB/s,12.776MiB/s,5.481GiB/1.918GiB/1.924GiB
+2022-12-16 15:36:25.401,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,12.480MiB/s,12.536MiB/s,12.532GiB/6.585GiB/6.614GiB
+2022-12-16 15:36:25.409,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:36:25.423,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.732MiB/s,12.776MiB/s,6.264GiB/1.918GiB/1.924GiB
+2022-12-16 15:36:25.429,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,12.480MiB/s,12.536MiB/s,12.532GiB/6.585GiB/6.614GiB
+2022-12-16 15:36:25.430,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:37:55.165,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,26.616MiB/s,26.685MiB/s,6.264GiB/4.298GiB/4.307GiB
+2022-12-16 15:37:55.170,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,12.603MiB/s,12.656MiB/s,12.532GiB/7.754GiB/7.787GiB
+2022-12-16 15:37:55.171,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:37:55.181,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,23.339MiB/s,23.399MiB/s,5.481GiB/3.518GiB/3.524GiB
+2022-12-16 15:37:55.194,submitted,1,T4,2.128 GiB,8.513 GiB,4,29,19.898GiB,5/5,20/20,12.603MiB/s,12.656MiB/s,15.665GiB/7.754GiB/7.787GiB
+2022-12-16 15:37:55.195,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:37:56.048,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,23.307MiB/s,23.371MiB/s,5.481GiB/3.530GiB/3.540GiB
+2022-12-16 15:37:56.051,completed,1,T4,2.128 GiB,8.513 GiB,3,29,19.898GiB,5/5,20/20,12.605MiB/s,14.989MiB/s,15.665GiB/7.766GiB/7.799GiB
+2022-12-16 15:37:56.055,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:37:56.065,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,20.030MiB/s,20.096MiB/s,4.698GiB/2.748GiB/2.757GiB
+2022-12-16 15:37:56.066,submitted,1,T4,2.128 GiB,8.513 GiB,4,30,20.585GiB,6/6,24/24,12.604MiB/s,14.962MiB/s,18.798GiB/7.766GiB/7.800GiB
+2022-12-16 15:37:56.067,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:37:58.129,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,19.970MiB/s,20.029MiB/s,4.698GiB/2.781GiB/2.788GiB
+2022-12-16 15:37:58.130,completed,1,T4,2.128 GiB,8.513 GiB,3,30,20.585GiB,6/6,24/24,17.063MiB/s,18.054MiB/s,18.798GiB/7.798GiB/7.833GiB
+2022-12-16 15:37:58.131,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:37:58.141,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,16.732MiB/s,16.779MiB/s,3.915GiB/2.001GiB/2.005GiB
+2022-12-16 15:37:58.142,submitted,1,T4,2.128 GiB,8.513 GiB,4,31,21.271GiB,7/7,28/28,17.037MiB/s,18.044MiB/s,21.931GiB/7.798GiB/7.833GiB
+2022-12-16 15:37:58.147,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:38:00.618,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,16.631MiB/s,16.696MiB/s,3.915GiB/2.030GiB/2.037GiB
+2022-12-16 15:38:00.621,completed,1,T4,2.128 GiB,8.513 GiB,3,31,21.271GiB,7/7,28/28,19.251MiB/s,20.235MiB/s,21.931GiB/7.839GiB/7.877GiB
+2022-12-16 15:38:00.621,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:38:00.625,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.114GiB,4/4,16/16,13.415MiB/s,13.479MiB/s,3.132GiB/1.247GiB/1.253GiB
+2022-12-16 15:38:00.625,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.957GiB,8/8,32/32,19.242MiB/s,20.225MiB/s,25.064GiB/7.839GiB/7.877GiB
+2022-12-16 15:38:00.625,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:38:58.308,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,5/5,20/20,12.800MiB/s,12.842MiB/s,3.915GiB/1.911GiB/1.917GiB
+2022-12-16 15:38:58.308,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,23.987MiB/s,24.192MiB/s,25.064GiB/9.151GiB/9.198GiB
+2022-12-16 15:38:58.308,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:38:58.315,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.800MiB/s,12.843MiB/s,5.481GiB/1.911GiB/1.918GiB
+2022-12-16 15:38:58.316,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,23.985MiB/s,24.193MiB/s,25.064GiB/9.151GiB/9.198GiB
+2022-12-16 15:38:58.316,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:38:58.319,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,12.799MiB/s,12.843MiB/s,6.264GiB/1.911GiB/1.918GiB
+2022-12-16 15:38:58.319,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,23.984MiB/s,24.194MiB/s,25.064GiB/9.151GiB/9.199GiB
+2022-12-16 15:38:58.319,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:38:58.321,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.799MiB/s,12.843MiB/s,6.264GiB/1.911GiB/1.918GiB
+2022-12-16 15:38:58.321,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,23.984MiB/s,24.195MiB/s,25.064GiB/9.151GiB/9.199GiB
+2022-12-16 15:38:58.321,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:40:33.919,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,25.434MiB/s,25.536MiB/s,6.264GiB/4.281GiB/4.296GiB
+2022-12-16 15:40:33.919,completed,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,24.561MiB/s,24.696MiB/s,25.064GiB/11.471GiB/11.526GiB
+2022-12-16 15:40:33.920,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:40:34.554,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.202MiB/s,22.310MiB/s,5.481GiB/3.514GiB/3.527GiB
+2022-12-16 15:40:34.554,completed,1,T4,2.128 GiB,8.513 GiB,1,33,22.643GiB,8/8,32/32,24.530MiB/s,24.697MiB/s,25.064GiB/11.481GiB/11.541GiB
+2022-12-16 15:40:34.554,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:40:37.606,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,18.980MiB/s,19.072MiB/s,4.698GiB/2.786GiB/2.796GiB
+2022-12-16 15:40:37.606,completed,1,T4,2.128 GiB,8.513 GiB,1,34,23.329GiB,8/8,32/32,24.538MiB/s,24.682MiB/s,25.064GiB/11.553GiB/11.610GiB
+2022-12-16 15:40:37.607,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:40:39.742,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.784MiB/s,15.871MiB/s,3.915GiB/2.034GiB/2.043GiB
+2022-12-16 15:40:39.742,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.015GiB,8/8,32/32,24.541MiB/s,24.675MiB/s,25.064GiB/11.600GiB/11.658GiB
+2022-12-16 15:40:39.749,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:41:39.268,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,12.665MiB/s,12.737MiB/s,4.698GiB/1.990GiB/2.002GiB
+2022-12-16 15:41:39.276,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,24.595MiB/s,24.759MiB/s,25.064GiB/13.038GiB/13.111GiB
+2022-12-16 15:41:39.276,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:41:39.282,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/6,32/24,12.663MiB/s,12.737MiB/s,6.264GiB/1.990GiB/2.002GiB
+2022-12-16 15:41:39.282,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,24.594MiB/s,24.759MiB/s,25.064GiB/13.038GiB/13.111GiB
+2022-12-16 15:41:39.282,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:41:39.294,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,12.662MiB/s,12.738MiB/s,6.264GiB/1.990GiB/2.002GiB
+2022-12-16 15:41:39.294,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,24.593MiB/s,24.760MiB/s,25.064GiB/13.038GiB/13.111GiB
+2022-12-16 15:41:39.294,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:41:39.300,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.662MiB/s,12.738MiB/s,6.264GiB/1.990GiB/2.002GiB
+2022-12-16 15:41:39.305,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,24.593MiB/s,24.760MiB/s,25.064GiB/13.038GiB/13.112GiB
+2022-12-16 15:41:39.305,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:43:14.622,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,24.642MiB/s,24.738MiB/s,6.264GiB/4.251GiB/4.263GiB
+2022-12-16 15:43:14.623,completed,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,24.418MiB/s,24.552MiB/s,25.064GiB/15.225GiB/15.302GiB
+2022-12-16 15:43:14.623,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:43:15.389,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,21.534MiB/s,21.586MiB/s,5.481GiB/3.487GiB/3.493GiB
+2022-12-16 15:43:15.390,completed,1,T4,2.128 GiB,8.513 GiB,2,37,25.388GiB,8/8,32/32,24.430MiB/s,24.547MiB/s,25.064GiB/15.248GiB/15.317GiB
+2022-12-16 15:43:15.390,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:43:16.304,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,18.419MiB/s,18.478MiB/s,4.698GiB/2.721GiB/2.727GiB
+2022-12-16 15:43:16.306,completed,1,T4,2.128 GiB,8.513 GiB,2,38,26.074GiB,8/8,32/32,24.428MiB/s,24.551MiB/s,25.064GiB/15.269GiB/15.341GiB
+2022-12-16 15:43:16.306,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:43:16.935,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.268MiB/s,15.380MiB/s,3.915GiB/1.943GiB/1.954GiB
+2022-12-16 15:43:16.935,completed,1,T4,2.128 GiB,8.513 GiB,2,39,26.761GiB,8/8,32/32,24.440MiB/s,24.555MiB/s,25.064GiB/15.291GiB/15.358GiB
+2022-12-16 15:43:16.935,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:44:15.451,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,5/5,20/20,12.557MiB/s,12.608MiB/s,3.915GiB/1.915GiB/1.923GiB
+2022-12-16 15:44:15.451,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.648MiB/s,24.775MiB/s,25.064GiB/16.785GiB/16.868GiB
+2022-12-16 15:44:15.451,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:44:15.470,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.555MiB/s,12.608MiB/s,5.482GiB/1.915GiB/1.923GiB
+2022-12-16 15:44:15.478,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.647MiB/s,24.775MiB/s,25.064GiB/16.785GiB/16.868GiB
+2022-12-16 15:44:15.479,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:44:15.489,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,12.554MiB/s,12.608MiB/s,6.264GiB/1.915GiB/1.923GiB
+2022-12-16 15:44:15.491,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.646MiB/s,24.775MiB/s,25.064GiB/16.785GiB/16.869GiB
+2022-12-16 15:44:15.491,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:44:15.509,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.552MiB/s,12.607MiB/s,6.264GiB/1.915GiB/1.923GiB
+2022-12-16 15:44:15.510,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.645MiB/s,24.775MiB/s,25.064GiB/16.785GiB/16.869GiB
+2022-12-16 15:44:15.510,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:44:29.184,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.693MiB/s,27.087MiB/s,6.264GiB/2.292GiB/2.305GiB
+2022-12-16 15:44:29.185,completed,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.726MiB/s,24.839MiB/s,25.064GiB/17.157GiB/17.231GiB
+2022-12-16 15:44:29.186,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.005GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:44:32.477,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.420MiB/s,27.025MiB/s,6.264GiB/2.379GiB/2.396GiB
+2022-12-16 15:44:32.477,completed,1,T4,2.128 GiB,8.513 GiB,2,36,24.702GiB,7/7,28/28,21.616MiB/s,21.726MiB/s,21.932GiB/14.100GiB/14.166GiB
+2022-12-16 15:44:32.478,completed,2,T4,8.513 GiB,34.052 GiB,2,10,13.757GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:44:37.217,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.768MiB/s,26.921MiB/s,6.264GiB/2.511GiB/2.519GiB
+2022-12-16 15:44:37.218,completed,1,T4,2.128 GiB,8.513 GiB,2,32,21.957GiB,6/6,24/24,18.505MiB/s,18.587MiB/s,18.798GiB/11.064GiB/11.110GiB
+2022-12-16 15:44:37.218,completed,2,T4,8.513 GiB,34.052 GiB,2,12,16.511GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:44:37.866,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.504MiB/s,26.901MiB/s,6.264GiB/2.521GiB/2.535GiB
+2022-12-16 15:44:37.866,completed,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,15.384MiB/s,15.461MiB/s,15.664GiB/7.937GiB/7.974GiB
+2022-12-16 15:44:37.866,completed,2,T4,8.513 GiB,34.052 GiB,2,14,19.265GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:45:55.902,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,24.751MiB/s,24.779MiB/s,6.264GiB/4.325GiB/4.330GiB
+2022-12-16 15:45:55.902,completed,1,T4,2.128 GiB,8.513 GiB,2,24,16.468GiB,4/4,16/16,12.116MiB/s,12.166MiB/s,12.532GiB/5.660GiB/5.684GiB
+2022-12-16 15:45:55.904,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:45:56.473,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,21.555MiB/s,21.682MiB/s,5.481GiB/3.547GiB/3.562GiB
+2022-12-16 15:45:56.479,completed,1,T4,2.128 GiB,8.513 GiB,3,25,17.154GiB,4/4,16/16,12.101MiB/s,12.170MiB/s,12.532GiB/5.660GiB/5.692GiB
+2022-12-16 15:45:56.479,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:45:56.543,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,18.426MiB/s,18.565MiB/s,4.698GiB/2.764GiB/2.780GiB
+2022-12-16 15:45:56.543,completed,1,T4,2.128 GiB,8.513 GiB,3,26,17.840GiB,4/4,16/16,12.099MiB/s,12.171MiB/s,12.532GiB/5.660GiB/5.693GiB
+2022-12-16 15:45:56.543,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:45:58.818,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.329MiB/s,15.464MiB/s,3.915GiB/2.018GiB/2.032GiB
+2022-12-16 15:45:58.825,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.526GiB,4/4,16/16,12.112MiB/s,12.174MiB/s,12.532GiB/5.693GiB/5.722GiB
+2022-12-16 15:45:58.826,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:46:46.334,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,5/5,20/20,12.897MiB/s,12.978MiB/s,3.915GiB/1.900GiB/1.912GiB
+2022-12-16 15:46:46.334,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.221MiB/s,12.293MiB/s,12.532GiB/6.311GiB/6.348GiB
+2022-12-16 15:46:46.334,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:46:46.349,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.896MiB/s,12.978MiB/s,5.481GiB/1.900GiB/1.912GiB
+2022-12-16 15:46:46.349,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.220MiB/s,12.293MiB/s,12.532GiB/6.311GiB/6.348GiB
+2022-12-16 15:46:46.350,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:46:46.362,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,12.895MiB/s,12.978MiB/s,6.264GiB/1.900GiB/1.912GiB
+2022-12-16 15:46:46.362,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.220MiB/s,12.293MiB/s,12.532GiB/6.311GiB/6.349GiB
+2022-12-16 15:46:46.362,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:46:46.364,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.895MiB/s,12.978MiB/s,6.264GiB/1.900GiB/1.912GiB
+2022-12-16 15:46:46.364,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.220MiB/s,12.293MiB/s,12.532GiB/6.311GiB/6.349GiB
+2022-12-16 15:46:46.364,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:48:23.510,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,25.275MiB/s,25.381MiB/s,6.264GiB/4.279GiB/4.293GiB
+2022-12-16 15:48:23.518,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.187MiB/s,12.251MiB/s,12.532GiB/7.450GiB/7.489GiB
+2022-12-16 15:48:23.518,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:48:23.526,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.039MiB/s,22.148MiB/s,5.481GiB/3.496GiB/3.510GiB
+2022-12-16 15:48:23.526,submitted,1,T4,2.128 GiB,8.513 GiB,4,29,19.899GiB,5/5,20/20,12.187MiB/s,12.250MiB/s,15.665GiB/7.450GiB/7.489GiB
+2022-12-16 15:48:23.526,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:48:25.845,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.121MiB/s,22.189MiB/s,5.481GiB/3.555GiB/3.565GiB
+2022-12-16 15:48:25.846,completed,1,T4,2.128 GiB,8.513 GiB,3,29,19.899GiB,5/5,20/20,15.996MiB/s,16.389MiB/s,15.665GiB/7.497GiB/7.530GiB
+2022-12-16 15:48:25.851,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:48:25.852,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,18.917MiB/s,18.985MiB/s,4.698GiB/2.773GiB/2.782GiB
+2022-12-16 15:48:25.852,submitted,1,T4,2.128 GiB,8.513 GiB,4,30,20.585GiB,6/6,24/24,15.985MiB/s,16.378MiB/s,18.797GiB/7.497GiB/7.531GiB
+2022-12-16 15:48:25.852,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:48:26.590,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,18.907MiB/s,19.055MiB/s,4.698GiB/2.786GiB/2.803GiB
+2022-12-16 15:48:26.590,completed,1,T4,2.128 GiB,8.513 GiB,3,30,20.585GiB,6/6,24/24,16.196MiB/s,21.281MiB/s,18.797GiB/7.505GiB/7.552GiB
+2022-12-16 15:48:26.591,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:48:26.592,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.711MiB/s,15.859MiB/s,3.914GiB/2.003GiB/2.019GiB
+2022-12-16 15:48:26.592,submitted,1,T4,2.128 GiB,8.513 GiB,4,31,21.271GiB,7/7,28/28,16.193MiB/s,21.267MiB/s,21.931GiB/7.505GiB/7.552GiB
+2022-12-16 15:48:26.592,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:48:28.744,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.811MiB/s,15.848MiB/s,3.914GiB/2.047GiB/2.051GiB
+2022-12-16 15:48:28.747,completed,1,T4,2.128 GiB,8.513 GiB,3,31,21.271GiB,7/7,28/28,21.484MiB/s,22.799MiB/s,21.931GiB/7.559GiB/7.594GiB
+2022-12-16 15:48:28.748,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:48:28.751,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.114GiB,4/4,16/16,12.646MiB/s,12.682MiB/s,3.132GiB/1.264GiB/1.268GiB
+2022-12-16 15:48:28.752,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.957GiB,8/8,32/32,21.462MiB/s,22.795MiB/s,25.063GiB/7.559GiB/7.594GiB
+2022-12-16 15:48:28.752,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:49:27.157,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,5/5,20/20,12.944MiB/s,12.989MiB/s,3.915GiB/2.032GiB/2.040GiB
+2022-12-16 15:49:27.157,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,25.581MiB/s,25.826MiB/s,25.063GiB/9.064GiB/9.114GiB
+2022-12-16 15:49:27.163,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:49:27.173,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/6,32/24,12.942MiB/s,12.989MiB/s,6.264GiB/2.032GiB/2.040GiB
+2022-12-16 15:49:27.175,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,25.577MiB/s,25.825MiB/s,25.063GiB/9.064GiB/9.114GiB
+2022-12-16 15:49:27.176,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:49:27.183,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.942MiB/s,12.989MiB/s,5.482GiB/2.032GiB/2.040GiB
+2022-12-16 15:49:27.187,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,25.574MiB/s,25.825MiB/s,25.063GiB/9.064GiB/9.114GiB
+2022-12-16 15:49:27.188,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:49:27.205,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.940MiB/s,12.988MiB/s,6.264GiB/2.032GiB/2.040GiB
+2022-12-16 15:49:27.205,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,25.569MiB/s,25.823MiB/s,25.063GiB/9.064GiB/9.115GiB
+2022-12-16 15:49:27.205,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:51:01.669,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,24.143MiB/s,24.251MiB/s,6.264GiB/4.185GiB/4.197GiB
+2022-12-16 15:51:01.676,completed,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,24.292MiB/s,24.446MiB/s,25.063GiB/11.168GiB/11.223GiB
+2022-12-16 15:51:01.677,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:51:01.874,completed,0,T4,0.000 B,2.128 GiB,2,36,6.171GiB,7/7,28/28,20.969MiB/s,21.110MiB/s,5.481GiB/3.402GiB/3.418GiB
+2022-12-16 15:51:01.874,completed,1,T4,2.128 GiB,8.513 GiB,1,33,22.643GiB,8/8,32/32,24.273MiB/s,24.445MiB/s,25.063GiB/11.168GiB/11.228GiB
+2022-12-16 15:51:01.875,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:51:03.464,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.872MiB/s,17.936MiB/s,4.698GiB/2.650GiB/2.658GiB
+2022-12-16 15:51:03.464,completed,1,T4,2.128 GiB,8.513 GiB,1,34,23.329GiB,8/8,32/32,24.268MiB/s,24.415MiB/s,25.063GiB/11.201GiB/11.259GiB
+2022-12-16 15:51:03.464,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:51:07.047,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,14.877MiB/s,14.947MiB/s,3.915GiB/1.934GiB/1.941GiB
+2022-12-16 15:51:07.047,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.015GiB,8/8,32/32,24.398MiB/s,24.512MiB/s,25.063GiB/11.320GiB/11.370GiB
+2022-12-16 15:51:07.048,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:52:10.570,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,12.268MiB/s,12.327MiB/s,4.697GiB/1.957GiB/1.967GiB
+2022-12-16 15:52:10.573,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.494MiB/s,24.639MiB/s,25.063GiB/12.873GiB/12.940GiB
+2022-12-16 15:52:10.573,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:52:10.580,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.267MiB/s,12.326MiB/s,5.481GiB/1.957GiB/1.967GiB
+2022-12-16 15:52:10.580,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.493MiB/s,24.639MiB/s,25.063GiB/12.873GiB/12.940GiB
+2022-12-16 15:52:10.580,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:52:10.589,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,12.267MiB/s,12.326MiB/s,6.264GiB/1.957GiB/1.967GiB
+2022-12-16 15:52:10.590,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.492MiB/s,24.638MiB/s,25.063GiB/12.873GiB/12.940GiB
+2022-12-16 15:52:10.590,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:52:10.592,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.267MiB/s,12.326MiB/s,6.264GiB/1.957GiB/1.967GiB
+2022-12-16 15:52:10.592,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.492MiB/s,24.639MiB/s,25.063GiB/12.873GiB/12.940GiB
+2022-12-16 15:52:10.592,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:53:45.698,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,24.302MiB/s,24.415MiB/s,6.264GiB/4.193GiB/4.209GiB
+2022-12-16 15:53:45.698,completed,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.345MiB/s,24.488MiB/s,25.063GiB/15.077GiB/15.160GiB
+2022-12-16 15:53:45.701,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:53:49.564,completed,0,T4,0.000 B,2.128 GiB,2,36,6.171GiB,7/7,28/28,21.343MiB/s,21.412MiB/s,5.480GiB/3.509GiB/3.518GiB
+2022-12-16 15:53:49.565,completed,1,T4,2.128 GiB,8.513 GiB,2,37,25.388GiB,8/8,32/32,24.383MiB/s,24.510MiB/s,25.063GiB/15.186GiB/15.260GiB
+2022-12-16 15:53:49.565,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:53:52.637,completed,0,T4,0.000 B,2.128 GiB,2,32,5.485GiB,6/6,24/24,18.316MiB/s,18.406MiB/s,4.697GiB/2.785GiB/2.795GiB
+2022-12-16 15:53:52.638,completed,1,T4,2.128 GiB,8.513 GiB,2,38,26.074GiB,8/8,32/32,24.387MiB/s,24.535MiB/s,25.063GiB/15.262GiB/15.346GiB
+2022-12-16 15:53:52.638,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:53:54.006,completed,0,T4,0.000 B,2.128 GiB,2,28,4.799GiB,5/5,20/20,15.260MiB/s,15.385MiB/s,3.914GiB/2.020GiB/2.033GiB
+2022-12-16 15:53:54.006,completed,1,T4,2.128 GiB,8.513 GiB,2,39,26.760GiB,8/8,32/32,24.425MiB/s,24.538MiB/s,25.063GiB/15.311GiB/15.379GiB
+2022-12-16 15:53:54.007,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:54:45.687,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,5/5,20/20,12.324MiB/s,12.375MiB/s,3.915GiB/1.867GiB/1.874GiB
+2022-12-16 15:54:45.687,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.385MiB/s,24.513MiB/s,25.063GiB/16.519GiB/16.601GiB
+2022-12-16 15:54:45.687,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:54:45.711,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/6,32/24,12.322MiB/s,12.375MiB/s,6.264GiB/1.867GiB/1.874GiB
+2022-12-16 15:54:45.711,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.384MiB/s,24.513MiB/s,25.063GiB/16.519GiB/16.601GiB
+2022-12-16 15:54:45.711,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:54:45.728,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.321MiB/s,12.374MiB/s,5.481GiB/1.867GiB/1.875GiB
+2022-12-16 15:54:45.728,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.383MiB/s,24.513MiB/s,25.063GiB/16.519GiB/16.601GiB
+2022-12-16 15:54:45.728,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:54:45.765,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.318MiB/s,12.372MiB/s,6.264GiB/1.867GiB/1.875GiB
+2022-12-16 15:54:45.766,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.381MiB/s,24.513MiB/s,25.063GiB/16.519GiB/16.602GiB
+2022-12-16 15:54:45.766,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:55:24.924,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,23.833MiB/s,24.081MiB/s,6.264GiB/2.762GiB/2.776GiB
+2022-12-16 15:55:24.924,completed,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,24.298MiB/s,24.414MiB/s,25.063GiB/17.399GiB/17.478GiB
+2022-12-16 15:55:24.925,completed,2,T4,8.513 GiB,34.052 GiB,2,16,22.017GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:55:31.437,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,24.293MiB/s,24.460MiB/s,6.264GiB/2.940GiB/2.955GiB
+2022-12-16 15:55:31.445,completed,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,21.276MiB/s,21.371MiB/s,21.930GiB/14.420GiB/14.483GiB
+2022-12-16 15:55:31.445,completed,2,T4,8.513 GiB,34.052 GiB,3,18,24.771GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:55:36.634,completed,0,T4,0.000 B,2.128 GiB,1,36,6.172GiB,8/8,32/32,24.315MiB/s,24.438MiB/s,6.264GiB/3.067GiB/3.077GiB
+2022-12-16 15:55:36.635,completed,1,T4,2.128 GiB,8.513 GiB,2,32,21.956GiB,6/6,24/24,18.221MiB/s,18.300MiB/s,18.797GiB/11.378GiB/11.427GiB
+2022-12-16 15:55:36.635,completed,2,T4,8.513 GiB,34.052 GiB,3,20,27.525GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:55:38.374,completed,0,T4,0.000 B,2.128 GiB,1,36,6.172GiB,8/8,32/32,24.185MiB/s,24.477MiB/s,6.264GiB/3.099GiB/3.121GiB
+2022-12-16 15:55:38.375,completed,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,15.169MiB/s,15.253MiB/s,15.664GiB/8.261GiB/8.304GiB
+2022-12-16 15:55:38.375,completed,2,T4,8.513 GiB,34.052 GiB,3,22,30.279GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:56:20.496,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,25.339MiB/s,25.449MiB/s,6.264GiB/4.251GiB/4.263GiB
+2022-12-16 15:56:20.497,completed,1,T4,2.128 GiB,8.513 GiB,2,24,16.467GiB,4/4,16/16,12.302MiB/s,12.357MiB/s,12.532GiB/5.698GiB/5.724GiB
+2022-12-16 15:56:20.498,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:56:24.485,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,22.264MiB/s,22.426MiB/s,5.481GiB/3.574GiB/3.590GiB
+2022-12-16 15:56:24.485,completed,1,T4,2.128 GiB,8.513 GiB,3,25,17.153GiB,4/4,16/16,12.332MiB/s,12.387MiB/s,12.532GiB/5.760GiB/5.786GiB
+2022-12-16 15:56:24.485,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:56:25.655,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,19.157MiB/s,19.248MiB/s,4.699GiB/2.816GiB/2.826GiB
+2022-12-16 15:56:25.655,completed,1,T4,2.128 GiB,8.513 GiB,3,26,17.839GiB,4/4,16/16,12.328MiB/s,12.386MiB/s,12.532GiB/5.772GiB/5.799GiB
+2022-12-16 15:56:25.657,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:56:29.222,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.987MiB/s,16.060MiB/s,3.915GiB/2.085GiB/2.093GiB
+2022-12-16 15:56:29.223,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.526GiB,4/4,16/16,12.306MiB/s,12.377MiB/s,12.532GiB/5.805GiB/5.838GiB
+2022-12-16 15:56:29.226,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:57:32.800,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,12.418MiB/s,12.442MiB/s,4.698GiB/2.026GiB/2.030GiB
+2022-12-16 15:57:32.800,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.240MiB/s,12.298MiB/s,12.532GiB/6.534GiB/6.565GiB
+2022-12-16 15:57:32.800,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:57:32.815,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,12.417MiB/s,12.442MiB/s,4.698GiB/2.026GiB/2.030GiB
+2022-12-16 15:57:32.815,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.240MiB/s,12.298MiB/s,12.532GiB/6.534GiB/6.565GiB
+2022-12-16 15:57:32.815,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:57:32.839,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,12.416MiB/s,12.442MiB/s,6.264GiB/2.026GiB/2.030GiB
+2022-12-16 15:57:32.839,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.239MiB/s,12.298MiB/s,12.532GiB/6.534GiB/6.565GiB
+2022-12-16 15:57:32.839,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:57:32.853,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.414MiB/s,12.441MiB/s,6.264GiB/2.026GiB/2.030GiB
+2022-12-16 15:57:32.853,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.239MiB/s,12.298MiB/s,12.532GiB/6.534GiB/6.565GiB
+2022-12-16 15:57:32.859,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:58:51.970,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,26.643MiB/s,26.760MiB/s,6.264GiB/4.168GiB/4.180GiB
+2022-12-16 15:58:51.971,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,12.389MiB/s,12.446MiB/s,12.532GiB/7.571GiB/7.606GiB
+2022-12-16 15:58:51.971,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:58:51.972,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.171GiB,7/7,28/28,23.394MiB/s,23.502MiB/s,5.481GiB/3.387GiB/3.397GiB
+2022-12-16 15:58:51.972,submitted,1,T4,2.128 GiB,8.513 GiB,4,29,19.898GiB,5/5,20/20,12.389MiB/s,12.446MiB/s,15.664GiB/7.571GiB/7.606GiB
+2022-12-16 15:58:51.972,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:58:52.514,completed,0,T4,0.000 B,2.128 GiB,2,36,6.171GiB,7/7,28/28,23.388MiB/s,23.471MiB/s,5.481GiB/3.395GiB/3.405GiB
+2022-12-16 15:58:52.516,completed,1,T4,2.128 GiB,8.513 GiB,3,29,19.898GiB,5/5,20/20,12.383MiB/s,14.463MiB/s,15.664GiB/7.574GiB/7.612GiB
+2022-12-16 15:58:52.516,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:58:52.519,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,20.177MiB/s,20.221MiB/s,4.698GiB/2.615GiB/2.622GiB
+2022-12-16 15:58:52.521,submitted,1,T4,2.128 GiB,8.513 GiB,4,30,20.584GiB,6/6,24/24,12.383MiB/s,14.476MiB/s,18.797GiB/7.574GiB/7.612GiB
+2022-12-16 15:58:52.522,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:58:54.731,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,20.083MiB/s,20.147MiB/s,4.698GiB/2.650GiB/2.657GiB
+2022-12-16 15:58:54.731,completed,1,T4,2.128 GiB,8.513 GiB,3,30,20.584GiB,6/6,24/24,15.911MiB/s,17.109MiB/s,18.797GiB/7.601GiB/7.644GiB
+2022-12-16 15:58:54.731,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:58:54.733,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,16.862MiB/s,16.927MiB/s,3.915GiB/1.867GiB/1.874GiB
+2022-12-16 15:58:54.741,submitted,1,T4,2.128 GiB,8.513 GiB,4,31,21.270GiB,7/7,28/28,15.908MiB/s,17.109MiB/s,21.930GiB/7.601GiB/7.644GiB
+2022-12-16 15:58:54.741,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:58:57.886,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,16.786MiB/s,16.890MiB/s,3.915GiB/1.913GiB/1.922GiB
+2022-12-16 15:58:57.886,completed,1,T4,2.128 GiB,8.513 GiB,3,31,21.270GiB,7/7,28/28,19.664MiB/s,20.854MiB/s,21.930GiB/7.660GiB/7.710GiB
+2022-12-16 15:58:57.886,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 15:58:57.888,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.114GiB,4/4,16/16,13.606MiB/s,13.710MiB/s,3.132GiB/1.130GiB/1.139GiB
+2022-12-16 15:58:57.888,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.956GiB,8/8,32/32,19.661MiB/s,20.852MiB/s,25.063GiB/7.660GiB/7.710GiB
+2022-12-16 15:58:57.888,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:00:08.158,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,12.818MiB/s,12.878MiB/s,4.698GiB/1.944GiB/1.953GiB
+2022-12-16 16:00:08.158,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.956GiB,8/8,32/32,23.805MiB/s,24.027MiB/s,25.063GiB/9.255GiB/9.310GiB
+2022-12-16 16:00:08.158,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:00:08.176,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.817MiB/s,12.879MiB/s,5.481GiB/1.944GiB/1.954GiB
+2022-12-16 16:00:08.177,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.956GiB,8/8,32/32,23.801MiB/s,24.027MiB/s,25.063GiB/9.255GiB/9.311GiB
+2022-12-16 16:00:08.177,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:00:08.202,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.815MiB/s,12.878MiB/s,5.481GiB/1.944GiB/1.954GiB
+2022-12-16 16:00:08.204,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.956GiB,8/8,32/32,23.797MiB/s,24.025MiB/s,25.063GiB/9.255GiB/9.311GiB
+2022-12-16 16:00:08.206,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:00:08.227,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.814MiB/s,12.878MiB/s,6.264GiB/1.944GiB/1.954GiB
+2022-12-16 16:00:08.227,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.956GiB,8/8,32/32,23.793MiB/s,24.026MiB/s,25.063GiB/9.255GiB/9.312GiB
+2022-12-16 16:00:08.228,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:01:41.288,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,24.751MiB/s,24.892MiB/s,6.264GiB/4.164GiB/4.181GiB
+2022-12-16 16:01:41.290,completed,1,T4,2.128 GiB,8.513 GiB,-1,32,21.956GiB,8/8,32/32,23.876MiB/s,24.054MiB/s,25.063GiB/11.395GiB/11.465GiB
+2022-12-16 16:01:41.291,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:01:45.528,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,21.519MiB/s,21.621MiB/s,5.481GiB/3.467GiB/3.479GiB
+2022-12-16 16:01:45.528,completed,1,T4,2.128 GiB,8.513 GiB,1,33,22.643GiB,8/8,32/32,23.870MiB/s,24.027MiB/s,25.063GiB/11.496GiB/11.559GiB
+2022-12-16 16:01:45.528,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:01:48.391,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,18.346MiB/s,18.436MiB/s,4.698GiB/2.735GiB/2.745GiB
+2022-12-16 16:01:48.391,completed,1,T4,2.128 GiB,8.513 GiB,1,34,23.328GiB,8/8,32/32,23.855MiB/s,24.004MiB/s,25.063GiB/11.557GiB/11.620GiB
+2022-12-16 16:01:48.391,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:01:55.015,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.222MiB/s,15.246MiB/s,3.915GiB/2.052GiB/2.054GiB
+2022-12-16 16:01:55.015,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.015GiB,8/8,32/32,23.864MiB/s,23.982MiB/s,25.063GiB/11.714GiB/11.768GiB
+2022-12-16 16:01:55.016,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:02:58.318,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,12.447MiB/s,12.554MiB/s,5.481GiB/2.068GiB/2.085GiB
+2022-12-16 16:02:58.318,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.208MiB/s,24.395MiB/s,25.063GiB/13.302GiB/13.390GiB
+2022-12-16 16:02:58.318,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:02:58.326,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.446MiB/s,12.555MiB/s,5.481GiB/2.068GiB/2.086GiB
+2022-12-16 16:02:58.326,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.207MiB/s,24.396MiB/s,25.063GiB/13.302GiB/13.390GiB
+2022-12-16 16:02:58.326,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:02:58.330,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.446MiB/s,12.557MiB/s,5.481GiB/2.068GiB/2.086GiB
+2022-12-16 16:02:58.330,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.207MiB/s,24.397MiB/s,25.063GiB/13.302GiB/13.390GiB
+2022-12-16 16:02:58.330,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:02:58.338,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.445MiB/s,12.558MiB/s,6.264GiB/2.068GiB/2.086GiB
+2022-12-16 16:02:58.338,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.207MiB/s,24.398MiB/s,25.063GiB/13.302GiB/13.391GiB
+2022-12-16 16:02:58.338,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:04:21.905,completed,0,T4,0.000 B,2.128 GiB,1,36,6.172GiB,8/8,32/32,24.363MiB/s,24.426MiB/s,6.264GiB/4.039GiB/4.047GiB
+2022-12-16 16:04:21.914,completed,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,24.183MiB/s,24.297MiB/s,25.063GiB/15.248GiB/15.318GiB
+2022-12-16 16:04:21.922,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:04:29.883,completed,0,T4,0.000 B,2.128 GiB,2,36,6.171GiB,7/7,28/28,21.215MiB/s,21.259MiB/s,5.481GiB/3.422GiB/3.427GiB
+2022-12-16 16:04:29.883,completed,1,T4,2.128 GiB,8.513 GiB,2,37,25.387GiB,8/8,32/32,24.177MiB/s,24.289MiB/s,25.063GiB/15.432GiB/15.501GiB
+2022-12-16 16:04:29.884,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:04:29.941,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,18.143MiB/s,18.193MiB/s,4.698GiB/2.639GiB/2.644GiB
+2022-12-16 16:04:29.942,completed,1,T4,2.128 GiB,8.513 GiB,2,38,26.073GiB,8/8,32/32,24.174MiB/s,24.288MiB/s,25.063GiB/15.432GiB/15.503GiB
+2022-12-16 16:04:29.942,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:04:31.055,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,15.062MiB/s,15.108MiB/s,3.915GiB/1.870GiB/1.875GiB
+2022-12-16 16:04:31.055,completed,1,T4,2.128 GiB,8.513 GiB,2,39,26.759GiB,8/8,32/32,24.168MiB/s,24.279MiB/s,25.063GiB/15.455GiB/15.524GiB
+2022-12-16 16:04:31.055,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:05:24.937,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,12.539MiB/s,12.612MiB/s,4.698GiB/1.795GiB/1.805GiB
+2022-12-16 16:05:24.941,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.445GiB,8/8,32/32,24.459MiB/s,24.597MiB/s,25.063GiB/16.871GiB/16.960GiB
+2022-12-16 16:05:24.941,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:05:24.951,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.538MiB/s,12.612MiB/s,5.481GiB/1.795GiB/1.806GiB
+2022-12-16 16:05:24.951,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.445GiB,8/8,32/32,24.458MiB/s,24.597MiB/s,25.063GiB/16.871GiB/16.960GiB
+2022-12-16 16:05:24.951,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:05:24.970,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.536MiB/s,12.612MiB/s,5.481GiB/1.795GiB/1.806GiB
+2022-12-16 16:05:24.973,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.445GiB,8/8,32/32,24.457MiB/s,24.597MiB/s,25.063GiB/16.871GiB/16.961GiB
+2022-12-16 16:05:24.973,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:05:24.983,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.535MiB/s,12.612MiB/s,6.264GiB/1.795GiB/1.806GiB
+2022-12-16 16:05:24.983,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,27.445GiB,8/8,32/32,24.457MiB/s,24.597MiB/s,25.063GiB/16.871GiB/16.961GiB
+2022-12-16 16:05:24.983,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:05:37.343,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.865MiB/s,27.313MiB/s,6.264GiB/2.134GiB/2.151GiB
+2022-12-16 16:05:37.343,completed,1,T4,2.128 GiB,8.513 GiB,2,40,27.445GiB,8/8,32/32,24.518MiB/s,24.645MiB/s,25.063GiB/17.199GiB/17.283GiB
+2022-12-16 16:05:37.343,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.032GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:05:37.345,submitted,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.862MiB/s,27.314MiB/s,6.264GiB/2.134GiB/2.151GiB
+2022-12-16 16:05:37.346,submitted,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,21.414MiB/s,21.529MiB/s,21.930GiB/14.066GiB/14.137GiB
+2022-12-16 16:05:37.351,submitted,2,T4,8.513 GiB,34.052 GiB,4,26,35.786GiB,1/1,4/4,0B/s,0B/s,6.290GiB/0B/0B
+2022-12-16 16:05:37.362,submitted,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.891MiB/s,27.310MiB/s,6.264GiB/2.142GiB/2.151GiB
+2022-12-16 16:05:37.362,submitted,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,21.425MiB/s,21.529MiB/s,21.930GiB/14.070GiB/14.138GiB
+2022-12-16 16:05:37.362,submitted,2,T4,8.513 GiB,34.052 GiB,4,26,35.786GiB,2/2,8/8,0B/s,0B/s,12.579GiB/0B/0B
+2022-12-16 16:05:47.217,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.545MiB/s,26.916MiB/s,6.264GiB/2.390GiB/2.404GiB
+2022-12-16 16:05:47.217,completed,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,21.431MiB/s,21.542MiB/s,21.930GiB/14.278GiB/14.348GiB
+2022-12-16 16:05:47.218,completed,2,T4,8.513 GiB,34.052 GiB,3,26,35.786GiB,2/2,8/8,5.996MiB/s,6.427MiB/s,12.579GiB/59.027MiB/63.271MiB
+2022-12-16 16:05:47.222,submitted,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.541MiB/s,26.920MiB/s,6.264GiB/2.390GiB/2.404GiB
+2022-12-16 16:05:47.222,submitted,1,T4,2.128 GiB,8.513 GiB,2,32,21.956GiB,6/6,24/24,18.351MiB/s,18.449MiB/s,18.797GiB/11.146GiB/11.203GiB
+2022-12-16 16:05:47.222,submitted,2,T4,8.513 GiB,34.052 GiB,4,28,38.540GiB,3/3,12/12,5.993MiB/s,6.427MiB/s,18.869GiB/59.027MiB/63.297MiB
+2022-12-16 16:05:47.224,submitted,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,26.540MiB/s,26.922MiB/s,6.264GiB/2.390GiB/2.405GiB
+2022-12-16 16:05:47.224,submitted,1,T4,2.128 GiB,8.513 GiB,2,32,21.956GiB,6/6,24/24,18.351MiB/s,18.449MiB/s,18.797GiB/11.146GiB/11.203GiB
+2022-12-16 16:05:47.224,submitted,2,T4,8.513 GiB,34.052 GiB,4,28,38.540GiB,4/4,16/16,5.992MiB/s,6.429MiB/s,25.158GiB/59.027MiB/63.335MiB
+2022-12-16 16:05:55.375,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,25.728MiB/s,26.090MiB/s,6.264GiB/2.566GiB/2.583GiB
+2022-12-16 16:05:55.375,completed,1,T4,2.128 GiB,8.513 GiB,2,32,21.956GiB,6/6,24/24,18.304MiB/s,18.403MiB/s,18.797GiB/11.270GiB/11.327GiB
+2022-12-16 16:05:55.375,completed,2,T4,8.513 GiB,34.052 GiB,3,28,38.540GiB,4/4,16/16,10.368MiB/s,11.374MiB/s,25.158GiB/142.447MiB/153.818MiB
+2022-12-16 16:05:55.380,submitted,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,25.725MiB/s,26.089MiB/s,6.264GiB/2.566GiB/2.583GiB
+2022-12-16 16:05:55.380,submitted,1,T4,2.128 GiB,8.513 GiB,2,28,19.211GiB,5/5,20/20,15.245MiB/s,15.331MiB/s,15.664GiB/8.137GiB/8.181GiB
+2022-12-16 16:05:55.380,submitted,2,T4,8.513 GiB,34.052 GiB,4,30,41.294GiB,6/5,24/20,10.363MiB/s,11.371MiB/s,37.736GiB/142.447MiB/153.857MiB
+2022-12-16 16:05:55.406,submitted,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,25.712MiB/s,26.083MiB/s,6.264GiB/2.566GiB/2.583GiB
+2022-12-16 16:05:55.413,submitted,1,T4,2.128 GiB,8.513 GiB,2,28,19.211GiB,5/5,20/20,15.244MiB/s,15.330MiB/s,15.664GiB/8.137GiB/8.181GiB
+2022-12-16 16:05:55.413,submitted,2,T4,8.513 GiB,34.052 GiB,4,30,41.294GiB,6/6,24/24,10.340MiB/s,11.362MiB/s,37.736GiB/142.447MiB/154.033MiB
+2022-12-16 16:05:58.542,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,25.421MiB/s,25.854MiB/s,6.264GiB/2.626GiB/2.649GiB
+2022-12-16 16:05:58.545,completed,1,T4,2.128 GiB,8.513 GiB,2,28,19.211GiB,5/5,20/20,15.224MiB/s,15.315MiB/s,15.664GiB/8.174GiB/8.220GiB
+2022-12-16 16:05:58.545,completed,2,T4,8.513 GiB,34.052 GiB,3,30,41.294GiB,6/6,24/24,16.714MiB/s,17.693MiB/s,37.736GiB/198.230MiB/210.533MiB
+2022-12-16 16:05:58.546,submitted,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,25.420MiB/s,25.854MiB/s,6.264GiB/2.626GiB/2.649GiB
+2022-12-16 16:05:58.546,submitted,1,T4,2.128 GiB,8.513 GiB,2,24,16.467GiB,4/4,16/16,12.168MiB/s,12.247MiB/s,12.531GiB/5.041GiB/5.074GiB
+2022-12-16 16:05:58.546,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/7,32/28,16.704MiB/s,17.684MiB/s,50.313GiB/198.230MiB/210.549MiB
+2022-12-16 16:05:58.568,submitted,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,25.410MiB/s,25.853MiB/s,6.264GiB/2.626GiB/2.649GiB
+2022-12-16 16:05:58.569,submitted,1,T4,2.128 GiB,8.513 GiB,2,24,16.467GiB,4/4,16/16,12.175MiB/s,12.246MiB/s,12.531GiB/5.045GiB/5.074GiB
+2022-12-16 16:05:58.569,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,16.782MiB/s,17.678MiB/s,50.313GiB/201.087MiB/210.842MiB
+2022-12-16 16:07:18.634,completed,0,T4,0.000 B,2.128 GiB,2,40,6.857GiB,8/8,32/32,23.829MiB/s,23.940MiB/s,6.264GiB/4.380GiB/4.397GiB
+2022-12-16 16:07:18.634,completed,1,T4,2.128 GiB,8.513 GiB,2,24,16.467GiB,4/4,16/16,11.995MiB/s,12.059MiB/s,12.531GiB/5.908GiB/5.939GiB
+2022-12-16 16:07:18.634,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,22.527MiB/s,22.809MiB/s,50.313GiB/1.959GiB/1.983GiB
+2022-12-16 16:07:21.056,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.863MiB/s,20.915MiB/s,5.481GiB/3.663GiB/3.671GiB
+2022-12-16 16:07:21.057,completed,1,T4,2.128 GiB,8.513 GiB,3,25,17.153GiB,4/4,16/16,11.997MiB/s,12.060MiB/s,12.531GiB/5.937GiB/5.968GiB
+2022-12-16 16:07:21.058,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,22.899MiB/s,23.037MiB/s,50.313GiB/2.045GiB/2.057GiB
+2022-12-16 16:07:24.385,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.763MiB/s,17.863MiB/s,4.698GiB/2.931GiB/2.944GiB
+2022-12-16 16:07:24.389,completed,1,T4,2.128 GiB,8.513 GiB,3,26,17.839GiB,4/4,16/16,11.993MiB/s,12.054MiB/s,12.531GiB/5.974GiB/6.005GiB
+2022-12-16 16:07:24.390,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,22.862MiB/s,23.081MiB/s,50.313GiB/2.116GiB/2.136GiB
+2022-12-16 16:07:26.181,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,14.803MiB/s,14.874MiB/s,3.915GiB/2.181GiB/2.189GiB
+2022-12-16 16:07:26.183,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.525GiB,4/4,16/16,11.998MiB/s,12.063MiB/s,12.531GiB/5.998GiB/6.030GiB
+2022-12-16 16:07:26.184,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,22.951MiB/s,23.106MiB/s,50.313GiB/2.164GiB/2.179GiB
+2022-12-16 16:08:17.905,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,12.211MiB/s,12.237MiB/s,4.697GiB/2.062GiB/2.066GiB
+2022-12-16 16:08:17.907,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.211GiB,4/4,16/16,12.085MiB/s,12.149MiB/s,12.531GiB/6.652GiB/6.687GiB
+2022-12-16 16:08:17.909,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,24.201MiB/s,24.281MiB/s,50.313GiB/3.504GiB/3.516GiB
+2022-12-16 16:08:17.921,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,12.209MiB/s,12.237MiB/s,4.697GiB/2.062GiB/2.067GiB
+2022-12-16 16:08:17.923,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.211GiB,4/4,16/16,12.084MiB/s,12.149MiB/s,12.531GiB/6.652GiB/6.687GiB
+2022-12-16 16:08:17.923,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,24.198MiB/s,24.282MiB/s,50.313GiB/3.504GiB/3.516GiB
+2022-12-16 16:08:17.928,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.209MiB/s,12.237MiB/s,5.481GiB/2.062GiB/2.067GiB
+2022-12-16 16:08:17.933,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.211GiB,4/4,16/16,12.084MiB/s,12.149MiB/s,12.531GiB/6.652GiB/6.687GiB
+2022-12-16 16:08:17.934,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,24.197MiB/s,24.283MiB/s,50.313GiB/3.504GiB/3.517GiB
+2022-12-16 16:08:17.958,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.207MiB/s,12.237MiB/s,6.264GiB/2.062GiB/2.067GiB
+2022-12-16 16:08:17.958,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.211GiB,4/4,16/16,12.084MiB/s,12.149MiB/s,12.531GiB/6.652GiB/6.687GiB
+2022-12-16 16:08:17.959,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,24.192MiB/s,24.282MiB/s,50.313GiB/3.504GiB/3.517GiB
+2022-12-16 16:09:51.011,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.508MiB/s,23.644MiB/s,6.264GiB/4.153GiB/4.170GiB
+2022-12-16 16:09:51.012,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.211GiB,4/4,16/16,11.973MiB/s,12.039MiB/s,12.531GiB/7.678GiB/7.721GiB
+2022-12-16 16:09:51.012,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.809MiB/s,23.956MiB/s,50.313GiB/5.612GiB/5.647GiB
+2022-12-16 16:09:51.014,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.494MiB/s,20.630MiB/s,5.481GiB/3.370GiB/3.386GiB
+2022-12-16 16:09:51.014,submitted,1,T4,2.128 GiB,8.513 GiB,4,29,19.898GiB,5/5,20/20,11.973MiB/s,12.039MiB/s,15.664GiB/7.678GiB/7.721GiB
+2022-12-16 16:09:51.014,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.809MiB/s,23.956MiB/s,50.313GiB/5.612GiB/5.647GiB
+2022-12-16 16:09:51.060,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.487MiB/s,20.630MiB/s,5.481GiB/3.370GiB/3.387GiB
+2022-12-16 16:09:51.060,completed,1,T4,2.128 GiB,8.513 GiB,3,29,19.898GiB,5/5,20/20,11.972MiB/s,12.039MiB/s,15.664GiB/7.678GiB/7.721GiB
+2022-12-16 16:09:51.060,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.815MiB/s,23.955MiB/s,50.313GiB/5.614GiB/5.648GiB
+2022-12-16 16:09:51.061,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.473MiB/s,17.615MiB/s,4.698GiB/2.587GiB/2.604GiB
+2022-12-16 16:09:51.061,submitted,1,T4,2.128 GiB,8.513 GiB,4,30,20.584GiB,6/6,24/24,11.972MiB/s,12.039MiB/s,18.797GiB/7.678GiB/7.721GiB
+2022-12-16 16:09:51.062,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.815MiB/s,23.955MiB/s,50.313GiB/5.614GiB/5.648GiB
+2022-12-16 16:09:53.278,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.500MiB/s,17.598MiB/s,4.698GiB/2.628GiB/2.639GiB
+2022-12-16 16:09:53.280,completed,1,T4,2.128 GiB,8.513 GiB,3,30,20.584GiB,6/6,24/24,17.799MiB/s,18.430MiB/s,18.797GiB/7.717GiB/7.759GiB
+2022-12-16 16:09:53.280,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.823MiB/s,23.937MiB/s,50.313GiB/5.668GiB/5.695GiB
+2022-12-16 16:09:53.283,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.513MiB/s,14.610MiB/s,3.915GiB/1.845GiB/1.856GiB
+2022-12-16 16:09:53.283,submitted,1,T4,2.128 GiB,8.513 GiB,4,31,21.270GiB,7/7,28/28,17.788MiB/s,18.419MiB/s,21.930GiB/7.717GiB/7.759GiB
+2022-12-16 16:09:53.284,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.823MiB/s,23.937MiB/s,50.313GiB/5.668GiB/5.695GiB
+2022-12-16 16:09:58.133,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.684MiB/s,14.727MiB/s,3.915GiB/1.933GiB/1.937GiB
+2022-12-16 16:09:58.136,completed,1,T4,2.128 GiB,8.513 GiB,3,31,21.270GiB,7/7,28/28,20.908MiB/s,21.722MiB/s,21.930GiB/7.826GiB/7.867GiB
+2022-12-16 16:09:58.137,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.918MiB/s,24.006MiB/s,50.313GiB/5.804GiB/5.826GiB
+2022-12-16 16:09:58.140,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,11.747MiB/s,11.791MiB/s,3.132GiB/1.149GiB/1.154GiB
+2022-12-16 16:09:58.149,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.956GiB,8/8,32/32,20.898MiB/s,21.719MiB/s,25.062GiB/7.826GiB/7.867GiB
+2022-12-16 16:09:58.149,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.917MiB/s,24.006MiB/s,50.313GiB/5.804GiB/5.826GiB
+2022-12-16 16:11:10.805,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,11.808MiB/s,11.843MiB/s,4.698GiB/1.993GiB/1.999GiB
+2022-12-16 16:11:10.805,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.956GiB,8/8,32/32,23.704MiB/s,23.917MiB/s,25.062GiB/9.478GiB/9.534GiB
+2022-12-16 16:11:10.805,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.788MiB/s,23.896MiB/s,50.313GiB/7.461GiB/7.495GiB
+2022-12-16 16:11:10.817,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,11.807MiB/s,11.842MiB/s,5.481GiB/1.993GiB/1.999GiB
+2022-12-16 16:11:10.817,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.956GiB,8/8,32/32,23.702MiB/s,23.916MiB/s,25.062GiB/9.478GiB/9.534GiB
+2022-12-16 16:11:10.818,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.787MiB/s,23.895MiB/s,50.313GiB/7.461GiB/7.495GiB
+2022-12-16 16:11:10.830,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,11.806MiB/s,11.842MiB/s,6.264GiB/1.993GiB/1.999GiB
+2022-12-16 16:11:10.830,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.956GiB,8/8,32/32,23.700MiB/s,23.916MiB/s,25.062GiB/9.478GiB/9.534GiB
+2022-12-16 16:11:10.830,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.786MiB/s,23.895MiB/s,50.313GiB/7.461GiB/7.495GiB
+2022-12-16 16:11:10.842,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,11.805MiB/s,11.843MiB/s,6.264GiB/1.993GiB/2.000GiB
+2022-12-16 16:11:10.842,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.956GiB,8/8,32/32,23.698MiB/s,23.918MiB/s,25.062GiB/9.478GiB/9.535GiB
+2022-12-16 16:11:10.842,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.786MiB/s,23.896MiB/s,50.313GiB/7.461GiB/7.496GiB
+2022-12-16 16:12:48.327,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.593MiB/s,23.789MiB/s,6.264GiB/4.241GiB/4.263GiB
+2022-12-16 16:12:48.327,completed,1,T4,2.128 GiB,8.513 GiB,4,32,21.956GiB,8/8,32/32,23.432MiB/s,23.584MiB/s,25.062GiB/11.664GiB/11.726GiB
+2022-12-16 16:12:48.327,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.698MiB/s,23.785MiB/s,50.313GiB/9.689GiB/9.725GiB
+2022-12-16 16:12:49.382,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.732MiB/s,20.818MiB/s,5.481GiB/3.491GiB/3.499GiB
+2022-12-16 16:12:49.383,completed,1,T4,2.128 GiB,8.513 GiB,1,33,22.643GiB,8/8,32/32,23.471MiB/s,23.601MiB/s,25.062GiB/11.696GiB/11.754GiB
+2022-12-16 16:12:49.383,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.709MiB/s,23.786MiB/s,50.313GiB/9.718GiB/9.750GiB
+2022-12-16 16:12:49.455,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.767MiB/s,17.861MiB/s,4.697GiB/2.708GiB/2.717GiB
+2022-12-16 16:12:49.463,completed,1,T4,2.128 GiB,8.513 GiB,1,34,23.329GiB,8/8,32/32,23.465MiB/s,23.603MiB/s,25.062GiB/11.696GiB/11.756GiB
+2022-12-16 16:12:49.464,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.705MiB/s,23.787MiB/s,50.313GiB/9.718GiB/9.752GiB
+2022-12-16 16:12:49.520,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,14.806MiB/s,14.902MiB/s,3.914GiB/1.925GiB/1.934GiB
+2022-12-16 16:12:49.520,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.015GiB,8/8,32/32,23.460MiB/s,23.602MiB/s,25.062GiB/11.696GiB/11.757GiB
+2022-12-16 16:12:49.522,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.701MiB/s,23.786MiB/s,50.313GiB/9.718GiB/9.753GiB
+2022-12-16 16:14:08.420,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.487GiB,5/5,20/20,11.603MiB/s,11.693MiB/s,3.915GiB/2.012GiB/2.028GiB
+2022-12-16 16:14:08.423,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.700GiB,8/8,32/32,23.242MiB/s,23.421MiB/s,25.062GiB/13.406GiB/13.494GiB
+2022-12-16 16:14:08.423,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.593MiB/s,23.700MiB/s,50.313GiB/11.492GiB/11.544GiB
+2022-12-16 16:14:08.439,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.487GiB,6/6,24/24,11.633MiB/s,11.694MiB/s,4.698GiB/2.017GiB/2.028GiB
+2022-12-16 16:14:08.439,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.700GiB,8/8,32/32,23.247MiB/s,23.421MiB/s,25.062GiB/13.411GiB/13.494GiB
+2022-12-16 16:14:08.440,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.592MiB/s,23.700MiB/s,50.313GiB/11.492GiB/11.545GiB
+2022-12-16 16:14:08.464,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.487GiB,7/7,28/28,11.632MiB/s,11.694MiB/s,5.482GiB/2.017GiB/2.028GiB
+2022-12-16 16:14:08.464,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.700GiB,8/8,32/32,23.265MiB/s,23.421MiB/s,25.062GiB/13.416GiB/13.495GiB
+2022-12-16 16:14:08.464,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.591MiB/s,23.700MiB/s,50.313GiB/11.492GiB/11.545GiB
+2022-12-16 16:14:08.476,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.487GiB,8/8,32/32,11.630MiB/s,11.694MiB/s,6.265GiB/2.017GiB/2.029GiB
+2022-12-16 16:14:08.481,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.700GiB,8/8,32/32,23.264MiB/s,23.421MiB/s,25.062GiB/13.416GiB/13.495GiB
+2022-12-16 16:14:08.481,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.590MiB/s,23.700MiB/s,50.313GiB/11.492GiB/11.545GiB
+2022-12-16 16:15:40.032,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.408MiB/s,23.465MiB/s,6.265GiB/4.131GiB/4.138GiB
+2022-12-16 16:15:40.043,completed,1,T4,2.128 GiB,8.513 GiB,1,36,24.700GiB,8/8,32/32,23.238MiB/s,23.349MiB/s,25.062GiB/15.466GiB/15.538GiB
+2022-12-16 16:15:40.043,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.599MiB/s,23.662MiB/s,50.313GiB/13.606GiB/13.642GiB
+2022-12-16 16:15:44.906,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.362MiB/s,20.402MiB/s,5.482GiB/3.434GiB/3.440GiB
+2022-12-16 16:15:44.906,completed,1,T4,2.128 GiB,8.513 GiB,2,37,25.387GiB,8/8,32/32,23.215MiB/s,23.322MiB/s,25.062GiB/15.564GiB/15.634GiB
+2022-12-16 16:15:44.907,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.575MiB/s,23.633MiB/s,50.313GiB/13.705GiB/13.739GiB
+2022-12-16 16:15:45.835,completed,0,T4,0.000 B,2.128 GiB,2,32,5.487GiB,6/6,24/24,17.434MiB/s,17.451MiB/s,4.699GiB/2.667GiB/2.669GiB
+2022-12-16 16:15:45.836,completed,1,T4,2.128 GiB,8.513 GiB,2,38,26.073GiB,8/8,32/32,23.217MiB/s,23.318MiB/s,25.062GiB/15.585GiB/15.653GiB
+2022-12-16 16:15:45.836,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.574MiB/s,23.629MiB/s,50.313GiB/13.726GiB/13.757GiB
+2022-12-16 16:15:47.685,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.397MiB/s,14.472MiB/s,3.915GiB/1.897GiB/1.905GiB
+2022-12-16 16:15:47.685,completed,1,T4,2.128 GiB,8.513 GiB,2,39,26.759GiB,8/8,32/32,23.202MiB/s,23.300MiB/s,25.062GiB/15.619GiB/15.684GiB
+2022-12-16 16:15:47.685,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.548MiB/s,23.610MiB/s,50.313GiB/13.753GiB/13.789GiB
+2022-12-16 16:16:56.093,completed,0,T4,0.000 B,2.128 GiB,3,28,4.801GiB,4/4,16/16,11.522MiB/s,11.567MiB/s,3.132GiB/1.886GiB/1.893GiB
+2022-12-16 16:16:56.097,completed,1,T4,2.128 GiB,8.513 GiB,2,40,27.445GiB,8/8,32/32,23.088MiB/s,23.208MiB/s,25.062GiB/17.087GiB/17.170GiB
+2022-12-16 16:16:56.097,completed,2,T4,8.513 GiB,34.052 GiB,4,32,44.048GiB,8/8,32/32,23.430MiB/s,23.504MiB/s,50.313GiB/15.249GiB/15.297GiB
+2022-12-16 16:17:05.070,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.487GiB,7/5,28/20,11.573MiB/s,11.613MiB/s,5.481GiB/1.996GiB/2.003GiB
+2022-12-16 16:17:05.070,submitted,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,20.158MiB/s,20.260MiB/s,21.929GiB/14.145GiB/14.214GiB
+2022-12-16 16:17:05.071,submitted,2,T4,8.513 GiB,34.052 GiB,1,34,46.802GiB,8/8,32/32,23.464MiB/s,23.528MiB/s,50.313GiB/15.477GiB/15.519GiB
+2022-12-16 16:17:05.099,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.487GiB,6/6,24/24,11.571MiB/s,11.612MiB/s,4.699GiB/1.996GiB/2.003GiB
+2022-12-16 16:17:05.107,submitted,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,20.157MiB/s,20.260MiB/s,21.929GiB/14.145GiB/14.214GiB
+2022-12-16 16:17:05.107,submitted,2,T4,8.513 GiB,34.052 GiB,1,34,46.802GiB,8/8,32/32,23.463MiB/s,23.528MiB/s,50.313GiB/15.477GiB/15.520GiB
+2022-12-16 16:17:05.114,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.487GiB,8/7,32/28,11.570MiB/s,11.613MiB/s,6.265GiB/1.996GiB/2.003GiB
+2022-12-16 16:17:05.114,submitted,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,20.157MiB/s,20.260MiB/s,21.929GiB/14.145GiB/14.215GiB
+2022-12-16 16:17:05.114,submitted,2,T4,8.513 GiB,34.052 GiB,1,34,46.802GiB,8/8,32/32,23.463MiB/s,23.528MiB/s,50.313GiB/15.477GiB/15.520GiB
+2022-12-16 16:17:05.130,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.487GiB,8/8,32/32,11.569MiB/s,11.612MiB/s,6.265GiB/1.996GiB/2.003GiB
+2022-12-16 16:17:05.130,submitted,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,20.156MiB/s,20.260MiB/s,21.929GiB/14.145GiB/14.215GiB
+2022-12-16 16:17:05.130,submitted,2,T4,8.513 GiB,34.052 GiB,1,34,46.802GiB,8/8,32/32,23.462MiB/s,23.528MiB/s,50.313GiB/15.477GiB/15.521GiB
+2022-12-16 16:17:06.372,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.487GiB,8/8,32/32,20.740MiB/s,24.438MiB/s,6.265GiB/2.022GiB/2.036GiB
+2022-12-16 16:17:06.372,completed,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,20.153MiB/s,20.259MiB/s,21.929GiB/14.168GiB/14.238GiB
+2022-12-16 16:17:06.372,completed,2,T4,8.513 GiB,34.052 GiB,1,34,46.802GiB,8/8,32/32,23.454MiB/s,23.532MiB/s,50.313GiB/15.501GiB/15.552GiB
+2022-12-16 16:17:12.234,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.487GiB,8/8,32/32,23.974MiB/s,24.766MiB/s,6.265GiB/2.175GiB/2.187GiB
+2022-12-16 16:17:12.234,completed,1,T4,2.128 GiB,8.513 GiB,2,32,21.957GiB,6/6,24/24,17.241MiB/s,17.330MiB/s,18.797GiB/11.143GiB/11.197GiB
+2022-12-16 16:17:12.234,completed,2,T4,8.513 GiB,34.052 GiB,1,36,49.556GiB,8/8,32/32,23.470MiB/s,23.545MiB/s,50.313GiB/15.645GiB/15.696GiB
+2022-12-16 16:17:14.631,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.487GiB,8/8,32/32,24.086MiB/s,24.799MiB/s,6.265GiB/2.234GiB/2.248GiB
+2022-12-16 16:17:14.631,completed,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,14.303MiB/s,14.389MiB/s,15.664GiB/8.040GiB/8.085GiB
+2022-12-16 16:17:14.632,completed,2,T4,8.513 GiB,34.052 GiB,1,38,52.310GiB,8/8,32/32,23.470MiB/s,23.551MiB/s,50.313GiB/15.700GiB/15.755GiB
+2022-12-16 16:18:41.397,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.248MiB/s,23.284MiB/s,6.265GiB/4.192GiB/4.197GiB
+2022-12-16 16:18:41.398,completed,1,T4,2.128 GiB,8.513 GiB,2,24,16.467GiB,4/4,16/16,11.368MiB/s,11.420MiB/s,12.531GiB/5.862GiB/5.889GiB
+2022-12-16 16:18:41.403,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.466MiB/s,23.525MiB/s,50.313GiB/17.686GiB/17.730GiB
+2022-12-16 16:18:43.994,completed,0,T4,0.000 B,2.128 GiB,2,36,6.173GiB,7/7,28/28,20.205MiB/s,20.328MiB/s,5.482GiB/3.448GiB/3.464GiB
+2022-12-16 16:18:43.995,completed,1,T4,2.128 GiB,8.513 GiB,3,25,17.153GiB,4/4,16/16,11.357MiB/s,11.422MiB/s,12.531GiB/5.885GiB/5.919GiB
+2022-12-16 16:18:43.995,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.461MiB/s,23.532MiB/s,50.313GiB/17.742GiB/17.796GiB
+2022-12-16 16:18:44.553,completed,0,T4,0.000 B,2.128 GiB,2,32,5.487GiB,6/6,24/24,17.364MiB/s,17.421MiB/s,4.699GiB/2.684GiB/2.690GiB
+2022-12-16 16:18:44.554,completed,1,T4,2.128 GiB,8.513 GiB,3,26,17.839GiB,4/4,16/16,11.368MiB/s,11.424MiB/s,12.531GiB/5.897GiB/5.926GiB
+2022-12-16 16:18:44.554,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.471MiB/s,23.536MiB/s,50.313GiB/17.762GiB/17.811GiB
+2022-12-16 16:18:47.091,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.431MiB/s,14.540MiB/s,3.915GiB/1.934GiB/1.945GiB
+2022-12-16 16:18:47.092,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.526GiB,4/4,16/16,11.360MiB/s,11.426MiB/s,12.531GiB/5.921GiB/5.956GiB
+2022-12-16 16:18:47.092,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.467MiB/s,23.540MiB/s,50.313GiB/17.817GiB/17.873GiB
+2022-12-16 16:20:04.082,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,12.268MiB/s,12.296MiB/s,4.698GiB/2.144GiB/2.149GiB
+2022-12-16 16:20:04.091,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,11.536MiB/s,11.596MiB/s,12.531GiB/6.880GiB/6.916GiB
+2022-12-16 16:20:04.091,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.694MiB/s,23.760MiB/s,50.313GiB/19.771GiB/19.826GiB
+2022-12-16 16:20:04.114,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,12.266MiB/s,12.295MiB/s,4.698GiB/2.144GiB/2.149GiB
+2022-12-16 16:20:04.121,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,11.535MiB/s,11.595MiB/s,12.531GiB/6.880GiB/6.916GiB
+2022-12-16 16:20:04.121,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.693MiB/s,23.760MiB/s,50.313GiB/19.771GiB/19.827GiB
+2022-12-16 16:20:04.136,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.265MiB/s,12.295MiB/s,5.481GiB/2.144GiB/2.149GiB
+2022-12-16 16:20:04.142,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,11.539MiB/s,11.595MiB/s,12.531GiB/6.883GiB/6.916GiB
+2022-12-16 16:20:04.142,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.692MiB/s,23.760MiB/s,50.313GiB/19.771GiB/19.827GiB
+2022-12-16 16:20:04.157,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.263MiB/s,12.294MiB/s,6.264GiB/2.144GiB/2.150GiB
+2022-12-16 16:20:04.157,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,11.539MiB/s,11.595MiB/s,12.531GiB/6.883GiB/6.916GiB
+2022-12-16 16:20:04.157,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.692MiB/s,23.760MiB/s,50.313GiB/19.771GiB/19.828GiB
+2022-12-16 16:21:29.495,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.439MiB/s,23.621MiB/s,6.264GiB/4.048GiB/4.068GiB
+2022-12-16 16:21:29.496,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,11.479MiB/s,11.540MiB/s,12.531GiB/7.804GiB/7.845GiB
+2022-12-16 16:21:29.496,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.606MiB/s,23.679MiB/s,50.313GiB/21.667GiB/21.733GiB
+2022-12-16 16:21:29.498,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.406MiB/s,20.587MiB/s,5.481GiB/3.265GiB/3.285GiB
+2022-12-16 16:21:29.498,submitted,1,T4,2.128 GiB,8.513 GiB,4,29,19.898GiB,5/5,20/20,11.479MiB/s,11.540MiB/s,15.664GiB/7.804GiB/7.845GiB
+2022-12-16 16:21:29.498,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.606MiB/s,23.679MiB/s,50.313GiB/21.667GiB/21.733GiB
+2022-12-16 16:21:31.241,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.462MiB/s,20.570MiB/s,5.481GiB/3.306GiB/3.317GiB
+2022-12-16 16:21:31.242,completed,1,T4,2.128 GiB,8.513 GiB,3,29,19.898GiB,5/5,20/20,12.505MiB/s,13.998MiB/s,15.664GiB/7.826GiB/7.866GiB
+2022-12-16 16:21:31.242,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.606MiB/s,23.671MiB/s,50.313GiB/21.707GiB/21.766GiB
+2022-12-16 16:21:31.243,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.448MiB/s,17.555MiB/s,4.698GiB/2.523GiB/2.534GiB
+2022-12-16 16:21:31.243,submitted,1,T4,2.128 GiB,8.513 GiB,4,30,20.585GiB,6/6,24/24,12.504MiB/s,13.996MiB/s,18.797GiB/7.826GiB/7.866GiB
+2022-12-16 16:21:31.245,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.606MiB/s,23.671MiB/s,50.313GiB/21.707GiB/21.766GiB
+2022-12-16 16:21:33.273,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.506MiB/s,17.592MiB/s,4.698GiB/2.565GiB/2.574GiB
+2022-12-16 16:21:33.273,completed,1,T4,2.128 GiB,8.513 GiB,3,30,20.585GiB,6/6,24/24,14.817MiB/s,17.552MiB/s,18.797GiB/7.860GiB/7.905GiB
+2022-12-16 16:21:33.274,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.607MiB/s,23.677MiB/s,50.313GiB/21.755GiB/21.820GiB
+2022-12-16 16:21:33.275,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.516MiB/s,14.602MiB/s,3.915GiB/1.782GiB/1.791GiB
+2022-12-16 16:21:33.275,submitted,1,T4,2.128 GiB,8.513 GiB,4,31,21.270GiB,7/7,28/28,14.815MiB/s,17.549MiB/s,21.930GiB/7.860GiB/7.905GiB
+2022-12-16 16:21:33.275,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.607MiB/s,23.677MiB/s,50.313GiB/21.755GiB/21.820GiB
+2022-12-16 16:21:35.160,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.563MiB/s,14.614MiB/s,3.915GiB/1.814GiB/1.818GiB
+2022-12-16 16:21:35.160,completed,1,T4,2.128 GiB,8.513 GiB,3,31,21.270GiB,7/7,28/28,17.924MiB/s,20.595MiB/s,21.930GiB/7.898GiB/7.943GiB
+2022-12-16 16:21:35.160,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.623MiB/s,23.686MiB/s,50.313GiB/21.813GiB/21.871GiB
+2022-12-16 16:21:35.163,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,11.594MiB/s,11.644MiB/s,3.132GiB/1.031GiB/1.035GiB
+2022-12-16 16:21:35.163,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.957GiB,8/8,32/32,17.918MiB/s,20.591MiB/s,25.063GiB/7.898GiB/7.943GiB
+2022-12-16 16:21:35.163,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.623MiB/s,23.686MiB/s,50.313GiB/21.813GiB/21.871GiB
+2022-12-16 16:22:49.836,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,5/5,20/20,11.627MiB/s,11.693MiB/s,3.915GiB/1.881GiB/1.892GiB
+2022-12-16 16:22:49.836,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,23.038MiB/s,23.252MiB/s,25.063GiB/9.578GiB/9.635GiB
+2022-12-16 16:22:49.836,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.614MiB/s,23.681MiB/s,50.313GiB/23.527GiB/23.594GiB
+2022-12-16 16:22:49.870,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,11.625MiB/s,11.693MiB/s,5.481GiB/1.881GiB/1.892GiB
+2022-12-16 16:22:49.870,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,23.033MiB/s,23.252MiB/s,25.063GiB/9.578GiB/9.635GiB
+2022-12-16 16:22:49.870,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.614MiB/s,23.681MiB/s,50.313GiB/23.527GiB/23.595GiB
+2022-12-16 16:22:49.887,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,11.623MiB/s,11.693MiB/s,5.481GiB/1.881GiB/1.893GiB
+2022-12-16 16:22:49.888,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,23.030MiB/s,23.250MiB/s,25.063GiB/9.578GiB/9.635GiB
+2022-12-16 16:22:49.891,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.613MiB/s,23.681MiB/s,50.313GiB/23.527GiB/23.595GiB
+2022-12-16 16:22:49.906,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,11.622MiB/s,11.693MiB/s,6.264GiB/1.881GiB/1.893GiB
+2022-12-16 16:22:49.906,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,23.027MiB/s,23.252MiB/s,25.063GiB/9.578GiB/9.636GiB
+2022-12-16 16:22:49.906,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.613MiB/s,23.681MiB/s,50.313GiB/23.527GiB/23.596GiB
+2022-12-16 16:24:33.731,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.330MiB/s,23.352MiB/s,6.264GiB/4.258GiB/4.262GiB
+2022-12-16 16:24:33.733,completed,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,22.922MiB/s,23.037MiB/s,25.063GiB/11.876GiB/11.931GiB
+2022-12-16 16:24:33.737,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.595MiB/s,23.649MiB/s,50.313GiB/25.902GiB/25.962GiB
+2022-12-16 16:24:38.074,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.229MiB/s,20.265MiB/s,5.482GiB/3.543GiB/3.548GiB
+2022-12-16 16:24:38.075,completed,1,T4,2.128 GiB,8.513 GiB,1,33,22.643GiB,8/8,32/32,22.869MiB/s,22.985MiB/s,25.063GiB/11.957GiB/12.014GiB
+2022-12-16 16:24:38.076,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.582MiB/s,23.639MiB/s,50.313GiB/25.988GiB/26.050GiB
+2022-12-16 16:24:43.323,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.221MiB/s,17.269MiB/s,4.698GiB/2.837GiB/2.843GiB
+2022-12-16 16:24:43.323,completed,1,T4,2.128 GiB,8.513 GiB,1,34,23.329GiB,8/8,32/32,22.816MiB/s,22.941MiB/s,25.063GiB/12.061GiB/12.120GiB
+2022-12-16 16:24:43.323,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.568MiB/s,23.626MiB/s,50.313GiB/26.093GiB/26.157GiB
+2022-12-16 16:24:43.833,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.295MiB/s,14.376MiB/s,3.915GiB/2.055GiB/2.064GiB
+2022-12-16 16:24:43.834,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.015GiB,8/8,32/32,22.779MiB/s,22.931MiB/s,25.063GiB/12.061GiB/12.128GiB
+2022-12-16 16:24:43.834,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.558MiB/s,23.625MiB/s,50.313GiB/26.093GiB/26.167GiB
+2022-12-16 16:25:53.601,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,11.208MiB/s,11.253MiB/s,4.698GiB/2.011GiB/2.019GiB
+2022-12-16 16:25:53.602,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,22.549MiB/s,22.683MiB/s,25.063GiB/13.517GiB/13.588GiB
+2022-12-16 16:25:53.613,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.453MiB/s,23.517MiB/s,50.313GiB/27.575GiB/27.650GiB
+2022-12-16 16:25:53.620,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,11.207MiB/s,11.253MiB/s,5.481GiB/2.011GiB/2.019GiB
+2022-12-16 16:25:53.635,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,22.548MiB/s,22.683MiB/s,25.063GiB/13.517GiB/13.588GiB
+2022-12-16 16:25:53.635,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.452MiB/s,23.517MiB/s,50.313GiB/27.575GiB/27.651GiB
+2022-12-16 16:25:53.645,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,11.206MiB/s,11.252MiB/s,6.264GiB/2.011GiB/2.019GiB
+2022-12-16 16:25:53.645,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,22.547MiB/s,22.683MiB/s,25.063GiB/13.517GiB/13.588GiB
+2022-12-16 16:25:53.647,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.452MiB/s,23.516MiB/s,50.313GiB/27.575GiB/27.651GiB
+2022-12-16 16:25:53.654,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,11.205MiB/s,11.252MiB/s,6.264GiB/2.011GiB/2.019GiB
+2022-12-16 16:25:53.669,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,22.546MiB/s,22.683MiB/s,25.063GiB/13.517GiB/13.589GiB
+2022-12-16 16:25:53.669,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.452MiB/s,23.517MiB/s,50.313GiB/27.575GiB/27.651GiB
+2022-12-16 16:27:31.042,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.023MiB/s,23.096MiB/s,6.264GiB/4.228GiB/4.237GiB
+2022-12-16 16:27:31.045,completed,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,22.605MiB/s,22.714MiB/s,25.063GiB/15.674GiB/15.749GiB
+2022-12-16 16:27:31.045,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.436MiB/s,23.492MiB/s,50.313GiB/29.785GiB/29.856GiB
+2022-12-16 16:27:31.939,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.149MiB/s,20.200MiB/s,5.481GiB/3.458GiB/3.464GiB
+2022-12-16 16:27:31.940,completed,1,T4,2.128 GiB,8.513 GiB,2,37,25.387GiB,8/8,32/32,22.588MiB/s,22.699MiB/s,25.063GiB/15.688GiB/15.761GiB
+2022-12-16 16:27:31.949,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.431MiB/s,23.486MiB/s,50.313GiB/29.799GiB/29.869GiB
+2022-12-16 16:27:33.633,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.265MiB/s,17.338MiB/s,4.698GiB/2.698GiB/2.707GiB
+2022-12-16 16:27:33.637,completed,1,T4,2.128 GiB,8.513 GiB,2,38,26.074GiB,8/8,32/32,22.575MiB/s,22.694MiB/s,25.063GiB/15.718GiB/15.796GiB
+2022-12-16 16:27:33.637,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.423MiB/s,23.482MiB/s,50.313GiB/29.828GiB/29.903GiB
+2022-12-16 16:27:34.510,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.443MiB/s,14.495MiB/s,3.915GiB/1.928GiB/1.933GiB
+2022-12-16 16:27:34.517,completed,1,T4,2.128 GiB,8.513 GiB,2,39,26.760GiB,8/8,32/32,22.578MiB/s,22.687MiB/s,25.063GiB/15.739GiB/15.812GiB
+2022-12-16 16:27:34.517,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.423MiB/s,23.480MiB/s,50.313GiB/29.848GiB/29.920GiB
+2022-12-16 16:28:29.851,completed,0,T4,0.000 B,2.128 GiB,3,28,4.801GiB,4/4,16/16,11.627MiB/s,11.664MiB/s,3.132GiB/1.774GiB/1.779GiB
+2022-12-16 16:28:29.852,completed,1,T4,2.128 GiB,8.513 GiB,2,40,27.446GiB,8/8,32/32,22.647MiB/s,22.762MiB/s,25.063GiB/16.996GiB/17.079GiB
+2022-12-16 16:28:29.853,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.065GiB,8/8,32/32,23.425MiB/s,23.480MiB/s,50.313GiB/31.117GiB/31.190GiB
+2022-12-16 16:28:36.659,completed,0,T4,0.000 B,2.128 GiB,3,28,4.801GiB,4/4,16/16,11.680MiB/s,11.737MiB/s,3.132GiB/1.859GiB/1.868GiB
+2022-12-16 16:28:36.660,completed,1,T4,2.128 GiB,8.513 GiB,2,36,24.701GiB,7/7,28/28,19.770MiB/s,19.883MiB/s,21.930GiB/13.992GiB/14.066GiB
+2022-12-16 16:28:36.660,completed,2,T4,8.513 GiB,34.052 GiB,2,42,57.819GiB,8/8,32/32,23.426MiB/s,23.489MiB/s,50.313GiB/31.274GiB/31.358GiB
+2022-12-16 16:28:42.403,completed,0,T4,0.000 B,2.128 GiB,3,28,4.801GiB,4/4,16/16,11.712MiB/s,11.760MiB/s,3.132GiB/1.930GiB/1.938GiB
+2022-12-16 16:28:42.403,completed,1,T4,2.128 GiB,8.513 GiB,2,32,21.957GiB,6/6,24/24,16.939MiB/s,17.022MiB/s,18.797GiB/10.962GiB/11.014GiB
+2022-12-16 16:28:42.403,completed,2,T4,8.513 GiB,34.052 GiB,2,44,60.573GiB,8/8,32/32,23.434MiB/s,23.492MiB/s,50.313GiB/31.416GiB/31.494GiB
+2022-12-16 16:28:52.278,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,11.836MiB/s,11.869MiB/s,4.698GiB/2.065GiB/2.070GiB
+2022-12-16 16:28:52.285,submitted,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,14.142MiB/s,14.204MiB/s,15.664GiB/7.986GiB/8.021GiB
+2022-12-16 16:28:52.286,submitted,2,T4,8.513 GiB,34.052 GiB,2,46,63.327GiB,8/8,32/32,23.462MiB/s,23.515MiB/s,50.313GiB/31.680GiB/31.751GiB
+2022-12-16 16:28:52.294,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,11.835MiB/s,11.869MiB/s,5.481GiB/2.065GiB/2.071GiB
+2022-12-16 16:28:52.295,submitted,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,14.141MiB/s,14.204MiB/s,15.664GiB/7.986GiB/8.021GiB
+2022-12-16 16:28:52.296,submitted,2,T4,8.513 GiB,34.052 GiB,2,46,63.327GiB,8/8,32/32,23.462MiB/s,23.515MiB/s,50.313GiB/31.680GiB/31.752GiB
+2022-12-16 16:28:52.304,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,11.835MiB/s,11.869MiB/s,5.481GiB/2.065GiB/2.071GiB
+2022-12-16 16:28:52.304,submitted,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,14.141MiB/s,14.204MiB/s,15.664GiB/7.986GiB/8.021GiB
+2022-12-16 16:28:52.304,submitted,2,T4,8.513 GiB,34.052 GiB,2,46,63.327GiB,8/8,32/32,23.462MiB/s,23.515MiB/s,50.313GiB/31.680GiB/31.752GiB
+2022-12-16 16:28:52.315,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,11.834MiB/s,11.870MiB/s,6.264GiB/2.065GiB/2.071GiB
+2022-12-16 16:28:52.315,submitted,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,14.141MiB/s,14.204MiB/s,15.664GiB/7.986GiB/8.021GiB
+2022-12-16 16:28:52.315,submitted,2,T4,8.513 GiB,34.052 GiB,2,46,63.327GiB,8/8,32/32,23.462MiB/s,23.516MiB/s,50.313GiB/31.680GiB/31.753GiB
+2022-12-16 16:28:56.278,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,22.553MiB/s,25.386MiB/s,6.264GiB/2.160GiB/2.177GiB
+2022-12-16 16:28:56.280,completed,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,14.156MiB/s,14.217MiB/s,15.664GiB/8.048GiB/8.082GiB
+2022-12-16 16:28:56.280,completed,2,T4,8.513 GiB,34.052 GiB,2,46,63.327GiB,8/8,32/32,23.473MiB/s,23.527MiB/s,50.313GiB/31.787GiB/31.859GiB
+2022-12-16 16:30:19.813,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.284MiB/s,23.347MiB/s,6.264GiB/4.025GiB/4.034GiB
+2022-12-16 16:30:19.813,completed,1,T4,2.128 GiB,8.513 GiB,2,24,16.468GiB,4/4,16/16,11.253MiB/s,11.311MiB/s,12.532GiB/5.797GiB/5.827GiB
+2022-12-16 16:30:19.813,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.424MiB/s,23.480MiB/s,50.313GiB/33.631GiB/33.711GiB
+2022-12-16 16:30:29.018,completed,0,T4,0.000 B,2.128 GiB,2,36,6.173GiB,7/7,28/28,20.344MiB/s,20.430MiB/s,5.482GiB/3.433GiB/3.442GiB
+2022-12-16 16:30:29.018,completed,1,T4,2.128 GiB,8.513 GiB,3,25,17.154GiB,4/4,16/16,11.254MiB/s,11.314MiB/s,12.532GiB/5.899GiB/5.930GiB
+2022-12-16 16:30:29.019,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.419MiB/s,23.476MiB/s,50.313GiB/33.834GiB/33.917GiB
+2022-12-16 16:30:31.266,completed,0,T4,0.000 B,2.128 GiB,2,32,5.487GiB,6/6,24/24,17.520MiB/s,17.576MiB/s,4.699GiB/2.697GiB/2.703GiB
+2022-12-16 16:30:31.268,completed,1,T4,2.128 GiB,8.513 GiB,3,26,17.840GiB,4/4,16/16,11.267MiB/s,11.321MiB/s,12.532GiB/5.930GiB/5.959GiB
+2022-12-16 16:30:31.268,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.427MiB/s,23.485MiB/s,50.313GiB/33.897GiB/33.981GiB
+2022-12-16 16:30:33.516,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.641MiB/s,14.718MiB/s,3.915GiB/1.947GiB/1.955GiB
+2022-12-16 16:30:33.516,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.527GiB,4/4,16/16,11.269MiB/s,11.325MiB/s,12.532GiB/5.956GiB/5.985GiB
+2022-12-16 16:30:33.516,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.432MiB/s,23.490MiB/s,50.313GiB/33.956GiB/34.040GiB
+2022-12-16 16:31:41.980,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,12.214MiB/s,12.261MiB/s,5.481GiB/2.024GiB/2.031GiB
+2022-12-16 16:31:41.983,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,11.383MiB/s,11.442MiB/s,12.532GiB/6.777GiB/6.812GiB
+2022-12-16 16:31:41.984,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.537MiB/s,23.595MiB/s,50.313GiB/35.682GiB/35.769GiB
+2022-12-16 16:31:42.004,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/6,28/24,12.212MiB/s,12.261MiB/s,5.481GiB/2.024GiB/2.032GiB
+2022-12-16 16:31:42.005,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,11.382MiB/s,11.442MiB/s,12.532GiB/6.777GiB/6.813GiB
+2022-12-16 16:31:42.005,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.537MiB/s,23.595MiB/s,50.313GiB/35.682GiB/35.769GiB
+2022-12-16 16:31:42.025,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.210MiB/s,12.262MiB/s,5.481GiB/2.024GiB/2.032GiB
+2022-12-16 16:31:42.030,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,11.382MiB/s,11.442MiB/s,12.532GiB/6.777GiB/6.813GiB
+2022-12-16 16:31:42.030,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.536MiB/s,23.595MiB/s,50.313GiB/35.682GiB/35.770GiB
+2022-12-16 16:31:42.041,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.209MiB/s,12.262MiB/s,6.264GiB/2.024GiB/2.032GiB
+2022-12-16 16:31:42.042,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,11.381MiB/s,11.442MiB/s,12.532GiB/6.777GiB/6.813GiB
+2022-12-16 16:31:42.042,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.536MiB/s,23.594MiB/s,50.313GiB/35.682GiB/35.770GiB
+2022-12-16 16:33:20.750,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.502MiB/s,23.555MiB/s,6.264GiB/4.237GiB/4.246GiB
+2022-12-16 16:33:20.751,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.213GiB,4/4,16/16,11.326MiB/s,11.379MiB/s,12.532GiB/7.836GiB/7.872GiB
+2022-12-16 16:33:20.751,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.479MiB/s,23.533MiB/s,50.313GiB/37.858GiB/37.945GiB
+2022-12-16 16:33:20.752,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.514MiB/s,20.566MiB/s,5.481GiB/3.454GiB/3.463GiB
+2022-12-16 16:33:20.752,submitted,1,T4,2.128 GiB,8.513 GiB,4,29,19.899GiB,5/5,20/20,11.326MiB/s,11.379MiB/s,15.665GiB/7.836GiB/7.872GiB
+2022-12-16 16:33:20.753,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.479MiB/s,23.533MiB/s,50.313GiB/37.858GiB/37.945GiB
+2022-12-16 16:33:21.573,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.405MiB/s,20.562MiB/s,5.481GiB/3.458GiB/3.478GiB
+2022-12-16 16:33:21.575,completed,1,T4,2.128 GiB,8.513 GiB,3,29,19.899GiB,5/5,20/20,11.313MiB/s,13.374MiB/s,15.665GiB/7.836GiB/7.882GiB
+2022-12-16 16:33:21.582,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.469MiB/s,23.532MiB/s,50.313GiB/37.861GiB/37.963GiB
+2022-12-16 16:33:21.584,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.487GiB,6/6,24/24,17.426MiB/s,17.584MiB/s,4.698GiB/2.675GiB/2.695GiB
+2022-12-16 16:33:21.586,submitted,1,T4,2.128 GiB,8.513 GiB,4,30,20.585GiB,6/6,24/24,11.313MiB/s,13.389MiB/s,18.798GiB/7.836GiB/7.882GiB
+2022-12-16 16:33:21.586,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.469MiB/s,23.532MiB/s,50.313GiB/37.861GiB/37.963GiB
+2022-12-16 16:33:22.679,completed,0,T4,0.000 B,2.128 GiB,2,32,5.487GiB,6/6,24/24,17.571MiB/s,17.627MiB/s,4.698GiB/2.710GiB/2.716GiB
+2022-12-16 16:33:22.681,completed,1,T4,2.128 GiB,8.513 GiB,3,30,20.585GiB,6/6,24/24,15.349MiB/s,16.864MiB/s,18.798GiB/7.858GiB/7.905GiB
+2022-12-16 16:33:22.681,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.480MiB/s,23.538MiB/s,50.313GiB/37.904GiB/37.998GiB
+2022-12-16 16:33:22.682,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.604MiB/s,14.660MiB/s,3.915GiB/1.927GiB/1.933GiB
+2022-12-16 16:33:22.682,submitted,1,T4,2.128 GiB,8.513 GiB,4,31,21.272GiB,7/7,28/28,15.340MiB/s,16.900MiB/s,21.932GiB/7.858GiB/7.905GiB
+2022-12-16 16:33:22.682,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.480MiB/s,23.538MiB/s,50.313GiB/37.904GiB/37.998GiB
+2022-12-16 16:33:22.774,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.635MiB/s,14.666MiB/s,3.915GiB/1.931GiB/1.934GiB
+2022-12-16 16:33:22.777,completed,1,T4,2.128 GiB,8.513 GiB,3,31,21.272GiB,7/7,28/28,16.486MiB/s,17.058MiB/s,21.932GiB/7.872GiB/7.908GiB
+2022-12-16 16:33:22.781,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.485MiB/s,23.539MiB/s,50.313GiB/37.915GiB/38.002GiB
+2022-12-16 16:33:22.786,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,11.670MiB/s,11.702MiB/s,3.132GiB/1.148GiB/1.151GiB
+2022-12-16 16:33:22.786,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.958GiB,8/8,32/32,16.446MiB/s,17.077MiB/s,25.064GiB/7.872GiB/7.908GiB
+2022-12-16 16:33:22.786,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.487MiB/s,23.539MiB/s,50.313GiB/37.918GiB/38.002GiB
+2022-12-16 16:34:31.489,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,5/5,20/20,11.765MiB/s,11.799MiB/s,3.915GiB/1.947GiB/1.953GiB
+2022-12-16 16:34:31.489,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.958GiB,8/8,32/32,23.185MiB/s,23.377MiB/s,25.064GiB/9.436GiB/9.487GiB
+2022-12-16 16:34:31.489,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.509MiB/s,23.566MiB/s,50.313GiB/39.531GiB/39.626GiB
+2022-12-16 16:34:31.498,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,11.764MiB/s,11.799MiB/s,4.698GiB/1.947GiB/1.953GiB
+2022-12-16 16:34:31.499,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.958GiB,8/8,32/32,23.184MiB/s,23.375MiB/s,25.064GiB/9.436GiB/9.487GiB
+2022-12-16 16:34:31.499,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.509MiB/s,23.566MiB/s,50.313GiB/39.531GiB/39.627GiB
+2022-12-16 16:34:31.502,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/7,32/28,11.764MiB/s,11.799MiB/s,6.264GiB/1.947GiB/1.953GiB
+2022-12-16 16:34:31.502,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.958GiB,8/8,32/32,23.183MiB/s,23.376MiB/s,25.064GiB/9.436GiB/9.487GiB
+2022-12-16 16:34:31.502,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.509MiB/s,23.566MiB/s,50.313GiB/39.531GiB/39.627GiB
+2022-12-16 16:34:31.515,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,11.763MiB/s,11.798MiB/s,6.264GiB/1.947GiB/1.953GiB
+2022-12-16 16:34:31.515,submitted,1,T4,2.128 GiB,8.513 GiB,-1,32,21.958GiB,8/8,32/32,23.181MiB/s,23.376MiB/s,25.064GiB/9.436GiB/9.487GiB
+2022-12-16 16:34:31.515,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.509MiB/s,23.566MiB/s,50.313GiB/39.531GiB/39.627GiB
+2022-12-16 16:36:13.104,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.262MiB/s,23.316MiB/s,6.264GiB/4.236GiB/4.244GiB
+2022-12-16 16:36:13.105,completed,1,T4,2.128 GiB,8.513 GiB,-1,32,21.958GiB,8/8,32/32,22.798MiB/s,22.939MiB/s,25.064GiB/11.657GiB/11.716GiB
+2022-12-16 16:36:13.106,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.477MiB/s,23.531MiB/s,50.313GiB/41.807GiB/41.903GiB
+2022-12-16 16:36:17.542,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.361MiB/s,20.448MiB/s,5.482GiB/3.549GiB/3.560GiB
+2022-12-16 16:36:17.543,completed,1,T4,2.128 GiB,8.513 GiB,1,33,22.643GiB,8/8,32/32,22.854MiB/s,22.988MiB/s,25.064GiB/11.770GiB/11.831GiB
+2022-12-16 16:36:17.543,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.483MiB/s,23.540MiB/s,50.313GiB/41.919GiB/42.021GiB
+2022-12-16 16:36:17.550,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.449MiB/s,17.536MiB/s,4.698GiB/2.766GiB/2.776GiB
+2022-12-16 16:36:17.551,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.016GiB,8/8,32/32,22.853MiB/s,22.987MiB/s,25.064GiB/11.770GiB/11.831GiB
+2022-12-16 16:36:17.551,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.483MiB/s,23.540MiB/s,50.313GiB/41.919GiB/42.021GiB
+2022-12-16 16:36:19.705,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,14.553MiB/s,14.650MiB/s,3.915GiB/2.016GiB/2.026GiB
+2022-12-16 16:36:19.705,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.016GiB,8/8,32/32,22.864MiB/s,23.006MiB/s,25.064GiB/11.822GiB/11.886GiB
+2022-12-16 16:36:19.706,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.485MiB/s,23.543MiB/s,50.313GiB/41.972GiB/42.076GiB
+2022-12-16 16:37:31.713,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/5,24/20,11.325MiB/s,11.376MiB/s,4.698GiB/1.993GiB/2.002GiB
+2022-12-16 16:37:31.717,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,22.571MiB/s,22.705MiB/s,25.064GiB/13.313GiB/13.382GiB
+2022-12-16 16:37:31.717,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.422MiB/s,23.478MiB/s,50.313GiB/43.507GiB/43.611GiB
+2022-12-16 16:37:31.731,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,11.324MiB/s,11.375MiB/s,4.698GiB/1.993GiB/2.002GiB
+2022-12-16 16:37:31.731,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,22.570MiB/s,22.705MiB/s,25.064GiB/13.313GiB/13.382GiB
+2022-12-16 16:37:31.731,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.422MiB/s,23.478MiB/s,50.313GiB/43.507GiB/43.612GiB
+2022-12-16 16:37:31.755,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,11.322MiB/s,11.375MiB/s,5.481GiB/1.993GiB/2.002GiB
+2022-12-16 16:37:31.755,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,22.569MiB/s,22.704MiB/s,25.064GiB/13.313GiB/13.382GiB
+2022-12-16 16:37:31.755,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.422MiB/s,23.478MiB/s,50.313GiB/43.507GiB/43.612GiB
+2022-12-16 16:37:31.769,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,11.321MiB/s,11.375MiB/s,6.264GiB/1.993GiB/2.002GiB
+2022-12-16 16:37:31.773,submitted,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,22.568MiB/s,22.705MiB/s,25.064GiB/13.313GiB/13.383GiB
+2022-12-16 16:37:31.773,submitted,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.421MiB/s,23.478MiB/s,50.313GiB/43.507GiB/43.612GiB
+2022-12-16 16:39:06.341,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,23.371MiB/s,23.454MiB/s,6.264GiB/4.180GiB/4.193GiB
+2022-12-16 16:39:06.341,completed,1,T4,2.128 GiB,8.513 GiB,1,36,24.702GiB,8/8,32/32,22.609MiB/s,22.737MiB/s,25.064GiB/15.426GiB/15.508GiB
+2022-12-16 16:39:06.341,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.428MiB/s,23.485MiB/s,50.313GiB/45.682GiB/45.793GiB
+2022-12-16 16:39:10.845,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,20.349MiB/s,20.438MiB/s,5.481GiB/3.476GiB/3.486GiB
+2022-12-16 16:39:10.861,completed,1,T4,2.128 GiB,8.513 GiB,2,37,25.389GiB,8/8,32/32,22.599MiB/s,22.706MiB/s,25.064GiB/15.521GiB/15.592GiB
+2022-12-16 16:39:10.861,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.425MiB/s,23.479MiB/s,50.313GiB/45.780GiB/45.885GiB
+2022-12-16 16:39:12.124,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,17.521MiB/s,17.552MiB/s,4.698GiB/2.719GiB/2.722GiB
+2022-12-16 16:39:12.124,completed,1,T4,2.128 GiB,8.513 GiB,2,38,26.075GiB,8/8,32/32,22.595MiB/s,22.709MiB/s,25.064GiB/15.547GiB/15.621GiB
+2022-12-16 16:39:12.124,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.425MiB/s,23.478MiB/s,50.313GiB/45.809GiB/45.913GiB
+2022-12-16 16:39:14.367,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,14.647MiB/s,14.691MiB/s,3.915GiB/1.967GiB/1.971GiB
+2022-12-16 16:39:14.367,completed,1,T4,2.128 GiB,8.513 GiB,2,39,26.761GiB,8/8,32/32,22.586MiB/s,22.711MiB/s,25.064GiB/15.592GiB/15.672GiB
+2022-12-16 16:39:14.367,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.422MiB/s,23.478MiB/s,50.313GiB/45.855GiB/45.965GiB
+2022-12-16 16:40:08.157,completed,0,T4,0.000 B,2.128 GiB,3,28,4.800GiB,4/4,16/16,12.072MiB/s,12.120MiB/s,3.132GiB/1.844GiB/1.851GiB
+2022-12-16 16:40:08.157,completed,1,T4,2.128 GiB,8.513 GiB,2,40,27.447GiB,8/8,32/32,22.795MiB/s,22.925MiB/s,25.064GiB/16.896GiB/16.985GiB
+2022-12-16 16:40:08.157,completed,2,T4,8.513 GiB,34.052 GiB,2,48,66.081GiB,8/8,32/32,23.468MiB/s,23.528MiB/s,50.313GiB/47.178GiB/47.297GiB
+2022-12-16 16:40:17.074,completed,0,T4,0.000 B,2.128 GiB,3,28,4.800GiB,4/4,16/16,12.404MiB/s,12.484MiB/s,3.132GiB/2.003GiB/2.015GiB
+2022-12-16 16:40:17.077,completed,1,T4,2.128 GiB,8.513 GiB,2,36,24.702GiB,7/7,28/28,20.096MiB/s,20.215MiB/s,21.931GiB/14.034GiB/14.109GiB
+2022-12-16 16:40:17.085,completed,2,T4,8.513 GiB,34.052 GiB,3,50,68.835GiB,8/8,32/32,23.527MiB/s,23.590MiB/s,50.313GiB/47.502GiB/47.628GiB
+2022-12-16 16:40:17.411,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,12.462MiB/s,12.493MiB/s,5.481GiB/2.016GiB/2.021GiB
+2022-12-16 16:40:17.411,submitted,1,T4,2.128 GiB,8.513 GiB,2,32,21.957GiB,6/6,24/24,17.277MiB/s,17.363MiB/s,18.797GiB/10.918GiB/10.971GiB
+2022-12-16 16:40:17.411,submitted,2,T4,8.513 GiB,34.052 GiB,3,52,71.591GiB,8/8,32/32,23.539MiB/s,23.591MiB/s,50.313GiB/47.532GiB/47.639GiB
+2022-12-16 16:40:17.423,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,12.461MiB/s,12.494MiB/s,4.698GiB/2.016GiB/2.021GiB
+2022-12-16 16:40:17.423,submitted,1,T4,2.128 GiB,8.513 GiB,2,32,21.957GiB,6/6,24/24,17.276MiB/s,17.363MiB/s,18.797GiB/10.918GiB/10.971GiB
+2022-12-16 16:40:17.423,submitted,2,T4,8.513 GiB,34.052 GiB,3,52,71.591GiB,8/8,32/32,23.538MiB/s,23.592MiB/s,50.313GiB/47.532GiB/47.639GiB
+2022-12-16 16:40:17.438,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,12.460MiB/s,12.494MiB/s,5.481GiB/2.016GiB/2.021GiB
+2022-12-16 16:40:17.438,submitted,1,T4,2.128 GiB,8.513 GiB,2,32,21.957GiB,6/6,24/24,17.276MiB/s,17.363MiB/s,18.797GiB/10.918GiB/10.971GiB
+2022-12-16 16:40:17.438,submitted,2,T4,8.513 GiB,34.052 GiB,3,52,71.591GiB,8/8,32/32,23.538MiB/s,23.592MiB/s,50.313GiB/47.532GiB/47.640GiB
+2022-12-16 16:40:17.448,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,12.459MiB/s,12.494MiB/s,6.264GiB/2.016GiB/2.022GiB
+2022-12-16 16:40:17.449,submitted,1,T4,2.128 GiB,8.513 GiB,2,32,21.957GiB,6/6,24/24,17.276MiB/s,17.363MiB/s,18.797GiB/10.918GiB/10.971GiB
+2022-12-16 16:40:17.449,submitted,2,T4,8.513 GiB,34.052 GiB,3,52,71.591GiB,8/8,32/32,23.538MiB/s,23.592MiB/s,50.313GiB/47.532GiB/47.640GiB
+2022-12-16 16:40:21.291,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,27.110MiB/s,29.662MiB/s,6.264GiB/2.129GiB/2.146GiB
+2022-12-16 16:40:21.291,completed,1,T4,2.128 GiB,8.513 GiB,2,32,21.957GiB,6/6,24/24,17.326MiB/s,17.419MiB/s,18.797GiB/11.008GiB/11.063GiB
+2022-12-16 16:40:21.291,completed,2,T4,8.513 GiB,34.052 GiB,3,52,71.591GiB,8/8,32/32,23.550MiB/s,23.607MiB/s,50.313GiB/47.644GiB/47.759GiB
+2022-12-16 16:40:28.770,completed,0,T4,0.000 B,2.128 GiB,-1,32,5.486GiB,8/8,32/32,31.874MiB/s,33.733MiB/s,6.264GiB/2.432GiB/2.462GiB
+2022-12-16 16:40:28.770,completed,1,T4,2.128 GiB,8.513 GiB,2,28,19.212GiB,5/5,20/20,14.650MiB/s,14.735MiB/s,15.664GiB/8.060GiB/8.103GiB
+2022-12-16 16:40:28.770,completed,2,T4,8.513 GiB,34.052 GiB,3,54,74.345GiB,8/8,32/32,23.612MiB/s,23.671MiB/s,50.313GiB/47.941GiB/48.063GiB
+2022-12-16 16:40:55.273,completed,0,T4,0.000 B,2.128 GiB,1,36,6.172GiB,8/8,32/32,44.575MiB/s,44.934MiB/s,6.264GiB/4.151GiB/4.179GiB
+2022-12-16 16:40:55.273,completed,1,T4,2.128 GiB,8.513 GiB,2,24,16.468GiB,4/4,16/16,13.005MiB/s,13.116MiB/s,12.532GiB/5.757GiB/5.806GiB
+2022-12-16 16:40:55.273,completed,2,T4,8.513 GiB,34.052 GiB,3,56,77.100GiB,8/8,32/32,24.135MiB/s,24.211MiB/s,50.313GiB/49.629GiB/49.785GiB
+2022-12-16 16:40:56.810,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,40.659MiB/s,41.135MiB/s,5.481GiB/3.454GiB/3.475GiB
+2022-12-16 16:40:56.816,completed,1,T4,2.128 GiB,8.513 GiB,3,25,17.154GiB,4/4,16/16,13.102MiB/s,13.188MiB/s,12.532GiB/5.820GiB/5.858GiB
+2022-12-16 16:40:56.816,completed,2,T4,8.513 GiB,34.052 GiB,3,56,77.100GiB,8/8,32/32,24.178MiB/s,24.239MiB/s,50.313GiB/49.753GiB/49.878GiB
+2022-12-16 16:40:56.914,completed,0,T4,0.000 B,2.128 GiB,2,32,5.487GiB,6/6,24/24,36.670MiB/s,37.269MiB/s,4.698GiB/2.671GiB/2.698GiB
+2022-12-16 16:40:56.915,completed,1,T4,2.128 GiB,8.513 GiB,3,26,17.840GiB,4/4,16/16,13.099MiB/s,13.193MiB/s,12.532GiB/5.820GiB/5.861GiB
+2022-12-16 16:40:56.915,completed,2,T4,8.513 GiB,34.052 GiB,3,56,77.100GiB,8/8,32/32,24.176MiB/s,24.242MiB/s,50.313GiB/49.753GiB/49.887GiB
+2022-12-16 16:40:57.126,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,32.601MiB/s,33.433MiB/s,3.915GiB/1.888GiB/1.925GiB
+2022-12-16 16:40:57.126,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.526GiB,4/4,16/16,13.093MiB/s,13.203MiB/s,12.532GiB/5.820GiB/5.869GiB
+2022-12-16 16:40:57.126,completed,2,T4,8.513 GiB,34.052 GiB,3,56,77.100GiB,8/8,32/32,24.174MiB/s,24.248MiB/s,50.313GiB/49.753GiB/49.904GiB
+2022-12-16 16:40:57.871,completed,0,T4,0.000 B,2.128 GiB,2,28,4.800GiB,5/5,20/20,33.019MiB/s,33.762MiB/s,3.915GiB/1.933GiB/1.962GiB
+2022-12-16 16:40:57.871,completed,1,T4,2.128 GiB,8.513 GiB,3,27,18.526GiB,4/4,16/16,13.145MiB/s,13.243MiB/s,12.532GiB/5.852GiB/5.896GiB
+2022-12-16 16:40:57.871,completed,2,T4,8.513 GiB,34.052 GiB,3,52,71.593GiB,7/7,28/28,21.165MiB/s,21.226MiB/s,44.023GiB/43.539GiB/43.664GiB
+2022-12-16 16:40:57.871,completed,3,T4,34.052 GiB,136.206 GiB,1,4,5.530GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:40:59.731,completed,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,30.084MiB/s,30.624MiB/s,3.132GiB/1.242GiB/1.265GiB
+2022-12-16 16:40:59.731,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,13.267MiB/s,13.359MiB/s,12.532GiB/5.931GiB/5.972GiB
+2022-12-16 16:40:59.731,completed,2,T4,8.513 GiB,34.052 GiB,3,52,71.593GiB,7/7,28/28,21.228MiB/s,21.279MiB/s,44.023GiB/43.707GiB/43.812GiB
+2022-12-16 16:40:59.731,completed,3,T4,34.052 GiB,136.206 GiB,1,4,5.530GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:40:59.771,completed,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,30.056MiB/s,30.644MiB/s,3.132GiB/1.242GiB/1.267GiB
+2022-12-16 16:40:59.773,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,13.266MiB/s,13.362MiB/s,12.532GiB/5.931GiB/5.974GiB
+2022-12-16 16:40:59.773,completed,2,T4,8.513 GiB,34.052 GiB,3,48,66.087GiB,6/6,24/24,18.179MiB/s,18.225MiB/s,37.734GiB/37.417GiB/37.512GiB
+2022-12-16 16:40:59.773,completed,3,T4,34.052 GiB,136.206 GiB,1,8,11.061GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:00.943,completed,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,30.295MiB/s,31.234MiB/s,3.132GiB/1.287GiB/1.327GiB
+2022-12-16 16:41:00.943,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,13.325MiB/s,13.446MiB/s,12.532GiB/5.973GiB/6.027GiB
+2022-12-16 16:41:00.943,completed,2,T4,8.513 GiB,34.052 GiB,3,44,60.581GiB,5/5,20/20,15.149MiB/s,15.192MiB/s,31.445GiB/31.183GiB/31.272GiB
+2022-12-16 16:41:00.943,completed,3,T4,34.052 GiB,136.206 GiB,1,12,16.591GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:02.066,completed,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,30.819MiB/s,31.745MiB/s,3.132GiB/1.343GiB/1.383GiB
+2022-12-16 16:41:02.067,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,13.421MiB/s,13.556MiB/s,12.532GiB/6.030GiB/6.091GiB
+2022-12-16 16:41:02.067,completed,2,T4,8.513 GiB,34.052 GiB,3,40,55.075GiB,4/4,16/16,12.135MiB/s,12.170MiB/s,25.155GiB/24.947GiB/25.018GiB
+2022-12-16 16:41:02.067,completed,3,T4,34.052 GiB,136.206 GiB,1,16,22.121GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:04.841,completed,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,32.591MiB/s,33.370MiB/s,3.132GiB/1.509GiB/1.545GiB
+2022-12-16 16:41:04.841,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,13.700MiB/s,13.813MiB/s,12.532GiB/6.193GiB/6.244GiB
+2022-12-16 16:41:04.841,completed,2,T4,8.513 GiB,34.052 GiB,3,36,49.569GiB,3/3,12/12,9.124MiB/s,9.148MiB/s,18.867GiB/18.776GiB/18.827GiB
+2022-12-16 16:41:04.842,completed,3,T4,34.052 GiB,136.206 GiB,1,20,27.651GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:07.122,completed,0,T4,0.000 B,2.128 GiB,3,28,4.801GiB,4/4,16/16,33.599MiB/s,34.645MiB/s,3.132GiB/1.630GiB/1.681GiB
+2022-12-16 16:41:07.122,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,13.883MiB/s,14.042MiB/s,12.532GiB/6.306GiB/6.379GiB
+2022-12-16 16:41:07.122,completed,2,T4,8.513 GiB,34.052 GiB,3,32,44.063GiB,2/2,8/8,6.090MiB/s,6.108MiB/s,12.578GiB/12.549GiB/12.586GiB
+2022-12-16 16:41:07.122,completed,3,T4,34.052 GiB,136.206 GiB,1,24,33.181GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:08.191,completed,0,T4,0.000 B,2.128 GiB,3,28,4.801GiB,4/4,16/16,34.127MiB/s,35.403MiB/s,3.132GiB/1.691GiB/1.755GiB
+2022-12-16 16:41:08.191,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,13.994MiB/s,14.163MiB/s,12.532GiB/6.371GiB/6.448GiB
+2022-12-16 16:41:08.191,completed,2,T4,8.513 GiB,34.052 GiB,3,28,38.558GiB,1/1,4/4,3.048MiB/s,3.055MiB/s,6.289GiB/6.289GiB/6.303GiB
+2022-12-16 16:41:08.191,completed,3,T4,34.052 GiB,136.206 GiB,1,28,38.710GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:11.254,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/5,28/20,36.535MiB/s,37.498MiB/s,5.481GiB/1.920GiB/1.971GiB
+2022-12-16 16:41:11.254,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,14.391MiB/s,14.597MiB/s,12.532GiB/6.596GiB/6.690GiB
+2022-12-16 16:41:11.254,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:11.255,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:11.257,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,6/6,24/24,36.533MiB/s,37.500MiB/s,4.698GiB/1.920GiB/1.971GiB
+2022-12-16 16:41:11.258,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,14.391MiB/s,14.598MiB/s,12.532GiB/6.596GiB/6.690GiB
+2022-12-16 16:41:11.258,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:11.258,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:11.261,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,7/7,28/28,36.530MiB/s,37.502MiB/s,5.481GiB/1.920GiB/1.971GiB
+2022-12-16 16:41:11.261,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,14.391MiB/s,14.598MiB/s,12.532GiB/6.596GiB/6.690GiB
+2022-12-16 16:41:11.261,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:11.261,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:11.263,submitted,0,T4,0.000 B,2.128 GiB,4,32,5.486GiB,8/8,32/32,36.868MiB/s,37.503MiB/s,6.264GiB/1.938GiB/1.971GiB
+2022-12-16 16:41:11.263,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,14.391MiB/s,14.598MiB/s,12.532GiB/6.596GiB/6.690GiB
+2022-12-16 16:41:11.263,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:11.264,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:26.180,completed,0,T4,0.000 B,2.128 GiB,2,40,6.858GiB,8/8,32/32,118.016MiB/s,124.531MiB/s,6.264GiB/4.063GiB/4.201GiB
+2022-12-16 16:41:26.180,completed,1,T4,2.128 GiB,8.513 GiB,3,28,19.212GiB,4/4,16/16,16.246MiB/s,16.451MiB/s,12.532GiB/7.682GiB/7.779GiB
+2022-12-16 16:41:26.180,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:26.180,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:26.181,submitted,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,106.343MiB/s,112.865MiB/s,5.481GiB/3.279GiB/3.418GiB
+2022-12-16 16:41:26.181,submitted,1,T4,2.128 GiB,8.513 GiB,4,29,19.899GiB,5/5,20/20,16.246MiB/s,16.451MiB/s,15.664GiB/7.682GiB/7.779GiB
+2022-12-16 16:41:26.181,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:26.181,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:27.089,completed,0,T4,0.000 B,2.128 GiB,2,36,6.172GiB,7/7,28/28,108.771MiB/s,113.045MiB/s,5.481GiB/3.446GiB/3.541GiB
+2022-12-16 16:41:27.089,completed,1,T4,2.128 GiB,8.513 GiB,3,29,19.899GiB,5/5,20/20,16.399MiB/s,49.157MiB/s,15.664GiB/7.769GiB/7.893GiB
+2022-12-16 16:41:27.089,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:27.089,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:27.090,submitted,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,97.253MiB/s,101.534MiB/s,4.698GiB/2.663GiB/2.758GiB
+2022-12-16 16:41:27.091,submitted,1,T4,2.128 GiB,8.513 GiB,4,30,20.585GiB,6/6,24/24,16.399MiB/s,49.169MiB/s,18.797GiB/7.769GiB/7.894GiB
+2022-12-16 16:41:27.091,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:27.091,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:27.856,completed,0,T4,0.000 B,2.128 GiB,2,32,5.486GiB,6/6,24/24,98.962MiB/s,102.343MiB/s,4.698GiB/2.791GiB/2.855GiB
+2022-12-16 16:41:27.856,completed,1,T4,2.128 GiB,8.513 GiB,3,30,20.585GiB,6/6,24/24,36.219MiB/s,71.233MiB/s,18.797GiB/7.905GiB/8.016GiB
+2022-12-16 16:41:27.857,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:27.857,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:27.858,submitted,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,87.569MiB/s,90.957MiB/s,3.915GiB/2.008GiB/2.072GiB
+2022-12-16 16:41:27.858,submitted,1,T4,2.128 GiB,8.513 GiB,4,31,21.271GiB,7/7,28/28,36.198MiB/s,71.236MiB/s,21.930GiB/7.905GiB/8.016GiB
+2022-12-16 16:41:27.858,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:27.858,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:28.082,completed,0,T4,0.000 B,2.128 GiB,2,28,4.801GiB,5/5,20/20,86.687MiB/s,90.517MiB/s,3.915GiB/2.020GiB/2.083GiB
+2022-12-16 16:41:28.082,completed,1,T4,2.128 GiB,8.513 GiB,3,31,21.271GiB,7/7,28/28,33.880MiB/s,78.431MiB/s,21.930GiB/7.905GiB/8.039GiB
+2022-12-16 16:41:28.082,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:28.082,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:28.084,submitted,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,75.324MiB/s,79.161MiB/s,3.132GiB/1.237GiB/1.300GiB
+2022-12-16 16:41:28.084,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,21.957GiB,8/8,32/32,33.865MiB/s,78.509MiB/s,25.063GiB/7.905GiB/8.039GiB
+2022-12-16 16:41:28.084,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:28.084,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:47.004,completed,0,T4,0.000 B,2.128 GiB,2,24,4.115GiB,4/4,16/16,84.455MiB/s,85.613MiB/s,3.132GiB/2.947GiB/2.988GiB
+2022-12-16 16:41:47.004,completed,1,T4,2.128 GiB,8.513 GiB,-1,32,21.957GiB,8/8,32/32,114.981MiB/s,118.795MiB/s,25.063GiB/11.371GiB/11.526GiB
+2022-12-16 16:41:47.004,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:47.004,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:47.452,completed,0,T4,0.000 B,2.128 GiB,2,20,3.429GiB,3/3,12/12,62.923MiB/s,63.064MiB/s,2.349GiB/2.224GiB/2.229GiB
+2022-12-16 16:41:47.452,completed,1,T4,2.128 GiB,8.513 GiB,1,33,22.643GiB,8/8,32/32,116.236MiB/s,118.726MiB/s,25.063GiB/11.439GiB/11.607GiB
+2022-12-16 16:41:47.452,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:47.452,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:49.836,completed,0,T4,0.000 B,2.128 GiB,2,16,2.743GiB,2/2,8/8,40.453MiB/s,40.690MiB/s,1.566GiB/1.524GiB/1.533GiB
+2022-12-16 16:41:49.836,completed,1,T4,2.128 GiB,8.513 GiB,1,34,23.329GiB,8/8,32/32,114.359MiB/s,117.684MiB/s,25.063GiB/11.862GiB/11.998GiB
+2022-12-16 16:41:49.837,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:49.837,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:51.736,completed,0,T4,0.000 B,2.128 GiB,2,12,2.057GiB,1/1,4/4,19.810MiB/s,19.815MiB/s,801.740MiB/801.740MiB/801.911MiB
+2022-12-16 16:41:51.737,completed,1,T4,2.128 GiB,8.513 GiB,1,35,24.015GiB,8/8,32/32,114.878MiB/s,118.246MiB/s,25.063GiB/12.199GiB/12.337GiB
+2022-12-16 16:41:51.737,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:41:51.737,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:15.753,completed,0,T4,0.000 B,2.128 GiB,2,8,1.372GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:15.753,completed,1,T4,2.128 GiB,8.513 GiB,1,36,24.701GiB,8/8,32/32,123.429MiB/s,125.924MiB/s,25.063GiB/16.840GiB/17.025GiB
+2022-12-16 16:42:15.753,completed,2,T4,8.513 GiB,34.052 GiB,3,24,33.051GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:15.753,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:15.754,submitted,0,T4,0.000 B,2.128 GiB,2,8,1.372GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:15.755,submitted,1,T4,2.128 GiB,8.513 GiB,1,32,21.956GiB,7/7,28/28,117.406MiB/s,119.878MiB/s,21.929GiB/13.706GiB/13.877GiB
+2022-12-16 16:42:15.755,submitted,2,T4,8.513 GiB,34.052 GiB,4,26,35.808GiB,1/1,4/4,0B/s,0B/s,6.293GiB/0B/0B
+2022-12-16 16:42:15.755,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:15.756,submitted,0,T4,0.000 B,2.128 GiB,2,8,1.372GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:15.756,submitted,1,T4,2.128 GiB,8.513 GiB,1,32,21.956GiB,7/7,28/28,117.402MiB/s,119.880MiB/s,21.929GiB/13.706GiB/13.878GiB
+2022-12-16 16:42:15.756,submitted,2,T4,8.513 GiB,34.052 GiB,4,26,35.808GiB,2/2,8/8,0B/s,0B/s,12.586GiB/0B/0B
+2022-12-16 16:42:15.757,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:18.717,completed,0,T4,0.000 B,2.128 GiB,2,8,1.372GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:18.717,completed,1,T4,2.128 GiB,8.513 GiB,1,32,21.956GiB,7/7,28/28,119.740MiB/s,121.547MiB/s,21.929GiB/14.347GiB/14.482GiB
+2022-12-16 16:42:18.718,completed,2,T4,8.513 GiB,34.052 GiB,3,26,35.808GiB,2/2,8/8,38.341MiB/s,55.428MiB/s,12.586GiB/113.512MiB/164.098MiB
+2022-12-16 16:42:18.718,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:18.719,submitted,0,T4,0.000 B,2.128 GiB,2,8,1.372GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:18.719,submitted,1,T4,2.128 GiB,8.513 GiB,1,28,19.212GiB,6/6,24/24,113.765MiB/s,115.548MiB/s,18.797GiB/11.215GiB/11.336GiB
+2022-12-16 16:42:18.719,submitted,2,T4,8.513 GiB,34.052 GiB,4,28,38.562GiB,4/3,16/12,38.319MiB/s,55.438MiB/s,25.169GiB/113.512MiB/164.223MiB
+2022-12-16 16:42:18.719,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:18.721,submitted,0,T4,0.000 B,2.128 GiB,2,8,1.372GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:18.721,submitted,1,T4,2.128 GiB,8.513 GiB,1,28,19.212GiB,6/6,24/24,113.762MiB/s,115.547MiB/s,18.797GiB/11.215GiB/11.336GiB
+2022-12-16 16:42:18.721,submitted,2,T4,8.513 GiB,34.052 GiB,4,28,38.562GiB,4/4,16/16,38.297MiB/s,55.435MiB/s,25.169GiB/113.512MiB/164.305MiB
+2022-12-16 16:42:18.721,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:20.080,completed,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:20.080,completed,1,T4,2.128 GiB,8.513 GiB,1,28,19.212GiB,6/6,24/24,113.878MiB/s,115.659MiB/s,18.797GiB/11.414GiB/11.539GiB
+2022-12-16 16:42:20.080,completed,2,T4,8.513 GiB,34.052 GiB,3,28,38.562GiB,4/4,16/16,93.498MiB/s,113.141MiB/s,25.169GiB/289.414MiB/329.818MiB
+2022-12-16 16:42:20.080,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:20.081,submitted,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:20.082,submitted,1,T4,2.128 GiB,8.513 GiB,1,24,16.467GiB,5/5,20/20,107.926MiB/s,109.684MiB/s,15.663GiB/8.281GiB/8.392GiB
+2022-12-16 16:42:20.082,submitted,2,T4,8.513 GiB,34.052 GiB,4,30,41.318GiB,6/5,24/20,93.429MiB/s,113.145MiB/s,37.753GiB/289.414MiB/330.023MiB
+2022-12-16 16:42:20.082,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:20.083,submitted,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:20.083,submitted,1,T4,2.128 GiB,8.513 GiB,1,24,16.467GiB,5/5,20/20,107.922MiB/s,109.683MiB/s,15.663GiB/8.281GiB/8.392GiB
+2022-12-16 16:42:20.083,submitted,2,T4,8.513 GiB,34.052 GiB,4,30,41.318GiB,6/6,24/24,93.357MiB/s,113.157MiB/s,37.753GiB/289.414MiB/330.245MiB
+2022-12-16 16:42:20.083,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:23.853,completed,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:23.853,completed,1,T4,2.128 GiB,8.513 GiB,1,24,16.467GiB,5/5,20/20,107.526MiB/s,109.506MiB/s,15.663GiB/8.741GiB/8.862GiB
+2022-12-16 16:42:23.853,completed,2,T4,8.513 GiB,34.052 GiB,3,30,41.318GiB,6/6,24/24,161.933MiB/s,172.271MiB/s,37.753GiB/923.131MiB/966.911MiB
+2022-12-16 16:42:23.853,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:23.854,submitted,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:23.855,submitted,1,T4,2.128 GiB,8.513 GiB,1,20,13.723GiB,4/4,16/16,101.596MiB/s,103.552MiB/s,12.531GiB/5.609GiB/5.716GiB
+2022-12-16 16:42:23.855,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.073GiB,7/7,28/28,161.885MiB/s,172.260MiB/s,44.045GiB/923.131MiB/967.139MiB
+2022-12-16 16:42:23.855,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:23.856,submitted,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:42:23.856,submitted,1,T4,2.128 GiB,8.513 GiB,1,20,13.723GiB,4/4,16/16,101.593MiB/s,103.552MiB/s,12.531GiB/5.609GiB/5.716GiB
+2022-12-16 16:42:23.856,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,44.073GiB,8/8,32/32,161.833MiB/s,172.255MiB/s,50.336GiB/923.131MiB/967.409MiB
+2022-12-16 16:42:23.856,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:43:41.590,completed,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:43:41.590,completed,1,T4,2.128 GiB,8.513 GiB,1,20,13.723GiB,4/4,16/16,92.379MiB/s,93.086MiB/s,12.531GiB/12.113GiB/12.206GiB
+2022-12-16 16:43:41.590,completed,2,T4,8.513 GiB,34.052 GiB,-1,32,44.073GiB,8/8,32/32,174.854MiB/s,176.330MiB/s,50.336GiB/14.011GiB/14.129GiB
+2022-12-16 16:43:41.590,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:43:45.372,completed,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:43:45.372,completed,1,T4,2.128 GiB,8.513 GiB,1,16,10.979GiB,3/3,12/12,68.213MiB/s,68.680MiB/s,9.399GiB/9.209GiB/9.272GiB
+2022-12-16 16:43:45.372,completed,2,T4,8.513 GiB,34.052 GiB,1,34,46.828GiB,8/8,32/32,174.940MiB/s,175.798MiB/s,50.336GiB/14.663GiB/14.735GiB
+2022-12-16 16:43:45.372,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:43:48.344,completed,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:43:48.344,completed,1,T4,2.128 GiB,8.513 GiB,1,12,8.234GiB,2/2,8/8,44.847MiB/s,45.134MiB/s,6.266GiB/6.206GiB/6.246GiB
+2022-12-16 16:43:48.344,completed,2,T4,8.513 GiB,34.052 GiB,1,36,49.583GiB,8/8,32/32,175.174MiB/s,176.158MiB/s,50.336GiB/15.191GiB/15.276GiB
+2022-12-16 16:43:48.345,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:43:50.894,completed,0,T4,0.000 B,2.128 GiB,3,12,1.972GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:43:50.894,completed,1,T4,2.128 GiB,8.513 GiB,1,8,5.490GiB,1/1,4/4,22.314MiB/s,22.413MiB/s,3.134GiB/3.134GiB/3.147GiB
+2022-12-16 16:43:50.894,completed,2,T4,8.513 GiB,34.052 GiB,1,38,52.338GiB,8/8,32/32,174.284MiB/s,176.507MiB/s,50.336GiB/15.548GiB/15.746GiB
+2022-12-16 16:43:50.895,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:10.808,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.658GiB,4/1,16/4,0B/s,0B/s,3.035GiB/0B/0B
+2022-12-16 16:46:10.809,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:10.809,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,
+2022-12-16 16:46:10.809,submitted,3,T4,34.052 GiB,8/8,32/32,177.708MiB/s,178.111MiB/s,50.336GiB/40.128GiB/40.219GiB136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:10.814,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.658GiB,2/2,8/8,0B/s,0B/s,1.517GiB/0B/0B
+2022-12-16 16:46:10.814,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:10.814,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,8/8,32/32,177.703MiB/s,178.111MiB/s,50.336GiB/40.128GiB/40.220GiB
+2022-12-16 16:46:10.815,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:10.816,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.658GiB,4/3,16/12,0B/s,0B/s,3.035GiB/0B/0B
+2022-12-16 16:46:10.816,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:10.816,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,8/8,32/32,177.702MiB/s,178.111MiB/s,50.336GiB/40.128GiB/40.220GiB
+2022-12-16 16:46:10.816,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:10.818,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.658GiB,4/4,16/16,0B/s,0B/s,3.035GiB/0B/0B
+2022-12-16 16:46:10.818,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:10.818,submitted,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,8/8,32/32,177.701MiB/s,178.111MiB/s,50.336GiB/40.128GiB/40.220GiB
+2022-12-16 16:46:10.818,submitted,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:50.206,completed,0,T4,0.000 B,2.128 GiB,-1,16,2.658GiB,4/4,16/16,78.033MiB/s,78.385MiB/s,3.035GiB/3.001GiB/3.015GiB
+2022-12-16 16:46:50.206,completed,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:50.206,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,8/8,32/32,174.071MiB/s,174.564MiB/s,50.336GiB/46.003GiB/46.134GiB
+2022-12-16 16:46:50.206,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:50.731,completed,0,T4,0.000 B,2.128 GiB,-1,12,1.994GiB,3/3,12/12,57.702MiB/s,58.193MiB/s,2.277GiB/2.249GiB/2.268GiB
+2022-12-16 16:46:50.731,completed,1,T4,2.128 GiB,8.513 GiB,2,5,3.410GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:50.731,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,8/8,32/32,173.734MiB/s,174.374MiB/s,50.336GiB/46.003GiB/46.173GiB
+2022-12-16 16:46:50.731,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:51.109,completed,0,T4,0.000 B,2.128 GiB,-1,8,1.329GiB,2/2,8/8,38.579MiB/s,38.654MiB/s,1.518GiB/1.518GiB/1.521GiB
+2022-12-16 16:46:51.109,completed,1,T4,2.128 GiB,8.513 GiB,2,6,4.076GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:51.109,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,8/8,32/32,173.918MiB/s,174.349MiB/s,50.336GiB/46.116GiB/46.231GiB
+2022-12-16 16:46:51.109,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:51.123,completed,0,T4,0.000 B,2.128 GiB,-1,4,680.756MiB,1/1,4/4,19.285MiB/s,19.323MiB/s,777.216MiB/777.216MiB/778.745MiB
+2022-12-16 16:46:51.123,completed,1,T4,2.128 GiB,8.513 GiB,2,7,4.742GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:46:51.123,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,8/8,32/32,173.908MiB/s,174.348MiB/s,50.336GiB/46.116GiB/46.233GiB
+2022-12-16 16:46:51.124,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:07.877,completed,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:07.877,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:07.877,completed,2,T4,8.513 GiB,34.052 GiB,1,40,55.094GiB,8/8,32/32,173.966MiB/s,174.703MiB/s,50.336GiB/48.978GiB/49.185GiB
+2022-12-16 16:47:07.877,completed,3,T4,34.052 GiB,136.206 GiB,1,32,44.241GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:08.927,completed,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:08.927,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:08.927,completed,2,T4,8.513 GiB,34.052 GiB,1,36,49.584GiB,7/7,28/28,151.964MiB/s,152.608MiB/s,44.044GiB/42.857GiB/43.038GiB
+2022-12-16 16:47:08.927,completed,3,T4,34.052 GiB,136.206 GiB,2,36,49.774GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:13.008,completed,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:13.009,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:13.009,completed,2,T4,8.513 GiB,34.052 GiB,1,32,44.074GiB,6/6,24/24,130.231MiB/s,130.829MiB/s,37.750GiB/37.153GiB/37.323GiB
+2022-12-16 16:47:13.009,completed,3,T4,34.052 GiB,136.206 GiB,2,40,55.307GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:14.036,completed,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:14.036,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:14.036,completed,2,T4,8.513 GiB,34.052 GiB,1,28,38.566GiB,5/5,20/20,108.521MiB/s,108.950MiB/s,31.460GiB/31.022GiB/31.145GiB
+2022-12-16 16:47:14.036,completed,3,T4,34.052 GiB,136.206 GiB,2,44,60.837GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:14.270,completed,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:14.270,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:14.270,completed,2,T4,8.513 GiB,34.052 GiB,1,24,33.057GiB,4/4,16/16,86.798MiB/s,86.982MiB/s,25.168GiB/24.806GiB/24.858GiB
+2022-12-16 16:47:14.270,completed,3,T4,34.052 GiB,136.206 GiB,2,48,66.369GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:16.761,completed,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:16.762,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:16.762,completed,2,T4,8.513 GiB,34.052 GiB,1,20,27.548GiB,3/3,12/12,65.026MiB/s,65.252MiB/s,18.875GiB/18.680GiB/18.745GiB
+2022-12-16 16:47:16.762,completed,3,T4,34.052 GiB,136.206 GiB,2,52,71.902GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:20.343,completed,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:20.343,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:20.343,completed,2,T4,8.513 GiB,34.052 GiB,1,16,22.038GiB,2/2,8/8,43.459MiB/s,43.540MiB/s,12.583GiB/12.583GiB/12.606GiB
+2022-12-16 16:47:20.343,completed,3,T4,34.052 GiB,136.206 GiB,2,56,77.435GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:20.375,completed,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:20.375,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:47:20.375,completed,2,T4,8.513 GiB,34.052 GiB,1,12,16.529GiB,1/1,4/4,21.726MiB/s,21.767MiB/s,6.291GiB/6.291GiB/6.303GiB
+2022-12-16 16:47:20.375,completed,3,T4,34.052 GiB,136.206 GiB,2,60,82.967GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.710,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,2/1,8/4,0B/s,0B/s,1.567GiB/0B/0B
+2022-12-16 16:55:23.710,submitted,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.710,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.710,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.712,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,2/2,8/8,0B/s,0B/s,1.567GiB/0B/0B
+2022-12-16 16:55:23.712,submitted,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.712,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.712,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.714,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,3/3,12/12,0B/s,0B/s,2.350GiB/0B/0B
+2022-12-16 16:55:23.714,submitted,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.714,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.714,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.716,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,4/4,16/16,0B/s,0B/s,3.133GiB/0B/0B
+2022-12-16 16:55:23.716,submitted,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.716,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:23.716,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:54.428,completed,0,T4,0.000 B,2.128 GiB,-1,16,2.744GiB,4/4,16/16,104.454MiB/s,104.858MiB/s,3.133GiB/3.133GiB/3.145GiB
+2022-12-16 16:55:54.428,completed,1,T4,2.128 GiB,8.513 GiB,2,8,5.408GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:54.428,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:54.428,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.158,completed,0,T4,0.000 B,2.128 GiB,-1,12,2.058GiB,3/3,12/12,76.518MiB/s,76.814MiB/s,2.350GiB/2.350GiB/2.359GiB
+2022-12-16 16:55:55.158,completed,1,T4,2.128 GiB,8.513 GiB,3,9,6.096GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.158,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.158,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.248,completed,0,T4,0.000 B,2.128 GiB,-1,8,1.372GiB,2/2,8/8,50.857MiB/s,51.054MiB/s,1.566GiB/1.566GiB/1.572GiB
+2022-12-16 16:55:55.248,completed,1,T4,2.128 GiB,8.513 GiB,3,10,6.785GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.248,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.248,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.265,completed,0,T4,0.000 B,2.128 GiB,-1,4,702.844MiB,1/1,4/4,25.428MiB/s,25.526MiB/s,802.298MiB/802.298MiB/805.396MiB
+2022-12-16 16:55:55.265,completed,1,T4,2.128 GiB,8.513 GiB,3,11,7.473GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.265,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 16:55:55.265,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.214,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,2/1,8/4,0B/s,0B/s,1.566GiB/0B/0B
+2022-12-16 17:04:41.214,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,8.161GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.214,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.214,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.216,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,3/2,12/8,0B/s,0B/s,2.350GiB/0B/0B
+2022-12-16 17:04:41.216,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,8.161GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.217,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.217,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.218,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,4/3,16/12,0B/s,0B/s,3.133GiB/0B/0B
+2022-12-16 17:04:41.218,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,8.161GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.219,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.219,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.221,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,4/4,16/16,0B/s,0B/s,3.133GiB/0B/0B
+2022-12-16 17:04:41.221,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,8.161GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.221,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:04:41.221,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.584,completed,0,T4,0.000 B,2.128 GiB,-1,16,2.744GiB,4/4,16/16,73.711MiB/s,74.826MiB/s,3.133GiB/3.050GiB/3.096GiB
+2022-12-16 17:05:23.584,completed,1,T4,2.128 GiB,8.513 GiB,3,12,8.161GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.585,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.585,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.586,submitted,0,T4,0.000 B,2.128 GiB,-1,12,2.058GiB,3/3,12/12,54.776MiB/s,55.822MiB/s,2.349GiB/2.266GiB/2.310GiB
+2022-12-16 17:05:23.586,submitted,1,T4,2.128 GiB,8.513 GiB,4,13,8.849GiB,1/1,4/4,0B/s,0B/s,3.116GiB/0B/0B
+2022-12-16 17:05:23.586,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.586,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.879,completed,0,T4,0.000 B,2.128 GiB,-1,12,2.058GiB,3/3,12/12,54.732MiB/s,55.750MiB/s,2.349GiB/2.280GiB/2.322GiB
+2022-12-16 17:05:23.880,completed,1,T4,2.128 GiB,8.513 GiB,3,13,8.849GiB,1/1,4/4,0B/s,18.710MiB/s,3.116GiB/0B/5.451MiB
+2022-12-16 17:05:23.880,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.880,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.881,submitted,0,T4,0.000 B,2.128 GiB,-1,8,1.372GiB,2/2,8/8,35.936MiB/s,36.885MiB/s,1.566GiB/1.497GiB/1.537GiB
+2022-12-16 17:05:23.881,submitted,1,T4,2.128 GiB,8.513 GiB,4,14,9.536GiB,2/2,8/8,0B/s,18.723MiB/s,6.232GiB/0B/5.493MiB
+2022-12-16 17:05:23.882,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:23.882,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:24.086,completed,0,T4,0.000 B,2.128 GiB,-1,8,1.372GiB,2/2,8/8,36.614MiB/s,36.828MiB/s,1.566GiB/1.533GiB/1.542GiB
+2022-12-16 17:05:24.086,completed,1,T4,2.128 GiB,8.513 GiB,3,14,9.536GiB,2/2,8/8,0B/s,33.667MiB/s,6.232GiB/0B/12.506MiB
+2022-12-16 17:05:24.086,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:24.086,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:24.088,submitted,0,T4,0.000 B,2.128 GiB,-1,4,702.585MiB,1/1,4/4,17.903MiB/s,18.049MiB/s,802.037MiB/767.553MiB/773.785MiB
+2022-12-16 17:05:24.088,submitted,1,T4,2.128 GiB,8.513 GiB,4,15,10.224GiB,3/3,12/12,0B/s,33.583MiB/s,9.348GiB/0B/12.540MiB
+2022-12-16 17:05:24.088,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:24.088,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:25.748,completed,0,T4,0.000 B,2.128 GiB,-1,4,702.585MiB,1/1,4/4,18.010MiB/s,18.077MiB/s,802.037MiB/802.037MiB/805.018MiB
+2022-12-16 17:05:25.748,completed,1,T4,2.128 GiB,8.513 GiB,3,15,10.224GiB,3/3,12/12,40.051MiB/s,58.783MiB/s,9.348GiB/77.709MiB/111.414MiB
+2022-12-16 17:05:25.748,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:25.748,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:25.749,submitted,0,T4,0.000 B,2.128 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:25.749,submitted,1,T4,2.128 GiB,8.513 GiB,4,16,10.912GiB,4/4,16/16,40.023MiB/s,58.777MiB/s,12.463GiB/77.709MiB/111.484MiB
+2022-12-16 17:05:25.749,submitted,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:05:25.749,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:19.665,completed,0,T4,0.000 B,2.128 GiB,3,12,2.058GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:19.665,completed,1,T4,2.128 GiB,8.513 GiB,-1,16,10.912GiB,4/4,16/16,71.857MiB/s,72.197MiB/s,12.463GiB/12.304GiB/12.362GiB
+2022-12-16 17:08:19.665,completed,2,T4,8.513 GiB,34.052 GiB,1,8,11.021GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:19.665,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:21.050,completed,0,T4,0.000 B,2.128 GiB,3,12,2.058GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:21.051,completed,1,T4,2.128 GiB,8.513 GiB,-1,12,8.184GiB,3/3,12/12,53.661MiB/s,53.983MiB/s,9.347GiB/9.257GiB/9.312GiB
+2022-12-16 17:08:21.051,completed,2,T4,8.513 GiB,34.052 GiB,2,10,13.760GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:21.051,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:22.484,completed,0,T4,0.000 B,2.128 GiB,3,12,2.058GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:22.485,completed,1,T4,2.128 GiB,8.513 GiB,-1,8,5.455GiB,2/2,8/8,35.857MiB/s,35.975MiB/s,6.231GiB/6.221GiB/6.242GiB
+2022-12-16 17:08:22.485,completed,2,T4,8.513 GiB,34.052 GiB,2,12,16.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:22.485,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:22.872,completed,0,T4,0.000 B,2.128 GiB,3,12,2.058GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:22.873,completed,1,T4,2.128 GiB,8.513 GiB,-1,4,2.728GiB,1/1,4/4,17.826MiB/s,17.863MiB/s,3.116GiB/3.116GiB/3.122GiB
+2022-12-16 17:08:22.873,completed,2,T4,8.513 GiB,34.052 GiB,2,14,19.236GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:08:22.873,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.278,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,2/1,8/4,0B/s,0B/s,1.566GiB/0B/0B
+2022-12-16 17:09:05.278,submitted,1,T4,2.128 GiB,8.513 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.278,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.279,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.281,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,3/2,12/8,0B/s,0B/s,2.350GiB/0B/0B
+2022-12-16 17:09:05.281,submitted,1,T4,2.128 GiB,8.513 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.281,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.281,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.282,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,4/3,16/12,0B/s,0B/s,3.132GiB/0B/0B
+2022-12-16 17:09:05.282,submitted,1,T4,2.128 GiB,8.513 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.282,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.282,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.286,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,4/4,16/16,0B/s,0B/s,3.132GiB/0B/0B
+2022-12-16 17:09:05.286,submitted,1,T4,2.128 GiB,8.513 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.286,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:05.286,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.150,completed,0,T4,0.000 B,2.128 GiB,-1,16,2.744GiB,4/4,16/16,72.811MiB/s,73.811MiB/s,3.132GiB/3.048GiB/3.090GiB
+2022-12-16 17:09:48.151,completed,1,T4,2.128 GiB,8.513 GiB,-1,0,0B,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.151,completed,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.151,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.917,completed,0,T4,0.000 B,2.128 GiB,-1,12,2.057GiB,3/3,12/12,54.783MiB/s,55.024MiB/s,2.349GiB/2.334GiB/2.344GiB
+2022-12-16 17:09:48.917,completed,1,T4,2.128 GiB,8.513 GiB,1,1,703.290MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.918,completed,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.918,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.966,completed,0,T4,0.000 B,2.128 GiB,-1,8,1.371GiB,2/2,8/8,36.369MiB/s,36.631MiB/s,1.566GiB/1.551GiB/1.562GiB
+2022-12-16 17:09:48.966,completed,1,T4,2.128 GiB,8.513 GiB,1,2,1.373GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.966,completed,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:48.966,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:49.301,completed,0,T4,0.000 B,2.128 GiB,-1,4,702.038MiB,1/1,4/4,18.207MiB/s,18.210MiB/s,801.491MiB/801.491MiB/801.606MiB
+2022-12-16 17:09:49.301,completed,1,T4,2.128 GiB,8.513 GiB,1,3,2.059GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:49.301,completed,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:09:49.301,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.945,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,2/1,8/4,0B/s,0B/s,1.567GiB/0B/0B
+2022-12-16 17:15:36.946,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.946,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.946,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.951,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,3/2,12/8,0B/s,0B/s,2.350GiB/0B/0B
+2022-12-16 17:15:36.951,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.951,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.951,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.954,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,4/3,16/12,0B/s,0B/s,3.132GiB/0B/0B
+2022-12-16 17:15:36.954,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.955,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.955,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.956,submitted,0,T4,0.000 B,2.128 GiB,4,16,2.744GiB,4/4,16/16,0B/s,0B/s,3.132GiB/0B/0B
+2022-12-16 17:15:36.956,submitted,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.956,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:15:36.956,submitted,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:20.251,completed,0,T4,0.000 B,2.128 GiB,-1,16,2.744GiB,4/4,16/16,72.450MiB/s,72.831MiB/s,3.132GiB/3.063GiB/3.079GiB
+2022-12-16 17:16:20.251,completed,1,T4,2.128 GiB,8.513 GiB,1,4,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:20.251,completed,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:20.251,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:20.898,completed,0,T4,0.000 B,2.128 GiB,-1,12,2.058GiB,3/3,12/12,53.450MiB/s,54.263MiB/s,2.349GiB/2.294GiB/2.328GiB
+2022-12-16 17:16:20.898,completed,1,T4,2.128 GiB,8.513 GiB,2,5,3.432GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:20.898,completed,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:20.898,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:21.330,completed,0,T4,0.000 B,2.128 GiB,-1,8,1.372GiB,2/2,8/8,35.778MiB/s,35.940MiB/s,1.567GiB/1.550GiB/1.557GiB
+2022-12-16 17:16:21.330,completed,1,T4,2.128 GiB,8.513 GiB,2,6,4.119GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:21.330,completed,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:21.330,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:22.087,completed,0,T4,0.000 B,2.128 GiB,-1,4,702.090MiB,1/1,4/4,17.760MiB/s,17.794MiB/s,801.495MiB/801.495MiB/803.051MiB
+2022-12-16 17:16:22.087,completed,1,T4,2.128 GiB,8.513 GiB,2,7,4.806GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:22.087,completed,2,T4,8.513 GiB,34.052 GiB,2,16,21.974GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-16 17:16:22.087,completed,3,T4,34.052 GiB,136.206 GiB,2,64,88.498GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
diff --git a/test/resources/org/apache/cassandra/tools/compaction_logs/compaction-UnifiedCompactionStrategy-density-blobs-2-unrepaired.csv b/test/resources/org/apache/cassandra/tools/compaction_logs/compaction-UnifiedCompactionStrategy-density-blobs-2-unrepaired.csv
new file mode 100644
index 000000000000..95ed11f96220
--- /dev/null
+++ b/test/resources/org/apache/cassandra/tools/compaction_logs/compaction-UnifiedCompactionStrategy-density-blobs-2-unrepaired.csv
@@ -0,0 +1,2123 @@
+Timestamp,Event,Level,W,Min Density,Max Density,Overlap,Tot. SSTables,Tot. size (bytes),Compactions,Comp. SSTables,Read (bytes/sec),Write (bytes/sec),Tot. comp. size/Read/Written (bytes)
+2022-12-08 14:37:00.610,submitted,0,T4,0.000 B,2.128 GiB,4,20,2.743GiB,2/1,8/4,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 14:37:00.675,submitted,0,T4,0.000 B,2.128 GiB,4,20,2.743GiB,2/2,8/8,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 14:37:00.694,submitted,0,T4,0.000 B,2.128 GiB,4,20,2.743GiB,4/3,16/12,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 14:37:00.710,submitted,0,T4,0.000 B,2.128 GiB,4,20,2.743GiB,4/4,16/16,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 14:37:00.724,submitted,0,T4,0.000 B,2.128 GiB,4,20,2.743GiB,5/5,20/20,0B/s,0B/s,3.133GiB/0B/0B
+2022-12-08 14:39:25.793,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,8/6,32/24,18.266MiB/s,18.303MiB/s,5.012GiB/2.588GiB/2.593GiB
+2022-12-08 14:39:25.825,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,9/7,36/28,18.262MiB/s,18.301MiB/s,5.639GiB/2.588GiB/2.594GiB
+2022-12-08 14:39:25.838,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,9/8,36/32,18.261MiB/s,18.300MiB/s,5.638GiB/2.588GiB/2.594GiB
+2022-12-08 14:39:25.859,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/9,40/36,18.258MiB/s,18.299MiB/s,6.266GiB/2.588GiB/2.594GiB
+2022-12-08 14:39:25.883,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/10,40/40,18.255MiB/s,18.297MiB/s,6.266GiB/2.588GiB/2.594GiB
+2022-12-08 14:39:56.449,completed,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/10,40/40,35.351MiB/s,35.926MiB/s,6.266GiB/3.618GiB/3.640GiB
+2022-12-08 14:39:57.359,completed,0,T4,0.000 B,2.128 GiB,-1,36,4.938GiB,9/9,36/36,31.713MiB/s,32.086MiB/s,5.639GiB/3.017GiB/3.032GiB
+2022-12-08 14:39:57.359,completed,1,T4,2.128 GiB,8.513 GiB,1,2,562.226MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:39:59.185,completed,0,T4,0.000 B,2.128 GiB,-1,32,4.389GiB,8/8,32/32,27.783MiB/s,28.048MiB/s,5.012GiB/2.425GiB/2.434GiB
+2022-12-08 14:39:59.186,completed,1,T4,2.128 GiB,8.513 GiB,1,4,1.098GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:39:59.331,completed,0,T4,0.000 B,2.128 GiB,-1,28,3.841GiB,7/7,28/28,24.109MiB/s,24.413MiB/s,4.386GiB/1.798GiB/1.809GiB
+2022-12-08 14:39:59.331,completed,1,T4,2.128 GiB,8.513 GiB,1,6,1.647GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:40:04.335,completed,0,T4,0.000 B,2.128 GiB,-1,24,3.292GiB,6/6,24/24,19.758MiB/s,19.926MiB/s,3.759GiB/1.238GiB/1.244GiB
+2022-12-08 14:40:04.340,completed,1,T4,2.128 GiB,8.513 GiB,1,8,2.196GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:02.661,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,28/24,16.782MiB/s,16.856MiB/s,4.386GiB/2.570GiB/2.581GiB
+2022-12-08 14:42:02.668,submitted,1,T4,2.128 GiB,8.513 GiB,1,10,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:02.683,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,28/28,16.779MiB/s,16.856MiB/s,4.386GiB/2.570GiB/2.581GiB
+2022-12-08 14:42:02.687,submitted,1,T4,2.128 GiB,8.513 GiB,1,10,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:02.699,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,16.778MiB/s,16.855MiB/s,5.012GiB/2.570GiB/2.582GiB
+2022-12-08 14:42:02.699,submitted,1,T4,2.128 GiB,8.513 GiB,1,10,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:02.739,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,16.773MiB/s,16.855MiB/s,6.265GiB/2.570GiB/2.582GiB
+2022-12-08 14:42:02.742,submitted,1,T4,2.128 GiB,8.513 GiB,1,10,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:02.755,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,16.772MiB/s,16.854MiB/s,6.265GiB/2.570GiB/2.582GiB
+2022-12-08 14:42:02.755,submitted,1,T4,2.128 GiB,8.513 GiB,1,10,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:38.225,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,32.541MiB/s,32.667MiB/s,6.265GiB/3.669GiB/3.677GiB
+2022-12-08 14:42:38.232,completed,1,T4,2.128 GiB,8.513 GiB,1,10,2.745GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:38.633,completed,0,T4,0.000 B,2.128 GiB,-1,36,4.938GiB,9/9,36/36,28.997MiB/s,29.274MiB/s,5.638GiB/3.042GiB/3.058GiB
+2022-12-08 14:42:38.634,completed,1,T4,2.128 GiB,8.513 GiB,2,12,3.294GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:39.615,completed,0,T4,0.000 B,2.128 GiB,-1,32,4.389GiB,8/8,32/32,25.647MiB/s,25.932MiB/s,5.012GiB/2.439GiB/2.455GiB
+2022-12-08 14:42:39.622,completed,1,T4,2.128 GiB,8.513 GiB,2,14,3.843GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:40.691,completed,0,T4,0.000 B,2.128 GiB,-1,28,3.840GiB,7/7,28/28,22.305MiB/s,22.560MiB/s,4.385GiB/1.836GiB/1.846GiB
+2022-12-08 14:42:40.691,completed,1,T4,2.128 GiB,8.513 GiB,2,16,4.392GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:42:41.099,completed,0,T4,0.000 B,2.128 GiB,-1,24,3.292GiB,6/6,24/24,19.122MiB/s,19.254MiB/s,3.759GiB/1.220GiB/1.226GiB
+2022-12-08 14:42:41.101,completed,1,T4,2.128 GiB,8.513 GiB,2,18,4.941GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:44:29.643,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,28/24,16.915MiB/s,16.966MiB/s,4.385GiB/2.427GiB/2.434GiB
+2022-12-08 14:44:29.643,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.491GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:44:29.659,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/7,36/28,16.912MiB/s,16.967MiB/s,5.638GiB/2.427GiB/2.435GiB
+2022-12-08 14:44:29.660,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.491GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:44:29.675,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,16.910MiB/s,16.968MiB/s,5.638GiB/2.427GiB/2.435GiB
+2022-12-08 14:44:29.675,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.491GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:44:29.685,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,16.909MiB/s,16.968MiB/s,6.264GiB/2.427GiB/2.435GiB
+2022-12-08 14:44:29.697,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.491GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:44:29.710,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,16.906MiB/s,16.968MiB/s,6.264GiB/2.427GiB/2.436GiB
+2022-12-08 14:44:29.711,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.491GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:45:14.243,completed,0,T4,0.000 B,2.128 GiB,1,45,6.171GiB,10/10,40/40,32.848MiB/s,33.007MiB/s,6.264GiB/3.808GiB/3.820GiB
+2022-12-08 14:45:14.252,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.491GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:45:14.424,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,29.430MiB/s,29.651MiB/s,5.638GiB/3.184GiB/3.198GiB
+2022-12-08 14:45:14.432,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.039GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:45:14.843,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,26.080MiB/s,26.294MiB/s,5.012GiB/2.568GiB/2.581GiB
+2022-12-08 14:45:14.844,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.588GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:45:17.596,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,23.061MiB/s,23.172MiB/s,4.385GiB/2.019GiB/2.027GiB
+2022-12-08 14:45:17.599,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.137GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:45:19.007,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,19.600MiB/s,19.924MiB/s,3.759GiB/1.413GiB/1.429GiB
+2022-12-08 14:45:19.007,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.686GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:46:54.335,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,6/6,24/24,17.248MiB/s,17.303MiB/s,3.759GiB/2.436GiB/2.444GiB
+2022-12-08 14:46:54.346,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:46:54.369,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/7,32/28,17.244MiB/s,17.301MiB/s,5.012GiB/2.436GiB/2.444GiB
+2022-12-08 14:46:54.371,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:46:54.388,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,17.241MiB/s,17.300MiB/s,5.638GiB/2.436GiB/2.445GiB
+2022-12-08 14:46:54.398,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:46:54.411,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,17.239MiB/s,17.299MiB/s,6.264GiB/2.436GiB/2.445GiB
+2022-12-08 14:46:54.415,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:46:54.428,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,17.237MiB/s,17.299MiB/s,6.264GiB/2.436GiB/2.445GiB
+2022-12-08 14:46:54.428,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:47:31.265,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,34.860MiB/s,35.205MiB/s,6.264GiB/3.692GiB/3.716GiB
+2022-12-08 14:47:31.265,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:47:31.268,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,31.324MiB/s,31.673MiB/s,5.638GiB/3.066GiB/3.089GiB
+2022-12-08 14:47:31.269,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.784GiB,2/1,8/4,0B/s,0B/s,2.507GiB/0B/0B
+2022-12-08 14:47:31.278,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,31.318MiB/s,31.684MiB/s,5.638GiB/3.066GiB/3.090GiB
+2022-12-08 14:47:31.279,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.784GiB,2/2,8/8,0B/s,0B/s,2.507GiB/0B/0B
+2022-12-08 14:47:36.168,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,31.318MiB/s,31.453MiB/s,5.638GiB/3.217GiB/3.225GiB
+2022-12-08 14:47:36.176,completed,1,T4,2.128 GiB,8.513 GiB,3,32,8.784GiB,2/2,8/8,4.890MiB/s,6.168MiB/s,2.507GiB/23.863MiB/30.095MiB
+2022-12-08 14:47:36.178,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,27.872MiB/s,28.008MiB/s,5.011GiB/2.590GiB/2.598GiB
+2022-12-08 14:47:36.178,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.334GiB,4/3,16/12,4.881MiB/s,6.164MiB/s,5.014GiB/23.863MiB/30.137MiB
+2022-12-08 14:47:36.189,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,27.867MiB/s,28.008MiB/s,5.011GiB/2.590GiB/2.599GiB
+2022-12-08 14:47:36.190,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.334GiB,4/4,16/16,4.869MiB/s,6.173MiB/s,5.014GiB/23.863MiB/30.251MiB
+2022-12-08 14:47:36.462,completed,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,27.833MiB/s,27.959MiB/s,5.011GiB/2.597GiB/2.603GiB
+2022-12-08 14:47:36.462,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.334GiB,4/4,16/16,6.018MiB/s,11.460MiB/s,5.014GiB/31.134MiB/33.547MiB
+2022-12-08 14:47:36.472,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,24.397MiB/s,24.526MiB/s,4.385GiB/1.971GiB/1.977GiB
+2022-12-08 14:47:36.484,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.882GiB,5/5,20/20,6.007MiB/s,11.665MiB/s,6.267GiB/31.134MiB/33.730MiB
+2022-12-08 14:47:36.490,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,24.388MiB/s,24.521MiB/s,4.385GiB/1.971GiB/1.977GiB
+2022-12-08 14:47:36.492,completed,1,T4,2.128 GiB,8.513 GiB,4,36,9.882GiB,5/5,20/20,5.986MiB/s,11.458MiB/s,6.267GiB/31.134MiB/33.839MiB
+2022-12-08 14:47:36.498,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,6/6,24/24,20.952MiB/s,21.087MiB/s,3.758GiB/1.344GiB/1.350GiB
+2022-12-08 14:47:36.507,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.431GiB,6/6,24/24,5.977MiB/s,11.505MiB/s,7.520GiB/31.134MiB/33.973MiB
+2022-12-08 14:47:36.525,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,20.940MiB/s,21.088MiB/s,3.758GiB/1.344GiB/1.351GiB
+2022-12-08 14:47:36.525,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.431GiB,8/7,32/28,5.946MiB/s,11.825MiB/s,10.027GiB/31.134MiB/34.417MiB
+2022-12-08 14:47:36.536,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,20.935MiB/s,21.086MiB/s,3.758GiB/1.344GiB/1.351GiB
+2022-12-08 14:47:36.536,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.431GiB,8/8,32/32,5.934MiB/s,11.826MiB/s,10.027GiB/31.134MiB/34.538MiB
+2022-12-08 14:47:37.733,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,20.802MiB/s,21.070MiB/s,3.758GiB/1.362GiB/1.374GiB
+2022-12-08 14:47:37.743,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.431GiB,8/8,32/32,19.947MiB/s,24.055MiB/s,10.027GiB/57.369MiB/64.976MiB
+2022-12-08 14:47:37.759,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,17.380MiB/s,17.657MiB/s,3.132GiB/753.448MiB/765.439MiB
+2022-12-08 14:47:37.762,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.980GiB,10/9,40/36,19.653MiB/s,24.099MiB/s,12.534GiB/57.369MiB/65.631MiB
+2022-12-08 14:47:37.791,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,17.368MiB/s,17.655MiB/s,3.132GiB/753.448MiB/765.933MiB
+2022-12-08 14:47:37.792,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.980GiB,10/10,40/40,19.304MiB/s,24.055MiB/s,12.534GiB/57.369MiB/66.355MiB
+2022-12-08 14:49:25.770,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/6,32/24,15.854MiB/s,15.920MiB/s,5.011GiB/2.343GiB/2.353GiB
+2022-12-08 14:49:25.771,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.980GiB,10/10,40/40,30.681MiB/s,30.980MiB/s,12.534GiB/3.299GiB/3.331GiB
+2022-12-08 14:49:25.791,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,28/28,15.871MiB/s,15.920MiB/s,4.384GiB/2.346GiB/2.353GiB
+2022-12-08 14:49:25.791,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.980GiB,10/10,40/40,30.771MiB/s,30.984MiB/s,12.534GiB/3.309GiB/3.332GiB
+2022-12-08 14:49:25.796,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,15.870MiB/s,15.920MiB/s,5.011GiB/2.346GiB/2.354GiB
+2022-12-08 14:49:25.797,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.980GiB,10/10,40/40,30.770MiB/s,30.989MiB/s,12.534GiB/3.309GiB/3.333GiB
+2022-12-08 14:49:25.803,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,15.870MiB/s,15.920MiB/s,6.264GiB/2.346GiB/2.354GiB
+2022-12-08 14:49:25.803,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.980GiB,10/10,40/40,30.768MiB/s,30.993MiB/s,12.534GiB/3.309GiB/3.333GiB
+2022-12-08 14:49:25.809,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.869MiB/s,15.921MiB/s,6.264GiB/2.346GiB/2.354GiB
+2022-12-08 14:49:25.809,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.980GiB,10/10,40/40,30.766MiB/s,30.998MiB/s,12.534GiB/3.309GiB/3.334GiB
+2022-12-08 14:50:15.119,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,31.746MiB/s,31.908MiB/s,6.264GiB/3.860GiB/3.871GiB
+2022-12-08 14:50:15.120,completed,1,T4,2.128 GiB,8.513 GiB,4,40,10.980GiB,10/10,40/40,30.761MiB/s,30.922MiB/s,12.534GiB/4.790GiB/4.815GiB
+2022-12-08 14:50:18.171,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,28.473MiB/s,28.628MiB/s,5.638GiB/3.311GiB/3.321GiB
+2022-12-08 14:50:18.172,completed,1,T4,2.128 GiB,8.513 GiB,1,42,11.529GiB,10/10,40/40,30.734MiB/s,30.887MiB/s,12.534GiB/4.878GiB/4.902GiB
+2022-12-08 14:50:18.212,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,25.313MiB/s,25.475MiB/s,5.012GiB/2.685GiB/2.696GiB
+2022-12-08 14:50:18.214,completed,1,T4,2.128 GiB,8.513 GiB,1,44,12.078GiB,10/10,40/40,30.727MiB/s,30.884MiB/s,12.534GiB/4.878GiB/4.902GiB
+2022-12-08 14:50:19.606,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,22.068MiB/s,22.243MiB/s,4.385GiB/2.082GiB/2.093GiB
+2022-12-08 14:50:19.606,completed,1,T4,2.128 GiB,8.513 GiB,1,46,12.627GiB,10/10,40/40,30.660MiB/s,30.866MiB/s,12.534GiB/4.909GiB/4.942GiB
+2022-12-08 14:50:20.512,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,18.982MiB/s,19.118MiB/s,3.759GiB/1.475GiB/1.482GiB
+2022-12-08 14:50:20.512,completed,1,T4,2.128 GiB,8.513 GiB,1,48,13.175GiB,10/10,40/40,30.678MiB/s,30.866MiB/s,12.534GiB/4.939GiB/4.969GiB
+2022-12-08 14:52:09.482,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,6/6,24/24,15.641MiB/s,15.675MiB/s,3.759GiB/2.500GiB/2.505GiB
+2022-12-08 14:52:09.483,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.725GiB,10/10,40/40,30.702MiB/s,30.823MiB/s,12.534GiB/8.210GiB/8.242GiB
+2022-12-08 14:52:09.497,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/7,36/28,15.640MiB/s,15.675MiB/s,5.638GiB/2.500GiB/2.506GiB
+2022-12-08 14:52:09.502,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.725GiB,10/10,40/40,30.701MiB/s,30.823MiB/s,12.534GiB/8.210GiB/8.243GiB
+2022-12-08 14:52:09.512,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,15.638MiB/s,15.674MiB/s,5.638GiB/2.500GiB/2.506GiB
+2022-12-08 14:52:09.512,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.725GiB,10/10,40/40,30.699MiB/s,30.822MiB/s,12.534GiB/8.210GiB/8.243GiB
+2022-12-08 14:52:09.537,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/9,36/36,15.636MiB/s,15.675MiB/s,5.638GiB/2.500GiB/2.506GiB
+2022-12-08 14:52:09.546,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.725GiB,10/10,40/40,30.696MiB/s,30.823MiB/s,12.534GiB/8.210GiB/8.244GiB
+2022-12-08 14:52:09.553,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.634MiB/s,15.675MiB/s,6.264GiB/2.500GiB/2.507GiB
+2022-12-08 14:52:09.554,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.725GiB,10/10,40/40,30.694MiB/s,30.822MiB/s,12.534GiB/8.210GiB/8.244GiB
+2022-12-08 14:52:48.471,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,31.860MiB/s,32.117MiB/s,6.264GiB/3.720GiB/3.733GiB
+2022-12-08 14:52:48.483,completed,1,T4,2.128 GiB,8.513 GiB,1,50,13.725GiB,10/10,40/40,30.705MiB/s,30.855MiB/s,12.534GiB/9.380GiB/9.426GiB
+2022-12-08 14:52:48.582,completed,0,T4,0.000 B,2.128 GiB,-1,36,4.938GiB,9/9,36/36,28.816MiB/s,28.941MiB/s,5.638GiB/3.101GiB/3.110GiB
+2022-12-08 14:52:48.582,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.273GiB,10/10,40/40,30.694MiB/s,30.855MiB/s,12.534GiB/9.380GiB/9.429GiB
+2022-12-08 14:52:49.965,completed,0,T4,0.000 B,2.128 GiB,-1,32,4.389GiB,8/8,32/32,25.517MiB/s,25.755MiB/s,5.012GiB/2.504GiB/2.515GiB
+2022-12-08 14:52:49.965,completed,1,T4,2.128 GiB,8.513 GiB,2,54,14.822GiB,10/10,40/40,30.732MiB/s,30.847MiB/s,12.534GiB/9.433GiB/9.468GiB
+2022-12-08 14:52:51.566,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,22.647MiB/s,22.786MiB/s,4.385GiB/1.924GiB/1.932GiB
+2022-12-08 14:52:51.568,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.371GiB,10/10,40/40,30.729MiB/s,30.906MiB/s,12.534GiB/9.480GiB/9.534GiB
+2022-12-08 14:52:52.703,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,19.578MiB/s,19.715MiB/s,3.759GiB/1.322GiB/1.328GiB
+2022-12-08 14:52:52.704,completed,1,T4,2.128 GiB,8.513 GiB,2,58,15.920GiB,10/10,40/40,30.733MiB/s,30.909MiB/s,12.534GiB/9.515GiB/9.570GiB
+2022-12-08 14:54:29.254,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.336MiB/s,15.426MiB/s,3.132GiB/2.093GiB/2.105GiB
+2022-12-08 14:54:29.260,completed,1,T4,2.128 GiB,8.513 GiB,2,60,16.470GiB,10/10,40/40,30.577MiB/s,30.722MiB/s,12.534GiB/12.350GiB/12.409GiB
+2022-12-08 14:54:30.592,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.362MiB/s,15.440MiB/s,3.132GiB/2.116GiB/2.127GiB
+2022-12-08 14:54:30.595,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.371GiB,9/9,36/36,27.528MiB/s,27.651MiB/s,11.280GiB/11.141GiB/11.191GiB
+2022-12-08 14:54:30.595,completed,2,T4,8.513 GiB,34.052 GiB,1,1,1.101GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:32.715,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.358MiB/s,15.431MiB/s,3.132GiB/2.148GiB/2.158GiB
+2022-12-08 14:54:32.715,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.273GiB,8/8,32/32,24.483MiB/s,24.574MiB/s,10.027GiB/9.945GiB/9.982GiB
+2022-12-08 14:54:32.716,completed,2,T4,8.513 GiB,34.052 GiB,1,2,2.202GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:34.188,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.388MiB/s,15.458MiB/s,3.132GiB/2.174GiB/2.184GiB
+2022-12-08 14:54:34.191,completed,1,T4,2.128 GiB,8.513 GiB,2,48,13.174GiB,7/7,28/28,21.391MiB/s,21.482MiB/s,8.773GiB/8.719GiB/8.756GiB
+2022-12-08 14:54:34.191,completed,2,T4,8.513 GiB,34.052 GiB,1,3,3.303GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:34.533,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.405MiB/s,15.455MiB/s,3.132GiB/2.181GiB/2.188GiB
+2022-12-08 14:54:34.538,completed,1,T4,2.128 GiB,8.513 GiB,2,44,12.077GiB,6/6,24/24,18.329MiB/s,18.400MiB/s,7.520GiB/7.475GiB/7.505GiB
+2022-12-08 14:54:34.539,completed,2,T4,8.513 GiB,34.052 GiB,1,4,4.403GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:35.082,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.347MiB/s,15.457MiB/s,3.132GiB/2.181GiB/2.197GiB
+2022-12-08 14:54:35.082,completed,1,T4,2.128 GiB,8.513 GiB,2,40,10.979GiB,5/5,20/20,15.256MiB/s,15.318MiB/s,6.266GiB/6.229GiB/6.255GiB
+2022-12-08 14:54:35.082,completed,2,T4,8.513 GiB,34.052 GiB,1,5,5.504GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:36.115,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.451MiB/s,15.496MiB/s,3.132GiB/2.212GiB/2.218GiB
+2022-12-08 14:54:36.115,completed,1,T4,2.128 GiB,8.513 GiB,2,36,9.881GiB,4/4,16/16,12.200MiB/s,12.252MiB/s,5.013GiB/4.992GiB/5.013GiB
+2022-12-08 14:54:36.115,completed,2,T4,8.513 GiB,34.052 GiB,1,6,6.604GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:37.202,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.468MiB/s,15.537MiB/s,3.132GiB/2.231GiB/2.241GiB
+2022-12-08 14:54:37.212,completed,1,T4,2.128 GiB,8.513 GiB,2,32,8.783GiB,3/3,12/12,9.156MiB/s,9.186MiB/s,3.760GiB/3.754GiB/3.766GiB
+2022-12-08 14:54:37.212,completed,2,T4,8.513 GiB,34.052 GiB,1,7,7.704GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:37.439,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.499MiB/s,15.541MiB/s,3.132GiB/2.239GiB/2.245GiB
+2022-12-08 14:54:37.439,completed,1,T4,2.128 GiB,8.513 GiB,2,28,7.685GiB,2/2,8/8,6.092MiB/s,6.115MiB/s,2.506GiB/2.500GiB/2.510GiB
+2022-12-08 14:54:37.439,completed,2,T4,8.513 GiB,34.052 GiB,1,8,8.805GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:39.147,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.482MiB/s,15.520MiB/s,3.132GiB/2.262GiB/2.268GiB
+2022-12-08 14:54:39.147,completed,1,T4,2.128 GiB,8.513 GiB,2,24,6.587GiB,1/1,4/4,3.046MiB/s,3.055MiB/s,1.253GiB/1.253GiB/1.257GiB
+2022-12-08 14:54:39.147,completed,2,T4,8.513 GiB,34.052 GiB,1,9,9.905GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.141,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,28/24,15.784MiB/s,15.875MiB/s,4.385GiB/2.445GiB/2.459GiB
+2022-12-08 14:54:48.141,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.141,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.165,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/7,32/28,15.803MiB/s,15.874MiB/s,5.011GiB/2.448GiB/2.459GiB
+2022-12-08 14:54:48.165,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.165,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.189,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,15.821MiB/s,15.873MiB/s,5.638GiB/2.451GiB/2.459GiB
+2022-12-08 14:54:48.189,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.189,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.210,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,15.820MiB/s,15.872MiB/s,6.264GiB/2.451GiB/2.459GiB
+2022-12-08 14:54:48.211,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.211,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.227,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.842MiB/s,15.873MiB/s,6.264GiB/2.455GiB/2.460GiB
+2022-12-08 14:54:48.231,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:54:48.232,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:23.161,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,34.782MiB/s,35.019MiB/s,6.264GiB/3.727GiB/3.739GiB
+2022-12-08 14:55:23.171,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:23.171,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:24.204,completed,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,31.340MiB/s,31.503MiB/s,5.638GiB/3.125GiB/3.135GiB
+2022-12-08 14:55:24.204,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.038GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:24.204,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:26.053,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,27.784MiB/s,28.069MiB/s,5.012GiB/2.541GiB/2.552GiB
+2022-12-08 14:55:26.056,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.587GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:26.064,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:26.195,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,24.602MiB/s,24.782MiB/s,4.385GiB/1.920GiB/1.928GiB
+2022-12-08 14:55:26.195,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.137GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:26.199,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:27.875,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,21.324MiB/s,21.555MiB/s,3.759GiB/1.327GiB/1.337GiB
+2022-12-08 14:55:27.876,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.685GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:55:27.876,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.363,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/6,32/24,17.515MiB/s,17.567MiB/s,5.012GiB/2.483GiB/2.490GiB
+2022-12-08 14:57:13.363,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.364,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.380,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,28/28,17.513MiB/s,17.566MiB/s,4.385GiB/2.483GiB/2.490GiB
+2022-12-08 14:57:13.380,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.380,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.398,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,17.511MiB/s,17.567MiB/s,5.011GiB/2.483GiB/2.491GiB
+2022-12-08 14:57:13.402,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.402,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.418,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/9,36/36,17.508MiB/s,17.567MiB/s,5.638GiB/2.483GiB/2.491GiB
+2022-12-08 14:57:13.426,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.427,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.435,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,17.506MiB/s,17.715MiB/s,6.264GiB/2.483GiB/2.491GiB
+2022-12-08 14:57:13.436,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:13.436,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:50.833,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,34.366MiB/s,34.513MiB/s,6.264GiB/3.702GiB/3.712GiB
+2022-12-08 14:57:50.838,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:50.839,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:50.855,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,30.845MiB/s,31.000MiB/s,5.639GiB/3.077GiB/3.086GiB
+2022-12-08 14:57:50.858,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.783GiB,1/1,4/4,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 14:57:50.858,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:50.865,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,30.840MiB/s,31.001MiB/s,5.639GiB/3.077GiB/3.086GiB
+2022-12-08 14:57:50.865,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.783GiB,2/2,8/8,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 14:57:50.866,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:53.656,completed,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,30.579MiB/s,30.872MiB/s,5.639GiB/3.144GiB/3.161GiB
+2022-12-08 14:57:53.657,completed,1,T4,2.128 GiB,8.513 GiB,3,32,8.783GiB,2/2,8/8,3.624MiB/s,5.868MiB/s,2.506GiB/10.042MiB/16.255MiB
+2022-12-08 14:57:53.657,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:53.666,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,27.117MiB/s,27.412MiB/s,5.012GiB/2.518GiB/2.534GiB
+2022-12-08 14:57:53.667,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.331GiB,4/3,16/12,3.611MiB/s,5.848MiB/s,5.012GiB/10.042MiB/16.255MiB
+2022-12-08 14:57:53.667,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:53.682,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,27.218MiB/s,27.408MiB/s,5.012GiB/2.522GiB/2.535GiB
+2022-12-08 14:57:53.682,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.331GiB,4/4,16/16,3.591MiB/s,5.838MiB/s,5.012GiB/10.042MiB/16.320MiB
+2022-12-08 14:57:53.683,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:54.877,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,27.194MiB/s,27.355MiB/s,5.012GiB/2.552GiB/2.562GiB
+2022-12-08 14:57:54.879,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.331GiB,4/4,16/16,8.829MiB/s,11.508MiB/s,5.012GiB/22.657MiB/30.540MiB
+2022-12-08 14:57:54.879,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:54.881,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.527GiB,7/7,28/28,23.754MiB/s,23.915MiB/s,4.386GiB/1.925GiB/1.935GiB
+2022-12-08 14:57:54.881,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.881GiB,5/5,20/20,8.810MiB/s,11.532MiB/s,6.265GiB/22.657MiB/30.632MiB
+2022-12-08 14:57:54.881,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:54.906,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.527GiB,7/7,28/28,23.743MiB/s,23.913MiB/s,4.386GiB/1.925GiB/1.935GiB
+2022-12-08 14:57:54.907,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.881GiB,6/6,24/24,8.693MiB/s,11.509MiB/s,7.519GiB/22.657MiB/30.839MiB
+2022-12-08 14:57:54.908,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:57.441,completed,0,T4,0.000 B,2.128 GiB,1,33,4.527GiB,7/7,28/28,23.442MiB/s,23.656MiB/s,4.386GiB/1.969GiB/1.979GiB
+2022-12-08 14:57:57.447,completed,1,T4,2.128 GiB,8.513 GiB,3,36,9.881GiB,6/6,24/24,14.205MiB/s,17.083MiB/s,7.519GiB/62.999MiB/73.316MiB
+2022-12-08 14:57:57.447,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:57.455,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,20.046MiB/s,20.265MiB/s,3.759GiB/1.343GiB/1.353GiB
+2022-12-08 14:57:57.456,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.430GiB,8/7,32/28,14.153MiB/s,17.079MiB/s,10.024GiB/62.999MiB/73.551MiB
+2022-12-08 14:57:57.456,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:57.473,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,20.039MiB/s,20.260MiB/s,3.759GiB/1.343GiB/1.353GiB
+2022-12-08 14:57:57.474,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.430GiB,8/8,32/32,14.088MiB/s,17.047MiB/s,10.024GiB/62.999MiB/73.712MiB
+2022-12-08 14:57:57.482,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:57.608,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,20.114MiB/s,20.244MiB/s,3.759GiB/1.349GiB/1.354GiB
+2022-12-08 14:57:57.608,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.430GiB,8/8,32/32,13.612MiB/s,17.807MiB/s,10.024GiB/62.999MiB/76.869MiB
+2022-12-08 14:57:57.608,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:57.614,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,16.725MiB/s,16.854MiB/s,3.132GiB/739.138MiB/744.867MiB
+2022-12-08 14:57:57.620,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.979GiB,9/9,36/36,13.597MiB/s,17.792MiB/s,11.278GiB/62.999MiB/76.938MiB
+2022-12-08 14:57:57.621,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:57:57.644,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,16.713MiB/s,16.850MiB/s,3.132GiB/739.138MiB/745.225MiB
+2022-12-08 14:57:57.655,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.979GiB,10/10,40/40,13.490MiB/s,17.942MiB/s,12.532GiB/62.999MiB/77.365MiB
+2022-12-08 14:57:57.655,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:59:52.345,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/6,32/24,16.063MiB/s,16.098MiB/s,5.012GiB/2.493GiB/2.498GiB
+2022-12-08 14:59:52.345,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,31.362MiB/s,31.603MiB/s,12.532GiB/3.596GiB/3.623GiB
+2022-12-08 14:59:52.345,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:59:52.366,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,28/28,16.061MiB/s,16.098MiB/s,4.386GiB/2.493GiB/2.499GiB
+2022-12-08 14:59:52.367,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,31.356MiB/s,31.605MiB/s,12.532GiB/3.596GiB/3.624GiB
+2022-12-08 14:59:52.367,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:59:52.371,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,16.061MiB/s,16.098MiB/s,5.012GiB/2.493GiB/2.499GiB
+2022-12-08 14:59:52.372,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,31.355MiB/s,31.606MiB/s,12.532GiB/3.596GiB/3.624GiB
+2022-12-08 14:59:52.372,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:59:52.393,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/9,36/36,16.059MiB/s,16.104MiB/s,5.638GiB/2.493GiB/2.500GiB
+2022-12-08 14:59:52.393,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,31.349MiB/s,31.615MiB/s,12.532GiB/3.596GiB/3.626GiB
+2022-12-08 14:59:52.393,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 14:59:52.409,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,16.058MiB/s,16.106MiB/s,6.264GiB/2.493GiB/2.501GiB
+2022-12-08 14:59:52.411,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,31.345MiB/s,31.617MiB/s,12.532GiB/3.596GiB/3.627GiB
+2022-12-08 14:59:52.414,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:00:35.112,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,29.393MiB/s,29.533MiB/s,6.264GiB/3.645GiB/3.653GiB
+2022-12-08 15:00:35.119,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,30.151MiB/s,30.325MiB/s,12.532GiB/4.716GiB/4.743GiB
+2022-12-08 15:00:35.120,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:00:40.224,completed,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,26.204MiB/s,26.288MiB/s,5.638GiB/3.140GiB/3.145GiB
+2022-12-08 15:00:40.226,completed,1,T4,2.128 GiB,8.513 GiB,1,42,11.528GiB,10/10,40/40,30.024MiB/s,30.186MiB/s,12.532GiB/4.846GiB/4.872GiB
+2022-12-08 15:00:40.226,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:00:40.888,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,22.978MiB/s,23.227MiB/s,5.012GiB/2.521GiB/2.535GiB
+2022-12-08 15:00:40.888,completed,1,T4,2.128 GiB,8.513 GiB,1,44,12.076GiB,10/10,40/40,29.989MiB/s,30.197MiB/s,12.532GiB/4.860GiB/4.894GiB
+2022-12-08 15:00:40.888,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:00:42.180,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,20.003MiB/s,20.098MiB/s,4.385GiB/1.926GiB/1.931GiB
+2022-12-08 15:00:42.185,completed,1,T4,2.128 GiB,8.513 GiB,1,46,12.625GiB,10/10,40/40,29.993MiB/s,30.166MiB/s,12.532GiB/4.899GiB/4.927GiB
+2022-12-08 15:00:42.186,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:00:43.064,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,16.885MiB/s,17.014MiB/s,3.759GiB/1.311GiB/1.317GiB
+2022-12-08 15:00:43.064,completed,1,T4,2.128 GiB,8.513 GiB,1,48,13.174GiB,10/10,40/40,29.930MiB/s,30.148MiB/s,12.532GiB/4.914GiB/4.950GiB
+2022-12-08 15:00:43.066,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:02:29.848,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/6,36/24,15.423MiB/s,15.477MiB/s,5.638GiB/2.371GiB/2.380GiB
+2022-12-08 15:02:29.848,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,30.621MiB/s,30.792MiB/s,12.532GiB/8.221GiB/8.267GiB
+2022-12-08 15:02:29.850,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:02:29.868,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,28/28,15.421MiB/s,15.476MiB/s,4.385GiB/2.371GiB/2.380GiB
+2022-12-08 15:02:29.871,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,30.618MiB/s,30.791MiB/s,12.532GiB/8.221GiB/8.267GiB
+2022-12-08 15:02:29.872,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:02:29.894,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,15.418MiB/s,15.476MiB/s,5.638GiB/2.371GiB/2.380GiB
+2022-12-08 15:02:29.894,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,30.616MiB/s,30.791MiB/s,12.532GiB/8.221GiB/8.268GiB
+2022-12-08 15:02:29.896,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:02:29.903,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,15.417MiB/s,15.476MiB/s,6.264GiB/2.371GiB/2.380GiB
+2022-12-08 15:02:29.913,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,30.615MiB/s,30.791MiB/s,12.532GiB/8.221GiB/8.268GiB
+2022-12-08 15:02:29.914,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:02:29.927,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.415MiB/s,15.475MiB/s,6.264GiB/2.371GiB/2.381GiB
+2022-12-08 15:02:29.927,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,30.612MiB/s,30.790MiB/s,12.532GiB/8.221GiB/8.268GiB
+2022-12-08 15:02:29.927,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:03:22.653,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,28.950MiB/s,29.059MiB/s,6.264GiB/3.797GiB/3.806GiB
+2022-12-08 15:03:22.653,completed,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,29.995MiB/s,30.140MiB/s,12.532GiB/9.599GiB/9.646GiB
+2022-12-08 15:03:22.654,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:03:26.820,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,25.894MiB/s,26.007MiB/s,5.637GiB/3.273GiB/3.280GiB
+2022-12-08 15:03:26.822,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.273GiB,10/10,40/40,29.984MiB/s,30.121MiB/s,12.532GiB/9.718GiB/9.762GiB
+2022-12-08 15:03:26.822,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:03:27.482,completed,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,22.894MiB/s,22.995MiB/s,5.011GiB/2.659GiB/2.665GiB
+2022-12-08 15:03:27.491,completed,1,T4,2.128 GiB,8.513 GiB,2,54,14.822GiB,10/10,40/40,29.979MiB/s,30.126MiB/s,12.532GiB/9.736GiB/9.783GiB
+2022-12-08 15:03:27.491,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:03:27.644,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.958MiB/s,20.008MiB/s,4.385GiB/2.038GiB/2.041GiB
+2022-12-08 15:03:27.644,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.371GiB,10/10,40/40,29.981MiB/s,30.127MiB/s,12.532GiB/9.741GiB/9.788GiB
+2022-12-08 15:03:27.645,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:03:30.104,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,16.866MiB/s,17.095MiB/s,3.758GiB/1.445GiB/1.458GiB
+2022-12-08 15:03:30.104,completed,1,T4,2.128 GiB,8.513 GiB,2,58,15.919GiB,10/10,40/40,29.956MiB/s,30.130MiB/s,12.532GiB/9.805GiB/9.862GiB
+2022-12-08 15:03:30.104,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:04:51.362,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.278MiB/s,15.351MiB/s,3.132GiB/2.111GiB/2.121GiB
+2022-12-08 15:04:51.363,completed,1,T4,2.128 GiB,8.513 GiB,2,60,16.468GiB,10/10,40/40,30.334MiB/s,30.497MiB/s,12.532GiB/12.336GiB/12.402GiB
+2022-12-08 15:04:51.364,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.006GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:04:53.911,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.279MiB/s,15.315MiB/s,3.132GiB/2.149GiB/2.154GiB
+2022-12-08 15:04:53.912,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.371GiB,9/9,36/36,27.276MiB/s,27.399MiB/s,11.279GiB/11.156GiB/11.207GiB
+2022-12-08 15:04:53.912,completed,2,T4,8.513 GiB,34.052 GiB,2,11,12.107GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:04:54.362,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.231MiB/s,15.299MiB/s,3.132GiB/2.149GiB/2.158GiB
+2022-12-08 15:04:54.363,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.273GiB,8/8,32/32,24.188MiB/s,24.315MiB/s,10.026GiB/9.903GiB/9.955GiB
+2022-12-08 15:04:54.363,completed,2,T4,8.513 GiB,34.052 GiB,2,12,13.208GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:04:56.633,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.207MiB/s,15.292MiB/s,3.132GiB/2.179GiB/2.191GiB
+2022-12-08 15:04:56.634,completed,1,T4,2.128 GiB,8.513 GiB,2,48,13.174GiB,7/7,28/28,21.106MiB/s,21.215MiB/s,8.772GiB/8.696GiB/8.741GiB
+2022-12-08 15:04:56.634,completed,2,T4,8.513 GiB,34.052 GiB,2,13,14.310GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:04:57.551,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.210MiB/s,15.297MiB/s,3.132GiB/2.193GiB/2.206GiB
+2022-12-08 15:04:57.551,completed,1,T4,2.128 GiB,8.513 GiB,2,44,12.076GiB,6/6,24/24,18.097MiB/s,18.199MiB/s,7.519GiB/7.461GiB/7.503GiB
+2022-12-08 15:04:57.552,completed,2,T4,8.513 GiB,34.052 GiB,2,14,15.411GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:04:59.087,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.261MiB/s,15.285MiB/s,3.132GiB/2.223GiB/2.227GiB
+2022-12-08 15:04:59.087,completed,1,T4,2.128 GiB,8.513 GiB,2,40,10.979GiB,5/5,20/20,15.070MiB/s,15.131MiB/s,6.266GiB/6.242GiB/6.267GiB
+2022-12-08 15:04:59.092,completed,2,T4,8.513 GiB,34.052 GiB,2,15,16.512GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:04:59.103,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.260MiB/s,15.285MiB/s,3.132GiB/2.223GiB/2.227GiB
+2022-12-08 15:04:59.103,completed,1,T4,2.128 GiB,8.513 GiB,2,36,9.881GiB,4/4,16/16,12.044MiB/s,12.094MiB/s,5.013GiB/4.989GiB/5.009GiB
+2022-12-08 15:04:59.104,completed,2,T4,8.513 GiB,34.052 GiB,2,16,17.614GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:00.707,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.184MiB/s,15.269MiB/s,3.132GiB/2.236GiB/2.249GiB
+2022-12-08 15:05:00.715,completed,1,T4,2.128 GiB,8.513 GiB,2,32,8.783GiB,3/3,12/12,9.039MiB/s,9.088MiB/s,3.760GiB/3.747GiB/3.767GiB
+2022-12-08 15:05:00.716,completed,2,T4,8.513 GiB,34.052 GiB,2,17,18.715GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:01.663,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.181MiB/s,15.280MiB/s,3.132GiB/2.250GiB/2.265GiB
+2022-12-08 15:05:01.666,completed,1,T4,2.128 GiB,8.513 GiB,2,28,7.685GiB,2/2,8/8,6.010MiB/s,6.040MiB/s,2.506GiB/2.501GiB/2.513GiB
+2022-12-08 15:05:01.667,completed,2,T4,8.513 GiB,34.052 GiB,2,18,19.817GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:02.952,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.242MiB/s,15.257MiB/s,3.132GiB/2.278GiB/2.280GiB
+2022-12-08 15:05:02.955,completed,1,T4,2.128 GiB,8.513 GiB,2,24,6.587GiB,1/1,4/4,2.990MiB/s,3.001MiB/s,1.253GiB/1.253GiB/1.258GiB
+2022-12-08 15:05:02.955,completed,2,T4,8.513 GiB,34.052 GiB,2,19,20.918GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.248,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,28/24,15.338MiB/s,15.388MiB/s,4.385GiB/2.492GiB/2.500GiB
+2022-12-08 15:05:16.248,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.251,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.263,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,28/28,15.359MiB/s,15.388MiB/s,4.385GiB/2.495GiB/2.500GiB
+2022-12-08 15:05:16.263,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.263,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.283,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,15.357MiB/s,15.389MiB/s,5.638GiB/2.495GiB/2.500GiB
+2022-12-08 15:05:16.283,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.283,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.304,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,15.355MiB/s,15.388MiB/s,6.264GiB/2.495GiB/2.501GiB
+2022-12-08 15:05:16.304,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.304,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.307,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.354MiB/s,15.389MiB/s,6.264GiB/2.495GiB/2.501GiB
+2022-12-08 15:05:16.308,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:16.308,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:55.106,completed,0,T4,0.000 B,2.128 GiB,1,45,6.171GiB,10/10,40/40,31.304MiB/s,31.696MiB/s,6.264GiB/3.689GiB/3.709GiB
+2022-12-08 15:05:55.107,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:55.107,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:56.368,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,28.435MiB/s,28.730MiB/s,5.638GiB/3.109GiB/3.125GiB
+2022-12-08 15:05:56.368,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.038GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:56.369,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:56.696,completed,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,25.202MiB/s,25.629MiB/s,5.011GiB/2.486GiB/2.506GiB
+2022-12-08 15:05:56.697,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.587GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:56.697,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:58.819,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,22.724MiB/s,22.979MiB/s,4.385GiB/1.935GiB/1.949GiB
+2022-12-08 15:05:58.819,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.136GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:05:58.819,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:06:00.454,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,19.725MiB/s,19.863MiB/s,3.758GiB/1.345GiB/1.351GiB
+2022-12-08 15:06:00.455,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.685GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:06:00.458,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.005,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/6,32/24,16.206MiB/s,16.261MiB/s,5.011GiB/2.385GiB/2.393GiB
+2022-12-08 15:07:47.005,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.005,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.027,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/7,32/28,16.204MiB/s,16.261MiB/s,5.011GiB/2.385GiB/2.394GiB
+2022-12-08 15:07:47.028,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.029,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.042,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,16.202MiB/s,16.261MiB/s,5.011GiB/2.385GiB/2.394GiB
+2022-12-08 15:07:47.043,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.044,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.055,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/9,36/36,16.201MiB/s,16.260MiB/s,5.638GiB/2.385GiB/2.394GiB
+2022-12-08 15:07:47.056,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.058,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.078,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,16.198MiB/s,16.260MiB/s,6.264GiB/2.385GiB/2.394GiB
+2022-12-08 15:07:47.079,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:07:47.079,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:31.653,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,32.595MiB/s,32.940MiB/s,6.264GiB/3.798GiB/3.820GiB
+2022-12-08 15:08:31.653,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:31.653,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:31.655,submitted,0,T4,0.000 B,2.128 GiB,-1,36,4.937GiB,9/9,36/36,29.309MiB/s,29.655MiB/s,5.637GiB/3.171GiB/3.193GiB
+2022-12-08 15:08:31.655,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.783GiB,2/1,8/4,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:08:31.656,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:31.672,submitted,0,T4,0.000 B,2.128 GiB,-1,36,4.937GiB,9/9,36/36,29.301MiB/s,29.655MiB/s,5.637GiB/3.171GiB/3.194GiB
+2022-12-08 15:08:31.673,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.783GiB,2/2,8/8,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:08:31.673,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:35.390,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,29.190MiB/s,29.390MiB/s,5.637GiB/3.268GiB/3.282GiB
+2022-12-08 15:08:35.390,completed,1,T4,2.128 GiB,8.513 GiB,3,32,8.783GiB,2/2,8/8,5.107MiB/s,6.130MiB/s,2.506GiB/18.977MiB/22.781MiB
+2022-12-08 15:08:35.391,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:35.395,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,25.968MiB/s,26.170MiB/s,5.011GiB/2.642GiB/2.656GiB
+2022-12-08 15:08:35.395,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.332GiB,3/3,12/12,5.101MiB/s,6.123MiB/s,3.759GiB/18.977MiB/22.781MiB
+2022-12-08 15:08:35.399,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:35.404,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,25.979MiB/s,26.169MiB/s,5.011GiB/2.645GiB/2.656GiB
+2022-12-08 15:08:35.404,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.332GiB,4/4,16/16,5.088MiB/s,6.108MiB/s,5.012GiB/18.977MiB/22.781MiB
+2022-12-08 15:08:35.404,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:35.715,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,25.875MiB/s,26.136MiB/s,5.011GiB/2.648GiB/2.662GiB
+2022-12-08 15:08:35.715,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.332GiB,4/4,16/16,5.980MiB/s,6.798MiB/s,5.012GiB/24.164MiB/24.653MiB
+2022-12-08 15:08:35.716,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:35.722,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,22.656MiB/s,22.919MiB/s,4.385GiB/2.021GiB/2.035GiB
+2022-12-08 15:08:35.723,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.880GiB,5/5,20/20,5.970MiB/s,6.898MiB/s,6.265GiB/24.164MiB/24.746MiB
+2022-12-08 15:08:35.724,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:35.731,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,22.653MiB/s,22.920MiB/s,4.385GiB/2.021GiB/2.036GiB
+2022-12-08 15:08:35.731,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.880GiB,6/6,24/24,5.957MiB/s,6.993MiB/s,7.519GiB/24.164MiB/24.824MiB
+2022-12-08 15:08:35.731,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:36.565,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,22.676MiB/s,22.901MiB/s,4.385GiB/2.040GiB/2.051GiB
+2022-12-08 15:08:36.567,completed,1,T4,2.128 GiB,8.513 GiB,3,36,9.880GiB,6/6,24/24,9.325MiB/s,18.762MiB/s,7.519GiB/29.209MiB/42.159MiB
+2022-12-08 15:08:36.574,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:36.580,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,19.468MiB/s,19.697MiB/s,3.759GiB/1.413GiB/1.425GiB
+2022-12-08 15:08:36.581,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.429GiB,8/7,32/28,9.251MiB/s,18.752MiB/s,10.025GiB/29.209MiB/42.488MiB
+2022-12-08 15:08:36.581,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:36.588,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,19.465MiB/s,19.698MiB/s,3.759GiB/1.413GiB/1.425GiB
+2022-12-08 15:08:36.589,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.429GiB,8/8,32/32,9.214MiB/s,18.784MiB/s,10.025GiB/29.209MiB/42.680MiB
+2022-12-08 15:08:36.589,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:36.611,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,19.458MiB/s,19.698MiB/s,3.759GiB/1.413GiB/1.425GiB
+2022-12-08 15:08:36.611,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.429GiB,8/8,32/32,9.111MiB/s,18.828MiB/s,10.025GiB/29.209MiB/43.226MiB
+2022-12-08 15:08:36.612,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:36.619,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,16.252MiB/s,16.493MiB/s,3.132GiB/805.516MiB/817.489MiB
+2022-12-08 15:08:36.620,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.979GiB,9/9,36/36,9.073MiB/s,18.791MiB/s,11.278GiB/29.209MiB/43.313MiB
+2022-12-08 15:08:36.620,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:08:36.627,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,16.249MiB/s,16.493MiB/s,3.132GiB/805.516MiB/817.599MiB
+2022-12-08 15:08:36.627,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.979GiB,10/10,40/40,9.038MiB/s,18.728MiB/s,12.532GiB/29.209MiB/43.408MiB
+2022-12-08 15:08:36.628,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:10:36.726,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/6,36/24,15.597MiB/s,15.647MiB/s,5.637GiB/2.584GiB/2.593GiB
+2022-12-08 15:10:36.726,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,29.819MiB/s,30.038MiB/s,12.532GiB/3.539GiB/3.565GiB
+2022-12-08 15:10:36.727,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:10:36.755,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/7,36/28,15.594MiB/s,15.647MiB/s,5.638GiB/2.584GiB/2.593GiB
+2022-12-08 15:10:36.756,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,29.812MiB/s,30.036MiB/s,12.532GiB/3.539GiB/3.565GiB
+2022-12-08 15:10:36.756,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:10:36.766,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,15.593MiB/s,15.647MiB/s,5.638GiB/2.584GiB/2.593GiB
+2022-12-08 15:10:36.766,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,29.809MiB/s,30.037MiB/s,12.532GiB/3.539GiB/3.566GiB
+2022-12-08 15:10:36.766,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:10:36.791,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/9,36/36,15.591MiB/s,15.646MiB/s,5.638GiB/2.584GiB/2.593GiB
+2022-12-08 15:10:36.793,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,29.803MiB/s,30.034MiB/s,12.532GiB/3.539GiB/3.566GiB
+2022-12-08 15:10:36.794,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:10:36.808,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.589MiB/s,15.646MiB/s,6.264GiB/2.584GiB/2.594GiB
+2022-12-08 15:10:36.809,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,29.799MiB/s,30.034MiB/s,12.532GiB/3.539GiB/3.567GiB
+2022-12-08 15:10:36.809,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:11:10.239,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,31.646MiB/s,31.772MiB/s,6.264GiB/3.626GiB/3.633GiB
+2022-12-08 15:11:10.239,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,30.028MiB/s,30.220MiB/s,12.532GiB/4.546GiB/4.575GiB
+2022-12-08 15:11:10.240,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:11:11.370,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,28.438MiB/s,28.598MiB/s,5.638GiB/3.027GiB/3.035GiB
+2022-12-08 15:11:11.372,completed,1,T4,2.128 GiB,8.513 GiB,1,42,11.527GiB,10/10,40/40,30.020MiB/s,30.228MiB/s,12.532GiB/4.578GiB/4.610GiB
+2022-12-08 15:11:11.380,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:11:13.437,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,25.241MiB/s,25.351MiB/s,5.011GiB/2.449GiB/2.453GiB
+2022-12-08 15:11:13.448,completed,1,T4,2.128 GiB,8.513 GiB,1,44,12.076GiB,10/10,40/40,30.038MiB/s,30.231MiB/s,12.532GiB/4.641GiB/4.671GiB
+2022-12-08 15:11:13.449,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:11:13.523,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,22.094MiB/s,22.239MiB/s,4.385GiB/1.823GiB/1.828GiB
+2022-12-08 15:11:13.526,completed,1,T4,2.128 GiB,8.513 GiB,1,46,12.625GiB,10/10,40/40,30.021MiB/s,30.229MiB/s,12.532GiB/4.641GiB/4.674GiB
+2022-12-08 15:11:13.526,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:11:13.660,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,18.925MiB/s,19.122MiB/s,3.759GiB/1.196GiB/1.203GiB
+2022-12-08 15:11:13.663,completed,1,T4,2.128 GiB,8.513 GiB,1,48,13.174GiB,10/10,40/40,29.995MiB/s,30.232MiB/s,12.532GiB/4.641GiB/4.678GiB
+2022-12-08 15:11:13.664,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:13:04.641,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,28/24,15.151MiB/s,15.230MiB/s,4.385GiB/2.188GiB/2.199GiB
+2022-12-08 15:13:04.644,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.723GiB,10/10,40/40,30.095MiB/s,30.252MiB/s,12.532GiB/7.918GiB/7.960GiB
+2022-12-08 15:13:04.645,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:13:04.663,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/7,32/28,15.149MiB/s,15.230MiB/s,5.011GiB/2.188GiB/2.199GiB
+2022-12-08 15:13:04.666,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.723GiB,10/10,40/40,30.093MiB/s,30.252MiB/s,12.532GiB/7.918GiB/7.960GiB
+2022-12-08 15:13:04.667,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:13:04.680,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,15.147MiB/s,15.230MiB/s,5.011GiB/2.188GiB/2.200GiB
+2022-12-08 15:13:04.680,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.723GiB,10/10,40/40,30.091MiB/s,30.251MiB/s,12.532GiB/7.918GiB/7.961GiB
+2022-12-08 15:13:04.691,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:13:04.713,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/9,36/36,15.144MiB/s,15.230MiB/s,5.638GiB/2.188GiB/2.200GiB
+2022-12-08 15:13:04.714,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.723GiB,10/10,40/40,30.087MiB/s,30.251MiB/s,12.532GiB/7.918GiB/7.961GiB
+2022-12-08 15:13:04.714,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:13:04.728,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.142MiB/s,15.230MiB/s,6.264GiB/2.188GiB/2.200GiB
+2022-12-08 15:13:04.729,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.723GiB,10/10,40/40,30.085MiB/s,30.250MiB/s,12.532GiB/7.918GiB/7.962GiB
+2022-12-08 15:13:04.729,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:14:05.791,completed,0,T4,0.000 B,2.128 GiB,1,45,6.171GiB,10/10,40/40,30.429MiB/s,30.558MiB/s,6.264GiB/4.006GiB/4.017GiB
+2022-12-08 15:14:05.791,completed,1,T4,2.128 GiB,8.513 GiB,1,50,13.723GiB,10/10,40/40,30.026MiB/s,30.200MiB/s,12.532GiB/9.693GiB/9.750GiB
+2022-12-08 15:14:05.791,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:14:07.570,completed,0,T4,0.000 B,2.128 GiB,1,41,5.622GiB,9/9,36/36,27.253MiB/s,27.467MiB/s,5.637GiB/3.416GiB/3.434GiB
+2022-12-08 15:14:07.577,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.272GiB,10/10,40/40,30.026MiB/s,30.201MiB/s,12.532GiB/9.746GiB/9.802GiB
+2022-12-08 15:14:07.577,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:14:09.220,completed,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,24.215MiB/s,24.351MiB/s,5.011GiB/2.829GiB/2.840GiB
+2022-12-08 15:14:09.220,completed,1,T4,2.128 GiB,8.513 GiB,2,54,14.821GiB,10/10,40/40,29.993MiB/s,30.182MiB/s,12.532GiB/9.783GiB/9.845GiB
+2022-12-08 15:14:09.220,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:14:09.956,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,21.257MiB/s,21.321MiB/s,4.385GiB/2.223GiB/2.228GiB
+2022-12-08 15:14:09.958,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.370GiB,10/10,40/40,30.001MiB/s,30.178MiB/s,12.532GiB/9.807GiB/9.865GiB
+2022-12-08 15:14:09.958,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:14:10.852,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,18.168MiB/s,18.291MiB/s,3.759GiB/1.607GiB/1.615GiB
+2022-12-08 15:14:10.852,completed,1,T4,2.128 GiB,8.513 GiB,2,58,15.918GiB,10/10,40/40,30.001MiB/s,30.171MiB/s,12.532GiB/9.834GiB/9.889GiB
+2022-12-08 15:14:10.852,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:35.000,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,14.984MiB/s,15.050MiB/s,3.132GiB/2.199GiB/2.209GiB
+2022-12-08 15:15:35.006,completed,1,T4,2.128 GiB,8.513 GiB,2,60,16.468GiB,10/10,40/40,29.759MiB/s,29.914MiB/s,12.532GiB/12.200GiB/12.263GiB
+2022-12-08 15:15:35.006,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.019GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:40.798,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.084MiB/s,15.149MiB/s,3.132GiB/2.299GiB/2.309GiB
+2022-12-08 15:15:40.799,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.370GiB,9/9,36/36,26.722MiB/s,26.870MiB/s,11.278GiB/11.110GiB/11.172GiB
+2022-12-08 15:15:40.799,completed,2,T4,8.513 GiB,34.052 GiB,3,21,23.121GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:42.085,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.102MiB/s,15.156MiB/s,3.132GiB/2.321GiB/2.329GiB
+2022-12-08 15:15:42.086,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.272GiB,8/8,32/32,23.748MiB/s,23.867MiB/s,10.025GiB/9.894GiB/9.943GiB
+2022-12-08 15:15:42.086,completed,2,T4,8.513 GiB,34.052 GiB,3,22,24.222GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:44.627,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.121MiB/s,15.151MiB/s,3.132GiB/2.361GiB/2.366GiB
+2022-12-08 15:15:44.627,completed,1,T4,2.128 GiB,8.513 GiB,2,48,13.174GiB,7/7,28/28,20.768MiB/s,20.867MiB/s,8.772GiB/8.693GiB/8.734GiB
+2022-12-08 15:15:44.627,completed,2,T4,8.513 GiB,34.052 GiB,3,23,25.324GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:44.867,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/6,32/24,15.098MiB/s,15.150MiB/s,5.011GiB/2.361GiB/2.369GiB
+2022-12-08 15:15:44.882,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.077GiB,6/6,24/24,17.770MiB/s,17.865MiB/s,7.519GiB/7.440GiB/7.480GiB
+2022-12-08 15:15:44.883,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,26.425GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:44.892,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/7,32/28,15.096MiB/s,15.149MiB/s,5.012GiB/2.361GiB/2.370GiB
+2022-12-08 15:15:44.902,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.077GiB,6/6,24/24,17.769MiB/s,17.865MiB/s,7.519GiB/7.440GiB/7.481GiB
+2022-12-08 15:15:44.902,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,26.425GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:44.907,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,15.094MiB/s,15.149MiB/s,5.012GiB/2.361GiB/2.370GiB
+2022-12-08 15:15:44.908,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.077GiB,6/6,24/24,17.768MiB/s,17.865MiB/s,7.519GiB/7.440GiB/7.481GiB
+2022-12-08 15:15:44.910,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,26.425GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:44.912,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,15.094MiB/s,15.149MiB/s,6.264GiB/2.361GiB/2.370GiB
+2022-12-08 15:15:44.912,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.077GiB,6/6,24/24,17.768MiB/s,17.866MiB/s,7.519GiB/7.440GiB/7.481GiB
+2022-12-08 15:15:44.912,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,26.425GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:44.936,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.091MiB/s,15.152MiB/s,6.264GiB/2.361GiB/2.371GiB
+2022-12-08 15:15:44.938,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.077GiB,6/6,24/24,17.770MiB/s,17.866MiB/s,7.519GiB/7.441GiB/7.482GiB
+2022-12-08 15:15:44.938,submitted,2,T4,8.513 GiB,34.052 GiB,3,24,26.425GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:45.333,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,15.080MiB/s,27.149MiB/s,6.264GiB/2.365GiB/2.383GiB
+2022-12-08 15:15:45.334,completed,1,T4,2.128 GiB,8.513 GiB,2,44,12.077GiB,6/6,24/24,17.770MiB/s,17.865MiB/s,7.519GiB/7.448GiB/7.488GiB
+2022-12-08 15:15:45.334,completed,2,T4,8.513 GiB,34.052 GiB,3,24,26.425GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:45.837,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,15.106MiB/s,32.319MiB/s,6.264GiB/2.377GiB/2.408GiB
+2022-12-08 15:15:45.837,completed,1,T4,2.128 GiB,8.513 GiB,2,40,10.978GiB,5/5,20/20,14.779MiB/s,14.872MiB/s,6.266GiB/6.201GiB/6.240GiB
+2022-12-08 15:15:45.837,completed,2,T4,8.513 GiB,34.052 GiB,3,25,27.527GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:46.259,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,27.862MiB/s,31.855MiB/s,6.264GiB/2.402GiB/2.420GiB
+2022-12-08 15:15:46.260,completed,1,T4,2.128 GiB,8.513 GiB,2,36,9.880GiB,4/4,16/16,11.808MiB/s,11.867MiB/s,5.012GiB/4.960GiB/4.985GiB
+2022-12-08 15:15:46.268,completed,2,T4,8.513 GiB,34.052 GiB,3,26,28.629GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:51.277,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,30.616MiB/s,32.026MiB/s,6.264GiB/2.562GiB/2.581GiB
+2022-12-08 15:15:51.277,completed,1,T4,2.128 GiB,8.513 GiB,2,32,8.782GiB,3/3,12/12,8.825MiB/s,8.869MiB/s,3.759GiB/3.752GiB/3.771GiB
+2022-12-08 15:15:51.277,completed,2,T4,8.513 GiB,34.052 GiB,3,27,29.731GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:51.783,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,29.492MiB/s,31.985MiB/s,6.264GiB/2.573GiB/2.597GiB
+2022-12-08 15:15:51.785,completed,1,T4,2.128 GiB,8.513 GiB,2,28,7.685GiB,2/2,8/8,5.881MiB/s,5.911MiB/s,2.506GiB/2.502GiB/2.514GiB
+2022-12-08 15:15:51.787,completed,2,T4,8.513 GiB,34.052 GiB,3,28,30.832GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:15:52.612,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,30.629MiB/s,32.119MiB/s,6.264GiB/2.611GiB/2.627GiB
+2022-12-08 15:15:52.612,completed,1,T4,2.128 GiB,8.513 GiB,2,24,6.587GiB,1/1,4/4,2.937MiB/s,2.949MiB/s,1.253GiB/1.253GiB/1.258GiB
+2022-12-08 15:15:52.612,completed,2,T4,8.513 GiB,34.052 GiB,3,29,31.934GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:26.825,completed,0,T4,0.000 B,2.128 GiB,1,45,6.171GiB,10/10,40/40,33.964MiB/s,34.070MiB/s,6.264GiB/3.849GiB/3.855GiB
+2022-12-08 15:16:26.828,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:26.829,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:27.478,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,30.544MiB/s,30.793MiB/s,5.637GiB/3.227GiB/3.241GiB
+2022-12-08 15:16:27.478,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.038GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:27.478,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:27.661,completed,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,27.296MiB/s,27.583MiB/s,5.011GiB/2.601GiB/2.617GiB
+2022-12-08 15:16:27.661,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.587GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:27.661,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:31.100,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,24.277MiB/s,24.377MiB/s,4.385GiB/2.066GiB/2.071GiB
+2022-12-08 15:16:31.101,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.136GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:31.101,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:32.061,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,21.104MiB/s,21.202MiB/s,3.758GiB/1.455GiB/1.460GiB
+2022-12-08 15:16:32.061,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.685GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:16:32.061,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.677,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/6,32/24,17.250MiB/s,17.369MiB/s,5.011GiB/2.371GiB/2.387GiB
+2022-12-08 15:18:05.679,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.680,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.692,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/7,36/28,17.247MiB/s,17.369MiB/s,5.637GiB/2.371GiB/2.388GiB
+2022-12-08 15:18:05.692,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.694,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.700,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,17.246MiB/s,17.370MiB/s,5.011GiB/2.371GiB/2.388GiB
+2022-12-08 15:18:05.703,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.703,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.714,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/9,36/36,17.245MiB/s,17.370MiB/s,5.637GiB/2.371GiB/2.388GiB
+2022-12-08 15:18:05.714,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.714,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.724,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,17.243MiB/s,17.371MiB/s,6.264GiB/2.371GiB/2.388GiB
+2022-12-08 15:18:05.724,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:05.724,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:45.693,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,35.538MiB/s,35.907MiB/s,6.264GiB/3.778GiB/3.801GiB
+2022-12-08 15:18:45.693,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:45.698,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:45.707,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,31.983MiB/s,32.363MiB/s,5.638GiB/3.152GiB/3.175GiB
+2022-12-08 15:18:45.707,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.782GiB,2/1,8/4,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:18:45.708,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:45.716,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,31.978MiB/s,32.365MiB/s,5.638GiB/3.152GiB/3.176GiB
+2022-12-08 15:18:45.718,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.782GiB,2/2,8/8,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:18:45.719,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:48.356,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,31.943MiB/s,32.063MiB/s,5.638GiB/3.232GiB/3.240GiB
+2022-12-08 15:18:48.356,completed,1,T4,2.128 GiB,8.513 GiB,3,32,8.782GiB,2/2,8/8,4.553MiB/s,6.580MiB/s,2.506GiB/11.981MiB/17.306MiB
+2022-12-08 15:18:48.356,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:48.364,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,28.442MiB/s,28.565MiB/s,5.011GiB/2.605GiB/2.614GiB
+2022-12-08 15:18:48.364,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.331GiB,4/3,16/12,4.538MiB/s,6.591MiB/s,5.012GiB/11.981MiB/17.391MiB
+2022-12-08 15:18:48.369,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:48.387,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,28.431MiB/s,28.564MiB/s,5.011GiB/2.605GiB/2.615GiB
+2022-12-08 15:18:48.392,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.331GiB,4/4,16/16,4.499MiB/s,6.591MiB/s,5.012GiB/11.981MiB/17.540MiB
+2022-12-08 15:18:48.392,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:51.231,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,28.267MiB/s,28.369MiB/s,5.011GiB/2.676GiB/2.681GiB
+2022-12-08 15:18:51.233,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.331GiB,4/4,16/16,10.294MiB/s,12.285MiB/s,5.012GiB/46.877MiB/53.772MiB
+2022-12-08 15:18:51.234,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:51.251,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,24.818MiB/s,24.927MiB/s,4.385GiB/2.050GiB/2.055GiB
+2022-12-08 15:18:51.251,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.880GiB,5/5,20/20,10.246MiB/s,12.265MiB/s,6.265GiB/46.877MiB/53.912MiB
+2022-12-08 15:18:51.251,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:51.264,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,24.812MiB/s,24.925MiB/s,4.385GiB/2.050GiB/2.056GiB
+2022-12-08 15:18:51.264,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.880GiB,6/6,24/24,10.216MiB/s,12.296MiB/s,7.518GiB/46.877MiB/54.172MiB
+2022-12-08 15:18:51.265,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:52.071,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,24.586MiB/s,24.804MiB/s,4.385GiB/2.056GiB/2.066GiB
+2022-12-08 15:18:52.072,completed,1,T4,2.128 GiB,8.513 GiB,3,36,9.880GiB,6/6,24/24,10.883MiB/s,16.740MiB/s,7.518GiB/57.755MiB/67.195MiB
+2022-12-08 15:18:52.072,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:52.074,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,21.156MiB/s,21.372MiB/s,3.758GiB/1.429GiB/1.439GiB
+2022-12-08 15:18:52.074,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.429GiB,7/7,28/28,10.878MiB/s,16.723MiB/s,8.772GiB/57.755MiB/67.217MiB
+2022-12-08 15:18:52.088,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:52.114,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,21.236MiB/s,21.365MiB/s,3.758GiB/1.433GiB/1.439GiB
+2022-12-08 15:18:52.114,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.429GiB,8/8,32/32,10.791MiB/s,16.762MiB/s,10.025GiB/57.755MiB/67.855MiB
+2022-12-08 15:18:52.114,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:52.191,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,21.265MiB/s,21.354MiB/s,3.758GiB/1.436GiB/1.440GiB
+2022-12-08 15:18:52.194,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.429GiB,8/8,32/32,10.627MiB/s,16.881MiB/s,10.025GiB/57.755MiB/69.121MiB
+2022-12-08 15:18:52.194,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:52.206,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,17.834MiB/s,17.926MiB/s,3.132GiB/829.095MiB/833.352MiB
+2022-12-08 15:18:52.207,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.978GiB,10/9,40/36,10.595MiB/s,16.912MiB/s,12.531GiB/57.755MiB/69.424MiB
+2022-12-08 15:18:52.207,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:18:52.221,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,17.828MiB/s,17.924MiB/s,3.132GiB/829.095MiB/833.533MiB
+2022-12-08 15:18:52.221,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.978GiB,10/10,40/40,10.565MiB/s,16.868MiB/s,12.531GiB/57.755MiB/69.569MiB
+2022-12-08 15:18:52.221,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:20:43.246,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,28/24,16.221MiB/s,16.242MiB/s,4.385GiB/2.495GiB/2.499GiB
+2022-12-08 15:20:43.246,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,30.655MiB/s,30.832MiB/s,12.531GiB/3.392GiB/3.412GiB
+2022-12-08 15:20:43.246,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:20:43.259,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/7,40/28,16.220MiB/s,16.242MiB/s,6.264GiB/2.495GiB/2.499GiB
+2022-12-08 15:20:43.260,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,30.652MiB/s,30.834MiB/s,12.531GiB/3.392GiB/3.412GiB
+2022-12-08 15:20:43.260,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:20:43.269,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,16.219MiB/s,16.243MiB/s,5.638GiB/2.495GiB/2.499GiB
+2022-12-08 15:20:43.271,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,30.649MiB/s,30.835MiB/s,12.531GiB/3.392GiB/3.413GiB
+2022-12-08 15:20:43.271,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:20:43.292,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,16.217MiB/s,16.242MiB/s,6.264GiB/2.495GiB/2.499GiB
+2022-12-08 15:20:43.294,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,30.643MiB/s,30.833MiB/s,12.531GiB/3.392GiB/3.413GiB
+2022-12-08 15:20:43.295,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:20:43.309,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,16.215MiB/s,16.242MiB/s,6.264GiB/2.495GiB/2.500GiB
+2022-12-08 15:20:43.319,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,30.638MiB/s,30.833MiB/s,12.531GiB/3.392GiB/3.414GiB
+2022-12-08 15:20:43.319,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:21:25.838,completed,0,T4,0.000 B,2.128 GiB,1,45,6.171GiB,10/10,40/40,30.592MiB/s,30.654MiB/s,6.264GiB/3.695GiB/3.701GiB
+2022-12-08 15:21:25.842,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,29.910MiB/s,30.098MiB/s,12.531GiB/4.554GiB/4.582GiB
+2022-12-08 15:21:25.849,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:21:29.036,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,27.042MiB/s,27.159MiB/s,5.637GiB/3.128GiB/3.137GiB
+2022-12-08 15:21:29.036,completed,1,T4,2.128 GiB,8.513 GiB,1,42,11.527GiB,10/10,40/40,29.736MiB/s,29.954MiB/s,12.531GiB/4.620GiB/4.654GiB
+2022-12-08 15:21:29.036,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:21:29.992,completed,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,23.886MiB/s,23.977MiB/s,5.011GiB/2.523GiB/2.530GiB
+2022-12-08 15:21:29.994,completed,1,T4,2.128 GiB,8.513 GiB,1,44,12.076GiB,10/10,40/40,29.743MiB/s,29.943MiB/s,12.531GiB/4.649GiB/4.680GiB
+2022-12-08 15:21:29.998,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:21:31.649,completed,0,T4,0.000 B,2.128 GiB,1,33,4.525GiB,7/7,28/28,20.517MiB/s,20.810MiB/s,4.384GiB/1.920GiB/1.934GiB
+2022-12-08 15:21:31.654,completed,1,T4,2.128 GiB,8.513 GiB,1,46,12.625GiB,10/10,40/40,29.714MiB/s,29.931MiB/s,12.531GiB/4.692GiB/4.727GiB
+2022-12-08 15:21:31.654,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:21:36.343,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,17.386MiB/s,17.516MiB/s,3.758GiB/1.369GiB/1.376GiB
+2022-12-08 15:21:36.344,completed,1,T4,2.128 GiB,8.513 GiB,1,48,13.174GiB,10/10,40/40,29.575MiB/s,29.808MiB/s,12.531GiB/4.806GiB/4.844GiB
+2022-12-08 15:21:36.346,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:23:24.929,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,8/6,32/24,15.198MiB/s,15.273MiB/s,5.010GiB/2.399GiB/2.411GiB
+2022-12-08 15:23:24.929,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,30.166MiB/s,30.320MiB/s,12.531GiB/8.101GiB/8.142GiB
+2022-12-08 15:23:24.930,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:23:24.945,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,8/7,32/28,15.215MiB/s,15.273MiB/s,5.010GiB/2.402GiB/2.411GiB
+2022-12-08 15:23:24.945,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,30.176MiB/s,30.320MiB/s,12.531GiB/8.104GiB/8.143GiB
+2022-12-08 15:23:24.946,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:23:24.976,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,9/8,36/32,15.212MiB/s,15.274MiB/s,5.637GiB/2.402GiB/2.412GiB
+2022-12-08 15:23:24.985,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,30.184MiB/s,30.319MiB/s,12.531GiB/8.107GiB/8.143GiB
+2022-12-08 15:23:24.986,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:23:24.996,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,9/9,36/36,15.210MiB/s,15.273MiB/s,5.637GiB/2.402GiB/2.412GiB
+2022-12-08 15:23:25.002,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,30.182MiB/s,30.318MiB/s,12.531GiB/8.107GiB/8.144GiB
+2022-12-08 15:23:25.003,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:23:25.021,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,10/10,40/40,15.208MiB/s,15.273MiB/s,6.263GiB/2.402GiB/2.412GiB
+2022-12-08 15:23:25.022,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,30.179MiB/s,30.318MiB/s,12.531GiB/8.107GiB/8.144GiB
+2022-12-08 15:23:25.024,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:24:18.798,completed,0,T4,0.000 B,2.128 GiB,1,45,6.171GiB,10/10,40/40,28.573MiB/s,28.640MiB/s,6.263GiB/3.835GiB/3.842GiB
+2022-12-08 15:24:18.798,completed,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,29.607MiB/s,29.760MiB/s,12.531GiB/9.508GiB/9.557GiB
+2022-12-08 15:24:18.798,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:24:19.652,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,25.540MiB/s,25.623MiB/s,5.638GiB/3.227GiB/3.233GiB
+2022-12-08 15:24:19.663,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.271GiB,10/10,40/40,29.607MiB/s,29.751MiB/s,12.531GiB/9.533GiB/9.579GiB
+2022-12-08 15:24:19.664,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:24:20.469,completed,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,22.491MiB/s,22.681MiB/s,5.011GiB/2.611GiB/2.624GiB
+2022-12-08 15:24:20.469,completed,1,T4,2.128 GiB,8.513 GiB,2,54,14.820GiB,10/10,40/40,29.604MiB/s,29.743MiB/s,12.531GiB/9.556GiB/9.600GiB
+2022-12-08 15:24:20.469,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:24:22.032,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.522MiB/s,19.640MiB/s,4.385GiB/2.012GiB/2.020GiB
+2022-12-08 15:24:22.033,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.369GiB,10/10,40/40,29.536MiB/s,29.702MiB/s,12.531GiB/9.579GiB/9.633GiB
+2022-12-08 15:24:22.033,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:24:22.813,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,16.651MiB/s,16.712MiB/s,3.759GiB/1.402GiB/1.406GiB
+2022-12-08 15:24:22.813,completed,1,T4,2.128 GiB,8.513 GiB,2,58,15.917GiB,10/10,40/40,29.545MiB/s,29.697MiB/s,12.531GiB/9.604GiB/9.654GiB
+2022-12-08 15:24:22.813,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:25:52.046,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.151MiB/s,15.195MiB/s,3.132GiB/2.176GiB/2.182GiB
+2022-12-08 15:25:52.051,completed,1,T4,2.128 GiB,8.513 GiB,2,60,16.467GiB,10/10,40/40,29.926MiB/s,30.073MiB/s,12.531GiB/12.336GiB/12.396GiB
+2022-12-08 15:25:52.051,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.035GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:25:52.053,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.150MiB/s,15.195MiB/s,3.132GiB/2.176GiB/2.182GiB
+2022-12-08 15:25:52.053,submitted,1,T4,2.128 GiB,8.513 GiB,2,56,15.369GiB,9/9,36/36,26.897MiB/s,27.032MiB/s,11.278GiB/11.083GiB/11.138GiB
+2022-12-08 15:25:52.053,submitted,2,T4,8.513 GiB,34.052 GiB,4,31,34.137GiB,1/1,4/4,0B/s,0B/s,5.033GiB/0B/0B
+2022-12-08 15:25:53.876,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.115MiB/s,15.148MiB/s,3.132GiB/2.198GiB/2.203GiB
+2022-12-08 15:25:53.877,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.369GiB,9/9,36/36,26.875MiB/s,27.013MiB/s,11.278GiB/11.122GiB/11.178GiB
+2022-12-08 15:25:53.877,completed,2,T4,8.513 GiB,34.052 GiB,3,31,34.137GiB,1/1,4/4,1.086MiB/s,2.170MiB/s,5.033GiB/1.962MiB/3.918MiB
+2022-12-08 15:25:53.885,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.114MiB/s,15.148MiB/s,3.132GiB/2.198GiB/2.203GiB
+2022-12-08 15:25:53.886,submitted,1,T4,2.128 GiB,8.513 GiB,2,52,14.271GiB,8/8,32/32,23.878MiB/s,24.003MiB/s,10.025GiB/9.869GiB/9.920GiB
+2022-12-08 15:25:53.886,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,35.238GiB,2/2,8/8,1.081MiB/s,2.182MiB/s,10.064GiB/1.962MiB/3.960MiB
+2022-12-08 15:25:56.845,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.090MiB/s,15.122MiB/s,3.132GiB/2.238GiB/2.243GiB
+2022-12-08 15:25:56.849,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.271GiB,8/8,32/32,23.865MiB/s,23.983MiB/s,10.025GiB/9.932GiB/9.981GiB
+2022-12-08 15:25:56.849,completed,2,T4,8.513 GiB,34.052 GiB,3,32,35.238GiB,2/2,8/8,4.208MiB/s,5.490MiB/s,10.064GiB/16.613MiB/21.191MiB
+2022-12-08 15:25:56.858,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.089MiB/s,15.121MiB/s,3.132GiB/2.238GiB/2.243GiB
+2022-12-08 15:25:56.859,submitted,1,T4,2.128 GiB,8.513 GiB,2,48,13.174GiB,7/7,28/28,20.870MiB/s,20.976MiB/s,8.772GiB/8.679GiB/8.723GiB
+2022-12-08 15:25:56.859,submitted,2,T4,8.513 GiB,34.052 GiB,4,33,36.340GiB,3/3,12/12,4.193MiB/s,5.488MiB/s,15.096GiB/16.613MiB/21.247MiB
+2022-12-08 15:25:58.287,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.032MiB/s,15.072MiB/s,3.132GiB/2.250GiB/2.256GiB
+2022-12-08 15:25:58.287,completed,1,T4,2.128 GiB,8.513 GiB,2,48,13.174GiB,7/7,28/28,20.847MiB/s,20.952MiB/s,8.772GiB/8.699GiB/8.743GiB
+2022-12-08 15:25:58.288,completed,2,T4,8.513 GiB,34.052 GiB,3,33,36.340GiB,3/3,12/12,7.185MiB/s,7.748MiB/s,15.096GiB/29.786MiB/31.327MiB
+2022-12-08 15:25:58.294,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.031MiB/s,15.072MiB/s,3.132GiB/2.250GiB/2.257GiB
+2022-12-08 15:25:58.295,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.076GiB,6/6,24/24,17.882MiB/s,17.974MiB/s,7.519GiB/7.447GiB/7.485GiB
+2022-12-08 15:25:58.297,submitted,2,T4,8.513 GiB,34.052 GiB,4,34,37.441GiB,4/4,16/16,7.166MiB/s,7.744MiB/s,20.126GiB/29.786MiB/31.373MiB
+2022-12-08 15:25:59.670,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,14.999MiB/s,15.056MiB/s,3.132GiB/2.266GiB/2.274GiB
+2022-12-08 15:25:59.679,completed,1,T4,2.128 GiB,8.513 GiB,2,44,12.076GiB,6/6,24/24,17.880MiB/s,17.963MiB/s,7.519GiB/7.470GiB/7.505GiB
+2022-12-08 15:25:59.679,completed,2,T4,8.513 GiB,34.052 GiB,3,34,37.441GiB,4/4,16/16,8.712MiB/s,10.654MiB/s,20.126GiB/40.720MiB/47.077MiB
+2022-12-08 15:25:59.682,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,14.998MiB/s,15.056MiB/s,3.132GiB/2.266GiB/2.274GiB
+2022-12-08 15:25:59.697,submitted,1,T4,2.128 GiB,8.513 GiB,2,40,10.978GiB,5/5,20/20,14.878MiB/s,14.949MiB/s,6.266GiB/6.216GiB/6.246GiB
+2022-12-08 15:25:59.697,submitted,2,T4,8.513 GiB,34.052 GiB,4,35,38.543GiB,5/5,20/20,8.679MiB/s,10.678MiB/s,25.158GiB/40.720MiB/47.239MiB
+2022-12-08 15:26:00.103,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,14.995MiB/s,15.049MiB/s,3.132GiB/2.272GiB/2.280GiB
+2022-12-08 15:26:00.103,completed,1,T4,2.128 GiB,8.513 GiB,2,40,10.978GiB,5/5,20/20,14.866MiB/s,14.945MiB/s,6.266GiB/6.217GiB/6.251GiB
+2022-12-08 15:26:00.103,completed,2,T4,8.513 GiB,34.052 GiB,3,35,38.543GiB,5/5,20/20,7.696MiB/s,11.456MiB/s,25.158GiB/40.720MiB/51.977MiB
+2022-12-08 15:26:00.113,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,14.994MiB/s,15.048MiB/s,3.132GiB/2.272GiB/2.280GiB
+2022-12-08 15:26:00.113,submitted,1,T4,2.128 GiB,8.513 GiB,2,36,9.880GiB,4/4,16/16,11.865MiB/s,11.932MiB/s,5.012GiB/4.964GiB/4.992GiB
+2022-12-08 15:26:00.114,submitted,2,T4,8.513 GiB,34.052 GiB,4,36,39.645GiB,6/6,24/24,7.676MiB/s,11.604MiB/s,30.190GiB/40.720MiB/52.209MiB
+2022-12-08 15:26:00.865,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.009MiB/s,15.035MiB/s,3.132GiB/2.285GiB/2.289GiB
+2022-12-08 15:26:00.865,completed,1,T4,2.128 GiB,8.513 GiB,2,36,9.880GiB,4/4,16/16,11.871MiB/s,11.932MiB/s,5.012GiB/4.975GiB/5.000GiB
+2022-12-08 15:26:00.865,completed,2,T4,8.513 GiB,34.052 GiB,3,36,39.645GiB,6/6,24/24,11.694MiB/s,14.722MiB/s,30.190GiB/55.932MiB/64.542MiB
+2022-12-08 15:26:00.870,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.009MiB/s,15.035MiB/s,3.132GiB/2.285GiB/2.289GiB
+2022-12-08 15:26:00.870,submitted,1,T4,2.128 GiB,8.513 GiB,2,32,8.782GiB,3/3,12/12,8.884MiB/s,8.932MiB/s,3.759GiB/3.722GiB/3.742GiB
+2022-12-08 15:26:00.871,submitted,2,T4,8.513 GiB,34.052 GiB,4,37,40.746GiB,7/7,28/28,11.673MiB/s,14.714MiB/s,35.222GiB/55.932MiB/64.619MiB
+2022-12-08 15:26:01.643,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.016MiB/s,15.085MiB/s,3.132GiB/2.297GiB/2.308GiB
+2022-12-08 15:26:01.648,completed,1,T4,2.128 GiB,8.513 GiB,2,32,8.782GiB,3/3,12/12,8.891MiB/s,8.935MiB/s,3.759GiB/3.732GiB/3.750GiB
+2022-12-08 15:26:01.648,completed,2,T4,8.513 GiB,34.052 GiB,3,37,40.746GiB,7/7,28/28,12.428MiB/s,18.816MiB/s,35.222GiB/68.942MiB/82.491MiB
+2022-12-08 15:26:01.655,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.015MiB/s,15.085MiB/s,3.132GiB/2.297GiB/2.308GiB
+2022-12-08 15:26:01.655,submitted,1,T4,2.128 GiB,8.513 GiB,2,28,7.684GiB,2/2,8/8,5.909MiB/s,5.941MiB/s,2.506GiB/2.478GiB/2.492GiB
+2022-12-08 15:26:01.655,submitted,2,T4,8.513 GiB,34.052 GiB,4,38,41.848GiB,8/8,32/32,12.389MiB/s,18.833MiB/s,40.254GiB/68.942MiB/82.693MiB
+2022-12-08 15:26:03.409,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,14.994MiB/s,15.095MiB/s,3.132GiB/2.320GiB/2.335GiB
+2022-12-08 15:26:03.409,completed,1,T4,2.128 GiB,8.513 GiB,2,28,7.684GiB,2/2,8/8,5.912MiB/s,5.937MiB/s,2.506GiB/2.490GiB/2.500GiB
+2022-12-08 15:26:03.410,completed,2,T4,8.513 GiB,34.052 GiB,3,38,41.848GiB,8/8,32/32,19.474MiB/s,22.959MiB/s,40.254GiB/113.978MiB/124.905MiB
+2022-12-08 15:26:03.421,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,14.992MiB/s,15.094MiB/s,3.132GiB/2.320GiB/2.335GiB
+2022-12-08 15:26:03.421,submitted,1,T4,2.128 GiB,8.513 GiB,2,24,6.586GiB,1/1,4/4,2.936MiB/s,2.949MiB/s,1.253GiB/1.236GiB/1.242GiB
+2022-12-08 15:26:03.424,submitted,2,T4,8.513 GiB,34.052 GiB,4,39,42.950GiB,9/9,36/36,19.417MiB/s,22.933MiB/s,45.285GiB/113.978MiB/125.059MiB
+2022-12-08 15:26:08.771,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.105MiB/s,15.135MiB/s,3.132GiB/2.416GiB/2.421GiB
+2022-12-08 15:26:08.771,completed,1,T4,2.128 GiB,8.513 GiB,2,24,6.586GiB,1/1,4/4,2.938MiB/s,2.950MiB/s,1.253GiB/1.253GiB/1.258GiB
+2022-12-08 15:26:08.771,completed,2,T4,8.513 GiB,34.052 GiB,3,39,42.950GiB,9/9,36/36,27.145MiB/s,28.315MiB/s,45.285GiB/272.621MiB/283.750MiB
+2022-12-08 15:26:08.779,submitted,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,15.120MiB/s,15.135MiB/s,3.132GiB/2.418GiB/2.421GiB
+2022-12-08 15:26:08.780,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:26:08.780,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,27.118MiB/s,28.321MiB/s,50.317GiB/272.621MiB/284.015MiB
+2022-12-08 15:26:13.048,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,6/6,24/24,15.076MiB/s,15.110MiB/s,3.758GiB/2.474GiB/2.480GiB
+2022-12-08 15:26:13.060,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:26:13.060,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.020MiB/s,30.350MiB/s,50.317GiB/392.585MiB/409.828MiB
+2022-12-08 15:26:13.088,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,8/7,32/28,15.072MiB/s,15.110MiB/s,5.011GiB/2.474GiB/2.480GiB
+2022-12-08 15:26:13.089,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:26:13.090,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.918MiB/s,30.330MiB/s,50.317GiB/392.585MiB/410.856MiB
+2022-12-08 15:26:13.108,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,8/8,32/32,15.070MiB/s,15.110MiB/s,5.011GiB/2.474GiB/2.481GiB
+2022-12-08 15:26:13.108,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:26:13.108,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.870MiB/s,30.311MiB/s,50.317GiB/392.585MiB/411.211MiB
+2022-12-08 15:26:13.132,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,10/9,40/36,15.068MiB/s,15.109MiB/s,6.263GiB/2.474GiB/2.481GiB
+2022-12-08 15:26:13.132,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:26:13.132,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.810MiB/s,30.297MiB/s,50.317GiB/392.585MiB/411.821MiB
+2022-12-08 15:26:13.141,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.485GiB,10/10,40/40,15.067MiB/s,15.109MiB/s,6.263GiB/2.474GiB/2.481GiB
+2022-12-08 15:26:13.150,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:26:13.150,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.787MiB/s,30.291MiB/s,50.317GiB/392.585MiB/412.006MiB
+2022-12-08 15:27:01.957,completed,0,T4,0.000 B,2.128 GiB,1,45,6.171GiB,10/10,40/40,28.082MiB/s,28.210MiB/s,6.263GiB/3.740GiB/3.750GiB
+2022-12-08 15:27:01.957,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:27:01.957,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,27.697MiB/s,27.979MiB/s,50.317GiB/1.688GiB/1.705GiB
+2022-12-08 15:27:03.091,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,25.077MiB/s,25.227MiB/s,5.637GiB/3.137GiB/3.148GiB
+2022-12-08 15:27:03.091,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.038GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:27:03.091,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,27.723MiB/s,28.005MiB/s,50.317GiB/1.720GiB/1.737GiB
+2022-12-08 15:27:05.116,completed,0,T4,0.000 B,2.128 GiB,1,37,5.074GiB,8/8,32/32,22.072MiB/s,22.191MiB/s,5.010GiB/2.548GiB/2.556GiB
+2022-12-08 15:27:05.117,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.587GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:27:05.118,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,27.650MiB/s,27.855MiB/s,50.317GiB/1.770GiB/1.783GiB
+2022-12-08 15:27:06.169,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.189MiB/s,19.267MiB/s,4.384GiB/1.942GiB/1.947GiB
+2022-12-08 15:27:06.170,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.135GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:27:06.170,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,27.652MiB/s,27.857MiB/s,50.317GiB/1.799GiB/1.812GiB
+2022-12-08 15:27:08.598,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,16.274MiB/s,16.407MiB/s,3.758GiB/1.352GiB/1.359GiB
+2022-12-08 15:27:08.602,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.685GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:27:08.602,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,27.739MiB/s,27.973MiB/s,50.317GiB/1.870GiB/1.886GiB
+2022-12-08 15:29:00.464,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,28/24,15.172MiB/s,15.214MiB/s,4.384GiB/2.479GiB/2.486GiB
+2022-12-08 15:29:00.467,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:29:00.467,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,30.177MiB/s,30.324MiB/s,50.317GiB/5.331GiB/5.357GiB
+2022-12-08 15:29:00.480,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/7,36/28,15.170MiB/s,15.213MiB/s,5.637GiB/2.479GiB/2.486GiB
+2022-12-08 15:29:00.481,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:29:00.483,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,30.174MiB/s,30.323MiB/s,50.317GiB/5.331GiB/5.357GiB
+2022-12-08 15:29:00.504,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,15.168MiB/s,15.212MiB/s,5.637GiB/2.479GiB/2.486GiB
+2022-12-08 15:29:00.510,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:29:00.775,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,30.170MiB/s,30.321MiB/s,50.317GiB/5.331GiB/5.357GiB
+2022-12-08 15:29:00.777,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,15.143MiB/s,15.189MiB/s,6.263GiB/2.479GiB/2.487GiB
+2022-12-08 15:29:00.777,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:29:00.777,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,30.125MiB/s,30.278MiB/s,50.317GiB/5.331GiB/5.358GiB
+2022-12-08 15:29:00.784,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.143MiB/s,15.188MiB/s,6.263GiB/2.479GiB/2.487GiB
+2022-12-08 15:29:00.784,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:29:00.784,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,30.124MiB/s,30.279MiB/s,50.317GiB/5.331GiB/5.358GiB
+2022-12-08 15:29:45.070,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,29.099MiB/s,29.448MiB/s,6.263GiB/3.701GiB/3.724GiB
+2022-12-08 15:29:45.070,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:29:45.070,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.672MiB/s,29.850MiB/s,50.317GiB/6.534GiB/6.573GiB
+2022-12-08 15:29:45.072,submitted,0,T4,0.000 B,2.128 GiB,-1,36,4.937GiB,9/9,36/36,26.074MiB/s,26.423MiB/s,5.637GiB/3.075GiB/3.097GiB
+2022-12-08 15:29:45.072,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.782GiB,2/1,8/4,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:29:45.072,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.672MiB/s,29.850MiB/s,50.317GiB/6.534GiB/6.573GiB
+2022-12-08 15:29:45.086,submitted,0,T4,0.000 B,2.128 GiB,-1,36,4.937GiB,9/9,36/36,26.068MiB/s,26.423MiB/s,5.637GiB/3.075GiB/3.098GiB
+2022-12-08 15:29:45.098,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.782GiB,2/2,8/8,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:29:45.100,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.670MiB/s,29.850MiB/s,50.317GiB/6.534GiB/6.573GiB
+2022-12-08 15:29:45.140,completed,0,T4,0.000 B,2.128 GiB,-1,36,4.937GiB,9/9,36/36,26.110MiB/s,26.419MiB/s,5.637GiB/3.078GiB/3.099GiB
+2022-12-08 15:29:45.143,completed,1,T4,2.128 GiB,8.513 GiB,3,32,8.782GiB,2/2,8/8,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:29:45.143,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.676MiB/s,29.849MiB/s,50.317GiB/6.537GiB/6.575GiB
+2022-12-08 15:29:45.147,submitted,0,T4,0.000 B,2.128 GiB,-1,32,4.388GiB,8/8,32/32,23.080MiB/s,23.391MiB/s,5.011GiB/2.451GiB/2.472GiB
+2022-12-08 15:29:45.147,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.331GiB,3/3,12/12,0B/s,0B/s,3.759GiB/0B/0B
+2022-12-08 15:29:45.152,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.675MiB/s,29.849MiB/s,50.317GiB/6.537GiB/6.575GiB
+2022-12-08 15:29:45.165,submitted,0,T4,0.000 B,2.128 GiB,-1,32,4.388GiB,8/8,32/32,23.074MiB/s,23.388MiB/s,5.011GiB/2.451GiB/2.472GiB
+2022-12-08 15:29:45.166,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.331GiB,4/4,16/16,0B/s,0B/s,5.012GiB/0B/0B
+2022-12-08 15:29:45.166,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.687MiB/s,29.848MiB/s,50.317GiB/6.540GiB/6.575GiB
+2022-12-08 15:29:48.652,completed,0,T4,0.000 B,2.128 GiB,-1,32,4.388GiB,8/8,32/32,23.224MiB/s,23.344MiB/s,5.011GiB/2.541GiB/2.548GiB
+2022-12-08 15:29:48.652,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.331GiB,4/4,16/16,9.350MiB/s,11.084MiB/s,5.012GiB/32.911MiB/39.009MiB
+2022-12-08 15:29:48.652,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.715MiB/s,29.828MiB/s,50.317GiB/6.647GiB/6.672GiB
+2022-12-08 15:29:48.655,submitted,0,T4,0.000 B,2.128 GiB,-1,28,3.840GiB,7/7,28/28,20.245MiB/s,20.365MiB/s,4.384GiB/1.914GiB/1.921GiB
+2022-12-08 15:29:48.655,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.880GiB,6/5,24/20,9.341MiB/s,11.080MiB/s,7.518GiB/32.911MiB/39.031MiB
+2022-12-08 15:29:48.655,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.714MiB/s,29.828MiB/s,50.317GiB/6.647GiB/6.673GiB
+2022-12-08 15:29:48.729,submitted,0,T4,0.000 B,2.128 GiB,-1,28,3.840GiB,7/7,28/28,20.221MiB/s,20.400MiB/s,4.384GiB/1.914GiB/1.925GiB
+2022-12-08 15:29:48.729,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.880GiB,6/6,24/24,9.150MiB/s,11.350MiB/s,7.518GiB/32.911MiB/40.836MiB
+2022-12-08 15:29:48.729,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.719MiB/s,29.838MiB/s,50.317GiB/6.650GiB/6.677GiB
+2022-12-08 15:29:49.812,completed,0,T4,0.000 B,2.128 GiB,-1,28,3.840GiB,7/7,28/28,20.263MiB/s,20.447MiB/s,4.384GiB/1.938GiB/1.947GiB
+2022-12-08 15:29:49.814,completed,1,T4,2.128 GiB,8.513 GiB,3,36,9.880GiB,6/6,24/24,12.555MiB/s,17.853MiB/s,7.518GiB/49.660MiB/61.293MiB
+2022-12-08 15:29:49.815,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.726MiB/s,29.856MiB/s,50.317GiB/6.683GiB/6.713GiB
+2022-12-08 15:29:49.820,submitted,0,T4,0.000 B,2.128 GiB,-1,24,3.291GiB,6/6,24/24,17.300MiB/s,17.486MiB/s,3.758GiB/1.311GiB/1.321GiB
+2022-12-08 15:29:49.821,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.429GiB,7/7,28/28,12.520MiB/s,17.823MiB/s,8.772GiB/49.660MiB/61.367MiB
+2022-12-08 15:29:49.825,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.725MiB/s,29.856MiB/s,50.317GiB/6.683GiB/6.713GiB
+2022-12-08 15:29:49.838,submitted,0,T4,0.000 B,2.128 GiB,-1,24,3.291GiB,6/6,24/24,17.295MiB/s,17.485MiB/s,3.758GiB/1.311GiB/1.321GiB
+2022-12-08 15:29:49.839,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.429GiB,8/8,32/32,15.755MiB/s,17.839MiB/s,10.025GiB/53.028MiB/61.671MiB
+2022-12-08 15:29:49.839,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.723MiB/s,29.855MiB/s,50.317GiB/6.683GiB/6.713GiB
+2022-12-08 15:29:51.519,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,17.369MiB/s,17.484MiB/s,3.758GiB/1.343GiB/1.349GiB
+2022-12-08 15:29:51.520,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.429GiB,8/8,32/32,18.385MiB/s,22.821MiB/s,10.025GiB/87.143MiB/99.085MiB
+2022-12-08 15:29:51.520,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.750MiB/s,29.843MiB/s,50.317GiB/6.738GiB/6.759GiB
+2022-12-08 15:29:51.521,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,14.432MiB/s,14.546MiB/s,3.132GiB/733.785MiB/739.590MiB
+2022-12-08 15:29:51.521,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.977GiB,9/9,36/36,18.372MiB/s,22.806MiB/s,11.278GiB/87.143MiB/99.092MiB
+2022-12-08 15:29:51.521,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.750MiB/s,29.843MiB/s,50.317GiB/6.738GiB/6.759GiB
+2022-12-08 15:29:51.537,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,14.427MiB/s,14.546MiB/s,3.132GiB/733.785MiB/739.813MiB
+2022-12-08 15:29:51.538,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.977GiB,10/10,40/40,18.292MiB/s,22.805MiB/s,12.530GiB/87.143MiB/99.439MiB
+2022-12-08 15:29:51.538,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,29.748MiB/s,29.842MiB/s,50.317GiB/6.738GiB/6.760GiB
+2022-12-08 15:32:02.792,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/6,32/24,13.659MiB/s,13.679MiB/s,5.011GiB/2.429GiB/2.433GiB
+2022-12-08 15:32:02.793,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.977GiB,10/10,40/40,26.882MiB/s,27.100MiB/s,12.530GiB/3.537GiB/3.565GiB
+2022-12-08 15:32:02.793,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.726MiB/s,28.814MiB/s,50.317GiB/10.189GiB/10.220GiB
+2022-12-08 15:32:02.812,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,28/28,13.658MiB/s,13.679MiB/s,4.384GiB/2.429GiB/2.433GiB
+2022-12-08 15:32:02.814,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.977GiB,10/10,40/40,26.879MiB/s,27.099MiB/s,12.530GiB/3.537GiB/3.565GiB
+2022-12-08 15:32:02.814,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.724MiB/s,28.814MiB/s,50.317GiB/10.189GiB/10.221GiB
+2022-12-08 15:32:02.827,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/8,40/32,13.657MiB/s,13.679MiB/s,6.264GiB/2.429GiB/2.433GiB
+2022-12-08 15:32:02.832,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.977GiB,10/10,40/40,26.876MiB/s,27.099MiB/s,12.530GiB/3.537GiB/3.566GiB
+2022-12-08 15:32:02.832,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.723MiB/s,28.814MiB/s,50.317GiB/10.189GiB/10.221GiB
+2022-12-08 15:32:02.849,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,13.655MiB/s,13.679MiB/s,6.264GiB/2.429GiB/2.434GiB
+2022-12-08 15:32:02.849,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.977GiB,10/10,40/40,26.887MiB/s,27.099MiB/s,12.530GiB/3.539GiB/3.566GiB
+2022-12-08 15:32:02.850,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.721MiB/s,28.814MiB/s,50.317GiB/10.189GiB/10.222GiB
+2022-12-08 15:32:02.864,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,13.654MiB/s,13.679MiB/s,6.264GiB/2.429GiB/2.434GiB
+2022-12-08 15:32:02.864,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.977GiB,10/10,40/40,26.884MiB/s,27.098MiB/s,12.530GiB/3.539GiB/3.567GiB
+2022-12-08 15:32:02.864,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.720MiB/s,28.813MiB/s,50.317GiB/10.189GiB/10.222GiB
+2022-12-08 15:32:52.818,completed,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,27.839MiB/s,28.031MiB/s,6.264GiB/3.789GiB/3.803GiB
+2022-12-08 15:32:52.820,completed,1,T4,2.128 GiB,8.513 GiB,4,40,10.977GiB,10/10,40/40,27.031MiB/s,27.236MiB/s,12.530GiB/4.877GiB/4.914GiB
+2022-12-08 15:32:52.820,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.627MiB/s,28.722MiB/s,50.317GiB/11.552GiB/11.591GiB
+2022-12-08 15:32:54.444,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,25.112MiB/s,25.211MiB/s,5.637GiB/3.203GiB/3.211GiB
+2022-12-08 15:32:54.446,completed,1,T4,2.128 GiB,8.513 GiB,1,42,11.526GiB,10/10,40/40,27.075MiB/s,27.236MiB/s,12.530GiB/4.928GiB/4.957GiB
+2022-12-08 15:32:54.446,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.631MiB/s,28.726MiB/s,50.317GiB/11.600GiB/11.638GiB
+2022-12-08 15:32:55.705,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,22.248MiB/s,22.402MiB/s,5.011GiB/2.595GiB/2.607GiB
+2022-12-08 15:32:55.705,completed,1,T4,2.128 GiB,8.513 GiB,1,44,12.075GiB,10/10,40/40,27.048MiB/s,27.223MiB/s,12.530GiB/4.956GiB/4.988GiB
+2022-12-08 15:32:55.706,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.619MiB/s,28.714MiB/s,50.317GiB/11.630GiB/11.668GiB
+2022-12-08 15:32:57.264,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.568MiB/s,19.688MiB/s,4.385GiB/2.004GiB/2.011GiB
+2022-12-08 15:32:57.275,completed,1,T4,2.128 GiB,8.513 GiB,1,46,12.623GiB,10/10,40/40,27.066MiB/s,27.240MiB/s,12.530GiB/5.000GiB/5.032GiB
+2022-12-08 15:32:57.276,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.643MiB/s,28.721MiB/s,50.317GiB/11.683GiB/11.715GiB
+2022-12-08 15:32:58.307,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,16.853MiB/s,17.042MiB/s,3.758GiB/1.393GiB/1.403GiB
+2022-12-08 15:32:58.307,completed,1,T4,2.128 GiB,8.513 GiB,1,48,13.173GiB,10/10,40/40,27.119MiB/s,27.274MiB/s,12.530GiB/5.038GiB/5.067GiB
+2022-12-08 15:32:58.307,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.645MiB/s,28.725MiB/s,50.317GiB/11.713GiB/11.746GiB
+2022-12-08 15:35:06.071,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/6,32/24,13.879MiB/s,13.939MiB/s,5.011GiB/2.483GiB/2.494GiB
+2022-12-08 15:35:06.071,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,27.123MiB/s,27.287MiB/s,12.530GiB/8.423GiB/8.474GiB
+2022-12-08 15:35:06.071,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.272MiB/s,28.356MiB/s,50.317GiB/15.088GiB/15.133GiB
+2022-12-08 15:35:06.095,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/7,32/28,13.877MiB/s,13.940MiB/s,5.011GiB/2.483GiB/2.495GiB
+2022-12-08 15:35:06.095,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,27.121MiB/s,27.287MiB/s,12.530GiB/8.423GiB/8.475GiB
+2022-12-08 15:35:06.095,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.271MiB/s,28.356MiB/s,50.317GiB/15.088GiB/15.134GiB
+2022-12-08 15:35:06.108,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/8,40/32,13.876MiB/s,13.939MiB/s,6.264GiB/2.483GiB/2.495GiB
+2022-12-08 15:35:06.108,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,27.119MiB/s,27.287MiB/s,12.530GiB/8.423GiB/8.475GiB
+2022-12-08 15:35:06.108,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.270MiB/s,28.356MiB/s,50.317GiB/15.088GiB/15.134GiB
+2022-12-08 15:35:06.122,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,13.875MiB/s,13.939MiB/s,6.264GiB/2.483GiB/2.495GiB
+2022-12-08 15:35:06.130,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,27.118MiB/s,27.287MiB/s,12.530GiB/8.423GiB/8.475GiB
+2022-12-08 15:35:06.130,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.269MiB/s,28.356MiB/s,50.317GiB/15.088GiB/15.135GiB
+2022-12-08 15:35:06.136,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,13.874MiB/s,13.938MiB/s,6.264GiB/2.483GiB/2.495GiB
+2022-12-08 15:35:06.136,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,27.117MiB/s,27.287MiB/s,12.530GiB/8.423GiB/8.476GiB
+2022-12-08 15:35:06.136,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.269MiB/s,28.356MiB/s,50.317GiB/15.088GiB/15.135GiB
+2022-12-08 15:35:52.733,completed,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,27.256MiB/s,27.449MiB/s,6.264GiB/3.709GiB/3.722GiB
+2022-12-08 15:35:52.733,completed,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,27.040MiB/s,27.183MiB/s,12.530GiB/9.629GiB/9.680GiB
+2022-12-08 15:35:52.733,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.153MiB/s,28.222MiB/s,50.317GiB/16.308GiB/16.348GiB
+2022-12-08 15:35:54.981,completed,0,T4,0.000 B,2.128 GiB,-1,36,4.937GiB,9/9,36/36,24.363MiB/s,24.554MiB/s,5.637GiB/3.127GiB/3.139GiB
+2022-12-08 15:35:54.981,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.270GiB,10/10,40/40,26.998MiB/s,27.153MiB/s,12.530GiB/9.674GiB/9.729GiB
+2022-12-08 15:35:54.981,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.122MiB/s,28.199MiB/s,50.317GiB/16.352GiB/16.396GiB
+2022-12-08 15:35:55.612,completed,0,T4,0.000 B,2.128 GiB,-1,32,4.389GiB,8/8,32/32,21.649MiB/s,21.779MiB/s,5.011GiB/2.516GiB/2.524GiB
+2022-12-08 15:35:55.612,completed,1,T4,2.128 GiB,8.513 GiB,2,54,14.819GiB,10/10,40/40,27.015MiB/s,27.151MiB/s,12.530GiB/9.696GiB/9.745GiB
+2022-12-08 15:35:55.613,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.125MiB/s,28.196MiB/s,50.317GiB/16.371GiB/16.412GiB
+2022-12-08 15:35:56.512,completed,0,T4,0.000 B,2.128 GiB,-1,28,3.840GiB,7/7,28/28,18.859MiB/s,18.967MiB/s,4.385GiB/1.903GiB/1.909GiB
+2022-12-08 15:35:56.514,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.368GiB,10/10,40/40,26.982MiB/s,27.140MiB/s,12.530GiB/9.708GiB/9.765GiB
+2022-12-08 15:35:56.514,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.100MiB/s,28.190MiB/s,50.317GiB/16.381GiB/16.433GiB
+2022-12-08 15:36:01.446,completed,0,T4,0.000 B,2.128 GiB,1,29,3.977GiB,6/6,24/24,16.015MiB/s,16.080MiB/s,3.758GiB/1.347GiB/1.350GiB
+2022-12-08 15:36:01.447,completed,1,T4,2.128 GiB,8.513 GiB,2,58,15.917GiB,10/10,40/40,26.948MiB/s,27.091MiB/s,12.530GiB/9.826GiB/9.878GiB
+2022-12-08 15:36:01.447,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,28.069MiB/s,28.144MiB/s,50.317GiB/16.498GiB/16.542GiB
+2022-12-08 15:37:31.295,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.541MiB/s,13.602MiB/s,3.132GiB/1.920GiB/1.928GiB
+2022-12-08 15:37:31.301,completed,1,T4,2.128 GiB,8.513 GiB,2,60,16.466GiB,10/10,40/40,26.991MiB/s,27.123MiB/s,12.530GiB/12.210GiB/12.270GiB
+2022-12-08 15:37:31.302,completed,2,T4,8.513 GiB,34.052 GiB,4,40,44.051GiB,10/10,40/40,27.950MiB/s,28.022MiB/s,50.317GiB/18.880GiB/18.929GiB
+2022-12-08 15:37:34.864,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.515MiB/s,13.591MiB/s,3.132GiB/1.963GiB/1.974GiB
+2022-12-08 15:37:34.864,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.368GiB,9/9,36/36,24.217MiB/s,24.347MiB/s,11.277GiB/11.032GiB/11.091GiB
+2022-12-08 15:37:34.864,completed,2,T4,8.513 GiB,34.052 GiB,1,41,45.153GiB,10/10,40/40,27.938MiB/s,28.019MiB/s,50.317GiB/18.969GiB/19.024GiB
+2022-12-08 15:37:40.020,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.545MiB/s,13.644MiB/s,3.132GiB/2.036GiB/2.051GiB
+2022-12-08 15:37:40.020,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.271GiB,8/8,32/32,21.506MiB/s,21.636MiB/s,10.024GiB/9.896GiB/9.956GiB
+2022-12-08 15:37:40.022,completed,2,T4,8.513 GiB,34.052 GiB,1,42,46.254GiB,10/10,40/40,27.942MiB/s,28.034MiB/s,50.317GiB/19.113GiB/19.176GiB
+2022-12-08 15:37:41.683,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.674MiB/s,13.725MiB/s,3.132GiB/2.077GiB/2.085GiB
+2022-12-08 15:37:41.687,completed,1,T4,2.128 GiB,8.513 GiB,2,48,13.173GiB,7/7,28/28,18.840MiB/s,18.941MiB/s,8.771GiB/8.690GiB/8.737GiB
+2022-12-08 15:37:41.690,completed,2,T4,8.513 GiB,34.052 GiB,1,43,47.355GiB,10/10,40/40,27.977MiB/s,28.060MiB/s,50.317GiB/19.182GiB/19.239GiB
+2022-12-08 15:37:42.869,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.679MiB/s,13.794MiB/s,3.132GiB/2.094GiB/2.112GiB
+2022-12-08 15:37:42.869,completed,1,T4,2.128 GiB,8.513 GiB,2,44,12.075GiB,6/6,24/24,16.155MiB/s,16.254MiB/s,7.519GiB/7.459GiB/7.505GiB
+2022-12-08 15:37:42.870,completed,2,T4,8.513 GiB,34.052 GiB,1,44,48.457GiB,10/10,40/40,27.985MiB/s,28.079MiB/s,50.317GiB/19.221GiB/19.285GiB
+2022-12-08 15:37:44.062,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.710MiB/s,13.772MiB/s,3.132GiB/2.115GiB/2.124GiB
+2022-12-08 15:37:44.063,completed,1,T4,2.128 GiB,8.513 GiB,2,40,10.978GiB,5/5,20/20,13.458MiB/s,13.528MiB/s,6.265GiB/6.226GiB/6.259GiB
+2022-12-08 15:37:44.063,completed,2,T4,8.513 GiB,34.052 GiB,1,45,49.559GiB,10/10,40/40,27.989MiB/s,28.073MiB/s,50.317GiB/19.256GiB/19.313GiB
+2022-12-08 15:37:44.323,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.716MiB/s,13.772MiB/s,3.132GiB/2.119GiB/2.128GiB
+2022-12-08 15:37:44.323,completed,1,T4,2.128 GiB,8.513 GiB,2,36,9.880GiB,4/4,16/16,10.752MiB/s,10.809MiB/s,5.012GiB/4.976GiB/5.003GiB
+2022-12-08 15:37:44.323,completed,2,T4,8.513 GiB,34.052 GiB,1,46,50.661GiB,10/10,40/40,27.987MiB/s,28.073MiB/s,50.317GiB/19.262GiB/19.321GiB
+2022-12-08 15:37:45.588,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.764MiB/s,13.791MiB/s,3.132GiB/2.143GiB/2.148GiB
+2022-12-08 15:37:45.588,completed,1,T4,2.128 GiB,8.513 GiB,2,32,8.782GiB,3/3,12/12,8.061MiB/s,8.098MiB/s,3.759GiB/3.736GiB/3.753GiB
+2022-12-08 15:37:45.588,completed,2,T4,8.513 GiB,34.052 GiB,1,47,51.762GiB,10/10,40/40,28.005MiB/s,28.075MiB/s,50.317GiB/19.309GiB/19.357GiB
+2022-12-08 15:37:48.477,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.790MiB/s,13.818MiB/s,3.132GiB/2.186GiB/2.191GiB
+2022-12-08 15:37:48.477,completed,1,T4,2.128 GiB,8.513 GiB,2,28,7.684GiB,2/2,8/8,5.366MiB/s,5.390MiB/s,2.505GiB/2.499GiB/2.510GiB
+2022-12-08 15:37:48.477,completed,2,T4,8.513 GiB,34.052 GiB,1,48,52.864GiB,10/10,40/40,28.028MiB/s,28.092MiB/s,50.317GiB/19.404GiB/19.448GiB
+2022-12-08 15:37:50.424,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,20/20,13.815MiB/s,13.869MiB/s,3.132GiB/2.217GiB/2.225GiB
+2022-12-08 15:37:50.427,completed,1,T4,2.128 GiB,8.513 GiB,2,24,6.586GiB,1/1,4/4,2.678MiB/s,2.689MiB/s,1.252GiB/1.252GiB/1.257GiB
+2022-12-08 15:37:50.427,completed,2,T4,8.513 GiB,34.052 GiB,1,49,53.966GiB,10/10,40/40,28.033MiB/s,28.104MiB/s,50.317GiB/19.460GiB/19.509GiB
+2022-12-08 15:38:16.342,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,28/24,14.006MiB/s,14.063MiB/s,4.385GiB/2.602GiB/2.612GiB
+2022-12-08 15:38:16.346,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:16.349,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.115MiB/s,28.202MiB/s,50.317GiB/20.228GiB/20.291GiB
+2022-12-08 15:38:16.360,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/7,32/28,14.005MiB/s,14.063MiB/s,5.011GiB/2.602GiB/2.613GiB
+2022-12-08 15:38:16.360,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:16.361,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.114MiB/s,28.202MiB/s,50.317GiB/20.228GiB/20.292GiB
+2022-12-08 15:38:16.365,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,14.005MiB/s,14.063MiB/s,5.011GiB/2.602GiB/2.613GiB
+2022-12-08 15:38:16.365,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:16.373,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.114MiB/s,28.202MiB/s,50.317GiB/20.228GiB/20.292GiB
+2022-12-08 15:38:16.387,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,14.003MiB/s,14.062MiB/s,6.264GiB/2.602GiB/2.613GiB
+2022-12-08 15:38:16.388,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:16.390,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.113MiB/s,28.202MiB/s,50.317GiB/20.228GiB/20.292GiB
+2022-12-08 15:38:16.399,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,14.002MiB/s,14.062MiB/s,6.264GiB/2.602GiB/2.613GiB
+2022-12-08 15:38:16.403,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:16.403,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.113MiB/s,28.202MiB/s,50.317GiB/20.228GiB/20.293GiB
+2022-12-08 15:38:55.135,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,27.473MiB/s,27.597MiB/s,6.264GiB/3.619GiB/3.626GiB
+2022-12-08 15:38:55.135,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:55.135,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.045MiB/s,28.109MiB/s,50.317GiB/21.240GiB/21.289GiB
+2022-12-08 15:38:56.140,completed,0,T4,0.000 B,2.128 GiB,-1,36,4.937GiB,9/9,36/36,24.663MiB/s,24.806MiB/s,5.637GiB/3.015GiB/3.023GiB
+2022-12-08 15:38:56.140,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.038GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:56.140,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.041MiB/s,28.109MiB/s,50.317GiB/21.265GiB/21.317GiB
+2022-12-08 15:38:57.823,completed,0,T4,0.000 B,2.128 GiB,-1,32,4.389GiB,8/8,32/32,21.725MiB/s,21.944MiB/s,5.011GiB/2.415GiB/2.428GiB
+2022-12-08 15:38:57.823,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.586GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:57.824,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.040MiB/s,28.106MiB/s,50.317GiB/21.310GiB/21.361GiB
+2022-12-08 15:38:59.150,completed,0,T4,0.000 B,2.128 GiB,-1,28,3.840GiB,7/7,28/28,19.049MiB/s,19.132MiB/s,4.385GiB/1.818GiB/1.822GiB
+2022-12-08 15:38:59.150,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.135GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:59.150,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.035MiB/s,28.100MiB/s,50.317GiB/21.343GiB/21.393GiB
+2022-12-08 15:38:59.367,completed,0,T4,0.000 B,2.128 GiB,-1,24,3.292GiB,6/6,24/24,16.224MiB/s,16.362MiB/s,3.758GiB/1.192GiB/1.198GiB
+2022-12-08 15:38:59.367,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.684GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:38:59.367,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.027MiB/s,28.099MiB/s,50.317GiB/21.343GiB/21.397GiB
+2022-12-08 15:41:03.948,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,6/6,24/24,14.956MiB/s,15.024MiB/s,3.758GiB/2.447GiB/2.458GiB
+2022-12-08 15:41:03.948,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:41:03.948,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.419MiB/s,28.501MiB/s,50.317GiB/25.098GiB/25.172GiB
+2022-12-08 15:41:03.968,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,9/7,36/28,14.954MiB/s,15.023MiB/s,5.638GiB/2.447GiB/2.459GiB
+2022-12-08 15:41:03.968,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:41:03.968,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.418MiB/s,28.502MiB/s,50.317GiB/25.098GiB/25.172GiB
+2022-12-08 15:41:03.995,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,8/8,32/32,14.952MiB/s,15.022MiB/s,5.012GiB/2.447GiB/2.459GiB
+2022-12-08 15:41:03.999,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:41:03.999,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.417MiB/s,28.502MiB/s,50.317GiB/25.098GiB/25.173GiB
+2022-12-08 15:41:04.012,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/9,40/36,14.950MiB/s,15.022MiB/s,6.264GiB/2.447GiB/2.459GiB
+2022-12-08 15:41:04.012,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:41:04.012,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.417MiB/s,28.502MiB/s,50.317GiB/25.098GiB/25.174GiB
+2022-12-08 15:41:04.024,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/10,40/40,14.949MiB/s,15.022MiB/s,6.264GiB/2.447GiB/2.459GiB
+2022-12-08 15:41:04.025,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:41:04.025,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.416MiB/s,28.502MiB/s,50.317GiB/25.098GiB/25.174GiB
+2022-12-08 15:41:48.606,completed,0,T4,0.000 B,2.128 GiB,1,45,6.173GiB,10/10,40/40,28.970MiB/s,29.366MiB/s,6.264GiB/3.676GiB/3.697GiB
+2022-12-08 15:41:48.610,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:41:48.610,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.404MiB/s,28.478MiB/s,50.317GiB/26.325GiB/26.393GiB
+2022-12-08 15:41:48.613,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,25.946MiB/s,26.343MiB/s,5.638GiB/3.050GiB/3.071GiB
+2022-12-08 15:41:48.615,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.782GiB,2/1,8/4,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:41:48.616,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.404MiB/s,28.478MiB/s,50.317GiB/26.325GiB/26.393GiB
+2022-12-08 15:41:48.630,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,25.940MiB/s,26.346MiB/s,5.638GiB/3.050GiB/3.072GiB
+2022-12-08 15:41:48.631,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.782GiB,2/2,8/8,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:41:48.631,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.403MiB/s,28.478MiB/s,50.317GiB/26.325GiB/26.394GiB
+2022-12-08 15:41:54.270,completed,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,26.278MiB/s,26.439MiB/s,5.638GiB/3.211GiB/3.222GiB
+2022-12-08 15:41:54.270,completed,1,T4,2.128 GiB,8.513 GiB,3,32,8.782GiB,2/2,8/8,5.931MiB/s,6.552MiB/s,2.506GiB/33.390MiB/36.884MiB
+2022-12-08 15:41:54.270,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.418MiB/s,28.491MiB/s,50.317GiB/26.494GiB/26.563GiB
+2022-12-08 15:41:54.272,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.076GiB,8/8,32/32,23.331MiB/s,23.494MiB/s,5.012GiB/2.585GiB/2.596GiB
+2022-12-08 15:41:54.272,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.331GiB,3/3,12/12,5.929MiB/s,6.551MiB/s,3.759GiB/33.390MiB/36.886MiB
+2022-12-08 15:41:54.274,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.418MiB/s,28.491MiB/s,50.317GiB/26.494GiB/26.563GiB
+2022-12-08 15:41:54.279,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.076GiB,8/8,32/32,23.329MiB/s,23.493MiB/s,5.012GiB/2.585GiB/2.596GiB
+2022-12-08 15:41:54.286,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.331GiB,4/4,16/16,5.922MiB/s,6.544MiB/s,5.012GiB/33.390MiB/36.899MiB
+2022-12-08 15:41:54.287,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.417MiB/s,28.491MiB/s,50.317GiB/26.494GiB/26.563GiB
+2022-12-08 15:41:55.100,completed,0,T4,0.000 B,2.128 GiB,1,37,5.076GiB,8/8,32/32,23.299MiB/s,23.363MiB/s,5.012GiB/2.600GiB/2.605GiB
+2022-12-08 15:41:55.107,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.331GiB,4/4,16/16,5.883MiB/s,8.440MiB/s,5.012GiB/37.998MiB/42.652MiB
+2022-12-08 15:41:55.107,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.414MiB/s,28.488MiB/s,50.317GiB/26.514GiB/26.583GiB
+2022-12-08 15:41:55.109,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.527GiB,7/7,28/28,20.361MiB/s,20.429MiB/s,4.385GiB/1.973GiB/1.978GiB
+2022-12-08 15:41:55.110,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.880GiB,6/5,24/20,5.875MiB/s,8.460MiB/s,7.519GiB/37.998MiB/42.755MiB
+2022-12-08 15:41:55.111,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.414MiB/s,28.488MiB/s,50.317GiB/26.514GiB/26.583GiB
+2022-12-08 15:41:55.123,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.527GiB,7/7,28/28,20.357MiB/s,20.428MiB/s,4.385GiB/1.973GiB/1.978GiB
+2022-12-08 15:41:55.124,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.880GiB,6/6,24/24,5.862MiB/s,8.524MiB/s,7.519GiB/37.998MiB/42.882MiB
+2022-12-08 15:41:55.124,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.413MiB/s,28.488MiB/s,50.317GiB/26.514GiB/26.583GiB
+2022-12-08 15:41:55.623,completed,0,T4,0.000 B,2.128 GiB,1,33,4.527GiB,7/7,28/28,20.203MiB/s,20.394MiB/s,4.385GiB/1.973GiB/1.985GiB
+2022-12-08 15:41:55.634,completed,1,T4,2.128 GiB,8.513 GiB,3,36,9.880GiB,6/6,24/24,7.437MiB/s,12.856MiB/s,7.519GiB/40.657MiB/49.714MiB
+2022-12-08 15:41:55.634,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.404MiB/s,28.486MiB/s,50.317GiB/26.519GiB/26.596GiB
+2022-12-08 15:41:55.636,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,17.274MiB/s,17.468MiB/s,3.759GiB/1.347GiB/1.358GiB
+2022-12-08 15:41:55.636,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.429GiB,7/7,28/28,7.409MiB/s,13.059MiB/s,8.772GiB/40.657MiB/50.005MiB
+2022-12-08 15:41:55.636,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.404MiB/s,28.486MiB/s,50.317GiB/26.519GiB/26.596GiB
+2022-12-08 15:41:55.658,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,17.267MiB/s,17.465MiB/s,3.759GiB/1.347GiB/1.359GiB
+2022-12-08 15:41:55.660,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.429GiB,8/8,32/32,7.360MiB/s,13.206MiB/s,10.025GiB/40.657MiB/50.363MiB
+2022-12-08 15:41:55.660,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.403MiB/s,28.486MiB/s,50.317GiB/26.519GiB/26.597GiB
+2022-12-08 15:41:57.164,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,17.463MiB/s,17.529MiB/s,3.759GiB/1.382GiB/1.386GiB
+2022-12-08 15:41:57.164,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.429GiB,8/8,32/32,18.593MiB/s,23.522MiB/s,10.025GiB/78.184MiB/90.960MiB
+2022-12-08 15:41:57.164,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.421MiB/s,28.492MiB/s,50.317GiB/26.577GiB/26.644GiB
+2022-12-08 15:41:57.165,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,14.558MiB/s,14.623MiB/s,3.132GiB/773.941MiB/777.430MiB
+2022-12-08 15:41:57.165,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.978GiB,9/9,36/36,18.583MiB/s,23.508MiB/s,11.278GiB/78.184MiB/90.960MiB
+2022-12-08 15:41:57.165,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.421MiB/s,28.492MiB/s,50.317GiB/26.577GiB/26.644GiB
+2022-12-08 15:41:57.179,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,14.554MiB/s,14.623MiB/s,3.132GiB/773.941MiB/777.589MiB
+2022-12-08 15:41:57.180,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.978GiB,10/10,40/40,18.487MiB/s,23.505MiB/s,12.531GiB/78.184MiB/91.264MiB
+2022-12-08 15:41:57.180,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.420MiB/s,28.492MiB/s,50.317GiB/26.577GiB/26.645GiB
+2022-12-08 15:43:53.596,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,6/6,24/24,13.837MiB/s,13.884MiB/s,3.759GiB/2.292GiB/2.299GiB
+2022-12-08 15:43:53.596,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,27.133MiB/s,27.389MiB/s,12.531GiB/3.165GiB/3.195GiB
+2022-12-08 15:43:53.596,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.264MiB/s,28.336MiB/s,50.317GiB/29.644GiB/29.720GiB
+2022-12-08 15:43:53.627,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,7/7,28/28,13.834MiB/s,13.883MiB/s,4.386GiB/2.292GiB/2.300GiB
+2022-12-08 15:43:53.627,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,27.126MiB/s,27.387MiB/s,12.531GiB/3.165GiB/3.196GiB
+2022-12-08 15:43:53.627,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.263MiB/s,28.336MiB/s,50.317GiB/29.644GiB/29.721GiB
+2022-12-08 15:43:53.645,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,9/8,36/32,13.833MiB/s,13.883MiB/s,5.638GiB/2.292GiB/2.300GiB
+2022-12-08 15:43:53.646,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,27.122MiB/s,27.386MiB/s,12.531GiB/3.165GiB/3.196GiB
+2022-12-08 15:43:53.646,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.262MiB/s,28.336MiB/s,50.317GiB/29.644GiB/29.722GiB
+2022-12-08 15:43:53.664,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/9,40/36,13.831MiB/s,13.882MiB/s,6.265GiB/2.292GiB/2.300GiB
+2022-12-08 15:43:53.665,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,27.118MiB/s,27.385MiB/s,12.531GiB/3.165GiB/3.196GiB
+2022-12-08 15:43:53.665,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.262MiB/s,28.336MiB/s,50.317GiB/29.644GiB/29.722GiB
+2022-12-08 15:43:53.685,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/10,40/40,13.829MiB/s,13.883MiB/s,6.265GiB/2.292GiB/2.300GiB
+2022-12-08 15:43:53.686,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,27.113MiB/s,27.384MiB/s,12.531GiB/3.165GiB/3.197GiB
+2022-12-08 15:43:53.686,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.261MiB/s,28.336MiB/s,50.317GiB/29.644GiB/29.722GiB
+2022-12-08 15:44:54.536,completed,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/10,40/40,28.309MiB/s,28.358MiB/s,6.265GiB/3.978GiB/3.982GiB
+2022-12-08 15:44:54.550,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.978GiB,10/10,40/40,27.110MiB/s,27.252MiB/s,12.531GiB/4.776GiB/4.801GiB
+2022-12-08 15:44:54.550,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.229MiB/s,28.290MiB/s,50.317GiB/31.288GiB/31.356GiB
+2022-12-08 15:44:55.387,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,25.344MiB/s,25.601MiB/s,5.638GiB/3.357GiB/3.377GiB
+2022-12-08 15:44:55.387,completed,1,T4,2.128 GiB,8.513 GiB,1,42,11.527GiB,10/10,40/40,27.110MiB/s,27.261MiB/s,12.531GiB/4.798GiB/4.825GiB
+2022-12-08 15:44:55.387,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.224MiB/s,28.294MiB/s,50.317GiB/31.306GiB/31.384GiB
+2022-12-08 15:44:56.403,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,22.630MiB/s,22.782MiB/s,5.011GiB/2.756GiB/2.768GiB
+2022-12-08 15:44:56.403,completed,1,T4,2.128 GiB,8.513 GiB,1,44,12.076GiB,10/10,40/40,27.138MiB/s,27.274MiB/s,12.531GiB/4.830GiB/4.855GiB
+2022-12-08 15:44:56.403,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.227MiB/s,28.295MiB/s,50.317GiB/31.338GiB/31.412GiB
+2022-12-08 15:44:56.961,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.876MiB/s,19.999MiB/s,4.385GiB/2.141GiB/2.149GiB
+2022-12-08 15:44:56.962,completed,1,T4,2.128 GiB,8.513 GiB,1,46,12.625GiB,10/10,40/40,27.071MiB/s,27.273MiB/s,12.531GiB/4.833GiB/4.869GiB
+2022-12-08 15:44:56.962,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.223MiB/s,28.293MiB/s,50.317GiB/31.348GiB/31.426GiB
+2022-12-08 15:44:57.254,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,17.054MiB/s,17.254MiB/s,3.759GiB/1.515GiB/1.527GiB
+2022-12-08 15:44:57.254,completed,1,T4,2.128 GiB,8.513 GiB,1,48,13.174GiB,10/10,40/40,27.028MiB/s,27.292MiB/s,12.531GiB/4.833GiB/4.880GiB
+2022-12-08 15:44:57.255,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.216MiB/s,28.295MiB/s,50.317GiB/31.348GiB/31.436GiB
+2022-12-08 15:47:07.836,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,6/6,24/24,13.919MiB/s,13.939MiB/s,3.759GiB/2.639GiB/2.643GiB
+2022-12-08 15:47:07.837,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,26.940MiB/s,27.083MiB/s,12.531GiB/8.252GiB/8.296GiB
+2022-12-08 15:47:07.837,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.115MiB/s,28.180MiB/s,50.317GiB/34.822GiB/34.901GiB
+2022-12-08 15:47:07.848,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,7/7,28/28,13.929MiB/s,13.939MiB/s,4.386GiB/2.641GiB/2.643GiB
+2022-12-08 15:47:07.848,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,26.939MiB/s,27.084MiB/s,12.531GiB/8.252GiB/8.297GiB
+2022-12-08 15:47:07.848,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.115MiB/s,28.180MiB/s,50.317GiB/34.822GiB/34.902GiB
+2022-12-08 15:47:07.851,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,8/8,32/32,13.929MiB/s,13.939MiB/s,5.012GiB/2.641GiB/2.643GiB
+2022-12-08 15:47:07.851,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,26.939MiB/s,27.084MiB/s,12.531GiB/8.252GiB/8.297GiB
+2022-12-08 15:47:07.851,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.115MiB/s,28.180MiB/s,50.317GiB/34.822GiB/34.902GiB
+2022-12-08 15:47:07.858,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,9/9,36/36,13.929MiB/s,13.939MiB/s,5.638GiB/2.641GiB/2.643GiB
+2022-12-08 15:47:07.858,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,26.938MiB/s,27.085MiB/s,12.531GiB/8.252GiB/8.297GiB
+2022-12-08 15:47:07.858,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.115MiB/s,28.180MiB/s,50.317GiB/34.822GiB/34.902GiB
+2022-12-08 15:47:07.867,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/10,40/40,13.928MiB/s,13.940MiB/s,6.265GiB/2.641GiB/2.644GiB
+2022-12-08 15:47:07.870,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,26.938MiB/s,27.085MiB/s,12.531GiB/8.252GiB/8.298GiB
+2022-12-08 15:47:07.871,submitted,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.115MiB/s,28.180MiB/s,50.317GiB/34.822GiB/34.903GiB
+2022-12-08 15:47:37.174,completed,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/10,40/40,29.213MiB/s,29.595MiB/s,6.265GiB/3.498GiB/3.517GiB
+2022-12-08 15:47:37.174,completed,1,T4,2.128 GiB,8.513 GiB,1,50,13.722GiB,10/10,40/40,27.044MiB/s,27.227MiB/s,12.531GiB/9.059GiB/9.120GiB
+2022-12-08 15:47:37.175,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.126MiB/s,28.202MiB/s,50.317GiB/35.640GiB/35.737GiB
+2022-12-08 15:47:40.876,completed,0,T4,0.000 B,2.128 GiB,-1,36,4.938GiB,9/9,36/36,26.482MiB/s,26.696MiB/s,5.638GiB/2.979GiB/2.989GiB
+2022-12-08 15:47:40.876,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.272GiB,10/10,40/40,27.130MiB/s,27.276MiB/s,12.531GiB/9.186GiB/9.235GiB
+2022-12-08 15:47:40.876,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.143MiB/s,28.209MiB/s,50.317GiB/35.765GiB/35.848GiB
+2022-12-08 15:47:42.203,completed,0,T4,0.000 B,2.128 GiB,-1,32,4.389GiB,8/8,32/32,23.498MiB/s,23.697MiB/s,5.012GiB/2.377GiB/2.385GiB
+2022-12-08 15:47:42.203,completed,1,T4,2.128 GiB,8.513 GiB,2,54,14.821GiB,10/10,40/40,27.100MiB/s,27.276MiB/s,12.531GiB/9.211GiB/9.271GiB
+2022-12-08 15:47:42.203,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.135MiB/s,28.207MiB/s,50.317GiB/35.790GiB/35.882GiB
+2022-12-08 15:47:42.830,completed,0,T4,0.000 B,2.128 GiB,-1,28,3.841GiB,7/7,28/28,20.641MiB/s,20.806MiB/s,4.386GiB/1.761GiB/1.767GiB
+2022-12-08 15:47:42.832,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.369GiB,10/10,40/40,27.127MiB/s,27.267MiB/s,12.531GiB/9.237GiB/9.284GiB
+2022-12-08 15:47:42.832,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.140MiB/s,28.203MiB/s,50.317GiB/35.814GiB/35.895GiB
+2022-12-08 15:47:46.559,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,17.511MiB/s,17.688MiB/s,3.759GiB/1.184GiB/1.191GiB
+2022-12-08 15:47:46.562,completed,1,T4,2.128 GiB,8.513 GiB,2,58,15.918GiB,10/10,40/40,27.104MiB/s,27.240MiB/s,12.531GiB/9.327GiB/9.374GiB
+2022-12-08 15:47:46.562,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.131MiB/s,28.192MiB/s,50.317GiB/35.905GiB/35.983GiB
+2022-12-08 15:49:43.159,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.019MiB/s,14.043MiB/s,3.132GiB/2.126GiB/2.130GiB
+2022-12-08 15:49:43.159,completed,1,T4,2.128 GiB,8.513 GiB,2,60,16.467GiB,10/10,40/40,26.927MiB/s,27.048MiB/s,12.531GiB/12.332GiB/12.388GiB
+2022-12-08 15:49:43.159,completed,2,T4,8.513 GiB,34.052 GiB,1,50,55.067GiB,10/10,40/40,28.017MiB/s,28.078MiB/s,50.317GiB/38.950GiB/39.034GiB
+2022-12-08 15:49:44.014,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,13.942MiB/s,14.039MiB/s,3.132GiB/2.126GiB/2.141GiB
+2022-12-08 15:49:44.021,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.370GiB,9/9,36/36,24.181MiB/s,24.333MiB/s,11.278GiB/11.080GiB/11.150GiB
+2022-12-08 15:49:44.021,completed,2,T4,8.513 GiB,34.052 GiB,2,51,56.169GiB,10/10,40/40,28.000MiB/s,28.078MiB/s,50.317GiB/38.950GiB/39.057GiB
+2022-12-08 15:49:46.312,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.045MiB/s,14.068MiB/s,3.132GiB/2.173GiB/2.177GiB
+2022-12-08 15:49:46.312,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.272GiB,8/8,32/32,21.542MiB/s,21.639MiB/s,10.024GiB/9.903GiB/9.948GiB
+2022-12-08 15:49:46.313,completed,2,T4,8.513 GiB,34.052 GiB,2,52,57.270GiB,10/10,40/40,28.030MiB/s,28.091MiB/s,50.317GiB/39.054GiB/39.139GiB
+2022-12-08 15:49:50.525,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.068MiB/s,14.104MiB/s,3.132GiB/2.235GiB/2.240GiB
+2022-12-08 15:49:50.525,completed,1,T4,2.128 GiB,8.513 GiB,2,48,13.173GiB,7/7,28/28,18.836MiB/s,18.923MiB/s,8.771GiB/8.735GiB/8.776GiB
+2022-12-08 15:49:50.525,completed,2,T4,8.513 GiB,34.052 GiB,2,53,58.373GiB,10/10,40/40,28.036MiB/s,28.099MiB/s,50.317GiB/39.178GiB/39.265GiB
+2022-12-08 15:49:50.669,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.056MiB/s,14.102MiB/s,3.132GiB/2.235GiB/2.242GiB
+2022-12-08 15:49:50.670,completed,1,T4,2.128 GiB,8.513 GiB,2,44,12.076GiB,6/6,24/24,16.130MiB/s,16.209MiB/s,7.518GiB/7.483GiB/7.519GiB
+2022-12-08 15:49:50.670,completed,2,T4,8.513 GiB,34.052 GiB,2,54,59.474GiB,10/10,40/40,28.033MiB/s,28.098MiB/s,50.317GiB/39.178GiB/39.268GiB
+2022-12-08 15:49:50.956,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.031MiB/s,14.114MiB/s,3.132GiB/2.235GiB/2.248GiB
+2022-12-08 15:49:50.960,completed,1,T4,2.128 GiB,8.513 GiB,2,40,10.978GiB,5/5,20/20,13.424MiB/s,13.493MiB/s,6.265GiB/6.231GiB/6.263GiB
+2022-12-08 15:49:50.960,completed,2,T4,8.513 GiB,34.052 GiB,2,55,60.576GiB,10/10,40/40,28.027MiB/s,28.099MiB/s,50.317GiB/39.178GiB/39.278GiB
+2022-12-08 15:49:51.157,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.052MiB/s,14.113MiB/s,3.132GiB/2.241GiB/2.251GiB
+2022-12-08 15:49:51.157,completed,1,T4,2.128 GiB,8.513 GiB,2,36,9.880GiB,4/4,16/16,10.733MiB/s,10.788MiB/s,5.012GiB/4.981GiB/5.007GiB
+2022-12-08 15:49:51.157,completed,2,T4,8.513 GiB,34.052 GiB,2,56,61.678GiB,10/10,40/40,28.032MiB/s,28.099MiB/s,50.317GiB/39.189GiB/39.283GiB
+2022-12-08 15:49:52.048,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.034MiB/s,14.118MiB/s,3.132GiB/2.250GiB/2.264GiB
+2022-12-08 15:49:52.048,completed,1,T4,2.128 GiB,8.513 GiB,2,32,8.782GiB,3/3,12/12,8.035MiB/s,8.081MiB/s,3.758GiB/3.734GiB/3.755GiB
+2022-12-08 15:49:52.048,completed,2,T4,8.513 GiB,34.052 GiB,2,57,62.780GiB,10/10,40/40,28.025MiB/s,28.103MiB/s,50.317GiB/39.204GiB/39.313GiB
+2022-12-08 15:49:55.250,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.154MiB/s,14.183MiB/s,3.132GiB/2.314GiB/2.318GiB
+2022-12-08 15:49:55.250,completed,1,T4,2.128 GiB,8.513 GiB,2,28,7.685GiB,2/2,8/8,5.367MiB/s,5.389MiB/s,2.506GiB/2.506GiB/2.516GiB
+2022-12-08 15:49:55.250,completed,2,T4,8.513 GiB,34.052 GiB,2,58,63.882GiB,10/10,40/40,28.054MiB/s,28.119MiB/s,50.317GiB/39.332GiB/39.423GiB
+2022-12-08 15:49:55.337,completed,0,T4,0.000 B,2.128 GiB,3,35,4.801GiB,5/5,20/20,14.170MiB/s,14.192MiB/s,3.132GiB/2.317GiB/2.321GiB
+2022-12-08 15:49:55.337,completed,1,T4,2.128 GiB,8.513 GiB,2,24,6.587GiB,1/1,4/4,2.683MiB/s,2.694MiB/s,1.253GiB/1.253GiB/1.258GiB
+2022-12-08 15:49:55.337,completed,2,T4,8.513 GiB,34.052 GiB,2,59,64.984GiB,10/10,40/40,28.059MiB/s,28.121MiB/s,50.317GiB/39.341GiB/39.429GiB
+2022-12-08 15:50:04.678,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,7/6,28/24,14.281MiB/s,14.322MiB/s,4.385GiB/2.466GiB/2.473GiB
+2022-12-08 15:50:04.678,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:04.678,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.089MiB/s,28.158MiB/s,50.317GiB/39.640GiB/39.737GiB
+2022-12-08 15:50:04.689,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,9/7,36/28,14.280MiB/s,14.323MiB/s,5.638GiB/2.466GiB/2.473GiB
+2022-12-08 15:50:04.690,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:04.690,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.089MiB/s,28.158MiB/s,50.317GiB/39.640GiB/39.738GiB
+2022-12-08 15:50:04.707,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/8,40/32,14.278MiB/s,14.323MiB/s,6.264GiB/2.466GiB/2.473GiB
+2022-12-08 15:50:04.709,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:04.709,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.088MiB/s,28.158MiB/s,50.317GiB/39.640GiB/39.738GiB
+2022-12-08 15:50:04.718,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,9/9,36/36,14.277MiB/s,14.323MiB/s,5.638GiB/2.466GiB/2.474GiB
+2022-12-08 15:50:04.718,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:04.718,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.088MiB/s,28.158MiB/s,50.317GiB/39.640GiB/39.739GiB
+2022-12-08 15:50:04.735,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.487GiB,10/10,40/40,14.276MiB/s,14.323MiB/s,6.264GiB/2.466GiB/2.474GiB
+2022-12-08 15:50:04.735,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:04.735,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.088MiB/s,28.158MiB/s,50.317GiB/39.640GiB/39.739GiB
+2022-12-08 15:50:52.872,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,27.764MiB/s,27.989MiB/s,6.264GiB/3.725GiB/3.743GiB
+2022-12-08 15:50:52.873,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.490GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:52.873,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.045MiB/s,28.118MiB/s,50.317GiB/40.898GiB/41.004GiB
+2022-12-08 15:50:54.647,completed,0,T4,0.000 B,2.128 GiB,1,41,5.623GiB,9/9,36/36,24.957MiB/s,25.064MiB/s,5.638GiB/3.142GiB/3.151GiB
+2022-12-08 15:50:54.647,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.039GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:54.647,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.050MiB/s,28.115MiB/s,50.317GiB/40.953GiB/41.049GiB
+2022-12-08 15:50:57.467,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,22.040MiB/s,22.208MiB/s,5.011GiB/2.572GiB/2.583GiB
+2022-12-08 15:50:57.467,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.588GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:57.467,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.051MiB/s,28.114MiB/s,50.317GiB/41.032GiB/41.125GiB
+2022-12-08 15:50:58.075,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.184MiB/s,19.410MiB/s,4.385GiB/1.952GiB/1.966GiB
+2022-12-08 15:50:58.076,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.137GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:50:58.076,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.039MiB/s,28.114MiB/s,50.317GiB/41.032GiB/41.141GiB
+2022-12-08 15:51:02.455,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,16.584MiB/s,16.689MiB/s,3.759GiB/1.407GiB/1.413GiB
+2022-12-08 15:51:02.455,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.685GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:51:02.455,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.052MiB/s,28.114MiB/s,50.317GiB/41.171GiB/41.261GiB
+2022-12-08 15:52:57.205,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,6/6,24/24,15.084MiB/s,15.136MiB/s,3.758GiB/2.541GiB/2.549GiB
+2022-12-08 15:52:57.206,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:52:57.206,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.270MiB/s,28.336MiB/s,50.317GiB/44.658GiB/44.762GiB
+2022-12-08 15:52:57.229,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/7,36/28,15.082MiB/s,15.136MiB/s,5.637GiB/2.541GiB/2.550GiB
+2022-12-08 15:52:57.229,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:52:57.229,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.272MiB/s,28.336MiB/s,50.317GiB/44.662GiB/44.763GiB
+2022-12-08 15:52:57.261,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,8/8,32/32,15.079MiB/s,15.135MiB/s,5.011GiB/2.541GiB/2.550GiB
+2022-12-08 15:52:57.261,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:52:57.261,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.271MiB/s,28.335MiB/s,50.317GiB/44.662GiB/44.763GiB
+2022-12-08 15:52:57.267,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,15.078MiB/s,15.135MiB/s,6.264GiB/2.541GiB/2.550GiB
+2022-12-08 15:52:57.267,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:52:57.271,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.271MiB/s,28.335MiB/s,50.317GiB/44.662GiB/44.763GiB
+2022-12-08 15:52:57.290,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,15.076MiB/s,15.135MiB/s,6.264GiB/2.541GiB/2.551GiB
+2022-12-08 15:52:57.290,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:52:57.297,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.271MiB/s,28.335MiB/s,50.317GiB/44.662GiB/44.764GiB
+2022-12-08 15:53:35.805,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,28.169MiB/s,28.630MiB/s,6.264GiB/3.540GiB/3.565GiB
+2022-12-08 15:53:35.805,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.234GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:53:35.805,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.215MiB/s,28.288MiB/s,50.317GiB/45.635GiB/45.753GiB
+2022-12-08 15:53:35.807,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,25.129MiB/s,25.591MiB/s,5.638GiB/2.913GiB/2.938GiB
+2022-12-08 15:53:35.807,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.783GiB,2/1,8/4,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:53:35.807,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.215MiB/s,28.288MiB/s,50.317GiB/45.635GiB/45.753GiB
+2022-12-08 15:53:35.811,submitted,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,25.127MiB/s,25.594MiB/s,5.638GiB/2.913GiB/2.939GiB
+2022-12-08 15:53:35.811,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,8.783GiB,2/2,8/8,0B/s,0B/s,2.506GiB/0B/0B
+2022-12-08 15:53:35.812,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.215MiB/s,28.288MiB/s,50.317GiB/45.635GiB/45.753GiB
+2022-12-08 15:53:38.960,completed,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,25.447MiB/s,25.537MiB/s,5.638GiB/3.006GiB/3.013GiB
+2022-12-08 15:53:38.960,completed,1,T4,2.128 GiB,8.513 GiB,3,32,8.783GiB,2/2,8/8,4.365MiB/s,4.532MiB/s,2.506GiB/13.726MiB/14.253MiB
+2022-12-08 15:53:38.960,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.220MiB/s,28.284MiB/s,50.317GiB/45.731GiB/45.833GiB
+2022-12-08 15:53:38.962,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,22.452MiB/s,22.543MiB/s,5.011GiB/2.379GiB/2.386GiB
+2022-12-08 15:53:38.970,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.332GiB,3/3,12/12,4.362MiB/s,4.530MiB/s,3.759GiB/13.726MiB/14.253MiB
+2022-12-08 15:53:38.970,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.220MiB/s,28.284MiB/s,50.317GiB/45.731GiB/45.833GiB
+2022-12-08 15:53:38.976,submitted,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,22.446MiB/s,22.544MiB/s,5.011GiB/2.379GiB/2.386GiB
+2022-12-08 15:53:38.977,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.332GiB,4/4,16/16,4.342MiB/s,4.532MiB/s,5.012GiB/13.726MiB/14.326MiB
+2022-12-08 15:53:38.977,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.220MiB/s,28.284MiB/s,50.317GiB/45.731GiB/45.834GiB
+2022-12-08 15:53:45.677,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,22.407MiB/s,22.685MiB/s,5.011GiB/2.524GiB/2.539GiB
+2022-12-08 15:53:45.677,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.332GiB,4/4,16/16,9.399MiB/s,10.886MiB/s,5.012GiB/77.686MiB/89.672MiB
+2022-12-08 15:53:45.677,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.216MiB/s,28.290MiB/s,50.317GiB/45.908GiB/46.030GiB
+2022-12-08 15:53:45.684,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.502MiB/s,19.782MiB/s,4.385GiB/1.898GiB/1.913GiB
+2022-12-08 15:53:45.684,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.881GiB,6/5,24/20,9.391MiB/s,10.885MiB/s,7.519GiB/77.686MiB/89.737MiB
+2022-12-08 15:53:45.684,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.216MiB/s,28.290MiB/s,50.317GiB/45.908GiB/46.030GiB
+2022-12-08 15:53:45.705,submitted,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.496MiB/s,19.786MiB/s,4.385GiB/1.898GiB/1.914GiB
+2022-12-08 15:53:45.705,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,9.881GiB,6/6,24/24,9.366MiB/s,10.890MiB/s,7.519GiB/77.686MiB/90.025MiB
+2022-12-08 15:53:45.706,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.215MiB/s,28.290MiB/s,50.317GiB/45.908GiB/46.030GiB
+2022-12-08 15:53:45.863,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,28/28,19.447MiB/s,19.806MiB/s,4.385GiB/1.898GiB/1.917GiB
+2022-12-08 15:53:45.874,completed,1,T4,2.128 GiB,8.513 GiB,3,36,9.881GiB,6/6,24/24,9.185MiB/s,12.353MiB/s,7.519GiB/77.686MiB/92.007MiB
+2022-12-08 15:53:45.874,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.215MiB/s,28.291MiB/s,50.317GiB/45.912GiB/46.036GiB
+2022-12-08 15:53:45.877,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,16.543MiB/s,16.903MiB/s,3.759GiB/1.272GiB/1.291GiB
+2022-12-08 15:53:45.878,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.430GiB,8/7,32/28,9.168MiB/s,12.326MiB/s,10.025GiB/77.686MiB/92.200MiB
+2022-12-08 15:53:45.881,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.214MiB/s,28.291MiB/s,50.317GiB/45.912GiB/46.036GiB
+2022-12-08 15:53:45.896,submitted,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,16.537MiB/s,16.901MiB/s,3.759GiB/1.272GiB/1.291GiB
+2022-12-08 15:53:45.896,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.430GiB,8/8,32/32,9.480MiB/s,12.974MiB/s,10.025GiB/81.046MiB/92.402MiB
+2022-12-08 15:53:45.897,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.214MiB/s,28.291MiB/s,50.317GiB/45.912GiB/46.037GiB
+2022-12-08 15:53:46.860,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,16.643MiB/s,16.837MiB/s,3.759GiB/1.293GiB/1.302GiB
+2022-12-08 15:53:46.861,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.430GiB,8/8,32/32,12.864MiB/s,19.330MiB/s,10.025GiB/95.813MiB/109.375MiB
+2022-12-08 15:53:46.862,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.219MiB/s,28.287MiB/s,50.317GiB/45.947GiB/46.057GiB
+2022-12-08 15:53:46.866,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,13.752MiB/s,13.949MiB/s,3.132GiB/682.096MiB/691.865MiB
+2022-12-08 15:53:46.870,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.979GiB,10/9,40/36,12.821MiB/s,19.334MiB/s,12.532GiB/95.813MiB/109.602MiB
+2022-12-08 15:53:46.871,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.219MiB/s,28.287MiB/s,50.317GiB/45.947GiB/46.057GiB
+2022-12-08 15:53:46.885,submitted,0,T4,0.000 B,2.128 GiB,1,25,3.429GiB,5/5,20/20,13.747MiB/s,13.951MiB/s,3.132GiB/682.096MiB/692.207MiB
+2022-12-08 15:53:46.888,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,10.979GiB,10/10,40/40,12.756MiB/s,19.404MiB/s,12.532GiB/95.813MiB/110.010MiB
+2022-12-08 15:53:46.896,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.219MiB/s,28.287MiB/s,50.317GiB/45.947GiB/46.058GiB
+2022-12-08 15:55:58.892,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,6/6,24/24,13.902MiB/s,13.930MiB/s,3.758GiB/2.466GiB/2.471GiB
+2022-12-08 15:55:58.893,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.419MiB/s,27.616MiB/s,12.532GiB/3.647GiB/3.673GiB
+2022-12-08 15:55:58.893,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.147MiB/s,28.209MiB/s,50.317GiB/49.457GiB/49.567GiB
+2022-12-08 15:55:58.919,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/7,36/28,13.900MiB/s,13.929MiB/s,5.638GiB/2.466GiB/2.471GiB
+2022-12-08 15:55:58.920,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.413MiB/s,27.615MiB/s,12.532GiB/3.647GiB/3.674GiB
+2022-12-08 15:55:58.920,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.146MiB/s,28.209MiB/s,50.317GiB/49.457GiB/49.567GiB
+2022-12-08 15:55:58.931,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,9/8,36/32,13.899MiB/s,13.930MiB/s,5.638GiB/2.466GiB/2.471GiB
+2022-12-08 15:55:58.936,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.411MiB/s,27.615MiB/s,12.532GiB/3.647GiB/3.674GiB
+2022-12-08 15:55:58.936,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.146MiB/s,28.209MiB/s,50.317GiB/49.457GiB/49.567GiB
+2022-12-08 15:55:58.941,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/9,40/36,13.898MiB/s,13.929MiB/s,6.264GiB/2.466GiB/2.471GiB
+2022-12-08 15:55:58.941,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.409MiB/s,27.615MiB/s,12.532GiB/3.647GiB/3.675GiB
+2022-12-08 15:55:58.942,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.146MiB/s,28.209MiB/s,50.317GiB/49.457GiB/49.568GiB
+2022-12-08 15:55:58.959,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,13.896MiB/s,13.929MiB/s,6.264GiB/2.466GiB/2.472GiB
+2022-12-08 15:55:58.959,submitted,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.405MiB/s,27.615MiB/s,12.532GiB/3.647GiB/3.675GiB
+2022-12-08 15:55:58.961,submitted,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.145MiB/s,28.209MiB/s,50.317GiB/49.457GiB/49.568GiB
+2022-12-08 15:56:14.876,completed,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,10/10,40/40,27.401MiB/s,27.986MiB/s,6.264GiB/2.884GiB/2.896GiB
+2022-12-08 15:56:14.876,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.282MiB/s,27.435MiB/s,12.532GiB/4.055GiB/4.078GiB
+2022-12-08 15:56:14.876,completed,2,T4,8.513 GiB,34.052 GiB,2,60,66.085GiB,10/10,40/40,28.137MiB/s,28.197MiB/s,50.317GiB/49.880GiB/49.987GiB
+2022-12-08 15:56:23.002,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,27.958MiB/s,28.295MiB/s,6.264GiB/3.118GiB/3.128GiB
+2022-12-08 15:56:23.003,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.281MiB/s,27.442MiB/s,12.532GiB/4.272GiB/4.297GiB
+2022-12-08 15:56:23.003,completed,2,T4,8.513 GiB,34.052 GiB,2,56,61.680GiB,9/9,36/36,25.301MiB/s,25.355MiB/s,45.285GiB/45.053GiB/45.150GiB
+2022-12-08 15:56:23.003,completed,3,T4,34.052 GiB,136.206 GiB,1,1,4.424GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:26.840,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,28.097MiB/s,28.302MiB/s,6.264GiB/3.218GiB/3.233GiB
+2022-12-08 15:56:26.840,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.187MiB/s,27.443MiB/s,12.532GiB/4.359GiB/4.400GiB
+2022-12-08 15:56:26.840,completed,2,T4,8.513 GiB,34.052 GiB,2,52,57.275GiB,8/8,32/32,22.462MiB/s,22.515MiB/s,40.253GiB/40.087GiB/40.183GiB
+2022-12-08 15:56:26.840,completed,3,T4,34.052 GiB,136.206 GiB,1,2,8.848GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:27.949,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,28.068MiB/s,28.481MiB/s,6.264GiB/3.256GiB/3.273GiB
+2022-12-08 15:56:27.949,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.252MiB/s,27.567MiB/s,12.532GiB/4.398GiB/4.449GiB
+2022-12-08 15:56:27.950,completed,2,T4,8.513 GiB,34.052 GiB,2,48,52.870GiB,7/7,28/28,19.638MiB/s,19.690MiB/s,35.222GiB/35.073GiB/35.166GiB
+2022-12-08 15:56:27.950,completed,3,T4,34.052 GiB,136.206 GiB,1,3,13.273GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:28.880,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,28.372MiB/s,28.577MiB/s,6.264GiB/3.290GiB/3.305GiB
+2022-12-08 15:56:28.887,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.312MiB/s,27.619MiB/s,12.532GiB/4.433GiB/4.483GiB
+2022-12-08 15:56:28.888,completed,2,T4,8.513 GiB,34.052 GiB,2,44,48.464GiB,6/6,24/24,16.823MiB/s,16.865MiB/s,30.189GiB/30.065GiB/30.140GiB
+2022-12-08 15:56:28.888,completed,3,T4,34.052 GiB,136.206 GiB,1,4,17.697GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:32.149,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,28.403MiB/s,28.605MiB/s,6.264GiB/3.387GiB/3.397GiB
+2022-12-08 15:56:32.149,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.496MiB/s,27.651MiB/s,12.532GiB/4.551GiB/4.577GiB
+2022-12-08 15:56:32.149,completed,2,T4,8.513 GiB,34.052 GiB,2,40,44.060GiB,5/5,20/20,14.015MiB/s,14.044MiB/s,25.158GiB/25.089GiB/25.141GiB
+2022-12-08 15:56:32.149,completed,3,T4,34.052 GiB,136.206 GiB,1,5,22.121GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:32.706,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,28.143MiB/s,28.574MiB/s,6.264GiB/3.390GiB/3.412GiB
+2022-12-08 15:56:32.707,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.406MiB/s,27.645MiB/s,12.532GiB/4.551GiB/4.591GiB
+2022-12-08 15:56:32.710,completed,2,T4,8.513 GiB,34.052 GiB,2,36,39.654GiB,4/4,16/16,11.204MiB/s,11.230MiB/s,20.126GiB/20.057GiB/20.103GiB
+2022-12-08 15:56:32.710,completed,3,T4,34.052 GiB,136.206 GiB,1,6,26.545GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:35.231,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,28.303MiB/s,28.566MiB/s,6.264GiB/3.472GiB/3.486GiB
+2022-12-08 15:56:35.231,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.514MiB/s,27.673MiB/s,12.532GiB/4.637GiB/4.663GiB
+2022-12-08 15:56:35.231,completed,2,T4,8.513 GiB,34.052 GiB,2,32,35.250GiB,3/3,12/12,8.407MiB/s,8.424MiB/s,15.096GiB/15.055GiB/15.087GiB
+2022-12-08 15:56:35.231,completed,3,T4,34.052 GiB,136.206 GiB,1,7,30.969GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:38.865,completed,0,T4,0.000 B,2.128 GiB,-1,40,5.486GiB,10/10,40/40,28.189MiB/s,28.654MiB/s,6.264GiB/3.571GiB/3.595GiB
+2022-12-08 15:56:38.866,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.465MiB/s,27.700MiB/s,12.532GiB/4.726GiB/4.766GiB
+2022-12-08 15:56:38.866,completed,2,T4,8.513 GiB,34.052 GiB,2,28,30.844GiB,2/2,8/8,5.609MiB/s,5.622MiB/s,10.063GiB/10.039GiB/10.062GiB
+2022-12-08 15:56:38.867,completed,3,T4,34.052 GiB,136.206 GiB,1,8,35.394GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:43.639,completed,0,T4,0.000 B,2.128 GiB,1,45,6.172GiB,10/10,40/40,28.355MiB/s,28.578MiB/s,6.264GiB/3.711GiB/3.724GiB
+2022-12-08 15:56:43.645,completed,1,T4,2.128 GiB,8.513 GiB,-1,40,10.979GiB,10/10,40/40,27.589MiB/s,27.749MiB/s,12.532GiB/4.876GiB/4.904GiB
+2022-12-08 15:56:43.646,completed,2,T4,8.513 GiB,34.052 GiB,2,24,26.439GiB,1/1,4/4,2.803MiB/s,2.808MiB/s,5.032GiB/5.022GiB/5.032GiB
+2022-12-08 15:56:43.647,completed,3,T4,34.052 GiB,136.206 GiB,1,9,39.817GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:46.939,completed,0,T4,0.000 B,2.128 GiB,1,41,5.624GiB,9/9,36/36,25.692MiB/s,25.795MiB/s,5.638GiB/3.174GiB/3.181GiB
+2022-12-08 15:56:46.939,completed,1,T4,2.128 GiB,8.513 GiB,1,42,11.528GiB,10/10,40/40,27.587MiB/s,27.760MiB/s,12.532GiB/4.964GiB/4.995GiB
+2022-12-08 15:56:46.939,completed,2,T4,8.513 GiB,34.052 GiB,2,24,26.439GiB,1/1,4/4,2.803MiB/s,2.809MiB/s,5.032GiB/5.032GiB/5.042GiB
+2022-12-08 15:56:46.939,completed,3,T4,34.052 GiB,136.206 GiB,1,9,39.817GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:47.192,completed,0,T4,0.000 B,2.128 GiB,1,37,5.075GiB,8/8,32/32,22.894MiB/s,22.980MiB/s,5.012GiB/2.552GiB/2.558GiB
+2022-12-08 15:56:47.192,completed,1,T4,2.128 GiB,8.513 GiB,1,44,12.076GiB,10/10,40/40,27.563MiB/s,27.754MiB/s,12.532GiB/4.967GiB/5.001GiB
+2022-12-08 15:56:47.192,completed,2,T4,8.513 GiB,34.052 GiB,2,24,26.439GiB,1/1,4/4,2.803MiB/s,2.808MiB/s,5.032GiB/5.032GiB/5.042GiB
+2022-12-08 15:56:47.192,completed,3,T4,34.052 GiB,136.206 GiB,1,9,39.817GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:47.744,completed,0,T4,0.000 B,2.128 GiB,1,33,4.527GiB,7/7,28/28,20.009MiB/s,20.237MiB/s,4.386GiB/1.934GiB/1.946GiB
+2022-12-08 15:56:47.744,completed,1,T4,2.128 GiB,8.513 GiB,1,46,12.625GiB,10/10,40/40,27.609MiB/s,27.766MiB/s,12.532GiB/4.990GiB/5.018GiB
+2022-12-08 15:56:47.744,completed,2,T4,8.513 GiB,34.052 GiB,2,24,26.439GiB,1/1,4/4,2.802MiB/s,2.807MiB/s,5.032GiB/5.032GiB/5.042GiB
+2022-12-08 15:56:47.744,completed,3,T4,34.052 GiB,136.206 GiB,1,9,39.817GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:48.433,completed,0,T4,0.000 B,2.128 GiB,1,33,4.527GiB,7/7,28/28,20.082MiB/s,20.280MiB/s,4.386GiB/1.951GiB/1.961GiB
+2022-12-08 15:56:48.433,completed,1,T4,2.128 GiB,8.513 GiB,1,46,12.625GiB,10/10,40/40,27.539MiB/s,27.785MiB/s,12.532GiB/4.996GiB/5.040GiB
+2022-12-08 15:56:48.433,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:48.433,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:50.838,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,24/24,17.429MiB/s,17.627MiB/s,3.759GiB/1.371GiB/1.381GiB
+2022-12-08 15:56:50.838,completed,1,T4,2.128 GiB,8.513 GiB,1,48,13.175GiB,10/10,40/40,27.717MiB/s,27.886MiB/s,12.532GiB/5.093GiB/5.124GiB
+2022-12-08 15:56:50.838,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:56:50.838,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:58:48.233,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,6/6,24/24,14.952MiB/s,14.999MiB/s,3.759GiB/2.472GiB/2.480GiB
+2022-12-08 15:58:48.233,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,28.294MiB/s,28.469MiB/s,12.532GiB/8.443GiB/8.495GiB
+2022-12-08 15:58:48.233,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:58:48.233,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:58:48.238,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,28/28,14.952MiB/s,15.000MiB/s,4.385GiB/2.472GiB/2.480GiB
+2022-12-08 15:58:48.240,submitted,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,28.293MiB/s,28.470MiB/s,12.532GiB/8.443GiB/8.496GiB
+2022-12-08 15:58:48.241,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:58:48.241,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.218,completed,0,T4,0.000 B,2.128 GiB,5,45,6.172GiB,8/7,33/28,21.613MiB/s,21.733MiB/s,5.168GiB/3.364GiB/3.374GiB
+2022-12-08 15:59:29.218,completed,1,T4,2.128 GiB,8.513 GiB,1,50,13.724GiB,10/10,40/40,28.640MiB/s,28.842MiB/s,12.532GiB/9.692GiB/9.761GiB
+2022-12-08 15:59:29.218,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.218,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.228,submitted,0,T4,0.000 B,2.128 GiB,5,41,5.623GiB,8/7,34/29,18.560MiB/s,18.680MiB/s,5.325GiB/2.737GiB/2.748GiB
+2022-12-08 15:59:29.229,submitted,1,T4,2.128 GiB,8.513 GiB,2,52,14.273GiB,10/10,40/40,28.648MiB/s,28.842MiB/s,12.532GiB/9.695GiB/9.761GiB
+2022-12-08 15:59:29.229,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.229,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.237,completed,0,T4,0.000 B,2.128 GiB,5,41,5.623GiB,8/7,34/29,18.558MiB/s,18.679MiB/s,5.325GiB/2.737GiB/2.748GiB
+2022-12-08 15:59:29.237,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.273GiB,10/10,40/40,28.647MiB/s,28.842MiB/s,12.532GiB/9.695GiB/9.761GiB
+2022-12-08 15:59:29.237,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.237,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.266,submitted,0,T4,0.000 B,2.128 GiB,5,37,5.075GiB,8/7,35/30,15.521MiB/s,15.626MiB/s,5.481GiB/2.115GiB/2.121GiB
+2022-12-08 15:59:29.266,submitted,1,T4,2.128 GiB,8.513 GiB,2,54,14.822GiB,10/10,40/40,28.663MiB/s,28.842MiB/s,12.532GiB/9.701GiB/9.762GiB
+2022-12-08 15:59:29.266,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.266,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.428,completed,0,T4,0.000 B,2.128 GiB,5,37,5.075GiB,8/7,35/30,15.501MiB/s,16.492MiB/s,5.481GiB/2.117GiB/2.123GiB
+2022-12-08 15:59:29.429,completed,1,T4,2.128 GiB,8.513 GiB,2,54,14.822GiB,10/10,40/40,28.694MiB/s,28.842MiB/s,12.532GiB/9.717GiB/9.767GiB
+2022-12-08 15:59:29.429,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.429,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.434,submitted,0,T4,0.000 B,2.128 GiB,5,33,4.526GiB,7/7,31/31,12.453MiB/s,13.473MiB/s,4.855GiB/1.491GiB/1.497GiB
+2022-12-08 15:59:29.434,submitted,1,T4,2.128 GiB,8.513 GiB,2,56,15.370GiB,10/10,40/40,28.693MiB/s,28.842MiB/s,12.532GiB/9.717GiB/9.767GiB
+2022-12-08 15:59:29.435,submitted,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:29.435,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:31.253,completed,0,T4,0.000 B,2.128 GiB,1,33,4.526GiB,7/7,31/31,19.379MiB/s,23.682MiB/s,4.855GiB/1.529GiB/1.542GiB
+2022-12-08 15:59:31.253,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.370GiB,10/10,40/40,28.715MiB/s,28.896MiB/s,12.532GiB/9.775GiB/9.837GiB
+2022-12-08 15:59:31.253,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:31.253,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:34.405,completed,0,T4,0.000 B,2.128 GiB,1,29,3.978GiB,6/6,27/27,19.160MiB/s,20.122MiB/s,4.228GiB/990.681MiB/998.875MiB
+2022-12-08 15:59:34.405,completed,1,T4,2.128 GiB,8.513 GiB,2,58,15.919GiB,10/10,40/40,28.797MiB/s,28.942MiB/s,12.532GiB/9.892GiB/9.941GiB
+2022-12-08 15:59:34.406,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 15:59:34.406,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:00:58.319,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.903MiB/s,14.953MiB/s,3.602GiB/1.540GiB/1.545GiB
+2022-12-08 16:00:58.319,completed,1,T4,2.128 GiB,8.513 GiB,2,60,16.468GiB,10/10,40/40,28.780MiB/s,28.931MiB/s,12.532GiB/12.245GiB/12.309GiB
+2022-12-08 16:00:58.319,completed,2,T4,8.513 GiB,34.052 GiB,2,20,22.034GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:00:58.319,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:02.311,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.819MiB/s,14.867MiB/s,3.602GiB/1.590GiB/1.595GiB
+2022-12-08 16:01:02.315,completed,1,T4,2.128 GiB,8.513 GiB,2,56,15.371GiB,9/9,36/36,25.869MiB/s,26.004MiB/s,11.279GiB/11.087GiB/11.145GiB
+2022-12-08 16:01:02.315,completed,2,T4,8.513 GiB,34.052 GiB,3,21,23.136GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:02.315,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:02.568,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.783MiB/s,14.861MiB/s,3.602GiB/1.590GiB/1.598GiB
+2022-12-08 16:01:02.568,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.273GiB,8/8,32/32,22.989MiB/s,23.114MiB/s,10.027GiB/9.837GiB/9.891GiB
+2022-12-08 16:01:02.570,completed,2,T4,8.513 GiB,34.052 GiB,3,22,24.238GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:02.570,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:06.473,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.871MiB/s,14.929MiB/s,3.602GiB/1.656GiB/1.662GiB
+2022-12-08 16:01:06.474,completed,1,T4,2.128 GiB,8.513 GiB,2,48,13.175GiB,7/7,28/28,20.123MiB/s,20.233MiB/s,8.774GiB/8.673GiB/8.720GiB
+2022-12-08 16:01:06.478,completed,2,T4,8.513 GiB,34.052 GiB,3,23,25.340GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:06.478,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:08.476,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.923MiB/s,14.981MiB/s,3.602GiB/1.691GiB/1.698GiB
+2022-12-08 16:01:08.476,completed,1,T4,2.128 GiB,8.513 GiB,2,44,12.077GiB,6/6,24/24,17.228MiB/s,17.320MiB/s,7.520GiB/7.460GiB/7.500GiB
+2022-12-08 16:01:08.476,completed,2,T4,8.513 GiB,34.052 GiB,3,24,26.443GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:08.476,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:08.480,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.923MiB/s,14.981MiB/s,3.602GiB/1.691GiB/1.698GiB
+2022-12-08 16:01:08.480,completed,1,T4,2.128 GiB,8.513 GiB,2,40,10.979GiB,5/5,20/20,14.372MiB/s,14.452MiB/s,6.266GiB/6.207GiB/6.241GiB
+2022-12-08 16:01:08.480,completed,2,T4,8.513 GiB,34.052 GiB,3,25,27.545GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:08.480,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:11.974,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.928MiB/s,15.012MiB/s,3.602GiB/1.743GiB/1.752GiB
+2022-12-08 16:01:11.978,completed,1,T4,2.128 GiB,8.513 GiB,2,36,9.881GiB,4/4,16/16,11.483MiB/s,11.550MiB/s,5.013GiB/4.997GiB/5.026GiB
+2022-12-08 16:01:11.978,completed,2,T4,8.513 GiB,34.052 GiB,3,26,28.647GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:11.978,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:12.432,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.981MiB/s,15.058MiB/s,3.602GiB/1.756GiB/1.764GiB
+2022-12-08 16:01:12.432,completed,1,T4,2.128 GiB,8.513 GiB,2,32,8.783GiB,3/3,12/12,8.608MiB/s,8.654MiB/s,3.760GiB/3.751GiB/3.772GiB
+2022-12-08 16:01:12.434,completed,2,T4,8.513 GiB,34.052 GiB,3,27,29.750GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:12.434,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:12.831,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.930MiB/s,15.063MiB/s,3.602GiB/1.756GiB/1.771GiB
+2022-12-08 16:01:12.832,completed,1,T4,2.128 GiB,8.513 GiB,2,28,7.685GiB,2/2,8/8,5.731MiB/s,5.761MiB/s,2.507GiB/2.502GiB/2.515GiB
+2022-12-08 16:01:12.835,completed,2,T4,8.513 GiB,34.052 GiB,3,28,30.852GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:12.835,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:13.233,completed,0,T4,0.000 B,2.128 GiB,3,35,4.800GiB,5/5,23/23,14.989MiB/s,15.177MiB/s,3.602GiB/1.767GiB/1.790GiB
+2022-12-08 16:01:13.233,completed,1,T4,2.128 GiB,8.513 GiB,2,24,6.587GiB,1/1,4/4,2.868MiB/s,2.881MiB/s,1.253GiB/1.253GiB/1.259GiB
+2022-12-08 16:01:13.234,completed,2,T4,8.513 GiB,34.052 GiB,3,29,31.954GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:13.234,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:30.383,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/6,31/27,15.356MiB/s,15.460MiB/s,4.855GiB/2.068GiB/2.081GiB
+2022-12-08 16:01:30.387,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:30.387,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:30.387,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:30.402,submitted,0,T4,0.000 B,2.128 GiB,4,40,5.486GiB,7/7,31/31,15.354MiB/s,15.459MiB/s,4.855GiB/2.068GiB/2.082GiB
+2022-12-08 16:01:30.402,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:30.402,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:01:30.402,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:04.738,submitted,0,T4,0.000 B,2.128 GiB,4,45,6.172GiB,9/8,39/35,23.171MiB/s,23.549MiB/s,6.107GiB/2.905GiB/2.931GiB
+2022-12-08 16:02:04.741,submitted,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:04.742,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:04.743,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.302,completed,0,T4,0.000 B,2.128 GiB,4,45,6.172GiB,9/8,39/35,25.629MiB/s,27.183MiB/s,6.107GiB/2.962GiB/2.976GiB
+2022-12-08 16:02:06.310,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.489GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.310,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.310,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.312,submitted,0,T4,0.000 B,2.128 GiB,4,41,5.623GiB,9/8,39/35,22.373MiB/s,23.960MiB/s,6.107GiB/2.336GiB/2.349GiB
+2022-12-08 16:02:06.313,submitted,1,T4,2.128 GiB,8.513 GiB,3,22,6.038GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.313,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.313,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.867,completed,0,T4,0.000 B,2.128 GiB,4,41,5.623GiB,9/8,39/35,24.012MiB/s,29.303MiB/s,6.107GiB/2.347GiB/2.368GiB
+2022-12-08 16:02:06.874,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.038GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.874,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.875,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.884,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.075GiB,8/8,35/35,20.748MiB/s,25.977MiB/s,5.481GiB/1.721GiB/1.742GiB
+2022-12-08 16:02:06.884,submitted,1,T4,2.128 GiB,8.513 GiB,3,24,6.587GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.885,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:02:06.885,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:44.668,completed,0,T4,0.000 B,2.128 GiB,3,47,6.446GiB,8/8,35/35,25.637MiB/s,25.728MiB/s,5.481GiB/4.136GiB/4.149GiB
+2022-12-08 16:03:44.668,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.587GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:44.668,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:44.668,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:46.067,completed,0,T4,0.000 B,2.128 GiB,3,42,5.761GiB,7/7,30/30,22.448MiB/s,22.610MiB/s,4.698GiB/3.378GiB/3.398GiB
+2022-12-08 16:03:46.067,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.273GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:46.067,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:46.067,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:46.726,completed,0,T4,0.000 B,2.128 GiB,3,37,5.075GiB,6/6,25/25,19.416MiB/s,19.490MiB/s,3.915GiB/2.618GiB/2.627GiB
+2022-12-08 16:03:46.735,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.959GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:46.735,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:46.735,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:53.517,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.075GiB,6/6,24/24,16.362MiB/s,16.391MiB/s,3.759GiB/1.950GiB/1.954GiB
+2022-12-08 16:03:53.517,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.646GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:53.518,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:53.518,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:53.533,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.075GiB,7/7,28/28,16.360MiB/s,16.390MiB/s,4.385GiB/1.950GiB/1.954GiB
+2022-12-08 16:03:53.534,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.646GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:53.534,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:03:53.534,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:26.737,submitted,0,T4,0.000 B,2.128 GiB,4,42,5.760GiB,9/8,36/32,24.215MiB/s,24.410MiB/s,5.638GiB/2.773GiB/2.788GiB
+2022-12-08 16:04:26.737,submitted,1,T4,2.128 GiB,8.513 GiB,3,30,8.646GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:26.737,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:26.737,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:37.999,completed,0,T4,0.000 B,2.128 GiB,4,42,5.760GiB,10/8,40/32,28.074MiB/s,28.259MiB/s,6.264GiB/3.097GiB/3.108GiB
+2022-12-08 16:04:37.999,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.646GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:37.999,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:37.999,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.006,submitted,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,9/8,36/32,24.649MiB/s,24.835MiB/s,5.637GiB/2.471GiB/2.481GiB
+2022-12-08 16:04:38.007,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,9.195GiB,1/0,4/0,0B/s,0B/s,1.254GiB/0B/0B
+2022-12-08 16:04:38.007,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.007,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.010,submitted,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,9/8,36/32,24.647MiB/s,24.839MiB/s,5.637GiB/2.471GiB/2.482GiB
+2022-12-08 16:04:38.010,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,9.195GiB,1/1,4/4,0B/s,0B/s,1.254GiB/0B/0B
+2022-12-08 16:04:38.010,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.010,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.049,submitted,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,9/8,36/32,24.624MiB/s,24.829MiB/s,5.637GiB/2.471GiB/2.482GiB
+2022-12-08 16:04:38.051,submitted,1,T4,2.128 GiB,8.513 GiB,4,32,9.195GiB,2/2,8/8,0B/s,0B/s,2.507GiB/0B/0B
+2022-12-08 16:04:38.052,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.052,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.797,completed,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,9/8,36/32,24.361MiB/s,26.838MiB/s,5.637GiB/2.481GiB/2.495GiB
+2022-12-08 16:04:38.797,completed,1,T4,2.128 GiB,8.513 GiB,3,32,9.195GiB,2/2,8/8,0B/s,3.893MiB/s,2.507GiB/0B/2.904MiB
+2022-12-08 16:04:38.797,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.797,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.803,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,8/8,32/32,20.954MiB/s,23.416MiB/s,5.011GiB/1.854GiB/1.869GiB
+2022-12-08 16:04:38.803,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.743GiB,4/2,16/8,0B/s,3.875MiB/s,5.012GiB/0B/2.919MiB
+2022-12-08 16:04:38.803,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.803,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.811,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,8/8,32/32,20.949MiB/s,23.391MiB/s,5.011GiB/1.854GiB/1.869GiB
+2022-12-08 16:04:38.811,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.743GiB,4/3,16/12,0B/s,3.907MiB/s,5.012GiB/0B/2.976MiB
+2022-12-08 16:04:38.814,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.814,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.829,submitted,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,20.940MiB/s,23.427MiB/s,5.011GiB/1.854GiB/1.869GiB
+2022-12-08 16:04:38.829,submitted,1,T4,2.128 GiB,8.513 GiB,4,34,9.743GiB,4/4,16/16,0B/s,3.955MiB/s,5.012GiB/0B/3.081MiB
+2022-12-08 16:04:38.829,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:04:38.829,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:23.235,completed,0,T4,0.000 B,2.128 GiB,2,39,5.349GiB,8/8,32/32,25.386MiB/s,25.551MiB/s,5.011GiB/2.882GiB/2.893GiB
+2022-12-08 16:05:23.236,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.743GiB,4/4,16/16,11.509MiB/s,11.776MiB/s,5.012GiB/515.659MiB/527.641MiB
+2022-12-08 16:05:23.236,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:23.236,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:23.237,submitted,0,T4,0.000 B,2.128 GiB,2,35,4.800GiB,7/7,28/28,22.153MiB/s,22.317MiB/s,4.385GiB/2.256GiB/2.266GiB
+2022-12-08 16:05:23.237,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,10.292GiB,6/5,24/20,11.509MiB/s,11.777MiB/s,7.676GiB/515.659MiB/527.677MiB
+2022-12-08 16:05:23.237,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:23.237,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:23.257,submitted,0,T4,0.000 B,2.128 GiB,2,35,4.800GiB,7/7,28/28,22.147MiB/s,22.317MiB/s,4.385GiB/2.256GiB/2.267GiB
+2022-12-08 16:05:23.257,submitted,1,T4,2.128 GiB,8.513 GiB,4,36,10.292GiB,6/6,24/24,11.504MiB/s,11.777MiB/s,7.676GiB/515.659MiB/527.929MiB
+2022-12-08 16:05:23.257,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:23.257,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:24.799,completed,0,T4,0.000 B,2.128 GiB,2,35,4.800GiB,7/7,28/28,22.191MiB/s,22.326MiB/s,4.385GiB/2.287GiB/2.299GiB
+2022-12-08 16:05:24.802,completed,1,T4,2.128 GiB,8.513 GiB,3,36,10.292GiB,6/6,24/24,14.646MiB/s,17.433MiB/s,7.676GiB/543.384MiB/554.275MiB
+2022-12-08 16:05:24.804,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:24.804,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:24.809,submitted,0,T4,0.000 B,2.128 GiB,2,31,4.251GiB,6/6,24/24,18.956MiB/s,19.093MiB/s,3.758GiB/1.661GiB/1.673GiB
+2022-12-08 16:05:24.816,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.841GiB,8/7,32/28,14.625MiB/s,17.434MiB/s,10.339GiB/543.384MiB/554.385MiB
+2022-12-08 16:05:24.816,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:24.816,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:24.832,submitted,0,T4,0.000 B,2.128 GiB,2,31,4.251GiB,6/6,24/24,18.950MiB/s,19.091MiB/s,3.758GiB/1.661GiB/1.673GiB
+2022-12-08 16:05:24.838,submitted,1,T4,2.128 GiB,8.513 GiB,4,38,10.841GiB,8/8,32/32,14.575MiB/s,17.406MiB/s,10.339GiB/543.384MiB/554.659MiB
+2022-12-08 16:05:24.838,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:24.838,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:26.022,completed,0,T4,0.000 B,2.128 GiB,2,31,4.251GiB,6/6,24/24,18.960MiB/s,19.039MiB/s,3.758GiB/1.686GiB/1.690GiB
+2022-12-08 16:05:26.023,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.841GiB,8/8,32/32,20.055MiB/s,22.570MiB/s,10.339GiB/567.280MiB/581.121MiB
+2022-12-08 16:05:26.023,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:05:26.023,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:06:27.869,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.074GiB,6/6,24/24,15.788MiB/s,15.869MiB/s,3.758GiB/2.008GiB/2.019GiB
+2022-12-08 16:06:27.870,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,11.390GiB,9/8,36/32,24.311MiB/s,24.620MiB/s,11.670GiB/2.050GiB/2.075GiB
+2022-12-08 16:06:27.870,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:06:27.870,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:06:27.892,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.074GiB,7/7,28/28,15.785MiB/s,15.868MiB/s,4.385GiB/2.008GiB/2.019GiB
+2022-12-08 16:06:27.893,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,11.390GiB,9/8,36/32,24.304MiB/s,24.618MiB/s,11.670GiB/2.050GiB/2.075GiB
+2022-12-08 16:06:27.893,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:06:27.893,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:04.032,submitted,0,T4,0.000 B,2.128 GiB,4,42,5.760GiB,8/8,32/32,22.949MiB/s,23.076MiB/s,5.011GiB/2.848GiB/2.856GiB
+2022-12-08 16:07:04.032,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,11.390GiB,9/8,36/32,24.969MiB/s,25.161MiB/s,11.670GiB/2.988GiB/3.010GiB
+2022-12-08 16:07:04.032,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:04.034,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:08.732,completed,0,T4,0.000 B,2.128 GiB,4,42,5.760GiB,10/8,40/32,25.131MiB/s,25.806MiB/s,6.264GiB/2.943GiB/2.954GiB
+2022-12-08 16:07:08.732,completed,1,T4,2.128 GiB,8.513 GiB,4,40,11.390GiB,9/8,36/32,24.859MiB/s,25.005MiB/s,11.670GiB/3.090GiB/3.108GiB
+2022-12-08 16:07:08.732,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:08.732,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:08.734,submitted,0,T4,0.000 B,2.128 GiB,4,38,5.211GiB,8/8,32/32,21.842MiB/s,22.517MiB/s,5.011GiB/2.316GiB/2.327GiB
+2022-12-08 16:07:08.734,submitted,1,T4,2.128 GiB,8.513 GiB,4,42,11.939GiB,10/8,40/32,24.858MiB/s,25.005MiB/s,13.001GiB/3.090GiB/3.108GiB
+2022-12-08 16:07:08.734,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:08.734,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:17.136,completed,0,T4,0.000 B,2.128 GiB,4,38,5.211GiB,9/8,36/32,24.457MiB/s,24.879MiB/s,5.637GiB/2.491GiB/2.505GiB
+2022-12-08 16:07:17.136,completed,1,T4,2.128 GiB,8.513 GiB,4,42,11.939GiB,9/8,36/32,24.606MiB/s,24.783MiB/s,11.670GiB/3.262GiB/3.285GiB
+2022-12-08 16:07:17.136,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:17.136,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:17.137,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,8/8,32/32,21.305MiB/s,21.726MiB/s,5.011GiB/1.864GiB/1.879GiB
+2022-12-08 16:07:17.137,submitted,1,T4,2.128 GiB,8.513 GiB,4,44,12.488GiB,10/8,40/32,24.606MiB/s,24.783MiB/s,13.001GiB/3.262GiB/3.285GiB
+2022-12-08 16:07:17.138,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:17.138,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:51.301,completed,0,T4,0.000 B,2.128 GiB,2,39,5.348GiB,8/8,32/32,23.987MiB/s,24.303MiB/s,5.011GiB/2.632GiB/2.651GiB
+2022-12-08 16:07:51.301,completed,1,T4,2.128 GiB,8.513 GiB,4,44,12.488GiB,9/8,36/32,24.286MiB/s,24.549MiB/s,11.670GiB/4.031GiB/4.074GiB
+2022-12-08 16:07:51.307,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:07:51.307,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:08:09.024,completed,0,T4,0.000 B,2.128 GiB,2,35,4.800GiB,7/7,28/28,20.638MiB/s,20.700MiB/s,4.385GiB/2.340GiB/2.345GiB
+2022-12-08 16:08:09.026,completed,1,T4,2.128 GiB,8.513 GiB,4,46,13.037GiB,10/8,40/32,24.096MiB/s,24.266MiB/s,13.001GiB/4.417GiB/4.448GiB
+2022-12-08 16:08:09.026,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:08:09.026,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:08:09.853,completed,0,T4,0.000 B,2.128 GiB,2,31,4.251GiB,6/6,24/24,17.619MiB/s,17.647MiB/s,3.758GiB/1.728GiB/1.731GiB
+2022-12-08 16:08:09.853,completed,1,T4,2.128 GiB,8.513 GiB,4,48,13.586GiB,10/8,40/32,24.096MiB/s,24.247MiB/s,13.001GiB/4.437GiB/4.464GiB
+2022-12-08 16:08:09.853,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:08:09.853,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:03.524,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.074GiB,7/6,28/24,15.409MiB/s,15.505MiB/s,4.385GiB/1.965GiB/1.977GiB
+2022-12-08 16:09:03.524,submitted,1,T4,2.128 GiB,8.513 GiB,5,50,14.135GiB,9/8,37/32,24.523MiB/s,24.702MiB/s,11.983GiB/5.801GiB/5.844GiB
+2022-12-08 16:09:03.524,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:03.524,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:03.549,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.074GiB,7/7,28/28,15.406MiB/s,15.505MiB/s,4.385GiB/1.965GiB/1.978GiB
+2022-12-08 16:09:03.549,submitted,1,T4,2.128 GiB,8.513 GiB,5,50,14.135GiB,9/8,37/32,24.520MiB/s,24.702MiB/s,11.983GiB/5.801GiB/5.844GiB
+2022-12-08 16:09:03.549,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:03.551,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:45.684,submitted,0,T4,0.000 B,2.128 GiB,4,42,5.760GiB,8/8,32/32,21.657MiB/s,21.811MiB/s,5.011GiB/2.854GiB/2.866GiB
+2022-12-08 16:09:45.684,submitted,1,T4,2.128 GiB,8.513 GiB,5,50,14.135GiB,10/8,42/32,24.474MiB/s,24.630MiB/s,13.628GiB/6.797GiB/6.840GiB
+2022-12-08 16:09:45.686,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:45.686,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:52.133,completed,0,T4,0.000 B,2.128 GiB,4,42,5.760GiB,9/8,36/32,23.822MiB/s,24.244MiB/s,5.637GiB/2.987GiB/3.007GiB
+2022-12-08 16:09:52.133,completed,1,T4,2.128 GiB,8.513 GiB,5,50,14.135GiB,10/8,42/32,24.422MiB/s,24.610MiB/s,13.628GiB/6.936GiB/6.989GiB
+2022-12-08 16:09:52.133,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:52.133,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:52.139,submitted,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,8/8,32/32,20.680MiB/s,21.103MiB/s,5.011GiB/2.361GiB/2.381GiB
+2022-12-08 16:09:52.139,submitted,1,T4,2.128 GiB,8.513 GiB,5,52,14.683GiB,9/8,37/32,24.421MiB/s,24.609MiB/s,11.983GiB/6.936GiB/6.989GiB
+2022-12-08 16:09:52.139,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:52.139,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:57.223,completed,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,9/8,36/32,23.944MiB/s,24.301MiB/s,5.637GiB/2.482GiB/2.494GiB
+2022-12-08 16:09:57.223,completed,1,T4,2.128 GiB,8.513 GiB,5,52,14.683GiB,10/8,42/32,24.421MiB/s,24.590MiB/s,13.628GiB/7.058GiB/7.106GiB
+2022-12-08 16:09:57.223,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:57.223,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:57.227,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,8/8,32/32,20.873MiB/s,21.239MiB/s,5.011GiB/1.855GiB/1.868GiB
+2022-12-08 16:09:57.227,submitted,1,T4,2.128 GiB,8.513 GiB,5,54,15.232GiB,10/8,42/32,24.421MiB/s,24.590MiB/s,13.628GiB/7.058GiB/7.106GiB
+2022-12-08 16:09:57.229,submitted,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:09:57.229,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:10:37.195,completed,0,T4,0.000 B,2.128 GiB,2,39,5.348GiB,8/8,32/32,23.656MiB/s,23.769MiB/s,5.011GiB/2.759GiB/2.768GiB
+2022-12-08 16:10:37.195,completed,1,T4,2.128 GiB,8.513 GiB,5,54,15.232GiB,10/8,42/32,24.155MiB/s,24.290MiB/s,13.628GiB/7.925GiB/7.969GiB
+2022-12-08 16:10:37.196,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:10:37.196,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:10:42.141,completed,0,T4,0.000 B,2.128 GiB,2,35,4.800GiB,7/7,28/28,20.899MiB/s,21.075MiB/s,4.384GiB/2.250GiB/2.262GiB
+2022-12-08 16:10:42.142,completed,1,T4,2.128 GiB,8.513 GiB,6,56,15.781GiB,9/8,38/32,24.229MiB/s,24.379MiB/s,12.297GiB/8.066GiB/8.116GiB
+2022-12-08 16:10:42.142,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:10:42.142,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:10:47.448,completed,0,T4,0.000 B,2.128 GiB,2,31,4.251GiB,6/6,24/24,18.074MiB/s,18.258MiB/s,3.758GiB/1.731GiB/1.744GiB
+2022-12-08 16:10:47.449,completed,1,T4,2.128 GiB,8.513 GiB,6,58,16.330GiB,9/8,38/32,24.298MiB/s,24.459MiB/s,12.297GiB/8.215GiB/8.269GiB
+2022-12-08 16:10:47.451,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:10:47.452,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:19.952,completed,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.154MiB/s,18.222MiB/s,3.131GiB/1.891GiB/1.898GiB
+2022-12-08 16:11:19.952,completed,1,T4,2.128 GiB,8.513 GiB,6,60,16.879GiB,10/8,44/32,25.501MiB/s,25.639MiB/s,14.254GiB/9.430GiB/9.481GiB
+2022-12-08 16:11:19.953,completed,2,T4,8.513 GiB,34.052 GiB,3,30,33.057GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:19.953,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:19.959,submitted,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.152MiB/s,18.222MiB/s,3.131GiB/1.891GiB/1.898GiB
+2022-12-08 16:11:19.960,submitted,1,T4,2.128 GiB,8.513 GiB,6,56,15.781GiB,9/8,40/34,22.308MiB/s,22.432MiB/s,13.001GiB/8.177GiB/8.222GiB
+2022-12-08 16:11:19.960,submitted,2,T4,8.513 GiB,34.052 GiB,4,31,34.159GiB,1/0,4/0,0B/s,0B/s,5.034GiB/0B/0B
+2022-12-08 16:11:19.962,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:19.966,submitted,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.151MiB/s,18.222MiB/s,3.131GiB/1.891GiB/1.898GiB
+2022-12-08 16:11:19.966,submitted,1,T4,2.128 GiB,8.513 GiB,6,56,15.781GiB,9/8,40/34,22.307MiB/s,22.432MiB/s,13.001GiB/8.177GiB/8.223GiB
+2022-12-08 16:11:19.966,submitted,2,T4,8.513 GiB,34.052 GiB,4,31,34.159GiB,1/1,4/4,0B/s,0B/s,5.034GiB/0B/0B
+2022-12-08 16:11:19.967,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:19.983,completed,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.148MiB/s,18.224MiB/s,3.131GiB/1.891GiB/1.898GiB
+2022-12-08 16:11:19.983,completed,1,T4,2.128 GiB,8.513 GiB,6,56,15.781GiB,9/8,40/34,22.306MiB/s,22.434MiB/s,13.001GiB/8.177GiB/8.224GiB
+2022-12-08 16:11:19.983,completed,2,T4,8.513 GiB,34.052 GiB,3,31,34.159GiB,1/1,4/4,0B/s,0B/s,5.034GiB/0B/0B
+2022-12-08 16:11:19.983,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:19.984,submitted,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.147MiB/s,18.224MiB/s,3.131GiB/1.891GiB/1.898GiB
+2022-12-08 16:11:19.984,submitted,1,T4,2.128 GiB,8.513 GiB,6,52,14.683GiB,8/8,36/36,19.113MiB/s,19.226MiB/s,11.748GiB/6.923GiB/6.964GiB
+2022-12-08 16:11:19.985,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,35.262GiB,2/1,8/4,0B/s,0B/s,10.068GiB/0B/0B
+2022-12-08 16:11:19.985,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:19.985,submitted,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.147MiB/s,18.225MiB/s,3.131GiB/1.891GiB/1.898GiB
+2022-12-08 16:11:19.985,submitted,1,T4,2.128 GiB,8.513 GiB,6,52,14.683GiB,8/8,36/36,19.113MiB/s,19.226MiB/s,11.748GiB/6.923GiB/6.964GiB
+2022-12-08 16:11:19.985,submitted,2,T4,8.513 GiB,34.052 GiB,4,32,35.262GiB,2/2,8/8,0B/s,0B/s,10.068GiB/0B/0B
+2022-12-08 16:11:19.985,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:21.141,completed,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.172MiB/s,18.331MiB/s,3.131GiB/1.913GiB/1.929GiB
+2022-12-08 16:11:21.141,completed,1,T4,2.128 GiB,8.513 GiB,2,52,14.683GiB,8/8,36/36,25.352MiB/s,27.269MiB/s,11.748GiB/6.962GiB/7.011GiB
+2022-12-08 16:11:21.141,completed,2,T4,8.513 GiB,34.052 GiB,3,32,35.262GiB,2/2,8/8,9.179MiB/s,10.687MiB/s,10.068GiB/10.567MiB/12.295MiB
+2022-12-08 16:11:21.141,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:21.151,submitted,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.170MiB/s,18.333MiB/s,3.131GiB/1.913GiB/1.929GiB
+2022-12-08 16:11:21.151,submitted,1,T4,2.128 GiB,8.513 GiB,2,48,13.586GiB,7/7,32/32,22.110MiB/s,24.114MiB/s,10.495GiB/5.709GiB/5.753GiB
+2022-12-08 16:11:21.151,submitted,2,T4,8.513 GiB,34.052 GiB,4,33,36.363GiB,3/3,12/12,9.100MiB/s,10.748MiB/s,15.103GiB/10.567MiB/12.473MiB
+2022-12-08 16:11:21.151,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:23.031,completed,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.472MiB/s,18.535MiB/s,3.131GiB/1.979GiB/1.985GiB
+2022-12-08 16:11:23.031,completed,1,T4,2.128 GiB,8.513 GiB,2,48,13.586GiB,7/7,32/32,28.387MiB/s,28.639MiB/s,10.495GiB/5.798GiB/5.830GiB
+2022-12-08 16:11:23.031,completed,2,T4,8.513 GiB,34.052 GiB,3,33,36.363GiB,3/3,12/12,14.080MiB/s,18.068MiB/s,15.103GiB/40.260MiB/47.880MiB
+2022-12-08 16:11:23.031,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:23.033,submitted,0,T4,0.000 B,2.128 GiB,3,32,4.388GiB,5/5,20/20,18.472MiB/s,18.535MiB/s,3.131GiB/1.979GiB/1.986GiB
+2022-12-08 16:11:23.033,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.488GiB,6/6,28/28,25.207MiB/s,25.454MiB/s,9.242GiB/4.545GiB/4.571GiB
+2022-12-08 16:11:23.033,submitted,2,T4,8.513 GiB,34.052 GiB,4,34,37.465GiB,4/4,16/16,14.072MiB/s,18.098MiB/s,20.137GiB/40.260MiB/47.982MiB
+2022-12-08 16:11:23.033,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:23.754,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.074GiB,7/6,28/24,18.406MiB/s,18.596MiB/s,4.384GiB/1.984GiB/2.005GiB
+2022-12-08 16:11:23.754,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.488GiB,6/6,28/28,22.814MiB/s,25.414MiB/s,9.242GiB/4.545GiB/4.596GiB
+2022-12-08 16:11:23.754,submitted,2,T4,8.513 GiB,34.052 GiB,3,34,37.465GiB,4/4,16/16,14.263MiB/s,22.906MiB/s,20.137GiB/48.244MiB/64.928MiB
+2022-12-08 16:11:23.754,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:23.756,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.074GiB,7/7,28/28,18.405MiB/s,18.596MiB/s,4.384GiB/1.984GiB/2.005GiB
+2022-12-08 16:11:23.756,submitted,1,T4,2.128 GiB,8.513 GiB,2,44,12.488GiB,6/6,28/28,22.809MiB/s,25.412MiB/s,9.242GiB/4.545GiB/4.596GiB
+2022-12-08 16:11:23.756,submitted,2,T4,8.513 GiB,34.052 GiB,3,34,37.465GiB,4/4,16/16,14.255MiB/s,22.891MiB/s,20.137GiB/48.244MiB/64.945MiB
+2022-12-08 16:11:23.762,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:44.727,completed,0,T4,0.000 B,2.128 GiB,3,37,5.074GiB,7/7,28/28,34.262MiB/s,34.798MiB/s,4.384GiB/2.954GiB/2.976GiB
+2022-12-08 16:11:44.728,completed,1,T4,2.128 GiB,8.513 GiB,2,44,12.488GiB,6/6,28/28,27.099MiB/s,27.558MiB/s,9.242GiB/5.386GiB/5.435GiB
+2022-12-08 16:11:44.728,completed,2,T4,8.513 GiB,34.052 GiB,3,34,37.465GiB,4/4,16/16,27.856MiB/s,28.415MiB/s,20.137GiB/659.510MiB/672.768MiB
+2022-12-08 16:11:44.729,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:45.367,completed,0,T4,0.000 B,2.128 GiB,3,33,4.526GiB,6/6,24/24,30.450MiB/s,30.818MiB/s,3.758GiB/2.358GiB/2.372GiB
+2022-12-08 16:11:45.367,completed,1,T4,2.128 GiB,8.513 GiB,3,46,13.037GiB,6/6,28/28,27.267MiB/s,27.508MiB/s,9.242GiB/5.420GiB/5.457GiB
+2022-12-08 16:11:45.367,completed,2,T4,8.513 GiB,34.052 GiB,3,34,37.465GiB,4/4,16/16,28.094MiB/s,28.326MiB/s,20.137GiB/683.096MiB/688.799MiB
+2022-12-08 16:11:45.367,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:45.941,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,6/6,24/24,26.465MiB/s,26.882MiB/s,3.758GiB/1.745GiB/1.764GiB
+2022-12-08 16:11:45.941,submitted,1,T4,2.128 GiB,8.513 GiB,3,48,13.586GiB,6/6,28/28,27.001MiB/s,27.515MiB/s,9.242GiB/5.441GiB/5.482GiB
+2022-12-08 16:11:45.942,submitted,2,T4,8.513 GiB,34.052 GiB,3,34,37.465GiB,4/4,16/16,27.444MiB/s,28.399MiB/s,20.137GiB/683.096MiB/706.886MiB
+2022-12-08 16:11:45.942,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:45.950,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,7/7,28/28,26.458MiB/s,26.882MiB/s,4.384GiB/1.745GiB/1.764GiB
+2022-12-08 16:11:45.951,submitted,1,T4,2.128 GiB,8.513 GiB,3,48,13.586GiB,6/6,28/28,26.996MiB/s,27.515MiB/s,9.242GiB/5.441GiB/5.482GiB
+2022-12-08 16:11:45.951,submitted,2,T4,8.513 GiB,34.052 GiB,3,34,37.465GiB,4/4,16/16,27.434MiB/s,28.397MiB/s,20.137GiB/683.096MiB/707.094MiB
+2022-12-08 16:11:45.951,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:45.953,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,8/8,32/32,26.456MiB/s,26.883MiB/s,5.011GiB/1.745GiB/1.764GiB
+2022-12-08 16:11:45.953,submitted,1,T4,2.128 GiB,8.513 GiB,3,48,13.586GiB,6/6,28/28,26.994MiB/s,27.516MiB/s,9.242GiB/5.441GiB/5.483GiB
+2022-12-08 16:11:45.954,submitted,2,T4,8.513 GiB,34.052 GiB,3,34,37.465GiB,4/4,16/16,27.431MiB/s,28.395MiB/s,20.137GiB/683.096MiB/707.127MiB
+2022-12-08 16:11:45.954,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:51.611,completed,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,46.943MiB/s,49.127MiB/s,5.011GiB/2.043GiB/2.073GiB
+2022-12-08 16:11:51.611,completed,1,T4,2.128 GiB,8.513 GiB,3,48,13.586GiB,6/6,28/28,27.588MiB/s,27.962MiB/s,9.242GiB/5.699GiB/5.745GiB
+2022-12-08 16:11:51.611,completed,2,T4,8.513 GiB,34.052 GiB,3,34,37.465GiB,4/4,16/16,27.796MiB/s,28.171MiB/s,20.137GiB/849.683MiB/861.139MiB
+2022-12-08 16:11:51.611,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:51.612,submitted,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,46.937MiB/s,49.126MiB/s,5.011GiB/2.043GiB/2.073GiB
+2022-12-08 16:11:51.612,submitted,1,T4,2.128 GiB,8.513 GiB,3,44,12.419GiB,5/5,24/24,24.076MiB/s,24.434MiB/s,7.910GiB/4.367GiB/4.407GiB
+2022-12-08 16:11:51.612,submitted,2,T4,8.513 GiB,34.052 GiB,4,35,38.637GiB,5/5,20/20,27.794MiB/s,28.169MiB/s,25.250GiB/849.683MiB/861.150MiB
+2022-12-08 16:11:51.612,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:52.064,completed,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,45.518MiB/s,48.871MiB/s,5.011GiB/2.058GiB/2.097GiB
+2022-12-08 16:11:52.064,completed,1,T4,2.128 GiB,8.513 GiB,3,44,12.419GiB,5/5,24/24,23.907MiB/s,24.418MiB/s,7.910GiB/4.381GiB/4.418GiB
+2022-12-08 16:11:52.064,completed,2,T4,8.513 GiB,34.052 GiB,3,35,38.637GiB,5/5,20/20,27.389MiB/s,33.634MiB/s,25.250GiB/849.683MiB/874.623MiB
+2022-12-08 16:11:52.064,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:52.066,submitted,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,45.511MiB/s,48.868MiB/s,5.011GiB/2.058GiB/2.097GiB
+2022-12-08 16:11:52.066,submitted,1,T4,2.128 GiB,8.513 GiB,3,40,11.252GiB,4/4,20/20,20.384MiB/s,20.879MiB/s,6.578GiB/3.049GiB/3.079GiB
+2022-12-08 16:11:52.066,submitted,2,T4,8.513 GiB,34.052 GiB,4,36,39.809GiB,6/6,24/24,27.387MiB/s,33.684MiB/s,30.365GiB/849.683MiB/874.710MiB
+2022-12-08 16:11:52.066,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:54.891,completed,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,47.544MiB/s,49.581MiB/s,5.011GiB/2.212GiB/2.246GiB
+2022-12-08 16:11:54.891,completed,1,T4,2.128 GiB,8.513 GiB,3,40,11.252GiB,4/4,20/20,20.448MiB/s,20.846MiB/s,6.578GiB/3.122GiB/3.147GiB
+2022-12-08 16:11:54.891,completed,2,T4,8.513 GiB,34.052 GiB,3,36,39.809GiB,6/6,24/24,39.599MiB/s,42.665MiB/s,30.365GiB/973.326MiB/1000.136MiB
+2022-12-08 16:11:54.892,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:54.893,submitted,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,47.540MiB/s,49.581MiB/s,5.011GiB/2.212GiB/2.246GiB
+2022-12-08 16:11:54.893,submitted,1,T4,2.128 GiB,8.513 GiB,3,36,10.086GiB,3/3,16/16,16.953MiB/s,17.336MiB/s,5.247GiB/1.791GiB/1.810GiB
+2022-12-08 16:11:54.893,submitted,2,T4,8.513 GiB,34.052 GiB,4,37,40.980GiB,7/7,28/28,39.592MiB/s,42.659MiB/s,35.478GiB/973.326MiB/1000.185MiB
+2022-12-08 16:11:54.893,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:54.895,completed,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,47.534MiB/s,49.581MiB/s,5.011GiB/2.212GiB/2.247GiB
+2022-12-08 16:11:54.895,completed,1,T4,2.128 GiB,8.513 GiB,3,36,10.086GiB,3/3,16/16,16.952MiB/s,17.337MiB/s,5.247GiB/1.791GiB/1.810GiB
+2022-12-08 16:11:54.895,completed,2,T4,8.513 GiB,34.052 GiB,4,37,40.980GiB,7/7,28/28,39.581MiB/s,42.657MiB/s,35.478GiB/973.326MiB/1000.291MiB
+2022-12-08 16:11:54.895,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:11:54.899,submitted,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,47.522MiB/s,49.581MiB/s,5.011GiB/2.212GiB/2.247GiB
+2022-12-08 16:11:54.899,submitted,1,T4,2.128 GiB,8.513 GiB,3,32,8.919GiB,2/2,12/12,13.468MiB/s,13.837MiB/s,3.915GiB/470.118MiB/482.999MiB
+2022-12-08 16:11:54.899,submitted,2,T4,8.513 GiB,34.052 GiB,4,38,42.151GiB,8/8,32/32,39.562MiB/s,42.661MiB/s,40.592GiB/973.326MiB/1000.460MiB
+2022-12-08 16:11:54.900,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:04.056,completed,0,T4,0.000 B,2.128 GiB,2,39,5.348GiB,8/8,32/32,49.917MiB/s,51.486MiB/s,5.011GiB/2.743GiB/2.782GiB
+2022-12-08 16:12:04.056,completed,1,T4,2.128 GiB,8.513 GiB,3,32,8.919GiB,2/2,12/12,13.677MiB/s,13.966MiB/s,3.915GiB/602.614MiB/615.354MiB
+2022-12-08 16:12:04.056,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,56.412MiB/s,58.179MiB/s,40.592GiB/1.471GiB/1.510GiB
+2022-12-08 16:12:04.056,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:06.830,completed,0,T4,0.000 B,2.128 GiB,2,35,4.800GiB,7/7,28/28,46.674MiB/s,48.043MiB/s,4.384GiB/2.281GiB/2.325GiB
+2022-12-08 16:12:06.831,completed,1,T4,2.128 GiB,8.513 GiB,3,34,9.468GiB,2/2,12/12,14.023MiB/s,14.333MiB/s,3.915GiB/656.802MiB/671.318MiB
+2022-12-08 16:12:06.831,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,58.440MiB/s,62.027MiB/s,40.592GiB/1.680GiB/1.744GiB
+2022-12-08 16:12:06.831,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:09.383,completed,0,T4,0.000 B,2.128 GiB,2,31,4.251GiB,6/6,24/24,44.284MiB/s,44.628MiB/s,3.758GiB/1.840GiB/1.851GiB
+2022-12-08 16:12:09.384,completed,1,T4,2.128 GiB,8.513 GiB,3,36,10.017GiB,2/2,12/12,14.360MiB/s,14.670MiB/s,3.915GiB/709.258MiB/724.548MiB
+2022-12-08 16:12:09.384,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,63.895MiB/s,65.569MiB/s,40.592GiB/1.938GiB/1.982GiB
+2022-12-08 16:12:09.384,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:22.993,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.074GiB,7/6,28/24,47.455MiB/s,48.774MiB/s,4.385GiB/2.092GiB/2.151GiB
+2022-12-08 16:12:22.993,submitted,1,T4,2.128 GiB,8.513 GiB,3,38,10.566GiB,2/2,12/12,16.894MiB/s,17.312MiB/s,3.915GiB/1.039GiB/1.065GiB
+2022-12-08 16:12:22.993,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,80.104MiB/s,82.424MiB/s,40.592GiB/3.447GiB/3.533GiB
+2022-12-08 16:12:22.993,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:23.003,submitted,0,T4,0.000 B,2.128 GiB,4,37,5.074GiB,7/7,28/28,47.443MiB/s,48.785MiB/s,4.385GiB/2.092GiB/2.152GiB
+2022-12-08 16:12:23.004,submitted,1,T4,2.128 GiB,8.513 GiB,3,38,10.566GiB,2/2,12/12,16.892MiB/s,17.314MiB/s,3.915GiB/1.039GiB/1.065GiB
+2022-12-08 16:12:23.004,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,80.083MiB/s,82.435MiB/s,40.592GiB/3.447GiB/3.534GiB
+2022-12-08 16:12:23.004,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:30.135,completed,0,T4,0.000 B,2.128 GiB,3,37,5.074GiB,7/7,28/28,82.564MiB/s,84.162MiB/s,4.385GiB/2.832GiB/2.886GiB
+2022-12-08 16:12:30.135,completed,1,T4,2.128 GiB,8.513 GiB,3,38,10.566GiB,2/2,12/12,18.310MiB/s,18.698MiB/s,3.915GiB/1.254GiB/1.281GiB
+2022-12-08 16:12:30.136,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.504MiB/s,88.734MiB/s,40.592GiB/4.344GiB/4.410GiB
+2022-12-08 16:12:30.136,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:30.137,submitted,0,T4,0.000 B,2.128 GiB,3,33,4.526GiB,6/6,24/24,72.892MiB/s,74.497MiB/s,3.758GiB/2.206GiB/2.260GiB
+2022-12-08 16:12:30.137,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,11.115GiB,3/3,16/16,18.310MiB/s,18.699MiB/s,5.168GiB/1.254GiB/1.281GiB
+2022-12-08 16:12:30.137,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.501MiB/s,88.736MiB/s,40.592GiB/4.344GiB/4.410GiB
+2022-12-08 16:12:30.137,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:30.147,submitted,0,T4,0.000 B,2.128 GiB,3,33,4.526GiB,6/6,24/24,72.836MiB/s,74.508MiB/s,3.758GiB/2.206GiB/2.261GiB
+2022-12-08 16:12:30.147,submitted,1,T4,2.128 GiB,8.513 GiB,4,40,11.115GiB,4/4,20/20,18.307MiB/s,18.700MiB/s,6.421GiB/1.254GiB/1.281GiB
+2022-12-08 16:12:30.148,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.481MiB/s,88.743MiB/s,40.592GiB/4.344GiB/4.411GiB
+2022-12-08 16:12:30.148,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:30.481,submitted,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,8/7,32/28,72.392MiB/s,74.574MiB/s,5.011GiB/2.266GiB/2.288GiB
+2022-12-08 16:12:30.482,submitted,1,T4,2.128 GiB,8.513 GiB,3,40,11.115GiB,4/4,20/20,18.387MiB/s,45.123MiB/s,6.421GiB/1.266GiB/1.300GiB
+2022-12-08 16:12:30.482,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.663MiB/s,88.982MiB/s,40.592GiB/4.387GiB/4.452GiB
+2022-12-08 16:12:30.482,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:30.491,submitted,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,8/8,32/32,72.341MiB/s,74.582MiB/s,5.011GiB/2.266GiB/2.289GiB
+2022-12-08 16:12:30.491,submitted,1,T4,2.128 GiB,8.513 GiB,3,40,11.115GiB,4/4,20/20,18.385MiB/s,45.532MiB/s,6.421GiB/1.266GiB/1.300GiB
+2022-12-08 16:12:30.492,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.644MiB/s,88.987MiB/s,40.592GiB/4.387GiB/4.453GiB
+2022-12-08 16:12:30.492,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:31.104,completed,0,T4,0.000 B,2.128 GiB,4,38,5.212GiB,9/8,36/32,73.540MiB/s,98.240MiB/s,5.637GiB/2.301GiB/2.347GiB
+2022-12-08 16:12:31.104,completed,1,T4,2.128 GiB,8.513 GiB,3,40,11.115GiB,4/4,20/20,18.450MiB/s,44.230MiB/s,6.421GiB/1.281GiB/1.330GiB
+2022-12-08 16:12:31.104,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.349MiB/s,89.262MiB/s,40.592GiB/4.432GiB/4.522GiB
+2022-12-08 16:12:31.104,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:31.105,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,8/8,32/32,64.010MiB/s,88.710MiB/s,5.011GiB/1.675GiB/1.720GiB
+2022-12-08 16:12:31.105,submitted,1,T4,2.128 GiB,8.513 GiB,4,42,11.663GiB,6/4,28/20,18.450MiB/s,44.240MiB/s,8.928GiB/1.281GiB/1.330GiB
+2022-12-08 16:12:31.105,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.346MiB/s,89.263MiB/s,40.592GiB/4.432GiB/4.522GiB
+2022-12-08 16:12:31.105,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:31.106,submitted,0,T4,0.000 B,2.128 GiB,4,34,4.663GiB,8/8,32/32,64.007MiB/s,88.707MiB/s,5.011GiB/1.675GiB/1.720GiB
+2022-12-08 16:12:31.106,submitted,1,T4,2.128 GiB,8.513 GiB,4,42,11.663GiB,6/5,28/24,18.449MiB/s,44.253MiB/s,8.928GiB/1.281GiB/1.330GiB
+2022-12-08 16:12:31.106,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.344MiB/s,89.263MiB/s,40.592GiB/4.432GiB/4.522GiB
+2022-12-08 16:12:31.110,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:31.111,submitted,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,63.982MiB/s,88.825MiB/s,5.011GiB/1.675GiB/1.721GiB
+2022-12-08 16:12:31.111,submitted,1,T4,2.128 GiB,8.513 GiB,4,42,11.663GiB,6/6,28/28,18.448MiB/s,44.321MiB/s,8.928GiB/1.281GiB/1.330GiB
+2022-12-08 16:12:31.111,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.334MiB/s,89.266MiB/s,40.592GiB/4.432GiB/4.523GiB
+2022-12-08 16:12:31.112,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:42.904,completed,0,T4,0.000 B,2.128 GiB,1,34,4.663GiB,8/8,32/32,91.404MiB/s,94.222MiB/s,5.011GiB/2.718GiB/2.770GiB
+2022-12-08 16:12:42.904,completed,1,T4,2.128 GiB,8.513 GiB,3,42,11.663GiB,6/6,28/28,65.750MiB/s,68.624MiB/s,8.928GiB/2.138GiB/2.189GiB
+2022-12-08 16:12:42.904,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,88.344MiB/s,89.223MiB/s,40.592GiB/5.525GiB/5.576GiB
+2022-12-08 16:12:42.904,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:43.956,completed,0,T4,0.000 B,2.128 GiB,1,30,4.115GiB,7/7,28/28,79.895MiB/s,82.265MiB/s,4.385GiB/2.171GiB/2.212GiB
+2022-12-08 16:12:43.958,completed,1,T4,2.128 GiB,8.513 GiB,3,44,12.212GiB,6/6,28/28,65.036MiB/s,67.886MiB/s,8.928GiB/2.196GiB/2.250GiB
+2022-12-08 16:12:43.958,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.992MiB/s,88.988MiB/s,40.592GiB/5.598GiB/5.655GiB
+2022-12-08 16:12:43.958,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:43.960,submitted,0,T4,0.000 B,2.128 GiB,1,26,3.566GiB,6/6,24/24,68.818MiB/s,71.199MiB/s,3.758GiB/1.544GiB/1.586GiB
+2022-12-08 16:12:43.960,submitted,1,T4,2.128 GiB,8.513 GiB,4,46,12.761GiB,7/7,32/32,65.023MiB/s,67.884MiB/s,10.181GiB/2.196GiB/2.250GiB
+2022-12-08 16:12:43.960,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.987MiB/s,88.988MiB/s,40.592GiB/5.598GiB/5.655GiB
+2022-12-08 16:12:43.960,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:43.965,submitted,0,T4,0.000 B,2.128 GiB,1,26,3.566GiB,6/6,24/24,68.797MiB/s,71.197MiB/s,3.758GiB/1.544GiB/1.586GiB
+2022-12-08 16:12:43.966,submitted,1,T4,2.128 GiB,8.513 GiB,4,46,12.761GiB,8/8,36/36,65.003MiB/s,67.882MiB/s,11.434GiB/2.196GiB/2.250GiB
+2022-12-08 16:12:43.966,submitted,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.979MiB/s,88.986MiB/s,40.592GiB/5.598GiB/5.656GiB
+2022-12-08 16:12:43.966,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:12:46.537,completed,0,T4,0.000 B,2.128 GiB,2,31,4.252GiB,6/6,24/24,67.475MiB/s,69.530MiB/s,3.758GiB/1.689GiB/1.725GiB
+2022-12-08 16:12:46.538,completed,1,T4,2.128 GiB,8.513 GiB,3,46,12.761GiB,8/8,36/36,79.998MiB/s,86.590MiB/s,11.434GiB/2.401GiB/2.450GiB
+2022-12-08 16:12:46.538,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,87.818MiB/s,88.564MiB/s,40.592GiB/5.810GiB/5.859GiB
+2022-12-08 16:12:46.538,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:14.376,completed,0,T4,0.000 B,2.128 GiB,2,27,3.703GiB,5/5,20/20,59.820MiB/s,60.344MiB/s,3.132GiB/2.738GiB/2.761GiB
+2022-12-08 16:13:14.376,completed,1,T4,2.128 GiB,8.513 GiB,4,48,13.310GiB,10/8,44/36,90.044MiB/s,91.090MiB/s,13.940GiB/4.967GiB/5.020GiB
+2022-12-08 16:13:14.376,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,89.443MiB/s,90.003MiB/s,40.592GiB/8.390GiB/8.444GiB
+2022-12-08 16:13:14.376,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:14.872,completed,0,T4,0.000 B,2.128 GiB,2,23,3.154GiB,4/4,16/16,47.493MiB/s,47.739MiB/s,2.505GiB/2.141GiB/2.152GiB
+2022-12-08 16:13:14.872,completed,1,T4,2.128 GiB,8.513 GiB,4,50,13.859GiB,10/8,44/36,89.190MiB/s,91.431MiB/s,13.940GiB/4.981GiB/5.080GiB
+2022-12-08 16:13:14.872,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,89.332MiB/s,90.176MiB/s,40.592GiB/8.430GiB/8.504GiB
+2022-12-08 16:13:14.872,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:23.004,completed,0,T4,0.000 B,2.128 GiB,2,19,2.606GiB,3/3,12/12,36.318MiB/s,36.573MiB/s,1.879GiB/1.855GiB/1.868GiB
+2022-12-08 16:13:23.005,completed,1,T4,2.128 GiB,8.513 GiB,4,52,14.408GiB,10/8,44/36,93.269MiB/s,95.360MiB/s,13.940GiB/5.914GiB/6.025GiB
+2022-12-08 16:13:23.005,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,91.528MiB/s,92.321MiB/s,40.592GiB/9.367GiB/9.445GiB
+2022-12-08 16:13:23.005,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:23.720,completed,0,T4,0.000 B,2.128 GiB,2,15,2.057GiB,2/2,8/8,24.242MiB/s,24.248MiB/s,1.253GiB/1.253GiB/1.253GiB
+2022-12-08 16:13:23.720,completed,1,T4,2.128 GiB,8.513 GiB,4,54,14.957GiB,10/8,44/36,94.051MiB/s,95.726MiB/s,13.940GiB/6.016GiB/6.113GiB
+2022-12-08 16:13:23.720,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,91.689MiB/s,92.569MiB/s,40.592GiB/9.445GiB/9.536GiB
+2022-12-08 16:13:23.720,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:23.943,completed,0,T4,0.000 B,2.128 GiB,2,11,1.509GiB,1/1,4/4,12.143MiB/s,12.147MiB/s,641.518MiB/641.518MiB/641.682MiB
+2022-12-08 16:13:23.944,completed,1,T4,2.128 GiB,8.513 GiB,5,56,15.505GiB,10/8,46/36,93.784MiB/s,95.889MiB/s,14.566GiB/6.029GiB/6.143GiB
+2022-12-08 16:13:23.944,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,91.766MiB/s,92.674MiB/s,40.592GiB/9.478GiB/9.567GiB
+2022-12-08 16:13:23.944,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:58.144,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:58.144,completed,1,T4,2.128 GiB,8.513 GiB,5,58,16.054GiB,10/8,46/36,108.139MiB/s,109.325MiB/s,14.566GiB/10.481GiB/10.598GiB
+2022-12-08 16:13:58.144,completed,2,T4,8.513 GiB,34.052 GiB,3,38,42.151GiB,8/8,32/32,101.769MiB/s,102.559MiB/s,40.592GiB/13.918GiB/14.024GiB
+2022-12-08 16:13:58.144,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:58.148,submitted,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:13:58.148,submitted,1,T4,2.128 GiB,8.513 GiB,5,54,14.957GiB,8/8,37/37,93.552MiB/s,94.683MiB/s,11.747GiB/9.228GiB/9.341GiB
+2022-12-08 16:13:58.148,submitted,2,T4,8.513 GiB,34.052 GiB,3,39,43.253GiB,8/8,32/32,101.766MiB/s,102.561MiB/s,40.592GiB/13.918GiB/14.025GiB
+2022-12-08 16:13:58.148,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:01.768,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:01.768,completed,1,T4,2.128 GiB,8.513 GiB,5,54,14.957GiB,9/8,42/37,110.934MiB/s,114.647MiB/s,13.313GiB/9.750GiB/9.868GiB
+2022-12-08 16:14:01.768,completed,2,T4,8.513 GiB,34.052 GiB,3,39,43.253GiB,8/8,32/32,103.006MiB/s,103.758MiB/s,40.592GiB/14.451GiB/14.557GiB
+2022-12-08 16:14:01.768,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:01.771,submitted,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:01.771,submitted,1,T4,2.128 GiB,8.513 GiB,5,50,13.859GiB,8/8,38/38,96.760MiB/s,100.435MiB/s,12.060GiB/8.497GiB/8.610GiB
+2022-12-08 16:14:01.771,submitted,2,T4,8.513 GiB,34.052 GiB,3,40,44.354GiB,8/8,32/32,103.003MiB/s,103.759MiB/s,40.592GiB/14.451GiB/14.557GiB
+2022-12-08 16:14:01.771,submitted,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:02.182,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:02.182,completed,1,T4,2.128 GiB,8.513 GiB,3,50,13.859GiB,8/8,38/38,99.248MiB/s,115.995MiB/s,12.060GiB/8.578GiB/8.665GiB
+2022-12-08 16:14:02.182,completed,2,T4,8.513 GiB,34.052 GiB,3,40,44.354GiB,8/8,32/32,103.086MiB/s,103.878MiB/s,40.592GiB/14.506GiB/14.616GiB
+2022-12-08 16:14:02.182,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:03.016,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:03.016,completed,1,T4,2.128 GiB,8.513 GiB,3,46,12.762GiB,7/7,34/34,96.823MiB/s,105.189MiB/s,10.808GiB/7.406GiB/7.524GiB
+2022-12-08 16:14:03.016,completed,2,T4,8.513 GiB,34.052 GiB,3,41,45.456GiB,8/8,32/32,103.361MiB/s,104.145MiB/s,40.592GiB/14.631GiB/14.738GiB
+2022-12-08 16:14:03.016,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:04.441,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:04.441,completed,1,T4,2.128 GiB,8.513 GiB,3,42,11.663GiB,6/6,30/30,86.043MiB/s,92.286MiB/s,9.554GiB/6.361GiB/6.434GiB
+2022-12-08 16:14:04.442,completed,2,T4,8.513 GiB,34.052 GiB,3,42,46.558GiB,8/8,32/32,104.137MiB/s,104.734MiB/s,40.592GiB/14.881GiB/14.970GiB
+2022-12-08 16:14:04.442,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:06.261,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:06.261,completed,1,T4,2.128 GiB,8.513 GiB,3,36,9.948GiB,5/5,24/24,78.547MiB/s,81.547MiB/s,7.596GiB/4.635GiB/4.676GiB
+2022-12-08 16:14:06.261,completed,2,T4,8.513 GiB,34.052 GiB,4,43,48.282GiB,9/8,36/32,104.506MiB/s,105.492MiB/s,46.333GiB/15.125GiB/15.264GiB
+2022-12-08 16:14:06.261,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:06.744,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:06.744,completed,1,T4,2.128 GiB,8.513 GiB,3,30,8.233GiB,4/4,18/18,63.738MiB/s,70.392MiB/s,5.638GiB/2.690GiB/2.751GiB
+2022-12-08 16:14:06.744,completed,2,T4,8.513 GiB,34.052 GiB,4,44,50.007GiB,10/8,40/32,104.923MiB/s,105.716MiB/s,52.077GiB/15.231GiB/15.346GiB
+2022-12-08 16:14:06.745,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:08.440,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:14:08.440,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.135GiB,3/3,14/14,52.708MiB/s,55.136MiB/s,4.385GiB/1.565GiB/1.588GiB
+2022-12-08 16:14:08.440,completed,2,T4,8.513 GiB,34.052 GiB,4,45,51.109GiB,10/8,40/32,105.766MiB/s,106.249MiB/s,52.077GiB/15.528GiB/15.601GiB
+2022-12-08 16:14:08.440,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:15:09.808,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:15:09.808,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.038GiB,2/2,10/10,45.410MiB/s,45.923MiB/s,3.132GiB/3.096GiB/3.131GiB
+2022-12-08 16:15:09.808,completed,2,T4,8.513 GiB,34.052 GiB,4,46,52.210GiB,10/8,40/32,125.726MiB/s,126.336MiB/s,52.077GiB/26.024GiB/26.150GiB
+2022-12-08 16:15:09.808,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:15:10.827,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:15:10.827,completed,1,T4,2.128 GiB,8.513 GiB,3,17,4.665GiB,1/1,5/5,23.224MiB/s,23.333MiB/s,1.566GiB/1.566GiB/1.573GiB
+2022-12-08 16:15:10.827,completed,2,T4,8.513 GiB,34.052 GiB,4,47,53.588GiB,9/8,36/32,125.888MiB/s,126.512MiB/s,46.335GiB/26.183GiB/26.313GiB
+2022-12-08 16:15:10.827,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:26.082,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:26.082,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:26.082,completed,2,T4,8.513 GiB,34.052 GiB,4,48,54.966GiB,10/8,40/32,138.515MiB/s,138.904MiB/s,52.077GiB/39.007GiB/39.115GiB
+2022-12-08 16:16:26.082,completed,3,T4,34.052 GiB,136.206 GiB,1,10,44.242GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:26.083,submitted,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:26.083,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:26.083,submitted,2,T4,8.513 GiB,34.052 GiB,4,44,50.559GiB,9/8,36/32,121.676MiB/s,122.036MiB/s,47.044GiB/33.973GiB/34.073GiB
+2022-12-08 16:16:26.083,submitted,3,T4,34.052 GiB,136.206 GiB,2,11,48.667GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.793,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.793,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.794,completed,2,T4,8.513 GiB,34.052 GiB,4,44,50.559GiB,9/8,36/32,135.871MiB/s,147.151MiB/s,47.044GiB/34.206GiB/34.377GiB
+2022-12-08 16:16:27.794,completed,3,T4,34.052 GiB,136.206 GiB,2,11,48.667GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.795,submitted,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.795,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.795,submitted,2,T4,8.513 GiB,34.052 GiB,4,40,46.151GiB,8/8,32/32,119.106MiB/s,130.367MiB/s,42.009GiB/29.172GiB/29.334GiB
+2022-12-08 16:16:27.795,submitted,3,T4,34.052 GiB,136.206 GiB,2,12,53.093GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.967,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.967,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:27.967,completed,2,T4,8.513 GiB,34.052 GiB,1,40,46.151GiB,8/8,32/32,118.060MiB/s,143.280MiB/s,42.009GiB/29.260GiB/29.358GiB
+2022-12-08 16:16:27.967,completed,3,T4,34.052 GiB,136.206 GiB,2,12,53.093GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:29.203,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:29.203,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:29.203,completed,2,T4,8.513 GiB,34.052 GiB,1,36,41.742GiB,7/7,28/28,128.297MiB/s,136.766MiB/s,36.974GiB/24.404GiB/24.504GiB
+2022-12-08 16:16:29.203,completed,3,T4,34.052 GiB,136.206 GiB,2,13,57.520GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:39.456,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:39.456,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:39.456,completed,2,T4,8.513 GiB,34.052 GiB,1,32,37.335GiB,6/6,24/24,120.116MiB/s,122.382MiB/s,31.941GiB/20.802GiB/20.889GiB
+2022-12-08 16:16:39.456,completed,3,T4,34.052 GiB,136.206 GiB,2,14,61.945GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:40.189,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:40.189,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:40.189,completed,2,T4,8.513 GiB,34.052 GiB,1,28,32.857GiB,5/5,20/20,102.911MiB/s,104.051MiB/s,26.827GiB/15.780GiB/15.850GiB
+2022-12-08 16:16:40.189,completed,3,T4,34.052 GiB,136.206 GiB,2,15,66.441GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:41.746,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:41.746,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:41.746,completed,2,T4,8.513 GiB,34.052 GiB,1,24,28.379GiB,4/4,16/16,83.271MiB/s,86.211MiB/s,21.712GiB/10.815GiB/10.877GiB
+2022-12-08 16:16:41.746,completed,3,T4,34.052 GiB,136.206 GiB,2,16,70.938GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:45.898,completed,0,T4,0.000 B,2.128 GiB,3,12,1.563GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:45.898,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:16:45.898,completed,2,T4,8.513 GiB,34.052 GiB,1,20,23.901GiB,3/3,12/12,67.262MiB/s,68.515MiB/s,16.599GiB/6.024GiB/6.056GiB
+2022-12-08 16:16:45.898,completed,3,T4,34.052 GiB,136.206 GiB,2,17,75.433GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:17:25.626,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.249GiB,2/1,8/4,0B/s,0B/s,1.215GiB/0B/0B
+2022-12-08 16:17:25.626,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:17:25.626,submitted,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.635MiB/s,53.201MiB/s,11.486GiB/3.016GiB/3.049GiB
+2022-12-08 16:17:25.626,submitted,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:17:25.627,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.249GiB,2/2,8/8,0B/s,0B/s,1.215GiB/0B/0B
+2022-12-08 16:17:25.627,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:17:25.627,submitted,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.634MiB/s,53.201MiB/s,11.486GiB/3.016GiB/3.049GiB
+2022-12-08 16:17:25.627,submitted,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:17:49.719,completed,0,T4,0.000 B,2.128 GiB,3,17,2.249GiB,2/2,8/8,50.481MiB/s,51.600MiB/s,1.215GiB/1.188GiB/1.214GiB
+2022-12-08 16:17:49.719,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.293GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:17:49.719,completed,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.660MiB/s,53.019MiB/s,11.486GiB/4.257GiB/4.286GiB
+2022-12-08 16:17:49.719,completed,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:17:50.044,completed,0,T4,0.000 B,2.128 GiB,3,13,1.717GiB,1/1,4/4,25.468MiB/s,25.519MiB/s,621.779MiB/621.779MiB/623.028MiB
+2022-12-08 16:17:50.044,completed,1,T4,2.128 GiB,8.513 GiB,3,14,3.826GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:17:50.044,completed,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.772MiB/s,53.006MiB/s,11.486GiB/4.283GiB/4.302GiB
+2022-12-08 16:17:50.045,completed,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:19:39.459,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.871GiB,1/1,4/4,0B/s,0B/s,622.417MiB/0B/0B
+2022-12-08 16:19:39.459,submitted,1,T4,2.128 GiB,8.513 GiB,3,16,4.359GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:19:39.459,submitted,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.918MiB/s,53.084MiB/s,11.486GiB/9.949GiB/9.980GiB
+2022-12-08 16:19:39.459,submitted,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:19:39.460,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.871GiB,2/2,8/8,0B/s,0B/s,1.216GiB/0B/0B
+2022-12-08 16:19:39.460,submitted,1,T4,2.128 GiB,8.513 GiB,3,16,4.359GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:19:39.460,submitted,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.917MiB/s,53.084MiB/s,11.486GiB/9.949GiB/9.980GiB
+2022-12-08 16:19:39.460,submitted,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:19:39.462,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.871GiB,3/3,12/12,0B/s,0B/s,1.823GiB/0B/0B
+2022-12-08 16:19:39.462,submitted,1,T4,2.128 GiB,8.513 GiB,3,16,4.359GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:19:39.462,submitted,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.917MiB/s,53.084MiB/s,11.486GiB/9.949GiB/9.980GiB
+2022-12-08 16:19:39.462,submitted,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:04.425,completed,0,T4,0.000 B,2.128 GiB,1,14,1.871GiB,3/3,12/12,74.032MiB/s,75.000MiB/s,1.823GiB/1.805GiB/1.828GiB
+2022-12-08 16:20:04.425,completed,1,T4,2.128 GiB,8.513 GiB,3,16,4.359GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:04.425,completed,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.310MiB/s,52.438MiB/s,11.486GiB/11.110GiB/11.137GiB
+2022-12-08 16:20:04.426,completed,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:04.427,submitted,0,T4,0.000 B,2.128 GiB,1,10,1.339GiB,2/2,8/8,49.098MiB/s,49.979MiB/s,1.215GiB/1.197GiB/1.218GiB
+2022-12-08 16:20:04.427,submitted,1,T4,2.128 GiB,8.513 GiB,4,18,4.893GiB,2/1,8/4,0B/s,0B/s,2.489GiB/0B/0B
+2022-12-08 16:20:04.427,submitted,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.310MiB/s,52.438MiB/s,11.486GiB/11.110GiB/11.137GiB
+2022-12-08 16:20:04.427,submitted,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:04.431,submitted,0,T4,0.000 B,2.128 GiB,1,10,1.339GiB,2/2,8/8,49.089MiB/s,49.975MiB/s,1.215GiB/1.197GiB/1.219GiB
+2022-12-08 16:20:04.431,submitted,1,T4,2.128 GiB,8.513 GiB,4,18,4.893GiB,2/2,8/8,0B/s,0B/s,2.489GiB/0B/0B
+2022-12-08 16:20:04.431,submitted,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.308MiB/s,52.438MiB/s,11.486GiB/11.110GiB/11.137GiB
+2022-12-08 16:20:04.431,submitted,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:04.905,completed,0,T4,0.000 B,2.128 GiB,1,10,1.339GiB,2/2,8/8,48.895MiB/s,49.068MiB/s,1.215GiB/1.215GiB/1.219GiB
+2022-12-08 16:20:04.905,completed,1,T4,2.128 GiB,8.513 GiB,2,18,4.893GiB,2/2,8/8,0B/s,39.268MiB/s,2.489GiB/0B/18.547MiB
+2022-12-08 16:20:04.905,completed,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.195MiB/s,52.401MiB/s,11.486GiB/11.110GiB/11.154GiB
+2022-12-08 16:20:04.905,completed,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:04.915,completed,0,T4,0.000 B,2.128 GiB,1,6,826.215MiB,1/1,4/4,24.449MiB/s,24.536MiB/s,622.272MiB/622.272MiB/624.473MiB
+2022-12-08 16:20:04.915,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.427GiB,2/2,8/8,0B/s,38.835MiB/s,2.489GiB/0B/18.724MiB
+2022-12-08 16:20:04.915,completed,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.192MiB/s,52.399MiB/s,11.486GiB/11.110GiB/11.154GiB
+2022-12-08 16:20:04.915,completed,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:11.284,completed,0,T4,0.000 B,2.128 GiB,1,2,281.128MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:11.284,completed,1,T4,2.128 GiB,8.513 GiB,2,22,5.960GiB,2/2,8/8,46.363MiB/s,52.959MiB/s,2.489GiB/317.642MiB/362.838MiB
+2022-12-08 16:20:11.284,completed,2,T4,8.513 GiB,34.052 GiB,1,16,19.424GiB,2/2,8/8,52.225MiB/s,52.311MiB/s,11.486GiB/11.442GiB/11.460GiB
+2022-12-08 16:20:11.284,completed,3,T4,34.052 GiB,136.206 GiB,2,18,79.928GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:13.053,completed,0,T4,0.000 B,2.128 GiB,1,2,281.128MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:13.053,completed,1,T4,2.128 GiB,8.513 GiB,2,22,5.960GiB,2/2,8/8,49.393MiB/s,53.455MiB/s,2.489GiB/425.773MiB/460.788MiB
+2022-12-08 16:20:13.053,completed,2,T4,8.513 GiB,34.052 GiB,1,12,14.396GiB,1/1,4/4,26.111MiB/s,26.149MiB/s,5.744GiB/5.744GiB/5.752GiB
+2022-12-08 16:20:13.053,completed,3,T4,34.052 GiB,136.206 GiB,2,19,84.976GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:50.633,completed,0,T4,0.000 B,2.128 GiB,1,2,281.128MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:50.633,completed,1,T4,2.128 GiB,8.513 GiB,2,22,5.960GiB,2/2,8/8,55.048MiB/s,55.255MiB/s,2.489GiB/2.484GiB/2.493GiB
+2022-12-08 16:20:50.633,completed,2,T4,8.513 GiB,34.052 GiB,1,8,9.365GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:50.633,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:50.959,completed,0,T4,0.000 B,2.128 GiB,1,2,281.128MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:50.959,completed,1,T4,2.128 GiB,8.513 GiB,2,18,4.870GiB,1/1,4/4,27.382MiB/s,27.467MiB/s,1.244GiB/1.244GiB/1.248GiB
+2022-12-08 16:20:50.959,completed,2,T4,8.513 GiB,34.052 GiB,1,9,10.460GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:20:50.959,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:22.153,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.333GiB,2/1,8/4,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:26:22.153,submitted,1,T4,2.128 GiB,8.513 GiB,2,14,3.780GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:22.153,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:22.153,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:22.154,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.333GiB,2/2,8/8,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:26:22.155,submitted,1,T4,2.128 GiB,8.513 GiB,2,14,3.780GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:22.155,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:22.155,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:45.856,completed,0,T4,0.000 B,2.128 GiB,3,17,2.333GiB,2/2,8/8,53.107MiB/s,54.074MiB/s,1.253GiB/1.229GiB/1.251GiB
+2022-12-08 16:26:45.856,completed,1,T4,2.128 GiB,8.513 GiB,2,14,3.780GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:45.856,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:45.856,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:46.272,completed,0,T4,0.000 B,2.128 GiB,3,13,1.784GiB,1/1,4/4,26.607MiB/s,26.704MiB/s,641.679MiB/641.679MiB/644.006MiB
+2022-12-08 16:26:46.272,completed,1,T4,2.128 GiB,8.513 GiB,3,16,4.331GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:46.272,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:26:46.272,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.369,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,2/1,8/4,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:28:36.369,submitted,1,T4,2.128 GiB,8.513 GiB,3,18,4.881GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.369,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.370,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.371,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,3/2,12/8,0B/s,0B/s,1.879GiB/0B/0B
+2022-12-08 16:28:36.371,submitted,1,T4,2.128 GiB,8.513 GiB,3,18,4.881GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.371,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.371,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.372,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,3/3,12/12,0B/s,0B/s,1.879GiB/0B/0B
+2022-12-08 16:28:36.372,submitted,1,T4,2.128 GiB,8.513 GiB,3,18,4.881GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.372,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:28:36.372,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.152,completed,0,T4,0.000 B,2.128 GiB,1,14,1.921GiB,3/3,12/12,77.671MiB/s,77.952MiB/s,1.879GiB/1.879GiB/1.886GiB
+2022-12-08 16:29:01.153,completed,1,T4,2.128 GiB,8.513 GiB,3,18,4.881GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.153,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.153,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.184,completed,0,T4,0.000 B,2.128 GiB,1,10,1.373GiB,2/2,8/8,51.735MiB/s,51.923MiB/s,1.253GiB/1.253GiB/1.258GiB
+2022-12-08 16:29:01.184,completed,1,T4,2.128 GiB,8.513 GiB,3,20,5.431GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.184,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.184,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.210,completed,0,T4,0.000 B,2.128 GiB,1,6,843.968MiB,1/1,4/4,25.869MiB/s,25.963MiB/s,642.462MiB/642.462MiB/644.790MiB
+2022-12-08 16:29:01.210,completed,1,T4,2.128 GiB,8.513 GiB,3,22,5.981GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.210,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:29:01.210,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:35:50.988,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.333GiB,1/1,4/4,0B/s,0B/s,641.278MiB/0B/0B
+2022-12-08 16:35:50.989,submitted,1,T4,2.128 GiB,8.513 GiB,3,24,6.532GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:35:50.989,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:35:50.989,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:35:50.990,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.333GiB,2/2,8/8,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:35:50.991,submitted,1,T4,2.128 GiB,8.513 GiB,3,24,6.532GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:35:50.991,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:35:50.991,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.123,completed,0,T4,0.000 B,2.128 GiB,3,17,2.333GiB,2/2,8/8,38.735MiB/s,38.877MiB/s,1.253GiB/1.253GiB/1.258GiB
+2022-12-08 16:36:24.123,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.532GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.123,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.123,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.124,submitted,0,T4,0.000 B,2.128 GiB,3,13,1.783GiB,1/1,4/4,19.354MiB/s,19.425MiB/s,641.278MiB/641.278MiB/643.619MiB
+2022-12-08 16:36:24.124,submitted,1,T4,2.128 GiB,8.513 GiB,4,26,7.083GiB,1/1,4/4,0B/s,0B/s,1.247GiB/0B/0B
+2022-12-08 16:36:24.124,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.124,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.131,submitted,0,T4,0.000 B,2.128 GiB,3,13,1.783GiB,1/1,4/4,19.350MiB/s,19.421MiB/s,641.278MiB/641.278MiB/643.619MiB
+2022-12-08 16:36:24.131,submitted,1,T4,2.128 GiB,8.513 GiB,4,26,7.083GiB,2/2,8/8,0B/s,0B/s,2.493GiB/0B/0B
+2022-12-08 16:36:24.131,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.131,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.178,completed,0,T4,0.000 B,2.128 GiB,3,13,1.783GiB,1/1,4/4,19.323MiB/s,19.393MiB/s,641.278MiB/641.278MiB/643.619MiB
+2022-12-08 16:36:24.178,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.083GiB,2/2,8/8,0B/s,20.059MiB/s,2.493GiB/0B/945.046KiB
+2022-12-08 16:36:24.178,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.178,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.180,submitted,0,T4,0.000 B,2.128 GiB,3,9,1.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.180,submitted,1,T4,2.128 GiB,8.513 GiB,4,28,7.633GiB,3/3,12/12,0B/s,20.505MiB/s,3.739GiB/0B/1000.260KiB
+2022-12-08 16:36:24.180,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.180,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.182,submitted,0,T4,0.000 B,2.128 GiB,3,9,1.235GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.182,submitted,1,T4,2.128 GiB,8.513 GiB,4,28,7.633GiB,4/4,16/16,0B/s,21.034MiB/s,4.986GiB/0B/1.046MiB
+2022-12-08 16:36:24.182,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:36:24.182,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:01.652,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,3/1,12/4,0B/s,0B/s,1.880GiB/0B/0B
+2022-12-08 16:37:01.652,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,7.633GiB,4/4,16/16,67.660MiB/s,68.547MiB/s,4.986GiB/2.477GiB/2.510GiB
+2022-12-08 16:37:01.652,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:01.652,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:01.663,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,2/2,8/8,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:37:01.663,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,7.633GiB,4/4,16/16,67.640MiB/s,68.548MiB/s,4.986GiB/2.477GiB/2.511GiB
+2022-12-08 16:37:01.663,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:01.663,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:01.665,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,3/3,12/12,0B/s,0B/s,1.880GiB/0B/0B
+2022-12-08 16:37:01.665,submitted,1,T4,2.128 GiB,8.513 GiB,3,28,7.633GiB,4/4,16/16,67.637MiB/s,68.548MiB/s,4.986GiB/2.477GiB/2.511GiB
+2022-12-08 16:37:01.665,submitted,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:01.665,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.224,completed,0,T4,0.000 B,2.128 GiB,1,14,1.921GiB,3/3,12/12,47.453MiB/s,47.713MiB/s,1.880GiB/1.833GiB/1.843GiB
+2022-12-08 16:37:41.224,completed,1,T4,2.128 GiB,8.513 GiB,3,28,7.633GiB,4/4,16/16,63.892MiB/s,64.431MiB/s,4.986GiB/4.808GiB/4.849GiB
+2022-12-08 16:37:41.224,completed,2,T4,8.513 GiB,34.052 GiB,1,10,11.553GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.224,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.267,completed,0,T4,0.000 B,2.128 GiB,1,14,1.921GiB,3/3,12/12,47.401MiB/s,47.679MiB/s,1.880GiB/1.833GiB/1.844GiB
+2022-12-08 16:37:41.267,completed,1,T4,2.128 GiB,8.513 GiB,3,24,6.541GiB,3/3,12/12,47.297MiB/s,47.828MiB/s,3.739GiB/3.562GiB/3.602GiB
+2022-12-08 16:37:41.267,completed,2,T4,8.513 GiB,34.052 GiB,2,11,12.648GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.267,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.307,completed,0,T4,0.000 B,2.128 GiB,1,10,1.372GiB,2/2,8/8,31.164MiB/s,31.389MiB/s,1.253GiB/1.206GiB/1.215GiB
+2022-12-08 16:37:41.307,completed,1,T4,2.128 GiB,8.513 GiB,3,26,7.092GiB,3/3,12/12,47.272MiB/s,47.826MiB/s,3.739GiB/3.562GiB/3.604GiB
+2022-12-08 16:37:41.307,completed,2,T4,8.513 GiB,34.052 GiB,2,11,12.648GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.307,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.309,submitted,0,T4,0.000 B,2.128 GiB,1,6,842.689MiB,1/1,4/4,14.975MiB/s,15.129MiB/s,641.307MiB/593.662MiB/599.755MiB
+2022-12-08 16:37:41.309,submitted,1,T4,2.128 GiB,8.513 GiB,4,28,7.642GiB,5/4,20/16,47.271MiB/s,47.826MiB/s,6.233GiB/3.562GiB/3.604GiB
+2022-12-08 16:37:41.309,submitted,2,T4,8.513 GiB,34.052 GiB,2,11,12.648GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.309,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.316,submitted,0,T4,0.000 B,2.128 GiB,1,6,842.689MiB,1/1,4/4,14.973MiB/s,15.130MiB/s,641.307MiB/593.662MiB/599.896MiB
+2022-12-08 16:37:41.316,submitted,1,T4,2.128 GiB,8.513 GiB,4,28,7.642GiB,5/5,20/20,47.267MiB/s,47.826MiB/s,6.233GiB/3.562GiB/3.604GiB
+2022-12-08 16:37:41.316,submitted,2,T4,8.513 GiB,34.052 GiB,2,11,12.648GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:41.316,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:43.109,completed,0,T4,0.000 B,2.128 GiB,1,6,842.689MiB,1/1,4/4,15.033MiB/s,15.127MiB/s,641.307MiB/622.991MiB/626.906MiB
+2022-12-08 16:37:43.109,completed,1,T4,2.128 GiB,8.513 GiB,2,28,7.642GiB,5/5,20/20,56.948MiB/s,69.944MiB/s,6.233GiB/3.654GiB/3.708GiB
+2022-12-08 16:37:43.109,completed,2,T4,8.513 GiB,34.052 GiB,2,11,12.648GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:43.109,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:43.356,completed,0,T4,0.000 B,2.128 GiB,1,6,842.689MiB,1/1,4/4,14.943MiB/s,15.129MiB/s,641.307MiB/622.991MiB/630.724MiB
+2022-12-08 16:37:43.357,completed,1,T4,2.128 GiB,8.513 GiB,2,24,6.551GiB,4/4,16/16,54.388MiB/s,55.143MiB/s,4.987GiB/2.468GiB/2.474GiB
+2022-12-08 16:37:43.357,completed,2,T4,8.513 GiB,34.052 GiB,2,12,13.743GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:43.357,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:44.249,completed,0,T4,0.000 B,2.128 GiB,1,6,842.689MiB,1/1,4/4,15.060MiB/s,15.127MiB/s,641.307MiB/641.307MiB/644.169MiB
+2022-12-08 16:37:44.250,completed,1,T4,2.128 GiB,8.513 GiB,2,20,5.460GiB,3/3,12/12,31.088MiB/s,42.892MiB/s,3.741GiB/1.222GiB/1.274GiB
+2022-12-08 16:37:44.250,completed,2,T4,8.513 GiB,34.052 GiB,2,13,14.838GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:44.250,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:47.567,completed,0,T4,0.000 B,2.128 GiB,1,2,280.933MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:47.567,completed,1,T4,2.128 GiB,8.513 GiB,3,22,6.010GiB,3/3,12/12,47.656MiB/s,49.205MiB/s,3.741GiB/1.445GiB/1.456GiB
+2022-12-08 16:37:47.567,completed,2,T4,8.513 GiB,34.052 GiB,2,13,14.838GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:37:47.567,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:38:49.119,completed,0,T4,0.000 B,2.128 GiB,2,7,983.330MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:38:49.120,completed,1,T4,2.128 GiB,8.513 GiB,3,18,4.918GiB,2/2,8/8,37.511MiB/s,37.714MiB/s,2.494GiB/2.484GiB/2.497GiB
+2022-12-08 16:38:49.120,completed,2,T4,8.513 GiB,34.052 GiB,2,14,15.934GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:38:49.120,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:38:49.222,completed,0,T4,0.000 B,2.128 GiB,2,7,983.330MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:38:49.223,completed,1,T4,2.128 GiB,8.513 GiB,3,14,3.827GiB,1/1,4/4,18.808MiB/s,18.838MiB/s,1.247GiB/1.247GiB/1.249GiB
+2022-12-08 16:38:49.223,completed,2,T4,8.513 GiB,34.052 GiB,2,15,17.029GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:38:49.223,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:11.647,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.332GiB,1/1,4/4,0B/s,0B/s,641.322MiB/0B/0B
+2022-12-08 16:40:11.647,submitted,1,T4,2.128 GiB,8.513 GiB,3,10,2.735GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:11.647,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:11.647,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:11.649,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.332GiB,2/2,8/8,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:40:11.649,submitted,1,T4,2.128 GiB,8.513 GiB,3,10,2.735GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:11.649,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:11.649,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:43.863,completed,0,T4,0.000 B,2.128 GiB,3,17,2.332GiB,2/2,8/8,39.357MiB/s,39.448MiB/s,1.253GiB/1.238GiB/1.241GiB
+2022-12-08 16:40:43.863,completed,1,T4,2.128 GiB,8.513 GiB,3,10,2.735GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:43.863,completed,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:43.863,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:44.587,completed,0,T4,0.000 B,2.128 GiB,3,13,1.783GiB,1/1,4/4,19.472MiB/s,19.475MiB/s,641.322MiB/641.322MiB/641.420MiB
+2022-12-08 16:40:44.587,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.284GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:44.587,completed,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:40:44.587,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.763,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,2/1,8/4,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:41:18.763,submitted,1,T4,2.128 GiB,8.513 GiB,3,14,3.833GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.763,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.763,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.765,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,2/2,8/8,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:41:18.765,submitted,1,T4,2.128 GiB,8.513 GiB,3,14,3.833GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.765,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.765,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.767,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.921GiB,3/3,12/12,0B/s,0B/s,1.880GiB/0B/0B
+2022-12-08 16:41:18.767,submitted,1,T4,2.128 GiB,8.513 GiB,3,14,3.833GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.767,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:18.767,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:49.552,completed,0,T4,0.000 B,2.128 GiB,1,14,1.921GiB,3/3,12/12,60.202MiB/s,61.411MiB/s,1.880GiB/1.810GiB/1.846GiB
+2022-12-08 16:41:49.552,completed,1,T4,2.128 GiB,8.513 GiB,3,14,3.833GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:49.552,completed,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:49.552,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:49.553,submitted,0,T4,0.000 B,2.128 GiB,1,10,1.372GiB,2/2,8/8,39.359MiB/s,40.567MiB/s,1.253GiB/1.183GiB/1.219GiB
+2022-12-08 16:41:49.553,submitted,1,T4,2.128 GiB,8.513 GiB,4,16,4.382GiB,2/1,8/4,0B/s,0B/s,2.494GiB/0B/0B
+2022-12-08 16:41:49.553,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:49.553,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:49.554,submitted,0,T4,0.000 B,2.128 GiB,1,10,1.372GiB,2/2,8/8,39.358MiB/s,40.568MiB/s,1.253GiB/1.183GiB/1.219GiB
+2022-12-08 16:41:49.554,submitted,1,T4,2.128 GiB,8.513 GiB,4,16,4.382GiB,2/2,8/8,0B/s,0B/s,2.494GiB/0B/0B
+2022-12-08 16:41:49.554,submitted,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:49.554,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:50.056,completed,0,T4,0.000 B,2.128 GiB,1,10,1.372GiB,2/2,8/8,40.618MiB/s,40.760MiB/s,1.253GiB/1.241GiB/1.245GiB
+2022-12-08 16:41:50.056,completed,1,T4,2.128 GiB,8.513 GiB,2,16,4.382GiB,2/2,8/8,0B/s,64.210MiB/s,2.494GiB/0B/32.173MiB
+2022-12-08 16:41:50.056,completed,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:50.056,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:50.452,completed,0,T4,0.000 B,2.128 GiB,1,6,842.677MiB,1/1,4/4,20.247MiB/s,20.250MiB/s,641.418MiB/641.418MiB/641.519MiB
+2022-12-08 16:41:50.452,completed,1,T4,2.128 GiB,8.513 GiB,3,18,4.931GiB,2/2,8/8,0B/s,68.039MiB/s,2.494GiB/0B/61.032MiB
+2022-12-08 16:41:50.452,completed,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:41:50.452,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:42:33.121,completed,0,T4,0.000 B,2.128 GiB,2,7,983.221MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:42:33.121,completed,1,T4,2.128 GiB,8.513 GiB,3,20,5.480GiB,2/2,8/8,57.946MiB/s,58.389MiB/s,2.494GiB/2.465GiB/2.484GiB
+2022-12-08 16:42:33.121,completed,2,T4,8.513 GiB,34.052 GiB,2,16,18.125GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:42:33.121,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:42:34.092,completed,0,T4,0.000 B,2.128 GiB,2,7,983.221MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:42:34.092,completed,1,T4,2.128 GiB,8.513 GiB,3,16,4.389GiB,1/1,4/4,28.676MiB/s,28.771MiB/s,1.247GiB/1.247GiB/1.251GiB
+2022-12-08 16:42:34.092,completed,2,T4,8.513 GiB,34.052 GiB,2,17,19.222GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:42:34.092,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:46:46.209,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.332GiB,1/1,4/4,0B/s,0B/s,642.093MiB/0B/0B
+2022-12-08 16:46:46.209,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,3.297GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:46:46.209,submitted,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:46:46.209,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:46:46.211,submitted,0,T4,0.000 B,2.128 GiB,4,17,2.332GiB,2/2,8/8,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:46:46.211,submitted,1,T4,2.128 GiB,8.513 GiB,3,12,3.297GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:46:46.211,submitted,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:46:46.211,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:47:20.152,completed,0,T4,0.000 B,2.128 GiB,3,17,2.332GiB,2/2,8/8,37.181MiB/s,37.543MiB/s,1.253GiB/1.232GiB/1.244GiB
+2022-12-08 16:47:20.152,completed,1,T4,2.128 GiB,8.513 GiB,3,12,3.297GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:47:20.152,completed,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:47:20.152,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:47:20.913,completed,0,T4,0.000 B,2.128 GiB,3,13,1.783GiB,1/1,4/4,18.488MiB/s,18.524MiB/s,641.478MiB/641.478MiB/642.742MiB
+2022-12-08 16:47:20.913,completed,1,T4,2.128 GiB,8.513 GiB,3,14,3.847GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:47:20.913,completed,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:47:20.913,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.234,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.920GiB,2/1,8/4,0B/s,0B/s,1.253GiB/0B/0B
+2022-12-08 16:48:39.235,submitted,1,T4,2.128 GiB,8.513 GiB,3,16,4.397GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.235,submitted,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.235,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.237,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.920GiB,3/2,12/8,0B/s,0B/s,1.879GiB/0B/0B
+2022-12-08 16:48:39.237,submitted,1,T4,2.128 GiB,8.513 GiB,3,16,4.397GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.237,submitted,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.237,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.239,submitted,0,T4,0.000 B,2.128 GiB,4,14,1.920GiB,3/3,12/12,0B/s,0B/s,1.879GiB/0B/0B
+2022-12-08 16:48:39.239,submitted,1,T4,2.128 GiB,8.513 GiB,3,16,4.397GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.239,submitted,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:48:39.239,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:12.542,completed,0,T4,0.000 B,2.128 GiB,1,14,1.920GiB,3/3,12/12,55.763MiB/s,56.199MiB/s,1.879GiB/1.813GiB/1.827GiB
+2022-12-08 16:49:12.542,completed,1,T4,2.128 GiB,8.513 GiB,3,16,4.397GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:12.542,completed,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:12.542,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.048,completed,0,T4,0.000 B,2.128 GiB,1,10,1.372GiB,2/2,8/8,36.386MiB/s,36.908MiB/s,1.253GiB/1.237GiB/1.254GiB
+2022-12-08 16:49:14.048,completed,1,T4,2.128 GiB,8.513 GiB,3,18,4.946GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.048,completed,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.048,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.332,completed,0,T4,0.000 B,2.128 GiB,1,6,843.086MiB,1/1,4/4,18.288MiB/s,18.338MiB/s,641.698MiB/641.698MiB/643.449MiB
+2022-12-08 16:49:14.332,completed,1,T4,2.128 GiB,8.513 GiB,3,20,5.496GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.332,completed,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.332,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.333,submitted,0,T4,0.000 B,2.128 GiB,1,2,280.955MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.333,submitted,1,T4,2.128 GiB,8.513 GiB,4,22,6.046GiB,1/1,4/4,0B/s,0B/s,1.257GiB/0B/0B
+2022-12-08 16:49:14.333,submitted,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.333,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.335,submitted,0,T4,0.000 B,2.128 GiB,1,2,280.955MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.335,submitted,1,T4,2.128 GiB,8.513 GiB,4,22,6.046GiB,2/2,8/8,0B/s,0B/s,2.514GiB/0B/0B
+2022-12-08 16:49:14.335,submitted,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:49:14.335,submitted,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:50:19.853,completed,0,T4,0.000 B,2.128 GiB,1,2,280.955MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:50:19.853,completed,1,T4,2.128 GiB,8.513 GiB,2,22,6.046GiB,2/2,8/8,38.750MiB/s,38.938MiB/s,2.514GiB/2.479GiB/2.491GiB
+2022-12-08 16:50:19.853,completed,2,T4,8.513 GiB,34.052 GiB,2,18,20.319GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:50:19.853,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:50:21.386,completed,0,T4,0.000 B,2.128 GiB,1,2,280.955MiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:50:21.386,completed,1,T4,2.128 GiB,8.513 GiB,2,18,4.946GiB,1/1,4/4,19.203MiB/s,19.271MiB/s,1.257GiB/1.257GiB/1.262GiB
+2022-12-08 16:50:21.386,completed,2,T4,8.513 GiB,34.052 GiB,2,19,21.425GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
+2022-12-08 16:50:21.386,completed,3,T4,34.052 GiB,136.206 GiB,2,20,90.025GiB,0/0,0/0,0B/s,0B/s,0B/0B/0B
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-CompressionInfo.db b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-CompressionInfo.db
new file mode 100644
index 000000000000..c87db27a121d
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-CompressionInfo.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Data.db b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Data.db
new file mode 100644
index 000000000000..9f545217f15e
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Data.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Digest.crc32 b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Digest.crc32
new file mode 100644
index 000000000000..9fd1e8cb3530
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Digest.crc32
@@ -0,0 +1 @@
+3063591096
\ No newline at end of file
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Filter.db b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Filter.db
new file mode 100644
index 000000000000..6cd025087cbf
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Filter.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Partitions.db b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Partitions.db
new file mode 100644
index 000000000000..872ca7ddabef
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Partitions.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Rows.db b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Statistics.db b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Statistics.db
new file mode 100644
index 000000000000..e7e9885a1621
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Statistics.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-TOC.txt b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-TOC.txt
new file mode 100644
index 000000000000..e5fc2427c7f0
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+Filter.db
+TOC.txt
+CompressionInfo.db
+Rows.db
+Statistics.db
+Digest.crc32
diff --git a/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/schema.txt b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/schema.txt
new file mode 100644
index 000000000000..56935346f661
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_1-7ab21491848211ee825e31532694b0d3/schema.txt
@@ -0,0 +1,24 @@
+CREATE TABLE IF NOT EXISTS schema_validation_tests.test_1 (
+	col_txt text PRIMARY KEY,
+	col_bool boolean,
+	col_dec decimal,
+	col_int int,
+	col_uuid uuid)
+	WITH ID = 7ab21491-8482-11ee-825e-31532694b0d3
+	AND additional_write_policy = '99PERCENTILE'
+	AND bloom_filter_fp_chance = 0.01
+	AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+	AND cdc = false
+	AND comment = ''
+	AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+	AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+	AND crc_check_chance = 1.0
+	AND default_time_to_live = 0
+	AND extensions = {}
+	AND gc_grace_seconds = 864000
+	AND max_index_interval = 2048
+	AND memtable_flush_period_in_ms = 0
+	AND min_index_interval = 128
+	AND nodesync = {'enabled': 'true', 'incremental': 'true'}
+	AND read_repair = 'BLOCKING'
+	AND speculative_retry = '99PERCENTILE';
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..ac0e354c1f14
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Data.db b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Data.db
new file mode 100644
index 000000000000..fa3eb5f88c43
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Data.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Digest.crc32 b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..d763779f758d
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+2160731571
\ No newline at end of file
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Filter.db b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..6cd025087cbf
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Filter.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Partitions.db b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..7e6075f03ea2
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Partitions.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Rows.db b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Statistics.db b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..ee4433d8ea39
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Statistics.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-TOC.txt b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..e5fc2427c7f0
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+Filter.db
+TOC.txt
+CompressionInfo.db
+Rows.db
+Statistics.db
+Digest.crc32
diff --git a/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/schema.txt b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/schema.txt
new file mode 100644
index 000000000000..cf32c48680ac
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_2-83ac0dd1848211ee825e31532694b0d3/schema.txt
@@ -0,0 +1,26 @@
+CREATE TABLE IF NOT EXISTS schema_validation_tests.test_2 (
+	col_txt text,
+	col_uuid uuid,
+	col_bool boolean,
+	col_dec decimal,
+	col_int int,
+	PRIMARY KEY (col_txt, col_uuid))
+	WITH ID = 83ac0dd1-8482-11ee-825e-31532694b0d3
+	AND CLUSTERING ORDER BY (col_uuid ASC)
+	AND additional_write_policy = '99PERCENTILE'
+	AND bloom_filter_fp_chance = 0.01
+	AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+	AND cdc = false
+	AND comment = ''
+	AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+	AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+	AND crc_check_chance = 1.0
+	AND default_time_to_live = 0
+	AND extensions = {}
+	AND gc_grace_seconds = 864000
+	AND max_index_interval = 2048
+	AND memtable_flush_period_in_ms = 0
+	AND min_index_interval = 128
+	AND nodesync = {'enabled': 'true', 'incremental': 'true'}
+	AND read_repair = 'BLOCKING'
+	AND speculative_retry = '99PERCENTILE';
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..87f694471fe7
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Data.db b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Data.db
new file mode 100644
index 000000000000..9efce0d4c519
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Data.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Digest.crc32 b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..7c3ffadbcf78
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1470262863
\ No newline at end of file
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Filter.db b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..063343275efa
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Filter.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Partitions.db b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..e95d8e88e766
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Partitions.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Rows.db b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Statistics.db b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..0479160bc4f8
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Statistics.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-TOC.txt b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..e5fc2427c7f0
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+Filter.db
+TOC.txt
+CompressionInfo.db
+Rows.db
+Statistics.db
+Digest.crc32
diff --git a/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/schema.txt b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/schema.txt
new file mode 100644
index 000000000000..3b6985b62a5c
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_3-88b6cb80848211ee825e31532694b0d3/schema.txt
@@ -0,0 +1,26 @@
+CREATE TABLE IF NOT EXISTS schema_validation_tests.test_3 (
+	col_txt text,
+	col_int int,
+	col_uuid uuid,
+	col_bool boolean,
+	col_dec decimal,
+	PRIMARY KEY ((col_txt, col_int), col_uuid))
+	WITH ID = 88b6cb80-8482-11ee-825e-31532694b0d3
+	AND CLUSTERING ORDER BY (col_uuid DESC)
+	AND additional_write_policy = '99PERCENTILE'
+	AND bloom_filter_fp_chance = 0.01
+	AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+	AND cdc = false
+	AND comment = ''
+	AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+	AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+	AND crc_check_chance = 1.0
+	AND default_time_to_live = 0
+	AND extensions = {}
+	AND gc_grace_seconds = 864000
+	AND max_index_interval = 2048
+	AND memtable_flush_period_in_ms = 0
+	AND min_index_interval = 128
+	AND nodesync = {'enabled': 'true', 'incremental': 'true'}
+	AND read_repair = 'BLOCKING'
+	AND speculative_retry = '99PERCENTILE';
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..41e66b5fb603
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Data.db b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Data.db
new file mode 100644
index 000000000000..1592254e8ae8
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Data.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Digest.crc32 b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..3741c1226738
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3445881969
\ No newline at end of file
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Filter.db b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..6cd025087cbf
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Filter.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Partitions.db b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..07199c89c1fd
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Partitions.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Rows.db b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Statistics.db b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..b85280a65121
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Statistics.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-TOC.txt b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..e5fc2427c7f0
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+Filter.db
+TOC.txt
+CompressionInfo.db
+Rows.db
+Statistics.db
+Digest.crc32
diff --git a/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/schema.txt b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/schema.txt
new file mode 100644
index 000000000000..acc727d0f876
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_4-8bf57bc0848211ee825e31532694b0d3/schema.txt
@@ -0,0 +1,26 @@
+CREATE TABLE IF NOT EXISTS schema_validation_tests.test_4 (
+	col_txt text,
+	col_int int,
+	col_uuid uuid,
+	col_bool boolean,
+	col_dec decimal,
+	PRIMARY KEY (col_txt, col_int, col_uuid))
+	WITH ID = 8bf57bc0-8482-11ee-825e-31532694b0d3
+	AND CLUSTERING ORDER BY (col_int DESC, col_uuid ASC)
+	AND additional_write_policy = '99PERCENTILE'
+	AND bloom_filter_fp_chance = 0.01
+	AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+	AND cdc = false
+	AND comment = ''
+	AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+	AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+	AND crc_check_chance = 1.0
+	AND default_time_to_live = 0
+	AND extensions = {}
+	AND gc_grace_seconds = 864000
+	AND max_index_interval = 2048
+	AND memtable_flush_period_in_ms = 0
+	AND min_index_interval = 128
+	AND nodesync = {'enabled': 'true', 'incremental': 'true'}
+	AND read_repair = 'BLOCKING'
+	AND speculative_retry = '99PERCENTILE';
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db
new file mode 100644
index 000000000000..9e2a63de591c
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-CompressionInfo.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Data.db b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Data.db
new file mode 100644
index 000000000000..844c5d3dc6b1
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Data.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Digest.crc32 b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Digest.crc32
new file mode 100644
index 000000000000..28094a0cf88d
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3527822699
\ No newline at end of file
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Filter.db b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Filter.db
new file mode 100644
index 000000000000..6cd025087cbf
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Filter.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Partitions.db b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Partitions.db
new file mode 100644
index 000000000000..8fbf52c90bd1
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Partitions.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Rows.db b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Rows.db
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Statistics.db b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Statistics.db
new file mode 100644
index 000000000000..a246bd720a81
Binary files /dev/null and b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Statistics.db differ
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-TOC.txt b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-TOC.txt
new file mode 100644
index 000000000000..e5fc2427c7f0
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Partitions.db
+Data.db
+Filter.db
+TOC.txt
+CompressionInfo.db
+Rows.db
+Statistics.db
+Digest.crc32
diff --git a/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/schema.txt b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/schema.txt
new file mode 100644
index 000000000000..2d7ed65a4bd3
--- /dev/null
+++ b/test/resources/schema/schema_validation_tests/test_5-9db47b90848211ee825e31532694b0d3/schema.txt
@@ -0,0 +1,36 @@
+CREATE TABLE IF NOT EXISTS schema_validation_tests.test_5 (
+	col_txt text PRIMARY KEY,
+	col_ascii ascii,
+	col_bigint bigint,
+	col_blob blob,
+	col_bool boolean,
+	col_date date,
+	col_dbl double,
+	col_dec decimal,
+	col_float float,
+	col_inet inet,
+	col_small smallint,
+	col_time time,
+	col_timestamp timestamp,
+	col_timeuuid timeuuid,
+	col_tinyint tinyint,
+	col_varchar text,
+	col_varint varint)
+	WITH ID = 9db47b90-8482-11ee-825e-31532694b0d3
+	AND additional_write_policy = '99PERCENTILE'
+	AND bloom_filter_fp_chance = 0.01
+	AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+	AND cdc = false
+	AND comment = ''
+	AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+	AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+	AND crc_check_chance = 1.0
+	AND default_time_to_live = 0
+	AND extensions = {}
+	AND gc_grace_seconds = 864000
+	AND max_index_interval = 2048
+	AND memtable_flush_period_in_ms = 0
+	AND min_index_interval = 128
+	AND nodesync = {'enabled': 'true', 'incremental': 'true'}
+	AND read_repair = 'BLOCKING'
+	AND speculative_retry = '99PERCENTILE';
diff --git a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
index f7f934638553..095c2ae5aca7 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
@@ -89,6 +89,7 @@
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.Closeable;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.ReflectionUtils;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.memory.BufferPool;
@@ -793,7 +794,7 @@ public synchronized void close() throws IOException
             Field field = Clock.Global.class.getDeclaredField("instance");
             field.setAccessible(true);
 
-            Field modifiersField = Field.class.getDeclaredField("modifiers");
+            Field modifiersField = ReflectionUtils.getModifiersField();
             modifiersField.setAccessible(true);
             modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
 
diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java
index 8344a65db78b..18e8bf710743 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java
@@ -97,7 +97,7 @@ private static void invokeRepair(String keyspaceName, boolean repairPaxos, boole
     {
         Collection<Range<Token>> ranges = rangesSupplier.call();
         // no need to wait for completion, as we track all task submissions and message exchanges, and ensure they finish before continuing to next action
-        StorageService.instance.repair(keyspaceName, new RepairOption(RepairParallelism.SEQUENTIAL, isPrimaryRangeOnly, false, false, 1, ranges, false, false, force, PreviewKind.NONE, false, true, repairPaxos, repairOnlyPaxos), singletonList((tag, event) -> {
+        StorageService.instance.repair(keyspaceName, new RepairOption(RepairParallelism.SEQUENTIAL, isPrimaryRangeOnly, false, false, 1, ranges, false, false, false, force, PreviewKind.NONE, false, true, repairPaxos, repairOnlyPaxos, false), singletonList((tag, event) -> {
             if (event.getType() == ProgressEventType.COMPLETE)
                 listener.run();
         }));
diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java
index c57f49341fbc..f86044641a6a 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java
@@ -115,7 +115,7 @@ public static LatestBallots read(Permit permit, DecoratedKey key, TableMetadata
                 promised.unixMicros(),
                 accepted == null || accepted.update.isEmpty() ? 0L : accepted.ballot.unixMicros(),
                 accepted == null || accepted.update.isEmpty() ? 0L : accepted.update.stats().minTimestamp,
-                latestBallot(committed.update.iterator()),
+                latestBallot(committed.update.unfilteredIterator()),
                 baseTable
             );
         });
diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedAction.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedAction.java
index 551616ff60a2..a8a2017a968b 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedAction.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedAction.java
@@ -19,11 +19,7 @@
 package org.apache.cassandra.simulator.systems;
 
 import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.Executor;
 
 import javax.annotation.Nullable;
@@ -452,7 +448,7 @@ Action applyTo(Object description, Kind kind, OrderOn orderOn, Modifiers self, V
     protected SimulatedAction setMessageModifiers(Verb verb, Modifiers self, Modifiers responses)
     {
         if (verbModifiers.isEmpty())
-            verbModifiers = new EnumMap<>(Verb.class);
+            verbModifiers = new HashMap<>();
         verbModifiers.put(verb, self);
         if (verb.responseVerb != null)
             verbModifiers.put(verb.responseVerb, responses);
diff --git a/test/unit/org/apache/cassandra/CassandraXMLJUnitResultFormatter.java b/test/unit/org/apache/cassandra/CassandraXMLJUnitResultFormatter.java
index 7a9df05a48dd..8a505d862e81 100644
--- a/test/unit/org/apache/cassandra/CassandraXMLJUnitResultFormatter.java
+++ b/test/unit/org/apache/cassandra/CassandraXMLJUnitResultFormatter.java
@@ -23,20 +23,30 @@
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.math.BigDecimal;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
 import java.util.Date;
 import java.util.Enumeration;
-import java.util.Hashtable;
+import java.util.Map;
+import java.util.NavigableMap;
 import java.util.Properties;
-
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.stream.Collectors;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
-import junit.framework.AssertionFailedError;  // checkstyle: permit this import
-import junit.framework.Test;  // checkstyle: permit this import
+import org.apache.commons.lang3.StringUtils;
 
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import junit.framework.AssertionFailedError; // checkstyle: permit this import
+import junit.framework.Test; // checkstyle: permit this import
+import junit.framework.TestSuite; // checkstyle: permit this import
 import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.taskdefs.optional.junit.FormatterElement;
 import org.apache.tools.ant.taskdefs.optional.junit.IgnoredTestListener;
 import org.apache.tools.ant.taskdefs.optional.junit.JUnitResultFormatter;
 import org.apache.tools.ant.taskdefs.optional.junit.JUnitTest;
@@ -50,10 +60,11 @@
 import org.w3c.dom.Element;
 import org.w3c.dom.Text;
 
-import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_SUITENAME;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_TESTTAG;
-import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_JAVA_COMMAND;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_FAIL_ON_FORBIDDEN_LOG_ENTRIES;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+import static org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.getFilteredTrace;
+import static org.apache.tools.ant.taskdefs.optional.junit.JUnitVersionHelper.getTestCaseClassName;
 
 /**
  * Prints XML output of the test to a specified Writer.
@@ -61,33 +72,37 @@
  * @see FormatterElement
  */
 
-public class CassandraXMLJUnitResultFormatter implements JUnitResultFormatter, XMLConstants, IgnoredTestListener {
-
-    private static final double ONE_SECOND = 1000.0;
-
-    /** constant for unnnamed testsuites/cases */
+public class CassandraXMLJUnitResultFormatter implements JUnitResultFormatter, XMLConstants, IgnoredTestListener
+{
+    /**
+     * constant for unnnamed testsuites/cases
+     */
     private static final String UNKNOWN = "unknown";
 
-    private static DocumentBuilder getDocumentBuilder() {
-        try {
+    private static DocumentBuilder getDocumentBuilder()
+    {
+        try
+        {
             return DocumentBuilderFactory.newInstance().newDocumentBuilder();
-        } catch (final Exception exc) {
+        }
+        catch (final Exception exc)
+        {
             throw new ExceptionInInitializerError(exc);
         }
     }
 
     private static final String tag = TEST_CASSANDRA_TESTTAG.getString();
 
-    /*
-     * Set the property for the test suite name so that log configuration can pick it up
-     * and log to a file specific to this test suite
-     */
-    static
-    {
-        String command = SUN_JAVA_COMMAND.getString();
-        String args[] = command.split(" ");
-        TEST_CASSANDRA_SUITENAME.setString(args[1]);
-    }
+//    /*
+//     * Set the property for the test suite name so that log configuration can pick it up
+//     * and log to a file specific to this test suite
+//     */
+//    static
+//    {
+//        String command = SUN_JAVA_COMMAND.getString();
+//        String args[] = command.split(" ");
+//        TEST_CASSANDRA_SUITENAME.setString(args[1]);
+//    }
 
     /**
      * The XML document.
@@ -101,73 +116,127 @@ private static DocumentBuilder getDocumentBuilder() {
 
     /**
      * Element for the current test.
-     *
+     * <p>
      * The keying of this map is a bit of a hack: tests are keyed by caseName(className) since
      * the Test we get for Test-start isn't the same as the Test we get during test-assumption-fail,
      * so we can't easily match Test objects without manually iterating over all keys and checking
      * individual fields.
      */
-    private final Hashtable<String, Element> testElements = new Hashtable<String, Element>();
+    private final ConcurrentMap<String, Element> testElements = new ConcurrentHashMap<>();
 
     private Element propsElement;
     private Element systemOutputElement;
 
     /**
-     * tests that failed.
+     * Tests that failed - see {@link #testElements} for keys interpretation
      */
-    private final Hashtable failedTests = new Hashtable();
+    private final ConcurrentMap<String, Element> failedTests = new ConcurrentHashMap<>();
 
     /**
-     * Tests that were skipped.
+     * Tests that were skipped - see {@link #testElements} for keys interpretation
      */
-    private final Hashtable<String, Test> skippedTests = new Hashtable<String, Test>();
+    private final ConcurrentMap<String, Test> skippedTests = new ConcurrentHashMap<>();
+
     /**
-     * Tests that were ignored. See the note above about the key being a bit of a hack.
+     * Tests that were ignored - see {@link #testElements} for keys interpretation
      */
-    private final Hashtable<String, Test> ignoredTests = new Hashtable<String, Test>();
+    private final ConcurrentMap<String, Test> ignoredTests = new ConcurrentHashMap<>();
+
     /**
-     * Timing helper.
+     * Times when the tests were started - see {@link #testElements} for keys interpretation
      */
-    private final Hashtable<String, Long> testStarts = new Hashtable<String, Long>();
+    private final ConcurrentMap<String, Long> testStarts = new ConcurrentHashMap<>();
+
+    /**
+     * Times when the tests were finished - see {@link #testElements} for keys interpretation
+     */
+    private final ConcurrentMap<String, Long> testEnds = new ConcurrentHashMap<>();
+
+    /**
+     * Forbbidden log entries (collected as throwables), recorded before, between and after test cases (for example
+     * during execution of @BeforeClass or @AfterClass blocks). Keys are the times in ms of the event
+     */
+    private final ConcurrentSkipListMap<Long, Throwable> suiteEvents = new ConcurrentSkipListMap<>();
+
+    /**
+     * Forbidden log entries (collected as throwawbles) recorded during tests execution - see {@link #suiteEvents} for keys interpretation
+     */
+    private final ConcurrentSkipListMap<Long, Throwable> testEvents = new ConcurrentSkipListMap<>();
+
+    /**
+     * The current events map - the listener registered in {@link ForbiddenLogEntriesFilter} writes the events to the
+     * map referenced by this variable
+     */
+    private volatile ConcurrentSkipListMap<Long, Throwable> events = suiteEvents;
+
     /**
      * Where to write the log to.
      */
     private OutputStream out;
 
-    /** No arg constructor. */
-    public CassandraXMLJUnitResultFormatter() {
+    private String curSuiteName = null;
+    private String classCaseDesc = null;
+
+    private ForbiddenLogEntriesFilter forbiddenLogEntriesFilter;
+
+    /**
+     * No arg constructor.
+     */
+    public CassandraXMLJUnitResultFormatter()
+    {
     }
 
-    /** {@inheritDoc}. */
-    public void setOutput(final OutputStream out) {
+    /**
+     * {@inheritDoc}.
+     */
+    @Override
+    public void setOutput(final OutputStream out)
+    {
         this.out = out;
     }
 
-    /** {@inheritDoc}. */
-    public void setSystemOutput(final String out) {
-        systemOutputElement = formatOutput(SYSTEM_OUT, out);
+    /**
+     * {@inheritDoc}.
+     */
+    @Override
+    public void setSystemOutput(final String out)
+    {
+        maybeAddClassCaseElement();
+        addOutputNode(SYSTEM_OUT, out);
     }
 
-    /** {@inheritDoc}. */
-    public void setSystemError(final String out) {
-        rootElement.appendChild(formatOutput(SYSTEM_ERR, out));
+    /**
+     * {@inheritDoc}.
+     */
+    @Override
+    public void setSystemError(final String out)
+    {
+        maybeAddClassCaseElement();
+        addOutputNode(SYSTEM_ERR, out);
     }
 
     /**
      * The whole testsuite started.
+     *
      * @param suite the testsuite.
      */
-    public void startTestSuite(final JUnitTest suite) {
+    @Override
+    public void startTestSuite(final JUnitTest suite)
+    {
+        forbiddenLogEntriesFilter = ForbiddenLogEntriesFilter.getInstanceIfUsed();
+        if (TEST_FAIL_ON_FORBIDDEN_LOG_ENTRIES.getBoolean() && forbiddenLogEntriesFilter != null)
+            forbiddenLogEntriesFilter.setListener(this::onForbiddenLogEvent);
+
+        long startTime = System.currentTimeMillis();
+        curSuiteName = suite.getName();
+        classCaseDesc = String.format("%s(%s)", formatName(curSuiteName), TestSuite.class.getName());
+
         doc = getDocumentBuilder().newDocument();
         rootElement = doc.createElement(TESTSUITE);
-        String n = suite.getName();
-        if (n != null && !tag.isEmpty())
-            n = n + "-" + tag;
-        rootElement.setAttribute(ATTR_NAME, n == null ? UNKNOWN : n);
+        rootElement.setAttribute(ATTR_NAME, formatName(suite.getName()));
 
         //add the timestamp
-        final String timestamp = DateUtils.format(new Date(),
-                DateUtils.ISO8601_DATETIME_PATTERN);
+        final String timestamp = DateUtils.format(new Date(startTime), DateUtils.ISO8601_DATETIME_PATTERN);
         rootElement.setAttribute(TIMESTAMP, timestamp);
         //and the hostname.
         rootElement.setAttribute(HOSTNAME, getHostname());
@@ -175,9 +244,11 @@ public void startTestSuite(final JUnitTest suite) {
         // Output properties
         propsElement = doc.createElement(PROPERTIES);
         final Properties props = suite.getProperties();
-        if (props != null) {
-            final Enumeration e = props.propertyNames();
-            while (e.hasMoreElements()) {
+        if (props != null)
+        {
+            final Enumeration<?> e = props.propertyNames();
+            while (e.hasMoreElements())
+            {
                 final String name = (String) e.nextElement();
                 final Element propElement = doc.createElement(PROPERTY);
                 propElement.setAttribute(ATTR_NAME, name);
@@ -187,217 +258,317 @@ public void startTestSuite(final JUnitTest suite) {
         }
     }
 
-    /**
-     * get the local hostname
-     * @return the name of the local host, or "localhost" if we cannot work it out
-     */
-    private String getHostname()  {
-        String hostname = "localhost";
-        try {
-            final InetAddress localHost = InetAddress.getLocalHost();
-            if (localHost != null) {
-                hostname = localHost.getHostName();
-            }
-        } catch (final UnknownHostException e) {
-            // fall back to default 'localhost'
-        }
-        return hostname;
-    }
-
-    /**
-     * The whole testsuite ended.
-     * @param suite the testsuite.
-     * @throws BuildException on error.
-     */
-    public void endTestSuite(final JUnitTest suite) throws BuildException {
-        rootElement.setAttribute(ATTR_TESTS, "" + suite.runCount());
-        rootElement.setAttribute(ATTR_FAILURES, "" + suite.failureCount());
-        rootElement.setAttribute(ATTR_ERRORS, "" + suite.errorCount());
-        rootElement.setAttribute(ATTR_SKIPPED, "" + suite.skipCount());
-        rootElement.setAttribute(ATTR_TIME, "" + (suite.getRunTime() / ONE_SECOND));
-        if (suite.failureCount() > 0 || suite.errorCount() > 0)
-        {
-            // only include properties and system-out if there's failure/error
-            rootElement.appendChild(propsElement);
-            if (null != systemOutputElement)
-                rootElement.appendChild(systemOutputElement);
-        }
-        if (out != null) {
-            Writer wri = null;
-            try {
-                wri = new BufferedWriter(new OutputStreamWriter(out, "UTF8"));
-                wri.write("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n");
-                (new DOMElementWriter()).write(rootElement, wri, 0, "  ");
-            } catch (final IOException exc) {
-                throw new BuildException("Unable to write log file", exc);
-            } finally {
-                if (wri != null) {
-                    try {
-                        wri.flush();
-                    } catch (final IOException ex) {
-                        // ignore
-                    }
-                }
-                if (out != System.out && out != System.err) {
-                    FileUtils.close(wri);
-                }
-            }
-        }
-    }
 
     /**
      * Interface TestListener.
      *
      * <p>A new Test is started.
-     * @param t the test.
+     *
+     * @param test the test.
      */
-    public void startTest(final Test t) {
-        testStarts.put(createDescription(t), currentTimeMillis());
-    }
-
-    private static String createDescription(final Test test) throws BuildException {
-        if (!tag.isEmpty())
-            return JUnitVersionHelper.getTestCaseName(test) + "-" + tag +"(" + JUnitVersionHelper.getTestCaseClassName(test) + ")";
-        return JUnitVersionHelper.getTestCaseName(test) + "(" + JUnitVersionHelper.getTestCaseClassName(test) + ")";
+    @Override
+    public void startTest(final Test test)
+    {
+        testEvents.clear();
+        events = testEvents;
+        long testStartTime = currentTimeMillis();
+        String desc = createDescription(test);
+        testStarts.put(desc, testStartTime);
     }
 
     /**
      * Interface TestListener.
      *
      * <p>A Test is finished.
+     *
      * @param test the test.
      */
-    public void endTest(final Test test) {
-        final String testDescription = createDescription(test);
+    @Override
+    public void endTest(final Test test)
+    {
+        events = suiteEvents;
+
+        long testEndTime = System.currentTimeMillis();
+
+        final String desc = createDescription(test);
 
         // Fix for bug #5637 - if a junit.extensions.TestSetup is
         // used and throws an exception during setUp then startTest
         // would never have been called
-        if (!testStarts.containsKey(testDescription)) {
-            startTest(test);
+        if (!testStarts.containsKey(desc))
+        {
+            testStarts.put(desc, testEndTime);
+            testEvents.clear();
         }
+
+        testEnds.put(desc, testEndTime);
+
+        long testStartTime = testStarts.get(desc);
+        long testDuration = testEndTime - testStartTime;
+
         Element currentTest;
-        if (!failedTests.containsKey(test) && !skippedTests.containsKey(testDescription) && !ignoredTests.containsKey(testDescription)) {
-            currentTest = doc.createElement(TESTCASE);
-            String n = JUnitVersionHelper.getTestCaseName(test);
-            if (n != null && !tag.isEmpty())
-                n = n + "-" + tag;
-            currentTest.setAttribute(ATTR_NAME,
-                                     n == null ? UNKNOWN : n);
-            // a TestSuite can contain Tests from multiple classes,
-            // even tests with the same name - disambiguate them.
-            currentTest.setAttribute(ATTR_CLASSNAME,
-                    JUnitVersionHelper.getTestCaseClassName(test));
+        if (!failedTests.containsKey(desc) && !skippedTests.containsKey(desc) && !ignoredTests.containsKey(desc))
+        {
+            currentTest = createTestCaseElement(getTestCaseClassName(test), resolveCaseName(test), testDuration);
             rootElement.appendChild(currentTest);
-            testElements.put(createDescription(test), currentTest);
-        } else {
-            currentTest = testElements.get(testDescription);
+            testElements.put(desc, currentTest);
+            maybeAddForbiddenEntriesFailureElement(desc, testEvents);
+        }
+        else
+        {
+            // the test is skipped / ignored / failed - we do not add another failure because there can be only one
+            // failure associated with a test case
+            currentTest = testElements.get(desc);
+            updateTime(currentTest, testDuration);
         }
-
-        final Long l = testStarts.get(createDescription(test));
-        currentTest.setAttribute(ATTR_TIME,
-            "" + ((currentTimeMillis() - l) / ONE_SECOND));
     }
 
     /**
-     * Interface TestListener for JUnit &lt;= 3.4.
+     * The whole testsuite ended.
      *
-     * <p>A Test failed.
-     * @param test the test.
-     * @param t the exception.
+     * @param suite the testsuite.
+     * @throws BuildException on error.
      */
-    public void addFailure(final Test test, final Throwable t) {
-        formatError(FAILURE, test, t);
+    @Override
+    public void endTestSuite(final JUnitTest suite) throws BuildException
+    {
+        if (TEST_FAIL_ON_FORBIDDEN_LOG_ENTRIES.getBoolean() && forbiddenLogEntriesFilter != null)
+            forbiddenLogEntriesFilter.setListener(null);
+
+        maybeAddClassCaseElement();
+        Element classCaseElement = testElements.get(classCaseDesc);
+        maybeAddForbiddenEntriesFailureElement(classCaseDesc, suiteEvents);
+        long testTime = testStarts.entrySet().stream().mapToLong(entry -> testEnds.getOrDefault(entry.getKey(), entry.getValue()) - entry.getValue()).sum();
+        long nonTestTime = Math.max(0L, suite.getRunTime() - testTime);
+        updateTime(classCaseElement, nonTestTime);
+
+        for (Map.Entry<String, Element> descAndFailureElem : failedTests.entrySet())
+        {
+            Element testElem = testElements.get(descAndFailureElem.getKey());
+            if (testElem != null)
+                testElem.appendChild(descAndFailureElem.getValue());
+        }
+
+        rootElement.setAttribute(ATTR_TESTS, String.valueOf(suite.runCount()));
+        rootElement.setAttribute(ATTR_FAILURES, String.valueOf(failedTests.size()));
+        rootElement.setAttribute(ATTR_ERRORS, String.valueOf(suite.errorCount()));
+        rootElement.setAttribute(ATTR_SKIPPED, String.valueOf(suite.skipCount()));
+        updateTime(rootElement, suite.getRunTime());
+        if (out != null)
+        {
+            Writer wri = null;
+            try
+            {
+                wri = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
+                wri.write("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n");
+                (new DOMElementWriter()).write(rootElement, wri, 0, "  ");
+            }
+            catch (final IOException exc)
+            {
+                throw new BuildException("Unable to write log file", exc);
+            }
+            finally
+            {
+                if (wri != null)
+                {
+                    try
+                    {
+                        wri.flush();
+                    }
+                    catch (final IOException ex)
+                    {
+                        // ignore
+                    }
+                }
+                if (out != System.out && out != System.err)
+                {
+                    FileUtils.close(wri);
+                }
+            }
+        }
     }
 
     /**
      * Interface TestListener for JUnit &gt; 3.4.
      *
      * <p>A Test failed.
+     *
      * @param test the test.
-     * @param t the assertion.
+     * @param t    the assertion.
      */
-    public void addFailure(final Test test, final AssertionFailedError t) {
-        addFailure(test, (Throwable) t);
+    @Override
+    public void addFailure(final Test test, final AssertionFailedError t)
+    {
+        if (test != null)
+        {
+            endTest(test);
+            failedTests.put(createDescription(test), getFailureOrError(t, getFilteredTrace(t)));
+        }
     }
 
     /**
      * Interface TestListener.
      *
      * <p>An error occurred while running the test.
+     *
      * @param test the test.
-     * @param t the error.
+     * @param t    the error.
      */
-    public void addError(final Test test, final Throwable t) {
-        formatError(ERROR, test, t);
-    }
-
-    private void formatError(final String type, final Test test, final Throwable t) {
-        if (test != null) {
+    @Override
+    public void addError(final Test test, final Throwable t)
+    {
+        if (test != null)
+        {
             endTest(test);
-            failedTests.put(test, test);
+            failedTests.put(createDescription(test), getFailureOrError(t, getFilteredTrace(t)));
         }
+    }
 
-        final Element nested = doc.createElement(type);
-        Element currentTest;
-        if (test != null) {
-            currentTest = testElements.get(createDescription(test));
-        } else {
-            currentTest = rootElement;
+    @Override
+    public void testIgnored(final Test test)
+    {
+        formatSkip(test, JUnitVersionHelper.getIgnoreMessage(test));
+        if (test != null)
+        {
+            ignoredTests.put(createDescription(test), test);
         }
+    }
 
-        currentTest.appendChild(nested);
+    @Override
+    public void testAssumptionFailure(final Test test, final Throwable failure)
+    {
+        formatSkip(test, failure.getMessage());
+        skippedTests.put(createDescription(test), test);
+    }
 
-        final String message = t.getMessage();
-        if (message != null && message.length() > 0) {
-            nested.setAttribute(ATTR_MESSAGE, t.getMessage());
+    private void maybeAddClassCaseElement()
+    {
+        if (!testElements.containsKey(classCaseDesc))
+        {
+            Element classElement = createTestCaseElement(TestSuite.class.getName(), formatName(curSuiteName), 0);
+            rootElement.appendChild(classElement);
+            testElements.put(classCaseDesc, classElement);
         }
-        nested.setAttribute(ATTR_TYPE, t.getClass().getName());
-
-        final String strace = JUnitTestRunner.getFilteredTrace(t);
-        final Text trace = doc.createTextNode(strace);
-        nested.appendChild(trace);
     }
 
-    private Element formatOutput(final String type, final String output) {
+    private void addOutputNode(final String type, final String output)
+    {
         final Element nested = doc.createElement(type);
+        rootElement.appendChild(nested);
         nested.appendChild(doc.createCDATASection(output));
-        return nested;
     }
 
-    public void testIgnored(final Test test) {
-        formatSkip(test, JUnitVersionHelper.getIgnoreMessage(test));
-        if (test != null) {
-            ignoredTests.put(createDescription(test), test);
-        }
+    private String formatName(String name)
+    {
+        name = name == null ? UNKNOWN : (StringUtils.isBlank(tag) ? name : String.format("%s-%s", name, tag));
+        return name;
     }
 
+    private String resolveCaseName(Test test)
+    {
+        return formatName(JUnitVersionHelper.getTestCaseName(test));
+    }
 
-    public void formatSkip(final Test test, final String message) {
-        if (test != null) {
+    private String createDescription(final Test test) throws BuildException
+    {
+        return String.format("%s(%s)", resolveCaseName(test), getTestCaseClassName(test));
+    }
+
+    private Element createTestCaseElement(String className, String caseName, long timeMs)
+    {
+        final Element testCaseElement = doc.createElement(TESTCASE);
+        testCaseElement.setAttribute(ATTR_CLASSNAME, className);
+        testCaseElement.setAttribute(ATTR_NAME, caseName);
+        updateTime(testCaseElement, timeMs);
+        return testCaseElement;
+    }
+
+    private void updateTime(Element element, long timeMs)
+    {
+        element.setAttribute(ATTR_TIME, BigDecimal.valueOf(timeMs).divide(BigDecimal.valueOf(1000)).toString());
+    }
+
+    private void formatSkip(final Test test, final String message)
+    {
+        if (test != null)
+        {
             endTest(test);
         }
 
         final Element nested = doc.createElement("skipped");
 
-        if (message != null) {
+        if (message != null)
+        {
             nested.setAttribute("message", message);
         }
 
         Element currentTest;
-        if (test != null) {
+        if (test != null)
+        {
             currentTest = testElements.get(createDescription(test));
-        } else {
+        }
+        else
+        {
             currentTest = rootElement;
         }
 
         currentTest.appendChild(nested);
+    }
+
+    private Element getFailureOrError(Throwable t, String content)
+    {
+        String type = t instanceof AssertionFailedError ? FAILURE : ERROR;
+        final Element nested = doc.createElement(type);
+
+        if (t.getMessage() != null && t.getMessage().length() > 0) nested.setAttribute(ATTR_MESSAGE, t.getMessage());
+        nested.setAttribute(ATTR_TYPE, t.getClass().getName());
+
+        final Text trace = doc.createTextNode(content);
+        nested.appendChild(trace);
 
+        return nested;
     }
 
-    public void testAssumptionFailure(final Test test, final Throwable failure) {
-        formatSkip(test, failure.getMessage());
-        skippedTests.put(createDescription(test), test);
+    private void maybeAddForbiddenEntriesFailureElement(String desc, NavigableMap<Long, Throwable> forbiddenEntries)
+    {
+        if (!forbiddenEntries.isEmpty() && !failedTests.containsKey(desc))
+        {
+            Element elem = getFailureOrError(new AssertionFailedError("Forbidden entries detected"),
+                                             forbiddenEntries.values().stream().map(JUnitTestRunner::getFilteredTrace).collect(Collectors.joining("\n", "\n", "")));
+            failedTests.putIfAbsent(desc, elem);
+            forbiddenEntries.clear();
+        }
+    }
 
+    private void onForbiddenLogEvent(ILoggingEvent event)
+    {
+        String timestamp = DateUtils.format(new Date(event.getTimeStamp()), "HH:mm:ss.SSS");
+        String msg = String.format("%s %s %s", timestamp, event.getLoggerName(), event.getFormattedMessage());
+        Throwable t = new AssertionFailedError(msg);
+        t.setStackTrace(event.getCallerData());
+        events.put(event.getTimeStamp(), t);
+    }
+
+    /**
+     * get the local hostname
+     *
+     * @return the name of the local host, or "localhost" if we cannot work it out
+     */
+    private String getHostname()
+    {
+        String hostname = "localhost";
+        try
+        {
+            final InetAddress localHost = InetAddress.getLocalHost();
+            if (localHost != null)
+            {
+                hostname = localHost.getHostName();
+            }
+        }
+        catch (final UnknownHostException e)
+        {
+            // fall back to default 'localhost'
+        }
+        return hostname;
     }
 } // XMLJUnitResultFormatter
diff --git a/test/unit/org/apache/cassandra/ForbiddenLogEntriesFilter.java b/test/unit/org/apache/cassandra/ForbiddenLogEntriesFilter.java
new file mode 100644
index 000000000000..215d8bafad1c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/ForbiddenLogEntriesFilter.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra;
+
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.function.Consumer;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.filter.Filter;
+import ch.qos.logback.core.spi.FilterReply;
+import io.netty.util.ResourceLeakDetector;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.logging.LogbackLoggingSupport;
+import org.apache.cassandra.utils.logging.LoggingSupportFactory;
+
+public class ForbiddenLogEntriesFilter extends Filter<ILoggingEvent>
+{
+    private static final Set<String> followedLoggerNames = Sets.newConcurrentHashSet(ImmutableSet.of(
+    LoggerFactory.getLogger(ResourceLeakDetector.class).getName(),
+    LoggerFactory.getLogger(Ref.class).getName()));
+
+    private volatile Consumer<ILoggingEvent> listener = null;
+
+    private final Set<String> expectedPhrases = new CopyOnWriteArraySet<>();
+
+    public static ForbiddenLogEntriesFilter getInstanceIfUsed()
+    {
+        if (!(LoggingSupportFactory.getLoggingSupport() instanceof LogbackLoggingSupport))
+            return null;
+
+        LogbackLoggingSupport loggingSupport = (LogbackLoggingSupport) LoggingSupportFactory.getLoggingSupport();
+        return loggingSupport.getAllLogbackFilters().stream().filter(ForbiddenLogEntriesFilter.class::isInstance)
+                             .map(ForbiddenLogEntriesFilter.class::cast).findFirst().orElse(null);
+    }
+
+    public void addLoggerToFollow(String name)
+    {
+        followedLoggerNames.add(name);
+    }
+
+    public void addLoggerToFollow(Class<?> clazz)
+    {
+        followedLoggerNames.add(LoggerFactory.getLogger(clazz).getName());
+    }
+
+    public void addLoggerToFollow(Logger logger)
+    {
+        followedLoggerNames.add(logger.getName());
+    }
+
+    public void addExpectedPhrase(String phrase)
+    {
+        expectedPhrases.add(phrase);
+    }
+
+    public void clearExpectedPhrases()
+    {
+        expectedPhrases.clear();
+    }
+
+    public void setListener(Consumer<ILoggingEvent> listener)
+    {
+        this.listener = listener;
+    }
+
+    @Override
+    public FilterReply decide(ILoggingEvent event)
+    {
+        Consumer<ILoggingEvent> listener = this.listener;
+        if (listener == null)
+            return FilterReply.NEUTRAL;
+
+        // we are only interested in error messages
+        if (event.getLevel() != Level.ERROR)
+            return FilterReply.NEUTRAL;
+
+        // we are only interested in messages from the specified classes
+        if (!followedLoggerNames.contains(event.getLoggerName()))
+            return FilterReply.NEUTRAL;
+
+        // we skip messages containing one of the specified phrases
+        if (expectedPhrases.stream().anyMatch(msg -> event.getFormattedMessage().contains(msg)))
+            return FilterReply.NEUTRAL;
+
+        listener.accept(event);
+
+        return FilterReply.NEUTRAL;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/LogbackStatusListener.java b/test/unit/org/apache/cassandra/LogbackStatusListener.java
index 834d51fca27b..1b965d9bb312 100644
--- a/test/unit/org/apache/cassandra/LogbackStatusListener.java
+++ b/test/unit/org/apache/cassandra/LogbackStatusListener.java
@@ -22,6 +22,8 @@
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Locale;
 
 import org.slf4j.ILoggerFactory;
@@ -44,7 +46,6 @@
  */
 public class LogbackStatusListener implements StatusListener, LoggerContextListener
 {
-
     public static final PrintStream originalOut = System.out;
     public static final PrintStream originalErr = System.err;
 
@@ -55,6 +56,23 @@ public class LogbackStatusListener implements StatusListener, LoggerContextListe
     private PrintStream replacementOut;
     private PrintStream replacementErr;
 
+    /*
+     * Set the property for the test suite name so that log configuration can pick it up
+     * and log to a file specific to this test suite
+     */
+    static
+    {
+        // checkstyle: suppress below 'blockSystemPropertyUsage'
+        String command = System.getProperty("sun.java.command");
+        List<String> args = Arrays.asList(command.split(" "));
+        int idx = args.lastIndexOf("-junit4");
+
+        if (idx > 0 && (idx + 1) < args.size())
+            System.setProperty("suitename", args.get(idx + 1));
+        else
+            System.setProperty("suitename", args.get(1));
+    }
+
     @Override
     public void addStatusEvent(Status s)
     {
@@ -141,7 +159,7 @@ public void flush() throws IOException
                 reset();
             }
         }
-    };
+    }
 
     private static class WrappedPrintStream extends PrintStream
     {
@@ -281,7 +299,7 @@ public void print(double d)
         @Override
         public void print(char[] s)
         {
-            if(isAsyncAppender())
+            if (isAsyncAppender())
                 original.println(s);
             else
                 super.print(s);
@@ -456,7 +474,8 @@ public PrintStream append(char c)
                 return original.append(c);
             else
                 return super.append(c);
-        }    }
+        }
+    }
 
     public boolean isResetResistant()
     {
@@ -510,7 +529,7 @@ public synchronized void onStop(LoggerContext loggerContext)
             haveRegisteredListener = false;
             if (haveRegisteredListener)
             {
-                ((LoggerContext)LoggerFactory.getILoggerFactory()).removeListener(this);
+                ((LoggerContext) LoggerFactory.getILoggerFactory()).removeListener(this);
             }
         }
     }
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index f341fb6eb803..0554c33bc0d8 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -35,6 +35,7 @@
 import org.apache.cassandra.config.*;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.db.marshal.*;
@@ -764,7 +765,38 @@ public static void cleanupSavedCaches()
         ServerTestUtils.cleanupSavedCaches();
     }
 
-    private static CompressionParams compressionParams(int chunkLength)
+    /**
+     * Simple method that allows creating a table given it's CQL definition.
+     *
+     * <p>The method also creates the keyspace of the table if needs be (using a simple strategy with 1 replica) and
+     * can also create a few UDT (also from their CQL definition) if needed for the created table.
+     *
+     * <p>This method does not complain if any of the created entity already exists.
+     */
+    public static void load(String keyspace, String schemaCQL, String... typesCQL)
+    {
+        KeyspaceMetadata ksm = KeyspaceMetadata.create(keyspace,
+                                                       KeyspaceParams.simple(1),
+                                                       Tables.none(),
+                                                       Views.none(),
+                                                       Types.none(),
+                                                       UserFunctions.none());
+        Schema.instance.transform(SchemaTransformations.addKeyspace(ksm, true));
+
+        for (String typeCQL : typesCQL)
+        {
+            Types types = Schema.instance.getKeyspaceMetadata(keyspace).types;
+            SchemaTransformation t = SchemaTransformations.addOrUpdateType(CreateTypeStatement.parse(typeCQL,
+                                                                                                     keyspace, types));
+            Schema.instance.transform(t);
+        }
+
+        Types types = Schema.instance.getKeyspaceMetadata(keyspace).types;
+        TableMetadata metadata = CreateTableStatement.parse(schemaCQL, keyspace, types).build();
+        Schema.instance.transform(SchemaTransformations.addTable(metadata, true));
+    }
+
+    public static CompressionParams compressionParams(int chunkLength)
     {
         String algo = TEST_COMPRESSION_ALGO.getString().toLowerCase();
         switch (algo)
diff --git a/test/unit/org/apache/cassandra/ServerTestUtils.java b/test/unit/org/apache/cassandra/ServerTestUtils.java
index fa59f0d67d83..89d45cb6e0ab 100644
--- a/test/unit/org/apache/cassandra/ServerTestUtils.java
+++ b/test/unit/org/apache/cassandra/ServerTestUtils.java
@@ -146,6 +146,7 @@ public static void cleanupAndLeaveDirs() throws IOException
         mkdirs(); // Creates the directories if they does not exists
         cleanup(); // Ensure that the directories are all empty
         CommitLog.instance.restartUnsafe();
+        CommitLog.instance.getSegmentManager().awaitManagementTasksCompletion();
     }
 
     /**
@@ -156,14 +157,15 @@ public static void cleanup()
         // clean up commitlog
         cleanupDirectory(DatabaseDescriptor.getCommitLogLocation());
 
-        String cdcDir = DatabaseDescriptor.getCDCLogLocation();
+        File cdcDir = DatabaseDescriptor.getCDCLogLocation();
         if (cdcDir != null)
             cleanupDirectory(cdcDir);
         cleanupDirectory(DatabaseDescriptor.getHintsDirectory());
+        cleanupDirectory(DatabaseDescriptor.getMetadataDirectory());
         cleanupSavedCaches();
 
         // clean up data directory which are stored as data directory/keyspace/data files
-        for (String dirName : DatabaseDescriptor.getAllDataFileLocations())
+        for (File dirName : DatabaseDescriptor.getAllDataFileLocations())
         {
             cleanupDirectory(dirName);
         }
diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java
index 49af3e1271ac..44fa54679e14 100644
--- a/test/unit/org/apache/cassandra/Util.java
+++ b/test/unit/org/apache/cassandra/Util.java
@@ -25,11 +25,16 @@
 import java.io.IOError;
 import java.io.IOException;
 import java.io.InputStream;
+import java.lang.reflect.Field;
 import java.math.BigInteger;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -37,6 +42,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -45,6 +51,7 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.IntFunction;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -61,6 +68,7 @@
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.FieldIdentifier;
 import org.apache.cassandra.db.AbstractReadCommandBuilder;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.ClusteringComparator;
@@ -78,14 +86,16 @@
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.compaction.AbstractCompactionTask;
-import org.apache.cassandra.db.compaction.ActiveCompactionsTracker;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionSSTable;
 import org.apache.cassandra.db.compaction.CompactionTasks;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.db.partitions.FilteredPartition;
 import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
 import org.apache.cassandra.db.partitions.Partition;
@@ -109,11 +119,14 @@
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.EndpointState;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.sstable.SSTableIdFactory;
 import org.apache.cassandra.io.sstable.SSTableLoader;
 import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
 import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
@@ -141,12 +154,15 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.FilterFactory;
 import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.Throwables;
+import org.assertj.core.api.Assertions;
 import org.awaitility.Awaitility;
 import org.hamcrest.Matcher;
 import org.mockito.Mockito;
 import org.mockito.internal.stubbing.defaultanswers.ForwardsInvocations;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertEquals;
@@ -160,6 +176,8 @@ public class Util
 
     private static List<UUID> hostIdPool = new ArrayList<>();
 
+    public final static TimeUnit supportedMTimeGranularity = getSupportedMTimeGranularity();
+
     public static IPartitioner testPartitioner()
     {
         return DatabaseDescriptor.getPartitioner();
@@ -308,7 +326,7 @@ public static void createInitialRing(StorageService ss, IPartitioner partitioner
 
         // check that all nodes are in token metadata
         for (int i=0; i<endpointTokens.size(); ++i)
-            assertTrue(ss.getTokenMetadata().isMember(hosts.get(i)));
+            assertThat(ss.getTokenMetadata().isMember(hosts.get(i))).withFailMessage("%s is not a member of token metadata", hosts.get(i)).isTrue();
     }
 
     public static Future<?> compactAll(ColumnFamilyStore cfs, long gcBefore)
@@ -322,13 +340,46 @@ public static Future<?> compactAll(ColumnFamilyStore cfs, long gcBefore)
     public static void compact(ColumnFamilyStore cfs, Collection<SSTableReader> sstables)
     {
         long gcBefore = cfs.gcBefore(FBUtilities.nowInSeconds());
-        try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstables, gcBefore))
+        try (CompactionTasks tasks = cfs.getCompactionStrategy().getUserDefinedTasks(sstables, gcBefore))
         {
             for (AbstractCompactionTask task : tasks)
-                task.execute(ActiveCompactionsTracker.NOOP);
+                task.execute();
+        }
+    }
+
+    /**
+     * Checks that the provided SSTable set does not overlap. The result of a major compaction should satisfy this.
+     */
+    public static void assertNoOverlap(Collection<? extends CompactionSSTable> liveSet)
+    {
+        for (CompactionSSTable rdr1 : liveSet)
+        {
+            for (CompactionSSTable rdr2 : liveSet)
+            {
+                if (rdr1 == rdr2)
+                    continue;
+
+                Range<Token> rdr2Range = new Range<>(rdr2.getFirst().getToken(), rdr2.getLast().getToken());
+                assertFalse(rdr1.getBounds().intersects(Collections.singletonList(rdr2Range)));
+            }
         }
     }
 
+    /**
+     * Perform full compaction, everything in the given CFS to one file. Unlike major compaction, this must also compact
+     * non-overlapping files, and should try to not split output.
+     */
+    public static void forceFullCompaction(ColumnFamilyStore cfs, int timeoutInSeconds)
+    {
+        Future<?> future = CompactionManager.instance.submitUserDefined(cfs,
+                cfs.getLiveSSTables()
+                        .stream()
+                        .map(s -> s.getDescriptor())
+                        .collect(Collectors.toList()),
+                FBUtilities.nowInSeconds());
+        FBUtilities.waitOnFutures(Collections.singletonList(future), timeoutInSeconds, TimeUnit.SECONDS);
+    }
+
     public static void expectEOF(Callable<?> callable)
     {
         expectException(callable, EOFException.class);
@@ -400,17 +451,17 @@ public static void assertEmpty(ReadCommand command)
         }
     }
 
-    public static List<ImmutableBTreePartition> getAllUnfiltered(ReadCommand command)
+    public static List<Partition> getAllUnfiltered(ReadCommand command)
     {
         try (ReadExecutionController controller = command.executionController())
         {
             return getAllUnfiltered(command, controller);
         }
     }
-    
-    public static List<ImmutableBTreePartition> getAllUnfiltered(ReadCommand command, ReadExecutionController controller)
+
+    public static List<Partition> getAllUnfiltered(ReadCommand command, ReadExecutionController controller)
     {
-        List<ImmutableBTreePartition> results = new ArrayList<>();
+        List<Partition> results = new ArrayList<>();
         try (UnfilteredPartitionIterator iterator = command.executeLocally(controller))
         {
             while (iterator.hasNext())
@@ -431,7 +482,7 @@ public static List<FilteredPartition> getAll(ReadCommand command)
             return getAll(command, controller);
         }
     }
-    
+
     public static List<FilteredPartition> getAll(ReadCommand command, ReadExecutionController controller)
     {
         List<FilteredPartition> results = new ArrayList<>();
@@ -483,15 +534,15 @@ public static Row getOnlyRow(ReadCommand cmd)
         }
     }
 
-    public static ImmutableBTreePartition getOnlyPartitionUnfiltered(ReadCommand cmd)
+    public static Partition getOnlyPartitionUnfiltered(ReadCommand cmd)
     {
         try (ReadExecutionController controller = cmd.executionController())
         {
             return getOnlyPartitionUnfiltered(cmd, controller);
         }
     }
-    
-    public static ImmutableBTreePartition getOnlyPartitionUnfiltered(ReadCommand cmd, ReadExecutionController controller)
+
+    public static Partition getOnlyPartitionUnfiltered(ReadCommand cmd, ReadExecutionController controller)
     {
         try (UnfilteredPartitionIterator iterator = cmd.executeLocally(controller))
         {
@@ -508,7 +559,7 @@ public static FilteredPartition getOnlyPartition(ReadCommand cmd)
     {
         return getOnlyPartition(cmd, false);
     }
-    
+
     public static FilteredPartition getOnlyPartition(ReadCommand cmd, boolean trackRepairedStatus)
     {
         try (ReadExecutionController executionController = cmd.executionController(trackRepairedStatus);
@@ -820,6 +871,15 @@ public static Closeable markDirectoriesUnwriteable(ColumnFamilyStore cfs)
         return () -> DisallowedDirectories.clearUnwritableUnsafe();
     }
 
+    public static boolean getDirectoriesWriteable(ColumnFamilyStore cfs)
+    {
+        boolean ret = true;
+        for (File dir : cfs.getDirectories().getCFDirectories())
+            ret &= !DisallowedDirectories.isUnwritable(dir);
+
+        return ret;
+    }
+
     public static PagingState makeSomePagingState(ProtocolVersion protocolVersion)
     {
         return makeSomePagingState(protocolVersion, Integer.MAX_VALUE);
@@ -1100,6 +1160,9 @@ public static void disableBloomFilter(ColumnFamilyStore cfs)
      */
     public static void setUpgradeFromVersion(String version)
     {
+        if (!Gossiper.instance.isEnabled())
+            Gossiper.instance.maybeInitializeLocalState(0);
+
         int v = Optional.ofNullable(Gossiper.instance.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddressAndPort()))
                         .map(ep -> ep.getApplicationState(ApplicationState.RELEASE_VERSION))
                         .map(rv -> rv.version)
@@ -1107,16 +1170,8 @@ public static void setUpgradeFromVersion(String version)
 
         Gossiper.instance.addLocalApplicationState(ApplicationState.RELEASE_VERSION,
                                                    VersionedValue.unsafeMakeVersionedValue(version, v + 1));
-        try
-        {
-            // add dummy host to avoid returning early in Gossiper.instance.upgradeFromVersionSupplier
-            Gossiper.instance.initializeNodeUnsafe(InetAddressAndPort.getByName("127.0.0.2"), UUID.randomUUID(), 1);
-        }
-        catch (UnknownHostException e)
-        {
-            throw new RuntimeException(e);
-        }
-        Gossiper.instance.expireUpgradeFromVersion();
+
+        Gossiper.instance.clusterVersionProvider.reset();
     }
 
     /**
@@ -1275,4 +1330,98 @@ public static RuntimeException testMustBeImplementedForSSTableFormat()
     {
         return new UnsupportedOperationException("Test must be implemented for sstable format " + DatabaseDescriptor.getSelectedSSTableFormat().getClass().getName());
     }
+
+    public static void assertSSTableIds(SSTableId v1, SSTableId v2, IntFunction<Boolean> predicate)
+    {
+        Assertions.assertThat(Pair.create(v1, v2))
+                  .matches(p -> predicate.apply(SSTableIdFactory.COMPARATOR.compare(p.left, p.right)));
+    }
+
+    private static TimeUnit getSupportedMTimeGranularity() {
+        try
+        {
+            Path p = Files.createTempFile(Util.class.getSimpleName(), "dummy-file");
+            FileTime ft = Files.getLastModifiedTime(p);
+            Files.deleteIfExists(p);
+            Field f = FileTime.class.getDeclaredField("unit");
+            f.setAccessible(true);
+            return (TimeUnit) f.get(ft);
+        }
+        catch (IOException |  NoSuchFieldException | IllegalAccessException e)
+        {
+            throw new AssertionError("Failed to read supported file modification time granularity");
+        }
+    }
+
+    public static void markNodeAsDead(InetAddressAndPort address)
+    {
+        EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(address);
+        Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.markDead(address, endpointState));
+        IFailureDetector.instance.report(address);
+        IFailureDetector.instance.interpret(address);
+        assertFalse("Node not convicted", IFailureDetector.instance.isAlive(address));
+    }
+
+    public static void joinNodeToRing(InetAddressAndPort address, Token token, IPartitioner partitioner)
+    {
+        joinNodeToRing(address, token, partitioner, UUID.randomUUID(), 1);
+    }
+
+    public static void joinNodeToRing(InetAddressAndPort address, Token token, IPartitioner partitioner, UUID hostId, int generationNbr)
+    {
+        Gossiper.instance.initializeNodeUnsafe(address, hostId, MessagingService.current_version, generationNbr);
+        Gossiper.instance.injectApplicationState(address, ApplicationState.TOKENS, new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(token)));
+        StorageService.instance.onChange(address,
+                    ApplicationState.STATUS_WITH_PORT,
+                    new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(token)));
+    }
+
+    public static boolean isListeningOn(InetSocketAddress address)
+    {
+        try (ServerSocket socket = new ServerSocket())
+        {
+            socket.bind(address);
+            return false;
+        }
+        catch (IOException ex)
+        {
+            return true;
+        }
+    }
+
+    public static UserType makeUDT(String name, Map<String, AbstractType<?>> fields, boolean multicell)
+    {
+        return makeUDT("ks", name, fields, multicell);
+    }
+
+    public static UserType makeUDT(String ks, String name, Map<String, AbstractType<?>> fields, boolean multicell)
+    {
+        List<FieldIdentifier> fieldNames = new ArrayList<>(fields.size());
+        List<AbstractType<?>> fieldTypes = new ArrayList<>(fields.size());
+        for (Map.Entry<String, AbstractType<?>> entry : fields.entrySet())
+        {
+            fieldNames.add(FieldIdentifier.forUnquoted(entry.getKey()));
+            fieldTypes.add(entry.getValue());
+        }
+        return new UserType(ks, UTF8Type.instance.decompose(name), fieldNames, fieldTypes, multicell);
+    }
+
+    public static void assumeAssertsEnabled()
+    {
+        Assume.assumeTrue("Asserts must be enabled for this test", assertsEnabled());
+    }
+
+    public static boolean assertsEnabled()
+    {
+        try
+        {
+            assert false;
+            return false;
+        }
+        catch (AssertionError e)
+        {
+            return true;
+        }
+    }
+
 }
diff --git a/test/unit/org/apache/cassandra/audit/AuditLogFilterTest.java b/test/unit/org/apache/cassandra/audit/AuditLogFilterTest.java
index 62bc767d6614..714e8163e43d 100644
--- a/test/unit/org/apache/cassandra/audit/AuditLogFilterTest.java
+++ b/test/unit/org/apache/cassandra/audit/AuditLogFilterTest.java
@@ -22,12 +22,22 @@
 import java.util.Set;
 
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+
 import static org.apache.cassandra.audit.AuditLogFilter.isFiltered;
 
 public class AuditLogFilterTest
 {
+    @BeforeClass
+    public static void setup()
+    {
+        // CNDB-9099: DatabaseDescriptor must be initialized before using FBUtilities.getBroadcastAddressAndPort()
+        DatabaseDescriptor.clientInitialization();
+    }
+    
     @Test
     public void testInputWithSpaces()
     {
diff --git a/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java b/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java
index 33a57eb29d91..11849e9ccd3f 100644
--- a/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java
+++ b/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java
@@ -34,6 +34,9 @@
 import com.datastax.driver.core.exceptions.SyntaxError;
 import com.datastax.driver.core.exceptions.UnauthorizedException;
 import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.auth.CassandraAuthorizer;
+import org.apache.cassandra.auth.CassandraRoleManager;
+import org.apache.cassandra.auth.PasswordAuthenticator;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.OverrideConfigurationLoader;
 import org.apache.cassandra.config.ParameterizedClass;
@@ -69,11 +72,11 @@ public class AuditLoggerAuthTest
     public static void setup() throws Exception
     {
         OverrideConfigurationLoader.override((config) -> {
-            config.authenticator = new ParameterizedClass("PasswordAuthenticator");
-            config.role_manager = new ParameterizedClass("CassandraRoleManager");
-            config.authorizer = new ParameterizedClass("CassandraAuthorizer");
+            config.authenticator = new ParameterizedClass(PasswordAuthenticator.class.getName());
+            config.role_manager = new ParameterizedClass(CassandraRoleManager.class.getName());
+            config.authorizer = new ParameterizedClass(CassandraAuthorizer.class.getName());
             config.audit_logging_options.enabled = true;
-            config.audit_logging_options.logger = new ParameterizedClass("InMemoryAuditLogger", null);
+            config.audit_logging_options.logger = new ParameterizedClass(InMemoryAuditLogger.class.getName(), null);
         });
 
         SUPERUSER_SETUP_DELAY_MS.setLong(0);
diff --git a/test/unit/org/apache/cassandra/auth/AuthCacheTest.java b/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
index 50a0a9c0fc7f..73d8ceff06b1 100644
--- a/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
+++ b/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
@@ -322,6 +322,38 @@ public void testCacheLoaderIsNotCalledOnGetAllWhenCacheIsNotEmpty()
         assertEquals(1, loadCounter);
     }
 
+    @Test
+    public void testMaybeInvalidateByFilter() {
+        TestCache authCache = new TestCache(this::countingLoader, this::emptyBulkLoader, this::setValidity, () -> validity, () -> isCacheEnabled);
+
+        // Load cache
+        int result = authCache.get("10");
+        assertEquals(10, result);
+        assertEquals(1, loadCounter);
+
+        result = authCache.get("20");
+        assertEquals(20, result);
+        assertEquals(2, loadCounter);
+
+        // Additional reads are from cache
+        assertEquals(10, authCache.get("10").longValue());
+        assertEquals(20, authCache.get("20").longValue());
+        assertEquals(2, loadCounter);
+
+        // Invalidate using a filter
+        authCache.maybeInvalidateByFilter(s -> s.equals("10"));
+
+        // Getting invalidated value requires loading
+        result = authCache.get("10");
+        assertEquals(10, result);
+        assertEquals(3, loadCounter);
+
+        // Other values are still cached
+        result = authCache.get("20");
+        assertEquals(20, result);
+        assertEquals(3, loadCounter);
+    }
+
     private void setValidity(int validity)
     {
         this.validity = validity;
diff --git a/test/unit/org/apache/cassandra/auth/AuthTestUtils.java b/test/unit/org/apache/cassandra/auth/AuthTestUtils.java
index 610832ffb466..9c5565dcdc89 100644
--- a/test/unit/org/apache/cassandra/auth/AuthTestUtils.java
+++ b/test/unit/org/apache/cassandra/auth/AuthTestUtils.java
@@ -18,8 +18,6 @@
 
 package org.apache.cassandra.auth;
 
-import java.util.List;
-import java.util.Map;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.InetAddress;
@@ -30,6 +28,8 @@
 import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 import java.util.Collection;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeoutException;
 
@@ -258,27 +258,27 @@ public static long getNetworkPermissionsReadCount()
     {
         ColumnFamilyStore networkPemissionsTable =
                 Keyspace.open(SchemaConstants.AUTH_KEYSPACE_NAME).getColumnFamilyStore(AuthKeyspace.NETWORK_PERMISSIONS);
-        return networkPemissionsTable.metric.readLatency.latency.getCount();
+        return networkPemissionsTable.metric.readLatency.tableOrKeyspaceMetric().latency.getCount();
     }
 
     public static long getCidrPermissionsReadCount()
     {
         ColumnFamilyStore cidrPemissionsTable =
         Keyspace.open(SchemaConstants.AUTH_KEYSPACE_NAME).getColumnFamilyStore(AuthKeyspace.CIDR_PERMISSIONS);
-        return cidrPemissionsTable.metric.readLatency.latency.getCount();
+        return cidrPemissionsTable.metric.readLatency.tableOrKeyspaceMetric().latency.getCount();
     }
 
     public static long getRolePermissionsReadCount()
     {
         ColumnFamilyStore rolesPemissionsTable =
                 Keyspace.open(SchemaConstants.AUTH_KEYSPACE_NAME).getColumnFamilyStore(AuthKeyspace.ROLE_PERMISSIONS);
-        return rolesPemissionsTable.metric.readLatency.latency.getCount();
+        return rolesPemissionsTable.metric.readLatency.tableOrKeyspaceMetric().latency.getCount();
     }
 
     public static long getRolesReadCount()
     {
         ColumnFamilyStore rolesTable = Keyspace.open(SchemaConstants.AUTH_KEYSPACE_NAME).getColumnFamilyStore(AuthKeyspace.ROLES);
-        return rolesTable.metric.readLatency.latency.getCount();
+        return rolesTable.metric.readLatency.tableOrKeyspaceMetric().latency.getCount();
     }
 
     public static RoleOptions getLoginRoleOptions()
diff --git a/test/unit/org/apache/cassandra/auth/CassandraRoleManagerTest.java b/test/unit/org/apache/cassandra/auth/CassandraRoleManagerTest.java
index 7b6b910a866e..290852ca1f1d 100644
--- a/test/unit/org/apache/cassandra/auth/CassandraRoleManagerTest.java
+++ b/test/unit/org/apache/cassandra/auth/CassandraRoleManagerTest.java
@@ -21,6 +21,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -80,19 +81,25 @@ public void getGrantedRolesImplMinimizesReads()
 
         // simple role with no grants
         fetchRolesAndCheckReadCount(roleManager, ROLE_A);
+        assertRoleMembers(roleManager, ImmutableSet.of(), ImmutableSet.of(ROLE_A));
+        
         // single level of grants
         grantRolesTo(roleManager, ROLE_A, ROLE_B, ROLE_C);
+        assertRoleMembers(roleManager, ImmutableSet.of(ROLE_A), ImmutableSet.of(ROLE_B, ROLE_C));
         fetchRolesAndCheckReadCount(roleManager, ROLE_A);
 
         // multi level role hierarchy
         grantRolesTo(roleManager, ROLE_B, ROLE_B_1, ROLE_B_2, ROLE_B_3);
+        assertRoleMembers(roleManager, ImmutableSet.of(ROLE_B), ImmutableSet.of(ROLE_B_1, ROLE_B_2, ROLE_B_3));
         grantRolesTo(roleManager, ROLE_C, ROLE_C_1, ROLE_C_2, ROLE_C_3);
+        assertRoleMembers(roleManager, ImmutableSet.of(ROLE_C), ImmutableSet.of(ROLE_C_1, ROLE_C_2, ROLE_C_3));
         fetchRolesAndCheckReadCount(roleManager, ROLE_A);
 
         // Check that when granted roles appear multiple times in parallel levels of the hierarchy, we don't
         // do redundant reads. E.g. here role_b_1, role_b_2 and role_b3 are granted to both role_b and role_c
         // but we only want to actually read them once
         grantRolesTo(roleManager, ROLE_C, ROLE_B_1, ROLE_B_2, ROLE_B_3);
+        assertRoleMembers(roleManager, ImmutableSet.of(ROLE_B, ROLE_C), ImmutableSet.of(ROLE_B_1, ROLE_B_2, ROLE_B_3));
         fetchRolesAndCheckReadCount(roleManager, ROLE_A);
     }
 
@@ -167,4 +174,12 @@ private void assertRoleSet(Set<Role> actual, RoleResource...expected)
         for (RoleResource expectedRole : expected)
             assertTrue(actual.stream().anyMatch(role -> role.resource.equals(expectedRole)));
     }
+    
+    private void assertRoleMembers(IRoleManager roleManager, Set<RoleResource> expectedMembers, Set<RoleResource> rolesToCheck)
+    {
+        for (RoleResource roleToCheck : rolesToCheck)
+        {
+            assertEquals(expectedMembers, roleManager.getMembersOf(roleToCheck));
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/auth/RoleOptionsTest.java b/test/unit/org/apache/cassandra/auth/RoleOptionsTest.java
index 8a224ebdcde3..5ae03cbfbfac 100644
--- a/test/unit/org/apache/cassandra/auth/RoleOptionsTest.java
+++ b/test/unit/org/apache/cassandra/auth/RoleOptionsTest.java
@@ -195,6 +195,11 @@ public Set<RoleResource> getRoles(RoleResource grantee,
                 return null;
             }
 
+            public Set<RoleResource> getMembersOf(RoleResource role)
+            {
+                return null;
+            }
+
             public Set<RoleResource> getAllRoles() throws RequestValidationException, RequestExecutionException
             {
                 return null;
diff --git a/test/unit/org/apache/cassandra/auth/StubAuthorizer.java b/test/unit/org/apache/cassandra/auth/StubAuthorizer.java
index e9f7d2218bcb..354278370bcc 100644
--- a/test/unit/org/apache/cassandra/auth/StubAuthorizer.java
+++ b/test/unit/org/apache/cassandra/auth/StubAuthorizer.java
@@ -102,11 +102,17 @@ public void revokeAllFrom(RoleResource revokee)
                 userPermissions.remove(key);
     }
 
-    public void revokeAllOn(IResource droppedResource)
+    public Set<RoleResource> revokeAllOn(IResource droppedResource)
     {
+        Set<RoleResource> roles = new HashSet<>();
         for (Pair<String, IResource> key : userPermissions.keySet())
             if (key.right.equals(droppedResource))
+            {
                 userPermissions.remove(key);
+                roles.add(RoleResource.role(key.left));
+            }
+        
+        return roles;
     }
 
     public Set<? extends IResource> protectedResources()
diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java
index 6d9a52fb65a7..e3615e698b34 100644
--- a/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java
+++ b/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java
@@ -18,35 +18,103 @@
 package org.apache.cassandra.batchlog;
 
 import java.net.UnknownHostException;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.DynamicEndpointSnitch;
+import org.apache.cassandra.locator.GossipingPropertyFileSnitch;
+import org.apache.cassandra.locator.IEndpointSnitch;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.ReplicaPlans;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
 
-import static org.hamcrest.CoreMatchers.is;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertTrue;
 
 public class BatchlogEndpointFilterTest
 {
     private static final String LOCAL = "local";
 
+    // Repeat all tests some more times since we're dealing with random stuff - i.e. increase the
+    // chance to hit issues.
+    private static final int repetitions = 100;
+    private static final InetAddressAndPort[] INET_ADDRESSES = new InetAddressAndPort[0];
+
+    private DynamicEndpointSnitch dsnitch;
+
     @BeforeClass
-    public static void initialiseServer()
+    public static void beforeClass()
     {
+        DatabaseDescriptor.setConfig(new Config());
         DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setBroadcastAddress(endpointAddress(0, 0).getAddress());
+    }
+
+    @Test
+    public void shouldUseLocalRackIfPreferLocalParameter() throws UnknownHostException
+    {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.random_remote);
+        Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
+                                                                          .put(LOCAL, InetAddressAndPort.getByName("0"))
+                                                                          .put(LOCAL, InetAddressAndPort.getByName("00"))
+                                                                          .put("1", InetAddressAndPort.getByName("1"))
+                                                                          .put("1", InetAddressAndPort.getByName("11"))
+                                                                          .put("2", InetAddressAndPort.getByName("2"))
+                                                                          .put("2", InetAddressAndPort.getByName("22"))
+                                                                          .build();
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(true, endpoints);
+        assertThat(result.size()).isEqualTo(2);
+        assertThat(result).containsAnyElementsOf(endpoints.get(LOCAL));
+        assertThat(result).containsAnyElementsOf(Iterables.concat(endpoints.get("1"), endpoints.get("2")));
+    }
+
+    @Test
+    public void shouldUseLocalRackIfPreferLocalStrategy() throws UnknownHostException
+    {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.prefer_local);
+        Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
+                .put(LOCAL, InetAddressAndPort.getByName("0"))
+                .put(LOCAL, InetAddressAndPort.getByName("00"))
+                .put("1", InetAddressAndPort.getByName("1"))
+                .put("1", InetAddressAndPort.getByName("11"))
+                .put("2", InetAddressAndPort.getByName("2"))
+                .put("2", InetAddressAndPort.getByName("22"))
+                .build();
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(false, endpoints);
+        assertThat(result.size()).isEqualTo(2);
+        assertThat(result).containsAnyElementsOf(endpoints.get(LOCAL));
+        assertThat(result).containsAnyElementsOf(Iterables.concat(endpoints.get("1"), endpoints.get("2")));
     }
 
     @Test
     public void shouldSelect2HostsFromNonLocalRacks() throws UnknownHostException
     {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.random_remote);
         Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
                 .put(LOCAL, InetAddressAndPort.getByName("0"))
                 .put(LOCAL, InetAddressAndPort.getByName("00"))
@@ -55,8 +123,8 @@ public void shouldSelect2HostsFromNonLocalRacks() throws UnknownHostException
                 .put("2", InetAddressAndPort.getByName("2"))
                 .put("2", InetAddressAndPort.getByName("22"))
                 .build();
-        Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
-        assertThat(result.size(), is(2));
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(false, endpoints);
+        assertThat(result.size()).isEqualTo(2);
         assertTrue(result.contains(InetAddressAndPort.getByName("11")));
         assertTrue(result.contains(InetAddressAndPort.getByName("22")));
     }
@@ -64,6 +132,7 @@ public void shouldSelect2HostsFromNonLocalRacks() throws UnknownHostException
     @Test
     public void shouldSelectLastHostsFromLastNonLocalRacks() throws UnknownHostException
     {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.random_remote);
         Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
                                                          .put(LOCAL, InetAddressAndPort.getByName("00"))
                                                          .put("1", InetAddressAndPort.getByName("11"))
@@ -73,8 +142,8 @@ public void shouldSelectLastHostsFromLastNonLocalRacks() throws UnknownHostExcep
                                                          .put("3", InetAddressAndPort.getByName("33"))
                                                          .build();
         
-        Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
-        assertThat(result.size(), is(2));
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(false, endpoints);
+        assertThat(result.size()).isEqualTo(2);
 
         // result should be the last replicas of the last two racks
         // (Collections.shuffle has been replaced with Collections.reverse for testing)
@@ -85,13 +154,14 @@ public void shouldSelectLastHostsFromLastNonLocalRacks() throws UnknownHostExcep
     @Test
     public void shouldSelectHostFromLocal() throws UnknownHostException
     {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.random_remote);
         Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
                 .put(LOCAL, InetAddressAndPort.getByName("0"))
                 .put(LOCAL, InetAddressAndPort.getByName("00"))
                 .put("1", InetAddressAndPort.getByName("1"))
                 .build();
-        Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
-        assertThat(result.size(), is(2));
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(false, endpoints);
+        assertThat(result.size()).isEqualTo(2);
         assertTrue(result.contains(InetAddressAndPort.getByName("1")));
         assertTrue(result.contains(InetAddressAndPort.getByName("0")));
     }
@@ -99,17 +169,19 @@ public void shouldSelectHostFromLocal() throws UnknownHostException
     @Test
     public void shouldReturnPassedEndpointForSingleNodeDC() throws UnknownHostException
     {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.random_remote);
         Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
                 .put(LOCAL, InetAddressAndPort.getByName("0"))
                 .build();
-        Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
-        assertThat(result.size(), is(1));
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(false, endpoints);
+        assertThat(result.size()).isEqualTo(1);
         assertTrue(result.contains(InetAddressAndPort.getByName("0")));
     }
 
     @Test
     public void shouldSelectTwoRandomHostsFromSingleOtherRack() throws UnknownHostException
     {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.random_remote);
         Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
                 .put(LOCAL, InetAddressAndPort.getByName("0"))
                 .put(LOCAL, InetAddressAndPort.getByName("00"))
@@ -117,10 +189,10 @@ public void shouldSelectTwoRandomHostsFromSingleOtherRack() throws UnknownHostEx
                 .put("1", InetAddressAndPort.getByName("11"))
                 .put("1", InetAddressAndPort.getByName("111"))
                 .build();
-        Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(false, endpoints);
         // result should be the last two non-local replicas
         // (Collections.shuffle has been replaced with Collections.reverse for testing)
-        assertThat(result.size(), is(2));
+        assertThat(result.size()).isEqualTo(2);
         assertTrue(result.contains(InetAddressAndPort.getByName("11")));
         assertTrue(result.contains(InetAddressAndPort.getByName("111")));
     }
@@ -128,16 +200,17 @@ public void shouldSelectTwoRandomHostsFromSingleOtherRack() throws UnknownHostEx
     @Test
     public void shouldSelectTwoRandomHostsFromSingleRack() throws UnknownHostException
     {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.random_remote);
         Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
                 .put(LOCAL, InetAddressAndPort.getByName("1"))
                 .put(LOCAL, InetAddressAndPort.getByName("11"))
                 .put(LOCAL, InetAddressAndPort.getByName("111"))
                 .put(LOCAL, InetAddressAndPort.getByName("1111"))
                 .build();
-        Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(false, endpoints);
         // result should be the last two non-local replicas
         // (Collections.shuffle has been replaced with Collections.reverse for testing)
-        assertThat(result.size(), is(2));
+        assertThat(result.size()).isEqualTo(2);
         assertTrue(result.contains(InetAddressAndPort.getByName("111")));
         assertTrue(result.contains(InetAddressAndPort.getByName("1111")));
     }
@@ -145,24 +218,813 @@ public void shouldSelectTwoRandomHostsFromSingleRack() throws UnknownHostExcepti
     @Test
     public void shouldSelectOnlyTwoHostsEvenIfLocal() throws UnknownHostException
     {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(Config.BatchlogEndpointStrategy.random_remote);
         Multimap<String, InetAddressAndPort> endpoints = ImmutableMultimap.<String, InetAddressAndPort> builder()
                                                          .put(LOCAL, InetAddressAndPort.getByName("1"))
                                                          .put(LOCAL, InetAddressAndPort.getByName("11"))
                                                          .build();
-        Collection<InetAddressAndPort> result = filterBatchlogEndpoints(endpoints);
-        assertThat(result.size(), is(2));
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsRandomForTests(false, endpoints);
+        assertThat(result.size()).isEqualTo(2);
         assertTrue(result.contains(InetAddressAndPort.getByName("1")));
         assertTrue(result.contains(InetAddressAndPort.getByName("11")));
     }
 
-    private Collection<InetAddressAndPort> filterBatchlogEndpoints(Multimap<String, InetAddressAndPort> endpoints)
+    private Collection<InetAddressAndPort> filterBatchlogEndpointsRandomForTests(boolean preferLocalRack, Multimap<String, InetAddressAndPort> endpoints)
+    {
+        return ReplicaPlans.filterBatchlogEndpointsRandom(preferLocalRack, LOCAL, endpoints,
+                // Reverse instead of shuffle
+                Collections::reverse,
+                // Always alive
+                (addr) -> true,
+                // Always pick the last
+                (size) -> size - 1);
+    }
+
+    private Collection<InetAddressAndPort> filterBatchlogEndpointsForTests(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        return DatabaseDescriptor.getBatchlogEndpointStrategy().useDynamicSnitchScores ?
+                filterBatchlogEndpointsDynamicForTests(endpoints) :
+                filterBatchlogEndpointsRandomForTests(false, endpoints);
+    }
+
+    private Collection<InetAddressAndPort> filterBatchlogEndpointsDynamicForTests(Multimap<String, InetAddressAndPort> endpoints) {
+        return ReplicaPlans.filterBatchlogEndpointsDynamic(false, LOCAL, endpoints, x -> true);
+    }
+
+
+    @Test
+    public void shouldUseCoordinatorForSingleNodeDC()
+    {
+        withConfigs(Stream.of(
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        1),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Collections.singletonList(LOCAL),
+                        1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Collections.singletonList(LOCAL),
+                        1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Collections.singletonList(LOCAL),
+                        1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        1)
+        ), this::shouldUseCoordinatorForSingleNodeDC);
+    }
+
+    private void shouldUseCoordinatorForSingleNodeDC(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsForTests(endpoints);
+        assertThat(result.size(), is(1));
+        assertThat(result, hasItem(endpointAddress(0, 0)));
+    }
+
+    @Test
+    public void shouldUseCoordinatorAndTheOtherForTwoNodesInOneRack()
+    {
+        withConfigs(Stream.of(
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        2),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Collections.singletonList(LOCAL),
+                        2),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        2),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Collections.singletonList(LOCAL),
+                        2),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Collections.singletonList(LOCAL),
+                        2),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        2)
+        ), this::shouldUseCoordinatorAndTheOtherForTwoNodesInOneRack);
+    }
+
+    private void shouldUseCoordinatorAndTheOtherForTwoNodesInOneRack(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsForTests(endpoints);
+        assertThat(result.size(), is(2));
+        assertThat(new HashSet<>(result).size(), is(2));
+        assertThat(result, hasItem(endpointAddress(0, 0)));
+        assertThat(result, hasItem(endpointAddress(0, 1)));
+    }
+
+    @Test
+    public void shouldUseCoordinatorAndTheOtherForTwoNodesInTwoRacks()
+    {
+        withConfigs(Stream.of(
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1)
+        ), this::shouldUseCoordinatorAndTheOtherForTwoNodesInTwoRacks);
+    }
+
+    private void shouldUseCoordinatorAndTheOtherForTwoNodesInTwoRacks(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        Collection<InetAddressAndPort> result = filterBatchlogEndpointsForTests(endpoints);
+        assertThat(result.size(), is(2));
+        assertThat(new HashSet<>(result).size(), is(2));
+        assertThat(result, hasItem(endpointAddress(0, 0)));
+        assertThat(result, hasItem(endpointAddress(1, 0)));
+    }
+
+    @Test
+    public void shouldSelectOneNodeFromLocalRackAndOneNodeFromTheOtherRack()
+    {
+        withConfigs(Stream.of(
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "1"),
+                        2, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Arrays.asList(LOCAL, "1"),
+                        2, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        2, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        2, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        2, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        2, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        1, 1)
+        ), this::shouldSelectOneNodeFromLocalRackAndOneNodeFromTheOtherRack);
+    }
+
+    private void shouldSelectOneNodeFromLocalRackAndOneNodeFromTheOtherRack(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        for (int i = 0; i < repetitions; i++)
+        {
+            Collection<InetAddressAndPort> result = filterBatchlogEndpointsForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(new HashSet<>(result).size(), is(2));
+            assertThat(result, hasItem(endpointAddress(1, 0)));
+            assertThat(result, either(hasItem(endpointAddress(0, 0)))
+                    .or(hasItem(endpointAddress(0, 1))));
+        }
+    }
+
+    @Test
+    public void shouldReturnNoBatchlogEnpointsIfAllAreUnavailable()
+    {
+        withConfigs(Stream.of(
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Collections.singletonList(LOCAL),
+                        3),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Collections.singletonList(LOCAL),
+                        15),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15)
+        ), this::shouldReturnNoBatchlogEnpointsIfAllAreUnavailable);
+    }
+
+    private void shouldReturnNoBatchlogEnpointsIfAllAreUnavailable(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        Predicate<InetAddressAndPort> isAlive = x -> x.equals(endpointAddress(0, 0));
+        for (int i = 0; i < repetitions; i++)
+        {
+            Collection<InetAddressAndPort> result = DatabaseDescriptor.getBatchlogEndpointStrategy().useDynamicSnitchScores ?
+                    ReplicaPlans.filterBatchlogEndpointsDynamic(false, LOCAL, endpoints, isAlive) :
+                    ReplicaPlans.filterBatchlogEndpointsRandom(false, LOCAL, endpoints, Collections::reverse, isAlive, (size) -> size - 1);
+            Assert.assertEquals(0, result.size());
+        }
+    }
+
+    @Test
+    public void shouldNotFailIfThereAreAtLeastTwoLiveNodesBesideCoordinator()
+    {
+        withConfigs(Stream.of(
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Collections.singletonList(LOCAL),
+                        3),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Collections.singletonList(LOCAL),
+                        15),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15)
+        ), this::shouldNotFailIfThereAreAtLeastTwoLiveNodesBesideCoordinator);
+    }
+
+    private void shouldNotFailIfThereAreAtLeastTwoLiveNodesBesideCoordinator(Multimap<String, InetAddressAndPort> endpoints)
     {
-        return ReplicaPlans.filterBatchlogEndpoints(LOCAL, endpoints,
-                                                    // Reverse instead of shuffle
-                                                    Collections::reverse,
-                                                    // Always alive
-                                                    (addr) -> true,
-                                                    // Always pick the last
-                                                    (size) -> size - 1);
+        Predicate<InetAddressAndPort> isAlive = x -> nodeInRack(x) >= endpoints.get(LOCAL).size() - 2;
+        for (int i = 0; i < repetitions; i++)
+        {
+            if (DatabaseDescriptor.getBatchlogEndpointStrategy().useDynamicSnitchScores) {
+                ReplicaPlans.filterBatchlogEndpointsDynamic(false, LOCAL, endpoints, isAlive);
+            } else {
+                ReplicaPlans.filterBatchlogEndpointsRandom(false, LOCAL, endpoints, Collections::reverse, isAlive, (size) -> size - 1);
+            }
+        }
     }
+
+    @Test
+    public void shouldNotFailIfThereAreAtLeastTwoLiveNodesIncludingCoordinator()
+    {
+        withConfigs(Stream.of(
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Collections.singletonList(LOCAL),
+                        3),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Collections.singletonList(LOCAL),
+                        3),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Collections.singletonList(LOCAL),
+                        15),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1"),
+                        15, 15),
+
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.prefer_local, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, false,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, false,
+                        Arrays.asList(LOCAL, "r1", "r2"),
+                        15, 15, 15)
+        ), this::shouldNotFailIfThereAreAtLeastTwoLiveNodesIncludingCoordinator);
+    }
+
+    private void shouldNotFailIfThereAreAtLeastTwoLiveNodesIncludingCoordinator(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        Predicate<InetAddressAndPort> isAlive = x -> nodeInRack(x) <= 1;
+        for (int i = 0; i < repetitions; i++)
+        {
+            if (DatabaseDescriptor.getBatchlogEndpointStrategy().useDynamicSnitchScores) {
+                ReplicaPlans.filterBatchlogEndpointsDynamic(false, LOCAL, endpoints, isAlive);
+            } else {
+                ReplicaPlans.filterBatchlogEndpointsRandom(false, LOCAL, endpoints, Collections::reverse, isAlive, (size) -> size - 1);
+            }
+        }
+    }
+
+    @Test
+    public void shouldSelectTwoHostsFromNonLocalRacks()
+    {
+        withConfigs(Stream.of(
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "1", "2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "1", "2"),
+                        15, 1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Arrays.asList(LOCAL, "1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.random_remote, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "1", "2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "1", "2"),
+                        15, 1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Arrays.asList(LOCAL, "1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                        Collections.singletonList(LOCAL),
+                        15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "1", "2"),
+                        15, 15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "1", "2"),
+                        15, 1, 1),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Arrays.asList(LOCAL, "1"),
+                        15, 15),
+                () -> configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                        Collections.singletonList(LOCAL),
+                        15)
+        ), this::assertTwoEndpointsWithoutCoordinator);
+    }
+
+    private void assertTwoEndpointsWithoutCoordinator(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        for (int i = 0; i < repetitions; i++)
+        {
+            Collection<InetAddressAndPort> result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            // result should be the last two non-local replicas
+            // (Collections.shuffle has been replaced with Collections.reverse for testing)
+            assertThat(result.size(), is(2));
+            assertThat(new HashSet<>(result).size(), is(2));
+            assertThat(result, not(hasItems(endpoints.get(LOCAL).toArray(INET_ADDRESSES))));
+        }
+    }
+
+    /**
+     * Test with {@link Config.BatchlogEndpointStrategy#dynamic}.
+     */
+    @Test
+    public void shouldSelectTwoFastestHostsFromSingleLocalRackWithDynamicSnitch()
+    {
+        for (int i = 0; i < repetitions; i++)
+        {
+            InetAddressAndPort host1 = endpointAddress(0, 1);
+            InetAddressAndPort host2 = endpointAddress(0, 2);
+            InetAddressAndPort host3 = endpointAddress(0, 3);
+            List<InetAddressAndPort> hosts = Arrays.asList(host1, host2, host3);
+
+            Multimap<String, InetAddressAndPort> endpoints = configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                    Collections.singletonList(LOCAL),
+                    20);
+
+            // ascending
+            setScores(endpoints, hosts, 10, 12, 14);
+            List<InetAddressAndPort> order = Arrays.asList(host1, host2, host3);
+            assertEquals(order, ReplicaPlans.sortByProximity(Arrays.asList(host3, host1, host2)));
+
+            Collection<InetAddressAndPort> result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(result, hasItem(host1));
+            assertThat(result, hasItem(host2));
+
+            // descending
+            setScores(endpoints, hosts, 50, 9, 1);
+            order = Arrays.asList(host3, host2, host1);
+            assertEquals(order, ReplicaPlans.sortByProximity(Arrays.asList(host1, host2, host3)));
+            result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(result, hasItem(host2));
+            assertThat(result, hasItem(host3));
+        }
+    }
+
+    /**
+     * Test with {@link Config.BatchlogEndpointStrategy#dynamic}.
+     */
+    @Test
+    public void shouldSelectOneFastestHostsFromNonLocalRackWithDynamicSnitch()
+    {
+        for (int i = 0; i < repetitions; i++)
+        {
+            // for each rack, get last host (only in test), then sort all endpoints from each rack by scores
+            Multimap<String, InetAddressAndPort> endpoints = configure(Config.BatchlogEndpointStrategy.dynamic, true,
+                    Arrays.asList(LOCAL, "r1", "r2"),
+                    15, 15, 15);
+            InetAddressAndPort r0h1 = endpointAddress(0, 1);
+            InetAddressAndPort r1h1 = endpointAddress(1, 0);
+            InetAddressAndPort r1h2 = endpointAddress(1, 1);
+            InetAddressAndPort r2h1 = endpointAddress(2, 0);
+            InetAddressAndPort r2h2 = endpointAddress(2, 1);
+            List<InetAddressAndPort> hosts = Arrays.asList(r0h1, r1h1, r1h2, r2h1, r2h2);
+
+            // ascending
+            setScores(endpoints, hosts, 11, 6/* r1h1 */, 12, 5/* r2h1 */, 10);
+            Collection<InetAddressAndPort> result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(result, hasItem(r1h1));
+            assertThat(result, hasItem(r2h1));
+
+            // descending
+            setScores(endpoints, hosts, 5/* r0h1 */, 20, 5, 0/* r2h1 */, 15);
+            result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(result, hasItem(r0h1));
+            assertThat(result, hasItem(r2h1));
+        }
+    }
+
+    /**
+     * Test with {@link Config.BatchlogEndpointStrategy#dynamic_remote}.
+     */
+    @Test
+    public void shouldSelectTwoFastestHostsFromSingleLocalRackWithDynamicSnitchRemote()
+    {
+        for (int i = 0; i < repetitions; i++)
+        {
+            InetAddressAndPort host1 = endpointAddress(0, 1);
+            InetAddressAndPort host2 = endpointAddress(0, 2);
+            InetAddressAndPort host3 = endpointAddress(0, 3);
+            List<InetAddressAndPort> hosts = Arrays.asList(host1, host2, host3);
+
+            Multimap<String, InetAddressAndPort> endpoints = configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                    Collections.singletonList(LOCAL),
+                    20);
+
+            // ascending
+            setScores(endpoints, hosts,
+                    10, 12, 14);
+            List<InetAddressAndPort> order = Arrays.asList(host1, host2, host3);
+            assertEquals(order, ReplicaPlans.sortByProximity(Arrays.asList(host3, host1, host2)));
+
+            Collection<InetAddressAndPort> result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(result, hasItem(host1));
+            assertThat(result, hasItem(host2));
+
+            // descending
+            setScores(endpoints, hosts,
+                    50, 9, 1);
+            order = Arrays.asList(host3, host2, host1);
+            assertEquals(order, ReplicaPlans.sortByProximity(Arrays.asList(host1, host2, host3)));
+
+            result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(result, hasItem(host2));
+            assertThat(result, hasItem(host3));
+        }
+    }
+
+    /**
+     * Test with {@link Config.BatchlogEndpointStrategy#dynamic_remote}.
+     */
+    @Test
+    public void shouldSelectOneFastestHostsFromNonLocalRackWithDynamicSnitchRemote()
+    {
+        for (int i = 0; i < repetitions; i++)
+        {
+            // for each rack, get last host (only in test), then sort all endpoints from each rack by scores
+            Multimap<String, InetAddressAndPort> endpoints = configure(Config.BatchlogEndpointStrategy.dynamic_remote, true,
+                    Arrays.asList(LOCAL, "r1", "r2"),
+                    15, 15, 15);
+            InetAddressAndPort r0h1 = endpointAddress(0, 1);
+            InetAddressAndPort r1h1 = endpointAddress(1, 0);
+            InetAddressAndPort r1h2 = endpointAddress(1, 1);
+            InetAddressAndPort r2h1 = endpointAddress(2, 0);
+            InetAddressAndPort r2h2 = endpointAddress(2, 1);
+            List<InetAddressAndPort> hosts = Arrays.asList(r0h1, r1h1, r1h2, r2h1, r2h2);
+
+            // ascending
+            setScores(endpoints, hosts,
+                    1,
+                    10, 12,
+                    5, 10);
+
+            Collection<InetAddressAndPort> result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            filterBatchlogEndpointsDynamicForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(result, hasItem(r1h1));
+            assertThat(result, hasItem(r2h1));
+
+            // descending
+            setScores(endpoints, hosts,
+                    1, // rack 0
+                    20, 5, // rack 1
+                    0, 15); // rack 2
+            result = filterBatchlogEndpointsDynamicForTests(endpoints);
+            assertThat(result.size(), is(2));
+            assertThat(result, hasItem(r1h2));
+            assertThat(result, hasItem(r2h1));
+        }
+    }
+
+    private void setScores(Multimap<String, InetAddressAndPort> endpoints,
+                           List<InetAddressAndPort> hosts,
+                           Integer... scores)
+    {
+        int maxScore = 0;
+
+        // set the requested scores for the requested hosts
+        for (int round = 0; round < 50; round++)
+        {
+            for (int i = 0; i < hosts.size(); i++)
+            {
+                dsnitch.receiveTiming(hosts.get(i), scores[i], MILLISECONDS);
+                maxScore = Math.max(maxScore, scores[i]);
+            }
+        }
+
+        // set some random (higher) scores for unrequested hosts
+        for (InetAddressAndPort ep : endpoints.values())
+        {
+            if (hosts.contains(ep))
+                continue;
+            for (int r = 0; r < 1; r++)
+                dsnitch.receiveTiming(ep, maxScore + ThreadLocalRandom.current().nextInt(100) + 1, MILLISECONDS);
+        }
+
+        dsnitch.updateScores();
+    }
+
+    private int nodeInRack(InetAddressAndPort input)
+    {
+        return input.getAddress().getAddress()[3];
+    }
+
+    private static InetAddressAndPort endpointAddress(int rack, int nodeInRack)
+    {
+        if (rack == 0 && nodeInRack == 0)
+            return FBUtilities.getBroadcastAddressAndPort();
+
+        try
+        {
+            return InetAddressAndPort.getByAddress(new byte[]{ 0, 0, (byte) rack, (byte) nodeInRack });
+        }
+        catch (UnknownHostException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private Multimap<String, InetAddressAndPort> configure(Config.BatchlogEndpointStrategy batchlogEndpointStrategy,
+                                                           boolean dynamicSnitch,
+                                                           List<String> racks,
+                                                           int... nodesPerRack)
+    {
+        DatabaseDescriptor.setDynamicBadnessThreshold(0.1);
+        StorageService.instance.unsafeInitialize();
+
+        // if any of the three assertions fires, your test is busted
+        assert !racks.isEmpty();
+        assert racks.size() <= 10;
+        assert racks.size() == nodesPerRack.length;
+
+        ImmutableMultimap.Builder<String, InetAddressAndPort> builder = ImmutableMultimap.builder();
+        for (int r = 0; r < racks.size(); r++)
+        {
+            String rack = racks.get(r);
+            for (int n = 0; n < nodesPerRack[r]; n++)
+                builder.put(rack, endpointAddress(r, n));
+        }
+
+        ImmutableMultimap<String, InetAddressAndPort> endpoints = builder.build();
+
+        reconfigure(batchlogEndpointStrategy, dynamicSnitch, endpoints);
+
+        return endpoints;
+    }
+
+    private void reconfigure(Config.BatchlogEndpointStrategy batchlogEndpointStrategy,
+                             boolean dynamicSnitch,
+                             Multimap<String, InetAddressAndPort> endpoints)
+    {
+        DatabaseDescriptor.setBatchlogEndpointStrategy(batchlogEndpointStrategy);
+
+        if (DatabaseDescriptor.getEndpointSnitch() instanceof DynamicEndpointSnitch)
+            ((DynamicEndpointSnitch) DatabaseDescriptor.getEndpointSnitch()).close();
+
+        Multimap<InetAddressAndPort, String> endpointRacks = Multimaps.invertFrom(endpoints, ArrayListMultimap.create());
+        GossipingPropertyFileSnitch gpfs = new GossipingPropertyFileSnitch()
+        {
+            @Override
+            public String getDatacenter(InetAddressAndPort endpoint)
+            {
+                return "dc1";
+            }
+
+            @Override
+            public String getRack(InetAddressAndPort endpoint)
+            {
+                return endpointRacks.get(endpoint).iterator().next();
+            }
+        };
+        IEndpointSnitch snitch;
+        if (dynamicSnitch)
+            snitch = dsnitch = new DynamicEndpointSnitch(gpfs, String.valueOf(gpfs.hashCode()));
+        else
+        {
+            dsnitch = null;
+            snitch = gpfs;
+        }
+
+        DatabaseDescriptor.setDynamicBadnessThreshold(0);
+        DatabaseDescriptor.setEndpointSnitch(snitch);
+
+        DatabaseDescriptor.setBatchlogEndpointStrategy(batchlogEndpointStrategy);
+    }
+
+    private void withConfigs(Stream<Supplier<Multimap<String, InetAddressAndPort>>> supplierStream,
+                             Consumer<Multimap<String, InetAddressAndPort>> testFunction)
+    {
+        supplierStream.map(Supplier::get)
+                .forEach(endpoints -> {
+                    try
+                    {
+                        testFunction.accept(endpoints);
+                    }
+                    catch (AssertionError e)
+                    {
+                        throw new AssertionError(configToString(endpoints), e);
+                    }
+                });
+    }
+
+    private String configToString(Multimap<String, InetAddressAndPort> endpoints)
+    {
+        return "strategy:" + DatabaseDescriptor.getBatchlogEndpointStrategy()
+                + " snitch:" + DatabaseDescriptor.getEndpointSnitch().getClass().getSimpleName()
+                + " nodes-per-rack: " + endpoints.asMap().entrySet().stream()
+                .map(e -> e.getKey() + '=' + e.getValue().size())
+                .collect(Collectors.joining());
+    }
+
 }
diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
index 90a1d73c2a2c..b4e94601a5bd 100644
--- a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
+++ b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
@@ -27,6 +27,7 @@
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.Util.PartitionerSwitcher;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -41,7 +42,6 @@
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -114,8 +114,8 @@ public void testDelete()
                 .applyUnsafe();
 
         DecoratedKey dk = cfs.decorateKey(ByteBufferUtil.bytes("1234"));
-        ImmutableBTreePartition results = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, dk).build());
-        Iterator<Row> iter = results.iterator();
+        Partition results = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, dk).build());
+        Iterator<Row> iter = results.rowIterator();
         assert iter.hasNext();
 
         Mutation mutation = new Mutation(PartitionUpdate.fullPartitionDelete(cfm,
@@ -226,7 +226,7 @@ public void testTruncatedReplay() throws InterruptedException, ExecutionExceptio
             long timestamp = currentTimeMillis() - BatchlogManager.getBatchlogTimeout();
 
             if (i == 500)
-                SystemKeyspace.saveTruncationRecord(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2),
+                SystemKeyspace.saveTruncationRecord(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).metadata.id,
                                                     timestamp,
                                                     CommitLogPosition.NONE);
 
diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogTest.java
index 194ae91ea141..e5bf16d7f4d9 100644
--- a/test/unit/org/apache/cassandra/batchlog/BatchlogTest.java
+++ b/test/unit/org/apache/cassandra/batchlog/BatchlogTest.java
@@ -27,7 +27,6 @@
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.RowUpdateBuilder;
@@ -36,8 +35,12 @@
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 
@@ -104,4 +107,55 @@ public void testSerialization() throws IOException
             }
         }
     }
+
+    @Test
+    public void testDseDeserialization() throws IOException
+    {
+        SchemaLoader.prepareServer();
+
+        String keyspace = "testDseDeserialization";
+
+        TableMetadata.Builder table = SchemaLoader.standardCFMD(keyspace, "batchlog", 1, BytesType.instance);
+        table.id(TableId.fromString("397cb220-bedc-11ee-a2a7-39f39072efe4"));
+
+        SchemaLoader.createKeyspace(keyspace,
+                                    KeyspaceParams.simple(1),
+                                    table);
+
+        TableMetadata cfm = Keyspace.open(keyspace).getColumnFamilyStore("batchlog").metadata();
+
+        // prepare a batch locally
+        long now = 1706556356256000L;
+        long mutationTimestamp = now + 10;
+        TimeUUID uuid = TimeUUID.fromString("398b0a00-bedc-11ee-a2a7-39f39072efe4");
+
+        List<Mutation> mutations = new ArrayList<>(10);
+        for (int i = 0; i < 10; i++)
+        {
+            mutations.add(new RowUpdateBuilder(cfm, mutationTimestamp, bytes(i))
+                          .clustering("name" + i)
+                          .add("val", "val" + i)
+                          .build());
+        }
+        Batch batch1 = Batch.createLocal(uuid, now, mutations);
+        assertEquals(uuid, batch1.id);
+        assertEquals(now, batch1.creationTime);
+        assertEquals(mutations, batch1.decodedMutations);
+
+        // deserialize the same (hopefully) batch which was serialized in dse 6.8
+        File f = new File("test/data/serialization/DSE_68/batch.bin");
+        assert f.exists() : f.path();
+
+        DataInputPlus dis = new FileInputStreamPlus(f);
+
+        int version = MessagingService.VERSION_DSE_68;
+        Batch batch2 = Batch.serializer.deserialize(dis, version);
+
+        // expect batches to be equal, i.e. downgrading from 6.8 batch to CC batch works
+        assertEquals(batch1.id, batch2.id);
+        assertEquals(batch1.creationTime, batch2.creationTime);
+        assertEquals(batch1.decodedMutations.size(), batch2.decodedMutations.size());
+
+        assertEquals(batch1.decodedMutations.toString(), batch2.decodedMutations.toString());
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
index 50145e10b363..7b07cec7baef 100644
--- a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
+++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
@@ -62,14 +62,14 @@ public static void defineSchema() throws ConfigurationException
     @Test
     public void testSerializeAndLoadKeyCache0kB() throws Exception
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         doTestSerializeAndLoadKeyCache();
     }
 
     @Test
     public void testSerializeAndLoadKeyCache() throws Exception
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         doTestSerializeAndLoadKeyCache();
     }
 
diff --git a/test/unit/org/apache/cassandra/cache/CachingRebuffererTest.java b/test/unit/org/apache/cassandra/cache/CachingRebuffererTest.java
new file mode 100644
index 000000000000..e3d45d2cd609
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cache/CachingRebuffererTest.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cache;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.ChunkReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.memory.BufferPool;
+import org.apache.cassandra.utils.memory.BufferPools;
+import org.github.jamm.MemoryMeter;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class CachingRebuffererTest
+{
+    private final int PAGE_SIZE = 4096;
+    private File file;
+    private ChunkReader chunkReader;
+    private BufferPool bufferPool;
+    private ChannelProxy blockingChannel;
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Before
+    public void setUp() throws IOException
+    {
+        assumeNotNull(ChunkCache.instance);
+
+        file = new File(java.io.File.createTempFile("CachingRebuffererTest", ""));
+        file.deleteOnExit();
+
+        blockingChannel = new ChannelProxy(file);
+        chunkReader = Mockito.mock(ChunkReader.class);
+        bufferPool = BufferPools.forChunkCache();
+
+        when(chunkReader.chunkSize()).thenReturn(PAGE_SIZE);
+        when(chunkReader.channel()).thenReturn(blockingChannel);
+        when(chunkReader.type()).thenReturn(ChunkReader.ReaderType.SIMPLE);
+
+        ChunkCache.instance.invalidateFile(file);
+    }
+
+    // Helper test to estimate the memory overhead caused by buffer cache
+    @Ignore
+    @Test
+    public void calculateMemoryOverhead() throws InterruptedException
+    {
+        // Allocate 1,5M items
+        long count = 1_500_000;
+
+        class EmptyAllocatingChunkReader implements ChunkReader
+        {
+            public void readChunk(long chunkOffset, ByteBuffer toBuffer)
+            {
+            }
+
+            public int chunkSize()
+            {
+                return PAGE_SIZE;
+            }
+
+            public BufferType preferredBufferType()
+            {
+                return BufferType.OFF_HEAP;
+            }
+
+            public Rebufferer instantiateRebufferer()
+            {
+                return null;
+            }
+
+            public void invalidateIfCached(long position)
+            {
+                // do nothing
+            }
+
+            public void close()
+            {
+
+            }
+
+            public ChannelProxy channel()
+            {
+                return blockingChannel;
+            }
+
+            public long fileLength()
+            {
+                return count * PAGE_SIZE; // enough to cover all keys requested or else the chunk length will overflow
+            }
+
+            public double getCrcCheckChance()
+            {
+                return 0;
+            }
+
+            public ReaderType type()
+            {
+                return ReaderType.SIMPLE;
+            }
+        }
+
+        Rebufferer rebufferer = ChunkCache.instance.maybeWrap(new EmptyAllocatingChunkReader()).instantiateRebufferer();
+        final MemoryMeter memoryMeter = MemoryMeter.builder()
+                                                   .withGuessing(MemoryMeter.Guess.UNSAFE)
+                                                   .build();
+        final long initialHeap = memoryMeter.measureDeep(ChunkCache.instance);
+        System.out.println("initial deepSize = " + FBUtilities.prettyPrintMemory(initialHeap));
+
+        // Cache them count times
+        for (long i = 0; i < count; i++)
+            rebufferer.rebuffer(i * PAGE_SIZE).release();
+
+        long queriedSize = 1L * PAGE_SIZE * count;
+        System.out.println("queriedSize = " + FBUtilities.prettyPrintMemory(queriedSize));
+
+        long cachedCount = ChunkCache.instance.size();
+        long cachedSize = ChunkCache.instance.weightedSize();
+
+        System.out.println("cachedCount = " + cachedCount);
+        System.out.println("cachedSize = " + FBUtilities.prettyPrintMemory(cachedSize));
+
+        final long populatedHeap = memoryMeter.measureDeep(ChunkCache.instance);
+        System.out.println("populated deepSize = " + FBUtilities.prettyPrintMemory(populatedHeap));
+        System.out.println("deepSizeDelta/cachedCount = " + FBUtilities.prettyPrintBinary((populatedHeap - initialHeap) * 1.0 / cachedCount, "B", ""));
+
+        ChunkCache.instance.clear();
+
+        System.out.println("cleared deepSize = " + FBUtilities.prettyPrintMemory(memoryMeter.measureDeep(ChunkCache.instance)));
+    }
+
+    @Test
+    public void testRebufferInSamePage()
+    {
+        when(chunkReader.chunkSize()).thenReturn(PAGE_SIZE);
+        doNothing().when(chunkReader).readChunk(anyLong(), any());
+
+        Rebufferer rebufferer = ChunkCache.instance.maybeWrap(chunkReader).instantiateRebufferer();
+        assertNotNull(rebufferer);
+
+        for (int i = 0; i < PAGE_SIZE; i++)
+        {
+            Rebufferer.BufferHolder buffer = rebufferer.rebuffer(i);
+            assertNotNull(buffer);
+            assertEquals(PAGE_SIZE, buffer.buffer().capacity());
+            assertEquals(0, buffer.offset());
+            buffer.release();
+        }
+
+        verify(chunkReader, times(1)).readChunk(anyLong(), any());
+    }
+
+    @Test
+    public void testRebufferSeveralPages()
+    {
+        Rebufferer rebufferer = ChunkCache.instance.maybeWrap(chunkReader).instantiateRebufferer();
+        assertNotNull(rebufferer);
+
+        doNothing().when(chunkReader).readChunk(anyLong(), any());
+
+        final int numPages = 10;
+
+        for (int j = 0; j < numPages; j++)
+        {
+            for (int i = j * PAGE_SIZE; i < (j + 1) * PAGE_SIZE; i ++)
+            {
+                Rebufferer.BufferHolder buffer = rebufferer.rebuffer(i);
+                assertNotNull(buffer);
+                assertEquals(PAGE_SIZE, buffer.buffer().capacity());
+                assertEquals(j * PAGE_SIZE, buffer.offset());
+                buffer.release();
+            }
+        }
+
+        verify(chunkReader, times(numPages)).readChunk(anyLong(), any());
+    }
+
+    @Test
+    public void testRebufferContendedPage() throws InterruptedException
+    {
+        final int numThreads = 15;
+        final int numAttempts = 1024;
+        final CountDownLatch doneLatch = new CountDownLatch(numThreads);
+        final AtomicReference<Throwable> error = new AtomicReference<>(null);
+        final AtomicInteger numBuffers = new AtomicInteger(0);
+
+        doAnswer(i -> {
+            numBuffers.incrementAndGet();
+            return null;
+        }).when(chunkReader).readChunk(anyLong(), any());
+
+        Rebufferer rebufferer = ChunkCache.instance.maybeWrap(chunkReader).instantiateRebufferer();
+        assertNotNull(rebufferer);
+
+        // using NamedThreadFactory ensures that the threads are InlinedThreadLocalThread, which means the pool will cache buffers in a thread local stash
+        ExecutorService executor = Executors.newFixedThreadPool(numThreads, new NamedThreadFactory("testMultipleThreadsOneSizeSepAllocFree"));
+
+        for (int j = 0; j < numThreads; j++)
+        {
+            executor.submit(() -> {
+                try
+                {
+                    for (int i = 0; i < numAttempts; i++)
+                    {
+                        Rebufferer.BufferHolder buffer = rebufferer.rebuffer(0);
+                        assertNotNull(buffer);
+                        assertEquals(PAGE_SIZE, buffer.buffer().capacity());
+                        assertEquals(0, buffer.offset());
+                        buffer.release();
+
+                        // removes the buffer from the cache, other threads should still be able to create a new one and
+                        // insert it into the cache thanks to the ref. counting mechanism
+                        ((ChunkCache.CachingRebufferer) rebufferer).invalidateIfCached(0);
+                    }
+                }
+                catch (Throwable t)
+                {
+                    t.printStackTrace();
+                    error.set(t);
+                }
+                finally
+                {
+                    doneLatch.countDown();
+                }
+            });
+        }
+
+        doneLatch.await(1, TimeUnit.MINUTES);
+        assertNull(error.get());
+        assertTrue(numBuffers.get() > 1); // there should be several buffers created, in the thousands, up to numThreads * numAttempts
+    }
+
+    @Test(expected = CorruptSSTableException.class)
+    public void testExceptionInReadChunk()
+    {
+        doThrow(CorruptSSTableException.class).when(chunkReader).readChunk(anyLong(), any());
+
+        Rebufferer rebufferer = ChunkCache.instance.maybeWrap(chunkReader).instantiateRebufferer();
+        assertNotNull(rebufferer);
+
+        rebufferer.rebuffer(0);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cache/ChunkCacheInterceptingTest.java b/test/unit/org/apache/cassandra/cache/ChunkCacheInterceptingTest.java
new file mode 100644
index 000000000000..7b4577096273
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cache/ChunkCacheInterceptingTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cache;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.ChunkReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.RebuffererFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ChunkCacheInterceptingTest
+{
+    Interceptor interceptor;
+
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void testChunkCacheInterception()
+    {
+        assumeTrue(ChunkCache.instance != null);
+
+        try
+        {
+            ChunkCache.instance.intercept(rf -> intercept(rf));
+
+            ChunkReader chunkReader = mock(ChunkReader.class);
+            when(chunkReader.chunkSize()).thenReturn(1024);
+            when(chunkReader.channel()).thenReturn(new ChannelProxy(new File("")));
+            when(chunkReader.type()).thenReturn(ChunkReader.ReaderType.SIMPLE);
+
+            RebuffererFactory rebuferrerFactory = ChunkCache.instance.maybeWrap(chunkReader);
+            assertTrue("chunk cache didn't create our interceptor?", interceptor != null);
+
+            rebuferrerFactory.instantiateRebufferer();
+            assertEquals("interceptor not used to create rebufferer?",1, interceptor.numInstantiations);
+        }
+        finally
+        {
+            ChunkCache.instance.enable(true);
+        }
+    }
+
+    private RebuffererFactory intercept(RebuffererFactory rf)
+    {
+        interceptor = new Interceptor(rf);
+        return interceptor;
+    }
+
+    private static class Interceptor implements RebuffererFactory
+    {
+        private RebuffererFactory wrapped;
+        private int numInstantiations = 0;
+
+        Interceptor(RebuffererFactory wrapped)
+        {
+            this.wrapped = wrapped;
+        }
+
+        @Override
+        public void close()
+        {
+            wrapped.close();
+        }
+
+        @Override
+        public ChannelProxy channel()
+        {
+            return wrapped.channel();
+        }
+
+        @Override
+        public long fileLength()
+        {
+            return wrapped.fileLength();
+        }
+
+        @Override
+        public double getCrcCheckChance()
+        {
+            return wrapped.getCrcCheckChance();
+        }
+
+        @Override
+        public Rebufferer instantiateRebufferer()
+        {
+            numInstantiations += 1;
+            return wrapped.instantiateRebufferer();
+        }
+
+        @Override
+        public void invalidateIfCached(long position)
+        {
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/cache/ChunkCacheLoadingTest.java b/test/unit/org/apache/cassandra/cache/ChunkCacheLoadingTest.java
new file mode 100644
index 000000000000..ec1ea3d998fc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cache/ChunkCacheLoadingTest.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cache;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.RebuffererFactory;
+import org.apache.cassandra.io.util.WrappingRebufferer;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitConfig;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeNotNull;
+
+/**
+ * Exercises loading and storing chunks in the chunk cache, as well as error conditions around that.
+ */
+@RunWith(BMUnitRunner.class)
+@BMUnitConfig(debug = true)
+public class ChunkCacheLoadingTest
+{
+    private static final String KEYSPACE = "db3050";
+    private static final String COUNTER = "counter";
+    private static final String NORMAL = "normal";
+
+    private static volatile RebufferInterceptingRebufferer rebufferInterceptor;
+
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.daemonInitialization();
+
+        assumeNotNull(ChunkCache.instance);
+
+        SchemaLoader.prepareServer();
+
+        // Set up chunk cache interception, so that rebuffering results can be inspected and consumed.
+        ChunkCache.instance.intercept(rf ->
+                                      {
+                                          rebufferInterceptor = new RebufferInterceptingRebufferer(rf.instantiateRebufferer());
+                                          return rebufferInterceptor;
+                                      });
+
+
+        // A counter table, used for setting up and triggering reads from the counter cache.
+        // With table params for a custom compressor that issues a read on each uncompression.
+        TableMetadata counterTable = TableMetadata.builder(KEYSPACE, COUNTER)
+                                                  .isCounter(true)
+                                                  .addPartitionKeyColumn("key", Int32Type.instance)
+                                                  .addRegularColumn("c", CounterColumnType.instance)
+                                                  .compression(CompressionParams.fromMap(ImmutableMap.of(CompressionParams.CLASS,
+                                                                                                         ReadingNopCompressor.class.getName())))
+                                                  .build();
+        // A regular table, used for setting up chunk cache collisions with different chunks.
+        TableMetadata normalTable = TableMetadata.builder(KEYSPACE, NORMAL)
+                                                 .addPartitionKeyColumn("key", Int32Type.instance)
+                                                 .addRegularColumn("val", Int32Type.instance)
+                                                 .build();
+
+        SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), counterTable, normalTable);
+    }
+
+    @Before
+    public void setUp()
+    {
+        assumeNotNull(ChunkCache.instance);
+    }
+
+    @AfterClass
+    public static void cleanup()
+    {
+        SchemaLoader.cleanupSavedCaches();
+    }
+
+    // See DB-3050
+    @BMRule(name = "Ensure chunk cache collisions",
+            targetClass = "org.apache.cassandra.cache.ChunkCache$Key",
+            targetMethod = "hashCode()",
+            // This is needed in order to ensure that loads of different chunks will result in colliding loads in the
+            // underlying Caffeine cache. We want to ensure that because in 6.0 and above (unlike in 5.1) doing a nested read
+            // of the same chunk is much harder to handle without resulting in a stall, mostly because of the changes related
+            // to the chunk cache now storing chunk futures instead of the chunks themselves.
+            condition = "$0.internedPath.contains(\"db3050\") && $0.internedPath.contains(\"Data.db\")",
+            action = "return 1")
+    @Test(timeout=5000)
+    public void testUncompressionReadCollision() throws Exception
+    {
+        // Write a single row into the counter table, then flush it, so that a read from it will trigger a chunk cache load.
+        ColumnFamilyStore counterCfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(COUNTER);
+        new CounterMutation(new RowUpdateBuilder(counterCfs.metadata(), 0, bytes(1)).add("c", 12L).build(), ConsistencyLevel.ONE).apply();
+        counterCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        // Write a single row into the normal table, then flush it, so that it can be used for the nested, colliding reads from
+        // within the custom compressor.
+        ColumnFamilyStore normalCfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(NORMAL);
+        new RowUpdateBuilder(normalCfs.metadata(), 0, bytes(1)).add("val", 34).build().apply();
+        normalCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        // Invalidate everything from the chunk cache but the SSTable partition index files, as invalidating these files will
+        // cause an async read for the partition index during the counter read on cache reload. This will cause a jump from the
+        // BackgroundIoStage thread to a TPC thread, and we want the BackgroundIoStage to keep going until it blocks due to a
+        // blocking secondary read done in some of the ReadingNopCompressor methods.
+        Set<File> sstableDataFilePaths = new HashSet<>();
+        for (var dataDirectory : Directories.dataDirectories)
+            addNonPartitionFiles(sstableDataFilePaths, dataDirectory.location);
+        sstableDataFilePaths.forEach(ChunkCache.instance::invalidateFile);
+
+        // Trigger a counter cache load by triggering a flush and then a load (without flushing the load will be a NOP). We
+        // don't even need to invalidate the cache after the flush. The load then is guaranteed to trigger the DB-3050 error
+        // condition - before the DB-3050 fix, this means recursively calling a chunk cache load (i.e. recursively calling a
+        // ConcurrentHashMap.computeIfAbsent) and hitting a collision with the reserved bucket from the first call in the second
+        // call. The guarantee comes from issuing a read for the exact same partition during the uncompression part of the
+        // chunk cache load for the first read.
+        CacheService.instance.counterCache.submitWrite(Integer.MAX_VALUE).get();
+        CacheService.instance.counterCache.loadSaved();
+
+        ColumnMetadata cm = counterCfs.metadata().getColumn(ByteBufferUtil.bytes("c"));
+        assertEquals(12L, counterCfs.getCachedCounter(CounterCacheKey.create(counterCfs.metadata(), bytes(1), Clustering.EMPTY, cm, null)).count);
+    }
+
+    private static void addNonPartitionFiles(Set<File> sstableDataFilePaths, File... roots)
+    {
+        if (roots == null)
+            return;
+
+        for (File file : roots)
+        {
+            if (file.isDirectory())
+            {
+                addNonPartitionFiles(sstableDataFilePaths, file.tryList());
+                continue;
+            }
+            assert file.isFile();
+            String absolutePath = file.path();
+            if (!absolutePath.contains("Partitions.db"))
+                sstableDataFilePaths.add(file);
+        }
+    }
+
+    /**
+     * A custom compressor that issues a read on each uncompression and each compressed buffer length check.
+     * Used to trigger a nested read during chunk cache loading.
+     */
+    public static class ReadingNopCompressor implements ICompressor
+    {
+        private static final AtomicInteger nestedReadsCounter = new AtomicInteger();
+
+        public static ReadingNopCompressor create(Map<String, String> options)
+        {
+            return new ReadingNopCompressor();
+        }
+
+        @Override
+        public int initialCompressedBufferLength(int chunkLength)
+        {
+            // Limit the number of nested reads to avoid stack overflow. It's important that the NORMAL table doesn't use
+            // the ReadingNopCompressor, otherwise it can issue nested reads for the same chunk which could cause stalling.
+            if (nestedReadsCounter.incrementAndGet() < 10)
+                // Using the COUNTER table instead of the NORMAL one will actually result in stalling the test.
+                QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE key=1;", KEYSPACE, NORMAL));
+            return chunkLength;
+        }
+
+        @Override
+        public int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset)
+        {
+            System.arraycopy(input, inputOffset, output, outputOffset, inputLength);
+            return inputLength;
+        }
+
+        @Override
+        public void compress(ByteBuffer input, ByteBuffer output)
+        {
+            ByteBufferUtil.put(input, output);
+        }
+
+        @Override
+        public void uncompress(ByteBuffer input, ByteBuffer output)
+        {
+            // Limit the number of nested reads to avoid stack overflow. It's important that the NORMAL table doesn't use
+            // the ReadingNopCompressor, otherwise it can issue nested reads for the same chunk which could cause stalling.
+            if (nestedReadsCounter.incrementAndGet() < 10)
+                // Using the COUNTER table instead of the NORMAL one will actually result in stalling the test.
+                QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE key=1;", KEYSPACE, NORMAL));
+            ByteBufferUtil.put(input, output);
+        }
+
+        @Override
+        public Set<String> supportedOptions()
+        {
+            return Collections.emptySet();
+        }
+
+        @Override
+        public BufferType preferredBufferType()
+        {
+            return BufferType.OFF_HEAP;
+        }
+
+        @Override
+        public boolean supports(BufferType bufferType)
+        {
+            return true;
+        }
+    }
+
+    @BMRule(name = "Throw once during async load of compressed chunk",
+            targetClass = "org.apache.cassandra.io.util.CompressedChunkReader$Standard",
+            targetMethod = "readChunk(long, java.nio.ByteBuffer)",
+            condition = "not flagged(\"throw\")" +
+                    "&& $0.toString().contains(\"db3050\")" +
+                    "&& $0.toString().contains(\"normal\")" +
+                    "&& $0.toString().contains(\"Data.db\")",
+            action = "flag(\"throw\");" +
+                    "throw new RuntimeException(\"Nope, no chunk for you!\")")
+    @Test
+    public void testChunkAsyncLoadThrows()
+    {
+        prepareChunkCache();
+
+        try
+        {
+            // A single disk read from the NORMAL table - chunk loading should be triggered, and it should throw a
+            // RuntimeException.
+            QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE key=1;", KEYSPACE, NORMAL));
+            fail("Expected read to throw due to RuntimeException during load");
+        }
+        catch (Throwable t)
+        {
+            // Expect the RuntimeException set up with Byteman to be thrown.
+            if (!(t instanceof RuntimeException) || !t.getMessage().contains("no chunk for you"))
+                fail(String.format("Expected cause to be RuntimeException, but got %s with message %s",
+                                   t.getClass().getSimpleName(),
+                                   t.getMessage() == null ? "" : t.getMessage()));
+        }
+
+        AtomicBoolean rebufferCalled = new AtomicBoolean();
+        rebufferInterceptor.setRebufferExecutionHandler((bufferHolder, throwable) ->
+                                                        {
+                                                            rebufferInterceptor.setRebufferExecutionHandler(null);
+                                                            Assert.assertTrue(rebufferCalled.compareAndSet(false, true));
+                                                            Assert.assertTrue(bufferHolder != null);
+                                                        });
+        // The same single disk read from the NORMAL table - chunk loading should be triggered again as the cache
+        // shouldn't contain a future for the previous unsuccessful read, and the read should be successful.
+        UntypedResultSet result = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE key=1;", KEYSPACE, NORMAL));
+        Assert.assertEquals(34, result.one().getInt("val"));
+        Assert.assertTrue(rebufferCalled.get());
+
+        // TODO dimitar.dimitrov Add a test about concurrent reads, where one throws, and the other tries to obtain the same
+        //      chunk/chunk future.
+    }
+
+    /**
+     * Creates a single-row SSTable for the NORMAL table and invalidates chunk cache entries about that SSTable.
+     */
+    private static void prepareChunkCache()
+    {
+        // Write a single row into the normal table, then flush it, so that follow-up reads for that table and partition will
+        // have to read from disk.
+        ColumnFamilyStore normalCfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(NORMAL);
+        new RowUpdateBuilder(normalCfs.metadata(), 0, bytes(1)).add("val", 34).build().apply();
+        normalCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        // Invalidate everything from the chunk cache (except the SSTable partition index files) to make sure that follow-up
+        // reads for that table trigger chunk cache loading.
+        Set<File> sstableDataFilePaths = new HashSet<>();
+        for (var dataDirectory : Directories.dataDirectories)
+            addNonPartitionFiles(sstableDataFilePaths, dataDirectory.location);
+        sstableDataFilePaths.forEach(ChunkCache.instance::invalidateFile);
+    }
+
+    /**
+     * A wrapping rebufferer that allows customizable inspection and consumption of the results of the
+     * {@link Rebufferer#rebuffer(long)} calls triggered by the inspected chunk cache.
+     * Also implements {@link RebuffererFactory} in order to be used with the {@link ChunkCache#intercept(Function)} API,
+     * even though it always returns itself as a {@link Rebufferer}.
+     */
+    private static class RebufferInterceptingRebufferer extends WrappingRebufferer implements RebuffererFactory
+    {
+        private volatile BiConsumer<BufferHolder, Throwable> rebufferExecutionHandler;
+
+        RebufferInterceptingRebufferer(Rebufferer source)
+        {
+            super(source);
+        }
+
+        @Override
+        public BufferHolder rebuffer(long position)
+        {
+            if (rebufferExecutionHandler == null)
+                return wrapped.rebuffer(position);
+
+            BufferHolder bufferHolder;
+            try
+            {
+                bufferHolder = wrapped.rebuffer(position);
+            }
+            catch (Throwable t)
+            {
+                rebufferExecutionHandler.accept(null, t);
+                throw t;
+            }
+            rebufferExecutionHandler.accept(bufferHolder, null);
+            return bufferHolder;
+        }
+
+        /**
+         * Allows the injection of a custom consumer for the results of the {@link #rebuffer(long)} calls
+         * to the wrapped rebufferer.
+         *
+         * @param rebufferExecutionHandler The consumer of the results of the {@link #rebuffer(long)} calls
+         *                                 to the wrapped rebufferer. If null, {@link #rebuffer(long)} calls
+         *                                 to {@code RebufferInterceptingRebufferer} will just call the wrapped rebufferer.
+         */
+        void setRebufferExecutionHandler(BiConsumer<BufferHolder, Throwable> rebufferExecutionHandler)
+        {
+            this.rebufferExecutionHandler = rebufferExecutionHandler;
+        }
+
+        @Override
+        public Rebufferer instantiateRebufferer()
+        {
+            return this;
+        }
+
+        public void invalidateIfCached(long position)
+        {
+            // do nothing
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cache/ChunkCacheNotPresentTest.java b/test/unit/org/apache/cassandra/cache/ChunkCacheNotPresentTest.java
new file mode 100644
index 000000000000..aa32da1fb5ef
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cache/ChunkCacheNotPresentTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cache;
+
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.CompactionRealm;
+import org.apache.cassandra.db.compaction.unified.RealEnvironment;
+import org.apache.cassandra.schema.CompressionParams;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class ChunkCacheNotPresentTest
+{
+    @BeforeClass
+    public static void setupDD()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.disableChunkCache();
+    }
+
+    @Test
+    public void testCompressionParams()
+    {
+        assertNull(ChunkCache.instance);
+        CompressionParams params = CompressionParams.fromMap(ImmutableMap.of("class", "LZ4Compressor", "chunk_length_in_kb", "1"));
+        assertNotNull(params);
+    }
+
+    @Test
+    public void testRealEnvironment()
+    {
+        assertNull(ChunkCache.instance);
+        CompactionRealm mockRealm = Mockito.mock(CompactionRealm.class);
+        RealEnvironment env = new RealEnvironment(mockRealm);
+        env.cacheMissRatio();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cache/ChunkCacheTest.java b/test/unit/org/apache/cassandra/cache/ChunkCacheTest.java
new file mode 100644
index 000000000000..5b69fbd2fdc3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cache/ChunkCacheTest.java
@@ -0,0 +1,570 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cache;
+
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Arrays;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import com.google.common.base.Throwables;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.metrics.ChunkCacheMetrics;
+import org.apache.cassandra.utils.memory.BufferPool;
+import org.awaitility.Awaitility;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+public class ChunkCacheTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(ChunkCacheTest.class);
+
+    @BeforeClass
+    public static void setupDD()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.enableChunkCache(512);
+    }
+
+    @Test
+    public void testRandomAccessReaderCanUseCache() throws IOException
+    {
+        File file = FileUtils.createTempFile("foo", null);
+        file.deleteOnExit();
+
+        ChunkCache.instance.clear();
+        assertEquals(0, ChunkCache.instance.size());
+        assertEquals(0, ChunkCache.instance.sizeOfFile(file));
+
+        try (SequentialWriter writer = new SequentialWriter(file))
+        {
+            writer.write(new byte[64]);
+            writer.flush();
+        }
+
+        FileHandle.Builder builder = new FileHandle.Builder(file).withChunkCache(ChunkCache.instance);
+        try (FileHandle h = builder.complete();
+             RandomAccessReader r = h.createReader())
+        {
+            r.reBuffer();
+
+            assertEquals(1, ChunkCache.instance.size());
+            assertEquals(1, ChunkCache.instance.sizeOfFile(file));
+        }
+
+        // We do not invalidate the file on close
+    }
+
+    @Test
+    public void testInvalidateFileNotInCache()
+    {
+        ChunkCache.instance.clear();
+        assertEquals(0, ChunkCache.instance.size());
+        ChunkCache.instance.invalidateFile(FileUtils.getTempDir().resolve("does/not/exist/in/cache/or/on/file/system"));
+    }
+
+    @Test
+    public void testRandomAccessReadersWithUpdatedFileAndMultipleChunksAndCacheInvalidation() throws IOException
+    {
+        File file = FileUtils.createTempFile("foo", null);
+        file.deleteOnExit();
+
+        ChunkCache.instance.clear();
+        assertEquals(0, ChunkCache.instance.size());
+        assertEquals(0, ChunkCache.instance.sizeOfFile(file));
+
+        writeBytes(file, new byte[RandomAccessReader.DEFAULT_BUFFER_SIZE * 3]);
+
+        FileHandle.Builder builder1 = new FileHandle.Builder(file).withChunkCache(ChunkCache.instance);
+             try (FileHandle handle1 = builder1.complete();
+                  RandomAccessReader reader1 = handle1.createReader())
+             {
+                 // Read 2 chunks and verify contents
+                 for (int i = 0; i < RandomAccessReader.DEFAULT_BUFFER_SIZE * 2; i++)
+                     assertEquals((byte) 0, reader1.readByte());
+
+                 // Overwrite the file's contents
+                 var bytes = new byte[RandomAccessReader.DEFAULT_BUFFER_SIZE * 3];
+                 Arrays.fill(bytes, (byte) 1);
+                 writeBytes(file, bytes);
+
+                 // Verify rebuffer pulls from cache for first 2 bytes and then from disk for third byte
+                 reader1.seek(0);
+                 for (int i = 0; i < RandomAccessReader.DEFAULT_BUFFER_SIZE * 2; i++)
+                     assertEquals((byte) 0, reader1.readByte());
+                 // Trigger read of next chunk and see it is the new data
+                 assertEquals((byte) 1, reader1.readByte());
+
+                 assertEquals(3, ChunkCache.instance.size());
+                 assertEquals(3, ChunkCache.instance.sizeOfFile(file));
+             }
+
+            // Invalidate cache for both chunks
+            ChunkCache.instance.invalidateFile(file);
+
+            // Verify cache does not contain an entry for the file
+            assertEquals(0, ChunkCache.instance.sizeOfFile(file));
+
+            // Existing handles and readers keep using the old file id. To make sure we get a new one, recreate the
+            // handle and reader.
+            try (FileHandle handle2 = builder1.complete();
+                 RandomAccessReader reader2 = handle2.createReader())
+            {
+                for (int i = 0; i < RandomAccessReader.DEFAULT_BUFFER_SIZE * 3; i++)
+                    assertEquals((byte) 1, reader2.readByte());
+                assertEquals(3, ChunkCache.instance.sizeOfFile(file));
+            }
+
+        // We do not invalidate the file on close
+    }
+
+    @Test
+    public void testRandomAccessReadersForDifferentFilesWithCacheInvalidation() throws IOException
+    {
+        File fileFoo = FileUtils.createTempFile("foo", null);
+        fileFoo.deleteOnExit();
+        File fileBar = FileUtils.createTempFile("bar", null);
+        fileBar.deleteOnExit();
+
+        ChunkCache.instance.clear();
+        assertEquals(0, ChunkCache.instance.size());
+        assertEquals(0, ChunkCache.instance.sizeOfFile(fileFoo));
+        assertEquals(0, ChunkCache.instance.sizeOfFile(fileBar));
+
+        writeBytes(fileFoo, new byte[64]);
+        // Write different bytes for meaningful content validation
+        var barBytes = new byte[64];
+        Arrays.fill(barBytes, (byte) 1);
+        writeBytes(fileBar, barBytes);
+
+        FileHandle.Builder builderFoo = new FileHandle.Builder(fileFoo).withChunkCache(ChunkCache.instance);
+        try (FileHandle handleFoo = builderFoo.complete();
+             RandomAccessReader readerFoo = handleFoo.createReader())
+        {
+            assertEquals((byte) 0, readerFoo.readByte());
+
+            assertEquals(1, ChunkCache.instance.size());
+            assertEquals(1, ChunkCache.instance.sizeOfFile(fileFoo));
+
+            FileHandle.Builder builderBar = new FileHandle.Builder(fileBar).withChunkCache(ChunkCache.instance);
+            try (
+                 FileHandle handleBar = builderBar.complete();
+                 RandomAccessReader readerBar = handleBar.createReader())
+            {
+                assertEquals((byte) 1, readerBar.readByte());
+
+                assertEquals(2, ChunkCache.instance.size());
+                assertEquals(1, ChunkCache.instance.sizeOfFile(fileFoo));
+                assertEquals(1, ChunkCache.instance.sizeOfFile(fileBar));
+
+                // Invalidate fileFoo and verify that only fileFoo's chunks are removed
+                ChunkCache.instance.invalidateFile(fileFoo);
+                assertEquals(0, ChunkCache.instance.sizeOfFile(fileFoo));
+                assertEquals(1, ChunkCache.instance.sizeOfFile(fileBar));
+            }
+        }
+        // We do not invalidate the file on close
+    }
+
+    private void writeBytes(File file, byte[] bytes) throws IOException
+    {
+        try (SequentialWriter writer = new SequentialWriter(file))
+        {
+            writer.write(bytes);
+            writer.flush();
+        }
+    }
+
+    static final class MockFileControl implements AutoCloseable
+    {
+        final File file;
+        final int fileSize;
+        FileChannel channel;
+        FileHandle fileHandle;
+        ChannelProxy proxy;
+        RandomAccessReader reader;
+        ChunkCache chunkCache;
+        volatile boolean reading;
+
+        CompletableFuture<?> waitOnRead = new CompletableFuture<>();
+
+        public MockFileControl(File file, int fileSize, ChunkCache chunkCache) throws Exception
+        {
+            this.file = file;
+            this.fileSize = fileSize;
+            this.chunkCache = chunkCache;
+        }
+
+        @Override
+        public void close() throws Exception
+        {
+            if (reader != null)
+                reader.close();
+            if (fileHandle != null)
+                fileHandle.close();
+            if (channel != null)
+                channel.close();
+        }
+
+        void createFile() throws Exception
+        {
+            file.deleteOnExit();
+
+            try (SequentialWriter writer = new SequentialWriter(file))
+            {
+                writer.write(new byte[fileSize]);
+                writer.flush();
+            }
+        }
+
+        RandomAccessReader openReader() throws Exception
+        {
+            assert reader == null;
+            channel = spy(FileChannel.class);
+            when(channel.read(any(ByteBuffer.class), anyLong())).thenAnswer(invocation -> {
+
+                reading = true;
+                logger.info("Waiting on read for file {}", file.path());
+                // this allows us to introduce a delay or a failure in the read
+                waitOnRead.join();
+                logger.info("Read completed for file {}", file.path());
+                reading = false;
+
+
+                ByteBuffer buffer = invocation.getArgument(0);
+                long position = invocation.getArgument(1);
+                int writen = buffer.remaining();
+                buffer.put(new byte[writen]);
+                return writen;
+            });
+            when(channel.size()).thenReturn(Long.valueOf(fileSize));
+
+            proxy = new ChannelProxy(file, channel);
+            FileHandle.Builder builder = new FileHandle.Builder(proxy.file())
+                                         .withChunkCache(chunkCache);
+            fileHandle = builder.complete(f -> proxy);
+            reader = fileHandle.createReader();
+
+            return reader;
+        }
+    }
+
+    /**
+     * This test asserts that in case of multiple threads reading from multiple files, the reads for one file
+     * are not blocked by the reads for another file.
+     * This is something that can happen on CNDB because we read data from the network (S3 or Storage Service)
+     * and it can be slow (or fail after some timeout).
+     */
+    @Test
+    public void testBlockReadsMultipleThreads() throws Exception
+    {
+        ChunkCache chunkCache = ChunkCache.instance;
+        chunkCache.clear();
+        assertEquals(0, chunkCache.size());
+        int numFiles = 64;
+        int fileSize = 64;
+
+        // reading from 1 file is very slow (blocked until we signal it to continue)
+        int slowFileIndex = 5;
+
+        MockFileControl[] files = new MockFileControl[numFiles];
+        try
+        {
+            for (int i = 0; i < numFiles; i++)
+            {
+                File file = FileUtils.createTempFile("foo" + i, ".tmp");
+                MockFileControl mockFileControl = new MockFileControl(file, fileSize, chunkCache);
+                files[i] = mockFileControl;
+                mockFileControl.createFile();
+                if (i != slowFileIndex)
+                {
+                    mockFileControl.waitOnRead.complete(null);
+                }
+                assertEquals(0, chunkCache.sizeOfFile(file));
+            }
+
+            ExecutorService threadPool = Executors.newFixedThreadPool(numFiles);
+
+            Future<?>[] results = new Future[numFiles];
+            for (int i = 0; i < numFiles; i++)
+            {
+                MockFileControl mockFileControl = files[i];
+                RandomAccessReader r = mockFileControl.openReader();
+                File file = mockFileControl.file;
+
+                results[i] = threadPool.submit(() -> {
+                    r.reBuffer();
+                    assertEquals(1, chunkCache.sizeOfFile(file));
+                });
+            }
+
+            // ensure that all the threads were able to complete, even if one was slow
+            for (int i = 0; i < numFiles; i++)
+            {
+                if (i != slowFileIndex)
+                {
+                    results[i].get();
+                }
+            }
+
+            // let the slow file finish
+            files[slowFileIndex].waitOnRead.complete(null);
+            results[slowFileIndex].get();
+        }
+        finally
+        {
+            for (MockFileControl file : files)
+            {
+                if (file != null)
+                {
+                    file.close();
+                }
+            }
+        }
+    }
+
+    /**
+     * This test asserts that in case of multiple threads reading from multiple files, the reads for one file
+     * are not blocked by the reads for another file.
+     * This is something that can happen on CNDB because we read data from the network (S3 or Storage Service)
+     * and it can be slow (or fail after some timeout).
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testNotCacheOnReadErrors() throws Exception
+    {
+        BufferPool pool = mock(BufferPool.class);
+        CopyOnWriteArrayList<ByteBuffer> allocated = new CopyOnWriteArrayList<>();
+        when(pool.get(anyInt(), any(BufferType.class))).thenAnswer(invocation -> {
+            int size = invocation.getArgument(0);
+            ByteBuffer buffer = ByteBuffer.allocateDirect(size);
+            allocated.add(buffer);
+            return buffer;
+        });
+
+        doAnswer(invocation -> {
+            ByteBuffer buffer = invocation.getArgument(0);
+            allocated.remove(buffer);
+            return true;
+        }).when(pool).put(any(ByteBuffer.class));
+        ChunkCache chunkCache = new ChunkCache(pool, 512, ChunkCacheMetrics::create);
+
+        assertEquals(0, chunkCache.size());
+        int fileSize = 64;
+        File file1 = FileUtils.createTempFile("foo1", ".tmp");
+        File file2 = FileUtils.createTempFile("foo2", ".tmp");
+        try (MockFileControl mockFileControl1 = new MockFileControl(file1, fileSize, chunkCache);
+             MockFileControl mockFileControl2 = new MockFileControl(file2, fileSize, chunkCache);)
+        {
+
+            mockFileControl1.createFile();
+            mockFileControl2.createFile();
+
+            // file 1 has an error during read, we shouldn't cache the handle
+            mockFileControl1.waitOnRead.completeExceptionally(new RuntimeException("some weird runtime error"));
+            RandomAccessReader r1 = mockFileControl1.openReader();
+            assertThrows(CompletionException.class, r1::reBuffer);
+            assertEquals(0, chunkCache.sizeOfFile(mockFileControl1.file));
+            assertEquals(0, chunkCache.size());
+            assertEquals(0, allocated.size());
+
+            // file 2 works fine, we should cache the handle
+            mockFileControl2.waitOnRead.complete(null);
+            RandomAccessReader r2 = mockFileControl2.openReader();
+            r2.reBuffer();
+            assertEquals(1, chunkCache.sizeOfFile(mockFileControl2.file));
+            assertEquals(1, chunkCache.size());
+            assertEquals(1, allocated.size());
+        }
+    }
+
+    @Test
+    public void testRacingReaders() throws Exception
+    {
+        testRacingReaders(false);
+    }
+
+    @Test
+    public void testRacingReadersWithError() throws Exception
+    {
+        testRacingReaders(true);
+    }
+
+    private void testRacingReaders(boolean injectReadError) throws Exception
+    {
+        BufferPool pool = mock(BufferPool.class);
+        CopyOnWriteArrayList<ByteBuffer> allocated = new CopyOnWriteArrayList<>();
+        when(pool.get(anyInt(), any(BufferType.class))).thenAnswer(invocation -> {
+            int size = invocation.getArgument(0);
+            ByteBuffer buffer = ByteBuffer.allocateDirect(size);
+            allocated.add(buffer);
+            return buffer;
+        });
+
+        doAnswer(invocation -> {
+            ByteBuffer buffer = invocation.getArgument(0);
+            allocated.remove(buffer);
+            return true;
+        }).when(pool).put(any(ByteBuffer.class));
+
+        ChunkCache chunkCache = new ChunkCache(pool, 512, ChunkCacheMetrics::create);
+        assertEquals(chunkCache.size(), 0);
+        int fileSize = 64;
+        File file1 = FileUtils.createTempFile("foo1", ".tmp");
+        try (MockFileControl mockFileControl1 = new MockFileControl(file1, fileSize, chunkCache);
+             MockFileControl mockFileControl2 = new MockFileControl(file1, fileSize, chunkCache);)
+        {
+
+            mockFileControl1.createFile();
+
+            RandomAccessReader r1 = mockFileControl1.openReader();
+            RandomAccessReader r2 = mockFileControl2.openReader();
+
+            // start 2 threads that will try to read from the same file, the same chunk
+            // they are racing to cache the chunk
+            CompletableFuture<?> thread1 = CompletableFuture.runAsync(r1::reBuffer);
+
+            Awaitility.await().until(() -> mockFileControl1.reading);
+            assertEquals(allocated.size(), 1);
+
+            CompletableFuture<?> thread2 = CompletableFuture.runAsync(r2::reBuffer);
+            if (injectReadError)
+            {
+                RuntimeException error = new RuntimeException("some weird runtime error");
+                mockFileControl1.waitOnRead.completeExceptionally(error);
+                assertSame(error, Throwables.getRootCause(assertThrows(CompletionException.class, thread1::join)));
+                assertSame(error, Throwables.getRootCause(assertThrows(CompletionException.class, thread2::join)));
+                // assert that we didn't leak the buffer
+                assertEquals(0, allocated.size());
+                assertEquals(0, chunkCache.size());
+            }
+            else
+            {
+                mockFileControl1.waitOnRead.complete(null);
+                thread1.join();
+                thread2.join();
+                // assert that we have only 1 buffer allocated
+                assertEquals(1, allocated.size());
+                assertEquals(1, chunkCache.size());
+            }
+
+            assertTrue(mockFileControl1.waitOnRead.isDone());
+            // assert that thread2 never performed the read
+            assertFalse(mockFileControl2.waitOnRead.isDone());
+        }
+
+        assertEquals(0, ChunkCache.instance.sizeOfFile(file1));
+    }
+
+    @Test
+    public void tstDontCacheErroredReads() throws Exception
+    {
+        BufferPool pool = mock(BufferPool.class);
+        CopyOnWriteArrayList<ByteBuffer> allocated = new CopyOnWriteArrayList<>();
+        when(pool.get(anyInt(), any(BufferType.class))).thenAnswer(invocation -> {
+            int size = invocation.getArgument(0);
+            ByteBuffer buffer = ByteBuffer.allocateDirect(size);
+            allocated.add(buffer);
+            return buffer;
+        });
+
+        doAnswer(invocation -> {
+            ByteBuffer buffer = invocation.getArgument(0);
+            allocated.remove(buffer);
+            return true;
+        }).when(pool).put(any(ByteBuffer.class));
+
+        ChunkCache chunkCache = new ChunkCache(pool, 512, ChunkCacheMetrics::create);
+        assertEquals(0, chunkCache.size());
+        int fileSize = 64;
+        File file1 = FileUtils.createTempFile("foo1", ".tmp");
+        try (MockFileControl mockFileControl1 = new MockFileControl(file1, fileSize, chunkCache);
+             MockFileControl mockFileControl2 = new MockFileControl(file1, fileSize, chunkCache);)
+        {
+
+            mockFileControl1.createFile();
+
+            RandomAccessReader r1 = mockFileControl1.openReader();
+            RandomAccessReader r2 = mockFileControl2.openReader();
+
+            // start 2 threads that will try to read from the same file, the same chunk
+            // they are racing to cache the chunk
+            CompletableFuture<?> thread1 = CompletableFuture.runAsync(r1::reBuffer);
+
+            Awaitility.await().until(() -> mockFileControl1.reading);
+            assertEquals(1, allocated.size());
+
+            // in this case thread1 errors before thread2 starts to read
+            RuntimeException error = new RuntimeException("some weird runtime error");
+            mockFileControl1.waitOnRead.completeExceptionally(error);
+            assertSame(error, assertThrows(CompletionException.class, thread1::join).getCause());
+
+            // assert that we didn't leak the buffer
+            assertEquals(0, allocated.size());
+            assertEquals(0, chunkCache.size());
+
+            // assert that the cache didn't cache the CompletableFuture that completed exceptionally the first time
+            CompletableFuture<?> thread2 = CompletableFuture.runAsync(r2::reBuffer);
+            mockFileControl2.waitOnRead.complete(null);
+            // threads2 completes without error
+            thread2.join();
+            // assert that we have only 1 buffer allocated
+            assertEquals(1, allocated.size());
+            assertEquals(1, chunkCache.size());
+
+            assertTrue(mockFileControl1.waitOnRead.isDone());
+            // assert that thread2 performed the read
+            assertTrue(mockFileControl2.waitOnRead.isDone());
+        }
+
+        assertEquals(0, ChunkCache.instance.sizeOfFile(file1));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/concurrent/CustomExecutorTest.java b/test/unit/org/apache/cassandra/concurrent/CustomExecutorTest.java
new file mode 100644
index 000000000000..83b20d12e8c7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/concurrent/CustomExecutorTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+//import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.WithResources;
+import org.apache.cassandra.utils.concurrent.Future;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class CustomExecutorTest
+{
+    @BeforeClass
+    public static void beforeClass()
+    {
+        CassandraRelevantProperties.CUSTOM_STAGE_EXECUTOR_FACTORY_PROPERTY.setString(CustomExecutorFactory.class.getName());
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    private static final HashMap<String, LocalAwareExecutorPlus> executors = new HashMap<>();
+
+    @Test
+    public void testCustomExecutor()
+    {
+        Arrays.stream(Stage.values()).forEach(stage -> stage.execute(() -> {}));
+
+        assertTrue(executors.get(Stage.MUTATION.jmxName) instanceof CustomExecutor);
+        assertEquals(Stage.MUTATION.jmxName, ((CustomExecutor) executors.get(Stage.MUTATION.jmxName)).name);
+
+        assertTrue(executors.get(Stage.READ.jmxName) instanceof CustomExecutor);
+        assertEquals(Stage.READ.jmxName, ((CustomExecutor) executors.get(Stage.READ.jmxName)).name);
+
+        assertTrue(executors.get(Stage.COUNTER_MUTATION.jmxName) instanceof CustomExecutor);
+        assertEquals(Stage.COUNTER_MUTATION.jmxName, ((CustomExecutor) executors.get(Stage.COUNTER_MUTATION.jmxName)).name);
+
+        assertTrue(executors.get(Stage.VIEW_MUTATION.jmxName) instanceof CustomExecutor);
+        assertEquals(Stage.VIEW_MUTATION.jmxName, ((CustomExecutor) executors.get(Stage.VIEW_MUTATION.jmxName)).name);
+
+        assertTrue(executors.get(Stage.REQUEST_RESPONSE.jmxName) instanceof CustomExecutor);
+        assertEquals(Stage.REQUEST_RESPONSE.jmxName, ((CustomExecutor) executors.get(Stage.REQUEST_RESPONSE.jmxName)).name);
+
+        assertTrue(executors.get(Stage.NATIVE_TRANSPORT_REQUESTS.jmxName) instanceof CustomExecutor);
+        assertEquals(Stage.NATIVE_TRANSPORT_REQUESTS.jmxName, ((CustomExecutor) executors.get(Stage.NATIVE_TRANSPORT_REQUESTS.jmxName)).name);
+    }
+
+    public static class CustomExecutorFactory implements StageExecutorFactory
+    {
+        @Override
+        public LocalAwareExecutorPlus init(String jmxName, String jmxType, int numThreads, LocalAwareExecutorPlus.MaximumPoolSizeListener onSetMaximumPoolSize)
+        {
+            return executors.computeIfAbsent(jmxName, k -> new CustomExecutor(jmxName));
+        }
+    }
+
+    public static class CustomExecutor implements LocalAwareExecutorPlus
+    {
+        private final String name;
+
+        public CustomExecutor(String name)
+        {
+            this.name = name;
+        }
+
+        @Override
+        public void execute(WithResources withResources, Runnable task)
+        {
+
+        }
+
+        @Override
+        public <T> Future<T> submit(WithResources withResources, Callable<T> task)
+        {
+            return null;
+        }
+
+        @Override
+        public Future<?> submit(WithResources withResources, Runnable task)
+        {
+            return null;
+        }
+
+        @Override
+        public <T> Future<T> submit(WithResources withResources, Runnable task, T result)
+        {
+            return null;
+        }
+
+        @Override
+        public boolean inExecutor()
+        {
+            return false;
+        }
+
+        @Override
+        public void maybeExecuteImmediately(Runnable command)
+        {
+
+        }
+
+        @Override
+        public int getActiveTaskCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public long getCompletedTaskCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public int getPendingTaskCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public void shutdown()
+        {
+
+        }
+
+        @Override
+        public List<Runnable> shutdownNow()
+        {
+            return List.of();
+        }
+
+        @Override
+        public boolean isShutdown()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean isTerminated()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+        {
+            return false;
+        }
+
+        @Override
+        public <T> Future<T> submit(Callable<T> task)
+        {
+            return null;
+        }
+
+        @Override
+        public <T> Future<T> submit(Runnable task, T result)
+        {
+            return null;
+        }
+
+        @Override
+        public Future<?> submit(Runnable task)
+        {
+            return null;
+        }
+
+        @Override
+        public <T> List<java.util.concurrent.Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) throws InterruptedException
+        {
+            return List.of();
+        }
+
+        @Override
+        public <T> List<java.util.concurrent.Future<T>> invokeAll(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) throws InterruptedException
+        {
+            return List.of();
+        }
+
+        @Override
+        public <T> T invokeAny(Collection<? extends Callable<T>> tasks) throws InterruptedException, ExecutionException
+        {
+            return null;
+        }
+
+        @Override
+        public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
+        {
+            return null;
+        }
+
+        @Override
+        public void execute(Runnable command)
+        {
+
+        }
+
+        @Override
+        public int getCorePoolSize()
+        {
+            return 0;
+        }
+
+        @Override
+        public void setCorePoolSize(int newCorePoolSize)
+        {
+
+        }
+
+        @Override
+        public int getMaximumPoolSize()
+        {
+            return 0;
+        }
+
+        @Override
+        public void setMaximumPoolSize(int newMaximumPoolSize)
+        {
+
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java b/test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java
index 061957d23d58..e1ae447a4d87 100644
--- a/test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java
+++ b/test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java
@@ -34,6 +34,7 @@
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.tracing.TraceState;
 import org.apache.cassandra.tracing.TraceStateImpl;
@@ -146,11 +147,12 @@ public static void checkClientWarningsArePropagated(ExecutorPlus executor, Runna
     }
 
     public static void checkTracingIsPropagated(ExecutorPlus executor, Runnable schedulingTask) {
+        ClientState clientState = ClientState.forInternalCalls();
         ClientWarn.instance.captureWarnings();
         assertThat(ClientWarn.instance.getWarnings()).isNullOrEmpty();
 
         ConcurrentLinkedQueue<String> q = new ConcurrentLinkedQueue<>();
-        Tracing.instance.set(new TraceState(FBUtilities.getLocalAddressAndPort(), nextTimeUUID(), Tracing.TraceType.NONE)
+        Tracing.instance.set(new TraceState(clientState, FBUtilities.getLocalAddressAndPort(), nextTimeUUID(), Tracing.TraceType.NONE)
         {
             @Override
             protected void traceImpl(String message)
@@ -248,7 +250,7 @@ private static void withTracing(Runnable fn)
     {
         TraceState state = Tracing.instance.get();
         try {
-            Tracing.instance.set(new TraceStateImpl(InetAddressAndPort.getByAddress(InetAddresses.forString("127.0.0.1")), nextTimeUUID(), Tracing.TraceType.NONE));
+            Tracing.instance.set(new TraceStateImpl(ClientState.forInternalCalls(), InetAddressAndPort.getByAddress(InetAddresses.forString("127.0.0.1")), nextTimeUUID(), Tracing.TraceType.NONE));
             fn.run();
         }
         finally
diff --git a/test/unit/org/apache/cassandra/concurrent/LocalAwareExecutorPlusTest.java b/test/unit/org/apache/cassandra/concurrent/LocalAwareExecutorPlusTest.java
index f47046c759a5..8ccf378e113a 100644
--- a/test/unit/org/apache/cassandra/concurrent/LocalAwareExecutorPlusTest.java
+++ b/test/unit/org/apache/cassandra/concurrent/LocalAwareExecutorPlusTest.java
@@ -25,7 +25,7 @@
 
 public class LocalAwareExecutorPlusTest extends AbstractExecutorPlusTest
 {
-    final ExecutorLocals locals = new ExecutorLocals(null, null);
+    final ExecutorLocals locals = new ExecutorLocals(null, null, null, null);
 
     @Test
     public void testPooled() throws Throwable
diff --git a/test/unit/org/apache/cassandra/concurrent/StageTimeMeasurementTest.java b/test/unit/org/apache/cassandra/concurrent/StageTimeMeasurementTest.java
new file mode 100644
index 000000000000..4dbd769f5ccb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/concurrent/StageTimeMeasurementTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.awaitility.Awaitility;
+
+import static org.junit.Assert.assertEquals;
+
+public class StageTimeMeasurementTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(StageTimeMeasurementTest.class);
+
+    public static final Stage TESTED_STAGE = Stage.READ;
+    private static final int MAX_CONCURRENCY = 2;
+    private static final long TASK_DURATION_NANOS = TimeUnit.MILLISECONDS.toNanos(100);
+    static TestTaskExecutionCallback callback;
+
+    @BeforeClass
+    public static void setup()
+    {
+        CassandraRelevantProperties.CUSTOM_TASK_EXECUTION_CALLBACK_CLASS.setString(TestTaskExecutionCallback.class.getName());
+        callback = (TestTaskExecutionCallback) TaskExecutionCallback.instance;
+        DatabaseDescriptor.daemonInitialization();
+        Stage.READ.setMaximumPoolSize(MAX_CONCURRENCY);
+
+        // prime the stage, so that the first task doesn't have to wait for the stage to be initialized
+        for (int i = 0; i < MAX_CONCURRENCY; i++)
+        {
+            TESTED_STAGE.execute(new LongRunnable());
+        }
+        Awaitility.await().until(() -> callback.executionTimes.size() == MAX_CONCURRENCY);
+    }
+
+    @Before
+    public void reset()
+    {
+        callback.executionTimes.clear();
+        callback.enqueuedTimes.clear();
+    }
+
+    @Test
+    public void executionAndQueueTimeAreCountedOnExecute()
+    {
+        testExecutionAndQueueTimeAreCounted(TESTED_STAGE::execute);
+    }
+
+    @Test
+    public void executionAndQueueTimeAreCountedOnExecuteWithLocals()
+    {
+        testExecutionAndQueueTimeAreCounted(r -> TESTED_STAGE.execute(r, ExecutorLocals.current()));
+    }
+
+    @Test
+    public void executionAndQueueTimeAreCountedOnMaybeExecuteImmediately()
+    {
+        testExecutionAndQueueTimeAreCounted(TESTED_STAGE::maybeExecuteImmediately);
+    }
+
+    @Test
+    public void executionAndQueueTimeAreCountedOnSubmit()
+    {
+        testExecutionAndQueueTimeAreCounted(TESTED_STAGE::submit);
+    }
+
+    @Test
+    public void executionAndQueueTimeAreCountedOnSubmitWithResult()
+    {
+        testExecutionAndQueueTimeAreCounted(r -> TESTED_STAGE.submit(r, null));
+    }
+
+    @Test
+    public void executionAndQueueTimeAreCountedOnSubmitCallable()
+    {
+        testExecutionAndQueueTimeAreCounted(r -> TESTED_STAGE.submit(() -> { r.run(); return null; }));
+    }
+
+    public void testExecutionAndQueueTimeAreCounted(Consumer<Runnable> runnableRunner)
+    {
+        int NUM_TASKS = 10;
+
+        for (int i = 0; i < NUM_TASKS; i++)
+        {
+            ForkJoinPool.commonPool().execute(() -> runnableRunner.accept(new LongRunnable()));
+        }
+
+        Awaitility.await().until(() -> callback.executionTimes.size() == NUM_TASKS);
+
+        logger.info("Completed tasks: {}", TESTED_STAGE.getCompletedTaskCount());
+        logger.info("Execution times: {}", callback.executionTimes);
+        logger.info("Queue times: {}", callback.enqueuedTimes);
+
+        final double MAX_ACCEPTABLE_MEASUREMENT_ERROR = 0.1 * TASK_DURATION_NANOS;
+
+        for (int i = 0; i < NUM_TASKS; i++)
+        {
+            // expect each task takes roughly TASK_DURATION_MS
+            assertEquals(TASK_DURATION_NANOS, callback.executionTimes.get(i), MAX_ACCEPTABLE_MEASUREMENT_ERROR);
+        }
+        for (int i = 0; i < NUM_TASKS; i += MAX_CONCURRENCY)
+        {
+            // expect in each iteration tasks are enqueued for TASK_DURATION_NANOS more
+            for (int concurrentTask = 0; concurrentTask < MAX_CONCURRENCY; concurrentTask++)
+            {
+                assertEquals((double) i / MAX_CONCURRENCY * TASK_DURATION_NANOS, callback.enqueuedTimes.get(i + concurrentTask), MAX_ACCEPTABLE_MEASUREMENT_ERROR);
+            }
+        }
+    }
+
+    public static class TestTaskExecutionCallback implements TaskExecutionCallback
+    {
+        private final List<Long> executionTimes = new CopyOnWriteArrayList<>();
+        private final List<Long> enqueuedTimes = new CopyOnWriteArrayList<>();
+
+        @Override
+        public void onCompleted(Stage stage, long executionDurationNanos)
+        {
+            assertEquals(TESTED_STAGE, stage);
+            executionTimes.add(executionDurationNanos);
+        }
+
+        @Override
+        public void onDequeue(Stage stage, long enqueuedDurationNanos)
+        {
+            assertEquals(TESTED_STAGE, stage);
+            enqueuedTimes.add(enqueuedDurationNanos);
+        }
+    }
+
+    private static class LongRunnable implements Runnable
+    {
+        @Override
+        public void run()
+        {
+            Uninterruptibles.sleepUninterruptibly(TASK_DURATION_NANOS, TimeUnit.NANOSECONDS);
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/config/CassandraRelevantPropertiesTest.java b/test/unit/org/apache/cassandra/config/CassandraRelevantPropertiesTest.java
index 7a3e0cd9be4b..544093ce884f 100644
--- a/test/unit/org/apache/cassandra/config/CassandraRelevantPropertiesTest.java
+++ b/test/unit/org/apache/cassandra/config/CassandraRelevantPropertiesTest.java
@@ -23,7 +23,9 @@
 
 import org.apache.cassandra.distributed.shared.WithProperties;
 import org.apache.cassandra.exceptions.ConfigurationException;
-import org.assertj.core.api.Assertions;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_RELEVANT_PROPERTIES;
 import static org.junit.Assert.assertEquals;
@@ -43,7 +45,7 @@ public void setup()
     public void testSystemPropertyisSet() {
         try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "test"))
         {
-            Assertions.assertThat(System.getProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey())).isEqualTo("test"); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
+            assertThat(System.getProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey())).isEqualTo("test"); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
         }
     }
 
@@ -52,38 +54,49 @@ public void testString()
     {
         try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "some-string"))
         {
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getString()).isEqualTo("some-string");
+            assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getString()).isEqualTo("some-string");
         }
     }
 
+    @Test
+    public void testString_null()
+    {
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getString()).isNull();
+    }
+
+    @Test
+    public void testString_override_default()
+    {
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getString("other-string")).isEqualTo("other-string");
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.setString("this-string");
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getString("other-string")).isEqualTo("this-string");
+    }
+
     @Test
     public void testBoolean()
     {
-        try
-        {
-            System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "true");
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isEqualTo(true);
-            System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "false");
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isEqualTo(false);
-            System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "junk");
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isEqualTo(false);
-            System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "");
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isEqualTo(false);
-        }
-        finally
-        {
-            System.clearProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey());
-        }
+        System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "true");
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isTrue();
+        System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "false");
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isFalse();
+        System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "junk");
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isFalse();
+        System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "");
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isFalse();
     }
 
     @Test
     public void testBoolean_null()
     {
-        try (WithProperties properties = new WithProperties())
-        {
-            TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean();
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isFalse();
-        }
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isFalse();
+    }
+
+    @Test
+    public void testBoolean_override_default()
+    {
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean(true)).isTrue();
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.setBoolean(false);
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean(true)).isFalse();
     }
 
     @Test
@@ -91,7 +104,7 @@ public void testDecimal()
     {
         try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "123456789"))
         {
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(123456789);
+            assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(123456789);
         }
     }
 
@@ -100,7 +113,7 @@ public void testHexadecimal()
     {
         try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "0x1234567a"))
         {
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(305419898);
+            assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(305419898);
         }
     }
 
@@ -109,7 +122,7 @@ public void testOctal()
     {
         try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "01234567"))
         {
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(342391);
+            assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(342391);
         }
     }
 
@@ -118,7 +131,7 @@ public void testInteger_empty()
     {
         try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, ""))
         {
-            Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(342391);
+            assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(342391);
         }
     }
 
@@ -131,6 +144,74 @@ public void testInteger_null()
         }
     }
 
+    @Test
+    public void testInteger_override_default()
+    {
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt(2345)).isEqualTo(2345);
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.setInt(1234);
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt(2345)).isEqualTo(1234);
+    }
+
+    @Test
+    public void testLong()
+    {
+        System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "1234");
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getLong()).isEqualTo(1234);
+    }
+
+    @Test(expected = ConfigurationException.class)
+    public void testLong_empty()
+    {
+        System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "");
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.getLong();
+        fail("Expected ConfigurationException");
+    }
+
+    @Test(expected = ConfigurationException.class)
+    public void testLong_null()
+    {
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.getLong();
+        fail("Expected NullPointerException");
+    }
+
+    @Test
+    public void testLong_override_default()
+    {
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getLong(2345)).isEqualTo(2345);
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.setLong(1234);
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getLong(2345)).isEqualTo(1234);
+    }
+
+    @Test
+    public void testDouble()
+    {
+        System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "1.567");
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getDouble()).isEqualTo(1.567);
+    }
+
+    @Test(expected = ConfigurationException.class)
+    public void testDouble_empty()
+    {
+        System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "");
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.getDouble();
+        fail("Expected ConfigurationException");
+    }
+
+    @Test(expected = ConfigurationException.class)
+    public void testDouble_null()
+    {
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.getDouble();
+        fail("Expected NullPointerException");
+    }
+
+    @Test
+    public void testDouble_override_default()
+    {
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getDouble(2.345)).isEqualTo(2.345);
+        TEST_CASSANDRA_RELEVANT_PROPERTIES.setDouble(1.234);
+        assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getDouble(2.345)).isEqualTo(1.234);
+    }
+
     @Test
     public void testClearProperty()
     {
diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
index e763e738d98e..a2fc733b4ddf 100644
--- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
+++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
@@ -27,9 +27,7 @@
 import java.lang.management.ThreadMXBean;
 import java.lang.reflect.Method;
 import java.net.URL;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -39,10 +37,7 @@
 
 import org.junit.Test;
 
-import org.apache.cassandra.utils.Pair;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import org.assertj.core.api.SoftAssertions;
 
 /**
  * Verifies that {@link DatabaseDescriptor#clientInitialization()} and a couple of <i>apply</i> methods
@@ -57,30 +52,28 @@ public class DatabaseDescriptorRefTest
     static final String[] validClasses = {
     "org.apache.cassandra.ConsoleAppender",
     "org.apache.cassandra.ConsoleAppender$1",
-    "org.apache.cassandra.ConsoleAppenderBeanInfo",
-    "org.apache.cassandra.ConsoleAppenderCustomizer",
+    "org.apache.cassandra.ForbiddenLogEntriesFilter",
     "org.apache.cassandra.LogbackStatusListener",
     "org.apache.cassandra.LogbackStatusListener$ToLoggerOutputStream",
     "org.apache.cassandra.LogbackStatusListener$WrappedPrintStream",
     "org.apache.cassandra.TeeingAppender",
     "org.apache.cassandra.audit.AuditLogOptions",
     "org.apache.cassandra.audit.BinAuditLogger",
-    "org.apache.cassandra.audit.BinLogAuditLogger",
     "org.apache.cassandra.audit.IAuditLogger",
     "org.apache.cassandra.auth.AllowAllInternodeAuthenticator",
     "org.apache.cassandra.auth.AuthCache$BulkLoader",
-    "org.apache.cassandra.auth.Cacheable",
     "org.apache.cassandra.auth.IAuthenticator",
     "org.apache.cassandra.auth.IAuthorizer",
-    "org.apache.cassandra.auth.IInternodeAuthenticator",
     "org.apache.cassandra.auth.ICIDRAuthorizer",
     "org.apache.cassandra.auth.ICIDRAuthorizer$CIDRAuthorizerMode",
+    "org.apache.cassandra.auth.IInternodeAuthenticator",
     "org.apache.cassandra.auth.INetworkAuthorizer",
     "org.apache.cassandra.auth.IRoleManager",
     "org.apache.cassandra.config.CassandraRelevantProperties",
     "org.apache.cassandra.config.CassandraRelevantProperties$PropertyConverter",
     "org.apache.cassandra.config.Config",
     "org.apache.cassandra.config.Config$1",
+    "org.apache.cassandra.config.Config$BatchlogEndpointStrategy",
     "org.apache.cassandra.config.Config$CommitFailurePolicy",
     "org.apache.cassandra.config.Config$CQLStartTime",
     "org.apache.cassandra.config.Config$CommitLogSync",
@@ -97,8 +90,6 @@ public class DatabaseDescriptorRefTest
     "org.apache.cassandra.config.Config$RepairCommandPoolFullStrategy",
     "org.apache.cassandra.config.Config$SSTableConfig",
     "org.apache.cassandra.config.Config$UserFunctionTimeoutPolicy",
-    "org.apache.cassandra.config.ConfigBeanInfo",
-    "org.apache.cassandra.config.ConfigCustomizer",
     "org.apache.cassandra.config.ConfigurationLoader",
     "org.apache.cassandra.config.DataRateSpec",
     "org.apache.cassandra.config.DataRateSpec$DataRateUnit",
@@ -118,87 +109,43 @@ public class DatabaseDescriptorRefTest
     "org.apache.cassandra.config.DataStorageSpec$LongBytesBound",
     "org.apache.cassandra.config.DataStorageSpec$LongMebibytesBound",
     "org.apache.cassandra.config.DatabaseDescriptor",
-    "org.apache.cassandra.config.DatabaseDescriptor$ByteUnit",
     "org.apache.cassandra.config.DurationSpec",
     "org.apache.cassandra.config.DurationSpec$IntMillisecondsBound",
     "org.apache.cassandra.config.DurationSpec$IntMinutesBound",
     "org.apache.cassandra.config.DurationSpec$IntSecondsBound",
     "org.apache.cassandra.config.DurationSpec$LongMillisecondsBound",
-    "org.apache.cassandra.config.DurationSpec$LongMicrosecondsBound",
     "org.apache.cassandra.config.DurationSpec$LongNanosecondsBound",
     "org.apache.cassandra.config.DurationSpec$LongSecondsBound",
     "org.apache.cassandra.config.EncryptionOptions",
-    "org.apache.cassandra.config.EncryptionOptions$ClientEncryptionOptions",
     "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions",
     "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions$InternodeEncryption",
-    "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions$OutgoingEncryptedPortSource",
-    "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsBeanInfo",
-    "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsCustomizer",
-    "org.apache.cassandra.config.EncryptionOptionsBeanInfo",
-    "org.apache.cassandra.config.EncryptionOptionsCustomizer",
     "org.apache.cassandra.config.GuardrailsOptions",
-    "org.apache.cassandra.config.GuardrailsOptions$Config",
-    "org.apache.cassandra.config.GuardrailsOptions$ConsistencyLevels",
-    "org.apache.cassandra.config.GuardrailsOptions$TableProperties",
     "org.apache.cassandra.config.ParameterizedClass",
     "org.apache.cassandra.config.RepairConfig",
     "org.apache.cassandra.config.RepairRetrySpec",
+    "org.apache.cassandra.config.ReplicaFilteringProtectionOptions",
     "org.apache.cassandra.config.RetrySpec",
     "org.apache.cassandra.config.RetrySpec$MaxAttempt",
-    "org.apache.cassandra.config.RetrySpec$Type",
-    "org.apache.cassandra.config.ReplicaFilteringProtectionOptions",
     "org.apache.cassandra.config.StartupChecksOptions",
+    "org.apache.cassandra.config.StorageAttachedIndexOptions",
+    "org.apache.cassandra.config.StorageFlagsConfig",
     "org.apache.cassandra.config.SubnetGroups",
-    "org.apache.cassandra.config.TrackWarnings",
-    "org.apache.cassandra.config.TransparentDataEncryptionOptions",
-    "org.apache.cassandra.config.YamlConfigurationLoader",
-    "org.apache.cassandra.config.YamlConfigurationLoader$CustomConstructor",
-    "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker",
-    "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker$1",
-    "org.apache.cassandra.config.YamlConfigurationLoader$CustomConstructor",
     "org.apache.cassandra.config.TransparentDataEncryptionOptions",
-    "org.apache.cassandra.config.StartupChecksOptions",
-    "org.apache.cassandra.config.SubnetGroups",
-    "org.apache.cassandra.config.TrackWarnings",
-    "org.apache.cassandra.config.StorageAttachedIndexOptions",
+    "org.apache.cassandra.cql3.PageSize",
     "org.apache.cassandra.db.ConsistencyLevel",
     "org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager",
     "org.apache.cassandra.db.commitlog.CommitLog",
     "org.apache.cassandra.db.commitlog.CommitLogMBean",
     "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC",
-    "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerFactory",
     "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard",
-    "org.apache.cassandra.db.commitlog.DefaultCommitLogSegmentMgrFactory",
+    "org.apache.cassandra.db.compaction.unified.Reservations$Type",
     "org.apache.cassandra.db.guardrails.GuardrailsConfig",
-    "org.apache.cassandra.db.guardrails.GuardrailsConfig$ConsistencyLevels",
-    "org.apache.cassandra.db.guardrails.GuardrailsConfig$TableProperties",
-    "org.apache.cassandra.db.guardrails.GuardrailsConfigMBean",
-    "org.apache.cassandra.db.guardrails.Values$Config",
-    "org.apache.cassandra.db.rows.UnfilteredSource",
     "org.apache.cassandra.dht.IPartitioner",
-    "org.apache.cassandra.distributed.api.IInstance",
-    "org.apache.cassandra.distributed.api.IInvokableInstance",
-    "org.apache.cassandra.distributed.api.IIsolatedExecutor",
-    "org.apache.cassandra.distributed.impl.InstanceConfig",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$CallableNoExcept",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$InstanceFunction",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$SerializableBiConsumer",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$SerializableBiFunction",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$SerializableCallable",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$SerializableConsumer",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$SerializableFunction",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$SerializableRunnable",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$SerializableTriFunction",
-    "org.apache.cassandra.distributed.impl.InvokableInstance$TriFunction",
-    "org.apache.cassandra.distributed.impl.Message",
-    "org.apache.cassandra.distributed.impl.NetworkTopology",
-    "org.apache.cassandra.distributed.shared.InstanceClassLoader",
     "org.apache.cassandra.exceptions.CassandraException",
     "org.apache.cassandra.exceptions.ConfigurationException",
     "org.apache.cassandra.exceptions.InvalidRequestException",
     "org.apache.cassandra.exceptions.RequestValidationException",
     "org.apache.cassandra.exceptions.TransportException",
-    "org.apache.cassandra.fql.FullQueryLogger",
     "org.apache.cassandra.fql.FullQueryLoggerOptions",
     "org.apache.cassandra.gms.IFailureDetector",
     "org.apache.cassandra.io.FSError",
@@ -210,7 +157,6 @@ public class DatabaseDescriptorRefTest
     "org.apache.cassandra.io.sstable.Component$Type",
     "org.apache.cassandra.io.sstable.IScrubber",
     "org.apache.cassandra.io.sstable.MetricsProviders",
-    "org.apache.cassandra.io.sstable.SSTable",
     "org.apache.cassandra.io.sstable.SSTable$Builder",
     "org.apache.cassandra.io.sstable.format.AbstractSSTableFormat",
     "org.apache.cassandra.io.sstable.format.SSTableFormat",
@@ -220,70 +166,53 @@ public class DatabaseDescriptorRefTest
     "org.apache.cassandra.io.sstable.format.SSTableFormat$KeyCacheValueSerializer",
     "org.apache.cassandra.io.sstable.format.SSTableFormat$SSTableReaderFactory",
     "org.apache.cassandra.io.sstable.format.SSTableFormat$SSTableWriterFactory",
-    "org.apache.cassandra.io.sstable.format.SSTableFormat$Type",
-    "org.apache.cassandra.io.sstable.format.SSTableReader",
     "org.apache.cassandra.io.sstable.format.SSTableReader$Builder",
     "org.apache.cassandra.io.sstable.format.SSTableReaderLoadingBuilder",
-    "org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter",
     "org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter$Builder",
-    "org.apache.cassandra.io.sstable.format.SortedTableReaderLoadingBuilder",
-    "org.apache.cassandra.io.sstable.format.SSTableWriter",
     "org.apache.cassandra.io.sstable.format.SSTableWriter$Builder",
-    "org.apache.cassandra.io.sstable.format.SortedTableWriter",
+    "org.apache.cassandra.io.sstable.format.SortedTableReaderLoadingBuilder",
     "org.apache.cassandra.io.sstable.format.SortedTableWriter$Builder",
     "org.apache.cassandra.io.sstable.format.Version",
-    "org.apache.cassandra.io.sstable.format.big.BigFormat",
-    "org.apache.cassandra.io.sstable.format.big.BigFormat$BigFormatFactory",
-    "org.apache.cassandra.io.sstable.format.big.BigFormat$BigTableReaderFactory",
-    "org.apache.cassandra.io.sstable.format.big.BigFormat$BigTableWriterFactory",
-    "org.apache.cassandra.io.sstable.format.big.BigFormat$BigVersion",
-    "org.apache.cassandra.io.sstable.format.big.BigFormat$Components",
-    "org.apache.cassandra.io.sstable.format.big.BigFormat$Components$Types",
-    "org.apache.cassandra.io.sstable.format.big.BigSSTableReaderLoadingBuilder",
-    "org.apache.cassandra.io.sstable.format.big.BigTableReader",
-    "org.apache.cassandra.io.sstable.format.big.BigTableReader$Builder",
-    "org.apache.cassandra.io.sstable.format.big.BigTableWriter",
-    "org.apache.cassandra.io.sstable.format.big.BigTableWriter$Builder",
-    "org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport",
-    "org.apache.cassandra.io.sstable.keycache.KeyCacheSupport",
-    "org.apache.cassandra.io.sstable.metadata.MetadataType",
+    "org.apache.cassandra.io.sstable.format.bti.BtiFormat",
+    "org.apache.cassandra.io.sstable.format.bti.BtiFormat$BtiFormatFactory",
+    "org.apache.cassandra.io.sstable.format.bti.BtiFormat$BtiTableReaderFactory",
+    "org.apache.cassandra.io.sstable.format.bti.BtiFormat$BtiTableWriterFactory",
+    "org.apache.cassandra.io.sstable.format.bti.BtiFormat$BtiVersion",
+    "org.apache.cassandra.io.sstable.format.bti.BtiFormat$Components",
+    "org.apache.cassandra.io.sstable.format.bti.BtiFormat$Components$Types",
+    "org.apache.cassandra.io.sstable.format.bti.BtiTableReaderLoadingBuilder",
+    "org.apache.cassandra.io.sstable.format.bti.BtiTableReader$Builder",
+    "org.apache.cassandra.io.sstable.format.bti.BtiTableWriter$Builder",
     "org.apache.cassandra.io.util.BufferedDataOutputStreamPlus",
-    "org.apache.cassandra.io.util.DataInputPlus",
-    "org.apache.cassandra.io.util.DataInputPlus$DataInputStreamPlus",
     "org.apache.cassandra.io.util.DataOutputBuffer",
     "org.apache.cassandra.io.util.DataOutputBufferFixed",
     "org.apache.cassandra.io.util.DataOutputPlus",
     "org.apache.cassandra.io.util.DataOutputStreamPlus",
     "org.apache.cassandra.io.util.DiskOptimizationStrategy",
     "org.apache.cassandra.io.util.File",
-    "org.apache.cassandra.io.util.FileInputStreamPlus",
-    "org.apache.cassandra.io.util.FileOutputStreamPlus",
     "org.apache.cassandra.io.util.PathUtils$IOToLongFunction",
-    "org.apache.cassandra.io.util.RebufferingInputStream",
     "org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy",
     "org.apache.cassandra.locator.IEndpointSnitch",
     "org.apache.cassandra.locator.InetAddressAndPort",
     "org.apache.cassandra.locator.Replica",
-    "org.apache.cassandra.locator.ReplicaCollection",
     "org.apache.cassandra.locator.SeedProvider",
-    "org.apache.cassandra.locator.SimpleSeedProvider",
+    "org.apache.cassandra.metrics.TableMetrics$MetricsAggregation",
+    "org.apache.cassandra.security.AbstractCryptoProvider",
     "org.apache.cassandra.security.EncryptionContext",
     "org.apache.cassandra.security.ISslContextFactory",
     "org.apache.cassandra.security.SSLFactory",
     "org.apache.cassandra.service.CacheService$CacheType",
-    "org.apache.cassandra.security.AbstractCryptoProvider",
     "org.apache.cassandra.transport.ProtocolException",
     "org.apache.cassandra.utils.Closeable",
     "org.apache.cassandra.utils.CloseableIterator",
     "org.apache.cassandra.utils.FBUtilities",
-    "org.apache.cassandra.utils.FBUtilities$1",
     "org.apache.cassandra.utils.Pair",
+    "org.apache.cassandra.utils.StorageCompatibilityMode",
     "org.apache.cassandra.utils.binlog.BinLogOptions",
+    "org.apache.cassandra.utils.bytecomparable.ByteComparable$Version",
+    "org.apache.cassandra.utils.concurrent.Ref",
     "org.apache.cassandra.utils.concurrent.RefCounted",
-    "org.apache.cassandra.utils.concurrent.SelfRefCounted",
-    "org.apache.cassandra.utils.concurrent.Transactional",
     "org.apache.cassandra.utils.concurrent.UncheckedInterruptedException",
-    "org.apache.cassandra.utils.StorageCompatibilityMode"
     };
 
     static final Set<String> checkedClasses = new HashSet<>(Arrays.asList(validClasses));
@@ -301,7 +230,8 @@ public void testDatabaseDescriptorRef() throws Throwable
 
         ClassLoader delegate = Thread.currentThread().getContextClassLoader();
 
-        List<Pair<String, Exception>> violations = Collections.synchronizedList(new ArrayList<>());
+        SoftAssertions violations = new SoftAssertions();
+        // List<Pair<String, Exception>> violations = Collections.synchronizedList(new ArrayList<>());
 
         ClassLoader cl = new ClassLoader(null)
         {
@@ -330,46 +260,49 @@ protected Class<?> findClass(String name) throws ClassNotFoundException
                 if (cls != null)
                     return cls;
 
-                if (name.startsWith("org.apache.cassandra."))
-                {
-                    // out.println(name);
-
-                    if (!checkedClasses.contains(name))
-                        violations.add(Pair.create(name, new Exception()));
-                }
-
                 URL url = delegate.getResource(name.replace('.', '/') + ".class");
-                if (url == null)
+                try
                 {
-                    // For Java 11: system class files are not readable via getResource(), so
-                    // try it this way
-                    cls = Class.forName(name, false, delegate);
-                    classMap.put(name, cls);
-                    return cls;
-                }
+                    if (url == null)
+                    {
+                        // For Java 11: system class files are not readable via getResource(), so
+                        // try it this way
+                        cls = Class.forName(name, false, delegate);
+                        classMap.put(name, cls);
+                        return cls;
+                    }
 
-                // Java8 way + all non-system class files
-                try (InputStream in = url.openConnection().getInputStream())
-                {
-                    ByteArrayOutputStream os = new ByteArrayOutputStream();
-                    int c;
-                    while ((c = in.read()) != -1)
-                        os.write(c);
-                    byte[] data = os.toByteArray();
-                    cls = defineClass(name, data, 0, data.length);
-                    classMap.put(name, cls);
-                    return cls;
+                    // Java8 way + all non-system class files
+                    try (InputStream in = url.openConnection().getInputStream())
+                    {
+                        ByteArrayOutputStream os = new ByteArrayOutputStream();
+                        int c;
+                        while ((c = in.read()) != -1)
+                            os.write(c);
+                        byte[] data = os.toByteArray();
+                        cls = defineClass(name, data, 0, data.length);
+                        classMap.put(name, cls);
+                        return cls;
+                    }
+                    catch (IOException e)
+                    {
+                        throw new ClassNotFoundException(name, e);
+                    }
                 }
-                catch (IOException e)
+                finally
                 {
-                    throw new ClassNotFoundException(name, e);
+                    if (name.startsWith("org.apache.cassandra."))
+                    {
+                        violations.assertThat(checkedClasses.contains(name)).describedAs(name).isTrue();
+                        out.println("\"" + name + "\",");
+                    }
                 }
             }
         };
 
         Thread.currentThread().setContextClassLoader(cl);
 
-        assertEquals("thread started", threadCount, threads.getThreadCount());
+        violations.assertThat(threads.getThreadCount()).describedAs("thread started").isEqualTo(threadCount);
 
         Class<?> databaseDescriptorClass = Class.forName("org.apache.cassandra.config.DatabaseDescriptor", true, cl);
 
@@ -416,26 +349,10 @@ protected Class<?> findClass(String name) throws ClassNotFoundException
             {
                 for (ThreadInfo threadInfo : threads.getThreadInfo(threads.getAllThreadIds()))
                     out.println("Thread #" + threadInfo.getThreadId() + ": " + threadInfo.getThreadName());
-                assertEquals("thread started in " + methodName, threadCount, ManagementFactory.getThreadMXBean().getThreadCount());
+                violations.assertThat(ManagementFactory.getThreadMXBean().getThreadCount()).describedAs("thread started in " + methodName).isEqualTo(threadCount);
             }
 
-            checkViolations(err, violations);
-        }
-    }
-
-    private void checkViolations(PrintStream err, List<Pair<String, Exception>> violations)
-    {
-        if (!violations.isEmpty())
-        {
-            StringBuilder sb = new StringBuilder();
-            for (Pair<String, Exception> violation : new ArrayList<>(violations))
-                sb.append("\n\n")
-                  .append("VIOLATION: ").append(violation.left); //.append('\n')
-                  //.append(Throwables.getStackTraceAsString(violation.right));
-            String msg = sb.toString();
-            err.println(msg);
-
-            fail(msg);
+            violations.assertAll();
         }
     }
 }
diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
index dcd617adcf9c..8358e63d1101 100644
--- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
+++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
@@ -24,6 +24,7 @@
 import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.nio.file.Files;
+import java.nio.file.FileStore;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumSet;
@@ -31,9 +32,13 @@
 import java.util.function.Consumer;
 
 import com.google.common.base.Throwables;
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.Multiset;
 import org.junit.Assert;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.distributed.shared.WithProperties;
@@ -44,6 +49,9 @@
 import org.apache.cassandra.security.EncryptionContextGenerator;
 import org.assertj.core.api.Assertions;
 import org.mockito.MockedStatic;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.MBeanWrapper;
+import org.mockito.Mockito;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNLIMITED_CONCURRENT_VALIDATIONS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_LOADER;
@@ -53,13 +61,20 @@
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
 
 public class DatabaseDescriptorTest
 {
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
     @BeforeClass
     public static void setupDatabaseDescriptor()
     {
@@ -81,6 +96,8 @@ public void testConfigurationLoader() throws Exception
 
         config = DatabaseDescriptor.loadConfig();
         assertEquals("ConfigurationLoader Test", config.cluster_name);
+
+        System.clearProperty("cassandra.config.loader");
     }
 
     public static class TestLoader implements ConfigurationLoader
@@ -282,7 +299,7 @@ public void testTokensFromString()
     public void testExceptionsForInvalidConfigValues() {
         try
         {
-            DatabaseDescriptor.setColumnIndexCacheSize(-1);
+            DatabaseDescriptor.setColumnIndexCacheSizeInKiB(-1);
             fail("Should have received a IllegalArgumentException column_index_cache_size = -1");
         }
         catch (IllegalArgumentException ignored) { }
@@ -290,8 +307,8 @@ public void testExceptionsForInvalidConfigValues() {
 
         try
         {
-            DatabaseDescriptor.setColumnIndexCacheSize(2 * 1024 * 1024);
-            fail("Should have received a ConfigurationException column_index_cache_size= 2GiB");
+            DatabaseDescriptor.setColumnIndexCacheSizeInKiB(2 * 1024 * 1024);
+            fail("Should have received a ConfigurationException column_index_cache_size = 2GiB");
         }
         catch (ConfigurationException ignored) { }
         Assert.assertEquals(2048, DatabaseDescriptor.getColumnIndexCacheSize());
@@ -364,6 +381,7 @@ public void testLowestAcceptableTimeouts() throws ConfigurationException
         testConfig.cas_contention_timeout = greaterThanLowestTimeout;
         testConfig.counter_write_request_timeout = greaterThanLowestTimeout;
         testConfig.request_timeout = greaterThanLowestTimeout;
+        testConfig.native_transport_timeout = greaterThanLowestTimeout;
 
         assertEquals(testConfig.read_request_timeout, greaterThanLowestTimeout);
         assertEquals(testConfig.range_request_timeout, greaterThanLowestTimeout);
@@ -372,6 +390,7 @@ public void testLowestAcceptableTimeouts() throws ConfigurationException
         assertEquals(testConfig.cas_contention_timeout, greaterThanLowestTimeout);
         assertEquals(testConfig.counter_write_request_timeout, greaterThanLowestTimeout);
         assertEquals(testConfig.request_timeout, greaterThanLowestTimeout);
+        assertEquals(testConfig.native_transport_timeout, greaterThanLowestTimeout);
 
         //set less than Lowest acceptable value
         DurationSpec.LongMillisecondsBound lowerThanLowestTimeout = new DurationSpec.LongMillisecondsBound(DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT.toMilliseconds() - 1);
@@ -383,6 +402,7 @@ public void testLowestAcceptableTimeouts() throws ConfigurationException
         testConfig.cas_contention_timeout = lowerThanLowestTimeout;
         testConfig.counter_write_request_timeout = lowerThanLowestTimeout;
         testConfig.request_timeout = lowerThanLowestTimeout;
+        testConfig.native_transport_timeout = lowerThanLowestTimeout;
 
         DatabaseDescriptor.checkForLowestAcceptedTimeouts(testConfig);
 
@@ -393,6 +413,7 @@ public void testLowestAcceptableTimeouts() throws ConfigurationException
         assertEquals(testConfig.cas_contention_timeout, DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
         assertEquals(testConfig.counter_write_request_timeout, DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
         assertEquals(testConfig.request_timeout, DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
+        assertEquals(testConfig.native_transport_timeout, DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT);
     }
 
     @Test
@@ -822,12 +843,12 @@ public void testCommitLogDiskAccessMode() throws IOException
         ParameterizedClass savedCompression = DatabaseDescriptor.getCommitLogCompression();
         EncryptionContext savedEncryptionContexg = DatabaseDescriptor.getEncryptionContext();
         Config.DiskAccessMode savedCommitLogDOS = DatabaseDescriptor.getCommitLogWriteDiskAccessMode();
-        String savedCommitLogLocation = DatabaseDescriptor.getCommitLogLocation();
+        File savedCommitLogLocation = DatabaseDescriptor.getCommitLogLocation();
 
         try
         {
             // block size available
-            DatabaseDescriptor.setCommitLogLocation(Files.createTempDirectory("testCommitLogDiskAccessMode").toString());
+            DatabaseDescriptor.setCommitLogLocation(new File(Files.createTempDirectory("testCommitLogDiskAccessMode")));
 
             // no encryption or compression
             DatabaseDescriptor.setCommitLogCompression(null);
@@ -922,4 +943,69 @@ else if (mode == Config.DiskAccessMode.auto)
                 assertThat(DatabaseDescriptor.getCommitLogWriteDiskAccessMode()).isEqualTo(mode);
         }
     }
+    
+    @Test
+    public void testDataFileDirectoriesMinTotalSpaceInGB() throws IOException
+    {
+        DatabaseDescriptor.setDataDirectories(new File[] {});
+        assertEquals(0L, DatabaseDescriptor.getDataFileDirectoriesMinTotalSpaceInGB());
+
+        DatabaseDescriptor.setDataDirectories(new File[] { new File(temporaryFolder.newFolder("data"))});
+        assertTrue(DatabaseDescriptor.getDataFileDirectoriesMinTotalSpaceInGB() > 0);
+
+        Multiset<FileStore> fileStoreMultiset = HashMultiset.create();
+
+        // single disk (i.e. mockFileStore1)
+        FileStore mockFileStore1 = Mockito.mock(FileStore.class);
+        when(mockFileStore1.getTotalSpace()).thenReturn(1L << 43); // 8 TB
+        fileStoreMultiset.add(mockFileStore1);
+        assertEquals(8192L, DatabaseDescriptor.getDataFileDirectoriesMinTotalSpaceInGB(fileStoreMultiset));
+
+        // two different disks (i.e. mockFileStore1, mockFileStore2)
+        FileStore mockFileStore2 = Mockito.mock(FileStore.class);
+        when(mockFileStore2.getTotalSpace()).thenReturn(1L << 41); // 2 TB
+        fileStoreMultiset.add(mockFileStore2);
+        assertEquals(4096L, DatabaseDescriptor.getDataFileDirectoriesMinTotalSpaceInGB(fileStoreMultiset));
+
+        // two different disks with three directories. Two directories are on disk 1 (i.e. mockFileStore1)
+        fileStoreMultiset.add(mockFileStore1);
+        assertEquals(6144L, DatabaseDescriptor.getDataFileDirectoriesMinTotalSpaceInGB(fileStoreMultiset));
+
+        fileStoreMultiset.clear();
+
+        FileStore mockLargeFileStore = Mockito.mock(FileStore.class);
+        when(mockLargeFileStore.getTotalSpace()).thenReturn(-1L);
+        fileStoreMultiset.add(mockLargeFileStore);
+        assertEquals(Long.MAX_VALUE >> 30, DatabaseDescriptor.getDataFileDirectoriesMinTotalSpaceInGB(fileStoreMultiset));
+
+        FileStore mockSmallFileStore = Mockito.mock(FileStore.class);
+        when(mockSmallFileStore.getTotalSpace()).thenReturn(1L << 29); // 512 MB
+        fileStoreMultiset.add(mockSmallFileStore);
+        assertEquals(0L, DatabaseDescriptor.getDataFileDirectoriesMinTotalSpaceInGB(fileStoreMultiset));
+    }
+
+    @Test
+    public void testResetUnsafe()
+    {
+        assertTrue(DatabaseDescriptor.isDaemonInitialized());
+        assertFalse(DatabaseDescriptor.isClientOrToolInitialized());
+        assertNotNull(DatabaseDescriptor.getPartitioner());
+        assertNotNull(DatabaseDescriptor.getEndpointSnitch());
+        assertTrue(MBeanWrapper.instance.isRegistered("org.apache.cassandra.db:type=EndpointSnitchInfo"));
+
+        try
+        {
+            DatabaseDescriptor.resetUnsafe();
+
+            assertFalse(DatabaseDescriptor.isDaemonInitialized());
+            assertFalse(DatabaseDescriptor.isClientOrToolInitialized());
+            assertNull(DatabaseDescriptor.getPartitioner());
+            assertNull(DatabaseDescriptor.getEndpointSnitch());
+            assertFalse(MBeanWrapper.instance.isRegistered("org.apache.cassandra.db:type=EndpointSnitchInfo"));
+        }
+        finally
+        {
+            DatabaseDescriptor.daemonInitialization();
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/config/ParameterizedClassExample.java b/test/unit/org/apache/cassandra/config/ParameterizedClassExample.java
new file mode 100644
index 000000000000..0bd3c7a6d4be
--- /dev/null
+++ b/test/unit/org/apache/cassandra/config/ParameterizedClassExample.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import java.util.Map;
+
+import org.junit.Assert;
+
+public class ParameterizedClassExample
+{
+    public ParameterizedClassExample()
+    {
+        Assert.fail("This constructor should not be called");
+    }
+
+    public ParameterizedClassExample(Map<String, String> parameters)
+    {
+        if (parameters == null)
+            throw new IllegalArgumentException("Parameters must not be null");
+
+        boolean simulateFailure = Boolean.parseBoolean(parameters.getOrDefault("fail", "false"));
+        if (simulateFailure)
+        {
+            throw new IllegalArgumentException("Simulated failure");
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/config/ParameterizedClassTest.java b/test/unit/org/apache/cassandra/config/ParameterizedClassTest.java
new file mode 100644
index 000000000000..732812aefd15
--- /dev/null
+++ b/test/unit/org/apache/cassandra/config/ParameterizedClassTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.cassandra.auth.AllowAllAuthorizer;
+import org.apache.cassandra.auth.IAuthorizer;
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class ParameterizedClassTest
+{
+    @Test
+    public void testParameterizedClassEmptyConstructorHasNullParameters()
+    {
+        ParameterizedClass parameterizedClass = new ParameterizedClass();
+        assertNull(parameterizedClass.parameters);
+    }
+
+    @Test
+    public void testParameterizedClassConstructorWithClassNameHasNonNullParameters()
+    {
+        ParameterizedClass parameterizedClass = new ParameterizedClass("TestClass");
+        assertNotNull(parameterizedClass.parameters);
+    }
+
+    @Test
+    public void testParameterizedClassConstructorWithClassNameAndParametersHasNullParamters()
+    {
+        ParameterizedClass parameterizedClass = new ParameterizedClass("TestClass", null);
+        assertNull(parameterizedClass.parameters);
+    }
+
+    @Test
+    public void testNewInstanceWithNonExistentClassFailsWithConfigurationException()
+    {
+        assertThatThrownBy(() -> ParameterizedClass.newInstance(new ParameterizedClass("NonExistentClass"),
+                                                                List.of("org.apache.cassandra.config")))
+        .hasMessage("Unable to find class NonExistentClass in packages [\"org.apache.cassandra.config\"]")
+        .isInstanceOf(ConfigurationException.class);
+    }
+
+    @Test
+    public void testNewInstanceWithSingleEmptyConstructorUsesEmptyConstructor()
+    {
+        ParameterizedClass parameterizedClass = new ParameterizedClass(AllowAllAuthorizer.class.getName());
+        IAuthorizer instance = ParameterizedClass.newInstance(parameterizedClass, null);
+        assertNotNull(instance);
+    }
+
+    @Test
+    public void testNewInstanceWithValidConstructorsFavorsMapConstructor()
+    {
+        ParameterizedClass parameterizedClass = new ParameterizedClass(ParameterizedClassExample.class.getName());
+        ParameterizedClassExample instance = ParameterizedClass.newInstance(parameterizedClass, null);
+        assertNotNull(instance);
+    }
+
+    @Test
+    public void testNewInstanceWithValidConstructorsUsingNullParamtersFavorsMapConstructor()
+    {
+        ParameterizedClass parameterizedClass = new ParameterizedClass(ParameterizedClassExample.class.getName());
+        parameterizedClass.parameters = null;
+
+        ParameterizedClassExample instance = ParameterizedClass.newInstance(parameterizedClass, null);
+        assertNotNull(instance);
+    }
+
+    @Test
+    public void testNewInstanceWithConstructorExceptionPreservesOriginalFailure()
+    {
+        assertThatThrownBy(() -> ParameterizedClass.newInstance(new ParameterizedClass(ParameterizedClassExample.class.getName(),
+                                                                                       Map.of("fail", "true")), null))
+        .hasMessageStartingWith("Failed to instantiate class")
+        .hasMessageContaining("Simulated failure")
+        .isInstanceOf(ConfigurationException.class);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/config/ParseAndConvertUnitsTest.java b/test/unit/org/apache/cassandra/config/ParseAndConvertUnitsTest.java
index 7d28f0945364..75220ae0ac18 100644
--- a/test/unit/org/apache/cassandra/config/ParseAndConvertUnitsTest.java
+++ b/test/unit/org/apache/cassandra/config/ParseAndConvertUnitsTest.java
@@ -41,12 +41,12 @@ public void testConfigurationLoaderParser()
         //Confirm duration parameters were successfully parsed with the default values in cassandra.yaml
         assertEquals(new DurationSpec.IntMillisecondsBound(10800000), config.max_hint_window);
         assertEquals(new DurationSpec.LongMillisecondsBound(0), config.native_transport_idle_timeout);
-        assertEquals(new DurationSpec.LongMillisecondsBound(10000), config.request_timeout);
-        assertEquals(new DurationSpec.LongMillisecondsBound(5000), config.read_request_timeout);
-        assertEquals(new DurationSpec.LongMillisecondsBound(10000), config.range_request_timeout);
-        assertEquals(new DurationSpec.LongMillisecondsBound(2000), config.write_request_timeout);
-        assertEquals(new DurationSpec.LongMillisecondsBound(5000), config.counter_write_request_timeout);
-        assertEquals(new DurationSpec.LongMillisecondsBound(1800), config.cas_contention_timeout);
+        assertEquals(new DurationSpec.LongMillisecondsBound(20000), config.request_timeout);
+        assertEquals(new DurationSpec.LongMillisecondsBound(20000), config.read_request_timeout);
+        assertEquals(new DurationSpec.LongMillisecondsBound(20000), config.range_request_timeout);
+        assertEquals(new DurationSpec.LongMillisecondsBound(20000), config.write_request_timeout);
+        assertEquals(new DurationSpec.LongMillisecondsBound(20000), config.counter_write_request_timeout);
+        assertEquals(new DurationSpec.LongMillisecondsBound(20000), config.cas_contention_timeout);
         assertEquals(new DurationSpec.LongMillisecondsBound(60000), config.truncate_request_timeout);
         assertEquals(new DurationSpec.IntSecondsBound(300), config.streaming_keep_alive_period);
         assertEquals(new DurationSpec.LongMillisecondsBound(500), config.slow_query_log_timeout);
@@ -75,6 +75,7 @@ public void testConfigurationLoaderParser()
         assertEquals(new DurationSpec.LongMillisecondsBound(1500), config.user_defined_functions_fail_timeout);
         assertEquals(new DurationSpec.LongMillisecondsBound(500), config.user_defined_functions_warn_timeout);
         assertEquals(new DurationSpec.IntSecondsBound(3600), config.validation_preview_purge_head_start);
+        assertEquals(new DurationSpec.LongMillisecondsBound(12000), config.native_transport_timeout);
 
         //Confirm space parameters were successfully parsed with the default values in cassandra.yaml
         assertNull(config.memtable_heap_space);
diff --git a/test/unit/org/apache/cassandra/config/StorageAttachedIndexOptionsTest.java b/test/unit/org/apache/cassandra/config/StorageAttachedIndexOptionsTest.java
deleted file mode 100644
index 753757014e26..000000000000
--- a/test/unit/org/apache/cassandra/config/StorageAttachedIndexOptionsTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.config;
-
-import org.junit.Test;
-
-import org.apache.cassandra.exceptions.ConfigurationException;
-
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-public class StorageAttachedIndexOptionsTest
-{
-    @Test
-    public void testStorageAttachedIndexOptionsValidation()
-    {
-        StorageAttachedIndexOptions saiOptions = new StorageAttachedIndexOptions();
-
-        saiOptions.segment_write_buffer_size = new DataStorageSpec.IntMebibytesBound(0);
-        saiOptions.validate();
-
-        saiOptions.segment_write_buffer_size = new DataStorageSpec.IntMebibytesBound(StorageAttachedIndexOptions.MAXIMUM_SEGMENT_BUFFER_MB);
-        saiOptions.validate();
-
-        saiOptions.segment_write_buffer_size = new DataStorageSpec.IntMebibytesBound(StorageAttachedIndexOptions.MAXIMUM_SEGMENT_BUFFER_MB + 1);
-        assertThatThrownBy(saiOptions::validate).isInstanceOf(ConfigurationException.class)
-                                                .hasMessage(StorageAttachedIndexOptions.INVALID_BUFFER_SIZE_ERROR);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/BasicReadTest.java b/test/unit/org/apache/cassandra/cql3/BasicReadTest.java
new file mode 100644
index 000000000000..fee8ef0daf1f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/BasicReadTest.java
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.memory.BufferPool;
+
+import static org.junit.Assert.assertEquals;
+
+
+/**
+ * Tests randomly causing re-read with NotInCacheException.
+ */
+public class BasicReadTest extends CQLTester
+{
+    final int BASE_COUNT = 700;
+    final int REPS = 150;
+    final int DELETIONS = 55;
+
+    Random rand;
+
+    @BeforeClass
+    public static void setupBasicReadTest()
+    {
+        // Make sure to go through the multiple page path for chunk cache entries
+        CassandraRelevantProperties.BUFFERPOOL_DISABLE_COMBINED_ALLOCATION.setBoolean(true);
+    }
+
+    @Before
+    public void setUp()
+    {
+        rand = new Random();
+    }
+
+    @Test
+    public void testWideIndexingForward() throws Throwable
+    {
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(16);  // 16 to allow exception within block
+        createTable("CREATE TABLE %s (k int, c int, v int, d text, PRIMARY KEY (k, c))");
+        int COUNT = rand.nextInt(BASE_COUNT / 10) + BASE_COUNT;
+
+        addDeletions(1, COUNT, rand.nextInt());
+
+        for (int i = 0; i < COUNT; i++)
+            execute("INSERT INTO %s (k, c, v, d) VALUES (?, ?, ?, ?)", 1, i, i, generateString(10 << (i % 12)));
+        flush();
+
+        interceptCache();
+        for (int rep = 0; rep < REPS; ++rep)
+        {
+            int i = rand.nextInt(COUNT);
+            int j = i + rand.nextInt(BASE_COUNT / 10);
+            if (j > COUNT)
+                j = COUNT;
+            Object[][] rows = getRows(execute("SELECT v FROM %s WHERE k = 1 and c >= ? and c < ?", i, j));
+            String message = String.format("%d<=c<%d rows returned %s", i, j, Arrays.deepToString(rows));
+            assertNoDeletions(message, rows);
+            assertEquals(message, j - i, rows.length);
+        }
+    }
+
+    @Test
+    public void testWideIndexingReversed() throws Throwable
+    {
+        DatabaseDescriptor.setColumnIndexSizeInKiB(16);
+        createTable("CREATE TABLE %s (k int, c int, v int, d text, PRIMARY KEY (k, c))");
+        int COUNT = rand.nextInt(BASE_COUNT / 10) + BASE_COUNT;
+
+        addDeletions(1, COUNT, rand.nextInt());
+
+        for (int i = 0; i < COUNT; i++)
+            execute("INSERT INTO %s (k, c, v, d) VALUES (?, ?, ?, ?)", 1, i, i, generateString(10 << (i % 12)));
+        flush();
+
+        interceptCache();
+        for (int rep = 0; rep < REPS; ++rep)
+        {
+            int i = rand.nextInt(COUNT);
+            int j = i + rand.nextInt(BASE_COUNT / 10);
+            if (j > COUNT)
+                j = COUNT;
+            Object[][] rows = getRows(execute("SELECT v FROM %s WHERE k = 1 and c >= ? and c < ? ORDER BY c DESC", i, j));
+            String message = String.format("%d<=c<%d rows returned %s", i, j, Arrays.deepToString(rows));
+            assertNoDeletions(message, rows);
+            assertEquals("Lookup between " + i + " and " + j + " count " + COUNT, j - i, rows.length);
+        }
+    }
+
+    @Test
+    public void testWideIndexForwardIn() throws Throwable
+    {
+        testWideIndexIn(false, true);
+    }
+
+    @Test
+    public void testWideIndexReversedIn() throws Throwable
+    {
+        testWideIndexIn(true, true);
+    }
+
+    @Test
+    public void testWideIndexForwardAllIn() throws Throwable
+    {
+        testWideIndexIn(false, false);
+    }
+
+    @Test
+    public void testWideIndexReversedAllIn() throws Throwable
+    {
+        testWideIndexIn(true, false);
+    }
+
+    private void testWideIndexIn(boolean reversed, boolean readSubset) throws Throwable
+    {
+        DatabaseDescriptor.setColumnIndexSizeInKiB(4);
+        createTable("CREATE TABLE %s (k int, c int, v int, d text, PRIMARY KEY (k, c, v))");
+        int COUNT = rand.nextInt(BASE_COUNT / 10) + BASE_COUNT;
+        int MULT = 5;
+
+        addDeletions(2, COUNT, rand.nextInt());
+
+        for (int i = 0; i < COUNT; i++)
+            for (int j = 0; j < MULT; ++j)
+                execute("INSERT INTO %s (k, c, v, d) VALUES (?, ?, ?, ?)", i % 3, i, j, generateString(100 << j));
+
+        flush();
+
+        interceptCache();
+        for (int rep = 0; rep < REPS; ++rep)
+        {
+            int[] arr;
+            if (readSubset)
+            {
+                Set<Integer> vals = new HashSet<>();
+                int sz = Math.max(5, rand.nextInt(BASE_COUNT / 50));
+                while (vals.size() < sz)
+                    vals.add(rand.nextInt(COUNT));
+                arr = vals.stream().mapToInt(i -> i).toArray();
+            }
+            else
+            {
+                arr = IntStream.range(0, COUNT).toArray();
+            }
+
+            String s = Arrays.stream(arr).mapToObj(Integer::toString).collect(Collectors.joining(","));
+            for (int i = 0; i < 3; ++i)
+            {
+                int ii = i;
+                Object[][] rows = getRows(execute(String.format("SELECT c,v FROM %%s WHERE k = ? and c IN (%s)%s", s, reversed ? " ORDER BY c DESC" : ""), i));
+                String message = String.format("c %s rows returned %s", s, Arrays.deepToString(rows));
+                assertNoDeletions(message, rows);
+                assertEquals("k = " + i + " IN " + s + " count " + COUNT + ": " +
+                        Arrays.stream(rows)
+                              .map(Arrays::toString)
+                              .collect(Collectors.joining("\n  ", "\n {", "\n }")),
+                        MULT * Arrays.stream(arr).filter(x -> x % 3 == ii).count(),
+                        rows.length);
+            }
+        }
+    }
+
+    @Test
+    public void testForward() throws Throwable
+    {
+        testForward("");
+    }
+
+    @Test
+    public void testForward_ChunkDirectlyAllocated() throws Throwable
+    {
+        testForward(String.format(" WITH compression = {'chunk_length_in_kb': '%d', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}",
+                                  BufferPool.NORMAL_CHUNK_SIZE / 1024 * 2));  // Note, this is currently 256k
+    }
+
+    @Test
+    public void testForward_128k() throws Throwable
+    {
+        testForward(" WITH compression = {'chunk_length_in_kb': '128', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testForward_64k() throws Throwable
+    {
+        testForward(" WITH compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testForward_32k() throws Throwable
+    {
+        testForward(" WITH compression = {'chunk_length_in_kb': '32', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testForward_16k() throws Throwable
+    {
+        testForward(" WITH compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testForward_8k() throws Throwable
+    {
+        testForward(" WITH compression = {'chunk_length_in_kb': '8', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testForward_4k() throws Throwable
+    {
+        testForward(" WITH compression = {'chunk_length_in_kb': '4', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testForward_2k() throws Throwable
+    {
+        testForward(" WITH compression = {'chunk_length_in_kb': '2', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testForward_1k() throws Throwable
+    {
+        testForward(" WITH compression = {'chunk_length_in_kb': '1', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    private void testForward(String compression) throws Throwable
+    {
+        int STEP = 32;
+        DatabaseDescriptor.setColumnIndexSizeInKiB(1000);   // make sure rows fit to test only non-indexed code
+        createTable("CREATE TABLE %s (k int, c int, v int, d text, PRIMARY KEY (k, c))" + compression);
+        int COUNT = rand.nextInt(BASE_COUNT / 10) + BASE_COUNT;
+
+        addDeletions(COUNT / STEP, STEP, rand.nextInt());
+        for (int i = 0; i < COUNT; i++)
+            execute("INSERT INTO %s (k, c, v, d) VALUES (?, ?, ?, ?)", i / STEP, i % STEP, i, generateString(10 << (i % 12)));
+        flush();
+
+        interceptCache();
+        for (int rep = 0; rep < REPS; ++rep)
+        {
+            int i = rand.nextInt(COUNT);
+            Object[][] rows = getRows(execute("SELECT v FROM %s WHERE k = ? and c >= ?", i / STEP, i % STEP));
+            int max = STEP;
+            if (i / STEP == COUNT / STEP)
+                max = COUNT % STEP;
+            String message = String.format("k %d c %d rows returned %s", i / STEP, i % STEP, Arrays.deepToString(rows));
+            assertNoDeletions(message, rows);
+            assertEquals(message, max - (i % STEP), rows.length);
+        }
+    }
+
+    @Test
+    public void testReversed() throws Throwable
+    {
+        testReversed("");
+    }
+
+    @Test
+    public void testReversed_ChunkDirectlyAllocated() throws Throwable
+    {
+        testReversed(String.format(" WITH compression = {'chunk_length_in_kb': '%d', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}",
+                                   BufferPool.NORMAL_CHUNK_SIZE / 1024 * 2)); // Note, this is currently 256k
+    }
+
+    @Test
+    public void testReversed_128k() throws Throwable
+    {
+        testReversed(" WITH compression = {'chunk_length_in_kb': '128', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testReversed_64k() throws Throwable
+    {
+        testReversed(" WITH compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testReversed_32k() throws Throwable
+    {
+        testReversed(" WITH compression = {'chunk_length_in_kb': '32', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testReversed_16k() throws Throwable
+    {
+        testReversed(" WITH compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testReversed_8k() throws Throwable
+    {
+        testReversed(" WITH compression = {'chunk_length_in_kb': '8', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testReversed_4k() throws Throwable
+    {
+        testReversed(" WITH compression = {'chunk_length_in_kb': '4', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testReversed_2k() throws Throwable
+    {
+        testReversed(" WITH compression = {'chunk_length_in_kb': '2', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    @Test
+    public void testReversed_1k() throws Throwable
+    {
+        testReversed(" WITH compression = {'chunk_length_in_kb': '1', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}");
+    }
+
+    private void testReversed(String compression) throws Throwable
+    {
+        int STEP = 32;
+        DatabaseDescriptor.setColumnIndexSizeInKiB(1000);   // make sure rows fit to test only non-indexed code
+        createTable("CREATE TABLE %s (k int, c int, v int, d text, PRIMARY KEY (k, c))" + compression);
+        int COUNT = rand.nextInt(BASE_COUNT / 10) + BASE_COUNT;
+
+        addDeletions(COUNT / STEP, STEP, rand.nextInt());
+        for (int i = 0; i < COUNT; i++)
+            execute("INSERT INTO %s (k, c, v, d) VALUES (?, ?, ?, ?)", i / STEP, i % STEP, i, generateString(10 << (i % 12)));
+        flush();
+
+        interceptCache();
+        for (int rep = 0; rep < REPS; ++rep)
+        {
+            int i = rand.nextInt(COUNT);
+            Object[][] rows = getRows(execute("SELECT v FROM %s WHERE k = ? and c < ? ORDER BY c DESC", i / STEP, i % STEP));
+            String message = String.format("k %d c %d rows returned %s", i / STEP, i % STEP, Arrays.deepToString(rows));
+            assertNoDeletions(message, rows);
+            assertEquals(i % STEP, rows.length);
+        }
+    }
+
+    public void addDeletions(int krange, int crange, int seed) throws Throwable
+    {
+        addDeletedDataTable(krange, crange, seed);
+        // Note: The loops here and in addDeletionsData need to fully match in their usage of rand
+        Random rand = new Random(seed);
+        for (int i = 0; i < DELETIONS; ++i)
+        {
+            int partition = rand.nextInt(krange) + 1;   // Note: partition 0 will not have tombstones intentionally
+            int left = rand.nextInt(crange + 1) - 1;
+            int right = left + rand.nextInt(crange + 1 - left);
+            boolean leftInclusive = left == right ? true : rand.nextBoolean();
+            boolean rightInclusive = left == right ? true : rand.nextBoolean();
+
+            int start = left + 1;
+            int range = right - left - 1;
+            int v = range > 0 ? start + rand.nextInt(range) : -1;
+            int c = rand.nextInt(3);
+
+            execute(String.format("DELETE FROM %%s WHERE k = ? AND c %s ? AND c %s ?",
+                                  leftInclusive ? ">=" : ">",
+                                  rightInclusive ? "<=" : "<"),
+                    partition,
+                    left,
+                    right);
+
+        }
+    }
+
+    public void addDeletedDataTable(int krange, int crange, int seed) throws Throwable
+    {
+        Random rand = new Random(seed);
+        for (int i = 0; i < DELETIONS; ++i)
+        {
+            int partition = rand.nextInt(krange) + 1;   // Note: partition 0 will not have tombstones intentionally
+            int left = rand.nextInt(crange + 1) - 1;
+            int right = left + rand.nextInt(crange + 1 - left);
+            boolean leftInclusive = left == right ? true : rand.nextBoolean();
+            boolean rightInclusive = left == right ? true : rand.nextBoolean();
+
+            int start = left + 1;
+            int range = right - left - 1;
+            int v = range > 0 ? start + rand.nextInt(range) : -1;
+            int c = rand.nextInt(3);
+            if (v == -1)
+            {
+                if (leftInclusive && rightInclusive)
+                    v = c < 1 ? left : right;
+                else if (leftInclusive)
+                    v = left;
+                else if (rightInclusive)
+                    v = right;
+                else    // nothing is covered
+                    continue;
+            }
+            else
+            {
+                if (leftInclusive && c == 0)
+                    v = left;
+                else if (rightInclusive && c == 2)
+                    v = right;
+            }
+
+            if (range > 0) // else right = left + 1, nothing inclusive
+                execute("INSERT INTO %s (k, c, v, d) VALUES (?, ?, ?, ?)", partition, v, v == 0 ? -11111  : -v, "DELETED");
+        }
+        flush();
+    }
+
+    public void assertNoDeletions(String message, Object[][] rows)
+    {
+        for (Object[] row : rows)
+        {
+            Assert.assertTrue("Deleted data resurfaced " + message, ((Number) row[0]).intValue() >= 0);
+        }
+    }
+
+    @Test
+    public void testRangeQueries() throws Throwable
+    {
+        DatabaseDescriptor.setColumnIndexSizeInKiB(16);
+        interceptCache();
+
+        createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c))");
+        int PARTITIONS = 20;
+        int ROWS = 10;
+        for (int i = 0; i < PARTITIONS; i++)
+            for (int j = 0; j < ROWS; j++)
+                execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", i, j, i * j);
+
+        flush();
+
+        for (int rep = 0; rep < REPS; ++rep)
+        {
+            Object[][] rows = getRows(execute("SELECT * FROM %s"));
+            assertEquals(PARTITIONS * ROWS, rows.length);
+        }
+
+        for (int rep = 0; rep < REPS; ++rep)
+        {
+            int from = rand.nextInt(PARTITIONS - 2);
+            int to = 2 + from + rand.nextInt(PARTITIONS - from - 2);
+
+            Object[][] rows = getRows(execute("SELECT k, c, v FROM %s WHERE k <= ? and k >= ? ALLOW FILTERING", to, from));
+            assertEquals((to - from + 1) * ROWS, rows.length);
+
+            rows = getRows(execute("SELECT k, c, v FROM %s WHERE k < ? and k >= ? ALLOW FILTERING", to, from));
+            assertEquals((to - from) * ROWS, rows.length);
+
+            rows = getRows(execute("SELECT k, c, v FROM %s WHERE k <= ? and k > ? ALLOW FILTERING", to, from));
+            assertEquals((to - from) * ROWS, rows.length);
+
+            rows = getRows(execute("SELECT k, c, v FROM %s WHERE k < ? and k > ? ALLOW FILTERING", to, from));
+            assertEquals((to - from - 1) * ROWS, rows.length);
+        }
+    }
+
+    String generateString(int length)
+    {
+        String s = "";
+        for (int i = 0; i < length; ++i)
+            s += (char) ('a' + (i % 26));
+        return s;
+    }
+
+    public void interceptCache()
+    {
+        // no additional cache modification
+    }
+
+    @After
+    public void clearIntercept()
+    {
+        // no additional cache modification
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/BatchTest.java b/test/unit/org/apache/cassandra/cql3/BatchTest.java
index 330271e982d3..8cf9ac9b86c3 100644
--- a/test/unit/org/apache/cassandra/cql3/BatchTest.java
+++ b/test/unit/org/apache/cassandra/cql3/BatchTest.java
@@ -17,6 +17,15 @@
  */
 package org.apache.cassandra.cql3;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
 import com.datastax.driver.core.BatchStatement;
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.PreparedStatement;
@@ -24,14 +33,13 @@
 import com.datastax.driver.core.exceptions.InvalidQueryException;
 import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.service.EmbeddedCassandraService;
+import org.apache.cassandra.utils.ByteBufferUtil;
 
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertSame;
 
 public class BatchTest extends CQLTester
 {
@@ -47,6 +55,10 @@ public class BatchTest extends CQLTester
     @BeforeClass()
     public static void setup() throws ConfigurationException, IOException
     {
+        // Set batch sizes for guardrails to the same values as in Apache
+        // Needed for testOversizedBatch()
+        DatabaseDescriptor.setBatchSizeWarnThresholdInKiB(5);
+        DatabaseDescriptor.setBatchSizeFailThresholdInKiB(50);
         cassandra = ServerTestUtils.startEmbeddedCassandraService();
 
         cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build();
@@ -171,6 +183,28 @@ public void testOversizedBatch()
         session.execute(b);
     }
 
+    @Test
+    public void testQueryOptionConsistency()
+    {
+        BatchQueryOptions queryOptions = BatchQueryOptions.withoutPerStatementVariables(QueryOptions.DEFAULT);
+        assertSame(ConsistencyLevel.ONE, queryOptions.getConsistency());
+        queryOptions.updateConsistency(ConsistencyLevel.ALL);
+        assertSame(ConsistencyLevel.ALL, queryOptions.getConsistency());
+    }
+
+    @Test
+    public void testGetVariables()
+    {
+        BatchQueryOptions queryOptions = BatchQueryOptions.withoutPerStatementVariables(QueryOptions.DEFAULT);
+        assertThat(queryOptions.getVariables()).isEmpty();
+
+        List<List<ByteBuffer>> variables = Collections.singletonList(Collections.singletonList(ByteBufferUtil.bytes(1)));
+        List<Object> queryOrIdList = Collections.singletonList(1);
+        queryOptions = BatchQueryOptions.withPerStatementVariables(QueryOptions.DEFAULT, variables, queryOrIdList);
+        assertThat(queryOptions.getVariables()).isEqualTo(variables);
+        assertThat(queryOptions.getQueryOrIdList()).isEqualTo(queryOrIdList);
+    }
+
     public void sendBatch(BatchStatement.Type type, boolean addCounter, boolean addNonCounter, boolean addClustering)
     {
 
diff --git a/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java b/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java
new file mode 100644
index 000000000000..416235b9c2aa
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java
@@ -0,0 +1,821 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.serializers.AsciiSerializer;
+import org.apache.cassandra.serializers.BytesSerializer;
+import org.apache.cassandra.serializers.CollectionSerializer;
+import org.apache.cassandra.serializers.DurationSerializer;
+import org.apache.cassandra.serializers.InetAddressSerializer;
+import org.apache.cassandra.serializers.SimpleDateSerializer;
+import org.apache.cassandra.serializers.TimeSerializer;
+import org.apache.cassandra.serializers.TimestampSerializer;
+import org.apache.cassandra.serializers.UTF8Serializer;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test functionality to re-create a CQL literal from its serialized representation.
+ * This test uses some randomness to generate the values and nested structures (collections,tuples,UDTs).
+ */
+public class CQL3TypeLiteralTest
+{
+    private static final Pattern QUOTE = Pattern.compile("'");
+
+    private static final Random r = new Random();
+    /**
+     * Container holding the expected CQL literal for a type and serialized value.
+     * The CQL literal is generated independently from the code in {@link CQL3Type}.
+     */
+    static class Value
+    {
+        final String expected;
+        final CQL3Type cql3Type;
+        final ByteBuffer value;
+
+        Value(String expected, CQL3Type cql3Type, ByteBuffer value)
+        {
+            this.expected = expected;
+            this.cql3Type = cql3Type;
+            this.value = value;
+        }
+    }
+
+    static final Map<CQL3Type.Native, List<Value>> nativeTypeValues = new EnumMap<>(CQL3Type.Native.class);
+
+    static void addNativeValue(String expected, CQL3Type.Native cql3Type, ByteBuffer value)
+    {
+        List<Value> l = nativeTypeValues.get(cql3Type);
+        if (l == null)
+            nativeTypeValues.put(cql3Type, l = new ArrayList<>());
+        l.add(new Value(expected, cql3Type, value));
+    }
+
+    static
+    {
+        // Add some (random) values for each native type.
+        // Also adds null values and empty values, if the type allows this.
+
+        for (int i = 0; i < 20; i++)
+        {
+            String v = randString(true);
+            addNativeValue(quote(v), CQL3Type.Native.ASCII, AsciiSerializer.instance.serialize(v));
+        }
+        addNativeValue("''", CQL3Type.Native.ASCII, AsciiSerializer.instance.serialize(""));
+        addNativeValue("''", CQL3Type.Native.ASCII, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.ASCII, null);
+
+        for (int i = 0; i < 20; i++)
+        {
+            String v = randString(false);
+            addNativeValue(quote(v), CQL3Type.Native.TEXT, UTF8Serializer.instance.serialize(v));
+        }
+        addNativeValue("''", CQL3Type.Native.TEXT, UTF8Serializer.instance.serialize(""));
+        addNativeValue("''", CQL3Type.Native.TEXT, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.TEXT, null);
+
+        for (int i = 0; i < 20; i++)
+        {
+            String v = randString(false);
+            addNativeValue(quote(v), CQL3Type.Native.VARCHAR, UTF8Serializer.instance.serialize(v));
+        }
+        addNativeValue("''", CQL3Type.Native.VARCHAR, UTF8Serializer.instance.serialize(""));
+        addNativeValue("''", CQL3Type.Native.VARCHAR, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.VARCHAR, null);
+
+        addNativeValue("0", CQL3Type.Native.BIGINT, LongType.instance.decompose(0L));
+        for (int i = 0; i < 20; i++)
+        {
+            long v = randLong();
+            addNativeValue(Long.toString(v), CQL3Type.Native.BIGINT, LongType.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.BIGINT, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.BIGINT, null);
+
+        addNativeValue("0", CQL3Type.Native.COUNTER, LongType.instance.decompose(0L));
+        for (int i = 0; i < 20; i++)
+        {
+            long v = randLong();
+            addNativeValue(Long.toString(v), CQL3Type.Native.COUNTER, LongType.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.COUNTER, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.COUNTER, null);
+
+        addNativeValue("0", CQL3Type.Native.INT, Int32Type.instance.decompose(0));
+        for (int i = 0; i < 20; i++)
+        {
+            int v = randInt();
+            addNativeValue(Integer.toString(v), CQL3Type.Native.INT, Int32Type.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.INT, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.INT, null);
+
+        addNativeValue("0", CQL3Type.Native.SMALLINT, ShortType.instance.decompose((short) 0));
+        for (int i = 0; i < 20; i++)
+        {
+            short v = randShort();
+            addNativeValue(Short.toString(v), CQL3Type.Native.SMALLINT, ShortType.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.SMALLINT, null);
+
+        addNativeValue("0", CQL3Type.Native.TINYINT, ByteType.instance.decompose((byte) 0));
+        for (int i = 0; i < 20; i++)
+        {
+            byte v = randByte();
+            addNativeValue(Short.toString(v), CQL3Type.Native.TINYINT, ByteType.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.TINYINT, null);
+
+        addNativeValue("0.0", CQL3Type.Native.FLOAT, FloatType.instance.decompose((float) 0));
+        for (int i = 0; i < 20; i++)
+        {
+            float v = randFloat();
+            addNativeValue(Float.toString(v), CQL3Type.Native.FLOAT, FloatType.instance.decompose(v));
+        }
+        addNativeValue("NaN", CQL3Type.Native.FLOAT, FloatType.instance.decompose(Float.NaN));
+        addNativeValue("null", CQL3Type.Native.FLOAT, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.FLOAT, null);
+
+        addNativeValue("0.0", CQL3Type.Native.DOUBLE, DoubleType.instance.decompose((double) 0));
+        for (int i = 0; i < 20; i++)
+        {
+            double v = randDouble();
+            addNativeValue(Double.toString(v), CQL3Type.Native.DOUBLE, DoubleType.instance.decompose(v));
+        }
+        addNativeValue("NaN", CQL3Type.Native.DOUBLE, DoubleType.instance.decompose(Double.NaN));
+        addNativeValue("null", CQL3Type.Native.DOUBLE, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.DOUBLE, null);
+
+        addNativeValue("0", CQL3Type.Native.DECIMAL, DecimalType.instance.decompose(BigDecimal.ZERO));
+        for (int i = 0; i < 20; i++)
+        {
+            BigDecimal v = BigDecimal.valueOf(randDouble());
+            addNativeValue(v.toString(), CQL3Type.Native.DECIMAL, DecimalType.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.DECIMAL, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.DECIMAL, null);
+
+        addNativeValue("0", CQL3Type.Native.VARINT, IntegerType.instance.decompose(BigInteger.ZERO));
+        for (int i = 0; i < 20; i++)
+        {
+            BigInteger v = BigInteger.valueOf(randLong());
+            addNativeValue(v.toString(), CQL3Type.Native.VARINT, IntegerType.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.VARINT, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.VARINT, null);
+
+        // boolean doesn't have that many possible values...
+        addNativeValue("false", CQL3Type.Native.BOOLEAN, BooleanType.instance.decompose(false));
+        addNativeValue("true", CQL3Type.Native.BOOLEAN, BooleanType.instance.decompose(true));
+        addNativeValue("null", CQL3Type.Native.BOOLEAN, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.BOOLEAN, null);
+
+        // (mostly generates date values with surreal values like in year 14273)
+        for (int i = 0; i < 20; i++)
+        {
+            int v = randInt();
+            addNativeValue(quote(SimpleDateSerializer.instance.toString(v)), CQL3Type.Native.DATE, SimpleDateSerializer.instance.serialize(v));
+        }
+        addNativeValue("null", CQL3Type.Native.DATE, null);
+
+        for (int i = 0; i < 100; i++)
+        {
+            long v = randLong(24L * 60 * 60 * 1000 * 1000 * 1000);
+            addNativeValue(quote(TimeSerializer.instance.toString(v)), CQL3Type.Native.TIME, TimeSerializer.instance.serialize(v));
+        }
+        addNativeValue("null", CQL3Type.Native.TIME, null);
+
+        for (int i = 0; i < 100; i++)
+        {
+            Duration duration = Duration.newInstance(Math.abs(randInt()), Math.abs(randInt()), Math.abs(randLong()));
+            addNativeValue(DurationSerializer.instance.toString(duration), CQL3Type.Native.DURATION, DurationSerializer.instance.serialize(duration));
+        }
+        addNativeValue("null", CQL3Type.Native.DURATION, null);
+
+        // (mostly generates timestamp values with surreal values like in year 14273)
+        for (int i = 0; i < 20; i++)
+        {
+            long v = randLong();
+            addNativeValue(quote(TimestampSerializer.instance.toStringUTC(new Date(v))), CQL3Type.Native.TIMESTAMP, TimestampType.instance.fromString(Long.toString(v)));
+        }
+        addNativeValue("null", CQL3Type.Native.TIMESTAMP, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.TIMESTAMP, null);
+
+        for (int i = 0; i < 20; i++)
+        {
+//            UUID v = UUIDGen.getTimeUUID(randLong(System.currentTimeMillis()));
+            TimeUUID v = TimeUUID.Generator.nextTimeUUID();
+            addNativeValue(v.toString(), CQL3Type.Native.TIMEUUID, TimeUUIDType.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.TIMEUUID, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.TIMEUUID, null);
+
+        for (int i = 0; i < 20; i++)
+        {
+            UUID v = UUID.randomUUID();
+            addNativeValue(v.toString(), CQL3Type.Native.UUID, UUIDType.instance.decompose(v));
+        }
+        addNativeValue("null", CQL3Type.Native.UUID, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.UUID, null);
+
+        for (int i = 0; i < 20; i++)
+        {
+            ByteBuffer v = randBytes();
+            addNativeValue("0x" + BytesSerializer.instance.toString(v), CQL3Type.Native.BLOB, BytesType.instance.decompose(v));
+        }
+        addNativeValue("0x", CQL3Type.Native.BLOB, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.BLOB, null);
+
+        for (int i = 0; i < 20; i++)
+        {
+            InetAddress v;
+            try
+            {
+                v = InetAddress.getByAddress(new byte[]{ randByte(), randByte(), randByte(), randByte() });
+            }
+            catch (UnknownHostException e)
+            {
+                throw new RuntimeException(e);
+            }
+            addNativeValue(quote(v.getHostAddress()), CQL3Type.Native.INET, InetAddressSerializer.instance.serialize(v));
+        }
+        addNativeValue("null", CQL3Type.Native.INET, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        addNativeValue("null", CQL3Type.Native.INET, null);
+    }
+
+    @Test
+    public void testNative()
+    {
+        // test each native type against each supported protocol version (although it doesn't make sense to
+        // iterate through all protocol versions as of C* 3.0).
+
+        for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        {
+            for (Map.Entry<CQL3Type.Native, List<Value>> entry : nativeTypeValues.entrySet())
+            {
+                for (Value value : entry.getValue())
+                {
+                    compareCqlLiteral(version, value);
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testCollectionWithNatives()
+    {
+        // test 100 collections with varying element/key/value types against each supported protocol version,
+        // type of collection is randomly chosen
+
+        for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        {
+            for (int n = 0; n < 100; n++)
+            {
+                Value value = generateCollectionValue(version, randomCollectionType(0), true);
+                compareCqlLiteral(version, value);
+            }
+        }
+    }
+
+    @Test
+    public void testCollectionNullAndEmpty()
+    {
+        // An empty collection is one with a size of 0 (note that rely on the fact that protocol version < 3 are not
+        // supported anymore and so the size of a collection is always on 4 bytes).
+        ByteBuffer emptyCollection = ByteBufferUtil.bytes(0);
+
+        for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        {
+            for (boolean frozen : Arrays.asList(true, false))
+            {
+                // empty
+                Value value = new Value("[]", ListType.getInstance(UTF8Type.instance, frozen).asCQL3Type(), emptyCollection);
+                compareCqlLiteral(version, value);
+                value = new Value("{}", SetType.getInstance(UTF8Type.instance, frozen).asCQL3Type(), emptyCollection);
+                compareCqlLiteral(version, value);
+                value = new Value("{}", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, frozen).asCQL3Type(), emptyCollection);
+                compareCqlLiteral(version, value);
+
+                // null
+                value = new Value("null", ListType.getInstance(UTF8Type.instance, frozen).asCQL3Type(), null);
+                compareCqlLiteral(version, value);
+                value = new Value("null", SetType.getInstance(UTF8Type.instance, frozen).asCQL3Type(), null);
+                compareCqlLiteral(version, value);
+                value = new Value("null", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, frozen).asCQL3Type(), null);
+                compareCqlLiteral(version, value);
+            }
+        }
+    }
+
+    @Test
+    public void testTupleWithNatives()
+    {
+        // test 100 tuples with varying element/key/value types against each supported protocol version
+
+        for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        {
+            for (int n = 0; n < 100; n++)
+            {
+                Value value = generateTupleValue(version, randomTupleType(0), true);
+                compareCqlLiteral(version, value);
+            }
+        }
+    }
+
+    @Test
+    public void testUserDefinedWithNatives()
+    {
+        // test 100 UDTs with varying element/key/value types against each supported protocol version
+
+        for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        {
+            for (int n = 0; n < 100; n++)
+            {
+                Value value = generateUserDefinedValue(version, randomUserType(0), true);
+                compareCqlLiteral(version, value);
+            }
+        }
+    }
+
+    @Test
+    public void testNested()
+    {
+        // This is the "nice" part of this unit test - it tests (probably) nested type structures
+        // like 'tuple<map, list<user>, tuple, user>' or 'map<tuple<int, text>, set<inet>>' with
+        // random types  against each supported protocol version.
+
+        for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        {
+            for (int n = 0; n < 100; n++)
+            {
+                Value value = randomNested(version);
+                compareCqlLiteral(version, value);
+            }
+        }
+    }
+
+    @Test
+    public void testForEachUserType()
+    {
+        UTName name1 = new UTName(new ColumnIdentifier("ks", true), new ColumnIdentifier("name1", true));
+        UTName name2 = new UTName(new ColumnIdentifier("ks", true), new ColumnIdentifier("name2", true));
+
+        checkForEachUserType(CQL3Type.Raw.from(UTF8Type.instance.asCQL3Type()));
+        checkForEachUserType(CQL3Type.Raw.userType(name1), name1);
+        checkForEachUserType(CQL3Type.Raw.list(CQL3Type.Raw.userType(name1)), name1);
+        checkForEachUserType(CQL3Type.Raw.set(CQL3Type.Raw.userType(name1)), name1);
+        checkForEachUserType(CQL3Type.Raw.map(CQL3Type.Raw.userType(name1), CQL3Type.Raw.userType(name2)), name1, name2);
+        checkForEachUserType(CQL3Type.Raw.tuple(Arrays.asList(CQL3Type.Raw.userType(name1), CQL3Type.Raw.userType(name2)), true), name1, name2);
+    }
+
+    private void checkForEachUserType(CQL3Type.Raw t, UTName... expectedNames)
+    {
+        Set<UTName> collectedNames = new HashSet<>();
+        t.forEachUserType(collectedNames::add);
+        assertThat(collectedNames).hasSameElementsAs(Arrays.stream(expectedNames).collect(Collectors.toSet()));
+    }
+
+    static void compareCqlLiteral(ProtocolVersion version, Value value)
+    {
+        ByteBuffer buffer = value.value != null ? value.value.duplicate() : null;
+        String msg = "Failed to get expected value for type " + value.cql3Type + " / " + value.cql3Type.getType() + " with protocol-version " + version + " expected:\"" + value.expected + '"';
+        try
+        {
+            assertEquals(msg,
+                         value.expected,
+                         value.cql3Type.toCQLLiteral(buffer));
+        }
+        catch (RuntimeException e)
+        {
+            throw new RuntimeException(msg, e);
+        }
+    }
+
+    static Value randomNested(ProtocolVersion version)
+    {
+        AbstractType type = randomNestedType(2);
+
+        return generateAnyValue(version, type.asCQL3Type());
+    }
+
+    /**
+     * Generates type of randomly nested type structures.
+     */
+    static AbstractType randomNestedType(int level)
+    {
+        if (level == 0)
+            return randomNativeType();
+        switch (randInt(level == 2 ? 3 : 4))
+        {
+            case 0:
+                return randomCollectionType(level - 1);
+            case 1:
+                return randomTupleType(level - 1);
+            case 2:
+                return randomUserType(level - 1);
+            case 3:
+                return randomNativeType();
+        }
+        throw new AssertionError();
+    }
+
+    static Value generateCollectionValue(ProtocolVersion version, CollectionType collectionType, boolean allowNull)
+    {
+        StringBuilder expected = new StringBuilder();
+        ByteBuffer buffer;
+
+        if (allowNull && randBool(0.05d))
+        {
+            expected.append("null");
+            buffer = null;
+        }
+        else
+        {
+            int size = randInt(20);
+
+            CQL3Type elements;
+            CQL3Type values = null;
+            char bracketOpen;
+            char bracketClose;
+            switch (collectionType.kind)
+            {
+                case LIST:
+                    elements = ((ListType) collectionType).getElementsType().asCQL3Type();
+                    bracketOpen = '[';
+                    bracketClose = ']';
+                    break;
+                case SET:
+                    elements = ((SetType) collectionType).getElementsType().asCQL3Type();
+                    bracketOpen = '{';
+                    bracketClose = '}';
+                    break;
+                case MAP:
+                    elements = ((MapType) collectionType).getKeysType().asCQL3Type();
+                    values = ((MapType) collectionType).getValuesType().asCQL3Type();
+                    bracketOpen = '{';
+                    bracketClose = '}';
+                    break;
+                default:
+                    throw new AssertionError();
+            }
+
+            expected.append(bracketOpen);
+            Collection<ByteBuffer> buffers = new ArrayList<>();
+            Set<ByteBuffer> added = new HashSet<>();
+            for (int i = 0; i < size; i++)
+            {
+                Value el = generateAnyValue(version, elements);
+                if (!added.add(el.value))
+                    continue;
+
+                buffers.add(el.value.duplicate());
+                if (expected.length() > 1)
+                    expected.append(", ");
+                expected.append(el.cql3Type.toCQLLiteral(el.value));
+
+                if (collectionType.kind == CollectionType.Kind.MAP)
+                {
+                    // add map value
+                    el = generateAnyValue(version, values);
+                    buffers.add(el.value.duplicate());
+                    expected.append(": ");
+                    expected.append(el.cql3Type.toCQLLiteral(el.value));
+                }
+            }
+            expected.append(bracketClose);
+            buffer = CollectionSerializer.pack(buffers, added.size());
+        }
+
+        return new Value(expected.toString(), collectionType.asCQL3Type(), buffer);
+    }
+
+    /**
+     * Generates a value for any type or type structure.
+     */
+    static Value generateAnyValue(ProtocolVersion version, CQL3Type type)
+    {
+        if (type instanceof CQL3Type.Native)
+            return generateNativeValue(type, false);
+        if (type instanceof CQL3Type.Tuple)
+            return generateTupleValue(version, (TupleType) type.getType(), false);
+        if (type instanceof CQL3Type.UserDefined)
+            return generateUserDefinedValue(version, (UserType) type.getType(), false);
+        if (type instanceof CQL3Type.Collection)
+            return generateCollectionValue(version, (CollectionType) type.getType(), false);
+        throw new AssertionError();
+    }
+
+    static Value generateTupleValue(ProtocolVersion version, TupleType tupleType, boolean allowNull)
+    {
+        StringBuilder expected = new StringBuilder();
+        ByteBuffer buffer;
+
+        if (allowNull && randBool(0.05d))
+        {
+            // generate 'null' collection
+            expected.append("null");
+            buffer = null;
+        }
+        else
+        {
+            expected.append('(');
+
+            // # of fields in this value
+            int fields = tupleType.size();
+            if (randBool(0.2d))
+                fields = randInt(fields);
+
+            ByteBuffer[] buffers = new ByteBuffer[fields];
+            for (int i = 0; i < fields; i++)
+            {
+                AbstractType<?> fieldType = tupleType.type(i);
+
+                if (i > 0)
+                    expected.append(", ");
+
+                if (allowNull && randBool(.1))
+                {
+                    expected.append("null");
+                    continue;
+                }
+
+                Value value = generateAnyValue(version, fieldType.asCQL3Type());
+                expected.append(value.expected);
+                buffers[i] = value.value.duplicate();
+            }
+            expected.append(')');
+            buffer = TupleType.buildValue(buffers);
+        }
+
+        return new Value(expected.toString(), tupleType.asCQL3Type(), buffer);
+    }
+
+    static Value generateUserDefinedValue(ProtocolVersion version, UserType userType, boolean allowNull)
+    {
+        StringBuilder expected = new StringBuilder();
+        ByteBuffer buffer;
+
+        if (allowNull && randBool(0.05d))
+        {
+            // generate 'null' collection
+            expected.append("null");
+            buffer = null;
+        }
+        else
+        {
+            expected.append('{');
+
+            // # of fields in this value
+            int fields = userType.size();
+            if (randBool(0.2d))
+                fields = randInt(fields);
+
+            ByteBuffer[] buffers = new ByteBuffer[fields];
+            for (int i = 0; i < fields; i++)
+            {
+                AbstractType<?> fieldType = userType.type(i);
+
+                if (i > 0)
+                    expected.append(", ");
+
+                expected.append(ColumnIdentifier.maybeQuote(userType.fieldNameAsString(i)));
+                expected.append(": ");
+
+                if (randBool(.1))
+                {
+                    expected.append("null");
+                    continue;
+                }
+
+                Value value = generateAnyValue(version, fieldType.asCQL3Type());
+                expected.append(value.expected);
+                buffers[i] = value.value.duplicate();
+            }
+            expected.append('}');
+            buffer = TupleType.buildValue(buffers);
+        }
+
+        return new Value(expected.toString(), userType.asCQL3Type(), buffer);
+    }
+
+    static Value generateNativeValue(CQL3Type type, boolean allowNull)
+    {
+        List<Value> values = nativeTypeValues.get(type);
+        assert values != null : type.toString() + " needs to be defined";
+        while (true)
+        {
+            Value v = values.get(randInt(values.size()));
+            if (allowNull || v.value != null)
+                return v;
+        }
+    }
+
+    static CollectionType randomCollectionType(int level)
+    {
+        CollectionType.Kind kind = CollectionType.Kind.values()[randInt(CollectionType.Kind.values().length)];
+        switch (kind)
+        {
+            case LIST:
+            case SET:
+                return ListType.getInstance(randomNestedType(level), randBool());
+            case MAP:
+                return MapType.getInstance(randomNestedType(level), randomNestedType(level), randBool());
+        }
+        throw new AssertionError();
+    }
+
+    static TupleType randomTupleType(int level)
+    {
+        int typeCount = 2 + randInt(5);
+        List<AbstractType<?>> types = new ArrayList<>();
+        for (int i = 0; i < typeCount; i++)
+            types.add(randomNestedType(level));
+        return new TupleType(types);
+    }
+
+    static UserType randomUserType(int level)
+    {
+        int typeCount = 2 + randInt(5);
+        List<FieldIdentifier> names = new ArrayList<>();
+        List<AbstractType<?>> types = new ArrayList<>();
+        for (int i = 0; i < typeCount; i++)
+        {
+            names.add(FieldIdentifier.forQuoted('f' + randLetters(i)));
+            types.add(randomNestedType(level));
+        }
+        return new UserType("ks", UTF8Type.instance.fromString("u" + randInt(1000000)), names, types, true);
+    }
+
+    //
+    // Following methods are just helper methods. Mostly to generate many kinds of random values.
+    //
+
+    private static String randLetters(int len)
+    {
+        StringBuilder sb = new StringBuilder(len);
+        while (len-- > 0)
+        {
+            int i = randInt(52);
+            if (i < 26)
+                sb.append((char) ('A' + i));
+            else
+                sb.append((char) ('a' + i - 26));
+        }
+        return sb.toString();
+    }
+
+    static AbstractType randomNativeType()
+    {
+        while (true)
+        {
+            CQL3Type.Native t = CQL3Type.Native.values()[randInt(CQL3Type.Native.values().length)];
+            if (t != CQL3Type.Native.EMPTY)
+                return t.getType();
+        }
+    }
+
+    static boolean randBool()
+    {
+        return randBool(0.5d);
+    }
+
+    static boolean randBool(double probability)
+    {
+        return r.nextDouble() < probability;
+    }
+
+    static long randLong()
+    {
+        return r.nextLong();
+    }
+
+    static long randLong(long max)
+    {
+        return r.nextLong() % max;
+    }
+
+    static int randInt()
+    {
+        return r.nextInt();
+    }
+
+    static int randInt(int max)
+    {
+        return r.nextInt(max);
+    }
+
+    static short randShort()
+    {
+        return (short) r.nextInt();
+    }
+
+    static byte randByte()
+    {
+        return (byte) r.nextInt();
+    }
+
+    static double randDouble()
+    {
+        return r.nextDouble();
+    }
+
+    static float randFloat()
+    {
+        return r.nextFloat();
+    }
+
+    static String randString(boolean ascii)
+    {
+        int l = randInt(20);
+        StringBuilder sb = new StringBuilder(l);
+        for (int i = 0; i < l; i++)
+        {
+            if (randBool(.05))
+                sb.append('\'');
+            else
+            {
+                char c = (char) (ascii ? randInt(128) : randShort());
+                sb.append(c);
+            }
+        }
+        return UTF8Serializer.instance.deserialize(UTF8Serializer.instance.serialize(sb.toString()));
+    }
+
+    static ByteBuffer randBytes()
+    {
+        int l = randInt(20);
+        byte[] v = new byte[l];
+        for (int i = 0; i < l; i++)
+        {
+            v[i] = randByte();
+        }
+        return ByteBuffer.wrap(v);
+    }
+
+    private static String quote(String v)
+    {
+        return '\'' + QUOTE.matcher(v).replaceAll("''") + '\'';
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index 6eade9333e1c..a918c93981ed 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -42,9 +42,12 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Random;
 import java.util.Set;
 import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
@@ -63,6 +66,7 @@
 
 import com.google.common.base.Objects;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import org.apache.commons.lang3.ArrayUtils;
@@ -72,17 +76,23 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.rules.TestName;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.codahale.metrics.Gauge;
 import com.datastax.driver.core.CloseFuture;
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.ColumnDefinitions;
 import com.datastax.driver.core.DataType;
 import com.datastax.driver.core.NettyOptions;
+import com.datastax.driver.core.PoolingOptions;
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.Session;
@@ -101,7 +111,6 @@
 import org.apache.cassandra.auth.AuthSchemaChangeListener;
 import org.apache.cassandra.auth.AuthTestUtils;
 import org.apache.cassandra.auth.IRoleManager;
-import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DataStorageSpec;
@@ -109,10 +118,13 @@
 import org.apache.cassandra.config.EncryptionOptions;
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.cql3.functions.types.ParseUtils;
+import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.BooleanType;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
@@ -153,8 +165,10 @@
 import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.metrics.ClientMetrics;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.SchemaKeyspace;
@@ -173,6 +187,7 @@
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JMXServerUtils;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.TimeUUID;
 import org.assertj.core.api.Assertions;
@@ -181,6 +196,7 @@
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DRIVER_CONNECTION_TIMEOUT_MS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DRIVER_READ_TIMEOUT_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_RANDOM_SEED;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_REUSE_PREPARED;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ROW_CACHE_SIZE;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_USE_PREPARED;
@@ -189,6 +205,8 @@
 import static org.apache.cassandra.cql3.SchemaElement.SchemaElementType.MATERIALIZED_VIEW;
 import static org.apache.cassandra.cql3.SchemaElement.SchemaElementType.TABLE;
 import static org.apache.cassandra.cql3.SchemaElement.SchemaElementType.TYPE;
+import static org.apache.cassandra.index.sai.SAITester.vector;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -222,7 +240,17 @@ public abstract class CQLTester
     public static final String DATA_CENTER = ServerTestUtils.DATA_CENTER;
     public static final String DATA_CENTER_REMOTE = ServerTestUtils.DATA_CENTER_REMOTE;
     public static final String RACK1 = ServerTestUtils.RACK1;
-    protected static final int ASSERTION_TIMEOUT_SECONDS = 15;
+    private static final int ASSERTION_TIMEOUT_SECONDS = 15;
+
+    /**
+     * Whether to use coordinator execution in {@link #execute(String, Object...)}, so queries get full validation and
+     * go through reconciliation. When enabled, calls to {@link #execute(String, Object...)} will behave as calls to
+     * {@link #executeWithCoordinator(String, Object...)}. Otherwise, they will behave as calls to
+     * {@link #executeInternal(String, Object...)}.
+     *
+     * @see #execute
+     */
+    private static boolean coordinatorExecution = false;
 
     private static org.apache.cassandra.transport.Server server;
     private static JMXConnectorServer jmxServer;
@@ -231,6 +259,8 @@ public abstract class CQLTester
     protected static MBeanServerConnection jmxConnection;
 
     protected static int nativePort;
+    private static Randomization random;
+
     protected static final InetAddress nativeAddr;
     protected static final Set<InetAddressAndPort> remoteAddrs = new HashSet<>();
     private static final Map<Pair<User, ProtocolVersion>, Cluster> clusters = new HashMap<>();
@@ -238,14 +268,19 @@ public abstract class CQLTester
 
     private static Consumer<Cluster.Builder> clusterBuilderConfigurator;
 
+    private static final ThreadPoolExecutor schemaCleanup =
+    new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
+
+
     public static final List<ProtocolVersion> PROTOCOL_VERSIONS = new ArrayList<>(ProtocolVersion.SUPPORTED.size());
 
     private static final String CREATE_INDEX_NAME_REGEX = "(\\s*(\\w*|\"\\w*\")\\s*)";
+    private static final String CREATE_INDEX_NAME_QUOTED_REGEX = "(\\s*(\\w*|\"[^\"]*\")\\s*)";
     private static final String CREATE_INDEX_REGEX = String.format("\\A\\s*CREATE(?:\\s+CUSTOM)?\\s+INDEX" +
                                                                    "(?:\\s+IF\\s+NOT\\s+EXISTS)?\\s*" +
                                                                    "%s?\\s*ON\\s+(%<s\\.)?%<s\\s*" +
-                                                                   "(\\((?:\\s*\\w+\\s*\\()?%<s\\))?",
-                                                                   CREATE_INDEX_NAME_REGEX);
+                                                                   "(\\((?:\\s*\\w+\\s*\\()?%s\\))?",
+                                                                   CREATE_INDEX_NAME_REGEX, CREATE_INDEX_NAME_QUOTED_REGEX);
     private static final Pattern CREATE_INDEX_PATTERN = Pattern.compile(CREATE_INDEX_REGEX, Pattern.CASE_INSENSITIVE);
 
     public static final NettyOptions IMMEDIATE_CONNECTION_SHUTDOWN_NETTY_OPTIONS = new NettyOptions()
@@ -283,7 +318,7 @@ public static final ProtocolVersion getDefaultVersion()
     private List<String> types = new ArrayList<>();
     private List<String> functions = new ArrayList<>();
     private List<String> aggregates = new ArrayList<>();
-
+    private List<String> indexes = new ArrayList<>();
     private User user;
 
     // We don't use USE_PREPARED_VALUES in the code below so some test can foce value preparation (if the result
@@ -397,6 +432,16 @@ public static JMXServiceURL getJMXServiceURL() throws MalformedURLException
         return new JMXServiceURL(String.format("service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi", jmxHost, jmxPort));
     }
 
+    public static Randomization getRandom()
+    {
+        if (random == null)
+            random = new Randomization();
+        return random;
+    }
+
+    @Rule
+    public FailureWatcher failureRule = new FailureWatcher();
+
     @BeforeClass
     public static void setUpClass()
     {
@@ -411,6 +456,8 @@ protected static void prePrepareServer()
         CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS.setLong(0);
         ServerTestUtils.daemonInitialization();
 
+        DatabaseDescriptor.setAutoSnapshot(false);
+
         if (ROW_CACHE_SIZE_IN_MIB > 0)
             DatabaseDescriptor.setRowCacheSizeInMiB(ROW_CACHE_SIZE_IN_MIB);
         StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
@@ -451,15 +498,13 @@ public static void tearDownClass()
     @Before
     public void beforeTest() throws Throwable
     {
-        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", KEYSPACE));
-        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", KEYSPACE_PER_TEST));
+        Schema.instance.transform(schema -> schema.withAddedOrUpdated(KeyspaceMetadata.create(KEYSPACE_PER_TEST, KeyspaceParams.simple(1)))
+                                                  .withAddedOrUpdated(KeyspaceMetadata.create(KEYSPACE, KeyspaceParams.simple(1))), false);
     }
 
     @After
     public void afterTest() throws Throwable
     {
-        dropPerTestKeyspace();
-
         // Restore standard behavior in case it was changed
         usePrepared = USE_PREPARED_VALUES;
         reusePrepared = REUSE_PREPARED;
@@ -467,9 +512,6 @@ public void afterTest() throws Throwable
         final List<String> keyspacesToDrop = copy(keyspaces);
         final List<String> tablesToDrop = copy(tables);
         final List<String> viewsToDrop = copy(views);
-        final List<String> typesToDrop = copy(types);
-        final List<String> functionsToDrop = copy(functions);
-        final List<String> aggregatesToDrop = copy(aggregates);
         keyspaces = null;
         tables = null;
         views = null;
@@ -478,54 +520,16 @@ public void afterTest() throws Throwable
         aggregates = null;
         user = null;
 
-        // We want to clean up after the test, but dropping a table is rather long so just do that asynchronously
-        ScheduledExecutors.optionalTasks.execute(new Runnable()
+        try
         {
-            public void run()
-            {
-                try
-                {
-                    for (int i = viewsToDrop.size() - 1; i >= 0; i--)
-                        schemaChange(String.format("DROP MATERIALIZED VIEW IF EXISTS %s.%s", KEYSPACE, viewsToDrop.get(i)));
-
-                    for (int i = tablesToDrop.size() - 1; i >= 0; i--)
-                        schemaChange(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, tablesToDrop.get(i)));
-
-                    for (int i = aggregatesToDrop.size() - 1; i >= 0; i--)
-                        schemaChange(String.format("DROP AGGREGATE IF EXISTS %s", aggregatesToDrop.get(i)));
-
-                    for (int i = functionsToDrop.size() - 1; i >= 0; i--)
-                        schemaChange(String.format("DROP FUNCTION IF EXISTS %s", functionsToDrop.get(i)));
-
-                    for (int i = typesToDrop.size() - 1; i >= 0; i--)
-                        schemaChange(String.format("DROP TYPE IF EXISTS %s.%s", KEYSPACE, typesToDrop.get(i)));
-
-                    for (int i = keyspacesToDrop.size() - 1; i >= 0; i--)
-                        schemaChange(String.format("DROP KEYSPACE IF EXISTS %s", keyspacesToDrop.get(i)));
-
-                    // Dropping doesn't delete the sstables. It's not a huge deal but it's cleaner to cleanup after us
-                    // Thas said, we shouldn't delete blindly before the TransactionLogs.SSTableTidier for the table we drop
-                    // have run or they will be unhappy. Since those taks are scheduled on StorageService.tasks and that's
-                    // mono-threaded, just push a task on the queue to find when it's empty. No perfect but good enough.
-
-                    final CountDownLatch latch = new CountDownLatch(1);
-                    ScheduledExecutors.nonPeriodicTasks.execute(new Runnable()
-                    {
-                        public void run()
-                        {
-                            latch.countDown();
-                        }
-                    });
-                    latch.await(2, TimeUnit.SECONDS);
-
-                    removeAllSSTables(KEYSPACE, tablesToDrop);
-                }
-                catch (Exception e)
-                {
-                    throw new RuntimeException(e);
-                }
-            }
-        });
+            Schema.instance.transform(schema -> schema.without(List.of(KEYSPACE_PER_TEST))
+                                                      .withAddedOrUpdated(KeyspaceMetadata.create(KEYSPACE, KeyspaceParams.simple(1)))
+                                                      .without(keyspacesToDrop), false);
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
     }
 
     protected void resetSchema() throws Throwable
@@ -536,6 +540,14 @@ protected void resetSchema() throws Throwable
         beforeTest();
     }
 
+    /**
+     * Blocks until the previous schema cleanup task finished.
+     */
+    public void waitForSchemaCleanupCompleted(long timeout, TimeUnit unit)
+    {
+        Awaitility.await().atMost(timeout, unit).until(() -> schemaCleanup.getActiveCount() == 0);
+    }
+
     public static List<String> buildNodetoolArgs(List<String> args)
     {
         int port = jmxPort == 0 ? CASSANDRA_JMX_LOCAL_PORT.getInt(7199) : jmxPort;
@@ -574,6 +586,15 @@ public static List<String> buildCassandraStressArgs(List<String> args)
         return allArgs;
     }
 
+    protected static void requireNetworkWithoutDriver()
+    {
+        if (server != null)
+            return;
+
+        startServices();
+        startServer(server -> {});
+    }
+
     protected static void requireAuthentication()
     {
         DatabaseDescriptor.setAuthenticator(new AuthTestUtils.LocalPasswordAuthenticator());
@@ -671,7 +692,7 @@ private static void startServer(Consumer<Server.Builder> decorator)
         server.start();
     }
 
-    private static Cluster initClientCluster(User user, ProtocolVersion version)
+    private Cluster initClientCluster(User user, ProtocolVersion version)
     {
         SocketOptions socketOptions =
                 new SocketOptions().setConnectTimeoutMillis(TEST_DRIVER_CONNECTION_TIMEOUT_MS.getInt()) // default is 5000
@@ -685,7 +706,8 @@ private static Cluster initClientCluster(User user, ProtocolVersion version)
                                          .withClusterName("Test Cluster")
                                          .withPort(nativePort)
                                          .withSocketOptions(socketOptions)
-                                         .withNettyOptions(IMMEDIATE_CONNECTION_SHUTDOWN_NETTY_OPTIONS);
+                                         .withNettyOptions(IMMEDIATE_CONNECTION_SHUTDOWN_NETTY_OPTIONS)
+                                         .withPoolingOptions(new PoolingOptions().setPoolTimeoutMillis(10000));
 
         if (user != null)
             builder.withCredentials(user.username, user.password);
@@ -695,15 +717,44 @@ private static Cluster initClientCluster(User user, ProtocolVersion version)
         else
             builder = builder.withProtocolVersion(com.datastax.driver.core.ProtocolVersion.fromInt(version.asInt()));
 
-        clusterBuilderConfigurator.accept(builder);
+        if (clusterBuilderConfigurator != null)
+            clusterBuilderConfigurator.accept(builder);
 
         Cluster cluster = builder.build();
 
-        logger.info("Started Java Driver instance for protocol version {}", version);
+        logger.info("Started Java Driver session for {} with protocol version {}", user, version);
 
         return cluster;
     }
 
+    protected void closeClientCluster(String username, String password)
+    {
+        // Close driver cluster belonging to user
+        User user = new User(username, password);
+        for (ProtocolVersion protocolVersion : PROTOCOL_VERSIONS)
+        {
+            closeClientCluster(user, protocolVersion);
+        }
+    }
+
+    private void closeClientCluster(User user, ProtocolVersion protocolVersion)
+    {
+        Pair<User, ProtocolVersion> key = Pair.create(user, protocolVersion);
+        Session session = sessions.remove(key);
+        if (session != null)
+        {
+            session.close();
+        }
+
+        Cluster cluster = clusters.remove(key);
+        if (cluster != null)
+        {
+            cluster.close();
+        }
+
+        logger.info("Closed Java Driver session for {} with protocol version {}", user, protocolVersion);
+    }
+
     protected void dropPerTestKeyspace() throws Throwable
     {
         execute(String.format("DROP KEYSPACE IF EXISTS %s", KEYSPACE_PER_TEST));
@@ -736,6 +787,16 @@ public ColumnFamilyStore getColumnFamilyStore(String keyspace, String table)
         return Keyspace.open(keyspace).getColumnFamilyStore(table);
     }
 
+    public ColumnMetadata getColumn(String name)
+    {
+        return getCurrentColumnFamilyStore().metadata.get().getColumn(ColumnIdentifier.getInterned(name, true));
+    }
+
+    public ColumnMetadata getDroppedColumn(String name)
+    {
+        return getCurrentColumnFamilyStore().metadata.get().getDroppedColumn(ColumnIdentifier.getInterned(name, true).bytes);
+    }
+
     public void flush(boolean forceFlush)
     {
         if (forceFlush)
@@ -749,7 +810,12 @@ public void flush()
 
     public void flush(String keyspace)
     {
-        ColumnFamilyStore store = getCurrentColumnFamilyStore(keyspace);
+        flush(keyspace, currentTable());
+    }
+
+    public void flush(String keyspace, String table)
+    {
+        ColumnFamilyStore store = getColumnFamilyStore(keyspace, table);
         if (store != null)
             Util.flush(store);
     }
@@ -771,16 +837,19 @@ private List<ColumnFamilyStore> getTables(String keyspace, String[] tables)
 
     public void disableCompaction(String keyspace)
     {
-        ColumnFamilyStore store = getCurrentColumnFamilyStore(keyspace);
-        if (store != null)
-            store.disableAutoCompaction();
+        disableCompaction(keyspace, currentTable());
     }
 
     public void compact()
     {
-         ColumnFamilyStore store = getCurrentColumnFamilyStore();
-         if (store != null)
-             store.forceMajorCompaction();
+        compact(KEYSPACE, currentTable());
+    }
+
+    public void compact(String keyspace, String table)
+    {
+        ColumnFamilyStore store = getColumnFamilyStore(keyspace, table);
+        if (store != null)
+            store.forceMajorCompaction();
     }
 
     public void compact(String keyspace, String table1, String... tables)
@@ -887,6 +956,26 @@ protected String currentKeyspace()
         return keyspaces.get(keyspaces.size() - 1);
     }
 
+    protected String currentIndex()
+    {
+        if (indexes.isEmpty())
+            return null;
+        return indexes.get(indexes.size() - 1);
+    }
+
+    protected String getIndex(int i)
+    {
+        return indexes.get(i);
+    }
+
+    protected Collection<String> currentTables()
+    {
+        if (tables == null || tables.isEmpty())
+            return ImmutableList.of();
+
+        return new ArrayList<>(tables);
+    }
+
     protected ByteBuffer unset()
     {
         return ByteBufferUtil.UNSET_BYTE_BUFFER;
@@ -1024,7 +1113,7 @@ private int numberOfDigits(int i)
         return i == 0 ? 1 : (int) (Math.log10(i) + 1);
     }
 
-    protected String createTable(String query)
+    public String createTable(String query)
     {
         return createTable(KEYSPACE, query);
     }
@@ -1158,8 +1247,8 @@ protected static void waitForViewMutations()
                   .atMost(10, TimeUnit.MINUTES)
                   .pollDelay(0, TimeUnit.MILLISECONDS)
                   .pollInterval(1, TimeUnit.MILLISECONDS)
-                  .until(() -> Stage.VIEW_MUTATION.executor().getPendingTaskCount() == 0 &&
-                               Stage.VIEW_MUTATION.executor().getActiveTaskCount() == 0);
+                  .until(() -> Stage.VIEW_MUTATION.getPendingTaskCount() == 0 &&
+                               Stage.VIEW_MUTATION.getActiveTaskCount() == 0);
     }
 
     /**
@@ -1212,7 +1301,7 @@ private void dropFormattedTable(String formattedQuery)
      * @param query the index creation query
      * @return the name of the created index
      */
-    protected String createIndex(String query)
+    public String createIndex(String query)
     {
         return createIndex(KEYSPACE, query);
     }
@@ -1260,6 +1349,7 @@ private Pair<String, String> createFormattedIndex(String keyspace, String format
     {
         logger.info(formattedQuery);
         Pair<String, String> qualifiedIndexName = getCreateIndexName(keyspace, formattedQuery);
+        indexes.add(qualifiedIndexName.right);
         schemaChange(formattedQuery);
         return qualifiedIndexName;
     }
@@ -1275,6 +1365,7 @@ protected static Pair<String, String> getCreateIndexName(String keyspace, String
             keyspace = parsedKeyspace;
 
         String index = matcher.group(2);
+        boolean isQuotedGeneratedIndexName = false;
         if (Strings.isNullOrEmpty(index))
         {
             String table = matcher.group(7);
@@ -1282,31 +1373,41 @@ protected static Pair<String, String> getCreateIndexName(String keyspace, String
                 throw new IllegalArgumentException("Table name should be specified: " + formattedQuery);
 
             String column = matcher.group(9);
+            isQuotedGeneratedIndexName = ParseUtils.isQuoted(column, '\"');
 
             String baseName = Strings.isNullOrEmpty(column)
-                              ? IndexMetadata.generateDefaultIndexName(table)
+                              ? IndexMetadata.generateDefaultIndexName(table, null)
                               : IndexMetadata.generateDefaultIndexName(table, new ColumnIdentifier(column, true));
 
             KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(keyspace);
             assertNotNull(ks);
             index = ks.findAvailableIndexName(baseName);
         }
-
         index = ParseUtils.isQuoted(index, '\"')
                 ? ParseUtils.unDoubleQuote(index)
-                : index.toLowerCase();
+                : isQuotedGeneratedIndexName ? index : index.toLowerCase();
 
         return Pair.create(keyspace, index);
     }
 
     public void waitForTableIndexesQueryable()
     {
-        waitForTableIndexesQueryable(currentTable());
+        waitForTableIndexesQueryable(60);
     }
 
-    public void waitForTableIndexesQueryable(String table)
+    public void waitForTableIndexesQueryable(int seconds)
     {
-        waitForTableIndexesQueryable(KEYSPACE, table);
+        waitForTableIndexesQueryable(currentTable(), seconds);
+    }
+
+    public void waitForTableIndexesQueryable(String table, int seconds)
+    {
+        waitForTableIndexesQueryable(KEYSPACE, table, seconds);
+    }
+
+    public void waitForTableIndexesQueryable(String keyspace, String table)
+    {
+        waitForTableIndexesQueryable(keyspace, table, 60);
     }
 
     /**
@@ -1314,10 +1415,11 @@ public void waitForTableIndexesQueryable(String table)
      *
      * @param keyspace the table keyspace name
      * @param table the table name
+     * @param seconds the maximum time to wait for the indexes to be queryable
      */
-    public void waitForTableIndexesQueryable(String keyspace, String table)
+    public void waitForTableIndexesQueryable(String keyspace, String table, int seconds)
     {
-        waitForAssert(() -> Assertions.assertThat(getNotQueryableIndexes(keyspace, table)).isEmpty(), 60, TimeUnit.SECONDS);
+        waitForAssert(() -> Assertions.assertThat(getNotQueryableIndexes(keyspace, table)).isEmpty(), seconds, TimeUnit.SECONDS);
     }
 
     public void waitForIndexQueryable(String index)
@@ -1333,7 +1435,20 @@ public void waitForIndexQueryable(String index)
      */
     public void waitForIndexQueryable(String keyspace, String index)
     {
-        waitForAssert(() -> assertTrue(isIndexQueryable(keyspace, index)), 60, TimeUnit.SECONDS);
+        waitForIndexQueryable(keyspace, index, 1, TimeUnit.MINUTES);
+    }
+
+    /**
+     * Index creation is asynchronous. This method waits until the specified index is queryable.
+     *
+     * @param keyspace the index keyspace name
+     * @param index the index name
+     * @param timeout the timeout
+     * @param unit the timeout unit
+     */
+    public void waitForIndexQueryable(String keyspace, String index, long timeout, TimeUnit unit)
+    {
+        waitForAssert(() -> assertTrue(isIndexQueryable(keyspace, index)), timeout, unit);
     }
 
     protected void waitForIndexBuilds(String index)
@@ -1395,6 +1510,34 @@ protected boolean isIndexQueryable(String keyspace, String indexName)
         return manager.isIndexQueryable(index);
     }
 
+    protected boolean areAllTableIndexesQueryable()
+    {
+        return areAllTableIndexesQueryable(KEYSPACE, currentTable());
+    }
+
+    protected boolean areAllTableIndexesQueryable(String keyspace, String table)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        for (Index index : cfs.indexManager.listIndexes())
+        {
+            if (!cfs.indexManager.isIndexQueryable(index))
+                return false;
+        }
+        return true;
+    }
+
+    protected boolean indexNeedsFullRebuild(String index)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        return cfs.indexManager.needsFullRebuild(index);
+    }
+
+    protected void verifyInitialIndexFailed(String indexName)
+    {
+        // Verify that the initial index build fails...
+        waitForAssert(() -> assertTrue(indexNeedsFullRebuild(indexName)));
+    }
+
     @Nullable
     protected SecondaryIndexManager getIndexManager(String keyspace, String indexName)
     {
@@ -1424,13 +1567,30 @@ protected void createIndexMayThrow(String query) throws Throwable
         QueryProcessor.executeOnceInternal(fullQuery);
     }
 
-    protected void dropIndex(String query) throws Throwable
+    public void dropIndex(String query)
     {
         String fullQuery = String.format(query, KEYSPACE);
         logger.info(fullQuery);
         schemaChange(fullQuery);
     }
 
+    /**
+     *  Because the tracing executor is single threaded, submitting an empty event should ensure
+     *  that all tracing events mutations have been applied.
+     */
+    protected void waitForTracingEvents()
+    {
+        try
+        {
+            Stage.TRACING.submit(() -> {}).get();
+        }
+        catch (Throwable t)
+        {
+            JVMStabilityInspector.inspectThrowable(t);
+            logger.error("Failed to wait for tracing events: {}", t);
+        }
+    }
+
     protected static void assertSchemaChange(String query,
                                              Event.SchemaChange.Change expectedChange,
                                              Event.SchemaChange.Target expectedTarget,
@@ -1501,8 +1661,8 @@ protected static ResultMessage schemaChange(String query)
         {
             logger.info("Error performing schema change", e);
             if (e instanceof InvalidRequestException)
-                throw new InvalidRequestException(String.format("Error setting schema for test (query was: %s)", query), e);
-            throw new RuntimeException("Error setting schema for test (query was: " + query + ")", e);
+                throw e;
+            throw new RuntimeException(String.format("Error setting schema for test (query was: %s)", query), e);
         }
     }
 
@@ -1531,6 +1691,11 @@ protected com.datastax.driver.core.ResultSet executeNet(ProtocolVersion protocol
         return sessionNet(protocolVersion).execute(statement);
     }
 
+    protected com.datastax.driver.core.ResultSet executeNet(Statement statement)
+    {
+        return sessionNet().execute(statement);
+    }
+
     protected com.datastax.driver.core.ResultSet executeNetWithPaging(ProtocolVersion version, String query, int pageSize, Object... values)
     {
         return sessionNet(version).execute(new SimpleStatement(formatQuery(query), values).setFetchSize(pageSize));
@@ -1551,7 +1716,7 @@ protected com.datastax.driver.core.ResultSet executeNetWithoutPaging(String quer
         return executeNetWithPaging(query, Integer.MAX_VALUE);
     }
 
-    protected Session sessionNet()
+    public Session sessionNet()
     {
         return sessionNet(getDefaultVersion());
     }
@@ -1580,7 +1745,7 @@ protected SimpleClient newSimpleClient(ProtocolVersion version) throws IOExcepti
                .connect(false, false);
     }
 
-    protected String formatQuery(String query)
+    public String formatQuery(String query)
     {
         return formatQuery(KEYSPACE, query);
     }
@@ -1602,14 +1767,93 @@ public String formatViewQuery(String keyspace, String query)
         return currentView == null ? query : String.format(query, keyspace + "." + currentView);
     }
 
+    protected CQLStatement parseStatement(String query)
+    {
+        String formattedQuery = formatQuery(query);
+        return QueryProcessor.parseStatement(formattedQuery, ClientState.forInternalCalls());
+    }
+
+    protected ReadCommand parseReadCommand(String query)
+    {
+        SelectStatement select = (SelectStatement) parseStatement(query);
+        return  (ReadCommand) select.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds());
+    }
+
     protected ResultMessage.Prepared prepare(String query) throws Throwable
     {
         return QueryProcessor.instance.prepare(formatQuery(query), ClientState.forInternalCalls());
     }
 
-    protected UntypedResultSet execute(String query, Object... values)
+    /**
+     * Enables coordinator execution in {@link #execute(String, Object...)}, so queries get full validation and go
+     * through reconciliation. This makes calling {@link #execute(String, Object...)} equivalent to calling
+     * {@link #executeWithCoordinator(String, Object...)}.
+     */
+    protected static void enableCoordinatorExecution()
+    {
+        requireNetworkWithoutDriver();
+        coordinatorExecution = true;
+    }
+
+    /**
+     * Disables coordinator execution in {@link #execute(String, Object...)}, so queries won't get full validation nor
+     * go through reconciliation.This makes calling {@link #execute(String, Object...)} equivalent to calling
+     * {@link #executeInternal(String, Object...)}.
+     */
+    protected static void disableCoordinatorExecution()
+    {
+        coordinatorExecution = false;
+    }
+
+    /**
+     * Execute the specified query as either an internal query or a coordinator query depending on the value of
+     * {@link #coordinatorExecution}.
+     *
+     * @param query a CQL query
+     * @param values the values to bind to the query
+     * @return the query results
+     * @see #execute
+     * @see #executeInternal
+     */
+    public UntypedResultSet execute(String query, Object... values)
+    {
+        return coordinatorExecution
+               ? executeWithCoordinator(query, values)
+               : executeInternal(query, values);
+    }
+
+    /**
+     * Execute the specified query as an internal query only for the local node. This will skip reconciliation and some
+     * validation.
+     * </p>
+     * For the particular case of {@code SELECT} queries using secondary indexes, the skipping of reconciliation means
+     * that the query {@link org.apache.cassandra.db.filter.RowFilter} might not be fully applied to the index results.
+     *
+     * @param query a CQL query
+     * @param values the values to bind to the query
+     * @return the query results
+     * @see CQLStatement#executeLocally
+     */
+    public UntypedResultSet executeInternal(String query, Object... values)
+    {
+        return executeFormattedQuery(formatQuery(query), false, values);
+    }
+
+    /**
+     * Execute the specified query as an coordinator-side query meant for all the relevant nodes in the cluster, even if
+     * {@link CQLTester} tests are single-node. This won't skip reconciliation and will do full validation.
+     * </p>
+     * For the particular case of {@code SELECT} queries using secondary indexes, applying reconciliation means that the
+     * query {@link org.apache.cassandra.db.filter.RowFilter} will be fully applied to the index results.
+     *
+     * @param query a CQL query
+     * @param values the values to bind to the query
+     * @return the query results
+     * @see CQLStatement#execute
+     */
+    public UntypedResultSet executeWithCoordinator(String query, Object... values)
     {
-        return executeFormattedQuery(formatQuery(query), values);
+        return executeFormattedQuery(formatQuery(query), true, values);
     }
 
     public UntypedResultSet executeView(String query, Object... values) throws Throwable
@@ -1623,14 +1867,27 @@ public UntypedResultSet executeView(String query, Object... values) throws Throw
      */
     public UntypedResultSet executeFormattedQuery(String query, Object... values)
     {
+        return executeFormattedQuery(query, coordinatorExecution, values);
+    }
+
+    private UntypedResultSet executeFormattedQuery(String query, boolean useCoordinator, Object... values)
+    {        
+        if (useCoordinator)
+            requireNetworkWithoutDriver();
+        
         UntypedResultSet rs;
         if (usePrepared)
         {
             if (logger.isTraceEnabled())
                 logger.trace("Executing: {} with values {}", query, formatAllValues(values));
+
+            Object[] transformedValues = transformValues(values);
+
             if (reusePrepared)
             {
-                rs = QueryProcessor.executeInternal(query, transformValues(values));
+                rs = useCoordinator
+                     ? QueryProcessor.execute(query, ConsistencyLevel.ONE, transformedValues)
+                     : QueryProcessor.executeInternal(query, transformedValues);
 
                 // If a test uses a "USE ...", then presumably its statements use relative table. In that case, a USE
                 // change the meaning of the current keyspace, so we don't want a following statement to reuse a previously
@@ -1641,15 +1898,21 @@ public UntypedResultSet executeFormattedQuery(String query, Object... values)
             }
             else
             {
-                rs = QueryProcessor.executeOnceInternal(query, transformValues(values));
+                rs = useCoordinator
+                     ? QueryProcessor.executeOnce(query, ConsistencyLevel.ONE, transformedValues)
+                     : QueryProcessor.executeOnceInternal(query, transformedValues);
             }
         }
         else
         {
             query = replaceValues(query, values);
+
             if (logger.isTraceEnabled())
                 logger.trace("Executing: {}", query);
-            rs = QueryProcessor.executeOnceInternal(query);
+
+            rs = useCoordinator
+                 ? QueryProcessor.executeOnce(query, ConsistencyLevel.ONE)
+                 : QueryProcessor.executeOnceInternal(query);
         }
         if (rs != null)
         {
@@ -1815,6 +2078,24 @@ protected void assertRowCountNet(ResultSet r1, int expectedCount)
 
     public static void assertRows(UntypedResultSet result, Object[]... rows)
     {
+        assertRows(result, false, rows);
+    }
+
+    public static void assertRows(UntypedResultSet result, boolean printAssertedRows, Object[]... rows)
+    {
+        // Useful for manual debugging, but generally unnecessary on
+        if (printAssertedRows)
+        {
+            // Print all the rows
+            for (Object[] row : rows)
+            {
+                System.out.print("Expected row:");
+                for (Object column : row)
+                    System.out.print("  " + column);
+                System.out.println();
+            }
+        }
+
         if (result == null)
         {
             if (rows.length > 0)
@@ -1827,10 +2108,13 @@ public static void assertRows(UntypedResultSet result, Object[]... rows)
         int i = 0;
         while (iter.hasNext() && i < rows.length)
         {
+            if (rows[i] == null)
+                throw new IllegalArgumentException(String.format("Invalid expected value for row: %d. A row cannot be null.", i));
+
             Object[] expected = rows[i];
             UntypedResultSet.Row actual = iter.next();
 
-            Assert.assertEquals(String.format("Invalid number of (expected) values provided for row %d", i), expected == null ? 1 : expected.length, meta.size());
+            Assert.assertEquals(String.format("Invalid number of (expected) values provided for row %d", i), expected.length, meta.size());
 
             StringBuilder error = new StringBuilder();
             for (int j = 0; j < meta.size(); j++)
@@ -2110,18 +2394,18 @@ public static Object[][] rows(Object[]... rows)
         return rows;
     }
 
-    protected void assertEmpty(UntypedResultSet result) throws Throwable
+    protected void assertEmpty(UntypedResultSet result)
     {
         if (result != null && !result.isEmpty())
             throw new AssertionError(String.format("Expected empty result but got %d rows: %s \n", result.size(), makeRowStrings(result)));
     }
 
-    protected void assertInvalid(String query, Object... values) throws Throwable
+    protected void assertInvalid(String query, Object... values)
     {
         assertInvalidMessage(null, query, values);
     }
 
-    protected void assertInvalidMessage(String errorMessage, String query, Object... values) throws Throwable
+    protected void assertInvalidMessage(String errorMessage, String query, Object... values)
     {
         assertInvalidThrowMessage(errorMessage, null, query, values);
     }
@@ -2131,12 +2415,12 @@ protected void assertInvalidMessageNet(String errorMessage, String query, Object
         assertInvalidThrowMessage(Optional.of(ProtocolVersion.CURRENT), errorMessage, null, query, values);
     }
 
-    protected void assertInvalidThrow(Class<? extends Throwable> exception, String query, Object... values) throws Throwable
+    protected void assertInvalidThrow(Class<? extends Throwable> exception, String query, Object... values)
     {
         assertInvalidThrowMessage(null, exception, query, values);
     }
 
-    protected void assertInvalidThrowMessage(String errorMessage, Class<? extends Throwable> exception, String query, Object... values) throws Throwable
+    protected void assertInvalidThrowMessage(String errorMessage, Class<? extends Throwable> exception, String query, Object... values)
     {
         assertInvalidThrowMessage(Optional.empty(), errorMessage, exception, query, values);
     }
@@ -2154,7 +2438,7 @@ protected void assertInvalidThrowMessage(Optional<ProtocolVersion> protocolVersi
                                              String errorMessage,
                                              Class<? extends Throwable> exception,
                                              String query,
-                                             Object... values) throws Throwable
+                                             Object... values)
     {
         try
         {
@@ -2183,6 +2467,13 @@ protected void assertInvalidThrowMessage(Optional<ProtocolVersion> protocolVersi
         }
     }
 
+    public static List<String> warningsFromResultSet(List<String> ignoredWarnings, ResultSet rs)
+    {
+        return rs.getExecutionInfo().getWarnings()
+                 .stream().filter(w -> ignoredWarnings.stream().noneMatch(w::contains))
+                 .collect(Collectors.toList());
+    }
+
     private static String queryInfo(String query, Object[] values)
     {
         return USE_PREPARED_VALUES
@@ -2239,8 +2530,7 @@ protected void assertInvalidRequestMessage(String errorMessage, String query, Ob
      */
     private static void assertMessageContains(String text, Exception e)
     {
-        Assert.assertTrue("Expected error message to contain '" + text + "', but got '" + e.getMessage() + "'",
-                e.getMessage().contains(text));
+        Assertions.assertThat(e.getMessage()).contains(text);
     }
 
     /**
@@ -2271,9 +2561,48 @@ public interface CheckedFunction {
      */
     public void beforeAndAfterFlush(CheckedFunction runnable) throws Throwable
     {
-        runnable.apply();
+        try
+        {
+            runnable.apply();
+        }
+        catch (Throwable t)
+        {
+            throw new AssertionError("Test failed before flush:\n" + t, t);
+        }
+
         flush();
-        runnable.apply();
+
+        try
+        {
+            runnable.apply();
+        }
+        catch (Throwable t)
+        {
+            throw new AssertionError("Test failed after flush:\n" + t, t);
+        }
+    }
+
+    /**
+     * Runs the given function before a flush, after a flush, and finally after a compaction of the table. This is
+     * useful for checking that behavior is the same whether data is in memtables, memtable-flushed-sstbales,
+     * compaction-built-sstables.
+     * @param runnable
+     * @throws Throwable
+     */
+    public void runThenFlushThenCompact(CheckedFunction runnable) throws Throwable
+    {
+        beforeAndAfterFlush(runnable);
+
+        compact();
+
+        try
+        {
+            runnable.apply();
+        }
+        catch (Throwable t)
+        {
+            throw new AssertionError("Test failed after compact:\n" + t, t);
+        }
     }
 
     private static String replaceValues(String query, Object[] values)
@@ -2540,18 +2869,45 @@ protected List<Object> list(Object...values)
         return Arrays.asList(values);
     }
 
-    @SafeVarargs
-    protected final <T> Vector<T> vector(T... values)
+    /** @return a normalized vector with the given dimension */
+    public static Vector<Float> randomVectorBoxed(int dimension)
     {
-        return new Vector<>(values);
+        var floats = randomVector(dimension);
+        return vector(floats);
     }
 
-    protected Vector<Float> vector(float[] v)
+    public static ByteBuffer randomVectorSerialized(int dimension)
     {
-        var v2 = new Float[v.length];
+        var rawVector = randomVectorBoxed(dimension);
+        return VectorType.getInstance(FloatType.instance, dimension).getSerializer().serialize(rawVector);
+    }
+
+    public static float[] randomVector(int dimension)
+    {
+        // this can be called from concurrent threads so don't use getRandom()
+        var R = ThreadLocalRandom.current();
+
+        var vector = new float[dimension];
+        for (int i = 0; i < dimension; i++)
+        {
+            vector[i] = R.nextFloat();
+        }
+        normalize(vector);
+        return vector;
+    }
+
+    /** Normalize the given vector in-place */
+    protected static void normalize(float[] v)
+    {
+        var sum = 0.0f;
+        for (int i = 0; i < v.length; i++)
+        {
+            sum += v[i] * v[i];
+        }
+
+        sum = (float) Math.sqrt(sum);
         for (int i = 0; i < v.length; i++)
-            v2[i] = v[i];
-        return new Vector<>(v2);
+            v[i] /= sum;
     }
 
     protected Set<Object> set(Object...values)
@@ -2608,6 +2964,7 @@ private String getTestMethodName()
                                                                             : "";
     }
 
+    @Ignore // Check TinySegmentFlushingFailureTest for details why this annotation is needed here despite this is not a test
     public static class Vector<T> extends AbstractList<T>
     {
         private final T[] values;
@@ -2859,6 +3216,11 @@ public boolean equals(Object o)
             return Objects.equal(username, u.username)
                 && Objects.equal(password, u.password);
         }
+
+        public String toString()
+        {
+            return username;
+        }
     }
 
     public static abstract class InMemory extends CQLTester
@@ -2888,4 +3250,112 @@ public void cleanupFileSystemListeners()
             fs.clearListeners();
         }
     }
+
+    @Ignore // Check TinySegmentFlushingFailureTest for details why this annotation is needed here despite this is not a test
+    public static class Randomization
+    {
+        private long seed;
+        private Random random;
+
+        Randomization()
+        {
+            if (random == null)
+            {
+                seed = TEST_RANDOM_SEED.getLong(nanoTime());
+                random = new Random(seed);
+            }
+        }
+
+        public void printSeedOnFailure()
+        {
+            System.err.println("Randomized test failed. To rerun test use -Dcassandra.test.random.seed=" + seed);
+        }
+
+        public Random getRandom()
+        {
+            return random;
+        }
+
+        public int nextInt()
+        {
+            return random.nextInt();
+        }
+
+        public int nextIntBetween(int minValue, int maxValue)
+        {
+            return RandomInts.randomIntBetween(random, minValue, maxValue);
+        }
+
+        public float nextFloatBetween(int minValue, int maxValue)
+        {
+            return random.nextFloat() * (maxValue - minValue) + minValue;
+        }
+
+        public long nextLong()
+        {
+            return random.nextLong();
+        }
+
+        public short nextShort()
+        {
+            return (short)random.nextInt(Short.MAX_VALUE + 1);
+        }
+
+        public byte nextByte()
+        {
+            return (byte)random.nextInt(Byte.MAX_VALUE + 1);
+        }
+
+        public BigInteger nextBigInteger(int minNumBits, int maxNumBits)
+        {
+            return new BigInteger(RandomInts.randomIntBetween(random, minNumBits, maxNumBits), random);
+        }
+
+        public BigDecimal nextBigDecimal(int minUnscaledValue, int maxUnscaledValue, int minScale, int maxScale)
+        {
+            return BigDecimal.valueOf(RandomInts.randomIntBetween(random, minUnscaledValue, maxUnscaledValue),
+                                      RandomInts.randomIntBetween(random, minScale, maxScale));
+        }
+
+        public float nextFloat()
+        {
+            return random.nextFloat();
+        }
+
+        public double nextDouble()
+        {
+            return random.nextDouble();
+        }
+
+        public String nextAsciiString(int minLength, int maxLength)
+        {
+            return RandomStrings.randomAsciiOfLengthBetween(random, minLength, maxLength);
+        }
+
+        public String nextTextString(int minLength, int maxLength)
+        {
+            return RandomStrings.randomRealisticUnicodeOfLengthBetween(random, minLength, maxLength);
+        }
+
+        public boolean nextBoolean()
+        {
+            return random.nextBoolean();
+        }
+
+        public void nextBytes(byte[] bytes)
+        {
+            random.nextBytes(bytes);
+        }
+    }
+
+    @Ignore // Check TinySegmentFlushingFailureTest for details why this annotation is needed here despite this is not a test
+    public static class FailureWatcher extends TestWatcher
+    {
+        @Override
+        protected void failed(Throwable e, Description description)
+        {
+            if (random != null)
+                random.printSeedOnFailure();
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/CompactionOutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/CompactionOutOfSpaceTest.java
new file mode 100644
index 000000000000..cb98bcd27210
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/CompactionOutOfSpaceTest.java
@@ -0,0 +1,314 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.service.CassandraDaemon;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMKiller;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+import org.assertj.core.api.Assertions;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+@RunWith(BMUnitRunner.class)
+public class CompactionOutOfSpaceTest extends CQLTester
+{
+    @BeforeClass
+    public static void setupClass()
+    {
+        CassandraDaemon d = new CassandraDaemon();
+        d.activate();
+
+        // these were moved after CassandraDaemon::activate to avoid
+        // race condition between compaction (triggered by setUpClass)
+        // and checkSSTablesFormat StartupCheck (and possible others that
+        // traverse the filesystem). See STAR-1294 for more info.
+        // If it turns out that CQLTester initialization must be run
+        // before activate() then perhaps we should try disabling
+        // compactions as the first step and enabling them only after activate();
+        // To disable compactions one may use ColumnFamilyStore::disableAutoCompaction
+        // (see also how compactions are being temporarily disabled in CassandraDaemon::setup
+        CQLTester.setUpClass();
+        CQLTester.requireNetwork();
+    }
+
+    @AfterClass
+    public static void tearDownClass()
+    {
+        StorageService.instance.registerDaemon(null);
+    }
+
+    @Before
+    public void setup()
+    {
+        // restart the services in case a previous test has stopped them
+
+        if (!StorageService.instance.isNativeTransportRunning())
+            StorageService.instance.startNativeTransport();
+
+        if (!StorageService.instance.isGossipActive())
+            StorageService.instance.startGossiping();
+    }
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOError(new java.io.IOException(\"No space left on device\"))")
+    public void testUcsBackgroundCompactionNoDiskSpaceIgnore() throws Throwable
+    {
+        String ucsCqlCompactionParams = "{'class':'UnifiedCompactionStrategy', 'num_shards':'1'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.ignore, ucsCqlCompactionParams, "No space left on device");
+    }
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction with only IOException",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOException(\"No space left on device\")")
+    public void testUcsBackgroundCompactionNoDiskSpaceIOExceptionIgnore() throws Throwable
+    {
+        String ucsCqlCompactionParams = "{'class':'UnifiedCompactionStrategy', 'num_shards':'1'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.ignore, ucsCqlCompactionParams, "No space left on device");
+    }
+
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOError(new java.io.IOException(\"No space left on device\"))")
+    public void testUcsBackgroundCompactionNoDiskSpaceStop() throws Throwable
+    {
+        String ucsCqlCompactionParams = "{'class':'UnifiedCompactionStrategy', 'num_shards':'1'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.stop, ucsCqlCompactionParams, "No space left on device");
+    }
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction with only IOException",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOException(\"No space left on device\")")
+    public void testUcsBackgroundCompactionNoDiskSpaceIOExceptionStop() throws Throwable
+    {
+        String ucsCqlCompactionParams = "{'class':'UnifiedCompactionStrategy', 'num_shards':'1'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.stop, ucsCqlCompactionParams, "No space left on device");
+    }
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOError(new java.io.IOException(\"No space left on device\"))")
+    public void testUcsBackgroundCompactionNoDiskSpaceDie() throws Throwable
+    {
+        String ucsCqlCompactionParams = "{'class':'UnifiedCompactionStrategy', 'num_shards':'1'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.die, ucsCqlCompactionParams, "No space left on device");
+    }
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction with only IOException",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOException(\"No space left on device\")")
+    public void testUcsBackgroundCompactionNoDiskSpacIOExceptioneDie() throws Throwable
+    {
+        String ucsCqlCompactionParams = "{'class':'UnifiedCompactionStrategy', 'num_shards':'1'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.die, ucsCqlCompactionParams, "No space left on device");
+    }
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOError(new java.io.IOException(\"No space left on device\"))")
+    public void testStcsBackgroundCompactionNoDiskSpaceIgnore() throws Throwable
+    {
+        String stcsCqlCompactionParams = "{'class':'SizeTieredCompactionStrategy', 'max_threshold':'4'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.ignore, stcsCqlCompactionParams, "No space left on device");
+    }
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOError(new java.io.IOException(\"No space left on device\"))")
+    public void testStcsBackgroundCompactionNoDiskSpaceStop() throws Throwable
+    {
+        String stcsCqlCompactionParams = "{'class':'SizeTieredCompactionStrategy', 'max_threshold':'4'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.stop, stcsCqlCompactionParams, "No space left on device");
+    }
+
+    @Test
+    @BMRule(name = "Simulate disk full during background compaction",
+    targetClass = "CompactionTask",
+    targetMethod = "runMayThrow",
+    targetLocation = "AT ENTRY",
+    condition = "org.apache.cassandra.cql3.CompactionOutOfSpaceTest.isKillerForTestsInstalled()",
+    action = "throw new java.io.IOError(new java.io.IOException(\"No space left on device\"))")
+    public void testStcsBackgroundCompactionNoDiskSpaceDie() throws Throwable
+    {
+        String stcsCqlCompactionParams = "{'class':'SizeTieredCompactionStrategy', 'max_threshold':'4'}";
+        flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy.die, stcsCqlCompactionParams, "No space left on device");
+    }
+
+    private void flush4SstablesAndEnableAutoCompaction(Config.DiskFailurePolicy policy, String cqlCompactionParams, String msg) throws Throwable
+    {
+        createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c)) WITH compaction = " + cqlCompactionParams);
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.disableAutoCompaction();
+
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 1, 1);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 2, 2);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 3, 3);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 4, 4);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(new KillerForTests());
+        Config.DiskFailurePolicy originalPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(policy);
+            Assertions.assertThatExceptionOfType(Exception.class).isThrownBy(() -> cfs.enableAutoCompaction(true))
+                      .withStackTraceContaining(msg);
+            verifyDiskFailurePolicy(policy);
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(originalPolicy);
+            JVMStabilityInspector.replaceKiller(originalKiller);
+        }
+    }
+
+    private void verifyDiskFailurePolicy(Config.DiskFailurePolicy policy)
+    {
+        switch (policy)
+        {
+            case stop:
+            case stop_paranoid:
+                verifyDiskFailurePolicyStop();
+                break;
+            case die:
+                verifyDiskFailurePolicyDie();
+                break;
+            case best_effort:
+                verifyDiskFailurePolicyBestEffort();
+                break;
+            case ignore:
+                verifyDiskFailurePolicyIgnore();
+                break;
+            default:
+                fail("Unsupported disk failure policy: " + policy);
+                break;
+        }
+    }
+
+    private void verifyDiskFailurePolicyStop()
+    {
+        verifyGossip(false);
+        verifyNativeTransports(false);
+        verifyJVMWasKilled(false);
+    }
+
+    private void verifyDiskFailurePolicyDie()
+    {
+        verifyJVMWasKilled(true);
+    }
+
+    private void verifyDiskFailurePolicyBestEffort()
+    {
+        assertFalse(Util.getDirectoriesWriteable(getCurrentColumnFamilyStore(KEYSPACE_PER_TEST)));
+        FBUtilities.sleepQuietly(10); // give them a chance to stop before verifying they were not stopped
+        verifyGossip(true);
+        verifyNativeTransports(true);
+        verifyJVMWasKilled(false);
+    }
+
+    private void verifyDiskFailurePolicyIgnore()
+    {
+        FBUtilities.sleepQuietly(10); // give them a chance to stop before verifying they were not stopped
+        verifyGossip(true);
+        verifyNativeTransports(true);
+        verifyJVMWasKilled(false);
+    }
+
+    private void verifyJVMWasKilled(boolean killed)
+    {
+        KillerForTests killer = (KillerForTests) JVMStabilityInspector.killer();
+        assertEquals(killed, killer.wasKilled());
+        if (killed)
+            assertFalse(killer.wasKilledQuietly()); // true only on startup
+    }
+
+    private void verifyGossip(boolean isEnabled)
+    {
+        assertEquals(isEnabled, Gossiper.instance.isEnabled());
+    }
+
+    private void verifyNativeTransports(boolean isRunning)
+    {
+        // Native transports are also stopped asynchronously, but isRunning is set synchronously
+        assertEquals(isRunning, StorageService.instance.isNativeTransportRunning());
+
+        // if the transport has been stopped, we wait for it to be fully stopped so that restarting it for
+        // the next test will not fail due to the port being already in use
+        if (!isRunning)
+            StorageService.instance.stopNativeTransport();
+    }
+
+    public static boolean isKillerForTestsInstalled()
+    {
+        logger.info("Checking if killer for tests is installed: {}", JVMStabilityInspector.killer().getClass().getName());
+        return JVMStabilityInspector.killer().getClass().getName().contains("KillerForTests");
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java b/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java
index dd18ac78648e..f974f50b8e49 100644
--- a/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java
@@ -204,7 +204,7 @@ public void testBatchMessage()
             statements.add((ModificationStatement) QueryProcessor.parseStatement(query, cs));
 
         BatchStatement batch =
-            new BatchStatement(BatchStatement.Type.UNLOGGED, VariableSpecifications.empty(), statements, Attributes.none());
+            new BatchStatement(null, BatchStatement.Type.UNLOGGED, VariableSpecifications.empty(), statements, Attributes.none());
 
         // execute an BATCH message with now set to [now + 1 day], with ttl = 1, making its effective ttl = 1 day + 1.
         QueryProcessor.instance.processBatch(batch, qs, batchQueryOptions(now + day), emptyMap(), Dispatcher.RequestTime.forImmediateExecution());
@@ -250,7 +250,7 @@ private static QueryOptions queryOptions(long nowInSeconds)
         return QueryOptions.create(ConsistencyLevel.ONE,
                                    Collections.emptyList(),
                                    false,
-                                   Integer.MAX_VALUE,
+                                   PageSize.NONE,
                                    null,
                                    null,
                                    ProtocolVersion.CURRENT,
diff --git a/test/unit/org/apache/cassandra/cql3/EmptyValuesTest.java b/test/unit/org/apache/cassandra/cql3/EmptyValuesTest.java
index babecffed2d6..a8496d07fac6 100644
--- a/test/unit/org/apache/cassandra/cql3/EmptyValuesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/EmptyValuesTest.java
@@ -80,6 +80,8 @@ private void verify(String emptyValue) throws Throwable
                 pb.redirectErrorStream(true);
                 if (CassandraRelevantProperties.CASSANDRA_CONFIG.isPresent())
                     pb.environment().put("JVM_OPTS", "-Dcassandra.config=" + CassandraRelevantProperties.CASSANDRA_CONFIG.getString());
+                String jvmOpts = pb.environment().getOrDefault("JVM_OPTS", "") + " -Dcassandra.disable_tcactive_openssl=true";
+                pb.environment().put("JVM_OPTS", jvmOpts);
                 Process process = pb.start();
                 exitValue = process.waitFor();
                 IOUtils.copy(process.getInputStream(), buf);
diff --git a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
index 31b303471e9d..06b79b495e2e 100644
--- a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
@@ -49,13 +49,13 @@ public class GcCompactionTest extends CQLTester
     // Test needs synchronous table drop to avoid flushes causing flaky failures
 
     @Override
-    protected String createTable(String query)
+    public String createTable(String query)
     {
         return super.createTable(KEYSPACE_PER_TEST, query);
     }
 
     @Override
-    protected UntypedResultSet execute(String query, Object... values)
+    public UntypedResultSet execute(String query, Object... values)
     {
         return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
     }
@@ -204,7 +204,7 @@ public void testGarbageCollectRetainsLCSLevel() throws Throwable
                   "  PRIMARY KEY ((key), column)" +
                   ") WITH compaction = { 'class' : 'LeveledCompactionStrategy' };");
 
-      assertEquals("LeveledCompactionStrategy", getCurrentColumnFamilyStore().getCompactionStrategyManager().getName());
+      assertEquals("LeveledCompactionStrategy", getCurrentColumnFamilyStore().getCompactionStrategyContainer().getName());
 
       for (int i = 0; i < KEY_COUNT; ++i)
           for (int j = 0; j < CLUSTERING_COUNT; ++j)
diff --git a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
index 3f5f75abd8df..7353da42af58 100644
--- a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
+++ b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
@@ -39,7 +39,7 @@
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaKeyspace;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.service.StorageService;
@@ -113,25 +113,25 @@ public static void setUpClass()
      * that we can assert on the key cache size and metrics.
      */
     @Override
-    protected String createTable(String query)
+    public String createTable(String query)
     {
         return super.createTable(KEYSPACE_PER_TEST, query + " WITH caching = { 'keys' : 'ALL', 'rows_per_partition' : '0' }");
     }
 
     @Override
-    protected UntypedResultSet execute(String query, Object... values)
+    public UntypedResultSet execute(String query, Object... values)
     {
         return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
     }
 
     @Override
-    protected String createIndex(String query)
+    public String createIndex(String query)
     {
         return createIndex(KEYSPACE_PER_TEST, query);
     }
 
     @Override
-    protected void dropTable(String query)
+    public void dropTable(String query)
     {
         dropTable(KEYSPACE_PER_TEST, query);
     }
@@ -139,14 +139,14 @@ protected void dropTable(String query)
     @Test
     public void testSliceQueriesShallowIndexEntry() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         testSliceQueries();
     }
 
     @Test
     public void testSliceQueriesIndexInfoOnHeap() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         testSliceQueries();
     }
 
@@ -236,14 +236,14 @@ private static String makeStringValue(String pk, int ck1, int ck2)
     @Test
     public void test2iKeyCachePathsShallowIndexEntry() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         test2iKeyCachePaths();
     }
 
     @Test
     public void test2iKeyCachePathsIndexInfoOnHeap() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         test2iKeyCachePaths();
     }
 
@@ -269,8 +269,8 @@ private void test2iKeyCachePaths() throws Throwable
             expectedRequests += recentBloomFilterFalsePositives() + 20;
         }
 
-        long hits = metrics.hits.getCount();
-        long requests = metrics.requests.getCount();
+        long hits = metrics.hits();
+        long requests = metrics.requests();
         assertEquals(0, hits);
         assertEquals(sstableImplCachesKeys ? expectedRequests : 0, requests);
 
@@ -287,8 +287,8 @@ private void test2iKeyCachePaths() throws Throwable
         }
 
         metrics = CacheService.instance.keyCache.getMetrics();
-        hits = metrics.hits.getCount();
-        requests = metrics.requests.getCount();
+        hits = metrics.hits();
+        requests = metrics.requests();
         assertEquals(sstableImplCachesKeys ? 200 : 0, hits);
         assertEquals(sstableImplCachesKeys ? expectedRequests : 0, requests);
 
@@ -315,7 +315,7 @@ private void test2iKeyCachePaths() throws Throwable
         }
 
         dropTable("DROP TABLE %s");
-        assert Schema.instance.isSameVersion(SchemaKeyspace.calculateSchemaDigest());
+        assert Schema.instance.isSameVersion(SchemaTestUtil.calculateSchemaDigest());
 
         //Test loading for a dropped 2i/table
         CacheService.instance.keyCache.clear();
@@ -329,14 +329,14 @@ private void test2iKeyCachePaths() throws Throwable
     @Test
     public void test2iKeyCachePathsSaveKeysForDroppedTableShallowIndexEntry() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         test2iKeyCachePathsSaveKeysForDroppedTable();
     }
 
     @Test
     public void test2iKeyCachePathsSaveKeysForDroppedTableIndexInfoOnHeap() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         test2iKeyCachePathsSaveKeysForDroppedTable();
     }
 
@@ -363,8 +363,8 @@ private void test2iKeyCachePathsSaveKeysForDroppedTable() throws Throwable
             expectedNumberOfRequests += recentBloomFilterFalsePositives() + 20;
         }
 
-        long hits = metrics.hits.getCount();
-        long requests = metrics.requests.getCount();
+        long hits = metrics.hits();
+        long requests = metrics.requests();
         assertEquals(0, hits);
         assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests);
 
@@ -382,8 +382,8 @@ private void test2iKeyCachePathsSaveKeysForDroppedTable() throws Throwable
         }
 
         metrics = CacheService.instance.keyCache.getMetrics();
-        hits = metrics.hits.getCount();
-        requests = metrics.requests.getCount();
+        hits = metrics.hits();
+        requests = metrics.requests();
         assertEquals(sstableImplCachesKeys ? 200 : 0, hits);
         assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests);
 
@@ -411,14 +411,14 @@ private void test2iKeyCachePathsSaveKeysForDroppedTable() throws Throwable
     @Test
     public void testKeyCacheNonClusteredShallowIndexEntry() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         testKeyCacheNonClustered();
     }
 
     @Test
     public void testKeyCacheNonClusteredIndexInfoOnHeap() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         testKeyCacheNonClustered();
     }
 
@@ -442,8 +442,8 @@ private void testKeyCacheNonClustered() throws Throwable
             expectedNumberOfRequests += recentBloomFilterFalsePositives() + 1;
         }
 
-        long hits = metrics.hits.getCount();
-        long requests = metrics.requests.getCount();
+        long hits = metrics.hits();
+        long requests = metrics.requests();
         assertEquals(0, hits);
         assertEquals(sstableImplCachesKeys ? 10 : 0, requests);
 
@@ -456,8 +456,8 @@ private void testKeyCacheNonClustered() throws Throwable
             expectedNumberOfRequests += recentBloomFilterFalsePositives() + 1;
         }
 
-        hits = metrics.hits.getCount();
-        requests = metrics.requests.getCount();
+        hits = metrics.hits();
+        requests = metrics.requests();
         assertEquals(sstableImplCachesKeys ? 10 : 0, hits);
         assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests);
     }
@@ -465,14 +465,14 @@ private void testKeyCacheNonClustered() throws Throwable
     @Test
     public void testKeyCacheClusteredShallowIndexEntry() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         testKeyCacheClustered();
     }
 
     @Test
     public void testKeyCacheClusteredIndexInfoOnHeap() throws Throwable
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         testKeyCacheClustered();
     }
 
@@ -493,8 +493,8 @@ private void testKeyCacheClustered() throws Throwable
         }
 
         CacheMetrics metrics = CacheService.instance.keyCache.getMetrics();
-        long hits = metrics.hits.getCount();
-        long requests = metrics.requests.getCount();
+        long hits = metrics.hits();
+        long requests = metrics.requests();
         assertEquals(0, hits);
         assertEquals(sstableImplCachesKeys ? 10 : 0, requests);
 
@@ -505,8 +505,8 @@ private void testKeyCacheClustered() throws Throwable
         }
 
         metrics = CacheService.instance.keyCache.getMetrics();
-        hits = metrics.hits.getCount();
-        requests = metrics.requests.getCount();
+        hits = metrics.hits();
+        requests = metrics.requests();
         assertEquals(sstableImplCachesKeys ? 10 : 0, hits);
         assertEquals(sstableImplCachesKeys ? 20 : 0, requests);
 
@@ -521,8 +521,8 @@ private void testKeyCacheClustered() throws Throwable
         }
 
         metrics = CacheService.instance.keyCache.getMetrics();
-        hits = metrics.hits.getCount();
-        requests = metrics.requests.getCount();
+        hits = metrics.hits();
+        requests = metrics.requests();
         assertEquals(sstableImplCachesKeys ? 10 + 100 : 0, hits);
         assertEquals(sstableImplCachesKeys ? 20 + 100 : 0, requests);
 
@@ -536,8 +536,8 @@ private void testKeyCacheClustered() throws Throwable
             }
         }
 
-        hits = metrics.hits.getCount();
-        requests = metrics.requests.getCount();
+        hits = metrics.hits();
+        requests = metrics.requests();
         assertEquals(sstableImplCachesKeys ? 110 + 4910 : 0, hits);
         assertEquals(sstableImplCachesKeys ? 120 + 5500 : 0, requests);
     }
@@ -602,10 +602,10 @@ private static void clearCache()
         CassandraMetricsRegistry.Metrics.getNames().forEach(CassandraMetricsRegistry.Metrics::remove);
         CacheService.instance.keyCache.clear();
         CacheMetrics metrics = CacheService.instance.keyCache.getMetrics();
-        Assert.assertEquals(0, metrics.entries.getValue().intValue());
-        Assert.assertEquals(0L, metrics.hits.getCount());
-        Assert.assertEquals(0L, metrics.requests.getCount());
-        Assert.assertEquals(0L, metrics.size.getValue().longValue());
+        Assert.assertEquals(0, metrics.entries());
+        Assert.assertEquals(0L, metrics.hits());
+        Assert.assertEquals(0L, metrics.requests());
+        Assert.assertEquals(0L, metrics.size());
     }
 
     private static void triggerBlockingFlush(Index index) throws Exception
diff --git a/test/unit/org/apache/cassandra/cql3/KeywordTestBase.java b/test/unit/org/apache/cassandra/cql3/KeywordTestBase.java
index aa6e508fa47d..15bc3a7ff340 100644
--- a/test/unit/org/apache/cassandra/cql3/KeywordTestBase.java
+++ b/test/unit/org/apache/cassandra/cql3/KeywordTestBase.java
@@ -27,9 +27,12 @@
 import com.google.common.collect.Sets;
 
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.service.StorageService;
+import org.assertj.core.api.Assertions;
 
 /**
  * This class tests all keywords which took a long time. Hence it was split into multiple
@@ -45,6 +48,12 @@ public abstract class KeywordTestBase extends CQLTester
                                                   })
                                                   .collect(Collectors.toList());
 
+    static
+    {
+        // ensure that ANN is a separate keyword, so it's included on this tests (see CNDB-12733)
+        Assertions.assertThat(keywords).contains(new Object[]{"ANN", false});
+    }
+    
     public static Collection<Object[]> getKeywordsForSplit(int split, int totalSplits)
     {
         return Sets.newHashSet(Lists.partition(KeywordTestBase.keywords, KeywordTestBase.keywords.size() / totalSplits)
@@ -59,6 +68,12 @@ public KeywordTestBase(String keyword, boolean isReserved)
         this.isReserved = isReserved;
     }
 
+    @BeforeClass
+    public static void beforeClass()
+    {
+        StorageService.instance.setUpDistributedSystemKeyspaces();
+    }
+
     @Test
     public void test() throws Throwable
     {
diff --git a/test/unit/org/apache/cassandra/cql3/MapsTest.java b/test/unit/org/apache/cassandra/cql3/MapsTest.java
new file mode 100644
index 000000000000..118485ad9191
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/MapsTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.NumberType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.utils.Pair;
+
+public class MapsTest extends CQLTester
+{
+    private final Function<NumberType<?>, AbstractType<?>> identityMapper = integerType -> integerType;
+
+    @Rule
+    public ExpectedException thrown = ExpectedException.none();
+
+    @Test
+    public void testGetExactMapTypeIfKnownWithDifferentTypes()
+    {
+        thrown.expect(InvalidRequestException.class);
+        thrown.expectMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals");
+
+        Maps.getExactMapTypeIfKnown(ImmutableList.of(
+            Pair.create(Int32Type.instance, Int32Type.instance),
+            Pair.create(Int32Type.instance, IntegerType.instance)
+        ), identityMapper);
+    }
+
+    @Test
+    public void testGetExactMapTypeIfKnownWithTheSameTypes()
+    {
+        AbstractType<?> exactType = Maps.getExactMapTypeIfKnown(ImmutableList.of(
+            Pair.create(Int32Type.instance, Int32Type.instance),
+            Pair.create(Int32Type.instance, Int32Type.instance)
+        ), identityMapper);
+
+        AbstractType<?> expected = MapType.getInstance(Int32Type.instance, Int32Type.instance, false).freeze();
+        Assert.assertEquals(expected, exactType);
+    }
+
+    @Test
+    public void testGetExactMapTypeIfKnownWithoutTypes()
+    {
+        AbstractType<?> exactType = Maps.getExactMapTypeIfKnown(ImmutableList.of(), identityMapper);
+
+        Assert.assertNull(exactType);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java b/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java
index a9f3d267df63..a0de5ba8a90a 100644
--- a/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java
+++ b/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java
@@ -21,12 +21,10 @@
 import org.junit.Test;
 
 import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.metrics.ClientRequestsMetricsProvider;
 
 import static org.apache.cassandra.db.ConsistencyLevel.NODE_LOCAL;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsForLevel;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetrics;
-import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetricsForLevel;
 import static org.junit.Assert.assertEquals;
 
 public class NodeLocalConsistencyTest extends CQLTester
@@ -40,15 +38,16 @@ public static void setUp() throws Exception
     @Test
     public void testModify()
     {
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
         createTable("CREATE TABLE %s (key text, val int, PRIMARY KEY(key));");
 
-        long beforeLevel  = writeMetricsForLevel(NODE_LOCAL).latency.getCount();
-        long beforeGlobal = writeMetrics.latency.getCount();
+        long beforeLevel  = metrics.writeMetricsForLevel(NODE_LOCAL).executionTimeMetrics.latency.getCount();
+        long beforeGlobal = metrics.writeMetrics.executionTimeMetrics.latency.getCount();
 
         QueryProcessor.process(formatQuery("INSERT INTO %s (key, val) VALUES ('key', 0);"), NODE_LOCAL);
 
-        long afterLevel  = writeMetricsForLevel(NODE_LOCAL).latency.getCount();
-        long afterGlobal = writeMetrics.latency.getCount();
+        long afterLevel  = metrics.writeMetricsForLevel(NODE_LOCAL).executionTimeMetrics.latency.getCount();
+        long afterGlobal = metrics.writeMetrics.executionTimeMetrics.latency.getCount();
 
         assertEquals(1, afterLevel - beforeLevel);
         assertEquals(1, afterGlobal - beforeGlobal);
@@ -57,15 +56,16 @@ public void testModify()
     @Test
     public void testBatch()
     {
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
         createTable("CREATE TABLE %s (key text, val int, PRIMARY KEY(key));");
 
-        long beforeLevel  = writeMetricsForLevel(NODE_LOCAL).latency.getCount();
-        long beforeGlobal = writeMetrics.latency.getCount();
+        long beforeLevel  = metrics.writeMetricsForLevel(NODE_LOCAL).executionTimeMetrics.latency.getCount();
+        long beforeGlobal = metrics.writeMetrics.executionTimeMetrics.latency.getCount();
 
         QueryProcessor.process(formatQuery("BEGIN BATCH INSERT INTO %s (key, val) VALUES ('key', 0); APPLY BATCH;"), NODE_LOCAL);
 
-        long afterLevel  = writeMetricsForLevel(NODE_LOCAL).latency.getCount();
-        long afterGlobal = writeMetrics.latency.getCount();
+        long afterLevel  = metrics.writeMetricsForLevel(NODE_LOCAL).executionTimeMetrics.latency.getCount();
+        long afterGlobal = metrics.writeMetrics.executionTimeMetrics.latency.getCount();
 
         assertEquals(1, afterLevel - beforeLevel);
         assertEquals(1, afterGlobal - beforeGlobal);
@@ -74,15 +74,16 @@ public void testBatch()
     @Test
     public void testSelect()
     {
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(null);
         createTable("CREATE TABLE %s (key text, val int, PRIMARY KEY(key));");
 
-        long beforeLevel  = readMetricsForLevel(NODE_LOCAL).latency.getCount();
-        long beforeGlobal = readMetrics.latency.getCount();
+        long beforeLevel  = metrics.readMetricsForLevel(NODE_LOCAL).executionTimeMetrics.latency.getCount();
+        long beforeGlobal = metrics.readMetrics.executionTimeMetrics.latency.getCount();
 
         QueryProcessor.process(formatQuery("SELECT * FROM %s;"), NODE_LOCAL);
 
-        long afterLevel  = readMetricsForLevel(NODE_LOCAL).latency.getCount();
-        long afterGlobal = readMetrics.latency.getCount();
+        long afterLevel  = metrics.readMetricsForLevel(NODE_LOCAL).executionTimeMetrics.latency.getCount();
+        long afterGlobal = metrics.readMetrics.executionTimeMetrics.latency.getCount();
 
         assertEquals(1, afterLevel - beforeLevel);
         assertEquals(1, afterGlobal - beforeGlobal);
diff --git a/test/unit/org/apache/cassandra/cql3/OperatorTest.java b/test/unit/org/apache/cassandra/cql3/OperatorTest.java
new file mode 100644
index 000000000000..182c3d9b6880
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/OperatorTest.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.Index;
+import org.assertj.core.api.Assertions;
+
+public class OperatorTest
+{
+    @Test
+    public void testAnalyzer()
+    {
+        // test with a text-based case-insensitive analyzer
+        UTF8Type utf8Type = UTF8Type.instance;
+        Index.Analyzer analyzer = value -> Collections.singletonList(utf8Type.decompose(utf8Type.compose(value).toUpperCase()));
+        testAnalyzer(utf8Type, utf8Type.decompose("FOO"), utf8Type.decompose("FOO"), analyzer, true);
+        testAnalyzer(utf8Type, utf8Type.decompose("FOO"), utf8Type.decompose("foo"), analyzer, true);
+        testAnalyzer(utf8Type, utf8Type.decompose("foo"), utf8Type.decompose("foo"), analyzer, true);
+        testAnalyzer(utf8Type, utf8Type.decompose("foo"), utf8Type.decompose("FOO"), analyzer, true);
+        testAnalyzer(utf8Type, utf8Type.decompose("foo"), utf8Type.decompose("abc"), analyzer, false);
+
+        // test with an int-based analyzer that decomposes an integer into its digits
+        Int32Type intType = Int32Type.instance;
+        analyzer = value -> intType.compose(value)
+                                   .toString()
+                                   .chars()
+                                   .boxed()
+                                   .map(intType::decompose)
+                                   .collect(Collectors.toList());
+        testAnalyzer(intType, intType.decompose(123), intType.decompose(123), analyzer, true);
+        testAnalyzer(intType, intType.decompose(123), intType.decompose(1), analyzer, true);
+        testAnalyzer(intType, intType.decompose(123), intType.decompose(2), analyzer, true);
+        testAnalyzer(intType, intType.decompose(123), intType.decompose(3), analyzer, true);
+        testAnalyzer(utf8Type, intType.decompose(123), intType.decompose(4), analyzer, false);
+        testAnalyzer(utf8Type, intType.decompose(123), intType.decompose(12), analyzer, true);
+        testAnalyzer(utf8Type, intType.decompose(123), intType.decompose(23), analyzer, true);
+        testAnalyzer(utf8Type, intType.decompose(123), intType.decompose(13), analyzer, true);
+        testAnalyzer(utf8Type, intType.decompose(123), intType.decompose(321), analyzer, true);
+        testAnalyzer(utf8Type, intType.decompose(123), intType.decompose(1234), analyzer, false);
+    }
+
+    private static void testAnalyzer(AbstractType<?> type,
+                                     ByteBuffer left,
+                                     ByteBuffer right,
+                                     Index.Analyzer analyzer,
+                                     boolean shouldBeSatisfied)
+    {
+        // test that EQ and ANALYZER_MATCHES are satisfied by the same value with an analyzer
+        for (Operator operator : Arrays.asList(Operator.EQ, Operator.ANALYZER_MATCHES))
+            Assertions.assertThat(operator.isSatisfiedBy(type, left, right, analyzer, analyzer)).isEqualTo(shouldBeSatisfied);
+
+        // test that EQ without an analyzer behaves as type-based identity
+        Assertions.assertThat(Operator.EQ.isSatisfiedBy(type, left, right, null, null))
+                  .isEqualTo(type.compareForCQL(left, right) == 0);
+
+        // test that ANALYZER_MATCHES throws an exception when no analyzer is provided
+        Assertions.assertThatThrownBy(() -> Operator.ANALYZER_MATCHES.isSatisfiedBy(type, left, right, null, null))
+                  .isInstanceOf(AssertionError.class)
+                  .hasMessageContaining(": operation can only be computed by an indexed column with a configured analyzer");
+
+        // test that all other operators ignore the analyzer
+        for (Operator operator : Operator.values())
+        {
+            if (operator == Operator.EQ || operator == Operator.ANALYZER_MATCHES)
+                continue;
+
+            boolean supported = false;
+            try
+            {
+                shouldBeSatisfied = operator.isSatisfiedBy(type, left, right, null, null);
+                supported = true;
+            }
+            catch (Exception e)
+            {
+                Assertions.assertThatThrownBy(() -> operator.isSatisfiedBy(type, left, right, analyzer, analyzer))
+                          .isInstanceOf(e.getClass())
+                          .hasMessage(e.getMessage());
+            }
+
+            if (supported)
+            {
+                Assertions.assertThat(operator.isSatisfiedBy(type, left, right, analyzer, analyzer))
+                          .isEqualTo(shouldBeSatisfied);
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
index 49195f57cf99..b675ae1d664d 100644
--- a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@ -18,40 +18,87 @@
 package org.apache.cassandra.cql3;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.util.concurrent.ExecutionException;
 
+import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.Config.DiskFailurePolicy;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.CommitLogSegment;
-import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.JVMKiller;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.KillerForTests;
 
 import static org.junit.Assert.fail;
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
 
 /**
  * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
  */
-public class OutOfSpaceTest extends CQLTester
+public class OutOfSpaceTest
 {
+    private final static String KEYSPACE = "ks";
+    private final static String TABLE = "tab";
+    private static EmbeddedCassandraService service; // we use EmbeddedCassandraService instead of CqlTester because we want CassandraDaemon
+
+    @BeforeClass
+    public static void beforeClass() throws IOException
+    {
+        DatabaseDescriptor.daemonInitialization();
+        ServerTestUtils.mkdirs();
+        ServerTestUtils.cleanup();
+        service = new EmbeddedCassandraService();
+        service.start();
+    }
+
+    public static void afterClass()
+    {
+        service.stop();
+    }
+
+    @Before
+    public void before()
+    {
+        if (!StorageService.instance.isNativeTransportRunning())
+            StorageService.instance.startNativeTransport();
+        if (!StorageService.instance.isGossipActive())
+            StorageService.instance.startGossiping();
+    }
+
+    @After
+    public void after()
+    {
+        SchemaTestUtil.dropKeyspaceIfExist(KEYSPACE, false);
+    }
+
     @Test
     public void testFlushUnwriteableDie() throws Throwable
     {
         makeTable();
 
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
-        try (Closeable c = Util.markDirectoriesUnwriteable(getCurrentColumnFamilyStore()))
+        try (Closeable c = Util.markDirectoriesUnwriteable(ColumnFamilyStore.getIfExists(KEYSPACE, TABLE)))
         {
             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
             flushAndExpectError();
@@ -71,9 +118,10 @@ public void testFlushUnwriteableStop() throws Throwable
         makeTable();
 
         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
-        try (Closeable c = Util.markDirectoriesUnwriteable(getCurrentColumnFamilyStore()))
+        try (Closeable c = Util.markDirectoriesUnwriteable(ColumnFamilyStore.getIfExists(KEYSPACE, TABLE)))
         {
             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+            Assert.assertTrue(Gossiper.instance.isEnabled()); // sanity check
             flushAndExpectError();
             Assert.assertFalse(Gossiper.instance.isEnabled());
         }
@@ -89,7 +137,7 @@ public void testFlushUnwriteableIgnore() throws Throwable
         makeTable();
 
         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
-        try (Closeable c = Util.markDirectoriesUnwriteable(getCurrentColumnFamilyStore()))
+        try (Closeable c = Util.markDirectoriesUnwriteable(ColumnFamilyStore.getIfExists(KEYSPACE, TABLE)))
         {
             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
             flushAndExpectError();
@@ -105,11 +153,12 @@ public void testFlushUnwriteableIgnore() throws Throwable
 
     public void makeTable() throws Throwable
     {
-        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+        SchemaTestUtil.announceNewKeyspace(KeyspaceMetadata.create(KEYSPACE, KeyspaceParams.simple(1)));
+        QueryProcessor.executeInternal(String.format("CREATE TABLE %s.%s (a text, b text, c text, PRIMARY KEY (a, b));", KEYSPACE, TABLE));
 
         // insert exactly the amount of tombstones that shouldn't trigger an exception
         for (int i = 0; i < 10; i++)
-            execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+            QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (a, b, c) VALUES ('key', 'column%d', null);", KEYSPACE, TABLE, i));
     }
 
     public void flushAndExpectError() throws InterruptedException, ExecutionException
@@ -117,8 +166,8 @@ public void flushAndExpectError() throws InterruptedException, ExecutionExceptio
         try
         {
             Keyspace.open(KEYSPACE)
-                    .getColumnFamilyStore(currentTable())
-                    .forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS)
+                    .getColumnFamilyStore(TABLE)
+                    .forceFlush(UNIT_TESTS)
                     .get();
             fail("FSWriteError expected.");
         }
@@ -130,9 +179,19 @@ public void flushAndExpectError() throws InterruptedException, ExecutionExceptio
 
         // Make sure commit log wasn't discarded.
         TableId tableId = currentTableMetadata().id;
-        for (CommitLogSegment segment : CommitLog.instance.segmentManager.getActiveSegments())
+        for (CommitLogSegment segment : CommitLog.instance.getSegmentManager().getActiveSegments())
             if (segment.getDirtyTableIds().contains(tableId))
                 return;
         fail("Expected commit log to remain dirty for the affected table.");
     }
+
+    private TableMetadata currentTableMetadata()
+    {
+        return ColumnFamilyStore.getIfExists(KEYSPACE, TABLE).metadata();
+    }
+
+    private void flush()
+    {
+        ColumnFamilyStore.getIfExists(KEYSPACE, TABLE).forceBlockingFlush(UNIT_TESTS);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/PagingQueryTest.java b/test/unit/org/apache/cassandra/cql3/PagingQueryTest.java
index 8f5f2828b482..eec789cd2adf 100644
--- a/test/unit/org/apache/cassandra/cql3/PagingQueryTest.java
+++ b/test/unit/org/apache/cassandra/cql3/PagingQueryTest.java
@@ -18,20 +18,63 @@
 
 package org.apache.cassandra.cql3;
 
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Supplier;
 
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
-import com.datastax.driver.core.*;
 import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.OperationExecutionException;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.pager.AggregationQueryPager;
+import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
 
+import static org.apache.commons.lang3.ArrayUtils.EMPTY_OBJECT_ARRAY;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+@RunWith(Parameterized.class)
 public class PagingQueryTest extends CQLTester
 {
+    final int ROW_SIZE = 53; // size of internal representation
+
+    @Parameterized.Parameters(name = "aggregation_sub_page_size={0}")
+    public static Collection<Object[]> generateParameters()
+    {
+        return Arrays.asList(new Object[]{ PageSize.inBytes(1024) }, new Object[]{ PageSize.NONE });
+    }
+
+    public PagingQueryTest(PageSize subPageSize)
+    {
+        DatabaseDescriptor.setAggregationSubPageSize(subPageSize);
+    }
+
     @Test
     public void pagingOnRegularColumn() throws Throwable
     {
@@ -59,53 +102,454 @@ public void pagingOnRegularColumn() throws Throwable
 
         flush();
 
-        try (Session session = sessionNet())
+        Session session = sessionNet();
+        SimpleStatement stmt = new SimpleStatement("SELECT c1, c2, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE k1 = 1");
+        stmt.setFetchSize(3);
+        ResultSet rs = session.execute(stmt);
+        Iterator<Row> iter = rs.iterator();
+        for (int c1 = 0; c1 < 100; c1++)
+        {
+            for (int c2 = 0; c2 < 100; c2++)
+            {
+                assertTrue(iter.hasNext());
+                Row row = iter.next();
+                String msg = "On " + c1 + ',' + c2;
+                assertEquals(msg, c1, row.getInt(0));
+                assertEquals(msg, c2, row.getInt(1));
+                assertEquals(msg, Integer.toString(c1), row.getString(2));
+                assertEquals(msg, Integer.toString(c2), row.getString(3));
+            }
+        }
+        assertFalse(iter.hasNext());
+
+        for (int c1 = 0; c1 < 100; c1++)
         {
-            SimpleStatement stmt = new SimpleStatement("SELECT c1, c2, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE k1 = 1");
+            stmt = new SimpleStatement("SELECT c1, c2, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE k1 = 1 AND c1 = ?", c1);
             stmt.setFetchSize(3);
-            ResultSet rs = session.execute(stmt);
-            Iterator<Row> iter = rs.iterator();
-            for (int c1 = 0; c1 < 100; c1++)
+            rs = session.execute(stmt);
+            iter = rs.iterator();
+            for (int c2 = 0; c2 < 100; c2++)
+            {
+                assertTrue(iter.hasNext());
+                Row row = iter.next();
+                String msg = "Within " + c1 + " on " + c2;
+                assertEquals(msg, c1, row.getInt(0));
+                assertEquals(msg, c2, row.getInt(1));
+                assertEquals(msg, Integer.toString(c1), row.getString(2));
+                assertEquals(msg, Integer.toString(c2), row.getString(3));
+            }
+            assertFalse(iter.hasNext());
+        }
+    }
+
+    // new paging-in-bytes tests
+
+    /**
+     * Returns a lambda that creates a pager for the query
+     */
+    private Supplier<Pair<QueryPager, SelectStatement>> getPager(String query, Object... args)
+    {
+        return () -> {
+            QueryHandler.Prepared prepared = QueryProcessor.prepareInternal(String.format(query, args));
+            SelectStatement select = (SelectStatement) prepared.statement;
+            ReadQuery readQuery = select.getQuery(QueryProcessor.makeInternalOptions(prepared.statement, EMPTY_OBJECT_ARRAY), FBUtilities.nowInSeconds());
+            QueryPager pager = select.getPager(readQuery, QueryOptions.forInternalCalls(ConsistencyLevel.LOCAL_ONE, Collections.emptyList()));
+            return Pair.create(pager, select);
+        };
+    }
+
+    /**
+     * Inovke the test and check for the expected number of rows
+     */
+    private void assertResults(Supplier<Pair<QueryPager, SelectStatement>> pagerSupplier, int expectedCount)
+    {
+        Pair<QueryPager, SelectStatement> pagerAndStmt = pagerSupplier.get();
+        QueryPager pager = pagerAndStmt.left;
+        SelectStatement select = pagerAndStmt.right;
+
+        List<List<ByteBuffer>> rows;
+
+        long nowInSec = FBUtilities.nowInSeconds();
+        assertThat(pager.isExhausted()).isFalse();
+        try (ReadExecutionController executionController = pager.executionController();
+             PartitionIterator iter = pager.fetchPageInternal(PageSize.NONE, executionController))
+        {
+            rows = select.process(iter, nowInSec, true, ClientState.forInternalCalls()).rows;
+        }
+
+        assertThat(rows.size()).isEqualTo(expectedCount);
+        assertThat(pager.isExhausted()).isTrue();
+    }
+
+    /**
+     * Invoke the tests with the provided page size. Firstly we just request the page size in rows as provided by the parameter.
+     * In the second test we convert (by multiplying) the requested number of rows on page to the number of bytes (assuming certain row size).
+     */
+    private void assertResults(Supplier<Pair<QueryPager, SelectStatement>> pagerSupplier, int requestedPageSizeInRows, int expectedCountOnFirstPage, int expectedCount)
+    {
+        assertResults(pagerSupplier, PageSize.inRows(requestedPageSizeInRows), expectedCountOnFirstPage, expectedCount);
+        assertResults(pagerSupplier, PageSize.inBytes(requestedPageSizeInRows * ROW_SIZE), expectedCountOnFirstPage, expectedCount);
+    }
+
+    /**
+     * Invoke the tests with the provided page size. Firstly we just request the page size in rows as provided by the parameter.
+     * In the second test we convert (by multiplying) the requested number of rows on page to the number of bytes (assuming certain row size).
+     */
+    private void assertResults(Supplier<Pair<QueryPager, SelectStatement>> pagerSupplier, int requestedPageSizeInRows, int expectedCountOnFirstPage, int expectedCount, int expectedValue)
+    {
+        List<List<ByteBuffer>> rows = assertResults(pagerSupplier, PageSize.inRows(requestedPageSizeInRows), expectedCountOnFirstPage, expectedCount);
+        assertThat(ByteBufferUtil.toLong(rows.get(0).get(0))).isEqualTo((long) expectedValue);
+        assertResults(pagerSupplier, PageSize.inBytes(requestedPageSizeInRows * ROW_SIZE), expectedCountOnFirstPage, expectedCount);
+    }
+
+    /**
+     * Invoke the test with the provided page size. Expect the exact number of rows on the first page and exact number of rows in total (all pages).
+     */
+    private List<List<ByteBuffer>> assertResults(Supplier<Pair<QueryPager, SelectStatement>> pagerSupplier, PageSize requestedPageSize, int expectedCountOnFirstPage, int expectedCount)
+    {
+        Pair<QueryPager, SelectStatement> pagerAndStmt = pagerSupplier.get();
+        QueryPager pager = pagerAndStmt.left;
+        SelectStatement select = pagerAndStmt.right;
+
+        List<List<ByteBuffer>> rows = null;
+
+        long nowInSec = FBUtilities.nowInSeconds();
+
+        int countOnFirstPage = -1;
+        int count = 0;
+
+        logger.info("Assertion on query {} with requested page size {} - expected count on first page = {}, expected count total = {}:", select.toString(), requestedPageSize, expectedCountOnFirstPage, expectedCount);
+
+        try
+        {
+            while (!pager.isExhausted())
             {
-                for (int c2 = 0; c2 < 100; c2++)
+                try (ReadExecutionController executionController = pager.executionController();
+                     PartitionIterator iter = pager.fetchPageInternal(requestedPageSize, executionController))
                 {
-                    assertTrue(iter.hasNext());
-                    Row row = iter.next();
-                    String msg = "On " + c1 + ',' + c2;
-                    assertEquals(msg, c1, row.getInt(0));
-                    assertEquals(msg, c2, row.getInt(1));
-                    assertEquals(msg, Integer.toString(c1), row.getString(2));
-                    assertEquals(msg, Integer.toString(c2), row.getString(3));
+                    rows = select.process(iter, nowInSec, true, ClientState.forInternalCalls()).rows;
+                    logger.info("Got page of {} rows with size: {}", rows.size(), rows.stream().mapToInt(cols -> cols.stream().mapToInt(Buffer::remaining).sum()).sum());
                 }
+
+                if (countOnFirstPage < 0)
+                    countOnFirstPage = rows.size();
+                count += rows.size();
             }
-            assertFalse(iter.hasNext());
 
-            for (int c1 = 0; c1 < 100; c1++)
+            assertThat(countOnFirstPage).isEqualTo(expectedCountOnFirstPage);
+            assertThat(count).isEqualTo(expectedCount);
+            assertThat(pager.isExhausted()).isTrue();
+        }
+        catch (InvalidRequestException ex)
+        {
+            if (pager instanceof AggregationQueryPager && requestedPageSize.getUnit() == PageSize.PageUnit.BYTES)
+                return null;
+        }
+
+        if (pager instanceof AggregationQueryPager && requestedPageSize.getUnit() == PageSize.PageUnit.BYTES)
+            fail("Expected " + OperationExecutionException.class.getSimpleName() + " to be thrown when paging is in bytes");
+
+        return rows;
+    }
+
+    private void testPagingCases(String query, int selPartitions, int selClusterings, int genPartitions, int genClusterings) throws Throwable
+    {
+        testPagingCases(query, selPartitions, selClusterings, genPartitions, genClusterings, 1);
+    }
+
+
+    private void testPagingCases(String query, int selPartitions, int selClusterings, int genPartitions, int genClusterings, int genClusterings2) throws Throwable
+    {
+        String table = generateData(genPartitions, genClusterings, genClusterings2);
+
+        flush(KEYSPACE, table);
+        Supplier<Pair<QueryPager, SelectStatement>> pagerSupplier;
+        query = String.format(query, KEYSPACE + '.' + table);
+        int selected = selPartitions * selClusterings;
+
+        // when there is a page size
+        pagerSupplier = getPager("%s ALLOW FILTERING", query);
+        assertResults(pagerSupplier, selected / 3, selected / 3, selected);
+
+        // when there is a query limit
+        pagerSupplier = getPager("%s LIMIT %d ALLOW FILTERING", query, selected / 3);
+        assertResults(pagerSupplier, selected / 3);
+
+
+        // when there is a per partition limit
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d ALLOW FILTERING", query, selClusterings / 2);
+        assertResults(pagerSupplier, selPartitions * (selClusterings / 2));
+
+
+        // when there is a page size and a query limit:
+
+        // - where query limit is == page size
+        pagerSupplier = getPager("%s LIMIT %d ALLOW FILTERING", query, selected / 2);
+        assertResults(pagerSupplier, selected / 2, selected / 2, selected / 2);
+
+        // - where query limit is < page size
+        pagerSupplier = getPager("%s LIMIT %d ALLOW FILTERING", query, selected / 3);
+        assertResults(pagerSupplier, selected / 2, selected / 3, selected / 3);
+
+        // - where query limit is > page size
+        pagerSupplier = getPager("%s LIMIT %d ALLOW FILTERING", query, selected / 2);
+        assertResults(pagerSupplier, selected / 3, selected / 3, selected / 2);
+
+
+        // when there is a per partition limit and a query limit:
+
+        // - where query limit is < per partition limit
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 2, selClusterings / 3);
+        assertResults(pagerSupplier, selClusterings / 3);
+
+        // - where query limit is > per partition limit (case for single partition and multiple partitions)
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 3, selClusterings / 2);
+        if (selPartitions == 1)
+            assertResults(pagerSupplier, selClusterings / 3);
+        else
+            assertResults(pagerSupplier, selClusterings / 2);
+
+        // - where query limit is == per partition limit
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 2, selClusterings / 2);
+        assertResults(pagerSupplier, selClusterings / 2);
+
+        // when there is a page size and a per partition limit,
+
+        // - where page size is < per partition limit
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d ALLOW FILTERING", query, selClusterings / 2);
+        assertResults(pagerSupplier, selClusterings / 3, selClusterings / 3, selPartitions * (selClusterings / 2));
+
+        // - where page size is == per partition limit
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d ALLOW FILTERING", query, selClusterings / 2);
+        assertResults(pagerSupplier, selClusterings / 2, selClusterings / 2, selPartitions * (selClusterings / 2));
+
+        // - where page size is > per partition limit (case for single partition and mulitple partitions)
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d ALLOW FILTERING", query, selClusterings / 3);
+        if (selPartitions == 1)
+            assertResults(pagerSupplier, selClusterings / 2, selClusterings / 3, selClusterings / 3);
+        else
+            assertResults(pagerSupplier, selClusterings / 2, selClusterings / 2, selPartitions * (selClusterings / 3));
+
+
+        // when there is a page size, a per partition limit and a query limit
+
+        // - where per partition limit == query limit == page size
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 2, selClusterings / 2);
+        assertResults(pagerSupplier, selClusterings / 2, selClusterings / 2, selClusterings / 2);
+
+        // - where per partition limit > query limit > page size
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 2, selClusterings / 3);
+        assertResults(pagerSupplier, selClusterings / 4, selClusterings / 4, selClusterings / 3);
+
+        // - where per partition limit > page size > query limit
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 2, selClusterings / 4);
+        assertResults(pagerSupplier, selClusterings / 3, selClusterings / 4, selClusterings / 4);
+
+        // - where per query limit > per partition limit > page size (case for single partition and mulitple partitions)
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 3, selClusterings / 2);
+        if (selPartitions == 1)
+            assertResults(pagerSupplier, selClusterings / 4, selClusterings / 4, selClusterings / 3);
+        else
+            assertResults(pagerSupplier, selClusterings / 4, selClusterings / 4, selClusterings / 2);
+
+        // - where per query limit > page size > per partition limit (case for single partition and mulitple partitions)
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 4, selClusterings / 2);
+        if (selPartitions == 1)
+            assertResults(pagerSupplier, selClusterings / 3, selClusterings / 4, selClusterings / 4);
+        else
+            assertResults(pagerSupplier, selClusterings / 3, selClusterings / 3, selClusterings / 2);
+
+        // - where page size > per partition limit > query limit (case for single partition and mulitple partitions)
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 3, selClusterings / 4);
+        if (selPartitions == 1)
+            assertResults(pagerSupplier, selClusterings / 2, selClusterings / 4, selClusterings / 4);
+        else
+            assertResults(pagerSupplier, selClusterings / 2, selClusterings / 4, selClusterings / 4);
+
+        // - where page size > query limit > per partition limit (case for single partition and mulitple partitions)
+        pagerSupplier = getPager("%s PER PARTITION LIMIT %d LIMIT %d ALLOW FILTERING", query, selClusterings / 4, selClusterings / 3);
+        if (selPartitions == 1)
+            assertResults(pagerSupplier, selClusterings / 2, selClusterings / 4, selClusterings / 4);
+        else
+            assertResults(pagerSupplier, selClusterings / 2, selClusterings / 3, selClusterings / 3);
+    }
+
+
+    private void testPagingCasesWithAggregateEverything(String query, int genPartitions, int genClusterings, int genClusterings2, int expectedResult) throws Throwable
+    {
+        String table = generateData(genPartitions, genClusterings, genClusterings2);
+
+        flush(KEYSPACE, table);
+        Supplier<Pair<QueryPager, SelectStatement>> pagerSupplier;
+        query = String.format(query, KEYSPACE + '.' + table);
+
+        // when there is a page size
+        pagerSupplier = getPager("%s ALLOW FILTERING", query);
+        assertResults(pagerSupplier, 1, 1, 1, expectedResult);
+    }
+
+    private String generateData(int genPartitions, int genClusterings, int genClusterings2) throws Throwable
+    {
+        String table = String.format("table_%d_%d_%d", genPartitions, genClusterings, genClusterings2);
+        if (Schema.instance.getTableMetadata(KEYSPACE, table) != null)
+            return table;
+
+        QueryProcessor.executeInternal(String.format("CREATE TABLE %s.%s (k INT, c INT, c2 INT, v INT, PRIMARY KEY (k, c, c2))", KEYSPACE, table));
+        for (int k = 0; k < genPartitions; k++)
+        {
+            for (int c = 0; c < genClusterings; c++)
             {
-                stmt = new SimpleStatement("SELECT c1, c2, v1, v2 FROM " + KEYSPACE + '.' + currentTable() + " WHERE k1 = 1 AND c1 = ?", c1);
-                stmt.setFetchSize(3);
-                rs = session.execute(stmt);
-                iter = rs.iterator();
-                for (int c2 = 0; c2 < 100; c2++)
+                for (int c2 = 0; c2 < genClusterings2; c2++)
                 {
-                    assertTrue(iter.hasNext());
-                    Row row = iter.next();
-                    String msg = "Within " + c1 + " on " + c2;
-                    assertEquals(msg, c1, row.getInt(0));
-                    assertEquals(msg, c2, row.getInt(1));
-                    assertEquals(msg, Integer.toString(c1), row.getString(2));
-                    assertEquals(msg, Integer.toString(c2), row.getString(3));
+                    execute(String.format("INSERT INTO %s.%s (k, c, c2, v) VALUES (?, ?, ?, ?)", KEYSPACE, table), k, c, c2, 1);
+                    if ((k * genClusterings + c) % (3 * (genClusterings + genPartitions) / 2) == 0)
+                        flush(KEYSPACE, table);
                 }
-                assertFalse(iter.hasNext());
             }
         }
+
+        return table;
+    }
+
+
+    @Test
+    public void testLimitsOnFullScanQuery() throws Throwable
+    {
+        testPagingCases("SELECT * FROM %s", 10, 10, 10, 10);
+    }
+
+    @Test
+    public void testLimitsOnSliceSelection() throws Throwable
+    {
+        testPagingCases("SELECT * FROM %s WHERE c > 2 AND c <= 7", 10, 5, 10, 10);
+    }
+
+    @Test
+    public void testLimitsOnClusteringsSelection() throws Throwable
+    {
+        testPagingCases("SELECT * FROM %s WHERE c IN (2, 4, 7, 8)", 10, 4, 10, 10);
+    }
+
+    @Test
+    public void testLimitsOnSliceAndKeyRangeSelection() throws Throwable
+    {
+        testPagingCases("SELECT * FROM %s WHERE c > 2 AND c <= 7 AND TOKEN(k) > TOKEN(0)", 6, 5, 10, 10);
+    }
+
+    @Test
+    public void testLimitsInSinglePartition() throws Throwable
+    {
+        testPagingCases("SELECT * FROM %s WHERE k = 5", 1, 100, 10, 100);
+    }
+
+    @Test
+    public void testLimitsInMultiplePartitions() throws Throwable
+    {
+        testPagingCases("SELECT * FROM %s WHERE k IN (5, 7, 9)", 3, 100, 10, 100);
+    }
+
+    @Test
+    public void testLimitsOnSliceInSinglePartition() throws Throwable
+    {
+        testPagingCases("SELECT * FROM %s WHERE c > 20 AND c <= 70 AND k = 5", 1, 50, 10, 100);
+    }
+
+    @Test
+    public void testLimitsOnClusteringsInSinglePartitionSelection() throws Throwable
+    {
+        testPagingCases("SELECT * FROM %s WHERE c IN (2, 4, 7, 8) AND k = 5", 1, 4, 10, 10);
+    }
+
+    @Test
+    public void testLimitsOnFullScanQueryWithGrouping() throws Throwable
+    {
+        testPagingCases("SELECT k, c, SUM(v) FROM %s GROUP BY k, c", 10, 10, 10, 10, 10);
+    }
+
+    @Test
+    public void testLimitsOnSliceSelectionWithGrouping() throws Throwable
+    {
+        testPagingCases("SELECT k, c, SUM(v) FROM %s WHERE c > 2 AND c <= 7 GROUP BY k, c", 10, 5, 10, 10, 10);
+    }
+
+    @Test
+    public void testLimitsOnClusteringsSelectionWithGrouping() throws Throwable
+    {
+        testPagingCases("SELECT k, c, SUM(v) FROM %s WHERE c IN (2, 4, 7, 8) GROUP BY k, c", 10, 4, 10, 10, 10);
+    }
+
+    @Test
+    public void testLimitsOnSliceAndKeyRangeSelectionWithGrouping() throws Throwable
+    {
+        testPagingCases("SELECT k, c, SUM(v) FROM %s WHERE c > 2 AND c <= 7 AND TOKEN(k) > TOKEN(0) GROUP BY k, c", 6, 5, 10, 10, 10);
+    }
+
+    @Test
+    public void testLimitsInSinglePartitionWithGrouping() throws Throwable
+    {
+        testPagingCases("SELECT k, c, SUM(v) FROM %s WHERE k = 5 GROUP BY k, c", 1, 100, 10, 100, 10);
+    }
+
+    @Test
+    public void testLimitsInMultiplePartitionsWithGrouping() throws Throwable
+    {
+        testPagingCases("SELECT k, c, SUM(v) FROM %s WHERE k IN (5, 7, 9) GROUP BY k, c", 3, 100, 10, 100, 10);
+    }
+
+    @Test
+    public void testLimitsOnSliceInSinglePartitionWithGrouping() throws Throwable
+    {
+        testPagingCases("SELECT k, c, SUM(v) FROM %s WHERE c > 20 AND c <= 70 AND k = 5 GROUP BY k, c", 1, 50, 10, 100, 10);
+    }
+
+
+    @Test
+    public void testLimitsOnFullScanQueryWithAggregateEverything() throws Throwable
+    {
+        testPagingCasesWithAggregateEverything("SELECT COUNT(*) FROM %s", 3, 3, 3, 27);
+    }
+
+    @Test
+    public void testLimitsOnSliceSelectionWithAggregateEverything() throws Throwable
+    {
+        testPagingCasesWithAggregateEverything("SELECT COUNT(*) FROM %s WHERE c > 2 AND c <= 7", 10, 10, 10, 500);
+    }
+
+    @Test
+    public void testLimitsOnClusteringsSelectionWithAggregateEverything() throws Throwable
+    {
+        testPagingCasesWithAggregateEverything("SELECT COUNT(*) FROM %s WHERE c IN (2, 4, 7, 8)", 10, 10, 10, 400);
+    }
+
+    @Test
+    public void testLimitsOnSliceAndKeyRangeSelectionWithAggregateEverything() throws Throwable
+    {
+        testPagingCasesWithAggregateEverything("SELECT COUNT(*) FROM %s WHERE c > 2 AND c <= 7 AND TOKEN(k) > TOKEN(0)", 10, 10, 10, 300);
+    }
+
+    @Test
+    public void testLimitsInSinglePartitionWithAggregateEverything() throws Throwable
+    {
+        testPagingCasesWithAggregateEverything("SELECT COUNT(*) FROM %s WHERE k = 5", 10, 10, 10, 100);
+    }
+
+    @Test
+    public void testLimitsInMultiplePartitionsWithAggregateEverything() throws Throwable
+    {
+        testPagingCasesWithAggregateEverything("SELECT COUNT(*) FROM %s WHERE k IN (5, 7, 9)", 10, 10, 10, 300);
+    }
+
+    @Test
+    public void testLimitsOnSliceInSinglePartitionWithAggregateEverything() throws Throwable
+    {
+        testPagingCasesWithAggregateEverything("SELECT COUNT(*) FROM %s WHERE c > 2 AND c <= 7 AND k = 5", 10, 10, 10, 50);
     }
 
     private static String someText()
     {
         char[] arr = new char[1024];
         for (int i = 0; i < arr.length; i++)
-            arr[i] = (char)(32 + ThreadLocalRandom.current().nextInt(95));
+            arr[i] = (char) (32 + ThreadLocalRandom.current().nextInt(95));
         return new String(arr);
     }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/PagingTest.java b/test/unit/org/apache/cassandra/cql3/PagingTest.java
index 75d73e5d0ded..b8ef073d26e5 100644
--- a/test/unit/org/apache/cassandra/cql3/PagingTest.java
+++ b/test/unit/org/apache/cassandra/cql3/PagingTest.java
@@ -33,7 +33,11 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 
 import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
-import org.apache.cassandra.locator.*;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
 import org.apache.cassandra.service.EmbeddedCassandraService;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
diff --git a/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java b/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java
index c799a286ab3f..04b76232a588 100644
--- a/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java
@@ -40,6 +40,7 @@
 import static org.apache.cassandra.service.QueryState.forInternalCalls;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PERSIST_PREPARED_STATEMENTS;
 
 public class PstmtPersistenceTest extends CQLTester
 {
@@ -118,6 +119,41 @@ public void testCachedPreparedStatements() throws Throwable
         assertEquals(3, numberOfStatementsOnDisk());
     }
 
+    @Test
+    public void testUnpersistedPreparedStatements() throws Throwable {
+        requireNetwork();
+        try
+        {
+            PERSIST_PREPARED_STATEMENTS.setBoolean(false);
+
+            assertEquals(0, numberOfStatementsOnDisk());
+
+            execute("CREATE KEYSPACE IF NOT EXISTS foo WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}");
+            execute("CREATE TABLE foo.unpersisted (key text PRIMARY KEY, val int)");
+
+            ClientState clientState = ClientState.forExternalCalls(InetSocketAddress.createUnresolved("127.0.0.1", 1234));
+
+            createTable("CREATE TABLE %s (pk int PRIMARY KEY, val text)");
+
+            String statement0 = "SELECT * FROM %s WHERE keyspace_name = ?";
+            String statement1 = "SELECT * FROM %s WHERE pk = ?";
+            String statement2 = "SELECT * FROM %s WHERE key = ?";
+            String statement3 = "SELECT * FROM %S WHERE key = ?";
+            prepareStatement(statement0, SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspaceTables.TABLES, clientState);
+            prepareStatement(statement1, clientState);
+            prepareStatement(statement2, "foo", "unpersisted", clientState);
+            clientState.setKeyspace("foo");
+            prepareStatement(statement1, clientState);
+            prepareStatement(statement3, "foo", "unpersisted", clientState);
+
+            assertEquals(0, numberOfStatementsOnDisk());
+        }
+        finally
+        {
+            PERSIST_PREPARED_STATEMENTS.setBoolean(true);
+        }
+    }
+
     private void validatePstmts(List<MD5Digest> stmtIds, QueryHandler handler)
     {
         QueryOptions optionsStr = QueryOptions.forInternalCalls(Collections.singletonList(UTF8Type.instance.fromString("foobar")));
diff --git a/test/unit/org/apache/cassandra/cql3/QueryInterceptorTest.java b/test/unit/org/apache/cassandra/cql3/QueryInterceptorTest.java
new file mode 100644
index 000000000000..0a298c482dc4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/QueryInterceptorTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.BatchStatement;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.SimpleStatement;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static org.junit.Assert.assertEquals;
+
+public class QueryInterceptorTest extends CQLTester
+{
+    @BeforeClass
+    public static void setup() throws Throwable
+    {
+        requireNetwork();
+    }
+
+    @After
+    public void cleanInterceptors() throws Throwable
+    {
+        QueryProcessor.instance.clearInterceptors();
+    }
+
+    @Test
+    public void returnsNoRows() throws Throwable
+    {
+        createTable("create table %s (id int primary key, v int)");
+        execute("insert into %s (id, v) values (0, 0)");
+        execute("insert into %s (id, v) values (1, 1)");
+
+        assertRows(execute("select * from %s where id = 1"), row(1, 1));
+
+        QueryProcessor.instance.registerInterceptor(new QueryInterceptor()
+        {
+            @Nullable
+            @Override
+            public ResultMessage interceptStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime)
+            {
+                if (statement instanceof SelectStatement)
+                {
+                    SelectStatement selectStatement = (SelectStatement)statement;
+                    if (selectStatement.table.keyspace.equals(keyspace()) && selectStatement.table.name.equals(currentTable()))
+                    {
+                        return generateResults();
+                    }
+                }
+                return null;
+            }
+        });
+
+        assertEquals(0, executeNet("select * from %s where id = 1").all().size());
+    }
+
+    @Test
+    public void altersExistingRows() throws Throwable
+    {
+        createTable("create table %s (id int primary key, v int)");
+        execute("insert into %s (id, v) values (0, 0)");
+        execute("insert into %s (id, v) values (1, 1)");
+
+        assertRows(execute("select * from %s where id = 1"), row(1, 1));
+
+        QueryProcessor.instance.registerInterceptor(new QueryInterceptor()
+        {
+            @Nullable
+            @Override
+            public ResultMessage interceptStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime)
+            {
+                if (statement instanceof SelectStatement)
+                {
+                    SelectStatement selectStatement = (SelectStatement)statement;
+                    if (selectStatement.table.keyspace.equals(keyspace()) && selectStatement.table.name.equals(currentTable()))
+                    {
+                        return generateResults(row(1, 2));
+                    }
+                }
+                return null;
+            }
+        });
+
+        List<Row> rows = executeNet("select * from %s where id = 1").all();
+
+        assertEquals(1, rows.size());
+        assertEquals(1, rows.get(0).getInt(0));
+        assertEquals(2, rows.get(0).getInt(1));
+    }
+
+    @Test
+    public void addsAdditionalRows() throws Throwable
+    {
+        createTable("create table %s (id int primary key, v int)");
+        execute("insert into %s (id, v) values (0, 0)");
+        execute("insert into %s (id, v) values (1, 1)");
+
+        assertRows(execute("select * from %s where id = 1"), row(1, 1));
+
+        QueryProcessor.instance.registerInterceptor(new QueryInterceptor()
+        {
+            @Nullable
+            @Override
+            public ResultMessage interceptStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime)
+            {
+                if (statement instanceof SelectStatement)
+                {
+                    SelectStatement selectStatement = (SelectStatement)statement;
+                    if (selectStatement.table.keyspace.equals(keyspace()) && selectStatement.table.name.equals(currentTable()))
+                    {
+                        return generateResults(row(1, 1), row(1, 2));
+                    }
+                }
+                return null;
+            }
+        });
+
+        List<Row> rows = executeNet("select * from %s where id = 1").all();
+
+        assertEquals(2, rows.size());
+        assertEquals(1, rows.get(0).getInt(0));
+        assertEquals(1, rows.get(0).getInt(1));
+        assertEquals(1, rows.get(1).getInt(0));
+        assertEquals(2, rows.get(1).getInt(1));
+    }
+
+    @Test
+    public void testInterceptBatchStatement() throws Throwable
+    {
+        createTable("create table %s (id int primary key, v int)");
+
+        BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED);
+        batch.add(new SimpleStatement(String.format("insert into %s.%s (id, v) values (0, 0)", keyspace(), currentTable())));
+        batch.add(new SimpleStatement(String.format("insert into %s.%s (id, v) values (1, 1)", keyspace(), currentTable())));
+        executeNet(batch);
+
+        assertRows(execute("select count(*) from %s"), row(2L));
+
+        // skip batch execution
+        QueryProcessor.instance.registerInterceptor(new QueryInterceptor()
+        {
+            @Override
+            public ResultMessage interceptBatchStatement(org.apache.cassandra.cql3.statements.BatchStatement batch,
+                                                         QueryState state,
+                                                         BatchQueryOptions options,
+                                                         Map<String, ByteBuffer> customPayload,
+                                                         Dispatcher.RequestTime requestTime)
+            {
+                return new ResultMessage.Void();
+            }
+        });
+
+        batch = new BatchStatement(BatchStatement.Type.LOGGED);
+        batch.add(new SimpleStatement(String.format("insert into %s.%s (id, v) values (3, 0)", keyspace(), currentTable())));
+        batch.add(new SimpleStatement(String.format("insert into %s.%s (id, v) values (4, 1)", keyspace(), currentTable())));
+        executeNet(batch);
+
+        // verify second batch is not inserted
+        assertRows(execute("select count(*) from %s"), row(2L));
+
+        // clear interceptor and inject default interceptor
+        QueryProcessor.instance.clearInterceptors();
+        QueryProcessor.instance.registerInterceptor(new QueryInterceptor() {});
+
+        batch = new BatchStatement(BatchStatement.Type.LOGGED);
+        batch.add(new SimpleStatement(String.format("insert into %s.%s (id, v) values (5, 0)", keyspace(), currentTable())));
+        batch.add(new SimpleStatement(String.format("insert into %s.%s (id, v) values (6, 1)", keyspace(), currentTable())));
+        executeNet(batch);
+
+        // verify third batch is inserted
+        assertRows(execute("select count(*) from %s"), row(4L));
+    }
+
+    private ResultMessage generateResults(Object[]... rows)
+    {
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        TableMetadata table = cfs.metadata();
+        ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(new ArrayList<>(table.columns()));
+        ResultSet resultSet = new ResultSet(resultMetadata);
+
+        for (int index = 0; index < rows.length; index++)
+        {
+            Object[] row = rows[index];
+            resultSet.addRow(Arrays.asList(Int32Type.instance.decompose((Integer)row[0]), Int32Type.instance.decompose((Integer)row[1])));
+        }
+        return new ResultMessage.Rows(resultSet);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java b/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java
index 2b2b78a7347a..1079b128165d 100644
--- a/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java
+++ b/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java
@@ -63,6 +63,7 @@ public class RandomSchemaTest extends CQLTester.InMemory
     {
         // make sure blob is always the same
         CassandraRelevantProperties.TEST_BLOB_SHARED_SEED.setInt(42);
+        CassandraRelevantProperties.VECTOR_FLOAT_ONLY.setBoolean(false);
 
         requireNetwork();
     }
diff --git a/test/unit/org/apache/cassandra/cql3/ReservedTypeNamesTest.java b/test/unit/org/apache/cassandra/cql3/ReservedTypeNamesTest.java
new file mode 100644
index 000000000000..fc2464084011
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/ReservedTypeNamesTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.exceptions.SyntaxException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+public class ReservedTypeNamesTest
+{
+    @Test
+    public void testUnquotedReservedTypeNames()
+    {
+        for (String reservedTypeName : ReservedTypeNames.reservedTypeNames)
+        {
+            assertTypeCreationParseFails(reservedTypeName);
+        }
+    }
+
+    @Test
+    public void testQuotedReservedTypeNames()
+    {
+        for (String reservedTypeName : ReservedTypeNames.reservedTypeNames)
+        {
+            String doubleQuotedName = CqlBuilder.maybeQuoteTypeName(reservedTypeName);
+            assertEquals('"' + reservedTypeName + '"', doubleQuotedName);
+
+            parseCreateType(doubleQuotedName);
+        }
+    }
+
+    @Test
+    public void testUnreservedNames()
+    {
+        Set<String> unreservedNames = Sets.newHashSet("mytype", "niceType", "hello", "cassandra");
+
+        for (String unreservedName: unreservedNames)
+        {
+            assertFalse(ReservedTypeNames.isReserved(unreservedName));
+            parseCreateType(unreservedName);
+        }
+    }
+
+    private void assertTypeCreationParseFails(String typeName)
+    {
+        try
+        {
+            parseCreateType(typeName);
+            fail(String.format("Reserved type name %s should not have parsed", typeName));
+        }
+        catch (SyntaxException exception)
+        {
+            // Expected
+        }
+    }
+
+    private void parseCreateType(String typeName)
+    {
+        QueryProcessor.parseStatement(String.format("CREATE TYPE ks.%s (id int)", typeName));
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java
index 3231d3b851a2..13f8b019c435 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java
@@ -27,6 +27,7 @@
 import java.util.Map;
 
 import com.google.common.base.Objects;
+
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
diff --git a/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java b/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java
index 93b09241ae5a..27d243dc045b 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java
@@ -32,6 +32,7 @@
 import org.apache.cassandra.exceptions.InvalidRequestException;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 
 /* ViewFilteringTest class has been split into multiple ones because of timeout issues (CASSANDRA-16670, CASSANDRA-17167)
  * Any changes here check if they apply to the other classes
@@ -371,15 +372,11 @@ public void testMVCreationSelectRestrictions() throws Throwable
 
         for (String badStatement : badStatements)
         {
-            try
+            assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() ->
             {
                 createView(badStatement);
                 Assert.fail("Create MV statement should have failed due to missing IS NOT NULL restriction: " + badStatement);
-            }
-            catch (RuntimeException e)
-            {
-                Assert.assertSame(InvalidRequestException.class, e.getCause().getClass());
-            }
+            });
         }
 
         List<String> goodStatements = Arrays.asList(
diff --git a/test/unit/org/apache/cassandra/cql3/ViewPKTest.java b/test/unit/org/apache/cassandra/cql3/ViewPKTest.java
index 1c2b9a6bb172..f51c7d284019 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewPKTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewPKTest.java
@@ -29,8 +29,9 @@
 import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
-import org.assertj.core.api.Assertions;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.junit.Assert.assertTrue;
 
 /*
@@ -112,24 +113,16 @@ public void testPrimaryKeyIsNotNull()
         catch (Exception e)
         {
             Throwable cause = e.getCause();
-            Assertions.assertThat(cause).isInstanceOf(SyntaxException.class);
-            Assertions.assertThat(cause.getMessage()).contains("mismatched input");
+            assertThat(cause).isInstanceOf(SyntaxException.class);
+            assertThat(cause.getMessage()).contains("mismatched input");
         }
 
         // Must include both when the partition key is composite
-        try
-        {
+        assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " +
                        "WHERE bigintval IS NOT NULL AND asciival IS NOT NULL " +
                        "PRIMARY KEY (bigintval, k, asciival)");
-            Assert.fail("Should fail if compound primary is not completely filtered as NOT NULL");
-        }
-        catch (Exception e)
-        {
-            Throwable cause = e.getCause();
-            Assertions.assertThat(cause).isInstanceOf(InvalidRequestException.class);
-            Assertions.assertThat(cause.getMessage()).contains("Primary key columns k must be restricted");
-        }
+        }).withMessageContaining("Primary key columns k must be restricted");
 
         dropTable("DROP TABLE %s");
 
@@ -146,24 +139,16 @@ public void testPrimaryKeyIsNotNull()
         catch (Exception e)
         {
             Throwable cause = e.getCause();
-            Assertions.assertThat(cause).isInstanceOf(SyntaxException.class);
-            Assertions.assertThat(cause.getMessage()).contains("mismatched input");
+            assertThat(cause).isInstanceOf(SyntaxException.class);
+            assertThat(cause.getMessage()).contains("mismatched input");
         }
 
         // Must still include both even when the partition key is composite
-        try
-        {
+        assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " +
                        "WHERE bigintval IS NOT NULL AND asciival IS NOT NULL " +
                        "PRIMARY KEY (bigintval, k, asciival)");
-            Assert.fail("Should fail if compound primary is not completely filtered as NOT NULL");
-        }
-        catch (Exception e)
-        {
-            Throwable cause = e.getCause();
-            Assertions.assertThat(cause).isInstanceOf(InvalidRequestException.class);
-            Assertions.assertThat(cause.getMessage()).contains("Primary key columns k must be restricted");
-        }
+        }).withMessageContaining("Primary key columns k must be restricted");
     }
 
     @Test
@@ -227,30 +212,17 @@ public void testCompoundPartitionKey() throws Throwable
                     Assert.fail("MV creation failed on " + def);
             }
 
-            try
-            {
+            assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> {
                 String query = "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL "
                                + asciival + "PRIMARY KEY ((" + def.name + ", k), asciival)";
                 createView("mv3_" + def.name, query);
+            });
 
-                Assert.fail("Should fail on duplicate name");
-            }
-            catch (Exception e)
-            {
-                Assertions.assertThat(e.getCause()).isInstanceOf(RequestValidationException.class);
-            }
-
-            try
-            {
+            assertThatExceptionOfType(RequestValidationException.class).isThrownBy(() -> {
                 String query = "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL "
                                + asciival + "PRIMARY KEY ((" + def.name + ", k), nonexistentcolumn)";
                 createView("mv4_" + def.name, query);
-                Assert.fail("Should fail with unknown base column");
-            }
-            catch (Exception e)
-            {
-                Assertions.assertThat(e.getCause()).isInstanceOf(RequestValidationException.class);
-            }
+            });
         }
 
         updateView("INSERT INTO %s (k, asciival, bigintval) VALUES (?, ?, from_json(?))", 0, "ascii text", "123123123123");
@@ -426,29 +398,13 @@ public void testMultipleNonPrimaryKeysInView()
                     "e int," +
                     "PRIMARY KEY ((a, b), c))");
 
-        try
-        {
+        assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL AND d IS NOT NULL AND e IS NOT NULL PRIMARY KEY ((d, a), b, e, c)");
-            Assert.fail("Should have rejected a query including multiple non-primary key base columns");
-        }
-        catch (Exception e)
-        {
-            Throwable cause = e.getCause();
-            Assertions.assertThat(cause).isInstanceOf(InvalidRequestException.class);
-            Assertions.assertThat(cause.getMessage()).contains("Cannot include more than one non-primary key column");
-        }
+        }).withMessageContaining("Cannot include more than one non-primary key column");
 
-        try
-        {
+        assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a IS NOT NULL AND b IS NOT NULL AND c IS NOT NULL AND d IS NOT NULL AND e IS NOT NULL PRIMARY KEY ((a, b), c, d, e)");
-            Assert.fail("Should have rejected a query including multiple non-primary key base columns");
-        }
-        catch (Exception e)
-        {
-            Throwable cause = e.getCause();
-            Assertions.assertThat(cause).isInstanceOf(InvalidRequestException.class);
-            Assertions.assertThat(cause.getMessage()).contains("Cannot include more than one non-primary key column");
-        }
+        }).withMessageContaining("Cannot include more than one non-primary key column");
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/cql3/ViewTest.java b/test/unit/org/apache/cassandra/cql3/ViewTest.java
index fa98a2f3be63..c318c971e750 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewTest.java
@@ -88,35 +88,17 @@ public void testStaticTable() throws Throwable
                     "val text, " +
                     "PRIMARY KEY(k,c))");
 
-        try
-        {
+        Assertions.assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE sval IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (sval,k,c)");
-            Assert.fail("Use of static column in a MV primary key should fail");
-        }
-        catch (Exception e)
-        {
-            Assert.assertTrue(e.getCause() instanceof InvalidRequestException);
-        }
+        });
 
-        try
-        {
+        Assertions.assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT val, sval FROM %s WHERE val IS NOT NULL AND  k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val, k, c)");
-            Assert.fail("Explicit select of static column in MV should fail");
-        }
-        catch (Exception e)
-        {
-            Assert.assertTrue(e.getCause() instanceof InvalidRequestException);
-        }
+        });
 
-        try
-        {
+        Assertions.assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
-            Assert.fail("Implicit select of static column in MV should fail");
-        }
-        catch (Exception e)
-        {
-            Assert.assertTrue(e.getCause() instanceof InvalidRequestException);
-        }
+        });
 
         createView("CREATE MATERIALIZED VIEW %s AS SELECT val,k,c FROM %s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
 
@@ -175,15 +157,9 @@ public void testCountersTable()
                     "k int PRIMARY KEY, " +
                     "count counter)");
 
-        try
-        {
+        Assertions.assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE count IS NOT NULL AND k IS NOT NULL PRIMARY KEY (count,k)");
-            Assert.fail("MV on counter should fail");
-        }
-        catch (Exception e)
-        {
-            Assert.assertTrue(e.getCause() instanceof InvalidRequestException);
-        }
+        });
     }
 
     @Test
@@ -193,16 +169,9 @@ public void testDurationsTable()
                     "k int PRIMARY KEY, " +
                     "result duration)");
 
-        try
-        {
+        Assertions.assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE result IS NOT NULL AND k IS NOT NULL PRIMARY KEY (result,k)");
-            Assert.fail("MV on duration should fail");
-        }
-        catch (Exception e)
-        {
-            Throwable cause = e.getCause();
-            Assert.assertEquals("duration type is not supported for PRIMARY KEY column 'result'", cause.getMessage());
-        }
+        }).withMessageContaining("duration type is not supported for PRIMARY KEY column 'result'");
     }
 
     @Test
@@ -462,14 +431,14 @@ private void testViewBuilderResume(int concurrentViewBuilders) throws Throwable
                                      "WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
 
         cfs.enableAutoCompaction();
-        List<? extends Future<?>> futures = CompactionManager.instance.submitBackground(cfs);
+        Future<?> future = CompactionManager.instance.submitBackground(cfs);
 
         //Force a second MV on the same base table, which will restart the first MV builder...
         createView("CREATE MATERIALIZED VIEW %s AS SELECT val, k, c FROM %s " +
                    "WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
 
         //Compact the base table
-        FBUtilities.waitOnFutures(futures);
+        FBUtilities.waitOnFuture(future);
 
         waitForViewBuild(mv1);
 
@@ -516,15 +485,10 @@ public void testDisableMaterializedViews() throws Throwable
         boolean enableMaterializedViews = DatabaseDescriptor.getMaterializedViewsEnabled();
         try
         {
-            DatabaseDescriptor.setMaterializedViewsEnabled(false);
-            createView("CREATE MATERIALIZED VIEW %s AS SELECT v FROM %s WHERE k IS NOT NULL AND v IS NOT NULL PRIMARY KEY (v, k)");
-            Assert.fail("Should not be able to create a materialized view if they are disabled");
-        }
-        catch (RuntimeException e)
-        {
-            Throwable cause = e.getCause();
-            Assertions.assertThat(cause).isInstanceOf(InvalidRequestException.class);
-            Assertions.assertThat(cause.getMessage()).contains("Materialized views are disabled");
+            Assertions.assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
+                DatabaseDescriptor.setMaterializedViewsEnabled(false);
+                createView("CREATE MATERIALIZED VIEW %s AS SELECT v FROM %s WHERE k IS NOT NULL AND v IS NOT NULL PRIMARY KEY (v, k)");
+            }).withMessageContaining("Materialized views are disabled");
         }
         finally
         {
diff --git a/test/unit/org/apache/cassandra/cql3/ViewTimesTest.java b/test/unit/org/apache/cassandra/cql3/ViewTimesTest.java
index 9c459363913d..c51f765b263d 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewTimesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewTimesTest.java
@@ -29,8 +29,8 @@
 import org.apache.cassandra.Util;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.assertj.core.api.Assertions;
 
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
@@ -263,18 +263,11 @@ public void testCreateMvWithTTL()
                     "val int) WITH default_time_to_live = 60");
 
         // Must NOT include "default_time_to_live" for Materialized View creation
-        try
-        {
+        assertThatExceptionOfType(InvalidRequestException.class).isThrownBy(() -> {
+
             createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " +
                        "WHERE k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (k,c) WITH default_time_to_live = 30");
-            fail("Should fail if TTL is provided for materialized view");
-        }
-        catch (RuntimeException e)
-        {
-            Throwable cause = e.getCause();
-            Assertions.assertThat(cause).isInstanceOf(InvalidRequestException.class);
-            Assertions.assertThat(cause.getMessage()).contains("Cannot set default_time_to_live for a materialized view");
-        }
+        }).withMessageContaining("Cannot set default_time_to_live for a materialized view");
     }
 
     @Test
@@ -296,8 +289,10 @@ public void testAlterMvWithNoZeroTTL() throws Throwable
         catch (Exception e)
         {
             // Make sure the message is clear. See CASSANDRA-16960
-            assertEquals("Forbidden default_time_to_live detected for a materialized view. Data in a materialized view always expire at the same time than the corresponding "
-                         + "data in the parent table. default_time_to_live must be set to zero, see CASSANDRA-12868 for more information",
+            assertEquals("Forbidden default_time_to_live detected for a materialized view. " +
+                         "Data in a materialized view always expires at the same time as " +
+                         "the corresponding data in the parent table. default_time_to_live " +
+                         "must be set to zero, see CASSANDRA-12868 for more information",
                          e.getMessage());
         }
     }
diff --git a/test/unit/org/apache/cassandra/cql3/WhereClauseExpressionTreeTest.java b/test/unit/org/apache/cassandra/cql3/WhereClauseExpressionTreeTest.java
new file mode 100644
index 000000000000..d4d0ff033726
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/WhereClauseExpressionTreeTest.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import org.junit.Test;
+
+import com.bpodgursky.jbool_expressions.parsers.ExprParser;
+import org.apache.cassandra.exceptions.SyntaxException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class WhereClauseExpressionTreeTest
+{
+    @Test(expected = SyntaxException.class)
+    public void cannotHaveEmptyWhereClause() throws Throwable
+    {
+        cqlParse("");
+    }
+
+    @Test
+    public void singleRelationWithoutEnclosure() throws Throwable
+    {
+        testExpression("a = 1");
+    }
+
+    @Test
+    public void singleRelationWithEnclosure() throws Throwable
+    {
+        testExpression("(a = 1)");
+    }
+
+    @Test
+    public void simpleAndExpressionWithRelationsWithoutEnclosure() throws Throwable
+    {
+        testExpression("a = 1 AND b = 1");
+    }
+
+    @Test
+    public void simpleAndExpressionWithRelationsWithEnclosure() throws Throwable
+    {
+        testExpression("(a = 1 AND b = 1)");
+    }
+
+    @Test
+    public void multipleAndExpressionWithRelations() throws Throwable
+    {
+        testExpression("a = 1 AND b = 1 AND c = 1");
+    }
+
+    @Test
+    public void disjunctionExpression() throws Throwable
+    {
+        testExpression("a = 1 AND b = 1 OR c = 1");
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        System.out.println(cqlParse("a = 1 OR b = 1 AND c = 1"));
+    }
+
+    @Test
+    public void precedenceIsMaintainedWithoutParentheses() throws Throwable
+    {
+        testExpression("a = 1 AND b = 1 OR c = 1");
+
+        testExpression("a = 1 OR b = 1 AND c = 1");
+
+        testExpression("a = 1 OR b = 1 OR c = 1 AND d = 1 OR e = 1");
+
+        testExpression("a = 1 AND b = 1 AND c = 1 OR d = 1 AND e = 1");
+    }
+
+    @Test
+    public void multipleDisjunctionExpression() throws Throwable
+    {
+        testExpression("(a = 1 AND b = 1) OR (c = 1 AND d = 1)");
+    }
+
+    @Test
+    public void disjunctionExpressionWithPrecedence() throws Throwable
+    {
+        testExpression("a = 1 AND (b = 1 OR (c = 1 AND d = 1 AND e = 1))");
+    }
+
+    @Test
+    public void flattenConjunction() throws Throwable
+    {
+        WhereClause clause = WhereClause.parse("a = 1 AND (b = 1 AND c = 1)");
+        WhereClause.ExpressionElement flattened = clause.root().flatten();
+        assertTrue(flattened instanceof WhereClause.AndElement);
+        assertEquals(3, ((WhereClause.ContainerElement) flattened).children.size());
+        assertEquals("a = 1 AND b = 1 AND c = 1", flattened.toString());
+    }
+
+    @Test
+    public void flattenDisjunction() throws Throwable
+    {
+        WhereClause clause = WhereClause.parse("a = 1 OR (b = 1 OR c = 1)");
+        WhereClause.ExpressionElement flattened = clause.root().flatten();
+        assertTrue(flattened instanceof WhereClause.OrElement);
+        assertEquals(3, ((WhereClause.ContainerElement) flattened).children.size());
+        assertEquals("a = 1 OR b = 1 OR c = 1", flattened.toString());
+    }
+
+    @Test
+    public void flattenDeeplyNested() throws Throwable
+    {
+        WhereClause.ExpressionElement flattened;
+
+        // deeper nesting, right
+        flattened = WhereClause.parse("a = 1 OR (b = 1 OR (c = 1 OR d = 1))").root().flatten();
+        assertEquals("a = 1 OR b = 1 OR c = 1 OR d = 1", flattened.toString());
+
+        // deeper nesting, left
+        flattened = WhereClause.parse("((a = 1 OR b = 1) OR c = 1) OR d = 1").root().flatten();
+        assertEquals("a = 1 OR b = 1 OR c = 1 OR d = 1", flattened.toString());
+    }
+
+
+    @Test
+    public void flattenMixed() throws Throwable
+    {
+        WhereClause.ExpressionElement flattened;
+
+        flattened = WhereClause.parse("a = 1 OR (b = 1 AND c = 1)").root().flatten();
+        assertEquals("a = 1 OR (b = 1 AND c = 1)", flattened.toString());
+
+        flattened = WhereClause.parse("(a = 1 OR (b = 1 OR c = 1)) AND (d = 1 AND (e = 1 OR f = 1))").root().flatten();
+        assertEquals("(a = 1 OR b = 1 OR c = 1) AND d = 1 AND (e = 1 OR f = 1)", flattened.toString());
+    }
+
+    @Test
+    public void conjunctiveFormEmpty() throws Throwable
+    {
+        WhereClause conj = WhereClause.empty().conjunctiveForm();
+        assertEquals(WhereClause.empty(), conj);
+        assertEquals(0, conj.root().expressions().size());
+        assertEquals(0, conj.root().relations().size());
+    }
+
+    @Test
+    public void conjunctiveFormSimple() throws Throwable
+    {
+        WhereClause.AndElement conj = WhereClause.parse("a = 1").root().conjunctiveForm();
+        assertEquals(1, conj.children.size());
+        assertEquals("a = 1", conj.toString());
+    }
+
+    @Test
+    public void conjunctiveFormSingleAnd() throws Throwable
+    {
+        WhereClause.AndElement conj = WhereClause.parse("a = 1 AND b = 1").root().conjunctiveForm();
+        assertEquals(2, conj.children.size());
+        assertEquals("a = 1 AND b = 1", conj.toString());
+    }
+
+    @Test
+    public void conjunctiveFormSingleOr() throws Throwable
+    {
+        WhereClause.AndElement conj = WhereClause.parse("a = 1 OR b = 1").root().conjunctiveForm();
+        assertEquals(1, conj.children.size());
+        assertEquals("a = 1 OR b = 1", conj.toString());
+    }
+
+    @Test
+    public void conjunctiveFormNested() throws Throwable
+    {
+        WhereClause.AndElement conj = WhereClause.parse("a = 1 AND (b = 1 AND c = 1)").root().conjunctiveForm();
+        assertEquals(3, conj.children.size());
+        assertEquals("a = 1 AND b = 1 AND c = 1", conj.toString());
+    }
+
+    @Test
+    public void rename() throws Throwable
+    {
+        WhereClause.ExpressionElement root = WhereClause.parse("a1 = 1 OR (b1 = 1 AND c1 = 1)").root();
+
+        WhereClause.ExpressionElement renamed1 =
+                root.rename(
+                        ColumnIdentifier.getInterned("a1", false),
+                        ColumnIdentifier.getInterned("a2", false));
+
+        assertEquals("a2 = 1 OR (b1 = 1 AND c1 = 1)", renamed1.toString());
+
+        WhereClause.ExpressionElement renamed2 =
+                root.rename(
+                        ColumnIdentifier.getInterned("b1", false),
+                        ColumnIdentifier.getInterned("b2", false));
+
+        assertEquals("a1 = 1 OR (b2 = 1 AND c1 = 1)", renamed2.toString());
+    }
+
+    @Test
+    public void randomTest() throws Throwable
+    {
+        for (int count = 0; count < CQLTester.getRandom().nextIntBetween(100, 1000); count++)
+            testExpression(randomExpression());
+    }
+
+    private void testExpression(String expression) throws Throwable
+    {
+        assertEquals("Failed to correctly parse: [" + expression + "]", jboolParse(expression), jboolParse(cqlParse(expression)));
+    }
+
+   private static String alphabet = "abcdefghijklmnopqrstuvwxyz";
+
+   private String randomExpression()
+   {
+       StringBuilder builder = new StringBuilder();
+
+       boolean applyPrecedence = CQLTester.getRandom().nextBoolean();
+
+       int numberOfElements = CQLTester.getRandom().nextIntBetween(1, 26);
+       int precedenceLevel = 0;
+       for (int element = 0; element < numberOfElements - 1; element++)
+       {
+           if (applyPrecedence && CQLTester.getRandom().nextIntBetween(0, 2) == 0)
+           {
+               builder.append("(");
+               precedenceLevel++;
+           }
+           builder.append(alphabet, element, element + 1);
+           builder.append(" = 1");
+           if (applyPrecedence && CQLTester.getRandom().nextIntBetween(0, 2) == 2 && precedenceLevel > 0)
+           {
+               builder.append(")");
+               precedenceLevel--;
+           }
+           builder.append(CQLTester.getRandom().nextBoolean() ? " AND " : " OR ");
+       }
+       builder.append(alphabet, numberOfElements - 1, numberOfElements);
+       builder.append(" = 1");
+       if (applyPrecedence)
+           while (precedenceLevel-- > 0)
+               builder.append(")");
+
+       return builder.toString();
+   }
+
+   private String cqlParse(String expression) throws Throwable
+   {
+       return WhereClause.parse(expression).root().toString();
+   }
+
+   private String jboolParse(String expression)
+   {
+       return ExprParser.parse(toJbool(expression)).toString();
+   }
+
+   private String toJbool(String cqlExpression)
+   {
+       return cqlExpression.replaceAll("AND", "&").replaceAll("OR", "|").replaceAll(" = 1", "");
+   }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/WhereClauseMutationTest.java b/test/unit/org/apache/cassandra/cql3/WhereClauseMutationTest.java
new file mode 100644
index 000000000000..fc071f8b6e98
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/WhereClauseMutationTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.Row;
+import org.apache.cassandra.cql3.statements.SelectOptions;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * The purpose of the test is to show the mutation of {@link WhereClause} relations
+ * by a {@link QueryInterceptor}.
+ *
+ * The test adds 2 rows with the keys of "original" and "mutated".
+ *
+ * Without the {@link QueryInterceptor} when "original" is select the value associated
+ * with the "original" row is returned.
+ *
+ * A {@link QueryInterceptor} is then added that changes the value of the primary key
+ * relation from "original" to "mutated" so when "original" is selected the value for
+ * the "mutated" row is returned.
+ */
+public class WhereClauseMutationTest extends CQLTester
+{
+    @BeforeClass
+    public static void setup() throws Throwable
+    {
+        requireNetwork();
+    }
+
+    @After
+    public void cleanInterceptors() throws Throwable
+    {
+        QueryProcessor.instance.clearInterceptors();
+    }
+
+    @Test
+    public void mutationTest() throws Throwable
+    {
+        createTable("create table %s (pk text, ck text, v text, primary key (pk, ck))");
+        execute("insert into %s (pk, ck, v) values ('original', 'clustering', 'value1')");
+        execute("insert into %s (pk, ck, v) values ('mutated', 'clustering', 'value2')");
+
+        assertRowsIgnoringOrder(execute("select * from %s where pk = 'original' and ck = 'clustering'"), row("original", "clustering", "value1"));
+
+        QueryProcessor.instance.registerInterceptor(new QueryInterceptor()
+        {
+            @Nullable
+            @Override
+            public ResultMessage interceptStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime)
+            {
+                if (statement instanceof SelectStatement)
+                {
+                    SelectStatement selectStatement = (SelectStatement)statement;
+                    // We only want to attempt mutation on our table
+                    if (selectStatement.table.keyspace.equals(keyspace()) && selectStatement.table.name.equals(currentTable()))
+                    {
+                        SelectStatement.RawStatement rawStatement = (SelectStatement.RawStatement)QueryProcessor.parseStatement(selectStatement.getRawCQLStatement());
+
+                        // Mutate the SelectStatement with a new WhereClause with the relations mutated
+                        rawStatement = new SelectStatement.RawStatement(new QualifiedName(rawStatement.keyspace(),
+                                                                                          rawStatement.name()),
+                                                                        rawStatement.parameters,
+                                                                        rawStatement.selectClause,
+                                                                        rawStatement.whereClause.mutateRelations(r -> mutateRelation(r)),
+                                                                        rawStatement.limit,
+                                                                        rawStatement.perPartitionLimit,
+                                                                        null,
+                                                                        SelectOptions.EMPTY);
+
+                        selectStatement = rawStatement.prepare(queryState.getClientState());
+                        return selectStatement.execute(queryState, options, requestTime);
+                    }
+                }
+                return null;
+            }
+        });
+
+        List<Row> rows = executeNet("select * from %s where pk = 'original' and ck = 'clustering'").all();
+        assertEquals(1, rows.size());
+        assertEquals("value2", rows.get(0).getString(2));
+    }
+
+    private Relation mutateRelation(Relation original)
+    {
+        // Only perform the mutation on single column relations
+        if (original instanceof SingleColumnRelation)
+        {
+            SingleColumnRelation singleColumnRelation = (SingleColumnRelation)original;
+            // Make sure we are only changing the primary key column
+            if (singleColumnRelation.getEntity().toCQLString().equals("pk"))
+                // Return a new SingleColumnRelation with the primary key column
+                // value changed to "mutated"
+                return new SingleColumnRelation(singleColumnRelation.getEntity(),
+                                                singleColumnRelation.operator(),
+                                                Constants.Literal.string("mutated"));
+        }
+        return original;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/CollectionFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/CollectionFctsTest.java
index 0ad299edee7e..f6b3704e159b 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/CollectionFctsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/CollectionFctsTest.java
@@ -45,45 +45,45 @@ public void testNotNumericCollection() throws Throwable
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v uuid, l list<text>, s set<boolean>, fl frozen<list<text>>, fs frozen<set<boolean>>)");
 
         // sum
-        assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_sum(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument v of type uuid",
                                   InvalidRequestException.class,
                                   "SELECT collection_sum(v) FROM %s");
-        assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_sum(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument l of type list<text>",
                                   InvalidRequestException.class,
                                   "SELECT collection_sum(l) FROM %s");
-        assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_sum(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument s of type set<boolean>",
                                   InvalidRequestException.class,
                                   "SELECT collection_sum(s) FROM %s");
-        assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_sum(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument fl of type frozen<list<text>>",
                                   InvalidRequestException.class,
                                   "SELECT collection_sum(fl) FROM %s");
-        assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_sum(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument fs of type frozen<set<boolean>>",
                                   InvalidRequestException.class,
                                   "SELECT collection_sum(fs) FROM %s");
 
         // avg
-        assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_avg(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument v of type uuid",
                                   InvalidRequestException.class,
                                   "SELECT collection_avg(v) FROM %s");
-        assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_avg(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument l of type list<text>",
                                   InvalidRequestException.class,
                                   "SELECT collection_avg(l) FROM %s");
-        assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_avg(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument s of type set<boolean>",
                                   InvalidRequestException.class,
                                   "SELECT collection_avg(s) FROM %s");
-        assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_avg(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument fl of type frozen<list<text>>",
                                   InvalidRequestException.class,
                                   "SELECT collection_avg(fl) FROM %s");
-        assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+        assertInvalidThrowMessage("Function system.collection_avg(numeric_set_or_list) requires a numeric set/list argument, " +
                                   "but found argument fs of type frozen<set<boolean>>",
                                   InvalidRequestException.class,
                                   "SELECT collection_avg(fs) FROM %s");
diff --git a/test/unit/org/apache/cassandra/cql3/functions/FunctionFactoryTest.java b/test/unit/org/apache/cassandra/cql3/functions/FunctionFactoryTest.java
index 8db1c182acd6..8fe941c138f9 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/FunctionFactoryTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/FunctionFactoryTest.java
@@ -30,6 +30,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.Duration;
 import org.apache.cassandra.cql3.UntypedResultSet;
@@ -48,7 +49,9 @@ public class FunctionFactoryTest extends CQLTester
     private static final FunctionFactory IDENTITY = new FunctionFactory("identity", FunctionParameter.anyType(true))
     {
         @Override
-        protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+        protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                       List<AbstractType<?>> argTypes,
+                                                       AbstractType<?> receiverType)
         {
             return new NativeScalarFunction(name.name, argTypes.get(0), argTypes.get(0))
             {
@@ -74,7 +77,9 @@ public ByteBuffer execute(Arguments arguments)
     private static final FunctionFactory TO_STRING = new FunctionFactory("tostring", FunctionParameter.anyType(false))
     {
         @Override
-        protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+        protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args,
+                                                       List<AbstractType<?>> argTypes,
+                                                       AbstractType<?> receiverType)
         {
             return new NativeScalarFunction(name.name, UTF8Type.instance, argTypes.get(0))
             {
diff --git a/test/unit/org/apache/cassandra/cql3/functions/IndexFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/IndexFctsTest.java
new file mode 100644
index 000000000000..c5b6b6dd7000
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/IndexFctsTest.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import org.junit.Test;
+
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.SAITester;
+
+public class IndexFctsTest extends SAITester
+{
+    @Test
+    public void testAnalyzeFunction() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        execute("INSERT INTO %s (k, v) VALUES (1, 'johnny apples seedlings')");
+        execute("INSERT INTO %s (k, v) VALUES (2, null)");
+
+        assertRows(execute("SELECT k, sai_analyze(v, ?) FROM %s",
+                           "{\n" +
+                           "\t\"tokenizer\":{\"name\":\"whitespace\"},\n" +
+                           "\t\"tokenizer\":{\"name\":\"whitespace\"},\n" +
+                           "\t\"filters\":[{\"name\":\"porterstem\"}]\n" +
+                           '}'),
+                   row(1, list("johnni", "appl", "seedl")),
+                   row(2, null));
+
+        assertInvalidThrowMessage("Function system.sai_analyze requires a non-null json_analyzer parameter (2nd argument)",
+                                  InvalidRequestException.class,
+                                  "SELECT sai_analyze(v, null) FROM %s");
+
+        assertInvalidThrowMessage("Function system.sai_analyze unable to analyze text=abc json_analyzer=def",
+                                  InvalidRequestException.class,
+                                  "SELECT sai_analyze('abc', 'def') FROM %s");
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/OperationFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/OperationFctsTest.java
index c46f6a92db9a..50d877542875 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/OperationFctsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/OperationFctsTest.java
@@ -866,6 +866,16 @@ public void testOperationsWithDuration() throws Throwable
                              "SELECT time / 10m FROM %s WHERE pk = 1");
         assertInvalidMessage("the operation 'date - duration' failed: The duration must have a day precision. Was: 10m",
                              "SELECT * FROM %s WHERE pk = 1 AND time > ? - 10m", toDate("2016-10-04"));
+
+        // test overflow errors
+        assertInvalidMessage("is greater than max supported date",
+                             "INSERT INTO %s (pk, time, v) VALUES (2, '+5881581-01-01', 7)");
+        assertInvalidMessage("is greater than max supported date",
+                             "INSERT INTO %s (pk, time, v) VALUES (4, '+5881580-01-01' + 1y, 9)");
+        assertInvalidMessage("is less than min supported date",
+                             "INSERT INTO %s (pk, time, v) VALUES (3, '-5877642-01-01', 8)");
+        assertInvalidMessage("is less than min supported date",
+                             "INSERT INTO %s (pk, time, v) VALUES (5, '-5877640-01-01' - 2y, 10)");
     }
 
     private Date toTimestamp(String timestampAsString)
diff --git a/test/unit/org/apache/cassandra/cql3/functions/UDAggregateTest.java b/test/unit/org/apache/cassandra/cql3/functions/UDAggregateTest.java
new file mode 100644
index 000000000000..123cc3d5f79c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/UDAggregateTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class UDAggregateTest extends CQLTester
+{
+    @Test
+    public void testNewKeyspace()
+    {
+        String oldKeyspaceName = "old_ks";
+
+        UserType userType = new UserType(oldKeyspaceName, ByteBufferUtil.bytes("a"),
+                                         Arrays.asList(FieldIdentifier.forUnquoted("a1"),
+                                                       FieldIdentifier.forUnquoted("a2"),
+                                                       FieldIdentifier.forUnquoted("a3")),
+                                         Arrays.asList(IntegerType.instance,
+                                                       IntegerType.instance,
+                                                       IntegerType.instance),
+                                         true);
+
+        String functionName = "my_func";
+        UDFunction oldFunction = createUDFunction(oldKeyspaceName, functionName, userType);
+        UDAggregate aggr = UDAggregate.create(Collections.singleton(oldFunction),
+                                              new FunctionName(oldKeyspaceName, "my_aggregate"),
+                                              Arrays.asList(userType, Int32Type.instance),
+                                              Int32Type.instance,
+                                              new FunctionName(oldKeyspaceName, functionName),
+                                              null,
+                                              Int32Type.instance,
+                                              null,
+                                              false);
+
+        String newKeyspaceName = "new_ks";
+        UDFunction newFunction = createUDFunction(newKeyspaceName, functionName, userType);
+        UDAggregate newAggr = aggr.withNewKeyspace(newKeyspaceName, Collections.singletonList(newFunction), Types.of(userType));
+
+        assertNotEquals(newAggr.name, aggr.name);
+        assertEquals(newAggr.elementKeyspace(), newKeyspaceName);
+    }
+
+    private UDFunction createUDFunction(String oldKeyspaceName, String functionName, UserType userType)
+    {
+        return UDFunction.create(new FunctionName(oldKeyspaceName, functionName),
+                                 Arrays.asList(ColumnIdentifier.getInterned("state", false),
+                                               ColumnIdentifier.getInterned("val", false),
+                                               ColumnIdentifier.getInterned("val2", false)),
+                                 Arrays.asList(Int32Type.instance, userType, Int32Type.instance),
+                                 Int32Type.instance,
+                                 true,
+                                 "java",
+                                 "return val2;",
+                                 false,
+                                 false,
+                                 Collections.emptyList());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/VectorFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/VectorFctsTest.java
index 691f8b6ef325..0f40bcf68e2e 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/VectorFctsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/VectorFctsTest.java
@@ -18,170 +18,153 @@
 
 package org.apache.cassandra.cql3.functions;
 
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.commons.lang3.ArrayUtils;
+import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
-import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.assertj.core.api.Assertions;
 
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
 
-@RunWith(Parameterized.class)
 public class VectorFctsTest extends CQLTester
 {
-    @Parameterized.Parameter
-    public String function;
-
-    @Parameterized.Parameter(1)
-    public VectorSimilarityFunction luceneFunction;
-
-    @Parameterized.Parameters(name = "{index}: function={0}")
-    public static Collection<Object[]> data()
+    @BeforeClass
+    public static void setupClass()
     {
-        return Arrays.asList(new Object[][]{
-        { "system.similarity_cosine", VectorSimilarityFunction.COSINE },
-        { "system.similarity_euclidean", VectorSimilarityFunction.EUCLIDEAN },
-        { "system.similarity_dot_product", VectorSimilarityFunction.DOT_PRODUCT }
-        });
+        VECTOR_FLOAT_ONLY.setBoolean(false);
     }
 
     @Test
-    public void testVectorSimilarityFunction()
+    public void randomVectorFunction() throws Throwable
     {
-        createTable(KEYSPACE, "CREATE TABLE %s (pk int PRIMARY KEY, value vector<float, 2>, " +
-                              "l list<float>, " + // lists shouldn't be accepted by the functions
-                              "fl frozen<list<float>>, " + // frozen lists shouldn't be accepted by the functions
-                              "v1 vector<float, 1>, " + // 1-dimension vector to test missmatching dimensions
-                              "v_int vector<int, 2>, " + // int vectors shouldn't be accepted by the functions
-                              "v_double vector<double, 2>)");// double vectors shouldn't be accepted by the functions
-
-        float[] values = new float[]{ 1f, 2f };
-        CQLTester.Vector<Float> vector = vector(ArrayUtils.toObject(values));
-        Object[] similarity = row(luceneFunction.compare(values, values));
-
-        // basic functionality
-        execute("INSERT INTO %s (pk, value, l, fl, v1, v_int, v_double) VALUES (0, ?, ?, ?, ?, ?, ?)",
-                vector, list(1f, 2f), list(1f, 2f), vector(1f), vector(1, 2), vector(1d, 2d));
-        assertRows(execute("SELECT " + function + "(value, value) FROM %s"), similarity);
-
-        // literals
-        assertRows(execute("SELECT " + function + "(value, [1, 2]) FROM %s"), similarity);
-        assertRows(execute("SELECT " + function + "([1, 2], value) FROM %s"), similarity);
-        assertRows(execute("SELECT " + function + "([1, 2], [1, 2]) FROM %s"), similarity);
-
-        // bind markers
-        assertRows(execute("SELECT " + function + "(value, ?) FROM %s", vector), similarity);
-        assertRows(execute("SELECT " + function + "(?, value) FROM %s", vector), similarity);
-        assertThatThrownBy(() -> execute("SELECT " + function + "(?, ?) FROM %s", vector, vector))
-        .hasMessageContaining("Cannot infer type of argument ?");
-
-        // bind markers with type hints
-        assertRows(execute("SELECT " + function + "((vector<float, 2>) ?, ?) FROM %s", vector, vector), similarity);
-        assertRows(execute("SELECT " + function + "(?, (vector<float, 2>) ?) FROM %s", vector, vector), similarity);
-        assertRows(execute("SELECT " + function + "((vector<float, 2>) ?, (vector<float, 2>) ?) FROM %s", vector, vector), similarity);
-
-        // bind markers and literals
-        assertRows(execute("SELECT " + function + "([1, 2], ?) FROM %s", vector), similarity);
-        assertRows(execute("SELECT " + function + "(?, [1, 2]) FROM %s", vector), similarity);
-        assertRows(execute("SELECT " + function + "([1, 2], ?) FROM %s", vector), similarity);
-
-        // wrong column types with columns
-        assertThatThrownBy(() -> execute("SELECT " + function + "(l, value) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument l of type list<float>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(fl, value) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument fl of type frozen<list<float>>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(value, l) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument l of type list<float>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(value, fl) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument fl of type frozen<list<float>>");
-
-        // wrong column types with columns and literals
-        assertThatThrownBy(() -> execute("SELECT " + function + "(l, [1, 2]) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument l of type list<float>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(fl, [1, 2]) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument fl of type frozen<list<float>>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], l) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument l of type list<float>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], fl) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument fl of type frozen<list<float>>");
-
-        // wrong column types with cast literals
-        assertThatThrownBy(() -> execute("SELECT " + function + "((List<Float>)[1, 2], [3, 4]) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument (list<float>)[1, 2] of type frozen<list<float>>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "((List<Float>)[1, 2], (List<Float>)[3, 4]) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument (list<float>)[1, 2] of type frozen<list<float>>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], (List<Float>)[3, 4]) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument (list<float>)[3, 4] of type frozen<list<float>>");
-
-        // wrong non-float vectors
-        assertThatThrownBy(() -> execute("SELECT " + function + "(v_int, [1, 2]) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument v_int of type vector<int, 2>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(v_double, [1, 2]) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument v_double of type vector<double, 2>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], v_int) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument v_int of type vector<int, 2>");
-        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], v_double) FROM %s"))
-        .hasMessageContaining("Function " + function + " requires a float vector argument, but found argument v_double of type vector<double, 2>");
-
-        // mismatching dimensions with literals
-        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], [3]) FROM %s", vector(1)))
-        .hasMessageContaining("All arguments must have the same vector dimensions");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(value, [1]) FROM %s", vector(1)))
-        .hasMessageContaining("All arguments must have the same vector dimensions");
-        assertThatThrownBy(() -> execute("SELECT " + function + "([1], value) FROM %s", vector(1)))
-        .hasMessageContaining("All arguments must have the same vector dimensions");
-
-        // mismatching dimensions with bind markers
-        assertThatThrownBy(() -> execute("SELECT " + function + "((vector<float, 1>) ?, value) FROM %s", vector(1)))
-        .hasMessageContaining("All arguments must have the same vector dimensions");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(value, (vector<float, 1>) ?) FROM %s", vector(1)))
-        .hasMessageContaining("All arguments must have the same vector dimensions");
-        assertThatThrownBy(() -> execute("SELECT " + function + "((vector<float, 2>) ?, (vector<float, 1>) ?) FROM %s", vector(1, 2), vector(1)))
-        .hasMessageContaining("All arguments must have the same vector dimensions");
-
-        // mismatching dimensions with columns
-        assertThatThrownBy(() -> execute("SELECT " + function + "(value, v1) FROM %s"))
-        .hasMessageContaining("All arguments must have the same vector dimensions");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(v1, value) FROM %s"))
-        .hasMessageContaining("All arguments must have the same vector dimensions");
-
-        // null arguments with literals
-        assertRows(execute("SELECT " + function + "(value, null) FROM %s"), row((Float) null));
-        assertRows(execute("SELECT " + function + "(null, value) FROM %s"), row((Float) null));
-        assertThatThrownBy(() -> execute("SELECT " + function + "(null, null) FROM %s"))
-        .hasMessageContaining("Cannot infer type of argument NULL in call to function " + function);
-
-        // null arguments with bind markers
-        assertRows(execute("SELECT " + function + "(value, ?) FROM %s", (CQLTester.Vector<Float>) null), row((Float) null));
-        assertRows(execute("SELECT " + function + "(?, value) FROM %s", (CQLTester.Vector<Float>) null), row((Float) null));
-        assertThatThrownBy(() -> execute("SELECT " + function + "(?, ?) FROM %s", null, null))
-        .hasMessageContaining("Cannot infer type of argument ? in call to function " + function);
-
-        // test all-zero vectors, only cosine similarity should reject them
-        if (luceneFunction == VectorSimilarityFunction.COSINE)
+        createTable("CREATE TABLE %s (pk int primary key, value vector<float, 2>)");
+
+        // correct usage
+        execute("INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, -1, 1))");
+        Assert.assertEquals(1, execute("SELECT value FROM %s WHERE pk = 0").size());
+
+        // wrong number of arguments
+        assertInvalidThrowMessage("Invalid number of arguments for function system.random_float_vector(literal_int, float, float)",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector())");
+        assertInvalidThrowMessage("Invalid number of arguments for function system.random_float_vector(literal_int, float, float)",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, -1))");
+        assertInvalidThrowMessage("Invalid number of arguments for function system.random_float_vector(literal_int, float, float)",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, -1, 1, 0))");
+
+        // mandatory arguments
+        assertInvalidThrowMessage("Function system.random_float_vector(literal_int, float, float) requires a literal_int argument, " +
+                                  "but found NULL",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(null, null, null))");
+        assertInvalidThrowMessage("Function system.random_float_vector(literal_int, float, float) requires a literal_int argument, " +
+                                  "but found NULL",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(null, -1, 1))");
+        assertInvalidThrowMessage("Min argument of function system.random_float_vector(literal_int, float, float) must not be null",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, null, null))");
+        assertInvalidThrowMessage("Max argument of function system.random_float_vector(literal_int, float, float) must not be null",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, -1, null))");
+        assertInvalidThrowMessage("Min argument of function system.random_float_vector(literal_int, float, float) must not be null",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, null, 1))");
+
+        // wrong argument types
+        assertInvalidThrowMessage("Function system.random_float_vector(literal_int, float, float) requires a literal_int argument, " +
+                                  "but found 'a'",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector('a', -1, 1))");
+        assertInvalidThrowMessage("Function system.random_float_vector(literal_int, float, float) requires a literal_int argument, " +
+                                  "but found system.\"_add\"(1, 1)",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(1 + 1, -1, 1))");
+        assertInvalidThrowMessage("Function system.random_float_vector(literal_int, float, float) requires a literal_int argument, " +
+                                  "but found value",
+                                  InvalidRequestException.class,
+                                  "SELECT random_float_vector(value, -1, 1) FROM %s");
+        assertInvalidThrowMessage("Function system.random_float_vector(literal_int, float, float) requires a literal_int argument, " +
+                                  "but found 1 + 1",
+                                  InvalidRequestException.class,
+                                  "SELECT random_float_vector(1 + 1, -1, 1) FROM %s");
+
+        // wrong argument values
+        assertInvalidThrowMessage("Max value must be greater than min value",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, 1, -1))");
+        assertInvalidThrowMessage("Max value must be finite",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, 0, " + Float.NaN + "))");
+        assertInvalidThrowMessage("Min value must be finite",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(2, " + Float.NEGATIVE_INFINITY + ", 0))");
+
+        // correct function with wrong receiver type
+        assertInvalidThrowMessage("Type error: cannot assign result of function system.random_float_vector " +
+                                  "(type vector<float, 1>) to value (type vector<float, 2>)",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, random_float_vector(1, -1, 1))");
+
+        // test select
+        for (int dimension : new int[]{ 1, 2, 3, 10, 1000 })
         {
-            String expected = "Function " + function + " doesn't support all-zero vectors";
-            assertThatThrownBy(() -> execute("SELECT " + function + "(value, [0, 0]) FROM %s")) .hasMessageContaining(expected);
-            assertThatThrownBy(() -> execute("SELECT " + function + "([0, 0], value) FROM %s")).hasMessageContaining(expected);
+            assertSelectRandomVectorFunction(dimension, -1, 1);
+            assertSelectRandomVectorFunction(dimension, 0, 1);
+            assertSelectRandomVectorFunction(dimension, -1.5f, 1.5f);
+            assertSelectRandomVectorFunction(dimension, 0.999999f, 1);
+            assertSelectRandomVectorFunction(dimension, 0, 0.000001f);
+            assertSelectRandomVectorFunction(dimension, -Float.MAX_VALUE, Float.MAX_VALUE);
         }
-        else
+    }
+
+    private void assertSelectRandomVectorFunction(int dimension, float min, float max)
+    {
+        String functionCall = String.format("random_float_vector(%d, %f, %f)", dimension, min, max);
+        String select = "SELECT " + functionCall + " FROM %s";
+
+        for (int i = 0; i < 100; i++)
         {
-            float expected = luceneFunction.compare(values, new float[]{ 0, 0 });
-            assertRows(execute("SELECT " + function + "(value, [0, 0]) FROM %s"), row(expected));
-            assertRows(execute("SELECT " + function + "([0, 0], value) FROM %s"), row(expected));
+            UntypedResultSet rs = execute(select);
+            Assertions.assertThat(rs).isNotEmpty();
+            Assertions.assertThat(rs.one().getVector("system." + functionCall, FloatType.instance, dimension))
+                      .hasSize(dimension)
+                      .allSatisfy(v -> Assertions.assertThat(v).isBetween(min, max));
         }
+    }
 
-        // not-assignable element types
-        assertThatThrownBy(() -> execute("SELECT " + function + "(value, ['a', 'b']) FROM %s WHERE pk=0"))
-            .hasMessageContaining("Type error: ['a', 'b'] cannot be passed as argument 1");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(['a', 'b'], value) FROM %s WHERE pk=0"))
-            .hasMessageContaining("Type error: ['a', 'b'] cannot be passed as argument 0");
-        assertThatThrownBy(() -> execute("SELECT " + function + "(['a', 'b'], ['a', 'b']) FROM %s WHERE pk=0"))
-            .hasMessageContaining("Type error: ['a', 'b'] cannot be passed as argument 0");
+    @Test
+    public void normalizeL2Function() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 2>)");
+
+        execute("INSERT INTO %s (k, v) VALUES (0, ?)", vector(3.0f, 4.0f));
+
+        assertRows(execute("SELECT normalize_l2(v) FROM %s"), row(vector(0.6f, 0.8f)));
+        assertRows(execute("SELECT k, normalize_l2((vector<float, 2>) null) FROM %s"), row(0, null));
+
+        assertInvalidThrowMessage("Invalid number of arguments for function system.normalize_l2(vector<float, n>)",
+                                  InvalidRequestException.class,
+                                  "SELECT normalize_l2() FROM %s");
+        assertInvalidThrowMessage("Invalid number of arguments for function system.normalize_l2(vector<float, n>)",
+                                  InvalidRequestException.class,
+                                  "SELECT normalize_l2(v, 1) FROM %s");
+        assertInvalidThrowMessage("Function system.normalize_l2(vector<float, n>) requires a float vector argument, " +
+                                  "but found argument 123 of type int",
+                                  InvalidRequestException.class,
+                                  "SELECT normalize_l2(123) FROM %s");
     }
+
+    @SafeVarargs
+    protected final <T> Vector<T> vector(T... values)
+    {
+        return new Vector<>(values);
+    }
+
 }
diff --git a/test/unit/org/apache/cassandra/cql3/functions/VectorSimilarityFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/VectorSimilarityFctsTest.java
new file mode 100644
index 000000000000..e2c3fd11310f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/VectorSimilarityFctsTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.cassandra.cql3.CQLTester;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@RunWith(Parameterized.class)
+public class VectorSimilarityFctsTest extends CQLTester
+{
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    @Parameterized.Parameter
+    public String function;
+
+    @Parameterized.Parameter(1)
+    public VectorSimilarityFunction rawFunction;
+
+    @Parameterized.Parameters(name = "{index}: function={0}")
+    public static Collection<Object[]> data()
+    {
+        return Arrays.asList(new Object[][]{
+        { "system.similarity_cosine", VectorSimilarityFunction.COSINE },
+        { "system.similarity_euclidean", VectorSimilarityFunction.EUCLIDEAN },
+        { "system.similarity_dot_product", VectorSimilarityFunction.DOT_PRODUCT }
+        });
+    }
+
+    @BeforeClass
+    public static void setupClass()
+    {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
+    }
+
+    @Test
+    public void testVectorSimilarityFunction()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int PRIMARY KEY, value vector<float, 2>, " +
+                              "l list<float>, " + // lists shouldn't be accepted by the functions
+                              "fl frozen<list<float>>, " + // frozen lists shouldn't be accepted by the functions
+                              "v1 vector<float, 1>, " + // 1-dimension vector to test missmatching dimensions
+                              "v_int vector<int, 2>, " + // int vectors shouldn't be accepted by the functions
+                              "v_double vector<double, 2>)");// double vectors shouldn't be accepted by the functions
+
+        float[] rawVector = new float[]{ 1f, 2f };
+        Vector<Float> vector = vector(ArrayUtils.toObject(rawVector));
+        VectorFloat<?> v = vts.createFloatVector(rawVector);
+        Object[] similarity = row(rawFunction.compare(v, v));
+
+        // basic functionality
+        execute("INSERT INTO %s (pk, value, l, fl, v1, v_int, v_double) VALUES (0, ?, ?, ?, ?, ?, ?)",
+                vector, list(1f, 2f), list(1f, 2f), vector(1f), vector(1, 2), vector(1d, 2d));
+        assertRows(execute("SELECT " + function + "(value, value) FROM %s"), similarity);
+
+        // literals
+        assertRows(execute("SELECT " + function + "(value, [1, 2]) FROM %s"), similarity);
+        assertRows(execute("SELECT " + function + "([1, 2], value) FROM %s"), similarity);
+        assertRows(execute("SELECT " + function + "([1, 2], [1, 2]) FROM %s"), similarity);
+
+        // bind markers
+        assertRows(execute("SELECT " + function + "(value, ?) FROM %s", vector), similarity);
+        assertRows(execute("SELECT " + function + "(?, value) FROM %s", vector), similarity);
+        assertThatThrownBy(() -> execute("SELECT " + function + "(?, ?) FROM %s", vector, vector))
+        .hasMessageContaining("Cannot infer type of argument ?");
+
+        // bind markers with type hints
+        assertRows(execute("SELECT " + function + "((vector<float, 2>) ?, ?) FROM %s", vector, vector), similarity);
+        assertRows(execute("SELECT " + function + "(?, (vector<float, 2>) ?) FROM %s", vector, vector), similarity);
+        assertRows(execute("SELECT " + function + "((vector<float, 2>) ?, (vector<float, 2>) ?) FROM %s", vector, vector), similarity);
+
+        // bind markers and literals
+        assertRows(execute("SELECT " + function + "([1, 2], ?) FROM %s", vector), similarity);
+        assertRows(execute("SELECT " + function + "(?, [1, 2]) FROM %s", vector), similarity);
+        assertRows(execute("SELECT " + function + "([1, 2], ?) FROM %s", vector), similarity);
+
+        // wrong column types with columns
+        assertThatThrownBy(() -> execute("SELECT " + function + "(l, value) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument l of type list<float>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(fl, value) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument fl of type frozen<list<float>>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(value, l) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument l of type list<float>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(value, fl) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument fl of type frozen<list<float>>");
+
+        // wrong column types with columns and literals
+        assertThatThrownBy(() -> execute("SELECT " + function + "(l, [1, 2]) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument l of type list<float>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(fl, [1, 2]) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument fl of type frozen<list<float>>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], l) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument l of type list<float>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], fl) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument fl of type frozen<list<float>>");
+
+        // wrong column types with cast literals
+        assertThatThrownBy(() -> execute("SELECT " + function + "((List<Float>)[1, 2], [3, 4]) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument (list<float>)[1, 2] of type frozen<list<float>>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "((List<Float>)[1, 2], (List<Float>)[3, 4]) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument (list<float>)[3, 4] of type frozen<list<float>>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], (List<Float>)[3, 4]) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument (list<float>)[3, 4] of type frozen<list<float>>");
+
+        // wrong non-float vectors
+        assertThatThrownBy(() -> execute("SELECT " + function + "(v_int, [1, 2]) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument v_int of type vector<int, 2>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(v_double, [1, 2]) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument v_double of type vector<double, 2>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], v_int) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument v_int of type vector<int, 2>");
+        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], v_double) FROM %s"))
+        .hasMessageContaining("requires a float vector argument, but found argument v_double of type vector<double, 2>");
+
+        // mismatching dimensions with literals
+        assertThatThrownBy(() -> execute("SELECT " + function + "([1, 2], [3]) FROM %s", vector(1)))
+        .hasMessageContaining("All arguments must have the same vector dimensions");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(value, [1]) FROM %s", vector(1)))
+        .hasMessageContaining("All arguments must have the same vector dimensions");
+        assertThatThrownBy(() -> execute("SELECT " + function + "([1], value) FROM %s", vector(1)))
+        .hasMessageContaining("All arguments must have the same vector dimensions");
+
+        // mismatching dimensions with bind markers
+        assertThatThrownBy(() -> execute("SELECT " + function + "((vector<float, 1>) ?, value) FROM %s", vector(1)))
+        .hasMessageContaining("All arguments must have the same vector dimensions");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(value, (vector<float, 1>) ?) FROM %s", vector(1)))
+        .hasMessageContaining("All arguments must have the same vector dimensions");
+        assertThatThrownBy(() -> execute("SELECT " + function + "((vector<float, 2>) ?, (vector<float, 1>) ?) FROM %s", vector(1, 2), vector(1)))
+        .hasMessageContaining("All arguments must have the same vector dimensions");
+
+        // mismatching dimensions with columns
+        assertThatThrownBy(() -> execute("SELECT " + function + "(value, v1) FROM %s"))
+        .hasMessageContaining("All arguments must have the same vector dimensions");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(v1, value) FROM %s"))
+        .hasMessageContaining("All arguments must have the same vector dimensions");
+
+        // null arguments with literals
+        assertRows(execute("SELECT " + function + "(value, null) FROM %s"), row((Float) null));
+        assertRows(execute("SELECT " + function + "(null, value) FROM %s"), row((Float) null));
+        assertThatThrownBy(() -> execute("SELECT " + function + "(null, null) FROM %s"))
+        .hasMessageContaining("Cannot infer type of argument NULL in call to function " + function);
+
+        // null arguments with bind markers
+        assertRows(execute("SELECT " + function + "(value, ?) FROM %s", (Vector<Float>) null), row((Float) null));
+        assertRows(execute("SELECT " + function + "(?, value) FROM %s", (Vector<Float>) null), row((Float) null));
+        assertThatThrownBy(() -> execute("SELECT " + function + "(?, ?) FROM %s", null, null))
+        .hasMessageContaining("Cannot infer type of argument ? in call to function " + function);
+
+        // test all-zero vectors, only cosine similarity should reject them
+        if (rawFunction == VectorSimilarityFunction.COSINE)
+        {
+            String expected = "doesn't support all-zero vectors";
+            assertThatThrownBy(() -> execute("SELECT " + function + "(value, [0, 0]) FROM %s")) .hasMessageContaining(expected);
+            assertThatThrownBy(() -> execute("SELECT " + function + "([0, 0], value) FROM %s")).hasMessageContaining(expected);
+        }
+        else
+        {
+            float expected = rawFunction.compare(v, vts.createFloatVector(new float[]{ 0, 0 }));
+            assertRows(execute("SELECT " + function + "(value, [0, 0]) FROM %s"), row(expected));
+            assertRows(execute("SELECT " + function + "([0, 0], value) FROM %s"), row(expected));
+        }
+
+        // not-assignable element types
+        assertThatThrownBy(() -> execute("SELECT " + function + "(value, ['a', 'b']) FROM %s WHERE pk=0"))
+            .hasMessageContaining("Type error: ['a', 'b'] cannot be passed as argument 1");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(['a', 'b'], value) FROM %s WHERE pk=0"))
+            .hasMessageContaining("Type error: ['a', 'b'] cannot be passed as argument 0");
+        assertThatThrownBy(() -> execute("SELECT " + function + "(['a', 'b'], ['a', 'b']) FROM %s WHERE pk=0"))
+            .hasMessageContaining("Type error: ['a', 'b'] cannot be passed as argument 0");
+    }
+
+    @SafeVarargs
+    protected final <T> Vector<T> vector(T... values)
+    {
+        return new Vector<>(values);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
index 1ca8da1163e5..94d5c7435f98 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
@@ -28,6 +28,8 @@
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.VectorType;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
+
 /**
  * {@link ColumnMaskQueryTester} for {@link DefaultMaskingFunction}.
  */
@@ -36,6 +38,8 @@ public class ColumnMaskQueryWithDefaultTest extends ColumnMaskQueryTester
     @Parameterized.Parameters(name = "order={0}, mask={1}, type={2}, value={3}")
     public static Collection<Object[]> options()
     {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
+
         List<Object[]> options = new ArrayList<>();
         for (String order : Arrays.asList("ASC", "DESC"))
         {
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
index fd139eeaf4d3..84d88e5444fe 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
@@ -29,6 +29,8 @@
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.VectorType;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
+
 /**
  * {@link ColumnMaskQueryTester} for {@link ReplaceMaskingFunction}.
  */
@@ -37,6 +39,8 @@ public class ColumnMaskQueryWithReplaceTest extends ColumnMaskQueryTester
     @Parameterized.Parameters(name = "order={0}, mask={1}, type={2}, value={3}")
     public static Collection<Object[]> options()
     {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
+
         List<Object[]> options = new ArrayList<>();
         for (String order : Arrays.asList("ASC", "DESC"))
         {
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
index 0f42a6a64389..a3b37566fec7 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
@@ -28,6 +28,7 @@
 import com.datastax.driver.core.PreparedStatement;
 import com.datastax.driver.core.Session;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.functions.Arguments;
 import org.apache.cassandra.cql3.functions.FunctionFactory;
@@ -36,10 +37,12 @@
 import org.apache.cassandra.cql3.functions.NativeFunctions;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.distributed.shared.WithProperties;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 
 import static java.lang.String.format;
 import static java.util.Collections.emptyList;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
 import static org.apache.cassandra.cql3.functions.masking.ColumnMask.DISABLED_ERROR_MESSAGE;
 
 /**
@@ -111,17 +114,20 @@ public void testUDTs() throws Throwable
     @Test
     public void testVectors() throws Throwable
     {
-        // Create table with mask
-        String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, 3> MASKED WITH DEFAULT)");
-        assertColumnIsMasked(table, "v", "mask_default", emptyList(), emptyList());
+        try (WithProperties properties = new WithProperties().set(VECTOR_FLOAT_ONLY, false))
+        {
+            // Create table with mask
+            String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, 3> MASKED WITH DEFAULT)");
+            assertColumnIsMasked(table, "v", "mask_default", emptyList(), emptyList());
 
-        // Alter column mask
-        alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_null()");
-        assertColumnIsMasked(table, "v", "mask_null", emptyList(), emptyList());
+            // Alter column mask
+            alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_null()");
+            assertColumnIsMasked(table, "v", "mask_null", emptyList(), emptyList());
 
-        // Drop mask
-        alterTable("ALTER TABLE %s ALTER v DROP MASKED");
-        assertTableColumnsAreNotMasked("v");
+            // Drop mask
+            alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+            assertTableColumnsAreNotMasked("v");
+        }
     }
 
     @Test
@@ -187,12 +193,12 @@ public void testInvalidMaskingFunctionArgumentTypes() throws Throwable
     {
         // create table
         createTableName();
-        assertInvalidMessage("Function system.mask_inner requires an argument of type int, but found argument 'a' of type ascii",
+        assertInvalidMessage("Function system.mask_inner(string, int, int, [text]) requires an argument of type int, but found argument 'b' of type ascii",
                              formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v text MASKED WITH mask_inner('a', 'b'))"));
 
         // alter table
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
-        assertInvalidMessage("Function system.mask_inner requires an argument of type int, but found argument 'a' of type ascii",
+        assertInvalidMessage("Function system.mask_inner(string, int, int, [text]) requires an argument of type int, but found argument 'b' of type ascii",
                              "ALTER TABLE %s ALTER v MASKED WITH mask_inner('a', 'b')");
         assertTableColumnsAreNotMasked("k", "v");
     }
@@ -574,7 +580,7 @@ private void assertRowsWithPaging(String query, Object[]... rows)
     private static final FunctionFactory NEGATIVE = new FunctionFactory("mask_negative", FunctionParameter.fixed(CQL3Type.Native.INT))
     {
         @Override
-        protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+        protected NativeFunction doGetOrCreateFunction(List<? extends AssignmentTestable> args, List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
         {
             return new MaskingFunction(name, argTypes.get(0), argTypes.get(0))
             {
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTester.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTester.java
index 93f8b001f664..b25152e2df3c 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTester.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTester.java
@@ -134,7 +134,7 @@ protected void assertColumnIsMasked(String table,
         UntypedResultSet columnRows = execute("SELECT * FROM system_schema.columns " +
                                               "WHERE keyspace_name = ? AND table_name = ? AND column_name = ?",
                                               KEYSPACE, table, column);
-        ColumnMetadata persistedColumn = SchemaKeyspace.createColumnFromRow(columnRows.one(), keyspaceMetadata.types, UserFunctions.none());
+        ColumnMetadata persistedColumn = SchemaKeyspace.createColumnFromRow(columnRows.one(), keyspaceMetadata.types, UserFunctions.none(), tableMetadata.isCounter());
 
         // Verify the column mask in the persisted schema
         ColumnMask savedMask = persistedColumn.getMask();
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/HashMaskingFunctionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/HashMaskingFunctionTest.java
index 10fe75546498..ef7ac85cb98e 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/masking/HashMaskingFunctionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/HashMaskingFunctionTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.cql3.functions.masking;
 
+import java.math.BigDecimal;
 import java.nio.ByteBuffer;
 
 import org.junit.Assert;
@@ -27,8 +28,11 @@
 import com.datastax.driver.core.DataType;
 import com.datastax.driver.core.ResultSet;
 import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.db.marshal.DecimalType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
 import static java.lang.String.format;
 
@@ -48,6 +52,13 @@ protected void testMaskingOnColumn(String name, CQL3Type type, Object value) thr
     {
         ByteBuffer serializedValue = serializedValue(type, value);
 
+        // See CNDB-13393; needed when TrieMemtable is the default in MemtableParams
+        if (value instanceof BigDecimal && "ck".equals(name))
+        {
+            ByteSource byteSource = DecimalType.instance.asComparableBytes(serializedValue, ByteComparable.Version.OSS50);
+            serializedValue = DecimalType.instance.fromComparableBytes(ByteSource.peekable(byteSource), ByteComparable.Version.OSS50);
+        }
+
         // with default algorithm
         assertRows(execute(format("SELECT mask_hash(%s) FROM %%s", name)),
                    row(HashMaskingFunction.hash(HashMaskingFunction.messageDigest(HashMaskingFunction.DEFAULT_ALGORITHM),
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java b/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
index fcbda1b8e0f3..7ac8ca07f5a7 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
@@ -28,6 +28,7 @@
 
 import com.google.common.collect.ImmutableList;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.CQL3Type;
@@ -51,6 +52,7 @@
 import org.apache.cassandra.utils.TimeUUID;
 
 import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
 
 /**
  * Abstract class for testing a specific implementation of {@link MaskingFunction}.
@@ -60,6 +62,12 @@
  */
 public abstract class MaskingFunctionTester extends CQLTester
 {
+    @BeforeClass
+    public static void setup()
+    {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
+    }
+
     /**
      * Tests the native masking function for all CQL native data types.
      */
@@ -110,7 +118,7 @@ public void testMaskingOnNative() throws Throwable
                 case DATE:
                     testMaskingOnAllColumns(type,
                                             SimpleDateSerializer.timeInMillisToDay(2),
-                                            SimpleDateSerializer.timeInMillisToDay(Long.MAX_VALUE));
+                                            SimpleDateSerializer.timeInMillisToDay(Integer.MAX_VALUE));
                     break;
                 case DURATION:
                     testMaskingOnNotKeyColumns(type, Duration.newInstance(1, 2, 3), Duration.newInstance(3, 2, 1));
@@ -302,4 +310,10 @@ protected boolean isNullOrEmptyMultiCell(CQL3Type type, Object value)
 
         return false;
     }
+
+    @SafeVarargs
+    protected final <T> Vector<T> vector(T... values)
+    {
+        return new Vector<>(values);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunctionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunctionTest.java
index 89d20d0d5821..c164fb7cd9a9 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunctionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunctionTest.java
@@ -46,6 +46,7 @@ protected void testMaskingOnColumn(String name, CQL3Type type, Object value) thr
 
     protected void testMaskingOnColumn(PartialMaskingFunction.Kind masker, String name, CQL3Type type, Object value) throws Throwable
     {
+        String functionSignature = PartialMaskingFunction.factory(masker).toString().toLowerCase();
         String functionName = SchemaConstants.SYSTEM_KEYSPACE_NAME + ".mask_" + masker.name().toLowerCase();
 
         if (type.getType() instanceof StringType)
@@ -91,9 +92,9 @@ protected void testMaskingOnColumn(PartialMaskingFunction.Kind masker, String na
         }
         else
         {
-            assertInvalidThrowMessage(format("Function %s requires an argument of type [text|varchar|ascii], " +
+            assertInvalidThrowMessage(format("Function %s requires an argument of type string, " +
                                              "but found argument %s of type %s",
-                                             functionName, name, type),
+                                             functionSignature, name, type),
                                       InvalidRequestException.class,
                                       format("SELECT %s(%s, 1, 2) FROM %%s", functionName, name));
         }
diff --git a/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java b/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
index bf8e1e495576..cefcb5e3c292 100644
--- a/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
@@ -54,7 +54,7 @@ public void testBoundsAsClusteringWithNoRestrictions()
     {
         TableMetadata tableMetadata = newTableMetadata(Sort.ASC);
 
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false).build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -76,7 +76,9 @@ public void testBoundsAsClusteringWithOneEqRestrictionsAndOneClusteringColumn()
         ByteBuffer clustering_0 = ByteBufferUtil.bytes(1);
         Restriction eq = newSingleEq(tableMetadata, 0, clustering_0);
 
-        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, eq);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(eq)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -98,7 +100,9 @@ public void testBoundsAsClusteringWithOneEqRestrictionsAndTwoClusteringColumns()
         ByteBuffer clustering_0 = ByteBufferUtil.bytes(1);
         Restriction eq = newSingleEq(tableMetadata, 0, clustering_0);
 
-        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, eq);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(eq)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -122,7 +126,10 @@ public void testBoundsAsClusteringWithOneInRestrictionsAndOneClusteringColumn()
         TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
 
         Restriction in = newSingleIN(tableMetadata, 0, value1, value2, value3);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, in);
+
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(in)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -149,7 +156,9 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneClusteringColumn()
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
 
         Restriction slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -160,7 +169,9 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneClusteringColumn()
         assertEmptyEnd(get(bounds, 0));
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -171,7 +182,9 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneClusteringColumn()
         assertEmptyEnd(get(bounds, 0));
 
         slice = newSingleSlice(tableMetadata, 0, Bound.END, true, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -182,7 +195,9 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneClusteringColumn()
         assertEndBound(get(bounds, 0), true, value1);
 
         slice = newSingleSlice(tableMetadata, 0, Bound.END, false, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -194,7 +209,10 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneClusteringColumn()
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
         Restriction slice2 = newSingleSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -206,7 +224,10 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneClusteringColumn()
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
         slice2 = newSingleSlice(tableMetadata, 0, Bound.END, true, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -229,7 +250,9 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneDescendingClusterin
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
 
         Restriction slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -240,7 +263,9 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneDescendingClusterin
         assertEndBound(get(bounds, 0), false, value1);
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -251,7 +276,9 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneDescendingClusterin
         assertEndBound(get(bounds, 0), true, value1);
 
         slice = newSingleSlice(tableMetadata, 0, Bound.END, true, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -262,7 +289,9 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneDescendingClusterin
         assertEmptyEnd(get(bounds, 0));
 
         slice = newSingleSlice(tableMetadata, 0, Bound.END, false, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -274,7 +303,10 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneDescendingClusterin
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
         Restriction slice2 = newSingleSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -286,7 +318,10 @@ public void testBoundsAsClusteringWithSliceRestrictionsAndOneDescendingClusterin
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
         slice2 = newSingleSlice(tableMetadata, 0, Bound.END, true, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -310,7 +345,10 @@ public void testBoundsAsClusteringWithEqAndInRestrictions()
         ByteBuffer value3 = ByteBufferUtil.bytes(3);
         Restriction eq = newSingleEq(tableMetadata, 0, value1);
         Restriction in = newSingleIN(tableMetadata, 1, value1, value2, value3);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, eq, in);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(eq)
+                                                                                .addRestriction(in)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -340,7 +378,10 @@ public void testBoundsAsClusteringWithEqAndSliceRestrictions()
         Restriction eq = newSingleEq(tableMetadata, 0, value3);
 
         Restriction slice = newSingleSlice(tableMetadata, 1, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, eq, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(eq)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -351,7 +392,10 @@ public void testBoundsAsClusteringWithEqAndSliceRestrictions()
         assertEndBound(get(bounds, 0), true, value3);
 
         slice = newSingleSlice(tableMetadata, 1, Bound.START, true, value1);
-        restrictions = restrictions(tableMetadata, eq, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(eq)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -362,7 +406,10 @@ public void testBoundsAsClusteringWithEqAndSliceRestrictions()
         assertEndBound(get(bounds, 0), true, value3);
 
         slice = newSingleSlice(tableMetadata, 1, Bound.END, true, value1);
-        restrictions =  restrictions(tableMetadata, eq, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(eq)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -373,7 +420,10 @@ public void testBoundsAsClusteringWithEqAndSliceRestrictions()
         assertEndBound(get(bounds, 0), true, value3, value1);
 
         slice = newSingleSlice(tableMetadata, 1, Bound.END, false, value1);
-        restrictions =  restrictions(tableMetadata, eq, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(eq)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -385,7 +435,11 @@ public void testBoundsAsClusteringWithEqAndSliceRestrictions()
 
         slice = newSingleSlice(tableMetadata, 1, Bound.START, false, value1);
         Restriction slice2 = newSingleSlice(tableMetadata, 1, Bound.END, false, value2);
-        restrictions =  restrictions(tableMetadata, eq, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(eq)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -397,7 +451,11 @@ public void testBoundsAsClusteringWithEqAndSliceRestrictions()
 
         slice = newSingleSlice(tableMetadata, 1, Bound.START, true, value1);
         slice2 = newSingleSlice(tableMetadata, 1, Bound.END, true, value2);
-        restrictions =  restrictions(tableMetadata, eq, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(eq)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -419,7 +477,9 @@ public void testBoundsAsClusteringWithMultiEqRestrictions()
         ByteBuffer value1 = ByteBufferUtil.bytes(1);
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
         Restriction eq = newMultiEq(tableMetadata, 0, value1, value2);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, eq);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(eq)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -442,7 +502,9 @@ public void testBoundsAsClusteringWithMultiInRestrictions()
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
         ByteBuffer value3 = ByteBufferUtil.bytes(3);
         Restriction in = newMultiIN(tableMetadata, 0, asList(value1, value2), asList(value2, value3));
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, in);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(in)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -468,7 +530,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneClusteringCol
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
 
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -479,7 +543,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneClusteringCol
         assertEmptyEnd(get(bounds, 0));
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -490,7 +556,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneClusteringCol
         assertEmptyEnd(get(bounds, 0));
 
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -501,7 +569,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneClusteringCol
         assertEndBound(get(bounds, 0), true, value1);
 
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -513,7 +583,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneClusteringCol
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -525,7 +598,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneClusteringCol
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -549,7 +625,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingClu
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
 
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -560,7 +638,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingClu
         assertEndBound(get(bounds, 0), false, value1);
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -571,7 +651,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingClu
         assertEndBound(get(bounds, 0), true, value1);
 
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -582,7 +664,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingClu
         assertEmptyEnd(get(bounds, 0));
 
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -594,7 +678,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingClu
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -606,7 +693,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingClu
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -630,7 +720,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoClusteringCol
 
         // (clustering_0, clustering1) > (1, 2)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
-        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -642,7 +734,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoClusteringCol
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -654,7 +748,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoClusteringCol
 
         // (clustering_0, clustering1) <= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -666,7 +762,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoClusteringCol
 
         // (clustering_0, clustering1) < (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -679,7 +777,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoClusteringCol
         // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -692,7 +793,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoClusteringCol
         // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -716,7 +820,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoDescendingClu
 
         // (clustering_0, clustering1) > (1, 2)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
-        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -728,7 +834,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoDescendingClu
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -740,7 +848,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoDescendingClu
 
         // (clustering_0, clustering1) <= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -752,8 +862,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoDescendingClu
 
         // (clustering_0, clustering1) < (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
-
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
         assertStartBound(get(bounds, 0), false, value1, value2);
@@ -766,7 +877,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoDescendingClu
         // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -779,7 +893,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoDescendingClu
         // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -804,7 +921,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingAnd
 
         // (clustering_0, clustering1) > (1, 2)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
-        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -818,7 +937,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingAnd
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -832,7 +953,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingAnd
 
         // (clustering_0, clustering1) <= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -846,7 +969,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingAnd
 
         // (clustering_0, clustering1) < (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -861,7 +986,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingAnd
         // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -876,7 +1004,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingAnd
         // (clustering_0) > (1) AND (clustering_0, clustering1) < (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value1);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -891,7 +1022,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneDescendingAnd
         // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -920,7 +1054,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneAscendingAndO
 
         // (clustering_0, clustering1) > (1, 2)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
-        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -934,7 +1070,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneAscendingAndO
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -948,7 +1086,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneAscendingAndO
 
         // (clustering_0, clustering1) <= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -962,7 +1102,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneAscendingAndO
 
         // (clustering_0, clustering1) < (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -977,7 +1119,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneAscendingAndO
         // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -992,7 +1137,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithOneAscendingAndO
         // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -1023,7 +1171,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
 
         // (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
-        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1038,7 +1188,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
         // clustering_0 = 1 AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
         Restriction eq = newSingleEq(tableMetadata, 0, value1);
         slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice, eq);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(eq)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1053,7 +1206,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
         // clustering_0 IN (1, 2) AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
         Restriction in = newSingleIN(tableMetadata, 0, value1, value2);
         slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice, in);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(in)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1071,7 +1227,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1083,7 +1241,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
 
         // (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1097,7 +1257,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
 
         // (clustering_0, clustering1, clustering_2, clustering_3) <= (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1111,7 +1273,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
 
         // (clustering_0, clustering1, clustering_2, clustering_3) < (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1126,7 +1290,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
         // (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4) AND (clustering_0, clustering_1) < (2, 3)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value3);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1141,7 +1308,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithTwoAscendingAndT
         // (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4) AND (clustering_0, clustering1, clustering_2, clustering_3) <= (4, 3, 2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value4, value3, value2, value1);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -1172,7 +1342,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescend
 
         // (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, slice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(slice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1192,7 +1364,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescend
         // clustering_0 = 1 AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
         Restriction eq = newSingleEq(tableMetadata, 0, value1);
         slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice, eq);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(eq)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -1208,7 +1383,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescend
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1222,7 +1399,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescend
 
         // (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1240,7 +1419,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescend
 
         // (clustering_0, clustering1, clustering_2, clustering_3) <= (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1258,7 +1439,9 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescend
 
         // (clustering_0, clustering1, clustering_2, clustering_3) < (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2, value3, value4);
-        restrictions = restrictions(tableMetadata, slice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1277,7 +1460,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescend
         // (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4) AND (clustering_0, clustering_1) < (2, 3)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value3);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(5, bounds.size());
@@ -1298,7 +1484,10 @@ public void testBoundsAsClusteringWithMultiSliceRestrictionsWithAscendingDescend
         // (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4) AND (clustering_0, clustering1, clustering_2, clustering_3) <= (4, 3, 2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value4, value3, value2, value1);
-        restrictions = restrictions(tableMetadata, slice, slice2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(slice)
+                                                   .addRestriction(slice2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(7, bounds.size());
@@ -1337,7 +1526,10 @@ public void testBoundsAsClusteringWithSingleEqAndMultiEqRestrictions()
         // clustering_0 = 1 AND (clustering_1, clustering_2) = (2, 3)
         Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction multiEq = newMultiEq(tableMetadata, 1, value2, value3);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, singleEq, multiEq);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(singleEq)
+                                                                                .addRestriction(multiEq)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1351,7 +1543,11 @@ public void testBoundsAsClusteringWithSingleEqAndMultiEqRestrictions()
         singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction singleEq2 = newSingleEq(tableMetadata, 1, value2);
         multiEq = newMultiEq(tableMetadata, 2, value3, value4);
-        restrictions = restrictions(tableMetadata, singleEq, singleEq2, multiEq);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(singleEq)
+                                                   .addRestriction(singleEq2)
+                                                   .addRestriction(multiEq)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1364,7 +1560,10 @@ public void testBoundsAsClusteringWithSingleEqAndMultiEqRestrictions()
         // (clustering_0, clustering_1) = (1, 2) AND clustering_2 = 3
         singleEq = newSingleEq(tableMetadata, 2, value3);
         multiEq = newMultiEq(tableMetadata, 0, value1, value2);
-        restrictions = restrictions(tableMetadata, singleEq, multiEq);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(singleEq)
+                                                   .addRestriction(multiEq)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1378,7 +1577,11 @@ public void testBoundsAsClusteringWithSingleEqAndMultiEqRestrictions()
         singleEq = newSingleEq(tableMetadata, 0, value1);
         singleEq2 = newSingleEq(tableMetadata, 3, value4);
         multiEq = newMultiEq(tableMetadata, 1, value2, value3);
-        restrictions = restrictions(tableMetadata, singleEq, multiEq, singleEq2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(singleEq)
+                                                   .addRestriction(multiEq)
+                                                   .addRestriction(singleEq2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1406,7 +1609,10 @@ public void testBoundsAsClusteringWithSingleEqAndMultiINRestrictions()
         // clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3), (4, 5))
         Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction multiIN = newMultiIN(tableMetadata, 1, asList(value2, value3), asList(value4, value5));
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, singleEq, multiIN);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(singleEq)
+                                                                                .addRestriction(multiIN)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1421,7 +1627,10 @@ public void testBoundsAsClusteringWithSingleEqAndMultiINRestrictions()
         // clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3))
         singleEq = newSingleEq(tableMetadata, 0, value1);
         multiIN = newMultiIN(tableMetadata, 1, asList(value2, value3));
-        restrictions = restrictions(tableMetadata, multiIN, singleEq);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(multiIN)
+                                                   .addRestriction(singleEq)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1435,7 +1644,11 @@ public void testBoundsAsClusteringWithSingleEqAndMultiINRestrictions()
         singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction singleEq2 = newSingleEq(tableMetadata, 1, value5);
         multiIN = newMultiIN(tableMetadata, 2, asList(value2, value3), asList(value4, value5));
-        restrictions = restrictions(tableMetadata, singleEq, multiIN, singleEq2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(singleEq)
+                                                   .addRestriction(multiIN)
+                                                   .addRestriction(singleEq2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1466,7 +1679,10 @@ public void testBoundsAsClusteringWithSingleEqAndSliceRestrictions()
         // clustering_0 = 1 AND (clustering_1, clustering_2) > (2, 3)
         Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, singleEq, multiSlice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(singleEq)
+                                                                                .addRestriction(multiSlice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1480,7 +1696,11 @@ public void testBoundsAsClusteringWithSingleEqAndSliceRestrictions()
         singleEq = newSingleEq(tableMetadata, 0, value1);
         multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3);
         Restriction multiSlice2 = newMultiSlice(tableMetadata, 1, Bound.END, false, value4);
-        restrictions = restrictions(tableMetadata, multiSlice2, singleEq, multiSlice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(multiSlice2)
+                                                   .addRestriction(singleEq)
+                                                   .addRestriction(multiSlice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1494,7 +1714,11 @@ public void testBoundsAsClusteringWithSingleEqAndSliceRestrictions()
         singleEq = newSingleEq(tableMetadata, 0, value1);
         multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, true, value2, value3);
         multiSlice2 = newMultiSlice(tableMetadata, 1, Bound.END, true, value4, value5);
-        restrictions = restrictions(tableMetadata, multiSlice2, singleEq, multiSlice);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(multiSlice2)
+                                                   .addRestriction(singleEq)
+                                                   .addRestriction(multiSlice)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1521,7 +1745,10 @@ public void testBoundsAsClusteringWithMultiEqAndSingleSliceRestrictions()
         // (clustering_0, clustering_1) = (1, 2) AND clustering_2 > 3
         Restriction multiEq = newMultiEq(tableMetadata, 0, value1, value2);
         Restriction singleSlice = newSingleSlice(tableMetadata, 2, Bound.START, false, value3);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, multiEq, singleSlice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(multiEq)
+                                                                                .addRestriction(singleSlice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1546,7 +1773,10 @@ public void testBoundsAsClusteringWithSeveralMultiColumnRestrictions()
         // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) > (3, 4)
         Restriction multiEq = newMultiEq(tableMetadata, 0, value1, value2);
         Restriction multiSlice = newMultiSlice(tableMetadata, 2, Bound.START, false, value3, value4);
-        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, multiEq, multiSlice);
+        ClusteringColumnRestrictions restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                                                .addRestriction(multiEq)
+                                                                                .addRestriction(multiSlice)
+                                                                                .build();
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1559,7 +1789,10 @@ public void testBoundsAsClusteringWithSeveralMultiColumnRestrictions()
         // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) IN ((3, 4), (4, 5))
         multiEq = newMultiEq(tableMetadata, 0, value1, value2);
         Restriction multiIN = newMultiIN(tableMetadata, 2, asList(value3, value4), asList(value4, value5));
-        restrictions = restrictions(tableMetadata, multiEq, multiIN);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(multiEq)
+                                                   .addRestriction(multiIN)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1574,7 +1807,10 @@ public void testBoundsAsClusteringWithSeveralMultiColumnRestrictions()
         // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) = (3, 4)
         multiEq = newMultiEq(tableMetadata, 0, value1, value2);
         Restriction multiEq2 = newMultiEq(tableMetadata, 2, value3, value4);
-        restrictions = restrictions(tableMetadata, multiEq, multiEq2);
+        restrictions = ClusteringColumnRestrictions.builder(tableMetadata, false)
+                                                   .addRestriction(multiEq)
+                                                   .addRestriction(multiEq2)
+                                                   .build();
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1711,7 +1947,7 @@ private static Restriction newMultiIN(TableMetadata tableMetadata, int firstInde
             columnMetadatas.add(getClusteringColumnDefinition(tableMetadata, firstIndex + i));
             terms.add(toMultiItemTerminal(values[i].toArray(new ByteBuffer[0])));
         }
-        return new MultiColumnRestriction.InRestrictionWithValues(columnMetadatas, terms);
+        return new MultiColumnRestriction.INRestriction(columnMetadatas, new MarkerOrTerms.Terms(terms));
     }
 
     /**
@@ -1725,7 +1961,7 @@ private static Restriction newMultiIN(TableMetadata tableMetadata, int firstInde
     private static Restriction newSingleIN(TableMetadata tableMetadata, int index, ByteBuffer... values)
     {
         ColumnMetadata columnDef = getClusteringColumnDefinition(tableMetadata, index);
-        return new SingleColumnRestriction.InRestrictionWithValues(columnDef, toTerms(values));
+        return new SingleColumnRestriction.INRestriction(columnDef, new MarkerOrTerms.Terms(toTerms(values)));
     }
 
     /**
@@ -1753,7 +1989,7 @@ private static ColumnMetadata getClusteringColumnDefinition(TableMetadata tableM
     private static Restriction newSingleSlice(TableMetadata tableMetadata, int index, Bound bound, boolean inclusive, ByteBuffer value)
     {
         ColumnMetadata columnDef = getClusteringColumnDefinition(tableMetadata, index);
-        return new SingleColumnRestriction.SliceRestriction(columnDef, bound, inclusive, toTerm(value));
+        return SingleColumnRestriction.SliceRestriction.fromBound(columnDef, bound, inclusive, toTerm(value));
     }
 
     /**
@@ -1773,7 +2009,7 @@ private static Restriction newMultiSlice(TableMetadata tableMetadata, int firstI
         {
             columnMetadatas.add(getClusteringColumnDefinition(tableMetadata, i + firstIndex));
         }
-        return new MultiColumnRestriction.SliceRestriction(columnMetadatas, bound, inclusive, toMultiItemTerminal(values));
+        return MultiColumnRestriction.SliceRestriction.fromBound(columnMetadatas, bound, inclusive, toMultiItemTerminal(values));
     }
 
     /**
@@ -1817,14 +2053,6 @@ private static <T> T get(SortedSet<T> set, int i)
         return Iterables.get(set, i);
     }
     
-    private static ClusteringColumnRestrictions restrictions(TableMetadata table, Restriction... restrictions)
-    {
-        ClusteringColumnRestrictions clusteringColumnRestrictions = new ClusteringColumnRestrictions(table, false);
-        for (Restriction restriction : restrictions)
-            clusteringColumnRestrictions = clusteringColumnRestrictions.mergeWith(restriction, null);
-        return clusteringColumnRestrictions;
-    }
-
     private enum Sort
     {
         ASC,
diff --git a/test/unit/org/apache/cassandra/cql3/restrictions/ExternalRestrictionTest.java b/test/unit/org/apache/cassandra/cql3/restrictions/ExternalRestrictionTest.java
new file mode 100644
index 000000000000..642526a69afc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/restrictions/ExternalRestrictionTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.restrictions;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryInterceptor;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+
+public class ExternalRestrictionTest extends CQLTester
+{
+    @BeforeClass
+    public static void setup() throws Throwable
+    {
+        requireNetwork();
+    }
+
+    @After
+    public void cleanInterceptors() throws Throwable
+    {
+        QueryProcessor.instance.clearInterceptors();
+    }
+
+    @Test
+    public void testExternalExpressionFiltering() throws Throwable
+    {
+        // This test tests the functionality of the ExternalExpression
+        // An ExternalExpression is added to the SelectStatement to only
+        // allow one row through rather that the 2 rows that should be returned
+        createTable("create table %s (pk int, ck int, v int, primary key(pk, ck))");
+        execute("insert into %s (pk, ck, v) values (0, 0, 0)");
+        execute("insert into %s (pk, ck, v) values (1, 1, 1)");
+        execute("insert into %s (pk, ck, v) values (1, 2, 2)");
+
+        assertRowsIgnoringOrder(execute("select * from %s where pk = 1"), row(1, 1, 1), row(1, 2, 2));
+
+        QueryProcessor.instance.registerInterceptor(new QueryInterceptor()
+        {
+            @Nullable
+            @Override
+            public ResultMessage interceptStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime)
+            {
+                if (statement instanceof SelectStatement)
+                {
+                    SelectStatement selectStatement = (SelectStatement)statement;
+                    if (selectStatement.table.keyspace.equals(keyspace()) && selectStatement.table.name.equals(currentTable()))
+                    {
+                        ColumnMetadata column = selectStatement.table.getColumn(ColumnIdentifier.getInterned("v", false));
+                        ByteBuffer value = Int32Type.instance.decompose(1);
+                        statement = selectStatement.addIndexRestrictions(Collections.singleton(new TestExternalRestriction(column, value)));
+                        return statement.execute(queryState, options, requestTime);
+                    }
+                }
+                return null;
+            }
+        });
+
+        List<com.datastax.driver.core.Row> rows = executeNet("select * from %s where pk = 1").all();
+
+        assertEquals(1, rows.size());
+        assertEquals(1, rows.get(0).getInt(2));
+    }
+
+    static class TestExternalRestriction implements ExternalRestriction
+    {
+        ColumnMetadata column;
+        ByteBuffer value;
+
+        TestExternalRestriction(ColumnMetadata column, ByteBuffer value)
+        {
+            this.column = column;
+            this.value = value;
+        }
+
+        @Override
+        public void addToRowFilter(RowFilter.Builder filter, TableMetadata table, QueryOptions options)
+        {
+            filter.addUserExpression(new TestFilterExpression(column, value));
+        }
+
+        @Override
+        public boolean needsFiltering(Index.Group indexGroup)
+        {
+            return false;
+        }
+    }
+
+    static class TestFilterExpression extends RowFilter.UserExpression
+    {
+        TestFilterExpression(ColumnMetadata column, ByteBuffer value)
+        {
+            super(column, Operator.EQ, value);
+        }
+
+        @Override
+        public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
+        {
+            return ByteBufferUtil.compareUnsigned(value, row.getCell(column).buffer()) == 0;
+        }
+
+        @Override
+        public String toString(boolean cql) {
+            return String.format("%s %s %s",
+                    cql ? column.name.toCQLString() : column.name.toString(),
+                    operator,
+                    ByteBufferUtil.bytesToHex(value));
+        }
+
+        @Override
+        protected void serialize(DataOutputPlus out, int version) throws IOException
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected long serializedSize(int version)
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java b/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java
index 530cb6761403..4328e859651b 100644
--- a/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java
@@ -23,6 +23,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.*;
@@ -89,10 +90,10 @@ public void testSerDes() throws IOException
         String typeName = createType("CREATE TYPE %s (f1 int, f2 int)");
 
         UserType type = new UserType(KEYSPACE, ByteBufferUtil.bytes(typeName),
-                                     asList(FieldIdentifier.forUnquoted("f1"),
-                                            FieldIdentifier.forUnquoted("f2")),
-                                     asList(Int32Type.instance,
-                                            Int32Type.instance),
+                                     ImmutableList.of(FieldIdentifier.forUnquoted("f1"),
+                                                      FieldIdentifier.forUnquoted("f2")),
+                                     ImmutableList.of(Int32Type.instance,
+                                                      Int32Type.instance),
                                      false);
 
         List<Pair<Selectable.Raw, Selectable.Raw>> list = asList(Pair.create(RawIdentifier.forUnquoted("f1"),
diff --git a/test/unit/org/apache/cassandra/cql3/selection/SortedRowsBuilderTest.java b/test/unit/org/apache/cassandra/cql3/selection/SortedRowsBuilderTest.java
new file mode 100644
index 000000000000..0c1fc852840d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/selection/SortedRowsBuilderTest.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import javax.annotation.Nullable;
+
+import com.google.common.math.IntMath;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.assertj.core.api.Assertions;
+
+/**
+ * Tests for {@link SortedRowsBuilder}.
+ */
+public class SortedRowsBuilderTest
+{
+    private static final Comparator<List<ByteBuffer>> comparator = (o1, o2) -> Int32Type.instance.compare(o1.get(0), o2.get(0));
+    private static final Comparator<List<ByteBuffer>> reverseComparator = comparator.reversed();
+
+    @Test
+    public void testRowBuilder()
+    {
+        test();
+        test(0);
+        test(0, 0, 0, 0);
+        test(0, 1, 2, 3, 5, 6, 7, 8);
+        test(8, 7, 6, 5, 3, 2, 1, 0);
+        test(1, 6, 2, 0, 7, 3, 5, 4);
+        test(1, 6, 2, 0, 7, 3, 5, 4, 4, 5, 3, 7, 0, 2, 6, 1);
+    }
+
+    private static void test(int... values)
+    {
+        List<List<ByteBuffer>> rows = toRows(values);
+
+        List<Integer> limits = IntStream.range(1, values.length + 1).boxed().collect(Collectors.toList());
+        limits.add(Integer.MAX_VALUE);
+        limits.add(Integer.MAX_VALUE - 1);
+        limits.add(Integer.MAX_VALUE / 2);
+        limits.add(Integer.MAX_VALUE / 4);
+
+        List<Integer> offsets = IntStream.range(0, values.length).boxed().collect(Collectors.toList());
+        offsets.add(Integer.MAX_VALUE);
+        offsets.add(Integer.MAX_VALUE - 1);
+        offsets.add(Integer.MAX_VALUE / 2);
+        offsets.add(Integer.MAX_VALUE / 4);
+
+        for (int limit : limits)
+        {
+            for (int offset : offsets)
+            {
+                int totalLimit = IntMath.saturatedAdd(limit, offset);
+                // with insertion order
+                test(rows, SortedRowsBuilder.create(limit, offset), null);
+
+                // with comparator
+                test(rows, SortedRowsBuilder.create(limit, offset, comparator), comparator);
+                test(rows, SortedRowsBuilder.create(limit, offset, reverseComparator), reverseComparator);
+                test(rows, SortedRowsBuilder.WithListSort.create(limit, offset, comparator), comparator);
+                test(rows, SortedRowsBuilder.WithListSort.create(limit, offset, reverseComparator), reverseComparator);
+                if (totalLimit < 1 << 20)
+                {
+                    test(rows, SortedRowsBuilder.WithHeapSort.create(limit, offset, comparator), comparator);
+                    test(rows, SortedRowsBuilder.WithHeapSort.create(limit, offset, reverseComparator), reverseComparator);
+                }
+                test(rows, SortedRowsBuilder.WithHybridSort.create(limit, offset, comparator), comparator);
+                test(rows, SortedRowsBuilder.WithHybridSort.create(limit, offset, reverseComparator), reverseComparator);
+            }
+        }
+    }
+
+    private static void test(List<List<ByteBuffer>> rows,
+                             SortedRowsBuilder builder,
+                             @Nullable Comparator<List<ByteBuffer>> comparator)
+    {
+        int offset = builder.offset;
+        int fetchLimit = builder.fetchLimit;
+
+        // get the expected values...
+        List<List<ByteBuffer>> expecetedRows = new ArrayList<>(rows);
+        if (comparator != null)
+            expecetedRows.sort(comparator);
+        expecetedRows = expecetedRows.subList(Math.min(offset, expecetedRows.size()),
+                                              Math.min(fetchLimit, expecetedRows.size()));
+        List<Integer> expectedValues = fromRows(expecetedRows);
+
+        // get the actual values...
+        rows.forEach(builder::add);
+        List<Integer> actualValues = fromRows(builder.build());
+
+        // ...and compare
+        Assertions.assertThat(actualValues).isEqualTo(expectedValues);
+    }
+
+    private static List<List<ByteBuffer>> toRows(int... values)
+    {
+        List<List<ByteBuffer>> rows = new ArrayList<>();
+        for (int value : values)
+            rows.add(Collections.singletonList(Int32Type.instance.decompose(value)));
+        return rows;
+    }
+
+    private static List<Integer> fromRows(List<List<ByteBuffer>> rows)
+    {
+        List<Integer> values = new ArrayList<>();
+        for (List<ByteBuffer> row : rows)
+            values.add(Int32Type.instance.compose(row.get(0)));
+        return values;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/selection/TermSelectionTest.java b/test/unit/org/apache/cassandra/cql3/selection/TermSelectionTest.java
index fb46809ff48b..4affbb815466 100644
--- a/test/unit/org/apache/cassandra/cql3/selection/TermSelectionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/selection/TermSelectionTest.java
@@ -19,12 +19,20 @@
 package org.apache.cassandra.cql3.selection;
 
 import java.math.BigDecimal;
-import java.util.*;
+import java.util.List;
+import java.util.UUID;
 
 import org.junit.Test;
 
-import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.Duration;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.TypeParser;
+import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
 import static org.junit.Assert.assertEquals;
@@ -134,6 +142,10 @@ public void testSelectLiteral() throws Throwable
                    row(list(set(1), set(3))),
                    row(list(set(1), set(2))),
                    row(list(set(1), set(1))));
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT [{pk, t}, {ck}] FROM %s WHERE pk = 1");
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT [{pk}, {t}] FROM %s WHERE pk = 1");
 
         // Test Maps nested within Lists
         assertRows(execute("SELECT [{}, (map<text, int>){'min' : min(ck), 'max' : max(ck)}] FROM %s"),
@@ -154,10 +166,50 @@ public void testSelectLiteral() throws Throwable
                    row(list(tuple(1, 3, timestampInMicros))));
         assertRows(execute("SELECT [(min(ck), max(ck))] FROM %s"),
                    row(list(tuple(1, 3))));
-        assertRows(execute("SELECT [(CAST(pk AS BIGINT), CAST(ck AS BIGINT)), (t, WRITETIME(t))] FROM %s"),
-                   row(list(tuple(1L, 1L), tuple("one", timestampInMicros))),
-                   row(list(tuple(1L, 2L), tuple("two", timestampInMicros))),
-                   row(list(tuple(1L, 3L), tuple("three", timestampInMicros))));
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT [(CAST(pk AS BIGINT), CAST(ck AS BIGINT)), (t, writetime(t))] FROM %s");
+
+        assertRows(execute("SELECT [(CAST(pk AS BIGINT), CAST(ck AS BIGINT), t), (CAST(pk AS BIGINT), CAST(ck AS BIGINT))] FROM %s"),
+                   row(list(tuple(1L, 1L, "one"), tuple(1L, 1L))),
+                   row(list(tuple(1L, 2L, "two"), tuple(1L, 2L))),
+                   row(list(tuple(1L, 3L, "three"), tuple(1L, 3L))));
+
+        assertRows(execute("SELECT [(CAST(pk AS BIGINT), CAST(ck AS BIGINT)), (CAST(pk AS BIGINT), CAST(ck AS BIGINT), t)] FROM %s"),
+                   row(list(tuple(1L, 1L), tuple(1L, 1L, "one"))),
+                   row(list(tuple(1L, 2L), tuple(1L, 2L, "two"))),
+                   row(list(tuple(1L, 3L), tuple(1L, 3L, "three"))));
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT [(CAST(pk AS BIGINT), t, CAST(ck AS BIGINT)), (CAST(pk AS BIGINT), CAST(ck AS BIGINT))] FROM %s");
+
+        // list of tuples of tuples
+        assertRows(execute("SELECT [((t,t, t), (t,t,CAST(ck AS BIGINT))), ((t,t), (t,t,CAST(ck AS BIGINT)))] FROM %s"),
+                   row(list(tuple(tuple("one", "one", "one"), tuple("one", "one", 1L)),
+                            tuple(tuple("one", "one"), tuple("one", "one", 1L)))),
+                   row(list(tuple(tuple("two", "two", "two"), tuple("two", "two", 2L)),
+                            tuple(tuple("two", "two"), tuple("two", "two", 2L)))),
+                   row(list(tuple(tuple("three", "three", "three"), tuple("three", "three", 3L)),
+                            tuple(tuple("three", "three"), tuple("three", "three", 3L)))));
+
+        assertRows(execute("SELECT [((t,t), (t,t,CAST(ck AS BIGINT))), ((t,t), (t,t,CAST(ck AS BIGINT),t))] FROM %s"),
+                   row(list(tuple(tuple("one", "one"), tuple("one", "one", 1L)),
+                            tuple(tuple("one", "one"), tuple("one", "one", 1L, "one")))),
+                   row(list(tuple(tuple("two", "two"), tuple("two", "two", 2L)),
+                            tuple(tuple("two", "two"), tuple("two", "two", 2L, "two")))),
+                   row(list(tuple(tuple("three", "three"), tuple("three", "three", 3L)),
+                            tuple(tuple("three", "three"), tuple("three", "three", 3L, "three")))));
+
+        // single element tuple: tuple(t) incompatible with tuple(long, long)
+        assertInvalidMessage("(t) is not of the expected type: tuple<bigint, bigint>",
+                             "SELECT [(CAST(pk AS BIGINT), CAST(ck AS BIGINT)), (t)] FROM %s");
+
+        assertInvalidMessage("(cast(ck as bigint)) is not of the expected type: tuple<text, text>",
+                             "SELECT [(t, t), (CAST(ck AS BIGINT))] FROM %s");
+
+        // single element tuple: tuple(long) compatible with tuple(long, long)
+        assertRows(execute("SELECT [(CAST(pk AS BIGINT), CAST(ck AS BIGINT)), (CAST(ck AS BIGINT))] FROM %s"),
+                   row(list(tuple(1L, 1L), tuple(1L))),
+                   row(list(tuple(1L, 2L), tuple(2L))),
+                   row(list(tuple(1L, 3L), tuple(3L))));
 
         // Test UDTs nested within Lists
         String type = createType("CREATE TYPE %s(a int, b int, c bigint)");
@@ -189,6 +241,10 @@ public void testSelectLiteral() throws Throwable
                    row(set(list(1), list(3))));
         assertRows(execute("SELECT {([min(ck)]), [max(ck)]} FROM %s"),
                    row(set(list(1), list(3))));
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT {[min(ck), writetime(t)], [max(ck)]} FROM %s");
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT {[writetime(t)], [max(ck)]} FROM %s");
 
         // Test Sets nested within Sets
         assertRows(execute("SELECT {{}, {min(ck), max(ck)}} FROM %s"),
@@ -223,6 +279,45 @@ public void testSelectLiteral() throws Throwable
                    row(set(tuple(1, 3, timestampInMicros))));
         assertRows(execute("SELECT {(min(ck), max(ck))} FROM %s"),
                    row(set(tuple(1, 3))));
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT {(min(ck), max(ck)), (t, writetime(t))} FROM %s");
+
+        assertRows(execute("SELECT {(CAST(pk AS BIGINT), CAST(ck AS BIGINT), t), (CAST(pk AS BIGINT), CAST(ck AS BIGINT))} FROM %s"),
+                   row(set(tuple(1L, 1L, "one"), tuple(1L, 1L))),
+                   row(set(tuple(1L, 2L, "two"), tuple(1L, 2L))),
+                   row(set(tuple(1L, 3L, "three"), tuple(1L, 3L))));
+
+        assertRows(execute("SELECT {(CAST(pk AS BIGINT), CAST(ck AS BIGINT)), (CAST(pk AS BIGINT), CAST(ck AS BIGINT), t)} FROM %s"),
+                   row(set(tuple(1L, 1L), tuple(1L, 1L, "one"))),
+                   row(set(tuple(1L, 2L), tuple(1L, 2L, "two"))),
+                   row(set(tuple(1L, 3L), tuple(1L, 3L, "three"))));
+
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT {(CAST(pk AS BIGINT), t, CAST(ck AS BIGINT)), (CAST(pk AS BIGINT), CAST(ck AS BIGINT))} FROM %s");
+
+        assertInvalidMessage("Invalid collection literal: all selectors must have the same CQL type inside collection literals",
+                             "SELECT {(CAST(pk AS BIGINT), t, CAST(ck AS BIGINT)), (CAST(pk AS BIGINT), CAST(ck AS BIGINT))} FROM %s");
+
+        // set of tuples of tuples
+        assertRows(execute("SELECT {((t,t, t), (t,t,CAST(ck AS BIGINT))), ((t,t), (t,t,CAST(ck AS BIGINT)))} FROM %s"),
+                   row(set(tuple(tuple("one", "one", "one"), tuple("one", "one", 1L)),
+                           tuple(tuple("one", "one"), tuple("one", "one", 1L)))),
+                   row(set(tuple(tuple("two", "two", "two"), tuple("two", "two", 2L)),
+                           tuple(tuple("two", "two"), tuple("two", "two", 2L)))),
+                   row(set(tuple(tuple("three", "three", "three"), tuple("three", "three", 3L)),
+                           tuple(tuple("three", "three"), tuple("three", "three", 3L)))));
+
+        assertRows(execute("SELECT {((t,t), (t,t,CAST(ck AS BIGINT))), ((t,t), (t,t,CAST(ck AS BIGINT),t))} FROM %s"),
+                   row(set(tuple(tuple("one", "one"), tuple("one", "one", 1L)),
+                           tuple(tuple("one", "one"), tuple("one", "one", 1L, "one")))),
+                   row(set(tuple(tuple("two", "two"), tuple("two", "two", 2L)),
+                           tuple(tuple("two", "two"), tuple("two", "two", 2L, "two")))),
+                   row(set(tuple(tuple("three", "three"), tuple("three", "three", 3L)),
+                           tuple(tuple("three", "three"), tuple("three", "three", 3L, "three")))));
+
+        // getExactType for (t) is null
+        assertInvalidMessage("(t) is not of the expected type: tuple<bigint, bigint>",
+                             "SELECT {(CAST(pk AS BIGINT), CAST(ck AS BIGINT)), (t), (CAST(pk AS BIGINT), CAST(ck AS BIGINT))} FROM %s");
 
         // Test UDTs nested within Sets
         assertRows(execute("SELECT {(" + type + "){a : min(ck), b: max(ck)}} FROM %s"),
@@ -378,7 +473,7 @@ public void testSelectLiteral() throws Throwable
 
 
         // Test Litteral Set with Duration elements
-        assertInvalidMessage("Durations are not allowed inside sets: set<duration>",
+        assertInvalidMessage("Durations are not allowed inside sets: frozen<set<duration>>",
                              "SELECT pk, ck, (set<duration>){2d, 1mo} FROM %s");
 
         assertInvalidMessage("Invalid field selection: system.min(ck) of type int is not a user type",
@@ -407,7 +502,7 @@ public void testCollectionLiteralsWithDurations() throws Throwable
                    row(map(2, Duration.from("10h"))),
                    row(map(3, Duration.from("11h"))));
 
-        assertInvalidMessage("Durations are not allowed as map keys: map<duration, int>",
+        assertInvalidMessage("Durations are not allowed as map keys: frozen<map<duration, int>>",
                              "SELECT (map<duration, int>){d1 : ck, d2 :ck} FROM %s");
     }
 
@@ -415,7 +510,7 @@ public void testCollectionLiteralsWithDurations() throws Throwable
     public void testSelectUDTLiteral() throws Throwable
     {
         String type = createType("CREATE TYPE %s(a int, b text)");
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + type + ")");
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + type + ')');
 
         execute("INSERT INTO %s(k, v) VALUES (?, ?)", 0, userType("a", 3, "b", "foo"));
 
@@ -492,7 +587,7 @@ public void testSelectPrepared() throws Throwable
         assertColumnSpec(boundNames.get(0), "[selection]", Int32Type.instance);
         assertColumnSpec(boundNames.get(1), "adecimal", DecimalType.instance);
         assertColumnSpec(boundNames.get(2), "[selection]", UTF8Type.instance);
-        assertColumnSpec(boundNames.get(3), "atuple", TypeParser.parse("TupleType(Int32Type,UTF8Type)"));
+        assertColumnSpec(boundNames.get(3), "atuple", TypeParser.parse("TupleType(Int32Type,UTF8Type)").freeze());
         assertColumnSpec(boundNames.get(4), "pk", Int32Type.instance);
 
 
@@ -503,7 +598,7 @@ public void testSelectPrepared() throws Throwable
         assertColumnSpec(resultNames.get(0), "(int)?", Int32Type.instance);
         assertColumnSpec(resultNames.get(1), "(decimal)?", DecimalType.instance);
         assertColumnSpec(resultNames.get(2), "(text)?", UTF8Type.instance);
-        assertColumnSpec(resultNames.get(3), "(tuple<int, text>)?", TypeParser.parse("TupleType(Int32Type,UTF8Type)"));
+        assertColumnSpec(resultNames.get(3), "(tuple<int, text>)?", TypeParser.parse("TupleType(Int32Type,UTF8Type)").freeze());
         assertColumnSpec(resultNames.get(4), "pk", Int32Type.instance);
         assertColumnSpec(resultNames.get(5), "ck", Int32Type.instance);
         assertColumnSpec(resultNames.get(6), "t", UTF8Type.instance);
diff --git a/test/unit/org/apache/cassandra/cql3/statements/AlterKeyspaceStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/AlterKeyspaceStatementTest.java
new file mode 100644
index 000000000000..66e44e14f092
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/AlterKeyspaceStatementTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.AlterKeyspaceStatement;
+import org.apache.cassandra.service.ClientState;
+
+import static org.junit.Assert.assertEquals;
+
+public class AlterKeyspaceStatementTest extends CQLTester
+{
+    @Test
+    public void testAttributeOverride()
+    {
+        String keyspaceName = createKeyspaceName();
+        CQLStatement.Raw raw = QueryProcessor.parseStatement(String.format("ALTER KEYSPACE %s WITH REPLICATION = " +
+                                                                           "{ 'class' : 'SimpleStrategy', 'replication_factor' : '1' }",
+                                                                           keyspaceName));
+
+        CQLStatement stm = raw.prepare(ClientState.forInternalCalls());
+        ((AlterKeyspaceStatement) stm).overrideAttribute("replication_factor", "replication_factor", "2");
+        assertEquals("2", ((AlterKeyspaceStatement) stm).getAttribute("replication_factor"));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/CreateIndexStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/CreateIndexStatementTest.java
new file mode 100644
index 000000000000..ebb347eac4dd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/CreateIndexStatementTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import java.util.Set;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateIndexStatement;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@RunWith(Parameterized.class)
+public class CreateIndexStatementTest extends CQLTester
+{
+    @Parameterized.Parameters(name = "index = {0}")
+    public static Set<String> dseIndexes()
+    {
+        return CreateIndexStatement.DSE_INDEXES;
+    }
+
+    @Parameterized.Parameter()
+    public String indexClass;
+
+    @BeforeClass
+    public static void setup() throws Exception
+    {
+        SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1));
+        QueryProcessor.executeOnceInternal("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c))");
+    }
+
+    private void assertNoIndex(String indexName) throws Throwable
+    {
+        try
+        {
+            executeNet("DESCRIBE INDEX ks." + indexName);
+            fail("Expected InvalidQueryException caused by a missing index");
+        }
+        catch (InvalidQueryException e)
+        {
+            assertTrue(e.getMessage().contains(indexName + "' not found"));
+        }
+    }
+
+    @Test
+    public void dseIndexCreationShouldBeIgonerWithWarning() throws Throwable
+    {
+        // should not throw
+        ResultSet rows = executeNet(String.format("CREATE CUSTOM INDEX index_name ON ks.tbl (v) USING '%s'", indexClass));
+
+        assertTrue(rows.wasApplied()); // the command is ignored
+
+        String warning = rows.getAllExecutionInfo().get(0).getWarnings().get(0);
+        assertTrue("Custom DSE index creation should cause a warning", warning.contains("DSE custom index"));
+
+        assertNoIndex("index_name");
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/CreateKeyspaceStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/CreateKeyspaceStatementTest.java
new file mode 100644
index 000000000000..0770e51ba4aa
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/CreateKeyspaceStatementTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import org.junit.Test;
+
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateKeyspaceStatement;
+import org.apache.cassandra.service.ClientState;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class CreateKeyspaceStatementTest extends CQLTester
+{
+    @Test
+    public void testAttributeOverride()
+    {
+        String keyspaceName = createKeyspaceName();
+        CQLStatement.Raw raw = QueryProcessor.parseStatement(String.format("CREATE KEYSPACE %s WITH REPLICATION = " +
+                                                                           "{ 'class' : 'SimpleStrategy', 'replication_factor' : '1' }",
+                                                                           keyspaceName));
+
+        CQLStatement stm = raw.prepare(ClientState.forInternalCalls());
+        ((CreateKeyspaceStatement) stm).overrideAttribute("replication_factor", "replication_factor", "2");
+        assertEquals("2", ((CreateKeyspaceStatement) stm).getAttribute("replication_factor"));
+    }
+
+    @Test
+    public void ignoreUnsupportedGraphEngineProperty() throws Throwable
+    {
+        String keyspaceName = createKeyspaceName();
+        String keyspaceOptions = "graph_engine = 'Core'";
+
+        // should not throw
+        ResultSet rows = executeNet(String.format("CREATE KEYSPACE %s WITH REPLICATION = " +
+                                                  "{ 'class' : 'SimpleStrategy', 'replication_factor' : '1' } AND %s",
+                                                  keyspaceName, keyspaceOptions));
+
+        assertTrue(rows.wasApplied());
+
+        String warning = rows.getAllExecutionInfo().get(0).getWarnings().get(0);
+        assertThat(warning, containsString("The unsupported graph property 'graph_engine' was ignored."));
+
+        assertNoGraphEngineKeyspaceProperty(keyspaceName);
+    }
+
+    private void assertNoGraphEngineKeyspaceProperty(String tableName) throws Throwable
+    {
+        ResultSet result = executeNet("DESCRIBE KEYSPACE " + tableName);
+
+        String createStatement = result.one().getString("create_statement");
+        assertThat(createStatement, not(containsString("graph_engine")));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementCompactionStrategiesTest.java b/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementCompactionStrategiesTest.java
new file mode 100644
index 000000000000..d620c6b0083b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementCompactionStrategiesTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class CreateTableStatementCompactionStrategiesTest extends CQLTester
+{
+    @Parameterized.Parameters(name = "compactionStrategy = {0}")
+    public static Set<String> strategies()
+    {
+        return ImmutableSet.of(
+            "{'class': 'org.apache.cassandra.db.compaction.MemoryOnlyStrategy', 'max_threshold': '32', 'min_threshold': '4'}",
+            "{'class': 'MemoryOnlyStrategy', 'max_threshold': '32', 'min_threshold': '4'}",
+            "{'class': 'org.apache.cassandra.db.compaction.TieredCompactionStrategy', 'tiering_strategy': 'TimeWindowStorageStrategy', 'config': 'strategy1', 'max_tier_ages': '3600,7200'}",
+            "{'class': 'TieredCompactionStrategy', 'tiering_strategy': 'TimeWindowStorageStrategy', 'config': 'strategy1', 'max_tier_ages': '3600,7200'}"
+        );
+    }
+
+    @Parameterized.Parameter()
+    public String compactionStrategy;
+
+    @BeforeClass
+    public static void setup() throws Exception
+    {
+        SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1));
+    }
+
+    @Test
+    public void dseCompactionStrategyShouldBeIgnoredWithWarning() throws Throwable
+    {
+        decorateCQLWithTestNames = false;
+        String tableName = createTableName();
+
+        // should not throw
+        ResultSet rows = executeNet(String.format("CREATE TABLE ks.%s (k int PRIMARY KEY, v int) WITH " +
+                                                  "compaction = %s;", tableName, compactionStrategy));
+
+        assertTrue(rows.wasApplied());
+
+        String warning = rows.getAllExecutionInfo().get(0).getWarnings().get(0);
+        assertThat(warning, containsString("The compaction strategy parameter was overridden with the default"));
+
+        assertDefaultCompactionStrategy(tableName);
+    }
+
+    private void assertDefaultCompactionStrategy(String tableName) throws Throwable
+    {
+        ResultSet result = executeNet("DESCRIBE TABLE ks." + tableName);
+
+        String createStatement = result.one().getString("create_statement");
+        assertThat(createStatement, containsString(CompactionParams.DEFAULT.klass().getCanonicalName()));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementGraphTest.java b/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementGraphTest.java
new file mode 100644
index 000000000000..ed2f27d1ecb1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementGraphTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class CreateTableStatementGraphTest extends CQLTester
+{
+    @Parameterized.Parameters(name = "tableOptions = {0}")
+    public static Set<String> tableOptions()
+    {
+        return ImmutableSet.of(
+            "VERTEX LABEL",
+            "vertex label",
+            "VERTEX LABEL person_label",
+            "VERTEX LABEL personlabel",
+            "VERTEX LABEL \"personlabel\"",
+            "VERTEX LABEL personlabel AND CLUSTERING ORDER BY (v DESC)",
+            "CLUSTERING ORDER BY (v DESC) AND VERTEX LABEL",
+            "EDGE LABEL person_authored_book FROM person(name,person_id) TO book(name, book_id,  cover)",
+            "EDGE LABEL person_authored_book FROM person((name),person_id) TO book((name), book_id,  cover)",
+            "EDGE LABEL person_authored_book FROM person((name,person_id)) TO book((name, book_id),  cover)",
+            "EDGE LABEL person_authored_book FROM person((name,person_id), address) TO book((name, book_id),  cover)",
+            "EDGE LABEL person_authored_book FROM person((name)) TO book((name))",
+            "EDGE LABEL person_authored_book FROM person(name) TO book(cover)",
+            "VERTEX LABEL AND EDGE LABEL person_authored_book FROM person(name) TO book(cover)"
+        );
+    }
+
+    @Parameterized.Parameter()
+    public String tableOptions;
+
+    @BeforeClass
+    public static void setup() throws Exception
+    {
+        decorateCQLWithTestNames = false;
+        SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1));
+    }
+
+    @Test
+    public void dseGraphShouldBeIgnoredWithWarning() throws Throwable
+    {
+        String tableName = createTableName();
+
+        // should not throw
+        ResultSet rows = executeNet(String.format("CREATE TABLE ks.%s (k int, v int, PRIMARY KEY (k, v)) WITH %s", tableName, tableOptions));
+
+        assertTrue(rows.wasApplied());
+
+        String warning = rows.getAllExecutionInfo().get(0).getWarnings().get(0);
+        assertThat(warning, containsString("The unsupported graph table property was ignored"));
+
+        assertNoGraphLabels(tableName);
+    }
+
+    private void assertNoGraphLabels(String tableName) throws Throwable
+    {
+        ResultSet result = executeNet("DESCRIBE TABLE ks." + tableName);
+
+        String createStatement = result.one().getString("create_statement");
+        assertThat(createStatement.toUpperCase(), not(containsString("LABEL")));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementNodeSyncTest.java b/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementNodeSyncTest.java
new file mode 100644
index 000000000000..85ae6c948ac2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/statements/CreateTableStatementNodeSyncTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.statements;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class CreateTableStatementNodeSyncTest extends CQLTester
+{
+    @Parameterized.Parameters(name = "tableOptions = {0}")
+    public static Set<String> tableOptions()
+    {
+        return ImmutableSet.of(
+            "WITH nodesync = { 'enabled' : 'true', 'incremental' : 'true' }",
+            "WITH nodesync = { 'enabled' : 'true' }",
+            "WITH nodesync = { 'enabled' : 'false' }",
+            "WITH nodesync = { 'enabled' : 'true', 'deadline_target_sec': 60 }"
+        );
+    }
+
+    @Parameterized.Parameter()
+    public String tableOptions;
+
+    @BeforeClass
+    public static void setup() throws Exception
+    {
+        decorateCQLWithTestNames = false;
+        SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1));
+    }
+
+    @Test
+    public void dseNodesyncShouldBeIgnoredWithWarning() throws Throwable
+    {
+        String tableName = createTableName();
+
+        // should not throw
+        ResultSet rows = executeNet(String.format("CREATE TABLE ks.%s (k int PRIMARY KEY, v int) %s", tableName, tableOptions));
+
+        assertTrue(rows.wasApplied());
+
+        String warning = rows.getAllExecutionInfo().get(0).getWarnings().get(0);
+        assertThat(warning, containsString("The unsupported 'nodesync' table option was ignored."));
+
+        assertNoNodesyncTableParamater(tableName);
+    }
+
+    private void assertNoNodesyncTableParamater(String tableName) throws Throwable
+    {
+        ResultSet result = executeNet("DESCRIBE TABLE ks." + tableName);
+
+        String createStatement = result.one().getString("create_statement");
+        assertThat(createStatement, not(containsString("nodesync")));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java
index 175a9f3809e1..d29f7b0e937e 100644
--- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java
+++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java
@@ -20,6 +20,7 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -84,6 +85,8 @@ public void testSchemaChangeDuringPaging()
     @Test
     public void testDescribeFunctionAndAggregate() throws Throwable
     {
+        waitForSchemaCleanupCompleted(60, TimeUnit.SECONDS);
+
         String fNonOverloaded = createFunction(KEYSPACE_PER_TEST,
                                                "",
                                                "CREATE OR REPLACE FUNCTION %s() " +
@@ -222,8 +225,8 @@ public void testDescribeFunctionWithTuples() throws Throwable
         assertRowsNet(executeDescribeNet("DESCRIBE FUNCTION " + function),
                       row(KEYSPACE_PER_TEST,
                           "function",
-                          shortFunctionName(function) + "(tuple<int>, list<frozen<tuple<int, text>>>, tuple<frozen<tuple<int, text>>, text>)",
-                          "CREATE FUNCTION " + function + "(t tuple<int>, l list<frozen<tuple<int, text>>>, nt tuple<frozen<tuple<int, text>>, text>)\n" +
+                              shortFunctionName(function) + "(tuple<int>, list<tuple<int, text>>, tuple<tuple<int, text>, text>)",
+                              "CREATE FUNCTION " + function + "(t tuple<int>, l list<tuple<int, text>>, nt tuple<tuple<int, text>, text>)\n" +
                           "    CALLED ON NULL INPUT\n" +
                           "    RETURNS tuple<int, text>\n" +
                           "    LANGUAGE java\n" +
@@ -270,6 +273,8 @@ public void testDescribe() throws Throwable
     {
         try
         {
+            waitForSchemaCleanupCompleted(60, TimeUnit.SECONDS);
+
             execute("CREATE KEYSPACE test WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1};");
             execute("CREATE TABLE test.users ( userid text PRIMARY KEY, firstname text, lastname text, age int);");
             execute("CREATE INDEX myindex ON test.users (age);");
@@ -492,25 +497,37 @@ public void testDescribeTableWithInternals() throws Throwable
                                       "    v3 int,\n" +
                                       "    PRIMARY KEY ((pk1, pk2), c)\n" +
                                       ") WITH ID = " + id + "\n" +
-                                      "    AND CLUSTERING ORDER BY (c ASC)\n" +
-                                      "    AND " + tableParametersCql();
+                                      "    AND CLUSTERING ORDER BY (c ASC)\n";
 
         assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE_PER_TEST + "." + table + " WITH INTERNALS"),
                       row(KEYSPACE_PER_TEST,
                           "table",
                           table,
-                          tableCreateStatement));
+                          tableCreateStatement +
+                          "    AND " + tableParametersCql()));
 
         String dropStatement = "ALTER TABLE " + KEYSPACE_PER_TEST + "." + table + " DROP v3 USING TIMESTAMP 1589286942065000;";
 
         execute(dropStatement);
 
+        String tableCreateStatementAfterDrop = "CREATE TABLE " + KEYSPACE_PER_TEST + "." + table + " (\n" +
+                                      "    pk1 text,\n" +
+                                      "    pk2 int,\n" +
+                                      "    c int,\n" +
+                                      "    s decimal static,\n" +
+                                      "    v1 text,\n" +
+                                      "    v2 int,\n" +
+                                      "    PRIMARY KEY ((pk1, pk2), c)\n" +
+                                      ") WITH ID = " + id + "\n" +
+                                      "    AND CLUSTERING ORDER BY (c ASC)\n";
+
         assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE_PER_TEST + "." + table + " WITH INTERNALS"),
                       row(KEYSPACE_PER_TEST,
                           "table",
                           table,
-                          tableCreateStatement + "\n" +
-                          dropStatement));
+                          tableCreateStatementAfterDrop +
+                          "    AND DROPPED COLUMN RECORD v3 int USING TIMESTAMP 1589286942065000" + "\n" +
+                          "    AND " + tableParametersCql()));
 
         String addStatement = "ALTER TABLE " + KEYSPACE_PER_TEST + "." + table + " ADD v3 int;";
 
@@ -520,9 +537,9 @@ public void testDescribeTableWithInternals() throws Throwable
                       row(KEYSPACE_PER_TEST,
                           "table",
                           table,
-                          tableCreateStatement + "\n" +
-                          dropStatement + "\n" +
-                          addStatement));
+                          tableCreateStatement +
+                          "    AND DROPPED COLUMN RECORD v3 int USING TIMESTAMP 1589286942065000" + "\n" +
+                          "    AND " + tableParametersCql()));
     }
 
     @Test
@@ -580,25 +597,31 @@ public void testPrimaryKeyPositionWithAndWithoutInternals() throws Throwable
                                       "    v1 text,\n" +
                                       "    v2 int,\n" +
                                       "    v3 int\n" +
-                                      ") WITH ID = " + id + "\n" +
-                                      "    AND " + tableParametersCql();
+                                      ") WITH ID = " + id;
 
         assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE_PER_TEST + "." + table + " WITH INTERNALS"),
                       row(KEYSPACE_PER_TEST,
                           "table",
                           table,
-                          tableCreateStatement));
-
+                          tableCreateStatement + "\n" +
+                          "    AND " + tableParametersCql()));
         String dropStatement = "ALTER TABLE " + KEYSPACE_PER_TEST + "." + table + " DROP v3 USING TIMESTAMP 1589286942065000;";
 
         execute(dropStatement);
 
+        String tableCreateStatementAfterDrop = "CREATE TABLE " + KEYSPACE_PER_TEST + "." + table + " (\n" +
+                                      "    pk text PRIMARY KEY,\n" +
+                                      "    v1 text,\n" +
+                                      "    v2 int\n" +
+                                      ") WITH ID = " + id;
+
         assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE_PER_TEST + "." + table + " WITH INTERNALS"),
                       row(KEYSPACE_PER_TEST,
                           "table",
                           table,
-                          tableCreateStatement + "\n" +
-                          dropStatement));
+                          tableCreateStatementAfterDrop + "\n" +
+                          "    AND DROPPED COLUMN RECORD v3 int USING TIMESTAMP 1589286942065000" + "\n" +
+                          "    AND " + tableParametersCql()));
 
         String tableCreateStatementWithoutDroppedColumn = "CREATE TABLE " + KEYSPACE_PER_TEST + "." + table + " (\n" +
                                                           "    pk text PRIMARY KEY,\n" +
@@ -990,7 +1013,7 @@ private static String allTypesTable()
                "    textcol text,\n" +
                "    timestampcol timestamp,\n" +
                "    tinyintcol tinyint,\n" +
-               "    tuplecol frozen<tuple<text, int, frozen<tuple<timestamp>>>>,\n" +
+               "    tuplecol tuple<text, int, tuple<timestamp>>,\n" +
                "    uuidcol uuid,\n" +
                "    varcharcol text,\n" +
                "    varintcol varint,\n" +
diff --git a/test/unit/org/apache/cassandra/cql3/statements/PropertyDefinitionsTest.java b/test/unit/org/apache/cassandra/cql3/statements/PropertyDefinitionsTest.java
index cba0c86bfe2e..529766d664d7 100644
--- a/test/unit/org/apache/cassandra/cql3/statements/PropertyDefinitionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/statements/PropertyDefinitionsTest.java
@@ -24,6 +24,8 @@
 
 import org.apache.cassandra.exceptions.SyntaxException;
 
+import static org.junit.Assert.assertEquals;
+
 import static org.apache.cassandra.cql3.statements.PropertyDefinitions.parseBoolean;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -51,6 +53,23 @@ public void testNegativeBooleanParsing()
         assertFalse(parseBoolean("prop6", "No"));
     }
 
+    @Test
+    public void testGetProperty()
+    {
+        String key = "k";
+        String value = "v";
+        PropertyDefinitions pd = new PropertyDefinitions();
+        pd.addProperty(key, value);
+        assertEquals(value, pd.getProperty(key).toString());
+    }
+
+    @Test(expected = SyntaxException.class)
+    public void testGetMissingProperty()
+    {
+        PropertyDefinitions pd = new PropertyDefinitions();
+        pd.getProperty("missing");
+    }
+
     @Test(expected = SyntaxException.class)
     public void testInvalidPositiveBooleanParsing()
     {
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CQLVectorNotAllowedTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CQLVectorNotAllowedTest.java
new file mode 100644
index 000000000000..1360d9f2c715
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CQLVectorNotAllowedTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.validation.entities;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_TYPE_ALLOWED;
+
+public class CQLVectorNotAllowedTest extends CQLTester
+{
+    @BeforeClass
+    public static void setupClass()
+    {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
+        VECTOR_TYPE_ALLOWED.setBoolean(false);
+    }
+
+    @Test(expected = InvalidRequestException.class)
+    public void testCreateTableVectorPresentNotAllowed() throws Throwable
+    {
+        createTableMayThrow("CREATE TABLE %s (a int, b int, pk vector<int, 2> primary key)");
+    }
+
+    @Test(expected = InvalidRequestException.class)
+    public void testCreateTableVectorInTupleNotAllowed() throws Throwable
+    {
+        createTableMayThrow("CREATE TABLE %s (a int, b int, pk tuple<int, vector<int, 2>> primary key)");
+    }
+
+    @Test(expected = InvalidRequestException.class)
+    public void testCreateUdtWithVectorNotAllowed() throws Throwable
+    {
+        createType("CREATE TYPE %s (uuid_type uuid, text_type text, vec vector<int, 2>)");
+    }
+
+    @Test(expected = InvalidRequestException.class)
+    public void testAlterTableAddVectorNotAllowed() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
+        alterTableMayThrow("ALTER TABLE %s ADD v3 vector<int, 2>;");
+    }
+
+    @Test
+    public void testCreateUdtWithoutVectorNotAllowed() throws Throwable
+    {
+        createType("CREATE TYPE %s (uuid_type uuid, text_type text)");
+    }
+
+    @Test
+    public void testCreateTableVectorNotPresentNotAllowed() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a int, b int, c tuple<int, int>, PRIMARY KEY(a, b))");
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CQLVectorTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CQLVectorTest.java
new file mode 100644
index 000000000000..2acea2dde243
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CQLVectorTest.java
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.validation.entities;
+
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.functions.Arguments;
+import org.apache.cassandra.cql3.functions.FunctionArguments;
+import org.apache.cassandra.cql3.functions.NativeFunctions;
+import org.apache.cassandra.cql3.functions.NativeScalarFunction;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
+import static java.lang.String.format;
+
+public class CQLVectorTest extends CQLTester
+{
+    @BeforeClass
+    public static void setupClass()
+    {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
+    }
+
+    @Test
+    public void select()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk vector<int, 2> primary key)");
+
+        execute("INSERT INTO %s (pk) VALUES ([1, 2])");
+
+        Vector<Integer> vector = vector(1, 2);
+        Object[] row = row(vector);
+
+        assertRows(execute("SELECT * FROM %s WHERE pk = [1, 2]"), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk = ?", vector), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk = [1, 1 + 1]"), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk = [1, ?]", 2), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk = [1, (int) ?]", 2), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk = [1, 1 + (int) ?]", 1), row);
+
+        assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, 2])"), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, 2], [1, 2])"), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk IN (?)", vector), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, 1 + 1])"), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, ?])", 2), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, (int) ?])", 2), row);
+        assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, 1 + (int) ?])", 1), row);
+
+        assertRows(execute("SELECT * FROM %s WHERE pk > [0, 0] AND pk < [1, 3] ALLOW FILTERING"), row);
+        assertRows(execute("SELECT * FROM %s WHERE token(pk) = token([1, 2])"), row);
+
+        assertRows(execute("SELECT * FROM %s"), row);
+        Assertions.assertThat(execute("SELECT * FROM %s").one().getVector("pk", Int32Type.instance, 2))
+                  .isEqualTo(vector);
+    }
+
+    @Test
+    public void insert()
+    {
+        Runnable test = () -> {
+            assertRows(execute("SELECT * FROM %s"), row(list(1, 2)));
+            execute("TRUNCATE %s");
+            assertRows(execute("SELECT * FROM %s"));
+        };
+
+        createTable(KEYSPACE, "CREATE TABLE %s (pk vector<int, 2> primary key)");
+
+        execute("INSERT INTO %s (pk) VALUES ([1, 2])");
+        test.run();
+
+        execute("INSERT INTO %s (pk) VALUES (?)", vector(1, 2));
+        test.run();
+
+        execute("INSERT INTO %s (pk) VALUES ([1, 1 + 1])");
+        test.run();
+
+        execute("INSERT INTO %s (pk) VALUES ([1, ?])", 2);
+        test.run();
+
+        execute("INSERT INTO %s (pk) VALUES ([1, (int) ?])", 2);
+        test.run();
+
+        execute("INSERT INTO %s (pk) VALUES ([1, 1 + (int) ?])", 1);
+        test.run();
+    }
+
+    @Test
+    public void insertNonPK()
+    {
+        Runnable test = () -> {
+            assertRows(execute("SELECT * FROM %s"), row(0, list(1, 2)));
+            execute("TRUNCATE %s");
+            assertRows(execute("SELECT * FROM %s"));
+        };
+
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<int, 2>)");
+
+        execute("INSERT INTO %s (pk, value) VALUES (0, [1, 2])");
+        test.run();
+
+        execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, 2));
+        test.run();
+
+        execute("INSERT INTO %s (pk, value) VALUES (0, [1, 1 + 1])");
+        test.run();
+
+        execute("INSERT INTO %s (pk, value) VALUES (0, [1, ?])", 2);
+        test.run();
+
+        execute("INSERT INTO %s (pk, value) VALUES (0, [1, (int) ?])", 2);
+        test.run();
+
+        execute("INSERT INTO %s (pk, value) VALUES (0, [1, 1 + (int) ?])", 1);
+        test.run();
+    }
+
+    @Test
+    public void invalidNumberOfDimensionsFixedWidth() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value vector<int, 2>)");
+
+        // fewer values than expected, with literals and bind markers
+        assertInvalidThrowMessage("Invalid vector literal for value of type vector<int, 2>; expected 2 elements, but given 1",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, [1])");
+        assertInvalidThrowMessage("Not enough bytes to read a vector<int, 2>",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1));
+
+        // more values than expected, with literals and bind markers
+        assertInvalidThrowMessage("Invalid vector literal for value of type vector<int, 2>; expected 2 elements, but given 3",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, [1, 2, 3])");
+        assertInvalidThrowMessage("Unexpected 4 extraneous bytes after vector<int, 2> value",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, 2, 3));
+    }
+
+    @Test
+    public void invalidNumberOfDimensionsVariableWidth() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value vector<text, 2>)");
+
+        // fewer values than expected, with literals and bind markers
+        assertInvalidThrowMessage("Invalid vector literal for value of type vector<text, 2>; expected 2 elements, but given 1",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ['a'])");
+        assertInvalidThrowMessage("Not enough bytes to read a vector<text, 2>",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ?)", vector("a"));
+
+        // more values than expected, with literals and bind markers
+        assertInvalidThrowMessage("Invalid vector literal for value of type vector<text, 2>; expected 2 elements, but given 3",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ['a', 'b', 'c'])");
+        assertInvalidThrowMessage("Unexpected 2 extraneous bytes after vector<text, 2> value",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ?)", vector("a", "b", "c"));
+    }
+
+    @Test
+    public void sandwichBetweenUDTs()
+    {
+        createType("CREATE TYPE cql_test_keyspace.b (y int);");
+        createType("CREATE TYPE cql_test_keyspace.a (z vector<frozen<b>, 2>);");
+
+        createTable("CREATE TABLE %s (pk int primary key, value a)");
+
+        execute("INSERT INTO %s (pk, value) VALUES (0, {z: [{y:1}, {y:2}]})");
+        assertRows(execute("SELECT * FROM %s"),
+                   row(0, userType("z", vector(userType("y", 1), userType("y", 2)))));
+    }
+
+    @Test
+    public void invalidElementTypeFixedWidth() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value vector<int, 2>)");
+
+        // fixed-length bigint instead of int, with literals and bind markers
+        assertInvalidThrowMessage("Invalid vector literal for value: value (bigint)1 is not of type int",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, [(bigint) 1, (bigint) 2])");
+        assertInvalidThrowMessage("Unexpected 8 extraneous bytes after vector<int, 2> value",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1L, Long.MAX_VALUE));
+
+        // variable-length text instead of int, with literals and bind markers
+        assertInvalidThrowMessage("Invalid vector literal for value: value 'a' is not of type int",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ['a', 'b'])");
+        assertInvalidThrowMessage("Not enough bytes to read a vector<int, 2>",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ?)", vector("a", "b"));
+    }
+
+    @Test
+    public void invalidElementTypeVariableWidth() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value vector<text, 2>)");
+
+        // fixed-length int instead of text, with literals and bind markers
+        assertInvalidThrowMessage("Invalid vector literal for value: value 1 is not of type text",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, [1, 2])");
+        assertInvalidThrowMessage("Unexpected 6 extraneous bytes after vector<text, 2> value",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, 2));
+
+        // variable-length varint instead of text, with literals and bind markers
+        assertInvalidThrowMessage("Invalid vector literal for value: value (varint)1 is not of type text",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, [(varint) 1, (varint) 2])");
+        assertInvalidThrowMessage("String didn't validate.",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (pk, value) VALUES (0, ?)",
+                                  vector(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE), BigInteger.ONE));
+    }
+
+    @Test
+    public void update()
+    {
+        Runnable test = () -> {
+            assertRows(execute("SELECT * FROM %s"), row(0, list(1, 2)));
+            execute("TRUNCATE %s");
+            assertRows(execute("SELECT * FROM %s"));
+        };
+
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<int, 2>)");
+
+        execute("UPDATE %s set VALUE = [1, 2] WHERE pk = 0");
+        test.run();
+
+        execute("UPDATE %s set VALUE = ? WHERE pk = 0", vector(1, 2));
+        test.run();
+
+        execute("UPDATE %s set VALUE = [1, 1 + 1] WHERE pk = 0");
+        test.run();
+
+        execute("UPDATE %s set VALUE = [1, ?] WHERE pk = 0", 2);
+        test.run();
+
+        execute("UPDATE %s set VALUE = [1, (int) ?] WHERE pk = 0", 2);
+        test.run();
+
+        execute("UPDATE %s set VALUE = [1, 1 + (int) ?] WHERE pk = 0", 1);
+        test.run();
+    }
+
+    @Test
+    public void nullValues()
+    {
+        assertAcceptsNullValues("int"); // fixed length
+        assertAcceptsNullValues("float"); // fixed length with special/optimized treatment
+        assertAcceptsNullValues("text"); // variable length
+    }
+
+    private void assertAcceptsNullValues(String type)
+    {
+        createTable(format("CREATE TABLE %%s (k int primary key, v vector<%s, 2>)", type));
+
+        execute("INSERT INTO %s (k, v) VALUES (0, null)");
+        assertRows(execute("SELECT * FROM %s"), row(0, null));
+
+        execute("INSERT INTO %s (k, v) VALUES (0, ?)", (List<Integer>) null);
+        assertRows(execute("SELECT * FROM %s"), row(0, null));
+    }
+
+    @Test
+    public void emptyValues() throws Throwable
+    {
+        assertRejectsEmptyValues("int"); // fixed length
+        assertRejectsEmptyValues("float"); // fixed length with special/optimized treatment
+        assertRejectsEmptyValues("text"); // variable length
+    }
+
+    private void assertRejectsEmptyValues(String type) throws Throwable
+    {
+        createTable(format("CREATE TABLE %%s (k int primary key, v vector<%s, 2>)", type));
+
+        assertInvalidThrowMessage(format("Invalid HEX constant (0x) for \"v\" of type vector<%s, 2>", type),
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (k, v) VALUES (0, 0x)");
+
+        assertInvalidThrowMessage("Invalid empty vector value",
+                                  InvalidRequestException.class,
+                                  "INSERT INTO %s (k, v) VALUES (0, ?)",
+                                  ByteBufferUtil.EMPTY_BYTE_BUFFER);
+    }
+
+    @Test
+    public void functions()
+    {
+        VectorType<Integer> type = VectorType.getInstance(Int32Type.instance, 2);
+        Vector<Integer> vector = vector(1, 2);
+
+        NativeFunctions.instance.add(new NativeScalarFunction("f", type, type)
+        {
+            @Override
+            public ByteBuffer execute(Arguments arguments) throws InvalidRequestException
+            {
+                return arguments.get(0);
+            }
+            
+            @Override
+            public Arguments newArguments(ProtocolVersion version)
+            {
+                return FunctionArguments.newNoopInstance(version, 1);
+            }
+        });
+
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<int, 2>)");
+        execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector);
+
+        assertRows(execute("SELECT f(value) FROM %s WHERE pk=0"), row(vector));
+        assertRows(execute("SELECT f([1, 2]) FROM %s WHERE pk=0"), row(vector));
+    }
+
+    @Test
+    public void specializedFunctions()
+    {
+        VectorType<Float> type = VectorType.getInstance(FloatType.instance, 2);
+        Vector<Float> vector = vector(1.0f, 2.0f);
+
+        NativeFunctions.instance.add(new NativeScalarFunction("f", type, type, type)
+        {
+            @Override
+            public ByteBuffer execute(Arguments arguments) throws InvalidRequestException
+            {
+                float[] left = arguments.get(0);
+                float[] right = arguments.get(1);
+                int size = Math.min(left.length, right.length);
+                float[] sum = new float[size];
+                for (int i = 0; i < size; i++)
+                    sum[i] = left[i] + right[i];
+                return type.getSerializer().serializeFloatArray(sum);
+            }
+
+            @Override
+            public Arguments newArguments(ProtocolVersion version)
+            {
+                return new FunctionArguments(version,
+                                             (v, b) -> type.composeAsFloat(b),
+                                             (v, b) -> type.composeAsFloat(b));
+            }
+        });
+
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<float, 2>)");
+        execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector);
+        execute("INSERT INTO %s (pk, value) VALUES (1, ?)", vector);
+
+        Object[][] expected = { row(vector(2f, 4f)), row(vector(2f, 4f)) };
+        assertRows(execute("SELECT f(value, [1.0, 2.0]) FROM %s"), expected);
+        assertRows(execute("SELECT f([1.0, 2.0], value) FROM %s"), expected);
+    }
+
+    @Test
+    public void token()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk vector<int, 2> primary key)");
+        execute("INSERT INTO %s (pk) VALUES (?)", vector(1, 2));
+        long tokenColumn = execute("SELECT token(pk) as t FROM %s").one().getLong("t");
+        long tokenTerminal = execute("SELECT token([1, 2]) as t FROM %s").one().getLong("t");
+        Assert.assertEquals(tokenColumn, tokenTerminal);
+    }
+
+    @Test
+    public void udf() throws Throwable
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<int, 2>)");
+        Vector<Integer> vector = vector(1, 2);
+        execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector);
+
+        // identity function
+        String f = createFunction(KEYSPACE,
+                                  "",
+                                  "CREATE FUNCTION %s (x vector<int, 2>) " +
+                                  "CALLED ON NULL INPUT " +
+                                  "RETURNS vector<int, 2> " +
+                                  "LANGUAGE java " +
+                                  "AS 'return x;'");
+        assertRows(execute(format("SELECT %s(value) FROM %%s", f)), row(vector));
+        assertRows(execute(format("SELECT %s([2, 3]) FROM %%s", f)), row(vector(2, 3)));
+        assertRows(execute(format("SELECT %s(null) FROM %%s", f)), row((Vector<Integer>) null));
+
+        // identitiy function with nested type
+        f = createFunction(KEYSPACE,
+                           "",
+                           "CREATE FUNCTION %s (x list<vector<int, 2>>) " +
+                           "CALLED ON NULL INPUT " +
+                           "RETURNS list<vector<int, 2>> " +
+                           "LANGUAGE java " +
+                           "AS 'return x;'");
+        assertRows(execute(format("SELECT %s([value]) FROM %%s", f)), row(list(vector)));
+        assertRows(execute(format("SELECT %s([[2, 3]]) FROM %%s", f)), row(list(vector(2, 3))));
+        assertRows(execute(format("SELECT %s(null) FROM %%s", f)), row((Vector<Integer>) null));
+
+        // identitiy function with elements of variable length
+        f = createFunction(KEYSPACE,
+                           "",
+                           "CREATE FUNCTION %s (x vector<text, 2>) " +
+                           "CALLED ON NULL INPUT " +
+                           "RETURNS vector<text, 2> " +
+                           "LANGUAGE java " +
+                           "AS 'return x;'");
+        assertRows(execute(format("SELECT %s(['abc', 'defghij']) FROM %%s", f)), row(vector("abc", "defghij")));
+        assertRows(execute(format("SELECT %s(null) FROM %%s", f)), row((Vector<Integer>) null));
+
+        // Test wrong types on function creation
+        assertInvalidThrowMessage("vectors may only have positive dimensions; given 0",
+                                  InvalidRequestException.class,
+                                  "CREATE FUNCTION %s (x vector<int, 0>) " +
+                                  "CALLED ON NULL INPUT " +
+                                  "RETURNS vector<int, 2> " +
+                                  "LANGUAGE java " +
+                                  "AS 'return x;'");
+        assertInvalidThrowMessage("vectors may only have positive dimensions; given 0",
+                                  InvalidRequestException.class,
+                                  "CREATE FUNCTION %s (x vector<int, 2>) " +
+                                  "CALLED ON NULL INPUT " +
+                                  "RETURNS vector<int, 0> " +
+                                  "LANGUAGE java " +
+                                  "AS 'return x;'");
+
+        // make sure the function referencing the UDT is dropped before dropping the UDT at cleanup
+        execute("DROP FUNCTION " + f);
+    }
+
+    @Test
+    public void explicitlyFrozen() throws Throwable
+    {
+        // explicitly frozen
+        execute(format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<vector<text, 2>>)", KEYSPACE));
+    }
+
+    @Test
+    public void invalidSyntax() throws Throwable
+    {
+        assertInvalidThrowMessage("mismatched input '>' expecting ','",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v vector<text>)", KEYSPACE));
+        assertInvalidThrowMessage("no viable alternative at input '2'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v vector<2>)", KEYSPACE));
+        assertInvalidThrowMessage("no viable alternative at input '>'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v vector<>)", KEYSPACE));
+        assertInvalidThrowMessage("Unknown type cql_test_keyspace.vector",
+                                  InvalidRequestException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v vector)", KEYSPACE));
+
+        assertInvalidThrowMessage("mismatched input '>' expecting ','",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<vector<text>>)", KEYSPACE));
+        assertInvalidThrowMessage("no viable alternative at input '2'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<vector<2>>)", KEYSPACE));
+        assertInvalidThrowMessage("no viable alternative at input '>'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<vector<>>)", KEYSPACE));
+        assertInvalidThrowMessage("Unknown type cql_test_keyspace.vector",
+                                  InvalidRequestException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<vector>)", KEYSPACE));
+    }
+
+    @SafeVarargs
+    protected final <T> Vector<T> vector(T... values)
+    {
+        return new Vector<>(values);
+    }}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
index e6ef163af256..50d146f67bd4 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
@@ -1751,15 +1751,15 @@ public void testUDTAndCollectionNestedAccess() throws Throwable
     {
         String type = createType("CREATE TYPE %s (s set<int>, m map<text, text>)");
 
-        assertInvalidMessage("Non-frozen UDTs are not allowed inside collections",
+        assertInvalidMessage("non-frozen user types are only supported at top-level",
                              "CREATE TABLE " + KEYSPACE + ".t (k int PRIMARY KEY, v map<text, " + type + ">)");
 
         String mapType = "map<text, frozen<" + type + ">>";
         for (boolean frozen : new boolean[]{false, true})
         {
-            mapType = frozen ? "frozen<" + mapType + ">" : mapType;
+            mapType = frozen ? "frozen<" + mapType + '>' : mapType;
 
-            createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + mapType + ")");
+            createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + mapType + ')');
 
             execute("INSERT INTO %s(k, v) VALUES (0, ?)", map("abc", userType("s", set(2, 4, 6), "m", map("a", "v1", "d", "v2"))));
 
@@ -1771,16 +1771,16 @@ public void testUDTAndCollectionNestedAccess() throws Throwable
             });
         }
 
-        assertInvalidMessage("Non-frozen UDTs with nested non-frozen collections are not supported",
-                             "CREATE TABLE " + KEYSPACE + ".t (k int PRIMARY KEY, v " + type + ")");
+        assertInvalidMessage("non-frozen collections are only supported at top-level",
+                             "CREATE TABLE " + KEYSPACE + ".t (k int PRIMARY KEY, v " + type + ')');
 
         type = createType("CREATE TYPE %s (s frozen<set<int>>, m frozen<map<text, text>>)");
 
         for (boolean frozen : new boolean[]{false, true})
         {
-            type = frozen ? "frozen<" + type + ">" : type;
+            type = frozen ? "frozen<" + type + '>' : type;
 
-            createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + type + ")");
+            createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + type + ')');
 
             execute("INSERT INTO %s(k, v) VALUES (0, ?)", userType("s", set(2, 4, 6), "m", map("a", "v1", "d", "v2")));
 
@@ -1878,7 +1878,7 @@ public void testInsertingCollectionsWithInvalidElements() throws Throwable
                              "INSERT INTO %s (k, s) VALUES (0, ?)",
                              set(tuple(1, "1", 1.0, 1), tuple(2, "2", 2.0, 2)));
 
-        assertInvalidMessage("Invalid set literal for s: value (1, '1', 1.0, 1) is not of type frozen<tuple<int, text, double>>",
+        assertInvalidMessage("Invalid set literal for s: value (1, '1', 1.0, 1) is not of type tuple<int, text, double>",
                              "INSERT INTO %s (k, s) VALUES (0, {(1, '1', 1.0, 1)})");
 
         createTable("CREATE TABLE %s (k int PRIMARY KEY, l frozen<list<tuple<int, text, double>>>)");
@@ -1886,7 +1886,7 @@ public void testInsertingCollectionsWithInvalidElements() throws Throwable
                              "INSERT INTO %s (k, l) VALUES (0, ?)",
                              list(tuple(1, "1", 1.0, 1), tuple(2, "2", 2.0, 2)));
 
-        assertInvalidMessage("Invalid list literal for l: value (1, '1', 1.0, 1) is not of type frozen<tuple<int, text, double>>",
+        assertInvalidMessage("Invalid list literal for l: value (1, '1', 1.0, 1) is not of type tuple<int, text, double>",
                              "INSERT INTO %s (k, l) VALUES (0, [(1, '1', 1.0, 1)])");
 
         createTable("CREATE TABLE %s (k int PRIMARY KEY, m frozen<map<tuple<int, text, double>, int>>)");
@@ -1894,7 +1894,7 @@ public void testInsertingCollectionsWithInvalidElements() throws Throwable
                              "INSERT INTO %s (k, m) VALUES (0, ?)",
                              map(tuple(1, "1", 1.0, 1), 1, tuple(2, "2", 2.0, 2), 2));
 
-        assertInvalidMessage("Invalid map literal for m: key (1, '1', 1.0, 1) is not of type frozen<tuple<int, text, double>>",
+        assertInvalidMessage("Invalid map literal for m: key (1, '1', 1.0, 1) is not of type tuple<int, text, double>",
                              "INSERT INTO %s (k, m) VALUES (0, {(1, '1', 1.0, 1) : 1})");
 
         createTable("CREATE TABLE %s (k int PRIMARY KEY, m frozen<map<int, tuple<int, text, double>>>)");
@@ -1902,7 +1902,7 @@ public void testInsertingCollectionsWithInvalidElements() throws Throwable
                              "INSERT INTO %s (k, m) VALUES (0, ?)",
                              map(1, tuple(1, "1", 1.0, 1), 2, tuple(2, "2", 2.0, 2)));
 
-        assertInvalidMessage("Invalid map literal for m: value (1, '1', 1.0, 1) is not of type frozen<tuple<int, text, double>>",
+        assertInvalidMessage("Invalid map literal for m: value (1, '1', 1.0, 1) is not of type tuple<int, text, double>",
                              "INSERT INTO %s (k, m) VALUES (0, {1 : (1, '1', 1.0, 1)})");
     }
 
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java
index b52caaa479eb..216b5982f570 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java
@@ -22,7 +22,8 @@
 
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.InvalidColumnTypeException;
+import org.apache.cassandra.exceptions.RequestValidationException;
 
 public class CountersTest extends CQLTester
 {
@@ -34,7 +35,7 @@ public class CountersTest extends CQLTester
     public void testRegularCounters() throws Throwable
     {
         assertInvalidThrowMessage("Cannot mix counter and non counter columns in the same table",
-                                  InvalidRequestException.class,
+                                  InvalidColumnTypeException.class,
                                   String.format("CREATE TABLE %s.%s (id bigint PRIMARY KEY, count counter, things set<text>)", KEYSPACE, createTableName()));
     }
 
@@ -42,10 +43,10 @@ public void testRegularCounters() throws Throwable
     public void testCannotAlterWithNonCounterColumn() throws Throwable
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, c counter)");
-        assertInvalidThrowMessage("Cannot have a non counter column (\"t\") in a counter table",
+        assertInvalidThrowMessage("Invalid type text for column t: Cannot mix counter and non counter columns in the same table",
                 ConfigurationException.class, formatQuery("ALTER TABLE %s ADD t text"));
         createTable("CREATE TABLE %s (k int PRIMARY KEY, t text)");
-        assertInvalidThrowMessage("Cannot have a counter column (\"c\") in a non counter table",
+        assertInvalidThrowMessage("Invalid type counter for column c: Cannot mix counter and non counter columns in the same table",
                                   ConfigurationException.class, formatQuery("ALTER TABLE %s ADD c counter"));
     }
 
@@ -56,18 +57,35 @@ public void testCannotAlterWithNonCounterColumn() throws Throwable
     public void testCountersOnCollections() throws Throwable
     {
         String tableName = KEYSPACE + "." + createTableName();
-        assertInvalidThrow(InvalidRequestException.class,
+        assertInvalidThrow(InvalidColumnTypeException.class,
                            String.format("CREATE TABLE %s (k int PRIMARY KEY, l list<counter>)", tableName));
 
         tableName = KEYSPACE + "." + createTableName();
-        assertInvalidThrow(InvalidRequestException.class,
+        assertInvalidThrow(InvalidColumnTypeException.class,
                            String.format("CREATE TABLE %s (k int PRIMARY KEY, s set<counter>)", tableName));
 
         tableName = KEYSPACE + "." + createTableName();
-        assertInvalidThrow(InvalidRequestException.class,
+        assertInvalidThrow(InvalidColumnTypeException.class,
                            String.format("CREATE TABLE %s (k int PRIMARY KEY, m map<text, counter>)", tableName));
     }
 
+    /**
+     * Migrated from user_types_test.py::TestUserTypes::test_no_counters_in_user_types
+     */
+    @Test
+    public void testCountersOnUserTypes() throws Throwable
+    {
+        String typeName = KEYSPACE + '.' + createTypeName();
+        assertInvalidThrowMessage("A user type cannot contain counters",
+                                  RequestValidationException.class,
+                                  String.format("CREATE TYPE %s (a counter)", typeName));
+
+        typeName = KEYSPACE + '.' + createType("CREATE TYPE %s (a int)");
+        assertInvalidThrowMessage("A user type cannot contain counters",
+                                  RequestValidationException.class,
+                                  String.format("ALTER TYPE %s ADD b counter", typeName));
+    }
+
     @Test
     public void testCounterUpdatesWithUnset() throws Throwable
     {
@@ -172,8 +190,8 @@ public void testCounterFilteringWithNull() throws Throwable
     @Test
     public void testProhibitReversedCounterAsPartOfPrimaryKey() throws Throwable
     {
-        assertInvalidThrowMessage("counter type is not supported for PRIMARY KEY column 'a'",
-                                  InvalidRequestException.class, String.format("CREATE TABLE %s.%s (a counter, b int, PRIMARY KEY (b, a)) WITH CLUSTERING ORDER BY (a desc);", KEYSPACE, createTableName()));
+        assertInvalidThrowMessage("counters are not supported within PRIMARY KEY columns",
+                                  InvalidColumnTypeException.class, String.format("CREATE TABLE %s.%s (a counter, b int, PRIMARY KEY (b, a)) WITH CLUSTERING ORDER BY (a desc);", KEYSPACE, createTableName()));
     }
 
     /**
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
index 369f90f70af0..04a4f4f47030 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
@@ -17,8 +17,8 @@
  */
 package org.apache.cassandra.cql3.validation.entities;
 
-import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
@@ -38,6 +38,7 @@
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 
+import static java.lang.String.format;
 import static org.junit.Assert.assertEquals;
 
 public class FrozenCollectionsTest extends CQLTester
@@ -806,10 +807,10 @@ public void testInvalidOperations() throws Throwable
         assertInvalid("DELETE m[?] FROM %s WHERE k=?", 0, 0);
 
         assertInvalidCreateWithMessage("CREATE TABLE %s (k int PRIMARY KEY, t set<set<int>>)",
-                "Non-frozen collections are not allowed inside collections");
+                                       "non-frozen collections are only supported at top-level");
 
         assertInvalidCreateWithMessage("CREATE TABLE %s (k int PRIMARY KEY, t frozen<set<counter>>)",
-                                       "Counters are not allowed inside collections");
+                                       "counters are not allowed within collections");
 
         assertInvalidCreateWithMessage("CREATE TABLE %s (k int PRIMARY KEY, t frozen<text>)",
                 "frozen<> is only allowed on collections, tuples, and user-defined types");
@@ -870,7 +871,7 @@ public void testSecondaryIndex() throws Throwable
                    row(0, list(1, 2, 3), set(1, 2, 3), map(1, "a")),
                    row(1, list(1, 2, 3), set(4, 5, 6), map(2, "b")));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "d"),
                              "SELECT * FROM %s WHERE d CONTAINS KEY ?", 1);
 
         assertRows(execute("SELECT * FROM %s WHERE b CONTAINS ? AND d CONTAINS KEY ? ALLOW FILTERING", 1, 1),
@@ -1378,10 +1379,9 @@ public void testToString()
         assertEquals("MapType(ListType(Int32Type),Int32Type)", clean(m.toString(true)));
 
         // tuple<set<int>>
-        List<AbstractType<?>> types = new ArrayList<>();
-        types.add(SetType.getInstance(Int32Type.instance, true));
+        List<AbstractType<?>> types = Collections.singletonList(SetType.getInstance(Int32Type.instance, true));
         TupleType tuple = new TupleType(types);
-        assertEquals("TupleType(SetType(Int32Type))", clean(tuple.toString()));
+        assertEquals("FrozenType(TupleType(SetType(Int32Type)))", clean(tuple.toString()));
     }
 
     @Test
@@ -1480,4 +1480,44 @@ public void testSetsWithElementsBiggerThan64K() throws Throwable
 
         assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set(largeText, "v1", "v2")));
     }
+
+    @Test
+    public void testInvalidSyntax() throws Throwable
+    {
+        // lists
+        assertInvalidThrowMessage("no viable alternative at input '>'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<list<>>)", KEYSPACE));
+        assertInvalidThrowMessage("mismatched input ',' expecting '>'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<list<int, int>>)", KEYSPACE));
+        assertInvalidThrowMessage("Unknown type cql_test_keyspace.list",
+                                  InvalidRequestException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<list>)", KEYSPACE));
+
+        // sets
+        assertInvalidThrowMessage("no viable alternative at input '>'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<set<>>)", KEYSPACE));
+        assertInvalidThrowMessage("mismatched input ',' expecting '>'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<set<int, int>>)", KEYSPACE));
+        assertInvalidThrowMessage("mismatched input '>' expecting '<'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<set>)", KEYSPACE));
+
+        // maps
+        assertInvalidThrowMessage("mismatched input '>' expecting ','",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<map<text>>)", KEYSPACE));
+        assertInvalidThrowMessage("no viable alternative at input '2'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<map<2>>)", KEYSPACE));
+        assertInvalidThrowMessage("no viable alternative at input '>'",
+                                  SyntaxException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<map<>>)", KEYSPACE));
+        assertInvalidThrowMessage("Unknown type cql_test_keyspace.map",
+                                  InvalidRequestException.class,
+                                  format("CREATE TABLE %s.t (k int PRIMARY KEY, v frozen<map>)", KEYSPACE));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
index 59a3000071d4..ee41c0135e2f 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
@@ -28,6 +28,12 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
@@ -45,6 +51,7 @@
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.index.IndexBuildInProgressException;
 import org.apache.cassandra.index.IndexNotAvailableException;
 import org.apache.cassandra.index.SecondaryIndexManager;
 import org.apache.cassandra.index.StubIndex;
@@ -81,21 +88,21 @@ public static void setDefaultSecondaryIndex()
     }
 
     @Test
-    public void testCreateAndDropIndex() throws Throwable
+    public void testCreateAndDropIndex()
     {
         testCreateAndDropIndex("test", false);
         testCreateAndDropIndex("test2", true);
     }
 
     @Test
-    public void testCreateAndDropIndexWithQuotedIdentifier() throws Throwable
+    public void testCreateAndDropIndexWithQuotedIdentifier()
     {
         testCreateAndDropIndex("\"quoted_ident\"", false);
         testCreateAndDropIndex("\"quoted_ident2\"", true);
     }
 
     @Test
-    public void testCreateAndDropIndexWithCamelCaseIdentifier() throws Throwable
+    public void testCreateAndDropIndexWithCamelCaseIdentifier()
     {
         testCreateAndDropIndex("CamelCase", false);
         testCreateAndDropIndex("CamelCase2", true);
@@ -106,9 +113,8 @@ public void testCreateAndDropIndexWithCamelCaseIdentifier() throws Throwable
      *
      * @param indexName         the index name
      * @param addKeyspaceOnDrop add the keyspace name in the drop statement
-     * @throws Throwable if an error occurs
      */
-    private void testCreateAndDropIndex(String indexName, boolean addKeyspaceOnDrop) throws Throwable
+    private void testCreateAndDropIndex(String indexName, boolean addKeyspaceOnDrop)
     {
         assertInvalidMessage(format("Index '%s.%s' doesn't exist",
                                     KEYSPACE,
@@ -285,7 +291,7 @@ public void testRangeQuery() throws Throwable
      * migrated from cql_tests.py:TestCQL.compression_option_validation_test()
      */
     @Test
-    public void testUnknownCompressionOptions() throws Throwable
+    public void testUnknownCompressionOptions()
     {
         String tableName = createTableName();
         assertInvalidThrow(SyntaxException.class, format("CREATE TABLE %s (key varchar PRIMARY KEY, password varchar, gender varchar) WITH compression_parameters:sstable_compressor = 'DeflateCompressor'", tableName));
@@ -344,7 +350,7 @@ public void testIndexOnComposite() throws Throwable
      * migrated from cql_tests.py:TestCQL.refuse_in_with_indexes_test()
      */
     @Test
-    public void testInvalidIndexSelect() throws Throwable
+    public void testInvalidIndexSelect()
     {
         createTable("create table %s (pk varchar primary key, col1 varchar, col2 varchar)");
         createIndex("create index on %s (col1)");
@@ -432,7 +438,7 @@ public void testIndexOnCompoundRowKey() throws Throwable
      * Migrated from cql_tests.py:TestCQL.secondary_index_counters()
      */
     @Test
-    public void testIndexOnCountersInvalid() throws Throwable
+    public void testIndexOnCountersInvalid()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, c counter)");
         assertInvalid("CREATE INDEX ON test(c)");
@@ -477,6 +483,104 @@ public void testIndexOnCollections() throws Throwable
         });
     }
 
+    private static void assertBackingTableKeyValidator(SecondaryIndexManager indexManager, String indexName, AbstractType<?> expectedType)
+    {
+        assertEquals(expectedType, indexManager.getIndexByName(indexName)
+                                               .getBackingTable()
+                                               .map(ColumnFamilyStore::metadata)
+                                               .map(m -> m.partitionKeyType)
+                                               .orElseThrow(AssertionError::new));
+    }
+
+    /**
+     * Test for DB-1121
+     */
+    @Test
+    public void testIndexOnCollectionsBackingTableKeyValidator()
+    {
+        createTable("CREATE TABLE %s (" +
+                    "k int PRIMARY KEY, " +
+                    "non_frozen_list list<int>, " +
+                    "non_frozen_set set<text>, " +
+                    "non_frozen_map map<text, int>," +
+                    "frozen_list frozen<list<int>>, " +
+                    "frozen_set frozen<set<text>>, " +
+                    "frozen_map frozen<map<text, int>>)");
+
+        createIndex("CREATE INDEX non_frozen_list_idx ON %s (non_frozen_list)");
+        createIndex("CREATE INDEX non_frozen_set_idx ON %s (non_frozen_set)");
+        createIndex("CREATE INDEX non_frozen_map_idx ON %s (non_frozen_map)");
+        createIndex("CREATE INDEX non_frozen_map_keys_idx ON %s (KEYS(non_frozen_map))");
+        createIndex("CREATE INDEX non_frozen_map_entries_idx ON %s (ENTRIES(non_frozen_map))");
+        createIndex("CREATE INDEX frozen_list_idx ON %s (FULL(frozen_list))");
+        createIndex("CREATE INDEX frozen_set_idx ON %s (FULL(frozen_set))");
+        createIndex("CREATE INDEX frozen_map_idx ON %s (FULL(frozen_map))");
+
+        SecondaryIndexManager indexManager = ColumnFamilyStore.getIfExists(keyspace(), currentTable()).indexManager;
+
+        assertBackingTableKeyValidator(indexManager, "non_frozen_list_idx", Int32Type.instance);
+        assertBackingTableKeyValidator(indexManager, "non_frozen_set_idx", UTF8Type.instance);
+        assertBackingTableKeyValidator(indexManager, "non_frozen_map_idx", Int32Type.instance);
+        assertBackingTableKeyValidator(indexManager, "non_frozen_map_keys_idx", UTF8Type.instance);
+        assertBackingTableKeyValidator(indexManager, "non_frozen_map_entries_idx", CompositeType.getInstance(UTF8Type.instance, Int32Type.instance));
+        assertBackingTableKeyValidator(indexManager, "frozen_list_idx", ListType.getInstance(Int32Type.instance, false));
+        assertBackingTableKeyValidator(indexManager, "frozen_set_idx", SetType.getInstance(UTF8Type.instance, false));
+        assertBackingTableKeyValidator(indexManager, "frozen_map_idx", MapType.getInstance(UTF8Type.instance, Int32Type.instance, false));
+
+        // Unsupported index types for non-frozen list
+        assertInvalidMessage("Cannot create index on keys of column non_frozen_list with non-map type",
+                             "CREATE INDEX ON %s (KEYS(non_frozen_list))");
+        assertInvalidMessage("Cannot create index on entries of column non_frozen_list with non-map type",
+                             "CREATE INDEX ON %s (ENTRIES(non_frozen_list))");
+        assertInvalidMessage("full() indexes can only be created on frozen collections",
+                             "CREATE INDEX ON %s (FULL(non_frozen_list))");
+
+        // Unsupported index types for non-frozen set
+        assertInvalidMessage("Cannot create index on keys of column non_frozen_set with non-map type",
+                             "CREATE INDEX ON %s (KEYS(non_frozen_set))");
+        assertInvalidMessage("Cannot create index on entries of column non_frozen_set with non-map type",
+                             "CREATE INDEX ON %s (ENTRIES(non_frozen_set))");
+        assertInvalidMessage("full() indexes can only be created on frozen collections",
+                             "CREATE INDEX ON %s (FULL(non_frozen_set))");
+
+        // Unsupported index types for non-frozen map
+        assertInvalidMessage("full() indexes can only be created on frozen collections",
+                             "CREATE INDEX ON %s (FULL(non_frozen_map))");
+
+        // Unsupported index types for frozen list
+        assertInvalidMessage("Cannot create keys() index on frozen column frozen_list. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_list)' modifier",
+                             "CREATE INDEX ON %s (KEYS(frozen_list))");
+        assertInvalidMessage("Cannot create entries() index on frozen column frozen_list. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_list)' modifier",
+                             "CREATE INDEX ON %s (ENTRIES(frozen_list))");
+        assertInvalidMessage("Cannot create values() index on frozen column frozen_list. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_list)' modifier",
+                             "CREATE INDEX ON %s (VALUES(frozen_list))");
+
+        // Unsupported index types for frozen set
+        assertInvalidMessage("Cannot create keys() index on frozen column frozen_set. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_set)' modifier",
+                             "CREATE INDEX ON %s (KEYS(frozen_set))");
+        assertInvalidMessage("Cannot create entries() index on frozen column frozen_set. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_set)' modifier",
+                             "CREATE INDEX ON %s (ENTRIES(frozen_set))");
+        assertInvalidMessage("Cannot create values() index on frozen column frozen_set. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_set)' modifier",
+                             "CREATE INDEX ON %s (VALUES(frozen_set))");
+
+        // Unsupported index types for frozen map
+        assertInvalidMessage("Cannot create keys() index on frozen column frozen_map. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_map)' modifier",
+                             "CREATE INDEX ON %s (KEYS(frozen_map))");
+        assertInvalidMessage("Cannot create entries() index on frozen column frozen_map. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_map)' modifier",
+                             "CREATE INDEX ON %s (ENTRIES(frozen_map))");
+        assertInvalidMessage("Cannot create values() index on frozen column frozen_map. Frozen collections " +
+                             "are immutable and must be fully indexed by using the 'full(frozen_map)' modifier",
+                             "CREATE INDEX ON %s (VALUES(frozen_map))");
+    }
+
     @Test
     public void testSelectOnMultiIndexOnCollectionsWithNull() throws Throwable
     {
@@ -593,7 +697,7 @@ public void testSelectCountOnIndexedColumn() throws Throwable
     }
 
     @Test
-    public void testSyntaxVariationsForIndexOnCollectionsValue() throws Throwable
+    public void testSyntaxVariationsForIndexOnCollectionsValue()
     {
         createTable("CREATE TABLE %s (k int, m map<int, int>, l list<int>, s set<int>, PRIMARY KEY (k))");
         createAndDropCollectionValuesIndex("m");
@@ -601,7 +705,7 @@ public void testSyntaxVariationsForIndexOnCollectionsValue() throws Throwable
         createAndDropCollectionValuesIndex("s");
     }
 
-    private void createAndDropCollectionValuesIndex(String columnName) throws Throwable
+    private void createAndDropCollectionValuesIndex(String columnName)
     {
         String indexName = columnName + "_idx";
         SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager;
@@ -616,7 +720,7 @@ private void createAndDropCollectionValuesIndex(String columnName) throws Throwa
     }
 
     @Test
-    public void testCreateIndexWithQuotedColumnNames() throws Throwable
+    public void testCreateIndexWithQuotedColumnNames()
     {
         createTable("CREATE TABLE %s (" +
                     " k int," +
@@ -638,7 +742,7 @@ public void testCreateIndexWithQuotedColumnNames() throws Throwable
         createAndDropIndexWithQuotedColumnIdentifier("\"column_name_with\"\"escaped quote\"");
     }
 
-    private void createAndDropIndexWithQuotedColumnIdentifier(String target) throws Throwable
+    private void createAndDropIndexWithQuotedColumnIdentifier(String target)
     {
         String indexName = "test_mixed_case_idx";
         createIndex(format("CREATE INDEX %s ON %%s(%s)", indexName, target));
@@ -660,7 +764,7 @@ private void createAndDropIndexWithQuotedColumnIdentifier(String target) throws
      * and restarting the node, here we just cleanup the cache.
      */
     @Test
-    public void testCanQuerySecondaryIndex() throws Throwable
+    public void testCanQuerySecondaryIndex()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v int,)");
 
@@ -680,7 +784,7 @@ public void testCanQuerySecondaryIndex() throws Throwable
     // make sure we check conditional and unconditional statements,
     // both singly and in batches (CASSANDRA-10536)
     @Test
-    public void testIndexOnCompositeValueOver64k() throws Throwable
+    public void testIndexOnCompositeValueOver64k()
     {
         createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY (a))");
         createIndex("CREATE INDEX ON %s(c)");
@@ -697,7 +801,7 @@ public void testIndexOnCompositeValueOver64k() throws Throwable
     }
 
     @Test
-    public void testIndexOnPartitionKeyInsertValueOver64k() throws Throwable
+    public void testIndexOnPartitionKeyInsertValueOver64k()
     {
         createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY ((a, b)))");
         createIndex("CREATE INDEX ON %s(a)");
@@ -727,7 +831,7 @@ public void testIndexOnPartitionKeyInsertValueOver64k() throws Throwable
     }
 
     @Test
-    public void testIndexOnPartitionKeyWithStaticColumnAndNoRows() throws Throwable
+    public void testIndexOnPartitionKeyWithStaticColumnAndNoRows()
     {
         createTable("CREATE TABLE %s (pk1 int, pk2 int, c int, s int static, v int, PRIMARY KEY((pk1, pk2), c))");
         createIndex("CREATE INDEX ON %s (pk2)");
@@ -750,7 +854,7 @@ public void testIndexOnPartitionKeyWithStaticColumnAndNoRows() throws Throwable
     }
 
     @Test
-    public void testIndexOnClusteringColumnInsertValueOver64k() throws Throwable
+    public void testIndexOnClusteringColumnInsertValueOver64k()
     {
         createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY (a, b))");
         createIndex("CREATE INDEX ON %s(b)");
@@ -780,7 +884,7 @@ public void testIndexOnClusteringColumnInsertValueOver64k() throws Throwable
     }
 
     @Test
-    public void testIndexOnFullCollectionEntryInsertCollectionValueOver64k() throws Throwable
+    public void testIndexOnFullCollectionEntryInsertCollectionValueOver64k()
     {
         createTable("CREATE TABLE %s(a int, b frozen<map<int, blob>>, PRIMARY KEY (a))");
         createIndex("CREATE INDEX ON %s(full(b))");
@@ -797,7 +901,7 @@ public void testIndexOnFullCollectionEntryInsertCollectionValueOver64k() throws
     }
 
     @Test
-    public void prepareStatementsWithLIKEClauses() throws Throwable
+    public void prepareStatementsWithLIKEClauses()
     {
         createTable("CREATE TABLE %s (a int, c1 text, c2 text, v1 text, v2 text, v3 int, PRIMARY KEY (a, c1, c2))");
         createIndex(format("CREATE CUSTOM INDEX c1_idx on %%s(c1) USING '%s' WITH OPTIONS = {'mode' : 'PREFIX'}",
@@ -841,21 +945,21 @@ public void prepareStatementsWithLIKEClauses() throws Throwable
 
         // LIKE is not supported on indexes of non-literal values
         // this is rejected before binding, so the value isn't available in the error message
-        assertInvalidMessage("LIKE restriction is only supported on properly indexed columns. v3 LIKE ? is not valid",
+        assertInvalidMessage("Index on column v3 does not support LIKE restrictions.",
                              "SELECT * FROM %s WHERE v3 LIKE ?",
                              "%abc");
-        assertInvalidMessage("LIKE restriction is only supported on properly indexed columns. v3 LIKE ? is not valid",
+        assertInvalidMessage("Index on column v3 does not support LIKE restrictions.",
                              "SELECT * FROM %s WHERE v3 LIKE ?",
                              "%abc%");
-        assertInvalidMessage("LIKE restriction is only supported on properly indexed columns. v3 LIKE ? is not valid",
+        assertInvalidMessage("Index on column v3 does not support LIKE restrictions.",
                              "SELECT * FROM %s WHERE v3 LIKE ?",
                              "%abc%");
-        assertInvalidMessage("LIKE restriction is only supported on properly indexed columns. v3 LIKE ? is not valid",
+        assertInvalidMessage("Index on column v3 does not support LIKE restrictions.",
                              "SELECT * FROM %s WHERE v3 LIKE ?",
                              "abc");
     }
 
-    public void failInsert(String insertCQL, Object...args) throws Throwable
+    public void failInsert(String insertCQL, Object...args)
     {
         try
         {
@@ -868,7 +972,7 @@ public void failInsert(String insertCQL, Object...args) throws Throwable
         }
     }
 
-    public void succeedInsert(String insertCQL, Object...args) throws Throwable
+    public void succeedInsert(String insertCQL, Object...args)
     {
         execute(insertCQL, args);
         flush();
@@ -878,7 +982,7 @@ public void succeedInsert(String insertCQL, Object...args) throws Throwable
      * Migrated from cql_tests.py:TestCQL.clustering_indexing_test()
      */
     @Test
-    public void testIndexesOnClustering() throws Throwable
+    public void testIndexesOnClustering()
     {
         createTable("CREATE TABLE %s ( id1 int, id2 int, author text, time bigint, v1 text, v2 text, PRIMARY KEY ((id1, id2), author, time))");
 
@@ -911,7 +1015,7 @@ public void testIndexesOnClustering() throws Throwable
     }
 
     @Test
-    public void testMultipleIndexesOnOneColumn() throws Throwable
+    public void testMultipleIndexesOnOneColumn()
     {
         String indexClassName = StubIndex.class.getName();
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a), b))");
@@ -952,7 +1056,7 @@ public void testMultipleIndexesOnOneColumn() throws Throwable
     }
 
     @Test
-    public void testDeletions() throws Throwable
+    public void testDeletions()
     {
         // Test for bugs like CASSANDRA-10694.  These may not be readily visible with the built-in secondary index
         // implementation because of the stale entry handling.
@@ -1005,7 +1109,7 @@ public void testDeletions() throws Throwable
     }
 
     @Test
-    public void testUpdatesToMemtableData() throws Throwable
+    public void testUpdatesToMemtableData()
     {
         // verify the contract specified by Index.Indexer::updateRow(oldRowData, newRowData),
         // when a row in the memtable is updated, the indexer should be informed of:
@@ -1075,7 +1179,7 @@ public void testUpdatesToMemtableData() throws Throwable
     }
 
     @Test
-    public void testIndexQueriesWithIndexNotReady() throws Throwable
+    public void testIndexQueriesWithIndexNotReady()
     {
         createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
 
@@ -1089,7 +1193,7 @@ public void testIndexQueriesWithIndexNotReady() throws Throwable
             execute("SELECT value FROM %s WHERE value = 2");
             fail();
         }
-        catch (IndexNotAvailableException e)
+        catch (IndexBuildInProgressException e)
         {
             assertTrue(true);
         }
@@ -1100,7 +1204,7 @@ public void testIndexQueriesWithIndexNotReady() throws Throwable
     }
 
     @Test // A Bad init could leave an index only accepting reads
-    public void testReadOnlyIndex() throws Throwable
+    public void testReadOnlyIndex()
     {
         // On successful initialization both reads and writes go through
         createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
@@ -1138,7 +1242,7 @@ public void testReadOnlyIndex() throws Throwable
     }
 
     @Test  // A Bad init could leave an index only accepting writes
-    public void testWriteOnlyIndex() throws Throwable
+    public void testWriteOnlyIndex()
     {
         // On successful initialization both reads and writes go through
         createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
@@ -1176,7 +1280,7 @@ public void testWriteOnlyIndex() throws Throwable
     }
 
     @Test
-    public void droppingIndexInvalidatesPreparedStatements() throws Throwable
+    public void droppingIndexInvalidatesPreparedStatements()
     {
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a), b))");
         String indexName = createIndex("CREATE INDEX ON %s(c)");
@@ -1191,7 +1295,7 @@ public void droppingIndexInvalidatesPreparedStatements() throws Throwable
 
     // See CASSANDRA-11021
     @Test
-    public void testIndexesOnNonStaticColumnsWhereSchemaIncludesStaticColumns() throws Throwable
+    public void testIndexesOnNonStaticColumnsWhereSchemaIncludesStaticColumns()
     {
         createTable("CREATE TABLE %s (a int, b int, c int static, d int, PRIMARY KEY (a, b))");
         createIndex("CREATE INDEX b_idx on %s(b)");
@@ -1417,7 +1521,7 @@ public void testAllowFilteringOnPartitionKeyWithIndexForContains() throws Throwa
     }
 
     @Test
-    public void testIndexOnStaticColumnWithPartitionWithoutRows() throws Throwable
+    public void testIndexOnStaticColumnWithPartitionWithoutRows()
     {
         createTable("CREATE TABLE %s (pk int, c int, s int static, v int, PRIMARY KEY(pk, c))");
         createIndex("CREATE INDEX ON %s (s)");
@@ -1443,7 +1547,7 @@ public void testIndexOnStaticColumnWithPartitionWithoutRows() throws Throwable
     }
 
     @Test
-    public void testIndexOnRegularColumnWithPartitionWithoutRows() throws Throwable
+    public void testIndexOnRegularColumnWithPartitionWithoutRows()
     {
         createTable("CREATE TABLE %s (pk int, c int, s int static, v int, PRIMARY KEY(pk, c))");
         createIndex("CREATE INDEX ON %s (v)");
@@ -1461,7 +1565,7 @@ public void testIndexOnRegularColumnWithPartitionWithoutRows() throws Throwable
     }
 
     @Test
-    public void testIndexOnDurationColumn() throws Throwable
+    public void testIndexOnDurationColumn()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, d duration)");
         assertInvalidMessage("Secondary indexes are not supported on duration columns",
@@ -1487,7 +1591,7 @@ public void testIndexOnDurationColumn() throws Throwable
     }
 
     @Test
-    public void testIndexOnFrozenUDT() throws Throwable
+    public void testIndexOnFrozenUDT()
     {
         String type = createType("CREATE TYPE %s (a int)");
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<" + type + ">)");
@@ -1515,7 +1619,7 @@ public void testIndexOnFrozenUDT() throws Throwable
     }
 
     @Test
-    public void testIndexOnFrozenCollectionOfUDT() throws Throwable
+    public void testIndexOnFrozenCollectionOfUDT()
     {
         String type = createType("CREATE TYPE %s (a int)");
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<set<frozen<" + type + ">>>)");
@@ -1530,7 +1634,7 @@ public void testIndexOnFrozenCollectionOfUDT() throws Throwable
 
         execute("INSERT INTO %s (k, v) VALUES (?, ?)", 2, set(udt2));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "v"),
                              "SELECT * FROM %s WHERE v CONTAINS ?", udt1);
 
         assertRows(execute("SELECT * FROM %s WHERE v = ?", set(udt1, udt2)), row(1, set(udt1, udt2)));
@@ -1547,7 +1651,7 @@ public void testIndexOnFrozenCollectionOfUDT() throws Throwable
     }
 
     @Test
-    public void testIndexOnNonFrozenCollectionOfFrozenUDT() throws Throwable
+    public void testIndexOnNonFrozenCollectionOfFrozenUDT()
     {
         String type = createType("CREATE TYPE %s (a int)");
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v set<frozen<" + type + ">>)");
@@ -1580,7 +1684,7 @@ public void testIndexOnNonFrozenCollectionOfFrozenUDT() throws Throwable
     }
 
     @Test
-    public void testIndexOnNonFrozenUDT() throws Throwable
+    public void testIndexOnNonFrozenUDT()
     {
         String type = createType("CREATE TYPE %s (a int)");
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v " + type + ")");
@@ -1633,7 +1737,7 @@ public void testIndexOnPartitionKeyOverridingExpiredRow() throws Throwable
     }
 
     @Test
-    public void testIndexOnPartitionKeyOverridingDeletedRow() throws Throwable
+    public void testIndexOnPartitionKeyOverridingDeletedRow()
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, c int, v int, PRIMARY KEY ((k1, k2), c))");
         createIndex("CREATE INDEX ON %s(k1)");
@@ -1689,7 +1793,7 @@ public void testIndexOnClusteringKeyOverridingExpiredRow() throws Throwable
     }
 
     @Test
-    public void testIndexOnClusteringKeyOverridingDeletedRow() throws Throwable
+    public void testIndexOnClusteringKeyOverridingDeletedRow()
     {
         createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
         createIndex("CREATE INDEX ON %s(ck)");
@@ -1762,7 +1866,7 @@ public void testIndexOnRegularColumnOverridingExpiredRow() throws Throwable
     }
 
     @Test
-    public void testIndexOnRegularColumnOverridingDeletedRow() throws Throwable
+    public void testIndexOnRegularColumnOverridingDeletedRow()
     {
         createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
         createIndex("CREATE INDEX ON %s(v)");
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
index 59f84de5f75f..4ada694cf063 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
@@ -29,6 +29,7 @@
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import com.google.common.collect.ImmutableMap;
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.CQLTester;
@@ -37,17 +38,22 @@
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.db.marshal.DecimalType;
 import org.apache.cassandra.db.marshal.DurationType;
+import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.utils.AbstractTypeGenerators;
 import org.apache.cassandra.utils.AbstractTypeGenerators.TypeSupport;
 import org.quicktheories.core.Gen;
 import org.quicktheories.generators.SourceDSL;
 
+import static java.util.Arrays.asList;
+import static org.apache.cassandra.Util.makeUDT;
 import static org.apache.cassandra.db.SchemaCQLHelper.toCqlType;
 import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport;
 import static org.apache.cassandra.utils.AbstractTypeGenerators.tupleTypeGen;
 import static org.apache.cassandra.utils.FailingConsumer.orFail;
 import static org.apache.cassandra.utils.Generators.filter;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.quicktheories.QuickTheory.qt;
 
 public class TupleTypeTest extends CQLTester
@@ -58,7 +64,7 @@ public void testTuplePutAndGet() throws Throwable
         String[] valueTypes = {"frozen<tuple<int, text, double>>", "tuple<int, text, double>"};
         for (String valueType : valueTypes)
         {
-            createTable("CREATE TABLE %s (k int PRIMARY KEY, t " + valueType + ")");
+            createTable("CREATE TABLE %s (k int PRIMARY KEY, t " + valueType + ')');
 
             execute("INSERT INTO %s (k, t) VALUES (?, ?)", 0, tuple(3, "foo", 3.4));
             execute("INSERT INTO %s (k, t) VALUES (?, ?)", 1, tuple(8, "bar", 0.2));
@@ -139,7 +145,7 @@ public void testTupleFromString() throws Throwable
             row(0, 4, tuple(null, "1"))
         );
 
-        assertInvalidMessage("Invalid tuple literal: too many elements. Type frozen<tuple<int, text>> expects 2 but got 3",
+        assertInvalidMessage("Invalid tuple literal: too many elements. Type tuple<int, text> expects 2 but got 3",
                              "INSERT INTO %s(k, t) VALUES (1,'1:2:3')");
     }
 
@@ -150,7 +156,7 @@ public void testInvalidQueries() throws Throwable
 
         assertInvalidSyntax("INSERT INTO %s (k, t) VALUES (0, ())");
 
-        assertInvalidMessage("Invalid tuple literal for t: too many elements. Type frozen<tuple<int, text, double>> expects 3 but got 4",
+        assertInvalidMessage("Invalid tuple literal for t: too many elements. Type tuple<int, text, double> expects 3 but got 4",
                              "INSERT INTO %s (k, t) VALUES (0, (2, 'foo', 3.1, 'bar'))");
 
         createTable("CREATE TABLE %s (k int PRIMARY KEY, t frozen<tuple<int, tuple<int, text, double>>>)");
@@ -158,7 +164,7 @@ public void testInvalidQueries() throws Throwable
                              "INSERT INTO %s (k, t) VALUES (0, ?)",
                              tuple(1, tuple(1, "1", 1.0, 1)));
 
-        assertInvalidMessage("Invalid tuple literal for t: component 1 is not of type frozen<tuple<int, text, double>>",
+        assertInvalidMessage("Invalid tuple literal for t: component 1 is not of type tuple<int, text, double>",
                              "INSERT INTO %s (k, t) VALUES (0, (1, (1, '1', 1.0, 1)))");
     }
 
@@ -323,6 +329,57 @@ private void tupleCkReadWrite(Order order)
         }));
     }
 
+    /**
+     * This test verifies if the tuple type is properly parsed from CQL. In particular, it checks that when a column is
+     * defined as a tuple, the tuple type is implicitly frozen, which applies also to all the nested tuples and UDTs.
+     * For dropped columns we chech that the top level type (and only it) is not automatically frozen.
+     */
+    @Test
+    public void testCreateTuples()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, t tuple<int, text>) " +
+                    "WITH DROPPED COLUMN RECORD dropped tuple<int, text> USING TIMESTAMP 1680702275400000 ");
+        assertThat(getColumn("t").type).isEqualTo(new TupleType(asList(Int32Type.instance, UTF8Type.instance), false));
+        assertThat(getDroppedColumn("dropped").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, UTF8Type.instance), true));
+
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, t frozen<tuple<int, text>>) " +
+                    "WITH DROPPED COLUMN RECORD dropped frozen<tuple<int, text>> USING TIMESTAMP 1680702275400000");
+        assertThat(getColumn("t").type).isEqualTo(new TupleType(asList(Int32Type.instance, UTF8Type.instance), false));
+        assertThat(getDroppedColumn("dropped").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, UTF8Type.instance), false));
+
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, t tuple<int, tuple<int, text>>) " +
+                    "WITH DROPPED COLUMN RECORD dropped tuple<int, tuple<int, text>> USING TIMESTAMP 1680702275400000");
+        assertThat(getColumn("t").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, new TupleType(asList(Int32Type.instance, UTF8Type.instance), false)), false));
+        assertThat(getDroppedColumn("dropped").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, new TupleType(asList(Int32Type.instance, UTF8Type.instance), false)), true));
+
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, t frozen<tuple<int, frozen<tuple<int, text>>>>) " +
+                    "WITH DROPPED COLUMN RECORD dropped frozen<tuple<int, frozen<tuple<int, text>>>> USING TIMESTAMP 1680702275400000");
+        assertThat(getColumn("t").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, new TupleType(asList(Int32Type.instance, UTF8Type.instance), false)), false));
+        assertThat(getDroppedColumn("dropped").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, new TupleType(asList(Int32Type.instance, UTF8Type.instance), false)), false));
+
+        String udt = createType("CREATE TYPE %s (a int, b text)");
+
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, t tuple<int, frozen<" + udt + ">>) " +
+                    "WITH DROPPED COLUMN RECORD dropped tuple<int, frozen<tuple<int, text>>> USING TIMESTAMP 1680702275400000");
+        assertThat(getColumn("t").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, makeUDT(keyspace(), udt, ImmutableMap.of("a", Int32Type.instance, "b", UTF8Type.instance), false)), false));
+        assertThat(getDroppedColumn("dropped").type).
+                isEqualTo(new TupleType(asList(Int32Type.instance, new TupleType(asList(Int32Type.instance, UTF8Type.instance), false)), true));
+
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, t tuple<int, " + udt + ">) " +
+                    "WITH DROPPED COLUMN RECORD dropped tuple<int, tuple<int, text>> USING TIMESTAMP 1680702275400000");
+        assertThat(getColumn("t").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, makeUDT(keyspace(), udt, ImmutableMap.of("a", Int32Type.instance, "b", UTF8Type.instance), false)), false));
+        assertThat(getDroppedColumn("dropped").type)
+                .isEqualTo(new TupleType(asList(Int32Type.instance, new TupleType(asList(Int32Type.instance, UTF8Type.instance), false)), true));
+    }
+
     private static final class TypeAndRows
     {
         TupleType type;
@@ -391,13 +448,5 @@ <T> Comparator<T> apply(Comparator<T> c)
 
         abstract <T> Comparator<T> apply(Comparator<T> c);
     }
-
-    private static List<Object[]> toObjects(UntypedResultSet results)
-    {
-        List<Object[]> rows = new ArrayList<>(results.size());
-        for (UntypedResultSet.Row row : results)
-            rows.add(results.metadata().stream().map(c -> c.type.compose(row.getBlob(c.name.toString()))).toArray());
-        return rows;
-    }
 }
 
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
index 968182749389..a853ae27c741 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
@@ -18,7 +18,9 @@
 package org.apache.cassandra.cql3.validation.entities;
 
 import java.lang.reflect.Field;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
@@ -26,15 +28,26 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.cassandra.auth.*;
+import org.apache.cassandra.auth.AuthenticatedUser;
+import org.apache.cassandra.auth.DataResource;
+import org.apache.cassandra.auth.FunctionResource;
+import org.apache.cassandra.auth.IAuthorizer;
+import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.auth.RoleResource;
+import org.apache.cassandra.auth.StubAuthorizer;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.functions.UserFunction;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.VariableSpecifications;
 import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UserFunction;
 import org.apache.cassandra.cql3.statements.BatchStatement;
 import org.apache.cassandra.cql3.statements.ModificationStatement;
-import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.UnauthorizedException;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.ClientState;
 
 import static org.junit.Assert.assertEquals;
@@ -226,7 +239,7 @@ public void testBatchStatement() throws Throwable
             functions.add(functionName);
             statements.add(stmt);
         }
-        BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
+        BatchStatement batch = new BatchStatement(null, BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
         assertUnauthorized(batch, functions);
 
         grantExecuteOnFunction(functions.get(0));
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java
index bba5c92b3b93..fbd05db99634 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java
@@ -17,7 +17,10 @@
  */
 package org.apache.cassandra.cql3.validation.entities;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.Iterables;
@@ -27,12 +30,12 @@
 
 import org.apache.cassandra.cql3.Attributes;
 import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.VariableSpecifications;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.statements.BatchStatement;
 import org.apache.cassandra.cql3.statements.ModificationStatement;
-import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.service.ClientState;
 
 import static org.junit.Assert.assertTrue;
@@ -308,7 +311,7 @@ public void testBatchStatement() throws Throwable
         statements.add(modificationStatement(cql("INSERT INTO %s (key, i_cc, t_cc) VALUES (2, 2, %s)",
                                                  functionCall(tFunc, "'foo'"))));
 
-        BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
+        BatchStatement batch = new BatchStatement(null, BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
         assertFunctions(batch, iFunc, iFunc2, tFunc);
     }
 
@@ -321,7 +324,7 @@ public void testBatchStatementWithConditions() throws Throwable
         statements.add(modificationStatement(cql("UPDATE %s SET i_val = %s WHERE key=0 AND i_cc=1 and t_cc='foo' IF s_val = %s",
                                                  functionCall(iFunc, "0"), functionCall(sFunc, "{1}"))));
 
-        BatchStatement batch = new BatchStatement(BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
+        BatchStatement batch = new BatchStatement(null, BatchStatement.Type.LOGGED, VariableSpecifications.empty(), statements, Attributes.none());
         assertFunctions(batch, iFunc, lFunc, sFunc);
     }
 
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
index ccb1513f19fd..ddb4ce82a7c7 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
@@ -41,12 +41,14 @@
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.cql3.functions.UDAggregate;
 import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.FunctionExecutionException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.transport.ProtocolVersion;
+import org.assertj.core.api.Assertions;
 
 public class UFJavaTest extends CQLTester
 {
@@ -806,19 +808,104 @@ public void testAllNativeTypes() throws Throwable
     @Test
     public void testUDFToCqlString()
     {
-        UDFunction function = UDFunction.create(new FunctionName("my_ks", "my_function"),
-                                                Arrays.asList(ColumnIdentifier.getInterned("column", false)),
-                                                Arrays.asList(UTF8Type.instance),
+        FunctionName functionName = new FunctionName("my_ks", "my_function");
+        ColumnIdentifier column = ColumnIdentifier.getInterned("column", false);
+        List<ColumnIdentifier> argNames = Collections.singletonList(column);
+        List<AbstractType<?>> argTypes = Collections.singletonList(UTF8Type.instance);
+
+        UDFunction function = UDFunction.create(functionName,
+                                                argNames,
+                                                argTypes,
                                                 Int32Type.instance,
                                                 false,
                                                 "java",
-                                                "return 0;");
+                                                "return 0;",
+                                                false,
+                                                false,
+                                                Collections.emptyList());
 
         Assert.assertTrue(function.toCqlString(true, true).contains("CREATE FUNCTION IF NOT EXISTS"));
         Assert.assertFalse(function.toCqlString(true, false).contains("CREATE FUNCTION IF NOT EXISTS"));
 
         Assert.assertEquals(function.toCqlString(true, true), function.toCqlString(false, true));
         Assert.assertEquals(function.toCqlString(true, false), function.toCqlString(false, false));
+
+        // test DETERMINISTIC
+        Assertions.assertThat(UDFunction.create(functionName,
+                                                argNames,
+                                                argTypes,
+                                                Int32Type.instance,
+                                                false,
+                                                "java",
+                                                "return 0;",
+                                                true,
+                                                false,
+                                                Collections.emptyList())
+                                        .toCqlString(true, false))
+                  .isEqualTo("CREATE FUNCTION my_ks.my_function(column text)\n" +
+                             "    RETURNS NULL ON NULL INPUT\n" +
+                             "    RETURNS int\n" +
+                             "    DETERMINISTIC\n" +
+                             "    LANGUAGE java\n" +
+                             "    AS $$return 0;$$;");
+
+        // test MONOTONIC
+        Assertions.assertThat(UDFunction.create(functionName,
+                                                argNames,
+                                                argTypes,
+                                                Int32Type.instance,
+                                                false,
+                                                "java",
+                                                "return 0;",
+                                                false,
+                                                true,
+                                                Collections.emptyList())
+                                        .toCqlString(true, false))
+                  .isEqualTo("CREATE FUNCTION my_ks.my_function(column text)\n" +
+                             "    RETURNS NULL ON NULL INPUT\n" +
+                             "    RETURNS int\n" +
+                             "    MONOTONIC\n" +
+                             "    LANGUAGE java\n" +
+                             "    AS $$return 0;$$;");
+
+        // test MONOTONIC ON
+        Assertions.assertThat(UDFunction.create(functionName,
+                                                argNames,
+                                                argTypes,
+                                                Int32Type.instance,
+                                                false,
+                                                "java",
+                                                "return 0;",
+                                                false,
+                                                false,
+                                                argNames)
+                                        .toCqlString(true, false))
+                  .isEqualTo("CREATE FUNCTION my_ks.my_function(column text)\n" +
+                             "    RETURNS NULL ON NULL INPUT\n" +
+                             "    RETURNS int\n" +
+                             "    MONOTONIC ON column\n" +
+                             "    LANGUAGE java\n" +
+                             "    AS $$return 0;$$;");
+
+        // test DETERMINISTIC and MONOTONIC ON
+        Assertions.assertThat(UDFunction.create(functionName,
+                                                argNames,
+                                                argTypes,
+                                                Int32Type.instance,
+                                                false,
+                                                "java",
+                                                "return 0;",
+                                                true,
+                                                false,
+                                                argNames)
+                                        .toCqlString(true, false))
+                  .isEqualTo("CREATE FUNCTION my_ks.my_function(column text)\n" +
+                             "    RETURNS NULL ON NULL INPUT\n" +
+                             "    RETURNS int\n" +
+                             "    DETERMINISTIC\n" +
+                             "    MONOTONIC ON column\n" +
+                             "    LANGUAGE java\n" +
+                             "    AS $$return 0;$$;");
     }
 
     @Test
@@ -834,29 +921,51 @@ public void testUDAToCqlString() throws Throwable
                                                   "        return state + val;\n" +
                                                   "    $$;");
 
-        // Java representation of state function so we can construct aggregate programmatically
-        UDFunction stateFunction = UDFunction.create(new FunctionName(KEYSPACE, stateFunctionName.split("\\.")[1]),
+        stateFunctionName = stateFunctionName.split("\\.")[1];
+
+        // Java representation of state function, so we can construct aggregate programmatically
+        UDFunction stateFunction = UDFunction.create(new FunctionName(KEYSPACE, stateFunctionName),
                                                      Arrays.asList(ColumnIdentifier.getInterned("state", false),
                                                                    ColumnIdentifier.getInterned("val", false)),
                                                      Arrays.asList(Int32Type.instance, Int32Type.instance),
                                                      Int32Type.instance,
                                                      true,
                                                      "java",
-                                                     "return state + val;");
+                                                     "return state + val;",
+                                                     false,
+                                                     false,
+                                                     Collections.emptyList());
 
         UDAggregate aggregate = UDAggregate.create(Collections.singleton(stateFunction),
                                                    new FunctionName(KEYSPACE, "my_aggregate"),
                                                    Collections.singletonList(Int32Type.instance),
                                                    Int32Type.instance,
-                                                   new FunctionName(KEYSPACE, stateFunctionName.split("\\.")[1]),
+                                                   new FunctionName(KEYSPACE, stateFunctionName),
                                                    null,
                                                    Int32Type.instance,
-                                                   null);
+                                                   null,
+                                                   false);
 
         Assert.assertTrue(aggregate.toCqlString(true, true).contains("CREATE AGGREGATE IF NOT EXISTS"));
         Assert.assertFalse(aggregate.toCqlString(true, false).contains("CREATE AGGREGATE IF NOT EXISTS"));
 
         Assert.assertEquals(aggregate.toCqlString(true, true), aggregate.toCqlString(false, true));
         Assert.assertEquals(aggregate.toCqlString(true, false), aggregate.toCqlString(false, false));
+
+        // test DETERMINISTIC
+        Assertions.assertThat(UDAggregate.create(Collections.singleton(stateFunction),
+                                                 new FunctionName(KEYSPACE, "my_aggregate"),
+                                                 Collections.singletonList(Int32Type.instance),
+                                                 Int32Type.instance,
+                                                 new FunctionName(KEYSPACE, stateFunctionName),
+                                                 null,
+                                                 Int32Type.instance,
+                                                 null,
+                                                 true)
+                                         .toCqlString(true, false))
+                  .isEqualTo("CREATE AGGREGATE cql_test_keyspace.my_aggregate(int)\n" +
+                             "    SFUNC " + stateFunctionName + '\n' +
+                             "    STYPE int\n" +
+                             "    DETERMINISTIC;");
     }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
index 56795c2dfc70..0694a185803e 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
@@ -20,6 +20,7 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 
@@ -27,12 +28,14 @@
 import org.junit.Test;
 
 import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.cql3.functions.UDFunction;
 import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
@@ -850,6 +853,9 @@ public void testBrokenFunction() throws Throwable
                                                             true,
                                                             "java",
                                                             f.body(),
+                                                            false,
+                                                            false,
+                                                            Collections.emptyList(),
                                                             new InvalidRequestException("foo bar is broken"));
         SchemaTestUtil.addOrUpdateKeyspace(ksm.withSwapped(ksm.userFunctions.without(f.name(), f.argTypes()).with(broken)), false);
 
@@ -1010,8 +1016,219 @@ public void testRejectInvalidFunctionNamesOnCreation()
                                        "RETURNS int " +
                                        "LANGUAGE JAVA\n" +
                                        "AS 'return val;'");
-            }).hasRootCauseInstanceOf(InvalidRequestException.class)
-              .hasRootCauseMessage("Function name '%s' is invalid", funcName);
+            }).isInstanceOf(InvalidRequestException.class)
+              .hasMessageContaining("Function name '%s' is invalid", funcName);
         }
     }
+
+    @Test
+    public void testRejectInvalidLanguageOnCreation()
+    {
+        for (String funcName : Arrays.asList("my/fancy/func", "my_other[fancy]func"))
+        {
+            assertThatThrownBy(() -> createFunctionOverload(String.format("%s.\"%s\"", KEYSPACE_PER_TEST, funcName),
+                                                            "int",
+                                                            "CREATE OR REPLACE FUNCTION %s(val int) " +
+                                                            "RETURNS NULL ON NULL INPUT " +
+                                                            "RETURNS int " +
+                                                            "LANGUAGE javascript\n" +
+                                                            "AS 'return val;'"))
+            .isInstanceOf(InvalidRequestException.class)
+            .hasMessageContaining("Currently only Java UDFs are available in Cassandra. " +
+                                  "For more information - CASSANDRA-18252");
+
+            assertThatThrownBy(() -> createFunctionOverload(String.format("%s.\"%s\"", KEYSPACE_PER_TEST, funcName),
+                                                            "int",
+                                                            "CREATE OR REPLACE FUNCTION %s(val int) " +
+                                                            "RETURNS NULL ON NULL INPUT " +
+                                                            "RETURNS int " +
+                                                            "LANGUAGE JAVASCRIPT\n" +
+                                                            "AS 'return val;'"))
+            .isInstanceOf(InvalidRequestException.class)
+            .hasMessageContaining("Currently only Java UDFs are available in Cassandra. " +
+                                  "For more information - CASSANDRA-18252");
+
+            // test with weird made up word for language
+            assertThatThrownBy(() -> createFunctionOverload(String.format("%s.\"%s\"", KEYSPACE_PER_TEST, funcName),
+                                                            "int",
+                                                            "CREATE OR REPLACE FUNCTION %s(val int) " +
+                                                            "RETURNS NULL ON NULL INPUT " +
+                                                            "RETURNS int " +
+                                                            "LANGUAGE JAVASCRI\n" +
+                                                            "AS 'return val;'"))
+            .isInstanceOf(InvalidRequestException.class)
+            .hasMessageContaining("Currently only Java UDFs are available in Cassandra. " +
+                                  "For more information - CASSANDRA-18252");
+        }
+    }
+
+    @Test
+    public void testParseDeterministic() throws Throwable
+    {
+        testParseDeterministic("", false);
+        testParseDeterministic("DETERMINISTIC", true);
+    }
+
+    private void testParseMonotonic(String arguments,
+                                    String monotonicModifier,
+                                    boolean expectedMonotonic,
+                                    String... expectedMonotonicOn) throws Throwable
+    {
+        String fName = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (" +
+                                                       arguments +
+                                                       ") CALLED ON NULL INPUT " +
+                                                       "RETURNS int " +
+                                                       monotonicModifier +
+                                                       " LANGUAGE java " +
+                                                       "AS 'return 1;'");
+
+        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+        Assert.assertNotNull(ksm);
+        UDFunction f = (UDFunction) ksm.userFunctions.get(parseFunctionName(fName)).iterator().next();
+        Assert.assertEquals(expectedMonotonic, f.isMonotonic());
+        Assert.assertArrayEquals(expectedMonotonicOn, f.monotonicOn().stream().map(Object::toString).toArray());
+    }
+
+    private void testParseDeterministic(String deterministicModifier, boolean expectedDeterministic) throws Throwable
+    {
+        String fName = createFunction(KEYSPACE, "double", "CREATE FUNCTION %s (input double) " +
+                                                          "CALLED ON NULL INPUT " +
+                                                          "RETURNS double " +
+                                                          deterministicModifier +
+                                                          " LANGUAGE java " +
+                                                          "AS 'return Math.sin(input);'");
+
+        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+        Assert.assertNotNull(ksm);
+        UDFunction f = (UDFunction) ksm.userFunctions.get(parseFunctionName(fName)).iterator().next();
+        Assert.assertEquals(expectedDeterministic, f.isDeterministic());
+    }
+
+    @Test
+    public void testParseMonotonic() throws Throwable
+    {
+        testParseMonotonic("a int", "", false);
+        testParseMonotonic("a int", "MONOTONIC", true, "a");
+        testParseMonotonic("a int", "MONOTONIC ON a", true, "a");
+        testParseMonotonic("a int",
+                           "MONOTONIC ON b",
+                           "Monotony should be declared on one of the arguments, 'b' is not an argument");
+
+        testParseMonotonic("a int, b int", "", false);
+        testParseMonotonic("a int, b int", "MONOTONIC", true, "a", "b");
+        testParseMonotonic("a int, b int", "MONOTONIC ON a", false, "a");
+        testParseMonotonic("a int, b int", "MONOTONIC ON b", false, "b");
+        testParseMonotonic("a int, b int",
+                           "MONOTONIC ON c",
+                           "Monotony should be declared on one of the arguments, 'c' is not an argument");
+    }
+
+    private void testParseMonotonic(String parameters, String monotonicModifier, String expectedMessage)
+    {
+        assertThatThrownBy(() -> createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (" +
+                                                                 parameters +
+                                                                 ") CALLED ON NULL INPUT " +
+                                                                 "RETURNS int " +
+                                                                 monotonicModifier +
+                                                                 " LANGUAGE java " +
+                                                                 "AS 'return 1;'"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(expectedMessage);
+    }
+
+    @Test
+    public void testUDFOnGroupByClause() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))");
+        execute("INSERT into %s (pk, ck) values (0, 0)");
+        execute("INSERT into %s (pk, ck) values (0, 1)");
+        execute("INSERT into %s (pk, ck) values (0, 2)");
+        execute("INSERT into %s (pk, ck) values (0, 3)");
+        execute("INSERT into %s (pk, ck) values (0, 4)");
+        execute("INSERT into %s (pk, ck) values (0, 5)");
+
+        // Verify that asynchronous UDF execution is enabled by default
+        Assert.assertTrue(DatabaseDescriptor.enableUserDefinedFunctionsThreads());
+
+        // Test that UDFs in GROUP BY clause are rejected due to the asynchronous UDF execution
+        String f = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (x int) " +
+                                                   "CALLED ON NULL INPUT " +
+                                                   "RETURNS int " +
+                                                   "MONOTONIC " +
+                                                   "LANGUAGE java " +
+                                                   "AS 'return x / 2;'");
+        assertInvalidMessage("User defined functions are not supported in the GROUP BY clause when asynchronous",
+                             String.format("SELECT pk, ck, %s(ck) FROM %%s GROUP BY pk, %<s(ck)", f));
+
+        // Test that native functions in GROUP BY clause are not rejected
+        assertRows(execute(String.format("SELECT pk, ck FROM %%s GROUP BY pk, %s(ck)", "tounixtimestamp")),
+                   row(0, 0), row(0, 1), row(0, 2), row(0, 3), row(0, 4), row(0, 5));
+
+        // Test with asynchronous UDF execution disabled
+        try
+        {
+            DatabaseDescriptor.enableUserDefinedFunctionsThreads(false);
+
+            // Test non-monotonic function
+            String f1 = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (x int) " +
+                                                        "CALLED ON NULL INPUT " +
+                                                        "RETURNS int " +
+                                                        "LANGUAGE java " +
+                                                        "AS 'return -1 * x;'");
+            assertInvalidMessage("Only monotonic functions are supported in the GROUP BY clause.",
+                                 String.format("SELECT pk, ck, %s(ck) FROM %%s GROUP BY pk, %<s(ck)", f1));
+
+            // Test fully monotonic function
+            String f2 = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (x int) " +
+                                                        "CALLED ON NULL INPUT " +
+                                                        "RETURNS int " +
+                                                        "MONOTONIC " +
+                                                        "LANGUAGE java " +
+                                                        "AS 'return x / 2;'");
+            assertRows(execute(String.format("SELECT pk, ck, %s(ck) FROM %%s GROUP BY pk, %<s(ck)", f2)),
+                       row(0, 0, 0),
+                       row(0, 2, 1),
+                       row(0, 4, 2));
+
+            // Test partially monotonic function
+            String f3 = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (dividend int, divisor int) " +
+                                                        "CALLED ON NULL INPUT " +
+                                                        "RETURNS int " +
+                                                        "MONOTONIC ON dividend " +
+                                                        "LANGUAGE java " +
+                                                        "AS 'return dividend / divisor;'");
+            assertRows(execute(String.format("SELECT pk, ck, %s(ck, 2) FROM %%s GROUP BY pk, %<s(ck, 2)", f3)),
+                       row(0, 0, 0),
+                       row(0, 2, 1),
+                       row(0, 4, 2));
+            assertRows(execute(String.format("SELECT pk, ck, %s(ck, 3) FROM %%s GROUP BY pk, %<s(ck, 3)", f3)),
+                       row(0, 0, 0),
+                       row(0, 3, 1));
+            assertInvalidMessage("Only monotonic functions are supported in the GROUP BY clause.",
+                                 String.format("SELECT pk, ck, %s(2, ck) FROM %%s GROUP BY pk, %<s(2, ck)", f3));
+        }
+        finally
+        {
+            DatabaseDescriptor.enableUserDefinedFunctionsThreads(true);
+        }
+    }
+
+    @Test
+    public void testFunctionsAreNonMonotonicAndNonDeterministicByDefault() throws Throwable
+    {
+        String name = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (x int) " +
+                                                      "CALLED ON NULL INPUT " +
+                                                      "RETURNS int " +
+                                                      "LANGUAGE java " +
+                                                      "AS 'return x / 2;'");
+
+        UntypedResultSet functions = execute("SELECT * FROM system_schema.functions " +
+                                             "WHERE keyspace_name=? AND function_name=?",
+                                             KEYSPACE, shortFunctionName(name));
+
+        Assert.assertEquals(1, functions.size());
+        Assert.assertFalse(functions.one().getBoolean("monotonic"));
+        Assert.assertFalse(functions.one().getBoolean("deterministic"));
+        Assert.assertEquals(Collections.emptyList(), functions.one().getList("monotonic_on", UTF8Type.instance));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
index 7a1d73ea3735..3cbaa6db5aae 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
@@ -39,6 +39,7 @@ public static void setUpClass()     // overrides CQLTester.setUpClass()
         StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
 
         prepareServer();
+        StorageService.instance.setUpDistributedSystemKeyspaces();
     }
 
     @Test
@@ -67,7 +68,7 @@ public void testInvalidInputForUserType() throws Throwable
                              "INSERT INTO %s (k, t) VALUES (0, ?)",
                              userType("a", 1, "b", tuple(1, "1", 1.0, 1)));
 
-        assertInvalidMessage("Invalid user type literal for t: field b is not of type frozen<tuple<int, text, double>>",
+        assertInvalidMessage("Invalid user type literal for t: field b is not of type tuple<int, text, double>",
                              "INSERT INTO %s (k, t) VALUES (0, {a: 1, b: (1, '1', 1.0, 1)})");
     }
 
@@ -113,21 +114,24 @@ public void testInvalidUDTStatements() throws Throwable
         String myType = KEYSPACE + '.' + typename;
 
         // non-frozen UDTs in a table PK
-        assertInvalidMessage("Invalid non-frozen user-defined type \"" + myType + "\" for PRIMARY KEY column 'k'",
+        assertInvalidMessage("for column k: non-frozen user types are not supported for PRIMARY KEY columns",
                 "CREATE TABLE " + KEYSPACE + ".wrong (k " + myType + " PRIMARY KEY , v int)");
-        assertInvalidMessage("Invalid non-frozen user-defined type \"" + myType + "\" for PRIMARY KEY column 'k2'",
+        assertInvalidMessage("for column k2: non-frozen user types are not supported for PRIMARY KEY columns",
                 "CREATE TABLE " + KEYSPACE + ".wrong (k1 int, k2 " + myType + ", v int, PRIMARY KEY (k1, k2))");
 
         // non-frozen UDTs in a collection
-        assertInvalidMessage("Non-frozen UDTs are not allowed inside collections: list<" + myType + ">",
+        assertInvalidMessage("for column v: non-frozen user types are only supported at top-level",
                 "CREATE TABLE " + KEYSPACE + ".wrong (k int PRIMARY KEY, v list<" + myType + ">)");
-        assertInvalidMessage("Non-frozen UDTs are not allowed inside collections: set<" + myType + ">",
+        assertInvalidMessage("for column v: non-frozen user types are only supported at top-level",
                 "CREATE TABLE " + KEYSPACE + ".wrong (k int PRIMARY KEY, v set<" + myType + ">)");
-        assertInvalidMessage("Non-frozen UDTs are not allowed inside collections: map<" + myType + ", int>",
+        assertInvalidMessage("for column v: non-frozen user types are only supported at top-level",
                 "CREATE TABLE " + KEYSPACE + ".wrong (k int PRIMARY KEY, v map<" + myType + ", int>)");
-        assertInvalidMessage("Non-frozen UDTs are not allowed inside collections: map<int, " + myType + ">",
+        assertInvalidMessage("for column v: non-frozen user types are only supported at top-level",
                 "CREATE TABLE " + KEYSPACE + ".wrong (k int PRIMARY KEY, v map<int, " + myType + ">)");
 
+        // the tests below are commented out - they are valid on OSS but CC automatically freezes the inner types
+        // so no exception is expected
+        /*
         // non-frozen UDT in a collection (as part of a UDT definition)
         assertInvalidMessage("Non-frozen UDTs are not allowed inside collections: list<" + myType + ">",
                 "CREATE TYPE " + KEYSPACE + ".wrong (a int, b list<" + myType + ">)");
@@ -139,6 +143,7 @@ public void testInvalidUDTStatements() throws Throwable
         String ut1 = createType(KEYSPACE, "CREATE TYPE %s (a int)");
         assertInvalidMessage("A user type cannot contain non-frozen UDTs",
                 "ALTER TYPE " + KEYSPACE + "." + ut1 + " ADD b " + myType);
+        */
 
         // referencing a UDT in another keyspace
         assertInvalidMessage("Statement on keyspace " + KEYSPACE + " cannot refer to a user type in keyspace otherkeyspace;" +
@@ -181,7 +186,7 @@ public void testInvalidUDTStatements() throws Throwable
         // non-frozen UDT with non-frozen nested collection
         String typename2 = createType("CREATE TYPE %s (bar int, foo list<int>)");
         String myType2 = KEYSPACE + '.' + typename2;
-        assertInvalidMessage("Non-frozen UDTs with nested non-frozen collections are not supported",
+        assertInvalidMessage("for column v: non-frozen collections are only supported at top-level",
                 "CREATE TABLE " + KEYSPACE + ".wrong (k int PRIMARY KEY, v " + myType2 + ")");
 
         String userType = createType("CREATE TYPE %s (userids SET<UUID>)");
@@ -515,6 +520,39 @@ public void testAlteringUserTypeNestedWithinUserType() throws Throwable
         );
     }
 
+    /**
+     * This is a test for CNDB-7789 that makes sure we reject anything non-frozen nested inside a user type, even if
+     * it's done by an ALTER TYPE.
+     */
+    @Test
+    public void testCreateInvalidNonFrozenNestedUserType() throws Throwable
+    {
+        String inner = createType("CREATE TYPE %s (a int)");
+        String outer1 = createType("CREATE TYPE %s (b int, c " + typeWithKs(inner) + ')');
+
+        // While outer1 can be created, it cannot be used as a column type.
+        assertInvalidMessage("non-frozen user types are only supported at top-level",
+                             "CREATE TABLE " + keyspace() + ".failed (k int PRIMARY KEY, u " + typeWithKs(outer1) + ')');
+
+        // Of course, it's allowed if frozen
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, u frozen<" + typeWithKs(outer1) + ">)");
+
+        // It's also allowed if we created the outer type with the inner one frozen
+        String outer2 = createType("CREATE TYPE %s (b int, c frozen<" + typeWithKs(inner) + ">)");
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, u frozen<" + typeWithKs(outer2) + ">)");
+
+        // Try adding a non-frozen inner UDT with ALTER TYPE after it is used by a table
+        String outer3 = createType("CREATE TYPE %s (b int)");
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, u " + typeWithKs(outer3) + ')');
+
+        assertInvalidMessage("A user type cannot contain non-frozen UDTs",
+                             "ALTER TYPE " + typeWithKs(outer3) + " ADD c " + typeWithKs(inner));
+
+        // Also try with a collection
+        assertInvalidMessage("non-frozen collections are only supported at top-level",
+                             "ALTER TYPE " + typeWithKs(outer3) + " ADD c list<text>");
+    }
+
     /**
      * Migrated from cql_tests.py:TestCQL.user_types_test()
      */
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/WritetimeOrTTLTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/WritetimeOrTTLTest.java
index 571196d444a5..b0310d645196 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/WritetimeOrTTLTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/WritetimeOrTTLTest.java
@@ -164,7 +164,7 @@ public void testFrozenList() throws Throwable
     }
 
     @Test
-    public void testSet() throws Throwable
+    public void testSet()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, s set<int>)");
 
@@ -257,7 +257,7 @@ public void testSet() throws Throwable
     }
 
     @Test
-    public void testFrozenSet() throws Throwable
+    public void testFrozenSet()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, s frozen<set<int>>)");
 
@@ -349,7 +349,7 @@ public void testFrozenSet() throws Throwable
     }
 
     @Test
-    public void testMap() throws Throwable
+    public void testMap()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<int, int>)");
 
@@ -443,7 +443,7 @@ public void testMap() throws Throwable
     }
 
     @Test
-    public void testFrozenMap() throws Throwable
+    public void testFrozenMap()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, m frozen<map<int,int>>)");
 
@@ -691,7 +691,7 @@ public void testFrozenNestedCollections() throws Throwable
     }
 
     @Test
-    public void testUDT() throws Throwable
+    public void testUDT()
     {
         String type = createType("CREATE TYPE %s (f1 int, f2 int)");
         createTable("CREATE TABLE %s (k int PRIMARY KEY, t " + type + ')');
@@ -750,7 +750,7 @@ public void testUDT() throws Throwable
     }
 
     @Test
-    public void testFrozenUDT() throws Throwable
+    public void testFrozenUDT()
     {
         String type = createType("CREATE TYPE %s (f1 int, f2 int)");
         createTable("CREATE TABLE %s (k int PRIMARY KEY, t frozen<" + type + ">)");
@@ -924,7 +924,7 @@ public void testNestedUDTs() throws Throwable
     }
 
     @Test
-    public void testFrozenNestedUDTs() throws Throwable
+    public void testFrozenNestedUDTs()
     {
         String nestedType = createType("CREATE TYPE %s (f1 int, f2 int)");
         String type = createType(format("CREATE TYPE %%s (f1 frozen<%s>, f2 frozen<%<s>)", nestedType));
@@ -1051,7 +1051,7 @@ public void testFrozenNestedUDTs() throws Throwable
     }
 
     @Test
-    public void testFunctions() throws Throwable
+    public void testFunctions()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v int, s set<int>, fs frozen<set<int>>)");
         execute("INSERT INTO %s (k, v, s, fs) VALUES (0, 0, {1, 2, 3}, {1, 2, 3}) USING TIMESTAMP 1 AND TTL 1000");
@@ -1097,23 +1097,22 @@ private static List<Long> timestamps(Long... a)
         return Arrays.asList(a);
     }
 
-    private void assertRows(String query, Object[]... rows) throws Throwable
+    private void assertRows(String query, Object[]... rows)
     {
         assertRows(execute(query), rows);
     }
 
-    private void assertWritetimeAndTTL(String column, Long timestamp, Integer ttl) throws Throwable
+    private void assertWritetimeAndTTL(String column, Long timestamp, Integer ttl)
     {
         assertWritetimeAndTTL(column, null, timestamp, ttl);
     }
 
-    private void assertWritetimeAndTTL(String column, List<Long> timestamps, List<Integer> ttls) throws Throwable
+    private void assertWritetimeAndTTL(String column, List<Long> timestamps, List<Integer> ttls)
     {
         assertWritetimeAndTTL(column, null, timestamps, ttls);
     }
 
     private void assertWritetimeAndTTL(String column, String where, Long timestamp, Integer ttl)
-    throws Throwable
     {
         where = where == null ? "" : " WHERE " + where;
 
@@ -1143,7 +1142,6 @@ private void assertWritetimeAndTTL(String column, String where, Long timestamp,
     }
 
     private void assertWritetimeAndTTL(String column, String where, List<Long> timestamps, List<Integer> ttls)
-    throws Throwable
     {
         where = where == null ? "" : " WHERE " + where;
 
@@ -1199,7 +1197,7 @@ private void assertTTL(Integer expected, Integer actual)
         }
     }
 
-    private void assertInvalidPrimaryKeySelection(String column) throws Throwable
+    private void assertInvalidPrimaryKeySelection(String column)
     {
         assertInvalidThrowMessage("Cannot use selection function writetime on PRIMARY KEY part " + column,
                                   InvalidRequestException.class,
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java
index 74a0a4b17f03..f40051c57bb6 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java
@@ -52,6 +52,8 @@ public void testChangingCrcCheckChance()
 
         ColumnFamilyStore cfs = Keyspace.open(CQLTester.KEYSPACE).getColumnFamilyStore(currentTable());
         ColumnFamilyStore indexCfs = Iterables.getFirst(cfs.indexManager.getAllIndexColumnFamilyStores(), null);
+        cfs.disableAutoCompaction();
+        indexCfs.disableAutoCompaction();
         Util.flush(cfs);
 
         Assert.assertEquals(0.99, cfs.getCrcCheckChance(), 0.0);
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java
index a1223c17c2b1..a18d745c1b3c 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java
@@ -43,11 +43,11 @@ private void executeAndCheck(String query, int numSSTables, Object[]... rows) th
     {
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore(KEYSPACE_PER_TEST);
 
-        ((ClearableHistogram) cfs.metric.sstablesPerReadHistogram.cf).clear(); // resets counts
+        ((ClearableHistogram) cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram()).clear(); // resets counts
 
         assertRows(execute(query), rows);
 
-        long numSSTablesIterated = cfs.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(); // max sstables read
+        long numSSTablesIterated = cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getSnapshot().getMax(); // max sstables read
         assertEquals(String.format("Expected %d sstables iterated but got %d instead, with %d live sstables",
                                    numSSTables, numSSTablesIterated, cfs.getLiveSSTables().size()),
                      numSSTables,
@@ -59,11 +59,11 @@ private void executeAndCheckRangeQuery(String query, int numSSTables, Object[]..
         logger.info("Executing query: {} with parameters: {}", query, Arrays.toString(rows));
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore(KEYSPACE_PER_TEST);
 
-        ((ClearableHistogram) cfs.metric.sstablesPerRangeReadHistogram.cf).clear(); // resets counts
+        ((ClearableHistogram) cfs.metric.sstablesPerRangeReadHistogram.tableOrKeyspaceHistogram()).clear(); // resets counts
 
         assertRows(execute(query), rows);
 
-        long numSSTablesIterated = cfs.metric.sstablesPerRangeReadHistogram.cf.getSnapshot().getMax(); // max sstables read
+        long numSSTablesIterated = cfs.metric.sstablesPerRangeReadHistogram.tableOrKeyspaceHistogram().getSnapshot().getMax(); // max sstables read
         assertEquals(String.format("Expected %d sstables iterated but got %d instead, with %d live sstables",
                                    numSSTables, numSSTablesIterated, cfs.getLiveSSTables().size()),
                      numSSTables,
@@ -71,7 +71,7 @@ private void executeAndCheckRangeQuery(String query, int numSSTables, Object[]..
     }
 
     @Override
-    protected String createTable(String query)
+    public String createTable(String query)
     {
         String ret = super.createTable(KEYSPACE_PER_TEST, query);
         disableCompaction(KEYSPACE_PER_TEST);
@@ -79,7 +79,7 @@ protected String createTable(String query)
     }
 
     @Override
-    protected UntypedResultSet execute(String query, Object... values)
+    public UntypedResultSet execute(String query, Object... values)
     {
         return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
     }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
index dc7cccfcb4af..716f51c38571 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
@@ -40,25 +40,22 @@
  */
 public class TombstonesTest extends CQLTester
 {
-    static final int ORIGINAL_FAILURE_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
+    static final int ORIGINAL_FAILURE_THRESHOLD = DatabaseDescriptor.getGuardrailsConfig().getTombstoneFailThreshold();
     static final int FAILURE_THRESHOLD = 100;
 
-    static final int ORIGINAL_WARN_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
     static final int WARN_THRESHOLD = 50;
 
     @BeforeClass
     public static void setUp() throws Throwable
     {
         DatabaseDescriptor.daemonInitialization();
-        DatabaseDescriptor.setTombstoneFailureThreshold(FAILURE_THRESHOLD);
-        DatabaseDescriptor.setTombstoneWarnThreshold(WARN_THRESHOLD);
+        DatabaseDescriptor.getGuardrailsConfig().setTombstonesThreshold(WARN_THRESHOLD, FAILURE_THRESHOLD);
     }
 
     @AfterClass
     public static void tearDown()
     {
-        DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_FAILURE_THRESHOLD);
-        DatabaseDescriptor.setTombstoneWarnThreshold(ORIGINAL_WARN_THRESHOLD);
+        DatabaseDescriptor.getGuardrailsConfig().setTombstonesThreshold(ORIGINAL_FAILURE_THRESHOLD, ORIGINAL_FAILURE_THRESHOLD);
     }
 
     @Test
@@ -72,13 +69,13 @@ public void testBelowThresholdSelect() throws Throwable
 
         // insert exactly the amount of tombstones that shouldn't trigger an exception
         for (int i = 0; i < FAILURE_THRESHOLD; i++)
-            execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+            execute("DELETE FROM %s WHERE a = 'key' and b = '" + i + "'");
 
         try
         {
             execute("SELECT * FROM %s WHERE a = 'key';");
             assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
-            assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
+            assertEquals(oldWarnings + 1, cfs.metric.tombstoneWarnings.getCount());
         }
         catch (Throwable e)
         {
@@ -96,7 +93,7 @@ public void testBeyondThresholdSelect() throws Throwable
 
         // insert exactly the amount of tombstones that *SHOULD* trigger an exception
         for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
-            execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+            execute("DELETE FROM %s WHERE a = 'key' and b = '" + i + "'");
 
         try
         {
@@ -218,7 +215,7 @@ public void testBeyondWarnThresholdSelect() throws Throwable
 
         // insert the number of tombstones that *SHOULD* trigger an Warning
         for (int i = 0; i < WARN_THRESHOLD + 1; i++)
-            execute("INSERT INTO %s (a, b, c ) VALUES ('key', 'cc" + i + "',  null);");
+            execute("DELETE FROM %s WHERE a = 'key' and b = '" + i + "'");
         try
         {
             execute("SELECT * FROM %s WHERE a = 'key';");
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
index 8202e8031a02..bae47ba651a0 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
@@ -28,20 +28,22 @@
 import java.util.Date;
 import java.util.Locale;
 import java.util.TimeZone;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang3.time.DateUtils;
-
+import org.junit.Assert;
 import org.junit.Test;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.classic.joran.ReconfigureOnChangeTask;
-import ch.qos.logback.classic.spi.TurboFilterList;
-import ch.qos.logback.classic.turbo.ReconfigureOnChangeFilter;
-import ch.qos.logback.classic.turbo.TurboFilter;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.QueryProcessor;
@@ -57,7 +59,6 @@
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
-import static ch.qos.logback.core.CoreConstants.RECONFIGURE_ON_CHANGE_TASK;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
@@ -1812,9 +1813,16 @@ public void testEmptyListAndNullInitcond() throws Throwable
     public void testLogbackReload() throws Throwable
     {
         // see https://issues.apache.org/jira/browse/CASSANDRA-11033
+        Logger l = LoggerFactory.getLogger(AggregationTest.class);
+        ch.qos.logback.classic.Logger logbackLogger = (ch.qos.logback.classic.Logger) l;
+        LoggerContext ctx = logbackLogger.getLoggerContext();
+
+        ReconfigureOnChangeTask rocTask = new ReconfigureOnChangeTask();
+        rocTask.setContext(ctx);
 
-        // make logback's scan interval 1ms - boilerplate, but necessary for this test
-        configureLogbackScanPeriod(1L);
+        ScheduledExecutorService scheduledExecutorService = ctx.getScheduledExecutorService();
+        ScheduledFuture<?> scheduledFuture = scheduledExecutorService.scheduleAtFixedRate(rocTask, 1, 1,
+                                                                                          TimeUnit.MILLISECONDS);
         try
         {
 
@@ -1863,43 +1871,10 @@ public void testLogbackReload() throws Throwable
         }
         finally
         {
-            configureLogbackScanPeriod(60000L);
+            scheduledFuture.cancel(true);
         }
     }
 
-    private static void configureLogbackScanPeriod(long millis)
-    {
-        Logger l = LoggerFactory.getLogger(AggregationTest.class);
-        ch.qos.logback.classic.Logger logbackLogger = (ch.qos.logback.classic.Logger) l;
-        LoggerContext ctx = logbackLogger.getLoggerContext();
-        TurboFilterList turboFilterList = ctx.getTurboFilterList();
-        boolean done = false;
-        for (TurboFilter turboFilter : turboFilterList)
-        {
-            if (turboFilter instanceof ReconfigureOnChangeFilter)
-            {
-                ReconfigureOnChangeFilter reconfigureFilter = (ReconfigureOnChangeFilter) turboFilter;
-                reconfigureFilter.setContext(ctx);
-                reconfigureFilter.setRefreshPeriod(millis);
-                reconfigureFilter.stop();
-                reconfigureFilter.start(); // start() sets the next check timestammp
-                done = true;
-                break;
-            }
-        }
-
-        ReconfigureOnChangeTask roct = (ReconfigureOnChangeTask) ctx.getObject(RECONFIGURE_ON_CHANGE_TASK);
-        if (roct != null)
-        {
-            // New functionality in logback - they replaced ReconfigureOnChangeFilter (which runs in the logging code)
-            // with an async ReconfigureOnChangeTask - i.e. in a thread that does not become sandboxed.
-            // Let the test run anyway, just we cannot reconfigure it (and it is pointless to reconfigure).
-            return;
-        }
-
-        assertTrue("ReconfigureOnChangeFilter not in logback's turbo-filter list - do that by adding scan=\"true\" to logback-test.xml's configuration element", done);
-    }
-
     @Test
     public void testOrReplaceOptionals() throws Throwable
     {
@@ -2170,8 +2145,55 @@ public void testRejectInvalidAggregateNamesOnCreation()
                                         " SFUNC func\n" +
                                         " STYPE map<text,bigint>\n" +
                                         " INITCOND { };");
-            }).hasRootCauseInstanceOf(InvalidRequestException.class)
-              .hasRootCauseMessage("Aggregate name '%s' is invalid", funcName);
+            }).isInstanceOf(InvalidRequestException.class)
+              .hasMessageContaining("Aggregate name '%s' is invalid", funcName);
         }
     }
+
+    @Test
+    public void testParseDeterministic() throws Throwable
+    {
+        testParseDeterministic(false);
+        testParseDeterministic(true);
+    }
+
+    private void testParseDeterministic(boolean deterministic) throws Throwable
+    {
+        createTable("CREATE TABLE %s (key int PRIMARY KEY, d double)");
+
+        String sfunc = shortFunctionName(createFunction(KEYSPACE,
+                                                        "int, int",
+                                                        "CREATE FUNCTION %s(a int, b int) " +
+                                                        "CALLED ON NULL INPUT " +
+                                                        "RETURNS int " +
+                                                        "LANGUAGE java " +
+                                                        "AS 'return a + b;'"));
+
+        String query = "CREATE AGGREGATE %s(int) SFUNC " + sfunc + " STYPE int";
+        if (deterministic)
+            query += " DETERMINISTIC";
+        String name = createAggregate(KEYSPACE, "int", query);
+
+        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace());
+        assertNotNull(ksm);
+        UDAggregate f = (UDAggregate) ksm.userFunctions.get(parseFunctionName(name)).iterator().next();
+        assertEquals(deterministic, f.isDeterministic());
+    }
+
+    @Test
+    public void testAggregatesAreNonDeterministicByDefault() throws Throwable
+    {
+        String fName = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s(i int, j int) RETURNS NULL ON NULL INPUT " +
+                                                       "RETURNS int " +
+                                                       "LANGUAGE java " +
+                                                       "AS 'return i + j;'");
+        String aName = createAggregate(KEYSPACE, "int", String.format("CREATE AGGREGATE %%s (int) SFUNC %s STYPE int INITCOND 1;", shortFunctionName(fName)));
+
+        UntypedResultSet aggregates = execute("SELECT * FROM system_schema.aggregates " +
+                                              "WHERE keyspace_name=? AND aggregate_name=?;",
+                                              KEYSPACE, shortFunctionName(aName));
+
+        Assert.assertEquals(1, aggregates.size());
+        Assert.assertFalse(aggregates.one().getBoolean("deterministic"));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
index c2cf8d1631d8..23678d1c8f96 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
@@ -20,16 +20,20 @@
 import java.util.UUID;
 
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.memtable.AbstractShardedMemtable;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.memtable.SkipListMemtable;
 import org.apache.cassandra.db.memtable.TestMemtable;
+import org.apache.cassandra.distributed.shared.WithProperties;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.SyntaxException;
@@ -42,6 +46,8 @@
 import org.apache.cassandra.utils.FBUtilities;
 
 import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
@@ -50,6 +56,16 @@
 
 public class AlterTest extends CQLTester
 {
+    @BeforeClass
+    public static void setUpClass()
+    {
+        // AlterTest uses Murmur3 partitioner, but injects OrderPreservingPartitioner.StringToken
+        // into TokenMetadata; expect trouble
+        MEMTABLE_SHARD_COUNT.setString("1");
+        CQLTester.setUpClass();
+        assertThat(AbstractShardedMemtable.getDefaultShardCount()).isEqualTo(1);;
+    }
+
     @Test
     public void testNonFrozenCollectionsAreIncompatibleWithBlob() throws Throwable
     {
@@ -499,6 +515,25 @@ public void testAlterKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() throw
                                            "ALTER KEYSPACE %s WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
     }
 
+    /**
+     * Test that when cassandra.dc_skip_name_validation=true nothing is thrown when altering a keyspace to invalid DC option in replication configuration.
+     */
+    @Test
+    public void testAlterKeyspaceWithNTSAcceptsAnyDataCenterNamesIfValidationIgnored() throws Throwable
+    {
+        try (WithProperties properties = new WithProperties().set(CassandraRelevantProperties.DATACENTER_SKIP_NAME_VALIDATION, true))
+        {
+            // Create a keyspace with expected DC name.
+            createKeyspace("CREATE KEYSPACE %s WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+
+            // try modifying the keyspace
+            alterKeyspace("ALTER KEYSPACE %s WITH replication = { 'class' : 'NetworkTopologyStrategy', 'INVALID_DC' : 2 }");
+
+            // Mix valid and invalid
+            alterKeyspace("ALTER KEYSPACE %s WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
+        }
+    }
+
     @Test
     public void testAlterKeyspaceWithMultipleInstancesOfSameDCThrowsSyntaxException() throws Throwable
     {
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java
index 2a0d665bbe96..2be2701b4fbd 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java
@@ -22,7 +22,9 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.CQLTester;
@@ -40,9 +42,21 @@
 import org.assertj.core.api.Assertions;
 
 import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
 
 public class CQLVectorTest extends CQLTester.InMemory
 {
+    @BeforeClass
+    public static void setUp()
+    {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
+    }
+    @AfterClass
+    public static void tearDown()
+    {
+        VECTOR_FLOAT_ONLY.setBoolean(true);
+    }
+    
     @Test
     public void select()
     {
@@ -77,12 +91,16 @@ public void select()
     }
 
     @Test
-    public void selectNonPk()
+    public void selectNonPk() throws Throwable
     {
         createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<int, 2>)");
 
         execute("INSERT INTO %s (pk, value) VALUES (0, [1, 2])");
         assertRows(execute("SELECT * FROM %s WHERE value=[1, 2] ALLOW FILTERING"), row(0, list(1, 2)));
+
+        assertInvalidThrowMessage("Cannot execute this query as it might involve data filtering and thus may have unpredictable performance. If you want to execute this query despite the performance unpredictability, use ALLOW FILTERING",
+                                  InvalidRequestException.class,
+                                  "SELECT * FROM %s WHERE value=[1, 2]");
     }
 
     @Test
@@ -554,4 +572,10 @@ public void udf() throws Throwable
         // make sure the function referencing the UDT is dropped before dropping the UDT at cleanup
         execute("DROP FUNCTION " + f);
     }
+
+    @SafeVarargs
+    protected final <T> Vector<T> vector(T... values)
+    {
+        return new Vector<>(values);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit2Test.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit2Test.java
index 1c4feb3840cf..40a8f5ca6892 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit2Test.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit2Test.java
@@ -155,7 +155,6 @@ public void testFilteringOnCompactTablesWithoutIndices() throws Throwable
             assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2) AND c IN (6, 7) ALLOW FILTERING"),
                        row(2, 1, 6));
 
-
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE c > 4");
 
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
index f671dd91593b..25ece3026372 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
@@ -17,15 +17,30 @@
  */
 package org.apache.cassandra.cql3.validation.operations;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.Duration;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.memtable.PersistentMemoryMemtable;
 import org.apache.cassandra.db.memtable.SkipListMemtable;
 import org.apache.cassandra.db.memtable.TestMemtable;
 import org.apache.cassandra.db.memtable.TrieMemtable;
 import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.distributed.shared.WithProperties;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.SyntaxException;
@@ -34,23 +49,25 @@
 import org.apache.cassandra.locator.IEndpointSnitch;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.MemtableParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaKeyspaceTables;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.triggers.ITrigger;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.UUID;
 
 import static java.lang.String.format;
-import static org.apache.cassandra.cql3.Duration.*;
-import static org.junit.Assert.*;
+import static org.apache.cassandra.cql3.Duration.NANOS_PER_HOUR;
+import static org.apache.cassandra.cql3.Duration.NANOS_PER_MICRO;
+import static org.apache.cassandra.cql3.Duration.NANOS_PER_MILLI;
+import static org.apache.cassandra.cql3.Duration.NANOS_PER_MINUTE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class CreateTest extends CQLTester
 {
@@ -109,13 +126,13 @@ public void testCreateTinyintColumns() throws Throwable
     @Test
     public void testCreateTableWithDurationColumns() throws Throwable
     {
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'a'",
+        assertInvalidMessage("Invalid type duration for column a: duration types are not supported within PRIMARY KEY columns",
                              "CREATE TABLE cql_test_keyspace.table0 (a duration PRIMARY KEY, b int);");
 
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'b'",
+        assertInvalidMessage("Invalid type duration for column b: duration types are not supported within PRIMARY KEY columns",
                              "CREATE TABLE cql_test_keyspace.table0 (a text, b duration, c duration, primary key (a, b));");
 
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'b'",
+        assertInvalidMessage("Invalid type duration for column b: duration types are not supported within PRIMARY KEY columns",
                              "CREATE TABLE cql_test_keyspace.table0 (a text, b duration, c duration, primary key (a, b)) with clustering order by (b DESC);");
 
         createTable("CREATE TABLE %s (a int, b int, c duration, primary key (a, b));");
@@ -195,7 +212,7 @@ public void testCreateTableWithDurationColumns() throws Throwable
         createTable("CREATE TABLE %s (a text PRIMARY KEY, duration duration);");
 
         // Test duration within Map
-        assertInvalidMessage("Durations are not allowed as map keys: map<duration, text>",
+        assertInvalidMessage("Invalid type map<duration, text> for column m: duration types are not supported within non-frozen map keys",
                              "CREATE TABLE cql_test_keyspace.table0(pk int PRIMARY KEY, m map<duration, text>)");
 
         createTable("CREATE TABLE %s(pk int PRIMARY KEY, m map<text, duration>)");
@@ -203,17 +220,17 @@ public void testCreateTableWithDurationColumns() throws Throwable
         assertRows(execute("SELECT * FROM %s"),
                    row(1, map("one month", Duration.from("1mo"), "60 days", Duration.from("60d"))));
 
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'm'",
-                "CREATE TABLE cql_test_keyspace.table0(m frozen<map<text, duration>> PRIMARY KEY, v int)");
+        assertInvalidMessage("Invalid type frozen<map<text, duration>> for column m: duration types are not supported within PRIMARY KEY columns",
+                             "CREATE TABLE cql_test_keyspace.table0(m frozen<map<text, duration>> PRIMARY KEY, v int)");
 
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'm'",
+        assertInvalidMessage("Invalid type frozen<map<text, duration>> for column m: duration types are not supported within PRIMARY KEY columns",
                              "CREATE TABLE cql_test_keyspace.table0(pk int, m frozen<map<text, duration>>, v int, PRIMARY KEY (pk, m))");
 
         // Test duration within Set
-        assertInvalidMessage("Durations are not allowed inside sets: set<duration>",
+        assertInvalidMessage("Invalid type set<duration> for column s: duration types are not supported within non-frozen",
                              "CREATE TABLE cql_test_keyspace.table0(pk int PRIMARY KEY, s set<duration>)");
 
-        assertInvalidMessage("Durations are not allowed inside sets: frozen<set<duration>>",
+        assertInvalidMessage("Invalid type frozen<set<duration>> for column s: duration types are not supported within PRIMARY KEY columns",
                              "CREATE TABLE cql_test_keyspace.table0(s frozen<set<duration>> PRIMARY KEY, v int)");
 
         // Test duration within List
@@ -222,7 +239,7 @@ public void testCreateTableWithDurationColumns() throws Throwable
         assertRows(execute("SELECT * FROM %s"),
                    row(1, list(Duration.from("1mo"), Duration.from("60d"))));
 
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'l'",
+        assertInvalidMessage("Invalid type frozen<list<duration>> for column l: duration types are not supported within PRIMARY KEY columns",
                              "CREATE TABLE cql_test_keyspace.table0(l frozen<list<duration>> PRIMARY KEY, v int)");
 
         // Test duration within Tuple
@@ -231,23 +248,23 @@ public void testCreateTableWithDurationColumns() throws Throwable
         assertRows(execute("SELECT * FROM %s"),
                    row(1, tuple(1, Duration.from("1mo"))));
 
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 't'",
+        assertInvalidMessage("Invalid type frozen<tuple<int, duration>> for column t: duration types are not supported within PRIMARY KEY columns",
                              "CREATE TABLE cql_test_keyspace.table0(t frozen<tuple<int, duration>> PRIMARY KEY, v int)");
 
         // Test duration within UDT
         String typename = createType("CREATE TYPE %s (a duration)");
         String myType = KEYSPACE + '.' + typename;
-        createTable("CREATE TABLE %s(pk int PRIMARY KEY, u " + myType + ")");
+        createTable("CREATE TABLE %s(pk int PRIMARY KEY, u " + myType + ')');
         execute("INSERT INTO %s (pk, u) VALUES (1, {a : 1mo})");
         assertRows(execute("SELECT * FROM %s"),
                    row(1, userType("a", Duration.from("1mo"))));
 
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'u'",
+        assertInvalidMessage("Invalid type frozen<" + typename + "> for column u: duration types are not supported within PRIMARY KEY columns",
                              "CREATE TABLE cql_test_keyspace.table0(pk int, u frozen<" + myType + ">, v int, PRIMARY KEY(pk, u))");
 
         // Test duration with several level of depth
-        assertInvalidMessage("duration type is not supported for PRIMARY KEY column 'm'",
-                "CREATE TABLE cql_test_keyspace.table0(pk int, m frozen<map<text, list<tuple<int, duration>>>>, v int, PRIMARY KEY (pk, m))");
+        assertInvalidMessage("Invalid type frozen<map<text, frozen<list<frozen<tuple<int, duration>>>>>> for column m: duration types are not supported within PRIMARY KEY columns",
+                             "CREATE TABLE cql_test_keyspace.table0(pk int, m frozen<map<text, list<tuple<int, duration>>>>, v int, PRIMARY KEY (pk, m))");
     }
 
     private ByteBuffer duration(long months, long days, long nanoseconds) throws IOException
@@ -371,8 +388,8 @@ public void testKeyspace() throws Throwable
 
         execute("CREATE KEYSPACE testXYZ WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
 
-        assertInvalid(
-                     "CREATE KEYSPACE My_much_much_too_long_identifier_that_should_not_work WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+        String tooLongKeyspace = IntStream.range(0, SchemaConstants.NAME_LENGTH + 1).mapToObj(i ->"x").collect(Collectors.joining());
+        assertInvalid("CREATE KEYSPACE " + tooLongKeyspace + " WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
 
         execute("DROP KEYSPACE testXYZ");
         assertInvalidThrow(InvalidRequestException.class, "DROP KEYSPACE non_existing");
@@ -384,7 +401,7 @@ public void testKeyspace() throws Throwable
     }
 
     /**
-     *  Test {@link ConfigurationException} is thrown on create keyspace with invalid DC option in replication configuration .
+     *  Test {@link ConfigurationException} is thrown on create keyspace with invalid DC option in replication configuration.
      */
     @Test
     public void testCreateKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() throws Throwable
@@ -400,6 +417,25 @@ public void testCreateKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() thro
         execute("DROP KEYSPACE IF EXISTS testXYZ");
     }
 
+    /**
+     *  Test that when cassandra.dc_skip_name_validation=true nothing is thrown on create keyspace with invalid DC option in replication configuration.
+     */
+    @Test
+    public void testCreateKeyspaceWithNTSAcceptsAnyDataCenterNamesIfValidationIgnored() throws Throwable
+    {
+        try (WithProperties properties = new WithProperties())
+        {
+            properties.set(CassandraRelevantProperties.DATACENTER_SKIP_NAME_VALIDATION, true);
+
+            execute("CREATE KEYSPACE testABC WITH replication = { 'class' : 'NetworkTopologyStrategy', 'INVALID_DC' : 2 }");
+            execute("CREATE KEYSPACE testXYZ WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
+
+            // clean-up
+            execute("DROP KEYSPACE IF EXISTS testABC");
+            execute("DROP KEYSPACE IF EXISTS testXYZ");
+        }
+    }
+
     /**
      * Test {@link ConfigurationException} is not thrown on create NetworkTopologyStrategy keyspace without any options.
      */
@@ -613,6 +649,13 @@ public void testCreateTableWithMemtable() throws Throwable
         testMemtableConfig("test_shortname", SkipListMemtable.FACTORY, SkipListMemtable.class);
         testMemtableConfig("default", MemtableParams.DEFAULT.factory(), defaultClass);
 
+        // Handle CC 4.0 memtable configuration given as a map
+        testMapMemtableConfig("", null, MemtableParams.DEFAULT.factory(), defaultClass);
+        testMapMemtableConfig("SkipListMemtable", "skiplist", MemtableParams.get("skiplist").factory(), SkipListMemtable.class);
+        testMapMemtableConfig("TrieMemtable","trie", MemtableParams.get("trie").factory(), TrieMemtable.class);
+        testMapMemtableConfig("TrieMemtableStage1", "trie", MemtableParams.get("trie").factory(), TrieMemtable.class);
+        testMapMemtableConfig("PersistentMemoryMemtable", "persistent_memory", MemtableParams.get("persistent_memory").factory(), PersistentMemoryMemtable.class);
+
         assertThrowsConfigurationException("The 'class_name' option must be specified.",
                                            "CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
                                            + " WITH memtable = 'test_empty_class';");
@@ -656,6 +699,17 @@ private void testMemtableConfig(String memtableConfig, Memtable.Factory factoryI
         assertSchemaOption("memtable", MemtableParams.DEFAULT.configurationKey().equals(memtableConfig) ? null : memtableConfig);
     }
 
+    private void testMapMemtableConfig(String memtableConfig, String expectedMemtableConfig, Memtable.Factory factoryInstance, Class<? extends Memtable> memtableClass) throws Throwable
+    {
+        String memtableMap = "".equals(memtableConfig) ? memtableConfig : String.format("'class' : '%s'", memtableConfig);
+        createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+                    + " WITH memtable = {" + memtableMap + "};");
+        assertSame(factoryInstance, getCurrentColumnFamilyStore().metadata().params.memtable.factory());
+        Assert.assertTrue(memtableClass.isInstance(getCurrentColumnFamilyStore().getTracker().getView().getCurrentMemtable()));
+
+        assertSchemaOption("memtable", expectedMemtableConfig);
+    }
+
     void assertSchemaOption(String option, Object expected) throws Throwable
     {
         assertRows(execute(format("SELECT " + option + " FROM %s.%s WHERE keyspace_name = ? and table_name = ?;",
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
index 645becd94c80..2d340fc58649 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
@@ -538,6 +538,29 @@ private void testDeleteWithOneClusteringColumns(boolean forceFlush) throws Throw
                    row(0, 2, 2),
                    row(0, 3, 3));
 
+        // test slices:
+        execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 0, 0)");
+        execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 1, 1)");
+        execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 2, 2)");
+        execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 3, 3)");
+        execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 4, 4)");
+        execute("INSERT INTO %s (partitionKey, clustering, value) VALUES (0, 5, 5)");
+        flush(forceFlush);
+
+        execute("DELETE FROM %s WHERE partitionKey = ? AND clustering > ?", 0, 3);
+        flush(forceFlush);
+        assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+                   row(0, 0, 0),
+                   row(0, 1, 1),
+                   row(0, 2, 2),
+                   row(0, 3, 3));
+
+        execute("DELETE FROM %s WHERE partitionKey = ? AND clustering NOT IN (?, ?)", 0, 1, 2);
+        flush(forceFlush);
+        assertRows(execute("SELECT * FROM %s WHERE partitionKey = ?", 0),
+                   row(0, 1, 1),
+                   row(0, 2, 2));
+
         // test invalid queries
 
         // missing primary key element
@@ -572,6 +595,7 @@ private void testDeleteWithOneClusteringColumns(boolean forceFlush) throws Throw
         // Non primary key in the where clause
         assertInvalidMessage("Non PRIMARY KEY columns found in where clause: value",
                              "DELETE FROM %s WHERE partitionKey = ? AND clustering = ? AND value = ?", 0, 1, 3);
+
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/DropRecreateAndRestoreTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/DropRecreateAndRestoreTest.java
index ea08d7878465..62f89b06096f 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/DropRecreateAndRestoreTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/DropRecreateAndRestoreTest.java
@@ -53,7 +53,7 @@ public void testCreateWithIdRestore() throws Throwable
 
         // Drop will flush and clean segments. Hard-link them so that they can be restored later.
         List<String> segments = CommitLog.instance.getActiveSegmentNames();
-        File logPath = new File(DatabaseDescriptor.getCommitLogLocation());
+        File logPath = DatabaseDescriptor.getCommitLogLocation();
         for (String segment: segments)
             FileUtils.createHardLink(new File(logPath, segment), new File(logPath, segment + ".save"));
 
@@ -65,7 +65,7 @@ public void testCreateWithIdRestore() throws Throwable
 
         // Restore saved segments
         for (String segment: segments)
-            FileUtils.renameWithConfirm(new File(logPath, segment + ".save"), new File(logPath, segment));
+            new File(logPath, segment + ".save").move(new File(logPath, segment));
         try
         {
             // Restore to point in time (microseconds granularity)
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertInvalidateSizedRecordsTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertInvalidateSizedRecordsTest.java
index dcfb9baa656d..71d4dcb2c4f9 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertInvalidateSizedRecordsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertInvalidateSizedRecordsTest.java
@@ -45,6 +45,7 @@ public class InsertInvalidateSizedRecordsTest extends CQLTester
     private static final ByteBuffer LARGE_BLOB = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT + 1);
     private static final ByteBuffer MEDIUM_BLOB = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT / 2 + 10);
 
+    static
     {
         requireNetwork();
     }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectMultiColumnRelationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectMultiColumnRelationTest.java
index 9c89678b8c05..b955751b192b 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectMultiColumnRelationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectMultiColumnRelationTest.java
@@ -36,7 +36,7 @@ public void testSingleClusteringInvalidQueries() throws Throwable
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
 
         assertInvalidSyntax("SELECT * FROM %s WHERE () = (?, ?)", 1, 2);
-        assertInvalidMessage("b cannot be restricted by more than one relation if it includes an Equal",
+        assertInvalidMessage("More than one restriction was found for the start bound on b",
                              "SELECT * FROM %s WHERE a = 0 AND (b) = (?) AND (b) > (?)", 0, 0);
         assertInvalidMessage("More than one restriction was found for the start bound on b",
                              "SELECT * FROM %s WHERE a = 0 AND (b) > (?) AND (b) > (?)", 0, 1);
@@ -368,9 +368,98 @@ public void testSingleClustering() throws Throwable
     @Test
     public void testNonEqualsRelation() throws Throwable
     {
-        createTable("CREATE TABLE %s (a int PRIMARY KEY, b int)");
-        assertInvalidMessage("Unsupported \"!=\" relation: (b) != (0)",
-                             "SELECT * FROM %s WHERE a = 0 AND (b) != (0)");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b, c))");
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 1);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 0);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
+
+        // Excluding subtrees
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b) != (?)", 0, 0),
+                   row(0, 1, 0),
+                   row(0, 1, 1)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b) != (?)", 0, 1),
+                   row(0, 0, 0),
+                   row(0, 0, 1)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b) != (?)", 0, -1),
+                   row(0, 0, 0),
+                   row(0, 0, 1),
+                   row(0, 1, 0),
+                   row(0, 1, 1)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b) != (?)", 0, 2),
+                   row(0, 0, 0),
+                   row(0, 0, 1),
+                   row(0, 1, 0),
+                   row(0, 1, 1)
+        );
+
+        // Excluding single rows
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b, c) != (?, ?)", 0, -1, -1),
+                   row(0, 0, 0),
+                   row(0, 0, 1),
+                   row(0, 1, 0),
+                   row(0, 1, 1)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b, c) != (?, ?)", 0, 0, 1),
+                   row(0, 0, 0),
+                   row(0, 1, 0),
+                   row(0, 1, 1)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b, c) != (?, ?)", 0, 2, 2),
+                   row(0, 0, 0),
+                   row(0, 0, 1),
+                   row(0, 1, 0),
+                   row(0, 1, 1)
+        );
+
+        // Merging multiple != =
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b, c) != ? AND (b, c) != ?", 0, tuple(0, 1), tuple(1, 0)),
+                   row(0, 0, 0),
+                   row(0, 1, 1)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND b = ? AND (b, c) != ?", 0, 0, tuple(0, 1)),
+                   row(0, 0, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b) = (?) AND (b, c) != ?", 0, 0, tuple(0, 1)),
+                   row(0, 0, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND b != ? AND (b, c) != ?", 0, 1, tuple(0, 1)),
+                   row(0, 0, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b) != (?) AND (b, c) != ?", 0, 1, tuple(0, 1)),
+                   row(0, 0, 0)
+        );
+
+        // Merging with < <= >= >
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND b < ? AND (b, c) != ?", 0, 1, tuple(0, 1)),
+                   row(0, 0, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND b <= ? AND (b, c) != ?", 0, 0, tuple(0, 1)),
+                   row(0, 0, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND b > ? AND (b, c) != ?", 0, 0, tuple(1, 1)),
+                   row(0, 1, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND b >= ? AND (b, c) != ?", 0, 1, tuple(1, 1)),
+                   row(0, 1, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b, c) < ? AND (b, c) != ?", 0, tuple(0, 2), tuple(0, 1)),
+                   row(0, 0, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b, c) <= ? AND (b, c) != ?", 0, tuple(0, 2), tuple(0, 1)),
+                   row(0, 0, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b, c) > ? AND (b, c) != ?", 0, tuple(0, 0), tuple(1, 1)),
+                   row(0, 0, 1),
+                   row(0, 1, 0)
+        );
+        assertRows(execute("SELECT a, b, c FROM %s WHERE a = ? AND (b, c) >= ? AND (b, c) != ?", 0, tuple(0, 1), tuple(1, 1)),
+                   row(0, 0, 1),
+                   row(0, 1, 0)
+        );
     }
 
     @Test
@@ -829,7 +918,7 @@ public void testMultipleClusteringWithIndex() throws Throwable
         assertRows(execute("SELECT * FROM %s WHERE (b) IN ((?)) AND e = ? ALLOW FILTERING", 1, 2),
                    row(0, 1, 1, 1, 2));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "b"),
                              "SELECT * FROM %s WHERE (b) IN ((?), (?)) AND e = ?", 0, 1, 2);
         assertRows(execute("SELECT * FROM %s WHERE (b) IN ((?), (?)) AND e = ? ALLOW FILTERING", 0, 1, 2),
                    row(0, 0, 1, 1, 2),
@@ -840,18 +929,18 @@ public void testMultipleClusteringWithIndex() throws Throwable
         assertRows(execute("SELECT * FROM %s WHERE (b, c) IN ((?, ?)) AND e = ? ALLOW FILTERING", 0, 1, 2),
                    row(0, 0, 1, 1, 2));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "b"),
                              "SELECT * FROM %s WHERE (b, c) IN ((?, ?), (?, ?)) AND e = ?", 0, 1, 1, 1, 2);
         assertRows(execute("SELECT * FROM %s WHERE (b, c) IN ((?, ?), (?, ?)) AND e = ? ALLOW FILTERING", 0, 1, 1, 1, 2),
                    row(0, 0, 1, 1, 2),
                    row(0, 1, 1, 1, 2));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "b"),
                              "SELECT * FROM %s WHERE (b) >= (?) AND e = ?", 1, 2);
         assertRows(execute("SELECT * FROM %s WHERE (b) >= (?) AND e = ? ALLOW FILTERING", 1, 2),
                    row(0, 1, 1, 1, 2));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "b"),
                              "SELECT * FROM %s WHERE (b, c) >= (?, ?) AND e = ?", 1, 1, 2);
         assertRows(execute("SELECT * FROM %s WHERE (b, c) >= (?, ?) AND e = ? ALLOW FILTERING", 1, 1, 2),
                    row(0, 1, 1, 1, 2));
@@ -936,23 +1025,23 @@ public void testMultiplePartitionKeyAndMultiClusteringWithIndex() throws Throwab
                    row(0, 0, 1, 1, 1, 5));
 
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND (c, d) IN ((?, ?)) ALLOW FILTERING", 0, 1, 1),
-                row(0, 0, 1, 1, 0, 4),
-                row(0, 0, 1, 1, 1, 5));
+                   row(0, 0, 1, 1, 0, 4),
+                   row(0, 0, 1, 1, 1, 5));
 
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND (c, d) >= (?, ?) ALLOW FILTERING", 0, 1, 1),
-                row(0, 0, 1, 1, 0, 4),
-                row(0, 0, 1, 1, 1, 5),
-                row(0, 0, 2, 0, 0, 5));
+                   row(0, 0, 1, 1, 0, 4),
+                   row(0, 0, 1, 1, 1, 5),
+                   row(0, 0, 2, 0, 0, 5));
 
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = ? AND (c) IN ((?)) AND f = ?", 0, 0, 1, 5),
                    row(0, 0, 1, 1, 1, 5));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "c"),
                              "SELECT * FROM %s WHERE a = ? AND (c) IN ((?), (?)) AND f = ?", 0, 1, 3, 5);
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND (c) IN ((?), (?)) AND f = ? ALLOW FILTERING", 0, 1, 3, 5),
                    row(0, 0, 1, 1, 1, 5));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "c"),
                              "SELECT * FROM %s WHERE a = ? AND (c) IN ((?), (?)) AND f = ?", 0, 1, 2, 5);
 
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = ? AND (c) IN ((?), (?)) AND f = ?", 0, 0, 1, 2, 5),
@@ -970,7 +1059,7 @@ public void testMultiplePartitionKeyAndMultiClusteringWithIndex() throws Throwab
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND (c, d) IN ((?, ?)) AND f = ? ALLOW FILTERING", 0, 1, 0, 3),
                    row(0, 0, 1, 0, 0, 3));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "c"),
                              "SELECT * FROM %s WHERE a = ? AND (c) >= (?) AND f = ?", 0, 1, 5);
 
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = ? AND (c) >= (?) AND f = ?", 0, 0, 1, 5),
@@ -984,7 +1073,7 @@ public void testMultiplePartitionKeyAndMultiClusteringWithIndex() throws Throwab
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = ? AND (c, d) >= (?, ?) AND f = ?", 0, 0, 1, 1, 5),
                    row(0, 0, 1, 1, 1, 5),
                    row(0, 0, 2, 0, 0, 5));
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "c"),
                              "SELECT * FROM %s WHERE a = ? AND (c, d) >= (?, ?) AND f = ?", 0, 1, 1, 5);
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND (c, d) >= (?, ?) AND f = ? ALLOW FILTERING", 0, 1, 1, 5),
                    row(0, 0, 1, 1, 1, 5),
@@ -1053,10 +1142,10 @@ public void testMixedOrderColumns1() throws Throwable
         execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, -1, 0);
         execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, 0, 0);
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)<=(?,?,?,?) " +
-        "AND (b)>(?)", 0, 2, 0, 1, 1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)<=(?,?,?,?) " +
+                   "AND (b)>(?)", 0, 2, 0, 1, 1, -1),
 
                    row(0, 2, 0, 1, 1),
                    row(0, 2, 0, -1, 0),
@@ -1079,10 +1168,10 @@ public void testMixedOrderColumns1() throws Throwable
 
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)<=(?,?,?,?) " +
-        "AND (b)>=(?)", 0, 2, 0, 1, 1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)<=(?,?,?,?) " +
+                   "AND (b)>=(?)", 0, 2, 0, 1, 1, -1),
 
                    row(0, 2, 0, 1, 1),
                    row(0, 2, 0, -1, 0),
@@ -1106,20 +1195,20 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d)>=(?,?,?)" +
-        "AND (b,c,d,e)<(?,?,?,?) ", 0, 1, 1, 0, 1, 1, 0, 1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d)>=(?,?,?)" +
+                   "AND (b,c,d,e)<(?,?,?,?) ", 0, 1, 1, 0, 1, 1, 0, 1),
                    row(0, 1, 1, 0, -1),
                    row(0, 1, 1, 0, 0)
 
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)>(?,?,?,?)" +
-        "AND (b,c,d)<=(?,?,?) ", 0, -1, 0, -1, -1, 2, 0, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)>(?,?,?,?)" +
+                   "AND (b,c,d)<=(?,?,?) ", 0, -1, 0, -1, -1, 2, 0, -1),
 
                    row(0, 2, 0, -1, 0),
                    row(0, 2, 0, -1, 1),
@@ -1142,27 +1231,27 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e) < (?,?,?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e) < (?,?,?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
                    row(0, 1, 0, 0, -1)
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e) <= (?,?,?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e) <= (?,?,?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
                    row(0, 1, 0, 0, -1),
                    row(0, 1, 0, 0, 0)
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b)<(?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b)<(?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, -1, 0, -1, -1),
 
                    row(0, 1, -1, 1, 0),
                    row(0, 1, -1, 1, 1),
@@ -1185,10 +1274,10 @@ public void testMixedOrderColumns1() throws Throwable
 
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b)<(?) " +
-        "AND (b)>(?)", 0, 2, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b)<(?) " +
+                   "AND (b)>(?)", 0, 2, -1),
 
                    row(0, 1, -1, 1, 0),
                    row(0, 1, -1, 1, 1),
@@ -1208,10 +1297,10 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b)<(?) " +
-        "AND (b)>=(?)", 0, 2, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b)<(?) " +
+                   "AND (b)>=(?)", 0, 2, -1),
 
                    row(0, 1, -1, 1, 0),
                    row(0, 1, -1, 1, 1),
@@ -1232,10 +1321,10 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)<=(?,?,?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)<=(?,?,?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
 
                    row(0, 2, 0, 1, 1),
                    row(0, 2, 0, -1, 0),
@@ -1259,10 +1348,10 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c)<=(?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c)<=(?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
 
                    row(0, 2, 0, 1, 1),
                    row(0, 2, 0, -1, 0),
@@ -1286,10 +1375,10 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d)<=(?,?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d)<=(?,?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, -1, 0, -1, -1),
 
                    row(0, 2, 0, -1, 0),
                    row(0, 2, 0, -1, 1),
@@ -1312,10 +1401,10 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)>(?,?,?,?)" +
-        "AND (b,c,d)<=(?,?,?) ", 0, -1, 0, -1, -1, 2, 0, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)>(?,?,?,?)" +
+                   "AND (b,c,d)<=(?,?,?) ", 0, -1, 0, -1, -1, 2, 0, -1),
 
                    row(0, 2, 0, -1, 0),
                    row(0, 2, 0, -1, 1),
@@ -1338,28 +1427,28 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d)>=(?,?,?)" +
-        "AND (b,c,d,e)<(?,?,?,?) ", 0, 1, 1, 0, 1, 1, 0, 1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d)>=(?,?,?)" +
+                   "AND (b,c,d,e)<(?,?,?,?) ", 0, 1, 1, 0, 1, 1, 0, 1),
                    row(0, 1, 1, 0, -1),
                    row(0, 1, 1, 0, 0)
         );
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)<(?,?,?,?) " +
-        "AND (b,c,d)>=(?,?,?)", 0, 1, 1, 0, 1, 1, 1, 0),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)<(?,?,?,?) " +
+                   "AND (b,c,d)>=(?,?,?)", 0, 1, 1, 0, 1, 1, 1, 0),
                    row(0, 1, 1, 0, -1),
                    row(0, 1, 1, 0, 0)
 
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c)<(?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c)<(?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
                    row(0, 1, -1, 1, 0),
                    row(0, 1, -1, 1, 1),
                    row(0, 1, -1, 0, 0),
@@ -1379,10 +1468,10 @@ public void testMixedOrderColumns1() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c)<(?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c)<(?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
                    row(0, 1, -1, 1, 0),
                    row(0, 1, -1, 1, 1),
                    row(0, 1, -1, 0, 0),
@@ -1593,10 +1682,10 @@ public void testMixedOrderColumns4() throws Throwable
         execute("INSERT INTO %s (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)", 0, -1, 0, 0, 0);
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)<(?,?,?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)<(?,?,?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
 
                    row(0, -1, 0, 0, 0),
                    row(0, -1, 0, -1, 0),
@@ -1623,28 +1712,28 @@ public void testMixedOrderColumns4() throws Throwable
 
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e) < (?,?,?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e) < (?,?,?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
                    row(0, 1, 0, 0, -1)
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e) <= (?,?,?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e) <= (?,?,?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 1, 0, 0, 0, 1, 0, -1, -1),
                    row(0, 1, 0, 0, -1),
                    row(0, 1, 0, 0, 0)
         );
 
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)<=(?,?,?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)<=(?,?,?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, 1, 1, -1, 0, -1, -1),
 
                    row(0, -1, 0, 0, 0),
                    row(0, -1, 0, -1, 0),
@@ -1670,10 +1759,10 @@ public void testMixedOrderColumns4() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c)<=(?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c)<=(?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
 
                    row(0, -1, 0, 0, 0),
                    row(0, -1, 0, -1, 0),
@@ -1699,10 +1788,10 @@ public void testMixedOrderColumns4() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c)<(?,?) " +
-        "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c)<(?,?) " +
+                   "AND (b,c,d,e)>(?,?,?,?)", 0, 2, 0, -1, 0, -1, -1),
                    row(0, -1, 0, 0, 0),
                    row(0, -1, 0, -1, 0),
                    row(0, 0, 0, 0, 0),
@@ -1724,10 +1813,10 @@ public void testMixedOrderColumns4() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)<=(?,?,?,?) " +
-        "AND (b)>=(?)", 0, 2, 0, 1, 1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)<=(?,?,?,?) " +
+                   "AND (b)>=(?)", 0, 2, 0, 1, 1, -1),
 
                    row(0, -1, 0, 0, 0),
                    row(0, -1, 0, -1, 0),
@@ -1753,10 +1842,10 @@ public void testMixedOrderColumns4() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e)<=(?,?,?,?) " +
-        "AND (b)>(?)", 0, 2, 0, 1, 1, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e)<=(?,?,?,?) " +
+                   "AND (b)>(?)", 0, 2, 0, 1, 1, -1),
 
                    row(0, 0, 0, 0, 0),
                    row(0, 1, 1, 0, -1),
@@ -1855,21 +1944,21 @@ public void testMixedOrderColumns4() throws Throwable
         );
 
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b) < (?) ", 0, 0),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b) < (?) ", 0, 0),
                    row(0, -1, 0, 0, 0), row(0, -1, 0, -1, 0)
         );
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b) <= (?) ", 0, -1),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b) <= (?) ", 0, -1),
                    row(0, -1, 0, 0, 0), row(0, -1, 0, -1, 0)
         );
         assertRows(execute(
-        "SELECT * FROM %s" +
-        " WHERE a = ? " +
-        "AND (b,c,d,e) < (?,?,?,?) and (b,c,d,e) > (?,?,?,?) ", 0, 2, 0, 0, 0, 2, -2, 0, 0),
+                   "SELECT * FROM %s" +
+                   " WHERE a = ? " +
+                   "AND (b,c,d,e) < (?,?,?,?) and (b,c,d,e) > (?,?,?,?) ", 0, 2, 0, 0, 0, 2, -2, 0, 0),
                    row(0, 2, 0, -1, 0),
                    row(0, 2, 0, -1, 1),
                    row(0, 2, -1, 1, 1)
@@ -1932,6 +2021,7 @@ public void testInvalidColumnNames() throws Throwable
         assertInvalidMessage("Undefined column name e", "SELECT * FROM %s WHERE (b, e) > (0, 1) and b <= 2");
         assertInvalidMessage("Undefined column name e", "SELECT c AS e FROM %s WHERE (b, e) = (0, 0)");
         assertInvalidMessage("Undefined column name e", "SELECT c AS e FROM %s WHERE (b, e) IN ((0, 1), (2, 4))");
+        assertInvalidMessage("Undefined column name e", "SELECT c AS e FROM %s WHERE (b, e) NOT IN ((0, 1), (2, 4))");
         assertInvalidMessage("Undefined column name e", "SELECT c AS e FROM %s WHERE (b, e) > (0, 1) and b <= 2");
     }
 
@@ -1976,4 +2066,477 @@ public void testInRestrictionsWithIndex() throws Throwable
         assertInvalidMessage("Multicolumn IN filters are not supported",
                              "SELECT * FROM %s WHERE (c2, c3) IN ((?, ?), (?, ?)) ALLOW FILTERING", 1, 0, 2, 0);
     }
- }
+
+    @Test
+    public void testNotInRestrictionsWithClustering() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, c1 int, c2 int, v int, primary key(pk, c1, c2))");
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 1, 11);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 2, 12);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 3, 13);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 4, 14);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 1, 21);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 2, 22);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 3, 23);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 4, 24);
+
+        // empty NOT IN
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN ()", 1),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+
+        // non existent NOT IN:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN ((?, ?))", 1, 2000, 2001),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+
+        // existing values in NOT IN, different ways of passing them:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN ((0, 2), (0, 3), (1, 1), (1, 4))", 1),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN ((?, ?), (?, ?), (?, ?), (?, ?))",
+                           1, 0, 2, 0, 3, 1, 1, 1, 4),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN (?, ?, ?, ?)",
+                           1, tuple(0, 2), tuple(0, 3), tuple(1, 1), tuple(1, 4)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN ?",
+                           1, list(tuple(0, 2), tuple(0, 3), tuple(1, 1), tuple(1, 4))),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        // Tuples given in arbitrary order:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN (?, ?, ?, ?)",
+                           1, tuple(0, 3), tuple(1, 4), tuple(1, 1), tuple(0, 2)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        // Multiple NOT IN:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN (?, ?) AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(0, 2), tuple(0, 3), tuple(1, 1), tuple(1, 4)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        // Multiple NOT IN, mixed markers:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN (?, ?) AND (c1, c2) NOT IN ?",
+                           1, tuple(0, 2), tuple(0, 3), list(tuple(1, 1), tuple(1, 4))),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        // Multiple NOT IN, mixed markers and values:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) NOT IN ((0, 2), (0, 3)) AND (c1, c2) NOT IN ?",
+                           1, list(tuple(1, 1), tuple(1, 4))),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        // Mixed single-column and multicolumn restrictions:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 NOT IN ? AND (c1, c2) NOT IN ?",
+                           1, list(0), list(tuple(1, 1), tuple(1, 4))),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 NOT IN (?) AND (c1, c2) NOT IN (?, ?)",
+                           1, 0, tuple(1, 1), tuple(1, 4)),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+    }
+
+    @Test
+    public void testNotInRestrictionsWithClusteringAndSlices() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, c1 int, c2 int, v int, primary key(pk, c1, c2))");
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 1, 11);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 2, 12);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 3, 13);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 4, 14);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 1, 21);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 2, 22);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 3, 23);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 4, 24);
+
+        // NOT IN values outside of slice bounds
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 2), tuple(2, 5)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 1, 21));
+
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 2), tuple(1, 1)),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+
+        // Empty result set
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(1, 2), tuple(1, 3), tuple(1, 4)));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(0, 3), tuple(0, 2), tuple(0, 1)));
+
+        // NOT IN values inside slice bounds
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 2), tuple(0, 2)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 1, 21));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) <= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 2), tuple(0, 2)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 1), tuple(1, 3)),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 1), tuple(1, 3)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 4, 24));
+
+
+        // One NOT IN value exactly the same as the slice bound
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(1, 2), tuple(0, 2), tuple(1, 2)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 1, 21));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) <= ? AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(1, 2), tuple(0, 2), tuple(1, 2)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14),
+                   row(1, 1, 1, 21));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 1), tuple(1, 1)),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 1), tuple(1, 1)),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+
+        // NOT IN with both upper and lower bound
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 2), tuple(0, 4)),
+                   row(1, 0, 3, 13),
+                   row(1, 1, 1, 21));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 2), tuple(0, 4)),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 1, 1, 21));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) <= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 2), tuple(0, 4)),
+                   row(1, 0, 3, 13),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) <= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 2), tuple(0, 4)),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22));
+
+        // Mixed multi-column NOT IN with single column slice restriction:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 > ? AND (c1, c2) NOT IN (?)",
+                           1, 0, tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 >= ? AND (c1, c2) NOT IN (?)",
+                           1, 1, tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 < ? AND (c1, c2) NOT IN (?)",
+                           1, 1, tuple(0, 4)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 <= ? AND (c1, c2) NOT IN (?)",
+                           1, 0, tuple(0, 4)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 >= ? AND c1 <= ? AND (c1, c2) NOT IN (?)",
+                           1, 0, 1, tuple(0, 4)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+
+        // Mixed single-column NOT IN with multi-column slice restriction:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND c1 NOT IN (?)",
+                           1, tuple(0, 1), 1),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND c1 NOT IN (?)",
+                           1, tuple(0, 1), 1),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND c1 NOT IN (?)",
+                           1, tuple(1, 3), 1),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND c1 NOT IN (?)",
+                           1, tuple(1, 3), 0),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) <= ? AND c1 NOT IN (?)",
+                           1, tuple(1, 3), 0),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) <= ? AND c1 NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 3), 0),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+    }
+
+    @Test
+    public void testNotInRestrictionsWithMixedOrderClusteringAndSlices() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, c1 int, c2 int, v int, primary key(pk, c1, c2)) WITH CLUSTERING ORDER BY (c1 DESC, c2 ASC)");
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 1, 11);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 2, 12);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 3, 13);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 0, 4, 14);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 1, 21);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 2, 22);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 3, 23);
+        execute("INSERT INTO %s (pk, c1, c2, v) values (?, ?, ?, ?)", 1, 1, 4, 24);
+
+        // NOT IN values outside of slice bounds
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 2), tuple(2, 5)),
+                   row(1, 1, 1, 21),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 2), tuple(1, 1)),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+
+        // Empty result set
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(1, 2), tuple(1, 3), tuple(1, 4)));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(0, 3), tuple(0, 2), tuple(0, 1)));
+
+        // NOT IN values inside slice bounds
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 2), tuple(0, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) <= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 2), tuple(0, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 1), tuple(1, 3)),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 1), tuple(1, 3)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 4, 24));
+
+        // One NOT IN value exactly the same as the slice bound
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(1, 2), tuple(0, 2), tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) <= ? AND (c1, c2) NOT IN (?, ?)",
+                           1, tuple(1, 2), tuple(0, 2), tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 1), tuple(1, 1)),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(1, 1), tuple(1, 1)),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+
+
+        // NOT IN with both upper and lower bound
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 2), tuple(0, 4)),
+                   row(1, 1, 1, 21),
+                   row(1, 0, 3, 13));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 2), tuple(0, 4)),
+                   row(1, 1, 1, 21),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND (c1, c2) <= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 2), tuple(0, 4)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 0, 3, 13));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) <= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 2), tuple(0, 4)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13));
+
+        // Mixed multi-column NOT IN with single column slice restriction:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 > ? AND (c1, c2) NOT IN (?)",
+                           1, 0, tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 >= ? AND (c1, c2) NOT IN (?)",
+                           1, 1, tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 < ? AND (c1, c2) NOT IN (?)",
+                           1, 1, tuple(0, 4)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 <= ? AND (c1, c2) NOT IN (?)",
+                           1, 0, tuple(0, 4)),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 >= ? AND c1 <= ? AND (c1, c2) NOT IN (?)",
+                           1, 0, 1, tuple(0, 4)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13));
+
+        // Mixed single-column NOT IN with multi-column slice restriction:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND c1 NOT IN (?)",
+                           1, tuple(0, 1), 1),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND c1 NOT IN (?)",
+                           1, tuple(0, 1), 1),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND c1 NOT IN (?)",
+                           1, tuple(1, 3), 1),
+                   row(1, 0, 1, 11),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 3, 13),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) < ? AND c1 NOT IN (?)",
+                           1, tuple(1, 3), 0),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) <= ? AND c1 NOT IN (?)",
+                           1, tuple(1, 3), 0),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) >= ? AND (c1, c2) <= ? AND c1 NOT IN (?)",
+                           1, tuple(0, 2), tuple(1, 3), 0),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 2, 22),
+                   row(1, 1, 3, 23));
+
+        // Mixed single-column and multi column slices with multi column NOT IN:
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND c1 < ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 1), 1, tuple(0, 3)),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND (c1, c2) > ? AND c1 <= ? AND (c1, c2) NOT IN (?)",
+                           1, tuple(0, 1), 0, tuple(0, 3)),
+                   row(1, 0, 2, 12),
+                   row(1, 0, 4, 14));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 > ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, 0, tuple(1, 4), tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 3, 23));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 >= ? AND (c1, c2) < ? AND (c1, c2) NOT IN (?)",
+                           1, 1, tuple(1, 4), tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 3, 23));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 > ? AND c1 < ? AND (c1, c2) NOT IN (?)",
+                           1, 0, 2, tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+        assertRows(execute("SELECT * FROM %s WHERE pk = ? AND c1 >= ? AND c1 <= ? AND (c1, c2) NOT IN (?)",
+                           1, 1, 1, tuple(1, 2)),
+                   row(1, 1, 1, 21),
+                   row(1, 1, 3, 23),
+                   row(1, 1, 4, 24));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOffsetTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOffsetTest.java
new file mode 100644
index 000000000000..86b4604906fc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOffsetTest.java
@@ -0,0 +1,485 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.validation.operations;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.math.IntMath;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.SyntaxException;
+import org.assertj.core.api.Assertions;
+
+/**
+ * Tests for {@code SELECT} queries with {@code LIMIT} and {@code OFFSET}.
+ */
+public class SelectOffsetTest extends CQLTester
+{
+    public static final Object[][] EMPTY_ROWS = new Object[0][];
+
+    private static final Logger logger = LoggerFactory.getLogger(SelectOffsetTest.class);
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        requireNetwork();
+
+        // disable offset guardrails for this test
+        DatabaseDescriptor.getGuardrailsConfig().setOffsetRowsThreshold(-1, -1);
+    }
+
+    @Test
+    public void testParseAndValidate() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c))");
+
+        // with LIMIT
+        execute("SELECT * FROM %s LIMIT 4 OFFSET 0");
+        execute("SELECT * FROM %s LIMIT 4 OFFSET 1");
+        assertRejectsNegativeOffset("SELECT * FROM %s LIMIT 4 OFFSET -1");
+        assertRejectsOffsetWithoutLimit("SELECT * FROM %s OFFSET 1");
+
+        // with PER PARTITION LIMIT
+        execute("SELECT * FROM %s PER PARTITION LIMIT 2 LIMIT 10 OFFSET 0");
+        execute("SELECT * FROM %s PER PARTITION LIMIT 2 LIMIT 10 OFFSET 1");
+        assertRejectsNegativeOffset("SELECT * FROM %s PER PARTITION LIMIT 2 LIMIT 10 OFFSET -1");
+        assertRejectsOffsetWithoutLimit("SELECT * FROM %s PER PARTITION LIMIT 2 OFFSET 1");
+
+        // with ALLOW FILTERING
+        execute("SELECT * FROM %s WHERE v=0 LIMIT 10 OFFSET 0 ALLOW FILTERING");
+        execute("SELECT * FROM %s WHERE v=0 LIMIT 10 OFFSET 1 ALLOW FILTERING");
+        assertRejectsNegativeOffset("SELECT * FROM %s WHERE v=0 LIMIT 10 OFFSET -1 ALLOW FILTERING");
+        assertRejectsOffsetWithoutLimit("SELECT * FROM %s WHERE v=0 OFFSET 1 ALLOW FILTERING");
+    }
+
+    private void assertRejectsNegativeOffset(String query) throws Throwable
+    {
+        assertInvalidThrowMessage("Offset must be positive",
+                                  InvalidRequestException.class,
+                                  query);
+    }
+
+    private void assertRejectsOffsetWithoutLimit(String query) throws Throwable
+    {
+        assertInvalidThrowMessage("[OFFSET]",
+                                  SyntaxException.class,
+                                  query);
+    }
+
+    @Test
+    public void testSkinnyTable() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)");
+
+        // test with empty table
+        testLimitAndOffset("SELECT * FROM %s");
+        testLimitAndOffset("SELECT * FROM %s WHERE k=2");
+        testLimitAndOffset("SELECT * FROM %s WHERE v=30");
+        testLimitAndOffset("SELECT k, v, sum(v) FROM %s", row(null, null, 0));
+
+        // write some data
+        execute("INSERT INTO %s (k, v) VALUES (1, 10)");
+        execute("INSERT INTO %s (k, v) VALUES (2, 20)");
+        execute("INSERT INTO %s (k, v) VALUES (3, 30)");
+        execute("INSERT INTO %s (k, v) VALUES (4, 40)");
+
+        testLimitAndOffset("SELECT * FROM %s", row(1, 10), row(2, 20), row(4, 40), row(3, 30));
+        testLimitAndOffset("SELECT * FROM %s WHERE k=2", row(2, 20));
+        testLimitAndOffset("SELECT * FROM %s WHERE k<2", row(1, 10));
+        testLimitAndOffset("SELECT * FROM %s WHERE k>2", row(4, 40), row(3, 30));
+        testLimitAndOffset("SELECT * FROM %s WHERE v=30", row(3, 30));
+        testLimitAndOffset("SELECT * FROM %s WHERE v<30", row(1, 10), row(2, 20));
+        testLimitAndOffset("SELECT * FROM %s WHERE v>30", row(4, 40));
+        testLimitAndOffset("SELECT k, v, sum(v) FROM %s", row(1, 10, 100));
+    }
+
+    @Test
+    public void testWideTable() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c1 int, c2 int, v int, PRIMARY KEY (k, c1, c2))");
+
+        // test with empty table
+        testLimitAndOffset("SELECT * FROM %s");
+        testLimitAndOffset("SELECT * FROM %s PER PARTITION LIMIT 3");
+        testLimitAndOffset("SELECT * FROM %s GROUP BY k, c1");
+        testLimitAndOffset("SELECT k, c1, c2, sum(v) FROM %s GROUP BY k, c1");
+        testLimitAndOffset("SELECT k, c1, c2, sum(v) FROM %s", row(null, null, null, 0));
+
+        // write some data
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (0, 0, 0, 0)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (0, 0, 1, 1)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (0, 0, 2, 2)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (0, 1, 0, 3)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (0, 1, 1, 4)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (0, 1, 2, 5)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (1, 0, 0, 6)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (1, 0, 1, 7)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (1, 0, 2, 8)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (1, 1, 0, 9)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (1, 1, 1, 10)");
+        execute("INSERT INTO %s (k, c1, c2, v) VALUES (1, 1, 2, 11)");
+
+        testLimitAndOffset("SELECT * FROM %s",
+                           row(1, 0, 0, 6),
+                           row(1, 0, 1, 7),
+                           row(1, 0, 2, 8),
+                           row(1, 1, 0, 9),
+                           row(1, 1, 1, 10),
+                           row(1, 1, 2, 11),
+                           row(0, 0, 0, 0),
+                           row(0, 0, 1, 1),
+                           row(0, 0, 2, 2),
+                           row(0, 1, 0, 3),
+                           row(0, 1, 1, 4),
+                           row(0, 1, 2, 5));
+
+        // With filtering restrictions
+        testLimitAndOffset("SELECT * FROM %s WHERE k=0",
+                           row(0, 0, 0, 0),
+                           row(0, 0, 1, 1),
+                           row(0, 0, 2, 2),
+                           row(0, 1, 0, 3),
+                           row(0, 1, 1, 4),
+                           row(0, 1, 2, 5));
+        testLimitAndOffset("SELECT * FROM %s WHERE k=0 AND c1=1",
+                           row(0, 1, 0, 3),
+                           row(0, 1, 1, 4),
+                           row(0, 1, 2, 5));
+        testLimitAndOffset("SELECT * FROM %s WHERE v>2 AND v<8",
+                           row(1, 0, 0, 6),
+                           row(1, 0, 1, 7),
+                           row(0, 1, 0, 3),
+                           row(0, 1, 1, 4),
+                           row(0, 1, 2, 5));
+        testLimitAndOffset("SELECT * FROM %s WHERE v<=2 OR v>=8",
+                           row(1, 0, 2, 8),
+                           row(1, 1, 0, 9),
+                           row(1, 1, 1, 10),
+                           row(1, 1, 2, 11),
+                           row(0, 0, 0, 0),
+                           row(0, 0, 1, 1),
+                           row(0, 0, 2, 2));
+
+        // With PER PARTITION LIMIT
+        testLimitAndOffset("SELECT * FROM %s PER PARTITION LIMIT 3",
+                           row(1, 0, 0, 6),
+                           row(1, 0, 1, 7),
+                           row(1, 0, 2, 8),
+                           row(0, 0, 0, 0),
+                           row(0, 0, 1, 1),
+                           row(0, 0, 2, 2));
+        testLimitAndOffset("SELECT * FROM %s PER PARTITION LIMIT 1",
+                           row(1, 0, 0, 6),
+                           row(0, 0, 0, 0));
+
+        // With aggregation
+        testLimitAndOffset("SELECT k, c1, c2, sum(v) FROM %s", row(1, 0, 0, 66));
+        testLimitAndOffset("SELECT count(*) FROM %s", row(12L));
+
+        // With GROUP BY
+        testLimitAndOffset("SELECT * FROM %s GROUP BY k, c1",
+                           row(1, 0, 0, 6),
+                           row(1, 1, 0, 9),
+                           row(0, 0, 0, 0),
+                           row(0, 1, 0, 3));
+        testLimitAndOffset("SELECT k, c1, c2, sum(v) FROM %s GROUP BY k, c1",
+                           row(1, 0, 0, 21),
+                           row(1, 1, 0, 30),
+                           row(0, 0, 0, 3),
+                           row(0, 1, 0, 12));
+
+        // With ORDER BY
+        testLimitAndOffset("SELECT * FROM %s WHERE k = 0 ORDER BY c1 DESC",
+                           row(0, 1, 2, 5),
+                           row(0, 1, 1, 4),
+                           row(0, 1, 0, 3),
+                           row(0, 0, 2, 2),
+                           row(0, 0, 1, 1),
+                           row(0, 0, 0, 0));
+        testLimitAndOffset("SELECT * FROM %s WHERE k = 0 ORDER BY c1 DESC PER PARTITION LIMIT 4",
+                           row(0, 1, 2, 5),
+                           row(0, 1, 1, 4),
+                           row(0, 1, 0, 3),
+                           row(0, 0, 2, 2));
+        testLimitAndOffset("SELECT * FROM %s WHERE k = 0 ORDER BY c1 DESC PER PARTITION LIMIT 1",
+                           row(0, 1, 2, 5));
+
+        // With keys IN
+        testLimitAndOffset("SELECT * FROM %s WHERE k IN (1, 0)",
+                           row(0, 0, 0, 0),
+                           row(0, 0, 1, 1),
+                           row(0, 0, 2, 2),
+                           row(0, 1, 0, 3),
+                           row(0, 1, 1, 4),
+                           row(0, 1, 2, 5),
+                           row(1, 0, 0, 6),
+                           row(1, 0, 1, 7),
+                           row(1, 0, 2, 8),
+                           row(1, 1, 0, 9),
+                           row(1, 1, 1, 10),
+                           row(1, 1, 2, 11));
+        testLimitAndOffsetWithoutPaging("SELECT * FROM %s WHERE k IN (1, 0) ORDER BY c1, c2",
+                                        row(0, 0, 0, 0),
+                                        row(1, 0, 0, 6),
+                                        row(0, 0, 1, 1),
+                                        row(1, 0, 1, 7),
+                                        row(0, 0, 2, 2),
+                                        row(1, 0, 2, 8),
+                                        row(0, 1, 0, 3),
+                                        row(1, 1, 0, 9),
+                                        row(0, 1, 1, 4),
+                                        row(1, 1, 1, 10),
+                                        row(0, 1, 2, 5),
+                                        row(1, 1, 2, 11));
+        testLimitAndOffsetWithoutPaging("SELECT * FROM %s WHERE k IN (1, 0) ORDER BY c1 DESC, c2 DESC",
+                                        row(0, 1, 2, 5),
+                                        row(1, 1, 2, 11),
+                                        row(0, 1, 1, 4),
+                                        row(1, 1, 1, 10),
+                                        row(0, 1, 0, 3),
+                                        row(1, 1, 0, 9),
+                                        row(0, 0, 2, 2),
+                                        row(1, 0, 2, 8),
+                                        row(0, 0, 1, 1),
+                                        row(1, 0, 1, 7),
+                                        row(0, 0, 0, 0),
+                                        row(1, 0, 0, 6));
+    }
+
+    @Test
+    public void testWideTableWithStatic() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c int, v int, s int static, PRIMARY KEY (k, c))");
+
+        // test with empty table
+        testLimitAndOffset("SELECT * FROM %s");
+        testLimitAndOffset("SELECT * FROM %s PER PARTITION LIMIT 1");
+
+        // write some data
+        execute("INSERT INTO %s (k, s) VALUES (0, 1)");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 0, 0)");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 1)");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 2, 0)");
+        execute("INSERT INTO %s (k, s) VALUES (1, 0)");
+        execute("INSERT INTO %s (k, c, v) VALUES (1, 0, 1)");
+        execute("INSERT INTO %s (k, c, v) VALUES (1, 1, 0)");
+        execute("INSERT INTO %s (k, c, v) VALUES (1, 2, 1)");
+
+        testLimitAndOffset("SELECT * FROM %s",
+                           row(1, 0, 0, 1),
+                           row(1, 1, 0, 0),
+                           row(1, 2, 0, 1),
+                           row(0, 0, 1, 0),
+                           row(0, 1, 1, 1),
+                           row(0, 2, 1, 0));
+        testLimitAndOffset("SELECT k, s FROM %s",
+                           row(1, 0),
+                           row(1, 0),
+                           row(1, 0),
+                           row(0, 1),
+                           row(0, 1),
+                           row(0, 1));
+        testLimitAndOffset("SELECT s FROM %s",
+                           row(0),
+                           row(0),
+                           row(0),
+                           row(1),
+                           row(1),
+                           row(1));
+
+        testLimitAndOffset("SELECT * FROM %s PER PARTITION LIMIT 2",
+                           row(1, 0, 0, 1),
+                           row(1, 1, 0, 0),
+                           row(0, 0, 1, 0),
+                           row(0, 1, 1, 1));
+        testLimitAndOffset("SELECT * FROM %s PER PARTITION LIMIT 1",
+                           row(1, 0, 0, 1),
+                           row(0, 0, 1, 0));
+    }
+
+    @Test
+    public void testANN()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 1>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        execute("INSERT INTO %s (k, v) VALUES (1, [1])");
+        execute("INSERT INTO %s (k, v) VALUES (2, [4])");
+        execute("INSERT INTO %s (k, v) VALUES (3, [2])");
+        execute("INSERT INTO %s (k, v) VALUES (4, [3])");
+        execute("INSERT INTO %s (k, v) VALUES (5, [2])");
+
+        // limit without an explicit offset should work
+        assertRows(execute("SELECT * FROM %s ORDER BY v ANN OF [0] LIMIT 10"),
+                   row(1, vector(1f)),
+                   row(5, vector(2f)),
+                   row(3, vector(2f)),
+                   row(4, vector(3f)),
+                   row(2, vector(4f)));
+        assertRows(execute("SELECT * FROM %s ORDER BY v ANN OF [0] LIMIT 5"),
+                   row(1, vector(1f)),
+                   row(5, vector(2f)),
+                   row(3, vector(2f)),
+                   row(4, vector(3f)),
+                   row(2, vector(4f)));
+        assertRows(execute("SELECT * FROM %s ORDER BY v ANN OF [0] LIMIT 4"),
+                   row(1, vector(1f)),
+                   row(5, vector(2f)),
+                   row(3, vector(2f)),
+                   row(4, vector(3f)));
+        assertRows(execute("SELECT * FROM %s ORDER BY v ANN OF [0] LIMIT 3"),
+                   row(1, vector(1f)),
+                   row(5, vector(2f)),
+                   row(3, vector(2f)));
+        assertRows(execute("SELECT * FROM %s ORDER BY v ANN OF [0] LIMIT 2"),
+                   row(1, vector(1f)),
+                   row(3, vector(2f)));
+        assertRows(execute("SELECT * FROM %s ORDER BY v ANN OF [0] LIMIT 1"),
+                   row(1, vector(1f)));
+
+        // offset >= 0 is not allowed
+        for (int offset : Arrays.asList(0, 1, 2))
+        {
+            String query = "SELECT * FROM %s ORDER BY v ANN OF [0] LIMIT 10 OFFSET " + offset;
+            String error = String.format(SelectStatement.TOPK_OFFSET_ERROR, offset);
+            Assertions.assertThatThrownBy(() -> execute(query))
+                      .isInstanceOf(InvalidRequestException.class)
+                      .hasMessage(error);
+            Assertions.assertThatThrownBy(() -> executeNet(query))
+                      .isInstanceOf(InvalidQueryException.class)
+                      .hasMessage(error);
+        }
+    }
+
+    @SafeVarargs
+    protected static <T> Vector<T> vector(T... values)
+    {
+        return new Vector<>(values);
+    }
+
+    private void testLimitAndOffset(String select, Object[]... rows) throws Throwable
+    {
+        testLimitAndOffset(select, true, rows);
+    }
+
+    private void testLimitAndOffsetWithoutPaging(String select, Object[]... rows) throws Throwable
+    {
+        testLimitAndOffset(select, false, rows);
+    }
+
+    private void testLimitAndOffset(String select, boolean paging, Object[]... rows) throws Throwable
+    {
+        List<Integer> limits = IntStream.range(1, rows.length + 1).boxed().collect(Collectors.toList());
+        limits.add(Integer.MAX_VALUE);
+        limits.add(Integer.MAX_VALUE - 1);
+
+        List<Integer> offsets = IntStream.range(0, rows.length).boxed().collect(Collectors.toList());
+        offsets.add(Integer.MAX_VALUE);
+        offsets.add(Integer.MAX_VALUE - 1);
+        offsets.add(null);
+
+        for (int limit : limits)
+        {
+            for (Integer offset : offsets)
+            {
+                // skip offset without limit, which is forbidden and tested in testParseAndValidate
+                if (limit == Integer.MAX_VALUE && offset != null)
+                    continue;
+
+                testLimitAndOffset(select, limit, offset, paging, rows);
+            }
+        }
+    }
+
+    private void testLimitAndOffset(String query, int limit, @Nullable Integer offset, boolean paging, Object[]... rows) throws Throwable
+    {
+        // test without a limit (nor offset)
+        assertRows(execute(query + " ALLOW FILTERING"), rows);
+
+        // append the specified limit and offset to the unrestricted query
+        StringBuilder sb = new StringBuilder(query);
+        sb.append(" LIMIT ").append(limit);
+        if (offset != null)
+            sb.append(" OFFSET ").append(offset);
+        sb.append(" ALLOW FILTERING");
+        String queryWithLimitAndOffset = sb.toString();
+
+        // trim the unrestricted query results according to the specified limit and offset
+        rows = trimRows(limit, offset, rows);
+
+        // test without paging
+        logger.debug("Executing test query without paging: {}", query);
+        assertRows(execute(queryWithLimitAndOffset), rows);
+
+        // test with paging (not all queries support it)
+        if (paging)
+        {
+            int numRows = rows.length;
+            for (int pageSize : ImmutableSet.of(Integer.MAX_VALUE, numRows + 1, numRows, numRows - 1, 1))
+            {
+                logger.debug("Executing test query with page size {}: {}", pageSize, query);
+                ResultSet rs = executeNetWithPaging(queryWithLimitAndOffset, pageSize);
+
+                // key-based paging should be disabled when limit/offset paging is used
+                if (offset != null)
+                {
+                    Assert.assertTrue(rs.isFullyFetched());
+                    Assert.assertNull(rs.getExecutionInfo().getPagingState());
+                }
+
+                assertRowsNet(rs, rows);
+            }
+        }
+
+        // test with bind markers
+        sb = new StringBuilder(query);
+        sb.append(" LIMIT ?");
+        if (offset != null)
+            sb.append(" OFFSET ?");
+        sb.append(" ALLOW FILTERING");
+        String queryWithBindMarkers = sb.toString();
+        assertRows(offset == null
+                   ? execute(queryWithBindMarkers, limit)
+                   : execute(queryWithBindMarkers, limit, offset),
+                   rows);
+    }
+
+    @SuppressWarnings("UnstableApiUsage")
+    private static Object[][] trimRows(Integer limit, @Nullable Integer offset, Object[]... rows)
+    {
+        offset = offset == null ? 0 : offset;
+        if (offset >= rows.length)
+            return EMPTY_ROWS;
+
+        int fetchLimit = Math.min(IntMath.saturatedAdd(limit, offset), rows.length);
+
+        return Arrays.copyOfRange(rows, offset, fetchLimit);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
index 0cbe221bebe5..bb3e3c06db2b 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
@@ -653,13 +653,13 @@ public void testAllowSkippingEqualityAndSingleValueInRestrictedClusteringColumns
         execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 1, 4);
         execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 2, 5);
 
-        assertInvalidMessage("Order by is currently only supported on the clustered columns of the PRIMARY KEY, got d",
+        assertInvalidMessage("Ordering on non-clustering column d requires the column to be indexed",
                              "SELECT * FROM %s WHERE a=? ORDER BY d DESC", 0);
 
-        assertInvalidMessage("Order by is currently only supported on the clustered columns of the PRIMARY KEY, got d",
+        assertInvalidMessage("Cannot combine clustering column ordering with non-clustering column ordering",
                              "SELECT * FROM %s WHERE a=? ORDER BY b ASC, c ASC, d ASC", 0);
 
-        String errorMsg = "Order by currently only supports the ordering of columns following their declared order in the PRIMARY KEY";
+        String errorMsg = "Ordering by clustered columns must follow the declared order in the PRIMARY KEY";
 
         assertRows(execute("SELECT * FROM %s WHERE a=? AND b=? ORDER BY c", 0, 0),
                    row(0, 0, 0, 0),
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
index d2546819e517..90924cbf585a 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
@@ -30,7 +30,7 @@
 public class SelectSingleColumnRelationTest extends CQLTester
 {
     @Test
-    public void testInvalidCollectionEqualityRelation() throws Throwable
+    public void testInvalidCollectionEqualityRelation()
     {
         createTable("CREATE TABLE %s (a int PRIMARY KEY, b set<int>, c list<int>, d map<int, int>)");
         createIndex("CREATE INDEX ON %s (b)");
@@ -46,7 +46,7 @@ public void testInvalidCollectionEqualityRelation() throws Throwable
     }
 
     @Test
-    public void testInvalidCollectionNonEQRelation() throws Throwable
+    public void testInvalidCollectionNonEQRelation()
     {
         createTable("CREATE TABLE %s (a int PRIMARY KEY, b set<int>, c int)");
         createIndex("CREATE INDEX ON %s (c)");
@@ -63,7 +63,9 @@ public void testInvalidCollectionNonEQRelation() throws Throwable
                              "SELECT * FROM %s WHERE c = 0 AND b <= ?", set(0));
         assertInvalidMessage("Collection column 'b' (set<int>) cannot be restricted by a 'IN' relation",
                              "SELECT * FROM %s WHERE c = 0 AND b IN (?)", set(0));
-        assertInvalidMessage("Unsupported \"!=\" relation: b != 5",
+        assertInvalidMessage("Collection column 'b' (set<int>) cannot be restricted by a 'NOT IN' relation",
+                             "SELECT * FROM %s WHERE c = 0 AND b NOT IN (?)", set(0));
+        assertInvalidMessage("Collection column 'b' (set<int>) cannot be restricted by a '!=' relation",
                 "SELECT * FROM %s WHERE c = 0 AND b != 5");
         assertInvalidMessage("Unsupported restriction: b IS NOT NULL",
                 "SELECT * FROM %s WHERE c = 0 AND b IS NOT NULL");
@@ -159,7 +161,7 @@ public void testClusteringColumnRelations() throws Throwable
     }
 
     @Test
-    public void testPartitionKeyColumnRelations() throws Throwable
+    public void testPartitionKeyColumnRelations()
     {
         createTable("CREATE TABLE %s (a text, b int, c int, d int, primary key((a, b), c))");
         execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 1, 1, 1);
@@ -205,7 +207,7 @@ public void testPartitionKeyColumnRelations() throws Throwable
     }
 
     @Test
-    public void testClusteringColumnRelationsWithClusteringOrder() throws Throwable
+    public void testClusteringColumnRelationsWithClusteringOrder()
     {
         createTable("CREATE TABLE %s (a text, b int, c int, d int, primary key(a, b, c)) WITH CLUSTERING ORDER BY (b DESC, c ASC);");
         execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "first", 1, 5, 1);
@@ -225,7 +227,7 @@ public void testClusteringColumnRelationsWithClusteringOrder() throws Throwable
     }
 
     @Test
-    public void testAllowFilteringWithClusteringColumn() throws Throwable
+    public void testAllowFilteringWithClusteringColumn()
     {
         createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c))");
 
@@ -264,7 +266,7 @@ public void testAllowFilteringWithClusteringColumn() throws Throwable
     }
 
     @Test
-    public void testAllowFilteringWithIndexedColumn() throws Throwable
+    public void testAllowFilteringWithIndexedColumn()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, a int, b int)");
         createIndex("CREATE INDEX ON %s(a)");
@@ -285,7 +287,7 @@ public void testAllowFilteringWithIndexedColumn() throws Throwable
     }
 
     @Test
-    public void testAllowFilteringWithIndexedColumnAndStaticColumns() throws Throwable
+    public void testAllowFilteringWithIndexedColumnAndStaticColumns()
     {
         createTable("CREATE TABLE %s (a int, b int, c int, s int static, PRIMARY KEY(a, b))");
         createIndex("CREATE INDEX ON %s(c)");
@@ -304,7 +306,7 @@ public void testAllowFilteringWithIndexedColumnAndStaticColumns() throws Throwab
     }
 
     @Test
-    public void testIndexQueriesOnComplexPrimaryKey() throws Throwable
+    public void testIndexQueriesOnComplexPrimaryKey()
     {
         createTable("CREATE TABLE %s (pk0 int, pk1 int, ck0 int, ck1 int, ck2 int, value int, PRIMARY KEY ((pk0, pk1), ck0, ck1, ck2))");
 
@@ -327,7 +329,7 @@ public void testIndexQueriesOnComplexPrimaryKey() throws Throwable
     }
 
     @Test
-    public void testIndexOnClusteringColumns() throws Throwable
+    public void testIndexOnClusteringColumns()
     {
         createTable("CREATE TABLE %s (id1 int, id2 int, author text, time bigint, v1 text, v2 text, PRIMARY KEY ((id1, id2), author, time))");
         createIndex("CREATE INDEX ON %s(time)");
@@ -357,6 +359,8 @@ public void testIndexOnClusteringColumns() throws Throwable
                             "SELECT v1 FROM %s WHERE time IN (1, 2)");
         assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                              "SELECT v1 FROM %s WHERE id2 IN (0, 2)");
+        assertRows(execute("SELECT v1 FROM %s WHERE id2 = 0 and time IN (1, 2) ALLOW FILTERING"),
+                   row("B"));
 
         // Checks that the IN queries works with filtering
         assertRows(execute("SELECT v1 FROM %s WHERE time IN (1, 2) ALLOW FILTERING"), row("B"), row("C"), row("E"));
@@ -397,7 +401,7 @@ public void testCompositeIndexWithPrimaryKey() throws Throwable
     }
 
     @Test
-    public void testRangeQueryOnIndex() throws Throwable
+    public void testRangeQueryOnIndex()
     {
         createTable("CREATE TABLE %s (id int primary key, row int, setid int);");
         createIndex("CREATE INDEX ON %s (setid)");
@@ -427,7 +431,7 @@ public void testEmptyIN() throws Throwable
     }
 
     @Test
-    public void testINWithDuplicateValue() throws Throwable
+    public void testINWithDuplicateValue()
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY (k1, k2))");
         execute("INSERT INTO %s (k1,  k2, v) VALUES (?, ?, ?)", 1, 1, 1);
@@ -443,7 +447,7 @@ public void testINWithDuplicateValue() throws Throwable
     }
 
     @Test
-    public void testLargeClusteringINValues() throws Throwable
+    public void testLargeClusteringINValues()
     {
         createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c))");
         execute("INSERT INTO %s (k, c, v) VALUES (0, 0, 0)");
@@ -455,7 +459,7 @@ public void testLargeClusteringINValues() throws Throwable
     }
 
     @Test
-    public void testMultiplePartitionKeyWithIndex() throws Throwable
+    public void testMultiplePartitionKeyWithIndex()
     {
         Util.assumeLegacySecondaryIndex(); // SAI does not allow multi-column slice restrictions
 
@@ -495,18 +499,18 @@ public void testMultiplePartitionKeyWithIndex() throws Throwable
                 row(0, 0, 1, 1, 1, 5),
                 row(0, 0, 2, 0, 0, 5));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'c'),
                              "SELECT * FROM %s WHERE a = ? AND c IN (?, ?) AND f = ?", 0, 0, 1, 5);
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND c IN (?, ?) AND f = ? ALLOW FILTERING", 0, 1, 3, 5),
                    row(0, 0, 1, 1, 1, 5));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'c'),
                              "SELECT * FROM %s WHERE a = ? AND c IN (?, ?) AND f = ?", 0, 1, 2, 5);
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND c IN (?, ?) AND f = ? ALLOW FILTERING", 0, 1, 2, 5),
                    row(0, 0, 1, 1, 1, 5),
                    row(0, 0, 2, 0, 0, 5));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'c'),
                              "SELECT * FROM %s WHERE a = ? AND c IN (?, ?) AND d IN (?) AND f = ?", 0, 1, 3, 0, 3);
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND c IN (?, ?) AND d IN (?) AND f = ? ALLOW FILTERING", 0, 1, 3, 0, 3),
                    row(0, 0, 1, 0, 0, 3));
@@ -517,7 +521,7 @@ public void testMultiplePartitionKeyWithIndex() throws Throwable
                 row(0, 0, 1, 1, 1, 5),
                 row(0, 0, 2, 0, 0, 5));
 
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'c'),
                              "SELECT * FROM %s WHERE a = ? AND c >= ? AND f = ?", 0, 1, 5);
         assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = ? AND c >= ? AND f = ?", 0, 0, 1, 5),
                    row(0, 0, 1, 1, 1, 5),
@@ -538,7 +542,7 @@ public void testMultiplePartitionKeyWithIndex() throws Throwable
     }
 
     @Test
-    public void testFunctionCallWithUnset() throws Throwable
+    public void testFunctionCallWithUnset()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, s text, i int)");
 
@@ -549,7 +553,7 @@ public void testFunctionCallWithUnset() throws Throwable
     }
 
     @Test
-    public void testLimitWithUnset() throws Throwable
+    public void testLimitWithUnset()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, i int)");
         execute("INSERT INTO %s (k, i) VALUES (1, 1)");
@@ -561,7 +565,7 @@ public void testLimitWithUnset() throws Throwable
     }
 
     @Test
-    public void testWithUnsetValues() throws Throwable
+    public void testWithUnsetValues()
     {
         createTable("CREATE TABLE %s (k int, i int, j int, s text, PRIMARY KEY(k,i,j))");
         createIndex("CREATE INDEX s_index ON %s (s)");
@@ -570,11 +574,17 @@ public void testWithUnsetValues() throws Throwable
         assertInvalidMessage("Invalid unset value for column k", "SELECT * from %s WHERE k IN ?", unset());
         assertInvalidMessage("Invalid unset value for column k", "SELECT * from %s WHERE k IN(?)", unset());
         assertInvalidMessage("Invalid unset value for column k", "SELECT * from %s WHERE k IN(?,?)", 1, unset());
+        assertInvalidMessage("Invalid unset value for column k", "SELECT * from %s WHERE k NOT IN ? ALLOW FILTERING", unset());
+        assertInvalidMessage("Unsupported unset value for column k", "SELECT * from %s WHERE k NOT IN(?) ALLOW FILTERING", unset());
+        assertInvalidMessage("Unsupported unset value for column k", "SELECT * from %s WHERE k NOT IN(?,?) ALLOW FILTERING", 1, unset());
         // clustering column
         assertInvalidMessage("Invalid unset value for column i", "SELECT * from %s WHERE k = 1 AND i = ?", unset());
         assertInvalidMessage("Invalid unset value for column i", "SELECT * from %s WHERE k = 1 AND i IN ?", unset());
         assertInvalidMessage("Invalid unset value for column i", "SELECT * from %s WHERE k = 1 AND i IN(?)", unset());
         assertInvalidMessage("Invalid unset value for column i", "SELECT * from %s WHERE k = 1 AND i IN(?,?)", 1, unset());
+        assertInvalidMessage("Invalid unset value for column i", "SELECT * from %s WHERE k = 1 AND i NOT IN ?", unset());
+        assertInvalidMessage("Invalid unset value for column i", "SELECT * from %s WHERE k = 1 AND i NOT IN(?)", unset());
+        assertInvalidMessage("Invalid unset value for column i", "SELECT * from %s WHERE k = 1 AND i NOT IN(?,?)", 1, unset());
         assertInvalidMessage("Invalid unset value for column i", "SELECT * from %s WHERE i = ? ALLOW FILTERING", unset());
         // indexed column
         assertInvalidMessage("Unsupported unset value for column s", "SELECT * from %s WHERE s = ?", unset());
@@ -583,7 +593,7 @@ public void testWithUnsetValues() throws Throwable
     }
 
     @Test
-    public void testInvalidSliceRestrictionOnPartitionKey() throws Throwable
+    public void testInvalidSliceRestrictionOnPartitionKey()
     {
         createTable("CREATE TABLE %s (a int PRIMARY KEY, b int, c text)");
         assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
@@ -593,7 +603,7 @@ public void testInvalidSliceRestrictionOnPartitionKey() throws Throwable
     }
 
     @Test
-    public void testInvalidMulticolumnSliceRestrictionOnPartitionKey() throws Throwable
+    public void testInvalidMulticolumnSliceRestrictionOnPartitionKey()
     {
         createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY ((a, b)))");
         assertInvalidMessage("Multi-column relations can only be applied to clustering columns but was applied to: a",
@@ -611,7 +621,7 @@ public void testInvalidMulticolumnSliceRestrictionOnPartitionKey() throws Throwa
     }
 
     @Test
-    public void testInvalidColumnNames() throws Throwable
+    public void testInvalidColumnNames()
     {
         createTable("CREATE TABLE %s (a int, b int, c map<int, int>, PRIMARY KEY (a, b))");
         assertInvalidMessage("Undefined column name d", "SELECT * FROM %s WHERE d = 0");
@@ -619,19 +629,24 @@ public void testInvalidColumnNames() throws Throwable
         assertInvalidMessage("Undefined column name d", "SELECT * FROM %s WHERE d > 0 and d <= 2");
         assertInvalidMessage("Undefined column name d", "SELECT * FROM %s WHERE d CONTAINS 0");
         assertInvalidMessage("Undefined column name d", "SELECT * FROM %s WHERE d CONTAINS KEY 0");
+        assertInvalidMessage("Undefined column name d", "SELECT * FROM %s WHERE d NOT CONTAINS 0");
+        assertInvalidMessage("Undefined column name d", "SELECT * FROM %s WHERE d NOT CONTAINS KEY 0");
         assertInvalidMessage("Undefined column name d", "SELECT a AS d FROM %s WHERE d = 0");
         assertInvalidMessage("Undefined column name d", "SELECT b AS d FROM %s WHERE d IN (0, 1)");
+        assertInvalidMessage("Undefined column name d", "SELECT b AS d FROM %s WHERE d NOT IN (0, 1)");
         assertInvalidMessage("Undefined column name d", "SELECT b AS d FROM %s WHERE d > 0 and d <= 2");
         assertInvalidMessage("Undefined column name d", "SELECT c AS d FROM %s WHERE d CONTAINS 0");
         assertInvalidMessage("Undefined column name d", "SELECT c AS d FROM %s WHERE d CONTAINS KEY 0");
+        assertInvalidMessage("Undefined column name d", "SELECT c AS d FROM %s WHERE d NOT CONTAINS 0");
+        assertInvalidMessage("Undefined column name d", "SELECT c AS d FROM %s WHERE d NOT CONTAINS KEY 0");
         assertInvalidMessage("Undefined column name d", "SELECT d FROM %s WHERE a = 0");
     }
 
     @Test
-    public void testInvalidNonFrozenUDTRelation() throws Throwable
+    public void testInvalidNonFrozenUDTRelation()
     {
         String type = createType("CREATE TYPE %s (a int)");
-        createTable("CREATE TABLE %s (a int PRIMARY KEY, b " + type + ")");
+        createTable("CREATE TABLE %s (a int PRIMARY KEY, b " + type + ')');
         Object udt = userType("a", 1);
 
         // All operators
@@ -642,17 +657,19 @@ public void testInvalidNonFrozenUDTRelation() throws Throwable
         assertInvalidMessage(msg, "SELECT * FROM %s WHERE b >= ?", udt);
         assertInvalidMessage(msg, "SELECT * FROM %s WHERE b <= ?", udt);
         assertInvalidMessage(msg, "SELECT * FROM %s WHERE b IN (?)", udt);
+        assertInvalidMessage(msg, "SELECT * FROM %s WHERE b NOT IN (?)", udt);
         assertInvalidMessage(msg, "SELECT * FROM %s WHERE b LIKE ?", udt);
-        assertInvalidMessage("Unsupported \"!=\" relation: b != {a: 0}",
-                             "SELECT * FROM %s WHERE b != {a: 0}", udt);
+        assertInvalidMessage(msg, "SELECT * FROM %s WHERE b != {a: 0}", udt);
         assertInvalidMessage("Unsupported restriction: b IS NOT NULL",
                              "SELECT * FROM %s WHERE b IS NOT NULL", udt);
         assertInvalidMessage("Cannot use CONTAINS on non-collection column b",
                              "SELECT * FROM %s WHERE b CONTAINS ?", udt);
+        assertInvalidMessage("Cannot use NOT CONTAINS on non-collection column b",
+                             "SELECT * FROM %s WHERE b NOT CONTAINS ?", udt);
     }
 
     @Test
-    public void testInRestrictionWithClusteringColumn() throws Throwable
+    public void testInRestrictionWithClusteringColumn()
     {
         createTable("CREATE TABLE %s (key int, c1 int, c2 int, s1 text static, PRIMARY KEY ((key, c1), c2))");
 
@@ -686,7 +703,7 @@ public void testInRestrictionWithClusteringColumn() throws Throwable
     }
 
     @Test
-    public void testInRestrictionsWithAllowFiltering() throws Throwable
+    public void testInRestrictionsWithAllowFiltering()
     {
         createTable("CREATE TABLE %s (pk1 int, pk2 int, c text, s int static, v int, primary key((pk1, pk2), c))");
         execute("INSERT INTO %s (pk1, pk2, c, s, v) values (?, ?, ?, ?, ?)", 1, 0, "5", 1, 3);
@@ -739,7 +756,7 @@ public void testInRestrictionsWithAllowFiltering() throws Throwable
     }
 
     @Test
-    public void testInRestrictionsWithAllowFilteringAndOrdering() throws Throwable
+    public void testInRestrictionsWithAllowFilteringAndOrdering()
     {
         createTable("CREATE TABLE %s (pk int, c text, v int, primary key(pk, c)) WITH CLUSTERING ORDER BY (c DESC)");
         execute("INSERT INTO %s (pk, c, v) values (?, ?, ?)", 1, "0", 5);
@@ -784,4 +801,528 @@ public void testInRestrictionsWithCountersAndAllowFiltering() throws Throwable
                    row(1, 1L),
                    row(3, 1L));
     }
+
+    @Test
+    public void testClusteringSlicesWithNotIn()
+    {
+        createTable("CREATE TABLE %s (a text, b int, c int, d int, primary key(a, b, c))");
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 1, 4, 1);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 2, 5, 2);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 2, 6, 3);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 2, 7, 4);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 3, 8, 5);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 3, 9, 6);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 4, 1, 7);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 4, 2, 8);
+
+        // restrict first clustering column by NOT IN
+        assertRows(execute("select * from %s where a = ? and b not in ?", "key", list(2, 4, 5)),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+        assertRows(execute("select * from %s where a = ? and b not in (?, ?, ?)", "key", 2, 4, 5),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+
+        // use different order of items in NOT IN list:
+        assertRows(execute("select * from %s where a = ? and b not in (?, ?, ?)", "key", 5, 2, 4),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+        assertRows(execute("select * from %s where a = ? and b not in (?, ?, ?)", "key", 5, 4, 2),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+
+        // restrict last clustering column by NOT IN
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in ?", "key", 2, list(5, 6)),
+                   row("key", 2, 7, 4));
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in (?, ?)", "key", 2, 5, 6),
+                   row("key", 2, 7, 4));
+
+        // empty NOT IN should have no effect:
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in ?", "key", 2, list()),
+                   row("key", 2, 5, 2),
+                   row("key", 2, 6, 3),
+                   row("key", 2, 7, 4));
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in ()", "key", 2),
+                   row("key", 2, 5, 2),
+                   row("key", 2, 6, 3),
+                   row("key", 2, 7, 4));
+
+        // NOT IN value that doesn't match any data should have no effect:
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in (?)", "key", 2, 0),
+                   row("key", 2, 5, 2),
+                   row("key", 2, 6, 3),
+                   row("key", 2, 7, 4));
+
+        // Duplicate NOT IN values:
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in (?, ?)", "key", 2, 5, 5),
+                   row("key", 2, 6, 3),
+                   row("key", 2, 7, 4));
+
+        // mix NOT IN and '<' and '<=' comparison on the same column
+        assertRows(execute("select * from %s where a = ? and b not in ? and b < ?", "key", list(2, 5), 1)); // empty
+        assertRows(execute("select * from %s where a = ? and b not in ? and b < ?", "key", list(2, 5), 3),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b <= ?", "key", list(2), 2),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b <= ?", "key", list(2), 3),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b <= ?", "key", list(2), 10),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+
+        // mix NOT IN and '>' and '>=' comparison on the same column
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ?", "key", list(2), 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ?", "key", list(2), 2),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ?", "key", list(2), 2),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ?", "key", list(2), 4),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ?", "key", list(2), 0),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+
+        // mix NOT IN and range slice
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ? and b < ?", "key", list(2), 1, 4),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ? and b < ?", "key", list(2), 1, 4),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ? and b <= ?", "key", list(2), 1, 4),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ? and b <= ?", "key", list(2), 1, 4),
+                   row("key", 1, 4, 1),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+
+        // Collision between a slice bound and NOT IN value:
+        assertRows(execute("select * from %s where a = ? and b not in ? and b < ?", "key", list(2), 2),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ?", "key", list(2), 2),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 4, 1, 7),
+                   row("key", 4, 2, 8));
+
+        // NOT IN value outside of the slice range:
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ? and b < ?", "key", list(0), 2, 4),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ? and b < ?", "key", list(10), 2, 4),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+
+        // multiple NOT IN on the same column, use different ways of passing a list
+        assertRows(execute("select * from %s where a = ? and b not in ? and b not in ?", "key", list(1, 2), list(4)),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+        assertRows(execute("select * from %s where a = ? and b not in (?, ?) and b not in (?)", "key", 1, 2, 4),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+        assertRows(execute("select * from %s where a = ? and b not in (?, ?) and b not in ?", "key", 1, 2, list(4)),
+                   row("key", 3, 8, 5),
+                   row("key", 3, 9, 6));
+
+        // mix IN and NOT IN
+        assertRows(execute("select * from %s where a = ? and b in ? and c not in ?", "key", list(2, 3), list(5, 6, 9)),
+                   row("key", 2, 7, 4),
+                   row("key", 3, 8, 5));
+
+    }
+
+    @Test
+    public void testClusteringSlicesWithNotInAndReverseOrdering()
+    {
+        createTable("CREATE TABLE %s (a text, b int, c int, d int, primary key(a, b, c)) with clustering order by (b desc, c desc)");
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 1, 4, 1);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 2, 5, 2);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 2, 6, 3);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 2, 7, 4);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 3, 8, 5);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 3, 9, 6);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 4, 1, 7);
+        execute("insert into %s (a, b, c, d) values (?, ?, ?, ?)", "key", 4, 2, 8);
+
+        // restrict first clustering column by NOT IN
+        assertRows(execute("select * from %s where a = ? and b not in ?", "key", list(2, 4, 5)),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in (?, ?, ?)", "key", 2, 4, 5),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5),
+                   row("key", 1, 4, 1));
+
+        // restrict last clustering column by NOT IN
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in ?", "key", 2, list(5, 6)),
+                   row("key", 2, 7, 4));
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in (?, ?)", "key", 2, 5, 6),
+                   row("key", 2, 7, 4));
+
+        // empty NOT IN should have no effect:
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in ?", "key", 2, list()),
+                   row("key", 2, 7, 4),
+                   row("key", 2, 6, 3),
+                   row("key", 2, 5, 2));
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in ()", "key", 2),
+                   row("key", 2, 7, 4),
+                   row("key", 2, 6, 3),
+                   row("key", 2, 5, 2));
+
+        // NOT IN value that doesn't match any data should have no effect:
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in (?)", "key", 2, 0),
+                   row("key", 2, 7, 4),
+                   row("key", 2, 6, 3),
+                   row("key", 2, 5, 2));
+
+        // Duplicate NOT IN values:
+        assertRows(execute("select * from %s where a = ? and b = ? and c not in (?, ?)", "key", 2, 5, 5),
+                   row("key", 2, 7, 4),
+                   row("key", 2, 6, 3));
+
+        // mix NOT IN and '<' and '<=' comparison on the same column
+        assertRows(execute("select * from %s where a = ? and b not in ? and b < ?", "key", list(2, 5), 1)); // empty
+        assertRows(execute("select * from %s where a = ? and b not in ? and b < ?", "key", list(2, 5), 3),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b <= ?", "key", list(2), 2),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b <= ?", "key", list(2), 3),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b <= ?", "key", list(2), 10),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5),
+                   row("key", 1, 4, 1));
+
+        // mix NOT IN and '>' and '>=' comparison on the same column
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ?", "key", list(2), 1),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ?", "key", list(2), 2),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ?", "key", list(2), 2),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ?", "key", list(2), 4),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ?", "key", list(2), 0),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5),
+                   row("key", 1, 4, 1));
+
+        // mix NOT IN and range slice
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ? and b < ?", "key", list(2), 1, 4),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ? and b < ?", "key", list(2), 1, 4),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ? and b <= ?", "key", list(2), 1, 4),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b >= ? and b <= ?", "key", list(2), 1, 4),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5),
+                   row("key", 1, 4, 1));
+
+        // Collision between a slice bound and NOT IN value:
+        assertRows(execute("select * from %s where a = ? and b not in ? and b < ?", "key", list(2), 2),
+                   row("key", 1, 4, 1));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ?", "key", list(2), 2),
+                   row("key", 4, 2, 8),
+                   row("key", 4, 1, 7),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+
+        // NOT IN value outside of the slice range:
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ? and b < ?", "key", list(0), 2, 4),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+        assertRows(execute("select * from %s where a = ? and b not in ? and b > ? and b < ?", "key", list(10), 2, 4),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+
+        // multiple NOT IN on the same column, use different ways of passing a list
+        assertRows(execute("select * from %s where a = ? and b not in ? and b not in ?", "key", list(1, 2), list(4)),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+        assertRows(execute("select * from %s where a = ? and b not in (?, ?) and b not in (?)", "key", 1, 2, 4),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+        assertRows(execute("select * from %s where a = ? and b not in (?, ?) and b not in ?", "key", 1, 2, list(4)),
+                   row("key", 3, 9, 6),
+                   row("key", 3, 8, 5));
+
+        // mix IN and NOT IN
+        assertRows(execute("select * from %s where a = ? and b in ? and c not in ?", "key", list(2, 3), list(5, 6, 9)),
+                   row("key", 3, 8, 5),
+                   row("key", 2, 7, 4));
+    }
+
+    @Test
+    public void testNotInRestrictionsWithAllowFiltering()
+    {
+        createTable("CREATE TABLE %s (pk int, c int, v int, primary key(pk, c))");
+        execute("insert into %s (pk, c, v) values (?, ?, ?)", 1, 1, 1);
+        execute("insert into %s (pk, c, v) values (?, ?, ?)", 1, 2, 2);
+        execute("insert into %s (pk, c, v) values (?, ?, ?)", 1, 3, 3);
+        execute("insert into %s (pk, c, v) values (?, ?, ?)", 1, 4, 4);
+        execute("insert into %s (pk, c, v) values (?, ?, ?)", 1, 5, 5);
+
+        // empty NOT IN set
+        assertRows(execute("select * from %s where pk = ? and v not in ? allow filtering", 1, list()),
+                   row(1, 1, 1),
+                   row(1, 2, 2),
+                   row(1, 3, 3),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+        assertRows(execute("select * from %s where pk = ? and v not in () allow filtering", 1),
+                   row(1, 1, 1),
+                   row(1, 2, 2),
+                   row(1, 3, 3),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+        // NOT IN with values that don't match any data
+        assertRows(execute("select * from %s where pk = ? and v not in (?, ?) allow filtering", 1, -6, 20),
+                   row(1, 1, 1),
+                   row(1, 2, 2),
+                   row(1, 3, 3),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+        // NOT IN that excludes a few values
+        assertRows(execute("select * from %s where pk = ? and v not in ? allow filtering", 1, list(2, 3)),
+                   row(1, 1, 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+        assertRows(execute("select * from %s where pk = ? and v not in (?, ?) allow filtering", 1, 2, 3),
+                   row(1, 1, 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+        // NOT IN with one-sided slice filters:
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v < ? allow filtering", 1, list(2, 3), 5),
+                   row(1, 1, 1),
+                   row(1, 4, 4));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v < ? allow filtering", 1, list(2, 3, 10), 5),
+                   row(1, 1, 1),
+                   row(1, 4, 4));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v <= ? allow filtering", 1, list(2, 3), 5),
+                   row(1, 1, 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v <= ? allow filtering", 1, list(2, 3, 10), 5),
+                   row(1, 1, 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v > ? allow filtering", 1, list(2, 3), 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v > ? allow filtering", 1, list(0, 2, 3), 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v >= ? allow filtering", 1, list(2, 3), 1),
+                   row(1, 1, 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v >= ? allow filtering", 1, list(0, 2, 3), 1),
+                   row(1, 1, 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+        // NOT IN with range filters:
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v > ? and v < ? allow filtering", 1, list(2, 3), 1, 4)); // empty
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v > ? and v < ? allow filtering", 1, list(2, 3), 1, 4)); // empty
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v > ? and v < ? allow filtering", 1, list(2, 3), 0, 5),
+                   row(1, 1, 1),
+                   row(1, 4, 4));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v >= ? and v < ? allow filtering", 1, list(2, 3), 1, 4),
+                   row(1, 1, 1));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v > ? and v <= ? allow filtering", 1, list(2, 3), 1, 4),
+                   row(1, 4, 4));
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v >= ? and v <= ? allow filtering", 1, list(2, 3), 1, 4),
+                   row(1, 1, 1),
+                   row(1, 4, 4));
+
+        // more than one NOT IN clause
+        assertRows(execute("select * from %s where pk = ? and v not in ? and v not in ? allow filtering", 1, list(2), list(3)),
+                   row(1, 1, 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+    }
+
+    @Test
+    public void testNotInRestrictionsWithOrAndAllowFiltering()
+    {
+        createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c))");
+        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", 1, 1, 1);
+        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", 1, 2, 2);
+        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", 1, 3, 3);
+        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", 1, 4, 4);
+        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", 1, 5, 5);
+
+        assertRows(execute("SELECT * FROM %s WHERE v = ? OR v not in ? ALLOW FILTERING", 0, list()),
+                   row(1, 1, 1),
+                   row(1, 2, 2),
+                   row(1, 3, 3),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+        assertRows(execute("SELECT * FROM %s WHERE v NOT IN ? OR v NOT IN ? ALLOW FILTERING", list(), list(1, 2)),
+                   row(1, 1, 1),
+                   row(1, 2, 2),
+                   row(1, 3, 3),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+        assertRows(execute("SELECT * FROM %s WHERE v = ? OR v NOT IN ? ALLOW FILTERING", 0, list(1, 2, 3)),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+        assertRows(execute("SELECT * FROM %s WHERE v = ? OR v NOT IN ? ALLOW FILTERING", 1, list(1, 2, 3)),
+                   row(1, 1, 1),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+        // Multiple NOT IN:
+        assertRows(execute("SELECT * FROM %s WHERE v NOT IN ? OR v NOT IN ? ALLOW FILTERING", list(1, 2), list(3, 4)),
+                   row(1, 1, 1),
+                   row(1, 2, 2),
+                   row(1, 3, 3),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+        assertRows(execute("SELECT * FROM %s WHERE v NOT IN ? OR v NOT IN ? ALLOW FILTERING", list(1, 2, 3, 4), list()),
+                   row(1, 1, 1),
+                   row(1, 2, 2),
+                   row(1, 3, 3),
+                   row(1, 4, 4),
+                   row(1, 5, 5));
+
+
+        // Mixed IN / NOT IN with AND and OR:
+        assertRows(execute("SELECT * FROM %s WHERE v IN ? OR v NOT IN ? AND v NOT IN ? ALLOW FILTERING", list(1), list(1, 2, 3), list(2, 3, 4)),
+                   row(1, 1, 1),
+                   row(1, 5, 5));
+
+        assertRows(execute("SELECT * FROM %s WHERE v NOT IN ? AND (v IN ? OR v NOT IN ?) ALLOW FILTERING", list(), list(3), list(5)),
+                   row(1, 1, 1),
+                   row(1, 2, 2),
+                   row(1, 3, 3),
+                   row(1, 4, 4));
+
+        assertRows(execute("SELECT * FROM %s WHERE v NOT IN ? AND (v IN ? OR v NOT IN ?) ALLOW FILTERING", list(1, 2), list(3), list(5)),
+                   row(1, 3, 3),
+                   row(1, 4, 4));
+
+        assertRows(execute("SELECT * FROM %s WHERE v IN ? AND (v IN ? OR v NOT IN ?) ALLOW FILTERING", list(1, 3), list(5), list(3)),
+                   row(1, 1, 1));
+
+        assertRows(execute("SELECT * FROM %s WHERE v IN ? AND (v IN ? OR v NOT IN ?) ALLOW FILTERING", list(1, 3), list(5), list()),
+                   row(1, 1, 1),
+                   row(1, 3, 3));
+    }
+
+
+    @Test
+    public void testNonEqualsRelationWithFiltering()
+    {
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 2, 2);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 3, 3);
+
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? ALLOW FILTERING", 0, 0),
+                   row(0, 1),
+                   row(0, 2),
+                   row(0, 3)
+        );
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? ALLOW FILTERING", 0, 1),
+                   row(0, 0),
+                   row(0, 2),
+                   row(0, 3)
+        );
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? ALLOW FILTERING", 0, -1),
+                   row(0, 0),
+                   row(0, 1),
+                   row(0, 2),
+                   row(0, 3)
+        );
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? ALLOW FILTERING", 0, 5),
+                   row(0, 0),
+                   row(0, 1),
+                   row(0, 2),
+                   row(0, 3)
+        );
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? AND c != ? ALLOW FILTERING", 0, 1, 2),
+                   row(0, 0),
+                   row(0, 3)
+        );
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? AND c < ? ALLOW FILTERING", 0, 1, 2),
+                   row(0, 0)
+        );
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? AND c <= ? ALLOW FILTERING", 0, 1, 2),
+                   row(0, 0),
+                   row(0, 2)
+        );
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? AND c > ? ALLOW FILTERING", 0, 2, 0),
+                   row(0, 1),
+                   row(0, 3)
+        );
+        assertRows(execute("SELECT a, b FROM %s WHERE a = ? AND c != ? AND c >= ? ALLOW FILTERING", 0, 2, 1),
+                   row(0, 1),
+                   row(0, 3)
+        );
+    }
+
 }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
index 639f6a514520..7e59facb9fa2 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
@@ -591,7 +591,7 @@ public void testContainsKeyAndContainsWithIndexOnMapKey() throws Throwable
         execute("INSERT INTO %s (account, id , categories) VALUES (?, ?, ?)", "test", 6, map("lmn", "foo2"));
 
         beforeAndAfterFlush(() -> {
-            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+            assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "categories"),
                                  "SELECT * FROM %s WHERE account = ? AND categories CONTAINS ?", "test", "foo");
 
             assertRows(execute("SELECT * FROM %s WHERE account = ? AND categories CONTAINS KEY ?", "test", "lmn"),
@@ -616,7 +616,7 @@ public void testContainsKeyAndContainsWithIndexOnMapValue() throws Throwable
         execute("INSERT INTO %s (account, id , categories) VALUES (?, ?, ?)", "test", 6, map("lmn2", "foo"));
 
         beforeAndAfterFlush(() -> {
-            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+            assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "categories"),
                                  "SELECT * FROM %s WHERE account = ? AND categories CONTAINS KEY ?", "test", "lmn");
 
             assertRows(execute("SELECT * FROM %s WHERE account = ? AND categories CONTAINS ?", "test", "foo"),
@@ -1381,6 +1381,9 @@ public void testFilteringWithoutIndicesWithCollections() throws Throwable
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE c CONTAINS 2");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE c NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
@@ -1388,10 +1391,23 @@ public void testFilteringWithoutIndicesWithCollections() throws Throwable
             assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS 3 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE c NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
+            assertRows(execute("SELECT * FROM %s WHERE c NOT CONTAINS 2 AND c NOT CONTAINS 3 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
+            assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c NOT CONTAINS 3 ALLOW FILTERING"),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
+
             // Checks filtering for sets
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE d CONTAINS 4");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE d NOT CONTAINS 4");
+
             assertRows(execute("SELECT * FROM %s WHERE d CONTAINS 4 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
@@ -1399,26 +1415,68 @@ public void testFilteringWithoutIndicesWithCollections() throws Throwable
             assertRows(execute("SELECT * FROM %s WHERE d CONTAINS 4 AND d CONTAINS 6 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE d NOT CONTAINS 4 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
+            assertRows(execute("SELECT * FROM %s WHERE d NOT CONTAINS 4 AND d NOT CONTAINS 6 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
+            assertRows(execute("SELECT * FROM %s WHERE d CONTAINS 4 AND d NOT CONTAINS 6 ALLOW FILTERING"),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
+
             // Checks filtering for maps
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE e CONTAINS 2");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE e NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE e CONTAINS 2 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE e NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE e CONTAINS KEY 1 ALLOW FILTERING"),
                        row(1, 2, list(1, 6), set(2, 12), map(1, 6)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE e NOT CONTAINS KEY 1 ALLOW FILTERING"),
+                       row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
+            assertRows(execute("SELECT * FROM %s WHERE e CONTAINS 2 AND e NOT CONTAINS KEY 1 ALLOW FILTERING"),
+                       row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
+
             assertRows(execute("SELECT * FROM %s WHERE e[1] = 6 ALLOW FILTERING"),
                        row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
 
+            assertRows(execute("SELECT * FROM %s WHERE e[1] != 6 ALLOW FILTERING"),
+                       row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
+            assertRows(execute("SELECT * FROM %s WHERE e[1] != 6 AND e[3] != 2 ALLOW FILTERING"),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
+            assertRows(execute("SELECT * FROM %s WHERE e CONTAINS KEY 1 AND e[1] != 6 ALLOW FILTERING"),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
+
             assertRows(execute("SELECT * FROM %s WHERE e CONTAINS KEY 1 AND e CONTAINS 2 ALLOW FILTERING"),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE e CONTAINS KEY 1 AND e NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND d CONTAINS 4 AND e CONTAINS KEY 3 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
+
+            assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND d CONTAINS 4 AND e NOT CONTAINS KEY 1 ALLOW FILTERING"),
+                       row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
         });
 
         // Checks filtering with null
@@ -1435,6 +1493,19 @@ public void testFilteringWithoutIndicesWithCollections() throws Throwable
         assertInvalidMessage("Unsupported null map value for column e",
                              "SELECT * FROM %s WHERE e[1] = null ALLOW FILTERING");
 
+        assertInvalidMessage("Unsupported null value for column c",
+                             "SELECT * FROM %s WHERE c NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column d",
+                             "SELECT * FROM %s WHERE d NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column e",
+                             "SELECT * FROM %s WHERE e NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column e",
+                             "SELECT * FROM %s WHERE e NOT CONTAINS KEY null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null map key for column e",
+                             "SELECT * FROM %s WHERE e[null] != 2 ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null map value for column e",
+                             "SELECT * FROM %s WHERE e[1] != null ALLOW FILTERING");
+
         // Checks filtering with unset
         assertInvalidMessage("Unsupported unset value for column c",
                              "SELECT * FROM %s WHERE c CONTAINS ? ALLOW FILTERING",
@@ -1454,6 +1525,26 @@ public void testFilteringWithoutIndicesWithCollections() throws Throwable
         assertInvalidMessage("Unsupported unset map value for column e",
                              "SELECT * FROM %s WHERE e[1] = ? ALLOW FILTERING",
                              unset());
+
+        assertInvalidMessage("Unsupported unset value for column c",
+                             "SELECT * FROM %s WHERE c NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column d",
+                             "SELECT * FROM %s WHERE d NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column e",
+                             "SELECT * FROM %s WHERE e NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column e",
+                             "SELECT * FROM %s WHERE e NOT CONTAINS KEY ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset map key for column e",
+                             "SELECT * FROM %s WHERE e[?] != 2 ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset map value for column e",
+                             "SELECT * FROM %s WHERE e[1] != ? ALLOW FILTERING",
+                             unset());
+
     }
 
     @Test
@@ -1489,13 +1580,23 @@ public void testFilteringWithoutIndicesWithFrozenCollections() throws Throwable
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE c CONTAINS 2");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE c NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE c NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND c CONTAINS 3 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE c NOT CONTAINS 2 AND c NOT CONTAINS 3 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
             // Checks filtering for sets
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE d = {6, 4}");
@@ -1516,13 +1617,23 @@ public void testFilteringWithoutIndicesWithFrozenCollections() throws Throwable
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE d CONTAINS 4");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE d NOT CONTAINS 4");
+
             assertRows(execute("SELECT * FROM %s WHERE d CONTAINS 4 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE d NOT CONTAINS 4 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE d CONTAINS 4 AND d CONTAINS 6 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE d NOT CONTAINS 4 AND d NOT CONTAINS 6 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
             // Checks filtering for maps
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE e = {1 : 2}");
@@ -1544,22 +1655,42 @@ public void testFilteringWithoutIndicesWithFrozenCollections() throws Throwable
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE e CONTAINS 2");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE e NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE e CONTAINS 2 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE e NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE e CONTAINS KEY 1 ALLOW FILTERING"),
                        row(1, 2, list(1, 6), set(2, 12), map(1, 6)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE e NOT CONTAINS KEY 1 ALLOW FILTERING"),
+                       row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)));
+
             assertInvalidMessage("Map-entry equality predicates on frozen map column e are not supported",
                                  "SELECT * FROM %s WHERE e[1] = 6 ALLOW FILTERING");
 
+            assertInvalidMessage("Map-entry equality predicates on frozen map column e are not supported",
+                                 "SELECT * FROM %s WHERE e[1] != 6 ALLOW FILTERING");
+
             assertRows(execute("SELECT * FROM %s WHERE e CONTAINS KEY 1 AND e CONTAINS 2 ALLOW FILTERING"),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE e CONTAINS KEY 1 AND e NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND d CONTAINS 4 AND e CONTAINS KEY 3 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
+
+            assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 2 AND d CONTAINS 4 AND e NOT CONTAINS KEY 1 ALLOW FILTERING"),
+                       row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
         });
 
         // Checks filtering with null
@@ -1582,6 +1713,19 @@ public void testFilteringWithoutIndicesWithFrozenCollections() throws Throwable
         assertInvalidMessage("Map-entry equality predicates on frozen map column e are not supported",
                              "SELECT * FROM %s WHERE e[1] = null ALLOW FILTERING");
 
+        assertInvalidMessage("Unsupported null value for column c",
+                             "SELECT * FROM %s WHERE c NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column d",
+                             "SELECT * FROM %s WHERE d NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column e",
+                             "SELECT * FROM %s WHERE e NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column e",
+                             "SELECT * FROM %s WHERE e NOT CONTAINS KEY null ALLOW FILTERING");
+        assertInvalidMessage("Map-entry equality predicates on frozen map column e are not supported",
+                             "SELECT * FROM %s WHERE e[null] != 2 ALLOW FILTERING");
+        assertInvalidMessage("Map-entry equality predicates on frozen map column e are not supported",
+                             "SELECT * FROM %s WHERE e[1] != null ALLOW FILTERING");
+
         // Checks filtering with unset
         assertInvalidMessage("Unsupported unset value for column c",
                              "SELECT * FROM %s WHERE c = ? ALLOW FILTERING",
@@ -1610,6 +1754,25 @@ public void testFilteringWithoutIndicesWithFrozenCollections() throws Throwable
         assertInvalidMessage("Map-entry equality predicates on frozen map column e are not supported",
                              "SELECT * FROM %s WHERE e[1] = ? ALLOW FILTERING",
                              unset());
+        assertInvalidMessage("Unsupported unset value for column c",
+                             "SELECT * FROM %s WHERE c NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column d",
+                             "SELECT * FROM %s WHERE d NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column e",
+                             "SELECT * FROM %s WHERE e NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column e",
+                             "SELECT * FROM %s WHERE e NOT CONTAINS KEY ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Map-entry equality predicates on frozen map column e are not supported",
+                             "SELECT * FROM %s WHERE e[?] != 2 ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Map-entry equality predicates on frozen map column e are not supported",
+                             "SELECT * FROM %s WHERE e[1] != ? ALLOW FILTERING",
+                             unset());
+
     }
 
 
@@ -1920,45 +2083,86 @@ public void testAllowFilteringOnPartitionKeyWithoutIndicesWithCollections() thro
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                     "SELECT * FROM %s WHERE b < 0 AND c CONTAINS 2");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE b < 0 AND c NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE b >= 4 AND c CONTAINS 2 ALLOW FILTERING"),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE b >= 4 AND c NOT CONTAINS 3 ALLOW FILTERING"),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
+
             assertRows(
                     execute("SELECT * FROM %s WHERE a > 0 AND b <= 3 AND c CONTAINS 2 AND c CONTAINS 3 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
 
+            assertRows(
+                    execute("SELECT * FROM %s WHERE a > 0 AND b <= 3 AND c NOT CONTAINS 1 AND c NOT CONTAINS 6 ALLOW FILTERING"),
+                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
+
             // Checks filtering for sets
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                     "SELECT * FROM %s WHERE a = 1 AND d CONTAINS 4");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE a = 1 AND d NOT CONTAINS 4");
+
             assertRows(execute("SELECT * FROM %s WHERE d CONTAINS 4 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE d NOT CONTAINS 4 ALLOW FILTERING"),
+                       row(2, 3, list(3, 6), set(6, 12), map(3, 6)),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE d CONTAINS 4 AND d CONTAINS 6 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE d NOT CONTAINS 4 AND d NOT CONTAINS 6 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
             // Checks filtering for maps
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                                  "SELECT * FROM %s WHERE e CONTAINS 2");
 
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                 "SELECT * FROM %s WHERE e NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE a < 2 AND b >= 3 AND e CONTAINS 2 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE a < 2 AND b >= 3 AND e NOT CONTAINS 6 ALLOW FILTERING"),
+                       row(1, 3, list(3, 2), set(6, 4), map(3, 2)),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
+
             assertRows(execute("SELECT * FROM %s WHERE a = 1 AND e CONTAINS KEY 1 ALLOW FILTERING"),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)),
                        row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
 
+            assertRows(execute("SELECT * FROM %s WHERE a = 1 AND e NOT CONTAINS KEY 3 ALLOW FILTERING"),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE a in (1) AND b in (2) AND e[1] = 6 ALLOW FILTERING"),
                        row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
 
+            assertRows(execute("SELECT * FROM %s WHERE a in (1) AND b in (2) AND e[1] != 2 ALLOW FILTERING"),
+                       row(1, 2, list(1, 6), set(2, 12), map(1, 6)));
+
             assertRows(execute("SELECT * FROM %s WHERE a = 1 AND e CONTAINS KEY 1 AND e CONTAINS 2 ALLOW FILTERING"),
                        row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
 
+            assertRows(execute("SELECT * FROM %s WHERE a = 1 AND e CONTAINS KEY 1 AND e NOT CONTAINS 6 ALLOW FILTERING"),
+                       row(1, 4, list(1, 2), set(2, 4), map(1, 2)));
+
             assertRows(
                     execute("SELECT * FROM %s WHERE a >= 1 AND b in (3) AND c CONTAINS 2 AND d CONTAINS 4 AND e CONTAINS KEY 3 ALLOW FILTERING"),
                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
+
+            assertRows(
+                    execute("SELECT * FROM %s WHERE a >= 1 AND b in (3) AND c CONTAINS 2 AND d CONTAINS 4 AND e NOT CONTAINS KEY 1 ALLOW FILTERING"),
+                        row(1, 3, list(3, 2), set(6, 4), map(3, 2)));
         });
 
         // Checks filtering with null
@@ -1974,6 +2178,18 @@ public void testAllowFilteringOnPartitionKeyWithoutIndicesWithCollections() thro
                              "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e[null] = 2 ALLOW FILTERING");
         assertInvalidMessage("Unsupported null map value for column e",
                              "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e[1] = null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column c",
+                             "SELECT * FROM %s WHERE a > 1 AND c NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column d",
+                             "SELECT * FROM %s WHERE b < 1 AND d NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column e",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e NOT CONTAINS null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null value for column e",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e NOT CONTAINS KEY null ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null map key for column e",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e[null] != 2 ALLOW FILTERING");
+        assertInvalidMessage("Unsupported null map value for column e",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e[1] != null ALLOW FILTERING");
 
         // Checks filtering with unset
         assertInvalidMessage("Unsupported unset value for column c",
@@ -1994,6 +2210,24 @@ public void testAllowFilteringOnPartitionKeyWithoutIndicesWithCollections() thro
         assertInvalidMessage("Unsupported unset map value for column e",
                              "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e[1] = ? ALLOW FILTERING",
                              unset());
+        assertInvalidMessage("Unsupported unset value for column c",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND c NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column d",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND d NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column e",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e NOT CONTAINS ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset value for column e",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e NOT CONTAINS KEY ? ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset map key for column e",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e[?] != 2 ALLOW FILTERING",
+                             unset());
+        assertInvalidMessage("Unsupported unset map value for column e",
+                             "SELECT * FROM %s WHERE a >= 1 AND b < 1 AND e[1] != ? ALLOW FILTERING",
+                             unset());
     }
 
     @Test
@@ -2158,18 +2392,32 @@ public void containsFilteringForClusteringKeys() throws Throwable
 
             assertRows(execute("SELECT * FROM %s WHERE a = 21 AND b CONTAINS 2 ALLOW FILTERING"),
                        row(21, list(2, 3), 24));
+
+            assertRows(execute("SELECT * FROM %s WHERE a = 21 AND b NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(21, list(3, 3), 34));
+
             assertInvalidMessage("Clustering columns can only be restricted with CONTAINS with a secondary index or filtering",
                                  "SELECT * FROM %s WHERE a = 21 AND b CONTAINS 2");
 
             assertRows(execute("SELECT * FROM %s WHERE b CONTAINS 2 ALLOW FILTERING"),
                        row(21, list(2, 3), 24));
+
+            assertRows(execute("SELECT * FROM %s WHERE b NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(11, list(1, 3), 14),
+                       row(21, list(3, 3), 34));
+
             assertInvalidMessage("Clustering columns can only be restricted with CONTAINS with a secondary index or filtering",
                                  "SELECT * FROM %s WHERE b CONTAINS 2");
 
+            assertInvalidMessage("Clustering columns can only be restricted with CONTAINS with a secondary index or filtering",
+                                 "SELECT * FROM %s WHERE b NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE b CONTAINS 3 ALLOW FILTERING"),
                        row(11, list(1, 3), 14),
                        row(21, list(2, 3), 24),
                        row(21, list(3, 3), 34));
+
+            assertRows(execute("SELECT * FROM %s WHERE b NOT CONTAINS 3 ALLOW FILTERING"));
         });
 
         // non-first clustering column
@@ -2183,18 +2431,34 @@ public void containsFilteringForClusteringKeys() throws Throwable
 
             assertRows(execute("SELECT * FROM %s WHERE a = 21 AND c CONTAINS 2 ALLOW FILTERING"),
                        row(21, 22, list(2, 3), 24));
+
+            assertRows(execute("SELECT * FROM %s WHERE a = 21 AND c NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(21, 22, list(3, 3), 34));
+
             assertInvalidMessage("Clustering columns can only be restricted with CONTAINS with a secondary index or filtering",
                                  "SELECT * FROM %s WHERE a = 21 AND c CONTAINS 2");
 
+            assertInvalidMessage("Clustering columns can only be restricted with CONTAINS with a secondary index or filtering",
+                                 "SELECT * FROM %s WHERE a = 21 AND c NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE b > 20 AND c CONTAINS 2 ALLOW FILTERING"),
                        row(21, 22, list(2, 3), 24));
+
+            assertRows(execute("SELECT * FROM %s WHERE b > 20 AND c NOT CONTAINS 2 ALLOW FILTERING"),
+                       row(21, 22, list(3, 3), 34));
+
             assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
                                  "SELECT * FROM %s WHERE b > 20 AND c CONTAINS 2");
 
+            assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
+                                 "SELECT * FROM %s WHERE b > 20 AND c NOT CONTAINS 2");
+
             assertRows(execute("SELECT * FROM %s WHERE c CONTAINS 3 ALLOW FILTERING"),
                        row(11, 12, list(1, 3), 14),
                        row(21, 22, list(2, 3), 24),
                        row(21, 22, list(3, 3), 34));
+
+            assertEmpty(execute("SELECT * FROM %s WHERE c NOT CONTAINS 3 ALLOW FILTERING"));
         });
 
         createTable("CREATE TABLE %s (a int, b int, c frozen<map<text, text>>, d int, PRIMARY KEY (a, b, c))");
@@ -2206,8 +2470,15 @@ public void containsFilteringForClusteringKeys() throws Throwable
         beforeAndAfterFlush(() -> {
             assertRows(execute("SELECT * FROM %s WHERE b > 20 AND c CONTAINS KEY '2' ALLOW FILTERING"),
                        row(21, 22, map("2", "3"), 24));
+
+            assertRows(execute("SELECT * FROM %s WHERE b > 20 AND c NOT CONTAINS KEY '2' ALLOW FILTERING"),
+                       row(21, 22, map("3", "3"), 34));
+
             assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
                                  "SELECT * FROM %s WHERE b > 20 AND c CONTAINS KEY '2'");
+
+            assertInvalidMessage("Clustering column \"c\" cannot be restricted (preceding column \"b\" is restricted by a non-EQ relation)",
+                                 "SELECT * FROM %s WHERE b > 20 AND c NOT CONTAINS KEY '2'");
         });
     }
 
@@ -2239,6 +2510,9 @@ private void testContainsOnPartitionKey(String schema) throws Throwable
         assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                              "SELECT * FROM %s WHERE pk CONTAINS KEY 1");
 
+        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                             "SELECT * FROM %s WHERE pk NOT CONTAINS KEY 1");
+
         beforeAndAfterFlush(() -> {
             assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk CONTAINS KEY 1 ALLOW FILTERING"),
                                     row(map(1, 2), 1, 1),
@@ -2260,6 +2534,24 @@ private void testContainsOnPartitionKey(String schema) throws Throwable
 
             assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk CONTAINS KEY 1 AND ck = 1 AND v = 3 ALLOW FILTERING"),
                                     row(map(1, 2, 3, 4), 1, 3));
+
+            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk NOT CONTAINS KEY 1 ALLOW FILTERING"),
+                                    row(map(5, 6), 5, 5),
+                                    row(map(7, 8), 6, 6));
+
+            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk CONTAINS KEY 1 AND pk NOT CONTAINS 4 ALLOW FILTERING"),
+                                    row(map(1, 2), 1, 1),
+                                    row(map(1, 2), 2, 2));
+
+            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk NOT CONTAINS KEY 1 AND pk CONTAINS 8 ALLOW FILTERING"),
+                                    row(map(7, 8), 6, 6));
+
+            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk NOT CONTAINS KEY 1 AND pk NOT CONTAINS 8 ALLOW FILTERING"),
+                                    row(map(5, 6), 5, 5));
+
+            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk NOT CONTAINS KEY 1 AND v = 5 ALLOW FILTERING"),
+                                    row(map(5, 6), 5, 5));
+
         });
     }
 
@@ -2341,8 +2633,16 @@ public void containsFilteringOnNonClusteringColumn() throws Throwable {
                        row(21, 22, 23, list(2, 4)),
                        row(21, 25, 26, list(2, 7)));
 
-            assertRows(executeFilteringOnly("SELECT a, b, c, d FROM %s WHERE b > 20 AND d CONTAINS 2 AND d contains 4"),
+            assertRows(executeFilteringOnly("SELECT a, b, c, d FROM %s WHERE b > 20 AND d NOT CONTAINS 2"),
+                       row(31, 32, 33, list(3, 4)));
+
+            assertRows(executeFilteringOnly("SELECT a, b, c, d FROM %s WHERE b > 20 AND d CONTAINS 2 AND d CONTAINS 4"),
                        row(21, 22, 23, list(2, 4)));
+
+            assertRows(executeFilteringOnly("SELECT a, b, c, d FROM %s WHERE b > 20 AND d NOT CONTAINS 2 AND d CONTAINS 4"),
+                       row(31, 32, 33, list(3, 4)));
+
+            assertRows(executeFilteringOnly("SELECT a, b, c, d FROM %s WHERE b > 20 AND d NOT CONTAINS 2 AND d NOT CONTAINS 4"));
         });
     }
 
@@ -2904,7 +3204,7 @@ public void testFilteringOnListContainingDurations() throws Throwable
             else
             {
                 assertInvalidMessage("Collection column 'l' (list<duration>) cannot be restricted by a 'IN' relation",
-                        "SELECT * FROM %s WHERE l IN ([1s, 2s], [2s, 3s]) ALLOW FILTERING");
+                                     "SELECT * FROM %s WHERE l IN ([1s, 2s], [2s, 3s]) ALLOW FILTERING");
             }
 
             assertInvalidMessage("Slice restrictions are not supported on collections containing durations",
@@ -2943,13 +3243,13 @@ public void testFilteringOnMapContainingDurations() throws Throwable
                            row(0, map(1, Duration.from("1s"), 2, Duration.from("2s"))));
 
                 assertRows(execute("SELECT * FROM %s WHERE m IN ({1:1s, 2:2s}, {1:1s, 3:3s}) ALLOW FILTERING"),
-                        row(0, map(1, Duration.from("1s"), 2, Duration.from("2s"))),
-                        row(2, map(1, Duration.from("1s"), 3, Duration.from("3s"))));
+                           row(0, map(1, Duration.from("1s"), 2, Duration.from("2s"))),
+                           row(2, map(1, Duration.from("1s"), 3, Duration.from("3s"))));
             }
             else
             {
                 assertInvalidMessage("Collection column 'm' (map<int, duration>) cannot be restricted by a 'IN' relation",
-                        "SELECT * FROM %s WHERE m IN ({1:1s, 2:2s}, {1:1s, 3:3s}) ALLOW FILTERING");
+                                     "SELECT * FROM %s WHERE m IN ({1:1s, 2:2s}, {1:1s, 3:3s}) ALLOW FILTERING");
             }
 
             assertInvalidMessage("Slice restrictions are not supported on collections containing durations",
@@ -2967,6 +3267,10 @@ public void testFilteringOnMapContainingDurations() throws Throwable
             assertRows(execute("SELECT * FROM %s WHERE m CONTAINS 1s ALLOW FILTERING"),
                        row(0, map(1, Duration.from("1s"), 2, Duration.from("2s"))),
                        row(2, map(1, Duration.from("1s"), 3, Duration.from("3s"))));
+
+            assertRows(execute("SELECT * FROM %s WHERE m NOT CONTAINS 1s ALLOW FILTERING"),
+                       row(1, map(2, Duration.from("2s"), 3, Duration.from("3s"))));
+
         }
     }
 
@@ -3018,16 +3322,16 @@ public void testFilteringOnUdtContainingDurations() throws Throwable
                            row(0, userType("i", 1, "d", Duration.from("2s"))));
 
                 assertRows(execute("SELECT * FROM %s WHERE u IN ({i: 2, d:3s}, {i: 1, d:3s}) ALLOW FILTERING"),
-                        row(1, userType("i", 2, "d", Duration.from("3s"))),
-                        row(2, userType("i", 1, "d", Duration.from("3s"))));
+                           row(1, userType("i", 2, "d", Duration.from("3s"))),
+                           row(2, userType("i", 1, "d", Duration.from("3s"))));
             }
             else
             {
                 assertInvalidMessage("Non-frozen UDT column 'u' (" + udt + ") cannot be restricted by any relation",
-                        "SELECT * FROM %s WHERE u = {i: 1, d:2s} ALLOW FILTERING");
+                                     "SELECT * FROM %s WHERE u = {i: 1, d:2s} ALLOW FILTERING");
 
                 assertInvalidMessage("Non-frozen UDT column 'u' (" + udt + ") cannot be restricted by any relation",
-                        "SELECT * FROM %s WHERE u IN ({i: 2, d:3s}, {i: 1, d:3s}) ALLOW FILTERING");
+                                     "SELECT * FROM %s WHERE u IN ({i: 2, d:3s}, {i: 1, d:3s}) ALLOW FILTERING");
             }
 
             assertInvalidMessage("Slice restrictions are not supported on UDTs containing durations",
@@ -3064,25 +3368,36 @@ public void testFilteringOnCollectionsWithNull() throws Throwable
         beforeAndAfterFlush(() -> {
             // lists
             assertRows(execute("SELECT k, v FROM %s WHERE l CONTAINS 1 ALLOW FILTERING"), row(1, 0), row(0, 0), row(0, 2));
+            assertRows(execute("SELECT k, v FROM %s WHERE l NOT CONTAINS 4 ALLOW FILTERING"), row(1, 2), row(0, 0), row(0, 2));
             assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND l CONTAINS 1 ALLOW FILTERING"), row(0, 0), row(0, 2));
+            assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND l NOT CONTAINS 4 ALLOW FILTERING"), row(0, 0), row(0, 2));
             assertRows(execute("SELECT k, v FROM %s WHERE l CONTAINS 2 ALLOW FILTERING"), row(1, 0), row(0, 0));
             assertEmpty(execute("SELECT k, v FROM %s WHERE l CONTAINS 6 ALLOW FILTERING"));
 
             // sets
             assertRows(execute("SELECT k, v FROM %s WHERE s CONTAINS 'a' ALLOW FILTERING" ), row(0, 0), row(0, 2));
+            assertRowsIgnoringOrder(execute("SELECT k, v FROM %s WHERE s NOT CONTAINS 'a' ALLOW FILTERING" ),  row(1, 2), row(0, 1), row(1, 0), row(1, 1));
             assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND s CONTAINS 'a' ALLOW FILTERING"), row(0, 0), row(0, 2));
+            assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND s NOT CONTAINS 'a' ALLOW FILTERING" ), row(0, 1));
             assertRows(execute("SELECT k, v FROM %s WHERE s CONTAINS 'd' ALLOW FILTERING"), row(1, 1));
             assertEmpty(execute("SELECT k, v FROM %s  WHERE s CONTAINS 'e' ALLOW FILTERING"));
+            assertRows(execute("SELECT k, v FROM %s  WHERE s NOT CONTAINS 'a' AND s NOT CONTAINS 'c' ALLOW FILTERING"), row(1, 0), row(1, 1), row(1, 2));
 
             // maps
             assertRows(execute("SELECT k, v FROM %s WHERE m CONTAINS 1 ALLOW FILTERING"), row(1, 0), row(1, 1), row(0, 0), row(0, 1));
+            assertRows(execute("SELECT k, v FROM %s WHERE m NOT CONTAINS 1 ALLOW FILTERING"), row(1, 2), row(0, 2));
             assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND m CONTAINS 1 ALLOW FILTERING"), row(0, 0), row(0, 1));
+            assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND m NOT CONTAINS 1 ALLOW FILTERING"), row(0, 2));
             assertRows(execute("SELECT k, v FROM %s WHERE m CONTAINS 2 ALLOW FILTERING"), row(0, 1));
             assertEmpty(execute("SELECT k, v FROM %s  WHERE m CONTAINS 4 ALLOW FILTERING"));
+            assertRows(execute("SELECT k, v FROM %s  WHERE m NOT CONTAINS 1 AND m NOT CONTAINS 3 ALLOW FILTERING"), row(1, 2));
 
             assertRows(execute("SELECT k, v FROM %s WHERE m CONTAINS KEY 'a' ALLOW FILTERING"), row(1, 1), row(0, 0), row(0, 1));
+            assertRows(execute("SELECT k, v FROM %s WHERE m NOT CONTAINS KEY 'a' ALLOW FILTERING"), row(1, 0), row(1, 2), row(0, 2));
             assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND m CONTAINS KEY 'a' ALLOW FILTERING"), row(0, 0), row(0, 1));
+            assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND m NOT CONTAINS KEY 'a' ALLOW FILTERING"), row(0, 2));
             assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND m CONTAINS KEY 'c' ALLOW FILTERING"), row(0, 2));
+            assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND m NOT CONTAINS KEY 'c' ALLOW FILTERING"), row(0, 0), row(0, 1));
         });
     }
 
@@ -3251,4 +3566,13 @@ public void testQuotedUDTData() throws Throwable
 
         assertRows(execute("SELECT udt_data FROM " + KEYSPACE + ".t4"), row(userType("random", "I'm newb")));
     }
+
+    // ensure that we can create composite types larger than signed short
+    @Test
+    public void compositeValuePk() throws Throwable
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (a blob, b blob, PRIMARY KEY ((a, b)))");
+        execute("INSERT INTO %s (a, b) VALUES (?, ?)", ByteBuffer.allocate(Short.MAX_VALUE + 1), EMPTY_BYTE_BUFFER);
+    }
+
 }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/TruncateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/TruncateTest.java
index 78a42fcb5913..58609853b2d0 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/TruncateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/TruncateTest.java
@@ -17,12 +17,58 @@
  */
 package org.apache.cassandra.cql3.validation.operations;
 
+import java.nio.ByteBuffer;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.TruncateStatement;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.TruncateException;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.TRUNCATE_STATEMENT_PROVIDER;
 
 public class TruncateTest extends CQLTester
 {
+    static {
+        TRUNCATE_STATEMENT_PROVIDER.setString(TestTruncateStatementProvider.class.getName());
+    }
+
+    public static final ByteBuffer[] VALUES = new ByteBuffer[0];
+    public static boolean testTruncateProvider = false;
+
+    @BeforeClass
+    public static void setup()
+    {
+        TRUNCATE_STATEMENT_PROVIDER.setString(TestTruncateStatementProvider.class.getName());
+    }
+
+    @AfterClass
+    public static void teardown()
+    {
+        System.clearProperty(TRUNCATE_STATEMENT_PROVIDER.getKey());        
+    }
+    
+    @After
+    public void afterTest()
+    {
+        testTruncateProvider = false;
+        TestTruncateStatementProvider.testTruncateStatement = null;
+    }
+
     @Test
     public void testTruncate() throws Throwable
     {
@@ -45,4 +91,132 @@ public void testTruncate() throws Throwable
             assertEmpty(execute("SELECT * FROM %s"));
         }
     }
+
+    @Test
+    public void testRemoteTruncateStmt() throws Throwable
+    {
+        testTruncateProvider = true;
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+
+        execute("TRUNCATE TABLE %s");
+        assertTrue(TestTruncateStatementProvider.testTruncateStatement.executeLocallyInvoked);
+    }
+
+    @Test
+    public void testTruncateUnknownTable() throws Throwable
+    {
+        testTruncateProvider = true;
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+
+        String query = "TRUNCATE TABLE doesnotexist";
+        try
+        {
+            // Check TruncateStatement.exceuteLocally path
+            execute(query);
+            fail("Expected TruncationException");
+        }
+        catch (TruncateException e)
+        {
+            assertEquals("Error during truncate: Unknown keyspace/table system.doesnotexist", e.getMessage());
+            assertTrue(TestTruncateStatementProvider.testTruncateStatement.executeLocallyInvoked);
+        }
+
+        try
+        {
+            // Check TruncateStatement.exceute path
+            TestTruncateStatementProvider.testTruncateStatement.execute(QueryState.forInternalCalls(), QueryOptions.DEFAULT, new Dispatcher.RequestTime(0L, 0L));
+            fail("Expected TruncationException");
+        }
+        catch (TruncateException e)
+        {
+            assertEquals("Error during truncate: Unknown keyspace/table system.doesnotexist", e.getMessage());
+            assertTrue(TestTruncateStatementProvider.testTruncateStatement.executeInvoked);
+        }
+    }
+
+    @Test
+    public void testTruncateView() throws Throwable
+    {
+        testTruncateProvider = true;
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
+        String qualifiedViewName = KEYSPACE + "." + createViewName();
+        execute("CREATE MATERIALIZED VIEW " + qualifiedViewName + " AS SELECT * "
+                + "FROM %s WHERE a IS NOT NULL and b IS NOT NULL "
+                + "PRIMARY KEY (a, b)");
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+
+        try
+        {
+            // Check TruncateStatement.exceuteLocally path
+            execute("TRUNCATE TABLE " + qualifiedViewName);
+            fail("Expected TruncationException");
+        }
+        catch (TruncateException e)
+        {
+            assertEquals("Error during truncate: Cannot TRUNCATE materialized view directly; must truncate base table instead", e.getMessage());
+            assertTrue(TestTruncateStatementProvider.testTruncateStatement.executeLocallyInvoked);
+        }
+
+        try
+        {
+            // Check TruncateStatement.exceute path
+            TestTruncateStatementProvider.testTruncateStatement.execute(QueryState.forInternalCalls(), QueryOptions.DEFAULT, new Dispatcher.RequestTime(0L, 0L));
+            fail("Expected TruncationException");
+        }
+        catch (TruncateException e)
+        {
+            assertEquals("Error during truncate: Cannot TRUNCATE materialized view directly; must truncate base table instead", e.getMessage());
+            assertTrue(TestTruncateStatementProvider.testTruncateStatement.executeInvoked);
+        }
+    }
+
+    public static class TestTruncateStatementProvider implements TruncateStatement.TruncateStatementProvider
+    {
+        public static TestTruncateStatement testTruncateStatement;
+
+        @Override
+        public TruncateStatement createTruncateStatement(String queryString, QualifiedName name)
+        {
+            if (TruncateTest.testTruncateProvider)
+            {
+                testTruncateStatement = new TestTruncateStatement(queryString, name);
+                return testTruncateStatement;
+            }
+            else
+                return new TruncateStatement(queryString, name);
+        }
+    }
+    
+    public static class TestTruncateStatement extends TruncateStatement
+    {
+        public boolean executeInvoked = false;
+        public boolean executeLocallyInvoked = false;
+        
+        public TestTruncateStatement(String queryString, QualifiedName name)
+        {
+            super(queryString, name);
+        }
+
+        @Override
+        public void validate(ClientState state) throws InvalidRequestException
+        {
+            // accept anything
+        }
+
+        @Override
+        public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime) throws InvalidRequestException, TruncateException
+        {
+            executeInvoked = true;
+            return super.execute(state, options, requestTime);
+        }
+
+        @Override
+        public ResultMessage executeLocally(QueryState state, QueryOptions options)
+        {
+            executeLocallyInvoked = true;
+            return super.executeLocally(state, options);
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/UpdateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/UpdateTest.java
index 59a061610601..36d8523ee720 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/UpdateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/UpdateTest.java
@@ -179,6 +179,10 @@ private void testUpdate(boolean forceFlush) throws Throwable
 
         assertInvalidMessage("Slice restrictions are not supported on the clustering columns in UPDATE statements",
                              "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 > ?", 7, 0, 1);
+
+        assertInvalidMessage("Slice restrictions are not supported on the clustering columns in UPDATE statements",
+                             "UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 NOT IN (?)", 7, 0, 1);
+
     }
 
     @Test
diff --git a/src/java/org/apache/cassandra/db/AbstractReadCommandBuilder.java b/test/unit/org/apache/cassandra/db/AbstractReadCommandBuilder.java
similarity index 89%
rename from src/java/org/apache/cassandra/db/AbstractReadCommandBuilder.java
rename to test/unit/org/apache/cassandra/db/AbstractReadCommandBuilder.java
index 30d1eda45080..27b754186598 100644
--- a/src/java/org/apache/cassandra/db/AbstractReadCommandBuilder.java
+++ b/test/unit/org/apache/cassandra/db/AbstractReadCommandBuilder.java
@@ -23,6 +23,8 @@
 
 import com.google.common.collect.Sets;
 
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.db.filter.ANNOptions;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.ColumnIdentifier;
@@ -39,11 +41,12 @@ public abstract class AbstractReadCommandBuilder
     protected long nowInSeconds;
 
     private int cqlLimit = -1;
-    private int pagingLimit = -1;
+    private PageSize pageSize = PageSize.NONE;
+    private int perPartitionLimit = -1;
     protected boolean reversed = false;
 
     protected Set<ColumnIdentifier> columns;
-    protected final RowFilter filter = RowFilter.create(true);
+    protected final RowFilter.Builder filter = RowFilter.builder(true);
 
     private ClusteringBound<?> lowerClusteringBound;
     private ClusteringBound<?> upperClusteringBound;
@@ -114,9 +117,15 @@ public AbstractReadCommandBuilder withLimit(int newLimit)
         return this;
     }
 
-    public AbstractReadCommandBuilder withPagingLimit(int newLimit)
+    public AbstractReadCommandBuilder withPageSize(PageSize pageSize)
     {
-        this.pagingLimit = newLimit;
+        this.pageSize = pageSize;
+        return this;
+    }
+
+    public AbstractReadCommandBuilder withPerPartitionLimit(int perPartitionLimit)
+    {
+        this.perPartitionLimit = perPartitionLimit;
         return this;
     }
 
@@ -174,7 +183,13 @@ public AbstractReadCommandBuilder filterOn(String column, Operator op, Object va
         else if (op == Operator.CONTAINS_KEY)
             type = forKeys(type);
 
-        this.filter.add(def, op, bb(value, type));
+        ByteBuffer bb = bb(value, type);
+
+        if (op == Operator.ANN)
+            filter.addANNExpression(def, bb, ANNOptions.NONE);
+        else
+            filter.add(def, op, bb);
+
         return this;
     }
 
@@ -212,9 +227,10 @@ protected ClusteringIndexFilter makeFilter()
 
     protected DataLimits makeLimits()
     {
-        DataLimits limits = cqlLimit < 0 ? DataLimits.NONE : DataLimits.cqlLimits(cqlLimit);
-        if (pagingLimit >= 0)
-            limits = limits.forPaging(pagingLimit);
+        DataLimits limits = DataLimits.cqlLimits(cqlLimit < 0 ? DataLimits.NO_LIMIT : cqlLimit,
+                                                 perPartitionLimit < 0 ? DataLimits.NO_LIMIT : perPartitionLimit);
+        if (pageSize.isDefined())
+            limits = limits.forPaging(pageSize);
         return limits;
     }
 
@@ -233,7 +249,7 @@ public SinglePartitionBuilder(ColumnFamilyStore cfs, DecoratedKey key)
         @Override
         public ReadCommand build()
         {
-            return SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, makeColumnFilter(), filter, makeLimits(), partitionKey, makeFilter());
+            return SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, makeColumnFilter(), filter.build(), makeLimits(), partitionKey, makeFilter());
         }
     }
 
@@ -307,7 +323,7 @@ else if (!startInclusive && endInclusive)
             else
                 bounds = new ExcludingBounds<>(start, end);
 
-            return PartitionRangeReadCommand.create(cfs.metadata(), nowInSeconds, makeColumnFilter(), filter, makeLimits(), new DataRange(bounds, makeFilter()));
+            return PartitionRangeReadCommand.create(cfs.metadata(), nowInSeconds, makeColumnFilter(), filter.build(), makeLimits(), new DataRange(bounds, makeFilter()));
         }
 
         static DecoratedKey makeKey(TableMetadata metadata, Object... partitionKey)
diff --git a/test/unit/org/apache/cassandra/db/AbstractReadQueryToCQLStringTest.java b/test/unit/org/apache/cassandra/db/AbstractReadQueryToCQLStringTest.java
index bfd743c39372..f0087c651350 100644
--- a/test/unit/org/apache/cassandra/db/AbstractReadQueryToCQLStringTest.java
+++ b/test/unit/org/apache/cassandra/db/AbstractReadQueryToCQLStringTest.java
@@ -86,11 +86,17 @@ public void testSkinnyTable() throws Throwable
         test("SELECT * FROM %s WHERE v1 > 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE v1 <= 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE v1 >= 1 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE v2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k = 0 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE k = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE k = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE k = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE k = 0 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE token(k) > 0 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
 
         // row filter with indexed column
         createIndex("CREATE INDEX ON %s (v1)");
@@ -100,9 +106,12 @@ public void testSkinnyTable() throws Throwable
         test("SELECT * FROM %s WHERE v1 <= 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE v1 >= 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE v2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1");
-        test("SELECT * FROM %s WHERE k = 0 AND v1 = 1");
+        test("SELECT * FROM %s WHERE k = 0 AND v1 = 1",
+             "SELECT * FROM %s WHERE token(k) >= token(0) AND token(k) <= token(0) AND v1 = 1 ALLOW FILTERING");
 
         // grouped partition-directed queries, maybe producing multiple queries
         test("SELECT * FROM %s WHERE k IN (0)",
@@ -156,13 +165,19 @@ public void testSkinnyTableWithMulticolumnKey() throws Throwable
         test("SELECT * FROM %s WHERE k1 = 1 AND v2 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k2 = 2 AND v2 = 1 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE v2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 0 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 0 AND v2 = 2 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE token(k1, k2) > 0 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
 
         // row filter with indexed column
         createIndex("CREATE INDEX ON %s (k1)");
@@ -181,12 +196,16 @@ public void testSkinnyTableWithMulticolumnKey() throws Throwable
         test("SELECT * FROM %s WHERE k1 = 1 AND v2 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k2 = 2 AND v2 = 1 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE v2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND k1 = 1");
         test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND k2 = 2");
         test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1");
         test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v2 = 2");
-        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1");
+//      test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1");
+        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1",
+                          "SELECT * FROM %s WHERE token(k1, k2) >= token(1, 2) AND token(k1, k2) <= token(1, 2) AND v1 = 1 ALLOW FILTERING");
 
         // grouped partition-directed queries, maybe producing multiple queries
         test("SELECT * FROM %s WHERE k1 IN (1) AND k2 = 2",
@@ -257,14 +276,24 @@ public void testWideTable() throws Throwable
         test("SELECT * FROM %s WHERE s = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE v2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k = 0 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k = 0 AND c = 1 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE k = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE k = 0 AND c = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE token(k) > 0 AND c = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE k = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE k = 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE k = 0 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE k = 0 AND c = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE k = 0 AND c = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE k = 0 AND c = 1 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE token(k) > 0 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE token(k) > 0 AND c = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE token(k) > 0 AND c = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE token(k) > 0 AND c = 1 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
 
         // expression filter with indexed column
         createIndex("CREATE INDEX ON %s (c)");
@@ -274,11 +303,14 @@ public void testWideTable() throws Throwable
         test("SELECT * FROM %s WHERE v1 = 1");
         test("SELECT * FROM %s WHERE s = 1");
         test("SELECT * FROM %s WHERE v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE v2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k) > 0 AND v1 = 1");
-        test("SELECT * FROM %s WHERE k = 0 AND v1 = 1");
+        test("SELECT * FROM %s WHERE k = 0 AND v1 = 1",
+             "SELECT * FROM %s WHERE token(k) >= token(0) AND token(k) <= token(0) AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k = 0 AND v1 = 1 AND c = 1",
-             "SELECT * FROM %s WHERE k = 0 AND c = 1 AND v1 = 1 ALLOW FILTERING");
+             "SELECT * FROM %s WHERE token(k) >= token(0) AND token(k) <= token(0) AND c = 1 AND v1 = 1 ALLOW FILTERING");
 
         // grouped partition-directed queries, maybe producing multiple queries
         test("SELECT * FROM %s WHERE k IN (0)",
@@ -392,14 +424,24 @@ public void testWideTableWithMulticolumnKey() throws Throwable
         test("SELECT * FROM %s WHERE c3 = 2 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE v2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND c1 = 1 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND c1 = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND c1 = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND c1 = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND c1 = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND c1 = 1 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE token(k1, k2) > 0 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND c1 = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE token(k1, k2) > 0 AND c1 = 1 AND v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE token(k1, k2) > 0 AND c1 = 1 AND v2 = 2 AND v1 = 1 ALLOW FILTERING");
 
         // expression filter with indexed column
         createIndex("CREATE INDEX ON %s (k1)");
@@ -420,10 +462,14 @@ public void testWideTableWithMulticolumnKey() throws Throwable
         test("SELECT * FROM %s WHERE c1 = 1 AND c2 = 2 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE c1 = 1 AND c2 = 2 AND c3 = 3 ALLOW FILTERING",
              "SELECT * FROM %s WHERE (c1, c2, c3) = (1, 2, 3) ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING", true,
+             "SELECT * FROM %s WHERE v1 = 1 AND v2 = 2 ALLOW FILTERING",
+             "SELECT * FROM %s WHERE v2 = 2 AND v1 = 1 ALLOW FILTERING");
         test("SELECT * FROM %s WHERE token(k1, k2) > 0 AND v1 = 1");
-        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1");
-        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND c1 = 1 AND v1 = 1");
+        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND v1 = 1",
+             "SELECT * FROM %s WHERE token(k1, k2) >= token(1, 2) AND token(k1, k2) <= token(1, 2) AND v1 = 1 ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE k1 = 1 AND k2 = 2 AND c1 = 1 AND v1 = 1",
+             "SELECT * FROM %s WHERE token(k1, k2) >= token(1, 2) AND token(k1, k2) <= token(1, 2) AND c1 = 1 AND v1 = 1 ALLOW FILTERING");
 
         // grouped partition-directed queries, maybe producing multiple queries
         test("SELECT * FROM %s WHERE k1 IN (1) AND k2 IN (2)",
@@ -648,11 +694,15 @@ public void testFrozenCollections() throws Throwable
              "SELECT m, s, t, u FROM %s");
 
         // filtering
-        test("SELECT * FROM %s WHERE l = ['a', 'b'] ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE s = {'a', 'b'} ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE m = {'a': 'b', 'c': 'd'} ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE l = ['a', 'b'] ALLOW FILTERING", false, false,
+             "SELECT * FROM %s WHERE l = ['a', ... ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE s = {'a', 'b'} ALLOW FILTERING", false, false,
+             "SELECT * FROM %s WHERE s = {'a', ... ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE m = {'a': 'b', 'c': 'd'} ALLOW FILTERING", false, false,
+             "SELECT * FROM %s WHERE m = {'a': ... ALLOW FILTERING");
         test("SELECT * FROM %s WHERE t = ('a', 1) ALLOW FILTERING");
-        test("SELECT * FROM %s WHERE u = {a: 'a', b: 1} ALLOW FILTERING");
+        test("SELECT * FROM %s WHERE u = {a: 'a', b: 1} ALLOW FILTERING", false, false,
+             "SELECT * FROM %s WHERE u = {a: 'a... ALLOW FILTERING");
         testInvalid("SELECT * FROM %s WHERE l['a'] = 'a' ALLOW FILTERING");
         testInvalid("SELECT * FROM %s WHERE s['a'] = 'a' ALLOW FILTERING");
         testInvalid("SELECT * FROM %s WHERE m['a'] = 'a' ALLOW FILTERING");
@@ -788,17 +838,34 @@ private void test(String query) throws Throwable
     }
 
     private void test(String query, String... expected)
+    {
+        test(query, false, expected);
+    }
+
+    private void test(String query, boolean matchAnyExpected, String... expected)
+    {
+        test(query, matchAnyExpected, true, expected);
+    }
+
+    private void test(String query, boolean matchAnyExpected, boolean executeExpected, String... expected)
     {
         List<String> actual = toCQLString(query);
         List<String> fullExpected = Stream.of(expected)
                                           .map(this::formatQuery)
                                           .map(s -> s.endsWith(" ALLOW FILTERING") ? s : s + " ALLOW FILTERING")
                                           .collect(Collectors.toList());
-        assertEquals(fullExpected, actual);
 
-        // execute both the expected output commands to verify that they are valid CQL
-        for (String q : expected)
-            execute(q);
+        if (matchAnyExpected)
+        {
+            assertTrue(fullExpected.stream().anyMatch(s -> actual.stream().anyMatch(s::equals)));
+        }
+        else
+           assertEquals(fullExpected, actual);
+
+        // execute all the expected output commands to verify that they are valid CQL
+        if (executeExpected)
+            for (String q : expected)
+                execute(q);
     }
 
     private void testInvalid(String query) throws Throwable
diff --git a/test/unit/org/apache/cassandra/db/CellSpecTest.java b/test/unit/org/apache/cassandra/db/CellSpecTest.java
index b14b74be2208..1ce5022f5e53 100644
--- a/test/unit/org/apache/cassandra/db/CellSpecTest.java
+++ b/test/unit/org/apache/cassandra/db/CellSpecTest.java
@@ -124,7 +124,7 @@ private static long valuePtrSize(Object value)
             return ObjectSizes.sizeOnHeapExcludingDataOf((ByteBuffer) value);
         else if (value instanceof byte[])
             return ObjectSizes.sizeOfArray((byte[]) value) - ((byte[]) value).length;
-        throw new IllegalArgumentException("Unsupported type: " + value.getClass());
+        throw new IllegalArgumentException("Unsupported type by valuePtrSize: " + value.getClass());
     }
 
     @Parameterized.Parameters(name = "{0}")
diff --git a/test/unit/org/apache/cassandra/db/CleanupTest.java b/test/unit/org/apache/cassandra/db/CleanupTest.java
index ce17562fe3d8..012d0044538c 100644
--- a/test/unit/org/apache/cassandra/db/CleanupTest.java
+++ b/test/unit/org/apache/cassandra/db/CleanupTest.java
@@ -24,6 +24,7 @@
 import java.util.AbstractMap;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -58,6 +59,7 @@
 import org.apache.cassandra.locator.SimpleStrategy;
 import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.service.StorageService;
@@ -73,6 +75,7 @@ public class CleanupTest
     public static final String KEYSPACE1 = "CleanupTest1";
     public static final String CF_INDEXED1 = "Indexed1";
     public static final String CF_STANDARD1 = "Standard1";
+    public static final String CF_STANDARD_UCS1 = "StandardUCS1";
 
     public static final String KEYSPACE2 = "CleanupTestMultiDc";
     public static final String CF_INDEXED2 = "Indexed2";
@@ -95,7 +98,8 @@ public static void defineSchema() throws ConfigurationException
         SchemaLoader.prepareServer();
         SchemaLoader.createKeyspace(KEYSPACE1,
                                     KeyspaceParams.simple(1),
-                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)
+                                                .compaction(CompactionParams.stcs(new HashMap<>())),
                                     SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED1, true));
 
 
@@ -116,8 +120,11 @@ public String getDatacenter(InetAddressAndPort endpoint)
 
         SchemaLoader.createKeyspace(KEYSPACE2,
                                     KeyspaceParams.nts("DC1", 1),
-                                    SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD2),
-                                    SchemaLoader.compositeIndexCFMD(KEYSPACE2, CF_INDEXED2, true));
+                                    SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD2)
+                                                .compaction(CompactionParams.stcs(new HashMap<>())),
+                                    SchemaLoader.compositeIndexCFMD(KEYSPACE2, CF_INDEXED2, true),
+                                    SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD_UCS1)
+                                                .compaction(CompactionParams.ucs(new HashMap<>())));
         SchemaLoader.createKeyspace(KEYSPACE3,
                                     KeyspaceParams.nts("DC1", 1),
                                     SchemaLoader.standardCFMD(KEYSPACE3, CF_STANDARD3));
@@ -169,7 +176,7 @@ public void testCleanupWithIndexes() throws IOException, ExecutionException, Int
         while (!cfs.getBuiltIndexes().contains(indexName) && nanoTime() - start < TimeUnit.SECONDS.toNanos(10))
             Thread.sleep(10);
 
-        RowFilter cf = RowFilter.create(true);
+        RowFilter.Builder cf = RowFilter.builder(true);
         cf.add(cdef, Operator.EQ, VALUE);
         assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).filterOn("birthdate", Operator.EQ, VALUE).build()).size());
 
@@ -219,18 +226,30 @@ public void testCleanupWithNewToken() throws ExecutionException, InterruptedExce
     }
 
     @Test
-    public void testCleanupWithNoTokenRange() throws Exception
+    public void testCleanupSTCSWithNoTokenRange() throws Exception
     {
-        testCleanupWithNoTokenRange(false);
+        testCleanupWithNoTokenRange(CF_STANDARD2, false);
     }
 
     @Test
-    public void testUserDefinedCleanupWithNoTokenRange() throws Exception
+    public void testUserDefinedCleanupSTCSWithNoTokenRange() throws Exception
     {
-        testCleanupWithNoTokenRange(true);
+        testCleanupWithNoTokenRange(CF_STANDARD2, true);
     }
 
-    private void testCleanupWithNoTokenRange(boolean isUserDefined) throws Exception
+    @Test
+    public void testCleanupUCSWithNoTokenRange() throws Exception
+    {
+        testCleanupWithNoTokenRange(CF_STANDARD_UCS1, false);
+    }
+
+    @Test
+    public void testUserDefinedCleanupUCSWithNoTokenRange() throws Exception
+    {
+        testCleanupWithNoTokenRange(CF_STANDARD_UCS1, true);
+    }
+
+    private void testCleanupWithNoTokenRange(String cfsName, boolean isUserDefined) throws Exception
     {
 
         TokenMetadata tmd = StorageService.instance.getTokenMetadata();
@@ -242,7 +261,7 @@ private void testCleanupWithNoTokenRange(boolean isUserDefined) throws Exception
 
         Keyspace keyspace = Keyspace.open(KEYSPACE2);
         keyspace.setMetadata(KeyspaceMetadata.create(KEYSPACE2, KeyspaceParams.nts("DC1", 1)));
-        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD2);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfsName);
 
         // insert data and verify we get it back w/ range query
         fillCF(cfs, "val", LOOPS);
diff --git a/test/unit/org/apache/cassandra/db/ClusteringHeapSizeTest.java b/test/unit/org/apache/cassandra/db/ClusteringHeapSizeTest.java
index 3ba4ae650e13..4eda7b950c6a 100644
--- a/test/unit/org/apache/cassandra/db/ClusteringHeapSizeTest.java
+++ b/test/unit/org/apache/cassandra/db/ClusteringHeapSizeTest.java
@@ -64,7 +64,7 @@ public void unsharedHeapSizeExcludingDataLTEUnsharedHeapSize()
     @Test
     public void testSingletonClusteringHeapSize()
     {
-        Clustering<?> clustering = this.clustering.accessor().factory().staticClustering();
+        Clustering<?> clustering = Clustering.STATIC_CLUSTERING;
         Assertions.assertThat(clustering.unsharedHeapSize())
                   .isEqualTo(0);
         Assertions.assertThat(clustering.unsharedHeapSizeExcludingData())
diff --git a/test/unit/org/apache/cassandra/db/ClusteringPrefixTest.java b/test/unit/org/apache/cassandra/db/ClusteringPrefixTest.java
index a295b2278694..460fbfef77fd 100644
--- a/test/unit/org/apache/cassandra/db/ClusteringPrefixTest.java
+++ b/test/unit/org/apache/cassandra/db/ClusteringPrefixTest.java
@@ -180,12 +180,12 @@ public <V> void testRetainable(ValueAccessor.ObjectFactory<V> factory,
 
     public <V> void testRetainable(ValueAccessor.ObjectFactory<V> factory,
                                    Function<String, V[]> allocator,
-                                   Function<ClusteringPrefix<V>, ClusteringPrefix<V>> mapper)
+                                   Function<ClusteringPrefix<?>, ClusteringPrefix<?>> mapper)
     {
-        ClusteringPrefix<V>[] clusterings = new ClusteringPrefix[]
+        ClusteringPrefix<?>[] clusterings = new ClusteringPrefix[]
         {
             factory.clustering(),
-            factory.staticClustering(),
+            Clustering.STATIC_CLUSTERING,
             factory.clustering(allocator.apply("test")),
             factory.bound(ClusteringPrefix.Kind.INCL_START_BOUND, allocator.apply("testA")),
             factory.bound(ClusteringPrefix.Kind.INCL_END_BOUND, allocator.apply("testB")),
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java
index 75e311e13a6f..5373c36d9690 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java
@@ -43,7 +43,11 @@
 import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class ColumnFamilyMetricTest
 {
@@ -107,18 +111,18 @@ public void testColUpdateTimeDeltaFiltering()
         ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2");
 
         // This confirms another test/set up did not overflow the histogram
-        store.metric.colUpdateTimeDeltaHistogram.cf.getSnapshot().get999thPercentile();
+        store.metric.colUpdateTimeDeltaHistogram.tableOrKeyspaceHistogram().getSnapshot().get999thPercentile();
 
         applyMutation(store.metadata(), "4242", ByteBufferUtil.bytes("0"), 0);
 
         // The histogram should not have overflowed on the first write
-        store.metric.colUpdateTimeDeltaHistogram.cf.getSnapshot().get999thPercentile();
+        store.metric.colUpdateTimeDeltaHistogram.tableOrKeyspaceHistogram().getSnapshot().get999thPercentile();
 
         // smallest time delta that would overflow the histogram if unfiltered
         applyMutation(store.metadata(), "4242", ByteBufferUtil.bytes("1"), 18165375903307L);
 
         // CASSANDRA-11117 - update with large timestamp delta should not overflow the histogram
-        store.metric.colUpdateTimeDeltaHistogram.cf.getSnapshot().get999thPercentile();
+        store.metric.colUpdateTimeDeltaHistogram.tableOrKeyspaceHistogram().getSnapshot().get999thPercentile();
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreClientModeTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreClientModeTest.java
index af77938fda07..8162ef3af110 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreClientModeTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreClientModeTest.java
@@ -30,6 +30,7 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.locator.SimpleSnitch;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
@@ -63,6 +64,8 @@ public static void setUpClass()
         DatabaseDescriptor.setEndpointSnitch(new SimpleSnitch());
         DatabaseDescriptor.getRawConfig().memtable_flush_writers = 1;
         DatabaseDescriptor.getRawConfig().local_system_data_file_directory = tempFolder.toString();
+        DatabaseDescriptor.setSpecificLocationForLocalSystemData(new File(tempFolder.getRoot()));
+        DatabaseDescriptor.setMetadataDirectory(new File(tempFolder.getRoot()));
         DatabaseDescriptor.getRawConfig().partitioner = "Murmur3Partitioner";
         DatabaseDescriptor.applyPartitioner();
     }
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 8855ceb0d3e3..bcc801e4306d 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collection;
@@ -30,8 +31,11 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterators;
@@ -44,15 +48,19 @@
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.UpdateBuilder;
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.ColumnFamilyStore.FlushReason;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.Tracker;
 import org.apache.cassandra.db.memtable.AbstractMemtable;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.Cell;
@@ -72,6 +80,7 @@
 import org.apache.cassandra.metrics.ClearableHistogram;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MockSchema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
 import org.apache.cassandra.service.snapshot.SnapshotManifest;
@@ -82,6 +91,8 @@
 import org.apache.cassandra.utils.concurrent.OpOrder.Barrier;
 import org.apache.cassandra.utils.concurrent.OpOrder.Group;
 
+import static org.apache.cassandra.db.ColumnFamilyStore.STATUS.INVALID_DROPPED;
+import static org.apache.cassandra.db.ColumnFamilyStore.STATUS.INVALID_UNLOADED;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -154,9 +165,9 @@ public void testTimeSortedQuery()
                 .applyUnsafe();
         Util.flush(cfs);
 
-        ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); // resets counts
+        ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram()).clear(); // resets counts
         Util.getAll(Util.cmd(cfs, "key1").includeRow("c1").build());
-        assertEquals(1, cfs.metric.sstablesPerReadHistogram.cf.getCount());
+        assertEquals(1, cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getCount());
     }
 
     @Test
@@ -202,7 +213,28 @@ public void runMayThrow() throws IOException
     }
 
     @Test
-    public void testDeleteStandardRowSticksAfterFlush()
+    public void testDiscardSSTables() throws ExecutionException, InterruptedException
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
+
+        new RowUpdateBuilder(cfs.metadata(), 0, "key1").clustering("Column1").build().applyUnsafe();
+        cfs.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS).get();
+
+        new RowUpdateBuilder(cfs.metadata(), 0, "key1").clustering("Column1").build().applyUnsafe();
+        cfs.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS).get();
+
+        Set<SSTableReader> sstables = cfs.getLiveSSTables();
+        assertEquals(2, sstables.size());
+
+        SSTableReader discarded = sstables.iterator().next();
+        cfs.discardSSTables(sstables, s -> s == discarded, OperationType.SSTABLE_DISCARD);
+
+        assertEquals(1, cfs.getLiveSSTables().size());
+        assertFalse(cfs.getLiveSSTables().contains(discarded));
+    }
+
+    @Test
+    public void testDeleteStandardRowSticksAfterFlush() throws Throwable
     {
         // test to make sure flushing after a delete doesn't resurrect delted cols.
         String keyspaceName = KEYSPACE1;
@@ -350,7 +382,7 @@ public void testBackupAfterFlush() throws Throwable
                                              CF_STANDARD1,
                                              liveSSTable.descriptor.id,
                                              liveSSTable.descriptor.version.format);
-            for (Component c : liveSSTable.getComponents())
+            for (Component c : liveSSTable.components())
                 assertTrue("Cannot find backed-up file:" + desc.fileFor(c), desc.fileFor(c).exists());
         }
     }
@@ -378,13 +410,13 @@ public void speculationThreshold()
         }
         // Sanity check the metrics - 50th percentile of linear 0-10000ms
         // remember, latencies are only an estimate - off by up to 20% by the 1.2 factor between buckets.
-        assertThat(cfs.metric.coordinatorReadLatency.getCount()).isEqualTo(count);
-        assertThat(cfs.metric.coordinatorReadLatency.getSnapshot().getValue(0.5))
+        assertThat(cfs.metric.coordinatorReadLatency.tableOrKeyspaceTimer().getCount()).isEqualTo(count);
+        assertThat(cfs.metric.coordinatorReadLatency.tableOrKeyspaceTimer().getSnapshot().getValue(0.5))
             .isBetween((double) TimeUnit.MILLISECONDS.toMicros(5839),
                        (double) TimeUnit.MILLISECONDS.toMicros(5840));
         // Sanity check the metrics - 75th percentileof linear 0-10000ms
-        assertThat(cfs.metric.coordinatorWriteLatency.getCount()).isEqualTo(count);
-        assertThat(cfs.metric.coordinatorWriteLatency.getSnapshot().getValue(0.75))
+        assertThat(cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount()).isEqualTo(count);
+        assertThat(cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getSnapshot().getValue(0.75))
         .isBetween((double) TimeUnit.MILLISECONDS.toMicros(8409),
                    (double) TimeUnit.MILLISECONDS.toMicros(8410));
 
@@ -539,7 +571,7 @@ private void assertRangeCount(ColumnFamilyStore cfs, ColumnMetadata col, ByteBuf
         {
             for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).filterOn(col.name.toString(), Operator.EQ, val).build()))
             {
-                for (Row r : partition)
+                for (Row r : partition.rows())
                 {
                     if (r.getCell(col).buffer().equals(val))
                         ++found;
@@ -578,12 +610,12 @@ public void testSnapshotWithoutFlushWithSecondaryIndexes() throws Exception
         assertThat(baseTableFile).isNotEqualTo(indexTableFile);
         assertThat(Directories.isSecondaryIndexFolder(new File(indexTableFile).parent())).isTrue();
 
-        Set<String> originalFiles = new HashSet<>();
+        Set<File> originalFiles = new HashSet<>();
         Iterables.toList(cfs.concatWithIndexes()).stream()
                  .flatMap(c -> c.getLiveSSTables().stream().map(t -> t.descriptor.fileFor(Components.DATA)))
-                 .forEach(e -> originalFiles.add(e.toString()));
-        assertThat(originalFiles.stream().anyMatch(f -> f.endsWith(indexTableFile))).isTrue();
-        assertThat(originalFiles.stream().anyMatch(f -> f.endsWith(baseTableFile))).isTrue();
+                 .forEach(originalFiles::add);
+        assertThat(originalFiles.stream().anyMatch(f -> f.toString().endsWith(indexTableFile))).isTrue();
+        assertThat(originalFiles.stream().anyMatch(f -> f.toString().endsWith(baseTableFile))).isTrue();
     }
 
     private void createSnapshotAndDelete(String ks, String table, boolean writeData)
@@ -643,7 +675,7 @@ public void testDataDirectoriesOfColumnFamily() throws Exception
     {
         ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
         List<String> dataPaths = cfs.getDataPaths();
-        Assert.assertFalse(dataPaths.isEmpty());
+        assertFalse(dataPaths.isEmpty());
 
         Path path = Paths.get(dataPaths.get(0));
 
@@ -697,6 +729,65 @@ public static long getSnapshotManifestAndSchemaFileSizes(TableSnapshot snapshot)
         return schemaAndManifestFileSizes;
     }
 
+    @Test
+    public void testMutateRepaired() throws IOException
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
+
+        new RowUpdateBuilder(cfs.metadata(), 0, "key1").clustering("Column1").add("val", "val1").build().applyUnsafe();
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        Set<SSTableReader> sstables = cfs.getLiveSSTables();
+        assertEquals(1, sstables.size());
+
+        SSTableReader sstable = sstables.iterator().next();
+        assertFalse(sstable.isRepaired());
+
+        int repaired = cfs.mutateRepaired(sstables, 1, null, false);
+        assertEquals(1, repaired);
+
+        sstables = cfs.getLiveSSTables();
+        sstable = sstables.iterator().next();
+        assertTrue(sstable.isRepaired());
+    }
+
+    @Test
+    public void testInvalidateWithDropping()
+    {
+        testInvalidateCFS(true);
+    }
+
+    @Test
+    public void testInvalidateWithoutDropping()
+    {
+        testInvalidateCFS(false);
+    }
+
+    private void testInvalidateCFS(boolean dropData)
+    {
+        DatabaseDescriptor.setIncrementalBackupsEnabled(false);
+        ColumnFamilyStore cfs = MockSchema.newCFS();
+        assertTrue(cfs.isValid());
+        Tracker tracker = cfs.getTracker();
+
+        Collection<SSTableReader> readers = IntStream.range(0, 10)
+                                                     .mapToObj(i -> MockSchema.sstable(i, 10, true, cfs))
+                                                     .collect(Collectors.toList());
+        tracker.addInitialSSTables(readers);
+        readers.forEach(reader -> assertEquals(1, reader.selfRef().globalCount()));
+
+        cfs.invalidate(false, dropData);
+        assertFalse(cfs.isValid());
+        assertThat(cfs.status()).isEqualTo(dropData ? INVALID_DROPPED : INVALID_UNLOADED);
+        System.gc();
+        System.gc();
+
+        readers.forEach(reader -> {
+            assertEquals(0, reader.selfRef().globalCount());
+            assertEquals(!dropData, Files.exists(reader.descriptor.pathFor(Components.DATA)));
+        });
+    }
+
     private Memtable fakeMemTableWithMinTS(ColumnFamilyStore cfs, long minTS)
     {
         return new AbstractMemtable(cfs.metadata, minTS)
@@ -708,6 +799,12 @@ public long put(PartitionUpdate update, UpdateTransaction indexer, Group opGroup
                 return 0;
             }
 
+            @Override
+            public Partition getPartition(DecoratedKey key)
+            {
+                return null;
+            }
+
             @Override
             public long partitionCount()
             {
@@ -720,11 +817,29 @@ public long getLiveDataSize()
                 return 0;
             }
 
+            @Override
+            public long getEstimatedAverageRowSize()
+            {
+                return 0;
+            }
+
             @Override
             public void addMemoryUsageTo(MemoryUsage usage)
             {
             }
 
+            @Override
+            public DecoratedKey minPartitionKey()
+            {
+                return null;
+            }
+
+            @Override
+            public DecoratedKey maxPartitionKey()
+            {
+                return null;
+            }
+
             @Override
             public void markExtraOnHeapUsed(long additionalSpace, Group opGroup)
             {
diff --git a/test/unit/org/apache/cassandra/db/ColumnsTest.java b/test/unit/org/apache/cassandra/db/ColumnsTest.java
index 4c8bcc0e4c44..7b30144f8b24 100644
--- a/test/unit/org/apache/cassandra/db/ColumnsTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnsTest.java
@@ -19,21 +19,28 @@
 package org.apache.cassandra.db;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.Predicate;
+import java.util.stream.IntStream;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
-
-import org.apache.cassandra.db.commitlog.CommitLog;
-
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.junit.Assert;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.marshal.UTF8Type;
@@ -48,6 +55,8 @@
 
 public class ColumnsTest
 {
+    private final static Logger logger = LoggerFactory.getLogger(ColumnsTest.class);
+
     static
     {
         DatabaseDescriptor.daemonInitialization();
@@ -57,8 +66,12 @@ public class ColumnsTest
     private static final TableMetadata TABLE_METADATA = MockSchema.newCFS().metadata();
 
     @Test
-    public void testDeserializeCorruption() throws IOException
+    public void testDeserializeCorruption()
     {
+        // Tests ability to detect deserialization corruption when a different superset was used for serialization
+        // In particular, when encoded a column with an index that is larger than the size of the superset
+        // (in other cases detecting corruption is rather impossible with the current serialization format)
+
         ColumnsCheck check = randomSmall(1, 0, 3, 0);
         Columns superset = check.columns;
         List<ColumnMetadata> minus1 = new ArrayList<>(check.definitions);
@@ -66,19 +79,47 @@ public void testDeserializeCorruption() throws IOException
         Columns minus2 = check.columns
                 .without(check.columns.getSimple(3))
                 .without(check.columns.getSimple(2));
+        assertDeserializationCorruption(minus1, superset, minus2);
+
+        // the missing column is the last one, so we encode 5th column in the bitmap but the superset has only 4 elements
+        testDeserializationCorruption(randomSmall(0, 0, 5, 0), IntStream.of(4), IntStream.of(0));
+
+        // the missing column is the last one, so we encode 100th column but the superset has only 99 elements
+        testDeserializationCorruption(randomHuge(0, 0, 100, 0), IntStream.of(99), IntStream.of(0));
+    }
+
+    private static void testDeserializationCorruption(ColumnsCheck check, IntStream dropFromSubset, IntStream dropFromSuperset)
+    {
+        List<ColumnMetadata> subset = new ArrayList<>(check.definitions);
+        for (Iterator<Integer> i = dropFromSubset.iterator(); i.hasNext(); )
+            subset.remove(check.definitions.get(i.next()));
+
+        Columns deserSuperset = check.columns;
+        for (Iterator<Integer> i = dropFromSuperset.iterator(); i.hasNext(); )
+            deserSuperset = deserSuperset.without(check.definitions.get(i.next()));
+
+        assertDeserializationCorruption(subset, check.columns, deserSuperset);
+    }
+
+    private static void assertDeserializationCorruption(Collection<ColumnMetadata> subset, Columns serSuperset, Columns deserSuperset)
+    {
         try (DataOutputBuffer out = new DataOutputBuffer())
         {
-            // serialize a subset
-            Columns.serializer.serializeSubset(minus1, superset, out);
+            Columns.serializer.serializeSubset(subset, serSuperset, out);
             try (DataInputBuffer in = new DataInputBuffer(out.toByteArray()))
             {
-                Columns.serializer.deserializeSubset(minus2, in);
-                Assert.assertFalse(true);
+                Columns.serializer.deserializeSubset(deserSuperset, in);
+                Assert.fail();
             }
             catch (IOException e)
             {
+                logger.info("Expected exception", e);
             }
         }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
     }
 
     // this tests most of our functionality, since each subset we perform
diff --git a/test/unit/org/apache/cassandra/db/ConsistencyLevelTest.java b/test/unit/org/apache/cassandra/db/ConsistencyLevelTest.java
new file mode 100644
index 000000000000..01d17724ec25
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ConsistencyLevelTest.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.net.UnknownHostException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.OrderPreservingPartitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.AbstractNetworkTopologySnitch;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.EverywhereStrategy;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.LocalStrategy;
+import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.locator.SimpleStrategy;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.utils.Pair;
+
+import static org.junit.Assert.*;
+
+public class ConsistencyLevelTest
+{
+    private static final String KS = "test";
+    private static final Map<String, String> RACK = new HashMap<>(), DATACENTER = new HashMap<>();
+    private static final IEndpointSnitch SNITCH = new AbstractNetworkTopologySnitch()
+    {
+        @Override
+        public String getRack(InetAddressAndPort endpoint)
+        {
+            return RACK.getOrDefault(endpoint.getHostAddress(false), "RC1");
+        }
+
+        @Override
+        public String getDatacenter(InetAddressAndPort endpoint)
+        {
+            return DATACENTER.getOrDefault(endpoint.getHostAddress(false), "DC1");
+        }
+    };
+
+    @BeforeClass
+    public static void setSnitch()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setEndpointSnitch(SNITCH);
+    }
+
+    @AfterClass
+    public static void resetSnitch()
+    {
+        DatabaseDescriptor.setEndpointSnitch(null);
+    }
+
+    @After
+    public void resetSnitchState()
+    {
+        RACK.clear();
+        DATACENTER.clear();
+    }
+
+    @Test
+    public void allButOne_shouldBe_2_forReplicationFactor_3()
+    {
+        testAllButOne(simpleStrategy(3), 2);
+    }
+
+    @Test
+    public void allButOne_shouldBe_1_forReplicationFactor_2()
+    {
+        testAllButOne(simpleStrategy(2), 1);
+    }
+
+    @Test
+    public void allButOne_shouldBe_1_forReplicationFactor_1()
+    {
+        testAllButOne(simpleStrategy(1), 1);
+    }
+
+    @Test
+    public void allButOne_shouldBe_1_forLocalStrategy()
+    {
+        testAllButOne(localStrategy(), 1);
+    }
+
+    @Test
+    public void allButOne_shouldBe_8_forReplicationFactor_3_3_3()
+    {
+        testAllButOne(networkTopologyStrategy(3, 3, 3), 8);
+    }
+
+    @Test
+    public void allButOne_shouldBe_11_forEverywhereStrategyOnClusterOf_12() throws Exception
+    {
+        testAllButOne(everywhereStrategy(
+                        dc(1, Pair.create("192.168.0.1", "A"), Pair.create("192.168.0.2", "E"), Pair.create("192.168.0.3", "H"),
+                                Pair.create("192.168.0.4", "C"), Pair.create("192.168.0.5", "I"), Pair.create("192.168.0.6", "J")),
+                        dc(2, Pair.create("192.168.1.1", "B"), Pair.create("192.168.1.2", "G"), Pair.create("192.168.1.3", "L"),
+                                Pair.create("192.168.1.4", "D"), Pair.create("192.168.1.5", "F"), Pair.create("192.168.1.6", "K"))),
+                11);
+    }
+
+    private void testAllButOne(AbstractReplicationStrategy replicationStrategy, int expected)
+    {
+        // when
+        int blockFor = ConsistencyLevel.allButOneFor(replicationStrategy);
+
+        // then
+        assertEquals("number of nodes to block for", expected, blockFor);
+    }
+
+    private static NetworkTopologyStrategy networkTopologyStrategy(int... dc)
+    {
+        Map<String, String> config = new HashMap<>();
+        for (int i = 0; i < dc.length; i++)
+        {
+            config.put("DC" + i, Integer.toString(dc[i]));
+        }
+        return new NetworkTopologyStrategy(KS, new TokenMetadata(), SNITCH, config);
+    }
+
+    private static AbstractReplicationStrategy simpleStrategy(int replicationFactory)
+    {
+        Map<String, String> config = Collections.singletonMap("replication_factor", Integer.toString(replicationFactory));
+        return new SimpleStrategy(KS, new TokenMetadata(), SNITCH, config);
+    }
+
+    @SafeVarargs
+    private static AbstractReplicationStrategy everywhereStrategy(Multimap<InetAddressAndPort, Token>... dcs)
+    {
+        TokenMetadata metadata = new TokenMetadata();
+        for (Multimap<InetAddressAndPort, Token> dc : dcs)
+        {
+            metadata.updateNormalTokens(dc);
+        }
+        return new EverywhereStrategy(KS, metadata, SNITCH, Collections.emptyMap());
+    }
+
+    private static AbstractReplicationStrategy localStrategy()
+    {
+        return new LocalStrategy(KS, new TokenMetadata(), SNITCH, Collections.emptyMap());
+    }
+
+    private static Multimap<InetAddressAndPort, Token> dc(int id, Pair<String, String>... addressToken) throws UnknownHostException
+    {
+        Multimap<InetAddressAndPort, Token> dc = HashMultimap.create();
+        for (Pair<String, String> pair : addressToken)
+        {
+            DATACENTER.put(pair.left, "DC" + id);
+            dc.put(InetAddressAndPort.getByName(pair.left), new OrderPreservingPartitioner.StringToken(pair.right));
+        }
+        return dc;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/CounterCacheTest.java b/test/unit/org/apache/cassandra/db/CounterCacheTest.java
index 2b743a9f59f1..5668ea374c6d 100644
--- a/test/unit/org/apache/cassandra/db/CounterCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/CounterCacheTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.db;
 
+import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.concurrent.ExecutionException;
 
@@ -33,7 +34,10 @@
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.cache.CounterCacheKey;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.rows.CellPath;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
@@ -77,25 +81,25 @@ public void testReadWrite()
         cfs.truncateBlocking();
         CacheService.instance.invalidateCounterCache();
 
-        Clustering<?> c1 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
-        Clustering<?> c2 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
+        Clustering<?> c1 = ClusteringBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
+        Clustering<?> c2 = ClusteringBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
         ColumnMetadata cd = cfs.metadata().getColumn(ByteBufferUtil.bytes("c"));
 
         assertEquals(0, CacheService.instance.counterCache.size());
-        assertNull(cfs.getCachedCounter(bytes(1), c1, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(1), c2, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(2), c1, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(2), c2, cd, null));
-
-        cfs.putCachedCounter(bytes(1), c1, cd, null, ClockAndCount.create(1L, 1L));
-        cfs.putCachedCounter(bytes(1), c2, cd, null, ClockAndCount.create(1L, 2L));
-        cfs.putCachedCounter(bytes(2), c1, cd, null, ClockAndCount.create(2L, 1L));
-        cfs.putCachedCounter(bytes(2), c2, cd, null, ClockAndCount.create(2L, 2L));
-
-        assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(bytes(1), c1, cd, null));
-        assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(bytes(1), c2, cd, null));
-        assertEquals(ClockAndCount.create(2L, 1L), cfs.getCachedCounter(bytes(2), c1, cd, null));
-        assertEquals(ClockAndCount.create(2L, 2L), cfs.getCachedCounter(bytes(2), c2, cd, null));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(1), c1, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(1), c2, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(2), c1, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(2), c2, cd, null)));
+
+        cfs.putCachedCounter(key(cfs, bytes(1), c1, cd, null), ClockAndCount.create(1L, 1L));
+        cfs.putCachedCounter(key(cfs, bytes(1), c2, cd, null), ClockAndCount.create(1L, 2L));
+        cfs.putCachedCounter(key(cfs, bytes(2), c1, cd, null), ClockAndCount.create(2L, 1L));
+        cfs.putCachedCounter(key(cfs, bytes(2), c2, cd, null), ClockAndCount.create(2L, 2L));
+
+        assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(key(cfs, bytes(1), c1, cd, null)));
+        assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(key(cfs, bytes(1), c2, cd, null)));
+        assertEquals(ClockAndCount.create(2L, 1L), cfs.getCachedCounter(key(cfs, bytes(2), c1, cd, null)));
+        assertEquals(ClockAndCount.create(2L, 2L), cfs.getCachedCounter(key(cfs, bytes(2), c2, cd, null)));
     }
 
     @Test
@@ -105,42 +109,42 @@ public void testCounterCacheInvalidate()
         cfs.truncateBlocking();
         CacheService.instance.invalidateCounterCache();
 
-        Clustering<?> c1 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
-        Clustering<?> c2 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
+        Clustering<?> c1 = ClusteringBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
+        Clustering<?> c2 = ClusteringBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
         ColumnMetadata cd = cfs.metadata().getColumn(ByteBufferUtil.bytes("c"));
 
         assertEquals(0, CacheService.instance.counterCache.size());
-        assertNull(cfs.getCachedCounter(bytes(1), c1, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(1), c2, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(2), c1, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(2), c2, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(3), c1, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(3), c2, cd, null));
-
-        cfs.putCachedCounter(bytes(1), c1, cd, null, ClockAndCount.create(1L, 1L));
-        cfs.putCachedCounter(bytes(1), c2, cd, null, ClockAndCount.create(1L, 2L));
-        cfs.putCachedCounter(bytes(2), c1, cd, null, ClockAndCount.create(2L, 1L));
-        cfs.putCachedCounter(bytes(2), c2, cd, null, ClockAndCount.create(2L, 2L));
-        cfs.putCachedCounter(bytes(3), c1, cd, null, ClockAndCount.create(3L, 1L));
-        cfs.putCachedCounter(bytes(3), c2, cd, null, ClockAndCount.create(3L, 2L));
-
-        assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(bytes(1), c1, cd, null));
-        assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(bytes(1), c2, cd, null));
-        assertEquals(ClockAndCount.create(2L, 1L), cfs.getCachedCounter(bytes(2), c1, cd, null));
-        assertEquals(ClockAndCount.create(2L, 2L), cfs.getCachedCounter(bytes(2), c2, cd, null));
-        assertEquals(ClockAndCount.create(3L, 1L), cfs.getCachedCounter(bytes(3), c1, cd, null));
-        assertEquals(ClockAndCount.create(3L, 2L), cfs.getCachedCounter(bytes(3), c2, cd, null));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(1), c1, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(1), c2, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(2), c1, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(2), c2, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(3), c1, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(3), c2, cd, null)));
+
+        cfs.putCachedCounter(key(cfs, bytes(1), c1, cd, null), ClockAndCount.create(1L, 1L));
+        cfs.putCachedCounter(key(cfs, bytes(1), c2, cd, null), ClockAndCount.create(1L, 2L));
+        cfs.putCachedCounter(key(cfs, bytes(2), c1, cd, null), ClockAndCount.create(2L, 1L));
+        cfs.putCachedCounter(key(cfs, bytes(2), c2, cd, null), ClockAndCount.create(2L, 2L));
+        cfs.putCachedCounter(key(cfs, bytes(3), c1, cd, null), ClockAndCount.create(3L, 1L));
+        cfs.putCachedCounter(key(cfs, bytes(3), c2, cd, null), ClockAndCount.create(3L, 2L));
+
+        assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(key(cfs, bytes(1), c1, cd, null)));
+        assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(key(cfs, bytes(1), c2, cd, null)));
+        assertEquals(ClockAndCount.create(2L, 1L), cfs.getCachedCounter(key(cfs, bytes(2), c1, cd, null)));
+        assertEquals(ClockAndCount.create(2L, 2L), cfs.getCachedCounter(key(cfs, bytes(2), c2, cd, null)));
+        assertEquals(ClockAndCount.create(3L, 1L), cfs.getCachedCounter(key(cfs, bytes(3), c1, cd, null)));
+        assertEquals(ClockAndCount.create(3L, 2L), cfs.getCachedCounter(key(cfs, bytes(3), c2, cd, null)));
 
         cfs.invalidateCounterCache(Collections.singleton(new Bounds<Token>(cfs.decorateKey(bytes(1)).getToken(),
                                                                            cfs.decorateKey(bytes(2)).getToken())));
 
         assertEquals(2, CacheService.instance.counterCache.size());
-        assertNull(cfs.getCachedCounter(bytes(1), c1, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(1), c2, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(2), c1, cd, null));
-        assertNull(cfs.getCachedCounter(bytes(2), c2, cd, null));
-        assertEquals(ClockAndCount.create(3L, 1L), cfs.getCachedCounter(bytes(3), c1, cd, null));
-        assertEquals(ClockAndCount.create(3L, 2L), cfs.getCachedCounter(bytes(3), c2, cd, null));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(1), c1, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(1), c2, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(2), c1, cd, null)));
+        assertNull(cfs.getCachedCounter(key(cfs, bytes(2), c2, cd, null)));
+        assertEquals(ClockAndCount.create(3L, 1L), cfs.getCachedCounter(key(cfs, bytes(3), c1, cd, null)));
+        assertEquals(ClockAndCount.create(3L, 2L), cfs.getCachedCounter(key(cfs, bytes(3), c2, cd, null)));
     }
 
     @Test
@@ -166,14 +170,14 @@ public void testSaveLoad() throws ExecutionException, InterruptedException, Writ
         CacheService.instance.counterCache.loadSaved();
         assertEquals(4, CacheService.instance.counterCache.size());
 
-        Clustering<?> c1 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
-        Clustering<?> c2 = CBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
+        Clustering<?> c1 = ClusteringBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(1)).build();
+        Clustering<?> c2 = ClusteringBuilder.create(cfs.metadata().comparator).add(ByteBufferUtil.bytes(2)).build();
         ColumnMetadata cd = cfs.metadata().getColumn(ByteBufferUtil.bytes("c"));
 
-        assertEquals(1L, cfs.getCachedCounter(bytes(1), c1, cd, null).count);
-        assertEquals(2L, cfs.getCachedCounter(bytes(1), c2, cd, null).count);
-        assertEquals(1L, cfs.getCachedCounter(bytes(2), c1, cd, null).count);
-        assertEquals(2L, cfs.getCachedCounter(bytes(2), c2, cd, null).count);
+        assertEquals(1L, cfs.getCachedCounter(key(cfs, bytes(1), c1, cd, null)).count);
+        assertEquals(2L, cfs.getCachedCounter(key(cfs, bytes(1), c2, cd, null)).count);
+        assertEquals(1L, cfs.getCachedCounter(key(cfs, bytes(2), c1, cd, null)).count);
+        assertEquals(2L, cfs.getCachedCounter(key(cfs, bytes(2), c2, cd, null)).count);
     }
 
     @Test
@@ -239,4 +243,8 @@ public void testDisabledSaveLoad() throws ExecutionException, InterruptedExcepti
         }
     }
 
+    private CounterCacheKey key(ColumnFamilyStore cfs, ByteBuffer bytes, Clustering<?> c1, ColumnMetadata cd, CellPath path)
+    {
+        return CounterCacheKey.create(cfs.metadata(), bytes, c1, cd, path);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/CounterMutationCallbackTest.java b/test/unit/org/apache/cassandra/db/CounterMutationCallbackTest.java
new file mode 100644
index 000000000000..d82d49346072
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/CounterMutationCallbackTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessageFlag;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.sensors.SensorsCustomParams;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.sensors.ActiveRequestSensors;
+import org.apache.cassandra.sensors.ActiveSensorsFactory;
+import org.apache.cassandra.sensors.Context;
+import org.apache.cassandra.sensors.RequestSensors;
+import org.apache.cassandra.sensors.Sensor;
+import org.apache.cassandra.sensors.SensorsRegistry;
+import org.apache.cassandra.sensors.Type;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.net.ParamType.TRACE_SESSION;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
+import static org.assertj.core.api.Assertions.assertThat;
+
+@RunWith(Parameterized.class)
+public class CounterMutationCallbackTest
+{
+    private static final String KEYSPACE1 = "CounterMutationCallbackTest";
+    private static final String CF_COUTNER = "Counter";
+    private static final double COUNTER_MUTATION_WRITE_BYTES = 56.0;
+    private static final double COUNTER_MUTATION_INTERNODE_BYTES = 72.0;
+
+    private CopyOnWriteArrayList<Message> capturedOutboundMessages;
+
+    @BeforeClass
+    public static void defineSchema() throws Exception
+    {
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(ActiveSensorsFactory.class.getName());
+        
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(3),
+                                    SchemaLoader.counterCFMD(KEYSPACE1, CF_COUTNER));
+
+        CompactionManager.instance.disableAutoCompaction();
+    }
+
+    @Before
+    public void beforeTest()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE1).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER).metadata());
+
+        capturedOutboundMessages = new CopyOnWriteArrayList<>();
+        MessagingService.instance().outboundSink.add((message, to) -> capturedOutboundMessages.add(message));
+    }
+
+    @After
+    public void afterTest()
+    {
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER).truncateBlocking();
+
+        SensorsRegistry.instance.clear();
+
+        CassandraRelevantProperties.BF_RECREATE_ON_FP_CHANCE_CHANGE.setBoolean(false);
+    }
+
+    @Parameterized.Parameter()
+    public Pair<Integer, Integer> replicaCountAndExpectedSensorValueMultiplier;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object> parameters()
+    {
+        // pairs of (replica count, expected sensor value multiplier)
+        return ImmutableList.of(
+        Pair.create(0, 1), // CL.ANY
+        Pair.create(1, 1), // CL.ONE
+        Pair.create(2, 2),  // CL.TWO
+        Pair.create(3, 3)  // CL.THREE
+        );
+    }
+
+    @Test
+    public void testCounterMutationCallback()
+    {
+        // dummy mutation
+        TableMetadata metadata = MockSchema.newTableMetadata(KEYSPACE1, CF_COUTNER);
+        Mutation mutation = new Mutation(PartitionUpdate.simpleBuilder(metadata, "").build());
+        CounterMutation counterMutation = new CounterMutation(mutation, ConsistencyLevel.ANY); // CL here just for serialization, otherwise ignored
+        Message<CounterMutation> msg = Message.builder(Verb.COUNTER_MUTATION_REQ, counterMutation)
+               .withId(1)
+               .from(FBUtilities.getLocalAddressAndPort())
+               .withCreatedAt(approxTime.now())
+               .withExpiresAt(approxTime.now() + TimeUnit.SECONDS.toNanos(1))
+               .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
+               .withParam(TRACE_SESSION, UUID.randomUUID())
+               .build();
+
+        RequestSensors requestSensors = new ActiveRequestSensors();
+
+        Context context = Context.from(Keyspace.open(KEYSPACE1).getMetadata().tables.get(CF_COUTNER).get());
+        requestSensors.registerSensor(context, Type.INTERNODE_BYTES);
+        requestSensors.registerSensor(context, Type.WRITE_BYTES);
+        requestSensors.incrementSensor(context, Type.WRITE_BYTES, COUNTER_MUTATION_WRITE_BYTES); // mimic a counter mutation of size COUNTER_MUTATION_WRITE_BYTES on the leader node
+        requestSensors.incrementSensor(context, Type.INTERNODE_BYTES, COUNTER_MUTATION_INTERNODE_BYTES); // mimic an inter-node payload of size COUNTER_MUTATION_INTERNODE_BYTES on the leader node
+        requestSensors.syncAllSensors();
+        CounterMutationCallback callback = new CounterMutationCallback(msg, FBUtilities.getLocalAddressAndPort(), requestSensors);
+        Integer replicaCount = replicaCountAndExpectedSensorValueMultiplier.left;
+        callback.setReplicaCount(replicaCount);
+
+        callback.run();
+
+        // Sensor values on the leader should not accommodate for replica sensors
+        Sensor localSensor = requestSensors.getSensor(context, Type.WRITE_BYTES).get();
+        assertThat(localSensor.getValue()).isEqualTo(COUNTER_MUTATION_WRITE_BYTES);
+        Sensor registerSensor = SensorsRegistry.instance.getSensor(context, Type.WRITE_BYTES).get();
+        assertThat(registerSensor.getValue()).isEqualTo(COUNTER_MUTATION_WRITE_BYTES);
+        localSensor = requestSensors.getSensor(context, Type.INTERNODE_BYTES).get();
+        assertThat(localSensor.getValue()).isEqualTo(COUNTER_MUTATION_INTERNODE_BYTES);
+        registerSensor = SensorsRegistry.instance.getSensor(context, Type.INTERNODE_BYTES).get();
+        assertThat(registerSensor.getValue()).isEqualTo(COUNTER_MUTATION_INTERNODE_BYTES);
+
+        // verify custom headers have the sensors values adjusted for the replica count
+        assertThat(capturedOutboundMessages).size().isEqualTo(1);
+        Map<String, byte[]> customParam = capturedOutboundMessages.get(0).header.customParams();
+        assertThat(customParam).isNotNull();
+        int expectedSensorValueMultiplier = replicaCountAndExpectedSensorValueMultiplier.right;
+        assertThat(customParam).hasEntrySatisfying(String.format("WRITE_BYTES_REQUEST.%s.%s", KEYSPACE1, CF_COUTNER),
+                                                   v -> {
+                                                       double actual = SensorsCustomParams.sensorValueFromBytes(v);
+                                                       assertThat(actual).isEqualTo(COUNTER_MUTATION_WRITE_BYTES * expectedSensorValueMultiplier);
+                                                   });
+        assertThat(customParam).hasEntrySatisfying(String.format("WRITE_BYTES_GLOBAL.%s.%s", KEYSPACE1, CF_COUTNER),
+                                                   v -> {
+                                                       double actual = SensorsCustomParams.sensorValueFromBytes(v);
+                                                       assertThat(actual).isEqualTo(COUNTER_MUTATION_WRITE_BYTES * expectedSensorValueMultiplier);
+                                                   });
+        assertThat(customParam).hasEntrySatisfying(String.format("INTERNODE_BYTES_REQUEST.%s.%s", KEYSPACE1, CF_COUTNER),
+                                                   v -> {
+                                                       double actual = SensorsCustomParams.sensorValueFromBytes(v);
+                                                       assertThat(actual).isEqualTo(COUNTER_MUTATION_INTERNODE_BYTES * expectedSensorValueMultiplier);
+                                                   });
+        assertThat(customParam).hasEntrySatisfying(String.format("INTERNODE_BYTES_GLOBAL.%s.%s", KEYSPACE1, CF_COUTNER),
+                                                   v -> {
+                                                       double actual = SensorsCustomParams.sensorValueFromBytes(v);
+                                                       assertThat(actual).isEqualTo(COUNTER_MUTATION_INTERNODE_BYTES * expectedSensorValueMultiplier);
+                                                   });
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/CounterMutationTest.java b/test/unit/org/apache/cassandra/db/CounterMutationTest.java
index 5f25c7218546..aade53335b55 100644
--- a/test/unit/org/apache/cassandra/db/CounterMutationTest.java
+++ b/test/unit/org/apache/cassandra/db/CounterMutationTest.java
@@ -17,10 +17,14 @@
  */
 package org.apache.cassandra.db;
 
+import java.nio.ByteBuffer;
+
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.CounterCacheKey;
+import org.apache.cassandra.db.rows.CellPath;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.db.rows.Row;
@@ -29,6 +33,7 @@
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CounterId;
 
 import static org.junit.Assert.assertEquals;
 
@@ -147,14 +152,14 @@ public void testBatch() throws WriteTimeoutException
         assertEquals(-2L, CounterContext.instance().total(row.getCell(c2cfs2)));
 
         // Check the caches, separately
-        CBuilder cb = CBuilder.create(cfsOne.metadata().comparator);
+        ClusteringBuilder cb = ClusteringBuilder.create(cfsOne.metadata().comparator);
         cb.add("cc");
 
-        assertEquals(1L, cfsOne.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c1cfs1, null).count);
-        assertEquals(-1L, cfsOne.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c2cfs1, null).count);
+        assertEquals(1L, cfsOne.getCachedCounter(key(cfsOne, Util.dk("key1").getKey(), cb.build(), c1cfs1, null)).count);
+        assertEquals(-1L, cfsOne.getCachedCounter(key(cfsOne, Util.dk("key1").getKey(), cb.build(), c2cfs1, null)).count);
 
-        assertEquals(2L, cfsTwo.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c1cfs2, null).count);
-        assertEquals(-2L, cfsTwo.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c2cfs2, null).count);
+        assertEquals(2L, cfsTwo.getCachedCounter(key(cfsTwo, Util.dk("key1").getKey(), cb.build(), c1cfs2, null)).count);
+        assertEquals(-2L, cfsTwo.getCachedCounter(key(cfsTwo, Util.dk("key1").getKey(), cb.build(), c2cfs2, null)).count);
     }
 
     @Test
@@ -215,4 +220,25 @@ public void testDeletes() throws WriteTimeoutException
             ConsistencyLevel.ONE).apply();
         Util.assertEmpty(Util.cmd(cfs).includeRow("cc").columns("val", "val2").build());
     }
+
+    @Test
+    public void testAddingWithoutLocks() throws WriteTimeoutException
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1);
+        cfs.truncateBlocking();
+        ColumnMetadata cDef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
+
+        // Do the initial update (+1)
+        long toAdd = 5;
+        Mutation m = new RowUpdateBuilder(cfs.metadata(), 5, "key1").clustering("cc").add("val", toAdd).build();
+        new CounterMutation(m, ConsistencyLevel.ONE).applyCounterMutationWithoutLocks(1234567, CounterId.getLocalId());
+
+        Row row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val").build());
+        assertEquals(toAdd, CounterContext.instance().total(row.getCell(cDef)));
+    }
+
+    private CounterCacheKey key(ColumnFamilyStore cfs, ByteBuffer bytes, Clustering<?> c1, ColumnMetadata cd, CellPath path)
+    {
+        return CounterCacheKey.create(cfs.metadata(), bytes, c1, cd, path);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java
index 2eb26bc8e9b5..a3de12cce346 100644
--- a/test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java
+++ b/test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java
@@ -111,7 +111,7 @@ public void acceptMutationForNaturalEndpoint() throws Exception
         int value = randomInt();
         int key = 30;
         CounterMutation mutation = mutation(key, value);
-        handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
+        handler.doVerb(Message.builder(Verb.COUNTER_MUTATION_REQ, mutation).from(node1).withId(messageId).build());
 
         // unlike non-counter mutations, we can't verify the response message for a successful write.
         // acting as the leader for the mutation, we'll try to forward the writes to the other replicas
@@ -138,7 +138,7 @@ public void acceptMutationForPendingEndpoint() throws Exception
         int value = randomInt();
         int key = 50;
         CounterMutation mutation = mutation(key, value);
-        handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
+        handler.doVerb(Message.builder(Verb.COUNTER_MUTATION_REQ, mutation).from(node1).withId(messageId).build());
         verifyWrite(key, value);
         assertEquals(startingTotalMetricCount, StorageMetrics.totalOpsForInvalidToken.getCount());
         assertEquals(startingKeyspaceMetricCount, keyspaceMetricValue());
@@ -170,7 +170,7 @@ public void acceptMutationIfRejectionNotEnabled() throws Exception
         // the node which is the actual natural endpoint for this mutation is not a real
         // node, but if we write at CL.ANY we'll generate a hint for it and StorageProxy's
         // counterWriterPerformer will blindly apply the mutation so we can verify it locally
-        handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
+        handler.doVerb(Message.builder(Verb.COUNTER_MUTATION_REQ, mutation).from(node1).withId(messageId).build());
 
         verifyWrite(key, value);
         assertEquals(startingTotalMetricCount + 1, StorageMetrics.totalOpsForInvalidToken.getCount());
diff --git a/test/unit/org/apache/cassandra/db/CustomStorageProviderTest.java b/test/unit/org/apache/cassandra/db/CustomStorageProviderTest.java
new file mode 100644
index 000000000000..ae1cd91837e9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/CustomStorageProviderTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import javax.annotation.Nullable;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.io.storage.StorageProvider;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.DefaultFSErrorHandler;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_STORAGE_PROVIDER;
+
+public class CustomStorageProviderTest
+{
+    static
+    {
+        CUSTOM_STORAGE_PROVIDER.setString(CustomStorageProviderTest.TestCustomStorageProvider.class.getName());
+    }
+
+    private static final String KS = "ks";
+    private static final String TABLE = "cf";
+    private static final String ASSERT_MESSAGE = "Should throw if the directory exists";
+    private static File tempDataDir;
+
+    private TableMetadata tableMetadata;
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        FileUtils.setFSErrorHandler(new DefaultFSErrorHandler());
+    }
+
+    @Before
+    public void beforeEach()
+    {
+        tempDataDir = FileUtils.createTempFile("cassandra", "unittest");
+        tempDataDir.tryDelete(); // hack to create a temp dir
+        tempDataDir.tryCreateDirectory();
+
+        tableMetadata = TableMetadata.builder(KS, TABLE)
+                                     .addPartitionKeyColumn("thekey", UTF8Type.instance)
+                                     .addClusteringColumn("thecolumn", UTF8Type.instance)
+                                     .build();
+    }
+
+    @AfterClass
+    public static void afterClass()
+    {
+        FileUtils.deleteRecursive(tempDataDir);
+        System.clearProperty("cassandra.custom_storage_provider");
+    }
+
+    private static Directories.DataDirectory[] toDataDirectories(File location)
+    {
+        return new Directories.DataDirectory[]{ new Directories.DataDirectory(location) };
+    }
+
+    @Test
+    public void testCustomStorageProvider()
+    {
+        Assertions.assertThat(StorageProvider.instance).isInstanceOf(TestCustomStorageProvider.class);
+        ((TestCustomStorageProvider) StorageProvider.instance).useCustomBehavior = true;
+
+        File newDir = new File(tempDataDir, "testCustomStorageProvider");
+        new Directories(tableMetadata, toDataDirectories(newDir));
+        Assert.assertTrue(Files.exists(newDir.toPath()));
+    }
+
+    @Test
+    public void testDirectoriesMock()
+    {
+        Assertions.assertThat(StorageProvider.instance).isInstanceOf(TestCustomStorageProvider.class);
+        ((TestCustomStorageProvider) StorageProvider.instance).useCustomBehavior = true;
+
+        File newDir = new File(tempDataDir, "testDirectoriesMock");
+        new Directories(tableMetadata, toDataDirectories(newDir));
+
+        // Call to normal constuctor on an existing directory.
+        Assertions.assertThatThrownBy(() -> new Directories(tableMetadata, toDataDirectories(newDir)))
+                  .isInstanceOf(IllegalArgumentException.class)
+                  .hasMessage(ASSERT_MESSAGE);
+
+        // Call to mock constructor, which doesn't call StorageProvider.instance, on an existing directory.
+        new Directories(tableMetadata, newDir.toPath());
+    }
+
+    public static class TestCustomStorageProvider extends StorageProvider.DefaultProvider
+    {
+        // Should be false during initialization, so the defaul behaviour is used
+        boolean useCustomBehavior = false;
+
+        /**
+         * This method is called from Directories constuctor by accessing StorageProvider.instance,
+         * which is expected to be set to this custom storage provider.
+         * The method is overriden with custom behavior that the same directory cannot be created twice.
+         */
+        @Override
+        public Directories.DataDirectory[] createDataDirectories(@Nullable KeyspaceMetadata ksMetadata,
+                                                                 TableMetadata tableMetadata,
+                                                                 Directories.DataDirectory[] dirs)
+        {
+            if (!useCustomBehavior)
+                return super.createDataDirectories(ksMetadata, tableMetadata, dirs);
+
+            for (Directories.DataDirectory d : dirs)
+            {
+                Path dir = d.location.toPath();
+                if (Files.exists(dir))
+                    throw new IllegalArgumentException(ASSERT_MESSAGE);
+                PathUtils.createDirectoriesIfNotExists(dir);
+            }
+            return dirs;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java
index 34a2b83a1266..92804748fecf 100644
--- a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java
+++ b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java
@@ -71,7 +71,7 @@ public void testDeletePartition(DecoratedKey key, boolean flushBeforeRemove, boo
         // validate that data's written
         FilteredPartition partition = Util.getOnlyPartition(Util.cmd(store, key).build());
         assertTrue(partition.rowCount() > 0);
-        Row r = partition.iterator().next();
+        Row r = partition.rowIterator().next();
         assertTrue(r.getCell(column).value().equals(ByteBufferUtil.bytes("asdf")));
 
         if (flushBeforeRemove)
@@ -87,8 +87,8 @@ public void testDeletePartition(DecoratedKey key, boolean flushBeforeRemove, boo
             Util.flush(store);
 
         // validate removal
-        ImmutableBTreePartition partitionUnfiltered = Util.getOnlyPartitionUnfiltered(Util.cmd(store, key).build());
+        Partition partitionUnfiltered = Util.getOnlyPartitionUnfiltered(Util.cmd(store, key).build());
         assertFalse(partitionUnfiltered.partitionLevelDeletion().isLive());
-        assertFalse(partitionUnfiltered.iterator().hasNext());
+        assertFalse(partitionUnfiltered.rowIterator().hasNext());
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
index 1794ae8843e3..7ac07355d590 100644
--- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
+++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
@@ -49,6 +49,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -66,11 +67,17 @@
 import ch.qos.logback.core.read.ListAppender;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaKeyspaceTables;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.auth.AuthKeyspace;
 import org.apache.cassandra.config.Config.DiskFailurePolicy;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.DurationSpec;
-import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.Directories.DataDirectories;
 import org.apache.cassandra.db.Directories.DataDirectory;
@@ -87,11 +94,6 @@
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.schema.Indexes;
-import org.apache.cassandra.schema.MockSchema;
-import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.schema.SchemaKeyspaceTables;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.DefaultFSErrorHandler;
 import org.apache.cassandra.service.snapshot.SnapshotManifest;
 import org.apache.cassandra.service.snapshot.TableSnapshot;
@@ -282,7 +284,7 @@ private List<File> createFakeSSTable(Descriptor desc)
         for (Component c : DatabaseDescriptor.getSelectedSSTableFormat().uploadComponents())
         {
             File f = desc.fileFor(c);
-            f.createFileIfNotExists();
+            assert f.createFileIfNotExists();
             components.add(f);
         }
         return components;
@@ -411,6 +413,22 @@ public void testMaybeManifestLoading() throws Exception {
         }
     }
 
+    @Test
+    public void testResolve() throws IOException
+    {
+        TableMetadata cfm = CFM.iterator().next();
+        Directories directories = new Directories(cfm, toDataDirectories(tempDataDir));
+
+        Descriptor resolved = directories.resolve("me-123-big-Data.db", 0);
+
+        assertEquals(cfm.keyspace, resolved.ksname);
+        assertEquals(cfm.name, resolved.cfname);
+        assertTrue(BigFormat.is(resolved.getFormat()));
+        assertEquals(BigFormat.getInstance().getVersion("me"), resolved.version);
+        assertEquals("123", resolved.id.toString());
+    }
+
+
     @Test
     public void testSecondaryIndexDirectories()
     {
@@ -482,6 +500,17 @@ private File createFile(File file, int size)
         return file;
     }
 
+    @Test
+    public void testVerifyFullPermissions() throws IOException
+    {
+        Assert.assertFalse(Directories.verifyFullPermissions(new File("non_directory.txt")));
+
+        Path tmpDir = Files.createTempDirectory(this.getClass().getSimpleName());
+        File dir = new File(tmpDir, "sub_dir");
+        dir.tryCreateDirectories();
+        Assert.assertTrue(Directories.verifyFullPermissions(dir));
+    }
+
     @Test
     public void testSSTableLister()
     {
@@ -815,6 +844,20 @@ public void testDirectoriesTableSymlink() throws IOException
         testDirectoriesSymlinksHelper(false);
     }
 
+    @Test
+    public void testFileActionHasPrivilege() throws IOException
+    {
+        Path p = Files.createTempDirectory("something");
+        File file = new File(p);
+        assertTrue(Directories.FileAction.hasPrivilege(file, Directories.FileAction.X));
+        assertTrue(Directories.FileAction.hasPrivilege(file, Directories.FileAction.W));
+        assertTrue(Directories.FileAction.hasPrivilege(file, Directories.FileAction.XW));
+        assertTrue(Directories.FileAction.hasPrivilege(file, Directories.FileAction.R));
+        assertTrue(Directories.FileAction.hasPrivilege(file, Directories.FileAction.XR));
+        assertTrue(Directories.FileAction.hasPrivilege(file, Directories.FileAction.RW));
+        assertTrue(Directories.FileAction.hasPrivilege(file, Directories.FileAction.XRW));
+    }
+
     /**
      * Makes sure we can find the data directory for a file when the table directory is a symlink
      *
@@ -837,7 +880,7 @@ private void testDirectoriesSymlinksHelper(boolean oldStyle) throws IOException
         for (TableMetadata tm : CFM)
         {
             Path keyspacedir = Files.createDirectories(ddir2.resolve(tm.keyspace));
-            String tabledir = tm.name + (oldStyle ? "" : Component.separator + tm.id.toHexString());
+            String tabledir = tm.name + (oldStyle ? "" : Component.SEPARATOR + tm.id.toHexString());
             Files.createSymbolicLink(keyspacedir.resolve(tabledir), symlinktarget);
         }
 
@@ -872,30 +915,28 @@ public void testIsStoredInLocalSystemKeyspacesDataLocation()
     public void testDataDirectoriesIterator() throws IOException
     {
         Path tmpDir = Files.createTempDirectory(this.getClass().getSimpleName());
-        Path subDir_1 = Files.createDirectory(tmpDir.resolve("a"));
-        Path subDir_2 = Files.createDirectory(tmpDir.resolve("b"));
-        Path subDir_3 = Files.createDirectory(tmpDir.resolve("c"));
+        File subDir_1 = new File(Files.createDirectory(tmpDir.resolve("a")));
+        File subDir_2 = new File(Files.createDirectory(tmpDir.resolve("b")));
+        File subDir_3 = new File(Files.createDirectory(tmpDir.resolve("c")));
 
-        DataDirectories directories = new DataDirectories(new String[]{subDir_1.toString(), subDir_2.toString()},
-                                                          new String[]{subDir_3.toString()});
+        DataDirectories directories = new DataDirectories(new File[]{subDir_1, subDir_2}, new File[]{subDir_3});
 
         Iterator<DataDirectory> iter = directories.iterator();
         assertTrue(iter.hasNext());
-        assertEquals(new DataDirectory(new File(subDir_1)), iter.next());
+        assertEquals(new DataDirectory(subDir_1), iter.next());
         assertTrue(iter.hasNext());
-        assertEquals(new DataDirectory(new File(subDir_2)), iter.next());
+        assertEquals(new DataDirectory(subDir_2), iter.next());
         assertTrue(iter.hasNext());
-        assertEquals(new DataDirectory(new File(subDir_3)), iter.next());
+        assertEquals(new DataDirectory(subDir_3), iter.next());
         assertFalse(iter.hasNext());
 
-        directories = new DataDirectories(new String[]{subDir_1.toString(), subDir_2.toString()},
-                                                          new String[]{subDir_1.toString()});
+        directories = new DataDirectories(new File[]{subDir_1, subDir_2}, new File[]{subDir_1});
 
         iter = directories.iterator();
         assertTrue(iter.hasNext());
-        assertEquals(new DataDirectory(new File(subDir_1)), iter.next());
+        assertEquals(new DataDirectory(subDir_1), iter.next());
         assertTrue(iter.hasNext());
-        assertEquals(new DataDirectory(new File(subDir_2)), iter.next());
+        assertEquals(new DataDirectory(subDir_2), iter.next());
         assertFalse(iter.hasNext());
     }
 
@@ -1106,7 +1147,7 @@ public void testHasAvailableSpaceSumming()
 
     private String getNewFilename(TableMetadata tm, boolean oldStyle)
     {
-        return tm.keyspace + File.pathSeparator() + tm.name + (oldStyle ? "" : Component.separator + tm.id.toHexString()) + "/na-1-big-Data.db";
+        return tm.keyspace + File.pathSeparator() + tm.name + (oldStyle ? "" : Component.SEPARATOR + tm.id.toHexString()) + "/na-1-big-Data.db";
     }
 
     private List<Directories.DataDirectoryCandidate> getWriteableDirectories(DataDirectory[] dataDirectories, long writeSize)
diff --git a/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java b/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
index 9fa824577144..68f7f92c68e3 100644
--- a/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
@@ -82,7 +82,7 @@ public void setup() throws IOException
     public void getBoundariesTest()
     {
         DiskBoundaries dbv = dbm.getDiskBoundaries(mock);
-        Assert.assertEquals(3, dbv.positions.size());
+        Assert.assertEquals(3, dbv.getPositions().size());
         assertEquals(dbv.directories, dirs.getWriteableLocations());
     }
 
@@ -90,11 +90,11 @@ public void getBoundariesTest()
     public void disallowedDirectoriesTest()
     {
         DiskBoundaries dbv = dbm.getDiskBoundaries(mock);
-        Assert.assertEquals(3, dbv.positions.size());
+        Assert.assertEquals(3, dbv.getPositions().size());
         assertEquals(dbv.directories, dirs.getWriteableLocations());
         DisallowedDirectories.maybeMarkUnwritable(new File(tmpDir, "3"));
         dbv = dbm.getDiskBoundaries(mock);
-        Assert.assertEquals(2, dbv.positions.size());
+        Assert.assertEquals(2, dbv.getPositions().size());
         Assert.assertEquals(Lists.newArrayList(new Directories.DataDirectory(new File(tmpDir, "1")),
                                                new Directories.DataDirectory(new File(tmpDir, "2"))),
                                  dbv.directories);
@@ -126,13 +126,13 @@ public void alterKeyspaceTest() throws Throwable
     @Test
     public void testGetDisksInBounds()
     {
-        List<PartitionPosition> pps = new ArrayList<>();
+        List<Token> pps = new ArrayList<>();
 
-        pps.add(pp(100));
-        pps.add(pp(200));
-        pps.add(pp(Long.MAX_VALUE)); // last position is always the max token
+        pps.add(t(100));
+        pps.add(t(200));
+        pps.add(t(Long.MAX_VALUE)); // last position is always the max token
 
-        DiskBoundaries diskBoundaries = new DiskBoundaries(mock, dirs.getWriteableLocations(), pps, 0, 0);
+        DiskBoundaries diskBoundaries = new DiskBoundaries(mock, dirs.getWriteableLocations(), pps, null, 0);
 
         Assert.assertEquals(Lists.newArrayList(datadirs.get(0)),                  diskBoundaries.getDisksInBounds(dk(10),  dk(50)));
         Assert.assertEquals(Lists.newArrayList(datadirs.get(2)),                  diskBoundaries.getDisksInBounds(dk(250), dk(500)));
@@ -152,7 +152,7 @@ public void testGetDisksInBounds()
     public void testGetDataDirectoriesForFiles()
     {
         int gen = 1;
-        List<Murmur3Partitioner.LongToken> tokens = mock.getDiskBoundaries().positions.stream().map(t -> (Murmur3Partitioner.LongToken)t.getToken()).collect(Collectors.toList());
+        List<Murmur3Partitioner.LongToken> tokens = mock.getDiskBoundaries().getPositions().stream().map(t -> (Murmur3Partitioner.LongToken)t.getToken()).collect(Collectors.toList());
         IPartitioner partitioner = Murmur3Partitioner.instance;
 
         Murmur3Partitioner.LongToken sstableFirstDisk1 = (Murmur3Partitioner.LongToken) partitioner.midpoint(partitioner.getMinimumToken(), tokens.get(0));
diff --git a/test/unit/org/apache/cassandra/db/ImportTest.java b/test/unit/org/apache/cassandra/db/ImportTest.java
index b5e63551069a..5fc118fb617a 100644
--- a/test/unit/org/apache/cassandra/db/ImportTest.java
+++ b/test/unit/org/apache/cassandra/db/ImportTest.java
@@ -40,13 +40,14 @@
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.dht.BootStrapper;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
 import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -55,7 +56,6 @@
 import org.apache.cassandra.io.util.PathUtils;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -252,7 +252,7 @@ private File moveToBackupDir(Set<SSTableReader> sstables, String keyspace, Strin
             sstable.selfRef().release();
             for (File f : sstable.descriptor.directory.tryList())
             {
-                if (f.toString().contains(sstable.descriptor.baseFile().toString()))
+                if (f.toUri().toString().contains(sstable.descriptor.baseFileUri()))
                 {
                     System.out.println("move " + f.toPath() + " to " + backupdir);
                     File moveFileTo = new File(backupdir, f.name());
@@ -593,7 +593,7 @@ public void testImportCacheEnabledWithoutSrcDir() throws Throwable
     @Test
     public void testRefreshCorrupt() throws Throwable
     {
-        createTable("create table %s (id int primary key, d int) WITH caching = { 'keys': 'NONE', 'rows_per_partition': 'ALL' }");
+        createTable("create table %s (id int primary key, d int) WITH compaction = {'class':'SizeTieredCompactionStrategy'} AND caching = { 'keys': 'NONE', 'rows_per_partition': 'ALL' }");
         for (int i = 0; i < 10; i++)
             execute("insert into %s (id, d) values (?, ?)", i, i);
         flush();
@@ -913,13 +913,13 @@ public void skipIndexChecksumOnSAITest() throws Throwable
 
             File[] dataFiles = backupDir.list(f -> f.name().endsWith('-' + BigFormat.Components.DATA.type.repr));
 
-            IndexDescriptor indexDescriptor = IndexDescriptor.create(Descriptor.fromFile(dataFiles[0]),
-                                                                     Murmur3Partitioner.instance,
-                                                                     Schema.instance.getTableMetadata(KEYSPACE, "sai_test").comparator);
-            IndexIdentifier indexIdentifier = new IndexIdentifier(KEYSPACE, "sai_test", "idx1");
+            Descriptor descriptor = Descriptor.fromFile(dataFiles[0]);
+            IndexDescriptor indexDescriptor = IndexDescriptor.empty(descriptor);
+            IndexContext indexContext = SAITester.createIndexContext("idx1", UTF8Type.instance);
 
             // corrupt one of index files
-            try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier))
+            IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+            try (IndexOutputWriter output = components.addOrGet(components.completionMarkerComponent()).openOutput())
             {
                 SAICodecUtils.writeHeader(output);
                 output.writeByte((byte) 0);
diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
index 55baba84695a..a0748cfef9c3 100644
--- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
@@ -36,6 +36,7 @@
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.exceptions.UnknownKeyspaceException;
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.big.BigTableReader;
@@ -45,6 +46,7 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.assertj.core.api.Assertions;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -55,13 +57,13 @@ public class KeyspaceTest extends CQLTester
     // Test needs synchronous table drop to avoid flushes causing flaky failures of testLimitSSTables
 
     @Override
-    protected String createTable(String query)
+    public String createTable(String query)
     {
         return super.createTable(KEYSPACE_PER_TEST, query);
     }
 
     @Override
-    protected UntypedResultSet execute(String query, Object... values)
+    public UntypedResultSet execute(String query, Object... values)
     {
         return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
     }
@@ -453,7 +455,7 @@ public void testLimitSSTables() throws Throwable
             Util.flush(cfs);
         }
 
-        ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear();
+        ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram()).clear();
 
         SinglePartitionReadCommand command = singlePartitionSlice(cfs, "0", slices(cfs, null, 1499, false), 1000);
         int[] expectedValues = new int[500];
@@ -461,16 +463,16 @@ public void testLimitSSTables() throws Throwable
             expectedValues[i] = i + 1000;
         assertRowsInResult(cfs, command, expectedValues);
 
-        assertEquals(5, cfs.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(), 0.1);
-        ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear();
+        assertEquals(5, cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getSnapshot().getMax(), 0.1);
+        ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram()).clear();
 
         command = singlePartitionSlice(cfs, "0", slices(cfs, 1500, 2000, false), 1000);
         for (int i = 0; i < 500; i++)
             expectedValues[i] = i + 1500;
         assertRowsInResult(cfs, command, expectedValues);
 
-        assertEquals(5, cfs.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(), 0.1);
-        ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear();
+        assertEquals(5, cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getSnapshot().getMax(), 0.1);
+        ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram()).clear();
 
         // reverse
         command = singlePartitionSlice(cfs, "0", slices(cfs, 1500, 2000, true), 1000);
@@ -520,13 +522,33 @@ private void validateSliceLarge(ColumnFamilyStore cfs)
         assertRowsInResult(cfs, command);
     }
 
+    @Test(expected = Keyspace.BarrierRejectionException.class)
+    public void testStopMutations() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))");
+        Keyspace keyspace = Keyspace.open(KEYSPACE_PER_TEST);
+        keyspace.stopMutations();
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", 0, 0);
+        fail();
+    }
+
+    @Test
+    public void testSetUnsetInitialized()
+    {
+        // dumb test to make sonar happy
+        Keyspace.unsetInitialized();
+        assertThat(Keyspace.isInitialized()).isFalse();
+        Keyspace.setInitialized();
+        assertThat(Keyspace.isInitialized()).isTrue();
+    }
+
     @Test
     public void shouldThrowOnMissingKeyspace()
     {
         String ksName = "MissingKeyspace";
 
         Assertions.assertThatThrownBy(() -> Keyspace.open(ksName, Schema.instance, false))
-                  .isInstanceOf(AssertionError.class)
-                  .hasMessage("Unknown keyspace " + ksName);
+                  .isInstanceOf(UnknownKeyspaceException.class)
+                  .hasMessage("Could not find a keyspace " + ksName);
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/MultiRangeReadCommandTest.java b/test/unit/org/apache/cassandra/db/MultiRangeReadCommandTest.java
new file mode 100644
index 000000000000..b9946861d0bb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/MultiRangeReadCommandTest.java
@@ -0,0 +1,413 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Iterables;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.reads.ReadCallback;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiRangeReadCommandTest
+{
+    public static final String KEYSPACE1 = "MultiRangeReadCommandTest";
+    public static final String CF_STANDARD1 = "Standard1";
+
+    private static IPartitioner partitioner;
+    private static ColumnFamilyStore cfs;
+
+    @BeforeClass
+    public static void defineSchema() throws ConfigurationException
+    {
+        SchemaLoader.prepareServer();
+        StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        partitioner = DatabaseDescriptor.getPartitioner();
+        SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1));
+        cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
+    }
+
+    @Before
+    public void setup()
+    {
+        cfs.clearUnsafe();
+    }
+
+    @Test
+    public void verifyNotFetchingRemainingRangesOverLimit() throws InterruptedException
+    {
+        int rowCount = 1000;
+        for (int i = 0; i < rowCount; ++i)
+        {
+            RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 10, String.valueOf(i));
+            builder.clustering("c");
+            builder.add("val", String.valueOf(i));
+            builder.build().applyUnsafe();
+        }
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        int tokens = 100;
+        DataLimits limits = DataLimits.cqlLimits(100);
+        PartitionRangeReadCommand partitionRangeCommand = PartitionRangeReadCommand.allDataRead(cfs.metadata(), 10).withUpdatedLimit(limits);
+        MultiRangeReadCommand command = MultiRangeReadCommand.create(partitionRangeCommand, ranges(tokens), true);
+
+        assert cfs.metric != null;
+        SSTableReader sstable = Iterables.getOnlyElement(cfs.getLiveSSTables());
+        long beforeMetricsRecorded = cfs.metric.liveScannedHistogram.tableOrKeyspaceHistogram().getCount();
+        long beforeSSTableRead = sstable.getReadMeter().count();
+        assertEquals(limits.count(), rows(command.executeLocally(command.executionController())).size());
+
+        long metricsRecorded = cfs.metric.liveScannedHistogram.tableOrKeyspaceHistogram().getCount() - beforeMetricsRecorded;
+        assertEquals(1, metricsRecorded);
+
+        long subrangeScanned =  sstable.getReadMeter().count() - beforeSSTableRead;
+        String errorMessage = String.format("Should only query enough ranges to satisfy limit, but queried %d ranges", subrangeScanned);
+        assertTrue( errorMessage, subrangeScanned > 1 && subrangeScanned < tokens);
+    }
+
+    @Test
+    public void testMultiRangeReadResponse()
+    {
+        int rowCount = 1000;
+        for (int i = 0; i < rowCount; ++i)
+        {
+            RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 10, String.valueOf(i));
+            builder.clustering("c");
+            builder.add("val", String.valueOf(i));
+            builder.build().applyUnsafe();
+        }
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        PartitionRangeReadCommand partitionRangeCommand = PartitionRangeReadCommand.allDataRead(cfs.metadata(), FBUtilities.nowInSeconds());
+        List<AbstractBounds<PartitionPosition>> ranges = ranges(100);
+        MultiRangeReadCommand command = MultiRangeReadCommand.create(partitionRangeCommand, ranges, true);
+
+        UnfilteredPartitionIterator data = command.executeLocally(command.executionController());
+        MultiRangeReadResponse response = (MultiRangeReadResponse) command.createResponse(data, null);
+
+        // verify subrange response from multi-range read responses contains all data in the subrange
+        for (AbstractBounds<PartitionPosition> range : ranges)
+        {
+            PartitionRangeReadCommand subrange = partitionRangeCommand.forSubRange(range, false);
+            ReadResponse subrangeResponse = response.subrangeResponse(command, range);
+
+            UnfilteredPartitionIterator actual = subrangeResponse.makeIterator(subrange);
+            UnfilteredPartitionIterator expected = subrange.executeLocally(subrange.executionController());
+            assertData(expected, actual);
+        }
+    }
+
+    @Test(expected = AssertionError.class)
+    public void testEmptyRangesAssertionInCreation()
+    {
+        PartitionRangeReadCommand partitionRangeCommand = PartitionRangeReadCommand.allDataRead(cfs.metadata(), FBUtilities.nowInSeconds());
+        List<AbstractBounds<PartitionPosition>> ranges = Collections.EMPTY_LIST;
+        MultiRangeReadCommand.create(partitionRangeCommand, ranges, true);
+    }
+
+    @Test(expected = AssertionError.class)
+    public void testEmptyHandlersAssertionInCreation()
+    {
+        List<ReadCallback<?, ?>> subrangeHandlers = Collections.EMPTY_LIST;
+        MultiRangeReadCommand.create(subrangeHandlers);
+    }
+
+    @Test
+    public void testIsLimitedToOnePartition()
+    {
+        // Multiple ranges isn't limited
+        assertFalse(command(ranges(2), true).isLimitedToOnePartition());
+
+        // Single row bounds with different keys isn't limited
+        List<AbstractBounds<PartitionPosition>> ranges = new ArrayList<>();
+        ranges.add(new Bounds<>(partitioner.decorateKey(UTF8Type.instance.decompose("B")),
+                                partitioner.decorateKey(UTF8Type.instance.decompose("A"))));
+        assertFalse(command(ranges, true).isLimitedToOnePartition());
+
+        // Single row bounds with different keys is limited
+        ranges = new ArrayList<>();
+        ranges.add(new Bounds<>(partitioner.decorateKey(UTF8Type.instance.decompose("A")),
+                                partitioner.decorateKey(UTF8Type.instance.decompose("A"))));
+        assertTrue(command(ranges, true).isLimitedToOnePartition());
+    }
+
+    @Test
+    public void testUpdatingLimitsIsReflectedInCommand()
+    {
+        ReadCommand command = command(ranges(2), true);
+        assertTrue(command.limits() == DataLimits.NONE);
+        command = command.withUpdatedLimit(DataLimits.cqlLimits(10));
+        assertEquals(10, command.limits().count());
+    }
+
+    @Test
+    public void testIsRangeRequestReturnsFalse()
+    {
+        assertFalse(command(ranges(2), true).isRangeRequest());
+    }
+
+    @Test
+    public void testTimeoutReturnsRangeTimeout()
+    {
+        assertEquals(DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.SECONDS), command(ranges(2), true).getTimeout(TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testSelectsKey()
+    {
+        PartitionRangeReadCommand partitionRangeCommand = PartitionRangeReadCommand.allDataRead(cfs.metadata(), FBUtilities.nowInSeconds());
+        List<AbstractBounds<PartitionPosition>> ranges = new ArrayList<>();
+        ranges.add(new Bounds<>(partitioner.decorateKey(UTF8Type.instance.decompose("A")),
+                                partitioner.decorateKey(UTF8Type.instance.decompose("A"))));
+        MultiRangeReadCommand command = MultiRangeReadCommand.create(partitionRangeCommand, ranges, true);
+
+        assertTrue(command.selectsKey(partitioner.decorateKey(UTF8Type.instance.decompose("A"))));
+        assertFalse(command.selectsKey(partitioner.decorateKey(UTF8Type.instance.decompose("B"))));
+    }
+
+    @Test(expected = AssertionError.class)
+    public void testCannotCreateResponseWithDigestQuery()
+    {
+        PartitionRangeReadCommand partitionRangeCommand = PartitionRangeReadCommand.allDataRead(cfs.metadata(), FBUtilities.nowInSeconds());
+        partitionRangeCommand = partitionRangeCommand.copyAsDigestQuery();
+        MultiRangeReadCommand command = MultiRangeReadCommand.create(partitionRangeCommand, ranges(2), true);
+        UnfilteredPartitionIterator data = command.executeLocally(command.executionController());
+        command.createResponse(data, null);
+    }
+
+    @Test
+    public void testResponseIsNotDigestResponse()
+    {
+        MultiRangeReadCommand command = command(ranges(2), true);
+        MultiRangeReadResponse response = (MultiRangeReadResponse)command.createResponse(command.executeLocally(command.executionController()), null);
+        assertFalse(response.isDigestResponse());
+    }
+
+    @Test
+    public void testResponseIsRepairedDigestConclusiveForLocalResponse()
+    {
+        MultiRangeReadCommand command = command(ranges(2), true);
+        MultiRangeReadResponse response = (MultiRangeReadResponse)command.createResponse(command.executeLocally(command.executionController()), null);
+        assertTrue(response.isRepairedDigestConclusive());
+    }
+
+    @Test
+    public void testRepairedDataDigestIsEmptyForLocalResponse()
+    {
+        MultiRangeReadCommand command = command(ranges(2), true);
+        MultiRangeReadResponse response = (MultiRangeReadResponse)command.createResponse(command.executeLocally(command.executionController()), null);
+        assertFalse(response.repairedDataDigest().hasRemaining());
+    }
+
+    @Test
+    public void testMaybeIncludeRepairedDigestForLocalResponse()
+    {
+        MultiRangeReadCommand command = command(ranges(2), true);
+        MultiRangeReadResponse response = (MultiRangeReadResponse)command.createResponse(command.executeLocally(command.executionController()), null);
+        assertTrue(response.mayIncludeRepairedDigest());
+    }
+
+    @Test(expected = AssertionError.class)
+    public void testCreateWithEmptyRanges()
+    {
+        command(Collections.EMPTY_LIST, true);
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testMultiRangeReadResponseDigest()
+    {
+        MultiRangeReadCommand command = command(ranges(10), true);
+        UnfilteredPartitionIterator data = command.executeLocally(command.executionController());
+        MultiRangeReadResponse response = (MultiRangeReadResponse) command.createResponse(data, null);
+        response.digest(null);
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testMultiRangeReadResponseToDebugString()
+    {
+        MultiRangeReadCommand command = command(ranges(10), true);
+        UnfilteredPartitionIterator data = command.executeLocally(command.executionController());
+        MultiRangeReadResponse response = (MultiRangeReadResponse) command.createResponse(data, null);
+        response.toDebugString(null, null);
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testCreateDigestCommand()
+    {
+        MultiRangeReadCommand command = command(ranges(10), true);
+        command.copyAsDigestQuery();
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testGetPager()
+    {
+        MultiRangeReadCommand command = command(ranges(10), true);
+        command.getPager(null, ProtocolVersion.CURRENT);
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testExecute()
+    {
+        MultiRangeReadCommand command = command(ranges(10), true);
+        command.execute(null, null, Dispatcher.RequestTime.forImmediateExecution());
+    }
+
+    @Test
+    public void testSerializationRoundTrip() throws Exception
+    {
+        for (int tokens : Arrays.asList(2, 3, 5, 10, 63, 128))
+        {
+            List<AbstractBounds<PartitionPosition>> ranges = ranges(tokens);
+
+            for (int i = 0; i < ranges.size() - 1; i++)
+            {
+                for (int j = i + 1; j < ranges.size(); j++)
+                {
+                    testSerializationRoundtrip(ranges.subList(i, j), true);
+                    testSerializationRoundtrip(ranges.subList(i, j), false);
+                }
+            }
+        }
+    }
+
+    private static MultiRangeReadCommand command(List<AbstractBounds<PartitionPosition>> subRanges, boolean isRangeContinuation)
+    {
+        PartitionRangeReadCommand partitionRangeCommand = PartitionRangeReadCommand.allDataRead(cfs.metadata(), FBUtilities.nowInSeconds());
+        return MultiRangeReadCommand.create(partitionRangeCommand, subRanges, isRangeContinuation);
+    }
+
+    private static void testSerializationRoundtrip(List<AbstractBounds<PartitionPosition>> subRanges, boolean isRangeContinuation) throws Exception
+    {
+        MultiRangeReadCommand command = command(subRanges, isRangeContinuation);
+        testSerializationRoundtrip(command, command);
+    }
+
+    private static void testSerializationRoundtrip(MultiRangeReadCommand command, MultiRangeReadCommand expected) throws Exception
+    {
+        DataOutputBuffer output = new DataOutputBuffer();
+        ReadCommand.serializer.serialize(command, output, MessagingService.current_version);
+        assertEquals(ReadCommand.serializer.serializedSize(command, MessagingService.current_version), output.position());
+
+        DataInputPlus input = new DataInputBuffer(output.buffer(), false);
+        MultiRangeReadCommand deserialized = (MultiRangeReadCommand)ReadCommand.serializer.deserialize(input, MessagingService.current_version);
+
+        assertEquals(expected.metadata().id, deserialized.metadata().id);
+        assertEquals(expected.nowInSec(), deserialized.nowInSec());
+        assertEquals(expected.limits(), deserialized.limits());
+        assertEquals(expected.indexQueryPlan == null ? null : expected.indexQueryPlan.getFirst().getIndexMetadata(),
+                     deserialized.indexQueryPlan == null ? null : deserialized.indexQueryPlan.getFirst().getIndexMetadata());
+        assertEquals(expected.digestVersion(), deserialized.digestVersion());
+        assertEquals(expected.ranges().size(), deserialized.ranges().size());
+        Iterator<DataRange> expectedRangeIterator = expected.ranges().iterator();
+        Iterator<DataRange> deserializedRangeIterator = expected.ranges().iterator();
+        while (expectedRangeIterator.hasNext())
+        {
+            DataRange expectedRange = expectedRangeIterator.next();
+            DataRange deserializedRange = deserializedRangeIterator.next();
+            assertEquals(expectedRange.keyRange, deserializedRange.keyRange);
+            assertEquals(expectedRange.clusteringIndexFilter, deserializedRange.clusteringIndexFilter);
+        }
+    }
+
+    private List<AbstractBounds<PartitionPosition>> ranges(int numTokens)
+    {
+        assert numTokens >= 2;
+
+        List<Token> tokens = new ArrayList<>(numTokens);
+        tokens.add(partitioner.getMinimumToken());
+        tokens.add(partitioner.getMaximumToken());
+
+        while (tokens.size() < numTokens)
+        {
+            Token next = partitioner.getRandomToken();
+            if (!tokens.contains(next))
+                tokens.add(next);
+        }
+        Collections.sort(tokens);
+
+        List<AbstractBounds<PartitionPosition>> ranges = new ArrayList<>();
+        for (int i = 0; i < tokens.size() - 1; i++)
+        {
+            Token.KeyBound left = tokens.get(i).maxKeyBound(); // exclusive
+            Token.KeyBound right = tokens.get(i + 1).maxKeyBound(); // inclusive
+            ranges.add(new Range<>(left, right));
+        }
+
+        return ranges;
+    }
+
+    private void assertData(UnfilteredPartitionIterator expectedResult, UnfilteredPartitionIterator actualResult)
+    {
+        List<Unfiltered> expected = rows(expectedResult);
+        List<Unfiltered> actual = rows(actualResult);
+        assertEquals(expected, actual);
+    }
+
+    private List<Unfiltered> rows(UnfilteredPartitionIterator iterator)
+    {
+        List<Unfiltered> unfiltered = new ArrayList<>();
+        while (iterator.hasNext())
+        {
+            try (UnfilteredRowIterator rowIterator = iterator.next())
+            {
+                while (rowIterator.hasNext())
+                {
+                    unfiltered.add(rowIterator.next());
+                }
+            }
+        }
+        iterator.close();
+        return unfiltered;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/NameSortTest.java b/test/unit/org/apache/cassandra/db/NameSortTest.java
index 2fdd73530077..e3e9f50bd478 100644
--- a/test/unit/org/apache/cassandra/db/NameSortTest.java
+++ b/test/unit/org/apache/cassandra/db/NameSortTest.java
@@ -92,7 +92,7 @@ private void validateNameSort(ColumnFamilyStore cfs) throws IOException
     {
         for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).build()))
         {
-            for (Row r : partition)
+            for (Row r : partition.rows())
             {
                 for (ColumnMetadata cd : r.columns())
                 {
diff --git a/test/unit/org/apache/cassandra/db/NativeCellTest.java b/test/unit/org/apache/cassandra/db/NativeCellTest.java
index d93dea42fecc..1fc5205443d5 100644
--- a/test/unit/org/apache/cassandra/db/NativeCellTest.java
+++ b/test/unit/org/apache/cassandra/db/NativeCellTest.java
@@ -117,7 +117,7 @@ private static ColumnMetadata rndcol()
         return new ColumnMetadata("",
                                   "",
                                   ColumnIdentifier.getInterned(uuid.toString(), false),
-                                    isComplex ? new SetType<>(BytesType.instance, true) : BytesType.instance,
+                                    isComplex ? SetType.getInstance(BytesType.instance, true) : BytesType.instance,
                                   -1,
                                   ColumnMetadata.Kind.REGULAR,
                                   null);
diff --git a/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java b/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java
index 20387488ba36..aa8855a810fd 100644
--- a/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java
+++ b/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java
@@ -166,26 +166,26 @@ public void testRangeSliceInclusionExclusion()
         // Start and end inclusive
         partitions = Util.getAll(Util.cmd(cfs).fromKeyIncl("2").toKeyIncl("7").build());
         assertEquals(6, partitions.size());
-        assertEquals(ByteBufferUtil.bytes("2"), partitions.get(0).iterator().next().getCell(cDef).buffer());
-        assertEquals(ByteBufferUtil.bytes("7"), partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).buffer());
+        assertEquals(ByteBufferUtil.bytes("2"), partitions.get(0).rowIterator().next().getCell(cDef).buffer());
+        assertEquals(ByteBufferUtil.bytes("7"), partitions.get(partitions.size() - 1).rowIterator().next().getCell(cDef).buffer());
 
         // Start and end excluded
         partitions = Util.getAll(Util.cmd(cfs).fromKeyExcl("2").toKeyExcl("7").build());
         assertEquals(4, partitions.size());
-        assertEquals(ByteBufferUtil.bytes("3"), partitions.get(0).iterator().next().getCell(cDef).buffer());
-        assertEquals(ByteBufferUtil.bytes("6"), partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).buffer());
+        assertEquals(ByteBufferUtil.bytes("3"), partitions.get(0).rowIterator().next().getCell(cDef).buffer());
+        assertEquals(ByteBufferUtil.bytes("6"), partitions.get(partitions.size() - 1).rowIterator().next().getCell(cDef).buffer());
 
         // Start excluded, end included
         partitions = Util.getAll(Util.cmd(cfs).fromKeyExcl("2").toKeyIncl("7").build());
         assertEquals(5, partitions.size());
-        assertEquals(ByteBufferUtil.bytes("3"), partitions.get(0).iterator().next().getCell(cDef).buffer());
-        assertEquals(ByteBufferUtil.bytes("7"), partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).buffer());
+        assertEquals(ByteBufferUtil.bytes("3"), partitions.get(0).rowIterator().next().getCell(cDef).buffer());
+        assertEquals(ByteBufferUtil.bytes("7"), partitions.get(partitions.size() - 1).rowIterator().next().getCell(cDef).buffer());
 
         // Start included, end excluded
         partitions = Util.getAll(Util.cmd(cfs).fromKeyIncl("2").toKeyExcl("7").build());
         assertEquals(5, partitions.size());
-        assertEquals(ByteBufferUtil.bytes("2"), partitions.get(0).iterator().next().getCell(cDef).buffer());
-        assertEquals(ByteBufferUtil.bytes("6"), partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).buffer());
+        assertEquals(ByteBufferUtil.bytes("2"), partitions.get(0).rowIterator().next().getCell(cDef).buffer());
+        assertEquals(ByteBufferUtil.bytes("6"), partitions.get(partitions.size() - 1).rowIterator().next().getCell(cDef).buffer());
     }
 }
 
diff --git a/test/unit/org/apache/cassandra/db/PartitionTest.java b/test/unit/org/apache/cassandra/db/PartitionTest.java
index edf2a8254ed4..dfda562f52b3 100644
--- a/test/unit/org/apache/cassandra/db/PartitionTest.java
+++ b/test/unit/org/apache/cassandra/db/PartitionTest.java
@@ -132,8 +132,8 @@ public void testDigest(int version) throws NoSuchAlgorithmException
 
             ReadCommand cmd1 = Util.cmd(cfs, "key1").build();
             ReadCommand cmd2 = Util.cmd(cfs, "key2").build();
-            ImmutableBTreePartition p1 = Util.getOnlyPartitionUnfiltered(cmd1);
-            ImmutableBTreePartition p2 = Util.getOnlyPartitionUnfiltered(cmd2);
+            Partition p1 = Util.getOnlyPartitionUnfiltered(cmd1);
+            Partition p2 = Util.getOnlyPartitionUnfiltered(cmd2);
 
             byte[] digest1 = getDigest(p1.unfilteredIterator(), version);
             byte[] digest2 = getDigest(p2.unfilteredIterator(), version);
@@ -178,7 +178,7 @@ public void testColumnStatsRecordsRowDeletesCorrectly()
         builder.build().applyUnsafe();
 
         RowUpdateBuilder.deleteRowAt(cfs.metadata(), 10L, localDeletionTime, "key1", "c").applyUnsafe();
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key1").build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key1").build());
         EncodingStats stats = partition.stats();
         assertEquals(localDeletionTime, stats.minLocalDeletionTime);
     }
diff --git a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
index 2fee75877949..f520d0a628f3 100644
--- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
+++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
@@ -24,9 +24,12 @@
 import java.util.concurrent.Future;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterators;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.UpdateBuilder;
@@ -35,30 +38,50 @@
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.index.StubIndex;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.OrderCheckingIterator;
 
 import static org.apache.cassandra.SchemaLoader.standardCFMD;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+@RunWith(Parameterized.class)
 public class RangeTombstoneTest
 {
     private static final String KSNAME = "RangeTombstoneTest";
     private static final String CFNAME = "StandardInteger1";
+    private static final String CFNAME_INDEXED = "StandardIntegerIndexed";
+    public static final int GC_GRACE = 5000;
+
+    @Parameterized.Parameters(name = "compaction={0}")
+    public static Iterable<CompactionParams> compactionParamSets()
+    {
+        return ImmutableSet.of(CompactionParams.stcs(ImmutableMap.of()),
+                               CompactionParams.ucs(ImmutableMap.of()));
+    }
 
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
@@ -66,7 +89,19 @@ public static void defineSchema() throws ConfigurationException
         SchemaLoader.prepareServer();
         SchemaLoader.createKeyspace(KSNAME,
                                     KeyspaceParams.simple(1),
-                                    standardCFMD(KSNAME, CFNAME, 1, UTF8Type.instance, Int32Type.instance, Int32Type.instance));
+                                    standardCFMD(KSNAME, CFNAME, 1, UTF8Type.instance, Int32Type.instance, Int32Type.instance),
+                                    standardCFMD(KSNAME, CFNAME_INDEXED, 1, UTF8Type.instance, Int32Type.instance, Int32Type.instance));
+    }
+
+    public RangeTombstoneTest(CompactionParams compactionParams)
+    {
+        Keyspace ks = Keyspace.open(KSNAME);
+        ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().compaction(compactionParams).build());
+        cfs.disableAutoCompaction(); // don't trigger compaction at 4 sstables
+        cfs = ks.getColumnFamilyStore(CFNAME_INDEXED);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().compaction(compactionParams).build());
+        cfs.disableAutoCompaction(); // don't trigger compaction at 4 sstables
     }
 
     @Test
@@ -147,7 +182,7 @@ public void rangeTombstoneFilteringTest() throws Exception
 
         new RowUpdateBuilder(cfs.metadata(), 2, key).addRangeTombstone(15, 20).build().applyUnsafe();
 
-        ImmutableBTreePartition partition;
+        Partition partition;
 
         partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(11).toIncl(14).build());
         Collection<RangeTombstone> rt = rangeTombstones(partition);
@@ -218,10 +253,23 @@ public void rangeTombstoneFilteringTest() throws Exception
         assertEquals(2, rt.size());
     }
 
-    private Collection<RangeTombstone> rangeTombstones(ImmutableBTreePartition partition)
+    private Collection<RangeTombstone> rangeTombstones(Partition partition)
     {
         List<RangeTombstone> tombstones = new ArrayList<>();
-        Iterators.addAll(tombstones, partition.deletionInfo().rangeIterator(false));
+        MutableDeletionInfo.Builder deletionInfoBuilder = MutableDeletionInfo.builder(partition.partitionLevelDeletion(),
+                                                                                      partition.metadata().comparator,
+                                                                                      false);
+        try (UnfilteredRowIterator iter = partition.unfilteredIterator())
+        {
+            while (iter.hasNext())
+            {
+                Unfiltered unfiltered = iter.next();
+                if (unfiltered.isRangeTombstoneMarker())
+                    deletionInfoBuilder.add((RangeTombstoneMarker) unfiltered);
+            }
+        }
+        DeletionInfo deletionInfo = deletionInfoBuilder.build();
+        Iterators.addAll(tombstones, deletionInfo.rangeIterator(false));
         return tombstones;
     }
 
@@ -273,7 +321,6 @@ public void testTrackTimesRangeTombstone() throws ExecutionException, Interrupte
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
         cfs.truncateBlocking();
         String key = "rt_times";
-
         long nowInSec = FBUtilities.nowInSeconds();
         new RowUpdateBuilder(cfs.metadata(), nowInSec, 1000L, key).addRangeTombstone(1, 2).build().apply();
         Util.flush(cfs);
@@ -321,6 +368,7 @@ public void test7810() throws ExecutionException, InterruptedException
         Keyspace ks = Keyspace.open(KSNAME);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
         SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build());
+        cfs.truncateBlocking();
 
         String key = "7810";
 
@@ -333,9 +381,78 @@ public void test7810() throws ExecutionException, InterruptedException
         new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(10, 11).build().apply();
         Util.flush(cfs);
 
-        Thread.sleep(5);
+        Thread.sleep(3000);
         cfs.forceMajorCompaction();
-        assertEquals(8, Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).rowCount());
+        checkUnfilteredContains(cfs, key, 8);
+    }
+
+
+    @Test
+    public void testDB4980_mid() throws ExecutionException, InterruptedException
+    {
+        testDB4980("4980_m", 9, 11, 12, 14, 10, 13);
+    }
+
+    @Test
+    public void testDB4980_left() throws ExecutionException, InterruptedException
+    {
+        testDB4980("4980_l", 10, 13, 12, 14, 9, 11);
+    }
+
+    @Test
+    public void testDB4980_right() throws ExecutionException, InterruptedException
+    {
+        testDB4980("4980_r", 9, 11, 10, 13, 12, 14);
+    }
+
+    public void testDB4980(String key, int start1, int end1, int start2, int end2, int start3, int end3) throws ExecutionException, InterruptedException
+    {
+        Keyspace ks = Keyspace.open(KSNAME);
+        ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(GC_GRACE).build());
+        cfs.truncateBlocking();
+
+        UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
+        for (int i = 10; i < 20; i ++)
+            builder.newRow(i).add("val", i);
+        builder.apply();
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        long localTime = FBUtilities.nowInSeconds();
+        new RowUpdateBuilder(cfs.metadata(), localTime - (GC_GRACE + 100), 1, 0, key)
+            .addRangeTombstone(start1, end1)
+            .build()
+            .apply();
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        new RowUpdateBuilder(cfs.metadata(), localTime - (GC_GRACE + 30), 2, 0, key)
+            .addRangeTombstone(start2, end2)
+            .build()
+            .apply();
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        // This one should stay
+        new RowUpdateBuilder(cfs.metadata(), localTime, 3, 0, key)
+            .addRangeTombstone(start3, end3)
+            .build()
+            .apply();
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        cfs.forceMajorCompaction();
+        checkUnfilteredContains(cfs, key, 7);
+    }
+
+    private void checkUnfilteredContains(ColumnFamilyStore cfs, String key, int expected)
+    {
+        assertEquals(1, cfs.getLiveSSTables().size());
+        DecoratedKey dkey = cfs.metadata().partitioner.decorateKey(((AbstractType<String>) cfs.metadata().partitionKeyType).decompose(key));
+        UnfilteredRowIterator iter = cfs.getLiveSSTables().iterator().next().rowIterator(dkey,
+                                                                                         Slices.ALL,
+                                                                                         ColumnFilter.NONE,
+                                                                                         false,
+                                                                                         SSTableReadsListener.NOOP_LISTENER);
+        iter = new OrderCheckingIterator(iter);
+        assertEquals(expected, Iterators.size(iter));
     }
 
     @Test
@@ -344,6 +461,7 @@ public void test7808_1() throws ExecutionException, InterruptedException
         Keyspace ks = Keyspace.open(KSNAME);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
         SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build());
+        cfs.truncateBlocking();
 
         String key = "7808_1";
         UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
@@ -364,6 +482,7 @@ public void test7808_2() throws ExecutionException, InterruptedException
         Keyspace ks = Keyspace.open(KSNAME);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
         SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build());
+        cfs.truncateBlocking();
 
         String key = "7808_2";
         UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
@@ -379,7 +498,7 @@ public void test7808_2() throws ExecutionException, InterruptedException
         Util.flush(cfs);
         Thread.sleep(5);
         cfs.forceMajorCompaction();
-        assertEquals(1, Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).rowCount());
+        checkUnfilteredContains(cfs, key, 1);
     }
 
     @Test
@@ -465,7 +584,7 @@ public void reverseQueryTest() throws Exception
     public void testRowWithRangeTombstonesUpdatesSecondaryIndex() throws Exception
     {
         Keyspace table = Keyspace.open(KSNAME);
-        ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME_INDEXED);
         ByteBuffer key = ByteBufferUtil.bytes("k5");
         ByteBuffer indexedColumnName = ByteBufferUtil.bytes("val");
 
@@ -571,7 +690,7 @@ public void testRangeTombstoneCompaction() throws Exception
     public void testOverwritesToDeletedColumns() throws Exception
     {
         Keyspace table = Keyspace.open(KSNAME);
-        ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME_INDEXED);
         ByteBuffer key = ByteBufferUtil.bytes("k6");
         ByteBuffer indexedColumnName = ByteBufferUtil.bytes("val");
 
diff --git a/test/unit/org/apache/cassandra/db/ReadCommandTest.java b/test/unit/org/apache/cassandra/db/ReadCommandTest.java
index 3701ae419339..54234880194d 100644
--- a/test/unit/org/apache/cassandra/db/ReadCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadCommandTest.java
@@ -46,11 +46,16 @@
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.ReversedType;
 import org.apache.cassandra.db.marshal.SetType;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.DeserializationHelper;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.RowIterator;
 import org.apache.cassandra.db.rows.Unfiltered;
-import org.apache.cassandra.db.rows.DeserializationHelper;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.dht.Range;
@@ -77,6 +82,7 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -359,7 +365,7 @@ public void testSinglePartitionGroupMerge() throws Exception
         List<ByteBuffer> buffers = new ArrayList<>(groups.length);
         long nowInSeconds = FBUtilities.nowInSeconds();
         ColumnFilter columnFilter = ColumnFilter.allRegularColumnsBuilder(cfs.metadata(), false).build();
-        RowFilter rowFilter = RowFilter.create(true);
+        RowFilter.Builder rowFilter = RowFilter.builder(true);
         Slice slice = Slice.make(BufferClusteringBound.BOTTOM, BufferClusteringBound.TOP);
         ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(Slices.with(cfs.metadata().comparator, slice), false);
 
@@ -383,7 +389,7 @@ public void testSinglePartitionGroupMerge() throws Exception
                 {
                     RowUpdateBuilder.deleteRow(cfs.metadata(), FBUtilities.timestampMicros(), ByteBufferUtil.bytes(data[1]), data[2]).apply();
                 }
-                commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter, DataLimits.NONE, Util.dk(data[1]), sliceFilter));
+                commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter.build(), DataLimits.NONE, Util.dk(data[1]), sliceFilter));
             }
 
             Util.flush(cfs);
@@ -469,7 +475,7 @@ public void testSerializer() throws IOException
         ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).includeRow("dd").build();
         int messagingVersion = MessagingService.current_version;
         FakeOutputStream out = new FakeOutputStream();
-        Tracing.instance.newSession(Tracing.TraceType.QUERY);
+        Tracing.instance.newSession(ClientState.forInternalCalls(), Tracing.TraceType.QUERY);
         Message<ReadCommand> messageOut = Message.out(Verb.READ_REQ, readCommand);
         long size = messageOut.serializedSize(messagingVersion);
         Message.serializer.serialize(messageOut, new WrappedDataOutputStreamPlus(out), messagingVersion);
@@ -526,7 +532,7 @@ public void testCountDeletedRows() throws Exception
         List<ByteBuffer> buffers = new ArrayList<>(groups.length);
         long nowInSeconds = FBUtilities.nowInSeconds();
         ColumnFilter columnFilter = ColumnFilter.allRegularColumnsBuilder(cfs.metadata(), false).build();
-        RowFilter rowFilter = RowFilter.create(true);
+        RowFilter.Builder rowFilter = RowFilter.builder(true);
         Slice slice = Slice.make(BufferClusteringBound.BOTTOM, BufferClusteringBound.TOP);
         ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(
                 Slices.with(cfs.metadata().comparator, slice), false);
@@ -552,7 +558,7 @@ public void testCountDeletedRows() throws Exception
                     RowUpdateBuilder.deleteRow(cfs.metadata(), FBUtilities.timestampMicros(),
                             ByteBufferUtil.bytes(data[1]), data[2]).apply();
                 }
-                commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter,
+                commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter.build(),
                         DataLimits.NONE, Util.dk(data[1]), sliceFilter));
             }
 
@@ -572,7 +578,7 @@ public void testCountDeletedRows() throws Exception
             }
         }
 
-        assertEquals(5, cfs.metric.tombstoneScannedHistogram.cf.getSnapshot().getMax());
+        assertEquals(5, cfs.metric.tombstoneScannedHistogram.tableOrKeyspaceHistogram().getSnapshot().getMax());
     }
 
     @Test
@@ -602,7 +608,7 @@ public void testCountWithNoDeletedRow() throws Exception
         List<ByteBuffer> buffers = new ArrayList<>(groups.length);
         long nowInSeconds = FBUtilities.nowInSeconds();
         ColumnFilter columnFilter = ColumnFilter.allRegularColumnsBuilder(cfs.metadata(), false).build();
-        RowFilter rowFilter = RowFilter.create(true);
+        RowFilter.Builder rowFilter = RowFilter.builder(true);
         Slice slice = Slice.make(BufferClusteringBound.BOTTOM, BufferClusteringBound.TOP);
         ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(
                 Slices.with(cfs.metadata().comparator, slice), false);
@@ -628,7 +634,7 @@ public void testCountWithNoDeletedRow() throws Exception
                     RowUpdateBuilder.deleteRow(cfs.metadata(), FBUtilities.timestampMicros(),
                             ByteBufferUtil.bytes(data[1]), data[2]).apply();
                 }
-                commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter,
+                commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter.build(),
                         DataLimits.NONE, Util.dk(data[1]), sliceFilter));
             }
 
@@ -648,7 +654,7 @@ public void testCountWithNoDeletedRow() throws Exception
             }
         }
 
-        assertEquals(1, cfs.metric.tombstoneScannedHistogram.cf.getSnapshot().getMax());
+        assertEquals(1, cfs.metric.tombstoneScannedHistogram.tableOrKeyspaceHistogram().getSnapshot().getMax());
     }
 
     @Test
@@ -937,8 +943,8 @@ private void setGCGrace(ColumnFamilyStore cfs, int gcGrace)
     private long getAndResetOverreadCount(ColumnFamilyStore cfs)
     {
         // always clear the histogram after reading to make comparisons & asserts easier
-        long rows = cfs.metric.repairedDataTrackingOverreadRows.cf.getSnapshot().getMax();
-        ((ClearableHistogram)cfs.metric.repairedDataTrackingOverreadRows.cf).clear();
+        long rows = cfs.metric.repairedDataTrackingOverreadRows.tableOrKeyspaceHistogram().getSnapshot().getMax();
+        ((ClearableHistogram)cfs.metric.repairedDataTrackingOverreadRows.tableOrKeyspaceHistogram()).clear();
         return rows;
     }
 
@@ -1016,7 +1022,7 @@ private void fullyPurgedPartitionCreatesEmptyDigest(ColumnFamilyStore cfs, ReadC
 
         try (ReadExecutionController controller = command.executionController(true))
         {
-            List<ImmutableBTreePartition> partitions = Util.getAllUnfiltered(command, controller);
+            List<Partition> partitions = Util.getAllUnfiltered(command, controller);
             assertEquals(1, partitions.size());
             ByteBuffer digestWithTombstones = controller.getRepairedDataDigest();
             assertTrue(ByteBufferUtil.compareUnsigned(EMPTY_BYTE_BUFFER, digestWithTombstones) != 0);
@@ -1033,7 +1039,7 @@ private void fullyPurgedPartitionCreatesEmptyDigest(ColumnFamilyStore cfs, ReadC
 
         try (ReadExecutionController controller = command.executionController(true))
         {
-            List<ImmutableBTreePartition> partitions = Util.getAllUnfiltered(command, controller);
+            List<Partition> partitions = Util.getAllUnfiltered(command, controller);
             assertTrue(partitions.isEmpty());
             ByteBuffer digestWithoutTombstones = controller.getRepairedDataDigest();
             assertEquals(0, ByteBufferUtil.compareUnsigned(EMPTY_BYTE_BUFFER, digestWithoutTombstones));
@@ -1072,7 +1078,7 @@ public void mixedPurgedAndNonPurgedPartitions()
         
         try (ReadExecutionController controller = command.executionController(true))
         {
-            List<ImmutableBTreePartition> partitions = Util.getAllUnfiltered(command, controller);
+            List<Partition> partitions = Util.getAllUnfiltered(command, controller);
             assertEquals(1, partitions.size());
             digestWithoutPurgedPartition = controller.getRepairedDataDigest();
             assertTrue(ByteBufferUtil.compareUnsigned(EMPTY_BYTE_BUFFER, digestWithoutPurgedPartition) != 0);
@@ -1086,7 +1092,7 @@ public void mixedPurgedAndNonPurgedPartitions()
 
         try (ReadExecutionController controller = command.executionController(true))
         {
-            List<ImmutableBTreePartition> partitions = Util.getAllUnfiltered(command, controller);
+            List<Partition> partitions = Util.getAllUnfiltered(command, controller);
             assertEquals(1, partitions.size());
             ByteBuffer digestWithPurgedPartition = controller.getRepairedDataDigest();
             assertEquals(0, ByteBufferUtil.compareUnsigned(digestWithPurgedPartition, digestWithoutPurgedPartition));
diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java
index d00550432dc3..b65ac6a6fae8 100644
--- a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.db;
 
+import java.util.TreeSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -29,6 +30,7 @@
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
@@ -208,7 +210,7 @@ private static class StubReadCommand extends SinglePartitionReadCommand
                   RowFilter.none(),
                   DataLimits.NONE,
                   key(tmd, key),
-                  null,
+                  new ClusteringIndexNamesFilter(new TreeSet<>(tmd.comparator), false),
                   null,
                   false,
                   null);
diff --git a/test/unit/org/apache/cassandra/db/ReadMessageTest.java b/test/unit/org/apache/cassandra/db/ReadMessageTest.java
index 5b052536fb27..51376ca085df 100644
--- a/test/unit/org/apache/cassandra/db/ReadMessageTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadMessageTest.java
@@ -29,6 +29,7 @@
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.commitlog.CommitLogTestReplayer;
@@ -173,7 +174,7 @@ public void testGetColumn()
         int found = 0;
         for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).build()))
         {
-            for (Row r : partition)
+            for (Row r : partition.rows())
             {
                 if (r.getCell(col).value().equals(ByteBufferUtil.bytes("abcd")))
                     ++found;
@@ -229,6 +230,9 @@ public boolean apply(Mutation mutation)
         {
             for (PartitionUpdate upd : mutation.getPartitionUpdates())
             {
+                if (SchemaConstants.isSystemKeyspace(upd.metadata().keyspace))
+                    continue;
+
                 Row r = upd.getRow(Clustering.make(ByteBufferUtil.bytes("c")));
                 if (r != null)
                 {
diff --git a/test/unit/org/apache/cassandra/db/ReadObserverTest.java b/test/unit/org/apache/cassandra/db/ReadObserverTest.java
new file mode 100644
index 000000000000..15531fe7ca7d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ReadObserverTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.repair.consistent.LocalSessionAccessor;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class ReadObserverTest
+{
+    public static class TestReadObserverFactory implements ReadObserverFactory
+    {
+        static List<Pair<TableMetadata, ReadObserver>> issuedObservers = new CopyOnWriteArrayList<>();
+
+        @Override
+        public ReadObserver create(TableMetadata table)
+        {
+            ReadObserver observer = mock(ReadObserver.class);
+            issuedObservers.add(Pair.create(table, observer));
+            return observer;
+        }
+    }
+
+    private static final String CF = "Standard";
+    private static final String KEYSPACE = "ReadObserverTest";
+
+    @BeforeClass
+    public static void defineSchema() throws ConfigurationException
+    {
+        CassandraRelevantProperties.CUSTOM_READ_OBSERVER_FACTORY.setString(TestReadObserverFactory.class.getName());
+
+        DatabaseDescriptor.daemonInitialization();
+
+        TableMetadata.Builder metadata =
+        TableMetadata.builder(KEYSPACE, CF)
+                     .addPartitionKeyColumn("key", BytesType.instance)
+                     .addStaticColumn("s", AsciiType.instance)
+                     .addClusteringColumn("col", AsciiType.instance)
+                     .addRegularColumn("a", AsciiType.instance)
+                     .addRegularColumn("b", AsciiType.instance)
+                     .caching(CachingParams.CACHE_EVERYTHING);
+
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE,
+                                    KeyspaceParams.simple(1),
+                                    metadata);
+
+        LocalSessionAccessor.startup();
+    }
+
+    @Test
+    public void testObserverCallbacks()
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF);
+
+        new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
+        .clustering("cc")
+        .add("a", ByteBufferUtil.bytes("regular"))
+        .build()
+        .apply();
+
+        new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key"))
+        .add("s", ByteBufferUtil.bytes("static"))
+        .build()
+        .apply();
+
+        ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build();
+        assertFalse(Util.getAll(readCommand).isEmpty());
+
+        List<Pair<TableMetadata, ReadObserver>> observers = TestReadObserverFactory.issuedObservers.stream()
+                                                                                                   .filter(p -> p.left.name.equals(CF))
+                                                                                                   .collect(Collectors.toList());
+        assertEquals(1, observers.size());
+        ReadObserver observer = observers.get(0).right;
+
+        verify(observer).onPartition(eq(Util.dk("key")), eq(DeletionTime.LIVE));
+        verify(observer).onUnfiltered(argThat(Unfiltered::isRow));
+        verify(observer).onStaticRow(argThat(row -> row.columns().stream().allMatch(col -> col.name.toCQLString().equals("s"))));
+        verify(observer).onComplete();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/ReadResponseTest.java b/test/unit/org/apache/cassandra/db/ReadResponseTest.java
index 93ca6a80e47b..5ddc8d90327e 100644
--- a/test/unit/org/apache/cassandra/db/ReadResponseTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadResponseTest.java
@@ -23,8 +23,10 @@
 import java.util.Random;
 
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
@@ -48,6 +50,12 @@ public class ReadResponseTest
     private final Random random = new Random();
     private TableMetadata metadata;
 
+    @BeforeClass
+    public static void setupClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
     @Before
     public void setup()
     {
@@ -211,7 +219,7 @@ private ReadCommand command(int key, TableMetadata metadata)
         return new StubReadCommand(key, metadata, false);
     }
 
-    private static class StubRepairedDataInfo extends RepairedDataInfo
+    public static class StubRepairedDataInfo extends RepairedDataInfo
     {
         private final ByteBuffer repairedDigest;
         private final boolean conclusive;
diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java
index 4445454b0f0d..ac526383be04 100644
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java
@@ -22,6 +22,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Map;
 import java.util.UUID;
 
 import org.junit.Before;
@@ -52,6 +53,8 @@
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 
+import static org.junit.Assert.assertEquals;
+
 @RunWith(Parameterized.class)
 public class RecoveryManagerFlushedTest
 {
@@ -126,8 +129,12 @@ public void testWithFlush() throws Exception
         logger.debug("begin manual replay");
         // replay the commit log (nothing on Standard1 should be replayed since everything was flushed, so only the row on Standard2
         // will be replayed)
-        int replayed = CommitLog.instance.resetUnsafe(false);
-        assert replayed == 1 : "Expecting only 1 replayed mutation, got " + replayed;
+        Map<Keyspace, Integer> replayed = CommitLog.instance.resetUnsafe(false);
+        assertEquals("Expecting only one keyspace with replayed mutations", 1, replayed.size());
+        Keyspace replayedKeyspace = replayed.keySet().iterator().next();
+        Integer keyspaceReplayedCount = replayed.values().iterator().next();
+        assertEquals(String.format("Expecting %s keyspace, not %s", KEYSPACE1, replayedKeyspace.getName()), KEYSPACE1, replayedKeyspace.getName());
+        assertEquals("Expecting only 1 replayed mutation, got " + replayed, 1, (int) keyspaceReplayedCount);
     }
 
     private void insertRow(String cfname, String key)
diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java
index e8444fba4fd9..be948929841f 100644
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java
@@ -36,8 +36,8 @@
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.compress.DeflateCompressor;
 import org.apache.cassandra.io.compress.LZ4Compressor;
@@ -113,7 +113,7 @@ public void testMissingHeader() throws IOException
         keyspace2.getColumnFamilyStore("Standard3").clearUnsafe();
 
         // nuke the header
-        for (File file : new File(DatabaseDescriptor.getCommitLogLocation()).tryList())
+        for (File file : DatabaseDescriptor.getCommitLogLocation().tryList())
         {
             if (file.name().endsWith(".header"))
                 FileUtils.deleteWithConfirm(file);
diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
index 3a805d279727..32ccffe82b82 100644
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
@@ -43,6 +43,7 @@
 import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.CommitLogArchiver;
+import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
 import org.apache.cassandra.db.commitlog.CommitLogReplayer;
 import org.apache.cassandra.db.context.CounterContext;
 import org.apache.cassandra.db.rows.Row;
@@ -237,7 +238,7 @@ public void testRecoverCounter() throws IOException
 
         keyspace1.getColumnFamilyStore("Counter1").clearUnsafe();
 
-        int replayed = CommitLog.instance.resetUnsafe(false);
+        CommitLog.instance.resetUnsafe(false);
 
         ColumnMetadata counterCol = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
         Row row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val").build());
@@ -398,14 +399,14 @@ private static class MockInitiator extends CommitLogReplayer.MutationInitiator
 
         @Override
         protected org.apache.cassandra.utils.concurrent.Future<Integer> initiateMutation(final Mutation mutation,
-                                                                                         final long segmentId,
+                                                                                         final CommitLogDescriptor desc,
                                                                                          final int serializedSize,
                                                                                          final int entryLocation,
                                                                                          final CommitLogReplayer clr)
         {
             final org.apache.cassandra.utils.concurrent.Future<Integer> toWrap =
                 super.initiateMutation(mutation,
-                                       segmentId,
+                                       desc,
                                        serializedSize,
                                        entryLocation,
                                        clr);
diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
index 6270b25eb789..8b40d443c3c3 100644
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
@@ -22,12 +22,23 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogReplayer;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.compress.DeflateCompressor;
 import org.apache.cassandra.io.compress.LZ4Compressor;
@@ -37,14 +48,9 @@
 import org.apache.cassandra.security.EncryptionContext;
 import org.apache.cassandra.security.EncryptionContextGenerator;
 
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Test for the truncate operation.
@@ -93,23 +99,128 @@ public static void defineSchema() throws ConfigurationException
     public void testTruncate() throws IOException
     {
         Keyspace keyspace = Keyspace.open(KEYSPACE1);
-        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
 
         // add a single cell
         new RowUpdateBuilder(cfs.metadata(), 0, "key1")
             .clustering("cc")
             .add("val", "val1")
             .build()
-            .applyUnsafe();
+            .apply();
 
         // Make sure data was written
         assertTrue(Util.getAll(Util.cmd(cfs).build()).size() > 0);
 
         // and now truncate it
         cfs.truncateBlocking();
-        assert 0 != CommitLog.instance.resetUnsafe(false);
+        Map<Keyspace, Integer> replayed = CommitLog.instance.resetUnsafe(false);
+        assertNull("Expected no mutations to be replayed for " + keyspace + " keyspace, but got " + replayed,
+                     replayed.get(keyspace));
 
         // and validate truncation.
         Util.assertEmptyUnfiltered(Util.cmd(cfs).build());
     }
+
+    @Test
+    public void testTruncateWithReplay() throws IOException
+    {
+        // Tests that a the recovery (commitlog replay) in combination with a truncate operation works.
+        //
+        // Test procedure:
+        // 1. add two mutations
+        // 2. perform truncate
+        // 3. add another mutation
+        // 4. replay CL - there must be exactly one replayed mutation and two skipped mutations
+        // 5. truncate again
+        // 6. replay CL - since there was no activity on the CL, there should be no segments and nothing being replayed or skipped
+
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
+
+        // add some data
+        new RowUpdateBuilder(cfs.metadata(), 0, "key1")
+        .clustering("cc")
+        .add("val", "val1")
+        .build()
+        .apply();
+        new RowUpdateBuilder(cfs.metadata(), 0, "key2")
+        .clustering("dd")
+        .add("val", "val2")
+        .build()
+        .apply();
+
+        // Make sure data was written
+        assertEquals(2, Util.getAll(Util.cmd(cfs).build()).size());
+
+        // and now truncate it
+        cfs.truncateBlocking();
+
+        // add another single cell
+        new RowUpdateBuilder(cfs.metadata(), 0, "key3")
+        .clustering("ee")
+        .add("val", "val3")
+        .build()
+        .apply();
+
+        CommitLogReplayer.MutationInitiator originalInitiator = CommitLogReplayer.mutationInitiator;
+        FilteringInitiator filteringInitiator = new FilteringInitiator();
+        CommitLogReplayer.mutationInitiator = filteringInitiator;
+        try
+        {
+            // Expect exactly three records, only one replayed (the 3rd row-update above)
+            CommitLog.instance.resetUnsafe(false);
+            assertEquals(1, filteringInitiator.replayed.get());
+            assertEquals(2, filteringInitiator.skipped.get());
+
+            // and validate truncation.
+            assertEquals(1, Util.getAll(Util.cmd(cfs).build()).size());
+
+            filteringInitiator.reset();
+
+            // another truncate
+            cfs.truncateBlocking();
+
+            // No replayed mutations this time
+            CommitLog.instance.resetUnsafe(false);
+            assertEquals(0, filteringInitiator.replayed.get());
+            assertEquals(0, filteringInitiator.skipped.get());
+        }
+        finally
+        {
+            CommitLogReplayer.mutationInitiator = originalInitiator;
+        }
+
+        // and validate truncation.
+        assertEquals(0, Util.getAll(Util.cmd(cfs).build()).size());
+    }
+
+    private static class FilteringInitiator extends CommitLogReplayer.MutationInitiator
+    {
+        private final AtomicInteger replayed = new AtomicInteger();
+        private final AtomicInteger skipped = new AtomicInteger();
+
+        @Override
+        protected void onReplayed(PartitionUpdate update, long segmentId, int position)
+        {
+            if (KEYSPACE1.equals(update.metadata().keyspace))
+            {
+                replayed.incrementAndGet();
+            }
+        }
+
+        @Override
+        protected void onSkipped(PartitionUpdate update)
+        {
+            if (KEYSPACE1.equals(update.metadata().keyspace))
+            {
+                skipped.incrementAndGet();
+            }
+        }
+
+        public void reset()
+        {
+            replayed.set(0);
+            skipped.set(0);
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/RowCacheTest.java b/test/unit/org/apache/cassandra/db/RowCacheTest.java
index 4b37e69414ae..74e0d0632921 100644
--- a/test/unit/org/apache/cassandra/db/RowCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/RowCacheTest.java
@@ -55,7 +55,9 @@
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ORG_CAFFINITAS_OHC_SEGMENTCOUNT;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public class RowCacheTest
 {
@@ -498,7 +500,7 @@ public void testSSTablesPerReadHistogramWhenRowCache()
         //force flush for confidence that SSTables exists
         Util.flush(cachedStore);
 
-        ((ClearableHistogram)cachedStore.metric.sstablesPerReadHistogram.cf).clear();
+        ((ClearableHistogram)cachedStore.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram()).clear();
 
         for (int i = 0; i < 100; i++)
         {
@@ -506,14 +508,14 @@ public void testSSTablesPerReadHistogramWhenRowCache()
 
             Util.getAll(Util.cmd(cachedStore, key).build());
 
-            long count_before = cachedStore.metric.sstablesPerReadHistogram.cf.getCount();
+            long count_before = cachedStore.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getCount();
             Util.getAll(Util.cmd(cachedStore, key).build());
 
             // check that SSTablePerReadHistogram has been updated by zero,
             // so count has been increased and in a 1/2 of requests there were zero read SSTables
-            long count_after = cachedStore.metric.sstablesPerReadHistogram.cf.getCount();
-            double belowMedian = cachedStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getValue(0.49D);
-            double mean_after = cachedStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getMean();
+            long count_after = cachedStore.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getCount();
+            double belowMedian = cachedStore.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getSnapshot().getValue(0.49D);
+            double mean_after = cachedStore.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getSnapshot().getMean();
             assertEquals("SSTablePerReadHistogram should be updated even key found in row cache", count_before + 1, count_after);
             assertTrue("In half of requests we have not touched SSTables, " +
                        "so 49 percentile (" + belowMedian + ") must be strongly less than 0.9", belowMedian < 0.9D);
@@ -521,7 +523,7 @@ public void testSSTablesPerReadHistogramWhenRowCache()
                        "so mean value (" + mean_after + ") must be strongly less than 1, but greater than 0", mean_after < 0.999D && mean_after > 0.001D);
         }
 
-        assertEquals("Min value of SSTablesPerRead should be zero", 0, cachedStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getMin());
+        assertEquals("Min value of SSTablesPerRead should be zero", 0, cachedStore.metric.sstablesPerReadHistogram.tableOrKeyspaceHistogram().getSnapshot().getMin());
 
         CacheService.instance.setRowCacheCapacityInMB(0);
     }
diff --git a/test/unit/org/apache/cassandra/db/RowTest.java b/test/unit/org/apache/cassandra/db/RowTest.java
index e7f2a25ff2cc..fc23ae69d6d8 100644
--- a/test/unit/org/apache/cassandra/db/RowTest.java
+++ b/test/unit/org/apache/cassandra/db/RowTest.java
@@ -85,13 +85,13 @@ public void setup()
     @Test
     public void testMergeRangeTombstones()
     {
-        PartitionUpdate.Builder update1 = new PartitionUpdate.Builder(metadata, dk, metadata.regularAndStaticColumns(), 1);
+        PartitionUpdate.Builder update1 = PartitionUpdate.builder(metadata, dk, metadata.regularAndStaticColumns(), 1);
         writeRangeTombstone(update1, "1", "11", 123, 123);
         writeRangeTombstone(update1, "2", "22", 123, 123);
         writeRangeTombstone(update1, "3", "31", 123, 123);
         writeRangeTombstone(update1, "4", "41", 123, 123);
 
-        PartitionUpdate.Builder update2 = new PartitionUpdate.Builder(metadata, dk, metadata.regularAndStaticColumns(), 1);
+        PartitionUpdate.Builder update2 = PartitionUpdate.builder(metadata, dk, metadata.regularAndStaticColumns(), 1);
         writeRangeTombstone(update2, "1", "11", 123, 123);
         writeRangeTombstone(update2, "111", "112", 1230, 123);
         writeRangeTombstone(update2, "2", "24", 123, 123);
diff --git a/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java b/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java
index f59c2a97be24..a52064cd637e 100644
--- a/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java
+++ b/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java
@@ -136,7 +136,7 @@ public static Mutation deleteRow(TableMetadata metadata, long timestamp, Object
 
     public static Mutation deleteRowAt(TableMetadata metadata, long timestamp, long localDeletionTime, Object key, Object... clusteringValues)
     {
-        PartitionUpdate.Builder update = new PartitionUpdate.Builder(metadata, makeKey(metadata, key), metadata.regularAndStaticColumns(), 0);
+        PartitionUpdate.Builder update = PartitionUpdate.builder(metadata, makeKey(metadata, key), metadata.regularAndStaticColumns(), 1);
         deleteRow(update, timestamp, localDeletionTime, clusteringValues);
         return new Mutation.PartitionUpdateCollector(update.metadata().keyspace, update.partitionKey()).add(update.build()).build();
     }
diff --git a/test/unit/org/apache/cassandra/db/SSTableIterationTest.java b/test/unit/org/apache/cassandra/db/SSTableIterationTest.java
new file mode 100644
index 000000000000..eb777539eb74
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/SSTableIterationTest.java
@@ -0,0 +1,384 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.compaction.SortedStringTableCursor;
+import org.apache.cassandra.io.sstable.compaction.IteratorFromCursor;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursor;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursorMerger;
+import org.apache.cassandra.io.sstable.compaction.SkipEmptyDataCursor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+public class SSTableIterationTest extends CQLTester
+{
+
+    public static final int PARTITIONS = 50;
+    public static final int ROWS = 100;
+
+    @Test
+    public void testRowIteration() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c))");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        for (int i = 0; i < PARTITIONS; ++i)
+            for (int j = 0; j < ROWS; j++)
+                if (i != j)
+                    execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", i, 0, j, i + j, (long) j);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        assertEquals((ROWS - 1) * PARTITIONS, execute("SELECT * FROM %s").size());
+
+        SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+        assertCursorMatchesScanner(reader);
+    }
+
+    @Test
+    public void testRowIterationWithComplexColumn() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, d set<int>, PRIMARY KEY (a, b, c))");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        for (int i = 0; i < PARTITIONS; ++i)
+            for (int j = 0; j < ROWS; j++)
+                if (i != j)
+                    execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", i, 0, j, ImmutableSet.of(i, j, i + j), (long) j);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        assertEquals((ROWS - 1) * PARTITIONS, execute("SELECT * FROM %s").size());
+
+        SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+        assertCursorMatchesScanner(reader);
+    }
+
+    @Test
+    public void testRowIterationWithDeletion() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c))");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        for (int i = 0; i < PARTITIONS; ++i)
+            for (int j = 0; j < ROWS; j++)
+                if (i != j)
+                    execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", i, 0, j, i + j, (long) j);
+                else
+                    execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ? AND b = ? AND c = ?", (long) j, i, 0, j);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        assertEquals((ROWS - 1) * PARTITIONS, execute("SELECT * FROM %s").size());
+
+        SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+        assertCursorMatchesScanner(reader);
+    }
+
+    @Test
+    public void testRowIterationWithRangeTombstone() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c))");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        for (int i = 0; i < PARTITIONS; ++i)
+            for (int j = 0; j < ROWS; j++)
+                if (i != j)
+                    execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", i, 0, j, i + j, (long) j);
+                else
+                    execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ? AND b = ? AND c >= ? AND c <= ?", (long) j, i, 0, j - 2, j + 2);
+
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        // We deleted two before c=a for a>1, 1 for a=1 and 0 for a=0.
+        assertEquals((ROWS - 3) * PARTITIONS + 3, execute("SELECT * FROM %s").size());
+
+        SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+        assertCursorMatchesScanner(reader);
+    }
+
+    @Test
+    public void testRowIterationWithStatic() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, d int, s int static, PRIMARY KEY (a, b, c))");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        for (int i = 0; i < PARTITIONS; ++i)
+            for (int j = 0; j < ROWS; j++)
+                if (i != j)
+                    if (i % 3 == 1)
+                        execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", i, 0, j, i + j, (long) j);
+                    else
+                        execute("INSERT INTO %s (a, b, c, d, s) VALUES (?, ?, ?, ?, ?) USING TIMESTAMP ?", i, 0, j, i + j, i + j, (long) j);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        assertEquals((ROWS - 1) * PARTITIONS, execute("SELECT * FROM %s").size());
+
+        SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+        assertCursorMatchesScanner(reader);
+    }
+
+    @Test
+    public void testRestrictedIteration() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c))");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        for (int i = 0; i < PARTITIONS; ++i)
+            for (int j = 0; j < ROWS; j++)
+                if (i != j)
+                    execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", i, 0, j, i + j, (long) j);
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        assertEquals((ROWS - 1) * PARTITIONS, execute("SELECT * FROM %s").size());
+
+        Random rand = new Random(1);
+        SSTableReader reader = cfs.getLiveSSTables().iterator().next();
+        assertCursorMatchesScanner(reader, 0.1, 0.1);
+        // one-sided bounds
+        assertCursorMatchesScanner(reader, Double.NaN, 0.7);
+        assertCursorMatchesScanner(reader, 0.6, Double.NaN);
+        // at ends of the sstable (start partition should be removed, end partition should be kept)
+        assertCursorMatchesScanner(reader, 0, 0.5);
+        assertCursorMatchesScanner(reader, 0, Double.NaN);
+        assertCursorMatchesScanner(reader, 0.4, 0);
+        assertCursorMatchesScanner(reader, Double.NaN, 0);
+        // beyond the ends of the sstable
+        assertCursorMatchesScanner(reader, -0.001, 0.3);
+        assertCursorMatchesScanner(reader, 0.7, -0.001);
+        // No bounds variations
+        assertCursorMatchesScanner(reader, Double.NaN, Double.NaN);
+        assertCursorMatchesScanner(reader, -0.001, -0.001);
+        // almost empty
+        assertCursorMatchesScanner(reader, 0.499, 0.5);
+        assertCursorMatchesScanner(reader, 0.0, 0.999);
+        assertCursorMatchesScanner(reader, 0.999, 0);
+        // outside span
+        assertCursorMatchesScanner(reader, Double.NaN, 1.001);
+        assertCursorMatchesScanner(reader, 1.001, Double.NaN);
+        assertCursorMatchesScanner(reader, -0.002, 1.001);
+        assertCursorMatchesScanner(reader, 1.001, -0.002);
+    }
+
+
+
+    @Test
+    public void testIteratorMerge1() throws Throwable
+    {
+        testIteratorMerge(1);
+    }
+
+    @Test
+    public void testIteratorMerge2() throws Throwable
+    {
+        testIteratorMerge(2);
+    }
+
+    @Test
+    public void testIteratorMerge3() throws Throwable
+    {
+        testIteratorMerge(3);
+    }
+
+    @Test
+    public void testIteratorMerge7() throws Throwable
+    {
+        testIteratorMerge(7);
+    }
+
+    @Test
+    public void testIteratorMerge15() throws Throwable
+    {
+        testIteratorMerge(15);
+    }
+
+    public void testIteratorMerge(int sstableCount) throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, d text, s text static, cc set<int>, PRIMARY KEY (a, b, c))");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        cfs.disableAutoCompaction();
+
+        for (int sstable = 0; sstable < sstableCount; ++sstable)
+        {
+            for (int i = 0; i < PARTITIONS; ++i)
+                for (int j = 0; j < ROWS; j++)
+                    if (i != j)
+                    {
+                        if (i % 3 != 1 || (sstable + i) % 5 == 4)
+                            execute("INSERT INTO %s (a, b, c, d, cc) VALUES (?, ?, ?, ?, ?) USING TIMESTAMP ?", i + sstable, 0, j * sstable, "" + i + ":" + j + ":" + sstable, ImmutableSet.of(i, j, sstable), (long) sstable);
+                        else
+                            execute("INSERT INTO %s (a, b, c, d, s) VALUES (?, ?, ?, ?, ?) USING TIMESTAMP ?", i + sstable, 0, j * sstable, "" + i + ":" + j + ":" + sstable, "" + i + ":" + j + ":" + sstable, (long) sstable);
+                    }
+                    else
+                    {
+                        if (i % 3 != 1 || (sstable + i) % 5 == 4)
+                            execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ? AND b = ? AND c >= ? AND c <= ?", (long) sstable, i, 0, j - 2, j + 2);
+                        else
+                            execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ? AND b = ? AND c = ?", (long) sstable, i, 0, j);
+                    }
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        }
+
+        Set<SSTableReader> sstables = cfs.getLiveSSTables();
+        for (SSTableReader rdr : sstables)
+        {
+            assertCursorMatchesScanner(rdr);
+        }
+
+        SSTableCursor mergedCursor = new SSTableCursorMerger(sstables.stream()
+                                                                     .map(SortedStringTableCursor::new)
+                                                                     .collect(Collectors.toList()),
+                                                             cfs.metadata());
+        mergedCursor = new SkipEmptyDataCursor(mergedCursor); // make sure we drop rows that end up empty
+
+        UnfilteredPartitionIterator mergedScanner = UnfilteredPartitionIterators.merge(sstables.stream()
+                                                                                               .map(SSTableReader::getScanner)
+                                                                                               .collect(Collectors.toList()),
+                                                                                       UnfilteredPartitionIterators.MergeListener.NOOP);
+        assertIteratorsEqual(mergedScanner, new IteratorFromCursor(cfs.metadata(), mergedCursor), true);
+    }
+
+
+
+    private void dumpCursor(SSTableCursor cursor, TableMetadata metadata) throws IOException
+    {
+        while (true)
+        {
+            switch (cursor.advance())
+            {
+                case COMPLEX_COLUMN_CELL:
+                    System.out.println("      " + cursor.cell());
+                    break;
+                case COMPLEX_COLUMN:
+                    System.out.println("    " + cursor.column() + " " + cursor.complexColumnDeletion());
+                    break;
+                case SIMPLE_COLUMN:
+                    System.out.println("    " + cursor.cell());
+                    break;
+                case ROW:
+                    System.out.println("  Row @" + cursor.clusteringKey().toString(metadata) + " " + cursor.rowLevelDeletion());
+                    break;
+                case RANGE_TOMBSTONE:
+                    System.out.println("  Range tombstone @" + cursor.clusteringKey().toString(metadata) + " " + cursor.rowLevelDeletion());
+                    break;
+                case PARTITION:
+                    System.out.println("Partition @" + cursor.partitionKey().toString());
+                    break;
+                case EXHAUSTED:
+                    System.out.println("Exhausted\n");
+                    return;
+            }
+        }
+    }
+
+    private void assertCursorMatchesScanner(SSTableReader reader) throws IOException
+    {
+        dumpSSTableCursor(reader);
+        try (UnfilteredPartitionIterator scanner = reader.getScanner();
+             UnfilteredPartitionIterator cursor = new IteratorFromCursor(reader.metadata(), new SortedStringTableCursor(reader)))
+        {
+            assertIteratorsEqual(scanner, cursor, false);
+        }
+    }
+
+    private void assertCursorMatchesScanner(SSTableReader reader, double cutLeft, double cutRight) throws IOException
+    {
+
+        final Token first = reader.first.getToken();
+        final Token last = reader.last.getToken();
+        final IPartitioner partitioner = first.getPartitioner();
+
+        // cut off the given ratio from start and end
+        final Token left = !Double.isNaN(cutLeft) ? partitioner.split(first, last, cutLeft) : partitioner.getMinimumToken();
+        final Token right = !Double.isNaN(cutRight) ? partitioner.split(first, last, 1 - cutRight) : partitioner.getMinimumToken();
+        Range<Token> range = new Range<>(left, right);
+        System.out.println(String.format("\nRange %.3f to %.3f: %s", cutLeft, 1 - cutRight, range));
+        assert !range.isTrulyWrapAround();
+        try (var scanner = reader.getScanner(ImmutableList.of(range));
+             var cursor = new SortedStringTableCursor(reader, range))
+        {
+            System.out.println(String.format("Length %s vs %s", FBUtilities.prettyPrintMemory(scanner.getLengthInBytes()),
+                                             FBUtilities.prettyPrintMemory(cursor.bytesTotal())));
+            System.out.println(String.format("Position %s vs %s", FBUtilities.prettyPrintMemory(scanner.getCurrentPosition()),
+                                             FBUtilities.prettyPrintMemory(cursor.bytesProcessed())));
+
+            var fromCursor = new IteratorFromCursor(reader.metadata(), cursor);
+            assertIteratorsEqual(scanner, fromCursor, false);
+
+            System.out.println(String.format("Length %s vs %s", FBUtilities.prettyPrintMemory(scanner.getLengthInBytes()),
+                                             FBUtilities.prettyPrintMemory(cursor.bytesTotal())));
+            System.out.println(String.format("Position %s vs %s", FBUtilities.prettyPrintMemory(scanner.getCurrentPosition()),
+                                             FBUtilities.prettyPrintMemory(cursor.bytesProcessed())));
+        }
+    }
+
+    private void dumpSSTableCursor(SSTableReader reader) throws IOException
+    {
+        try (SSTableCursor cursor = new SortedStringTableCursor(reader))
+        {
+            dumpCursor(cursor, reader.metadata());
+        }
+    }
+
+    private void assertIteratorsEqual(UnfilteredPartitionIterator scanner, UnfilteredPartitionIterator cursor, boolean dump)
+    {
+        assertEquals(scanner.metadata(), cursor.metadata());
+        while (scanner.hasNext())
+        {
+            assertTrue(cursor.hasNext());
+            try (UnfilteredRowIterator scannerRows = scanner.next();
+                 UnfilteredRowIterator cursorRows = cursor.next())
+            {
+                assertEquals(scannerRows.isEmpty(), cursorRows.isEmpty());
+                assertEquals(scannerRows.partitionKey(), cursorRows.partitionKey());
+                assertEquals(scannerRows.partitionLevelDeletion(), cursorRows.partitionLevelDeletion());
+                assertEquals(scannerRows.metadata(), cursorRows.metadata());
+                assertEquals(scannerRows.staticRow(), cursorRows.staticRow());
+                if (dump) System.out.println("Partition @" + cursorRows.partitionKey());
+
+                if (dump && !cursorRows.staticRow().isEmpty())
+                    System.out.println("  " + cursorRows.staticRow().toString(cursorRows.metadata()));
+
+                while (scannerRows.hasNext())
+                {
+                    assertTrue(cursorRows.hasNext());
+                    Unfiltered next = cursorRows.next();
+                    if (dump) System.out.println("  " + next.toString(cursorRows.metadata()));
+                    assertEquals(scannerRows.next(), next);
+                }
+                assertFalse(cursorRows.hasNext());
+            }
+        }
+        assertFalse(cursor.hasNext());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
index 7b83b50beb30..76c8a2c5295d 100644
--- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
+++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
@@ -27,15 +27,11 @@
 import org.junit.Test;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.cassandra.*;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
-import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.index.sasi.SASIIndex;
-import org.apache.cassandra.schema.*;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
@@ -47,11 +43,34 @@
 import java.util.Collections;
 import java.util.stream.Collectors;
 
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.schema.Types;
+import org.assertj.core.api.Assertions;
+
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
 
 public class SchemaCQLHelperTest extends CQLTester
 {
@@ -128,8 +147,8 @@ public void testUserTypesCQL()
     @Test
     public void testDroppedColumnsCQL()
     {
-        String keyspace = "cql_test_keyspace_dropped_columns";
-        String table = "test_table_dropped_columns";
+        String keyspace = createKeyspaceName();
+        String table = createTableName();
 
         TableMetadata.Builder builder =
         TableMetadata.builder(keyspace, table)
@@ -138,50 +157,82 @@ public void testDroppedColumnsCQL()
                      .addStaticColumn("st1", IntegerType.instance)
                      .addRegularColumn("reg1", IntegerType.instance)
                      .addRegularColumn("reg2", IntegerType.instance)
-                     .addRegularColumn("reg3", IntegerType.instance);
+                     .addRegularColumn("reg3", IntegerType.instance)
+                     .addRegularColumn(ColumnIdentifier.getInterned("Reg3", true), IntegerType.instance); // Mixed case column
 
         ColumnMetadata st1 = builder.getColumn(ByteBufferUtil.bytes("st1"));
         ColumnMetadata reg1 = builder.getColumn(ByteBufferUtil.bytes("reg1"));
         ColumnMetadata reg2 = builder.getColumn(ByteBufferUtil.bytes("reg2"));
         ColumnMetadata reg3 = builder.getColumn(ByteBufferUtil.bytes("reg3"));
+        ColumnMetadata reg3MixedCase = builder.getColumn(ByteBufferUtil.bytes("Reg3"));
 
         builder.removeRegularOrStaticColumn(st1.name)
                .removeRegularOrStaticColumn(reg1.name)
                .removeRegularOrStaticColumn(reg2.name)
-               .removeRegularOrStaticColumn(reg3.name);
+               .removeRegularOrStaticColumn(reg3.name)
+               .removeRegularOrStaticColumn(reg3MixedCase.name);
 
         builder.recordColumnDrop(st1, 5000)
                .recordColumnDrop(reg1, 10000)
                .recordColumnDrop(reg2, 20000)
-               .recordColumnDrop(reg3, 30000);
+               .recordColumnDrop(reg3, 30000)
+               .recordColumnDrop(reg3MixedCase, 40000);
 
         SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), builder);
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
 
-        String expected = "CREATE TABLE IF NOT EXISTS cql_test_keyspace_dropped_columns.test_table_dropped_columns (\n" +
+        String expected = "CREATE TABLE IF NOT EXISTS " + keyspace + '.' + table + " (\n" +
                           "    pk1 varint,\n" +
                           "    ck1 varint,\n" +
-                          "    reg1 varint,\n" +
-                          "    reg3 varint,\n" +
-                          "    reg2 varint,\n" +
-                          "    st1 varint static,\n" +
                           "    PRIMARY KEY (pk1, ck1)\n) WITH ID =";
         String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
 
         assertThat(actual,
                    allOf(startsWith(expected),
-                         containsString("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg1 USING TIMESTAMP 10000;"),
-                         containsString("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg3 USING TIMESTAMP 30000;"),
-                         containsString("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP reg2 USING TIMESTAMP 20000;"),
-                         containsString("ALTER TABLE cql_test_keyspace_dropped_columns.test_table_dropped_columns DROP st1 USING TIMESTAMP 5000;")));
+                         containsString("DROPPED COLUMN RECORD reg1 varint USING TIMESTAMP 10000"),
+                         containsString("DROPPED COLUMN RECORD reg2 varint USING TIMESTAMP 20000"),
+                         containsString("DROPPED COLUMN RECORD reg3 varint USING TIMESTAMP 30000"),
+                         containsString("DROPPED COLUMN RECORD \"Reg3\" varint USING TIMESTAMP 40000"),
+                         containsString("DROPPED COLUMN RECORD st1 varint static USING TIMESTAMP 5000")));
+    }
+
+    @Test
+    public void testDroppedColumnsCQLWithEarlierTimestamp()
+    {
+        String keyspace = createKeyspaceName();
+        String table = createTableName();
+
+        TableMetadata.Builder builder =
+        TableMetadata.builder(keyspace, table)
+                     .addPartitionKeyColumn("pk1", IntegerType.instance)
+                     .addClusteringColumn("ck1", IntegerType.instance)
+                     .addStaticColumn("st1", IntegerType.instance)
+                     .addRegularColumn("reg1", IntegerType.instance)
+                     .addRegularColumn("reg2", IntegerType.instance)
+                     .addRegularColumn("reg3", IntegerType.instance);
+
+        ColumnMetadata st1 = builder.getColumn(ByteBufferUtil.bytes("st1"));
+        builder.removeRegularOrStaticColumn(st1.name);
+
+        String expectedMessage = String.format("Invalid dropped column record for column st1 in %s at 5000: pre-existing record at 1000 is newer", table);
+        try
+        {
+            builder.recordColumnDrop(st1, 5000)
+                   .recordColumnDrop(st1, 1000);
+            fail("Expected an ConfigurationException: " + expectedMessage);
+        }
+        catch (ConfigurationException e)
+        {
+            assertThat(e.getMessage(), containsString(expectedMessage));
+        }
     }
 
     @Test
     public void testReaddedColumns()
     {
-        String keyspace = "cql_test_keyspace_readded_columns";
-        String table = "test_table_readded_columns";
+        String keyspace = createKeyspaceName();
+        String table = createTableName();
 
         TableMetadata.Builder builder =
         TableMetadata.builder(keyspace, table)
@@ -209,21 +260,19 @@ public void testReaddedColumns()
 
         // when re-adding, column is present as both column and as dropped column record.
         String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
-        String expected = "CREATE TABLE IF NOT EXISTS cql_test_keyspace_readded_columns.test_table_readded_columns (\n" +
+        String expected = "CREATE TABLE IF NOT EXISTS " + keyspace + '.' + table + " (\n" +
                           "    pk1 varint,\n" +
                           "    ck1 varint,\n" +
-                          "    reg2 varint,\n" +
-                          "    reg1 varint,\n" +
                           "    st1 varint static,\n" +
+                          "    reg1 varint,\n" +
+                          "    reg2 varint,\n" +
                           "    PRIMARY KEY (pk1, ck1)\n" +
                           ") WITH ID";
 
         assertThat(actual,
                    allOf(startsWith(expected),
-                         containsString("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns DROP reg1 USING TIMESTAMP 10000;"),
-                         containsString("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns ADD reg1 varint;"),
-                         containsString("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns DROP st1 USING TIMESTAMP 20000;"),
-                         containsString("ALTER TABLE cql_test_keyspace_readded_columns.test_table_readded_columns ADD st1 varint static;")));
+                         containsString("DROPPED COLUMN RECORD reg1 varint USING TIMESTAMP 10000"),
+                         containsString("DROPPED COLUMN RECORD st1 varint static USING TIMESTAMP 20000")));
     }
 
     @Test
@@ -295,7 +344,8 @@ public void testCfmOptionsCQL()
         ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
 
         assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata()),
-                   containsString("CLUSTERING ORDER BY (cl1 ASC)\n" +
+                   containsString("AND CLUSTERING ORDER BY (cl1 ASC)\n" +
+                            "    AND DROPPED COLUMN RECORD reg1 ascii USING TIMESTAMP " + droppedTimestamp +"\n" +
                             "    AND additional_write_policy = 'ALWAYS'\n" +
                             "    AND allow_auto_snapshot = true\n" +
                             "    AND bloom_filter_fp_chance = 1.0\n" +
@@ -435,16 +485,14 @@ public void testSnapshot() throws Throwable
                           "    ck1 varint,\n" +
                           "    ck2 varint,\n" +
                           "    reg2 int,\n" +
-                          "    reg1 " + typeC+ ",\n" +
                           "    reg3 int,\n" +
+                          "    reg1 " + typeC + ",\n" +
                           "    PRIMARY KEY ((pk1, pk2), ck1, ck2)\n" +
                           ") WITH ID = " + cfs.metadata.id + "\n" +
-                          "    AND CLUSTERING ORDER BY (ck1 ASC, ck2 DESC)";
+                          "    AND CLUSTERING ORDER BY (ck1 ASC, ck2 DESC)" + "\n" +
+                          "    AND DROPPED COLUMN RECORD reg3 int USING TIMESTAMP 10000";
 
-        assertThat(schema,
-                   allOf(startsWith(expected),
-                         containsString("ALTER TABLE " + keyspace() + "." + tableName + " DROP reg3 USING TIMESTAMP 10000;"),
-                         containsString("ALTER TABLE " + keyspace() + "." + tableName + " ADD reg3 int;")));
+        assertThat(schema, startsWith(expected));
 
         final boolean isIndexLegacy = DatabaseDescriptor.getDefaultSecondaryIndex().equals(CassandraIndex.NAME);
         assertThat(schema, containsString(
@@ -490,16 +538,14 @@ public void testSnapshotWithDroppedColumnsWithoutReAdding() throws Throwable
                           "    ck1 varint,\n" +
                           "    ck2 varint,\n" +
                           "    reg1 int,\n" +
-                          "    reg3 int,\n" +
-                          "    reg2 int,\n" +
                           "    PRIMARY KEY ((pk1, pk2), ck1, ck2)\n" +
                           ") WITH ID = " + cfs.metadata.id + "\n" +
                           "    AND CLUSTERING ORDER BY (ck1 ASC, ck2 DESC)";
 
         assertThat(schema,
                    allOf(startsWith(expected),
-                         containsString("ALTER TABLE " + keyspace() + "." + tableName + " DROP reg2 USING TIMESTAMP 10000;"),
-                         containsString("ALTER TABLE " + keyspace() + "." + tableName + " DROP reg3 USING TIMESTAMP 10000;")));
+                         containsString("DROPPED COLUMN RECORD reg3 int USING TIMESTAMP 10000"),
+                         containsString("DROPPED COLUMN RECORD reg2 int USING TIMESTAMP 10000")));
 
         JsonNode manifest = JsonUtils.JSON_OBJECT_MAPPER.readTree(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).toJavaIOFile());
         JsonNode files = manifest.get("files");
@@ -529,15 +575,13 @@ public void testSnapshotWithDroppedColumnsWithoutReAddingOnSingleKeyTable() thro
         schema = schema.substring(schema.indexOf("CREATE TABLE")); // trim to ensure order
         String expected = "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
                           "    pk1 varint PRIMARY KEY,\n" +
-                          "    reg1 int,\n" +
-                          "    reg3 int,\n" +
-                          "    reg2 int\n" +
+                          "    reg1 int\n" +
                           ") WITH ID = " + cfs.metadata.id + "\n";
 
         assertThat(schema,
                    allOf(startsWith(expected),
-                         containsString("ALTER TABLE " + keyspace() + "." + tableName + " DROP reg2 USING TIMESTAMP 10000;"),
-                         containsString("ALTER TABLE " + keyspace() + "." + tableName + " DROP reg3 USING TIMESTAMP 10000;")));
+                         containsString("DROPPED COLUMN RECORD reg3 int USING TIMESTAMP 10000"),
+                         containsString("DROPPED COLUMN RECORD reg2 int USING TIMESTAMP 10000")));
 
         JsonNode manifest = JsonUtils.JSON_OBJECT_MAPPER.readTree(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).toJavaIOFile());
         JsonNode files = manifest.get("files");
@@ -555,6 +599,57 @@ public void testSystemKsSnapshot()
         Assert.assertFalse(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).exists());
     }
 
+    @Test
+    public void testDroppedType()
+    {
+        String typeA = createType("CREATE TYPE %s (a1 varint, a2 varint, a3 varint);");
+        String typeB = createType("CREATE TYPE %s (b1 frozen<" + typeA + ">, b2 frozen<" + typeA + ">, b3 frozen<" + typeA + ">);");
+
+        String tableName = createTable("CREATE TABLE IF NOT EXISTS %s (" +
+                                       "pk1 varint," +
+                                       "ck1 varint," +
+                                       "reg1 " + typeB + ',' +
+                                       "reg2 varint," +
+                                       "PRIMARY KEY (pk1, ck1));");
+
+        alterTable("ALTER TABLE %s DROP reg1 USING TIMESTAMP 10000;");
+
+        Runnable validate = () -> {
+            try
+            {
+                ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
+                cfs.snapshot(SNAPSHOT);
+                String schema = Files.asCharSource(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).toJavaIOFile(),
+                                                   Charset.defaultCharset()).read();
+
+                Assertions.assertThat(schema)
+                          .startsWith("CREATE TABLE IF NOT EXISTS " + keyspace() + '.' + tableName + " (\n" +
+                                      "    pk1 varint,\n" +
+                                      "    ck1 varint,\n" +
+                                      "    reg2 varint,\n" +
+                                      "    PRIMARY KEY (pk1, ck1)\n)");
+
+                // Note that the dropped record will have converted the initial UDT to a tuple. Further, that tuple
+                // will be genuinely non-frozen (the parsing code will interpret it as non-frozen).
+                Assertions.assertThat(schema)
+                          .contains("DROPPED COLUMN RECORD reg1 tuple<" +
+                                    "frozen<tuple<varint, varint, varint>>, " +
+                                    "frozen<tuple<varint, varint, varint>>, " +
+                                    "frozen<tuple<varint, varint, varint>>>");
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        };
+
+        // Validate before and after the type drop
+        validate.run();
+        schemaChange("DROP TYPE " + keyspace() + '.' + typeB);
+        schemaChange("DROP TYPE " + keyspace() + '.' + typeA);
+        validate.run();
+    }
+
     @Test
     public void testBooleanCompositeKey() throws Throwable
     {
@@ -572,4 +667,44 @@ public void testBooleanCompositeKey() throws Throwable
         execute("insert into %s (t_id, id, ck, nk) VALUES (true, true, false, true)");
         assertRows(execute("select t_id, id, ck, nk from %s"), row(true, true, false, true), row(true, false, false, true));
     }
+
+    @Test
+    public void testParseCreateTableWithDroppedColumns()
+    {
+        String keyspace = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+        String createTable = "CREATE TABLE IF NOT EXISTS %s (\n" +
+                             "    pk1 varint,\n" +
+                             "    ck1 varint,\n" +
+                             "    PRIMARY KEY (pk1, ck1)\n" +
+                             ") WITH ID = 552f4510-b8fd-11eb-aef4-518b3b328020\n" +
+                             "    AND CLUSTERING ORDER BY (ck1 ASC)\n" +
+                             "    AND DROPPED COLUMN RECORD reg1 varint USING TIMESTAMP 10000\n" +
+                             "    AND DROPPED COLUMN RECORD st1 varint static USING TIMESTAMP 5000\n";
+        createTable(keyspace, createTable);
+    }
+
+    @Test
+    public void testParseCreateTableWithDuplicateDroppedColumns()
+    {
+        String keyspace = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+        String createTable = "CREATE TABLE IF NOT EXISTS %s (\n" +
+                             "    pk1 varint,\n" +
+                             "    ck1 varint,\n" +
+                             "    PRIMARY KEY (pk1, ck1)\n" +
+                             ") WITH ID = 552f4510-b8fd-11eb-aef4-518b3b328020\n" +
+                             "    AND CLUSTERING ORDER BY (ck1 ASC)\n" +
+                             "    AND DROPPED COLUMN RECORD reg1 varint USING TIMESTAMP 10000\n" +
+                             "    AND DROPPED COLUMN RECORD reg1 varint static USING TIMESTAMP 5000\n";
+        try
+        {
+            createTable(keyspace, createTable);
+            fail("Expected an InvalidRequestException: Cannot have multiple dropped column record for column reg1");
+        }
+        catch (RuntimeException e)
+        {
+            assertThat(e, instanceOf(org.apache.cassandra.exceptions.InvalidRequestException.class));
+            assertThat(e.getMessage(),
+                       containsString("Cannot have multiple dropped column record for column"));
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
index c13ef6001873..a37d4e06d505 100644
--- a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
@@ -133,7 +133,7 @@ public void testIndexScan()
         int rowCount = 0;
         for (FilteredPartition partition : partitions)
         {
-            for (Row row : partition)
+            for (Row row : partition.rows())
             {
                 ++rowCount;
                 assert ByteBufferUtil.toLong(Util.cell(cfs, row, "birthdate").buffer()) > 1L;
@@ -304,8 +304,7 @@ public void testDeleteOfInconsistentValuesInKeysIndex() throws Exception
 
         // now apply another update, but force the index update to be skipped
         keyspace.apply(new RowUpdateBuilder(cfs.metadata(), 2, "k1").noRowMarker().add("birthdate", 2L).build(),
-                       true,
-                       false);
+                WriteOptions.SKIP_INDEXES_AND_COMMITLOG);
 
         // Now searching the index for either the old or new value should return 0 rows
         // because the new value was not indexed and the old value should be ignored
@@ -317,8 +316,7 @@ public void testDeleteOfInconsistentValuesInKeysIndex() throws Exception
         // now, reset back to the original value, still skipping the index update, to
         // make sure the value was expunged from the index when it was discovered to be inconsistent
         keyspace.apply(new RowUpdateBuilder(cfs.metadata(), 3, "k1").noRowMarker().add("birthdate", 1L).build(),
-                       true,
-                       false);
+                WriteOptions.SKIP_INDEXES_AND_COMMITLOG);
         assertIndexedNone(cfs, col, 1L);
         ColumnFamilyStore indexCfs = cfs.indexManager.getAllIndexColumnFamilyStores().iterator().next();
         assertIndexCfsIsEmpty(indexCfs);
@@ -364,7 +362,7 @@ private void runDeleteOfInconsistentValuesFromCompositeIndexTest(boolean isStati
         if (!isStatic)
             builder = builder.clustering("c");
         builder.add(colName, 20l);
-        keyspace.apply(builder.build(), true, false);
+        keyspace.apply(builder.build(), WriteOptions.SKIP_INDEXES_AND_COMMITLOG);
 
         // Now searching the index for either the old or new value should return 0 rows
         // because the new value was not indexed and the old value should be ignored
@@ -380,7 +378,7 @@ private void runDeleteOfInconsistentValuesFromCompositeIndexTest(boolean isStati
         if (!isStatic)
             builder = builder.clustering("c");
         builder.add(colName, 10L);
-        keyspace.apply(builder.build(), true, false);
+        keyspace.apply(builder.build(), WriteOptions.SKIP_INDEXES_AND_COMMITLOG);
         assertIndexedNone(cfs, col, 20l);
 
         ColumnFamilyStore indexCfs = cfs.indexManager.getAllIndexColumnFamilyStores().iterator().next();
diff --git a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
index 661e3651d389..591f09fc6bc0 100644
--- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
+++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
@@ -19,12 +19,16 @@
 package org.apache.cassandra.db;
 
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import com.google.common.collect.Iterables;
 import com.google.common.io.Files;
 import org.junit.Assert;
 import org.junit.Test;
@@ -34,33 +38,52 @@
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.TupleType;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.BTreeRow;
 import org.apache.cassandra.db.rows.BufferCell;
 import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.exceptions.UnknownColumnException;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException;
 
 public class SerializationHeaderTest
 {
     private static final String KEYSPACE = "SerializationHeaderTest";
 
+    private final static Version versionWithExplicitFrozenTuples;
+    private final static Version versionWithImplicitFrozenTuples;
+
     static
     {
         DatabaseDescriptor.daemonInitialization();
+
+        versionWithImplicitFrozenTuples = BtiFormat.getInstance().getVersion("cb");
+        assertThat(versionWithImplicitFrozenTuples.hasImplicitlyFrozenTuples()).isTrue();
+
+        versionWithExplicitFrozenTuples = BtiFormat.getInstance().getVersion("cc");
+        assertThat(versionWithExplicitFrozenTuples.hasImplicitlyFrozenTuples()).isFalse();
     }
     
     @Test
@@ -94,7 +117,7 @@ public void testWrittenAsDifferentKind() throws Exception
                 Descriptor descriptor = new Descriptor(format.getLatestVersion(), dir, schema.keyspace, schema.name, id.get());
 
                 SerializationHeader header = SerializationHeader.makeWithoutStats(schema);
-                try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
+                try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE, TableMetadataRef.forOfflineTools(schema));
                      SSTableWriter sstableWriter = descriptor.getFormat().getWriterFactory()
                                                              .builder(descriptor)
                                                              .setTableMetadataRef(TableMetadataRef.forOfflineTools(schema))
@@ -110,7 +133,7 @@ public void testWrittenAsDifferentKind() throws Exception
                         Cell<?> cell = BufferCell.live(cd, 1L, value);
                         Clustering<?> clustering = clusteringFunction.apply(value);
                         Row row = BTreeRow.singleCellRow(clustering, cell);
-                        sstableWriter.append(PartitionUpdate.singleRowUpdate(schema, value, row).unfilteredIterator());
+                        sstableWriter.append(PartitionUpdate.singleRowUpdate(schema, schema.partitioner.decorateKey(value), row).unfilteredIterator());
                     }
                     sstableWriter.finish(false);
                     txn.finish();
@@ -155,4 +178,61 @@ public void testWrittenAsDifferentKind() throws Exception
         }
     }
 
+    @Test
+    public void testDecodeFrozenTuplesEncodedAsNonFrozen() throws UnknownColumnException
+    {
+        TupleType multicellTupleType = new TupleType(Arrays.asList(Int32Type.instance, Int32Type.instance), true);
+        AbstractType<?> frozenTupleType = multicellTupleType.freeze();
+
+        TableMetadata metadata = TableMetadata.builder("ks", "tab")
+                                              .addPartitionKeyColumn("k", frozenTupleType)
+                                              .addStaticColumn("s", frozenTupleType)
+                                              .addClusteringColumn("c", frozenTupleType)
+                                              .addRegularColumn("v", frozenTupleType)
+                                              .build();
+
+        SerializationHeader.Component component = SerializationHeader.Component.buildComponentForTools(multicellTupleType,
+                                                                                                       Arrays.asList(multicellTupleType),
+                                                                                                       new LinkedHashMap<>(Map.of(ByteBufferUtil.bytes("s"), multicellTupleType)),
+                                                                                                       new LinkedHashMap<>(Map.of(ByteBufferUtil.bytes("v"), multicellTupleType)),
+                                                                                                       EncodingStats.NO_STATS);
+
+        SerializationHeader header = component.toHeader("asdfa", metadata, versionWithImplicitFrozenTuples, false);
+        assertThat(header.keyType().isMultiCell()).isFalse();
+        assertThat(header.clusteringTypes().get(0).isMultiCell()).isFalse();
+        assertThat(header.columns().statics.iterator().next().type.isMultiCell()).isFalse();
+        assertThat(header.columns().regulars.iterator().next().type.isMultiCell()).isFalse();
+
+        assertThatIllegalArgumentException().isThrownBy(() -> component.toHeader("asdfa", metadata, versionWithExplicitFrozenTuples, false));
+    }
+
+    @Test
+    public void testDecodeDroppedUTDsEncodedAsNonFrozenTuples() throws UnknownColumnException
+    {
+        TupleType multicellTupleType = new TupleType(Arrays.asList(Int32Type.instance, Int32Type.instance), true);
+
+        TableMetadata metadata = TableMetadata.builder("ks", "tab")
+                                              .addPartitionKeyColumn("k", Int32Type.instance)
+                                              .addStaticColumn("s", Int32Type.instance)
+                                              .addClusteringColumn("c", Int32Type.instance)
+                                              .addRegularColumn("v", Int32Type.instance)
+                                              .recordColumnDrop(ColumnMetadata.regularColumn("ks", "tab", "dv", multicellTupleType), 0L)
+                                              .recordColumnDrop(ColumnMetadata.staticColumn("ks", "tab", "ds", multicellTupleType), 0L)
+                                              .build();
+
+        SerializationHeader.Component component = SerializationHeader.Component.buildComponentForTools(multicellTupleType,
+                                                                                                       Arrays.asList(Int32Type.instance),
+                                                                                                       new LinkedHashMap<>(Map.of(ByteBufferUtil.bytes("s"), Int32Type.instance,
+                                                                                                                                  ByteBufferUtil.bytes("ds"), multicellTupleType)),
+                                                                                                       new LinkedHashMap<>(Map.of(ByteBufferUtil.bytes("v"), Int32Type.instance,
+                                                                                                                                  ByteBufferUtil.bytes("dv"), multicellTupleType)),
+                                                                                                       EncodingStats.NO_STATS);
+
+        SerializationHeader header = component.toHeader("tab", metadata, versionWithImplicitFrozenTuples, false);
+        assertThat(Iterables.find(header.columns().regulars, md -> md.name.toString().equals("dv")).type.isMultiCell()).isTrue();
+        assertThat(Iterables.find(header.columns().statics, md -> md.name.toString().equals("ds")).type.isMultiCell()).isTrue();
+
+        assertThatIllegalArgumentException().isThrownBy(() -> component.toHeader("tab", metadata, versionWithExplicitFrozenTuples, false));
+    }
+
 }
diff --git a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
index c65258c34777..6a57e4e3dc4c 100644
--- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
@@ -21,6 +21,7 @@
 package org.apache.cassandra.db;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -38,8 +39,6 @@
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.ColumnIdentifier;
@@ -52,6 +51,8 @@
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.IntegerType;
 import org.apache.cassandra.db.marshal.UTF8Type;
@@ -62,20 +63,28 @@
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.btree.BTreeSet;
+import org.mockito.Mockito;
 
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public class SinglePartitionSliceCommandTest
@@ -231,7 +240,7 @@ private void checkForS(UnfilteredPartitionIterator pi)
         Cell<?> cell = cellIterator.next();
         Assert.assertEquals(s, cell.column());
         Assert.assertEquals(ByteBufferUtil.bytesToHex(cell.buffer()), ByteBufferUtil.bytes("s"), cell.buffer());
-        Assert.assertFalse(cellIterator.hasNext());
+        assertFalse(cellIterator.hasNext());
     }
 
     @Test
@@ -240,7 +249,7 @@ public void staticColumnsAreReturned() throws IOException
         DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
 
         QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, s) VALUES ('k1', 's')");
-        Assert.assertFalse(QueryProcessor.executeInternal("SELECT s FROM ks.tbl WHERE k='k1'").isEmpty());
+        assertFalse(QueryProcessor.executeInternal("SELECT s FROM ks.tbl WHERE k='k1'").isEmpty());
 
         ColumnFilter columnFilter = ColumnFilter.selection(RegularAndStaticColumns.of(s));
         ClusteringIndexSliceFilter sliceFilter = new ClusteringIndexSliceFilter(Slices.NONE, false);
@@ -279,7 +288,7 @@ public void staticColumnsAreReturned() throws IOException
         }
 
         // check (de)serialized iterator for sstable static cell
-        Schema.instance.getColumnFamilyStoreInstance(metadata.id).forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        Schema.instance.getColumnFamilyStoreInstance(metadata.id).forceBlockingFlush(UNIT_TESTS);
         try (ReadExecutionController executionController = cmd.executionController(); UnfilteredPartitionIterator pi = cmd.executeLocally(executionController))
         {
             response = ReadResponse.createDataResponse(pi, cmd, executionController.getRepairedDataInfo());
@@ -481,7 +490,7 @@ public void toCQLStringIsSafeToCall() throws IOException
                                                             sliceFilter);
         String ret = cmd.toCQLString();
         Assert.assertNotNull(ret);
-        Assert.assertFalse(ret.isEmpty());
+        assertFalse(ret.isEmpty());
     }
 
     public static UnfilteredRowIterator getIteratorFromSinglePartition(String q)
@@ -555,7 +564,7 @@ public void sstableFiltering()
         RangeTombstone rt = new RangeTombstone(slice, DeletionTime.build(TimeUnit.MILLISECONDS.toMicros(nowMillis),
                                                                        Ints.checkedCast(TimeUnit.MILLISECONDS.toSeconds(nowMillis))));
 
-        PartitionUpdate.Builder builder = new PartitionUpdate.Builder(metadata, bb(100), metadata.regularAndStaticColumns(), 1);
+        PartitionUpdate.Builder builder = PartitionUpdate.builder(metadata, metadata.partitioner.decorateKey(bb(100)), metadata.regularAndStaticColumns(), 1);
         builder.add(rt);
         new Mutation(builder.build()).apply();
 
@@ -566,6 +575,171 @@ public void sstableFiltering()
 
     }
 
+    @Test
+    public void testLowerBoundApplicableSingleColumnAsc()
+    {
+        String query = "INSERT INTO %s.%s (k, i) VALUES ('k1', %s)";
+        SSTableReader sstable = createSSTable(metadata, KEYSPACE, TABLE, query);
+        assertEquals(Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(0)),
+                                Util.clustering(metadata.comparator, BigInteger.valueOf(9))),
+                     sstable.getSSTableMetadata().coveredClustering);
+        DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+
+        Slice slice1 = Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(3)).asStartBound(), ClusteringBound.TOP);
+        assertFalse(lowerBoundApplicable(metadata, key, slice1, sstable, false));
+        assertTrue(lowerBoundApplicable(metadata, key, slice1, sstable, true));
+
+        Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, BigInteger.valueOf(3)).asEndBound());
+        assertTrue(lowerBoundApplicable(metadata, key, slice2, sstable, false));
+        assertFalse(lowerBoundApplicable(metadata, key, slice2, sstable, true));
+
+        // corner cases
+        Slice slice3 = Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(0)).asStartBound(), ClusteringBound.TOP);
+        assertFalse(lowerBoundApplicable(metadata, key, slice3, sstable, false));
+        assertTrue(lowerBoundApplicable(metadata, key, slice3, sstable, true));
+
+        Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, BigInteger.valueOf(9)).asEndBound());
+        assertTrue(lowerBoundApplicable(metadata, key, slice4, sstable, false));
+        assertFalse(lowerBoundApplicable(metadata, key, slice4, sstable, true));
+    }
+
+    @Test
+    public void testLowerBoundApplicableSingleColumnDesc()
+    {
+        String TABLE_REVERSED = "tbl_reversed";
+        String createTable = String.format(
+        "CREATE TABLE %s.%s (k text, i varint, v int, primary key (k, i)) WITH CLUSTERING ORDER BY (i DESC)",
+        KEYSPACE, TABLE_REVERSED);
+        QueryProcessor.executeOnceInternal(createTable);
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE_REVERSED);
+        TableMetadata metadata = cfs.metadata();
+        String query = "INSERT INTO %s.%s (k, i) VALUES ('k1', %s)";
+        SSTableReader sstable = createSSTable(metadata, KEYSPACE, TABLE_REVERSED, query);
+        assertEquals(Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(9)),
+                                Util.clustering(metadata.comparator, BigInteger.valueOf(0))),
+                     sstable.getSSTableMetadata().coveredClustering);
+        DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+
+        Slice slice1 = Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(8)).asStartBound(), ClusteringBound.TOP);
+        assertFalse(lowerBoundApplicable(metadata, key, slice1, sstable, false));
+        assertTrue(lowerBoundApplicable(metadata, key, slice1, sstable, true));
+
+        Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, BigInteger.valueOf(8)).asEndBound());
+        assertTrue(lowerBoundApplicable(metadata, key, slice2, sstable, false));
+        assertFalse(lowerBoundApplicable(metadata, key, slice2, sstable, true));
+
+        // corner cases
+        Slice slice3 = Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(9)).asStartBound(), ClusteringBound.TOP);
+        assertFalse(lowerBoundApplicable(metadata, key, slice3, sstable, false));
+        assertTrue(lowerBoundApplicable(metadata, key, slice3, sstable, true));
+
+        Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, BigInteger.valueOf(0)).asEndBound());
+        assertTrue(lowerBoundApplicable(metadata, key, slice4, sstable, false));
+        assertFalse(lowerBoundApplicable(metadata, key, slice4, sstable, true));
+    }
+
+    @Test
+    public void testLowerBoundApplicableMultipleColumnsAsc()
+    {
+        String query = "INSERT INTO %s.%s (k, c1, c2) VALUES ('k1', 0, %s)";
+        SSTableReader sstable = createSSTable(CFM_SLICES, KEYSPACE, TABLE_SCLICES, query);
+        assertEquals(Slice.make(Util.clustering(CFM_SLICES.comparator, 0, 0),
+                                Util.clustering(CFM_SLICES.comparator, 0, 9)),
+                     sstable.getSSTableMetadata().coveredClustering);
+        DecoratedKey key = CFM_SLICES.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+
+        Slice slice1 = Slice.make(Util.clustering(CFM_SLICES.comparator, 0, 3).asStartBound(), ClusteringBound.TOP);
+        assertFalse(lowerBoundApplicable(CFM_SLICES, key, slice1, sstable, false));
+        assertTrue(lowerBoundApplicable(CFM_SLICES, key, slice1, sstable, true));
+
+        Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(CFM_SLICES.comparator, 0, 3).asEndBound());
+        assertTrue(lowerBoundApplicable(CFM_SLICES, key, slice2, sstable, false));
+        assertFalse(lowerBoundApplicable(CFM_SLICES, key, slice2, sstable, true));
+
+        // corner cases
+        Slice slice3 = Slice.make(Util.clustering(CFM_SLICES.comparator, 0, 0).asStartBound(), ClusteringBound.TOP);
+        assertFalse(lowerBoundApplicable(CFM_SLICES, key, slice3, sstable, false));
+        assertTrue(lowerBoundApplicable(CFM_SLICES, key, slice3, sstable, true));
+
+        Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(CFM_SLICES.comparator, 0, 9).asEndBound());
+        assertTrue(lowerBoundApplicable(CFM_SLICES, key, slice4, sstable, false));
+        assertFalse(lowerBoundApplicable(CFM_SLICES, key, slice4, sstable, true));
+    }
+
+    @Test
+    public void testLowerBoundApplicableMultipleColumnsDesc()
+    {
+        String TABLE_REVERSED = "tbl_slices_reversed";
+        String createTable = String.format(
+        "CREATE TABLE %s.%s (k text, c1 int, c2 int, v int, primary key (k, c1, c2)) WITH CLUSTERING ORDER BY (c1 ASC, c2 DESC)",
+        KEYSPACE, TABLE_REVERSED);
+        QueryProcessor.executeOnceInternal(createTable);
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE_REVERSED);
+        TableMetadata metadata = cfs.metadata();
+
+        String query = "INSERT INTO %s.%s (k, c1, c2) VALUES ('k1', 0, %s)";
+        SSTableReader sstable = createSSTable(metadata, KEYSPACE, TABLE_REVERSED, query);
+        assertEquals(Slice.make(Util.clustering(metadata.comparator, 0, 9),
+                                Util.clustering(metadata.comparator, 0, 0)),
+                     sstable.getSSTableMetadata().coveredClustering);
+        DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+
+        Slice slice1 = Slice.make(Util.clustering(metadata.comparator, 0, 8).asStartBound(), ClusteringBound.TOP);
+        assertFalse(lowerBoundApplicable(metadata, key, slice1, sstable, false));
+        assertTrue(lowerBoundApplicable(metadata, key, slice1, sstable, true));
+
+        Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, 0, 8).asEndBound());
+        assertTrue(lowerBoundApplicable(metadata, key, slice2, sstable, false));
+        assertFalse(lowerBoundApplicable(metadata, key, slice2, sstable, true));
+
+        // corner cases
+        Slice slice3 = Slice.make(Util.clustering(metadata.comparator, 0, 9).asStartBound(), ClusteringBound.TOP);
+        assertFalse(lowerBoundApplicable(metadata, key, slice3, sstable, false));
+        assertTrue(lowerBoundApplicable(metadata, key, slice3, sstable, true));
+
+        Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, 0, 0).asEndBound());
+        assertTrue(lowerBoundApplicable(metadata, key, slice4, sstable, false));
+        assertFalse(lowerBoundApplicable(metadata, key, slice4, sstable, true));
+    }
+
+    private SSTableReader createSSTable(TableMetadata metadata, String keyspace, String table, String query)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        for (int i = 0; i < 10; i++)
+            QueryProcessor.executeInternal(String.format(query, keyspace, table, i));
+        cfs.forceBlockingFlush(UNIT_TESTS);
+        DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+        ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, key));
+        assertEquals(1, view.sstables.size());
+        return view.sstables.get(0);
+    }
+
+    private boolean lowerBoundApplicable(TableMetadata metadata, DecoratedKey key, Slice slice, SSTableReader sstable, boolean isReversed)
+    {
+        Slices.Builder slicesBuilder = new Slices.Builder(metadata.comparator);
+        slicesBuilder.add(slice);
+        Slices slices = slicesBuilder.build();
+        ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, isReversed);
+
+        SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(metadata,
+                                                                           FBUtilities.nowInSeconds(),
+                                                                           ColumnFilter.all(metadata),
+                                                                           RowFilter.none(),
+                                                                           DataLimits.NONE,
+                                                                           key,
+                                                                           filter);
+
+        try (UnfilteredRowIteratorWithLowerBound iter = new UnfilteredRowIteratorWithLowerBound(key,
+                                                                                                sstable,
+                                                                                                slices,
+                                                                                                isReversed,
+                                                                                                ColumnFilter.all(metadata),
+                                                                                                Mockito.mock(SSTableReadsListener.class)))
+        {
+            return iter.lowerBound() != null;
+        }
+    }
+
     private String toString(List<Unfiltered> unfiltereds, TableMetadata metadata)
     {
         return unfiltereds.stream().map(u -> u.toString(metadata, true)).collect(Collectors.toList()).toString();
diff --git a/test/unit/org/apache/cassandra/db/SortedLocalRangesTest.java b/test/unit/org/apache/cassandra/db/SortedLocalRangesTest.java
new file mode 100644
index 000000000000..f94866550a58
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/SortedLocalRangesTest.java
@@ -0,0 +1,221 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Random;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.dht.Splitter;
+import org.apache.cassandra.dht.SplitterTest;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import static org.apache.cassandra.dht.SplitterTest.getSplitter;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
+
+public class SortedLocalRangesTest
+{
+    final private static Random random = new Random(1047504572034957L);
+
+    @Mock
+    ColumnFamilyStore cfs;
+
+    @Mock
+    Keyspace keyspace;
+
+    @Mock
+    StorageService storageService;
+
+    @Mock
+    TokenMetadata tmd;
+
+    @Mock
+    AbstractReplicationStrategy replicationStrategy;
+
+    IPartitioner partitioner;
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+    }
+
+    @Before
+    public void setUp() throws IllegalAccessException
+    {
+        MockitoAnnotations.initMocks(this);
+
+        partitioner = DatabaseDescriptor.getPartitioner();
+
+        when(cfs.getPartitioner()).thenReturn(partitioner);
+        when(cfs.getKeyspaceName()).thenReturn("keyspace");
+        when(cfs.getTableName()).thenReturn("table");
+
+        FBUtilities.getProtectedField(ColumnFamilyStore.class, "keyspace").set(cfs, keyspace);
+
+        when(cfs.getKeyspaceReplicationStrategy()).thenReturn(replicationStrategy);
+        when(storageService.getTokenMetadataForKeyspace(eq("keyspace"))).thenReturn(tmd);
+        when(replicationStrategy.getTokenMetadata()).thenReturn(tmd);
+    }
+
+    SortedLocalRanges makeRanges(long ringVersion, List<Splitter.WeightedRange> ranges)
+    {
+        when(tmd.getRingVersion()).thenReturn(ringVersion);
+        return new SortedLocalRanges(cfs, ringVersion, ranges);
+    }
+
+    @Test
+    public void testNoRanges()
+    {
+        long ringVersion = 1;
+        List<Splitter.WeightedRange> ranges = ImmutableList.of();
+        SortedLocalRanges sortedRanges = makeRanges(ringVersion, ranges);
+
+        assertNotNull(sortedRanges);
+        assertNotNull(sortedRanges.toString());
+
+        assertEquals(sortedRanges, sortedRanges);
+        assertEquals(sortedRanges.hashCode(), sortedRanges.hashCode());
+
+        assertEquals(sortedRanges.getRanges(), ranges);
+
+        assertFalse(sortedRanges.isOutOfDate());
+        assertEquals(0, sortedRanges.getRanges().size());
+        assertEquals(ringVersion, sortedRanges.getRingVersion());
+
+        // split(x) returns 1 range for all x when empty
+        assertEquals(1, sortedRanges.split(0).size());
+        for (int i = 0; i <= 10; i++)
+            assertEquals(1, sortedRanges.split(i).size());
+    }
+
+    @Test
+    public void testSplit()
+    {
+        long ringVersion = 1;
+
+        for (int i = 1; i <= 100; i++)
+        {
+            int numTokens = 172 + random.nextInt(128);
+            int rf = random.nextInt(4) + 2;
+            int parts = random.nextInt(5) + 1;
+            List<Splitter.WeightedRange> ranges = SplitterTest.generateLocalRanges(numTokens,
+                                                                                   rf,
+                                                                                   getSplitter(partitioner),
+                                                                                   random,
+                                                                                   partitioner instanceof RandomPartitioner);
+            SortedLocalRanges sortedRanges = makeRanges(ringVersion, ranges);
+
+            List<Token> boundaries = sortedRanges.split(parts);
+            assertNotNull(boundaries);
+            assertEquals(parts, boundaries.size());
+        }
+    }
+
+    @Test
+    public void testSplitNoSplitter()
+    {
+        long ringVersion = 1;
+        int numTokens = 172 + random.nextInt(128);
+        int rf = random.nextInt(4) + 2;
+        int parts = random.nextInt(5) + 1;
+        List<Splitter.WeightedRange> ranges = SplitterTest.generateLocalRanges(numTokens,
+                                                                               rf,
+                                                                               getSplitter(partitioner),
+                                                                               random,
+                                                                               partitioner instanceof RandomPartitioner);
+
+        // mock a partitioner without the splitter and verify split ranges are the same as the local ranges
+        IPartitioner partitioner = Mockito.mock(IPartitioner.class);
+        when(cfs.getPartitioner()).thenReturn(partitioner);
+        when(partitioner.splitter()).thenReturn(Optional.empty());
+
+        SortedLocalRanges sortedRanges = makeRanges(ringVersion, ranges);
+
+        List<Token> boundaries = sortedRanges.split(parts);
+        assertNotNull(boundaries);
+        assertEquals(ranges.size(), boundaries.size()); // it ignores the parts and just returns the ranges
+    }
+
+    @Test
+    public void testEquals()
+    {
+        long ringVersion = 1;
+        int numTokens = 172 + random.nextInt(128);
+        int rf = random.nextInt(4) + 2;
+        List<Splitter.WeightedRange> ranges = SplitterTest.generateLocalRanges(numTokens,
+                                                                               rf,
+                                                                               getSplitter(partitioner),
+                                                                               random,
+                                                                               partitioner instanceof RandomPartitioner);
+
+        SortedLocalRanges sortedRanges1 = makeRanges(ringVersion, ranges);
+        SortedLocalRanges sortedRanges2 = makeRanges(ringVersion, ranges);
+
+        assertEquals(sortedRanges1, sortedRanges2);
+        assertEquals(sortedRanges1.hashCode(), sortedRanges2.hashCode());
+        assertEquals(sortedRanges1.toString(), sortedRanges2.toString());
+
+        sortedRanges1.invalidate();
+        assertEquals(sortedRanges1, sortedRanges2);
+
+        sortedRanges2.invalidate();
+        assertEquals(sortedRanges1, sortedRanges2);
+
+        ringVersion++;
+
+        // different ring version
+        SortedLocalRanges sortedRanges3 = makeRanges(ringVersion, ranges);
+        assertNotEquals(sortedRanges1, sortedRanges3);
+        assertNotEquals(sortedRanges1.hashCode(), sortedRanges3.hashCode());
+        assertNotEquals(sortedRanges1.toString(), sortedRanges3.toString());
+
+        // different ranges
+        ranges = SplitterTest.generateLocalRanges(numTokens,
+                                                  rf,
+                                                  getSplitter(partitioner),
+                                                  random,
+                                                  partitioner instanceof RandomPartitioner);
+        SortedLocalRanges sortedRanges4 = makeRanges(ringVersion, ranges);
+        assertNotEquals(sortedRanges1, sortedRanges4);
+        assertNotEquals(sortedRanges1.hashCode(), sortedRanges4.hashCode());
+        assertNotEquals(sortedRanges1.toString(), sortedRanges4.toString());
+
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java
index 2a1561b8a1a5..a804e152ae45 100644
--- a/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java
+++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java
@@ -186,7 +186,7 @@ public void testMigrateTransferredRanges() throws Throwable
             assertEquals(InetAddress.getByName("127.0.0.1"), row.getInetAddress("peer"));
             assertEquals(DatabaseDescriptor.getStoragePort(), row.getInt("peer_port"));
             assertEquals("bar", row.getString("keyspace_name"));
-            assertEquals(ImmutableSet.of(ByteBuffer.wrap(new byte[] { 42 })), row.getSet("ranges", BytesType.instance));
+            assertEquals(ImmutableSet.of(ByteBuffer.wrap(new byte[]{ 42 })), row.getSet("ranges", BytesType.instance));
         }
         assertEquals(1, rowCount);
 
diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
index 0bade4bee886..bb1e8e1e1110 100644
--- a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
@@ -21,16 +21,19 @@
 import java.net.UnknownHostException;
 import java.util.*;
 
+import com.google.common.collect.Sets;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.nodes.ILocalInfo;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.SchemaKeyspace;
 import org.apache.cassandra.service.StorageService;
@@ -81,7 +84,7 @@ public void testNonLocalToken() throws UnknownHostException
     {
         BytesToken token = new BytesToken(ByteBufferUtil.bytes("token3"));
         InetAddressAndPort address = InetAddressAndPort.getByName("127.0.0.2");
-        SystemKeyspace.updateTokens(address, Collections.<Token>singletonList(token));
+        SystemKeyspace.updateTokens(address, Sets.newHashSet(token));
         assert SystemKeyspace.loadTokens().get(address).contains(token);
         SystemKeyspace.removeEndpoint(address);
         assert !SystemKeyspace.loadTokens().containsValue(token);
@@ -186,17 +189,14 @@ private void setupReleaseVersion(String version)
     {
         // besides the release_version, we also need to insert the cluster_name or the check
         // in SystemKeyspace.checkHealth were we verify it matches DatabaseDescriptor will fail
-        QueryProcessor.executeInternal(String.format("INSERT INTO system.local(key, release_version, cluster_name) " +
-                                                     "VALUES ('local', '%s', '%s')",
-                                                     version,
-                                                     DatabaseDescriptor.getClusterName()));
+        Nodes.local().update(current -> current.setReleaseVersion(new CassandraVersion(version)).setClusterName(DatabaseDescriptor.getClusterName()), true);
         String r = readLocalVersion();
         assertEquals(String.format("Expected %s, got %s", version, r), version, r);
     }
 
     private String readLocalVersion()
     {
-        UntypedResultSet rs = QueryProcessor.executeInternal("SELECT release_version FROM system.local WHERE key='local'");
-        return rs.isEmpty() || !rs.one().has("release_version") ? null : rs.one().getString("release_version");
+        ILocalInfo info = Nodes.local().get();
+        return info != null && info.getReleaseVersion() != null ? info.getReleaseVersion().toString() : null;
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/WriteOptionsTest.java b/test/unit/org/apache/cassandra/db/WriteOptionsTest.java
new file mode 100644
index 000000000000..4e62d0fde139
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/WriteOptionsTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+
+import static org.apache.cassandra.db.WriteOptions.DEFAULT;
+import static org.apache.cassandra.db.WriteOptions.DEFAULT_WITHOUT_COMMITLOG;
+import static org.apache.cassandra.db.WriteOptions.FOR_BATCH_REPLAY;
+import static org.apache.cassandra.db.WriteOptions.FOR_BOOTSTRAP_STREAMING;
+import static org.apache.cassandra.db.WriteOptions.FOR_HINT_REPLAY;
+import static org.apache.cassandra.db.WriteOptions.FOR_PAXOS_COMMIT;
+import static org.apache.cassandra.db.WriteOptions.FOR_READ_REPAIR;
+import static org.apache.cassandra.db.WriteOptions.FOR_STREAMING;
+import static org.apache.cassandra.db.WriteOptions.FOR_VIEW_BUILD;
+import static org.apache.cassandra.db.WriteOptions.SKIP_INDEXES_AND_COMMITLOG;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class WriteOptionsTest extends CQLTester
+{
+    @Test
+    public void testShouldWriteCommitLog() throws Throwable
+    {
+        String DURABLE_KEYSPACE = "durable_ks";
+        String NON_DURABLE_KEYSPACE = "non_durable_ks";
+        String DEFAULT_DURABLE_KEYSPACE = "ks_with_default_durability";
+
+        Set<WriteOptions> WRITE_COMMIT_LOG_TRUE = Sets.newHashSet(FOR_BOOTSTRAP_STREAMING,
+                FOR_STREAMING,
+                FOR_PAXOS_COMMIT,
+                FOR_VIEW_BUILD);
+        Set<WriteOptions> WRITE_COMMIT_LOG_AUTO = Sets.newHashSet(DEFAULT,
+                FOR_BATCH_REPLAY,
+                FOR_HINT_REPLAY,
+                FOR_READ_REPAIR);
+
+        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true", DURABLE_KEYSPACE));
+        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = false", NON_DURABLE_KEYSPACE));
+        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", DEFAULT_DURABLE_KEYSPACE));
+        try
+        {
+            for (WriteOptions opt : WriteOptions.values())
+            {
+                if (WRITE_COMMIT_LOG_AUTO.contains(opt))
+                {
+                    assertTrue(String.format("%s should write commit log", opt), opt.shouldWriteCommitLog(DURABLE_KEYSPACE));
+                    assertTrue(String.format("%s should write commit log", opt), opt.shouldWriteCommitLog(DEFAULT_DURABLE_KEYSPACE));
+                    assertFalse(String.format("%s should NOT write commit log", opt), opt.shouldWriteCommitLog(NON_DURABLE_KEYSPACE));
+                }
+                else
+                {
+                    assertEquals("CommitLog write for non durable keyspace for " + opt, WRITE_COMMIT_LOG_TRUE.contains(opt), opt.shouldWriteCommitLog(NON_DURABLE_KEYSPACE));
+                    assertEquals("CommitLog write for default duratility for " + opt, WRITE_COMMIT_LOG_TRUE.contains(opt), opt.shouldWriteCommitLog(DEFAULT_DURABLE_KEYSPACE));
+                }
+            }
+        }
+        finally
+        {
+            execute(String.format("DROP KEYSPACE IF EXISTS %s", DURABLE_KEYSPACE));
+            execute(String.format("DROP KEYSPACE IF EXISTS %s", NON_DURABLE_KEYSPACE));
+        }
+    }
+
+    @Test
+    public void testUpdateIndexes()
+    {
+        Set<WriteOptions> SKIP_INDEXES = Sets.newHashSet(SKIP_INDEXES_AND_COMMITLOG);
+
+        for (WriteOptions opt : WriteOptions.values())
+        {
+            assertEquals(!SKIP_INDEXES.contains(opt), opt.updateIndexes);
+        }
+    }
+
+    @Test
+    public void testRequiresViewUpdate() throws Throwable
+    {
+        Set<WriteOptions> DO_NOT_REQUIRE_VIEW_UPDATE = Sets.newHashSet(FOR_BOOTSTRAP_STREAMING);
+
+        String TABLE_WITHOUT_VIEW = "table_without_view";
+        executeNet(String.format("CREATE TABLE %s.%s (k1 int primary key, v1 int)", keyspace(), TABLE_WITHOUT_VIEW));
+
+        String TABLE_WITH_VIEW = "base_table";
+        String VIEW = "mv";
+        executeNet(String.format("CREATE TABLE %s.%s (k1 int primary key, v1 int)", keyspace(), TABLE_WITH_VIEW));
+        executeNet(String.format("CREATE MATERIALIZED VIEW %s.%s AS SELECT * FROM %s WHERE k1 IS NOT NULL AND v1 IS NOT NULL PRIMARY KEY (v1, k1)", keyspace(), VIEW, TABLE_WITH_VIEW));
+
+        Mutation tableWithoutViewMutation = createMutation(TABLE_WITHOUT_VIEW);
+        Mutation tableWithViewMutation = createMutation(TABLE_WITH_VIEW);
+        Mutation mixedMutation = createMutation(TABLE_WITH_VIEW, TABLE_WITHOUT_VIEW);
+
+        Keyspace ks = Keyspace.open(keyspace());
+
+        for (WriteOptions opt : WriteOptions.values())
+        {
+            if (DO_NOT_REQUIRE_VIEW_UPDATE.contains(opt))
+            {
+                assertFalse(opt.requiresViewUpdate(ks.viewManager, tableWithoutViewMutation));
+                assertFalse(opt.requiresViewUpdate(ks.viewManager, tableWithViewMutation));
+                assertFalse(opt.requiresViewUpdate(ks.viewManager, mixedMutation));
+            }
+            else
+            {
+                assertFalse(opt.requiresViewUpdate(ks.viewManager, tableWithoutViewMutation));
+                assertEquals(opt.updateIndexes, opt.requiresViewUpdate(ks.viewManager, tableWithViewMutation));
+            }
+        }
+    }
+
+    private Mutation createMutation(String... tables)
+    {
+        SimpleBuilders.MutationBuilder builder = new SimpleBuilders.MutationBuilder(keyspace(), DatabaseDescriptor.getPartitioner().decorateKey(ByteBuffer.wrap("key".getBytes())));
+        for (String table : tables)
+            builder.update(table).row().add("v1", 1);
+        return builder.build();
+    }
+
+    @Test
+    public void testUsePairedViewReplication()
+    {
+        Set<WriteOptions> USE_PAIRED_VIEW_REPLICATION = Sets.newHashSet(DEFAULT, DEFAULT_WITHOUT_COMMITLOG, FOR_PAXOS_COMMIT, FOR_VIEW_BUILD);
+        for (WriteOptions opt : WriteOptions.values())
+        {
+            assertEquals(USE_PAIRED_VIEW_REPLICATION.contains(opt), opt.usePairedViewReplication);
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/commitlog/AbstractCommitLogServiceTest.java b/test/unit/org/apache/cassandra/db/commitlog/AbstractCommitLogServiceTest.java
index a918e3c70fbc..f4379c7217e8 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/AbstractCommitLogServiceTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/AbstractCommitLogServiceTest.java
@@ -29,6 +29,7 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.commitlog.AbstractCommitLogService.SyncRunnable;
 import org.apache.cassandra.utils.FreeRunningClock;
+import org.apache.cassandra.utils.MonotonicClock;
 
 import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
 import static org.apache.cassandra.concurrent.Interruptible.State.SHUTTING_DOWN;
@@ -102,7 +103,7 @@ private static class FakeCommitLogService extends AbstractCommitLogService
     {
         FakeCommitLogService(long syncIntervalMillis)
         {
-            super(new FakeCommitLog(), "This is not a real commit log", syncIntervalMillis, true);
+            super(new FakeCommitLog(), "This is not a real commit log", syncIntervalMillis, MonotonicClock.Global.preciseTime, true);
             lastSyncedAt = 0;
         }
 
diff --git a/test/unit/org/apache/cassandra/db/commitlog/BatchCommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/BatchCommitLogTest.java
index e9ec640e79eb..1e33270382f9 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/BatchCommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/BatchCommitLogTest.java
@@ -34,16 +34,16 @@
 import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.security.EncryptionContext;
 
-import static org.junit.Assert.assertEquals;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.junit.Assert.assertEquals;
 
 public class BatchCommitLogTest extends CommitLogTest
 {
     private static final long CL_BATCH_SYNC_WINDOW = 1000; // 1 second
     
-    public BatchCommitLogTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext)
+    public BatchCommitLogTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext, Config.DiskAccessMode diskAccessMode)
     {
-        super(commitLogCompression, encryptionContext);
+        super(commitLogCompression, encryptionContext, diskAccessMode);
     }
 
     @BeforeClass
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CDCTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CDCTestReplayer.java
index 762459097cc8..1ea671b59117 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CDCTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CDCTestReplayer.java
@@ -19,7 +19,6 @@
 
 import java.io.IOException;
 
-import org.apache.cassandra.io.util.File;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,7 +44,7 @@ public CDCTestReplayer() throws IOException
 
     public void examineCommitLog() throws IOException
     {
-        replayFiles(new File(DatabaseDescriptor.getCommitLogLocation()).tryList());
+        replayFiles(DatabaseDescriptor.getCommitLogLocation().tryList());
     }
 
     private class CommitLogTestReader extends CommitLogReader
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogApiTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogApiTest.java
new file mode 100644
index 000000000000..6fdbe170d897
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogApiTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+public class CommitLogApiTest
+{
+    @BeforeClass
+    public static void beforeClass() throws ConfigurationException
+    {
+        // Disable durable writes for system keyspaces to prevent system mutations, e.g. sstable_activity,
+        // to end up in CL segments and cause unexpected results in this test wrt counting CL segments,
+        // see CASSANDRA-12854
+        KeyspaceParams.DEFAULT_LOCAL_DURABLE_WRITES = false;
+        SchemaLoader.prepareServer();
+    }
+
+    @Before
+    public void before() throws IOException
+    {
+        CommitLog.instance.resetUnsafe(true);
+    }
+
+    @Test
+    public void testForPath()
+    {
+        AbstractCommitLogSegmentManager original = CommitLog.instance.getSegmentManager();
+        File location = FileUtils.getTempDir();
+        CommitLog.instance.forPath(location);
+        Assert.assertNotEquals(original, CommitLog.instance.getSegmentManager());
+        Assert.assertEquals(location, CommitLog.instance.getSegmentManager().storageDirectory);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogArchiverTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogArchiverTest.java
index 4a491c86b189..45eb29c8f2d7 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogArchiverTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogArchiverTest.java
@@ -39,6 +39,7 @@
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
 import static org.apache.cassandra.io.util.PathUtils.forEach;
 import static org.junit.Assert.assertTrue;
 
@@ -112,7 +113,7 @@ public void testArchiver()
         }
 
         CommitLog.instance.forceRecycleAllSegments();
-        CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
+        CommitLog.instance.getSegmentManager().awaitManagementTasksCompletion();
         // If the number of files that under backup dir is bigger than 1, that means the
         // archiver for commitlog is effective.
         assertTrue(dir.isDirectory() && dir.tryList().length > 0);
@@ -133,15 +134,15 @@ public void testRestoreInDifferentPrecision() throws Throwable
         assertRows(execute("SELECT * FROM %s"), row(4, 0, 0), row(4, 1, 1), row(3, 0, 0), row(3, 1, 1));
 
         CommitLog.instance.forceRecycleAllSegments();
-        CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
+        CommitLog.instance.getSegmentManager().awaitManagementTasksCompletion();
         execute("TRUNCATE TABLE %s");
         assertRowCount(execute("SELECT * FROM %s"), 0);
 
         // replay log
         CommitLog.instance.archiver.maybeRestoreArchive();
-        CommitLogSegment.resetReplayLimit();
+        CommitLog.instance.getSegmentManager().resetReplayLimit();
         // restore archived files
-        CommitLog.instance.recoverFiles(CommitLog.instance.getUnmanagedFiles());
+        CommitLog.instance.recoverFiles(UNIT_TESTS, CommitLog.instance.getUnmanagedFiles());
         // restore poin time is rpiTime in microseconds , so row(4, 0, 0) and row(4, 1, 1) is skipped
         assertRows(execute("SELECT * FROM %s"), row(3, 0, 0), row(3, 1, 1));
 
@@ -161,13 +162,13 @@ public void testRestoreInDifferentPrecision() throws Throwable
         assertRows(execute("SELECT * FROM %s"), row(1, 0, 0), row(1, 1, 1), row(2, 0, 0), row(2, 1, 1));
 
         CommitLog.instance.forceRecycleAllSegments();
-        CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
+        CommitLog.instance.getSegmentManager().awaitManagementTasksCompletion();
         execute("TRUNCATE TABLE %s");
         assertRowCount(execute("SELECT * FROM %s"), 0);
         // replay log
         CommitLog.instance.archiver.maybeRestoreArchive();
-        CommitLogSegment.resetReplayLimit();
-        CommitLog.instance.recoverFiles(CommitLog.instance.getUnmanagedFiles());
+        CommitLog.instance.getSegmentManager().resetReplayLimit();
+        CommitLog.instance.recoverFiles(UNIT_TESTS, CommitLog.instance.getUnmanagedFiles());
         // restore poin time is rpiTime in millseconds, so row(2, 0, 0) and row(2, 1, 1) is skipped
         assertRows(execute("SELECT * FROM %s"), row(1, 0, 0), row(1, 1, 1));
     }
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogAwaitAsyncAtTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogAwaitAsyncAtTest.java
new file mode 100644
index 000000000000..e024954a7958
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogAwaitAsyncAtTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.commitlog;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.utils.FreeRunningClock;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.mockito.Mockito;
+
+public class CommitLogAwaitAsyncAtTest
+{
+    @BeforeClass
+    public static void beforeClass() throws ConfigurationException
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    /**
+     * syncTime (awaitSyncAt param) is in the past, now value overflowed, awaitSyncAt should not block,
+     * no clock advance calls.
+     */
+    @Test
+    public void notBlockIfSyncTimeIsInPast() throws InterruptedException
+    {
+        testResumingAwaitSyncAt(Long.MIN_VALUE + 10,
+                              Long.MAX_VALUE - 10,
+                                0);
+    }
+
+    /**
+     * syncTime (awaitSyncAt param) is in the future, awaitSyncAt should block, unblocking is caused by the flush
+     */
+    @Test
+    public void flushShouldUnblockAwaitSync() throws InterruptedException
+    {
+        testResumingAwaitSyncAt(Long.MAX_VALUE - 10,
+                              Long.MAX_VALUE - 5,
+                                1000);
+    }
+
+    /**
+     * Creates a CommitLogService instance and a new thread that calls awaitSyncAt. Awaits for at most a minute
+     * for the call to return.
+     * Uses artificial clock to progress through the commit flush. One clock advance is performed after the service and
+     * the thread are started.
+     *
+     * @param nowNanos test start time nanoseconds
+     * @param syncAtNanos awaitSyncAt parameter nanoseconds
+     * @param advanceMillis clock step in milliseconds
+     */
+    private void testResumingAwaitSyncAt(long nowNanos, long syncAtNanos, long advanceMillis) throws InterruptedException
+    {
+        FreeRunningClock clock = new FreeRunningClock(nowNanos);
+        AbstractCommitLogService service = getCommitLogService(clock);
+
+        Thread awaitForSync = new Thread(CommitLogAwaitAsyncAtTest.class.getSimpleName() + " commit log waiting thread")
+        {
+            @Override
+            public void run()
+            {
+                service.awaitSyncAt(syncAtNanos, null);
+            }
+        };
+        awaitForSync.start();
+
+        service.start();
+
+        // move clock once with advance millis
+        clock.advance(advanceMillis, TimeUnit.MILLISECONDS);
+
+        // wait at most 1 minute for awaitSyncAt to unblock
+        awaitForSync.join(60 * 1000);
+        if (awaitForSync.isAlive())
+            Assert.fail("awaitSyncAt should be unblocked by now, check commit log code for bugs in nanoseconds" +
+                        "comparisons");
+    }
+
+    private AbstractCommitLogService getCommitLogService(MonotonicClock clock) {
+        CommitLog commitLog = Mockito.mock(CommitLog.class);
+        return new AbstractCommitLogService(commitLog, "testService", 100, clock)
+        {
+            @Override
+            protected void maybeWaitForSync(CommitLogSegment.Allocation alloc)
+            {
+            }
+        };
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java
index 2fd38d8e8c14..0250c897ce25 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java
@@ -56,11 +56,11 @@ public void testTruncateSegmentDiscard() throws Throwable
 
         execute("INSERT INTO %s (idx, data) VALUES (?, ?)", 15, Integer.toString(17));
 
-        Collection<CommitLogSegment> active = new ArrayList<>(CommitLog.instance.segmentManager.getActiveSegments());
+        Collection<CommitLogSegment> active = new ArrayList<>(CommitLog.instance.getSegmentManager().getActiveSegments());
         CommitLog.instance.forceRecycleAllSegments();
 
         // If one of the previous segments remains, it wasn't clean.
-        active.retainAll(CommitLog.instance.segmentManager.getActiveSegments());
+        active.retainAll(CommitLog.instance.getSegmentManager().getActiveSegments());
         assert active.isEmpty();
     }
     
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogDescriptorTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogDescriptorTest.java
index 5ef83b48a13b..6fa76861c415 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogDescriptorTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogDescriptorTest.java
@@ -85,7 +85,7 @@ public void testVersions()
         Assert.assertEquals(1340512736956320000L, CommitLogDescriptor.fromFileName("CommitLog-2-1340512736956320000.log").id);
 
         Assert.assertEquals(MessagingService.current_version, new CommitLogDescriptor(1340512736956320000L, null, neverEnabledEncryption).getMessagingVersion());
-        String newCLName = "CommitLog-" + CommitLogDescriptor.current_version + "-1340512736956320000.log";
+        String newCLName = "CommitLog-" + CommitLogDescriptor.CURRENT_VERSION + "-1340512736956320000.log";
         Assert.assertEquals(MessagingService.current_version, CommitLogDescriptor.fromFileName(newCLName).getMessagingVersion());
     }
 
@@ -113,7 +113,7 @@ private void testDescriptorPersistence(CommitLogDescriptor desc) throws IOExcept
         // Put some extra data in the stream.
         buf.putDouble(0.1);
         buf.flip();
-        FileDataInput input = new FileSegmentInputStream(buf, "input", 0);
+        FileDataInput input = new FileSegmentInputStream(buf, new File("input"), 0);
         CommitLogDescriptor read = CommitLogDescriptor.readHeader(input, neverEnabledEncryption);
         Assert.assertEquals("Descriptor length", length, input.getFilePointer());
         Assert.assertEquals("Descriptors", desc, read);
@@ -124,10 +124,10 @@ private void testDescriptorPersistence(CommitLogDescriptor desc) throws IOExcept
     public void testDescriptorPersistence() throws IOException
     {
         testDescriptorPersistence(new CommitLogDescriptor(11, null, neverEnabledEncryption));
-        testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.current_version, 13, null, neverEnabledEncryption));
-        testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.current_version, 15, null, neverEnabledEncryption));
-        testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.current_version, 17, new ParameterizedClass("LZ4Compressor", null), neverEnabledEncryption));
-        testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.current_version, 19,
+        testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 13, null, neverEnabledEncryption));
+        testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 15, null, neverEnabledEncryption));
+        testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 17, new ParameterizedClass("LZ4Compressor", null), neverEnabledEncryption));
+        testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 19,
                                                           new ParameterizedClass("StubbyCompressor", ImmutableMap.of("parameter1", "value1", "flag2", "55", "argument3", "null")
                                                           ), neverEnabledEncryption));
     }
@@ -140,7 +140,7 @@ public void testDescriptorInvalidParametersSize() throws IOException
         for (int i=0; i<65535; ++i)
             params.put("key"+i, Integer.toString(i, 16));
         try {
-            CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.current_version,
+            CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION,
                                                                21,
                                                                new ParameterizedClass("LZ4Compressor", params),
                                                                neverEnabledEncryption);
@@ -177,7 +177,7 @@ public void constructParametersString_WithCompressionAndEncryption()
     @Test
     public void writeAndReadHeader_NoCompressionOrEncryption() throws IOException
     {
-        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, neverEnabledEncryption);
+        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, neverEnabledEncryption);
         ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
         CommitLogDescriptor.writeHeader(buffer, descriptor);
         buffer.flip();
@@ -191,7 +191,7 @@ public void writeAndReadHeader_NoCompressionOrEncryption() throws IOException
     @Test
     public void writeAndReadHeader_OnlyCompression() throws IOException
     {
-        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, neverEnabledEncryption);
+        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, neverEnabledEncryption);
         ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
         CommitLogDescriptor.writeHeader(buffer, descriptor);
         buffer.flip();
@@ -205,7 +205,7 @@ public void writeAndReadHeader_OnlyCompression() throws IOException
     @Test
     public void writeAndReadHeader_WithEncryptionHeader_EncryptionEnabledInYaml() throws IOException
     {
-        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, enabledEncryption);
+        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, enabledEncryption);
         ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
         CommitLogDescriptor.writeHeader(buffer, descriptor);
         buffer.flip();
@@ -223,7 +223,7 @@ public void writeAndReadHeader_WithEncryptionHeader_EncryptionEnabledInYaml() th
     @Test
     public void writeAndReadHeader_WithEncryptionHeader_EncryptionDisabledInYaml() throws IOException
     {
-        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, enabledEncryption);
+        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, enabledEncryption);
         ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
         CommitLogDescriptor.writeHeader(buffer, descriptor);
         buffer.flip();
@@ -242,7 +242,7 @@ public void writeAndReadHeader_WithEncryptionHeader_EncryptionDisabledInYaml() t
     @Test
     public void writeAndReadHeader_WithCompressionAndEncryption() throws IOException
     {
-        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, enabledEncryption);
+        CommitLogDescriptor descriptor = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, enabledEncryption);
         ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
         CommitLogDescriptor.writeHeader(buffer, descriptor);
         buffer.flip();
@@ -258,60 +258,60 @@ public void writeAndReadHeader_WithCompressionAndEncryption() throws IOException
     @Test
     public void equals_NoCompressionOrEncryption()
     {
-        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, null);
+        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, null);
         Assert.assertEquals(desc1, desc1);
 
-        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, null);
+        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, null);
         Assert.assertEquals(desc1, desc2);
 
-        desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, neverEnabledEncryption);
+        desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, neverEnabledEncryption);
         Assert.assertEquals(desc1, desc1);
-        desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, neverEnabledEncryption);
+        desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, neverEnabledEncryption);
         Assert.assertEquals(desc1, desc2);
 
-        desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, previouslyEnabledEncryption);
+        desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, previouslyEnabledEncryption);
         Assert.assertEquals(desc1, desc1);
-        desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, previouslyEnabledEncryption);
+        desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, previouslyEnabledEncryption);
         Assert.assertEquals(desc1, desc2);
     }
 
     @Test
     public void equals_OnlyCompression()
     {
-        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, null);
+        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, null);
         Assert.assertEquals(desc1, desc1);
 
-        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, null);
+        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, null);
         Assert.assertEquals(desc1, desc2);
 
-        desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, neverEnabledEncryption);
+        desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, neverEnabledEncryption);
         Assert.assertEquals(desc1, desc1);
-        desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, neverEnabledEncryption);
+        desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, neverEnabledEncryption);
         Assert.assertEquals(desc1, desc2);
 
-        desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, previouslyEnabledEncryption);
+        desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, previouslyEnabledEncryption);
         Assert.assertEquals(desc1, desc1);
-        desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, previouslyEnabledEncryption);
+        desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, previouslyEnabledEncryption);
         Assert.assertEquals(desc1, desc2);
     }
 
     @Test
     public void equals_OnlyEncryption()
     {
-        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, enabledEncryption);
+        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, enabledEncryption);
         Assert.assertEquals(desc1, desc1);
 
-        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, enabledEncryption);
+        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, enabledEncryption);
         Assert.assertEquals(desc1, desc2);
 
-        desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, neverEnabledEncryption);
+        desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, neverEnabledEncryption);
         Assert.assertEquals(desc1, desc1);
-        desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, neverEnabledEncryption);
+        desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, neverEnabledEncryption);
         Assert.assertEquals(desc1, desc2);
 
-        desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, previouslyEnabledEncryption);
+        desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, previouslyEnabledEncryption);
         Assert.assertEquals(desc1, desc1);
-        desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, previouslyEnabledEncryption);
+        desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, previouslyEnabledEncryption);
         Assert.assertEquals(desc1, desc2);
     }
 
@@ -321,10 +321,10 @@ public void equals_OnlyEncryption()
     @Test
     public void equals_BothCompressionAndEncryption()
     {
-        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, enabledEncryption);
+        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, enabledEncryption);
         Assert.assertEquals(desc1, desc1);
 
-        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, compression, enabledEncryption);
+        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, compression, enabledEncryption);
         Assert.assertEquals(desc1, desc2);
     }
 
@@ -341,5 +341,12 @@ public void testInferCDCIndexFile()
         File invalidCdcLink = new File(fileNameSuffix + ".invalidlog");
         inferredIndexFile = CommitLogDescriptor.inferCdcIndexFile(invalidCdcLink);
         Assert.assertNull(inferredIndexFile);
+    }   
+    
+    @Test
+    public void testDSE68MessagingVersion()
+    {
+        CommitLogDescriptor descriptor = new CommitLogDescriptor(680, 1, null, null);
+        Assert.assertEquals(MessagingService.VERSION_DSE_68, descriptor.getMessagingVersion());
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogFailurePolicyTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogFailurePolicyTest.java
index 3dc4e5f69e7d..abb0e8964bdc 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogFailurePolicyTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogFailurePolicyTest.java
@@ -29,6 +29,7 @@
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.service.CassandraDaemon;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.JVMKiller;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.KillerForTests;
 
@@ -75,7 +76,7 @@ public void testCommitFailurePolicy_die()
         StorageService.instance.registerDaemon(daemon);
 
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
         Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
         try
         {
@@ -99,7 +100,7 @@ public void testCommitFailurePolicy_ignore_beforeStartup()
         StorageService.instance.registerDaemon(daemon);
 
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
         Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
         try
         {
@@ -124,7 +125,7 @@ public void testCommitFailurePolicy_ignore_afterStartup() throws Exception
         StorageService.instance.registerDaemon(daemon);
 
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
         Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
         try
         {
@@ -139,4 +140,28 @@ public void testCommitFailurePolicy_ignore_afterStartup() throws Exception
             JVMStabilityInspector.replaceKiller(originalKiller);
         }
     }
+
+    @Test
+    public void testCommitFailurePolicy_fail_writes()
+    {
+        CassandraDaemon daemon = new CassandraDaemon();
+        daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+        StorageService.instance.registerDaemon(daemon);
+
+        KillerForTests killerForTests = new KillerForTests();
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.fail_writes);
+            CommitLog.handleCommitError("Testing fail writes policy", new Throwable());
+            //error policy is set to fail_writes, so JVM must not be killed if error occurs after startup
+            Assert.assertFalse(killerForTests.wasKilled());
+        }
+        finally
+        {
+            DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+            JVMStabilityInspector.replaceKiller(originalKiller);
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogInitWithExceptionTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogInitWithExceptionTest.java
index b3cff94c66af..26bf7c42ad54 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogInitWithExceptionTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogInitWithExceptionTest.java
@@ -28,6 +28,7 @@
 import org.apache.cassandra.CassandraIsolatedJunit4ClassRunner;
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
 
@@ -77,7 +78,7 @@ public void testCommitLogInitWithException() {
 
     private static class MockCommitLogSegmentMgr extends CommitLogSegmentManagerStandard {
 
-        public MockCommitLogSegmentMgr(CommitLog commitLog, String storageDirectory)
+        public MockCommitLogSegmentMgr(CommitLog commitLog, File storageDirectory)
         {
             super(commitLog, storageDirectory);
         }
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogPolicyBytemanTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogPolicyBytemanTest.java
new file mode 100644
index 000000000000..a034395ccecf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogPolicyBytemanTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MemtableParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMKiller;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.awaitility.Awaitility.await;
+
+@RunWith(BMUnitRunner.class)
+public class CommitLogPolicyBytemanTest
+{
+    protected static final String KEYSPACE = "CommitLogBytemanTest";
+    protected static final String STANDARD = "Standard";
+    private static final String CUSTOM = "Custom";
+
+    private static JVMKiller oldKiller;
+    private static KillerForTests testKiller;
+    private static Config.CommitFailurePolicy oldPolicy;
+
+    public static final AtomicBoolean failSync = new AtomicBoolean(false);
+
+    @BeforeClass
+    public static void beforeClass() throws ConfigurationException
+    {
+        // Disable durable writes for system keyspaces to prevent system mutations, e.g. sstable_activity,
+        // to end up in CL segments and cause unexpected results in this test wrt counting CL segments,
+        // see CASSANDRA-12854
+        KeyspaceParams.DEFAULT_LOCAL_DURABLE_WRITES = false;
+
+        SchemaLoader.prepareServer();
+        StorageService.instance.getTokenMetadata().updateHostId(UUID.randomUUID(), FBUtilities.getBroadcastAddressAndPort());
+
+        MemtableParams skipListMemtable = MemtableParams.get("skiplist");
+
+        TableMetadata.Builder custom =
+                TableMetadata.builder(KEYSPACE, CUSTOM)
+                        .addPartitionKeyColumn("k", IntegerType.instance)
+                        .addClusteringColumn("c1", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false))
+                        .addClusteringColumn("c2", SetType.getInstance(UTF8Type.instance, false))
+                        .addStaticColumn("s", IntegerType.instance)
+                        .memtable(skipListMemtable);
+
+        SchemaLoader.createKeyspace(KEYSPACE,
+                KeyspaceParams.simple(1),
+                SchemaLoader.standardCFMD(KEYSPACE, STANDARD, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable),
+                custom);
+        CompactionManager.instance.disableAutoCompaction();
+
+        testKiller = new KillerForTests();
+
+        // While we don't want the JVM to be nuked from under us on a test failure, we DO want some indication of
+        // an error. If we hit a "Kill the JVM" condition while working with the CL when we don't expect it, an aggressive
+        // KillerForTests will assertion out on us.
+        oldKiller = JVMStabilityInspector.replaceKiller(testKiller);
+
+        oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+    }
+
+    @AfterClass
+    public static void afterClass()
+    {
+        JVMStabilityInspector.replaceKiller(oldKiller);
+    }
+
+    @Before
+    public void beforeTest() throws IOException
+    {
+        CommitLog.instance.resetUnsafe(true);
+    }
+
+    @After
+    public void afterTest()
+    {
+        DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+        testKiller.reset();
+    }
+
+    @Test
+    @BMRules(rules = { @BMRule(name = "Fail sync in CommitLog",
+            targetClass = "CommitLog",
+            targetMethod = "sync",
+            condition = "org.apache.cassandra.db.commitlog.CommitLogPolicyBytemanTest.failSync.get()",
+            action = "throw new java.lang.RuntimeException(\"Fail CommitLog.sync to test fail_writes policy\");") } )
+    public void testFailWritesPolicies() throws IOException
+    {
+        DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.fail_writes);
+
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(STANDARD);
+        Mutation m = new RowUpdateBuilder(cfs.metadata.get(), 0, "key")
+                .clustering("bytes")
+                .add("val", ByteBuffer.allocate(10 * 1024))
+                .build();
+
+        CommitLog.instance.add(m);
+        Assert.assertFalse(CommitLog.instance.shouldRejectMutations());
+
+        failSync.set(true);
+        await().atMost(2, TimeUnit.SECONDS)
+                .until(() -> CommitLog.instance.shouldRejectMutations());
+        Assert.assertThrows(FSWriteError.class, () -> CommitLog.instance.add(m));
+
+        failSync.set(false);
+        // Force a sync to clear the error
+        CommitLog.instance.sync(false);
+        await().atMost(10, TimeUnit.SECONDS)
+                .until(() -> !CommitLog.instance.shouldRejectMutations());
+        CommitLog.instance.add(m);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java
index 23440ebfa89c..4d4ffa1c4c79 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java
@@ -27,17 +27,18 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.KillerForTests;
 
@@ -185,7 +186,7 @@ private void confirmReadOrder(TestCLRHandler handler, int offset)
                 continue;
             }
 
-            for (Row r : pu)
+            for (Row r : pu.rows())
             {
                 String expected = Integer.toString(i + offset);
                 String seen = new String(r.getCell(cd).buffer().array());
@@ -198,7 +199,7 @@ private void confirmReadOrder(TestCLRHandler handler, int offset)
 
     static ArrayList<File> getCommitLogs()
     {
-        File dir = new File(DatabaseDescriptor.getCommitLogLocation());
+        File dir = DatabaseDescriptor.getCommitLogLocation();
         File[] files = dir.tryList();
         ArrayList<File> results = new ArrayList<>();
         for (File f : files)
@@ -247,6 +248,8 @@ public void handleMutation(Mutation m, int size, int entryLocation, CommitLogDes
             }
         }
 
+        public void handleInvalidMutation(TableId id){}
+
         public int seenMutationCount() { return seenMutations.size(); }
     }
 
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogReplayerTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogReplayerTest.java
new file mode 100644
index 000000000000..a65ccfa79965
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogReplayerTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(BMUnitRunner.class)
+public class CommitLogReplayerTest
+{
+    @BeforeClass
+    public static void before()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+    @Test
+    @BMRules(rules = { @BMRule(name = "Fail applying mutation",
+            targetClass = "org.apache.cassandra.concurrent.Stage",
+            targetMethod = "submit",
+            action = "return org.apache.cassandra.utils.concurrent.ImmediateFuture.failure(new RuntimeException(\"mutation failed\"));") } )
+    public void testTrackingSegmentsWhenMutationFails()
+    {
+        CommitLogReplayer.MutationInitiator mutationInitiator = new CommitLogReplayer.MutationInitiator();
+        CommitLogReplayer replayer = new CommitLogReplayer(CommitLog.instance, CommitLogPosition.NONE, null, CommitLogReplayer.ReplayFilter.create());
+        CommitLogDescriptor descriptor = mock(CommitLogDescriptor.class);
+        String failedSegment = "failedSegment";
+        when(descriptor.fileName()).thenReturn(failedSegment);
+        Future<Integer> mutationFuture = mutationInitiator.initiateMutation(mock(Mutation.class), descriptor, 0, 0, replayer);
+        Assert.assertThrows(ExecutionException.class, () -> mutationFuture.get());
+        Assert.assertTrue(!replayer.getSegmentWithInvalidOrFailedMutations().isEmpty());
+        Assert.assertTrue(replayer.getSegmentWithInvalidOrFailedMutations().contains(failedSegment));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentBackpressureTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentBackpressureTest.java
index 6a8521485a25..3f3fb36161da 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentBackpressureTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentBackpressureTest.java
@@ -112,7 +112,7 @@ public void testCompressedCommitLogBackpressure() throws Throwable
 
             dummyThread.start();
 
-            AbstractCommitLogSegmentManager clsm = CommitLog.instance.segmentManager;
+            AbstractCommitLogSegmentManager clsm = CommitLog.instance.getSegmentManager();
 
             Util.spinAssertEquals(3, () -> clsm.getActiveSegments().size(), 5);
 
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
index e964efa2d8dd..7b602ffcc48f 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
@@ -23,25 +23,28 @@
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.cassandra.ServerTestUtils;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileReader;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.WriteOptions;
 import org.apache.cassandra.db.commitlog.CommitLogSegment.CDCState;
 import org.apache.cassandra.exceptions.CDCWriteException;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileReader;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.TableMetadata;
 
@@ -66,7 +69,7 @@ public void beforeTest() throws Throwable
         // Need to clean out any files from previous test runs. Prevents flaky test failures.
         CommitLog.instance.stopUnsafe(true);
         CommitLog.instance.start();
-        ((CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager).updateCDCTotalSize();
+        ((CommitLogSegmentManagerCDC)CommitLog.instance.getSegmentManager()).updateCDCTotalSize();
     }
 
     @Test
@@ -81,7 +84,7 @@ public void testCDCWriteFailure() throws Throwable
             execute("INSERT INTO %s (idx, data) VALUES (1, '1');");
 
             // Confirm that, on flush+recyle, we see files show up in cdc_raw
-            CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager;
+            CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.getSegmentManager();
             Keyspace.open(keyspace())
                     .getColumnFamilyStore(currentTable())
                     .forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
@@ -90,7 +93,7 @@ public void testCDCWriteFailure() throws Throwable
             Assert.assertTrue("Expected files to be moved to overflow.", getCDCRawCount() > 0);
 
             // Simulate a CDC consumer reading files then deleting them
-            for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).tryList())
+            for (File f : DatabaseDescriptor.getCDCLogLocation().tryList())
                 FileUtils.deleteWithConfirm(f);
 
             // Update size tracker to reflect deleted files. Should flip flag on current allocatingFrom to allow.
@@ -119,7 +122,7 @@ public void testNonblockingShouldMaintainSteadyDiskUsage() throws Throwable
         final long cdcSizeLimit = commitlogSize * targetFilesCount;
         final int mutationSize = DatabaseDescriptor.getCommitLogSegmentSize() / 3;
         testWithNonblockingMode(() -> testWithCDCSpaceInMb((int) cdcSizeLimit, () -> {
-            CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager;
+            CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.getSegmentManager();
 
             createTableAndBulkWrite(mutationSize);
 
@@ -154,7 +157,7 @@ public void testCDCIndexFileWriteOnSync() throws IOException
             .build().apply();
 
         CommitLog.instance.sync(true);
-        CommitLogSegment currentSegment = CommitLog.instance.segmentManager.allocatingFrom();
+        CommitLogSegment currentSegment = CommitLog.instance.getSegmentManager().allocatingFrom();
         int syncOffset = currentSegment.lastSyncedOffset;
 
         // Confirm index file is written
@@ -190,7 +193,7 @@ public void testCDCIndexFileWriteOnSync() throws IOException
     public void testCompletedFlag() throws Throwable
     {
         String tableName = createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=true;");
-        CommitLogSegment initialSegment = CommitLog.instance.segmentManager.allocatingFrom();
+        CommitLogSegment initialSegment = CommitLog.instance.getSegmentManager().allocatingFrom();
 
         testWithCDCSpaceInMb(8, () -> bulkWrite(tableName));
 
@@ -215,7 +218,7 @@ public void testDeleteLinkOnDiscardNoCDC() throws Throwable
         new RowUpdateBuilder(currentTableMetadata(), 0, 1)
             .add("data", randomizeBuffer(DatabaseDescriptor.getCommitLogSegmentSize() / 3))
             .build().apply();
-        CommitLogSegment currentSegment = CommitLog.instance.segmentManager.allocatingFrom();
+        CommitLogSegment currentSegment = CommitLog.instance.getSegmentManager().allocatingFrom();
 
         // Confirm that, with no CDC data present, we've hard-linked but have no index file
         Path linked = new File(DatabaseDescriptor.getCDCLogLocation(), currentSegment.logFile.name()).toPath();
@@ -230,7 +233,7 @@ public void testDeleteLinkOnDiscardNoCDC() throws Throwable
 
         // Force a full recycle and confirm hard-link is deleted
         CommitLog.instance.forceRecycleAllSegments();
-        CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
+        CommitLog.instance.getSegmentManager().awaitManagementTasksCompletion();
         Assert.assertFalse("Expected hard link to CLS to be deleted on non-cdc segment: " + linked, Files.exists(linked));
     }
 
@@ -238,7 +241,7 @@ public void testDeleteLinkOnDiscardNoCDC() throws Throwable
     public void testRetainLinkOnDiscardCDC() throws Throwable
     {
         createTable("CREATE TABLE %s (idx int, data text, primary key(idx)) WITH cdc=true;");
-        CommitLogSegment currentSegment = CommitLog.instance.segmentManager.allocatingFrom();
+        CommitLogSegment currentSegment = CommitLog.instance.getSegmentManager().allocatingFrom();
         File cdcIndexFile = currentSegment.getCDCIndexFile();
         Assert.assertFalse("Expected no index file before flush but found: " + cdcIndexFile, cdcIndexFile.exists());
 
@@ -272,13 +275,13 @@ public void testReplayLogic() throws Throwable
 
         // Build up a list of expected index files after replay and then clear out cdc_raw
         List<CDCIndexData> oldData = parseCDCIndexData();
-        for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).tryList())
+        for (File f : DatabaseDescriptor.getCDCLogLocation().tryList())
             FileUtils.deleteWithConfirm(f.absolutePath());
 
         try
         {
             Assert.assertEquals("Expected 0 files in CDC folder after deletion. ",
-                                0, new File(DatabaseDescriptor.getCDCLogLocation()).tryList().length);
+                                0, DatabaseDescriptor.getCDCLogLocation().tryList().length);
         }
         finally
         {
@@ -286,14 +289,14 @@ public void testReplayLogic() throws Throwable
             // hang in the shutdown on CQLTester trying to clean up / drop keyspaces / tables and hanging applying
             // mutations.
             CommitLog.instance.start();
-            CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
+            CommitLog.instance.getSegmentManager().awaitManagementTasksCompletion();
         }
         CDCTestReplayer replayer = new CDCTestReplayer();
         replayer.examineCommitLog();
 
         // Rough sanity check -> should be files there now.
         Assert.assertTrue("Expected non-zero number of files in CDC folder after restart.",
-                          new File(DatabaseDescriptor.getCDCLogLocation()).tryList().length > 0);
+                          DatabaseDescriptor.getCDCLogLocation().tryList().length > 0);
 
         // Confirm all the old indexes in old are present and >= the original offset, as we flag the entire segment
         // as cdc written on a replay.
@@ -339,7 +342,7 @@ private List<CDCIndexData> parseCDCIndexData()
         List<CDCIndexData> results = new ArrayList<>();
         try
         {
-            for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).tryList())
+            for (File f : DatabaseDescriptor.getCDCLogLocation().tryList())
             {
                 if (f.name().contains("_cdc.idx"))
                     results.add(new CDCIndexData(f));
@@ -395,16 +398,16 @@ private ByteBuffer randomizeBuffer(int size)
 
     private int getCDCRawCount()
     {
-        return new File(DatabaseDescriptor.getCDCLogLocation()).tryList().length;
+        return DatabaseDescriptor.getCDCLogLocation().tryList().length;
     }
 
     private void expectCurrentCDCState(CDCState expectedState)
     {
-        CDCState currentState = CommitLog.instance.segmentManager.allocatingFrom().getCDCState();
+        CDCState currentState = CommitLog.instance.getSegmentManager().allocatingFrom().getCDCState();
         if (currentState != expectedState)
         {
             logger.error("expectCurrentCDCState violation! Expected state: {}. Found state: {}. Current CDC allocation: {}",
-                         expectedState, currentState, ((CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager).updateCDCTotalSize());
+                         expectedState, currentState, ((CommitLogSegmentManagerCDC)CommitLog.instance.getSegmentManager()).updateCDCTotalSize());
             Assert.fail(String.format("Received unexpected CDCState on current allocatingFrom segment. Expected: %s. Received: %s",
                         expectedState, currentState));
         }
@@ -466,7 +469,7 @@ private void bulkWrite(String tableName, int mutationSize) throws Throwable
             {
                 new RowUpdateBuilder(ccfm, 0, i)
                 .add("data", randomizeBuffer(mutationSize))
-                .build().applyFuture().get();
+                .build().applyFuture(WriteOptions.DEFAULT).get();
             }
             if (blockWrites)
                 Assert.fail("Expected CDCWriteException from full CDC but did not receive it.");
@@ -485,7 +488,7 @@ private void testSegmentFlaggingOnCreation0() throws Throwable
 
             createTableAndBulkWrite();
 
-            CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager;
+            CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.getSegmentManager();
             expectCurrentCDCState(blockWrites? CDCState.FORBIDDEN : CDCState.CONTAINS);
 
             // When block writes, releasing CDC commit logs should update the CDC state to PERMITTED
@@ -495,16 +498,20 @@ private void testSegmentFlaggingOnCreation0() throws Throwable
 
                 cdcMgr.awaitManagementTasksCompletion();
                 // Delete all files in cdc_raw
-                for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).tryList())
-                    f.delete();
+                for (File f : DatabaseDescriptor.getCDCLogLocation().tryList())
+                {
+                    logger.debug("delete {}", f.absolutePath());
+                    FileUtils.deleteWithConfirm(f);
+                }
                 cdcMgr.updateCDCTotalSize();
                 // Confirm cdc update process changes flag on active segment
                 expectCurrentCDCState(CDCState.PERMITTED);
             }
 
             // Clear out archived CDC files
-            for (File f : new File(DatabaseDescriptor.getCDCLogLocation()).tryList()) {
-                FileUtils.deleteWithConfirm(f);
+            for (File f : DatabaseDescriptor.getCDCLogLocation().tryList()) {
+                logger.debug("delete {}", f.absolutePath());
+                FileUtils.delete(f);
             }
         });
     }
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
index a5968a8f6aec..721efbc76ab6 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
@@ -36,38 +36,34 @@
 import java.util.stream.Collectors;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
-
 import javax.crypto.Cipher;
 
 import com.google.common.collect.Iterables;
 import com.google.common.io.Files;
-
-import org.apache.cassandra.io.util.FileOutputStreamPlus;
-
-import org.junit.*;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
-import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.db.memtable.SkipListMemtable;
-import org.apache.cassandra.io.compress.ZstdCompressor;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.schema.MemtableParams;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.schema.SchemaTestUtil;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.Config.DiskFailurePolicy;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
-import org.apache.cassandra.config.Config.DiskFailurePolicy;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.commitlog.CommitLogReplayer.CommitLogReplayException;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.memtable.SkipListMemtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.exceptions.ConfigurationException;
@@ -75,26 +71,38 @@
 import org.apache.cassandra.io.compress.DeflateCompressor;
 import org.apache.cassandra.io.compress.LZ4Compressor;
 import org.apache.cassandra.io.compress.SnappyCompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MemtableParams;
+import org.apache.cassandra.schema.SchemaTestUtil;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.security.CipherFactory;
+import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.security.EncryptionContext;
 import org.apache.cassandra.security.EncryptionContextGenerator;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Hex;
+import org.apache.cassandra.utils.JVMKiller;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.KillerForTests;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.vint.VIntCoding;
 
 import static java.lang.String.format;
-import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_IGNORE_REPLAY_ERRORS;
-import static org.apache.cassandra.config.CassandraRelevantProperties.COMMIT_LOG_REPLAY_LIST;
+import static org.apache.cassandra.config.CassandraRelevantProperties.*;
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.STARTUP;
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
+import static org.apache.cassandra.db.commitlog.CommitLogSegment.SYNC_MARKER_SIZE;
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -120,28 +128,34 @@ public abstract class CommitLogTest
     private static final String KEYSPACE2_REPLAY = "CommitLogTestReplay2";
     private static final String KEYSPACE2_REPLAY_TABLE2 = "CommitLogTestReplay2Table2";
 
-    private static JVMStabilityInspector.Killer oldKiller;
+    private static JVMKiller oldKiller;
     private static KillerForTests testKiller;
 
-    public CommitLogTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext)
+    public CommitLogTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext, Config.DiskAccessMode diskAccessMode)
     {
         DatabaseDescriptor.setCommitLogCompression(commitLogCompression);
         DatabaseDescriptor.setEncryptionContext(encryptionContext);
         DatabaseDescriptor.initializeCommitLogDiskAccessMode();
+        if (diskAccessMode != null)
+            DatabaseDescriptor.setDiskAccessMode(diskAccessMode);
     }
 
     @Parameters()
     public static Collection<Object[]> generateData() throws Exception
     {
-        return Arrays.asList(new Object[][]
-                             {
-                             { null, EncryptionContextGenerator.createDisabledContext() }, // No compression, no encryption
-                             { null, newEncryptionContext() }, // Encryption
-                             { new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() },
-                             { new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() },
-                             { new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() },
-                             { new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() }
-                             });
+        List<Object[]> params = new ArrayList<>();
+        for (Config.DiskAccessMode mode : Config.DiskAccessMode.values())
+        {
+            params.addAll(Arrays.asList(new Object[][]{
+            { null, EncryptionContextGenerator.createDisabledContext(), mode }, // No compression, no encryption
+            { null, newEncryptionContext(), mode }, // Encryption
+            { new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext(), mode },
+            { new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext(), mode },
+            { new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext(), mode },
+            { new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext(), mode }
+                                        }));
+        }
+        return params;
     }
 
     private static EncryptionContext newEncryptionContext() throws Exception
@@ -220,6 +234,8 @@ public void afterTest()
     {
         CommitLogSegmentReader.setAllowSkipSyncMarkerCrc(false);
         COMMIT_LOG_REPLAY_LIST.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+        COMMIT_LOG_REPLAY_LIST.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+        CUSTOM_REPLAY_FILTER_CLASS.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
         testKiller.reset();
     }
 
@@ -227,8 +243,8 @@ public void afterTest()
     public void testRecoveryWithEmptyLog() throws Exception
     {
         runExpecting(() -> {
-            CommitLog.instance.recoverFiles(tmpFile(CommitLogDescriptor.current_version),
-                                            tmpFile(CommitLogDescriptor.current_version));
+            CommitLog.instance.recoverFiles(STARTUP, tmpFile(CommitLogDescriptor.CURRENT_VERSION),
+                                            tmpFile(CommitLogDescriptor.CURRENT_VERSION));
             return null;
         }, CommitLogReplayException.class);
     }
@@ -236,7 +252,7 @@ public void testRecoveryWithEmptyLog() throws Exception
     @Test
     public void testRecoveryWithEmptyFinalLog() throws Exception
     {
-        CommitLog.instance.recoverFiles(tmpFile(CommitLogDescriptor.current_version));
+        CommitLog.instance.recoverFiles(STARTUP, tmpFile(CommitLogDescriptor.CURRENT_VERSION));
     }
 
     /**
@@ -251,8 +267,8 @@ public void testHeaderOnlyFileFiltering() throws Exception
 
         File directory = new File(Files.createTempDir());
 
-        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, DatabaseDescriptor.getEncryptionContext());
-        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 2, null, DatabaseDescriptor.getEncryptionContext());
+        CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 1, null, DatabaseDescriptor.getEncryptionContext());
+        CommitLogDescriptor desc2 = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION, 2, null, DatabaseDescriptor.getEncryptionContext());
 
         ByteBuffer buffer;
 
@@ -282,13 +298,13 @@ public void testHeaderOnlyFileFiltering() throws Exception
 
         // one corrupt file and one header only file should be ok
         runExpecting(() -> {
-            CommitLog.instance.recoverFiles(file1, file2);
+            CommitLog.instance.recoverFiles(STARTUP, file1, file2);
             return null;
         }, null);
 
         // 2 corrupt files and one header only file should fail
         runExpecting(() -> {
-            CommitLog.instance.recoverFiles(file1, file1, file2);
+            CommitLog.instance.recoverFiles(STARTUP, file1, file1, file2);
             return null;
         }, CommitLogReplayException.class);
     }
@@ -310,11 +326,33 @@ public void testRecoveryWithShortLog() throws Exception
     public void testRecoveryWithShortSize() throws Exception
     {
         runExpecting(() -> {
-            testRecovery(new byte[2], CommitLogDescriptor.current_version);
+            testRecovery(new byte[2], CommitLogDescriptor.CURRENT_VERSION);
             return null;
         }, CommitLogReplayException.class);
     }
 
+    @Test
+    public void testRecoveryWithTruncatedFileAndTruncationToleration() throws Exception
+    {
+        CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION,
+                                                           CommitLog.instance.getSegmentManager().getNextId(),
+                                                           DatabaseDescriptor.getCommitLogCompression(),
+                                                           DatabaseDescriptor.getEncryptionContext());
+
+        byte[] randomData = new byte[100];
+        (new java.util.Random()).nextBytes(randomData);
+
+        // Simulates a truncated log segment section by writing a segment section marker with a section end offset
+        // that is greater than the log file size.
+        //
+        // This is achieved by using a data length greater than the actual data contents, which will be used when
+        // writing the segment marker.
+        int dataLength = randomData.length * 2;
+
+        // Recovery should succeed when truncation toleration is specified
+        testRecovery(desc, randomData, dataLength, true);
+    }
+
     @Test
     public void testRecoveryWithShortMutationSize() throws Exception
     {
@@ -325,7 +363,7 @@ private void testRecoveryWithGarbageLog() throws Exception
     {
         byte[] garbage = new byte[100];
         (new java.util.Random()).nextBytes(garbage);
-        testRecovery(garbage, CommitLogDescriptor.current_version);
+        testRecovery(garbage, CommitLogDescriptor.CURRENT_VERSION);
     }
 
     @Test
@@ -388,13 +426,13 @@ public void testDontDeleteIfDirty() throws Exception
                       .build();
         CommitLog.instance.add(m2);
 
-        assertEquals(2, CommitLog.instance.segmentManager.getActiveSegments().size());
+        assertEquals(2, CommitLog.instance.getSegmentManager().getActiveSegments().size());
 
         TableId id2 = m2.getTableIds().iterator().next();
         CommitLog.instance.discardCompletedSegments(id2, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
 
         // Assert we still have both our segments
-        assertEquals(2, CommitLog.instance.segmentManager.getActiveSegments().size());
+        assertEquals(2, CommitLog.instance.getSegmentManager().getActiveSegments().size());
     }
 
     @Test
@@ -414,14 +452,14 @@ public void testDeleteIfNotDirty() throws Exception
         CommitLog.instance.add(rm);
         CommitLog.instance.add(rm);
 
-        assertEquals(1, CommitLog.instance.segmentManager.getActiveSegments().size());
+        assertEquals(1, CommitLog.instance.getSegmentManager().getActiveSegments().size());
 
         // "Flush": this won't delete anything
         TableId id1 = rm.getTableIds().iterator().next();
         CommitLog.instance.sync(true);
         CommitLog.instance.discardCompletedSegments(id1, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
 
-        assertEquals(1, CommitLog.instance.segmentManager.getActiveSegments().size());
+        assertEquals(1, CommitLog.instance.getSegmentManager().getActiveSegments().size());
 
         // Adding new mutation on another CF, large enough (including CL entry overhead) that a new segment is created
         Mutation rm2 = new RowUpdateBuilder(cfs2.metadata(), 0, "k")
@@ -433,7 +471,7 @@ public void testDeleteIfNotDirty() throws Exception
         CommitLog.instance.add(rm2);
         CommitLog.instance.add(rm2);
 
-        Collection<CommitLogSegment> segments = CommitLog.instance.segmentManager.getActiveSegments();
+        Collection<CommitLogSegment> segments = CommitLog.instance.getSegmentManager().getActiveSegments();
 
         assertEquals(format("Expected 3 segments but got %d (%s)", segments.size(), getDirtyCFIds(segments)),
                      3,
@@ -445,7 +483,7 @@ public void testDeleteIfNotDirty() throws Exception
         TableId id2 = rm2.getTableIds().iterator().next();
         CommitLog.instance.discardCompletedSegments(id2, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
 
-        segments = CommitLog.instance.segmentManager.getActiveSegments();
+        segments = CommitLog.instance.getSegmentManager().getActiveSegments();
 
         // Assert we still have both our segment
         assertEquals(format("Expected 1 segment but got %d (%s)", segments.size(), getDirtyCFIds(segments)),
@@ -596,8 +634,8 @@ protected void testRecoveryWithBadSizeArgument(int size, int dataSize, long chec
     protected Pair<File, Integer> tmpFile() throws IOException
     {
         EncryptionContext encryptionContext = DatabaseDescriptor.getEncryptionContext();
-        CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.current_version,
-                                                           CommitLogSegment.getNextId(),
+        CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.CURRENT_VERSION,
+                                                           CommitLog.instance.getSegmentManager().getNextId(),
                                                            DatabaseDescriptor.getCommitLogCompression(),
                                                            encryptionContext);
 
@@ -647,20 +685,31 @@ protected Void testRecovery(byte[] logData, int version) throws Exception
         return null;
     }
 
-    protected Void testRecovery(CommitLogDescriptor desc, byte[] logData) throws Exception
+    protected Void testRecovery(CommitLogDescriptor desc, byte[] logData, int dataLength, boolean tolerateTruncation) throws Exception
     {
         File logFile = tmpFile(desc.version);
         CommitLogDescriptor fromFile = CommitLogDescriptor.fromFileName(logFile.name());
         // Change id to match file.
         desc = new CommitLogDescriptor(desc.version, fromFile.id, desc.compression, desc.getEncryptionContext());
+
         ByteBuffer buf = ByteBuffer.allocate(1024);
         CommitLogDescriptor.writeHeader(buf, desc, getAdditionalHeaders(desc.getEncryptionContext()));
+
+        // Write a section marker using the given data length
+        CommitLogSegment.writeSyncMarker(fromFile.id, buf, buf.position(), buf.position(), buf.position() + SYNC_MARKER_SIZE + dataLength);
+
+        // Update buffer position for sync marker
+        buf.position(buf.position() + SYNC_MARKER_SIZE);
+
+        // Add data to byte buffer
+        buf.put(logData);
+
         try (OutputStream lout = new FileOutputStreamPlus(logFile))
         {
             lout.write(buf.array(), 0, buf.position());
-            lout.write(logData);
+
             //statics make it annoying to test things correctly
-            CommitLog.instance.recover(logFile.path()); //CASSANDRA-1119 / CASSANDRA-1179 throw on failure*/
+            CommitLog.instance.recoverPath(logFile.path(), tolerateTruncation); //CASSANDRA-1119 / CASSANDRA-1179 throw on failure
         }
         return null;
     }
@@ -688,7 +737,7 @@ public void testRecoveryWithBadCompressor() throws Exception
     {
         CommitLogDescriptor desc = new CommitLogDescriptor(4, new ParameterizedClass("UnknownCompressor", null), EncryptionContextGenerator.createDisabledContext());
         runExpecting(() -> {
-            testRecovery(desc, new byte[0]);
+            testRecovery(desc, new byte[0], 0, false);
             return null;
         }, CommitLogReplayException.class);
     }
@@ -714,7 +763,7 @@ protected void runExpecting(Callable<Void> r, Class<?> expected)
 
     protected void testRecovery(final byte[] logData, Class<?> expected) throws Exception
     {
-        runExpecting(() -> testRecovery(logData, CommitLogDescriptor.current_version), expected);
+        runExpecting(() -> testRecovery(logData, CommitLogDescriptor.CURRENT_VERSION), expected);
     }
 
     @Test
@@ -740,13 +789,13 @@ public void testTruncateWithoutSnapshot()
             for (int i = 0; i < 5; i++)
                 CommitLog.instance.add(m2);
 
-            assertEquals(2, CommitLog.instance.segmentManager.getActiveSegments().size());
+            assertEquals(2, CommitLog.instance.getSegmentManager().getActiveSegments().size());
             CommitLogPosition position = CommitLog.instance.getCurrentPosition();
             for (Keyspace keyspace : Keyspace.system())
                 for (ColumnFamilyStore syscfs : keyspace.getColumnFamilyStores())
                     CommitLog.instance.discardCompletedSegments(syscfs.metadata().id, CommitLogPosition.NONE, position);
             CommitLog.instance.discardCompletedSegments(cfs2.metadata().id, CommitLogPosition.NONE, position);
-            assertEquals(1, CommitLog.instance.segmentManager.getActiveSegments().size());
+            assertEquals(1, CommitLog.instance.getSegmentManager().getActiveSegments().size());
         }
         finally
         {
@@ -808,7 +857,7 @@ public void replaySimple() throws IOException
         List<String> activeSegments = CommitLog.instance.getActiveSegmentNames();
         assertFalse(activeSegments.isEmpty());
 
-        File[] files = new File(CommitLog.instance.segmentManager.storageDirectory).tryList((file, name) -> activeSegments.contains(name));
+        File[] files = CommitLog.instance.getSegmentManager().storageDirectory.tryList((file, name) -> activeSegments.contains(name));
         replayer.replayFiles(files);
 
         assertEquals(cellCount, replayer.cells);
@@ -924,7 +973,7 @@ private void assertReplay(int expectedReplayedMutations, CassandraRelevantProper
             }};
 
             List<String> activeSegments = CommitLog.instance.getActiveSegmentNames();
-            File[] files = new File(CommitLog.instance.segmentManager.storageDirectory).tryList((file, name) -> activeSegments.contains(name));
+            File[] files = CommitLog.instance.getSegmentManager().storageDirectory.tryList((file, name) -> activeSegments.contains(name));
             ReplayListPropertyReplayer replayer = new ReplayListPropertyReplayer(CommitLog.instance, CommitLogPosition.NONE, cfPersisted, CommitLogReplayer.ReplayFilter.create());
             replayer.replayFiles(files);
 
@@ -946,7 +995,7 @@ public void replayWithBadSyncMarkerCRC() throws IOException
         List<String> activeSegments = CommitLog.instance.getActiveSegmentNames();
         assertFalse(activeSegments.isEmpty());
 
-        File directory = new File(CommitLog.instance.segmentManager.storageDirectory);
+        File directory = CommitLog.instance.getSegmentManager().storageDirectory;
         File firstActiveFile = Objects.requireNonNull(directory.tryList((file, name) -> activeSegments.contains(name)))[0];
         zeroFirstSyncMarkerCRC(firstActiveFile);
 
@@ -957,7 +1006,7 @@ public void replayWithBadSyncMarkerCRC() throws IOException
             // If compression or encryption are enabled, expect an error, and do not attempt to replay using only mutation CRCs.
             runExpecting(() ->
                          {
-                             CommitLog.instance.recoverFiles(firstActiveFile);
+                             CommitLog.instance.recoverFiles(STARTUP, firstActiveFile);
                              return null;
                          },
                          CommitLogReplayException.class);
@@ -1000,6 +1049,29 @@ private void zeroFirstSyncMarkerCRC(File file) throws IOException
         }
     }
 
+    @Test
+    public void failedToReplayMultipleTimes() throws IOException
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+        final Mutation rm1 = new RowUpdateBuilder(cfs.metadata(), 0, "k1")
+                             .clustering("bytes")
+                             .add("val", bytes("this is a string"))
+                             .build();
+        CommitLog.instance.add(rm1);
+        CommitLog.instance.sync(true);
+
+        SimpleCountingReplayer replayer = new SimpleCountingReplayer(CommitLog.instance, CommitLogPosition.NONE, cfs.metadata());
+        List<String> activeSegments = CommitLog.instance.getActiveSegmentNames();
+        assertFalse(activeSegments.isEmpty());
+
+        File[] files = CommitLog.instance.getSegmentManager().storageDirectory.tryList((file, name) -> activeSegments.contains(name));
+        replayer.replayFiles(files);
+        assertEquals(1, replayer.cells);
+
+        // replayer twice should fail
+        assertThatThrownBy(() -> replayer.replayFiles(files)).hasMessageContaining("CommitlogReplayer can only replay once");
+    }
+
     @Test
     public void replayWithDiscard() throws IOException
     {
@@ -1031,7 +1103,7 @@ public void replayWithDiscard() throws IOException
         List<String> activeSegments = CommitLog.instance.getActiveSegmentNames();
         assertFalse(activeSegments.isEmpty());
 
-        File[] files = new File(CommitLog.instance.segmentManager.storageDirectory).tryList((file, name) -> activeSegments.contains(name));
+        File[] files = CommitLog.instance.getSegmentManager().storageDirectory.tryList((file, name) -> activeSegments.contains(name));
         replayer.replayFiles(files);
 
         assertEquals(cellCount, replayer.cells);
@@ -1070,15 +1142,14 @@ public void handleMutation(Mutation m, int size, int entryLocation, CommitLogDes
                 // whether or not system keyspaces will be mutated during a test.
                 if (partitionUpdate.metadata().name.equals(metadata.name))
                 {
-                    for (Row row : partitionUpdate)
+                    for (Row row : partitionUpdate.rows())
                         cells += Iterables.size(row.cells());
                 }
             }
         }
     }
 
-    @Test
-    public void testUnwriteableFlushRecovery() throws ExecutionException, InterruptedException, IOException
+    private void prepareUnwriteableFlushRecovery() throws ExecutionException, InterruptedException, IOException
     {
         CommitLog.instance.resetUnsafe(true);
 
@@ -1119,11 +1190,76 @@ public void testUnwriteableFlushRecovery() throws ExecutionException, Interrupte
         }
 
         CommitLog.instance.sync(true);
+    }
+
+    @Test
+    public void testUnwriteableFlushRecoveryNoFilter() throws ExecutionException, InterruptedException, IOException
+    {
+        prepareUnwriteableFlushRecovery();
+
+        Map<Keyspace, Integer> replayedKeyspaces = CommitLog.instance.resetUnsafe(false);
+        Assert.assertEquals(1, replayedKeyspaces.size());
+        Map.Entry<Keyspace, Integer> firstKeyspace = replayedKeyspaces.entrySet().iterator().next();
+        Assert.assertEquals(KEYSPACE1, firstKeyspace.getKey().getName());
+        Assert.assertEquals(1, (long)firstKeyspace.getValue());
+    }
+
+    @Test
+    public void testUnwriteableFlushRecoveryIncludingFilter() throws ExecutionException, InterruptedException, IOException
+    {
+        prepareUnwriteableFlushRecovery();
+
         try (WithProperties properties = new WithProperties().set(COMMIT_LOG_REPLAY_LIST, KEYSPACE1 + '.' + STANDARD1))
         {
             // Currently we don't attempt to re-flush a memtable that failed, thus make sure data is replayed by commitlog.
             // If retries work subsequent flushes should clear up error and this should change to expect 0.
-            assertEquals(1, CommitLog.instance.resetUnsafe(false));
+            Map<Keyspace, Integer> replayedKeyspaces = CommitLog.instance.resetUnsafe(false);
+            Assert.assertEquals(1, replayedKeyspaces.size());
+            Map.Entry<Keyspace, Integer> firstKeyspace = replayedKeyspaces.entrySet().iterator().next();
+            Assert.assertEquals(KEYSPACE1, firstKeyspace.getKey().getName());
+            Assert.assertEquals(1, (long)firstKeyspace.getValue());
+        }
+    }
+
+    @Test
+    public void testUnwriteableFlushRecoveryNotmachingFilter() throws ExecutionException, InterruptedException, IOException
+    {
+        prepareUnwriteableFlushRecovery();
+
+        try (WithProperties properties = new WithProperties().set(COMMIT_LOG_REPLAY_LIST, KEYSPACE2 + '.' + STANDARD2))
+        {
+            Map<Keyspace, Integer> replayedKeyspaces = CommitLog.instance.resetUnsafe(false);
+            Assert.assertEquals(0, replayedKeyspaces.size());
+        }
+    }
+
+    /**
+     * Test that Custom filter class is being called by the fact that the custom filter has 
+     * different behavior than default. It filters everything and nothing is replayed.
+     */
+    @Test
+    public void testUnwriteableFlushRecoveryCustomExcludingFilter() throws ExecutionException, InterruptedException, IOException
+    {
+        prepareUnwriteableFlushRecovery();
+
+        // Test the custom filter, which excludes everything.
+        CUSTOM_REPLAY_FILTER_CLASS.setString(NeverReplayFilter.class.getName());
+        Map<Keyspace, Integer> replayedKeyspaces = CommitLog.instance.resetUnsafe(false);
+        Assert.assertEquals(0, replayedKeyspaces.size());
+    }
+
+    public static class NeverReplayFilter extends CommitLogReplayer.ReplayFilter
+    {
+        @Override
+        public Iterable<PartitionUpdate> filter(Mutation mutation)
+        {
+            return Collections.emptySet();
+        }
+
+        @Override
+        public boolean includes(TableMetadataRef tableMetadataRef)
+        {
+            return false;
         }
     }
 
@@ -1158,7 +1294,8 @@ public void testOutOfOrderFlushRecovery(BiConsumer<ColumnFamilyStore, Memtable>
         // In the absence of error, this should be 0 because forceBlockingFlush/forceRecycleAllSegments would have
         // persisted all data in the commit log. Because we know there was an error, there must be something left to
         // replay.
-        assertEquals(1, CommitLog.instance.resetUnsafe(false));
+        Assert.assertEquals(1, CommitLog.instance.resetUnsafe(false).size());
+        System.clearProperty("cassandra.replayList");
     }
 
     BiConsumer<ColumnFamilyStore, Memtable> flush = (cfs, current) ->
@@ -1229,14 +1366,14 @@ public void testRecoveryWithCollectionClusteringKeysStatic() throws Exception
         Mutation rm = rb.build();
         CommitLog.instance.add(rm);
 
-        int replayed = 0;
+        Map<Keyspace, Integer> replayed;
 
         try (WithProperties properties = new WithProperties().set(COMMITLOG_IGNORE_REPLAY_ERRORS, true))
         {
             replayed = CommitLog.instance.resetUnsafe(false);
         }
-        
-        assertEquals(replayed, 1);
+
+        Assert.assertEquals(replayed.size(), 1);
     }
 }
 
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index 0519af925c80..e4b91ae96fbb 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@ -18,7 +18,6 @@
 */
 package org.apache.cassandra.db.commitlog;
 
-import org.apache.cassandra.io.util.File;
 import java.io.IOException;
 
 import com.google.common.base.Predicate;
@@ -48,7 +47,7 @@ public CommitLogTestReplayer(Predicate<Mutation> processor) throws IOException
 
     public void examineCommitLog() throws IOException
     {
-        replayFiles(new File(DatabaseDescriptor.getCommitLogLocation()).tryList());
+        replayFiles(DatabaseDescriptor.getCommitLogLocation().tryList());
     }
 
     private class CommitLogTestReader extends CommitLogReader
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java
index c3a827aae628..40d0777db809 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java
@@ -52,6 +52,7 @@
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Tables;
 import org.apache.cassandra.security.EncryptionContextGenerator;
+import org.apache.cassandra.utils.JVMKiller;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.KillerForTests;
 
@@ -76,7 +77,7 @@ public class CommitLogUpgradeTest
     static final String KEYSPACE = "Keyspace1";
     static final String CELLNAME = "name";
 
-    private JVMStabilityInspector.Killer originalKiller;
+    private JVMKiller originalKiller;
     private KillerForTests killerForTests;
     private boolean shouldBeKilled = false;
 
@@ -186,7 +187,7 @@ public boolean apply(Mutation mutation)
         {
             for (PartitionUpdate update : mutation.getPartitionUpdates())
             {
-                for (Row row : update)
+                for (Row row : update.rows())
                     if (row.clustering().size() > 0 &&
                         AsciiType.instance.compose(row.clustering().bufferAt(0)).startsWith(CELLNAME))
                     {
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java
index b97cad2c09ea..3d8e980a1c83 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java
@@ -39,17 +39,24 @@
 import org.junit.Assert;
 
 import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
 import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.utils.FBUtilities;
 
-import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.*;
+import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.CELLS_PROPERTY;
+import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.CFID_PROPERTY;
+import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.HASH_PROPERTY;
+import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.KEYSPACE;
+import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.PROPERTIES_FILE;
+import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.TABLE;
+import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.hash;
+import static org.apache.cassandra.db.commitlog.CommitLogUpgradeTest.metadata;
 
 public class CommitLogUpgradeTestMaker
 {
@@ -132,7 +139,7 @@ public void makeLog() throws IOException, InterruptedException
             FileUtils.deleteRecursive(dataDir);
 
         dataDir.tryCreateDirectories();
-        for (File f : new File(DatabaseDescriptor.getCommitLogLocation()).tryList())
+        for (File f : DatabaseDescriptor.getCommitLogLocation().tryList())
             FileUtils.createHardLink(f, new File(dataDir, f.name()));
 
         Properties prop = new Properties();
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitlogShutdownTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitlogShutdownTest.java
index 9ec6efca5334..4f973ae6903c 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitlogShutdownTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitlogShutdownTest.java
@@ -22,7 +22,6 @@
 import java.util.Random;
 
 import com.google.common.collect.ImmutableMap;
-import org.apache.cassandra.io.util.File;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -94,6 +93,6 @@ public void testShutdownWithPendingTasks() throws Exception
         CommitLog.instance.shutdownBlocking();
 
         // the shutdown should block until all logs except the currently active one and perhaps a new, empty one are gone
-        Assert.assertTrue(new File(DatabaseDescriptor.getCommitLogLocation()).tryList().length <= 2);
+        Assert.assertTrue(DatabaseDescriptor.getCommitLogLocation().tryList().length <= 2);
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/commitlog/DirectIOSegmentTest.java b/test/unit/org/apache/cassandra/db/commitlog/DirectIOSegmentTest.java
index 27d3946b2aa2..b228b4e5c7b8 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/DirectIOSegmentTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/DirectIOSegmentTest.java
@@ -23,6 +23,7 @@
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.junit.BeforeClass;
@@ -38,6 +39,7 @@
 import org.mockito.internal.creation.MockSettingsImpl;
 import sun.nio.ch.DirectBuffer;
 
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.doAnswer;
@@ -75,6 +77,9 @@ public void testFlushBuffer()
         doCallRealMethod().when(manager).getConfiguration();
         when(bufferPool.createBuffer()).thenReturn(ByteBuffer.allocate(bufSize + fsBlockSize));
         doNothing().when(manager).addSize(anyLong());
+        long idBase = currentTimeMillis();
+        AtomicInteger nextId = new AtomicInteger(1);
+        doReturn(idBase + nextId.getAndIncrement()).when(manager).getNextId();
 
         qt().forAll(Generators.forwardRanges(0, bufSize))
             .checkAssert(startEnd -> {
@@ -125,6 +130,9 @@ public void testFlushSize()
         doCallRealMethod().when(manager).getConfiguration();
         when(bufferPool.createBuffer()).thenReturn(ByteBuffer.allocate(bufSize + fsBlockSize));
         doNothing().when(manager).addSize(anyLong());
+        long idBase = currentTimeMillis();
+        AtomicInteger nextId = new AtomicInteger(1);
+        doReturn(idBase + nextId.getAndIncrement()).when(manager).getNextId();
 
         FileChannel channel = mock(FileChannel.class);
         ThrowingFunction<Path, FileChannel, IOException> channelFactory = path -> channel;
diff --git a/test/unit/org/apache/cassandra/db/commitlog/GroupCommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/GroupCommitLogTest.java
index 8b0a506a20d6..19488cf1172d 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/GroupCommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/GroupCommitLogTest.java
@@ -27,9 +27,9 @@
 
 public class GroupCommitLogTest extends CommitLogTest
 {
-    public GroupCommitLogTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext)
+    public GroupCommitLogTest(ParameterizedClass commitLogCompression, EncryptionContext encryptionContext, Config.DiskAccessMode diskAccessMode)
     {
-        super(commitLogCompression, encryptionContext);
+        super(commitLogCompression, encryptionContext, diskAccessMode);
     }
 
     @BeforeClass
diff --git a/test/unit/org/apache/cassandra/db/commitlog/MemoryMappedSegmentTest.java b/test/unit/org/apache/cassandra/db/commitlog/MemoryMappedSegmentTest.java
new file mode 100644
index 000000000000..f6ce34bad170
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/commitlog/MemoryMappedSegmentTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.commitlog;
+
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.io.util.File;
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+public class MemoryMappedSegmentTest
+{
+    @BeforeClass
+    public static void beforeClass() throws Exception
+    {
+        SchemaLoader.prepareServer();
+    }
+
+    @Test
+    public void shouldNotSkipFileAdviseToFreeSystemCache()
+    {
+        //given
+        MemoryMappedSegment.skipFileAdviseToFreePageCache = false;
+        MemoryMappedSegment memoryMappedSegment = memoryMappedSegment();
+        int startMarker = 0;
+        int nextMarker = 1024;
+
+        //when
+        memoryMappedSegment.flush(startMarker, nextMarker);
+
+        //then
+        verify(memoryMappedSegment)
+                .adviceOnFileToFreePageCache(eq(memoryMappedSegment.fd), eq(startMarker), eq(nextMarker), eq(memoryMappedSegment.logFile));
+    }
+
+    @Test
+    public void shouldSkipFileAdviseToFreeSystemCache()
+    {
+        //given
+        MemoryMappedSegment.skipFileAdviseToFreePageCache = true;
+        MemoryMappedSegment memoryMappedSegment = memoryMappedSegment();
+
+        //when
+        memoryMappedSegment.flush(0, 1024);
+
+        //then
+        verify(memoryMappedSegment, never())
+                .adviceOnFileToFreePageCache(anyInt(), anyInt(), anyInt(), any(File.class));
+    }
+
+    private MemoryMappedSegment memoryMappedSegment()
+    {
+        return Mockito.spy(new MemoryMappedSegment.MemoryMappedSegmentBuilder(CommitLog.instance.getSegmentManager()).build());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/commitlog/SegmentReaderTest.java b/test/unit/org/apache/cassandra/db/commitlog/SegmentReaderTest.java
index 416675907028..3939a723ac9f 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/SegmentReaderTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/SegmentReaderTest.java
@@ -24,7 +24,6 @@
 import java.util.Collections;
 import java.util.Random;
 import java.util.function.BiFunction;
-
 import javax.crypto.Cipher;
 
 import org.junit.Assert;
@@ -41,9 +40,9 @@
 import org.apache.cassandra.io.compress.SnappyCompressor;
 import org.apache.cassandra.io.compress.ZstdCompressor;
 import org.apache.cassandra.security.CipherFactory;
-import org.apache.cassandra.security.EncryptionUtils;
 import org.apache.cassandra.security.EncryptionContext;
 import org.apache.cassandra.security.EncryptionContextGenerator;
+import org.apache.cassandra.security.EncryptionUtils;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 public class SegmentReaderTest
diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
deleted file mode 100644
index 1fc43bdcc3d1..000000000000
--- a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.compaction;
-
-import java.util.Collections;
-
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.junit.Assert;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.schema.CompactionParams;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.FBUtilities;
-
-public class AbstractCompactionStrategyTest
-{
-    private static final String KEYSPACE1 = "Keyspace1";
-    private static final String LCS_TABLE = "LCS_TABLE";
-    private static final String STCS_TABLE = "STCS_TABLE";
-    private static final String TWCS_TABLE = "TWCS_TABLE";
-
-    @BeforeClass
-    public static void loadData() throws ConfigurationException
-    {
-        SchemaLoader.prepareServer();
-        SchemaLoader.createKeyspace(KEYSPACE1,
-                                    KeyspaceParams.simple(1),
-                                    SchemaLoader.standardCFMD(KEYSPACE1, LCS_TABLE)
-                                                .compaction(CompactionParams.lcs(Collections.emptyMap())),
-                                    SchemaLoader.standardCFMD(KEYSPACE1, STCS_TABLE)
-                                                .compaction(CompactionParams.stcs(Collections.emptyMap())),
-                                    SchemaLoader.standardCFMD(KEYSPACE1, TWCS_TABLE)
-                                                .compaction(CompactionParams.create(TimeWindowCompactionStrategy.class, Collections.emptyMap())));
-        Keyspace.open(KEYSPACE1).getColumnFamilyStore(LCS_TABLE).disableAutoCompaction();
-        Keyspace.open(KEYSPACE1).getColumnFamilyStore(STCS_TABLE).disableAutoCompaction();
-        Keyspace.open(KEYSPACE1).getColumnFamilyStore(TWCS_TABLE).disableAutoCompaction();
-    }
-
-    @After
-    public void tearDown()
-    {
-
-        Keyspace.open(KEYSPACE1).getColumnFamilyStore(LCS_TABLE).truncateBlocking();
-        Keyspace.open(KEYSPACE1).getColumnFamilyStore(STCS_TABLE).truncateBlocking();
-        Keyspace.open(KEYSPACE1).getColumnFamilyStore(TWCS_TABLE).truncateBlocking();
-    }
-
-    @Test(timeout=30000)
-    public void testGetNextBackgroundTaskDoesNotBlockLCS()
-    {
-        testGetNextBackgroundTaskDoesNotBlock(LCS_TABLE);
-    }
-
-    @Test(timeout=30000)
-    public void testGetNextBackgroundTaskDoesNotBlockSTCS()
-    {
-        testGetNextBackgroundTaskDoesNotBlock(STCS_TABLE);
-    }
-
-    @Test(timeout=30000)
-    public void testGetNextBackgroundTaskDoesNotBlockTWCS()
-    {
-        testGetNextBackgroundTaskDoesNotBlock(TWCS_TABLE);
-    }
-
-    public void testGetNextBackgroundTaskDoesNotBlock(String table)
-    {
-        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(table);
-        AbstractCompactionStrategy strategy = cfs.getCompactionStrategyManager().getStrategies().get(1).get(0);
-
-        // Add 4 sstables
-        for (int i = 1; i <= 4; i++)
-        {
-            insertKeyAndFlush(table, i);
-        }
-
-        // Check they are returned on the next background task
-        try (LifecycleTransaction txn = strategy.getNextBackgroundTask(FBUtilities.nowInSeconds()).transaction)
-        {
-            Assert.assertEquals(cfs.getLiveSSTables(), txn.originals());
-        }
-
-        // now remove sstables on the tracker, to simulate a concurrent transaction
-        cfs.getTracker().removeUnsafe(cfs.getLiveSSTables());
-
-        // verify the compaction strategy will return null
-        Assert.assertNull(strategy.getNextBackgroundTask(FBUtilities.nowInSeconds()));
-    }
-
-
-    private static void insertKeyAndFlush(String table, int key)
-    {
-        long timestamp = System.currentTimeMillis();
-        DecoratedKey dk = Util.dk(String.format("%03d", key));
-        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(table);
-        new RowUpdateBuilder(cfs.metadata(), timestamp, dk.getKey())
-        .clustering(String.valueOf(key))
-        .add("val", "val")
-        .build()
-        .applyUnsafe();
-        Util.flush(cfs);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
index c2ec223fbd4f..fadddc4249c5 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Random;
 import java.util.Set;
 
 import org.junit.Before;
@@ -41,18 +42,21 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.service.StorageService;
 
 @Ignore
-public class AbstractPendingRepairTest extends AbstractRepairTest
+public abstract class AbstractPendingRepairTest extends AbstractRepairTest
 {
     protected String ks;
     protected final String tbl = "tbl";
     protected TableMetadata cfm;
     protected ColumnFamilyStore cfs;
-    protected CompactionStrategyManager csm;
+    protected CompactionStrategyFactory strategyFactory;
+    protected CompactionStrategyContainer compactionStrategyContainer;
     protected static ActiveRepairService ARS;
 
-    private int nextSSTableKey = 0;
+    protected int nextSSTableKey = 0;
+    public abstract String createTableCql();
 
     @BeforeClass
     public static void setupClass()
@@ -64,20 +68,28 @@ public static void setupClass()
         // cutoff messaging service
         MessagingService.instance().outboundSink.add((message, to) -> false);
         MessagingService.instance().inboundSink.add((message) -> false);
+        StorageService.instance.initServer();
     }
 
     @Before
     public void setup()
     {
         ks = "ks_" + System.currentTimeMillis();
-        cfm = CreateTableStatement.parse(String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, tbl), ks).build();
+        cfm = CreateTableStatement.parse(createTableCql(), ks).build();
         SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm);
         cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
-        csm = cfs.getCompactionStrategyManager();
+        strategyFactory = cfs.getCompactionFactory();
+        compactionStrategyContainer = cfs.getCompactionStrategyContainer();
         nextSSTableKey = 0;
         cfs.disableAutoCompaction();
     }
 
+    void handleOrphan(SSTableReader sstable)
+    {
+        compactionStrategyContainer.getStrategies(false, null)
+                                   .forEach(acs -> ((LegacyAbstractCompactionStrategy) acs).removeSSTable(sstable));
+    }
+
     /**
      * creates and returns an sstable
      *
@@ -85,9 +97,14 @@ public void setup()
      */
     SSTableReader makeSSTable(boolean orphan)
     {
-        int pk = nextSSTableKey++;
+        // store a few shuffled keys to avoid non-overlap
         Set<SSTableReader> pre = cfs.getLiveSSTables();
-        QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES(?, ?)", ks, tbl), pk, pk);
+        Random rand = new Random(nextSSTableKey++);
+        for (int i = 0; i < 10; ++i)
+        {
+            int pk = rand.nextInt();
+            QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES(?, ?)", ks, tbl), pk, pk);
+        }
         Util.flush(cfs);
         Set<SSTableReader> post = cfs.getLiveSSTables();
         Set<SSTableReader> diff = new HashSet<>(post);
@@ -96,7 +113,7 @@ SSTableReader makeSSTable(boolean orphan)
         SSTableReader sstable = diff.iterator().next();
         if (orphan)
         {
-            csm.getUnrepairedUnsafe().allStrategies().forEach(acs -> acs.removeSSTable(sstable));
+            handleOrphan(sstable);
         }
         return sstable;
     }
diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractTableOperationTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractTableOperationTest.java
new file mode 100644
index 000000000000..303ec51cdf51
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/AbstractTableOperationTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.repair.AbstractPendingAntiCompactionTest;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.TimeUUID;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+
+public class AbstractTableOperationTest extends AbstractPendingAntiCompactionTest
+{
+    @Test
+    public void testAbstractTableOperationToStringContainsTaskId()
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS();
+        TimeUUID expectedTaskId = nextTimeUUID();
+        AbstractTableOperation.OperationProgress task = new AbstractTableOperation.OperationProgress(cfs.metadata(), OperationType.COMPACTION, 0, 1000, expectedTaskId, new ArrayList<>());
+        Assertions.assertThat(task.toString())
+                  .contains(expectedTaskId.toString());
+    }
+
+    @Test
+    public void testCompactionInfoToStringFormat()
+    {
+        UUID tableId = UUID.randomUUID();
+        TimeUUID taskId = nextTimeUUID();
+        ColumnFamilyStore cfs = MockSchema.newCFS(builder -> builder.id(TableId.fromUUID(tableId)));
+        AbstractTableOperation.OperationProgress task = new AbstractTableOperation.OperationProgress(cfs.metadata(), OperationType.COMPACTION, 0, 1000, taskId, new ArrayList<>());
+        Assertions.assertThat(task.toString())
+                  .isEqualTo("Compaction(%s, 0 / 1000 bytes)@%s(%s, %s)",
+                             taskId, tableId, cfs.getKeyspaceName(), cfs.getTableName());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
deleted file mode 100644
index 4f915f7956d0..000000000000
--- a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.compaction;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.junit.Test;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.cache.AutoSavingCache;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.view.View;
-import org.apache.cassandra.db.view.ViewBuilderTask;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.SecondaryIndexBuilder;
-import org.apache.cassandra.io.sstable.IScrubber;
-import org.apache.cassandra.io.sstable.IVerifier;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.service.CacheService;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-public class ActiveCompactionsTest extends CQLTester
-{
-    @Test
-    public void testActiveCompactionTrackingRaceWithIndexBuilder() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
-        String idxName = createIndex("CREATE INDEX on %s(a)");
-        getCurrentColumnFamilyStore().disableAutoCompaction();
-        for (int i = 0; i < 5; i++)
-        {
-            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
-            getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
-        }
-
-        Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName);
-        Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
-
-        ExecutorService es = Executors.newFixedThreadPool(2);
-
-        final int loopCount = 3500;
-        for (int ii = 0; ii < loopCount; ii++)
-        {
-            CountDownLatch trigger = new CountDownLatch(1);
-            SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables, true);
-            Future<?> f1 = es.submit(() -> {
-                Uninterruptibles.awaitUninterruptibly(trigger);
-                try
-                {
-                    CompactionManager.instance.submitIndexBuild(builder).get();
-                }
-                catch (Exception e)
-                {
-                    throw new RuntimeException(e);
-                }
-            });
-            Future<?> f2 = es.submit(() -> {
-                Uninterruptibles.awaitUninterruptibly(trigger);
-                CompactionManager.instance.active.getCompactionsForSSTable(null, null);
-            });
-            trigger.countDown();
-            FBUtilities.waitOnFutures(Arrays.asList(f1, f2));
-        }
-        es.shutdown();
-        es.awaitTermination(1, TimeUnit.MINUTES);
-    }
-
-    @Test
-    public void testSecondaryIndexTracking() throws Throwable
-    {
-        Util.assumeLegacySecondaryIndex();
-        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
-        String idxName = createIndex("CREATE INDEX on %s(a)");
-        getCurrentColumnFamilyStore().disableAutoCompaction();
-        for (int i = 0; i < 5; i++)
-        {
-            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
-            flush();
-        }
-
-        Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName);
-        Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
-        SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables, false);
-
-        MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
-        CompactionManager.instance.submitIndexBuild(builder, mockActiveCompactions).get();
-
-        assertTrue(mockActiveCompactions.finished);
-        assertNotNull(mockActiveCompactions.holder);
-        assertEquals(sstables, mockActiveCompactions.holder.getCompactionInfo().getSSTables());
-    }
-
-    @Test
-    public void testIndexSummaryRedistributionTracking() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
-        getCurrentColumnFamilyStore().disableAutoCompaction();
-        for (int i = 0; i < 5; i++)
-        {
-            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
-            flush();
-        }
-        Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
-        try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstables, OperationType.INDEX_SUMMARY))
-        {
-            Map<TableId, LifecycleTransaction> transactions = ImmutableMap.<TableId, LifecycleTransaction>builder().put(getCurrentColumnFamilyStore().metadata().id, txn).build();
-            IndexSummaryRedistribution isr = new IndexSummaryRedistribution(transactions, 0, 1000);
-            MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
-            mockActiveCompactions.beginCompaction(isr);
-            try
-            {
-                isr.redistributeSummaries();
-            }
-            finally
-            {
-                mockActiveCompactions.finishCompaction(isr);
-            }
-            assertTrue(mockActiveCompactions.finished);
-            assertNotNull(mockActiveCompactions.holder);
-            // index redistribution operates over all keyspaces/tables, we always cancel them
-            assertTrue(mockActiveCompactions.holder.getCompactionInfo().getSSTables().isEmpty());
-            assertTrue(mockActiveCompactions.holder.getCompactionInfo().shouldStop((sstable) -> false));
-        }
-    }
-
-    @Test
-    public void testViewBuildTracking() throws Throwable
-    {
-        createTable("CREATE TABLE %s (k1 int, c1 int , val int, PRIMARY KEY (k1, c1))");
-        getCurrentColumnFamilyStore().disableAutoCompaction();
-        for (int i = 0; i < 5; i++)
-        {
-            execute("INSERT INTO %s (k1, c1, val) VALUES (" + i + ", 2, 3)");
-            flush();
-        }
-        execute(String.format("CREATE MATERIALIZED VIEW %s.view1 AS SELECT k1, c1, val FROM %s.%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)", keyspace(), keyspace(), currentTable()));
-        View view = Iterables.getOnlyElement(getCurrentColumnFamilyStore().viewManager);
-
-        Token token = DatabaseDescriptor.getPartitioner().getMinimumToken();
-        ViewBuilderTask vbt = new ViewBuilderTask(getCurrentColumnFamilyStore(), view, new Range<>(token, token), token, 0);
-
-        MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
-        CompactionManager.instance.submitViewBuilder(vbt, mockActiveCompactions).get();
-        assertTrue(mockActiveCompactions.finished);
-        assertTrue(mockActiveCompactions.holder.getCompactionInfo().getSSTables().isEmpty());
-        // this should stop for all compactions, even if it doesn't pick any sstables;
-        assertTrue(mockActiveCompactions.holder.getCompactionInfo().shouldStop((sstable) -> false));
-    }
-
-    @Test
-    public void testScrubOne() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
-        getCurrentColumnFamilyStore().disableAutoCompaction();
-        for (int i = 0; i < 5; i++)
-        {
-            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
-            flush();
-        }
-
-        SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null);
-        try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstable, OperationType.SCRUB))
-        {
-            MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
-            CompactionManager.instance.scrubOne(getCurrentColumnFamilyStore(), txn, IScrubber.options().skipCorrupted().build(), mockActiveCompactions);
-
-            assertTrue(mockActiveCompactions.finished);
-            assertEquals(mockActiveCompactions.holder.getCompactionInfo().getSSTables(), Sets.newHashSet(sstable));
-            assertFalse(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> false));
-            assertTrue(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> true));
-        }
-
-    }
-
-    @Test
-    public void testVerifyOne() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
-        getCurrentColumnFamilyStore().disableAutoCompaction();
-        for (int i = 0; i < 5; i++)
-        {
-            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
-            flush();
-        }
-
-        SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null);
-        MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
-        CompactionManager.instance.verifyOne(getCurrentColumnFamilyStore(), sstable, IVerifier.options().build(), mockActiveCompactions);
-        assertTrue(mockActiveCompactions.finished);
-        assertEquals(mockActiveCompactions.holder.getCompactionInfo().getSSTables(), Sets.newHashSet(sstable));
-        assertFalse(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> false));
-        assertTrue(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> true));
-    }
-
-    @Test
-    public void testSubmitCacheWrite() throws ExecutionException, InterruptedException
-    {
-        AutoSavingCache.Writer writer = CacheService.instance.keyCache.getWriter(100);
-        MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
-        CompactionManager.instance.submitCacheWrite(writer, mockActiveCompactions).get();
-        assertTrue(mockActiveCompactions.finished);
-        assertTrue(mockActiveCompactions.holder.getCompactionInfo().getSSTables().isEmpty());
-    }
-
-    private static class MockActiveCompactions implements ActiveCompactionsTracker
-    {
-        public CompactionInfo.Holder holder;
-        public boolean finished = false;
-        public void beginCompaction(CompactionInfo.Holder ci)
-        {
-            holder = ci;
-        }
-
-        public void finishCompaction(CompactionInfo.Holder ci)
-        {
-            finished = true;
-        }
-    }
-}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/ActiveOperationsConcurrencyTest.java b/test/unit/org/apache/cassandra/db/compaction/ActiveOperationsConcurrencyTest.java
new file mode 100644
index 000000000000..3bc1cf81fdec
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/ActiveOperationsConcurrencyTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.LockSupport;
+import java.util.stream.IntStream;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.NonThrowingCloseable;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ActiveOperationsConcurrencyTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(ActiveOperationsConcurrencyTest.class);
+    volatile Object blackhole;
+
+    @Test
+    public void noConcurrentModificationTest() throws InterruptedException
+    {
+        DatabaseDescriptor.daemonInitialization();
+
+        int NUM_THREADS = 32;
+        AtomicBoolean exit = new AtomicBoolean(false);
+        CopyOnWriteArrayList<Object> errors = new CopyOnWriteArrayList<>();
+
+        ActiveOperations testedObject = new ActiveOperations();
+
+        class TestThread extends Thread
+        {
+            public TestThread(int idx)
+            {
+                super("test-thread-" + idx);
+            }
+
+            @Override
+            public void run()
+            {
+                try
+                {
+                    while (!exit.get())
+                    {
+                        switch (ThreadLocalRandom.current().nextInt(4))
+                        {
+                            case 0: getTableOperationsTest(); break;
+                            case 1: onOperationStartTest(); break;
+                            case 2: getOperationsForSSTableTest(); break;
+                            case 3: isActiveTest(); break;
+                        }
+                    }
+                }
+                catch (Throwable e)
+                {
+                    logger.error("Stopping test due to error ", e);
+                    errors.add(e);
+                    exit.set(true);
+                }
+            }
+
+            private void isActiveTest()
+            {
+                LockSupport.parkNanos(ThreadLocalRandom.current().nextInt(1000000));
+                TableOperation operation = mock(TableOperation.class);
+                blackhole = testedObject.isActive(operation);
+            }
+
+            private void getOperationsForSSTableTest()
+            {
+                LockSupport.parkNanos(ThreadLocalRandom.current().nextInt(1000000));
+                SSTableReader reader = mock(SSTableReader.class);
+                blackhole = testedObject.getOperationsForSSTable(reader, OperationType.COMPACTION);
+            }
+
+            private void onOperationStartTest()
+            {
+                LockSupport.parkNanos(ThreadLocalRandom.current().nextInt(1000000));
+                TableOperation.Progress progress = mock(TableOperation.Progress.class);
+                when(progress.total()).thenReturn(ThreadLocalRandom.current().nextLong());
+                TableOperation operation = mock(TableOperation.class);
+                when(operation.getProgress()).thenReturn(progress);
+                try (NonThrowingCloseable release = testedObject.onOperationStart(operation))
+                {
+                    blackhole = release;
+                    LockSupport.parkNanos(ThreadLocalRandom.current().nextInt(1000000));
+                }
+            }
+
+            private void getTableOperationsTest()
+            {
+                LockSupport.parkNanos(ThreadLocalRandom.current().nextInt(1000000));
+                blackhole = testedObject.getTableOperations();
+            }
+        }
+
+        Thread[] threads = new Thread[NUM_THREADS];
+
+        IntStream.range(0, NUM_THREADS)
+                 .forEach(idx -> threads[idx] = new TestThread(idx));
+
+        for (Thread thread : threads)
+        {
+            thread.start();
+        }
+
+        int NUM_TICKS = 10;
+        for (int tick = 0; tick < NUM_TICKS && !exit.get(); tick++)
+        {
+            Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+            logger.info("Tick {}...", tick);
+        }
+        exit.set(true);
+
+        for (Thread thread : threads)
+        {
+            thread.join();
+        }
+
+        if (!errors.isEmpty())
+        {
+            errors.forEach(error -> logger.error("Error: ", error));
+            fail("Unexpected errors in the test");
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/ActiveOperationsTest.java b/test/unit/org/apache/cassandra/db/compaction/ActiveOperationsTest.java
new file mode 100644
index 000000000000..efdbedc86eb7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/ActiveOperationsTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.AutoSavingCache;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.view.View;
+import org.apache.cassandra.db.view.ViewBuilderTask;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.SecondaryIndexBuilder;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NonThrowingCloseable;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class ActiveOperationsTest extends CQLTester
+{
+    @BeforeClass
+    public static void beforeClass()
+    {
+        StorageService.instance.setUpDistributedSystemKeyspaces();
+    }
+
+    @Test
+    public void testActiveCompactionTrackingRaceWithIndexBuilder() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+        String idxName = createIndex("CREATE INDEX on %s(a)");
+        getCurrentColumnFamilyStore().disableAutoCompaction();
+        for (int i = 0; i < 5; i++)
+        {
+            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
+            getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        }
+
+        Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName);
+        Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+
+        ExecutorService es = Executors.newFixedThreadPool(2);
+
+        final int loopCount = 3500;
+        for (int ii = 0; ii < loopCount; ii++)
+        {
+            CountDownLatch trigger = new CountDownLatch(1);
+            SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables, true);
+            Future<?> f1 = es.submit(() -> {
+                Uninterruptibles.awaitUninterruptibly(trigger);
+                try
+                {
+                    CompactionManager.instance.submitIndexBuild(builder).get();
+                }
+                catch (Exception e)
+                {
+                    throw new RuntimeException(e);
+                }
+            });
+            Future<?> f2 = es.submit(() -> {
+                Uninterruptibles.awaitUninterruptibly(trigger);
+                CompactionManager.instance.active.getOperationsForSSTable(null, null);
+            });
+            trigger.countDown();
+            FBUtilities.waitOnFutures(Arrays.asList(f1, f2));
+        }
+        es.shutdown();
+        es.awaitTermination(1, TimeUnit.MINUTES);
+    }
+
+    @Test
+    public void testSecondaryIndexTracking() throws Throwable
+    {
+        Util.assumeLegacySecondaryIndex();
+        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+        String idxName = createIndex("CREATE INDEX on %s(a)");
+        getCurrentColumnFamilyStore().disableAutoCompaction();
+        for (int i = 0; i < 5; i++)
+        {
+            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
+            flush();
+        }
+
+        Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName);
+        Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+        SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables, false);
+
+        MockTableOperations mockActiveCompactions = new MockTableOperations();
+        CompactionManager.instance.submitIndexBuild(builder, mockActiveCompactions).get();
+
+        assertTrue(mockActiveCompactions.finished);
+        assertNotNull(mockActiveCompactions.operation);
+        assertEquals(sstables, mockActiveCompactions.operation.getProgress().sstables());
+    }
+
+    @Test
+    public void testIndexSummaryRedistributionTracking() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+        getCurrentColumnFamilyStore().disableAutoCompaction();
+        for (int i = 0; i < 5; i++)
+        {
+            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
+            flush();
+        }
+        Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+        try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstables, OperationType.INDEX_SUMMARY))
+        {
+            Map<TableId, LifecycleTransaction> transactions = ImmutableMap.<TableId, LifecycleTransaction>builder().put(getCurrentColumnFamilyStore().metadata().id, txn).build();
+            IndexSummaryRedistribution isr = new IndexSummaryRedistribution(transactions, 0, 1000);
+            MockTableOperations mockActiveCompactions = new MockTableOperations();
+            CompactionManager.instance.runIndexSummaryRedistribution(isr, mockActiveCompactions);
+            assertTrue(mockActiveCompactions.finished);
+            assertNotNull(mockActiveCompactions.operation);
+            // index redistribution operates over all keyspaces/tables, we always cancel them
+            assertTrue(mockActiveCompactions.operation.getProgress().sstables().isEmpty());
+            assertTrue(mockActiveCompactions.operation.shouldStop((sstable) -> false));
+        }
+    }
+
+    @Test
+    public void testViewBuildTracking() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, c1 int , val int, PRIMARY KEY (k1, c1))");
+        getCurrentColumnFamilyStore().disableAutoCompaction();
+        for (int i = 0; i < 5; i++)
+        {
+            execute("INSERT INTO %s (k1, c1, val) VALUES (" + i + ", 2, 3)");
+            flush();
+        }
+        execute(String.format("CREATE MATERIALIZED VIEW %s.view1 AS SELECT k1, c1, val FROM %s.%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)", keyspace(), keyspace(), currentTable()));
+        View view = Iterables.getOnlyElement(getCurrentColumnFamilyStore().viewManager);
+
+        Token token = DatabaseDescriptor.getPartitioner().getMinimumToken();
+        ViewBuilderTask vbt = new ViewBuilderTask(getCurrentColumnFamilyStore(), view, new Range<>(token, token), token, 0);
+
+        MockTableOperations mockActiveCompactions = new MockTableOperations();
+        CompactionManager.instance.submitViewBuilder(vbt, mockActiveCompactions).get();
+        assertTrue(mockActiveCompactions.finished);
+        assertTrue(mockActiveCompactions.operation.getProgress().sstables().isEmpty());
+        // this should stop for all compactions, even if it doesn't pick any sstables;
+        assertTrue(mockActiveCompactions.operation.shouldStop((sstable) -> false));
+    }
+
+    @Test
+    public void testScrubOne() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+        getCurrentColumnFamilyStore().disableAutoCompaction();
+        for (int i = 0; i < 5; i++)
+        {
+            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
+            flush();
+        }
+
+        SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null);
+        try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstable, OperationType.SCRUB))
+        {
+            MockTableOperations mockActiveCompactions = new MockTableOperations();
+            CompactionManager.instance.scrubOne(getCurrentColumnFamilyStore(), txn, IScrubber.options().skipCorrupted().build(), mockActiveCompactions);
+
+            assertTrue(mockActiveCompactions.finished);
+            assertEquals(mockActiveCompactions.operation.getProgress().sstables(), Sets.newHashSet(sstable));
+            assertFalse(mockActiveCompactions.operation.shouldStop((s) -> false));
+            assertTrue(mockActiveCompactions.operation.shouldStop((s) -> true));
+        }
+
+    }
+
+    @Test
+    public void testVerifyOne() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
+        getCurrentColumnFamilyStore().disableAutoCompaction();
+        for (int i = 0; i < 5; i++)
+        {
+            execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
+            flush();
+        }
+
+        SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null);
+        MockTableOperations mockActiveCompactions = new MockTableOperations();
+        CompactionManager.instance.verifyOne(getCurrentColumnFamilyStore(), sstable, IVerifier.options().build(), mockActiveCompactions);
+        assertTrue(mockActiveCompactions.finished);
+        assertEquals(mockActiveCompactions.operation.getProgress().sstables(), Sets.newHashSet(sstable));
+        assertFalse(mockActiveCompactions.operation.shouldStop((s) -> false));
+        assertTrue(mockActiveCompactions.operation.shouldStop((s) -> true));
+    }
+
+    @Test
+    public void testSubmitCacheWrite() throws ExecutionException, InterruptedException
+    {
+        AutoSavingCache.Writer writer = CacheService.instance.keyCache.getWriter(100);
+        MockTableOperations mockActiveCompactions = new MockTableOperations();
+        CompactionManager.instance.submitCacheWrite(writer, mockActiveCompactions).get();
+        assertTrue(mockActiveCompactions.finished);
+        assertTrue(mockActiveCompactions.operation.getProgress().sstables().isEmpty());
+    }
+
+    private static class MockTableOperations implements TableOperationObserver
+    {
+        public TableOperation operation;
+        public boolean finished = false;
+
+        public NonThrowingCloseable onOperationStart(TableOperation op)
+        {
+            this.operation = op;
+            return () -> finished = true;
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
index 97da2a4076db..665d0788105d 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
@@ -34,38 +34,43 @@
 
 import org.apache.cassandra.Util;
 import org.apache.cassandra.io.util.File;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.BeforeClass;
-import org.junit.After;
 import org.junit.Test;
 
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.RangesAtEndpoint;
-import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.repair.NoSuchRepairSessionException;
-import org.apache.cassandra.schema.MockSchema;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken;
+import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableTxnWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Refs;
-import org.apache.cassandra.UpdateBuilder;
 import org.apache.cassandra.utils.concurrent.Transactional;
 
 import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
diff --git a/test/unit/org/apache/cassandra/db/compaction/BackgroundCompactionRunnerTest.java b/test/unit/org/apache/cassandra/db/compaction/BackgroundCompactionRunnerTest.java
new file mode 100644
index 000000000000..99cf388ac9cf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/BackgroundCompactionRunnerTest.java
@@ -0,0 +1,550 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOError;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.*;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.concurrent.WrappedExecutorPlus;
+import org.apache.cassandra.utils.concurrent.Promise;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.BackgroundCompactionRunner.RequestResult;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.assertj.core.util.Lists;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+
+import static org.assertj.core.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.notNull;
+import static org.mockito.ArgumentMatchers.same;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class BackgroundCompactionRunnerTest
+{
+    private final WrappedExecutorPlus compactionExecutor = Mockito.mock(WrappedExecutorPlus.class);
+    private final ScheduledExecutorPlus checkExecutor = Mockito.mock(ScheduledExecutorPlus.class);
+    private final ActiveOperations activeOperations = Mockito.mock(ActiveOperations.class);
+    private final ColumnFamilyStore cfs = Mockito.mock(ColumnFamilyStore.class);
+    private final CompactionStrategy compactionStrategy = Mockito.mock(CompactionStrategy.class);
+
+    private BackgroundCompactionRunner runner;
+    public int pendingTaskCount;
+    private List<AbstractCompactionTask> compactionTasks;
+    private ArgumentCaptor<Runnable> capturedCompactionRunnables, capturedCheckRunnables;
+
+    private static boolean savedAutomaticSSTableUpgrade;
+    private static int savedMaxConcurrentAuotUpgradeTasks;
+
+    @BeforeClass
+    public static void initClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        savedAutomaticSSTableUpgrade = DatabaseDescriptor.automaticSSTableUpgrade();
+        savedMaxConcurrentAuotUpgradeTasks = DatabaseDescriptor.maxConcurrentAutoUpgradeTasks();
+    }
+
+    @AfterClass
+    public static void tearDownClass()
+    {
+        DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(savedAutomaticSSTableUpgrade);
+        DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(savedMaxConcurrentAuotUpgradeTasks);
+    }
+
+    @Before
+    public void initTest()
+    {
+        DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(true);
+        DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(2);
+
+        pendingTaskCount = 0;
+        runner = new BackgroundCompactionRunner(compactionExecutor, checkExecutor, activeOperations);
+        compactionTasks = new ArrayList<>();
+        capturedCompactionRunnables = ArgumentCaptor.forClass(Runnable.class);
+        capturedCheckRunnables = ArgumentCaptor.forClass(Runnable.class);
+
+        assertThat(runner.getOngoingCompactionsCount()).isZero();
+        assertThat(runner.getOngoingUpgradesCount()).isZero();
+
+        when(compactionExecutor.getMaximumPoolSize()).thenReturn(2);
+        when(cfs.isAutoCompactionDisabled()).thenReturn(false);
+        when(cfs.isValid()).thenReturn(true);
+        when(checkExecutor.getPendingTaskCount()).thenAnswer(i-> pendingTaskCount);
+        when(cfs.getCompactionStrategy()).thenReturn(compactionStrategy);
+        when(compactionStrategy.getNextBackgroundTasks(ArgumentMatchers.anyLong())).thenReturn(compactionTasks);
+        doNothing().when(checkExecutor).execute(capturedCheckRunnables.capture());
+        doNothing().when(compactionExecutor).execute(capturedCompactionRunnables.capture());
+    }
+
+    @After
+    public void tearDownTest()
+    {
+        reset(compactionExecutor, checkExecutor, activeOperations, cfs, compactionStrategy);
+    }
+
+
+    // when cfs is invalid we should immediately return ABORTED
+    @Test
+    public void invalidCFS() throws Exception
+    {
+        when(cfs.isAutoCompactionDisabled()).thenReturn(false);
+        when(cfs.isValid()).thenReturn(false);
+
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.ABORTED);
+        assertThat(runner.getMarkedCFSs()).isEmpty();
+        verify(checkExecutor, never()).execute(notNull());
+    }
+
+
+    // when automatic compactions are disabled for cfs, we should immediately return ABORTED
+    @Test
+    public void automaticCompactionsDisabled() throws Exception
+    {
+        when(cfs.isAutoCompactionDisabled()).thenReturn(true);
+        when(cfs.isValid()).thenReturn(true);
+
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.ABORTED);
+        assertThat(runner.getMarkedCFSs()).isEmpty();
+        verify(checkExecutor, never()).execute(notNull());
+    }
+
+
+    // we should mark cfs for compaction and schedule a check
+    @Test
+    public void markCFSForCompactionAndScheduleCheck() throws Exception
+    {
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+
+        assertThat(result).isNotDone();
+
+        verify(checkExecutor).execute(notNull());
+        assertThat(runner.getMarkedCFSs()).contains(cfs);
+    }
+
+
+    // when cfs is invalid we should immediately return ABORTED
+    @Test
+    public void invalidCFSs() throws Exception
+    {
+        when(cfs.isAutoCompactionDisabled()).thenReturn(false);
+        when(cfs.isValid()).thenReturn(false);
+
+        runner.markForCompactionCheck(ImmutableSet.of(cfs));
+
+        assertThat(runner.getMarkedCFSs()).isEmpty();
+        verify(checkExecutor, never()).execute(notNull());
+    }
+
+
+    // when automatic compactions are disabled for cfs, we should immediately return ABORTED
+    @Test
+    public void automaticCompactionsDisabledForCFSs() throws Exception
+    {
+        when(cfs.isAutoCompactionDisabled()).thenReturn(true);
+        when(cfs.isValid()).thenReturn(true);
+
+        runner.markForCompactionCheck(ImmutableSet.of(cfs));
+
+        assertThat(runner.getMarkedCFSs()).isEmpty();
+        verify(checkExecutor, never()).execute(notNull());
+    }
+
+
+    // we should mark cfs for compaction and schedule a check
+    @Test
+    public void markCFSsForCompactionAndScheduleCheck() throws Exception
+    {
+        runner.markForCompactionCheck(ImmutableSet.of(cfs));
+
+        verify(checkExecutor).execute(notNull());
+        assertThat(runner.getMarkedCFSs()).contains(cfs);
+    }
+
+
+    // we should mark cfs for compaction but not schedule new check if there is one already scheduled
+    @Test
+    public void markCFSForCompactionAndNotScheduleCheck() throws Exception
+    {
+        pendingTaskCount = 100;
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+
+        assertThat(result).isNotDone();
+
+        verify(checkExecutor, never()).execute(notNull());
+        assertThat(runner.getMarkedCFSs()).contains(cfs);
+    }
+
+
+    // we should immeditatlly return ABORTED if the executor is shutdown
+    @Test
+    public void immediatelyReturnIfExecutorIsDown() throws Exception
+    {
+        when(checkExecutor.isShutdown()).thenReturn(true);
+        doThrow(new RejectedExecutionException("rejected")).when(checkExecutor).execute(ArgumentMatchers.notNull());
+
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.ABORTED);
+
+        verify(checkExecutor).execute(notNull());
+    }
+
+
+    // shutdown should shut down the check executor and should not shut down the compaction executor
+    @Test
+    public void shutdown() throws Exception
+    {
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+
+        assertThat(result.isDone()).isFalse();
+
+        runner.shutdown();
+
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.ABORTED);
+
+        verify(checkExecutor).shutdown();
+        verify(compactionExecutor, never()).shutdown();
+    }
+
+
+    // a check should make a task finish with NOT_NEEDED if there are no compaction tasks and upgrades are disabled
+    @Test
+    public void finishWithNotNeededWhenNoCompactionTasksAndUpgradesDisabled() throws Exception
+    {
+        DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false);
+
+        when(cfs.getCandidatesForUpgrade()).thenReturn(ImmutableList.of(mock(SSTableReader.class)));
+
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+        verifyCFSWasMarkedForCompaction();
+        capturedCheckRunnables.getValue().run();
+
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.NOT_NEEDED);
+        verify(checkExecutor, never()).execute(notNull());
+        assertThat(runner.getMarkedCFSs()).isEmpty();
+    }
+
+    // a check should make a task finish with NOT_NEEDED if there are no compaction tasks and no upgrade tasks
+    @Test
+    public void finishWithNotNeededWhenNoCompactionTasksAndNoUpgradeTasks() throws Exception
+    {
+        when(cfs.getCandidatesForUpgrade()).thenReturn(Lists.emptyList());
+
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+        verifyCFSWasMarkedForCompaction();
+        capturedCheckRunnables.getValue().run();
+
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.NOT_NEEDED);
+        verify(checkExecutor, never()).execute(notNull());
+        assertThat(runner.getMarkedCFSs()).isEmpty();
+    }
+
+
+    // a check should start a compaction task if there is some
+    @Test
+    public void startCompactionTask() throws Exception
+    {
+        // although it is possible to run upgrade tasks, we make sure that compaction tasks are selected
+        Promise<RequestResult> result = markCFSAndRunCheck();
+
+        // check the task was scheduled on compaction executor
+        verifyTaskScheduled(compactionExecutor);
+        verifyState(1, 0);
+        assertThat(result).isNotDone();
+
+        // ... we immediatelly marked that CFS for compaction again
+        verifyCFSWasMarkedForCompaction();
+
+        // now we will execute the task
+        capturedCompactionRunnables.getValue().run();
+
+        // so we expect that:
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.COMPLETED);
+        verifyState(0, 0);
+
+        // another check should be schedued upon task completion
+        verifyCFSWasMarkedForCompaction();
+
+        // make sure we haven't even attempted to check for upgrade tasks (because there were compaction tasks to run)
+        verify(cfs, Mockito.never()).getCandidatesForUpgrade();
+    }
+
+
+    // a check should start an upgrade task if there is some and there is no compaction task
+    @Test
+    public void startUpgradeTask() throws Exception
+    {
+        AbstractCompactionTask compactionTask = mock(AbstractCompactionTask.class);
+        // although it is possible to run upgrade tasks, we make sure that compaction tasks are selected
+        DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(true);
+        SSTableReader sstable = mock(SSTableReader.class);
+        Tracker tracker = mock(Tracker.class);
+        LifecycleTransaction txn = mock(LifecycleTransaction.class);
+        when(cfs.getCandidatesForUpgrade()).thenReturn(Collections.singletonList(sstable));
+        when(cfs.getTracker()).thenReturn(tracker);
+        when(tracker.tryModify(sstable, OperationType.UPGRADE_SSTABLES)).thenReturn(txn);
+        when(compactionStrategy.createCompactionTask(same(txn), anyLong(), anyLong())).thenReturn(compactionTask);
+
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+        verifyCFSWasMarkedForCompaction();
+
+        capturedCheckRunnables.getValue().run();
+
+        // make sure we did check for the upgrade candidates
+        verify(cfs).getCandidatesForUpgrade();
+
+        // check the task was scheduled on compaction executor
+        verifyTaskScheduled(compactionExecutor);
+        verifyState(1, 1);
+        assertThat(result).isNotDone();
+
+        // ... we immediatelly marked that CFS for compaction again
+        verifyCFSWasMarkedForCompaction();
+
+        // now we will execute the task
+        capturedCompactionRunnables.getValue().run();
+
+        // so we expect that:
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.COMPLETED);
+        verifyState(0, 0);
+
+        // another check should be schedued upon task completion
+        verifyCFSWasMarkedForCompaction();
+    }
+
+
+    // we should run multiple compactions for a CFS in parallel if possible
+    @Test
+    public void startMultipleCompactionTasksInParallel() throws Exception
+    {
+        // first task
+        Promise<RequestResult> result1 = markCFSAndRunCheck();
+        verifyTaskScheduled(compactionExecutor);
+        verifyState(1, 0);
+        verifyCFSWasMarkedForCompaction();
+
+        // second task
+        Promise<RequestResult> result2 = markCFSAndRunCheck();
+        verifyTaskScheduled(compactionExecutor);
+        verifyState(2, 0);
+        verifyCFSWasMarkedForCompaction();
+
+        assertThat(result2).isNotSameAs(result1);
+
+        // now we will execute the first task
+        assertThat(result1).isNotDone();
+        capturedCompactionRunnables.getAllValues().get(0).run();
+        assertThat(result1).isDone();
+        assertThat(result1.get()).isEqualTo(RequestResult.COMPLETED);
+
+        // so we expect that:
+        verifyState(1, 0);
+
+        // execute the second task
+        assertThat(result2).isNotDone();
+        capturedCompactionRunnables.getAllValues().get(1).run();
+        assertThat(result2).isDone();
+        assertThat(result2.get()).isEqualTo(RequestResult.COMPLETED);
+
+        // so we expect that:
+        verifyState(0, 0);
+
+        verifyState(0, 0);
+    }
+
+
+    // postpone execution if the thread pool is busy
+    @Test
+    public void postponeCompactionTasksIfPoolIsBusy() throws Exception
+    {
+        // first task
+        Promise<RequestResult> result1 = markCFSAndRunCheck();
+        verifyTaskScheduled(compactionExecutor);
+        verifyState(1, 0);
+        verifyCFSWasMarkedForCompaction();
+
+        // second task
+        Promise<RequestResult> result2 = markCFSAndRunCheck();
+        verifyTaskScheduled(compactionExecutor);
+        verifyState(2, 0);
+        verifyCFSWasMarkedForCompaction();
+
+        // third task, but now the task should not be scheduled for execution because of the pool size (2)
+        clearInvocations(compactionStrategy);
+        Promise<RequestResult> result3 = markCFSAndRunCheck();
+        verifyState(2, 0);
+        // we should not execute a new task, actually not even attempt to get a new compaction task
+        verify(compactionStrategy, never()).getNextBackgroundTasks(anyInt());
+        verify(compactionExecutor, never()).execute(notNull());
+        // we should also not schedule a new check or remove mark for the CFS
+        verify(checkExecutor, never()).execute(notNull());
+        assertThat(runner.getMarkedCFSs()).contains(cfs);
+
+        assertThat(result3).isNotSameAs(result1);
+        assertThat(result3).isNotSameAs(result2);
+
+        // now we will execute the task 1
+        assertThat(result1).isNotDone();
+        capturedCompactionRunnables.getAllValues().get(0).run();
+        // so we expect that:
+        assertThat(result1).isDone();
+        assertThat(result1.get()).isEqualTo(RequestResult.COMPLETED);
+        verifyState(1, 0);
+
+        // execute the check, so that the thrid task is submitted
+        clearInvocations(checkExecutor);
+        capturedCheckRunnables.getValue().run();
+        verifyTaskScheduled(compactionExecutor);
+        verifyState(2, 0);
+        verifyCFSWasMarkedForCompaction();
+
+        // execute the rest of the tasks
+        assertThat(result2).isNotDone();
+        capturedCompactionRunnables.getAllValues().get(1).run();
+        assertThat(result2).isDone();
+        assertThat(result2.get()).isEqualTo(RequestResult.COMPLETED);
+
+        assertThat(result3).isNotDone();
+        capturedCompactionRunnables.getAllValues().get(2).run();
+        assertThat(result3).isDone();
+        assertThat(result3.get()).isEqualTo(RequestResult.COMPLETED);
+
+        verifyState(0, 0);
+    }
+
+
+    // returned future should not support complete or cancel
+    @Test
+    public void futureRequestResultNotSupportForTermination()
+    {
+        Promise<RequestResult> result = markCFSAndRunCheck();
+
+        assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> result.setSuccess(RequestResult.COMPLETED));
+        assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> result.setFailure(new RuntimeException()));
+        assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> result.cancel(false));
+
+        runner.shutdown();
+    }
+
+
+    // handling submission failure
+    @Test
+    public void handleTaskSubmissionFailure() throws Exception
+    {
+        doThrow(new RejectedExecutionException()).when(compactionExecutor).execute(notNull());
+
+        Promise<RequestResult> result = markCFSAndRunCheck();
+        clearInvocations(checkExecutor);
+
+        // so we expect that:
+        assertThat(result).isDone();
+        assertThat(result.get()).isEqualTo(RequestResult.COMPLETED);
+        verifyState(0, 0);
+
+        verify(checkExecutor, never()).execute(notNull());
+    }
+
+
+    // handling task failure
+    @Test
+    public void handleTaskFailure() throws Exception
+    {
+        Promise<RequestResult> result = markCFSAndRunCheck();
+        clearInvocations(checkExecutor);
+
+        doThrow(new IOError(new RuntimeException())).when(compactionTasks.get(0)).execute(activeOperations);
+        capturedCompactionRunnables.getValue().run();
+
+        // so we expect that:
+        assertThatThrownBy(() -> result.get()).isInstanceOf(ExecutionException.class);
+        verifyState(0, 0);
+
+        // another check should be schedued upon task completion
+        verifyCFSWasMarkedForCompaction();
+    }
+
+
+    private void verifyTaskScheduled(Executor executor)
+    {
+        verify(executor).execute(notNull());
+        clearInvocations(executor);
+    }
+
+    private void verifyState(int ongoingCompactions, int ongoingUpgrades)
+    {
+        assertThat(runner.getOngoingCompactionsCount()).isEqualTo(ongoingCompactions);
+        assertThat(runner.getOngoingUpgradesCount()).isEqualTo(ongoingUpgrades);
+    }
+
+    private void verifyCFSWasMarkedForCompaction()
+    {
+        verifyTaskScheduled(checkExecutor);
+        assertThat(runner.getMarkedCFSs()).contains(cfs);
+    }
+
+    private Promise<RequestResult> markCFSAndRunCheck()
+    {
+        AbstractCompactionTask compactionTask = mock(AbstractCompactionTask.class);
+
+        compactionTasks.clear();
+        compactionTasks.add(compactionTask);
+
+        Promise<RequestResult> result = runner.markForCompactionCheck(cfs);
+        verifyCFSWasMarkedForCompaction();
+
+        capturedCheckRunnables.getValue().run();
+        return result;
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/BackgroundCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/BackgroundCompactionsTest.java
new file mode 100644
index 000000000000..2ed5a6a0c03e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/BackgroundCompactionsTest.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.TreeMap;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.unified.AdaptiveController;
+import org.apache.cassandra.db.compaction.unified.StaticController;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TimeUUID;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+public class BackgroundCompactionsTest
+{
+    private final String keyspace = "ks";
+    private final String table = "table";
+
+    @Mock
+    private ColumnFamilyStore cfs;
+
+    @Mock
+    private CompactionStrategyContainer strategyContainer;
+
+    @Mock
+    private CompactionLogger compactionLogger;
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        DatabaseDescriptor.daemonInitialization(); // because of all the static initialization in CFS
+    }
+
+    @Before
+    public void setUp()
+    {
+        MockitoAnnotations.initMocks(this);
+
+        TableMetadata metadata = TableMetadata.builder(keyspace, table)
+                                              .addPartitionKeyColumn("pk", AsciiType.instance)
+                                              .build();
+
+        when(cfs.metadata()).thenReturn(metadata);
+        when(cfs.getKeyspaceName()).thenReturn(keyspace);
+        when(cfs.getTableName()).thenReturn(table);
+        when(compactionLogger.enabled()).thenReturn(true);
+        when(strategyContainer.getCompactionLogger()).thenReturn(compactionLogger);
+    }
+
+    private CompactionAggregate mockAggregate(long key, int numCompactions, int numCompacting)
+    {
+        if (numCompacting > numCompactions)
+            throw new IllegalArgumentException("Cannot have more compactions in progress than total compactions");
+
+        CompactionAggregate ret = Mockito.mock(CompactionAggregate.class);
+        when(ret.getKey()).thenReturn(new CompactionAggregate.Key(key));
+
+        List<CompactionPick> compactions = new ArrayList<>(numCompactions);
+        for (int i = 0; i < numCompactions; i++)
+            compactions.add(Mockito.mock(CompactionPick.class));
+
+        when(ret.numEstimatedCompactions()).thenReturn(numCompactions);
+        when(ret.getActive()).thenReturn(compactions);
+        when(ret.getInProgress()).thenReturn(compactions.subList(0, numCompacting));
+        when(ret.toString()).thenReturn(String.format("Key: %d, compactions: %d/%d", key, numCompactions, numCompacting));
+
+        return ret;
+    }
+
+    @Test
+    public void testNoCompaction()
+    {
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(0, backgroundCompactions.getTotalCompactions());
+
+        CompactionStrategyStatistics statistics = backgroundCompactions.getStatistics(strategyContainer);
+        assertNotNull(statistics);
+        assertTrue(statistics.aggregates().isEmpty());
+        assertEquals(keyspace, statistics.keyspace());
+        assertEquals(table, statistics.table());
+        assertEquals(strategyContainer.getClass().getSimpleName(), statistics.strategy());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testNullPendingCompactions()
+    {
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+        backgroundCompactions.setPending(strategyContainer, null);
+    }
+
+    @Test
+    public void testDuplicatePendingCompactionsAreMerged()
+    {
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+
+        List<CompactionAggregate> pending = new ArrayList<>(0);
+        CompactionAggregate prev = null;
+        for (int i = 0; i < 5; i++)
+        {
+            CompactionAggregate aggregate = mockAggregate(1, 1, 0);
+            pending.add(aggregate);
+
+            if (prev != null)
+            {
+                CompactionAggregate combinedAggregate = mockAggregate(1, i + 1, 0);
+                when(prev.mergeWith(any())).thenReturn(combinedAggregate);
+            }
+
+
+            prev = aggregate;
+        }
+
+        // Compactions with the same key are merged
+        backgroundCompactions.setPending(strategyContainer, pending);
+
+        assertEquals(pending.size(), backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+    }
+
+    @Test
+    public void testPendingCompactions()
+    {
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+
+        List<CompactionAggregate> pending = new ArrayList<>(0);
+        for (int i = 0; i < 5; i++)
+            pending.add(mockAggregate(i, 1, 0));
+
+        backgroundCompactions.setPending(strategyContainer, pending);
+
+        Mockito.verify(compactionLogger, never()).statistics(eq(strategyContainer), eq("pending"), any(CompactionStrategyStatistics.class));
+        Mockito.verify(compactionLogger, times(1)).pending(eq(strategyContainer), eq(pending.size()));
+
+        assertEquals(pending.size(), backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+
+        // Remove the previous pending compactions, none should be kept since they don't have in progress compactions
+        backgroundCompactions.setPending(strategyContainer, ImmutableList.of());
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(0, backgroundCompactions.getTotalCompactions());
+    }
+
+    @Test
+    public void testCompactionFromPending()
+    {
+        // Add some pending compactions, and then submit one of them, the most common case
+
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+
+        List<CompactionAggregate> pending = new ArrayList<>(0);
+        for (int i = 0; i < 5; i++)
+            pending.add(mockAggregate(i, 1, 0));
+
+        backgroundCompactions.setPending(strategyContainer, pending);
+
+        Mockito.verify(compactionLogger, never()).statistics(eq(strategyContainer), eq("pending"), any(CompactionStrategyStatistics.class));
+        Mockito.verify(compactionLogger, times(1)).pending(eq(strategyContainer), eq(pending.size()));
+
+        assertEquals(pending.size(), backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+
+        TimeUUID uuid = TimeUUID.Generator.nextTimeUUID();
+        CompactionAggregate aggregate = pending.get(0);
+        CompactionPick compaction = Mockito.mock(CompactionPick.class);
+        when(aggregate.getSelected()).thenReturn(compaction);
+        when(aggregate.getMatching(any(TreeMap.class))).thenReturn(aggregate);
+        when(aggregate.containsSameInstance(eq(compaction))).thenReturn(Pair.create(true, compaction)); // ensure the aggregate already has the compaction
+
+        backgroundCompactions.setSubmitted(strategyContainer, uuid, aggregate);
+
+        Mockito.verify(compaction, times(1)).setSubmitted(eq(uuid));
+        Mockito.verify(compactionLogger, times(1)).statistics(eq(strategyContainer), eq("submitted"), any(CompactionStrategyStatistics.class));
+
+        when(pending.get(0).numEstimatedCompactions()).thenReturn(0);
+        assertEquals(pending.size() - 1, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+
+        CompactionProgress progress = Mockito.mock(CompactionProgress.class);
+        when(progress.operationId()).thenReturn(uuid);
+
+        backgroundCompactions.onInProgress(progress);
+        Mockito.verify(compaction, times(1)).setProgress(eq(progress));
+
+        assertEquals(pending.size() - 1, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+
+        // Remove the previous pending compactions, the one submitted should be kept
+        backgroundCompactions.setPending(strategyContainer, ImmutableList.of());
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(1, backgroundCompactions.getTotalCompactions());
+
+        backgroundCompactions.onCompleted(strategyContainer, uuid);
+
+        Mockito.verify(compaction, times(1)).setCompleted();
+        Mockito.verify(compactionLogger, times(1)).statistics(eq(strategyContainer), eq("completed"), any(CompactionStrategyStatistics.class));
+
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(0, backgroundCompactions.getTotalCompactions());
+    }
+
+    @Test
+    public void testCompactionWithMatchingPending()
+    {
+        // Add some pending compactions, and then submit a compaction from an aggregate that is not in the pending
+        // but for which there is a matching aggregate, this would happen if two threads raced and created equivalent
+        // but not identical pending aggregates
+
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+
+        List<CompactionAggregate> pending = new ArrayList<>(0);
+        for (int i = 0; i < 5; i++)
+            pending.add(mockAggregate(i, 1, 0));
+
+        backgroundCompactions.setPending(strategyContainer, pending);
+
+        Mockito.verify(compactionLogger, never()).statistics(eq(strategyContainer), eq("pending"), any(CompactionStrategyStatistics.class));
+        Mockito.verify(compactionLogger, times(1)).pending(eq(strategyContainer), eq(pending.size()));
+
+        assertEquals(pending.size(), backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+
+        TimeUUID uuid = TimeUUID.Generator.nextTimeUUID();
+        CompactionAggregate aggregate = mockAggregate(0, 1, 0);
+        CompactionPick compaction = Mockito.mock(CompactionPick.class);
+        when(aggregate.getSelected()).thenReturn(compaction);
+        when(aggregate.getMatching(any(TreeMap.class))).thenReturn(pending.get(0));
+
+        CompactionPick existingCompaction = pending.get(0).getActive().get(0); // ensure the matching aggregate does not have the compaction
+        when(pending.get(0).containsSameInstance(eq(compaction))).thenReturn(Pair.create(false, existingCompaction));
+        when(pending.get(0).withReplacedCompaction(eq(compaction), eq(existingCompaction))).thenReturn(pending.get(0));
+
+        backgroundCompactions.setSubmitted(strategyContainer, uuid, aggregate);
+
+        Mockito.verify(compaction, times(1)).setSubmitted(eq(uuid));
+        Mockito.verify(compactionLogger, times(1)).statistics(eq(strategyContainer), eq("submitted"), any(CompactionStrategyStatistics.class));
+
+        when(pending.get(0).numEstimatedCompactions()).thenReturn(0);
+        assertEquals(pending.size() - 1, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+
+        CompactionProgress progress = Mockito.mock(CompactionProgress.class);
+        when(progress.operationId()).thenReturn(uuid);
+
+        backgroundCompactions.onInProgress(progress);
+        Mockito.verify(compaction, times(1)).setProgress(eq(progress));
+
+        assertEquals(pending.size() - 1, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+
+        // Remove the previous pending compactions, the one submitted should be kept
+        backgroundCompactions.setPending(strategyContainer, ImmutableList.of());
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(1, backgroundCompactions.getTotalCompactions());
+
+        backgroundCompactions.onCompleted(strategyContainer, uuid);
+
+        Mockito.verify(compaction, times(1)).setCompleted();
+        Mockito.verify(compactionLogger, times(1)).statistics(eq(strategyContainer), eq("completed"), any(CompactionStrategyStatistics.class));
+
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(0, backgroundCompactions.getTotalCompactions());
+    }
+
+    @Test
+    public void testCompactionNotInPending()
+    {
+        // Submit a compaction that is not part of a pending aggregate, this normally happens for tombstone compactions,
+        // in this case the pending aggregates are empty but a tombstone compaction is submitted
+
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+
+        backgroundCompactions.setPending(strategyContainer, ImmutableList.of());
+
+        Mockito.verify(compactionLogger, never()).statistics(eq(strategyContainer), eq("pending"), any(CompactionStrategyStatistics.class));
+        Mockito.verify(compactionLogger, times(1)).pending(eq(strategyContainer), eq(0));
+
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(0, backgroundCompactions.getTotalCompactions());
+
+        TimeUUID uuid = TimeUUID.Generator.nextTimeUUID();
+        CompactionAggregate aggregate = mockAggregate(-1, 0, 0);
+        CompactionPick compaction = Mockito.mock(CompactionPick.class);
+        when(aggregate.getSelected()).thenReturn(compaction);
+        when(aggregate.getMatching(any(TreeMap.class))).thenReturn(null);
+
+        backgroundCompactions.setSubmitted(strategyContainer, uuid, aggregate);
+
+        Mockito.verify(compaction, times(1)).setSubmitted(eq(uuid));
+        Mockito.verify(compactionLogger, times(1)).statistics(eq(strategyContainer), eq("submitted"), any(CompactionStrategyStatistics.class));
+
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(1, backgroundCompactions.getTotalCompactions());
+
+        CompactionProgress progress = Mockito.mock(CompactionProgress.class);
+        when(progress.operationId()).thenReturn(uuid);
+
+        backgroundCompactions.onInProgress(progress);
+        Mockito.verify(compaction, times(1)).setProgress(eq(progress));
+
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(1, backgroundCompactions.getTotalCompactions());
+
+        // Remove the previous pending compactions, the one submitted should be kept
+        backgroundCompactions.setPending(strategyContainer, ImmutableList.of());
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(1, backgroundCompactions.getTotalCompactions());
+
+        backgroundCompactions.onCompleted(strategyContainer, uuid);
+
+        Mockito.verify(compaction, times(1)).setCompleted();
+        Mockito.verify(compactionLogger, times(1)).statistics(eq(strategyContainer), eq("completed"), any(CompactionStrategyStatistics.class));
+
+        assertEquals(0, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(0, backgroundCompactions.getTotalCompactions());
+    }
+
+    @Test
+    public void testReplacePending()
+    {
+        // Add som pending aggregates, then replace them with aggregates with different keys, verify that only
+        // those with compactions are kept, partially overlap the keys between the old and new aggregates
+
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+
+        List<CompactionAggregate> pending = new ArrayList<>(0);
+        int key = 0;
+        for (int i = 0; i < 5; i++)
+        {
+            pending.add(mockAggregate(key++, 1, 0)); // these aggregates have no compactions
+        }
+
+        // this aggregates have a compaction
+        for (int i = 0; i < 5; i++)
+        {
+            CompactionAggregate aggregateWithComps = mockAggregate(key++, 1, 1);
+            when(aggregateWithComps.withOnlyTheseCompactions(any(Collection.class))).thenReturn(aggregateWithComps);
+            when(aggregateWithComps.getMatching(any(TreeMap.class))).thenCallRealMethod();
+            pending.add(aggregateWithComps);
+        }
+
+        backgroundCompactions.setPending(strategyContainer, pending);
+
+        assertEquals(pending.size(), backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size(), backgroundCompactions.getTotalCompactions());
+
+        pending.clear();
+
+        key -= 2; //overlap the aggregates by 2 keys
+
+        for (int i = 0; i < 5; i++)
+        {
+            // those that overlap the key need to report 2 compactions because they take the one from the old aggregate
+            // when addCompacting is called
+            CompactionAggregate aggregate = mockAggregate(key++, i < 2 ? 2 : 1, 0);
+            when(aggregate.withAdditionalCompactions(any(Collection.class))).thenReturn(aggregate);
+            pending.add(aggregate);
+        }
+
+        backgroundCompactions.setPending(strategyContainer, pending);
+
+        // the extra compactions are those from the old aggregates with a compaction regardless of whether
+        // the keys overlapped or not (when the keys overlap the new one has a compaction added, when they do
+        // not the old aggregate is used)
+        assertEquals(pending.size() + 5, backgroundCompactions.getEstimatedRemainingTasks());
+        assertEquals(pending.size() + 5, backgroundCompactions.getTotalCompactions());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetSubmittedNoId()
+    {
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+        backgroundCompactions.setSubmitted(strategyContainer, null, Mockito.mock(CompactionAggregate.class));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetSubmittedNoAggregate()
+    {
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+        backgroundCompactions.setSubmitted(strategyContainer, TimeUUID.Generator.nextTimeUUID(), null);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetSubmittedDuplicateId()
+    {
+        TimeUUID uuid = TimeUUID.Generator.nextTimeUUID();
+        CompactionAggregate aggregate = mockAggregate(1, 1, 0);
+        when(aggregate.getSelected()).thenReturn(CompactionPick.EMPTY);
+
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+        backgroundCompactions.setSubmitted(strategyContainer, uuid, aggregate);
+        backgroundCompactions.setSubmitted(strategyContainer, uuid, aggregate);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetInProgressNoProgress()
+    {
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+        backgroundCompactions.onInProgress(null);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetCompletedNoId()
+    {
+        BackgroundCompactions backgroundCompactions = new BackgroundCompactions(cfs);
+        backgroundCompactions.onCompleted(strategyContainer, null);
+    }
+
+    @Test
+    public void periodicReportsTest()
+    {
+        CompactionStrategyOptions options = mock(CompactionStrategyOptions.class);
+        BackgroundCompactions backgroundCompactions = mock(BackgroundCompactions.class);
+        UnifiedCompactionStrategy ucs = mock(UnifiedCompactionStrategy.class);
+        CompactionStrategyStatistics stats = mock(CompactionStrategyStatistics.class);
+
+        when(ucs.getOptions()).thenReturn(options);
+        when(ucs.getBackgroundCompactions()).thenReturn(backgroundCompactions);
+        when(options.isLogAll()).thenReturn(true);
+        when(options.getLogPeriodMinutes()).thenReturn(1);
+        when(backgroundCompactions.getStatistics(ucs)).thenReturn(stats);
+        when(ucs.getCompactionLogger()).thenReturn(compactionLogger);
+        doCallRealMethod().when(ucs).periodicReport();
+
+        ucs.periodicReport();
+        Mockito.verify(compactionLogger, times(1)).statistics(eq(ucs), eq("periodic"), any(CompactionStrategyStatistics.class));
+    }
+
+    @Test
+    public void controllerConfigTest()
+    {
+        UnifiedCompactionStrategy ucs = mock(UnifiedCompactionStrategy.class);
+        doCallRealMethod().when(ucs).storeControllerConfig();
+
+        AdaptiveController adaptiveController = mock(AdaptiveController.class);
+        when(ucs.getController()).thenReturn(adaptiveController);
+        ucs.storeControllerConfig();
+        Mockito.verify(adaptiveController, times(1)).storeControllerConfig();
+
+        StaticController staticController = mock(StaticController.class);
+        when(ucs.getController()).thenReturn(staticController);
+        ucs.storeControllerConfig();
+        Mockito.verify(staticController, times(1)).storeControllerConfig();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/BaseCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/BaseCompactionStrategyTest.java
new file mode 100644
index 000000000000..8b799213cb08
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/BaseCompactionStrategyTest.java
@@ -0,0 +1,392 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import org.apache.commons.math3.random.JDKRandomGenerator;
+import org.junit.Ignore;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DiskBoundaries;
+import org.apache.cassandra.db.SortedLocalRanges;
+import org.apache.cassandra.db.compaction.unified.RealEnvironment;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Splitter;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+import org.mockito.Answers;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyIterable;
+import static org.mockito.Mockito.RETURNS_SMART_NULLS;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
+
+/**
+ * A class that contains common mocks and test utilities for unit tests of compaction strategies
+ * that involve mocking compactions and sstables.
+ */
+@Ignore
+public class BaseCompactionStrategyTest
+{
+    static final double epsilon = 0.00000001;
+    static final JDKRandomGenerator random = new JDKRandomGenerator();
+
+    final String keyspace = "ks";
+    final String table = "tbl";
+
+    @Mock(answer = Answers.RETURNS_SMART_NULLS)
+    CompactionRealm realm;
+
+    @Mock
+    CompactionStrategyFactory strategyFactory;
+
+    @Mock(stubOnly = true)
+    DiskBoundaries diskBoundaries;
+
+    // Returned by diskBoundaries.getPositions() and modified by UnifiedCompactionStrategyTest
+    protected List<Token> diskBoundaryPositions = null;
+
+    int diskIndexes = 0;
+
+    Map<SSTableReader, Integer> diskIndexMap = new HashMap<>();
+
+    SortedLocalRanges localRanges;
+
+    Tracker dataTracker;
+
+    long repairedAt;
+
+    CompactionLogger compactionLogger;
+
+    IPartitioner partitioner;
+
+    Splitter splitter;
+
+    protected static void setUpClass()
+    {
+        long seed = System.currentTimeMillis();
+        random.setSeed(seed);
+        System.out.println("Random seed: " + seed);
+
+        DatabaseDescriptor.daemonInitialization(); // because of all the static initialization in CFS
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+    }
+
+    protected void setUp()
+    {
+        setUp(1);
+    }
+
+    protected void setUp(int numShards)
+    {
+        MockitoAnnotations.initMocks(this);
+
+        TableMetadata metadata = TableMetadata.builder(keyspace, table)
+                                              .addPartitionKeyColumn("pk", AsciiType.instance)
+                                              .build();
+
+        dataTracker = Tracker.newDummyTracker(TableMetadataRef.forOfflineTools(metadata));
+        repairedAt = System.currentTimeMillis();
+        partitioner = DatabaseDescriptor.getPartitioner();
+        splitter = partitioner.splitter().orElse(null);
+        if (numShards > 1)
+            assertNotNull("Splitter is required with multiple compaction shards", splitter);
+
+        when(realm.getPartitioner()).thenReturn(partitioner);
+        localRanges = SortedLocalRanges.forTestingFull(realm);
+
+        when(realm.metadata()).thenReturn(metadata);
+        when(realm.makeUCSEnvironment()).thenAnswer(invocation -> new RealEnvironment(realm));
+        when(realm.getKeyspaceName()).thenReturn(keyspace);
+        when(realm.getTableName()).thenReturn(table);
+        when(realm.getDiskBoundaries()).thenReturn(diskBoundaries);
+        when(realm.buildShardManager()).thenCallRealMethod();
+        when(diskBoundaries.getLocalRanges()).thenReturn(localRanges);
+        when(diskBoundaries.isOutOfDate()).thenReturn(false);
+        when(realm.getLiveSSTables()).thenAnswer(request -> dataTracker.getLiveSSTables());
+        when(realm.getCompactingSSTables()).thenAnswer(request -> dataTracker.getCompacting());
+        when(realm.getSSTables(any())).thenAnswer(request -> dataTracker.getView().select(request.getArgument(0)));
+        when(realm.getNoncompactingSSTables(anyIterable())).thenAnswer(request -> dataTracker.getNoncompacting(request.getArgument(0)));
+        when(realm.tryModify(anyIterable(), any())).thenAnswer(
+            request -> dataTracker.tryModify(request.getArgument(0, Iterable.class),
+                                             request.getArgument(1)));
+        when(realm.tryModify(anyIterable(), any(), any())).thenAnswer(
+        request -> dataTracker.tryModify(request.getArgument(0, Iterable.class),
+                                         request.getArgument(1),
+                                         request.getArgument(2)));
+
+        // use a real compaction logger to execute that code too, even though we don't really check
+        // the content of the files, at least we cover the code. The files will be overwritten next
+        // time the test is run or by a gradle clean task, so they will not grow indefinitely
+        compactionLogger = new CompactionLogger(realm.metadata());
+        compactionLogger.enable();
+
+        when(strategyFactory.getRealm()).thenReturn(realm);
+        when(strategyFactory.getCompactionLogger()).thenReturn(compactionLogger);
+
+        when(diskBoundaries.getNumBoundaries()).thenAnswer(invocation -> diskIndexes);
+        when(diskBoundaries.getPositions()).thenAnswer(invocationOnMock -> diskBoundaryPositions);
+        when(diskBoundaries.getDiskIndexFromKey(any())).thenAnswer(
+            request -> diskIndexMap.get(request.getArgument(0)).intValue()
+        );
+    }
+
+    /**
+     * Add sstables to the tracker, which is enough for {@link UnifiedCompactionStrategy}, but for
+     * {@link LegacyAbstractCompactionStrategy} we also need to add the sstables directly to the strategy.
+     */
+    void addSSTablesToStrategy(AbstractCompactionStrategy strategy, Iterable<SSTableReader> sstables)
+    {
+        dataTracker.addInitialSSTables(sstables);
+
+        if (strategy instanceof LegacyAbstractCompactionStrategy)
+        {
+            LegacyAbstractCompactionStrategy legacyStrategy = (LegacyAbstractCompactionStrategy) strategy;
+            for (SSTableReader sstable : sstables)
+                legacyStrategy.addSSTable(sstable);
+        }
+    }
+
+    /**
+     * Remove sstables from the tracker, which should be enough for {@link UnifiedCompactionStrategy}, but for
+     * {@link LegacyAbstractCompactionStrategy} we also need to remove the sstables directly from the strategy.
+     */
+    void removeSSTablesFromStrategy(AbstractCompactionStrategy strategy, Set<SSTableReader> sstables)
+    {
+        dataTracker.removeCompactingUnsafe(sstables);
+
+        if (strategy instanceof LegacyAbstractCompactionStrategy)
+        {
+            LegacyAbstractCompactionStrategy legacyStrategy = (LegacyAbstractCompactionStrategy) strategy;
+            for (SSTableReader sstable : sstables)
+                legacyStrategy.removeSSTable(sstable);
+        }
+    }
+
+    SSTableReader mockSSTable(int level, long bytesOnDisk, long timestamp, double hotness, DecoratedKey first, DecoratedKey last)
+    {
+        return mockSSTable(level, bytesOnDisk, timestamp, hotness, first, last,  0, true, null, 0);
+    }
+
+    SSTableReader mockSSTable(long bytesOnDisk, long timestamp, DecoratedKey first, DecoratedKey last)
+    {
+        return mockSSTable(0, bytesOnDisk, timestamp, 0, first, last,  0, true, null, 0);
+    }
+
+    SSTableReader mockSSTable(int level,
+                              long bytesOnDisk,
+                              long timestamp,
+                              double hotness,
+                              DecoratedKey first,
+                              DecoratedKey last,
+                              int diskIndex,
+                              boolean repaired,
+                              TimeUUID pendingRepair,
+                              int ttl)
+    {
+        // We create a ton of mock SSTables that mockito is going to keep until the end of the test suite without stubOnly.
+        // Mockito keeps them alive to preserve the history of invocations which is not available for stubs. If we ever
+        // need history of invocations and remove stubOnly, we should also manually reset mocked SSTables in tearDown.
+        // FIXME: This should eventually be CompactionSSTable
+        SSTableReader ret = Mockito.mock(SSTableReader.class, withSettings().stubOnly()
+                                                                            .defaultAnswer(RETURNS_SMART_NULLS));
+
+        when(ret.isSuitableForCompaction()).thenReturn(true);
+        when(ret.getSSTableLevel()).thenReturn(level);
+        when(ret.onDiskLength()).thenReturn(bytesOnDisk);
+        when(ret.uncompressedLength()).thenReturn(bytesOnDisk); // let's assume no compression
+        when(ret.hotness()).thenReturn(hotness);
+        when(ret.getMaxTimestamp()).thenReturn(timestamp);
+        when(ret.getMinTimestamp()).thenReturn(timestamp);
+        when(ret.getFirst()).thenReturn(first);
+        when(ret.getLast()).thenReturn(last);
+        when(ret.isMarkedSuspect()).thenReturn(false);
+        when(ret.isRepaired()).thenReturn(repaired);
+        when(ret.getRepairedAt()).thenReturn(repairedAt);
+        when(ret.getPendingRepair()).thenReturn(pendingRepair);
+        when(ret.isPendingRepair()).thenReturn(pendingRepair != null);
+        when(ret.getColumnFamilyName()).thenReturn(table);
+        when(ret.getKeyspaceName()).thenReturn(keyspace);
+        when(ret.getId()).thenReturn(new SequenceBasedSSTableId(level));
+        when(ret.getDataFile()).thenReturn(new File(UUIDBasedSSTableId.Builder.instance.generator(Stream.empty()).get() + ".db"));
+        when(ret.toString()).thenReturn(String.format("Bytes on disk: %s, level %d, hotness %f, timestamp %d, first %s, last %s, disk index: %d, repaired: %b, pend. repair: %b",
+                                                      FBUtilities.prettyPrintMemory(bytesOnDisk), level, hotness, timestamp, first, last, diskIndex, repaired, pendingRepair));
+        long deletionTime;
+        if (ttl > 0)
+            deletionTime = TimeUnit.MILLISECONDS.toSeconds(timestamp) + ttl;
+        else
+            deletionTime = Long.MAX_VALUE;
+
+        when(ret.getMinLocalDeletionTime()).thenReturn(deletionTime);
+        when(ret.getMaxLocalDeletionTime()).thenReturn(deletionTime);
+        when(ret.getMinTTL()).thenReturn(ttl);
+        when(ret.getMaxTTL()).thenReturn(ttl);
+
+        when(ret.estimatedKeys()).thenReturn(ShardManager.PER_PARTITION_SPAN_THRESHOLD * 2);
+
+        diskIndexMap.put(ret, diskIndex);
+        if (diskIndex >= diskIndexes)
+            diskIndexes = diskIndex + 1;
+        return ret;
+    }
+
+    List<SSTableReader> mockSSTables(int numSSTables, long bytesOnDisk, double hotness, long timestamp)
+    {
+        return mockSSTables(numSSTables, bytesOnDisk, hotness, timestamp, 0, true,null);
+    }
+
+    List<SSTableReader> mockSSTables(int numSSTables, long bytesOnDisk, double hotness, long timestamp, int diskIndex, boolean repaired, TimeUUID pendingRepair)
+    {
+        final Token minimumToken = partitioner.getMinimumToken();
+        final Token maximumToken = partitioner.getMaximumToken();
+        DecoratedKey first = new BufferDecoratedKey(partitioner.split(minimumToken, maximumToken, 0.000001 + random.nextDouble() * 0.0002),
+                                                    ByteBuffer.allocate(0));
+        DecoratedKey last = new BufferDecoratedKey(partitioner.split(minimumToken, maximumToken, 0.999999 - random.nextDouble() * 0.0002),
+                                                   ByteBuffer.allocate(0));
+        // Span may be a few fractions of percent smaller, thus density may be greater than the size by that amount
+
+        List<SSTableReader> sstables = new ArrayList<>();
+        for (int i = 0; i < numSSTables; i++)
+        {
+            long b = (long)(bytesOnDisk * 0.94 + bytesOnDisk * 0.05 * random.nextDouble()); // leave 5% variability and 1% for density
+            double h = hotness * 0.95 + hotness * 0.05 * random.nextDouble(); // leave 5% variability
+            sstables.add(mockSSTable(0, b, timestamp, h, first, last, diskIndex, repaired, pendingRepair, 0));
+        }
+
+        return sstables;
+    }
+
+    List<SSTableReader> mockNonOverlappingSSTables(int numSSTables, int level, long bytesOnDisk)
+    {
+        if (!partitioner.splitter().isPresent())
+            throw new IllegalStateException(String.format("Cannot split ranges with current partitioner %s", partitioner));
+
+        ByteBuffer emptyBuffer = ByteBuffer.allocate(0);
+
+        long timestamp = System.currentTimeMillis();
+        List<SSTableReader> sstables = new ArrayList<>(numSSTables);
+        for (int i = 0; i < numSSTables; i++)
+        {
+            DecoratedKey first = new BufferDecoratedKey(boundary(numSSTables, i + 0.0001), emptyBuffer);
+            DecoratedKey last =  new BufferDecoratedKey(boundary(numSSTables, i + 0.9999), emptyBuffer);
+            sstables.add(mockSSTable(level, bytesOnDisk, timestamp, 0., first, last));
+
+            timestamp+=10;
+        }
+
+        return sstables;
+    }
+
+    private Token boundary(int numSSTables, double i)
+    {
+        return partitioner.split(partitioner.getMinimumToken(), partitioner.getMaximumToken(), i / numSSTables);
+    }
+
+    CompactionProgress mockCompletedCompactionProgress(Set<SSTableReader> compacting, TimeUUID id)
+    {
+        CompactionProgress progress = Mockito.mock(CompactionProgress.class);
+
+        long compactingLen = totUncompressedLength(compacting);
+        when(progress.operationId()).thenReturn(id);
+        when(progress.inSSTables()).thenReturn(compacting);
+        when(progress.uncompressedBytesRead()).thenReturn(compactingLen);
+        when(progress.uncompressedBytesWritten()).thenReturn(compactingLen);
+        when(progress.durationInMillis()).thenReturn(TimeUnit.SECONDS.toMillis(30));
+
+        return progress;
+    }
+
+    void addSizeTieredOptions(Map<String, String> options)
+    {
+        addSizeTieredOptions(options, SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE);
+    }
+
+    void addSizeTieredOptions(Map<String, String> options, long minSSTableSize)
+    {
+        options.put(SizeTieredCompactionStrategyOptions.MIN_SSTABLE_SIZE_KEY, Long.toString(minSSTableSize));
+        options.put(SizeTieredCompactionStrategyOptions.BUCKET_LOW_KEY, Double.toString(SizeTieredCompactionStrategyOptions.DEFAULT_BUCKET_LOW));
+        options.put(SizeTieredCompactionStrategyOptions.BUCKET_HIGH_KEY, Double.toString(SizeTieredCompactionStrategyOptions.DEFAULT_BUCKET_HIGH));
+    }
+
+    void addTimeTieredOptions(Map<String, String> options)
+    {
+        addSizeTieredOptions(options, SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE);
+
+        options.put(TimeWindowCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, TimeUnit.MILLISECONDS.toString());
+        options.put(TimeWindowCompactionStrategyOptions.COMPACTION_WINDOW_SIZE_KEY, "30");
+        options.put(TimeWindowCompactionStrategyOptions.COMPACTION_WINDOW_UNIT_KEY, "MINUTES");
+        options.put(TimeWindowCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, Long.toString(Long.MAX_VALUE)); // disable check for expired sstables
+    }
+
+    void addLeveledOptions(Map<String, String> options, long maxSSTableSizeBytes)
+    {
+        addLeveledOptions(options, SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE, maxSSTableSizeBytes, 10);
+    }
+
+    void addLeveledOptions(Map<String, String> options, long minSSTableSizeBytes, long maxSSTableSizeBytes, int fanout)
+    {
+        addSizeTieredOptions(options, minSSTableSizeBytes);
+
+        options.put(LeveledCompactionStrategy.SSTABLE_SIZE_OPTION, Long.toString(maxSSTableSizeBytes >> 20)); // Bytes to MB
+        options.put(LeveledCompactionStrategy.LEVEL_FANOUT_SIZE_OPTION, Integer.toString(fanout));
+    }
+
+    long totUncompressedLength(Collection<? extends CompactionSSTable> sstables)
+    {
+        long ret = 0;
+        for (CompactionSSTable sstable : sstables)
+            ret += sstable.uncompressedLength();
+
+        return ret;
+    }
+
+    double totHotness(Collection<? extends CompactionSSTable> sstables)
+    {
+        double ret = 0;
+        for (CompactionSSTable sstable : sstables)
+            ret += sstable.hotness();
+
+        return ret;
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CQLUnifiedCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/CQLUnifiedCompactionTest.java
new file mode 100644
index 000000000000..afcffe418253
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CQLUnifiedCompactionTest.java
@@ -0,0 +1,374 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.stream.Collectors;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.compaction.unified.AdaptiveController;
+import org.apache.cassandra.db.compaction.unified.Controller;
+import org.apache.cassandra.db.compaction.unified.StaticController;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_L0_SHARDS_ENABLED;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * CQL tests on a table configured with Unified Compaction.
+ *
+ * The unified compaction strategy is described in this design document:
+ *
+ * TODO: link to design doc or SEP
+ *
+ * It has properties of  both tiered and leveled compactions and it adapts to the workload
+ * by switching between strategies or increasing / decreasing the fanout factor.
+ *
+ * The essential formulae are the calculations of buckets:
+ *
+ * S = ⌊log_oF(size / m)⌋ = ⌊(ln size - ln m) / (ln F + ln o)⌋
+ *
+ * where log_oF is the log with oF as the base
+ * o is the survival factor, currently fixed to 1
+ * F is the fanout factor calculated below
+ * m is the minimal size, fixed in the strategy options
+ * size is the sorted run size (sum of all the sizes of the sstables in the sorted run)
+ *
+ * Also, T is the number of sorted runs that trigger compaction.
+ *
+ * Give a parameter W, which is fixed in these tests, then T and F are calculated as follows:
+ *
+ * - W < 0 then T = 2 and F = 2 - W (leveled merge policy)
+ * - W > 0 then T = F and F = 2 + W (tiered merge policy)
+ * - W = 0 then T = F = 2 (middle ground)
+ */
+public class CQLUnifiedCompactionTest extends CQLTester
+{
+    @BeforeClass
+    public static void beforeClass()
+    {
+        UCS_L0_SHARDS_ENABLED.setBoolean(true);
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setReadThresholdsEnabled(false);
+
+        CQLTester.setUpClass();
+        StorageService.instance.initServer();
+    }
+
+    @After
+    public void tearDown()
+    {
+        // This prevents unwanted flushing in future tests
+        // Dirty CL segments cause memtables to be flushed after a schema change and we don't want this
+        // to happen asynchronously in CQLTester.afterTest() because it would interfere with the tests
+        // that rely on an exact number of sstables
+
+        for (String table: currentTables())
+        {
+            logger.debug("Dropping {} synchronously to prevent unwanted flushing due to CL dirty", table);
+            schemaChange(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, table));
+        }
+
+        CommitLog.instance.forceRecycleAllSegments();
+    }
+
+    @Test
+    public void testCreateTable()
+    {
+        createTable("create table %s (id int primary key, val text) with compaction = {'class':'UnifiedCompactionStrategy'}");
+        assertTrue(getCurrentCompactionStrategy() instanceof UnifiedCompactionStrategy);
+    }
+
+    @Test
+    public void testStaticOptions()
+    {
+        testStaticOptions(512, 2, 50, -2);
+        testStaticOptions(1024, 4, 150, 0);
+        testStaticOptions(2048, 10, 250, 2);
+    }
+
+    private void testStaticOptions(int dataSetSizeGB, int numShards, int minSSTableSize, int ... Ws)
+    {
+        testStaticOptions(false, dataSetSizeGB, numShards, minSSTableSize, Ws);
+        testStaticOptions(true, dataSetSizeGB, numShards, minSSTableSize, Ws);
+    }
+
+    private void testStaticOptions(boolean useSiUnits, long dataSetSizeGB, int numShards, long sstableSizeMB, int ... Ws)
+    {
+        String scalingParametersStr = String.join(",", Arrays.stream(Ws)
+                                                          .mapToObj(i -> Integer.toString(i))
+                                                          .collect(Collectors.toList()));
+
+        long minSizeMB = sstableSizeMB * 10 / 15;
+        createTable("create table %s (id int primary key, val text) with compaction = " +
+                    "{'class':'UnifiedCompactionStrategy', 'adaptive' : 'false', " +
+                    (useSiUnits
+                     ? String.format("'dataset_size' : '%s', ", FBUtilities.prettyPrintMemory(dataSetSizeGB << 30))
+                     : String.format("'dataset_size_in_gb' : '%d', ", dataSetSizeGB)) +
+                    String.format("'base_shard_count' : '%d', ", numShards) +
+                    (useSiUnits
+                     ? String.format("'min_sstable_size' : '%s', ", FBUtilities.prettyPrintMemory(minSizeMB << 20))
+                     : String.format("'min_sstable_size_in_mb' : '%d', ", minSizeMB)) +
+                    String.format("'target_sstable_size' : '%s', ", FBUtilities.prettyPrintMemory(sstableSizeMB << 20)) +
+                    String.format("'scaling_parameters' : '%s'}", scalingParametersStr));
+
+        CompactionStrategy strategy = getCurrentCompactionStrategy();
+        assertTrue(strategy instanceof UnifiedCompactionStrategy);
+
+        UnifiedCompactionStrategy unifiedCompactionStrategy = (UnifiedCompactionStrategy) strategy;
+        Controller controller = unifiedCompactionStrategy.getController();
+        assertEquals(dataSetSizeGB << 30, controller.getDataSetSizeBytes());
+        assertEquals(numShards, controller.getNumShards(numShards * sstableSizeMB << 20));
+        assertEquals(minSizeMB << 20, controller.getMinSstableSizeBytes());
+        assertEquals(sstableSizeMB << 20, controller.getTargetSSTableSize());
+
+        assertTrue(unifiedCompactionStrategy.getController() instanceof StaticController);
+        for (int i = 0; i < Ws.length; i++)
+            assertEquals(Ws[i], unifiedCompactionStrategy.getW(i));
+    }
+
+    @Test
+    public void testAdaptiveOptions()
+    {
+        testAdaptiveOptions(512, 2, 50, -2);
+        testAdaptiveOptions(1024, 4, 150, 0);
+        testAdaptiveOptions(2048, 10, 250, 2);
+    }
+
+    private void testAdaptiveOptions(int dataSetSizeGB, int numShards, int sstableSizeMB, int w)
+    {
+        testAdaptiveOptions(false, dataSetSizeGB, numShards, sstableSizeMB, w);
+        testAdaptiveOptions(true, dataSetSizeGB, numShards, sstableSizeMB, w);
+    }
+
+    private void testAdaptiveOptions(boolean useSiUnits, long dataSetSizeGB, int numShards, long sstableSizeMB, int w)
+    {
+        long minSizeMB = sstableSizeMB * 10 / 15;
+        createTable("create table %s (id int primary key, val text) with compaction = " +
+                    "{'class':'UnifiedCompactionStrategy', 'adaptive' : 'true', " +
+                    (useSiUnits
+                        ? String.format("'dataset_size' : '%s', ", FBUtilities.prettyPrintMemory(dataSetSizeGB << 30))
+                        : String.format("'dataset_size_in_gb' : '%d', ", dataSetSizeGB)) +
+                    String.format("'base_shard_count' : '%d', ", numShards) +
+                    (useSiUnits
+                        ? String.format("'min_sstable_size' : '%s', ", FBUtilities.prettyPrintMemory(minSizeMB << 20))
+                        : String.format("'min_sstable_size_in_mb' : '%d', ", minSizeMB)) +
+                    String.format("'target_sstable_size' : '%s', ", FBUtilities.prettyPrintMemory(sstableSizeMB << 20)) +
+                    String.format("'scaling_parameters' : '%s', ", w) +
+                    String.format("'adaptive_min_scaling_parameter' : '%s', ", -6) +
+                    String.format("'adaptive_max_scaling_parameter' : '%s', ", 16) +
+                    String.format("'adaptive_interval_sec': '%d', ", 300) +
+                    String.format("'adaptive_threshold': '%f', ", 0.25) +
+                    String.format("'max_adaptive_compactions': '%d', ", 5) +
+                    String.format("'adaptive_min_cost': '%d'}", 1));
+
+        CompactionStrategy strategy = getCurrentCompactionStrategy();
+        assertTrue(strategy instanceof UnifiedCompactionStrategy);
+
+        UnifiedCompactionStrategy unifiedCompactionStrategy = (UnifiedCompactionStrategy) strategy;
+
+        assertTrue(unifiedCompactionStrategy.getController() instanceof AdaptiveController);
+        for (int i = 0; i < 10; i++)
+            assertEquals(w, unifiedCompactionStrategy.getW(i));
+
+        AdaptiveController controller = (AdaptiveController) unifiedCompactionStrategy.getController();
+        assertEquals(dataSetSizeGB << 30, controller.getDataSetSizeBytes());
+        assertEquals(numShards, controller.getNumShards(numShards * sstableSizeMB << 20));
+        assertEquals(minSizeMB << 20, controller.getMinSstableSizeBytes());
+        assertEquals(sstableSizeMB << 20, controller.getTargetSSTableSize());
+        assertEquals(-6, controller.getMinScalingParameter());
+        assertEquals(16, controller.getMaxScalingParameter());
+        assertEquals(300, controller.getInterval());
+        assertEquals(0.25, controller.getThreshold(), 0.000001);
+        assertEquals(1, controller.getMinCost());
+        assertEquals(5, controller.getMaxRecentAdaptiveCompactions());
+    }
+
+    @Test
+    public void testAlterTable()
+    {
+        createTable("create table %s (id int primary key, val text) with compaction = {'class' : 'SizeTieredCompactionStrategy'}");
+        assertFalse(getCurrentCompactionStrategy() instanceof UnifiedCompactionStrategy);
+
+        alterTable("alter table %s with compaction = {'class' : 'UnifiedCompactionStrategy', 'adaptive' : 'true'}");
+        assertTrue(getCurrentCompactionStrategy() instanceof UnifiedCompactionStrategy);
+        assertTrue(((UnifiedCompactionStrategy) getCurrentCompactionStrategy()).getController() instanceof AdaptiveController);
+
+        alterTable("alter table %s with compaction = {'class' : 'UnifiedCompactionStrategy', 'adaptive' : 'false'}");
+        assertTrue(getCurrentCompactionStrategy() instanceof UnifiedCompactionStrategy);
+        assertTrue(((UnifiedCompactionStrategy) getCurrentCompactionStrategy()).getController() instanceof StaticController);
+    }
+
+    @Test
+    public void testSingleCompaction() throws Throwable
+    {
+        testSingleCompaction(4, 6); // W = 4 => T = 6 sstables required to trigger a compaction, see doc for formula
+        testSingleCompaction(2, 4); // W = 2 => T = 4
+        testSingleCompaction(0, 2); // W = 0 => T = 2
+        testSingleCompaction(-2, 2); // W = -2 => T = 2
+        testSingleCompaction(-4, 2); // W = -4 => T = 2
+    }
+
+    private void testSingleCompaction(int W, int T) throws Throwable
+    {
+        // Start with sstables whose size is minimal_size_in_mb, 1mb, ensure that there are no overlaps between sstables
+        int numInserts = 1024;
+        int valSize = 1024;
+
+        createTable("create table %s (id int primary key, val blob) with compaction = {'class':'UnifiedCompactionStrategy', 'adaptive' : 'false', " +
+                    String.format("'scaling_parameters' : '%d', 'min_sstable_size_in_mb' : '1', 'base_shard_count': '1', 'log_all' : 'true'}", W));
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.disableAutoCompaction();
+
+        assertEquals(0, cfs.getLiveSSTables().size());
+
+        int key = 0;
+        ByteBuffer val = ByteBuffer.wrap(new byte[valSize]);
+        for (int i = 0; i < T; i++)
+            key = insertAndFlush(numInserts, key, val);
+
+        int expectedInserts = numInserts * T;
+
+        assertEquals(T, cfs.getLiveSSTables().size());
+        assertEquals(expectedInserts, getRows(execute("SELECT * FROM %s")).length);
+
+        cfs.enableAutoCompaction(true);
+
+        assertEquals(1, cfs.getLiveSSTables().size());
+        assertEquals(expectedInserts, getRows(execute("SELECT * FROM %s")).length);
+    }
+
+    @Test
+    public void testMultipleCompactionsSingleW_Static() throws Throwable
+    {
+        // tiered tests with W = 2 and T = F = 4
+        testMultipleCompactions(4, 1, 1, new int[] {2});  //  4 sstables should be compacted into 1
+        testMultipleCompactions(8, 1, 1, new int[] {2});  //  8 sstables should be compacted into 1
+        testMultipleCompactions(16, 1, 1, new int[] {2}); // 16 sstables should be compacted into 1
+
+        // middle-point tests between tiered and leveled with W = 0, T = F = 2
+        testMultipleCompactions(2, 1, 1, new int[] {0});   // 2 sstables should be compacted into 1
+        testMultipleCompactions(4, 1, 1, new int[] {0});   // 4 sstables should be compacted into 1
+        testMultipleCompactions(8, 1, 1, new int[] {0});   // 2 sstables should be compacted into 1
+        testMultipleCompactions(16, 1, 1, new int[] {0});  // 16 sstables should be compacted into 1
+
+        // leveled tests with W = -2 and T = 2, F = 4
+        testMultipleCompactions(2, 1, 1, new int[] {-2});  //  2 sstables should be compacted into 1
+        testMultipleCompactions(4, 1, 1, new int[] {-2});  //  4 sstables should be compacted into 1
+        testMultipleCompactions(8, 1, 1, new int[] {-2});  //  8 sstables should be compacted into 1
+        testMultipleCompactions(9, 1, 1, new int[] {-2});  //  9 sstables should be compacted into 2
+        testMultipleCompactions(16, 1, 1, new int[] {-2}); // 12 sstables should be compacted into 1
+    }
+
+    @Test
+    public void testMultipleCompactionsDifferentWs_Static() throws Throwable
+    {
+        // tiered tests with W = [4, -6] and T = [6, 2], F = [6, 8]
+        testMultipleCompactions(12, 1, 1, new int[] {4, -6});  //  sstables: 12 -> (6,6) => 2 => 1
+
+        // tiered tests with W = [30, 2, -6] and T = [32, 4, 2], F = [32, 4, 8]
+        testMultipleCompactions(128, 1, 1, new int[] {30, 2, -6});  //  sstables: 128 -> (32,32, 32, 32) => 4 => (4) => 1
+    }
+
+    @Test
+    public void testMultipleCompactionsSingleW_TwoShards() throws Throwable
+    {
+        testMultipleCompactions(4, 1, 2, new int[]{2});  //  4 sstables should be compacted into 1
+        testMultipleCompactions(8, 1, 2, new int[]{2});  //  8 sstables should be compacted into 1
+    }
+
+    private void testMultipleCompactions(int numInitialSSTables, int numFinalSSTables, int numShards, int[] Ws) throws Throwable
+    {
+        int numInserts = 1024 * numShards;
+        int valSize = 2048;
+
+        String scalingParamsStr = Arrays.stream(Ws)
+                                        .mapToObj(Integer::toString)
+                                        .collect(Collectors.joining(","));
+
+        createTable("create table %s (id int primary key, val blob) with compression = { 'enabled' : false } AND " +
+                    "compaction = {'class':'UnifiedCompactionStrategy', 'adaptive' : 'false', 'max_sstables_to_compact': 256, " +
+                    String.format("'scaling_parameters' : '%s', 'min_sstable_size' : '0B', 'base_shard_count': '%d', 'log_all' : 'true'}",
+                                  scalingParamsStr, numShards));
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.disableAutoCompaction();
+
+        int key = 0;
+        byte[] bytes = new byte[valSize];
+        (new Random(87652)).nextBytes(bytes);
+        ByteBuffer val = ByteBuffer.wrap(bytes);
+
+        for (int i = 0; i < numInitialSSTables; i++)
+            key = insertAndFlush(numInserts, key, val);
+
+        int expectedInserts = numInserts * numInitialSSTables;
+
+        assertEquals(expectedInserts, getRows(execute("SELECT * FROM %s")).length);
+        assertEquals(numInitialSSTables * numShards, cfs.getLiveSSTables().size());
+
+        // trigger a compaction, wait for the future because otherwise the check below
+        // may be called before the strategy has executed getNextBackgroundTask()
+        cfs.enableAutoCompaction(true);
+
+        int numChecks = 0;
+        int numTimesWithNoCompactions = 0;
+        while(numTimesWithNoCompactions < 10  && numChecks < 1500) // 15 seconds
+        {
+            // check multiple times because we don't look ahead to future buckets at the moment so there is a brief
+            // window without pending compactions and without compactions in progress, this may make the test flaky on slow J2
+            if (cfs.getCompactionStrategy().getTotalCompactions() == 0)
+                numTimesWithNoCompactions++;
+
+            FBUtilities.sleepQuietly(10);
+            numChecks++;
+        }
+
+        assertEquals(expectedInserts, getRows(execute("SELECT * FROM %s")).length);
+        assertEquals(numFinalSSTables * numShards, cfs.getLiveSSTables().size());
+    }
+
+    private int insertAndFlush(int numInserts, int key, ByteBuffer val) throws Throwable
+    {
+        for (int i = 0; i < numInserts; i++)
+            execute("INSERT INTO %s (id, val) VALUES(?,?)", key++, val);
+
+        flush();
+        return key;
+    }
+
+    private CompactionStrategy getCurrentCompactionStrategy()
+    {
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        return cfs.getCompactionStrategyContainer()
+                  .getStrategies()
+                  .get(0);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
index b3a182f18634..4c1af0ac3fcc 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
@@ -18,7 +18,10 @@
 
 package org.apache.cassandra.db.compaction;
 
+import java.io.Closeable;
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -60,8 +63,11 @@
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.NonThrowingCloseable;
+import org.apache.cassandra.utils.Throwables;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.apache.cassandra.db.compaction.TableOperation.StopTrigger.UNIT_TESTS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -85,12 +91,12 @@ public void cancelTest() throws InterruptedException
         {
             tct.start();
 
-            List<CompactionInfo.Holder> activeCompactions = getActiveCompactionsForTable(cfs);
+            List<TableOperation> activeCompactions = getActiveCompactionsForTable(cfs);
             assertEquals(1, activeCompactions.size());
-            assertEquals(activeCompactions.get(0).getCompactionInfo().getSSTables(), toMarkCompacting);
+            assertEquals(activeCompactions.get(0).getProgress().sstables(), toMarkCompacting);
             // predicate requires the non-compacting sstables, should not cancel the one currently compacting:
             cfs.runWithCompactionsDisabled(() -> null, (sstable) -> !toMarkCompacting.contains(sstable),
-                                           OperationType.P0, false, false, true);
+                                           OperationType.P0, false, false, true, UNIT_TESTS);
             assertEquals(1, activeCompactions.size());
             assertFalse(activeCompactions.get(0).isStopRequested());
 
@@ -98,7 +104,7 @@ public void cancelTest() throws InterruptedException
             // compaction we actually run the callable (countdown the latch)
             CountDownLatch cdl = new CountDownLatch(1);
             Thread t = new Thread(() -> cfs.runWithCompactionsDisabled(() -> { cdl.countDown(); return null; }, toMarkCompacting::contains,
-                                                                       OperationType.P0, false, false, true));
+                                                                       OperationType.P0, false, false, true, UNIT_TESTS));
             t.start();
             while (!activeCompactions.get(0).isStopRequested())
                 Thread.sleep(100);
@@ -133,40 +139,40 @@ public void multipleCompactionsCancelTest() throws InterruptedException
         {
             tcts.forEach(TestCompactionTask::start);
 
-            List<CompactionInfo.Holder> activeCompactions = getActiveCompactionsForTable(cfs);
+            List<TableOperation> activeCompactions = getActiveCompactionsForTable(cfs);
             assertEquals(2, activeCompactions.size());
 
             Set<Set<SSTableReader>> compactingSSTables = new HashSet<>();
-            compactingSSTables.add(activeCompactions.get(0).getCompactionInfo().getSSTables());
-            compactingSSTables.add(activeCompactions.get(1).getCompactionInfo().getSSTables());
+            compactingSSTables.add(activeCompactions.get(0).getProgress().sstables());
+            compactingSSTables.add(activeCompactions.get(1).getProgress().sstables());
             Set<Set<SSTableReader>> expectedSSTables = new HashSet<>();
             expectedSSTables.add(new HashSet<>(sstables.subList(0, 3)));
             expectedSSTables.add(new HashSet<>(sstables.subList(6, 9)));
             assertEquals(compactingSSTables, expectedSSTables);
 
             cfs.runWithCompactionsDisabled(() -> null, (sstable) -> false,
-                                           OperationType.P0, false, false, true);
+                                           OperationType.P0, false, false, true, UNIT_TESTS);
             assertEquals(2, activeCompactions.size());
-            assertTrue(activeCompactions.stream().noneMatch(CompactionInfo.Holder::isStopRequested));
+            assertTrue(activeCompactions.stream().noneMatch(TableOperation::isStopRequested));
 
             CountDownLatch cdl = new CountDownLatch(1);
             // start a compaction which only needs the sstables where first token is > 50 - these are the sstables compacted by tcts.get(1)
             Thread t = new Thread(() -> cfs.runWithCompactionsDisabled(() -> { cdl.countDown(); return null; },
                                                                        (sstable) -> first(sstable) > 50,
-                                                                       OperationType.P0, false, false, true));
+                                                                       OperationType.P0, false, false, true, UNIT_TESTS));
             t.start();
             activeCompactions = getActiveCompactionsForTable(cfs);
             assertEquals(2, activeCompactions.size());
             Thread.sleep(500);
-            for (CompactionInfo.Holder holder : activeCompactions)
+            for (TableOperation compaction : activeCompactions)
             {
-                if (holder.getCompactionInfo().getSSTables().containsAll(sstables.subList(6, 9)))
-                    assertTrue(holder.isStopRequested());
+                if (compaction.getProgress().sstables().containsAll(sstables.subList(6, 9)))
+                    assertTrue(compaction.isStopRequested());
                 else
-                    assertFalse(holder.isStopRequested());
+                    assertFalse(compaction.isStopRequested());
             }
             tcts.get(1).abort();
-            assertEquals(1, CompactionManager.instance.active.getCompactions().size());
+            assertEquals(1, CompactionManager.instance.active.getTableOperations().size());
             cdl.await();
             t.join();
         }
@@ -194,7 +200,7 @@ public void testSubrangeCompaction() throws InterruptedException
         {
             tcts.forEach(TestCompactionTask::start);
 
-            List<CompactionInfo.Holder> activeCompactions = getActiveCompactionsForTable(cfs);
+            List<TableOperation> activeCompactions = getActiveCompactionsForTable(cfs);
             assertEquals(4, activeCompactions.size());
             Range<Token> range = new Range<>(token(0), token(49));
             Thread t = new Thread(() -> {
@@ -213,17 +219,17 @@ public void testSubrangeCompaction() throws InterruptedException
             Thread.sleep(500);
             assertEquals(4, getActiveCompactionsForTable(cfs).size());
             List<TestCompactionTask> toAbort = new ArrayList<>();
-            for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs))
+            for (TableOperation compaction : getActiveCompactionsForTable(cfs))
             {
-                if (holder.getCompactionInfo().getSSTables().stream().anyMatch(sstable -> sstable.intersects(Collections.singleton(range))))
+                if (compaction.getProgress().sstables().stream().anyMatch(sstable -> sstable.intersects(Collections.singleton(range))))
                 {
-                    assertTrue(holder.isStopRequested());
+                    assertTrue(compaction.isStopRequested());
                     for (TestCompactionTask tct : tcts)
-                        if (tct.sstables.equals(holder.getCompactionInfo().getSSTables()))
+                        if (tct.sstables.equals(compaction.getProgress().sstables()))
                             toAbort.add(tct);
                 }
                 else
-                    assertFalse(holder.isStopRequested());
+                    assertFalse(compaction.isStopRequested());
             }
             assertEquals(2, toAbort.size());
             toAbort.forEach(TestCompactionTask::abort);
@@ -237,7 +243,7 @@ public void testSubrangeCompaction() throws InterruptedException
     }
 
     @Test
-    public void testAnticompaction() throws InterruptedException, ExecutionException
+    public void testAnticompaction() throws InterruptedException, ExecutionException, IOException
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
         List<SSTableReader> sstables = createSSTables(cfs, 10, 0);
@@ -258,7 +264,7 @@ public void testAnticompaction() throws InterruptedException, ExecutionException
         {
             tcts.forEach(TestCompactionTask::start);
             nonAffectedTcts.forEach(TestCompactionTask::start);
-            List<CompactionInfo.Holder> activeCompactions = getActiveCompactionsForTable(cfs);
+            List<TableOperation> activeCompactions = getActiveCompactionsForTable(cfs);
             assertEquals(5, activeCompactions.size());
             // make sure that sstables are fully contained so that the metadata gets mutated
             Range<Token> range = new Range<>(token(-1), token(49));
@@ -273,17 +279,17 @@ public void testAnticompaction() throws InterruptedException, ExecutionException
             Future<?> fut = pac.run();
             Thread.sleep(600);
             List<TestCompactionTask> toAbort = new ArrayList<>();
-            for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs))
+            for (TableOperation compaction : getActiveCompactionsForTable(cfs))
             {
-                if (holder.getCompactionInfo().getSSTables().stream().anyMatch(sstable -> sstable.intersects(Collections.singleton(range)) && !sstable.isRepaired() && !sstable.isPendingRepair()))
+                if (compaction.getProgress().sstables().stream().anyMatch(sstable -> sstable.intersects(Collections.singleton(range)) && !sstable.isRepaired() && !sstable.isPendingRepair()))
                 {
-                    assertTrue(holder.isStopRequested());
+                    assertTrue(compaction.isStopRequested());
                     for (TestCompactionTask tct : tcts)
-                        if (tct.sstables.equals(holder.getCompactionInfo().getSSTables()))
+                        if (tct.sstables.equals(compaction.getProgress().sstables()))
                             toAbort.add(tct);
                 }
                 else
-                    assertFalse(holder.isStopRequested());
+                    assertFalse(compaction.isStopRequested());
             }
             assertEquals(2, toAbort.size());
             toAbort.forEach(TestCompactionTask::abort);
@@ -332,26 +338,26 @@ public boolean hasNext()
         indexBuildStarted.await();
         assertEquals(1, getActiveCompactionsForTable(cfs).size());
         boolean foundCompaction = false;
-        for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs))
+        for (TableOperation compaction : getActiveCompactionsForTable(cfs))
         {
-            if (holder.getCompactionInfo().getSSTables().equals(new HashSet<>(sstables)))
+            if (compaction.getProgress().sstables().equals(new HashSet<>(sstables)))
             {
-                assertFalse(holder.isStopRequested());
+                assertFalse(compaction.isStopRequested());
                 foundCompaction = true;
             }
         }
         assertTrue(foundCompaction);
         cfs.runWithCompactionsDisabled(() -> { compactionsStopped.countDown(); return null; },
-                                       (sstable) -> true, OperationType.P0, false, false, true);
+                                       (sstable) -> true, OperationType.P0, false, false, true, UNIT_TESTS);
         // wait for the runWithCompactionsDisabled callable
         compactionsStopped.await();
         assertEquals(1, getActiveCompactionsForTable(cfs).size());
         foundCompaction = false;
-        for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs))
+        for (TableOperation compaction : getActiveCompactionsForTable(cfs))
         {
-            if (holder.getCompactionInfo().getSSTables().equals(new HashSet<>(sstables)))
+            if (compaction.getProgress().sstables().equals(new HashSet<>(sstables)))
             {
-                assertTrue(holder.isStopRequested());
+                assertTrue(compaction.isStopRequested());
                 foundCompaction = true;
             }
         }
@@ -394,6 +400,7 @@ private static class TestCompactionTask
         private CompactionController controller;
         private CompactionIterator ci;
         private List<ISSTableScanner> scanners;
+        private Closeable closeable;
 
         public TestCompactionTask(ColumnFamilyStore cfs, Set<SSTableReader> sstables)
         {
@@ -408,7 +415,8 @@ public void start()
             assertNotNull(txn);
             controller = new CompactionController(cfs, sstables, Integer.MIN_VALUE);
             ci = new CompactionIterator(txn.opType(), scanners, controller, FBUtilities.nowInSeconds(), nextTimeUUID());
-            CompactionManager.instance.active.beginCompaction(ci);
+            TableOperation op = ci.getOperation();
+            closeable = CompactionManager.instance.active.onOperationStart(op);
         }
 
         public void abort()
@@ -421,8 +429,8 @@ public void abort()
                 txn.abort();
             if (scanners != null)
                 scanners.forEach(ISSTableScanner::close);
-            CompactionManager.instance.active.finishCompaction(ci);
-
+            if (closeable != null)
+                Throwables.maybeFail(Throwables.close(null, closeable));
         }
     }
 
@@ -442,7 +450,7 @@ public void test2iCancellation() throws Throwable
         try (LifecycleTransaction txn = idx.getTracker().tryModify(idx.getLiveSSTables(), OperationType.COMPACTION))
         {
             getCurrentColumnFamilyStore().runWithCompactionsDisabled(() -> true, (sstable) -> { sstables.add(sstable); return true;},
-                                                                     OperationType.P0, false, false, false);
+                                                                     OperationType.P0, false, false, false, UNIT_TESTS);
         }
         // the predicate only gets compacting sstables, and we are only compacting the 2i sstables - with interruptIndexes = false we should see no sstables here
         assertTrue(sstables.isEmpty());
@@ -473,7 +481,7 @@ public void testStandardCompactionTaskCancellation() throws Throwable
         createTable("create table %s (id int primary key, something int)");
         getCurrentColumnFamilyStore().disableAutoCompaction();
 
-        for (int i = 0; i < 10; i++)
+        for (int i = 0; i < 15; i++)
         {
             for (int j = 0; j < 3; ++j) // write more than once to ensure overlap for UCS
                 execute("insert into %s (id, something) values (?,?)", i * (j+1), i + j);
@@ -481,15 +489,15 @@ public void testStandardCompactionTaskCancellation() throws Throwable
         }
         AbstractCompactionTask ct = null;
 
-        for (List<AbstractCompactionStrategy> css : getCurrentColumnFamilyStore().getCompactionStrategyManager().getStrategies())
+        for (CompactionStrategy cs : getCurrentColumnFamilyStore().getCompactionStrategyContainer().getStrategies())
         {
-            for (AbstractCompactionStrategy cs : css)
+            Collection<AbstractCompactionTask> tasks = cs.getNextBackgroundTasks(0);
+            if (!tasks.isEmpty())
             {
-                ct = cs.getNextBackgroundTask(0);
+                ct = tasks.iterator().next();
                 if (ct != null)
                     break;
             }
-            if (ct != null) break;
         }
         assertNotNull(ct);
 
@@ -503,24 +511,24 @@ public void testStandardCompactionTaskCancellation() throws Throwable
          */
         Thread t = new Thread(() -> {
             Uninterruptibles.awaitUninterruptibly(waitForBeginCompaction);
-            getCurrentColumnFamilyStore().getCompactionStrategyManager().pause();
-            CompactionManager.instance.interruptCompactionFor(metadatas, (s) -> true, false);
+            getCurrentColumnFamilyStore().getCompactionStrategyContainer().pause();
+            CompactionManager.instance.interruptCompactionFor(metadatas, (s) -> true, false, UNIT_TESTS);
             waitForStart.countDown();
             CompactionManager.instance.waitForCessation(Collections.singleton(getCurrentColumnFamilyStore()), (s) -> true);
-            getCurrentColumnFamilyStore().getCompactionStrategyManager().resume();
+            getCurrentColumnFamilyStore().getCompactionStrategyContainer().resume();
         });
         t.start();
 
         try
         {
-            ct.execute(new ActiveCompactions()
+            ct.execute(new ActiveOperations()
             {
                 @Override
-                public void beginCompaction(CompactionInfo.Holder ci)
+                public NonThrowingCloseable onOperationStart(TableOperation op)
                 {
                     waitForBeginCompaction.countDown();
                     Uninterruptibles.awaitUninterruptibly(waitForStart);
-                    super.beginCompaction(ci);
+                    return super.onOperationStart(op);
                 }
             });
             fail("execute should throw CompactionInterruptedException");
@@ -536,11 +544,11 @@ public void beginCompaction(CompactionInfo.Holder ci)
         }
     }
 
-    private List<CompactionInfo.Holder> getActiveCompactionsForTable(ColumnFamilyStore cfs)
+    private List<TableOperation> getActiveCompactionsForTable(ColumnFamilyStore cfs)
     {
-        return CompactionManager.instance.active.getCompactions()
+        return CompactionManager.instance.active.getTableOperations()
                                                 .stream()
-                                                .filter(holder -> holder.getCompactionInfo().getTable().orElse("unknown").equalsIgnoreCase(cfs.name))
+                                                .filter(operation -> operation.getProgress().table().orElse("unknown").equalsIgnoreCase(cfs.name))
                                                 .collect(Collectors.toList());
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAggregateTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAggregateTest.java
new file mode 100644
index 000000000000..3076093ce9df
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAggregateTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Pair;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CompactionAggregateTest
+{
+    @Test
+    public void testContainsSameInstance()
+    {
+        // Create a CompactionAggregate
+        SSTableReader sstableReader = mock(SSTableReader.class);
+        CompactionAggregate agg = CompactionAggregate.createForTombstones(sstableReader);
+        // Non-existnig compaction
+        Pair<Boolean, CompactionPick> res = agg.containsSameInstance(mock(CompactionPick.class));
+        
+        assertFalse(res.left);
+        assertNull(res.right);
+        
+        // Existing compaction and same instance
+        CompactionPick existing = agg.getSelected();
+        res = agg.containsSameInstance(existing);
+        
+        assertTrue(res.left); // same instance
+        assertNotNull(res.right);
+        assertEquals(existing, res.right);
+        assertSame(existing, res.right);
+
+        // Existing compaction but different instance
+        CompactionPick otherInstance = existing.withParent(existing.parent());
+        res = agg.containsSameInstance(otherInstance);
+        
+        assertFalse(res.left); // different instance
+        assertNotNull(res.right);
+        assertEquals(otherInstance, res.right);
+        assertNotSame(otherInstance, res.right);
+    }
+
+    @Test
+    public void testWithReplacedCompaction()
+    {
+        // Create a CompactionAggregate with two compactions
+        CompactionPick anotherCompaction = Mockito.mock(CompactionPick.class);
+        when(anotherCompaction.sstables()).thenReturn(ImmutableSet.of());
+        SSTableReader sstableReader = mock(SSTableReader.class);
+        CompactionAggregate agg = CompactionAggregate.createForTombstones(sstableReader)
+                                                     .withAdditionalCompactions(ImmutableList.of(anotherCompaction));
+
+        // Setup existing and replacement CompactionPick
+        CompactionPick existing = agg.getSelected();
+        CompactionPick replacement = existing.withParent(existing.parent() + 1);
+
+        // Initial conditions
+        assertEquals(2, agg.compactions.size());
+        assertFalse(agg.compactions.contains(replacement));
+
+        // No existing CompactionPick to replace - replacement is added
+        CompactionAggregate res = agg.withReplacedCompaction(replacement, null);
+        
+        assertEquals(3, res.compactions.size());
+        assertTrue(res.compactions.contains(replacement));
+
+        // Existing CompactionPick is replaced
+        res = agg.withReplacedCompaction(replacement, existing);
+        
+        assertEquals(2, res.compactions.size());
+        assertFalse(res.compactions.contains(existing));
+        assertTrue(res.compactions.contains(replacement));
+    }
+
+    @Test
+    public void testAggregatedStatistics()
+    {
+        UnifiedCompactionStrategy.Arena arena = Mockito.mock(UnifiedCompactionStrategy.Arena.class);
+        UnifiedCompactionStrategy.Level level = Mockito.mock(UnifiedCompactionStrategy.Level.class);
+        int picksCount = 15;
+        int perPick = 4;
+        long readTput = 1_000_000;
+        long writeTput = 800_000;
+        long sstableSize = picksCount * perPick * readTput;
+        int totalTime = 0;
+        int inProgressCount = 0;
+        ArrayList<CompactionPick> compactions = new ArrayList<>();
+        for (int i = 0; i < picksCount; ++i)
+        {
+            CompactionPick pickMock = Mockito.mock(CompactionPick.class);
+            final ImmutableSet<CompactionSSTable> pickSSTables = mockSSTables(4, sstableSize);
+            when(pickMock.sstables()).thenReturn(pickSSTables);
+            final boolean inProgress = (i & 1) == 0;
+            when(pickMock.inProgress()).thenReturn(inProgress);
+            when(pickMock.submitted()).thenReturn(inProgress);
+            final boolean completed = i % 10 == 9;
+            when(pickMock.completed()).thenReturn(completed);
+            if (inProgress && !completed)
+            {
+                CompactionProgress progress = Mockito.mock(CompactionTask.CompactionOperation.class);
+                when(pickMock.progress()).thenReturn(progress);
+                final int timeInSecs = picksCount - i;
+                when(progress.durationInMillis()).thenReturn(TimeUnit.SECONDS.toMillis(timeInSecs));
+                when(progress.readThroughput()).thenCallRealMethod();
+                when(progress.writeThroughput()).thenCallRealMethod();
+                when(progress.uncompressedBytesRead()).thenReturn(timeInSecs * readTput);
+                when(progress.uncompressedBytesWritten()).thenReturn(timeInSecs * writeTput);
+                totalTime += timeInSecs;
+                ++inProgressCount;
+            }
+            compactions.add(pickMock);
+        }
+        List<CompactionSSTable> sstables = compactions.stream().flatMap(pick -> pick.sstables().stream()).collect(Collectors.toList());
+        CompactionPick selectedPick = compactions.remove(0);
+        ImmutableSet<CompactionSSTable> expired = mockSSTables(7, sstableSize);
+        sstables.addAll(expired);
+        when(selectedPick.expired()).thenReturn(expired);
+        for (CompactionPick pending : compactions)
+            when(pending.expired()).thenReturn(ImmutableSet.of());
+        CompactionAggregate agg = CompactionAggregate.createUnified(sstables, 4, selectedPick, compactions, arena, level);
+        CompactionAggregateStatistics stats = agg.getStatistics();
+        final long incompletePicks = picksCount - compactions.stream().filter(CompactionPick::completed).count();
+        assertEquals(incompletePicks, stats.numCompactions());
+        assertEquals(inProgressCount, stats.numCompactionsInProgress());
+        assertEquals(incompletePicks * perPick * sstableSize, stats.tot());
+        assertEquals(totalTime * readTput, stats.read());
+        assertEquals(totalTime * writeTput, stats.written());
+        assertEquals(inProgressCount * readTput, stats.readThroughput(), 0);
+        assertEquals(inProgressCount * writeTput, stats.writeThroughput(), 0);
+        assertEquals(7, stats.numExpiredSSTables);
+        assertEquals(7 * sstableSize, stats.totalBytesToDrop);
+    }
+
+    private static ImmutableSet<CompactionSSTable> mockSSTables(int perPick, long sstableSize)
+    {
+        Set<CompactionSSTable> sstables = Sets.newHashSet();
+        for (int i = 0; i < perPick; ++i)
+        {
+            CompactionSSTable sstableMock = Mockito.mock(CompactionSSTable.class, Mockito.withSettings().stubOnly());
+            when(sstableMock.uncompressedLength()).thenReturn(sstableSize);
+            sstables.add(sstableMock);
+        }
+        return ImmutableSet.copyOf(sstables);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
index f1777cca099e..c2ee854f20ee 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.TimeUnit;
 import java.util.function.LongPredicate;
 
@@ -35,7 +36,6 @@
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
-import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
@@ -49,10 +49,13 @@
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.WrappedRunnable;
 import org.jboss.byteman.contrib.bmunit.BMRule;
 import org.jboss.byteman.contrib.bmunit.BMRules;
 import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION;
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -108,7 +111,7 @@ public void testMaxPurgeableTimestamp()
         {
             assertPurgeBoundary(controller.getPurgeEvaluator(key), timestamp1); //memtable only
 
-            Util.flush(cfs);
+            cfs.forceBlockingFlush(UNIT_TESTS);
             assertTrue(controller.getPurgeEvaluator(key).test(Long.MAX_VALUE)); //no memtables and no sstables
         }
 
@@ -116,7 +119,7 @@ public void testMaxPurgeableTimestamp()
 
         // create another sstable
         applyMutation(cfs.metadata(), key, timestamp2);
-        Util.flush(cfs);
+        cfs.forceBlockingFlush(UNIT_TESTS);
 
         // check max purgeable timestamp when compacting the first sstable with and without a memtable
         try (CompactionController controller = new CompactionController(cfs, compacting, 0))
@@ -129,7 +132,7 @@ public void testMaxPurgeableTimestamp()
         }
 
         // check max purgeable timestamp again without any sstables but with different insertion orders on the memtable
-        Util.flush(cfs);
+        cfs.forceBlockingFlush(UNIT_TESTS);
 
         //newest to oldest
         try (CompactionController controller = new CompactionController(cfs, null, 0))
@@ -141,7 +144,7 @@ public void testMaxPurgeableTimestamp()
             assertPurgeBoundary(controller.getPurgeEvaluator(key), timestamp3); //memtable only
         }
 
-        Util.flush(cfs);
+        cfs.forceBlockingFlush(UNIT_TESTS);
 
         //oldest to newest
         try (CompactionController controller = new CompactionController(cfs, null, 0))
@@ -169,33 +172,33 @@ public void testGetFullyExpiredSSTables()
 
         // create sstable with tombstone that should be expired in no older timestamps
         applyDeleteMutation(cfs.metadata(), key, timestamp2);
-        Util.flush(cfs);
+        cfs.forceBlockingFlush(UNIT_TESTS);
 
         // first sstable with tombstone is compacting
         Set<SSTableReader> compacting = Sets.newHashSet(cfs.getLiveSSTables());
 
         // create another sstable with more recent timestamp
         applyMutation(cfs.metadata(), key, timestamp1);
-        Util.flush(cfs);
+        cfs.forceBlockingFlush(UNIT_TESTS);
 
         // second sstable is overlapping
         Set<SSTableReader> overlapping = Sets.difference(Sets.newHashSet(cfs.getLiveSSTables()), compacting);
 
         // the first sstable should be expired because the overlapping sstable is newer and the gc period is later
-        long gcBefore = (System.currentTimeMillis() / 1000) + 5;
-        Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(cfs, compacting, overlapping, gcBefore);
+        int gcBefore = (int) (System.currentTimeMillis() / 1000) + 5;
+        Set<CompactionSSTable> expired = CompactionController.getFullyExpiredSSTables(cfs, compacting, x -> overlapping, gcBefore);
         assertNotNull(expired);
         assertEquals(1, expired.size());
         assertEquals(compacting.iterator().next(), expired.iterator().next());
 
         // however if we add an older mutation to the memtable then the sstable should not be expired
         applyMutation(cfs.metadata(), key, timestamp3);
-        expired = CompactionController.getFullyExpiredSSTables(cfs, compacting, overlapping, gcBefore);
+        expired = CompactionController.getFullyExpiredSSTables(cfs, compacting, x -> overlapping, gcBefore);
         assertNotNull(expired);
         assertEquals(0, expired.size());
 
         // Now if we explicitly ask to ignore overlaped sstables, we should get back our expired sstable
-        expired = CompactionController.getFullyExpiredSSTables(cfs, compacting, overlapping, gcBefore, true);
+        expired = CompactionController.getFullyExpiredSSTables(cfs, compacting, x -> overlapping, gcBefore, true);
         assertNotNull(expired);
         assertEquals(1, expired.size());
     }
@@ -205,15 +208,15 @@ public void testGetFullyExpiredSSTables()
     @BMRule(name = "Pause compaction",
     targetClass = "CompactionTask",
     targetMethod = "runMayThrow",
-    targetLocation = "INVOKE getCompactionAwareWriter",
+    targetLocation = "INVOKE createCompactionOperation",
     condition = "Thread.currentThread().getName().equals(\"compaction1\")",
     action = "org.apache.cassandra.db.compaction.CompactionControllerTest.createCompactionControllerLatch.countDown();" +
              "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly" +
              "(org.apache.cassandra.db.compaction.CompactionControllerTest.compaction2FinishLatch);"),
     @BMRule(name = "Check overlaps",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
-    targetLocation = "INVOKE finish",
+    targetClass = "CompactionAwareWriter",
+    targetMethod = "finish",
+    targetLocation = "INVOKE finished",
     condition = "Thread.currentThread().getName().equals(\"compaction1\")",
     action = "org.apache.cassandra.db.compaction.CompactionControllerTest.compaction1RefreshLatch.countDown();" +
              "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly" +
@@ -224,109 +227,99 @@ public void testGetFullyExpiredSSTables()
     condition = "Thread.currentThread().getName().equals(\"compaction1\")",
     action = "org.apache.cassandra.db.compaction.CompactionControllerTest.incrementOverlapRefreshCounter();")
     })
-    public void testIgnoreOverlapsTrue() throws Exception
+    public void testIgnoreOverlaps() throws Exception
     {
-        resetCounters();
         testOverlapIterator(true);
-    }
-
-    @Test
-    @BMRules(rules = {
-    @BMRule(name = "Pause compaction",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
-    targetLocation = "INVOKE getCompactionAwareWriter",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.createCompactionControllerLatch.countDown();" +
-             "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly" +
-             "(org.apache.cassandra.db.compaction.CompactionControllerTest.compaction2FinishLatch);"),
-    @BMRule(name = "Check overlaps",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
-    targetLocation = "INVOKE finish",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.compaction1RefreshLatch.countDown();" +
-             "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly" +
-             "(org.apache.cassandra.db.compaction.CompactionControllerTest.refreshCheckLatch);"),
-    @BMRule(name = "Increment overlap refresh counter",
-    targetClass = "ColumnFamilyStore",
-    targetMethod = "getAndReferenceOverlappingLiveSSTables",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.incrementOverlapRefreshCounter();")
-    })
-    public void testIgnoreOverlapsFalse() throws Exception
-    {
-        resetCounters();
+        overlapRefreshCounter = 0;
+        compaction2FinishLatch = new CountDownLatch(1);
+        createCompactionControllerLatch = new CountDownLatch(1);
+        compaction1RefreshLatch = new CountDownLatch(1);
+        refreshCheckLatch = new CountDownLatch(1);
         testOverlapIterator(false);
     }
 
+    static CountDownLatch memtableRaceStartLatch = new CountDownLatch(1);
+    static CountDownLatch memtableRaceFinishLatch = new CountDownLatch(1);
+
     @Test
     @BMRules(rules = {
-    @BMRule(name = "Pause compaction",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
-    targetLocation = "INVOKE getCompactionAwareWriter",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.createCompactionControllerLatch.countDown();" +
-             "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly" +
-             "(org.apache.cassandra.db.compaction.CompactionControllerTest.compaction2FinishLatch);"),
-    @BMRule(name = "Check overlaps",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
-    targetLocation = "INVOKE finish",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.compaction1RefreshLatch.countDown();" +
-             "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly" +
-             "(org.apache.cassandra.db.compaction.CompactionControllerTest.refreshCheckLatch);"),
-    @BMRule(name = "Increment overlap refresh counter",
-    targetClass = "ColumnFamilyStore",
-    targetMethod = "getAndReferenceOverlappingLiveSSTables",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.incrementOverlapRefreshCounter();")
+      @BMRule(name = "Pause between getting and processing partition",
+              targetClass = "org.apache.cassandra.db.partitions.TrieBackedPartition",
+              targetMethod = "create",
+              targetLocation = "AT ENTRY",
+              condition = "Thread.currentThread().getName().matches(\"CompactionExecutor:.*\")",
+              action = "System.out.println(\"Byteman rule firing\");" +
+                       "org.apache.cassandra.db.compaction.CompactionControllerTest.memtableRaceStartLatch.countDown();" +
+                       "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly(" +
+                       "    org.apache.cassandra.db.compaction.CompactionControllerTest.memtableRaceFinishLatch, " +
+                       "    5, java.util.concurrent.TimeUnit.SECONDS);")
     })
-    public void testIgnoreOverlapsUCSTrue() throws Exception
+    public void testMemtableRace() throws Exception
     {
-        resetCounters();
-        testOverlapIteratorUCS(true);
+        // If CompactionController does not take an OpOrder group for reading the memtable, it is open to a race
+        // making its reads corrupted. See CNDB-11398
+
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF1);
+        cfs.truncateBlocking();
+        cfs.disableAutoCompaction();
+
+        DecoratedKey pk = Util.dk("k1");
+        for (int j = 0; j < 3; ++j)
+        {
+            writeRows(cfs, pk, j, j + 100);
+            deleteRows(cfs, pk, 0, j + 1); // make sure we have some tombstones to trigger purging
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        }
+        writeRows(cfs, pk, 5, 5 + 5000);
+
+        // We have a few sstables and a memtable, let's compact and have compaction sleep while we insert more data.
+        Runnable addDataToMemtable = new WrappedRunnable()
+        {
+            @Override
+            public void runMayThrow() throws Exception
+            {
+                assertTrue("BMRule did not trigger", memtableRaceStartLatch.await(5, TimeUnit.SECONDS));
+                writeRows(cfs, pk, 8, 8 + 5000);
+                memtableRaceFinishLatch.countDown();
+            }
+        };
+        var addDataFuture = ForkJoinPool.commonPool().submit(addDataToMemtable);
+
+        // Submit a compaction where all tombstones are expired to make compactor read memtables.
+        FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false));
+        // We must have had a signal and written data to the memtable.
+        addDataFuture.get();
+        // Compaction must have succeeded.
+        assertEquals(1, cfs.getLiveSSTables().size());
     }
 
-    @Test
-    @BMRules(rules = {
-    @BMRule(name = "Pause compaction",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
-    targetLocation = "INVOKE getCompactionAwareWriter",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.createCompactionControllerLatch.countDown();" +
-             "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly" +
-             "(org.apache.cassandra.db.compaction.CompactionControllerTest.compaction2FinishLatch);"),
-    @BMRule(name = "Check overlaps",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
-    targetLocation = "INVOKE finish",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.compaction1RefreshLatch.countDown();" +
-             "com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly" +
-             "(org.apache.cassandra.db.compaction.CompactionControllerTest.refreshCheckLatch);"),
-    @BMRule(name = "Increment overlap refresh counter",
-    targetClass = "ColumnFamilyStore",
-    targetMethod = "getAndReferenceOverlappingLiveSSTables",
-    condition = "Thread.currentThread().getName().equals(\"compaction1\")",
-    action = "org.apache.cassandra.db.compaction.CompactionControllerTest.incrementOverlapRefreshCounter();")
-    })
-    public void testIgnoreOverlapsUCSFalse() throws Exception
+    private static void writeRows(ColumnFamilyStore cfs, DecoratedKey pk, int start, int end)
     {
-        resetCounters();
-        testOverlapIteratorUCS(false);
+        for (int i = start; i < end; ++i)
+        {
+            TableMetadata cfm = cfs.metadata();
+            long timestamp = FBUtilities.timestampMicros();
+            ByteBuffer val = ByteBufferUtil.bytes(1L);
+
+            new RowUpdateBuilder(cfm, timestamp, pk)
+            .clustering("ck" + i)
+            .add("val", val)
+            .build()
+            .applyUnsafe();
+        }
     }
 
-    private void resetCounters()
+    private static void deleteRows(ColumnFamilyStore cfs, DecoratedKey pk, int start, int end)
     {
-        overlapRefreshCounter = 0;
-        compaction2FinishLatch = new CountDownLatch(1);
-        createCompactionControllerLatch = new CountDownLatch(1);
-        compaction1RefreshLatch = new CountDownLatch(1);
-        refreshCheckLatch = new CountDownLatch(1);
+        for (int i = start; i < end; ++i)
+        {
+            TableMetadata cfm = cfs.metadata();
+            long timestamp = FBUtilities.timestampMicros();
+
+            RowUpdateBuilder.deleteRowAt(cfm, timestamp, (int) (timestamp / 1000000), pk, "ck" + i)
+                            .applyUnsafe();
+        }
     }
 
     public void testOverlapIterator(boolean ignoreOverlaps) throws Exception
@@ -351,101 +344,22 @@ public void testOverlapIterator(boolean ignoreOverlaps) throws Exception
         assertEquals(cfs.getLiveSSTables().size(), 2);
         String sstable2 = cfs.getLiveSSTables().iterator().next().getFilename();
 
-        CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.setBoolean(true);
+        ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.setBoolean(true);
         Map<String, String> options = new HashMap<>();
         options.put(TimeWindowCompactionStrategyOptions.COMPACTION_WINDOW_SIZE_KEY, "30");
         options.put(TimeWindowCompactionStrategyOptions.COMPACTION_WINDOW_UNIT_KEY, "SECONDS");
         options.put(TimeWindowCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
         options.put(TimeWindowCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0");
         options.put(TimeWindowCompactionStrategyOptions.UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY, Boolean.toString(ignoreOverlaps));
-        TimeWindowCompactionStrategy twcs = new TimeWindowCompactionStrategy(cfs, options);
+        TimeWindowCompactionStrategy twcs = new TimeWindowCompactionStrategy(new CompactionStrategyFactory(cfs), options);
         for (SSTableReader sstable : cfs.getLiveSSTables())
             twcs.addSSTable(sstable);
 
         twcs.startup();
 
-        CompactionTask task = (CompactionTask)twcs.getUserDefinedTask(sstables, 0);
-        assertFalse(task.keepOriginals);
-
-        assertNotNull(task);
-        assertEquals(1, Iterables.size(task.transaction.originals()));
-
-        //start a compaction for the first sstable (compaction1)
-        //the overlap iterator should contain sstable2
-        //this compaction will be paused by the BMRule
-        Thread t = new Thread(() -> {
-            task.execute(null);
-        });
-
-        //start a compaction for the second sstable (compaction2)
-        //the overlap iterator should contain sstable1
-        //this compaction should complete as normal
-        Thread t2 = new Thread(() -> {
-            Uninterruptibles.awaitUninterruptibly(createCompactionControllerLatch);
-            assertEquals(1, overlapRefreshCounter);
-            CompactionManager.instance.forceUserDefinedCompaction(sstable2);
-
-            //after compaction2 is finished, wait 1 minute and then resume compaction1 (this gives enough time for the overlapIterator to be refreshed)
-            //after resuming, the overlap iterator for compaction1 should be updated to include the new sstable created by compaction2,
-            //and it should not contain sstable2
-            try
-            {
-                TimeUnit.MINUTES.sleep(1);
-            }
-            catch (InterruptedException e)
-            {
-                throw new RuntimeException(e);
-            }
-            compaction2FinishLatch.countDown();
-        });
-
-        t.setName("compaction1");
-        t.start();
-        t2.start();
-
-        compaction1RefreshLatch.await();
-        //at this point, the overlap iterator for compaction1 should be refreshed
-
-        //verify that the overlap iterator for compaction1 is refreshed twice, (once during the constructor, and again after compaction2 finishes)
-        assertEquals(2, overlapRefreshCounter);
-
-        refreshCheckLatch.countDown();
-        t.join();
-    }
+        CompactionTasks tasks = twcs.getUserDefinedTasks(sstables, 0);
 
-    public void testOverlapIteratorUCS(boolean ignoreOverlaps) throws Exception
-    {
-
-        Keyspace keyspace = Keyspace.open(KEYSPACE);
-        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF1);
-        cfs.truncateBlocking();
-        cfs.disableAutoCompaction();
-
-        //create 2 overlapping sstables
-        DecoratedKey key = Util.dk("k1");
-        long timestamp1 = FBUtilities.timestampMicros();
-        long timestamp2 = timestamp1 - 5;
-        applyMutation(cfs.metadata(), key, timestamp1);
-        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
-        assertEquals(cfs.getLiveSSTables().size(), 1);
-        Set<SSTableReader> sstables = cfs.getLiveSSTables();
-
-        applyMutation(cfs.metadata(), key, timestamp2);
-        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
-        assertEquals(cfs.getLiveSSTables().size(), 2);
-        String sstable2 = cfs.getLiveSSTables().iterator().next().getFilename();
-
-        CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.setBoolean(true);
-        Map<String, String> options = new HashMap<>();
-        options.put(TimeWindowCompactionStrategyOptions.UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY, Boolean.toString(ignoreOverlaps));
-        UnifiedCompactionStrategy ucs = new UnifiedCompactionStrategy(cfs, options);
-        for (SSTableReader sstable : cfs.getLiveSSTables())
-            ucs.addSSTable(sstable);
-
-        ucs.startup();
-
-        CompactionTask task = (CompactionTask)ucs.getUserDefinedTask(sstables, 0);
-        assertFalse(task.keepOriginals);
+        CompactionTask task = (CompactionTask) tasks.iterator().next();
 
         assertNotNull(task);
         assertEquals(1, Iterables.size(task.transaction.originals()));
@@ -454,7 +368,7 @@ public void testOverlapIteratorUCS(boolean ignoreOverlaps) throws Exception
         //the overlap iterator should contain sstable2
         //this compaction will be paused by the BMRule
         Thread t = new Thread(() -> {
-            task.execute(null);
+            task.executeInternal();
         });
 
         //start a compaction for the second sstable (compaction2)
@@ -520,37 +434,4 @@ public static void incrementOverlapRefreshCounter()
     {
         overlapRefreshCounter++;
     }
-
-    @Test
-    public void testDisableNeverPurgeTombstones()
-    {
-        Keyspace keyspace = Keyspace.open(KEYSPACE);
-        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);
-        cfs.truncateBlocking();
-
-        DecoratedKey key = Util.dk("k1");
-        long timestamp = System.currentTimeMillis();
-        applyMutation(cfs.metadata(), key, timestamp);
-        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
-        Set<SSTableReader> toCompact = Sets.newHashSet(cfs.getLiveSSTables());
-        cfs.setNeverPurgeTombstones(true);
-        applyMutation(cfs.metadata(), key, timestamp + 1);
-
-        try (CompactionController cc = new CompactionController(cfs, toCompact, (int)(System.currentTimeMillis()/1000)))
-        {
-            assertFalse(cc.getPurgeEvaluator(key).test(timestamp));
-            assertFalse(cc.getPurgeEvaluator(key).test(timestamp + 1));
-            assertTrue(cc.getFullyExpiredSSTables().isEmpty());
-
-            cfs.setNeverPurgeTombstones(false);
-            assertFalse(cc.getPurgeEvaluator(key).test(timestamp));
-            assertFalse(cc.getPurgeEvaluator(key).test(timestamp + 1));
-            assertTrue(cc.getFullyExpiredSSTables().isEmpty());
-
-            cc.maybeRefreshOverlaps();
-            assertTrue(cc.getPurgeEvaluator(key).test(timestamp));
-            assertFalse(cc.getPurgeEvaluator(key).test(timestamp + 1));
-            assertTrue(cc.getFullyExpiredSSTables().isEmpty());
-        }
-    }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionInfoTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionInfoTest.java
deleted file mode 100644
index 753a18505dec..000000000000
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionInfoTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.compaction;
-
-import java.util.ArrayList;
-import java.util.UUID;
-
-import org.junit.Test;
-
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.repair.AbstractPendingAntiCompactionTest;
-import org.apache.cassandra.schema.MockSchema;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.utils.TimeUUID;
-import org.assertj.core.api.Assertions;
-
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-
-public class CompactionInfoTest extends AbstractPendingAntiCompactionTest
-{
-    @Test
-    public void testCompactionInfoToStringContainsTaskId()
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS();
-        TimeUUID expectedTaskId = nextTimeUUID();
-        CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, expectedTaskId, new ArrayList<>());
-        Assertions.assertThat(compactionInfo.toString())
-                  .contains(expectedTaskId.toString());
-    }
-
-    @Test
-    public void testCompactionInfoToStringFormat()
-    {
-        UUID tableId = UUID.randomUUID();
-        TimeUUID taskId = nextTimeUUID();
-        ColumnFamilyStore cfs = MockSchema.newCFS(builder -> builder.id(TableId.fromUUID(tableId)));
-        CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, taskId, new ArrayList<>());
-        Assertions.assertThat(compactionInfo.toString())
-                  .isEqualTo("Compaction(%s, 0 / 1000 bytes)@%s(mockks, mockcf1)", taskId, tableId);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
index 2dc4e12df5fb..5d25db2809bb 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
@@ -17,18 +17,25 @@
  */
 package org.apache.cassandra.db.compaction;
 
-import static org.apache.cassandra.config.CassandraRelevantProperties.DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
 import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.assertCommandIssued;
 import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.makeRow;
-import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.partition;
-import static org.junit.Assert.*;
-
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.collect.*;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
@@ -42,17 +49,27 @@
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowsGenerator;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS;
+import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.partition;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 public class CompactionIteratorTest extends CQLTester
 {
 
@@ -245,22 +262,7 @@ private void verifyEquivalent(List<List<Unfiltered>> sources, List<Unfiltered> r
 
     private List<List<Unfiltered>> parse(String[] inputs, UnfilteredRowsGenerator generator)
     {
-        return ImmutableList.copyOf(Lists.transform(Arrays.asList(inputs), x -> parse(x, generator)));
-    }
-
-    private List<Unfiltered> parse(String input, UnfilteredRowsGenerator generator)
-    {
-        Matcher m = Pattern.compile("D(\\d+)\\|").matcher(input);
-        if (m.lookingAt())
-        {
-            int del = Integer.parseInt(m.group(1));
-            input = input.substring(m.end());
-            List<Unfiltered> list = generator.parse(input, NOW - 1);
-            deletionTimes.put(list, DeletionTime.build(del, del));
-            return list;
-        }
-        else
-            return generator.parse(input, NOW - 1);
+        return ImmutableList.copyOf(Lists.transform(Arrays.asList(inputs), x -> generator.parse(x, NOW - 1, deletionTimes)));
     }
 
     private List<Unfiltered> compact(Iterable<List<Unfiltered>> sources, Iterable<List<Unfiltered>> tombstoneSources)
@@ -335,12 +337,13 @@ public void transformTest()
                                                               Lists.transform(content, x -> new Scanner(x)),
                                                               controller, NOW, null))
         {
+            TableOperation op = iter.getOperation();
             assertTrue(iter.hasNext());
             UnfilteredRowIterator rows = iter.next();
             assertTrue(rows.hasNext());
             assertNotNull(rows.next());
 
-            iter.stop();
+            op.stop(TableOperation.StopTrigger.UNIT_TESTS);
             try
             {
                 // Will call Transformation#applyToRow
@@ -368,7 +371,8 @@ public void transformPartitionTest()
                                                               Lists.transform(content, x -> new Scanner(x)),
                                                               controller, NOW, null))
         {
-            iter.stop();
+            TableOperation op = iter.getOperation();
+            op.stop(TableOperation.StopTrigger.UNIT_TESTS);
             try
             {
                 // Will call Transformation#applyToPartition
@@ -456,6 +460,12 @@ public Set<SSTableReader> getBackingSSTables()
         {
             return ImmutableSet.of();
         }
+
+        @Override
+        public int level()
+        {
+            return 0;
+        }
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionSimulationTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionSimulationTest.java
new file mode 100644
index 000000000000..5706d0cd44cc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionSimulationTest.java
@@ -0,0 +1,1517 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.inject.Inject;
+
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.RateLimiter;
+import org.apache.commons.math3.distribution.AbstractIntegerDistribution;
+import org.apache.commons.math3.distribution.UniformIntegerDistribution;
+import org.apache.commons.math3.distribution.ZipfDistribution;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.clearspring.analytics.hash.MurmurHash;
+import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
+import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+import com.clearspring.analytics.stream.cardinality.ICardinality;
+import io.airlift.airline.Command;
+import io.airlift.airline.HelpOption;
+import io.airlift.airline.Option;
+import io.airlift.airline.SingleCommand;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.compaction.unified.AdaptiveController;
+import org.apache.cassandra.db.compaction.unified.Controller;
+import org.apache.cassandra.db.compaction.unified.CostsCalculator;
+import org.apache.cassandra.db.compaction.unified.Environment;
+import org.apache.cassandra.db.compaction.unified.Reservations;
+import org.apache.cassandra.db.compaction.unified.StaticController;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.utils.ExpMovingAverage;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.MovingAverage;
+import org.apache.cassandra.utils.Overlaps;
+import org.apache.cassandra.utils.TimeUUID;
+import org.mockito.Mockito;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOG_DIR;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.when;
+
+/**
+ * A test that simulates compactions to see how strategies behave.
+ * <p/>
+ * SSTables are mocked with a specific cardinality {@link ICardinality} that provides
+ * an estimated number of keys in the sstable, e.g. {@link HyperLogLogPlus}.
+ * <p/>
+ * Integers are sampled from a probability distribution such us {@link UniformIntegerDistribution} or {@link ZipfDistribution}
+ * and offered to a cardinality object. When the estimated number of objects in the cardinality reaches a threshold,
+ * an sstable is mocked using this cardinality and the compaction strategy is given the sstable and asked to check for
+ * compactions. If there is a compaction task, then it is placed in a queue and the operation is repeated
+ * from the beginning until the desired number of sampled integers has been reached.
+ * <p/>
+ * Another thread waits for compaction tasks that are put on the queue. When there is a compaction task,
+ * the cardinalities of the sstables in the compaction task are merged and a new sstable is created with the
+ * merged cardinality and given to the strategy, which checks again for any compaction events so that they can
+ * be put on the queue as well. The process then continues until the queue is empty.
+ * <p/>
+ * The simulation completes when both threads have terminated.
+ * <p/>
+ * The size of the sstables is given by the estimated number of objects in the cardinality times a fixed size.
+ * <p/>
+ * The following values are calculated and reported at the end of the simulation:
+ *
+ * <li> Write Amplification (WA): number of entries written (by either flushing or compacting) / number of inserts </li>
+ * <li> Read or Space Amplification (RA): histogram of sorted runs (anything better?) </li>
+ * <li> Sorted runs existing at the end of the simulation </li>
+ * <li> Compaction strategy statistics for the entire simulation </li>
+ */
+@Command(name = "compactionSim", description = "Compaction Simulation Tests")
+@Ignore
+public class CompactionSimulationTest extends BaseCompactionStrategyTest
+{
+    private final static Logger logger = LoggerFactory.getLogger(CompactionSimulationTest.class);
+
+    private static final String logDirectory = LOG_DIR.getString();
+
+    /**
+     * The average time for flushing 1kb of data, as measured on Fallout tests ran on ironic.
+     */
+    private long flushTimeMicros = 20;
+
+    /**
+     * The average time for compacting 1kb of data, as measured on Fallout tests ran on ironic.
+     */
+    private long compactionTimeMicros = 45;
+
+    /**
+     * The average time for reading an entire partition, as measured on Fallout tests ran on ironic.
+     */
+    private long partitionReadLatencyMicros = 150;
+
+    /** How often we append values to the csv file */
+    private static final int csvUpdatePeriodMs = 500;
+
+    /** Only collect values for final averages after this warmup period */
+    private static final int warmupPeriodSec = 15;
+
+    /** The minimum sstable size in bytes */
+    private static final long sstableSize = 500 << 20; // 50 MB
+
+    /** The number of unique keys that cause an sstable to be flushed, the value size is calculated by dividing
+     * {@link this#sstableSize} by this value. The smaller this value is, the greater the number of sstables generated.
+     */
+    private static final int uniqueKeysPerSStable = 50000;
+
+    /** When calculating the read cost, we multiply by this factor to simulate a Bloom Filter false positive rate of 1%. So
+     * we estimate that we'll access 1% of the live sstables.
+     */
+    private static final double bfFactor = 0.01;
+
+    /**
+     * When calculating the read cost, we multiply by this factor to simulate a cache hit rate of 1 - cacheFactor.
+     */
+    private static final double cacheFactor = 0.05;
+
+    @Inject
+    public HelpOption helpOption;
+
+    @Option(name = { "-wl", "--workload" }, description = "Workload type specified as RXX_WXX, e.g. R50_W50")
+    String workload = "R50_W50";
+
+    @Option(name = { "-t", "--type" }, description = "The test type: either \"static\" or \"adaptive\"")
+    String type = "adaptive";
+
+    @Option(name= {"-min"}, description = "The minimum value of W")
+    int minW = -10;
+
+    @Option(name= {"-max"}, description = "The maximum value of W")
+    int maxW = 32;
+
+    @Option(name= {"--data-size"}, description = "The data set size in GB")
+    int datasetSizeGB = 32;
+
+    @Option(name= {"--num-shards"}, description = "The number of compaction shards")
+    int numShards = 4;
+
+    @Option(name= {"--min-cost"}, description = "The minimum cost for adaptive analysis")
+    int minCost = 5;
+
+    @Option(name= {"--max-adaptive-compactions"}, description = "The max nunmber of concurrent adaptive compactions")
+    int maxAdaptiveCompactions = 5;
+
+    @Option(name= {"--gain"}, description = "The gain for adaptive analysis")
+    double gain = 0.15;
+
+    @Option(name= {"-step"}, description = "The step size for W for static analysis")
+    int stepW = 2;
+
+    @Option(name= {"-w"}, description = "The initial value of W for adaptive analysis")
+    int W = 0;
+
+    @Option(name= {"-update-time"}, description = "The update interval in seconds for adaptive analysis")
+    int updateTimeSec = 15;
+
+    @Option(name= {"-duration"}, description = "The duration in minutes for adaptive analysis or for each step in static analysis")
+    int durationMinutes = 1;
+
+    @Option(name= {"-expired-sstable-check-frequency"}, description = "How often to check for expired SSTables")
+    long expiredSSTableCheckFrequency = 600;
+
+    @Option(name= {"-unsafe-aggressive-sstable-expiration"}, description = "Whether to drop expired SSTables without checking if the partitions appear in other SSTables")
+    boolean ignoreOverlaps = false;
+
+    @Option(name= {"-base-shard-count"}, description = "Base shard count, 4 by default")
+    int baseShardCount = 4;
+
+    @Option(name= {"-target_sstable_size_mb"}, description = "Target sstable size in mb, 1024 by default")
+    long targetSSTableSizeMB = 1024;
+
+    @Option(name= {"-overlap-inclusion-method"}, description = "Overlap inclusion method, NONE, SINGLE or TRANSITIVE")
+    Overlaps.InclusionMethod overlapInclusionMethod = Overlaps.InclusionMethod.TRANSITIVE;
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        BaseCompactionStrategyTest.setUpClass();
+    }
+
+    @Before
+    public void setUp()
+    {
+        setUp(numShards);
+        logger.info("Simulation set up for data size of {} GiB, {} shards", datasetSizeGB, numShards);
+    }
+
+    public static void main(String[] args) throws Exception
+    {
+        setUpClass();
+
+        CompactionSimulationTest test = SingleCommand.singleCommand(CompactionSimulationTest.class).parse(args);
+
+        if (test.helpOption.showHelpIfRequested())
+            return;
+
+        test.setUp();
+        test.run();
+    }
+
+    public void run() throws Exception
+    {
+        Pattern WL_REGEX = Pattern.compile("^R(\\d+)_W(\\d+)$");
+        Matcher matcher = WL_REGEX.matcher(workload.toUpperCase());
+        if (!matcher.matches())
+            throw new IllegalArgumentException(String.format("Invalid workload %s.", workload));
+
+        int readRowsSec = Integer.parseInt(matcher.group(1)) * 10000;
+        int writeRowsSec = Integer.parseInt(matcher.group(2)) * 10000;
+        System.out.println(String.format("Running %s with %d read rows / sec and %d write rows /sec", workload, readRowsSec, writeRowsSec));
+        logger.info("Running {} with {} read rows / sec and {} write rows /sec", workload, readRowsSec, writeRowsSec);
+
+        if (type.toLowerCase().equals("static"))
+            testStaticAnalysis(workload, readRowsSec, writeRowsSec);
+        else if (type.toLowerCase().equals("adaptive"))
+            testAdaptiveController(workload, readRowsSec, writeRowsSec);
+        else
+            throw new IllegalArgumentException("Invalid type: " + type);
+
+    }
+
+    @Test
+    public void testAdaptiveController_R50_W50() throws Exception
+    {
+        int readRowsSec = 50_000;
+        int writeRowsSec = 50_000;
+
+        testAdaptiveController("R50_W50", readRowsSec, writeRowsSec);
+    }
+
+    @Test
+    public void testStaticAnalysis_R50_W50() throws Exception
+    {
+        int readRowsSec = 50_000;
+        int writeRowsSec = 50_000;
+
+        testStaticAnalysis("R50_W50", readRowsSec, writeRowsSec);
+    }
+
+    @Test
+    public void testSingleW() throws Exception
+    {
+        int W = 2; // similar to tiered with 4 sorted runs per bucket
+        int writeRowsSec = 1_000_000;
+        int readRowsSec = 1_000_000;
+        int maxKey = 30_000_000;
+
+        CsvWriter csvWriter = CsvWriter.make("testUniform_UnifiedStrategy");
+        testUniform(false, csvWriter, W, sstableSize, TimeUnit.MINUTES.toMillis(1), maxKey, readRowsSec, writeRowsSec, NO_OP_OBSERVER);
+    }
+
+    /**
+     * Run a simulation using {@link UnifiedCompactionStrategy} with an initial value of W and let the adaptive
+     * controller choose the best value depending on the workloa
+     */
+    private void testAdaptiveController(String dataSetName, int readRowsSec, int writeRowsSec) throws Exception
+    {
+        int maxKey = 100_000_000;
+
+        String csvFileName = "testAdaptiveController_" + dataSetName;
+        CsvWriter csvWriter = CsvWriter.make(csvFileName);
+
+        testUniform(true, csvWriter, W, sstableSize, TimeUnit.MINUTES.toMillis(durationMinutes), maxKey, readRowsSec, writeRowsSec, NO_OP_OBSERVER);
+        clearSSTables();
+    }
+
+    /**
+     * Run a simulation using {@link UnifiedCompactionStrategy} with different values of W and for a different number of
+     * trials. Report the average IO cost over the period. This can then be plotted as a function of W to show the
+     * impact that W has on the IO cost depending on the workload type (see callers).
+     */
+    private void testStaticAnalysis(String dataSetName, int readRowsSec, int writeRowsSec) throws Exception
+    {
+        int maxKey = 50_000_000;
+
+        String csvFileName = "testStaticAnalysis_" + dataSetName;
+        CsvWriter csvWriter = CsvWriter.make(csvFileName);
+
+        for (int w = minW; w <= maxW; w += stepW)
+        {
+            testUniform(false, csvWriter, w, sstableSize, TimeUnit.MINUTES.toMillis(durationMinutes), maxKey, readRowsSec, writeRowsSec, NO_OP_OBSERVER);
+            clearSSTables();
+        }
+    }
+
+    private void testUniform(boolean adaptive,
+                             CsvWriter csvWriter,
+                             int W,
+                             long sstableSize,
+                             long durationMillis,
+                             int maxKey,
+                             int readRowsSec,
+                             int writeRowsSec,
+                             SimulationObserver observer) throws Exception
+    {
+        if (maxKey <= 0)
+            fail("Maxkey should be positive");
+
+        int valueSize = (int) Math.ceil(sstableSize / (double) uniqueKeysPerSStable); // value length for each key
+
+        logger.debug("Running simulation with uniform distribution, max key: {}, duration: {} ms, maxKey: {}, keys/sstable: {}, value size: {}, min sstable size: {}",
+                     maxKey, durationMillis, maxKey, uniqueKeysPerSStable, valueSize, FBUtilities.prettyPrintMemory(sstableSize));
+
+        AbstractIntegerDistribution distribution = new UniformIntegerDistribution(random, 0, maxKey);
+
+        Counters counters = new Counters();
+        UnifiedCompactionStrategy strategy = createUnifiedCompactionStrategy(counters, adaptive, W, sstableSize, valueSize);
+
+        Simulation simulation = new Simulation(strategy,
+                                               distribution,
+                                               csvWriter,
+                                               counters,
+                                               maxKey,
+                                               uniqueKeysPerSStable,
+                                               valueSize,
+                                               durationMillis,
+                                               readRowsSec, writeRowsSec,
+                                               observer);
+        simulation.run();
+    }
+
+    private void clearSSTables()
+    {
+        Iterable<SSTableReader> sstables = Iterables.concat(dataTracker.getLiveSSTables(), dataTracker.getCompacting());
+        for (SSTableReader sstable : sstables)
+            Mockito.reset(sstable);
+
+        dataTracker.removeUnsafe(dataTracker.getLiveSSTables());
+        dataTracker.removeCompactingUnsafe(dataTracker.getCompacting());
+        repairedAt = System.currentTimeMillis();
+
+        assertTrue(dataTracker.getLiveSSTables().isEmpty());
+        assertTrue(dataTracker.getCompacting().isEmpty());
+    }
+
+    private UnifiedCompactionStrategy createUnifiedCompactionStrategy(Counters counters, boolean adaptive, int W, long sstableSize, int valueSize)
+    {
+        double o = 1.0;
+        int[] Ws = new int[] { W };
+        int[] previousWs = new int[] { W };
+        double maxSpaceOverhead = 0.2;
+
+        Controller controller = adaptive
+                                ? new AdaptiveController(MonotonicClock.Global.preciseTime,
+                                                         new SimulatedEnvironment(counters, valueSize), Ws, previousWs,
+                                                         new double[] { o },
+                                                         ((long) datasetSizeGB) << 33,  // leave some room
+                                                         sstableSize,
+                                                         0,
+                                                         0,
+                                                         maxSpaceOverhead,
+                                                         0,
+                                                         expiredSSTableCheckFrequency,
+                                                         ignoreOverlaps,
+                                                         baseShardCount,
+                                                         false,
+                                                         targetSSTableSizeMB << 20,
+                                                         0,
+                                                         0,
+                                                         Reservations.Type.PER_LEVEL,
+                                                         overlapInclusionMethod,
+                                                         true,
+                                                         false,
+                                                         updateTimeSec,
+                                                         minW,
+                                                         maxW,
+                                                         gain,
+                                                         minCost,
+                                                         maxAdaptiveCompactions,
+                                                         "ks",
+                                                         "tbl")
+                                : new StaticController(new SimulatedEnvironment(counters, valueSize),
+                                                       Ws,
+                                                       new double[] { o },
+                                                       ((long) datasetSizeGB) << 33,  // leave some room
+                                                       sstableSize,
+                                                       0,
+                                                       0,
+                                                       maxSpaceOverhead, // MB
+                                                       0,
+                                                       expiredSSTableCheckFrequency,
+                                                       ignoreOverlaps,
+                                                       baseShardCount,
+                                                       false,
+                                                       targetSSTableSizeMB << 20,
+                                                       0,
+                                                       0,
+                                                       Reservations.Type.PER_LEVEL,
+                                                       overlapInclusionMethod,
+                                                       true,
+                                                       false,
+                                                       "ks",
+                                                       "tbl");
+
+        return new UnifiedCompactionStrategy(strategyFactory, controller);
+    }
+
+    private final static class CsvWriter
+    {
+        private final OutputStreamWriter updateWriter;
+        private final OutputStreamWriter averagesWriter;
+        private boolean headerWritten;
+
+        private CsvWriter(String fileName) throws IOException
+        {
+            this.updateWriter =  new OutputStreamWriter(Files.newOutputStream(Paths.get(logDirectory, fileName + ".csv"), StandardOpenOption.CREATE, StandardOpenOption.WRITE));
+            this.averagesWriter =  new OutputStreamWriter(Files.newOutputStream(Paths.get(logDirectory, fileName + "-avg.csv"), StandardOpenOption.CREATE, StandardOpenOption.WRITE));
+            this.headerWritten = false;
+        }
+
+        static CsvWriter make(String fileName) throws IOException
+        {
+            return new CsvWriter(fileName);
+        }
+
+        void writeHeader(String toWrite)
+        {
+            if (!headerWritten)
+            {
+                performWrite(toWrite, updateWriter);
+                performWrite(toWrite, averagesWriter);
+                headerWritten = true;
+            }
+        }
+
+        void write(String toWrite)
+        {
+            performWrite(toWrite, updateWriter);
+        }
+
+        void writeAverages(String toWrite)
+        {
+            performWrite(toWrite, averagesWriter);
+        }
+
+        private synchronized void performWrite(String toWrite, OutputStreamWriter writer)
+        {
+            try
+            {
+                writer.write(toWrite);
+                writer.flush();
+            }
+            catch (IOException ex)
+            {
+                logger.error("Failed to write to csv: ", ex);
+            }
+        }
+    }
+
+    /**
+     * Some counters for the simulation
+     */
+    private final static class Counters
+    {
+        /** The simulated number of rows inserted by the user. */
+        final AtomicLong numInserted = new AtomicLong(0L);
+
+        /** The simulated number of rows requested by the user. */
+        final AtomicLong numRequested = new AtomicLong(0L);
+
+        /** The simulated number of rows flushed */
+        final AtomicLong numFlushed = new AtomicLong(0L);
+
+        /** The simulated number of rows read during compaction */
+        final AtomicLong numReadForCompaction = new AtomicLong(0L);
+
+        /** The simulated number of rows written during compaction */
+        final AtomicLong numWrittenForCompaction = new AtomicLong(0L);
+
+        /** The simulated number of rows written to disk (by flushing or compactions). */
+        final AtomicLong numWritten = new AtomicLong(0L);
+
+        /** The number of compactions simulated */
+        final AtomicLong numCompactions = new AtomicLong(0L);
+
+        /** The number of compactions submitted but not yet executed */
+        final AtomicInteger numCompactionsPending = new AtomicInteger(0);
+
+        /** The number of sstables that have been compacted away */
+        final AtomicLong numCompactedSSTables = new AtomicLong(0L);
+
+        void reset()
+        {
+            numInserted.set(0);
+            numRequested.set(0);
+            numFlushed.set(0);
+            numReadForCompaction.set(0);
+            numWrittenForCompaction.set(0);
+            numWritten.set(0);
+            numCompactions.set(0);
+            numCompactionsPending.set(0);
+            numCompactedSSTables.set(0);
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("Ins: %d (%d%%), Req: %d (%d%%), Flushed: %d, Written: %d",
+                                 numInserted.get(),
+                                 percentageInserted(),
+                                 numRequested.get(),
+                                 percentageRead(),
+                                 numFlushed.get(),
+                                 numWritten.get());
+        }
+
+        int percentageInserted()
+        {
+            double tot = Math.max(1, numInserted.get() + numRequested.get());
+            return (int) ((numInserted.get() / tot) * 100);
+        }
+
+        int percentageRead()
+        {
+            double tot = Math.max(1, numInserted.get() + numRequested.get());
+            return (int) ((numRequested.get() / tot) * 100);
+        }
+    }
+
+    /**
+     * An implementation of {@link Environment} that uses simulated values.
+     */
+    private class SimulatedEnvironment implements Environment
+    {
+        final Counters counters;
+        final int valueSize;
+
+        SimulatedEnvironment(Counters counters, int valueSize)
+        {
+            this.counters = counters;
+            this.valueSize = valueSize;
+        }
+
+        @Override
+        public MovingAverage makeExpMovAverage()
+        {
+            return ExpMovingAverage.decayBy100();
+        }
+
+        @Override
+        public double cacheMissRatio()
+        {
+            return cacheFactor;
+        }
+
+        @Override
+        public double bloomFilterFpRatio()
+        {
+            return bfFactor;
+        }
+
+        @Override
+        public int chunkSize()
+        {
+            return PageAware.PAGE_SIZE;
+        }
+
+        @Override
+        public long bytesInserted()
+        {
+            return counters.numInserted.get() * valueSize;
+        }
+
+        @Override
+        public long partitionsRead()
+        {
+            return counters.numRequested.get();
+        }
+
+        @Override
+        public double sstablePartitionReadLatencyNanos()
+        {
+            return TimeUnit.MICROSECONDS.toNanos(partitionReadLatencyMicros);
+        }
+
+        @Override
+        public double compactionTimePerKbInNanos()
+        {
+            // this is slightly incorrect, we would need to measure the size of compacted sstables
+            return TimeUnit.MICROSECONDS.toNanos(compactionTimeMicros);
+        }
+
+        @Override
+        public double flushTimePerKbInNanos()
+        {
+            return TimeUnit.MICROSECONDS.toNanos(flushTimeMicros);
+        }
+
+        @Override
+        public double WA()
+        {
+            double bytesFlushed = counters.numFlushed.get() * valueSize;
+            double bytesCompacted = counters.numWrittenForCompaction.get() * valueSize;
+            return bytesFlushed <= 0 ? 0 : (bytesFlushed + bytesCompacted) / bytesFlushed;
+        }
+
+        @Override
+        public double flushSize()
+        {
+            return uniqueKeysPerSStable * valueSize; // a rough estimation should be fine
+        }
+
+        @Override
+        public int maxConcurrentCompactions()
+        {
+            return DatabaseDescriptor.getConcurrentCompactors();
+        }
+
+        @Override
+        public double maxThroughput()
+        {
+            return Double.MAX_VALUE;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("Read latency: %d us / partition, flush latency: %d us / KiB, compaction latency: %d us / KiB, bfpr: %f, measured WA: %.2f, flush size %s",
+                                 TimeUnit.NANOSECONDS.toMicros((long) sstablePartitionReadLatencyNanos()),
+                                 TimeUnit.NANOSECONDS.toMicros((long) flushTimePerKbInNanos()),
+                                 TimeUnit.NANOSECONDS.toMicros((long) compactionTimePerKbInNanos()),
+                                 bloomFilterFpRatio(),
+                                 WA(),
+                                 FBUtilities.prettyPrintMemory((long)flushSize()));
+        }
+    }
+
+    /**
+     * The output of the simulation
+     */
+    private final class SimulationOutput
+    {
+        /** The initial timestamp */
+        private final long start;
+
+        /** The compaction strategy */
+        private final UnifiedCompactionStrategy strategy;
+
+        /** The compaction cost calculator */
+        private final CostsCalculator calculator;
+
+        /** Save the read IO costs after the warm-up period for calculating the final average and stddev, TODO - can we do it wihtout a list? */
+        private final List<Double> readIOCosts;
+
+        /** Save the write IO costs after the warm-up period for calculating the final average and stddev, TODO - can we do it wihtout a list? */
+        private final List<Double> writeIOCosts;
+
+        /**
+         * Creates an initial empty status and writes the header to the CSV file.
+         */
+        SimulationOutput(long start, CsvWriter writer, UnifiedCompactionStrategy strategy)
+        {
+            this.start = start;
+            this.strategy = strategy;
+            this.calculator = strategy.getController().getCalculator();
+            this.readIOCosts = new ArrayList<>();
+            this.writeIOCosts = new ArrayList<>();
+
+            writeCSVHeader(writer);
+        }
+
+        private void writeCSVHeader(CsvWriter writer)
+        {
+            writer.writeHeader(String.join(",",
+                                           "timestamp ms",
+                                           "W",
+                                           "num compactions",
+                                           "live sstables",
+                                           "space used bytes",
+                                           "Tot Num inserted",
+                                           "Tot Num read",
+                                           "% inserted",
+                                           "% read",
+                                           "Read IO",
+                                           "Read IO stddev",
+                                           "Write IO",
+                                           "Write IO stddev",
+                                           "Tot IO",
+                                           "Tot IO stddev",
+                                           "Num. pending",
+                                           "WA")
+                               + System.lineSeparator());
+        }
+
+        private void write(CsvWriter writer, Counters counters)
+        {
+            int W = strategy.getW(0);
+            long length = (long) Math.ceil(calculator.spaceUsed());
+            int RA = strategy.getController().readAmplification(length, W);
+            int WA = strategy.getController().writeAmplification(length, W);
+
+            double readIOCost = calculator.getReadCostForQueries(RA);
+            double writeIOCost = calculator.getWriteCostForQueries(WA);
+
+            if (System.currentTimeMillis() - start >= TimeUnit.SECONDS.toMillis(warmupPeriodSec))
+            {
+                this.readIOCosts.add(readIOCost);
+                this.writeIOCosts.add(writeIOCost);
+            }
+
+            String toWrite = String.join(",",
+                                         toString(System.currentTimeMillis() - start),
+                                         toString(W),
+                                         toString(counters.numCompactions.get()),
+                                         toString(calculator.numSSTables()),
+                                         toString(length),
+                                         toString(counters.numInserted.get()),
+                                         toString(counters.numRequested.get()),
+                                         toString(counters.percentageInserted()),
+                                         toString(counters.percentageRead()),
+                                         toString(readIOCost),
+                                         "0",
+                                         toString(writeIOCost),
+                                         "0",
+                                         toString(readIOCost + writeIOCost),
+                                         "0",
+                                         toString(counters.numCompactionsPending.get() + strategy.getEstimatedRemainingTasks()),
+                                         toString(calculator.getEnv().WA()))
+                             + System.lineSeparator();
+
+            writer.write(toWrite);
+        }
+
+        private void writeAverages(CsvWriter writer, Counters counters)
+        {
+            double writeIOCostAvg = average(writeIOCosts);
+            double writeIOCostStd = stddev(writeIOCostAvg, writeIOCosts);
+
+            double readIOCostAvg = average(readIOCosts);
+            double readIOCostStd = stddev(readIOCostAvg, readIOCosts);
+
+
+            String toWrite = String.join(",",
+                                         toString(System.currentTimeMillis() - start),
+                                         toString(strategy.getW(0)),
+                                         toString(counters.numCompactions.get()),
+                                         toString(calculator.numSSTables()),
+                                         toString(calculator.spaceUsed()),
+                                         toString(counters.numInserted.get()),
+                                         toString(counters.numRequested.get()),
+                                         toString(counters.percentageInserted()),
+                                         toString(counters.percentageRead()),
+                                         toString(readIOCostAvg),
+                                         toString(readIOCostStd),
+                                         toString(writeIOCostAvg),
+                                         toString(writeIOCostStd),
+                                         toString(readIOCostAvg + writeIOCostAvg),
+                                         toString(readIOCostStd + writeIOCostStd),
+                                         toString(counters.numCompactionsPending.get() + strategy.getEstimatedRemainingTasks()),
+                                         toString(calculator.getEnv().WA()))
+                             + System.lineSeparator();
+
+            writer.writeAverages(toWrite);
+        }
+
+        public double average(List<Double> vals)
+        {
+            return vals.isEmpty() ? 0 : vals.stream().reduce(Double::sum).get() / vals.size();
+        }
+
+        public double stddev(double avg, List<Double> vals)
+        {
+            if (vals.isEmpty())
+                return 0;
+
+            double sd = 0;
+            for (double v : vals)
+                sd += Math.pow(v - avg, 2);
+
+            return Math.sqrt(sd / vals.size());
+        }
+
+        private String toString(long val)
+        {
+            return String.format("%d", val);
+        }
+
+        private String toString(double val)
+        {
+            return String.format("%.6f", val);
+        }
+
+        @Override
+        public String toString()
+        {
+            long elapsed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - start);
+            return String.format("W: %d, num. sstables: %d, elapsed: %d s",
+                                 strategy.getW(0),
+                                 dataTracker.getLiveSSTables().size(),
+                                 elapsed);
+        }
+    }
+
+    /**
+     * A simple state machine for the simulation
+     */
+    private enum SimulationState
+    {
+        NONE, // the simulation hasn't yet started or is pre-loading data
+        SETTING_UP, // the simulation is setting up, e.g. pre-loading data and waiting for initial set of compactions
+        RUNNING, // the simulation is running: inserting data and/or reading data and reporting the output
+        TEARING_DOWN, //the simulation is tearing down (waiting for threads to complete)
+        DONE // the simulation is finishing compactions or done
+    }
+
+    /**
+     * Implemented by tests that need to react to simulation progress
+     */
+    private interface SimulationObserver
+    {
+        void onChange(SimulationState state);
+    }
+
+    private static SimulationObserver NO_OP_OBSERVER = state -> {};
+
+    /**
+     * The implementation of the simulation
+     */
+    private final class Simulation
+    {
+        /** The strategy to test */
+        private final UnifiedCompactionStrategy strategy;
+
+        /** The distribution is used to generated values to be inserted */
+        private final AbstractIntegerDistribution distribution;
+
+        /** The simulation output will be passed to this csv writer */
+        private final CsvWriter csvWriter;
+
+        /** The maximum key to insert when pre-loading data, this is also normally the maximum key value of the data distribution */
+        private final int maxKey;
+
+        /** The number of unique keys that trigger an sstable to be created */
+        private final int uniqueKeysPerSStable;
+
+        /** The fixed value size for each key inserted */
+        private final int valueSize;
+
+        /** The simulation duration in milliseconds, it will keep on reading and writing for this period of time
+         * according to the rate limiters below */
+        private final long durationMillis;
+
+        /** The insertion rate limited is based on the insert rows / sec received in the c.tor */
+        private final RateLimiter writeRate;
+
+        /** The read rate limited is based on the read rows / sec received in the c.tor */
+        private final RateLimiter readRate;
+
+        /** These are the compactions that have been submitted by the strategy */
+        private final BlockingQueue<AbstractCompactionTask> compactions;
+
+        /** The cardinalities for the sstables to be flushed */
+        private final BlockingQueue<ICardinality> flushing;
+
+        /** The simulation counters */
+        private final Counters counters;
+
+        /** This is set in case of error to fail the test */
+        private final AtomicReference<Throwable> error;
+
+        /** A simulation observer */
+        private final SimulationObserver observer;
+
+        /** The simulation state */
+        private final AtomicReference<SimulationState> state;
+
+        /** The start of the simulation */
+        private volatile long start;
+
+        /** Contains output parameters that the simulation should produce periodically. */
+        private volatile SimulationOutput output;
+
+        /**
+         * Create a new simulation
+         * @param strategy the strategy to test
+         * @param distribution the distribution to generate random integer keys
+         * @param csvWriter writes statistics to a csv file
+         * @param maxKey the maximum value of the key,
+         * @param uniqueKeysPerSStable the number of unique keys that trigger an sstable to be created
+         * @param valueSize the fixed size for the value associated to each key
+         * @param durationMillis the duration of the simulation read and write phases
+         * @param readRowsSec the simulated number of rows to be read every second
+         * @param writeRowsSec the simulated number of rows to be inserted every second
+         */
+        Simulation(UnifiedCompactionStrategy strategy,
+                   AbstractIntegerDistribution distribution,
+                   CsvWriter csvWriter,
+                   Counters counters,
+                   int maxKey,
+                   int uniqueKeysPerSStable,
+                   int valueSize,
+                   long durationMillis,
+                   int readRowsSec, int writeRowsSec,
+                   SimulationObserver observer)
+        {
+            this.strategy = strategy;
+            this.distribution = distribution;
+            this.csvWriter = csvWriter;
+            this.maxKey = maxKey;
+            this.uniqueKeysPerSStable = uniqueKeysPerSStable;
+            this.valueSize = valueSize;
+            this.durationMillis = durationMillis;
+            this.writeRate = writeRowsSec > 0 ? RateLimiter.create(writeRowsSec) : null;
+            this.readRate = readRowsSec > 0 ? RateLimiter.create(readRowsSec) : null;
+            this.compactions =  new ArrayBlockingQueue<>(512); // flushing / compaction thread will be blocked when queue is full
+            this.flushing = new ArrayBlockingQueue<>(256); // insert thread will be blocked when queue is full
+            this.counters = counters;
+
+            this.error = new AtomicReference<>(null);
+            this.state = new AtomicReference<>(SimulationState.NONE);
+            this.observer = observer;
+        }
+
+        void run() throws Exception
+        {
+            if (state.get() != SimulationState.NONE)
+                throw new IllegalStateException("Simulation already run!");
+
+            try
+            {
+                NamedThreadFactory threadFactory = new NamedThreadFactory("Simulation-worker");
+
+                setState(SimulationState.NONE, SimulationState.SETTING_UP);
+
+                this.start = System.currentTimeMillis();
+                strategy.getController().startup(strategy, ScheduledExecutors.scheduledTasks);
+                this.output = new SimulationOutput(start, csvWriter, strategy);
+
+                int numThreads = strategy.getController().maxConcurrentCompactions();
+
+                CountDownLatch settingUpDone = new CountDownLatch(1);
+                CountDownLatch runningDone = new CountDownLatch(2);
+                CountDownLatch tearingDownDone = new CountDownLatch(3 + numShards); // 1 reporter, 2 flusher and num shards compacting threads
+
+                threadFactory.newThread(new RunAndCountDown(settingUpDone, "preload", this::preloadData)).start();
+                threadFactory.newThread(new RunAndCountDown(tearingDownDone, "report", this::reportOutput)).start();
+
+                for (int i = 0; i < numThreads; i++)
+                    threadFactory.newThread(new RunAndCountDown(tearingDownDone, "compact " + i, this::compactData)).start();
+
+                settingUpDone.await();
+
+                if (error.get() != null)
+                    throw new RuntimeException("Simulation has failed");
+
+                waitForCompactionsToSettle();
+
+                setState(SimulationState.SETTING_UP, SimulationState.RUNNING);
+                this.start = System.currentTimeMillis();
+                //this.counters.reset();
+
+                threadFactory.newThread(new RunAndCountDown(tearingDownDone, "flush 1", this::flushData)).start();
+                threadFactory.newThread(new RunAndCountDown(tearingDownDone, "flush 2", this::flushData)).start();
+                threadFactory.newThread(new RunAndCountDown(runningDone, "insert", () -> runOrWait(this::insertData, writeRate))).start();
+                threadFactory.newThread(new RunAndCountDown(runningDone, "read", () -> runOrWait(this::readData, readRate))).start();
+
+                runningDone.await();
+
+                if (error.get() != null)
+                    throw new RuntimeException("Simulation has failed");
+
+                setState(SimulationState.RUNNING, SimulationState.TEARING_DOWN);
+
+                waitForCompactionsToSettle();
+
+                tearingDownDone.await();
+
+                summarize();
+            }
+            finally
+            {
+                setState(SimulationState.TEARING_DOWN, SimulationState.DONE);
+
+                if (strategy.getController().isRunning())
+                    strategy.getController().shutdown();
+            }
+        }
+
+        private class RunAndCountDown implements Runnable
+        {
+            CountDownLatch done;
+            String what;
+            Runnable task;
+
+            RunAndCountDown(CountDownLatch done, String what, Runnable task)
+            {
+                this.done = done;
+                this.what = what;
+                this.task = task;
+            }
+
+            @Override
+            public void run()
+            {
+                try
+                {
+                    logger.debug("Running \"{}\"", what);
+                    task.run();
+                }
+                catch (Throwable t)
+                {
+                    SimulationState currentState = state.get();
+                    logger.error("Unexpected error during \"{}\" with state {}:", what, currentState, t);
+
+                    error.compareAndSet(null, t);
+
+                    if (currentState.ordinal() < SimulationState.TEARING_DOWN.ordinal())
+                        setState(currentState, SimulationState.TEARING_DOWN);
+                }
+                finally
+                {
+                    logger.debug("Finished \"{}\"", what);
+                    done.countDown();
+                }
+            }
+        }
+
+
+        private void setState(SimulationState from, SimulationState to)
+        {
+            logger.debug("Updating simulation state from {} to {}", from, to);
+
+            if (state.compareAndSet(from, to))
+                observer.onChange(to);
+            else
+                throw new IllegalStateException(String.format("Failed to update simulation state from %s to %s", from, to));
+        }
+
+        void waitForCompactionsToSettle()
+        {
+            logger.debug("Waiting for compactions to settle...");
+
+            for (int i = 0; i < 3; i++)
+            { // 3 attempts in case the queue is temporarily empty before submitting a new compaction
+                while (!compactions.isEmpty())
+                {
+                    FBUtilities.sleepQuietly(1000);
+                    logger.debug("{}, live sstables: {}, compacting: {}, pending compactions: {}, pending flushing: {}, elapsed: {} s",
+                                 counters,
+                                 dataTracker.getLiveSSTables().size(),
+                                 dataTracker.getCompacting().size(),
+                                 compactions.size() + strategy.getEstimatedRemainingTasks(),
+                                 flushing.size(),
+                                 TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - start));
+                }
+            }
+
+            logger.debug("Compactions settled, live sstables: {}", dataTracker.getLiveSSTables().size());
+        }
+
+        void summarize()
+        {
+            if (error.get() != null)
+            {
+                Throwable err = error.get();
+                err.printStackTrace();
+                fail("Simulation failed with exception: " + err.getClass().getCanonicalName() + '/' + err.getMessage());
+
+                return;
+            }
+
+            long elapsedMs = System.currentTimeMillis() - start;
+            logger.info("Total time: {}s  WA : {}", TimeUnit.SECONDS.convert(elapsedMs, TimeUnit.MILLISECONDS), strategy.getController().getEnv().WA());
+            logger.info("Final outputs: {} {}", counters, output);
+
+            logger.info("Strategy aggregated statistics:");
+            logger.info(strategy.getStatistics().toString());
+        }
+
+        /**
+         * If the rate limiter is null simply sleep for the entire duration, otherwise run the task.
+         */
+        private void runOrWait(Runnable task, RateLimiter rateLimiter)
+        {
+            if (rateLimiter != null)
+                task.run();
+            else
+                FBUtilities.sleepQuietly(durationMillis);
+        }
+
+        /**
+         * Insert the entire key space.
+         */
+        private void preloadData()
+        {
+            ICardinality cardinality = newCardinality();
+
+            byte[] scratchBytes = new byte[8];
+            ByteBuffer scratch = ByteBuffer.wrap(scratchBytes);
+            long numToFlush;
+            int lastFlushed = 0;
+            long lastLogged = System.currentTimeMillis();
+            long maxBytesToInsert = (long) datasetSizeGB << 30;
+            long bytesInserted = 0;
+            int i = 0;
+
+            logger.info("Inserting up to {}", FBUtilities.prettyPrintMemory(maxBytesToInsert));
+
+            try
+            {
+                while(bytesInserted < maxBytesToInsert)
+                {
+                    scratch.clear();
+                    scratch.putLong(0, i);
+                    long hash = MurmurHash.hash64(scratchBytes, scratchBytes.length);
+                    cardinality.offerHashed(hash);
+
+                    counters.numInserted.incrementAndGet();
+                    bytesInserted += valueSize;
+
+                    i++;
+                    if (i == maxKey)
+                        i = 0;
+
+                    if (System.currentTimeMillis()- lastLogged >= TimeUnit.SECONDS.toMillis(1))
+                    {
+                        lastLogged = System.currentTimeMillis();
+                        logger.debug("Ins: {}, keys: {}, live sstables: {}, compacting: {}, pending compactions: {}",
+                                     FBUtilities.prettyPrintMemory(bytesInserted),
+                                     i,
+                                     dataTracker.getLiveSSTables().size(),
+                                     dataTracker.getCompacting().size(),
+                                     compactions.size() + strategy.getEstimatedRemainingTasks());
+                    }
+
+                    if (i >= (lastFlushed + uniqueKeysPerSStable) && // no point in checking the cardinality until we've inserted uniqueKeysPerSStable more entries
+                        (numToFlush = cardinality.cardinality()) >= uniqueKeysPerSStable)
+                    {
+                        counters.numFlushed.addAndGet(numToFlush);
+                        lastFlushed = i;
+                        generateSSTables(cardinality, numToFlush, partitioner.getMinimumToken(), partitioner.getMaximumToken(), "preload", true);
+
+                        cardinality = newCardinality();
+                    }
+
+                    if (i % 1000 == 0 && state.get() == SimulationState.TEARING_DOWN)
+                    { // this happens if the compaction threads fail
+                        logger.debug("Interrupting preload, simulation is tearing down");
+                        break;
+                    }
+                }
+
+                if ((numToFlush = cardinality.cardinality()) > 0)
+                {
+                    counters.numFlushed.addAndGet(numToFlush);
+                    generateSSTables(cardinality, numToFlush, partitioner.getMinimumToken(), partitioner.getMaximumToken(), "preload", true);
+                }
+            }
+            catch (Exception e)
+            {
+                logger.error("Exception happen during preloading", e);
+            }
+        }
+
+        /**
+         * Simulate inserting data and generating sstables when the cardinality has reached {@link this#uniqueKeysPerSStable} unique entries.
+         */
+        private void insertData()
+        {
+            ICardinality cardinality = newCardinality();
+
+            int numSSTables = 0;
+            long numFlushed = 0;
+
+            byte[] scratchBytes = new byte[8];
+            ByteBuffer scratch = ByteBuffer.wrap(scratchBytes);
+
+            long now;
+            long lastLogged = start;
+            try
+            {
+                while((now = System.currentTimeMillis()) - start <= durationMillis)
+                {
+                    scratch.clear();
+                    scratch.putLong(0, distribution.sample());
+                    long hash = MurmurHash.hash64(scratchBytes, scratchBytes.length);
+                    cardinality.offerHashed(hash);
+
+                    counters.numInserted.incrementAndGet();
+                    writeRate.acquire();
+
+                    if (now - lastLogged >= TimeUnit.SECONDS.toMillis(1))
+                    {
+                        lastLogged = now;
+                        logger.debug("{}, live sstables: {}, compacting: {}, pending compactions: {}, pending flushing: {}, elapsed: {} s",
+                                     counters,
+                                     dataTracker.getLiveSSTables().size(),
+                                     dataTracker.getCompacting().size(),
+                                     compactions.size() + strategy.getEstimatedRemainingTasks(),
+                                     flushing.size(),
+                                     TimeUnit.MILLISECONDS.toSeconds(now - start));
+
+                        if (state.get() == SimulationState.TEARING_DOWN)
+                            break;
+                    }
+
+                    if (counters.numInserted.get() >= (numFlushed + uniqueKeysPerSStable) && // no point in checking the cardinality until we've inserted uniqueKeysPerSStable more entries
+                        cardinality.cardinality() >= uniqueKeysPerSStable)
+                    {
+                        numFlushed = counters.numInserted.get();
+                        numSSTables++;
+
+                        flushing.put(cardinality);
+                        cardinality = newCardinality();
+                    }
+                }
+
+                // generate one final sstable
+                if (cardinality.cardinality() > 0)
+                {
+                    numSSTables++;
+                    flushing.put(cardinality);
+                }
+
+                logger.debug("Status: {} {}, sstables: {}, completed inserting data", counters, output, numSSTables);
+            }
+            catch (InterruptedException e)
+            {
+                logger.error("Exception happen during insertion", e);
+            }
+        }
+
+        /** Simulate reading some data */
+        private void readData()
+        {
+            while(System.currentTimeMillis() - start <= durationMillis)
+            {
+                counters.numRequested.incrementAndGet();
+                readRate.acquire();
+
+                if (state.get() == SimulationState.TEARING_DOWN)
+                    break;
+            }
+        }
+
+        /**
+         * Convert the compaction statistics to the simulation output and append it to the csv file.
+         */
+        void reportOutput()
+        {
+            while(state.get().ordinal() < SimulationState.TEARING_DOWN.ordinal())
+            {
+                FBUtilities.sleepQuietly(csvUpdatePeriodMs);
+                doReportOutput(false);
+            }
+
+            doReportOutput(true);
+        }
+
+        private void doReportOutput(boolean isLast)
+        {
+            if (isLast)
+                output.writeAverages(csvWriter, counters);
+            else
+                output.write(csvWriter, counters);
+
+            logger.trace("{} {}", counters, output);
+        }
+
+        /**
+         * Take the cardinalities from the flushing queue and generate sstables.
+         * @throws Exception
+         */
+        private void flushData()
+        {
+            try
+            {
+                while(state.get().ordinal() < SimulationState.TEARING_DOWN.ordinal() || !flushing.isEmpty())
+                {
+                    ICardinality cardinality = flushing.poll(1, TimeUnit.MILLISECONDS);
+                    if (cardinality == null)
+                        continue;
+
+                    long numToFlush = cardinality.cardinality();
+                    counters.numFlushed.addAndGet(numToFlush);
+                    generateSSTables(cardinality, numToFlush, partitioner.getMinimumToken(), partitioner.getMaximumToken(), "flushing", true);
+                }
+            }
+            catch (InterruptedException e)
+            {
+                logger.error("Exception happen during flushing", e);
+            }
+        }
+
+        /**
+         * Perform the following:
+         *
+         * <li>Take compaction tasks from {@link this#compactions}</li>
+         * <li>Merge the cardinality of the txn sstables</li>
+         * <li>Generate a new merged sstable</li>
+         * <li>Pass it to the strategy and live sstables</li>
+         * <li>Check with the strategy if there is a new compaction task</li>
+         */
+        private void compactData()
+        {
+            try
+            {
+                while(state.get().ordinal() < SimulationState.TEARING_DOWN.ordinal() || !compactions.isEmpty())
+                {
+                    AbstractCompactionTask task = compactions.poll(1, TimeUnit.SECONDS);
+                    if (task == null)
+                    {
+                        logger.info("no task");
+                        continue;
+                    }
+
+                    ILifecycleTransaction txn = task.getTransaction();
+                    Set<SSTableReader> candidates = txn.originals();
+                    for (SSTableReader candidate : candidates)
+                        counters.numReadForCompaction.addAndGet(candidate.keyCardinalityEstimator().cardinality());
+
+                    TimeUUID id = txn.opId();
+
+                    //strategy.getBackgroundCompactions().setInProgress(mockCompletedCompactionProgress(candidates, id));
+                    ICardinality merged = getMerged(candidates);
+
+                    counters.numWrittenForCompaction.addAndGet(merged.cardinality());
+
+                    // first remove the sstables to avoid overlaps when adding the new one for LCS
+                    dataTracker.removeUnsafe(candidates);
+                    dataTracker.removeCompactingUnsafe(candidates);
+
+                    // first create the new merged sstable
+                    generateSSTables(merged, merged.cardinality(),
+                                     candidates.stream().map(x -> x.getFirst().getToken()).min(Comparator.naturalOrder()).get(),
+                                     candidates.stream().map(x -> x.getLast().getToken()).max(Comparator.naturalOrder()).get(),
+                                     "compacting", false);
+                    //Thread.sleep(5);
+
+                    // then remove the old sstables
+                    strategy.onCompleted(id, null);
+                    counters.numCompactions.incrementAndGet();
+                    counters.numCompactionsPending.decrementAndGet();
+                    counters.numCompactedSSTables.addAndGet(candidates.size());
+
+                    logger.debug("Executed {} compactions, live sstables: {}, compacting sstables: {}, compacted sstables: {}",
+                                 counters.numCompactions, dataTracker.getLiveSSTables().size(), dataTracker.getCompacting().size(), counters.numCompactedSSTables);
+
+                    maybeSubmitCompaction();
+
+                    ((LifecycleTransaction)txn).unsafeClose();
+                }
+                logger.debug("...completed monitoring compactions");
+            }
+            catch (InterruptedException | CardinalityMergeException e)
+            {
+                logger.error("Exception happen during compaction", e);
+            }
+        }
+
+        /**
+         * Merge the cardinalities of the input sstables.
+         *
+         * @return the merged cardinality
+         *
+         * @throws CardinalityMergeException
+         */
+        private ICardinality getMerged(Set<SSTableReader> candidates) throws CardinalityMergeException
+        {
+            ICardinality[] cardinalities = new ICardinality[candidates.size() - 1];
+            int i = 0;
+            ICardinality first = null;
+
+            for (SSTableReader sstable : candidates)
+            {
+                if (first == null)
+                    first = sstable.keyCardinalityEstimator();
+                else
+                    cardinalities[i++] = sstable.keyCardinalityEstimator();
+            }
+
+            return first.merge(cardinalities);
+        }
+
+        /**
+         * Create a new cardinality with similar parameters as those used in {@link MetadataCollector}.
+         * See CASSANDRA-5906 for error and size details. Instead of using 12, 25 we use 12,24 since that
+         * halves the memory used (2.7k instead of 5.5k for 10k entries) and we can tollerate a slightly larger error.
+         *
+         * @return a newly constructed cardinality
+         */
+        private ICardinality newCardinality()
+        {
+            return new HyperLogLogPlus(12, 24); // for real sstables in MetadataCollector we use 13, 25
+        }
+
+        /**
+         * Create one or more mocked sstables based on the cardinality received and the value size. The theoretical sstable size
+         * (numEntries * valueSize) will be split across multiple compaction shards.
+         *
+         * @param cardinality - the cardinality used to simulate the sstable
+         * @param numEntries - the total number of entries to write to disk
+         * @param reason - the reason (flushing, compacting, etc)
+         * @param checkForCompaction - if true we check if a compaction needs to be submitted
+         */
+        private void generateSSTables(ICardinality cardinality, long numEntries, Token minToken, Token maxToken, String reason, boolean checkForCompaction) throws InterruptedException
+        {
+            // The theoretical sstable size that is being mocked
+            long sstableSize = numEntries * valueSize;
+            IPartitioner partitioner = minToken.getPartitioner();
+
+            int shards = strategy.getController().getNumShards(valueSize * numEntries / minToken.size(maxToken));
+            ShardTracker boundaries = strategy.getShardManager().boundaries(shards);
+
+            int numSStables = 0;
+            boundaries.advanceTo(minToken);
+            while (true)
+            {
+                ++numSStables;
+                if (boundaries.shardEnd() == null || maxToken.compareTo(boundaries.shardEnd()) <= 0)
+                    break;
+                boundaries.advanceTo(boundaries.shardEnd().nextValidToken());
+            }
+
+            boundaries = strategy.getShardManager().boundaries(shards);
+
+            List<SSTableReader> sstables = new ArrayList<>(numSStables);
+            long keyCount = (long) Math.ceil(numEntries / (double) numSStables);
+            long bytesOnDisk = valueSize * keyCount;
+            long timestamp = System.currentTimeMillis();
+
+            boundaries.advanceTo(minToken);
+            while (true)
+            {
+                Range<Token> span = boundaries.shardSpan();
+                Token firstToken = span.left.nextValidToken();
+                if (minToken.compareTo(firstToken) > 0)
+                    firstToken = minToken;
+                Token lastToken = partitioner.split(span.left, span.right, 1 - Math.scalb(1, -24)); // something that is < span.right
+                if (maxToken.compareTo(lastToken) < 0)
+                    lastToken = maxToken;
+                DecoratedKey first = new BufferDecoratedKey(firstToken, ByteBuffer.allocate(0));
+                DecoratedKey last = new BufferDecoratedKey(lastToken, ByteBuffer.allocate(0));
+
+                SSTableReader sstable = mockSSTable(0, bytesOnDisk, timestamp, 0, first, last, 0, true, null, 0);
+                when(sstable.keyCardinalityEstimator()).thenReturn(cardinality);
+                when(sstable.estimatedKeys()).thenReturn(keyCount);
+                sstables.add(sstable);
+
+                if (boundaries.shardEnd() == null || maxToken.compareTo(boundaries.shardEnd()) <= 0)
+                    break;
+                boundaries.advanceTo(boundaries.shardEnd().nextValidToken());
+            }
+
+            counters.numWritten.addAndGet(numEntries);
+            dataTracker.addInitialSSTablesWithoutUpdatingSize(sstables);
+            logger.debug("Generated {} new sstables for {}, live: {}, compacting: {}, tot sstable size {}",
+                         sstables.size(), reason, dataTracker.getLiveSSTables().size(), dataTracker.getCompacting().size(),
+                         sstableSize);
+
+            if (checkForCompaction)
+                maybeSubmitCompaction();
+        }
+
+        private void maybeSubmitCompaction() throws InterruptedException
+        {
+            Collection<AbstractCompactionTask> tasks = strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+            for (AbstractCompactionTask task : tasks)
+            {
+                compactions.put(task);
+                counters.numCompactionsPending.incrementAndGet();
+                logger.debug("Submitted new compaction, live sstables: {}, compacting sstables: {}, compacted sstables: {}",
+                             dataTracker.getLiveSSTables().size(), dataTracker.getCompacting().size(), counters.numCompactedSSTables);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyContainerPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyContainerPendingRepairTest.java
new file mode 100644
index 000000000000..756b4fd4c54d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyContainerPendingRepairTest.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+
+import org.junit.Ignore;
+
+import org.apache.cassandra.repair.NoSuchRepairSessionException;
+
+@Ignore
+public interface CompactionStrategyContainerPendingRepairTest
+{
+    /**
+     * Pending repair strategy should be created when we encounter a new pending id
+     */
+    void testSstableAdded() throws IOException;
+
+    void testSstableDeleted() throws IOException;
+
+    void testSstableListChangedAddAndRemove() throws IOException;
+
+    void testSstableRepairStatusChanged() throws IOException;
+
+    /**
+     * {@link CompactionStrategyContainer} should include
+     * pending repair strategies when appropriate
+     */
+    void testStrategiesContainsPendingRepair() throws IOException;
+
+    /**
+     * Tests that finalized repairs result in cleanup compaction tasks
+     * which reclassify the sstables as repaired
+     */
+    void testCleanupCompactionFinalized() throws Exception;
+
+    /**
+     * Tests that finalized repairs racing with compactions on the same set of sstables don't leave unrepaired sstables behind
+     */
+    void testFinalizedAndCompactionRace() throws IOException, NoSuchRepairSessionException;
+
+    void testFinalizedSessionTransientCleanup() throws IOException;
+
+    void testFailedSessionTransientCleanup() throws IOException;
+
+    void testCleanupCompactionFailed() throws IOException;
+
+    void testSessionCompleted() throws IOException;
+
+    void testSessionCompletedWithDifferentSSTables() throws IOException;
+
+
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerBoundaryReloadTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerBoundaryReloadTest.java
deleted file mode 100644
index 0d3b0d0e324d..000000000000
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerBoundaryReloadTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.compaction;
-
-import java.net.UnknownHostException;
-import java.util.List;
-
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DiskBoundaries;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.service.StorageService;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
-
-public class CompactionStrategyManagerBoundaryReloadTest extends CQLTester
-{
-    @Test
-    public void testNoReload()
-    {
-        createTable("create table %s (id int primary key)");
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        List<List<AbstractCompactionStrategy>> strategies = cfs.getCompactionStrategyManager().getStrategies();
-        DiskBoundaries db = cfs.getDiskBoundaries();
-        StorageService.instance.getTokenMetadata().invalidateCachedRings();
-        // make sure the strategy instances are the same (no reload)
-        assertTrue(isSame(strategies, cfs.getCompactionStrategyManager().getStrategies()));
-        // but disk boundaries are not .equal (ring version changed)
-        assertNotEquals(db, cfs.getDiskBoundaries());
-        assertTrue(db.isEquivalentTo(cfs.getDiskBoundaries()));
-
-        db = cfs.getDiskBoundaries();
-        alterTable("alter table %s with comment = 'abcd'");
-        assertTrue(isSame(strategies, cfs.getCompactionStrategyManager().getStrategies()));
-        // disk boundaries don't change because of alter
-        assertEquals(db, cfs.getDiskBoundaries());
-    }
-
-    @Test
-    public void testReload() throws UnknownHostException
-    {
-        createTable("create table %s (id int primary key)");
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        List<List<AbstractCompactionStrategy>> strategies = cfs.getCompactionStrategyManager().getStrategies();
-        DiskBoundaries db = cfs.getDiskBoundaries();
-        TokenMetadata tmd = StorageService.instance.getTokenMetadata();
-        tmd.updateNormalToken(tmd.partitioner.getMinimumToken(), InetAddressAndPort.getByName("127.0.0.1"));
-        tmd.updateNormalToken(tmd.partitioner.getMaximumToken(), InetAddressAndPort.getByName("127.0.0.2"));
-        // make sure the strategy instances have been reloaded
-        assertFalse(isSame(strategies,
-                           cfs.getCompactionStrategyManager().getStrategies()));
-        assertNotEquals(db, cfs.getDiskBoundaries());
-        db = cfs.getDiskBoundaries();
-
-        strategies = cfs.getCompactionStrategyManager().getStrategies();
-        alterTable("alter table %s with compaction = {'class': 'SizeTieredCompactionStrategy', 'enabled': false}");
-        assertFalse(isSame(strategies,
-                           cfs.getCompactionStrategyManager().getStrategies()));
-        assertEquals(db, cfs.getDiskBoundaries());
-
-    }
-
-    private boolean isSame(List<List<AbstractCompactionStrategy>> a, List<List<AbstractCompactionStrategy>> b)
-    {
-        if (a.size() != b.size())
-            return false;
-        for (int i = 0; i < a.size(); i++)
-        {
-            if (a.get(i).size() != b.get(i).size())
-                return false;
-            for (int j = 0; j < a.get(i).size(); j++)
-                if (a.get(i).get(j) != b.get(i).get(j))
-                    return false;
-        }
-        return true;
-    }
-}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
index d87b384afd99..0ad7ca6987bb 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
@@ -19,9 +19,13 @@
 package org.apache.cassandra.db.compaction;
 
 import java.util.ArrayList;
+import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.repair.consistent.LocalSession;
@@ -29,6 +33,7 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.notifications.SSTableAddedNotification;
 import org.apache.cassandra.notifications.SSTableDeletingNotification;
@@ -40,51 +45,83 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
 /**
- * Tests CompactionStrategyManager's handling of pending repair sstables
+ * Tests CompactionStrategyContainer's handling of pending repair sstables
  */
-public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingRepairTest
+public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingRepairTest implements CompactionStrategyContainerPendingRepairTest
 {
+    @Override
+    public String createTableCql()
+    {
+        // Note: This test is tightly coupled to the LegacyAbstractCompactionStrategy so cannot use the default UCS
+        // UCS is tested in UnifiedCompactionContainerPendingRepairTest
+        return String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT) WITH COMPACTION={'class':'SizeTieredCompactionStrategy'}",
+                             ks, tbl);
+    }
+
 
     private boolean transientContains(SSTableReader sstable)
     {
-        return csm.getTransientRepairsUnsafe().containsSSTable(sstable);
+        return ((CompactionStrategyManager) compactionStrategyContainer)
+               .getTransientRepairsUnsafe()
+               .containsSSTable(sstable);
     }
 
     private boolean pendingContains(SSTableReader sstable)
     {
-        return csm.getPendingRepairsUnsafe().containsSSTable(sstable);
+        return ((CompactionStrategyManager) compactionStrategyContainer)
+               .getPendingRepairsUnsafe()
+               .containsSSTable(sstable);
     }
 
     private boolean repairedContains(SSTableReader sstable)
     {
-        return csm.getRepairedUnsafe().containsSSTable(sstable);
+        return ((CompactionStrategyManager) compactionStrategyContainer)
+               .getRepairedUnsafe()
+               .containsSSTable(sstable);
     }
 
     private boolean unrepairedContains(SSTableReader sstable)
     {
-        return csm.getUnrepairedUnsafe().containsSSTable(sstable);
+        return ((CompactionStrategyManager) compactionStrategyContainer)
+               .getUnrepairedUnsafe()
+               .containsSSTable(sstable);
     }
 
     private boolean hasPendingStrategiesFor(TimeUUID sessionID)
     {
-        return !Iterables.isEmpty(csm.getPendingRepairsUnsafe().getStrategiesFor(sessionID));
+        return !Iterables.isEmpty(((CompactionStrategyManager) compactionStrategyContainer)
+                                  .getPendingRepairsUnsafe()
+                                  .getStrategiesFor(sessionID));
     }
 
     private boolean hasTransientStrategiesFor(TimeUUID sessionID)
     {
-        return !Iterables.isEmpty(csm.getTransientRepairsUnsafe().getStrategiesFor(sessionID));
+        return !Iterables.isEmpty(((CompactionStrategyManager) compactionStrategyContainer)
+                                  .getTransientRepairsUnsafe()
+                                  .getStrategiesFor(sessionID));
+    }
+
+    private void assertCompactionStrategyManagerPendingRepairs(boolean expectedEmpty)
+    {
+        assertEquals(expectedEmpty, ((CompactionStrategyManager) cfs.getCompactionStrategy()).pendingRepairs().isEmpty());
     }
 
     /**
      * Pending repair strategy should be created when we encounter a new pending id
      */
+    @Override
     @Test
-    public void sstableAdded()
+    public void testSstableAdded() throws IOException
     {
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
-        Assert.assertTrue(Iterables.isEmpty(csm.getPendingRepairsUnsafe().allStrategies()));
+        Assert.assertTrue(Iterables.isEmpty(((CompactionStrategyManager) compactionStrategyContainer)
+                                            .getPendingRepairsUnsafe()
+                                            .allStrategies()));
 
         SSTableReader sstable = makeSSTable(true);
         Assert.assertFalse(sstable.isRepaired());
@@ -97,7 +134,7 @@ public void sstableAdded()
         Assert.assertFalse(hasTransientStrategiesFor(repairID));
 
         // add the sstable
-        csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+        compactionStrategyContainer.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
         Assert.assertFalse(repairedContains(sstable));
         Assert.assertFalse(unrepairedContains(sstable));
         Assert.assertTrue(pendingContains(sstable));
@@ -105,8 +142,9 @@ public void sstableAdded()
         Assert.assertFalse(hasTransientStrategiesFor(repairID));
     }
 
+    @Override
     @Test
-    public void sstableListChangedAddAndRemove()
+    public void testSstableListChangedAddAndRemove() throws IOException
     {
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -128,8 +166,9 @@ public void sstableListChangedAddAndRemove()
         SSTableListChangedNotification notification;
         notification = new SSTableListChangedNotification(Collections.singleton(sstable1),
                                                           Collections.emptyList(),
-                                                          OperationType.COMPACTION);
-        csm.handleNotification(notification, cfs.getTracker());
+                                                          OperationType.COMPACTION,
+                                                          Optional.empty());
+        compactionStrategyContainer.handleNotification(notification, cfs.getTracker());
 
         Assert.assertFalse(repairedContains(sstable1));
         Assert.assertFalse(unrepairedContains(sstable1));
@@ -143,8 +182,9 @@ public void sstableListChangedAddAndRemove()
         // remove and add
         notification = new SSTableListChangedNotification(Collections.singleton(sstable2),
                                                           Collections.singleton(sstable1),
-                                                          OperationType.COMPACTION);
-        csm.handleNotification(notification, cfs.getTracker());
+                                                          OperationType.COMPACTION,
+                                                          Optional.empty());
+        compactionStrategyContainer.handleNotification(notification, cfs.getTracker());
 
         Assert.assertFalse(repairedContains(sstable1));
         Assert.assertFalse(unrepairedContains(sstable1));
@@ -154,8 +194,9 @@ public void sstableListChangedAddAndRemove()
         Assert.assertTrue(pendingContains(sstable2));
     }
 
+    @Override
     @Test
-    public void sstableRepairStatusChanged()
+    public void testSstableRepairStatusChanged() throws IOException
     {
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -172,7 +213,7 @@ public void sstableRepairStatusChanged()
         // change to pending repaired
         mutateRepaired(sstable, repairID, false);
         notification = new SSTableRepairStatusChanged(Collections.singleton(sstable));
-        csm.handleNotification(notification, cfs.getTracker());
+        compactionStrategyContainer.handleNotification(notification, cfs.getTracker());
         Assert.assertFalse(unrepairedContains(sstable));
         Assert.assertFalse(repairedContains(sstable));
         Assert.assertTrue(hasPendingStrategiesFor(repairID));
@@ -182,26 +223,27 @@ public void sstableRepairStatusChanged()
         // change to repaired
         mutateRepaired(sstable, System.currentTimeMillis());
         notification = new SSTableRepairStatusChanged(Collections.singleton(sstable));
-        csm.handleNotification(notification, cfs.getTracker());
+        compactionStrategyContainer.handleNotification(notification, cfs.getTracker());
         Assert.assertFalse(unrepairedContains(sstable));
         Assert.assertTrue(repairedContains(sstable));
         Assert.assertFalse(pendingContains(sstable));
     }
 
+    @Override
     @Test
-    public void sstableDeleted()
+    public void testSstableDeleted() throws IOException
     {
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
 
         SSTableReader sstable = makeSSTable(true);
         mutateRepaired(sstable, repairID, false);
-        csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+        compactionStrategyContainer.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
         Assert.assertTrue(pendingContains(sstable));
 
         // delete sstable
         SSTableDeletingNotification notification = new SSTableDeletingNotification(sstable);
-        csm.handleNotification(notification, cfs.getTracker());
+        compactionStrategyContainer.handleNotification(notification, cfs.getTracker());
         Assert.assertFalse(pendingContains(sstable));
         Assert.assertFalse(unrepairedContains(sstable));
         Assert.assertFalse(repairedContains(sstable));
@@ -211,39 +253,35 @@ public void sstableDeleted()
      * CompactionStrategyManager.getStrategies should include
      * pending repair strategies when appropriate
      */
+    @Override
     @Test
-    public void getStrategies()
+    public void testStrategiesContainsPendingRepair() throws IOException
     {
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
 
-        List<List<AbstractCompactionStrategy>> strategies;
-
-        strategies = csm.getStrategies();
-        Assert.assertEquals(3, strategies.size());
-        Assert.assertTrue(strategies.get(2).isEmpty());
+        Assert.assertTrue(compactionStrategyContainer.getStrategies(false, repairID).isEmpty());
 
         SSTableReader sstable = makeSSTable(true);
         mutateRepaired(sstable, repairID, false);
-        csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+        compactionStrategyContainer.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
 
-        strategies = csm.getStrategies();
-        Assert.assertEquals(3, strategies.size());
-        Assert.assertFalse(strategies.get(2).isEmpty());
+        Assert.assertFalse(compactionStrategyContainer.getStrategies(false, repairID).isEmpty());
     }
 
     /**
      * Tests that finalized repairs result in cleanup compaction tasks
      * which reclassify the sstables as repaired
      */
+    @Override
     @Test
-    public void cleanupCompactionFinalized() throws NoSuchRepairSessionException
+    public void testCleanupCompactionFinalized() throws NoSuchRepairSessionException
     {
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
         SSTableReader sstable = makeSSTable(true);
         mutateRepaired(sstable, repairID, false);
-        csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+        compactionStrategyContainer.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
         LocalSessionAccessor.finalizeUnsafe(repairID);
         Assert.assertTrue(hasPendingStrategiesFor(repairID));
         Assert.assertFalse(hasTransientStrategiesFor(repairID));
@@ -251,13 +289,15 @@ public void cleanupCompactionFinalized() throws NoSuchRepairSessionException
         Assert.assertTrue(sstable.isPendingRepair());
         Assert.assertFalse(sstable.isRepaired());
 
-        cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
-        AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
+        cfs.getCompactionStrategyContainer().enable(); // enable compaction to fetch next background task
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
         Assert.assertNotNull(compactionTask);
-        Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
+        Assert.assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
 
         // run the compaction
-        compactionTask.execute(ActiveCompactionsTracker.NOOP);
+        compactionTask.execute();
 
         Assert.assertTrue(repairedContains(sstable));
         Assert.assertFalse(unrepairedContains(sstable));
@@ -269,46 +309,7 @@ public void cleanupCompactionFinalized() throws NoSuchRepairSessionException
         long expectedRepairedAt = ActiveRepairService.instance().getParentRepairSession(repairID).repairedAt;
         Assert.assertFalse(sstable.isPendingRepair());
         Assert.assertTrue(sstable.isRepaired());
-        Assert.assertEquals(expectedRepairedAt, sstable.getSSTableMetadata().repairedAt);
-    }
-
-    /**
-     * Tests that failed repairs result in cleanup compaction tasks
-     * which reclassify the sstables as unrepaired
-     */
-    @Test
-    public void cleanupCompactionFailed()
-    {
-        TimeUUID repairID = registerSession(cfs, true, true);
-        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
-        SSTableReader sstable = makeSSTable(true);
-        mutateRepaired(sstable, repairID, false);
-        csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
-        LocalSessionAccessor.failUnsafe(repairID);
-
-        Assert.assertTrue(hasPendingStrategiesFor(repairID));
-        Assert.assertFalse(hasTransientStrategiesFor(repairID));
-        Assert.assertTrue(pendingContains(sstable));
-        Assert.assertTrue(sstable.isPendingRepair());
-        Assert.assertFalse(sstable.isRepaired());
-
-        cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
-        AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
-        Assert.assertNotNull(compactionTask);
-        Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
-
-        // run the compaction
-        compactionTask.execute(ActiveCompactionsTracker.NOOP);
-
-        Assert.assertFalse(repairedContains(sstable));
-        Assert.assertTrue(unrepairedContains(sstable));
-        Assert.assertFalse(hasPendingStrategiesFor(repairID));
-        Assert.assertFalse(hasTransientStrategiesFor(repairID));
-
-        // sstable should have pendingRepair cleared, and repairedAt set correctly
-        Assert.assertFalse(sstable.isPendingRepair());
-        Assert.assertFalse(sstable.isRepaired());
-        Assert.assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().repairedAt);
+        assertEquals(expectedRepairedAt, sstable.getSSTableMetadata().repairedAt);
     }
 
     /**
@@ -318,7 +319,7 @@ public void cleanupCompactionFailed()
      * compaction task is issued for that repair session.
      */
     @Test
-    public void testFinalizedAndCompactionRace() throws NoSuchRepairSessionException
+    public void testFinalizedAndCompactionRace() throws IOException, NoSuchRepairSessionException
     {
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -334,21 +335,22 @@ public void testFinalizedAndCompactionRace() throws NoSuchRepairSessionException
         }
 
         // change to pending repair
-        mutateRepaired(sstables, repairID, false);
-        csm.handleNotification(new SSTableAddedNotification(sstables, null), cfs.getTracker());
+        cfs.mutateRepaired(sstables, 0, repairID, false);
+        compactionStrategyContainer.handleNotification(new SSTableAddedNotification(sstables, null), cfs.getTracker());
         for (SSTableReader sstable : sstables)
         {
             Assert.assertFalse(sstable.isRepaired());
             Assert.assertTrue(sstable.isPendingRepair());
-            Assert.assertEquals(repairID, sstable.getPendingRepair());
+            assertEquals(repairID, sstable.getPendingRepair());
         }
 
         // Get a compaction taks based on the sstables marked as pending repair
-        cfs.getCompactionStrategyManager().enable();
-        for (SSTableReader sstable : sstables)
-            pendingContains(sstable);
-        AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
-        Assert.assertNotNull(compactionTask);
+        compactionStrategyContainer.enable();
+        assertEquals(numberOfSStables, cfs.getPendingRepairSSTables(repairID).size());
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
 
         // Finalize the repair session
         LocalSessionAccessor.finalizeUnsafe(repairID);
@@ -357,38 +359,45 @@ public void testFinalizedAndCompactionRace() throws NoSuchRepairSessionException
         Assert.assertTrue(hasPendingStrategiesFor(repairID));
 
         // run the compaction
-        compactionTask.execute(ActiveCompactionsTracker.NOOP);
+        compactionTask.execute();
 
         // The repair session is finalized but there is an sstable left behind pending repair!
-        SSTableReader compactedSSTable = cfs.getLiveSSTables().iterator().next();
-        Assert.assertEquals(repairID, compactedSSTable.getPendingRepair());
-        Assert.assertEquals(1, cfs.getLiveSSTables().size());
+        SSTableReader compactedSSTable = cfs.getPendingRepairSSTables(repairID).iterator().next();
+        assertEquals(repairID, compactedSSTable.getPendingRepair());
+        assertEquals(1, cfs.getLiveSSTables().size());
+        assertEquals(1, cfs.getPendingRepairSSTables(repairID).size());
 
         System.out.println("*********************************************************************************************");
         System.out.println(compactedSSTable);
         System.out.println("Pending repair UUID: " + compactedSSTable.getPendingRepair());
         System.out.println("Repaired at: " + compactedSSTable.getRepairedAt());
-        System.out.println("Creation time: " + compactedSSTable.getDataCreationTime());
+        System.out.println("Creation time: " + compactedSSTable.getCreationTimeFor(SSTableFormat.Components.DATA));
         System.out.println("Live sstables: " + cfs.getLiveSSTables().size());
+        System.out.println("Pending repair sstables: " + cfs.getPendingRepairSSTables(repairID).size());
         System.out.println("*********************************************************************************************");
 
         // Run compaction again. It should pick up the pending repair sstable
-        compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
-        if (compactionTask != null)
+        compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        if (!compactionTasks.isEmpty())
         {
-            Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
-            compactionTask.execute(ActiveCompactionsTracker.NOOP);
+            assertEquals(1, compactionTasks.size());
+            compactionTask = compactionTasks.iterator().next();
+            assertNotNull(compactionTask);
+            Assert.assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
+            compactionTask.execute();
         }
 
         System.out.println("*********************************************************************************************");
         System.out.println(compactedSSTable);
         System.out.println("Pending repair UUID: " + compactedSSTable.getPendingRepair());
         System.out.println("Repaired at: " + compactedSSTable.getRepairedAt());
-        System.out.println("Creation time: " + compactedSSTable.getDataCreationTime());
+        System.out.println("Creation time: " + compactedSSTable.getCreationTimeFor(SSTableFormat.Components.DATA));
         System.out.println("Live sstables: " + cfs.getLiveSSTables().size());
+        System.out.println("Pending repair sstables: " + cfs.getPendingRepairSSTables(repairID).size());
         System.out.println("*********************************************************************************************");
 
-        Assert.assertEquals(1, cfs.getLiveSSTables().size());
+        assertEquals(0, cfs.getPendingRepairSSTables(repairID).size());
+        assertEquals(1, cfs.getLiveSSTables().size());
         Assert.assertFalse(hasPendingStrategiesFor(repairID));
         Assert.assertFalse(hasTransientStrategiesFor(repairID));
         Assert.assertTrue(repairedContains(compactedSSTable));
@@ -398,18 +407,159 @@ public void testFinalizedAndCompactionRace() throws NoSuchRepairSessionException
         long expectedRepairedAt = ActiveRepairService.instance().getParentRepairSession(repairID).repairedAt;
         Assert.assertFalse(compactedSSTable.isPendingRepair());
         Assert.assertTrue(compactedSSTable.isRepaired());
-        Assert.assertEquals(expectedRepairedAt, compactedSSTable.getSSTableMetadata().repairedAt);
+        assertEquals(expectedRepairedAt, compactedSSTable.getSSTableMetadata().repairedAt);
+    }
+
+    /**
+     * Tests that failed repairs result in cleanup compaction tasks
+     * which reclassify the sstables as unrepaired
+     */
+    @Override
+    @Test
+    public void testCleanupCompactionFailed() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+        SSTableReader sstable = makeSSTable(true);
+        mutateRepaired(sstable, repairID, false);
+        compactionStrategyContainer.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+        LocalSessionAccessor.failUnsafe(repairID);
+
+        Assert.assertTrue(hasPendingStrategiesFor(repairID));
+        Assert.assertFalse(hasTransientStrategiesFor(repairID));
+        Assert.assertTrue(pendingContains(sstable));
+        Assert.assertTrue(sstable.isPendingRepair());
+        Assert.assertFalse(sstable.isRepaired());
+
+        cfs.getCompactionStrategyContainer().enable(); // enable compaction to fetch next background task
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
+        Assert.assertNotNull(compactionTask);
+        Assert.assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
+
+        // run the compaction
+        compactionTask.execute();
+
+        Assert.assertFalse(repairedContains(sstable));
+        Assert.assertTrue(unrepairedContains(sstable));
+        Assert.assertFalse(hasPendingStrategiesFor(repairID));
+        Assert.assertFalse(hasTransientStrategiesFor(repairID));
+
+        // sstable should have pendingRepair cleared, and repairedAt set correctly
+        Assert.assertFalse(sstable.isPendingRepair());
+        Assert.assertFalse(sstable.isRepaired());
+        assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().repairedAt);
+    }
+
+    @Override
+    @Test
+    public void testSessionCompleted() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+        assertCompactionStrategyManagerPendingRepairs(true);
+
+        // add sstable as unrepaired
+        final boolean isOrphan = false;
+        SSTableReader sstable = makeSSTable(isOrphan);
+
+        // change to pending repair
+        mutateRepaired(sstable, repairID, false);
+        SSTableRepairStatusChanged notification = new SSTableRepairStatusChanged(Collections.singleton(sstable));
+        compactionStrategyContainer.handleNotification(notification, cfs.getTracker());
+        Assert.assertFalse(unrepairedContains(sstable));
+        Assert.assertFalse(repairedContains(sstable));
+        Assert.assertTrue(hasPendingStrategiesFor(repairID));
+        Assert.assertFalse(hasTransientStrategiesFor(repairID));
+        Assert.assertTrue(pendingContains(sstable));
+
+        // finalize
+        LocalSessionAccessor.finalizeUnsafe(repairID);
+
+        // complete session
+        ARS.consistent.local.sessionCompleted(ARS.consistent.local.getSession(repairID));
+
+        // sstable is repaired
+        Assert.assertFalse(unrepairedContains(sstable));
+        Assert.assertTrue(repairedContains(sstable));
+        Assert.assertFalse(pendingContains(sstable));
+    }
+
+    @Override
+    @Test
+    public void testSessionCompletedWithDifferentSSTables() throws IOException
+    {
+        TimeUUID repairID1 = registerSession(cfs, true, true);
+        TimeUUID repairID2 = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID1, COORDINATOR, PARTICIPANTS);
+        LocalSessionAccessor.prepareUnsafe(repairID2, COORDINATOR, PARTICIPANTS);
+        assertCompactionStrategyManagerPendingRepairs(true);
+
+        // add sstables as unrepaired
+        final boolean isOrphan = false;
+        SSTableReader sstable1 = makeSSTable(isOrphan);
+        Assert.assertTrue(unrepairedContains(sstable1));
+
+        SSTableReader sstable2 = makeSSTable(isOrphan);
+        Assert.assertTrue(unrepairedContains(sstable2));
+
+        SSTableReader sstable3 = makeSSTable(isOrphan);
+        Assert.assertTrue(unrepairedContains(sstable3));
+
+        // change sstable1 to pending repair for session 1
+        mutateRepaired(sstable1, repairID1, false);
+        SSTableRepairStatusChanged notification = new SSTableRepairStatusChanged(ImmutableList.of(sstable1));
+        compactionStrategyContainer.handleNotification(notification, cfs.getTracker());
+        Assert.assertFalse(sstable1.isRepaired());
+        Assert.assertTrue(sstable1.isPendingRepair());
+        Assert.assertTrue(hasPendingStrategiesFor(repairID1));
+        Assert.assertFalse(hasTransientStrategiesFor(repairID1));
+
+        // change sstable2 to pending repair for session 2
+        mutateRepaired(sstable2, repairID2, false);
+        notification = new SSTableRepairStatusChanged(ImmutableList.of(sstable2));
+        compactionStrategyContainer.handleNotification(notification, cfs.getTracker());
+        Assert.assertFalse(sstable2.isRepaired());
+        Assert.assertTrue(sstable2.isPendingRepair());
+        Assert.assertTrue(hasPendingStrategiesFor(repairID2));
+        Assert.assertFalse(hasTransientStrategiesFor(repairID2));
+
+        // change sstable3 to repaired
+        mutateRepaired(sstable3, System.currentTimeMillis());
+        Assert.assertTrue(sstable3.isRepaired());
+        Assert.assertFalse(sstable3.isPendingRepair());
+
+        // finalize session 1
+        LocalSessionAccessor.finalizeUnsafe(repairID1);
+
+        // simulate compaction on repaired sstable3
+        cfs.getTracker().tryModify(sstable3, OperationType.COMPACTION);
+
+        // completing session 1 will not require to disable compactions because:
+        // * sstable2 belongs to a different session
+        // * sstable3 is repaired
+        ARS.consistent.local.sessionCompleted(ARS.consistent.local.getSession(repairID1));
+
+        // now sstable1 and sstable3 are repaired
+        Assert.assertTrue(sstable1.isRepaired());
+        Assert.assertTrue(sstable3.isRepaired());
+        Assert.assertTrue(sstable2.isPendingRepair());
+
+        assertEquals(Collections.singleton(repairID2),
+                     ((CompactionStrategyManager) compactionStrategyContainer).pendingRepairs());
     }
 
+    @Override
     @Test
-    public void finalizedSessionTransientCleanup()
+    public void testFinalizedSessionTransientCleanup() throws IOException
     {
         Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
         SSTableReader sstable = makeSSTable(true);
         mutateRepaired(sstable, repairID, true);
-        csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+        compactionStrategyContainer.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
         LocalSessionAccessor.finalizeUnsafe(repairID);
 
         Assert.assertFalse(hasPendingStrategiesFor(repairID));
@@ -419,28 +569,31 @@ public void finalizedSessionTransientCleanup()
         Assert.assertFalse(repairedContains(sstable));
         Assert.assertFalse(unrepairedContains(sstable));
 
-        cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
-        AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
+        cfs.getCompactionStrategyContainer().enable(); // enable compaction to fetch next background task
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
         Assert.assertNotNull(compactionTask);
-        Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
+        Assert.assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
 
         // run the compaction
-        compactionTask.execute(ActiveCompactionsTracker.NOOP);
+        compactionTask.execute();
 
         Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
         Assert.assertFalse(hasPendingStrategiesFor(repairID));
         Assert.assertFalse(hasTransientStrategiesFor(repairID));
     }
 
+    @Override
     @Test
-    public void failedSessionTransientCleanup()
+    public void testFailedSessionTransientCleanup() throws IOException
     {
         Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
         SSTableReader sstable = makeSSTable(true);
         mutateRepaired(sstable, repairID, true);
-        csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
+        compactionStrategyContainer.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
         LocalSessionAccessor.failUnsafe(repairID);
 
         Assert.assertFalse(hasPendingStrategiesFor(repairID));
@@ -450,13 +603,15 @@ public void failedSessionTransientCleanup()
         Assert.assertFalse(repairedContains(sstable));
         Assert.assertFalse(unrepairedContains(sstable));
 
-        cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
-        AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds());
+        cfs.getCompactionStrategyContainer().enable(); // enable compaction to fetch next background task
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
         Assert.assertNotNull(compactionTask);
-        Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
+        Assert.assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
 
         // run the compaction
-        compactionTask.execute(ActiveCompactionsTracker.NOOP);
+        compactionTask.execute();
 
         Assert.assertFalse(cfs.getLiveSSTables().isEmpty());
         Assert.assertFalse(hasPendingStrategiesFor(repairID));
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
index d66c357c8c15..d6469204b8e2 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
@@ -24,8 +24,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 import com.google.common.collect.ImmutableList;
@@ -50,11 +48,12 @@
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.DiskBoundaries;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SortedLocalRanges;
 import org.apache.cassandra.db.compaction.AbstractStrategyHolder.GroupedSSTableContainer;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.notifications.SSTableAddedNotification;
@@ -63,6 +62,7 @@
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.mockito.Mockito;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 import static org.junit.Assert.assertEquals;
@@ -71,6 +71,7 @@
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.when;
 
 public class CompactionStrategyManagerTest
 {
@@ -130,12 +131,12 @@ public void testSSTablesAssignedToCorrectCompactionStrategy() throws IOException
             if (i % 3 == 0)
             {
                 //make 1 third of sstables repaired
-                cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, System.currentTimeMillis(), null, false);
+                cfs.mutateRepaired(newSSTables, System.currentTimeMillis(), null, false);
             }
             else if (i % 3 == 1)
             {
                 //make 1 third of sstables pending repair
-                cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, 0, nextTimeUUID(), false);
+                cfs.mutateRepaired(newSSTables, 0, nextTimeUUID(), false);
             }
             previousSSTables = currentSSTables;
         }
@@ -152,6 +153,7 @@ public void testSSTablesAssignedToCorrectCompactionStrategy(int numSSTables, int
     {
         // Create a mock CFS with the given number of disks
         MockCFS cfs = createJBODMockCFS(numDisks);
+        CompactionStrategyFactory strategyFactory = new CompactionStrategyFactory(cfs);
         //Check that CFS will contain numSSTables
         assertEquals(numSSTables, cfs.getLiveSSTables().size());
 
@@ -160,8 +162,11 @@ public void testSSTablesAssignedToCorrectCompactionStrategy(int numSSTables, int
 
         MockBoundaryManager mockBoundaryManager = new MockBoundaryManager(cfs, boundaries);
         logger.debug("Boundaries for {} disks is {}", numDisks, Arrays.toString(boundaries));
-        CompactionStrategyManager csm = new CompactionStrategyManager(cfs, mockBoundaryManager::getBoundaries,
+        CompactionStrategyManager csm = new CompactionStrategyManager(strategyFactory,
+                                                                      mockBoundaryManager::getBoundaries,
+                                                                      true,
                                                                       true);
+        csm.reload(csm, cfs.metadata().params.compaction, CompactionStrategyContainer.ReloadReason.FULL);
 
         // Check that SSTables are assigned to the correct Compaction Strategy
         for (SSTableReader reader : cfs.getLiveSSTables())
@@ -201,82 +206,10 @@ public void testSSTablesAssignedToCorrectCompactionStrategy(int numSSTables, int
         }
     }
 
-    @Test
-    public void testAutomaticUpgradeConcurrency() throws Exception
-    {
-        ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
-        DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(true);
-        DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(1);
-
-        // latch to block CompactionManager.BackgroundCompactionCandidate#maybeRunUpgradeTask
-        // inside the currentlyBackgroundUpgrading check - with max_concurrent_auto_upgrade_tasks = 1 this will make
-        // sure that BackgroundCompactionCandidate#maybeRunUpgradeTask returns false until the latch has been counted down
-        CountDownLatch latch = new CountDownLatch(1);
-        AtomicInteger upgradeTaskCount = new AtomicInteger(0);
-        MockCFSForCSM mock = new MockCFSForCSM(cfs, latch, upgradeTaskCount);
-
-        CompactionManager.BackgroundCompactionCandidate r = CompactionManager.instance.getBackgroundCompactionCandidate(mock);
-        CompactionStrategyManager mgr = mock.getCompactionStrategyManager();
-        // basic idea is that we start a thread which will be able to get in to the currentlyBackgroundUpgrading-guarded
-        // code in CompactionManager, then we try to run a bunch more of the upgrade tasks which should return false
-        // due to the currentlyBackgroundUpgrading count being >= max_concurrent_auto_upgrade_tasks
-        Thread t = new Thread(() -> r.maybeRunUpgradeTask(mgr));
-        t.start();
-        Thread.sleep(100); // let the thread start and grab the task
-        assertEquals(1, CompactionManager.instance.currentlyBackgroundUpgrading.get());
-        assertFalse(r.maybeRunUpgradeTask(mgr));
-        assertFalse(r.maybeRunUpgradeTask(mgr));
-        latch.countDown();
-        t.join();
-        assertEquals(1, upgradeTaskCount.get()); // we should only call findUpgradeSSTableTask once when concurrency = 1
-        assertEquals(0, CompactionManager.instance.currentlyBackgroundUpgrading.get());
-
-        DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false);
-    }
-
-    @Test
-    public void testAutomaticUpgradeConcurrency2() throws Exception
-    {
-        ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
-        DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(true);
-        DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(2);
-        // latch to block CompactionManager.BackgroundCompactionCandidate#maybeRunUpgradeTask
-        // inside the currentlyBackgroundUpgrading check - with max_concurrent_auto_upgrade_tasks = 1 this will make
-        // sure that BackgroundCompactionCandidate#maybeRunUpgradeTask returns false until the latch has been counted down
-        CountDownLatch latch = new CountDownLatch(1);
-        AtomicInteger upgradeTaskCount = new AtomicInteger();
-        MockCFSForCSM mock = new MockCFSForCSM(cfs, latch, upgradeTaskCount);
-
-        CompactionManager.BackgroundCompactionCandidate r = CompactionManager.instance.getBackgroundCompactionCandidate(mock);
-        CompactionStrategyManager mgr = mock.getCompactionStrategyManager();
-
-        // basic idea is that we start 2 threads who will be able to get in to the currentlyBackgroundUpgrading-guarded
-        // code in CompactionManager, then we try to run a bunch more of the upgrade task which should return false
-        // due to the currentlyBackgroundUpgrading count being >= max_concurrent_auto_upgrade_tasks
-        Thread t = new Thread(() -> r.maybeRunUpgradeTask(mgr));
-        t.start();
-        Thread t2 = new Thread(() -> r.maybeRunUpgradeTask(mgr));
-        t2.start();
-        Thread.sleep(100); // let the threads start and grab the task
-        assertEquals(2, CompactionManager.instance.currentlyBackgroundUpgrading.get());
-        assertFalse(r.maybeRunUpgradeTask(mgr));
-        assertFalse(r.maybeRunUpgradeTask(mgr));
-        assertFalse(r.maybeRunUpgradeTask(mgr));
-        assertEquals(2, CompactionManager.instance.currentlyBackgroundUpgrading.get());
-        latch.countDown();
-        t.join();
-        t2.join();
-        assertEquals(2, upgradeTaskCount.get());
-        assertEquals(0, CompactionManager.instance.currentlyBackgroundUpgrading.get());
-
-        DatabaseDescriptor.setMaxConcurrentAutoUpgradeTasks(1);
-        DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false);
-    }
-
     private static void assertHolderExclusivity(boolean isRepaired, boolean isPendingRepair, boolean isTransient, Class<? extends AbstractStrategyHolder> expectedType)
     {
         ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
-        CompactionStrategyManager csm = cfs.getCompactionStrategyManager();
+        CompactionStrategyManager csm = (CompactionStrategyManager) cfs.getCompactionStrategy();
 
         AbstractStrategyHolder holder = csm.getHolder(isRepaired, isPendingRepair, isTransient);
         assertNotNull(holder);
@@ -297,7 +230,7 @@ private static void assertHolderExclusivity(boolean isRepaired, boolean isPendin
     private static void assertInvalieHolderConfig(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
     {
         ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
-        CompactionStrategyManager csm = cfs.getCompactionStrategyManager();
+        CompactionStrategyManager csm = (CompactionStrategyManager) cfs.getCompactionStrategy();
         try
         {
             csm.getHolder(isRepaired, isPendingRepair, isTransient);
@@ -326,10 +259,10 @@ public void testMutualExclusiveHolderClassification() throws Exception
         assertInvalieHolderConfig(true, false, true);
     }
 
-    PartitionPosition forKey(int key)
+    Token forKey(int key)
     {
         DecoratedKey dk = Util.dk(String.format("%04d", key));
-        return dk.getToken().minKeyBound();
+        return dk.getToken();
     }
 
     /**
@@ -340,7 +273,8 @@ public void groupSSTables() throws Exception
     {
         final int numDir = 4;
         ColumnFamilyStore cfs = createJBODMockCFS(numDir);
-        Keyspace.open(cfs.getKeyspaceName()).getColumnFamilyStore(cfs.name).disableAutoCompaction();
+        CompactionStrategyFactory strategyFactory = new CompactionStrategyFactory(cfs);
+        Keyspace.open(cfs.keyspace.getName()).getColumnFamilyStore(cfs.name).disableAutoCompaction();
         assertTrue(cfs.getLiveSSTables().isEmpty());
         List<SSTableReader> transientRepairs = new ArrayList<>();
         List<SSTableReader> pendingRepair = new ArrayList<>();
@@ -349,28 +283,33 @@ public void groupSSTables() throws Exception
 
         for (int i = 0; i < numDir; i++)
         {
-            int key = 100 * i;
+            int key = 100 * i + 1;  // key must not fall on boundary where it would be taken to belong to previous disk
             transientRepairs.add(createSSTableWithKey(cfs.getKeyspaceName(), cfs.name, key++));
             pendingRepair.add(createSSTableWithKey(cfs.getKeyspaceName(), cfs.name, key++));
             unrepaired.add(createSSTableWithKey(cfs.getKeyspaceName(), cfs.name, key++));
             repaired.add(createSSTableWithKey(cfs.getKeyspaceName(), cfs.name, key++));
         }
 
-        cfs.getCompactionStrategyManager().mutateRepaired(transientRepairs, 0, nextTimeUUID(), true);
-        cfs.getCompactionStrategyManager().mutateRepaired(pendingRepair, 0, nextTimeUUID(), false);
-        cfs.getCompactionStrategyManager().mutateRepaired(repaired, 1000, null, false);
+        cfs.mutateRepaired(transientRepairs, 0, nextTimeUUID(), true);
+        cfs.mutateRepaired(pendingRepair, 0, nextTimeUUID(), false);
+        cfs.mutateRepaired(repaired, 1000, null, false);
+
+
+        SortedLocalRanges localRanges = Mockito.mock(SortedLocalRanges.class);
+        when(localRanges.getRingVersion()).thenReturn(10L);
 
         DiskBoundaries boundaries = new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(),
                                                        Lists.newArrayList(forKey(100), forKey(200), forKey(300)),
-                                                       10, 10);
+                                                       localRanges, 10);
 
-        CompactionStrategyManager csm = new CompactionStrategyManager(cfs, () -> boundaries, true);
+        CompactionStrategyManager csm = new CompactionStrategyManager(strategyFactory, () -> boundaries, true, true);
+        csm.reload(csm, cfs.metadata().params.compaction, CompactionStrategyContainer.ReloadReason.FULL);
 
-        List<GroupedSSTableContainer> grouped = csm.groupSSTables(Iterables.concat( transientRepairs, pendingRepair, repaired, unrepaired));
+        List<GroupedSSTableContainer<SSTableReader>> grouped = csm.groupSSTables(Iterables.concat( transientRepairs, pendingRepair, repaired, unrepaired));
 
         for (int x=0; x<grouped.size(); x++)
         {
-            GroupedSSTableContainer group = grouped.get(x);
+            GroupedSSTableContainer<SSTableReader> group = grouped.get(x);
             AbstractStrategyHolder holder = csm.getHolders().get(x);
             for (int y=0; y<numDir; y++)
             {
@@ -472,7 +411,7 @@ private void verifySSTableIsAssignedToCorrectStrategy(Integer[] boundaries, Comp
         int index = getSSTableIndex(boundaries, reader);
         assertEquals(index, csm.compactionStrategyIndexFor(reader));
         // Check that compaction strategy actually contains SSTable
-        assertTrue(((SizeTieredCompactionStrategy)csm.compactionStrategyFor(reader)).sstables.contains(reader));
+        assertTrue(((SizeTieredCompactionStrategy) csm.compactionStrategyFor(reader)).sstables.contains(reader));
     }
 
     /**
@@ -496,13 +435,13 @@ private int getSSTableIndex(Integer[] boundaries, SSTableReader reader)
     {
         int index = 0;
         int firstKey = Integer.parseInt(new String(ByteBufferUtil.getArray(reader.getFirst().getKey())));
-        while (boundaries[index] <= firstKey)
+        while (boundaries[index] < firstKey)
             index++;
         logger.debug("Index for SSTable {} on boundary {} is {}", reader.descriptor.id, Arrays.toString(boundaries), index);
         return index;
     }
 
-    class MockBoundaryManager
+    private class MockBoundaryManager
     {
         private final ColumnFamilyStore cfs;
         private Integer[] positions;
@@ -529,8 +468,10 @@ public DiskBoundaries getBoundaries()
 
         private DiskBoundaries createDiskBoundaries(ColumnFamilyStore cfs, Integer[] boundaries)
         {
-            List<PartitionPosition> positions = Arrays.stream(boundaries).map(b -> Util.token(String.format(String.format("%04d", b))).minKeyBound()).collect(Collectors.toList());
-            return new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(), positions, 0, 0);
+            List<Token> positions = Arrays.stream(boundaries).map(b -> Util.token(String.format(String.format("%04d", b)))).collect(Collectors.toList());
+            SortedLocalRanges localRanges = Mockito.mock(SortedLocalRanges.class);
+            when(localRanges.getRingVersion()).thenReturn(0L);
+            return new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(), positions, localRanges, 0);
         }
     }
 
@@ -558,50 +499,4 @@ private static class MockCFS extends ColumnFamilyStore
             super(cfs.keyspace, cfs.getTableName(), Util.newSeqGen(), cfs.metadata, dirs, false, false, true);
         }
     }
-
-    private static class MockCFSForCSM extends ColumnFamilyStore
-    {
-        private final CountDownLatch latch;
-        private final AtomicInteger upgradeTaskCount;
-
-        private MockCFSForCSM(ColumnFamilyStore cfs, CountDownLatch latch, AtomicInteger upgradeTaskCount)
-        {
-            super(cfs.keyspace, cfs.name, Util.newSeqGen(10), cfs.metadata, cfs.getDirectories(), true, false, false);
-            this.latch = latch;
-            this.upgradeTaskCount = upgradeTaskCount;
-        }
-        @Override
-        public CompactionStrategyManager getCompactionStrategyManager()
-        {
-            return new MockCSM(this, latch, upgradeTaskCount);
-        }
-    }
-
-    private static class MockCSM extends CompactionStrategyManager
-    {
-        private final CountDownLatch latch;
-        private final AtomicInteger upgradeTaskCount;
-
-        private MockCSM(ColumnFamilyStore cfs, CountDownLatch latch, AtomicInteger upgradeTaskCount)
-        {
-            super(cfs);
-            this.latch = latch;
-            this.upgradeTaskCount = upgradeTaskCount;
-        }
-
-        @Override
-        public AbstractCompactionTask findUpgradeSSTableTask()
-        {
-            try
-            {
-                latch.await();
-                upgradeTaskCount.incrementAndGet();
-            }
-            catch (InterruptedException e)
-            {
-                throw new RuntimeException(e);
-            }
-            return null;
-        }
-    }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyStatisticsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyStatisticsTest.java
new file mode 100644
index 000000000000..fadc5869905c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyStatisticsTest.java
@@ -0,0 +1,775 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.db.compaction.unified.Controller;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Overlaps;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TimeUUID;
+import org.mockito.Mockito;
+
+import static org.apache.cassandra.db.compaction.LeveledManifest.MAX_COMPACTING_L0;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyDouble;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test for the compaction statistics for all strategies that support them.
+ */
+public class CompactionStrategyStatisticsTest extends BaseCompactionStrategyTest
+{
+    private final int minCompactionThreshold = 4;
+    private final int maxCompactionThreshold = 32;
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        BaseCompactionStrategyTest.setUpClass();
+    }
+
+    @Before
+    public void setUp()
+    {
+        super.setUp();
+
+        when(realm.getMinimumCompactionThreshold()).thenReturn(minCompactionThreshold);
+        when(realm.getMaximumCompactionThreshold()).thenReturn(maxCompactionThreshold);
+    }
+
+    /**
+     * Creates 5 buckets with T sorted runs in each using W = 2 and o = 1 (the default)
+     */
+    @Test
+    public void testUnifiedCompactionStrategy_tiered_twoShards_fiveBuckets_W2()
+    {
+        int W = 2; // W = 2 => T = F = 4
+        int T = 4;
+        int F = 4;
+        final long minSstableSizeBytes = 2L << 20; // 2 MB
+        final int numBuckets = 5;
+
+        Controller controller = Mockito.mock(Controller.class);
+        when(controller.getScalingParameter(anyInt())).thenReturn(W);
+        when(controller.getFanout(anyInt())).thenReturn(F);
+        when(controller.getThreshold(anyInt())).thenReturn(T);
+        when(controller.getMinSstableSizeBytes()).thenReturn(minSstableSizeBytes);
+        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minSstableSizeBytes);
+        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.TRANSITIVE);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
+        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE);
+        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.random()).thenCallRealMethod();
+        when(controller.getNumShards(anyDouble())).thenReturn(1);
+        // Calculate the minimum shard size such that the top bucket compactions won't be considered "oversized" and
+        // all will be allowed to run. The calculation below assumes (1) that compactions are considered "oversized"
+        // if they are more than 1/2 of the max shard size; (2) that mockSSTables uses 15% less than the max SSTable
+        // size for that bucket.
+//        long topBucketMaxSstableSize = (long) (minSstableSizeBytes * Math.pow(F, numBuckets));
+//        long minShardSizeWithoutOversizedCompactions = T * topBucketMaxSstableSize * 2;
+//        when(controller.getShardSizeBytes()).thenReturn(minShardSizeWithoutOversizedCompactions);
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+        List<Collection<SSTableReader>> testBuckets = new ArrayList<>(numBuckets * 2);
+
+        // The order is repaired false, disk 0, then repaired true, disk 1, one bucket per shard, lowest to highest
+        // because the test picks from the end of the test buckets, we need to revert this order
+        for (int i = numBuckets - 1; i >= 0; i--)
+        {
+            for (boolean repaired : new boolean[] { false, true })
+            {
+                for (int diskIndex = 1; diskIndex >= 0; diskIndex--)
+                {
+                    // calculate the max size then mockSSTables will remove 15% to this value,
+                    // this assumes o = 1, which is the default
+                    long size = (long) (minSstableSizeBytes * Math.pow(F, i + 1));
+                    List<SSTableReader> sstables = mockSSTables(T,
+                                                                size,
+                                                                0,
+                                                                System.currentTimeMillis(),
+                                                                diskIndex,
+                                                                repaired,
+                                                                null);
+                    testBuckets.add(sstables);
+                }
+            }
+        }
+
+        testCompactionStatistics(testBuckets, strategy);
+    }
+
+    /**
+     * Creates 5 buckets with T sorted runs in each using W = 2 and o = 1 (the default)
+     */
+    @Test
+    public void testUnifiedCompactionStrategy_leveled_one_shard_oneBucket_F8()
+    {
+        int W = -6; // W = 2 => T = 2, F = 8
+        int T = 2;
+        int F = 8;
+        int m = 2; // m = 2 MB
+        long minSize = m << 20; // MB to bytes
+
+        Controller controller = Mockito.mock(Controller.class);
+        when(controller.getScalingParameter(anyInt())).thenReturn(W);
+        when(controller.getFanout(anyInt())).thenReturn(F);
+        when(controller.getThreshold(anyInt())).thenReturn(T);
+        when(controller.getMinSstableSizeBytes()).thenReturn(minSize);
+        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minSize);
+        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.TRANSITIVE);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
+        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE);
+        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.random()).thenCallRealMethod();
+        when(controller.getNumShards(anyDouble())).thenReturn(1);
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+
+        // put F sstables in the first bucket
+        List<SSTableReader> ssTablesList = new LinkedList<>();
+        for (int i = 0; i < F; i++)
+            ssTablesList.addAll(mockSSTables(1, minSize, 0, System.currentTimeMillis()));
+
+        Collections.sort(ssTablesList, Comparator.comparing(SSTableReader::onDiskLength).reversed());
+        Set<SSTableReader> sstables = new LinkedHashSet<>(F);
+        sstables.addAll(ssTablesList);
+
+        // sort by size and add 2 by 2 from largest to smallest, normally the sstable resulting from the 1 compaction
+        // would be added back to the same bucket and be selected for the next compaction but we don't simulate this
+        // so next time a compaction is invoked it will pick the next two largest sstables, that's why there will be
+        // F/2 compactions rather than F-1
+//        LinkedList<Collection<SSTableReader>> compactions = new LinkedList();
+//        for (int i = 0; (i + T) <= ssTablesList.size(); i += T)
+//        {
+//            List<SSTableReader> candidates = ssTablesList.subList(i, i + T);
+//            compactions.addFirst(candidates); // we want the first 2 sstables (the largest) to be the last in the list
+//        }
+
+        testCompactionStatistics(sstables, ImmutableList.of(sstables), 1, strategy);
+    }
+
+    /**
+     * Creates 5 STCS buckets with a single compaction pick (<= max threshold tables) and
+     * increasing hotness so that the highest test bucket will be compacted first.
+     */
+    @Test
+    public void testSizeTieredCompactionStrategy_fiveBucketsOnePick()
+    {
+        Map<String, String> options = new HashMap<>();
+        addSizeTieredOptions(options);
+
+        SizeTieredCompactionStrategy strategy = new SizeTieredCompactionStrategy(strategyFactory, options);
+
+        final int numCompactions = 5;
+        long minSize = SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE;
+        double hotness = 1000;
+
+        List<Collection<SSTableReader>> testBuckets = new ArrayList<>(numCompactions);
+        for (int i = 0; i < numCompactions; i++)
+        {
+            List<SSTableReader> sstables = mockSSTables(maxCompactionThreshold,
+                                                        minSize,
+                                                        hotness,
+                                                        System.currentTimeMillis());
+            testBuckets.add(sstables);
+
+            minSize *= 10;
+            hotness *= 2;
+        }
+
+        testCompactionStatistics(testBuckets, strategy);
+    }
+
+    /**
+     * Creates a single STCS bucket with enough sstables to fill 5 picks.
+     */
+    @Test
+    public void testSizeTieredCompactionStrategy_oneBucketFivePicks()
+    {
+        Map<String, String> options = new HashMap<>();
+        addSizeTieredOptions(options);
+
+        SizeTieredCompactionStrategy strategy = new SizeTieredCompactionStrategy(strategyFactory, options);
+
+        final int numCompactions = 5;
+        long size = SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE * 2;
+        double hotness = 1000;
+
+        List<Collection<SSTableReader>> testBuckets = new ArrayList<>(numCompactions);
+        for (int i = 0; i < numCompactions; i++)
+        {
+            List<SSTableReader> sstables = mockSSTables(maxCompactionThreshold,
+                                                        size,
+                                                        hotness,
+                                                        System.currentTimeMillis());
+            testBuckets.add(sstables);
+            hotness *= 2;
+        }
+
+        testCompactionStatistics(testBuckets, strategy);
+    }
+
+    /**
+     * Creates 3 STCS buckets with enough sstables to have 2 compactions per bucket and increasing
+     * hotness so that the highest test buckets will be compacted first.
+     */
+    @Test
+    public void testSizeTieredCompactionStrategy_threeBucketsTwoPicks()
+    {
+        Map<String, String> options = new HashMap<>();
+        addSizeTieredOptions(options);
+
+        SizeTieredCompactionStrategy strategy = new SizeTieredCompactionStrategy(strategyFactory, options);
+
+        long minSize = SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE;
+        double hotness = 1000;
+
+        List<Collection<SSTableReader>> testBuckets = new ArrayList<>(3);
+        for (int i = 0; i < 3; i++) // STCS buckets
+        {
+            for (int j = 0; j < 2; j++) // picks
+            {
+                List<SSTableReader> sstables = mockSSTables(maxCompactionThreshold,
+                                                            minSize,
+                                                            hotness,
+                                                            System.currentTimeMillis());
+                testBuckets.add(sstables);
+                hotness *= 2;
+            }
+
+            minSize *= 10;
+        }
+
+        testCompactionStatistics(testBuckets, strategy);
+    }
+
+
+    /**
+     * Creates 5 TWCS buckets with increasing timestamp so that the higher buckets will be compacted first.
+     * Each bucket only has a single compaction pick (<= max threshold tables).
+     */
+    @Test
+    public void testTimeWindowCompactionStrategy_fiveBucketsOnePick()
+    {
+        Map<String, String> options = new HashMap<>();
+        addTimeTieredOptions(options);
+
+        TimeWindowCompactionStrategy strategy = new TimeWindowCompactionStrategy(strategyFactory, options);
+
+        final int numCompactions = 5;
+        long size = SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE * 5;
+        double hotness = 1000;
+        long timestap = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(10); // 10 hours ago
+
+        List<Collection<SSTableReader>> testBuckets = new ArrayList<>(numCompactions);
+        for (int i = 0; i < numCompactions; i++)
+        {
+            List<SSTableReader> sstables = mockSSTables(maxCompactionThreshold, size, hotness, timestap);
+            testBuckets.add(sstables);
+
+            timestap += TimeUnit.HOURS.toMillis(2);
+        }
+
+        testCompactionStatistics(testBuckets, strategy);
+    }
+
+    /**
+     * Creates a single TWCS bucket with enough sstables to fill 5 picks.
+     */
+    @Test
+    public void testTimeWindowCompactionStrategy_oneBucketFivePicks()
+    {
+        Map<String, String> options = new HashMap<>();
+        addTimeTieredOptions(options);
+
+        TimeWindowCompactionStrategy strategy = new TimeWindowCompactionStrategy(strategyFactory, options);
+
+        final int numCompactions = 5;
+        long size = SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE * 5;
+        double hotness = 100;
+        long timestap = System.currentTimeMillis();
+
+        List<Collection<SSTableReader>> testBuckets = new ArrayList<>(numCompactions);
+        for (int i = 0; i < numCompactions; i++)
+        {
+            List<SSTableReader> sstables = mockSSTables(maxCompactionThreshold, size, hotness, timestap);
+            testBuckets.add(sstables);
+
+            hotness *= 2; // hottest tables should be picked first because TWCS uses STCS in the latest bucket
+        }
+
+        testCompactionStatistics(testBuckets, strategy);
+    }
+
+    /**
+     * Creates 3 TWCS buckets with enough sstables to have 2 compactions per bucket.
+     */
+    @Test
+    public void testTimeWindowCompactionStrategy_threeBucketsTwoPicks()
+    {
+        Map<String, String> options = new HashMap<>();
+        addTimeTieredOptions(options);
+
+        TimeWindowCompactionStrategy strategy = new TimeWindowCompactionStrategy(strategyFactory, options);
+
+        long size = SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE * 10;
+        double hotness = 1000;
+        long timestap = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(10); // 10 hours ago
+
+        List<Collection<SSTableReader>> testBuckets = new ArrayList<>(3 * 2);
+        for (int i = 0; i < 3; i++)
+        {
+            for (int j = 0; j < 2; j++)
+            {
+                List<SSTableReader> sstables = mockSSTables(maxCompactionThreshold, size, hotness, timestap);
+                testBuckets.add(sstables);
+
+                hotness *= 2; // hottest tables should be picked first in the newest bucket because of STCS
+                size -= SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE; // smaller sstables are picked first in other TWCS buckets
+            }
+
+            size = SizeTieredCompactionStrategyOptions.DEFAULT_MIN_SSTABLE_SIZE * 10;
+            timestap += TimeUnit.HOURS.toMillis(2);
+        }
+
+        testCompactionStatistics(testBuckets, strategy);
+    }
+
+    /**
+     * A utility method for determining the overlapping sstables similarly to what {@link LeveledManifest} does
+     * when selecting sstables from the next level that overlap with a candidate of the previous level.
+     *
+     * @param sstable the sstables from the previous level
+     * @param candidates the candidates sstables from the next level
+     *
+     * @return a set containing the sstable passed in and all the sstables that overlap from the candidates
+     */
+    private static Set<SSTableReader> overlapping(SSTableReader sstable, List<SSTableReader> candidates)
+    {
+        Map<CompactionSSTable, Bounds<Token>> candidatesWithBounds = LeveledManifest.genBounds(candidates);
+        Set<CompactionSSTable> overlapping = LeveledManifest.overlappingWithBounds(sstable,
+                                                                                   candidatesWithBounds);
+        Set<SSTableReader> overlappingReaders = new HashSet<>();
+        overlappingReaders.add(sstable);
+        for (CompactionSSTable s : overlapping)
+            overlappingReaders.add((SSTableReader) s);
+        return overlappingReaders;
+    }
+
+    /**
+     * Creates 3 LCS levels. Checks that L2 and L3 are able to compact in parallel but L0 gets blocked by the
+     * L1 compaction. Once the L2 and L3 compactions have finished, then the L0 compaction can proceed.
+     */
+    @Test
+    public void testLeveledCompactionStrategy_threeLevels()
+    {
+        Map<String, String> options = new HashMap<>();
+        long maxSSTableSize = 160 << 20; // 160 MB in bytes
+        addLeveledOptions(options, maxSSTableSize);
+
+        LeveledCompactionStrategy strategy = new LeveledCompactionStrategy(strategyFactory, options);
+
+        final int numLevels = 3;
+        List<List<SSTableReader>> ssTablesByLevel = new ArrayList<>(numLevels);
+        for (int i = 0; i < numLevels; i++)
+        {
+            // level zero maximum size is 4 times maxSSTableSize, and for other levels it is
+            // the fan-out size (10) to the power of the level number, times maxSSTableSize.
+            long maxLevelSize = (long) ((i == 0 ? 4 : Math.pow(10, i)) * maxSSTableSize);
+
+            // we add one to ensure the score will be > 1 so that only one sstable (and no more!) will be selected for compaction
+            int numSSTables = (int) Math.ceil(maxLevelSize / maxSSTableSize) + 1;
+
+            List<SSTableReader> sstables = mockNonOverlappingSSTables(numSSTables, i, maxSSTableSize);
+            ssTablesByLevel.add(sstables);
+        }
+
+        // all sstables flattened
+        Set<SSTableReader> sstables = ssTablesByLevel.stream().flatMap(bucket -> bucket.stream()).collect(Collectors.toSet());
+
+        // Organize the sstables into the expected compactions
+        // LCS will always compact the highest level first unless L0 has more than 32 sstables in which case
+        // it compacts using STCS
+        List<Collection<SSTableReader>> compactions = new ArrayList(3);
+
+        //L0 will compact all its sstables and the ones of L1 since they all overlap and the total is below the max threshold
+        compactions.add(Sets.union(Sets.newLinkedHashSet(ssTablesByLevel.get(0)), Sets.newLinkedHashSet(ssTablesByLevel.get(1))));
+
+        // L1 will compact the first sstable that it finds not overlapping with L2 sstables that are not suspect or already
+        // compacting. Because the next line will select the first sstable in L2 to compact, L1 will pick the first sstable
+        // that does not overlap with it
+        SSTableReader candidate = null;
+        for (SSTableReader c : ssTablesByLevel.get(1))
+        {
+            if (c.getFirst().compareTo(ssTablesByLevel.get(2).get(0).getLast()) > 0)
+            {
+                candidate = c;
+                break;
+            }
+        }
+        assertNotNull(candidate);
+        // compact the candidate with all the overlapping sstables of L2
+        compactions.add(overlapping(candidate, ssTablesByLevel.get(2)));
+
+        // L2 will compact the first sstable because the score is > 1 but no other overlapping sstables since L3 is empty
+        compactions.add(overlapping(ssTablesByLevel.get(2).get(0), ImmutableList.of()));
+
+        // L2 and L1 compactions can proceed in parallel but L0 will refuse to compact due to overlapping sstables in L1
+        // already compacting, hence we can only test 2 compactions initially
+        testCompactionStatistics(sstables, compactions, 2, strategy);
+
+        // Now check L0 compaction can proceed, the other levels won't compact since the score should be <= 1
+        ssTablesByLevel.get(1).remove(candidate); // remove the L1 sstable that was already compacted
+        Set<SSTableReader> candidates = Sets.union(Sets.newLinkedHashSet(ssTablesByLevel.get(0)), Sets.newLinkedHashSet(ssTablesByLevel.get(1)));
+        long totLength = totUncompressedLength(candidates);
+
+        Collection<AbstractCompactionTask> tasks = strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertFalse(tasks.isEmpty());
+
+        for (AbstractCompactionTask task : tasks)
+        {
+            assertNotNull(task);
+            TimeUUID id = task.getTransaction().opId();
+
+            verifyStatistics(strategy,
+                             1,
+                             1,
+                             candidates.size(),
+                             candidates.size(),
+                             totLength,
+                             0,
+                             0,
+                             0);
+
+            CompactionProgress progress = mockCompletedCompactionProgress(candidates, id);
+            strategy.onInProgress(progress);
+
+            verifyStatistics(strategy,
+                             1,
+                             1,
+                             candidates.size(),
+                             candidates.size(),
+                             totLength,
+                             totLength,
+                             totLength,
+                             0);
+
+            strategy.backgroundCompactions.onCompleted(strategy, id);
+        }
+
+        // Now we should have L1 again...
+    }
+
+    /**
+     *  Test the case where L0 has enough sstables to trigger STCS, plus also add some tables in L1.
+     */
+    @Test
+    public void testLeveledCompactionStrategy_stcsL0()
+    {
+        Map<String, String> options = new HashMap<>();
+        long maxSSTableSize = 160 << 20; // 160 MB in bytes
+        addLeveledOptions(options, maxSSTableSize);
+
+        LeveledCompactionStrategy strategy = new LeveledCompactionStrategy(strategyFactory, options);
+
+        int level = 1;
+        long maxLevelSize = (long) (Math.pow(10, level) * maxSSTableSize);
+        int numSSTables = (int) Math.ceil(maxLevelSize / maxSSTableSize) + 1;
+        List<SSTableReader> l1SSTables = mockNonOverlappingSSTables(numSSTables, level, maxSSTableSize);
+
+        List<SSTableReader> l0SSTables = mockSSTables(MAX_COMPACTING_L0 + 1, maxSSTableSize, 0.0, System.currentTimeMillis());
+
+        Set<SSTableReader> sstables = Sets.newHashSet(l0SSTables);
+        sstables.addAll(l1SSTables);
+
+        // Organize the sstables into the expected compactions
+        // LCS will always compact the highest level first unless L0 has more than 32 sstables in which case
+        // it compacts using STCS
+        List<Collection<SSTableReader>> compactions = new ArrayList(2);
+
+        // L1 will compact the first sstable because the score is > 1 but no other overlapping sstables since L2 is empty
+        compactions.add(overlapping(l1SSTables.get(0), ImmutableList.of()));
+
+        // L0 should use STCS to compact them all up to the max threshold, since all sstables have the same hotness,
+        // they will be sorted by size
+        Collections.sort(l0SSTables, Comparator.comparing(SSTableReader::onDiskLength));
+        compactions.add(l0SSTables.subList(0, Math.min(maxCompactionThreshold, l0SSTables.size())));
+
+        testCompactionStatistics(sstables, compactions, compactions.size(), strategy);
+    }
+
+    private void testCompactionStatistics(List<Collection<SSTableReader>> compactions, AbstractCompactionStrategy strategy)
+    {
+        Set<SSTableReader> sstables = compactions.stream().flatMap(Collection::stream).collect(Collectors.toSet());
+        testCompactionStatistics(sstables, compactions, compactions.size(), strategy);
+    }
+
+    /**
+     * Tests the statistics for a given strategy. It is expected that the compactions passed in will contain a set of sstables
+     * to be compacted together, with the highest index being picked first, then the second highest and so forth.
+     *
+     * @param compactions sstables grouped by compaction, each compaction is expected to be compacted fully (no splitting currently
+     *                    supported), the highest index compaction should be picked first by the strategy
+     * @param numExpectedCompactions the expected number of compactions that can occur in parallel
+     * @param strategy the compaction strategy
+     */
+    private void testCompactionStatistics(Set<SSTableReader> sstables,
+                                          List<Collection<SSTableReader>> compactions,
+                                          int numExpectedCompactions,
+                                          AbstractCompactionStrategy strategy)
+    {
+        // Add the tables to the strategy and the data tracker
+        addSSTablesToStrategy(strategy, sstables);
+
+        List<CompactionSSTable> sstablesForCompaction = compactions.stream().flatMap(Collection::stream).collect(Collectors.toList());
+
+        int numSSTables = sstablesForCompaction.size();
+        long totLength = totUncompressedLength(sstablesForCompaction);
+        double totHotness = totHotness(sstablesForCompaction);
+
+        Set<CompactionSSTable> compacting = new HashSet<>();
+        List<Pair<Set<SSTableReader>, TimeUUID>> submittedCompactions = new ArrayList<>(compactions.size());
+
+        long totRead = 0;
+        long totWritten = 0;
+        int numSSTablesCompacting = 0;
+        int numCompactions = compactions.size();
+        int numCompactionsInProgress = 0;
+
+        // Create a compaction task and start the compaction for each bucket starting with the highest index
+        int i = 0;
+        while (i < numExpectedCompactions)
+        {
+            List<Pair<Set<SSTableReader>, TimeUUID>> tasksCompactions = new ArrayList<>(compactions.size());
+            Collection<AbstractCompactionTask> tasks = strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+            assertFalse(tasks.isEmpty());
+
+            // Keep track of all the tasks that were submitted (one per shard)
+            for (AbstractCompactionTask task : tasks)
+            {
+                Set<SSTableReader> candidates = Sets.newHashSet(task.transaction.originals());
+
+                i++;
+
+                assertNotNull(task);
+                TimeUUID id = task.getTransaction().opId();
+
+                numCompactionsInProgress++;
+                numSSTablesCompacting += candidates.size();
+                tasksCompactions.add(Pair.create(candidates, id));
+            }
+
+            // after mocking the compactions the list of pending compactions has been updated in the strategy
+            // and this will be reflected in the statistics but the compaction task has not started yet
+            verifyStatistics(strategy,
+                             numCompactions,
+                             numCompactionsInProgress,
+                             numSSTables,
+                             numSSTablesCompacting,
+                             totLength,
+                             totRead,
+                             totWritten,
+                             totHotness);
+
+            // Start the compactions and check the statistics are updated
+            for (Pair<Set<SSTableReader>, TimeUUID> pair : tasksCompactions)
+            {
+                TimeUUID id = pair.right;
+                Set<SSTableReader> candidates = pair.left;
+
+                // Now we simulate starting the compaction task
+                CompactionProgress progress = mockCompletedCompactionProgress(candidates, id);
+                strategy.onInProgress(progress);
+
+                // The compaction has started and so we must updated the following expected values
+                totRead += progress.uncompressedBytesRead();
+                totWritten += progress.uncompressedBytesWritten();
+
+                // Now check that the statistics reflect the compaction in progress
+                verifyStatistics(strategy,
+                                 numCompactions,
+                                 numCompactionsInProgress,
+                                 numSSTables,
+                                 numSSTablesCompacting,
+                                 totLength,
+                                 totRead,
+                                 totWritten,
+                                 totHotness);
+
+                // update compacting for the next iteration
+                compacting.addAll(candidates);
+            }
+
+            submittedCompactions.addAll(tasksCompactions);
+        }
+
+        assertEquals(numExpectedCompactions, submittedCompactions.size());
+
+        // Terminate the compactions one by one by closing the AutoCloseable and check
+        // that the statistics are updated
+        for (Pair<Set<SSTableReader>, TimeUUID> pair : submittedCompactions)
+        {
+            Set<SSTableReader> compSSTables = pair.left;
+            long totSSTablesLen = totUncompressedLength(compSSTables);
+            strategy.onCompleted(pair.right, null);
+
+            numCompactions--;
+            numCompactionsInProgress--;
+            numSSTables -= compSSTables.size();
+            numSSTablesCompacting -= compSSTables.size();
+
+            totLength -= totSSTablesLen;
+            totRead -= totSSTablesLen;
+            totWritten -= totSSTablesLen;
+            totHotness -= totHotness(compSSTables);
+
+            removeSSTablesFromStrategy(strategy, pair.left);
+            sstables.removeAll(pair.left);
+            compacting.removeAll(pair.left);
+
+            verifyStatistics(strategy,
+                             numCompactions,
+                             numCompactionsInProgress,
+                             numSSTables,
+                             numSSTablesCompacting,
+                             totLength,
+                             totRead,
+                             totWritten,
+                             totHotness);
+        }
+
+        assertTrue(String.format("Data tracker still had compacting sstables: %s", dataTracker.getCompacting()),
+                   dataTracker.getCompacting().isEmpty());
+    }
+
+    private void verifyStatistics(CompactionStrategy strategy,
+                                  int expectedCompactions,
+                                  int expectedCompacting,
+                                  int expectedSSTables,
+                                  int expectedSSTablesCompacting,
+                                  long expectedTotBytes,
+                                  long expectedReadBytes,
+                                  long expectedWrittenBytes,
+                                  double expectedTotHotness)
+    {
+        CompactionStrategyStatistics stats = strategy.getStatistics().get(0);
+        System.out.println(stats.toString());
+
+        assertEquals(keyspace, stats.keyspace());
+        assertEquals(table, stats.table());
+        assertEquals(strategy.getClass().getSimpleName(), stats.strategy());
+
+        assertEquals(expectedCompactions, strategy.getTotalCompactions());
+
+        int numCompactions = 0;
+        int numCompacting = 0;
+        int numSSTables = 0;
+        int numCompactingSSTables = 0;
+        long totBytes = 0;
+        long writtenBytes = 0;
+        long readBytes = 0;
+        double hotness = 0;
+
+        for (CompactionAggregateStatistics compactionStatistics : stats.aggregates())
+        {
+            numCompactions += compactionStatistics.numCompactions();
+            numCompacting += compactionStatistics.numCompactionsInProgress();
+            numSSTables += compactionStatistics.numCandidateSSTables();
+            numCompactingSSTables += compactionStatistics.numCompactingSSTables();
+
+            if (compactionStatistics instanceof TieredCompactionStatistics)
+            {
+                TieredCompactionStatistics tieredStatistics = (TieredCompactionStatistics) compactionStatistics;
+
+                totBytes += tieredStatistics.tot();
+                writtenBytes += tieredStatistics.written();
+                readBytes += tieredStatistics.read();
+                hotness += tieredStatistics.hotness;
+            }
+            else if (compactionStatistics instanceof LeveledCompactionStatistics)
+            {
+                LeveledCompactionStatistics leveledStatistics = (LeveledCompactionStatistics) compactionStatistics;
+
+                totBytes += leveledStatistics.tot();
+                writtenBytes += leveledStatistics.written();
+                readBytes += leveledStatistics.read();
+            }
+            else
+            {
+                UnifiedCompactionStatistics tieredStatistics = (UnifiedCompactionStatistics) compactionStatistics;
+
+                totBytes += tieredStatistics.tot();
+                writtenBytes += tieredStatistics.written();
+                readBytes += tieredStatistics.read();
+            }
+        }
+
+        assertEquals(expectedCompactions, numCompactions);
+        assertEquals(expectedCompacting, numCompacting);
+
+        if (!(strategy instanceof LeveledCompactionStrategy))
+        { // LCS won't report pending sstables but only pending tasks
+            assertEquals(expectedSSTables, numSSTables);
+            assertEquals(expectedSSTablesCompacting, numCompactingSSTables);
+            assertEquals(expectedTotBytes, totBytes);
+        }
+
+        assertEquals(expectedReadBytes, readBytes);
+        assertEquals(expectedWrittenBytes, writtenBytes);
+
+        if (expectedTotHotness > 0)
+            assertEquals(expectedTotHotness, hotness, epsilon);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
index 066aa92067f1..f04251f6cadc 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
@@ -24,10 +24,13 @@
 import java.util.List;
 import java.util.Set;
 
+import com.google.common.collect.ImmutableSet;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
@@ -37,24 +40,52 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Transactional;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
 
 import static java.lang.String.format;
+import static org.apache.cassandra.db.lifecycle.View.updateCompacting;
 import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyCollection;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
+@RunWith(Parameterized.class)
 public class CompactionTaskTest
 {
     private static TableMetadata cfm;
     private static ColumnFamilyStore cfs;
 
+    @Parameterized.Parameters(name = "useCursors={0}")
+    public static Iterable<Boolean> useCursorChoices()
+    {
+        return ImmutableSet.of(false, true);
+    }
+
+    private final CompactionStrategy mockStrategy;
+
+    public CompactionTaskTest(boolean useCursors)
+    {
+        this.mockStrategy = mockStrategy(cfs, useCursors);
+    }
+
     @BeforeClass
     public static void setUpClass() throws Exception
     {
@@ -67,7 +98,7 @@ public static void setUpClass() throws Exception
     @Before
     public void setUp() throws Exception
     {
-        cfs.getCompactionStrategyManager().enable();
+        cfs.getCompactionStrategyContainer().enable();
         cfs.truncateBlocking();
     }
 
@@ -88,7 +119,7 @@ public void testTaskIdIsPersistedInCompactionHistory()
         try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION))
         {
             id = txn.opId();
-            CompactionTask task = new CompactionTask(cfs, txn, 0);
+            CompactionTask task = new CompactionTask(cfs, txn, 0, false, null);
             task.execute(CompactionManager.instance.active);
         }
 
@@ -106,9 +137,9 @@ public void testTaskIdIsPersistedInCompactionHistory()
     }
 
     @Test
-    public void compactionInterruption() throws Exception
+    public void compactionDisabled() throws Exception
     {
-        cfs.getCompactionStrategyManager().disable();
+        cfs.getCompactionStrategyContainer().disable();
         QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);");
         QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);");
         Util.flush(cfs);
@@ -121,9 +152,10 @@ public void compactionInterruption() throws Exception
 
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
         Assert.assertNotNull(txn);
-        CompactionTask task = new CompactionTask(cfs, txn, 0);
+
+        AbstractCompactionTask task = new CompactionTask(cfs, txn, 0, false, mockStrategy);
         Assert.assertNotNull(task);
-        cfs.getCompactionStrategyManager().pause();
+        cfs.getCompactionStrategyContainer().pause();
         try
         {
             task.execute(CompactionManager.instance.active);
@@ -136,7 +168,42 @@ public void compactionInterruption() throws Exception
         Assert.assertEquals(Transactional.AbstractTransactional.State.ABORTED, txn.state());
     }
 
-    private static void mutateRepaired(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair, boolean isTransient) throws IOException
+    @Test
+    public void compactionInterruption()
+    {
+        cfs.getCompactionStrategyContainer().disable();
+        Set<SSTableReader> sstables = generateData(2, 2);
+
+        LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+        assertNotNull(txn);
+
+        AbstractCompactionTask task = new CompactionTask(cfs, txn, 0, false, mockStrategy);
+        assertNotNull(task);
+
+        TableOperationObserver obs = Mockito.mock(TableOperationObserver.class);
+        NonThrowingCloseable cls = Mockito.mock(NonThrowingCloseable.class);
+
+        when(obs.onOperationStart(any(TableOperation.class))).thenAnswer(invocation -> {
+            TableOperation op = invocation.getArgument(0);
+            op.stop(TableOperation.StopTrigger.UNIT_TESTS);
+            return cls;
+        });
+
+        try
+        {
+            task.execute(obs);
+            Assert.fail("Expected CompactionInterruptedException");
+        }
+        catch (CompactionInterruptedException e)
+        {
+            // pass
+        }
+
+        verify(cls, times(1)).close();
+        assertEquals(Transactional.AbstractTransactional.State.ABORTED, txn.state());
+    }
+
+    static void mutateRepaired(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair, boolean isTransient) throws IOException
     {
         sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair, isTransient);
         sstable.reloadSSTableMetadata();
@@ -149,7 +216,7 @@ private static void mutateRepaired(SSTableReader sstable, long repairedAt, TimeU
     @Test
     public void mixedSSTableFailure() throws Exception
     {
-        cfs.getCompactionStrategyManager().disable();
+        cfs.getCompactionStrategyContainer().disable();
         QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);");
         Util.flush(cfs);
         QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);");
@@ -171,34 +238,46 @@ public void mixedSSTableFailure() throws Exception
         mutateRepaired(pending1, UNREPAIRED_SSTABLE, nextTimeUUID(), false);
         mutateRepaired(pending2, UNREPAIRED_SSTABLE, nextTimeUUID(), false);
 
-        LifecycleTransaction txn = null;
-        List<SSTableReader> toCompact = new ArrayList<>(sstables);
-        for (int i=0; i<sstables.size(); i++)
+        for (int mask = 1; mask < 16; ++mask) // test all combinations of two or more sstables with different repair marking
         {
-            try
-            {
-                txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
-                Assert.assertNotNull(txn);
-                CompactionTask task = new CompactionTask(cfs, txn, 0);
-                Assert.fail("Expected IllegalArgumentException");
-            }
-            catch (IllegalArgumentException e)
+            if (Integer.bitCount(mask) <= 1)
+                continue;
+
+            List<SSTableReader> toCompact = new ArrayList<>(sstables);
+            for (int i = 3; i >= 0; i--)
             {
-                // expected
+                if ((mask & (1<<i)) == 0)
+                    toCompact.remove(i);
             }
-            finally
+
+            LifecycleTransaction txn = null;
+            for (int order = 0; order < toCompact.size(); order++)
             {
-                if (txn != null)
-                    txn.abort();
+                try
+                {
+                    txn = cfs.getTracker().tryModify(toCompact, OperationType.COMPACTION);
+                    Assert.assertNotNull(txn);
+                    new CompactionTask(cfs, txn, 0, false, mockStrategy);
+                    Assert.fail("Expected IllegalArgumentException");
+                }
+                catch (IllegalArgumentException e)
+                {
+                    // expected
+                }
+                finally
+                {
+                    if (txn != null)
+                        txn.abort();
+                }
+                Collections.rotate(toCompact, 1);
             }
-            Collections.rotate(toCompact, 1);
         }
     }
 
     @Test
     public void testOfflineCompaction()
     {
-        cfs.getCompactionStrategyManager().disable();
+        cfs.getCompactionStrategyContainer().disable();
         QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);");
         Util.flush(cfs);
         QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);");
@@ -211,15 +290,21 @@ public void testOfflineCompaction()
         Set<SSTableReader> sstables = cfs.getLiveSSTables();
         Assert.assertEquals(4, sstables.size());
 
-        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, sstables))
+        Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
+        tracker.addInitialSSTables(sstables);
+        tracker.apply(updateCompacting(Collections.emptySet(), sstables));
+        try (LifecycleTransaction txn = new LifecycleTransaction(tracker,
+                                                                 OperationType.COMPACTION,
+                                                                 sstables,
+                                                                 LifecycleTransaction.newId()))
         {
-            Assert.assertEquals(4, txn.tracker.getView().liveSSTables().size());
-            CompactionTask task = new CompactionTask(cfs, txn, 1000);
-            task.execute(null);
+            Assert.assertEquals(4, tracker.getView().liveSSTables().size());
+            CompactionTask task = new CompactionTask(cfs, txn, 1000, false, null);
+            task.execute(new ActiveOperations());
 
             // Check that new SSTable was not released
-            Assert.assertEquals(1, txn.tracker.getView().liveSSTables().size());
-            SSTableReader newSSTable = txn.tracker.getView().liveSSTables().iterator().next();
+            Assert.assertEquals(1, tracker.getView().liveSSTables().size());
+            SSTableReader newSSTable = tracker.getView().liveSSTables().iterator().next();
             Assert.assertNotNull(newSSTable.tryRef());
         }
         finally
@@ -228,12 +313,75 @@ public void testOfflineCompaction()
             cfs.getTracker().removeUnsafe(sstables);
         }
     }
-    
+
     @Test
     public void testMajorCompactTask()
     {
-        //major compact without range/pk specified 
-        CompactionTasks compactionTasks = cfs.getCompactionStrategyManager().getMaximalTasks(Integer.MAX_VALUE, false, OperationType.MAJOR_COMPACTION);
+        //major compact without range/pk specified
+        CompactionTasks compactionTasks = cfs.getCompactionStrategyContainer().getMaximalTasks(Integer.MAX_VALUE, false, 0);
         Assert.assertTrue(compactionTasks.stream().allMatch(task -> task.compactionType.equals(OperationType.MAJOR_COMPACTION)));
     }
+
+    @Test
+    public void testCompactionReporting()
+    {
+        cfs.getCompactionStrategyContainer().disable();
+        Set<SSTableReader> sstables = generateData(2, 2);
+        LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+        assertNotNull(txn);
+        TableOperationObserver operationObserver = Mockito.mock(TableOperationObserver.class);
+        CompactionObserver compObserver = Mockito.mock(CompactionObserver.class);
+        final ArgumentCaptor<TableOperation> tableOpCaptor = ArgumentCaptor.forClass(AbstractTableOperation.class);
+        final ArgumentCaptor<CompactionProgress> compactionCaptor = ArgumentCaptor.forClass(CompactionProgress.class);
+        AbstractCompactionTask task = new CompactionTask(cfs, txn, 0, false, mockStrategy);
+        task.addObserver(compObserver);
+        assertNotNull(task);
+        task.execute(operationObserver);
+
+        verify(operationObserver, times(1)).onOperationStart(tableOpCaptor.capture());
+        verify(compObserver, times(1)).onInProgress(compactionCaptor.capture());
+        verify(compObserver, times(1)).onCompleted(eq(txn.opId()), eq(null));
+    }
+
+    @Test
+    public void testFailCompactionTask()
+    {
+        Set<SSTableReader> sstables = generateData(2, 2);
+        LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+        AbstractCompactionTask task = new CompactionTask(cfs, txn, 0, false, mockStrategy);
+        AbstractCompactionTask taskMock = Mockito.spy(task);
+        CompactionObserver compObserver = Mockito.mock(CompactionObserver.class);
+        taskMock.addObserver(compObserver);
+        Mockito.doThrow(new RuntimeException("Test throw")).when(taskMock).executeInternal();
+        Assert.assertThrows(RuntimeException.class, () ->  taskMock.execute());
+        Mockito.verify(compObserver, times(1)).onCompleted(any(TimeUUID.class), any(Throwable.class));
+    }
+
+    private Set<SSTableReader> generateData(int numSSTables, int numKeys)
+    {
+        for (int i = 0; i < numSSTables; i++)
+        {
+            for (int j = 0; j < numKeys; j++)
+                QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (?, ?);", j + i * numKeys, j + i * numKeys);
+
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        }
+
+        Set<SSTableReader> sstables = cfs.getLiveSSTables();
+        Assert.assertEquals(numSSTables, sstables.size());
+        return sstables;
+    }
+
+    static CompactionStrategy mockStrategy(ColumnFamilyStore cfs, boolean useCursors)
+    {
+        CompactionStrategy mock = Mockito.mock(CompactionStrategy.class);
+        CompactionLogger logger = new CompactionLogger(cfs.metadata());
+        Mockito.when(mock.supportsCursorCompaction()).thenReturn(useCursors);
+        Mockito.when(mock.getCompactionLogger()).thenReturn(logger);
+        Mockito.when(mock.getScanners(anyCollection()))
+               .thenAnswer(answ -> ScannerList.of(answ.getArgument(0), null));
+        Mockito.when(mock.getScanners(anyCollection(), any()))
+               .thenAnswer(answ -> ScannerList.of(answ.getArgument(0), answ.getArgument(1)));
+        return mock;
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
index d338c8b1690e..1e9842e6b9eb 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
@@ -20,10 +20,14 @@
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
@@ -34,6 +38,7 @@
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Throwables;
 import org.jboss.byteman.contrib.bmunit.BMRule;
@@ -44,9 +49,32 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+// TODO this test is broken since UCS as some of the BM rules became irrelevant and target not existing locations
 @RunWith(BMUnitRunner.class)
 public class CompactionsBytemanTest extends CQLTester
 {
+    @Before
+    public void setUp()
+    {
+        for (String ksname : Schema.instance.getKeyspaces())
+        {
+            for (ColumnFamilyStore cfs : Keyspace.open(ksname).getColumnFamilyStores())
+                cfs.disableAutoCompaction();
+        }
+    }
+
+    @After
+    public void tearDown()
+    {
+        while (STARTED != null && STARTED.getQueueLength() > 0)
+            STARTED.release();
+        STARTED = null;
+
+        while (PROCEED != null && PROCEED.getQueueLength() > 0)
+            PROCEED.release();
+        PROCEED = null;
+    }
+
     /*
     Return false for the first time hasAvailableDiskSpace is called. i.e first SSTable is too big
     Create 5 SSTables. After compaction, there should be 2 left - 1 as the 9 SStables which were merged,
@@ -126,15 +154,15 @@ public void testCompactingCFCounting() throws Throwable
     {
         createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c))");
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        cfs.enableAutoCompaction();
+        cfs.enableAutoCompaction(true);
 
         execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 1, 1);
-        Util.spinAssertEquals(true, () -> CompactionManager.instance.compactingCF.count(cfs) == 0, 5);
+        Util.spinAssertEquals(true, () -> CompactionManager.instance.getOngoingBackgroundCompactionsCount() == 0, 5);
         Util.flush(cfs);
 
-        Util.spinAssertEquals(true, () -> CompactionManager.instance.compactingCF.count(cfs) == 0, 5);
-        FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(cfs));
-        assertEquals(0, CompactionManager.instance.compactingCF.count(cfs));
+        Util.spinAssertEquals(true, () -> cfs.getCompactingSSTables().size() == 0, 5);
+        FBUtilities.waitOnFuture(CompactionManager.instance.submitBackground(cfs));
+        assertEquals(0, CompactionManager.instance.getOngoingBackgroundCompactionsCount());
     }
 
     private void createPossiblyExpiredSSTable(final ColumnFamilyStore cfs, final boolean expired) throws Throwable
@@ -157,10 +185,10 @@ private void createLowGCGraceTable(){
 
     @Test
     @BMRule(name = "Stop all compactions",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
+    targetClass = "CompactionTask$CompactionOperation",
+    targetMethod = "<init>",
     targetLocation = "AT INVOKE getCompactionAwareWriter",
-    action = "$ci.stop()")
+    action = "$this.op.stop(org.apache.cassandra.db.compaction.TableOperation$StopTrigger.UNIT_TESTS)")
     public void testStopUserDefinedCompactionRepaired() throws Throwable
     {
         testStopCompactionRepaired((cfs) -> {
@@ -171,10 +199,10 @@ public void testStopUserDefinedCompactionRepaired() throws Throwable
 
     @Test
     @BMRule(name = "Stop all compactions",
-    targetClass = "CompactionTask",
-    targetMethod = "runMayThrow",
+    targetClass = "CompactionTask$CompactionOperation",
+    targetMethod = "<init>",
     targetLocation = "AT INVOKE getCompactionAwareWriter",
-    action = "$ci.stop()")
+    action = "$this.op.stop(org.apache.cassandra.db.compaction.TableOperation$StopTrigger.UNIT_TESTS)")
     public void testStopSubRangeCompactionRepaired() throws Throwable
     {
         testStopCompactionRepaired((cfs) -> {
@@ -186,7 +214,8 @@ public void testStopSubRangeCompactionRepaired() throws Throwable
 
     public void testStopCompactionRepaired(Consumer<ColumnFamilyStore> compactionRunner) throws Throwable
     {
-        String table = createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c))");
+        String table = createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c)) WITH COMPACTION={'class':'SizeTieredCompactionStrategy'}");
+//        String table = createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c))");
         ColumnFamilyStore cfs = Keyspace.open(CQLTester.KEYSPACE).getColumnFamilyStore(table);
         cfs.disableAutoCompaction();
         for (int i = 0; i < 5; i++)
@@ -197,7 +226,7 @@ public void testStopCompactionRepaired(Consumer<ColumnFamilyStore> compactionRun
             }
             Util.flush(cfs);
         }
-        cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), System.currentTimeMillis(), null, false);
+        cfs.mutateRepaired(cfs.getLiveSSTables(), System.currentTimeMillis(), null, false);
         for (int i = 0; i < 5; i++)
         {
             for (int j = 0; j < 10; j++)
@@ -208,7 +237,7 @@ public void testStopCompactionRepaired(Consumer<ColumnFamilyStore> compactionRun
         }
 
         assertTrue(cfs.getTracker().getCompacting().isEmpty());
-        assertTrue(CompactionManager.instance.active.getCompactions().stream().noneMatch(h -> h.getCompactionInfo().getTableMetadata().equals(cfs.metadata)));
+        assertTrue(CompactionManager.instance.active.getTableOperations().stream().noneMatch(h -> h.getProgress().metadata().equals(cfs.metadata)));
 
         try
         {
@@ -223,7 +252,118 @@ public void testStopCompactionRepaired(Consumer<ColumnFamilyStore> compactionRun
         }
 
         assertTrue(cfs.getTracker().getCompacting().isEmpty());
-        assertTrue(CompactionManager.instance.active.getCompactions().stream().noneMatch(h -> h.getCompactionInfo().getTableMetadata().equals(cfs.metadata)));
+        assertTrue(CompactionManager.instance.active.getTableOperations().stream().noneMatch(h -> h.getProgress().metadata().equals(cfs.metadata)));
+
+    }
+
+    static Semaphore STARTED;
+    static Semaphore PROCEED;
 
+    @Test
+    @BMRule(name = "Delay compaction task execution",
+            targetClass = "AbstractCompactionTask",
+            targetMethod = "execute()",
+            action = "org.apache.cassandra.db.compaction.CompactionsBytemanTest.STARTED.release();\n" +
+                     "org.apache.cassandra.db.compaction.CompactionsBytemanTest.PROCEED.acquireUninterruptibly();")
+    public void testCompactionReloadDoesNotLoseHistory() throws Throwable
+    {
+        STARTED = new Semaphore(0);
+        PROCEED = new Semaphore(0);
+
+        try
+        {
+            createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c)) WITH COMPACTION={'class': 'UnifiedCompactionStrategy'}");
+            ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+
+            for (int i = 0; i < 4; ++i)
+            {
+                execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 1, 1);
+                cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+            }
+
+            // This is probably already started when we flushed the 4th sstable, but let's make sure.
+            CompactionManager.instance.submitBackground(cfs);
+            STARTED.acquireUninterruptibly();
+            List<CompactionStrategyStatistics> statistics = cfs.getCompactionStrategy().getStatistics();
+            assertEquals(1, statistics.size());
+            assertEquals(1, statistics.get(0).aggregates().size());
+
+            execute("ALTER TABLE %s WITH COMPACTION={'class': 'UnifiedCompactionStrategy', 'scaling_parameters': '4'}");
+            statistics = cfs.getCompactionStrategy().getStatistics();
+            assertEquals(1, statistics.size());
+            assertEquals(1, statistics.get(0).aggregates().size());
+        }
+        finally
+        {
+            // allow the task to continue
+            PROCEED.release();
+            dropTable("DROP TABLE %s");
+        }
+    }
+
+    @Test
+    @BMRule(name = "Delay compaction task execution",
+            targetClass = "AbstractCompactionTask",
+            targetMethod = "execute()",
+            action = "org.apache.cassandra.db.compaction.CompactionsBytemanTest.STARTED.release();\n" +
+                     "org.apache.cassandra.db.compaction.CompactionsBytemanTest.PROCEED.acquireUninterruptibly();")
+    public void testTotalCompactionsLCS() throws Throwable
+    {
+        testTotalCompactions("{'class': 'LeveledCompactionStrategy'}");
+    }
+
+    @BMRule(name = "Delay compaction task execution",
+            targetClass = "AbstractCompactionTask",
+            targetMethod = "execute()",
+            action = "org.apache.cassandra.db.compaction.CompactionsBytemanTest.STARTED.release();\n" +
+                     "org.apache.cassandra.db.compaction.CompactionsBytemanTest.PROCEED.acquireUninterruptibly();")
+    @Test
+    public void testTotalCompactionsSTCS() throws Throwable
+    {
+        testTotalCompactions("{'class': 'SizeTieredCompactionStrategy'}");
+    }
+
+    @Test
+    @BMRule(name = "Delay compaction task execution",
+            targetClass = "AbstractCompactionTask",
+            targetMethod = "execute()",
+            action = "org.apache.cassandra.db.compaction.CompactionsBytemanTest.STARTED.release();\n" +
+                     "org.apache.cassandra.db.compaction.CompactionsBytemanTest.PROCEED.acquireUninterruptibly();")
+    public void testTotalCompactionsUCS() throws Throwable
+    {
+        testTotalCompactions("{'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 1, 'base_shard_count': 1}");
+    }
+
+    private void testTotalCompactions(String compactionOption) throws Throwable
+    {
+        STARTED = new Semaphore(0);
+        PROCEED = new Semaphore(0);
+
+        try
+        {
+            createTable("CREATE TABLE %s (k INT, c INT, v INT, PRIMARY KEY (k, c)) WITH COMPACTION=" + compactionOption);
+            ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+            cfs.disableAutoCompaction();
+            int numSSTables = 10;
+            for (int i = 0; i < numSSTables; i++)
+            {
+                // Write more than one key to ensure overlap.
+                execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", i, 1, 1);
+                execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", i + 1, 1, 1);
+                execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", i + 2, 1, 1);
+                cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+            }
+            assertEquals(numSSTables, cfs.getLiveSSTables().size());
+
+            cfs.enableAutoCompaction(false);
+            STARTED.acquireUninterruptibly();
+            assertEquals(1, cfs.getCompactionStrategy().getTotalCompactions());
+        }
+        finally
+        {
+            // allow the task to continue
+            PROCEED.release();
+            dropTable("DROP TABLE %s");
+        }
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
index ca9f5bbcd16a..711aaa6e8f7e 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
@@ -22,16 +22,15 @@
 import java.nio.file.FileStore;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
-
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
@@ -56,11 +55,12 @@
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.LegacySSTableTest;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.PathUtils;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 import org.assertj.core.api.Assertions;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -79,6 +79,13 @@ public class CompactionsCQLTest extends CQLTester
     private static File testSStablesDir = new File(NEGATIVE_LDTS_INVALID_DELETES_TEST_DIR);
 
 
+    @BeforeClass
+    public static void beforeClass()
+    {
+        CQLTester.setUpClass();
+        StorageService.instance.initServer();
+    }
+
     @Before
     public void before() throws IOException
     {
@@ -97,7 +104,7 @@ public void after()
     public void testTriggerMinorCompactionSTCS() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY)  WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold':2};");
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("insert into %s (id) values ('1')");
         flush();
         execute("insert into %s (id) values ('1')");
@@ -109,7 +116,7 @@ public void testTriggerMinorCompactionSTCS() throws Throwable
     public void testTriggerMinorCompactionLCS() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY) WITH compaction = {'class':'LeveledCompactionStrategy', 'sstable_size_in_mb':1, 'fanout_size':5};");
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("insert into %s (id) values ('1')");
         flush();
         execute("insert into %s (id) values ('1')");
@@ -121,7 +128,7 @@ public void testTriggerMinorCompactionLCS() throws Throwable
     public void testTriggerMinorCompactionTWCS() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY) WITH compaction = {'class':'TimeWindowCompactionStrategy', 'min_threshold':2};");
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("insert into %s (id) values ('1')");
         flush();
         execute("insert into %s (id) values ('1')");
@@ -134,7 +141,7 @@ public void testTriggerMinorCompactionTWCS() throws Throwable
     public void testTriggerNoMinorCompactionSTCSDisabled() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY)  WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold':2, 'enabled':false};");
-        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("insert into %s (id) values ('1')");
         flush();
         execute("insert into %s (id) values ('1')");
@@ -146,14 +153,13 @@ public void testTriggerNoMinorCompactionSTCSDisabled() throws Throwable
     public void testTriggerMinorCompactionSTCSNodetoolEnabled() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY)  WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold':2, 'enabled':false};");
-        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         getCurrentColumnFamilyStore().enableAutoCompaction();
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
 
         // Alter keyspace replication settings to force compaction strategy reload and check strategy is still enabled
         execute("alter keyspace "+keyspace()+" with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 }");
-        getCurrentColumnFamilyStore().getCompactionStrategyManager().maybeReloadDiskBoundaries();
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
 
         execute("insert into %s (id) values ('1')");
         flush();
@@ -166,9 +172,9 @@ public void testTriggerMinorCompactionSTCSNodetoolEnabled() throws Throwable
     public void testTriggerNoMinorCompactionSTCSNodetoolDisabled() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY)  WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold':2, 'enabled':true};");
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         getCurrentColumnFamilyStore().disableAutoCompaction();
-        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("insert into %s (id) values ('1')");
         flush();
         execute("insert into %s (id) values ('1')");
@@ -180,9 +186,9 @@ public void testTriggerNoMinorCompactionSTCSNodetoolDisabled() throws Throwable
     public void testTriggerNoMinorCompactionSTCSAlterTable() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY)  WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold':2, 'enabled':true};");
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("ALTER TABLE %s WITH compaction = {'class': 'SizeTieredCompactionStrategy', 'enabled': false}");
-        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("insert into %s (id) values ('1')");
         flush();
         execute("insert into %s (id) values ('1')");
@@ -194,9 +200,9 @@ public void testTriggerNoMinorCompactionSTCSAlterTable() throws Throwable
     public void testTriggerMinorCompactionSTCSAlterTable() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY)  WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold':2, 'enabled':false};");
-        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("ALTER TABLE %s WITH compaction = {'class': 'SizeTieredCompactionStrategy', 'min_threshold': 2, 'enabled': true}");
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         execute("insert into %s (id) values ('1')");
         flush();
         execute("insert into %s (id) values ('1')");
@@ -205,7 +211,7 @@ public void testTriggerMinorCompactionSTCSAlterTable() throws Throwable
     }
 
     @Test
-    public void testSetLocalCompactionStrategy() throws Throwable
+    public void testSetLocalCompactionStrategySTCS() throws Throwable
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY)");
         testSetLocalCompactionStrategy(SizeTieredCompactionStrategy.class);
@@ -217,27 +223,27 @@ public void testSetLocalCompactionStrategyUCS() throws Throwable
         testSetLocalCompactionStrategy(UnifiedCompactionStrategy.class);
     }
 
-    private void testSetLocalCompactionStrategy(Class<? extends AbstractCompactionStrategy> strategy) throws Throwable
+    private void testSetLocalCompactionStrategy(Class<? extends CompactionStrategy> strategy) throws Throwable
     {
         createTable(String.format("CREATE TABLE %%s (id text PRIMARY KEY) with compaction = {'class': '%s'}", strategy.getSimpleName()));
         Map<String, String> localOptions = new HashMap<>();
         localOptions.put("class", "SizeTieredCompactionStrategy");
         getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
-        assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), SizeTieredCompactionStrategy.class));
+        assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyContainer(), SizeTieredCompactionStrategy.class));
         // Invalidate disk boundaries to ensure that boundary invalidation will not cause the old strategy to be reloaded
-        getCurrentColumnFamilyStore().invalidateLocalRanges();
+        getCurrentColumnFamilyStore().invalidateLocalRangesAndDiskBoundaries();
         // altering something non-compaction related
         execute("ALTER TABLE %s WITH gc_grace_seconds = 1000");
         // should keep the local compaction strat
-        assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), SizeTieredCompactionStrategy.class));
+        assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyContainer(), SizeTieredCompactionStrategy.class));
         // Alter keyspace replication settings to force compaction strategy reload
         execute("alter keyspace "+keyspace()+" with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 }");
         // should keep the local compaction strat
-        assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), SizeTieredCompactionStrategy.class));
+        assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyContainer(), SizeTieredCompactionStrategy.class));
         // altering a compaction option
         execute("ALTER TABLE %s WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold': 3}");
         // will use the new option
-        assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), SizeTieredCompactionStrategy.class));
+        assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyContainer(), SizeTieredCompactionStrategy.class));
     }
 
     @Test
@@ -248,12 +254,12 @@ public void testSetLocalCompactionStrategyDisable() throws Throwable
         localOptions.put("class", "SizeTieredCompactionStrategy");
         localOptions.put("enabled", "false");
         getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
-        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
         localOptions.clear();
         localOptions.put("class", "SizeTieredCompactionStrategy");
         // localOptions.put("enabled", "true"); - this is default!
         getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
     }
 
     @Test
@@ -264,10 +270,10 @@ public void testSetLocalCompactionStrategyEnable() throws Throwable
         localOptions.put("class", "LeveledCompactionStrategy");
 
         getCurrentColumnFamilyStore().disableAutoCompaction();
-        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
 
         getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
-        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
+        assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyContainer().isEnabled());
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -490,11 +496,14 @@ public void testLCSThresholdParams() throws Throwable
             Util.flush(cfs);
         }
         assertEquals(50, cfs.getLiveSSTables().size());
-        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
-        AbstractCompactionTask act = lcs.getNextBackgroundTask(0);
+        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) ((CompactionStrategyManager) cfs.getCompactionStrategyContainer())
+                                                                    .getUnrepairedUnsafe().first();
+        Collection<AbstractCompactionTask> tasks = lcs.getNextBackgroundTasks(0);
+        assertEquals(1, tasks.size());
+        AbstractCompactionTask act = tasks.iterator().next();
         // we should be compacting all 50 sstables:
         assertEquals(50, act.transaction.originals().size());
-        act.execute(ActiveCompactionsTracker.NOOP);
+        act.execute();
     }
 
     @Test
@@ -524,14 +533,79 @@ public void testSTCSinL0() throws Throwable
 
         // mark the L1 sstable as compacting to make sure we trigger STCS in L0:
         LifecycleTransaction txn = cfs.getTracker().tryModify(l1sstable, OperationType.COMPACTION);
-        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
-        AbstractCompactionTask act = lcs.getNextBackgroundTask(0);
+        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) ((CompactionStrategyManager) cfs.getCompactionStrategyContainer())
+                                                                    .getUnrepairedUnsafe()
+                                                                    .first();
+        Collection<AbstractCompactionTask> tasks = lcs.getNextBackgroundTasks(0);
+        assertEquals(1, tasks.size());
+        AbstractCompactionTask act = tasks.iterator().next();
         // note that max_threshold is 60 (more than the amount of L0 sstables), but MAX_COMPACTING_L0 is 32, which means we will trigger STCS with at most max_threshold sstables
         assertEquals(50, act.transaction.originals().size());
         assertEquals(0, ((LeveledCompactionTask)act).getLevel());
         assertTrue(act.transaction.originals().stream().allMatch(s -> s.getSSTableLevel() == 0));
         txn.abort(); // unmark the l1 sstable compacting
-        act.execute(ActiveCompactionsTracker.NOOP);
+        act.execute();
+    }
+
+    @Test
+    public void testABAReloadUCS()
+    {
+        testABAReload(UnifiedCompactionStrategy.class);
+    }
+
+    @Test
+    public void testABAReloadSTCS()
+    {
+        testABAReload(SizeTieredCompactionStrategy.class);
+    }
+
+    @Test
+    public void testABAReloadLCS()
+    {
+        testABAReload(LeveledCompactionStrategy.class);
+    }
+
+    private void testABAReload(Class<? extends CompactionStrategy> strategyClass)
+    {
+        createTable(String.format("CREATE TABLE %%s (id text PRIMARY KEY) WITH compaction = {'class':'%s'};", strategyClass.getSimpleName()));
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        assertEquals(strategyClass, cfs.getCompactionStrategyContainer().getCompactionParams().klass());
+        alterTable("ALTER TABLE %s WITH compaction = {'class': 'TimeWindowCompactionStrategy'}");
+        assertEquals(TimeWindowCompactionStrategy.class, cfs.getCompactionStrategyContainer().getCompactionParams().klass());
+        alterTable(String.format("ALTER TABLE %%s WITH compaction = {'class': '%s'}", strategyClass.getSimpleName()));
+        assertEquals(strategyClass, cfs.getCompactionStrategyContainer().getCompactionParams().klass());
+    }
+
+    @Test
+    public void testWithSecondaryIndexUCS() throws Throwable
+    {
+        testWithSecondaryIndex(UnifiedCompactionStrategy.class);
+    }
+
+    @Test
+    public void testWithSecondaryIndexSTCS() throws Throwable
+    {
+        testWithSecondaryIndex(SizeTieredCompactionStrategy.class);
+    }
+
+    @Test
+    public void testWithSecondaryIndexLCS() throws Throwable
+    {
+        testWithSecondaryIndex(LeveledCompactionStrategy.class);
+    }
+
+    public void testWithSecondaryIndex(Class<? extends CompactionStrategy> strategyClass) throws Throwable
+    {
+        createTable(String.format("CREATE TABLE %%s (pk int, c int, s int static, v int, PRIMARY KEY(pk, c)) WITH compaction = {'class':'%s'};", strategyClass.getSimpleName()));
+        createIndex("CREATE INDEX ON %s (v)");
+
+        execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 1, 9, 1);
+        execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 2, 9, 2);
+        execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 3, 1, 9, 1);
+        execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 4, 1, 9, 1);
+        flush();
+
+        compact();
     }
 
     @Test
@@ -554,13 +628,16 @@ public void testAbortNotifications() throws Throwable
         }
         getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
 
-        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) getCurrentColumnFamilyStore().getCompactionStrategyManager().getUnrepairedUnsafe().first();
+        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) ((CompactionStrategyManager) getCurrentColumnFamilyStore().getCompactionStrategyContainer())
+                                                                    .getUnrepairedUnsafe()
+                                                                    .first();
         LeveledCompactionTask lcsTask;
         while (true)
         {
-            lcsTask = (LeveledCompactionTask) lcs.getNextBackgroundTask(0);
-            if (lcsTask != null)
+            Collection<AbstractCompactionTask> tasks = lcs.getNextBackgroundTasks(0);
+            if (tasks.size() > 0)
             {
+                lcsTask = (LeveledCompactionTask) tasks.iterator().next();
                 lcsTask.execute(CompactionManager.instance.active);
                 break;
             }
@@ -595,7 +672,9 @@ public void testAbortNotifications() throws Throwable
         // sstables have been removed.
         try
         {
-            AbstractCompactionTask task = new NotifyingCompactionTask((LeveledCompactionTask) lcs.getNextBackgroundTask(0));
+            Collection<AbstractCompactionTask> tasks = lcs.getNextBackgroundTasks(0);
+            assertEquals(1, tasks.size());
+            AbstractCompactionTask task = new NotifyingCompactionTask(lcs, (LeveledCompactionTask) tasks.iterator().next());
             task.execute(CompactionManager.instance.active);
             fail("task should throw exception");
         }
@@ -604,42 +683,33 @@ public void testAbortNotifications() throws Throwable
             // ignored
         }
 
-        lcsTask = (LeveledCompactionTask) lcs.getNextBackgroundTask(0);
-        try
-        {
-            assertNotNull(lcsTask);
-        }
-        finally
-        {
-            if (lcsTask != null)
-                lcsTask.transaction.abort();
-        }
+        Collection<AbstractCompactionTask> tasks = lcs.getNextBackgroundTasks(0);
+        assertEquals(1, tasks.size());
+        lcsTask = (LeveledCompactionTask) tasks.iterator().next();
+        lcsTask.transaction.abort();
     }
 
     private static class NotifyingCompactionTask extends LeveledCompactionTask
     {
-        public NotifyingCompactionTask(LeveledCompactionTask task)
+        public NotifyingCompactionTask(LeveledCompactionStrategy lcs, LeveledCompactionTask task)
         {
-            super(task.cfs, task.transaction, task.getLevel(), task.gcBefore, task.getLevel(), false);
+            super(lcs, task.transaction, task.getLevel(), task.gcBefore, task.getLevel(), false);
         }
 
         @Override
-        public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs,
+        public CompactionAwareWriter getCompactionAwareWriter(CompactionRealm realm,
                                                               Directories directories,
-                                                              LifecycleTransaction txn,
                                                               Set<SSTableReader> nonExpiredSSTables)
         {
-            return new MaxSSTableSizeWriter(cfs, directories, txn, nonExpiredSSTables, 1 << 20, 1)
+            return new MaxSSTableSizeWriter(realm, directories, transaction, nonExpiredSSTables, 1 << 20, 1)
             {
                 int switchCount = 0;
-
-                @Override
-                public SSTableWriter sstableWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
+                public void switchCompactionWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
                 {
                     switchCount++;
                     if (switchCount > 5)
                         throw new RuntimeException("Throw after a few sstables have had their starts moved");
-                    return super.sstableWriter(directory, nextKey);
+                    super.switchCompactionWriter(directory, nextKey);
                 }
             };
         }
@@ -826,16 +896,15 @@ public void testProvidesTombstoneOptionverifiation()
          localOptions.put("provide_overlapping_tombstones","row");
 
          getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
-         assertEquals(CompactionParams.TombstoneOption.ROW, getCurrentColumnFamilyStore().getCompactionStrategyManager().getCompactionParams().tombstoneOption());
+         assertEquals(CompactionParams.TombstoneOption.ROW, getCurrentColumnFamilyStore().getCompactionParams().tombstoneOption());
      }
 
-
-    public boolean verifyStrategies(CompactionStrategyManager manager, Class<? extends AbstractCompactionStrategy> expected)
+    public boolean verifyStrategies(CompactionStrategyContainer strategyContainer, Class<? extends AbstractCompactionStrategy> expected)
     {
         boolean found = false;
-        for (List<AbstractCompactionStrategy> strategies : manager.getStrategies())
+        for (CompactionStrategy strategy : strategyContainer.getStrategies())
         {
-            if (!strategies.stream().allMatch((strategy) -> strategy.getClass().equals(expected)))
+            if (!strategy.getClass().equals(expected))
                 return false;
             found = true;
         }
@@ -872,9 +941,8 @@ public void testNoDiskspace() throws Throwable
             execute("insert into %s (id, i) values (?,?)", i, i);
             getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
         }
-        CompactionInfo.Holder holder = holder(OperationType.COMPACTION);
-        CompactionManager.instance.active.beginCompaction(holder);
-        try
+        AbstractTableOperation holder = holder(OperationType.COMPACTION);
+        try (NonThrowingCloseable c = CompactionManager.instance.active.onOperationStart(holder))
         {
             getCurrentColumnFamilyStore().forceMajorCompaction();
             fail("Exception expected");
@@ -883,39 +951,30 @@ public void testNoDiskspace() throws Throwable
         {
             // expected
         }
-        finally
-        {
-            CompactionManager.instance.active.finishCompaction(holder);
-        }
         // don't block compactions if there is a huge validation
         holder = holder(OperationType.VALIDATION);
-        CompactionManager.instance.active.beginCompaction(holder);
-        try
+        try (NonThrowingCloseable c = CompactionManager.instance.active.onOperationStart(holder))
         {
             getCurrentColumnFamilyStore().forceMajorCompaction();
         }
-        finally
-        {
-            CompactionManager.instance.active.finishCompaction(holder);
-        }
     }
 
-    private CompactionInfo.Holder holder(OperationType opType)
+    private AbstractTableOperation holder(OperationType opType)
     {
-        CompactionInfo.Holder holder = new CompactionInfo.Holder()
+        AbstractTableOperation holder = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
                 long availableSpace = 0;
                 for (File f : getCurrentColumnFamilyStore().getDirectories().getCFDirectories())
                     availableSpace += PathUtils.tryGetSpace(f.toPath(), FileStore::getUsableSpace);
 
-                return new CompactionInfo(getCurrentColumnFamilyStore().metadata(),
-                                          opType,
-                                          +0,
-                                          +availableSpace * 2,
-                                          nextTimeUUID(),
-                                          getCurrentColumnFamilyStore().getLiveSSTables());
+                return new OperationProgress(getCurrentColumnFamilyStore().metadata(), 
+                                             opType,                                               
+                                             +0,                                               
+                                             +availableSpace * 2, 
+                                             nextTimeUUID(),                                                                                                                   
+                                             getCurrentColumnFamilyStore().getLiveSSTables());
             }
 
             public boolean isGlobal()
@@ -926,6 +985,16 @@ public boolean isGlobal()
         return holder;
     }
 
+    @Test
+    public void testPeriodicCompactionsCall()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY) WITH compaction = {'class': 'TestCompactionClass'}");
+        TestCompactionClass cs = (TestCompactionClass) getCurrentColumnFamilyStore().getCompactionStrategyContainer().getStrategies().get(0);
+        int prCount = cs.periodicReportsCalled;
+        CompactionManager.periodicReports();
+        assertTrue(cs.periodicReportsCalled > prCount);
+    }
+
     private void loadTestSStables(ColumnFamilyStore cfs, File ksDir) throws IOException
     {
         Keyspace.open(cfs.getKeyspaceName()).getColumnFamilyStore(cfs.name).truncateBlocking();
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
index 4f1b639d7081..bb6cb8edecf2 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
@@ -31,7 +31,7 @@
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.exceptions.ConfigurationException;
@@ -123,7 +123,7 @@ public void testMajorCompactionPurge()
         FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false));
         cfs.invalidateCachedPartition(dk(key));
 
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
         assertEquals(1, partition.rowCount());
     }
 
@@ -168,7 +168,7 @@ public void testMajorCompactionPurgeTombstonesWithMaxTimestamp()
 
         cfs.invalidateCachedPartition(dk(key));
 
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
         assertEquals(1, partition.rowCount());
     }
 
@@ -212,7 +212,7 @@ public void testMajorCompactionPurgeTopLevelTombstoneWithMaxTimestamp()
 
         cfs.invalidateCachedPartition(dk(key));
 
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
         assertEquals(1, partition.rowCount());
     }
 
@@ -254,7 +254,7 @@ public void testMajorCompactionPurgeRangeTombstoneWithMaxTimestamp()
 
         cfs.invalidateCachedPartition(dk(key));
 
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
         assertEquals(1, partition.rowCount());
     }
 
@@ -303,9 +303,9 @@ public void testMinorCompactionPurge()
                 .build().applyUnsafe();
 
         Util.flush(cfs);
-        try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE))
+        try (CompactionTasks tasks = cfs.getCompactionStrategyContainer().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE))
         {
-            Iterables.getOnlyElement(tasks).execute(ActiveCompactionsTracker.NOOP);
+            Iterables.getOnlyElement(tasks).execute();
         }
 
         // verify that minor compaction does GC when key is provably not
@@ -314,7 +314,7 @@ public void testMinorCompactionPurge()
 
         // verify that minor compaction still GC when key is present
         // in a non-compacted sstable but the timestamp ensures we won't miss anything
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key1).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key1).build());
         assertEquals(1, partition.rowCount());
     }
 
@@ -355,16 +355,16 @@ public void testMinTimestampPurge()
         Util.flush(cfs);
 
         // compact the sstables with the c1/c2 data and the c1 tombstone
-        try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE))
+        try (CompactionTasks tasks = cfs.getCompactionStrategyContainer().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE))
         {
-            Iterables.getOnlyElement(tasks).execute(ActiveCompactionsTracker.NOOP);
+            Iterables.getOnlyElement(tasks).execute();
         }
 
         // We should have both the c1 and c2 tombstones still. Since the min timestamp in the c2 tombstone
         // sstable is older than the c1 tombstone, it is invalid to throw out the c1 tombstone.
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key3).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key3).build());
         assertEquals(2, partition.rowCount());
-        for (Row row : partition)
+        for (Row row : partition.rows())
             assertFalse(row.hasLiveData(FBUtilities.nowInSeconds(), enforceStrictLiveness));
     }
 
@@ -470,7 +470,7 @@ public void testCompactionPurgeTombstonedRow() throws ExecutionException, Interr
         rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata(), dk(key), 4, FBUtilities.nowInSeconds()));
         rm.build().applyUnsafe();
 
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
         assertFalse(partition.partitionLevelDeletion().isLive());
 
         // flush and major compact (with tombstone purging)
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index 79f01f9a59bb..b7c0d1830f22 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -22,9 +22,14 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import org.apache.cassandra.utils.TimeUUID;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -51,7 +56,7 @@
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.marshal.ValueAccessors;
 import org.apache.cassandra.db.partitions.FilteredPartition;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
@@ -70,15 +75,25 @@
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MockSchema;
 import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NonThrowingCloseable;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.mockito.Mockito;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class CompactionsTest
 {
@@ -155,7 +170,7 @@ public void testSingleSSTableCompaction() throws Exception
 
         // enable compaction, submit background and wait for it to complete
         store.enableAutoCompaction();
-        FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
+        FBUtilities.waitOnFuture(CompactionManager.instance.submitBackground(store));
         do
         {
             TimeUnit.SECONDS.sleep(1);
@@ -207,7 +222,7 @@ public void testUncheckedTombstoneSizeTieredCompaction() throws Exception
 
         // enable compaction, submit background and wait for it to complete
         store.enableAutoCompaction();
-        FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
+        FBUtilities.waitOnFuture(CompactionManager.instance.submitBackground(store));
         do
         {
             TimeUnit.SECONDS.sleep(1);
@@ -229,7 +244,7 @@ public void testUncheckedTombstoneSizeTieredCompaction() throws Exception
         SchemaTestUtil.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build());
 
         //submit background task again and wait for it to complete
-        FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
+        FBUtilities.waitOnFuture(CompactionManager.instance.submitBackground(store));
         do
         {
             TimeUnit.SECONDS.sleep(1);
@@ -398,7 +413,7 @@ private void testDontPurgeAccidentally(String k, String cfname) throws Interrupt
 
         Collection<SSTableReader> sstablesBefore = cfs.getLiveSSTables();
 
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
         assertTrue(!partition.isEmpty());
 
         RowUpdateBuilder deleteRowBuilder = new RowUpdateBuilder(table, 2, key);
@@ -407,7 +422,7 @@ private void testDontPurgeAccidentally(String k, String cfname) throws Interrupt
         // Remove key
 
         partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
-        assertTrue(partition.iterator().next().cells().iterator().next().isTombstone());
+        assertTrue(partition.rowIterator().next().cells().iterator().next().isTombstone());
 
         // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
         Thread.sleep(1000);
@@ -572,4 +587,116 @@ public void testConcurrencySettings()
         CompactionManager.instance.setConcurrentCompactors(1);
         assertEquals(1, CompactionManager.instance.getCoreCompactorThreads());
     }
-}
\ No newline at end of file
+
+    @Test
+    public void testCompactionsCanBeInterrupted() throws Exception
+    {
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
+        store.clearUnsafe();
+
+        // disable compaction while flushing
+        store.disableAutoCompaction();
+
+        // Write a bit of data
+        for (int j = 0; j < 2; j++)
+        {
+            for (int i = 1; i < 100; i++)
+            {
+                new RowUpdateBuilder(store.metadata(), 0, ByteBufferUtil.bytes("key" + i))
+                .clustering("Column1")
+                .add("val", ByteBufferUtil.bytes("abcd"))
+                .build()
+                .apply();
+            }
+
+            store.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        }
+
+        assertTrue(store.getLiveSSTables().size() >= 2);
+
+        // Enable compaction but do not submit any background compactions
+        store.getCompactionStrategyContainer().enable();
+
+        CountDownLatch compactionRegistered = new CountDownLatch(1);
+        CountDownLatch resumeCompaction = new CountDownLatch(1);
+        TableOperationObserver obs = Mockito.mock(TableOperationObserver.class);
+
+        when(obs.onOperationStart(any(TableOperation.class))).thenAnswer(invocation -> {
+            NonThrowingCloseable ret = CompactionManager.instance.active.onOperationStart(invocation.getArgument(0));
+            compactionRegistered.countDown(); // this makes sure we don't attempt to interrupt a compaction before it has registered
+            resumeCompaction.await(); // this will block the compaction just after it has registered so that we can interrupt it before it even starts
+            return ret;
+        });
+
+        List<Future<?>> compactions = CompactionManager.instance.submitMaximal(store, FBUtilities.nowInSeconds(), false, obs, OperationType.MAJOR_COMPACTION);
+        assertEquals("Expected one compaction to be submitted", 1, compactions.size());
+
+        // Wait for compaction to register with its operation observer (the metrics)
+        compactionRegistered.await(1, TimeUnit.MINUTES);
+
+        // Interrupt the compaction, this only works if CompactionManager.instance.active.onOperationStart() has already been called
+        boolean ret = CompactionManager.instance.interruptCompactionFor(ImmutableList.of(store.metadata()), TableOperation.StopTrigger.UNIT_TESTS);
+        assertTrue("Compaction should have been interrupted", ret);
+
+        // Let the compaction continue running
+        resumeCompaction.countDown();
+
+        // Make sure the compactions was interrupted
+        try
+        {
+            compactions.get(0).get();
+            fail("Compaction should have been interrupted");
+        }
+        catch(Throwable t)
+        {
+            t = Throwables.getRootCause(t);
+            assertTrue(t.getMessage(), t instanceof CompactionInterruptedException);
+        }
+    }
+
+    @Test
+    public void testCompactionListener()
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS();
+        cfs.addSSTable(MockSchema.sstable(1, true, cfs));
+        ActiveOperations.CompactionProgressListener listener = Mockito.mock(ActiveOperations
+                                                                            .CompactionProgressListener.class);
+        TableOperation.Progress progress = new AbstractTableOperation
+                                               .OperationProgress(cfs.metadata(),
+                                                                  OperationType.ANTICOMPACTION,
+                                                                  0,
+                                                                  0,
+                                                                  TimeUUID.Generator.nextTimeUUID(),
+                                                                  cfs.getLiveSSTables());
+
+        AbstractTableOperation operation = new AbstractTableOperation()
+        {
+            public Progress getProgress()
+            {
+                return progress;
+            }
+
+            public boolean isGlobal()
+            {
+                return false;
+            }
+        };
+        CompactionManager.instance.active.registerListener(listener);
+
+        try (NonThrowingCloseable cls = CompactionManager.instance.active.onOperationStart(operation))
+        {
+            verify(listener).onStarted(progress);
+            verify(listener, never()).onCompleted(progress);
+        }
+        verify(listener, times(1)).onStarted(progress);
+        verify(listener, times(1)).onCompleted(progress);
+
+        try (NonThrowingCloseable cls = CompactionManager.instance.active.onOperationStart(operation))
+        {
+            CompactionManager.instance.active.unregisterListener(listener);
+        }
+        verify(listener, times(2)).onStarted(progress);
+        verify(listener, times(1)).onCompleted(progress);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompositeCompactionTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/CompositeCompactionTaskTest.java
new file mode 100644
index 000000000000..e583f81ea145
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CompositeCompactionTaskTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.utils.TimeUUID;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class CompositeCompactionTaskTest
+{
+    private CompactionRealm mockRealm;
+    private LifecycleTransaction mockTransaction;
+    private CompositeCompactionTask compositeCompactionTask;
+    private AbstractCompactionTask mockTask1;
+    private AbstractCompactionTask mockTask2;
+    private TableOperationObserver mockOpObserver;
+
+    @Before
+    public void setUp() {
+        mockRealm = Mockito.mock(CompactionRealm.class);
+        mockTransaction = Mockito.mock(LifecycleTransaction.class);
+        when(mockTransaction.isOffline()).thenReturn(true);
+        when(mockTransaction.opId()).thenReturn(TimeUUID.Generator.nextTimeUUID());
+        when(mockRealm.tryModify(any(), any(), any())).thenReturn(mockTransaction);
+
+        mockTask1 = Mockito.mock(AbstractCompactionTask.class, Mockito.withSettings().useConstructor(mockRealm, mockTransaction));
+        mockTask2 = Mockito.mock(AbstractCompactionTask.class, Mockito.withSettings().useConstructor(mockRealm, mockTransaction));
+        mockOpObserver = Mockito.mock(TableOperationObserver.class);
+        compositeCompactionTask = CompositeCompactionTask.combineTasks(mockTask1, mockTask2);
+        compositeCompactionTask.setOpObserver(mockOpObserver);
+    }
+
+    @Test
+    public void testExecute() {
+        // Testing executeInternal() instead of execute() because we cannot mock transaction.close()
+        compositeCompactionTask.executeInternal();
+        verify(mockTask1, times(1)).execute(mockOpObserver);
+        verify(mockTask2, times(1)).execute(mockOpObserver);
+    }
+
+    @Test
+    public void testRejected() {
+        compositeCompactionTask.rejected(null);
+        verify(mockTask1, times(1)).rejected(null);
+        verify(mockTask2, times(1)).rejected(null);
+    }
+
+    @Test
+    public void testSetUserDefined() {
+        compositeCompactionTask.setUserDefined(true);
+        verify(mockTask1, times(1)).setUserDefined(true);
+        verify(mockTask2, times(1)).setUserDefined(true);
+    }
+
+    @Test
+    public void testSetCompactionType() {
+        OperationType compactionType = OperationType.COMPACTION;
+        compositeCompactionTask.setCompactionType(compactionType);
+        verify(mockTask1, times(1)).setCompactionType(compactionType);
+        verify(mockTask2, times(1)).setCompactionType(compactionType);
+    }
+
+    @Test
+    public void testAddObserver() {
+        CompactionObserver compObserver = Mockito.mock(CompactionObserver.class);
+        compositeCompactionTask.addObserver(compObserver);
+        verify(mockTask1, times(1)).addObserver(compObserver);
+        verify(mockTask2, times(1)).addObserver(compObserver);
+    }
+
+    @Test
+    public void testExecuteWithException() {
+        doThrow(new RuntimeException("Test Exception")).when(mockTask1).execute(Mockito.any());
+        assertThrows(RuntimeException.class, () -> compositeCompactionTask.executeInternal());
+        verify(mockTask1, times(1)).execute(mockOpObserver);
+        verify(mockTask2, times(1)).execute(mockOpObserver);
+    }
+
+    @Test
+    public void testApplyParallelismLimit_NoLimit() {
+        testApplyParallelismLimit(3, 0);
+    }
+
+    @Test
+    public void testApplyParallelismLimit_LimitGreaterThanTasks() {
+        testApplyParallelismLimit(3, 3);
+        testApplyParallelismLimit(5, 6);
+    }
+
+    @Test
+    public void testApplyParallelismLimit_LimitLessThanTasks()
+    {
+        testApplyParallelismLimit(5, 2);
+        testApplyParallelismLimit(8, 4);
+    }
+
+    private void testApplyParallelismLimit(int taskCount, int limit) {
+        List<AbstractCompactionTask> tasks = createMockTasks(taskCount);
+        List<AbstractCompactionTask> result = CompositeCompactionTask.applyParallelismLimit(tasks, limit);
+
+        assertEquals(limit > 0 ? Math.min(limit, taskCount) : taskCount, result.size());
+        assertEquals(taskCount, result.stream().flatMap(t -> t instanceof CompositeCompactionTask ? ((CompositeCompactionTask) t).tasks.stream()
+                                                                                                  : Stream.of(t)).count());
+
+        for (AbstractCompactionTask task : result) {
+            if (task instanceof CompositeCompactionTask)
+            {
+                task.setOpObserver(mockOpObserver);
+                task.executeInternal(); // can't call execute() because it will call transaction.close()
+            }
+            else
+                task.execute(mockOpObserver);
+        }
+        for (AbstractCompactionTask task : tasks) {
+            verify(task, times(1)).execute(mockOpObserver);
+        }
+    }
+
+    private List<AbstractCompactionTask> createMockTasks(int count) {
+        List<AbstractCompactionTask> tasks = new ArrayList<>();
+        for (int i = 0; i < count; i++) {
+            AbstractCompactionTask task = mock(AbstractCompactionTask.class, Mockito.withSettings().useConstructor(mockRealm, mockTransaction));
+            tasks.add(task);
+        }
+        return tasks;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java
index dc78276254f6..3cce9ebaf0dd 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java
@@ -21,15 +21,19 @@
  */
 
 
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.*;
-
+import java.io.RandomAccessFile;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableMap;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,16 +44,27 @@
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.cache.ChunkCache;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.lifecycle.PartialLifecycleTransaction;
 import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursor;
+import org.apache.cassandra.io.sstable.compaction.SortedStringTableCursor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
 
 import static org.junit.Assert.assertTrue;
 
@@ -63,6 +78,7 @@ public class CorruptedSSTablesCompactionsTest
     private static final String STANDARD_STCS = "Standard_STCS";
     private static final String STANDARD_LCS = "Standard_LCS";
     private static final String STANDARD_UCS = "Standard_UCS";
+    private static final String STANDARD_UCS_PARALLEL = "Standard_UCS_Parallel";
     private static int maxValueSize;
 
     @After
@@ -77,6 +93,9 @@ public void leakDetect() throws InterruptedException
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
     {
+        DatabaseDescriptor.daemonInitialization(); // because of all the static initialization in CFS
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+
         long seed = nanoTime();
 
         //long seed = 754271160974509L; // CASSANDRA-9530: use this seed to reproduce compaction failures if reading empty rows
@@ -91,7 +110,8 @@ public static void defineSchema() throws ConfigurationException
                                     KeyspaceParams.simple(1),
                                     makeTable(STANDARD_STCS).compaction(CompactionParams.stcs(Collections.emptyMap())),
                                     makeTable(STANDARD_LCS).compaction(CompactionParams.lcs(Collections.emptyMap())),
-                                    makeTable(STANDARD_UCS).compaction(CompactionParams.ucs(Collections.emptyMap())));
+                                    makeTable(STANDARD_UCS).compaction(CompactionParams.ucs(Collections.emptyMap())),
+                                    makeTable(STANDARD_UCS_PARALLEL).compaction(CompactionParams.ucs(new HashMap<>(ImmutableMap.of("min_sstable_size", "1KiB")))));
 
         maxValueSize = DatabaseDescriptor.getMaxValueSize();
         DatabaseDescriptor.setMaxValueSize(1024 * 1024);
@@ -123,32 +143,39 @@ public static void closeStdErr()
     }
 
     @Test
-    public void testCorruptedSSTablesWithSizeTieredCompactionStrategy() throws Exception
+    public void testCorruptedSSTablesWithSizeTieredCompactionStrategy() throws Throwable
     {
         testCorruptedSSTables(STANDARD_STCS);
     }
 
     @Test
-    public void testCorruptedSSTablesWithLeveledCompactionStrategy() throws Exception
+    public void testCorruptedSSTablesWithLeveledCompactionStrategy() throws Throwable
     {
         testCorruptedSSTables(STANDARD_LCS);
     }
 
     @Test
-    public void testCorruptedSSTablesWithUnifiedCompactionStrategy() throws Exception
+    public void testCorruptedSSTablesWithUnifiedCompactionStrategy() throws Throwable
     {
         testCorruptedSSTables(STANDARD_UCS);
     }
 
+    @Test
+    public void testCorruptedSSTablesWithUnifiedCompactionStrategyParallelized() throws Throwable
+    {
+        testCorruptedSSTables(STANDARD_UCS_PARALLEL);
+    }
+
+    static final int COMPACTION_FAIL = -1;
 
-    public void testCorruptedSSTables(String tableName) throws Exception
+    public void testCorruptedSSTables(String tableName) throws Throwable
     {
         // this test does enough rows to force multiple block indexes to be used
         Keyspace keyspace = Keyspace.open(KEYSPACE1);
         final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(tableName);
 
-        final int ROWS_PER_SSTABLE = 10;
-        final int SSTABLES = cfs.metadata().params.minIndexInterval * 2 / ROWS_PER_SSTABLE;
+        final int ROWS_PER_SSTABLE = 1000; // enough data so that compression does not try to open the same chunk for multiple partial compaction tasks
+        final int SSTABLES = 25;
         final int SSTABLES_TO_CORRUPT = 8;
 
         assertTrue(String.format("Not enough sstables (%d), expected at least %d sstables to corrupt", SSTABLES, SSTABLES_TO_CORRUPT),
@@ -189,30 +216,33 @@ public void testCorruptedSSTables(String tableName) throws Exception
             if (currentSSTable + 1 > SSTABLES_TO_CORRUPT)
                 break;
 
-            FileChannel fc = null;
-
-            try
+            do
             {
-                int corruptionSize = 100;
-                fc = new File(sstable.getFilename()).newReadWriteChannel();
-                assertNotNull(fc);
-                assertTrue(fc.size() > corruptionSize);
-                long pos = random.nextInt((int)(fc.size() - corruptionSize));
-                logger.info("Corrupting sstable {} [{}] at pos {} / {}", currentSSTable, sstable.getFilename(), pos, fc.size());
-                fc.position(pos);
-                // We want to write something large enough that the corruption cannot get undetected
-                // (even without compression)
-                byte[] corruption = new byte[corruptionSize];
-                random.nextBytes(corruption);
-                fc.write(ByteBuffer.wrap(corruption));
+                RandomAccessFile raf = null;
+
+                try
+                {
+                    int corruptionSize = 25;
+                    raf = new RandomAccessFile(sstable.getFilename(), "rw");
+                    assertNotNull(raf);
+                    assertTrue(raf.length() > corruptionSize);
+                    long pos = random.nextInt((int) (raf.length() - corruptionSize));
+                    logger.info("Corrupting sstable {} [{}] at pos {} / {}", currentSSTable, sstable.getFilename(), pos, raf.length());
+                    raf.seek(pos);
+                    // We want to write something large enough that the corruption cannot get undetected
+                    // (even without compression)
+                    byte[] corruption = new byte[corruptionSize];
+                    random.nextBytes(corruption);
+                    raf.write(corruption);
+                }
+                finally
+                {
+                    FileUtils.closeQuietly(raf);
+                }
                 if (ChunkCache.instance != null)
-                    ChunkCache.instance.invalidateFile(sstable.getFilename());
-
-            }
-            finally
-            {
-                FileUtils.closeQuietly(fc);
+                    ChunkCache.instance.invalidateFileNow(sstable.getDataFile());
             }
+            while (readsWithoutError(sstable));
 
             currentSSTable++;
         }
@@ -227,16 +257,120 @@ public void testCorruptedSSTables(String tableName) throws Exception
                 cfs.forceMajorCompaction();
                 break; // After all corrupted sstables are marked as such, compaction of the rest should succeed.
             }
-            catch (Exception e)
+            catch (Throwable e)
             {
-                // This is the expected path. The SSTable should be marked corrupted, and retrying the compaction
-                // should move on to the next corruption.
-                Throwables.assertAnyCause(e, CorruptSSTableException.class);
-                failures++;
+                System.out.println(e);
+                // This is the expected path.
+                int fails = processException(e);
+                if (fails == COMPACTION_FAIL)
+                {
+                    logger.info("Completing test after {} failures because of non-sstable-specific AssertionError\n{}", failures, e);
+                    failures = SSTABLES_TO_CORRUPT;
+                    break;
+                }
+                else
+                {
+                    failures += fails;
+                }
             }
         }
 
         cfs.truncateBlocking();
         assertEquals(SSTABLES_TO_CORRUPT, failures);
     }
+
+    private int processException(Throwable e) throws Throwable
+    {
+        return processException(e, new HashSet<>());
+    }
+
+    private int processException(Throwable e, Set<File> countedFiles) throws Throwable
+    {
+        Throwable cause = e;
+        int failures = 0;
+        boolean foundCause = false;
+        while (cause != null)
+        {
+            // The SSTable should be marked corrupted, and retrying the compaction
+            // should move on to the next corruption.
+            if (cause instanceof CorruptSSTableException)
+            {
+                if (countedFiles.add(((CorruptSSTableException) cause).file))
+                    failures++;
+                foundCause = true;
+                break;
+            }
+
+            // If we are compacting with cursors, we may be unable to identify the sstable at the source of the
+            // corruption, sometimes failing with an AssertionError in the compaction class. If so, complete the
+            // test.
+            if (CassandraRelevantProperties.ALLOW_CURSOR_COMPACTION.getBoolean() &&
+                cause instanceof AssertionError &&
+                cause.getMessage().contains("nodetool scrub"))
+            {
+                return COMPACTION_FAIL;
+            }
+
+            // If the compactions are parallelized, the error message should contain all failures of the current path.
+            for (var t : cause.getSuppressed())
+            {
+                final int childFailures = processException(t, countedFiles);
+                if (childFailures == COMPACTION_FAIL)
+                    return COMPACTION_FAIL;
+                failures += childFailures;
+            }
+            if (cause instanceof PartialLifecycleTransaction.AbortedException)
+            {
+                foundCause = true;
+                break;
+            }
+            cause = cause.getCause();
+        }
+        if (!foundCause)
+            throw e;
+        return failures;
+    }
+
+    private boolean readsWithoutError(SSTableReader sstable)
+    {
+        if (CassandraRelevantProperties.ALLOW_CURSOR_COMPACTION.getBoolean())
+            return readsWithoutErrorCursor(sstable);
+        else
+            return readsWithoutErrorIterator(sstable);
+    }
+
+    private boolean readsWithoutErrorIterator(SSTableReader sstable)
+    {
+        try
+        {
+            ISSTableScanner scanner = sstable.getScanner();
+            while (scanner.hasNext())
+            {
+                UnfilteredRowIterator iter = scanner.next();
+                while (iter.hasNext())
+                    iter.next();
+            }
+            return true;
+        }
+        catch (Throwable t)
+        {
+            sstable.unmarkSuspect();
+            return false;
+        }
+    }
+
+    private boolean readsWithoutErrorCursor(SSTableReader sstable)
+    {
+        try
+        {
+            SSTableCursor cursor = new SortedStringTableCursor(sstable);
+            while (cursor.advance() != SSTableCursor.Type.EXHAUSTED) {}
+            return true;
+        }
+        catch (Throwable t)
+        {
+            sstable.unmarkSuspect();
+            return false;
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/DelegatingShardManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/DelegatingShardManagerTest.java
new file mode 100644
index 000000000000..063e750e79b0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/DelegatingShardManagerTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.SortedLocalRanges;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+public class DelegatingShardManagerTest
+{
+    final IPartitioner partitioner = Murmur3Partitioner.instance;
+
+    @Test
+    public void testWrappingShardManagerNoDisks()
+    {
+        CompactionRealm realm = Mockito.mock(CompactionRealm.class);
+        when(realm.getPartitioner()).thenReturn(partitioner);
+        when(realm.estimatedPartitionCountInSSTables()).thenReturn(1L << 16);
+        SortedLocalRanges localRanges = SortedLocalRanges.forTestingFull(realm);
+        ShardManager delegate = new ShardManagerNoDisks(localRanges);
+
+        DelegatingShardManager wrapper = new DelegatingShardManager((x) -> consumeTokens(delegate.boundaries(x)), realm);
+
+        var range = new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken());
+        assertEquals(1, wrapper.rangeSpanned(range), 0);
+        assertEquals(1, wrapper.localSpaceCoverage(), 0);
+        assertEquals(1, wrapper.shardSetCoverage(), 0);
+        assertEquals(1D / (1L << 16), wrapper.minimumPerPartitionSpan(), 0);
+
+        // We expect the same shards because the wrapper delegates.
+        for (int i = 1; i < 512; i++)
+        {
+            var wrapperTokenTracker = wrapper.boundaries(i);
+
+            // Make assertion about the first shard's fraction of the whole token range. This assertion relies
+            // on the fact that the delegate shard manager splits the space evenly for the given number of tokens.
+            var fractionInShard = wrapperTokenTracker.fractionInShard(range);
+            assertEquals(1d / i, fractionInShard, 0.001);
+
+            Token[] actualTokens = consumeTokens(wrapperTokenTracker);
+            Token[] expectedTokens = consumeTokens(delegate.boundaries(i));
+            assertArrayEquals(actualTokens, expectedTokens);
+        }
+    }
+
+    private Token[] consumeTokens(ShardTracker iterator)
+    {
+        Token[] actualTokens = new Token[iterator.count()];
+        actualTokens[iterator.shardIndex()] = iterator.shardStart();
+        for (Token end = iterator.shardEnd(); end != null; end = iterator.shardEnd())
+        {
+            assertFalse(iterator.advanceTo(end));
+            assertTrue(iterator.advanceTo(end.nextValidToken()));
+            actualTokens[iterator.shardIndex()] = (end);
+        }
+        return actualTokens;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/DisabledRepairStateCheckingTest.java b/test/unit/org/apache/cassandra/db/compaction/DisabledRepairStateCheckingTest.java
new file mode 100644
index 000000000000..b3633fa92ce4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/DisabledRepairStateCheckingTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
+import static org.apache.cassandra.db.compaction.CompactionTaskTest.mockStrategy;
+import static org.apache.cassandra.db.compaction.CompactionTaskTest.mutateRepaired;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class DisabledRepairStateCheckingTest
+{
+    private static TableMetadata cfm;
+    private static ColumnFamilyStore cfs;
+
+    private final CompactionStrategy mockStrategy;
+
+    public DisabledRepairStateCheckingTest()
+    {
+        this.mockStrategy = mockStrategy(cfs, true);
+    }
+
+    @BeforeClass
+    public static void setUpClass() throws Exception
+    {
+        CassandraRelevantProperties.COMPACTION_SKIP_REPAIR_STATE_CHECKING.setBoolean(true);
+        SchemaLoader.prepareServer();
+        cfm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", "ks").build();
+        SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), cfm);
+        cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
+    }
+
+    @Before
+    public void setUp() throws Exception
+    {
+        cfs.getCompactionStrategyContainer().enable();
+        cfs.truncateBlocking();
+    }
+
+    /**
+     * Duplicate of {@link CompactionTaskTest#mixedSSTableFailure()} with disabled repair state checking. Creating the
+     * task should succeed.
+     */
+    @Test
+    public void mixedSSTableFailure() throws Exception
+    {
+        cfs.getCompactionStrategyContainer().disable();
+        for (int m = 1; m < 16; ++m) // test all combinations of two or more sstables with different repair marking
+        {
+            if (Integer.bitCount(m) <= 1)
+                continue;
+            for (int order = 0; order < Integer.bitCount(m); order++)
+            {
+
+                cfs.truncateBlocking();
+                QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);");
+                cfs.forceBlockingFlush(UNIT_TESTS);
+                QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);");
+                cfs.forceBlockingFlush(UNIT_TESTS);
+                QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (3, 3);");
+                cfs.forceBlockingFlush(UNIT_TESTS);
+                QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (4, 4);");
+                cfs.forceBlockingFlush(UNIT_TESTS);
+
+                List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+                Assert.assertEquals(4, sstables.size());
+
+                SSTableReader unrepaired = sstables.get(0);
+                SSTableReader repaired = sstables.get(1);
+                SSTableReader pending1 = sstables.get(2);
+                SSTableReader pending2 = sstables.get(3);
+
+                mutateRepaired(repaired, FBUtilities.nowInSeconds(), ActiveRepairService.NO_PENDING_REPAIR, false);
+                mutateRepaired(pending1, ActiveRepairService.UNREPAIRED_SSTABLE, TimeUUID.Generator.nextTimeUUID(), false);
+                mutateRepaired(pending2, ActiveRepairService.UNREPAIRED_SSTABLE, TimeUUID.Generator.nextTimeUUID(), false);
+
+                for (int i = 3; i >= 0; i--)
+                {
+                    if ((m & (1 << i)) == 0)
+                        sstables.remove(i);
+                }
+                Collections.rotate(sstables, order);
+
+                LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+                assertNotNull(txn);
+                CompactionTask task = new CompactionTask(cfs, txn, 0, false, mockStrategy);
+                assertNotNull(task); // task must be successfully created
+                task.executeInternal(); // and run
+                for (SSTableReader s : txn.current())
+                {
+                    assertFalse(s.isRepaired()); // and the resulting files must be marked unrepaired
+                    assertNull(s.getPendingRepair());
+                }
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/LegacyAbstractCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LegacyAbstractCompactionStrategyTest.java
new file mode 100644
index 000000000000..6e06fb1ccd4b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/LegacyAbstractCompactionStrategyTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class LegacyAbstractCompactionStrategyTest
+{
+    private static final String KEYSPACE1 = "Keyspace1";
+    private static final String LCS_TABLE = "LCS_TABLE";
+    private static final String STCS_TABLE = "STCS_TABLE";
+    private static final String TWCS_TABLE = "TWCS_TABLE";
+
+    @BeforeClass
+    public static void loadData() throws ConfigurationException
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, LCS_TABLE)
+                                                .compaction(CompactionParams.lcs(Collections.emptyMap())),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, STCS_TABLE)
+                                                .compaction(CompactionParams.stcs(Collections.emptyMap())),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, TWCS_TABLE)
+                                                .compaction(CompactionParams.create(TimeWindowCompactionStrategy.class, Collections.emptyMap())));
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(LCS_TABLE).disableAutoCompaction();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(STCS_TABLE).disableAutoCompaction();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(TWCS_TABLE).disableAutoCompaction();
+    }
+
+    @After
+    public void tearDown()
+    {
+
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(LCS_TABLE).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(STCS_TABLE).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(TWCS_TABLE).truncateBlocking();
+    }
+
+    @Test(timeout=30000)
+    public void testGetNextBackgroundTaskDoesNotBlockLCS()
+    {
+        testGetNextBackgroundTaskDoesNotBlock(LCS_TABLE);
+    }
+
+    @Test(timeout=30000)
+    public void testGetNextBackgroundTaskDoesNotBlockSTCS()
+    {
+        testGetNextBackgroundTaskDoesNotBlock(STCS_TABLE);
+    }
+
+    @Test(timeout=30000)
+    public void testGetNextBackgroundTaskDoesNotBlockTWCS()
+    {
+        testGetNextBackgroundTaskDoesNotBlock(TWCS_TABLE);
+    }
+
+    public void testGetNextBackgroundTaskDoesNotBlock(String table)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(table);
+        CompactionStrategy strategy = cfs.getCompactionStrategyContainer()
+                                         .getStrategies(false, null)
+                                         .get(0);
+
+        // Add 4 sstables
+        for (int i = 1; i <= 4; i++)
+        {
+            insertKeyAndFlush(table, i);
+        }
+
+        // Check they are returned on the next background task
+        Collection<AbstractCompactionTask> tasks = strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        Assert.assertEquals(1, tasks.size());
+        AbstractCompactionTask task = tasks.iterator().next();
+        Assert.assertNotNull(task);
+        try (var txn = task.transaction)
+        {
+            Assert.assertEquals(cfs.getLiveSSTables(), txn.originals());
+        }
+
+        // now remove sstables on the tracker, to simulate a concurrent transaction
+        cfs.getTracker().removeUnsafe(cfs.getLiveSSTables());
+
+        // verify the compaction strategy will return null
+        Assert.assertTrue(strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).isEmpty());
+    }
+
+
+    private static void insertKeyAndFlush(String table, int key)
+    {
+        long timestamp = System.currentTimeMillis();
+        DecoratedKey dk = Util.dk(String.format("%03d", key));
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(table);
+        new RowUpdateBuilder(cfs.metadata(), timestamp, dk.getKey())
+        .clustering(String.valueOf(key))
+        .add("val", "val")
+        .build()
+        .applyUnsafe();
+        Util.flush(cfs);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index d56003ae2a4a..27568b7b81e5 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@ -32,6 +32,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.google.common.collect.Collections2;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import org.junit.After;
@@ -146,11 +147,11 @@ public void testGrouperLevels() throws Exception{
         }
 
         waitForLeveling(cfs);
-        CompactionStrategyManager strategyManager = cfs.getCompactionStrategyManager();
+        CompactionStrategyContainer strategyContainer = cfs.getCompactionStrategyContainer();
         // Checking we're not completely bad at math
 
-        int l1Count = strategyManager.getSSTableCountPerLevel()[1];
-        int l2Count = strategyManager.getSSTableCountPerLevel()[2];
+        int l1Count = strategyContainer.getSSTableCountPerLevel()[1];
+        int l2Count = strategyContainer.getSSTableCountPerLevel()[2];
         if (l1Count == 0 || l2Count == 0)
         {
             logger.error("L1 or L2 has 0 sstables. Expected > 0 on both.");
@@ -159,15 +160,15 @@ public void testGrouperLevels() throws Exception{
             Assert.fail();
         }
 
-        Collection<Collection<SSTableReader>> groupedSSTables = cfs.getCompactionStrategyManager().groupSSTablesForAntiCompaction(cfs.getLiveSSTables());
-        for (Collection<SSTableReader> sstableGroup : groupedSSTables)
+        Collection<Collection<CompactionSSTable>> groupedSSTables = cfs.getCompactionStrategyContainer().groupSSTablesForAntiCompaction(cfs.getLiveSSTables());
+        for (Collection<CompactionSSTable> sstableGroup : groupedSSTables)
         {
             int groupLevel = -1;
-            Iterator<SSTableReader> it = sstableGroup.iterator();
+            Iterator<CompactionSSTable> it = sstableGroup.iterator();
             while (it.hasNext())
             {
 
-                SSTableReader sstable = it.next();
+                CompactionSSTable sstable = it.next();
                 int tableLevel = sstable.getSSTableLevel();
                 if (groupLevel == -1)
                     groupLevel = tableLevel;
@@ -201,10 +202,10 @@ public void testValidationMultipleSSTablePerLevel() throws Exception
         }
 
         waitForLeveling(cfs);
-        CompactionStrategyManager strategyManager = cfs.getCompactionStrategyManager();
+        CompactionStrategyContainer strategyContainer = cfs.getCompactionStrategyContainer();
         // Checking we're not completely bad at math
-        assertTrue(strategyManager.getSSTableCountPerLevel()[1] > 0);
-        assertTrue(strategyManager.getSSTableCountPerLevel()[2] > 0);
+        assertTrue(strategyContainer.getSSTableCountPerLevel()[1] > 0);
+        assertTrue(strategyContainer.getSSTableCountPerLevel()[2] > 0);
 
         Range<Token> range = new Range<>(Util.token(""), Util.token(""));
         long gcBefore = keyspace.getColumnFamilyStore(CF_STANDARDDLEVELED).gcBefore(FBUtilities.nowInSeconds());
@@ -228,7 +229,7 @@ public void testValidationMultipleSSTablePerLevel() throws Exception
      */
     public static void waitForLeveling(ColumnFamilyStore cfs) throws InterruptedException
     {
-        CompactionStrategyManager strategyManager = cfs.getCompactionStrategyManager();
+        CompactionStrategyContainer strategyContainer = cfs.getCompactionStrategyContainer();
         while (true)
         {
             // since we run several compaction strategies we wait until L0 in all strategies is empty and
@@ -236,19 +237,16 @@ public static void waitForLeveling(ColumnFamilyStore cfs) throws InterruptedExce
             // so it should be good enough
             boolean allL0Empty = true;
             boolean anyL1NonEmpty = false;
-            for (List<AbstractCompactionStrategy> strategies : strategyManager.getStrategies())
+            for (CompactionStrategy strategy : strategyContainer.getStrategies())
             {
-                for (AbstractCompactionStrategy strategy : strategies)
-                {
-                    if (!(strategy instanceof LeveledCompactionStrategy))
-                        return;
-                    // note that we check > 1 here, if there is too little data in L0, we don't compact it up to L1
-                    if (((LeveledCompactionStrategy)strategy).getLevelSize(0) > 1)
-                        allL0Empty = false;
-                    for (int i = 1; i < 5; i++)
-                        if (((LeveledCompactionStrategy)strategy).getLevelSize(i) > 0)
-                            anyL1NonEmpty = true;
-                }
+                if (!(strategy instanceof LeveledCompactionStrategy))
+                    return;
+                // note that we check > 1 here, if there is too little data in L0, we don't compact it up to L1
+                if (((LeveledCompactionStrategy)strategy).getLevelSize(0) > 1)
+                    allL0Empty = false;
+                for (int i = 1; i < 5; i++)
+                    if (((LeveledCompactionStrategy)strategy).getLevelSize(i) > 0)
+                        anyL1NonEmpty = true;
             }
             if (allL0Empty && anyL1NonEmpty)
                 return;
@@ -275,11 +273,13 @@ public void testCompactionProgress() throws Exception
         }
 
         waitForLeveling(cfs);
-        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getStrategies().get(1).get(0);
+        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) cfs.getCompactionStrategyContainer()
+                                                                            .getStrategies(false, null)
+                                                                            .get(0);
         assert strategy.getLevelSize(1) > 0;
 
         // get LeveledScanner for level 1 sstables
-        Collection<SSTableReader> sstables = strategy.manifest.getLevel(1);
+        Collection<SSTableReader> sstables = Collections2.transform(strategy.manifest.getLevel(1), SSTableReader.class::cast);
         List<ISSTableScanner> scanners = strategy.getScanners(sstables).scanners;
         assertEquals(1, scanners.size()); // should be one per level
         ISSTableScanner scanner = scanners.get(0);
@@ -288,7 +288,7 @@ public void testCompactionProgress() throws Exception
             scanner.next();
 
         // scanner.getCurrentPosition should be equal to total bytes of L1 sstables
-        assertEquals(scanner.getCurrentPosition(), SSTableReader.getTotalUncompressedBytes(sstables));
+        assertEquals(scanner.getCurrentPosition(), CompactionSSTable.getTotalUncompressedBytes(sstables));
     }
 
     @Test
@@ -311,7 +311,9 @@ public void testMutateLevel() throws Exception
             Util.flush(cfs);
         }
         Util.flush(cfs);
-        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getStrategies().get(1).get(0);
+        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) cfs.getCompactionStrategyContainer()
+                                                                            .getStrategies(false, null)
+                                                                            .get(0);
         cfs.forceMajorCompaction();
 
         for (SSTableReader s : cfs.getLiveSSTables())
@@ -357,14 +359,17 @@ public void testNewRepairedSSTable() throws Exception
         while(CompactionManager.instance.isCompacting(Arrays.asList(cfs), (sstable) -> true))
             Thread.sleep(100);
 
-        CompactionStrategyManager manager = cfs.getCompactionStrategyManager();
-        List<List<AbstractCompactionStrategy>> strategies = manager.getStrategies();
-        LeveledCompactionStrategy repaired = (LeveledCompactionStrategy) strategies.get(0).get(0);
-        LeveledCompactionStrategy unrepaired = (LeveledCompactionStrategy) strategies.get(1).get(0);
+        CompactionStrategyContainer strategyContainer = cfs.getCompactionStrategyContainer();
+        LeveledCompactionStrategy repaired = (LeveledCompactionStrategy) strategyContainer
+                                                                         .getStrategies(true, null)
+                                                                         .get(0);
+        LeveledCompactionStrategy unrepaired = (LeveledCompactionStrategy) strategyContainer
+                                                                           .getStrategies(false, null)
+                                                                           .get(0);
         assertEquals(0, repaired.manifest.getLevelCount() );
         assertEquals(2, unrepaired.manifest.getLevelCount());
-        assertTrue(manager.getSSTableCountPerLevel()[1] > 0);
-        assertTrue(manager.getSSTableCountPerLevel()[2] > 0);
+        assertTrue(strategyContainer.getSSTableCountPerLevel()[1] > 0);
+        assertTrue(strategyContainer.getSSTableCountPerLevel()[2] > 0);
 
         for (SSTableReader sstable : cfs.getLiveSSTables())
             assertFalse(sstable.isRepaired());
@@ -373,14 +378,14 @@ public void testNewRepairedSSTable() throws Exception
         // we only have unrepaired sstables:
         assertEquals(sstableCount, cfs.getLiveSSTables().size());
 
-        SSTableReader sstable1 = unrepaired.manifest.getLevel(2).iterator().next();
-        SSTableReader sstable2 = unrepaired.manifest.getLevel(1).iterator().next();
+        SSTableReader sstable1 = (SSTableReader) unrepaired.manifest.getLevel(2).iterator().next();
+        SSTableReader sstable2 = (SSTableReader) unrepaired.manifest.getLevel(1).iterator().next();
 
         sstable1.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable1.descriptor, System.currentTimeMillis(), null, false);
         sstable1.reloadSSTableMetadata();
         assertTrue(sstable1.isRepaired());
 
-        manager.handleNotification(new SSTableRepairStatusChanged(Arrays.asList(sstable1)), this);
+        strategyContainer.handleNotification(new SSTableRepairStatusChanged(Arrays.asList(sstable1)), this);
 
         int repairedSSTableCount = repaired.manifest.getSSTables().size();
         assertEquals(1, repairedSSTableCount);
@@ -390,7 +395,7 @@ public void testNewRepairedSSTable() throws Exception
         assertFalse(unrepaired.manifest.getLevel(2).contains(sstable1));
 
         unrepaired.removeSSTable(sstable2);
-        manager.handleNotification(new SSTableAddedNotification(singleton(sstable2), null), this);
+        strategyContainer.handleNotification(new SSTableAddedNotification(singleton(sstable2), null), this);
         assertTrue(unrepaired.manifest.getLevel(1).contains(sstable2));
         assertFalse(repaired.manifest.getLevel(1).contains(sstable2));
     }
@@ -514,7 +519,7 @@ public void testTokenRangeCompaction() throws Exception
     }
 
     @Test
-    public void testCompactionCandidateOrdering() throws Exception
+    public void testCompactionCandidateOrdering()
     {
         // add some data
         byte [] b = new byte[100 * 1024];
@@ -532,13 +537,15 @@ public void testCompactionCandidateOrdering() throws Exception
             update.applyUnsafe();
             Util.flush(cfs);
         }
-        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) (cfs.getCompactionStrategyManager()).getStrategies().get(1).get(0);
+        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) (cfs.getCompactionStrategyContainer())
+                                                                         .getStrategies(false, null)
+                                                                         .get(0);
         // get readers for level 0 sstables
-        Collection<SSTableReader> sstables = strategy.manifest.getLevel(0);
-        Collection<SSTableReader> sortedCandidates = strategy.manifest.ageSortedSSTables(sstables);
+        Set<CompactionSSTable> sstables = strategy.manifest.getLevel(0);
+        List<CompactionSSTable> sortedCandidates = strategy.manifest.ageSortedSSTables(sstables);
         assertTrue(String.format("More than 1 sstable required for test, found: %d .", sortedCandidates.size()), sortedCandidates.size() > 1);
         long lastMaxTimeStamp = Long.MIN_VALUE;
-        for (SSTableReader sstable : sortedCandidates)
+        for (CompactionSSTable sstable : sortedCandidates)
         {
             assertTrue(String.format("SStables not sorted into oldest to newest by maxTimestamp. Current sstable: %d , last sstable: %d", sstable.getMaxTimestamp(), lastMaxTimeStamp),
                        sstable.getMaxTimestamp() > lastMaxTimeStamp);
@@ -589,23 +596,22 @@ public void testDisableSTCSInL0() throws IOException
     private int getTaskLevel(ColumnFamilyStore cfs)
     {
         int level = -1;
-        for (List<AbstractCompactionStrategy> strategies : cfs.getCompactionStrategyManager().getStrategies())
+        for (CompactionStrategy strategy : cfs.getCompactionStrategyContainer().getStrategies())
         {
-            for (AbstractCompactionStrategy strategy : strategies)
+            Collection<AbstractCompactionTask> tasks = strategy.getNextBackgroundTasks(0);
+            if (!tasks.isEmpty())
             {
-                AbstractCompactionTask task = strategy.getNextBackgroundTask(0);
-                if (task != null)
+                assertEquals(1, tasks.size());
+                AbstractCompactionTask task = tasks.iterator().next();
+                try
                 {
-                    try
-                    {
-                        assertTrue(task instanceof LeveledCompactionTask);
-                        LeveledCompactionTask lcsTask = (LeveledCompactionTask) task;
-                        level = Math.max(level, lcsTask.getLevel());
-                    }
-                    finally
-                    {
-                        task.transaction.abort();
-                    }
+                    assertTrue(task instanceof LeveledCompactionTask);
+                    LeveledCompactionTask lcsTask = (LeveledCompactionTask) task;
+                    level = Math.max(level, lcsTask.getLevel());
+                }
+                finally
+                {
+                    task.transaction.abort();
                 }
             }
         }
@@ -729,14 +735,14 @@ public void randomMultiLevelAddTest()
         for (int i = 0; i < levelCount; i++)
         {
             actualSSTableCount += lm.getLevelSize(i);
-            List<SSTableReader> level = new ArrayList<>(lm.getLevel(i));
+            List<CompactionSSTable> level = new ArrayList<>(lm.getLevel(i));
             int lvl = i;
             assertTrue(level.stream().allMatch(s -> s.getSSTableLevel() == lvl));
             if (i > 0)
             {
-                level.sort(SSTableReader.firstKeyComparator);
-                SSTableReader prev = null;
-                for (SSTableReader sstable : level)
+                level.sort(CompactionSSTable.firstKeyComparator);
+                CompactionSSTable prev = null;
+                for (CompactionSSTable sstable : level)
                 {
                     if (prev != null && sstable.getFirst().compareTo(prev.getLast()) <= 0)
                     {
@@ -803,7 +809,9 @@ public void testPerLevelSizeBytes() throws IOException
 
         assertEquals(sstable.onDiskLength(), cfs.getPerLevelSizeBytes()[0]);
 
-        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) ( cfs.getCompactionStrategyManager()).getStrategies().get(1).get(0);
+        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) cfs.getCompactionStrategyContainer()
+                                                                            .getStrategies(false, null)
+                                                                            .get(0);
         strategy.manifest.remove(sstable);
         sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 2);
         sstable.reloadSSTableMetadata();
@@ -840,15 +848,15 @@ private static int[] canAdd(LeveledManifest lm, List<SSTableReader> newSSTables,
                 continue;
             }
 
-            List<SSTableReader> newLevel = new ArrayList<>(lm.getLevel(level));
-            for (SSTableReader sstable : lvlGroup.getValue())
+            List<CompactionSSTable> newLevel = new ArrayList<>(lm.getLevel(level));
+            for (CompactionSSTable sstable : lvlGroup.getValue())
             {
                 newLevel.add(sstable);
-                newLevel.sort(SSTableReader.firstKeyComparator);
+                newLevel.sort(CompactionSSTable.firstKeyComparator);
 
-                SSTableReader prev = null;
+                CompactionSSTable prev = null;
                 boolean kept = true;
-                for (SSTableReader sst : newLevel)
+                for (CompactionSSTable sst : newLevel)
                 {
                     if (prev != null && prev.getLast().compareTo(sst.getFirst()) >= 0)
                     {
@@ -867,7 +875,7 @@ private static int[] canAdd(LeveledManifest lm, List<SSTableReader> newSSTables,
         return canAdd;
     }
 
-    private static void assertLevelsEqual(Collection<SSTableReader> l1, Collection<SSTableReader> l2)
+    private static void assertLevelsEqual(Collection<? extends CompactionSSTable> l1, Collection<? extends CompactionSSTable> l2)
     {
         assertEquals(l1.size(), l2.size());
         assertEquals(new HashSet<>(l1), new HashSet<>(l2));
@@ -896,7 +904,7 @@ public void testHighestLevelHasMoreDataThanSupported()
 
         // compaction for L8 sstables is not supposed to be run because there is no upper level to promote sstables
         // that's why we expect compaction candidates for L7 only
-        Collection<SSTableReader> compactionCandidates = lm.getCompactionCandidates().sstables;
+        Collection<CompactionSSTable> compactionCandidates = lm.getCompactionCandidate().sstables;
         assertThat(compactionCandidates).containsAll(sstablesOnL7);
         assertThat(compactionCandidates).doesNotContainAnyElementsOf(sstablesOnL8);
     }
@@ -920,10 +928,10 @@ public void testReduceScopeL0L1() throws IOException
         List<SSTableReader> l0sstables = new ArrayList<>();
         for (int i = 10; i < 20; i++)
             l0sstables.add(MockSchema.sstable(i, (i + 1) * 1024 * 1024, cfs));
-        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, Iterables.concat(l0sstables, l1sstables)))
+        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, cfs.metadata, Iterables.concat(l0sstables, l1sstables)))
         {
-            Set<SSTableReader> nonExpired = Sets.difference(txn.originals(), Collections.emptySet());
-            CompactionTask task = new LeveledCompactionTask(cfs, txn, 1, 0, 1024*1024, false);
+            Set<SSTableReader> nonExpired = new HashSet<>(Sets.difference(txn.originals(), Collections.emptySet()));
+            CompactionTask task = new LeveledCompactionTask(cfs, txn, 1, 0, 1024*1024, false, null);
             SSTableReader lastRemoved = null;
             boolean removed = true;
             for (int i = 0; i < l0sstables.size(); i++)
@@ -965,16 +973,17 @@ public void testReduceScopeL0()
         for (int i = 10; i < 20; i++)
             l0sstables.add(MockSchema.sstable(i, (i + 1) * 1024 * 1024, cfs));
 
-        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, l0sstables))
+        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, cfs.metadata, l0sstables))
         {
-            CompactionTask task = new LeveledCompactionTask(cfs, txn, 0, 0, 1024*1024, false);
+            CompactionTask task = new LeveledCompactionTask(cfs, txn, 0, 0, 1024*1024, false, null);
 
             SSTableReader lastRemoved = null;
             boolean removed = true;
             for (int i = 0; i < l0sstables.size(); i++)
             {
                 Set<SSTableReader> before = new HashSet<>(txn.originals());
-                removed = task.reduceScopeForLimitedSpace(before, 0);
+                Set<SSTableReader> sources = new HashSet<>(before);
+                removed = task.reduceScopeForLimitedSpace(sources, 0);
                 SSTableReader removedSSTable = Sets.difference(before, txn.originals()).stream().findFirst().orElse(null);
                 if (removed)
                 {
@@ -1015,9 +1024,9 @@ public void testNoHighLevelReduction() throws IOException
             sstable.reloadSSTableMetadata();
             sstables.add(sstable);
         }
-        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, sstables))
+        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, cfs.metadata, sstables))
         {
-            CompactionTask task = new LeveledCompactionTask(cfs, txn, 0, 0, 1024 * 1024, false);
+            CompactionTask task = new LeveledCompactionTask(cfs, txn, 0, 0, 1024 * 1024, false, null);
             assertFalse(task.reduceScopeForLimitedSpace(Sets.newHashSet(sstables), 0));
             assertEquals(Sets.newHashSet(sstables), txn.originals());
         }
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledGenerationsTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledGenerationsTest.java
index 11592f01d55a..7aabb811adfa 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledGenerationsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledGenerationsTest.java
@@ -37,8 +37,8 @@
 import org.apache.cassandra.schema.MockSchema;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 public class LeveledGenerationsTest extends CQLTester
@@ -163,21 +163,21 @@ public void testFillLevels()
         {}
     }
 
-    private void assertIter(Iterator<SSTableReader> iter, long first, long last, int expectedCount)
+    private void assertIter(Iterator<? extends CompactionSSTable> iter, long first, long last, int expectedCount)
     {
-        List<SSTableReader> drained = Lists.newArrayList(iter);
+        List<CompactionSSTable> drained = Lists.newArrayList(iter);
         assertEquals(expectedCount, drained.size());
         assertEquals(dk(first).getToken(), first(drained).getFirst().getToken());
         assertEquals(dk(last).getToken(), last(drained).getFirst().getToken()); // we sort by first token, so this is the first token of the last sstable in iter
     }
 
-    private SSTableReader last(Iterable<SSTableReader> iter)
+    private CompactionSSTable last(Iterable<CompactionSSTable> iter)
     {
         return Iterables.getLast(iter);
     }
-    private SSTableReader first(Iterable<SSTableReader> iter)
+    private CompactionSSTable first(Iterable<CompactionSSTable> iter)
     {
-        SSTableReader first = Iterables.getFirst(iter, null);
+        CompactionSSTable first = Iterables.getFirst(iter, null);
         if (first == null)
             throw new RuntimeException();
         return first;
diff --git a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
index 3f5e4b6f5dfb..00825d9db75e 100644
--- a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
@@ -65,9 +65,9 @@ public void neverPurgePartitionTombstoneTest() throws Throwable
     }
 
     @Test
-    public void minorNeverPurgeTombstonesTest() throws Throwable
+    public void minorNeverPurgeTombstonesWithSizeTieredCompactionTest() throws Throwable
     {
-        createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 0");
+        createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 0 AND COMPACTION={'class':'SizeTieredCompactionStrategy'}");
         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
         cfs.disableAutoCompaction();
         for (int i = 0; i < 4; i++)
diff --git a/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java
index 090038146842..17e52ad82806 100644
--- a/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java
@@ -76,7 +76,7 @@ private void testCompaction(String columnFamilyName, int insertsPerTable)
             assertEquals(inserted.size(), Util.getAll(Util.cmd(store).build()).size());
         }
         FBUtilities.waitOnFuture(Util.compactAll(store, FBUtilities.nowInSeconds()));
-        assertEquals(1, store.getLiveSSTables().size());
+        Util.assertNoOverlap(store.getLiveSSTables());
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/db/compaction/OperationTypeTest.java b/test/unit/org/apache/cassandra/db/compaction/OperationTypeTest.java
new file mode 100644
index 000000000000..1b6c25a4991f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/OperationTypeTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class OperationTypeTest
+{
+    @Test
+    public void stuffNeededByCNDBShouldNotBeRemovedFromTheCodebase()
+    {
+        // this "test" is made solely to explicitly reference elements used
+        // by cndb (and [likely] unused in Cassandra)
+        assertEquals(OperationType.RESTORE.toString(), OperationType.RESTORE.type);
+        assertEquals(OperationType.REMOTE_RELOAD.toString(), OperationType.REMOTE_RELOAD.type);
+        assertEquals(OperationType.REMOTE_COMPACTION.toString(), OperationType.REMOTE_COMPACTION.type);
+        assertEquals(OperationType.TRUNCATE_TABLE.toString(), OperationType.TRUNCATE_TABLE.type);
+        assertEquals(OperationType.DROP_TABLE.toString(), OperationType.DROP_TABLE.type);
+        assertEquals(OperationType.REMOVE_UNREADEABLE.toString(), OperationType.REMOVE_UNREADEABLE.type);
+        assertEquals(OperationType.REGION_BOOTSTRAP.toString(), OperationType.REGION_BOOTSTRAP.type);
+        assertEquals(OperationType.REGION_DECOMMISSION.toString(), OperationType.REGION_DECOMMISSION.type);
+        assertEquals(OperationType.REGION_REPAIR.toString(), OperationType.REGION_REPAIR.type);
+        assertEquals(OperationType.SSTABLE_DISCARD.toString(), OperationType.SSTABLE_DISCARD.type);
+        assertTrue(OperationType.SSTABLE_DISCARD.localOnly);
+        assertTrue(OperationType.KEY_CACHE_SAVE.isCacheSave());
+        assertFalse(OperationType.EXCEPT_VALIDATIONS.test(OperationType.VALIDATION));
+        assertTrue(OperationType.COMPACTIONS_ONLY.test(OperationType.COMPACTION));
+        assertTrue(OperationType.REWRITES_SSTABLES.test(OperationType.UPGRADE_SSTABLES));
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java
index d2b3693032fc..7460a3e57d71 100644
--- a/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java
@@ -35,13 +35,22 @@
 
 public class PendingRepairManagerTest extends AbstractPendingRepairTest
 {
+    @Override
+    public String createTableCql()
+    {
+        // Note: This test is tightly coupled to the LegacyAbstractCompactionStrategy so cannot use the default UCS
+        // UCS is tested in UnifiedCompactionContainerPendingRepairTest
+        return String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT) WITH COMPACTION={'class':'SizeTieredCompactionStrategy'}",
+                             ks, tbl);
+    }
+
     /**
      * If a local session is ongoing, it should not be cleaned up
      */
     @Test
     public void needsCleanupInProgress()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
 
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -59,7 +68,7 @@ public void needsCleanupInProgress()
     @Test
     public void needsCleanupFinalized()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
 
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -78,7 +87,7 @@ public void needsCleanupFinalized()
     @Test
     public void needsCleanupFailed()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
 
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -95,14 +104,14 @@ public void needsCleanupFailed()
     public void needsCleanupNoSession()
     {
         TimeUUID fakeID = nextTimeUUID();
-        PendingRepairManager prm = new PendingRepairManager(cfs, null, false);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, null, false);
         Assert.assertTrue(prm.canCleanup(fakeID));
     }
 
     @Test
     public void estimateRemainingTasksInProgress()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
 
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -118,7 +127,7 @@ public void estimateRemainingTasksInProgress()
     @Test
     public void estimateRemainingFinishedRepairTasks()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
 
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -136,7 +145,7 @@ public void estimateRemainingFinishedRepairTasks()
     @Test
     public void getNextBackgroundTask()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
 
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -152,13 +161,15 @@ public void getNextBackgroundTask()
         LocalSessionAccessor.finalizeUnsafe(repairID);
 
         Assert.assertEquals(2, prm.getSessions().size());
-        Assert.assertNull(prm.getNextBackgroundTask(FBUtilities.nowInSeconds()));
-        AbstractCompactionTask compactionTask = prm.getNextRepairFinishedTask();
+        Assert.assertTrue(prm.getNextBackgroundTasks(FBUtilities.nowInSeconds()).isEmpty());
+        Collection<AbstractCompactionTask> compactionTasks = prm.getNextRepairFinishedTasks();
+        Assert.assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
         try
         {
             Assert.assertNotNull(compactionTask);
-            Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
-            PendingRepairManager.RepairFinishedCompactionTask cleanupTask = (PendingRepairManager.RepairFinishedCompactionTask) compactionTask;
+            Assert.assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
+            RepairFinishedCompactionTask cleanupTask = (RepairFinishedCompactionTask) compactionTask;
             Assert.assertEquals(repairID, cleanupTask.getSessionID());
         }
         finally
@@ -170,17 +181,17 @@ public void getNextBackgroundTask()
     @Test
     public void getNextBackgroundTaskNoSessions()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
-        Assert.assertNull(prm.getNextBackgroundTask(FBUtilities.nowInSeconds()));
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
+        Assert.assertTrue(prm.getNextBackgroundTasks(FBUtilities.nowInSeconds()).isEmpty());
     }
 
     /**
      * If all sessions should be cleaned up, getNextBackgroundTask should return null
      */
     @Test
-    public void getNextBackgroundTaskAllCleanup() throws Exception
+    public void getNextBackgroundTaskAllCleanup()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
 
@@ -191,14 +202,14 @@ public void getNextBackgroundTaskAllCleanup() throws Exception
         Assert.assertNotNull(prm.get(repairID));
         LocalSessionAccessor.finalizeUnsafe(repairID);
 
-        Assert.assertNull(prm.getNextBackgroundTask(FBUtilities.nowInSeconds()));
+        Assert.assertTrue(prm.getNextBackgroundTasks(FBUtilities.nowInSeconds()).isEmpty());
 
     }
 
     @Test
     public void maximalTaskNeedsCleanup()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
 
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -209,7 +220,7 @@ public void maximalTaskNeedsCleanup()
         Assert.assertNotNull(prm.get(repairID));
         LocalSessionAccessor.finalizeUnsafe(repairID);
 
-        Collection<AbstractCompactionTask> tasks = prm.getMaximalTasks(FBUtilities.nowInSeconds(), false);
+        Collection<AbstractCompactionTask> tasks = prm.getMaximalTasks(FBUtilities.nowInSeconds(), false, 0);
         try
         {
             Assert.assertEquals(1, tasks.size());
@@ -223,22 +234,20 @@ public void maximalTaskNeedsCleanup()
     @Test
     public void userDefinedTaskTest()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
         TimeUUID repairId = registerSession(cfs, true, true);
         SSTableReader sstable = makeSSTable(true);
         mutateRepaired(sstable, repairId, false);
         prm.addSSTable(sstable);
 
-        try (CompactionTasks tasks = csm.getUserDefinedTasks(Collections.singleton(sstable), 100))
-        {
-            Assert.assertEquals(1, tasks.size());
-        }
+        Collection<AbstractCompactionTask> tasks = prm.createUserDefinedTasks(Collections.singleton(sstable), 100);
+        Assert.assertEquals(1, tasks.size());
     }
 
     @Test
     public void mixedPendingSessionsTest()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
         TimeUUID repairId = registerSession(cfs, true, true);
         TimeUUID repairId2 = registerSession(cfs, true, true);
         SSTableReader sstable = makeSSTable(true);
@@ -248,10 +257,8 @@ public void mixedPendingSessionsTest()
         mutateRepaired(sstable2, repairId2, false);
         prm.addSSTable(sstable);
         prm.addSSTable(sstable2);
-        try (CompactionTasks tasks = csm.getUserDefinedTasks(Lists.newArrayList(sstable, sstable2), 100))
-        {
-            Assert.assertEquals(2, tasks.size());
-        }
+        Collection<AbstractCompactionTask> tasks = prm.createUserDefinedTasks(Lists.newArrayList(sstable, sstable2), 100);
+        Assert.assertEquals(2, tasks.size());
     }
 
     /**
@@ -261,7 +268,7 @@ public void mixedPendingSessionsTest()
     @Test(expected = PendingRepairManager.IllegalSSTableArgumentException.class)
     public void getScannersInvalidSSTable() throws Exception
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
         SSTableReader sstable = makeSSTable(true);
         prm.getScanners(Collections.singleton(sstable), Collections.singleton(RANGE1));
     }
@@ -273,7 +280,7 @@ public void getScannersInvalidSSTable() throws Exception
     @Test(expected = PendingRepairManager.IllegalSSTableArgumentException.class)
     public void getOrCreateInvalidSSTable() throws Exception
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
         SSTableReader sstable = makeSSTable(true);
         prm.getOrCreate(sstable);
     }
@@ -281,7 +288,7 @@ public void getOrCreateInvalidSSTable() throws Exception
     @Test
     public void sessionHasData()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
 
         TimeUUID repairID = registerSession(cfs, true, true);
         LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
@@ -296,14 +303,14 @@ public void sessionHasData()
     @Test
     public void noEmptyCompactionTask()
     {
-        PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
+        PendingRepairManager prm = new PendingRepairManager(cfs, strategyFactory, cfs.getCompactionParams(), false);
         SSTableReader sstable = makeSSTable(false);
         TimeUUID id = nextTimeUUID();
         mutateRepaired(sstable, id, false);
         prm.getOrCreate(sstable);
         cfs.truncateBlocking();
         Assert.assertFalse(cfs.getSSTables(SSTableSet.LIVE).iterator().hasNext());
-        Assert.assertNull(cfs.getCompactionStrategyManager().getNextBackgroundTask(0));
+        Assert.assertTrue(cfs.getCompactionStrategy().getNextBackgroundTasks(0).isEmpty());
 
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/ShardManagerReplicaAwareTest.java b/test/unit/org/apache/cassandra/db/compaction/ShardManagerReplicaAwareTest.java
new file mode 100644
index 000000000000..dc6a6d47b8f9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/ShardManagerReplicaAwareTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.dht.tokenallocator.TokenAllocation;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.locator.RackInferringSnitch;
+import org.apache.cassandra.locator.TokenMetadata;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ShardManagerReplicaAwareTest
+{
+
+    @Test
+    public void testRangeEndsForShardCountEqualtToNumTokensPlusOne() throws UnknownHostException
+    {
+        var mockCompationRealm = mock(CompactionRealm.class);
+        when(mockCompationRealm.estimatedPartitionCountInSSTables()).thenReturn(1L<<16);
+
+        for (int numTokens = 1; numTokens < 32; numTokens++)
+        {
+            var rs = buildStrategy(numTokens, 1, 1, 1);
+            var expectedTokens = rs.getTokenMetadata().sortedTokens();
+            var shardManager = new ShardManagerReplicaAware(rs, mockCompationRealm);
+
+            var shardCount = numTokens + 1;
+            var iterator = shardManager.boundaries(shardCount);
+            assertEquals(Murmur3Partitioner.instance.getMinimumToken(), iterator.shardStart());
+            var actualTokens = new ArrayList<Token>();
+            for (Token end = iterator.shardEnd(); end != null; end = iterator.shardEnd())
+            {
+                assertFalse(iterator.advanceTo(end));
+                assertTrue(iterator.advanceTo(end.nextValidToken()));
+                actualTokens.add(end);
+            }
+            assertEquals(expectedTokens, actualTokens);
+        }
+    }
+
+    @Test
+    public void testRangeEndsAreFromTokenListAndContainLowerRangeEnds() throws UnknownHostException
+    {
+        var mockCompationRealm = mock(CompactionRealm.class);
+        when(mockCompationRealm.estimatedPartitionCountInSSTables()).thenReturn(1L<<16);
+
+        for (int nodeCount = 1; nodeCount <= 6; nodeCount++)
+        {
+            for (int numTokensPerNode = 1; numTokensPerNode < 16; numTokensPerNode++)
+            {
+                // Confirm it works for multiple base shard counts.
+                for (int baseShardCount = 1; baseShardCount <= 3; baseShardCount++)
+                {
+                    // Testing with 1 rack, nodeCount nodes, and rf 1.
+                    var rs = buildStrategy(numTokensPerNode, 1, nodeCount, 1);
+                    var initialSplitPoints = rs.getTokenMetadata().sortedTokens();
+                    // Confirm test set up is correct.
+                    assertEquals(numTokensPerNode * nodeCount, initialSplitPoints.size());
+                    // Use a shared instance to
+                    var shardManager = new ShardManagerReplicaAware(rs, mockCompationRealm);
+
+                    // The tokens for one level lower.
+                    var lowerTokens = new ArrayList<Token>();
+                    var tokenLimit = numTokensPerNode * nodeCount * 8;
+                    for (int shardExponent = 0; baseShardCount * Math.pow(2, shardExponent) <= tokenLimit; shardExponent++)
+                    {
+                        var shardCount = baseShardCount * (int) Math.pow(2, shardExponent);
+                        var iterator = shardManager.boundaries(shardCount);
+                        assertEquals(Murmur3Partitioner.instance.getMinimumToken(), iterator.shardStart());
+                        assertEquals(shardCount, iterator.count());
+                        var actualSplitPoints = new ArrayList<Token>();
+                        var shardSpanSize = 0d;
+                        var index = 0;
+                        for (Token end = iterator.shardEnd(); end != null; end = iterator.shardEnd())
+                        {
+                            shardSpanSize += iterator.shardSpanSize();
+                            assertEquals(index++, iterator.shardIndex());
+                            assertFalse(iterator.advanceTo(end));
+                            assertTrue(iterator.advanceTo(end.nextValidToken()));
+                            actualSplitPoints.add(end);
+                        }
+                        // Need to add the last shard span size because we exit the above loop before adding it.
+                        shardSpanSize += iterator.shardSpanSize();
+                        // Confirm the shard span size adds to about 1
+                        assertEquals(1d, shardSpanSize, 0.001);
+
+                        // If we have more split points than the initialSplitPoints, we had to compute additional
+                        // tokens, so the best we can do is confirm containment.
+                        if (actualSplitPoints.size() >= initialSplitPoints.size())
+                            assertTrue(actualSplitPoints + " does not contain " + initialSplitPoints,
+                                       actualSplitPoints.containsAll(initialSplitPoints));
+                        else
+                            assertTrue(initialSplitPoints + " does not contain " + actualSplitPoints,
+                                       initialSplitPoints.containsAll(actualSplitPoints));
+
+                        // Higher tokens must always contain lower tokens.
+                        assertTrue(actualSplitPoints + " does not contain " + lowerTokens,
+                                   actualSplitPoints.containsAll(lowerTokens));
+                        lowerTokens = actualSplitPoints;
+                    }
+                }
+            }
+        }
+    }
+
+
+    private AbstractReplicationStrategy buildStrategy(int numTokens, int numRacks, int numNodes, int rf) throws UnknownHostException
+    {
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        DatabaseDescriptor.setEndpointSnitch(new RackInferringSnitch());
+        var config = new Config();
+        config.num_tokens = numTokens;
+        DatabaseDescriptor.setConfig(config);
+        var tokenMetadata = new TokenMetadata();
+        var snitch = DatabaseDescriptor.getEndpointSnitch();
+        var dc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+        // Configure rf
+        var options = Map.of(dc, Integer.toString(rf));
+        var networkTopology = new NetworkTopologyStrategy("0", tokenMetadata, snitch, options);
+
+        for (int i = 0; i < numRacks; i++)
+            generateFakeEndpoints(tokenMetadata, networkTopology, 1, numNodes, numTokens, dc, Integer.toString(i));
+
+        return networkTopology;
+    }
+
+    // Generates endpoints and adds them to the tmd and the rs.
+    private List<Token> generateFakeEndpoints(TokenMetadata tmd, AbstractReplicationStrategy rs, int firstNodeId, int lastNodId, int vnodes, String dc, String rack) throws UnknownHostException
+    {
+        System.out.printf("Adding nodes %d through %d to dc=%s, rack=%s.%n", firstNodeId, lastNodId, dc, rack);
+        var result = new ArrayList<Token>();
+        for (int i = firstNodeId; i <= lastNodId; i++)
+        {
+            // leave .1 for myEndpoint
+            InetAddressAndPort addr = InetAddressAndPort.getByName("127." + dc + '.' + rack + '.' + (i + 1));
+            var tokens = TokenAllocation.allocateTokens(tmd, rs, addr, vnodes);
+            // TODO why don't we need addBootstrapTokens here? The test only passes with updateNormalTokens.
+            // tmd.addBootstrapTokens(tokens, addr);
+            tmd.updateNormalTokens(tokens, addr);
+            result.addAll(tokens);
+        }
+        return result;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/ShardManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/ShardManagerTest.java
index 9ea03e695b87..36a4eaa876f1 100644
--- a/test/unit/org/apache/cassandra/db/compaction/ShardManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/ShardManagerTest.java
@@ -21,27 +21,28 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.google.common.collect.ImmutableList;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 import org.agrona.collections.IntArrayList;
-import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.BufferDecoratedKey;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DiskBoundaries;
-import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.SortedLocalRanges;
+import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.utils.Pair;
 import org.mockito.Mockito;
 
 import static org.junit.Assert.assertEquals;
@@ -54,23 +55,27 @@ public class ShardManagerTest
     final IPartitioner partitioner = Murmur3Partitioner.instance;
     final Token minimumToken = partitioner.getMinimumToken();
 
-    ColumnFamilyStore.VersionedLocalRanges weightedRanges;
+    SortedLocalRanges localRanges;
+    List<Splitter.WeightedRange> weightedRanges;
 
     static final double delta = 1e-15;
 
     @Before
     public void setUp()
     {
-        DatabaseDescriptor.daemonInitialization(); // because of all the static initialization in CFS
-        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
-        weightedRanges = new ColumnFamilyStore.VersionedLocalRanges(-1, 16);
+        weightedRanges = new ArrayList<>();
+        var realm = Mockito.mock(CompactionRealm.class);
+        localRanges = Mockito.mock(SortedLocalRanges.class, Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS));
+        Mockito.when(localRanges.getRanges()).thenAnswer(invocation -> weightedRanges);
+        Mockito.when(localRanges.getRealm()).thenReturn(realm);
+        Mockito.when(realm.estimatedPartitionCountInSSTables()).thenReturn(10000L);
     }
 
     @Test
     public void testRangeSpannedFullOwnership()
     {
         weightedRanges.add(new Splitter.WeightedRange(1.0, new Range<>(minimumToken, minimumToken)));
-        ShardManager shardManager = new ShardManagerNoDisks(weightedRanges);
+        ShardManager shardManager = new ShardManagerNoDisks(localRanges);
 
         // sanity check
         assertEquals(0.4, tokenAt(0.1).size(tokenAt(0.5)), delta);
@@ -79,8 +84,12 @@ public void testRangeSpannedFullOwnership()
         assertEquals(0.2, shardManager.rangeSpanned(range(0.3, 0.5)), delta);
 
         assertEquals(0.2, shardManager.rangeSpanned(mockedTable(0.5, 0.7, Double.NaN)), delta);
-        // single-partition correction
-        assertEquals(1.0, shardManager.rangeSpanned(mockedTable(0.3, 0.3, Double.NaN)), delta);
+        // single-token-span correction
+        assertEquals(0.02, shardManager.rangeSpanned(mockedTable(0.3, 0.3, Double.NaN, 200)), delta);
+        // small partition count correction
+        assertEquals(0.0001, shardManager.rangeSpanned(mockedTable(0.3, 0.30001, Double.NaN, 1)), delta);
+        assertEquals(0.001, shardManager.rangeSpanned(mockedTable(0.3, 0.30001, Double.NaN, 10)), delta);
+        assertEquals(0.01, shardManager.rangeSpanned(mockedTable(0.3, 0.31, Double.NaN, 10)), delta);
 
         // reported coverage
         assertEquals(0.1, shardManager.rangeSpanned(mockedTable(0.5, 0.7, 0.1)), delta);
@@ -89,7 +98,7 @@ public void testRangeSpannedFullOwnership()
         assertEquals(0.2, shardManager.rangeSpanned(mockedTable(0.5, 0.7, -1)), delta);
 
         // correction over coverage
-        assertEquals(1.0, shardManager.rangeSpanned(mockedTable(0.3, 0.5, 1e-50)), delta);
+        assertEquals(0.02, shardManager.rangeSpanned(mockedTable(0.3, 0.5, 1e-50, 200)), delta);
     }
 
     @Test
@@ -105,7 +114,7 @@ public void testRangeSpannedPartialOwnership()
         weightedRanges.add(new Splitter.WeightedRange(1.0, new Range<>(tokenAt(0.98), tokenAt(1.0))));
         double total = weightedRanges.stream().mapToDouble(wr -> wr.range().left.size(wr.range().right)).sum();
 
-        ShardManager shardManager = new ShardManagerNoDisks(weightedRanges);
+        ShardManager shardManager = new ShardManagerNoDisks(localRanges);
 
         // sanity check
         assertEquals(0.4, tokenAt(0.1).size(tokenAt(0.5)), delta);
@@ -119,10 +128,11 @@ public void testRangeSpannedPartialOwnership()
         assertEquals(0.1, shardManager.rangeSpanned(mockedTable(0.5, 0.8, Double.NaN)), delta);
 
         // single-partition correction
-        assertEquals(1.0, shardManager.rangeSpanned(mockedTable(0.3, 0.3, Double.NaN)), delta);
+        assertEquals(0.02 * total, shardManager.rangeSpanned(mockedTable(0.3, 0.3, Double.NaN, 200)), delta);
         // out-of-local-range correction
-        assertEquals(1.0, shardManager.rangeSpanned(mockedTable(0.6, 0.7, Double.NaN)), delta);
-        assertEquals(0.001, shardManager.rangeSpanned(mockedTable(0.6, 0.701, Double.NaN)), delta);
+        assertEquals(0.03, shardManager.rangeSpanned(mockedTable(0.6, 0.73, Double.NaN, 200)), delta);
+        // completely outside should use partition-based count
+        assertEquals(0.02 * total, shardManager.rangeSpanned(mockedTable(0.6, 0.7, Double.NaN, 200)), delta);
 
         // reported coverage
         assertEquals(0.1, shardManager.rangeSpanned(mockedTable(0.5, 0.7, 0.1)), delta);
@@ -131,7 +141,7 @@ public void testRangeSpannedPartialOwnership()
         assertEquals(0.1, shardManager.rangeSpanned(mockedTable(0.5, 0.8, -1)), delta);
 
         // correction over coverage, no recalculation
-        assertEquals(1.0, shardManager.rangeSpanned(mockedTable(0.5, 0.8, 1e-50)), delta);
+        assertEquals(0.02 * total, shardManager.rangeSpanned(mockedTable(0.5, 0.8, 1e-50, 200)), delta);
     }
 
     @Test
@@ -147,7 +157,7 @@ public void testRangeSpannedWeighted()
         weightedRanges.add(new Splitter.WeightedRange(1.0, new Range<>(tokenAt(0.98), tokenAt(1.0))));
         double total = weightedRanges.stream().mapToDouble(wr -> wr.size()).sum();
 
-        ShardManager shardManager = new ShardManagerNoDisks(weightedRanges);
+        ShardManager shardManager = new ShardManagerNoDisks(localRanges);
 
         // sanity check
         assertEquals(0.4, tokenAt(0.1).size(tokenAt(0.5)), delta);
@@ -161,10 +171,10 @@ public void testRangeSpannedWeighted()
         assertEquals(0.06, shardManager.rangeSpanned(mockedTable(0.5, 0.8, Double.NaN)), delta);
 
         // single-partition correction
-        assertEquals(1.0, shardManager.rangeSpanned(mockedTable(0.3, 0.3, Double.NaN)), delta);
+        assertEquals(0.02 * total, shardManager.rangeSpanned(mockedTable(0.3, 0.3, Double.NaN, 200)), delta);
         // out-of-local-range correction
-        assertEquals(1.0, shardManager.rangeSpanned(mockedTable(0.6, 0.7, Double.NaN)), delta);
-        assertEquals(0.001, shardManager.rangeSpanned(mockedTable(0.6, 0.701, Double.NaN)), delta);
+        assertEquals(0.02 * total, shardManager.rangeSpanned(mockedTable(0.6, 0.7, Double.NaN, 200)), delta);
+        assertEquals(0.03, shardManager.rangeSpanned(mockedTable(0.6, 0.73, Double.NaN)), delta);
 
         // reported coverage
         assertEquals(0.1, shardManager.rangeSpanned(mockedTable(0.5, 0.7, 0.1)), delta);
@@ -173,7 +183,7 @@ public void testRangeSpannedWeighted()
         assertEquals(0.06, shardManager.rangeSpanned(mockedTable(0.5, 0.8, -1)), delta);
 
         // correction over coverage, no recalculation
-        assertEquals(1.0, shardManager.rangeSpanned(mockedTable(0.5, 0.8, 1e-50)), delta);
+        assertEquals(0.02 * total, shardManager.rangeSpanned(mockedTable(0.5, 0.8, 1e-50, 200)), delta);
     }
 
     Token tokenAt(double pos)
@@ -181,23 +191,23 @@ Token tokenAt(double pos)
         return partitioner.split(minimumToken, minimumToken, pos);
     }
 
-    DecoratedKey keyAt(double pos)
+    Range<Token> range(double start, double end)
     {
-        Token token = tokenAt(pos);
-        return new BufferDecoratedKey(token, ByteBuffer.allocate(0));
+        return new Range<>(tokenAt(start), tokenAt(end));
     }
 
-    Range<Token> range(double start, double end)
+    CompactionSSTable mockedTable(double start, double end, double reportedCoverage)
     {
-        return new Range<>(tokenAt(start), tokenAt(end));
+        return mockedTable(start, end, reportedCoverage, ShardManager.PER_PARTITION_SPAN_THRESHOLD * 2);
     }
 
-    SSTableReader mockedTable(double start, double end, double reportedCoverage)
+    CompactionSSTable mockedTable(double start, double end, double reportedCoverage, long estimatedKeys)
     {
-        SSTableReader mock = Mockito.mock(SSTableReader.class);
-        Mockito.when(mock.getFirst()).thenReturn(keyAt(start));
-        Mockito.when(mock.getLast()).thenReturn(keyAt(end));
+        CompactionSSTable mock = Mockito.mock(CompactionSSTable.class);
+        Mockito.when(mock.getFirst()).thenReturn(tokenAt(start).minKeyBound());
+        Mockito.when(mock.getLast()).thenReturn(tokenAt(end).minKeyBound());
         Mockito.when(mock.tokenSpaceCoverage()).thenReturn(reportedCoverage);
+        Mockito.when(mock.estimatedKeys()).thenReturn(estimatedKeys);
         return mock;
     }
 
@@ -275,57 +285,57 @@ private int[] ints(int... values)
 
     private void testShardBoundaries(int[] expected, int numShards, int numDisks, int[] rangeBounds)
     {
-        ColumnFamilyStore cfs = Mockito.mock(ColumnFamilyStore.class);
-        when(cfs.getPartitioner()).thenReturn(partitioner);
+        CompactionRealm realm = Mockito.mock(CompactionRealm.class);
+        when(realm.getPartitioner()).thenReturn(partitioner);
 
-        List<Range<Token>> ranges = new ArrayList<>();
+        List<Splitter.WeightedRange> ranges = new ArrayList<>();
         for (int i = 0; i < rangeBounds.length; i += 2)
-            ranges.add(new Range<>(getToken(rangeBounds[i + 0]), getToken(rangeBounds[i + 1])));
-        ranges = Range.sort(ranges);
-        ColumnFamilyStore.VersionedLocalRanges sortedRanges = localRanges(ranges.stream().map(x -> new Splitter.WeightedRange(1.0, x)).collect(Collectors.toList()));
+            ranges.add(new Splitter.WeightedRange(1.0, new Range<>(getToken(rangeBounds[i + 0]), getToken(rangeBounds[i + 1]))));
+        SortedLocalRanges sortedRanges = SortedLocalRanges.forTesting(realm, ranges);
 
-        List<Token> diskBoundaries = splitRanges(sortedRanges, numDisks);
-        int[] result = getShardBoundaries(cfs, numShards, diskBoundaries, sortedRanges);
+        List<Token> diskBoundaries = sortedRanges.split(numDisks);
+        int[] result = getShardBoundaries(numShards, diskBoundaries, sortedRanges);
         Assert.assertArrayEquals("Disks " + numDisks + " shards " + numShards + " expected " + Arrays.toString(expected) + " was " + Arrays.toString(result), expected, result);
     }
 
     private void testShardBoundariesWeighted(int[] expected, int numShards, int numDisks, int[] rangeBounds)
     {
-        ColumnFamilyStore cfs = Mockito.mock(ColumnFamilyStore.class);
-        when(cfs.getPartitioner()).thenReturn(partitioner);
+        CompactionRealm realm = Mockito.mock(CompactionRealm.class);
+        when(realm.getPartitioner()).thenReturn(partitioner);
 
         List<Splitter.WeightedRange> ranges = new ArrayList<>();
         for (int i = 0; i < rangeBounds.length; i += 2)
             ranges.add(new Splitter.WeightedRange(2.0 / (rangeBounds.length - i), new Range<>(getToken(rangeBounds[i + 0]), getToken(rangeBounds[i + 1]))));
-        ColumnFamilyStore.VersionedLocalRanges sortedRanges = localRanges(ranges);
+        SortedLocalRanges sortedRanges = SortedLocalRanges.forTesting(realm, ranges);
 
-        List<Token> diskBoundaries = splitRanges(sortedRanges, numDisks);
-        int[] result = getShardBoundaries(cfs, numShards, diskBoundaries, sortedRanges);
+        List<Token> diskBoundaries = sortedRanges.split(numDisks);
+        int[] result = getShardBoundaries(numShards, diskBoundaries, sortedRanges);
         Assert.assertArrayEquals("Disks " + numDisks + " shards " + numShards + " expected " + Arrays.toString(expected) + " was " + Arrays.toString(result), expected, result);
     }
 
     private void testShardBoundaries(int[] expected, int numShards, int[] diskPositions, int[] rangeBounds)
     {
-        ColumnFamilyStore cfs = Mockito.mock(ColumnFamilyStore.class);
-        when(cfs.getPartitioner()).thenReturn(partitioner);
+        CompactionRealm realm = Mockito.mock(CompactionRealm.class);
+        when(realm.getPartitioner()).thenReturn(partitioner);
 
         List<Splitter.WeightedRange> ranges = new ArrayList<>();
         for (int i = 0; i < rangeBounds.length; i += 2)
             ranges.add(new Splitter.WeightedRange(1.0, new Range<>(getToken(rangeBounds[i + 0]), getToken(rangeBounds[i + 1]))));
-        ColumnFamilyStore.VersionedLocalRanges sortedRanges = localRanges(ranges);
+        SortedLocalRanges sortedRanges = SortedLocalRanges.forTesting(realm, ranges);
 
         List<Token> diskBoundaries = Arrays.stream(diskPositions).mapToObj(this::getToken).collect(Collectors.toList());
-        int[] result = getShardBoundaries(cfs, numShards, diskBoundaries, sortedRanges);
+        int[] result = getShardBoundaries(numShards, diskBoundaries, sortedRanges);
         Assert.assertArrayEquals("Disks " + Arrays.toString(diskPositions) + " shards " + numShards + " expected " + Arrays.toString(expected) + " was " + Arrays.toString(result), expected, result);
     }
 
-    private int[] getShardBoundaries(ColumnFamilyStore cfs, int numShards, List<Token> diskBoundaries, ColumnFamilyStore.VersionedLocalRanges sortedRanges)
+    private int[] getShardBoundaries(int numShards, List<Token> diskBoundaries, SortedLocalRanges sortedRanges)
     {
-        DiskBoundaries db = makeDiskBoundaries(cfs, diskBoundaries);
-        when(cfs.localRangesWeighted()).thenReturn(sortedRanges);
-        when(cfs.getDiskBoundaries()).thenReturn(db);
+        DiskBoundaries db = Mockito.mock(DiskBoundaries.class);
+        when(db.getLocalRanges()).thenReturn(sortedRanges);
+        when(db.getPositions()).thenReturn(diskBoundaries);
 
-        final ShardTracker shardTracker = ShardManager.create(cfs)
+        var rs = Mockito.mock(AbstractReplicationStrategy.class);
+        final ShardTracker shardTracker = ShardManager.create(db, rs, false)
                                                       .boundaries(numShards);
         IntArrayList list = new IntArrayList();
         for (int i = 0; i < 100; ++i)
@@ -336,35 +346,6 @@ private int[] getShardBoundaries(ColumnFamilyStore cfs, int numShards, List<Toke
         return list.toIntArray();
     }
 
-    ColumnFamilyStore.VersionedLocalRanges localRanges(List<Splitter.WeightedRange> ranges)
-    {
-        ColumnFamilyStore.VersionedLocalRanges versionedLocalRanges = new ColumnFamilyStore.VersionedLocalRanges(-1, ranges.size());
-        versionedLocalRanges.addAll(ranges);
-        return versionedLocalRanges;
-    }
-
-    ColumnFamilyStore.VersionedLocalRanges localRangesFull()
-    {
-        List<Splitter.WeightedRange> ranges = ImmutableList.of(new Splitter.WeightedRange(1.0,
-                                                                                          new Range<>(partitioner.getMinimumToken(),
-                                                                                                      partitioner.getMinimumToken())));
-        ColumnFamilyStore.VersionedLocalRanges versionedLocalRanges = new ColumnFamilyStore.VersionedLocalRanges(-1, ranges.size());
-        versionedLocalRanges.addAll(ranges);
-        return versionedLocalRanges;
-    }
-
-    List<Token> splitRanges(ColumnFamilyStore.VersionedLocalRanges ranges, int numDisks)
-    {
-        return ranges.get(0).left().getPartitioner().splitter().get().splitOwnedRanges(numDisks, ranges, false);
-    }
-
-    private static DiskBoundaries makeDiskBoundaries(ColumnFamilyStore cfs, List<Token> diskBoundaries)
-    {
-        List<PartitionPosition> diskPositions = diskBoundaries.stream().map(Token::maxKeyBound).collect(Collectors.toList());
-        DiskBoundaries db = new DiskBoundaries(cfs, null, diskPositions, -1, -1);
-        return db;
-    }
-
     private Token getToken(int x)
     {
         return tokenAt(x / 100.0);
@@ -378,18 +359,20 @@ private int fromToken(Token t)
     @Test
     public void testRangeEnds()
     {
-        ColumnFamilyStore cfs = Mockito.mock(ColumnFamilyStore.class);
-        when(cfs.getPartitioner()).thenReturn(partitioner);
-        ColumnFamilyStore.VersionedLocalRanges sortedRanges = localRangesFull();
+        CompactionRealm realm = Mockito.mock(CompactionRealm.class);
+        when(realm.getPartitioner()).thenReturn(partitioner);
+        SortedLocalRanges sortedRanges = SortedLocalRanges.forTestingFull(realm);
 
         for (int numDisks = 1; numDisks <= 3; ++numDisks)
         {
-            List<Token> diskBoundaries = splitRanges(sortedRanges, numDisks);
-            DiskBoundaries db = makeDiskBoundaries(cfs, diskBoundaries);
-            when(cfs.localRangesWeighted()).thenReturn(sortedRanges);
-            when(cfs.getDiskBoundaries()).thenReturn(db);
+            List<Token> diskBoundaries = sortedRanges.split(numDisks);
+            DiskBoundaries db = Mockito.mock(DiskBoundaries.class);
+            when(db.getLocalRanges()).thenReturn(sortedRanges);
+            when(db.getPositions()).thenReturn(diskBoundaries);
+
+            var rs = Mockito.mock(AbstractReplicationStrategy.class);
 
-            ShardManager shardManager = ShardManager.create(cfs);
+            ShardManager shardManager = ShardManager.create(db, rs, false);
             for (int numShards = 1; numShards <= 3; ++numShards)
             {
                 ShardTracker iterator = shardManager.boundaries(numShards);
@@ -403,7 +386,233 @@ public void testRangeEnds()
                     ++count;
                 }
                 assertEquals(numDisks * numShards, count);
+
+                assertEquals(numDisks * numShards, iterator.count());
+            }
+        }
+    }
+
+    @Test
+    public void testSpannedShardCount()
+    {
+        CompactionRealm realm = Mockito.mock(CompactionRealm.class);
+        when(realm.getPartitioner()).thenReturn(partitioner);
+        SortedLocalRanges sortedRanges = SortedLocalRanges.forTestingFull(realm);
+
+        for (int numDisks = 1; numDisks <= 3; ++numDisks)
+        {
+            List<Token> diskBoundaries = sortedRanges.split(numDisks);
+            DiskBoundaries db = Mockito.mock(DiskBoundaries.class);
+            when(db.getLocalRanges()).thenReturn(sortedRanges);
+            when(db.getPositions()).thenReturn(diskBoundaries);
+
+            var rs = Mockito.mock(AbstractReplicationStrategy.class);
+
+            ShardManager shardManager = ShardManager.create(db, rs, false);
+            for (int numShards = 1; numShards <= 3; ++numShards)
+            {
+                checkCoveredCount(shardManager, numDisks, numShards, 0.01, 0.99);
+                checkCoveredCount(shardManager, numDisks, numShards, 0.01, 0.49);
+                checkCoveredCount(shardManager, numDisks, numShards, 0.51, 0.99);
+                checkCoveredCount(shardManager, numDisks, numShards, 0.26, 0.74);
+
+                for (double l = 0; l <= 1; l += 0.05)
+                    for (double r = l; r <= 1; r += 0.05)
+                        checkCoveredCount(shardManager, numDisks, numShards, l, r);
+            }
+        }
+    }
+
+    private void checkCoveredCount(ShardManager shardManager, int numDisks, int numShards, double left, double right)
+    {
+        Token min = partitioner.getMinimumToken();
+        int totalSplits = numDisks * numShards;
+        int leftIdx = left == 0 ? 0 : (int) Math.ceil(left * totalSplits) - 1; // to reflect end-inclusiveness of ranges
+        int rightIdx = right == 0 ? 0 : (int) Math.ceil(right * totalSplits) - 1;
+        assertEquals(String.format("numDisks %d numShards %d left %f right %f", numDisks, numShards, left, right),
+                     rightIdx - leftIdx + 1,
+                     shardManager.coveredShardCount(partitioner.split(min, min, left).maxKeyBound(),
+                                                    partitioner.split(min, min, right).maxKeyBound(),
+                                                    numShards));
+    }
+
+    @Test
+    public void testGetShardRanges()
+    {
+        CompactionRealm realm = Mockito.mock(CompactionRealm.class);
+        when(realm.getPartitioner()).thenReturn(partitioner);
+        SortedLocalRanges sortedRanges = SortedLocalRanges.forTestingFull(realm);
+
+        for (int numDisks = 1; numDisks <= 3; ++numDisks)
+        {
+            List<Token> diskBoundaries = sortedRanges.split(numDisks);
+            DiskBoundaries db = Mockito.mock(DiskBoundaries.class);
+            when(db.getLocalRanges()).thenReturn(sortedRanges);
+            when(db.getPositions()).thenReturn(diskBoundaries);
+
+            var rs = Mockito.mock(AbstractReplicationStrategy.class);
+
+            ShardManager shardManager = ShardManager.create(db, rs, false);
+            for (int numShardsPerDisk = 1; numShardsPerDisk <= 3; ++numShardsPerDisk)
+            {
+                var ranges = shardManager.getShardRanges(numShardsPerDisk);
+                var boundaries = shardManager.boundaries(numShardsPerDisk);
+                assertEquals(numShardsPerDisk * numDisks, ranges.size());
+                for (int i = 0; i < ranges.size(); ++i)
+                {
+                    Range<Token> range = ranges.get(i);
+                    boundaries.advanceTo(range.left.nextValidToken());
+                    assertEquals(i, boundaries.shardIndex());
+                    boundaries.advanceTo(partitioner.split(range.left, range.right, 0.5));
+                    assertEquals(i, boundaries.shardIndex());
+                    boundaries.advanceTo(range.right);
+                    assertEquals(i, boundaries.shardIndex());
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testSplitSSTablesInRanges()
+    {
+        testSplitSSTablesInRanges(8, ints(1, 2, 4));
+        testSplitSSTablesInRanges(4, ints(1, 2, 4));
+        testSplitSSTablesInRanges(2, ints(1, 2, 4));
+        testSplitSSTablesInRanges(5, ints(1, 2, 4));
+        testSplitSSTablesInRanges(5, ints(2, 4, 8));
+        testSplitSSTablesInRanges(3, ints(1, 3, 5));
+        testSplitSSTablesInRanges(3, ints(3, 3, 3));
+
+        testSplitSSTablesInRanges(1, ints(1, 2, 3));
+
+        testSplitSSTablesInRanges(3, ints());
+    }
+
+    @Test
+    public void testSplitSSTablesInRangesMissingParts()
+    {
+        // Drop some sstables without losing ranges
+        testSplitSSTablesInRanges(8, ints(2, 4, 8),
+                                ints(1));
+
+        testSplitSSTablesInRanges(8, ints(2, 4, 8),
+                                ints(1), ints(0), ints(2, 7));
+
+        testSplitSSTablesInRanges(5, ints(2, 4, 8),
+                                ints(1), ints(0), ints(2, 7));
+    }
+
+    @Test
+    public void testSplitSSTablesInRangesOneRange()
+    {
+        // Drop second half
+        testSplitSSTablesInRanges(2, ints(2, 4, 8),
+                                ints(1), ints(2, 3), ints(4, 5, 6, 7));
+        // Drop all except center, within shard
+        testSplitSSTablesInRanges(3, ints(5, 7, 9),
+                                ints(0, 1, 3, 4), ints(0, 1, 2, 4, 5, 6), ints(0, 1, 2, 6, 7, 8));
+    }
+
+    @Test
+    public void testSplitSSTablesInRangesSkippedRange()
+    {
+        // Drop all sstables containing the 4/8-5/8 range.
+        testSplitSSTablesInRanges(8, ints(2, 4, 8),
+                                ints(1), ints(2), ints(4));
+        // Drop all sstables containing the 4/8-6/8 range.
+        testSplitSSTablesInRanges(8, ints(2, 4, 8),
+                                ints(1), ints(2), ints(4, 5));
+        // Drop all sstables containing the 4/8-8/8 range.
+        testSplitSSTablesInRanges(8, ints(2, 4, 8),
+                                ints(1), ints(2, 3), ints(4, 5, 6, 7));
+
+        // Drop all sstables containing the 0/8-2/8 range.
+        testSplitSSTablesInRanges(5, ints(2, 4, 8),
+                                ints(0), ints(0), ints(0, 1));
+        // Drop all sstables containing the 6/8-8/8 range.
+        testSplitSSTablesInRanges(5, ints(2, 4, 8),
+                                ints(1), ints(3), ints(6, 7));
+        // Drop sstables on both ends.
+        testSplitSSTablesInRanges(5, ints(3, 4, 8),
+                                ints(0, 2), ints(0, 3), ints(0, 1, 6, 7));
+    }
+
+    public void testSplitSSTablesInRanges(int numShards, int[] perLevelCounts, int[]... dropsPerLevel)
+    {
+        weightedRanges.clear();
+        weightedRanges.add(new Splitter.WeightedRange(1.0, new Range<>(minimumToken, minimumToken)));
+        ShardManager manager = new ShardManagerNoDisks(localRanges);
+
+        Set<CompactionSSTable> allSSTables = new HashSet<>();
+        int levelNum = 0;
+        for (int perLevelCount : perLevelCounts)
+        {
+            List<CompactionSSTable> ssTables = mockNonOverlappingSSTables(perLevelCount);
+            if (levelNum < dropsPerLevel.length)
+            {
+                for (int i = dropsPerLevel[levelNum].length - 1; i >= 0; i--)
+                    ssTables.remove(dropsPerLevel[levelNum][i]);
             }
+            allSSTables.addAll(ssTables);
+            ++levelNum;
+        }
+
+        var results = new ArrayList<Pair<Range<Token>, Set<CompactionSSTable>>>();
+        manager.splitSSTablesInShards(allSSTables, numShards, (sstables, range) -> results.add(Pair.create(range, Set.copyOf(sstables))));
+        int i = 0;
+        int[] expectedSSTablesInTasks = new int[results.size()];
+        int[] collectedSSTablesPerTask = new int[results.size()];
+        for (var t : results)
+        {
+            collectedSSTablesPerTask[i] = t.right().size();
+            expectedSSTablesInTasks[i] = (int) allSSTables.stream().filter(x -> intersects(x, t.left())).count();
+            ++i;
+        }
+        Assert.assertEquals(Arrays.toString(expectedSSTablesInTasks), Arrays.toString(collectedSSTablesPerTask));
+        System.out.println(Arrays.toString(expectedSSTablesInTasks));
+    }
+
+    private boolean intersects(CompactionSSTable r, Range<Token> range)
+    {
+        if (range == null)
+            return true;
+        return range.intersects(range(r));
+    }
+
+
+    private Bounds<Token> range(CompactionSSTable x)
+    {
+        return new Bounds<>(x.getFirst().getToken(), x.getLast().getToken());
+    }
+
+    List<CompactionSSTable> mockNonOverlappingSSTables(int numSSTables)
+    {
+        if (!partitioner.splitter().isPresent())
+            throw new IllegalStateException(String.format("Cannot split ranges with current partitioner %s", partitioner));
+
+        ByteBuffer emptyBuffer = ByteBuffer.allocate(0);
+
+        List<CompactionSSTable> sstables = new ArrayList<>(numSSTables);
+        for (int i = 0; i < numSSTables; i++)
+        {
+            DecoratedKey first = new BufferDecoratedKey(boundary(numSSTables, i).nextValidToken(), emptyBuffer);
+            DecoratedKey last =  new BufferDecoratedKey(boundary(numSSTables, i+1), emptyBuffer);
+            sstables.add(mockSSTable(first, last));
         }
+
+        return sstables;
+    }
+
+    private Token boundary(int numSSTables, int i)
+    {
+        return partitioner.split(partitioner.getMinimumToken(), partitioner.getMaximumToken(), i * 1.0 / numSSTables);
+    }
+
+    private CompactionSSTable mockSSTable(DecoratedKey first, DecoratedKey last)
+    {
+        CompactionSSTable sstable = Mockito.mock(CompactionSSTable.class);
+        when(sstable.getFirst()).thenReturn(first);
+        when(sstable.getLast()).thenReturn(last);
+        return sstable;
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/SharedCompactionObserverTest.java b/test/unit/org/apache/cassandra/db/compaction/SharedCompactionObserverTest.java
new file mode 100644
index 000000000000..13a2826927d6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/SharedCompactionObserverTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.mockito.Mockito.*;
+
+public class SharedCompactionObserverTest
+{
+    private SharedCompactionObserver sharedCompactionObserver;
+    private CompactionObserver mockObserver;
+    private CompactionProgress mockProgress;
+    private TimeUUID operationId;
+
+
+    @Before
+    public void setUp()
+    {
+        mockObserver = Mockito.mock(CompactionObserver.class);
+        sharedCompactionObserver = new SharedCompactionObserver(mockObserver);
+        operationId = TimeUUID.Generator.nextTimeUUID();
+        mockProgress = Mockito.mock(CompactionProgress.class);
+        when(mockProgress.operationId()).thenReturn(operationId);
+    }
+
+    @Test
+    public void testOnInProgress()
+    {
+        sharedCompactionObserver.onInProgress(mockProgress);
+        verify(mockObserver, times(1)).onInProgress(mockProgress);
+    }
+
+    @Test
+    public void testOnCompleted()
+    {
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.onInProgress(mockProgress);
+        sharedCompactionObserver.onCompleted(operationId, null);
+        verify(mockObserver, times(1)).onCompleted(operationId, null);
+    }
+
+    @Test
+    public void testOnCompletedFailure()
+    {
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.onInProgress(mockProgress);
+
+        Exception err = new RuntimeException();
+        sharedCompactionObserver.onCompleted(operationId, err);
+        verify(mockObserver, times(1)).onCompleted(operationId, err);
+    }
+
+    @Test
+    public void testMultipleSubtasksCompletion()
+    {
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.onInProgress(mockProgress);
+        sharedCompactionObserver.onInProgress(mockProgress);
+
+        sharedCompactionObserver.onCompleted(operationId, null);
+        verify(mockObserver, never()).onCompleted(any(TimeUUID.class), any());
+
+        sharedCompactionObserver.onCompleted(operationId, null);
+        verify(mockObserver, times(1)).onCompleted(operationId, null);
+    }
+
+    @Test
+    public void testMultipleSubtasksInProgressAfterCompletion()
+    {
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.onInProgress(mockProgress);
+        sharedCompactionObserver.onCompleted(operationId, null);
+        verify(mockObserver, never()).onCompleted(any(TimeUUID.class), any());
+
+        sharedCompactionObserver.onInProgress(mockProgress);
+        sharedCompactionObserver.onCompleted(operationId, null);
+        verify(mockObserver, times(1)).onCompleted(operationId, null);
+    }
+
+    @Test
+    public void testMultipleSubtasksCompletionWithFailure()
+    {
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.onInProgress(mockProgress);
+        sharedCompactionObserver.onInProgress(mockProgress);
+
+        sharedCompactionObserver.onCompleted(operationId, null);
+        verify(mockObserver, never()).onCompleted(any(TimeUUID.class), any());
+
+        Exception err = new RuntimeException();
+        sharedCompactionObserver.onCompleted(operationId, err);
+        verify(mockObserver, times(1)).onCompleted(operationId, err);
+    }
+
+    @Test
+    public void testConcurrentAccess() throws InterruptedException, ExecutionException
+    {
+        int threadCount = 100;
+        ExecutorService executor = Executors.newFixedThreadPool(threadCount);
+        CountDownLatch latch = new CountDownLatch(threadCount);
+
+        for (int i = 0; i < threadCount; i++)
+            sharedCompactionObserver.registerExpectedSubtask();
+        List<Future<?>> futures = new ArrayList<>();
+
+        for (int i = 0; i < threadCount; i++)
+        {
+            futures.add(executor.submit(() ->
+                                        {
+                                            sharedCompactionObserver.onInProgress(mockProgress);
+                                            if (ThreadLocalRandom.current().nextBoolean())
+                                                FBUtilities.sleepQuietly(ThreadLocalRandom.current().nextInt(1));
+                                            sharedCompactionObserver.onCompleted(operationId, null);
+                                            latch.countDown();
+                                        }));
+        }
+
+        for (Future<?> future : futures)
+            future.get();
+
+        verify(mockObserver, times(1)).onInProgress(mockProgress);
+        verify(mockObserver, times(1)).onCompleted(operationId, null);
+        executor.shutdown();
+    }
+
+    @Test
+    public void testErrorNoRegister()
+    {
+        Util.assumeAssertsEnabled();
+        Assert.assertThrows(AssertionError.class, () -> sharedCompactionObserver.onCompleted(operationId, null));
+    }
+
+    @Test
+    public void testErrorNoInProgress()
+    {
+        Util.assumeAssertsEnabled();
+        sharedCompactionObserver.registerExpectedSubtask();
+        Assert.assertThrows(AssertionError.class, () -> sharedCompactionObserver.onCompleted(operationId, null));
+    }
+
+    @Test
+    public void testErrorWrongProgress()
+    {
+        Util.assumeAssertsEnabled();
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.registerExpectedSubtask();
+        sharedCompactionObserver.onInProgress(mockProgress);
+        var mockProgress2 = Mockito.mock(CompactionProgress.class);
+        when(mockProgress2.operationId()).thenReturn(TimeUUID.Generator.nextTimeUUID());
+        Assert.assertThrows(AssertionError.class, () -> sharedCompactionObserver.onInProgress(mockProgress2));
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/SharedCompactionProgressTest.java b/test/unit/org/apache/cassandra/db/compaction/SharedCompactionProgressTest.java
new file mode 100644
index 000000000000..a3c4c8073fc2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/SharedCompactionProgressTest.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+/// Tests partially written by Copilot.
+public class SharedCompactionProgressTest
+{
+    private SharedCompactionProgress sharedCompactionProgress;
+    private TimeUUID operationId;
+
+    int id = 0;
+
+    @Before
+    public void setUp()
+    {
+        operationId = TimeUUID.Generator.nextTimeUUID();
+        sharedCompactionProgress = new SharedCompactionProgress(operationId, OperationType.COMPACTION, TableOperation.Unit.BYTES);
+    }
+
+    private CompactionProgress getMockProgress()
+    {
+        CompactionProgress mockProgress = Mockito.mock(CompactionProgress.class);
+        when(mockProgress.operationId()).thenReturn(TimeUUID.Generator.withSequence(operationId, id++));
+        when(mockProgress.operationType()).thenReturn(OperationType.COMPACTION);
+        when(mockProgress.unit()).thenReturn(TableOperation.Unit.BYTES);
+        var input = mockSSTable("i");
+        when(mockProgress.inSSTables()).thenReturn(input);
+        var output = mockSSTable("o");
+        when(mockProgress.outSSTables()).thenReturn(output);
+        var sstables = mockSSTable("s");
+        when(mockProgress.sstables()).thenReturn(sstables);
+        when(mockProgress.inputDiskSize()).thenReturn(100L);
+        when(mockProgress.outputDiskSize()).thenReturn(200L);
+        when(mockProgress.uncompressedBytesRead()).thenReturn(300L);
+        when(mockProgress.uncompressedBytesWritten()).thenReturn(400L);
+        when(mockProgress.partitionsRead()).thenReturn(500L);
+        when(mockProgress.rowsRead()).thenReturn(600L);
+        when(mockProgress.completed()).thenReturn(700L);
+        when(mockProgress.total()).thenReturn(800L);
+        when(mockProgress.startTimeMillis()).thenReturn(900L);
+        when(mockProgress.inputUncompressedSize()).thenReturn(1000L);
+        when(mockProgress.adjustedInputDiskSize()).thenReturn(1100L);
+        when(mockProgress.partitionsHistogram()).thenReturn(new long[]{1, 2, 3});
+        when(mockProgress.rowsHistogram()).thenReturn(new long[]{4, 5, 6});
+        return mockProgress;
+    }
+
+    private void checkProgress(CompactionProgress progress, int count, int countForTotal)
+    {
+        assertEquals(count, progress.inSSTables().size());
+        assertTrue(progress.inSSTables().stream().map(Object::toString).allMatch(s -> s.startsWith("i")));
+        assertEquals(count, progress.outSSTables().size());
+        assertTrue(progress.outSSTables().stream().map(Object::toString).allMatch(s -> s.startsWith("o")));
+        assertEquals(count, progress.sstables().size());
+        assertTrue(progress.sstables().stream().map(Object::toString).allMatch(s -> s.startsWith("s")));
+        assertEquals(100L * countForTotal, progress.inputDiskSize());
+        assertEquals(200L * count, progress.outputDiskSize());
+        assertEquals(300L * count, progress.uncompressedBytesRead());
+        assertEquals(400L * count, progress.uncompressedBytesWritten());
+        assertEquals(500L * count, progress.partitionsRead());
+        assertEquals(600L * count, progress.rowsRead());
+        assertEquals(700L * count, progress.completed());
+        assertEquals(800L * countForTotal, progress.total());
+        assertEquals(900L, progress.startTimeMillis());
+        assertEquals(1000L * countForTotal, progress.inputUncompressedSize());
+        assertEquals(1100L * count, progress.adjustedInputDiskSize());
+        assertArrayEquals(new long[]{1 * count, 2 * count, 3 * count}, progress.partitionsHistogram());
+        assertArrayEquals(new long[]{4 * count, 5 * count, 6 * count}, progress.rowsHistogram());
+    }
+
+    private Set<SSTableReader> mockSSTable(String nameprefix)
+    {
+        SSTableReader readerMock = Mockito.mock(SSTableReader.class);
+        when(readerMock.toString()).thenReturn(nameprefix + ThreadLocalRandom.current().nextInt());
+        return ImmutableSet.of(readerMock);
+    }
+
+    @Test
+    public void testCompleteSubtask()
+    {
+        CompactionProgress mockProgress = getMockProgress();
+        sharedCompactionProgress.registerExpectedSubtask(800L, 100L, 1000L);
+        sharedCompactionProgress.addSubtask(mockProgress);
+        checkProgress(sharedCompactionProgress, 1, 1);
+        boolean isComplete = sharedCompactionProgress.completeSubtask(mockProgress);
+        checkProgress(sharedCompactionProgress, 1, 1);
+        assertTrue(isComplete);
+    }
+
+    @Test
+    public void testComplete2Subtasks()
+    {
+        sharedCompactionProgress.registerExpectedSubtask(800L, 100L, 1000L);
+        sharedCompactionProgress.registerExpectedSubtask(800L, 100L, 1000L);
+        CompactionProgress mockProgress1 = getMockProgress();
+        CompactionProgress mockProgress2 = getMockProgress();
+        sharedCompactionProgress.addSubtask(mockProgress1);
+        checkProgress(sharedCompactionProgress, 1, 2);
+        sharedCompactionProgress.addSubtask(mockProgress2);
+        boolean isComplete = sharedCompactionProgress.completeSubtask(mockProgress2);
+        assertFalse(isComplete);
+        isComplete = sharedCompactionProgress.completeSubtask(mockProgress1);
+        assertTrue(isComplete);
+        checkProgress(sharedCompactionProgress, 2, 2);
+    }
+
+    @Test
+    public void testComplete2SubtasksLateStart()
+    {
+        sharedCompactionProgress.registerExpectedSubtask(800L, 100L, 1000L);
+        sharedCompactionProgress.registerExpectedSubtask(800L, 100L, 1000L);
+        CompactionProgress mockProgress = getMockProgress();
+        sharedCompactionProgress.addSubtask(mockProgress);
+        boolean isComplete = sharedCompactionProgress.completeSubtask(mockProgress);
+        assertFalse(isComplete);
+        checkProgress(sharedCompactionProgress, 1, 2);
+        mockProgress = getMockProgress();
+        sharedCompactionProgress.addSubtask(mockProgress);
+        isComplete = sharedCompactionProgress.completeSubtask(mockProgress);
+        assertTrue(isComplete);
+        checkProgress(sharedCompactionProgress, 2, 2);
+    }
+
+    @Test
+    public void testConcurrentAccess() throws InterruptedException, ExecutionException
+    {
+        int threadCount = 100;
+        ExecutorService executor = Executors.newFixedThreadPool(threadCount);
+
+        for (int i = 0; i < threadCount; ++i)
+            sharedCompactionProgress.registerExpectedSubtask(800L, 100L, 1000L);
+
+        AtomicInteger completed = new AtomicInteger(0);
+        List<Future<?>> futures = new ArrayList<>();
+
+        for (int i = 0; i < threadCount; i++)
+        {
+            futures.add(executor.submit(() ->
+                {
+                    CompactionProgress mockProgress = getMockProgress();
+                    sharedCompactionProgress.addSubtask(mockProgress);
+                    if (ThreadLocalRandom.current().nextBoolean())
+                        FBUtilities.sleepQuietly(ThreadLocalRandom.current().nextInt(1));
+                    if (sharedCompactionProgress.completeSubtask(mockProgress))
+                        completed.incrementAndGet();
+                }));
+        }
+
+        for (Future<?> future : futures)
+            future.get();
+
+        assertEquals(1, completed.get());
+        executor.shutdown();
+        checkProgress(sharedCompactionProgress, threadCount, threadCount);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/SharedTableOperationTest.java b/test/unit/org/apache/cassandra/db/compaction/SharedTableOperationTest.java
new file mode 100644
index 000000000000..fb21e0052f77
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/SharedTableOperationTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.apache.cassandra.utils.NonThrowingCloseable;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+public class SharedTableOperationTest {
+
+    private SharedTableOperation sharedTableOperation;
+    private TableOperationObserver mockObserver;
+    private TableOperation mockOperation1;
+    private TableOperation mockOperation2;
+    private TableOperation mockOperation3;
+    private TableOperation.Progress mockProgress;
+    private NonThrowingCloseable mockCloseable;
+
+    @Before
+    public void setUp() {
+        mockProgress = Mockito.mock(SharedTableOperation.Progress.class);
+        sharedTableOperation = new SharedTableOperation(mockProgress);
+        mockObserver = Mockito.mock(TableOperationObserver.class);
+        mockCloseable = Mockito.mock(NonThrowingCloseable.class);
+        mockOperation1 = Mockito.mock(TableOperation.class);
+        mockOperation2 = Mockito.mock(TableOperation.class);
+        mockOperation3 = Mockito.mock(TableOperation.class);
+
+        when(mockObserver.onOperationStart(sharedTableOperation)).thenReturn(mockCloseable);
+    }
+
+    @Test
+    public void testGetProgress() {
+        assertEquals(mockProgress, sharedTableOperation.getProgress());
+    }
+
+    @Test
+    public void testOneChild() {
+        // Register expected subtask
+        sharedTableOperation.registerExpectedSubtask();
+
+        // Wrap observer
+        TableOperationObserver wrappedObserver = sharedTableOperation.wrapObserver(mockObserver);
+
+        // Start operation
+        NonThrowingCloseable closeable = wrappedObserver.onOperationStart(mockOperation1);
+        assertNotNull(closeable);
+
+        // Verify observer communication
+        verify(mockObserver, times(1)).onOperationStart(sharedTableOperation);
+
+        // Close operation
+        closeable.close();
+        verify(mockCloseable, times(1)).close();
+    }
+
+    @Test
+    public void testOneChildStop() {
+        // Register expected subtask
+        sharedTableOperation.registerExpectedSubtask();
+
+        // Wrap observer
+        TableOperationObserver wrappedObserver = sharedTableOperation.wrapObserver(mockObserver);
+
+        // Start operation
+        NonThrowingCloseable closeable = wrappedObserver.onOperationStart(mockOperation1);
+        assertNotNull(closeable);
+        // When stopped, an operation will also close itself
+        Mockito.doAnswer(inv -> {
+            closeable.close();
+            return null;
+        }).when(mockOperation1).stop(any());
+
+        // Verify observer communication
+        verify(mockObserver, times(1)).onOperationStart(sharedTableOperation);
+
+        sharedTableOperation.stop(TableOperation.StopTrigger.CLEANUP);
+        verify(mockOperation1, times(1)).stop(TableOperation.StopTrigger.CLEANUP);
+
+        // Close operation
+        verify(mockCloseable, times(1)).close();
+    }
+
+    @Test
+    public void testThreeChildren() {
+        // Register expected subtasks
+        sharedTableOperation.registerExpectedSubtask();
+        sharedTableOperation.registerExpectedSubtask();
+        sharedTableOperation.registerExpectedSubtask();
+
+        // Wrap observer
+        TableOperationObserver wrappedObserver = sharedTableOperation.wrapObserver(mockObserver);
+
+        // Start operations
+        NonThrowingCloseable closeable1 = wrappedObserver.onOperationStart(mockOperation1);
+        NonThrowingCloseable closeable2 = wrappedObserver.onOperationStart(mockOperation2);
+        assertNotNull(closeable1);
+        assertNotNull(closeable2);
+        closeable1.close();
+
+        verify(mockObserver, times(1)).onOperationStart(sharedTableOperation);
+        verify(mockCloseable, times(0)).close();
+
+        NonThrowingCloseable closeable3 = wrappedObserver.onOperationStart(mockOperation3);
+        assertNotNull(closeable3);
+
+        // Close operations
+        closeable2.close();
+        closeable3.close();
+
+        verify(mockObserver, times(1)).onOperationStart(sharedTableOperation);
+        verify(mockCloseable, times(1)).close();
+    }
+
+    @Test
+    public void testThreeChildrenStop() {
+        // Register expected subtasks
+        sharedTableOperation.registerExpectedSubtask();
+        sharedTableOperation.registerExpectedSubtask();
+        sharedTableOperation.registerExpectedSubtask();
+
+        // Wrap observer
+        TableOperationObserver wrappedObserver = sharedTableOperation.wrapObserver(mockObserver);
+
+        // Start first operation
+        NonThrowingCloseable closeable1 = wrappedObserver.onOperationStart(mockOperation1);
+        assertNotNull(closeable1);
+        // When stopped, an operation will also close itself
+        Mockito.doAnswer(inv -> {
+            closeable1.close();
+            return null;
+        }).when(mockOperation1).stop(any());
+
+        // Issue stop before starting the next operations
+        sharedTableOperation.stop(TableOperation.StopTrigger.CLEANUP);
+        verify(mockObserver, times(1)).onOperationStart(sharedTableOperation);
+        verify(mockCloseable, times(0)).close();
+        verify(mockOperation1, times(1)).stop(TableOperation.StopTrigger.CLEANUP);
+
+        // Start remaining operations
+        NonThrowingCloseable closeable2 = wrappedObserver.onOperationStart(mockOperation2);
+        assertNotNull(closeable2);
+        verify(mockOperation2, times(1)).stop(TableOperation.StopTrigger.CLEANUP);
+
+        NonThrowingCloseable closeable3 = wrappedObserver.onOperationStart(mockOperation3);
+        assertNotNull(closeable3);
+        verify(mockOperation3, times(1)).stop(TableOperation.StopTrigger.CLEANUP);
+
+        // In response to a stop request, the child process will finish and close itself as it executes
+        closeable3.close();
+        verify(mockCloseable, times(0)).close();
+        closeable2.close(); // simulating a bit of disorder here
+
+        // Verify observer communication
+        verify(mockObserver, times(1)).onOperationStart(sharedTableOperation);
+        verify(mockCloseable, times(1)).close();
+        verify(mockOperation1, times(1)).stop(TableOperation.StopTrigger.CLEANUP);
+        verify(mockOperation2, times(1)).stop(TableOperation.StopTrigger.CLEANUP);
+        verify(mockOperation3, times(1)).stop(TableOperation.StopTrigger.CLEANUP);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/SimpleShardTrackerTest.java b/test/unit/org/apache/cassandra/db/compaction/SimpleShardTrackerTest.java
new file mode 100644
index 000000000000..0667400ac744
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/SimpleShardTrackerTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.junit.Test;
+
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+
+public class SimpleShardTrackerTest {
+
+    @Test(expected = AssertionError.class)
+    public void testZeroLengthTokenArray()
+    {
+        new SimpleShardTracker(new Token[0]);
+    }
+
+    @Test(expected = AssertionError.class)
+    public void testNonMinimumFirstToken()
+    {
+        new SimpleShardTracker(new Token[] { Murmur3Partitioner.instance.getMaximumToken() });
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java
index 9e03ec3a9a5e..00bbad35cf6f 100644
--- a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java
@@ -20,6 +20,7 @@
 
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.util.Collection;
 import java.util.Random;
 
 import org.apache.commons.lang3.StringUtils;
@@ -34,6 +35,7 @@
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 public class SingleSSTableLCSTaskTest extends CQLTester
@@ -46,19 +48,22 @@ public void basicTest() throws Throwable
         execute("insert into %s (id, t) values (1, 'meep')");
         Util.flush(cfs);
         SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyContainer()
+                                                                       .getStrategies(false, null)
+                                                                       .get(0);
 
         try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.COMPACTION))
         {
             if (txn != null)
             {
-                SingleSSTableLCSTask task = new SingleSSTableLCSTask(cfs, txn, 2);
-                task.executeInternal(null);
+                SingleSSTableLCSTask task = new SingleSSTableLCSTask(lcs, txn, 2);
+                task.executeInternal();
             }
         }
         assertEquals(1, cfs.getLiveSSTables().size());
         cfs.getLiveSSTables().forEach(s -> assertEquals(2, s.getSSTableLevel()));
         // make sure compaction strategy is notified:
-        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
+
         for (int i = 0; i < lcs.manifest.getLevelCount(); i++)
         {
             if (i == 2)
@@ -100,18 +105,34 @@ private void compactionTestHelper(boolean singleSSTUplevel) throws Throwable
                 Util.flush(cfs);
         }
         // now we have a bunch of data in L0, first compaction will be a normal one, containing all sstables:
-        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
-        AbstractCompactionTask act = lcs.getNextBackgroundTask(0);
-        act.execute(ActiveCompactionsTracker.NOOP);
+        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) ((CompactionStrategyManager) cfs.getCompactionStrategyContainer())
+                                                                    .getUnrepairedUnsafe()
+                                                                    .first();
+        Collection<AbstractCompactionTask> tasks = lcs.getNextBackgroundTasks(0);
+        assertEquals(1, tasks.size());
+        AbstractCompactionTask act = tasks.iterator().next();
+        assertNotNull(act);
+        act.execute();
 
         // now all sstables are laid out non-overlapping in L1, this means that the rest of the compactions
         // will be single sstable ones, make sure that we use SingleSSTableLCSTask if singleSSTUplevel is true:
-        while (lcs.getEstimatedRemainingTasks() > 0)
+        tasks = lcs.getNextBackgroundTasks(0);
+        while (!tasks.isEmpty())
         {
-            act = lcs.getNextBackgroundTask(0);
+            assertEquals(1, tasks.size());
+            act = tasks.iterator().next();
+            assertNotNull(act);
+
+            assertTrue(lcs.getTotalCompactions() > 0);
             assertEquals(singleSSTUplevel, act instanceof SingleSSTableLCSTask);
-            act.execute(ActiveCompactionsTracker.NOOP);
+            act.execute();
+
+            tasks = lcs.getNextBackgroundTasks(0);
         }
+
+        assertEquals(0, lcs.getTotalCompactions());
+        assertEquals(0, lcs.getEstimatedRemainingTasks());
+
         assertEquals(0, lcs.getLevelSize(0));
         int l1size = lcs.getLevelSize(1);
         // this should be 10, but it might vary a bit depending on partition sizes etc
@@ -134,12 +155,16 @@ public void corruptMetadataTest() throws Throwable
             fc.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
         }
         boolean gotException = false;
+        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) ((CompactionStrategyManager) cfs.getCompactionStrategyContainer())
+                                                                    .getUnrepairedUnsafe()
+                                                                    .first();
+
         try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.COMPACTION))
         {
             if (txn != null)
             {
-                SingleSSTableLCSTask task = new SingleSSTableLCSTask(cfs, txn, 2);
-                task.executeInternal(null);
+                SingleSSTableLCSTask task = new SingleSSTableLCSTask(lcs, txn, 2);
+                task.executeInternal();
             }
         }
         catch (Throwable t)
@@ -150,7 +175,7 @@ public void corruptMetadataTest() throws Throwable
         assertEquals(1, cfs.getLiveSSTables().size());
         for (SSTableReader sst : cfs.getLiveSSTables())
             assertEquals(0, sst.getSSTableMetadata().sstableLevel);
-        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first();
+
         assertEquals(1, lcs.getLevelSize(0));
         assertTrue(cfs.getTracker().getCompacting().isEmpty());
     }
diff --git a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
index dac331e1170d..2c11cd50be67 100644
--- a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
@@ -19,11 +19,17 @@
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import com.google.common.collect.ImmutableList;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -37,21 +43,27 @@
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.metrics.RestorableMeter;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.Pair;
+import org.mockito.Mockito;
 
-import static org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.getBuckets;
-import static org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.mostInterestingBucket;
-import static org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.trimToThresholdWithHotness;
 import static org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.validateOptions;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.when;
 
 public class SizeTieredCompactionStrategyTest
 {
     public static final String KEYSPACE1 = "SizeTieredCompactionStrategyTest";
     private static final String CF_STANDARD1 = "Standard1";
 
+    private static final Random random = new Random(98752945723L);
+
+    private final int minThreshold = 4; //same as the default
+    private final int maxThreshold = 32; //same as the default
+    private final double bucketLow = 0.5; //same as the default
+    private final double bucketHigh = 1.5; //same as the default
+    private final int minSSTableSize = 10; // small enough not to interfere
+
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
     {
@@ -94,62 +106,67 @@ public void testOptionsValidation() throws ConfigurationException
     @Test
     public void testGetBuckets()
     {
-        List<Pair<String, Long>> pairs = new ArrayList<>();
-        String[] strings = { "a", "bbbb", "cccccccc", "cccccccc", "bbbb", "a" };
-        for (String st : strings)
+        List<SSTableReader> sstables = new ArrayList<>();
+        long[] sstableLengths = { 1L, 4L, 8L, 8L, 4L, 1L };
+        for (long len : sstableLengths)
         {
-            Pair<String, Long> pair = Pair.create(st, (long) st.length());
-            pairs.add(pair);
+            SSTableReader sstable = Mockito.mock(SSTableReader.class);
+            when(sstable.onDiskLength()).thenReturn(len);
+            when(sstable.hotness()).thenReturn(0.);
+            sstables.add(sstable);
         }
 
-        List<List<String>> buckets = getBuckets(pairs, 1.5, 0.5, 2);
+        SizeTieredCompactionStrategyOptions stcsOptions = new SizeTieredCompactionStrategyOptions(2, bucketLow, bucketHigh);
+        SizeTieredCompactionStrategy.SizeTieredBuckets sizeTieredBuckets = new SizeTieredCompactionStrategy.SizeTieredBuckets(sstables, stcsOptions, minThreshold, maxThreshold);
+        List<List<CompactionSSTable>> buckets = sizeTieredBuckets.buckets();
         assertEquals(3, buckets.size());
-
-        for (List<String> bucket : buckets)
+        for (List<CompactionSSTable> bucket : buckets)
         {
             assertEquals(2, bucket.size());
-            assertEquals(bucket.get(0).length(), bucket.get(1).length());
-            assertEquals(bucket.get(0).charAt(0), bucket.get(1).charAt(0));
         }
 
-        pairs.clear();
+        sstables.clear();
         buckets.clear();
 
-        String[] strings2 = { "aaa", "bbbbbbbb", "aaa", "bbbbbbbb", "bbbbbbbb", "aaa" };
-        for (String st : strings2)
+        long[] sstableLengths2 = { 3L, 8L, 3L, 8L, 8L, 3L };
+        for (long len : sstableLengths2)
         {
-            Pair<String, Long> pair = Pair.create(st, (long) st.length());
-            pairs.add(pair);
+            SSTableReader sstable = Mockito.mock(SSTableReader.class);
+            when(sstable.onDiskLength()).thenReturn(len);
+            when(sstable.hotness()).thenReturn(0.);
+            sstables.add(sstable);
         }
 
-        buckets = getBuckets(pairs, 1.5, 0.5, 2);
+        sizeTieredBuckets = new SizeTieredCompactionStrategy.SizeTieredBuckets(sstables, stcsOptions, minThreshold, maxThreshold);
+        buckets = sizeTieredBuckets.buckets();
         assertEquals(2, buckets.size());
-
-        for (List<String> bucket : buckets)
+        for (List<CompactionSSTable> bucket : buckets)
         {
             assertEquals(3, bucket.size());
-            assertEquals(bucket.get(0).charAt(0), bucket.get(1).charAt(0));
-            assertEquals(bucket.get(1).charAt(0), bucket.get(2).charAt(0));
         }
 
         // Test the "min" functionality
-        pairs.clear();
+        sstables.clear();
         buckets.clear();
 
-        String[] strings3 = { "aaa", "bbbbbbbb", "aaa", "bbbbbbbb", "bbbbbbbb", "aaa" };
-        for (String st : strings3)
+        long[] sstableLengths3 = { 3L, 8L, 3L, 8L, 8L, 3L };
+        for (long len : sstableLengths3)
         {
-            Pair<String, Long> pair = Pair.create(st, (long) st.length());
-            pairs.add(pair);
+            SSTableReader sstable = Mockito.mock(SSTableReader.class);
+            when(sstable.onDiskLength()).thenReturn(len);
+            when(sstable.hotness()).thenReturn(0.);
+            sstables.add(sstable);
         }
 
-        buckets = getBuckets(pairs, 1.5, 0.5, 10);
+        stcsOptions = new SizeTieredCompactionStrategyOptions(10, bucketLow, bucketHigh);
+        sizeTieredBuckets = new SizeTieredCompactionStrategy.SizeTieredBuckets(sstables, stcsOptions, minThreshold, maxThreshold);
+        buckets = sizeTieredBuckets.buckets();
         assertEquals(1, buckets.size());
     }
 
     @SuppressWarnings("UnnecessaryLocalVariable")
     @Test
-    public void testPrepBucket()
+    public void testSingleBucketWith3IdenticalFilesRealSSTables()
     {
         String ksname = KEYSPACE1;
         String cfname = "Standard1";
@@ -172,10 +189,11 @@ public void testPrepBucket()
         }
         Util.flush(cfs);
 
+        SizeTieredCompactionStrategyOptions stcsOptions = new SizeTieredCompactionStrategyOptions();
         List<SSTableReader> sstrs = new ArrayList<>(cfs.getLiveSSTables());
-        Pair<List<SSTableReader>, Double> bucket;
+        SizeTieredCompactionStrategy.SizeTieredBuckets sizeTieredBuckets = new SizeTieredCompactionStrategy.SizeTieredBuckets(sstrs.subList(0, 2), stcsOptions, 4, 32);
 
-        List<SSTableReader> interestingBucket = mostInterestingBucket(Collections.singletonList(sstrs.subList(0, 2)), 4, 32);
+        List<CompactionSSTable> interestingBucket = new ArrayList<>(CompactionAggregate.getSelected(sizeTieredBuckets.getAggregates()).sstables());
         assertTrue("nothing should be returned when all buckets are below the min threshold", interestingBucket.isEmpty());
 
         sstrs.get(0).overrideReadMeter(new RestorableMeter(100.0, 100.0));
@@ -185,10 +203,303 @@ public void testPrepBucket()
         long estimatedKeys = sstrs.get(0).estimatedKeys();
 
         // if we have more than the max threshold, the coldest should be dropped
-        bucket = trimToThresholdWithHotness(sstrs, 2);
-        assertEquals("one bucket should have been dropped", 2, bucket.left.size());
+        sizeTieredBuckets = new SizeTieredCompactionStrategy.SizeTieredBuckets(sstrs, stcsOptions, 1, 2);
+        sizeTieredBuckets.aggregate();
+
+        List<CompactionPick> compactions = sizeTieredBuckets.getCompactions();
+        CompactionPick selected = CompactionAggregate.getSelected(sizeTieredBuckets.getAggregates());
+        if (!selected.isEmpty())
+            assertEquals(selected, compactions.get(0));
+        List<CompactionPick> pending = compactions.isEmpty() ? ImmutableList.of() : compactions.subList(1, compactions.size());
+
+        assertEquals("one bucket should have been dropped", 2, selected.sstables().size());
+        assertEquals("there should be one pending task", 1, pending.size());
+
         double expectedBucketHotness = (200.0 + 300.0) / estimatedKeys;
-        assertEquals(String.format("bucket hotness (%f) should be close to %f", bucket.right, expectedBucketHotness),
-                     expectedBucketHotness, bucket.right, 1.0);
+        assertEquals(String.format("bucket hotness (%f) should be close to %f",
+                                   CompactionAggregate.getSelected(sizeTieredBuckets.getAggregates()).hotness(), expectedBucketHotness),
+                     expectedBucketHotness, CompactionAggregate.getSelected(sizeTieredBuckets.getAggregates()).hotness(), 1.0);
+    }
+
+
+    @Test
+    public void testTwoBucketsDifferentHotness()
+    {
+        List<SSTableReader> bucket1 = mockBucket(8, 2000, 100);
+        List<SSTableReader> bucket2 = mockBucket(8, 1000, 200); // hottest bucket with hotness 200 per table should be selected
+
+        List<SSTableReader> sstables = Stream.concat(bucket1.stream(), bucket2.stream()).collect(Collectors.toList());
+        for (int i = 0; i < 5; i++)
+        {
+            Collections.shuffle(sstables, random);
+            testBuckets(sstables, bucket2, ImmutableList.of(bucket1), 2);
+        }
+    }
+
+    @Test
+    public void testTwoBucketsSameHotness()
+    {
+        List<SSTableReader> bucket1 = mockBucket(8, 1000, 100);
+        List<SSTableReader> bucket2 = mockBucket(8, 4000, 100); // bucket with largest sstables should be selected if same hotness
+
+        List<SSTableReader> sstables = Stream.concat(bucket1.stream(), bucket2.stream()).collect(Collectors.toList());
+        for (int i = 0; i < 5; i++)
+        {
+            Collections.shuffle(sstables, random);
+            testBuckets(sstables, bucket2, ImmutableList.of(bucket1), 2);
+        }
+    }
+
+    @Test
+    public void testSplitLargeBucketExactMultiple()
+    {
+        List<SSTableReader> bucket1 = mockBucket(maxThreshold, 1000, 100);
+        List<SSTableReader> bucket2 = mockBucket(maxThreshold, 1000, 200);
+        List<SSTableReader> bucket3 = mockBucket(maxThreshold, 1000, 300);
+        List<SSTableReader> bucket4 = mockBucket(maxThreshold, 1000, 400); // hottest bucket
+
+        List<SSTableReader> largeBucket = new ArrayList<>(maxThreshold * 4);
+        largeBucket.addAll(bucket1);
+        largeBucket.addAll(bucket2);
+        largeBucket.addAll(bucket3);
+        largeBucket.addAll(bucket4);
+
+        Collections.shuffle(largeBucket, random);
+
+        testBuckets(largeBucket, bucket4, ImmutableList.of(bucket3, bucket2, bucket1), 1);
+    }
+
+    @Test
+    public void testSplitLargeBucketNotExactMultiple()
+    {
+        List<SSTableReader> bucket1 = mockBucket(maxThreshold / 2, 1000, 100);
+        List<SSTableReader> bucket2 = mockBucket(maxThreshold, 1000, 200);
+        List<SSTableReader> bucket3 = mockBucket(maxThreshold, 1000, 300);
+        List<SSTableReader> bucket4 = mockBucket(maxThreshold, 1000, 400); // hottest bucket
+
+        List<SSTableReader> largeBucket = new ArrayList<>(maxThreshold * 4);
+        largeBucket.addAll(bucket1);
+        largeBucket.addAll(bucket2);
+        largeBucket.addAll(bucket3);
+        largeBucket.addAll(bucket4);
+
+        Collections.shuffle(largeBucket, random);
+
+        testBuckets(largeBucket, bucket4, ImmutableList.of(bucket3, bucket2, bucket1), 1);
+    }
+
+    @Test
+    public void testSplitLargeBucketWithLeftOverBelowMinThreshold()
+    {
+        List<SSTableReader> bucket1 = mockBucket(minThreshold - 1, 1000, 100); // should be ignored
+        List<SSTableReader> bucket2 = mockBucket(maxThreshold, 1000, 200); // hottest bucket
+
+        List<SSTableReader> largeBucket = new ArrayList<>(maxThreshold * 4);
+        largeBucket.addAll(bucket1);
+        largeBucket.addAll(bucket2);
+
+        Collections.shuffle(largeBucket, random);
+
+        testBuckets(largeBucket, bucket2, ImmutableList.of(), 1);
+    }
+
+    @Test
+    public void testIgnoreBucketsBelowMinThreshold()
+    {
+        List<SSTableReader> sstables = new ArrayList<>();
+        long bytesOnDisk = 1000;
+        double hotness = 200;
+        for (int i = 0; i < minThreshold; i++)
+        {
+            sstables.addAll(mockBucket(i, bytesOnDisk, hotness));
+            bytesOnDisk *= 2;
+            hotness *= 2;
+        }
+
+        // all buckets with sstables should be considered and so the number of expected aggregates
+        // is minThreshold - 1 (because one has no sstables)
+        testBuckets(sstables, ImmutableList.of(), ImmutableList.of(), minThreshold - 1);
+    }
+
+    @Test
+    public void testIgnoreBucketsBelowMinThresholdExceptOne()
+    {
+        List<SSTableReader> sstables = new ArrayList<>();
+        long bytesOnDisk = 1000;
+        double hotness = 200;
+        for (int i = 0; i < minThreshold; i++)
+        {
+            sstables.addAll(mockBucket(i, bytesOnDisk, hotness));
+            bytesOnDisk *= 2;
+            hotness *= 2;
+        }
+
+        List<SSTableReader> bucket = mockBucket(minThreshold, bytesOnDisk, hotness);
+        sstables.addAll(bucket); // this is the only bucket that should be picked up
+
+        // all buckets with sstables should be considered and so the number of expected aggregates
+        // is minThreshold (because one has no sstables)
+        testBuckets(sstables, bucket, ImmutableList.of(), minThreshold);
+    }
+
+    @Test
+    public void testManySmallSSTables()
+    {
+        // SStables smaller than minSSTableSize should all be grouped in the same bucket
+
+        int minSSTableSize = 1000;
+        List<SSTableReader> sstables = new ArrayList<>();
+
+        for (int i = 0; i < 10; i++)
+        {
+            List<SSTableReader> bucket = mockBucket(minThreshold + random.nextInt(maxThreshold), random.nextInt(minSSTableSize), 100);
+            sstables.addAll(bucket);
+        }
+
+        Collections.sort(sstables, Comparator.comparing(sstable -> sstable.onDiskLength()));
+
+        List<List<SSTableReader>> buckets = new ArrayList<>();
+        int i = 0;
+        while ((sstables.size() - i) >= minThreshold)
+        {
+            buckets.add(sstables.subList(i, Math.min(i+ maxThreshold, sstables.size())));
+            i += maxThreshold;
+        }
+
+        SizeTieredCompactionStrategyOptions stcsOptions = new SizeTieredCompactionStrategyOptions(minSSTableSize, bucketLow, bucketHigh);
+        testBuckets(stcsOptions, sstables, buckets.get(0), buckets.subList(1, buckets.size()), 1);
+    }
+
+    @Test
+    public void testThreeBucketsOnlyLargestSizeHasComps()
+    {
+        List<SSTableReader> bucket1 = mockBucket(2, 1000, 0); // no compaction
+        List<SSTableReader> bucket2 = mockBucket(2, 4000, 0); // no compaction
+        List<SSTableReader> bucket3 = mockBucket(4, 8000, 0); // one compaction
+
+        List<SSTableReader> sstables = new ArrayList<>(bucket1.size() + bucket2.size() + bucket3.size());
+        sstables.addAll(bucket1);
+        sstables.addAll(bucket2);
+        sstables.addAll(bucket3);
+
+        for (int i = 0; i < 5; i++)
+        {
+            Collections.shuffle(sstables, random);
+            testBuckets(sstables, bucket3, ImmutableList.of(), 3);
+        }
+    }
+
+    @Test
+    public void testThreeBucketsOnlySmallestSizeHasComps()
+    {
+        List<SSTableReader> bucket1 = mockBucket(4, 1000, 0); // one compaction
+        List<SSTableReader> bucket2 = mockBucket(2, 4000, 0); // no compaction
+        List<SSTableReader> bucket3 = mockBucket(2, 8000, 0); // no compaction
+
+        List<SSTableReader> sstables = new ArrayList<>(bucket1.size() + bucket2.size() + bucket3.size());
+        sstables.addAll(bucket1);
+        sstables.addAll(bucket2);
+        sstables.addAll(bucket3);
+
+        for (int i = 0; i < 5; i++)
+        {
+            Collections.shuffle(sstables, random);
+            testBuckets(sstables, bucket1, ImmutableList.of(), 3);
+        }
+    }
+
+    /**
+     * Sort the buckets by calling {@link SizeTieredCompactionStrategy.SizeTieredBuckets#aggregate()} and then verify
+     * that the selected bucket is {@code expectedBucket} and that the pending buckets are {@code expectedPending}.
+     *
+     * @param sstables - the input sstables to aggregate into buckets
+     * @param expectedSelected - the expected bucket that should be selected for compaction
+     * @param expectedPending - the expected pending buckets
+     */
+    private void testBuckets(List<SSTableReader> sstables, List<SSTableReader> expectedSelected, List<List<SSTableReader>> expectedPending, int numExpectedAggregates)
+    {
+        SizeTieredCompactionStrategyOptions stcsOptions = new SizeTieredCompactionStrategyOptions(minSSTableSize, bucketLow, bucketHigh);
+        testBuckets(stcsOptions, sstables, expectedSelected, expectedPending, numExpectedAggregates);
+    }
+
+    private void testBuckets(SizeTieredCompactionStrategyOptions stcsOptions,
+                             List<SSTableReader> sstables,
+                             List<SSTableReader> expectedSelected,
+                             List<List<SSTableReader>> expectedPending,
+                             int numExpectedAggregates)
+    {
+        SizeTieredCompactionStrategy.SizeTieredBuckets buckets = new SizeTieredCompactionStrategy.SizeTieredBuckets(sstables,
+                                                                                                                    stcsOptions,
+                                                                                                                    minThreshold,
+                                                                                                                    maxThreshold);
+        buckets.aggregate();
+
+        List<CompactionPick> compactions = buckets.getCompactions();
+        CompactionPick selected = CompactionAggregate.getSelected(buckets.getAggregates());
+        if (!selected.isEmpty())
+            assertEquals(selected, compactions.get(0));
+        List<CompactionPick> pending = compactions.isEmpty() ? ImmutableList.of() : compactions.subList(1, compactions.size());
+
+        compareBucketToCandidate(expectedSelected, selected);
+        assertEquals(expectedPending.size(), pending.size());
+
+        for (int i = 0; i < expectedPending.size(); i++)
+            compareBucketToCandidate(expectedPending.get(i), pending.get(i));
+
+        assertEquals(numExpectedAggregates, buckets.getAggregates().size());
+    }
+
+    private List<SSTableReader> mockBucket(int numSSTables, long bytesOnDisk, double hotness)
+    {
+        List<SSTableReader> ret = new ArrayList<>(numSSTables);
+        int h = 0;
+        for (int i = 0; i < numSSTables; i++)
+            ret.add(mockSSTable(bytesOnDisk, hotness));
+
+        return ret;
+    }
+
+    private SSTableReader mockSSTable(long bytesOnDisk, double hotness)
+    {
+        SSTableReader ret = Mockito.mock(SSTableReader.class);
+        when(ret.hotness()).thenReturn(hotness);
+        when(ret.onDiskLength()).thenReturn(bytesOnDisk);
+        when(ret.bytesOnDisk()).thenReturn(bytesOnDisk);
+        when(ret.toString()).thenReturn(String.format("Bytes on disk: %d, hotness %f, hashcode %d", bytesOnDisk, hotness, ret.hashCode()));
+
+        return ret;
+    }
+
+    private void compareBucketToCandidate(Collection<SSTableReader> bucket, CompactionPick candidate)
+    {
+        List<CompactionSSTable> sortedBucket = new ArrayList<>(bucket);
+        List<CompactionSSTable> sortedCandidate = new ArrayList<>(candidate.sstables());
+
+        // Sort by hash code because sorting by hotness may not work if several sstables have the
+        // same hotness and length on disk
+        Collections.sort(sortedBucket, Comparator.comparingLong(CompactionSSTable::hashCode));
+        Collections.sort(sortedCandidate, Comparator.comparingLong(CompactionSSTable::hashCode));
+
+        assertEquals(sortedBucket, sortedCandidate);
+        assertEquals(getBucketHotness(bucket), candidate.hotness(), 0.000001);
+        assertEquals(bucket.size() > 0 ? getBucketSize(bucket) / (double) bucket.size() : 0, candidate.avgSizeInBytes(), 1);
+    }
+
+    private double getBucketHotness(Collection<SSTableReader> bucket)
+    {
+        double ret = 0;
+        for (SSTableReader sstable : bucket)
+            ret += sstable.hotness();
+
+        return ret;
+    }
+
+    private long getBucketSize(Collection<SSTableReader> bucket)
+    {
+        long ret = 0;
+        for (SSTableReader sstable : bucket)
+            ret += sstable.onDiskLength();
+
+        return ret;
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index e7e97bed9988..d9b8c2ba706f 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@ -39,6 +39,7 @@
 import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.marshal.MapType;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
@@ -138,10 +139,10 @@ public void testAggressiveFullyExpired()
         Set<SSTableReader> sstables = Sets.newHashSet(cfs.getLiveSSTables());
         long now = FBUtilities.nowInSeconds();
         long gcBefore = now + 2;
-        Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(
+        Set<CompactionSSTable> expired = CompactionController.getFullyExpiredSSTables(
                 cfs,
                 sstables,
-                Collections.EMPTY_SET,
+                c -> Collections.<SSTableReader>emptySet(),
                 gcBefore);
         assertEquals(2, expired.size());
 
@@ -235,9 +236,9 @@ public void testNoExpire() throws InterruptedException, IOException
             .build()
             .applyUnsafe();
         Util.flush(cfs);
-        String noTTLKey = "nottl";
-        new RowUpdateBuilder(cfs.metadata(), timestamp, noTTLKey)
-            .add("col311", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+        final String noTTLColumn = "col311";
+        new RowUpdateBuilder(cfs.metadata(), timestamp, key)
+            .add(noTTLColumn, ByteBufferUtil.EMPTY_BYTE_BUFFER)
             .build()
             .applyUnsafe();
         // also write to other key to ensure overlap for UCS
@@ -259,7 +260,9 @@ public void testNoExpire() throws InterruptedException, IOException
         while(scanner.hasNext())
         {
             UnfilteredRowIterator iter = scanner.next();
-            assertEquals(Util.dk(noTTLKey), iter.partitionKey());
+            assertEquals(Util.dk(key), iter.partitionKey());
+            Row row = (Row) iter.next();
+            assertEquals(noTTLColumn, row.cells().iterator().next().column().toString());
         }
         scanner.close();
     }
diff --git a/test/unit/org/apache/cassandra/db/compaction/TestCompactionClass.java b/test/unit/org/apache/cassandra/db/compaction/TestCompactionClass.java
new file mode 100644
index 000000000000..e5f876c70a84
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/TestCompactionClass.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Map;
+
+public class TestCompactionClass extends SizeTieredCompactionStrategy
+{
+    int periodicReportsCalled = 0;
+
+    public TestCompactionClass(CompactionStrategyFactory factory, Map<String, String> options)
+    {
+        super(factory, options);
+    }
+
+    @Override
+    public void periodicReport()
+    {
+        super.periodicReport();
+        ++periodicReportsCalled;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
index 41a4c475786a..ec2e61790206 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
@@ -23,9 +23,11 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
-import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import org.junit.BeforeClass;
@@ -42,17 +44,16 @@
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.MockSchema;
-import org.apache.cassandra.utils.Pair;
 
+import static java.util.concurrent.TimeUnit.HOURS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION;
+import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.getBucketAggregates;
 import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.getWindowBoundsInMillis;
-import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.newestBucket;
 import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.validateOptions;
 import static org.apache.cassandra.utils.FBUtilities.nowInSeconds;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -130,14 +131,14 @@ public void testOptionsValidation() throws ConfigurationException
             options.put(TimeWindowCompactionStrategyOptions.UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY, "true");
         }
         
-        options.put(AbstractCompactionStrategy.UNCHECKED_TOMBSTONE_COMPACTION_OPTION, "true");
+        options.put(CompactionStrategyOptions.UNCHECKED_TOMBSTONE_COMPACTION_OPTION, "true");
         Keyspace keyspace = Keyspace.open(KEYSPACE1);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
-        TimeWindowCompactionStrategy twcs = new TimeWindowCompactionStrategy(cfs, options);
-        assertFalse(twcs.disableTombstoneCompactions);
-        options.put(AbstractCompactionStrategy.UNCHECKED_TOMBSTONE_COMPACTION_OPTION, "false");
-        twcs = new TimeWindowCompactionStrategy(cfs, options);
-        assertTrue(twcs.disableTombstoneCompactions);
+        TimeWindowCompactionStrategy twcs = new TimeWindowCompactionStrategy(new CompactionStrategyFactory(cfs), options);
+        assertFalse(twcs.options.isDisableTombstoneCompactions());
+        options.put(CompactionStrategyOptions.UNCHECKED_TOMBSTONE_COMPACTION_OPTION, "false");
+        twcs = new TimeWindowCompactionStrategy(new CompactionStrategyFactory(cfs), options);
+        assertTrue(twcs.options.isDisableTombstoneCompactions());
 
         options.put("bad_option", "1.0");
         unvalidated = validateOptions(options);
@@ -150,19 +151,19 @@ public void testTimeWindows()
     {
         long tstamp1 = 1451001601000L; // 2015-12-25 @ 00:00:01, in milliseconds
         long tstamp2 = 1451088001000L; // 2015-12-26 @ 00:00:01, in milliseconds
-        Long lowHour = 1451001600000L; // 2015-12-25 @ 00:00:00, in milliseconds
+        long lowHour = 1451001600000L; // 2015-12-25 @ 00:00:00, in milliseconds
 
         // A 1 hour window should round down to the beginning of the hour
-        assertEquals(0, getWindowBoundsInMillis(TimeUnit.HOURS, 1, tstamp1).left.compareTo(lowHour));
+        assertEquals(lowHour, getWindowBoundsInMillis(HOURS, 1, tstamp1));
 
         // A 1 minute window should round down to the beginning of the hour
-        assertEquals(0, getWindowBoundsInMillis(TimeUnit.MINUTES, 1, tstamp1).left.compareTo(lowHour));
+        assertEquals(lowHour, getWindowBoundsInMillis(TimeUnit.MINUTES, 1, tstamp1));
 
         // A 1 day window should round down to the beginning of the hour
-        assertEquals(0, getWindowBoundsInMillis(TimeUnit.DAYS, 1, tstamp1).left.compareTo(lowHour));
+        assertEquals(lowHour, getWindowBoundsInMillis(TimeUnit.DAYS, 1, tstamp1));
 
         // The 2 day window of 2015-12-25 + 2015-12-26 should round down to the beginning of 2015-12-25
-        assertEquals(0, getWindowBoundsInMillis(TimeUnit.DAYS, 2, tstamp2).left.compareTo(lowHour));
+        assertEquals(lowHour, getWindowBoundsInMillis(TimeUnit.DAYS, 2, tstamp2));
     }
 
     @Test
@@ -200,30 +201,29 @@ public void testPrepBucket()
 
         Util.flush(cfs);
 
-        HashMultimap<Long, SSTableReader> buckets = HashMultimap.create();
-        List<SSTableReader> sstrs = new ArrayList<>(cfs.getLiveSSTables());
+        TreeMap<Long, List<CompactionSSTable>> buckets = new TreeMap<>(Long::compare);
+        List<CompactionSSTable> sstrs = new ArrayList<>(cfs.getLiveSSTables());
 
         // We'll put 3 sstables into the newest bucket
         for (int i = 0; i < 3; i++)
         {
-            Pair<Long, Long> bounds = getWindowBoundsInMillis(TimeUnit.HOURS, 1, tstamp);
-            buckets.put(bounds.left, sstrs.get(i));
+            TimeWindowCompactionStrategy.addToBuckets(buckets, sstrs.get(i), tstamp, TimeUnit.HOURS, 1);
         }
 
-        TimeWindowCompactionStrategy.NewestBucket newBucket = newestBucket(buckets, 4, 32, new SizeTieredCompactionStrategyOptions(), getWindowBoundsInMillis(TimeUnit.HOURS, 1, System.currentTimeMillis()).left);
-        assertTrue("incoming bucket should not be accepted when it has below the min threshold SSTables", newBucket.sstables.isEmpty());
-        assertEquals("there should be no estimated remaining tasks when bucket is below min threshold SSTables", 0, newBucket.estimatedRemainingTasks);
+        List<CompactionAggregate> aggregates = getBucketAggregates(buckets, 4, 32, new SizeTieredCompactionStrategyOptions(), getWindowBoundsInMillis(HOURS, 1, System.currentTimeMillis()));
+        Set<CompactionPick> compactions = toCompactions(aggregates);
+        assertTrue("No selected compactions when fewer than min threshold SSTables in the newest bucket", CompactionAggregate.getSelected(aggregates).isEmpty());
+        assertTrue("No compactions when fewer than min threshold SSTables in the newest bucket", compactions.isEmpty());
 
-
-        newBucket = newestBucket(buckets, 2, 32, new SizeTieredCompactionStrategyOptions(), getWindowBoundsInMillis(TimeUnit.HOURS, 1, System.currentTimeMillis()).left);
-        assertFalse("incoming bucket should be accepted when it is larger than the min threshold SSTables", newBucket.sstables.isEmpty());
-        assertEquals("there should be one estimated remaining task when bucket is larger than the min threshold SSTables", 1, newBucket.estimatedRemainingTasks);
+        aggregates = getBucketAggregates(buckets, 2, 32, new SizeTieredCompactionStrategyOptions(), getWindowBoundsInMillis(HOURS, 1, System.currentTimeMillis()));
+        compactions = toCompactions(aggregates);
+        assertFalse("There should be one selected compaction when bucket is larger than the min but smaller than max threshold", CompactionAggregate.getSelected(aggregates).isEmpty());
+        assertEquals("There should be one compaction when bucket is larger than the min but smaller than max threshold", 1,  compactions.size());
 
         // And 2 into the second bucket (1 hour back)
         for (int i = 3; i < 5; i++)
         {
-            Pair<Long, Long> bounds = getWindowBoundsInMillis(TimeUnit.HOURS, 1, tstamp2);
-            buckets.put(bounds.left, sstrs.get(i));
+            TimeWindowCompactionStrategy.addToBuckets(buckets, sstrs.get(i), tstamp2, TimeUnit.HOURS, 1);
         }
 
         assertEquals("an sstable with a single value should have equal min/max timestamps", sstrs.get(0).getMinTimestamp(), sstrs.get(0).getMaxTimestamp());
@@ -248,15 +248,15 @@ public void testPrepBucket()
         sstrs = new ArrayList<>(cfs.getLiveSSTables());
         for (int i = 0; i < 40; i++)
         {
-            Pair<Long, Long> bounds = getWindowBoundsInMillis(TimeUnit.HOURS, 1, sstrs.get(i).getMaxTimestamp());
-            buckets.put(bounds.left, sstrs.get(i));
+            TimeWindowCompactionStrategy.addToBuckets(buckets, sstrs.get(i), sstrs.get(i).getMaxTimestamp(), TimeUnit.HOURS, 1);
         }
 
-        newBucket = newestBucket(buckets, 4, 32, new SizeTieredCompactionStrategyOptions(), getWindowBoundsInMillis(TimeUnit.HOURS, 1, System.currentTimeMillis()).left);
-        assertEquals("new bucket should be trimmed to max threshold of 32", newBucket.sstables.size(), 32);
+        aggregates = getBucketAggregates(buckets, 4, 32, new SizeTieredCompactionStrategyOptions(), getWindowBoundsInMillis(HOURS, 1, System.currentTimeMillis()));
+        compactions = toCompactions(aggregates);
+        assertEquals("new bucket should be split by max threshold of 32", buckets.keySet().size() + 1, compactions.size());
 
-        // one per bucket because they are all eligible and one more for the sstables that were trimmed
-        assertEquals("there should be one estimated remaining task per eligible bucket", buckets.keySet().size() + 1, newBucket.estimatedRemainingTasks);
+        CompactionPick selected = CompactionAggregate.getSelected(aggregates);
+        assertEquals("first pick should be trimmed to max threshold of 32", 32, selected.sstables().size());
     }
 
 
@@ -294,16 +294,18 @@ public void testDropExpiredSSTables() throws InterruptedException
         options.put(TimeWindowCompactionStrategyOptions.COMPACTION_WINDOW_UNIT_KEY, "SECONDS");
         options.put(TimeWindowCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
         options.put(TimeWindowCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0");
-        TimeWindowCompactionStrategy twcs = new TimeWindowCompactionStrategy(cfs, options);
+        TimeWindowCompactionStrategy twcs = new TimeWindowCompactionStrategy(new CompactionStrategyFactory(cfs), options);
         for (SSTableReader sstable : cfs.getLiveSSTables())
             twcs.addSSTable(sstable);
 
         twcs.startup();
-        assertNull(twcs.getNextBackgroundTask(nowInSeconds()));
+        assertTrue(twcs.getNextBackgroundTasks(nowInSeconds()).isEmpty());
 
         // Wait for the expiration of the first sstable
         Thread.sleep(TimeUnit.SECONDS.toMillis(TTL_SECONDS + 1));
-        AbstractCompactionTask t = twcs.getNextBackgroundTask(nowInSeconds());
+        Collection<AbstractCompactionTask> tasks = twcs.getNextBackgroundTasks(nowInSeconds());
+        assertEquals(1, tasks.size());
+        AbstractCompactionTask t = tasks.iterator().next();
         assertNotNull(t);
         assertEquals(1, Iterables.size(t.transaction.originals()));
         SSTableReader sstable = t.transaction.originals().iterator().next();
@@ -349,24 +351,26 @@ public void testDropOverlappingExpiredSSTables() throws InterruptedException
         options.put(TimeWindowCompactionStrategyOptions.COMPACTION_WINDOW_UNIT_KEY, "SECONDS");
         options.put(TimeWindowCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
         options.put(TimeWindowCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0");
-        TimeWindowCompactionStrategy twcs = new TimeWindowCompactionStrategy(cfs, options);
+        TimeWindowCompactionStrategy twcs = new TimeWindowCompactionStrategy(new CompactionStrategyFactory(cfs), options);
         for (SSTableReader sstable : cfs.getLiveSSTables())
             twcs.addSSTable(sstable);
 
         twcs.startup();
-        assertNull(twcs.getNextBackgroundTask(nowInSeconds()));
+        assertTrue(twcs.getNextBackgroundTasks(nowInSeconds()).isEmpty());
 
         // Wait for the expiration of the first sstable
         Thread.sleep(TimeUnit.SECONDS.toMillis(TTL_SECONDS + 1));
-        assertNull(twcs.getNextBackgroundTask(nowInSeconds()));
+        assertTrue(twcs.getNextBackgroundTasks(nowInSeconds()).isEmpty());
 
         options.put(TimeWindowCompactionStrategyOptions.UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY, "true");
-        twcs = new TimeWindowCompactionStrategy(cfs, options);
+        twcs = new TimeWindowCompactionStrategy(new CompactionStrategyFactory(cfs), options);
         for (SSTableReader sstable : cfs.getLiveSSTables())
             twcs.addSSTable(sstable);
 
         twcs.startup();
-        AbstractCompactionTask t = twcs.getNextBackgroundTask(nowInSeconds());
+        Collection<AbstractCompactionTask> tasks = twcs.getNextBackgroundTasks(nowInSeconds());
+        assertEquals(1, tasks.size());
+        AbstractCompactionTask t = tasks.iterator().next();
         assertNotNull(t);
         assertEquals(1, Iterables.size(t.transaction.originals()));
         SSTableReader sstable = t.transaction.originals().iterator().next();
@@ -390,9 +394,17 @@ public void testGroupForAntiCompaction()
             sstables.add(MockSchema.sstableWithTimestamp(i, curr + TimeUnit.MILLISECONDS.convert(i, TimeUnit.MINUTES), cfs));
 
         cfs.addSSTables(sstables);
-        Collection<Collection<SSTableReader>> groups = cfs.getCompactionStrategyManager().getCompactionStrategyFor(sstables.get(0)).groupSSTablesForAntiCompaction(sstables);
+        CompactionStrategyContainer compactionStrategyContainer = cfs.getCompactionStrategyContainer();
+        assert compactionStrategyContainer instanceof CompactionStrategyManager;
+        CompactionStrategyManager compactionStrategyManager = (CompactionStrategyManager) compactionStrategyContainer;
+        Collection<Collection<CompactionSSTable>> groups = compactionStrategyManager.getCompactionStrategyFor(sstables.get(0)).groupSSTablesForAntiCompaction(sstables);
         assertTrue(groups.size() > 0);
-        for (Collection<SSTableReader> group : groups)
+        for (Collection<CompactionSSTable> group : groups)
             assertEquals(1, group.size());
     }
+
+    private static Set<CompactionPick> toCompactions(List<CompactionAggregate> aggregates)
+    {
+        return aggregates.stream().flatMap(aggr -> aggr.getActive().stream()).collect(Collectors.toSet());
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionContainerPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionContainerPendingRepairTest.java
new file mode 100644
index 000000000000..6a2f3f8eb961
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionContainerPendingRepairTest.java
@@ -0,0 +1,804 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy.Arena;
+import org.apache.cassandra.db.compaction.unified.UnifiedCompactionTask;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.repair.consistent.LocalSession;
+import org.apache.cassandra.repair.consistent.LocalSessionAccessor;
+import org.apache.cassandra.repair.consistent.LocalSessions;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests UnifiedCompactionContainer's handling of pending repair sstables
+ */
+public class UnifiedCompactionContainerPendingRepairTest extends AbstractPendingRepairTest implements CompactionStrategyContainerPendingRepairTest
+{
+    @Override
+    public String createTableCql()
+    {
+        return String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, tbl);
+    }
+
+    @Override
+    void handleOrphan(SSTableReader sstable)
+    {
+        // UCS is stateless, so nothing to do
+    }
+
+    @Override
+    @Test
+    public void testSstableAdded() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        SSTableReader sstable = makeSSTable(true);
+        Assert.assertFalse(sstable.isRepaired());
+        Assert.assertFalse(sstable.isPendingRepair());
+        assertShardContainsSstable(sstable, false, false, false,null,true, true);
+
+        cfs.mutateRepaired(ImmutableList.of(sstable), 0, repairID, false);
+
+        Assert.assertFalse(sstable.isRepaired());
+        assertTrue(sstable.isPendingRepair());
+        assertEquals(repairID, sstable.getPendingRepair());
+        assertShardContainsSstable(sstable, false, true, false, repairID, true,true);
+    }
+
+    @Override
+    @Test
+    public void testSstableDeleted() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        final boolean isOrphan = true;
+        SSTableReader sstable = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable.isRepaired());
+        Assert.assertFalse(sstable.isPendingRepair());
+        assertShardContainsSstable(sstable, false, false, false, null,true, true);
+
+        cfs.mutateRepaired(ImmutableList.of(sstable), 0, repairID, false);
+
+        Assert.assertFalse(sstable.isRepaired());
+        Assert.assertTrue(sstable.isPendingRepair());
+        assertEquals(repairID, sstable.getPendingRepair());
+        assertShardContainsSstable(sstable, false, true, false, repairID, true,true);
+
+        // delete sstable
+        cfs.markObsolete(Collections.singletonList(sstable), OperationType.UNKNOWN);
+
+        assertShardContainsSstable(sstable, false, true, false, repairID, false,false);
+    }
+
+    @Override
+    @Test
+    public void testSstableListChangedAddAndRemove() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        final boolean isOrphan = true;
+        SSTableReader sstable1 = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable1.isRepaired());
+        Assert.assertFalse(sstable1.isPendingRepair());
+        assertShardContainsSstable(sstable1, false, false, false, null,true, true);
+
+        SSTableReader sstable2 = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable2.isRepaired());
+        Assert.assertFalse(sstable2.isPendingRepair());
+        assertShardContainsSstable(sstable2, false, false, false, null,true, true);
+
+        cfs.mutateRepaired(ImmutableList.of(sstable1, sstable2), 0, repairID, false);
+
+        Assert.assertFalse(sstable1.isRepaired());
+        Assert.assertTrue(sstable1.isPendingRepair());
+        assertEquals(repairID, sstable1.getPendingRepair());
+        assertShardContainsSstable(sstable1, false, true, false, repairID, true,true);
+
+        Assert.assertFalse(sstable2.isRepaired());
+        Assert.assertTrue(sstable2.isPendingRepair());
+        assertEquals(repairID, sstable2.getPendingRepair());
+        assertShardContainsSstable(sstable2, false, true, false, repairID,true,true);
+
+        // remove sstable1
+        cfs.markObsolete(Collections.singletonList(sstable1), OperationType.UNKNOWN);
+
+        assertShardContainsSstable(sstable1, false, true, false, repairID,false,false);
+        assertShardContainsSstable(sstable2, false, true, false, repairID,true,true);
+    }
+
+    @Override
+    @Test
+    public void testSstableRepairStatusChanged() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        // add as unrepaired
+        final boolean isOrphan = false;
+        SSTableReader sstable = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable.isRepaired());
+        Assert.assertFalse(sstable.isPendingRepair());
+        assertShardContainsSstable(sstable, false, false, false, null,true, true);
+
+        // change to pending repair
+        cfs.mutateRepaired(Collections.singletonList(sstable), 0, repairID, false);
+
+        Assert.assertFalse(sstable.isRepaired());
+        Assert.assertTrue(sstable.isPendingRepair());
+        assertEquals(repairID, sstable.getPendingRepair());
+        assertShardContainsSstable(sstable, false, true, false, repairID, true,true);
+
+        // change to repaired
+        long repairedAt = System.currentTimeMillis();
+        cfs.mutateRepaired(Collections.singletonList(sstable), repairedAt, null, false);
+
+        Assert.assertTrue(sstable.isRepaired());
+        Assert.assertFalse(sstable.isPendingRepair());
+        assertShardContainsSstable(sstable, true, false, false, null,true,true);
+    }
+
+    @Override
+    @Test
+    public void testStrategiesContainsPendingRepair() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        final boolean isOrphan = true;
+        SSTableReader sstable = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable.isRepaired());
+        Assert.assertFalse(sstable.isPendingRepair());
+        assertShardContainsSstable(sstable, false, false, false, null,true, true);
+
+        assertFalse(cfs.hasPendingRepairSSTables(repairID));
+
+        cfs.mutateRepaired(Collections.singletonList(sstable), 0, repairID, false);
+
+        Assert.assertFalse(sstable.isRepaired());
+        Assert.assertTrue(sstable.isPendingRepair());
+        assertEquals(repairID, sstable.getPendingRepair());
+        assertShardContainsSstable(sstable, false, true, false, repairID,true,true);
+
+        assertTrue(cfs.hasPendingRepairSSTables(repairID));
+    }
+
+
+    /**
+     * Tests that finalized repairs racing with compactions on the same set of sstables don't leave unrepaired sstables behind
+     * 
+     * This test checks that when a repair has been finalized but there are still pending sstables a finalize repair
+     * compaction task is issued for that repair session.
+     */
+    @Test
+    public void testFinalizedAndCompactionRace() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        final boolean isOrphan = true;
+        int numberOfSStables = 4; // this has to be >= T
+        List<SSTableReader> sstables = new ArrayList<>(numberOfSStables);
+        for (int i = 0; i < numberOfSStables; i++)
+        {
+            SSTableReader sstable = makeSSTable(isOrphan);
+            sstables.add(sstable);
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertFalse(sstable.isPendingRepair());
+            assertShardContainsSstable(sstable, false, false, false, null, true, true);
+        }
+
+        // change to pending repair
+        cfs.mutateRepaired(sstables, 0, repairID, false);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            assertEquals(repairID, sstable.getPendingRepair());
+            assertShardContainsSstable(sstable, false, true, false, repairID, true, true);
+        }
+
+        // Get a reference to compact the sstables that are pending repair in the same pending repair session ID
+        assertEquals(numberOfSStables, cfs.getPendingRepairSSTables(repairID).size());
+        compactionStrategyContainer.enable();
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
+        assertSame(UnifiedCompactionTask.class, compactionTask.getClass());
+
+        // Finalize & complete the repair session before the compaction executes
+        LocalSessionAccessor.finalizeUnsafe(repairID);
+        LocalSession session = ARS.consistent.local.getSession(repairID);
+        ARS.consistent.local.sessionCompleted(session);
+
+        // Complete the compaction
+        compactionTask.execute();
+
+        // The repair session is finalized but there is an sstable left behind pending repair!
+        SSTableReader compactedSSTable = cfs.getPendingRepairSSTables(repairID).iterator().next();
+
+        System.out.println("*********************************************************************************************");
+        System.out.println(compactedSSTable);
+        System.out.println("Pending repair UUID: " + compactedSSTable.getPendingRepair());
+        System.out.println("Repaired at: " + compactedSSTable.getRepairedAt());
+        System.out.println("Creation time: " + compactedSSTable.getCreationTimeFor(SSTableFormat.Components.DATA));
+        System.out.println("Live sstables: " + cfs.getLiveSSTables().size());
+        System.out.println("Pending repair sstables: " + cfs.getPendingRepairSSTables(repairID).size());
+        System.out.println("*********************************************************************************************");
+
+        // Run compaction again. It should pick up the pending repair sstables
+        compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        if (!compactionTasks.isEmpty())
+        {
+            assertEquals(1, compactionTasks.size());
+            compactionTask = compactionTasks.iterator().next();
+            assertNotNull(compactionTask);
+            Assert.assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
+            compactionTask.execute();
+        }
+
+        System.out.println("*********************************************************************************************");
+        System.out.println(compactedSSTable);
+        System.out.println("Pending repair UUID: " + compactedSSTable.getPendingRepair());
+        System.out.println("Repaired at: " + compactedSSTable.getRepairedAt());
+        System.out.println("Creation time: " + compactedSSTable.getCreationTimeFor(SSTableFormat.Components.DATA));
+        System.out.println("Live sstables: " + cfs.getLiveSSTables().size());
+        System.out.println("Pending repair sstables: " + cfs.getPendingRepairSSTables(repairID).size());
+        System.out.println("*********************************************************************************************");
+
+        compactionStrategyContainer.disable();
+
+        assertEquals(0, cfs.getPendingRepairSSTables(repairID).size());
+        assertEquals(1, cfs.getLiveSSTables().size());
+    }
+
+    /**
+     * Tests that finalized repairs result in {@link LocalSessions#sessionCompleted}
+     * which reclassify the sstables as repaired
+     */
+    @Override
+    @Test
+    public void testCleanupCompactionFinalized() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        final boolean isOrphan = true;
+        int numberOfSStables = 4; // this has to be >= T
+        List<SSTableReader> sstables = new ArrayList<>(numberOfSStables);
+        for (int i = 0; i < numberOfSStables; i++)
+        {
+            SSTableReader sstable = makeSSTable(isOrphan);
+            sstables.add(sstable);
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertFalse(sstable.isPendingRepair());
+            assertShardContainsSstable(sstable, false, false, false, null, true, true);
+        }
+
+        // change to pending repair
+        cfs.mutateRepaired(sstables, 0, repairID, false);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            assertEquals(repairID, sstable.getPendingRepair());
+            assertShardContainsSstable(sstable, false, true, false, repairID, true, true);
+        }
+
+        // finalize
+        LocalSessionAccessor.finalizeUnsafe(repairID);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            assertEquals(repairID, sstable.getPendingRepair());
+        }
+
+        // enable compaction to fetch next background task
+        compactionStrategyContainer.enable();
+
+        // Finish repair for any pending repair sstables for a finalized session
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
+        assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
+        assertEquals(0L, compactionTask.getSpaceOverhead());
+        compactionTask.execute();
+
+        // Compact any remaining sstables
+        compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
+        assertSame(UnifiedCompactionTask.class, compactionTask.getClass());
+        compactionTask.execute();
+
+        // sstables should not be found in any shards after compacted
+        for (SSTableReader sstable : sstables)
+        {
+            assertShardContainsSstable(sstable, false, true, false, repairID, false, false);
+            assertFalse(cfs.getLiveSSTables().contains(sstable));
+            assertFalse(cfs.getPendingRepairSSTables(repairID).contains(sstable));
+        }
+
+        // complete session
+        LocalSession session = ARS.consistent.local.getSession(repairID);
+        ARS.consistent.local.sessionCompleted(session);
+
+        assertEquals(0, cfs.getPendingRepairSSTables(repairID).size());
+        assertEquals(1, cfs.getLiveSSTables().size());
+    }
+
+    @Override
+    @Test
+    public void testFinalizedSessionTransientCleanup() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        final boolean isOrphan = true;
+        int numberOfSStables = 4; // this has to be >= T
+        List<SSTableReader> sstables = new ArrayList<>(numberOfSStables);
+        for (int i = 0; i < numberOfSStables; i++)
+        {
+            SSTableReader sstable = makeSSTable(isOrphan);
+            sstables.add(sstable);
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertFalse(sstable.isPendingRepair());
+            assertShardContainsSstable(sstable, false, false, false, null, true, true);
+        }
+
+        // change to pending repair
+        cfs.mutateRepaired(sstables, 0, repairID, true);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            Assert.assertTrue(sstable.isTransient());
+            assertEquals(repairID, sstable.getPendingRepair());
+            assertShardContainsSstable(sstable, false, true, true, repairID, true, true);
+        }
+
+        // finalize
+        LocalSessionAccessor.finalizeUnsafe(repairID);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            Assert.assertTrue(sstable.isTransient());
+            assertEquals(repairID, sstable.getPendingRepair());
+        }
+
+        // enable compaction to fetch next background task
+        compactionStrategyContainer.enable();
+
+        // pending repair sstables should be compacted
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
+        assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
+
+        // run the compaction
+        compactionTask.execute();
+
+        // sstables should not be found in any shards after compacted
+        for (SSTableReader sstable : sstables)
+        {
+            assertShardContainsSstable(sstable, false, true, true, repairID, false, false);
+            assertFalse(cfs.getLiveSSTables().contains(sstable));
+            assertFalse(cfs.getPendingRepairSSTables(repairID).contains(sstable));
+        }
+
+        // complete session
+        LocalSession session = ARS.consistent.local.getSession(repairID);
+        ARS.consistent.local.sessionCompleted(session);
+
+        assertTrue(cfs.getLiveSSTables().isEmpty());
+        assertEquals(0, cfs.getPendingRepairSSTables(repairID).size());
+    }
+
+    @Override
+    @Test
+    public void testFailedSessionTransientCleanup() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        final boolean isOrphan = true;
+        int numberOfSStables = 4; // this has to be >= T
+        List<SSTableReader> sstables = new ArrayList<>(numberOfSStables);
+        for (int i = 0; i < numberOfSStables; i++)
+        {
+            SSTableReader sstable = makeSSTable(isOrphan);
+            sstables.add(sstable);
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertFalse(sstable.isPendingRepair());
+            assertShardContainsSstable(sstable, false, false, false, null, true, true);
+        }
+
+        // change to pending repair
+        cfs.mutateRepaired(sstables, 0, repairID, true);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            Assert.assertEquals(repairID, sstable.getPendingRepair());
+            assertShardContainsSstable(sstable, false, true, true, repairID, true, true);
+        }
+        // fail
+        LocalSessionAccessor.failUnsafe(repairID);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            Assert.assertEquals(repairID, sstable.getPendingRepair());
+            assertShardContainsSstable(sstable, false, true, true, repairID, true, true);
+        }
+
+        // enable compaction to fetch next background task
+        compactionStrategyContainer.enable();
+
+        // Finish repair for any pending repair sstables for a finalized session
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
+        assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
+        compactionTask.execute();
+
+        // Compact any remaining sstables
+        compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
+        assertSame(UnifiedCompactionTask.class, compactionTask.getClass());
+        compactionTask.execute();
+
+        // sstables should not be found in any shards after compacted
+        for (SSTableReader sstable : sstables)
+        {
+            assertShardContainsSstable(sstable, false, true, true, repairID, false, false);
+            assertFalse(cfs.getLiveSSTables().contains(sstable));
+            assertFalse(cfs.getPendingRepairSSTables(repairID).contains(sstable));
+        }
+
+        // complete session
+        LocalSession session = ARS.consistent.local.getSession(repairID);
+        ARS.consistent.local.sessionCompleted(session);
+
+        assertEquals(0, cfs.getPendingRepairSSTables(repairID).size());
+        assertEquals(1, cfs.getLiveSSTables().size());
+    }
+
+    /**
+     * Tests that failed repairs result in {@link LocalSessions#sessionCompleted}
+     * which reclassify the sstables as unrepaired
+     */
+    @Override
+    @Test
+    public void testCleanupCompactionFailed() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        final boolean isOrphan = true;
+        int numberOfSStables = 4; // this has to be >= T
+        List<SSTableReader> sstables = new ArrayList<>(numberOfSStables);
+        for (int i = 0; i < numberOfSStables; i++)
+        {
+            SSTableReader sstable = makeSSTable(isOrphan);
+            sstables.add(sstable);
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertFalse(sstable.isPendingRepair());
+            assertShardContainsSstable(sstable, false, false, false, null, true, true);
+        }
+
+        // change to pending repair
+        cfs.mutateRepaired(sstables, 0, repairID, false);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            Assert.assertEquals(repairID, sstable.getPendingRepair());
+            assertShardContainsSstable(sstable, false, true, false, repairID, true, true);
+        }
+
+        // fail
+        LocalSessionAccessor.failUnsafe(repairID);
+
+        for (SSTableReader sstable : sstables)
+        {
+            Assert.assertFalse(sstable.isRepaired());
+            Assert.assertTrue(sstable.isPendingRepair());
+            Assert.assertEquals(repairID, sstable.getPendingRepair());
+            assertShardContainsSstable(sstable, false, true, false, repairID, true, true);
+        }
+
+        // enable compaction to fetch next background task
+        compactionStrategyContainer.enable();
+
+        // Finish repair for any pending repair sstables for a finalized session
+        Collection<AbstractCompactionTask> compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        AbstractCompactionTask compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
+        assertSame(RepairFinishedCompactionTask.class, compactionTask.getClass());
+        compactionTask.execute();
+
+        // Compact any remaining sstables
+        compactionTasks = compactionStrategyContainer.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals(1, compactionTasks.size());
+        compactionTask = compactionTasks.iterator().next();
+        assertNotNull(compactionTask);
+        assertSame(UnifiedCompactionTask.class, compactionTask.getClass());
+        compactionTask.execute();
+
+        // sstables should not be found in any shards after compacted
+        for (SSTableReader sstable : sstables)
+        {
+            assertShardContainsSstable(sstable, false, true, false, repairID, false, false);
+            assertFalse(cfs.getLiveSSTables().contains(sstable));
+            assertFalse(cfs.getPendingRepairSSTables(repairID).contains(sstable));
+        }
+
+        // complete session
+        LocalSession session = ARS.consistent.local.getSession(repairID);
+        ARS.consistent.local.sessionCompleted(session);
+
+        assertEquals(0, cfs.getPendingRepairSSTables(repairID).size());
+        assertEquals(1, cfs.getLiveSSTables().size());
+    }
+
+    @Override
+    @Test
+    public void testSessionCompleted() throws IOException
+    {
+        TimeUUID repairID = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
+
+        // add sstables as unrepaired
+        final boolean isOrphan = false;
+        SSTableReader sstable1 = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable1.isRepaired());
+        Assert.assertFalse(sstable1.isPendingRepair());
+        assertShardContainsSstable(sstable1, false, false, false, null, true, true);
+
+        SSTableReader sstable2 = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable2.isRepaired());
+        Assert.assertFalse(sstable2.isPendingRepair());
+        assertShardContainsSstable(sstable2, false, false, false, null, true, true);
+        SSTableReader sstable3 = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable3.isRepaired());
+        Assert.assertFalse(sstable3.isPendingRepair());
+        assertShardContainsSstable(sstable3, false, false, false, null, true, true);
+
+        // change to pending repair
+        cfs.mutateRepaired(ImmutableList.of(sstable1, sstable2, sstable3), 0, repairID, false);
+
+        Assert.assertFalse(sstable1.isRepaired());
+        Assert.assertTrue(sstable1.isPendingRepair());
+        Assert.assertEquals(repairID, sstable1.getPendingRepair());
+        assertShardContainsSstable(sstable1, false, true, false, repairID, true, true);
+        Assert.assertFalse(sstable2.isRepaired());
+        Assert.assertTrue(sstable2.isPendingRepair());
+        Assert.assertEquals(repairID, sstable2.getPendingRepair());
+        assertShardContainsSstable(sstable2, false, true, false, repairID, true, true);
+
+        Assert.assertFalse(sstable3.isRepaired());
+        Assert.assertTrue(sstable3.isPendingRepair());
+        Assert.assertEquals(repairID, sstable3.getPendingRepair());
+        assertShardContainsSstable(sstable3, false, true, false, repairID, true, true);
+
+        // finalize
+        LocalSessionAccessor.finalizeUnsafe(repairID);
+
+        // complete (repair) session and sstables should be marked as repaired
+        LocalSession session = ARS.consistent.local.getSession(repairID);
+        ARS.consistent.local.sessionCompleted(session);
+
+        // sstables are repaired
+        assertShardContainsSstable(sstable1, true, false, false, null, true, true);
+        assertShardContainsSstable(sstable2, true, false, false, null, true, true);
+        assertShardContainsSstable(sstable3, true, false, false, null, true, true);
+    }
+
+    @Override
+    @Test
+    public void testSessionCompletedWithDifferentSSTables() throws IOException
+    {
+        TimeUUID repairID1 = registerSession(cfs, true, true);
+        TimeUUID repairID2 = registerSession(cfs, true, true);
+        LocalSessionAccessor.prepareUnsafe(repairID1, COORDINATOR, PARTICIPANTS);
+        LocalSessionAccessor.prepareUnsafe(repairID2, COORDINATOR, PARTICIPANTS);
+
+        // add sstables as unrepaired
+        final boolean isOrphan = false;
+        SSTableReader sstable1 = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable1.isRepaired());
+        Assert.assertFalse(sstable1.isPendingRepair());
+        assertShardContainsSstable(sstable1, false, false, false, null, true, true);
+        SSTableReader sstable2 = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable2.isRepaired());
+        Assert.assertFalse(sstable2.isPendingRepair());
+        assertShardContainsSstable(sstable2, false, false, false, null, true, true);
+        SSTableReader sstable3 = makeSSTable(isOrphan);
+        Assert.assertFalse(sstable3.isRepaired());
+        Assert.assertFalse(sstable3.isPendingRepair());
+        assertShardContainsSstable(sstable3, false, false, false, null, true, true);
+
+        // change sstable1 to pending repair for session 1
+        cfs.mutateRepaired(Collections.singletonList(sstable1), 0, repairID1, false);
+        Assert.assertFalse(sstable1.isRepaired());
+        Assert.assertTrue(sstable1.isPendingRepair());
+        Assert.assertEquals(repairID1, sstable1.getPendingRepair());
+        assertShardContainsSstable(sstable1, false, true, false, repairID1, true, true);
+        assertNumberOfShards(2);
+
+        // change sstable2 to pending repair for session 2
+        cfs.mutateRepaired(Collections.singletonList(sstable2), 0, repairID2, false);
+        Assert.assertFalse(sstable2.isRepaired());
+        Assert.assertTrue(sstable2.isPendingRepair());
+        Assert.assertEquals(repairID2, sstable2.getPendingRepair());
+        assertNumberOfShards(3);
+        assertShardContainsSstable(sstable2, false, true, false, repairID2, true, true);
+
+        // change sstable3 to repaired
+        long repairedAt3 = System.currentTimeMillis();
+        cfs.mutateRepaired(Collections.singletonList(sstable3), repairedAt3, null, false);
+        Assert.assertTrue(sstable3.isRepaired());
+        Assert.assertFalse(sstable3.isPendingRepair());
+        assertNumberOfShards(3);
+        assertShardContainsSstable(sstable3, true, false, false, null, true, true);
+
+        // finalize session 1
+        LocalSessionAccessor.finalizeUnsafe(repairID1);
+
+        // simulate index build on pending sstable for session 1
+        cfs.getTracker().tryModify(sstable1, OperationType.INDEX_BUILD);
+
+        // completing session 1 will not require to disable compactions because:
+        // * sstable1 is building index (and considered as compacting), which would not be found in any shards
+        // * sstable2 belongs to a different session
+        // * sstable3 is repaired
+        LocalSession session1 = ARS.consistent.local.getSession(repairID1);
+        ARS.consistent.local.sessionCompleted(session1);
+
+        // expecting sstable1 not found in any shards
+        assertShardContainsSstable(sstable1, false, true, false, repairID1, false, true);
+        // expecting sstable2 exists in pending repair shard
+        assertShardContainsSstable(sstable2, false, true, false, repairID2, true, true);
+        // expecting sstable3 exists in repaired shards
+        assertShardContainsSstable(sstable3, true, false, false, null, true, true);    }
+
+    private void assertNumberOfShards(int expectedNumberOfShards)
+    {
+        Collection<CompactionStrategy> compactionStrategies = compactionStrategyContainer.getStrategies();
+        assertEquals(1, compactionStrategies.size());
+        compactionStrategies.forEach(cs -> {
+            assertTrue(cs instanceof UnifiedCompactionStrategy);
+
+            UnifiedCompactionStrategy ucs = ((UnifiedCompactionStrategy) cs);
+            assertEquals("Expecting number of shards in the strategy.",
+                         expectedNumberOfShards,
+                         ucs.getLevels().keySet().size());
+        });
+    }
+
+    private void assertShardContainsSstable(SSTableReader sstable,
+                                            boolean expectedIsRepaired,
+                                            boolean expectedIsPending,
+                                            boolean expectedIsTransient,
+                                            TimeUUID expectedRepairId,
+                                            boolean expectedRepairStatus,
+                                            boolean expectedContainsSstable)
+    {
+        List<CompactionStrategy> compactionStrategies = compactionStrategyContainer.getStrategies();
+        // CompactionStrategyContainer should always contains 1 UnifiedCompactionStrategy
+        assertEquals(1, compactionStrategies.size());
+        compactionStrategies.forEach(cs -> {
+            assertTrue(cs instanceof UnifiedCompactionStrategy);
+
+            UnifiedCompactionStrategy ucs = ((UnifiedCompactionStrategy) cs);
+            Set<CompactionSSTable> ucsSstables = ucs.getSSTables()
+                                                    .stream()
+                                                    .filter(sst -> sst.equals(sstable))
+                                                    .collect(Collectors.toSet());
+
+            assertEquals("Expecting strategy contains sstable.", expectedContainsSstable, ucsSstables.size() == 1);
+
+            Map<Arena, List<UnifiedCompactionStrategy.Level>> shardListMap = ucs.getLevels();
+            Set<Arena> arenas = shardListMap.keySet();
+
+            if (expectedRepairStatus)
+            {
+                Set<Arena> shardsWithPrefix = arenas.stream()
+                                                    .filter(shard -> {
+                                                        if (shard.getSSTables().isEmpty())
+                                                            return false;
+
+                                                        SSTableReader shardSSTable = (SSTableReader) shard.getSSTables().get(0);
+                                                        return shardSSTable.isRepaired() == expectedIsRepaired &&
+                                                               shardSSTable.isTransient() == expectedIsTransient &&
+                                                               shardSSTable.isPendingRepair() == expectedIsPending &&
+                                                               (shardSSTable.getPendingRepair() == null
+                                                                ? expectedRepairId == null
+                                                                : shardSSTable.getPendingRepair().equals(expectedRepairId));
+                                                    })
+                                                    .collect(Collectors.toSet());
+
+                assertEquals(String.format("Expecting a shard with repair status: pending=%s repaired=%s but found %s of it.",
+                                           expectedIsPending, expectedIsRepaired, shardsWithPrefix.size()),
+                             1,
+                             shardsWithPrefix.size());
+
+                Arena shardWithPrefix = shardsWithPrefix.iterator().next();
+                assertEquals(String.format("Expecting a shard with repair status: %s contains the sstable is %s.",
+                                           expectedRepairStatus,
+                                           expectedContainsSstable),
+                             expectedContainsSstable,
+                             shardWithPrefix.getSSTables().contains(sstable));
+            }
+            else
+            {
+                // not expecting any shard would contain the sstable
+                Set<Arena> shardsContainsSstable = arenas.stream()
+                                                         .filter(shard -> shard.getSSTables().contains(sstable))
+                                                         .collect(Collectors.toSet());
+
+                assertTrue(String.format("Expecting no shard should contain the sstable but found exists in %s",
+                                         shardsContainsSstable),
+                           shardsContainsSstable.isEmpty());
+            }
+        });
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionTest.java b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionTest.java
new file mode 100644
index 000000000000..25d99cb4cd5e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionTest.java
@@ -0,0 +1,427 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.compaction.unified.Controller;
+import org.apache.cassandra.db.compaction.unified.Reservations;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Overlaps;
+import org.apache.cassandra.utils.TimeUUID;
+import org.hamcrest.Matchers;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.when;
+
+@RunWith(Parameterized.class)
+public class UnifiedCompactionStrategyGetSelectionTest extends BaseCompactionStrategyTest
+{
+    @Parameterized.Parameter(0)
+    public double modifier;
+
+    @Parameterized.Parameter(1)
+    public int reservations;
+
+    @Parameterized.Parameter(2)
+    public Reservations.Type reservationsType;
+
+    @Parameterized.Parameter(3)
+    public int levels;
+
+    @Parameterized.Parameter(4)
+    public int compactors;
+
+    static final long START_SIZE = 1L << 30;
+
+
+    @Parameterized.Parameters(name = "Type {2} Reservations {1} Modifier {0} Levels {3} Compactors {4}")
+    public static List<Object[]> params()
+    {
+        ArrayList<Object[]> params = new ArrayList<>();
+        for (Reservations.Type reservationsType : Reservations.Type.values())
+            for (int reservations : new int[]{ 0, 1, Integer.MAX_VALUE })
+            {
+                if (reservations == 0 && reservationsType == Reservations.Type.LEVEL_OR_BELOW)
+                    continue; // if we don't have reservations, the type doesn't matter, save some time
+                for (double modifier : new double[]{ 0.0, 0.5, 1.0 })
+                    for (int levels = 1; levels < 5; ++levels)
+                        for (int compactors : new int[]{ 1, 4, 15, 30 })
+                            params.add(new Object[]{ modifier, reservations, reservationsType, levels, compactors });
+            }
+        return params;
+    }
+
+    @Test
+    public void testGetSelection()
+    {
+        testGetSelection(generateCompactions(levels, 8 + compactors * 4, START_SIZE, modifier),
+                         requestedParallelism(),
+                         reservations,
+                         reservationsType,
+                         compactors,
+                         levels,
+                         100L << 30,
+                         random.nextInt(20) + 1);
+    }
+
+    boolean ignoreRepeats()
+    {
+        return true;
+    }
+
+    int requestedParallelism()
+    {
+        return 1;
+    }
+
+    List<CompactionSSTable> getSSTablesSet(List<List<CompactionSSTable>> sets, int levels, int perLevel, int level, int sstableInLevel)
+    {
+        return sets.get(0);
+    }
+
+    List<List<CompactionSSTable>> prepareSSTablesSets(int levels, int perLevel)
+    {
+        // We are reusing the same set, the construction will ignore it
+        final List<CompactionSSTable> fakeSet = ImmutableList.of(Mockito.mock(CompactionSSTable.class));
+        return ImmutableList.of(fakeSet);
+    }
+
+    List<CompactionAggregate.UnifiedAggregate> generateCompactions(int levels, int perLevel, long startSize, double sizeModifier)
+    {
+        double growth = Math.pow(2, 1 - sizeModifier);
+        List<CompactionAggregate.UnifiedAggregate> list = new ArrayList<>();
+        List<List<CompactionSSTable>> sets = prepareSSTablesSets(levels, perLevel);
+        long size = startSize;
+        for (int i = 0; i < levels; ++i)
+        {
+            for (int j = 0; j < perLevel; ++j)
+            {
+                int overlap = (int) Math.max(0, random.nextGaussian() * 5 + 15);
+                CompactionPick pick = CompactionPick.create(TimeUUID.Generator.nextTimeUUID(),
+                                                            i,
+                                                            getSSTablesSet(sets, levels, perLevel, i, j),
+                                                            Collections.emptySet(),
+                                                            random.nextInt(20) == 0 ? -1 : 1,
+                                                            size,
+                                                            size,
+                                                            size);
+                CompactionAggregate.UnifiedAggregate aggregate = Mockito.mock(CompactionAggregate.UnifiedAggregate.class, Mockito.withSettings().stubOnly());
+                when(aggregate.getSelected()).thenReturn(pick);
+                when(aggregate.maxOverlap()).thenReturn(overlap);
+                when(aggregate.toString()).thenAnswer(inv -> toString((CompactionAggregate) inv.getMock()));
+                Mockito.doCallRealMethod().when(aggregate).setPermittedParallelism(anyInt());
+                Mockito.doCallRealMethod().when(aggregate).getPermittedParallelism();
+                list.add(aggregate);
+            }
+            size *= growth;
+        }
+        return list;
+    }
+
+    static String toString(CompactionAggregate a)
+    {
+        CompactionAggregate.UnifiedAggregate u = (CompactionAggregate.UnifiedAggregate) a;
+        CompactionPick p = u.getSelected();
+        return String.format("level %d size %s overlap %d parallelism %d%s", levelOf(p), FBUtilities.prettyPrintMemory(p.totSizeInBytes()), u.maxOverlap(), u.getPermittedParallelism(), p.hotness() < 0 ? " adaptive" : "");
+    }
+
+    public void testGetSelection(List<CompactionAggregate.UnifiedAggregate> compactions,
+                                 int requestedParallelism,
+                                 int reservations,
+                                 Reservations.Type reservationType,
+                                 int totalCount,
+                                 int levelCount,
+                                 long spaceAvailable,
+                                 int adaptiveLimit)
+    {
+        System.out.println(String.format("Starting testGetSelection: reservations %d, totalCount %d, levelCount %d, requestedParallelism %d, spaceAvailable %s, adaptiveLimit %d",
+                                         reservations,
+                                         totalCount,
+                                         levelCount,
+                                         requestedParallelism,
+                                         FBUtilities.prettyPrintMemory(spaceAvailable),
+                                         adaptiveLimit));
+        boolean ignoreRepeats = ignoreRepeats();
+
+        Controller controller = Mockito.mock(Controller.class, Mockito.withSettings().stubOnly());
+        when(controller.random()).thenAnswer(inv -> ThreadLocalRandom.current());
+        when(controller.prioritize(anyList())).thenCallRealMethod();
+        when(controller.getReservedThreads()).thenReturn(reservations);
+        when(controller.getReservationsType()).thenReturn(reservationType);
+        when(controller.getOverheadSizeInBytes(any(), anyLong())).thenAnswer(inv -> ((Long)inv.getArgument(1)).longValue());
+        when(controller.isRecentAdaptive(any())).thenAnswer(inv -> ((CompactionPick) inv.getArgument(0)).hotness() < 0);  // hotness is used to mock adaptive
+        when(controller.overlapInclusionMethod()).thenReturn(ignoreRepeats ? Overlaps.InclusionMethod.TRANSITIVE : Overlaps.InclusionMethod.NONE);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
+
+        UnifiedCompactionStrategy.ShardingStats stats = new UnifiedCompactionStrategy.ShardingStats(null, null, 0, 1.0, 0, 0, 0, requestedParallelism);
+        UnifiedCompactionStrategy strategy = Mockito.mock(UnifiedCompactionStrategy.class, Mockito.withSettings().stubOnly());
+        when(strategy.getController()).thenReturn(controller);
+        when(strategy.getShardingStats(any())).thenReturn(stats);
+        when(strategy.getSelection(any(), anyInt(), any(), anyLong(), anyInt())).thenCallRealMethod();
+
+        int[] perLevel = new int[levelCount];
+        int maxReservations = totalCount / levelCount;
+        boolean oneExtra = maxReservations < reservations;
+        reservations = Math.min(reservations, maxReservations);
+        int remainder = totalCount - levelCount * reservations;
+
+        List<CompactionAggregate> running = new ArrayList<>();
+
+        while (!compactions.isEmpty())
+        {
+            Arrays.fill(perLevel, 0);
+            long spaceTaken = 0;
+            int adaptiveUsed = 0;
+            for (CompactionAggregate aggregate : running)
+            {
+                CompactionPick compaction = aggregate.getSelected();
+                final int level = levelOf(compaction);
+                final int threads = ((CompactionAggregate.UnifiedAggregate) aggregate).getPermittedParallelism();
+                perLevel[level] += threads;
+                spaceTaken += compaction.totSizeInBytes();
+                if (controller.isRecentAdaptive(compaction))
+                    adaptiveUsed += threads;
+            }
+
+            List<CompactionAggregate> result = strategy.getSelection(compactions,
+                                                                     totalCount,
+                                                                     perLevel,
+                                                                     spaceAvailable - spaceTaken,
+                                                                     adaptiveLimit - adaptiveUsed);
+
+            System.out.println("Selected " + result.size() + ": " + result.stream()
+                                                                          .map(a -> toString(a))
+                                                                          .collect(Collectors.joining(", ")));
+            if (result.isEmpty())
+            {
+                Assert.assertFalse(running.isEmpty());
+                // if running is not empty, run through to remove something from it and try again
+            }
+
+
+            compactions.removeAll(result);
+            running.addAll(result);
+
+            Arrays.fill(perLevel, 0);
+            spaceTaken = 0;
+            adaptiveUsed = 0;
+            for (CompactionAggregate aggregate : running)
+            {
+                CompactionPick compaction = aggregate.getSelected();
+                final int level = levelOf(compaction);
+                final int threads = ((CompactionAggregate.UnifiedAggregate) aggregate).getPermittedParallelism();
+                perLevel[level] += threads;
+                spaceTaken += compaction.totSizeInBytes();
+                if (controller.isRecentAdaptive(compaction))
+                    adaptiveUsed += threads;
+            }
+
+            // Check that restrictions are honored
+            Assert.assertThat(running.size(), Matchers.lessThanOrEqualTo(totalCount));
+            Assert.assertThat(spaceTaken, Matchers.lessThanOrEqualTo(spaceAvailable));
+            Assert.assertThat(adaptiveUsed, Matchers.lessThanOrEqualTo(adaptiveLimit));
+            boolean extrasExhausted = verifyReservations(reservationType, reservations, levelCount, perLevel, remainder, oneExtra);
+
+            // Check that we do select what we can select
+            if (running.size() < totalCount)
+            {
+                for (int i = 0; i < levelCount; ++i)
+                {
+                    if (hasRoomInLevel(reservationType, reservations, remainder, oneExtra, extrasExhausted, perLevel, i))
+                    {
+                        List<CompactionAggregate.UnifiedAggregate> failures = getSelectablePicks(compactions,
+                                                                                                 ignoreRepeats
+                                                                                                   ? Collections.emptySet()
+                                                                                                   : running.stream().flatMap(a -> a.getSelected().sstables().stream()).collect(Collectors.toSet()),
+                                                                                                 spaceAvailable - spaceTaken,
+                                                                                                 adaptiveUsed == adaptiveLimit,
+                                                                                                 controller,
+                                                                                                 i);
+                        Assert.assertThat(failures, Matchers.hasSize(0));
+                    }
+                }
+            }
+
+            // Check priorities were respected
+            for (CompactionAggregate c : result)
+            {
+                CompactionPick p = c.getSelected();
+                int level = levelOf(p);
+                for (CompactionAggregate.UnifiedAggregate other : getSelectablePicks(compactions,
+                                                                                     ignoreRepeats
+                                                                                       ? Collections.emptySet()
+                                                                                       : running.stream().flatMap(a -> a.getSelected().sstables().stream()).collect(Collectors.toSet()),
+                                                                                     spaceAvailable - spaceTaken + p.totSizeInBytes(),
+                                                                                     controller.isRecentAdaptive(p) ? false : adaptiveUsed == adaptiveLimit,
+                                                                                     controller,
+                                                                                     level))
+                {
+                    final CompactionAggregate.UnifiedAggregate unifiedAggregate = (CompactionAggregate.UnifiedAggregate) c;
+                    Assert.assertThat(other.maxOverlap(), Matchers.lessThanOrEqualTo(unifiedAggregate.maxOverlap()));
+                    if (other.maxOverlap() == unifiedAggregate.maxOverlap())
+                        Assert.assertThat(other.bucketIndex(), Matchers.lessThanOrEqualTo(unifiedAggregate.bucketIndex()));
+                }
+            }
+
+            // Check that we don't assign higher parallelism than requested
+            for (CompactionAggregate c : result)
+                Assert.assertThat(((CompactionAggregate.UnifiedAggregate) c).getPermittedParallelism(), Matchers.lessThanOrEqualTo(requestedParallelism));
+
+
+            // randomly simulate some of them completing
+            int toRemove = (running.size() + 1) / 2; // round up, to remove one for size == 1
+            for (int i = 0; i < toRemove; ++i)
+                running.remove(random.nextInt(running.size()));
+        }
+    }
+
+    private static boolean verifyReservations(Reservations.Type type, int reservations, int levelCount, int[] perLevel, int remainder, boolean oneExtra)
+    {
+        switch (type)
+        {
+            case PER_LEVEL:
+                return verifyReservationsPerLevel(reservations, levelCount, perLevel, remainder, oneExtra);
+            case LEVEL_OR_BELOW:
+                return verifyReservationsLevelOrBelow(reservations, levelCount, perLevel, remainder, oneExtra);
+            default:
+                throw new AssertionError();
+        }
+    }
+    private static boolean verifyReservationsPerLevel(int reservations, int levelCount, int[] perLevel, int remainder, boolean oneExtra)
+    {
+        int remainderUsed = 0;
+        int allowedExtra = oneExtra ? 1 : remainder;
+        for (int i = 0; i < levelCount; ++i)
+        {
+            Assert.assertThat(perLevel[i], Matchers.lessThanOrEqualTo(reservations + allowedExtra));
+            if (perLevel[i] > reservations)
+                remainderUsed += perLevel[i] - reservations;
+        }
+        Assert.assertThat(remainderUsed, Matchers.lessThanOrEqualTo(remainder));
+        return remainderUsed >= remainder;
+    }
+
+    private static boolean verifyReservationsLevelOrBelow(int reservations, int levelCount, int[] perLevel, long remainder, boolean oneExtra)
+    {
+        long sum = 0;
+        long allowed = oneExtra ? 0 : remainder;
+        int count = 0;
+        for (int i = levelCount - 1; i >= 0; --i)
+        {
+            sum += perLevel[i];
+            allowed += reservations;
+            if (++count <= remainder && oneExtra)
+                ++allowed;
+            Assert.assertThat(sum, Matchers.lessThanOrEqualTo(allowed));
+        }
+        Assert.assertThat(sum, Matchers.lessThanOrEqualTo(remainder + levelCount * reservations));
+        assertEquals(allowed, remainder + levelCount * reservations);   // if failed, the problem is in the test
+        return sum >= remainder + levelCount * reservations;
+    }
+
+    private static boolean isAcceptableLevelOrBelow(int reservations, int levelCount, int[] perLevel, long remainder, boolean oneExtra)
+    {
+        long sum = 0;
+        long allowed = oneExtra ? 0 : remainder;
+        int count = 0;
+        for (int i = levelCount - 1; i >= 0; --i)
+        {
+            sum += perLevel[i];
+            allowed += reservations;
+            if (++count <= remainder && oneExtra)
+                ++allowed;
+            if (sum > allowed)
+                return false;
+        }
+        return true;
+    }
+
+    private static boolean hasRoomInLevel(Reservations.Type type, int reservations, int remainder, boolean oneExtra, boolean extrasExhausted, int perLevel[], int level)
+    {
+        switch (type)
+        {
+            case PER_LEVEL:
+                return hasRoomInLevelPerLevel(reservations, remainder, oneExtra, extrasExhausted, perLevel, level);
+            case LEVEL_OR_BELOW:
+                return hasRoomInLevelOrAbove(reservations, remainder, oneExtra, extrasExhausted, perLevel, level);
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    private static boolean hasRoomInLevelPerLevel(int reservations, int remainder, boolean oneExtra, boolean extrasExhausted, int perLevel[], int level)
+    {
+        int allowedExtra = extrasExhausted ? 0 : (oneExtra ? 1 : remainder);
+        return perLevel[level] < reservations + allowedExtra;
+    }
+
+    private static boolean hasRoomInLevelOrAbove(int reservations, int remainder, boolean oneExtra, boolean extrasExhausted, int perLevel[], int level)
+    {
+        if (extrasExhausted)
+            return false;
+        ++perLevel[level];
+        boolean result = isAcceptableLevelOrBelow(reservations, perLevel.length, perLevel, remainder, oneExtra);
+        --perLevel[level];
+        return result;
+    }
+
+    private static <T extends CompactionAggregate> List<T> getSelectablePicks(List<T> compactions, Set<CompactionSSTable> rejectIfContained, long spaceRemaining, boolean adaptiveAtLimit, Controller controller, int level)
+    {
+        List<T> failures = new ArrayList<>();
+        for (T compaction : compactions)
+        {
+            CompactionPick x = compaction.getSelected();
+            if (!isSelectable(rejectIfContained, spaceRemaining, adaptiveAtLimit, controller, level, x))
+                continue;
+
+            failures.add(compaction);
+        }
+        return failures;
+    }
+
+    private static boolean isSelectable(Set<CompactionSSTable> rejectIfContained, long spaceRemaining, boolean adaptiveAtLimit, Controller controller, int level, CompactionPick x)
+    {
+        if (levelOf(x) != level) return false;
+        if (x.totSizeInBytes() > spaceRemaining) return false;
+        if (adaptiveAtLimit && controller.isRecentAdaptive(x)) return false;
+        if (!Collections.disjoint(x.sstables(), rejectIfContained)) return false;
+        return true;
+    }
+
+    private static int levelOf(CompactionPick x)
+    {
+        return (int) x.parent();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionWithParallelismTest.java b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionWithParallelismTest.java
new file mode 100644
index 000000000000..6b2c66b9d722
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionWithParallelismTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+public class UnifiedCompactionStrategyGetSelectionWithParallelismTest extends UnifiedCompactionStrategyGetSelectionTest
+{
+    @Override
+    int requestedParallelism()
+    {
+        return 4;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionWithRepeatsTest.java b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionWithRepeatsTest.java
new file mode 100644
index 000000000000..e780f8192892
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyGetSelectionWithRepeatsTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.collect.ImmutableList;
+
+import org.mockito.Mockito;
+
+public class UnifiedCompactionStrategyGetSelectionWithRepeatsTest extends UnifiedCompactionStrategyGetSelectionTest
+{
+    @Override
+    boolean ignoreRepeats()
+    {
+        return false;
+    }
+
+    @Override
+    List<CompactionSSTable> getSSTablesSet(List<List<CompactionSSTable>> sets, int levels, int perLevel, int level, int inLevel)
+    {
+        return sets.get(getRepeatIndex(levels * perLevel, level * perLevel + inLevel));
+    }
+
+    @Override
+    List<List<CompactionSSTable>> prepareSSTablesSets(int levels, int perLevel)
+    {
+        return IntStream.range(0, levels * perLevel)
+                        .mapToObj(i -> ImmutableList.of(Mockito.mock(CompactionSSTable.class)))
+                        .collect(Collectors.toList());
+    }
+
+    private int getRepeatIndex(int size, int index)
+    {
+        double d = random.nextGaussian();
+        if (d <= 0.5 || d > 1)
+            return index;
+        else
+            return (int) (d * size - 1);    // high likelihood of hitting the same index
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java
index a673b27b7ceb..2ca4be9f0b62 100644
--- a/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java
@@ -20,7 +20,9 @@
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -28,40 +30,47 @@
 import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
-import org.apache.commons.math3.random.JDKRandomGenerator;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
-import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.BufferDecoratedKey;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.DiskBoundaries;
 import org.apache.cassandra.db.compaction.unified.Controller;
+import org.apache.cassandra.db.compaction.unified.Reservations;
 import org.apache.cassandra.db.compaction.unified.UnifiedCompactionTask;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.lifecycle.Tracker;
-import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.PartialLifecycleTransaction;
+import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Overlaps;
 import org.apache.cassandra.utils.Pair;
-import org.mockito.Answers;
-import org.mockito.Mock;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.Transactional;
 import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
@@ -69,90 +78,41 @@
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyDouble;
 import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.Mockito.RETURNS_SMART_NULLS;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.withSettings;
 
 /**
  * The unified compaction strategy is described in this design document:
  *
  * See CEP-26: https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-26%3A+Unified+Compaction+Strategy
  */
-public class UnifiedCompactionStrategyTest
+@RunWith(Parameterized.class)
+public class UnifiedCompactionStrategyTest extends BaseCompactionStrategyTest
 {
     private final static long ONE_MB = 1 << 20;
 
     // Multiple disks can be used both with and without disk boundaries. We want to test both cases.
 
-    final String keyspace = "ks";
-    final String table = "tbl";
-
-    @Mock(answer = Answers.RETURNS_SMART_NULLS)
-    ColumnFamilyStore cfs;
-
-    @Mock(answer = Answers.RETURNS_SMART_NULLS)
-    CompactionStrategyManager csm;
-
-    ColumnFamilyStore.VersionedLocalRanges localRanges;
-
-    Tracker dataTracker;
-
-    long repairedAt;
-
-    IPartitioner partitioner;
+    @Parameterized.Parameters(name = "useDiskBoundaries {0}")
+    public static Iterable<Object[]> params()
+    {
+        return Arrays.asList(new Object[][] { {false}, {true} });
+    }
 
-    Splitter splitter;
+    @Parameterized.Parameter
+    public boolean useDiskBoundaries = true;
 
     @BeforeClass
     public static void setUpClass()
     {
-        long seed = System.currentTimeMillis();
-        random.setSeed(seed);
-        System.out.println("Random seed: " + seed);
-
-        DatabaseDescriptor.daemonInitialization(); // because of all the static initialization in CFS
-        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        BaseCompactionStrategyTest.setUpClass();
     }
 
-
-    static final JDKRandomGenerator random = new JDKRandomGenerator();
-
     @Before
     public void setUp()
     {
-        setUp(1);
-    }
-
-    protected void setUp(int numShards)
-    {
-        MockitoAnnotations.initMocks(this);
-
-        TableMetadata metadata = TableMetadata.builder(keyspace, table)
-                                              .addPartitionKeyColumn("pk", AsciiType.instance)
-                                              .build();
-
-        dataTracker = Tracker.newDummyTracker();
-        repairedAt = System.currentTimeMillis();
-        partitioner = DatabaseDescriptor.getPartitioner();
-        splitter = partitioner.splitter().orElse(null);
-        if (numShards > 1)
-            assertNotNull("Splitter is required with multiple compaction shards", splitter);
-
-        when(cfs.getPartitioner()).thenReturn(partitioner);
-        localRanges = cfs.fullWeightedRange(0, partitioner);
-
-        when(cfs.metadata()).thenReturn(metadata);
-        when(cfs.getTableName()).thenReturn(table);
-        when(cfs.localRangesWeighted()).thenReturn(localRanges);
-        when(cfs.getTracker()).thenReturn(dataTracker);
-        when(cfs.getLiveSSTables()).thenAnswer(request -> dataTracker.getView().select(SSTableSet.LIVE));
-        when(cfs.getSSTables(any())).thenAnswer(request -> dataTracker.getView().select(request.getArgument(0)));
-        when(cfs.getCompactionStrategyManager()).thenReturn(csm);
-
-        DiskBoundaries db = new DiskBoundaries(cfs, new Directories.DataDirectory[0], 0);
-        when(cfs.getDiskBoundaries()).thenReturn(db);
-
-        when(csm.onlyPurgeRepairedTombstones()).thenReturn(false);
+        super.setUp();
     }
 
     @Test
@@ -160,6 +120,7 @@ public void testNoSSTables()
     {
         Controller controller = Mockito.mock(Controller.class);
         long minimalSizeBytes = 2 << 20;
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
         when(controller.getScalingParameter(anyInt())).thenReturn(4);
         when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
         when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
@@ -169,11 +130,12 @@ public void testNoSSTables()
         when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
         when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
         when(controller.maxSSTablesToCompact()).thenReturn(1000);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
         when(controller.random()).thenCallRealMethod();
 
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
 
-        assertNull(strategy.getNextBackgroundTask(FBUtilities.nowInSeconds()));
+        assertTrue(strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).isEmpty());
         assertEquals(0, strategy.getEstimatedRemainingTasks());
     }
 
@@ -240,11 +202,17 @@ private void testGetBucketsOneArena(Map<Integer, Integer> sstableMap, int[] Ws,
         long minimalSizeBytes = m << 20;
 
         Controller controller = Mockito.mock(Controller.class);
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
         when(controller.getNumShards(anyDouble())).thenReturn(1);
         when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
         when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
         when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
         when(controller.maxSSTablesToCompact()).thenReturn(1000);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.getReservedThreads()).thenReturn(Integer.MAX_VALUE);
+        when(controller.getReservationsType()).thenReturn(Reservations.Type.PER_LEVEL);
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.SINGLE);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
 
         when(controller.getScalingParameter(anyInt())).thenAnswer(answer -> {
             int index = answer.getArgument(0);
@@ -257,9 +225,9 @@ private void testGetBucketsOneArena(Map<Integer, Integer> sstableMap, int[] Ws,
         when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
         when(controller.random()).thenCallRealMethod();
 
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
 
-        IPartitioner partitioner = cfs.getPartitioner();
+        IPartitioner partitioner = realm.getPartitioner();
         DecoratedKey first = new BufferDecoratedKey(partitioner.getMinimumToken(), ByteBuffer.allocate(0));
         DecoratedKey last = new BufferDecoratedKey(partitioner.getMaximumToken(), ByteBuffer.allocate(0));
 
@@ -277,21 +245,265 @@ private void testGetBucketsOneArena(Map<Integer, Integer> sstableMap, int[] Ws,
                 sstables.add(mockSSTable(sizeOnDiskBytes, System.currentTimeMillis(), first, last));
             }
         }
-        strategy.addSSTables(sstables);
         dataTracker.addInitialSSTables(sstables);
 
-        List<UnifiedCompactionStrategy.Level> levels = strategy.getLevels();
-        assertEquals(expectedTs.length, levels.size());
+        Map<UnifiedCompactionStrategy.Arena, List<UnifiedCompactionStrategy.Level>> arenas = strategy.getLevels();
+        assertNotNull(arenas);
+        assertEquals(1, arenas.size());
+
+        for (Map.Entry<UnifiedCompactionStrategy.Arena, List<UnifiedCompactionStrategy.Level>> entry : arenas.entrySet())
+        {
+            List<UnifiedCompactionStrategy.Level> levels = entry.getValue();
+            assertEquals(expectedTs.length, levels.size());
+
+            for (int i = 0; i < expectedTs.length; i++)
+            {
+                UnifiedCompactionStrategy.Level level = levels.get(i);
+                assertEquals(i, level.getIndex());
+
+                Collection<CompactionAggregate.UnifiedAggregate> compactionAggregates =
+                level.getCompactionAggregates(entry.getKey(), controller, dataSetSizeBytes);
+
+                long selectedCount = compactionAggregates.stream()
+                                                         .filter(a -> !a.isEmpty())
+                                                         .count();
+                int expectedCount = level.getSSTables().size() >= expectedTs[i] ? 1 : 0;
+                assertEquals(expectedCount, selectedCount);
+            }
+        }
+        // Make sure getMaxOverlapsMap does not fail.
+        System.out.println(strategy.getMaxOverlapsMap());
+    }
+
+    @Test
+    public void testOverlapSetsWithDuplicatedSSTablesProducesNonDuplicatedAggregatesNone()
+    {
+        testOverlapSetsWithDuplicatedSSTablesProducesNonDuplicatedAggregates(Overlaps.InclusionMethod.NONE);
+    }
+
+    @Test
+    public void testOverlapSetsWithDuplicatedSSTablesProducesNonDuplicatedAggregatesSingle()
+    {
+        testOverlapSetsWithDuplicatedSSTablesProducesNonDuplicatedAggregates(Overlaps.InclusionMethod.SINGLE);
+    }
+
+    @Test
+    public void testOverlapSetsWithDuplicatedSSTablesProducesNonDuplicatedAggregatesTransitive()
+    {
+        testOverlapSetsWithDuplicatedSSTablesProducesNonDuplicatedAggregates(Overlaps.InclusionMethod.TRANSITIVE);
+    }
+
+    private void testOverlapSetsWithDuplicatedSSTablesProducesNonDuplicatedAggregates(Overlaps.InclusionMethod inclusionMethod)
+    {
+        final int m = 2; // minimal sorted run size in MB m
+        final Map<Integer, Integer> sstables = new TreeMap<>();
+        // 50MB, 100 sstables
+        sstables.put(50, 100);
+
+        // populate multiple overlapSets including duplicated sstables
+        AtomicLong leftToken = new AtomicLong(0);
+        Supplier<Pair<DecoratedKey, DecoratedKey>> keysSupplier = () -> {
+            // make sure any sstable is overlapping only part of all sstables, thus creating multiple overlapSets that
+            // include duplicated sstable
+            Pair<DecoratedKey, DecoratedKey> p = Pair.create(key(leftToken.get()), key(leftToken.get() + 80));
+            leftToken.incrementAndGet();
+            return p;
+        };
+
+        testGetMultipleBucketsOneArenaNonOverlappingAggregates(sstables, new int[]{ 30, 2, -6 }, m, 1, keysSupplier, inclusionMethod);
+    }
+
+    private void testGetMultipleBucketsOneArenaNonOverlappingAggregates(Map<Integer, Integer> sstableMap, int[] Ws, int m, int expectedLevels,
+                                                                        Supplier<Pair<DecoratedKey, DecoratedKey>> keysSupplier,
+                                                                        Overlaps.InclusionMethod inclusionMethod)
+    {
+        long minimalSizeBytes = m << 20;
+
+        Controller controller = Mockito.mock(Controller.class);
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
+        when(controller.getNumShards(anyDouble())).thenReturn(1);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
+        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.maxSSTablesToCompact()).thenReturn(1000);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.getReservedThreads()).thenReturn(Integer.MAX_VALUE);
+        when(controller.getReservationsType()).thenReturn(Reservations.Type.PER_LEVEL);
+        when(controller.overlapInclusionMethod()).thenReturn(inclusionMethod);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
+
+        when(controller.getScalingParameter(anyInt())).thenAnswer(answer -> {
+            int index = answer.getArgument(0);
+            return Ws[index < Ws.length ? index : Ws.length - 1];
+        });
+        when(controller.getFanout(anyInt())).thenCallRealMethod();
+        when(controller.getThreshold(anyInt())).thenCallRealMethod();
+        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+
+        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
+        when(controller.random()).thenCallRealMethod();
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+        List<SSTableReader> sstables = new ArrayList<>();
+        long dataSetSizeBytes = 0;
+        for (Map.Entry<Integer, Integer> entry : sstableMap.entrySet())
+        {
+            for (int i = 0; i < entry.getValue(); i++)
+            {
+                // we want a number > 0 and < 1 so that the sstable has always some size and never crosses the boundary to the next bucket
+                // so we leave a 1% margin, picking a number from 0.01 to 0.99
+                double rand = 0.01 + 0.98 * random.nextDouble();
+                long sizeOnDiskBytes = (entry.getKey() << 20) + (long) (minimalSizeBytes * rand);
+                dataSetSizeBytes += sizeOnDiskBytes;
+                Pair<DecoratedKey, DecoratedKey> keys = keysSupplier.get();
+                sstables.add(mockSSTable(sizeOnDiskBytes, System.currentTimeMillis(), keys.left, keys.right));
+            }
+        }
+        dataTracker.addInitialSSTables(sstables);
+
+        Map<UnifiedCompactionStrategy.Arena, List<UnifiedCompactionStrategy.Level>> arenas = strategy.getLevels();
+        assertNotNull(arenas);
+        assertEquals(1, arenas.size());
 
-        for (int i = 0; i < expectedTs.length; i++)
+        for (Map.Entry<UnifiedCompactionStrategy.Arena, List<UnifiedCompactionStrategy.Level>> entry : arenas.entrySet())
         {
-            UnifiedCompactionStrategy.Level level = levels.get(i);
-            assertEquals(i, level.getIndex());
-            UnifiedCompactionStrategy.SelectionContext context = new UnifiedCompactionStrategy.SelectionContext(strategy.getController());
-            UnifiedCompactionStrategy.CompactionPick pick = level.getCompactionPick(context);
+            List<UnifiedCompactionStrategy.Level> levels = entry.getValue();
+            assertEquals(expectedLevels, levels.size());
+
+            for (int i = 0; i < expectedLevels; i++)
+            {
+                UnifiedCompactionStrategy.Level level = levels.get(i);
+                assertEquals(i, level.getIndex());
+
+                Collection<CompactionAggregate.UnifiedAggregate> compactionAggregates =
+                    level.getCompactionAggregates(entry.getKey(), controller, dataSetSizeBytes);
+
+                Set<CompactionSSTable> selectedSSTables = new HashSet<>();
+                for (CompactionAggregate.UnifiedAggregate aggregate : compactionAggregates)
+                {
+                    for (CompactionSSTable sstable : aggregate.getSelected().sstables())
+                    {
+                        if (selectedSSTables.contains(sstable))
+                            throw new RuntimeException("Found duplicated sstable " + sstable);
+                        selectedSSTables.add(sstable);
+                    }
+                }
 
-            assertEquals(level.getSSTables().size() >= expectedTs[i], pick != null);
+                // at least one aggregate is selected
+                long selectedCount = compactionAggregates.stream().filter(a -> !a.isEmpty()).count();
+                assertThat(selectedCount).isGreaterThanOrEqualTo(1);
+            }
         }
+        // Make sure getMaxOverlapsMap does not fail.
+        System.out.println(strategy.getMaxOverlapsMap());
+    }
+
+    private BufferDecoratedKey key(long token)
+    {
+        return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(token), ByteBuffer.allocate(0));
+    }
+
+    private BufferDecoratedKey key(Token token)
+    {
+        return new BufferDecoratedKey(token, ByteBuffer.allocate(0));
+    }
+
+    @Test
+    public void testOversizedCompactions_limitingTriggered_maxSpaceOverhead1pct()
+    {
+        testLimitOversizedCompactions(true, 0.01);
+    }
+
+    @Test
+    public void testOversizedCompactions_limitingTriggered_maxSpaceOverhead10pct()
+    {
+        testLimitOversizedCompactions(true, 0.1);
+    }
+
+    @Test
+    public void testOversizedCompactions_limitingTriggered_maxSpaceOverhead20pct()
+    {
+        testLimitOversizedCompactions(true, 0.2);
+    }
+
+    @Test
+    public void testOversizedCompactions_limitingTriggered_maxSpaceOverhead50pct()
+    {
+        testLimitOversizedCompactions(true, 0.5);
+    }
+
+    @Test
+    public void testOversizedCompactions_limitingTriggered_maxSpaceOverhead90pct()
+    {
+        testLimitOversizedCompactions(true, 0.9);
+    }
+
+    void testLimitOversizedCompactions(boolean triggerOversizedLimiting, double maxSpaceOverhead)
+    {
+        testLimitCompactions(1000, true, triggerOversizedLimiting, maxSpaceOverhead);
+    }
+
+    @Test
+    public void testLimitCompactions_noLimiting()
+    {
+        testLimitCompactionsCount(true, 1000);
+    }
+
+    @Test
+    public void testLimitCompactionsCount_1()
+    {
+        testLimitCompactionsCount(false, 1);
+    }
+
+    @Test
+    public void testLimitCompactionsCount_3()
+    {
+        testLimitCompactionsCount(false, 3);
+    }
+
+    @Test
+    public void testLimitCompactionsCount_PerLevel_1()
+    {
+        testLimitCompactionsCount(true, 1);
+    }
+
+    @Test
+    public void testLimitCompactionsCount_PerLevel_5()
+    {
+        testLimitCompactionsCount(true, 5);
+    }
+
+    @Test
+    public void testLimitCompactionsCount_PerLevel_11()
+    {
+        testLimitCompactionsCount(true, 11);
+    }
+
+    void testLimitCompactionsCount(boolean topLevelOnly, int count)
+    {
+        testLimitCompactions(count, topLevelOnly, false, 1.0);
+    }
+
+    public void testLimitCompactions(int maxCount, boolean topLevelOnly, boolean triggerOversizedLimiting, double maxSpaceOverhead)
+    {
+        final int numBuckets = 4;
+        UnifiedCompactionStrategy strategy = prepareStrategyWithLimits(maxCount,
+                                                                       topLevelOnly,
+                                                                       triggerOversizedLimiting,
+                                                                       maxSpaceOverhead,
+                                                                       Double.MAX_VALUE,
+                                                                       numBuckets);
+
+        int numArenas = strategy.getController().getNumShards(1);
+        // Without limiting oversized compactions kicking in, we expect one compaction per shard, otherwise we expect
+        // a fraction of the number of all shards, proportional to the max allowed space amplification fraction.
+        int expectedCompactionTasks = triggerOversizedLimiting
+                                      ? (int) (Math.floor(numArenas * maxSpaceOverhead))
+                                      : topLevelOnly
+                                        ? Math.min((maxCount + numBuckets - 1) / numBuckets, numArenas)
+                                        : Math.min(maxCount, numArenas);
+        // TODO: Check that a warning was issued if space overhead limit was too low.
+        assertEquals(expectedCompactionTasks, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size());
     }
 
     @Test
@@ -365,44 +577,64 @@ public void testLayout(int W, int numSSTables, int maxSSTablesToCompact)
         final int levels = (int) Math.floor(Math.log(numSSTables) / Math.log(F)) + 1;
 
         Controller controller = Mockito.mock(Controller.class);
+        when(controller.getMinSstableSizeBytes()).thenReturn(minSstableSizeBytes);
         when(controller.getScalingParameter(anyInt())).thenReturn(W);
         when(controller.getFanout(anyInt())).thenCallRealMethod();
         when(controller.getThreshold(anyInt())).thenCallRealMethod();
         when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
         when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
         when(controller.getNumShards(anyDouble())).thenReturn(numShards);
+        when(controller.getMaxSpaceOverhead()).thenReturn(1.0);
         when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minSstableSizeBytes);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.getReservedThreads()).thenReturn(Integer.MAX_VALUE);
+        when(controller.getReservationsType()).thenReturn(Reservations.Type.PER_LEVEL);
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.SINGLE);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
 
         if (maxSSTablesToCompact >= numSSTables)
             when(controller.maxConcurrentCompactions()).thenReturn(levels * (W < 0 ? 1 : F)); // make sure the work is assigned to different levels
         else
             when(controller.maxConcurrentCompactions()).thenReturn(1000); // make sure the work is assigned to different levels
 
+        when(controller.maxCompactionSpaceBytes()).thenCallRealMethod();
         when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.getDataSetSizeBytes()).thenReturn(minSstableSizeBytes * numSSTables * numShards);
         when(controller.maxSSTablesToCompact()).thenReturn(maxSSTablesToCompact);
         Random random = Mockito.mock(Random.class);
         when(random.nextInt(anyInt())).thenReturn(0);
         when(controller.random()).thenReturn(random);
 
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
         List<SSTableReader> allSstables = new ArrayList<>();
 
         List<SSTableReader> sstables = mockSSTables(numSSTables,
+                                                    minSstableSizeBytes,
                                                     0,
                                                     System.currentTimeMillis(),
-                                                    0);
+                                                    0,
+                                                    true,
+                                                    null);
         allSstables.addAll(sstables);
-        strategy.addSSTables(allSstables);
         dataTracker.addInitialSSTables(allSstables);
 
         int num = numSSTables;
-        UnifiedCompactionStrategy.CompactionPick task;
+        Collection<AbstractCompactionTask> tasks;
+        boolean headerPrinted = false;
         while (true)
         {
-            task = strategy.getNextCompactionPick(0); // do not check expiration
-            if (task == null)
+            tasks = strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+            if (tasks.isEmpty())
                 break;
 
+            for (CompactionAggregate aggregate : strategy.getAggregates())
+            {
+                if (!headerPrinted)
+                    System.out.println(aggregate.getStatistics().header());
+                headerPrinted = true;
+                System.out.println(aggregate.getStatistics().data());
+            }
+
             boolean layout = Math.min(num, maxSSTablesToCompact) > F * F;
             int limit;
             if (layout)
@@ -414,169 +646,759 @@ public void testLayout(int W, int numSSTables, int maxSSTablesToCompact)
             else
                 limit = maxSSTablesToCompact;
 
-            int expected = num;
-            if (layout)
+            for (AbstractCompactionTask task : tasks)
             {
-                int forTopLevel = (int) (Math.pow(F, Math.floor(Math.log(num) / Math.log(F))));
-                expected = W > 0
-                           ? forTopLevel
-                           : num / forTopLevel * forTopLevel;
+                int expected = num;
+                if (layout)
+                {
+                    int forTopLevel = (int) (Math.pow(F, Math.floor(Math.log(num) / Math.log(F))));
+                    expected = W > 0
+                               ? forTopLevel
+                               : num / forTopLevel * forTopLevel;
 
-            }
-            expected = Math.min(expected, limit);
+                }
+                expected = Math.min(expected, limit);
 
-            int count = task.size();
-            assertEquals(expected, count);
-            for (SSTableReader rdr : task)
-                strategy.removeSSTable(rdr);
-            num -= count;
+                int count = task.transaction.originals().size();
+                assertEquals(expected, count);
+                num -= count;
+            }
         }
         // Check that we issue all the compactions
         assertTrue(num < T);
     }
 
-    private static Map<Long, Integer> mapFromPair(Pair<Long, Integer> ... pairs)
+    @Test
+    public void testLimitCompactionsThroughput_1()
     {
-        Map<Long, Integer> ret = new HashMap<>();
-        for (Pair<Long, Integer> pair : pairs)
-        {
-            ret.put(pair.left, pair.right);
-        }
+        testLimitCompactionsThroughput(1000, 1);
+    }
 
-        return ret;
+    @Test
+    public void testLimitCompactionsThroughput_3()
+    {
+        testLimitCompactionsThroughput(1000, 3);
     }
 
     @Test
-    public void testGetNextBackgroundTasks()
+    public void testOversizedCompactions_limitingNotTriggered()
+    {
+        testLimitOversizedCompactions(false, 1.0);
+    }
+
+    void testLimitCompactionsThroughput(int maxCount, int maxThroughput)
+    {
+        UnifiedCompactionStrategy strategy = prepareStrategyWithLimits(maxCount, false, false, 1.0, maxThroughput, 4);
+
+        // first call should return a pilot task
+        assertEquals(1, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size());
+
+        // if task hasn't progressed, no new tasks should be produced
+        assertEquals(0, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size());
+
+        for (CompactionPick pick : strategy.backgroundCompactions.getCompactionsInProgress())
+            strategy.backgroundCompactions.onInProgress(mockProgress(strategy, pick.id()));
+
+        // now that we have a rate, make sure we produce tasks to fill up the limit
+        assertEquals(Math.min(maxThroughput, maxCount) - 1, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size());
+
+        // and don't create any new ones when the limit is filled, before they make progress
+        assertEquals(0, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size());
+
+        for (CompactionPick pick : strategy.backgroundCompactions.getCompactionsInProgress())
+            if (!pick.inProgress())
+                strategy.backgroundCompactions.onInProgress(mockProgress(strategy, pick.id()));
+
+        // and also when they do
+        assertEquals(0, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size());
+
+        for (int remaining = strategy.getController().getNumShards(1) - Math.min(maxThroughput, maxCount);
+             remaining > 0;
+             --remaining)
+        {
+            // mark a task as completed
+            strategy.backgroundCompactions.onCompleted(strategy, Iterables.get(strategy.backgroundCompactions.getCompactionsInProgress(), 0).id());
+
+            // and check that we get a new one
+            assertEquals(1, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size());
+        }
+    }
+
+    private UnifiedCompactionStrategy prepareStrategyWithLimits(int maxCount,
+                                                                boolean topBucketOnly,
+                                                                boolean triggerOversizedLimiting,
+                                                                double maxSpaceOverhead,
+                                                                double maxThroughput,
+                                                                int numBuckets)
     {
+        int W = 2; // W = 2 => T = F = 4
+        int T = 4;
+        int F = 4;
+        final long minSstableSizeBytes = 2L << 20; // 2 MB
+        final int numShards = 5;
+
         Controller controller = Mockito.mock(Controller.class);
-        long minimalSizeBytes = 2 << 20;
-        when(controller.getScalingParameter(anyInt())).thenReturn(0);
+        when(controller.getMinSstableSizeBytes()).thenReturn(minSstableSizeBytes);
+        when(controller.getScalingParameter(anyInt())).thenReturn(W);
         when(controller.getFanout(anyInt())).thenCallRealMethod();
         when(controller.getThreshold(anyInt())).thenCallRealMethod();
         when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
         when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
-        when(controller.getNumShards(anyDouble())).thenReturn(1);
-        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
-        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
-        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.getNumShards(anyDouble())).thenReturn(numShards);
+        when(controller.getMaxSpaceOverhead()).thenReturn(maxSpaceOverhead);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minSstableSizeBytes);
+        when(controller.maxConcurrentCompactions()).thenReturn(maxCount);
+        when(controller.maxCompactionSpaceBytes()).thenCallRealMethod();
+        when(controller.maxThroughput()).thenReturn(maxThroughput);
         when(controller.maxSSTablesToCompact()).thenReturn(1000);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.getReservedThreads()).thenReturn(Integer.MAX_VALUE);
+        when(controller.getReservationsType()).thenReturn(Reservations.Type.PER_LEVEL);
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.SINGLE);
+        when(controller.parallelizeOutputShards()).thenReturn(false);   // We want to count compactions issued, not individual tasks
+        // Calculate the minimum shard size such that the top bucket compactions won't be considered "oversized" and
+        // all will be allowed to run. The calculation below assumes (1) that compactions are considered "oversized"
+        // if they are more than 1/2 of the max shard size; (2) that mockSSTables uses 15% less than the max SSTable
+        // size for that bucket.
+        long topBucketMaxSstableSize = (long) (minSstableSizeBytes * Math.pow(F, numBuckets));
+        long topBucketMaxCompactionSize = T * topBucketMaxSstableSize;
+        when(controller.getDataSetSizeBytes()).thenReturn(topBucketMaxCompactionSize * numShards);
         when(controller.random()).thenCallRealMethod();
 
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
-
-        IPartitioner partitioner = cfs.getPartitioner();
-
-        List<SSTableReader> sstables = createSStables(partitioner);
+        when(controller.getOverheadSizeInBytes(any(), anyLong())).thenAnswer(inv -> ((Long)inv.getArgument(1)).longValue());
 
-        strategy.addSSTables(sstables);
-        dataTracker.addInitialSSTables(sstables);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+        List<SSTableReader> allSstables = new ArrayList<>();
 
-        AbstractCompactionTask task = strategy.getNextBackgroundTask(0);
-        assertSame(UnifiedCompactionTask.class, task.getClass());
-        task.transaction.abort();
+        for (int i = numBuckets; i > 0; i--)
+        {
+            // Set compactions only in the top bucket of each shard
+            int numSstables = (!topBucketOnly || i == numBuckets) ? T : T - 1;
+            long size = (long) (minSstableSizeBytes * Math.pow(F, i));
+            // Simulate shards by using different disk indexes
+            for (int j = numShards; j > 0; j--)
+            {
+                List<SSTableReader> sstables = mockSSTables(numSstables,
+                                                            size,
+                                                            0,
+                                                            System.currentTimeMillis(),
+                                                            j - 1,
+                                                            true,
+                                                            null);
+                allSstables.addAll(sstables);
+            }
+        }
+        dataTracker.addInitialSSTables(allSstables);
+        return strategy;
     }
 
-    private List<SSTableReader> createSStables(IPartitioner partitioner)
+    private CompactionProgress mockProgress(UnifiedCompactionStrategy strategy, TimeUUID id)
     {
-        return createSStables(partitioner, mapFromPair(Pair.create(4 * ONE_MB, 4)), 10000);
+        CompactionProgress progress = Mockito.mock(CompactionProgress.class);
+        when(progress.durationInMillis()).thenReturn(1000L);
+        when(progress.outputDiskSize()).thenReturn(1L);
+        when(progress.operationId()).thenReturn(id);
+        return progress;
     }
 
-    private List<SSTableReader> createSStables(IPartitioner partitioner, int ttl)
+    private static final class ArenaSpecs
     {
-        return createSStables(partitioner, mapFromPair(Pair.create(4 * ONE_MB, 4)), ttl);
-    }
+        private List<SSTableReader> sstables;
+        private int[] expectedBuckets;
 
-    private List<SSTableReader> createSStables(IPartitioner partitioner, Map<Long, Integer> sstablesMap)
-    {
-        return createSStables(partitioner, sstablesMap, 10000);
+        ArenaSpecs(int[] expectedBuckets)
+        {
+            this.sstables = new ArrayList<>();
+            this.expectedBuckets = expectedBuckets;
+        }
     }
 
-    // Used to make sure timestamps are not exactly the same, which disables expiration
-    int millisAdjustment = 0;
-
-    private List<SSTableReader> createSStables(IPartitioner partitioner,
-                                               Map<Long, Integer> sstablesMap,
-                                               int ttl)
+    private ArenaSpecs mockArena(Token min,
+                                 Token max,
+                                 Map<Long, Integer> sstables,
+                                 boolean repaired,
+                                 TimeUUID pendingRepair,
+                                 int diskIndex,
+                                 int[] expectedBuckets)
     {
-        List<SSTableReader> mockSSTables = new ArrayList<>();
-        Token min = partitioner.getMinimumToken();
-        Token max = partitioner.getMaximumToken();
+        ArenaSpecs arena = new ArenaSpecs(expectedBuckets);
         ByteBuffer bb = ByteBuffer.allocate(0);
-        sstablesMap.forEach((size, num) -> {
+
+        sstables.forEach((size, num) -> {
             Token first = min.getPartitioner().split(min, max, 0.01);
+            Token last = min.getPartitioner().split(min, max, 0.99);
+            double tokenSpan = first.size(last);
 
             for (int i = 0; i < num; i++)
             {
-                // pending repair
-                mockSSTables.add(mockSSTable(0,
-                                             size,
-                                             System.currentTimeMillis() + millisAdjustment++,
-                                             0.0,
-                                             new BufferDecoratedKey(first, bb),
-                                             new BufferDecoratedKey(max, bb),
-                                             ttl));
+                arena.sstables.add(mockSSTable(0,
+                                               (long) (size * tokenSpan * 1.01),    // adjust slightly bigger to avoid rounding issues
+                                               System.currentTimeMillis(),
+                                               0.0,
+                                               new BufferDecoratedKey(first, bb),
+                                               new BufferDecoratedKey(last, bb),
+                                               diskIndex,
+                                               repaired,
+                                               pendingRepair,
+                                               0));
                 first = first.nextValidToken();
             }
         });
-        return mockSSTables;
+
+        return arena;
     }
 
-    @Test
-    public void testDropExpiredSSTables()
+    private List<Token> makeBoundaries(int numShards, int numDisks)
     {
-        testDropExpiredFromBucket(1);
-        testDropExpiredAndCompactNonExpired();
+        IPartitioner partitioner = realm.getPartitioner();
+        assert numShards >= 1;
+        assert numDisks >= 1;
+
+        if (numShards * numDisks == 1)
+            return ImmutableList.of(partitioner.getMaximumToken());
+
+        Splitter splitter = partitioner.splitter().orElse(null);
+        assertNotNull("The partitioner must support a splitter", splitter);
+
+        int numBoundaries = useDiskBoundaries ? numDisks * numShards : numShards;
+        Splitter.WeightedRange range = new Splitter.WeightedRange(1.0, new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken()));
+        final List<Token> shards = splitter.splitOwnedRanges(numBoundaries, ImmutableList.of(range), Splitter.SplitType.ALWAYS_SPLIT)
+                                   .boundaries
+                                   .stream()
+                                   .collect(Collectors.toList());
+        if (useDiskBoundaries)
+        {
+            diskBoundaryPositions = new ArrayList<>(numDisks);
+            for (int i = 0; i < numDisks; ++i)
+                diskBoundaryPositions.add(shards.get((i + 1) * numShards - 1));
+        }
+        return shards;
     }
 
-    private void testDropExpiredFromBucket(int numShards)
+    private List<ArenaSpecs> mockArenas(int diskIndex,
+                                        int diskCount,
+                                        boolean repaired,
+                                        TimeUUID pendingRepair,
+                                        List<Token> boundaries,
+                                        Map<Long, Integer> sstables,
+                                        int[] buckets)
     {
-        Controller controller = Mockito.mock(Controller.class);
-        long minimalSizeBytes = 2 << 20;
-        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
-        when(controller.getScalingParameter(anyInt())).thenReturn(3); // T=5
-        when(controller.getFanout(anyInt())).thenCallRealMethod();
-        when(controller.getThreshold(anyInt())).thenCallRealMethod();
-        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
-        when(controller.getNumShards(anyDouble())).thenReturn(numShards);
-        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
-        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
-        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
-        when(controller.maxSSTablesToCompact()).thenReturn(1000);
-        when(controller.getIgnoreOverlapsInExpirationCheck()).thenReturn(false);
-        when(controller.random()).thenCallRealMethod();
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
-        strategy.startup();
+        List<ArenaSpecs> arenasList = new ArrayList<>();
 
-        List<SSTableReader> sstables = createSStables(cfs.getPartitioner());
-        // Tracker#addSSTables also tries to backup SSTables, so we use addInitialSSTables and notify explicitly
-        strategy.addSSTables(sstables);
-        dataTracker.addInitialSSTables(sstables);
+        int numShards = boundaries.size() / diskCount;
+        List<Token> shardPositions = useDiskBoundaries
+                                                 ? boundaries.subList(diskIndex * numShards, (diskIndex + 1) * numShards)
+                                                 : boundaries;
+        Token min = useDiskBoundaries && diskIndex > 0
+                    ? boundaries.get(diskIndex * numShards - 1).getToken()
+                    : partitioner.getMinimumToken();
+        Token max = shardPositions.get(shardPositions.size() - 1).getToken();
 
-        try
-        {
-            // nothing to compact yet
-            assertNull(strategy.getNextCompactionPick(0));
+        arenasList.add(mockArena(min, max, sstables, repaired, pendingRepair, diskIndex, buckets));
 
-            long timestamp = sstables.get(sstables.size() - 1).getMaxLocalDeletionTime();
-            long expirationPoint = timestamp + 1;
+        return arenasList;
+    }
 
-            UnifiedCompactionStrategy.CompactionPick pick = strategy.getNextCompactionPick(expirationPoint);
-            assertNotNull(pick);
-            assertEquals(sstables.size(), pick.size());
-            assertEquals(-1, pick.level);
+    private static Map<Long, Integer> mapFromPair(Pair<Long, Integer> ... pairs)
+    {
+        Map<Long, Integer> ret = new HashMap<>();
+        for (Pair<Long, Integer> pair : pairs)
+        {
+            ret.put(pair.left, pair.right);
+        }
+
+        return ret;
+    }
+
+    @Test
+    public void testAllArenasOneBucket_NoShards()
+    {
+        testAllArenasOneBucket(1);
+    }
+
+    @Test
+    public void testAllArenasOneBucket_MultipleShards()
+    {
+        testAllArenasOneBucket(5);
+    }
+
+    private void testAllArenasOneBucket(int numShards)
+    {
+        final int m = 2; // minimal sorted run size in MB
+        final int W = 2; // => o = 1 => F = 4, T = 4: 0-8m, 8-32m, 32-128m
+
+        List<Token> boundaries = makeBoundaries(numShards, 2);
+        List<ArenaSpecs> arenasList = new ArrayList<>();
+
+        Map<Long, Integer> sstables = mapFromPair(Pair.create(4 * ONE_MB, 4));
+        int[] buckets = new int[]{4};
+
+        TimeUUID pendingRepair = TimeUUID.Generator.nextTimeUUID();
+        arenasList.addAll(mockArenas(0, 2, false, pendingRepair, boundaries, sstables, buckets)); // pending repair
+
+        arenasList.addAll(mockArenas(0, 2, false, null, boundaries, sstables, buckets)); // unrepaired
+        arenasList.addAll(mockArenas(1, 2, false, null, boundaries, sstables, buckets)); // unrepaired, next disk
+
+        arenasList.addAll(mockArenas(0, 2, true, null, boundaries, sstables, buckets)); // repaired
+        arenasList.addAll(mockArenas(1, 2, true, null, boundaries, sstables, buckets)); // repaired, next disk
+
+        testGetBucketsMultipleArenas(arenasList, W, m, boundaries);
+    }
+
+    @Test
+    public void testRepairedOneDiskOneBucket_NoShards()
+    {
+        testRepairedOneDiskOneBucket(1);
+    }
+
+    @Test
+    public void testRepairedOneDiskOneBucket_MultipleShards()
+    {
+        testRepairedOneDiskOneBucket(5);
+    }
+
+    private void testRepairedOneDiskOneBucket(int numShards)
+    {
+        final int m = 2; // minimal sorted run size in MB
+        final int W = 2; // => o = 1 => F = 4, T = 4: 0-8m, 8-32m, 32-128m
+
+        Map<Long, Integer> sstables = mapFromPair(Pair.create(4 * ONE_MB, 4));
+        int[] buckets = new int[]{4};
+
+        List<Token> boundaries = makeBoundaries(numShards, 1);
+        List<ArenaSpecs> arenas = mockArenas(0, 1, true, null, boundaries, sstables, buckets);
+        testGetBucketsMultipleArenas(arenas, W, m, boundaries);
+    }
+
+    @Test
+    public void testRepairedTwoDisksOneBucket_NoShards()
+    {
+        testRepairedTwoDisksOneBucket(1);
+    }
+
+    @Test
+    public void testRepairedTwoDisksOneBucket_MultipleShards()
+    {
+        testRepairedTwoDisksOneBucket(5);
+    }
+
+    private void testRepairedTwoDisksOneBucket(int numShards)
+    {
+        final int m = 2; // minimal sorted run size in MB
+        final int W = 2; // => o = 1 => F = 4, T = 4: 0-8m, 8-32m, 32-128m
+
+        Map<Long, Integer> sstables = mapFromPair(Pair.create(4 * ONE_MB, 4));
+        int[] buckets = new int[]{4};
+
+        List<Token> boundaries = makeBoundaries(numShards, 2);
+        List<ArenaSpecs> arenas = new ArrayList<>();
+
+        arenas.addAll(mockArenas(0, 2, true, null, boundaries, sstables, buckets));
+        arenas.addAll(mockArenas(1, 2, true, null, boundaries, sstables, buckets));
+
+        testGetBucketsMultipleArenas(arenas, W, m, boundaries);
+    }
+
+    @Test
+    public void testRepairedMultipleDisksMultipleBuckets_NoShards()
+    {
+        testRepairedMultipleDisksMultipleBuckets(1);
+    }
+
+    @Test
+    public void testRepairedMultipleDisksMultipleBuckets_MultipleShards()
+    {
+        testRepairedMultipleDisksMultipleBuckets(15);
+    }
+
+    private void testRepairedMultipleDisksMultipleBuckets(int numShards)
+    {
+        final int m = 2; // minimal sorted run size in MB
+        final int W = 2; // => o = 1 => F = 4, T = 4: 0-8m, 8-32m, 32-128m
+
+        List<Token> boundaries = makeBoundaries(numShards, 6);
+        List<ArenaSpecs> arenasList = new ArrayList<>();
+
+        Map<Long, Integer> sstables1 = mapFromPair(Pair.create(2 * ONE_MB, 4), Pair.create(8 * ONE_MB, 4), Pair.create(32 * ONE_MB, 4));
+        int[] buckets1 = new int[]{4,4,4};
+
+        Map<Long, Integer> sstables2 = mapFromPair(Pair.create(8 * ONE_MB, 4), Pair.create(32 * ONE_MB, 8));
+        int[] buckets2 = new int[]{0,4,8};
+
+        for (int i = 0; i < 6; i++)
+        {
+            if (i % 2 == 0)
+                arenasList.addAll(mockArenas(i, 6, true, null, boundaries, sstables1, buckets1));
+            else
+                arenasList.addAll(mockArenas(i, 6, true, null, boundaries, sstables2, buckets2));
+
+        }
+
+        testGetBucketsMultipleArenas(arenasList, W, m, boundaries);
+    }
+
+    @Test
+    public void testRepairedUnrepairedOneDiskMultipleBuckets_NoShards()
+    {
+        testRepairedUnrepairedOneDiskMultipleBuckets(1);
+    }
+
+    @Test
+    public void testRepairedUnrepairedOneDiskMultipleBuckets_MultipleShards()
+    {
+        testRepairedUnrepairedOneDiskMultipleBuckets(10);
+    }
+
+    private void testRepairedUnrepairedOneDiskMultipleBuckets(int numShards)
+    {
+        final int m = 2; // minimal sorted run size in MB
+        final int W = 2; // => o = 1 => F = 4, T = 4: 0-8m, 8-32m, 32-128m
+
+        List<Token> boundaries = makeBoundaries(numShards, 1);
+        List<ArenaSpecs> arenasList = new ArrayList<>();
+
+        Map<Long, Integer> sstables1 = mapFromPair(Pair.create(2 * ONE_MB, 4), Pair.create(8 * ONE_MB, 4), Pair.create(32 * ONE_MB, 4));
+        int[] buckets1 = new int[]{4,4,4};
+
+        Map<Long, Integer> sstables2 = mapFromPair(Pair.create(8 * ONE_MB, 4), Pair.create(32 * ONE_MB, 8));
+        int[] buckets2 = new int[]{0,4,8};
+
+        arenasList.addAll(mockArenas(0, 1, true, null, boundaries, sstables2, buckets2)); // repaired
+        arenasList.addAll(mockArenas(0, 1, false, null, boundaries, sstables1, buckets1)); // unrepaired
+
+        testGetBucketsMultipleArenas(arenasList, W, m, boundaries);
+    }
+
+    @Test
+    public void testRepairedUnrepairedTwoDisksMultipleBuckets_NoShards()
+    {
+        testRepairedUnrepairedTwoDisksMultipleBuckets(1);
+    }
+
+    @Test
+    public void testRepairedUnrepairedTwoDisksMultipleBuckets_MultipleShards()
+    {
+        testRepairedUnrepairedTwoDisksMultipleBuckets(5);
+    }
+
+    private void testRepairedUnrepairedTwoDisksMultipleBuckets(int numShards)
+    {
+        final int m = 2; // minimal sorted run size in MB
+        final int W = 2; // => o = 1 => F = 4, T = 4: 0-8m, 8-32m, 32-128m
+
+        List<Token> boundaries = makeBoundaries(numShards, 2);
+        List<ArenaSpecs> arenasList = new ArrayList<>();
+
+        Map<Long, Integer> sstables1 = mapFromPair(Pair.create(2 * ONE_MB, 4), Pair.create(8 * ONE_MB, 4), Pair.create(32 * ONE_MB, 4));
+        int[] buckets1 = new int[]{4,4,4};
+
+        Map<Long, Integer> sstables2 = mapFromPair(Pair.create(8 * ONE_MB, 4), Pair.create(32 * ONE_MB, 8));
+        int[] buckets2 = new int[]{0,4,8};
+
+        arenasList.addAll(mockArenas(0, 2, true, null, boundaries, sstables2, buckets2));  // repaired, first disk
+        arenasList.addAll(mockArenas(1, 2, true, null, boundaries, sstables1, buckets1));  // repaired, second disk
+
+        arenasList.addAll(mockArenas(0, 2, false, null, boundaries, sstables1, buckets1));  // unrepaired, first disk
+        arenasList.addAll(mockArenas(1, 2, false, null, boundaries, sstables2, buckets2));  // unrepaired, second disk
+
+        testGetBucketsMultipleArenas(arenasList, W, m, boundaries);
+    }
+
+    private void testGetBucketsMultipleArenas(List<ArenaSpecs> arenaSpecs, int W, int m, List<Token> shards)
+    {
+        long minimalSizeBytes = m << 20;
+
+        Controller controller = Mockito.mock(Controller.class);
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
+        when(controller.getScalingParameter(anyInt())).thenReturn(W);
+        when(controller.getFanout(anyInt())).thenCallRealMethod();
+        when(controller.getThreshold(anyInt())).thenCallRealMethod();
+        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
+        when(controller.getNumShards(anyDouble())).thenReturn(shards.size());
+        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.maxSSTablesToCompact()).thenReturn(1000);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.random()).thenCallRealMethod();
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+
+        List<SSTableReader> sstables = arenaSpecs.stream().flatMap(a -> a.sstables.stream()).collect(Collectors.toList());
+        dataTracker.addInitialSSTables(sstables);
+
+        Map<UnifiedCompactionStrategy.Arena, List<UnifiedCompactionStrategy.Level>> arenas = strategy.getLevels();
+        assertNotNull(arenas);
+        assertEquals(arenaSpecs.size(), arenas.size());
+
+        int idx = 0;
+        for (Map.Entry<UnifiedCompactionStrategy.Arena, List<UnifiedCompactionStrategy.Level>> entry : arenas.entrySet())
+        {
+            List<UnifiedCompactionStrategy.Level> levels = entry.getValue();
+            ArenaSpecs currentArenaSpecs = arenaSpecs.get(idx++);
+
+            assertEquals(currentArenaSpecs.expectedBuckets.length, levels.size());
+            for (int i = 0; i < currentArenaSpecs.expectedBuckets.length; i++)
+                assertEquals(currentArenaSpecs.expectedBuckets[i], levels.get(i).sstables.size());
+        }
+        // Make sure getMaxOverlapsMap does not fail.
+        System.out.println(strategy.getMaxOverlapsMap());
+    }
+
+    @Test
+    public void testGetNextBackgroundTasksParallelizeOutputShards() throws Exception
+    {
+        assertCompactionTask(1, 3, true, UnifiedCompactionTask.class);
+        assertCompactionTask(3, 9, true, UnifiedCompactionTask.class);
+    }
+
+    @Test
+    public void testGetNextBackgroundTasksNoParallelization() throws Exception
+    {
+        assertCompactionTask(1, 3, false, UnifiedCompactionTask.class);
+        assertCompactionTask(3, 3, false, UnifiedCompactionTask.class);
+    }
+
+    private void assertCompactionTask(final int numShards, final int expectedNumOfTasks, boolean parallelizeOutputShards, Class expectedClass)
+    {
+        Controller controller = Mockito.mock(Controller.class);
+        long minimalSizeBytes = 2 << 20;
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
+        when(controller.getScalingParameter(anyInt())).thenReturn(0);
+        when(controller.getFanout(anyInt())).thenCallRealMethod();
+        when(controller.getThreshold(anyInt())).thenCallRealMethod();
+        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
+        when(controller.getNumShards(anyDouble())).thenReturn(numShards);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
+        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE);
+        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.maxSSTablesToCompact()).thenReturn(1000);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.getReservedThreads()).thenReturn(Integer.MAX_VALUE);
+        when(controller.getReservationsType()).thenReturn(Reservations.Type.PER_LEVEL);
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.SINGLE);
+        when(controller.parallelizeOutputShards()).thenReturn(parallelizeOutputShards);
+        when(controller.random()).thenCallRealMethod();
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+
+        IPartitioner partitioner = realm.getPartitioner();
+
+        List<SSTableReader> sstables = createSStables(partitioner);
+
+        dataTracker.addInitialSSTables(sstables);
+
+        Collection<AbstractCompactionTask> tasks = strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+        assertEquals("Expecting number of next background tasks:", expectedNumOfTasks, tasks.size());
+        for (AbstractCompactionTask task : tasks)
+        {
+            assertSame(expectedClass, task.getClass());
+        }
+    }
+
+    @Test
+    public void testGetNextCompactionAggregates()
+    {
+        Controller controller = Mockito.mock(Controller.class);
+        long minimalSizeBytes = 2 << 20;
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
+        when(controller.getScalingParameter(anyInt())).thenReturn(0);
+        when(controller.getFanout(anyInt())).thenCallRealMethod();
+        when(controller.getThreshold(anyInt())).thenCallRealMethod();
+        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
+        when(controller.getNumShards(anyDouble())).thenReturn(1);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
+        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE);
+        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.maxSSTablesToCompact()).thenReturn(1000);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.random()).thenCallRealMethod();
+        when(controller.getMaxRecentAdaptiveCompactions()).thenReturn(-1);
+        when(controller.isRecentAdaptive(any())).thenReturn(true);
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+
+        CompactionPick compaction = Mockito.mock(CompactionPick.class);
+        when(compaction.isEmpty()).thenReturn(false);
+        when(compaction.hasExpiredOnly()).thenReturn(false);
+        List<SSTableReader> nonExpiredSSTables = createSStables(realm.getPartitioner());
+        when(compaction.sstables()).thenReturn(ImmutableSet.copyOf(nonExpiredSSTables));
+        when(compaction.totalOverheadInBytes()).thenReturn(minimalSizeBytes);   // doesn't really matter
+
+        CompactionAggregate.UnifiedAggregate aggregate = Mockito.mock(CompactionAggregate.UnifiedAggregate.class);
+        when(aggregate.getSelected()).thenReturn(compaction);
+
+        Collection<CompactionAggregate> compactionAggregates = strategy.getNextCompactionAggregates(ImmutableList.of(aggregate), 1000);
+        assertNotNull(compactionAggregates);
+        assertEquals(1, compactionAggregates.size());
+    }
+
+    private List<SSTableReader> createSStables(IPartitioner partitioner)
+    {
+        return createSStables(partitioner, mapFromPair(Pair.create(4 * ONE_MB, 4)), 10000, TimeUUID.Generator.nextTimeUUID());
+    }
+
+    private List<SSTableReader> createSStables(IPartitioner partitioner, int ttl, TimeUUID pendingRepair)
+    {
+        return createSStables(partitioner, mapFromPair(Pair.create(4 * ONE_MB, 4)), ttl, pendingRepair);
+    }
+
+    private List<SSTableReader> createSStables(IPartitioner partitioner, Map<Long, Integer> sstablesMap)
+    {
+        return createSStables(partitioner, sstablesMap, 10000, TimeUUID.Generator.nextTimeUUID());
+    }
+
+    private List<SSTableReader> createSStables(IPartitioner partitioner,
+                                               Map<Long, Integer> sstablesMap,
+                                               int ttl,
+                                               TimeUUID pendingRepair)
+    {
+        List<SSTableReader> mockSSTables = new ArrayList<>();
+        Token min = partitioner.getMinimumToken();
+        Token max = partitioner.getMaximumToken();
+        ByteBuffer bb = ByteBuffer.allocate(0);
+        sstablesMap.forEach((size, num) -> {
+            Token first = min.getPartitioner().split(min, max, 0.01);
+
+            for (int i = 0; i < num; i++)
+            {
+                // pending repair
+                mockSSTables.add(mockSSTable(0,
+                                             size,
+                                             System.currentTimeMillis(),
+                                             0.0,
+                                             new BufferDecoratedKey(first, bb),
+                                             new BufferDecoratedKey(max, bb),
+                                             0,
+                                             false,
+                                             pendingRepair,
+                                             ttl));
+                first = first.nextValidToken();
+            }
+
+            for (int i = 0; i < num; i++)
+            {
+                // unrepaired
+                mockSSTables.add(mockSSTable(0,
+                                             size,
+                                             System.currentTimeMillis(),
+                                             0.0,
+                                             new BufferDecoratedKey(first, bb),
+                                             new BufferDecoratedKey(max, bb),
+                                             0,
+                                             false,
+                                             null,
+                                             ttl));
+                first = first.nextValidToken();
+            }
+
+            for (int i = 0; i < num; i++)
+            {
+                // repaired
+                mockSSTables.add(mockSSTable(0,
+                                             size,
+                                             System.currentTimeMillis(),
+                                             0.0,
+                                             new BufferDecoratedKey(first, bb),
+                                             new BufferDecoratedKey(max, bb),
+                                             0,
+                                             true,
+                                             null,
+                                             ttl));
+                first = first.nextValidToken();
+            }
+        });
+        return mockSSTables;
+    }
+
+    @Test
+    public void testDropExpiredSSTables1Shard() throws Exception
+    {
+        testDropExpiredFromBucket(1, true);
+        testDropExpiredAndCompactNonExpired(true);
+    }
+
+    @Test
+    public void testDropExpiredSSTables3Shards() throws Exception
+    {
+        // We don't want separate tasks for each output shard here
+        testDropExpiredFromBucket(3, false);
+    }
+
+    private void testDropExpiredFromBucket(int numShards, boolean parallelizeOutputShards) throws Exception
+    {
+        Controller controller = Mockito.mock(Controller.class);
+        long minimalSizeBytes = 2 << 20;
+        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
+        when(controller.getScalingParameter(anyInt())).thenReturn(3); // T=5
+        when(controller.getFanout(anyInt())).thenCallRealMethod();
+        when(controller.getThreshold(anyInt())).thenCallRealMethod();
+        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
+        when(controller.getNumShards(anyDouble())).thenReturn(numShards);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
+        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE);
+        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.maxSSTablesToCompact()).thenReturn(1000);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.getReservedThreads()).thenReturn(Integer.MAX_VALUE);
+        when(controller.getReservationsType()).thenReturn(Reservations.Type.PER_LEVEL);
+        when(controller.getIgnoreOverlapsInExpirationCheck()).thenReturn(false);
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.SINGLE);
+        when(controller.parallelizeOutputShards()).thenReturn(parallelizeOutputShards);
+        when(controller.random()).thenCallRealMethod();
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+        strategy.startup();
+
+        List<SSTableReader> sstables = createSStables(realm.getPartitioner());
+        // Tracker#addSSTables also tries to backup SSTables, so we use addInitialSSTables and notify explicitly
+        dataTracker.addInitialSSTables(sstables);
+
+        try
+        {
+            // nothing to compact yet
+            assertEquals(0, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size());
+
+            long timestamp = sstables.get(sstables.size() - 1).getMaxLocalDeletionTime();
+            long expirationPoint = timestamp + 1;
+
+            var tasks = strategy.getNextBackgroundTasks(expirationPoint);
+            assertEquals(3, tasks.size()); // repaired, unrepaired, pending
+
+            Collection<CompactionPick> picks = strategy.backgroundCompactions.getCompactionsInProgress();
+            assertEquals(0, picks.size()); // Expiration tasks are quick and not tracked
+
+            assertEquals(sstables.size(), dataTracker.getLiveSSTables().size());
+            assertEquals(sstables.size(), dataTracker.getCompacting().size());
+
+            var tableset = new HashSet<>(sstables);
+            for (var t : tasks)
+            {
+                assertTrue(tableset.containsAll(t.transaction.originals()));
+                tableset.removeAll(t.transaction.originals());
+            }
+            assertTrue(tableset.isEmpty());
         }
         finally
         {
             strategy.shutdown();
+            dataTracker.removeCompactingUnsafe(dataTracker.getCompacting());
+            dataTracker.removeUnsafe(dataTracker.getLiveSSTables());
         }
     }
 
-    private void testDropExpiredAndCompactNonExpired()
+    private void testDropExpiredAndCompactNonExpired(boolean parallelizeOutputShards)
     {
         Controller controller = Mockito.mock(Controller.class);
         long minimalSizeBytes = 2 << 20;
         when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
         when(controller.getScalingParameter(anyInt())).thenReturn(2);
         when(controller.getFanout(anyInt())).thenCallRealMethod();
         when(controller.getThreshold(anyInt())).thenCallRealMethod();
@@ -584,106 +1406,441 @@ private void testDropExpiredAndCompactNonExpired()
         when(controller.getNumShards(anyDouble())).thenReturn(1);
         when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
         when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE);
         when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
         when(controller.getIgnoreOverlapsInExpirationCheck()).thenReturn(false);
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.SINGLE);
+        when(controller.parallelizeOutputShards()).thenReturn(parallelizeOutputShards);
         when(controller.maxSSTablesToCompact()).thenReturn(1000);
 
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
         when(controller.random()).thenCallRealMethod();
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
         strategy.startup();
 
-        List<SSTableReader> expiredSSTables = createSStables(cfs.getPartitioner(), 1000);
-        List<SSTableReader> nonExpiredSSTables = createSStables(cfs.getPartitioner(), 0);
-        strategy.addSSTables(expiredSSTables);
-        strategy.addSSTables(nonExpiredSSTables.subList(0, 3));
-        dataTracker.addInitialSSTables(Iterables.concat(expiredSSTables, nonExpiredSSTables));
+        TimeUUID pendingRepair = TimeUUID.Generator.nextTimeUUID();
+        List<SSTableReader> expiredSSTables = createSStables(realm.getPartitioner(), 1000, pendingRepair);
+        long millis = System.currentTimeMillis();
+        while (millis == System.currentTimeMillis()) // make sure we have different timestamps
+        {
+            Thread.yield();
+        }
+
+        List<SSTableReader> nonExpiredSSTables = createSStables(realm.getPartitioner(), 0, pendingRepair);
+        List<SSTableReader> allSSTables = Stream.concat(expiredSSTables.stream(), nonExpiredSSTables.stream())
+                                                .collect(Collectors.toList());
+        dataTracker.addInitialSSTables(allSSTables);
 
         long timestamp = expiredSSTables.get(expiredSSTables.size() - 1).getMaxLocalDeletionTime();
         long expirationPoint = timestamp + 1;
 
         try
         {
-            UnifiedCompactionStrategy.CompactionPick pick = strategy.getNextCompactionPick(expirationPoint);
-
-            assertEquals(expiredSSTables.size(), pick.size());
-            assertEquals(-1, pick.level);
+            for (var task : strategy.getNextBackgroundTasks(expirationPoint))
+            {
+                assertTrue(task instanceof ExpirationTask);
+                assertEquals(4, task.transaction.originals().size());
+                assertEquals(0L, task.getSpaceOverhead());
+            }
+            Collection<CompactionPick> picks = strategy.backgroundCompactions.getCompactionsInProgress();
+            assertEquals(0, picks.size()); // expiration tasks are not tracked
 
-            strategy.addSSTables(nonExpiredSSTables);   // duplicates should be skipped
-            pick = strategy.getNextCompactionPick(expirationPoint);
+            // Try again, expired SSTables are in compacting state and should not be picked
+            strategy.getNextBackgroundTasks(expirationPoint);
+            picks = strategy.backgroundCompactions.getCompactionsInProgress();
+            assertNotEquals(0, picks.size());
 
-            assertEquals(expiredSSTables.size() + nonExpiredSSTables.size(), pick.size());
-            assertEquals(0, pick.level);
+            for (CompactionPick pick : picks)
+            {
+                assertEquals(4, pick.sstables().size());
+                assertEquals(0, pick.expired().size());
+                Set<CompactionSSTable> nonExpired = pick.sstables();
+                long expectedTotSize = nonExpired.stream()
+                                                 .mapToLong(CompactionSSTable::onDiskLength)
+                                                 .sum();
+                assertEquals(expectedTotSize, pick.totSizeInBytes());
+                assertEquals(expectedTotSize / nonExpired.size(), pick.avgSizeInBytes());
+                assertEquals(0, pick.parent());
+            }
         }
         finally
         {
             strategy.shutdown();
+            dataTracker.removeCompactingUnsafe(dataTracker.getCompacting());
+            dataTracker.removeUnsafe(dataTracker.getLiveSSTables());
         }
     }
 
+    @Test
+    public void testPrioritizeLocallyAvailableSSTables()
+    {
+        Set<SSTableReader> sstables0 = new HashSet<>(createSSTalesWithDiskIndex(realm.getPartitioner(), 0));
+        Set<SSTableReader> sstables1 = new HashSet<>(createSSTalesWithDiskIndex(realm.getPartitioner(), 1));
+        Set<SSTableReader> sstables = Sets.union(sstables0, sstables1);
+        dataTracker.addInitialSSTables(sstables);
+
+        for (SSTableReader sstable : sstables)
+        {
+            long onDiskLength;
+            if (sstables1.contains(sstable))
+                onDiskLength = sstable.onDiskLength();
+            else
+                onDiskLength = 0L;
+            when(sstable.onDiskLength()).thenReturn(onDiskLength);
+        }
+
+        Controller controller = Mockito.mock(Controller.class);
+        long minimalSizeBytes = 2 << 20;
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
+        when(controller.getScalingParameter(anyInt())).thenReturn(0);
+        when(controller.getFanout(anyInt())).thenCallRealMethod();
+        when(controller.getThreshold(anyInt())).thenCallRealMethod();
+        when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
+        when(controller.getNumShards(anyDouble())).thenReturn(1);
+        when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
+        when(controller.maxConcurrentCompactions()).thenReturn(1);
+        when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE);
+        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
+        when(controller.maxSSTablesToCompact()).thenReturn(2);
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.SINGLE);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
+        when(controller.prioritize(anyList())).thenAnswer(answ -> {
+            List<CompactionAggregate.UnifiedAggregate> pending = answ.getArgument(0);
+            pending.sort(Comparator.comparingLong(a -> ((CompactionAggregate.UnifiedAggregate) a).sstables.stream().mapToLong(CompactionSSTable::onDiskLength).sum()).reversed());
+            return pending;
+        });
+        when(controller.random()).thenCallRealMethod();
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+        Collection<AbstractCompactionTask> tasks = strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds());
+
+        assertEquals(1, tasks.size());
+        var compacting = realm.getCompactingSSTables();
+        assertEquals(2, compacting.size());
+        assertEquals(new HashSet<>(sstables1), compacting);
+    }
+
+    private Set<SSTableReader> createSSTalesWithDiskIndex(IPartitioner partitioner, int diskIndex)
+    {
+        Set<SSTableReader> mockSSTables = new HashSet<>();
+        Map<Long, Integer> sstablesMap = mapFromPair(Pair.create(4 * ONE_MB, 2));
+        Token min = partitioner.getMinimumToken();
+        Token max = partitioner.getMaximumToken();
+        ByteBuffer bb = ByteBuffer.allocate(0);
+        int ttl = 0;
+        TimeUUID pendingRepair = null;
+        sstablesMap.forEach((size, num) -> {
+            Token first = min.getPartitioner().split(min, max, 0.01);
+
+            for (int i = 0; i < num; i++)
+            {
+                mockSSTables.add(mockSSTable(0,
+                                             size,
+                                             FBUtilities.nowInSeconds(),
+                                             0.0,
+                                             new BufferDecoratedKey(first, bb),
+                                             new BufferDecoratedKey(max, bb),
+                                             diskIndex,
+                                             false,
+                                             pendingRepair,
+                                             ttl
+                ));
+                first = first.nextValidToken();
+            }
+        });
+        return mockSSTables;
+    }
+
     @Test
     public void testPending()
     {
         Controller controller = Mockito.mock(Controller.class);
-        when(controller.getScalingParameter(anyInt())).thenReturn(8); // F=10, T=10
+        when(controller.getScalingParameter(anyInt())).thenReturn(-8); // F=10, T=2
         when(controller.getFanout(anyInt())).thenCallRealMethod();
         when(controller.getThreshold(anyInt())).thenCallRealMethod();
         when(controller.maxSSTablesToCompact()).thenReturn(10); // same as fanout
 
         long minimalSizeBytes = 2 << 20;
         when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
+        when(controller.getMinSstableSizeBytes()).thenReturn(minimalSizeBytes);
         when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
         when(controller.getNumShards(anyDouble())).thenReturn(1);
         when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes);
         when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
+        when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE);
         when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
         when(controller.getIgnoreOverlapsInExpirationCheck()).thenReturn(false);
         when(controller.random()).thenCallRealMethod();
+        when(controller.prioritize(anyList())).thenAnswer(answ -> answ.getArgument(0));
+        when(controller.getReservedThreads()).thenReturn(Integer.MAX_VALUE);
+        when(controller.getReservationsType()).thenReturn(Reservations.Type.PER_LEVEL);
+        when(controller.overlapInclusionMethod()).thenReturn(Overlaps.InclusionMethod.SINGLE);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
 
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
         strategy.startup();
 
-        int count = 91;
-        List<SSTableReader> sstables = createSStables(cfs.getPartitioner(),
-                                                      mapFromPair(Pair.create(4 * ONE_MB, count)));
-        strategy.addSSTables(sstables);
+        List<SSTableReader> sstables = createSStables(realm.getPartitioner(),
+                                                      mapFromPair(Pair.create(4 * ONE_MB, 91)));
         dataTracker.addInitialSSTables(sstables);
 
-        UnifiedCompactionStrategy.CompactionPick pick = strategy.getNextCompactionPick(0);
-        assertNotNull(pick);
-        assertEquals(9, strategy.getEstimatedRemainingTasks());
+        assertEquals(3, strategy.getNextBackgroundTasks(FBUtilities.nowInSeconds()).size()); // repaired, unrepaired, pending
+        Collection<CompactionAggregate> aggregates = strategy.backgroundCompactions.getAggregates();
+        assertEquals(3, aggregates.size());
+        for (CompactionAggregate aggregate : aggregates)
+            assertEquals(8, aggregate.getPending().size());
     }
 
     @Test
-    public void testMaximalSelection()
+    public void testCreateParallelTasks()
     {
+        testCreateParallelTasks(8, arr(1, 2, 4));
+        testCreateParallelTasks(4, arr(1, 2, 4));
+        testCreateParallelTasks(2, arr(1, 2, 4));
+        testCreateParallelTasks(5, arr(1, 2, 4));
+        testCreateParallelTasks(5, arr(2, 4, 8));
+        testCreateParallelTasks(3, arr(1, 3, 5));
+        testCreateParallelTasks(3, arr(3, 3, 3));
+
+        testCreateParallelTasks(1, arr(1, 2, 3));
+    }
+
+    @Test
+    public void testCreateParallelTasksMissingParts()
+    {
+        // Drop some sstables without losing ranges
+        testCreateParallelTasks(8, arr(2, 4, 8),
+                                arr(1));
+
+        testCreateParallelTasks(8, arr(2, 4, 8),
+                                arr(1), arr(0), arr(2, 7));
+
+        testCreateParallelTasks(5, arr(2, 4, 8),
+                                arr(1), arr(0), arr(2, 7));
+    }
+
+    @Test
+    public void testCreateParallelTasksOneRange()
+    {
+        // Drop second half
+        testCreateParallelTasks(2, arr(2, 4, 8),
+                                arr(1), arr(2, 3), arr(4, 5, 6, 7));
+        // Drop all except center, within shard
+        testCreateParallelTasks(3, arr(5, 7, 9),
+                                arr(0, 1, 3, 4), arr(0, 1, 2, 4, 5, 6), arr(0, 1, 2, 6, 7, 8));
+    }
+
+    @Test
+    public void testCreateParallelTasksSkippedRange()
+    {
+        // Drop all sstables containing the 4/8-5/8 range.
+        testCreateParallelTasks(8, arr(2, 4, 8),
+                                arr(1), arr(2), arr(4));
+        // Drop all sstables containing the 4/8-6/8 range.
+        testCreateParallelTasks(8, arr(2, 4, 8),
+                                arr(1), arr(2), arr(4, 5));
+        // Drop all sstables containing the 4/8-8/8 range.
+        testCreateParallelTasks(8, arr(2, 4, 8),
+                                arr(1), arr(2, 3), arr(4, 5, 6, 7));
+
+        // Drop all sstables containing the 0/8-2/8 range.
+        testCreateParallelTasks(5, arr(2, 4, 8),
+                                arr(0), arr(0), arr(0, 1));
+        // Drop all sstables containing the 6/8-8/8 range.
+        testCreateParallelTasks(5, arr(2, 4, 8),
+                                arr(1), arr(3), arr(6, 7));
+        // Drop sstables on both ends.
+        testCreateParallelTasks(5, arr(3, 4, 8),
+                                arr(0, 2), arr(0, 3), arr(0, 1, 6, 7));
+    }
+
+    public void testCreateParallelTasks(int numShards, int[] perLevelCounts, int[]... dropsPerLevel)
+    {
+        for (int parallelism = numShards; parallelism >= 1; --parallelism)
+            testCreateParallelTasks(numShards, parallelism, perLevelCounts, dropsPerLevel);
+    }
+
+    public void testCreateParallelTasks(int numShards, int concurrencyLimit, int[] perLevelCounts, int[]... dropsPerLevel)
+    {
+        // Note: This test has a counterpart in ShardManagerTest that exercises splitSSTablesInShards directly and more thoroughly.
+        // This one ensures the data is correctly passed to and presented in compaction tasks.
+        Set<SSTableReader> allSSTables = new HashSet<>();
+        int levelNum = 0;
+        for (int perLevelCount : perLevelCounts)
+        {
+            List<SSTableReader> ssTables = mockNonOverlappingSSTables(perLevelCount, levelNum, 100 << (20 + levelNum));
+            if (levelNum < dropsPerLevel.length)
+            {
+                for (int i = dropsPerLevel[levelNum].length - 1; i >= 0; i--)
+                    ssTables.remove(dropsPerLevel[levelNum][i]);
+            }
+            allSSTables.addAll(ssTables);
+            ++levelNum;
+        }
+        dataTracker.addInitialSSTables(allSSTables);
+
+        Controller controller = Mockito.mock(Controller.class);
+        when(controller.getNumShards(anyDouble())).thenReturn(numShards);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
+        when(controller.getOverheadSizeInBytes(any(), anyLong())).thenAnswer(inv -> (long) (inv.getArgument(1)) * 4 / 3);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+        strategy.startup();
+        LifecycleTransaction txn = dataTracker.tryModify(allSSTables, OperationType.COMPACTION);
+        var tasks = new ArrayList<CompactionTask>();
+        strategy.createAndAddTasks(0, txn, strategy.makeShardingStats(txn), concurrencyLimit, tasks);
+        int i = 0;
+        int[] expectedSSTablesInTasks = new int[tasks.size()];
+        int[] collectedSSTablesPerTask = new int[tasks.size()];
+        for (CompactionTask t : tasks)
+        {
+            assertTrue(t instanceof UnifiedCompactionTask);
+            assertFalse(t.inputSSTables().isEmpty());
+            collectedSSTablesPerTask[i] = t.inputSSTables().size();
+            expectedSSTablesInTasks[i] = (int) allSSTables.stream().filter(x -> intersects(x, t.tokenRange())).count();
+            t.rejected(null); // close transaction
+            ++i;
+        }
+        if (tasks.size() == 1)
+            assertNull(tasks.get(0).tokenRange()); // make sure single-task compactions are not ranged
+        Assert.assertEquals(Arrays.toString(expectedSSTablesInTasks), Arrays.toString(collectedSSTablesPerTask));
+        System.out.println(Arrays.toString(expectedSSTablesInTasks));
+        assertThat(tasks.size()).isLessThanOrEqualTo(concurrencyLimit);
+        assertEquals(allSSTables, tasks.stream().map(CompactionTask::inputSSTables).flatMap(Set::stream).collect(Collectors.toSet()));
+        for (var t : tasks)
+        {
+            for (var q : tasks)
+            {
+                if (t != q)
+                    assertFalse("Subranges " + t.tokenRange() + " and " + q.tokenRange() + "intersect", t.tokenRange().intersects(q.tokenRange()));
+            }
+            long compactionSize = t.totals != null ? t.totals.inputDiskSize : CompactionSSTable.getTotalDataBytes(t.inputSSTables());
+            assertEquals(4.0/3 * compactionSize, t.getSpaceOverhead(), 0.001 * compactionSize);
+        }
+
+        // make sure the composite transaction has the correct number of tasks
+        assertEquals(Transactional.AbstractTransactional.State.ABORTED, txn.state());
+    }
+
+    private boolean intersects(SSTableReader r, Range<Token> range)
+    {
+        if (range == null)
+            return true;
+        return range.intersects(range(r));
+    }
+
+
+    private Bounds<Token> range(SSTableReader x)
+    {
+        return new Bounds<>(x.getFirst().getToken(), x.getLast().getToken());
+    }
+
+    @Test
+    public void testDontCreateParallelTasks()
+    {
+        int numShards = 5;
         Set<SSTableReader> allSSTables = new HashSet<>();
         allSSTables.addAll(mockNonOverlappingSSTables(10, 0, 100 << 20));
         allSSTables.addAll(mockNonOverlappingSSTables(15, 1, 200 << 20));
         allSSTables.addAll(mockNonOverlappingSSTables(25, 2, 400 << 20));
-
+        dataTracker.addInitialSSTables(allSSTables);
         Controller controller = Mockito.mock(Controller.class);
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
-        strategy.addSSTables(allSSTables);
+        when(controller.getNumShards(anyDouble())).thenReturn(numShards);
+        when(controller.parallelizeOutputShards()).thenReturn(false);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+        strategy.startup();
+        LifecycleTransaction txn = dataTracker.tryModify(allSSTables, OperationType.COMPACTION);
+        var tasks = new ArrayList<CompactionTask>();
+        strategy.createAndAddTasks(0, txn, strategy.makeShardingStats(txn), 1000, tasks);
+        assertEquals(1, tasks.size());
+        assertEquals(allSSTables, tasks.get(0).inputSSTables());
+    }
+    @Test
+    public void testMaximalSelection()
+    {
+        // shared transaction, all tasks refer to the same input sstables
+        testMaximalSelection(1, 1, 0, false, 12 + 18 + 30, ((12 * 100L + 18 * 200 + 30 * 400) << 20));
+        testMaximalSelection(5, 5, 0, true, 12 + 18 + 30, ((12 * 100L + 18 * 200 + 30 * 400) << 20));
+        // when there's a common split point of existing and new sharding (i.e. gcd(num_shards,12,18,30) > 1), it should be used
+        testMaximalSelection(3, 3, 0, false, 4 + 6 + 10, ((4 * 100L + 6 * 200 + 10 * 400) << 20));
+        testMaximalSelection(9, 3, 0, false, 4 + 6 + 10, ((4 * 100L + 6 * 200 + 10 * 400) << 20));
+        testMaximalSelection(9, 9, 0, true, 4 + 6 + 10, ((4 * 100L + 6 * 200 + 10 * 400) << 20));
+        testMaximalSelection(2, 2, 0, false, 6 + 9 + 15, ((6 * 100L + 9 * 200 + 15 * 400) << 20));
+        testMaximalSelection(4, 2, 0, false, 6 + 9 + 15, ((6 * 100L + 9 * 200 + 15 * 400) << 20));
+        testMaximalSelection(4, 4, 0, true, 6 + 9 + 15, ((6 * 100L + 9 * 200 + 15 * 400) << 20));
+        testMaximalSelection(18, 6, 0, false, 2 + 3 + 5, ((2 * 100L + 3 * 200 + 5 * 400) << 20));
+        testMaximalSelection(18, 18, 0, true, 2 + 3 + 5, ((2 * 100L + 3 * 200 + 5 * 400) << 20));
+    }
+
+    @Test
+    public void testMaximalSelectionWithLimit()
+    {
+        // shared transaction, all tasks refer to the same input sstables
+        testMaximalSelection(5, 2, 2, true, 12 + 18 + 30, ((12 * 100L + 18 * 200 + 30 * 400) << 20));
+        // when there's a common split point of existing and new sharding (i.e. gcd(num_shards,12,18,30) > 1), it should be used
+        testMaximalSelection(3, 3, 2, false, 4 + 6 + 10, ((4 * 100L + 6 * 200 + 10 * 400) << 20));
+        testMaximalSelection(9, 3, 1, false, 4 + 6 + 10, ((4 * 100L + 6 * 200 + 10 * 400) << 20));
+        testMaximalSelection(9, 9, 3, true, 4 + 6 + 10, ((4 * 100L + 6 * 200 + 10 * 400) << 20));
+        testMaximalSelection(18, 6, 3, false, 2 + 3 + 5, ((2 * 100L + 3 * 200 + 5 * 400) << 20));
+        testMaximalSelection(18, 6, 2, false, 2 + 3 + 5, ((2 * 100L + 3 * 200 + 5 * 400) << 20));
+        testMaximalSelection(18, 12, 2, true, 2 + 3 + 5, ((2 * 100L + 3 * 200 + 5 * 400) << 20));
+        testMaximalSelection(18, 18, 3, true, 2 + 3 + 5, ((2 * 100L + 3 * 200 + 5 * 400) << 20));
+    }
+
+    private void testMaximalSelection(int numShards, int expectedTaskCount, int parallelismLimit, boolean parallelize, int originalsCount, long onDiskLength)
+    {
+        Set<SSTableReader> allSSTables = new HashSet<>();
+        allSSTables.addAll(mockNonOverlappingSSTables(12, 0, 100 << 20));
+        allSSTables.addAll(mockNonOverlappingSSTables(18, 1, 200 << 20));
+        allSSTables.addAll(mockNonOverlappingSSTables(30, 2, 400 << 20));
         dataTracker.addInitialSSTables(allSSTables);
 
-        Collection<AbstractCompactionTask> tasks = strategy.getMaximalTask(0, false);
-        assertEquals(5, tasks.size());  // 5 (gcd of 10,15,25) common boundaries
-        for (AbstractCompactionTask task : tasks)
+        Controller controller = Mockito.mock(Controller.class);
+        when(controller.getNumShards(anyDouble())).thenReturn(numShards);
+        when(controller.parallelizeOutputShards()).thenReturn(parallelize);
+        when(controller.maxConcurrentCompactions()).thenReturn(1000);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
+
+        CompactionTasks limitedParallelismTasks = strategy.getMaximalTasks(0, false, parallelismLimit);
+        Collection<AbstractCompactionTask> allTasks = (parallelismLimit > 0)
+                                             ? limitedParallelismTasks.stream().flatMap(t -> t instanceof CompositeCompactionTask
+                                                                                             ? ((CompositeCompactionTask) t).tasks.stream()
+                                                                                             : Stream.of(t)).collect(Collectors.toList())
+                                             : limitedParallelismTasks;
+        assertEquals(expectedTaskCount, allTasks.size());
+        for (AbstractCompactionTask task : allTasks)
         {
-            Set<SSTableReader> compacting = task.transaction.originals();
-            assertEquals(2 + 3 + 5, compacting.size()); // count / gcd sstables of each level
-            assertEquals((2 * 100L + 3 * 200 + 5 * 400) << 20, compacting.stream().mapToLong(SSTableReader::onDiskLength).sum());
+            Set<SSTableReader> compacting = task.getTransaction().originals();
+            assertEquals(originalsCount, compacting.size()); // count / gcd sstables of each level
+            assertEquals(onDiskLength, compacting.stream().mapToLong(CompactionSSTable::onDiskLength).sum());
 
-            // None of the selected sstables may intersect any in any other set.
-            for (AbstractCompactionTask task2 : tasks)
+            if (!(task.getTransaction() instanceof PartialLifecycleTransaction))
             {
-                if (task == task2)
-                    continue;
+                // None of the selected sstables may intersect any in any other set.
+                for (AbstractCompactionTask task2 : allTasks)
+                {
+                    if (task == task2)
+                        continue;
 
-                Set<SSTableReader> compacting2 = task2.transaction.originals();
-                for (SSTableReader r1 : compacting)
-                    for (SSTableReader r2 : compacting2)
-                        assertTrue(r1 + " intersects " + r2, r1.getFirst().compareTo(r2.getLast()) > 0 || r1.getLast().compareTo(r2.getFirst()) < 0);
+                    Set<SSTableReader> compacting2 = task2.getTransaction().originals();
+                    for (SSTableReader r1 : compacting)
+                        for (SSTableReader r2 : compacting2)
+                            assertTrue(r1 + " intersects " + r2, r1.getFirst().compareTo(r2.getLast()) > 0 || r1.getLast().compareTo(r2.getFirst()) < 0);
+                }
+            }
+        }
+
+        if (parallelismLimit > 0)
+        {
+            assertTrue(limitedParallelismTasks.size() <= parallelismLimit);
+            for (AbstractCompactionTask task : limitedParallelismTasks)
+            {
+                if (task instanceof CompositeCompactionTask)
+                {
+                    CompositeCompactionTask cct = (CompositeCompactionTask) task;
+                    // We want each compaction's parts to be split among the composite tasks, i.e. each op should appear at most once in each
+                    assertEquals(cct.tasks.size(), cct.tasks.stream().map(t -> t.getTransaction().opId()).distinct().count());
+                }
             }
         }
     }
@@ -707,7 +1864,7 @@ public void testBucketSelectionHalved()
     public void testBucketSelectionFives()
     {
         testBucketSelection(arr(25, 15, 10), repeats(5, arr(10)), Overlaps.InclusionMethod.TRANSITIVE);
-        testBucketSelection(arr(25, 15, 10), repeats(10, arr(6, 4)), Overlaps.InclusionMethod.SINGLE);
+        testBucketSelection(arr(25, 15, 10), new int [] {6, 4, 6, 6, 6, 6, 4, 4, 4, 4}, Overlaps.InclusionMethod.SINGLE);
         // When we take large sstables for one compaction, remaining overlaps don't have enough to trigger next
         testBucketSelection(arr(25, 15, 10), repeats(10, arr(3)), Overlaps.InclusionMethod.NONE, 20);
     }
@@ -722,8 +1879,7 @@ public void testBucketSelectionMissing()
     public void testBucketSelectionHalvesMissing()
     {
         // Drop one half: still compact because of overlap
-        // Note: picks are returned right-to-left because the random mock always returns 0, picking the last bucket.
-        testBucketSelection(repeats(4, arr(6, 3)), arr(6, 6, 5), Overlaps.InclusionMethod.TRANSITIVE, 0, 1);
+        testBucketSelection(repeats(4, arr(6, 3)), arr(5, 6, 6), Overlaps.InclusionMethod.TRANSITIVE, 0, 1);
         // Drop one full: don't compact
         testBucketSelection(repeats(4, arr(3, 6)), arr(6, 6), Overlaps.InclusionMethod.TRANSITIVE, 5, 1);
         // Drop two adjacent halves: don't compact
@@ -744,6 +1900,21 @@ private int[] repeats(int count, int... values)
         return rep;
     }
 
+    private int[] concat(int[]... arrays)
+    {
+        int total = 0;
+        for (int[] array : arrays)
+            total += array.length;
+        int[] result = new int[total];
+        int offset = 0;
+        for (int[] array : arrays)
+        {
+            System.arraycopy(array, 0, result, offset, array.length);
+            offset += array.length;
+        }
+        return result;
+    }
+
     public void testBucketSelection(int[] counts, int[] expecteds, Overlaps.InclusionMethod overlapInclusionMethod)
     {
         testBucketSelection(counts, expecteds, overlapInclusionMethod, 0);
@@ -756,7 +1927,7 @@ public void testBucketSelection(int[] counts, int[] expecteds, Overlaps.Inclusio
         for (int i = 0; i < fanout; ++i)
         {
             final int count = counts[i];
-            final List<SSTableReader> list = mockNonOverlappingSSTables(count, 0, (100 << 20) / count);
+            final List<SSTableReader> list = mockNonOverlappingSSTables(count, /*ignored*/ 0, (100 << 20) / count);
             if (i == 0)
             {
                 for (int k = dropFromFirst.length - 1; k >= 0; --k)
@@ -770,145 +1941,78 @@ public void testBucketSelection(int[] counts, int[] expecteds, Overlaps.Inclusio
         when(controller.getThreshold(anyInt())).thenCallRealMethod();
         when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod();
         when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0);
-        when(controller.getNumShards(anyDouble())).thenReturn(1);
         when(controller.getBaseSstableSize(anyInt())).thenReturn((double) (90 << 20));
-        when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can
-        when(controller.maxThroughput()).thenReturn(Double.MAX_VALUE);
-        when(controller.getIgnoreOverlapsInExpirationCheck()).thenReturn(false);
         when(controller.overlapInclusionMethod()).thenReturn(overlapInclusionMethod);
+        when(controller.parallelizeOutputShards()).thenReturn(true);
+        when(controller.getOverheadSizeInBytes(any(), anyLong())).thenAnswer(inv -> (long) (inv.getArgument(1)) * 4 / 3);
         Random randomMock = Mockito.mock(Random.class);
         when(randomMock.nextInt(anyInt())).thenReturn(0);
         when(controller.random()).thenReturn(randomMock);
-        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(cfs, new HashMap<>(), controller);
-        strategy.addSSTables(allSSTables);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
         dataTracker.addInitialSSTables(allSSTables);
 
-        List<UnifiedCompactionStrategy.CompactionPick> picks = new ArrayList<>();
-        while (true)
+        List<CompactionPick> picks = new ArrayList<>();
+        boolean compactionFound;
+        do
         {
-            UnifiedCompactionStrategy.CompactionPick pick = strategy.getNextCompactionPick(0);
-            if (pick == null)
-                break;
-            strategy.removeSSTables(pick);
-            picks.add(pick);
+            Collection<CompactionAggregate.UnifiedAggregate> aggregates = strategy.getPendingCompactionAggregates();
+            compactionFound = false;
+            for (CompactionAggregate a : aggregates)
+            {
+                CompactionPick pick = a.getSelected();
+                if (pick == null || pick.isEmpty())
+                    continue;
+                if (realm.tryModify(pick.sstables(), OperationType.COMPACTION) != null) // This will fail if the sstables are in multiple buckets (for Overlaps.InclusionMethod.NONE)
+                {
+                    compactionFound = true;
+                    picks.add(pick);
+                }
+            }
         }
-        assertEquals(expectedRemaining, strategy.getSSTables().size());
+        while (compactionFound);
+        assertEquals(expectedRemaining, Iterables.size(dataTracker.getNoncompacting()));
 
         assertEquals(expecteds.length, picks.size());
         int buckIdx = 0;
-        for (UnifiedCompactionStrategy.CompactionPick pick : picks)
+        for (CompactionPick pick : picks)
+            System.out.println("## " + pick.sstables().size());
+        for (CompactionPick pick : picks)
         {
             int expectedCount = expecteds[buckIdx++];
-            assertEquals(expectedCount, pick.size()); // count / gcd sstables of each level
+            assertEquals(expectedCount, pick.sstables().size()); // count / gcd sstables of each level
 
             if (overlapInclusionMethod == Overlaps.InclusionMethod.TRANSITIVE)
             {
                 // None of the selected sstables may intersect any in any other set.
-                for (UnifiedCompactionStrategy.CompactionPick pick2 : picks)
+                for (CompactionPick pick2 : picks)
                 {
                     if (pick == pick2)
                         continue;
 
-                    for (SSTableReader r1 : pick)
-                        for (SSTableReader r2 : pick2)
+                    for (CompactionSSTable r1 : pick.sstables())
+                        for (CompactionSSTable r2 : pick2.sstables())
                             assertTrue(r1 + " intersects " + r2, r1.getFirst().compareTo(r2.getLast()) > 0 || r1.getLast().compareTo(r2.getFirst()) < 0);
                 }
             }
+            assertEquals(1.3333, pick.overheadToDataRatio(), 0.0001);
         }
-    }
-
-    SSTableReader mockSSTable(int level, long bytesOnDisk, long timestamp, double hotness, DecoratedKey first, DecoratedKey last)
-    {
-        return mockSSTable(level, bytesOnDisk, timestamp, hotness, first, last, 0);
-    }
-
-    SSTableReader mockSSTable(long bytesOnDisk, long timestamp, DecoratedKey first, DecoratedKey last)
-    {
-        return mockSSTable(0, bytesOnDisk, timestamp, 0, first, last, 0);
-    }
-
-    SSTableReader mockSSTable(int level,
-                              long bytesOnDisk,
-                              long timestamp,
-                              double hotness,
-                              DecoratedKey first,
-                              DecoratedKey last,
-                              int ttl)
-    {
-        // We create a ton of mock SSTables that mockito is going to keep until the end of the test suite without stubOnly.
-        // Mockito keeps them alive to preserve the history of invocations which is not available for stubs. If we ever
-        // need history of invocations and remove stubOnly, we should also manually reset mocked SSTables in tearDown.
-        SSTableReader ret = Mockito.mock(SSTableReader.class, withSettings().stubOnly()
-                                                                            .defaultAnswer(RETURNS_SMART_NULLS));
-
-        when(ret.getSSTableLevel()).thenReturn(level);
-        when(ret.onDiskLength()).thenReturn(bytesOnDisk);
-        when(ret.uncompressedLength()).thenReturn(bytesOnDisk); // let's assume no compression
-        when(ret.getMaxTimestamp()).thenReturn(timestamp);
-        when(ret.getMinTimestamp()).thenReturn(timestamp);
-        when(ret.getFirst()).thenReturn(first);
-        when(ret.getLast()).thenReturn(last);
-        when(ret.isMarkedSuspect()).thenReturn(false);
-        when(ret.isRepaired()).thenReturn(false);
-        when(ret.getRepairedAt()).thenReturn(repairedAt);
-        when(ret.getPendingRepair()).thenReturn(null);
-        when(ret.isPendingRepair()).thenReturn(false);
-        when(ret.getColumnFamilyName()).thenReturn(table);
-        when(ret.toString()).thenReturn(String.format("Bytes on disk: %s, level %d, hotness %f, timestamp %d, first %s, last %s",
-                                                      FBUtilities.prettyPrintMemory(bytesOnDisk), level, hotness, timestamp, first, last));
-        long deletionTime;
-        if (ttl > 0)
-            deletionTime = TimeUnit.MILLISECONDS.toSeconds(timestamp) + ttl;
-        else
-            deletionTime = Long.MAX_VALUE;
-
-        when(ret.getMinLocalDeletionTime()).thenReturn(deletionTime);
-        when(ret.getMaxLocalDeletionTime()).thenReturn(deletionTime);
-        when(ret.getMinTTL()).thenReturn(ttl);
-        when(ret.getMaxTTL()).thenReturn(ttl);
-
-        return ret;
-    }
-
-    List<SSTableReader> mockSSTables(int numSSTables, long bytesOnDisk, double hotness, long timestamp)
-    {
-        DecoratedKey first = new BufferDecoratedKey(partitioner.getMinimumToken(), ByteBuffer.allocate(0));
-        DecoratedKey last = new BufferDecoratedKey(partitioner.getMinimumToken(), ByteBuffer.allocate(0));
 
-        List<SSTableReader> sstables = new ArrayList<>();
-        for (int i = 0; i < numSSTables; i++)
-        {
-            long b = (long)(bytesOnDisk * 0.95 + bytesOnDisk * 0.05 * random.nextDouble()); // leave 5% variability
-            double h = hotness * 0.95 + hotness * 0.05 * random.nextDouble(); // leave 5% variability
-            sstables.add(mockSSTable(0, b, timestamp, h, first, last, 0));
-        }
+        Mockito.when(controller.getNumShards(anyDouble())).thenReturn(16);  // co-prime with counts to ensure multiple sstables fall in each shard
+        // Make sure getMaxOverlapsMap does not fail.
+        System.out.println(strategy.getMaxOverlapsMap());
 
-        return sstables;
+        dataTracker.removeUnsafe(allSSTables);
     }
 
-    List<SSTableReader> mockNonOverlappingSSTables(int numSSTables, int level, long bytesOnDisk)
+    @Test
+    public void testGetLevel()
     {
-        if (!partitioner.splitter().isPresent())
-            throw new IllegalStateException(String.format("Cannot split ranges with current partitioner %s", partitioner));
-
-        ByteBuffer emptyBuffer = ByteBuffer.allocate(0);
-
-        long timestamp = System.currentTimeMillis();
-        List<SSTableReader> sstables = new ArrayList<>(numSSTables);
-        for (int i = 0; i < numSSTables; i++)
-        {
-            DecoratedKey first = new BufferDecoratedKey(boundary(numSSTables, i).nextValidToken(), emptyBuffer);
-            DecoratedKey last =  new BufferDecoratedKey(boundary(numSSTables, i+1), emptyBuffer);
-            sstables.add(mockSSTable(level, bytesOnDisk, timestamp, 0., first, last));
-
-            timestamp+=10;
-        }
-
-        return sstables;
-    }
+        Controller controller = Mockito.mock(Controller.class);
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller);
 
-    private Token boundary(int numSSTables, int i)
-    {
-        return partitioner.split(partitioner.getMinimumToken(), partitioner.getMaximumToken(), i * 1.0 / numSSTables);
+        UnifiedCompactionStrategy.Level level = strategy.getLevel(1, 0.25d, 0.5d);
+        assertEquals(1, level.index);
+        assertEquals(0.25d, level.min, 0);
+        assertEquals(0.5d, level.max, 0);
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/ZombieSSTablesTest.java b/test/unit/org/apache/cassandra/db/compaction/ZombieSSTablesTest.java
new file mode 100644
index 000000000000..b79bc59e2fe5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/ZombieSSTablesTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Sets;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ *
+ */
+public class ZombieSSTablesTest
+{
+    private static final String KEYSPACE1 = "BlacklistingCompactionsTest";
+    private static final String STANDARD_STCS = "Standard_STCS";
+    private static final String STANDARD_LCS = "Standard_LCS";
+    private static final String STANDARD_TWCS = "Standard_TWCS";
+    private static final String MAXIMAL = "_Maximal";
+    private static int maxValueSize;
+
+    @After
+    public void leakDetect() throws InterruptedException
+    {
+        System.gc();
+        System.gc();
+        System.gc();
+        Thread.sleep(10);
+    }
+
+    @BeforeClass
+    public static void defineSchema() throws ConfigurationException
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                KeyspaceParams.simple(1),
+                makeTable(STANDARD_STCS).compaction(CompactionParams.DEFAULT),
+                makeTable(STANDARD_LCS).compaction(CompactionParams.lcs(Collections.emptyMap())),
+                makeTable(STANDARD_TWCS).compaction(CompactionParams.twcs(Collections.emptyMap())),
+                makeTable(STANDARD_STCS + MAXIMAL).compaction(CompactionParams.DEFAULT),
+                makeTable(STANDARD_LCS + MAXIMAL).compaction(CompactionParams.lcs(Collections.emptyMap())),
+                makeTable(STANDARD_TWCS + MAXIMAL).compaction(CompactionParams.twcs(Collections.emptyMap())));
+
+        maxValueSize = DatabaseDescriptor.getMaxValueSize();
+        DatabaseDescriptor.setMaxValueSize(1024 * 1024);
+    }
+
+    /**
+     * Return a table metadata, we use types with fixed size to increase the chance of detecting corrupt data
+     */
+    private static TableMetadata.Builder makeTable(String tableName)
+    {
+        return SchemaLoader.standardCFMD(KEYSPACE1, tableName, 1, LongType.instance, LongType.instance, LongType.instance);
+    }
+
+    @AfterClass
+    public static void tearDown()
+    {
+        DatabaseDescriptor.setMaxValueSize(maxValueSize);
+    }
+
+    @Test
+    public void testWithSizeTieredCompactionStrategy() throws Exception
+    {
+        testZombieSSTables(STANDARD_STCS);
+    }
+
+    @Test
+    public void testWithLeveledCompactionStrategy() throws Exception
+    {
+        testZombieSSTables(STANDARD_LCS);
+    }
+
+    @Test
+    public void testWithTimeWindowCompactionStrategy() throws Exception
+    {
+        testZombieSSTables(STANDARD_TWCS);
+    }
+
+    @Test
+    public void testWithSizeTieredCompactionStrategyMaximal() throws Exception
+    {
+        testZombieSSTablesMaximal(STANDARD_STCS);
+    }
+
+    @Test
+    public void testWithLeveledCompactionStrategyMaximal() throws Exception
+    {
+        testZombieSSTablesMaximal(STANDARD_LCS);
+    }
+
+    @Test
+    public void testWithTimeWindowCompactionStrategyMaximal() throws Exception
+    {
+        testZombieSSTablesMaximal(STANDARD_TWCS);
+    }
+
+    private void prepareZombieSSTables(ColumnFamilyStore cfs) throws Exception
+    {
+        final int ROWS_PER_SSTABLE = 10;
+        final int SSTABLES = 15;
+        final int SSTABLES_TO_DELETE = 2;
+
+        cfs.truncateBlocking();
+
+        // disable compaction while flushing
+        cfs.disableAutoCompaction();
+        //test index corruption
+        //now create a few new SSTables
+        long maxTimestampExpected = Long.MIN_VALUE;
+        Set<DecoratedKey> inserted = new HashSet<>();
+
+        for (int j = 0; j < SSTABLES; j++)
+        {
+            for (int i = 0; i < ROWS_PER_SSTABLE; i++)
+            {
+                DecoratedKey key = Util.dk(String.valueOf(i));
+                long timestamp = j * ROWS_PER_SSTABLE + i;
+                new RowUpdateBuilder(cfs.metadata(), timestamp, key.getKey())
+                        .clustering(Long.valueOf(i))
+                        .add("val", Long.valueOf(i))
+                        .build()
+                        .applyUnsafe();
+                maxTimestampExpected = Math.max(timestamp, maxTimestampExpected);
+                inserted.add(key);
+            }
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+            CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected);
+            assertEquals(inserted.toString(), inserted.size(), Util.getAll(Util.cmd(cfs).build()).size());
+        }
+
+        Collection<SSTableReader> sstables = cfs.getLiveSSTables();
+
+        // delete first 'sstablesToDelete' SSTables, but make it so that the compaction strategy still thinks they
+        // are present by not sending the removal notification (this can normally happen due to a race between the add
+        // and remove notification for an sstable).
+        Set<SSTableReader> toDrop = sstables.stream().limit(SSTABLES_TO_DELETE).collect(Collectors.toSet());
+        cfs.getTracker().removeUnsafe(toDrop);
+        toDrop.stream().forEach(sstable -> sstable.selfRef().release());    // avoid leak
+        assertTrue(Sets.intersection(cfs.getLiveSSTables(), toDrop).isEmpty());
+    }
+
+    private void testZombieSSTablesMaximal(String tableName) throws Exception
+    {
+        // this test does enough rows to force multiple block indexes to be used
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(tableName + MAXIMAL);
+
+        prepareZombieSSTables(cfs);
+        Collection<AbstractCompactionTask> maximalTasks = cfs.getCompactionStrategy().getMaximalTasks(0, false, 0);
+        assertNotNull(maximalTasks);
+        assertFalse(maximalTasks.isEmpty());
+        maximalTasks.stream().forEach(task -> task.transaction.abort());    // avoid leak
+    }
+
+    private void testZombieSSTables(String tableName) throws Exception
+    {
+        // this test does enough rows to force multiple block indexes to be used
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(tableName);
+
+        prepareZombieSSTables(cfs);
+
+        cfs.getCompactionStrategyContainer().enable();
+        Collection<AbstractCompactionTask> nextBackgroundTasks = cfs.getCompactionStrategy().getNextBackgroundTasks(0);
+        assertNotNull(nextBackgroundTasks);
+        assertFalse(nextBackgroundTasks.isEmpty());
+        nextBackgroundTasks.stream().forEach(task -> task.transaction.abort());    // avoid leak
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/AdaptiveControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/AdaptiveControllerTest.java
new file mode 100644
index 000000000000..87e6ed71a6bc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/AdaptiveControllerTest.java
@@ -0,0 +1,432 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FixedMonotonicClock;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.when;
+
+public class AdaptiveControllerTest extends ControllerTest
+{
+    private CostsCalculator calculator;
+    private FixedMonotonicClock clock;
+
+    private final int minW = -10;
+    private final int maxW = 64;
+    private final int[] Ws = {0};
+    private final int[] previousWs = {0};
+    private final int interval = 60;
+    private final int minCost = 5;
+    private final int maxAdaptiveCompactions = 2;
+    private final double baseCost = minCost * 5;
+    private final double threshold = 0.15;
+
+    @Before
+    public void setup()
+    {
+        calculator = Mockito.mock(CostsCalculator.class);
+        clock = new FixedMonotonicClock();
+    }
+
+    private AdaptiveController makeController()
+    {
+        return makeController(dataSizeGB, numShards, sstableSizeMB, 0);
+    }
+
+    private AdaptiveController makeController(long dataSizeGB, int numShards, long sstableSizeMB, long minSSTableSizeMB)
+    {
+        return new AdaptiveController(clock,
+                                      env,
+                                      Ws,
+                                      previousWs,
+                                      Controller.DEFAULT_SURVIVAL_FACTORS,
+                                      dataSizeGB << 30,
+                                      minSSTableSizeMB << 20,
+                                      0,
+                                      0,
+                                      Controller.DEFAULT_MAX_SPACE_OVERHEAD,
+                                      0,
+                                      Controller.DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS,
+                                      Controller.DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION,
+                                      numShards,
+                                      false,
+                                      sstableSizeMB << 20,
+                                      Controller.DEFAULT_SSTABLE_GROWTH,
+                                      Controller.DEFAULT_RESERVED_THREADS,
+                                      Controller.DEFAULT_RESERVED_THREADS_TYPE,
+                                      Controller.DEFAULT_OVERLAP_INCLUSION_METHOD,
+                                      true,
+                                      false,
+                                      interval,
+                                      minW,
+                                      maxW,
+                                      threshold,
+                                      minCost,
+                                      maxAdaptiveCompactions,
+                                      keyspaceName,
+                                      tableName);
+    }
+
+    @Test
+    public void testLongTableNameFromOptions()
+    {
+        String longTableName = "test_create_k8yq1r75bpzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
+        when(cfs.getTableName()).thenReturn(longTableName);
+
+        Map<String, String> options = new HashMap<>();
+        options.put(AdaptiveController.THRESHOLD, "0.15");
+        // Calls fromOptions on long table name, which tries to read options from a file.
+        // The too long file name should not lead to a failure.
+        Controller controller = testFromOptions(true, options);
+        assertTrue(controller instanceof AdaptiveController);
+
+        when(cfs.getTableName()).thenReturn(tableName);
+    }
+
+    @Test
+    public void testFromOptions()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(AdaptiveController.MIN_SCALING_PARAMETER, "-10");
+        options.put(AdaptiveController.MAX_SCALING_PARAMETER, "32");
+        options.put(AdaptiveController.INTERVAL_SEC, "120");
+        options.put(AdaptiveController.THRESHOLD, "0.15");
+        options.put(AdaptiveController.MIN_COST, "5");
+        options.put(AdaptiveController.MAX_ADAPTIVE_COMPACTIONS, "-1");
+        options.put(Controller.SCALING_PARAMETERS_OPTION, "T5");
+        options.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+
+        int[] scalingParameters = new int[30];
+        Arrays.fill(scalingParameters, 1);
+        AdaptiveController.storeOptions(keyspaceName, tableName, scalingParameters, 10 << 20);
+
+        Controller controller = testFromOptions(true, options);
+        assertTrue(controller instanceof AdaptiveController);
+
+        for (int i = 0; i < 10; i++)
+        {
+            assertEquals(1, controller.getScalingParameter(i));
+            assertEquals(1, controller.getPreviousScalingParameter(i));
+        }
+        int[] emptyScalingParameters = {};
+        AdaptiveController.storeOptions(keyspaceName, tableName, emptyScalingParameters, 10 << 20);
+
+        Controller controller2 = testFromOptions(true, options);
+        assertTrue(controller2 instanceof AdaptiveController);
+
+        for (int i = 0; i < 10; i++)
+        {
+            assertEquals(3, controller2.getScalingParameter(i));
+            assertEquals(3, controller2.getPreviousScalingParameter(i));
+        }
+        AdaptiveController.getControllerConfigPath(keyspaceName, tableName).delete();
+
+        Map<String, String> options2 = new HashMap<>();
+        options2.put(AdaptiveController.MIN_SCALING_PARAMETER, "-10");
+        options2.put(AdaptiveController.MAX_SCALING_PARAMETER, "32");
+        options2.put(AdaptiveController.INTERVAL_SEC, "120");
+        options2.put(AdaptiveController.THRESHOLD, "0.15");
+        options2.put(AdaptiveController.MIN_COST, "5");
+        options2.put(AdaptiveController.MAX_ADAPTIVE_COMPACTIONS, "-1");
+        options2.put(Controller.SCALING_PARAMETERS_OPTION, "L5");
+        options2.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+
+        Controller controller3 = testFromOptions(true, options2);
+        assertTrue(controller3 instanceof AdaptiveController);
+
+        for (int i = 0; i < 10; i++)
+        {
+            assertEquals(-3, controller3.getScalingParameter(i));
+            assertEquals(-3, controller3.getPreviousScalingParameter(i));
+        }
+
+        Map<String, String> options3 = new HashMap<>();
+        options3.put(AdaptiveController.MIN_SCALING_PARAMETER, "-10");
+        options3.put(AdaptiveController.MAX_SCALING_PARAMETER, "32");
+        options3.put(AdaptiveController.INTERVAL_SEC, "120");
+        options3.put(AdaptiveController.THRESHOLD, "0.15");
+        options3.put(AdaptiveController.MIN_COST, "5");
+        options3.put(AdaptiveController.MAX_ADAPTIVE_COMPACTIONS, "-1");
+        options3.put(Controller.STATIC_SCALING_FACTORS_OPTION, "4");
+        options3.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+
+        Controller controller4 = testFromOptions(true, options3);
+        assertTrue(controller4 instanceof AdaptiveController);
+
+        for (int i = 0; i < 10; i++)
+        {
+            assertEquals(4, controller4.getScalingParameter(i));
+            assertEquals(4, controller4.getPreviousScalingParameter(i));
+        }
+    }
+
+    @Test
+    public void testValidateOptions()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(AdaptiveController.MIN_SCALING_PARAMETER, "-10");
+        options.put(AdaptiveController.MAX_SCALING_PARAMETER, "32");
+        options.put(AdaptiveController.INTERVAL_SEC, "120");
+        options.put(AdaptiveController.THRESHOLD, "0.15");
+        options.put(AdaptiveController.MIN_COST, "5");
+        options.put(AdaptiveController.MAX_ADAPTIVE_COMPACTIONS, "-1");
+
+        super.testValidateOptions(options, true);
+
+        Map<String, String> options2 = new HashMap<>();
+        options2.put(AdaptiveController.MIN_SCALING_PARAMETER, "-10");
+        options2.put(AdaptiveController.MAX_SCALING_PARAMETER, "32");
+        options2.put(AdaptiveController.INTERVAL_SEC, "120");
+        options2.put(AdaptiveController.THRESHOLD, "0.15");
+        options2.put(AdaptiveController.MIN_COST, "5");
+        options2.put(AdaptiveController.MAX_ADAPTIVE_COMPACTIONS, "-1");
+        options2.put(Controller.STATIC_SCALING_FACTORS_OPTION, "1,2,3");
+
+        super.testValidateOptions(options2, true);
+
+        Map<String, String> options3 = new HashMap<>();
+        options3.put(AdaptiveController.MIN_SCALING_PARAMETER, "-10");
+        options3.put(AdaptiveController.MAX_SCALING_PARAMETER, "32");
+        options3.put(AdaptiveController.INTERVAL_SEC, "120");
+        options3.put(AdaptiveController.THRESHOLD, "0.15");
+        options3.put(AdaptiveController.MIN_COST, "5");
+        options3.put(AdaptiveController.MAX_ADAPTIVE_COMPACTIONS, "-1");
+        options3.put(Controller.SCALING_PARAMETERS_OPTION, "1,2,3");
+
+        super.testValidateOptions(options3, true);
+    }
+
+    @Test
+    public void testValidateCompactionStrategyOptions()
+    {
+        super.testValidateCompactionStrategyOptions(true);
+    }
+
+    @Test
+    public void testStartShutdown()
+    {
+        AdaptiveController controller = makeController();
+        testStartShutdown(controller);
+    }
+
+    @Test
+    public void testShutdownNotStarted()
+    {
+        AdaptiveController controller = makeController();
+        testShutdownNotStarted(controller);
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testStartAlreadyStarted()
+    {
+        AdaptiveController controller = makeController();
+        testStartAlreadyStarted(controller);
+    }
+
+    @Test
+    public void testMinSSTableSizeDynamic()
+    {
+        // <= 50 MB, round up to 50 MB
+        testMinSSTableSizeDynamic(1, 50);
+        testMinSSTableSizeDynamic((50 << 20) - 1, 50);
+        testMinSSTableSizeDynamic(50 << 20, 50);
+
+        // <= 100 MB, round up to 100 MB
+        testMinSSTableSizeDynamic((50 << 20) + 1, 100);
+        testMinSSTableSizeDynamic((100 << 20) - 1, 100);
+        testMinSSTableSizeDynamic(100 << 20, 100);
+
+        // no flush size, 50 MB, then flush size of 100 MB + 1 returns 150MB
+        testMinSSTableSizeDynamic(0, 50, (100 << 20) + 1, 150);
+    }
+
+    private void testMinSSTableSizeDynamic(long flushSizeBytes1, int minSSTableSizeMB1)
+    {
+        // The most common case, the second calculation is skipped so even if the env returns zero the second time, the result won't change
+        testMinSSTableSizeDynamic(flushSizeBytes1, minSSTableSizeMB1, 0, minSSTableSizeMB1);
+    }
+
+    private void testMinSSTableSizeDynamic(long flushSizeBytes1, int minSSTableSizeMB1, long flushSizeBytes2, int minSSTableSizeMB2)
+    {
+        // create a controller with minSSTableSizeMB set to zero so that it will calculate the min sstable size from the flush size
+        AdaptiveController controller = makeController(dataSizeGB, numShards, Integer.MAX_VALUE, -1);
+
+        when(env.flushSize()).thenReturn(flushSizeBytes1 * 1.0);
+        assertEquals(minSSTableSizeMB1 << 20, controller.getMinSstableSizeBytes());
+
+        when(env.flushSize()).thenReturn(flushSizeBytes2 * 1.0);
+        assertEquals(minSSTableSizeMB2 << 20, controller.getMinSstableSizeBytes());
+    }
+
+
+    @Test
+    public void testUpdateNotEnoughTimeElapsed()
+    {
+        AdaptiveController controller = makeController();
+        controller.startup(strategy, calculator);
+
+        // no update, not enough time elapsed
+        controller.onStrategyBackgroundTaskRequest();
+        assertEquals(Ws[0], controller.getScalingParameter(0));
+    }
+
+    @Test
+    public void testUpdateBelowMinCost() throws InterruptedException
+    {
+        AdaptiveController controller = makeController();
+        controller.startup(strategy, calculator);
+
+        // no update, <= min cost
+        when(calculator.getReadCostForQueries(anyInt())).thenReturn((double) minCost);
+        when(calculator.getReadCostForQueries(anyInt())).thenReturn(0.);
+        when(calculator.spaceUsed()).thenReturn(1.0);
+
+        clock.setNowInNanos(clock.now() + TimeUnit.SECONDS.toNanos(interval + 1));
+        controller.onStrategyBackgroundTaskRequest();
+        assertEquals(Ws[0], controller.getScalingParameter(0));
+    }
+
+    @Test
+    public void testUpdateWithSize_min() throws InterruptedException
+    {
+        long totSize = (long) sstableSizeMB << 20;
+        testUpdateWithSize(totSize, new double[]{ baseCost, 0, baseCost }, new double[]{ 0, baseCost, baseCost }, new int[]{ 0, 0, 0 });
+    }
+
+    @Test
+    public void testUpdateWithSize_1GB() throws InterruptedException
+    {
+        long totSize = 1L << 31;
+        testUpdateWithSize(totSize, new double[]{ baseCost, 0, baseCost }, new double[]{ 0, baseCost, baseCost }, new int[]{ -9, 31, 1 });
+    }
+
+    @Test
+    public void testUpdateWithSize_2GB() throws InterruptedException
+    {
+        long totSize = 2L << 31;
+        testUpdateWithSize(totSize, new double[]{ baseCost, 0, baseCost }, new double[]{ 0, baseCost, baseCost }, new int[]{ -5, 44, 1 } );
+    }
+
+    @Test
+    public void testUpdateWithSize_128GB() throws InterruptedException
+    {
+        long totSize = 1L << 37;
+        testUpdateWithSize(totSize, new double[] {baseCost, 0, baseCost}, new double[] {0, baseCost, baseCost}, new int[] {-8, 39, 1});
+    }
+
+    @Test
+    public void testUpdateWithSize_512GB() throws InterruptedException
+    {
+        long totSize = 1L << 39;
+        testUpdateWithSize(totSize, new double[] {baseCost, 0, baseCost}, new double[] {0, baseCost, baseCost}, new int[] {-7, 63, 1});
+    }
+
+    @Test
+    public void testUpdateWithSize_1TB() throws InterruptedException
+    {
+        long totSize = 1L << 40;
+        testUpdateWithSize(totSize, new double[] {baseCost, 0, baseCost}, new double[] {0, baseCost, baseCost}, new int[] {-7, 25, 1});
+    }
+
+    @Test
+    public void testUpdateWithSize_5TB() throws InterruptedException
+    {
+        long totSize = 5 * (1L << 40);
+        testUpdateWithSize(totSize, new double[] {baseCost, 0, baseCost}, new double[] {0, baseCost, baseCost}, new int[] {-10, 39, 1});
+    }
+
+    @Test
+    public void testUpdateWithSize_10TB() throws InterruptedException
+    {
+        long totSize = 10 * (1L << 40);
+        testUpdateWithSize(totSize, new double[] { baseCost, 0, baseCost}, new double[] { 0, baseCost, baseCost}, new int[] { -8, 46, 1});
+    }
+
+    @Test
+    public void testUpdateWithSize_20TB() throws InterruptedException
+    {
+        long totSize = 20 * (1L << 49);
+        testUpdateWithSize(totSize, new double[] { baseCost, 0, baseCost}, new double[] { 0, baseCost, baseCost}, new int[] { -8, 40, 1});
+    }
+
+    private void testUpdateWithSize(long totSize, double[] readCosts, double[] writeCosts, int[] expectedWs) throws InterruptedException
+    {
+        int shardSizeGB = (int) (totSize >> 30);
+        AdaptiveController controller = makeController(shardSizeGB, 1, sstableSizeMB, 0); // one unique shard
+        controller.startup(strategy, calculator);
+
+        assertEquals(readCosts.length, writeCosts.length);
+        assertEquals(writeCosts.length, expectedWs.length);
+
+        when(calculator.spaceUsed()).thenReturn((double) totSize);
+
+        for (int i = 0; i < readCosts.length; i++)
+        {
+            final double readCost = readCosts[i];
+            final double writeCost = writeCosts[i];
+
+            when(calculator.getReadCostForQueries(anyInt())).thenAnswer(answ -> (int) answ.getArgument(0) * readCost);
+            when(calculator.getWriteCostForQueries(anyInt())).thenAnswer(answ -> (int) answ.getArgument(0) * writeCost);
+
+            clock.setNowInNanos(clock.now() + TimeUnit.SECONDS.toNanos(interval + 1));
+
+            controller.onStrategyBackgroundTaskRequest();
+            assertEquals(expectedWs[i], controller.getScalingParameter(0));
+        }
+    }
+
+    @Test
+    public void testMetrics()
+    {
+        TableMetadata metadata = TableMetadata.builder("ks", "table")
+                                              .partitioner(Murmur3Partitioner.instance)
+                                              .addPartitionKeyColumn("key", UTF8Type.instance)
+                                              .addClusteringColumn("col", UTF8Type.instance)
+                                              .addRegularColumn("value", UTF8Type.instance)
+                                              .caching(CachingParams.CACHE_NOTHING)
+                                              .build();
+        Controller.Metrics metrics = new Controller.Metrics(metadata);
+        AdaptiveController controller = makeController();
+        metrics.setController(controller);
+
+        double wa = metrics.getMeasuredWA();
+        double readIo = metrics.getReadIOCost();
+        double writeIo = metrics.getWriteIOCost();
+        double totalIo = metrics.getTotalIOCost();
+
+        assertEquals(0, wa, 0);
+        assertEquals(0, readIo, 0);
+        assertEquals(0, writeIo, 0);
+        assertEquals(0, totalIo, 0);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/BackgroundCompactionTrackingTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/BackgroundCompactionTrackingTest.java
new file mode 100644
index 000000000000..6af5e7a4da65
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/BackgroundCompactionTrackingTest.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionStrategy;
+import org.apache.cassandra.db.compaction.CompactionStrategyStatistics;
+import org.apache.cassandra.db.compaction.TableOperation;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStatistics;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitConfig;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+@RunWith(BMUnitRunner.class)
+@BMUnitConfig(debug=true)
+@BMRule(
+name = "Get stats before task completion",
+targetClass = "org.apache.cassandra.db.compaction.ActiveOperations",
+targetMethod = "completeOperation",
+targetLocation = "AT ENTRY",
+action = "org.apache.cassandra.db.compaction.unified.BackgroundCompactionTrackingTest.getStats()"
+)
+public class BackgroundCompactionTrackingTest extends CQLTester
+{
+    // Get rid of commitlog noise
+    @Before
+    public void disableCommitlog()
+    {
+        schemaChange("ALTER KEYSPACE " + KEYSPACE + " WITH durable_writes = false");
+    }
+    @After
+    public void enableCommitlog()
+    {
+        schemaChange("ALTER KEYSPACE " + KEYSPACE + " WITH durable_writes = true");
+    }
+
+    @Test
+    public void testBackgroundCompactionTrackingIterators() throws InterruptedException
+    {
+        testBackgroundCompactionTracking(false, false, 5);
+    }
+
+    @Test
+    public void testBackgroundCompactionTrackingIteratorsParallelized() throws InterruptedException
+    {
+        testBackgroundCompactionTracking(true, false, 5);
+    }
+
+    @Test
+    public void testBackgroundCompactionTrackingCursors() throws InterruptedException
+    {
+        testBackgroundCompactionTracking(false, true,5);
+    }
+
+    @Test
+    public void testBackgroundCompactionTrackingCursorsParallelized() throws InterruptedException
+    {
+        testBackgroundCompactionTracking(true, true,5);
+    }
+
+    public void testBackgroundCompactionTracking(boolean parallelize, boolean useCursors, int shards) throws InterruptedException
+    {
+        CompactionManager.instance.setMaximumCompactorThreads(50);
+        CompactionManager.instance.setCoreCompactorThreads(50);
+        CassandraRelevantProperties.ALLOW_CURSOR_COMPACTION.setBoolean(useCursors);
+        String table = createTable(String.format("CREATE TABLE %%s (k int, t int, v blob, PRIMARY KEY (k, t))" +
+                                                 " with compaction = {" +
+                                                 "'class': 'UnifiedCompactionStrategy', " +
+                                                 "'parallelize_output_shards': '%s', " +
+                                                 "'num_shards': %d, " +
+                                                 "'min_sstable_size': '1KiB', " +
+                                                 "'log': 'all', " +
+                                                 "'scaling_parameters': 'T4, T7'" +
+                                   "}",
+                                   parallelize, shards));
+        ColumnFamilyStore cfs = getColumnFamilyStore(KEYSPACE, table);
+        cfs.disableAutoCompaction();
+        strategy = cfs.getCompactionStrategy();
+        int partitions = 5000;
+        int rows_per_partition = 10;
+
+        for (int iter = 1; iter <= 5; ++iter)
+        {
+            byte [] payload = new byte[5000];
+            new Random(42).nextBytes(payload);
+            ByteBuffer b = ByteBuffer.wrap(payload);
+            Set<SSTableReader> before = new HashSet<>(cfs.getLiveSSTables());
+
+            for (int i = 0; i < partitions; i++)
+            {
+                for (int j = 0; j < rows_per_partition; j++)
+                    execute(String.format("INSERT INTO %s.%s(k, t, v) VALUES (?, ?, ?)", KEYSPACE, table), i, j, b);
+
+                if ((i + 1) % ((partitions  + 3) / 4) == 0)
+                    cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+            }
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+            operations = new ArrayList<>();
+            statistics = new ArrayList<>();
+            Set<SSTableReader> newSSTables = new HashSet<>(cfs.getLiveSSTables());
+            newSSTables.removeAll(before);
+            long totalSize = newSSTables.stream().mapToLong(SSTableReader::onDiskLength).sum();
+            long uncompressedSize = newSSTables.stream().mapToLong(SSTableReader::uncompressedLength).sum();
+
+            cfs.enableAutoCompaction(true); // since the trigger is hit, this initiates an L0 compaction
+            CompactionManager.instance.submitBackground(cfs).await();    // no more compactions to run, refresh stats
+            cfs.disableAutoCompaction();
+
+            // Check that the background compactions state is correct during the compaction
+            Assert.assertTrue("Byteman rule did not fire", !operations.isEmpty());
+            printStats();
+            assertEquals(1, operations.size());
+            var ops = operations.get(0)
+                                .stream()
+                                .filter(op -> op.metadata() == cfs.metadata())
+                                .collect(Collectors.toList());
+            assertEquals(1, ops.size());
+            var op = ops.get(0);
+            {
+                assertSame(cfs.metadata(), op.metadata());
+
+                assertEquals(uncompressedSize, op.total());
+                assertEquals(uncompressedSize, op.completed());
+            }
+
+            var stats = statistics.get(0).get(0); // unrepaired
+            if (stats.aggregates().size() > 1)
+            {
+                var L1 = (UnifiedCompactionStatistics) stats.aggregates().get(1);
+                assertEquals(1, L1.bucket());
+                assertEquals(shards * (iter - 1), L1.numSSTables());  // pre-compaction state
+                assertEquals(totalSize * 1.0 * (iter - 1), L1.sizeInBytes(), totalSize * 0.03);
+                assertEquals(iter - 1, L1.maxOverlap());
+            }
+            var L0 = (UnifiedCompactionStatistics) stats.aggregates().get(0);
+            assertEquals(0, L0.bucket());
+            assertEquals(totalSize * 1.0, L0.sizeInBytes(), totalSize * 0.03);
+            assertEquals(uncompressedSize * 1.0, L0.tot(), uncompressedSize * 0.03);
+            assertEquals(uncompressedSize * 1.0, L0.written(), uncompressedSize * 0.03);
+            assertEquals(uncompressedSize * 1.0, L0.read(), uncompressedSize * 0.03);
+            assertEquals(1, L0.numCompactionsInProgress());
+            assertEquals(4, L0.numCompactingSSTables());
+            assertEquals(4, L0.numSSTables());
+            assertEquals(4, L0.maxOverlap());
+
+            assertEquals(iter * shards, cfs.getLiveSSTables().size());
+
+            // Check that the background compactions state is correct after the compaction
+            operations.clear();
+            statistics.clear();
+            getStats();
+            printStats();
+            assertEquals(0, operations.get(0).size());
+            stats = statistics.get(statistics.size() - 1).get(0); // unrepaired
+            var L1 = (UnifiedCompactionStatistics) stats.aggregates().get(0);
+            assertEquals(1, L1.bucket());
+            assertEquals(shards * iter, L1.numSSTables());  // pre-compaction state
+            assertEquals(totalSize * 1.0 * iter, L1.sizeInBytes(), totalSize * 0.03);
+            assertEquals(iter, L1.maxOverlap());
+        }
+    }
+
+    private void printStats()
+    {
+        for (int i = 0; i < operations.size(); ++i)
+        {
+            System.out.println(operations.get(i).stream().map(Object::toString).collect(Collectors.joining("\n")));
+            System.out.println(statistics.get(i));
+        }
+    }
+
+    public static synchronized void getStats()
+    {
+        operations.add(CompactionManager.instance.getSSTableTasks()
+                                                 .stream()
+                                                 .map(BackgroundCompactionTrackingTest::snapshot)
+                                                 .collect(Collectors.toList()));
+        statistics.add(strategy.getStatistics());
+    }
+
+    private static TableOperation.Progress snapshot(TableOperation.Progress progress)
+    {
+        // Take a snapshot to make sure we are capturing the values at the time ActiveOperations is called.
+        // This is to make sure we report the completed state then, and not end up okay because they were corrected
+        // when some component closed at a later time.
+        return new AbstractTableOperation.OperationProgress(progress.metadata(),
+                                                            progress.operationType(),
+                                                            progress.completed(),
+                                                            progress.total(),
+                                                            progress.unit(),
+                                                            progress.operationId(),
+                                                            progress.sstables(),
+                                                            null);
+    }
+
+    static CompactionStrategy strategy;
+    static List<List<CompactionStrategyStatistics>> statistics;
+    static List<List<TableOperation.Progress>> operations;
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/CassandraRelevantPropertiesLegacyFallbackTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/CassandraRelevantPropertiesLegacyFallbackTest.java
new file mode 100644
index 000000000000..a41d85a02553
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/CassandraRelevantPropertiesLegacyFallbackTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.shared.WithProperties;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_PREFIX;
+import static org.apache.cassandra.config.CassandraRelevantProperties.LEGACY_PREFIX;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_ADAPTIVE_MIN_COST;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_MIN_SSTABLE_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_SHARED_STORAGE;
+import static org.apache.cassandra.db.compaction.unified.Controller.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+// checkstyle: suppress below 'blockSystemPropertyUsage'
+
+public class CassandraRelevantPropertiesLegacyFallbackTest
+{
+
+    /**
+     * Making sure to start each test with clean system property state
+     */
+    @After
+    public void doAfter()
+    {
+        CassandraRelevantProperties[] options = new CassandraRelevantProperties[]{ UCS_MIN_SSTABLE_SIZE, UCS_ADAPTIVE_MIN_COST, UCS_SHARED_STORAGE };
+        String[] prefixes = new String[]{ CASSANDRA_PREFIX, LEGACY_PREFIX };
+
+        for (CassandraRelevantProperties option : options)
+        {
+            option.clearValueWithLegacyPrefix();
+        }
+    }
+
+    @Test
+    public void testGetStringProperty()
+    {
+        try (WithProperties properties = new WithProperties().clear(UCS_MIN_SSTABLE_SIZE))
+        {
+            String propValue = UCS_MIN_SSTABLE_SIZE.getStringWithLegacyFallback();
+            assertEquals(DEFAULT_MIN_SSTABLE_SIZE, FBUtilities.parseHumanReadableBytes(propValue));
+
+            UCS_MIN_SSTABLE_SIZE.setString("50MiB");
+            assertEquals("50MiB", UCS_MIN_SSTABLE_SIZE.getStringWithLegacyFallback());
+
+            System.setProperty(UCS_MIN_SSTABLE_SIZE.getKeyWithLegacyPrefix(), "1MiB");
+            // Since PREFIX is set it takes precedence
+            assertEquals("50MiB", UCS_MIN_SSTABLE_SIZE.getStringWithLegacyFallback());
+
+            UCS_MIN_SSTABLE_SIZE.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+            assertEquals("1MiB", UCS_MIN_SSTABLE_SIZE.getStringWithLegacyFallback());
+        }
+    }
+
+    @Test
+    public void testGetIntegerSystemProperty()
+    {
+        try (WithProperties properties = new WithProperties().clear(UCS_ADAPTIVE_MIN_COST))
+        {
+            int minCost = UCS_ADAPTIVE_MIN_COST.getIntWithLegacyFalback();
+            assertEquals(1000, minCost);
+
+            UCS_ADAPTIVE_MIN_COST.setString("500");
+            assertEquals(500, UCS_ADAPTIVE_MIN_COST.getIntWithLegacyFalback());
+
+            System.setProperty(UCS_ADAPTIVE_MIN_COST.getKeyWithLegacyPrefix(), "100");
+            assertEquals(500, UCS_ADAPTIVE_MIN_COST.getIntWithLegacyFalback());
+
+            UCS_ADAPTIVE_MIN_COST.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+            assertEquals(100, UCS_ADAPTIVE_MIN_COST.getIntWithLegacyFalback());
+        }
+    }
+
+    @Test
+    public void testGetBooleanSystemProperty()
+    {
+        try (WithProperties properties = new WithProperties().clear(UCS_SHARED_STORAGE))
+        {
+            boolean sharedStorage = UCS_SHARED_STORAGE.getBooleanWithLegacyFallback();
+            assertFalse(sharedStorage);
+
+            System.setProperty(UCS_SHARED_STORAGE.getKeyWithLegacyPrefix(), "false");
+            assertFalse(UCS_SHARED_STORAGE.getBooleanWithLegacyFallback());
+
+            System.setProperty(UCS_SHARED_STORAGE.getKeyWithLegacyPrefix(), "true");
+            assertTrue(UCS_SHARED_STORAGE.getBooleanWithLegacyFallback());
+
+            UCS_SHARED_STORAGE.setString("false");
+            assertFalse(UCS_SHARED_STORAGE.getBooleanWithLegacyFallback());
+
+            UCS_SHARED_STORAGE.setString("true");
+            assertTrue(UCS_SHARED_STORAGE.getBooleanWithLegacyFallback());
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/ControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/ControllerTest.java
index 202bbc0f984a..ea3f30270009 100644
--- a/test/unit/org/apache/cassandra/db/compaction/unified/ControllerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/ControllerTest.java
@@ -1,13 +1,11 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Copyright DataStax, Inc.
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -21,42 +19,67 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
+import java.util.function.Predicate;
 
-import com.google.common.collect.ImmutableList;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
-import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DiskBoundaries;
-import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.compaction.CompactionStrategyOptions;
 import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.ReplicationFactor;
+import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MovingAverage;
 import org.apache.cassandra.utils.Overlaps;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 
-import static java.lang.String.format;
-import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.when;
 
-public class ControllerTest
+@Ignore
+public abstract class ControllerTest
 {
     static final double epsilon = 0.00000001;
+    static final long dataSizeGB = 512;
+    static final int numShards = 4; // pick it so that dataSizeGB is exactly divisible or tests will break
+    static final long sstableSizeMB = 2;
+    static final double maxSpaceOverhead = 0.3d;
     static final boolean allowOverlaps = false;
     static final long checkFrequency= 600L;
+    static final float tombstoneThresholdOption = 1;
+    static final long tombstoneCompactionIntervalOption = 1;
+    static final boolean uncheckedTombstoneCompactionOption = true;
+    static final boolean logAllOption = true;
+    static final String logTypeOption = "all";
+    static final int logPeriodMinutesOption = 1;
+    static final boolean compactionEnabled = true;
+    static final double readMultiplier = 0.5;
+    static final double writeMultiplier = 1.0;
+    static final String tableName = "tbl";
 
     @Mock
     ColumnFamilyStore cfs;
@@ -67,12 +90,31 @@ public class ControllerTest
     @Mock
     UnifiedCompactionStrategy strategy;
 
+    @Mock
+    ScheduledExecutorService executorService;
+
+    @Mock
+    ScheduledFuture fut;
+
+    @Mock
+    Environment env;
+
+    @Mock
+    AbstractReplicationStrategy replicationStrategy;
+
+    @Mock
+    DiskBoundaries boundaries;
+
     protected String keyspaceName = "TestKeyspace";
-    protected DiskBoundaries diskBoundaries = new DiskBoundaries(cfs, null, null, 0, 0);
+    protected int numDirectories = 1;
+    protected boolean useVector = false;
 
     @BeforeClass
     public static void setUpClass()
     {
+        // The def below should match Controller.PREFIX + Controller.OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES
+        // We can't reference these, because it would initialize Controller (and get the value before we modify it).
+        UCS_OVERRIDE_UCS_CONFIG_FOR_VECTOR_TABLES.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
     }
 
@@ -85,64 +127,150 @@ public void setUp()
         when(strategy.getEstimatedRemainingTasks()).thenReturn(0);
 
         when(metadata.toString()).thenReturn("");
+        when(replicationStrategy.getReplicationFactor()).thenReturn(ReplicationFactor.fullOnly(3));
+        when(cfs.makeUCSEnvironment()).thenAnswer(invocation -> new RealEnvironment(cfs));
+        when(cfs.getKeyspaceReplicationStrategy()).thenReturn(replicationStrategy);
         when(cfs.getKeyspaceName()).thenAnswer(invocation -> keyspaceName);
-        when(cfs.getDiskBoundaries()).thenAnswer(invocation -> diskBoundaries);
+        when(cfs.getDiskBoundaries()).thenReturn(boundaries);
+        when(cfs.buildShardManager()).thenCallRealMethod();
+        when(cfs.makeUCSEnvironment()).thenCallRealMethod();
+        when(cfs.getTableName()).thenReturn(tableName);
+        when(boundaries.getNumBoundaries()).thenAnswer(invocation -> numDirectories);
+
+        when(executorService.scheduleAtFixedRate(any(Runnable.class), anyLong(), anyLong(), any(TimeUnit.class))).thenReturn(fut);
+
+        when(env.flushSize()).thenReturn((double) (sstableSizeMB << 20));
+        when(cfs.metadata()).thenReturn(metadata);
+        when(metadata.hasVectorType()).thenAnswer(invocation -> useVector);
     }
 
-    Controller testFromOptions(Map<String, String> options)
+    Controller testFromOptions(boolean adaptive, Map<String, String> options)
     {
-        addOptions(false, options);
+        addOptions(adaptive, options);
         Controller.validateOptions(options);
 
         Controller controller = Controller.fromOptions(cfs, options);
         assertNotNull(controller);
         assertNotNull(controller.toString());
 
+        assertEquals(dataSizeGB << 30, controller.getDataSetSizeBytes());
+        assertFalse(controller.isRunning());
         for (int i = 0; i < 5; i++) // simulate 5 levels
             assertEquals(Controller.DEFAULT_SURVIVAL_FACTOR, controller.getSurvivalFactor(i), epsilon);
-        assertEquals(1, controller.getNumShards(0));
-        assertEquals(4, controller.getNumShards(16 * 100 << 20));
-        assertEquals(Overlaps.InclusionMethod.SINGLE, controller.overlapInclusionMethod());
+        assertNull(controller.getCalculator());
+        if (!options.containsKey(Controller.NUM_SHARDS_OPTION))
+        {
+            assertEquals(1, controller.getNumShards(0));
+            assertEquals(4, controller.getNumShards(16 * 100 << 20));
+            assertEquals(Overlaps.InclusionMethod.SINGLE, controller.overlapInclusionMethod());
+        }
+        else
+        {
+            int numShards = Integer.parseInt(options.get(Controller.NUM_SHARDS_OPTION));
+            long minSSTableSize = FBUtilities.parseHumanReadableBytes(options.get(Controller.MIN_SSTABLE_SIZE_OPTION));
+            assertEquals(1, controller.getNumShards(0));
+            assertEquals(numShards, controller.getNumShards(numShards * minSSTableSize));
+            assertEquals(numShards, controller.getNumShards(16 * 100 << 20));
+        }
 
         return controller;
     }
 
-    @Test
-    public void testValidateOptions()
+    Controller testFromOptionsVector(boolean adaptive, Map<String, String> options)
     {
-        testValidateOptions(false);
-    }
+        useVector = true;
+        addOptions(adaptive, options);
+        Controller.validateOptions(options);
 
-    @Test
-    public void testValidateOptionsIntegers()
-    {
-        testValidateOptions(true);
+        Controller controller = Controller.fromOptions(cfs, options);
+        assertNotNull(controller);
+        assertNotNull(controller.toString());
+
+        assertEquals(dataSizeGB << 30, controller.getDataSetSizeBytes());
+        assertFalse(controller.isRunning());
+        for (int i = 0; i < 5; i++) // simulate 5 levels
+            assertEquals(Controller.DEFAULT_SURVIVAL_FACTOR, controller.getSurvivalFactor(i), epsilon);
+        assertNull(controller.getCalculator());
+
+        return controller;
     }
 
-    void testValidateOptions(boolean useIntegers)
+    void testValidateOptions(Map<String, String> options, boolean adaptive)
     {
-        Map<String, String> options = new HashMap<>();
-        addOptions(useIntegers, options);
+        addOptions(adaptive, options);
         options = Controller.validateOptions(options);
         assertTrue(options.toString(), options.isEmpty());
     }
 
-    private static void addOptions(boolean useIntegers, Map<String, String> options)
+    private static void putWithAlt(Map<String, String> options, String opt, String alt, int altShift, long altVal)
     {
-        String wStr = Arrays.stream(Ws)
-                            .mapToObj(useIntegers ? Integer::toString : UnifiedCompactionStrategy::printScalingParameter)
-                            .collect(Collectors.joining(","));
-        options.putIfAbsent(Controller.SCALING_PARAMETERS_OPTION, wStr);
+        if (options.containsKey(opt) || options.containsKey(alt))
+            return;
+        if (ThreadLocalRandom.current().nextBoolean())
+            options.put(opt, FBUtilities.prettyPrintMemory(altVal << altShift));
+        else
+            options.put(alt, Long.toString(altVal));
+    }
+
+    private static void addOptions(boolean adaptive, Map<String, String> options)
+    {
+        options.putIfAbsent(Controller.ADAPTIVE_OPTION, Boolean.toString(adaptive));
+        putWithAlt(options, Controller.DATASET_SIZE_OPTION, Controller.DATASET_SIZE_OPTION_GB, 30, dataSizeGB);
+
+        if (ThreadLocalRandom.current().nextBoolean())
+            options.putIfAbsent(Controller.MAX_SPACE_OVERHEAD_OPTION, Double.toString(maxSpaceOverhead));
+        else
+            options.putIfAbsent(Controller.MAX_SPACE_OVERHEAD_OPTION, String.format("%.1f%%", maxSpaceOverhead * 100));
 
         options.putIfAbsent(Controller.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION, Boolean.toString(allowOverlaps));
         options.putIfAbsent(Controller.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION, Long.toString(checkFrequency));
 
-        options.putIfAbsent(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(2));
-        options.putIfAbsent(Controller.TARGET_SSTABLE_SIZE_OPTION, FBUtilities.prettyPrintMemory(100 << 20));
-        // The below value is based on the value in the above statement. Decreasing the above statement should result in a decrease below.
-        options.putIfAbsent(Controller.MIN_SSTABLE_SIZE_OPTION, "70.710MiB");
+        if (!options.containsKey(Controller.NUM_SHARDS_OPTION))
+        {
+            options.putIfAbsent(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(4));
+            options.putIfAbsent(Controller.TARGET_SSTABLE_SIZE_OPTION, FBUtilities.prettyPrintMemory(1 << 30));
+        }
         options.putIfAbsent(Controller.OVERLAP_INCLUSION_METHOD_OPTION, Overlaps.InclusionMethod.SINGLE.toString().toLowerCase());
-        options.putIfAbsent(Controller.SSTABLE_GROWTH_OPTION, "0.5");
+    }
+
+    void testStartShutdown(Controller controller)
+    {
+        assertNotNull(controller);
+
+        assertEquals((long) dataSizeGB << 30, controller.getDataSetSizeBytes());
+        assertEquals(numShards, controller.getNumShards(1));
+        assertEquals((long) sstableSizeMB << 20, controller.getTargetSSTableSize());
+        assertFalse(controller.isRunning());
+        assertEquals(Controller.DEFAULT_SURVIVAL_FACTOR, controller.getSurvivalFactor(0), epsilon);
+        assertNull(controller.getCalculator());
+
+        controller.startup(strategy, executorService);
+        assertTrue(controller.isRunning());
+        assertNotNull(controller.getCalculator());
+
+        controller.shutdown();
+        assertFalse(controller.isRunning());
+        assertNull(controller.getCalculator());
+
+        controller.shutdown(); // no op
+    }
+
+    void testShutdownNotStarted(Controller controller)
+    {
+        assertNotNull(controller);
+
+        controller.shutdown(); // no op.
+    }
+
+    void testStartAlreadyStarted(Controller controller)
+    {
+        assertNotNull(controller);
+
+        controller.startup(strategy, executorService);
+        assertTrue(controller.isRunning());
+        assertNotNull(controller.getCalculator());
+
+        controller.startup(strategy, executorService);
     }
 
     @Test
@@ -201,39 +329,6 @@ void testScalingParameterError(String definition)
         }
     }
 
-    @Test
-    public void testGetNumShards()
-    {
-        Map<String, String> options = new HashMap<>();
-        options.putIfAbsent(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(3));
-        options.putIfAbsent(Controller.TARGET_SSTABLE_SIZE_OPTION, FBUtilities.prettyPrintMemory(100 << 20));
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "0B");
-        options.put(Controller.SSTABLE_GROWTH_OPTION, "0.0");
-        Controller.validateOptions(options);
-        Controller controller = Controller.fromOptions(cfs, options);
-
-        // Easy ones
-        // x00 MiB = x * 100
-        assertEquals(6, controller.getNumShards(Math.scalb(600, 20)));
-        assertEquals(24, controller.getNumShards(Math.scalb(2400, 20)));
-        assertEquals(6 * 1024, controller.getNumShards(Math.scalb(600, 30)));
-        // Check rounding
-        assertEquals(6, controller.getNumShards(Math.scalb(800, 20)));
-        assertEquals(12, controller.getNumShards(Math.scalb(900, 20)));
-        assertEquals(6 * 1024, controller.getNumShards(Math.scalb(800, 30)));
-        assertEquals(12 * 1024, controller.getNumShards(Math.scalb(900, 30)));
-        // Check lower limit
-        assertEquals(3, controller.getNumShards(Math.scalb(200, 20)));
-        assertEquals(3, controller.getNumShards(Math.scalb(100, 20)));
-        assertEquals(3, controller.getNumShards(Math.scalb(10, 20)));
-        assertEquals(3, controller.getNumShards(5));
-        assertEquals(3, controller.getNumShards(0));
-        // Check upper limit
-        assertEquals(3 * (int) Controller.MAX_SHARD_SPLIT, controller.getNumShards(Math.scalb(600, 40)));
-        assertEquals(3 * (int) Controller.MAX_SHARD_SPLIT, controller.getNumShards(Math.scalb(10, 60)));
-        assertEquals(3 * (int) Controller.MAX_SHARD_SPLIT, controller.getNumShards(Double.POSITIVE_INFINITY));
-    }
-
     @Test
     public void testGetNumShards_growth_0()
     {
@@ -241,7 +336,7 @@ public void testGetNumShards_growth_0()
         options.put(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(3));
         options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "100MiB");
         options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "10MiB");
-        options.put(Controller.SSTABLE_GROWTH_OPTION, "0.0");
+        options.put(Controller.SSTABLE_GROWTH_OPTION, "0");
         Controller controller = Controller.fromOptions(cfs, options);
         assertEquals(0.0, controller.sstableGrowthModifier, 0.0);
 
@@ -283,6 +378,7 @@ public void testGetNumShards_growth_1()
         options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "10MiB");
         options.put(Controller.SSTABLE_GROWTH_OPTION, "1.0");
         Controller controller = Controller.fromOptions(cfs, options);
+        assertEquals(1.0, controller.sstableGrowthModifier, 0.0);
 
         // Easy ones
         // x00 MiB = x * 100
@@ -313,6 +409,80 @@ public void testGetNumShards_growth_1()
         assertEquals(1, controller.getNumShards(Double.NaN));
     }
 
+    @Test
+    public void testGetNumShards_legacy()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(Controller.NUM_SHARDS_OPTION, Integer.toString(3));
+        mockFlushSize(100);
+        Controller controller = Controller.fromOptions(cfs, options);
+
+        // Easy ones
+        // x00 MiB = x * 100
+        assertEquals(3, controller.getNumShards(Math.scalb(600, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(2400, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(600, 30)));
+        // Check rounding
+        assertEquals(3, controller.getNumShards(Math.scalb(800, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(900, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(800, 30)));
+        assertEquals(3, controller.getNumShards(Math.scalb(900, 30)));
+        // Check lower limit
+        assertEquals(3, controller.getNumShards(Math.scalb(600, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(500, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(400, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(300, 20)));
+        // Check min size
+        assertEquals(1, controller.getNumShards(Math.scalb(290, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(200, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(190, 20)));
+        assertEquals(1, controller.getNumShards(5));
+        assertEquals(1, controller.getNumShards(0));
+        // Check upper limit
+        assertEquals(3, controller.getNumShards(Math.scalb(600, 40)));
+        assertEquals(3, controller.getNumShards(Math.scalb(10, 60)));
+        assertEquals(3, controller.getNumShards(Double.POSITIVE_INFINITY));
+        // Check NaN
+        assertEquals(1, controller.getNumShards(Double.NaN));
+
+        assertEquals(Integer.MAX_VALUE, controller.getReservedThreads());
+    }
+
+    @Test
+    public void testGetNumShards_legacy_disabled()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(Controller.NUM_SHARDS_OPTION, Integer.toString(-1));
+        mockFlushSize(100);
+        Controller controller = Controller.fromOptions(cfs, options);
+
+        // The number of shards grows with local density, the controller works as if number of shards was not defined
+        assertEquals(2, controller.getNumShards(Math.scalb(200, 20)));
+        assertEquals(4, controller.getNumShards(Math.scalb(200, 25)));
+    }
+
+    @Test
+    public void testGetNumShards_legacy_validation()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(Controller.NUM_SHARDS_OPTION, Integer.toString(-1));
+        Map<String, String> validatedOptions = Controller.validateOptions(options);
+        assertTrue("-1 should be a valid option: " + validatedOptions, validatedOptions.isEmpty());
+
+        options = new HashMap<>();
+        options.put(Controller.NUM_SHARDS_OPTION, Integer.toString(-1));
+        options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "128MB");
+        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "0B");
+        validatedOptions = Controller.validateOptions(options);
+        assertTrue("-1 num of shards should be acceptable with V2 params: " + validatedOptions, validatedOptions.isEmpty());
+
+        Map<String, String> invalidOptions = new HashMap<>();
+        invalidOptions.put(Controller.NUM_SHARDS_OPTION, Integer.toString(32));
+        invalidOptions.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "128MB");
+        assertThrows("Positive num of shards should not be acceptable with V2 params",
+                     ConfigurationException.class, () -> Controller.validateOptions(invalidOptions));
+    }
+
     @Test
     public void testGetNumShards_growth_1_2()
     {
@@ -322,6 +492,7 @@ public void testGetNumShards_growth_1_2()
         options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "10MiB");
         options.put(Controller.SSTABLE_GROWTH_OPTION, "0.5");
         Controller controller = Controller.fromOptions(cfs, options);
+        assertEquals(0.5, controller.sstableGrowthModifier, 0.0);
 
         // Easy ones
         // x00 MiB = x * 3 * 100
@@ -391,202 +562,126 @@ public void testGetNumShards_growth_1_3()
     }
 
     @Test
-    public void testGetNumShards_minSize_10MiB_b_3()
+    public void testMinSizeAuto()
     {
         Map<String, String> options = new HashMap<>();
         options.put(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(3));
-        options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "100MiB");
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "10MiB");
-        options.put(Controller.SSTABLE_GROWTH_OPTION, "0.333");
+        options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "200MiB");
+        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "auto");
+//        options.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+        mockFlushSize(45); // rounds up to 50MiB
         Controller controller = Controller.fromOptions(cfs, options);
-        // Check min size
-        assertEquals(1, controller.getNumShards(Math.scalb(29, 20)));
-        assertEquals(1, controller.getNumShards(Math.scalb(20, 20)));
-        assertEquals(1, controller.getNumShards(Math.scalb(19, 20)));
-        assertEquals(1, controller.getNumShards(5));
-        assertEquals(1, controller.getNumShards(0));
-    }
 
-    @Test
-    public void testGetNumShards_minSize_10MiB_b_20()
-    {
-        Map<String, String> options = new HashMap<>();
-        options.put(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(20));
-        options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "100MiB");
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "10MiB");
-        options.put(Controller.SSTABLE_GROWTH_OPTION, "0.333");
-        Controller controller = Controller.fromOptions(cfs, options);
         // Check min size
-        assertEquals(2, controller.getNumShards(Math.scalb(29, 20)));
-        assertEquals(2, controller.getNumShards(Math.scalb(20, 20)));
-        assertEquals(1, controller.getNumShards(Math.scalb(19, 20)));
-        assertEquals(1, controller.getNumShards(5));
-        assertEquals(1, controller.getNumShards(0));
+        assertEquals(1, controller.getNumShards(Math.scalb(149, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(100, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(99, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(50, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(49, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(10, 20)));
+
+        // sanity check
+        assertEquals(3, controller.getNumShards(Math.scalb(600, 20)));
+        assertEquals(6, controller.getNumShards(Math.scalb(2400, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(400, 20)));
+        assertEquals(3, controller.getNumShards(Math.scalb(200, 20)));
     }
 
-    @Test
-    public void testGetNumShards_minSize_10MiB_b_8()
+    private void mockFlushSize(double d)
     {
-        Map<String, String> options = new HashMap<>();
-        options.put(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(8));
-        options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "100MiB");
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "10MiB");
-        options.put(Controller.SSTABLE_GROWTH_OPTION, "0.333");
-        Controller controller = Controller.fromOptions(cfs, options);
-        // Check min size
-        assertEquals(2, controller.getNumShards(Math.scalb(29, 20)));
-        assertEquals(2, controller.getNumShards(Math.scalb(20, 20)));
-        assertEquals(1, controller.getNumShards(Math.scalb(19, 20)));
-        assertEquals(1, controller.getNumShards(5));
-        assertEquals(1, controller.getNumShards(0));
+        TableMetrics metrics = Mockito.mock(TableMetrics.class);
+        MovingAverage flushSize = Mockito.mock(MovingAverage.class);
+        when(cfs.metrics()).thenReturn(metrics);
+        when(metrics.flushSizeOnDisk()).thenReturn(flushSize);
+        when(flushSize.get()).thenReturn(Math.scalb(d, 20)); // rounds up to 50MiB
     }
 
     @Test
-    public void testGetNumShards_minSize_3MiB_b_20()
+    public void testMinSizeAutoAtMostTargetMin()
     {
         Map<String, String> options = new HashMap<>();
-        options.put(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(20));
-        options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "100MiB");
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "3MiB");
-        options.put(Controller.SSTABLE_GROWTH_OPTION, "0.333");
+        options.put(Controller.BASE_SHARD_COUNT_OPTION, Integer.toString(3));
+        options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, "200MiB");
+        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "Auto");
+        mockFlushSize(300); // above target min, set to 141MiB
         Controller controller = Controller.fromOptions(cfs, options);
+
         // Check min size
-        assertEquals(4, controller.getNumShards(Math.scalb(29, 20)));
-        assertEquals(4, controller.getNumShards(Math.scalb(20, 20)));
-        assertEquals(4, controller.getNumShards(Math.scalb(19, 20)));
-        assertEquals(1, controller.getNumShards(5));
-        assertEquals(1, controller.getNumShards(0));
+        assertEquals(1, controller.getNumShards(Math.scalb(400, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(300, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(200, 20)));
+        assertEquals(1, controller.getNumShards(Math.scalb(100, 20)));
+        // sanity check
+        assertEquals(3, controller.getNumShards(Math.scalb(600, 20)));
+        assertEquals(6, controller.getNumShards(Math.scalb(2400, 20)));
     }
 
-    static final int[] Ws = new int[] { 30, 2, 0, -6};
-
     @Test
-    public void testFromOptions()
+    public void testParallelizeOutputShards()
     {
-        Map<String, String> options = new HashMap<>();
-        addOptions(false, options);
-
-        Controller controller = testFromOptions(options);
-
-        for (int i = 0; i < Ws.length; i++)
-            assertEquals(Ws[i], controller.getScalingParameter(i));
-
-        assertEquals(Ws[Ws.length-1], controller.getScalingParameter(Ws.length));
+        testBooleanOption(Controller.PARALLELIZE_OUTPUT_SHARDS_OPTION, Controller.DEFAULT_PARALLELIZE_OUTPUT_SHARDS, Controller::parallelizeOutputShards);
     }
 
-    @Test
-    public void testFromOptionsIntegers()
+    public void testBooleanOption(String name, boolean defaultValue, Predicate<Controller> getter, String... extraSettings)
     {
-        Map<String, String> options = new HashMap<>();
-        addOptions(true, options);
-
-        Controller controller = testFromOptions(options);
-
-        for (int i = 0; i < Ws.length; i++)
-            assertEquals(Ws[i], controller.getScalingParameter(i));
-
-        assertEquals(Ws[Ws.length-1], controller.getScalingParameter(Ws.length));
+        Controller controller = Controller.fromOptions(cfs, newOptions(extraSettings));
+        assertEquals(defaultValue, getter.test(controller));
+        for (boolean b : new boolean[] { true, false })
+        {
+            Map<String, String> options = newOptions(extraSettings);
+            options.put(name, Boolean.toString(b));
+            controller = Controller.fromOptions(cfs, options);
+            assertEquals(b, getter.test(controller));
+        }
     }
 
-    @Test
-    public void testMaxSSTablesToCompact()
+    private HashMap<String, String> newOptions(String... settings)
     {
-        Map<String, String> options = new HashMap<>();
-        Controller controller = testFromOptions(options);
-        assertTrue(controller.maxSSTablesToCompact == Integer.MAX_VALUE);
-
-        options.put(Controller.MAX_SSTABLES_TO_COMPACT_OPTION, "100");
-        controller = testFromOptions(options);
-        assertEquals(100, controller.maxSSTablesToCompact);
+        HashMap<String, String> options = new HashMap<>();
+        for (int i = 0; i < settings.length; i += 2)
+            options.put(settings[i], settings[i + 1]);
+        return options;
     }
 
-    @Test
-    public void testExpiredSSTableCheckFrequency()
+    void testValidateCompactionStrategyOptions(boolean testLogType)
     {
         Map<String, String> options = new HashMap<>();
-
-        Controller controller = testFromOptions(options);
-        assertEquals(TimeUnit.MILLISECONDS.convert(Controller.DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS, TimeUnit.SECONDS),
-                     controller.getExpiredSSTableCheckFrequency());
-
-        options.put(Controller.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION, "5");
-        controller = testFromOptions(options);
-        assertEquals(5000L, controller.getExpiredSSTableCheckFrequency());
-
-        try
+        options.put(CompactionStrategyOptions.TOMBSTONE_THRESHOLD_OPTION, Float.toString(tombstoneThresholdOption));
+        options.put(CompactionStrategyOptions.TOMBSTONE_COMPACTION_INTERVAL_OPTION, Long.toString(tombstoneCompactionIntervalOption));
+        options.put(CompactionStrategyOptions.UNCHECKED_TOMBSTONE_COMPACTION_OPTION, Boolean.toString(uncheckedTombstoneCompactionOption));
+
+        if (testLogType)
+            options.put(CompactionStrategyOptions.LOG_TYPE_OPTION, logTypeOption);
+        else
+            options.put(CompactionStrategyOptions.LOG_ALL_OPTION, Boolean.toString(logAllOption));
+
+        options.put(CompactionStrategyOptions.LOG_PERIOD_MINUTES_OPTION, Integer.toString(logPeriodMinutesOption));
+        options.put(CompactionStrategyOptions.COMPACTION_ENABLED, Boolean.toString(compactionEnabled));
+        options.put(CompactionStrategyOptions.READ_MULTIPLIER_OPTION, Double.toString(readMultiplier));
+        options.put(CompactionStrategyOptions.WRITE_MULTIPLIER_OPTION, Double.toString(writeMultiplier));
+
+        CompactionStrategyOptions compactionStrategyOptions = new CompactionStrategyOptions(UnifiedCompactionStrategy.class, options, true);
+        assertNotNull(compactionStrategyOptions);
+        assertNotNull(compactionStrategyOptions.toString());
+        assertEquals(tombstoneThresholdOption, compactionStrategyOptions.getTombstoneThreshold(), epsilon);
+        assertEquals(tombstoneCompactionIntervalOption, compactionStrategyOptions.getTombstoneCompactionInterval());
+        assertEquals(uncheckedTombstoneCompactionOption, compactionStrategyOptions.isUncheckedTombstoneCompaction());
+
+        if (testLogType)
         {
-            options.put(Controller.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION, "0");
-            testFromOptions(options);
-            fail("Exception should be thrown");
+            assertEquals((logTypeOption.equals("all") || logTypeOption.equals("events_only")), compactionStrategyOptions.isLogEnabled());
+            assertEquals(logTypeOption.equals("all"), compactionStrategyOptions.isLogAll());
         }
-        catch (ConfigurationException e)
+        else
         {
-            // valid path
+            assertEquals(logAllOption, compactionStrategyOptions.isLogEnabled());
+            assertEquals(logAllOption, compactionStrategyOptions.isLogAll());
         }
-    }
-
-    @Test
-    public void testAllowOverlaps()
-    {
-        Map<String, String> options = new HashMap<>();
-
-        Controller controller = testFromOptions(options);
-        assertEquals(Controller.DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION, controller.getIgnoreOverlapsInExpirationCheck());
-
-        options.put(Controller.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION, "true");
-        controller = testFromOptions(options);
-        assertEquals(Controller.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION, controller.getIgnoreOverlapsInExpirationCheck());
-    }
-
-    @Test
-    public void testBaseShardCountDefault()
-    {
-        Map<String, String> options = new HashMap<>();
-        Controller controller = Controller.fromOptions(cfs, options);
-        assertEquals(Controller.DEFAULT_BASE_SHARD_COUNT, controller.baseShardCount);
-
-        PartitionPosition min = Util.testPartitioner().getMinimumToken().minKeyBound();
-        diskBoundaries = new DiskBoundaries(cfs, null, ImmutableList.of(min, min, min), 0, 0);
-        controller = Controller.fromOptions(cfs, options);
-        assertEquals(4, controller.baseShardCount);
-
-        diskBoundaries = new DiskBoundaries(cfs, null, ImmutableList.of(min), 0, 0);
-        controller = Controller.fromOptions(cfs, options);
-        assertEquals(Controller.DEFAULT_BASE_SHARD_COUNT, controller.baseShardCount);
-    }
-
-    @Test
-    public void testMinSSTableSize()
-    {
-        Map<String, String> options = new HashMap<>();
-
-        // verify 0 is acceptable
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, format("%sB", 0));
-        Controller.validateOptions(options);
+        assertEquals(logPeriodMinutesOption, compactionStrategyOptions.getLogPeriodMinutes());
+        assertEquals(readMultiplier, compactionStrategyOptions.getReadMultiplier(), epsilon);
+        assertEquals(writeMultiplier, compactionStrategyOptions.getWriteMultiplier(), epsilon);
 
-        // test min < 0 failes
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "-1B");
-        assertThatExceptionOfType(ConfigurationException.class)
-        .describedAs("Should have thrown a ConfigurationException when min_sstable_size is less than 0")
-        .isThrownBy(() -> Controller.validateOptions(options))
-        .withMessageContaining("greater than or equal to 0");
-
-        // test min < default target sstable size * INV_SQRT_2
-        int limit = (int) Math.ceil(Controller.DEFAULT_TARGET_SSTABLE_SIZE * Controller.INVERSE_SQRT_2);
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, format("%sB", limit + 1));
-        assertThatExceptionOfType(ConfigurationException.class)
-        .describedAs("Should have thrown a ConfigurationException when min_sstable_size is greater than target_sstable_size")
-        .isThrownBy(() -> Controller.validateOptions(options))
-        .withMessageContaining(format("less than the target size minimum: %s", FBUtilities.prettyPrintMemory(limit)));
-
-        // test min < configured target table size * INV_SQRT_2
-        limit = (int) Math.ceil(Controller.MIN_TARGET_SSTABLE_SIZE * 2 * Controller.INVERSE_SQRT_2);
-        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, format("%sB", limit + 1));
-        options.put(Controller.TARGET_SSTABLE_SIZE_OPTION, format("%sB", Controller.MIN_TARGET_SSTABLE_SIZE * 2));
-
-        assertThatExceptionOfType(ConfigurationException.class)
-        .describedAs("Should have thrown a ConfigurationException when min_sstable_size is greater than target_sstable_size")
-        .isThrownBy(() -> Controller.validateOptions(options))
-        .withMessageContaining(format("less than the target size minimum: %s", FBUtilities.prettyPrintMemory(limit)));
+        Map<String, String> uncheckedOptions = CompactionStrategyOptions.validateOptions(options);
+        assertNotNull(uncheckedOptions);
     }
 }
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/CostsCalculatorTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/CostsCalculatorTest.java
new file mode 100644
index 000000000000..7e26a708f90b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/CostsCalculatorTest.java
@@ -0,0 +1,288 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.Random;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.db.compaction.BackgroundCompactions;
+import org.apache.cassandra.db.compaction.CompactionStrategyOptions;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FixedMonotonicClock;
+import org.apache.cassandra.utils.MovingAverage;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+public class CostsCalculatorTest
+{
+    private static final double epsilon = 0.00000001;
+    private static final Random random = new Random(0L);
+
+    @Mock
+    private Environment environment;
+
+    @Mock
+    private UnifiedCompactionStrategy strategy;
+
+    @Mock
+    private CompactionStrategyOptions options;
+
+    @Mock
+    private TableMetadata metadata;
+
+    @Mock
+    private BackgroundCompactions backgroundCompactions;
+
+    @Mock
+    private ScheduledExecutorService executorService;
+
+    @Mock
+    private ScheduledFuture fut;
+
+    @Mock
+    private SSTableReader sstable;
+
+    private FixedMonotonicClock clock;
+
+    @Before
+    public void setUp()
+    {
+        MockitoAnnotations.initMocks(this);
+
+        when(executorService.scheduleAtFixedRate(any(Runnable.class), anyLong(), anyLong(), any(TimeUnit.class))).thenReturn(fut);
+        when(strategy.getSSTables()).thenAnswer(invocation -> Sets.newHashSet(sstable));
+        when(strategy.getMetadata()).thenReturn(metadata);
+
+        when(sstable.onDiskLength()).thenReturn((long) PageAware.PAGE_SIZE);
+        when(backgroundCompactions.getAggregates()).thenReturn(ImmutableList.of());
+
+        clock = new FixedMonotonicClock();
+        when(environment.makeExpMovAverage()).thenAnswer(ts -> new MovingAverageMock());
+    }
+
+    @Test
+    public void testCreateAndClose()
+    {
+        CostsCalculator cost = new CostsCalculator(environment, strategy, executorService);
+        assertNotNull(cost);
+        assertNotNull(cost.toString());
+
+        Mockito.verify(executorService, times(1)).scheduleAtFixedRate(any(Runnable.class), anyLong(), anyLong(), any(TimeUnit.class));
+
+        cost.close();
+        Mockito.verify(fut, times(1)).cancel(anyBoolean());
+    }
+
+    @Test
+    public void testUpdate() throws InterruptedException
+    {
+        testCosts(100, 100, PageAware.PAGE_SIZE, 1, 1);
+    }
+
+    @Test
+    public void testDoubleReadTime() throws InterruptedException
+    {
+        testCosts(200, 100, PageAware.PAGE_SIZE, 1, 1);
+    }
+
+    @Test
+    public void testDoubleWriteTime() throws InterruptedException
+    {
+        testCosts(100, 200, PageAware.PAGE_SIZE, 1, 1);
+    }
+
+    @Test
+    public void testLargerChunkSize() throws InterruptedException
+    {
+        testCosts(100, 100, 64 << 10, 1, 1);
+    }
+
+    @Test
+    public void testHalfCacheMissRatio() throws InterruptedException
+    {
+        testCosts(100, 100, PageAware.PAGE_SIZE, 1, 1);
+    }
+
+    @Test
+    public void testReadMultiplier() throws InterruptedException
+    {
+        testCosts(1000, 100, PageAware.PAGE_SIZE, 0.1, 1);
+    }
+
+    @Test
+    public void testWriteMultiplier() throws InterruptedException
+    {
+        testCosts(100, 100, PageAware.PAGE_SIZE, 1, 10);
+    }
+
+    private void testCosts(long readTimeMicros,
+                           long writeTimeMicros,
+                           int chunkSize,
+                           double readMultiplier,
+                           double writeMultiplier) throws InterruptedException
+    {
+        int blockSize = PageAware.PAGE_SIZE;
+        long totPartitionsRead = 1 + random.nextInt(32);
+        long totBytesInserted = blockSize + random(blockSize);
+
+        when(environment.partitionsRead()).thenReturn(totPartitionsRead);
+        when(environment.bytesInserted()).thenReturn(totBytesInserted);
+        when(environment.chunkSize()).thenReturn(chunkSize);
+        when(environment.sstablePartitionReadLatencyNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(readTimeMicros));
+        when(environment.flushTimePerKbInNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(writeTimeMicros));
+        when(environment.compactionTimePerKbInNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(writeTimeMicros));
+        when(strategy.getOptions()).thenReturn(options);
+        when(options.getReadMultiplier()).thenReturn(readMultiplier);
+        when(options.getWriteMultiplier()).thenReturn(writeMultiplier);
+
+        CostsCalculator cost = new CostsCalculator(environment, strategy, executorService);
+        assertNotNull(cost);
+        assertNotNull(cost.toString());
+
+        cost.sampleValues();
+        assertNotNull(cost.toString());
+
+        for (int i = 0; i < 32; i++)
+        {
+            long bytesInserted = (i * blockSize + random(blockSize));
+            totPartitionsRead += (1 + i);
+            totBytesInserted += bytesInserted;
+
+            when(environment.partitionsRead()).thenReturn(totPartitionsRead);
+            when(environment.bytesInserted()).thenReturn(totBytesInserted);
+
+            clock.setNowInNanos(clock.now() + TimeUnit.MILLISECONDS.toNanos(CostsCalculator.samplingPeriodMs));
+            cost.sampleValues();
+            assertNotNull(cost.toString());
+
+            // the WA is 2 and the flush and compaction times for now are the same and equal to writeTimeMicros
+            double writeCost = ((bytesInserted / (double) (1 << 10)) * TimeUnit.MICROSECONDS.toNanos(writeTimeMicros)) / (double) TimeUnit.MILLISECONDS.toNanos(1);
+            assertEquals((writeCost + writeCost * 2) * writeMultiplier, cost.getWriteCostForQueries(2), epsilon);
+
+            // the RA is 2, the delta partitions read is i + 1
+            assertEquals((((i + 1) * readTimeMicros) / (double) TimeUnit.MILLISECONDS.toMicros(1)) * 2 * readMultiplier, cost.getReadCostForQueries(2), epsilon);
+        }
+    }
+
+    @Test
+    public void testNoBytesInserted()
+    {
+        int blockSize = PageAware.PAGE_SIZE;
+        long totPartitionsRead = 1 + random.nextInt(32);
+        long totBytesInserted = blockSize + random(blockSize);
+
+        when(environment.partitionsRead()).thenReturn(totPartitionsRead);
+        when(environment.bytesInserted()).thenReturn(totBytesInserted);
+        when(environment.chunkSize()).thenReturn(4096);
+        when(environment.cacheMissRatio()).thenReturn(0.05);
+        when(environment.bloomFilterFpRatio()).thenReturn(0.01);
+        when(environment.sstablePartitionReadLatencyNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(20));
+        when(environment.flushTimePerKbInNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(20));
+        when(environment.compactionTimePerKbInNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(20));
+        when(strategy.getOptions()).thenReturn(options);
+        when(options.getWriteMultiplier()).thenReturn(1.0);
+
+        CostsCalculator cost = new CostsCalculator(environment, strategy, executorService);
+        assertNotNull(cost);
+        assertNotNull(cost.toString());
+
+        cost.sampleValues();
+        assertNotNull(cost.toString());
+
+        when(environment.bytesInserted()).thenReturn(0L);
+        for (int i = 0; i < 10; i++)
+            assertEquals(0, cost.getWriteCostForQueries(i), epsilon);
+    }
+
+    @Test
+    public void testNoPartitionsRead()
+    {
+        int blockSize = PageAware.PAGE_SIZE;
+        long totPartitionsRead = 1 + random.nextInt(32);
+        long totBytesInserted = blockSize + random(blockSize);
+
+        when(environment.partitionsRead()).thenReturn(totPartitionsRead);
+        when(environment.bytesInserted()).thenReturn(totBytesInserted);
+        when(environment.chunkSize()).thenReturn(4096);
+        when(environment.cacheMissRatio()).thenReturn(0.05);
+        when(environment.bloomFilterFpRatio()).thenReturn(0.01);
+        when(environment.sstablePartitionReadLatencyNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(20));
+        when(environment.flushTimePerKbInNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(20));
+        when(environment.compactionTimePerKbInNanos()).thenReturn((double) TimeUnit.MICROSECONDS.toNanos(20));
+        when(strategy.getOptions()).thenReturn(options);
+        when(options.getReadMultiplier()).thenReturn(0.5);
+
+        CostsCalculator cost = new CostsCalculator(environment, strategy, executorService);
+        assertNotNull(cost);
+        assertNotNull(cost.toString());
+
+        cost.sampleValues();
+        assertNotNull(cost.toString());
+
+        when(environment.partitionsRead()).thenReturn(0L);
+        for (int i = 0; i < 10; i++)
+            assertEquals(0, cost.getReadCostForQueries(i), epsilon);
+    }
+
+    private static long random(int blockSize)
+    {
+        return 1 + random.nextInt(blockSize - 1);
+    }
+
+    private static class MovingAverageMock implements MovingAverage
+    {
+        private double val = 0;
+
+        @Override
+        public MovingAverage update(double val)
+        {
+            this.val = val;
+            return this;
+        }
+
+        @Override
+        public double get()
+        {
+            return val;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%.02f", val);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/ParallelizedTasksTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/ParallelizedTasksTest.java
new file mode 100644
index 000000000000..9db77e0e63f8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/ParallelizedTasksTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SortedLocalRanges;
+import org.apache.cassandra.db.compaction.AbstractCompactionTask;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionStrategyFactory;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.ShardManager;
+import org.apache.cassandra.db.compaction.ShardManagerNoDisks;
+import org.apache.cassandra.db.compaction.SharedCompactionObserver;
+import org.apache.cassandra.db.compaction.SharedCompactionProgress;
+import org.apache.cassandra.db.compaction.SharedTableOperation;
+import org.apache.cassandra.db.compaction.TableOperation;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.db.lifecycle.CompositeLifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.PartialLifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.Transactional;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ParallelizedTasksTest extends ShardingTestBase
+{
+    @Test
+    public void testOneSSTablePerShardIterators() throws Throwable
+    {
+        int numShards = 5;
+        testParallelized(numShards, PARTITIONS, numShards, true, false);
+    }
+
+    @Test
+    public void testMultipleInputSSTablesIterators() throws Throwable
+    {
+        int numShards = 3;
+        testParallelized(numShards, PARTITIONS, numShards, false, false);
+    }
+
+    @Test
+    public void testOneSSTablePerShardCursors() throws Throwable
+    {
+        int numShards = 5;
+        testParallelized(numShards, PARTITIONS, numShards, true, true);
+    }
+
+    @Test
+    public void testMultipleInputSSTablesCursors() throws Throwable
+    {
+        int numShards = 3;
+        testParallelized(numShards, PARTITIONS, numShards, false, true);
+    }
+
+    private void testParallelized(int numShards, int rowCount, int numOutputSSTables, boolean compact, boolean useCursors) throws Throwable
+    {
+        CassandraRelevantProperties.ALLOW_CURSOR_COMPACTION.setBoolean(useCursors);
+        ColumnFamilyStore cfs = getColumnFamilyStore();
+        cfs.disableAutoCompaction();
+
+        populate(rowCount, compact);
+
+        LifecycleTransaction transaction = cfs.getTracker().tryModify(cfs.getLiveSSTables(), OperationType.COMPACTION, UnifiedCompactionStrategy.nextTimeUUID());
+
+        ShardManager shardManager = new ShardManagerNoDisks(SortedLocalRanges.forTestingFull(cfs));
+
+        Controller mockController = Mockito.mock(Controller.class);
+        Mockito.when(mockController.getNumShards(Mockito.anyDouble())).thenReturn(numShards);
+
+        Collection<SSTableReader> sstables = transaction.originals();
+        CompositeLifecycleTransaction compositeTransaction = new CompositeLifecycleTransaction(transaction);
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(new CompactionStrategyFactory(cfs), mockController);
+        UnifiedCompactionStrategy mockStrategy = strategy;
+        strategy.getCompactionLogger().enable();
+        SharedCompactionProgress sharedProgress = new SharedCompactionProgress(transaction.opId(), transaction.opType(), TableOperation.Unit.BYTES);
+        SharedCompactionObserver sharedObserver = new SharedCompactionObserver(strategy);
+        SharedTableOperation sharedOperation = new SharedTableOperation(sharedProgress);
+
+        List<AbstractCompactionTask> tasks = shardManager.splitSSTablesInShards(
+            sstables,
+            numShards,
+            (rangeSSTables, range) ->
+            new UnifiedCompactionTask(cfs,
+                                      mockStrategy,
+                                      new PartialLifecycleTransaction(compositeTransaction),
+                                      0,
+                                      false,
+                                      shardManager,
+                                      new UnifiedCompactionStrategy.ShardingStats(rangeSSTables, shardManager, mockController),
+                                      range,
+                                      rangeSSTables,
+                                      sharedProgress,
+                                      sharedObserver,
+                                      sharedOperation)
+        );
+        compositeTransaction.completeInitialization();
+        assertEquals(numOutputSSTables, tasks.size());
+
+        List<Future<?>> futures = tasks.stream()
+                                       .map(t -> ForkJoinPool.commonPool()
+                                                             .submit(() -> {
+                                                                 t.execute(CompactionManager.instance.active);
+                                                             }))
+                                       .collect(Collectors.toList());
+
+        FBUtilities.waitOnFutures(futures);
+        assertTrue(transaction.state() == Transactional.AbstractTransactional.State.COMMITTED);
+
+        verifySharding(numShards, rowCount, numOutputSSTables, cfs);
+        cfs.truncateBlocking();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/RangedAggregatesTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/RangedAggregatesTest.java
new file mode 100644
index 000000000000..4c0df8392903
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/RangedAggregatesTest.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.AbstractCompactionTask;
+import org.apache.cassandra.db.compaction.CompactionAggregate;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionStrategyFactory;
+import org.apache.cassandra.db.compaction.ShardManager;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.utils.FBUtilities;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+
+/// This tests UnifiedWithRange aggregates that pre-split compactions into shards and is used by CNDB.
+public class RangedAggregatesTest extends ShardingTestBase
+{
+    @Test
+    public void testOneSSTablePerShard() throws Throwable
+    {
+        testRangedAggregates(6, 6, PARTITIONS, 6, true, true);
+    }
+
+    @Test
+    public void testOneSSTablePerShardFurtherSplit() throws Throwable
+    {
+        testRangedAggregates(6, 3, PARTITIONS, 6, true, true);
+    }
+
+    @Test
+    public void testOneSSTablePerShardOnlyFurtherSplit() throws Throwable
+    {
+        testRangedAggregates(6, 1, PARTITIONS, 6, true, true);
+    }
+
+    @Test
+    public void testMultipleInputSSTables() throws Throwable
+    {
+        testRangedAggregates(6, 6, PARTITIONS, 6, false, true);
+    }
+
+    @Test
+    public void testMultipleInputSSTablesFurtherSplit() throws Throwable
+    {
+        testRangedAggregates(6, 3, PARTITIONS, 6, false, true);
+    }
+
+    @Test
+    public void testMultipleInputSSTablesOnlyFurtherSplit() throws Throwable
+    {
+        testRangedAggregates(6, 1, PARTITIONS, 6, false, true);
+    }
+
+    private void testRangedAggregates(int numShards, int aggregateParallelism, int rowCount, int numOutputSSTables, boolean compact, boolean useCursors) throws Throwable
+    {
+        CassandraRelevantProperties.ALLOW_CURSOR_COMPACTION.setBoolean(useCursors);
+        ColumnFamilyStore cfs = getColumnFamilyStore();
+        cfs.disableAutoCompaction();
+
+        populate(rowCount, compact);
+        var originals = new HashSet<>(cfs.getLiveSSTables());
+
+        Controller mockController = Mockito.mock(Controller.class);
+        Mockito.when(mockController.getNumShards(Mockito.anyDouble())).thenReturn(numShards);
+        Mockito.when(mockController.parallelizeOutputShards()).thenReturn(true);
+
+        UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(new CompactionStrategyFactory(cfs), mockController);
+        ShardManager shardManager = strategy.getShardManager();
+        Collection<CompactionAggregate.UnifiedAggregate> maximals = strategy.getMaximalAggregates();
+        maximals = maximals.stream()
+                           .flatMap(agg ->
+                               shardManager.splitSSTablesInShardsLimited
+                               (
+                                   agg.getSelected().sstables(),
+                                   null,
+                                   numShards,
+                                   numShards,
+                                   aggregateParallelism,
+                                   (rangeSSTables, range) ->
+                                       CompactionAggregate.createUnifiedWithRange
+                                       (
+                                           agg,
+                                           rangeSSTables,
+                                           range,
+                                           numShards    // this should further split
+                                       )
+                               ).stream())
+                           .collect(Collectors.toList());
+
+        int totalTaskCount = 0;
+        for (CompactionAggregate.UnifiedAggregate maximal : maximals)
+        {
+            // execute each partial aggregate separately because we can only mark the inputs compacting once
+            List<AbstractCompactionTask> tasks = new ArrayList<>();
+            strategy.createAndAddTasks(0, maximal, tasks);
+            totalTaskCount += tasks.size();
+            List<Future<?>> futures = tasks.stream()
+                                           .map(t -> ForkJoinPool.commonPool()
+                                                                 .submit(() -> {
+                                                                     t.execute(CompactionManager.instance.active);
+                                                                 }))
+                                           .collect(Collectors.toList());
+            FBUtilities.waitOnFutures(futures);
+        }
+        assertEquals(numOutputSSTables, totalTaskCount);
+
+
+        // make sure the partial aggregates are not deleting sstables
+        Assert.assertTrue(cfs.getLiveSSTables().containsAll(originals));
+        cfs.getTracker().removeUnsafe(originals);
+
+        verifySharding(numShards, rowCount, numOutputSSTables, cfs);
+        cfs.truncateBlocking();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriterTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriterTest.java
index 9eb05dc50392..a7f478fcec7e 100644
--- a/test/unit/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriterTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriterTest.java
@@ -1,13 +1,11 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Copyright DataStax, Inc.
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,78 +16,33 @@
 
 package org.apache.cassandra.db.compaction.unified;
 
-import java.nio.ByteBuffer;
-import java.util.Collection;
 import java.util.List;
-import java.util.Random;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.compaction.CompactionController;
-import org.apache.cassandra.db.compaction.CompactionIterator;
+import org.apache.cassandra.db.SortedLocalRanges;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.compaction.ShardManager;
 import org.apache.cassandra.db.compaction.ShardManagerDiskAware;
 import org.apache.cassandra.db.compaction.ShardManagerNoDisks;
-import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.TimeUUID;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
-public class ShardedCompactionWriterTest extends CQLTester
+public class ShardedCompactionWriterTest extends ShardingTestBase
 {
-    private static final String KEYSPACE = "cawt_keyspace";
-    private static final String TABLE = "cawt_table";
-
-    private static final int ROW_PER_PARTITION = 10;
-
-    @BeforeClass
-    public static void beforeClass()
-    {
-        CQLTester.setUpClass();
-        CQLTester.prepareServer();
-        StorageService.instance.initServer();
-
-        // Disabling durable write since we don't care
-        schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes=false");
-        schemaChange(String.format("CREATE TABLE %s.%s (k int, t int, v blob, PRIMARY KEY (k, t))", KEYSPACE, TABLE));
-    }
-
-    @AfterClass
-    public static void tearDownClass()
-    {
-        QueryProcessor.executeInternal("DROP KEYSPACE IF EXISTS " + KEYSPACE);
-    }
-
-    private ColumnFamilyStore getColumnFamilyStore()
-    {
-        return Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
-    }
-
     @Test
     public void testOneSSTablePerShard() throws Throwable
     {
         // If we set the minSSTableSize ratio to 0.5, because this gets multiplied by the shard size to give the min sstable size,
         // assuming evenly distributed data, it should split at each boundary and so we should end up with numShards sstables
         int numShards = 5;
-        int rowCount = 5000;
-        testShardedCompactionWriter(numShards, rowCount, numShards, true);
+        testShardedCompactionWriter(numShards, PARTITIONS, numShards, true);
     }
 
 
@@ -97,10 +50,8 @@ public void testOneSSTablePerShard() throws Throwable
     public void testMultipleInputSSTables() throws Throwable
     {
         int numShards = 3;
-        int rowCount = 5000;
-        testShardedCompactionWriter(numShards, rowCount, numShards, false);
+        testShardedCompactionWriter(numShards, PARTITIONS, numShards, false);
     }
-
     private void testShardedCompactionWriter(int numShards, int rowCount, int numOutputSSTables, boolean majorCompaction) throws Throwable
     {
         ColumnFamilyStore cfs = getColumnFamilyStore();
@@ -110,34 +61,16 @@ private void testShardedCompactionWriter(int numShards, int rowCount, int numOut
 
         LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getLiveSSTables(), OperationType.COMPACTION);
 
-        ShardManager boundaries = new ShardManagerNoDisks(ColumnFamilyStore.fullWeightedRange(-1, cfs.getPartitioner()));
-        ShardedCompactionWriter writer = new ShardedCompactionWriter(cfs, cfs.getDirectories(), txn, txn.originals(), false, boundaries.boundaries(numShards));
+        ShardManager boundaries = new ShardManagerNoDisks(SortedLocalRanges.forTestingFull(cfs));
+        ShardedCompactionWriter writer = new ShardedCompactionWriter(cfs, cfs.getDirectories(), txn, txn.originals(), 1, false, true, boundaries.boundaries(numShards));
 
         int rows = compact(cfs, txn, writer);
-        assertEquals(numOutputSSTables, cfs.getLiveSSTables().size());
         assertEquals(rowCount, rows);
 
-        long totalOnDiskLength = cfs.getLiveSSTables().stream().mapToLong(SSTableReader::onDiskLength).sum();
-        long totalBFSize = cfs.getLiveSSTables().stream().mapToLong(ShardedCompactionWriterTest::getFilterSize).sum();
-        assert totalBFSize > 16 * numOutputSSTables : "Bloom Filter is empty"; // 16 is the size of empty bloom filter
-        for (SSTableReader rdr : cfs.getLiveSSTables())
-        {
-            assertEquals((double) rdr.onDiskLength() / totalOnDiskLength,
-                         (double) getFilterSize(rdr) / totalBFSize, 0.1);
-            assertEquals(1.0 / numOutputSSTables, rdr.tokenSpaceCoverage(), 0.05);
-        }
-
-        validateData(cfs, rowCount);
+        verifySharding(numShards, rowCount, numOutputSSTables, cfs);
         cfs.truncateBlocking();
     }
 
-    static long getFilterSize(SSTableReader rdr)
-    {
-        if (!(rdr instanceof SSTableReaderWithFilter))
-            return 0;
-        return ((SSTableReaderWithFilter) rdr).getFilterSerializedSize();
-    }
-
     @Test
     public void testDiskAdvance() throws Throwable
     {
@@ -149,8 +82,8 @@ public void testDiskAdvance() throws Throwable
 
         populate(rowCount, false);
 
-        final ColumnFamilyStore.VersionedLocalRanges localRanges = cfs.localRangesWeighted();
-        final List<Token> diskBoundaries = cfs.getPartitioner().splitter().get().splitOwnedRanges(numDisks, localRanges, false);
+        final SortedLocalRanges localRanges = SortedLocalRanges.forTestingFull(cfs);
+        final List<Token> diskBoundaries = localRanges.split(numDisks);
         ShardManager shardManager = new ShardManagerDiskAware(localRanges, diskBoundaries);
         int rows = compact(1, cfs, shardManager, cfs.getLiveSSTables());
 
@@ -168,19 +101,19 @@ public void testDiskAdvance() throws Throwable
         // is to create on-partition sstables at the start because shard wasn't advanced at the right time.
         Set<SSTableReader> liveSSTables = cfs.getLiveSSTables();
         List<SSTableReader> selection = liveSSTables.stream()
-                                                    .filter(rdr -> rdr.getFirst().getToken().compareTo(selectionStart) > 0 &&
-                                                                   rdr.getLast().getToken().compareTo(selectionEnd) <= 0)
-                                                    .collect(Collectors.toList());
+                .filter(rdr -> rdr.getFirst().getToken().compareTo(selectionStart) > 0 &&
+                        rdr.getLast().getToken().compareTo(selectionEnd) <= 0)
+                .collect(Collectors.toList());
         List<SSTableReader> remainder = liveSSTables.stream()
-                                                    .filter(rdr -> !selection.contains(rdr))
-                                                    .collect(Collectors.toList());
+                .filter(rdr -> !selection.contains(rdr))
+                .collect(Collectors.toList());
 
         rows = compact(numShards, cfs, shardManager, selection);
 
         List<SSTableReader> compactedSelection = cfs.getLiveSSTables()
-                                                    .stream()
-                                                    .filter(rdr -> !remainder.contains(rdr))
-                                                    .collect(Collectors.toList());
+                .stream()
+                .filter(rdr -> !remainder.contains(rdr))
+                .collect(Collectors.toList());
         // We must now have numShards sstables per each of the two disk sections
         assertEquals(numShards * 2, compactedSelection.size());
         assertEquals(rowCount * 2.0 / numDisks, rows * 1.0, rowCount / 20.0); // should end up with roughly this many rows
@@ -196,102 +129,12 @@ public void testDiskAdvance() throws Throwable
             verifyNoSpannedBoundaries(diskBoundaries, rdr);
 
             assertEquals((double) rdr.onDiskLength() / totalOnDiskLength,
-                         (double) getFilterSize(rdr) / totalBFSize, 0.1);
+                    (double) getFilterSize(rdr) / totalBFSize, 0.1);
             assertEquals(expectedTokenShare, rdr.tokenSpaceCoverage(), expectedTokenShare * 0.05);
             assertEquals(expectedSize, rdr.onDiskLength(), expectedSize * 0.1);
         }
 
-        validateData(cfs, rowCount);
+        validateData(rowCount);
         cfs.truncateBlocking();
     }
-
-    private int compact(int numShards, ColumnFamilyStore cfs, ShardManager shardManager, Collection<SSTableReader> selection)
-    {
-        int rows;
-        LifecycleTransaction txn = cfs.getTracker().tryModify(selection, OperationType.COMPACTION);
-        ShardedCompactionWriter writer = new ShardedCompactionWriter(cfs,
-                                                                     cfs.getDirectories(),
-                                                                     txn,
-                                                                     txn.originals(),
-                                                                     false,
-                                                                     shardManager.boundaries(numShards));
-
-        rows = compact(cfs, txn, writer);
-        return rows;
-    }
-
-    private static void verifyNoSpannedBoundaries(List<Token> diskBoundaries, SSTableReader rdr)
-    {
-        for (int i = 0; i < diskBoundaries.size(); ++i)
-        {
-            Token boundary = diskBoundaries.get(i);
-            // rdr cannot span a boundary. I.e. it must be either fully before (last <= boundary) or fully after
-            // (first > boundary).
-            assertTrue(rdr.getFirst().getToken().compareTo(boundary) > 0 ||
-                       rdr.getLast().getToken().compareTo(boundary) <= 0);
-        }
-    }
-
-    private int compact(ColumnFamilyStore cfs, LifecycleTransaction txn, CompactionAwareWriter writer)
-    {
-        //assert txn.originals().size() == 1;
-        int rowsWritten = 0;
-        long nowInSec = FBUtilities.nowInSeconds();
-        try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(txn.originals());
-             CompactionController controller = new CompactionController(cfs, txn.originals(), cfs.gcBefore(nowInSec));
-             CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, TimeUUID.minAtUnixMillis(System.currentTimeMillis())))
-        {
-            while (ci.hasNext())
-            {
-                if (writer.append(ci.next()))
-                    rowsWritten++;
-            }
-        }
-        writer.finish();
-        return rowsWritten;
-    }
-
-    private void populate(int count, boolean compact) throws Throwable
-    {
-        byte [] payload = new byte[5000];
-        new Random(42).nextBytes(payload);
-        ByteBuffer b = ByteBuffer.wrap(payload);
-
-        ColumnFamilyStore cfs = getColumnFamilyStore();
-        for (int i = 0; i < count; i++)
-        {
-            for (int j = 0; j < ROW_PER_PARTITION; j++)
-                execute(String.format("INSERT INTO %s.%s(k, t, v) VALUES (?, ?, ?)", KEYSPACE, TABLE), i, j, b);
-
-            if (i % (count / 4) == 0)
-                cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
-        }
-
-        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
-        if (compact && cfs.getLiveSSTables().size() > 1)
-        {
-            // we want just one big sstable to avoid doing actual compaction in compact() above
-            try
-            {
-                cfs.forceMajorCompaction();
-            }
-            catch (Throwable t)
-            {
-                throw new RuntimeException(t);
-            }
-            assert cfs.getLiveSSTables().size() == 1 : cfs.getLiveSSTables();
-        }
-    }
-
-    private void validateData(ColumnFamilyStore cfs, int rowCount) throws Throwable
-    {
-        for (int i = 0; i < rowCount; i++)
-        {
-            Object[][] expected = new Object[ROW_PER_PARTITION][];
-            for (int j = 0; j < ROW_PER_PARTITION; j++)
-                expected[j] = row(i, j);
-
-            assertRows(execute(String.format("SELECT k, t FROM %s.%s WHERE k = :i", KEYSPACE, TABLE), i), expected);
-        }
-    }
 }
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/ShardedMultiWriterTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/ShardedMultiWriterTest.java
index 5a9a0155f0c1..a8b961097579 100644
--- a/test/unit/org/apache/cassandra/db/compaction/unified/ShardedMultiWriterTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/ShardedMultiWriterTest.java
@@ -1,13 +1,11 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Copyright DataStax, Inc.
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -23,21 +21,38 @@
 
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.ShardManager;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.service.StorageService;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_L0_SHARDS_ENABLED;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 
+@RunWith(Parameterized.class)
 public class ShardedMultiWriterTest extends CQLTester
 {
     private static final int ROW_PER_PARTITION = 10;
 
+    @Parameterized.Parameter
+    public boolean isReplicaAware;
+
+    @Parameterized.Parameters(name = "isReplicaAware={0}")
+    public static Object[] parameters()
+    {
+        return new Object[] { true, false };
+    }
+
     @BeforeClass
     public static void beforeClass()
     {
+        UCS_L0_SHARDS_ENABLED.setBoolean(true);
         CQLTester.setUpClass();
         StorageService.instance.initServer();
     }
@@ -50,7 +65,7 @@ public void testShardedCompactionWriter_fiveShards() throws Throwable
         long totSizeBytes = ((minSSTableSizeMB << 20) * numShards) * 2;
 
         // We have double the data required for 5 shards so we should get 5 shards
-        testShardedCompactionWriter(numShards, totSizeBytes, numShards, minSSTableSizeMB);
+        testShardedCompactionWriter(numShards, totSizeBytes, numShards);
     }
 
     @Test
@@ -61,7 +76,7 @@ public void testShardedCompactionWriter_oneShard() throws Throwable
         long totSizeBytes = (minSSTableSizeMB << 20);
 
         // there should be only 1 shard if there is <= minSSTableSize
-        testShardedCompactionWriter(numShards, totSizeBytes, 1, minSSTableSizeMB);
+        testShardedCompactionWriter(numShards, totSizeBytes, 1);
     }
 
     @Test
@@ -72,25 +87,63 @@ public void testShardedCompactionWriter_threeShard() throws Throwable
         long totSizeBytes = (minSSTableSizeMB << 20) * 3;
 
         // there should be only 3 shards if there is minSSTableSize * 3 data
-        testShardedCompactionWriter(numShards, totSizeBytes, 3, minSSTableSizeMB);
+        testShardedCompactionWriter(numShards, totSizeBytes, 3);
     }
 
-    private void testShardedCompactionWriter(int numShards, long totSizeBytes, int numOutputSSTables, int minSSTableSizeMB) throws Throwable
+    private void testShardedCompactionWriter(int numShards, long totSizeBytes, int numOutputSSTables) throws Throwable
     {
         createTable(String.format("CREATE TABLE %%s (k int, t int, v blob, PRIMARY KEY (k, t)) with compaction = " +
-                                  "{'class':'UnifiedCompactionStrategy', 'base_shard_count' : '%d', 'min_sstable_size': '" + minSSTableSizeMB + "MiB'} ", numShards));
+                                  "{'class':'UnifiedCompactionStrategy', 'base_shard_count' : '%d', " +
+                                  "'min_sstable_size' : '0B', 'is_replica_aware': '%s'} ", numShards, isReplicaAware));
 
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
         cfs.disableAutoCompaction();
 
         int rowCount = insertData(totSizeBytes);
-        cfs.metric.flushSizeOnDisk.update(totSizeBytes); // flush size is only updated after the flush completes; set here so that flush uses correct size
+        cfs.metric.flushSizeOnDisk().update(totSizeBytes); // flush size is only updated after the flush completes; set here so that flush uses correct size
         cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
 
         assertEquals(numOutputSSTables, cfs.getLiveSSTables().size());
-        for (SSTableReader rdr : cfs.getLiveSSTables())
+
+        if (isReplicaAware)
         {
-            assertEquals(1.0 / numOutputSSTables, rdr.tokenSpaceCoverage(), 0.05);
+            // Assert that the space does not cross token boundaries
+            var tokenMetadata = StorageService.instance.getTokenMetadataForKeyspace(keyspace());
+            var tokenSpaceCoverage = 0d;
+            var spannedTokens = 0;
+            for (SSTableReader rdr : cfs.getLiveSSTables())
+            {
+                final double coverage = rdr.first.getToken().size(rdr.last.getToken());
+                // the coverage reported by rdr.tokenSpaceCoverage() may be adjusted upwards if the sstable spans too
+                // few partitions
+                if (rdr.estimatedKeys() >= ShardManager.PER_PARTITION_SPAN_THRESHOLD
+                    && coverage >= ShardManager.MINIMUM_TOKEN_COVERAGE)
+                    assertEquals(coverage, rdr.tokenSpaceCoverage(), 0.01);
+
+                tokenSpaceCoverage += coverage;
+                for (var token : tokenMetadata.sortedTokens())
+                    if (rdr.getBounds().contains(token))
+                        spannedTokens++;
+            }
+            // We don't have an even distribution because the first token is selected at random and we split along
+            // token boundaries, so we don't assert even distribution. We do however konw that the coverage should
+            // add up to about 1 without crossing that boundary. The coverage is measured by measuring the distance
+            // between the min and the max token in each shard, so we have a large delta in the assertion.
+            assertThat(tokenSpaceCoverage).isLessThanOrEqualTo(1.0);
+            assertEquals(1.0, tokenSpaceCoverage, 0.1);
+            // If we have more split points than tokens, the sstables must be split along token boundaries
+            var numSplitPoints = numShards - 1;
+            var expectedSpannedTokens = Math.max(0, tokenMetadata.sortedTokens().size() - numSplitPoints);
+            // There is a chance that the sstable bounds don't contain a token boundary due to the random selection
+            // of the first token, so we can only assert that we don't have more spanned tokens than expected.
+            assertThat(spannedTokens).isLessThanOrEqualTo(expectedSpannedTokens);
+        }
+        else
+        {
+            for (SSTableReader rdr : cfs.getLiveSSTables())
+            {
+                assertEquals(1.0 / numOutputSSTables, rdr.tokenSpaceCoverage(), 0.05);
+            }
         }
 
         validateData(rowCount);
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/ShardingTestBase.java b/test/unit/org/apache/cassandra/db/compaction/unified/ShardingTestBase.java
new file mode 100644
index 000000000000..558accc14051
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/ShardingTestBase.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionIterator;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.ShardManager;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.ScannerList;
+import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ShardingTestBase extends CQLTester
+{
+    static final String KEYSPACE = "cawt_keyspace";
+    static final String TABLE = "cawt_table";
+
+    static final int ROW_PER_PARTITION = 10;
+    static final int PARTITIONS = 5000;
+
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        CQLTester.setUpClass();
+        CQLTester.prepareServer();
+        StorageService.instance.initServer();
+
+        // Disabling durable write since we don't care
+        schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes=false");
+        schemaChange(String.format("CREATE TABLE %s.%s (k int, t int, v blob, PRIMARY KEY (k, t))", KEYSPACE, TABLE));
+    }
+
+    @AfterClass
+    public static void tearDownClass()
+    {
+        QueryProcessor.executeInternal("DROP KEYSPACE IF EXISTS " + KEYSPACE);
+    }
+
+    ColumnFamilyStore getColumnFamilyStore()
+    {
+        return Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
+    }
+
+    void verifySharding(int numShards, int rowCount, int numOutputSSTables, ColumnFamilyStore cfs) throws Throwable
+    {
+        assertEquals(numOutputSSTables, cfs.getLiveSSTables().size());
+
+        long totalOnDiskLength = cfs.getLiveSSTables().stream().mapToLong(SSTableReader::onDiskLength).sum();
+        long totalBFSize = cfs.getLiveSSTables().stream().mapToLong(ShardingTestBase::getFilterSize).sum();
+        long totalKeyCount = cfs.getLiveSSTables().stream().mapToLong(SSTableReader::estimatedKeys).sum();
+        assert totalBFSize > 16 * numOutputSSTables : "Bloom Filter is empty"; // 16 is the size of empty bloom filter
+        for (SSTableReader rdr : cfs.getLiveSSTables())
+        {
+            assertEquals((double) rdr.onDiskLength() / totalOnDiskLength,
+                         (double) getFilterSize(rdr) / totalBFSize, 0.1);
+            assertEquals(1.0 / numOutputSSTables, rdr.tokenSpaceCoverage(), 0.05);
+        }
+
+        System.out.println("Total on disk length: " + FBUtilities.prettyPrintMemory(totalOnDiskLength));
+        System.out.println("Total BF size: " + FBUtilities.prettyPrintMemory(totalBFSize));
+        System.out.println("Total key count: " + FBUtilities.prettyPrintDecimal(totalKeyCount, "", ""));
+        try (var filter = FilterFactory.getFilter(totalKeyCount, 0.01))
+        {
+            System.out.println("Optimal total BF size: " + FBUtilities.prettyPrintMemory(filter.serializedSize(false)));
+        }
+        try (var filter = FilterFactory.getFilter(totalKeyCount / numShards, 0.01))
+        {
+            System.out.println("Sharded optimal total BF size: " + FBUtilities.prettyPrintMemory(filter.serializedSize(false) * numShards));
+        }
+
+        cfs.getLiveSSTables().forEach(s -> System.out.println("SSTable: " + s.toString() + " covers " + s.getFirst() + " to " + s.getLast()));
+
+        validateData(rowCount);
+    }
+
+    static long getFilterSize(SSTableReader rdr)
+    {
+        SSTable.Owner owner = rdr.owner().orElse(null);
+        assert owner instanceof ColumnFamilyStore || owner == null;
+        ColumnFamilyStore cfs = (ColumnFamilyStore) owner;
+        return BloomFilterMetrics.instance.bloomFilterDiskSpaceUsed.getTableGauge(cfs).getValue();
+    }
+
+    int compact(int numShards, ColumnFamilyStore cfs, ShardManager shardManager, Collection<SSTableReader> selection)
+    {
+        int rows;
+        LifecycleTransaction txn = cfs.getTracker().tryModify(selection, OperationType.COMPACTION);
+        ShardedCompactionWriter writer = new ShardedCompactionWriter(cfs,
+                                                                     cfs.getDirectories(),
+                                                                     txn,
+                                                                     txn.originals(),
+                                                                     1.0,
+                                                                     false,
+                                                                     true,
+                                                                     shardManager.boundaries(numShards));
+
+        rows = compact(cfs, txn, writer);
+        return rows;
+    }
+
+    static void verifyNoSpannedBoundaries(List<Token> diskBoundaries, SSTableReader rdr)
+    {
+        for (int i = 0; i < diskBoundaries.size(); ++i)
+        {
+            Token boundary = diskBoundaries.get(i);
+            // rdr cannot span a boundary. I.e. it must be either fully before (last <= boundary) or fully after
+            // (first > boundary).
+            assertTrue(rdr.getFirst().getToken().compareTo(boundary) > 0 ||
+                       rdr.getLast().getToken().compareTo(boundary) <= 0);
+        }
+    }
+
+    int compact(ColumnFamilyStore cfs, LifecycleTransaction txn, CompactionAwareWriter writer)
+    {
+        //assert txn.originals().size() == 1;
+        int rowsWritten = 0;
+        long nowInSec = FBUtilities.nowInSeconds();
+        try (ScannerList scanners = cfs.getCompactionStrategy().getScanners(txn.originals());
+             CompactionController controller = new CompactionController(cfs, txn.originals(), cfs.gcBefore(nowInSec));
+             CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec,
+                                                            TimeUUID.maxAtUnixMillis(System.currentTimeMillis())))
+        {
+            while (ci.hasNext())
+            {
+                if (writer.append(ci.next()) != null)
+                    rowsWritten++;
+            }
+        }
+        writer.finish();
+        return rowsWritten;
+    }
+
+    void populate(int count, boolean compact) throws Throwable
+    {
+        byte [] payload = new byte[5000];
+        new Random(42).nextBytes(payload);
+        ByteBuffer b = ByteBuffer.wrap(payload);
+
+        ColumnFamilyStore cfs = getColumnFamilyStore();
+        for (int i = 0; i < count; i++)
+        {
+            for (int j = 0; j < ROW_PER_PARTITION; j++)
+                execute(String.format("INSERT INTO %s.%s(k, t, v) VALUES (?, ?, ?)", KEYSPACE, TABLE), i, j, b);
+
+            if (i % (count / 4) == 0)
+                cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        }
+
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        if (compact && cfs.getLiveSSTables().size() > 1)
+        {
+            // we want no overlapping sstables to avoid doing actual compaction in compact() above
+            try
+            {
+                cfs.forceMajorCompaction();
+            }
+            catch (Throwable t)
+            {
+                throw new RuntimeException(t);
+            }
+        }
+    }
+
+    void validateData(int rowCount) throws Throwable
+    {
+        for (int i = 0; i < rowCount; i++)
+        {
+            Object[][] expected = new Object[ROW_PER_PARTITION][];
+            for (int j = 0; j < ROW_PER_PARTITION; j++)
+                expected[j] = row(i, j);
+
+            assertRows(execute(String.format("SELECT k, t FROM %s.%s WHERE k = :i", KEYSPACE, TABLE), i), expected);
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/StaticControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/StaticControllerTest.java
new file mode 100644
index 000000000000..8db32151e6eb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/StaticControllerTest.java
@@ -0,0 +1,412 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.distributed.shared.WithProperties;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.ReplicationFactor;
+import org.apache.cassandra.schema.SchemaConstants;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.UCS_SHARED_STORAGE;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.when;
+
+public class StaticControllerTest extends ControllerTest
+{
+    static final int[] Ws = new int[] { 30, 2, 0, -6};
+
+    @Test
+    public void testFromOptions()
+    {
+        Map<String, String> options = new HashMap<>();
+        addOptions(false, options);
+
+        Controller controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+
+        for (int i = 0; i < Ws.length; i++)
+            assertEquals(Ws[i], controller.getScalingParameter(i));
+
+        assertEquals(Ws[Ws.length-1], controller.getScalingParameter(Ws.length));
+    }
+
+    private static void addOptions(boolean useIntegers, Map<String, String> options)
+    {
+        String wStr = Arrays.stream(Ws)
+                            .mapToObj(useIntegers ? Integer::toString : UnifiedCompactionStrategy::printScalingParameter)
+                            .collect(Collectors.joining(","));
+        options.put(StaticController.SCALING_PARAMETERS_OPTION, wStr);
+        options.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+    }
+
+    @Test
+    public void testFromOptionsIntegers()
+    {
+        Map<String, String> options = new HashMap<>();
+        addOptions(true, options);
+
+        Controller controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+
+
+        for (int i = 0; i < Ws.length; i++)
+            assertEquals(Ws[i], controller.getScalingParameter(i));
+
+        assertEquals(Ws[Ws.length-1], controller.getScalingParameter(Ws.length));
+    }
+
+    @Test
+    public void testFromOptionsIntegersDeprecatedName()
+    {
+        Map<String, String> options = new HashMap<>();
+        addOptions(true, options);
+        options.put(StaticController.STATIC_SCALING_FACTORS_OPTION,
+                    options.remove(StaticController.SCALING_PARAMETERS_OPTION));
+
+        Controller controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+
+
+        for (int i = 0; i < Ws.length; i++)
+            assertEquals(Ws[i], controller.getScalingParameter(i));
+
+        assertEquals(Ws[Ws.length-1], controller.getScalingParameter(Ws.length));
+    }
+
+    @Test
+    public void testFromOptionsVectorTable()
+    {
+        useVector = true;
+        Map<String, String> options = new HashMap<>();
+        Controller controller = Controller.fromOptions(cfs, new HashMap<>());
+        assertNotNull(controller);
+        assertNotNull(controller.toString());
+
+        assertEquals(Controller.DEFAULT_VECTOR_BASE_SHARD_COUNT, controller.baseShardCount);
+        assertEquals(Controller.DEFAULT_VECTOR_SSTABLE_GROWTH, controller.sstableGrowthModifier, 0.01);
+        assertEquals(Controller.DEFAULT_VECTOR_MIN_SSTABLE_SIZE, controller.minSSTableSize);
+        assertEquals(Controller.DEFAULT_VECTOR_RESERVED_THREADS, controller.getReservedThreads());
+        assertEquals(Controller.DEFAULT_VECTOR_TARGET_SSTABLE_SIZE, controller.getTargetSSTableSize());
+        int[] vectorScalingParameter = Controller.parseScalingParameters(StaticController.DEFAULT_VECTOR_STATIC_SCALING_PARAMETERS);
+        for (int i = 0; i < vectorScalingParameter.length; i++)
+            assertEquals(vectorScalingParameter[i], controller.getScalingParameter(i));
+
+        addOptions(false, options);
+
+        // Test overrides still work.
+        controller = testFromOptionsVector(false, options);
+        assertTrue(controller instanceof StaticController);
+
+        for (int i = 0; i < Ws.length; i++)
+            assertEquals(Ws[i], controller.getScalingParameter(i));
+
+        assertEquals(Ws[Ws.length-1], controller.getScalingParameter(Ws.length));
+    }
+
+    @Test
+    public void testValidateOptions()
+    {
+        Map<String, String> options = new HashMap<>();
+        addOptions(false, options);
+
+        super.testValidateOptions(options, false);
+    }
+
+    @Test
+    public void testValidateOptionsIntegers()
+    {
+        Map<String, String> options = new HashMap<>();
+        addOptions(true, options);
+
+        super.testValidateOptions(options, false);
+    }
+
+    @Test
+    public void testValidateOptionsIntegersDeprecatedName()
+    {
+        Map<String, String> options = new HashMap<>();
+        addOptions(true, options);
+        options.put(StaticController.STATIC_SCALING_FACTORS_OPTION,
+                    options.remove(StaticController.SCALING_PARAMETERS_OPTION));
+
+        super.testValidateOptions(options, false);
+    }
+
+    @Test
+    public void testValidateCompactionStrategyOptions()
+    {
+        super.testValidateCompactionStrategyOptions(true);
+    }
+
+    @Test
+    public void testSurvivalFactorForSharedStorage()
+    {
+        try (WithProperties ignore = new WithProperties().set(UCS_SHARED_STORAGE, true))
+        {
+            final int rf = 3;
+            when(replicationStrategy.getReplicationFactor()).thenReturn(ReplicationFactor.fullOnly(rf));
+
+            Controller controller = Controller.fromOptions(cfs,  new HashMap<>());
+            assertNotNull(controller);
+            assertNotNull(controller.toString());
+
+            assertEquals(Controller.DEFAULT_SURVIVAL_FACTOR / rf, controller.getSurvivalFactor(0), epsilon);
+            assertEquals(Controller.DEFAULT_SURVIVAL_FACTOR, controller.getSurvivalFactor(1), epsilon);
+            assertEquals(Controller.DEFAULT_SURVIVAL_FACTOR, controller.getSurvivalFactor(2), epsilon);
+
+            assertThatThrownBy(() -> controller.getSurvivalFactor(-1)).isInstanceOf(IllegalArgumentException.class);
+
+        }
+    }
+
+    @Test
+    public void testStartShutdown()
+    {
+        StaticController controller = new StaticController(env,
+                                                           Ws,
+                                                           Controller.DEFAULT_SURVIVAL_FACTORS,
+                                                           dataSizeGB << 30,
+                                                           0,
+                                                           0,
+                                                           0,
+                                                           Controller.DEFAULT_MAX_SPACE_OVERHEAD,
+                                                           0,
+                                                           Controller.DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS,
+                                                           Controller.DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION,
+                                                           numShards,
+                                                           false,
+                                                           sstableSizeMB << 20,
+                                                           Controller.DEFAULT_SSTABLE_GROWTH,
+                                                           Controller.DEFAULT_RESERVED_THREADS,
+                                                           Controller.DEFAULT_RESERVED_THREADS_TYPE,
+                                                           Controller.DEFAULT_OVERLAP_INCLUSION_METHOD,
+                                                           true,
+                                                           false,
+                                                           keyspaceName,
+                                                           tableName);
+        super.testStartShutdown(controller);
+    }
+
+    @Test
+    public void testShutdownNotStarted()
+    {
+        StaticController controller = new StaticController(env,
+                                                           Ws,
+                                                           Controller.DEFAULT_SURVIVAL_FACTORS,
+                                                           dataSizeGB << 30,
+                                                           0,
+                                                           0,
+                                                           0,
+                                                           Controller.DEFAULT_MAX_SPACE_OVERHEAD,
+                                                           0,
+                                                           Controller.DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS,
+                                                           Controller.DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION,
+                                                           numShards,
+                                                           false,
+                                                           sstableSizeMB << 20,
+                                                           Controller.DEFAULT_SSTABLE_GROWTH,
+                                                           Controller.DEFAULT_RESERVED_THREADS,
+                                                           Controller.DEFAULT_RESERVED_THREADS_TYPE,
+                                                           Controller.DEFAULT_OVERLAP_INCLUSION_METHOD,
+                                                           true,
+                                                           false,
+                                                           keyspaceName,
+                                                           tableName);
+        super.testShutdownNotStarted(controller);
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testStartAlreadyStarted()
+    {
+        StaticController controller = new StaticController(env,
+                                                           Ws,
+                                                           Controller.DEFAULT_SURVIVAL_FACTORS,
+                                                           dataSizeGB << 30,
+                                                           0,
+                                                           0,
+                                                           0,
+                                                           Controller.DEFAULT_MAX_SPACE_OVERHEAD,
+                                                           0,
+                                                           Controller.DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS,
+                                                           Controller.DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION,
+                                                           numShards,
+                                                           false,
+                                                           sstableSizeMB << 20,
+                                                           Controller.DEFAULT_SSTABLE_GROWTH,
+                                                           Controller.DEFAULT_RESERVED_THREADS,
+                                                           Controller.DEFAULT_RESERVED_THREADS_TYPE,
+                                                           Controller.DEFAULT_OVERLAP_INCLUSION_METHOD,
+                                                           true,
+                                                           false,
+                                                           keyspaceName,
+                                                           tableName);
+        super.testStartAlreadyStarted(controller);
+    }
+
+    @Test
+    public void testV1MaxSpaceOverhead()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(Controller.NUM_SHARDS_OPTION, Integer.toString(numShards));
+        options.put(Controller.MIN_SSTABLE_SIZE_OPTION, "20MiB");
+
+        Controller controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+
+        assertEquals(maxSpaceOverhead, controller.getMaxSpaceOverhead(), 0.0d);
+
+        options.put(Controller.MAX_SPACE_OVERHEAD_OPTION, "0.5");
+        controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+
+        assertEquals(0.5d, controller.getMaxSpaceOverhead(), 0.0d);
+
+        options.put(Controller.MAX_SPACE_OVERHEAD_OPTION, "0.1");
+        controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+
+        assertEquals(1.0d / ControllerTest.numShards, controller.getMaxSpaceOverhead(), 0.0d);
+
+        for (Double d : ImmutableList.of(0.0, 10.0, -10.0))
+        {
+            String s = d.toString();
+            try
+            {
+                options.put(Controller.MAX_SPACE_OVERHEAD_OPTION, s);
+                testFromOptions(false, options);
+                fail(String.format("%s validation must have failed for the value %s", Controller.MAX_SPACE_OVERHEAD_OPTION, s));
+            }
+            catch (ConfigurationException ce)
+            {
+                // expected
+                assertEquals(ce.getMessage(), String.format("Invalid configuration, %s must be between %f and %f: %s",
+                                                            Controller.MAX_SPACE_OVERHEAD_OPTION,
+                                                            Controller.MAX_SPACE_OVERHEAD_LOWER_BOUND,
+                                                            Controller.MAX_SPACE_OVERHEAD_UPPER_BOUND,
+                                                            s));
+            }
+        }
+    }
+
+    @Test
+    public void testMaxSSTablesToCompact()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+        Controller controller = testFromOptions(false, options);
+        assertTrue(controller.maxSSTablesToCompact <= controller.dataSetSize * controller.maxSpaceOverhead / controller.minSSTableSize);
+
+        options.put(Controller.MAX_SPACE_OVERHEAD_OPTION, "0.1");
+        controller = testFromOptions(false, options);
+        assertTrue(controller.maxSSTablesToCompact <= controller.dataSetSize * controller.maxSpaceOverhead / controller.minSSTableSize);
+
+        options.put(Controller.MAX_SSTABLES_TO_COMPACT_OPTION, "100");
+        controller = testFromOptions(false, options);
+        assertEquals(100, controller.maxSSTablesToCompact);
+
+        options.put(Controller.MAX_SSTABLES_TO_COMPACT_OPTION, "0");
+        controller = testFromOptions(false, options);
+        assertTrue(controller.maxSSTablesToCompact <= controller.dataSetSize * controller.maxSpaceOverhead / controller.minSSTableSize);
+    }
+
+    @Test
+    public void testExpiredSSTableCheckFrequency()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+
+        Controller controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+        assertEquals(TimeUnit.MILLISECONDS.convert(Controller.DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS, TimeUnit.SECONDS),
+                     controller.getExpiredSSTableCheckFrequency());
+
+        options.put(Controller.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION, "5");
+        controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+        assertEquals(5000L, controller.getExpiredSSTableCheckFrequency());
+
+        try
+        {
+            options.put(Controller.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION, "0");
+            testFromOptions(false, options);
+            fail("Exception should be thrown");
+        }
+        catch (ConfigurationException e)
+        {
+            // valid path
+        }
+    }
+
+    @Test
+    public void testAllowOverlaps()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+
+        Controller controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+        assertEquals(Controller.DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION, controller.getIgnoreOverlapsInExpirationCheck());
+
+        options.put(Controller.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION, "true");
+        controller = testFromOptions(false, options);
+        assertTrue(controller instanceof StaticController);
+        assertEquals(Controller.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION, controller.getIgnoreOverlapsInExpirationCheck());
+    }
+
+    @Test
+    public void testBaseShardCountDefault()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(Controller.SSTABLE_GROWTH_OPTION, "0");
+        Controller controller = Controller.fromOptions(cfs, options);
+        assertEquals(Controller.DEFAULT_BASE_SHARD_COUNT, controller.baseShardCount);
+
+        String prevKS = keyspaceName;
+        try
+        {
+            keyspaceName = SchemaConstants.SYSTEM_KEYSPACE_NAME;
+            controller = controller.fromOptions(cfs, options);
+            assertEquals(4, controller.baseShardCount);
+        }
+        finally
+        {
+            keyspaceName = prevKS;
+        }
+
+        numDirectories = 3;
+        controller = controller.fromOptions(cfs, options);
+        assertEquals(4, controller.baseShardCount);
+
+        numDirectories = 1;
+        controller = controller.fromOptions(cfs, options);
+        assertEquals(Controller.DEFAULT_BASE_SHARD_COUNT, controller.baseShardCount);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java b/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java
index c9d79c16a131..fa455b4badbe 100644
--- a/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java
@@ -41,11 +41,11 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
 import org.apache.cassandra.db.compaction.CompactionController;
 import org.apache.cassandra.db.compaction.CompactionIterator;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.schema.MockSchema;
@@ -208,7 +208,7 @@ public void testMultiDatadirCheck() throws IOException
             sstables.add(MockSchema.sstable(i, 1000, getCurrentColumnFamilyStore()));
 
         Directories dirs = new Directories(getCurrentColumnFamilyStore().metadata(), dataDirs);
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, sstables);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.COMPACTION, getColumnFamilyStore().metadata, sstables);
         CompactionAwareWriter writer = new MaxSSTableSizeWriter(getCurrentColumnFamilyStore(), dirs, txn, sstables, 2000, 1);
         // init case
         writer.maybeSwitchWriter(null);
@@ -232,13 +232,13 @@ private int compact(ColumnFamilyStore cfs, LifecycleTransaction txn, CompactionA
         assert txn.originals().size() == 1;
         int rowsWritten = 0;
         long nowInSec = FBUtilities.nowInSeconds();
-        try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(txn.originals());
+        try (ScannerList scanners = cfs.getCompactionStrategy().getScanners(txn.originals());
              CompactionController controller = new CompactionController(cfs, txn.originals(), cfs.gcBefore(nowInSec));
              CompactionIterator ci = new CompactionIterator(COMPACTION, scanners.scanners, controller, nowInSec, nextTimeUUID()))
         {
             while (ci.hasNext())
             {
-                if (writer.append(ci.next()))
+                if (writer.append(ci.next()) != null)
                     rowsWritten++;
             }
         }
diff --git a/test/unit/org/apache/cassandra/db/counters/CounterLockManagerTest.java b/test/unit/org/apache/cassandra/db/counters/CounterLockManagerTest.java
new file mode 100644
index 000000000000..112ae0d06764
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/counters/CounterLockManagerTest.java
@@ -0,0 +1,320 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.counters;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * Unit Tests for {@link CounterLockManager} implementations
+ */
+public class CounterLockManagerTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(CounterLockManagerTest.class);
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        DatabaseDescriptor.setConfig(new Config());
+    }
+
+    @Test
+    public void basicTestCachedCounterLockManager() throws Exception
+    {
+        basicTest(new CachedCounterLockManager());
+    }
+
+
+    @Test
+    public void basicTestStripedCounterLockManager() throws Exception
+    {
+        basicTest(new StripedCounterLockManager());
+    }
+
+
+    private static void basicTest(CounterLockManager manager) throws Exception
+    {
+        // please note that keys are not sorted and there is one duplicate
+        List<Integer> keys = List.of(1, 2, 3, 4, 5, 4);
+        List<CounterLockManager.LockHandle> lockHandleHandles = manager.grabLocks(keys);
+        for (CounterLockManager.LockHandle l : lockHandleHandles)
+            assertThat(l.tryLock(1, TimeUnit.SECONDS)).isTrue();
+
+        if (manager.hasNumKeys())
+            assertThat(manager.getNumKeys()).isEqualTo(keys.size() - 1); // one key is duplicated
+
+        lockHandleHandles.forEach(CounterLockManager.LockHandle::release);
+
+        if (manager.hasNumKeys())
+            assertThat(manager.getNumKeys()).isZero();
+
+        // double release is not allowed (expecting IllegalMonitorStateException)
+        for (CounterLockManager.LockHandle l : lockHandleHandles)
+            assertThatThrownBy(l::release).isInstanceOf(IllegalMonitorStateException.class);
+
+        // the number of keys is still zero
+        if (manager.hasNumKeys())
+            assertThat(manager.getNumKeys()).isZero();
+    }
+
+
+    @Test
+    public void lockTimeoutTestCachedCounterLockManager() throws Exception
+    {
+        lockTimeout(new CachedCounterLockManager());
+    }
+
+    @Test
+    public void lockTimeoutTestStripedCounterLockManager() throws Exception
+    {
+        lockTimeout(new StripedCounterLockManager());
+    }
+
+    private static void lockTimeout(CounterLockManager manager) throws Exception
+    {
+        List<Integer> keys = List.of(1, 2, 3, 4, 5);
+        List<CounterLockManager.LockHandle> lockHandleHandles = manager.grabLocks(keys);
+        for (CounterLockManager.LockHandle l : lockHandleHandles)
+            assertThat(l.tryLock(1, TimeUnit.SECONDS)).isTrue();
+
+        if (manager.hasNumKeys())
+            assertThat(manager.getNumKeys()).isEqualTo(keys.size());
+
+        // some implementations use reentrant locks, so we have to try to get the lock from another thread
+        int count = CompletableFuture.supplyAsync(() -> {
+            try
+            {
+                int numAcquired = 0;
+                List<CounterLockManager.LockHandle> newLockHandleHandles = manager.grabLocks(keys);
+                try
+                {
+                    for (CounterLockManager.LockHandle l : newLockHandleHandles)
+                    {
+                        if (l.tryLock(1, TimeUnit.SECONDS))
+                            numAcquired++;
+                    }
+                }
+                finally
+                {
+                    newLockHandleHandles.forEach(CounterLockManager.LockHandle::release);
+                }
+                return numAcquired;
+            }
+            catch (InterruptedException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }).join();
+        assertThat(count).isZero();
+
+        lockHandleHandles.forEach(CounterLockManager.LockHandle::release);
+
+        if (manager.hasNumKeys())
+            assertThat(manager.getNumKeys()).isZero();
+
+        count = CompletableFuture.supplyAsync(() -> {
+            try
+            {
+                int numAcquired = 0;
+                List<CounterLockManager.LockHandle> newLockHandleHandles = manager.grabLocks(keys);
+                try
+                {
+                    for (CounterLockManager.LockHandle l : newLockHandleHandles)
+                    {
+                        if (l.tryLock(1, TimeUnit.SECONDS))
+                            numAcquired++;
+                    }
+                }
+                finally
+                {
+                    newLockHandleHandles.forEach(CounterLockManager.LockHandle::release);
+                }
+                return numAcquired;
+            }
+            catch (InterruptedException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }).join();
+        assertThat(count).isEqualTo(keys.size());
+
+        if (manager.hasNumKeys())
+            assertThat(manager.getNumKeys()).isZero();
+    }
+
+    @Test
+    public void testInterruptedExceptionCachedCounterLockManager() throws Exception
+    {
+        testInterruptedException(new CachedCounterLockManager());
+    }
+
+    @Test
+    public void testInterruptedExceptionStripedCounterLockManager() throws Exception
+    {
+        testInterruptedException(new StripedCounterLockManager());
+    }
+
+    private static void testInterruptedException(CounterLockManager manager) throws Exception
+    {
+        List<Integer> keys = List.of(1, 2, 3, 4, 5);
+        List<CounterLockManager.LockHandle> lockHandleHandles = manager.grabLocks(keys);
+        for (CounterLockManager.LockHandle l : lockHandleHandles)
+            assertThat(l.tryLock(1, TimeUnit.SECONDS)).isTrue();
+
+        if (manager.hasNumKeys())
+            assertThat(manager.getNumKeys()).isEqualTo(keys.size());
+
+        CompletableFuture<Void> result = new CompletableFuture<>();
+        Thread otherThread = new Thread(() -> {
+            List<CounterLockManager.LockHandle> newLockHandleHandles = manager.grabLocks(keys);
+            try
+            {
+                try
+                {
+                    for (CounterLockManager.LockHandle l : newLockHandleHandles)
+                    {
+                        // the first of these locks will be interrupted
+                        l.tryLock(1, TimeUnit.HOURS);
+                    }
+                    result.complete(null);
+                }
+                catch (InterruptedException error)
+                {
+                    result.completeExceptionally(error);
+                }
+            }
+            finally
+            {
+                // in any case all the locks have to be released
+                newLockHandleHandles.forEach(CounterLockManager.LockHandle::release);
+            }
+        });
+
+        otherThread.start();
+        otherThread.interrupt();
+        assertThatThrownBy(result::join).hasCauseInstanceOf(InterruptedException.class);
+
+        lockHandleHandles.forEach(CounterLockManager.LockHandle::release);
+
+        if (manager.hasNumKeys())
+            assertThat(manager.getNumKeys()).isZero();
+    }
+
+
+    @Test
+    public void stressTestCachedCounterLockManager() throws Exception
+    {
+        stressTest(new CachedCounterLockManager());
+    }
+
+    @Test
+    public void stressTestStripedCounterLockManager() throws Exception
+    {
+        stressTest(new StripedCounterLockManager());
+    }
+
+    private static void stressTest(CounterLockManager manager) throws Exception
+    {
+        Random randon = new Random(12313);
+        int numThreads = 40;
+        int numKeys = 20;
+        int numIterations = 1000;
+        AtomicReference<Throwable> oneError = new AtomicReference<>();
+        CountDownLatch allDone = new CountDownLatch(numIterations);
+        ExecutorService threadPool = Executors.newFixedThreadPool(numThreads);
+        AtomicInteger threadId = new AtomicInteger();
+        try
+        {
+            for (int i = 0; i < numIterations; i++)
+            {
+                threadPool.submit(() -> {
+                    String id = "t" + threadId.incrementAndGet();
+                    int numKeysToLock = randon.nextInt(numKeys);
+
+                    List<Integer> keys = new ArrayList<>(numKeysToLock);
+                    try
+                    {
+                        for (int k = 0; k < numKeysToLock; k++)
+                            keys.add(randon.nextInt(numKeys));
+
+                        //logger.info("Thread {} grabbing locks for  {} keys, {}", id, numKeysToLock, keys);
+
+                        List<CounterLockManager.LockHandle> lockHandles = manager.grabLocks(keys);
+                        //logger.info("Thread {} got locks {}", id, lockHandles);
+                        try
+                        {
+                            for (CounterLockManager.LockHandle l : lockHandles)
+                                assertThat(l.tryLock(1, TimeUnit.MINUTES)).isTrue();
+
+                            //logger.info("Thread {} locked {}", id, lockHandles);
+
+                            // simlate some work
+                            Thread.sleep(randon.nextInt(10));
+                        }
+                        finally
+                        {
+                            // release in inverse order, like Cassandra does
+                            // iterate over all locks in reverse order and unlock them
+                            for (int j = lockHandles.size() - 1; j >= 0; j--)
+                                lockHandles.get(j).release();
+
+                            //logger.info("Thread {} released {} lockHandles, {}", id, numKeysToLock, lockHandles);
+                        }
+                    }
+                    catch (Throwable error)
+                    {
+                        logger.error("Iteration {} failed to acquire {}", id, keys, error);
+                        oneError.set(error);
+                    }
+                    finally
+                    {
+                        allDone.countDown();
+                    }
+                });
+            }
+            assertThat(allDone.await(10, TimeUnit.MINUTES)).isTrue();
+            assertThat(oneError.get()).isNull();
+
+            // the number of keys is zero in the end
+            if (manager.hasNumKeys())
+                assertThat(manager.getNumKeys()).isZero();
+        }
+        finally
+        {
+            threadPool.shutdown();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/filter/ANNOptionsTest.java b/test/unit/org/apache/cassandra/db/filter/ANNOptionsTest.java
new file mode 100644
index 000000000000..9c1a2d6a7bad
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/filter/ANNOptionsTest.java
@@ -0,0 +1,472 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.filter;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Optional;
+
+import javax.annotation.Nullable;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.StubIndex;
+import org.apache.cassandra.index.TargetParser;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.Pair;
+import org.assertj.core.api.Assertions;
+import org.quicktheories.QuickTheory;
+
+import static org.quicktheories.generators.SourceDSL.integers;
+
+/**
+ * Tests for {@link ANNOptions}, independent of the specific underlying index implementation.
+ */
+public class ANNOptionsTest extends CQLTester
+{
+    @BeforeClass
+    public static void setUpClass()
+    {
+        // Set the messaging version that adds support for the new ANN options before starting the server
+        CassandraRelevantProperties.DS_CURRENT_MESSAGING_VERSION.setInt(MessagingService.VERSION_DS_11);
+        CQLTester.setUpClass();
+    }
+
+    /**
+     * Test parsing and validation of ANN options in {@code SELECT} queries.
+     */
+    @Test
+    public void testParseAndValidate()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, n int, v vector<float, 2>)");
+
+        // invalid queries with ALLOW FILTERING before creating the ANN index
+        assertInvalidThrowMessage(StatementRestrictions.ANN_OPTIONS_WITHOUT_ORDER_BY_ANN,
+                                  InvalidRequestException.class,
+                                  "SELECT * FROM %s WHERE v = [1, 1] ALLOW FILTERING WITH ann_options = {}");
+        assertInvalidThrowMessage(String.format(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE, 'v'),
+                                  InvalidRequestException.class,
+                                  "SELECT * FROM %s ORDER BY v ANN OF [1, 1] ALLOW FILTERING WITH ann_options = {}");
+
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v) USING '%s'", ANNIndex.class.getName()));
+
+        // correct queries without specific ANN options
+        execute("SELECT * FROM %s ORDER BY v ANN OF [1, 1] WITH ann_options = {}");
+        execute("SELECT * FROM %s ORDER BY v ANN OF [1, 1] WITH ANN_OPTIONS = {}");
+        execute("SELECT * FROM %s ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {}");
+        execute("SELECT * FROM %s ORDER BY v ANN OF [1, 1] ALLOW FILTERING WITH ann_options = {}");
+        execute("SELECT * FROM %s WHERE k=0 ORDER BY v ANN OF [1, 1] WITH ann_options = {}");
+
+        // correct queries with specific ANN options
+        execute("SELECT * FROM %s ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'rerank_k': 10}");
+        execute("SELECT * FROM %s ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'rerank_k': 11}");
+        execute("SELECT * FROM %s ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'rerank_k': 1000}");
+        execute("SELECT * FROM %s ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'rerank_k': '1000'}");
+
+        // Queries with invalid ann options that will eventually be valid when we support disabling reranking
+        assertInvalidThrowMessage("Invalid rerank_k value -1 lesser than limit 100",
+                                  InvalidRequestException.class,
+                                  "SELECT * FROM %s ORDER BY v ANN OF [1, 1] LIMIT 100 WITH ann_options = {'rerank_k': -1}");
+        assertInvalidThrowMessage("Invalid rerank_k value 0 lesser than limit 100",
+                                  InvalidRequestException.class,
+                                  "SELECT * FROM %s ORDER BY v ANN OF [1, 1] LIMIT 100 WITH ann_options = {'rerank_k': 0}");
+
+        // Queries that exceed the failure threshold for the guardrail. Specifies a protocol version to trigger
+        // validation in the coordinator.
+        assertInvalidThrowMessage(Optional.of(ProtocolVersion.V5),
+                                  "ANN options specifies rerank_k=5000, this exceeds the failure threshold of 4000.",
+                                  InvalidQueryException.class,
+                                  "SELECT * FROM %s ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'rerank_k': 5000}");
+
+        String baseQuery = "SELECT * FROM %s ORDER BY v ANN OF [1, 1]";
+
+        // unknown SELECT options
+        assertInvalidThrowMessage("Unknown property 'unknown_options'",
+                                  SyntaxException.class,
+                                  baseQuery + " WITH unknown_options = {}");
+
+        // mixed known and unknown SELECT options
+        assertInvalidThrowMessage("Unknown property 'unknown_options'",
+                                  SyntaxException.class,
+                                  baseQuery + " WITH ann_options = {'rerank_k': 0} AND unknown_options = {}");
+
+        // duplicated SELECT options
+        assertInvalidThrowMessage("Multiple definitions for property 'ann_options'",
+                                  SyntaxException.class,
+                                  baseQuery + " WITH ann_options = {'rerank_k': 0} AND ann_options = {'rerank_k': 0}");
+
+        // unknown ANN options
+        assertInvalidThrowMessage("Unknown ANN option: unknown",
+                                  InvalidRequestException.class,
+                                  baseQuery + " WITH ann_options = {'unknown': 0}");
+
+        // mixed known and unknown ANN options
+        assertInvalidThrowMessage("Unknown ANN option: unknown",
+                                  InvalidRequestException.class,
+                                  baseQuery + " WITH ann_options = {'rerank_k': 0, 'unknown': 0}");
+
+        // invalid ANN options (not a number)
+        assertInvalidThrowMessage("Invalid 'rerank_k' ANN option. Expected a positive int but found: a",
+                                  InvalidRequestException.class,
+                                  baseQuery + " WITH ann_options = {'rerank_k': 'a'}");
+
+        // ANN options with rerank lesser than limit
+        assertInvalidThrowMessage("Invalid rerank_k value 10 lesser than limit 100",
+                                  InvalidRequestException.class,
+                                  baseQuery + "LIMIT 100 WITH ann_options = {'rerank_k': 10}");
+
+        // ANN options without ORDER BY ANN with empty options
+        assertInvalidThrowMessage(StatementRestrictions.ANN_OPTIONS_WITHOUT_ORDER_BY_ANN,
+                                  InvalidRequestException.class,
+                                  "SELECT * FROM %s WITH ann_options = {}");
+
+        // ANN options without ORDER BY ANN with non-empty options
+        assertInvalidThrowMessage(StatementRestrictions.ANN_OPTIONS_WITHOUT_ORDER_BY_ANN,
+                                  InvalidRequestException.class,
+                                  "SELECT * FROM %s WITH ann_options = {'rerank_k': 10}");
+
+        // ANN options without ORDER BY ANN with other expressions
+        assertInvalidThrowMessage(StatementRestrictions.ANN_OPTIONS_WITHOUT_ORDER_BY_ANN,
+                                  InvalidRequestException.class,
+                                  "SELECT * FROM %s WHERE n = 0 ALLOW FILTERING WITH ann_options = {'rerank_k': 10}");
+    }
+
+    /**
+     * Test that ANN options are considered when generating the CQL string representation of a {@link ReadCommand}.
+     */
+    @Test
+    public void testToCQLString()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, n int, v vector<float, 2>)");
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v) USING '%s'", ANNIndex.class.getName()));
+
+        // without ANN options
+        String formattedQuery = formatQuery("SELECT * FROM %%s ORDER BY v ANN OF [1, 1]");
+        ReadCommand command = parseReadCommand(formattedQuery);
+        Assertions.assertThat(command.toCQLString()).doesNotContain("WITH ann_options");
+
+        // with ANN options
+        formattedQuery = formatQuery("SELECT * FROM %%s ORDER BY v ANN OF [1, 1] LIMIT 1 WITH ann_options = {'rerank_k': 2}");
+        command = parseReadCommand(formattedQuery);
+        Assertions.assertThat(command.toCQLString()).contains("WITH ann_options = {'rerank_k': 2}");
+    }
+
+    /**
+     * Verify that the ANN options in a query get to the {@link ReadCommand} and the {@link Index},
+     * even after serialization.
+     */
+    @Test
+    public void testTransport()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, n int, v vector<float, 2>)");
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v) USING '%s'", ANNIndex.class.getName()));
+
+        // unespecified ANN options, should be mapped to NONE
+        testTransport("SELECT * FROM %s ORDER BY v ANN OF [1, 1]", ANNOptions.NONE);
+        testTransport("SELECT * FROM %s ORDER BY v ANN OF [1, 1] WITH ann_options = {}", ANNOptions.NONE);
+
+        // TODO re-enable this test when we support negative rerank_k values
+        // some random negative values, all should be accepted and not be mapped to NONE
+//        String negativeQuery = "SELECT * FROM %%s ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'rerank_k': %d}";
+//        QuickTheory.qt()
+//                   .withExamples(100)
+//                   .forAll(integers().allPositive())
+//                   .checkAssert(i -> testTransport(String.format(negativeQuery, -i), ANNOptions.create(-i)));
+
+        // some random positive values, all should be accepted
+        String positiveQuery = "SELECT * FROM %%s ORDER BY v ANN OF [1, 1] LIMIT %d WITH ann_options = {'rerank_k': %<d}";
+        QuickTheory.qt()
+                   .withExamples(100)
+                   .forAll(integers().allPositive())
+                   .checkAssert(i -> testTransport(String.format(positiveQuery, i), ANNOptions.create(i)));
+    }
+
+    private void testTransport(String query, ANNOptions expectedOptions)
+    {
+        // verify that the options arrive correctly at the index
+        execute(query);
+        Assertions.assertThat(ANNIndex.lastQueryAnnOptions).isEqualTo(expectedOptions);
+
+        // verify that the options are correctly parsed and stored in the ReadCommand
+        String formattedQuery = formatQuery(query);
+        ReadCommand command = parseReadCommand(formattedQuery);
+        ANNOptions actualOptions = command.rowFilter().annOptions();
+        Assertions.assertThat(actualOptions).isEqualTo(expectedOptions);
+
+        // serialize and deserialize the command to check if the options are preserved...
+        try
+        {
+            // ...with a version that supports ANN options
+            DataOutputBuffer out = new DataOutputBuffer();
+            ReadCommand.serializer.serialize(command, out, MessagingService.VERSION_DS_11);
+            Assertions.assertThat(ReadCommand.serializer.serializedSize(command, MessagingService.VERSION_DS_11))
+                      .isEqualTo(out.buffer().remaining());
+            DataInputBuffer in = new DataInputBuffer(out.buffer(), true);
+            command = ReadCommand.serializer.deserialize(in, MessagingService.VERSION_DS_11);
+            actualOptions = command.rowFilter().annOptions();
+            Assertions.assertThat(actualOptions).isEqualTo(expectedOptions);
+
+            // ...with a version that doesn't support ANN options
+            out = new DataOutputBuffer();
+            if (expectedOptions != ANNOptions.NONE) {
+                try
+                {
+                    ReadCommand.serializer.serialize(command, out, MessagingService.VERSION_DS_10);
+                }
+                catch (IllegalStateException e)
+                {
+                    // expected
+                    Assertions.assertThat(e)
+                              .hasMessageContaining("Unable to serialize ANN options with messaging version: " + MessagingService.VERSION_DS_10);
+                }
+            } else {
+                ReadCommand.serializer.serialize(command, out, MessagingService.VERSION_DS_10);
+                Assertions.assertThat(ReadCommand.serializer.serializedSize(command, MessagingService.VERSION_DS_10))
+                          .isEqualTo(out.buffer().remaining());
+                in = new DataInputBuffer(out.buffer(), true);
+                command = ReadCommand.serializer.deserialize(in, MessagingService.VERSION_DS_10);
+                actualOptions = command.rowFilter().annOptions();
+                Assertions.assertThat(actualOptions).isEqualTo(ANNOptions.NONE);
+            }
+        }
+        catch (IOException e)
+        {
+            throw new AssertionError(e);
+        }
+    }
+
+    /**
+     * Tests that any future versions of {@link ANNOptions} can be able to read the current options.
+     */
+    @Test
+    public void testSerializationForFutureVersions() throws IOException
+    {
+        // the current version of the ANN options...
+        ANNOptions sentOptions = ANNOptions.create(7);
+        DataOutputBuffer out = new DataOutputBuffer();
+        ANNOptions.serializer.serialize(sentOptions, out, MessagingService.current_version);
+        int serializedSize = out.buffer().remaining();
+        Assertions.assertThat(ANNOptions.serializer.serializedSize(sentOptions, MessagingService.current_version))
+                  .isEqualTo(serializedSize);
+
+        // ...should be readable with the future serializer
+        DataInputBuffer in = new DataInputBuffer(out.buffer(), true);
+        FutureANNOptions receivedOptions = FutureANNOptions.serializer.deserialize(in);
+        Assertions.assertThat(receivedOptions).isEqualTo(new FutureANNOptions(sentOptions));
+        Assertions.assertThat(FutureANNOptions.serializer.serializedSize(receivedOptions))
+                  .isEqualTo(serializedSize);
+    }
+
+    /**
+     * Tests that we will be able to deserialize future versions of {@link ANNOptions} with new properties if those new
+     * properties are defaults.
+     */
+    @Test
+    public void testDeserializationOfCompatibleFutureVersions() throws IOException
+    {
+        // a future new version of the ANN options with default new properties...
+        FutureANNOptions sentOptions = new FutureANNOptions(7, FutureANNOptions.NEW_PROPERTY_DEFAULT);
+        DataOutputBuffer out = new DataOutputBuffer();
+        FutureANNOptions.serializer.serialize(sentOptions, out);
+        int serializedSize = out.buffer().remaining();
+        Assertions.assertThat(FutureANNOptions.serializer.serializedSize(sentOptions))
+                  .isEqualTo(serializedSize);
+
+        // ...should be readable with the current serializer
+        DataInputBuffer in = new DataInputBuffer(out.buffer(), true);
+        ANNOptions receivedOptions = ANNOptions.serializer.deserialize(in, MessagingService.current_version);
+        Assertions.assertThat(receivedOptions).isEqualTo(ANNOptions.create(sentOptions.rerankK));
+        Assertions.assertThat(ANNOptions.serializer.serializedSize(receivedOptions, MessagingService.current_version))
+                  .isEqualTo(serializedSize);
+    }
+
+    /**
+     * Tests that we won't be able to desrialize future versions of {@link ANNOptions} with new properties if those new
+     * properties are not defaults.
+     */
+    @Test
+    public void testDeserializationOfNonCompatibleFutureVersions() throws IOException
+    {
+        // a future new version of the ANN options with non-default new properties...
+        FutureANNOptions sentOptions = new FutureANNOptions(7, "newProperty");
+        DataOutputBuffer out = new DataOutputBuffer();
+        FutureANNOptions.serializer.serialize(sentOptions, out);
+        int serializedSize = out.buffer().remaining();
+        Assertions.assertThat(FutureANNOptions.serializer.serializedSize(sentOptions))
+                  .isEqualTo(serializedSize);
+
+        // ...should fail in a controlled manner with the current serializer
+        try (DataInputBuffer in = new DataInputBuffer(out.buffer(), true))
+        {
+            Assertions.assertThatThrownBy(() -> ANNOptions.serializer.deserialize(in, MessagingService.current_version))
+                      .isInstanceOf(IOException.class)
+                      .hasMessageContaining("Found unsupported ANN options");
+        }
+    }
+
+    /**
+     * Class simulating a future version of the ANN options, with an additional property.
+     */
+    private static class FutureANNOptions
+    {
+        public static final String NEW_PROPERTY_DEFAULT = "default";
+
+        public static final Serializer serializer = new Serializer();
+
+        @Nullable
+        public final Integer rerankK;
+        public final String newProperty;
+
+        public FutureANNOptions(@Nullable Integer rerankK, String newProperty)
+        {
+            this.rerankK = rerankK;
+            this.newProperty = newProperty;
+        }
+
+        public FutureANNOptions(ANNOptions options)
+        {
+            this.rerankK = options.rerankK;
+            this.newProperty = NEW_PROPERTY_DEFAULT;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            FutureANNOptions that = (FutureANNOptions) o;
+            return Objects.equals(rerankK, that.rerankK) && Objects.equals(newProperty, that.newProperty);
+        }
+
+        public static class Serializer
+        {
+            private static final int RERANK_K_MASK = 1;
+            private static final int NEW_PROPERTY_MASK = 1 << 31;
+            private static final int UNKNOWN_OPTIONS_MASK = ~(RERANK_K_MASK | NEW_PROPERTY_MASK);
+
+            public void serialize(FutureANNOptions options, DataOutputPlus out) throws IOException
+            {
+                int flags = flags(options);
+                out.writeInt(flags);
+
+                if (options.rerankK != null)
+                    out.writeUnsignedVInt32(options.rerankK);
+
+                if (hasNewProperty(flags))
+                    out.writeUTF(options.newProperty);
+            }
+
+            public FutureANNOptions deserialize(DataInputPlus in) throws IOException
+            {
+                int flags = in.readInt();
+
+                if ((flags & UNKNOWN_OPTIONS_MASK) != 0)
+                    throw new IOException("Found unsupported ANN options");
+
+                Integer rerankK = hasRerankK(flags) ? (int) in.readUnsignedVInt() : null;
+                String newProperty = hasNewProperty(flags) ? in.readUTF() : NEW_PROPERTY_DEFAULT;
+
+                return new FutureANNOptions(rerankK, newProperty);
+            }
+
+            public long serializedSize(FutureANNOptions options)
+            {
+                int flags = flags(options);
+                long size = TypeSizes.sizeof(flags);
+
+                if (options.rerankK != null)
+                    size += TypeSizes.sizeofUnsignedVInt(options.rerankK);
+
+                if (hasNewProperty(flags))
+                    size += TypeSizes.sizeof(options.newProperty);
+
+                return size;
+            }
+
+            private static int flags(FutureANNOptions options)
+            {
+                int flags = 0;
+
+                if (options.rerankK != null)
+                    flags |= RERANK_K_MASK;
+
+                if (!Objects.equals(options.newProperty, NEW_PROPERTY_DEFAULT))
+                    flags |= NEW_PROPERTY_MASK;
+
+                return flags;
+            }
+
+            private static boolean hasRerankK(long flags)
+            {
+                return (flags & RERANK_K_MASK) == RERANK_K_MASK;
+            }
+
+            public static boolean hasNewProperty(long flags)
+            {
+                return (flags & NEW_PROPERTY_MASK) == NEW_PROPERTY_MASK;
+            }
+        }
+    }
+
+    /**
+     * Mock index with dummy ANN support.
+     */
+    public static final class ANNIndex extends StubIndex
+    {
+        private final ColumnMetadata indexedColumn;
+        public static volatile ANNOptions lastQueryAnnOptions;
+
+        public ANNIndex(ColumnFamilyStore baseCfs, IndexMetadata metadata)
+        {
+            super(baseCfs, metadata);
+            Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(baseCfs.metadata(), metadata);
+            indexedColumn = target.left;
+        }
+
+        @Override
+        public boolean supportsExpression(ColumnMetadata column, Operator operator)
+        {
+            return indexedColumn.name.equals(column.name) && operator == Operator.ANN;
+        }
+
+        @Override
+        public Searcher searcherFor(ReadCommand command)
+        {
+            lastQueryAnnOptions = command.rowFilter().annOptions();
+            return super.searcherFor(command);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java
index 3cf7c59e9f27..319baae3b04e 100644
--- a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java
+++ b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java
@@ -31,7 +31,6 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.db.RegularAndStaticColumns;
-
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.rows.CellPath;
@@ -45,8 +44,10 @@
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.Throwables;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 
 public class ColumnFilterTest
@@ -83,7 +84,6 @@ public static void beforeClass()
         DatabaseDescriptor.setSeedProvider(Arrays::asList);
         DatabaseDescriptor.setEndpointSnitch(new SimpleSnitch());
         DatabaseDescriptor.setDefaultFailureDetector();
-        DatabaseDescriptor.setPartitionerUnsafe(new Murmur3Partitioner());
         Gossiper.instance.start(0);
     }
 
@@ -91,6 +91,7 @@ public static void beforeClass()
     public void before()
     {
         Util.setUpgradeFromVersion("4.0");
+        assertThat(Gossiper.instance.getMinVersion()).isEqualTo(new CassandraVersion("4.0"));
     }
 
     // Select all
diff --git a/test/unit/org/apache/cassandra/db/filter/DataLimitsTest.java b/test/unit/org/apache/cassandra/db/filter/DataLimitsTest.java
new file mode 100644
index 000000000000..ca0ef3e9b1de
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/filter/DataLimitsTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.filter;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.db.aggregation.AggregationSpecification;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.db.filter.DataLimits.NO_LIMIT;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+public class DataLimitsTest
+{
+    private final static Logger logger = LoggerFactory.getLogger(DataLimitsTest.class);
+
+    ByteBuffer lastReturnedKey = ByteBufferUtil.bytes("lastReturnedKey");
+
+    DataLimits cqlLimits = DataLimits.cqlLimits(19, 17);
+    DataLimits cqlLimitsForPagingInRows = cqlLimits.forPaging(PageSize.inRows(13));
+    DataLimits cqlLimitsForPagingInBytes = cqlLimits.forPaging(PageSize.inBytes(13));
+    DataLimits cqlLimitsForPagingInRowsWithLastRow = cqlLimits.forPaging(PageSize.inRows(13), lastReturnedKey, 5);
+    DataLimits cqlLimitsForPagingInBytesWithLastRow = cqlLimits.forPaging(PageSize.inBytes(13), lastReturnedKey, 5);
+    DataLimits groupByLimits = DataLimits.groupByLimits(19, 17, NO_LIMIT, NO_LIMIT, AggregationSpecification.AGGREGATE_EVERYTHING);
+    DataLimits groupByLimitsForPagingInRows = groupByLimits.forPaging(PageSize.inRows(13));
+    DataLimits groupByLimitsForPagingInBytes = groupByLimits.forPaging(PageSize.inBytes(13));
+    DataLimits groupByLimitsForPagingInRowsWithLastRow = groupByLimits.forPaging(PageSize.inRows(13), lastReturnedKey, 5);
+    DataLimits groupByLimitsForPagingInBytesWithLastRow = groupByLimits.forPaging(PageSize.inBytes(13), lastReturnedKey, 5);
+
+    @BeforeClass
+    public static void initClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void serializationTest()
+    {
+        TableMetadata metadata = SchemaLoader.standardCFMD("ks", "cf", 1, IntegerType.instance, IntegerType.instance, IntegerType.instance).build();
+        for (MessagingService.Version version : MessagingService.Version.values())
+        {
+            checkSerialization(version, cqlLimits, "cql limits", metadata);
+            checkSerialization(version, cqlLimitsForPagingInRows, "cql limits for paging in rows", metadata);
+            checkSerialization(version, cqlLimitsForPagingInBytes, "cql limits for paging in bytes", metadata);
+            checkSerialization(version, cqlLimitsForPagingInRowsWithLastRow, "cql limits for paging in rows with last row", metadata);
+            checkSerialization(version, cqlLimitsForPagingInBytesWithLastRow, "cql limits for paging in bytes with last row", metadata);
+            checkSerialization(version, groupByLimits, "group by limits", metadata);
+            checkSerialization(version, groupByLimitsForPagingInRows, "group by limits for paging in rows", metadata);
+            checkSerialization(version, groupByLimitsForPagingInBytes, "group by limits for paging in bytes", metadata);
+            checkSerialization(version, groupByLimitsForPagingInRowsWithLastRow, "group by limits for paging in rows with last row", metadata);
+            checkSerialization(version, groupByLimitsForPagingInBytesWithLastRow, "group by limits for paging in bytes with last row", metadata);
+        }
+    }
+
+    @Test
+    public void toStringTest()
+    {
+        String lastRetKeyStr = String.format("lastReturnedKey=%s", ByteBufferUtil.bytesToHex(lastReturnedKey));
+        String lastRetKeyRemainingStr = "lastReturnedKeyRemaining=5";
+
+        assertThat(cqlLimits.toString()).contains("ROWS LIMIT 19").contains("PER PARTITION LIMIT 17").doesNotContain("BYTES LIMIT");
+        assertThat(cqlLimitsForPagingInRows.toString()).contains("ROWS LIMIT 13").contains("PER PARTITION LIMIT 17").doesNotContain("BYTES LIMIT");
+        assertThat(cqlLimitsForPagingInBytes.toString()).contains("BYTES LIMIT 13").contains("ROWS LIMIT 19").contains("PER PARTITION LIMIT 17");
+        assertThat(cqlLimitsForPagingInRowsWithLastRow.toString()).contains("ROWS LIMIT 13").contains("PER PARTITION LIMIT 17").doesNotContain("BYTES LIMIT").contains(lastRetKeyStr).contains(lastRetKeyRemainingStr);
+        assertThat(cqlLimitsForPagingInBytesWithLastRow.toString()).contains("BYTES LIMIT 13").contains("ROWS LIMIT 19").contains("PER PARTITION LIMIT 17").contains(lastRetKeyStr).contains(lastRetKeyRemainingStr);
+
+        assertThat(groupByLimits.toString()).contains("GROUP LIMIT 19").contains("GROUP PER PARTITION LIMIT 17").doesNotContain("ROWS LIMIT").doesNotContain("BYTES LIMIT");
+        assertThat(groupByLimitsForPagingInRows.toString()).contains("GROUP LIMIT 19").contains("GROUP PER PARTITION LIMIT 17").contains("ROWS LIMIT 13").doesNotContain("BYTES LIMIT");
+        assertThat(groupByLimitsForPagingInBytes.toString()).contains("GROUP LIMIT 19").contains("GROUP PER PARTITION LIMIT 17").doesNotContain("ROWS LIMIT").contains("BYTES LIMIT 13");
+        assertThat(groupByLimitsForPagingInRowsWithLastRow.toString()).contains("GROUP LIMIT 19").contains("GROUP PER PARTITION LIMIT 17").contains("ROWS LIMIT 13").doesNotContain("BYTES LIMIT").contains(lastRetKeyStr).contains(lastRetKeyRemainingStr);
+        assertThat(groupByLimitsForPagingInBytesWithLastRow.toString()).contains("GROUP LIMIT 19").contains("GROUP PER PARTITION LIMIT 17").doesNotContain("ROWS LIMIT").contains("BYTES LIMIT 13").contains(lastRetKeyStr).contains(lastRetKeyRemainingStr);
+    }
+
+    private void checkSerialization(MessagingService.Version version, DataLimits limits, String name, TableMetadata metadata)
+    {
+        String msg = String.format("serialization of %s for version %s", name, version);
+        int size = (int) DataLimits.serializer.serializedSize(limits, version.value, metadata.comparator);
+        try (DataOutputBuffer out = new DataOutputBuffer(2 * size))
+        {
+            DataLimits.serializer.serialize(limits, out, version.value, metadata.comparator);
+            out.flush();
+            assertThat(out.getLength()).describedAs(msg).isEqualTo(size);
+            try (DataInputBuffer in = new DataInputBuffer(out.getData()))
+            {
+                DataLimits deserializedLimits = DataLimits.serializer.deserialize(in, version.value, metadata);
+                assertThat(deserializedLimits.count()).describedAs(msg).isEqualTo(limits.count());
+
+                if (version.value >= MessagingService.VERSION_DS_10)
+                    assertThat(deserializedLimits.bytes()).describedAs(msg).isEqualTo(limits.bytes());
+                else
+                    assertThat(deserializedLimits.bytes()).describedAs(msg).isEqualTo(NO_LIMIT);
+
+                assertThat(deserializedLimits.rows()).describedAs(msg).isEqualTo(limits.rows());
+                assertThat(deserializedLimits.perPartitionCount()).describedAs(msg).isEqualTo(limits.perPartitionCount());
+                assertThat(deserializedLimits.isDistinct()).describedAs(msg).isEqualTo(limits.isDistinct());
+                assertThat(deserializedLimits.isUnlimited()).describedAs(msg).isEqualTo(limits.isUnlimited());
+                assertThat(deserializedLimits.kind()).describedAs(msg).isEqualTo(limits.kind());
+                assertThat(deserializedLimits.isGroupByLimit()).describedAs(msg).isEqualTo(limits.isGroupByLimit());
+            }
+            catch (IOException | RuntimeException e)
+            {
+                logger.error("Failed to deserialize: " + msg, e);
+                fail(msg);
+            }
+        }
+        catch (IOException | RuntimeException e)
+        {
+            logger.error("Failed to serialize: " + msg, e);
+            fail(msg);
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java b/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
index 8952262e2be0..6042967aed61 100644
--- a/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
+++ b/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
@@ -19,7 +19,6 @@
 package org.apache.cassandra.db.filter;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.junit.Assert;
@@ -49,9 +48,8 @@
 
 public class RowFilterTest
 {
-
     @Test
-    public void testCQLFilterClose()
+    public void testRowFilterClose()
     {
         // CASSANDRA-15126
         TableMetadata metadata = TableMetadata.builder("testks", "testcf")
@@ -63,10 +61,10 @@ public void testCQLFilterClose()
         ColumnMetadata r = metadata.getColumn(new ColumnIdentifier("r", true));
 
         ByteBuffer one = Int32Type.instance.decompose(1);
-        RowFilter filter = RowFilter.none().withNewExpressions(new ArrayList<>());
+        RowFilter.Builder filter = RowFilter.builder(false);
         filter.add(s, Operator.NEQ, one);
         AtomicBoolean closed = new AtomicBoolean();
-        UnfilteredPartitionIterator iter = filter.filter(new SingletonUnfilteredPartitionIterator(new UnfilteredRowIterator()
+        UnfilteredPartitionIterator iter = filter.build().filter(new SingletonUnfilteredPartitionIterator(new UnfilteredRowIterator()
         {
             public DeletionTime partitionLevelDeletion() { return null; }
             public EncodingStats stats() { return null; }
@@ -91,10 +89,10 @@ public void close()
         Assert.assertFalse(iter.hasNext());
         Assert.assertTrue(closed.get());
 
-        filter = RowFilter.none().withNewExpressions(new ArrayList<>());
+        filter = RowFilter.builder(false);
         filter.add(r, Operator.NEQ, one);
         closed.set(false);
-        iter = filter.filter(new SingletonUnfilteredPartitionIterator(new UnfilteredRowIterator()
+        iter = filter.build().filter(new SingletonUnfilteredPartitionIterator(new UnfilteredRowIterator()
         {
             boolean hasNext = true;
             public DeletionTime partitionLevelDeletion() { return null; }
diff --git a/test/unit/org/apache/cassandra/db/filter/SliceTest.java b/test/unit/org/apache/cassandra/db/filter/SliceTest.java
index 18233f09f5cb..0acf6cd01982 100644
--- a/test/unit/org/apache/cassandra/db/filter/SliceTest.java
+++ b/test/unit/org/apache/cassandra/db/filter/SliceTest.java
@@ -43,6 +43,8 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+// TODO refactor this test - clustering comparator should be tested independently from slices intersections
+// TODO each intersection test should be done in both directions as "intersects" relation is symmetric
 public class SliceTest
 {
     @Test
diff --git a/test/unit/org/apache/cassandra/db/guardrails/CustomUserKeyspaceFilterProviderTest.java b/test/unit/org/apache/cassandra/db/guardrails/CustomUserKeyspaceFilterProviderTest.java
new file mode 100644
index 000000000000..18812d3582e0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/CustomUserKeyspaceFilterProviderTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+
+import org.apache.cassandra.service.ClientState;
+import org.junit.Test;
+
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class CustomUserKeyspaceFilterProviderTest
+{
+    public static class TestUserKeyspaceFilterProvider implements UserKeyspaceFilterProvider
+    {
+        @Override
+        public UserKeyspaceFilter get(ClientState clientState)
+        {
+            return null;
+        }
+    }
+
+    @Test
+    public void testMakeWithValidClass()
+    {
+        String validClassName = TestUserKeyspaceFilterProvider.class.getName();
+        UserKeyspaceFilterProvider provider = CustomUserKeyspaceFilterProvider.make(validClassName);
+        assertTrue(provider instanceof TestUserKeyspaceFilterProvider);
+    }
+
+    @Test
+    public void testMakeWithInvalidClass()
+    {
+        String invalidClassName = "com.example.NonExistingClass";
+        assertThrows(IllegalStateException.class, () -> CustomUserKeyspaceFilterProvider.make(invalidClassName));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailCollectionSizeTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailCollectionSizeTest.java
index b15f85a1255b..78c9c47dc4ef 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailCollectionSizeTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailCollectionSizeTest.java
@@ -26,6 +26,8 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.config.DataStorageSpec;
@@ -47,6 +49,7 @@ public class GuardrailCollectionSizeTest extends ThresholdTester
 {
     private static final int WARN_THRESHOLD = 1024; // bytes
     private static final int FAIL_THRESHOLD = WARN_THRESHOLD * 4; // bytes
+    private static long savedMinNotifyInterval;
 
     public GuardrailCollectionSizeTest()
     {
@@ -60,6 +63,19 @@ public GuardrailCollectionSizeTest()
               size -> new DataStorageSpec.LongBytesBound(size).toBytes());
     }
 
+    @BeforeClass
+    public static void setup()
+    {
+        savedMinNotifyInterval = Guardrails.collectionSize.minNotifyIntervalInMs();
+        Guardrails.collectionSize.minNotifyIntervalInMs(0);
+    }
+
+    @AfterClass
+    public static void tearDown()
+    {
+        Guardrails.collectionSize.minNotifyIntervalInMs(savedMinNotifyInterval);
+    }
+    
     @After
     public void after()
     {
@@ -219,8 +235,35 @@ public void testMapSizeWithUpdates() throws Throwable
         assertWarns("UPDATE %s SET v = v + ? WHERE k = 6", map(allocate(FAIL_THRESHOLD / 4 + 1), allocate(FAIL_THRESHOLD / 4)));
     }
 
+    @Test
+    public void testGuardrailRespectsMinimumNotificationInterval() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v set<text>)");
+
+        assertValid("INSERT INTO %s (k, v) VALUES (0, null)");
+        assertWarns("INSERT INTO %s (k, v) VALUES (1, ?)", set(allocate(WARN_THRESHOLD)));
+
+        long previousNotifyInterval = Guardrails.collectionSize.minNotifyIntervalInMs();
+        Guardrails.collectionSize.minNotifyIntervalInMs(2000L);
+        Guardrails.collectionSize.resetLastNotifyTime();
+
+        try
+        {
+            assertWarns("INSERT INTO %s (k, v) VALUES (2, ?)", set(allocate(WARN_THRESHOLD)));
+            assertValid("INSERT INTO %s (k, v) VALUES (3, ?)", set(allocate(WARN_THRESHOLD)));
+
+            Thread.sleep(2500L);
+
+            assertWarns("INSERT INTO %s (k, v) VALUES (2, ?)", set(allocate(WARN_THRESHOLD)));
+        }
+        finally
+        {
+            Guardrails.collectionSize.minNotifyIntervalInMs(previousNotifyInterval);
+        }
+    }
+
     @Override
-    protected String createTable(String query)
+    public String createTable(String query)
     {
         String table = super.createTable(query);
         disableCompaction();
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailCounterTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailCounterTest.java
new file mode 100644
index 000000000000..7ef06cbfc260
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailCounterTest.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+
+public class GuardrailCounterTest extends GuardrailTester
+{
+    private static boolean counterEnabled;
+
+    @BeforeClass
+    public static void setup()
+    {
+        counterEnabled = DatabaseDescriptor.getGuardrailsConfig().getCounterEnabled();
+    }
+
+    @AfterClass
+    public static void tearDown()
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setCounterEnabled(counterEnabled);
+    }
+
+    private void setGuardrails(boolean counterEnabled)
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setCounterEnabled(counterEnabled);
+    }
+
+    @Test
+    public void testCounterEnabled() throws Throwable
+    {
+        setGuardrails(true);
+        assertValid(String.format("CREATE TABLE %s (pk int PRIMARY KEY, c counter)", createTableName()));
+        execute("UPDATE %s SET c = c + 1 WHERE pk = 10");
+        assertRows(execute("SELECT c FROM %s WHERE pk = 10"), row(1L));
+    }
+
+    @Test
+    public void testCounterDisabled() throws Throwable
+    {
+        setGuardrails(false);
+        assertFails(String.format("CREATE TABLE %s (pk int PRIMARY KEY, c counter)", createTableName()),
+                "Guardrail counter violated: Counter is not allowed");
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailDiskUsageTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailDiskUsageTest.java
index 86e4df07b8dd..3409f2c4d2dc 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailDiskUsageTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailDiskUsageTest.java
@@ -490,6 +490,7 @@ public void testWriteRequests() throws Throwable
         InetAddressAndPort node2 = InetAddressAndPort.getByName("127.0.0.21");
         InetAddressAndPort node3 = InetAddressAndPort.getByName("127.0.0.31");
 
+        // avoid noise due to test machines
         Guardrails.replicaDiskUsage.resetLastNotifyTime();
         guardrails().setDataDiskUsagePercentageThreshold(98, 99);
 
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailIgnoredPropertiesTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailIgnoredPropertiesTest.java
new file mode 100644
index 000000000000..6a6be065b6b6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailIgnoredPropertiesTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.statements.schema.TableAttributes;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class GuardrailIgnoredPropertiesTest extends GuardrailTester
+{
+    private static Set<String> defaultTablePropertiesIgnored;
+
+    @Before
+    public void before()
+    {
+        defaultTablePropertiesIgnored = DatabaseDescriptor.getGuardrailsConfig().getTablePropertiesIgnored();
+
+        // but actually ignore "comment"
+        DatabaseDescriptor.getGuardrailsConfig().setTablePropertiesIgnored(ImmutableSet.of("comment"));
+    }
+
+    @After
+    public void after()
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setTablePropertiesIgnored(defaultTablePropertiesIgnored);
+    }
+
+    @Test
+    public void testPropertySkipping()
+    {
+        TableAttributes tableAttributes = new TableAttributes();
+        tableAttributes.addProperty("bloom_filter_fp_chance", "0.01");
+        tableAttributes.addProperty("comment", "");
+        tableAttributes.addProperty("crc_check_chance", "1.0");
+        tableAttributes.addProperty("default_time_to_live", "0");
+        tableAttributes.addProperty("gc_grace_seconds", "864000");
+        tableAttributes.addProperty("max_index_interval", "2048");
+        tableAttributes.addProperty("memtable_flush_period_in_ms", "0");
+        tableAttributes.addProperty("min_index_interval", "128");
+        tableAttributes.addProperty("read_repair","BLOCKING'");
+        tableAttributes.addProperty("speculative_retry", "99p");
+
+        assertTrue(tableAttributes.hasProperty("comment"));
+        assertEquals(10, tableAttributes.updatedProperties().size());
+
+        // Should not throw ConcurrentModificationException (CNDB-7724)
+        Guardrails.tableProperties.guard(tableAttributes.updatedProperties(), tableAttributes::removeProperty, null);
+//        Guardrails.ignoredTableProperties.maybeIgnoreAndWarn(tableAttributes.updatedProperties(), tableAttributes::removeProperty, null);
+
+        assertFalse(tableAttributes.hasProperty("comment"));
+        assertEquals(9, tableAttributes.updatedProperties().size());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailItemsPerCollectionTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailItemsPerCollectionTest.java
index a13e9b33d5e4..c5db282f4650 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailItemsPerCollectionTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailItemsPerCollectionTest.java
@@ -25,6 +25,8 @@
 import java.util.stream.IntStream;
 
 import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.db.marshal.Int32Type;
@@ -42,6 +44,7 @@ public class GuardrailItemsPerCollectionTest extends ThresholdTester
 {
     private static final int WARN_THRESHOLD = 10;
     private static final int FAIL_THRESHOLD = 20;
+    private static long savedMinNotifyInterval;
 
     public GuardrailItemsPerCollectionTest()
     {
@@ -53,6 +56,19 @@ public GuardrailItemsPerCollectionTest()
               Guardrails::getItemsPerCollectionFailThreshold);
     }
 
+    @BeforeClass
+    public static void setup()
+    {
+        savedMinNotifyInterval = Guardrails.itemsPerCollection.minNotifyIntervalInMs();
+        Guardrails.itemsPerCollection.minNotifyIntervalInMs(0);
+    }
+
+    @AfterClass
+    public static void tearDown()
+    {
+        Guardrails.itemsPerCollection.minNotifyIntervalInMs(savedMinNotifyInterval);
+    }
+
     @After
     public void after()
     {
@@ -218,7 +234,7 @@ public void testMapSizeWithUpdates() throws Throwable
     }
 
     @Override
-    protected String createTable(String query)
+    public String createTable(String query)
     {
         String table = super.createTable(query);
         disableCompaction();
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailLoggedBatchTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailLoggedBatchTest.java
new file mode 100644
index 000000000000..3229f5adf021
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailLoggedBatchTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.BatchStatement;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+public class GuardrailLoggedBatchTest extends GuardrailTester
+{
+    private static boolean loggedBatchEnabled;
+
+    @BeforeClass
+    public static void setup()
+    {
+        loggedBatchEnabled = DatabaseDescriptor.getGuardrailsConfig().getLoggedBatchEnabled();
+    }
+
+    @AfterClass
+    public static void tearDown()
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setLoggedBatchEnabled(loggedBatchEnabled);
+    }
+
+    @Before
+    public void setupTest()
+    {
+        createTable("CREATE TABLE IF NOT EXISTS %s (k INT, c INT, v TEXT, PRIMARY KEY(k, c))");
+    }
+
+    private void setGuardrails(boolean logged_batch_enabled)
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setLoggedBatchEnabled(logged_batch_enabled);
+    }
+
+    private void insertBatchAndAssertValid(boolean loggedBatchEnabled, boolean logged) throws Throwable
+    {
+        setGuardrails(loggedBatchEnabled);
+
+        BatchStatement batch = new BatchStatement(logged ? BatchStatement.Type.LOGGED : BatchStatement.Type.UNLOGGED);
+        batch.add(new SimpleStatement(String.format("INSERT INTO %s.%s (k, c, v) VALUES (1, 2, 'val')", keyspace(), currentTable())));
+        batch.add(new SimpleStatement(String.format("INSERT INTO %s.%s (k, c, v) VALUES (3, 4, 'val')", keyspace(), currentTable())));
+
+        assertValid(() -> executeNet(getDefaultVersion(), batch));
+    }
+
+    @Test
+    public void testInsertUnloggedBatch() throws Throwable
+    {
+        insertBatchAndAssertValid(false, false);
+        insertBatchAndAssertValid(true, false);
+    }
+
+    @Test(expected = InvalidQueryException.class)
+    public void testDisabledLoggedBatch() throws Throwable
+    {
+        insertBatchAndAssertValid(false, true);
+    }
+
+    @Test
+    public void testEnabledLoggedBatch() throws Throwable
+    {
+        insertBatchAndAssertValid(true, true);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailOffsetRowsTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailOffsetRowsTest.java
new file mode 100644
index 000000000000..703cc7a37e9d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailOffsetRowsTest.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.guardrails;
+
+
+import org.junit.Test;
+
+import static java.lang.String.format;
+
+/**
+ * Tests the guardrail for the number of rows that a LIMIT/OFFSET SELECT query can skip, {@link Guardrails#offsetRows}.
+ */
+public class GuardrailOffsetRowsTest extends ThresholdTester
+{
+    private static final int WARN_THRESHOLD = 2;
+    private static final int FAIL_THRESHOLD = 4;
+
+    public GuardrailOffsetRowsTest()
+    {
+        super(WARN_THRESHOLD,
+              FAIL_THRESHOLD,
+              Guardrails.offsetRows,
+              Guardrails::setOffsetRowsThreshold,
+              Guardrails::getOffsetRowsWarnThreshold,
+              Guardrails::getOffsetRowsFailThreshold);
+    }
+
+    @Test
+    public void testOffset() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c1 int, c2 int, v int, PRIMARY KEY (k, c1, c2))");
+
+        testGuardrail("SELECT * FROM %s LIMIT 100 OFFSET %d");
+        testGuardrail("SELECT * FROM %s PER PARTITION LIMIT 3 LIMIT 100 OFFSET %d");
+        testGuardrail("SELECT * FROM %s WHERE k = 0 GROUP BY k, c1 LIMIT 100 OFFSET %d");
+        testGuardrail("SELECT k, c1, c2, sum(v) FROM %s WHERE k = 0 GROUP BY k, c1 LIMIT 100 OFFSET %d");
+        testGuardrail("SELECT k, c1, c2, sum(v) FROM %s WHERE k = 0 LIMIT 100 OFFSET %d");
+    }
+
+    @Test
+    public void testExcludedUsers() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c1 int, c2 int, v int, PRIMARY KEY (k, c1, c2))");
+        int offset = FAIL_THRESHOLD + 1;
+        testExcludedUsers(() -> formatQuery("SELECT * FROM %s LIMIT 100 OFFSET %d", offset),
+                          () -> formatQuery("SELECT * FROM %s PER PARTITION LIMIT 3 LIMIT 100 OFFSET %d", offset),
+                          () -> formatQuery("SELECT * FROM %s WHERE k = 0 GROUP BY k, c1 LIMIT 100 OFFSET %d", offset),
+                          () -> formatQuery("SELECT k, c1, c2, sum(v) FROM %s WHERE k = 0 GROUP BY k, c1 LIMIT 100 OFFSET %d", offset),
+                          () -> formatQuery("SELECT k, c1, c2, sum(v) FROM %s WHERE k = 0 LIMIT 100 OFFSET %d", offset));
+    }
+
+    private void testGuardrail(String query) throws Throwable
+    {
+        assertValid(query, 1);
+        assertValid(query, WARN_THRESHOLD);
+        assertWarns(query, WARN_THRESHOLD + 1);
+        assertWarns(query, FAIL_THRESHOLD);
+        assertFails(query, FAIL_THRESHOLD + 1);
+        assertFails(query, Integer.MAX_VALUE);
+    }
+
+    private String formatQuery(String query, int offset)
+    {
+        return format(query, currentTable(), offset);
+    }
+
+    private void assertValid(String query, int offset) throws Throwable
+    {
+        super.assertValid(formatQuery(query, offset));
+    }
+
+    private void assertWarns(String query, int offset) throws Throwable
+    {
+        assertWarns(formatQuery(query, offset),
+                    format("Select query requested to skip %s rows, this exceeds the warning threshold of %s.",
+                           offset, WARN_THRESHOLD));
+    }
+
+    private void assertFails(String query, int offset) throws Throwable
+    {
+        assertFails(formatQuery(query, offset),
+                    format("Select query requested to skip %s rows, this exceeds the failure threshold of %s.",
+                           offset, FAIL_THRESHOLD));
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailPageSizeTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailPageSizeTest.java
index 68122f2bff9a..021c68c30b66 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailPageSizeTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailPageSizeTest.java
@@ -24,6 +24,7 @@
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.db.ConsistencyLevel;
@@ -136,7 +137,7 @@ private void executeWithPaging(ClientState state, String query, int pageSize)
         QueryOptions options = QueryOptions.create(ConsistencyLevel.ONE,
                                                    Collections.emptyList(),
                                                    false,
-                                                   pageSize,
+                                                   PageSize.inRows(pageSize),
                                                    null,
                                                    null,
                                                    ProtocolVersion.CURRENT,
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailPageWeightTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailPageWeightTest.java
new file mode 100644
index 000000000000..fac583c13cea
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailPageWeightTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.Collections;
+
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.service.ClientWarn;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.BYTES;
+import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.KIBIBYTES;
+
+/**
+ * Tests the guardrail for the page size, {@link Guardrails#pageSize}.
+ */
+public class GuardrailPageWeightTest extends ThresholdTester
+{
+    private static final int PAGE_WEIGHT_WARN_THRESHOLD_KIB = 4;
+    private static final int PAGE_WEIGHT_FAIL_THRESHOLD_KIB = 7;
+
+    public GuardrailPageWeightTest()
+    {
+        super(Math.toIntExact(KIBIBYTES.toBytes(PAGE_WEIGHT_WARN_THRESHOLD_KIB)) + "B",
+                Math.toIntExact(KIBIBYTES.toBytes(PAGE_WEIGHT_FAIL_THRESHOLD_KIB)) + "B",
+              Guardrails.pageWeight,
+              Guardrails::setPageWeightThreshold,
+              Guardrails::getPageWeightWarnThreshold,
+              Guardrails::getPageWeightFailThreshold,
+                bytes -> new DataStorageSpec.LongBytesBound(bytes, BYTES).toString(),
+                size -> new DataStorageSpec.LongBytesBound(size).toBytes(),
+                Integer.MAX_VALUE - 1);
+    }
+
+    @Before
+    public void setupTest()
+    {
+        createTable("CREATE TABLE IF NOT EXISTS %s (k INT, c INT, v TEXT, PRIMARY KEY(k, c))");
+    }
+
+    @Test
+    public void testSelectStatementAgainstPageWeight() throws Throwable
+    {
+        // regular query
+        String query = "SELECT * FROM %s";
+        assertPagingValid(query, 3);
+        assertPagingValid(query, PAGE_WEIGHT_WARN_THRESHOLD_KIB);
+        assertPagingWarns(query, 6);
+        assertPagingWarns(query, PAGE_WEIGHT_FAIL_THRESHOLD_KIB);
+        assertPagingFails(query, 11);
+
+        // aggregation query
+        query = "SELECT COUNT(*) FROM %s WHERE k=0";
+        assertPagingNotSupported(query, 3);
+        assertPagingNotSupported(query, PAGE_WEIGHT_WARN_THRESHOLD_KIB);
+        assertPagingNotSupported(query, 6);
+        assertPagingNotSupported(query, PAGE_WEIGHT_FAIL_THRESHOLD_KIB);
+        assertPagingFails(query, 11);
+
+        // query with limit over thresholds does not affect page weight guardrail
+        query = "SELECT * FROM %s LIMIT 100";
+        assertPagingValid(query, 3);
+        assertPagingValid(query, PAGE_WEIGHT_WARN_THRESHOLD_KIB);
+        assertPagingWarns(query, 6);
+        assertPagingWarns(query, PAGE_WEIGHT_FAIL_THRESHOLD_KIB);
+        assertPagingFails(query, 11);
+
+        // query with limit under thresholds does not affect page weight guardrail
+        query = "SELECT * FROM %s LIMIT 1";
+        assertPagingValid(query, 3);
+        assertPagingValid(query, PAGE_WEIGHT_WARN_THRESHOLD_KIB);
+        assertPagingWarns(query, 6);
+        assertPagingWarns(query, PAGE_WEIGHT_FAIL_THRESHOLD_KIB);
+        assertPagingFails(query, 11);
+    }
+
+    @Test
+    public void testExcludedUsers() throws Throwable
+    {
+        assertPagingIgnored("SELECT * FROM %s", PAGE_WEIGHT_WARN_THRESHOLD_KIB + 1);
+        assertPagingIgnored("SELECT * FROM %s", PAGE_WEIGHT_FAIL_THRESHOLD_KIB + 1);
+    }
+
+    private void assertPagingValid(String query, int pageWeight) throws Throwable
+    {
+        assertValid(() -> executeWithPaging(userClientState, query, pageWeight));
+    }
+
+    private void assertPagingIgnored(String query, int pageWeight) throws Throwable
+    {
+        assertValid(() -> executeWithPaging(superClientState, query, pageWeight));
+        assertValid(() -> executeWithPaging(systemClientState, query, pageWeight));
+    }
+
+    private void assertPagingWarns(String query, int pageWeight) throws Throwable
+    {
+        assertWarns(() -> executeWithPaging(userClientState, query, pageWeight),
+                    format("Query for table %s with page weight %s bytes exceeds warning threshold of %s bytes.",
+                           currentTable(), toBytes(pageWeight), toBytes(PAGE_WEIGHT_WARN_THRESHOLD_KIB)));
+    }
+
+    private void assertPagingFails(String query, int pageWeight) throws Throwable
+    {
+        assertFails(() -> executeWithPaging(userClientState, query, pageWeight),
+                    format("Aborting query for table %s, page weight %s bytes exceeds fail threshold of %s bytes.",
+                           currentTable(), toBytes(pageWeight), toBytes(PAGE_WEIGHT_FAIL_THRESHOLD_KIB)));
+    }
+
+    private void assertPagingNotSupported(String query, int pageWeight) throws Throwable
+    {
+        assertThrows(() -> executeWithPaging(userClientState, query, pageWeight), InvalidRequestException.class,
+                    format("Paging in bytes is not supported for aggregation queries. Please specify the page size in rows."));
+        ClientWarn.instance.resetWarnings();
+        listener.clear();
+    }
+
+    private void executeWithPaging(ClientState state, String query, int pageWeightBytes)
+    {
+        QueryState queryState = new QueryState(state);
+
+        String formattedQuery = formatQuery(query);
+        CQLStatement statement = QueryProcessor.parseStatement(formattedQuery, queryState.getClientState());
+        statement.validate(state);
+
+        QueryOptions options = QueryOptions.create(ConsistencyLevel.ONE,
+                                                   Collections.emptyList(),
+                                                   false,
+                                                   PageSize.inBytes(toBytes(pageWeightBytes)),
+                                                   null,
+                                                   null,
+                                                   ProtocolVersion.CURRENT,
+                                                   KEYSPACE);
+
+        statement.executeLocally(queryState, options);
+    }
+
+    private int toBytes(int kib)
+    {
+        return Math.toIntExact(KIBIBYTES.toBytes(kib));
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailPagingTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailPagingTest.java
new file mode 100644
index 000000000000..8b2931f8902f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailPagingTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.auth.AuthenticatedUser;
+import org.apache.cassandra.config.DataStorageSpec.IntBytesBound;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.cql3.QueryHandler;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.KIBIBYTES;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class GuardrailPagingTest extends GuardrailTester
+{
+    private static final String PARTITION_RANGE_QUERY = "SELECT * FROM %s.%s";
+    private static final String SINGLE_PARTITION_QUERY = "SELECT * FROM %s.%s WHERE k = 5";
+    private static final String MULTI_PARTITION_QUERY = "SELECT * FROM %s.%s WHERE k IN (1, 3, 5)";
+
+    private static int savedPageSizeWarnThreshold;
+    private static int savedPageSizeFailureThreshold;
+    private static IntBytesBound savedPageWeightWarnThreshold;
+    private static IntBytesBound savedPageWeightFailureThreshold;
+
+    private static final IntBytesBound testPageWeightFailureThreshold = new IntBytesBound(5, KIBIBYTES);
+
+    private static final int partitionCount = 10;
+    private static final int rowsPerPartition = 100;
+
+    @Parameterized.Parameters(name = "q={0},size={1}")
+    public static Collection<Object[]> parameters()
+    {
+        return Arrays.asList(new Object[]{ PARTITION_RANGE_QUERY, partitionCount * rowsPerPartition },
+                             new Object[]{ SINGLE_PARTITION_QUERY, rowsPerPartition },
+                             new Object[]{ MULTI_PARTITION_QUERY, 3 * rowsPerPartition });
+    }
+
+    @Parameterized.Parameter(0)
+    public String query;
+
+    @Parameterized.Parameter(1)
+    public int limit;
+
+    @BeforeClass
+    public static void setup()
+    {
+        savedPageSizeWarnThreshold = DatabaseDescriptor.getGuardrailsConfig().getPageSizeWarnThreshold();
+        savedPageSizeFailureThreshold = DatabaseDescriptor.getGuardrailsConfig().getPageSizeFailThreshold();
+        savedPageWeightWarnThreshold = DatabaseDescriptor.getGuardrailsConfig().getPageWeightWarnThreshold();
+        savedPageWeightFailureThreshold = DatabaseDescriptor.getGuardrailsConfig().getPageWeightFailThreshold();
+    }
+
+    @After
+    public void tearDown()
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setPageSizeThreshold(savedPageSizeWarnThreshold, savedPageSizeFailureThreshold);
+        DatabaseDescriptor.getGuardrailsConfig().setPageWeightThreshold(savedPageWeightWarnThreshold, savedPageWeightFailureThreshold);
+    }
+
+    @Before
+    public void setUp() throws Throwable
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setPageWeightThreshold(DatabaseDescriptor.getGuardrailsConfig().getPageWeightWarnThreshold(), testPageWeightFailureThreshold);
+
+        createTable("CREATE TABLE IF NOT EXISTS %s (k INT, c INT, v TEXT, PRIMARY KEY(k, c))");
+
+        for (int i = 0; i < partitionCount; i++)
+            for (int j = 0; j < rowsPerPartition; j++)
+                execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", i, j, "long long test message bla bla bla bla bla bla bla bla bla bla bla");
+    }
+
+    private ResultMessage.Rows selectWithPaging(String query, PageSize pageSize, ClientState clientState) throws InvalidRequestException
+    {
+        QueryOptions options = QueryOptions.create(ConsistencyLevel.LOCAL_QUORUM,
+                                                   Collections.emptyList(),
+                                                   false,
+                                                   pageSize,
+                                                   null,
+                                                   ConsistencyLevel.LOCAL_SERIAL,
+                                                   ProtocolVersion.CURRENT,
+                                                   KEYSPACE);
+
+        clientState.setKeyspace(KEYSPACE);
+        QueryState queryState = new QueryState(clientState);
+
+        QueryHandler.Prepared prepared = QueryProcessor.prepareInternal(String.format(query, KEYSPACE, currentTable()));
+        return (ResultMessage.Rows) prepared.statement.execute(queryState, options, Dispatcher.RequestTime.forImmediateExecution());
+    }
+
+    private ResultMessage.Rows testQueryWithPagedByRows(String query, PageSize pageSize, int rowLimit) throws Throwable
+    {
+        ResultMessage.Rows result = selectWithPaging(query, pageSize, ClientState.forExternalCalls(AuthenticatedUser.ANONYMOUS_USER));
+        Assertions.assertThat(result.result.rows.size()).isLessThan(rowLimit);
+        return result;
+    }
+
+    /**
+     * Test that the number of returned rows per page is silently limited to fit into the guardrail hard limit
+     */
+    @Test
+    public void testPartitionQueryWithPagedByRows() throws Throwable
+    {
+        // ask for more rows per page than can fit with the current guardrail
+        testQueryWithPagedByRows(query, PageSize.inRows(limit), limit);
+    }
+
+    /**
+     * Test that a query throws with page size that is bigger than the guardrail hard limit
+     */
+    @Test(expected = InvalidRequestException.class)
+    public void testQueryWithLargeBytePagesThrows() throws Throwable
+    {
+        testQueryWithPagedByRows(query, PageSize.inBytes(10 * 1024), limit);
+    }
+
+    /**
+     * Test that a query does not throw with page size that is smaller than the guardrail hard limit
+     */
+    @Test
+    public void testQueryWithSmallBytePagesWorks() throws Throwable
+    {
+        int maxPageSize = 2 * 1024;
+        ResultMessage.Rows result = testQueryWithPagedByRows(query, PageSize.inBytes(maxPageSize), limit);
+        // technically incorrect as we compare a size of encoded message to be sent to a client to the page size,
+        // but we can't know the page at this point.
+        assertTrue(ResultMessage.codec.encodedSize(result, ProtocolVersion.CURRENT) < maxPageSize);
+    }
+
+    /**
+     * Test that superusers and internal queries are excluded from the guardrail.
+     */
+    @Test
+    public void testExcludedUsers()
+    {
+        selectWithPaging(query, PageSize.inBytes(10 * 1024), ClientState.forInternalCalls());
+        selectWithPaging(query, PageSize.inBytes(10 * 1024), ClientState.forExternalCalls(new AuthenticatedUser("cassandra")));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailQueryFiltersTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailQueryFiltersTest.java
new file mode 100644
index 000000000000..4dc05334825a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailQueryFiltersTest.java
@@ -0,0 +1,249 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.guardrails;
+
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static java.lang.String.format;
+
+/**
+ * Tests the guardrail for the number of column value filters per SELECT query, {@link Guardrails#queryFilters}.
+ */
+public class GuardrailQueryFiltersTest extends ThresholdTester
+{
+    private static final int WARN_THRESHOLD = 2;
+    private static final int FAIL_THRESHOLD = 4;
+
+    public GuardrailQueryFiltersTest()
+    {
+        super(WARN_THRESHOLD,
+              FAIL_THRESHOLD,
+              Guardrails.queryFilters,
+              Guardrails::setQueryFiltersThreshold,
+              Guardrails::getQueryFiltersWarnThreshold,
+              Guardrails::getQueryFiltersFailThreshold);
+    }
+
+    @Test
+    public void testQueryFilters() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, x text, y text, z text, PRIMARY KEY((k1, k2), c1, c2))");
+
+        String x = createIndex("CREATE CUSTOM INDEX ON %s(x) " +
+                               "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                               "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        String y = createIndex("CREATE CUSTOM INDEX ON %s(y) " +
+                               "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                               "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(z) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+
+        // single column, single expression (analyzed)
+        assertValid("SELECT * FROM %s WHERE x : '1'");
+        assertValid("SELECT * FROM %s WHERE x : '1 2'");
+        assertWarns("SELECT * FROM %s WHERE x : '1 2 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1 2 3 4'", 4);
+        assertFails("SELECT * FROM %s WHERE x : '1 2 3 4 5'", 5);
+        assertFails("SELECT * FROM %s WHERE x : '1 2 3 4 5 6'", 6);
+
+        // single column, single expression (not analyzed)
+        assertValid("SELECT * FROM %s WHERE z = '1'");
+        assertValid("SELECT * FROM %s WHERE z = '1 2'");
+        assertValid("SELECT * FROM %s WHERE z = '1 2 3'");
+        assertValid("SELECT * FROM %s WHERE z = '1 2 3 4'");
+        assertValid("SELECT * FROM %s WHERE z = '1 2 3 4 5'");
+        assertValid("SELECT * FROM %s WHERE z = '1 2 3 4 5 6'");
+
+        // single column, multiple expressions (analyzed, AND)
+        assertValid("SELECT * FROM %s WHERE x : '1' AND x : '2'");
+        assertWarns("SELECT * FROM %s WHERE x : '1' AND x : '2 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1' AND x : '2 3 4'", 4);
+        assertFails("SELECT * FROM %s WHERE x : '1' AND x : '2 3 4 5'", 5);
+        assertFails("SELECT * FROM %s WHERE x : '1' AND x : '2 3 4 5 6'", 6);
+
+        // single column, multiple expressions (analyzed, OR)
+        assertValid("SELECT * FROM %s WHERE x : '1' OR x : '2'");
+        assertWarns("SELECT * FROM %s WHERE x : '1' OR x : '2 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1' OR x : '2 3 4'", 4);
+        assertFails("SELECT * FROM %s WHERE x : '1' OR x : '2 3 4 5'", 5);
+        assertFails("SELECT * FROM %s WHERE x : '1' OR x : '2 3 4 5 6'", 6);
+
+        // multiple columns (analyzed, AND)
+        assertValid("SELECT * FROM %s WHERE x : '1' AND y : '2'");
+        assertWarns("SELECT * FROM %s WHERE x : '1' AND y : '2 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1' AND y : '2 3 4'", 4);
+        assertFails("SELECT * FROM %s WHERE x : '1' AND y : '2 3 4 5'", 5);
+        assertFails("SELECT * FROM %s WHERE x : '1' AND y : '2 3 4 5 6'", 6);
+
+        // multiple columns (analyzed, OR)
+        assertValid("SELECT * FROM %s WHERE x : '1' OR y : '2'");
+        assertWarns("SELECT * FROM %s WHERE x : '1' OR y : '2 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1' OR y : '2 3 4'", 4);
+        assertFails("SELECT * FROM %s WHERE x : '1' OR y : '2 3 4 5'", 5);
+        assertFails("SELECT * FROM %s WHERE x : '1' OR y : '2 3 4 5 6'", 6);
+
+        // multiple columns (analyzed and not analyzed, AND)
+        assertWarns("SELECT * FROM %s WHERE x : '1' AND y : '2' AND z = '3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1' AND y : '2' AND z = '3 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1' AND y : '2 3' AND z = '4'", 4);
+        assertWarns("SELECT * FROM %s WHERE x : '1' AND y : '2 3' AND z = '4 4'", 4);
+        assertFails("SELECT * FROM %s WHERE x : '1' AND y : '2 3 4' AND z = '5'", 5);
+        assertFails("SELECT * FROM %s WHERE x : '1' AND y : '2 3 4' AND z = '5 5'", 5);
+
+        // multiple columns (analyzed and not analyzed, OR)
+        assertWarns("SELECT * FROM %s WHERE x : '1' OR y : '2' OR z = '3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1' OR y : '2' OR z = '3 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE x : '1' OR y : '2 3' OR z = '4'", 4);
+        assertWarns("SELECT * FROM %s WHERE x : '1' OR y : '2 3' OR z = '4 4'", 4);
+        assertFails("SELECT * FROM %s WHERE x : '1' OR y : '2 3 4' OR z = '5'", 5);
+        assertFails("SELECT * FROM %s WHERE x : '1' OR y : '2 3 4' OR z = '5 5'", 5);
+
+        // full partition key restrictions don't count as filters
+        assertValid("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND x : '1'");
+        assertValid("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND x : '1 2'");
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND x : '1 2 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND x : '1 2 3 4'", 4);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND x : '1 2 3 4 5'", 5);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND x : '1 2 3 4 5 6'", 6);
+
+        // partial partition key restrictions do count as filters
+        assertValid("SELECT * FROM %s WHERE k1 = 0 AND x : '1' ALLOW FILTERING");
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND x : '1 2' ALLOW FILTERING", 3);
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND x : '1 2 3' ALLOW FILTERING", 4);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND x : '1 2 3 4' ALLOW FILTERING", 5);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND x : '1 2 3 4 5' ALLOW FILTERING", 6);
+        assertValid("SELECT * FROM %s WHERE k2 = 0 AND x : '1' ALLOW FILTERING");
+        assertWarns("SELECT * FROM %s WHERE k2 = 0 AND x : '1 2' ALLOW FILTERING", 3);
+        assertWarns("SELECT * FROM %s WHERE k2 = 0 AND x : '1 2 3' ALLOW FILTERING", 4);
+        assertFails("SELECT * FROM %s WHERE k2 = 0 AND x : '1 2 3 4' ALLOW FILTERING", 5);
+        assertFails("SELECT * FROM %s WHERE k2 = 0 AND x : '1 2 3 4 5' ALLOW FILTERING", 6);
+
+        // full primary key restrictions don't count as filters
+        assertValid("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND x : '1'");
+        assertValid("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND x : '1 2'");
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND x : '1 2 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND x : '1 2 3 4'", 4);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND x : '1 2 3 4 5'", 5);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND x : '1 2 3 4 5 6'", 6);
+        assertValid("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND c2 = 0 AND x : '1'");
+        assertValid("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND c2 = 0 AND x : '1 2'");
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND c2 = 0 AND x : '1 2 3'", 3);
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND c2 = 0 AND x : '1 2 3 4'", 4);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND c2 = 0 AND x : '1 2 3 4 5'", 5);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c1 = 0 AND c2 = 0 AND x : '1 2 3 4 5 6'", 6);
+
+        // partial primary key restrictions do count as filters
+        assertValid("SELECT * FROM %s WHERE c2 = 0 AND x : '1' ALLOW FILTERING");
+        assertWarns("SELECT * FROM %s WHERE c2 = 0 AND x : '1 2' ALLOW FILTERING", 3);
+        assertWarns("SELECT * FROM %s WHERE c2 = 0 AND x : '1 2 3' ALLOW FILTERING", 4);
+        assertFails("SELECT * FROM %s WHERE c2 = 0 AND x : '1 2 3 4 5' ALLOW FILTERING", 6);
+        assertFails("SELECT * FROM %s WHERE c2 = 0 AND x : '1 2 3 4 5 6' ALLOW FILTERING", 7);
+        assertValid("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c2 = 0 AND x : '1' ALLOW FILTERING");
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c2 = 0 AND x : '1 2' ALLOW FILTERING", 3);
+        assertWarns("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c2 = 0 AND x : '1 2 3' ALLOW FILTERING", 4);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c2 = 0 AND x : '1 2 3 4 5' ALLOW FILTERING", 6);
+        assertFails("SELECT * FROM %s WHERE k1 = 0 AND k2 = 0 AND c2 = 0 AND x : '1 2 3 4 5 6' ALLOW FILTERING", 7);
+
+        // without the analyzed indexes
+        dropIndex("DROP INDEX %s." + x);
+        dropIndex("DROP INDEX %s." + y);
+        assertValid("SELECT * FROM %s WHERE x = '1' ALLOW FILTERING");
+        assertValid("SELECT * FROM %s WHERE x = '1 2' ALLOW FILTERING");
+        assertValid("SELECT * FROM %s WHERE x = '1 2 3' ALLOW FILTERING");
+        assertValid("SELECT * FROM %s WHERE x = '1' AND y = '2' ALLOW FILTERING");
+        assertValid("SELECT * FROM %s WHERE x = '1 2' AND y = '3 4' ALLOW FILTERING");
+        assertValid("SELECT * FROM %s WHERE x = '1 2 3' AND y = '4 5 6' ALLOW FILTERING");
+        assertWarns("SELECT * FROM %s WHERE x = '1' AND y = '2' AND z = '3' ALLOW FILTERING", 3);
+        assertWarns("SELECT * FROM %s WHERE x = '1 2' AND y = '3 4' AND z = '5 6' ALLOW FILTERING", 3);
+        assertWarns("SELECT * FROM %s WHERE x = '1 2 3' AND y = '4 5 6' AND z = '7 8 9' ALLOW FILTERING", 3);
+    }
+
+    @Test
+    public void testQueryFiltersWithIndexAndQueryAnalyzers() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': '{\n" +
+                    "\t\"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"1\", \"maxGramSize\":\"10\"}}," +
+                    "\t\"filters\":[{\"name\":\"lowercase\"}]\n" +
+                    "}'," +
+                    "'query_analyzer': '{\n" +
+                    "\t\"tokenizer\":{\"name\":\"whitespace\"},\n" +
+                    "\t\"filters\":[{\"name\":\"porterstem\"}]\n" +
+                    "}'};");
+
+        // only the query analyzer should be used to calculate the number of filters
+        assertValid("SELECT * FROM %s WHERE v : 'abcdef'");
+        assertValid("SELECT * FROM %s WHERE v : 'abcdef ghijkl'");
+        assertWarns("SELECT * FROM %s WHERE v : 'abcdef ghijkl mnopqr'", 3);
+        assertWarns("SELECT * FROM %s WHERE v : 'abcdef ghijkl mnopqr stuvwx'", 4);
+        assertFails("SELECT * FROM %s WHERE v : 'abcdef ghijkl mnopqr stuvwx xyz'", 5);
+    }
+
+    @Test
+    public void testExcludedUsers() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, x text, y text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(x) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(y) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        testExcludedUsers(() -> "SELECT * FROM %s WHERE x : '1 2 3'",
+                          () -> "SELECT * FROM %s WHERE x : '1 2 3' AND y : '4 5 6'");
+    }
+
+    @Test
+    public void testDisabledGuardrail() throws Throwable
+    {
+        DatabaseDescriptor.getRawConfig().query_filters_warn_threshold = -1;
+        DatabaseDescriptor.getRawConfig().query_filters_fail_threshold = -1;
+
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(v) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        assertValid("SELECT * FROM %s WHERE v : '1'");
+        assertValid("SELECT * FROM %s WHERE v : '1 2 3 4 5 6'");
+    }
+
+    private void assertWarns(String query, int operations) throws Throwable
+    {
+        assertWarns(query,
+                    format("Select query has %s column value filters after analysis, this exceeds the warning threshold of %s.",
+                           operations, WARN_THRESHOLD));
+//                    query);
+    }
+
+    private void assertFails(String query, int operations) throws Throwable
+    {
+        assertFails(query,
+                    format("Select query has %s column value filters after analysis, this exceeds the failure threshold of %s.",
+                           operations, FAIL_THRESHOLD));
+//                    query);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSAIAnnRerankKTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSAIAnnRerankKTest.java
new file mode 100644
index 000000000000..0ae27be470b5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSAIAnnRerankKTest.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import org.junit.After;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static org.junit.Assert.assertEquals;
+
+public class GuardrailSAIAnnRerankKTest extends ThresholdTester
+{
+    private static final int WARN_THRESHOLD = 50;
+    private static final int FAIL_THRESHOLD = 100;
+
+    private int defaultWarnThreshold = DatabaseDescriptor.getGuardrailsConfig().getSaiAnnRerankKWarnThreshold();
+    private int defaultFailThreshold = DatabaseDescriptor.getGuardrailsConfig().getSaiAnnRerankKFailThreshold();
+    private int defaultMaxTopK = CassandraRelevantProperties.SAI_VECTOR_SEARCH_MAX_TOP_K.getInt();
+
+    public GuardrailSAIAnnRerankKTest()
+    {
+        super(WARN_THRESHOLD,
+              FAIL_THRESHOLD,
+              Guardrails.annRerankKMaxValue,
+              Guardrails::setSaiAnnRerankKThreshold,
+              Guardrails::getSaiAnnRerankKWarnThreshold,
+              Guardrails::getSaiAnnRerankKFailThreshold);
+    }
+
+    @After
+    public void after()
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setSaiAnnRerankKThreshold(defaultWarnThreshold, defaultFailThreshold);
+        CassandraRelevantProperties.SAI_VECTOR_SEARCH_MAX_TOP_K.setInt(defaultMaxTopK);
+    }
+
+    @Test
+    public void testDefaultValues()
+    {
+        // Reset to defaults
+        DatabaseDescriptor.getGuardrailsConfig().setSaiAnnRerankKThreshold(defaultWarnThreshold, defaultFailThreshold);
+
+        // Test that default failure threshold is 4 times the max top K
+        int maxTopK = CassandraRelevantProperties.SAI_VECTOR_SEARCH_MAX_TOP_K.getInt();
+        assertEquals(-1, (int) DatabaseDescriptor.getGuardrailsConfig().getSaiAnnRerankKWarnThreshold());
+        assertEquals(4 * maxTopK, (int) DatabaseDescriptor.getGuardrailsConfig().getSaiAnnRerankKFailThreshold());
+    }
+
+    @Test
+    public void testSAIAnnRerankKThresholds() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 3>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        // Test values below and at warning threshold
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': 10}");
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': " + (WARN_THRESHOLD - 1) + '}');
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': " + WARN_THRESHOLD + '}');
+
+        // Test values between warning and failure thresholds
+        assertWarns("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': " + (WARN_THRESHOLD + 1) + '}',
+                    String.format("ANN options specifies rerank_k=%d, this exceeds the warning threshold of %d.",
+                                  WARN_THRESHOLD + 1, WARN_THRESHOLD));
+
+        // Test values at failure threshold (should still warn)
+        assertWarns("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': " + FAIL_THRESHOLD + '}',
+                    String.format("ANN options specifies rerank_k=%d, this exceeds the warning threshold of %d.",
+                                FAIL_THRESHOLD, WARN_THRESHOLD));
+
+        // Test values above failure threshold
+        assertFails("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': " + (FAIL_THRESHOLD + 1) + '}',
+                    String.format("ANN options specifies rerank_k=%d, this exceeds the failure threshold of %d.",
+                                FAIL_THRESHOLD + 1, FAIL_THRESHOLD));
+    }
+
+    @Test
+    public void testDisabledThresholds() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 3>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        // Test with warning threshold disabled
+        int failThreashold = DatabaseDescriptor.getGuardrailsConfig().getSaiAnnRerankKFailThreshold();
+        DatabaseDescriptor.getGuardrailsConfig().setSaiAnnRerankKThreshold(-1, failThreashold);
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': " + (WARN_THRESHOLD + 1) + '}');
+
+        // Test with failure threshold disabled
+        DatabaseDescriptor.getGuardrailsConfig().setSaiAnnRerankKThreshold(-1, -1);
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': " + (FAIL_THRESHOLD + 1) + '}');
+    }
+
+    @Ignore // TODO: e-enable this test when we support negative rerank_k values
+    public void testNegativeRerankK() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 3>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        // Negative rerank_k values should be valid and not trigger warnings
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': -1}");
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {'rerank_k': -1000}");
+    }
+
+    @Test
+    public void testMissingRerankK() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 3>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        // Queries without rerank_k should be valid and not trigger warnings.
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10");
+        assertValid("SELECT * FROM %s ORDER BY v ANN OF [1.0, 1.0, 1.0] LIMIT 10 WITH ann_options = {}");
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSAIIndexesTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSAIIndexesTest.java
new file mode 100644
index 000000000000..85a473fb589a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSAIIndexesTest.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.stream.StreamSupport;
+
+import com.google.common.base.Strings;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertEquals;
+
+public class GuardrailSAIIndexesTest extends GuardrailTester
+{
+    private static int totalExistingIndexes;
+    private int defaultSAIPerTableFailureThreshold;
+    private int defaultSAITotalFailureThreshold;
+
+    @BeforeClass
+    public static void setup()
+    {
+        // Some existing system tables may have indexes (e.g. Paxos.PaxosUncommittedIndex)
+        totalExistingIndexes = totalIndexes();
+    }
+    
+    @Before
+    public void before()
+    {
+        defaultSAIPerTableFailureThreshold = DatabaseDescriptor.getGuardrailsConfig().getStorageAttachedIndexesPerTableFailThreshold();
+        defaultSAITotalFailureThreshold = DatabaseDescriptor.getGuardrailsConfig().getStorageAttachedIndexesTotalFailThreshold();
+        DatabaseDescriptor.getGuardrailsConfig().setStorageAttachedIndexesPerTableThreshold(-1, 1);
+        DatabaseDescriptor.getGuardrailsConfig().setStorageAttachedIndexesTotalThreshold(-1, 2);
+    }
+
+    @After
+    public void after()
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setStorageAttachedIndexesPerTableThreshold(-1, defaultSAIPerTableFailureThreshold);
+        DatabaseDescriptor.getGuardrailsConfig().setStorageAttachedIndexesTotalThreshold(-1, defaultSAITotalFailureThreshold);
+    }
+
+//    @Test
+//    public void testDefaultsOnPrem()
+//    {
+//        testDefaults(false);
+//    }
+//
+//    @Test
+//    public void testDefaultsDBAAS()
+//    {
+//        testDefaults(true);
+//    }
+//
+//    public void testDefaults(boolean dbaas)
+//    {
+//        boolean previous = DatabaseDescriptor.isEmulateDbaasDefaults();
+//        try
+//        {
+//            DatabaseDescriptor.setEmulateDbaasDefaults(dbaas);
+//
+//            GuardrailsConfig config = new GuardrailsConfig();
+//            config.applyConfig();
+//
+//            assertEquals(GuardrailsConfig.DEFAULT_INDEXES_PER_TABLE_THRESHOLD, (int) config.sai_indexes_per_table_failure_threshold);
+//            assertEquals(GuardrailsConfig.DEFAULT_INDEXES_TOTAL_THRESHOLD, (int) config.sai_indexes_total_failure_threshold);
+//        }
+//        finally
+//        {
+//            DatabaseDescriptor.setEmulateDbaasDefaults(previous);
+//        }
+//    }
+
+    @Test
+    public void testPerTableFailureThreshold() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int primary key, v1 int, v2 int, v3 int)");
+        String indexName = createIndex(getCreateIndexStatement("v1"));
+        assertIndexesOnCurrentTable(1);
+
+        assertIndexCreationFails("", "v2");
+        assertIndexCreationFails("custom_index_name", "v2");
+        assertIndexesOnCurrentTable(1);
+
+        // guardrail should not affect indexes of other types
+        assertValid(getDifferentCreateIndexStatement("idx2", "v2"));
+        assertIndexesOnCurrentTable(2);
+
+        // drop the first index, we should be able to create new index again
+        dropIndex(format("DROP INDEX %s.%s", keyspace(), indexName));
+        assertIndexesOnCurrentTable(1);
+
+        execute(getCreateIndexStatement("v3"));
+        assertIndexesOnCurrentTable(2);
+
+        // previous guardrail should not apply to another base table
+        createTable("CREATE TABLE %s (k int primary key, v1 int, v2 int)");
+        assertValid(getCreateIndexStatement("v1"));
+        assertIndexesOnCurrentTable(1);
+
+        assertIndexCreationFails("custom_index_name2", "v2");
+        assertIndexesOnCurrentTable(1);
+    }
+
+    @Test
+    public void testTotalFailureThreshold() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int primary key, v1 int, v2 int)");
+        String indexName = createIndex(getCreateIndexStatement("v1"));
+        assertTotalIndexesOfTheSameType(1);
+        assertGlobalIndexes(1);
+
+        // Create index on new table
+        createTable("CREATE TABLE %s (k int primary key, v1 int, v2 int)");
+        createIndex(getCreateIndexStatement("v1"));
+        assertTotalIndexesOfTheSameType(2);
+        assertGlobalIndexes(2);
+
+        // Trying create new indexes on current table should fail
+        assertIndexCreationFails("", "v2");
+        assertIndexCreationFails("custom_index_name", "v2");
+        assertTotalIndexesOfTheSameType(2);
+        assertGlobalIndexes(2);
+
+        // Trying to create indexes on new table should also fail
+        createTable("CREATE TABLE %s (k int primary key, v1 int, v2 int)");
+        assertIndexCreationFails("", "v1");
+
+        // Trying to create different index type should not fail
+        assertValid(getDifferentCreateIndexStatement("idx2", "v2"));
+        assertTotalIndexesOfTheSameType(2);
+        assertGlobalIndexes(3);
+
+        // drop the first index, we should be able to create new index again
+        dropIndex(format("DROP INDEX %s.%s", keyspace(), indexName));
+        assertTotalIndexesOfTheSameType(1);
+        assertGlobalIndexes(2);
+
+        // Now index creation should succeed
+        createIndex(getCreateIndexStatement("v1"));
+        assertTotalIndexesOfTheSameType(2);
+        assertGlobalIndexes(3);
+    }
+
+    @Test
+    public void testExcludedUsers() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int primary key, v1 int, v2 int)");
+        testExcludedUsers(() -> getCreateIndexStatement("excluded_1", "v1"),
+                          () -> getCreateIndexStatement("excluded_2", "v2"),
+                          () -> "DROP INDEX excluded_1",
+                          () -> "DROP INDEX excluded_2");
+    }
+
+    private void assertIndexesOnCurrentTable(int count)
+    {
+        assertEquals(count, getCurrentColumnFamilyStore().indexManager.listIndexes().size());
+    }
+
+    private void assertGlobalIndexes(int count)
+    {
+        assertEquals(totalExistingIndexes + count, totalIndexes());
+    }
+
+    private static int totalIndexes()
+    {
+        return StreamSupport.stream(Keyspace.all().spliterator(), false).flatMap(k -> k.getColumnFamilyStores().stream()).mapToInt(t -> t.indexManager.listIndexes().size()).sum();
+    }
+
+    private void assertTotalIndexesOfTheSameType(int count)
+    {
+        int totalIndexes = (int) StreamSupport.stream(Keyspace.all().spliterator(), false).flatMap(k -> k.getColumnFamilyStores().stream())
+                                              .flatMap(t -> t.indexManager.listIndexes().stream())
+                                              .filter(i -> i.getIndexMetadata().getIndexClassName().equals(getIndexClassName())).count();
+        assertEquals(count, totalIndexes);
+    }
+
+    private void assertIndexCreationFails(String indexName, String column) throws Throwable
+    {
+        String expectedMessage = String.format("aborting the creation of secondary index %son table %s",
+                                               Strings.isNullOrEmpty(indexName) ? "" : indexName + " ", currentTable());
+        assertFails(getCreateIndexStatement(indexName, column), expectedMessage);
+    }
+
+    protected String getIndexClassName()
+    {
+        return StorageAttachedIndex.class.getName();
+    }
+
+    String getCreateIndexStatement(String column)
+    {
+        return String.format("CREATE CUSTOM INDEX ON %%s (%s) USING '%s'", column, StorageAttachedIndex.class.getCanonicalName());
+    }
+
+    String getCreateIndexStatement(String indexName, String column)
+    {
+        return String.format("CREATE CUSTOM INDEX %s ON %%s (%s) USING '%s'", indexName, column, StorageAttachedIndex.class.getCanonicalName());
+    }
+
+    String getDifferentCreateIndexStatement(String indexName, String column)
+    {
+        return String.format("CREATE INDEX %s ON %%s (%s)", indexName, column);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSASIIndexesPerTableTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSASIIndexesPerTableTest.java
new file mode 100644
index 000000000000..e036d7e5e217
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSASIIndexesPerTableTest.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+import static org.junit.Assert.assertEquals;
+
+public class GuardrailSASIIndexesPerTableTest extends GuardrailTester
+{
+    private int defaultSASIPerTableFailureThreshold;
+
+    @Before
+    public void before()
+    {
+        defaultSASIPerTableFailureThreshold = DatabaseDescriptor.getGuardrailsConfig().getSasiIndexesPerTableFailThreshold();
+    }
+
+    @After
+    public void after()
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setSasiIndexesPerTableThreshold(-1, defaultSASIPerTableFailureThreshold);
+    }
+
+    @Test
+    public void testCreateIndex() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int primary key, v1 int, v2 int)");
+
+        DatabaseDescriptor.setSASIIndexesEnabled(false);
+        assertCreationDisabled("v1");
+        assertNumIndexes(0);
+
+        DatabaseDescriptor.setSASIIndexesEnabled(true);
+        DatabaseDescriptor.getGuardrailsConfig().setSasiIndexesPerTableThreshold(-1, 1);
+        createIndex(getCreateIndexStatement("v1"));
+        assertNumIndexes(1);
+        assertCreationFailed("v2");
+        assertNumIndexes(1);
+    }
+
+    @Test
+    public void testExcludedUsers() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int primary key, v1 int, v2 int)");
+
+        DatabaseDescriptor.getGuardrailsConfig().setSasiIndexesPerTableThreshold(-1, 1);
+        testExcludedUsers(() -> getCreateIndexStatement("excluded_1", "v1"),
+                          () -> getCreateIndexStatement("excluded_2", "v2"),
+                          () -> "DROP INDEX excluded_1",
+                          () -> "DROP INDEX excluded_2");
+    }
+
+    private void assertNumIndexes(int count)
+    {
+        assertEquals(count, getCurrentColumnFamilyStore().indexManager.listIndexes().size());
+    }
+
+    private void assertCreationFailed(String column) throws Throwable
+    {
+        String expectedMessage = String.format("aborting the creation of secondary index on table %s", currentTable());
+        assertFails(getCreateIndexStatement(column), expectedMessage);
+    }
+
+    private void assertCreationDisabled(String column) throws Throwable
+    {
+        String expectedMessage = String.format("failed to create SASI index on table %s", currentTable());
+        assertThrows(() -> execute(getCreateIndexStatement(column)), InvalidRequestException.class, "SASI indexes are disabled. Enable in cassandra.yaml to use.");
+    }
+
+    private String getCreateIndexStatement(String column)
+    {
+        return String.format("CREATE CUSTOM INDEX ON %%s (%s) USING 'org.apache.cassandra.index.sasi.SASIIndex'", column);
+    }
+
+    private String getCreateIndexStatement(String indexName, String column)
+    {
+        return String.format("CREATE CUSTOM INDEX %s ON %%s (%s) USING 'org.apache.cassandra.index.sasi.SASIIndex'", indexName, column);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiFrozenTermSizeTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiFrozenTermSizeTest.java
index 50cdb2ab5855..0cae4fa2ce53 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiFrozenTermSizeTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiFrozenTermSizeTest.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.cassandra.config.DataStorageSpec;
@@ -65,6 +66,7 @@ protected int failThreshold()
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testTuple() throws Throwable
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, t tuple<text, text>)");
@@ -75,6 +77,7 @@ public void testTuple() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testFrozenUDT() throws Throwable
     {
         String udt = createType("CREATE TYPE %s (a text, b text)");
@@ -90,6 +93,7 @@ public void testFrozenUDT() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testFrozenList() throws Throwable
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, fl frozen<list<text>>)");
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiStringTermSizeTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiStringTermSizeTest.java
index 868d736fad35..d84222df9652 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiStringTermSizeTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiStringTermSizeTest.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.cassandra.config.DataStorageSpec;
@@ -65,6 +66,7 @@ protected int failThreshold()
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testCompositePartitionKey() throws Throwable
     {
         createTable("CREATE TABLE %s (k1 int, k2 text, v int, PRIMARY KEY((k1, k2)))");
@@ -75,6 +77,7 @@ public void testCompositePartitionKey() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testSimpleClustering() throws Throwable
     {
         createTable("CREATE TABLE %s (k int, c text, v int, PRIMARY KEY(k, c))");
@@ -85,6 +88,7 @@ public void testSimpleClustering() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testRegularColumn() throws Throwable
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
@@ -95,6 +99,7 @@ public void testRegularColumn() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testStaticColumn() throws Throwable
     {
         createTable("CREATE TABLE %s (k int, c int, s text STATIC, r int, PRIMARY KEY(k, c))");
@@ -107,6 +112,7 @@ public void testStaticColumn() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testList() throws Throwable
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<text>)");
@@ -128,6 +134,7 @@ public void testList() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testBatch() throws Throwable
     {
         createTable("CREATE TABLE %s (k text, c text, r text, s text STATIC, PRIMARY KEY(k, c))");
@@ -146,6 +153,7 @@ public void testBatch() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testCASWithIfNotExistsCondition() throws Throwable
     {
         createTable("CREATE TABLE %s (k text, c text, v text, s text STATIC, PRIMARY KEY(k, c))");
@@ -198,6 +206,7 @@ public void testWarningTermOnBuild()
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testFailingTermOnBuild()
     {
         ByteBuffer oversizedTerm = allocate(failThreshold() + 1);
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiVectorTermSizeTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiVectorTermSizeTest.java
index 6269f4d98c4b..cf852ba9dc92 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiVectorTermSizeTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSaiVectorTermSizeTest.java
@@ -22,6 +22,7 @@
 import java.util.List;
 
 import com.google.common.primitives.Floats;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.cassandra.config.DataStorageSpec;
@@ -67,6 +68,7 @@ protected int failThreshold()
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testWarn() throws Throwable
     {
         int warnDimensions = warnThreshold() / 4; // 4 bytes per dimension
@@ -81,6 +83,7 @@ public void testWarn() throws Throwable
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testFail() throws Throwable
     {
         int failDimensions = failThreshold() / 4; // 4 bytes per dimension
@@ -113,6 +116,7 @@ public void testWarningVectorOnBuild()
     }
 
     @Test
+    @Ignore("CNDB-12704")
     public void testFailingVectorOnBuild()
     {
         int failDimensions = failThreshold() / 4; // 4 bytes per dimension
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSecondaryIndexesPerTableTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSecondaryIndexesPerTableTest.java
index 628aead4eb9f..af25df84725a 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSecondaryIndexesPerTableTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSecondaryIndexesPerTableTest.java
@@ -60,15 +60,11 @@ public void testCreateIndex() throws Throwable
         assertCreateIndexFails("v2", "v2_idx");
         assertCurrentValue(3);
 
-        // 2i guardrail will also affect custom indexes
-        assertCreateCustomIndexFails("v2");
-
         // drop the two first indexes, we should be able to create new indexes again
         dropIndex(format("DROP INDEX %s.%s", keyspace(), "v3_idx"));
         assertCurrentValue(2);
 
         assertCreateIndexWarns("v3", "");
-        assertCreateCustomIndexFails("v4");
         assertCurrentValue(3);
 
         // previous guardrail should not apply to another base table
@@ -113,11 +109,4 @@ private void assertCreateIndexFails(String column, String indexName) throws Thro
                                     Strings.isNullOrEmpty(indexName) ? "" : indexName + " ", currentTable())
         );
     }
-
-    private void assertCreateCustomIndexFails(String column) throws Throwable
-    {
-        assertThresholdFails(format("CREATE CUSTOM INDEX ON %%s (%s) USING 'org.apache.cassandra.index.sasi.SASIIndex'", column),
-                             format("aborting the creation of secondary index on table %s", currentTable())
-        );
-    }
 }
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailTester.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailTester.java
index 58860d828f10..68c1dafe8dca 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailTester.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailTester.java
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.TreeSet;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -45,8 +46,10 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.guardrails.GuardrailEvent.GuardrailEventType;
 import org.apache.cassandra.db.view.View;
@@ -59,6 +62,7 @@
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.assertj.core.api.Assertions;
+import org.awaitility.Awaitility;
 
 import static java.lang.String.format;
 import static org.junit.Assert.assertEquals;
@@ -140,6 +144,14 @@ public void beforeGuardrailTest() throws Throwable
         useSuperUser();
         executeNet(format("CREATE USER IF NOT EXISTS %s WITH PASSWORD '%s'", USERNAME, PASSWORD));
         executeNet(format("GRANT ALL ON KEYSPACE %s TO %s", KEYSPACE, USERNAME));
+
+        // Make sure keyspace permissions have been applied
+        Awaitility.await()
+                  .atMost(10, TimeUnit.SECONDS)
+                  .with()
+                  .pollInterval(500, TimeUnit.MILLISECONDS)
+                  .until(() -> !executeNet("LIST ALL OF " + USERNAME).all().isEmpty());
+
         useUser(USERNAME, PASSWORD);
 
         String useKeyspaceQuery = "USE " + keyspace();
@@ -154,6 +166,7 @@ public void beforeGuardrailTest() throws Throwable
     public void afterGuardrailTest() throws Throwable
     {
         DiagnosticEventService.instance().unsubscribe(listener);
+        closeClientCluster(USERNAME, PASSWORD);
     }
 
     static Guardrails guardrails()
@@ -462,7 +475,9 @@ protected List<String> getWarnings()
         return warnings == null
                ? Collections.emptyList()
                : warnings.stream()
-                         .filter(w -> !w.equals(View.USAGE_WARNING) && !w.equals(SASIIndex.USAGE_WARNING))
+                         .filter(w -> !w.equals(View.USAGE_WARNING) 
+                                 && !w.equals(SASIIndex.USAGE_WARNING)
+                                 && !w.startsWith(SelectStatement.USAGE_WARNING_PAGE_WEIGHT))
                          .collect(Collectors.toList());
     }
 
@@ -508,7 +523,7 @@ protected ResultMessage execute(ClientState state, String query, ConsistencyLeve
         QueryOptions options = QueryOptions.create(cl,
                                                    Collections.emptyList(),
                                                    false,
-                                                   10,
+                                                   PageSize.inRows(10),
                                                    null,
                                                    serialCl,
                                                    ProtocolVersion.CURRENT,
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailTruncateTableTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailTruncateTableTest.java
new file mode 100644
index 000000000000..91c7eddb4b1a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailTruncateTableTest.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+public class GuardrailTruncateTableTest extends GuardrailTester
+{
+    private static boolean truncateTableEnabled;
+
+    public GuardrailTruncateTableTest()
+    {
+        super(Guardrails.dropTruncateTableEnabled);
+    }
+    @BeforeClass
+    public static void setup()
+    {
+        truncateTableEnabled = DatabaseDescriptor.getGuardrailsConfig().getDropTruncateTableEnabled();
+    }
+
+    @AfterClass
+    public static void tearDown()
+    {
+        setGuardrails(truncateTableEnabled);
+    }
+
+    private static void setGuardrails(boolean truncate_table_enabled)
+    {
+        DatabaseDescriptor.getGuardrailsConfig().setDropTruncateTableEnabled(truncate_table_enabled);
+    }
+
+    private void createTestTable()
+    {
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
+
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 1, 0, 2);
+        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 1, 1, 3);
+
+        assertRows(execute("SELECT * FROM %s"), row(1, 0, 2), row(1, 1, 3), row(0, 0, 0), row(0, 1, 1));
+    }
+
+    @Test
+    public void testEnabledTruncateTable() throws Throwable
+    {
+        createTestTable();
+        setGuardrails(true);
+        assertValid("TRUNCATE %s");
+        assertEmpty(execute("SELECT * FROM %s"));
+    }
+
+    @Test
+    public void testDisabledTruncateTable() throws Throwable
+    {
+        createTestTable();
+        setGuardrails(false);
+        assertFails("TRUNCATE %s", "Guardrail drop_truncate_table_enabled violated: DROP and TRUNCATE TABLE functionality is not allowed");
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java
index 8cd56587689c..89763b95afe1 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java
@@ -21,8 +21,11 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
+
 import static java.lang.String.format;
 
 /**
@@ -69,6 +72,12 @@ public GuardrailVectorDimensionsTest()
         "CREATE TABLE %s (k int PRIMARY KEY, v tuple<frozen<tuple<vector<int, %%d>, int>>, int>)"
     );
 
+    @BeforeClass
+    public static void setupClass()
+    {
+        CassandraRelevantProperties.VECTOR_FLOAT_ONLY.setBoolean(false);
+    }
+
     @Test
     public void testCreateTable() throws Throwable
     {
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorTypeEnabledTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorTypeEnabledTest.java
index 38654fa27a7d..2f3b56bf6baa 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorTypeEnabledTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorTypeEnabledTest.java
@@ -21,11 +21,13 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
 
 /**
  * Tests the guardrail for the usage of the vector type, {@link Guardrails#vectorTypeEnabled}.
@@ -44,6 +46,12 @@ public static List<Object[]> params()
         );
     }
 
+    @BeforeClass
+    public static void setupClass()
+    {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
+    }
+
     public GuardrailVectorTypeEnabledTest(boolean enabled)
     {
         super(Guardrails.vectorTypeEnabled);
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailWriteConsistencyLevelsTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailWriteConsistencyLevelsTest.java
index f2ad40fa998b..9f3ffd7642bc 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailWriteConsistencyLevelsTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailWriteConsistencyLevelsTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.db.guardrails;
 
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -165,7 +166,7 @@ private void testLWTQuery(String query, ConsistencyLevel cl) throws Throwable
         disableConsistencyLevels(SERIAL);
         assertFails(query, cl, SERIAL, SERIAL);
         assertValid(query, cl, LOCAL_SERIAL);
-        assertFails(query, cl, null, SERIAL);
+        assertValid(query, cl, null);
 
         disableConsistencyLevels(LOCAL_SERIAL);
         assertValid(query, cl, SERIAL);
@@ -175,7 +176,7 @@ private void testLWTQuery(String query, ConsistencyLevel cl) throws Throwable
         disableConsistencyLevels(SERIAL, LOCAL_SERIAL);
         assertFails(query, cl, SERIAL, SERIAL);
         assertFails(query, cl, LOCAL_SERIAL, LOCAL_SERIAL);
-        assertFails(query, cl, null, SERIAL);
+        assertThrows(query, cl, null, SERIAL);
     }
 
     private void assertValid(String query, ConsistencyLevel cl, ConsistencyLevel serialCl) throws Throwable
@@ -201,6 +202,13 @@ private void assertFails(String query, ConsistencyLevel cl, ConsistencyLevel ser
         assertExcludedUsers(query, cl, serialCl);
     }
 
+    private void assertThrows(String query, ConsistencyLevel cl, ConsistencyLevel serialCl, ConsistencyLevel rejectedCl) throws Throwable
+    {
+        assertThrows(() -> execute(userClientState, query, cl, serialCl),
+                InvalidRequestException.class,
+                    "Serial consistency levels are disallowed by disallowedWriteConsistencies Guardrail");
+    }
+
     private void assertExcludedUsers(String query, ConsistencyLevel cl, ConsistencyLevel serialCl) throws Throwable
     {
         assertValid(() -> execute(superClientState, query, cl, serialCl));
diff --git a/test/unit/org/apache/cassandra/db/guardrails/ThresholdTester.java b/test/unit/org/apache/cassandra/db/guardrails/ThresholdTester.java
index 7f790787333b..c2e5fbdb176d 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/ThresholdTester.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/ThresholdTester.java
@@ -103,6 +103,28 @@ protected ThresholdTester(String warnThreshold,
         disabledValue = null;
     }
 
+    protected ThresholdTester(String warnThreshold,
+                              String failThreshold,
+                              Threshold threshold,
+                              TriConsumer<Guardrails, String, String> setter,
+                              Function<Guardrails, String> warnGetter,
+                              Function<Guardrails, String> failGetter,
+                              Function<Long, String> stringFormatter,
+                              ToLongFunction<String> stringParser,
+                              long maxValue)
+    {
+        super(threshold);
+        this.warnThreshold = stringParser.applyAsLong(warnThreshold);
+        this.failThreshold = stringParser.applyAsLong(failThreshold);
+        this.setter = (g, w, f) -> setter.accept(g,
+                                                 w == null ? null : stringFormatter.apply(w),
+                                                 f == null ? null : stringFormatter.apply(f));
+        this.warnGetter = g -> stringParser.applyAsLong(warnGetter.apply(g));
+        this.failGetter = g -> stringParser.applyAsLong(failGetter.apply(g));
+        this.maxValue = maxValue;
+        disabledValue = null;
+    }
+
     protected long currentValue()
     {
         throw new UnsupportedOperationException();
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/CompositeLifecycleTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/CompositeLifecycleTransactionTest.java
new file mode 100644
index 000000000000..b2383f003d36
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/lifecycle/CompositeLifecycleTransactionTest.java
@@ -0,0 +1,287 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.IntStream;
+
+import com.google.common.util.concurrent.Runnables;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.monitoring.runtime.instrumentation.common.collect.Sets;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.concurrent.Transactional.AbstractTransactional.State;
+
+import static com.google.monitoring.runtime.instrumentation.common.collect.ImmutableSet.copyOf;
+
+public class CompositeLifecycleTransactionTest
+{
+    @BeforeClass
+    public static void setUp()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        CommitLog.instance.start();
+        MockSchema.cleanup();
+    }
+
+    @Test
+    public void testUpdates()
+    {
+        int count = 3;
+        ColumnFamilyStore cfs = MockSchema.newCFS();
+        Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
+        SSTableReader[] readers = readersArray(0, 3, cfs);
+        SSTableReader[] readers2 = readersArray(0, 5, cfs);
+        SSTableReader[] readers3 = readersArray(5, 5 + count, cfs);
+        tracker.addInitialSSTables(copyOf(readers));
+        LifecycleTransaction txn = tracker.tryModify(copyOf(readers), OperationType.UNKNOWN);
+
+        CompositeLifecycleTransaction composite = new CompositeLifecycleTransaction(txn);
+        var partials = IntStream.range(0, count).mapToObj(i -> new PartialLifecycleTransaction(composite)).toArray(PartialLifecycleTransaction[]::new);
+        composite.completeInitialization();
+
+        partials[0].update(readers2[3], false);
+
+        testBadUpdate(partials[2], readers2[0], false);  // same reader && instances
+        testBadUpdate(partials[2], readers2[1], true);  // early open unsupported
+
+        Assert.assertEquals(3, tracker.getView().compacting.size());
+        partials[0].checkpoint();
+        for (int i = 0 ; i < count ; i++)
+            partials[i].update(readers3[i], false);
+
+        for (var partial : partials)
+        {
+            Assert.assertEquals(State.IN_PROGRESS, txn.state());
+            partial.checkpoint();
+            partial.obsoleteOriginals();
+            partial.commit();
+        }
+
+        Assert.assertEquals(State.COMMITTED, txn.state());
+        Assert.assertEquals(0, tracker.getView().compacting.size());
+        var result = new HashSet<SSTableReader>();
+        result.addAll(Arrays.asList(readers3));
+        result.add(readers2[3]);
+        Assert.assertEquals(result, tracker.getView().sstables);
+
+        testThrows(() -> txn.abort());
+        testThrows(() -> txn.prepareToCommit());
+        testThrows(() -> txn.commit());
+
+        testThrows(() -> partials[1].abort());
+        testThrows(() -> partials[2].prepareToCommit());
+        testThrows(() -> partials[0].commit());
+    }
+
+    @Test
+    public void testCommit()
+    {
+        testPartialTransactions(300, true, none(), none());
+    }
+
+    @Test
+    public void testCommitPreserveOriginals()
+    {
+        testPartialTransactions(300, false, none(), none());
+    }
+
+    @Test
+    public void testAbort()
+    {
+        testPartialTransactions(300, true, arr(33), none());
+    }
+
+    @Test
+    public void testAbortAll()
+    {
+        int count = 300;
+        testPartialTransactions(count, true, all(count), none());
+    }
+
+    @Test
+    public void testOnlyClose()
+    {
+        testPartialTransactions(300, true, none(), arr(55));
+    }
+
+    @Test
+    public void testOnlyCloseAll()
+    {
+        int count = 300;
+        testPartialTransactions(count, true, none(), all(count));
+    }
+
+    @Test
+    public void testAbortAndOnlyClose()
+    {
+        testPartialTransactions(300, true, arr(89), arr(98));
+    }
+
+    public void testPartialTransactions(int count, boolean obsoleteOriginals, int[] indexesToAbort, int[] indexesToOnlyClose)
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS();
+        Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
+        SSTableReader[] inputs = readersArray(0, count, cfs);
+        SSTableReader[] outputs = readersArray(count, 2*count, cfs);
+        tracker.addInitialSSTables(copyOf(inputs));
+        LifecycleTransaction txn = tracker.tryModify(copyOf(inputs), OperationType.UNKNOWN);
+
+        CompositeLifecycleTransaction composite = new CompositeLifecycleTransaction(txn);
+        // register partial transactions before we launch committing threads
+        var partials = new PartialLifecycleTransaction[count];
+        for (int i = 0; i < count; ++i)
+            partials[i] = new PartialLifecycleTransaction(composite);
+        composite.completeInitialization();
+
+        var futures = new ArrayList<CompletableFuture<Void>>();
+        for (int i = 0; i < count; ++i)
+        {
+            boolean abort = in(indexesToAbort, i);
+            boolean onlyClose = in(indexesToOnlyClose, i);
+            PartialLifecycleTransaction partial = partials[i];
+            SSTableReader output = outputs[i];
+            Runnable r = onlyClose ? Runnables.doNothing()
+                                   : () ->
+                                     {
+                                         partial.update(output, false);
+                                         partial.checkpoint();
+                                         if (obsoleteOriginals)
+                                             partial.obsoleteOriginals();
+                                         if (abort)
+                                             partial.abort();
+                                         else
+                                         {
+                                             partial.prepareToCommit();
+                                             partial.commit();
+                                         }
+
+                                         testThrows(() -> partial.abort());
+                                         testThrows(() -> partial.prepareToCommit());
+                                         testThrows(() -> partial.commit());
+                                     };
+            futures.add(CompletableFuture.runAsync(r)
+                                         .whenComplete((v, t) -> partial.close()));
+        }
+
+        if (indexesToAbort.length == 0 && indexesToOnlyClose.length == 0)
+        {
+            CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)).join();
+
+            Assert.assertEquals(State.COMMITTED, txn.state());
+            Assert.assertEquals(0, tracker.getView().compacting.size());
+            Assert.assertEquals(obsoleteOriginals ? copyOf(outputs)
+                                                  : Sets.union(copyOf(inputs), copyOf(outputs)),
+                                tracker.getView().sstables);
+        }
+        else
+        {
+            try
+            {
+                CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)).join();
+                // Abort may have happened last; this is okay.
+            }
+            catch(Throwable t)
+            {
+                if (!Throwables.isCausedBy(t, PartialLifecycleTransaction.AbortedException.class))
+                    throw t;
+                // expected path where other tasks are aborted by an exception
+                System.out.println("Got expected exception: " + t);
+            }
+            Assert.assertEquals(State.ABORTED, txn.state());
+            Assert.assertEquals(0, tracker.getView().compacting.size());
+            Assert.assertEquals(copyOf(inputs), tracker.getView().sstables);
+        }
+    }
+
+    private static void testThrows(Runnable r)
+    {
+        boolean failed = false;
+        try
+        {
+            r.run();
+        }
+        catch (Throwable t)
+        {
+            failed = true;
+        }
+        Assert.assertTrue(failed);
+    }
+
+    private static void testBadUpdate(ILifecycleTransaction txn, SSTableReader update, boolean original)
+    {
+        boolean failed = false;
+        try
+        {
+            txn.update(update, original);
+        }
+        catch (Throwable t)
+        {
+            failed = true;
+        }
+        Assert.assertTrue(failed);
+    }
+
+
+    private static SSTableReader[] readersArray(int lb, int ub, ColumnFamilyStore cfs)
+    {
+        List<SSTableReader> readers = new ArrayList<>();
+        for (int i = lb ; i < ub ; i++)
+            readers.add(MockSchema.sstable(i, i, true, cfs));
+        return readers.toArray(SSTableReader[]::new);
+    }
+
+    private int[] arr(int... values)
+    {
+        return values;
+    }
+
+    private int[] none()
+    {
+        return new int[0];
+    }
+
+    private boolean in(int[] arr, int value)
+    {
+        for (int i : arr)
+            if (i == value)
+                return true;
+        return false;
+    }
+
+    private int[] all(int count)
+    {
+        int[] result = new int[count];
+        for (int i = 0 ; i < count ; i++)
+            result[i] = i;
+        return result;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
index 20af2a0c18fe..d6485d505a7d 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
@@ -26,11 +26,10 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
-
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.junit.Assert;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.commitlog.CommitLog;
@@ -164,12 +163,14 @@ public void testSetupDeletionNotification()
     public void testMarkObsolete()
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
-        LogTransaction txnLogs = new LogTransaction(OperationType.UNKNOWN);
+        AbstractLogTransaction txnLogs = ILogTransactionsFactory.instance.createLogTransaction(OperationType.UNKNOWN,
+                                                                                               LifecycleTransaction.newId(),
+                                                                                               cfs.metadata);
         Iterable<SSTableReader> readers = Lists.newArrayList(MockSchema.sstable(1, cfs), MockSchema.sstable(2, cfs));
         Iterable<SSTableReader> readersToKeep = Lists.newArrayList(MockSchema.sstable(3, cfs), MockSchema.sstable(4, cfs));
 
-        List<LogTransaction.Obsoletion> obsoletions = new ArrayList<>();
-        Helpers.prepareForObsoletion(readers, txnLogs, obsoletions, null);
+        List<AbstractLogTransaction.Obsoletion> obsoletions = new ArrayList<>();
+        Helpers.prepareForObsoletion(readers, txnLogs, obsoletions, null, null);
         assertNotNull(obsoletions);
         assertEquals(2, obsoletions.size());
 
@@ -191,7 +192,9 @@ public void testMarkObsolete()
     public void testObsoletionPerformance()
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
-        LogTransaction txnLogs = new LogTransaction(OperationType.UNKNOWN);
+        AbstractLogTransaction txnLogs = ILogTransactionsFactory.instance.createLogTransaction(OperationType.UNKNOWN,
+                                                                                               LifecycleTransaction.newId(),
+                                                                                               cfs.metadata);
         List<SSTableReader> readers = new ArrayList<>();
 
         for (int i = 0; i < 10000; i++)
@@ -200,7 +203,7 @@ public void testObsoletionPerformance()
         }
         long start = System.currentTimeMillis();
 
-        Helpers.prepareForObsoletion(readers.subList(0, 500), txnLogs, new ArrayList<>(),null );
+        Helpers.prepareForObsoletion(readers.subList(0, 500), txnLogs, new ArrayList<>(),null, null);
         txnLogs.finish();
         long time = System.currentTimeMillis() - start;
         assertTrue(time < 20000);
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
index 781a46884d2e..edff5782c9ef 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
@@ -20,6 +20,8 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.junit.After;
@@ -39,6 +41,7 @@
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
 import org.apache.cassandra.utils.concurrent.Transactional.AbstractTransactional.State;
+import org.awaitility.Awaitility;
 
 import static com.google.common.base.Predicates.in;
 import static com.google.common.collect.ImmutableList.copyOf;
@@ -76,7 +79,7 @@ public void restoreIncrementalBackup()
     public void testUpdates() // (including obsoletion)
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
-        Tracker tracker = Tracker.newDummyTracker();
+        Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
         SSTableReader[] readers = readersArray(0, 3, cfs);
         SSTableReader[] readers2 = readersArray(0, 4, cfs);
         SSTableReader[] readers3 = readersArray(0, 4, cfs);
@@ -140,7 +143,7 @@ public void testUpdates() // (including obsoletion)
     public void testCancellation()
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
-        Tracker tracker = Tracker.newDummyTracker();
+        Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
         List<SSTableReader> readers = readers(0, 3, cfs);
         tracker.addInitialSSTables(readers);
         LifecycleTransaction txn = tracker.tryModify(readers, OperationType.UNKNOWN);
@@ -184,7 +187,7 @@ public void testCancellation()
     public void testSplit()
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
-        Tracker tracker = Tracker.newDummyTracker();
+        Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
         List<SSTableReader> readers = readers(0, 4, cfs);
         tracker.addInitialSSTables(readers);
         LifecycleTransaction txn = tracker.tryModify(readers, OperationType.UNKNOWN);
@@ -207,6 +210,19 @@ public void testSplit()
         Assert.assertTrue(failed);
     }
 
+    @Test
+    public void testRescheduleFailedDeletions()
+    {
+        AtomicLong counter = new AtomicLong(0);
+        LogTransaction.failedDeletions.add(counter::incrementAndGet);
+
+        LifecycleTransaction.rescheduleFailedDeletions();
+        Awaitility.await("failed deletion").atMost(10, TimeUnit.SECONDS)
+                  .until(() -> counter.get() == 1);
+
+        Assert.assertEquals(0, LogTransaction.failedDeletions.size());
+    }
+
     private static void testBadUpdate(LifecycleTransaction txn, SSTableReader update, boolean original)
     {
         boolean failed = false;
@@ -251,7 +267,7 @@ private static void testBadCancel(LifecycleTransaction txn, SSTableReader cancel
 
     protected TestableTransaction newTest()
     {
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
         SSTableReader.resetTidying();
         return new TxnTest();
     }
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogReplicationSetTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogReplicationSetTest.java
new file mode 100644
index 000000000000..85dd9f933b47
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/lifecycle/LogReplicationSetTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.ArrayList;
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.assertj.core.api.Assertions;
+import org.mockito.Mockito;
+
+public class LogReplicationSetTest
+{
+    @Test
+    public void shouldThrowIfAppendFailedToAllReplicas() throws Throwable
+    {
+        int nrReplicas = 2;
+        LogReplicaSet replicas = new LogReplicaSet();
+        ArrayList<File> spyFiles = getSpyFiles("testAppendFailedToAll", nrReplicas);
+
+        replicas.addReplicas(spyFiles);
+        spyFiles.forEach(f -> Mockito.when(f.exists()).thenThrow(new RuntimeException()));
+
+        Assertions.assertThatExceptionOfType(RuntimeException.class)
+                  .isThrownBy(() -> replicas.append(LogRecord.makeAbort(System.currentTimeMillis())));
+    }
+
+    @Test
+    public void shouldNotThrowIfAppendFailedToSomeReplicas() throws Throwable
+    {
+        int nrReplicas = 2;
+        LogReplicaSet replicas = new LogReplicaSet();
+        ArrayList<File> spyFiles = getSpyFiles("testAppendFailedToSome", nrReplicas);
+
+        replicas.addReplicas(spyFiles);
+        Mockito.when(spyFiles.get(0).exists()).thenThrow(new RuntimeException());
+    }
+
+    private ArrayList<File> getSpyFiles(String testName, int nrReplicas)
+    {
+        ArrayList<File> files = new ArrayList<>(nrReplicas);
+        for (int i = 0; i < nrReplicas; i++)
+        {
+            files.add(Mockito.spy(createTempFile(testName, i)));
+        }
+        return files;
+    }
+
+    private static File createTempFile(String testName, int id)
+    {
+        String prefix = String.format("%s_%d", testName, id);
+        File dir = new File(FileUtils.getTempDir(), prefix);
+
+        FileUtils.createDirectory(dir);
+        File file = FileUtils.createTempFile(prefix, "tmp", dir);
+
+        file.deleteOnExit();
+        dir.deleteOnExit();
+        return file;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
index c70b0ea0de3b..647762cf8c5b 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
@@ -20,8 +20,11 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.spi.FileSystemProvider;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -29,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
@@ -38,10 +42,13 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Runnables;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
@@ -50,6 +57,7 @@
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTable;
@@ -70,11 +78,17 @@
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.FilterFactory;
 import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
 import org.apache.cassandra.utils.concurrent.Transactional;
+import org.mockito.Mockito;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -95,7 +109,7 @@ public static void setUp()
 
     protected AbstractTransactionalTest.TestableTransaction newTest() throws Exception
     {
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
         SSTableReader.resetTidying();
         return new TxnTest();
     }
@@ -109,7 +123,7 @@ private final static class Transaction extends Transactional.AbstractTransaction
             final File dataFolder;
             final SSTableReader sstableOld;
             final SSTableReader sstableNew;
-            final LogTransaction.SSTableTidier tidier;
+            final AbstractLogTransaction.ReaderTidier tidier;
 
             Transaction(ColumnFamilyStore cfs, LogTransaction txnLogs) throws IOException
             {
@@ -121,7 +135,7 @@ private final static class Transaction extends Transactional.AbstractTransaction
 
                 assertNotNull(txnLogs);
                 assertNotNull(txnLogs.id());
-                assertEquals(OperationType.COMPACTION, txnLogs.type());
+                assertEquals(OperationType.COMPACTION, txnLogs.opType());
 
                 txnLogs.trackNew(sstableNew);
                 tidier = txnLogs.obsoleted(sstableOld);
@@ -132,7 +146,7 @@ protected Throwable doCommit(Throwable accumulate)
             {
                 sstableOld.markObsolete(tidier);
                 sstableOld.selfRef().release();
-                LogTransaction.waitForDeletions();
+                LifecycleTransaction.waitForDeletions();
 
                 Throwable ret = txnLogs.commit(accumulate);
 
@@ -142,8 +156,8 @@ protected Throwable doCommit(Throwable accumulate)
 
             protected Throwable doAbort(Throwable accumulate)
             {
-                tidier.abort();
-                LogTransaction.waitForDeletions();
+                accumulate = tidier.abort(accumulate);
+                LifecycleTransaction.waitForDeletions();
 
                 Throwable ret = txnLogs.abort(accumulate);
 
@@ -159,8 +173,8 @@ protected void doPrepare()
 
             void assertInProgress() throws Exception
             {
-                assertFiles(dataFolder.path(), Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths(),
-                                                                                sstableOld.getAllFilePaths(),
+                assertFiles(dataFolder.path(), Sets.newHashSet(Iterables.concat(getAllFilePaths(sstableNew),
+                                                                                getAllFilePaths(sstableOld),
                                                                                 txnLogs.logFilePaths())));
             }
 
@@ -170,12 +184,12 @@ void assertPrepared() throws Exception
 
             void assertAborted() throws Exception
             {
-                assertFiles(dataFolder.path(), new HashSet<>(sstableOld.getAllFilePaths()));
+                assertFiles(dataFolder.path(), getAllFilePaths(sstableOld));
             }
 
             void assertCommitted() throws Exception
             {
-                assertFiles(dataFolder.path(), new HashSet<>(sstableNew.getAllFilePaths()));
+                assertFiles(dataFolder.path(), getAllFilePaths(sstableNew));
             }
         }
 
@@ -188,7 +202,7 @@ private TxnTest() throws IOException
 
         private TxnTest(ColumnFamilyStore cfs) throws IOException
         {
-            this(cfs, new LogTransaction(OperationType.COMPACTION));
+            this(cfs, createLogTransaction(OperationType.COMPACTION, cfs.metadata));
         }
 
         private TxnTest(ColumnFamilyStore cfs, LogTransaction txnLogs) throws IOException
@@ -231,7 +245,7 @@ public void testUntrack() throws Throwable
         SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
 
         // complete a transaction without keep the new files since they were untracked
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstableNew);
@@ -241,9 +255,9 @@ public void testUntrack() throws Throwable
 
         sstableNew.selfRef().release();
         Thread.sleep(1);
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
 
-        assertFiles(dataFolder.path(), Collections.<String>emptySet());
+        assertFiles(dataFolder.path(), Collections.<File>emptySet());
     }
 
     @Test
@@ -260,7 +274,7 @@ public void testUntrackIdenticalLogFilesOnDisk() throws Throwable
                                                                                   (log) -> log.obsoleted(sstable2),
                                                                                   (log) -> log.txnFile().addAll(LogRecord.Type.ADD, Collections.singleton(sstable2))))
         {
-            try (LogTransaction log = new LogTransaction(OperationType.COMPACTION))
+            try (LogTransaction log = new LogTransaction(OperationType.COMPACTION, LifecycleTransaction.newId()))
             {
                 log.trackNew(sstable1); // creates a log file in datadir1
                 log.untrackNew(sstable1); // removes sstable1 from `records`, but still on disk & in `onDiskRecords`
@@ -274,7 +288,7 @@ public void testUntrackIdenticalLogFilesOnDisk() throws Throwable
         sstable1.selfRef().release();
         sstable2.selfRef().release();
         Thread.sleep(1);
-        LogTransaction.waitForDeletions();
+        FBUtilities.waitOnFuture(ScheduledExecutors.nonPeriodicTasks.schedule(Runnables.doNothing(), 0, TimeUnit.MILLISECONDS));
     }
 
     @Test
@@ -286,14 +300,14 @@ public void testCommitSameDesc() throws Throwable
         SSTableReader sstableOld2 = sstable(dataFolder, cfs, 0, 256);
         SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstableNew);
 
         sstableOld1.setReplaced();
 
-        LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld2);
+        AbstractLogTransaction.ReaderTidier tidier = log.obsoleted(sstableOld2);
         assertNotNull(tidier);
 
         log.finish();
@@ -303,7 +317,7 @@ public void testCommitSameDesc() throws Throwable
         sstableOld1.selfRef().release();
         sstableOld2.selfRef().release();
 
-        assertFiles(dataFolder.path(), new HashSet<>(sstableNew.getAllFilePaths()));
+        assertFiles(dataFolder.path(), new HashSet<>(getAllFilePaths(sstableNew)));
 
         sstableNew.selfRef().release();
     }
@@ -315,13 +329,13 @@ public void testCommitOnlyNew() throws Throwable
         File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
         SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstable);
         log.finish();
 
-        assertFiles(dataFolder.path(), new HashSet<>(sstable.getAllFilePaths()));
+        assertFiles(dataFolder.path(), getAllFilePaths(sstable));
 
         sstable.selfRef().release();
     }
@@ -333,10 +347,10 @@ public void testCommitOnlyOld() throws Throwable
         File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
         SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
-        LogTransaction.SSTableTidier tidier = log.obsoleted(sstable);
+        AbstractLogTransaction.ReaderTidier tidier = log.obsoleted(sstable);
         assertNotNull(tidier);
 
         log.finish();
@@ -363,10 +377,10 @@ public void testCommitMultipleFolders() throws Throwable
                                      sstable(dataFolder2, cfs, 3, 128)
         };
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
-        LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
+        AbstractLogTransaction.ReaderTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
 
         log.trackNew(sstables[1]);
         log.trackNew(sstables[3]);
@@ -377,10 +391,10 @@ public void testCommitMultipleFolders() throws Throwable
         sstables[2].markObsolete(tidiers[1]);
 
         Arrays.stream(sstables).forEach(s -> s.selfRef().release());
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
 
-        assertFiles(dataFolder1.path(), new HashSet<>(sstables[1].getAllFilePaths()));
-        assertFiles(dataFolder2.path(), new HashSet<>(sstables[3].getAllFilePaths()));
+        assertFiles(dataFolder1.path(), getAllFilePaths(sstables[1]));
+        assertFiles(dataFolder2.path(), getAllFilePaths(sstables[3]));
     }
 
     @Test
@@ -390,7 +404,7 @@ public void testAbortOnlyNew() throws Throwable
         File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
         SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstable);
@@ -408,18 +422,18 @@ public void testAbortOnlyOld() throws Throwable
         File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
         SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
-        LogTransaction.SSTableTidier tidier = log.obsoleted(sstable);
+        AbstractLogTransaction.ReaderTidier tidier = log.obsoleted(sstable);
         assertNotNull(tidier);
 
-        tidier.abort();
+        tidier.abort(null);
         log.abort();
 
         sstable.selfRef().release();
 
-        assertFiles(dataFolder.path(), new HashSet<>(sstable.getAllFilePaths()));
+        assertFiles(dataFolder.path(), getAllFilePaths(sstable));
     }
 
     @Test
@@ -439,22 +453,22 @@ public void testAbortMultipleFolders() throws Throwable
                                      sstable(dataFolder2, cfs, 3, 128)
         };
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
-        LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
+        AbstractLogTransaction.ReaderTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
 
         log.trackNew(sstables[1]);
         log.trackNew(sstables[3]);
 
-        Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::abort);
+        Arrays.stream(tidiers).forEach(tider -> tider.abort(null));
         log.abort();
 
         Arrays.stream(sstables).forEach(s -> s.selfRef().release());
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
 
-        assertFiles(dataFolder1.path(), new HashSet<>(sstables[0].getAllFilePaths()));
-        assertFiles(dataFolder2.path(), new HashSet<>(sstables[2].getAllFilePaths()));
+        assertFiles(dataFolder1.path(), getAllFilePaths(sstables[0]));
+        assertFiles(dataFolder2.path(), getAllFilePaths(sstables[2]));
     }
 
 
@@ -467,13 +481,13 @@ public void testRemoveUnfinishedLeftovers_abort() throws Throwable
         SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
 
         // simulate tracking sstables with a failed transaction (new log file NOT deleted)
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstableNew);
-        LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
+        AbstractLogTransaction.ReaderTidier tidier = log.obsoleted(sstableOld);
 
-        Set<File> tmpFiles = sstableNew.getAllFilePaths().stream().map(File::new).collect(Collectors.toSet());
+        Set<File> tmpFiles = getAllFilePaths(sstableNew);
 
         sstableNew.selfRef().release();
         sstableOld.selfRef().release();
@@ -481,17 +495,17 @@ public void testRemoveUnfinishedLeftovers_abort() throws Throwable
         assertEquals(tmpFiles, getTemporaryFiles(sstableNew.descriptor.directory));
 
         // normally called at startup
-        LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
+        LifecycleTransaction.removeUnfinishedLeftovers(cfs.metadata());
 
         // sstableOld should be only table left
         Directories directories = new Directories(cfs.metadata());
         Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
         assertEquals(1, sstables.size());
 
-        assertFiles(dataFolder.path(), new HashSet<>(sstableOld.getAllFilePaths()));
+        assertFiles(dataFolder.path(), getAllFilePaths(sstableOld));
 
         // complete the transaction before releasing files
-        tidier.run();
+        tidier.commit();
         log.close();
     }
 
@@ -504,16 +518,16 @@ public void testRemoveUnfinishedLeftovers_commit() throws Throwable
         SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
 
         // simulate tracking sstables with a committed transaction (new log file deleted)
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstableNew);
-        LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
+        AbstractLogTransaction.ReaderTidier tidier = log.obsoleted(sstableOld);
 
         //Fake a commit
         log.txnFile().commit();
 
-        Set<File> tmpFiles = sstableOld.getAllFilePaths().stream().map(File::new).collect(Collectors.toSet());
+        Set<File> tmpFiles = getAllFilePaths(sstableOld);
 
         sstableNew.selfRef().release();
         sstableOld.selfRef().release();
@@ -521,17 +535,17 @@ public void testRemoveUnfinishedLeftovers_commit() throws Throwable
         assertEquals(tmpFiles, getTemporaryFiles(sstableOld.descriptor.directory));
 
         // normally called at startup
-        LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
+        LifecycleTransaction.removeUnfinishedLeftovers(cfs.metadata());
 
         // sstableNew should be only table left
         Directories directories = new Directories(cfs.metadata());
         Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
         assertEquals(1, sstables.size());
 
-        assertFiles(dataFolder.path(), new HashSet<>(sstableNew.getAllFilePaths()));
+        assertFiles(dataFolder.path(), getAllFilePaths(sstableNew));
 
         // complete the transaction to avoid LEAK errors
-        tidier.run();
+        tidier.commit();
         assertNull(log.complete(null));
     }
 
@@ -552,10 +566,10 @@ public void testRemoveUnfinishedLeftovers_commit_multipleFolders() throws Throwa
                                      sstable(dataFolder2, cfs, 3, 128)
         };
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
-        LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
+        AbstractLogTransaction.ReaderTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
 
         log.trackNew(sstables[1]);
         log.trackNew(sstables[3]);
@@ -569,20 +583,18 @@ public void testRemoveUnfinishedLeftovers_commit_multipleFolders() throws Throwa
         Arrays.stream(sstables).forEach(s -> s.selfRef().release());
 
         // test listing
-        assertEquals(sstables[0].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()),
-                            getTemporaryFiles(dataFolder1));
-        assertEquals(sstables[2].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()),
-                            getTemporaryFiles(dataFolder2));
+        assertEquals(getAllFilePaths(sstables[0]), getTemporaryFiles(dataFolder1));
+        assertEquals(getAllFilePaths(sstables[2]), getTemporaryFiles(dataFolder2));
 
         // normally called at startup
-        assertTrue(LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
+        assertTrue(LifecycleTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
 
         // new tables should be only table left
-        assertFiles(dataFolder1.path(), new HashSet<>(sstables[1].getAllFilePaths()));
-        assertFiles(dataFolder2.path(), new HashSet<>(sstables[3].getAllFilePaths()));
+        assertFiles(dataFolder1.path(), getAllFilePaths(sstables[1]));
+        assertFiles(dataFolder2.path(), getAllFilePaths(sstables[3]));
 
         // complete the transaction to avoid LEAK errors
-        Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
+        Arrays.stream(tidiers).forEach(AbstractLogTransaction.ReaderTidier::commit);
         assertNull(log.complete(null));
     }
 
@@ -603,10 +615,10 @@ public void testRemoveUnfinishedLeftovers_abort_multipleFolders() throws Throwab
                                      sstable(dataFolder2, cfs, 3, 128)
         };
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
-        LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
+        AbstractLogTransaction.ReaderTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
 
         log.trackNew(sstables[1]);
         log.trackNew(sstables[3]);
@@ -620,23 +632,30 @@ public void testRemoveUnfinishedLeftovers_abort_multipleFolders() throws Throwab
         Arrays.stream(sstables).forEach(s -> s.selfRef().release());
 
         // test listing
-        assertEquals(sstables[1].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()),
-                            getTemporaryFiles(dataFolder1));
-        assertEquals(sstables[3].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()),
-                            getTemporaryFiles(dataFolder2));
+        assertEquals(getAllFilePaths(sstables[1]), getTemporaryFiles(dataFolder1));
+        assertEquals(getAllFilePaths(sstables[3]), getTemporaryFiles(dataFolder2));
 
         // normally called at startup
-        assertTrue(LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
+        assertTrue(LifecycleTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
 
         // old tables should be only table left
-        assertFiles(dataFolder1.path(), new HashSet<>(sstables[0].getAllFilePaths()));
-        assertFiles(dataFolder2.path(), new HashSet<>(sstables[2].getAllFilePaths()));
+        assertFiles(dataFolder1.path(), getAllFilePaths(sstables[0]));
+        assertFiles(dataFolder2.path(), getAllFilePaths(sstables[2]));
 
         // complete the transaction to avoid LEAK errors
-        Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
+        Arrays.stream(tidiers).forEach(AbstractLogTransaction.ReaderTidier::commit);
         assertNull(log.complete(null));
     }
 
+    private static LogTransaction createLogTransaction(OperationType type, TableMetadataRef metadata)
+    {
+        LogTransaction txn = (LogTransaction) ILogTransactionsFactory.instance.createLogTransaction(type,
+                                                                                                    LifecycleTransaction.newId(),
+                                                                                                    metadata);
+        assertEquals(type, txn.opType());
+        return txn;
+    }
+
     @Test
     public void testRemoveUnfinishedLeftovers_multipleFolders_mismatchedFinalRecords() throws Throwable
     {
@@ -778,10 +797,10 @@ private static void testRemoveUnfinishedLeftovers_multipleFolders_errorCondition
                                      sstable(dataFolder2, cfs, 3, 128)
         };
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
-        LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
+        AbstractLogTransaction.ReaderTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
 
         log.trackNew(sstables[1]);
         log.trackNew(sstables[3]);
@@ -792,29 +811,29 @@ private static void testRemoveUnfinishedLeftovers_multipleFolders_errorCondition
         Arrays.stream(sstables).forEach(s -> s.selfRef().release());
 
         // if shouldCommit is true then it should remove the leftovers and return true, false otherwise
-        assertEquals(shouldCommit, LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
-        LogTransaction.waitForDeletions();
+        assertEquals(shouldCommit, LifecycleTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
+        LifecycleTransaction.waitForDeletions();
 
         if (shouldCommit)
         {
             // only new sstables should still be there
-            assertFiles(dataFolder1.path(), new HashSet<>(sstables[1].getAllFilePaths()));
-            assertFiles(dataFolder2.path(), new HashSet<>(sstables[3].getAllFilePaths()));
+            assertFiles(dataFolder1.path(), getAllFilePaths(sstables[1]));
+            assertFiles(dataFolder2.path(), getAllFilePaths(sstables[3]));
         }
         else
         {
             // all files should still be there
-            assertFiles(dataFolder1.path(), Sets.newHashSet(Iterables.concat(sstables[0].getAllFilePaths(),
-                                                                             sstables[1].getAllFilePaths(),
+            assertFiles(dataFolder1.path(), Sets.newHashSet(Iterables.concat(getAllFilePaths(sstables[0]),
+                                                                             getAllFilePaths(sstables[1]),
                                                                              Collections.singleton(log.logFilePaths().get(0)))));
-            assertFiles(dataFolder2.path(), Sets.newHashSet(Iterables.concat(sstables[2].getAllFilePaths(),
-                                                                             sstables[3].getAllFilePaths(),
+            assertFiles(dataFolder2.path(), Sets.newHashSet(Iterables.concat(getAllFilePaths(sstables[2]),
+                                                                             getAllFilePaths(sstables[3]),
                                                                              Collections.singleton(log.logFilePaths().get(1)))));
         }
 
 
         // complete the transaction to avoid LEAK errors
-        Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
+        Arrays.stream(tidiers).forEach(AbstractLogTransaction.ReaderTidier::commit);
         log.txnFile().commit(); // just anything to make sure transaction tidier will finish
         assertNull(log.complete(null));
     }
@@ -830,7 +849,7 @@ public void testGetTemporaryFiles() throws IOException
         assertNotNull(tmpFiles);
         assertEquals(0, tmpFiles.size());
 
-        try(LogTransaction log = new LogTransaction(OperationType.WRITE))
+        try(LogTransaction log = createLogTransaction(OperationType.WRITE, cfs.metadata))
         {
             Directories directories = new Directories(cfs.metadata());
 
@@ -846,7 +865,7 @@ public void testGetTemporaryFiles() throws IOException
             File[] afterSecondSSTable = dataFolder.tryList(pathname -> !pathname.isDirectory());
 
             int numNewFiles = afterSecondSSTable.length - beforeSecondSSTable.length;
-            assertEquals(numNewFiles - 1, sstable2.getAllFilePaths().size()); // new files except for transaction log file
+            assertEquals(numNewFiles - 1, getAllFilePaths(sstable2).size()); // new files except for transaction log file
 
             tmpFiles = getTemporaryFiles(dataFolder);
             assertNotNull(tmpFiles);
@@ -899,9 +918,9 @@ public void testGetTemporaryFilesMultipleFolders() throws IOException
         };
 
         // they should all have the same number of files since they are created in the same way
-        int numSStableFiles = sstables[0].getAllFilePaths().size();
+        int numSStableFiles = getAllFilePaths(sstables[0]).size();
 
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         for (File dataFolder : new File[] {dataFolder1, dataFolder2})
@@ -911,7 +930,7 @@ public void testGetTemporaryFilesMultipleFolders() throws IOException
             assertEquals(0, tmpFiles.size());
         }
 
-        LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
+        AbstractLogTransaction.ReaderTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
 
         log.trackNew(sstables[1]);
         log.trackNew(sstables[3]);
@@ -936,7 +955,7 @@ public void testGetTemporaryFilesMultipleFolders() throws IOException
         sstables[2].markObsolete(tidiers[1]);
 
         Arrays.stream(sstables).forEach(s -> s.selfRef().release());
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
 
         for (File dataFolder : new File[] {dataFolder1, dataFolder2})
         {
@@ -975,11 +994,11 @@ public void testWrongChecksumLastLineMissingFile() throws IOException
     {
         testCorruptRecord((t, s) ->
                           { // Fake a commit with invalid checksum and also delete one of the old files
-                              for (String filePath : s.getAllFilePaths())
+                              for (File filePath : getAllFilePaths(s))
                               {
-                                  if (filePath.endsWith("Data.db"))
+                                  if (filePath.name().endsWith("Data.db"))
                                   {
-                                      assertTrue(FileUtils.delete(filePath));
+                                      assertTrue(filePath.tryDelete());
                                       assertNull(t.txnFile().syncDirectory(null));
                                       break;
                                   }
@@ -1042,7 +1061,24 @@ public void testUnparsableFirstRecord() throws IOException
         }), false);
     }
 
+    @Test
+    public void testUnparsableFirstRecordThrows()
+    {
+        assertThatThrownBy(() -> {
+            testCorruptRecord((t, s) -> t.logFiles().forEach(f -> {
+                List<String> lines = FileUtils.readLines(f);
+                lines.add(0, "add:[a,b,c][12345678]");
+                FileUtils.replace(f, lines.toArray(new String[lines.size()]));
+            }), false, Directories.OnTxnErr.THROW);
+        }).hasCauseInstanceOf(LogTransaction.CorruptTransactionLogException.class);
+    }
+
     private static void testCorruptRecord(BiConsumer<LogTransaction, SSTableReader> modifier, boolean isRecoverable) throws IOException
+    {
+        testCorruptRecord(modifier, isRecoverable, Directories.OnTxnErr.IGNORE);
+    }
+
+    private static void testCorruptRecord(BiConsumer<LogTransaction, SSTableReader> modifier, boolean isRecoverable, Directories.OnTxnErr onTxnErr) throws IOException
     {
         ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
         File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
@@ -1050,11 +1086,11 @@ private static void testCorruptRecord(BiConsumer<LogTransaction, SSTableReader>
         SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
 
         // simulate tracking sstables with a committed transaction except the checksum will be wrong
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstableNew);
-        LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
+        AbstractLogTransaction.ReaderTidier tidier = log.obsoleted(sstableOld);
 
         // Modify the transaction log or disk state for sstableOld
         modifier.accept(log, sstableOld);
@@ -1069,18 +1105,18 @@ private static void testCorruptRecord(BiConsumer<LogTransaction, SSTableReader>
         sstableNew.selfRef().release();
 
         // The files on disk, for old files make sure to exclude the files that were deleted by the modifier
-        Set<String> newFiles = sstableNew.getAllFilePaths().stream().collect(Collectors.toSet());
-        Set<String> oldFiles = sstableOld.getAllFilePaths().stream().filter(p -> new File(p).exists()).collect(Collectors.toSet());
+        Set<File> newFiles = getAllFilePaths(sstableNew);
+        Set<File> oldFiles = getAllFilePaths(sstableOld, true);
 
         //This should filter as in progress since the last record is corrupt
-        assertFiles(newFiles, getTemporaryFiles(dataFolder));
+        assertFiles(newFiles, getTemporaryFiles(dataFolder, onTxnErr));
         assertFiles(oldFiles, getFinalFiles(dataFolder));
 
         if (isRecoverable)
         { // the corruption is recoverable but the commit record is unreadable so the transaction is still in progress
 
             //This should remove new files
-            LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
+            LifecycleTransaction.removeUnfinishedLeftovers(cfs.metadata());
 
             // make sure to exclude the old files that were deleted by the modifier
             assertFiles(dataFolder.path(), oldFiles);
@@ -1089,7 +1125,7 @@ private static void testCorruptRecord(BiConsumer<LogTransaction, SSTableReader>
         { // if an intermediate line was also modified, it should ignore the tx log file
 
             //This should not remove any files
-            LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
+            LifecycleTransaction.removeUnfinishedLeftovers(cfs.metadata());
 
             assertFiles(dataFolder.path(), Sets.newHashSet(Iterables.concat(newFiles,
                                                                             oldFiles,
@@ -1097,7 +1133,31 @@ private static void testCorruptRecord(BiConsumer<LogTransaction, SSTableReader>
         }
 
         // make sure to run the tidier to avoid any leaks in the logs
-        tidier.run();
+        tidier.commit();
+    }
+
+    @Test
+    public void testDeleteNonExistingFile()
+    {
+        File nonExisting = new File("a/b/c.txt");
+        Assert.assertFalse(nonExisting.exists());
+        LogTransaction.delete(nonExisting);
+    }
+
+    @Test
+    public void testDeleteWithIOException() throws IOException
+    {
+        File file = Mockito.mock(File.class);
+        Path path = Mockito.mock(Path.class);
+        FileSystem fs = Mockito.mock(FileSystem.class);
+        FileSystemProvider fsp = Mockito.mock(FileSystemProvider.class);
+
+        Mockito.when(file.toPath()).thenReturn(path);
+        Mockito.when(path.getFileSystem()).thenReturn(fs);
+        Mockito.when(fs.provider()).thenReturn(fsp);
+        Mockito.doThrow(new IOException("mock exception")).when(fsp).delete(path);
+
+        Assert.assertThrows(FSWriteError.class, () -> LogTransaction.delete(file));
     }
 
     @Test
@@ -1106,10 +1166,10 @@ public void testObsoletedDataFileUpdateTimeChanged() throws IOException
         testObsoletedFilesChanged(sstable ->
                                   {
                                       // increase the modification time of the Data file
-                                      for (String filePath : sstable.getAllFilePaths())
+                                      for (File filePath : getAllFilePaths(sstable))
                                       {
-                                          if (filePath.endsWith("Data.db"))
-                                              assertTrue(new File(filePath).trySetLastModified(System.currentTimeMillis() + 60000)); //one minute later
+                                          if (filePath.name().endsWith("Data.db"))
+                                              assertTrue(filePath.trySetLastModified(System.currentTimeMillis() + 60000)); //one minute later
                                       }
                                   });
     }
@@ -1122,11 +1182,11 @@ private static void testObsoletedFilesChanged(Consumer<SSTableReader> modifier)
         SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
 
         // simulate tracking sstables with a committed transaction except the checksum will be wrong
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstableNew);
-        LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
+        AbstractLogTransaction.ReaderTidier tidier = log.obsoleted(sstableOld);
 
         //modify the old sstable files
         modifier.accept(sstableOld);
@@ -1135,12 +1195,11 @@ private static void testObsoletedFilesChanged(Consumer<SSTableReader> modifier)
         log.txnFile().commit();
 
         //This should not remove the old files
-        LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
+        LifecycleTransaction.removeUnfinishedLeftovers(cfs.metadata());
 
-        assertFiles(dataFolder.path(), Sets.newHashSet(Iterables.concat(
-                                                                          sstableNew.getAllFilePaths(),
-                                                                          sstableOld.getAllFilePaths(),
-                                                                          log.logFilePaths())));
+        assertFiles(dataFolder.path(), Sets.newHashSet(Iterables.concat(getAllFilePaths(sstableNew),
+                                                                        getAllFilePaths(sstableOld),
+                                                                        log.logFilePaths())));
 
         sstableOld.selfRef().release();
         sstableNew.selfRef().release();
@@ -1148,12 +1207,12 @@ private static void testObsoletedFilesChanged(Consumer<SSTableReader> modifier)
         // complete the transaction to avoid LEAK errors
         assertNull(log.complete(null));
 
-        assertFiles(dataFolder.path(), Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths(),
-                                                                        sstableOld.getAllFilePaths(),
+        assertFiles(dataFolder.path(), Sets.newHashSet(Iterables.concat(getAllFilePaths(sstableNew),
+                                                                        getAllFilePaths(sstableOld),
                                                                         log.logFilePaths())));
 
         // make sure to run the tidier to avoid any leaks in the logs
-        tidier.run();
+        tidier.commit();
     }
 
     @Test
@@ -1165,11 +1224,10 @@ public void testTruncateFileUpdateTime() throws IOException
         testTruncatedModificationTimesHelper(sstable ->
                                   {
                                       // increase the modification time of the Data file
-                                      for (String filePath : sstable.getAllFilePaths())
+                                      for (File filePath : getAllFilePaths(sstable))
                                       {
-                                          File f = new File(filePath);
-                                          long lastModified = f.lastModified();
-                                          f.trySetLastModified(lastModified - (lastModified % 1000));
+                                          long lastModified = filePath.lastModified();
+                                          filePath.trySetLastModified(lastModified - (lastModified % 1000));
                                       }
                                   });
     }
@@ -1182,11 +1240,11 @@ private static void testTruncatedModificationTimesHelper(Consumer<SSTableReader>
         SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
 
         // simulate tracking sstables with a committed transaction except the checksum will be wrong
-        LogTransaction log = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction log = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(log);
 
         log.trackNew(sstableNew);
-        LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
+        AbstractLogTransaction.ReaderTidier tidier = log.obsoleted(sstableOld);
 
         //modify the old sstable files
         modifier.accept(sstableOld);
@@ -1194,19 +1252,19 @@ private static void testTruncatedModificationTimesHelper(Consumer<SSTableReader>
         //Fake a commit
         log.txnFile().commit();
 
-        LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
+        LifecycleTransaction.removeUnfinishedLeftovers(cfs.metadata());
 
         // only the new files should be there
-        assertFiles(dataFolder.path(), Sets.newHashSet(sstableNew.getAllFilePaths()));
+        assertFiles(dataFolder.path(), Sets.newHashSet(getAllFilePaths(sstableNew)));
         sstableNew.selfRef().release();
 
         // complete the transaction to avoid LEAK errors
         assertNull(log.complete(null));
 
-        assertFiles(dataFolder.path(), Sets.newHashSet(sstableNew.getAllFilePaths()));
+        assertFiles(dataFolder.path(), Sets.newHashSet(getAllFilePaths(sstableNew)));
 
         // make sure to run the tidier to avoid any leaks in the logs
-        tidier.run();
+        tidier.commit();
     }
 
     @Test
@@ -1216,10 +1274,10 @@ public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable
         File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
         SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
 
-        LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction logs = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(logs);
 
-        LogTransaction.SSTableTidier tidier = logs.obsoleted(sstable);
+        AbstractLogTransaction.ReaderTidier tidier = logs.obsoleted(sstable);
 
         logs.finish();
 
@@ -1240,15 +1298,15 @@ public void testGetTemporaryFilesThrowsIfCompletingAfterObsoletion() throws Thro
         File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
         SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
 
-        LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
+        LogTransaction logs = createLogTransaction(OperationType.COMPACTION, cfs.metadata);
         assertNotNull(logs);
 
-        LogTransaction.SSTableTidier tidier = logs.obsoleted(sstable);
+        AbstractLogTransaction.ReaderTidier tidier = logs.obsoleted(sstable);
 
         sstable.markObsolete(tidier);
         sstable.selfRef().release();
 
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
 
         try
         {
@@ -1329,7 +1387,7 @@ else if (BtiFormat.isSelected())
             SSTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components)
                                                                          .setTableMetadataRef(cfs.metadata)
                                                                          .setDataFile(dFile)
-                                                                         .setPartitionIndex(new PartitionIndex(iFile, 0, 0, MockSchema.readerBounds(generation), MockSchema.readerBounds(generation)))
+                                                                         .setPartitionIndex(new PartitionIndex(iFile, 0, 0, MockSchema.readerBounds(generation), MockSchema.readerBounds(generation), ByteComparable.Version.OSS50))
                                                                          .setRowIndexFile(rFile)
                                                                          .setFilter(FilterFactory.AlwaysPresent)
                                                                          .setMaxDataAge(1L)
@@ -1347,14 +1405,42 @@ else if (BtiFormat.isSelected())
         }
     }
 
-    private static void assertFiles(String dirPath, Set<String> expectedFiles) throws IOException
+    static Set<File> getAllFilePaths(SSTableReader sstable)
+    {
+        return getAllFilePaths(sstable, false);
+    }
+
+    /**
+     * @param sstable the sstable for which we want to check the files
+     * @param existingOnly  if true then only return files that do exist on disk
+     *
+     * @return the files expected to exist according to the sstable components
+     */
+    static Set<File> getAllFilePaths(SSTableReader sstable, boolean existingOnly)
+    {
+        Set<File> ret = new HashSet<>(sstable.components().size());
+        for (Component component : sstable.components())
+        {
+            File path = sstable.descriptor.fileFor(component);
+
+            // other components are expected to exist unless the test is explicitly checking
+            // a case where a component was not created, in which case existingOnly will be true
+            // and the file will only be added if it exists
+            if (!existingOnly || path.exists())
+                ret.add(path);
+        }
+
+        return ret;
+    }
+
+    private static void assertFiles(String dirPath, Set<File> expectedFiles) throws IOException
     {
         assertFiles(dirPath, expectedFiles, false);
     }
 
-    private static void assertFiles(String dirPath, Set<String> expectedFiles, boolean excludeNonExistingFiles) throws IOException
+    private static void assertFiles(String dirPath, Set<File> expectedFiles, boolean excludeNonExistingFiles) throws IOException
     {
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
 
         File dir = new File(dirPath).toCanonical();
         File[] files = dir.tryList();
@@ -1365,19 +1451,17 @@ private static void assertFiles(String dirPath, Set<String> expectedFiles, boole
                 if (file.isDirectory())
                     continue;
 
-                String filePath = file.path();
-                assertTrue(String.format("%s not in [%s]", filePath, expectedFiles), expectedFiles.contains(filePath));
-                expectedFiles.remove(filePath);
+                assertTrue(String.format("%s not in [%s]", file, expectedFiles), expectedFiles.contains(file));
+                expectedFiles.remove(file);
             }
         }
 
         if (excludeNonExistingFiles)
         {
-            for (String filePath : expectedFiles)
+            for (File file : expectedFiles)
             {
-                File file = new File(filePath);
                 if (!file.exists())
-                    expectedFiles.remove(filePath);
+                    expectedFiles.remove(file);
             }
         }
 
@@ -1386,13 +1470,12 @@ private static void assertFiles(String dirPath, Set<String> expectedFiles, boole
 
     // Check either that a temporary file is expected to exist (in the existingFiles) or that
     // it does not exist any longer.
-    private static void assertFiles(Iterable<String> existingFiles, Set<File> temporaryFiles)
+    private static void assertFiles(Iterable<File> existingFiles, Set<File> temporaryFiles)
     {
-        for (String filePath : existingFiles)
+        for (File filePath : existingFiles)
         {
-            File file = new File(filePath);
-            assertTrue(filePath, temporaryFiles.contains(file));
-            temporaryFiles.remove(file);
+            assertTrue(filePath.toString(), temporaryFiles.contains(filePath));
+            temporaryFiles.remove(filePath);
         }
 
         for (File file : temporaryFiles)
@@ -1409,6 +1492,11 @@ static Set<File> getTemporaryFiles(File folder)
         return listFiles(folder, Directories.FileType.TEMPORARY);
     }
 
+    static Set<File> getTemporaryFiles(File folder, Directories.OnTxnErr onTxnErr)
+    {
+        return listFiles(folder, onTxnErr, Directories.FileType.TEMPORARY);
+    }
+
     static Set<File> getFinalFiles(File folder)
     {
         return listFiles(folder, Directories.FileType.FINAL);
@@ -1437,14 +1525,18 @@ private static Stream<File> toCanonicalIgnoringNotFound(File file)
     }
 
     static Set<File> listFiles(File folder, Directories.FileType... types)
+    {
+        return listFiles(folder, Directories.OnTxnErr.IGNORE, types);
+    }
+
+    static Set<File> listFiles(File folder, Directories.OnTxnErr onTxnErr, Directories.FileType... types)
     {
         Collection<Directories.FileType> match = Arrays.asList(types);
-        return new LogAwareFileLister(folder.toPath(),
-                                      (file, type) -> match.contains(type),
-                                      Directories.OnTxnErr.IGNORE).list()
-                       .stream()
-                       .flatMap(LogTransactionTest::toCanonicalIgnoringNotFound)
-                       .collect(Collectors.toSet());
+        return ILogTransactionsFactory.instance.createLogAwareFileLister()
+                                               .list(folder.toPath(), (file, type) -> match.contains(type), onTxnErr)
+                                               .stream()
+                                               .map(File::toCanonical)
+                                               .collect(Collectors.toSet());
     }
 
     static final String DUMMY_KS = "ks";
@@ -1473,7 +1565,7 @@ SSTable dummySSTable()
     @Test(expected = TransactionAlreadyCompletedException.class)
     public void useAfterCompletedTest()
     {
-        try (LogTransaction txnFile = new LogTransaction(OperationType.STREAM))
+        try (LogTransaction txnFile = new LogTransaction(OperationType.STREAM, TimeUUID.Generator.nextTimeUUID()))
         {
             txnFile.abort(); // this should complete the txn
             txnFile.trackNew(dummySSTable()); // expect an IllegalStateException here
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
index 28aab8556ebc..d6eb9a29d8ad 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
@@ -32,13 +32,13 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
 import org.apache.cassandra.db.compaction.CompactionController;
 import org.apache.cassandra.db.compaction.CompactionIterator;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.sstable.CQLSSTableWriter;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableRewriter;
+import org.apache.cassandra.io.sstable.ScannerList;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.util.File;
@@ -48,6 +48,7 @@
 import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.db.lifecycle.LogTransactionTest.getAllFilePaths;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -82,11 +83,11 @@ public void testRewriteFinished() throws IOException
         SSTableReader oldSSTable = getSSTable(cfs, 1);
         LifecycleTransaction txn = cfs.getTracker().tryModify(oldSSTable, OperationType.COMPACTION);
         SSTableReader newSSTable = replaceSSTable(cfs, txn, false);
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
 
         // both sstables are in the same folder
-        assertFiles(oldSSTable.descriptor.directory.path(), new HashSet<>(newSSTable.getAllFilePaths()));
-        assertFiles(newSSTable.descriptor.directory.path(), new HashSet<>(newSSTable.getAllFilePaths()));
+        assertFiles(oldSSTable.descriptor.directory, getAllFilePaths(newSSTable));
+        assertFiles(newSSTable.descriptor.directory, getAllFilePaths(newSSTable));
     }
 
     @Test
@@ -99,9 +100,9 @@ public void testRewriteAborted() throws IOException
         LifecycleTransaction txn = cfs.getTracker().tryModify(oldSSTable, OperationType.COMPACTION);
 
         replaceSSTable(cfs, txn, true);
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
 
-        assertFiles(oldSSTable.descriptor.directory.path(), new HashSet<>(oldSSTable.getAllFilePaths()));
+        assertFiles(oldSSTable.descriptor.directory, getAllFilePaths(oldSSTable));
     }
 
     @Test
@@ -112,8 +113,8 @@ public void testFlush() throws IOException
 
         SSTableReader ssTableReader = getSSTable(cfs, 100);
 
-        String dataFolder = cfs.getLiveSSTables().iterator().next().descriptor.directory.path();
-        assertFiles(dataFolder, new HashSet<>(ssTableReader.getAllFilePaths()));
+        File dataFolder = cfs.getLiveSSTables().iterator().next().descriptor.directory;
+        assertFiles(dataFolder, getAllFilePaths(ssTableReader));
     }
 
     private SSTableReader getSSTable(ColumnFamilyStore cfs, int numPartitions) throws IOException
@@ -152,7 +153,7 @@ private SSTableReader replaceSSTable(ColumnFamilyStore cfs, LifecycleTransaction
         try (CompactionController controller = new CompactionController(cfs, txn.originals(), cfs.gcBefore(FBUtilities.nowInSeconds())))
         {
             try (SSTableRewriter rewriter = SSTableRewriter.constructKeepingOriginals(txn, false, 1000);
-                 AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(txn.originals());
+                 ScannerList scanners = cfs.getCompactionStrategy().getScanners(txn.originals());
                  CompactionIterator ci = new CompactionIterator(txn.opType(), scanners.scanners, controller, nowInSec, txn.opId())
             )
             {
@@ -197,17 +198,15 @@ private SSTableReader replaceSSTable(ColumnFamilyStore cfs, LifecycleTransaction
         return null;
     }
 
-    private void assertFiles(String dirPath, Set<String> expectedFiles)
+    private void assertFiles(File dirPath, Set<File> expectedFiles)
     {
-        File dir = new File(dirPath);
-        for (File file : dir.tryList())
+        for (File file : dirPath.tryList())
         {
             if (file.isDirectory())
                 continue;
 
-            String filePath = file.path();
-            assertTrue(filePath, expectedFiles.contains(filePath));
-            expectedFiles.remove(filePath);
+            assertTrue(file.toString(), expectedFiles.contains(file));
+            expectedFiles.remove(file);
         }
 
         assertTrue(expectedFiles.isEmpty());
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 7192d50031e8..cf4b71ed0a3a 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -30,9 +30,11 @@
 import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import org.apache.cassandra.utils.TimeUUID;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -49,20 +51,23 @@
 import org.apache.cassandra.notifications.MemtableRenewedNotification;
 import org.apache.cassandra.notifications.MemtableSwitchedNotification;
 import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableAddingNotification;
 import org.apache.cassandra.notifications.SSTableDeletingNotification;
 import org.apache.cassandra.notifications.SSTableListChangedNotification;
-import org.apache.cassandra.notifications.SSTableMetadataChanged;
 import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
 import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.MockSchema;
 import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+import org.mockito.Mockito;
 
 import static com.google.common.collect.ImmutableSet.copyOf;
 import static java.util.Collections.singleton;
 
+@RunWith(BMUnitRunner.class)
 public class TrackerTest
 {
-
     private static final class MockListener implements INotificationConsumer
     {
         final boolean throwException;
@@ -95,7 +100,7 @@ public static void setUp()
     public void testTryModify()
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
-        Tracker tracker = Tracker.newDummyTracker();
+        Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
         List<SSTableReader> readers = ImmutableList.of(MockSchema.sstable(0, true, cfs), MockSchema.sstable(1, cfs), MockSchema.sstable(2, cfs));
         tracker.addInitialSSTables(copyOf(readers));
         Assert.assertNull(tracker.tryModify(ImmutableList.of(MockSchema.sstable(0, cfs)), OperationType.COMPACTION));
@@ -118,7 +123,7 @@ public void testTryModify()
     public void testApply()
     {
         final ColumnFamilyStore cfs = MockSchema.newCFS();
-        final Tracker tracker = Tracker.newDummyTracker();
+        final Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
         final View resultView = ViewTest.fakeView(0, 0, cfs);
         final AtomicInteger count = new AtomicInteger();
         tracker.apply(new Predicate<View>()
@@ -168,9 +173,11 @@ public void testAddInitialSSTables()
         tracker.addInitialSSTables(copyOf(readers));
 
         Assert.assertEquals(3, tracker.view.get().sstables.size());
-        Assert.assertEquals(1, listener.senders.size());
-        Assert.assertEquals(1, listener.received.size());
-        Assert.assertTrue(listener.received.get(0) instanceof InitialSSTableAddedNotification);
+        Assert.assertEquals(2, listener.senders.size()); // one sender sent two notifications
+        Assert.assertEquals(listener.senders.get(0), listener.senders.get(1));
+        Assert.assertEquals(2, listener.received.size());
+        Assert.assertTrue(listener.received.get(0) instanceof SSTableAddingNotification);
+        Assert.assertTrue(listener.received.get(1) instanceof InitialSSTableAddedNotification);
 
         for (SSTableReader reader : readers)
             if (reader instanceof KeyCacheSupport<?>)
@@ -191,7 +198,7 @@ public void testAddSSTables()
         List<SSTableReader> readers = ImmutableList.of(MockSchema.sstable(0, 17, cfs),
                                                        MockSchema.sstable(1, 121, cfs),
                                                        MockSchema.sstable(2, 9, cfs));
-        tracker.addSSTables(copyOf(readers));
+        tracker.addSSTables(copyOf(readers), OperationType.UNKNOWN);
 
         Assert.assertEquals(3, tracker.view.get().sstables.size());
 
@@ -202,20 +209,78 @@ public void testAddSSTables()
         }
 
         Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount());
-        Assert.assertEquals(1, listener.senders.size());
-        Assert.assertEquals(1, listener.received.size());
+        Assert.assertEquals(2, listener.senders.size()); // one tracker issued two notifications
+        Assert.assertEquals(2, listener.received.size()); // 'adding' and 'added' notifications
         Assert.assertEquals(tracker, listener.senders.get(0));
-        Assert.assertTrue(listener.received.get(0) instanceof SSTableAddedNotification);
+        Assert.assertEquals(tracker, listener.senders.get(1));
+        Assert.assertTrue(listener.received.get(0) instanceof SSTableAddingNotification);
+        Assert.assertTrue(listener.received.get(1) instanceof SSTableAddedNotification);
         DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
     }
 
+    @Test
+    public void testLateSubscribers()
+    {
+        boolean backups = DatabaseDescriptor.isIncrementalBackupsEnabled();
+        DatabaseDescriptor.setIncrementalBackupsEnabled(false);
+        try
+        {
+            ColumnFamilyStore cfs = MockSchema.newCFS();
+            Tracker tracker = cfs.getTracker();
+
+            OrderTestingMockListener listener1 = new OrderTestingMockListener("l1");
+            OrderTestingMockListener listener2 = new OrderTestingMockListener("l2");
+            OrderTestingMockListener lateListener1 = new OrderTestingMockListener("ll1");
+            OrderTestingMockListener lateListener2 = new OrderTestingMockListener("ll2");
+
+            int initialCount = OrderTestingMockListener.counter.get();
+
+            tracker.subscribeLateConsumer(lateListener1);
+            tracker.subscribe(listener1);
+            tracker.subscribeLateConsumer(lateListener2);
+            tracker.subscribe(listener2);
+
+            List<SSTableReader> readers = ImmutableList.of(MockSchema.sstable(0, 17, cfs));
+            tracker.addSSTables(copyOf(readers), OperationType.UNKNOWN);
+
+            Assert.assertEquals(initialCount, listener1.captured);
+            Assert.assertEquals(initialCount + 1, listener2.captured);
+            Assert.assertEquals(initialCount + 2, lateListener1.captured);
+            Assert.assertEquals(initialCount + 3, lateListener2.captured);
+        }
+        finally
+        {
+            DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
+        }
+    }
+
+    // Mock listener that let us test the order of notifications by capturing and incrementing a common counter.
+    // Please note that this only capture the counter value for the first time a notification is triggered
+    private static final class OrderTestingMockListener implements INotificationConsumer
+    {
+        static final AtomicInteger counter = new AtomicInteger();
+        volatile int captured = -1;
+        private final String name;
+
+        OrderTestingMockListener(String name)
+        {
+            this.name = name;
+        }
+
+        public void handleNotification(INotification notification, Object sender)
+        {
+            if (captured < 0)
+                captured = counter.getAndIncrement();
+        }
+    }
+
     @Test
     public void testDropSSTables()
     {
         testDropSSTables(false);
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
         testDropSSTables(true);
-        LogTransaction.waitForDeletions();
+        LifecycleTransaction.waitForDeletions();
     }
 
     private void testDropSSTables(boolean invalidate)
@@ -238,7 +303,7 @@ private void testDropSSTables(boolean invalidate)
             else
             {
                 tracker.dropSSTables();
-                LogTransaction.waitForDeletions();
+                LifecycleTransaction.waitForDeletions();
             }
             Assert.assertEquals(9, cfs.metric.totalDiskSpaceUsed.getCount());
             Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
@@ -259,16 +324,17 @@ private void testDropSSTables(boolean invalidate)
             Assert.assertNull(tracker.dropSSTables(reader -> reader != readers.get(0), OperationType.UNKNOWN, null));
 
             Assert.assertEquals(1, tracker.getView().sstables.size());
-            Assert.assertEquals(4, listener.received.size());
+            Assert.assertEquals(5, listener.received.size());
             Assert.assertEquals(tracker, listener.senders.get(0));
-            Assert.assertTrue(listener.received.get(0) instanceof InitialSSTableAddedNotification);
-            Assert.assertTrue(listener.received.get(1) instanceof SSTableDeletingNotification);
-            Assert.assertTrue(listener.received.get(2) instanceof  SSTableDeletingNotification);
-            Assert.assertTrue(listener.received.get(3) instanceof SSTableListChangedNotification);
-            Assert.assertEquals(readers.get(1), ((SSTableDeletingNotification) listener.received.get(1)).deleting);
-            Assert.assertEquals(readers.get(2), ((SSTableDeletingNotification)listener.received.get(2)).deleting);
-            Assert.assertEquals(2, ((SSTableListChangedNotification) listener.received.get(3)).removed.size());
-            Assert.assertEquals(0, ((SSTableListChangedNotification) listener.received.get(3)).added.size());
+            Assert.assertTrue(listener.received.get(0) instanceof SSTableAddingNotification);
+            Assert.assertTrue(listener.received.get(1) instanceof InitialSSTableAddedNotification);
+            Assert.assertTrue(listener.received.get(2) instanceof SSTableDeletingNotification);
+            Assert.assertTrue(listener.received.get(3) instanceof  SSTableDeletingNotification);
+            Assert.assertTrue(listener.received.get(4) instanceof SSTableListChangedNotification);
+            Assert.assertEquals(readers.get(1), ((SSTableDeletingNotification) listener.received.get(2)).deleting);
+            Assert.assertEquals(readers.get(2), ((SSTableDeletingNotification)listener.received.get(3)).deleting);
+            Assert.assertEquals(2, ((SSTableListChangedNotification) listener.received.get(4)).removed.size());
+            Assert.assertEquals(0, ((SSTableListChangedNotification) listener.received.get(4)).added.size());
             Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
             readers.get(0).selfRef().release();
         }
@@ -281,6 +347,64 @@ private void testDropSSTables(boolean invalidate)
         }
     }
 
+    @Test
+    public void testDropSSTablesWithTxnCommitFailureLiveSSTables()
+    {
+        dropSSTablesWithTxnCommitFailure(true, false);
+    }
+
+    @Test
+    public void testDropSSTablesWithTxnCommitFailure()
+    {
+        dropSSTablesWithTxnCommitFailure(false, false);
+    }
+
+    @Test
+    @BMRule(name = "fail abort obsoletion",
+            targetClass = "Helpers",
+            targetMethod = "abortObsoletion",
+            action = "return new java.lang.RuntimeException(\"failed to abort obsoletions\")")
+    public void testDropSSTablesWithTxnCommitFailureAndAbortedObsoletions()
+    {
+        dropSSTablesWithTxnCommitFailure(true, true);
+    }
+
+    private void dropSSTablesWithTxnCommitFailure(boolean liveSSTables, boolean abortObsoletions)
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS();
+        Tracker tracker = Mockito.spy(cfs.getTracker());
+
+        Mockito.doThrow(new RuntimeException("Test throw")).when(tracker).notifySSTablesChanged(
+        Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any());
+
+        if (!liveSSTables)
+            cfs.invalidate(false, false);
+
+        MockListener listener = new MockListener(false);
+        tracker.subscribe(listener);
+        final List<SSTableReader> readers = ImmutableList.of(MockSchema.sstable(0, 9, true, cfs),
+                                                             MockSchema.sstable(1, 15, true, cfs),
+                                                             MockSchema.sstable(2, 71, true, cfs));
+        tracker.addInitialSSTables(copyOf(readers));
+
+        try (LifecycleTransaction txn = tracker.tryModify(readers.get(0), OperationType.COMPACTION))
+        {
+            Assert.assertThrows(RuntimeException.class, () -> tracker.dropSSTables());
+        }
+        catch (RuntimeException ex)
+        {
+            Assert.assertTrue(abortObsoletions);
+        }
+
+        // If obsoletions were aborted then we can't make any guarantees about the state of the SSTable liveset.
+        if (abortObsoletions)
+            return;
+
+        // Make sure that all SSTables are still live after the commit failed, unless we invalidated the CFS first.
+        int numSSTables = liveSSTables ? readers.size() : 0;
+        Assert.assertEquals(numSSTables, tracker.getLiveSSTables().size());
+    }
+
     @Test
     public void testMemtableReplacement()
     {
@@ -319,17 +443,17 @@ public void testMemtableReplacement()
         Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev1));
         Assert.assertEquals(2, tracker.getView().flushingMemtables.size());
 
-        tracker.replaceFlushed(prev1, Collections.emptyList());
+        tracker.replaceFlushed(prev1, Collections.emptyList(), Optional.empty());
         Assert.assertEquals(1, tracker.getView().flushingMemtables.size());
         Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev2));
 
         SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
-        tracker.replaceFlushed(prev2, singleton(reader));
+        tracker.replaceFlushed(prev2, singleton(reader), Optional.empty());
         Assert.assertEquals(1, tracker.getView().sstables.size());
-        Assert.assertEquals(2, listener.received.size());
-        Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
-        Assert.assertEquals(Optional.of(prev2), ((SSTableAddedNotification) listener.received.get(0)).memtable());
-        Assert.assertEquals(prev2, ((MemtableDiscardedNotification) listener.received.get(1)).memtable);
+        Assert.assertEquals(3, listener.received.size());
+        Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added);
+        Assert.assertEquals(Optional.of(prev2), ((SSTableAddedNotification) listener.received.get(1)).memtable());
+        Assert.assertEquals(prev2, ((MemtableDiscardedNotification) listener.received.get(2)).memtable);
         listener.received.clear();
         if (reader instanceof KeyCacheSupport<?>)
             Assert.assertTrue(((KeyCacheSupport<?>) reader).getKeyCache().isEnabled());
@@ -344,17 +468,17 @@ public void testMemtableReplacement()
         tracker.markFlushing(prev1);
         reader = MockSchema.sstable(0, 10, true, cfs);
         cfs.invalidate(false);
-        tracker.replaceFlushed(prev1, singleton(reader));
+        tracker.replaceFlushed(prev1, singleton(reader), Optional.empty());
         Assert.assertEquals(0, tracker.getView().sstables.size());
         Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
         Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
-        Assert.assertEquals(5, listener.received.size());
+        Assert.assertEquals(6, listener.received.size());
         Assert.assertEquals(prev1, ((MemtableSwitchedNotification) listener.received.get(0)).memtable);
-        Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added);
-        Assert.assertEquals(Optional.of(prev1), ((SSTableAddedNotification) listener.received.get(1)).memtable());
-        Assert.assertEquals(prev1, ((MemtableDiscardedNotification) listener.received.get(2)).memtable);
-        Assert.assertTrue(listener.received.get(3) instanceof SSTableDeletingNotification);
-        Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(4)).removed.size());
+        Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(2)).added);
+        Assert.assertEquals(Optional.of(prev1), ((SSTableAddedNotification) listener.received.get(2)).memtable());
+        Assert.assertEquals(prev1, ((MemtableDiscardedNotification) listener.received.get(3)).memtable);
+        Assert.assertTrue(listener.received.get(4) instanceof SSTableDeletingNotification);
+        Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(5)).removed.size());
         DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
     }
 
@@ -363,41 +487,59 @@ public void testNotifications()
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
         SSTableReader r1 = MockSchema.sstable(0, cfs), r2 = MockSchema.sstable(1, cfs);
-        Tracker tracker = Tracker.newDummyTracker();
+        Tracker tracker = Tracker.newDummyTracker(cfs.metadata);
+        Memtable memtable = MockSchema.memtable(cfs);
         MockListener listener = new MockListener(false);
         tracker.subscribe(listener);
-        tracker.notifyAdded(singleton(r1), false);
+        tracker.notifyAdded(singleton(r1), OperationType.UNKNOWN, false);
+        Assert.assertEquals(singleton(r1), ((SSTableAddedNotification) listener.received.get(0)).added);
+        Assert.assertFalse(((SSTableAddedNotification) listener.received.get(0)).fromStreaming());
+        Assert.assertFalse(((SSTableAddedNotification) listener.received.get(0)).operationId.isPresent());
+        listener.received.clear();
+        TimeUUID flushOperationId = TimeUUID.Generator.nextTimeUUID();
+        tracker.notifyAdded(singleton(r1), OperationType.FLUSH, Optional.of(flushOperationId), false, memtable, null);
         Assert.assertEquals(singleton(r1), ((SSTableAddedNotification) listener.received.get(0)).added);
+        Assert.assertEquals(memtable, (((SSTableAddedNotification) listener.received.get(0)).memtable().get()));
+        Assert.assertTrue(((SSTableAddedNotification) listener.received.get(0)).operationId.isPresent());
+        Assert.assertEquals(flushOperationId, (((SSTableAddedNotification) listener.received.get(0)).operationId.get()));
         listener.received.clear();
         tracker.notifyDeleting(r1);
         Assert.assertEquals(r1, ((SSTableDeletingNotification) listener.received.get(0)).deleting);
         listener.received.clear();
-        Assert.assertNull(tracker.notifySSTablesChanged(singleton(r1), singleton(r2), OperationType.COMPACTION, null));
+        Assert.assertNull(tracker.notifySSTablesChanged(singleton(r1), singleton(r2), OperationType.COMPACTION, Optional.empty(), null));
         Assert.assertEquals(singleton(r1), ((SSTableListChangedNotification) listener.received.get(0)).removed);
         Assert.assertEquals(singleton(r2), ((SSTableListChangedNotification) listener.received.get(0)).added);
         listener.received.clear();
         tracker.notifySSTableRepairedStatusChanged(singleton(r1));
         Assert.assertEquals(singleton(r1), ((SSTableRepairStatusChanged) listener.received.get(0)).sstables);
         listener.received.clear();
-        Memtable memtable = MockSchema.memtable(cfs);
         tracker.notifyRenewed(memtable);
         Assert.assertEquals(memtable, ((MemtableRenewedNotification) listener.received.get(0)).renewed);
         listener.received.clear();
-        tracker.notifySSTableMetadataChanged(r1, r1.getSSTableMetadata());
-        Assert.assertEquals(((SSTableMetadataChanged)listener.received.get(0)).sstable, r1);
-        Assert.assertEquals(r1.getSSTableMetadata(), ((SSTableMetadataChanged)listener.received.get(0)).oldMetadata);
-        listener.received.clear();
         tracker.unsubscribe(listener);
         MockListener failListener = new MockListener(true);
         tracker.subscribe(failListener);
         tracker.subscribe(listener);
-        Assert.assertNotNull(tracker.notifyAdded(singleton(r1), false, null, null));
+        Assert.assertNotNull(tracker.notifyAdded(singleton(r1), OperationType.REGION_DECOMMISSION, Optional.empty(), false, null, null));
         Assert.assertEquals(singleton(r1), ((SSTableAddedNotification) listener.received.get(0)).added);
+        Assert.assertEquals(OperationType.REGION_DECOMMISSION, ((SSTableAddedNotification) listener.received.get(0)).operationType);
         Assert.assertFalse(((SSTableAddedNotification) listener.received.get(0)).memtable().isPresent());
+        Assert.assertTrue(((SSTableAddedNotification) listener.received.get(0)).fromStreaming());
         listener.received.clear();
-        Assert.assertNotNull(tracker.notifySSTablesChanged(singleton(r1), singleton(r2), OperationType.COMPACTION, null));
+        Assert.assertNotNull(tracker.notifySSTablesChanged(singleton(r1), singleton(r2), OperationType.COMPACTION, Optional.empty(), null));
         Assert.assertEquals(singleton(r1), ((SSTableListChangedNotification) listener.received.get(0)).removed);
         Assert.assertEquals(singleton(r2), ((SSTableListChangedNotification) listener.received.get(0)).added);
+        Assert.assertFalse(((SSTableListChangedNotification) listener.received.get(0)).operationId.isPresent());
+        listener.received.clear();
+        TimeUUID compactionOperationId = TimeUUID.Generator.nextTimeUUID();
+        Assert.assertNotNull(tracker.notifySSTablesChanged(singleton(r1), singleton(r2), OperationType.COMPACTION, Optional.of(compactionOperationId), null));
+        Assert.assertEquals(singleton(r1), ((SSTableListChangedNotification) listener.received.get(0)).removed);
+        Assert.assertEquals(singleton(r2), ((SSTableListChangedNotification) listener.received.get(0)).added);
+        Assert.assertTrue(((SSTableListChangedNotification) listener.received.get(0)).operationId.isPresent());
+        Assert.assertEquals(compactionOperationId, ((SSTableListChangedNotification) listener.received.get(0)).operationId.get());
+        listener.received.clear();
+        Assert.assertNotNull(tracker.notifyAdded(singleton(r1), OperationType.UNKNOWN, Optional.empty(), true, null, null));
+        Assert.assertEquals(singleton(r1), ((InitialSSTableAddedNotification) listener.received.get(0)).added);
         listener.received.clear();
     }
 
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index eb162d59b9f7..3bf2b2b0fb19 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@ -103,8 +103,8 @@ public void testCompaction()
         Assert.assertFalse(View.permitCompacting(readers.subList(0, 2)).apply(cur));
         Assert.assertFalse(View.permitCompacting(readers.subList(0, 1)).apply(cur));
         Assert.assertFalse(View.permitCompacting(readers.subList(1, 2)).apply(cur));
-        Assert.assertTrue(readers.subList(2, 5).containsAll(copyOf(cur.getUncompacting(readers))));
-        Assert.assertEquals(3, copyOf(cur.getUncompacting(readers)).size());
+        Assert.assertTrue(readers.subList(2, 5).containsAll(copyOf(cur.getNoncompacting(readers))));
+        Assert.assertEquals(3, copyOf(cur.getNoncompacting(readers)).size());
         Assert.assertTrue(ImmutableSet.copyOf(cur.select(SSTableSet.NONCOMPACTING)).containsAll(readers.subList(2, 5)));
         Assert.assertEquals(3, ImmutableSet.copyOf(cur.select(SSTableSet.NONCOMPACTING)).size());
 
@@ -132,8 +132,8 @@ public void testCompaction()
         Assert.assertFalse(View.permitCompacting(readers.subList(1, 2)).apply(cur));
         testFailure(View.updateCompacting(emptySet(), readers.subList(1, 2)), cur);
         testFailure(View.updateCompacting(copyOf(readers.subList(0, 2)), emptySet()), cur);
-        Assert.assertTrue(copyOf(concat(readers.subList(0, 1), readers.subList(2, 5))).containsAll(copyOf(cur.getUncompacting(readers))));
-        Assert.assertEquals(4, copyOf(cur.getUncompacting(readers)).size());
+        Assert.assertTrue(copyOf(concat(readers.subList(0, 1), readers.subList(2, 5))).containsAll(copyOf(cur.getNoncompacting(readers))));
+        Assert.assertEquals(4, copyOf(cur.getNoncompacting(readers)).size());
         Set<SSTableReader> nonCompacting = ImmutableSet.copyOf(cur.select(SSTableSet.NONCOMPACTING));
         Assert.assertTrue(nonCompacting.containsAll(readers.subList(2, 5)));
         Assert.assertTrue(nonCompacting.containsAll(readers.subList(0, 1)));
diff --git a/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java
index 499470904d7b..7654c399943a 100644
--- a/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java
@@ -40,6 +40,7 @@
 import java.util.Set;
 import java.util.function.BiPredicate;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -104,7 +105,6 @@
 import org.apache.cassandra.utils.AbstractTypeGenerators.TypeGenBuilder;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.CassandraVersion;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.FastByteOperations;
 import org.apache.cassandra.utils.Generators;
 import org.apache.cassandra.utils.asserts.SoftAssertionsWithLimit;
@@ -120,6 +120,7 @@
 import org.reflections.util.ConfigurationBuilder;
 
 import static org.apache.cassandra.db.marshal.AbstractType.ComparisonType.CUSTOM;
+import static org.apache.cassandra.utils.AbstractTypeGenerators.DSE_CUSTOM_TYPES;
 import static org.apache.cassandra.utils.AbstractTypeGenerators.TypeKind.COMPOSITE;
 import static org.apache.cassandra.utils.AbstractTypeGenerators.TypeKind.COUNTER;
 import static org.apache.cassandra.utils.AbstractTypeGenerators.TypeKind.DYNAMIC_COMPOSITE;
@@ -153,6 +154,8 @@ public class AbstractTypeTest
 
     static
     {
+        CassandraRelevantProperties.VECTOR_FLOAT_ONLY.setBoolean(false);
+        
         // make sure blob is always the same
         CassandraRelevantProperties.TEST_BLOB_SHARED_SEED.setInt(42);
     }
@@ -170,8 +173,12 @@ public class AbstractTypeTest
     private static LoadedTypesCompatibility cassandra40TypesCompatibility;
     private static LoadedTypesCompatibility cassandra41TypesCompatibility;
     private static LoadedTypesCompatibility cassandra50TypesCompatibility;
+    private static LoadedTypesCompatibility dse68cndbTypesCompatibility;
+    private static LoadedTypesCompatibility legacyCC40TypesCompatibility;
+    private static LoadedTypesCompatibility cc40TypesCompatibility;
+    private static LoadedTypesCompatibility cc50TypesCompatibility;
 
-    private final static String CASSANDRA_VERSION = new CassandraVersion(FBUtilities.getReleaseVersionString()).toMajorMinorString();
+    private final static String CASSANDRA_VERSION = "cc-5.0";
     private final static Path BASE_OUTPUT_PATH = Paths.get("test", "data", "types-compatibility");
 
     @BeforeClass
@@ -181,6 +188,10 @@ public static void beforeClass() throws IOException
         cassandra40TypesCompatibility = new LoadedTypesCompatibility(compatibilityFile(CassandraVersion.CASSANDRA_4_0.toMajorMinorString()), Set.of());
         cassandra41TypesCompatibility = new LoadedTypesCompatibility(compatibilityFile(CassandraVersion.CASSANDRA_4_1.toMajorMinorString()), Set.of());
         cassandra50TypesCompatibility = new LoadedTypesCompatibility(compatibilityFile(CassandraVersion.CASSANDRA_5_0.toMajorMinorString()), Set.of());
+        dse68cndbTypesCompatibility = new LoadedTypesCompatibility(compatibilityFile("dse-6.8-cndb"), ImmutableSet.of());
+        legacyCC40TypesCompatibility = new LoadedTypesCompatibility(compatibilityFile("legacy-cc-4.0"), ImmutableSet.of());
+        cc40TypesCompatibility = new LoadedTypesCompatibility(compatibilityFile("cc-4.0"), ImmutableSet.of());
+        cc50TypesCompatibility = new LoadedTypesCompatibility(compatibilityFile("cc-5.0"), ImmutableSet.of());
         currentTypesCompatibility = new CurrentTypesCompatibility();
     }
 
@@ -271,79 +282,51 @@ public void unsafeSharedSerializer()
             // org.apache.cassandra.db.marshal.AbstractType.comparatorSet needs to match the serializer, but when serialziers
             // break this mapping they may cause the wrong comparator (happened in cases like uuid and lexecal uuid; which have different orderings!).
             // Frozen types (as of this writing) do not change the sort ordering, so this simplification is fine...
-            if (old != null && !old.unfreeze().equals(t.unfreeze()))
+            if (old != null && !unfreeze(old).equals(unfreeze(t)))
                 throw new AssertionError(String.format("Different types detected that shared the same serializer: %s != %s", old.asCQL3Type(), t.asCQL3Type()));
         });
     }
 
     @Test
-    @SuppressWarnings({"rawtypes", "unchecked"})
     public void eqHashSafe()
     {
-        StringBuilder sb = new StringBuilder();
-        outter: for (Class<? extends AbstractType> type : reflections.getSubTypesOf(AbstractType.class))
-        {
-            if (Modifier.isAbstract(type.getModifiers()) || isTestType(type) || AbstractTypeGenerators.UNSUPPORTED.containsKey(type))
-                continue;
-            boolean hasEq = false;
-            boolean hasHashCode = false;
-            for (Class<? extends AbstractType> t = type; !t.equals(AbstractType.class); t = (Class<? extends AbstractType>) t.getSuperclass())
+        forEachTypesPair(true,(left, right) ->{
+            if (left.equals(right))
             {
-                try
-                {
-                    t.getDeclaredMethod("getInstance");
-                    continue outter;
-                }
-                catch (NoSuchMethodException e)
-                {
-                    // ignore
-                }
-                try
-                {
-                    t.getDeclaredField("instance");
-                    continue outter;
-                }
-                catch (NoSuchFieldException e)
-                {
-                    // ignore
-                }
-                try
-                {
-                    t.getDeclaredMethod("equals", Object.class);
-                    hasEq = true;
-                }
-                catch (NoSuchMethodException e)
-                {
-                    // ignore
-                }
-                try
+                assertThat(left.hashCode()).isEqualTo(right.hashCode());
+
+                assertThat(left.subTypes()).isEqualTo(right.subTypes());
+                assertThat(left.isMultiCell()).isEqualTo(right.isMultiCell());
+                assertThat(left.isCollection()).isEqualTo(right.isCollection());
+                assertThat(left.isUDT()).isEqualTo(right.isUDT());
+                assertThat(left.isTuple()).isEqualTo(right.isTuple());
+                assertThat(left.isCounter()).isEqualTo(right.isCounter());
+                assertThat(left.isReversed()).isEqualTo(right.isReversed());
+                assertThat(left.isVector()).isEqualTo(right.isVector());
+
+                if (left.isVector())
+                    assertThat(((VectorType) left).dimension).isEqualTo(((VectorType) right).dimension);
+
+                if (left.isUDT())
                 {
-                    t.getDeclaredMethod("hashCode");
-                    hasHashCode = true;
+                    assertThat(((UserType) left).name).isEqualTo(((UserType) right).name);
+                    assertThat(((UserType) left).keyspace).isEqualTo(((UserType) right).keyspace);
+                    assertThat(((UserType) left).fieldNames()).isEqualTo(((UserType) right).fieldNames());
                 }
-                catch (NoSuchMethodException e)
+
+                if (left.getClass() == CompositeType.class)
+                    assertThat(right.getClass()).isEqualTo(CompositeType.class);
+
+                if (left.getClass() == DynamicCompositeType.class)
                 {
-                    // ignore
+                    assertThat(right.getClass()).isEqualTo(DynamicCompositeType.class);
+                    assertThat(((DynamicCompositeType) left).aliases).isEqualTo(((DynamicCompositeType) right).aliases);
                 }
-                if (hasEq && hasHashCode)
-                    continue outter;
-            }
-            sb.append("AbstractType must be safe for map keys, so must either be a singleton or define ");
-            if (!hasEq)
-                sb.append("equals");
-            if (!hasHashCode)
-            {
-                if (!hasEq)
-                    sb.append('/');
-                sb.append("hashCode");
+
+                if (left.isCollection())
+                    assertThat(((CollectionType) left).kind).isEqualTo(((CollectionType) right).kind);
             }
-            sb.append("; ").append(type).append('\n');
-        }
-        if (sb.length() != 0)
-        {
-            sb.setLength(sb.length() - 1);
-            throw new AssertionError(sb.toString());
-        }
+        });
     }
 
     @Test
@@ -386,7 +369,7 @@ public void comparableBytes()
 
                     // test byte[] api
                     byte[] bytes = ByteSourceInverse.readBytes(type.asComparableBytes(bb, bcv));
-                    assertBytesEquals(type.fromComparableBytes(ByteSource.peekable(ByteSource.fixedLength(bytes)), bcv), bb, "fromOrderedBytes(toOrderedBytes(bb)) != bb");
+                    assertBytesEquals(type.fromComparableBytes(ByteSource.peekable(ByteSource.preencoded(bytes)), bcv), bb, "fromOrderedBytes(toOrderedBytes(bb)) != bb");
                 }
             }
         });
@@ -452,8 +435,8 @@ public void nested()
         Map<Class<? extends AbstractType>, Function<? super AbstractType<?>, Integer>> complexTypes = ImmutableMap.of(MapType.class, ignore -> 2,
                                                                                                                       TupleType.class, t -> ((TupleType) t).size(),
                                                                                                                       UserType.class, t -> ((UserType) t).size(),
-                                                                                                                      CompositeType.class, t -> ((CompositeType) t).types.size(),
-                                                                                                                      DynamicCompositeType.class, t -> ((DynamicCompositeType) t).size());
+                                                                                                                      CompositeType.class, t -> t.subTypes.size(),
+                                                                                                                      DynamicCompositeType.class, t -> t.subTypes.size());
         qt().withShrinkCycles(0).forAll(AbstractTypeGenerators.builder().withoutTypeKinds(PRIMITIVE, COUNTER).build()).checkAssert(type -> {
             int expectedSize = complexTypes.containsKey(type.getClass()) ? complexTypes.get(type.getClass()).apply(type) : 1;
             assertThat(type.subTypes()).hasSize(expectedSize);
@@ -679,7 +662,7 @@ private static Types toTypes(Set<UserType> udts)
             return Types.none();
         Types.Builder builder = Types.builder();
         for (UserType udt : udts)
-            builder.add(udt.unfreeze());
+            builder.add((UserType) unfreeze(udt));
         return builder.build();
     }
 
@@ -693,13 +676,14 @@ private static ByteComparable fromBytes(AbstractType<?> type, ByteBuffer bb)
     public void ordering()
     {
         TypeGenBuilder baseline = genBuilder()
-                                  .withoutPrimitive(DurationType.instance) // this uses byte ordering and vint, which makes the ordering effectivlly random from a user's point of view
+                                  .withoutPrimitive(DurationType.instance) // this uses byte ordering and vint, which makes the ordering effectively random from a user's point of view
                                   .withoutTypeKinds(COUNTER); // counters don't allow ordering
         // composite requires all elements fit into Short.MAX_VALUE bytes
         // so try to limit the possible expansion of types
         Gen<AbstractType<?>> types = baseline.withCompositeElementGen(new TypeGenBuilder(baseline).withDefaultSizeGen(1).withMaxDepth(1).build())
                                              .build();
         qt().withShrinkCycles(0).forAll(examples(10, types)).checkAssert(example -> {
+            logger.info("Testing type {}", example.type);
             AbstractType type = example.type;
             List<ByteBuffer> actual = decompose(type, example.samples);
             actual.sort(type);
@@ -832,35 +816,168 @@ public void testAssumedCompatibility()
     }
 
     @Test
-    public void testBackwardCompatibility()
+    @Ignore("To be discussed")
+    public void testMigrationFromCassandra40ToCurrent()
+    {
+        cassandra40TypesCompatibility.assertLoaded();
+        testBackwardCompatibility(currentTypesCompatibility, cassandra40TypesCompatibility, true);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCassandra41ToCurrent()
+    {
+        cassandra41TypesCompatibility.assertLoaded();
+        testBackwardCompatibility(currentTypesCompatibility, cassandra41TypesCompatibility, true);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCassandra50ToCurrent()
+    {
+        cassandra50TypesCompatibility.assertLoaded();
+        testBackwardCompatibility(currentTypesCompatibility, cassandra50TypesCompatibility, true);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromDse68CndbToCurrent()
+    {
+        dse68cndbTypesCompatibility.assertLoaded();
+        testBackwardCompatibility(currentTypesCompatibility, dse68cndbTypesCompatibility, false);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromLegacyCC40ToCurrent()
+    {
+        legacyCC40TypesCompatibility.assertLoaded();
+        testBackwardCompatibility(currentTypesCompatibility, legacyCC40TypesCompatibility, false);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCC40ToCurrent()
+    {
+        cc40TypesCompatibility.assertLoaded();
+        testBackwardCompatibility(currentTypesCompatibility, cc40TypesCompatibility, false);
+    }
+
+    @Test
+    public void testMigrationFromCC50ToCurrent()
+    {
+        cc50TypesCompatibility.assertLoaded();
+        testBackwardCompatibility(currentTypesCompatibility, cc50TypesCompatibility, true);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCurrentToCassandra40()
     {
         cassandra40TypesCompatibility.assertLoaded();
-        testBackwardCompatibility(currentTypesCompatibility, cassandra40TypesCompatibility);
+        Set<Class<? extends AbstractType>> skippedTypes = Sets.union(DSE_CUSTOM_TYPES, Set.of(LegacyTimeUUIDType.class, VectorType.class));
+        testBackwardCompatibility(cassandra40TypesCompatibility, currentTypesCompatibility,
+                                  c -> !skippedTypes.contains(c),
+                                  t -> !skippedTypes.contains(t.getClass()),
+                                  true);
+    }
 
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCurrentToCassandra41()
+    {
         cassandra41TypesCompatibility.assertLoaded();
-        testBackwardCompatibility(currentTypesCompatibility, cassandra41TypesCompatibility);
+        Set<Class<? extends AbstractType>> skippedTypes = Sets.union(DSE_CUSTOM_TYPES, Set.of(LegacyTimeUUIDType.class, VectorType.class));
+        testBackwardCompatibility(cassandra41TypesCompatibility, currentTypesCompatibility,
+                                  c -> !skippedTypes.contains(c),
+                                  t -> !skippedTypes.contains(t.getClass()),
+                                  true);
+    }
 
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCurrentToCassandra50()
+    {
         cassandra50TypesCompatibility.assertLoaded();
-        testBackwardCompatibility(currentTypesCompatibility, cassandra50TypesCompatibility);
+        Set<Class<? extends AbstractType>> skippedTypes = DSE_CUSTOM_TYPES;
+        testBackwardCompatibility(cassandra50TypesCompatibility, currentTypesCompatibility,
+                                  c -> !skippedTypes.contains(c),
+                                  t -> !skippedTypes.contains(t.getClass()),
+                                  true);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCurrentToDse68Cndb()
+    {
+        dse68cndbTypesCompatibility.assertLoaded();
+        Set<Class<? extends AbstractType>> skippedTypes = Set.of(LegacyTimeUUIDType.class, VectorType.class);
+        testBackwardCompatibility(dse68cndbTypesCompatibility, currentTypesCompatibility,
+                                  c -> !skippedTypes.contains(c),
+                                  t -> !skippedTypes.contains(t.getClass()),
+                                  false);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCurrentToLegacyCC40()
+    {
+        legacyCC40TypesCompatibility.assertLoaded();
+        Set<Class<? extends AbstractType>> skippedTypes = Set.of(LegacyTimeUUIDType.class, VectorType.class);
+        testBackwardCompatibility(legacyCC40TypesCompatibility, currentTypesCompatibility,
+                                  c -> !skippedTypes.contains(c),
+                                  t -> !skippedTypes.contains(t.getClass()),
+                                  false);
+    }
+
+    @Test
+    @Ignore("To be discussed")
+    public void testMigrationFromCurrentToCC40()
+    {
+        cc40TypesCompatibility.assertLoaded();
+        testBackwardCompatibility(cc40TypesCompatibility, currentTypesCompatibility,
+                                  t -> t != LegacyTimeUUIDType.class,
+                                  t -> t != LegacyTimeUUIDType.instance,
+                                  false);
+    }
+
+    @Test
+    public void testMigrationFromCurrentToCC50()
+    {
+        cc50TypesCompatibility.assertLoaded();
+        testBackwardCompatibility(cc50TypesCompatibility, currentTypesCompatibility, true);
+    }
+
+    private static void testBackwardCompatibility(TypesCompatibility upgradeTo, TypesCompatibility upgradeFrom, boolean serializationCompatibleWithSupported)
+    {
+        testBackwardCompatibility(upgradeTo, upgradeFrom, t -> true, t -> true, serializationCompatibleWithSupported);
     }
 
-    public void testBackwardCompatibility(TypesCompatibility upgradeTo, TypesCompatibility upgradeFrom)
+    public static void testBackwardCompatibility(TypesCompatibility upgradeTo, TypesCompatibility upgradeFrom, Predicate<Class<? extends AbstractType>> classFilter, Predicate<AbstractType<?>> typeFilter, boolean serializationCompatibleWithSupported)
     {
         SoftAssertions assertions = new SoftAssertionsWithLimit(100);
 
-        assertions.assertThat(upgradeTo.knownTypes()).containsAll(upgradeFrom.knownTypes());
-        assertions.assertThat(upgradeTo.primitiveTypes()).containsAll(upgradeFrom.primitiveTypes());
+        assertions.assertThat(upgradeTo.knownTypes().stream().filter(classFilter).collect(Collectors.toSet()))
+                  .containsAll(upgradeFrom.knownTypes().stream().filter(classFilter).collect(Collectors.toSet()));
+        assertions.assertThat(upgradeTo.primitiveTypes().stream().filter(typeFilter).collect(Collectors.toSet()))
+                  .containsAll(upgradeFrom.primitiveTypes().stream().filter(typeFilter).collect(Collectors.toSet()));
 
         // for compatibility, we ensure that this version can read values of all the types the previous version can write
         assertions.assertThat(upgradeTo.multiCellSupportingTypesForReading()).containsAll(upgradeFrom.multiCellSupportingTypes());
 
+
         forEachTypesPair(true, (l, r) -> {
-            if (upgradeFrom.expectCompatibleWith(l, r))
-                assertions.assertThat(upgradeTo.expectCompatibleWith(l, r)).describedAs(isCompatibleWithDesc(l, r)).isTrue();
-            if (upgradeFrom.expectSerializationCompatibleWith(l, r))
-                assertions.assertThat(upgradeTo.expectSerializationCompatibleWith(l, r)).describedAs(isSerializationCompatibleWithDesc(l, r)).isTrue();
-            if (upgradeFrom.expectValueCompatibleWith(l, r))
-                assertions.assertThat(upgradeTo.expectValueCompatibleWith(l, r)).describedAs(isValueCompatibleWithDesc(l, r)).isTrue();
+            if (l.equals(r))
+                return;
+            if (typeFilter.test(l) && typeFilter.test(r))
+            {
+                if (upgradeFrom.expectCompatibleWith(l, r))
+                    assertions.assertThat(upgradeTo.expectCompatibleWith(l, r)).describedAs(isCompatibleWithDesc(l, r)).isTrue();
+                if (serializationCompatibleWithSupported && upgradeFrom.expectSerializationCompatibleWith(l, r))
+                    assertions.assertThat(upgradeTo.expectSerializationCompatibleWith(l, r)).describedAs(isSerializationCompatibleWithDesc(l, r)).isTrue();
+                if (upgradeFrom.expectValueCompatibleWith(l, r))
+                    assertions.assertThat(upgradeTo.expectValueCompatibleWith(l, r)).describedAs(isValueCompatibleWithDesc(l, r)).isTrue();
+            }
         });
 
         assertions.assertAll();
@@ -872,6 +989,10 @@ public void testImplementedCompatibility()
         SoftAssertions assertions = new SoftAssertionsWithLimit(100);
 
         forEachTypesPair(true, (l, r) -> {
+            if (l instanceof MultiCellCapableType && l.isMultiCell && ((MultiCellCapableType<?>) l).nameComparator().referencesDuration())
+                return;
+            if (r instanceof MultiCellCapableType && r.isMultiCell && ((MultiCellCapableType<?>) r).nameComparator().referencesDuration())
+                return;
             assertions.assertThat(l.equals(r)).describedAs("equals symmetricity for %s and %s", l, r).isEqualTo(r.equals(l));
             verifyTypesCompatibility(l, r, getTypeSupport(r).valueGen, assertions);
         });
@@ -879,6 +1000,11 @@ public void testImplementedCompatibility()
         assertions.assertAll();
     }
 
+    private static Path compatibilityFile(CassandraVersion version)
+    {
+        return BASE_OUTPUT_PATH.resolve(String.format("%s.%s.json.gz", version.major, version.minor));
+    }
+
     private static Path compatibilityFile(String version)
     {
         return BASE_OUTPUT_PATH.resolve(String.format("%s.json.gz", version));
@@ -944,7 +1070,9 @@ else if (currentTypesCompatibility.multiCellSupportingTypes().contains(left.getC
         if (!left.isMultiCell() && !right.isMultiCell())
         {
             // make sure that frozen<left> isCompatibleWith frozen<right> ==> left isCompatibleWith right
-            assertions.assertThat(unfreeze(left).isCompatibleWith(unfreeze(right))).isTrue();
+            assertions.assertThat(unfreeze(left).isCompatibleWith(unfreeze(right)))
+                      .describedAs(typeRelDesc("isCompatibleWith", unfreeze(left), unfreeze(right)))
+                      .isTrue();
 
             assertions.assertThatCode(() -> qt().withExamples(10)
                                                 .forAll(rightGen, rightGen)
@@ -1160,7 +1288,7 @@ public void testMultiCellSupport()
                     {
                         assertions.assertThat(l.freeze()).isSameAs(l);
                         assertions.assertThat(unfreeze(l)).isSameAs(l);
-                        assertions.assertThat(unfreeze(l)).isEqualTo(l.unfreeze());
+                        assertions.assertThat(unfreeze(l)).isEqualTo(unfreeze(l));
                     }
                 }
 
@@ -1551,6 +1679,10 @@ private CurrentTypesCompatibility()
             primitiveValueCompatibleWith.put(BytesType.instance, TimestampType.instance);
             primitiveValueCompatibleWith.put(BytesType.instance, UTF8Type.instance);
             primitiveValueCompatibleWith.put(BytesType.instance, UUIDType.instance);
+            primitiveValueCompatibleWith.put(BytesType.instance, LineStringType.instance);
+            primitiveValueCompatibleWith.put(BytesType.instance, PointType.instance);
+            primitiveValueCompatibleWith.put(BytesType.instance, PolygonType.instance);
+            primitiveValueCompatibleWith.put(BytesType.instance, DateRangeType.instance);
             primitiveValueCompatibleWith.put(IntegerType.instance, Int32Type.instance);
             primitiveValueCompatibleWith.put(IntegerType.instance, LongType.instance);
             primitiveValueCompatibleWith.put(IntegerType.instance, TimestampType.instance);
@@ -1571,6 +1703,10 @@ private CurrentTypesCompatibility()
             primitiveSerializationCompatibleWith.put(BytesType.instance, SimpleDateType.instance);
             primitiveSerializationCompatibleWith.put(BytesType.instance, TimeType.instance);
             primitiveSerializationCompatibleWith.put(BytesType.instance, UTF8Type.instance);
+            primitiveSerializationCompatibleWith.put(BytesType.instance, LineStringType.instance);
+            primitiveSerializationCompatibleWith.put(BytesType.instance, PointType.instance);
+            primitiveSerializationCompatibleWith.put(BytesType.instance, PolygonType.instance);
+            primitiveSerializationCompatibleWith.put(BytesType.instance, DateRangeType.instance);
             primitiveSerializationCompatibleWith.put(LongType.instance, TimestampType.instance);
             primitiveSerializationCompatibleWith.put(TimestampType.instance, LongType.instance);
             primitiveSerializationCompatibleWith.put(UTF8Type.instance, AsciiType.instance);
diff --git a/test/unit/org/apache/cassandra/db/marshal/CompositeAndTupleTypesTest.java b/test/unit/org/apache/cassandra/db/marshal/CompositeAndTupleTypesTest.java
index 178c2910198c..9757332a976d 100644
--- a/test/unit/org/apache/cassandra/db/marshal/CompositeAndTupleTypesTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/CompositeAndTupleTypesTest.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Random;
 
+import com.google.common.collect.ImmutableList;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -133,6 +134,6 @@ public void userType()
     @Test
     public void composite()
     {
-        testSerializationDeserialization(CompositeType::new, CompositeType::build);
+        testSerializationDeserialization(types -> new CompositeType(ImmutableList.copyOf(types)), CompositeType::build);
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java
index 0f3714870af7..c757dc4f6e85 100644
--- a/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java
@@ -20,7 +20,12 @@
 
 import java.nio.ByteBuffer;
 import java.nio.charset.CharacterCodingException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
 
 import com.google.common.collect.Lists;
 import org.junit.Assert;
@@ -34,17 +39,21 @@
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.serializers.UUIDSerializer;
 import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
 import org.assertj.core.api.Assertions;
 import org.quicktheories.generators.SourceDSL;
 
@@ -209,8 +218,8 @@ public void testFullRound() throws Exception
 
         ColumnMetadata cdef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
 
-        ImmutableBTreePartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
-        Iterator<Row> iter = readPartition.iterator();
+        Partition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Iterator<Row> iter = readPartition.rowIterator();
 
         compareValues(iter.next().getCell(cdef), "cname1");
         compareValues(iter.next().getCell(cdef), "cname2");
diff --git a/test/unit/org/apache/cassandra/db/marshal/DateRangeIntegrationTest.java b/test/unit/org/apache/cassandra/db/marshal/DateRangeIntegrationTest.java
new file mode 100644
index 000000000000..d444296dbc84
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/DateRangeIntegrationTest.java
@@ -0,0 +1,287 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import com.datastax.driver.core.CodecRegistry;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.TupleValue;
+import com.datastax.driver.core.UDTValue;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.marshal.datetime.DateRange;
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBound.Precision;
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBuilder;
+import org.apache.cassandra.db.marshal.datetime.DateRangeCodec;
+import org.hamcrest.Matchers;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+
+public class DateRangeIntegrationTest extends CQLTester
+{
+    @BeforeClass
+    public static void setup()
+    {
+        CodecRegistry.DEFAULT_INSTANCE.register(DateRangeCodec.instance);
+    }
+
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
+
+    @Test
+    public void testDateRangeAsPrimaryKey() throws Throwable
+    {
+        String keyspace = randomKeyspace();
+        executeNet(String.format("CREATE KEYSPACE %s WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}", keyspace));
+        executeNet("USE " + keyspace);
+        executeNet("CREATE TABLE dr (k 'DateRangeType' PRIMARY KEY, v int)");
+        executeNet("INSERT INTO dr (k, v) VALUES ('[2010-12-03 TO 2010-12-04]', 1)");
+        executeNet("INSERT INTO dr (k, v) VALUES ('[2015-12-03T10:15:30.001Z TO 2016-01-01T00:05:11.967Z]', 2)");
+
+        ResultSet results = executeNet(String.format("SELECT * FROM %s.dr", keyspace));
+        List<Row> rows = results.all();
+
+        assertEquals(2, rows.size());
+        DateRange expected = dateRange("2010-12-03T00:00:00.000Z", Precision.DAY, "2010-12-04T23:59:59.999Z", Precision.DAY);
+        assertEquals(expected, rows.get(0).get("k", DateRange.class));
+        expected = dateRange("2015-12-03T10:15:30.001Z", Precision.MILLISECOND, "2016-01-01T00:05:11.967Z", Precision.MILLISECOND);
+        assertEquals(expected, rows.get(1).get("k", DateRange.class));
+
+        results = executeNet("SELECT * FROM dr WHERE k = '[2015-12-03T10:15:30.001Z TO 2016-01-01T00:05:11.967]'");
+        rows = results.all();
+
+        assertEquals(1, rows.size());
+        assertEquals(2, rows.get(0).getInt("v"));
+
+
+        flush(keyspace, "dr");
+
+        results = executeNet("SELECT * FROM dr");
+        rows = results.all();
+
+        assertEquals(2, rows.size());
+        expected = dateRange("2015-12-03T10:15:30.001Z", Precision.MILLISECOND, "2016-01-01T00:05:11.967Z", Precision.MILLISECOND);
+        assertEquals(expected, rows.get(1).get("k", DateRange.class));
+    }
+
+    @Test
+    public void testCreateDateRange() throws Throwable
+    {
+        String keyspace = randomKeyspace();
+        executeNet(String.format("CREATE KEYSPACE %s WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}", keyspace));
+        executeNet("USE " + keyspace);
+        executeNet("CREATE TABLE dr (k int PRIMARY KEY, v 'DateRangeType')");
+        executeNet("INSERT INTO dr (k, v) VALUES (1, '[2000-01-01T10:15:30.301Z TO *]')");
+        executeNet("INSERT INTO dr (k, v) VALUES (2, '[2000-02 TO 2000-03]')");
+        executeNet("INSERT INTO dr (k, v) VALUES (3, '[* TO 2020]')");
+        executeNet("INSERT INTO dr (k, v) VALUES (4, null)");
+        executeNet("INSERT INTO dr (k) VALUES (5)");
+
+        ResultSet results = executeNet("SELECT * FROM dr");
+        List<Row> rows = results.all();
+        assertEquals(5, rows.size());
+        DateRange dateRange = rows.get(4).get("v", DateRange.class);
+        assertNotNull(dateRange);
+        DateRange expected = DateRangeBuilder.dateRange()
+                .withUnboundedLowerBound()
+                .withUpperBound("2020-12-31T23:59:59.999Z", Precision.YEAR)
+                .build();
+        assertEquals(expected, dateRange);
+    }
+
+    @Test
+    public void testInvalidDateRangeOrder() throws Throwable
+    {
+        String keyspace = randomKeyspace();
+        executeNet(String.format("CREATE KEYSPACE %s WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}", keyspace));
+        executeNet("USE " + keyspace);
+        executeNet("CREATE TABLE dr (k int PRIMARY KEY, v 'DateRangeType')");
+
+        expectedException.expect(InvalidQueryException.class);
+        expectedException.expectMessage("Wrong order: 2020-01-01T10:15:30.009Z TO 2010-01-01T00:05:11.031Z");
+        expectedException.expectMessage("Could not parse date range: [2020-01-01T10:15:30.009Z TO 2010-01-01T00:05:11.031Z]");
+        executeNet("INSERT INTO dr (k, v) VALUES (1, '[2020-01-01T10:15:30.009Z TO 2010-01-01T00:05:11.031Z]')");
+    }
+
+    @Test
+    public void testDateRangeInTuples() throws Throwable
+    {
+        String keyspace = randomKeyspace();
+        executeNet(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy' , 'replication_factor': '1'}", keyspace));
+        executeNet("USE " + keyspace);
+        executeNet("CREATE TYPE IF NOT EXISTS test_udt (i int, range 'DateRangeType')");
+        executeNet("CREATE TABLE dr (k int PRIMARY KEY, u test_udt, uf frozen<test_udt>, t tuple<'DateRangeType', int>, tf frozen<tuple<'DateRangeType', int>>)");
+
+        executeNet("INSERT INTO dr (k, u, uf, t, tf) VALUES (" +
+                "1, " +
+                "{i: 10, range: '[2000-01-01T10:15:30.003Z TO 2020-01-01T10:15:30.001Z]'}, " +
+                "{i: 20, range: '[2000-01-01T10:15:30.003Z TO 2020-01-01T10:15:30.001Z]'}, " +
+                "('[2000-01-01T10:15:30.003Z TO 2020-01-01T10:15:30.001Z]', 30), " +
+                "('[2000-01-01T10:15:30.003Z TO 2020-01-01T10:15:30.001Z]', 40))");
+
+        DateRange expected = dateRange("2000-01-01T10:15:30.003Z", Precision.MILLISECOND, "2020-01-01T10:15:30.001Z", Precision.MILLISECOND);
+        ResultSet results = executeNet("SELECT * FROM dr");
+        List<Row> rows = results.all();
+        assertEquals(1, rows.size());
+
+        UDTValue u = rows.get(0).get("u", UDTValue.class);
+        DateRange dateRange = u.get("range", DateRange.class);
+        assertEquals(expected, dateRange);
+        assertEquals(10, u.getInt("i"));
+
+        u = rows.get(0).get("uf", UDTValue.class);
+        dateRange = u.get("range", DateRange.class);
+        assertEquals(expected, dateRange);
+        assertEquals(20, u.getInt("i"));
+
+        TupleValue t = rows.get(0).get("t", TupleValue.class);
+        dateRange = t.get(0, DateRange.class);
+        assertEquals(expected, dateRange);
+        assertEquals(30, t.getInt(1));
+
+        t = rows.get(0).get("tf", TupleValue.class);
+        dateRange = t.get(0, DateRange.class);
+        assertEquals(expected, dateRange);
+        assertEquals(40, t.getInt(1));
+    }
+
+    @Test
+    public void testDateRangeInCollections() throws Throwable
+    {
+        String keyspace = randomKeyspace();
+        executeNet(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy' , 'replication_factor': '1'}", keyspace));
+        executeNet("USE " + keyspace);
+        executeNet("CREATE TABLE dr (k int PRIMARY KEY, l list<'DateRangeType'>, s set<'DateRangeType'>, dr2i map<'DateRangeType', int>, i2dr map<int, 'DateRangeType'>)");
+
+        executeNet("INSERT INTO dr (k, l, s, i2dr, dr2i) VALUES (" +
+                "1, " +
+                "['[2000-01-01T10:15:30.001Z TO 2020]', '[2010-01-01T10:15:30.001Z TO 2020]', '2001-01-02'], " +
+                "{'[2000-01-01T10:15:30.001Z TO 2020]', '[2000-01-01T10:15:30.001Z TO 2020]', '[2010-01-01T10:15:30.001Z TO 2020]'}, " +
+                "{1: '[2000-01-01T10:15:30.001Z TO 2020]', 2: '[2010-01-01T10:15:30.001Z TO 2020]'}, " +
+                "{'[2000-01-01T10:15:30.001Z TO 2020]': 1, '[2010-01-01T10:15:30.001Z TO 2020]': 2})");
+
+        ResultSet results = executeNet("SELECT * FROM dr");
+        List<Row> rows = results.all();
+        assertEquals(1, rows.size());
+
+        List<DateRange> drList = rows.get(0).getList("l", DateRange.class);
+        assertEquals(3, drList.size());
+        assertEquals(dateRange("2000-01-01T10:15:30.001Z", Precision.MILLISECOND, "2020-12-31T23:59:59.999Z", Precision.YEAR), drList.get(0));
+        assertEquals(dateRange("2010-01-01T10:15:30.001Z", Precision.MILLISECOND, "2020-12-31T23:59:59.999Z", Precision.YEAR), drList.get(1));
+        assertEquals(DateRangeBuilder.dateRange().withLowerBound("2001-01-02T00:00:00.000Z", Precision.DAY).build(), drList.get(2));
+
+        Set<DateRange> drSet = rows.get(0).getSet("s", DateRange.class);
+        assertEquals(2, drSet.size());
+        assertEquals(
+                Sets.newHashSet(
+                        dateRange("2000-01-01T10:15:30.001Z", Precision.MILLISECOND, "2020-12-31T23:59:59.999Z", Precision.YEAR),
+                        dateRange("2010-01-01T10:15:30.001Z", Precision.MILLISECOND, "2020-12-31T23:59:59.999Z", Precision.YEAR)),
+                drSet);
+
+        Map<DateRange, Integer> dr2i = rows.get(0).getMap("dr2i", DateRange.class, Integer.class);
+        assertEquals(2, dr2i.size());
+        assertEquals(1, (int) dr2i.get(dateRange("2000-01-01T10:15:30.001Z", Precision.MILLISECOND, "2020-12-31T23:59:59.999Z", Precision.YEAR)));
+        assertEquals(2, (int) dr2i.get(dateRange("2010-01-01T10:15:30.001Z", Precision.MILLISECOND, "2020-12-31T23:59:59.999Z", Precision.YEAR)));
+
+        Map<Integer, DateRange> i2dr = rows.get(0).getMap("i2dr", Integer.class, DateRange.class);
+        assertEquals(2, i2dr.size());
+        assertEquals(dateRange("2000-01-01T10:15:30.001Z", Precision.MILLISECOND, "2020-12-31T23:59:59.999Z", Precision.YEAR), i2dr.get(1));
+        assertEquals(dateRange("2010-01-01T10:15:30.001Z", Precision.MILLISECOND, "2020-12-31T23:59:59.999Z", Precision.YEAR), i2dr.get(2));
+    }
+
+    @Test
+    public void testPreparedStatementsWithDateRange() throws Throwable
+    {
+        String keyspace = randomKeyspace();
+        executeNet(String.format("CREATE KEYSPACE %s WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}", keyspace));
+        executeNet("USE " + keyspace);
+        executeNet("CREATE TABLE dr (k int PRIMARY KEY, v 'DateRangeType')");
+
+        Session session = sessionNet();
+        PreparedStatement statement = session.prepare(String.format("INSERT INTO %s.dr (k,v) VALUES(?,?)", keyspace));
+
+        DateRange dateRange = dateRange("2007-12-03T00:00:00.000Z", Precision.DAY, "2007-12-17T00:00:00.000Z", Precision.MONTH);
+        session.execute(statement.bind(1, dateRange));
+
+        ResultSet results = executeNet("SELECT * FROM dr");
+        List<Row> rows = results.all();
+        assertEquals(1, rows.size());
+
+        DateRange actual = rows.get(0).get("v", DateRange.class);
+        assertEquals(Precision.DAY, actual.getLowerBound().getPrecision());
+        assertEquals(Precision.MONTH, actual.getUpperBound().getPrecision());
+        assertEquals("[2007-12-03 TO 2007-12]", actual.formatToSolrString());
+
+        results = executeNet("SELECT JSON * FROM dr");
+        assertThat(results.all().get(0).toString(), Matchers.containsString("\"v\": \"[2007-12-03 TO 2007-12]\""));
+    }
+
+    @Test
+    public void testSemanticallyEquivalentDateRanges() throws Throwable
+    {
+        String keyspace = randomKeyspace();
+        executeNet(String.format("CREATE KEYSPACE %s WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}", keyspace));
+        executeNet("USE " + keyspace);
+        executeNet("CREATE TABLE dr (k int, c0 'DateRangeType', PRIMARY KEY (k, c0))");
+
+        executeNet("INSERT INTO dr (k, c0) VALUES (1, '2016-01-01')");
+        executeNet("INSERT INTO dr (k, c0) VALUES (1, '[2016-01-01 TO 2016-01-01]')");
+        executeNet("INSERT INTO dr (k, c0) VALUES (1, '[2016-01-01T00:00:00.000Z TO 2016-01-01]')");
+        executeNet("INSERT INTO dr (k, c0) VALUES (1, '[2016-01-01T00:00:00.000Z TO 2016-01-01:23:59:59.999Z]')");
+        executeNet("INSERT INTO dr (k, c0) VALUES (1, '[2016-01-01 TO 2016-01-01:23:59:59.999Z]')");
+
+        ResultSet results = executeNet("SELECT * FROM dr");
+        assertEquals(5, results.all().size());
+
+        results = executeNet("SELECT * FROM dr WHERE c0 = '2016-01-01' ALLOW FILTERING");
+        assertEquals(1, results.all().size());
+
+        results = executeNet("SELECT * FROM dr WHERE k = 1 AND c0 = '[2016-01-01T00:00:00.000Z TO 2016-01-01:23:59:59.999Z]'");
+        assertEquals(1, results.all().size());
+    }
+
+    private String randomKeyspace()
+    {
+        return "ks" + System.nanoTime();
+    }
+
+    private DateRange dateRange(String lowerBound, Precision lowerBoundPrecision, String upperBound, Precision upperBoundPrecision)
+    {
+        return DateRangeBuilder.dateRange()
+                .withLowerBound(lowerBound, lowerBoundPrecision)
+                .withUpperBound(upperBound, upperBoundPrecision)
+                .build();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/DateRangeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/DateRangeTypeTest.java
new file mode 100644
index 000000000000..fd622e7bb745
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/DateRangeTypeTest.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.datetime.DateRange;
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBound.Precision;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBuilder.dateRange;
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Parameterized.class)
+public class DateRangeTypeTest
+{
+    private final DateRangeType dateRangeType = DateRangeType.instance;
+
+    @Parameterized.Parameter(0)
+    public DateRange dataRange;
+
+    @Parameterized.Parameter(1)
+    public String source;
+
+    @Test
+    public void shouldFormatDateRangeAsJson()
+    {
+        ByteBuffer bytes = dateRangeType.decompose(dataRange);
+        String actualJson = dateRangeType.toJSONString(bytes, ProtocolVersion.CURRENT);
+        assertEquals('"' + source + '"', actualJson);
+    }
+
+    @Test
+    public void shouldCreateProperDateRangeFromString()
+    {
+        ByteBuffer dateRangeBytes = dateRangeType.fromString(source);
+        DateRange actual = dateRangeType.getSerializer().deserialize(dateRangeBytes);
+        assertEquals(dataRange, actual);
+    }
+
+    @SuppressWarnings("unused")
+    @Parameterized.Parameters(name = "dataRange = {0}, source = {1}")
+    public static Collection<Object[]> testData()
+    {
+        return Arrays.asList(
+                new Object[]{
+                        dateRange()
+                                .withLowerBound("1950-01-01T00:00:00.000Z", Precision.YEAR)
+                                .withUnboundedUpperBound()
+                                .build(),
+                        "[1950 TO *]"
+                },
+                new Object[]{
+                        dateRange()
+                                .withLowerBound("1998-01-01T00:00:00.000Z", Precision.MILLISECOND)
+                                .withUpperBound("1999-02-01T00:00:00.000Z", Precision.DAY)
+                                .build(),
+                        "[1998-01-01T00:00:00.000Z TO 1999-02-01]"
+                },
+                new Object[]{
+                        dateRange()
+                                .withLowerBound("1930-12-03T01:01:01.003Z", Precision.DAY)
+                                .withUpperBound("1951-01-02T00:00:00.003Z", Precision.MILLISECOND)
+                                .build(),
+                        "[1930-12-03 TO 1951-01-02T00:00:00.003Z]"
+                },
+                new Object[]{
+                        dateRange()
+                                .withUnboundedLowerBound()
+                                .withUpperBound("2014-01-02T00:00:00.003Z", Precision.YEAR)
+                                .build(),
+                        "[* TO 2014]"
+                },
+                new Object[]{
+                        dateRange()
+                                .withUnboundedLowerBound()
+                                .withUnboundedUpperBound()
+                                .build(),
+                        "[* TO *]"
+                },
+                new Object[]{
+                        dateRange()
+                                .withLowerBound("1966-03-03T03:30:30.030Z", Precision.YEAR)
+                                .build(),
+                        "1966"
+                },
+                new Object[]{
+                        dateRange()
+                                .withLowerBound("1700-01-01T00:00:00.000Z", Precision.MILLISECOND)
+                                .build(),
+                        "1700-01-01T00:00:00.000Z"
+                },
+                new Object[]{
+                        dateRange()
+                                .withLowerBound("-0009-01-01T00:00:00.000Z", Precision.YEAR)
+                                .build(),
+                        "-0009"
+                }
+        );
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
index c7952e6d84ec..5246e3995692 100644
--- a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
@@ -20,11 +20,12 @@
 
 import java.nio.ByteBuffer;
 import java.nio.charset.CharacterCodingException;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.UUID;
+import java.util.stream.Stream;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -34,40 +35,37 @@
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.serializers.MarshalException;
-import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
 import org.assertj.core.api.Assertions;
 
 public class DynamicCompositeTypeTest
 {
     private static final String KEYSPACE1 = "DynamicCompositeType";
     private static final String CF_STANDARDDYNCOMPOSITE = "StandardDynamicComposite";
-    public static Map<Byte, AbstractType<?>> aliases = new HashMap<>();
 
-    private static final DynamicCompositeType comparator;
-    static
-    {
-        aliases.put((byte)'b', BytesType.instance);
-        aliases.put((byte)'B', ReversedType.getInstance(BytesType.instance));
-        aliases.put((byte)'t', TimeUUIDType.instance);
-        aliases.put((byte)'T', ReversedType.getInstance(TimeUUIDType.instance));
-        comparator = DynamicCompositeType.getInstance(aliases);
-    }
+    public final static Map<Byte, AbstractType<?>> aliases = ImmutableMap.<Byte, AbstractType<?>>builder()
+                                                             .put((byte) 'b', BytesType.instance)
+                                                             .put((byte) 'B', ReversedType.getInstance(BytesType.instance))
+                                                             .put((byte) 't', TimeUUIDType.instance)
+                                                             .put((byte) 'T', ReversedType.getInstance(TimeUUIDType.instance))
+                                                             .build();
 
-    private static final int UUID_COUNT = 3;
-    public static final UUID[] uuids = new UUID[UUID_COUNT];
-    static
-    {
-        for (int i = 0; i < UUID_COUNT; ++i)
-            uuids[i] = nextTimeUUID().asUUID();
-    }
+    public static final DynamicCompositeType comparator = DynamicCompositeType.getInstance(aliases);
+
+    public static final int UUID_COUNT = 3;
+    public static final UUID[] uuids = Stream.generate(() -> nextTimeUUID().asUUID()).limit(UUID_COUNT).toArray(UUID[]::new);
 
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
@@ -209,8 +207,8 @@ public void testFullRound() throws Exception
 
         ColumnMetadata cdef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
 
-        ImmutableBTreePartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
-        Iterator<Row> iter = readPartition.iterator();
+        Partition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Iterator<Row> iter = readPartition.rowIterator();
 
         compareValues(iter.next().getCell(cdef), "cname1");
         compareValues(iter.next().getCell(cdef), "cname2");
@@ -246,8 +244,8 @@ public void testFullRoundReversed() throws Exception
 
         ColumnMetadata cdef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val"));
 
-        ImmutableBTreePartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
-        Iterator<Row> iter = readPartition.iterator();
+        Partition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+        Iterator<Row> iter = readPartition.rowIterator();
 
         compareValues(iter.next().getCell(cdef), "cname5");
         compareValues(iter.next().getCell(cdef), "cname4");
@@ -327,12 +325,13 @@ private static ByteBuffer createDynamicCompositeKey(String s, UUID uuid, int i,
         return createDynamicCompositeKey(s, uuid, i, lastIsOne, false);
     }
 
+    @VisibleForTesting
     public static ByteBuffer createDynamicCompositeKey(String s, UUID uuid, int i, boolean lastIsOne, boolean reversed)
     {
         String intType = (reversed ? "ReversedType(IntegerType)" : "IntegerType");
-        ByteBuffer bytes = ByteBufferUtil.bytes(s);
+        ByteBuffer bytes = s != null ? ByteBufferUtil.bytes(s) : null;
         int totalSize = 0;
-        if (s != null)
+        if (bytes != null)
         {
             totalSize += 2 + 2 + bytes.remaining() + 1;
             if (uuid != null)
@@ -347,7 +346,7 @@ public static ByteBuffer createDynamicCompositeKey(String s, UUID uuid, int i, b
 
         ByteBuffer bb = ByteBuffer.allocate(totalSize);
 
-        if (s != null)
+        if (bytes != null)
         {
             bb.putShort((short)(0x8000 | (reversed ? 'B' : 'b')));
             bb.putShort((short) bytes.remaining());
diff --git a/test/unit/org/apache/cassandra/db/marshal/GeometricTypeTests.java b/test/unit/org/apache/cassandra/db/marshal/GeometricTypeTests.java
new file mode 100644
index 000000000000..cbc0a4975a49
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/GeometricTypeTests.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+
+import com.esri.core.geometry.Polyline;
+import com.esri.core.geometry.ogc.OGCLineString;
+import com.esri.core.geometry.ogc.OGCPolygon;
+import org.apache.cassandra.db.marshal.geometry.LineString;
+import org.apache.cassandra.db.marshal.geometry.OgcGeometry;
+import org.apache.cassandra.db.marshal.geometry.Point;
+import org.apache.cassandra.db.marshal.geometry.Polygon;
+
+public class GeometricTypeTests
+{
+    public static Point p(double x, double y)
+    {
+        return new Point(x, y);
+    }
+
+    public static com.esri.core.geometry.Point ep(double x, double y)
+    {
+        return new com.esri.core.geometry.Point(x, y);
+    }
+
+    public static com.esri.core.geometry.Point ep(Point p)
+    {
+        return new com.esri.core.geometry.Point(p.getOgcPoint().X(), p.getOgcPoint().Y());
+    }
+
+    public static LineString lineString(Point p1, Point p2, Point... pn)
+    {
+        Polyline polyline = new Polyline(ep(p1), ep(p2));
+        for (Point p : pn)
+        {
+            polyline.lineTo(ep(p));
+        }
+
+        return new LineString(new OGCLineString(polyline, 0, OgcGeometry.SPATIAL_REFERENCE_4326));
+    }
+
+    public static Polygon polygon(Point p1, Point p2, Point p3, Point... pn)
+    {
+        com.esri.core.geometry.Polygon polygon = new com.esri.core.geometry.Polygon();
+        polygon.startPath(ep(p1));
+        polygon.lineTo(ep(p2));
+        polygon.lineTo(ep(p3));
+        for (Point p : pn)
+        {
+            polygon.lineTo(ep(p));
+        }
+        return new Polygon(new OGCPolygon(polygon, OgcGeometry.SPATIAL_REFERENCE_4326));
+    }
+
+    /**
+     * pads the buffer with some leading and trailing data to aid testing
+     * proper deserialization from continuous buffers
+     */
+    public static ByteBuffer padBuffer(ByteBuffer bb)
+    {
+        ByteBuffer padded = ByteBuffer.allocate(8 + bb.limit()).putInt(49).put(bb).putInt(50);
+        padded.position(4);
+        return padded;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/GeometryCodecTest.java b/test/unit/org/apache/cassandra/db/marshal/GeometryCodecTest.java
new file mode 100644
index 000000000000..efd02e2c0aeb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/GeometryCodecTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.datastax.driver.core.ProtocolVersion;
+import org.apache.cassandra.db.marshal.geometry.OgcGeometry;
+import org.apache.cassandra.db.marshal.geometry.Point;
+
+public class GeometryCodecTest
+{
+    private final GeometryCodec<Point> codec = new GeometryCodec<>(PointType.instance);
+
+    @Test
+    public void testFormat()
+    {
+        Assert.assertEquals("NULL", codec.format(null));
+        Assert.assertEquals("POINT (5.4 1)", codec.format(new Point(5.4, 1.0)));
+    }
+
+    @Test
+    public void testParse()
+    {
+        Assert.assertEquals(null, codec.parse(null));
+        Assert.assertEquals(new Point(5.4, 1.0), codec.parse("POINT (5.4 1)"));
+    }
+
+    @Test
+    public void testSerializationRoundTrip()
+    {
+        Point point = new Point(5.4, 1.0);
+        ByteBuffer serialized = codec.serialize(point, ProtocolVersion.NEWEST_SUPPORTED);
+        OgcGeometry deserialized = codec.deserialize(serialized, ProtocolVersion.NEWEST_SUPPORTED);
+        Assert.assertEquals(point, deserialized);
+    }
+
+    @Test
+    public void testEmptyValuesSerialization()
+    {
+        Assert.assertEquals(null, codec.serialize(null, ProtocolVersion.NEWEST_SUPPORTED));
+        Assert.assertEquals(null, codec.deserialize(null, ProtocolVersion.NEWEST_SUPPORTED));
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/marshal/GeometryIntegrationTest.java b/test/unit/org/apache/cassandra/db/marshal/GeometryIntegrationTest.java
new file mode 100644
index 000000000000..f25fd3f0430a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/GeometryIntegrationTest.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.CodecRegistry;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.marshal.geometry.LineString;
+import org.apache.cassandra.db.marshal.geometry.OgcGeometry;
+import org.apache.cassandra.db.marshal.geometry.Point;
+import org.apache.cassandra.db.marshal.geometry.Polygon;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.lineString;
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.p;
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.polygon;
+
+public class GeometryIntegrationTest extends CQLTester
+{
+
+    @BeforeClass
+    public static void setupCluster() throws IOException
+    {
+        CodecRegistry.DEFAULT_INSTANCE.register(GeometryCodec.pointCodec,
+                                                GeometryCodec.lineStringCodec,
+                                                GeometryCodec.polygonCodec);
+    }
+
+    @Before
+    public void setUpKeyspace() throws Throwable
+    {
+        executeNet("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}");
+    }
+
+    @After
+    public void teardown() throws Throwable
+    {
+        executeNet("DROP KEYSPACE ks;");
+    }
+
+    private <T extends OgcGeometry> void testType(T expected, Class<T> klass, AbstractGeometricType<T> type, String tableName, String wkt, String columnType) throws Throwable
+    {
+        executeNet(String.format("CREATE TABLE ks.%s (k INT PRIMARY KEY, g '%s')", tableName, columnType));
+        executeNet(String.format("INSERT INTO ks.%s (k, g) VALUES (1, '%s')", tableName, wkt));
+
+        ResultSet results = executeNet(String.format("SELECT * FROM ks.%s", tableName));
+        List<Row> rows = results.all();
+        Assert.assertEquals(1, rows.size());
+        Row row = rows.get(0);
+        T actual = row.get("g", klass);
+        Assert.assertEquals(expected, actual);
+        results = executeNet(String.format("SELECT toJson(g) FROM ks.%s", tableName));
+        rows = results.all();
+        Assert.assertEquals(1, rows.size());
+        row = rows.get(0);
+        String actualJson = row.getString("system.tojson(g)");
+        String expectedJson = type.toJSONString(type.getSerializer().serialize(expected), ProtocolVersion.CURRENT);
+        Assert.assertEquals(expectedJson, actualJson);
+    }
+
+    @Test
+    public void pointTest() throws Throwable
+    {
+        executeNet("CREATE TABLE ks.point (k INT PRIMARY KEY, g 'PointType')");
+        String wkt = "POINT(1.1 2.2)";
+        executeNet(String.format("INSERT INTO ks.point (k, g) VALUES (1, '%s')", wkt));
+
+        ResultSet results = executeNet("SELECT * FROM ks.point");
+        List<Row> rows = results.all();
+        Assert.assertEquals(1, rows.size());
+        Row row = rows.get(0);
+        Point point = row.get("g", Point.class);
+        Assert.assertEquals(new Point(1.1, 2.2), point);
+    }
+
+    @Test
+    public void lineStringTest() throws Throwable
+    {
+        LineString expected = lineString(p(30, 10), p(10, 30), p(40, 40));
+        String wkt = "linestring(30 10, 10 30, 40 40)";
+        testType(expected, LineString.class, LineStringType.instance, "linestring", wkt, "LineStringType");
+    }
+
+    @Test
+    public void polygonTest() throws Throwable
+    {
+        Polygon expected = polygon(p(30, 10), p(10, 20), p(20, 40), p(40, 40));
+        String wkt = "polygon((30 10, 40 40, 20 40, 10 20, 30 10))";
+        testType(expected, Polygon.class, PolygonType.instance, "polygon", wkt, "PolygonType");
+    }
+
+    @Test
+    public void primaryKeyTest() throws Throwable
+    {
+        executeNet("CREATE TABLE ks.geo (k 'PointType', c 'LineStringType', g 'PointType', PRIMARY KEY (k, c))");
+        executeNet("INSERT INTO ks.geo (k, c, g) VALUES ('POINT(1 2)', 'linestring(30 10, 10 30, 40 40)', 'POINT(1.1 2.2)')");
+        ResultSet results = executeNet("SELECT * FROM ks.geo");
+        List<Row> rows = results.all();
+        Assert.assertEquals(1, rows.size());
+        Row row = rows.get(0);
+
+        Point point1 = row.get("k", Point.class);
+        Assert.assertEquals(new Point(1, 2), point1);
+
+        LineString lineString = row.get("c", LineString.class);
+        Assert.assertEquals(lineString(p(30, 10), p(10, 30), p(40, 40)), lineString);
+
+        Point point = row.get("g", Point.class);
+        Assert.assertEquals(new Point(1.1, 2.2), point);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/marshal/LineStringTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/LineStringTypeTest.java
new file mode 100644
index 000000000000..8c8667939aa6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/LineStringTypeTest.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.db.marshal.geometry.LineString;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.lineString;
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.p;
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.padBuffer;
+
+public class LineStringTypeTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(LineStringTypeTest.class);
+
+    LineStringType type = LineStringType.instance;
+
+    @Test
+    public void successCase2d()
+    {
+        ByteBuffer actual = type.fromString("linestring(30 10, 10 30, 40 40)");
+
+        ByteBuffer expected = ByteBuffer.allocate(1024).order(ByteOrder.nativeOrder());
+        expected.position(0);
+
+        expected.put((byte) (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? 1 : 0)); // endianness
+        expected.putInt(2);  // type
+        expected.putInt(3);  // num points
+        expected.putDouble(30);  // x1
+        expected.putDouble(10);  // y1
+        expected.putDouble(10);  // x2
+        expected.putDouble(30);  // y2
+        expected.putDouble(40);  // x3
+        expected.putDouble(40);  // y3
+        expected.flip();
+
+        logger.debug("expected: {}", ByteBufferUtil.bytesToHex(expected));
+        logger.debug("actual:   {}", ByteBufferUtil.bytesToHex(actual));
+        String failMsg = String.format("%s != %s", ByteBufferUtil.bytesToHex(actual), ByteBufferUtil.bytesToHex(expected));
+        Assert.assertEquals(failMsg, expected, actual);
+
+        LineString expectedGeometry = lineString(p(30, 10), p(10, 30), p(40, 40));
+        LineString actualGeometry = type.getSerializer().deserialize(actual);
+        logger.debug("expected: {}", expectedGeometry);
+        logger.debug("actual:   {}", actualGeometry);
+        Assert.assertEquals(expectedGeometry, actualGeometry);
+    }
+
+    @Test(expected=MarshalException.class)
+    public void emptyFailure()
+    {
+        type.fromString("linestring()");
+    }
+
+    @Test(expected=MarshalException.class)
+    public void failure3d()
+    {
+        type.fromString("linestring(30 10 20, 10 30 20)");
+    }
+
+    /**
+     * Line strings that cross themselves shouldn't validate
+     */
+    @Test(expected=MarshalException.class)
+    public void simpleFailure()
+    {
+        type.fromString("linestring(0 0, 1 1, 0 1, 1 0)");
+    }
+
+    @Test(expected=MarshalException.class)
+    public void parseFailure()
+    {
+        type.fromString("superlinestring(30 10, 10 30, 40 40)");
+    }
+
+    @Test
+    public void jsonWktInput()
+    {
+        Constants.Value value = (Constants.Value) type.fromJSONObject("linestring(30 10, 10 30, 40 40)");
+        Assert.assertEquals(lineString(p(30, 10), p(10, 30), p(40, 40)), type.getSerializer().deserialize(value.bytes));
+    }
+
+    @Test
+    public void geoJsonInput()
+    {
+        String json = "{\"type\":\"LineString\",\"coordinates\":[[30.0,10.0],[10.0,30.0],[40.0,40.0]]}";
+        Constants.Value value = (Constants.Value) type.fromJSONObject(json);
+        Assert.assertEquals(lineString(p(30, 10), p(10, 30), p(40, 40)), type.getSerializer().deserialize(value.bytes));
+    }
+
+    @Test
+    public void geoJsonOutput()
+    {
+        String json = type.toJSONString(type.getSerializer().serialize(lineString(p(30, 10), p(10, 30), p(40, 40))), ProtocolVersion.CURRENT);
+        Assert.assertEquals("{\"type\":\"LineString\",\"coordinates\":[[30,10],[10,30],[40,40]]}", json);
+        logger.debug(json);
+    }
+
+    /**
+     * Use of absolute indexing in deserializers shouldn't cause problems
+     */
+    @Test
+    public void bufferOffset()
+    {
+        LineString expected = lineString(p(30, 10), p(10, 30), p(40, 40));
+        ByteBuffer bb = padBuffer(type.getSerializer().serialize(expected));
+        type.getSerializer().validate(bb);
+        LineString actual = type.getSerializer().deserialize(bb);
+        Assert.assertEquals(expected, actual);
+    }
+
+    @Test
+    public void bufferBigEndianess()
+    {
+        LineString expected = lineString(p(30, 10), p(10, 30), p(40, 40));
+        ByteBuffer bb = padBuffer(type.getSerializer().serialize(expected));
+        Assert.assertEquals(ByteOrder.BIG_ENDIAN, bb.order());
+    }
+
+}
+
diff --git a/test/unit/org/apache/cassandra/db/marshal/ListTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/ListTypeTest.java
new file mode 100644
index 000000000000..b15569f2c63b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/ListTypeTest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.cassandra.utils.AbstractTypeGenerators;
+import org.assertj.core.api.Assertions;
+import org.quicktheories.core.Gen;
+
+import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport;
+import static org.quicktheories.QuickTheory.qt;
+
+public class ListTypeTest
+{
+    @Test
+    public void testContains()
+    {
+        qt().forAll(AbstractTypeGenerators.primitiveTypeGen())
+            .checkAssert(ListTypeTest::testContains);
+    }
+
+    private static <V> void testContains(AbstractType<V> type)
+    {
+        ListType<V> listType = ListType.getInstance(type, false);
+
+        // generate a list of random values
+        List<V> values = new ArrayList<>();
+        List<ByteBuffer> bytes = new ArrayList<>();
+        Gen<ByteBuffer> gen = getTypeSupport(type).bytesGen();
+        qt().withExamples(100).forAll(gen).checkAssert(v -> {
+            values.add(type.compose(v));
+            bytes.add(v);
+        });
+        ByteBuffer list = listType.decompose(values);
+
+        // verify that the list contains its own elements
+        bytes.forEach(v -> assertContains(listType, list, v, true));
+
+        // verify some random values, contained or not
+        qt().withExamples(100)
+            .forAll(gen)
+            .checkAssert(v -> assertContains(listType, list, v, contains(type, bytes, v)));
+    }
+
+    private static void assertContains(ListType<?> type, ByteBuffer list, ByteBuffer value, boolean expected)
+    {
+        Assertions.assertThat(type.contains(list, value))
+                  .isEqualTo(expected);
+    }
+
+    private static boolean contains(AbstractType<?> type, Iterable<ByteBuffer> values, ByteBuffer value)
+    {
+        for (ByteBuffer v : values)
+        {
+            if (type.compare(v, value) == 0)
+                return true;
+        }
+        return false;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/MapTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/MapTypeTest.java
new file mode 100644
index 000000000000..a68f05dee6c7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/MapTypeTest.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.cassandra.utils.AbstractTypeGenerators;
+import org.assertj.core.api.Assertions;
+import org.quicktheories.core.Gen;
+
+import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport;
+import static org.quicktheories.QuickTheory.qt;
+
+public class MapTypeTest
+{
+    @Test
+    public void testContains()
+    {
+        Gen<AbstractType<?>> primitiveTypeGen = AbstractTypeGenerators.primitiveTypeGen();
+
+        qt().forAll(primitiveTypeGen, primitiveTypeGen)
+            .checkAssert(MapTypeTest::testContains);
+    }
+
+    private static <K, V> void testContains(AbstractType<K> keyType, AbstractType<V> valType)
+    {
+        MapType<K, V> mapType = MapType.getInstance(keyType, valType, false);
+
+        // generate a map of random key-value pairs
+        Map<K, V> entries = new HashMap<>();
+        Map<ByteBuffer, ByteBuffer> bytes = new HashMap<>();
+        Gen<ByteBuffer> keyGen = getTypeSupport(keyType).bytesGen();
+        Gen<ByteBuffer> valGen = getTypeSupport(valType).bytesGen();
+        qt().withExamples(100).forAll(keyGen, valGen).checkAssert((k, v) -> {
+            entries.put(keyType.compose(k), valType.compose(v));
+            bytes.put(k, v);
+        });
+        ByteBuffer map = mapType.decompose(entries);
+
+        // verify that the map contains its own keys and values
+        bytes.values().forEach(v -> assertContains(mapType, map, v, true));
+        bytes.keySet().forEach(k -> assertContainsKey(mapType, map, k, true));
+
+        // verify some random keys and values, contained or not
+        qt().withExamples(100)
+            .forAll(keyGen, valGen)
+            .checkAssert((k, v) -> {
+                assertContains(mapType, map, v, contains(valType, bytes.values(), v));
+                assertContainsKey(mapType, map, k, contains(keyType, bytes.keySet(), k));
+            });
+    }
+
+    private static void assertContains(MapType<?, ?> type, ByteBuffer map, ByteBuffer value, boolean expected)
+    {
+        Assertions.assertThat(type.contains(map, value))
+                  .isEqualTo(expected);
+    }
+
+    private static void assertContainsKey(MapType<?, ?> type, ByteBuffer map, ByteBuffer key, boolean expected)
+    {
+        Assertions.assertThat(type.containsKey(map, key))
+                  .isEqualTo(expected);
+    }
+
+    private static boolean contains(AbstractType<?> type, Iterable<ByteBuffer> values, ByteBuffer value)
+    {
+        for (ByteBuffer v : values)
+        {
+            if (type.compare(v, value) == 0)
+                return true;
+        }
+        return false;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/PointTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/PointTypeTest.java
new file mode 100644
index 000000000000..37c60fb857f9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/PointTypeTest.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.db.marshal.geometry.Point;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.p;
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.padBuffer;
+
+public class PointTypeTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(PointTypeTest.class);
+
+    PointType type = PointType.instance;
+
+    @Test
+    public void successCase2d()
+    {
+        ByteBuffer actual = type.fromString("point(1.1 2.2)");
+
+        ByteBuffer expected = ByteBuffer.allocate(1024).order(ByteOrder.nativeOrder());
+        expected.position(0);
+
+        expected.put((byte) (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? 1 : 0)); // endianness
+        expected.putInt(1);  // type
+        expected.putDouble(1.1); // x
+        expected.putDouble(2.2); // y
+        expected.flip();
+
+        logger.debug("expected: {}", ByteBufferUtil.bytesToHex(expected));
+        logger.debug("actual:   {}", ByteBufferUtil.bytesToHex(actual));
+        String failMsg = String.format("%s != %s", ByteBufferUtil.bytesToHex(actual), ByteBufferUtil.bytesToHex(expected));
+        Assert.assertEquals(failMsg, expected, actual);
+
+        Point point = type.getSerializer().deserialize(actual);
+        Assert.assertEquals(p(1.1, 2.2), point);
+    }
+
+    @Test(expected=MarshalException.class)
+    public void parseFailure()
+    {
+        type.fromString("superpoint(1.1 2.2 3.3)");
+    }
+
+    @Test
+    public void jsonWktInput()
+    {
+        Constants.Value value = (Constants.Value) type.fromJSONObject("point(1 2)");
+        Assert.assertEquals(p(1, 2), type.getSerializer().deserialize(value.bytes));
+    }
+
+    @Test
+    public void geoJsonInput()
+    {
+        String json = "{\"type\":\"Point\",\"coordinates\":[1.0,2.0]}";
+        Constants.Value value = (Constants.Value) type.fromJSONObject(json);
+        Assert.assertEquals(p(1, 2), type.getSerializer().deserialize(value.bytes));
+    }
+
+    @Test
+    public void geoJsonInputWithoutPrecision()
+    {
+        String json = "{\"type\":\"Point\",\"coordinates\":[1,2]}";
+        Constants.Value value = (Constants.Value) type.fromJSONObject(json);
+        Assert.assertEquals(p(1, 2), type.getSerializer().deserialize(value.bytes));
+    }
+
+    @Test
+    public void geoJsonOutput()
+    {
+        String json = type.toJSONString(type.getSerializer().serialize(p(1, 2)), ProtocolVersion.CURRENT);
+        Assert.assertEquals("{\"type\":\"Point\",\"coordinates\":[1,2]}", json);
+        logger.debug(json);
+    }
+
+    /**
+     * Use of absolute indexing in deserializers shouldn't cause problems
+     */
+    @Test
+    public void bufferOffset()
+    {
+        Point expected = p(1, 2);
+        ByteBuffer bb = padBuffer(type.getSerializer().serialize(expected));
+        type.getSerializer().validate(bb);
+        Point actual = type.getSerializer().deserialize(bb);
+        Assert.assertEquals(expected, actual);
+    }
+
+    private static ByteBuffer getExpectedSerialization(Point point, ByteOrder order)
+    {
+        ByteBuffer expected = ByteBuffer.allocate(1024).order(order);
+        expected.put((byte) (order == ByteOrder.LITTLE_ENDIAN ? 1 : 0)); // endianness
+        expected.putInt(1);  // type
+        expected.putDouble(point.getOgcPoint().X()); // x
+        expected.putDouble(point.getOgcPoint().Y()); // y
+        expected.flip();
+        return expected;
+    }
+
+    @Test
+    public void bufferBigEndianess()
+    {
+        Point expected = p(1, 2);
+        ByteBuffer bb = padBuffer(type.getSerializer().serialize(expected));
+        Assert.assertEquals(ByteOrder.BIG_ENDIAN, bb.order());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/PolygonTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/PolygonTypeTest.java
new file mode 100644
index 000000000000..cf414c4ec817
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/PolygonTypeTest.java
@@ -0,0 +1,246 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.esri.core.geometry.ogc.OGCGeometry;
+import com.esri.core.geometry.ogc.OGCPolygon;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.db.marshal.geometry.Polygon;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.p;
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.padBuffer;
+import static org.apache.cassandra.db.marshal.GeometricTypeTests.polygon;
+
+public class PolygonTypeTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(LineStringTypeTest.class);
+
+    private static final PolygonType type = PolygonType.instance;
+
+    @Test
+    public void successCase()
+    {
+        ByteBuffer actualBB = type.fromString("polygon((30 10, 40 40, 20 40, 10 20, 30 10))");
+
+        ByteBuffer expectedBB = ByteBuffer.allocate(1024).order(ByteOrder.nativeOrder());
+        expectedBB.position(0);
+
+        expectedBB.put((byte) (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? 1 : 0)); // endianness
+        expectedBB.putInt(3);  // type
+        expectedBB.putInt(1);  // num rings
+        expectedBB.putInt(5);  // num points (ring 1/1)
+        expectedBB.putDouble(30);  // x1
+        expectedBB.putDouble(10);  // y1
+        expectedBB.putDouble(40);  // x2
+        expectedBB.putDouble(40);  // y2
+        expectedBB.putDouble(20);  // x3
+        expectedBB.putDouble(40);  // y3
+        expectedBB.putDouble(10);  // x4
+        expectedBB.putDouble(20);  // y4
+        expectedBB.putDouble(30);  // x5
+        expectedBB.putDouble(10);  // y5
+        expectedBB.flip();
+
+        logger.debug("expected: {}", ByteBufferUtil.bytesToHex(expectedBB));
+        logger.debug("actual:   {}", ByteBufferUtil.bytesToHex(actualBB));
+        String failMsg = String.format("%s != %s", ByteBufferUtil.bytesToHex(actualBB), ByteBufferUtil.bytesToHex(expectedBB));
+        Assert.assertEquals(failMsg, expectedBB, actualBB);
+
+        Polygon expectedGeometry = polygon(p(30, 10), p(10, 20), p(20, 40), p(40, 40));
+        Polygon actualGeometry = type.getSerializer().deserialize(actualBB);
+        Assert.assertEquals(expectedGeometry, actualGeometry);
+    }
+
+    @Test(expected=MarshalException.class)
+    public void emptyFailure()
+    {
+        type.fromString("polygon(())");
+    }
+
+    @Test(expected=MarshalException.class)
+    public void failure3d()
+    {
+        type.fromString("polygon((30 10 1, 40 40 1, 20 40 1, 10 20 1, 30 10 1))");
+    }
+
+    /**
+     * Line strings that cross themselves shouldn't validate
+     */
+    @Test(expected=MarshalException.class)
+    public void simpleFailure()
+    {
+        type.fromString("polygon((0 0, 1 1, 0 1, 1 0, 0 0))");
+    }
+
+    @Test(expected=MarshalException.class)
+    public void parseFailure()
+    {
+        type.fromString("polygon123((30 10, 40 40, 20 40, 10 20, 30 10))");
+    }
+
+    @Test
+    public void jsonWktInput()
+    {
+        Constants.Value value = (Constants.Value) type.fromJSONObject("polygon((30 10, 40 40, 20 40, 10 20, 30 10))");
+        Assert.assertEquals(polygon(p(30, 10), p(10, 20), p(20, 40), p(40, 40)), type.getSerializer().deserialize(value.bytes));
+    }
+
+    @Test
+    public void geoJsonInput()
+    {
+        String json = "{\"type\":\"Polygon\",\"coordinates\":[[[30.0,10.0],[10.0,20.0],[20.0,40.0],[40.0,40.0],[30.0,10.0]]]}";
+        Constants.Value value = (Constants.Value) type.fromJSONObject(json);
+        Assert.assertEquals(polygon(p(30, 10), p(10, 20), p(20, 40), p(40, 40)), type.getSerializer().deserialize(value.bytes));
+    }
+
+    @Test
+    public void geoJsonInputNoPrecision()
+    {
+        String json = "{\"type\":\"Polygon\",\"coordinates\":[[[30,10],[10,20],[20,40],[40,40],[30,10]]]}";
+        Constants.Value value = (Constants.Value) type.fromJSONObject(json);
+        Assert.assertEquals(polygon(p(30, 10), p(10, 20), p(20, 40), p(40, 40)), type.getSerializer().deserialize(value.bytes));
+    }
+
+    @Test
+    public void geoJsonOutputWithDoubles()
+    {
+        String json = type.toJSONString(type.getSerializer().serialize(polygon(p(30.1111, 10.2), p(10.3, 20.4), p(20.5, 40.6), p(40.7, 40.8))), ProtocolVersion.CURRENT);
+        logger.debug(json);
+        Assert.assertEquals("{\"type\":\"Polygon\",\"coordinates\":[[[30.1111,10.2],[40.7,40.8],[20.5,40.6],[10.3,20.4],[30.1111,10.2]]]}", json);
+    }
+
+    @Test
+    public void geoJsonOutput()
+    {
+        String json = type.toJSONString(type.getSerializer().serialize(polygon(p(30, 10), p(10, 20), p(20, 40), p(40, 40))), ProtocolVersion.CURRENT);
+        logger.debug(json);
+        Assert.assertEquals("{\"type\":\"Polygon\",\"coordinates\":[[[30,10],[40,40],[20,40],[10,20],[30,10]]]}", json);
+    }
+
+    /**
+     * Use of absolute indexing in deserializers shouldn't cause problems
+     */
+    @Test
+    public void bufferOffset()
+    {
+        Polygon expected = polygon(p(30, 10), p(10, 20), p(20, 40), p(40, 40));
+        ByteBuffer bb = padBuffer(type.getSerializer().serialize(expected));
+        type.getSerializer().validate(bb);
+        Polygon actual = type.getSerializer().deserialize(bb);
+        Assert.assertEquals(expected, actual);
+    }
+
+    /**
+     * Duplicates DSP-10070
+     * There are some cases where esri can parse wkt into an invalid geometry object, but
+     * can't then convert the invalid geometry to a wkt string. We should catch these cases
+     * and throw a MarshalException
+     */
+    @Test(expected=MarshalException.class)
+    public void invalidInnerRingWkt()
+    {
+        String wkt = "POLYGON ((0.0 0.0, 0.0 10.0, 10.0 10.0, 10.0 0.0, 0.0 0.0), (1.0 10.0, 9.0 0.0, 9.0 9.0, 0.0 9.0, 0.0 0.0))";
+        OGCGeometry geometry = OGCGeometry.fromText(wkt);
+        Assert.assertTrue(geometry instanceof OGCPolygon);
+        new Polygon((OGCPolygon) geometry);
+    }
+
+    /**
+     * Duplicates DSP-10070
+     * There are some cases where esri can parse wkb into an invalid geometry object, but
+     * can't then convert the invalid geometry back to wkb. We should catch these cases
+     * and throw a MarshalException
+     */
+    @Test(expected=MarshalException.class)
+    public void invalidInnerRingWkb()
+    {
+        ByteBuffer bb = ByteBuffer.allocate(1024).order(ByteOrder.nativeOrder());
+        bb.position(0);
+
+        bb.put((byte) (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? 1 : 0)); // endianness
+        bb.putInt(3);  // type
+        bb.putInt(2);  // num rings
+        bb.putInt(5);  // num points (ring 1/2)
+        bb.putDouble(0).putDouble(0);
+        bb.putDouble(0).putDouble(10);
+        bb.putDouble(10).putDouble(10);
+        bb.putDouble(10).putDouble(0);
+        bb.putDouble(0).putDouble(0);
+        bb.putInt(5);  // num points (ring 1/2)
+        bb.putDouble(1).putDouble(10);
+        bb.putDouble(9).putDouble(0);
+        bb.putDouble(9).putDouble(9);
+        bb.putDouble(0).putDouble(9);
+        bb.putDouble(0).putDouble(0);
+        bb.flip();
+
+        type.getSerializer().validate(bb);
+    }
+
+    /**
+     * DSP-10092
+     * Tests that a polygon serialized with a clockwise outer ring and no closing point fails validation, since
+     * it's normalized form has the closing point with points defined counterclockwise.
+     *
+     * Esri 'helps' us by normalizing the geometries it deserializes. Since some values are reserialized from the
+     * deserialized objects, and some aren't, we need to make sure that binary data we get can be reserialized into
+     * equal bytes. Otherwise, users can run into issues with data appearing to disappear.
+     */
+    @Test(expected=MarshalException.class)
+    public void denormalizedPolygon()
+    {
+        ByteBuffer bb = ByteBuffer.allocate(1024).order(ByteOrder.nativeOrder());
+        bb.position(0);
+
+        bb.put((byte) (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? 1 : 0)); // endianness
+        bb.putInt(3);  // type
+        bb.putInt(1);  // num rings
+        bb.putInt(3);  // num points (ring 1/1)
+        bb.putDouble(0);  // x1
+        bb.putDouble(0);  // y1
+        bb.putDouble(1);  // x2
+        bb.putDouble(1);  // y2
+        bb.putDouble(1);  // x3
+        bb.putDouble(0);  // y3
+        bb.flip();
+
+        type.getSerializer().validate(bb);
+    }
+
+    @Test
+    public void bufferBigEndianess()
+    {
+        Polygon expected = polygon(p(30, 10), p(10, 20), p(20, 40), p(40, 40));
+        ByteBuffer bb = padBuffer(type.getSerializer().serialize(expected));
+        Assert.assertEquals(ByteOrder.BIG_ENDIAN, bb.order());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/ReversedTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/ReversedTypeTest.java
index 7dfbc2746bbe..df54ddb441a1 100644
--- a/test/unit/org/apache/cassandra/db/marshal/ReversedTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/ReversedTypeTest.java
@@ -20,6 +20,9 @@
 
 import org.junit.Test;
 
+import org.apache.cassandra.cql3.CQL3Type;
+import org.assertj.core.api.Assertions;
+
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
 import static org.apache.cassandra.utils.ByteBufferUtil.EMPTY_BYTE_BUFFER;
 
@@ -28,7 +31,7 @@ public class ReversedTypeTest
     @Test
     public void testReverseComparison()
     {
-        ReversedType<Long> t = ReversedType.getInstance(LongType.instance);
+        AbstractType<Long> t = ReversedType.getInstance(LongType.instance);
 
         assert t.compare(bytes(2L), bytes(4L)) > 0;
         assert t.compare(bytes(4L), bytes(2L)) < 0;
@@ -37,4 +40,19 @@ public void testReverseComparison()
         assert t.compare(EMPTY_BYTE_BUFFER, bytes(2L)) > 0;
         assert t.compare(bytes(2L), EMPTY_BYTE_BUFFER) < 0;
     }
+
+    @Test
+    public void testReverseOfReversed()
+    {
+        for (CQL3Type.Native cql3Type : CQL3Type.Native.values())
+        {
+            AbstractType<?> type = cql3Type.getType();
+            AbstractType<?> reversed = ReversedType.getInstance(type);
+            Assertions.assertThat(type)
+                      .isNotEqualTo(reversed)
+                      .isEqualTo(reversed.unwrap());
+
+            Assertions.assertThatIllegalArgumentException().isThrownBy(() -> ReversedType.getInstance(reversed));
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/marshal/SetTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/SetTypeTest.java
new file mode 100644
index 000000000000..787e6a46b50d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/SetTypeTest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal;
+
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.junit.Test;
+
+import org.apache.cassandra.utils.AbstractTypeGenerators;
+import org.assertj.core.api.Assertions;
+import org.quicktheories.core.Gen;
+
+import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport;
+import static org.quicktheories.QuickTheory.qt;
+
+public class SetTypeTest
+{
+    @Test
+    public void testContains()
+    {
+        qt().forAll(AbstractTypeGenerators.primitiveTypeGen())
+            .checkAssert(SetTypeTest::testContains);
+    }
+
+    private static <V> void testContains(AbstractType<V> type)
+    {
+        SetType<V> setType = SetType.getInstance(type, false);
+
+        // generate a set of random values
+        Set<V> values = new HashSet<>();
+        Set<ByteBuffer> bytes = new HashSet<>();
+        Gen<ByteBuffer> gen = getTypeSupport(type).bytesGen();
+        qt().withExamples(100).forAll(gen).checkAssert(x -> {
+            values.add(type.compose(x));
+            bytes.add(x);
+        });
+        ByteBuffer set = setType.decompose(values);
+
+        // verify that the set contains its own elements
+        bytes.forEach(v -> assertContains(setType, set, v, true));
+
+        // verify some random values, contained or not
+        qt().withExamples(100)
+            .forAll(gen)
+            .checkAssert(v -> assertContains(setType, set, v, contains(type, bytes, v)));
+    }
+
+    private static void assertContains(SetType<?> type, ByteBuffer set, ByteBuffer value, boolean expected)
+    {
+        Assertions.assertThat(type.contains(set, value))
+                  .isEqualTo(expected);
+    }
+
+    private static boolean contains(AbstractType<?> type, Iterable<ByteBuffer> values, ByteBuffer value)
+    {
+        for (ByteBuffer v : values)
+        {
+            if (type.compare(v, value) == 0)
+                return true;
+        }
+        return false;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java b/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java
index 0247681c1dd0..f03adf55af44 100644
--- a/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java
@@ -18,21 +18,30 @@
  */
 package org.apache.cassandra.db.marshal;
 
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Consumer;
+
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.dht.*;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.OrderPreservingPartitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.SyntaxException;
 
-import java.util.function.Consumer;
+import static org.apache.cassandra.Util.makeUDT;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class TypeParserTest
 {
@@ -70,11 +79,11 @@ public void testParse() throws ConfigurationException, SyntaxException
 
         type = TypeParser.parse("LongType(reversed=true)");
         assert type == ReversedType.getInstance(LongType.instance);
-        assert ((ReversedType<?>)type).baseType == LongType.instance;
+        assert type.unwrap() == LongType.instance;
 
         type = TypeParser.parse("LongType(reversed)");
         assert type == ReversedType.getInstance(LongType.instance);
-        assert ((ReversedType<?>)type).baseType == LongType.instance;
+        assert type.unwrap() == LongType.instance;
     }
 
     @Test
@@ -106,7 +115,7 @@ public void testParsePartitionerOrder() throws ConfigurationException, SyntaxExc
         });
         assertEquals(DatabaseDescriptor.getPartitioner().partitionOrdering(null), TypeParser.parse("PartitionerDefinedOrder"));
     }
-    
+
     @Test
     public void testParsePartitionerOrderWithBaseType()
     {
@@ -219,4 +228,52 @@ public static void assertForEachPartitioner(Consumer<IPartitioner> consumer)
             consumer.accept(partitioner);
         }
     }
+
+    @Test
+    public void testTuple()
+    {
+        List<TupleType> tupleTypes = Arrays.asList(
+//                new TupleType(Arrays.asList(UTF8Type.instance, Int32Type.instance), false),
+//                new TupleType(Arrays.asList(UTF8Type.instance, Int32Type.instance), true),
+//                new TupleType(Arrays.asList(UTF8Type.instance, new TupleType(Arrays.asList(Int32Type.instance, LongType.instance), false)), false),
+//                new TupleType(Arrays.asList(UTF8Type.instance, new TupleType(Arrays.asList(Int32Type.instance, LongType.instance), false)), true),
+                new TupleType(Arrays.asList(UTF8Type.instance, new TupleType(Arrays.asList(Int32Type.instance, LongType.instance), true)), false),
+//                new TupleType(Arrays.asList(UTF8Type.instance, new TupleType(Arrays.asList(Int32Type.instance, LongType.instance), true)), true),
+//                new TupleType(Arrays.asList(UTF8Type.instance, makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), false)), false),
+//                new TupleType(Arrays.asList(UTF8Type.instance, makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), false)), true),
+//                new TupleType(Arrays.asList(UTF8Type.instance, makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), true)), false),
+                new TupleType(Arrays.asList(UTF8Type.instance, makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), true)), true)
+        );
+
+        for (TupleType tupleType : tupleTypes)
+        {
+            assertThat(TypeParser.parse(tupleType.toString())).describedAs(tupleType.toString()).isEqualTo(tupleType);
+            assertThat(TypeParser.parse(tupleType.freeze().toString())).describedAs(tupleType.toString()).isEqualTo(tupleType.freeze());
+            assertThat(TypeParser.parse(tupleType.expandUserTypes().toString())).describedAs(tupleType.toString()).isEqualTo(tupleType.expandUserTypes());
+        }
+    }
+
+    @Test
+    public void testParseUDT()
+    {
+        List<UserType> userTypes = Arrays.asList(
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), false),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), true),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", new TupleType(Arrays.asList(Int32Type.instance, LongType.instance), false)), false),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", new TupleType(Arrays.asList(Int32Type.instance, LongType.instance), false)), true),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", new TupleType(Arrays.asList(Int32Type.instance, LongType.instance), true)), false),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", new TupleType(Arrays.asList(Int32Type.instance, LongType.instance), true)), true),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", makeUDT("udt2", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), false)), false),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", makeUDT("udt2", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), false)), true),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", makeUDT("udt2", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), true)), false),
+                makeUDT("udt", ImmutableMap.of("a", UTF8Type.instance, "b", makeUDT("udt2", ImmutableMap.of("a", UTF8Type.instance, "b", LongType.instance), true)), true)
+        );
+
+        for (UserType userType : userTypes)
+        {
+            assertEquals(userType, TypeParser.parse(userType.toString()));
+            assertEquals(userType.freeze(), TypeParser.parse(userType.freeze().toString()));
+            assertEquals(userType.expandUserTypes(), TypeParser.parse(userType.expandUserTypes().toString()));
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java b/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java
index 474b867007f1..3916eb6e350b 100644
--- a/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.db.marshal;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.utils.AbstractTypeGenerators;
@@ -46,6 +47,11 @@
 
 public class TypeValidationTest
 {
+    static
+    {
+        CassandraRelevantProperties.VECTOR_FLOAT_ONLY.setBoolean(false);
+    }
+    
     @Test(expected = MarshalException.class)
     public void testInvalidAscii()
     {
diff --git a/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeCodec.java b/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeCodec.java
new file mode 100644
index 000000000000..146c6e47f926
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeCodec.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.datetime;
+
+import java.nio.ByteBuffer;
+import java.text.ParseException;
+
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.ProtocolVersion;
+import com.datastax.driver.core.TypeCodec;
+import com.datastax.driver.core.exceptions.InvalidTypeException;
+import org.apache.cassandra.db.marshal.DateRangeType;
+import org.apache.cassandra.serializers.DateRangeSerializer;
+
+/**
+ * {@link TypeCodec} that maps binary representation of {@link DateRangeType} to {@link DateRange}.
+ */
+public class DateRangeCodec extends TypeCodec<DateRange>
+{
+    public static final DateRangeCodec instance = new DateRangeCodec();
+
+    private DateRangeCodec()
+    {
+        super(DataType.custom(DateRangeType.class.getName()), DateRange.class);
+    }
+
+    @Override
+    public ByteBuffer serialize(DateRange dateRange, ProtocolVersion protocolVersion) throws InvalidTypeException
+    {
+        if (dateRange == null)
+        {
+            return null;
+        }
+        return DateRangeSerializer.instance.serialize(dateRange);
+    }
+
+    @Override
+    public DateRange deserialize(ByteBuffer bytes, ProtocolVersion protocolVersion) throws InvalidTypeException
+    {
+        if (bytes == null || bytes.remaining() == 0)
+        {
+            return null;
+        }
+        return DateRangeSerializer.instance.deserialize(bytes);
+    }
+
+    @Override
+    public DateRange parse(String value) throws InvalidTypeException
+    {
+        if (value == null || value.isEmpty() || value.equalsIgnoreCase("NULL"))
+        {
+            return null;
+        }
+        try
+        {
+            return DateRangeUtil.parseDateRange(value);
+        }
+        catch (ParseException e)
+        {
+            throw new IllegalArgumentException(String.format("Invalid date range literal: '%s'", value), e);
+        }
+    }
+
+    @Override
+    public String format(DateRange dateRange) throws InvalidTypeException
+    {
+        return dateRange == null ? "NULL" : dateRange.formatToSolrString();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeCodecTest.java b/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeCodecTest.java
new file mode 100644
index 000000000000..7b4b80c83f79
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeCodecTest.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.marshal.datetime;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import com.datastax.driver.core.ProtocolVersion;
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBound.Precision;
+
+import static org.junit.Assert.assertEquals;
+
+public class DateRangeCodecTest
+{
+    private final DateRangeCodec codec = DateRangeCodec.instance;
+
+    @Test
+    public void testSerializeRoundTrip()
+    {
+        DateRange expected = DateRange.DateRangeBuilder.dateRange()
+                .withLowerBound("2015-12-03T10:15:30.00Z", Precision.SECOND)
+                .withUpperBound("2016-01-01T00:00:01.00Z", Precision.MILLISECOND)
+                .build();
+
+        ByteBuffer serialized = codec.serialize(expected, ProtocolVersion.NEWEST_SUPPORTED);
+
+        // For UDT or tuple type buffer contains whole cell payload, and codec can't rely on absolute byte addressing
+        ByteBuffer payload = ByteBuffer.allocate(5 + serialized.capacity());
+        // put serialized date range in between other data
+        payload.putInt(44).put(serialized).put((byte) 1);
+        payload.position(4);
+
+        DateRange actual = codec.deserialize(payload, ProtocolVersion.NEWEST_SUPPORTED);
+
+        assertEquals(expected, actual);
+        //provided ByteBuffer should never be consumed by read operations that modify its current position
+        assertEquals(4, payload.position());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeTest.java b/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeTest.java
new file mode 100644
index 000000000000..3f839ce1d4d2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeTest.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal.datetime;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBound;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class DateRangeTest
+{
+    @Test
+    public void testDateRangeEquality()
+    {
+        DateRange first = DateRange.DateRangeBuilder.dateRange()
+                .withLowerBound("2015-12-03T10:15:30.00Z", DateRangeBound.Precision.SECOND)
+                .withUpperBound("2016-01-01T00:00:01.00Z", DateRangeBound.Precision.MILLISECOND)
+                .build();
+        DateRange second = DateRange.DateRangeBuilder.dateRange()
+                // millis are off, but precision is higher so we skip them
+                .withLowerBound("2015-12-03T10:15:30.01Z", DateRangeBound.Precision.SECOND)
+                .withUpperBound("2016-01-01T00:00:01.00Z", DateRangeBound.Precision.MILLISECOND)
+                .build();
+        DateRange third = DateRange.DateRangeBuilder.dateRange()
+                .withLowerBound("2015-12-03T10:15:30.00Z", DateRangeBound.Precision.MILLISECOND)
+                .withUpperBound("2016-01-01T00:00:01.00Z", DateRangeBound.Precision.MILLISECOND)
+                .build();
+
+        assertEquals(first, second);
+        assertEquals(first, first);
+        assertEquals(first.hashCode(), second.hashCode());
+        assertEquals(first.hashCode(), first.hashCode());
+        assertEquals(first.formatToSolrString(), second.formatToSolrString());
+        assertNotEquals(first, third);
+        assertNotEquals(first.hashCode(), third.hashCode());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeUtilTest.java b/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeUtilTest.java
new file mode 100644
index 000000000000..391f7d77d41f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/marshal/datetime/DateRangeUtilTest.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.db.marshal.datetime;
+
+import java.text.ParseException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBound.Precision;
+import org.apache.cassandra.serializers.DateRangeSerializer;
+
+import static org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBuilder.dateRange;
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Enclosed.class)
+public class DateRangeUtilTest
+{
+    @RunWith(Parameterized.class)
+    public static class ParameterizedCases
+    {
+        @Parameterized.Parameter(0)
+        public String source;
+
+        @Parameterized.Parameter(1)
+        public DateRange expected;
+
+        @Test
+        public void shouldParseAndFormatSolrDateRangeFormat() throws ParseException
+        {
+            DateRange parsedSource = DateRangeUtil.parseDateRange(source);
+
+            assertEquals(expected, parsedSource);
+            assertEquals(source, parsedSource.formatToSolrString());
+        }
+
+        @Test
+        public void shouldSerializeAndDeserializeDateRange()
+        {
+            DateRange parsed = DateRangeSerializer.instance.deserialize(DateRangeSerializer.instance.serialize(expected));
+            assertEquals(expected, parsed);
+        }
+        
+        @SuppressWarnings("unused")
+        @Parameterized.Parameters(name = "source = {0}, expected = {1}")
+        public static Collection<Object[]> testData()
+        {
+            return Arrays.asList(
+                    new Object[]{
+                            "[2011-01 TO 2015]",
+                            dateRange()
+                                    .withLowerBound("2011-01-01T00:00:00.000Z", Precision.MONTH)
+                                    .withUpperBound("2015-12-31T23:59:59.999Z", Precision.YEAR)
+                                    .build()
+                    },
+                    new Object[]{
+                            "[2010-01-02 TO 2015-05-05T13]",
+                            dateRange()
+                                    .withLowerBound("2010-01-02T00:00:00.000Z", Precision.DAY)
+                                    .withUpperBound("2015-05-05T13:59:59.999Z", Precision.HOUR)
+                                    .build()
+                    },
+                    new Object[]{
+                            "[1973-06-30T13:57:28.123Z TO 1999-05-05T14:14:59]",
+                            dateRange()
+                                    .withLowerBound("1973-06-30T13:57:28.123Z", Precision.MILLISECOND)
+                                    .withUpperBound("1999-05-05T14:14:59.999Z", Precision.SECOND)
+                                    .build()
+                    },
+                    // leap year
+                    new Object[]{
+                            "[2010-01-01T15 TO 2016-02]",
+                            dateRange()
+                                    .withLowerBound("2010-01-01T15:00:00.000Z", Precision.HOUR)
+                                    .withUpperBound("2016-02-29T23:59:59.999Z", Precision.MONTH)
+                                    .build()
+                    },
+                    // pre-epoch
+                    new Object[]{
+                            "[1500 TO 1501]",
+                            dateRange()
+                                    .withLowerBound("1500-01-01T00:00:00.000Z", Precision.YEAR)
+                                    .withUpperBound("1501-12-31T23:59:59.999Z", Precision.YEAR)
+                                    .build()
+                    },
+                    // AD/BC era boundary
+                    new Object[]{
+                            "[0001-01-01 TO 0001-01-01]",
+                            dateRange()
+                                    .withLowerBound("0001-01-01T00:00:00.000Z", Precision.DAY)
+                                    .withUpperBound("0001-01-01T00:00:00.000Z", Precision.DAY)
+                                    .build()
+                    },
+                    new Object[]{
+                            "[0001-01-01 TO 0001-01-02]",
+                            dateRange()
+                                    .withLowerBound("0001-01-01T00:00:00.000Z", Precision.DAY)
+                                    .withUpperBound("0001-01-02T23:59:59.999Z", Precision.DAY)
+                                    .build()
+                    },
+                    new Object[]{
+                            "[0000-01-01 TO 0000-01-01]",
+                            dateRange()
+                                    .withLowerBound("0000-01-01T00:00:00.000Z", Precision.DAY)
+                                    .withUpperBound("0000-01-01T00:00:00.000Z", Precision.DAY)
+                                    .build()
+                    },
+                    new Object[]{
+                            "[0000-01-01 TO 0000-01-02]",
+                            dateRange()
+                                    .withLowerBound("0000-01-01T00:00:00.000Z", Precision.DAY)
+                                    .withUpperBound("0000-01-02T23:59:59.999Z", Precision.DAY)
+                                    .build()
+                    },
+                    new Object[]{
+                            "[-0001-01-01 TO -0001-01-01]",
+                            dateRange()
+                                    .withLowerBound("-0001-01-01T00:00:00.000Z", Precision.DAY)
+                                    .withUpperBound("-0001-01-01T00:00:00.000Z", Precision.DAY)
+                                    .build()
+                    },
+                    new Object[]{
+                            "[-0001-01-01 TO -0001-01-02]",
+                            dateRange()
+                                    .withLowerBound("-0001-01-01T00:00:00.000Z", Precision.DAY)
+                                    .withUpperBound("-0001-01-02T23:59:59.999Z", Precision.DAY)
+                                    .build()
+                    },
+                    // unbounded
+                    new Object[]{
+                            "[* TO 2014-12-01]",
+                            dateRange()
+                                    .withUnboundedLowerBound()
+                                    .withUpperBound("2014-12-01T23:59:59.999Z", Precision.DAY)
+                                    .build()
+                    },
+                    new Object[]{
+                            "[1999 TO *]",
+                            dateRange()
+                                    .withLowerBound("1999-01-01T00:00:00Z", Precision.YEAR)
+                                    .withUnboundedUpperBound()
+                                    .build()
+                    },
+                    new Object[]{
+                            "[* TO *]",
+                            dateRange()
+                                    .withUnboundedLowerBound()
+                                    .withUnboundedUpperBound()
+                                    .build()
+                    },
+                    new Object[]{
+                            "*",
+                            dateRange()
+                                    .withUnboundedLowerBound()
+                                    .build()
+                    },
+                    // unit shapes
+                    new Object[]{
+                            "-0009",
+                            dateRange()
+                                    .withLowerBound("-0009-01-01T00:00:00.000Z", Precision.YEAR)
+                                    .build()
+                    },
+                    new Object[]{
+                            "2000-11",
+                            dateRange()
+                                    .withLowerBound("2000-11-01T00:00:00.000Z", Precision.MONTH)
+                                    .build()
+                    }
+            );
+        }
+    }
+
+    public static class RoundingCases
+    {
+        @Rule
+        public ExpectedException expectedException = ExpectedException.none();
+
+        @Test
+        public void shouldNotParseDateRangeWithWrongDateOrder() throws ParseException
+        {
+            expectedException.expect(IllegalArgumentException.class);
+            expectedException.expectMessage("Wrong order: 2010 TO 2009");
+            DateRangeUtil.parseDateRange("[2010 TO 2009]");
+        }
+
+        @Test
+        public void shouldRoundUpperBoundToTheGivenPrecision()
+        {
+            ZonedDateTime timestamp = ZonedDateTime.ofInstant(Instant.parse("2011-02-03T04:05:16.789Z"), ZoneOffset.UTC);
+            assertEquals("2011-02-03T04:05:16.789Z", DateRangeUtil.roundUpperBoundTimestampToPrecision(timestamp, Precision.MILLISECOND).toInstant().toString());
+            assertEquals("2011-02-03T04:05:16.999Z", DateRangeUtil.roundUpperBoundTimestampToPrecision(timestamp, Precision.SECOND).toInstant().toString());
+            assertEquals("2011-02-03T04:05:59.999Z", DateRangeUtil.roundUpperBoundTimestampToPrecision(timestamp, Precision.MINUTE).toInstant().toString());
+            assertEquals("2011-02-03T04:59:59.999Z", DateRangeUtil.roundUpperBoundTimestampToPrecision(timestamp, Precision.HOUR).toInstant().toString());
+            assertEquals("2011-02-03T23:59:59.999Z", DateRangeUtil.roundUpperBoundTimestampToPrecision(timestamp, Precision.DAY).toInstant().toString());
+            assertEquals("2011-02-28T23:59:59.999Z", DateRangeUtil.roundUpperBoundTimestampToPrecision(timestamp, Precision.MONTH).toInstant().toString());
+            assertEquals("2011-12-31T23:59:59.999Z", DateRangeUtil.roundUpperBoundTimestampToPrecision(timestamp, Precision.YEAR).toInstant().toString());
+        }
+
+        @Test
+        public void shouldRoundLowerBoundToTheGivenPrecision()
+        {
+            ZonedDateTime timestamp = ZonedDateTime.ofInstant(Instant.parse("2011-02-03T04:05:16.789Z"), ZoneOffset.UTC);
+            assertEquals("2011-02-03T04:05:16.789Z", DateRangeUtil.roundLowerBoundTimestampToPrecision(timestamp, Precision.MILLISECOND).toInstant().toString());
+            assertEquals("2011-02-03T04:05:16Z", DateRangeUtil.roundLowerBoundTimestampToPrecision(timestamp, Precision.SECOND).toInstant().toString());
+            assertEquals("2011-02-03T04:05:00Z", DateRangeUtil.roundLowerBoundTimestampToPrecision(timestamp, Precision.MINUTE).toInstant().toString());
+            assertEquals("2011-02-03T04:00:00Z", DateRangeUtil.roundLowerBoundTimestampToPrecision(timestamp, Precision.HOUR).toInstant().toString());
+            assertEquals("2011-02-03T00:00:00Z", DateRangeUtil.roundLowerBoundTimestampToPrecision(timestamp, Precision.DAY).toInstant().toString());
+            assertEquals("2011-02-01T00:00:00Z", DateRangeUtil.roundLowerBoundTimestampToPrecision(timestamp, Precision.MONTH).toInstant().toString());
+            assertEquals("2011-01-01T00:00:00Z", DateRangeUtil.roundLowerBoundTimestampToPrecision(timestamp, Precision.YEAR).toInstant().toString());
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/memtable/FlushingTest.java b/test/unit/org/apache/cassandra/db/memtable/FlushingTest.java
new file mode 100644
index 000000000000..e70be3bb4e1d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/FlushingTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitConfig;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+@RunWith(BMUnitRunner.class)
+@BMUnitConfig(debug = true)
+public class FlushingTest extends CQLTester
+{
+    List<Token> ranges;
+    List<Directories.DataDirectory> locations;
+    ColumnFamilyStore cfs;
+    Memtable memtable;
+    ExecutorService executor;
+    int nThreads;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        createTable(KEYSPACE_PER_TEST, "CREATE TABLE %s (pk int PRIMARY KEY, value int)");
+
+        for (int i = 0; i < 10000; i++)
+            execute(String.format("INSERT INTO %s.%s (pk, value) VALUES (?, ?)", KEYSPACE_PER_TEST, currentTable()), i, i);
+
+        cfs = getCurrentColumnFamilyStore(KEYSPACE_PER_TEST);
+        memtable = cfs.getTracker().getView().getCurrentMemtable();
+
+        OpOrder.Barrier barrier = cfs.keyspace.writeOrder.newBarrier();
+        Memtable.LastCommitLogPosition position = new Memtable.LastCommitLogPosition(CommitLog.instance.getCurrentPosition());
+        memtable.switchOut(barrier, new AtomicReference<>(position));
+        barrier.issue();
+
+        ranges = new ArrayList<>();
+        locations = new ArrayList<>();
+        // this determines the number of flush writers created, the FlushRunnable will convert a null location into an sstable location for us
+        int rangeCount = 24;
+        for (int i = 0; i < rangeCount; ++i)
+        {
+            // split the range to ensure there are partitions to write
+            ranges.add(cfs.getPartitioner().split(cfs.getPartitioner().getMinimumToken(),
+                                                  cfs.getPartitioner().getMaximumToken(),
+                                                  (i+1) * 1.0 / rangeCount));
+            locations.add(null);
+        }
+        nThreads = locations.size() / 2;
+        executor = Executors.newFixedThreadPool(nThreads);
+    }
+
+    @Test
+    public void testAbortingFlushRunnablesWithoutStarting() throws Throwable
+    {
+        // abort without starting
+        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH, cfs.metadata))
+        {
+            List<Flushing.FlushRunnable> flushRunnables = Flushing.flushRunnables(cfs, memtable, ranges, locations, txn);
+            assertNotNull(flushRunnables);
+
+            for (Flushing.FlushRunnable flushRunnable : flushRunnables)
+                assertEquals(Flushing.FlushRunnableWriterState.IDLE, flushRunnable.state());
+
+            for (Flushing.FlushRunnable flushRunnable : flushRunnables)
+                assertNull(flushRunnable.abort(null));
+
+            for (Flushing.FlushRunnable flushRunnable : flushRunnables)
+                assertEquals(Flushing.FlushRunnableWriterState.ABORTED, flushRunnable.state());
+        }
+    }
+
+    static Semaphore stopSignal = null;
+    static Semaphore continueSignal;
+
+    public static void stopAndWait() throws InterruptedException
+    {
+        if (stopSignal != null)
+        {
+            stopSignal.release();
+            continueSignal.acquire();
+        }
+    }
+
+    @Test
+    @BMRule(name = "Wait before loop",
+    targetClass = "Flushing$FlushRunnable",
+    targetMethod = "writeSortedContents",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.db.memtable.FlushingTest.stopAndWait()")
+    public void testAbortingFlushRunnablesAfterStarting() throws Throwable
+    {
+        // abort after starting
+        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH, cfs.metadata))
+        {
+            List<Flushing.FlushRunnable> flushRunnables = Flushing.flushRunnables(cfs, memtable, ranges, locations, txn);
+
+            stopSignal = new Semaphore(0);
+            continueSignal = new Semaphore(0);
+
+            List<Future<SSTableMultiWriter>> futures = flushRunnables.stream().map(executor::submit).collect(Collectors.toList());
+
+            stopSignal.acquire(nThreads);
+            for (Flushing.FlushRunnable flushRunnable : flushRunnables)
+                assertNull(flushRunnable.abort(null));
+            continueSignal.release(flushRunnables.size());  // release all, including the ones that have not started yet
+
+            FBUtilities.waitOnFutures(futures);
+
+            for (Flushing.FlushRunnable flushRunnable : flushRunnables)
+                assertEquals(Flushing.FlushRunnableWriterState.ABORTED, flushRunnable.state());
+        }
+    }
+
+    @Test
+    public void testAbortingFlushRunnablesBeforeStarting() throws Throwable
+    {
+        // abort before starting
+        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH, cfs.metadata))
+        {
+            List<Flushing.FlushRunnable> flushRunnables = Flushing.flushRunnables(cfs, memtable, ranges, locations, txn);
+
+            for (Flushing.FlushRunnable flushRunnable : flushRunnables)
+                assertNull(flushRunnable.abort(null));
+
+            List<Future<SSTableMultiWriter>> futures = flushRunnables.stream().map(executor::submit).collect(Collectors.toList());
+
+            FBUtilities.waitOnFutures(futures);
+
+            for (Flushing.FlushRunnable flushRunnable : flushRunnables)
+                assertEquals(Flushing.FlushRunnableWriterState.ABORTED, flushRunnable.state());
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableQuickTest.java b/test/unit/org/apache/cassandra/db/memtable/MemtableQuickTest.java
index c442a4c036e8..279ec0992096 100644
--- a/test/unit/org/apache/cassandra/db/memtable/MemtableQuickTest.java
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableQuickTest.java
@@ -63,7 +63,9 @@ public static List<Object> parameters()
         return ImmutableList.of("skiplist",
                                 "skiplist_sharded",
                                 "skiplist_sharded_locking",
-                                "trie");
+                                "trie",
+                                "trie_stage1",
+                                "persistent_memory");
     }
 
     @BeforeClass
@@ -139,6 +141,13 @@ public void testMemtable() throws Throwable
         UntypedResultSet result = execute("SELECT * FROM " + table);
         assertRowCount(result, rowsPerPartition * (partitions - deletedPartitions) - deletedRows);
 
+        Memtable memtable = cfs.getCurrentMemtable();
+        Memtable.FlushablePartitionSet<?> flushSet = memtable.getFlushSet(null, null);
+        Assert.assertEquals(partitions, flushSet.partitionCount());
+        double expectedKeySize = partitions * 8;
+        // expected key size must be within 5% of actual
+        Assert.assertEquals(expectedKeySize, flushSet.partitionKeysSize(), expectedKeySize * 0.05);
+
         Util.flush(cfs);
 
         logger.info("Selecting *");
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTestBase.java b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTestBase.java
index efc992ee7f32..0b9d8922092d 100644
--- a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTestBase.java
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTestBase.java
@@ -1,3 +1,4 @@
+/*
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -58,6 +59,10 @@ public abstract class MemtableSizeTestBase extends CQLTester
 
     static final Logger logger = LoggerFactory.getLogger(MemtableSizeTestBase.class);
 
+    static String keyspace;
+    String table;
+    ColumnFamilyStore cfs;
+
     static final int partitions = 50_000;
     static final int rowsPerPartition = 4;
 
@@ -74,6 +79,7 @@ public static List<Object> parameters()
     {
         return ImmutableList.of("skiplist",
                                 "skiplist_sharded",
+                                "trie_stage1",
                                 "trie");
     }
 
@@ -82,6 +88,8 @@ public static List<Object> parameters()
     final int MAX_DIFFERENCE_PERCENT = 3;
     // Slab overhead, added when the memtable uses heap_buffers.
     final int SLAB_OVERHEAD = 1024 * 1024;
+    // Extra leniency for unslabbed buffers. We are not as precise there, and it's not a mode in real use.
+    final int UNSLABBED_EXTRA_PERCENT = 2;
 
     public static void setup(Config.MemtableAllocationType allocationType, IPartitioner partitioner)
     {
@@ -109,32 +117,42 @@ void checkMemtablePool()
         // overridden by instances
     }
 
-    @Test
-    public void testSize() throws Throwable
+    private void buildAndFillTable(String memtableClass) throws Throwable
     {
         // Make sure memtables use the correct allocation type, i.e. that setup has worked.
         // If this fails, make sure the test is not reusing an already-initialized JVM.
         checkMemtablePool();
 
         CQLTester.disablePreparedReuseForTest();
-        String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+        keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+
+        table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))" +
+                                      " with compression = {'enabled': false}" +
+                                      " and memtable = '" + memtableClass + "'");
+        execute("use " + keyspace + ';');
+
+        forcePreparedValues();
+
+        cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        cfs.disableAutoCompaction();
+        Util.flush(cfs);
+    }
+
+    @Test
+    public void testSize() throws Throwable
+    {
+
         try
         {
-            String table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))" +
-                                                 " with compression = {'enabled': false}" +
-                                                 " and memtable = '" + memtableClass + "'");
-            execute("use " + keyspace + ';');
+            buildAndFillTable(memtableClass);
 
             String writeStatement = "INSERT INTO " + table + "(userid,picid,commentid)VALUES(?,?,?)";
-            forcePreparedValues();
-
-            ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-            cfs.disableAutoCompaction();
-            Util.flush(cfs);
 
             Memtable memtable = cfs.getTracker().getView().getCurrentMemtable();
             long deepSizeBefore = meter.measureDeep(memtable);
-            logger.info("Memtable deep size before {}", FBUtilities.prettyPrintMemory(deepSizeBefore));
+            System.out.println("Memtable deep size before " +
+                               FBUtilities.prettyPrintMemory(deepSizeBefore));
+
             long i;
             long limit = partitions;
             logger.info("Writing {} partitions of {} rows", partitions, rowsPerPartition);
@@ -165,43 +183,65 @@ public void testSize() throws Throwable
                               cfs.getTracker().getView().getCurrentMemtable());
 
             Memtable.MemoryUsage usage = Memtable.getMemoryUsage(memtable);
-            long actualHeap = usage.ownsOnHeap;
-            logger.info(String.format("Memtable in %s mode: %d ops, %s serialized bytes, %s",
-                                      DatabaseDescriptor.getMemtableAllocationType(),
-                                      memtable.operationCount(),
-                                      FBUtilities.prettyPrintMemory(memtable.getLiveDataSize()),
-                                      usage));
+            long reportedHeap = usage.ownsOnHeap;
+            System.out.println(String.format("Memtable in %s mode: %d ops, %s serialized bytes, %s",
+                                             DatabaseDescriptor.getMemtableAllocationType(),
+                                             memtable.operationCount(),
+                                             FBUtilities.prettyPrintMemory(memtable.getLiveDataSize()),
+                                             usage));
+
+            if (memtable instanceof TrieMemtable)
+                ((TrieMemtable) memtable).releaseReferencesUnsafe();
+
+//            System.out.println("Take jmap -histo:live <pid>");
+//            Thread.sleep(10000);
 
             long deepSizeAfter = meter.measureDeep(memtable);
             logger.info("Memtable deep size {}", FBUtilities.prettyPrintMemory(deepSizeAfter));
 
-            long expectedHeap = deepSizeAfter - deepSizeBefore;
-            long max_difference = MAX_DIFFERENCE_PERCENT * expectedHeap / 100;
-            long trie_overhead = memtable instanceof TrieMemtable ? ((TrieMemtable) memtable).unusedReservedMemory() : 0;
+            long actualHeap = deepSizeAfter - deepSizeBefore;
+            long maxDifference = MAX_DIFFERENCE_PERCENT * actualHeap / 100;
+            long unusedReserved = memtable.unusedReservedOnHeapMemory();
+            System.out.println("Unused reserved " + FBUtilities.prettyPrintMemory(unusedReserved));
+            reportedHeap += unusedReserved;
+
             switch (DatabaseDescriptor.getMemtableAllocationType())
             {
-                case heap_buffers:
-                    max_difference += SLAB_OVERHEAD;
-                    actualHeap += trie_overhead;    // adjust trie memory with unused buffer space if on-heap
-                    break;
                 case unslabbed_heap_buffers:
-                    actualHeap += trie_overhead;    // adjust trie memory with unused buffer space if on-heap
+                    // add a hardcoded slack factor
+                    maxDifference += actualHeap * UNSLABBED_EXTRA_PERCENT / 100;
                     break;
             }
-            double deltaPerPartition = (expectedHeap - actualHeap) / (double) totalPartitions;
-            String message = String.format("Expected heap usage close to %s, got %s, %s difference. " +
-                                           "Delta per partition: %.2f bytes",
-                                           FBUtilities.prettyPrintMemory(expectedHeap),
+            String message = String.format("Actual heap usage is %s, got %s, %s difference.\n",
                                            FBUtilities.prettyPrintMemory(actualHeap),
-                                           FBUtilities.prettyPrintMemory(expectedHeap - actualHeap),
-                                           deltaPerPartition);
-            logger.info(message);
-
-            Assert.assertTrue(message, Math.abs(actualHeap - expectedHeap) <= max_difference);
+                                           FBUtilities.prettyPrintMemory(reportedHeap),
+                                           FBUtilities.prettyPrintMemory(actualHeap - reportedHeap));
+            System.out.println(message);
+            Assert.assertTrue(message, Math.abs(reportedHeap - actualHeap) <= maxDifference);
         }
         finally
         {
             execute(String.format("DROP KEYSPACE IF EXISTS %s", keyspace));
         }
     }
+
+    @Test
+    public void testRowSize() throws Throwable
+    {
+        buildAndFillTable(memtableClass);
+
+        String writeStatement = "INSERT INTO " + table + "(userid,picid,commentid)VALUES(?,?,?)";
+
+        Memtable memtable = cfs.getTracker().getView().getCurrentMemtable();
+        System.out.println("Writing " + partitions + " partitions of " + rowsPerPartition + " rows");
+        for (long i = 0; i < partitions; ++i)
+        {
+            for (long j = 0; j < rowsPerPartition; ++j)
+                execute(writeStatement, i, j, i + j);
+        }
+
+        long rowSize = memtable.getEstimatedAverageRowSize();
+        double expectedRowSize = (double) memtable.getLiveDataSize() / (partitions * rowsPerPartition);
+        Assert.assertEquals(expectedRowSize, rowSize, (partitions * rowsPerPartition) * 0.05);  // 5% accuracy
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableThreadedTest.java b/test/unit/org/apache/cassandra/db/memtable/MemtableThreadedTest.java
new file mode 100644
index 000000000000..27252e52c124
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableThreadedTest.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
+
+/// This test is a counterpart to InMemoryTrieThreadedTest that makes sure TrieMemtable is wiring the trie consistency
+/// machinery correctly. Note that this test always applies mutations the same way (with partition-level forced copying)
+/// and is effectively doing the same test but checking different correctness properties.
+///
+/// A problem with this will only appear as intermittent failures, never treat this test as flaky.
+@RunWith(Parameterized.class)
+public class MemtableThreadedTest extends CQLTester
+{
+    @Parameterized.Parameter()
+    public String memtableClass;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object> parameters()
+    {
+        return ImmutableList.of("SkipListMemtable",
+                                "TrieMemtable",
+                                "TrieMemtableStage1",
+                                "PersistentMemoryMemtable");
+    }
+
+    @BeforeClass
+    public static void setUp()
+    {
+        CQLTester.setUpClass();
+        CQLTester.prepareServer();
+        CQLTester.disablePreparedReuseForTest();
+        System.err.println("setupClass done.");
+    }
+
+    static String keyspace;
+    String table;
+    ColumnFamilyStore cfs;
+
+    private static final int COUNT = 45678;
+    private static final int PROGRESS_UPDATE = COUNT / 15;
+    private static final int READERS = 17;
+    private static final int WALKERS = 3;
+
+    @Test
+    public void testConsistentAndAtomicUpdates() throws Exception
+    {
+        // Note: Intermittent failures of this test other than timeout should be treated as a bug.
+
+        // Check that multi-row mutations are safe for concurrent readers,
+        // and that content is atomically applied, i.e. that readers see either nothing from the update or all of it,
+        // and consistent, i.e. that it is not possible to receive some newer updates while missing
+        // older ones. (For example, if the sequence of additions is 3, 1, 5, without this requirement a reader
+        // could see an enumeration which lists 3 and 5 but not 1.)
+        testAtomicUpdates(3, true, true);
+        // Note: using 3 per mutation, so that the first and second update fit in a sparse in-memory trie block.
+    }
+
+    @Test
+    public void testConsistentUpdates() throws Exception
+    {
+        // Note: Intermittent failures of this test other than timeout should be treated as a bug.
+
+        // Check that multi-row mutations are safe for concurrent readers,
+        // and that content is consistent, i.e. that it is not possible to receive some newer updates while missing
+        // older ones. (For example, if the sequence of additions is 3, 1, 5, without this requirement a reader
+        // could see an enumeration which lists 3 and 5 but not 1.)
+        testAtomicUpdates(3, false, true);
+        // Note: using 3 per mutation, so that the first and second update fit in a sparse in-memory trie block.
+    }
+
+    @Test
+    public void testAtomicUpdates() throws Exception
+    {
+        // Note: Intermittent failures of this test other than timeout should be treated as a bug.
+
+        // Check that multi-row mutations are safe for concurrent readers,
+        // and that content is atomically applied, i.e. that reader see either nothing from the update or all of it.
+        testAtomicUpdates(3, true, false);
+    }
+
+    @Test
+    public void testSafeUpdates() throws Exception
+    {
+        // Note: Intermittent failures of this test other than timeout should be treated as a bug.
+
+        // Check that multi row mutations are safe for concurrent readers.
+        testAtomicUpdates(3, false, false);
+    }
+
+    @Test
+    public void testConsistentAndAtomicSinglePathUpdates() throws Exception
+    {
+        // Note: Intermittent failures of this test other than timeout should be treated as a bug.
+
+        // Check that single row mutations are safe for concurrent readers,
+        // and that content is atomically applied, i.e. that readers see either nothing from the update or all of it,
+        // and consistent, i.e. that it is not possible to receive some newer updates while missing
+        // older ones. (For example, if the sequence of additions is 3, 1, 5, without this requirement a reader
+        // could see an enumeration which lists 3 and 5 but not 1.)
+        testAtomicUpdates(1, true, true);
+    }
+
+    @Test
+    public void testConsistentSinglePathUpdates() throws Exception
+    {
+        // Note: Intermittent failures of this test other than timeout should be treated as a bug.
+
+        // Check that single row mutations are safe for concurrent readers,
+        // and that content is consistent, i.e. that it is not possible to receive some newer updates while missing
+        // older ones. (For example, if the sequence of additions is 3, 1, 5, without this requirement a reader
+        // could see an enumeration which lists 3 and 5 but not 1.)
+        testAtomicUpdates(1, false, true);
+    }
+
+    @Test
+    public void testAtomicSinglePathUpdates() throws Exception
+    {
+        // Note: Intermittent failures of this test other than timeout should be treated as a bug.
+
+        // When doing single path updates atomicity comes for free. This only checks that the branching checker is
+        // not doing anything funny.
+        testAtomicUpdates(1, true, false);
+    }
+
+    @Test
+    public void testSafeSinglePathUpdates() throws Exception
+    {
+        // Check that single path updates without additional copying are safe for concurrent readers.
+        testAtomicUpdates(1, true, false);
+    }
+
+    public void testAtomicUpdates(int PER_MUTATION,
+                                  boolean checkAtomicity,
+                                  boolean checkSequence)
+    throws Exception
+    {
+        keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+        table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, ck bigint, value bigint, seq bigint, PRIMARY KEY(pk, ck))" +
+                                      " with compression = {'enabled': false}" +
+                                      " and memtable = { 'class': '" + memtableClass + "'}" +
+                                      " and compaction = { 'class': 'UnifiedCompactionStrategy', 'min_sstable_size_in_mb': '1' }"); // to trigger splitting of sstables, STAR-1826
+        execute("use " + keyspace + ';');
+
+        cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        cfs.disableAutoCompaction();
+        cfs.forceBlockingFlush(UNIT_TESTS);
+
+        int ckCount = COUNT;
+        int pkCount = Math.min(100, COUNT / 10);  // to guarantee repetition
+
+        String ws;
+        if (PER_MUTATION == 1)
+            ws = "INSERT INTO " + table + "(pk,ck,value,seq) VALUES (?,?,?,?)";
+        else
+        {
+            ws = "BEGIN UNLOGGED BATCH\n";
+            for (int i = 0; i < PER_MUTATION; ++i)
+                ws += "INSERT INTO " + table + "(pk,ck,value,seq) VALUES (?,?,?,?)\n";
+            ws += "APPLY BATCH";
+        }
+        String writeStatement = ws;
+
+        /*
+         * Adds COUNT partitions each with perPartition separate clusterings, where the sum of the values
+         * of all clusterings is 0.
+         * If the sum for any walk covering whole partitions is non-zero, we have had non-atomic updates.
+         */
+
+        ConcurrentLinkedQueue<Throwable> errors = new ConcurrentLinkedQueue<>();
+        List<Thread> threads = new ArrayList<Thread>();
+        AtomicBoolean writeCompleted = new AtomicBoolean(false);
+        AtomicInteger writeProgress = new AtomicInteger(0);
+
+        for (int i = 0; i < WALKERS; ++i)
+            threads.add(new Thread()
+            {
+                public void run()
+                {
+                    try
+                    {
+                        while (!writeCompleted.get())
+                        {
+                            int min = writeProgress.get();
+                            var results = execute("SELECT * FROM " + table);
+                            checkEntries("", min, checkAtomicity, checkSequence, PER_MUTATION, results);
+                        }
+                    }
+                    catch (Throwable t)
+                    {
+                        t.printStackTrace();
+                        errors.add(t);
+                    }
+                }
+            });
+
+        for (int i = 0; i < READERS; ++i)
+        {
+            threads.add(new Thread()
+            {
+                public void run()
+                {
+                    try
+                    {
+                        // await at least one ready partition
+                        while (writeProgress.get() == 0) {}
+
+                        Random r = ThreadLocalRandom.current();
+                        while (!writeCompleted.get())
+                        {
+                            long pk = r.nextInt(pkCount);
+                            int min = writeProgress.get() / (pkCount * PER_MUTATION) * PER_MUTATION;
+                            var results = execute("SELECT * FROM " + table + " WHERE pk = ?", pk);
+                            checkEntries(" in partition " + pk, min, checkAtomicity, checkSequence, PER_MUTATION, results);
+                        }
+                    }
+                    catch (Throwable t)
+                    {
+                        t.printStackTrace();
+                        errors.add(t);
+                    }
+                }
+            });
+        }
+
+        threads.add(new Thread()
+        {
+            public void run()
+            {
+                try
+                {
+                    int lastUpdate = 0;
+                    Object[] values = new Object[PER_MUTATION * 4];
+
+                    for (int i = 0; i < COUNT; i += PER_MUTATION)
+                    {
+                        long pk = (i / PER_MUTATION) % pkCount;
+                        int vidx = 0;
+                        for (int j = 0; j < PER_MUTATION; ++j)
+                        {
+
+                            long ck = i + j;
+                            long value = j == 0 ? -PER_MUTATION + 1 : 1;
+                            long seq = (i / PER_MUTATION / pkCount) * PER_MUTATION + j;
+                            values[vidx++] = pk;
+                            values[vidx++] = ck;
+                            values[vidx++] = value;
+                            values[vidx++] = seq;
+                        }
+                        execute(writeStatement, values);
+
+                        if (i >= pkCount * PER_MUTATION && i - lastUpdate >= PROGRESS_UPDATE)
+                        {
+                            writeProgress.set(i);
+                            lastUpdate = i;
+                        }
+                    }
+                }
+                catch (Throwable t)
+                {
+                    t.printStackTrace();
+                    errors.add(t);
+                }
+                finally
+                {
+                    writeCompleted.set(true);
+                }
+            }
+        });
+
+        for (Thread t : threads)
+            t.start();
+
+        for (Thread t : threads)
+            t.join();
+
+        if (!errors.isEmpty())
+            Assert.fail("Got errors:\n" + errors);
+    }
+
+    public void checkEntries(String location,
+                             int min,
+                             boolean checkAtomicity,
+                             boolean checkConsecutiveIds,
+                             int PER_MUTATION,
+                             UntypedResultSet entries)
+    {
+        long sum = 0;
+        int count = 0;
+        long idSum = 0;
+        long idMax = 0;
+        int currentPk = -1;
+        for (var en : entries)
+        {
+            long pk = en.getLong("pk");
+            if (pk != currentPk)
+            {
+                currentPk = (int) pk;
+                idMax = idSum = sum = 0;
+            }
+            ++count;
+            sum += en.getLong("value");
+            long seq = en.getLong("seq");
+            idSum += seq;
+            if (seq > idMax)
+                idMax = seq;
+        }
+
+        Assert.assertTrue("Values" + location + " should be at least " + min + ", got " + count, min <= count);
+
+        if (checkAtomicity)
+        {
+            // If mutations apply atomically, the row count is always a multiple of the mutation size...
+            Assert.assertTrue("Values" + location + " should be a multiple of " + PER_MUTATION + ", got " + count, count % PER_MUTATION == 0);
+            // ... and the sum of the values is 0 (as the sum for each individual mutation is 0).
+            Assert.assertEquals("Value sum" + location, 0, sum);
+        }
+
+        if (checkConsecutiveIds)
+        {
+            // If mutations apply consistently for the partition, for any row we see we have to have seen all rows that
+            // were applied before that. In other words, the id sum should be the sum of the integers from 1 to the
+            // highest id seen in the partition.
+            Assert.assertEquals("Id sum" + location, idMax * (idMax + 1) / 2, idSum);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/memtable/ShardedMemtableConfigTest.java b/test/unit/org/apache/cassandra/db/memtable/ShardedMemtableConfigTest.java
index ef5079b35ea6..8dc17915f5cf 100644
--- a/test/unit/org/apache/cassandra/db/memtable/ShardedMemtableConfigTest.java
+++ b/test/unit/org/apache/cassandra/db/memtable/ShardedMemtableConfigTest.java
@@ -50,7 +50,7 @@ public static void setup() throws Exception
     public void testDefaultShardCountSetByJMX() throws MalformedObjectNameException, ReflectionException, AttributeNotFoundException, InstanceNotFoundException, MBeanException, IOException, InvalidAttributeValueException, InterruptedException
     {
         // check the default, but also make sure the class is initialized if the default memtable is not sharded
-        assertEquals(FBUtilities.getAvailableProcessors(), AbstractShardedMemtable.getDefaultShardCount());
+        assertEquals(4 * FBUtilities.getAvailableProcessors(), AbstractShardedMemtable.getDefaultShardCount());
         jmxConnection.setAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), new Attribute("DefaultShardCount", "7"));
         assertEquals(7, AbstractShardedMemtable.getDefaultShardCount());
         assertEquals("7", jmxConnection.getAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), "DefaultShardCount"));
@@ -61,8 +61,8 @@ public void testAutoShardCount() throws MalformedObjectNameException, Reflection
     {
         AbstractShardedMemtable.getDefaultShardCount();    // initialize class
         jmxConnection.setAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), new Attribute("DefaultShardCount", "auto"));
-        assertEquals(FBUtilities.getAvailableProcessors(), AbstractShardedMemtable.getDefaultShardCount());
-        assertEquals(Integer.toString(FBUtilities.getAvailableProcessors()),
+        assertEquals(4 * FBUtilities.getAvailableProcessors(), AbstractShardedMemtable.getDefaultShardCount());
+        assertEquals(Integer.toString(4 * FBUtilities.getAvailableProcessors()),
                      jmxConnection.getAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), "DefaultShardCount"));
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/memtable/TestMemtable.java b/test/unit/org/apache/cassandra/db/memtable/TestMemtable.java
index ec10c1a81572..c1fb288785a4 100644
--- a/test/unit/org/apache/cassandra/db/memtable/TestMemtable.java
+++ b/test/unit/org/apache/cassandra/db/memtable/TestMemtable.java
@@ -31,5 +31,5 @@ public static Memtable.Factory factory(Map<String, String> options)
             return FACTORY;
     }
 
-    public static Memtable.Factory FACTORY = SkipListMemtable::new;
+    public static Memtable.Factory FACTORY = (commitLogLowerBound, metadataRef, owner) -> new SkipListMemtable(commitLogLowerBound, metadataRef, owner);
 }
diff --git a/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java b/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java
index 2b4ab06309c9..e3d9c01b3b54 100644
--- a/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java
+++ b/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java
@@ -33,6 +33,8 @@
 
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
@@ -42,17 +44,41 @@
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.partitions.AbstractBTreePartition;
 import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
 import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.TrieBackedPartition;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.db.rows.Row.Deletion;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
+@RunWith(Parameterized.class)
 public class PartitionImplementationTest
 {
+    enum Implementation
+    {
+        BTREE(ImmutableBTreePartition::create),
+        TRIE(TrieBackedPartition::fromIterator);
+
+        final Function<UnfilteredRowIterator, Partition> creator;
+
+        Implementation(Function<UnfilteredRowIterator, Partition> creator)
+        {
+            this.creator = creator;
+        }
+    }
+
+    @Parameterized.Parameters(name="{0}")
+    public static Object[] generateData()
+    {
+        return Implementation.values();
+    }
+
+    @Parameterized.Parameter(0)
+    public static Implementation implementation = Implementation.BTREE;
+
     private static final String KEYSPACE = "PartitionImplementationTest";
     private static final String CF = "Standard";
 
@@ -259,10 +285,10 @@ private void testIter(Supplier<Collection<? extends Unfiltered>> contentSupplier
     {
         NavigableSet<Clusterable> sortedContent = new TreeSet<Clusterable>(metadata.comparator);
         sortedContent.addAll(contentSupplier.get());
-        AbstractBTreePartition partition;
+        Partition partition;
         try (UnfilteredRowIterator iter = new Util.UnfilteredSource(metadata, Util.dk("pk"), staticRow, sortedContent.stream().map(x -> (Unfiltered) x).iterator()))
         {
-            partition = ImmutableBTreePartition.create(iter);
+            partition = implementation.creator.apply(iter);
         }
 
         ColumnMetadata defCol = metadata.getColumn(new ColumnIdentifier("col", true));
@@ -292,9 +318,13 @@ private void testIter(Supplier<Collection<? extends Unfiltered>> contentSupplier
         assertEquals(sortedContent.stream().anyMatch(x -> x instanceof Row),
                      partition.hasRows());
 
+        // rowCount
+        assertEquals(sortedContent.stream().filter(x -> x instanceof Row).count(),
+                     partition.rowCount());
+
         // iterator
         assertIteratorsEqual(sortedContent.stream().filter(x -> x instanceof Row).iterator(),
-                             partition.iterator());
+                             partition.rowIterator());
 
         // unfiltered iterator
         assertIteratorsEqual(sortedContent.iterator(),
@@ -401,7 +431,7 @@ private NavigableSet<Clustering<?>> makeClusterings(boolean reversed)
         return clusterings;
     }
 
-    private void testSlicingOfIterators(NavigableSet<Clusterable> sortedContent, AbstractBTreePartition partition, ColumnFilter cf, boolean reversed)
+    private void testSlicingOfIterators(NavigableSet<Clusterable> sortedContent, Partition partition, ColumnFilter cf, boolean reversed)
     {
         Function<? super Clusterable, ? extends Clusterable> colFilter = x -> x instanceof Row ? ((Row) x).filter(cf, metadata) : x;
         Slices slices = makeSlices();
@@ -568,4 +598,19 @@ public void testUnfiltereds()
         test(this::generateUnfiltereds, makeStaticRow());
     }
 
+    @Test
+    public void checkStaticPath()
+    {
+        ByteComparable sp = metadata.comparator.asByteComparable(Clustering.STATIC_CLUSTERING);
+        for (ByteComparable.Version v : ByteComparable.Version.values())
+            assertEquals(0, ByteComparable.compare(sp, TrieBackedPartition.STATIC_CLUSTERING_PATH, v));
+    }
+
+    @Test
+    public void checkBottomPath()
+    {
+        ByteComparable sp = metadata.comparator.asByteComparable(ClusteringBound.BOTTOM);
+        for (ByteComparable.Version v : ByteComparable.Version.values())
+            assertEquals(0, ByteComparable.compare(sp, TrieBackedPartition.BOTTOM_PATH, v));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java b/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java
index 8c1c8ce7ba76..2b5877028700 100644
--- a/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java
+++ b/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java
@@ -58,7 +58,7 @@ public void testMutationSize()
         builder.newRow().add("s", 1);
         builder.newRow(1).add("a", 2);
         int size1 = builder.build().dataSize();
-        Assert.assertEquals(102, size1);
+        Assert.assertEquals(94, size1);
 
         builder = UpdateBuilder.create(cfm, "key0");
         builder.newRow(1).add("a", 2);
@@ -80,7 +80,7 @@ public void testUpdateAllTimestamp()
         long timestamp = FBUtilities.timestampMicros();
         RowUpdateBuilder rub = new RowUpdateBuilder(cfm, timestamp, "key0").clustering(1).add("a", 1);
         PartitionUpdate pu = rub.buildUpdate();
-        PartitionUpdate pu2 = new PartitionUpdate.Builder(pu, 0).updateAllTimestamp(0).build();
+        PartitionUpdate pu2 = pu.withUpdatedTimestamps(0);
 
         Assert.assertTrue(pu.maxTimestamp() > 0);
         Assert.assertTrue(pu2.maxTimestamp() == 0);
diff --git a/test/unit/org/apache/cassandra/db/partitions/AtomicBTreePartitionMemtableAccountingTest.java b/test/unit/org/apache/cassandra/db/partitions/AtomicBTreePartitionMemtableAccountingTest.java
index fa3c5d8c3b89..00a7d6f79a56 100644
--- a/test/unit/org/apache/cassandra/db/partitions/AtomicBTreePartitionMemtableAccountingTest.java
+++ b/test/unit/org/apache/cassandra/db/partitions/AtomicBTreePartitionMemtableAccountingTest.java
@@ -226,14 +226,14 @@ void execute()
         {
             // Test regular row updates
             Pair<Row, Row> regularRows = makeInitialAndUpdate(r1md, c2md);
-            PartitionUpdate initial = PartitionUpdate.singleRowUpdate(metadata, partitionKey, regularRows.left, null);
-            PartitionUpdate update = PartitionUpdate.singleRowUpdate(metadata, partitionKey, regularRows.right, null);
+            PartitionUpdate initial = PartitionUpdate.singleRowUpdate(metadata, partitionKey, regularRows.left);
+            PartitionUpdate update = PartitionUpdate.singleRowUpdate(metadata, partitionKey, regularRows.right);
             validateUpdates(metadata, partitionKey, Arrays.asList(initial, update));
 
             // Test static row updates
             Pair<Row, Row> staticRows = makeInitialAndUpdate(s3md, c4md);
-            PartitionUpdate staticInitial = PartitionUpdate.singleRowUpdate(metadata, partitionKey, null, staticRows.left);
-            PartitionUpdate staticUpdate = PartitionUpdate.singleRowUpdate(metadata, partitionKey, null, staticRows.right);
+            PartitionUpdate staticInitial = PartitionUpdate.singleRowUpdate(metadata, partitionKey, staticRows.left);
+            PartitionUpdate staticUpdate = PartitionUpdate.singleRowUpdate(metadata, partitionKey, staticRows.right);
             validateUpdates(metadata, partitionKey, Arrays.asList(staticInitial, staticUpdate));
         }
 
@@ -303,7 +303,8 @@ void validateUpdates(TableMetadata metadata, DecoratedKey partitionKey, List<Par
             AtomicBTreePartition partition = new AtomicBTreePartition(metadataRef, partitionKey, allocator);
 
             // For each update, apply it and verify the allocator is positive
-            long unreleasable = updates.stream().mapToLong(update -> {
+            long unreleasable = updates.stream().mapToLong(updateUntyped -> {
+                BTreePartitionUpdate update = BTreePartitionUpdate.asBTreeUpdate(updateUntyped);
                 DeletionTime exsDeletion = partition.deletionInfo().getPartitionDeletion();
                 DeletionTime updDeletion = update.deletionInfo().getPartitionDeletion();
                 long updateUnreleasable = 0;
@@ -334,7 +335,7 @@ void validateUpdates(TableMetadata metadata, DecoratedKey partitionKey, List<Par
             AtomicBTreePartition recreated = new AtomicBTreePartition(metadataRef, partitionKey, recreatedAllocator);
             try (UnfilteredRowIterator iter = partition.unfilteredIterator())
             {
-                PartitionUpdate update = PartitionUpdate.fromIterator(iter, ColumnFilter.NONE);
+                BTreePartitionUpdate update = BTreePartitionUpdate.fromIterator(iter, ColumnFilter.NONE);
                 opOrder.newBarrier().issue();
                 OpOrder.Group writeOp = opOrder.getCurrent();
                 Cloner cloner = recreatedAllocator.cloner(writeOp);
diff --git a/test/unit/org/apache/cassandra/db/partitions/PartitionIteratorsTest.java b/test/unit/org/apache/cassandra/db/partitions/PartitionIteratorsTest.java
new file mode 100644
index 000000000000..73186795c42b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/partitions/PartitionIteratorsTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.partitions;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class PartitionIteratorsTest
+{
+    private SinglePartitionReadQuery mockReadQuery()
+    {
+        SinglePartitionReadQuery readQuery = mock(SinglePartitionReadQuery.class);
+        TableMetadata metadata = mock(TableMetadata.class);
+        when(readQuery.metadata()).thenReturn(metadata);
+        DecoratedKey key = mock(DecoratedKey.class);
+        when(readQuery.partitionKey()).thenReturn(key);
+        ClusteringIndexFilter filter = mock(ClusteringIndexFilter.class);
+        when(readQuery.clusteringIndexFilter()).thenReturn(filter);
+        return readQuery;
+    }
+
+    @Test
+    public void testPartitionIteratorIsClosed()
+    {
+        PartitionIterator partitionIterator = mock(PartitionIterator.class);
+        SinglePartitionReadQuery readQuery = mockReadQuery();
+
+        Row r = mock(Row.class);
+        when(r.isRow()).thenReturn(true);
+
+        RowIterator ri = mock(RowIterator.class);
+        Iterator<Row> i1 = List.of(r).iterator();
+        when(ri.next()).thenAnswer(i -> i1.next());
+        when(ri.hasNext()).thenAnswer(i -> i1.hasNext());
+
+        when(partitionIterator.hasNext()).thenReturn(true, false);
+        when(partitionIterator.next()).thenReturn(ri);
+
+        try (RowIterator rowIterator = PartitionIterators.getOnlyElement(partitionIterator, readQuery))
+        {
+            while (rowIterator.hasNext()) rowIterator.next();
+        }
+
+        verify(partitionIterator).close();
+    }
+
+    @Test
+    public void testPartitionIteratorIsClosedInCaseOfError()
+    {
+        PartitionIterator partitionIterator = mock(PartitionIterator.class);
+        SinglePartitionReadQuery readQuery = mockReadQuery();
+
+        Row r = mock(Row.class);
+        when(r.isRow()).thenReturn(true);
+
+        RowIterator ri = mock(RowIterator.class);
+        Iterator<Row> i1 = List.of(r).iterator();
+        when(ri.next()).thenAnswer(i -> i1.next());
+        when(ri.hasNext()).thenAnswer(i -> i1.hasNext());
+
+        when(partitionIterator.hasNext()).thenReturn(true, false);
+        when(partitionIterator.next()).thenThrow(new RuntimeException("expected"));
+
+        try (RowIterator rowIterator = PartitionIterators.getOnlyElement(partitionIterator, readQuery))
+        {
+            while (rowIterator.hasNext()) rowIterator.next();
+            fail();
+        }
+        catch (RuntimeException e)
+        {
+            // expected
+        }
+
+        verify(partitionIterator).close();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/partitions/RowTrackingIteratorTest.java b/test/unit/org/apache/cassandra/db/partitions/RowTrackingIteratorTest.java
new file mode 100644
index 000000000000..198d329b4073
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/partitions/RowTrackingIteratorTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.partitions;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+public class RowTrackingIteratorTest
+{
+    private static final int EMPTY_ROW = -1;
+
+    Partition[] testData = new Partition[]
+    {
+        partition(key(1), staticRow(3), rows(1, 2, 3)),
+        partition(key(11), staticRow(), rows(10)),
+        partition(key(21), staticRow(), rows()),
+        partition(key(31), staticRow(), rows(20, 21, 22))
+    };
+
+    @Mock
+    Consumer<DecoratedKey> partitionVerifier;
+    @Mock
+    Consumer<Row> staticRowVerifier;
+    @Mock
+    Consumer<Row> rowVerifier;
+
+    @Captor
+    private ArgumentCaptor<DecoratedKey> partitionCaptor;
+    @Captor
+    private ArgumentCaptor<Row> staticRowCaptor;
+    @Captor
+    private ArgumentCaptor<Row> rowCaptor;
+
+    @Test
+    public void testRowTrackingIterator()
+    {
+        initMocks(this);
+        PartitionIterator partitionIterator = makePartitionIterator(testData);
+
+        PartitionIterator testedIterator = PartitionIterators.filteredRowTrackingIterator(partitionIterator,
+                                                                                          partitionVerifier,
+                                                                                          staticRowVerifier,
+                                                                                          rowVerifier);
+
+        // iterate over partitions and rows
+        while (testedIterator.hasNext())
+        {
+            RowIterator rowIterator = testedIterator.next();
+            while (rowIterator.hasNext())
+            {
+                rowIterator.next();
+            }
+        }
+
+        // verify mock expectations
+        verify(partitionVerifier, times(4)).accept(partitionCaptor.capture());
+        verify(staticRowVerifier, times(1)).accept(staticRowCaptor.capture());
+        verify(rowVerifier, times(7)).accept(rowCaptor.capture());
+
+        int staticRowIndex = 0;
+        int rowIndex = 0;
+        for (int partitionIndex = 0; partitionIndex < testData.length; partitionIndex++)
+        {
+            Partition partition = testData[partitionIndex];
+            assertEquals(partition.key, partitionCaptor.getAllValues().get(partitionIndex));
+            if (!partition.staticRow.isEmpty())
+            {
+                // in the test we're identifying rows by the number of columns they bear
+                assertEquals(partition.staticRow.columnCount(), staticRowCaptor.getAllValues().get(staticRowIndex++).columnCount());
+            }
+            for (int idx = 0; idx < partition.rows.size(); idx++)
+            {
+                // in the test we're identifying rows by the number of columns they bear
+                assertEquals(partition.rows.get(idx).columnCount(), rowCaptor.getAllValues().get(rowIndex++).columnCount());
+            }
+        }
+
+        verifyNoMoreInteractions(partitionVerifier, staticRowVerifier, rowVerifier);
+    }
+
+    // below are helper methods to make the test data more readable
+    private static class Partition
+    {
+        private final DecoratedKey key;
+        private final Row staticRow;
+        private final List<Row> rows;
+
+        public Partition(int key, int staticRowDiscriminator, int[] rows)
+        {
+            this.key = new Murmur3Partitioner().decorateKey(ByteBufferUtil.bytes(key));
+            Row staticRow = mock(Row.class);
+            if (staticRowDiscriminator != EMPTY_ROW)
+            {
+                when(staticRow.columnCount()).thenReturn(staticRowDiscriminator);
+            }
+            else
+            {
+                when(staticRow.isEmpty()).thenReturn(true);
+            }
+            this.staticRow = staticRow;
+            this.rows = Arrays.stream(rows).boxed().map(rowDiscriminator -> {
+                Row row = mock(Row.class);
+                when(row.columnCount()).thenReturn(rowDiscriminator);
+                return row;
+            }).collect(Collectors.toList());
+        }
+    }
+
+    private Partition partition(int key, int staticRowDiscriminator, int... rows)
+    {
+        return new Partition(key, staticRowDiscriminator, rows);
+    }
+
+    private static int[] rows(int... rowDiscriminators)
+    {
+        return rowDiscriminators;
+    }
+
+    private static int key(int keyDiscriminator)
+    {
+        return keyDiscriminator;
+    }
+
+    private static int staticRow(int staticRowDiscriminator)
+    {
+        return staticRowDiscriminator;
+    }
+
+    private static int staticRow()
+    {
+        return EMPTY_ROW;
+    }
+
+    private static PartitionIterator makePartitionIterator(Partition[] partitions)
+    {
+        return new PartitionIterator()
+        {
+            int i = 0;
+            public boolean hasNext()
+            {
+                return i < partitions.length;
+            }
+            public RowIterator next()
+            {
+                return makeRowIterator(partitions[i++]);
+            }
+            public void close()
+            {
+            }
+        };
+    }
+
+    private static RowIterator makeRowIterator(Partition partition)
+    {
+        Iterator<Row> rows = partition.rows.iterator();
+
+        RowIterator iter = mock(RowIterator.class);
+        when(iter.partitionKey()).thenReturn(partition.key);
+        when(iter.staticRow()).thenReturn(partition.staticRow);
+        when(iter.hasNext()).thenAnswer(invocation -> rows.hasNext());
+        when(iter.next()).thenAnswer(invocation -> rows.next());
+
+        return iter;
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionBytemanTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionBytemanTest.java
index 795c7fe82c47..931937d4ff26 100644
--- a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionBytemanTest.java
+++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionBytemanTest.java
@@ -20,29 +20,41 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 import com.google.common.collect.Lists;
+import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.RangesAtEndpoint;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.assertj.core.api.Assertions;
 import org.jboss.byteman.contrib.bmunit.BMRule;
 import org.jboss.byteman.contrib.bmunit.BMRules;
 import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
 
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 @RunWith(BMUnitRunner.class)
 public class PendingAntiCompactionBytemanTest extends AbstractPendingAntiCompactionTest
@@ -50,9 +62,10 @@ public class PendingAntiCompactionBytemanTest extends AbstractPendingAntiCompact
     @BMRules(rules = { @BMRule(name = "Throw exception anticompaction",
                                targetClass = "Range$OrderedRangeContainmentChecker",
                                targetMethod = "test",
-                               action = "throw new org.apache.cassandra.db.compaction.CompactionInterruptedException(\"antiCompactionExceptionTest\");")} )
+                               action = "throw new org.apache.cassandra.db.compaction.CompactionInterruptedException" +
+                                        "(\"antiCompactionExceptionTest\", org.apache.cassandra.db.compaction.TableOperation$StopTrigger.UNIT_TESTS);")} )
     @Test
-    public void testExceptionAnticompaction() throws InterruptedException
+    public void testExceptionAnticompaction()
     {
         cfs.disableAutoCompaction();
         cfs2.disableAutoCompaction();
@@ -66,16 +79,11 @@ public void testExceptionAnticompaction() throws InterruptedException
             ranges.add(new Range<>(sstable.getFirst().getToken(), sstable.getLast().getToken()));
         }
         TimeUUID prsid = prepareSession();
-        try
-        {
-            PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Lists.newArrayList(cfs, cfs2), atEndpoint(ranges, NO_RANGES), es, () -> false);
-            pac.run().get();
-            fail("PAC should throw exception when anticompaction throws exception!");
-        }
-        catch (ExecutionException e)
-        {
-            assertTrue(e.getCause() instanceof CompactionInterruptedException);
-        }
+        PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Lists.newArrayList(cfs, cfs2), atEndpoint(ranges, NO_RANGES), es, () -> false);
+        Assertions.assertThatThrownBy(() -> pac.run().get())
+                  .hasCauseInstanceOf(CompactionInterruptedException.class)
+                  .hasMessageContaining("Compaction interrupted due to unit tests");
+
         // Note that since we fail the PAC immediately when any of the anticompactions fail we need to wait for the other
         // AC to finish as well before asserting that we have nothing compacting.
         CompactionManager.instance.waitForCessation(Lists.newArrayList(cfs, cfs2), (sstable) -> true);
@@ -97,4 +105,58 @@ private static RangesAtEndpoint atEndpoint(Collection<Range<Token>> full, Collec
 
         return builder.build();
     }
+
+    @BMRules(rules = { @BMRule(name = "Abort anti-compaction after first call to onOperationStart",
+            targetClass = "CompactionManager",
+            targetMethod = "antiCompactGroup",
+            condition = "not flagged(\"done\")",
+            targetLocation = "AFTER INVOKE compactionRateLimiterAcquire",
+            action = "org.apache.cassandra.db.compaction.CompactionManager.instance.stopCompaction(\"ANTICOMPACTION\");") } )
+    @Test
+    public void testStopAntiCompaction()
+    {
+        Assert.assertSame(ByteOrderedPartitioner.class, DatabaseDescriptor.getPartitioner().getClass());
+        cfs.disableAutoCompaction();
+
+        // create 2 sstables, one that will be split, and another that will be moved
+        for (int i = 0; i < 10; i++)
+        {
+            QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", ks, tbl), i, i);
+        }
+        cfs.forceBlockingFlush(UNIT_TESTS);
+        for (int i = 10; i < 20; i++)
+        {
+            QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", ks, tbl), i, i);
+        }
+        cfs.forceBlockingFlush(UNIT_TESTS);
+
+        assertEquals(2, cfs.getLiveSSTables().size());
+        assertEquals(0, cfs.getLiveSSTables().stream().filter(SSTableReader::isPendingRepair).count());
+
+        Token left = ByteOrderedPartitioner.instance.getToken(ByteBufferUtil.bytes(5));
+        Token right = ByteOrderedPartitioner.instance.getToken(ByteBufferUtil.bytes(15));
+        List<Range<Token>> ranges = Collections.singletonList(new Range<>(left, right));
+        List<ColumnFamilyStore> tables = Collections.singletonList(cfs);
+
+        // create a repair session so the anti-compaction job can find it
+        TimeUUID sessionID = TimeUUID.Generator.nextTimeUUID();
+        ActiveRepairService.instance().registerParentRepairSession(sessionID, InetAddressAndPort.getLocalHost(), tables, ranges, true, 1, true, PreviewKind.NONE);
+
+        ExecutorService executor = Executors.newSingleThreadExecutor();
+        try
+        {
+            PendingAntiCompaction pac = new PendingAntiCompaction(sessionID, tables, atEndpoint(ranges, NO_RANGES), executor, () -> false);
+            Future<?> future = pac.run();
+            Assertions.assertThatThrownBy(future::get)
+                      .hasCauseInstanceOf(CompactionInterruptedException.class)
+                      .hasMessageContaining("Compaction interrupted due to user request");
+        }
+        finally
+        {
+            executor.shutdown();
+        }
+
+        assertEquals(2, cfs.getLiveSSTables().size());
+        assertEquals(0, cfs.getLiveSSTables().stream().filter(SSTableReader::isPendingRepair).count());
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
index 4960bfd18888..73a279855b13 100644
--- a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
@@ -18,6 +18,8 @@
 
 package org.apache.cassandra.db.repair;
 
+import java.io.Closeable;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -53,12 +55,13 @@
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.AbstractPendingRepairTest;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionController;
-import org.apache.cassandra.db.compaction.CompactionInfo;
 import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.compaction.CompactionIterator;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -79,8 +82,10 @@
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 import org.apache.cassandra.utils.WrappedRunnable;
 import org.apache.cassandra.utils.concurrent.Transactional;
+import org.assertj.core.api.Assertions;
 
 import static java.util.Collections.emptyList;
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
@@ -149,7 +154,7 @@ public void successCase() throws Exception
         try
         {
             pac = new PendingAntiCompaction(sessionID, tables, atEndpoint(ranges, NO_RANGES), executor, () -> false);
-            pac.run().get();
+            pac.run().get(30, TimeUnit.SECONDS);
         }
         finally
         {
@@ -430,7 +435,7 @@ Future submitPendingAntiCompaction(AcquireResult result)
                         {
                             protected void runMayThrow()
                             {
-                                throw new CompactionInterruptedException("antiCompactionExceptionTest");
+                                throw new CompactionInterruptedException("antiCompactionExceptionTest", TableOperation.StopTrigger.UNIT_TESTS);
                             }
                         };
                         return es.submit(r);
@@ -439,14 +444,9 @@ protected void runMayThrow()
             }
         };
         ListenableFuture<?> fut = pac.run();
-        try
-        {
-            fut.get();
-            fail("Should throw exception");
-        }
-        catch(Throwable t)
-        {
-        }
+        Assertions.assertThatThrownBy(fut::get)
+                  .hasRootCauseInstanceOf(CompactionInterruptedException.class)
+                  .hasMessageContaining("Compaction interrupted due to unit tests");
     }
 
     @Test
@@ -463,27 +463,30 @@ public void testBlockedAcquisition() throws ExecutionException, InterruptedExcep
         {
             try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
                  CompactionController controller = new CompactionController(cfs, sstables, 0);
-                 CompactionIterator ci = CompactionManager.getAntiCompactionIterator(scanners, controller, 0, nextTimeUUID(), CompactionManager.instance.active, () -> false))
+                 CompactionIterator ci = new CompactionIterator(OperationType.ANTICOMPACTION, scanners, controller, 0, nextTimeUUID()))
             {
-                // `ci` is our imaginary ongoing anticompaction which makes no progress until after 30s
-                // now we try to start a new AC, which will try to cancel all ongoing compactions
-
-                CompactionManager.instance.active.beginCompaction(ci);
-                PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs), atEndpoint(FULL_RANGE, NO_RANGES), 0, 0, es, () -> false);
-                ListenableFuture fut = pac.run();
-                try
+                TableOperation op = ci.getOperation();
+                try (NonThrowingCloseable cls = CompactionManager.instance.active.onOperationStart(op))
                 {
-                    fut.get(30, TimeUnit.SECONDS);
-                    fail("the future should throw exception since we try to start a new anticompaction when one is already running");
-                }
-                catch (ExecutionException e)
-                {
-                    assertTrue(e.getCause() instanceof PendingAntiCompaction.SSTableAcquisitionException);
-                }
+                    // `ci` is our imaginary ongoing anticompaction which makes no progress until after 30s
+                    // now we try to start a new AC, which will try to cancel all ongoing compactions
 
-                assertEquals(1, getCompactionsFor(cfs).size());
-                for (CompactionInfo.Holder holder : getCompactionsFor(cfs))
-                    assertFalse(holder.isStopRequested());
+                    PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs), atEndpoint(FULL_RANGE, NO_RANGES), 0, 0, es, () -> false);
+                    ListenableFuture fut = pac.run();
+                    try
+                    {
+                        fut.get(30, TimeUnit.SECONDS);
+                        fail("the future should throw exception since we try to start a new anticompaction when one is already running");
+                    }
+                    catch (ExecutionException e)
+                    {
+                        assertTrue(e.getCause() instanceof PendingAntiCompaction.SSTableAcquisitionException);
+                    }
+
+                    assertEquals(1, getCompactionsFor(cfs).size());
+                    for (TableOperation compaction : getCompactionsFor(cfs))
+                        assertFalse(compaction.isStopRequested());
+                }
             }
         }
         finally
@@ -493,13 +496,13 @@ public void testBlockedAcquisition() throws ExecutionException, InterruptedExcep
         }
     }
 
-    private List<CompactionInfo.Holder> getCompactionsFor(ColumnFamilyStore cfs)
+    private List<TableOperation> getCompactionsFor(ColumnFamilyStore cfs)
     {
-        List<CompactionInfo.Holder> compactions = new ArrayList<>();
-        for (CompactionInfo.Holder holder : CompactionManager.instance.active.getCompactions())
+        List<TableOperation> compactions = new ArrayList<>();
+        for (TableOperation compaction : CompactionManager.instance.active.getTableOperations())
         {
-            if (holder.getCompactionInfo().getTableMetadata().equals(cfs.metadata()))
-                compactions.add(holder);
+            if (compaction.getProgress().metadata().equals(cfs.metadata()))
+                compactions.add(compaction);
         }
         return compactions;
     }
@@ -519,46 +522,48 @@ public void testUnblockedAcquisition() throws ExecutionException, InterruptedExc
                  CompactionController controller = new CompactionController(cfs, sstables, 0);
                  CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners, controller, 0, nextTimeUUID()))
             {
-                // `ci` is our imaginary ongoing anticompaction which makes no progress until after 5s
-                // now we try to start a new AC, which will try to cancel all ongoing compactions
-
-                CompactionManager.instance.active.beginCompaction(ci);
-                PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs), atEndpoint(FULL_RANGE, NO_RANGES), es, () -> false);
-                ListenableFuture fut = pac.run();
-                try
+                TableOperation op = ci.getOperation();
+                try (NonThrowingCloseable cls = CompactionManager.instance.active.onOperationStart(op))
                 {
-                    fut.get(5, TimeUnit.SECONDS);
-                }
-                catch (TimeoutException e)
-                {
-                    // expected, we wait 1 minute for compactions to get cancelled in runWithCompactionsDisabled, but we are not iterating
-                    // CompactionIterator so the compaction is not actually cancelled
-                }
-                try
-                {
-                    assertTrue(ci.hasNext());
-                    ci.next();
-                    fail("CompactionIterator should be abortable");
-                }
-                catch (CompactionInterruptedException e)
-                {
-                    CompactionManager.instance.active.finishCompaction(ci);
-                    txn.abort();
-                    // expected
-                }
-                CountDownLatch cdl = new CountDownLatch(1);
-                Futures.addCallback(fut, new FutureCallback<Object>()
-                {
-                    public void onSuccess(@Nullable Object o)
+                    // `ci` is our imaginary ongoing anticompaction which makes no progress until after 5s
+                    // now we try to start a new AC, which will try to cancel all ongoing compactions
+
+                    PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs), atEndpoint(FULL_RANGE, NO_RANGES), es, () -> false);
+                    ListenableFuture fut = pac.run();
+                    try
                     {
-                        cdl.countDown();
+                        fut.get(5, TimeUnit.SECONDS);
                     }
-
-                    public void onFailure(Throwable throwable)
+                    catch (TimeoutException e)
                     {
+                        // expected, we wait 1 minute for compactions to get cancelled in runWithCompactionsDisabled, but we are not iterating
+                        // CompactionIterator so the compaction is not actually cancelled
                     }
-                }, MoreExecutors.directExecutor());
-                assertTrue(cdl.await(1, TimeUnit.MINUTES));
+                    try
+                    {
+                        assertTrue(ci.hasNext());
+                        ci.next();
+                        fail("CompactionIterator should be abortable");
+                    }
+                    catch (CompactionInterruptedException e)
+                    {
+                        txn.abort();
+                        // expected
+                    }
+                    CountDownLatch cdl = new CountDownLatch(1);
+                    Futures.addCallback(fut, new FutureCallback<Object>()
+                    {
+                        public void onSuccess(@Nullable Object o)
+                        {
+                            cdl.countDown();
+                        }
+
+                        public void onFailure(Throwable throwable)
+                        {
+                        }
+                    }, MoreExecutors.directExecutor());
+                    assertTrue(cdl.await(1, TimeUnit.MINUTES));
+                }
             }
         }
         finally
@@ -605,11 +610,11 @@ public void testSSTablePredicateOngoingAntiCompaction()
 
     private void tryPredicate(ColumnFamilyStore cfs, List<SSTableReader> compacting, List<SSTableReader> expectedLive, boolean shouldFail)
     {
-        CompactionInfo.Holder holder = new CompactionInfo.Holder()
+        TableOperation operation = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, nextTimeUUID(), compacting);
+                return new OperationProgress(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, nextTimeUUID(), compacting);
             }
 
             public boolean isGlobal()
@@ -617,8 +622,7 @@ public boolean isGlobal()
                 return false;
             }
         };
-        CompactionManager.instance.active.beginCompaction(holder);
-        try
+        try(Closeable c = CompactionManager.instance.active.onOperationStart(operation))
         {
             PendingAntiCompaction.AntiCompactionPredicate predicate =
             new PendingAntiCompaction.AntiCompactionPredicate(Collections.singleton(new Range<>(new Murmur3Partitioner.LongToken(0), new Murmur3Partitioner.LongToken(100))),
@@ -628,29 +632,25 @@ public boolean isGlobal()
                 fail("should fail - we try to grab already anticompacting sstables for anticompaction");
             assertEquals(live, new HashSet<>(expectedLive));
         }
-        catch (PendingAntiCompaction.SSTableAcquisitionException e)
+        catch (PendingAntiCompaction.SSTableAcquisitionException | IOException e)
         {
             if (!shouldFail)
                 fail("We should not fail filtering sstables");
         }
-        finally
-        {
-            CompactionManager.instance.active.finishCompaction(holder);
-        }
     }
 
     @Test
-    public void testRetries() throws InterruptedException, ExecutionException
+    public void testRetries() throws InterruptedException, ExecutionException, TimeoutException
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
         cfs.addSSTable(MockSchema.sstable(1, true, cfs));
         CountDownLatch cdl = new CountDownLatch(5);
         ExecutorPlus es = executorFactory().sequential("test");
-        CompactionInfo.Holder holder = new CompactionInfo.Holder()
+        AbstractTableOperation operation = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
+                return new OperationProgress(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
             }
 
             public boolean isGlobal()
@@ -668,8 +668,7 @@ public boolean apply(SSTableReader sstable)
                     return true;
                 }
             };
-
-            CompactionManager.instance.active.beginCompaction(holder);
+            NonThrowingCloseable closeable = CompactionManager.instance.active.onOperationStart(operation);
             PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, nextTimeUUID(), 10, 1, acp)
             {
                 protected PendingAntiCompaction.AcquireResult acquireSSTables()
@@ -677,33 +676,32 @@ protected PendingAntiCompaction.AcquireResult acquireSSTables()
                     cdl.countDown();
                     if (cdl.getCount() > 0)
                         throw new PendingAntiCompaction.SSTableAcquisitionException("blah");
-                    else
-                        CompactionManager.instance.active.finishCompaction(holder);
+                    else 
+                        closeable.close();
                     return super.acquireSSTables();
                 }
             };
             Future f = es.submit(acquisitionCallable);
             cdl.await();
-            assertNotNull(f.get());
+            assertNotNull(f.get(30, TimeUnit.SECONDS));
         }
         finally
         {
             es.shutdown();
-            CompactionManager.instance.active.finishCompaction(holder);
         }
     }
 
     @Test
-    public void testRetriesTimeout() throws InterruptedException, ExecutionException
+    public void testRetriesTimeout() throws InterruptedException, ExecutionException, IOException, TimeoutException
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
         cfs.addSSTable(MockSchema.sstable(1, true, cfs));
         ExecutorPlus es = executorFactory().sequential("test");
-        CompactionInfo.Holder holder = new CompactionInfo.Holder()
+        TableOperation operation = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
+                return new OperationProgress(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
             }
 
             public boolean isGlobal()
@@ -711,7 +709,7 @@ public boolean isGlobal()
                 return false;
             }
         };
-        try
+        try (Closeable c = CompactionManager.instance.active.onOperationStart(operation))
         {
             PendingAntiCompaction.AntiCompactionPredicate acp = new PendingAntiCompaction.AntiCompactionPredicate(FULL_RANGE, nextTimeUUID())
             {
@@ -721,20 +719,18 @@ public boolean apply(SSTableReader sstable)
                     throw new PendingAntiCompaction.SSTableAcquisitionException("blah");
                 }
             };
-            CompactionManager.instance.active.beginCompaction(holder);
             PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, nextTimeUUID(), 2, 1000, acp);
             Future fut = es.submit(acquisitionCallable);
-            assertNull(fut.get());
+            assertNull(fut.get(30, TimeUnit.SECONDS));
         }
         finally
         {
             es.shutdown();
-            CompactionManager.instance.active.finishCompaction(holder);
         }
     }
 
     @Test
-    public void testWith2i() throws ExecutionException, InterruptedException
+    public void testWith2i() throws ExecutionException, InterruptedException, TimeoutException
     {
         cfs2.disableAutoCompaction();
         makeSSTables(2, cfs2, 100);
@@ -750,7 +746,7 @@ public void testWith2i() throws ExecutionException, InterruptedException
             try (LifecycleTransaction txn = idx.getTracker().tryModify(idx.getLiveSSTables(), OperationType.COMPACTION))
             {
                 PendingAntiCompaction pac = new PendingAntiCompaction(prsid, Collections.singleton(cfs2), atEndpoint(FULL_RANGE, NO_RANGES), es, () -> false);
-                pac.run().get();
+                pac.run().get(30, TimeUnit.SECONDS);
             }
             // and make sure it succeeded;
             for (SSTableReader sstable : cfs2.getLiveSSTables())
diff --git a/test/unit/org/apache/cassandra/db/rows/BTreeRowTest.java b/test/unit/org/apache/cassandra/db/rows/BTreeRowTest.java
new file mode 100644
index 000000000000..cfdb9c3ae5f8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/rows/BTreeRowTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.rows;
+
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+
+public class BTreeRowTest
+{
+    private final TableMetadata metadata = TableMetadata.builder("", "")
+                                                        .addPartitionKeyColumn("pk", Int32Type.instance)
+                                                        .addClusteringColumn("ck", Int32Type.instance)
+                                                        .addRegularColumn("v1", Int32Type.instance)
+                                                        .addRegularColumn("v2", Int32Type.instance)
+                                                        .build();
+    private final ColumnMetadata v2Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(1);
+    private final ColumnMetadata v1Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(0);
+
+    private BTreeRow.Builder row(int ck, Cell<?>... columns)
+    {
+        BTreeRow.Builder builder = new BTreeRow.Builder(true);
+        builder.newRow(Util.clustering(metadata.comparator, ck));
+        for (Cell<?> cell : columns)
+            builder.addCell(cell);
+        return builder;
+    }
+
+    private Cell<?> cell(ColumnMetadata metadata, int v, long timestamp)
+    {
+        return new BufferCell(metadata,
+                              timestamp,
+                              BufferCell.NO_TTL,
+                              BufferCell.NO_DELETION_TIME,
+                              ByteBufferUtil.bytes(v),
+                              null);
+    }
+
+    @Test
+    public void testRowMinTimespampFromCells()
+    {
+        int v1CellTimestamp = 1000;
+        int v2CellTimestamp = 500;
+        int primaryKeyTimestamp = 2000;
+        BTreeRow.Builder builder = row(1, cell(v1Metadata, 1, v1CellTimestamp), cell(v2Metadata, 1, v2CellTimestamp));
+        builder.addPrimaryKeyLivenessInfo(LivenessInfo.create(primaryKeyTimestamp, FBUtilities.nowInSeconds()));
+        Row row = builder.build();
+        assertEquals(v2CellTimestamp, row.minTimestamp());
+    }
+
+    @Test
+    public void testRowMinTimespampFromPrimaryKeyListener()
+    {
+        int v1CellTimestamp = 1000;
+        int v2CellTimestamp = 500;
+        int primaryKeyTimestamp = 100;
+        BTreeRow.Builder builder = row(2, cell(v1Metadata, 1, v1CellTimestamp), cell(v2Metadata, 1, v2CellTimestamp));
+        builder.addPrimaryKeyLivenessInfo(LivenessInfo.create(primaryKeyTimestamp, FBUtilities.nowInSeconds()));
+        Row row = builder.build();
+        assertEquals(primaryKeyTimestamp, row.minTimestamp());
+    }
+
+    @Test
+    public void testRowMinTimespampFromDeletion()
+    {
+        int v1CellTimestamp = 1000;
+        int v2CellTimestamp = 500;
+        int primaryKeyTimestamp = 100;
+        int localDeletionTime = 50;
+        BTreeRow.Builder builder = row(3, cell(v1Metadata, 1, v1CellTimestamp), cell(v2Metadata, 1, v2CellTimestamp));
+        builder.addPrimaryKeyLivenessInfo(LivenessInfo.create(primaryKeyTimestamp, FBUtilities.nowInSeconds()));
+        builder.addRowDeletion(new Row.Deletion(DeletionTime.build(localDeletionTime, FBUtilities.nowInSeconds()), true));
+        Row row = builder.build();
+        assertEquals(primaryKeyTimestamp, row.minTimestamp());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/rows/ComplexColumnDataTest.java b/test/unit/org/apache/cassandra/db/rows/ComplexColumnDataTest.java
new file mode 100644
index 000000000000..d91f110210ca
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/rows/ComplexColumnDataTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.rows;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.btree.BTree;
+
+public class ComplexColumnDataTest
+{
+    private ColumnMetadata complexColumn = ColumnMetadata.regularColumn("ks", "tab", "col0",
+                                                                        MapType.getInstance(Int32Type.instance, Int32Type.instance, true));
+
+    private ColumnMetadata simpleColumn = ColumnMetadata.regularColumn("ks", "tab", "col1",
+                                                                       Int32Type.instance);
+
+    @Test
+    public void testEmptyComplexColumn()
+    {
+        ComplexColumnData data = new ComplexColumnData(complexColumn,
+                                                       BTree.empty(),
+                                                       DeletionTime.LIVE);
+        Assert.assertFalse(data.hasCells());
+    }
+
+    @Test
+    public void testNonEmptyComplexColumn()
+    {
+
+        ComplexColumnData data = new ComplexColumnData(complexColumn,
+                                                       BTree.singleton("ignored value"),
+                                                       DeletionTime.LIVE);
+        Assert.assertTrue(data.hasCells());
+    }
+
+    @Test
+    public void testComplexColumnMinTimestampWithDeletion()
+    {
+        ComplexColumnData data = new ComplexColumnData(complexColumn,
+                                                       BTree.empty(),
+                                                       DeletionTime.build(500, 1000));
+        Assert.assertEquals("Min timestamp must be equal to deletion timestamp", 500, data.minTimestamp());
+    }
+
+    @Test
+    public void testComplexColumnMinTimestampWithCells()
+    {
+        ComplexColumnData data = new ComplexColumnData(complexColumn,
+                                                       new Cell[]{ new BufferCell(simpleColumn, 100, 0, 200, null, null) },
+                                                       DeletionTime.build(500, 1000));
+        Assert.assertEquals("Min timestamp must be equal to min cell timestamp", 100, data.minTimestamp());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/rows/RowAndDeletionMergeIteratorTest.java b/test/unit/org/apache/cassandra/db/rows/RowAndDeletionMergeIteratorTest.java
index ce80b88160b3..03b7b7c92338 100644
--- a/test/unit/org/apache/cassandra/db/rows/RowAndDeletionMergeIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/RowAndDeletionMergeIteratorTest.java
@@ -360,7 +360,7 @@ public void testWithNoopBoundaryMarkers()
 
         Assert.assertEquals(3, rtl.size());
 
-        try (UnfilteredRowIterator partition = createMergeIterator(update.iterator(), rtl.iterator(), false))
+        try (UnfilteredRowIterator partition = createMergeIterator(update.rowIterator(), rtl.iterator(), false))
         {
             assertRtMarker(partition.next(), ClusteringPrefix.Kind.INCL_START_BOUND, 0);
             assertRtMarker(partition.next(), ClusteringPrefix.Kind.INCL_END_BOUND, 8);
@@ -392,11 +392,11 @@ private Iterator<RangeTombstone> createRangeTombstoneIterator(RangeTombstone...
 
     private Iterator<Row> createRowIterator()
     {
-        PartitionUpdate.Builder update = new PartitionUpdate.Builder(cfm, dk, cfm.regularAndStaticColumns(), 1);
+        PartitionUpdate.Builder update = PartitionUpdate.builder(cfm, dk, cfm.regularAndStaticColumns(), 1);
         for (int i = 0; i < 5; i++)
             addRow(update, i, i);
 
-        return update.build().iterator();
+        return update.build().rowIterator();
     }
 
     private UnfilteredRowIterator createMergeIterator(Iterator<Row> rows, Iterator<RangeTombstone> tombstones, boolean reversed)
diff --git a/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java b/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java
index d2a9aa782400..96ecb67d9ec4 100644
--- a/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java
@@ -52,7 +52,7 @@
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.TrieBackedPartition;
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -651,7 +651,7 @@ public void testThrottledIteratorWithRangeDeletions() throws Exception
                 while (throttled.hasNext())
                 {
                     UnfilteredRowIterator next = throttled.next();
-                    ImmutableBTreePartition materializedPartition = ImmutableBTreePartition.create(next);
+                    TrieBackedPartition materializedPartition = TrieBackedPartition.fromIterator(next);
                     int unfilteredCount = Iterators.size(materializedPartition.unfilteredIterator());
 
                     System.out.println("batchsize " + batchSize + " unfilteredCount " + unfilteredCount + " materializedPartition " + materializedPartition);
@@ -680,7 +680,7 @@ public void testThrottledIteratorWithRangeDeletions() throws Exception
             }
 
             // Verify throttled data after merge
-            Partition partition = ImmutableBTreePartition.create(UnfilteredRowIterators.merge(unfilteredRowIterators));
+            Partition partition = TrieBackedPartition.fromIterator(UnfilteredRowIterators.merge(unfilteredRowIterators));
 
             long nowInSec = FBUtilities.nowInSeconds();
 
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsMergeTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsMergeTest.java
index 246a0baac205..ae378109cd1b 100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsMergeTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsMergeTest.java
@@ -18,25 +18,45 @@
 package org.apache.cassandra.db.rows;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Random;
 import java.util.function.IntUnaryOperator;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterators;
-
+import com.google.common.collect.Maps;
 import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.BufferClusteringBound;
+import org.apache.cassandra.db.Clusterable;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
 import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.rows.Unfiltered.Kind;
+import org.apache.cassandra.io.sstable.compaction.IteratorFromCursor;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursor;
+import org.apache.cassandra.io.sstable.compaction.SSTableCursorMerger;
+import org.apache.cassandra.io.sstable.compaction.SkipEmptyDataCursor;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.FBUtilities;
 
 public class UnfilteredRowIteratorsMergeTest
@@ -46,7 +66,7 @@ public class UnfilteredRowIteratorsMergeTest
         DatabaseDescriptor.daemonInitialization();
     }
     static DecoratedKey partitionKey = Util.dk("key");
-    static DeletionTime partitionLevelDeletion = DeletionTime.LIVE;
+    static Map<List<Unfiltered>, DeletionTime> partitionLevelDeletions = Maps.newHashMap();
     static TableMetadata metadata =
         TableMetadata.builder("UnfilteredRowIteratorsMergeTest", "Test")
                      .addPartitionKeyColumn("key", AsciiType.instance)
@@ -71,25 +91,37 @@ public UnfilteredRowIteratorsMergeTest()
     @Test
     public void testTombstoneMerge()
     {
-        testTombstoneMerge(false, false);
+        testTombstoneMerge(false, false, false);
     }
 
     @Test
     public void testTombstoneMergeReversed()
     {
-        testTombstoneMerge(true, false);
+        testTombstoneMerge(true, false, false);
+    }
+    @Test
+    public void testTombstoneMergeCursor()
+    {
+        testTombstoneMerge(false, false, true);
     }
 
+
     @Test
     public void testTombstoneMergeIterative()
     {
-        testTombstoneMerge(false, true);
+        testTombstoneMerge(false, true, false);
     }
 
     @Test
     public void testTombstoneMergeReversedIterative()
     {
-        testTombstoneMerge(true, true);
+        testTombstoneMerge(true, true, false);
+    }
+
+    @Test
+    public void testTombstoneMergeCursorIterative()
+    {
+        testTombstoneMerge(false, true, true);
     }
 
     @Test
@@ -100,8 +132,15 @@ public void testDuplicateRangeCase()
                      "66<[13] [13]<67");
     }
 
+    @Test
+    public void testWithPartitionLevelDeletion()
+    {
+        testForInput("D5|68[7]",
+                     "67<=[11] [11]<69" );
+    }
+
     @SuppressWarnings("unused")
-    public void testTombstoneMerge(boolean reversed, boolean iterations)
+    public void testTombstoneMerge(boolean reversed, boolean iterations, boolean throughCursor)
     {
         this.reversed = reversed;
         UnfilteredRowsGenerator generator = new UnfilteredRowsGenerator(comparator, reversed);
@@ -122,8 +161,8 @@ public void testTombstoneMerge(boolean reversed, boolean iterations)
                 System.out.println("Merging");
             for (int i=0; i<ITERATORS; ++i)
                 sources.add(generator.generateSource(r, ITEMS, RANGE, DEL_RANGE, timeGenerators.get(r.nextInt(timeGenerators.size()))));
-            List<Unfiltered> merged = merge(sources, iterations);
-    
+            List<Unfiltered> merged = merge(sources, iterations, throughCursor);
+
             if (ITEMS <= 20)
                 System.out.println("results in");
             if (ITEMS <= 20)
@@ -138,11 +177,23 @@ public void testTombstoneMerge(boolean reversed, boolean iterations)
         }
     }
 
-    private List<Unfiltered> merge(List<List<Unfiltered>> sources, boolean iterations)
+    private List<Unfiltered> merge(List<List<Unfiltered>> sources, boolean iterations, boolean throughCursors)
+    {
+        if (throughCursors)
+            return mergeThroughCursors(sources,  iterations);
+        else
+            return mergeThroughIterators(sources, iterations);
+    }
+
+    private List<Unfiltered> mergeThroughIterators(List<List<Unfiltered>> sources, boolean iterations)
     {
         List<UnfilteredRowIterator> us = sources.
                 stream().
-                map(l -> new UnfilteredRowsGenerator.Source(l.iterator(), metadata, partitionKey, DeletionTime.LIVE, reversed)).
+                map(l -> new UnfilteredRowsGenerator.Source(l.iterator(),
+                                                            metadata,
+                                                            partitionKey,
+                                                            partitionLevelDeletions.computeIfAbsent(l, v -> DeletionTime.LIVE),
+                                                            reversed)).
                 collect(Collectors.toList());
         List<Unfiltered> merged = new ArrayList<>();
         Iterators.addAll(merged, mergeIterators(us, iterations));
@@ -167,6 +218,155 @@ public UnfilteredRowIterator mergeIterators(List<UnfilteredRowIterator> us, bool
         }
     }
 
+    private List<Unfiltered> mergeThroughCursors(List<List<Unfiltered>> sources, boolean iterations)
+    {
+        List<SSTableCursor> us = sources.stream()
+                                        .map(l -> cursor(l,
+                                                         partitionKey,
+                                                         partitionLevelDeletions.computeIfAbsent(l, v -> DeletionTime.LIVE)))
+                                        .collect(Collectors.toList());
+        List<Unfiltered> merged = new ArrayList<>();
+        Iterators.addAll(merged, new IteratorFromCursor(metadata, mergeCursors(us, iterations)).next());
+        return merged;
+    }
+
+    private SSTableCursor mergeCursors(List<SSTableCursor> us, boolean iterations)
+    {
+        if (iterations)
+        {
+            SSTableCursor mi = us.get(0);
+            int i;
+            for (i = 1; i + 2 <= ITERATORS; i += 2)
+                mi = new SSTableCursorMerger(ImmutableList.of(mi, us.get(i), us.get(i+1)), metadata);
+            if (i + 1 <= ITERATORS)
+                mi = new SSTableCursorMerger(ImmutableList.of(mi, us.get(i)), metadata);
+            return new SkipEmptyDataCursor(mi);
+        }
+        else
+        {
+            return new SkipEmptyDataCursor(new SSTableCursorMerger(us, metadata));
+        }
+    }
+
+    private SSTableCursor cursor(List<Unfiltered> content, DecoratedKey partitionKey, DeletionTime partitionLevelDeletion)
+    {
+        return new SSTableCursor() {
+            Type type = Type.UNINITIALIZED;
+            Iterator<Unfiltered> iterator = content.iterator();
+            DeletionTime activeRangeDeletion = DeletionTime.LIVE;
+            DeletionTime rowLevelDeletion;
+            Unfiltered current;
+            Iterator<Cell<?>> cellIterator;
+            Cell currentCell;
+
+            public Type advance()
+            {
+                if (type == Type.RANGE_TOMBSTONE)
+                    activeRangeDeletion = rowLevelDeletion;
+
+                switch (type)
+                {
+                    case UNINITIALIZED:
+                        return type = Type.PARTITION;
+                    case ROW:
+                    case SIMPLE_COLUMN:
+                        if (cellIterator.hasNext())
+                        {
+                            currentCell = cellIterator.next();
+                            return type = Type.SIMPLE_COLUMN;
+                        }
+                        // else fall through
+                    case RANGE_TOMBSTONE:
+                    case PARTITION:
+                        if (!iterator.hasNext())
+                            return type = Type.EXHAUSTED;
+
+                        current = iterator.next();
+                        if (current.isRow())
+                        {
+                            Row row = (Row) this.current;
+                            cellIterator = row.cells().iterator();
+                            rowLevelDeletion = row.deletion().time();
+                            return type = Type.ROW;
+                        }
+                        else
+                        {
+                            RangeTombstoneMarker marker = (RangeTombstoneMarker) this.current;
+                            rowLevelDeletion = marker.isOpen(false) ? marker.openDeletionTime(false) : DeletionTime.LIVE;
+                            return type = Type.RANGE_TOMBSTONE;
+                        }
+                    default:
+                        throw new AssertionError();
+                }
+            }
+
+            public Type type()
+            {
+                return type;
+            }
+
+            public DecoratedKey partitionKey()
+            {
+                return partitionKey;
+            }
+
+            public DeletionTime partitionLevelDeletion()
+            {
+                return partitionLevelDeletion;
+            }
+
+            public ClusteringPrefix clusteringKey()
+            {
+                return current.clustering();
+            }
+
+            public LivenessInfo clusteringKeyLivenessInfo()
+            {
+                return ((Row) current).primaryKeyLivenessInfo();
+            }
+
+            public DeletionTime rowLevelDeletion()
+            {
+                return rowLevelDeletion;
+            }
+
+            public DeletionTime activeRangeDeletion()
+            {
+                return activeRangeDeletion;
+            }
+
+            public DeletionTime complexColumnDeletion()
+            {
+                return null;    // we don't return complex columns
+            }
+
+            public ColumnMetadata column()
+            {
+                return cell().column;
+            }
+
+            public long bytesProcessed()
+            {
+                return 0;
+            }
+
+            public long bytesTotal()
+            {
+                return 0;
+            }
+
+            public Cell cell()
+            {
+                return currentCell;
+            }
+
+            public void close()
+            {
+                // nothing
+            }
+        };
+    }
+
     @SuppressWarnings("unused")
     private List<Unfiltered> generateSource(Random r, IntUnaryOperator timeGenerator)
     {
@@ -410,35 +610,12 @@ private String str(Clusterable curr)
             RangeTombstoneMarker marker = (RangeTombstoneMarker) curr;
             if (marker.isClose(reversed))
                 val = "[" + marker.closeDeletionTime(reversed).markedForDeleteAt() + "]" + (marker.closeIsInclusive(reversed) ? "<=" : "<") + val;
-            if (marker.isOpen(reversed)) 
+            if (marker.isOpen(reversed))
                 val = val + (marker.openIsInclusive(reversed) ? "<=" : "<") + "[" + marker.openDeletionTime(reversed).markedForDeleteAt() + "]";
         }
         return val;
     }
 
-    class Source extends AbstractUnfilteredRowIterator implements UnfilteredRowIterator
-    {
-        Iterator<Unfiltered> content;
-
-        protected Source(Iterator<Unfiltered> content)
-        {
-            super(UnfilteredRowIteratorsMergeTest.metadata,
-                  UnfilteredRowIteratorsMergeTest.partitionKey,
-                  UnfilteredRowIteratorsMergeTest.partitionLevelDeletion,
-                  UnfilteredRowIteratorsMergeTest.metadata.regularAndStaticColumns(),
-                  null,
-                  reversed,
-                  EncodingStats.NO_STATS);
-            this.content = content;
-        }
-
-        @Override
-        protected Unfiltered computeNext()
-        {
-            return content.hasNext() ? content.next() : endOfData();
-        }
-    }
-
     public void testForInput(String... inputs)
     {
         reversed = false;
@@ -447,67 +624,23 @@ public void testForInput(String... inputs)
         List<List<Unfiltered>> sources = new ArrayList<>();
         for (String input : inputs)
         {
-            List<Unfiltered> source = generator.parse(input, DEL_RANGE);
+            List<Unfiltered> source = generator.parse(input, DEL_RANGE, partitionLevelDeletions);
             generator.dumpList(source);
             generator.verifyValid(source);
             sources.add(source);
         }
 
-        List<Unfiltered> merged = merge(sources, false);
-        System.out.println("Merge to:");
+        List<Unfiltered> merged = merge(sources, false, false);
+        System.out.println("Merge through iterator to:");
+        generator.dumpList(merged);
+        verifyEquivalent(sources, merged, generator);
+        generator.verifyValid(merged);
+        System.out.println();
+        merged = merge(sources, false, true);
+        System.out.println("Merge through cursors to:");
         generator.dumpList(merged);
         verifyEquivalent(sources, merged, generator);
         generator.verifyValid(merged);
         System.out.println();
-    }
-
-    List<Unfiltered> parse(String input)
-    {
-        String[] split = input.split(" ");
-        Pattern open = Pattern.compile("(\\d+)<(=)?\\[(\\d+)\\]");
-        Pattern close = Pattern.compile("\\[(\\d+)\\]<(=)?(\\d+)");
-        Pattern row = Pattern.compile("(\\d+)(\\[(\\d+)\\])?");
-        List<Unfiltered> out = new ArrayList<>(split.length);
-        for (String s : split)
-        {
-            Matcher m = open.matcher(s);
-            if (m.matches())
-            {
-                out.add(openMarker(Integer.parseInt(m.group(1)), Long.parseLong(m.group(3)), m.group(2) != null));
-                continue;
-            }
-            m = close.matcher(s);
-            if (m.matches())
-            {
-                out.add(closeMarker(Integer.parseInt(m.group(3)), Long.parseLong(m.group(1)), m.group(2) != null));
-                continue;
-            }
-            m = row.matcher(s);
-            if (m.matches())
-            {
-                int live = m.group(3) != null ? Integer.parseInt(m.group(3)) : DEL_RANGE;
-                out.add(emptyRowAt(Integer.parseInt(m.group(1)), x -> live));
-                continue;
-            }
-            Assert.fail("Can't parse " + s);
-        }
-        return out;
-    }
-
-    private RangeTombstoneMarker openMarker(int pos, long delTime, boolean inclusive)
-    {
-        return marker(pos, delTime, true, inclusive);
-    }
-
-    private RangeTombstoneMarker closeMarker(int pos, long delTime, boolean inclusive)
-    {
-        return marker(pos, delTime, false, inclusive);
-    }
-
-    private RangeTombstoneMarker marker(int pos, long delTime, boolean isStart, boolean inclusive)
-    {
-        return new RangeTombstoneBoundMarker(BufferClusteringBound.create(ClusteringBound.boundKind(isStart, inclusive),
-                                                                    new ByteBuffer[] {clusteringFor(pos).bufferAt(0)}),
-                                             DeletionTime.build(delTime, delTime));
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
index 9a40823db983..e2f85e7e0063 100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
@@ -21,10 +21,12 @@
 import java.util.Arrays;
 import java.util.Iterator;
 
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.junit.Assert;
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.BufferDecoratedKey;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionTime;
@@ -56,6 +58,11 @@ public class UnfilteredRowIteratorsTest
         v2Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(1);
     }
 
+    @BeforeClass
+    public static void setupClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
 
     @Test
     public void concatTest()
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowsGenerator.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowsGenerator.java
index 4941d0fa875d..79ab91a7914a 100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowsGenerator.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowsGenerator.java
@@ -18,14 +18,28 @@
 package org.apache.cassandra.db.rows;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
 import java.util.function.IntUnaryOperator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.junit.Assert;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.BufferClusteringBound;
+import org.apache.cassandra.db.Clusterable;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.rows.Unfiltered.Kind;
 import org.apache.cassandra.schema.TableMetadata;
@@ -219,6 +233,28 @@ public List<Unfiltered> parse(String input, int default_liveness)
         return out;
     }
 
+    /**
+     * As above, but also parses "Dxx|" prefix which specifies deletion time to be put in the specified deletion times map.
+     * @param input
+     * @param defaultLiveness
+     * @param deletionTimes
+     * @return
+     */
+    public List<Unfiltered> parse(String input, int defaultLiveness, Map<List<Unfiltered>, DeletionTime> deletionTimes)
+    {
+        Matcher m = Pattern.compile("D(\\d+)\\|").matcher(input);
+        if (m.lookingAt())
+        {
+            long del = Long.parseLong(m.group(1));
+            input = input.substring(m.end());
+            List<Unfiltered> list = parse(input, defaultLiveness);
+            deletionTimes.put(list, DeletionTime.build(del, del));
+            return list;
+        }
+        else
+            return parse(input, defaultLiveness);
+    }
+
     static Row emptyRowAt(int pos, IntUnaryOperator timeGenerator)
     {
         final Clustering<?> clustering = clusteringFor(pos);
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredSerializerTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredSerializerTest.java
index 835e613b9e0a..c2d42f9c9ce9 100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredSerializerTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredSerializerTest.java
@@ -21,7 +21,6 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Random;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 
@@ -31,13 +30,12 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.db.marshal.IntegerType;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
 
 import static org.assertj.core.api.Assertions.assertThatIOException;
 import static org.junit.Assert.assertEquals;
@@ -59,20 +57,41 @@ public static void beforeClass()
     }
 
     @Test
-    public void testRowSerDe() throws IOException
+    public void testSmallRowSerDe() throws IOException
     {
-        // test serialization and deserialization of a row body
-        testRowBodySerDe(10, Function.identity());
+        // test serialization and deserialization of a row body when row size is smaller than the preload threshold (1kb)
+        // so it is not preloaded into the buffer cache
+        testRowBodySerDe(100, Function.identity());
     }
 
     @Test
-    public void testRowSerDeWithCorruption() throws IOException
+    public void testLargeRowSerDe() throws IOException
     {
-        // test serialization and deserialization of a row body when the row is corrupted in the way that the actual
-        // row content is larger than the row size serialized in the preamble
-        ByteBuffer largeRow = getSerializedRow(50);
-        assertThatIOException().isThrownBy(() -> testRowBodySerDe(10, buf -> replaceRowContent(buf, largeRow)))
-                               .withMessageMatching("EOF after \\d+ bytes out of 50");
+        // test serialization and deserialization of a row body when row size is larger than the preload threshold (1kb)
+        // so it is preloaded into the buffer cache
+        testRowBodySerDe(1000, Function.identity());
+    }
+
+    @Test
+    public void testSmallRowSerDeWithCorruption() throws IOException
+    {
+        // test serialization and deserialization of a row body when row size is smaller than the preload threshold (1kb)
+        // so it is not preloaded into the buffer cache; also, the row is corrupted in the way that the actual row content
+        // is larger than the row size serialized in the preamble
+        ByteBuffer largeRow = getSerializedRow(10000);
+        assertThatIOException().isThrownBy(() ->
+                testRowBodySerDe(100, buf -> replaceRowContent(buf, largeRow))).withMessageMatching("EOF after \\d+ bytes out of 10000");
+    }
+
+    @Test
+    public void testLargeRowSerDeWithCorruption() throws IOException
+    {
+        // test serialization and deserialization of a row body when row size is larger than the preload threshold (1kb)
+        // so it is preloaded into the buffer cache; also, the row is corrupted in the way that the actual row content
+        // is larger than the row size serialized in the preamble
+        ByteBuffer largeRow = getSerializedRow(10000);
+        assertThatIOException().isThrownBy(() ->
+                testRowBodySerDe(1000, buf -> replaceRowContent(buf, largeRow))).withMessageMatching("EOF after \\d+ bytes out of 10000");
     }
 
     public static void testRowBodySerDe(int cellSize, Function<ByteBuffer, ByteBuffer> transform) throws IOException
@@ -85,19 +104,19 @@ public static void testRowBodySerDe(int cellSize, Function<ByteBuffer, ByteBuffe
 
         Row.Builder builder = BTreeRow.sortedBuilder();
         builder.newRow(Clustering.EMPTY);
-        builder.addCell(BufferCell.live(md.regularColumns().getSimple(0), TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()), data1.duplicate()));
-        builder.addCell(BufferCell.live(md.regularColumns().getSimple(1), TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()), data2.duplicate()));
+        builder.addCell(BufferCell.live(md.regularColumns().getSimple(0), FBUtilities.timestampMicros(), data1.duplicate()));
+        builder.addCell(BufferCell.live(md.regularColumns().getSimple(1), FBUtilities.timestampMicros(), data2.duplicate()));
         Row writtenRow = builder.build();
 
         try (DataOutputBuffer out = new DataOutputBuffer())
         {
-            UnfilteredSerializer.serializer.serialize(writtenRow, new SerializationHelper(SerializationHeader.makeWithoutStats(md)), out, 0, MessagingService.current_version);
-            out.flush();
-            try (DataInputBuffer in = new DataInputBuffer(transform.apply(out.asNewBuffer()), false))
+            SerializationHeader header = new SerializationHeader(true, md, md.regularAndStaticColumns(), EncodingStats.NO_STATS);
+            UnfilteredSerializer.serializer.serialize(writtenRow, new SerializationHelper(header), out, 0, 0);
+            try (DataInputBuffer in = new DataInputBuffer(transform.apply(out.buffer()), false))
             {
                 builder = BTreeRow.sortedBuilder();
-                DeserializationHelper helper = new DeserializationHelper(md, MessagingService.current_version, DeserializationHelper.Flag.LOCAL, ColumnFilter.all(md));
-                Unfiltered readRow = UnfilteredSerializer.serializer.deserialize(in, SerializationHeader.makeWithoutStats(md), helper, builder);
+                DeserializationHelper helper = new DeserializationHelper(md, 0, DeserializationHelper.Flag.LOCAL);
+                Unfiltered readRow = UnfilteredSerializer.serializer.deserialize(in, header, helper, builder);
                 assertEquals(writtenRow, readRow);
             }
         }
@@ -125,7 +144,7 @@ private static ByteBuffer replaceRowContent(ByteBuffer original, ByteBuffer repl
             replIn.readUnsignedVInt(); // skip row size
             out.write(replacement);
             out.close();
-            return out.asNewBuffer();
+            return out.buffer();
         }
         catch (IOException ex)
         {
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java
index 2921b5ae1749..369c298bec35 100644
--- a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java
@@ -100,7 +100,7 @@ public void validateFullyContainedIn_SingleContiguousRange_Succeeds()
     }
 
     @Test
-    public void validateFullyContainedIn_PartialOverlap_Fails()
+    public void validateFullyContainedIn_PartialOverlap_Fails() throws IOException
     {
         List<Range<Token>> requestedRanges = Arrays.asList(new Range<>(store.getPartitioner().getMinimumToken(), getTokenAtIndex(2)));
 
@@ -113,7 +113,7 @@ public void validateFullyContainedIn_PartialOverlap_Fails()
     }
 
     @Test
-    public void validateFullyContainedIn_SplitRange_Succeeds()
+    public void validateFullyContainedIn_SplitRange_Succeeds() throws IOException
     {
         List<Range<Token>> requestedRanges = Arrays.asList(new Range<>(store.getPartitioner().getMinimumToken(), getTokenAtIndex(4)),
                                                          new Range<>(getTokenAtIndex(2), getTokenAtIndex(6)),
@@ -128,7 +128,7 @@ public void validateFullyContainedIn_SplitRange_Succeeds()
         assertTrue(cof.contained(sections, sstable));
     }
 
-    private DecoratedKey getKeyAtIndex(int i)
+    private DecoratedKey getKeyAtIndex(int i) throws IOException
     {
         int count = 0;
         DecoratedKey key;
@@ -148,7 +148,7 @@ private DecoratedKey getKeyAtIndex(int i)
         return key;
     }
 
-    private Token getTokenAtIndex(int i)
+    private Token getTokenAtIndex(int i) throws IOException
     {
         return getKeyAtIndex(i).getToken();
     }
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
index 41dcdcb66b88..1230b3044819 100644
--- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
@@ -146,7 +146,7 @@ private static Set<SSTableReader> sstablesFromStreams(Collection<OutgoingStream>
         Set<SSTableReader> sstables = new HashSet<>();
         for (OutgoingStream stream: streams)
         {
-            Ref<SSTableReader> ref = CassandraOutgoingFile.fromStream(stream).getRef();
+            Ref<? extends SSTableReader> ref = CassandraOutgoingFile.fromStream(stream).getRef();
             sstables.add(ref.get());
             ref.release();
         }
diff --git a/test/unit/org/apache/cassandra/db/tries/CellReuseTest.java b/test/unit/org/apache/cassandra/db/tries/CellReuseTest.java
new file mode 100644
index 000000000000..dff8ce005a13
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/CellReuseTest.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Random;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import com.google.common.collect.Streams;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.asString;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.assertMapEquals;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.byteComparableVersion;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.generateKeys;
+
+public class CellReuseTest
+{
+    static Predicate<InMemoryTrie.NodeFeatures<Object>> FORCE_COPY_PARTITION = features -> {
+        var c = features.content();
+        if (c != null && c instanceof Boolean)
+            return (Boolean) c;
+        else
+            return false;
+    };
+
+    static Predicate<InMemoryTrie.NodeFeatures<Object>> NO_ATOMICITY = features -> false;
+
+    private static final int COUNT = 10000;
+    Random rand = new Random(2);
+
+    @Test
+    public void testCellReusePartitionCopying() throws Exception
+    {
+        testCellReuse(FORCE_COPY_PARTITION);
+    }
+
+    @Test
+    public void testCellReuseNoCopying() throws Exception
+    {
+        testCellReuse(NO_ATOMICITY);
+    }
+
+    public void testCellReuse(Predicate<InMemoryTrie.NodeFeatures<Object>> forceCopyPredicate) throws Exception
+    {
+        ByteComparable[] src = generateKeys(rand, COUNT);
+        InMemoryTrie<Object> trieLong = makeInMemoryTrie(src, opOrder -> InMemoryTrie.longLived(byteComparableVersion, BufferType.ON_HEAP, opOrder),
+                                                             forceCopyPredicate);
+
+        // dump some information first
+        System.out.println(String.format(" LongLived ON_HEAP sizes %10s %10s count %d",
+                                         FBUtilities.prettyPrintMemory(trieLong.usedSizeOnHeap()),
+                                         FBUtilities.prettyPrintMemory(trieLong.usedSizeOffHeap()),
+                                         Streams.stream(trieLong.values()).count()));
+
+        Pair<BitSet, BitSet> longReachable = reachableCells(trieLong);
+        BitSet reachable = longReachable.left;
+        int lrcells = reachable.cardinality();
+        int lrobjs = longReachable.right.cardinality();
+        System.out.println(String.format(" LongLived reachable cells %,d objs %,d cell space %,d obj space %,d",
+                                         lrcells,
+                                         lrobjs,
+                                         lrcells * 32,
+                                         lrobjs * 4
+        ));
+
+        IntArrayList availableList = ((MemoryAllocationStrategy.OpOrderReuseStrategy) trieLong.cellAllocator).indexesInPipeline();
+        BitSet available = new BitSet(reachable.size());
+        for (int v : availableList)
+            available.set(v >> 5);
+
+        // Check no reachable cell is marked for reuse
+        BitSet intersection = new BitSet(available.size());
+        intersection.or(available);
+        intersection.and(reachable);
+        assertCellSetEmpty(intersection, trieLong, " reachable cells marked as available");
+
+        // Check all unreachable cells are marked for reuse
+        BitSet unreachable = new BitSet(reachable.size());
+        unreachable.or(reachable);
+        unreachable.flip(0, trieLong.getAllocatedPos() >> 5);
+        unreachable.andNot(available);
+        assertCellSetEmpty(unreachable, trieLong, " unreachable cells not marked as available");
+    }
+
+    static class TestException extends RuntimeException
+    {
+    }
+
+    @Test
+    public void testAbortedMutation() throws Exception
+    {
+        ByteComparable[] src = generateKeys(rand, COUNT);
+        OpOrder order = new OpOrder();
+        InMemoryTrie<Object> trie = InMemoryTrie.longLived(byteComparableVersion, order);
+        InMemoryTrie<Object> check = InMemoryTrie.shortLived(byteComparableVersion);
+        int step = Math.min(100, COUNT / 100);
+        int throwStep = (COUNT + 10) / 5;   // do 4 throwing inserts
+        int nextThrow = throwStep;
+
+        for (int i = 0; i < src.length; i += step)
+            try (OpOrder.Group g = order.start())
+            {
+                int last = Math.min(i + step, src.length);
+                addToInMemoryTrie(Arrays.copyOfRange(src, i, last), trie, FORCE_COPY_PARTITION);
+                addToInMemoryTrie(Arrays.copyOfRange(src, i, last), check, NO_ATOMICITY);
+                if (i >= nextThrow)
+                {
+                    nextThrow += throwStep;
+                    try
+                    {
+                        addThrowingEntry(src[rand.nextBoolean() ? last : i],    // try both inserting new value and
+                                                                                // overwriting existing
+                                         trie, FORCE_COPY_PARTITION);
+                        ++i;
+                        Assert.fail("Expected failed mutation");
+                    }
+                    catch (TestException e)
+                    {
+                        // expected
+                    }
+                }
+            }
+
+        assertMapEquals(trie.filteredEntrySet(ByteBuffer.class).iterator(),
+                        check.filteredEntrySet(ByteBuffer.class).iterator());
+    }
+
+    private void assertCellSetEmpty(BitSet set, InMemoryTrie<?> trie, String message)
+    {
+        if (set.isEmpty())
+            return;
+
+        for (int i = set.nextSetBit(0); i >= 0; i = set.nextSetBit(i + 1))
+        {
+            System.out.println(String.format("Cell at %d: %08x %08x %08x %08x %08x %08x %08x %08x",
+                                             (i << 5),
+                                             trie.getIntVolatile((i << 5) + 0),
+                                             trie.getIntVolatile((i << 5) + 4),
+                                             trie.getIntVolatile((i << 5) + 8),
+                                             trie.getIntVolatile((i << 5) + 12),
+                                             trie.getIntVolatile((i << 5) + 16),
+                                             trie.getIntVolatile((i << 5) + 20),
+                                             trie.getIntVolatile((i << 5) + 24),
+                                             trie.getIntVolatile((i << 5) + 28)
+            ));
+
+        }
+        Assert.fail(set.cardinality() + message);
+    }
+
+    private Pair<BitSet, BitSet> reachableCells(InMemoryTrie<?> trie)
+    {
+//        System.out.println(trie.dump());
+        BitSet set = new BitSet();
+        BitSet objs = new BitSet();
+        mark(trie, trie.root, set, objs);
+        return Pair.create(set, objs);
+    }
+
+    private void mark(InMemoryTrie<?> trie, int node, BitSet set, BitSet objs)
+    {
+        set.set(node >> 5);
+//        System.out.println(trie.dumpNode(node));
+        switch (trie.offset(node))
+        {
+            case InMemoryTrie.SPLIT_OFFSET:
+                for (int i = 0; i < InMemoryTrie.SPLIT_START_LEVEL_LIMIT; ++i)
+                {
+                    int mid = trie.getSplitCellPointer(node, i, InMemoryTrie.SPLIT_START_LEVEL_LIMIT);
+                    if (mid != InMemoryTrie.NONE)
+                    {
+//                        System.out.println(trie.dumpNode(mid));
+                        set.set(mid >> 5);
+                        for (int j = 0; j < InMemoryTrie.SPLIT_OTHER_LEVEL_LIMIT; ++j)
+                        {
+                            int tail = trie.getSplitCellPointer(mid, j, InMemoryTrie.SPLIT_OTHER_LEVEL_LIMIT);
+                            if (tail != InMemoryTrie.NONE)
+                            {
+//                                System.out.println(trie.dumpNode(tail));
+                                set.set(tail >> 5);
+                                for (int k = 0; k < InMemoryTrie.SPLIT_OTHER_LEVEL_LIMIT; ++k)
+                                    markChild(trie, trie.getSplitCellPointer(tail, k, InMemoryTrie.SPLIT_OTHER_LEVEL_LIMIT), set, objs);
+                            }
+                        }
+                    }
+                }
+                break;
+            case InMemoryTrie.SPARSE_OFFSET:
+                for (int i = 0; i < InMemoryTrie.SPARSE_CHILD_COUNT; ++i)
+                    markChild(trie, trie.getIntVolatile(node + InMemoryTrie.SPARSE_CHILDREN_OFFSET + i * 4), set, objs);
+                break;
+            case InMemoryTrie.PREFIX_OFFSET:
+                int content = trie.getIntVolatile(node + InMemoryTrie.PREFIX_CONTENT_OFFSET);
+                if (content < 0)
+                    objs.set(~content);
+                else
+                    markChild(trie, content, set, objs);
+
+                markChild(trie, trie.followContentTransition(node), set, objs);
+                break;
+            default:
+                assert trie.offset(node) <= InMemoryTrie.CHAIN_MAX_OFFSET && trie.offset(node) >= InMemoryTrie.CHAIN_MIN_OFFSET;
+                markChild(trie, trie.getIntVolatile((node & -32) + InMemoryTrie.LAST_POINTER_OFFSET), set, objs);
+                break;
+        }
+    }
+
+    private void markChild(InMemoryTrie<?> trie, int child, BitSet set, BitSet objs)
+    {
+        if (child == InMemoryTrie.NONE)
+            return;
+        if (child > 0)
+            mark(trie, child, set, objs);
+        else
+            objs.set(~child);
+    }
+
+    static InMemoryTrie<Object> makeInMemoryTrie(ByteComparable[] src,
+                                                 Function<OpOrder, InMemoryTrie<Object>> creator,
+                                                 Predicate<InMemoryTrie.NodeFeatures<Object>> forceCopyPredicate) throws TrieSpaceExhaustedException
+    {
+        OpOrder order = new OpOrder();
+        InMemoryTrie<Object> trie = creator.apply(order);
+        int step = Math.max(Math.min(100, COUNT / 100), 1);
+        for (int i = 0; i < src.length; i += step)
+            try (OpOrder.Group g = order.start())
+            {
+                addToInMemoryTrie(Arrays.copyOfRange(src, i, i + step), trie, forceCopyPredicate);
+            }
+
+        return trie;
+    }
+
+    static void addToInMemoryTrie(ByteComparable[] src,
+                                  InMemoryTrie<Object> trie,
+                                  Predicate<InMemoryTrie.NodeFeatures<Object>> forceCopyPredicate) throws TrieSpaceExhaustedException
+    {
+        for (ByteComparable b : src)
+        {
+            // Note: Because we don't ensure order when calling resolve, just use a hash of the key as payload
+            // (so that all sources have the same value).
+            int payload = asString(b).hashCode();
+            ByteBuffer v = ByteBufferUtil.bytes(payload);
+            Trie<Object> update = Trie.singleton(b, byteComparableVersion, v);
+            update = InMemoryTrieThreadedTest.withRootMetadata(update, Boolean.TRUE);
+            update = update.prefixedBy(source("prefix"));
+            applyUpdating(trie, update, forceCopyPredicate);
+        }
+    }
+
+    static ByteComparable source(String key)
+    {
+        return ByteComparable.preencoded(byteComparableVersion, key.getBytes(StandardCharsets.UTF_8));
+    }
+
+    static void addThrowingEntry(ByteComparable b,
+                                 InMemoryTrie<Object> trie,
+                                 Predicate<InMemoryTrie.NodeFeatures<Object>> forceCopyPredicate) throws TrieSpaceExhaustedException
+    {
+        int payload = asString(b).hashCode();
+        ByteBuffer v = ByteBufferUtil.bytes(payload);
+        Trie<Object> update = Trie.singleton(b, byteComparableVersion, v);
+
+        // Create an update with two metadata entries, so that the lower is already a copied node.
+        // Abort processing on the lower metadata, where the new branch is not attached yet (so as not to affect the
+        // contents).
+        update = InMemoryTrieThreadedTest.withRootMetadata(update, Boolean.FALSE);
+        update = update.prefixedBy(source("fix"));
+        update = InMemoryTrieThreadedTest.withRootMetadata(update, Boolean.TRUE);
+        update = update.prefixedBy(source("pre"));
+
+        trie.apply(update,
+                   (existing, upd) ->
+                   {
+                       if (upd instanceof Boolean)
+                       {
+                           if (upd != null && !((Boolean) upd))
+                               throw new TestException();
+                           return null;
+                       }
+                       else
+                           return upd;
+                   },
+                   forceCopyPredicate);
+    }
+
+    public static <T> void applyUpdating(InMemoryTrie<T> trie,
+                                         Trie<T> mutation,
+                                         final Predicate<InMemoryTrie.NodeFeatures<T>> needsForcedCopy)
+    throws TrieSpaceExhaustedException
+    {
+        trie.apply(mutation, (x, y) -> y, needsForcedCopy);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/CollectionMergeTrieTest.java b/test/unit/org/apache/cassandra/db/tries/CollectionMergeTrieTest.java
index 94903a74d7e7..df6f28c6e19a 100644
--- a/test/unit/org/apache/cassandra/db/tries/CollectionMergeTrieTest.java
+++ b/test/unit/org/apache/cassandra/db/tries/CollectionMergeTrieTest.java
@@ -20,7 +20,6 @@
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
 import java.util.SortedMap;
@@ -45,8 +44,8 @@ public void testDirect()
     {
         ByteComparable[] src1 = generateKeys(rand, COUNT);
         ByteComparable[] src2 = generateKeys(rand, COUNT);
-        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
-        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>(forwardComparator);
+        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>(forwardComparator);
 
         InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
         InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
@@ -63,8 +62,8 @@ public void testWithDuplicates()
     {
         ByteComparable[] src1 = generateKeys(rand, COUNT);
         ByteComparable[] src2 = generateKeys(rand, COUNT);
-        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
-        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>(forwardComparator);
+        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>(forwardComparator);
 
         InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
         InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
@@ -82,12 +81,12 @@ public void testWithDuplicates()
     public void testDistinct()
     {
         ByteComparable[] src1 = generateKeys(rand, COUNT);
-        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>(forwardComparator);
         InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
 
         ByteComparable[] src2 = generateKeys(rand, COUNT);
         src2 = removeDuplicates(src2, content1);
-        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>(forwardComparator);
         InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
 
         content1.putAll(content2);
@@ -129,12 +128,6 @@ public void testMerge5()
         testMultiple(5, COUNT / 10);
     }
 
-    @Test
-    public void testMerge0()
-    {
-        testMultiple(0, COUNT / 10);
-    }
-
     public void testMultiple(int mergeCount, int count)
     {
         testMultipleDistinct(mergeCount, count);
@@ -144,7 +137,7 @@ public void testMultiple(int mergeCount, int count)
     public void testMultipleDistinct(int mergeCount, int count)
     {
         List<Trie<ByteBuffer>> tries = new ArrayList<>(mergeCount);
-        SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>(forwardComparator);
 
         for (int i = 0; i < mergeCount; ++i)
         {
@@ -160,22 +153,12 @@ public void testMultipleDistinct(int mergeCount, int count)
     public void testMultipleWithDuplicates(int mergeCount, int count)
     {
         List<Trie<ByteBuffer>> tries = new ArrayList<>(mergeCount);
-        SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
-        ByteComparable[][] keys = new ByteComparable[count][];
-        for (int i = 0; i < mergeCount; ++i)
-            keys[i] = generateKeys(rand, count);
+        SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>(forwardComparator);
 
         for (int i = 0; i < mergeCount; ++i)
         {
-            ByteComparable[] src = Arrays.copyOf(keys[i], count + count / 10);
-            // add duplicates from other tries
-            if (mergeCount > 1)
-            {
-                for (int j = count; j < src.length; ++j)
-                    src[j] = keys[randomButNot(rand, mergeCount, i)][rand.nextInt(count)];
-            }
-
-            Trie<ByteBuffer> trie = makeInMemoryTrie(keys[i], content, true);
+            ByteComparable[] src = generateKeys(rand, count);
+            Trie<ByteBuffer> trie = makeInMemoryTrie(src, content, true);
             tries.add(trie);
         }
 
diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTriePutTest.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTriePutTest.java
index 51b23d83c7c8..e5f0044f3612 100644
--- a/test/unit/org/apache/cassandra/db/tries/InMemoryTriePutTest.java
+++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTriePutTest.java
@@ -25,7 +25,6 @@
 import org.junit.Ignore;
 import org.junit.Test;
 
-import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 import static org.junit.Assert.fail;
@@ -39,9 +38,9 @@ boolean usePut()
     }
 
     @Test
-    public void testLongKey_StackOverflow() throws InMemoryTrie.SpaceExhaustedException
+    public void testLongKey_StackOverflow() throws TrieSpaceExhaustedException
     {
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.ON_HEAP);
+        InMemoryTrie<String> trie = strategy.create();
         Random rand = new Random(1);
         byte[] key = new byte[40960];
         rand.nextBytes(key);
@@ -49,7 +48,7 @@ public void testLongKey_StackOverflow() throws InMemoryTrie.SpaceExhaustedExcept
 
         try
         {
-            trie.putRecursive(ByteComparable.fixedLength(buf), "value", (x, y) -> y);
+            trie.putRecursive(ByteComparable.preencoded(byteComparableVersion, buf), "value", (x, y) -> y);
             Assert.fail("StackOverflowError expected with a recursive put for very long keys!");
         }
         catch (StackOverflowError soe)
@@ -57,7 +56,7 @@ public void testLongKey_StackOverflow() throws InMemoryTrie.SpaceExhaustedExcept
             // Expected.
         }
         // Using non-recursive put should work.
-        putSimpleResolve(trie, ByteComparable.fixedLength(buf), "value", (x, y) -> y, false);
+        putSimpleResolve(trie, ByteComparable.preencoded(byteComparableVersion, buf), "value", (x, y) -> y, false);
     }
 
     // This tests that trie space allocation works correctly close to the 2G limit. It is normally disabled because
@@ -65,9 +64,9 @@ public void testLongKey_StackOverflow() throws InMemoryTrie.SpaceExhaustedExcept
     // InMemoryTrie.allocateBlock is modified.
     @Ignore
     @Test
-    public void testOver1GSize() throws InMemoryTrie.SpaceExhaustedException
+    public void testOver1GSize() throws TrieSpaceExhaustedException
     {
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.ON_HEAP);
+        InMemoryTrie<String> trie = strategy.create();
         trie.advanceAllocatedPos(0x20000000);
         String t1 = "test1";
         String t2 = "testing2";
@@ -95,7 +94,7 @@ public void testOver1GSize() throws InMemoryTrie.SpaceExhaustedException
             trie.putRecursive(ByteComparable.of(t3), t3, (x, y) -> y);  // should put it over the edge
             fail("InMemoryTrie.SpaceExhaustedError was expected");
         }
-        catch (InMemoryTrie.SpaceExhaustedException e)
+        catch (TrieSpaceExhaustedException e)
         {
             // expected
         }
@@ -110,7 +109,7 @@ public void testOver1GSize() throws InMemoryTrie.SpaceExhaustedException
             trie.advanceAllocatedPos(Integer.MAX_VALUE);
             fail("InMemoryTrie.SpaceExhaustedError was expected");
         }
-        catch (InMemoryTrie.SpaceExhaustedException e)
+        catch (TrieSpaceExhaustedException e)
         {
             // expected
         }
diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java
index d1c17114231c..adfad17c4ab2 100644
--- a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java
+++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.google.common.collect.HashMultiset;
@@ -29,20 +30,27 @@
 import com.google.common.collect.Multiset;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
 import static org.junit.Assert.assertEquals;
 
+@RunWith(Parameterized.class)
 public abstract class InMemoryTrieTestBase
 {
     // Set this to true (in combination with smaller count) to dump the tries while debugging a problem.
     // Do not commit the code with VERBOSE = true.
     private static final boolean VERBOSE = false;
 
+    // Set to true by some tests that need prefix-free keys.
+    static boolean prefixFree = false;
+
     private static final int COUNT = 100000;
     private static final int KEY_CHOICE = 25;
     private static final int MIN_LENGTH = 10;
@@ -50,21 +58,76 @@ public abstract class InMemoryTrieTestBase
 
     Random rand = new Random();
 
-    static final ByteComparable.Version VERSION = InMemoryTrie.BYTE_COMPARABLE_VERSION;
-
     abstract boolean usePut();
 
+    static ByteComparable invert(ByteComparable b)
+    {
+        return version -> invert(b.asComparableBytes(version));
+    }
+
+    static ByteSource invert(ByteSource src)
+    {
+        return () ->
+        {
+            int v = src.next();
+            if (v == ByteSource.END_OF_STREAM)
+                return v;
+            return v ^ 0xFF;
+        };
+    }
+
     @Test
     public void testSingle()
     {
         ByteComparable e = ByteComparable.of("test");
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+        InMemoryTrie<String> trie = strategy.create();
         putSimpleResolve(trie, e, "test", (x, y) -> y);
         System.out.println("Trie " + trie.dump());
         assertEquals("test", trie.get(e));
         assertEquals(null, trie.get(ByteComparable.of("teste")));
     }
 
+    public enum ReuseStrategy
+    {
+        SHORT_LIVED
+        {
+            <T> InMemoryTrie<T> create()
+            {
+                return InMemoryTrie.shortLived(byteComparableVersion);
+            }
+        },
+        LONG_LIVED
+        {
+            <T> InMemoryTrie<T> create()
+            {
+                return InMemoryTrie.longLived(byteComparableVersion, BufferType.OFF_HEAP, null);
+            }
+        };
+
+        abstract <T> InMemoryTrie<T> create();
+    }
+
+    @Parameterized.Parameters(name="{0} version {1}")
+    public static List<Object[]> generateData()
+    {
+        var list = new ArrayList<Object[]>();
+        for (var s : ReuseStrategy.values())
+            for (var v : ByteComparable.Version.values())
+                list.add(new Object[] {s, v});
+        return list;
+    }
+
+    @Parameterized.Parameter(0)
+    public static ReuseStrategy strategy = ReuseStrategy.LONG_LIVED;
+
+    @Parameterized.Parameter(1)
+    public static ByteComparable.Version byteComparableVersion = ByteComparable.Version.OSS50;
+
+    public static Comparator<ByteComparable> forwardComparator =
+        (bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, byteComparableVersion);
+    public static Comparator<ByteComparable> reverseComparator =
+        (bytes1, bytes2) -> ByteComparable.compare(invert(bytes1), invert(bytes2), byteComparableVersion);
+
     @Test
     public void testSplitMulti()
     {
@@ -86,10 +149,10 @@ public void testSparse00bug()
         "40bdd47ec043641f2b403131323400",
         "40bd00bf5ae8cf9d1d403133323800",
         };
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+        InMemoryTrie<String> trie = strategy.create();
         for (String test : tests)
         {
-            ByteComparable e = ByteComparable.fixedLength(ByteBufferUtil.hexToBytes(test));
+            ByteComparable e = ByteComparable.preencoded(byteComparableVersion, ByteBufferUtil.hexToBytes(test));
             System.out.println("Adding " + asString(e) + ": " + test);
             putSimpleResolve(trie, e, test, (x, y) -> y);
         }
@@ -97,7 +160,7 @@ public void testSparse00bug()
         System.out.println(trie.dump());
 
         for (String test : tests)
-            assertEquals(test, trie.get(ByteComparable.fixedLength(ByteBufferUtil.hexToBytes(test))));
+            assertEquals(test, trie.get(ByteComparable.preencoded(byteComparableVersion, ByteBufferUtil.hexToBytes(test))));
 
         Arrays.sort(tests);
 
@@ -116,7 +179,7 @@ public void testUpdateContent()
     {
         String[] tests = new String[] {"testing", "tests", "trials", "trial", "testing", "trial", "trial"};
         String[] values = new String[] {"testing", "tests", "trials", "trial", "t2", "x2", "y2"};
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+        InMemoryTrie<String> trie = strategy.create();
         for (int i = 0; i < tests.length; ++i)
         {
             String test = tests[i];
@@ -146,12 +209,12 @@ static class SpecStackEntry
         Object content;
         SpecStackEntry parent;
 
-        public SpecStackEntry(Object[] spec, Object content, SpecStackEntry parent)
+        public SpecStackEntry(Object[] spec, Object content, SpecStackEntry parent, Direction direction)
         {
             this.children = spec;
             this.content = content;
             this.parent = parent;
-            this.curChild = -1;
+            this.curChild = direction.select(-1, spec.length);
         }
     }
 
@@ -159,17 +222,19 @@ public static class CursorFromSpec implements Trie.Cursor<ByteBuffer>
     {
         SpecStackEntry stack;
         int depth;
+        Direction direction;
 
-        CursorFromSpec(Object[] spec)
+        CursorFromSpec(Object[] spec, Direction direction)
         {
-            stack = new SpecStackEntry(spec, null, null);
+            this.direction = direction;
+            stack = new SpecStackEntry(spec, null, null, direction);
             depth = 0;
         }
 
         public int advance()
         {
             SpecStackEntry current = stack;
-            while (current != null && ++current.curChild >= current.children.length)
+            while (current != null && !direction.inLoop(current.curChild += direction.increase, 0, current.children.length - 1))
             {
                 current = current.parent;
                 --depth;
@@ -182,37 +247,31 @@ public int advance()
 
             Object child = current.children[current.curChild];
             if (child instanceof Object[])
-                stack = new SpecStackEntry((Object[]) child, null, current);
+                stack = new SpecStackEntry((Object[]) child, null, current, direction);
             else
-                stack = new SpecStackEntry(new Object[0], child, current);
+                stack = new SpecStackEntry(new Object[0], child, current, direction);
 
             return ++depth;
         }
 
-        public int advanceMultiple()
+        public int skipTo(int skipDepth, int skipTransition)
         {
-            if (++stack.curChild >= stack.children.length)
-                return skipChildren();
+            assert skipDepth <= depth + 1 : "skipTo descends more than one level";
 
-            Object child = stack.children[stack.curChild];
-            while (child instanceof Object[])
+            while (skipDepth < depth)
             {
-                stack = new SpecStackEntry((Object[]) child, null, stack);
-                ++depth;
-                if (stack.children.length == 0)
-                    return depth;
-                child = stack.children[0];
+                --depth;
+                stack = stack.parent;
             }
-            stack = new SpecStackEntry(new Object[0], child, stack);
-
-
-            return ++depth;
-        }
-
-        public int skipChildren()
-        {
-            --depth;
-            stack = stack.parent;
+            int index = skipTransition - 0x30;
+            assert direction.gt(index, stack.curChild) : "Backwards skipTo";
+            if (direction.gt(index, direction.select(stack.children.length - 1, 0)))
+            {
+                --depth;
+                stack = stack.parent;
+                return advance();
+            }
+            stack.curChild = index - direction.increase;
             return advance();
         }
 
@@ -231,6 +290,24 @@ public int incomingTransition()
             SpecStackEntry parent = stack.parent;
             return parent != null ? parent.curChild + 0x30 : -1;
         }
+
+        @Override
+        public Direction direction()
+        {
+            return direction;
+        }
+
+        @Override
+        public ByteComparable.Version byteComparableVersion()
+        {
+            return byteComparableVersion;
+        }
+
+        @Override
+        public Trie<ByteBuffer> tailTrie()
+        {
+            throw new UnsupportedOperationException("tailTrie on test cursor");
+        }
     }
 
     static Trie<ByteBuffer> specifiedTrie(Object[] nodeDef)
@@ -238,9 +315,9 @@ static Trie<ByteBuffer> specifiedTrie(Object[] nodeDef)
         return new Trie<ByteBuffer>()
         {
             @Override
-            protected Cursor<ByteBuffer> cursor()
+            protected Cursor<ByteBuffer> cursor(Direction direction)
             {
-                return new CursorFromSpec(nodeDef);
+                return new CursorFromSpec(nodeDef, direction);
             }
         };
     }
@@ -274,7 +351,7 @@ public void testEntriesNullChildBug()
                                            ByteBufferUtil.bytes(6) // 6
                                    };
 
-        SortedMap<ByteComparable, ByteBuffer> expected = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> expected = new TreeMap<>(forwardComparator);
         expected.put(comparable("00"), ByteBufferUtil.bytes(1));
         expected.put(comparable("01"), ByteBufferUtil.bytes(2));
         expected.put(comparable("2"), ByteBufferUtil.bytes(3));
@@ -290,24 +367,24 @@ public void testEntriesNullChildBug()
     static ByteComparable comparable(String s)
     {
         ByteBuffer b = ByteBufferUtil.bytes(s);
-        return ByteComparable.fixedLength(b);
+        return ByteComparable.preencoded(byteComparableVersion, b);
     }
 
     @Test
     public void testDirect()
     {
         ByteComparable[] src = generateKeys(rand, COUNT);
-        SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>(forwardComparator);
         InMemoryTrie<ByteBuffer> trie = makeInMemoryTrie(src, content, usePut());
         int keysize = Arrays.stream(src)
-                            .mapToInt(src1 -> ByteComparable.length(src1, VERSION))
+                            .mapToInt(src1 -> ByteComparable.length(src1, byteComparableVersion))
                             .sum();
         long ts = ObjectSizes.measureDeep(content);
         long onh = ObjectSizes.measureDeep(trie.contentArrays);
         System.out.format("Trie size on heap %,d off heap %,d measured %,d keys %,d treemap %,d\n",
-                          trie.sizeOnHeap(), trie.sizeOffHeap(), onh, keysize, ts);
+                          trie.usedSizeOnHeap(), trie.usedSizeOffHeap(), onh, keysize, ts);
         System.out.format("per entry on heap %.2f off heap %.2f measured %.2f keys %.2f treemap %.2f\n",
-                          trie.sizeOnHeap() * 1.0 / COUNT, trie.sizeOffHeap() * 1.0 / COUNT, onh * 1.0 / COUNT, keysize * 1.0 / COUNT, ts * 1.0 / COUNT);
+                          trie.usedSizeOnHeap() * 1.0 / COUNT, trie.usedSizeOffHeap() * 1.0 / COUNT, onh * 1.0 / COUNT, keysize * 1.0 / COUNT, ts * 1.0 / COUNT);
         if (VERBOSE)
             System.out.println("Trie " + trie.dump(ByteBufferUtil::bytesToHex));
 
@@ -379,7 +456,7 @@ private void testEntries(String[] tests)
     {
         for (Function<String, ByteComparable> mapping :
                 ImmutableList.<Function<String, ByteComparable>>of(ByteComparable::of,
-                                                                   s -> ByteComparable.fixedLength(s.getBytes())))
+                                                                   s -> ByteComparable.preencoded(byteComparableVersion, s.getBytes())))
         {
             testEntries(tests, mapping);
         }
@@ -387,7 +464,7 @@ private void testEntries(String[] tests)
 
     private void testEntriesHex(String[] tests)
     {
-        testEntries(tests, s -> ByteComparable.fixedLength(ByteBufferUtil.hexToBytes(s)));
+        testEntries(tests, s -> ByteComparable.preencoded(byteComparableVersion, ByteBufferUtil.hexToBytes(s)));
         // Run the other translations just in case.
         testEntries(tests);
     }
@@ -395,7 +472,7 @@ private void testEntriesHex(String[] tests)
     private void testEntries(String[] tests, Function<String, ByteComparable> mapping)
 
     {
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+        InMemoryTrie<String> trie = strategy.create();
         for (String test : tests)
         {
             ByteComparable e = mapping.apply(test);
@@ -413,16 +490,56 @@ static InMemoryTrie<ByteBuffer> makeInMemoryTrie(ByteComparable[] src,
                                                      boolean usePut)
 
     {
-        InMemoryTrie<ByteBuffer> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+        InMemoryTrie<ByteBuffer> trie = strategy.create();
         addToInMemoryTrie(src, content, trie, usePut);
         return trie;
     }
 
     static void addToInMemoryTrie(ByteComparable[] src,
                                   Map<ByteComparable, ByteBuffer> content,
-                                  InMemoryTrie<ByteBuffer> trie,
+                                  InMemoryTrie<? super ByteBuffer> trie,
                                   boolean usePut)
 
+    {
+        for (ByteComparable b : src)
+            addToInMemoryTrie(content, trie, usePut, b);
+    }
+
+    static void addNthToInMemoryTrie(ByteComparable[] src,
+                                     Map<ByteComparable, ByteBuffer> content,
+                                     InMemoryTrie<? super ByteBuffer> trie,
+                                     boolean usePut,
+                                     int divisor,
+                                     int remainder)
+
+    {
+        int i = 0;
+        for (ByteComparable b : src)
+        {
+            if (i++ % divisor != remainder)
+                continue;
+
+            addToInMemoryTrie(content, trie, usePut, b);
+        }
+    }
+
+    private static void addToInMemoryTrie(Map<ByteComparable, ByteBuffer> content, InMemoryTrie<? super ByteBuffer> trie, boolean usePut, ByteComparable b)
+    {
+        // Note: Because we don't ensure order when calling resolve, just use a hash of the key as payload
+        // (so that all sources have the same value).
+        int payload = asString(b).hashCode();
+        ByteBuffer v = ByteBufferUtil.bytes(payload);
+        content.put(b, v);
+        if (VERBOSE)
+            System.out.println("Adding " + asString(b) + ": " + ByteBufferUtil.bytesToHex(v));
+        putSimpleResolve(trie, b, v, (x, y) -> y, usePut);
+        if (VERBOSE)
+            System.out.println(trie.dump(x -> string(x)));
+    }
+
+    static void addToMap(ByteComparable[] src,
+                         Map<ByteComparable, ByteBuffer> content)
+
     {
         for (ByteComparable b : src)
         {
@@ -431,29 +548,38 @@ static void addToInMemoryTrie(ByteComparable[] src,
             int payload = asString(b).hashCode();
             ByteBuffer v = ByteBufferUtil.bytes(payload);
             content.put(b, v);
-            if (VERBOSE)
-                System.out.println("Adding " + asString(b) + ": " + ByteBufferUtil.bytesToHex(v));
-            putSimpleResolve(trie, b, v, (x, y) -> y, usePut);
-            if (VERBOSE)
-                System.out.println(trie.dump(ByteBufferUtil::bytesToHex));
         }
     }
 
-    static void checkGet(InMemoryTrie<ByteBuffer> trie, Map<ByteComparable, ByteBuffer> items)
+    private static String string(Object x)
+    {
+        return x instanceof ByteBuffer
+               ? ByteBufferUtil.bytesToHex((ByteBuffer) x)
+               : x instanceof ByteComparable
+                 ? ((ByteComparable) x).byteComparableAsString(byteComparableVersion)
+                 : x.toString();
+    }
+
+    static void checkGet(Trie<? super ByteBuffer> trie, Map<ByteComparable, ByteBuffer> items)
     {
         for (Map.Entry<ByteComparable, ByteBuffer> en : items.entrySet())
         {
+            if (VERBOSE)
+                System.out.println("Checking " + asString(en.getKey()) + ": " + ByteBufferUtil.bytesToHex(en.getValue()));
             assertEquals(en.getValue(), trie.get(en.getKey()));
         }
     }
 
     static void assertSameContent(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
     {
-        assertMapEquals(trie, map);
-        assertForEachEntryEquals(trie, map);
+        assertMapEquals(trie, map, Direction.FORWARD);
+        assertForEachEntryEquals(trie, map, Direction.FORWARD);
         assertValuesEqual(trie, map);
         assertForEachValueEquals(trie, map);
         assertUnorderedValuesEqual(trie, map);
+        assertMapEquals(trie, map, Direction.REVERSE);
+        assertForEachEntryEquals(trie, map, Direction.REVERSE);
+        checkGet(trie, map);
     }
 
     private static void assertValuesEqual(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
@@ -478,13 +604,30 @@ private static void assertUnorderedValuesEqual(Trie<ByteBuffer> trie, SortedMap<
         assertEquals("", errors.toString());
     }
 
-    private static void assertForEachEntryEquals(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
+    static Collection<ByteComparable> maybeReversed(Direction direction, Collection<ByteComparable> data)
+    {
+        return direction.isForward() ? data : reorderBy(data, reverseComparator);
+    }
+
+    static <V> Map<ByteComparable, V> maybeReversed(Direction direction, Map<ByteComparable, V> data)
+    {
+        return direction.isForward() ? data : reorderBy(data, reverseComparator);
+    }
+
+    private static <V> Map<ByteComparable, V> reorderBy(Map<ByteComparable, V> data, Comparator<ByteComparable> comparator)
+    {
+        Map<ByteComparable, V> newMap = new TreeMap<>(comparator);
+        newMap.putAll(data);
+        return newMap;
+    }
+
+    private static void assertForEachEntryEquals(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map, Direction direction)
     {
-        Iterator<Map.Entry<ByteComparable, ByteBuffer>> it = map.entrySet().iterator();
-        trie.forEachEntry((key, value) -> {
+        Iterator<Map.Entry<ByteComparable, ByteBuffer>> it = maybeReversed(direction, map).entrySet().iterator();
+        trie.forEachEntry(direction, (key, value) -> {
             Assert.assertTrue("Map exhausted first, key " + asString(key), it.hasNext());
             Map.Entry<ByteComparable, ByteBuffer> entry = it.next();
-            assertEquals(0, ByteComparable.compare(entry.getKey(), key, Trie.BYTE_COMPARABLE_VERSION));
+            assertEquals(0, ByteComparable.compare(entry.getKey(), key, byteComparableVersion));
             assertEquals(entry.getValue(), value);
         });
         Assert.assertFalse("Trie exhausted first", it.hasNext());
@@ -501,36 +644,41 @@ private static void assertForEachValueEquals(Trie<ByteBuffer> trie, SortedMap<By
         Assert.assertFalse("Trie exhausted first", it.hasNext());
     }
 
-    static void assertMapEquals(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
+    static void assertMapEquals(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map, Direction direction)
+    {
+        assertMapEquals(trie.entryIterator(direction), maybeReversed(direction, map).entrySet().iterator());
+    }
+
+    static <E> Collection<E> reorderBy(Collection<E> original, Comparator<E> comparator)
     {
-        assertMapEquals(trie.entrySet(), map.entrySet());
+        List<E> list = original.stream().collect(Collectors.toList());
+        list.sort(comparator);
+        return list;
     }
 
-    static void assertMapEquals(Iterable<Map.Entry<ByteComparable, ByteBuffer>> container1,
-                                Iterable<Map.Entry<ByteComparable, ByteBuffer>> container2)
+    static <B extends ByteComparable, C extends ByteComparable>
+    void assertMapEquals(Iterator<Map.Entry<B, ByteBuffer>> it1, Iterator<Map.Entry<C, ByteBuffer>> it2)
     {
-        Iterator<Map.Entry<ByteComparable, ByteBuffer>> it1 = container1.iterator();
-        Iterator<Map.Entry<ByteComparable, ByteBuffer>> it2 = container2.iterator();
         List<ByteComparable> failedAt = new ArrayList<>();
         StringBuilder b = new StringBuilder();
         while (it1.hasNext() && it2.hasNext())
         {
-            Map.Entry<ByteComparable, ByteBuffer> en1 = it1.next();
-            Map.Entry<ByteComparable, ByteBuffer> en2 = it2.next();
+            Map.Entry<? extends ByteComparable, ByteBuffer> en1 = it1.next();
+            Map.Entry<? extends ByteComparable, ByteBuffer> en2 = it2.next();
             b.append(String.format("TreeSet %s:%s\n", asString(en2.getKey()), ByteBufferUtil.bytesToHex(en2.getValue())));
             b.append(String.format("Trie    %s:%s\n", asString(en1.getKey()), ByteBufferUtil.bytesToHex(en1.getValue())));
-            if (ByteComparable.compare(en1.getKey(), en2.getKey(), VERSION) != 0 || ByteBufferUtil.compareUnsigned(en1.getValue(), en2.getValue()) != 0)
+            if (ByteComparable.compare(en1.getKey(), en2.getKey(), byteComparableVersion) != 0 || ByteBufferUtil.compareUnsigned(en1.getValue(), en2.getValue()) != 0)
                 failedAt.add(en1.getKey());
         }
         while (it1.hasNext())
         {
-            Map.Entry<ByteComparable, ByteBuffer> en1 = it1.next();
+            Map.Entry<? extends ByteComparable, ByteBuffer> en1 = it1.next();
             b.append(String.format("Trie    %s:%s\n", asString(en1.getKey()), ByteBufferUtil.bytesToHex(en1.getValue())));
             failedAt.add(en1.getKey());
         }
         while (it2.hasNext())
         {
-            Map.Entry<ByteComparable, ByteBuffer> en2 = it2.next();
+            Map.Entry<? extends ByteComparable, ByteBuffer> en2 = it2.next();
             b.append(String.format("TreeSet %s:%s\n", asString(en2.getKey()), ByteBufferUtil.bytesToHex(en2.getValue())));
             failedAt.add(en2.getKey());
         }
@@ -560,7 +708,7 @@ else if (actual.hasNext())
     static ByteComparable[] generateKeys(Random rand, int count)
     {
         ByteComparable[] sources = new ByteComparable[count];
-        TreeSet<ByteComparable> added = new TreeSet<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        TreeSet<ByteComparable> added = new TreeSet<>(forwardComparator);
         for (int i = 0; i < count; ++i)
         {
             sources[i] = generateKey(rand);
@@ -593,12 +741,13 @@ static ByteComparable generateKey(Random rand, int minLength, int maxLength)
             while (p < m)
                 bytes[p++] = (byte) r2.nextInt(256);
         }
-        return ByteComparable.fixedLength(bytes);
+        return prefixFree ? v -> ByteSource.withTerminator(ByteSource.TERMINATOR, ByteSource.of(bytes, v))
+                          : ByteComparable.preencoded(byteComparableVersion, bytes);
     }
 
     static String asString(ByteComparable bc)
     {
-        return bc != null ? bc.byteComparableAsString(VERSION) : "null";
+        return bc != null ? bc.byteComparableAsString(byteComparableVersion) : "null";
     }
 
     <T, M> void putSimpleResolve(InMemoryTrie<T> trie,
@@ -622,7 +771,7 @@ static <T, M> void putSimpleResolve(InMemoryTrie<T> trie,
                               (existing, update) -> existing != null ? resolver.resolve(existing, update) : update,
                               usePut);
         }
-        catch (InMemoryTrie.SpaceExhaustedException e)
+        catch (TrieSpaceExhaustedException e)
         {
             // Should not happen, test stays well below size limit.
             throw new AssertionError(e);
diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java
index c8b6ed57f0c7..8435772bd067 100644
--- a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java
+++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.db.tries;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -26,35 +27,66 @@
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
 
 import org.junit.Assert;
 import org.junit.Test;
 
-import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 
-import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.VERSION;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.byteComparableVersion;
 import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.generateKeys;
 
 public class InMemoryTrieThreadedTest
 {
-    private static final int COUNT = 300000;
+    private static final int COUNT = 30000;
     private static final int OTHERS = COUNT / 10;
     private static final int PROGRESS_UPDATE = COUNT / 15;
     private static final int READERS = 8;
     private static final int WALKERS = 2;
     private static final Random rand = new Random();
 
+    static
+    {
+        InMemoryTrieTestBase.prefixFree = true;
+    }
+
+    /**
+     * Force copy every modified cell below the partition/enumeration level. Provides atomicity of mutations within the
+     * partition level as well as consistency.
+     */
+    public static final Predicate<InMemoryTrie.NodeFeatures<Content>> FORCE_COPY_PARTITION = features -> isPartition(features.content());
+    /**
+     * Force copy every modified cell below the earliest branching point. Provides atomicity of mutations at any level,
+     * but readers/walkers may see inconsistent views of the data, in the sense that older mutations may be missed
+     * while newer ones are returned.
+     */
+    public static final Predicate<InMemoryTrie.NodeFeatures<Content>> FORCE_ATOMIC = features -> features.isBranching();
+    /**
+     * Do not do any additional copying beyond what is required to build the tries safely for concurrent readers.
+     * Mutations may be partially seen by readers, and older mutations may be missed while newer ones are returned.
+     */
+    public static final Predicate<InMemoryTrie.NodeFeatures<Content>> NO_ATOMICITY = features -> false;
+
+    static Value value(ByteComparable b, ByteComparable cprefix, ByteComparable c, int add, int seqId)
+    {
+        return new Value(b.byteComparableAsString(byteComparableVersion),
+                         (cprefix != null ? cprefix.byteComparableAsString(byteComparableVersion) : "") + c.byteComparableAsString(byteComparableVersion), add, seqId);
+    }
+
     static String value(ByteComparable b)
     {
-        return b.byteComparableAsString(VERSION);
+        return b.byteComparableAsString(byteComparableVersion);
     }
 
     @Test
     public void testThreaded() throws InterruptedException
     {
+        OpOrder readOrder = new OpOrder();
         ByteComparable[] src = generateKeys(rand, COUNT + OTHERS);
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.ON_HEAP);
+        InMemoryTrie<String> trie = InMemoryTrie.longLived(byteComparableVersion, readOrder);
         ConcurrentLinkedQueue<Throwable> errors = new ConcurrentLinkedQueue<>();
         List<Thread> threads = new ArrayList<>();
         AtomicBoolean writeCompleted = new AtomicBoolean(false);
@@ -68,11 +100,14 @@ public void testThreaded() throws InterruptedException
                     {
                         int min = writeProgress.get();
                         int count = 0;
-                        for (Map.Entry<ByteComparable, String> en : trie.entrySet())
+                        try (OpOrder.Group group = readOrder.start())
                         {
-                            String v = value(en.getKey());
-                            Assert.assertEquals(en.getKey().byteComparableAsString(VERSION), v, en.getValue());
-                            ++count;
+                            for (Map.Entry<? extends ByteComparable, String> en : trie.entrySet())
+                            {
+                                String v = value(en.getKey());
+                                Assert.assertEquals(en.getKey().byteComparableAsString(byteComparableVersion), v, en.getValue());
+                                ++count;
+                            }
                         }
                         Assert.assertTrue("Got only " + count + " while progress is at " + min, count >= min);
                     }
@@ -99,15 +134,18 @@ public void testThreaded() throws InterruptedException
                             int index = r.nextInt(COUNT + OTHERS);
                             ByteComparable b = src[index];
                             String v = value(b);
-                            String result = trie.get(b);
-                            if (result != null)
+                            try (OpOrder.Group group = readOrder.start())
                             {
-                                Assert.assertTrue("Got not added " + index + " when COUNT is " + COUNT,
-                                                  index < COUNT);
-                                Assert.assertEquals("Failed " + index, v, result);
+                                String result = trie.get(b);
+                                if (result != null)
+                                {
+                                    Assert.assertTrue("Got not added " + index + " when COUNT is " + COUNT,
+                                                      index < COUNT);
+                                    Assert.assertEquals("Failed " + index, v, result);
+                                }
+                                else if (index < min)
+                                    Assert.fail("Failed index " + index + " while progress is at " + min);
                             }
-                            else if (index < min)
-                                Assert.fail("Failed index " + index + " while progress is at " + min);
                         }
                     }
                 }
@@ -129,10 +167,7 @@ else if (index < min)
                     // Note: Because we don't ensure order when calling resolve, just use a hash of the key as payload
                     // (so that all sources have the same value).
                     String v = value(b);
-                    if (i % 2 == 0)
-                        trie.apply(Trie.singleton(b, v), (x, y) -> y);
-                    else
-                        trie.putRecursive(b, v, (x, y) -> y);
+                    trie.putSingleton(b, v, (x, y) -> y, i % 2 != 0);
 
                     if (i % PROGRESS_UPDATE == 0)
                         writeProgress.set(i);
@@ -158,4 +193,417 @@ else if (index < min)
         if (!errors.isEmpty())
             Assert.fail("Got errors:\n" + errors);
     }
+
+    static abstract class Content
+    {
+        final String pk;
+
+        Content(String pk)
+        {
+            this.pk = pk;
+        }
+
+        abstract boolean isPartition();
+    }
+
+    static class Value extends Content
+    {
+        final String ck;
+        final int value;
+        final int seq;
+
+        Value(String pk, String ck, int value, int seq)
+        {
+            super(pk);
+            this.ck = ck;
+            this.value = value;
+            this.seq = seq;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Value{" +
+                   "pk='" + pk + '\'' +
+                   ", ck='" + ck + '\'' +
+                   ", value=" + value +
+                   ", seq=" + seq +
+                   '}';
+        }
+
+        @Override
+        boolean isPartition()
+        {
+            return false;
+        }
+    }
+
+    static class Metadata extends Content
+    {
+        int updateCount;
+
+        Metadata(String pk)
+        {
+            super(pk);
+            updateCount = 1;
+        }
+
+        @Override
+        boolean isPartition()
+        {
+            return true;
+        }
+
+        Metadata mergeWith(Metadata other)
+        {
+            Metadata m = new Metadata(pk);
+            m.updateCount = updateCount + other.updateCount;
+            return m;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Metadata{" +
+                   "pk='" + pk + '\'' +
+                   ", updateCount=" + updateCount +
+                   '}';
+        }
+    }
+
+    static boolean isPartition(Content c)
+    {
+        return c != null && c.isPartition();
+    }
+
+    @Test
+    public void testConsistentUpdates() throws Exception
+    {
+        // Check that multi-path updates with below-partition-level copying are safe for concurrent readers,
+        // and that content is atomically applied, i.e. that reader see either nothing from the update or all of it,
+        // and consistent, i.e. that it is not possible to receive some newer updates while missing
+        // older ones. (For example, if the sequence of additions is 3, 1, 5, without this requirement a reader
+        // could see an enumeration which lists 3 and 5 but not 1.)
+        testAtomicUpdates(3, FORCE_COPY_PARTITION, true, true);
+        // Note: using 3 per mutation, so that the first and second update fit in a sparse in-memory trie block.
+    }
+
+    @Test
+    public void testAtomicUpdates() throws Exception
+    {
+        // Check that multi-path updates with below-branching-point copying are safe for concurrent readers,
+        // and that content is atomically applied, i.e. that reader see either nothing from the update or all of it.
+        testAtomicUpdates(3, FORCE_ATOMIC, true, false);
+    }
+
+    @Test
+    public void testSafeUpdates() throws Exception
+    {
+        // Check that multi path updates without additional copying are safe for concurrent readers.
+        testAtomicUpdates(3, NO_ATOMICITY, false, false);
+    }
+
+    @Test
+    public void testConsistentSinglePathUpdates() throws Exception
+    {
+        // Check that single path updates with below-partition-level copying are safe for concurrent readers,
+        // and that content is consistent, i.e. that it is not possible to receive some newer updates while missing
+        // older ones. (For example, if the sequence of additions is 3, 1, 5, without this requirement a reader
+        // could see an enumeration which lists 3 and 5 but not 1.)
+        testAtomicUpdates(1, FORCE_COPY_PARTITION, true, true);
+    }
+
+
+    @Test
+    public void testAtomicSinglePathUpdates() throws Exception
+    {
+        // When doing single path updates atomicity comes for free. This only checks that the branching checker is
+        // not doing anything funny.
+        testAtomicUpdates(1, FORCE_ATOMIC, true, false);
+    }
+
+    @Test
+    public void testSafeSinglePathUpdates() throws Exception
+    {
+        // Check that single path updates without additional copying are safe for concurrent readers.
+        testAtomicUpdates(1, NO_ATOMICITY, true, false);
+    }
+
+    // The generated keys all start with NEXT_COMPONENT, which makes it impossible to test the precise behavior of the
+    // partition-level force copying. Strip that byte.
+    private static ByteComparable[] skipFirst(ByteComparable[] keys)
+    {
+        ByteComparable[] result = new ByteComparable[keys.length];
+        for (int i = 0; i < keys.length; ++i)
+            result[i] = skipFirst(keys[i]);
+        return result;
+    }
+
+    private static ByteComparable skipFirst(ByteComparable key)
+    {
+        return v -> {
+            var bs = key.asComparableBytes(v);
+            int n = bs.next();
+            assert n != ByteSource.END_OF_STREAM;
+            return bs;
+        };
+    }
+
+    public void testAtomicUpdates(int PER_MUTATION,
+                                  Predicate<InMemoryTrie.NodeFeatures<Content>> forcedCopyChecker,
+                                  boolean checkAtomicity,
+                                  boolean checkSequence)
+    throws Exception
+    {
+        ByteComparable[] ckeys = skipFirst(generateKeys(rand, COUNT));
+        ByteComparable[] pkeys = skipFirst(generateKeys(rand, Math.min(100, COUNT / 10)));  // to guarantee repetition
+
+        /*
+         * Adds COUNT partitions each with perPartition separate clusterings, where the sum of the values
+         * of all clusterings is 0.
+         * If the sum for any walk covering whole partitions is non-zero, we have had non-atomic updates.
+         */
+
+        OpOrder readOrder = new OpOrder();
+//        InMemoryTrie<Content> trie = new InMemoryTrie<>(new MemtableAllocationStrategy.NoReuseStrategy(BufferType.OFF_HEAP));
+        InMemoryTrie<Content> trie = InMemoryTrie.longLived(byteComparableVersion, readOrder);
+        ConcurrentLinkedQueue<Throwable> errors = new ConcurrentLinkedQueue<>();
+        List<Thread> threads = new ArrayList<Thread>();
+        AtomicBoolean writeCompleted = new AtomicBoolean(false);
+        AtomicInteger writeProgress = new AtomicInteger(0);
+
+        for (int i = 0; i < WALKERS; ++i)
+            threads.add(new Thread()
+            {
+                public void run()
+                {
+                    try
+                    {
+                        Random r = ThreadLocalRandom.current();
+                        while (!writeCompleted.get())
+                        {
+                            int min = writeProgress.get();
+                            try (OpOrder.Group group = readOrder.start())
+                            {
+                                Iterable<Map.Entry<ByteComparable.Preencoded, Content>> entries = trie.entrySet();
+                                checkEntries("", min, true, checkAtomicity, false, PER_MUTATION, entries);
+                            }
+                        }
+                    }
+                    catch (Throwable t)
+                    {
+                        t.printStackTrace();
+                        errors.add(t);
+                    }
+                }
+            });
+
+        for (int i = 0; i < READERS; ++i)
+        {
+            ByteComparable[] srcLocal = pkeys;
+            threads.add(new Thread()
+            {
+                public void run()
+                {
+                    try
+                    {
+                        // await at least one ready partition
+                        while (writeProgress.get() == 0) {}
+
+                        Random r = ThreadLocalRandom.current();
+                        while (!writeCompleted.get())
+                        {
+                            ByteComparable key = srcLocal[r.nextInt(srcLocal.length)];
+                            int min = writeProgress.get() / (pkeys.length * PER_MUTATION) * PER_MUTATION;
+                            Iterable<Map.Entry<ByteComparable.Preencoded, Content>> entries;
+
+                            try (OpOrder.Group group = readOrder.start())
+                            {
+                                entries = trie.tailTrie(key).entrySet();
+                                checkEntries(" in tail " + key.byteComparableAsString(byteComparableVersion), min, false, checkAtomicity, checkSequence, PER_MUTATION, entries);
+                            }
+
+                            try (OpOrder.Group group = readOrder.start())
+                            {
+                                entries = trie.subtrie(key, nextBranch(key)).entrySet();
+                                checkEntries(" in branch " + key.byteComparableAsString(byteComparableVersion), min, true, checkAtomicity, checkSequence, PER_MUTATION, entries);
+                            }
+                        }
+                    }
+                    catch (Throwable t)
+                    {
+                        t.printStackTrace();
+                        errors.add(t);
+                    }
+                }
+            });
+        }
+
+        threads.add(new Thread()
+        {
+            public void run()
+            {
+                ThreadLocalRandom r = ThreadLocalRandom.current();
+                final Trie.CollectionMergeResolver<Content> mergeResolver = new Trie.CollectionMergeResolver<Content>()
+                {
+                    @Override
+                    public Content resolve(Content c1, Content c2)
+                    {
+                        if (c1.isPartition() && c2.isPartition())
+                            return ((Metadata) c1).mergeWith((Metadata) c2);
+                        throw new AssertionError("Test error, keys should be distinct.");
+                    }
+
+                    public Content resolve(Collection<Content> contents)
+                    {
+                        return contents.stream().reduce(this::resolve).get();
+                    }
+                };
+
+                try
+                {
+                    int lastUpdate = 0;
+                    for (int i = 0; i < COUNT; i += PER_MUTATION)
+                    {
+                        ByteComparable b = pkeys[(i / PER_MUTATION) % pkeys.length];
+                        Metadata partitionMarker = new Metadata(b.byteComparableAsString(byteComparableVersion));
+                        ByteComparable cprefix = null;
+                        if (r.nextBoolean())
+                            cprefix = ckeys[i]; // Also test branching point below the partition level
+
+                        List<Trie<Content>> sources = new ArrayList<>();
+                        for (int j = 0; j < PER_MUTATION; ++j)
+                        {
+
+                            ByteComparable k = ckeys[i + j];
+                            Trie<Content> row = Trie.singleton(k, byteComparableVersion,
+                                                               value(b, cprefix, k,
+                                                                     j == 0 ? -PER_MUTATION + 1 : 1,
+                                                                     (i / PER_MUTATION / pkeys.length) * PER_MUTATION + j));
+
+                            if (cprefix != null)
+                                row = row.prefixedBy(cprefix);
+
+                            row = withRootMetadata(row, partitionMarker);
+                            row = row.prefixedBy(b);
+                            sources.add(row);
+                        }
+
+                        final Trie<Content> mutation = Trie.merge(sources, mergeResolver);
+
+                        trie.apply(mutation,
+                                   (existing, update) -> existing == null ? update : mergeResolver.resolve(existing, update),
+                                   forcedCopyChecker);
+
+                        if (i >= pkeys.length * PER_MUTATION && i - lastUpdate >= PROGRESS_UPDATE)
+                        {
+                            writeProgress.set(i);
+                            lastUpdate = i;
+                        }
+                    }
+                }
+                catch (Throwable t)
+                {
+                    t.printStackTrace();
+                    errors.add(t);
+                }
+                finally
+                {
+                    writeCompleted.set(true);
+                }
+            }
+        });
+
+        for (Thread t : threads)
+            t.start();
+
+        for (Thread t : threads)
+            t.join();
+
+        System.out.format("Reuse %s %s atomicity %s on-heap %,d (+%,d) off-heap %,d\n",
+                          trie.cellAllocator.getClass().getSimpleName(),
+                          trie.bufferType,
+                          forcedCopyChecker == NO_ATOMICITY ? "none" :
+                          forcedCopyChecker == FORCE_ATOMIC ? "atomic" : "consistent partition",
+                          trie.usedSizeOnHeap(),
+                          trie.unusedReservedOnHeapMemory(),
+                          trie.usedSizeOffHeap());
+
+        if (!errors.isEmpty())
+            Assert.fail("Got errors:\n" + errors);
+    }
+
+    static ByteComparable nextBranch(ByteComparable key)
+    {
+        return version -> {
+            byte[] bytes = key.asByteComparableArray(version);
+            int last = bytes.length - 1;
+            while (last >= 0 && bytes[last] == ((byte) 0xFF))
+                --last;
+            if (last < 0)
+                return null;
+            ++bytes[last];
+            return ByteSource.preencoded(bytes, 0, last + 1);
+        };
+    }
+
+    static <T> Trie<T> withRootMetadata(Trie<T> wrapped, T metadata)
+    {
+        return wrapped.mergeWith(Trie.singleton(ByteComparable.EMPTY, byteComparableVersion, metadata), Trie.throwingResolver());
+    }
+
+    public void checkEntries(String location,
+                             int min,
+                             boolean usePk,
+                             boolean checkAtomicity,
+                             boolean checkConsecutiveIds,
+                             int PER_MUTATION,
+                             Iterable<Map.Entry<ByteComparable.Preencoded, Content>> entries)
+    {
+        long sum = 0;
+        int count = 0;
+        long idSum = 0;
+        long idMax = 0;
+        int updateCount = 0;
+        for (var en : entries)
+        {
+            String path = en.getKey().byteComparableAsString(byteComparableVersion);
+            if (en.getValue().isPartition())
+            {
+                Metadata m = (Metadata) en.getValue();
+                Assert.assertEquals("Partition metadata" + location, (usePk ? m.pk : ""), path);
+                updateCount += m.updateCount;
+                continue;
+            }
+            final Value value = (Value) en.getValue();
+            String valueKey = (usePk ? value.pk : "") + value.ck;
+            Assert.assertEquals(location, valueKey, path);
+            ++count;
+            sum += value.value;
+            int seq = value.seq;
+            idSum += seq;
+            if (seq > idMax)
+                idMax = seq;
+        }
+
+        Assert.assertTrue("Values" + location + " should be at least " + min + ", got " + count, min <= count);
+
+        if (checkAtomicity)
+        {
+            // If mutations apply atomically, the row count is always a multiple of the mutation size...
+            Assert.assertTrue("Values" + location + " should be a multiple of " + PER_MUTATION + ", got " + count, count % PER_MUTATION == 0);
+            // ... and the sum of the values is 0 (as the sum for each individual mutation is 0).
+            Assert.assertEquals("Value sum" + location, 0, sum);
+        }
+
+        if (checkConsecutiveIds)
+        {
+            // The update count reflected in the partition metadata must match the row count.
+            Assert.assertEquals("Update count" + location, count, updateCount);
+            // If mutations apply consistently for the partition, for any row we see we have to have seen all rows that
+            // were applied before that. In other words, the id sum should be the sum of the integers from 1 to the
+            // highest id seen in the partition.
+            Assert.assertEquals("Id sum" + location, idMax * (idMax + 1) / 2, idSum);
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/tries/MergeTrieTest.java b/test/unit/org/apache/cassandra/db/tries/MergeTrieTest.java
index cc08401102f5..1bd025b213c3 100644
--- a/test/unit/org/apache/cassandra/db/tries/MergeTrieTest.java
+++ b/test/unit/org/apache/cassandra/db/tries/MergeTrieTest.java
@@ -40,8 +40,8 @@ public void testDirect()
     {
         ByteComparable[] src1 = generateKeys(rand, COUNT);
         ByteComparable[] src2 = generateKeys(rand, COUNT);
-        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
-        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>(forwardComparator);
+        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>(forwardComparator);
 
         InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
         InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
@@ -57,11 +57,11 @@ public void testWithDuplicates()
     {
         ByteComparable[] src1 = generateKeys(rand, COUNT);
         ByteComparable[] src2 = generateKeys(rand, COUNT);
-        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
-        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>(forwardComparator);
+        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>(forwardComparator);
 
-        InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
-        InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
+        InMemoryTrie trie1 = makeInMemoryTrie(src1, content1, true);
+        InMemoryTrie trie2 = makeInMemoryTrie(src2, content2, true);
 
         addToInMemoryTrie(generateKeys(new Random(5), COUNT), content1, trie1, true);
         addToInMemoryTrie(generateKeys(new Random(5), COUNT), content2, trie2, true);
@@ -76,12 +76,12 @@ public void testWithDuplicates()
     public void testDistinct()
     {
         ByteComparable[] src1 = generateKeys(rand, COUNT);
-        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>(forwardComparator);
         InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
 
         ByteComparable[] src2 = generateKeys(rand, COUNT);
         src2 = removeDuplicates(src2, content1);
-        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+        SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>(forwardComparator);
         InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
 
         content1.putAll(content2);
diff --git a/test/unit/org/apache/cassandra/db/tries/PrefixTailTrieTest.java b/test/unit/org/apache/cassandra/db/tries/PrefixTailTrieTest.java
new file mode 100644
index 000000000000..1424a8d1f530
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/PrefixTailTrieTest.java
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Objects;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.primitives.Bytes;
+import org.junit.Test;
+
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Hex;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.byteComparableVersion;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.addNthToInMemoryTrie;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.addToInMemoryTrie;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.assertIterablesEqual;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.assertMapEquals;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.checkGet;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.generateKey;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.generateKeys;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class PrefixTailTrieTest
+{
+    private static final int COUNT_TAIL = 5000;
+    private static final int COUNT_HEAD = 25;
+    public static final Comparator<ByteComparable> BYTE_COMPARABLE_COMPARATOR = (a, b) -> ByteComparable.compare(a, b, byteComparableVersion);
+    Random rand = new Random();
+
+    static
+    {
+        // Use prefix-free keys to avoid putting partitions within partitions
+        InMemoryTrieTestBase.prefixFree = true;
+    }
+
+    static final InMemoryTrie.UpsertTransformer<Object, Object> THROWING_UPSERT = (e, u) -> {
+        if (e != null) throw new AssertionError();
+        return u;
+    };
+
+    static final Function<Object, String> CONTENT_TO_STRING = x -> x instanceof ByteBuffer
+                                                                   ? ByteBufferUtil.bytesToHex((ByteBuffer) x)
+                                                                   : x.toString();
+
+    static class Tail
+    {
+        byte[] prefix;
+        NavigableMap<ByteComparable, ByteBuffer> data;
+
+        public Tail(byte[] prefix, NavigableMap<ByteComparable, ByteBuffer> map)
+        {
+            this.prefix = prefix;
+            this.data = map;
+        }
+
+        public String toString()
+        {
+            return "Tail{" + ByteBufferUtil.bytesToHex(ByteBuffer.wrap(prefix)) + '}';
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Tail tail = (Tail) o;
+            return Arrays.equals(prefix, tail.prefix) && Objects.equals(data, tail.data);
+        }
+    }
+
+    static <T> T getRootContent(Trie<T> trie)
+    {
+        return trie.get(ByteComparable.EMPTY);
+    }
+
+    @Test
+    public void testPrefixTail() throws Exception
+    {
+        testPrefixTail(1, false);
+    }
+
+    @Test
+    public void testPrefixTailMerge2InHead() throws Exception
+    {
+        testPrefixTail(2, false);
+    }
+
+    @Test
+    public void testPrefixTailMerge2InTail() throws Exception
+    {
+        testPrefixTail(2, true);
+    }
+
+    @Test
+    public void testPrefixTailMerge5InHead() throws Exception
+    {
+        testPrefixTail(5, false);
+    }
+
+    @Test
+    public void testPrefixTailMerge5InTail() throws Exception
+    {
+        testPrefixTail(5, true);
+    }
+
+    static Tail combineTails(Object x, Object y)
+    {
+        // Cast failure is a test problem
+        Tail tx = (Tail) x;
+        Tail ty = (Tail) y;
+        var map = new TreeMap<ByteComparable, ByteBuffer>(BYTE_COMPARABLE_COMPARATOR);
+        map.putAll(tx.data);
+        map.putAll(ty.data);
+        return new Tail(tx.prefix, map);
+    }
+
+    public void testPrefixTail(int splits, boolean splitInTail) throws Exception
+    {
+        ByteComparable[] prefixes = generateKeys(rand, COUNT_HEAD);
+
+        NavigableMap<ByteComparable, Tail> data = new TreeMap<>(BYTE_COMPARABLE_COMPARATOR);
+        final Trie<Object> trie = splitInTail ? prepareSplitInTailTrie(splits, prefixes, data)
+                                              : prepareSplitInHeadTrie(splits, prefixes, data);
+//        System.out.println(trie.dump(CONTENT_TO_STRING));
+
+        // Test tailTrie for known prefix
+        for (int i = 0; i < COUNT_HEAD; ++i)
+        {
+            Tail t = data.get(prefixes[i]);
+            Trie<Object> tail = trie.tailTrie(prefixes[i]);
+            assertEquals(t, getRootContent(tail));
+            checkContent(tail, t.data);
+        }
+
+        // Test tail iteration for given class
+        for (Direction td : Direction.values())
+        {
+            long count = 0;
+            for (var en : trie.tailTries(td, Tail.class))
+            {
+                System.out.println(en.getKey().byteComparableAsString(byteComparableVersion));
+                Trie<Object> tail = en.getValue();
+                Tail t = data.get(en.getKey());
+                assertNotNull(t);
+                assertEquals(t, getRootContent(tail));
+                checkContent(tail, t.data);
+                ++count;
+            }
+            assertEquals(COUNT_HEAD, count);
+        }
+
+        // test a sample of tail slices
+        for (int i = rand.nextInt(7); i < COUNT_HEAD; i += 1 + rand.nextInt(7))
+        {
+            Tail t = data.get(prefixes[i]);
+            int keyCount = t.data.keySet().size();
+            int firstIndex = rand.nextInt(keyCount - 1);
+            int lastIndex = firstIndex + rand.nextInt(keyCount - firstIndex);
+            ByteComparable first = rand.nextInt(5) > 0 ? Iterables.get(t.data.keySet(), firstIndex) : null;
+            ByteComparable last = rand.nextInt(5) > 0 ? Iterables.get(t.data.keySet(), lastIndex) : null;
+            ByteComparable prefix = prefixes[i];
+            final ByteComparable leftWithPrefix = concat(prefix, first, rand.nextBoolean() ? prefix
+                                                                                           : rand.nextBoolean()
+                                                                                             ? data.lowerKey(prefix)
+                                                                                             : null);
+            final ByteComparable rightWithPrefix = concat(prefix, last, rand.nextBoolean() ? data.higherKey(prefix)
+                                                                                           : null);
+            Trie<Object> tail = trie.subtrie(leftWithPrefix,
+                                             rightWithPrefix)
+                                    .tailTrie(prefixes[i]);
+            System.out.println("Between " + (leftWithPrefix == null ? "null" : leftWithPrefix.byteComparableAsString(byteComparableVersion)) + " and " + (rightWithPrefix == null ? "null" : rightWithPrefix.byteComparableAsString(byteComparableVersion)));
+            assertEquals(first == null ? t : null, getRootContent(tail));   // this behavior will change soon to report all prefixes
+            checkContent(tail, subMap(t.data, first, last));
+        }
+
+        // Test processSkippingBranches variations
+        for (Direction td : Direction.values())
+        {
+            final AtomicLong count = new AtomicLong(0);
+            trie.forEachValueSkippingBranches(td, v -> count.incrementAndGet());
+            assertEquals(COUNT_HEAD, count.get());
+
+            count.set(0);
+            trie.forEachEntrySkippingBranches(td, (key, tail) ->
+            {
+                assertArrayEquals(((Tail) tail).prefix, key.asByteComparableArray(byteComparableVersion));
+                count.incrementAndGet();
+            });
+            assertEquals(COUNT_HEAD, count.get());
+        }
+    }
+
+    private static void checkContent(Trie<Object> tail, NavigableMap<ByteComparable, ByteBuffer> data)
+    {
+
+        assertMapEquals(tail.filteredEntryIterator(Direction.FORWARD, ByteBuffer.class),
+                        data.entrySet().iterator());
+        assertIterablesEqual(tail.filteredValues(Direction.FORWARD, ByteBuffer.class),
+                             data.values());
+        // As the keys are prefix-free, reverse iteration is the inverse of forward.
+        assertMapEquals(tail.filteredEntryIterator(Direction.REVERSE, ByteBuffer.class),
+                        data.descendingMap().entrySet().iterator());
+        assertIterablesEqual(tail.filteredValues(Direction.REVERSE, ByteBuffer.class),
+                             data.descendingMap().values());
+        checkGet(tail, data);
+    }
+
+    private static <K, V> NavigableMap<K, V> subMap(NavigableMap<K, V> data, K left, K right)
+    {
+        if (left == null)
+            return right == null ? data : data.headMap(right, false);
+        else
+            return right == null
+                   ? data.tailMap(left, true)
+                   : data.subMap(left, true, right, false);
+    }
+
+    private static ByteComparable concat(ByteComparable a, ByteComparable b, ByteComparable ifBNull)
+    {
+        if (b == null)
+            return ifBNull;
+        return ByteComparable.preencoded(byteComparableVersion,
+                                         Bytes.concat(a.asByteComparableArray(byteComparableVersion),
+                                                      b.asByteComparableArray(byteComparableVersion)));
+    }
+
+    private Trie<Object> prepareSplitInTailTrie(int splits, ByteComparable[] prefixes, Map<ByteComparable, Tail> data) throws TrieSpaceExhaustedException
+    {
+        InMemoryTrie<Object>[] tries = new InMemoryTrie[splits];
+        for (int i = 0; i < splits; ++i)
+            tries[i] = InMemoryTrie.shortLived(byteComparableVersion);
+        for (int i = 0; i < COUNT_HEAD; ++i)
+        {
+            ByteComparable[] src = generateKeys(rand, COUNT_TAIL);
+            NavigableMap<ByteComparable, ByteBuffer> allContent = new TreeMap<>(BYTE_COMPARABLE_COMPARATOR);
+            for (int k = 0; k < splits; ++k)
+            {
+                NavigableMap<ByteComparable, ByteBuffer> content = new TreeMap<>(BYTE_COMPARABLE_COMPARATOR);
+                InMemoryTrie<Object> tail = InMemoryTrie.shortLived(byteComparableVersion);
+                addNthToInMemoryTrie(src, content, tail, true, splits, k);
+
+                Tail t = new Tail(prefixes[i].asByteComparableArray(byteComparableVersion), content);
+                allContent.putAll(content);
+                tail.putRecursive(ByteComparable.EMPTY, t, THROWING_UPSERT);
+//            System.out.println(tail.dump(CONTENT_TO_STRING));
+                tries[k].apply(tail.prefixedBy(prefixes[i]), THROWING_UPSERT, Predicates.alwaysFalse());
+            }
+            Tail t = new Tail(prefixes[i].asByteComparableArray(byteComparableVersion), allContent);
+            data.put(ByteComparable.preencoded(byteComparableVersion, t.prefix), t);
+        }
+
+        return Trie.merge(Arrays.asList(tries), c -> c.stream().reduce(PrefixTailTrieTest::combineTails).get());
+    }
+
+
+    private Trie<Object> prepareSplitInHeadTrie(int splits, ByteComparable[] prefixes, Map<ByteComparable, Tail> data) throws TrieSpaceExhaustedException
+    {
+        InMemoryTrie<Object>[] tries = new InMemoryTrie[splits];
+        for (int i = 0; i < splits; ++i)
+            tries[i] = InMemoryTrie.shortLived(byteComparableVersion);
+        int trieIndex = 0;
+        for (int i = 0; i < prefixes.length; ++i)
+        {
+            ByteComparable[] src = generateKeys(rand, COUNT_TAIL);
+
+            NavigableMap<ByteComparable, ByteBuffer> content = new TreeMap<>(BYTE_COMPARABLE_COMPARATOR);
+            InMemoryTrie<Object> tail = InMemoryTrie.shortLived(byteComparableVersion);
+            addToInMemoryTrie(src, content, tail, true);
+
+            Tail t = new Tail(prefixes[i].asByteComparableArray(byteComparableVersion), content);
+            tail.putRecursive(ByteComparable.EMPTY, t, THROWING_UPSERT);
+//            System.out.println(tail.dump(CONTENT_TO_STRING));
+            tries[trieIndex].apply(tail.prefixedBy(prefixes[i]), THROWING_UPSERT, Predicates.alwaysFalse());
+
+            data.put(ByteComparable.preencoded(byteComparableVersion, t.prefix), t);
+            trieIndex = (trieIndex + 1) % splits;
+        }
+
+        return Trie.mergeDistinct(Arrays.asList(tries));
+    }
+
+    // also do same prefix updates
+
+    @Test
+    public void testTailMerge() throws Exception
+    {
+        ByteComparable prefix = generateKey(rand);
+        InMemoryTrie<Object> trie = InMemoryTrie.shortLived(byteComparableVersion);
+        NavigableMap<ByteComparable, ByteBuffer> content = new TreeMap<>(BYTE_COMPARABLE_COMPARATOR);
+
+        for (int i = 0; i < COUNT_HEAD; ++i)
+        {
+            ByteComparable[] src = generateKeys(rand, COUNT_TAIL);
+            InMemoryTrie<Object> tail = InMemoryTrie.shortLived(byteComparableVersion);
+            addToInMemoryTrie(src, content, tail, true);
+//                        System.out.println(tail.dump(CONTENT_TO_STRING));
+            tail.putRecursive(ByteComparable.EMPTY, 1, THROWING_UPSERT);
+            trie.apply(tail.prefixedBy(prefix),
+                       (x, y) -> x instanceof Integer ? (Integer) x + (Integer) y : y,
+                       Predicates.alwaysFalse());
+        }
+
+//                System.out.println(trie.dump(CONTENT_TO_STRING));
+
+        Trie<Object> tail = trie.tailTrie(prefix);
+        assertEquals(COUNT_HEAD, ((Integer) getRootContent(tail)).intValue());
+        assertMapEquals(tail.filteredEntryIterator(Direction.FORWARD, ByteBuffer.class),
+                        content.entrySet().iterator());
+        assertIterablesEqual(tail.filteredValues(Direction.FORWARD, ByteBuffer.class),
+                             content.values());
+
+
+        // Test tail iteration for metadata
+        long count = 0;
+        for (var en : trie.tailTries(Direction.FORWARD, Integer.class))
+        {
+            System.out.println(en.getKey().byteComparableAsString(byteComparableVersion));
+            Trie<Object> tt = en.getValue();
+            assertNotNull(tt);
+            assertEquals(COUNT_HEAD, ((Integer) getRootContent(tail)).intValue());
+            assertMapEquals(tt.filteredEntryIterator(Direction.FORWARD, ByteBuffer.class),
+                            content.entrySet().iterator());
+            assertIterablesEqual(tt.filteredValues(Direction.FORWARD, ByteBuffer.class),
+                                 content.values());
+            ++count;
+        }
+        assertEquals(1, count);
+    }
+
+    @Test
+    public void testKeyProducer() throws Exception
+    {
+
+        testKeyProducer(generateKeys(rand, COUNT_HEAD));
+    }
+
+    @Test
+    public void testKeyProducerMarkedRoot() throws Exception
+    {
+        // Check that path construction works correctly also when the root is the starting position.
+        testKeyProducer(new ByteComparable[] { ByteComparable.EMPTY });
+    }
+
+    private void testKeyProducer(ByteComparable[] prefixes) throws TrieSpaceExhaustedException
+    {
+        NavigableMap<ByteComparable, Tail> data = new TreeMap<>(BYTE_COMPARABLE_COMPARATOR);
+        final Trie<Object> trie = prepareSplitInHeadTrie(1, prefixes, data);
+//        System.out.println(trie.dump(CONTENT_TO_STRING));
+
+        InMemoryTrie<Object> dest = InMemoryTrie.shortLived(byteComparableVersion);
+        InclusionChecker checker = new InclusionChecker();
+        dest.apply(trie, checker, Predicates.alwaysFalse());
+        assertEquals("", checker.output.toString());
+    }
+
+    static class InclusionChecker implements InMemoryTrie.UpsertTransformerWithKeyProducer<Object, Object>
+    {
+        Tail currentTail = null;
+        StringBuilder output = new StringBuilder();
+
+        @Override
+        public Object apply(Object existing, Object update, InMemoryTrie.KeyProducer<Object> keyProducer)
+        {
+            if (existing != null)
+                output.append("Non-null existing\n");
+
+            byte[] tailPath = keyProducer.getBytes(Tail.class::isInstance);
+            byte[] fullPath = keyProducer.getBytes();
+            String tail = Hex.bytesToHex(tailPath);
+            String full = Hex.bytesToHex(fullPath);
+            if (!full.endsWith(tail))
+            {
+                output.append("Tail " + tail + " is not suffix of full path " + full + "\n");
+                return update; // can't continue
+            }
+
+            String msg = "\n@key " + full.substring(0, full.length() - tail.length()) + ":" + tail + "\n";
+
+            if (update instanceof Tail)
+            {
+                // At
+                if (tailPath.length != fullPath.length)
+                    output.append("Prefix not empty on tail root" + msg);
+                Tail t = (Tail) update;
+                if (!Arrays.equals(t.prefix, fullPath))
+                    output.append("Tail root path expected " + Hex.bytesToHex(t.prefix) + msg);
+                currentTail = t;
+            }
+            else
+            {
+                byte[] prefix = Arrays.copyOfRange(fullPath, 0, fullPath.length - tailPath.length);
+                if (currentTail == null)
+                    output.append("Null currentTail" + msg);
+                if (!Arrays.equals(currentTail.prefix, prefix))
+                    output.append("Prefix expected " + Hex.bytesToHex(currentTail.prefix) + msg);
+
+                if (!(update instanceof ByteBuffer))
+                    output.append("Not ByteBuffer " + update + msg);
+                ByteBuffer expected = currentTail.data.get(ByteComparable.preencoded(byteComparableVersion, tailPath));
+                if (expected == null)
+                    output.append("Suffix not found" + msg);
+                if (!expected.equals(update))
+                    output.append("Data mismatch " + ByteBufferUtil.bytesToHex((ByteBuffer) update) + " expected " + ByteBufferUtil.bytesToHex(expected) + msg);
+            }
+            return update;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/SlicedTrieTest.java b/test/unit/org/apache/cassandra/db/tries/SlicedTrieTest.java
index 07bae0ed9f17..df811a2a41ca 100644
--- a/test/unit/org/apache/cassandra/db/tries/SlicedTrieTest.java
+++ b/test/unit/org/apache/cassandra/db/tries/SlicedTrieTest.java
@@ -37,6 +37,7 @@
 
 import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.asString;
 import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.assertSameContent;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.byteComparableVersion;
 import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.generateKeys;
 import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.makeInMemoryTrie;
 import static java.util.Arrays.asList;
@@ -55,13 +56,13 @@ public class SlicedTrieTest
     "s",
     "q",
     "\000",
-    "\777",
-    "\777\000",
-    "\000\777",
+    "\377",
+    "\377\000",
+    "\000\377",
     "\000\000",
     "\000\000\000",
-    "\000\000\777",
-    "\777\777"
+    "\000\000\377",
+    "\377\377"
     });
     public static final ByteComparable[] KEYS = toByteComparable(new String[]{
     "test1",
@@ -75,14 +76,15 @@ public class SlicedTrieTest
     "sort",
     "sorting",
     "square",
-    "\777\000",
-    "\000\777",
+    "\377\000",
+    "\000\377",
     "\000\000",
     "\000\000\000",
-    "\000\000\777",
-    "\777\777"
+    "\000\000\377",
+    "\377\377"
     });
-    public static final Comparator<ByteComparable> BYTE_COMPARABLE_COMPARATOR = (bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, Trie.BYTE_COMPARABLE_VERSION);
+
+    public static final Comparator<ByteComparable> BYTE_COMPARABLE_COMPARATOR = (bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, byteComparableVersion);
     private static final int COUNT = 15000;
     Random rand = new Random();
 
@@ -95,7 +97,7 @@ public void testIntersectRangeDirect()
     public void testIntersectRange(int count)
     {
         ByteComparable[] src1 = generateKeys(rand, count);
-        NavigableMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, Trie.BYTE_COMPARABLE_VERSION));
+        NavigableMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, byteComparableVersion));
 
         InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
 
@@ -106,7 +108,7 @@ public void testIntersectRange(int count)
         {
             ByteComparable l = rand.nextBoolean() ? InMemoryTrieTestBase.generateKey(rand) : src1[rand.nextInt(src1.length)];
             ByteComparable r = rand.nextBoolean() ? InMemoryTrieTestBase.generateKey(rand) : src1[rand.nextInt(src1.length)];
-            int cmp = ByteComparable.compare(l, r, Trie.BYTE_COMPARABLE_VERSION);
+            int cmp = ByteComparable.compare(l, r, byteComparableVersion);
             if (cmp > 0)
             {
                 ByteComparable t = l;
@@ -125,14 +127,14 @@ public void testIntersectRange(int count)
     private static ByteComparable[] toByteComparable(String[] keys)
     {
         return Arrays.stream(keys)
-                     .map(x -> ByteComparable.fixedLength(x.getBytes(StandardCharsets.UTF_8)))
+                     .map(x -> ByteComparable.preencoded(byteComparableVersion, x.getBytes(StandardCharsets.UTF_8)))
                      .toArray(ByteComparable[]::new);
     }
 
     @Test
     public void testSingletonSubtrie()
     {
-        Arrays.sort(BOUNDARIES, (a, b) -> ByteComparable.compare(a, b, ByteComparable.Version.OSS50));
+        Arrays.sort(BOUNDARIES, (a, b) -> ByteComparable.compare(a, b, byteComparableVersion));
         for (int li = -1; li < BOUNDARIES.length; ++li)
         {
             ByteComparable l = li < 0 ? null : BOUNDARIES[li];
@@ -147,9 +149,9 @@ public void testSingletonSubtrie()
 
                     for (ByteComparable key : KEYS)
                     {
-                        int cmp1 = l != null ? ByteComparable.compare(key, l, ByteComparable.Version.OSS50) : 1;
-                        int cmp2 = r != null ? ByteComparable.compare(r, key, ByteComparable.Version.OSS50) : 1;
-                        Trie<Boolean> ix = new SlicedTrie<>(Trie.singleton(key, true), l, includeLeft, r, includeRight);
+                        int cmp1 = l != null ? ByteComparable.compare(key, l, byteComparableVersion) : 1;
+                        int cmp2 = r != null ? ByteComparable.compare(r, key, byteComparableVersion) : 1;
+                        Trie<Boolean> ix = new SlicedTrie<>(Trie.singleton(key, byteComparableVersion, true), l, includeLeft, r, includeRight);
                         boolean expected = true;
                         if (cmp1 < 0 || cmp1 == 0 && !includeLeft)
                             expected = false;
@@ -162,10 +164,10 @@ public void testSingletonSubtrie()
                             System.err.println(ix.dump());
                             Assert.fail(String.format("Failed on range %s%s,%s%s key %s expected %s got %s\n",
                                                       includeLeft ? "[" : "(",
-                                                      l != null ? l.byteComparableAsString(ByteComparable.Version.OSS50) : null,
-                                                      r != null ? r.byteComparableAsString(ByteComparable.Version.OSS50) : null,
+                                                      l != null ? l.byteComparableAsString(byteComparableVersion) : null,
+                                                      r != null ? r.byteComparableAsString(byteComparableVersion) : null,
                                                       includeRight ? "]" : ")",
-                                                      key.byteComparableAsString(ByteComparable.Version.OSS50),
+                                                      key.byteComparableAsString(byteComparableVersion),
                                                       expected,
                                                       actual));
                         }
@@ -228,8 +230,8 @@ public void testMergeSubtrie(int mergeCount)
             tries.add(makeInMemoryTrie(Arrays.copyOfRange(KEYS,
                                                            KEYS.length * i / mergeCount,
                                                            KEYS.length * (i + 1) / mergeCount),
-                                        content1,
-                                        true));
+                                       content1,
+                                       true));
         }
         Trie<ByteBuffer> trie1 = Trie.mergeDistinct(tries);
 
@@ -304,34 +306,51 @@ private static Trie<Integer> singleLevelIntTrie(int childs)
         return new Trie<Integer>()
         {
             @Override
-            protected Cursor<Integer> cursor()
+            protected Cursor<Integer> cursor(Direction direction)
             {
-                return new singleLevelCursor();
+                return new singleLevelCursor(direction);
             }
 
             class singleLevelCursor implements Cursor<Integer>
             {
+                final Direction direction;
                 int current = -1;
 
+                singleLevelCursor(Direction direction)
+                {
+                    this.direction = direction;
+                    current = direction.select(-1, childs);
+                }
+
                 @Override
                 public int advance()
                 {
-                    ++current;
+                    current += direction.increase;
                     return depth();
                 }
 
                 @Override
-                public int skipChildren()
+                public int skipTo(int depth, int transition)
                 {
-                    return advance();
+                    if (depth > 1)
+                        return advance();
+                    if (depth < 1)
+                        transition = direction.select(childs, -1);
+
+                    if (direction.isForward())
+                        current = Math.max(0, transition);
+                    else
+                        current = Math.min(childs - 1, transition);
+
+                    return depth();
                 }
 
                 @Override
                 public int depth()
                 {
-                    if (current == -1)
+                    if (current == direction.select(-1, childs))
                         return 0;
-                    if (current < childs)
+                    if (direction.inLoop(current, 0, childs - 1))
                         return 1;
                     return -1;
                 }
@@ -339,13 +358,31 @@ public int depth()
                 @Override
                 public int incomingTransition()
                 {
-                    return current;
+                    return current >= childs ? -1 : current;
                 }
 
                 @Override
                 public Integer content()
                 {
-                    return current;
+                    return current == direction.select(-1, childs) ? -1 : current;
+                }
+
+                @Override
+                public Direction direction()
+                {
+                    return direction;
+                }
+
+                @Override
+                public ByteComparable.Version byteComparableVersion()
+                {
+                    return byteComparableVersion;
+                }
+
+                @Override
+                public Trie<Integer> tailTrie()
+                {
+                    throw new UnsupportedOperationException("tailTrie on test cursor");
                 }
             }
         };
@@ -355,7 +392,7 @@ public Integer content()
     private static ByteComparable of(int value)
     {
         assert value >= 0 && value <= Byte.MAX_VALUE;
-        return ByteComparable.fixedLength(new byte[]{ (byte)value });
+        return ByteComparable.preencoded(byteComparableVersion, new byte[]{ (byte)value });
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/db/tries/TrieToDotTest.java b/test/unit/org/apache/cassandra/db/tries/TrieToDotTest.java
index 41c66aaaeb60..b10de587bba3 100644
--- a/test/unit/org/apache/cassandra/db/tries/TrieToDotTest.java
+++ b/test/unit/org/apache/cassandra/db/tries/TrieToDotTest.java
@@ -20,14 +20,14 @@
 
 import org.junit.Test;
 
-import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 public class TrieToDotTest
 {
     @Test
     public void testToDotContent() throws Exception
     {
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+        InMemoryTrie<String> trie = InMemoryTrie.shortLived(ByteComparable.Version.OSS50);
         String s = "Trie node types and manipulation mechanisms. The main purpose of this is to allow for handling tries directly as" +
                    " they are on disk without any serialization, and to enable the creation of such files.";
         s = s.toLowerCase();
@@ -36,6 +36,6 @@ public void testToDotContent() throws Exception
 
         System.out.println(trie.process(new TrieToDot(Object::toString,
                                                       x -> Character.toString((char) ((int) x)),
-                                                      true)));
+                                                      true), Direction.FORWARD));
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/tries/TrieToMermaidTest.java b/test/unit/org/apache/cassandra/db/tries/TrieToMermaidTest.java
index bd691bbd6e91..f19c1eb53e09 100644
--- a/test/unit/org/apache/cassandra/db/tries/TrieToMermaidTest.java
+++ b/test/unit/org/apache/cassandra/db/tries/TrieToMermaidTest.java
@@ -20,14 +20,14 @@
 
 import org.junit.Test;
 
-import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 public class TrieToMermaidTest
 {
     @Test
     public void testToMermaidContent() throws Exception
     {
-        InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+        InMemoryTrie<Object> trie = InMemoryTrie.shortLived(ByteComparable.Version.OSS50);
         // This was used as a basis the graphs in BTIFormat.md
         String s = "a allow an and any are as node of on the this to trie types with without";
         s = s.toLowerCase();
@@ -36,6 +36,6 @@ public void testToMermaidContent() throws Exception
 
         System.out.println(trie.process(new TrieToMermaid(Object::toString,
                                                       x -> Character.toString((char) ((int) x)),
-                                                      false)));
+                                                      false), Direction.FORWARD));
     }
 }
diff --git a/test/unit/org/apache/cassandra/db/virtual/CachesTableTest.java b/test/unit/org/apache/cassandra/db/virtual/CachesTableTest.java
new file mode 100644
index 000000000000..0cda1a0f6826
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/virtual/CachesTableTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+
+import static org.junit.Assert.assertFalse;
+
+public class CachesTableTest
+{
+    @BeforeClass
+    public static void setup() throws Exception
+    {
+        CQLTester.setUpClass();
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        CachesTable cachesTable = new CachesTable("test");
+        AbstractVirtualTable.DataSet dataSet = cachesTable.data();
+        assertFalse(dataSet.isEmpty());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java b/test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java
index 3863cbc18972..cf15e2121714 100644
--- a/test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java
@@ -57,102 +57,94 @@ public void testSelectAllWhenGossipInfoIsEmpty() throws Throwable
     @Test
     public void testSelectAllWithStateTransitions() throws Throwable
     {
-        try
-        {
-            requireNetwork(); // triggers gossiper
-
-            ConcurrentMap<InetAddressAndPort, EndpointState> states = Gossiper.instance.endpointStateMap;
-            Awaitility.await().until(() -> !states.isEmpty());
-            Map.Entry<InetAddressAndPort, EndpointState> entry = states.entrySet().stream().findFirst()
-                    .orElseThrow(AssertionError::new);
-            InetAddressAndPort endpoint = entry.getKey();
-            EndpointState localState = new EndpointState(entry.getValue());
-
-            Supplier<Map<InetAddressAndPort, EndpointState>> endpointStateMapSupplier = () -> new HashMap<InetAddressAndPort, EndpointState>() {{put(endpoint, localState);}};
-
-            VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace("vts_2",
-                                                                          of(new GossipInfoTable("vts_2", endpointStateMapSupplier))));
-
-            UntypedResultSet resultSet = execute("SELECT * FROM vts_2.gossip_info");
-
-            assertThat(resultSet.size()).isEqualTo(1);
-            UntypedResultSet.Row row = resultSet.one();
-            assertThat(row.getColumns().size()).isEqualTo(66);
-
-            assertThat(endpoint).isNotNull();
-            assertThat(localState).isNotNull();
-            assertThat(row.getInetAddress("address")).isEqualTo(endpoint.getAddress());
-            assertThat(row.getInt("port")).isEqualTo(endpoint.getPort());
-            assertThat(row.getString("hostname")).isEqualTo(endpoint.getHostName());
-            assertThat(row.getInt("generation")).isEqualTo(localState.getHeartBeatState().getGeneration());
-            assertThat(row.getInt("heartbeat")).isNotNull();
-
-            assertValue(row, "status", localState, ApplicationState.STATUS);
-            assertValue(row, "load", localState, ApplicationState.LOAD);
-            assertValue(row, "schema", localState, ApplicationState.SCHEMA);
-            assertValue(row, "dc", localState, ApplicationState.DC);
-            assertValue(row, "rack", localState, ApplicationState.RACK);
-            assertValue(row, "release_version", localState, ApplicationState.RELEASE_VERSION);
-            assertValue(row, "removal_coordinator", localState, ApplicationState.REMOVAL_COORDINATOR);
-            assertValue(row, "internal_ip", localState, ApplicationState.INTERNAL_IP);
-            assertValue(row, "rpc_address", localState, ApplicationState.RPC_ADDRESS);
-            assertValue(row, "severity", localState, ApplicationState.SEVERITY);
-            assertValue(row, "net_version", localState, ApplicationState.NET_VERSION);
-            assertValue(row, "host_id", localState, ApplicationState.HOST_ID);
-            assertValue(row, "rpc_ready", localState, ApplicationState.RPC_READY);
-            assertValue(row, "internal_address_and_port", localState, ApplicationState.INTERNAL_ADDRESS_AND_PORT);
-            assertValue(row, "native_address_and_port", localState, ApplicationState.NATIVE_ADDRESS_AND_PORT);
-            assertValue(row, "status_with_port", localState, ApplicationState.STATUS_WITH_PORT);
-            assertValue(row, "sstable_versions", localState, ApplicationState.SSTABLE_VERSIONS);
-            assertValue(row, "disk_usage", localState, ApplicationState.DISK_USAGE);
-            assertValue(row, "x_11_padding", localState, ApplicationState.X_11_PADDING);
-            assertValue(row, "x1", localState, ApplicationState.X1);
-            assertValue(row, "x2", localState, ApplicationState.X2);
-            assertValue(row, "x3", localState, ApplicationState.X3);
-            assertValue(row, "x4", localState, ApplicationState.X4);
-            assertValue(row, "x5", localState, ApplicationState.X5);
-            assertValue(row, "x6", localState, ApplicationState.X6);
-            assertValue(row, "x7", localState, ApplicationState.X7);
-            assertValue(row, "x8", localState, ApplicationState.X8);
-            assertValue(row, "x9", localState, ApplicationState.X9);
-            assertValue(row, "x10", localState, ApplicationState.X10);
-
-            assertVersion(row, "status_version", localState, ApplicationState.STATUS);
-            assertVersion(row, "load_version", localState, ApplicationState.LOAD);
-            assertVersion(row, "schema_version", localState, ApplicationState.SCHEMA);
-            assertVersion(row, "dc_version", localState, ApplicationState.DC);
-            assertVersion(row, "rack_version", localState, ApplicationState.RACK);
-            assertVersion(row, "release_version_version", localState, ApplicationState.RELEASE_VERSION);
-            assertVersion(row, "removal_coordinator_version", localState, ApplicationState.REMOVAL_COORDINATOR);
-            assertVersion(row, "internal_ip_version", localState, ApplicationState.INTERNAL_IP);
-            assertVersion(row, "rpc_address_version", localState, ApplicationState.RPC_ADDRESS);
-            assertVersion(row, "severity_version", localState, ApplicationState.SEVERITY);
-            assertVersion(row, "net_version_version", localState, ApplicationState.NET_VERSION);
-            assertVersion(row, "host_id_version", localState, ApplicationState.HOST_ID);
-            assertVersion(row, "tokens_version", localState, ApplicationState.TOKENS);
-            assertVersion(row, "rpc_ready_version", localState, ApplicationState.RPC_READY);
-            assertVersion(row, "internal_address_and_port_version", localState, ApplicationState.INTERNAL_ADDRESS_AND_PORT);
-            assertVersion(row, "native_address_and_port_version", localState, ApplicationState.NATIVE_ADDRESS_AND_PORT);
-            assertVersion(row, "status_with_port_version", localState, ApplicationState.STATUS_WITH_PORT);
-            assertVersion(row, "sstable_versions_version", localState, ApplicationState.SSTABLE_VERSIONS);
-            assertVersion(row, "disk_usage_version", localState, ApplicationState.DISK_USAGE);
-            assertVersion(row, "x_11_padding", localState, ApplicationState.X_11_PADDING);
-            assertVersion(row, "x1", localState, ApplicationState.X1);
-            assertVersion(row, "x2", localState, ApplicationState.X2);
-            assertVersion(row, "x3", localState, ApplicationState.X3);
-            assertVersion(row, "x4", localState, ApplicationState.X4);
-            assertVersion(row, "x5", localState, ApplicationState.X5);
-            assertVersion(row, "x6", localState, ApplicationState.X6);
-            assertVersion(row, "x7", localState, ApplicationState.X7);
-            assertVersion(row, "x8", localState, ApplicationState.X8);
-            assertVersion(row, "x9", localState, ApplicationState.X9);
-            assertVersion(row, "x10", localState, ApplicationState.X10);
-        }
-        finally
-        {
-            // clean up the gossip states
-            Gossiper.instance.clearUnsafe();
-        }
+        requireNetwork(); // triggers gossiper
+
+        ConcurrentMap<InetAddressAndPort, EndpointState> states = Gossiper.instance.endpointStateMap;
+        Awaitility.await().until(() -> !states.isEmpty());
+        Map.Entry<InetAddressAndPort, EndpointState> entry = states.entrySet().stream().findFirst()
+                .orElseThrow(AssertionError::new);
+        InetAddressAndPort endpoint = entry.getKey();
+        EndpointState localState = new EndpointState(entry.getValue());
+
+        Supplier<Map<InetAddressAndPort, EndpointState>> endpointStateMapSupplier = () -> new HashMap<InetAddressAndPort, EndpointState>() {{put(endpoint, localState);}};
+
+        VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace("vts_2",
+                                                                      of(new GossipInfoTable("vts_2", endpointStateMapSupplier))));
+
+        UntypedResultSet resultSet = execute("SELECT * FROM vts_2.gossip_info");
+
+        assertThat(resultSet.size()).isEqualTo(1);
+        UntypedResultSet.Row row = resultSet.one();
+        assertThat(row.getColumns().size()).isEqualTo(66);
+
+        assertThat(endpoint).isNotNull();
+        assertThat(localState).isNotNull();
+        assertThat(row.getInetAddress("address")).isEqualTo(endpoint.getAddress());
+        assertThat(row.getInt("port")).isEqualTo(endpoint.getPort());
+        assertThat(row.getString("hostname")).isEqualTo(endpoint.getHostName());
+        assertThat(row.getInt("generation")).isEqualTo(localState.getHeartBeatState().getGeneration());
+        assertThat(row.getInt("heartbeat")).isNotNull();
+
+        assertValue(row, "status", localState, ApplicationState.STATUS);
+        assertValue(row, "load", localState, ApplicationState.LOAD);
+        assertValue(row, "schema", localState, ApplicationState.SCHEMA);
+        assertValue(row, "dc", localState, ApplicationState.DC);
+        assertValue(row, "rack", localState, ApplicationState.RACK);
+        assertValue(row, "release_version", localState, ApplicationState.RELEASE_VERSION);
+        assertValue(row, "removal_coordinator", localState, ApplicationState.REMOVAL_COORDINATOR);
+        assertValue(row, "internal_ip", localState, ApplicationState.INTERNAL_IP);
+        assertValue(row, "rpc_address", localState, ApplicationState.RPC_ADDRESS);
+        assertValue(row, "severity", localState, ApplicationState.SEVERITY);
+        assertValue(row, "net_version", localState, ApplicationState.NET_VERSION);
+        assertValue(row, "host_id", localState, ApplicationState.HOST_ID);
+        assertValue(row, "rpc_ready", localState, ApplicationState.RPC_READY);
+        assertValue(row, "internal_address_and_port", localState, ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+        assertValue(row, "native_address_and_port", localState, ApplicationState.NATIVE_ADDRESS_AND_PORT);
+        assertValue(row, "status_with_port", localState, ApplicationState.STATUS_WITH_PORT);
+        assertValue(row, "sstable_versions", localState, ApplicationState.SSTABLE_VERSIONS);
+        assertValue(row, "disk_usage", localState, ApplicationState.DISK_USAGE);
+        assertValue(row, "x_11_padding", localState, ApplicationState.X_11_PADDING);
+        assertValue(row, "x1", localState, ApplicationState.X1);
+        assertValue(row, "x2", localState, ApplicationState.X2);
+        assertValue(row, "x3", localState, ApplicationState.X3);
+        assertValue(row, "x4", localState, ApplicationState.X4);
+        assertValue(row, "x5", localState, ApplicationState.X5);
+        assertValue(row, "x6", localState, ApplicationState.X6);
+        assertValue(row, "x7", localState, ApplicationState.X7);
+        assertValue(row, "x8", localState, ApplicationState.X8);
+        assertValue(row, "x9", localState, ApplicationState.X9);
+        assertValue(row, "x10", localState, ApplicationState.X10);
+
+        assertVersion(row, "status_version", localState, ApplicationState.STATUS);
+        assertVersion(row, "load_version", localState, ApplicationState.LOAD);
+        assertVersion(row, "schema_version", localState, ApplicationState.SCHEMA);
+        assertVersion(row, "dc_version", localState, ApplicationState.DC);
+        assertVersion(row, "rack_version", localState, ApplicationState.RACK);
+        assertVersion(row, "release_version_version", localState, ApplicationState.RELEASE_VERSION);
+        assertVersion(row, "removal_coordinator_version", localState, ApplicationState.REMOVAL_COORDINATOR);
+        assertVersion(row, "internal_ip_version", localState, ApplicationState.INTERNAL_IP);
+        assertVersion(row, "rpc_address_version", localState, ApplicationState.RPC_ADDRESS);
+        assertVersion(row, "severity_version", localState, ApplicationState.SEVERITY);
+        assertVersion(row, "net_version_version", localState, ApplicationState.NET_VERSION);
+        assertVersion(row, "host_id_version", localState, ApplicationState.HOST_ID);
+        assertVersion(row, "tokens_version", localState, ApplicationState.TOKENS);
+        assertVersion(row, "rpc_ready_version", localState, ApplicationState.RPC_READY);
+        assertVersion(row, "internal_address_and_port_version", localState, ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+        assertVersion(row, "native_address_and_port_version", localState, ApplicationState.NATIVE_ADDRESS_AND_PORT);
+        assertVersion(row, "status_with_port_version", localState, ApplicationState.STATUS_WITH_PORT);
+        assertVersion(row, "sstable_versions_version", localState, ApplicationState.SSTABLE_VERSIONS);
+        assertVersion(row, "disk_usage_version", localState, ApplicationState.DISK_USAGE);
+        assertVersion(row, "x_11_padding", localState, ApplicationState.X_11_PADDING);
+        assertVersion(row, "x1", localState, ApplicationState.X1);
+        assertVersion(row, "x2", localState, ApplicationState.X2);
+        assertVersion(row, "x3", localState, ApplicationState.X3);
+        assertVersion(row, "x4", localState, ApplicationState.X4);
+        assertVersion(row, "x5", localState, ApplicationState.X5);
+        assertVersion(row, "x6", localState, ApplicationState.X6);
+        assertVersion(row, "x7", localState, ApplicationState.X7);
+        assertVersion(row, "x8", localState, ApplicationState.X8);
+        assertVersion(row, "x9", localState, ApplicationState.X9);
+        assertVersion(row, "x10", localState, ApplicationState.X10);
     }
 
     private void assertValue(UntypedResultSet.Row row, String column, EndpointState localState, ApplicationState key)
diff --git a/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java b/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java
index 6e8a13612b35..bf72a9fcec32 100644
--- a/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java
@@ -30,11 +30,12 @@
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -76,11 +77,11 @@ public void testSelectAll() throws Throwable
 
         String directory = String.format("/some/datadir/%s/%s-%s", cfs.metadata.keyspace, cfs.metadata.name, cfs.metadata.id.asUUID());
 
-        CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
+        AbstractTableOperation compactionHolder = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, directory);
+                return new OperationProgress(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, directory);
             }
 
             public boolean isGlobal()
@@ -89,14 +90,14 @@ public boolean isGlobal()
             }
         };
 
-        CompactionManager.instance.active.beginCompaction(compactionHolder);
+        try (NonThrowingCloseable c = CompactionManager.instance.active.onOperationStart(compactionHolder))
+        {
+            UntypedResultSet result = execute("SELECT * FROM vts.sstable_tasks");
+            assertRows(result, row(CQLTester.KEYSPACE, currentTable(), compactionId, 1.0 * bytesCompacted / bytesTotal,
+                                   OperationType.COMPACTION.toString().toLowerCase(), bytesCompacted, sstables.size(),
+                                   directory, bytesTotal, AbstractTableOperation.Unit.BYTES.toString()));
+        }
         UntypedResultSet result = execute("SELECT * FROM vts.sstable_tasks");
-        assertRows(result, row(CQLTester.KEYSPACE, currentTable(), compactionId, 1.0 * bytesCompacted / bytesTotal,
-                OperationType.COMPACTION.toString().toLowerCase(), bytesCompacted, sstables.size(),
-                directory, bytesTotal, CompactionInfo.Unit.BYTES.toString()));
-
-        CompactionManager.instance.active.finishCompaction(compactionHolder);
-        result = execute("SELECT * FROM vts.sstable_tasks");
         assertEmpty(result);
     }
 }
diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
index f23e50521ac7..240361b0e2e3 100644
--- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
+++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
@@ -20,6 +20,7 @@
 import java.net.UnknownHostException;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
@@ -28,8 +29,10 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
 import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.RangeStreamer.FetchReplica;
@@ -42,23 +45,43 @@
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.streaming.StreamOperation;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
-
+@RunWith(BMUnitRunner.class)
 public class BootStrapperTest
 {
     static IPartitioner oldPartitioner;
 
     static Predicate<Replica> originalAlivePredicate = RangeStreamer.ALIVE_PREDICATE;
+    private static AtomicBoolean nonOptimizationHit = new AtomicBoolean(false);
+    private static AtomicBoolean optimizationHit = new AtomicBoolean(false);
+    private static final IFailureDetector mockFailureDetector = new IFailureDetector()
+    {
+        public boolean isAlive(InetAddressAndPort ep)
+        {
+            return true;
+        }
+
+        public void interpret(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
+        public void report(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
+        public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener) { throw new UnsupportedOperationException(); }
+        public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener) { throw new UnsupportedOperationException(); }
+        public void remove(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
+        public void forceConviction(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
+    };
+
     @BeforeClass
     public static void setup() throws ConfigurationException
     {
         DatabaseDescriptor.daemonInitialization();
         oldPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
-        SchemaLoader.startGossiper();
         SchemaLoader.prepareServer();
+        SchemaLoader.startGossiper();
         SchemaLoader.schemaDefinition("BootStrapperTest");
         RangeStreamer.ALIVE_PREDICATE = Predicates.alwaysTrue();
     }
@@ -83,6 +106,63 @@ public void testSourceTargetComputation() throws UnknownHostException
         }
     }
 
+    @Test
+    @BMRules(rules = { @BMRule(name = "Make sure the non-optimized path is picked up for some operations",
+                               targetClass = "org.apache.cassandra.dht.RangeStreamer",
+                               targetMethod = "convertPreferredEndpointsToWorkMap(EndpointsByReplica)",
+                               action = "org.apache.cassandra.dht.BootStrapperTest.nonOptimizationHit()"),
+                       @BMRule(name = "Make sure the optimized path is picked up for some operations",
+                               targetClass = "org.apache.cassandra.dht.RangeStreamer",
+                               targetMethod = "getOptimizedWorkMap(EndpointsByReplica,Collection,String)",
+                               action = "org.apache.cassandra.dht.BootStrapperTest.optimizationHit()") })
+    public void testStreamingCandidatesOptmizationSkip() throws UnknownHostException
+    {
+        testSkipStreamingCandidatesOptmizationFeatureFlag(true, true, false);
+        testSkipStreamingCandidatesOptmizationFeatureFlag(false, true, true);
+    }
+
+    private void testSkipStreamingCandidatesOptmizationFeatureFlag(boolean disableOptimization, boolean nonOptimizedPathHit, boolean optimizedPathHit) throws UnknownHostException
+    {
+        try
+        {
+            nonOptimizationHit.set(false);
+            optimizationHit.set(false);
+            CassandraRelevantProperties.SKIP_OPTIMAL_STREAMING_CANDIDATES_CALCULATION.setBoolean(disableOptimization);
+
+            for (String keyspaceName : Schema.instance.distributedKeyspaces().names())
+            {
+                StorageService ss = StorageService.instance;
+                TokenMetadata tmd = ss.getTokenMetadata();
+
+                generateFakeEndpoints(10);
+                Token myToken = tmd.partitioner.getRandomToken();
+                InetAddressAndPort myEndpoint = InetAddressAndPort.getByName("127.0.0.1");
+
+                assertEquals(10, tmd.sortedTokens().size());
+                RangeStreamer s = new RangeStreamer(tmd, null, myEndpoint, StreamOperation.BOOTSTRAP, true, DatabaseDescriptor.getEndpointSnitch(), new StreamStateStore(), mockFailureDetector, false, 1);
+                s.addRanges(keyspaceName, Keyspace.open(keyspaceName).getReplicationStrategy().getPendingAddressRanges(tmd, myToken, myEndpoint));
+            }
+
+            assertEquals(nonOptimizedPathHit, nonOptimizationHit.get());
+            assertEquals(optimizedPathHit, optimizationHit.get());
+        }
+        finally
+        {
+            CassandraRelevantProperties.SKIP_OPTIMAL_STREAMING_CANDIDATES_CALCULATION.reset();
+        }
+    }
+
+    // used by byteman
+    private static void nonOptimizationHit()
+    {
+        nonOptimizationHit.set(true);
+    }
+
+    private static void optimizationHit()
+    {
+        optimizationHit.set(true);
+    }
+
     private RangeStreamer testSourceTargetComputation(String keyspaceName, int numOldNodes, int replicationFactor) throws UnknownHostException
     {
         StorageService ss = StorageService.instance;
@@ -93,20 +173,6 @@ private RangeStreamer testSourceTargetComputation(String keyspaceName, int numOl
         InetAddressAndPort myEndpoint = InetAddressAndPort.getByName("127.0.0.1");
 
         assertEquals(numOldNodes, tmd.sortedTokens().size());
-        IFailureDetector mockFailureDetector = new IFailureDetector()
-        {
-            public boolean isAlive(InetAddressAndPort ep)
-            {
-                return true;
-            }
-
-            public void interpret(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
-            public void report(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
-            public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener) { throw new UnsupportedOperationException(); }
-            public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener) { throw new UnsupportedOperationException(); }
-            public void remove(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
-            public void forceConviction(InetAddressAndPort ep) { throw new UnsupportedOperationException(); }
-        };
         RangeStreamer s = new RangeStreamer(tmd, null, myEndpoint, StreamOperation.BOOTSTRAP, true, DatabaseDescriptor.getEndpointSnitch(), new StreamStateStore(), mockFailureDetector, false, 1);
         assertNotNull(Keyspace.open(keyspaceName));
         s.addRanges(keyspaceName, Keyspace.open(keyspaceName).getReplicationStrategy().getPendingAddressRanges(tmd, myToken, myEndpoint));
diff --git a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
index c0573d61f7dc..caa800c9a432 100644
--- a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
+++ b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.dht;
 
-import java.math.BigInteger;
 import java.util.List;
 
 import org.junit.AfterClass;
@@ -27,18 +26,15 @@
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.partitions.FilteredPartition;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.FBUtilities;
 
 /**
@@ -101,43 +97,4 @@ private void insert(String key)
         RowUpdateBuilder builder = new RowUpdateBuilder(Schema.instance.getTableMetadata(KEYSPACE1, CF), FBUtilities.timestampMicros(), key);
         builder.clustering("c").add("val", "asdf").build().applyUnsafe();
     }
-
-    static class BigIntegerToken extends ComparableObjectToken<BigInteger>
-    {
-        private static final long serialVersionUID = 1L;
-
-        public BigIntegerToken(BigInteger token)
-        {
-            super(token);
-        }
-
-        // convenience method for testing
-        public BigIntegerToken(String token) {
-            this(new BigInteger(token));
-        }
-
-        @Override
-        public IPartitioner getPartitioner()
-        {
-            return LengthPartitioner.instance;
-        }
-
-        @Override
-        public long getHeapSize()
-        {
-            return 0;
-        }
-
-        @Override
-        public long getLongValue()
-        {
-            return token.longValue();
-        }
-
-        @Override
-        public ByteSource asComparableBytes(ByteComparable.Version version)
-        {
-            return IntegerType.instance.asComparableBytes(IntegerType.instance.decompose(token), version);
-        }
-    }
 }
diff --git a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
index 24671e385666..a5d8c3c42ab9 100644
--- a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
+++ b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
@@ -22,25 +22,37 @@
 import java.util.*;
 import java.util.concurrent.ThreadLocalRandom;
 
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.db.BufferDecoratedKey;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.IntegerType;
 import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
-import org.apache.cassandra.dht.KeyCollisionTest.BigIntegerToken;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.*;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
 public class LengthPartitioner implements IPartitioner
 {
-    public static final BigInteger ZERO = new BigInteger("0");
-    public static final BigIntegerToken MINIMUM = new BigIntegerToken("-1");
+    public static final Long ZERO = 0L;
+    public static final BigIntegerToken MINIMUM = new BigIntegerToken(-1L);
+    public static final BigIntegerToken MAXIMUM = new BigIntegerToken(Long.MAX_VALUE);
+
+    private final Splitter splitter = new Splitter(this)
+    {
+        public Token tokenForValue(BigInteger value)
+        {
+            return new BigIntegerToken(value.longValue());
+        }
+
+        public BigInteger valueForToken(Token token)
+        {
+            return BigInteger.valueOf(((BigIntegerToken) token).getTokenValue());
+        }
+    };
 
     public static LengthPartitioner instance = new LengthPartitioner();
 
@@ -52,16 +64,31 @@ public DecoratedKey decorateKey(ByteBuffer key)
     public BigIntegerToken midpoint(Token ltoken, Token rtoken)
     {
         // the symbolic MINIMUM token should act as ZERO: the empty bit array
-        BigInteger left = ltoken.equals(MINIMUM) ? ZERO : ((BigIntegerToken)ltoken).token;
-        BigInteger right = rtoken.equals(MINIMUM) ? ZERO : ((BigIntegerToken)rtoken).token;
-        Pair<BigInteger,Boolean> midpair = FBUtilities.midpoint(left, right, 127);
+        Long left = ltoken.equals(MINIMUM) ? ZERO : ((BigIntegerToken) ltoken).token;
+        Long right = rtoken.equals(MINIMUM) ? ZERO : ((BigIntegerToken) rtoken).token;
+        Pair<BigInteger, Boolean> midpair = FBUtilities.midpoint(BigInteger.valueOf(left), BigInteger.valueOf(right), 127);
         // discard the remainder
-        return new BigIntegerToken(midpair.left);
+        return new BigIntegerToken(midpair.left.longValue());
     }
 
-    public Token split(Token left, Token right, double ratioToLeft)
+    public Token split(Token tleft, Token tright, double ratio)
     {
-        throw new UnsupportedOperationException();
+        assert ratio >= 0.0 && ratio <= 1.0;
+        BigIntegerToken ltoken = (BigIntegerToken) tleft;
+        BigIntegerToken rtoken = (BigIntegerToken) tright;
+
+        long left = ltoken.token;
+        long right = rtoken.token;
+
+        if (left < right)
+        {
+            return new BigIntegerToken((long) (((right - left) * ratio) + left));
+        }
+        else
+        {  // wrapping case
+            Long max = MAXIMUM.token;
+            return new BigIntegerToken((long) (((max + right) - left) * ratio) + left);
+        }
     }
 
     public BigIntegerToken getMinimumToken()
@@ -72,7 +99,7 @@ public BigIntegerToken getMinimumToken()
     @Override
     public Token getMaximumToken()
     {
-        return null;
+        return MAXIMUM;
     }
 
     public BigIntegerToken getRandomToken()
@@ -82,24 +109,35 @@ public BigIntegerToken getRandomToken()
 
     public BigIntegerToken getRandomToken(Random random)
     {
-        return new BigIntegerToken(BigInteger.valueOf(random.nextInt(15)));
+        return new BigIntegerToken((long) random.nextInt(15));
     }
 
-    private final Token.TokenFactory tokenFactory = new Token.TokenFactory() {
+    private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
+    {
         public ByteBuffer toByteArray(Token token)
         {
             BigIntegerToken bigIntegerToken = (BigIntegerToken) token;
-            return ByteBuffer.wrap(bigIntegerToken.token.toByteArray());
+            return ByteBufferUtil.bytes(bigIntegerToken.token);
         }
 
         public Token fromByteArray(ByteBuffer bytes)
         {
-            return new BigIntegerToken(new BigInteger(ByteBufferUtil.getArray(bytes)));
+            return new BigIntegerToken(ByteBufferUtil.toLong(bytes));
         }
 
+        @Override
         public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
         {
-            return fromByteArray(IntegerType.instance.fromComparableBytes(comparableBytes, version));
+            switch (version)
+            {
+                case LEGACY:
+                case OSS41:
+                    return new BigIntegerToken(ByteSourceInverse.getSignedLong(comparableBytes));
+                case OSS50:
+                    return new BigIntegerToken(ByteSourceInverse.getVariableLengthInteger(comparableBytes));
+                default:
+                    throw new AssertionError();
+            }
         }
 
         public String toString(Token token)
@@ -110,10 +148,12 @@ public String toString(Token token)
 
         public Token fromString(String string)
         {
-            return new BigIntegerToken(new BigInteger(string));
+            return new BigIntegerToken(Long.valueOf(string));
         }
 
-        public void validate(String token) {}
+        public void validate(String token)
+        {
+        }
     };
 
     public Token.TokenFactory getTokenFactory()
@@ -130,7 +170,7 @@ public BigIntegerToken getToken(ByteBuffer key)
     {
         if (key.remaining() == 0)
             return MINIMUM;
-        return new BigIntegerToken(BigInteger.valueOf(key.remaining()));
+        return new BigIntegerToken((long) key.remaining());
     }
 
     public Map<Token, Float> describeOwnership(List<Token> sortedTokens)
@@ -180,8 +220,72 @@ public AbstractType<?> partitionOrdering()
         return new PartitionerDefinedOrder(this);
     }
 
-    public AbstractType<?> partitionOrdering(AbstractType<?> partitionKeyType)
+    public Optional<Splitter> splitter()
+    {
+        return Optional.of(splitter);
+    }
+
+    static class BigIntegerToken extends ComparableObjectToken<Long>
     {
-        return new PartitionerDefinedOrder(this, partitionKeyType);
+        private static final long serialVersionUID = 1L;
+
+        public BigIntegerToken(Long token)
+        {
+            super(token);
+        }
+
+        // convenience method for testing
+        public BigIntegerToken(String token) {
+            this(Long.valueOf(token));
+        }
+
+        public ByteSource asComparableBytes(ByteComparable.Version version)
+        {
+            switch (version)
+            {
+                case LEGACY:
+                case OSS41:
+                    return ByteSource.of(token);
+                case OSS50:
+                    return ByteSource.variableLengthInteger(token);
+                default:
+                    throw new AssertionError();
+            }
+        }
+
+        @Override
+        public IPartitioner getPartitioner()
+        {
+            return LengthPartitioner.instance;
+        }
+
+        @Override
+        public long getHeapSize()
+        {
+            return 0;
+        }
+
+        @Override
+        public Token nextValidToken()
+        {
+            Long next = token.equals(Long.MAX_VALUE) ? ZERO
+                    : token + 1;
+            return new BigIntegerToken(next);
+        }
+
+        @Override
+        public long getLongValue()
+        {
+            return token;
+        }
+
+        @Override
+        public double size(Token next)
+        {
+            BigIntegerToken n = (BigIntegerToken) next;
+            long v = n.token - token;  // Overflow acceptable and desired.
+            double d = Math.scalb((double)v, -127); // Scale so that the full range is 1.
+            return d > 0.0 ? d : (d + 1.0); // Adjust for signed long, also making sure t.size(t) == 1.
+        }
     }
 }
diff --git a/test/unit/org/apache/cassandra/dht/OwnedRangesTest.java b/test/unit/org/apache/cassandra/dht/OwnedRangesTest.java
index c1fa621b4a3a..30de4ab8c385 100644
--- a/test/unit/org/apache/cassandra/dht/OwnedRangesTest.java
+++ b/test/unit/org/apache/cassandra/dht/OwnedRangesTest.java
@@ -23,13 +23,22 @@
 import java.util.List;
 
 import org.apache.commons.collections.CollectionUtils;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+
 import static org.junit.Assert.assertTrue;
 import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRanges;
 
 public class OwnedRangesTest
 {
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
     @Test
     public void testFilterRangesWithEmptySuperset()
     {
diff --git a/test/unit/org/apache/cassandra/dht/RangeIntersectsBoundsTest.java b/test/unit/org/apache/cassandra/dht/RangeIntersectsBoundsTest.java
new file mode 100644
index 000000000000..a60633707d70
--- /dev/null
+++ b/test/unit/org/apache/cassandra/dht/RangeIntersectsBoundsTest.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class RangeIntersectsBoundsTest
+{
+    @BeforeClass
+    public static void setupDD()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void rangeIntersectsBounds() throws Exception
+    {
+        Range<Token> all = new Range<>(new BigIntegerToken("0"), new BigIntegerToken("0"));
+        Range<Token> some = new Range<>(new BigIntegerToken("4"), new BigIntegerToken("8"));
+        Range<Token> someWrapped = new Range<>(some.right, some.left);
+
+        // Coda:
+        // l - matches left token of some range
+        // r - matches right token of some range
+        // b - below left token of some range
+        // a - above right token of some range
+        // i - inside some range (above left and below right
+        Bounds<Token> lr = new Bounds<>(new BigIntegerToken("4"), new BigIntegerToken("8"));
+        Bounds<Token> br = new Bounds<>(new BigIntegerToken("3"), new BigIntegerToken("8"));
+        Bounds<Token> bi = new Bounds<>(new BigIntegerToken("3"), new BigIntegerToken("7"));
+        Bounds<Token> ba = new Bounds<>(new BigIntegerToken("3"), new BigIntegerToken("9"));
+        Bounds<Token> la = new Bounds<>(new BigIntegerToken("4"), new BigIntegerToken("9"));
+        Bounds<Token> li = new Bounds<>(new BigIntegerToken("4"), new BigIntegerToken("7"));
+        Bounds<Token> ii = new Bounds<>(new BigIntegerToken("5"), new BigIntegerToken("7"));
+        Bounds<Token> ir = new Bounds<>(new BigIntegerToken("5"), new BigIntegerToken("8"));
+        Bounds<Token> bb = new Bounds<>(new BigIntegerToken("2"), new BigIntegerToken("3"));
+        Bounds<Token> aa = new Bounds<>(new BigIntegerToken("9"), new BigIntegerToken("10"));
+        Bounds<Token> bl = new Bounds<>(new BigIntegerToken("3"), new BigIntegerToken("4"));
+        Bounds<Token> ra = new Bounds<>(new BigIntegerToken("8"), new BigIntegerToken("9"));
+
+        assertTrue(all.intersects(lr));
+        assertTrue(all.intersects(br));
+        assertTrue(all.intersects(bi));
+        assertTrue(all.intersects(ba));
+        assertTrue(all.intersects(la));
+        assertTrue(all.intersects(li));
+        assertTrue(all.intersects(ii));
+        assertTrue(all.intersects(ir));
+        assertTrue(all.intersects(bb));
+        assertTrue(all.intersects(aa));
+        assertTrue(all.intersects(bl));
+        assertTrue(all.intersects(ra));
+
+        assertTrue(some.intersects(lr));
+        assertTrue(some.intersects(br));
+        assertTrue(some.intersects(bi));
+        assertTrue(some.intersects(ba));
+        assertTrue(some.intersects(la));
+        assertTrue(some.intersects(li));
+        assertTrue(some.intersects(ii));
+        assertTrue(some.intersects(ir));
+        assertFalse(some.intersects(bb));
+        assertFalse(some.intersects(aa));
+        assertFalse(some.intersects(bl));
+        assertTrue(some.intersects(ra));
+
+        assertTrue(someWrapped.intersects(lr));
+        assertTrue(someWrapped.intersects(br));
+        assertTrue(someWrapped.intersects(bi));
+        assertTrue(someWrapped.intersects(ba));
+        assertTrue(someWrapped.intersects(la));
+        assertTrue(someWrapped.intersects(li));
+        assertFalse(someWrapped.intersects(ii));
+        assertFalse(someWrapped.intersects(ir));
+        assertTrue(someWrapped.intersects(bb));
+        assertTrue(someWrapped.intersects(aa));
+        assertTrue(someWrapped.intersects(bl));
+        assertTrue(someWrapped.intersects(ra));
+    }
+
+    @Test
+    public void rangeIntersectsExcludingBounds() throws Exception
+    {
+        Range<Token> all = new Range<>(new BigIntegerToken("0"), new BigIntegerToken("0"));
+        Range<Token> some = new Range<>(new BigIntegerToken("4"), new BigIntegerToken("8"));
+        Range<Token> someWrapped = new Range<>(some.right, some.left);
+
+        // Coda:
+        // l - matches left token of some range
+        // r - matches right token of some range
+        // b - below left token of some range
+        // a - above right token of some range
+        // i - inside some range (above left and below right
+        ExcludingBounds<Token> lr = new ExcludingBounds<>(new BigIntegerToken("4"), new BigIntegerToken("8"));
+        ExcludingBounds<Token> br = new ExcludingBounds<>(new BigIntegerToken("3"), new BigIntegerToken("8"));
+        ExcludingBounds<Token> bi = new ExcludingBounds<>(new BigIntegerToken("3"), new BigIntegerToken("7"));
+        ExcludingBounds<Token> ba = new ExcludingBounds<>(new BigIntegerToken("3"), new BigIntegerToken("9"));
+        ExcludingBounds<Token> la = new ExcludingBounds<>(new BigIntegerToken("4"), new BigIntegerToken("9"));
+        ExcludingBounds<Token> li = new ExcludingBounds<>(new BigIntegerToken("4"), new BigIntegerToken("7"));
+        ExcludingBounds<Token> ii = new ExcludingBounds<>(new BigIntegerToken("5"), new BigIntegerToken("7"));
+        ExcludingBounds<Token> ir = new ExcludingBounds<>(new BigIntegerToken("5"), new BigIntegerToken("8"));
+        ExcludingBounds<Token> bb = new ExcludingBounds<>(new BigIntegerToken("2"), new BigIntegerToken("3"));
+        ExcludingBounds<Token> aa = new ExcludingBounds<>(new BigIntegerToken("9"), new BigIntegerToken("10"));
+        ExcludingBounds<Token> bl = new ExcludingBounds<>(new BigIntegerToken("3"), new BigIntegerToken("4"));
+        ExcludingBounds<Token> ra = new ExcludingBounds<>(new BigIntegerToken("8"), new BigIntegerToken("9"));
+
+        assertTrue(all.intersects(lr));
+        assertTrue(all.intersects(br));
+        assertTrue(all.intersects(bi));
+        assertTrue(all.intersects(ba));
+        assertTrue(all.intersects(la));
+        assertTrue(all.intersects(li));
+        assertTrue(all.intersects(ii));
+        assertTrue(all.intersects(ir));
+        assertTrue(all.intersects(bb));
+        assertTrue(all.intersects(aa));
+        assertTrue(all.intersects(bl));
+        assertTrue(all.intersects(ra));
+
+        assertTrue(some.intersects(lr));
+        assertTrue(some.intersects(br));
+        assertTrue(some.intersects(bi));
+        assertTrue(some.intersects(ba));
+        assertTrue(some.intersects(la));
+        assertTrue(some.intersects(li));
+        assertTrue(some.intersects(ii));
+        assertTrue(some.intersects(ir));
+        assertFalse(some.intersects(bb));
+        assertFalse(some.intersects(aa));
+        assertFalse(some.intersects(bl));
+        assertFalse(some.intersects(ra));
+
+        assertTrue(someWrapped.intersects(lr));
+        assertTrue(someWrapped.intersects(br));
+        assertTrue(someWrapped.intersects(bi));
+        assertTrue(someWrapped.intersects(ba));
+        assertTrue(someWrapped.intersects(la));
+        assertTrue(someWrapped.intersects(li));
+        assertFalse(someWrapped.intersects(ii));
+        assertFalse(someWrapped.intersects(ir));
+        assertTrue(someWrapped.intersects(bb));
+        assertTrue(someWrapped.intersects(aa));
+        assertFalse(someWrapped.intersects(bl));
+        assertFalse(someWrapped.intersects(ra));
+
+        Range<Token> range = new Range<>(Murmur3Partitioner.MINIMUM, new Murmur3Partitioner.LongToken(-1));
+        ExcludingBounds<Token> bounds = new ExcludingBounds<>(new Murmur3Partitioner.LongToken(-3248873570005575792L), Murmur3Partitioner.MINIMUM);
+
+        assertTrue(range.intersects(bounds));
+
+        range = new Range<>(new Murmur3Partitioner.LongToken(-1), Murmur3Partitioner.MINIMUM);
+
+        assertTrue(range.intersects(bounds));
+
+    }
+
+    @Test
+    public void rangeIntersectsIncludingExcludingBounds()
+    {
+        Range<Token> all = new Range<>(new BigIntegerToken("0"), new BigIntegerToken("0"));
+        Range<Token> some = new Range<>(new BigIntegerToken("4"), new BigIntegerToken("8"));
+        Range<Token> someWrapped = new Range<>(some.right, some.left);
+
+        // Coda:
+        // l - matches left token of some range
+        // r - matches right token of some range
+        // b - below left token of some range
+        // a - above right token of some range
+        // i - inside some range (above left and below right
+        IncludingExcludingBounds<Token> lr = new IncludingExcludingBounds<>(new BigIntegerToken("4"), new BigIntegerToken("8"));
+        IncludingExcludingBounds<Token> br = new IncludingExcludingBounds<>(new BigIntegerToken("3"), new BigIntegerToken("8"));
+        IncludingExcludingBounds<Token> bi = new IncludingExcludingBounds<>(new BigIntegerToken("3"), new BigIntegerToken("7"));
+        IncludingExcludingBounds<Token> ba = new IncludingExcludingBounds<>(new BigIntegerToken("3"), new BigIntegerToken("9"));
+        IncludingExcludingBounds<Token> la = new IncludingExcludingBounds<>(new BigIntegerToken("4"), new BigIntegerToken("9"));
+        IncludingExcludingBounds<Token> li = new IncludingExcludingBounds<>(new BigIntegerToken("4"), new BigIntegerToken("7"));
+        IncludingExcludingBounds<Token> ii = new IncludingExcludingBounds<>(new BigIntegerToken("5"), new BigIntegerToken("7"));
+        IncludingExcludingBounds<Token> ir = new IncludingExcludingBounds<>(new BigIntegerToken("5"), new BigIntegerToken("8"));
+        IncludingExcludingBounds<Token> bb = new IncludingExcludingBounds<>(new BigIntegerToken("2"), new BigIntegerToken("3"));
+        IncludingExcludingBounds<Token> aa = new IncludingExcludingBounds<>(new BigIntegerToken("9"), new BigIntegerToken("10"));
+        IncludingExcludingBounds<Token> bl = new IncludingExcludingBounds<>(new BigIntegerToken("3"), new BigIntegerToken("4"));
+        IncludingExcludingBounds<Token> ra = new IncludingExcludingBounds<>(new BigIntegerToken("8"), new BigIntegerToken("9"));
+
+        assertTrue(all.intersects(lr));
+        assertTrue(all.intersects(br));
+        assertTrue(all.intersects(bi));
+        assertTrue(all.intersects(ba));
+        assertTrue(all.intersects(la));
+        assertTrue(all.intersects(li));
+        assertTrue(all.intersects(ii));
+        assertTrue(all.intersects(ir));
+        assertTrue(all.intersects(bb));
+        assertTrue(all.intersects(aa));
+        assertTrue(all.intersects(bl));
+        assertTrue(all.intersects(ra));
+
+        assertTrue(some.intersects(lr));
+        assertTrue(some.intersects(br));
+        assertTrue(some.intersects(bi));
+        assertTrue(some.intersects(ba));
+        assertTrue(some.intersects(la));
+        assertTrue(some.intersects(li));
+        assertTrue(some.intersects(ii));
+        assertTrue(some.intersects(ir));
+        assertFalse(some.intersects(bb));
+        assertFalse(some.intersects(aa));
+        assertFalse(some.intersects(bl));
+        assertTrue(some.intersects(ra));
+
+        assertTrue(someWrapped.intersects(lr));
+        assertTrue(someWrapped.intersects(br));
+        assertTrue(someWrapped.intersects(bi));
+        assertTrue(someWrapped.intersects(ba));
+        assertTrue(someWrapped.intersects(la));
+        assertTrue(someWrapped.intersects(li));
+        assertFalse(someWrapped.intersects(ii));
+        assertFalse(someWrapped.intersects(ir));
+        assertTrue(someWrapped.intersects(bb));
+        assertTrue(someWrapped.intersects(aa));
+        assertFalse(someWrapped.intersects(bl));
+        assertTrue(someWrapped.intersects(ra));
+    }
+
+    /**
+     * Test that we handle partial bounds of the type x > n or x >= n which specifically have
+     * their right value as minimum.
+     */
+    @Test
+    public void rangeIntersectsPartialBounds()
+    {
+        Range<Token> range = new Range<>(Murmur3Partitioner.MINIMUM, new Murmur3Partitioner.LongToken(-1L));
+
+        Bounds<Token> boundsMatch = new Bounds<>(new Murmur3Partitioner.LongToken(-2L), Murmur3Partitioner.MINIMUM);
+        Bounds<Token> boundsNoMatch = new Bounds<>(new Murmur3Partitioner.LongToken(0L), Murmur3Partitioner.MINIMUM);
+
+        assertTrue(range.intersects(boundsMatch));
+        assertFalse(range.intersects(boundsNoMatch));
+
+        ExcludingBounds<Token> excBoundsMatch = new ExcludingBounds<>(new Murmur3Partitioner.LongToken(-2L), Murmur3Partitioner.MINIMUM);
+        ExcludingBounds<Token> excBoundsNoMatch = new ExcludingBounds<>(new Murmur3Partitioner.LongToken(-1L), Murmur3Partitioner.MINIMUM);
+
+        assertTrue(range.intersects(excBoundsMatch));
+        assertFalse(range.intersects(excBoundsNoMatch));
+
+        IncludingExcludingBounds<Token> incExcBoundsMatch = new IncludingExcludingBounds<>(new Murmur3Partitioner.LongToken(-2L), Murmur3Partitioner.MINIMUM);
+        IncludingExcludingBounds<Token> incExcBoundsNoMatch = new IncludingExcludingBounds<>(new Murmur3Partitioner.LongToken(0L), Murmur3Partitioner.MINIMUM);
+
+        assertTrue(range.intersects(incExcBoundsMatch));
+        assertFalse(range.intersects(incExcBoundsNoMatch));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/dht/SplitterTest.java b/test/unit/org/apache/cassandra/dht/SplitterTest.java
index 1de22ff8fc69..4d4e460f0d6c 100644
--- a/test/unit/org/apache/cassandra/dht/SplitterTest.java
+++ b/test/unit/org/apache/cassandra/dht/SplitterTest.java
@@ -36,6 +36,7 @@
 
 import static com.google.common.collect.Sets.newHashSet;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -78,7 +79,7 @@ public void testSplitOwnedRanges() {
 
         for (int i = 1; i <= (rt - lt); i++)
         {
-            List<Token> splits = splitter.splitOwnedRanges(i, Arrays.asList(new Splitter.WeightedRange(1.0d, range)), false);
+            List<Token> splits = splitter.splitOwnedRanges(i, Arrays.asList(new Splitter.WeightedRange(1.0d, range)), Splitter.SplitType.ALWAYS_SPLIT).boundaries;
             logger.info("{} splits of {} are: {}", i, range, splits);
             Assertions.assertThat(splits).hasSize(i);
         }
@@ -105,8 +106,7 @@ public void testWithWeight()
         IPartitioner partitioner = Murmur3Partitioner.instance;
         Splitter splitter = partitioner.splitter().get();
 
-        assertEquals(splitter.splitOwnedRanges(2, ranges, false), splitter.splitOwnedRanges(2, ranges2, false));
-        assertEquals(splitter.splitOwnedRanges(2, ranges, false), splitter.splitOwnedRanges(2, ranges3, false));
+        assertEquals(splitter.splitOwnedRanges(2, ranges, Splitter.SplitType.ALWAYS_SPLIT), splitter.splitOwnedRanges(2, ranges2, Splitter.SplitType.ALWAYS_SPLIT));
     }
 
     @Test
@@ -124,7 +124,7 @@ public void testWithWeight2()
         IPartitioner partitioner = Murmur3Partitioner.instance;
         Splitter splitter = partitioner.splitter().get();
 
-        assertEquals(splitter.splitOwnedRanges(2, ranges, false), splitter.splitOwnedRanges(2, ranges2, false));
+        assertEquals(splitter.splitOwnedRanges(2, ranges, Splitter.SplitType.ALWAYS_SPLIT), splitter.splitOwnedRanges(2, ranges2, Splitter.SplitType.ALWAYS_SPLIT));
     }
 
     private Range<Token> t(long left, long right)
@@ -139,8 +139,8 @@ private static void randomSplitTestNoVNodes(IPartitioner partitioner)
         for (int i = 0; i < 10000; i++)
         {
             List<Splitter.WeightedRange> localRanges = generateLocalRanges(1, r.nextInt(4) + 1, splitter, r, partitioner instanceof RandomPartitioner);
-            List<Token> boundaries = splitter.splitOwnedRanges(r.nextInt(9) + 1, localRanges, false);
-            assertTrue("boundaries = " + boundaries + " ranges = " + localRanges, assertRangeSizeEqual(localRanges, boundaries, partitioner, splitter, true));
+            Splitter.SplitResult result = splitter.splitOwnedRanges(r.nextInt(9) + 1, localRanges, Splitter.SplitType.ALWAYS_SPLIT);
+            assertTrue("boundaries = " + result.boundaries + " ranges = " + localRanges, assertRangeSizeEqual(localRanges, result, partitioner, splitter, Splitter.SplitType.ALWAYS_SPLIT));
         }
     }
 
@@ -148,27 +148,50 @@ private static void randomSplitTestVNodes(IPartitioner partitioner)
     {
         Splitter splitter = getSplitter(partitioner);
         Random r = new Random();
-        for (int i = 0; i < 10000; i++)
+        for (Splitter.SplitType splitType : Splitter.SplitType.values())
         {
-            // we need many tokens to be able to split evenly over the disks
-            int numTokens = 172 + r.nextInt(128);
-            int rf = r.nextInt(4) + 2;
-            int parts = r.nextInt(5) + 1;
-            List<Splitter.WeightedRange> localRanges = generateLocalRanges(numTokens, rf, splitter, r, partitioner instanceof RandomPartitioner);
-            List<Token> boundaries = splitter.splitOwnedRanges(parts, localRanges, true);
-            if (!assertRangeSizeEqual(localRanges, boundaries, partitioner, splitter, false))
-                fail(String.format("Could not split %d tokens with rf=%d into %d parts (localRanges=%s, boundaries=%s)", numTokens, rf, parts, localRanges, boundaries));
+            for (int i = 0; i < 10000; i++)
+            {
+                // we need many tokens to be able to split evenly over the disks
+                int numTokens = 172 + r.nextInt(128);
+                int rf = r.nextInt(4) + 2;
+                int parts = r.nextInt(5) + 1;
+                List<Splitter.WeightedRange> localRanges = generateLocalRanges(numTokens, rf, splitter, r, partitioner instanceof RandomPartitioner);
+
+                Splitter.SplitResult result = splitter.splitOwnedRanges(parts, localRanges, splitType);
+                if (!assertRangeSizeEqual(localRanges, result, partitioner, splitter, splitType))
+                    fail(String.format("Could not split %d tokens with rf=%d into %d parts (localRanges=%s, boundaries=%s, splitType=%s)",
+                                       numTokens, rf, parts, localRanges, result.boundaries, splitType));
+            }
         }
     }
 
-    private static boolean assertRangeSizeEqual(List<Splitter.WeightedRange> localRanges, List<Token> tokens, IPartitioner partitioner, Splitter splitter, boolean splitIndividualRanges)
+    private static boolean assertRangeSizeEqual(List<Splitter.WeightedRange> localRanges,
+                                                Splitter.SplitResult splitResult,
+                                                IPartitioner partitioner,
+                                                Splitter splitter,
+                                                Splitter.SplitType splitType)
     {
+        List<Token> boundaries = splitResult.boundaries;
+        boolean splitIndividualRanges = splitResult.rangesWereSplit;
+
+        // Check if the split type was respected. This is only relevant if there are two or more tokens because
+        // if the splitter cannot split at all, then the split result will indicate that no ranges were split regardless
+        // of the split type
+        if (boundaries.size() > 1)
+        {
+            if (splitType == Splitter.SplitType.ALWAYS_SPLIT)
+                assertTrue("Local ranges can only be split when SplitType forces it", splitIndividualRanges);
+            else if (splitType == Splitter.SplitType.ONLY_WHOLE)
+                assertFalse("Local ranges should not be split when SplitType doesn't force it", splitIndividualRanges);
+        }
+
         Token start = partitioner.getMinimumToken();
         List<BigInteger> splits = new ArrayList<>();
 
-        for (int i = 0; i < tokens.size(); i++)
+        for (int i = 0; i < boundaries.size(); i++)
         {
-            Token end = i == tokens.size() - 1 ? partitioner.getMaximumToken() : tokens.get(i);
+            Token end = i == boundaries.size() - 1 ? partitioner.getMaximumToken() : boundaries.get(i);
             splits.add(sumOwnedBetween(localRanges, start, end, splitter, splitIndividualRanges));
             start = end;
         }
@@ -209,7 +232,11 @@ private static BigInteger sumOwnedBetween(List<Splitter.WeightedRange> localRang
         return sum;
     }
 
-    private static List<Splitter.WeightedRange> generateLocalRanges(int numTokens, int rf, Splitter splitter, Random r, boolean randomPartitioner)
+    public static List<Splitter.WeightedRange> generateLocalRanges(int numTokens,
+                                                                   int rf,
+                                                                   Splitter splitter,
+                                                                   Random r,
+                                                                   boolean randomPartitioner)
     {
         int localTokens = numTokens * rf;
         List<Token> randomTokens = new ArrayList<>();
@@ -277,7 +304,6 @@ private static void testSplit(IPartitioner partitioner)
         // single range too small to be partitioned
         testSplit(partitioner, 1, newHashSet(Pair.create(1, 2)), newHashSet(Pair.create(1, 2)));
         testSplit(partitioner, 2, newHashSet(Pair.create(1, 2)), newHashSet(Pair.create(1, 2)));
-        testSplit(partitioner, 4, newHashSet(Pair.create(1, 4)), newHashSet(Pair.create(1, 4)));
         testSplit(partitioner, 8, newHashSet(Pair.create(1, 2)), newHashSet(Pair.create(1, 2)));
 
         // single wrapping range
@@ -290,7 +316,7 @@ private static void testSplit(IPartitioner partitioner)
         testSplit(partitioner, 2,
                   newHashSet(Pair.create(max.subtract(BigInteger.valueOf(8)), min)),
                   newHashSet(Pair.create(max.subtract(BigInteger.valueOf(8)), max.subtract(BigInteger.valueOf(4))),
-                             Pair.create(max.subtract(BigInteger.valueOf(4)), isRandom ? first : max)));
+                             Pair.create(max.subtract(BigInteger.valueOf(4)), min)));
         testSplit(partitioner, 2,
                   newHashSet(Pair.create(max.subtract(BigInteger.valueOf(8)), max)),
                   newHashSet(Pair.create(max.subtract(BigInteger.valueOf(8)), max.subtract(BigInteger.valueOf(4))),
@@ -544,7 +570,7 @@ private static Token getWrappedToken(IPartitioner partitioner, BigInteger positi
         return splitter.tokenForValue(position);
     }
 
-    private static Splitter getSplitter(IPartitioner partitioner)
+    public static Splitter getSplitter(IPartitioner partitioner)
     {
         return partitioner.splitter().orElseThrow(() -> new AssertionError(partitioner.getClass() + " must have a splitter"));
     }
diff --git a/test/unit/org/apache/cassandra/dht/tokenallocator/IsolatedTokenAllocatorTest.java b/test/unit/org/apache/cassandra/dht/tokenallocator/IsolatedTokenAllocatorTest.java
new file mode 100644
index 000000000000..0691df32d96b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/dht/tokenallocator/IsolatedTokenAllocatorTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht.tokenallocator;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.locator.RackInferringSnitch;
+import org.apache.cassandra.locator.TokenMetadata;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class IsolatedTokenAllocatorTest
+{
+    // This test ensures that as we increase shards, we maintain the invariant that lower level splits are
+    // always higher level splits.
+    @Test
+    public void testTokenAllocationForSingleNode() throws UnknownHostException
+    {
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        DatabaseDescriptor.setEndpointSnitch(new RackInferringSnitch());
+        var config = new Config();
+        config.num_tokens = 8;
+        DatabaseDescriptor.setConfig(config);
+        var tokenMetadata = new TokenMetadata();
+        var snitch = DatabaseDescriptor.getEndpointSnitch();
+        var networkTopology = new NetworkTopologyStrategy("0", tokenMetadata, snitch, Map.of());
+
+        var dc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+        // Initialize the token metadata with 8 nodes
+        var tokens = generateFakeEndpoints(tokenMetadata, networkTopology, 1, 1, 8, dc, "0");
+        tokens.sort(Token::compareTo);
+
+        var nodeTokens = tokenMetadata.sortedTokens();
+        assertEquals(8, nodeTokens.size());
+        assertArrayEquals(nodeTokens.toArray(), tokens.toArray());
+
+        // Inductively demonstrate that each higher level split contains all the previous splits.
+        List<Token> previousTokens = List.of();
+        for (int i = 1; i < 256; i++)
+        {
+            var newTokens = IsolatedTokenAllocator.allocateTokens(i, networkTopology);
+            assertEquals(i, newTokens.size());
+            assertTrue(newTokens.containsAll(previousTokens));
+            // The original 8 tokens must not be in the "new" tokens
+            assertTrue(newTokens.stream().noneMatch(nodeTokens::contains));
+            previousTokens = newTokens;
+        }
+    }
+
+    // Test confirms that the IsolatedTokenAllocator generates token splits in the same way that we generate new
+    // tokens for added nodes.
+    @Test
+    public void testTokenAllocationForMultipleNodesOneRack() throws UnknownHostException
+    {
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        DatabaseDescriptor.setEndpointSnitch(new RackInferringSnitch());
+        var config = new Config();
+        config.num_tokens = 8;
+        DatabaseDescriptor.setConfig(config);
+        var tokenMetadata = new TokenMetadata();
+        var snitch = DatabaseDescriptor.getEndpointSnitch();
+        var networkTopology = new NetworkTopologyStrategy("0", tokenMetadata, snitch, Map.of());
+
+        var dc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+
+        // Add the first node
+        var tokens1 = generateFakeEndpoints(tokenMetadata, networkTopology, 1, 1, 8, dc, "0");
+        // Generate the next set of tokens
+        var nextTokens = IsolatedTokenAllocator.allocateTokens(8, networkTopology);
+        // The newTokens should not include any tokens from tokens1
+        assertTrue(nextTokens.stream().noneMatch(tokens1::contains));
+        // We expect the newly added tokens to contain the splits defined in nextTokens
+        var tokens2 = generateFakeEndpoints(tokenMetadata, networkTopology, 2, 2, 8, dc, "0");
+        // tokens2 should contain nextTokens
+        assertTrue(tokens2.containsAll(nextTokens));
+
+        // Confirm for next 24 tokens (3 nodes worth)
+        nextTokens = IsolatedTokenAllocator.allocateTokens(24, networkTopology);
+        assertEquals(24, nextTokens.size());
+        var tokens345 = generateFakeEndpoints(tokenMetadata, networkTopology, 3, 5, 8, dc, "0");
+        assertTrue(tokens345.containsAll(nextTokens));
+    }
+
+    // Test confirms that the IsolatedTokenAllocator generates token splits in the same way that we generate new
+    // tokens for added nodes.
+    @Test
+    public void testTokenAllocationForMultiNodeMultiRack() throws UnknownHostException
+    {
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        DatabaseDescriptor.setEndpointSnitch(new RackInferringSnitch());
+        var config = new Config();
+        config.num_tokens = 8;
+        DatabaseDescriptor.setConfig(config);
+        var tokenMetadata = new TokenMetadata();
+        var snitch = DatabaseDescriptor.getEndpointSnitch();
+        var dc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
+        var rf = Map.of(dc, "3");
+        var networkTopology = new NetworkTopologyStrategy("0", tokenMetadata, snitch, rf);
+
+
+        var existingTokens = new ArrayList<>();
+        // Set up 1 node in each of 3 racks
+        for (int i = 0; i < 3; i++)
+           existingTokens.addAll(generateFakeEndpoints(tokenMetadata, networkTopology, 1, 1, 8, dc, Integer.toString(i)));
+
+        // Generate the next set of tokens
+        var nextTokens = IsolatedTokenAllocator.allocateTokens(16, networkTopology);
+        // The nextTokens should not include any tokens from existingTokens
+        assertTrue(nextTokens.stream().noneMatch(existingTokens::contains));
+        var newlyAddedTokens = new ArrayList<>();
+        for (int i = 0; i < 3; i++)
+            newlyAddedTokens.addAll(generateFakeEndpoints(tokenMetadata, networkTopology, 2, 3, 8, dc, Integer.toString(i)));
+        assertTrue(newlyAddedTokens.containsAll(nextTokens));
+    }
+
+    // Generates endpoints and adds them to the tmd and the rs.
+    private List<Token> generateFakeEndpoints(TokenMetadata tmd, AbstractReplicationStrategy rs, int firstNodeId, int lastNodId, int vnodes, String dc, String rack) throws UnknownHostException
+    {
+        System.out.printf("Adding nodes %d through %d to dc=%s, rack=%s.%n", firstNodeId, lastNodId, dc, rack);
+        var result = new ArrayList<Token>();
+        for (int i = firstNodeId; i <= lastNodId; i++)
+        {
+            // leave .1 for myEndpoint
+            InetAddressAndPort addr = InetAddressAndPort.getByName("127." + dc + '.' + rack + '.' + (i + 1));
+            var tokens = TokenAllocation.allocateTokens(tmd, rs, addr, vnodes);
+            // TODO why don't we need addBootstrapTokens here? The test only passes with updateNormalTokens.
+            // tmd.addBootstrapTokens(tokens, addr);
+            tmd.updateNormalTokens(tokens, addr);
+            result.addAll(tokens);
+        }
+        return result;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorFastTest.java b/test/unit/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorFastTest.java
new file mode 100644
index 000000000000..8e12c9a8dbe9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorFastTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht.tokenallocator;
+
+import java.util.Random;
+import java.util.TreeMap;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class NoReplicationTokenAllocatorFastTest
+{
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.clientInitialization();
+    }
+
+    @Test
+    public void testOverridenSeedToken()
+    {
+        var random = new Random();
+        var initialToken = Murmur3Partitioner.instance.getToken(UTF8Type.instance.fromString("initial"));
+        // Confirm that up to num_tokens 32, we get the same first token.
+        for (int i = 0; i < 32; i++)
+        {
+            var allocator = new NoReplicationTokenAllocator<>(new TreeMap<>(), new BasicReplicationStrategy(),
+                                                              Murmur3Partitioner.instance, () -> initialToken);
+            var tokens = allocator.addUnit(random.nextInt(), i);
+            var first = tokens.stream().findFirst();
+            assertTrue(first.isPresent());
+            assertEquals(first.get().getToken(), initialToken);
+        }
+    }
+
+    private static class BasicReplicationStrategy implements ReplicationStrategy<Integer>
+    {
+        @Override
+        public int replicas()
+        {
+            return 1;
+        }
+
+        @Override
+        public Object getGroup(Integer unit)
+        {
+            return unit;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationOverrideTest.java b/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationOverrideTest.java
new file mode 100644
index 000000000000..321c361053af
--- /dev/null
+++ b/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationOverrideTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht.tokenallocator;
+
+import java.net.UnknownHostException;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+
+public class TokenAllocationOverrideTest extends TokenAllocationTest
+{
+    @BeforeClass
+    public static void init()
+    {
+        CassandraRelevantProperties.USE_RANDOM_ALLOCATION_IF_NOT_SUPPORTED.setBoolean(true);
+    }
+
+    //With the random allocation flag enabled we can now support two racks with RF=3
+    @Override
+    @Test
+    public void testAllocateTokensNetworkStrategyTwoRacks() throws UnknownHostException
+    {
+        testAllocateTokensNetworkStrategy(2, 3);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java b/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java
index ac372e482794..edfccee76031 100644
--- a/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java
+++ b/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java
@@ -62,8 +62,8 @@ public static void setup() throws ConfigurationException
     {
         DatabaseDescriptor.daemonInitialization();
         oldPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
-        SchemaLoader.startGossiper();
         SchemaLoader.prepareServer();
+        SchemaLoader.startGossiper();
         SchemaLoader.schemaDefinition("TokenAllocationTest");
     }
 
diff --git a/test/unit/org/apache/cassandra/exceptions/RequestFailureReasonTest.java b/test/unit/org/apache/cassandra/exceptions/RequestFailureReasonTest.java
new file mode 100644
index 000000000000..3bbb21d3aeb8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/exceptions/RequestFailureReasonTest.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.exceptions;
+import org.junit.Test;
+
+import org.apache.cassandra.schema.TableId;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class RequestFailureReasonTest
+{
+    private static final RequestFailureReason[] REASONS = RequestFailureReason.values();
+    private static final Object[][] EXPECTED_VALUES =
+    {
+    { 0, "UNKNOWN" },
+    { 1, "READ_TOO_MANY_TOMBSTONES" },
+    { 2, "TIMEOUT" },
+    { 3, "INCOMPATIBLE_SCHEMA" },
+    { 4, "READ_SIZE" },
+    { 5, "NODE_DOWN" },
+    { 6, "INDEX_NOT_AVAILABLE" },
+    { 7, "READ_TOO_MANY_INDEXES" },
+    { 500, "UNKNOWN_COLUMN" },
+    { 501, "UNKNOWN_TABLE" },
+    { 502, "REMOTE_STORAGE_FAILURE" },
+    { 503, "INDEX_BUILD_IN_PROGRESS" }
+    };
+    @Test
+    public void testEnumCodesAndNames()
+    {
+        for (int i = 0; i < REASONS.length; i++)
+        {
+            assertEquals("RequestFailureReason code mismatch for " +
+                         REASONS[i].name(), EXPECTED_VALUES[i][0], REASONS[i].code);
+            assertEquals("RequestFailureReason name mismatch for code " +
+                         REASONS[i].code, EXPECTED_VALUES[i][1], REASONS[i].name());
+        }
+        assertEquals("Number of RequestFailureReason enum constants has changed. Update the test.",
+                     EXPECTED_VALUES.length, REASONS.length);
+    }
+
+    @Test
+    public void testFromCode()
+    {
+        // Test valid codes
+        for (Object[] expected : EXPECTED_VALUES)
+        {
+            int code = (Integer) expected[0];
+            RequestFailureReason expectedReason = RequestFailureReason.valueOf((String) expected[1]);
+            assertEquals(expectedReason, RequestFailureReason.fromCode(code));
+        }
+
+        // Test invalid codes
+        assertEquals(RequestFailureReason.UNKNOWN, RequestFailureReason.fromCode(200));
+        assertEquals(RequestFailureReason.UNKNOWN, RequestFailureReason.fromCode(999));
+        assertThrows(IllegalArgumentException.class, () -> RequestFailureReason.fromCode(-1));
+
+        // Below codes will map to UKNOWN until we rebase on the newer Apache Cassandra version, where they are not UNKNOWN.
+        // We leave them UNKNOWN for now to prevent future conflicts with Apache
+        assertEquals(RequestFailureReason.UNKNOWN, RequestFailureReason.fromCode(9));
+        assertEquals(RequestFailureReason.UNKNOWN, RequestFailureReason.fromCode(10));
+        assertEquals(RequestFailureReason.UNKNOWN, RequestFailureReason.fromCode(11));
+    }
+
+    @Test
+    public void testExceptionSubclassMapping()
+    {
+        // Create a subclass of UnknownTableException
+        class CustomUnknownTableException extends UnknownTableException 
+        {
+            public CustomUnknownTableException()
+            {
+                super("ks", TableId.generate());
+            }
+        }
+
+        // Verify the subclass maps correctly
+        // `UnknownTableException` extends` IncompatibleSchemaException`
+        RequestFailureReason result = RequestFailureReason.forException(new CustomUnknownTableException());
+        assertTrue("Expected either UNKNOWN_TABLE or INCOMPATIBLE_SCHEMA but got " + result,
+                   result == RequestFailureReason.UNKNOWN_TABLE || 
+                   result == RequestFailureReason.INCOMPATIBLE_SCHEMA);
+        
+        // Verify the parent class still maps correctly
+        assertEquals(RequestFailureReason.UNKNOWN_TABLE,
+                     RequestFailureReason.forException(new UnknownTableException("ks", TableId.generate())));
+        
+        // Test unmapped exception returns UNKNOWN
+        assertEquals(RequestFailureReason.UNKNOWN, 
+                     RequestFailureReason.forException(new RuntimeException("test")));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/fql/FullQueryLoggerTest.java b/test/unit/org/apache/cassandra/fql/FullQueryLoggerTest.java
index e95eb0c8b41a..2034bf026241 100644
--- a/test/unit/org/apache/cassandra/fql/FullQueryLoggerTest.java
+++ b/test/unit/org/apache/cassandra/fql/FullQueryLoggerTest.java
@@ -28,7 +28,6 @@
 import java.util.List;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicInteger;
-
 import javax.annotation.Nullable;
 
 import org.apache.cassandra.io.util.File;
@@ -39,9 +38,9 @@
 
 import io.netty.buffer.Unpooled;
 import net.openhft.chronicle.queue.ChronicleQueue;
-import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.queue.ExcerptTailer;
 import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.wire.ValueIn;
 import net.openhft.chronicle.wire.WireOut;
 import org.apache.cassandra.Util;
@@ -61,10 +60,6 @@
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.binlog.BinLogTest;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
 import static org.apache.cassandra.fql.FullQueryLogger.BATCH;
 import static org.apache.cassandra.fql.FullQueryLogger.BATCH_TYPE;
 import static org.apache.cassandra.fql.FullQueryLogger.GENERATED_NOW_IN_SECONDS;
@@ -79,6 +74,9 @@
 import static org.apache.cassandra.fql.FullQueryLogger.VALUES;
 import static org.apache.cassandra.fql.FullQueryLogger.VERSION;
 import static org.junit.Assert.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public class FullQueryLoggerTest extends CQLTester
 {
@@ -709,7 +707,7 @@ private static void compareQueryOptions(QueryOptions a, QueryOptions b)
         assertEquals(a.getConsistency(), b.getConsistency());
         assertEquals(a.getPagingState(), b.getPagingState());
         assertEquals(a.getValues(), b.getValues());
-        assertEquals(a.getSerialConsistency(), b.getSerialConsistency());
+        assertEquals(a.getSerialConsistency(null), b.getSerialConsistency(null));
     }
 
     private void configureFQL() throws Exception
diff --git a/test/unit/org/apache/cassandra/gms/ArrivalWindowTest.java b/test/unit/org/apache/cassandra/gms/ArrivalWindowTest.java
index 3a07ea36d7fc..8daf26542a3c 100644
--- a/test/unit/org/apache/cassandra/gms/ArrivalWindowTest.java
+++ b/test/unit/org/apache/cassandra/gms/ArrivalWindowTest.java
@@ -21,21 +21,23 @@
  */
 
 
-import static org.junit.Assert.*;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.FBUtilities;
 
+import static org.junit.Assert.assertEquals;
+
 public class ArrivalWindowTest
 {
     @BeforeClass
     public static void beforeClass()
     {
         DatabaseDescriptor.setDefaultFailureDetector();
+        DatabaseDescriptor.setConfig(new Config());
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/gms/CustomFailureDetectorTest.java b/test/unit/org/apache/cassandra/gms/CustomFailureDetectorTest.java
new file mode 100644
index 000000000000..8b30ac7bad5b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/gms/CustomFailureDetectorTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.gms;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_FAILURE_DETECTOR_PROPERTY;
+import static org.junit.Assert.assertTrue;
+
+public class CustomFailureDetectorTest
+{
+    static String oldValueCustomProperty = null;
+
+    @BeforeClass
+    public static void setProperty()
+    {
+        oldValueCustomProperty =CUSTOM_FAILURE_DETECTOR_PROPERTY.getString();
+        CUSTOM_FAILURE_DETECTOR_PROPERTY.setString(TestFailureDetector.class.getName());
+    }
+
+    @AfterClass
+    public static void resetProperty()
+    {
+        if (oldValueCustomProperty != null)
+            CUSTOM_FAILURE_DETECTOR_PROPERTY.setString(oldValueCustomProperty);
+        else
+            System.clearProperty(CUSTOM_FAILURE_DETECTOR_PROPERTY.getKey());
+    }
+
+    @Test
+    public void testCustomFailureDetector()
+    {
+        assertTrue(IFailureDetector.instance instanceof TestFailureDetector);
+    }
+
+    public static class TestFailureDetector implements IFailureDetector
+    {
+        @Override
+        public boolean isAlive(InetAddressAndPort ep)
+        {
+            return false;
+        }
+
+        @Override
+        public void interpret(InetAddressAndPort ep)
+        {
+        }
+
+        @Override
+        public void report(InetAddressAndPort ep)
+        {
+        }
+
+        @Override
+        public void remove(InetAddressAndPort ep)
+        {
+        }
+
+        @Override
+        public void forceConviction(InetAddressAndPort ep)
+        {
+        }
+
+        @Override
+        public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener)
+        {
+        }
+
+        @Override
+        public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener)
+        {
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/gms/ExpireEndpointTest.java b/test/unit/org/apache/cassandra/gms/ExpireEndpointTest.java
index 298549e39440..9dc518d293c1 100644
--- a/test/unit/org/apache/cassandra/gms/ExpireEndpointTest.java
+++ b/test/unit/org/apache/cassandra/gms/ExpireEndpointTest.java
@@ -28,6 +28,7 @@
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.service.StorageService;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -37,6 +38,7 @@ public class ExpireEndpointTest
     @BeforeClass
     public static void setup()
     {
+        NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
     }
 
diff --git a/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java b/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
index 8ec3dae913c2..b4882ad06054 100644
--- a/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
+++ b/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
@@ -35,10 +35,14 @@
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.StorageService;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.MAX_LOCAL_PAUSE_IN_MS;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class FailureDetectorTest
 {
@@ -74,7 +78,7 @@ public void testConvictAfterLeft() throws UnknownHostException
 
         InetAddressAndPort leftHost = hosts.get(1);
 
-        FailureDetector.instance.report(leftHost);
+        IFailureDetector.instance.report(leftHost);
 
         // trigger handleStateLeft in StorageService
         ss.onChange(leftHost, ApplicationState.STATUS_WITH_PORT,
@@ -84,7 +88,142 @@ public void testConvictAfterLeft() throws UnknownHostException
         assertFalse("Left endpoint not removed from TokenMetadata", tmd.isMember(leftHost));
 
         // confirm the FD's history for leftHost didn't get wiped by status jump to LEFT
-        FailureDetector.instance.interpret(leftHost);
-        assertFalse("Left endpoint not convicted", FailureDetector.instance.isAlive(leftHost));
+        IFailureDetector.instance.interpret(leftHost);
+        assertFalse("Left endpoint not convicted", IFailureDetector.instance.isAlive(leftHost));
+    }
+
+    @Test
+    public void testConvictAfterReplace() throws UnknownHostException
+    {
+        StorageService ss = StorageService.instance;
+        TokenMetadata tmd = ss.getTokenMetadata();
+        tmd.clearUnsafe();
+        IPartitioner partitioner = new RandomPartitioner();
+        VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
+
+        ArrayList<Token> endpointTokens = new ArrayList<>();
+        ArrayList<Token> keyTokens = new ArrayList<>();
+        List<InetAddressAndPort> hosts = new ArrayList<>();
+        List<UUID> hostIds = new ArrayList<>();
+
+        // We want to convict if there is any heartbeat data present in the FD
+        DatabaseDescriptor.setPhiConvictThreshold(0);
+
+        // Create a ring of 3 nodes
+        Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 3);
+
+        // Add a new node with old node's tokens
+        InetAddressAndPort oldNode = hosts.get(1);
+        InetAddressAndPort newNode = InetAddressAndPort.getByName("127.0.0.100");
+        Token oldToken = endpointTokens.get(1);
+        Gossiper.instance.initializeNodeUnsafe(newNode, UUID.randomUUID(), MessagingService.current_version, 1);
+        Gossiper.instance.injectApplicationState(newNode, ApplicationState.TOKENS, new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(oldToken)));
+        ss.onChange(newNode,
+                    ApplicationState.STATUS_WITH_PORT,
+                    new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(oldToken)));
+
+        // Mark the old node as dead.
+        Util.markNodeAsDead(oldNode);
+
+        // Trigger handleStateBootreplacing in StorageService
+        ss.onChange(newNode, ApplicationState.STATUS_WITH_PORT,
+                    valueFactory.bootReplacingWithPort(oldNode));
+
+        assertEquals("Old node did not replace new node", newNode, tmd.getReplacementNode(oldNode).get());
+    }
+
+    @Test
+    public void testStateBootReplacingFailsForLiveNode() throws UnknownHostException
+    {
+        StorageService ss = StorageService.instance;
+        TokenMetadata tmd = ss.getTokenMetadata();
+        tmd.clearUnsafe();
+        IPartitioner partitioner = new RandomPartitioner();
+        VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
+
+        ArrayList<Token> endpointTokens = new ArrayList<>();
+        ArrayList<Token> keyTokens = new ArrayList<>();
+        List<InetAddressAndPort> hosts = new ArrayList<>();
+        List<UUID> hostIds = new ArrayList<>();
+
+        // We want to convict if there is any heartbeat data present in the FD
+        DatabaseDescriptor.setPhiConvictThreshold(0);
+
+        // Create a ring of 3 nodes
+        Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 3);
+
+        // Add a new node with old node's tokens
+        InetAddressAndPort oldNode = hosts.get(1);
+        InetAddressAndPort newNode = InetAddressAndPort.getByName("127.0.0.100");
+        Token token = endpointTokens.get(1);
+
+        Util.joinNodeToRing(newNode, token, partitioner);
+
+        EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(oldNode);
+        Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.realMarkAlive(oldNode, endpointState));
+        assertTrue(Gossiper.instance.isAlive(oldNode));
+
+        // Trigger handleStateBootreplacing in StorageService
+        try
+        {
+            ss.onChange(newNode, ApplicationState.STATUS_WITH_PORT,
+                        valueFactory.bootReplacingWithPort(oldNode));
+            fail();
+        }
+        catch (RuntimeException ex)
+        {
+            String msg = ex.getMessage();
+            final String expected = "trying to replace alive node";
+            assertTrue(String.format("Didn't see expected '%s' message", expected), msg.contains(expected));
+        }
+    }
+
+    @Test
+    public void testReplacingLiveNodeFails() throws UnknownHostException
+    {
+        StorageService ss = StorageService.instance;
+        TokenMetadata tmd = ss.getTokenMetadata();
+        tmd.clearUnsafe();
+        IPartitioner partitioner = new RandomPartitioner();
+        VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
+
+        ArrayList<Token> endpointTokens = new ArrayList<>();
+        ArrayList<Token> keyTokens = new ArrayList<>();
+        List<InetAddressAndPort> hosts = new ArrayList<>();
+        List<UUID> hostIds = new ArrayList<>();
+
+        // We want to convict if there is any heartbeat data present in the FD
+        DatabaseDescriptor.setPhiConvictThreshold(0);
+
+        // Create a ring of 3 nodes
+        Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 3);
+
+        // Add a new node with old node's tokens
+        InetAddressAndPort oldNode = hosts.get(1);
+        InetAddressAndPort newNode = InetAddressAndPort.getByName("127.0.0.100");
+        Token token = endpointTokens.get(1);
+
+        Gossiper.instance.initializeNodeUnsafe(newNode, UUID.randomUUID(), MessagingService.current_version, 1);
+        Gossiper.instance.injectApplicationState(newNode, ApplicationState.TOKENS, new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(token)));
+
+        // Mark the old node as dead.
+        Util.markNodeAsDead(oldNode);
+
+        // Trigger handleStateBootreplacing in StorageService
+        ss.onChange(newNode, ApplicationState.STATUS_WITH_PORT,
+                    valueFactory.bootReplacingWithPort(oldNode));
+
+        assertEquals("Old node did not replace new node", newNode, tmd.getReplacementNode(oldNode).get());
+
+        // Resurrect old node and mark alive
+        EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(oldNode);
+        Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.realMarkAlive(oldNode, endpointState));
+
+        // Trigger handleStateNormal in StorageService which should fail and cause the old node to still be
+        // marked as a live endpoint.
+        ss.onChange(newNode,
+                    ApplicationState.STATUS_WITH_PORT,
+                    new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(token)));
+        assertTrue("Expected old node to be live but it was removed", Gossiper.instance.liveEndpoints.contains(oldNode));
     }
 }
diff --git a/test/unit/org/apache/cassandra/gms/GossiperTest.java b/test/unit/org/apache/cassandra/gms/GossiperTest.java
index 1c9584608d29..a2e554acbd1e 100644
--- a/test/unit/org/apache/cassandra/gms/GossiperTest.java
+++ b/test/unit/org/apache/cassandra/gms/GossiperTest.java
@@ -39,42 +39,47 @@
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.VersionedValue.VersionedValueFactory;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.SeedProvider;
 import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.CassandraGenerators;
 import org.apache.cassandra.utils.CassandraVersion;
-import org.apache.cassandra.utils.FBUtilities;
 import org.assertj.core.api.Assertions;
 import org.quicktheories.core.Gen;
 import org.quicktheories.impl.Constraint;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
+import org.mockito.Mockito;
+
+import static org.apache.cassandra.db.SystemKeyspace.CURRENT_VERSION;
+import static org.apache.cassandra.gms.ApplicationState.RELEASE_VERSION;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.quicktheories.QuickTheory.qt;
+import static org.mockito.Mockito.when;
 
 public class GossiperTest
 {
     static
     {
+        CassandraRelevantProperties.CLUSTER_VERSION_PROVIDER_MIN_STABLE_DURATION.setLong(30000);
         GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
         CommitLog.instance.start();
     }
 
-    private static final CassandraVersion CURRENT_VERSION = new CassandraVersion(FBUtilities.getReleaseVersionString());
-
     static final IPartitioner partitioner = new RandomPartitioner();
     StorageService ss = StorageService.instance;
     TokenMetadata tmd = StorageService.instance.getTokenMetadata();
@@ -85,11 +90,18 @@ public class GossiperTest
 
     private SeedProvider originalSeedProvider;
 
+    final VersionedValueFactory factory = new VersionedValueFactory(null);
+
+
     @Before
     public void setup()
     {
         tmd.clearUnsafe();
         originalSeedProvider = DatabaseDescriptor.getSeedProvider();
+        if (Gossiper.instance.isEnabled())
+            Gossiper.instance.stop();
+        Gossiper.instance.liveEndpoints.clear();
+        Gossiper.instance.endpointStateMap.clear();
     }
 
     @After
@@ -120,75 +132,117 @@ public void testPaddingIntact() throws Exception
         assert ApplicationState.X10 == ApplicationState.X10;
     }
 
-    @Test
-    public void testHasVersion3Nodes() throws Exception
+    private void setLiveEndpoint(String address, String version) throws UnknownHostException
     {
-        Gossiper.instance.start(0);
-        Gossiper.instance.expireUpgradeFromVersion();
-
-        VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(null);
-        EndpointState es = new EndpointState((HeartBeatState) null);
-        es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion(CURRENT_VERSION.toString()));
-        Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.1"), es);
-        Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.1"));
-
-
-        es = new EndpointState((HeartBeatState) null);
-        es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("3.11.3"));
-        Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.2"), es);
-        Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.2"));
-
-        es = new EndpointState((HeartBeatState) null);
-        es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("3.0.0"));
-        Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.3"), es);
-        Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.3"));
-
-        assertFalse(Gossiper.instance.upgradeFromVersionSupplier.get().value().compareTo(new CassandraVersion("3.0")) < 0);
-        assertTrue(Gossiper.instance.upgradeFromVersionSupplier.get().value().compareTo(new CassandraVersion("3.1")) < 0);
-        assertTrue(Gossiper.instance.hasMajorVersion3OrUnknownNodes());
-
-        Gossiper.instance.endpointStateMap.remove(InetAddressAndPort.getByName("127.0.0.3"));
-        Gossiper.instance.liveEndpoints.remove(InetAddressAndPort.getByName("127.0.0.3"));
+        if (version != null)
+        {
+            EndpointState es = new EndpointState((HeartBeatState) null);
+            es.addApplicationState(RELEASE_VERSION, factory.releaseVersion(version));
+            Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName(address), es);
+        }
+        else
+        {
+            Gossiper.instance.endpointStateMap.remove(InetAddressAndPort.getByName(address));
+        }
+        Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName(address));
+    }
 
-        assertFalse(Gossiper.instance.upgradeFromVersionSupplier.get().value().compareTo(new CassandraVersion("3.0")) < 0);
-        assertFalse(Gossiper.instance.upgradeFromVersionSupplier.get().value().compareTo(new CassandraVersion("3.1")) < 0);
-        assertTrue(Gossiper.instance.upgradeFromVersionSupplier.get().value().compareTo(new CassandraVersion("3.12")) < 0);
-        assertTrue(Gossiper.instance.hasMajorVersion3OrUnknownNodes());
+    private void removeEndpoint(String address) throws UnknownHostException
+    {
+        Gossiper.instance.endpointStateMap.remove(InetAddressAndPort.getByName(address));
+        Gossiper.instance.liveEndpoints.remove(InetAddressAndPort.getByName(address));
+    }
 
-        Gossiper.instance.endpointStateMap.remove(InetAddressAndPort.getByName("127.0.0.2"));
-        Gossiper.instance.liveEndpoints.remove(InetAddressAndPort.getByName("127.0.0.2"));
+    @Test
+    public void testHasVersion3Nodes()
+    {
+        IClusterVersionProvider cvp = Mockito.mock(IClusterVersionProvider.class);
+        Gossiper g = new Gossiper(false, cvp);
+
+        when(cvp.isUpgradeInProgress()).thenReturn(false);
+        when(cvp.getMinClusterVersion()).thenReturn(new CassandraVersion("3.0.0"));
+        assertThat(g.getMinVersion()).isEqualTo(new CassandraVersion("3.0.0"));
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("3.11.0"))).isTrue();
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("3.0.0"))).isFalse();
+        assertThat(Gossiper.instance.hasMajorVersion3OrUnknownNodes()).isTrue();
+
+        when(cvp.isUpgradeInProgress()).thenReturn(true);
+        when(cvp.getMinClusterVersion()).thenReturn(new CassandraVersion("3.0.0"));
+        assertThat(g.getMinVersion()).isEqualTo(new CassandraVersion("3.0.0"));
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("3.11.0"))).isTrue();
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("3.0.0"))).isFalse();
+        assertThat(Gossiper.instance.hasMajorVersion3OrUnknownNodes()).isTrue();
+
+        when(cvp.isUpgradeInProgress()).thenReturn(true);
+        when(cvp.getMinClusterVersion()).thenReturn(new CassandraVersion("3.11.0"));
+        assertThat(g.getMinVersion()).isEqualTo(new CassandraVersion("3.11.0"));
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("3.11.0"))).isFalse();
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("3.0.0"))).isFalse();
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("4.0.0"))).isTrue();
+        assertThat(Gossiper.instance.hasMajorVersion3OrUnknownNodes()).isTrue();
+
+        when(cvp.isUpgradeInProgress()).thenReturn(true);
+        when(cvp.getMinClusterVersion()).thenReturn(new CassandraVersion(CURRENT_VERSION.toString()));
+        assertThat(g.getMinVersion()).isEqualTo(CassandraVersion.NULL_VERSION);
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("3.0.0"))).isTrue();
+        assertThat(Gossiper.instance.hasMajorVersion3OrUnknownNodes()).isTrue();
+    }
 
-        assertEquals(SystemKeyspace.CURRENT_VERSION, Gossiper.instance.upgradeFromVersionSupplier.get().value());
+    @Test
+    public void testDefaultClusterVersionProvider() throws UnknownHostException
+    {
+        Gossiper g = Gossiper.instance;
+        g.stop();
+        g.liveEndpoints.clear();
+        g.endpointStateMap.clear();
+        IClusterVersionProvider cvp = g.clusterVersionProvider;
+
+        cvp.reset();
+        assertThat(cvp.getMinClusterVersion()).isEqualTo(CURRENT_VERSION);
+        assertThat(cvp.isUpgradeInProgress()).isTrue();
+
+        g.start(0);
+
+        cvp.reset();
+        assertThat(cvp.getMinClusterVersion()).isEqualTo(CURRENT_VERSION);
+        assertThat(cvp.isUpgradeInProgress()).isTrue();
+
+        g.setNotUpgradingSinceMillisUnsafe(System.currentTimeMillis() - CassandraRelevantProperties.CLUSTER_VERSION_PROVIDER_MIN_STABLE_DURATION.getLong() - 1);
+        assertThat(cvp.getMinClusterVersion()).isEqualTo(CURRENT_VERSION);
+        assertThat(cvp.isUpgradeInProgress()).isFalse();
+
+        // set one version missing
+        cvp.reset();
+        setLiveEndpoint("127.0.0.1", CURRENT_VERSION.toString());
+        setLiveEndpoint("127.0.0.2", null);
+        assertThat(cvp.getMinClusterVersion()).isEqualTo(CURRENT_VERSION);
+        assertThat(cvp.isUpgradeInProgress()).isTrue();
+
+        // set one version lower
+        cvp.reset();
+        setLiveEndpoint("127.0.0.1", CURRENT_VERSION.toString());
+        setLiveEndpoint("127.0.0.2", "3.0.0");
+        assertThat(cvp.getMinClusterVersion()).isEqualTo(new CassandraVersion("3.0.0"));
+        assertThat(cvp.isUpgradeInProgress()).isTrue();
     }
 
     @Test
     public void testHasVersion3NodesShouldReturnFalseWhenNoVersion3NodesDetectedAndCassandra4UpgradeInProgress() throws Exception
     {
-        Gossiper.instance.start(0);
-        Gossiper.instance.expireUpgradeFromVersion();
-
-        VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(null);
-        EndpointState es = new EndpointState((HeartBeatState) null);
-        es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion(CURRENT_VERSION.toString()));
-        Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.1"), es);
-        Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.1"));
-
-        es = new EndpointState((HeartBeatState) null);
-        String previousPatchVersion = String.valueOf(CURRENT_VERSION.major) + '.' + (CURRENT_VERSION.minor) + '.' + Math.max(CURRENT_VERSION.patch - 1, 0);
-        es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion(previousPatchVersion));
-        Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.2"), es);
-        Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.2"));
-        assertFalse(Gossiper.instance.hasMajorVersion3OrUnknownNodes());
-
-        Gossiper.instance.endpointStateMap.remove(InetAddressAndPort.getByName("127.0.0.2"));
-        Gossiper.instance.liveEndpoints.remove(InetAddressAndPort.getByName("127.0.0.2"));
+        IClusterVersionProvider cvp = Mockito.mock(IClusterVersionProvider.class);
+        Gossiper g = new Gossiper(false, cvp);
+
+        when(cvp.isUpgradeInProgress()).thenReturn(true);
+        when(cvp.getMinClusterVersion()).thenReturn(new CassandraVersion("4.1.0"));
+        assertThat(g.getMinVersion()).isEqualTo(new CassandraVersion("4.1.0"));
+        assertThat(g.isUpgradingFromVersionLowerThan(new CassandraVersion("3.0.0"))).isFalse();
+        assertThat(g.hasMajorVersion3OrUnknownNodes()).isFalse();
     }
 
     @Test
     public void testHasVersion3NodesShouldReturnTrueWhenNoVersion3NodesDetectedButNotAllVersionsKnown() throws Exception
     {
         Gossiper.instance.start(0);
-        Gossiper.instance.expireUpgradeFromVersion();
 
         VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(null);
         EndpointState es = new EndpointState((HeartBeatState) null);
@@ -214,17 +268,14 @@ public void testAssassinatedNodeWillNotContributeToVersionCalculation() throws E
         Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, initialNodeCount);
         for (int i = 0; i < initialNodeCount; i++)
         {
-            Gossiper.instance.injectApplicationState(hosts.get(i), ApplicationState.RELEASE_VERSION, new VersionedValue.VersionedValueFactory(null).releaseVersion(SystemKeyspace.CURRENT_VERSION.toString()));
+            Gossiper.instance.injectApplicationState(hosts.get(i), ApplicationState.RELEASE_VERSION, new VersionedValue.VersionedValueFactory(null).releaseVersion("4.1.1"));
         }
         Gossiper.instance.start(1);
-        Gossiper.instance.expireUpgradeFromVersion();
 
         // assassinate a non-existing node
         Gossiper.instance.assassinateEndpoint("127.0.0.4");
 
         assertTrue(Gossiper.instance.endpointStateMap.containsKey(InetAddressAndPort.getByName("127.0.0.4")));
-        assertNull(Gossiper.instance.upgradeFromVersionSupplier.get().value());
-        assertTrue(Gossiper.instance.upgradeFromVersionSupplier.get().canMemoize());
         assertFalse(Gossiper.instance.hasMajorVersion3OrUnknownNodes());
         assertFalse(Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_3_4));
     }
@@ -275,8 +326,8 @@ public void testLargeGenerationJump() throws UnknownHostException, InterruptedEx
     @Test
     public void testDuplicatedStateUpdate() throws Exception
     {
-        VersionedValue.VersionedValueFactory valueFactory =
-            new VersionedValue.VersionedValueFactory(DatabaseDescriptor.getPartitioner());
+        VersionedValueFactory valueFactory =
+        new VersionedValueFactory(DatabaseDescriptor.getPartitioner());
 
         SimpleStateChangeListener stateChangeListener = null;
         Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 2);
@@ -434,8 +485,8 @@ public void testReloadSeeds() throws UnknownHostException
     @Test
     public void testNotFireDuplicatedNotificationsWithUpdateContainsOldAndNewState() throws UnknownHostException
     {
-        VersionedValue.VersionedValueFactory valueFactory =
-            new VersionedValue.VersionedValueFactory(DatabaseDescriptor.getPartitioner());
+        VersionedValueFactory valueFactory =
+        new VersionedValueFactory(DatabaseDescriptor.getPartitioner());
 
         Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 2);
         SimpleStateChangeListener stateChangeListener = null;
@@ -641,4 +692,35 @@ public List<InetAddressAndPort> getSeeds()
             return new ArrayList<>();
         }
     }
+
+    @Test
+    public void testShutdownMarksNodeAsDead() throws UnknownHostException
+    {
+        Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 2);
+        try
+        {
+            InetAddressAndPort remoteHostAddress = hosts.get(1);
+
+            EndpointState initialRemoteState = Gossiper.instance.getEndpointStateForEndpoint(remoteHostAddress);
+            HeartBeatState initialRemoteHeartBeat = initialRemoteState.getHeartBeatState();
+
+            // Util.createInitialRing should have initialized remoteHost's HeartBeatState's generation to 1
+            assertEquals(initialRemoteHeartBeat.getGeneration(), 1);
+
+            VersionedValueFactory factory = new VersionedValueFactory(null);
+            HeartBeatState proposedRemoteHeartBeat = new HeartBeatState(initialRemoteHeartBeat.getGeneration() + Gossiper.MAX_GENERATION_DIFFERENCE + 1);
+            EndpointState proposedRemoteState = new EndpointState(proposedRemoteHeartBeat);
+            proposedRemoteState.addApplicationState(ApplicationState.STATUS_WITH_PORT, factory.shutdown(true));
+
+            assertTrue(Gossiper.instance.isAlive(remoteHostAddress));
+            Gossiper.instance.applyStateLocally(ImmutableMap.of(remoteHostAddress, proposedRemoteState));
+            assertFalse(Gossiper.instance.isAlive(remoteHostAddress));
+        }
+        finally
+        {
+            // clean up the gossip states
+            Gossiper.instance.endpointStateMap.clear();
+        }
+
+    }
 }
diff --git a/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java b/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
index 1f7587668e64..4b8a34b1f015 100644
--- a/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
+++ b/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
@@ -24,6 +24,10 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.LockSupport;
 import java.util.concurrent.locks.ReentrantLock;
 
 import org.junit.BeforeClass;
@@ -31,17 +35,26 @@
 import org.junit.runner.RunWith;
 
 import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.PendingRangeCalculatorService;
 import org.apache.cassandra.service.StorageService;
 import org.jboss.byteman.contrib.bmunit.BMRule;
 import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+import org.mockito.Mockito;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.mockito.Mockito.when;
 
 
 /**
@@ -131,4 +144,47 @@ private Map<InetAddressAndPort, EndpointState> getStates(InetAddressAndPort othe
         states.put(otherNodeAddr, state);
         return states;
     }
+
+    @Test
+    public void testPendingRangesCalculatedForAllRequestedKeyspaces() throws InterruptedException, TimeoutException
+    {
+        DatabaseDescriptor.daemonInitialization();
+
+        // mock schema with 100 keyspaces ks0, ks1, ..., ks99
+        Schema schema = Mockito.mock(Schema.class);
+        Keyspaces.Builder keyspaces = Keyspaces.builder();
+        for (int i = 0; i < 100; i++)
+            keyspaces.add(KeyspaceMetadata.create("ks" + i, KeyspaceParams.simple(1)));
+        when(schema.distributedKeyspaces()).thenReturn(keyspaces.build());
+
+        // a set of keyspaces for which pending ranges have been calculated - once the calculation is requested, we put a keyspace name into this set
+        Map<String, Boolean> processedKeyspaces = new ConcurrentHashMap<>();
+
+        // create a PendingRangeCalculatorService that will take 1 ms to calculate pending ranges for each keyspace
+        PendingRangeCalculatorService prcs = new PendingRangeCalculatorService("PendingRangeCalculator" + System.currentTimeMillis(), schema) {
+            @Override
+            public void calculatePendingRanges(String keyspace)
+            {
+                processedKeyspaces.put(keyspace, true);
+                LockSupport.parkNanos(1000000); // 1 ms processing time
+            }
+        };
+
+        // request pending range calculation for each keyspace with 100 µs interval
+        // Note that, those parkNanos are optional and are added to increse the visibility of the problem
+        for (int i = 0; i < 100; i++)
+        {
+            String name = "ks" + i;
+            prcs.update(ks -> ks.equals(name));
+            LockSupport.parkNanos(100000); // 100 µs schedule interval
+        }
+
+        // wait for all pending range calculations to finish
+        prcs.blockUntilFinished();
+        prcs.shutdownAndWait(10, TimeUnit.SECONDS);
+
+        // verify that pending ranges have been calculated for all keyspaces
+        assertThat(processedKeyspaces).withFailMessage("Test is broken or outdated. Expected at least 2 keyspaces to be calculated.").hasSizeGreaterThan(1);
+        assertThat(processedKeyspaces).hasSize(100);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java b/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
index 33d9a1206a79..d7595ffc673d 100644
--- a/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
+++ b/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
@@ -122,8 +122,11 @@ public void testDelayedResponse()
             assertThat(e.getMessage()).startsWith("Unable to contact any seeds");
         }
 
-        // we expect one SYN for each seed during shadow round + additional SYNs after gossiper has been enabled
-        assertTrue(spySyn.messagesIntercepted() > noOfSeeds);
+        // We expect one SYN for each seed during shadow round +
+        // there may be some additional SYNs after gossiper has been enabled
+        int messagesIntercepted = spySyn.messagesIntercepted();
+        assertTrue(String.format("Intercepted messages count should be >= number of seeds %d, but got %d instead",
+                                 noOfSeeds, messagesIntercepted), messagesIntercepted >= noOfSeeds);
 
         // we don't expect to emit any GOSSIP_DIGEST_ACK2 or SCHEMA_PULL messages
         assertEquals(0, spyAck2.messagesIntercepted());
diff --git a/test/unit/org/apache/cassandra/hints/CustomHintTest.java b/test/unit/org/apache/cassandra/hints/CustomHintTest.java
new file mode 100644
index 000000000000..6bcd0f5ba8cb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/hints/CustomHintTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+import java.util.UUID;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.Util.dk;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_HINTS_HANDLER;
+import static org.apache.cassandra.net.Verb.HINT_REQ;
+import static org.junit.Assert.assertTrue;
+
+public class CustomHintTest
+{
+    private static final String KEYSPACE = "custom_hint_test";
+    private static final String TABLE = "table";
+
+    private static boolean customVerbCalled = false;
+
+    @BeforeClass
+    public static void defineSchema()
+    {
+        CUSTOM_HINTS_HANDLER.setString(CustomHintVerbHandler.class.getName());
+        SchemaLoader.prepareServer();
+        StorageService.instance.initServer();
+        SchemaLoader.createKeyspace(KEYSPACE,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE));
+    }
+
+    @AfterClass
+    public static void resetCustom()
+    {
+        System.clearProperty(CUSTOM_HINTS_HANDLER.getKey());
+    }
+
+    @After
+    public void resetAfter()
+    {
+        customVerbCalled = false;
+    }
+
+    @Test
+    public void testChangedTopology() throws Exception
+    {
+        Hint hint = createHint();
+        UUID localId = StorageService.instance.getLocalHostUUID();
+        HintMessage message = new HintMessage(localId, hint);
+
+        HINT_REQ.handler().doVerb(Message.out(HINT_REQ, message));
+        assertTrue(customVerbCalled);
+    }
+
+    private Hint createHint()
+    {
+        long now = System.currentTimeMillis();
+        DecoratedKey dkey = dk(String.valueOf(1));
+        TableMetadata metadata = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
+
+        PartitionUpdate.SimpleBuilder builder = PartitionUpdate.simpleBuilder(metadata, dkey).timestamp(now);
+        builder.row("column0").add("val", "value0");
+        
+        return Hint.create(builder.buildAsMutation(), now);
+    }
+
+    public static class CustomHintVerbHandler implements IVerbHandler<HintMessage>
+    {
+        @Override
+        public void doVerb(Message<HintMessage> message)
+        {
+            customVerbCalled = true;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/hints/HintMessageTest.java b/test/unit/org/apache/cassandra/hints/HintMessageTest.java
index 3565a1274494..794a680ac668 100644
--- a/test/unit/org/apache/cassandra/hints/HintMessageTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintMessageTest.java
@@ -39,6 +39,7 @@
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
 
 public class HintMessageTest
 {
@@ -58,17 +59,17 @@ public void testSerializer() throws IOException
         UUID hostId = UUID.randomUUID();
         long now = FBUtilities.timestampMicros();
         TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
-        
-        Mutation mutation = 
+
+        Mutation mutation =
             new RowUpdateBuilder(table, now, bytes("key")).clustering("column").add("val", "val" + 1234).build();
-        
+
         Hint hint = Hint.create(mutation, now / 1000);
         HintMessage message = new HintMessage(hostId, hint);
 
         // serialize
         int serializedSize = (int) HintMessage.serializer.serializedSize(message, MessagingService.current_version);
         HintMessage deserializedMessage;
-        
+
         try (DataOutputBuffer dob = new DataOutputBuffer())
         {
             HintMessage.serializer.serialize(message, dob, MessagingService.current_version);
@@ -80,9 +81,9 @@ public void testSerializer() throws IOException
         }
 
         // compare before/after
-        assertEquals(hostId, deserializedMessage.hostId);
-        assertNotNull(deserializedMessage.hint);
-        assertHintsEqual(hint, deserializedMessage.hint);
+        assertEquals(hostId, deserializedMessage.hostId());
+        assertNotNull(deserializedMessage.hint());
+        assertHintsEqual(hint, deserializedMessage.hint());
     }
 
     @Test
@@ -91,13 +92,13 @@ public void testEncodedSerializer() throws IOException
         UUID hostId = UUID.randomUUID();
         long now = FBUtilities.timestampMicros();
         TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
-        
+
         Mutation mutation =
             new RowUpdateBuilder(table, now, bytes("key")).clustering("column").add("val", "val" + 1234) .build();
-        
+
         Hint hint = Hint.create(mutation, now / 1000);
         HintMessage.Encoded message;
-        
+
         try (DataOutputBuffer dob = new DataOutputBuffer())
         {
             Hint.serializer.serialize(hint, dob, MessagingService.current_version);
@@ -115,8 +116,35 @@ public void testEncodedSerializer() throws IOException
         HintMessage deserializedMessage = HintMessage.serializer.deserialize(dip, MessagingService.current_version);
 
         // compare before/after
-        assertEquals(hostId, deserializedMessage.hostId);
-        assertNotNull(deserializedMessage.hint);
-        assertHintsEqual(hint, deserializedMessage.hint);
+        assertEquals(hostId, deserializedMessage.hostId());
+        assertNotNull(deserializedMessage.hint());
+        assertHintsEqual(hint, deserializedMessage.hint());
+    }
+
+    @Test
+    public void testEncodedVersionMatching() throws IOException
+    {
+        int messageVersion = MessagingService.current_version;
+        int serializerVersion = MessagingService.VERSION_30;
+
+        UUID hostId = UUID.randomUUID();
+        long now = FBUtilities.timestampMicros();
+        TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
+
+        Mutation mutation =
+        new RowUpdateBuilder(table, now, bytes("key")).clustering("column").add("val", "val" + 1234).build();
+
+        Hint hint = Hint.create(mutation, now / 1000);
+        HintMessage.Encoded message;
+
+        try (DataOutputBuffer dob = new DataOutputBuffer())
+        {
+            Hint.serializer.serialize(hint, dob, messageVersion);
+            message = new HintMessage.Encoded(hostId, dob.buffer(), messageVersion);
+        }
+
+        assertThrows("Mismatched message and serializer version should cause an error",
+                     IllegalArgumentException.class,
+                     () -> HintMessage.serializer.serializedSize(message, serializerVersion));
     }
 }
diff --git a/test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java b/test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java
index b7f431dd81be..3f50c6d81daf 100644
--- a/test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java
@@ -31,6 +31,7 @@
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.MockMessagingService;
 import org.apache.cassandra.net.MockMessagingSpy;
@@ -38,6 +39,7 @@
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
 import org.awaitility.Awaitility;
 import org.jboss.byteman.contrib.bmunit.BMRule;
 import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
@@ -86,7 +88,7 @@ public void reinstanciateService() throws Throwable
 
         failureDetector.isAlive = true;
 
-        HintsService.instance = new HintsService(failureDetector);
+        HintsService.instance = new HintsService(e -> failureDetector.isAlive);
 
         HintsService.instance.startDispatch();
     }
@@ -111,6 +113,10 @@ public void testListPendingHints() throws InterruptedException, ExecutionExcepti
         assertEquals(1, info.totalFiles);
         assertEquals(info.oldestTimestamp, info.newestTimestamp); // there is 1 descriptor with only 1 timestamp
 
+        // Set the endpoint version for the HintsDispatchExecutor to be able to dispatch the hints
+        InetAddressAndPort knownEndpointAddress = FBUtilities.getBroadcastAddressAndPort();
+        MessagingService.instance().versions.set(knownEndpointAddress, MessagingService.current_version);
+
         spy.interceptMessageOut(20000).get();
         assertEquals(Collections.emptyList(), HintsService.instance.getPendingHints());
     }
diff --git a/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java b/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java
index 1aec822f2892..268c168919ce 100644
--- a/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java
@@ -63,7 +63,7 @@ private static Hint makeHint(TableMetadata tbm, int key, long creationTime, int
 
     private static DecoratedKey hintKey(Hint hint)
     {
-        return hint.mutation.key();
+        return hint.mutation().key();
     }
 
     private static Hint deserialize(ByteBuffer bb) throws IOException
diff --git a/test/unit/org/apache/cassandra/hints/HintsBufferTest.java b/test/unit/org/apache/cassandra/hints/HintsBufferTest.java
index 9509262553c0..792eef3a76ea 100644
--- a/test/unit/org/apache/cassandra/hints/HintsBufferTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsBufferTest.java
@@ -182,7 +182,7 @@ private static int validateEntry(UUID hostId, ByteBuffer buffer, long baseTimest
         int idx = (int) (hint.creationTime - baseTimestamp);
         assertEquals(hostId, load[idx]);
 
-        Row row = hint.mutation.getPartitionUpdates().iterator().next().iterator().next();
+        Row row = hint.mutation().getPartitionUpdates().iterator().next().rowIterator().next();
         assertEquals(1, Iterables.size(row.cells()));
 
         ValueAccessors.assertDataEquals(bytes(idx), row.clustering().get(0));
diff --git a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
index 1406e25ed981..cc9cd93da717 100644
--- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
@@ -97,6 +97,11 @@ private void loadCompletenessAndOrderTest(File directory) throws IOException
         HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
         assertEquals(2, catalog.stores().count());
 
+        // verify hint data size is set for descriptors created from local hint files
+        catalog.stores()
+               .flatMap(HintsStore::descriptors)
+               .forEach(desc -> assertTrue(desc.getDataSize() > 0));
+
         HintsStore store1 = catalog.get(hostId1);
         assertNotNull(store1);
         assertEquals(descriptor1, store1.poll());
@@ -110,6 +115,38 @@ private void loadCompletenessAndOrderTest(File directory) throws IOException
         assertNull(store2.poll());
     }
 
+    @Test
+    public void multipleHostsHintsTotalSizeTest() throws IOException
+    {
+        File directory = new File(testFolder.newFolder());
+        HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
+
+        long totalSize = 0;
+        int hosts = 10;
+        int filePerHost = 5;
+        long now = System.currentTimeMillis();
+        for (int i = 0; i < hosts; i++)
+        {
+            long sizePerHost = 0;
+            UUID hostId = UUID.randomUUID();
+            HintsStore store = catalog.get(hostId);
+            assertEquals(sizePerHost, store.getTotalFileSize());
+            for (int f = 0; f < filePerHost; f++)
+            {
+                HintsDescriptor descriptor = new HintsDescriptor(hostId, now + f);
+                writeDescriptor(directory, descriptor);
+                store.offerLast(descriptor);
+
+                assertTrue(descriptor.getDataSize() > 0);
+                sizePerHost += descriptor.getDataSize();
+            }
+            totalSize += sizePerHost;
+            assertEquals(sizePerHost, store.getTotalFileSize());
+        }
+
+        assertEquals(totalSize, catalog.stores().mapToLong(HintsStore::getTotalFileSize).sum());
+    }
+
     @Test
     public void deleteHintsTest() throws IOException
     {
diff --git a/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java b/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java
index 596727fad11b..4fd4e44c702b 100644
--- a/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java
@@ -22,6 +22,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
+import java.util.Optional;
 import java.util.UUID;
 
 import com.google.common.collect.ImmutableMap;
@@ -110,8 +111,8 @@ public void testReadFromFile() throws IOException
             try (HintsWriter ignored = HintsWriter.create(directory, expected))
             {
             }
-            HintsDescriptor actual = HintsDescriptor.readFromFile(expected.file(directory));
-            assertEquals(expected, actual);
+            Optional<HintsDescriptor> actual = HintsDescriptor.readFromFileQuietly(new File(directory, expected.fileName()).toPath());
+            assertEquals(Optional.of(expected), actual);
         }
         finally
         {
@@ -150,6 +151,30 @@ public void testHandleIOE() throws IOException
         newFile.deleteOnExit();
     }
 
+    @Test
+    public void testStatistics() throws IOException
+    {
+        UUID hostId = UUID.randomUUID();
+        int version = HintsDescriptor.CURRENT_VERSION;
+        long timestamp = System.currentTimeMillis();
+        ImmutableMap<String, Object> parameters = ImmutableMap.of();
+        HintsDescriptor expected = new HintsDescriptor(hostId, version, timestamp, parameters);
+
+        File directory = new File(Files.createTempDirectory("hints"));
+        directory.deleteOnExit();
+        try (HintsWriter ignored = HintsWriter.create(directory, expected))
+        {
+            ignored.totalHintsWritten.set(1234567L);
+        }
+        HintsDescriptor actual = HintsDescriptor.readFromFileQuietly(new File(directory, expected.fileName()).toPath()).get();
+        actual.loadStatsComponent(directory.toPath());
+        assertThat(actual.statistics().totalCount()).isEqualTo(1234567L);
+
+        new File(directory, HintsDescriptor.statisticsFileName(actual.hostId, actual.timestamp, actual.version)).tryDelete();
+        actual.loadStatsComponent(directory.toPath());
+        assertThat(actual.statistics()).isEqualTo(HintsDescriptor.EMPTY_STATS);
+    }
+
     private static void testSerializeDeserializeLoop(HintsDescriptor descriptor) throws IOException
     {
         // serialize to a byte array
diff --git a/test/unit/org/apache/cassandra/hints/HintsEndpointProviderTest.java b/test/unit/org/apache/cassandra/hints/HintsEndpointProviderTest.java
new file mode 100644
index 000000000000..bff8e64d8456
--- /dev/null
+++ b/test/unit/org/apache/cassandra/hints/HintsEndpointProviderTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.hints;
+
+import java.util.Optional;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class HintsEndpointProviderTest
+{
+    @BeforeClass
+    public static void setup()
+    {
+        SchemaLoader.prepareServer();
+    }
+
+    @Test
+    public void testVersionForUnknownEndpoint()
+    {
+        HintsEndpointProvider.DefaultHintsEndpointProvider provider = new HintsEndpointProvider.DefaultHintsEndpointProvider();
+
+        Optional<Integer> version = provider.versionForEndpoint(FBUtilities.getBroadcastAddressAndPort());
+
+        assertFalse("Version should not be present for unknown endpoint", version.isPresent());
+    }
+
+    @Test
+    public void testVersionForKnownEndpoint()
+    {
+        InetAddressAndPort knownEndpointAddress = FBUtilities.getBroadcastAddressAndPort().withPort(9999);
+        int version = 333;
+        MessagingService.instance().versions.set(knownEndpointAddress, version);
+
+        HintsEndpointProvider.DefaultHintsEndpointProvider provider = new HintsEndpointProvider.DefaultHintsEndpointProvider();
+        Optional<Integer> versionOption = provider.versionForEndpoint(knownEndpointAddress);
+
+        assertTrue("Version should be present for a known endpoint", versionOption.isPresent());
+        assertEquals(version, versionOption.get().intValue());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/hints/HintsReaderTest.java b/test/unit/org/apache/cassandra/hints/HintsReaderTest.java
index 56154fd95026..01f344ff9b06 100644
--- a/test/unit/org/apache/cassandra/hints/HintsReaderTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsReaderTest.java
@@ -44,6 +44,7 @@
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.metrics.HintsServiceMetrics;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaTestUtil;
@@ -54,6 +55,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.apache.cassandra.Util.dk;
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+import static org.assertj.core.api.Assertions.assertThat;
 
 public class HintsReaderTest
 {
@@ -86,6 +88,7 @@ private void generateHints(int num, String ks) throws IOException
         try (HintsWriter writer = HintsWriter.create(directory, descriptor))
         {
             ByteBuffer buffer = ByteBuffer.allocateDirect(256 * 1024);
+            long cnt0 = HintsServiceMetrics.hintsOnDisk.getCount();
             try (HintsWriter.Session session = writer.newSession(buffer))
             {
                 for (int i = 0; i < num; i++)
@@ -97,6 +100,8 @@ private void generateHints(int num, String ks) throws IOException
                     session.append(Hint.create(m, timestamp));
                 }
             }
+            assertThat(HintsServiceMetrics.hintsOnDisk.getCount()).isEqualTo(cnt0 + num * 2L);
+            assertThat(writer.totalHintsWritten.get()).isEqualTo(num * 2L);
             FileUtils.clean(buffer);
         }
 
@@ -141,7 +146,7 @@ private void verifyHint(Hint hint, long baseTimestamp, int i)
         assertEquals(timestamp, hint.creationTime);
         assertEquals(dk(bytes(i)), mutation.key());
 
-        Row row = mutation.getPartitionUpdates().iterator().next().iterator().next();
+        Row row = mutation.getPartitionUpdates().iterator().next().rowIterator().next();
         assertEquals(1, Iterables.size(row.cells()));
         ValueAccessors.assertDataEquals(bytes(i), row.clustering().get(0));
         Cell<?> cell = row.cells().iterator().next();
diff --git a/test/unit/org/apache/cassandra/hints/HintsServiceTest.java b/test/unit/org/apache/cassandra/hints/HintsServiceTest.java
index dd0eb5a6edde..81ea5d858715 100644
--- a/test/unit/org/apache/cassandra/hints/HintsServiceTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsServiceTest.java
@@ -17,33 +17,46 @@
  */
 package org.apache.cassandra.hints;
 
+import java.net.UnknownHostException;
+import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.Nullable;
 
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.MoreExecutors;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.datastax.driver.core.utils.MoreFutures;
 import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.metrics.HintsServiceMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.metrics.StorageMetrics;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.MockMessagingService;
 import org.apache.cassandra.net.MockMessagingSpy;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
 
-import static org.apache.cassandra.hints.HintsTestUtil.MockFailureDetector;
-import static org.apache.cassandra.hints.HintsTestUtil.sendHintsAndResponses;
+import static org.apache.cassandra.hints.HintsTestUtil.*;
 import static org.junit.Assert.assertEquals;
+
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionFactory;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_REWRITING_HINTS_ON_HOST_LEFT;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public class HintsServiceTest
@@ -53,10 +66,12 @@ public class HintsServiceTest
 
     private final MockFailureDetector failureDetector = new MockFailureDetector();
     private static TableMetadata metadata;
+    private final AtomicBoolean isAlive = new AtomicBoolean(true);
 
     @BeforeClass
     public static void defineSchema()
     {
+        SKIP_REWRITING_HINTS_ON_HOST_LEFT.setBoolean(true);
         SchemaLoader.prepareServer();
         StorageService.instance.initServer();
         SchemaLoader.createKeyspace(KEYSPACE,
@@ -65,6 +80,12 @@ public static void defineSchema()
         metadata = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
     }
 
+    @AfterClass
+    public static void tearDown()
+    {
+        System.clearProperty(SKIP_REWRITING_HINTS_ON_HOST_LEFT.getKey());
+    }
+
     @After
     public void cleanup()
     {
@@ -77,19 +98,93 @@ public void reinstanciateService() throws Throwable
         MessagingService.instance().inboundSink.clear();
         MessagingService.instance().outboundSink.clear();
 
+        // Hint service has to know the endpoint version to be able to send the hints. Otherwise,
+        // it wouldn't be able to make a decision whether to decode (rewrap) or not.
+        MessagingService.instance().versions.set(
+            HintsEndpointProvider.instance.endpointForHost(StorageService.instance.getLocalHostUUID()),
+            MessagingService.current_version);
+
         if (!HintsService.instance.isShutDown())
         {
             HintsService.instance.shutdownBlocking();
             HintsService.instance.deleteAllHints();
         }
 
-        failureDetector.isAlive = true;
+        isAlive.set(true);
 
-        HintsService.instance = new HintsService(failureDetector);
+        HintsService.instance = new HintsService(e -> isAlive.get());
 
         HintsService.instance.startDispatch();
     }
 
+    @Test
+    public void testHintsDroppedForUnknownHost()
+    {
+        // pause the scheduled dispatch before writing hints
+        HintsService.instance.pauseDispatch();
+
+        long totalHints = StorageMetrics.totalHints.getCount();
+
+        // write 100 hints on disk for host that is not part of cluster
+        UUID randomHost = UUID.randomUUID();
+        int numHints = 100;
+
+        HintsStore store = writeAndFlushHints(metadata, randomHost, numHints);
+        assertTrue(store.hasFiles());
+
+        // metrics should have been updated with number of create hints
+        assertEquals(totalHints + numHints, StorageMetrics.totalHints.getCount());
+
+        // re-enable dispatching
+        HintsService.instance.resumeDispatch();
+
+        // trigger a manual dispatching on host that is not part of cluster: hints should be dropped
+        HintsService.instance.dispatcherExecutor().dispatch(store);
+        ConditionFactory hints_dropped = Awaitility.await("Hints dropped");
+        hints_dropped.atMost(30, TimeUnit.SECONDS);
+        hints_dropped.untilAsserted(() ->
+                                    assertEquals(totalHints + numHints,
+                                                 StorageMetrics.totalHints.getCount())
+        );
+    }
+
+    @Test
+    public void testCountingCorruptedHints()
+    {
+        // pause the scheduled dispatch before writing hints
+        HintsService.instance.pauseDispatch();
+        try
+        {
+            long hintsOnDisk0 = HintsServiceMetrics.hintsOnDisk.getCount();
+            long corruptedHintsOnDisk0 = HintsServiceMetrics.corruptedHintsOnDisk.getCount();
+
+            UUID randomHost1 = UUID.randomUUID();
+            UUID randomHost2 = UUID.randomUUID();
+            int numHints = 10;
+
+            HintsStore store1 = writeAndFlushHints(metadata, randomHost1, numHints);
+            HintsStore store2 = writeAndFlushHints(metadata, randomHost2, numHints);
+
+            HintsDescriptor desc1 = store1.poll();
+            store1.markCorrupted(desc1);
+
+            assertThat(HintsServiceMetrics.hintsOnDisk.getCount()).isEqualTo(hintsOnDisk0 + 20);
+            assertThat(HintsServiceMetrics.corruptedHintsOnDisk.getCount()).isEqualTo(corruptedHintsOnDisk0 + 10);
+
+            store1.deleteAllHints();
+            store2.deleteAllHints();
+
+            assertThat(HintsServiceMetrics.hintsOnDisk.getCount()).isEqualTo(hintsOnDisk0);
+            assertThat(HintsServiceMetrics.corruptedHintsOnDisk.getCount()).isEqualTo(corruptedHintsOnDisk0);
+        }
+        finally
+        {
+            // re-enable dispatching
+            HintsService.instance.resumeDispatch();
+        }
+
+    }
+
     @Test
     public void testDispatchHints() throws InterruptedException, ExecutionException
     {
@@ -148,7 +243,7 @@ public void testPageRetry() throws InterruptedException, ExecutionException, Tim
         ).get();
 
         // marking the destination node as dead should stop sending hints
-        failureDetector.isAlive = false;
+        isAlive.set(false);
         spy.interceptNoMsg(20, TimeUnit.SECONDS).get();
     }
 
@@ -173,4 +268,39 @@ public void testPageSeek() throws InterruptedException, ExecutionException
         assertTrue(dispatchOffset != null);
         assertTrue(((ChecksummedDataInput.Position) dispatchOffset).sourcePosition > 0);
     }
+
+    @Test
+    public void testDeleteHintsForEndpoint() throws UnknownHostException
+    {
+        int numHints = 10;
+        TokenMetadata tokenMeta = StorageService.instance.getTokenMetadata();
+        InetAddressAndPort endpointToDeleteHints = InetAddressAndPort.getByName("1.1.1.1");
+        UUID hostIdToDeleteHints = UUID.randomUUID();
+        tokenMeta.updateHostId(hostIdToDeleteHints, endpointToDeleteHints);
+        InetAddressAndPort anotherEndpoint = InetAddressAndPort.getByName("1.1.1.2");
+        UUID anotherHostId = UUID.randomUUID();
+        tokenMeta.updateHostId(anotherHostId, anotherEndpoint);
+
+        HintsStore storeToDeleteHints = writeAndFlushHints(metadata, hostIdToDeleteHints, numHints);
+        assertTrue(storeToDeleteHints.hasFiles());
+        HintsStore anotherStore = writeAndFlushHints(metadata, anotherHostId, numHints);
+        assertTrue(anotherStore.hasFiles());
+        assertThat(HintsService.instance.getTotalFilesNum()).isEqualTo(2);
+        assertThat(HintsService.instance.getCorruptedFilesNum()).isEqualTo(0);
+        assertThat(HintsServiceMetrics.hintsOnDisk.getCount()).isEqualTo(20);
+        assertThat(HintsServiceMetrics.corruptedHintsOnDisk.getCount()).isZero();
+
+        HintsService.instance.deleteAllHintsForEndpoint(endpointToDeleteHints);
+        assertFalse(storeToDeleteHints.hasFiles());
+        assertTrue(anotherStore.hasFiles());
+        assertTrue(HintsService.instance.getCatalog().hasFiles());
+        assertThat(HintsServiceMetrics.hintsOnDisk.getCount()).isEqualTo(10);
+        assertThat(HintsServiceMetrics.corruptedHintsOnDisk.getCount()).isZero();
+
+        HintsService.instance.deleteAllHints();
+        assertEquals(0, HintsService.instance.getTotalHintsSize());
+        assertFalse(anotherStore.hasFiles());
+        assertThat(HintsServiceMetrics.hintsOnDisk.getCount()).isEqualTo(0);
+        assertThat(HintsServiceMetrics.corruptedHintsOnDisk.getCount()).isZero();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/hints/HintsTestUtil.java b/test/unit/org/apache/cassandra/hints/HintsTestUtil.java
index 727404e6e8e5..15b65fa8d134 100644
--- a/test/unit/org/apache/cassandra/hints/HintsTestUtil.java
+++ b/test/unit/org/apache/cassandra/hints/HintsTestUtil.java
@@ -17,12 +17,13 @@
  */
 package org.apache.cassandra.hints;
 
+import java.util.Collections;
 import java.util.UUID;
 
 import com.google.common.collect.Iterators;
 
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.partitions.AbstractBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.gms.IFailureDetectionEventListener;
 import org.apache.cassandra.gms.IFailureDetector;
@@ -44,12 +45,11 @@
 
 final class HintsTestUtil
 {
-    static void assertPartitionsEqual(AbstractBTreePartition expected, AbstractBTreePartition actual)
+    static void assertPartitionsEqual(Partition expected, Partition actual)
     {
         assertEquals(expected.partitionKey(), actual.partitionKey());
-        assertEquals(expected.deletionInfo(), actual.deletionInfo());
         assertEquals(expected.columns(), actual.columns());
-        assertTrue(Iterators.elementsEqual(expected.iterator(), actual.iterator()));
+        assertTrue(Iterators.elementsEqual(expected.unfilteredIterator(), actual.unfilteredIterator()));
     }
 
     static void assertHintsEqual(Hint expected, Hint actual)
@@ -78,8 +78,25 @@ static MockMessagingSpy sendHintsAndResponses(TableMetadata metadata, int noOfHi
             spy = MockMessagingService.when(verb(HINT_REQ)).respond(message);
         }
 
+        writeHints(metadata, StorageService.instance.getLocalHostUUID(), noOfHints);
+        return spy;
+    }
+
+    static HintsStore writeAndFlushHints(TableMetadata metadata, UUID hostId, int noOfHints)
+    {
+        writeHints(metadata, hostId, noOfHints);
+        HintsService.instance.flushAndFsyncBlockingly(Collections.singleton(hostId));
+
+        // close the write so hints are available for dispatching
+        HintsStore store = HintsService.instance.getCatalog().get(hostId);
+        store.closeWriter();
+
+        return store;
+    }
+
+    static void writeHints(TableMetadata metadata, UUID hostId, int noOfHints)
+    {
         // create and write noOfHints using service
-        UUID hostId = StorageService.instance.getLocalHostUUID();
         for (int i = 0; i < noOfHints; i++)
         {
             long now = Clock.Global.currentTimeMillis();
@@ -89,7 +106,6 @@ static MockMessagingSpy sendHintsAndResponses(TableMetadata metadata, int noOfHi
             Hint hint = Hint.create(builder.buildAsMutation(), now);
             HintsService.instance.write(hostId, hint);
         }
-        return spy;
     }
 
     static class MockFailureDetector implements IFailureDetector
diff --git a/test/unit/org/apache/cassandra/hints/HintsUpgradeTest.java b/test/unit/org/apache/cassandra/hints/HintsUpgradeTest.java
index 78b8f56b5034..ae9d295915ef 100644
--- a/test/unit/org/apache/cassandra/hints/HintsUpgradeTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsUpgradeTest.java
@@ -171,7 +171,7 @@ public void accept(Mutation mutation)
         {
             for (PartitionUpdate update : mutation.getPartitionUpdates())
             {
-                for (Row row : update)
+                for (Row row : update.rows())
                 {
                     if (row.clustering().size() > 0 &&
                         AsciiType.instance.compose(row.clustering().bufferAt(0)).startsWith(CELLNAME))
diff --git a/test/unit/org/apache/cassandra/index/AllIndexImplementationsTest.java b/test/unit/org/apache/cassandra/index/AllIndexImplementationsTest.java
new file mode 100644
index 000000000000..b815e4213ca3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/AllIndexImplementationsTest.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.internal.CassandraIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sasi.SASIIndex;
+
+import static org.apache.cassandra.cql3.restrictions.StatementRestrictions.INDEX_DOES_NOT_SUPPORT_DISJUNCTION;
+import static org.apache.cassandra.cql3.restrictions.StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE;
+
+/**
+ * Tests common functionality across all included index implementations.
+ */
+@RunWith(Parameterized.class)
+public class AllIndexImplementationsTest extends CQLTester
+{
+    @Parameterized.Parameter
+    public String alias;
+
+    @Parameterized.Parameter(1)
+    public Class<Index> indexClass;
+
+    @Parameterized.Parameter(2)
+    public String createIndexQuery;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> parameters()
+    {
+        List<Object[]> parameters = new LinkedList<>();
+        parameters.add(new Object[]{ "none", null, null });
+        parameters.add(new Object[]{ "legacy", CassandraIndex.class, "CREATE INDEX ON %%s(%s)" });
+        parameters.add(new Object[]{ "SASI", SASIIndex.class, "CREATE CUSTOM INDEX ON %%s(%s) USING 'org.apache.cassandra.index.sasi.SASIIndex'" });
+        parameters.add(new Object[]{ "SAI", StorageAttachedIndex.class, "CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'" });
+        return parameters;
+    }
+
+    @Test
+    public void testDisjunction()
+    {
+        createTable("CREATE TABLE %s (pk int, a int, b int, PRIMARY KEY(pk))");
+
+        boolean indexSupportsDisjuntion = StorageAttachedIndex.class.equals(indexClass);
+        boolean hasIndex = createIndexQuery != null;
+        if (hasIndex)
+            createIndex(String.format(createIndexQuery, 'a'));
+
+        execute("INSERT INTO %s (pk, a, b) VALUES (?, ?, ?)", 1, 1, 1);
+        execute("INSERT INTO %s (pk, a, b) VALUES (?, ?, ?)", 2, 2, 2);
+
+        // query with disjunctions on both columns when only one of them is indexed
+        assertDisjunction("a = 1 OR a = 2",
+                          !indexSupportsDisjuntion,
+                          hasIndex && indexSupportsDisjuntion,
+                          hasIndex ? INDEX_DOES_NOT_SUPPORT_DISJUNCTION : REQUIRES_ALLOW_FILTERING_MESSAGE,
+                          row(1), row(2));
+        assertDisjunction("a = 1 OR b = 2", true, false, row(1), row(2));
+        assertDisjunction("a = 1 AND (a = 1 OR b = 1)", true, hasIndex, row(1));
+        assertDisjunction("a = 1 AND (a = 1 OR b = 2)", true, hasIndex, row(1));
+        assertDisjunction("a = 1 AND (a = 2 OR b = 1)", true, hasIndex, row(1));
+        assertDisjunction("a = 1 AND (a = 2 OR b = 2)", true, hasIndex);
+        assertDisjunction("a = 2 AND (a = 1 OR b = 1)", true, hasIndex);
+        assertDisjunction("a = 2 AND (a = 1 OR b = 2)", true, hasIndex, row(2));
+        assertDisjunction("a = 2 AND (a = 2 OR b = 1)", true, hasIndex, row(2));
+        assertDisjunction("a = 2 AND (a = 2 OR b = 2)", true, hasIndex, row(2));
+        assertDisjunction("a = 1 OR (a = 1 AND b = 1)", true, indexSupportsDisjuntion, row(1));
+        assertDisjunction("a = 1 OR (a = 1 AND b = 2)", true, indexSupportsDisjuntion, row(1));
+        assertDisjunction("a = 1 OR (a = 2 AND b = 1)", true, indexSupportsDisjuntion, row(1));
+        assertDisjunction("a = 1 OR (a = 2 AND b = 2)", true, indexSupportsDisjuntion, row(1), row(2));
+        assertDisjunction("a = 2 OR (a = 1 AND b = 1)", true, indexSupportsDisjuntion, row(1), row(2));
+        assertDisjunction("a = 2 OR (a = 1 AND b = 2)", true, indexSupportsDisjuntion, row(2));
+        assertDisjunction("a = 2 OR (a = 2 AND b = 1)", true, indexSupportsDisjuntion, row(2));
+        assertDisjunction("a = 2 OR (a = 2 AND b = 2)", true, indexSupportsDisjuntion, row(2));
+
+        // create a second index in the remaining column, so all columns are indexed
+        if (hasIndex)
+            createIndex(String.format(createIndexQuery, 'b'));
+
+        // test with all columns indexed
+        assertDisjunction("a = 1 OR a = 2",
+                          !indexSupportsDisjuntion,
+                          hasIndex && indexSupportsDisjuntion,
+                          hasIndex ? INDEX_DOES_NOT_SUPPORT_DISJUNCTION : REQUIRES_ALLOW_FILTERING_MESSAGE,
+                          row(1), row(2));
+        assertDisjunction("a = 1 OR b = 2", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(1), row(2));
+        assertDisjunction("a = 1 AND (a = 1 OR b = 1)", !indexSupportsDisjuntion, hasIndex, row(1));
+        assertDisjunction("a = 1 AND (a = 1 OR b = 2)", !indexSupportsDisjuntion, hasIndex, row(1));
+        assertDisjunction("a = 1 AND (a = 2 OR b = 1)", !indexSupportsDisjuntion, hasIndex, row(1));
+        assertDisjunction("a = 1 AND (a = 2 OR b = 2)", !indexSupportsDisjuntion, hasIndex);
+        assertDisjunction("a = 2 AND (a = 1 OR b = 1)", !indexSupportsDisjuntion, hasIndex);
+        assertDisjunction("a = 2 AND (a = 1 OR b = 2)", !indexSupportsDisjuntion, hasIndex, row(2));
+        assertDisjunction("a = 2 AND (a = 2 OR b = 1)", !indexSupportsDisjuntion, hasIndex, row(2));
+        assertDisjunction("a = 2 AND (a = 2 OR b = 2)", !indexSupportsDisjuntion, hasIndex, row(2));
+        assertDisjunction("a = 1 OR (a = 1 AND b = 1)", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(1));
+        assertDisjunction("a = 1 OR (a = 1 AND b = 2)", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(1));
+        assertDisjunction("a = 1 OR (a = 2 AND b = 1)", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(1));
+        assertDisjunction("a = 1 OR (a = 2 AND b = 2)", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(1), row(2));
+        assertDisjunction("a = 2 OR (a = 1 AND b = 1)", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(1), row(2));
+        assertDisjunction("a = 2 OR (a = 1 AND b = 2)", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(2));
+        assertDisjunction("a = 2 OR (a = 2 AND b = 1)", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(2));
+        assertDisjunction("a = 2 OR (a = 2 AND b = 2)", !indexSupportsDisjuntion, indexSupportsDisjuntion, row(2));
+    }
+
+    private void assertDisjunction(String restrictions,
+                                   boolean requiresFiltering,
+                                   boolean shouldUseIndexes,
+                                   Object[]... rows)
+    {
+        assertDisjunction(restrictions, requiresFiltering, shouldUseIndexes, REQUIRES_ALLOW_FILTERING_MESSAGE, rows);
+    }
+
+    private void assertDisjunction(String restrictions,
+                                   boolean requiresFiltering,
+                                   boolean shouldUseIndexes,
+                                   String error,
+                                   Object[]... rows)
+    {
+        // without ALLOW FILTERING
+        String query = "SELECT pk FROM %s WHERE " + restrictions;
+        if (requiresFiltering)
+            assertInvalidThrowMessage(error, InvalidRequestException.class, query);
+        else
+            assertRowsIgnoringOrder(execute(query), rows);
+
+        // with ALLOW FILTERING
+        query += " ALLOW FILTERING";
+        assertRowsIgnoringOrder(execute(query), rows);
+
+        // verify whether the indexes are used
+        Index.QueryPlan plan = parseReadCommand(query).indexQueryPlan();
+        Assert.assertEquals(shouldUseIndexes, plan != null);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/CustomIndexTest.java b/test/unit/org/apache/cassandra/index/CustomIndexTest.java
index 9f568a877394..6d2b83f328bd 100644
--- a/test/unit/org/apache/cassandra/index/CustomIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/CustomIndexTest.java
@@ -20,7 +20,16 @@
  */
 package org.apache.cassandra.index;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -38,34 +47,46 @@
 import com.datastax.driver.core.exceptions.QueryValidationException;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.restrictions.IndexRestrictions;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
-import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.cql3.statements.ModificationStatement;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.CassandraWriteContext;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ColumnFamilyStore.FlushReason;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.index.transactions.IndexTransaction;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.Indexes;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
@@ -75,6 +96,8 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -121,11 +144,11 @@ public void indexControlsIfIncludedInBuildOnNewSSTables() throws Throwable
         flush();
 
         SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager;
-        IndexIncludedInBuild included = (IndexIncludedInBuild)indexManager.getIndexByName(toInclude);
+        IndexIncludedInBuild included = (IndexIncludedInBuild) indexManager.getIndexByName(toInclude);
         included.reset();
         assertTrue(included.rowsInserted.isEmpty());
 
-        IndexExcludedFromBuild excluded = (IndexExcludedFromBuild)indexManager.getIndexByName(toExclude);
+        IndexExcludedFromBuild excluded = (IndexExcludedFromBuild) indexManager.getIndexByName(toExclude);
         excluded.reset();
         assertTrue(excluded.rowsInserted.isEmpty());
 
@@ -149,7 +172,7 @@ public void indexReceivesWriteTimeDeletionsCorrectly() throws Throwable
         execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 3, 3);
 
         SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager;
-        StubIndex index = (StubIndex)indexManager.getIndexByName(indexName);
+        StubIndex index = (StubIndex) indexManager.getIndexByName(indexName);
         assertEquals(4, index.rowsInserted.size());
         assertTrue(index.partitionDeletions.isEmpty());
         assertTrue(index.rangeTombstones.isEmpty());
@@ -162,6 +185,7 @@ public void indexReceivesWriteTimeDeletionsCorrectly() throws Throwable
         assertEquals(1, index.partitionDeletions.size());
         assertEquals(1, index.rangeTombstones.size());
     }
+
     @Test
     public void nonCustomIndexesRequireExactlyOneTargetColumn() throws Throwable
     {
@@ -171,7 +195,7 @@ public void nonCustomIndexesRequireExactlyOneTargetColumn() throws Throwable
 
         assertInvalidMessage("Only CUSTOM indexes support multiple columns", "CREATE INDEX multi_idx on %s(v1,v2)");
         assertInvalidMessage("Only CUSTOM indexes can be created without specifying a target column",
-                           "CREATE INDEX no_targets on %s()");
+                             "CREATE INDEX no_targets on %s()");
 
         createIndex(String.format("CREATE CUSTOM INDEX multi_idx ON %%s(v1, v2) USING '%s'", StubIndex.class.getName()));
         assertIndexCreated("multi_idx", "v1", "v2");
@@ -443,7 +467,7 @@ public void customIndexDoesntSupportCustomExpressions() throws Throwable
                                   indexName,
                                   NoCustomExpressionsIndex.class.getName()));
         assertInvalidThrowMessage(Optional.of(ProtocolVersion.CURRENT),
-                                  String.format( IndexRestrictions.CUSTOM_EXPRESSION_NOT_SUPPORTED, indexName),
+                                  String.format(IndexRestrictions.CUSTOM_EXPRESSION_NOT_SUPPORTED, indexName),
                                   QueryValidationException.class,
                                   String.format("SELECT * FROM %%s WHERE expr(%s, 'foo bar baz')", indexName));
     }
@@ -504,9 +528,9 @@ public void indexSelectionPrefersMostSelectiveIndex() throws Throwable
                                   currentTable(),
                                   SettableSelectivityIndex.class.getName()));
         SettableSelectivityIndex moreSelective =
-            (SettableSelectivityIndex)getCurrentColumnFamilyStore().indexManager.getIndexByName(currentTable() + "_more_selective");
+        (SettableSelectivityIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(currentTable() + "_more_selective");
         SettableSelectivityIndex lessSelective =
-            (SettableSelectivityIndex)getCurrentColumnFamilyStore().indexManager.getIndexByName(currentTable() + "_less_selective");
+        (SettableSelectivityIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(currentTable() + "_less_selective");
         assertEquals(0, moreSelective.searchersProvided);
         assertEquals(0, lessSelective.searchersProvided);
 
@@ -535,9 +559,9 @@ public void customExpressionForcesIndexSelection() throws Throwable
                                   currentTable(),
                                   SettableSelectivityIndex.class.getName()));
         SettableSelectivityIndex moreSelective =
-            (SettableSelectivityIndex)getCurrentColumnFamilyStore().indexManager.getIndexByName(currentTable() + "_more_selective");
+        (SettableSelectivityIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(currentTable() + "_more_selective");
         SettableSelectivityIndex lessSelective =
-            (SettableSelectivityIndex)getCurrentColumnFamilyStore().indexManager.getIndexByName(currentTable() + "_less_selective");
+        (SettableSelectivityIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(currentTable() + "_less_selective");
         assertEquals(0, moreSelective.searchersProvided);
         assertEquals(0, lessSelective.searchersProvided);
 
@@ -586,7 +610,7 @@ public void reloadIndexMetadataOnBaseCfsReload()
         createIndex(String.format("CREATE CUSTOM INDEX reload_counter ON %%s() USING '%s'",
                                   CountMetadataReloadsIndex.class.getName()));
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        CountMetadataReloadsIndex index = (CountMetadataReloadsIndex)cfs.indexManager.getIndexByName("reload_counter");
+        CountMetadataReloadsIndex index = (CountMetadataReloadsIndex) cfs.indexManager.getIndexByName("reload_counter");
         assertEquals(0, index.reloads.get());
 
         // reloading the CFS, even without any metadata changes invokes the index's metadata reload task
@@ -600,7 +624,7 @@ public void notifyIndexersOfPartitionAndRowRemovalDuringCleanup() throws Throwab
         createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k,c))");
         createIndex(String.format("CREATE CUSTOM INDEX cleanup_index ON %%s() USING '%s'", StubIndex.class.getName()));
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        StubIndex index  = (StubIndex)cfs.indexManager.getIndexByName("cleanup_index");
+        StubIndex index = (StubIndex) cfs.indexManager.getIndexByName("cleanup_index");
 
         execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 0, 0);
         execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 1, 1);
@@ -629,7 +653,7 @@ public void notifyIndexersOfExpiredRowsDuringCompaction() throws Throwable
         createTable("CREATE TABLE %s (k int, c int, PRIMARY KEY (k,c))");
         createIndex(String.format("CREATE CUSTOM INDEX row_ttl_test_index ON %%s() USING '%s'", StubIndex.class.getName()));
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        StubIndex index  = (StubIndex)cfs.indexManager.getIndexByName("row_ttl_test_index");
+        StubIndex index = (StubIndex) cfs.indexManager.getIndexByName("row_ttl_test_index");
 
         execute("INSERT INTO %s (k, c) VALUES (?, ?) USING TTL 1", 0, 0);
         execute("INSERT INTO %s (k, c) VALUES (?, ?)", 0, 1);
@@ -715,7 +739,7 @@ public void indexBuildingPagesLargePartitions() throws Throwable
         // Index the partition with an Indexer which artificially simulates additional concurrent
         // flush activity by periodically issuing barriers on the read & write op groupings
         DecoratedKey targetKey = getCurrentColumnFamilyStore().decorateKey(ByteBufferUtil.bytes(0));
-        indexManager.indexPartition(targetKey, Collections.singleton(index), totalRows / 10);
+        indexManager.indexPartition(targetKey, Collections.singleton(index), PageSize.inRows(totalRows / 10));
 
         // When indexing is done check that:
         // * The base table's read ordering at finish was > the one at the start (i.e. that
@@ -774,7 +798,7 @@ public void partitionIndexTest() throws Throwable
         for (int pageSize = 1; pageSize <= 5; pageSize++)
         {
             targetKey = getCurrentColumnFamilyStore().decorateKey(ByteBufferUtil.bytes(1));
-            indexManager.indexPartition(targetKey, Collections.singleton(index), pageSize);
+            indexManager.indexPartition(targetKey, Collections.singleton(index), PageSize.inRows(pageSize));
             assertEquals(3, index.rowsInserted.size());
             assertEquals(0, index.rangeTombstones.size());
             assertTrue(index.partitionDeletions.get(0).isLive());
@@ -784,7 +808,7 @@ public void partitionIndexTest() throws Throwable
         for (int pageSize = 1; pageSize <= 5; pageSize++)
         {
             targetKey = getCurrentColumnFamilyStore().decorateKey(ByteBufferUtil.bytes(2));
-            indexManager.indexPartition(targetKey, Collections.singleton(index), pageSize);
+            indexManager.indexPartition(targetKey, Collections.singleton(index), PageSize.inRows(pageSize));
             assertEquals(1, index.rowsInserted.size());
             assertEquals(0, index.rangeTombstones.size());
             assertTrue(index.partitionDeletions.get(0).isLive());
@@ -794,7 +818,7 @@ public void partitionIndexTest() throws Throwable
         for (int pageSize = 1; pageSize <= 5; pageSize++)
         {
             targetKey = getCurrentColumnFamilyStore().decorateKey(ByteBufferUtil.bytes(3));
-            indexManager.indexPartition(targetKey, Collections.singleton(index), pageSize);
+            indexManager.indexPartition(targetKey, Collections.singleton(index), PageSize.inRows(pageSize));
             assertEquals(1, index.rowsInserted.size());
             assertEquals(2, index.rangeTombstones.size());
             assertTrue(index.partitionDeletions.get(0).isLive());
@@ -804,7 +828,7 @@ public void partitionIndexTest() throws Throwable
         for (int pageSize = 1; pageSize <= 5; pageSize++)
         {
             targetKey = getCurrentColumnFamilyStore().decorateKey(ByteBufferUtil.bytes(5));
-            indexManager.indexPartition(targetKey, Collections.singleton(index), pageSize);
+            indexManager.indexPartition(targetKey, Collections.singleton(index), PageSize.inRows(pageSize));
             assertEquals(1, index.partitionDeletions.size());
             assertFalse(index.partitionDeletions.get(0).isLive());
             index.reset();
@@ -833,7 +857,7 @@ public void partitionIsNotOverIndexed() throws Throwable
 
         // Index the partition
         DecoratedKey targetKey = getCurrentColumnFamilyStore().decorateKey(ByteBufferUtil.bytes(0));
-        indexManager.indexPartition(targetKey, Collections.singleton(index), totalRows);
+        indexManager.indexPartition(targetKey, Collections.singleton(index), PageSize.inRows(totalRows));
 
         // Assert only one partition is counted
         assertEquals(1, index.beginCalls);
@@ -864,7 +888,7 @@ public void rangeTombstoneTest() throws Throwable
 
         // Index the partition
         DecoratedKey targetKey = getCurrentColumnFamilyStore().decorateKey(ByteBufferUtil.bytes(1));
-        indexManager.indexPartition(targetKey, Sets.newHashSet(index, index2), 1);
+        indexManager.indexPartition(targetKey, Sets.newHashSet(index, index2), PageSize.inRows(1));
 
         // and both indexes should have the same range tombstone
         assertEquals(index.rangeTombstones, index2.rangeTombstones);
@@ -1007,8 +1031,8 @@ public long getEstimatedResultRows()
 
         public Searcher searcherFor(ReadCommand command)
         {
-                searchersProvided++;
-                return super.searcherFor(command);
+            searchersProvided++;
+            return super.searcherFor(command);
         }
     }
 
@@ -1169,14 +1193,21 @@ public void finish()
                         readOrderingAtFinish = baseCfs.readOrdering.getCurrent();
                 }
 
-                public void partitionDelete(DeletionTime deletionTime) { }
-
-                public void rangeTombstone(RangeTombstone tombstone) { }
+                public void partitionDelete(DeletionTime deletionTime)
+                {
+                }
 
-                public void updateRow(Row oldRowData, Row newRowData) { }
+                public void rangeTombstone(RangeTombstone tombstone)
+                {
+                }
 
-                public void removeRow(Row row) { }
+                public void updateRow(Row oldRowData, Row newRowData)
+                {
+                }
 
+                public void removeRow(Row row)
+                {
+                }
             };
         }
     }
@@ -1256,7 +1287,8 @@ public void reset()
         @Override
         public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker)
         {
-            return new SSTableFlushObserver() {
+            return new SSTableFlushObserver()
+            {
 
                 @Override
                 public void begin()
@@ -1283,7 +1315,7 @@ public void nextUnfilteredCluster(Unfiltered unfiltered)
                 }
 
                 @Override
-                public void complete()
+                public void complete(SSTable sstable)
                 {
                     completeFlushCalls.incrementAndGet();
                 }
@@ -1411,12 +1443,12 @@ public void testIndexGroupsInstancesManagement() throws Throwable
         // create two indexes belonging to the same group and verify that only one group is added to the manager
         String idx1 = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", indexClassName));
         String idx2 = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", indexClassName));
-        Supplier<IndexWithSharedGroup.Group> groupSupplier =
-                () -> indexManager.listIndexGroups().stream()
-                                                    .filter(g -> g instanceof IndexWithSharedGroup.Group)
-                                                    .map(g -> (IndexWithSharedGroup.Group) g)
-                                                    .findAny()
-                                                    .orElse(null);
+        Supplier<IndexWithSharedGroup.Group> groupSupplier = () -> indexManager.listIndexGroups()
+                                                                               .stream()
+                                                                               .filter(g -> g instanceof IndexWithSharedGroup.Group)
+                                                                               .map(g -> (IndexWithSharedGroup.Group) g)
+                                                                               .findAny().orElse(null);
+
         IndexWithSharedGroup.Group group = groupSupplier.get();
         // verify that only one group has been added to the manager
         assertEquals(2, indexManager.listIndexes().size());
@@ -1447,25 +1479,22 @@ public void testIndexGroupsInstancesManagement() throws Throwable
         assertEquals(2, indexManager.listIndexes().size());
         assertEquals(1, indexManager.listIndexGroups().size());
 
-        // drop the remaining members of the shared group and verify that it no longer exists in the manager
+        // drop the remaining members of the shared group and verify that it is removed manager
         dropIndex("DROP INDEX %s." + idx2);
         dropIndex("DROP INDEX %s." + idx5);
         assertEquals(0, indexManager.listIndexes().size());
         assertEquals(0, indexManager.listIndexGroups().size());
-        assertEquals(0, group.indexes.size());
+        assertNull(groupSupplier.get());
 
-        // create the sharing group members again and verify that they are added to a new group instance
+        // create the sharing group members again and verify that they are added to the new group instance
         createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(v1) USING '%s'", idx1, indexClassName));
         createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(v2) USING '%s'", idx2, indexClassName));
         createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(v3) USING '%s'", idx3, indexClassName));
-        IndexWithSharedGroup.Group newGroup = indexManager.listIndexGroups()
-                                                          .stream()
-                                                          .filter(g -> g instanceof IndexWithSharedGroup.Group)
-                                                          .map(g -> (IndexWithSharedGroup.Group) g)
-                                                          .findAny()
-                                                          .orElseThrow(AssertionError::new);
         assertEquals(3, indexManager.listIndexes().size());
         assertEquals(1, indexManager.listIndexGroups().size());
+
+        IndexWithSharedGroup.Group newGroup = groupSupplier.get();
+        assertNotSame(group, newGroup);
         assertEquals(3, newGroup.indexes.size());
     }
 
@@ -1557,12 +1586,6 @@ public boolean containsIndex(Index index)
                 return indexes.containsKey(index.getIndexMetadata().name);
             }
 
-            @Override
-            public boolean isSingleton()
-            {
-                return false;
-            }
-
             @Override
             public Index.Indexer indexerFor(Predicate<Index> indexSelector,
                                             DecoratedKey key,
@@ -1579,7 +1602,8 @@ public Index.Indexer indexerFor(Predicate<Index> indexSelector,
                                                      .filter(Objects::nonNull)
                                                      .collect(Collectors.toSet());
 
-                return indexers.isEmpty() ? null : new Index.Indexer() {
+                return indexers.isEmpty() ? null : new Index.Indexer()
+                {
 
                     @Override
                     public void begin()
@@ -1639,7 +1663,7 @@ public QueryPlan queryPlanFor(RowFilter rowFilter)
             }
 
             @Override
-            public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+            public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata, long keyCount)
             {
                 Set<SSTableFlushObserver> observers = indexes.values()
                                                              .stream()
@@ -1647,7 +1671,8 @@ public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNew
                                                              .filter(Objects::nonNull)
                                                              .collect(Collectors.toSet());
 
-                return new SSTableFlushObserver() {
+                return new SSTableFlushObserver()
+                {
 
                     @Override
                     public void begin()
@@ -1678,16 +1703,22 @@ public void nextUnfilteredCluster(Unfiltered unfiltered)
                     }
 
                     @Override
-                    public void complete()
+                    public void complete(SSTable sstable)
                     {
                         completeFlushCalls.incrementAndGet();
-                        observers.forEach(SSTableFlushObserver::complete);
+                        observers.forEach(obs -> obs.complete(sstable));
                     }
                 };
             }
 
             @Override
-            public Set<Component> getComponents()
+            public Set<Component> componentsForNewSSTable()
+            {
+                return Collections.emptySet();
+            }
+
+            @Override
+            public Set<Component> activeComponents(SSTableReader sstable)
             {
                 return Collections.emptySet();
             }
diff --git a/test/unit/org/apache/cassandra/index/ExpressionFilteringIndex.java b/test/unit/org/apache/cassandra/index/ExpressionFilteringIndex.java
new file mode 100644
index 000000000000..c2df17b4dbb2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/ExpressionFilteringIndex.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+
+/**
+ * An {@link Index} that selects rows whose indexed column value is equals to the requested custom expression.
+ * The implementation relies only on {@link #customExpressionFor(TableMetadata, ByteBuffer)}, while the searcher
+ * returns all the rows satisfying the key range.
+ */
+public final class ExpressionFilteringIndex extends StubIndex
+{
+    private final TableMetadata table;
+    private final ColumnMetadata column;
+    public final AtomicInteger searches = new AtomicInteger(0);
+
+    public ExpressionFilteringIndex(ColumnFamilyStore baseCfs, IndexMetadata metadata)
+    {
+        super(baseCfs, metadata);
+        this.table = baseCfs.metadata();
+        String columnName = metadata.options.get(IndexTarget.TARGET_OPTION_NAME);
+        assert columnName != null;
+        column = table.getColumn(UTF8Type.instance.decompose(columnName));
+    }
+
+    @Override
+    public AbstractType<?> customExpressionValueType()
+    {
+        return Int32Type.instance;
+    }
+
+    @Override
+    public RowFilter.CustomExpression customExpressionFor(TableMetadata cfm, ByteBuffer value)
+    {
+        return new RowFilter.CustomExpression(cfm, getIndexMetadata(), value)
+        {
+            @Override
+            public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
+            {
+                Cell cell = row.getCell(ExpressionFilteringIndex.this.column);
+                return cell != null && ByteBufferUtil.compareUnsigned(cell.buffer(), value) == 0;
+            }
+        };
+    }
+
+    @Override
+    public Searcher searcherFor(ReadCommand command)
+    {
+        return new Searcher(command)
+        {
+            @Override
+            public ReadCommand command()
+            {
+                return command;
+            }
+
+            @Override
+            public UnfilteredPartitionIterator search(ReadExecutionController executionController)
+            {
+                searches.incrementAndGet();
+
+                ReadCommand all;
+                if (command instanceof SinglePartitionReadCommand)
+                {
+                    SinglePartitionReadCommand cmd = (SinglePartitionReadCommand) command;
+                    all = SinglePartitionReadCommand.create(table,
+                                                            cmd.nowInSec(),
+                                                            cmd.partitionKey(),
+                                                            cmd.clusteringIndexFilter().getSlices(cmd.metadata()));
+                }
+                else if (command instanceof PartitionRangeReadCommand)
+                {
+                    PartitionRangeReadCommand cmd = (PartitionRangeReadCommand) command;
+                    all = PartitionRangeReadCommand.create(table,
+                                                           cmd.nowInSec(),
+                                                           ColumnFilter.all(table),
+                                                           RowFilter.none(),
+                                                           DataLimits.NONE,
+                                                           cmd.dataRange());
+                }
+                else
+                {
+                    throw new UnsupportedOperationException();
+                }
+                return all.executeLocally(ReadExecutionController.empty());
+            }
+        };
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/IndexNameTest.java b/test/unit/org/apache/cassandra/index/IndexNameTest.java
new file mode 100644
index 000000000000..6fefea75d5b3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/IndexNameTest.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index;
+
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.schema.SchemaConstants;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Parameterized.class)
+public class IndexNameTest extends CQLTester
+{
+    @Parameterized.Parameter()
+    public String createIndexQuery;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> parameters()
+    {
+        return List.of(
+        new Object[]{ "CREATE INDEX %s ON %s(%s)" },
+        new Object[]{ "CREATE CUSTOM INDEX %s ON %s(%s) USING 'org.apache.cassandra.index.sasi.SASIIndex'" },
+        new Object[]{ "CREATE CUSTOM INDEX %s ON %s(%s) USING 'StorageAttachedIndex'" }
+        );
+    }
+
+    private String intoColumnDefs(String[] columnNames)
+    {
+        StringBuilder sb = new StringBuilder();
+        for (String columnName : columnNames)
+        {
+            sb.append(columnName).append(" int,");
+        }
+        return sb.toString();
+    }
+
+    @Test
+    public void testQuotedAndLongColumnNames() throws Throwable
+    {
+        String[] columnNames
+        = new String[]{ "\"user name\"",
+                        "\"userCountry\"",
+                        "\"user-age\"",
+                        "\"/user/age\"",
+                        "\"userage\"",
+                        "\"a very very very very very very very very long field\"",
+                        "\"   a_very_very_very_very_very_very_very_very_"
+                        + "very_very_very_very_very_very_very_very_very_"
+                        + "very_very_very_very(very)very_very "
+                        + "_very_very_very_very_very_very_very_"
+                        + "very_very_very_very_very_very_very_very_"
+                        + "very_very_very_very_very_very_very_very_very \"",
+                        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
+        };
+
+        createTable("CREATE TABLE %s (key int,"
+                    + intoColumnDefs(columnNames)
+                    + "PRIMARY KEY (key))");
+        for (String columnName : columnNames)
+            createIndex(String.format(createIndexQuery, "", "%s", columnName));
+
+        for (int i = 0; i < columnNames.length; i++)
+            execute("INSERT INTO %s (key, " + columnNames[i] + ") VALUES (" + i + ", " + i + ')');
+
+        beforeAndAfterFlush(() -> {
+            for (int i = 0; i < columnNames.length; i++)
+                assertRows(execute("SELECT key, " + columnNames[i] +
+                                   " FROM %s WHERE " + columnNames[i] + " = " + i),
+                           row(i, i));
+        });
+    }
+
+    @Test
+    public void testAllLongNames()
+    {
+        String longName = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+        execute(String.format("CREATE KEYSPACE IF NOT EXISTS %s with replication = " +
+                              "{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }",
+                              longName));
+        execute(String.format("DROP TABLE IF EXISTS %s.%<s", longName));
+        createTable(String.format("CREATE TABLE %s.%<s (" +
+                                  "key int PRIMARY KEY," +
+                                  "%<s int)",
+                                  longName));
+        createIndex(String.format(createIndexQuery, "", longName + '.' + longName, longName));
+        execute(String.format("INSERT INTO %s.%<s (\"key\", %<s) VALUES (1, 1)", longName));
+        execute(String.format("INSERT INTO %s.%<s (\"key\", %<s) VALUES (2, 2)", longName));
+
+        assertRows(execute(String.format("SELECT key, %s FROM %<s.%<s WHERE %<s = 1", longName)), row(1, 1));
+
+        flush(longName, longName);
+
+        assertRows(execute(String.format("SELECT key, %s FROM %<s.%<s WHERE %<s = 1", longName)), row(1, 1));
+    }
+
+    @Test
+    public void failOnBadCharIndexName()
+    {
+        String columnName = "value";
+        createTable(String.format("CREATE TABLE %%s (key int PRIMARY KEY, %s int)", columnName));
+        assertThatThrownBy(() -> execute(String.format(createIndexQuery, "\"unacceptable index name\"", "%s", columnName)))
+        .isInstanceOf(ConfigurationException.class);
+    }
+
+    @Test
+    public void testTooLongNamesInternal() throws Throwable
+    {
+        String longName = "a".repeat(183);
+
+        createTable("CREATE TABLE %s (" +
+                    "key int PRIMARY KEY," +
+                    "value int)"
+        );
+        createIndex(String.format(createIndexQuery, longName, "%s", "value"));
+        execute(String.format("INSERT INTO %%s (\"key\", %s) VALUES (1, 1)", "value"));
+        execute(String.format("INSERT INTO %%s (\"key\", %s) VALUES (2, 2)", "value"));
+
+        beforeAndAfterFlush(() -> assertRows(execute(String.format("SELECT key, %s FROM %%s WHERE %<s = 1", "value")), row(1, 1)));
+    }
+
+    @Test
+    public void testMaxAcceptableLongNamesNewIndex() throws Throwable
+    {
+        assertEquals(182, Version.calculateIndexNameAllowedLength());
+        String longName = "a".repeat(182);
+        createTable("CREATE TABLE %s (" +
+                    "key int PRIMARY KEY," +
+                    "value int)"
+        );
+        executeNet(String.format(createIndexQuery, longName, "%s", "value"));
+
+        execute(String.format("INSERT INTO %%s (\"key\", %s) VALUES (1, 1)", "value"));
+        execute(String.format("INSERT INTO %%s (\"key\", %s) VALUES (2, 2)", "value"));
+
+        beforeAndAfterFlush(() -> assertRows(execute(String.format("SELECT key, %s FROM %%s WHERE %<s = 1", "value")), row(1, 1)));
+    }
+
+    @Test
+    public void failTooLongNamesNewIndex()
+    {
+        String longName = "a".repeat(183);
+        createTable("CREATE TABLE %s (" +
+                    "key int PRIMARY KEY," +
+                    "value int)"
+        );
+        assertThatThrownBy(() -> executeNet(String.format(createIndexQuery, longName, "%s", "value")))
+        .isInstanceOf(InvalidQueryException.class)
+        .hasMessage(String.format("Index name shouldn't be more than %s characters long (got %s chars for %s)",
+                                  SchemaConstants.INDEX_NAME_LENGTH, longName.length(), longName));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/IndexStatusManagerTest.java b/test/unit/org/apache/cassandra/index/IndexStatusManagerTest.java
index d08fec974a19..f13d0eb01744 100644
--- a/test/unit/org/apache/cassandra/index/IndexStatusManagerTest.java
+++ b/test/unit/org/apache/cassandra/index/IndexStatusManagerTest.java
@@ -24,7 +24,10 @@
 import java.util.TreeSet;
 import java.util.stream.Collectors;
 
+import org.junit.BeforeClass;
 import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.mockito.Mockito;
 
 import org.apache.cassandra.db.ConsistencyLevel;
@@ -124,6 +127,12 @@ public String getDatacenter(InetAddressAndPort endpoint)
         }
     };
 
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
     @Test
     public void shouldPrioritizeSuccessfulEndpoints() throws UnknownHostException
     {
@@ -330,8 +339,8 @@ public void shouldThrowWhenNotEnoughQueryableEndpoints()
                         .build()))
                 .isInstanceOf(ReadFailureException.class)
                 .hasMessageStartingWith("Operation failed")
-                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.252:7000")
-                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.254:7000");
+                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.252:7012")
+                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.254:7012");
     }
 
     @Test
@@ -367,9 +376,9 @@ public void shouldThrowWhenNoQueryableEndpoints()
                         .build()))
                 .isInstanceOf(ReadFailureException.class)
                 .hasMessageStartingWith("Operation failed")
-                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.253:7000")
-                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.254:7000")
-                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.255:7000");
+                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.253:7012")
+                .hasMessageContaining("INDEX_NOT_AVAILABLE from /127.0.0.254:7012")
+                .hasMessageContaining("INDEX_BUILD_IN_PROGRESS from /127.0.0.255:7012");
     }
 
     void runTest(Testcase testcase)
diff --git a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
index 45485876f984..5a756df92eba 100644
--- a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
+++ b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
@@ -18,39 +18,66 @@
 package org.apache.cassandra.index;
 
 import java.io.FileNotFoundException;
+import java.io.IOError;
 import java.net.SocketException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.collect.Sets;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
 import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.utils.JVMKiller;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.KillerForTests;
 import org.apache.cassandra.utils.concurrent.Refs;
+import org.mockito.Mockito;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class SecondaryIndexManagerTest extends CQLTester
 {
+    private static boolean backups;
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        backups = DatabaseDescriptor.isIncrementalBackupsEnabled();
+        DatabaseDescriptor.setIncrementalBackupsEnabled(false);
+    }
+
+    @AfterClass
+    public static void afterClass()
+    {
+        DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
+    }
+
     @After
     public void after()
     {
@@ -78,6 +105,38 @@ public void creatingIndexMarksTheIndexAsBuilt()
         assertMarkedAsBuilt(indexName);
     }
 
+    @Test
+    public void testIndexStatusPropagation()
+    {
+        assertFalse(Gossiper.instance.isEnabled());
+
+        // create index with Gossiper not enabled: no index status propagation threads
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createIndex("CREATE INDEX ON %s(b)");
+
+        assertTrue(Thread.getAllStackTraces().keySet()
+                         .stream()
+                         .filter(t -> t.getName().contains("StatusPropagationExecutor"))
+                         .findFirst().isEmpty());
+
+        Gossiper.instance.start(0);
+        try
+        {
+            // create index again with Gossiper started to submit index status propagation task
+            createIndex("CREATE INDEX ON %s(c)");
+
+            Thread statusPropagationThread = Thread.getAllStackTraces().keySet()
+                                                   .stream()
+                                                   .filter(t -> t.getName().contains("StatusPropagationExecutor"))
+                                                   .findFirst().get();
+            assertTrue(statusPropagationThread.isDaemon());
+        }
+        finally
+        {
+            Gossiper.instance.stop();
+        }
+    }
+
     @Test
     public void rebuilOrRecoveringIndexMarksTheIndexAsBuilt() throws Throwable
     {
@@ -91,7 +150,7 @@ public void rebuilOrRecoveringIndexMarksTheIndexAsBuilt() throws Throwable
     }
 
     @Test
-    public void recreatingIndexMarksTheIndexAsBuilt() throws Throwable
+    public void recreatingIndexMarksTheIndexAsBuilt()
     {
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String indexName = createIndex("CREATE INDEX ON %s(c)");
@@ -127,12 +186,108 @@ public void addingSSTablesMarksTheIndexAsBuilt()
         }
     }
 
+    @Test
+    public void testIndexRebuildWhenAddingSStableViaRemoteReload()
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
+
+        assertMarkedAsBuilt(indexName);
+
+        execute("Insert into %s(a,b,c) VALUES(1,1,1)");
+        assertRows(execute("SELECT * FROM %s WHERE c=1"), row(1, 1, 1));
+        flush(KEYSPACE);
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        Collection<SSTableReader> sstables = cfs.getLiveSSTables();
+
+        // unlink sstable and index context: expect no rows to be read by base and index
+        cfs.clearUnsafe();
+        IndexMetadata indexMetadata = cfs.metadata().indexes.iterator().next();
+        ((StorageAttachedIndex) cfs.getIndexManager().getIndex(indexMetadata)).getIndexContext().prepareSSTablesForRebuild(sstables);
+        assertEmpty(execute("SELECT * FROM %s WHERE a=1"));
+        assertEmpty(execute("SELECT * FROM %s WHERE c=1"));
+
+        // track sstable again: expect the query that needs the index cannot execute
+        cfs.getTracker().addInitialSSTables(sstables);
+        assertRows(execute("SELECT * FROM %s WHERE a=1"), row(1, 1, 1));
+        assertThrows(IOError.class, () -> execute("SELECT * FROM %s WHERE c=1"));
+
+        // remote reload should trigger index rebuild
+        cfs.getTracker().notifySSTablesChanged(Collections.emptySet(), sstables, OperationType.REMOTE_RELOAD, Optional.empty(), null);
+        waitForIndexBuilds(KEYSPACE, indexName); // this is needed because index build on remote reload is async
+        assertRows(execute("SELECT * FROM %s WHERE a=1"), row(1, 1, 1));
+        assertRows(execute("SELECT * FROM %s WHERE c=1"), row(1, 1, 1));
+    }
+
+    @Test
+    public void remoteReloadOnSSTableAddMarksTheIndexAsBuilt()
+    {
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+        assertMarkedAsBuilt(indexName);
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.indexManager.markAllIndexesRemoved();
+        assertNotMarkedAsBuilt(indexName);
+
+        try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+        {
+            cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, null, OperationType.REMOTE_RELOAD, Optional.empty()), cfs.getTracker());
+            waitForIndexBuilds(KEYSPACE, indexName); // this is needed because index build on remote reload is async
+            assertMarkedAsBuilt(indexName);
+        }
+    }
+
+    @Test
+    public void flushedSSTableDoesntBuildIndex()
+    {
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+        assertMarkedAsBuilt(indexName);
+
+        // Mark index removed to later chack the sstable added notification for a flushed sstable
+        // doesn't build the index:
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.indexManager.markAllIndexesRemoved();
+        assertNotMarkedAsBuilt(indexName);
+
+        try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+        {
+            cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, Mockito.mock(Memtable.class), OperationType.FLUSH, Optional.empty()), cfs.getTracker());
+            waitForIndexBuilds(KEYSPACE, indexName); // this is needed because index build on remote reload is async
+            assertNotMarkedAsBuilt(indexName);
+        }
+    }
+
+    @Test
+    public void remoteReloadOnSSTableListChangeMarksTheIndexAsBuilt()
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+        assertMarkedAsBuilt(indexName);
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.indexManager.markAllIndexesRemoved();
+        assertNotMarkedAsBuilt(indexName);
+
+        try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+        {
+            cfs.indexManager.handleNotification(new SSTableListChangedNotification(sstables, null, OperationType.REMOTE_RELOAD, Optional.empty()), cfs.getTracker());
+            waitForIndexBuilds(KEYSPACE, indexName); // this is needed because index build on remote reload is async
+            assertMarkedAsBuilt(indexName);
+        }
+    }
+
     @Test
     public void cannotRebuildRecoverWhileInitializationIsInProgress() throws Throwable
     {
         // create an index which blocks on creation
         TestingIndex.blockCreate();
-        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String defaultIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
         String readOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
         String writeOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
@@ -316,7 +471,7 @@ public void addingSSTableWithBuildFailureWhileRebuildIsInProgress() throws Throw
         final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
         final AtomicBoolean error = new AtomicBoolean();
 
-        // verify it's built after initialization:
+        // verify it's built:
         assertMarkedAsBuilt(indexName);
 
         // rebuild the index in another thread, but make it block:
@@ -419,7 +574,7 @@ public void initializingIndexNotQueryableButMaybeWritable()
     public void initializingIndexNotQueryableButMaybeNotWritableAfterPartialRebuild()
     {
         TestingIndex.blockCreate();
-        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String defaultIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
         String readOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
         String writeOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
@@ -444,6 +599,7 @@ public void initializingIndexNotQueryableButMaybeNotWritableAfterPartialRebuild(
         }
         catch (Throwable ex)
         {
+            ex.printStackTrace();
             assertTrue(ex.getMessage().contains("configured to fail"));
         }
         assertFalse(isQueryable(defaultIndexName));
@@ -550,7 +706,7 @@ public void handleJVMStablityOnFailedCreate()
     private void handleJVMStablityOnFailedCreate(Throwable throwable, boolean shouldKillJVM)
     {
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
 
         try
         {
@@ -590,7 +746,7 @@ private void handleJVMStablityOnFailedRebuild(Throwable throwable, boolean shoul
         String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
 
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
 
         try
         {
@@ -792,9 +948,9 @@ public void build()
                             }
 
                             @Override
-                            public CompactionInfo getCompactionInfo()
+                            public Progress getProgress()
                             {
-                                return builder.getCompactionInfo();
+                                return builder.getProgress();
                             }
                         };
                     }
diff --git a/test/unit/org/apache/cassandra/index/StubIndex.java b/test/unit/org/apache/cassandra/index/StubIndex.java
index cfaff698ec10..7d62479790ac 100644
--- a/test/unit/org/apache/cassandra/index/StubIndex.java
+++ b/test/unit/org/apache/cassandra/index/StubIndex.java
@@ -20,8 +20,10 @@
 
 import java.util.*;
 import java.util.concurrent.Callable;
+import java.util.function.BiFunction;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.Operator;
@@ -92,6 +94,10 @@ public AbstractType<?> customExpressionValueType()
         return UTF8Type.instance;
     }
 
+    public RowFilter postIndexQueryFilter(RowFilter filter)
+    {
+        return filter;
+    }
     public RowFilter getPostIndexQueryFilter(RowFilter filter)
     {
         return filter;
@@ -212,6 +218,11 @@ public Searcher searcherFor(final ReadCommand command)
         return new Searcher(command);
     }
 
+    public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand readCommand)
+    {
+        return (iter, command) -> iter;
+    }
+
     protected class Searcher implements Index.Searcher
     {
         private final ReadCommand command;
diff --git a/test/unit/org/apache/cassandra/index/StubIndexGroup.java b/test/unit/org/apache/cassandra/index/StubIndexGroup.java
index 22dfbe262b80..01d7464e154b 100644
--- a/test/unit/org/apache/cassandra/index/StubIndexGroup.java
+++ b/test/unit/org/apache/cassandra/index/StubIndexGroup.java
@@ -34,6 +34,7 @@
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.TableMetadata;
 
 /**
@@ -67,12 +68,6 @@ public boolean containsIndex(Index index)
         return indexes.contains(index);
     }
 
-    @Override
-    public boolean isSingleton()
-    {
-        return false;
-    }
-
     @Override
     public Index.Indexer indexerFor(Predicate<Index> indexSelector,
                                     DecoratedKey key,
@@ -93,12 +88,19 @@ public Index.QueryPlan queryPlanFor(RowFilter rowFilter)
     }
 
     @Override
-    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata, long keyCount)
     {
         return null;
     }
 
-    public Set<Component> getComponents()
+    @Override
+    public Set<Component> componentsForNewSSTable()
+    {
+        return Collections.emptySet();
+    }
+
+    @Override
+    public Set<Component> activeComponents(SSTableReader sstable)
     {
         return Collections.emptySet();
     }
diff --git a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
index 672777cc2524..c1dd07a3bea1 100644
--- a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
+++ b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
@@ -24,13 +24,12 @@
 import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
+import java.util.function.BiFunction;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
 import com.google.common.collect.ImmutableSet;
 
-import org.apache.cassandra.Util;
-import org.apache.cassandra.index.TargetParser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,17 +40,20 @@
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.index.SecondaryIndexBuilder;
+import org.apache.cassandra.index.TargetParser;
 import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.io.sstable.ReducingKeyIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -60,6 +62,7 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Refs;
+import org.apache.cassandra.Util;
 
 import static org.apache.cassandra.index.internal.CassandraIndex.getFunctions;
 import static org.apache.cassandra.index.internal.CassandraIndex.indexCfsMetadata;
@@ -209,15 +212,27 @@ public long getEstimatedResultRows()
         return indexCfs.getMeanEstimatedCellPerPartitionCount();
     }
 
+    /**
+     * No post processing of query results, just return them unchanged
+     */
+    public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command)
+    {
+        return (partitionIterator, readCommand) -> partitionIterator;
+    }
+
     public RowFilter getPostIndexQueryFilter(RowFilter filter)
     {
-        return getTargetExpression(filter.getExpressions()).map(filter::without)
-                                                           .orElse(filter);
+        return getTargetExpression(filter).map(filter::without).orElse(filter);
     }
 
-    private Optional<RowFilter.Expression> getTargetExpression(List<RowFilter.Expression> expressions)
+    private Optional<RowFilter.Expression> getTargetExpression(RowFilter rowFilter)
     {
-        return expressions.stream().filter(this::supportsExpression).findFirst();
+        for (RowFilter.Expression expression : rowFilter.expressions())
+        {
+            if (supportsExpression(expression))
+                return Optional.of(expression);
+        }
+        return Optional.empty();
     }
 
     public Index.Searcher searcherFor(ReadCommand command)
@@ -237,7 +252,7 @@ public void validate(PartitionUpdate update, ClientState state) throws InvalidRe
                 validateClusterings(update);
                 break;
             case REGULAR:
-                validateRows(update);
+                validateRows(update.rows());
                 break;
             case STATIC:
                 validateRows(Collections.singleton(update.staticRow()));
@@ -245,11 +260,11 @@ public void validate(PartitionUpdate update, ClientState state) throws InvalidRe
         }
     }
 
-    protected CBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
-                                               ClusteringPrefix<?> prefix,
-                                               CellPath path)
+    protected ClusteringBuilder buildIndexClusteringPrefix(ByteBuffer partitionKey,
+                                                           ClusteringPrefix<?> prefix,
+                                                           CellPath path)
     {
-        CBuilder builder = CBuilder.create(getIndexComparator());
+        ClusteringBuilder builder = ClusteringBuilder.create(getIndexComparator());
         builder.add(partitionKey);
         return builder;
     }
@@ -514,7 +529,7 @@ private void validatePartitionKey(DecoratedKey partitionKey) throws InvalidReque
     private void validateClusterings(PartitionUpdate update) throws InvalidRequestException
     {
         assert indexedColumn.isClusteringColumn();
-        for (Row row : update)
+        for (Row row : update.rows())
             validateIndexedValue(getIndexedValue(null, row.clustering(), null));
     }
 
@@ -588,7 +603,7 @@ private void invalidate()
     {
         // interrupt in-progress compactions
         Collection<ColumnFamilyStore> cfss = Collections.singleton(indexCfs);
-        CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true);
+        CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true, TableOperation.StopTrigger.INVALIDATE_INDEX);
         CompactionManager.instance.waitForCessation(cfss, (sstable) -> true);
         indexCfs.keyspace.writeOrder.awaitNewBarrier();
         Util.flush(indexCfs);
diff --git a/test/unit/org/apache/cassandra/index/sai/IndexingSchemaLoader.java b/test/unit/org/apache/cassandra/index/sai/IndexingSchemaLoader.java
new file mode 100644
index 000000000000..c6fab79ee259
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/IndexingSchemaLoader.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai;
+
+import java.util.HashMap;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class IndexingSchemaLoader extends SchemaLoader
+{
+    public static TableMetadata.Builder ndiCFMD(String ksName, String cfName)
+    {
+        TableMetadata.Builder builder =
+                TableMetadata.builder(ksName, cfName)
+                             .addPartitionKeyColumn("id", UTF8Type.instance)
+                             .addRegularColumn("first_name", UTF8Type.instance)
+                             .addRegularColumn("last_name", UTF8Type.instance)
+                             .addRegularColumn("age", Int32Type.instance)
+                             .addRegularColumn("height", Int32Type.instance)
+                             .addRegularColumn("timestamp", LongType.instance)
+                             .addRegularColumn("address", UTF8Type.instance)
+                             .addRegularColumn("score", DoubleType.instance)
+                             .addRegularColumn("comment", UTF8Type.instance)
+                             .addRegularColumn("comment_suffix_split", UTF8Type.instance)
+                             .addRegularColumn("/output/full-name/", UTF8Type.instance)
+                             .addRegularColumn("/data/output/id", UTF8Type.instance)
+                             .addRegularColumn("first_name_prefix", UTF8Type.instance);
+
+        Indexes.Builder indexes = Indexes.builder();
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_first_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "first_name");
+        }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_last_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "last_name");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_age", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "age");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_timestamp", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "timestamp");
+
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_address", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "address");
+                    put("case_sensitive", "false");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_score", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "score");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_comment", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "comment");
+                    put("case_sensitive", "true");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_comment_suffix_split", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "comment_suffix_split");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_output_full_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "/output/full-name/");
+                    put("case_sensitive", "false");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_data_output_id", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "/data/output/id");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_first_name_prefix", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "first_name_prefix");
+                }}));
+
+        return builder.indexes(indexes.build());
+    }
+
+    public static TableMetadata.Builder clusteringNDICFMD(String ksName, String cfName)
+    {
+        return clusteringNDICFMD(ksName, cfName, "location", "age", "height", "score");
+    }
+
+    public static TableMetadata.Builder clusteringNDICFMD(String ksName, String cfName, String...indexedColumns)
+    {
+        Indexes.Builder indexes = Indexes.builder();
+        for (String indexedColumn : indexedColumns)
+        {
+            indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_" + indexedColumn, IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+            {{
+                put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                put(IndexTarget.TARGET_OPTION_NAME, indexedColumn);
+            }}));
+        }
+
+        return TableMetadata.builder(ksName, cfName)
+                            .addPartitionKeyColumn("name", UTF8Type.instance)
+                            .addClusteringColumn("location", UTF8Type.instance)
+                            .addClusteringColumn("age", Int32Type.instance)
+                            .addRegularColumn("height", Int32Type.instance)
+                            .addRegularColumn("score", DoubleType.instance)
+                            .addStaticColumn("nickname", UTF8Type.instance)
+                            .indexes(indexes.build());
+    }
+
+    public static TableMetadata.Builder staticNDICFMD(String ksName, String cfName)
+    {
+        TableMetadata.Builder builder =
+                TableMetadata.builder(ksName, cfName)
+                             .addPartitionKeyColumn("sensor_id", Int32Type.instance)
+                             .addStaticColumn("sensor_type", UTF8Type.instance)
+                             .addClusteringColumn("date", LongType.instance)
+                             .addRegularColumn("value", DoubleType.instance)
+                             .addRegularColumn("variance", Int32Type.instance);
+
+        Indexes.Builder indexes = Indexes.builder();
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_sensor_type", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "sensor_type");
+            put("case_sensitive", "false");
+        }}));
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_value", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "value");
+        }}));
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_variance", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "variance");
+        }}));
+
+        return builder.indexes(indexes.build());
+    }
+
+    public static TableMetadata.Builder fullTextSearchNDICFMD(String ksName, String cfName)
+    {
+        TableMetadata.Builder builder =
+                TableMetadata.builder(ksName, cfName)
+                             .addPartitionKeyColumn("song_id", UUIDType.instance)
+                             .addRegularColumn("title", UTF8Type.instance)
+                             .addRegularColumn("artist", UTF8Type.instance);
+
+        Indexes.Builder indexes = Indexes.builder();
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_title", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "title");
+        }}));
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_artist", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "artist");
+            put("case_sensitive", "false");
+
+        }}));
+
+        return builder.indexes(indexes.build());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/MultiVersionComposabilityTest.java b/test/unit/org/apache/cassandra/index/sai/MultiVersionComposabilityTest.java
new file mode 100644
index 000000000000..d92999453dcd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/MultiVersionComposabilityTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+
+public class MultiVersionComposabilityTest extends SAITester
+{
+    @Test
+    public void testMultiVersionMapRangeQuery() throws Throwable
+    {
+        // The map type is selected because the encoding changes over time.
+        createTable("CREATE TABLE %s (pk int, data map<text,int>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(data)) USING 'StorageAttachedIndex'");
+
+        // Don't want compaction changing versions on us
+        disableCompaction();
+        waitForTableIndexesQueryable();
+
+        // Copy and randomize versions to ensure we can cover different orders of versions.
+        var versions = new ArrayList<>(Version.ALL);
+        Collections.shuffle(versions, getRandom().getRandom());
+
+        // Maps from pk to value for each key in the map.
+        var a = new HashMap<Integer, Integer>();
+        var b = new HashMap<Integer, Integer>();
+        var c = new HashMap<Integer, Integer>();
+
+        boolean compactFirst = true;
+        for (Version version : versions)
+        {
+            // Flush before writing so we have a memtable to test as well.
+            if (!compactFirst)
+                flush();
+            SAIUtil.setLatestVersion(version);
+            // Insert 100 random rows with mostly the same keys.
+            for (int i = 0; i < 100; i++)
+            {
+                var data = new HashMap<String, Integer>();
+                // Using random int opens us up to the case where we might have some overwrites, which is
+                // a good thing to cover in this test
+                var pk = getRandom().nextIntBetween(0, 1000);
+                addRandomValue(pk, a, "a", data);
+                addRandomValue(pk, b, "b", data);
+                addRandomValue(pk, c, "c", data);
+                execute("INSERT INTO %s (pk, data) VALUES (?, ?)", pk, data);
+            }
+
+            // Compact the first sstable so we can trigger it to have many segments, which
+            // some additional logic.
+            if (compactFirst)
+            {
+                // Creat 3 segments per flush
+                SegmentBuilder.updateLastValidSegmentRowId(10);
+                flush();
+                compact();
+                compactFirst = false;
+            }
+        }
+
+        var invertedA = invertMap(a);
+        var invertedB = invertMap(b);
+        var invertedC = invertMap(c);
+
+        beforeAndAfterFlush(() -> {
+            // Confirm that we get the correct results for equality searches on the extrema of each map key.
+            // Note: the min/max queries caught a bug in the way we compare terms when the current version
+            // is different from the index's version.
+            assertMinMaxValues("a", invertedA);
+            assertMinMaxValues("b", invertedB);
+            assertMinMaxValues("c", invertedC);
+            // Run some range queries.
+            assertRangeSlice("a", invertedA);
+            assertRangeSlice("b", invertedB);
+            assertRangeSlice("c", invertedC);
+        });
+    }
+
+    private void assertMinMaxValues(String key, NavigableMap<Integer, SortedSet<Integer>> map)
+    {
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE data[?] = ?", key, map.firstKey()), row(map.firstEntry().getValue().toArray()));
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE data[?] = ?", key, map.lastKey()), row(map.lastEntry().getValue().toArray()));
+    }
+
+    private void assertRangeSlice(String key, NavigableMap<Integer, SortedSet<Integer>> map)
+    {
+        for (int i = 0; i < 10; i++)
+        {
+            var from = getRandom().nextIntBetween(map.firstKey(), map.lastKey());
+            var to = getRandom().nextIntBetween(map.tailMap(from).firstKey(), map.lastKey());
+            var rows = map.subMap(from, true, to, true).values().stream()
+                          .flatMap(SortedSet::stream).map(CQLTester::row).toArray(Object[][]::new);
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE data[?] >= ? AND data[?] <= ?", key, from, key, to), rows);
+        }
+    }
+
+    private void addRandomValue(int pk, Map<Integer, Integer> rows, String key, Map<String, Integer> data)
+    {
+        var value = getRandom().nextInt();
+        data.put(key, value);
+        rows.put(pk, value);
+    }
+
+    private NavigableMap<Integer, SortedSet<Integer>> invertMap(Map<Integer, Integer> rows)
+    {
+        var inverted = new TreeMap<Integer, SortedSet<Integer>>();
+        for (Map.Entry<Integer, Integer> entry : rows.entrySet())
+        {
+            inverted.computeIfAbsent(entry.getValue(), k -> new TreeSet<>()).add(entry.getKey());
+        }
+        return inverted;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/SAITester.java b/test/unit/org/apache/cassandra/index/sai/SAITester.java
index ffa3d6a75e9c..5c0f4a3e002a 100644
--- a/test/unit/org/apache/cassandra/index/sai/SAITester.java
+++ b/test/unit/org/apache/cassandra/index/sai/SAITester.java
@@ -20,307 +20,298 @@
  */
 package org.apache.cassandra.index.sai;
 
-import java.io.DataInput;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.io.RandomAccessFile;
-import java.math.BigDecimal;
-import java.math.BigInteger;
 import java.nio.channels.FileChannel;
 import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 import javax.management.AttributeNotFoundException;
+import javax.management.MalformedObjectNameException;
 import javax.management.ObjectName;
 
+import com.google.common.base.Predicates;
 import com.google.common.collect.Sets;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.rules.TestRule;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.datastax.driver.core.QueryTrace;
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Session;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.config.CassandraRelevantProperties;
+import com.datastax.driver.core.exceptions.ReadFailureException;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.format.OnDiskFormat;
 import org.apache.cassandra.index.sai.disk.format.Version;
-import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.index.sai.utils.ResourceLeakDetector;
+import org.apache.cassandra.inject.ActionBuilder;
+import org.apache.cassandra.inject.Expression;
 import org.apache.cassandra.inject.Injection;
 import org.apache.cassandra.inject.Injections;
 import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.TOCComponent;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.MockSchema;
 import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.snapshot.TableSnapshot;
-import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.lucene.codecs.CodecUtil;
 
-import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_RANDOM_SEED;
 import static org.apache.cassandra.inject.ActionBuilder.newActionBuilder;
+import static org.apache.cassandra.inject.Expression.expr;
 import static org.apache.cassandra.inject.Expression.quote;
 import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
-public abstract class SAITester extends CQLTester
+public class SAITester extends CQLTester
 {
-    protected static final Logger logger = LoggerFactory.getLogger(SAITester.class);
+    static
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
 
-    protected static final String CREATE_KEYSPACE_TEMPLATE = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = " +
-                                                             "{'class': 'SimpleStrategy', 'replication_factor': '1'}";
+    protected static final String CREATE_KEYSPACE_TEMPLATE = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}";
 
     protected static final String CREATE_TABLE_TEMPLATE = "CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT, v2 TEXT) WITH compaction = " +
-                                                          "{'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
-    protected static final String CREATE_INDEX_TEMPLATE = "CREATE INDEX IF NOT EXISTS ON %%s(%s) USING 'sai'";
-
-    protected static final ColumnIdentifier V1_COLUMN_IDENTIFIER = ColumnIdentifier.getInterned("v1", true);
-    protected static final ColumnIdentifier V2_COLUMN_IDENTIFIER = ColumnIdentifier.getInterned("v2", true);
+            "{'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
+    protected static final String CREATE_INDEX_TEMPLATE = "CREATE CUSTOM INDEX IF NOT EXISTS ON %%s(%s) USING 'StorageAttachedIndex'";
 
-    protected static final Injections.Counter indexBuildCounter = Injections.newCounter("IndexBuildCounter")
-                                                                            .add(newInvokePoint().onClass(CompactionManager.class)
-                                                                                                 .onMethod("submitIndexBuild",
-                                                                                                           "SecondaryIndexBuilder",
-                                                                                                           "ActiveCompactionsTracker"))
-                                                                            .build();
+    protected static int ASSERTION_TIMEOUT_SECONDS = 15;
 
-    protected static final Injections.Counter perSSTableValidationCounter = Injections.newCounter("PerSSTableValidationCounter")
-                                                                                      .add(newInvokePoint().onClass(OnDiskFormat.class)
-                                                                                                           .onMethod("validatePerSSTableIndexComponents"))
-                                                                                      .build();
-
-    protected static final Injections.Counter perColumnValidationCounter = Injections.newCounter("PerColumnValidationCounter")
-                                                                                     .add(newInvokePoint().onClass(OnDiskFormat.class)
-                                                                                                          .onMethod("validatePerColumnIndexComponents"))
-                                                                                     .build();
+    protected static Injections.Counter.CounterBuilder addConditions(Injections.Counter.CounterBuilder builder, Consumer<ActionBuilder.ConditionsBuilder> adder)
+    {
+        adder.accept(builder.lastActionBuilder().conditions());
+        return builder;
+    }
 
-    private static Randomization random;
 
-    public static final ClusteringComparator EMPTY_COMPARATOR = new ClusteringComparator();
+    protected static final Injections.Counter INDEX_BUILD_COUNTER = Injections.newCounter("IndexBuildCounter")
+                                                                              .add(newInvokePoint().onClass(CompactionManager.class)
+                                                                                                   .onMethod("submitIndexBuild", "SecondaryIndexBuilder", "TableOperationObserver"))
+                                                                              .build();
 
-    public static final PrimaryKey.Factory TEST_FACTORY = new PrimaryKey.Factory(Murmur3Partitioner.instance, EMPTY_COMPARATOR);
+    protected static final Injections.Counter perSSTableValidationCounter = addConditions(Injections.newCounter("PerSSTableValidationCounter")
+                                                                                      .add(newInvokePoint().onClass("IndexDescriptor$IndexComponentsImpl")
+                                                                                                           .onMethod("validateComponents")),
+                                                                                          b -> b.not().when(expr(Expression.THIS).method("isPerIndexGroup").args())
+    ).build();
 
-    @BeforeClass
-    public static void setUpClass()
-    {
-        CQLTester.setUpClass();
+    protected static final Injections.Counter perColumnValidationCounter = addConditions(Injections.newCounter("PerColumnValidationCounter")
+                                                                                     .add(newInvokePoint().onClass("IndexDescriptor$IndexComponentsImpl")
+                                                                                                          .onMethod("validateComponents")),
+                                                                                         b -> b.when(expr(Expression.THIS).method("isPerIndexGroup").args())
+    ).build();
 
-        // Ensure that the on-disk format statics are loaded before the test run
-        Version.LATEST.onDiskFormat();
-    }
+    protected static ColumnIdentifier V1_COLUMN_IDENTIFIER = ColumnIdentifier.getInterned("v1", true);
+    protected static ColumnIdentifier V2_COLUMN_IDENTIFIER = ColumnIdentifier.getInterned("v2", true);
 
-    @Rule
-    public TestRule testRules = new ResourceLeakDetector();
+    public static final ClusteringComparator EMPTY_COMPARATOR = new ClusteringComparator();
 
-    @Rule
-    public FailureWatcher failureRule = new FailureWatcher();
+    public static final PrimaryKey.Factory TEST_FACTORY = Version.latest().onDiskFormat().newPrimaryKeyFactory(EMPTY_COMPARATOR);
 
-    @After
-    public void removeAllInjections()
-    {
-        Injections.deleteAll();
-        CassandraRelevantProperties.SAI_MINIMUM_POSTINGS_LEAVES.reset();
-        CassandraRelevantProperties.SAI_POSTINGS_SKIP.reset();
-        V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER.setLimitBytes(V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMIT);
-    }
 
-    public static Randomization getRandom()
+    static
     {
-        if (random == null)
-            random = new Randomization();
-        return random;
+        Version.ALL.size();
     }
 
     public enum CorruptionType
     {
         REMOVED
-        {
-            @Override
-            public void corrupt(File file) throws IOException
-            {
-                if (!file.tryDelete())
-                    throw new IOException("Unable to delete file: " + file);
-            }
-        },
+                {
+                    @Override
+                    public void corrupt(File file) throws IOException
+                    {
+                        if (!file.tryDelete())
+                            throw new IOException("Unable to delete file: " + file);
+                    }
+                },
         EMPTY_FILE
-        {
-            @Override
-            public void corrupt(File file) throws IOException
-            {
-                try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE))
                 {
-                    channel.truncate(0);
-                }
-            }
-        },
+                    @Override
+                    public void corrupt(File file) throws IOException
+                    {
+                        FileChannel.open(file.toPath(), StandardOpenOption.WRITE).truncate(0).close();
+                    }
+                },
         TRUNCATED_HEADER
-        {
-            @Override
-            public void corrupt(File file) throws IOException
-            {
-                try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE))
                 {
-                    channel.truncate(2);
-                }
-            }
-        },
+                    @Override
+                    public void corrupt(File file) throws IOException
+                    {
+                        FileChannel.open(file.toPath(), StandardOpenOption.WRITE).truncate(2).close();
+                    }
+                },
         TRUNCATED_DATA
-        {
-            @Override
-            public void corrupt(File file) throws IOException
-            {
-                // header length is not fixed, use footer length to navigate a given data position
-                try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE))
                 {
-                    channel.truncate(file.length() - CodecUtil.footerLength() - 2);
-                }
-            }
-        },
+                    @Override
+                    public void corrupt(File file) throws IOException
+                    {
+                        // header length is not fixed, use footer length to navigate a given data position
+                        FileChannel.open(file.toPath(), StandardOpenOption.WRITE).truncate(file.length() - CodecUtil.footerLength() - 2).close();
+                    }
+                },
         TRUNCATED_FOOTER
-        {
-            @Override
-            public void corrupt(File file) throws IOException
-            {
-                try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE))
                 {
-                    channel.truncate(file.length() - CodecUtil.footerLength() + 2);
-                }
-            }
-        },
+                    @Override
+                    public void corrupt(File file) throws IOException
+                    {
+                        FileChannel.open(file.toPath(), StandardOpenOption.WRITE).truncate(file.length() - CodecUtil.footerLength() + 2).close();
+                    }
+                },
         APPENDED_DATA
-        {
-            @Override
-            public void corrupt(File file) throws IOException
-            {
-                try (RandomAccessFile raf = new RandomAccessFile(file.toJavaIOFile(), "rw"))
                 {
-                    raf.seek(file.length());
-
-                    byte[] corruptedData = new byte[100];
-                    new Random().nextBytes(corruptedData);
-                    raf.write(corruptedData);
-                }
-            }
-        };
+                    @Override
+                    public void corrupt(File file) throws IOException
+                    {
+                        try (RandomAccessFile raf = new RandomAccessFile(file.toJavaIOFile(), "rw"))
+                        {
+                            raf.seek(file.length());
+
+                            byte[] corruptedData = new byte[100];
+                            new Random().nextBytes(corruptedData);
+                            raf.write(corruptedData);
+                        }
+                    }
+                };
 
         public abstract void corrupt(File file) throws IOException;
     }
 
-    public static StorageAttachedIndex createMockIndex(ColumnMetadata column)
-    {
-        TableMetadata table = TableMetadata.builder(column.ksName, column.cfName)
-                                           .addPartitionKeyColumn("pk", Int32Type.instance)
-                                           .addRegularColumn(column.name, column.type)
-                                           .partitioner(Murmur3Partitioner.instance)
-                                           .caching(CachingParams.CACHE_NOTHING)
-                                           .build();
-
-        Map<String, String> options = new HashMap<>();
-        options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName());
-        options.put("target", column.name.toString());
-
-        IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata(column.name.toString(), IndexMetadata.Kind.CUSTOM, options);
-
-        ColumnFamilyStore cfs = MockSchema.newCFS(table);
+    @Rule
+    public TestRule testRules = new ResourceLeakDetector();
 
-        return new StorageAttachedIndex(cfs, indexMetadata);
+    @After
+    public void removeAllInjections()
+    {
+        Injections.deleteAll();
     }
 
-    public static StorageAttachedIndex createMockIndex(AbstractType<?> cellType)
+    /**
+     * Enable external execution of all queries because we want to use reconciliation in SELECT queries so that we can
+     * simulate the application of the entire row filter in the coordinator node, even if unit tests are not multinode.
+     */
+    @BeforeClass
+    public static void setUpClass()
     {
-        TableMetadata table = TableMetadata.builder("test", "test")
-                                           .addPartitionKeyColumn("pk", Int32Type.instance)
-                                           .addRegularColumn("val", cellType)
-                                           .partitioner(Murmur3Partitioner.instance)
-                                           .caching(CachingParams.CACHE_NOTHING)
-                                           .build();
-
-        Map<String, String> options = new HashMap<>();
-        options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName());
-        options.put("target", "val");
+        CQLTester.setUpClass();
+        CQLTester.enableCoordinatorExecution();
+    }
 
-        IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata("val", IndexMetadata.Kind.CUSTOM, options);
+    /**
+     * Creates a SAI index on the current table, waiting for it to become queryable.
+     *
+     * @param column the name of the indexed column, maybe with {@code FULL()}, {@code KEYS()} or {@code VALUES()} spec
+     * @param options the index options, of the form {@code "{'option1': value1, 'option2': value2, ...}"}.
+     * @return the name of the created index
+     */
+    public String createSAIIndex(String column, @Nullable String options)
+    {
+        String query = String.format(CREATE_INDEX_TEMPLATE, column);
 
-        ColumnFamilyStore cfs = MockSchema.newCFS(table);
+        if (options != null)
+            query += " WITH OPTIONS = " + options;
 
-        return new StorageAttachedIndex(cfs, indexMetadata);
+        return createIndex(query);
     }
 
-    public static IndexTermType createIndexTermType(AbstractType<?> cellType)
+    public static IndexContext createIndexContext(String name, AbstractType<?> validator, ColumnFamilyStore cfs)
     {
-        return IndexTermType.create(ColumnMetadata.regularColumn("sai", "internal", "val", cellType), Collections.emptyList(), IndexTarget.Type.SIMPLE);
+        return new IndexContext(cfs.getKeyspaceName(),
+                                cfs.getTableName(),
+                                cfs.metadata().id,
+                                UTF8Type.instance,
+                                new ClusteringComparator(),
+                                ColumnMetadata.regularColumn("sai", "internal", name, validator),
+                                IndexTarget.Type.SIMPLE,
+                                IndexMetadata.fromSchemaMetadata(name, IndexMetadata.Kind.CUSTOM, null),
+                                cfs);
     }
 
-    public IndexIdentifier createIndexIdentifier(String indexName)
+    public static IndexContext createIndexContext(String name, AbstractType<?> validator)
     {
-        return createIndexIdentifier(keyspace(), currentTable(), indexName);
+        return new IndexContext("test_ks",
+                                "test_cf",
+                                TableId.generate(),
+                                UTF8Type.instance,
+                                new ClusteringComparator(),
+                                ColumnMetadata.regularColumn("sai", "internal", name, validator),
+                                IndexTarget.Type.SIMPLE,
+                                IndexMetadata.fromSchemaMetadata(name, IndexMetadata.Kind.CUSTOM, null),
+                                MockSchema.newCFS("test_ks"));
     }
 
-    public static IndexIdentifier createIndexIdentifier(String keyspaceName, String tableName, String indexName)
+    public static IndexContext createIndexContext(String columnName, String indexName, AbstractType<?> validator)
     {
-        return new IndexIdentifier(keyspaceName, tableName, indexName);
+        return new IndexContext("test_ks",
+                                "test_cf",
+                                TableId.generate(),
+                                UTF8Type.instance,
+                                new ClusteringComparator(),
+                                ColumnMetadata.regularColumn("sai", "internal", columnName, validator),
+                                IndexTarget.Type.SIMPLE,
+                                IndexMetadata.fromSchemaMetadata(indexName, IndexMetadata.Kind.CUSTOM, null),
+                                MockSchema.newCFS("test_ks"));
     }
 
-    protected StorageAttachedIndexGroup getCurrentIndexGroup()
+    public IndexContext getIndexContext(String indexName)
     {
-        return StorageAttachedIndexGroup.getIndexGroup(getCurrentColumnFamilyStore());
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        return StorageAttachedIndexGroup.getIndexGroup(cfs)
+                                 .getIndexes()
+                                 .stream()
+                                 .map(StorageAttachedIndex::getIndexContext)
+                                 .filter(ctx -> ctx.getIndexName().equals(indexName))
+                                 .findFirst()
+                                 .orElseThrow();
     }
 
-    protected void dropIndex(IndexIdentifier indexIdentifier) throws Throwable
+    public static Vector<Float> vector(float... v)
     {
-        dropIndex("DROP INDEX %s." + indexIdentifier.indexName);
+        var v2 = new Float[v.length];
+        for (int i = 0; i < v.length; i++)
+            v2[i] = v[i];
+        return new Vector<>(v2);
     }
 
     protected void simulateNodeRestart()
@@ -330,9 +321,11 @@ protected void simulateNodeRestart()
 
     protected void simulateNodeRestart(boolean wait)
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        cfs.indexManager.listIndexes().forEach(index -> ((StorageAttachedIndexGroup)cfs.indexManager.getIndexGroup(index)).reset());
-        cfs.indexManager.listIndexes().forEach(cfs.indexManager::buildIndex);
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        cfs.indexManager.listIndexes().forEach(index -> {
+            ((StorageAttachedIndexGroup)cfs.indexManager.getIndexGroup(index)).reset();
+        });
+        cfs.indexManager.listIndexes().forEach(index -> cfs.indexManager.buildIndex(index));
         cfs.indexManager.executePreJoinTasksBlocking(true);
         if (wait)
         {
@@ -340,50 +333,89 @@ protected void simulateNodeRestart(boolean wait)
         }
     }
 
-    protected void corruptIndexComponent(IndexComponent indexComponent, CorruptionType corruptionType) throws Exception
+    protected static IndexDescriptor loadDescriptor(SSTableReader sstable, ColumnFamilyStore cfs)
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        return IndexDescriptor.load(sstable,
+                                    StorageAttachedIndexGroup.getIndexGroup(cfs).getIndexes().stream().map(StorageAttachedIndex::getIndexContext).collect(Collectors.toSet()));
+    }
+
+    protected void corruptIndexComponent(IndexComponentType indexComponentType, CorruptionType corruptionType) throws Exception
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+
         for (SSTableReader sstable : cfs.getLiveSSTables())
         {
-            File file = IndexDescriptor.create(sstable).fileFor(indexComponent);
+            File file = loadDescriptor(sstable, cfs).perSSTableComponents().get(indexComponentType).file();
             corruptionType.corrupt(file);
         }
     }
 
-    protected void corruptIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier, CorruptionType corruptionType) throws Exception
+    protected void corruptIndexComponent(IndexComponentType indexComponentType, IndexContext indexContext, CorruptionType corruptionType) throws Exception
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+
         for (SSTableReader sstable : cfs.getLiveSSTables())
         {
-            File file = IndexDescriptor.create(sstable).fileFor(indexComponent, indexIdentifier);
+            File file = loadDescriptor(sstable, cfs).perIndexComponents(indexContext).get(indexComponentType).file();
             corruptionType.corrupt(file);
         }
     }
 
-    protected boolean indexNeedsFullRebuild(String index)
+    protected boolean verifyChecksum(IndexContext context)
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        return cfs.indexManager.needsFullRebuild(index);
-    }
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
 
-    protected void verifyInitialIndexFailed(String indexName)
-    {
-        // Verify that the initial index build fails...
-        waitForAssert(() -> assertTrue(indexNeedsFullRebuild(indexName)));
+        try (ColumnFamilyStore.RefViewFragment rvf = cfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL)))
+        {
+            for (SSTableReader sstable : rvf.sstables)
+            {
+                IndexDescriptor indexDescriptor = loadDescriptor(sstable, cfs);
+                if (indexDescriptor.isIndexEmpty(context))
+                    continue;
+                if (!indexDescriptor.perSSTableComponents().validateComponents(sstable, cfs.getTracker(), true, false)
+                    || !indexDescriptor.perIndexComponents(context).validateComponents(sstable, cfs.getTracker(), true, false))
+                    return false;
+            }
+        }
+
+        return true;
     }
 
-    protected boolean verifyChecksum(IndexTermType indexContext, IndexIdentifier indexIdentifier)
+    protected void verifySAIVersionInUse(Version expectedVersion, IndexContext... contexts)
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
 
         for (SSTableReader sstable : cfs.getLiveSSTables())
         {
-            IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
-            if (!indexDescriptor.validatePerSSTableComponents(IndexValidation.CHECKSUM, true, false)
-                || !indexDescriptor.validatePerIndexComponents(indexContext, indexIdentifier, IndexValidation.CHECKSUM, true, false))
-                return false;
+            IndexDescriptor indexDescriptor = loadDescriptor(sstable, cfs);
+
+            assertEquals(indexDescriptor.perSSTableComponents().version(), expectedVersion);
+            SSTableContext ssTableContext = group.sstableContextManager().getContext(sstable);
+            // This is to make sure the context uses the actual files we think
+            assertEquals(ssTableContext.usedPerSSTableComponents().version(), expectedVersion);
+
+            for (IndexContext indexContext : contexts)
+            {
+                assertEquals(indexDescriptor.perIndexComponents(indexContext).version(), expectedVersion);
+
+                for (SSTableIndex sstableIndex : indexContext.getView())
+                {
+                    if (sstableIndex.isEmpty())
+                        continue;
+
+                    // Make sure the index does use components of the proper version.
+                    assertEquals(sstableIndex.usedPerIndexComponents().version(), expectedVersion);
+                }
+            }
         }
-        return true;
+    }
+
+    protected static void assertFailureReason(ReadFailureException e, RequestFailureReason reason)
+    {
+        int expected = reason.codeForNativeProtocol();
+        int actual = e.getFailuresMap().get(FBUtilities.getBroadcastAddressAndPort().getAddress());
+        assertEquals(expected, actual);
     }
 
     protected Object getMBeanAttribute(ObjectName name, String attribute) throws Exception
@@ -413,9 +445,19 @@ protected Object getMetricValue(ObjectName metricObjectName)
         return metricValue;
     }
 
+    protected void startCompaction() throws Throwable
+    {
+        Iterable<ColumnFamilyStore> tables = StorageService.instance.getValidColumnFamilies(true, false, KEYSPACE, currentTable());
+        tables.forEach(table ->
+        {
+            long gcBefore = CompactionManager.getDefaultGcBefore(table, FBUtilities.nowInSeconds());
+            CompactionManager.instance.submitMaximal(table, gcBefore, false);
+        });
+    }
+
     public void waitForCompactions()
     {
-        waitForAssert(() -> assertFalse(CompactionManager.instance.isCompacting(ColumnFamilyStore.all(), ssTableReader -> true)), 10, TimeUnit.SECONDS);
+        waitForAssert(() -> assertFalse(CompactionManager.instance.isCompacting(ColumnFamilyStore.all(), Predicates.alwaysTrue())), 10, TimeUnit.SECONDS);
     }
 
     protected void waitForCompactionsFinished()
@@ -423,6 +465,18 @@ protected void waitForCompactionsFinished()
         waitForAssert(() -> assertEquals(0, getCompactionTasks()), 10, TimeUnit.SECONDS);
     }
 
+    protected void waitForEquals(ObjectName name, ObjectName name2)
+    {
+        waitForAssert(() -> {
+            long jmxValue = ((Number) getMetricValue(name)).longValue();
+            long jmxValue2 = ((Number) getMetricValue(name2)).longValue();
+
+            jmxValue2 += 2; // add 2 for the first 2 queries in setupCluster
+
+            assertEquals(jmxValue, jmxValue2);
+        }, 10, TimeUnit.SECONDS);
+    }
+
     protected void waitForEquals(ObjectName name, long value)
     {
         waitForAssert(() -> assertEquals(value, ((Number) getMetricValue(name)).longValue()), 10, TimeUnit.SECONDS);
@@ -432,9 +486,8 @@ protected ObjectName objectName(String name, String keyspace, String table, Stri
     {
         try
         {
-            return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex," +
-                                                "keyspace=%s,table=%s,index=%s,scope=%s,name=%s",
-                                                keyspace, table, index, type, name));
+            return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex,keyspace=%s,table=%s,index=%s,scope=%s,name=%s",
+                    keyspace, table, index, type, name));
         }
         catch (Throwable ex)
         {
@@ -446,9 +499,8 @@ protected ObjectName objectNameNoIndex(String name, String keyspace, String tabl
     {
         try
         {
-            return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex," +
-                                                "keyspace=%s,table=%s,scope=%s,name=%s",
-                                                keyspace, table, type, name));
+            return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex,keyspace=%s,table=%s,scope=%s,name=%s",
+                    keyspace, table, type, name));
         }
         catch (Throwable ex)
         {
@@ -456,21 +508,6 @@ protected ObjectName objectNameNoIndex(String name, String keyspace, String tabl
         }
     }
 
-    protected long getSegmentBufferSpaceLimit()
-    {
-        return V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER.limitBytes();
-    }
-
-    protected long getSegmentBufferUsedBytes()
-    {
-        return V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER.currentBytesUsed();
-    }
-
-    protected int getColumnIndexBuildsInProgress()
-    {
-        return SegmentBuilder.getActiveBuilderCount();
-    }
-
     protected void upgradeSSTables()
     {
         try
@@ -485,23 +522,26 @@ protected void upgradeSSTables()
 
     protected long totalDiskSpaceUsed()
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
         return cfs.metric.totalDiskSpaceUsed.getCount();
     }
 
     protected long indexDiskSpaceUse()
     {
-        return getCurrentIndexGroup().totalDiskUsage();
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        return Objects.requireNonNull(StorageAttachedIndexGroup.getIndexGroup(cfs)).totalDiskUsage();
     }
 
     protected int getOpenIndexFiles()
     {
-        return getCurrentIndexGroup().openIndexFiles();
+        ColumnFamilyStore cfs = Schema.instance.getKeyspaceInstance(KEYSPACE).getColumnFamilyStore(currentTable());
+        return StorageAttachedIndexGroup.getIndexGroup(cfs).openIndexFiles();
     }
 
     protected long getDiskUsage()
     {
-        return getCurrentIndexGroup().diskUsage();
+        ColumnFamilyStore cfs = Schema.instance.getKeyspaceInstance(KEYSPACE).getColumnFamilyStore(currentTable());
+        return StorageAttachedIndexGroup.getIndexGroup(cfs).diskUsage();
     }
 
     protected void verifyNoIndexFiles()
@@ -509,135 +549,183 @@ protected void verifyNoIndexFiles()
         assertTrue(indexFiles().isEmpty());
     }
 
-    protected void verifyIndexFiles(IndexTermType indexTermType,
-                                    IndexIdentifier indexIdentifier,
-                                    int indexFiles)
+    // Verify every sstables is indexed correctly and the components are valid.
+    protected void verifyIndexComponentFiles(@Nullable IndexContext numericIndexContext, @Nullable IndexContext stringIndexContext)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        for (SSTableReader sstable : cfs.getLiveSSTables())
+        {
+            // We create a descriptor from scratch, to ensure this discover from disk directly.
+            IndexDescriptor descriptor = loadDescriptor(sstable, cfs);
+
+            // Note that validation makes sure that all expected components exists, on top of validating those.
+            descriptor.perSSTableComponents().validateComponents(sstable, cfs.getTracker(), true, false);
+            if (numericIndexContext != null)
+                descriptor.perIndexComponents(numericIndexContext).validateComponents(sstable, cfs.getTracker(), true, false);
+            if (stringIndexContext != null)
+                descriptor.perIndexComponents(stringIndexContext).validateComponents(sstable, cfs.getTracker(), true, false);
+        }
+    }
+
+
+    // Note: this assumes the checked component files are at generation 0, which is not always the case with rebuild.
+    // The `verifyIndexComponentFiles` method is probably a safer replacement overall, but many test still use this so
+    // we keep it for now.
+    protected void verifyIndexFiles(IndexContext numericIndexContext, IndexContext literalIndexContext, int numericFiles, int literalFiles)
     {
-        verifyIndexFiles(indexTermType, indexIdentifier, indexFiles, indexFiles, indexFiles);
+        verifyIndexFiles(numericIndexContext,
+                         literalIndexContext,
+                         Math.max(numericFiles, literalFiles),
+                         numericFiles,
+                         literalFiles,
+                         numericFiles,
+                         literalFiles);
     }
 
-    protected void verifyIndexFiles(IndexTermType indexTermType,
-                                    IndexIdentifier indexIdentifier,
+    // Same as namesake
+    protected void verifyIndexFiles(IndexContext numericIndexContext,
+                                    IndexContext literalIndexContext,
                                     int perSSTableFiles,
-                                    int perColumnFiles,
-                                    int completionMarkers)
+                                    int numericFiles,
+                                    int literalFiles,
+                                    int numericCompletionMarkers,
+                                    int literalCompletionMarkers)
     {
         Set<File> indexFiles = indexFiles();
 
-        for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perSSTableIndexComponents(false))
+        for (IndexComponentType indexComponentType : Version.latest().onDiskFormat().perSSTableComponentTypes())
         {
-            Component component = SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent, null));
-            Set<File> tableFiles = componentFiles(indexFiles, component);
+            Set<File> tableFiles = componentFiles(indexFiles, new Component(SSTableFormat.Components.Types.CUSTOM, Version.latest().fileNameFormatter().format(indexComponentType, (String)null, 0)));
             assertEquals(tableFiles.toString(), perSSTableFiles, tableFiles.size());
         }
 
-        for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType))
+        if (literalIndexContext != null)
         {
-            String componentName = Version.LATEST.fileNameFormatter().format(indexComponent, indexIdentifier);
-            Component component = SSTableFormat.Components.Types.CUSTOM.createComponent(componentName);
-            Set<File> stringIndexFiles = componentFiles(indexFiles, component);
-            if (isBuildCompletionMarker(indexComponent))
-                assertEquals(completionMarkers, stringIndexFiles.size());
-            else
-                assertEquals(stringIndexFiles.toString(), perColumnFiles, stringIndexFiles.size());
+            for (IndexComponentType indexComponentType : Version.latest().onDiskFormat().perIndexComponentTypes(literalIndexContext))
+            {
+                Set<File> stringIndexFiles = componentFiles(indexFiles,
+                                                            new Component(SSTableFormat.Components.Types.CUSTOM,
+                                                                          Version.latest().fileNameFormatter().format(indexComponentType,
+                                                                                                                      literalIndexContext,
+                                                                                                                      0)));
+                if (isBuildCompletionMarker(indexComponentType))
+                    assertEquals(literalCompletionMarkers, stringIndexFiles.size());
+                else
+                    assertEquals(stringIndexFiles.toString(), literalFiles, stringIndexFiles.size());
+            }
         }
-    }
 
-    protected void verifySSTableIndexes(IndexIdentifier indexIdentifier, int count)
-    {
-        try
-        {
-            verifySSTableIndexes(indexIdentifier, count, count);
-        }
-        catch (Exception e)
+        if (numericIndexContext != null)
         {
-            throw Throwables.unchecked(e);
+            for (IndexComponentType indexComponentType : Version.latest().onDiskFormat().perIndexComponentTypes(numericIndexContext))
+            {
+                Set<File> numericIndexFiles = componentFiles(indexFiles,
+                                                             new Component(SSTableFormat.Components.Types.CUSTOM,
+                                                                           Version.latest().fileNameFormatter().format(indexComponentType,
+                                                                                                                       numericIndexContext,
+                                                                                                                       0)));
+                if (isBuildCompletionMarker(indexComponentType))
+                    assertEquals(numericCompletionMarkers, numericIndexFiles.size());
+                else
+                    assertEquals(numericIndexFiles.toString(), numericFiles, numericIndexFiles.size());
+            }
         }
     }
 
-    protected void verifySSTableIndexes(IndexIdentifier indexIdentifier, int sstableContextCount, int sstableIndexCount)
+    protected boolean isBuildCompletionMarker(IndexComponentType indexComponentType)
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        StorageAttachedIndexGroup indexGroup = getCurrentIndexGroup();
-        int contextCount = indexGroup.sstableContextManager().size();
-        assertEquals("Expected " + sstableContextCount +" SSTableContexts, but got " + contextCount, sstableContextCount, contextCount);
+        return (indexComponentType == IndexComponentType.GROUP_COMPLETION_MARKER) ||
+               (indexComponentType == IndexComponentType.COLUMN_COMPLETION_MARKER);
 
-        StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexIdentifier.indexName);
-        Collection<SSTableIndex> sstableIndexes = sai == null ? Collections.emptyList() : sai.view().getIndexes();
-        assertEquals("Expected " + sstableIndexCount +" SSTableIndexes, but got " + sstableIndexes.toString(), sstableIndexCount, sstableIndexes.size());
     }
 
-    protected boolean isBuildCompletionMarker(IndexComponent indexComponent)
+    protected Set<File> indexFiles()
     {
-        return (indexComponent == IndexComponent.GROUP_COMPLETION_MARKER) ||
-               (indexComponent == IndexComponent.COLUMN_COMPLETION_MARKER);
-
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        return cfs.getDirectories().getCFDirectories()
+                  .stream()
+                  .flatMap(dir -> Arrays.stream(dir.tryList()))
+                  .filter(File::isFile)
+                  .filter(file -> Version.tryParseFileName(file.name()).isPresent())
+                  .collect(Collectors.toSet());
     }
 
-    protected Set<File> indexFiles()
+    /**
+     * Checks that the set of all SAI index files in the TOC for all sstables (of the {@link #currentTable()}) are
+     * exactly the provided files.
+     *
+     * @param files expected SAI index files (typically the result of {@link #indexFiles()} above). Should not contain
+     *              non-SAI sstable files (or the test will fail).
+     */
+    protected void assertIndexFilesInToc(Set<File> files) throws IOException
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        Set<Component> components = cfs.indexManager.listIndexGroups()
-                                                    .stream()
-                                                    .filter(g -> g instanceof StorageAttachedIndexGroup)
-                                                    .map(Index.Group::getComponents)
-                                                    .flatMap(Set::stream)
-                                                    .collect(Collectors.toSet());
-
-        Set<File> indexFiles = new HashSet<>();
-        for (Component component : components)
+        Set<String> found = files.stream().map(File::name).collect(Collectors.toSet());
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        for (SSTableReader sstable : cfs.getLiveSSTables())
         {
-            List<File> files = cfs.getDirectories().getCFDirectories()
-                                  .stream()
-                                  .flatMap(dir -> Arrays.stream(dir.tryList()))
-                                  .filter(File::isFile)
-                                  .filter(f -> f.name().endsWith(component.name))
-                                  .collect(Collectors.toList());
-            indexFiles.addAll(files);
+            for (Component component : TOCComponent.loadTOC(sstable.descriptor, false))
+            {
+                if (component.type != SSTableFormat.Components.Types.CUSTOM || !component.name.startsWith(Version.SAI_DESCRIPTOR))
+                    continue;
+
+                String tocFile = sstable.descriptor.fileFor(component).name();
+                if (!found.remove(tocFile))
+                    fail(String.format("TOC of %s contains unexpected SAI index file %s (all expected: %s)", sstable, tocFile, files));
+            }
         }
-        return indexFiles;
+        assertTrue("The following files could not be found in the sstable TOC files: " + found, found.isEmpty());
     }
 
-    protected Set<File> componentFiles(Collection<File> indexFiles, Component component)
+    protected ObjectName bufferSpaceObjectName(String name) throws MalformedObjectNameException
     {
-        return indexFiles.stream().filter(c -> c.name().endsWith(component.name)).collect(Collectors.toSet());
+        return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex,name=%s", name));
     }
 
-    public String createTable(String query)
+    protected long getSegmentBufferSpaceLimit() throws Exception
     {
-        return createTable(KEYSPACE, query);
+        ObjectName limitBytesName = bufferSpaceObjectName("SegmentBufferSpaceLimitBytes");
+        return (long) (Long) getMetricValue(limitBytesName);
     }
 
-    @Override
-    public UntypedResultSet execute(String query, Object... values)
+    protected Object getSegmentBufferUsedBytes() throws Exception
     {
-        return executeFormattedQuery(formatQuery(query), values);
+        ObjectName usedBytesName = bufferSpaceObjectName("SegmentBufferSpaceUsedBytes");
+        return getMetricValue(usedBytesName);
     }
 
-    @Override
-    public Session sessionNet()
+    protected Object getColumnIndexBuildsInProgress() throws Exception
     {
-        return sessionNet(getDefaultVersion());
+        ObjectName buildersInProgressName = bufferSpaceObjectName("ColumnIndexBuildsInProgress");
+        return getMetricValue(buildersInProgressName);
     }
 
-    public void flush(String keyspace, String table)
+    protected void verifySSTableIndexes(String indexName, int count)
     {
-        ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(table);
-        if (store != null)
-            store.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        try
+        {
+            verifySSTableIndexes(indexName, count, count);
+        }
+        catch (Exception e)
+        {
+            throw Throwables.unchecked(e);
+        }
     }
 
-    public void compact(String keyspace, String table)
+    protected void verifySSTableIndexes(String indexName, int sstableContextCount, int sstableIndexCount)
     {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(cfs);
+        int contextCount = indexGroup == null ? 0 : indexGroup.sstableContextManager().size();
+        assertEquals("Expected " + sstableContextCount +" SSTableContexts, but got " + contextCount, sstableContextCount, contextCount);
 
-        ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(table);
-        if (store != null)
-            store.forceMajorCompaction();
+        StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName);
+        Collection<SSTableIndex> sstableIndexes = sai == null ? Collections.emptyList() : sai.getIndexContext().getView().getIndexes();
+        assertEquals("Expected " + sstableIndexCount +" SSTableIndexes, but got " + sstableIndexes.toString(), sstableIndexCount, sstableIndexes.size());
     }
 
     protected void truncate(boolean snapshot)
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
         if (snapshot)
             cfs.truncateBlocking();
         else
@@ -651,12 +739,22 @@ protected void rebuildIndexes(String... indexes)
 
     protected void reloadSSTableIndex()
     {
-        getCurrentIndexGroup().unsafeReload();
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        StorageAttachedIndexGroup.getIndexGroup(cfs).unsafeReload();
+    }
+
+    // `reloadSSTalbleIndex` calls `unsafeReload`, which clear all contexts, and then recreate from scratch. This method
+    // simply signal updates to every sstable without previously clearing anything.
+    protected void reloadSSTableIndexInPlace()
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+        group.onSSTableChanged(Collections.emptySet(), cfs.getLiveSSTables(), group.getIndexes(), true);
     }
 
     protected void runInitializationTask() throws Exception
     {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
         for (Index i : cfs.indexManager.listIndexes())
         {
             assert i instanceof StorageAttachedIndex;
@@ -670,52 +768,21 @@ protected int getCompactionTasks()
         return CompactionManager.instance.getActiveCompactions() + CompactionManager.instance.getPendingTasks();
     }
 
-    protected int snapshot(String snapshotName)
-    {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        TableSnapshot snapshot = cfs.snapshot(snapshotName);
-        return snapshot.getDirectories().size();
-    }
-
-    protected void restoreSnapshot(String snapshot)
-    {
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        Directories.SSTableLister lister = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(snapshot);
-        restore(cfs, lister);
-    }
-
-    protected void restore(ColumnFamilyStore cfs, Directories.SSTableLister lister)
-    {
-        File dataDirectory = cfs.getDirectories().getDirectoryForNewSSTables();
-
-        for (File file : lister.listFiles())
-        {
-            file.tryMove(new File(dataDirectory.absolutePath() + File.pathSeparator() + file.name()));
-        }
-        cfs.loadNewSSTables();
-    }
-
-    public static void copyTo(DataInput in, OutputStream out, int length) throws IOException
+    protected String getSingleTraceStatement(Session session, String query, String contains) throws Throwable
     {
-        byte[] buffer = new byte[64 * 1024];
-        int copiedBytes = 0;
-
-        while (copiedBytes + buffer.length < length)
-        {
-            in.readFully(buffer);
-            out.write(buffer);
-            copiedBytes += buffer.length;
-        }
+        query = String.format(query, KEYSPACE + "." + currentTable());
+        QueryTrace trace = session.execute(session.prepare(query).bind().enableTracing()).getExecutionInfo().getQueryTrace();
+        waitForTracingEvents();
 
-        if (copiedBytes < length)
+        for (QueryTrace.Event event : trace.getEvents())
         {
-            int left = length - copiedBytes;
-            in.readFully(buffer, 0, left);
-            out.write(buffer, 0, left);
+            if (event.getDescription().contains(contains))
+                return event.getDescription();
         }
+        return null;
     }
 
-    protected void assertNumRows(int expected, String query, Object... args)
+    protected void assertNumRows(int expected, String query, Object... args) throws Throwable
     {
         ResultSet rs = executeNet(String.format(query, args));
         assertEquals(expected, rs.all().size());
@@ -729,6 +796,36 @@ protected static Injection newFailureOnEntry(String name, Class<?> invokeClass,
                          .build();
     }
 
+    protected int snapshot(String snapshotName) throws IOException
+    {
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        TableSnapshot snapshot = cfs.snapshot(snapshotName);
+        return snapshot.getDirectories().size();
+    }
+
+    protected List<String> restoreSnapshot(String snapshot)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        Directories.SSTableLister lister = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(snapshot);
+        return restore(cfs, lister);
+    }
+
+    protected List<String> restore(ColumnFamilyStore cfs, Directories.SSTableLister lister)
+    {
+        File dataDirectory = cfs.getDirectories().getDirectoryForNewSSTables();
+
+        List<String> fileNames = new ArrayList<>();
+        for (File file : lister.listFiles())
+        {
+            if (file.tryMove(new File(dataDirectory.absolutePath() + File.pathSeparator() + file.name())))
+            {
+                fileNames.add(file.name());
+            }
+        }
+        cfs.loadNewSSTables();
+        return fileNames;
+    }
+
     protected void assertValidationCount(int perSSTable, int perColumn)
     {
         Assert.assertEquals(perSSTable, perSSTableValidationCounter.get());
@@ -759,11 +856,11 @@ protected void verifyIndexComponentsNotIncludedInSSTable() throws Exception
     private void verifySSTableComponents(String table, boolean indexComponentsExist) throws Exception
     {
         ColumnFamilyStore cfs = Objects.requireNonNull(Schema.instance.getKeyspaceInstance(KEYSPACE)).getColumnFamilyStore(table);
-        for (SSTable sstable : cfs.getLiveSSTables())
+        for (SSTableReader sstable : cfs.getLiveSSTables())
         {
-            Set<Component> components = sstable.getComponents();
+            Set<Component> components = sstable.components();
             StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
-            Set<Component> ndiComponents = group == null ? Collections.emptySet() : group.getComponents();
+            Set<Component> ndiComponents = group == null ? Collections.emptySet() : group.activeComponents(sstable);
 
             Set<Component> diff = Sets.difference(ndiComponents, components);
             if (indexComponentsExist)
@@ -777,162 +874,22 @@ private void verifySSTableComponents(String table, boolean indexComponentsExist)
         }
     }
 
-    protected static void setBDKPostingsWriterSizing(int minimumPostingsLeaves, int postingsSkip)
-    {
-        CassandraRelevantProperties.SAI_MINIMUM_POSTINGS_LEAVES.setString(Integer.toString(minimumPostingsLeaves));
-        CassandraRelevantProperties.SAI_POSTINGS_SKIP.setString(Integer.toString(postingsSkip));
-    }
-
-    protected static void setSegmentWriteBufferSpace(final int segmentSize)
-    {
-        V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER.setLimitBytes(segmentSize);
-    }
-
-    protected String getSingleTraceStatement(Session session, String query, String contains)
-    {
-        query = String.format(query, KEYSPACE + '.' + currentTable());
-        QueryTrace trace = session.execute(session.prepare(query).bind().enableTracing()).getExecutionInfo().getQueryTrace();
-        waitForTracingEvents();
-
-        for (QueryTrace.Event event : trace.getEvents())
-        {
-            if (event.getDescription().contains(contains))
-                return event.getDescription();
-        }
-        return null;
-    }
-
-    protected ByteComparable integerToByteComparable(int value)
-    {
-        return v -> Int32Type.instance.asComparableBytes(Int32Type.instance.decompose(value), v);
-    }
-
-    /**
-     *  Because the tracing executor is single threaded, submitting an empty event should ensure
-     *  that all tracing events mutations have been applied.
-     */
-    protected void waitForTracingEvents()
+    protected Set<File> componentFiles(Collection<File> indexFiles, Component component)
     {
-        try
-        {
-            Stage.TRACING.executor().submit(() -> {}).get();
-        }
-        catch (Throwable t)
-        {
-            JVMStabilityInspector.inspectThrowable(t);
-            logger.error("Failed to wait for tracing events", t);
-        }
+        return indexFiles.stream().filter(c -> c.name().endsWith(component.name)).collect(Collectors.toSet());
     }
 
-    public static class Randomization
+    protected Set<File> componentFiles(Collection<File> indexFiles, IndexComponentType indexComponentType, IndexContext indexContext)
     {
-        private final long seed;
-        private final Random random;
-
-        Randomization()
-        {
-            seed = TEST_RANDOM_SEED.getLong(System.nanoTime());
-            random = new Random(seed);
-        }
-
-        public void printSeedOnFailure()
-        {
-            logger.error("Randomized test failed. To rerun test use -D{}={}", TEST_RANDOM_SEED.getKey(), seed);
-        }
-
-        public int nextInt()
-        {
-            return random.nextInt();
-        }
-
-        public int nextInt(int max)
-        {
-            return RandomInts.randomInt(random, max);
-        }
-
-        public int nextIntBetween(int minValue, int maxValue)
-        {
-            return RandomInts.randomIntBetween(random, minValue, maxValue);
-        }
-
-        public long nextLong()
-        {
-            return random.nextLong();
-        }
-
-        public short nextShort()
-        {
-            return (short)random.nextInt(Short.MAX_VALUE + 1);
-        }
-
-        public byte nextByte()
-        {
-            return (byte)random.nextInt(Byte.MAX_VALUE + 1);
-        }
-
-        public BigInteger nextBigInteger(int maxNumBits)
-        {
-            return new BigInteger(RandomInts.randomInt(random, maxNumBits), random);
-        }
-
-        public BigInteger nextBigInteger(int minNumBits, int maxNumBits)
-        {
-            return new BigInteger(RandomInts.randomIntBetween(random, minNumBits, maxNumBits), random);
-        }
-
-        public BigDecimal nextBigDecimal(int minUnscaledValue, int maxUnscaledValue, int minScale, int maxScale)
-        {
-            return BigDecimal.valueOf(RandomInts.randomIntBetween(random, minUnscaledValue, maxUnscaledValue),
-                                      RandomInts.randomIntBetween(random, minScale, maxScale));
-        }
-
-        public float nextFloat()
-        {
-            return random.nextFloat();
-        }
-
-        public double nextDouble()
-        {
-            return random.nextDouble();
-        }
-
-        public String nextAsciiString(int minLength, int maxLength)
-        {
-            return RandomStrings.randomAsciiOfLengthBetween(random, minLength, maxLength);
-        }
-
-        public String nextTextString(int minLength, int maxLength)
-        {
-            return RandomStrings.randomRealisticUnicodeOfLengthBetween(random, minLength, maxLength);
-        }
-
-        public boolean nextBoolean()
-        {
-            return random.nextBoolean();
-        }
-
-        public void nextBytes(byte[] bytes)
-        {
-            random.nextBytes(bytes);
-        }
+        String componentName = Version.latest().fileNameFormatter().format(indexComponentType, indexContext, 0);
+        return indexFiles.stream().filter(c -> c.name().endsWith(componentName)).collect(Collectors.toSet());
     }
 
-    public static class FailureWatcher extends TestWatcher
-    {
-        @Override
-        protected void failed(Throwable e, Description description)
-        {
-            if (random != null)
-                random.printSeedOnFailure();
-        }
-    }
     /**
      * Run repeated verification task concurrently with target test
      */
     protected static class TestWithConcurrentVerification
     {
-        private static final int verificationMaxInMs = 300_000; // 300s
-
         private final Runnable verificationTask;
         private final CountDownLatch verificationStarted = new CountDownLatch(1);
 
@@ -940,6 +897,7 @@ protected static class TestWithConcurrentVerification
         private final CountDownLatch taskCompleted = new CountDownLatch(1);
 
         private final int verificationIntervalInMs;
+        private final int verificationMaxInMs = 300_000; // 300s
 
         public TestWithConcurrentVerification(Runnable verificationTask, Runnable targetTask)
         {
@@ -960,6 +918,7 @@ public TestWithConcurrentVerification(Runnable verificationTask, Runnable target
 
         public void start()
         {
+            AtomicReference<RuntimeException> verificationExeption = new AtomicReference<>();
             Thread verificationThread = new Thread(() -> {
                 verificationStarted.countDown();
 
@@ -974,7 +933,8 @@ public void start()
                     }
                     catch (Throwable e)
                     {
-                        throw Throwables.unchecked(e);
+                        verificationExeption.set(Throwables.unchecked(e));
+                        return;
                     }
                 }
             });
@@ -988,6 +948,9 @@ public void start()
                 taskCompleted.countDown();
 
                 verificationThread.join(verificationMaxInMs);
+                RuntimeException rte = verificationExeption.get();
+                if (rte != null)
+                    throw rte;
             }
             catch (InterruptedException e)
             {
diff --git a/test/unit/org/apache/cassandra/index/sai/SAIUtil.java b/test/unit/org/apache/cassandra/index/sai/SAIUtil.java
new file mode 100644
index 000000000000..c6ff0b91625f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/SAIUtil.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.lang.reflect.Field;
+
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.utils.ReflectionUtils;
+
+public class SAIUtil
+{
+    public static void setLatestVersion(Version version)
+    {
+        Field latest = null;
+        try
+        {
+            latest = Version.class.getDeclaredField("LATEST");
+            latest.setAccessible(true);
+            Field modifiersField = ReflectionUtils.getModifiersField();
+            modifiersField.setAccessible(true);
+            latest.set(null, version);
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/StorageAttachedIndexTest.java b/test/unit/org/apache/cassandra/index/sai/StorageAttachedIndexTest.java
new file mode 100644
index 000000000000..75ece22dadbb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/StorageAttachedIndexTest.java
@@ -0,0 +1,159 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Lists;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction;
+import org.apache.cassandra.cql3.selection.SortedRowsBuilder;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+import static org.junit.Assert.assertEquals;
+import static org.apache.cassandra.index.sai.SAITester.vector;
+
+
+public class StorageAttachedIndexTest
+{
+    private static final String KEYSPACE = "ks";
+    private static final String TABLE = "tab";
+    private static final int DIMENSION = 2;
+    private static final int numSSTables = 2;
+    private ColumnFamilyStore cfs;
+    private List<ByteBuffer> byteBufferList;
+    private List<CQLTester.Vector<Float>> vectorFloatList;
+    private SingleColumnRestriction.AnnRestriction testRestriction;
+    private StorageAttachedIndex sai;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        DatabaseDescriptor.daemonInitialization(() -> {
+            Config config = DatabaseDescriptor.loadConfig();
+            config.partitioner = Murmur3Partitioner.class.getName();
+            return config;
+        });
+
+        SchemaLoader.prepareServer();
+        Gossiper.instance.maybeInitializeLocalState(0);
+        SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1));
+        QueryProcessor.executeInternal(String.format("CREATE TABLE %s.%s (key int primary key, value vector<float, %s>)", KEYSPACE, TABLE, DIMENSION));
+        QueryProcessor.executeInternal(String.format("CREATE CUSTOM INDEX ON %s.%s(value) USING 'StorageAttachedIndex' WITH OPTIONS = { 'similarity_function': 'dot_product'}", KEYSPACE, TABLE));
+
+        vectorFloatList = new ArrayList<>();
+        CQLTester.Vector<Float> vector1 = vector(1f, 2f);
+        CQLTester.Vector<Float> vector2 = vector(3f, 4f);
+        CQLTester.Vector<Float> vector3 = vector(5f, 6f);
+        vectorFloatList.add(vector1);
+        vectorFloatList.add(vector2);
+        vectorFloatList.add(vector3);
+
+        cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+
+        for (int i = 0; i < numSSTables ; i++)
+        {
+            for (CQLTester.Vector<Float> vectorFloat : vectorFloatList)
+            {
+                QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, value) VALUES (?, ?)", KEYSPACE, TABLE), i, vectorFloat);
+            }
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        }
+
+        TableMetadata tableMetadata = cfs.metadata();
+        String columnName = "value";
+        ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnName, true);
+        ColumnMetadata columnDef = tableMetadata.getExistingColumn(columnIdentifier);
+        if (!(columnDef.type instanceof VectorType))
+            throw invalidRequest("ANN is only supported against DENSE FLOAT32 columns");
+
+        // Convert List<CQLTester.Vector<Float>> to List<ByteBuffer>
+        byteBufferList = new ArrayList<>();
+        for (CQLTester.Vector<Float> vector : vectorFloatList) {
+            ByteBuffer buffer = floatVectorToByteBuffer(vector);
+            byteBufferList.add(buffer);
+        }
+
+        Term terms = new Lists.Value(byteBufferList);
+
+        testRestriction = new SingleColumnRestriction.AnnRestriction(columnDef, terms);
+
+        sai = (StorageAttachedIndex) cfs.getIndexManager().getIndexByName(String.format("%s_value_idx", TABLE));
+    }
+
+    private static ByteBuffer floatVectorToByteBuffer(CQLTester.Vector<Float> vector) {
+        ByteBuffer buffer = ByteBuffer.allocate(vector.size() * 4);  // 4 bytes per float
+        for (Float value : vector) {
+            buffer.putFloat(value);
+        }
+        buffer.flip();
+        return buffer;
+    }
+
+    @Test
+    public void testOrderResults()
+    {
+        List<List<ByteBuffer>> rows = new ArrayList<>();
+        rows.add(byteBufferList);
+
+        SelectStatement selectStatementInstance = (SelectStatement) QueryProcessor.prepareInternal("SELECT key, value FROM " + KEYSPACE + '.' + TABLE).statement;
+
+        SortedRowsBuilder builder = selectStatementInstance.sortedRowsBuilder(Integer.MAX_VALUE, 0);
+        rows.forEach(builder::add);
+        List<List<ByteBuffer>> sortedRows = builder.build();
+
+        Comparator<List<ByteBuffer>> descendingComparator = (o1, o2) -> {
+            ByteBuffer value1 = o1.get(0);
+            ByteBuffer value2 = o2.get(0);
+            return value2.compareTo(value1);
+        };
+
+        rows.sort(descendingComparator);
+
+        for (int i = 0; i < sortedRows.size(); i++)
+        {
+            List<ByteBuffer> expectedRow = rows.get(i);
+            List<ByteBuffer> actualRow = sortedRows.get(i);
+            assertEquals(expectedRow, actualRow);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/analyzer/AnalyzerEqOperatorSupportTest.java b/test/unit/org/apache/cassandra/index/sai/analyzer/AnalyzerEqOperatorSupportTest.java
new file mode 100644
index 000000000000..3373ef7db30b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/analyzer/AnalyzerEqOperatorSupportTest.java
@@ -0,0 +1,430 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.conditions.ColumnCondition;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.service.ClientWarn;
+import org.assertj.core.api.Assertions;
+import org.assertj.core.api.ListAssert;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.index.sai.analyzer.AnalyzerEqOperatorSupport.EQ_RESTRICTION_ON_ANALYZED_WARNING;
+import static org.apache.cassandra.index.sai.analyzer.AnalyzerEqOperatorSupport.LWT_CONDITION_ON_ANALYZED_WARNING;
+
+/**
+ * Tests for {@link AnalyzerEqOperatorSupport}.
+ */
+public class AnalyzerEqOperatorSupportTest extends SAITester
+{
+    @Before
+    public void createTable()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text, f text)");
+    }
+
+    private void populateTable()
+    {
+        execute("INSERT INTO %s (k, v) VALUES (1, 'Quick fox')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'Lazy fox')");
+    }
+
+    @Test
+    public void testWithoutAnyIndex()
+    {
+        populateTable();
+
+        // equals (=)
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'Quick fox' ALLOW FILTERING", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'quick fox' ALLOW FILTERING");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'fox' ALLOW FILTERING");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'Quick fox' OR v = 'Lazy fox' ALLOW FILTERING", row(1), row(2));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'quick fox' OR v = 'lazy fox' ALLOW FILTERING");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'Quick' OR v = 'Lazy' ALLOW FILTERING");
+        assertInvalidMessage("v cannot be restricted by more than one relation if it includes an Equal",
+                             "SELECT k FROM %s WHERE v = 'Quick' AND v = 'fox' ALLOW FILTERING");
+
+        // matches (:)
+        assertInvalidMessage(": restriction is only supported on properly indexed columns. v : 'Quick fox' is not valid.",
+                             "SELECT k FROM %s WHERE v : 'Quick fox' ALLOW FILTERING");
+
+        // LWT
+        assertRowsWithoutWarning("UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'Quick fox'", row(true));
+        assertRowsWithoutWarning("UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'fox'", row(false, "Quick fox"));
+        assertInvalidMessage(ColumnCondition.ANALYZER_MATCHES_ERROR, "UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v : 'Quick fox'");
+    }
+
+    @Test
+    public void testWithLegacyIndex()
+    {
+        populateTable();
+
+        createIndex("CREATE INDEX ON %s(v)");
+
+        // equals (=)
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'Quick fox'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'quick fox'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'fox'");
+        assertInvalidMessage("v cannot be restricted by more than one relation if it includes an Equal",
+                             "SELECT k FROM %s WHERE v = 'Quick' AND v = 'fox'");
+        assertInvalidMessage(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_DISJUNCTION,
+                             "SELECT k FROM %s WHERE v = 'Quick fox' OR v = 'Lazy fox'");
+
+        // matches (:)
+        assertInvalidMessage(format(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_ANALYZER_MATCHES_MESSAGE, 'v'),
+                             "SELECT k FROM %s WHERE v : 'Quick fox'");
+
+        // LWT
+        assertRowsWithoutWarning("UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'Quick fox'", row(true));
+        assertRowsWithoutWarning("UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'fox'", row(false, "Quick fox"));
+        assertInvalidMessage(ColumnCondition.ANALYZER_MATCHES_ERROR, "UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v : 'Quick fox'");
+    }
+
+    @Test
+    public void testNonAnalyzedIndexWithDefaults()
+    {
+        assertIndexQueries("{}", () -> {
+
+            // equals (=)
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'Quick fox'", row(1));
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'quick fox'");
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'fox'");
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'Quick fox' OR v = 'Lazy fox'", row(1), row(2));
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'quick fox' OR v = 'lazy fox'");
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'Quick' OR v = 'Lazy'");
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'quick' OR v = 'lazy'");
+            assertInvalidMessage("v cannot be restricted by more than one relation if it includes an Equal",
+                                 "SELECT k FROM %s WHERE v = 'Quick' AND v = 'fox'");
+
+            // matches (:)
+            assertIndexDoesNotSupportMatches();
+
+            // LWT
+            assertRowsWithoutWarning("UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'Quick fox'", row(true));
+            assertRowsWithoutWarning("UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'fox'", row(false, "Quick fox"));
+            assertInvalidMessage(ColumnCondition.ANALYZER_MATCHES_ERROR, "UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v : 'Quick fox'");
+        });
+    }
+
+    @Test
+    public void testNonAnalyzedIndexWithMatch()
+    {
+        assertIndexThrowsNotAnalyzedError("{'equals_behaviour_when_analyzed': 'MATCH'}");
+    }
+
+    @Test
+    public void testNonAnalyzedIndexWithUnsupported()
+    {
+        assertIndexThrowsNotAnalyzedError("{'equals_behaviour_when_analyzed': 'UNSUPPORTED'}");
+    }
+
+    @Test
+    public void testNonAnalyzedIndexWithWrongValue()
+    {
+        assertIndexThrowsNotAnalyzedError("{'equals_behaviour_when_analyzed': 'WRONG'}");
+    }
+
+    @Test
+    public void testNonTokenizedIndexWithDefaults()
+    {
+        assertIndexQueries("{'case_sensitive': 'false'}", () -> {
+            assertNonTokenizedIndexSupportsEquality();
+            assertNonTokenizedIndexSupportsMatches();
+            assertNonTokenizedIndexSupportsMixedEqualityAndMatches();
+        });
+    }
+
+    @Test
+    public void testNonTokenizedIndexWithMatch()
+    {
+        assertIndexQueries("{'case_sensitive': 'false', 'equals_behaviour_when_analyzed': 'MATCH'}", () -> {
+            assertNonTokenizedIndexSupportsEquality();
+            assertNonTokenizedIndexSupportsMatches();
+            assertNonTokenizedIndexSupportsMixedEqualityAndMatches();
+        });
+    }
+
+    @Test
+    public void testNonTokenizedIndexWithUnsupported()
+    {
+        assertIndexQueries("{'case_sensitive': 'false', 'equals_behaviour_when_analyzed': 'UNSUPPORTED'}", () -> {
+            assertIndexDoesNotSupportEquals();
+            assertNonTokenizedIndexSupportsMatches();
+        });
+    }
+
+    @Test
+    public void testNonTokenizedIndexWithWrongValue()
+    {
+        assertIndexThrowsUnrecognizedOptionError("{'case_sensitive': 'false', 'equals_behaviour_when_analyzed': 'WRONG'}");
+    }
+
+    private void assertNonTokenizedIndexSupportsEquality()
+    {
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'fox'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick fox' OR v = 'Lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick fox' OR v = 'lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' OR v = 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' OR v = 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' AND v = 'fox'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' AND v = 'fox'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' AND v = 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' AND v = 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v = 'quick' AND v = 'fox') OR v = 'dog'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v = 'quick' AND v = 'fox') OR v = 'Lazy'");
+
+        // LWT
+        assertRowsWithLWTWarning("UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'Quick fox'", row(true));
+        assertRowsWithLWTWarning("UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'fox'", row(false, "Quick fox"));
+    }
+
+    private void assertNonTokenizedIndexSupportsMatches()
+    {
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick fox'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick fox'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'fox'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick fox' OR v : 'Lazy fox'", row(1), row(2));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick fox' OR v : 'lazy fox'", row(1), row(2));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick' OR v : 'Lazy'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick' OR v : 'lazy'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick' AND v : 'fox'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick' AND v : 'fox'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick' AND v : 'Lazy'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick' AND v : 'lazy'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE (v : 'quick' AND v : 'fox') OR v : 'dog'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE (v : 'quick' AND v : 'fox') OR v : 'Lazy'");
+
+        // LWT
+        assertInvalidMessage(ColumnCondition.ANALYZER_MATCHES_ERROR, "UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v : 'Quick fox'");
+    }
+
+    private void assertNonTokenizedIndexSupportsMixedEqualityAndMatches()
+    {
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick fox' OR v : 'Lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick fox' OR v : 'lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' OR v : 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' OR v : 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' AND v : 'fox'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' AND v : 'fox'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' AND v : 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' AND v : 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v = 'quick' AND v : 'fox') OR v = 'dog'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v = 'quick' AND v : 'fox') OR v = 'Lazy'");
+
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'Quick fox' OR v = 'Lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'quick fox' OR v = 'lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'Quick' OR v = 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'quick' OR v = 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'Quick' AND v = 'fox'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'quick' AND v = 'fox'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'Quick' AND v = 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'quick' AND v = 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v : 'quick' AND v = 'fox') OR v : 'dog'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v : 'quick' AND v = 'fox') OR v : 'Lazy'");
+
+        // LWT
+        assertInvalidMessage(ColumnCondition.ANALYZER_MATCHES_ERROR,
+                             "UPDATE %s SET v = 'Quick fox' WHERE k = 1 IF v = 'Quick fox' AND v : 'Quick fox'");
+    }
+
+    @Test
+    public void testTokenizedIndexWithDefaults()
+    {
+        assertIndexQueries("{'index_analyzer': 'standard'}", () -> {
+            assertTokenizedIndexSupportsEquality();
+            assertTokenizedIndexSupportsMatches();
+            assertTokenizedIndexSupportsMixedEqualityAndMatches();
+        });
+    }
+
+    @Test
+    public void testTokenizedIndexWithMatch()
+    {
+        assertIndexQueries("{'index_analyzer': 'standard', 'equals_behaviour_when_analyzed': 'MATCH'}", () -> {
+            assertTokenizedIndexSupportsEquality();
+            assertTokenizedIndexSupportsMatches();
+            assertTokenizedIndexSupportsMixedEqualityAndMatches();
+        });
+    }
+
+    @Test
+    public void testTokenizedIndexWithUnsupported()
+    {
+        assertIndexQueries("{'index_analyzer': 'standard', 'equals_behaviour_when_analyzed': 'UNSUPPORTED'}", () -> {
+            assertIndexDoesNotSupportEquals();
+            assertTokenizedIndexSupportsMatches();
+
+            // test mixed with another non-indexed column
+            assertInvalidMessage(": restriction is only supported on properly indexed columns",
+                                 "SELECT k FROM %s WHERE v : 'Quick' OR f : 'Lazy' ALLOW FILTERING");
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick' OR f = 'Lazy' ALLOW FILTERING", row(1));
+            assertInvalidMessage(": restriction is only supported on properly indexed columns",
+                                 "SELECT k FROM %s WHERE v = 'Quick' OR f : 'Lazy' ALLOW FILTERING");
+            assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'Quick' OR f = 'Lazy' ALLOW FILTERING");
+        });
+    }
+
+    @Test
+    public void testTokenizedIndexWithWrongValue()
+    {
+        assertIndexThrowsUnrecognizedOptionError("{'index_analyzer': 'standard', 'equals_behaviour_when_analyzed': 'WRONG'}");
+    }
+
+    private void assertTokenizedIndexSupportsEquality()
+    {
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick fox' OR v = 'Lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick fox' OR v = 'lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' OR v = 'Lazy'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' OR v = 'lazy'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' AND v = 'fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' AND v = 'fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' AND v = 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' AND v = 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v = 'quick' AND v = 'fox') OR v = 'dog'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v = 'quick' AND v = 'fox') OR v = 'Lazy'", row(1), row(2));
+    }
+
+    private void assertTokenizedIndexSupportsMatches()
+    {
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick fox'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick fox'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'fox'", row(1), row(2));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick fox' OR v : 'Lazy fox'", row(1), row(2));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick fox' OR v : 'lazy fox'", row(1), row(2));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick' OR v : 'Lazy'", row(1), row(2));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick' OR v : 'lazy'", row(1), row(2));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick' AND v : 'fox'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick' AND v : 'fox'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick' AND v : 'Lazy'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'quick' AND v : 'lazy'");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE (v : 'quick' AND v : 'fox') OR v : 'dog'", row(1));
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE (v : 'quick' AND v : 'fox') OR v : 'Lazy'", row(1), row(2));
+    }
+
+    private void assertTokenizedIndexSupportsMixedEqualityAndMatches()
+    {
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick fox' OR v : 'Lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick fox' OR v : 'lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' OR v : 'Lazy'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' OR v : 'lazy'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' AND v : 'fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' AND v : 'fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' AND v : 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'quick' AND v : 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v = 'quick' AND v : 'fox') OR v = 'dog'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v = 'quick' AND v : 'fox') OR v = 'Lazy'", row(1), row(2));
+
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'Quick fox' OR v = 'Lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'quick fox' OR v = 'lazy fox'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'Quick' OR v = 'Lazy'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'quick' OR v = 'lazy'", row(1), row(2));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'Quick' AND v = 'fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'quick' AND v = 'fox'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'Quick' AND v = 'Lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v : 'quick' AND v = 'lazy'");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v : 'quick' AND v = 'fox') OR v : 'dog'", row(1));
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE (v : 'quick' AND v = 'fox') OR v : 'Lazy'", row(1), row(2));
+
+        // test mixed with another non-indexed column
+        String errorMsg = ": restriction is only supported on properly indexed columns";
+        assertInvalidMessage(errorMsg, "SELECT k FROM %s WHERE v : 'Quick' OR f : 'Lazy' ALLOW FILTERING");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v : 'Quick' OR f = 'Lazy' ALLOW FILTERING", row(1));
+        assertInvalidMessage(errorMsg, "SELECT k FROM %s WHERE v = 'Quick' OR f : 'Lazy' ALLOW FILTERING");
+        assertRowsWithSelectWarning("SELECT k FROM %s WHERE v = 'Quick' OR f = 'Lazy' ALLOW FILTERING", row(1));
+    }
+
+    private void assertIndexDoesNotSupportEquals()
+    {
+        // the EQ query should not be supported by the index
+        String query = "SELECT k FROM %s WHERE v = 'Quick fox'";
+        assertInvalidMessage("Column 'v' has an index but does not support the operators specified in the query", query);
+
+        // the EQ query should stil be supported with filtering without index intervention
+        assertRowsWithoutWarning(query + " ALLOW FILTERING", row(1));
+
+        // the EQ query should not use any kind of transformation when filtered without index intervention
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'quick fox' ALLOW FILTERING");
+        assertRowsWithoutWarning("SELECT k FROM %s WHERE v = 'fox' ALLOW FILTERING");
+    }
+
+    private void assertIndexDoesNotSupportMatches()
+    {
+        String query = "SELECT k FROM %s WHERE v : 'Quick fox'";
+        String errorMessage = "Index on column v does not support ':' restrictions.";
+        assertInvalidMessage(errorMessage, query);
+        assertInvalidMessage(errorMessage, query + " ALLOW FILTERING");
+    }
+
+    private void assertIndexThrowsNotAnalyzedError(String indexOptions)
+    {
+        assertInvalidMessage(AnalyzerEqOperatorSupport.NOT_ANALYZED_ERROR,
+                             "CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS =" + indexOptions);
+    }
+
+    private void assertIndexThrowsUnrecognizedOptionError(String indexOptions)
+    {
+        assertInvalidMessage(AnalyzerEqOperatorSupport.WRONG_OPTION_ERROR,
+                             "CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS =" + indexOptions);
+    }
+
+    private void assertIndexQueries(String indexOptions, Runnable queries)
+    {
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = " + indexOptions);
+        populateTable();
+
+        queries.run();
+        flush();
+        queries.run();
+    }
+
+    private void assertRowsWithoutWarning(String query, Object[]... rows)
+    {
+        assertRows(query, rows).isNullOrEmpty();
+    }
+
+    private void assertRowsWithSelectWarning(String query, Object[]... rows)
+    {
+        assertRows(query, rows).hasSize(1).contains(format(EQ_RESTRICTION_ON_ANALYZED_WARNING, 'v', currentIndex()));
+    }
+
+    private void assertRowsWithLWTWarning(String query, Object[]... rows)
+    {
+        assertRows(query, rows).hasSize(1).contains(format(LWT_CONDITION_ON_ANALYZED_WARNING, 'v'));
+    }
+
+    private ListAssert<String> assertRows(String query, Object[]... rows)
+    {
+        ClientWarn.instance.captureWarnings();
+        CQLTester.disablePreparedReuseForTest();
+        assertRows(execute(query), rows);
+        ListAssert<String> assertion = Assertions.assertThat(ClientWarn.instance.getWarnings());
+        ClientWarn.instance.resetWarnings();
+        return assertion;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/analyzer/LuceneAnalyzerTest.java b/test/unit/org/apache/cassandra/index/sai/analyzer/LuceneAnalyzerTest.java
new file mode 100644
index 000000000000..f4fce2c8d6c5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/analyzer/LuceneAnalyzerTest.java
@@ -0,0 +1,247 @@
+
+/*
+ * All changes to the original code are Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Charsets;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class LuceneAnalyzerTest
+{
+    @Test
+    public void testCzechStem() throws Exception
+    {
+        String json = "{\n" +
+                      "\"tokenizer\":{\"name\":\"standard\"},\n" +
+                      "\"filters\":[{\"name\":\"czechstem\"}]\n" +
+                      "}\n";
+        String testString = "pánové";
+        String[] expected = new String[]{ "pán" };
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(expected, list.toArray(new String[0]));
+    }
+
+    @Test
+    public void testPattern() throws Exception
+    {
+        String json = "{\n" +
+                      "\"tokenizer\":{\"name\":\"simplepattern\", \"args\":{\"pattern\":\"[0123456789]{3}\"}}}";
+        String testString = "fd-786-335-514-x";
+        String[] expected = new String[]{ "786", "335", "514"  };
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(expected, list.toArray(new String[0]));
+    }
+
+    @Test
+    public void testNgram() throws Exception
+    {
+        String json = "{\n" +
+                      "\"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}},\n" +
+                      "\"filters\":[{\"name\":\"lowercase\"}]\n" +
+                      "}";
+        String testString = "DoG";
+        String[] expected = new String[]{ "do", "dog", "og" };
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(expected, list.toArray(new String[0]));
+    }
+
+    // We override the defaults for ngram. This test ensures that our defaults are used and not lucene's
+    @Test
+    public void testNgramTokenizerDefaults() throws Exception
+    {
+        String json = "{\n" +
+                      "\"tokenizer\":{\"name\":\"ngram\"},\n" +
+                      "\"filters\":[{\"name\":\"lowercase\"}]\n" +
+                      "}";
+        String testString = "DoGs";
+        // Default minGramSize is 3
+        String[] expected = new String[]{ "dog", "dogs", "ogs" };
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(expected, list.toArray(new String[0]));
+    }
+
+    @Test
+    public void testPorterStem1() throws Exception
+    {
+        String json = "{\n" +
+                      "\"tokenizer\":{\"name\":\"whitespace\"},\n" +
+                      "\"filters\":[{\"name\":\"porterstem\"}]\n" +
+                      "}";
+        String testString = "dogs withering in the windy";
+        String[] expected = new String[]{ "dog", "wither", "in", "the", "windi" };
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(expected, list.toArray(new String[0]));
+    }
+
+    @Test
+    public void testPorterStem2() throws Exception
+    {
+        String json = "{\n" +
+                      "\"tokenizer\":{\"name\":\"whitespace\"},\n" +
+                      "\"filters\":[{\"name\":\"porterstem\"}]\n" +
+                      "}";
+        String testString = "apples orcharding";
+        String[] expected = new String[]{ "appl", "orchard"};
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(expected, list.toArray(new String[0]));
+    }
+
+    @Test
+    public void testStopwordWithEscapedComma() throws Exception
+    {
+        // Need 4 backslashes to get through all of the parsing... this is an unlikely scenario, so it seems
+        // acceptable. Note that in CQL, it'll just need 2 backslashes.
+        String json = "{\"tokenizer\":{\"name\" : \"whitespace\"}," +
+                      "\"filters\":[{\"name\":\"stop\", \"args\": " +
+                      "{\"words\": \"one\\\\,stopword, test\"}}]}";
+        // Put result in the middle to make sure that one,stopword and test are broken up and applied individually
+        String testString = "one,stopword result test";
+        String[] expected = new String[]{ "result" };
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(expected, list.toArray(new String[0]));
+    }
+
+    @Test
+    public void testStopwordWithSpace() throws Exception
+    {
+        // Need 4 backslashes to get through all of the parsing... this is an unlikely scenario, so it seems
+        // acceptable. Note that in CQL, it'll just need 2 backslashes.
+        String json = "{\"tokenizer\":{\"name\" : \"keyword\"}," +
+                      "\"filters\":[{\"name\":\"stop\", \"args\": " +
+                      "{\"words\": \"one stopword, test\"}}]}";
+        // 'one stopword' is a single stopword, so it gets filtered out (note that the tokenizer is keyword, so
+        // it doesn't get broken up into multiple tokens)
+        String testString = "one stopword";
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(new String[]{}, list.toArray(new String[0]));
+    }
+
+    @Test
+    public void testStopwordWithoutArgsDefaultsToEnglish() throws Exception
+    {
+        String json = "{\"tokenizer\":{\"name\" : \"whitespace\"}," +
+                      "\"filters\":[{\"name\":\"stop\"}]}";
+        // Assert that when we do not pass any arguments to the stop filter, it defaults to english, and
+        // show this by joining the stop words together and asserting it produces no tokens
+        String testString = EnglishAnalyzer.ENGLISH_STOP_WORDS_SET.stream()
+                                                                  .map(s -> ((char[]) s))
+                                                                  .map(String::new)
+                                                                  .reduce((a, b) -> a + " " + b)
+                                                                  .get();
+        assertFalse(testString.isEmpty());
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(new String[]{}, list.toArray(new String[0]));
+
+        // Let's also confirm the stop words are the expected ones. (We rely on this set for some indexes, so if
+        // it were to change (it shouldn't), that would create inconsistencies in existing indexes which means
+        // we want to know before any changes get released.)
+        var expectedStopWords = Arrays.asList("a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in",
+                                              "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", "the",
+                                              "their", "then", "there", "these", "they", "this", "to", "was", "will",
+                                              "with");
+        var actualStopWords = EnglishAnalyzer.ENGLISH_STOP_WORDS_SET.stream()
+                                                                    .map(s -> ((char[]) s))
+                                                                    .map(String::new)
+                                                                    .sorted()
+                                                                    .collect(Collectors.toList());
+        assertEquals(expectedStopWords, actualStopWords);
+    }
+
+    @Test(expected = InvalidRequestException.class)
+    public void testMissingSynonymArg() throws Exception
+    {
+        // The synonym filter takes a 'synonyms' argument, not a 'words' argument
+        String json = "{\"tokenizer\":{\"name\" : \"keyword\"}," +
+                      "\"filters\":[{\"name\":\"synonym\", \"args\": " +
+                      "{\"words\": \"as => like\"}}]}";
+        tokenize("irrelevant test string", json);
+    }
+
+    @Test
+    public void testSynonmyMapping() throws Exception
+    {
+        // Need 4 backslashes to get through all of the parsing... this is an unlikely scenario, so it seems
+        // acceptable. Note that in CQL, it'll just need 2 backslashes.
+        String json = "{\"tokenizer\":{\"name\" : \"keyword\"}," +
+                      "\"filters\":[{\"name\":\"synonym\", \"args\": " +
+                      "{\"synonyms\": \"as => like\", \"analyzer\":\"" + WhitespaceAnalyzer.class.getName() + "\"}}]}";
+        String testString = "as";
+        String[] expected = new String[]{ "like" };
+        List<String> list = tokenize(testString, json);
+        assertArrayEquals(expected, list.toArray(new String[0]));
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testMissingClassException() throws Exception
+    {
+        // Need 4 backslashes to get through all of the parsing... this is an unlikely scenario, so it seems
+        // acceptable. Note that in CQL, it'll just need 2 backslashes.
+        String json = "{\"tokenizer\":{\"name\" : \"keyword\"}," +
+                      "\"filters\":[{\"name\":\"synonym\", \"args\": " +
+                      "{\"synonyms\": \"as => like\", \"analyzer\":\"not-a-class\"}}]}";
+        tokenize("irrelevant text", json);
+    }
+
+    public static List<String> tokenize(String testString, String json) throws Exception
+    {
+        Analyzer luceneAnalyzer = JSONAnalyzerParser.parse(json).left;
+        LuceneAnalyzer analyzer = new LuceneAnalyzer(UTF8Type.instance, luceneAnalyzer, new HashMap<>());
+
+        ByteBuffer toAnalyze = ByteBuffer.wrap(testString.getBytes(Charsets.UTF_8));
+        analyzer.reset(toAnalyze);
+        ByteBuffer analyzed = null;
+
+        List<String> list = new ArrayList<>();
+
+        while (analyzer.hasNext())
+        {
+            analyzed = analyzer.next();
+            list.add(ByteBufferUtil.string(analyzed, Charsets.UTF_8));
+        }
+
+        analyzer.end();
+
+        return list;
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java b/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java
index 9a7f8f9e42a5..6d108e320f3d 100644
--- a/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java
@@ -23,11 +23,14 @@
 import org.junit.Test;
 
 import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 
+/**
+ * Tests for the non-tokenizing analyzer
+ */
 public class NonTokenizingAnalyzerTest
 {
     @Test
@@ -36,8 +39,23 @@ public void asciiAnalyzer() throws Exception
         NonTokenizingOptions options = NonTokenizingOptions.getDefaultOptions();
         options.setCaseSensitive(false);
         options.setAscii(true);
+        NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(UTF8Type.instance, options);
+
+        String testString = "Éppinger";
+        ByteBuffer toAnalyze = ByteBuffer.wrap(testString.getBytes());
+        analyzer.reset(toAnalyze);
+        ByteBuffer analyzed = null;
+
+        while (analyzer.hasNext())
+        {
+            analyzed = analyzer.next();
+        }
 
-        assertEquals("eppinger", getAnalyzedString("Éppinger", options));
+        String good = "eppinger";
+
+        String result = ByteBufferUtil.string(analyzed);
+
+        assertEquals(good, result);
     }
 
     @Test
@@ -46,8 +64,23 @@ public void asciiAnalyzerFalse() throws Exception
         NonTokenizingOptions options = NonTokenizingOptions.getDefaultOptions();
         options.setCaseSensitive(true);
         options.setAscii(false);
+        NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(UTF8Type.instance, options);
+
+        String testString = "Éppinger";
+        ByteBuffer toAnalyze = ByteBuffer.wrap(testString.getBytes());
+        analyzer.reset(toAnalyze);
+        ByteBuffer analyzed = null;
 
-        assertEquals("Éppinger", getAnalyzedString("Éppinger", options));
+        while (analyzer.hasNext())
+        {
+            analyzed = analyzer.next();
+        }
+
+        String good = "Éppinger";
+
+        String result = ByteBufferUtil.string(analyzed);
+
+        assertEquals(good, result);
     }
 
     @Test
@@ -55,22 +88,37 @@ public void caseInsensitiveAnalyzer() throws Exception
     {
         NonTokenizingOptions options = NonTokenizingOptions.getDefaultOptions();
         options.setCaseSensitive(false);
+        NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(UTF8Type.instance, options);
 
-        assertEquals("nip it in the bud", getAnalyzedString("Nip it in the bud", options));
+        String testString = "Nip it in the bud";
+        ByteBuffer toAnalyze = ByteBuffer.wrap(testString.getBytes());
+        analyzer.reset(toAnalyze);
+        ByteBuffer analyzed = null;
+        
+        while (analyzer.hasNext())
+        {
+            analyzed = analyzer.next();
+        }
+        
+        assertEquals(testString.toLowerCase(), ByteBufferUtil.string(analyzed));
     }
 
     @Test
     public void caseSensitiveAnalyzer() throws Exception
     {
         NonTokenizingOptions options = NonTokenizingOptions.getDefaultOptions();
+        NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(UTF8Type.instance, options);
 
-        assertEquals("Nip it in the bud", getAnalyzedString("Nip it in the bud", options));
-    }
-
-    private String getAnalyzedString(String input, NonTokenizingOptions options) throws Exception
-    {
-        NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(SAITester.createIndexTermType(UTF8Type.instance), options);
-        analyzer.reset(ByteBuffer.wrap(input.getBytes()));
-        return analyzer.hasNext() ? ByteBufferUtil.string(analyzer.next) : null;
+        String testString = "Nip it in the bud";
+        ByteBuffer toAnalyze = ByteBuffer.wrap(testString.getBytes());
+        analyzer.reset(toAnalyze);
+        ByteBuffer analyzed = null;
+        
+        while (analyzer.hasNext())
+        {
+            analyzed = analyzer.next();
+        }
+        
+        assertNotEquals(testString.toLowerCase(), ByteBufferUtil.string(analyzed));
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/analyzer/filter/BasicResultFiltersTest.java b/test/unit/org/apache/cassandra/index/sai/analyzer/filter/BasicResultFiltersTest.java
new file mode 100644
index 000000000000..1340f93c9c15
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/analyzer/filter/BasicResultFiltersTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+import java.text.Normalizer;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+
+import static org.junit.Assert.assertEquals;
+
+public class BasicResultFiltersTest
+{
+    @Test
+    public void testLowerCase()
+    {
+        BasicResultFilters.LowerCase lowerCase = new BasicResultFilters.LowerCase();
+        
+        for (int count = 0; count < CQLTester.getRandom().nextIntBetween(100, 1000); count++)
+        {
+            String actual = CQLTester.getRandom().nextTextString(10, 50);
+            assertEquals(actual.toLowerCase(), lowerCase.process(actual));
+        }
+    }
+    
+    @Test
+    public void testNormalize()
+    {
+        BasicResultFilters.Normalize normalize = new BasicResultFilters.Normalize();
+
+        for (int count = 0; count < CQLTester.getRandom().nextIntBetween(100, 1000); count++)
+        {
+            String actual = CQLTester.getRandom().nextTextString(10, 50);
+            assertEquals(Normalizer.normalize(actual, Normalizer.Form.NFC), normalize.process(actual));
+        }
+    }
+    
+    @Test
+    public void testAscii()
+    {
+        BasicResultFilters.Ascii ascii = new BasicResultFilters.Ascii();
+
+        for (int count = 0; count < CQLTester.getRandom().nextIntBetween(100, 1000); count++)
+        {
+            String actual = CQLTester.getRandom().nextTextString(100, 5000);
+
+            char[] actualChars = actual.toCharArray();
+            char[] expectedChars = new char[actualChars.length * 4];
+            int expectedSize = BasicResultFilters.foldToASCII(actualChars, 0, expectedChars, 0, actualChars.length);
+            String expected = new String(expectedChars, 0, expectedSize);
+
+            assertEquals(expected, ascii.process(actual));
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/AbstractQueryTester.java b/test/unit/org/apache/cassandra/index/sai/cql/AbstractQueryTester.java
deleted file mode 100644
index 78f52c303984..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/AbstractQueryTester.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import com.google.common.collect.ImmutableList;
-import org.junit.Before;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
-import org.apache.cassandra.inject.Injections;
-
-import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
-
-@RunWith(Parameterized.class)
-public class AbstractQueryTester extends SAITester
-{
-    public static final Injections.Counter INDEX_QUERY_COUNTER = Injections.newCounter("IndexQueryCounter")
-                                                                              .add(newInvokePoint().onClass(StorageAttachedIndexSearcher.class).onMethod("search"))
-                                                                              .build();
-
-    @Parameterized.Parameter
-    public BaseDataModel dataModel;
-    @Parameterized.Parameter(1)
-    public List<IndexQuerySupport.BaseQuerySet> sets;
-
-    protected BaseDataModel.Executor executor;
-
-    @Before
-    public void setup() throws Throwable
-    {
-        requireNetwork();
-
-        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", BaseDataModel.KEYSPACE));
-
-        Injections.inject(INDEX_QUERY_COUNTER);
-
-        executor = new SingleNodeExecutor(this, INDEX_QUERY_COUNTER);
-    }
-
-    @SuppressWarnings("unused")
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> params()
-    {
-        List<Object[]> scenarios = new LinkedList<>();
-
-        scenarios.add(new Object[]{ new BaseDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA), IndexQuerySupport.BASE_QUERY_SETS });
-
-        scenarios.add(new Object[]{ new BaseDataModel.CompoundKeyDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA), IndexQuerySupport.BASE_QUERY_SETS });
-
-        scenarios.add(new Object[]{ new BaseDataModel.CompoundKeyWithStaticsDataModel(BaseDataModel.STATIC_COLUMNS, BaseDataModel.STATIC_COLUMN_DATA), IndexQuerySupport.STATIC_QUERY_SETS });
-
-        scenarios.add(new Object[]{ new BaseDataModel.CompositePartitionKeyDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA),
-                                    ImmutableList.builder().addAll(IndexQuerySupport.BASE_QUERY_SETS).addAll(IndexQuerySupport.COMPOSITE_PARTITION_QUERY_SETS).build()});
-
-        return scenarios;
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/AbstractRebuildAndImmutableComponentsTester.java b/test/unit/org/apache/cassandra/index/sai/cql/AbstractRebuildAndImmutableComponentsTester.java
new file mode 100644
index 000000000000..f4c5b0afd6b3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/AbstractRebuildAndImmutableComponentsTester.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.IMMUTABLE_SAI_COMPONENTS;
+import static org.junit.Assert.assertEquals;
+
+@Ignore
+public abstract class AbstractRebuildAndImmutableComponentsTester extends SAITester
+{
+     private Boolean defaultImmutableSetting;
+
+    protected abstract boolean useImmutableComponents();
+
+    @Before
+    public void setup() throws Throwable
+    {
+        defaultImmutableSetting = IMMUTABLE_SAI_COMPONENTS.getBoolean();
+        IMMUTABLE_SAI_COMPONENTS.setBoolean(useImmutableComponents());
+        requireNetwork();
+    }
+
+    @After
+    public void tearDown()
+    {
+        if (defaultImmutableSetting != null)
+            IMMUTABLE_SAI_COMPONENTS.setBoolean(defaultImmutableSetting);
+    }
+
+    @Test
+    public void rebuildCreateNewGenerationFiles() throws Throwable
+    {
+        // Setup: create index, insert data, flush, and make sure everything is correct.
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+        String name = createIndex("CREATE CUSTOM INDEX test_index ON %s(val) USING 'StorageAttachedIndex'");
+
+        IndexContext context = createIndexContext(name, UTF8Type.instance);
+
+        execute("INSERT INTO %s (id, val) VALUES ('0', 'testValue')");
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'otherValue')");
+        execute("INSERT INTO %s (id, val) VALUES ('2', 'testValue')");
+        execute("INSERT INTO %s (id, val) VALUES ('3', 'otherValue')");
+
+        flush();
+
+        assertEquals(2, execute("SELECT id FROM %s WHERE val = 'testValue'").size());
+
+        // Rebuild the index
+        rebuildIndexes(name);
+
+        assertEquals(2, execute("SELECT id FROM %s WHERE val = 'testValue'").size());
+
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        validateSSTables(cfs, context);
+    }
+
+    protected abstract void validateSSTables(ColumnFamilyStore cfs, IndexContext context) throws Exception;
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java b/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java
index f80d07eeb774..ee8afd65235f 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java
@@ -15,16 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.cassandra.index.sai.cql;
 
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.index.IndexBuildInProgressException;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.inject.InvokePointBuilder;
 
-import static java.lang.String.format;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertNotNull;
 
 /**
@@ -33,11 +35,11 @@
 public class AllowFilteringTest extends SAITester
 {
     @Test
-    public void testAllowFilteringOnFirstClusteringKeyColumn() throws Throwable
+    public void testAllowFilteringOnFirstClusteringKeyColumn()
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, v1 int, " +
                     "PRIMARY KEY ((k1, k2), c1, c2, c3))");
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(c1) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c1) USING '%s'", StorageAttachedIndex.class.getName()));
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE c1=0", false);
@@ -77,16 +79,17 @@ public void testAllowFilteringOnFirstClusteringKeyColumn() throws Throwable
     }
 
     @Test
-    public void testAllowFilteringOnNotFirstClusteringKeyColumn() throws Throwable
+    public void testAllowFilteringOnNotFirstClusteringKeyColumn()
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, c4 int, v1 int, " +
                     "PRIMARY KEY ((k1, k2), c1, c2, c3, c4))");
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(c3) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c3) USING '%s'", StorageAttachedIndex.class.getName()));
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE c3=0", false);
         test("SELECT * FROM %s WHERE c3>0", false);
         test("SELECT * FROM %s WHERE c3>0 AND c3<1", false);
+        test("SELECT * FROM %s WHERE c3!=0", false);
 
         // with additional simple filtering restrictions
         test("SELECT * FROM %s WHERE c3=0 AND k1=0", true);
@@ -121,12 +124,12 @@ public void testAllowFilteringOnNotFirstClusteringKeyColumn() throws Throwable
     }
 
     @Test
-    public void testAllowFilteringOnMultipleClusteringKeyColumns() throws Throwable
+    public void testAllowFilteringOnMultipleClusteringKeyColumns()
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, c4 int, v1 int, " +
                     "PRIMARY KEY ((k1, k2), c1, c2, c3, c4))");
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(c2) USING '%s'", StorageAttachedIndex.class.getName()));
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(c4) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c2) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c4) USING '%s'", StorageAttachedIndex.class.getName()));
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE c2=0 AND c4=0", false);
@@ -136,6 +139,7 @@ public void testAllowFilteringOnMultipleClusteringKeyColumns() throws Throwable
         test("SELECT * FROM %s WHERE c2>0 AND c2<1 AND c4=0", false);
         test("SELECT * FROM %s WHERE c2>0 AND c4>0", false);
         test("SELECT * FROM %s WHERE c2>0 AND c2<1 AND c4>0 AND c4<1", false);
+        test("SELECT * FROM %s WHERE c2!=0 AND c4!=1", false);
 
         // with additional simple filtering restrictions
         test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND k1=0", true);
@@ -168,15 +172,16 @@ public void testAllowFilteringOnMultipleClusteringKeyColumns() throws Throwable
     }
 
     @Test
-    public void testAllowFilteringOnSingleRegularColumn() throws Throwable
+    public void testAllowFilteringOnSingleRegularColumn()
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, v1 int, v2 int, PRIMARY KEY ((k1, k2), c1, c2))");
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE v1=0", false);
         test("SELECT * FROM %s WHERE v1>0", false);
         test("SELECT * FROM %s WHERE v1>0 AND v1<1", false);
+        test("SELECT * FROM %s WHERE v1!=0", false);
 
         // with additional simple filtering restrictions
         test("SELECT * FROM %s WHERE v1=0 AND k1=0", true);
@@ -209,12 +214,12 @@ public void testAllowFilteringOnSingleRegularColumn() throws Throwable
     }
 
     @Test
-    public void testAllowFilteringOnMultipleRegularColumns() throws Throwable
+    public void testAllowFilteringOnMultipleRegularColumns()
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, v1 int, v2 int, v3 int, " +
                     "PRIMARY KEY ((k1, k2), c1, c2))");
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE v1=0 AND v2=0", false);
@@ -223,6 +228,7 @@ public void testAllowFilteringOnMultipleRegularColumns() throws Throwable
         test("SELECT * FROM %s WHERE v1=0 AND v2>0", false);
         test("SELECT * FROM %s WHERE v1=0 AND v2>0 AND v2<1", false);
         test("SELECT * FROM %s WHERE v1>0 AND v1<1 AND v2>0 AND v2<1", false);
+        test("SELECT * FROM %s WHERE v1!=0 AND v2!=0", false);
 
         // with additional simple filtering restrictions
         test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0", true);
@@ -255,25 +261,27 @@ public void testAllowFilteringOnMultipleRegularColumns() throws Throwable
     }
 
     @Test
-    public void testAllowFilteringOnClusteringAndRegularColumns() throws Throwable
+    public void testAllowFilteringOnClusteringAndRegularColumns()
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, c4 int, v1 int, v2 int, v3 int, " +
                     "PRIMARY KEY ((k1, k2), c1, c2, c3, c4))");
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(c2) USING '%s'", StorageAttachedIndex.class.getName()));
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(c4) USING '%s'", StorageAttachedIndex.class.getName()));
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
-        createIndex(format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c2) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c4) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0", false);
         test("SELECT * FROM %s WHERE c2>0 AND c4>0 AND v1>0 AND v2>0", false);
         test("SELECT * FROM %s WHERE c2>0 AND c2<1 AND c4>0 AND c4<1 AND v1>0 AND v1<0 AND v2>0 AND v2<1", false);
+        test("SELECT * FROM %s WHERE c2!=0 AND c4!=1 AND v1!=0 AND v2!=0", false);
 
         // with additional simple filtering restrictions
         test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k1=0", true);
         test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k2=0", true);
         test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND c3=0", true);
         test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND v3=0", true);
+        test("SELECT * FROM %s WHERE c2!=0 AND c4!=0 AND v1!=0 AND v2!=0 AND v3=0", true);
 
         // with token restrictions
         test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND token(k1, k2) = token(0, 0)", false);
@@ -296,7 +304,50 @@ public void testAllowFilteringOnClusteringAndRegularColumns() throws Throwable
         test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND (c1, c2, c3, c4) = (0, 0, 0, 0) AND v3=0", true);
     }
 
-    private void test(String query, boolean requiresAllowFiltering) throws Throwable
+    @Test
+    public void testAllowFilteringOnCollectionColumn()
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, l list<int>, s set<int>, m_k map<int,int>,"
+                    + " m_v map<int,int>, m_en map<int, int>, not_indexed list<int>, PRIMARY KEY ((k1, k2), c1, c2))");
+        createIndex("CREATE CUSTOM INDEX ON %s(l) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(s) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(keys(m_k)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(values(m_v)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(m_en)) USING 'StorageAttachedIndex'");
+
+        // single contains
+        test("SELECT * FROM %s WHERE l contains 1", false);
+        test("SELECT * FROM %s WHERE s contains 1", false);
+        test("SELECT * FROM %s WHERE m_k contains key 1", false);
+        test("SELECT * FROM %s WHERE m_v contains 1", false);
+        test("SELECT * FROM %s WHERE m_en[1] = 1", false);
+
+        // multiple contains on different indexed columns
+        test("SELECT * FROM %s WHERE l contains 1 and s contains 2", false);
+        test("SELECT * FROM %s WHERE l contains 1 and m_k contains key 2", false);
+        test("SELECT * FROM %s WHERE l contains 1 and m_v contains 2", false);
+        test("SELECT * FROM %s WHERE l contains 1 and m_en[2] = 2", false);
+        test("SELECT * FROM %s WHERE s contains 1 and s contains 2", false);
+        test("SELECT * FROM %s WHERE s contains 1 and m_k contains key 2", false);
+        test("SELECT * FROM %s WHERE s contains 1 and m_v contains 2", false);
+        test("SELECT * FROM %s WHERE s contains 1 and m_en[2] = 2", false);
+
+        // multiple contains on the same column
+        test("SELECT * FROM %s WHERE l contains 1 and l contains 2", false);
+        test("SELECT * FROM %s WHERE s contains 1 and s contains 2", false);
+        test("SELECT * FROM %s WHERE m_k contains key 1 and m_k contains key 2", false);
+        test("SELECT * FROM %s WHERE m_v contains 1 and m_v contains 2", false);
+        test("SELECT * FROM %s WHERE m_en[1] = 1 and m_en[2] = 2", false);
+
+        // multiple contains on different columns with not indexed column
+        test("SELECT * FROM %s WHERE l contains 1 and not_indexed contains 2", true);
+        test("SELECT * FROM %s WHERE s contains 1 and not_indexed contains 2", true);
+        test("SELECT * FROM %s WHERE m_k contains key 1 and not_indexed contains 2", true);
+        test("SELECT * FROM %s WHERE m_v contains 1 and not_indexed contains 2", true);
+        test("SELECT * FROM %s WHERE m_en[1] = 1 and not_indexed contains 2", true);
+    }
+
+    private void test(String query, boolean requiresAllowFiltering)
     {
         if (requiresAllowFiltering)
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, query);
@@ -305,4 +356,193 @@ private void test(String query, boolean requiresAllowFiltering) throws Throwable
 
         assertNotNull(execute(query + " ALLOW FILTERING"));
     }
+
+    @Test
+    public void testUnsupportedIndexRestrictions()
+    {
+        createTable("CREATE TABLE %s (a text, b text, c text, d text, PRIMARY KEY (a, b))");
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(d) USING '%s'", StorageAttachedIndex.class.getName()));
+
+        execute("INSERT INTO %s (a, b, c, d) VALUES ('Test1', 'Test1', 'Test1', 'Test1')");
+        execute("INSERT INTO %s (a, b, c, d) VALUES ('Test2', 'Test2', 'Test2', 'Test2')");
+        execute("INSERT INTO %s (a, b, c, d) VALUES ('Test3', 'Test3', 'Test3', 'Test3')");
+
+        // Single restriction
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'b'), "SELECT * FROM %s WHERE b > 'Test'");
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'c'), "SELECT * FROM %s WHERE c > 'Test'");
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'd'), "SELECT * FROM %s WHERE d > 'Test'");
+
+        // Supported and unsupported restriction
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'b'), "SELECT * FROM %s WHERE b > 'Test' AND c = 'Test1'");
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'c'), "SELECT * FROM %s WHERE c > 'Test' AND d = 'Test1'");
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'd'), "SELECT * FROM %s WHERE d > 'Test' AND b = 'Test1'");
+
+        // Two unsupported restrictions
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'b'), "SELECT * FROM %s WHERE b > 'Test' AND b < 'Test3'");
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_MULTI, "[b, c]"), "SELECT * FROM %s WHERE c > 'Test' AND b < 'Test3'");
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_MULTI, "[b, d]"), "SELECT * FROM %s WHERE d > 'Test' AND b < 'Test3'");
+
+        // The same queries with ALLOW FILTERING should work
+
+        // Single restriction
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE b > 'Test' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"),
+                                                                                                    row("Test2", "Test2", "Test2", "Test2"),
+                                                                                                    row("Test3", "Test3", "Test3", "Test3"));
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE c > 'Test' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"),
+                                                                                                    row("Test2", "Test2", "Test2", "Test2"),
+                                                                                                    row("Test3", "Test3", "Test3", "Test3"));
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE d > 'Test' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"),
+                                                                                                    row("Test2", "Test2", "Test2", "Test2"),
+                                                                                                    row("Test3", "Test3", "Test3", "Test3"));
+
+        // Supported and unsupported restriction
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE b > 'Test' AND c = 'Test1' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"));
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE c > 'Test' AND d = 'Test1' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"));
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE d > 'Test' AND b = 'Test1' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"));
+
+        // Two unsupported restrictions
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE b > 'Test' AND b < 'Test3' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"),
+                                                                                                                    row("Test2", "Test2", "Test2", "Test2"));
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE c > 'Test' AND b < 'Test3' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"),
+                                                                                                                    row("Test2", "Test2", "Test2", "Test2"));
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE d > 'Test' AND b < 'Test3' ALLOW FILTERING"), row("Test1", "Test1", "Test1", "Test1"),
+                                                                                                                    row("Test2", "Test2", "Test2", "Test2"));
+    }
+
+    @Test
+    public void testIndexedColumnDoesNotSupportLikeRestriction()
+    {
+        createTable("CREATE TABLE %s (a text, b text, c text, d text, PRIMARY KEY (a, b))");
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(d) USING '%s'", StorageAttachedIndex.class.getName()));
+
+        // LIKE restriction
+        assertInvalidMessage(String.format(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_LIKE_MESSAGE, 'b'), "SELECT * FROM %s WHERE b LIKE 'Test'");
+        assertInvalidMessage(String.format(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_LIKE_MESSAGE, 'c'), "SELECT * FROM %s WHERE c LIKE 'Test'");
+        assertInvalidMessage(String.format(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_LIKE_MESSAGE, 'd'), "SELECT * FROM %s WHERE d LIKE 'Test'");
+    }
+
+    @Test
+    public void testIndexedColumnDoesNotSupportAnalyzerRestriction()
+    {
+        createTable("CREATE TABLE %s (a text, b text, c text, d text, PRIMARY KEY (a, b))");
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(d) USING '%s'", StorageAttachedIndex.class.getName()));
+
+        // Analyzer restriction
+        assertInvalidMessage(": restriction is only supported on properly indexed columns. a : 'Test' is not valid.", "SELECT * FROM %s WHERE a : 'Test'");
+        assertInvalidMessage(String.format(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_ANALYZER_MATCHES_MESSAGE, 'b'), "SELECT * FROM %s WHERE b : 'Test'");
+        assertInvalidMessage(String.format(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_ANALYZER_MATCHES_MESSAGE, 'c'), "SELECT * FROM %s WHERE c : 'Test'");
+        assertInvalidMessage(String.format(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_ANALYZER_MATCHES_MESSAGE, 'd'), "SELECT * FROM %s WHERE d : 'Test'");
+    }
+
+    @Test
+    public void testQueryRequiresFilteringButHasANNRestriction()
+    {
+        createTable("CREATE TABLE %s (pk text, i int, j int, k int, vec vector<float, 3>, PRIMARY KEY((pk, i), j))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+
+        // Should not fail because allow filtering is set but not required
+        assertRows(execute("SELECT * FROM %s ORDER BY vec ANN OF [1,1,1] LIMIT 10 ALLOW FILTERING;"));
+
+        // Do not recommend ALLOW FILTERING for non-primary key, non clustering column restrictions
+        assertInvalidMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE,
+                             "SELECT * FROM %s WHERE k > 0 ORDER BY vec ANN OF [2.5, 3.5, 4.5] LIMIT 10;");
+
+        // Do not let ALLOW FILTERING to lead to query execution for non-primary key, non clustering column restrictions
+        assertInvalidMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE,
+                             "SELECT * FROM %s WHERE k > 0 ORDER BY vec ANN OF [2.5, 3.5, 4.5] LIMIT 10 ALLOW FILTERING;");
+
+        // Do not recommend ALLOW FILTERING for clustering column restrictions
+        assertInvalidMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE,
+                             "SELECT * FROM %s WHERE j > 0 ORDER BY vec ANN OF [2.5, 3.5, 4.5] LIMIT 10;");
+
+        // Do not let ALLOW FILTERING lead to query execution for clustering column restrictions
+        assertInvalidMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE,
+                             "SELECT * FROM %s WHERE j > 0 ORDER BY vec ANN OF [2.5, 3.5, 4.5] LIMIT 10 ALLOW FILTERING;");
+
+        // Do not recommend ALLOW FILTERING for partial partition key restrictions
+        assertInvalidMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE,
+                             "SELECT * FROM %s WHERE pk > 'A' AND pk < 'C' ORDER BY vec ANN OF [2.5, 3.5, 4.5] LIMIT 10;");
+
+        // Do not let ALLOW FILTERING lead to query execution for partial partition key restrictions
+        assertInvalidMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE,
+                             "SELECT * FROM %s WHERE pk > 'A' AND pk < 'C' ORDER BY vec ANN OF [2.5, 3.5, 4.5] LIMIT 10 ALLOW FILTERING;");
+
+        // Do not recommend ALLOW FILTERING for complete partition key restrictions
+        assertInvalidMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE,
+                             "SELECT * FROM %s WHERE pk > 'A' AND pk < 'C' AND i > 0 ORDER BY vec ANN OF [2.5, 3.5, 4.5] LIMIT 10;");
+
+        // Do not let ALLOW FILTERING lead to query execution for complete partition key restrictions
+        assertInvalidMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE,
+                             "SELECT * FROM %s WHERE pk > 'A' AND pk < 'C' AND i > 0 ORDER BY vec ANN OF [2.5, 3.5, 4.5] LIMIT 10 ALLOW FILTERING;");
+    }
+
+    @Test
+    public void testMapRangeQueries()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(keys(item_cost)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(values(item_cost)) USING 'StorageAttachedIndex'");
+
+        // Insert data for later
+        execute("INSERT INTO %s (partition, item_cost) VALUES (0, {'apple': 2, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 1, 'orange': 3})");
+        flush();
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 4, 'orange': 2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 3, 'orange': 1})");
+
+        // Gen an ALLOW FILTERING recommendation.
+        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "item_cost"),
+                             "SELECT partition FROM %s WHERE item_cost['apple'] < 6");
+
+        // Show that filtering works correctly
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 1 ALLOW FILTERING"),
+                   row(0), row(2), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] >= 1 ALLOW FILTERING"),
+                   row(1), row(0), row(2), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 3 ALLOW FILTERING"),
+                   row(1), row(0));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] <= 3 ALLOW FILTERING"),
+                   row(1), row(0), row(3));
+
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 3 AND item_cost['apple'] > 1 ALLOW FILTERING"), row(0));
+
+
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        // Show that we're now able to execute the query.
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 3 AND item_cost['apple'] > 1"), row(0));
+    }
+
+    private final Injections.Barrier blockIndexBuild = Injections.newBarrier("block_index_build", 2, false)
+                                                                 .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndex.class)
+                                                                 .onMethod("startInitialBuild"))
+                                                                 .build();
+
+    @Test
+    public void testAllowFilteringDuringIndexBuild() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)");
+        Injections.inject(blockIndexBuild);
+        String idx = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(v) USING '%s'", StorageAttachedIndex.class.getName()));
+
+        assertThatThrownBy(() -> executeInternal("SELECT * FROM %s WHERE v=0"))
+                .hasMessage("The secondary index '" + idx + "' is not yet available as it is building")
+                .isInstanceOf(IndexBuildInProgressException.class);
+
+        assertThatThrownBy(() -> executeInternal("SELECT * FROM %s WHERE v=0 ALLOW FILTERING"))
+                .hasMessage("The secondary index '" + idx + "' is not yet available as it is building")
+                .isInstanceOf(IndexBuildInProgressException.class);
+
+        blockIndexBuild.countDown();
+        blockIndexBuild.disable();
+        waitForIndexQueryable(idx);
+        execute("SELECT * FROM %s WHERE v=0");
+        execute("SELECT * FROM %s WHERE v=0 ALLOW FILTERING");
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/AnalyzerTest.java b/test/unit/org/apache/cassandra/index/sai/cql/AnalyzerTest.java
new file mode 100644
index 000000000000..78a5a2a249e9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/AnalyzerTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.util.Arrays;
+
+public class AnalyzerTest extends SAITester
+{
+    @Test
+    public void createAnalyzerWrongTypeTest()
+    {
+        createTable("CREATE TABLE %s (pk1 int, pk2 text, val int, val2 int, PRIMARY KEY((pk1, pk2)))");
+        createIndex("CREATE CUSTOM INDEX ON %s(pk1) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(pk2) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (-1, 'b', 1)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (0, 'b', 2)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, 'b', 3)");
+
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (-1, 'a', -1)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (0, 'a', -2)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, 'a', -3)");
+
+        flush();
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer': 'standard'};");
+
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (-1, 'd', 1)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (0, 'd', 2)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, 'd', 3)");
+
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (-1, 'c', -1)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (0, 'c', -2)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, 'c', -3)");
+    }
+
+    /**
+     * Test that we cannot use an analyzer, tokenizing or not, on a frozen collection.
+     */
+    @Test
+    public void analyzerOnFrozenCollectionTest()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, l frozen<list<text>>, s frozen<set<text>>, m frozen<map<text, text>>)");
+
+        for (String column : Arrays.asList("l", "s", "m"))
+        {
+            assertRejectsNonFullIndexCreationOnFrozenCollection(column);
+            column = String.format("full(%s)", column);
+
+            // non-tokenizing options that produce an analyzer should be rejected
+            assertRejectsAnalyzerOnFrozenCollection(column, String.format("{'%s': %s}", NonTokenizingOptions.CASE_SENSITIVE, false));
+            assertRejectsAnalyzerOnFrozenCollection(column, String.format("{'%s': %s}", NonTokenizingOptions.NORMALIZE, true));
+            assertRejectsAnalyzerOnFrozenCollection(column, String.format("{'%s': %s}", NonTokenizingOptions.ASCII, true));
+
+            // non-tokenizing options that do not produce an analyzer should be accepted
+            assertAcceptsIndexOptions(column, String.format("{'%s': %s}", NonTokenizingOptions.CASE_SENSITIVE, true));
+            assertAcceptsIndexOptions(column, String.format("{'%s': %s}", NonTokenizingOptions.NORMALIZE, false));
+            assertAcceptsIndexOptions(column, String.format("{'%s': %s}", NonTokenizingOptions.ASCII, false));
+
+            // Lucene analyzer should always be rejected
+            assertRejectsAnalyzerOnFrozenCollection(column, "{'index_analyzer': 'standard'}");
+            assertRejectsAnalyzerOnFrozenCollection(column,
+                    "{'index_analyzer': " +
+                    "  '{\"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}," +
+                    "    \"filters\":[{\"name\":\"lowercase\"}]}'}");
+            assertRejectsAnalyzerOnFrozenCollection(column,
+                    "{'index_analyzer':'\n" +
+                            "  {\"tokenizer\":{\"name\" : \"whitespace\"},\n" +
+                            "   \"filters\":[{\"name\":\"stop\", \"args\": {\"words\": \"the, test\", \"format\": \"wordset\"}}]}'}");
+
+            // no options should be accepted
+            assertAcceptsIndexOptions(column, null);
+            assertAcceptsIndexOptions(column, "{}");
+        }
+    }
+
+    private void assertRejectsNonFullIndexCreationOnFrozenCollection(String column)
+    {
+        Assertions.assertThatThrownBy(() -> createSAIIndex(column, null))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining("Cannot create values() index on frozen column " + column);
+
+        Assertions.assertThatThrownBy(() -> createSAIIndex("KEYS(" + column + ')', null))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining("Cannot create keys() index on frozen column " + column);
+
+        Assertions.assertThatThrownBy(() -> createSAIIndex("VALUES(" + column + ')', null))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining("Cannot create values() index on frozen column " + column);
+    }
+
+    private void assertAcceptsIndexOptions(String column, @Nullable String options)
+    {
+        String index = createSAIIndex(column, options);
+        dropIndex("DROP INDEX %s." + index); // clear for further tests
+    }
+
+    private void assertRejectsAnalyzerOnFrozenCollection(String column, String options)
+    {
+        Assertions.assertThatThrownBy(() -> createSAIIndex(column, options))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining("Cannot use an analyzer on " + column + " because it's a frozen collection.");
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/BM25Test.java b/test/unit/org/apache/cassandra/index/sai/cql/BM25Test.java
new file mode 100644
index 000000000000..8f43b0f66ab3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/BM25Test.java
@@ -0,0 +1,931 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+import org.apache.cassandra.index.sai.plan.QueryController;
+
+import static org.apache.cassandra.index.sai.analyzer.AnalyzerEqOperatorSupport.EQ_AMBIGUOUS_ERROR;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
+import static org.junit.Assert.assertEquals;
+
+public class BM25Test extends SAITester
+{
+    @Before
+    public void setup() throws Throwable
+    {
+        SAIUtil.setLatestVersion(Version.EC);
+    }
+
+    @Test
+    public void testTwoIndexes()
+    {
+        // create un-analyzed index
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+
+        // BM25 should fail with only an equality index
+        assertInvalidMessage("BM25 ordering on column v requires an analyzed index",
+                             "SELECT k FROM %s WHERE v : 'apple' ORDER BY v BM25 OF 'apple' LIMIT 3");
+
+        createAnalyzedIndex();
+        // BM25 query should work now
+        var result = execute("SELECT k FROM %s WHERE v : 'apple' ORDER BY v BM25 OF 'apple' LIMIT 3");
+        assertRows(result, row(1));
+    }
+
+    @Test
+    public void testDeletedRow() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        createAnalyzedIndex();
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'apple juice')");
+        var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3");
+        assertThat(result).hasSize(2);
+        execute("DELETE FROM %s WHERE k=2");
+        String select = "SELECT k FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3";
+        beforeAndAfterFlush(() -> assertRows(execute(select), row(1)));
+    }
+
+    @Test
+    public void testDeletedColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        createAnalyzedIndex();
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'apple juice')");
+        String select = "SELECT k FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3";
+        assertRows(execute(select), row(1), row(2));
+        execute("DELETE v FROM %s WHERE k = 2");
+        beforeAndAfterFlush(() -> assertRows(execute(select), row(1)));
+    }
+
+    @Test
+    public void testDeletedRowWithPredicate() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text, n int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+        createAnalyzedIndex();
+        execute("INSERT INTO %s (k, v, n) VALUES (1, 'apple', 0)");
+        execute("INSERT INTO %s (k, v, n) VALUES (2, 'apple juice', 0)");
+        String select = "SELECT k FROM %s WHERE n = 0 ORDER BY v BM25 OF 'apple' LIMIT 3";
+        assertRows(execute(select), row(1), row(2));
+        execute("DELETE FROM %s WHERE k=2");
+        beforeAndAfterFlush(() -> assertRows(execute(select), row(1)));
+    }
+
+    @Test
+    public void testTwoIndexesAmbiguousPredicate() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+
+        createAnalyzedIndex();
+        // Create  un-analyzed indexes
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'apple juice')");
+        execute("INSERT INTO %s (k, v) VALUES (3, 'orange juice')");
+
+        // equality predicate is ambiguous (both analyzed and un-analyzed indexes could support it) so it should
+        // be rejected
+        beforeAndAfterFlush(() -> {
+            // Single predicate
+            assertInvalidMessage(String.format(EQ_AMBIGUOUS_ERROR, 'v', getIndex(0), getIndex(1)),
+                                 "SELECT k FROM %s WHERE v = 'apple'");
+
+            // AND
+            assertInvalidMessage(String.format(EQ_AMBIGUOUS_ERROR, 'v', getIndex(0), getIndex(1)),
+                                 "SELECT k FROM %s WHERE v = 'apple' AND v : 'juice'");
+
+            // OR
+            assertInvalidMessage(String.format(EQ_AMBIGUOUS_ERROR, 'v', getIndex(0), getIndex(1)),
+                                 "SELECT k FROM %s WHERE v = 'apple' OR v : 'juice'");
+        });
+    }
+
+    @Test
+    public void testTwoIndexesWithEqualsUnsupported() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+        // analyzed index with equals_behavior:unsupported option
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'equals_behaviour_when_analyzed': 'unsupported', " +
+                    "'index_analyzer':'{\"tokenizer\":{\"name\":\"standard\"},\"filters\":[{\"name\":\"porterstem\"}]}' }");
+
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'apple juice')");
+
+        beforeAndAfterFlush(() -> {
+            // combining two EQ predicates is not allowed
+            assertInvalid("SELECT k FROM %s WHERE v = 'apple' AND v = 'juice'");
+
+            // combining EQ and MATCH predicates is also not allowed (when we're not converting EQ to MATCH)
+            assertInvalid("SELECT k FROM %s WHERE v = 'apple' AND v : 'apple'");
+
+            // combining two MATCH predicates is fine
+            assertRows(execute("SELECT k FROM %s WHERE v : 'apple' AND v : 'juice'"),
+                       row(2));
+
+            // = operator should use un-analyzed index since equals is unsupported in analyzed index
+            assertRows(execute("SELECT k FROM %s WHERE v = 'apple'"),
+                       row(1));
+
+            // : operator should use analyzed index
+            assertRows(execute("SELECT k FROM %s WHERE v : 'apple'"),
+                       row(1), row(2));
+        });
+    }
+
+    @Test
+    public void testComplexQueriesWithMultipleIndexes() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 text, v2 text, v3 int)");
+
+        // Create mix of analyzed, unanalyzed, and non-text indexes
+        createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+        createAnalyzedIndex("v2");
+        createIndex("CREATE CUSTOM INDEX ON %s(v3) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (k, v1, v2, v3) VALUES (1, 'apple', 'orange juice', 5)");
+        execute("INSERT INTO %s (k, v1, v2, v3) VALUES (2, 'apple juice', 'apple', 10)");
+        execute("INSERT INTO %s (k, v1, v2, v3) VALUES (3, 'banana', 'grape juice', 5)");
+
+        beforeAndAfterFlush(() -> {
+            // Complex query mixing different types of indexes and operators
+            assertRows(execute("SELECT k FROM %s WHERE v1 = 'apple' AND v2 : 'juice' AND v3 = 5"),
+                       row(1));
+
+            // Mix of AND and OR conditions across different index types
+            assertRows(execute("SELECT k FROM %s WHERE v3 = 5 AND (v1 = 'apple' OR v2 : 'apple')"),
+                       row(1));
+
+            // Multi-term analyzed query
+            assertRows(execute("SELECT k FROM %s WHERE v2 : 'orange juice'"),
+                       row(1));
+
+            // Range query with text match
+            assertRows(execute("SELECT k FROM %s WHERE v3 >= 5 AND v2 : 'juice'"),
+                       row(1), row(3));
+        });
+    }
+
+    @Test
+    public void testMatchingAllowed() throws Throwable
+    {
+        // match operator should be allowed with BM25 on the same column
+        // (seems obvious but exercises a corner case in the internal RestrictionSet processing)
+        createSimpleTable();
+
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+
+        beforeAndAfterFlush(() ->
+        {
+            var result = execute("SELECT k FROM %s WHERE v : 'apple' ORDER BY v BM25 OF 'apple' LIMIT 3");
+            assertRows(result, row(1));
+        });
+    }
+
+    @Test
+    public void testUnknownQueryTerm() throws Throwable
+    {
+        createSimpleTable();
+
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+
+        beforeAndAfterFlush(() ->
+                            {
+                                var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'orange' LIMIT 1");
+                                assertEmpty(result);
+                            });
+    }
+
+    @Test
+    public void testDuplicateQueryTerm() throws Throwable
+    {
+        createSimpleTable();
+
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+
+        beforeAndAfterFlush(() ->
+                            {
+                                var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'apple apple' LIMIT 1");
+                                assertRows(result, row(1));
+                            });
+    }
+
+    @Test
+    public void testEmptyQuery() throws Throwable
+    {
+        createSimpleTable();
+
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+
+        beforeAndAfterFlush(() ->
+                            assertInvalidMessage("BM25 query must contain at least one term (perhaps your analyzer is discarding tokens you didn't expect)",
+                                                 "SELECT k FROM %s ORDER BY v BM25 OF '+' LIMIT 1"));
+    }
+
+    @Test
+    public void testTermFrequencyOrdering() throws Throwable
+    {
+        createSimpleTable();
+
+        // Insert documents with varying frequencies of the term "apple"
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'apple apple')");
+        execute("INSERT INTO %s (k, v) VALUES (3, 'apple apple apple')");
+
+        beforeAndAfterFlush(() ->
+        {
+            // Results should be ordered by term frequency (highest to lowest)
+            var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3");
+            assertRows(result,
+                       row(3),  // 3 occurrences
+                       row(2),  // 2 occurrences
+                       row(1)); // 1 occurrence
+        });
+    }
+
+    @Test
+    public void testTermFrequenciesWithOverwrites() throws Throwable
+    {
+        createSimpleTable();
+
+        // Insert documents with varying frequencies of the term "apple", but overwrite the first term
+        // This exercises the code that is supposed to reset frequency counts for overwrites
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'apple apple')");
+        execute("INSERT INTO %s (k, v) VALUES (3, 'apple apple apple')");
+
+        beforeAndAfterFlush(() ->
+                            {
+                                // Results should be ordered by term frequency (highest to lowest)
+                                var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3");
+                                assertRows(result,
+                                           row(3),  // 3 occurrences
+                                           row(2),  // 2 occurrences
+                                           row(1)); // 1 occurrence
+                            });
+    }
+
+    @Test
+    public void testDocumentLength() throws Throwable
+    {
+        createSimpleTable();
+        // Create documents with same term frequency but different lengths
+        execute("INSERT INTO %s (k, v) VALUES (1, 'test test')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'test test other words here to make it longer')");
+        execute("INSERT INTO %s (k, v) VALUES (3, 'test test extremely long document with many additional words to significantly increase the document length while maintaining the same term frequency for our target term')");
+
+        beforeAndAfterFlush(() ->
+        {
+            // Documents with same term frequency should be ordered by length (shorter first)
+            var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'test' LIMIT 3");
+            assertRows(result,
+                       row(1),
+                       row(2),
+                       row(3));
+        });
+    }
+
+    @Test
+    public void testMultiTermQueryScoring() throws Throwable
+    {
+        createSimpleTable();
+        // Two terms, but "apple" appears in fewer documents
+        execute("INSERT INTO %s (k, v) VALUES (1, 'apple banana')");
+        execute("INSERT INTO %s (k, v) VALUES (2, 'apple apple banana')");
+        execute("INSERT INTO %s (k, v) VALUES (3, 'apple banana banana')");
+        execute("INSERT INTO %s (k, v) VALUES (4, 'apple apple banana banana')");
+        execute("INSERT INTO %s (k, v) VALUES (5, 'banana banana')");
+
+        beforeAndAfterFlush(() ->
+        {
+            var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'apple banana' LIMIT 4");
+            assertRows(result,
+                       row(2),  // Highest frequency of most important term
+                       row(4),  // More mentions of both terms
+                       row(1),  // One of each term
+                       row(3)); // Low frequency of most important term
+        });
+    }
+
+    @Test
+    public void testIrrelevantRowsScoring() throws Throwable
+    {
+        createSimpleTable();
+        // Insert pizza reviews with varying relevance to "crispy crust"
+        execute("INSERT INTO %s (k, v) VALUES (1, 'The pizza had a crispy crust and was delicious')"); // Basic mention
+        execute("INSERT INTO %s (k, v) VALUES (2, 'Very crispy crispy crust, perfectly cooked')"); // Emphasized crispy
+        execute("INSERT INTO %s (k, v) VALUES (3, 'The crust crust crust was okay, nothing special')"); // Only crust mentions
+        execute("INSERT INTO %s (k, v) VALUES (4, 'Super crispy crispy crust crust, best pizza ever!')");  // Most mentions of both
+        execute("INSERT INTO %s (k, v) VALUES (5, 'The toppings were good but the pizza was soggy')"); // Irrelevant review
+
+        beforeAndAfterFlush(this::assertIrrelevantRowsCorrect);
+    }
+
+    private void assertIrrelevantRowsCorrect()
+    {
+        var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'crispy crust' LIMIT 5");
+        assertRows(result,
+                   row(4), // Highest frequency of both terms
+                   row(2), // High frequency of 'crispy', one 'crust'
+                   row(1)); // One mention of each term
+        // Rows 4 and 5 do not contain all terms
+    }
+
+    @Test
+    public void testIrrelevantRowsWithCompaction()
+    {
+        // same dataset as testIrrelevantRowsScoring, but split across two sstables
+        createSimpleTable();
+        disableCompaction();
+
+        execute("INSERT INTO %s (k, v) VALUES (1, 'The pizza had a crispy crust and was delicious')"); // Basic mention
+        execute("INSERT INTO %s (k, v) VALUES (2, 'Very crispy crispy crust, perfectly cooked')"); // Emphasized crispy
+        flush();
+
+        execute("INSERT INTO %s (k, v) VALUES (3, 'The crust crust crust was okay, nothing special')"); // Only crust mentions
+        execute("INSERT INTO %s (k, v) VALUES (4, 'Super crispy crispy crust crust, best pizza ever!')");  // Most mentions of both
+        execute("INSERT INTO %s (k, v) VALUES (5, 'The toppings were good but the pizza was soggy')"); // Irrelevant review
+        flush();
+
+        assertIrrelevantRowsCorrect();
+
+        compact();
+        assertIrrelevantRowsCorrect();
+
+        // Force segmentation and requery
+        SegmentBuilder.updateLastValidSegmentRowId(2);
+        compact();
+        assertIrrelevantRowsCorrect();
+    }
+
+    private void createSimpleTable()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        createAnalyzedIndex();
+    }
+
+    private String createAnalyzedIndex()
+    {
+        return createAnalyzedIndex("v");
+    }
+
+    private String createAnalyzedIndex(String column)
+    {
+        return createAnalyzedIndex(column, false);
+    }
+
+    private String createAnalyzedIndex(String column, boolean lowercase)
+    {
+        return createIndex("CREATE CUSTOM INDEX ON %s(" + column + ") " +
+                           "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                           "WITH OPTIONS = {" +
+                           "'index_analyzer': '{" +
+                           "\"tokenizer\" : {\"name\" : \"standard\"}, " +
+                           "\"filters\" : [{\"name\" : \"porterstem\"}" +
+                           (lowercase ? ", {\"name\" : \"lowercase\"}]" : "]")
+                           + "}'}"
+        );
+    }
+
+    @Test
+    public void testWithPredicate() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, p int, v text)");
+        createAnalyzedIndex();
+        execute("CREATE CUSTOM INDEX ON %s(p) USING 'StorageAttachedIndex'");
+
+        // Insert documents with varying frequencies of the term "apple"
+        execute("INSERT INTO %s (k, p, v) VALUES (1, 5, 'apple')");
+        execute("INSERT INTO %s (k, p, v) VALUES (2, 5, 'apple apple')");
+        execute("INSERT INTO %s (k, p, v) VALUES (3, 5, 'apple apple apple')");
+        execute("INSERT INTO %s (k, p, v) VALUES (4, 6, 'apple apple apple')");
+        execute("INSERT INTO %s (k, p, v) VALUES (5, 7, 'apple apple apple')");
+
+        beforeAndAfterFlush(() ->
+        {
+            // Results should be ordered by term frequency (highest to lowest)
+            var result = execute("SELECT k FROM %s WHERE p = 5 ORDER BY v BM25 OF 'apple' LIMIT 3");
+            assertRows(result,
+                       row(3),  // 3 occurrences
+                       row(2),  // 2 occurrences
+                       row(1)); // 1 occurrence
+        });
+    }
+
+    @Test
+    public void testWidePartition() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, v text, PRIMARY KEY (k1, k2))");
+        createAnalyzedIndex();
+
+        // Insert documents with varying frequencies of the term "apple"
+        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 1, 'apple')");
+        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 2, 'apple apple')");
+        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 3, 'apple apple apple')");
+
+        beforeAndAfterFlush(() ->
+        {
+            // Results should be ordered by term frequency (highest to lowest)
+            var result = execute("SELECT k2 FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3");
+            assertRows(result,
+                       row(3),  // 3 occurrences
+                       row(2),  // 2 occurrences
+                       row(1)); // 1 occurrence
+        });
+    }
+
+    @Test
+    public void testWidePartitionWithPkPredicate() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, v text, PRIMARY KEY (k1, k2))");
+        createAnalyzedIndex();
+
+        // Insert documents with varying frequencies of the term "apple"
+        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 1, 'apple')");
+        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 2, 'apple apple')");
+        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 3, 'apple apple apple')");
+        execute("INSERT INTO %s (k1, k2, v) VALUES (1, 3, 'apple apple apple')");
+        execute("INSERT INTO %s (k1, k2, v) VALUES (2, 3, 'apple apple apple')");
+
+        beforeAndAfterFlush(() ->
+        {
+            // Results should be ordered by term frequency (highest to lowest)
+            var result = execute("SELECT k2 FROM %s WHERE k1 = 0 ORDER BY v BM25 OF 'apple' LIMIT 3");
+            assertRows(result,
+                       row(3),  // 3 occurrences
+                       row(2),  // 2 occurrences
+                       row(1)); // 1 occurrence
+        });
+    }
+
+    @Test
+    public void testWidePartitionWithPredicate() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, p int, v text, PRIMARY KEY (k1, k2))");
+        createAnalyzedIndex();
+        execute("CREATE CUSTOM INDEX ON %s(p) USING 'StorageAttachedIndex'");
+
+        // Insert documents with varying frequencies of the term "apple"
+        execute("INSERT INTO %s (k1, k2, p, v) VALUES (0, 1, 5, 'apple')");
+        execute("INSERT INTO %s (k1, k2, p, v) VALUES (0, 2, 5, 'apple apple')");
+        execute("INSERT INTO %s (k1, k2, p, v) VALUES (0, 3, 5, 'apple apple apple')");
+        execute("INSERT INTO %s (k1, k2, p, v) VALUES (0, 4, 6, 'apple apple apple')");
+        execute("INSERT INTO %s (k1, k2, p, v) VALUES (0, 5, 7, 'apple apple apple')");
+
+        beforeAndAfterFlush(() ->
+        {
+            // Results should be ordered by term frequency (highest to lowest)
+            var result = execute("SELECT k2 FROM %s WHERE p = 5 ORDER BY v BM25 OF 'apple' LIMIT 3");
+            assertRows(result,
+                       row(3),  // 3 occurrences
+                       row(2),  // 2 occurrences
+                       row(1)); // 1 occurrence
+        });
+    }
+
+    @Test
+    public void testWithPredicateSearchThenOrder() throws Throwable
+    {
+        QueryController.QUERY_OPT_LEVEL = 0;
+        testWithPredicate();
+    }
+
+    @Test
+    public void testWidePartitionWithPredicateOrderThenSearch() throws Throwable
+    {
+        QueryController.QUERY_OPT_LEVEL = 1;
+        testWidePartitionWithPredicate();
+    }
+
+    @Test
+    public void testQueryWithNulls() throws Throwable
+    {
+        createSimpleTable();
+
+        execute("INSERT INTO %s (k, v) VALUES (0, null)");
+        execute("INSERT INTO %s (k, v) VALUES (1, 'test document')");
+        beforeAndAfterFlush(() ->
+        {
+            var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'test' LIMIT 1");
+            assertRows(result, row(1));
+        });
+    }
+
+    @Test
+    public void testQueryEmptyTable()
+    {
+        createSimpleTable();
+        var result = execute("SELECT k FROM %s ORDER BY v BM25 OF 'test' LIMIT 1");
+        assertThat(result).hasSize(0);
+    }
+
+    @Test
+    public void testBM25RaceConditionConcurrentQueriesInInvertedIndexSearcher() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, v text, PRIMARY KEY (pk))");
+        createAnalyzedIndex();
+
+        // Create 3 docs that have the same BM25 score and will be our top docs
+        execute("INSERT INTO %s (pk, v) VALUES (1, 'apple apple apple')");
+        execute("INSERT INTO %s (pk, v) VALUES (2, 'apple apple apple')");
+        execute("INSERT INTO %s (pk, v) VALUES (3, 'apple apple apple')");
+
+        // Now insert a lot of docs that will hit the query, but will be lower in frequency and therefore in score
+        for (int i = 4; i < 10000; i++)
+            execute("INSERT INTO %s (pk, v) VALUES (?, 'apple apple')", i);
+
+        // Bug only present in sstable
+        flush();
+
+        // Trigger many concurrent queries
+        final ExecutorService executor = Executors.newFixedThreadPool(10);
+        String select = "SELECT pk FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3";
+        var futures = new ArrayList<Future<UntypedResultSet>>();
+        for (int i = 0; i < 1000; i++)
+            futures.add(executor.submit(() -> execute(select)));
+
+        // The top results are always the same rows
+        for (Future<UntypedResultSet> future : futures)
+            assertRowsIgnoringOrder(future.get(), row(1), row(2), row(3));
+
+        // Shutdown executor
+        assertEquals(0, executor.shutdownNow().size());
+    }
+
+    @Test
+    public void testWildcardSelection()
+    {
+        createTable("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY (k, c))");
+        createAnalyzedIndex();
+        execute("INSERT INTO %s (k, c, v) VALUES (1, 1, 'apple')");
+
+        var result = execute("SELECT * FROM %s ORDER BY v BM25 OF 'apple' LIMIT 3");
+        assertThat(result).hasSize(1);
+    }
+
+    @Test
+    public void cannotHaveAggregationOnBM25Query()
+    {
+        createSimpleTable();
+
+        execute("INSERT INTO %s (k, v) VALUES (1, '4')");
+        execute("INSERT INTO %s (k, v) VALUES (2, '3')");
+        execute("INSERT INTO %s (k, v) VALUES (3, '2')");
+        execute("INSERT INTO %s (k, v) VALUES (4, '1')");
+
+        assertThatThrownBy(() -> execute("SELECT max(v) FROM %s ORDER BY v BM25 OF 'apple' LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
+
+        assertThatThrownBy(() -> execute("SELECT max(v) FROM %s WHERE k = 1 ORDER BY v BM25 OF 'apple' LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
+
+        assertThatThrownBy(() -> execute("SELECT * FROM %s GROUP BY k ORDER BY v BM25 OF 'apple' LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
+
+        assertThatThrownBy(() -> execute("SELECT count(*) FROM %s ORDER BY v BM25 OF 'apple' LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
+    }
+
+    @Test
+    public void testBM25andFilterz() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, category text, score int, title text, body text)");
+        createAnalyzedIndex("body");
+        createIndex("CREATE CUSTOM INDEX ON %s (score) USING 'StorageAttachedIndex'");
+        insertPrimitiveData();
+        beforeAndAfterFlush(
+                () -> {
+                    // 10 docs have score 3 and 3 of those have "health"
+                    var result = execute("SELECT * FROM %s WHERE score = 3 ORDER BY body BM25 OF ? LIMIT 10",
+                                         "health");
+                    assertThat(result).hasSize(3);
+
+                    // 4 docs have score 2 and one of those has "discussed"
+                    result = execute("SELECT * FROM %s WHERE score = 2 ORDER BY body BM25 OF ? LIMIT 10",
+                                         "discussed");
+                    assertThat(result).hasSize(1);
+                });
+    }
+
+    @Test
+    public void testErrorMessages()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, category text, score int, " +
+                    "title text, body text, bodyset set<text>, " +
+                    "map_category map<int, text>, map_body map<text, text>)");
+        createAnalyzedIndex("body", true);
+        createAnalyzedIndex("bodyset", true);
+        createAnalyzedIndex("map_body", true);
+
+        // Improve message issue CNDB-13514
+        assertInvalidMessage("BM25 ordering on column bodyset requires an analyzed index",
+                             "SELECT * FROM %s ORDER BY bodyset BM25 OF ? LIMIT 10");
+
+        // Discussion of message incosistency CNDB-13526
+        assertInvalidMessage("Ordering on non-clustering column requires each restricted column to be indexed except for fully-specified partition keys",
+                             "SELECT * FROM %s WHERE map_body CONTAINS KEY 'Climate' ORDER BY body BM25 OF ? LIMIT 10");
+    }
+
+    @Test
+    public void testWithLowercase() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, body text)");
+        createAnalyzedIndex("body", true);
+        execute("INSERT INTO %s (id, body) VALUES (?, ?)", 1, "Hi hi");
+        execute("INSERT INTO %s (id, body) VALUES (?, ?)", 2, "hi hi longer");
+        executeQuery(Arrays.asList(1, 2), "SELECT * FROM %s ORDER BY body BM25 OF 'hi' LIMIT 4");
+    }
+
+    @Test
+    public void testCollections() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, category text, score int, " +
+                    "title text, body text, bodyset set<text>, " +
+                    "map_category map<int, text>, map_body map<text, text>)");
+        createAnalyzedIndex("body", true);
+        createAnalyzedIndex("bodyset", true);
+        createAnalyzedIndex("map_body", true);
+        createIndex("CREATE CUSTOM INDEX ON %s (score) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s (category) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s (map_category) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s (KEYS(map_body)) USING 'StorageAttachedIndex'");
+        insertCollectionData();
+        analyzeDataset("climate");
+        analyzeDataset("health");
+
+        beforeAndAfterFlush(
+        () -> {
+            // ID 11: total words = 12, climate occurrences = 4
+            // ID 19: total words = 13, climate occurrences = 4
+            // ID 1: total words = 16, climate occurrences = 3
+            // ID 16: total words = 11, climate occurrences = 2
+            // ID 6: total words = 13, climate occurrences = 2
+            // ID 12: total words = 12, climate occurrences = 1
+            // ID 18: total words = 14, climate occurrences = 1
+            executeQuery(Arrays.asList(11, 19, 1, 16, 6, 12, 18), "SELECT * FROM %s  ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+            executeQuery(Arrays.asList(11, 19, 1), "SELECT * FROM %s WHERE score = 5 ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+            executeQuery(Arrays.asList(11, 19, 1, 16, 6, 12, 18), "SELECT * FROM %s WHERE bodyset CONTAINS 'climate' ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+            executeQuery(Arrays.asList(16, 6, 12, 18), "SELECT * FROM %s WHERE bodyset CONTAINS 'health' ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+            executeQuery(Arrays.asList(11, 19, 1, 16, 6, 12, 18), "SELECT * FROM %s WHERE map_category CONTAINS 'Climate' ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+            executeQuery(Arrays.asList(19, 16, 6, 12, 18), "SELECT * FROM %s WHERE map_category CONTAINS 'Health' ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+            executeQuery(Arrays.asList(11, 19, 1, 16, 6, 12, 18), "SELECT * FROM %s WHERE map_body CONTAINS 'Climate' ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+            executeQuery(Arrays.asList(11, 19, 16, 6, 12, 18), "SELECT * FROM %s WHERE map_body CONTAINS 'health' ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+            executeQuery(Arrays.asList(11, 19, 16, 6, 12, 18), "SELECT * FROM %s WHERE map_body CONTAINS KEY 'Health' ORDER BY body BM25 OF ? LIMIT 10",
+                         "climate");
+
+            // ID 4: total words = 15, health occurrences = 3
+            // ID 12: total words = 12, health occurrences = 2
+            // ID 6: total words = 13, health occurrences = 2
+            // ID 9: total words = 13, health occurrences = 2
+            // ID 18: total words = 14, health occurrences = 2
+            // ID 14: total words = 11, health occurrences = 1
+            // ID 16: total words = 11, health occurrences = 1
+            executeQuery(Arrays.asList(6, 16), "SELECT * FROM %s WHERE score > 3 ORDER BY body BM25 OF ? LIMIT 10",
+                         "health");
+            executeQuery(Arrays.asList(4, 12, 9, 18, 14), "SELECT * FROM %s WHERE category = 'Health' " +
+                                                          "ORDER BY body BM25 OF ? LIMIT 10",
+                         "Health");
+            executeQuery(Arrays.asList(4, 12, 9, 18, 14), "SELECT * FROM %s WHERE score <= 3 AND category = 'Health' " +
+                                                          "ORDER BY body BM25 OF ? LIMIT 10",
+                         "health");
+        });
+    }
+
+    @Test
+    public void testOrderingSeveralSSTablesWithMapPredicate() throws Throwable
+    {
+        // Force search-then-sort
+        QueryController.QUERY_OPT_LEVEL = 0;
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, category text, map_category map<int, int>)");
+        createAnalyzedIndex("category", true);
+        createIndex("CREATE CUSTOM INDEX ON %s (entries(map_category)) USING 'StorageAttachedIndex'");
+        // We don't want compaction to merge the two sstables since they are key to testing this code path.
+        disableCompaction();
+
+        // Insert documents so that they all have the same bm25 score and are easy to query across sstables
+        for (int i = 0; i < 10; i++)
+        {
+            execute("INSERT INTO %s (id, category, map_category) VALUES (?, ?, ?)",
+                    i, "Health", map(0, i));
+            if (i == 4)
+                flush();
+        }
+
+        // Confirm that the memtable/sstable and sstable/sstable pairings work as expected.
+        beforeAndAfterFlush(() -> {
+            // Submit a query that will fetch keys from 2 overlapping sstables. The key is that they are overlapping
+            // because we have optimizations that will skip keys that are out of the sstable's range. In this case,
+            // the actual bm25 data doesn't matter because we are covering the edge case of mapping PrK back to
+            // its value here.
+            assertRowsIgnoringOrder(execute("SELECT id FROM %s WHERE map_category[0] >= 4 AND map_category[0] <= 6 ORDER BY category BM25 OF 'health' LIMIT 10"),
+                                   row(4), row(5), row(6));
+        });
+    }
+
+    @Test
+    public void testOrderingSeveralSegments() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, category text, score int," +
+                    "title text, body text)");
+        createAnalyzedIndex("body", true);
+        createIndex("CREATE CUSTOM INDEX ON %s (score) USING 'StorageAttachedIndex'");
+        insertPrimitiveData(0, 10);
+        flush();
+        insertPrimitiveData(10, 20);
+
+        // One memtable, one sstable - different result from the reference in testCollections
+        // ID 1 and 6 contain 3 and 2 climate occurrences correspondingly,
+        // while ID 11 and 19 - 4 climate occurrences. However,
+        // since the segment with 0-9 IDs have only 2 rows with climate and 10-19 - 5,
+        // 1 and 6 win over 11 and 19.
+        executeQuery(Arrays.asList(1, 6, 11, 19, 16, 12, 18), "SELECT * FROM %s  ORDER BY body BM25 OF ? LIMIT 10",
+                "climate");
+        executeQuery(Arrays.asList(1, 11, 19), "SELECT * FROM %s WHERE score = 5 ORDER BY body BM25 OF ? LIMIT 10",
+                "climate");
+
+        // Flush into Two sstables - same result as the different above
+        flush();
+        executeQuery(Arrays.asList(1, 6, 11, 19, 16, 12, 18), "SELECT * FROM %s  ORDER BY body BM25 OF ? LIMIT 10",
+                "climate");
+        executeQuery(Arrays.asList(1, 11, 19), "SELECT * FROM %s WHERE score = 5 ORDER BY body BM25 OF ? LIMIT 10",
+                "climate");
+
+        // Compact into one sstable - same as reference from testCollections
+        compact();
+        executeQuery(Arrays.asList(11, 19, 1, 16, 6, 12, 18), "SELECT * FROM %s  ORDER BY body BM25 OF ? LIMIT 10",
+                "climate");
+        executeQuery(Arrays.asList(11, 19, 1), "SELECT * FROM %s WHERE score = 5 ORDER BY body BM25 OF ? LIMIT 10",
+                "climate");
+    }
+
+    private final static Object[][] DATASET =
+    {
+    { 1, "Climate", 5, "Climate change is a pressing issue. Climate patterns are shifting globally. Scientists study climate data daily.", 1 },
+    { 2, "Technology", 3, "Technology is advancing. New technology in AI and robotics is groundbreaking.", 1 },
+    { 3, "Economy", 4, "The economy is recovering. Economy experts are optimistic. However, the global economy still faces risks.", 1 },
+    { 4, "Health", 3, "Health is wealth. Health policies need to be improved to ensure better public health outcomes.", 1 },
+    { 5, "Education", 2, "Education is the foundation of success. Online education is booming.", 4 },
+    { 6, "Climate", 4, "Climate and health are closely linked. Climate affects air quality and health outcomes.", 2 },
+    { 7, "Education", 3, "Technology and education go hand in hand. EdTech is revolutionizing education through technology.", 3 },
+    { 8, "Economy", 3, "The global economy is influenced by technology. Fintech is a key part of the economy today.", 2 },
+    { 9, "Health", 3, "Education and health programs must be prioritized. Health education is vital in schools.", 2 },
+    { 10, "Mixed", 3, "Technology, economy, and education are pillars of development.", 2 },
+    { 11, "Climate", 5, "Climate climate climate. It's everywhere. Climate drives political and economic decisions.", 1 },
+    { 12, "Health", 2, "Health concerns rise with climate issues. Health organizations are sounding the alarm.", 2 },
+    { 13, "Economy", 3, "The economy is fluctuating. Uncertainty looms over the economy.", 1 },
+    { 14, "Health", 3, "Cutting-edge technology is transforming healthcare. Healthtech merges health and technology.", 1 },
+    { 15, "Education", 2, "Education reforms are underway. Education experts suggest holistic changes.", 1 },
+    { 16, "Climate", 4, "Climate affects the economy and health. Climate events cost billions annually.", 1 },
+    { 17, "Technology", 3, "Technology is the backbone of the modern economy. Without technology, economic growth stagnates.", 2 },
+    { 18, "Health", 2, "Health is discussed less than economy or climate or technology, but health matters deeply.", 1 },
+    { 19, "Climate", 5, "Climate change, climate policies, climate research—climate is the buzzword of our time.", 2 },
+    { 20, "Mixed", 3, "Investments in education and technology will shape the future of the global economy.", 1 }
+    };
+    
+    private void analyzeDataset(String term)
+    {
+        final Pattern PATTERN = Pattern.compile("\\W+");
+        for (Object[] row : DATASET)
+        {
+            String body = (String) row[3];
+            String[] words = PATTERN.split(body.toLowerCase());
+
+            long totalWords = words.length;
+            long termCount = Arrays.stream(words)
+                                   .filter(word -> word.equals(term))
+                                   .count();
+
+            if (termCount > 0)
+                System.out.printf("            // ID %d: total words = %d, %s occurrences = %d%n",
+                                  (Integer) row[0], totalWords, term, termCount);
+        }
+    }
+
+    private void insertPrimitiveData()
+    {
+        insertPrimitiveData(0, DATASET.length);
+    }
+
+    private void insertPrimitiveData(int start, int end)
+    {
+        for (int i = start; i < end; i++)
+        {
+            Object[] row = DATASET[i];
+            execute(
+            "INSERT INTO %s (id, category, score, body) VALUES (?, ?, ?, ?)",
+            row[0],
+            row[1],
+            row[2],
+            row[3]
+            );
+        }
+    }
+
+    private void insertCollectionData()
+    {
+        int setsize = 1;
+        for (int row = 0; row < DATASET.length; row++)
+        {
+            var set = new HashSet<String>();
+            for (int j = 0; j < setsize; j++)
+                set.add((String) DATASET[row - j][3]);
+            if (setsize >= 3)
+                setsize -= 2;
+            else
+                setsize++;
+            var map = new HashMap<Integer, String>();
+            var map_text = new HashMap<String, String>();
+            for (int j = 0; j <= row && j < 3; j++)
+            {
+                map.putIfAbsent((Integer) DATASET[row - j][2], (String) DATASET[row - j][1]);
+                map_text.putIfAbsent((String) DATASET[row - j][1], (String) DATASET[row - j][3]);
+            }
+
+            execute(
+            "INSERT INTO %s (id, category, score, body, bodyset, map_category, map_body) " +
+            "VALUES (?, ?, ?, ?, ?, ?, ?)",
+            DATASET[row][0],
+            DATASET[row][1],
+            DATASET[row][2],
+            DATASET[row][3],
+            set,
+            map,
+            map_text
+            );
+        }
+    }
+
+    private void executeQuery(List<Integer> expected, String query, Object... values) throws Throwable
+    {
+        assertResult(execute(query, values), expected);
+        prepare(query);
+        assertResult(execute(query, values), expected);
+    }
+
+    private void assertResult(UntypedResultSet result, List<Integer> expected)
+    {
+        Assertions.assertThat(result).hasSize(expected.size());
+        var ids = result.stream()
+                        .map(row -> row.getInt("id"))
+                        .collect(Collectors.toList());
+        Assertions.assertThat(ids).isEqualTo(expected);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java b/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java
index 75e72b1686e7..2fb93b8f3009 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java
@@ -27,7 +27,7 @@
 public class BooleanTypeTest extends SAITester
 {
     @Test
-    public void test() throws Throwable
+    public void test()
     {
         createTable("CREATE TABLE %s (id text PRIMARY KEY, val boolean)");
 
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/ClusteringKeyIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/ClusteringKeyIndexTest.java
index 58443fe0fbaf..9434cce1ac2b 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/ClusteringKeyIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/ClusteringKeyIndexTest.java
@@ -29,24 +29,25 @@ public class ClusteringKeyIndexTest extends SAITester
     @Before
     public void createTableAndIndex()
     {
-        createTable("CREATE TABLE %s (pk1 int, pk2 text, val int, PRIMARY KEY((pk1), pk2)) WITH CLUSTERING ORDER BY (pk2 DESC)");
+        createTable("CREATE TABLE %s (pk1 int, pk2 text, val int, val2 int, PRIMARY KEY((pk1), pk2)) WITH CLUSTERING ORDER BY (pk2 DESC)");
         createIndex("CREATE CUSTOM INDEX pk2_idx ON %s(pk2) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX val2_idx ON %s(val2) USING 'StorageAttachedIndex'");
 
         disableCompaction();
     }
 
     private void insertData1() throws Throwable
     {
-        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, '1', 1)");
-        execute("INSERT INTO %s (pk1, pk2, val) VALUES (2, '2', 2)");
-        execute("INSERT INTO %s (pk1, pk2, val) VALUES (3, '3', 3)");
+        execute("INSERT INTO %s (pk1, pk2, val, val2) VALUES (1, '1', 1, 1)");
+        execute("INSERT INTO %s (pk1, pk2, val, val2) VALUES (2, '2', 2, 2)");
+        execute("INSERT INTO %s (pk1, pk2, val, val2) VALUES (3, '3', 3, 3)");
     }
 
     private void insertData2() throws Throwable
     {
-        execute("INSERT INTO %s (pk1, pk2, val) VALUES (4, '4', 4)");
-        execute("INSERT INTO %s (pk1, pk2, val) VALUES (5, '5', 5)");
-        execute("INSERT INTO %s (pk1, pk2, val) VALUES (6, '6', 6)");
+        execute("INSERT INTO %s (pk1, pk2, val, val2) VALUES (4, '4', 4, 4)");
+        execute("INSERT INTO %s (pk1, pk2, val, val2) VALUES (5, '5', 5, 5)");
+        execute("INSERT INTO %s (pk1, pk2, val, val2) VALUES (6, '6', 6, 6)");
     }
 
     @Test
@@ -57,9 +58,48 @@ public void queryFromMemtable() throws Throwable
         runQueries();
     }
 
+    @Test
+    public void queryFromSingleSSTable() throws Throwable
+    {
+        insertData1();
+        insertData2();
+        flush();
+        runQueries();
+    }
+
+    @Test
+    public void queryFromMultipleSSTables() throws Throwable
+    {
+        insertData1();
+        flush();
+        insertData2();
+        flush();
+        runQueries();
+    }
+
+    @Test
+    public void queryFromMemtableAndSSTables() throws Throwable
+    {
+        insertData1();
+        flush();
+        insertData2();
+        runQueries();
+    }
+
+    @Test
+    public void queryFromCompactedSSTable() throws Throwable
+    {
+        insertData1();
+        flush();
+        insertData2();
+        flush();
+        compact();
+        runQueries();
+    }
+
     private Object[] expectedRow(int index)
     {
-        return row(index, Integer.toString(index), index);
+        return row(index, Integer.toString(index), index, index);
     }
 
     private void runQueries() throws Throwable
@@ -72,5 +112,7 @@ private void runQueries() throws Throwable
 
         assertThatThrownBy(()->execute("SELECT * FROM %s WHERE pk1 = -1 AND val = 2")).hasMessageContaining("use ALLOW FILTERING");
 
+        // Add an assertion that covers searching a non-primary key column
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE val2 = 1"), expectedRow(1));
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java b/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
index b490dfa94673..5004e6e127ed 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
@@ -23,20 +23,18 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.index.sai.SAITester;
 
 import static org.junit.Assert.assertEquals;
 
-/**
- * This test is primarily handling edge conditions, error conditions
- * and basic functionality. Comprehensive type testing of collections
- * is in the cql/types/collections package
- */
+// This test is primarily handling edge conditions, error conditions
+// and basic functionality. Comprehensive type testing of collections
+// is in the cql/types/collections package
+//TODO Sort out statement restrictions assertion
 public class CollectionIndexingTest extends SAITester
 {
     @Before
-    public void setup()
+    public void setup() throws Throwable
     {
         requireNetwork();
     }
@@ -44,43 +42,125 @@ public void setup()
     @Test
     public void indexMap()
     {
-        createPopulatedMap(createIndexDDL("value"));
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
         assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS 'v1'").size());
+
+        assertEmpty(execute("SELECT pk FROM %s WHERE value NOT CONTAINS 'v1'"));
+
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS 'v2'"),
+                                row(2));
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS 'v3'"),
+                                row(1));
+
+        flush();
+
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS 'v2'"),
+                                row(2));
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS 'v3'"),
+                                row(1));
+    }
+
+    @Test
+    public void indexEmptyMaps()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value map<int, text>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
+
+        // Test memtable index:
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v2");
+        }});
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, new HashMap<Integer, String>());
+
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value CONTAINS 'v1'"),
+                                row(1));
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS 'v1'"),
+                                row(2));
+
+        // Test sstable index:
+        flush();
+
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value CONTAINS 'v1'"),
+                                row(1));
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS 'v1'"),
+                                row(2));
+
+        // Add one more row with an empty map and flush.
+        // This will create an sstable with no index.
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 3, new HashMap<Integer, String>());
+        flush();
+
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value CONTAINS 'v1'"),
+                                row(1));
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS 'v1'"),
+                                row(2), row(3));
+    }
+
+    @Test
+    public void indexQueryEmpty()
+    {
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
+        assertEquals(0, execute("SELECT * FROM %s WHERE value CONTAINS ''").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value CONTAINS '' AND value CONTAINS 'v1'").size());
     }
 
     @Test
     public void indexMapKeys()
     {
-        createPopulatedMap(createIndexDDL("KEYS(value)"));
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
         assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS KEY 1").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value NOT CONTAINS KEY 1").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE value NOT CONTAINS KEY 5").size());
+
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 3, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(3, "v4");
+        }});
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS KEY 2"), row(3));
+
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 4, new HashMap<Integer, String>());
+        assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE value NOT CONTAINS KEY 2"), row(3), row(4));
     }
 
     @Test
     public void indexMapValues()
     {
-        createPopulatedMap(createIndexDDL("VALUES(value)"));
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
         assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS 'v1'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value NOT CONTAINS 'v1'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE value NOT CONTAINS 'v5'").size());
     }
 
     @Test
     public void indexMapEntries()
     {
-        createPopulatedMap(createIndexDDL("ENTRIES(value)"));
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
         assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1'").size());
         assertEquals(1, execute("SELECT * FROM %s WHERE value[1] = 'v1' AND value[2] = 'v2'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value[1] != 'v1'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE value[1] != 'v2' AND value[2] != 'v2'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] != 'v3'").size());
     }
 
     @Test
     public void indexFrozenList()
     {
-        createPopulatedFrozenList(createIndexDDL("FULL(value)"));
+        createPopulatedFrozenList();
+        createIndex("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
         assertEquals(2, execute("SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2, 3)).size());
     }
 
     @Test
-    public void indexFrozenMap() throws Throwable
+    public void indexFrozenMap()
     {
-        createPopulatedFrozenMap(createIndexDDL("FULL(value)"));
+        createPopulatedFrozenMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
         assertEquals(1, execute("SELECT * FROM %s WHERE value = ?", new HashMap<Integer, String>() {{
             put(1, "v1");
             put(2, "v2");
@@ -89,125 +169,162 @@ public void indexFrozenMap() throws Throwable
     }
 
     @Test
-    public void indexFrozenMapQueryKeys() throws Throwable
+    public void indexFrozenMapQueryKeys()
     {
-        createPopulatedFrozenMap(createIndexDDL("FULL(value)"));
-        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains key 1");
+        createPopulatedFrozenMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value not contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
     }
 
     @Test
-    public void indexFrozenMapQueryValues() throws Throwable
+    public void indexFrozenMapQueryValues()
     {
-        createPopulatedFrozenMap(createIndexDDL("FULL(value)"));
-        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains 'v1'");
+        createPopulatedFrozenMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value not contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value not contains 'v1' ALLOW FILTERING").size());
     }
 
     @Test
-    public void indexFrozenMapQueryEntries() throws Throwable
+    public void indexFrozenMapQueryEntries()
     {
-        createPopulatedFrozenMap(createIndexDDL("FULL(value)"));
+        createPopulatedFrozenMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
         assertInvalidMessage("Map-entry equality predicates on frozen map column value are not supported",
                 "SELECT * FROM %s WHERE value[1] = 'v1'");
     }
 
     @Test
-    public void indexMapEntriesQueryEq() throws Throwable
+    public void indexMapEntriesQueryEq()
     {
-        createPopulatedMap(createIndexDDL("ENTRIES(value)"));
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
         assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
                 "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
     }
 
     @Test
-    public void indexMapEntriesQueryKeys() throws Throwable
+    public void indexMapEntriesQueryKeys()
     {
-        createPopulatedMap(createIndexDDL("ENTRIES(value)"));
-        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains key 1");
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value not contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value not contains key 1 ALLOW FILTERING").size());
     }
 
     @Test
-    public void indexMapEntriesQueryValues() throws Throwable
+    public void indexMapEntriesQueryValues()
     {
-        createPopulatedMap(createIndexDDL("ENTRIES(value)"));
-        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains 'v1'");
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value not contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value not contains 'v1' ALLOW FILTERING").size());
     }
 
     @Test
-    public void indexMapKeysQueryEq() throws Throwable
+    public void indexMapKeysQueryEq()
     {
-        createPopulatedMap(createIndexDDL("KEYS(value)"));
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
         assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
                 "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
     }
 
     @Test
-    public void indexMapKeysQueryValues() throws Throwable
+    public void indexMapKeysQueryValues()
     {
-        createPopulatedMap(createIndexDDL("KEYS(value)"));
-        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains 'v1'");
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
     }
 
     @Test
-    public void indexMapKeysQueryEntries() throws Throwable
+    public void indexMapKeysQueryEntries()
     {
-        createPopulatedMap(createIndexDDL("KEYS(value)"));
-        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value[1] = 'v1'");
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] = 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1' ALLOW FILTERING").size());
     }
 
     @Test
-    public void indexMapValuesQueryEq() throws Throwable
+    public void indexMapValuesQueryEq()
     {
-        createPopulatedMap(createIndexDDL("VALUES(value)"));
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
         assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
                 "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
     }
 
     @Test
-    public void indexMapValuesQueryKeys() throws Throwable
+    public void indexMapValuesQueryKeys()
     {
-        createPopulatedMap(createIndexDDL("VALUES(value)"));
-        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains key 1");
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value not contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value not contains key 1 ALLOW FILTERING").size());
     }
 
     @Test
-    public void indexMapValuesQueryEntries() throws Throwable
+    public void indexMapValuesQueryEntries()
     {
-        createPopulatedMap(createIndexDDL("VALUES(value)"));
-        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value[1] = 'v1'");
+        createPopulatedMap();
+        createIndex("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] = 'v1'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] != 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1' ALLOW FILTERING").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE value[1] != 'v1' ALLOW FILTERING").size());
     }
 
     @Test
-    public void unindexedContainsExpressions()
+    public void notContainsShouldReturnUpdatedRows() throws Throwable
     {
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v int, m map<int, int>)");
-        createIndex("CREATE INDEX ON %s(v) USING 'SAI'"); // just to make sure that SAI is involved
-
-        Object[] row = row(0, 1, map(2, 3));
-        execute("INSERT INTO %s (k, v, m) VALUES (?, ?, ?)", row);
-        execute("INSERT INTO %s (k, v, m) VALUES (?, ?, ?)", 1, 1, map(12, 13));
-
-        // try without any indexes on the map
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS 3 ALLOW FILTERING"), row);
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS KEY 2 ALLOW FILTERING"), row);
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS KEY 2 AND m CONTAINS 3 ALLOW FILTERING"), row);
-
-        // try with index on map values
-        createIndex("CREATE INDEX ON %s(m) USING 'SAI'");
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS 3"), row);
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS KEY 2 ALLOW FILTERING"), row);
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS KEY 2 AND m CONTAINS 3 ALLOW FILTERING"), row);
-
-        // try with index on map keys
-        createIndex("CREATE INDEX ON %s(KEYS(m)) USING 'SAI'");
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS 3"), row);
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS KEY 2"), row);
-        assertRows(execute("SELECT k, v, m FROM %s WHERE v = 1 AND m CONTAINS KEY 2 AND m CONTAINS 3"), row);
+        createTable("CREATE TABLE %s(id int PRIMARY KEY, text_map map<text, text>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(values(text_map)) USING 'StorageAttachedIndex'");
+        execute("INSERT INTO %s(id, text_map) values (1, {'k1':'v1'})");
+        flush();
+        // This update overwrites 'v1', so now the map does not contain 'v1' and the row should be returned
+        // by the NOT CONTAINS 'v1' query. We purposefuly make this update after flush, so it ends up in a separate
+        // index than the original row.
+        execute("INSERT INTO %s(id, text_map) values (1, {'k2':'v2'})");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT id FROM %s WHERE text_map NOT CONTAINS 'v1'"), row(1));
+        });
     }
 
-    private void createPopulatedMap(String createIndex)
+    @Test
+    public void testUpdateMapToNullValue() throws Throwable
+    {
+        createTable("CREATE TABLE %s(id int PRIMARY KEY, text_map map<text, text>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(values(text_map)) USING 'StorageAttachedIndex'");
+        waitForTableIndexesQueryable();
+        execute("INSERT INTO %s(id, text_map) values (1, {'k1':'v1'})");
+        execute("INSERT INTO %s(id, text_map) values (2, {'k1':'v2'})");
+        assertRows(execute("SELECT id FROM %s WHERE text_map CONTAINS 'v1'"), row(1));
+        assertRows(execute("SELECT id FROM %s WHERE text_map NOT CONTAINS 'v1'"), row(2));
+        // Overwrite with null
+        execute("INSERT INTO %s(id, text_map) values (1, null)");
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT id FROM %s WHERE text_map CONTAINS 'v1'"));
+            assertRows(execute("SELECT id FROM %s WHERE text_map NOT CONTAINS 'v1'"), row(1), row(2));
+        });
+    }
+
+    private void createPopulatedMap()
     {
         createTable("CREATE TABLE %s (pk int primary key, value map<int, text>)");
-        createIndex(createIndex);
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
             put(1, "v1");
             put(2, "v2");
@@ -218,11 +335,9 @@ private void createPopulatedMap(String createIndex)
         }});
     }
 
-    @SuppressWarnings("SameParameterValue")
-    private void createPopulatedFrozenMap(String createIndex)
+    private void createPopulatedFrozenMap()
     {
         createTable("CREATE TABLE %s (pk int primary key, value frozen<map<int, text>>)");
-        createIndex(createIndex);
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
             put(1, "v1");
             put(2, "v2");
@@ -233,26 +348,18 @@ private void createPopulatedFrozenMap(String createIndex)
         }});
     }
 
-    @SuppressWarnings("SameParameterValue")
-    private void createPopulatedFrozenList(String createIndex)
+    private void createPopulatedFrozenList()
     {
         createTable("CREATE TABLE %s (pk int primary key, value frozen<list<int>>)");
-        createIndex(createIndex);
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, Arrays.asList(1, 2, 3));
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, Arrays.asList(1, 2, 3));
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 3, Arrays.asList(4, 5, 6));
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 4, Arrays.asList(1, 2, 7));
     }
 
-    @SuppressWarnings("SameParameterValue")
-    private void assertUnsupportedIndexOperator(int expectedSize, String query, Object... values) throws Throwable
-    {
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, query, values);
-        assertEquals(expectedSize, execute(query + " ALLOW FILTERING").size());
-    }
-
-    private static String createIndexDDL(String target)
+    private void assertUnsupportedIndexOperator(String query, Object... values)
     {
-        return "CREATE INDEX ON %s(" + target + ") USING 'sai'";
+//        assertInvalidMessage(String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, "value"),
+//                query, values);
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/ComplexQueryTest.java b/test/unit/org/apache/cassandra/index/sai/cql/ComplexQueryTest.java
index 0b4a053d1232..1cf3e2800b32 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/ComplexQueryTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/ComplexQueryTest.java
@@ -21,8 +21,13 @@
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.sai.SAITester;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+
 public class ComplexQueryTest extends SAITester
 {
     @Test
@@ -65,4 +70,212 @@ public void splitRowsWithBooleanLogic()
         var result = execute("SELECT pk FROM %s WHERE str_val = 'A' AND val = 'A'");
         assertRows(result, row(3));
     }
+
+    @Test
+    public void basicOrTest()
+    {
+        createTable("CREATE TABLE %s (pk int, a int, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 1, 1);
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 2, 2);
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 3, 3);
+
+        UntypedResultSet resultSet = execute("SELECT pk FROM %s WHERE a = 1 or a = 3");
+
+        assertRowsIgnoringOrder(resultSet, row(1), row(3) );
+    }
+
+    @Test
+    public void basicInTest()
+    {
+        createTable("CREATE TABLE %s (pk int, a int, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 1, 1);
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 2, 2);
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 3, 3);
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 4, 4);
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 5, 5);
+
+        UntypedResultSet resultSet = execute("SELECT pk FROM %s WHERE a in (1, 3, 5)");
+
+        assertRowsIgnoringOrder(resultSet, row(1), row(3), row(5));
+    }
+
+    @Test
+    public void complexQueryTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, a int, b int, c int, d int, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(d) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, a, b, c, d) VALUES (?, ?, ?, ?, ?)", 1, 1, 1, 1, 1);
+        execute("INSERT INTO %s (pk, a, b, c, d) VALUES (?, ?, ?, ?, ?)", 2, 2, 1, 1, 1);
+        execute("INSERT INTO %s (pk, a, b, c, d) VALUES (?, ?, ?, ?, ?)", 3, 3, 2, 1, 1);
+        execute("INSERT INTO %s (pk, a, b, c, d) VALUES (?, ?, ?, ?, ?)", 4, 4, 2, 2, 1);
+        execute("INSERT INTO %s (pk, a, b, c, d) VALUES (?, ?, ?, ?, ?)", 5, 5, 3, 2, 1);
+        execute("INSERT INTO %s (pk, a, b, c, d) VALUES (?, ?, ?, ?, ?)", 6, 6, 3, 2, 2);
+        execute("INSERT INTO %s (pk, a, b, c, d) VALUES (?, ?, ?, ?, ?)", 7, 7, 4, 3, 2);
+        execute("INSERT INTO %s (pk, a, b, c, d) VALUES (?, ?, ?, ?, ?)", 8, 8, 4, 3, 3);
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE (a = 1 AND c = 1) OR (b IN (3, 4) AND d = 2)"), row(1), row(7), row(6));
+            // Shows that IN with an empty list produces no rows
+            assertRows(execute("SELECT pk FROM %s WHERE (a = 1 AND c = 1) OR (b IN () AND d = 2)"), row(1));
+            assertRows(execute("SELECT pk FROM %s WHERE b IN () AND d = 2"));
+            assertRows(execute("SELECT pk FROM %s WHERE b NOT IN () AND d = 2"), row(7), row(6));
+        });
+
+    }
+
+    @Test
+    public void disjunctionWithClusteringKey()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, PRIMARY KEY(pk, ck))");
+
+        execute("INSERT INTO %s (pk, ck, a) VALUES (?, ?, ?)", 1, 1, 1);
+        execute("INSERT INTO %s (pk, ck, a) VALUES (?, ?, ?)", 2, 2, 2);
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE a = 1 or ck = 2"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+
+        UntypedResultSet resultSet = execute("SELECT pk FROM %s WHERE a = 1 or ck = 2 ALLOW FILTERING");
+
+        assertRowsIgnoringOrder(resultSet, row(1), row(2));
+    }
+
+    @Test
+    public void disjunctionWithIndexOnClusteringKey()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(ck) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, ck, a) VALUES (?, ?, ?)", 1, 1, 1);
+        execute("INSERT INTO %s (pk, ck, a) VALUES (?, ?, ?)", 2, 2, 2);
+
+        UntypedResultSet resultSet = execute("SELECT pk FROM %s WHERE a = 1 or ck = 2");
+
+        assertRowsIgnoringOrder(resultSet, row(1), row(2));
+    }
+
+    @Test
+    public void complexQueryWithMultipleClusterings()
+    {
+        createTable("CREATE TABLE %s (pk int, ck0 int, ck1 int, a int, b int, c int, d int, e int, PRIMARY KEY(pk, ck0, ck1))");
+        createIndex("CREATE CUSTOM INDEX ON %s(ck0) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(ck1) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(d) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(e) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, ck0, ck1, a, b, c, d, e) VALUES (?, ?, ?, ?, ?, ? ,?, ?)", 1, 1, 1, 1, 1, 1, 1, 1);
+        execute("INSERT INTO %s (pk, ck0, ck1, a, b, c, d, e) VALUES (?, ?, ?, ?, ?, ? ,?, ?)", 2, 2, 2, 2, 2, 2, 2, 2);
+        execute("INSERT INTO %s (pk, ck0, ck1, a, b, c, d, e) VALUES (?, ?, ?, ?, ?, ? ,?, ?)", 3, 3, 3, 3, 3, 3, 3, 3);
+        execute("INSERT INTO %s (pk, ck0, ck1, a, b, c, d, e) VALUES (?, ?, ?, ?, ?, ? ,?, ?)", 4, 4, 4, 4, 4, 4, 4, 4);
+        execute("INSERT INTO %s (pk, ck0, ck1, a, b, c, d, e) VALUES (?, ?, ?, ?, ?, ? ,?, ?)", 5, 5, 5, 5, 5, 5, 5, 5);
+
+        UntypedResultSet resultSet = execute("SELECT pk FROM %s WHERE b = 6 AND d = 6 OR (a = 6 OR (c = 3 OR ck0 = 5))");
+
+        assertRowsIgnoringOrder(resultSet, row(3), row(5));
+
+        resultSet = execute("SELECT pk FROM %s WHERE ck0 = 1 AND (b = 6 AND c = 6 OR (d = 6 OR e = 6))");
+
+        assertEquals(0 , resultSet.size());
+
+        resultSet = execute("SELECT pk FROM %s WHERE b = 4 OR a = 3 OR c = 5");
+
+        assertRowsIgnoringOrder(resultSet, row(3), row(4), row(5));
+    }
+
+    @Test
+    public void complexQueryWithPartitionKeyRestriction()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY(pk, ck))");
+
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 1, 1, 5);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 2, 2, 6);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 2, 1, 3, 7);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 2, 2, 4, 8);
+
+
+        assertThatThrownBy(() -> execute("SELECT pk, ck FROM %s WHERE pk = 1 AND (a = 2 OR b = 7)"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+
+        UntypedResultSet resultSet = execute("SELECT pk, ck FROM %s WHERE pk = 1 AND (a = 2 OR b = 7) ALLOW FILTERING");
+
+        assertRowsIgnoringOrder(resultSet, row(1, 2));
+
+        assertThatThrownBy(() -> execute("SELECT pk, ck FROM %s WHERE pk = 1 OR a = 2 OR b = 7 ALLOW FILTERING"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(String.format(StatementRestrictions.PARTITION_KEY_RESTRICTION_MUST_BE_TOP_LEVEL, "pk"));
+
+        // Here pk = 1 is directly under AND operation, so a simple isDisjunction check on it would not be enough
+        // to reject it ;)
+        assertThatThrownBy(() -> execute("SELECT pk, ck FROM %s WHERE a = 2 OR (pk = 1 AND b = 7) ALLOW FILTERING"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(String.format(StatementRestrictions.PARTITION_KEY_RESTRICTION_MUST_BE_TOP_LEVEL, "pk"));
+    }
+
+    @Test
+    public void complexQueryWithPartitionKeyRestrictionAndIndexes()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 1, 1, 5);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 2, 2, 6);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 2, 1, 3, 7);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 2, 2, 4, 8);
+
+        UntypedResultSet resultSet = execute("SELECT pk, ck FROM %s WHERE pk = 1 AND (a = 2 OR b = 7)");
+
+        assertRowsIgnoringOrder(resultSet, row(1, 2));
+
+        assertThatThrownBy(() -> execute("SELECT pk, ck FROM %s WHERE pk = 1 OR a = 2 OR b = 7 ALLOW FILTERING"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(String.format(StatementRestrictions.PARTITION_KEY_RESTRICTION_MUST_BE_TOP_LEVEL, "pk"));
+    }
+
+    @Test
+    public void indexNotSupportingDisjunctionTest()
+    {
+        createTable("CREATE TABLE %s (pk int, a int, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'org.apache.cassandra.index.sasi.SASIIndex'");
+
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 1, 1);
+        execute("INSERT INTO %s (pk, a) VALUES (?, ?)", 2, 2);
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE a = 1 or a = 2")).isInstanceOf(InvalidRequestException.class)
+                                                                                   .hasMessage(StatementRestrictions.INDEX_DOES_NOT_SUPPORT_DISJUNCTION);
+
+        assertRows(execute("SELECT pk FROM %s WHERE a = 1 or a = 2 ALLOW FILTERING"), row(1), row(2));
+    }
+
+    @Test
+    public void complexQueryWithMultipleNEQ()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 1, 1, 5);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 2, 2, 6);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 3, 3, 7);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 4, 4, 8);
+        execute("INSERT INTO %s (pk, ck, a, b) VALUES (?, ?, ?, ?)", 1, 5, null, null);
+
+        assertRowsIgnoringOrder(execute("SELECT ck FROM %s WHERE pk = 1 AND a != 2 AND b != 7"), row(1), row(4));
+        assertRowsIgnoringOrder(execute("SELECT ck FROM %s WHERE pk = 1 AND a != 2 AND a != 3"), row(1), row(4));
+        assertRowsIgnoringOrder(execute("SELECT ck FROM %s WHERE pk = 1 AND a NOT IN (2, 3)"), row(1), row(4));
+        assertRowsIgnoringOrder(execute("SELECT ck FROM %s WHERE pk = 1 AND a NOT IN (2, 3) AND b NOT IN (7, 8)"), row(1));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java
index 99fcdf8319d1..bd0622edbb67 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java
@@ -17,92 +17,127 @@
  */
 package org.apache.cassandra.index.sai.cql;
 
+import org.junit.Before;
 import org.junit.Test;
 
-import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.index.sai.SAITester;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
 public class CompositePartitionKeyIndexTest extends SAITester
 {
-    @Test
-    public void testCompositePartitionIndex() throws Throwable
+    @Before
+    public void createTableAndIndex()
     {
         createTable("CREATE TABLE %s (pk1 int, pk2 text, val int, PRIMARY KEY((pk1, pk2)))");
-        createIndex("CREATE INDEX ON %s(pk1) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(pk2) USING 'sai'");
+        createIndex("CREATE CUSTOM INDEX ON %s(pk1) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(pk2) USING 'StorageAttachedIndex'");
 
+        disableCompaction();
+    }
+
+    private void insertData1()
+    {
         execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, '1', 1)");
         execute("INSERT INTO %s (pk1, pk2, val) VALUES (2, '2', 2)");
         execute("INSERT INTO %s (pk1, pk2, val) VALUES (3, '3', 3)");
+    }
+
+    private void insertData2()
+    {
         execute("INSERT INTO %s (pk1, pk2, val) VALUES (4, '4', 4)");
         execute("INSERT INTO %s (pk1, pk2, val) VALUES (5, '5', 5)");
         execute("INSERT INTO %s (pk1, pk2, val) VALUES (6, '6', 6)");
+    }
 
-        beforeAndAfterFlush(() -> {
-            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = 2"),
-                                    expectedRow(2));
-
-            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1"),
-                                    expectedRow(2),
-                                    expectedRow(3),
-                                    expectedRow(4),
-                                    expectedRow(5),
-                                    expectedRow(6));
-
-            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 >= 3"),
-                                    expectedRow(3),
-                                    expectedRow(4),
-                                    expectedRow(5),
-                                    expectedRow(6));
-
-            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 < 3"),
-                                    expectedRow(1),
-                                    expectedRow(2));
-
-            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 <= 3"),
-                                    expectedRow(1),
-                                    expectedRow(2),
-                                    expectedRow(3));
-
-            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk2 = '2'"),
-                                    expectedRow(2));
-
-            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1 AND pk2 = '2'"),
-                                    expectedRow(2));
+    @Test
+    public void queryFromMemtable() throws Throwable
+    {
+        insertData1();
+        insertData2();
+        runQueries();
+    }
 
-            assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = -1 AND pk2 = '2'"));
+    @Test
+    public void queryFromSingleSSTable() throws Throwable
+    {
+        insertData1();
+        insertData2();
+        flush();
+        runQueries();
+    }
 
-            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT * FROM %s WHERE pk1 = -1 AND val = 2");
-        });
+    @Test
+    public void queryFromMultipleSSTables() throws Throwable
+    {
+        insertData1();
+        flush();
+        insertData2();
+        flush();
+        runQueries();
     }
 
     @Test
-    public void testFilterWithIndexForContains() throws Throwable
+    public void queryFromMemtableAndSSTables() throws Throwable
     {
-        createTable("CREATE TABLE %s (k1 int, k2 int, v set<int>, PRIMARY KEY ((k1, k2)))");
-        createIndex("CREATE INDEX ON %s(k2) USING 'sai'");
-
-        execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", 0, 0, set(1, 2, 3));
-        execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", 0, 1, set(2, 3, 4));
-        execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", 1, 0, set(3, 4, 5));
-        execute("INSERT INTO %s (k1, k2, v) VALUES (?, ?, ?)", 1, 1, set(4, 5, 6));
-
-        beforeAndAfterFlush(() -> {
-            assertRows(execute("SELECT * FROM %s WHERE k2 = ?", 1),
-                       row(0, 1, set(2, 3, 4)),
-                       row(1, 1, set(4, 5, 6))
-            );
-
-            assertRows(execute("SELECT * FROM %s WHERE k2 = ? AND v CONTAINS ? ALLOW FILTERING", 1, 6),
-                       row(1, 1, set(4, 5, 6))
-            );
-
-            assertEmpty(execute("SELECT * FROM %s WHERE k2 = ? AND v CONTAINS ? ALLOW FILTERING", 1, 7));
-        });
+        insertData1();
+        flush();
+        insertData2();
+        runQueries();
     }
 
-    private Object[] expectedRow(int index)
+    @Test
+    public void queryFromCompactedSSTable() throws Throwable
     {
+        insertData1();
+        flush();
+        insertData2();
+        flush();
+        compact();
+        runQueries();
+    }
+
+    private Object[] expectedRow(int index) {
         return row(index, Integer.toString(index), index);
     }
+
+    private void runQueries() throws Throwable
+    {
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = 2"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1"),
+                expectedRow(2),
+                expectedRow(3),
+                expectedRow(4),
+                expectedRow(5),
+                expectedRow(6));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 >= 3"),
+                expectedRow(3),
+                expectedRow(4),
+                expectedRow(5),
+                expectedRow(6));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 < 3"),
+                expectedRow(1),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 <= 3"),
+                expectedRow(1),
+                expectedRow(2),
+                expectedRow(3));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk2 = '2'"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1 AND pk2 = '2'"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = -1 AND pk2 = '2'"));
+
+        assertThatThrownBy(()->execute("SELECT * FROM %s WHERE pk1 = -1 AND val = 2"))
+                .hasMessageContaining("use ALLOW FILTERING");
+
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/DBPE14965Test.java b/test/unit/org/apache/cassandra/index/sai/cql/DBPE14965Test.java
new file mode 100644
index 000000000000..00c3cb69bad1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/DBPE14965Test.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.assertj.core.api.Assertions;
+
+public class DBPE14965Test extends SAITester
+{
+    @Test
+    public void testDBPE14965()
+    {
+        createTable("CREATE TABLE %s (" +
+                    "    pk1 int," +
+                    "    pk2 int," +
+                    "    ck int," +
+                    "    r int," +
+                    "    s int static," +
+                    "    PRIMARY KEY ((pk1, pk2), ck))");
+
+        createIndex("CREATE CUSTOM INDEX ON %s (pk1) USING 'StorageAttachedIndex'");
+
+        disableCompaction();
+        execute("INSERT INTO %s (pk1, pk2, ck, r, s) VALUES (0, ?, ?, ?, ?)", 1, 1, 1, 1);
+        flush();
+        compact();
+
+        Assertions.assertThat(execute("SELECT * FROM %s WHERE pk1=0").size()).isEqualTo(1); // finds 2!!
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java b/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java
index 107dae1c0a76..b4b351b90287 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java
@@ -42,7 +42,7 @@ public void createTableAndIndex()
 
     /**
      * This test tries to induce rounding errors involving decimal values with wide significands.
-     *
+     * </p>
      * Two values are indexed:
      * <ul>
      * <li>1.0</li>
@@ -50,11 +50,11 @@ public void createTableAndIndex()
      * </ul>
      */
     @Test
-    public void runQueriesWithDecimalValueCollision() throws Throwable
+    public void runQueriesWithDecimalValueCollision()
     {
         final int significandSizeInDecimalDigits = 512;
         // String.repeat(int) exists in JDK 11 and later, but this line was introduced on JDK 8
-        String wideDecimalString = "1." + StringUtils.repeat('0', significandSizeInDecimalDigits - 2) + '1';
+        String wideDecimalString = "1." + StringUtils.repeat('0', significandSizeInDecimalDigits - 2) + "1";
         BigDecimal wideDecimal = new BigDecimal(wideDecimalString);
         // Sanity checks that this value was actually constructed as intended
         Preconditions.checkState(wideDecimal.precision() == significandSizeInDecimalDigits,
@@ -64,7 +64,7 @@ public void runQueriesWithDecimalValueCollision() throws Throwable
                                  "expected: %s; actual: %s", wideDecimalString, wideDecimal.toPlainString());
 
         execute("INSERT INTO %s (pk, ck, dec) VALUES (0, 1, 1.0)");
-        execute("INSERT INTO %s (pk, ck, dec) VALUES (2, 0, " + wideDecimalString + ')');
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (2, 0, " + wideDecimalString + ")");
 
         // EQ queries
         assertRows(execute("SELECT * FROM %s WHERE dec = 1.0"),
@@ -98,12 +98,19 @@ public void runQueriesWithDecimalValueCollision() throws Throwable
         assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec >= 1.0"),
                 row(0, 1, BigDecimal.valueOf(1.0D)),
                 row(2, 0, wideDecimal));
+
+        // NEQ queries
+        assertRows(execute("SELECT * FROM %s WHERE dec != 1.0"),
+                row(2, 0, wideDecimal));
+
+        assertRows(execute("SELECT * FROM %s WHERE dec != " + wideDecimalString),
+                row(0, 1, BigDecimal.valueOf(1.0D)));
     }
     /**
      * This is a control method with small (two-significant-digit) values.
      */
     @Test
-    public void runQueriesWithoutCollisions() throws Throwable
+    public void runQueriesWithoutCollisions()
     {
         execute("INSERT INTO %s (pk, ck, dec) VALUES (-2, 1, 2.2)");
         execute("INSERT INTO %s (pk, ck, dec) VALUES (-2, 2, 2.2)");
@@ -147,5 +154,13 @@ public void runQueriesWithoutCollisions() throws Throwable
                 row(1, 1, BigDecimal.valueOf(1.1D)),
                 row(2, 1, BigDecimal.valueOf(2.2D)),
                 row(2, 2, BigDecimal.valueOf(2.2D)));
+
+        // NEQ queries
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec != 1.1"),
+                row(-2, 1, BigDecimal.valueOf(2.2D)),
+                row(-2, 2, BigDecimal.valueOf(2.2D)),
+                row(0, 1, BigDecimal.valueOf(0)),
+                row(2, 1, BigDecimal.valueOf(2.2D)),
+                row(2, 2, BigDecimal.valueOf(2.2D)));
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/DropIndexWhileQueryingTest.java b/test/unit/org/apache/cassandra/index/sai/cql/DropIndexWhileQueryingTest.java
new file mode 100644
index 000000000000..6d48e411974b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/DropIndexWhileQueryingTest.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.plan.QueryController;
+import org.apache.cassandra.index.sai.plan.TopKProcessor;
+import org.apache.cassandra.inject.ActionBuilder;
+import org.apache.cassandra.inject.Injection;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.inject.InvokePointBuilder;
+
+import org.junit.Test;
+
+import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertTrue;
+
+public class DropIndexWhileQueryingTest extends SAITester
+{
+    // See CNDB-10732
+    @Test
+    public void testDropIndexWhileQuerying() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k text PRIMARY KEY, x int, y text, z text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(y) USING 'StorageAttachedIndex'");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(x) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(z) USING 'StorageAttachedIndex'");
+        waitForTableIndexesQueryable();
+
+        injectIndexDrop("drop_index", indexName, "buildPlan", true);
+
+        execute("INSERT INTO %s (k, x, y, z) VALUES (?, ?, ?, ?)", "car", 0, "y0", "z0");
+        String query = "SELECT * FROM %s WHERE x IN (0, 1) OR (y IN ('Y0', 'Y1' ) OR z IN ('z1', 'z2'))";
+        assertThatThrownBy(() -> executeInternal(query)).hasMessage(QueryController.INDEX_MAY_HAVE_BEEN_DROPPED);
+        assertThatThrownBy(() -> executeInternal(query)).hasMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+    }
+
+    @Test
+    public void testFallbackToAnotherIndex() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k text PRIMARY KEY, x int, y text, z text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(y) USING 'StorageAttachedIndex'");
+        String indexName1 = createIndex("CREATE CUSTOM INDEX ON %s(x) USING 'StorageAttachedIndex'");
+        String indexName2 = createIndex("CREATE CUSTOM INDEX ON %s(z) USING 'StorageAttachedIndex'");
+        waitForTableIndexesQueryable();
+
+        injectIndexDrop("drop_index_1", indexName1, "buildIterator", true);
+        injectIndexDrop("drop_index_2", indexName2, "buildIterator", true);
+
+        execute("INSERT INTO %s (k, x, y, z) VALUES (?, ?, ?, ?)", "k1", 0, "y0", "z0"); // match
+        execute("INSERT INTO %s (k, x, y, z) VALUES (?, ?, ?, ?)", "k2", 0, "y1", "z2"); // no match
+        execute("INSERT INTO %s (k, x, y, z) VALUES (?, ?, ?, ?)", "k3", 5, "y2", "z0"); // no match
+        String query = "SELECT * FROM %s WHERE x = 0 AND y = 'y0' AND z = 'z0'";
+        assertRowCount(execute(query), 1);
+    }
+
+    // See CNDB-10535
+    @Test
+    public void testDropVectorIndexWhileQuerying() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        waitForTableIndexesQueryable();
+
+        injectIndexDrop("drop_index2", indexName, "buildPlan", false);
+
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
+
+        String query = "SELECT pk FROM %s ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 2";
+        assertThatThrownBy(() -> executeInternal(query)).hasMessage(TopKProcessor.INDEX_MAY_HAVE_BEEN_DROPPED);
+        assertThatThrownBy(() -> executeInternal(query)).hasMessage(String.format(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE, "val"));
+    }
+
+    private static void injectIndexDrop(String injectionName, String indexName, String methodName, boolean atEntry) throws Throwable
+    {
+        InvokePointBuilder invokePoint = newInvokePoint().onClass(QueryController.class).onMethod(methodName);
+        Injection injection = Injections.newCustom(injectionName)
+                                        .add(atEntry ? invokePoint.atEntry() : invokePoint.atExit())
+                                        .add(ActionBuilder
+                                             .newActionBuilder()
+                                             .actions()
+                                             .doAction("org.apache.cassandra.index.sai.cql.DropIndexWhileQueryingTest" +
+                                                       ".dropIndexForBytemanInjections(\"" + indexName + "\")"))
+                                        .build();
+        Injections.inject(injection);
+        injection.enable();
+        assertTrue("Injection should be enabled", injection.isEnabled());
+    }
+
+    // the method is used by the byteman rule to drop the index
+    @SuppressWarnings("unused")
+    public static void dropIndexForBytemanInjections(String indexName)
+    {
+        String fullQuery = String.format("DROP INDEX IF EXISTS %s.%s", KEYSPACE, indexName);
+        logger.info(fullQuery);
+        schemaChange(fullQuery);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java b/test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java
index fd13e334734c..f1251a4834cb 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java
@@ -36,11 +36,12 @@ public static void setupCluster()
     }
 
     @Test
-    public void shouldTolerateDuplicatedRowIDsAfterMemtableUpdates() throws Throwable
+    public void shouldTolerateDuplicatedRowIDsAfterMemtableUpdates()
     {
         createTable("CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT)");
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
 
+        // fill 2 bkd leaves
         for (int i = 0; i < 2048; ++i)
         {
             execute("INSERT INTO %s (id1, v1) VALUES (?, ?)", Integer.toString(i % 10), i);
@@ -50,6 +51,14 @@ public void shouldTolerateDuplicatedRowIDsAfterMemtableUpdates() throws Throwabl
         List<Row> rows = executeNet("SELECT * FROM %s WHERE v1 > 0").all();
         assertEquals(10, rows.size());
 
+        flush();
+
+        // tolerate duplicates from 1 sstable
+        // query will match both leaves, one entirely and one with filtering, as it contains a single entry with v1 == 0
+        rows = executeNet("SELECT * FROM %s WHERE v1 > 0").all();
+        assertEquals(10, rows.size());
+
+        // fill 2 bkd leaves again
         for (int i = 0; i < 2048; ++i)
         {
             execute("INSERT INTO %s (id1, v1) VALUES (?, ?)", Integer.toString(i % 10), i);
@@ -58,5 +67,11 @@ public void shouldTolerateDuplicatedRowIDsAfterMemtableUpdates() throws Throwabl
         // tolerate duplicates from memtable and sstable
         rows = executeNet("SELECT * FROM %s WHERE v1 > 0").all();
         assertEquals(10, rows.size());
+
+        flush();
+
+        // tolerate duplicates from 2 sstables
+        rows = executeNet("SELECT * FROM %s WHERE v1 > 0").all();
+        assertEquals(10, rows.size());
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/EmptyMemtableFlushTest.java b/test/unit/org/apache/cassandra/index/sai/cql/EmptyMemtableFlushTest.java
new file mode 100644
index 000000000000..d09b141a3de0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/EmptyMemtableFlushTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Set;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.io.util.File;
+
+import static org.junit.Assert.assertEquals;
+
+public class EmptyMemtableFlushTest extends SAITester
+{
+    @Test
+    public void numericIndexTest() throws Throwable
+    {
+        requireNetwork();
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val1 int, val2 int)");
+        IndexContext val1IndexContext = createIndexContext(createIndex("CREATE CUSTOM INDEX ON %s(val1) USING 'StorageAttachedIndex'"), Int32Type.instance);
+        IndexContext val2IndexContext = createIndexContext(createIndex("CREATE CUSTOM INDEX ON %s(val2) USING 'StorageAttachedIndex'"), Int32Type.instance);
+        execute("INSERT INTO %s (id, val1, val2) VALUES (0, 0, 0)");
+        execute("INSERT INTO %s (id, val2) VALUES (1, 1)");
+        execute("DELETE FROM %s WHERE id = 0");
+        flush();
+        // After this we should have only 1 set of index files but 2 completion markers
+        Set<File> indexFiles = indexFiles();
+        assertEquals(0, componentFiles(indexFiles, IndexComponentType.KD_TREE, val1IndexContext).size());
+        assertEquals(0, componentFiles(indexFiles, IndexComponentType.KD_TREE_POSTING_LISTS, val1IndexContext).size());
+        assertEquals(0, componentFiles(indexFiles, IndexComponentType.META, val1IndexContext).size());
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.COLUMN_COMPLETION_MARKER, val1IndexContext).size());
+
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.KD_TREE, val2IndexContext).size());
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.KD_TREE_POSTING_LISTS, val2IndexContext).size());
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.META, val2IndexContext).size());
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.COLUMN_COMPLETION_MARKER, val2IndexContext).size());
+
+        assertEquals(0, execute("SELECT * from %s WHERE val1 = 0").size());
+        assertEquals(1, execute("SELECT * from %s WHERE val2 = 1").size());
+
+        assertIndexFilesInToc(indexFiles);
+    }
+
+    @Test
+    public void literalIndexTest() throws Throwable
+    {
+        requireNetwork();
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val1 text, val2 text)");
+        IndexContext val1IndexContext = createIndexContext(createIndex("CREATE CUSTOM INDEX ON %s(val1) USING 'StorageAttachedIndex'"), UTF8Type.instance);
+        IndexContext val2IndexContext = createIndexContext(createIndex("CREATE CUSTOM INDEX ON %s(val2) USING 'StorageAttachedIndex'"), UTF8Type.instance);
+        execute("INSERT INTO %s (id, val1, val2) VALUES (0, '0', '0')");
+        execute("INSERT INTO %s (id, val2) VALUES (1, '1')");
+        execute("DELETE FROM %s WHERE id = 0");
+        flush();
+        // After this we should have only 1 set of index files but 2 completion markers
+        Set<File> indexFiles = indexFiles();
+        assertEquals(0, componentFiles(indexFiles, IndexComponentType.TERMS_DATA, val1IndexContext).size());
+        assertEquals(0, componentFiles(indexFiles, IndexComponentType.POSTING_LISTS, val1IndexContext).size());
+        assertEquals(0, componentFiles(indexFiles, IndexComponentType.META, val1IndexContext).size());
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.COLUMN_COMPLETION_MARKER, val1IndexContext).size());
+
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.TERMS_DATA, val2IndexContext).size());
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.POSTING_LISTS, val2IndexContext).size());
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.META, val2IndexContext).size());
+        assertEquals(1, componentFiles(indexFiles, IndexComponentType.COLUMN_COMPLETION_MARKER, val2IndexContext).size());
+
+        assertEquals(0, execute("SELECT * from %s WHERE val1 = '0'").size());
+        assertEquals(1, execute("SELECT * from %s WHERE val2 = '1'").size());
+
+        assertIndexFilesInToc(indexFiles);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/EmptyStringLifecycleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/EmptyStringLifecycleTest.java
index 7ace636e4f7a..94820b379843 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/EmptyStringLifecycleTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/EmptyStringLifecycleTest.java
@@ -76,4 +76,17 @@ public void testAfterCompaction()
         UntypedResultSet rows = execute("SELECT * FROM %s WHERE v = ''");
         assertRows(rows, row(1, ""), row(0, ""));
     }
+
+    @Test
+    public void testOrderBy()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        disableCompaction(KEYSPACE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, 'v'));
+
+        execute("INSERT INTO %s (k, v) VALUES (0, '')");
+
+        UntypedResultSet rows = execute("SELECT * FROM %s ORDER BY v LIMIT 10");
+        assertRows(rows, row(0, ""));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/EmptyValuesTest.java b/test/unit/org/apache/cassandra/index/sai/cql/EmptyValuesTest.java
new file mode 100644
index 000000000000..77982675a597
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/EmptyValuesTest.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.AbstractTypeGenerators;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.EMPTY_BYTE_BUFFER;
+import static org.quicktheories.QuickTheory.qt;
+
+/**
+ * Tests that empty values are only indexed for literal indexes. See CNDB-12774 for more details.
+ */
+public class EmptyValuesTest extends SAITester
+{
+    @Test
+    public void testEmptyValues()
+    {
+        qt().forAll(AbstractTypeGenerators.primitiveTypeGen()).checkAssert(type -> {
+
+            CQL3Type cql3Type = type.asCQL3Type();
+            if (type.allowsEmpty() && StorageAttachedIndex.SUPPORTED_TYPES.contains(cql3Type))
+            {
+                testEmptyValues(cql3Type);
+            }
+        });
+    }
+
+    private void testEmptyValues(CQL3Type type)
+    {
+        createTable(String.format("CREATE TABLE %%s (k int PRIMARY KEY, v %s)", type));
+        execute("INSERT INTO %s (k, v) VALUES (0, ?)", EMPTY_BYTE_BUFFER);
+        flush();
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, 'v'));
+
+        boolean indexed = TypeUtil.isLiteral(type.getType());
+
+        Assertions.assertThat(execute("SELECT * FROM %s WHERE v = ?", EMPTY_BYTE_BUFFER)).hasSize(indexed ? 1 : 0);
+
+        execute("INSERT INTO %s (k, v) VALUES (1, ?)", EMPTY_BYTE_BUFFER);
+        Assertions.assertThat(execute("SELECT * FROM %s WHERE v = ?", EMPTY_BYTE_BUFFER)).hasSize(indexed ? 2 : 0);
+
+        flush();
+        Assertions.assertThat(execute("SELECT * FROM %s WHERE v = ?", EMPTY_BYTE_BUFFER)).hasSize(indexed ? 2 : 0);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByInvalidQueryTest.java b/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByInvalidQueryTest.java
new file mode 100644
index 000000000000..d7020ab7f98c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByInvalidQueryTest.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Collections;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.ResultSet;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static org.apache.cassandra.cql3.restrictions.StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class GenericOrderByInvalidQueryTest extends SAITester
+{
+    @BeforeClass
+    public static void setupClass()
+    {
+        requireNetwork();
+    }
+
+    @Test
+    public void cannotOrderVarintColumn()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, val varint)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        executeOrderByAndAssertInvalidRequestException("varint");
+    }
+
+    @Test
+    public void cannotOrderWithAnalyzedIndex()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, val text)");
+        createIndex("CREATE CUSTOM INDEX test_v1_idx ON %s(val) USING 'StorageAttachedIndex'" +
+                    " WITH OPTIONS = {'index_analyzer': '{\"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }}'}");
+
+        execute("INSERT INTO %s (pk, val) VALUES (1, 'ciao amico')");
+        execute("INSERT INTO %s (pk, val) VALUES (2, 'ciao amico')");
+        assertRows(execute("SELECT * FROM %s"), row(1, "ciao amico"), row(2, "ciao amico"));
+
+        // Verify ORDER BY fails with analyzed index
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY val LIMIT 10"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(String.format(NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE, "val"));
+
+        // Verify ORDER BY works with non-analyzed index
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        assertRows(execute("SELECT * FROM %s ORDER BY val LIMIT 10"), row(1, "ciao amico"), row(2, "ciao amico"));
+    }
+
+    @Test
+    public void cannotOrderDecimalColumn()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, val decimal)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        executeOrderByAndAssertInvalidRequestException("decimal");
+    }
+
+    private void executeOrderByAndAssertInvalidRequestException(String cqlType)
+    {
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY val ASC LIMIT 1"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("SAI based ordering on column val of type " + cqlType + " is not supported");
+    }
+
+    @Test
+    public void cannotOrderTextColumnWithoutIndex()
+    {
+        createTable("CREATE TABLE %s (pk int, val text, PRIMARY KEY(pk))");
+
+        assertInvalidMessage(String.format(NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE, "val"),
+                             "SELECT * FROM %s ORDER BY val ASC LIMIT 1");
+        // Also confirm filtering does not make it work.
+        assertInvalidMessage(String.format(NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE, "val"),
+                             "SELECT * FROM %s ORDER BY val LIMIT 5 ALLOW FILTERING");
+    }
+
+    @Test
+    public void testTextOrderingIsNotAllowedWithClusteringOrdering()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, val text, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        assertInvalidMessage("Cannot combine clustering column ordering with non-clustering column ordering",
+                             "SELECT * FROM %s ORDER BY val, ck ASC LIMIT 2");
+    }
+
+    @Test
+    public void textOrderingMustHaveLimit()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        assertInvalidMessage("SAI based ORDER BY clause requires a LIMIT that is not greater than 1000. LIMIT was NO LIMIT",
+                             "SELECT * FROM %s ORDER BY val");
+
+    }
+
+    @Test
+    public void testInvalidColumnName()
+    {
+        String table = createTable(KEYSPACE, "CREATE TABLE %s (k int, c int, v int, primary key (k, c))");
+        assertInvalidMessage(String.format("Undefined column name bad_col in table %s", KEYSPACE + '.' + table),
+                             "SELECT k from %s ORDER BY bad_col LIMIT 1");
+    }
+
+    @Test
+    public void disallowClusteringColumnPredicateWithoutSupportingIndex()
+    {
+        createTable("CREATE TABLE %s (pk int, num int, v text, PRIMARY KEY(pk, num))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+        execute("INSERT INTO %s (pk, num, v) VALUES (3, 1, 'a')");
+        execute("INSERT INTO %s (pk, num, v) VALUES (3, 4, 'b')");
+        flush();
+
+        // If we didn't have the query planner fail this query, we would get incorrect results for both queries
+        // because the clustering columns are not yet available to restrict the ORDER BY result set.
+        assertThatThrownBy(() -> execute("SELECT num FROM %s WHERE pk=3 AND num > 3 ORDER BY v LIMIT 1"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
+
+        assertThatThrownBy(() -> execute("SELECT num FROM %s WHERE pk=3 AND num = 4 ORDER BY v LIMIT 1"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
+
+        // Cover the alternative code path
+        createIndex("CREATE CUSTOM INDEX ON %s(num) USING 'StorageAttachedIndex'");
+        assertRows(execute("SELECT num FROM %s WHERE pk=3 AND num > 3 ORDER BY v LIMIT 1"), row(4));
+    }
+
+    @Test
+    public void canOnlyExecuteWithCorrectConsistencyLevel()
+    {
+        createTable("CREATE TABLE %s (k int primary key, c int, v text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (k, c, v) VALUES (1, 1, 'a')");
+        execute("INSERT INTO %s (k, c, v) VALUES (2, 2, 'b')");
+        execute("INSERT INTO %s (k, c, v) VALUES (3, 3, 'c')");
+
+        executeConsistencyLevelQueries("c");
+        executeConsistencyLevelQueries("v");
+    }
+
+    private void executeConsistencyLevelQueries(String column)
+    {
+        var query = String.format("SELECT * FROM %%s ORDER BY %s LIMIT 3", column);
+        ClientWarn.instance.captureWarnings();
+        execute(query);
+        ResultSet result = execute(query, ConsistencyLevel.ONE);
+        assertEquals(3, result.size());
+        assertNull(ClientWarn.instance.getWarnings());
+
+        result = execute(query, ConsistencyLevel.LOCAL_ONE);
+        assertEquals(3, result.size());
+        assertNull(ClientWarn.instance.getWarnings());
+
+        result = execute(query, ConsistencyLevel.QUORUM);
+        assertEquals(3, result.size());
+        assertEquals(1, ClientWarn.instance.getWarnings().size());
+        assertEquals(String.format(SelectStatement.TOPK_CONSISTENCY_LEVEL_WARNING, ConsistencyLevel.QUORUM, ConsistencyLevel.ONE),
+                     ClientWarn.instance.getWarnings().get(0));
+
+        ClientWarn.instance.captureWarnings();
+        result = execute(query, ConsistencyLevel.LOCAL_QUORUM);
+        assertEquals(3, result.size());
+        assertEquals(1, ClientWarn.instance.getWarnings().size());
+        assertEquals(String.format(SelectStatement.TOPK_CONSISTENCY_LEVEL_WARNING, ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.LOCAL_ONE),
+                     ClientWarn.instance.getWarnings().get(0));
+
+        assertThatThrownBy(() -> execute(query, ConsistencyLevel.SERIAL))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(String.format(SelectStatement.TOPK_CONSISTENCY_LEVEL_ERROR, ConsistencyLevel.SERIAL));
+
+        assertThatThrownBy(() -> execute(query, ConsistencyLevel.LOCAL_SERIAL))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(String.format(SelectStatement.TOPK_CONSISTENCY_LEVEL_ERROR, ConsistencyLevel.LOCAL_SERIAL));
+    }
+
+    protected ResultSet execute(String query, ConsistencyLevel consistencyLevel)
+    {
+        ClientState state = ClientState.forInternalCalls();
+        QueryState queryState = new QueryState(state);
+
+        CQLStatement statement = QueryProcessor.parseStatement(formatQuery(query), queryState.getClientState());
+        statement.validate(queryState.getClientState());
+
+        QueryOptions options = QueryOptions.forInternalCalls(Collections.emptyList());
+        options.updateConsistency(consistencyLevel);
+
+        return ((ResultMessage.Rows)statement.execute(queryState, options, Dispatcher.RequestTime.forImmediateExecution())).result;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByTest.java b/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByTest.java
new file mode 100644
index 000000000000..4bf43bf1571a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByTest.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.TreeMap;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.plan.QueryController;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class GenericOrderByTest extends SAITester
+{
+    @Test
+    public void testOrderingAcrossManySstables()
+    {
+        // Disable query optimizer to prevent skipping hybrid query logic.
+        QueryController.QUERY_OPT_LEVEL = 0;
+        // We don't want our sstables getting compacted away
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, val int, str_val ascii)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        disableCompaction();
+
+        var expectedResults = new TreeMap<String, Integer>();
+
+        // Put the first and last ones in first to put them in sstables to guarantee we hit each for ASC and DESC, respectively.
+        execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", -1, -1, "AA");
+        expectedResults.put("AA", -1);
+        execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", -2, -2, "zz");
+        expectedResults.put("zz", -2);
+
+        for (int i = 0; i < 200; i++)
+        {
+            // Use ascii because its ordering works the way we expect.
+            var str = getRandom().nextAsciiString(10, 30);
+            execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", i, i, str);
+            expectedResults.put(str, i);
+            if (getRandom().nextIntBetween(0, 100) < 2)
+                flush();
+        }
+
+        // Put the first and last ones in a memtable to guarantee we hit each for ASC and DESC, respectively.
+        execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", -3, -1, "A");
+        expectedResults.put("A", -3);
+        execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", -4, -2, "z");
+        expectedResults.put("z", -4);
+
+        assertRows(execute("SELECT pk FROM %s ORDER BY str_val ASC LIMIT 1"),
+                   expectedResults.values().stream().map(CQLTester::row).limit(1).toArray(Object[][]::new));
+        assertRows(execute("SELECT pk FROM %s WHERE val < 15 ORDER BY str_val ASC LIMIT 1"),
+                   expectedResults.values().stream().filter(x -> x < 15).map(CQLTester::row).limit(1).toArray(Object[][]::new));
+
+        assertRows(execute("SELECT pk FROM %s ORDER BY str_val DESC LIMIT 1"),
+                   expectedResults.descendingMap().values().stream().map(CQLTester::row).limit(1).toArray(Object[][]::new));
+        assertRows(execute("SELECT pk FROM %s WHERE val < 15 ORDER BY str_val DESC LIMIT 1"),
+                   expectedResults.descendingMap().values().stream().filter(x -> x < 15).map(CQLTester::row).limit(1).toArray(Object[][]::new));
+    }
+
+    @Test
+    public void testOrderingAcrossMemtableAndSSTable() throws Throwable
+    {
+        QueryController.QUERY_OPT_LEVEL = 0;
+        // We don't want our sstables getting compacted away
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, val int, str_val ascii)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        disableCompaction();
+
+        // 'A' will be first
+        execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", -1, -1, "A");
+        execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", -2, -2, "z");
+
+        flush();
+
+        // 'zz' will be last
+        execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", -3, -3, "AA");
+        execute("INSERT INTO %s (pk, val, str_val) VALUES (?, ?, ?)", -4, -4, "zz");
+
+        beforeAndAfterFlush(() -> {
+            // Query for limit 1 to make sure that we don't reorder the results later on in the stack.
+            // This test verifies the correctness of the text encoding.
+            assertRows(execute("SELECT pk FROM %s ORDER BY str_val ASC LIMIT 1"), row(-1));
+            assertRows(execute("SELECT pk FROM %s WHERE val < 0 ORDER BY str_val ASC LIMIT 1"), row(-1));
+
+            assertRows(execute("SELECT pk FROM %s ORDER BY str_val DESC LIMIT 1"), row(-4));
+            assertRows(execute("SELECT pk FROM %s WHERE val < 0 ORDER BY str_val DESC LIMIT 1"), row(-4));
+        });
+    }
+
+    @Test
+    public void testPrimaryKeyRestrictionToEnsureBoundsAreCorrectlyHandled() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, x int, val int, str_val ascii)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+
+        // Insert many rows and then ensure we can get each of them when querying with specific bounds.
+        for (int i = 0; i < 100; i++)
+            execute("INSERT INTO %s (pk, x, val, str_val) VALUES (?, ?, ?, ?)", i, i, i, i);
+
+        // Test caught a bug in the way we created boundaries.
+        beforeAndAfterFlush(() -> {
+            for (int i = 0; i < 100; i++)
+            {
+                assertRows(execute("SELECT pk FROM %s WHERE pk = ? ORDER BY str_val ASC LIMIT 1", i), row(i));
+                assertRows(execute("SELECT pk FROM %s WHERE pk = ? ORDER BY val ASC LIMIT 1", i), row(i));
+                assertRows(execute("SELECT pk FROM %s WHERE pk = ? ORDER BY str_val DESC LIMIT 1", i), row(i));
+                assertRows(execute("SELECT pk FROM %s WHERE pk = ? ORDER BY val DESC LIMIT 1", i), row(i));
+            }
+        });
+    }
+
+    @Test
+    public void testMultiplePrimaryKeysForSameTerm() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, x int, val int, str_val ascii, PRIMARY KEY (pk, x))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+
+        // We use a primary key with the same partition column value to ensure it goes to the same shard in the
+        // memtable, which reproduces a bug we hit.
+        execute("INSERT INTO %s (pk, x, val, str_val) VALUES (?, ?, ?, ?)", 1, 1, 1, "A");
+        execute("INSERT INTO %s (pk, x, val, str_val) VALUES (?, ?, ?, ?)", 1, 2, 1, "A");
+        // Goes to a different shard in the memtable
+        execute("INSERT INTO %s (pk, x, val, str_val) VALUES (?, ?, ?, ?)", 2, 3, 2, "B");
+
+        beforeAndAfterFlush(() -> {
+            // Literal order by
+            assertRows(execute("SELECT x FROM %s ORDER BY str_val ASC LIMIT 2"), row(1), row(2));
+            assertRows(execute("SELECT x FROM %s ORDER BY str_val DESC LIMIT 1"), row(3));
+            assertRows(execute("SELECT x FROM %s WHERE val = 1 ORDER BY str_val ASC LIMIT 2"), row(1), row(2));
+            // Numeric order by
+            assertRows(execute("SELECT x FROM %s ORDER BY val ASC LIMIT 2"), row(1), row(2));
+            assertRows(execute("SELECT x FROM %s ORDER BY val DESC LIMIT 1"), row(3));
+            assertRows(execute("SELECT x FROM %s WHERE str_val = 'A' ORDER BY val ASC LIMIT 2"), row(1), row(2));
+        });
+    }
+
+    @Test
+    public void testSelectionAndOrderByOnTheSameColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, x int, v int, PRIMARY KEY (pk, x))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", 1, 1, 1);
+        execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", 1, 2, 5);
+        execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", 1, 3, 2);
+        execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", 1, 4, 4);
+        execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", 2, 1, 7);
+        execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", 2, 2, 6);
+        execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", 2, 3, 8);
+        execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", 2, 4, 3);
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT v FROM %s WHERE v >= -10 ORDER BY v ASC LIMIT 4"), row(1), row(2), row(3), row(4));
+            assertRows(execute("SELECT v FROM %s WHERE v > 1 ORDER BY v ASC LIMIT 4"), row(2), row(3), row(4), row(5));
+            assertRows(execute("SELECT v FROM %s WHERE v <= 3 ORDER BY v ASC LIMIT 4"), row(1), row(2), row(3));
+            assertRows(execute("SELECT v FROM %s WHERE v >= 4 AND v <= 6 ORDER BY v ASC LIMIT 4"), row(4), row(5), row(6));
+            assertRows(execute("SELECT v FROM %s WHERE v >= 7 ORDER BY v ASC LIMIT 4"), row(7), row(8));
+            assertRows(execute("SELECT v FROM %s WHERE v >= 10 ORDER BY v ASC LIMIT 4"));
+
+            assertRows(execute("SELECT v FROM %s WHERE v >= -10 ORDER BY v DESC LIMIT 4"), row(8), row(7), row(6), row(5));
+            assertRows(execute("SELECT v FROM %s WHERE v > 1 ORDER BY v DESC LIMIT 4"), row(8), row(7), row(6), row(5));
+            assertRows(execute("SELECT v FROM %s WHERE v <= 3 ORDER BY v DESC LIMIT 4"), row(3), row(2), row(1));
+            assertRows(execute("SELECT v FROM %s WHERE v >= 4 AND v <= 6 ORDER BY v DESC LIMIT 4"), row(6), row(5), row(4));
+            assertRows(execute("SELECT v FROM %s WHERE v >= 7 ORDER BY v DESC LIMIT 4"), row(8), row(7));
+            assertRows(execute("SELECT v FROM %s WHERE v >= 10 ORDER BY v DESC LIMIT 4"));
+        });
+    }
+
+    private void testSelectionAndOrderByOnTheSameColumnWithLargeRowCount(boolean asc) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, x int, v int, PRIMARY KEY (pk, x))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        Object[][] rows = new Object[100][];
+        var lowerBound = asc ? 0 : 4900;
+        var upperBound = asc ? 100 : 5000;
+        for (int i = 0; i < 10000; i++)
+        {
+            execute("INSERT INTO %s (pk, x, v) VALUES (?, ?, ?)", i, i, i);
+            if (i >= lowerBound && i < upperBound)
+            {
+                var pos = asc ? i - lowerBound : upperBound - i - 1;
+                rows[pos] = row(i);
+            }
+        }
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT v FROM %s WHERE v < 5000 ORDER BY v " + (asc ? "ASC" : "DESC") + " LIMIT 100"), rows);
+        });
+    }
+
+
+    /*
+     * The following two tests show that we can correctly select and order by a column for which the table contains
+     * sufficient rows to stress ranges within the backing data structure (e.g., BKDReader spanning multiple leaves).
+     */
+    @Test
+    public void testSelectionAndOrderByOnTheSameColumnWithLargeRowCountAsc() throws Throwable
+    {
+        testSelectionAndOrderByOnTheSameColumnWithLargeRowCount(true);
+    }
+
+    @Test
+    public void testSelectionAndOrderByOnTheSameColumnWithLargeRowCountDesc() throws Throwable
+    {
+        testSelectionAndOrderByOnTheSameColumnWithLargeRowCount(false);
+    }
+
+    @Test
+    public void cannotHaveAggregationOnOrderByQuery()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (k, v) VALUES (1, 4)");
+        execute("INSERT INTO %s (k, v) VALUES (2, 3)");
+        execute("INSERT INTO %s (k, v) VALUES (3, 2)");
+        execute("INSERT INTO %s (k, v) VALUES (4, 1)");
+
+        assertThatThrownBy(() -> execute("SELECT sum(v) FROM %s ORDER BY v LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
+
+        assertThatThrownBy(() -> execute("SELECT sum(v) FROM %s WHERE k = 1 ORDER BY v LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
+
+        assertThatThrownBy(() -> execute("SELECT * FROM %s GROUP BY k ORDER BY v LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
+
+        assertThatThrownBy(() -> execute("SELECT count(*) FROM %s ORDER BY v LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByUpdateDeleteTest.java b/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByUpdateDeleteTest.java
new file mode 100644
index 000000000000..3219c39054a0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/GenericOrderByUpdateDeleteTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.plan.QueryController;
+
+public class GenericOrderByUpdateDeleteTest extends SAITester
+{
+
+    @Before
+    public void setup() throws Throwable
+    {
+        // Enable the optimizer by default. If there are any tests that need to disable it, they can do so explicitly.
+        QueryController.QUERY_OPT_LEVEL = 1;
+    }
+
+    @Test
+    public void testPreparedQueries() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, val int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // Insert some data
+        for (int i = -100; i < 100; i++)
+            execute("INSERT INTO %s (pk, val) VALUES (?, ?)", i, i);
+
+        forcePreparedValues();
+        var query1 = "SELECT pk FROM %s WHERE val > ? ORDER BY val ASC LIMIT 4";
+        var query2 = "SELECT pk FROM %s WHERE val > ? ORDER BY val DESC LIMIT 4";
+        var query3 = "SELECT pk FROM %s ORDER BY val ASC LIMIT 4";
+        var query4 = "SELECT pk FROM %s ORDER BY val DESC LIMIT 4";
+        prepare(query1);
+        prepare(query2);
+        prepare(query3);
+        prepare(query4);
+
+        assertRows(execute(query1, 0), row(1), row(2), row(3), row(4));
+        assertRows(execute(query2, 0), row(99), row(98), row(97), row(96));
+        assertRows(execute(query3), row(-100), row(-99), row(-98), row(-97));
+        assertRows(execute(query4), row(99), row(98), row(97), row(96));
+    }
+
+    @Test
+    public void testOrderingWhereRowHasComplexColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, val int, m map<text, text>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, val, m) VALUES (?, ?, ?)", 1, 1, map("a", "b"));
+        execute("INSERT INTO %s (pk, val, m) VALUES (?, ?, ?)", 2, 2, map("a", "b"));
+        flush();
+
+        // Add more data to pk 1 in another sstable. This triggers merging cells in the RowWithSourceTable.
+        execute("INSERT INTO %s (pk, m) VALUES (?, ?)", 1, map("b", "c"));
+
+        beforeAndAfterFlush(() -> {
+            assertRowsInBothOrder("SELECT pk FROM %s ORDER BY val", 2, row(1), row(2));
+        });
+    }
+
+    @Test
+    public void endToEndTextOrderingTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, str_val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, str_val) VALUES (0, 'a')");
+        execute("INSERT INTO %s (pk, str_val) VALUES (1, 'z')");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsInBothOrder("SELECT pk FROM %s ORDER BY str_val", 10, row(0), row(1));
+        });
+
+        execute("INSERT INTO %s (pk, str_val) VALUES (2, 'b')");
+        execute("INSERT INTO %s (pk, str_val) VALUES (3, 'A')");
+
+        // Now we get memtable + sstable and then two sstables in the query, which confirms merging index results.
+        beforeAndAfterFlush(() -> {
+            assertRowsInBothOrder("SELECT pk FROM %s ORDER BY str_val", 4,
+                                  row(3), row(0), row(2), row(1));
+        });
+    }
+
+    @Test
+    public void endToEndIntOrderingTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, str_val int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, str_val) VALUES (0, -10)");
+        execute("INSERT INTO %s (pk, str_val) VALUES (1, 0)");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsInBothOrder("SELECT pk FROM %s ORDER BY str_val", 10, row(0), row(1));
+        });
+
+        execute("INSERT INTO %s (pk, str_val) VALUES (2, -5)");
+        execute("INSERT INTO %s (pk, str_val) VALUES (3, 100)");
+
+        // Now we get memtable + sstable and then two sstables in the query, which confirms merging index results.
+        beforeAndAfterFlush(() -> {
+            assertRowsInBothOrder("SELECT pk FROM %s ORDER BY str_val", 4,
+                                  row(0), row(2), row(1), row(3));
+        });
+    }
+
+    @Test
+    public void testTextOverwrittenRowsInDifferentMemtableOrSSTable() throws Throwable
+    {
+        testTextOverwritten(true);
+    }
+
+    @Test
+    public void testTextOverwrittenRowsInSameMemtableOrSSTable() throws Throwable
+    {
+        testTextOverwritten(false);
+    }
+
+    private void testTextOverwritten(boolean shouldFlush) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, str_val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        disableCompaction();
+
+        execute("INSERT INTO %s (pk, str_val) VALUES (0, 'a')");
+        execute("INSERT INTO %s (pk, str_val) VALUES (1, 'b')");
+        if (shouldFlush)
+            flush();
+        execute("INSERT INTO %s (pk, str_val) VALUES (0, 'c')");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsInBothOrder("SELECT pk FROM %s ORDER BY str_val", 10, row(1), row(0));
+        });
+    }
+
+    @Test
+    public void testIntOverwrittenRowsInDifferentMemtableOrSSTable() throws Throwable
+    {
+        testIntOverwritten(true);
+    }
+
+    @Test
+    public void testIntOverwrittenRowsInSameMemtableOrSSTable() throws Throwable
+    {
+        testIntOverwritten(false);
+    }
+
+    private void testIntOverwritten(boolean shouldFlush) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, str_val int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, str_val) VALUES (0, 1)");
+        execute("INSERT INTO %s (pk, str_val) VALUES (1, 2)");
+        if (shouldFlush)
+            flush();
+        execute("INSERT INTO %s (pk, str_val) VALUES (0, 3)");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsInBothOrder("SELECT pk FROM %s ORDER BY str_val", 10, row(1), row(0));
+        });
+    }
+
+    private void assertRowsInBothOrder(String query, int limit, Object[]... rowsInAscendingOrder)
+    {
+        assertRows(execute(query + " ASC LIMIT " + limit), rowsInAscendingOrder);
+        assertRows(execute(query + " DESC LIMIT " + limit), reverse(rowsInAscendingOrder));
+    }
+
+    private static Object[][] reverse(Object[][] rows)
+    {
+        Object[][] reversed = new Object[rows.length][];
+        for (int i = 0; i < rows.length; i++)
+        {
+            reversed[i] = rows[rows.length - i - 1];
+        }
+        return reversed;
+    }
+
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceAccuracyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceAccuracyTest.java
new file mode 100644
index 000000000000..6fcb976582c2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceAccuracyTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.utils.Pair;
+import org.apache.lucene.geo.GeoUtils;
+import org.apache.lucene.util.SloppyMath;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertTrue;
+
+public class GeoDistanceAccuracyTest extends VectorTester
+{
+    private static final Logger logger = LoggerFactory.getLogger(GeoDistanceAccuracyTest.class.getName());
+
+    // Number represents the number of results that are within the search radius divided by the number of expected results
+    // Note that this recall number is just for random vectors in a box around NYC. These vectors might not
+    // be representative of real data, so this test mostly serves to verify the status quo.
+    private final static float MIN_EXPECTED_RECALL = 0.85f;
+
+    // Number represents the percent of actual results that are incorrect (i.e. outside the search radius)
+    private final static float MAX_EXPECTED_FALSE_POSITIVE_RATE = 0.0001f;
+
+    @Test
+    public void testRandomVectorsAgainstHaversineDistance()
+    {
+        createTable("CREATE TABLE %s (pk int, val vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        int numVectors = 20000;
+        var vectors = IntStream.range(0, numVectors).mapToObj(s -> Pair.create(s, createRandomNYCVector())).collect(Collectors.toList());
+
+        // Insert the vectors
+        for (var vector : vectors)
+            execute("INSERT INTO %s (pk, val) VALUES (?, ?)", vector.left, vector(vector.right));
+
+        double recallRate = 0;
+        double falsePositiveRate = 0;
+        int queryCount = 100;
+        for (int i = 0; i < queryCount; i++)
+        {
+            var searchVector = createRandomNYCVector();
+            // Pick a random distance between 1km and 10km
+            var distanceInMeters = getRandom().nextIntBetween(1000, 10000);
+            // Get the "correct" results using the great circle distance or the haversine formula
+            var closeVectors = vectors.stream()
+                                      .filter(v -> isWithinDistance(v.right, searchVector, distanceInMeters))
+                                      .collect(Collectors.toList());
+
+            var results = execute("SELECT pk FROM %s WHERE GEO_DISTANCE(val, ?) < " + distanceInMeters, vector(searchVector));
+
+            // Get the collection of expected rows. This uses a more expensive and more correct haversine distance
+            // formula, which is why we calculate the false-positive ratio as well.
+            var expected = closeVectors.stream().map(v -> v.left).collect(Collectors.toSet());
+            var actual = results.stream().map(r -> r.getInt("pk")).collect(Collectors.toSet());
+            var incorrectMatches = actual.stream().filter(pk -> !expected.contains(pk)).count();
+            var correctMatches = actual.size() - incorrectMatches;
+
+            if (!expected.isEmpty())
+                recallRate += correctMatches / (double) expected.size();
+            else if (actual.isEmpty())
+                // We expected none (recall was empty) and we got none, so that is 100% recall. If we got some,
+                // recall is 0%, which is a no-op here.
+                recallRate += 1;
+
+            // If actual is empty, then we have no false positives, so this is a no-op.
+            if (!actual.isEmpty())
+                falsePositiveRate += incorrectMatches / (double) actual.size();
+        }
+        double observedRecall = recallRate / queryCount;
+        double observedFalsePositiveAccuracy = falsePositiveRate / queryCount;
+        logger.info("Observed recall rate: {}", observedRecall);
+        logger.info("Observed false positive rate: {}", observedFalsePositiveAccuracy);
+        assertTrue("Recall should be greater than " + MIN_EXPECTED_RECALL + " but found " + observedRecall,
+                   observedRecall > MIN_EXPECTED_RECALL);
+        assertTrue("False positive rate should be less than " + MAX_EXPECTED_FALSE_POSITIVE_RATE + " but found " + observedFalsePositiveAccuracy,
+                   observedFalsePositiveAccuracy < MAX_EXPECTED_FALSE_POSITIVE_RATE);
+    }
+
+    @Test
+    public void haversineBenchmark()
+    {
+        // Run 1 million iterations
+        int iterations = 1000000;
+        double strictHaversineDuration = 0;
+        double sloppyHaversineDuration = 0;
+        for (int i = 0; i < iterations; i++)
+        {
+            float searchLat = SAITester.getRandom().nextFloatBetween(-90, 90);
+            float searchLon = SAITester.getRandom().nextFloatBetween(-180, 180);
+            float pointLat = SAITester.getRandom().nextFloatBetween(-90, 90);
+            float pointLon = SAITester.getRandom().nextFloatBetween(-180, 180);
+
+            // Get the haversine distance using a strict algorithm
+            var nowStrict = System.nanoTime();
+            GeoDistanceAccuracyTest.strictHaversineDistance(searchLat, searchLon, pointLat, pointLon);
+            strictHaversineDuration += System.nanoTime() - nowStrict;
+
+            // Calculate the sloppy distance (the one used in the code)
+            var nowSloppy = System.nanoTime();
+            SloppyMath.haversinMeters(searchLat, searchLon, pointLat, pointLon);
+            sloppyHaversineDuration += System.nanoTime() - nowSloppy;
+        }
+
+        double strictHaversineAverage = strictHaversineDuration / iterations;
+        double sloppyHaversineAverage = sloppyHaversineDuration / iterations;
+        logger.info("Average duration for strict haversine: " + strictHaversineAverage);
+        logger.info("Average duration for sloppy haversine: " + sloppyHaversineAverage);
+        assertTrue("Sloppy haversine distance should be at least as fast as strict haversine distance.",
+                   sloppyHaversineAverage <= strictHaversineAverage);
+    }
+
+    public static boolean isWithinDistance(float[] vector, float[] searchVector, float distanceInMeters)
+    {
+        return strictHaversineDistance(vector[0], vector[1], searchVector[0], searchVector[1]) < distanceInMeters;
+    }
+
+    private float[] createRandomNYCVector()
+    {
+        // Approximate bounding box for contiguous NYC locations
+        var lat = getRandom().nextFloatBetween(39, 41);
+        var lon = getRandom().nextFloatBetween(-74, -72);
+        return new float[] {lat, lon};
+    }
+
+    // In the production code, we use a haversine distance formula from lucene, which prioritizes speed over some
+    // accuracy. This is the strict formula.
+    private static double strictHaversineDistance(float lat1, float lon1, float lat2, float lon2)
+    {
+        // This implementation is based on information from https://www.movable-type.co.uk/scripts/latlong.html
+        double phi1 = lat1 * Math.PI/180; // phi, lambda in radians
+        double phi2 = lat2 * Math.PI/180;
+        double deltaPhi = (lat2 - lat1) * Math.PI/180;
+        double deltaLambda = (lon2 - lon1) * Math.PI/180;
+
+        double a = Math.sin(deltaPhi / 2.0) * Math.sin(deltaPhi / 2.0) +
+                   Math.cos(phi1) * Math.cos(phi2) *
+                   Math.sin(deltaLambda / 2.0) * Math.sin(deltaLambda / 2.0);
+        double c = 2.0 * Math.atan2(Math.sqrt(a), Math.sqrt(1.0 - a));
+
+        return GeoUtils.EARTH_MEAN_RADIUS_METERS * c; // in meters
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceInvalidQueryTest.java b/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceInvalidQueryTest.java
new file mode 100644
index 000000000000..65345a3813ed
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceInvalidQueryTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.SyntaxException;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class GeoDistanceInvalidQueryTest extends VectorTester
+{
+    @Test
+    public void geoDistanceRequiresSearchVectorSizeTwo()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [5]) < 1000"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Invalid vector literal for v of type vector<float, 2>; expected 2 elements, but given 1");
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1, 1]) < 1000"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Invalid vector literal for v of type vector<float, 2>; expected 2 elements, but given 3");
+    }
+
+    @Test
+    public void geoDistanceRequiresVectorIndexSizeTwo()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        // Even though the search vector size is 2, the index vector size is not 2, so the query is not valid.
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1]) < 1000"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE is only supported against vector<float, 2> columns");
+
+        // Even though the search vector matches the index vector size, the index vector size is not 2, so the query is
+        // not valid.
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1, 1]) < 1000"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE is only supported against vector<float, 2> columns");
+    }
+
+    @Test
+    public void geoDistanceRequiresPositiveSearchRadius()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1]) < 0"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE radius must be positive, got 0.0");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1]) <= 0"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE radius must be positive, got 0.0");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1]) < -1"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE radius must be positive, got -1.0");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1]) <= -1"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE radius must be positive, got -1.0");
+    }
+
+    @Test
+    public void geoDistanceRequiresValidLatLonPositions()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [-90.1, 1]) < 100"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE latitude must be between -90 and 90 degrees, got -90.1");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [90.1, 0]) < 100"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE latitude must be between -90 and 90 degrees, got 90.1");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0, 180.1]) < 100"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE longitude must be between -180 and 180 degrees, got 180.1");
+
+        assertThatThrownBy(() -> execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0, -180.1]) < 100"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("GEO_DISTANCE longitude must be between -180 and 180 degrees, got -180.1");
+    }
+
+    @Test
+    public void geoDistanceMissingOrIncorrectlyConfiguredIndex()
+    {
+        createTable("CREATE TABLE %s (pk int, x int, v vector<float, 2>, PRIMARY KEY(pk))");
+
+        // Query without index
+        assertThatThrownBy(() -> execute( "SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1]) < 1000"))
+        .hasMessage(StatementRestrictions.GEO_DISTANCE_REQUIRES_INDEX_MESSAGE)
+        .isInstanceOf(InvalidRequestException.class);
+
+        // Intentionally create index with incorrect similarity function
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        // Query with incorrectly configured index
+        assertThatThrownBy(() -> execute( "SELECT pk FROM %s WHERE GEO_DISTANCE(v, [1, 1]) < 1000"))
+        .hasMessage(StatementRestrictions.VECTOR_INDEX_PRESENT_NOT_SUPPORT_GEO_DISTANCE_MESSAGE)
+        .isInstanceOf(InvalidRequestException.class);
+    }
+
+    @Test
+    public void geoDistanceUsageInIfClause()
+    {
+        createTable("CREATE TABLE %s (pk int, x int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        // GEO_DISTANCE is not parsable at this part of the CQL
+        assertThatThrownBy(() -> execute("UPDATE %s SET x = 100 WHERE pk = 1 IF GEO_DISTANCE(v, [1, 1]) < 1000"))
+        .isInstanceOf(SyntaxException.class);
+    }
+
+    @Test
+    public void geoDistanceWithANNRequiresAllSearchColumnsIndexed()
+    {
+        createTable("CREATE TABLE %s (pk int, x int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        // All of these queries fail with and without filtering
+        assertThatThrownBy(() -> execute("select pk from %s WHERE pk > 4 AND geo_distance(v,[5,5]) <= 1000000 ORDER BY v ANN of [5,5] limit 3"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
+
+        assertThatThrownBy(() -> execute("select pk from %s WHERE pk > 4 AND geo_distance(v,[5,5]) <= 1000000 ORDER BY v ANN of [5,5] limit 3 ALLOW FILTERING"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
+
+        assertThatThrownBy(() -> execute("select pk from %s WHERE x > 4 AND geo_distance(v,[5,5]) <= 1000000 ORDER BY v ANN of [5,5] limit 3"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
+
+        assertThatThrownBy(() -> execute("select pk from %s WHERE x > 4 AND geo_distance(v,[5,5]) <= 1000000 ORDER BY v ANN of [5,5] limit 3 ALLOW FILTERING"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
+
+        assertThatThrownBy(() -> execute("select pk from %s WHERE geo_distance(v,[5,5]) <= 1000000 AND v = [5,5] ORDER BY v ANN of [5,5] limit 3"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("v cannot be restricted by both BOUNDED_ANN and EQ");
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceRestrictionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceRestrictionTest.java
new file mode 100644
index 000000000000..94eab4107255
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/GeoDistanceRestrictionTest.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+public class GeoDistanceRestrictionTest extends VectorTester
+{
+    @Test
+    public void testBasicGeoDistanceQuery() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        // Distances computed using GeoDistanceAccuracyTest#strictHaversineDistance
+        execute("INSERT INTO %s (pk, v) VALUES (0, [1, 2])"); // distance is 555661 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (1, [4, 4])"); // distance is 157010 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (2, [5, 5])"); // distance is 0 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (3, [6, 6])"); // distance is 156891 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (4, [8, 9])"); // distance is 553647 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (5, [10, 10])"); // distance is 782780 m from [5,5]
+
+        beforeAndAfterFlush(() -> {
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [5,5]) < 157000"),
+                                    row(2), row(3));
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [5,5]) < 157011"),
+                                    row(1), row(2), row(3));
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [5,5]) <= 600000"),
+                                    row(0), row(1), row(2), row(3), row(4));
+        });
+    }
+
+    @Test
+    public void testPointCloseToBondaryAt1DegreeLatitude() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        // Points chosen to be close to the boundary of the search radius. The assertion failed based on earlier
+        // versions of the math used to determine whether the square distance was sufficient to short circuit
+        // the logic and skip performing the haversine distance calculation.
+        execute("INSERT INTO %s (pk, v) VALUES (0, [0.99, 0])"); // distance is 110.1 km from [0,0]
+        execute("INSERT INTO %s (pk, v) VALUES (1, [0.998, 0])"); // distance is 110.9 km from [0,0]
+        execute("INSERT INTO %s (pk, v) VALUES (2, [0.9982, 0])"); // distance is 110995 m from [0,0]
+        execute("INSERT INTO %s (pk, v) VALUES (3, [0.9983, 0])"); // distance is 111006.05 m from [0,0]
+        execute("INSERT INTO %s (pk, v) VALUES (4, [0.999, 0])"); // distance is 111.1 km from [0,0]
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0,0]) < 111000"),
+                       row(1), row(0), row(2));
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0,0]) <= 111006"),
+                       row(1), row(0), row(2));
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0,0]) < 111007"),
+                       row(1), row(0), row(2), row(3));
+        });
+    }
+
+
+    @Test
+    public void testPointCloseToBondaryAtOneTenthDegreeLatitude() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        execute("INSERT INTO %s (pk, v) VALUES (0, [0.10999, 0])"); // distance is 12230.3 m from [0,0]
+        execute("INSERT INTO %s (pk, v) VALUES (1, [0.11000, 0])"); // distance is 12231.4 m from [0,0]
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0,0]) < 12231"), row(0));
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0,0]) <= 12231"), row(0));
+        });
+    }
+
+    @Test
+    public void testPointCloseToBondaryAtOneTenThousandthsDegreeLatitude() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        execute("INSERT INTO %s (pk, v) VALUES (0, [0.00009, 0])"); // distance is 10.007 m from [0,0]
+        execute("INSERT INTO %s (pk, v) VALUES (1, [0.00010, 0])"); // distance is 11.120 m from [0,0]
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0,0]) < 11"), row(0));
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0,0]) <= 11"), row(0));
+        });
+    }
+
+    @Test
+    public void testIntersectedPredicateWithGeoDistanceQuery() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, num int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        createIndex("CREATE CUSTOM INDEX ON %s(num) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, num, v) VALUES (0, 0, [1, 2])"); // distance is 555661 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (1, 1, [4, 4])"); // distance is 157010 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (2, 2, [5, 5])"); // distance is 0 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (3, 3, [6, 6])"); // distance is 156891 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (4, 4, [8, 9])"); // distance is 553647 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (5, 5, [10, 10])"); // distance is 782780 m from [5,5]
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [5,5]) < 200000 AND num < 2"), row(1));
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [5,5]) <= 600000 AND num > 3"), row(4));
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [5,5]) <= 200000 AND num = 3"), row(3));
+        });
+    }
+
+    @Test
+    public void testGeoDistanceTopKQuery() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, point vector<float, 2>, v vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(point) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, point, v) VALUES (0, [1, 2], [1, 2, 1])"); // distance is 555661 m from [5,5]
+        execute("INSERT INTO %s (pk, point, v) VALUES (1, [4, 4], [4, 4, 1])"); // distance is 157010 m from [5,5]
+        execute("INSERT INTO %s (pk, point, v) VALUES (2, [5, 5], [5, 5, 1])"); // distance is 0 m from [5,5]
+        execute("INSERT INTO %s (pk, point, v) VALUES (3, [6, 6], [6, 6, 1])"); // distance is 156891 m from [5,5]
+        execute("INSERT INTO %s (pk, point, v) VALUES (4, [8, 9], [8, 9, 1])"); // distance is 553647 m from [5,5]
+        execute("INSERT INTO %s (pk, point, v) VALUES (5, [10, 10], [10, 10, 1])"); // distance is 782780 m from [5,5]
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(point, [0,0]) < 400000 ORDER BY v ANN OF [0, 1, 2] LIMIT 1"), row(0));
+        });
+    }
+
+    @Test
+    public void testPreparedIntersectedPredicateWithGeoDistanceQuery() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, num int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        createIndex("CREATE CUSTOM INDEX ON %s(num) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, num, v) VALUES (0, 0, [1, 2])"); // distance is 555661 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (1, 1, [4, 4])"); // distance is 157010 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (2, 2, [5, 5])"); // distance is 0 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (3, 3, [6, 6])"); // distance is 156891 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (4, 4, [8, 9])"); // distance is 553647 m from [5,5]
+        execute("INSERT INTO %s (pk, num, v) VALUES (5, 5, [10, 10])"); // distance is 782780 m from [5,5]
+
+        var query = "SELECT pk FROM %s WHERE GEO_DISTANCE(v, ?) < ? AND num < ?";
+        prepare(query);
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute(query, vector(5,5), 200000f, 2), row(1));
+            assertRows(execute(query, vector(5,5), 200000.0f, 2), row(1));
+        });
+    }
+
+    @Test
+    public void testNestedGeoDistanceQueries() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, num int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        createIndex("CREATE CUSTOM INDEX ON %s(num) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, v) VALUES (0, [1, 2])"); // distance is 555661 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (1, [4, 4])"); // distance is 157010 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (2, [5, 5])"); // distance is 0 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (3, [6, 6])"); // distance is 156891 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (4, [8, 9])"); // distance is 553647 m from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (5, [10, 10])"); // distance is 782780 m from [5,5]
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [10,10]) < 1 OR GEO_DISTANCE(v, [1,2]) < 1"),
+                       row(5), row(0));
+        });
+    }
+
+    @Test
+    public void testLongRangeGeoDistanceWithRealLocationsQuery() throws Throwable
+    {
+        createTable("CREATE TABLE %s (city text primary key, coordinates vector<float, 2>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(coordinates) USING 'StorageAttachedIndex' WITH OPTIONS = { 'similarity_function' : 'euclidean' }");
+
+        // coordinates are [latitude, longitude]
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Washington DC', [38.8951, -77.0364])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('New York City', [40.7128, -74.0060])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('San Francisco', [37.7749, -122.4194])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Los Angeles', [34.0522, -118.2437])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Chicago', [41.8781, -87.6298])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Boston', [42.3601, -71.0589])");
+
+        beforeAndAfterFlush(() -> {
+            // Cities within 5 meters of Boston
+            assertRowsIgnoringOrder(execute("SELECT city FROM %s WHERE GEO_DISTANCE(coordinates, [42.3601, -71.0589]) < 5"),
+                                    row("Boston"));
+
+
+            assertRowsIgnoringOrder(execute("SELECT city FROM %s WHERE GEO_DISTANCE(coordinates, [42.3601, -71.0589]) < 5 LIMIT 1"),
+                                    row("Boston"));
+
+            // Cities within 328.4 km of Washington DC
+            assertRowsIgnoringOrder(execute("SELECT city FROM %s WHERE GEO_DISTANCE(coordinates, [38.8951, -77.0364]) < 328400"),
+                                    row("New York City"), row("Washington DC"));
+
+            // Cities within 500 km of New York City
+            assertRowsIgnoringOrder(execute("SELECT city FROM %s WHERE GEO_DISTANCE(coordinates, [40.7128, -74.0060]) < 500000"),
+                                    row("Boston"), row("New York City"), row("Washington DC"));
+
+            // Cities within 1000 km of New York City
+            assertRowsIgnoringOrder(execute("SELECT city FROM %s WHERE GEO_DISTANCE(coordinates, [40.7128, -74.0060]) < 500000"),
+                                    row("Boston"), row("New York City"), row("Washington DC"));
+        });
+    }
+
+    @Test
+    public void testCloseRangeGeoDistanceWithRealLocationsQuery() throws Throwable
+    {
+        createTable("CREATE TABLE %s (city text primary key, coordinates vector<float, 2>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(coordinates) USING 'StorageAttachedIndex' WITH OPTIONS = { 'similarity_function' : 'euclidean' }");
+
+        // coordinates are [latitude, longitude]
+        // These are from NYC's Central Park
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Rec Center', [40.791186,-73.959591])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Baseball Field 11', [40.791597,-73.958059])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Baseball Field 7', [40.792847,-73.957105])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Baseball Field 6', [40.793018,-73.957565])");
+        execute("INSERT INTO %s (city, coordinates) VALUES ('Baseball Field 5', [40.793193,-73.958644])");
+
+        beforeAndAfterFlush(() -> {
+            // Point within 40 meters of field 6
+            assertRowsIgnoringOrder(execute("SELECT city FROM %s WHERE GEO_DISTANCE(coordinates, [40.793018,-73.957565]) < 40"),
+                                    row("Baseball Field 6"));
+
+            // Point within 43 meters of field 6 (field 7 is 43.14 meters away)
+            assertRowsIgnoringOrder(execute("SELECT city FROM %s WHERE GEO_DISTANCE(coordinates, [40.793018,-73.957565]) < 43.5"),
+                                    row("Baseball Field 6"), row("Baseball Field 7"));
+
+            // Point within 95 meters of field 6 (field 5 is 93 meters away)
+            assertRowsIgnoringOrder(execute("SELECT city FROM %s WHERE GEO_DISTANCE(coordinates, [40.793018,-73.957565]) < 95"),
+                                    row("Baseball Field 6"), row("Baseball Field 7"), row("Baseball Field 5"));
+        });
+    }
+
+    @Test
+    public void testGeoAndANNOnSameColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        // Distances computed using https://www.nhc.noaa.gov/gccalc.shtml
+        execute("INSERT INTO %s (pk, v) VALUES (0, [1, 2])"); // distance is 555 km from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (1, [4, 4])"); // distance is 157 km from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (2, [5, 5])"); // distance is 0 km from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (3, [6, 6])"); // distance is 157 km from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (4, [8, 9])"); // distance is 553 from [5,5]
+        execute("INSERT INTO %s (pk, v) VALUES (5, [10, 10])"); // distance is 782 km from [5,5]
+
+        beforeAndAfterFlush(() -> {
+            // GEO_DISTANCE gets all rows and then the limit gets the top 3
+            assertRows(execute("select pk from %s WHERE geo_distance(v,[5,5]) <= 1000000 ORDER BY v ANN of [5,5] limit 3"),
+                                    row(2), row(1), row(3));
+        });
+
+        // Delete a row
+        execute("DELETE FROM %s WHERE pk = 2");
+
+        // VSTODO this test asserts a slightly surprising result. PK 0 is further from the search point than PK 4, but by
+        // euclidean distance it is closer, so 0 is the third result. We still run the test because it
+        // validates that geo_distance and ANN OF two can be used at once. We have a ticket open to consider adding a
+        // HAVERSINE similarity function, which would give us correct results.
+        beforeAndAfterFlush(() -> {
+            // GEO_DISTANCE gets all rows and then the limit gets the top 3
+            assertRows(execute("select pk from %s WHERE geo_distance(v,[5,5]) <= 1000000 ORDER BY v ANN of [5,5] limit 3"),
+                       row(1), row(3), row(0));
+        });
+    }
+
+    @Test
+    public void testGeoDistanceNearAntiMerridianQueriesForLargeDistances() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, num int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        createIndex("CREATE CUSTOM INDEX ON %s(num) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, num, v) VALUES (0, 1, [0, -179])");
+        execute("INSERT INTO %s (pk, num, v) VALUES (1, 1, [0, 179])");
+        execute("INSERT INTO %s (pk, num, v) VALUES (2, 1, [45, -179])");
+        execute("INSERT INTO %s (pk, num, v) VALUES (3, 1, [45, 179])");
+        execute("INSERT INTO %s (pk, num, v) VALUES (4, 1, [90, -179])");
+        execute("INSERT INTO %s (pk, num, v) VALUES (5, 1, [90, 179])");
+        execute("INSERT INTO %s (pk, num, v) VALUES (6, 1, [0, 0])");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0, -179]) < 6000000"),
+                       row(0), row(1), row(2), row(3));
+
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [45, 179]) < 6000000"),
+                                    row(0), row(1), row(2), row(3), row(4), row(5));
+
+            // Search using AND and OR to cover different code paths
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [0, -179]) < 6000000 AND num = 1"),
+                                    row(0), row(1), row(2), row(3));
+
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(v, [45, 179]) < 6000000 OR num = 0"),
+                                    row(0), row(1), row(2), row(3), row(4), row(5));
+        });
+    }
+
+    @Test
+    public void testGeoDistanceNearAntiMerridianQueriesForCloseDistances() throws Throwable
+    {
+        createTable("CREATE TABLE %s (location text PRIMARY KEY, coords vector<float, 2>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(coords) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        // Here are the distances (these distances a not transitive, but do provide general motivation for the
+        // results observed in the test)
+        // Suva to Tubou is 292 km
+        // Suva to Dakuiloa is 328 km
+        // Tubou to Dakuiloa is 41.3 km
+        execute("INSERT INTO %s (location, coords) VALUES ('suva', [-18.1236146,178.4217888])"); // Suva, Fiji
+        execute("INSERT INTO %s (location, coords) VALUES ('tubou', [-18.2357357,-178.8109825])"); // Tubou, Fiji
+        execute("INSERT INTO %s (location, coords) VALUES ('dakuiloa', [-18.4452221,-178.4884278])"); // Dakuiloa, Fiji
+
+        beforeAndAfterFlush(() -> {
+            // Search from Suva
+            assertRowsIgnoringOrder(execute("SELECT location FROM %s WHERE GEO_DISTANCE(coords, [-18.1236146,178.4217888]) < 293000"),
+                                    row("suva"), row("tubou"));
+            // Search from Tubou
+            assertRowsIgnoringOrder(execute("SELECT location FROM %s WHERE GEO_DISTANCE(coords, [-18.2357357,-178.8109825]) < 293000"),
+                                    row("suva"), row("tubou"), row("dakuiloa"));
+            // Search from Dakuiloa
+            assertRowsIgnoringOrder(execute("SELECT location FROM %s WHERE GEO_DISTANCE(coords, [-18.4452221,-178.4884278]) < 293000"),
+                                    row("tubou"), row("dakuiloa"));
+            // Search from Dakuiloa with a smaller radius
+            assertRowsIgnoringOrder(execute("SELECT location FROM %s WHERE GEO_DISTANCE(coords, [-18.4452221,-178.4884278]) < 40000"),
+                                    row("dakuiloa"));
+            // Search from a point in between all three on the antimeridian
+            assertRowsIgnoringOrder(execute("SELECT location FROM %s WHERE GEO_DISTANCE(coords, [-18.3,-180]) < 170000"),
+                                    row("suva"), row("tubou"), row("dakuiloa"));
+            // Search from a point in between all three on the antimeridian
+            assertRowsIgnoringOrder(execute("SELECT location FROM %s WHERE GEO_DISTANCE(coords, [-18.3,180]) < 170000"),
+                                    row("suva"), row("tubou"), row("dakuiloa"));
+        });
+    }
+
+    @Test
+    public void testSearchesNearPoles() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, coords vector<float, 2>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(coords) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        execute("INSERT INTO %s (pk, coords) VALUES (0, [90,0])");
+        execute("INSERT INTO %s (pk, coords) VALUES (1, [89.99999, 0])");
+        execute("INSERT INTO %s (pk, coords) VALUES (2, [89.99999, 179])");
+        execute("INSERT INTO %s (pk, coords) VALUES (3, [89.99999, -179])");
+        execute("INSERT INTO %s (pk, coords) VALUES (4, [89.999, 0])");
+        execute("INSERT INTO %s (pk, coords) VALUES (5, [89.999, 179])");
+        execute("INSERT INTO %s (pk, coords) VALUES (5, [89.999, -179])");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(coords, [90, 0]) < 0.01"),
+                                    row(0));
+            assertRowsIgnoringOrder(execute("SELECT pk FROM %s WHERE GEO_DISTANCE(coords, [90, 0]) < 1"),
+                                    row(0), row(1), row(2), row(3));
+        });
+
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/IndexCombinationsTester.java b/test/unit/org/apache/cassandra/index/sai/cql/IndexCombinationsTester.java
new file mode 100644
index 000000000000..fd870502d3aa
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/IndexCombinationsTester.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+/**
+ * Utility class for testing different combinations of indexes on a table, to verify that the expected results are the
+ * same independently of what columns are indexed.
+ */
+public class IndexCombinationsTester
+{
+    private static final Logger logger = LoggerFactory.getLogger(IndexCombinationsTester.class);
+
+    private final SAITester tester;
+    private final Set<Index> indexes = new HashSet<>();
+
+    public IndexCombinationsTester(SAITester tester)
+    {
+        this.tester = tester;
+    }
+
+    public IndexCombinationsTester withIndexOn(String... columns)
+    {
+        IndexCombinationsTester tester = this;
+        for (String column : columns)
+        {
+            tester = withIndexOn(column, "CREATE CUSTOM INDEX ON %s(" + column + ") USING 'StorageAttachedIndex'");
+        }
+        return tester;
+    }
+
+    public IndexCombinationsTester withIndexOn(String column, String indexCreationQuery)
+    {
+        indexes.add(new Index(column, indexCreationQuery));
+        return this;
+    }
+
+    @SuppressWarnings("UnstableApiUsage")
+    public void test(Consumer<Set<String>> test)
+    {
+        for (int i = 0; i <= indexes.size(); i++)
+        {
+            for (Set<Index> combination : Sets.combinations(indexes, i))
+            {
+                Set<String> indexedColumns = combination.stream().map(idx -> idx.column).collect(Collectors.toSet());
+                logger.debug("Running test with indexes on: {}", indexedColumns);
+
+                combination.forEach(Index::create);
+                tester.waitForTableIndexesQueryable();
+                test.accept(indexedColumns);
+                combination.forEach(Index::drop);
+            }
+        }
+    }
+
+    private class Index
+    {
+        public final String column;
+        public final String createQuery;
+        public String name;
+
+        public Index(String column, String createQuery)
+        {
+            this.column = column;
+            this.createQuery = createQuery;
+        }
+
+        public void create()
+        {
+            name = tester.createIndex(createQuery);
+        }
+
+        public void drop()
+        {
+            assert name != null;
+            tester.dropIndex("DROP INDEX %s." + name);
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Index index = (Index) o;
+            return Objects.equals(column, index.column);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(column);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/IndexGroupLifecycleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/IndexGroupLifecycleTest.java
deleted file mode 100644
index 13965439abca..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/IndexGroupLifecycleTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql;
-
-import org.junit.Test;
-
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.lifecycle.Tracker;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
-
-import static java.lang.String.format;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-public class IndexGroupLifecycleTest extends SAITester
-{
-    @Test
-    public void testDropAndRecreate() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk text, value text, PRIMARY KEY (pk))");
-        populateOneSSTable();
-
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        cfs.disableAutoCompaction();
-        Tracker tracker = cfs.getTracker();
-
-        // create index and drop it: StorageAttachedIndexGroup should be removed
-        createIndex("CREATE CUSTOM INDEX sai ON %s(value) USING 'StorageAttachedIndex'");
-
-        StorageAttachedIndexGroup group = (StorageAttachedIndexGroup) cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.GROUP_KEY);
-        assertTrue(tracker.contains(group));
-        assertEquals(1, group.sstableContextManager().size());
-
-        dropIndex(format("DROP INDEX %s.sai", KEYSPACE));
-        assertFalse(tracker.contains(group));
-        assertEquals(0, group.sstableContextManager().size()); // sstable should be cleared from old group
-        assertNull(cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.GROUP_KEY));
-
-        // populate 2nd sstable. Old group should not track it
-        populateOneSSTable();
-        assertEquals(0, group.sstableContextManager().size());
-
-        // create index again: expect a new StorageAttachedIndexGroup to be registered into tracker
-        createIndex("CREATE CUSTOM INDEX sai ON %s(value) USING 'StorageAttachedIndex'");
-
-        StorageAttachedIndexGroup newGroup = (StorageAttachedIndexGroup) cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.GROUP_KEY);
-        assertNotSame(group, newGroup);
-        assertTrue(tracker.contains(newGroup));
-        assertEquals(2, newGroup.sstableContextManager().size());
-
-        // populate 3rd sstable. new group should  track it
-        populateOneSSTable();
-        assertEquals(3, newGroup.sstableContextManager().size());
-    }
-
-    private void populateOneSSTable()
-    {
-        execute("INSERT INTO %s(pk, value) VALUES('k', 'v')");
-        flush();
-    }
-}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/IndexGroupRemovalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/IndexGroupRemovalTest.java
new file mode 100644
index 000000000000..62a510f1369d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/IndexGroupRemovalTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class IndexGroupRemovalTest extends SAITester
+{
+    @Before
+    public void setup()
+    {
+        requireNetwork();
+    }
+
+    @Test
+    public void testDropAndRecreate() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk text, value text, PRIMARY KEY (pk))");
+        populateOneSSTable();
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.disableAutoCompaction();
+        Tracker tracker = cfs.getTracker();
+
+        // create index and drop it: StorageAttachedIndexGroup should be removed
+        createIndex("CREATE CUSTOM INDEX sai ON %s(value) USING 'StorageAttachedIndex'");
+
+        StorageAttachedIndexGroup group = (StorageAttachedIndexGroup) cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.GROUP_KEY);
+        assertTrue(tracker.contains(group));
+        assertEquals(1, group.sstableContextManager().size());
+
+        dropIndex(format("DROP INDEX %s.sai", KEYSPACE));
+        assertFalse(tracker.contains(group));
+        assertEquals(0, group.sstableContextManager().size()); // sstable should be cleared from old group
+        assertNull(cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.GROUP_KEY));
+
+        // populate 2nd sstable. Old group should not track it
+        populateOneSSTable();
+        assertEquals(0, group.sstableContextManager().size());
+
+        // create index again: expect a new StorageAttachedIndexGroup to be registered into tracker
+        createIndex("CREATE CUSTOM INDEX sai ON %s(value) USING 'StorageAttachedIndex'");
+
+        StorageAttachedIndexGroup newGroup = (StorageAttachedIndexGroup) cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.GROUP_KEY);
+        assertNotSame(group, newGroup);
+        assertTrue(tracker.contains(newGroup));
+        assertEquals(2, newGroup.sstableContextManager().size());
+
+        // populate 3rd sstable. new group should  track it
+        populateOneSSTable();
+        assertEquals(3, newGroup.sstableContextManager().size());
+    }
+
+    private void populateOneSSTable() throws Throwable
+    {
+        execute("INSERT INTO %s(pk, value) VALUES('k', 'v')");
+        flush();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/IndexParallelBuildTest.java b/test/unit/org/apache/cassandra/index/sai/cql/IndexParallelBuildTest.java
new file mode 100644
index 000000000000..1eb23c83a16d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/IndexParallelBuildTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Collections;
+
+import com.google.common.collect.Iterables;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.SimpleStatement;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.inject.InvokePointBuilder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class IndexParallelBuildTest extends SAITester
+{
+    private static final String CREATE_TABLE_TEMPLATE = "CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT, v2 TEXT)" +
+                                                        "  WITH compaction = {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
+
+    private static final int PAGE_SIZE = 5000;
+
+    @BeforeClass
+    public static void setupCQLTester()
+    {
+        Config conf = DatabaseDescriptor.loadConfig();
+        conf.num_tokens = 16;
+        conf.incremental_backups = false;
+        DatabaseDescriptor.daemonInitialization(() -> conf);
+    }
+
+    @Before
+    public void setup()
+    {
+        requireNetwork();
+    }
+
+    @After
+    public void resetCountersAndInjections()
+    {
+        Injections.deleteAll();
+    }
+
+    @Test
+    public void testParallelIndexBuild() throws Throwable
+    {
+        // populate 10 sstables
+        createTable(CREATE_TABLE_TEMPLATE);
+        int sstable = 10;
+        int rowsPerSSTable = 100;
+        int key = 0;
+        for (int s = 0; s < sstable; s++)
+        {
+            for (int i = 0; i < rowsPerSSTable; i++)
+            {
+                execute("INSERT INTO %s (id1, v1, v2) VALUES (?, 0, '0')", Integer.toString(key++));
+            }
+            flush();
+        }
+
+        // leave sstable files on disk
+        getCurrentColumnFamilyStore().getTracker().unloadSSTables();
+        assertTrue(getCurrentColumnFamilyStore().getLiveSSTables().isEmpty());
+
+        // create indexes
+        String index = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        assertTrue(getCurrentColumnFamilyStore().getLiveSSTables().isEmpty());
+
+        Injections.Counter parallelBuildCounter = Injections.newCounter("IndexParallelBuildCounter")
+                                                            .add(InvokePointBuilder.newInvokePoint().onClass("org.apache.cassandra.index.sai.StorageAttachedIndex$StorageAttachedIndexBuildingSupport")
+                                                                                   .onMethod("getParallelIndexBuildTasks"))
+                                                            .build();
+
+        Injections.inject(parallelBuildCounter);
+        assertEquals(0, parallelBuildCounter.get());
+
+        // reload on-disk sstables and wait for parallel index build
+        getCurrentColumnFamilyStore().loadNewSSTables();
+        waitForAssert(() -> {
+            assertEquals(getCurrentColumnFamilyStore().getLiveSSTables().size(), sstable);
+            StorageAttachedIndex sai = (StorageAttachedIndex) Iterables.getOnlyElement(getCurrentColumnFamilyStore().indexManager.listIndexes());
+            assertEquals(sai.getIndexContext().getView().getIndexes().size(), sstable);
+        });
+
+        // verify parallel index build is invoked once
+        assertEquals(1, parallelBuildCounter.get());
+
+        // verify index can be read
+        assertRowCount(sstable * rowsPerSSTable, "SELECT id1 FROM %s WHERE v1>=0");
+
+        // reset the counter
+        parallelBuildCounter.reset();
+        assertEquals(0, parallelBuildCounter.get());
+
+        // verify parallel index build again, this time with a full rebuild with the sstables already loaded
+        getCurrentColumnFamilyStore().indexManager.rebuildIndexesBlocking(Collections.singleton(index));
+        assertEquals(1, parallelBuildCounter.get());
+
+        // verify index can still be read
+        assertRowCount(sstable * rowsPerSSTable, "SELECT id1 FROM %s WHERE v1>=0");
+    }
+
+    private void assertRowCount(int count, String query)
+    {
+        assertEquals(count,
+                     sessionNet(getDefaultVersion())
+                     .execute(new SimpleStatement(formatQuery(query))
+                              .setFetchSize(PAGE_SIZE)).all().size());
+    }
+}
+
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/IndexQuerySupport.java b/test/unit/org/apache/cassandra/index/sai/cql/IndexQuerySupport.java
deleted file mode 100644
index cd8a047dafd8..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/IndexQuerySupport.java
+++ /dev/null
@@ -1,584 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.cassandra.index.sai.cql;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import com.google.common.base.MoreObjects;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-import org.junit.Assert;
-
-import com.datastax.driver.core.exceptions.InvalidQueryException;
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
-import org.apache.cassandra.db.marshal.InetAddressType;
-import org.apache.cassandra.db.marshal.SimpleDateType;
-import org.apache.cassandra.db.marshal.TimeType;
-import org.apache.cassandra.db.marshal.TimestampType;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
-import org.apache.cassandra.utils.Pair;
-import org.assertj.core.api.Assertions;
-import org.hamcrest.Matchers;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-
-/**
- * A CQL-based test framework for simulating queries across as much of the index state space as possible.
- *
- * This includes, but need not be limited to...
- *
- * 1.) ...queries on the same data as it migrates through the write path and storage engine.
- * 2.) ...queries across all supported native data types.
- * 3.) ...queries for all supported operators and value boundaries.
- * 4.) ...queries for varying write, update, delete, and TTL workloads.
- * 5.) ...queries across varying primary key and table structures.
- * 6.) ...queries across static, normal, and clustering column types.
- * 7.) ...queries across various paging and limit settings.
- *
- * IMPORTANT: This class is shared between the single-node SAITester based classes and the
- * multi-node distributed classes. It must not reference SAITester or CQLTester directly
- * to avoid static loading and initialisation.
- */
-public class IndexQuerySupport
-{
-    public static final List<BaseQuerySet> BASE_QUERY_SETS = ImmutableList.of(new BaseQuerySet(10, 5),
-                                                                              new BaseQuerySet(10, 9),
-                                                                              new BaseQuerySet(10, 10),
-                                                                              new BaseQuerySet(10, Integer.MAX_VALUE),
-                                                                              new BaseQuerySet(24, 10),
-                                                                              new BaseQuerySet(24, 100),
-                                                                              new BaseQuerySet(24, Integer.MAX_VALUE));
-
-    public static final List<BaseQuerySet> COMPOSITE_PARTITION_QUERY_SETS = ImmutableList.of(new CompositePartitionQuerySet(10, 5),
-                                                                                             new CompositePartitionQuerySet(10, 10),
-                                                                                             new CompositePartitionQuerySet(10, Integer.MAX_VALUE),
-                                                                                             new CompositePartitionQuerySet(24, 10),
-                                                                                             new CompositePartitionQuerySet(24, 100),
-                                                                                             new CompositePartitionQuerySet(24, Integer.MAX_VALUE));
-
-    public static final List<BaseQuerySet> STATIC_QUERY_SETS = ImmutableList.of(new StaticColumnQuerySet(10, 5),
-                                                                                new StaticColumnQuerySet(10, 10),
-                                                                                new StaticColumnQuerySet(10, Integer.MAX_VALUE),
-                                                                                new StaticColumnQuerySet(24, 10),
-                                                                                new StaticColumnQuerySet(24, 100),
-                                                                                new StaticColumnQuerySet(24, Integer.MAX_VALUE));
-
-    public static void writeLifecycle(BaseDataModel.Executor executor, BaseDataModel dataModel, List<BaseQuerySet> sets) throws Throwable
-    {
-        dataModel.createTables(executor);
-
-        dataModel.disableCompaction(executor);
-
-        dataModel.createIndexes(executor);
-
-        // queries against Memtable adjacent in-memory indexes
-        dataModel.insertRows(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries with Memtable flushed to SSTable on disk
-        dataModel.flush(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries across memory and disk indexes
-        dataModel.insertRows(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries w/ multiple SSTable indexes
-        dataModel.flush(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries after compacting to a single SSTable index
-        dataModel.compact(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries against Memtable updates and the existing SSTable index
-        dataModel.updateCells(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries against the newly flushed SSTable index and the existing SSTable index
-        dataModel.flush(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries after compacting updates into to a single SSTable index
-        dataModel.compact(executor);
-        executeQueries(dataModel, executor, sets);
-    }
-
-    public static void rowDeletions(BaseDataModel.Executor executor, BaseDataModel dataModel, List<BaseQuerySet> sets) throws Throwable
-    {
-        dataModel.createTables(executor);
-
-        dataModel.disableCompaction(executor);
-
-        dataModel.createIndexes(executor);
-        dataModel.insertRows(executor);
-        dataModel.flush(executor);
-        dataModel.compact(executor);
-
-        // baseline queries
-        executeQueries(dataModel, executor, sets);
-
-        // queries against Memtable deletes and the existing SSTable index
-        dataModel.deleteRows(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries against the newly flushed SSTable index and the existing SSTable index
-        dataModel.flush(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries after compacting deletes into to a single SSTable index
-        dataModel.compact(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // truncate, reload, and verify that the load is clean
-        dataModel.truncateTables(executor);
-        dataModel.insertRows(executor);
-        executeQueries(dataModel, executor, sets);
-    }
-
-    public static void cellDeletions(BaseDataModel.Executor executor, BaseDataModel dataModel, List<BaseQuerySet> sets) throws Throwable
-    {
-        dataModel.createTables(executor);
-
-        dataModel.disableCompaction(executor);
-
-        dataModel.createIndexes(executor);
-        dataModel.insertRows(executor);
-        dataModel.flush(executor);
-        dataModel.compact(executor);
-
-        // baseline queries
-        executeQueries(dataModel, executor, sets);
-
-        // queries against Memtable deletes and the existing SSTable index
-        dataModel.deleteCells(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries against the newly flushed SSTable index and the existing SSTable index
-        dataModel.flush(executor);
-        executeQueries(dataModel, executor, sets);
-
-        // queries after compacting deletes into to a single SSTable index
-        dataModel.compact(executor);
-        executeQueries(dataModel, executor, sets);
-    }
-
-    public static void timeToLive(BaseDataModel.Executor executor, BaseDataModel dataModel, List<BaseQuerySet> sets) throws Throwable
-    {
-        dataModel.createTables(executor);
-
-        dataModel.createIndexes(executor);
-        dataModel.insertRowsWithTTL(executor);
-
-        // Wait for the TTL to become effective:
-        TimeUnit.SECONDS.sleep(BaseDataModel.DEFAULT_TTL_SECONDS);
-
-        // Make sure TTLs are reflected in our query results from the Memtable:
-        executeQueries(dataModel, executor, sets);
-    }
-
-    private static void executeQueries(BaseDataModel dataModel, BaseDataModel.Executor executor, List<BaseQuerySet> sets) throws Throwable
-    {
-        for (BaseQuerySet set : sets)
-        {
-            set.execute(executor, dataModel);
-        }
-    }
-
-    static class StaticColumnQuerySet extends BaseQuerySet
-    {
-        StaticColumnQuerySet(int limit, int fetchSize)
-        {
-            super(limit, fetchSize);
-        }
-
-        public void execute(BaseDataModel.Executor tester, BaseDataModel model) throws Throwable
-        {
-            super.execute(tester, model);
-
-            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.EQ, 1845);
-            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.LT, 1845);
-            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.LTE, 1845);
-            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.GT, 1845);
-            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.GTE, 1845);
-            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.EQ, 1909);
-            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.LT, 1787);
-            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.GT, 1910);
-
-            rangeQuery(tester, model, BaseDataModel.STATIC_INT_COLUMN, 1845, 1909);
-        }
-    }
-
-    static class CompositePartitionQuerySet extends BaseQuerySet
-    {
-        CompositePartitionQuerySet(int limit, int fetchSize)
-        {
-            super(limit, fetchSize);
-        }
-
-        public void execute(BaseDataModel.Executor tester, BaseDataModel model) throws Throwable
-        {
-            super.execute(tester, model);
-
-            for(Pair<String, String> partitionKeyComponent: model.keyColumns)
-            {
-                String partitionKeyComponentName = partitionKeyComponent.left;
-                query(tester, model, partitionKeyComponentName, Operator.EQ, 0);
-                query(tester, model, partitionKeyComponentName, Operator.GT, 0);
-                query(tester, model, partitionKeyComponentName, Operator.LTE, 2);
-                query(tester, model, partitionKeyComponentName, Operator.GTE, -1);
-                query(tester, model, partitionKeyComponentName, Operator.LT, 50);
-                query(tester, model, partitionKeyComponentName, Operator.GT, 0);
-            }
-
-            String firstPartitionKey = model.keyColumns.get(0).left;
-            String secondPartitionKey = model.keyColumns.get(1).left;
-            List<Operator> numericOperators = Arrays.asList(Operator.EQ, Operator.GT, Operator.LT, Operator.GTE, Operator.LTE);
-            List<List<Operator>> combinations = Lists.cartesianProduct(numericOperators, numericOperators).stream()
-                                                     .filter(p-> p.get(0) != Operator.EQ || p.get(1) != Operator.EQ) //If both are EQ the entire partition is specified
-                                                     .collect(Collectors.toList());
-            for(List<Operator> operators : combinations)
-            {
-                andQuery(tester,
-                         model,
-                         firstPartitionKey, operators.get(0), 2,
-                         secondPartitionKey, operators.get(1), 2,
-                         false);
-            }
-        }
-    }
-
-    public static class BaseQuerySet
-    {
-        final int limit;
-        final int fetchSize;
-
-        BaseQuerySet(int limit, int fetchSize)
-        {
-            this.limit = limit;
-            this.fetchSize = fetchSize;
-        }
-
-        void execute(BaseDataModel.Executor tester, BaseDataModel model) throws Throwable
-        {
-            query(tester, model, BaseDataModel.ASCII_COLUMN, Operator.EQ, "MA");
-            query(tester, model, BaseDataModel.ASCII_COLUMN, Operator.EQ, "LA");
-            query(tester, model, BaseDataModel.ASCII_COLUMN, Operator.EQ, "XX");
-
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.EQ, 4800000000L);
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.EQ, 5000000000L);
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.LT, 5000000000L);
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.LTE, 5000000000L);
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.GT, 5000000000L);
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.GTE, 5000000000L);
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.EQ, 22L);
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.LT, 400000000L);
-            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.GT, 10000000000L);
-
-            rangeQuery(tester, model, BaseDataModel.BIGINT_COLUMN, 3000000000L, 7000000000L);
-
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2013-06-10"));
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2013-06-17"));
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.LT, SimpleDateType.instance.fromString("2013-06-17"));
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.LTE, SimpleDateType.instance.fromString("2013-06-17"));
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.GT, SimpleDateType.instance.fromString("2013-06-17"));
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.GTE, SimpleDateType.instance.fromString("2013-06-17"));
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2017-01-01"));
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.LT, SimpleDateType.instance.fromString("2000-01-01"));
-            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.GT, SimpleDateType.instance.fromString("2020-01-01"));
-
-            rangeQuery(tester, model, BaseDataModel.DATE_COLUMN, SimpleDateType.instance.fromString("2013-06-17"), SimpleDateType.instance.fromString("2018-06-19"));
-
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 43203.90);
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 7800.06);
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.LT, 82169.62);
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.LTE, 82169.62);
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.GT, 82169.62);
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.GTE, 82169.62);
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 82169.60);
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.LT, 1948.54);
-            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.GT, 570640.95);
-
-            rangeQuery(tester, model, BaseDataModel.DOUBLE_COLUMN, 56538.90, 113594.08);
-
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.EQ, 10.2f);
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.EQ, 1.9f);
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.LT, 5.3f);
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.LTE, 5.3f);
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.GT, 5.3f);
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.GTE, 5.3f);
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.EQ, 5.9f);
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.LT, 1.8f);
-            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.GT, 10.2f);
-
-            rangeQuery(tester, model, BaseDataModel.FLOAT_COLUMN, 4.6f, 6.7f);
-
-            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("170.63.206.57"));
-            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("170.63.206.56"));
-            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("205.204.196.65"));
-            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("164.165.67.10"));
-            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("204.196.242.71"));
-
-            rangeQuery(tester, model, BaseDataModel.INT_COLUMN, 2977853, 6784240);
-
-            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.EQ, (short) 164);
-            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.LT, (short) 164);
-            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.LTE, (short) 164);
-            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.GT, (short) 164);
-            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.GTE, (short) 164);
-            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.EQ, (short) 2);
-            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.LT, (short) 30);
-            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.GT, (short) 1861);
-
-            rangeQuery(tester, model, BaseDataModel.SMALLINT_COLUMN, (short) 126, (short) 383);
-
-            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.EQ, (byte) 16);
-            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.LT, (byte) 16);
-            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.LTE, (byte) 16);
-            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.GT, (byte) 16);
-            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.GTE, (byte) 16);
-            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.EQ, (byte) 1);
-            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.LT, (byte) 2);
-            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.GT, (byte) 117);
-
-            rangeQuery(tester, model, BaseDataModel.TINYINT_COLUMN, (byte) 12, (byte) 47);
-
-            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska");
-            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Wyoming");
-            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Franklin");
-            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "State of Michigan");
-            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Michigan");
-            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Louisiana");
-            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Massachusetts");
-
-            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.EQ, TimeType.instance.fromString("00:43:07"));
-            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.LT, TimeType.instance.fromString("00:43:07"));
-            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.LTE, TimeType.instance.fromString("00:43:07"));
-            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.GT, TimeType.instance.fromString("00:43:07"));
-            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.GTE, TimeType.instance.fromString("00:43:07"));
-            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.EQ, TimeType.instance.fromString("00:15:57"));
-            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.LT, TimeType.instance.fromString("00:15:50"));
-            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.GT, TimeType.instance.fromString("01:30:45"));
-
-            rangeQuery(tester, model, BaseDataModel.TIME_COLUMN, TimeType.instance.fromString("00:38:13"), TimeType.instance.fromString("00:56:07"));
-
-            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.EQ, TimestampType.instance.fromString("2013-06-17T00:00:00"));
-            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.LT, TimestampType.instance.fromString("2013-06-17T00:00:00"));
-            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.LTE, TimestampType.instance.fromString("2013-06-17T00:00:00"));
-            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.GT, TimestampType.instance.fromString("2013-06-17T00:00:00"));
-            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2013-06-17T00:00:00"));
-            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.EQ, TimestampType.instance.fromString("2017-01-01T00:00:00"));
-            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.LT, TimestampType.instance.fromString("2000-01-01T00:00:00"));
-            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.GT, TimestampType.instance.fromString("2020-01-01T00:00:00"));
-
-            rangeQuery(tester, model, BaseDataModel.TIMESTAMP_COLUMN,
-                       TimestampType.instance.fromString("2013-6-17T00:00:00"),
-                       TimestampType.instance.fromString("2018-6-19T00:00:00"));
-
-            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"));
-            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("752355f8-405b-4d94-88f3-9992cda30f1e"));
-            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("ac0aa734-d17f-11e8-a8d5-f2801f1b9fd1"));
-            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("c6eec0b0-0eef-40e8-ac38-3a82110443e4"));
-            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"));
-
-            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
-            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.LT, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
-            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.LTE, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
-            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.GT, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
-            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.GTE, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
-            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("2a421a68-d182-11e8-a8d5-f2801f1b9fd1"));
-
-            andQuery(tester, model,
-                     BaseDataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2013-06-20T00:00:00"),
-                     BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("752355f8-405b-4d94-88f3-9992cda30f1e"),
-                     false);
-
-            andQuery(tester, model,
-                     BaseDataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2018-06-20T00:00:00"),
-                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Texas",
-                     false);
-
-            andQuery(tester, model,
-                     BaseDataModel.SMALLINT_COLUMN, Operator.LTE, (short) 126,
-                     BaseDataModel.TINYINT_COLUMN, Operator.LTE, (byte) 9,
-                     false);
-
-            andQuery(tester, model,
-                     BaseDataModel.SMALLINT_COLUMN, Operator.LTE, (short) 126,
-                     BaseDataModel.NON_INDEXED_COLUMN, Operator.GT, 0,
-                     true);
-
-            andQuery(tester, model,
-                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
-                     BaseDataModel.NON_INDEXED_COLUMN, Operator.EQ, 2,
-                     true);
-
-
-            andQuery(tester, model,
-                     BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"),
-                     BaseDataModel.NON_INDEXED_COLUMN, Operator.LT, 3,
-                     true);
-
-            // with partition column filtering
-            String firstPartitionKey = model.keyColumns().get(0).left;
-
-            andQuery(tester, model,
-                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
-                     firstPartitionKey, Operator.EQ, 0,
-                     false);
-
-            boolean hasSimplePartitionKey = !(model instanceof BaseDataModel.CompositePartitionKeyDataModel);
-
-            andQuery(tester, model,
-                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Kentucky",
-                     firstPartitionKey, Operator.GT, 4,
-                     hasSimplePartitionKey);
-
-            andQuery(tester, model,
-                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Wyoming",
-                     firstPartitionKey, Operator.LT, 200,
-                     hasSimplePartitionKey);
-
-            if (model.keyColumns().size() > 1)
-            {
-                String secondPrimaryKey = model.keyColumns().get(1).left;
-
-                andQuery(tester, model,
-                         BaseDataModel.BIGINT_COLUMN, Operator.EQ, 4800000000L,
-                         secondPrimaryKey, Operator.EQ, 0,
-                         hasSimplePartitionKey);
-
-                andQuery(tester, model,
-                         BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 82169.60,
-                         secondPrimaryKey, Operator.GT, 0,
-                         hasSimplePartitionKey);
-
-                andQuery(tester, model,
-                         BaseDataModel.DOUBLE_COLUMN, Operator.LT, 1948.54,
-                         secondPrimaryKey, Operator.LTE, 2,
-                         hasSimplePartitionKey);
-
-                andQuery(tester, model,
-                         BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
-                         firstPartitionKey, Operator.EQ, 0,
-                         secondPrimaryKey, Operator.GTE, -1,
-                         false);
-
-                andQuery(tester, model,
-                         BaseDataModel.TEXT_COLUMN, Operator.EQ, "Kentucky",
-                         firstPartitionKey, Operator.GT, 4,
-                         secondPrimaryKey, Operator.LT, 50,
-                         hasSimplePartitionKey);
-
-                andQuery(tester, model,
-                         BaseDataModel.TEXT_COLUMN, Operator.EQ, "Wyoming",
-                         firstPartitionKey, Operator.LT, 200,
-                         secondPrimaryKey, Operator.GT, 0,
-                         hasSimplePartitionKey);
-            }
-        }
-
-        void query(BaseDataModel.Executor tester, BaseDataModel model, String column, Operator operator, Object value)
-        {
-            String query = String.format(BaseDataModel.SIMPLE_SELECT_TEMPLATE, BaseDataModel.ASCII_COLUMN, column, operator);
-            validate(tester, model, query, false, value, limit);
-        }
-
-        void andQuery(BaseDataModel.Executor tester, BaseDataModel model,
-                      String column1, Operator operator1, Object value1,
-                      String column2, Operator operator2, Object value2,
-                      boolean filtering)
-        {
-            String query = String.format(BaseDataModel.TWO_CLAUSE_AND_QUERY_TEMPLATE,
-                                         BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2);
-
-            validate(tester, model, query, filtering, value1, value2, limit);
-        }
-
-        void andQuery(BaseDataModel.Executor tester, BaseDataModel model,
-                      String column1, Operator operator1, Object value1,
-                      String column2, Operator operator2, Object value2,
-                      String column3, Operator operator3, Object value3,
-                      boolean filtering)
-        {
-            String query = String.format(BaseDataModel.THREE_CLAUSE_AND_QUERY_TEMPLATE,
-                                         BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2, column3, operator3);
-
-            validate(tester, model, query, filtering, value1, value2, value3, limit);
-        }
-
-        void rangeQuery(BaseDataModel.Executor tester, BaseDataModel model, String column, Object value1, Object value2)
-        {
-            String query = String.format(BaseDataModel.RANGE_QUERY_TEMPLATE, BaseDataModel.ASCII_COLUMN, column);
-            validate(tester, model, query, false, value1, value2, limit);
-        }
-
-        private void validate(BaseDataModel.Executor tester, BaseDataModel model, String query, boolean needsAllowFiltering, Object... values)
-        {
-            try
-            {
-                tester.counterReset();
-
-                // The non indexed query we use to validate the indexed query results is just the very same query but
-                // with ALLOW FILTERING appended. It might happen that the non indexed query also requires ALLOW
-                // FILTERING because it combines indexed and unindexed columns.
-                Assert.assertFalse(query.contains("ALLOW FILTERING"));
-                String validationQuery = query + " ALLOW FILTERING";
-                String indexedQuery = needsAllowFiltering ? validationQuery : query;
-
-                List<Object> actual = model.executeIndexed(tester, indexedQuery, fetchSize, values);
-
-                // This could be more strict, but it serves as a reasonable paging-aware lower bound:
-                int pageCount = (int) Math.ceil(actual.size() / (double) Math.min(actual.size(), fetchSize));
-                assertThat("Expected more calls to " + StorageAttachedIndexSearcher.class, tester.getCounter(), Matchers.greaterThanOrEqualTo((long) Math.max(1, pageCount)));
-
-                List<Object> expected = model.executeNonIndexed(tester, validationQuery, fetchSize, values);
-                assertEquals(expected, actual);
-
-                // verify that the query actually requires ALLOW FILTERING
-                if (needsAllowFiltering)
-                {
-                    Assertions.assertThatThrownBy(() -> model.executeIndexed(tester, query, fetchSize, values))
-                              .isInstanceOf(InvalidQueryException.class)
-                              .hasMessageContaining(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
-                }
-            }
-            catch (Throwable ex)
-            {
-                ex.printStackTrace();
-                throw ex;
-            }
-        }
-
-        @Override
-        public String toString()
-        {
-            return MoreObjects.toStringHelper(this).add("limit", limit).add("fetchSize", fetchSize).toString();
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/InetAddressTypeEquivalencyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/InetAddressTypeEquivalencyTest.java
index e8ca8c0f1896..9ebbd90f674b 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/InetAddressTypeEquivalencyTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/InetAddressTypeEquivalencyTest.java
@@ -18,20 +18,30 @@
 package org.apache.cassandra.index.sai.cql;
 
 import java.net.InetAddress;
-
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.cql.types.InetTest;
 
 /**
  * This is testing that we can query ipv4 addresses using ipv6 equivalent addresses.
- *
+ * </p>
  * The remaining InetAddressType tests are now handled by {@link InetTest}
  */
 public class InetAddressTypeEquivalencyTest extends SAITester
 {
+    // TODO: Disables coordinator execution because we know SAI indexing for inet works differently than RowFilter,
+    //  which can wrongly discard rows in the coordinator. This is reported in CNDB-12978, and we should enable
+    //  distributed execution again once we have a fix.
+    @BeforeClass
+    public static void disableCoordinatorExecution()
+    {
+        CQLTester.disableCoordinatorExecution();
+    }
+
     @Before
     public void createTableAndIndex()
     {
@@ -51,8 +61,14 @@ public void mixedWorkloadQueryTest() throws Throwable
         execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 2, '127.0.0.1')");
         execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 3, '127.0.0.2')");
         execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 4, '::ffff:7f00:3')");
+
+        flush();
+
         execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 5, '2002:4559:1fe2::4559:1fe2')");
         execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 6, '2002:4559:1fe2::4559:1fe2')");
+
+        flush();
+
         execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 7, '2002:4559:1fe2::4559:1fe2')");
         execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 8, '2002:4559:1fe2::4559:1fe3')");
 
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/LegacyIndexCqlTest.java b/test/unit/org/apache/cassandra/index/sai/cql/LegacyIndexCqlTest.java
new file mode 100644
index 000000000000..bf7f39897b6f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/LegacyIndexCqlTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.FileUtils;
+import org.apache.cassandra.io.util.File;
+
+import static org.junit.Assert.assertEquals;
+
+public class LegacyIndexCqlTest extends SAITester
+{
+    @Test
+    public void loadLegacyIndexAndRunCqlQuery() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, int_value int, text_value text, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX int_index ON %s(int_value) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX text_index ON %s(text_value) USING 'StorageAttachedIndex'");
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+
+        File directory = cfs.getDirectories().getDirectoryForNewSSTables();
+
+        FileUtils.copySSTablesAndIndexes(directory.toJavaIOFile(), "aa");
+
+        cfs.loadNewSSTables();
+
+        UntypedResultSet resultSet = execute("SELECT * FROM %s WHERE int_value > 10 AND int_value < 20");
+        assertEquals(9, resultSet.size());
+
+        resultSet = execute("SELECT * FROM %s WHERE text_value = '10'");
+        assertEquals(1, resultSet.size());
+
+        for (int row = 100; row < 200; row++)
+            execute("INSERT INTO %s (pk, int_value, text_value) VALUES (?, ?, ?)", row, row, Integer.toString(row));
+
+        resultSet = execute("SELECT * FROM %s WHERE int_value > 90 AND int_value < 110");
+        assertEquals(19, resultSet.size());
+
+        resultSet = execute("SELECT * FROM %s WHERE text_value = '10'");
+        assertEquals(1, resultSet.size());
+
+        resultSet = execute("SELECT * FROM %s WHERE text_value = '110'");
+        assertEquals(1, resultSet.size());
+
+        flush();
+
+        resultSet = execute("SELECT * FROM %s WHERE int_value > 90 AND int_value < 110");
+        assertEquals(19, resultSet.size());
+
+        resultSet = execute("SELECT * FROM %s WHERE text_value = '10'");
+        assertEquals(1, resultSet.size());
+
+        resultSet = execute("SELECT * FROM %s WHERE text_value = '110'");
+        assertEquals(1, resultSet.size());
+
+        compact();
+        waitForCompactions();
+
+        resultSet = execute("SELECT * FROM %s WHERE int_value > 90 AND int_value < 110");
+        assertEquals(19, resultSet.size());
+
+        resultSet = execute("SELECT * FROM %s WHERE text_value = '10'");
+        assertEquals(1, resultSet.size());
+
+        resultSet = execute("SELECT * FROM %s WHERE text_value = '110'");
+        assertEquals(1, resultSet.size());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/LiteralOrderTest.java b/test/unit/org/apache/cassandra/index/sai/cql/LiteralOrderTest.java
new file mode 100644
index 000000000000..30ce84c240f2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/LiteralOrderTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.TreeMap;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.index.sai.SAITester;
+
+public class LiteralOrderTest extends SAITester
+{
+    @Test
+    public void endToEndTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, str_val text, num int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(num) USING 'StorageAttachedIndex'");
+
+        // Each iteration overwrites the previous data, so this test deals with a lot of overwrites.
+        for (int i = 0; i < 5; i++)
+            insertRowsAndMakeAssertions();
+    }
+
+    private void insertRowsAndMakeAssertions() throws Throwable
+    {
+        var sortedMap = new TreeMap<String, Integer>();
+        for (int i = 0; i < 1000; i++)
+        {
+            var value = getRandom().nextAsciiString(10, 100);
+            sortedMap.put(value, i);
+            execute("INSERT INTO %s (pk, str_val, num) VALUES (?, ?, ?)", i, value, i);
+        }
+
+        beforeAndAfterFlush(() -> {
+            for (int i = 0; i < 1000; i++)
+            {
+                var randomLimit = getRandom().nextIntBetween(5, 100);
+                var expectedRows = sortedMap.values().stream().limit(randomLimit).map(CQLTester::row).toArray(Object[][]::new);
+                assertRows(execute("SELECT pk FROM %s ORDER BY str_val LIMIT ?", randomLimit), expectedRows);
+                // Utilize the fact that the num is also the PK
+                var randomUpperBound = getRandom().nextIntBetween(5, 100);
+                var expectedHybridRows = sortedMap.values().stream()
+                                                  .filter(n -> n < randomUpperBound)
+                                                  .map(CQLTester::row)
+                                                  .limit(randomLimit)
+                                                  .toArray(Object[][]::new);
+                assertRows(execute("SELECT pk FROM %s WHERE num < ? ORDER BY str_val LIMIT ?", randomUpperBound, randomLimit), expectedHybridRows);
+            }
+        });
+    }
+
+    // Because we store values in a trie, we want to confirm that we always descend the trie before returning
+    // values on the DESC path.
+    @Test
+    public void testTextPrefixes() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, str_val ascii)");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        disableCompaction();
+
+        execute("INSERT INTO %s (pk, str_val) VALUES (?, ?)", 1, "a");
+        execute("INSERT INTO %s (pk, str_val) VALUES (?, ?)", 2, "ab");
+        execute("INSERT INTO %s (pk, str_val) VALUES (?, ?)", 3, "abc");
+        execute("INSERT INTO %s (pk, str_val) VALUES (?, ?)", 4, "ad");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s ORDER BY str_val LIMIT 2"), row(1), row(2));
+            assertRows(execute("SELECT pk FROM %s ORDER BY str_val DESC LIMIT 2"), row(4), row(3));
+        });
+
+        // Compaction triggers index build via a different mechanism, so best to test explicitly.
+        compact();
+
+        assertRows(execute("SELECT pk FROM %s ORDER BY str_val LIMIT 2"), row(1), row(2));
+        assertRows(execute("SELECT pk FROM %s ORDER BY str_val DESC LIMIT 2"), row(4), row(3));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/LuceneAnalyzerTest.java b/test/unit/org/apache/cassandra/index/sai/cql/LuceneAnalyzerTest.java
new file mode 100644
index 000000000000..7c40007b0cef
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/LuceneAnalyzerTest.java
@@ -0,0 +1,1076 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.analyzer.AnalyzerEqOperatorSupport;
+import org.apache.cassandra.index.sai.analyzer.filter.BuiltInAnalyzers;
+import org.apache.cassandra.service.ClientWarn;
+import org.assertj.core.api.Assertions;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class LuceneAnalyzerTest extends SAITester
+{
+    @Test
+    public void testQueryAnalyzer()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': '{\n" +
+                    "\t\"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}," +
+                    "\t\"filters\":[{\"name\":\"lowercase\"}]\n" +
+                    "}'," +
+                    "'query_analyzer': '{\n" +
+                    "\t\"tokenizer\":{\"name\":\"whitespace\"},\n" +
+                    "\t\"filters\":[{\"name\":\"porterstem\"}]\n" +
+                    "}'};");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'the query')");
+
+        flush();
+
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'query'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val = 'query'").size());
+    }
+
+    /**
+     * See CNDB-12739 for more details.
+     */
+    @Test
+    public void testQueryAnalyzerWithExtraData() throws Throwable
+    {
+        createTable("CREATE TABLE %s (c1 int PRIMARY KEY , c2 text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(c2) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                "'index_analyzer': '{" +
+                "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                "  \"filters\" : [ { \"name\" : \"lowercase\", \"args\": {} }, " +
+                "                  { \"name\" : \"edgengram\", \"args\": { \"minGramSize\":\"1\", \"maxGramSize\":\"30\" } }]," +
+                "  \"charFilters\" : []}', " +
+                "'query_analyzer': '{" +
+                "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                "  \"filters\" : [ {\"name\" : \"lowercase\",\"args\": {}} ]}'}");
+
+        execute("INSERT INTO %s(c1,c2) VALUES (1, 'astra quick fox')");
+        execute("INSERT INTO %s(c1,c2) VALUES (2, 'astra quick foxes')");
+        execute("INSERT INTO %s(c1,c2) VALUES (3, 'astra1')");
+        execute("INSERT INTO %s(c1,c2) VALUES (4, 'astra4 -1@a#')");
+
+        beforeAndAfterFlush(() -> assertEquals(4, execute("SELECT * FROM %s WHERE c2 :'ast' ").size()));
+    }
+
+    @Test
+    public void testStandardQueryAnalyzer()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': 'standard'};");
+
+        execute("INSERT INTO %s (id, val) VALUES (1, 'some row')");
+        execute("INSERT INTO %s (id, val) VALUES (2, 'a different row')");
+        execute("INSERT INTO %s (id, val) VALUES (3, 'a row with some and different but not together')");
+        execute("INSERT INTO %s (id, val) VALUES (4, 'a row with some different together')");
+        execute("INSERT INTO %s (id, val) VALUES (5, 'a row with some Different together but not same casing')");
+
+        flush();
+
+        // The query is parsed by the standard analyzer, so the query is tokenized by whitespace and lowercased
+        // and then we do an intersection on the results and get docs that have 'some' and 'different'
+        assertRows(execute("SELECT id FROM %s WHERE val : 'Some different'"), row(5), row(4), row(3));
+        assertRows(execute("SELECT id FROM %s WHERE val : 'some different'"), row(5), row(4), row(3));
+    }
+
+    @Test
+    public void testQueryAnalyzerBuiltIn()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': 'standard', 'query_analyzer': 'lowercase'};");
+
+        execute("INSERT INTO %s (id, val) VALUES (1, 'the query')");
+        execute("INSERT INTO %s (id, val) VALUES (2, 'my test Query')");
+        execute("INSERT INTO %s (id, val) VALUES (3, 'The Big Dog')");
+
+        // Some in sstable and some in memory
+        flush();
+
+        execute("INSERT INTO %s (id, val) VALUES (4, 'another QUERY')");
+        execute("INSERT INTO %s (id, val) VALUES (5, 'the fifth insert')");
+        execute("INSERT INTO %s (id, val) VALUES (6, 'MY LAST ENTRY')");
+
+        // Shows that the query term is lowercased to match all 'query' terms in the index
+        UntypedResultSet resultSet = execute("SELECT id FROM %s WHERE val : 'QUERY'");
+        assertRows(resultSet, row(1), row(2), row(4));
+
+        // add whitespace in front of query term and since it isn't tokenized by whitespace, we get no results
+        resultSet = execute("SELECT id FROM %s WHERE val : ' query'");
+        assertRows(resultSet);
+
+        // similarly, phrases do not match because index tokenized by whitespace (among other things) but the query
+        // is not
+        resultSet = execute("SELECT id FROM %s WHERE val : 'the query'");
+        assertRows(resultSet);
+    }
+
+    @Test
+    public void testDifferentIndexAndQueryAnalyzersWhenAppliedDuringPostFiltering()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, c1 text)");
+        // This test verifies a bug fix where the query analyzer was incorrectly used in place of the index analyzer.
+        // The analyzers are selected in conjunction with the column values and the query. Specifically,
+        // the index analyzer includes a lowercase filter but the query analyzer does not.
+        createIndex("CREATE CUSTOM INDEX ON %s(c1) USING 'StorageAttachedIndex' WITH OPTIONS =" +
+                    "{'index_analyzer': 'standard', 'query_analyzer': 'whitespace'}");
+
+        // The standard analyzer maps this to just one output 'the', but the query analyzer would map this to 'THE'
+        execute("INSERT INTO %s (pk, c1) VALUES (?, ?)", 1, "THE");
+
+        UntypedResultSet resultSet = execute("SELECT pk FROM %s WHERE c1 : 'the'");
+        assertRows(resultSet, row(1));
+    }
+
+    @Test
+    public void testCreateIndexWithQueryAnalyzerAndNoIndexAnalyzerFails()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, c1 text)");
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(c1) USING 'StorageAttachedIndex' WITH OPTIONS = " +
+                    "{'query_analyzer': 'whitespace'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Cannot specify query_analyzer without an index_analyzer option or any combination of " +
+                             "case_sensitive, normalize, or ascii options. options={query_analyzer=whitespace, target=c1}");
+    }
+
+    @Test
+    public void testCreateIndexWithNormalizersWorks()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, c1 text, c2 text, c3 text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(c1) USING 'StorageAttachedIndex' WITH OPTIONS = " +
+                    "{'query_analyzer': 'whitespace', 'case_sensitive': false}");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(c2) USING 'StorageAttachedIndex' WITH OPTIONS = " +
+                    "{'query_analyzer': 'whitespace', 'normalize': true}");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(c3) USING 'StorageAttachedIndex' WITH OPTIONS = " +
+                    "{'query_analyzer': 'whitespace', 'ascii': true}");
+    }
+
+    @Test
+    public void testStandardAnalyzerWithFullConfig()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': '{" +
+                    "    \"tokenizer\" : {\"name\" : \"standard\"}," +
+                    "    \"filters\" : [ {\"name\" : \"lowercase\"}] \n" +
+                    "  }'}");
+        standardAnalyzerTest();
+    }
+
+    @Test
+    public void testStandardAnalyzerWithBuiltInName()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard'}");
+        standardAnalyzerTest();
+    }
+
+    private void standardAnalyzerTest()
+    {
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'The quick brown fox jumps over the lazy DOG.')");
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val = 'The quick brown fox jumps over the lazy DOG.' ALLOW FILTERING").size());
+
+        flush();
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog' OR val : 'missing'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'missing1' OR val : 'missing2'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'dog' AND val : 'missing'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog' AND val : 'lazy'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog' AND val : 'quick' AND val : 'fox'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog' AND val : 'quick' OR val : 'missing'").size());
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog' AND (val : 'quick' OR val : 'missing')").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'missing' AND (val : 'quick' OR val : 'dog')").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog' OR (val : 'quick' AND val : 'missing')").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'missing' OR (val : 'quick' AND val : 'dog')").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'missing' OR (val : 'quick' AND val : 'missing')").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'missing cat' OR val : 'dog'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'missing cat' OR val : 'missing dog'").size());
+
+        // EQ operator support is reintroduced for analyzed columns, it should work as ':' operator
+        assertEquals(1, execute("SELECT * FROM %s WHERE val = 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val = 'The quick brown fox jumps over the lazy DOG.' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val = 'dog' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void testEmptyAnalyzerFailsAtCreation()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                                             "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                                             "WITH OPTIONS = { 'index_analyzer': '{}'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Analzyer config requires at least a tokenizer, a filter, or a charFilter, but none found. config={}");
+    }
+
+// FIXME re-enable exception detection once incompatible options have been purged from prod DBs
+    @Test
+    public void testIndexAnalyzerAndNonTokenizingAnalyzerFailsAtCreation()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX val_idx ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard', 'ascii': true}");
+        dropIndex("DROP INDEX %s.val_idx");
+
+        createIndex("CREATE CUSTOM INDEX val_idx ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard', 'normalize': true}");
+        dropIndex("DROP INDEX %s.val_idx");
+
+        createIndex("CREATE CUSTOM INDEX val_idx ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard', 'case_sensitive': false}");
+    }
+
+    // Technically, the NoopAnalyzer is applied, but that maps each field without modification, so any operator
+    // that matches the SAI field will also match the PK field when compared later in the search (there are two phases).
+    @Test
+    public void testNoAnalyzerOnClusteredColumn()
+    {
+        createTable("CREATE TABLE %s (id int, val text, PRIMARY KEY (id, val))");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (id, val) VALUES (1, 'dog')");
+
+        assertThatThrownBy(() -> execute("SELECT * FROM %s WHERE val : 'dog'"))
+        .isInstanceOf(InvalidRequestException.class);
+
+        // Equality still works because indexed value is not analyzed, and so the search can be performed without
+        // filtering.
+        assertEquals(1, execute("SELECT * FROM %s WHERE val = 'dog'").size());
+    }
+
+    @Test
+    public void testStandardAnalyzerInClusteringColumns()
+    {
+        createTable("CREATE TABLE %s (id int, val text, PRIMARY KEY (id, val))");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) WITH OPTIONS = { 'ascii': true }"
+        )).isInstanceOf(InvalidRequestException.class);
+
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                                             "WITH OPTIONS = { 'case_sesnsitive': false }"
+        )).isInstanceOf(InvalidRequestException.class);
+
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                                             "WITH OPTIONS = { 'normalize': true }"
+        )).isInstanceOf(InvalidRequestException.class);
+    }
+
+    @Test
+    public void testBogusAnalyzer()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(
+        () -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'lalalalaal'}"
+        )).isInstanceOf(InvalidQueryException.class);
+
+        assertThatThrownBy(
+        () -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = " +
+                         "{'index_analyzer':'{\"tokenizer\" : {\"name\" : \"lalala\"}}'}"
+        )).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void testStopFilterNoFormat()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'\n" +
+                                            "\t{\"tokenizer\":{\"name\" : \"whitespace\"},\n" +
+                                            "\t \"filters\":[{\"name\":\"stop\", \"args\": {\"words\": \"the,test\"}}]}'}");
+        verifyStopWordsLoadedCorrectly();
+    }
+
+    @Test
+    public void testStopFilterWordSet()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'\n" +
+                   "\t{\"tokenizer\":{\"name\" : \"whitespace\"},\n" +
+                   "\t \"filters\":[{\"name\":\"stop\", \"args\": {\"words\": \"the, test\", \"format\": \"wordset\"}}]}'}");
+        verifyStopWordsLoadedCorrectly();
+    }
+
+    @Test
+    public void testStopFilterSnowball()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        // snowball allows multiple words on the same line--they are broken up by whitespace
+        executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'\n" +
+                   "\t{\"tokenizer\":{\"name\" : \"whitespace\"},\n" +
+                   "\t \"filters\":[{\"name\":\"stop\", \"args\": {\"words\": \"the test\", \"format\": \"snowball\"}}]}'}");
+        verifyStopWordsLoadedCorrectly();
+
+    }
+
+    private void verifyStopWordsLoadedCorrectly()
+    {
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'the big test')");
+
+        flush();
+
+        assertRows(execute("SELECT id FROM %s WHERE val : 'the'"));
+        assertRows(execute("SELECT id FROM %s WHERE val : 'the test'"));
+        assertRows(execute("SELECT id FROM %s WHERE val : 'test'"));
+        assertRows(execute("SELECT id FROM %s WHERE val : 'the big'"), row("1"));
+        assertRows(execute("SELECT id FROM %s WHERE val : 'big'"), row("1"));
+        // the extra words shouldn't change the outcome because tokenizer is whitespace and tokens are matched then unioned
+        assertRows(execute("SELECT id FROM %s WHERE val : 'test some other words'"));
+    }
+
+    @Test
+    public void verifyEmptyStringIndexingBehaviorOnNonAnalyzedColumn()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, v text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+        execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 0, "");
+        flush();
+        assertRows(execute("SELECT * FROM %s WHERE v = ''"), row(0, ""));
+    }
+
+    @Test
+    public void verifyEmptyStringIndexingBehaviorOnAnalyzedColumn()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, v text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'standard'}");
+        execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 0, "");
+        execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 1, "some text to analyze");
+        flush();
+        assertRows(execute("SELECT * FROM %s WHERE v : ''"));
+        assertRows(execute("SELECT * FROM %s WHERE v : ' '"));
+    }
+
+    // The english analyzer has a default set of stop words. This test relies on "the" being one of those stop words.
+    @Test
+    public void testStopWordFilteringEdgeCases()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                   "WITH OPTIONS = {'index_analyzer':'english'}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'the test')");
+        // When indexing a document with only stop words, the document should not be indexed.
+        // Note: from looking at the collections implementation, these rows are filtered out before getting
+        // to the NoOpAnalyzer, which would otherwise return an empty buffer, which would lead to incorrectly
+        // indexing documents at the base of the trie.
+        execute("INSERT INTO %s (id, val) VALUES ('2', 'the')");
+
+        flush();
+
+        // Ensure row is there
+        assertRows(execute("SELECT id FROM %s WHERE val : 'test'"), row("1"));
+        // Ensure a query with only stop words results in no rows
+        assertRows(execute("SELECT id FROM %s WHERE val : 'the'"));
+        // Ensure that the AND is correctly applied so that we get no results
+        assertRows(execute("SELECT id FROM %s WHERE val : 'the' AND val : 'test'"));
+    }
+
+    @Test
+    public void testCharfilter()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'{\n" +
+                    "\t\"tokenizer\":{\"name\":\"keyword\"},\n" +
+                    "\t\"charFilters\":[{\"name\":\"htmlstrip\"}]\n" +
+                    "}'}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', '<b>hello</b>')");
+
+        flush();
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'hello'").size());
+    }
+
+    @Test
+    public void testNGramfilter()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        String ddl = "CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'\n" +
+                     "\t{\"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}," +
+                     "\t\"filters\":[{\"name\":\"lowercase\"}]}'}";
+        createIndex(ddl);
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'DoG')");
+
+        flush();
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'do'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'og'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+    }
+
+    @Test
+    public void testNGramfilterNoFlush()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'\n" +
+                    "\t{\"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}," +
+                    "\t\"filters\":[{\"name\":\"lowercase\"}]}'}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'DoG')");
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'do'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'og'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+    }
+
+    @Test
+    public void testEdgeNgramFilterWithOR() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': '{\n" +
+                    "\t\"tokenizer\":{\"name\":\"standard\", \"args\":{}}," +
+                    "\t\"filters\":[{\"name\":\"lowercase\", \"args\":{}}, " +
+                    "{\"name\":\"edgengram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"30\"}}],\n" +
+                    "\t\"charFilters\":[]" +
+                    "}'};");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 some other words')");
+        execute("INSERT INTO %s (id, val) VALUES ('4', 'WFS7093AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('5', 'WFS0565AU')");
+
+        beforeAndAfterFlush(() -> {
+            // match (:)
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL0133AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'WFS2684AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : ''").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val : 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : '' OR val : 'WFS2684AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : '' AND val : 'WFS2684AU'").size());
+
+            // equals (=)
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL0133AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'WFS2684AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = ''").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val = 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = '' OR val = 'WFS2684AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = '' AND val = 'WFS2684AU'").size());
+
+            // mixed match (:) and equals (=)
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val : 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = '' OR val : 'WFS2684AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = '' AND val : 'WFS2684AU'").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val = 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : '' OR val = 'WFS2684AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : '' AND val = 'WFS2684AU'").size());
+        });
+    }
+
+    @Test
+    public void testNgramFilterWithOR() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': '{\n" +
+                    "\t\"tokenizer\":{\"name\":\"standard\", \"args\":{}}," +
+                    "\t\"filters\":[{\"name\":\"lowercase\", \"args\":{}}, " +
+                    "{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"30\"}}],\n" +
+                    "\t\"charFilters\":[]" +
+                    "}'};");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 some other words')");
+        execute("INSERT INTO %s (id, val) VALUES ('4', 'WFS7093AU')");
+        execute("INSERT INTO %s (id, val) VALUES ('5', 'WFS0565AU')");
+
+        beforeAndAfterFlush(() -> {
+            // match (:)
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL0133AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : '268'").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val : 'WFS2684AU'").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : '133' OR val : 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL' AND val : 'AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : 'XYZ' AND val : 'AU'").size());
+
+            // equals (=)
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL0133AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = '268'").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val = 'WFS2684AU'").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = '133' OR val = 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL' AND val = 'AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = 'XYZ' AND val = 'AU'").size());
+
+            // mixed match (:) and equals (=)
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val = 'WFS2684AU'").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val : '133' OR val = 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL' AND val = 'AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val : 'XYZ' AND val = 'AU'").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val : 'WFS2684AU'").size());
+            assertEquals(2, execute("SELECT val FROM %s WHERE val = '133' OR val : 'WFS2684AU'").size());
+            assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL' AND val : 'AU'").size());
+            assertEquals(0, execute("SELECT val FROM %s WHERE val = 'XYZ' AND val : 'AU'").size());
+        });
+    }
+    @Test
+    public void testWhitespace()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS =" +
+                    "{'index_analyzer':'whitespace'}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'hello world twice the and')");
+
+        flush();
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'hello'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'twice'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'the'").size()); // test stop word
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'and'").size()); // test stop word
+    }
+
+    @Test
+    public void testWhitespaceLowercase()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'\n" +
+                    "\t{\"tokenizer\":{\"name\":\"whitespace\"}," +
+                    "\t\"filters\":[{\"name\":\"lowercase\"}]}'}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'hELlo woRlD tWice tHe aNd')");
+
+        flush();
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'hello'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'twice'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'the'").size()); // test stop word
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'and'").size()); // test stop word
+    }
+
+    @Test
+    public void testTokenizer()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'\n" +
+                    "\t{\"tokenizer\":{\"name\":\"whitespace\"}," +
+                    "\t\"filters\":[{\"name\":\"porterstem\"}]}'}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'the queries test')");
+
+        flush();
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'the'").size()); // stop word test
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'query'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'query' OR val : 'missing'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'queries' AND val : 'the' AND val : 'test'").size());
+    }
+
+    @Test
+    public void testMixedAnalyzerMatchesAndEquality() // there are more detailed tests in AnalyzerEqOperatorSupportTest
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        String createIndexQuery = "CREATE CUSTOM INDEX ON %%s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                                  "'index_analyzer':'{\n" +
+                                  "   \"tokenizer\":{\"name\":\"whitespace\"}," +
+                                  "   \"filters\":[{\"name\":\"porterstem\"}]" +
+                                  "}'," +
+                                  "'equals_behaviour_when_analyzed': '%s'}";
+        createIndex(String.format(createIndexQuery, AnalyzerEqOperatorSupport.Value.MATCH));
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'the queries test')");
+
+        // we'll test AND and OR, with both MATCH and EQ and the first operator in the mix
+        final String conjunctionQueryMatchEq = "SELECT id FROM %s WHERE val : 'queries' AND val : 'the' AND val = 'the queries test'";
+        final String conjunctionQueryEqMatch = "SELECT id FROM %s WHERE val = 'queries' AND val = 'the' AND val : 'the queries test'";
+        final String disjunctionQueryMatchEq = "SELECT id FROM %s WHERE val : 'queries' OR val : 'the' OR val = 'blah, blah, blah'";
+        final String disjunctionQueryEqMatch = "SELECT id FROM %s WHERE val = 'queries' OR val = 'the' OR val : 'blah, blah, blah'";
+
+        // if the index supports EQ, the mixed queries should work as the operators are considered the same
+        for (String query : Arrays.asList(conjunctionQueryMatchEq, conjunctionQueryEqMatch, disjunctionQueryMatchEq, disjunctionQueryEqMatch))
+        {
+            assertRows(execute(query), row("1"));
+            assertRows(execute(query + "ALLOW FILTERING"), row("1"));
+        }
+
+        // recreate the index with 'equals_behaviour_when_analyzed': 'UNSUPPORTED'
+        dropIndex("DROP INDEX %s." + currentIndex());
+        createIndex(String.format(createIndexQuery, AnalyzerEqOperatorSupport.Value.UNSUPPORTED));
+
+        // If the index does not support EQ, the mixed queries should fail.
+        // The error message will slightly change depending on whether EQ or MATCH are before in the query.
+
+        Assertions.assertThatThrownBy(() -> execute(conjunctionQueryMatchEq))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining(String.format(SingleColumnRestriction.AnalyzerMatchesRestriction.CANNOT_BE_MERGED_ERROR, "val"));
+        Assertions.assertThatThrownBy(() -> execute(conjunctionQueryMatchEq + "ALLOW FILTERING"))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining(String.format(SingleColumnRestriction.AnalyzerMatchesRestriction.CANNOT_BE_MERGED_ERROR, "val"));
+
+        Assertions.assertThatThrownBy(() -> execute(conjunctionQueryEqMatch))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining(String.format(SingleColumnRestriction.EQRestriction.CANNOT_BE_MERGED_ERROR, "val"));
+        Assertions.assertThatThrownBy(() -> execute(conjunctionQueryEqMatch + "ALLOW FILTERING"))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining(String.format(SingleColumnRestriction.EQRestriction.CANNOT_BE_MERGED_ERROR, "val"));
+
+        Assertions.assertThatThrownBy(() -> execute(disjunctionQueryMatchEq))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+         assertRows(execute(disjunctionQueryMatchEq + "ALLOW FILTERING"), row("1"));
+
+        Assertions.assertThatThrownBy(() -> execute(disjunctionQueryEqMatch))
+                  .isInstanceOf(InvalidRequestException.class)
+                  .hasMessageContaining(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+         assertRows(execute(disjunctionQueryEqMatch + "ALLOW FILTERING"));
+    }
+
+    @Test
+    public void testBuiltInAlyzerIndexCreation()
+    {
+        for (BuiltInAnalyzers builtInAnalyzer : BuiltInAnalyzers.values())
+            testBuiltInAlyzerIndexCreationFor(builtInAnalyzer.name());
+    }
+
+    private void testBuiltInAlyzerIndexCreationFor(String builtInAnalyzerName)
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = " +
+                    "{'index_analyzer':'" + builtInAnalyzerName + "'}");
+    }
+
+    @Test
+    public void testInvalidQueryOnNumericColumn()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, some_num tinyint)");
+        createIndex("CREATE CUSTOM INDEX ON %s(some_num) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (id, some_num) VALUES (1, 1)");
+        flush();
+
+        assertThatThrownBy(() -> execute("SELECT * FROM %s WHERE some_num : 1"))
+        .isInstanceOf(InvalidRequestException.class);
+    }
+
+    @Test
+    public void testLegacyEqQueryOnNormalizedTextColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'ascii': 'true', 'case_sensitive': 'false', 'normalize': 'true'}");
+
+        execute("INSERT INTO %s (id, val) VALUES (1, 'Aaą')");
+
+        beforeAndAfterFlush(() -> assertEquals(1, execute("SELECT * FROM %s WHERE val = 'aaa'").size()));
+    }
+
+    @Test
+    public void testAnalyzerThatProducesTooManyBytesIsRejectedAtWriteTime()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'{" +
+                    "\"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"1\", \"maxGramSize\":\"26\"}},\n" +
+                    "\"filters\":[{\"name\":\"lowercase\"}]}'}");
+        String query = "INSERT INTO %s (id, val) VALUES (0, 'abcdedfghijklmnopqrstuvwxyz abcdedfghijklmnopqrstuvwxyz')";
+
+        boolean validate = CassandraRelevantProperties.SAI_VALIDATE_MAX_TERM_SIZE_AT_COORDINATOR.getBoolean();
+        try
+        {
+            CassandraRelevantProperties.SAI_VALIDATE_MAX_TERM_SIZE_AT_COORDINATOR.setBoolean(false);
+            execute(query);
+
+            CassandraRelevantProperties.SAI_VALIDATE_MAX_TERM_SIZE_AT_COORDINATOR.setBoolean(true);
+            assertThatThrownBy(() -> execute(query))
+                    .hasMessage("Term's analyzed size for column val exceeds the cumulative limit for index. Max allowed size 8.000KiB.")
+                    .isInstanceOf(InvalidRequestException.class);
+        }
+        finally
+        {
+            CassandraRelevantProperties.SAI_VALIDATE_MAX_TERM_SIZE_AT_COORDINATOR.setBoolean(validate);
+        }
+    }
+
+    @Test
+    public void testInvalidNamesOnConfig()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+
+        // Empty config
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'{}'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Analzyer config requires at least a tokenizer, a filter, or a charFilter, but none found. config={}");
+
+        var invalidCharfilters = "{\"tokenizer\" : {\"name\" : \"keyword\"},\"charfilters\" : [{\"name\" : \"htmlstrip\"}]}";
+
+        // Invalid config name, charfilters should be charFilters
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'" + invalidCharfilters + "'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Invalid field name 'charfilters' in analyzer config. Valid fields are: [tokenizer, filters, charFilters]");
+
+        // Invalid config name on query_analyzer, charfilters should be charFilters
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'standard', 'query_analyzer':'" + invalidCharfilters + "'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Invalid field name 'charfilters' in analyzer config. Valid fields are: [tokenizer, filters, charFilters]");
+
+        // Invalid tokenizer name
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'{\"tokenizer\":{\"name\" : \"invalid\"}}'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Unknown tokenizer 'invalid'. Valid options: [");
+
+        // Invalid filter name
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'{\"tokenizer\":{\"name\" : \"keyword\"},\n" +
+                                             "    \"filters\":[{\"name\" : \"invalid\"}]}'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Unknown filter 'invalid'. Valid options: [");
+
+        // Invalid charFilter name
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'{\"tokenizer\":{\"name\" : \"keyword\"},\n" +
+                                             "    \"charFilters\":[{\"name\" : \"invalid\"}]}'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Unknown charFilter 'invalid'. Valid options: [");
+
+        // Missing one of the params in the args field
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'{\"tokenizer\":{\"name\" : \"keyword\"},\n" +
+                                             "    \"filters\":[{\"name\" : \"synonym\", \"args\" : {\"words\" : \"as => like\"}},\n" +
+                                             "    {\"name\" : \"lowercase\"}]}'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Error configuring analyzer's filter 'synonym': Configuration Error: missing parameter 'synonyms'");
+
+
+        // Missing one of the params in the args field
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer':'{\"tokenizer\":{\"name\" : \"keyword\"},\n" +
+                                             "    \"filters\":[{\"name\" : \"synonym\", \"args\" : {\"synonyms\" : \"as => like\", \"extraParam\": \"xyz\"}},\n" +
+                                             "    {\"name\" : \"lowercase\"}]}'}"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Error configuring analyzer's filter 'synonym': Unknown parameters: {extraParam=xyz}");
+    }
+
+    @Test
+    public void testHighNumberOfMatchPredicates()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': '{" +
+                    "    \"tokenizer\" : {\n" +
+                    "          \"name\" : \"ngram\",\n" +
+                    "          \"args\" : {\n" +
+                    "            \"minGramSize\":\"2\",\n" +
+                    "            \"maxGramSize\":\"3\"\n" +
+                    "          }\n" +
+                    "    }," +
+                    "    \"filters\" : [ {\"name\" : \"lowercase\"}] \n" +
+                    "  }'}");
+
+        // Long enough to generate several tens of thousands of ngrams:
+        String longParam = "The quick brown fox jumps over the lazy DOG.".repeat(250);
+
+        // Generally we expect this query to run in < 25 ms each on reasonably performant
+        // hardware, but because CI performance can have a lot of variability,
+        // we take the minimum, and we allow a large margin to avoid random failures.
+        var count = 5;
+        var minElapsed = Long.MAX_VALUE;
+        for (int i = 0; i < count; i++)
+        {
+            var startTime = System.currentTimeMillis();
+            execute("SELECT * FROM %s WHERE val : '" + longParam + '\'');
+            var elapsed = System.currentTimeMillis() - startTime;
+            if (elapsed < minElapsed)
+                minElapsed = elapsed;
+            // In extreme case we just want to bail out after the first iteration
+            if (elapsed > 10000)
+                break;
+        }
+        assertTrue("Query too slow: " + minElapsed + " ms", minElapsed < 1000);
+    }
+
+    @Test
+    public void testClientWarningOnNGram()
+    {
+        // no explicit analyzer
+        assertNoWarning("{}");
+        assertNoWarning("{'ascii': 'true', 'case_sensitive': 'false', 'normalize': 'true'}");
+
+        // standard analyzer
+        assertNoWarning("{'index_analyzer': 'standard'}");
+        assertNoWarning("{'index_analyzer': 'whitespace'}");
+
+        // custom non-ngram analyzer
+
+        // ngram analyzer without query_analyzer
+        assertClientWarningOnNGram("{'index_analyzer': '{" +
+                                   "   \"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}," +
+                                   "   \"filters\":[{\"name\":\"lowercase\"}]}'}");
+        assertClientWarningOnNGram("{'index_analyzer': '{" +
+                                   "   \"tokenizer\":{\"name\":\"whitespace\"}," +
+                                   "   \"filters\":[{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}]}'}");
+
+        // ngram analyzer with ngram query_analyzer
+        assertNoWarning("{'index_analyzer': '{" +
+                        "   \"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}}'," +
+                        "'query_analyzer': '{" +
+                        "   \"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}}'}");
+
+        // ngram analyzer with non-ngram query_analyzer
+        assertNoWarning("{'index_analyzer': '{" +
+                        "   \"tokenizer\":{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"3\"}}," +
+                        "   \"filters\":[{\"name\":\"lowercase\"}]}'," +
+                        "'query_analyzer': '{" +
+                        "   \"tokenizer\":{\"name\":\"whitespace\"}," +
+                        "   \"filters\":[{\"name\":\"porterstem\"}]}'}");
+    }
+
+    @Test
+    public void testAnalyzerOnSet() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, genres set<text>)");
+        execute("INSERT INTO %s (id, genres) VALUES ('1', {'Horror', 'comedy'})");
+
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS 'Horror' ALLOW FILTERING"), row("1"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS 'Horror' ALLOW FILTERING"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS 'horror' ALLOW FILTERING"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS 'horror' ALLOW FILTERING"), row("1"));
+
+        createIndex("CREATE CUSTOM INDEX ON %s(genres) USING 'StorageAttachedIndex' WITH OPTIONS = { 'index_analyzer':'STANDARD'}");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS 'horror'"), row("1"));
+            assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS 'horror'"));
+        });
+    }
+
+    @Test
+    public void testAnalyzerOnSetWithDistinctQueryAnalyzer() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c int, v set<text>, PRIMARY KEY(k, c))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': '{" +
+                    "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                    "  \"filters\" : [ { \"name\" : \"lowercase\", \"args\": {} }, " +
+                    "                  { \"name\" : \"edgengram\", \"args\": { \"minGramSize\":\"1\", \"maxGramSize\":\"30\" } }]," +
+                    "  \"charFilters\" : []}', " +
+                    "'query_analyzer': '{" +
+                    "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                    "  \"filters\" : [ {\"name\" : \"lowercase\",\"args\": {}} ]}'}");
+
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, {'astra quick fox', 'astra quick foxes', 'astra4', 'astra5 -1@a#', 'lazy dog'})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 2, {'astra quick fox'})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 3, {'astra quick foxes'})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 4, {'astra4'})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 5, {'astra5 -1@a#'})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 6, {'lazy dog'})");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'ast'"), row(1), row(2), row(3), row(4), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'astra'"), row(1), row(2), row(3), row(4), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'astra4'"), row(1), row(4));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'astra5'"), row(1), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'astra9'"));
+
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'ast'"), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'astra'"), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'astra4'"), row(2), row(3), row(5), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'astra5'"), row(2), row(3), row(4), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'astra9'"), row(1), row(2), row(3), row(4), row(5), row(6));
+        });
+    }
+
+    @Test
+    public void testAnalyzerOnList() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, genres list<text>)");
+        execute("INSERT INTO %s (id, genres) VALUES ('1', ['Horror', 'comedy'])");
+
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS 'Horror' ALLOW FILTERING"), row("1"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS 'Horror' ALLOW FILTERING"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS 'horror' ALLOW FILTERING"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS 'horror' ALLOW FILTERING"), row("1"));
+
+        createIndex("CREATE CUSTOM INDEX ON %s(genres) USING 'StorageAttachedIndex' WITH OPTIONS = { 'index_analyzer':'STANDARD'}");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS 'horror'"), row("1"));
+            assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS 'horror'"));
+        });
+    }
+
+    @Test
+    public void testAnalyzerOnListWithDistinctQueryAnalyzer() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c int, v list<text>, PRIMARY KEY(k, c))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': '{" +
+                    "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                    "  \"filters\" : [ { \"name\" : \"lowercase\", \"args\": {} }, " +
+                    "                  { \"name\" : \"edgengram\", \"args\": { \"minGramSize\":\"1\", \"maxGramSize\":\"30\" } }]," +
+                    "  \"charFilters\" : []}', " +
+                    "'query_analyzer': '{" +
+                    "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                    "  \"filters\" : [ {\"name\" : \"lowercase\",\"args\": {}} ]}'}");
+
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, ['astra quick fox', 'astra quick foxes', 'astra4', 'astra5 -1@a#', 'lazy dog'])");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 2, ['astra quick fox'])");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 3, ['astra quick foxes'])");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 4, ['astra4'])");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 5, ['astra5 -1@a#'])");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 6, ['lazy dog'])");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'ast'"), row(1), row(2), row(3), row(4), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'astra'"), row(1), row(2), row(3), row(4), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'astra4'"), row(1), row(4));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'astra5'"), row(1), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS 'astra9'"));
+
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'ast'"), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'astra'"), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'astra4'"), row(2), row(3), row(5), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'astra5'"), row(2), row(3), row(4), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS 'astra9'"), row(1), row(2), row(3), row(4), row(5), row(6));
+        });
+    }
+
+    @Test
+    public void testAnalyzerOnMapKeys() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, genres map<text, int>)");
+        execute("INSERT INTO %s (id, genres) VALUES ('1', {'Horror' : 1, 'comedy' : 2})");
+
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS KEY 'Horror' ALLOW FILTERING"), row("1"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS KEY 'Horror' ALLOW FILTERING"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS KEY 'horror' ALLOW FILTERING"));
+        assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS KEY 'horror' ALLOW FILTERING"), row("1"));
+
+        createIndex("CREATE CUSTOM INDEX ON %s(KEYS(genres)) USING 'StorageAttachedIndex' WITH OPTIONS = { 'index_analyzer':'STANDARD'}");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsNet(executeNet("SELECT id FROM %s WHERE genres CONTAINS KEY 'horror'"), row("1"));
+            assertRowsNet(executeNet("SELECT id FROM %s WHERE genres NOT CONTAINS KEY 'horror'"));
+        });
+    }
+
+    @Test
+    public void testAnalyzerOnMapKeysWithDistinctQueryAnalyzer() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c int, v map<text, int>, PRIMARY KEY(k, c))");
+        createIndex("CREATE CUSTOM INDEX ON %s(KEYS(v)) USING 'StorageAttachedIndex' WITH OPTIONS = {" +
+                    "'index_analyzer': '{" +
+                    "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                    "  \"filters\" : [ { \"name\" : \"lowercase\", \"args\": {} }, " +
+                    "                  { \"name\" : \"edgengram\", \"args\": { \"minGramSize\":\"1\", \"maxGramSize\":\"30\" } }]," +
+                    "  \"charFilters\" : []}', " +
+                    "'query_analyzer': '{" +
+                    "  \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
+                    "  \"filters\" : [ {\"name\" : \"lowercase\",\"args\": {}} ]}'}");
+
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 1, {'astra quick fox':0, 'astra quick foxes':0, 'astra4':0, 'astra5 -1@a#':0, 'lazy dog':0})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 2, {'astra quick fox':0})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 3, {'astra quick foxes':0})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 4, {'astra4':0})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 5, {'astra5 -1@a#':0})");
+        execute("INSERT INTO %s (k, c, v) VALUES (0, 6, {'lazy dog':0})");
+
+        beforeAndAfterFlush(() -> {
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS KEY 'ast'"), row(1), row(2), row(3), row(4), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS KEY 'astra'"), row(1), row(2), row(3), row(4), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS KEY 'astra4'"), row(1), row(4));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS KEY 'astra5'"), row(1), row(5));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v CONTAINS KEY 'astra9'"));
+
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS KEY 'ast'"), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS KEY 'astra'"), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS KEY 'astra4'"), row(2), row(3), row(5), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS KEY 'astra5'"), row(2), row(3), row(4), row(6));
+            assertRowsNet(executeNet("SELECT c FROM %s WHERE v NOT CONTAINS KEY 'astra9'"), row(1), row(2), row(3), row(4), row(5), row(6));
+        });
+    }
+
+    private void assertClientWarningOnNGram(String indexOptions)
+    {
+        createIndexFromOptions(indexOptions);
+        Assertions.assertThat(ClientWarn.instance.getWarnings())
+                  .hasSize(1)
+                  .allMatch(w -> w.contains(StorageAttachedIndex.NGRAM_WITHOUT_QUERY_ANALYZER_WARNING));
+    }
+
+    private void assertNoWarning(String indexOptions)
+    {
+        createIndexFromOptions(indexOptions);
+        Assertions.assertThat(ClientWarn.instance.getWarnings()).isNull();
+    }
+
+    private void createIndexFromOptions(String options)
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        ClientWarn.instance.captureWarnings();
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = " + options);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/LuceneUpdateDeleteTest.java b/test/unit/org/apache/cassandra/index/sai/cql/LuceneUpdateDeleteTest.java
new file mode 100644
index 000000000000..227e5bd38160
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/LuceneUpdateDeleteTest.java
@@ -0,0 +1,641 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.conditions.ColumnCondition;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+
+import static org.apache.cassandra.index.sai.cql.VectorTypeTest.assertContainsInt;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class LuceneUpdateDeleteTest extends SAITester
+{
+    @Test
+    public void updateAndDeleteWithAnalyzerRestrictionQueryShouldFail()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (id, val) VALUES (0, 'a sad doG.')");
+
+        // Prove we can get the row back
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+
+        // DELETE fails
+        assertThatThrownBy(() -> execute("DELETE FROM %s WHERE val : 'dog'"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Invalid query. DELETE does not support use of secondary indices, but val : 'dog' restriction requires a secondary index.");
+
+        // UPDATE fails
+        assertThatThrownBy(() -> execute("UPDATE %s SET val = 'something new' WHERE val : 'dog'"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Invalid query. UPDATE does not support use of secondary indices, but val : 'dog' restriction requires a secondary index.");
+
+        // UPDATE with LWT fails (different error message because it fails at a different point)
+        assertThatThrownBy(() -> execute("UPDATE %s SET val = 'something new' WHERE id = 0 IF val : 'dog'"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining(ColumnCondition.ANALYZER_MATCHES_ERROR);
+    }
+
+    // No flushes
+    @Test
+    public void removeUpdateAndDeleteTextInMemoryTest()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        // The analyzed text column will result in overlapping and non-overlapping tokens in the in memory trie map.
+        // Note that capitalization is covered as well as tokenization.
+        execute("INSERT INTO %s (id, val) VALUES (0, 'a sad doG.')");
+        execute("INSERT INTO %s (id, val) VALUES (1, 'A Happy DOG.')");
+
+        // Prove initial assumptions about data structures are correct.
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("UPDATE %s SET val = null WHERE id = 0");
+
+        // Prove that we can remove a row when we update the data
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("UPDATE %s SET val = 'the dog' WHERE id = 0");
+
+        // Prove that we can remove a row when we update the data
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'the'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("DELETE from %s WHERE id = 1");
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+    }
+
+    // Flush after every insert/update/delete
+    @Test
+    public void removeUpdateAndDeleteTextOnDiskTest()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        // The analyzed text column will result in overlapping and non-overlapping tokens in the in memory trie map.
+        // Note that capitalization is covered as well as tokenization.
+        execute("INSERT INTO %s (id, val) VALUES (0, 'a sad doG.')");
+        execute("INSERT INTO %s (id, val) VALUES (1, 'A Happy DOG.')");
+
+        flush();
+
+        // Prove initial assumptions about data structures are correct.
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("UPDATE %s SET val = null WHERE id = 0");
+        flush();
+
+        // Prove that we can remove a row when we update the data
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("UPDATE %s SET val = 'the dog' WHERE id = 0");
+        flush();
+
+        // Prove that we can remove a row when we update the data
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'the'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("DELETE from %s WHERE id = 1");
+        flush();
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("INSERT INTO %s (id, val) VALUES (1, 'A Happy DOG.')");
+        flush();
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'the'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+    }
+
+    // Insert entries, flush them, then perform updates without flushing.
+    @Test
+    public void removeUpdateAndDeleteTextMixInMemoryOnDiskTest()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        // The analyzed text column will result in overlapping and non-overlapping tokens in the in memory trie map.
+        // Note that capitalization is covered as well as tokenization.
+        execute("INSERT INTO %s (id, val) VALUES (0, 'a sad doG.')");
+        execute("INSERT INTO %s (id, val) VALUES (1, 'A Happy DOG.')");
+
+        flush();
+
+        // Prove initial assumptions about data structures are correct.
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("UPDATE %s SET val = null WHERE id = 0");
+
+        // Prove that we can remove a row when we update the data
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("UPDATE %s SET val = 'the dog' WHERE id = 0");
+
+        // Prove that we can remove a row when we update the data
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'the'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("DELETE from %s WHERE id = 1");
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+
+        execute("INSERT INTO %s (id, val) VALUES (1, 'A Happy DOG.')");
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'the'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE val : 'dog'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'a'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE val : 'happy'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE val : 'sad'").size());
+    }
+
+    // row delete will trigger UpdateTransaction#onUpdated
+    @Test
+    public void rowDeleteRowInMemoryAndFlushTest()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, str_val text, val text, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, ck, str_val, val) VALUES (0, 0, 'A', 'dog 0')");
+        execute("INSERT INTO %s (pk, ck, str_val, val) VALUES (1, 1, 'B', 'dog 1')");
+        execute("DELETE from %s WHERE pk = 1 and ck = 1");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE val : 'dog'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+
+        flush();
+
+        result = execute("SELECT * FROM %s WHERE val : 'dog'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+    }
+
+    // range delete won't trigger UpdateTransaction#onUpdated
+    @Test
+    public void rangeDeleteRowInMemoryAndFlushTest()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, ck2 int, str_val text, val text, PRIMARY KEY(pk, ck, ck2))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, ck, ck2, str_val, val) VALUES (0, 0, 0, 'A', 'first insert')");
+        execute("INSERT INTO %s (pk, ck, ck2, str_val, val) VALUES (1, 1, 1, 'B', 'second insert')");
+        execute("DELETE from %s WHERE pk = 1 and ck = 1");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+
+        flush();
+
+        result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+    }
+
+    @Test
+    public void updateRowInMemoryAndFlushTest()
+    {
+        createTable("CREATE TABLE %s (pk int, str_val text, val text, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', 'first insert')");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', 'second insert')");
+        execute("UPDATE %s SET val = null WHERE pk = 1");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+
+        flush();
+
+        result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+    }
+
+    @Test
+    public void deleteRowPostFlushTest()
+    {
+        createTable("CREATE TABLE %s (pk int, str_val text, val text, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', 'first insert')");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', 'second insert')");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(2);
+        flush();
+
+        execute("UPDATE %s SET val = null WHERE pk = 0");
+        result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 1);
+
+        execute("DELETE from %s WHERE pk = 1");
+        result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).isEmpty();
+        flush();
+
+        result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).isEmpty();
+    }
+
+    @Test
+    public void deletedInOtherSSTablesTest()
+    {
+        createTable("CREATE TABLE %s (pk int, str_val text, val text, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', 'first insert')");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', 'second insert')");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'C', 'third insert')");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE val : 'first'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+        flush();
+
+        execute("DELETE from %s WHERE pk = 0");
+        execute("DELETE from %s WHERE pk = 1");
+        result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 2);
+    }
+
+    @Test
+    public void deletedInOtherSSTablesMultiIndexTest()
+    {
+        createTable("CREATE TABLE %s (pk int, str_val text, val text, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', 'first insert')");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'A', 'second insert')");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'A', 'third insert')");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE str_val = 'A' AND val : 'first'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+        flush();
+
+        execute("DELETE from %s WHERE pk = 0");
+        execute("DELETE from %s WHERE pk = 1");
+        result = execute("SELECT * FROM %s WHERE str_val = 'A' AND val : 'insert'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 2);
+    }
+
+    @Test
+    public void rangeDeletedInOtherSSTablesTest()
+    {
+        createTable("CREATE TABLE %s (pk int, ck1 int, ck2 int, str_val text, val text, PRIMARY KEY(pk, ck1, ck2))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, ck1, ck2, str_val, val) VALUES (0, 0, 1, 'A', 'first insert')");
+        execute("INSERT INTO %s (pk, ck1, ck2, str_val, val) VALUES (0, 0, 2, 'B', 'second insert')");
+        execute("INSERT INTO %s (pk, ck1, ck2, str_val, val) VALUES (0, 1, 3, 'C', 'third insert')");
+        execute("INSERT INTO %s (pk, ck1, ck2, str_val, val) VALUES (0, 1, 4, 'D', 'fourth insert')");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(4);
+        flush();
+
+        execute("DELETE from %s WHERE pk = 0 and ck1 = 0");
+
+        result = execute("SELECT * FROM %s WHERE val : 'insert'");
+        assertThat(result).hasSize(2);
+        assertContainsInt(result, "ck2", 3);
+        assertContainsInt(result, "ck2", 4);
+    }
+
+    @Test
+    public void partitionDeletedInOtherSSTablesTest()
+    {
+        createTable("CREATE TABLE %s (pk int, ck1 int, ck2 int, str_val text, val text, PRIMARY KEY(pk, ck1, ck2))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, ck1, ck2, str_val, val) VALUES (0, 0, 1, 'A', 'some text')");
+        execute("INSERT INTO %s (pk, ck1, ck2, str_val, val) VALUES (0, 0, 2, 'B', 'updated text')");
+        execute("INSERT INTO %s (pk, ck1, ck2, str_val, val) VALUES (1, 1, 3, 'C', 'another text')");
+        execute("INSERT INTO %s (pk, ck1, ck2, str_val, val) VALUES (1, 1, 4, 'D', 'more text')");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE val : 'updated'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+        flush();
+
+        execute("DELETE from %s WHERE pk = 0");
+
+        result = execute("SELECT * FROM %s WHERE val : 'another'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 1);
+
+
+        result = execute("SELECT * FROM %s WHERE val : 'text'");
+        assertThat(result).hasSize(2);
+    }
+
+    @Test
+    public void upsertTest()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, not_analyzed text, val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (1, 'B', 'different tokenized text')");
+
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE val : 'tokenized'");
+        assertThat(result).hasSize(2);
+        assertContainsInt(result, "pk", 0);
+        assertContainsInt(result, "pk", 1);
+        flush();
+
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        execute("INSERT INTO %s (pk, not_analyzed, val) VALUES (0, 'A', 'this will be tokenized')");
+        result = execute("SELECT * FROM %s WHERE val : 'tokenized'");
+        assertThat(result).hasSize(2);
+        assertContainsInt(result, "pk", 0);
+        assertContainsInt(result, "pk", 1);
+        flush();
+
+        result = execute("SELECT * FROM %s WHERE val : 'tokenized'");
+        assertThat(result).hasSize(2);
+        assertContainsInt(result, "pk", 0);
+        assertContainsInt(result, "pk", 1);
+    }
+
+    @Test
+    public void updateOtherColumnsTest()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, val text, not_analyzed text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (id, val, not_analyzed) VALUES (0, 'a sad doG.', 'more text')");
+        execute("INSERT INTO %s (id, val, not_analyzed) VALUES (1, 'A Happy DOG.', 'different text')");
+        execute("UPDATE %s SET not_analyzed='A' WHERE id=0");
+
+        var result = execute("SELECT * FROM %s WHERE val : 'dog'");
+        assertThat(result).hasSize(2);
+    }
+
+    @Test
+    public void updateManySSTablesTest()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'this is')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'a test')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'of the emergency')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'broadcast system')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'this is only')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'a test')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'if this were')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'a real emergency')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'you would be instructed')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'where to tune in your area')");
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'for news and official information')");
+        flush();
+
+        var result = execute("SELECT * FROM %s WHERE val : 'news'");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+        result = execute("SELECT * FROM %s WHERE val : 'this'");
+        assertThat(result).hasSize(0);
+    }
+
+    @Test
+    public void shadowedPrimaryKeyInDifferentSSTable()
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, str_val text, val text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) " +
+                    "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+        disableCompaction(KEYSPACE);
+
+        // flush a sstable with one vector
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', 'an indexed phrase')");
+        flush();
+
+        // flush another sstable to shadow the vector row
+        execute("DELETE FROM %s where pk = 0");
+        flush();
+
+        // flush another sstable with one new vector row
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', 'something different')");
+        flush();
+
+        // the shadow vector has the highest score
+        var result = execute("SELECT * FROM %s WHERE val : 'something'");
+        assertThat(result).hasSize(1);
+    }
+
+    @Test
+    public void testRangeDeletionThenOverwrite() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, x int, val text, primary key(pk, x))");
+        var indexName = createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+        waitForTableIndexesQueryable();
+
+        execute("INSERT INTO %s (pk, x, val) VALUES (0, 0, 'an indexed phrase')");
+        execute("INSERT INTO %s (pk, x, val) VALUES (0, 1, 'something random')");
+        execute("INSERT INTO %s (pk, x, val) VALUES (1, 1, 'random phrase')");
+
+        // Make assertion on value
+        assertRows(execute("SELECT x FROM %s WHERE val : 'phrase'"), row(1), row(0));
+        assertRows(execute("SELECT x FROM %s WHERE val : 'indexed'"), row(0));
+
+        searchMemtable(indexName, "indexed", 0);
+        searchMemtable(indexName, "random", 1, 0);
+
+        // delete range
+        execute("DELETE FROM %s WHERE pk = 0");
+
+        // Still expect both rows to be in the index because range deletion doesn't remove from index
+        searchMemtable(indexName, "indexed", 0);
+        searchMemtable(indexName, "random", 1, 0);
+
+        // Overwrite the value for the first of the 2 rows in partition 0
+        execute("INSERT INTO %s (pk, x, val) VALUES (0, 0, 'random')");
+
+        // Confirm the expected behavior
+        searchMemtable(indexName, "indexed"); // overwritten, and the update removes the value
+        searchMemtable(indexName, "something", 0); // range deleted, but not yet removed
+        searchMemtable(indexName, "random", 1, 0, 0); // random is in all 3 memtable index rows
+        searchMemtable(indexName, "phrase", 1); // was deleted/overwritten in 0, so just in 1 now
+    }
+
+    @Test
+    public void testOverwriteWithTTL() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, val text)");
+        var indexName = createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                    "WITH OPTIONS = { 'index_analyzer': 'standard' }");
+        waitForTableIndexesQueryable();
+
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'an indexed phrase') USING TTL 1");
+        execute("INSERT INTO %s (pk, val) VALUES (1, 'something random')");
+
+        // TTL is not applied in this path, so we get the result
+        searchMemtable(indexName, "indexed", 0);
+
+        // Run update and remove 'indexed' from the trie
+        execute("INSERT INTO %s (pk, val) VALUES (0, 'random')");
+
+        // Validate that we get no results
+        searchMemtable(indexName, "indexed");
+    }
+
+    private void searchMemtable(String indexName, String value, int... expectedResults) throws Throwable
+    {
+
+        var sai = (StorageAttachedIndex) Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).getIndexManager().getIndexByName(indexName);
+        var expression = new Expression(sai.getIndexContext()).add(Operator.ANALYZER_MATCHES,
+                                                                   UTF8Type.instance.decompose(value));
+        var queryContext = new QueryContext();
+        var range = Range.unbounded(sai.getIndexContext().getPartitioner());
+        var builder = KeyRangeUnionIterator.builder();
+        // Because there are many
+        for (var memtableIndex : sai.getIndexContext().getLiveMemtables().values())
+            builder.add(memtableIndex.search(queryContext, expression, range, 10));
+        try (var rangeIterator = builder.build())
+        {
+            for (Integer expectedResult : expectedResults)
+            {
+                assertTrue(rangeIterator.hasNext());
+                var pk = Int32Type.instance.getSerializer().deserialize(rangeIterator.next().partitionKey().getKey());
+                assertEquals(expectedResult, pk);
+            }
+            assertFalse(rangeIterator.hasNext());
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/MapEntriesIndexInvalidQueryTest.java b/test/unit/org/apache/cassandra/index/sai/cql/MapEntriesIndexInvalidQueryTest.java
new file mode 100644
index 000000000000..21f03d2e7618
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/MapEntriesIndexInvalidQueryTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+public class MapEntriesIndexInvalidQueryTest extends SAITester
+{
+
+    @Test
+    public void testConflictingBounds()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        // Invalid queries
+        assertInvalidMessage("More than one restriction was found for the end bound on item_cost",
+                             "SELECT partition FROM %s WHERE item_cost['apple'] <= 6 AND item_cost['apple'] < 10");
+        assertInvalidMessage("More than one restriction was found for the start bound on item_cost",
+                             "SELECT partition FROM %s WHERE item_cost['apple'] > 0 AND item_cost['apple'] > 1");
+        assertInvalidMessage("Column \"item_cost\" cannot be restricted by both an inequality " +
+                             "relation and \"CONTAINS(values=[], keys=[], entryKeys=[6170706c65], entryValues=[00000001])\"",
+                             "SELECT partition FROM %s WHERE item_cost['apple'] > 0 AND item_cost['apple'] = 1");
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/MapEntriesIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/MapEntriesIndexTest.java
new file mode 100644
index 000000000000..2555b6188fcd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/MapEntriesIndexTest.java
@@ -0,0 +1,514 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.InvalidColumnTypeException;
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.junit.Assert.assertEquals;
+
+public class MapEntriesIndexTest extends SAITester
+{
+    @Test
+    public void createEntriesIndexEqualityTest()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 1, 'orange': -2})");
+        flush();
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 2, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 1, 'orange': 3})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (4, {'apple': 10, 'orange': -7})");
+
+        // Test equality over both sstable and memtable
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] = 1"), row(1), row(3));
+        // Test sstable read
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] = -2"), row(1));
+        // Test memtable read
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] = -7"), row(4));
+        // Test miss
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] = -3"));
+    }
+
+    @Test
+    public void basicIntegerEntriesIndexRangeTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        // We intentionally use apple, banana, and orange to deal with multiple keys in the trie.
+        // We then search against banana to show that we only get results for banana
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 100, 'banana': 1, 'orange': 2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (4, {'apple': -10, 'banana': 3, 'orange': 2})");
+        flush();
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 50, 'banana': 2, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 10, 'banana': 1, 'orange': 3})");
+
+        // Test range over both sstable and memtable, then over two sstables
+        beforeAndAfterFlush(this::assertIntRangeQueries);
+    }
+
+    private void assertIntRangeQueries() {
+        // GT cases with all, some, and no results
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] > " + Integer.MIN_VALUE),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] > -1"),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] > 0"),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] > 1"), row(2), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] > 2"), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] > 3"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] > " + Integer.MAX_VALUE));
+
+        // GTE cases with all, some, and no results
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] >= " + Integer.MIN_VALUE),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] >= -1"),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] >= 0"),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] >= 1"),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] >= 2"), row(2), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] >= 3"),
+                   row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] >= " + Integer.MAX_VALUE));
+
+        // LT cases with all, some, and no results
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] < " + Integer.MAX_VALUE),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] < 4"),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] < 2"),
+                   row(1), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] < 1"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] < 0"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] < " + Integer.MIN_VALUE));
+
+        // LTE cases with all, some, and no results
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] <= " + Integer.MAX_VALUE),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] <= 4"),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] <= 2"),
+                   row(1), row(2), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] <= 1"),
+                   row(1), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] <= 0"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] <= " + Integer.MIN_VALUE));
+    }
+
+    @Test
+    public void basicIntegerEntriesNeqTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        // We intentionally use apple, banana, and orange to deal with multiple keys in the trie.
+        // We then search against banana to show that we only get results for banana
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 100, 'banana': 1, 'orange': 2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (4, {'apple': -10, 'banana': 3, 'orange': 2})");
+        flush();
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 50, 'banana': 2, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 10, 'banana': 1, 'orange': 3})");
+
+        // Test range over both sstable and memtable, then over two sstables
+        beforeAndAfterFlush(this::assertIntNeqQueries);
+    }
+
+    private void assertIntNeqQueries()
+    {
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] != 1"), row(2), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] != 2 AND item_cost['banana'] != 3"), row(1), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['banana'] != 1 OR item_cost['apple'] >= 20"), row(1), row(2), row(4));
+    }
+
+    @Test
+    public void queryMissingKeyTest()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, coordinates map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(coordinates)) USING 'StorageAttachedIndex'");
+
+        // No rows in table yet, so definitely no results
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] > 10"));
+
+        // Insert row with keys surrounding x but not including it so that we test the empty iterator case
+        execute("INSERT INTO %s (partition, coordinates) VALUES (1, {'w': 100, 'y': 2})");
+
+        // Make sure we still get no results
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] > 10"));
+        flush();
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] > 10"));
+    }
+
+    @Test
+    public void entriesIndexRangeNestedPredicatesTest()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, coordinates map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(coordinates)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, coordinates) VALUES (1, {'x': -1000000, 'y': 1000000})");
+        execute("INSERT INTO %s (partition, coordinates) VALUES (4, {'x': -100, 'y': 2})");
+
+        entriesIndexRangeNestedPredicatesTestAssertions();
+        flush();
+        entriesIndexRangeNestedPredicatesTestAssertions();
+    }
+
+    private void entriesIndexRangeNestedPredicatesTestAssertions()
+    {
+        // Intersections for x and y
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] <= 0 AND coordinates['y'] > 0"),
+                   row(1), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] < -100 AND coordinates['y'] > 0"),
+                   row(1));
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] < -100 AND coordinates['y'] > 1000000"));
+
+        // Intersections for x (setting upper and lower bounds)
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] < -100 AND coordinates['x'] > -1000000"));
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] <= -100 AND coordinates['x'] >= -1000000"),
+                   row(1), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] < -99 AND coordinates['x'] >= -101"),
+                   row(4));
+
+        // Unions
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] > -101 OR coordinates['y'] > 2"),
+                   row(1), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] > 0 OR coordinates['y'] > 0"),
+                   row(1), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] < -100 OR coordinates['y'] < 0"),
+                   row(1));
+        assertRows(execute("SELECT partition FROM %s WHERE coordinates['x'] < -1000000 OR coordinates['y'] > 1000000"));
+    }
+
+    @Test
+    public void basicDateEntriesIndexRangeTest()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, dates map<text, date>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(dates)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, dates) VALUES (1, {'a': '2000-02-03'})");
+        execute("INSERT INTO %s (partition, dates) VALUES (4, {'a': '2001-02-03'})");
+        flush();
+        execute("INSERT INTO %s (partition, dates) VALUES (2, {'a': '1999-02-03'})");
+        execute("INSERT INTO %s (partition, dates) VALUES (3, {'a': '2000-01-01'})");
+
+        // Test range over both sstable and memtable
+        assertDateRangeQueries();
+        // Make two sstables
+        flush();
+        assertDateRangeQueries();
+    }
+
+    private void assertDateRangeQueries()
+    {
+        assertRows(execute("SELECT partition FROM %s WHERE dates['a'] > '2000-01-01'"),
+                   row(1), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE dates['a'] >= '2000-01-01'"),
+                   row(1), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE dates['a'] < '2000-02-03'"),
+                   row(2), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE dates['a'] <= '2000-02-03'"),
+                   row(1), row(2), row(3));
+    }
+
+    @Test
+    public void basicTimestampEntriesIndexRangeTest()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, timestamps map<text, timestamp>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(timestamps)) USING 'StorageAttachedIndex'");
+
+        // 2011-02-03 04:05+0000 is 1296705900000
+        execute("INSERT INTO %s (partition, timestamps) VALUES (1, {'a': '2011-02-03 04:05+0000'})");
+        // 1299038900000 is on 2011-03-02
+        execute("INSERT INTO %s (partition, timestamps) VALUES (4, {'a': '1299038900000'})");
+        flush();
+        execute("INSERT INTO %s (partition, timestamps) VALUES (2, {'a': '1000000000000'})");
+        execute("INSERT INTO %s (partition, timestamps) VALUES (3, {'a': '1999999900000'})");
+
+        // Test range over both sstable and memtable
+        assertTimestampRangeQueries();
+        // Make two sstables
+        flush();
+        assertTimestampRangeQueries();
+    }
+
+    private void assertTimestampRangeQueries()
+    {
+        assertRows(execute("SELECT partition FROM %s WHERE timestamps['a'] > '0'"),
+                   row(1), row(2), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE timestamps['a'] > '1000000000000'"),
+                   row(1), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE timestamps['a'] > '1296705900000'"),
+                   row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE timestamps['a'] >= '1296705900000'"),
+                   row(1), row(4), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE timestamps['a'] < '2011-02-03 04:05+0000'"),
+                   row(2));
+        assertRows(execute("SELECT partition FROM %s WHERE timestamps['a'] <= '2011-02-03 04:05+0000'"),
+                   row(1), row(2));
+    }
+
+    // This test requires the ability to reverse lookup multiple rows from a single trie node
+    // The indexing works by having a trie map that maps from a term to an ordinal in the posting list
+    // and the posting list's ordinal maps to a list of primary keys.
+    @Test
+    public void testDifferentEntryWithSameValueInSameSSTable()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 1, 'orange': 2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 2, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 1, 'orange': 3})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (4, {'apple': 3, 'orange': 2})");
+        flush();
+
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 0"),
+                   row(1), row(2), row(4), row(3));
+    }
+
+    @Test
+    public void testUpdateInvalidatesRowInResultSet()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 1, 'orange': 2})");
+
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 2"), row(1));
+
+        // Push row from memtable to sstable and expect the same result
+        flush();
+
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 2"), row(1));
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 2, 'orange': 1})");
+
+        // Expect no rows then make sure we can still get the row
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 2"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 3"), row(1));
+
+        // Push row from memtable to sstable and expect the same result
+        flush();
+
+        // Expect no rows then make sure we can still get the row
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 2"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 3"), row(1));
+
+        // Now remove the key from the result
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'orange': 1})");
+
+        // Don't get anything
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 2"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 3"));
+
+        // Push row from memtable to sstable and expect the same result
+        flush();
+
+        // Don't get anything
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 2"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 3"));
+    }
+
+    @Test
+    public void queryLargeEntriesWithZeroes()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 101, 'orange': 2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 302, 'orange': 2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': -1000000, 'orange': 2})");
+        flush();
+        execute("INSERT INTO %s (partition, item_cost) VALUES (4, {'apple': 1000000, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (5, {'apple': -1000001, 'orange': 3})");
+
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 2"),
+                   row(1), row(2), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 0"), row(5), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] <= 0"), row(5), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < -100"), row(5), row(3));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] <= -100"), row(5), row(3));
+    }
+
+    @Test
+    public void queryTextEntriesWithZeroes()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, text>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': '', 'orange': '2'})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (4, {'apple': 'a', 'orange': '2'})");
+        flush();
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 'abv', 'orange': '1'})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 'z', 'orange': '3'})");
+
+        assertRowsIgnoringOrder(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 'a'"),
+                                row(2), row(3));
+        assertRowsIgnoringOrder(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 'a'"),
+                                row(1));
+    }
+
+    @Test
+    public void testUpdatesAndDeletes()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 1, 'orange': -2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 2, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 1, 'orange': 3})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (4, {'apple': 10, 'orange': -7})");
+        flush();
+
+        // Set a baseline to make sure data is as expected
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 1"),
+                   row(2), row(4));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 1 AND item_cost['apple'] < 10"),
+                   row(2));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] < 1"),
+                   row(1), row(4));
+
+        // Delete and update some rows
+        execute("DELETE FROM %s WHERE partition = 2");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (4, {'apple': 1, 'orange': 3})");
+
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 1"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] < 1"), row(1));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] >= 3"), row(4), row(3));
+
+        flush();
+
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 1"));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] < 1"), row(1));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] >= 3"), row(4), row(3));
+    }
+
+    // NOTE: this works without touching the SAI code. This is worth testing to make sure we don't need to reject
+    // these queries.
+    @Test
+    public void testLWTConditionalDelete() throws Throwable
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 1, 'orange': -2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 2, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 10000, 'orange': 1})");
+        flush();
+
+        // Attempt to delete rows, but the conditional is not satisfied
+        execute("DELETE FROM %s WHERE partition = 2 IF item_cost['apple'] > 2");
+
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 0"), row(1), row(2), row(3));
+
+        // Actually delete row 2 this time
+        execute("DELETE FROM %s WHERE partition = 2 IF item_cost['apple'] > 1");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 0"), row(1), row(3));
+            assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] > 1"), row(3));
+            // Show that it also works for the other map entry
+            assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] > 0"), row(3));
+        });
+    }
+
+    // NOTE: this works without touching the SAI code. This is worth testing to make sure we don't need to reject
+    // these queries.
+    @Test
+    public void testLWTConditionalUpdate()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 1, 'orange': -2})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (2, {'apple': 2, 'orange': 1})");
+        execute("INSERT INTO %s (partition, item_cost) VALUES (3, {'apple': 10000, 'orange': 1})");
+        flush();
+
+        var row3 = execute("SELECT item_cost FROM %s WHERE partition = 3");
+        assertEquals("{apple=10000, orange=1}", row3.one().getMap("item_cost", UTF8Type.instance, Int32Type.instance).toString());
+
+        // Attempt to update rows, but only one of the updates is successful in updating the value of apple
+        execute("UPDATE %s SET item_cost['apple'] = 3 WHERE partition = 2 IF item_cost['apple'] > 100");
+        execute("UPDATE %s SET item_cost['apple'] = 3 WHERE partition = 3 IF item_cost['apple'] > 100");
+
+        // observe the change
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] = 3"), row(3));
+        row3 = execute("SELECT item_cost FROM %s WHERE partition = 3");
+        assertEquals("{apple=3, orange=1}", row3.one().getMap("item_cost", UTF8Type.instance, Int32Type.instance).toString());
+    }
+
+    @Test
+    public void testRangeQueriesCombinedWithQueryAgainstOtherIndexes()
+    {
+        createTable("CREATE TABLE %s (partition int primary key, store_name text, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(store_name) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        // We intentionally use apple, banana, and orange to deal with multiple keys in the trie.
+        // We then search against banana to show that we only get results for banana
+        execute("INSERT INTO %s (partition, store_name, item_cost) VALUES (0, 'Partial Foods', {'apple': 5, 'orange': 7})");
+        flush();
+        execute("INSERT INTO %s (partition, store_name, item_cost) VALUES (1, 'Stale Market', {'apple': 6, 'orange': 4})");
+
+        // Test basic ranges
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] < 6"), row(0));
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['orange'] < 7"), row(1));
+
+        // Combine ranges with query on other index
+        assertRows(execute("SELECT partition FROM %s WHERE item_cost['apple'] <= 6 AND store_name = 'Partial Foods'"), row(0));
+    }
+
+    @Test
+    public void testPreparedQuery() throws Throwable
+    {
+        createTable("CREATE TABLE %s (partition int primary key, item_cost map<text, int>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(item_cost)) USING 'StorageAttachedIndex'");
+
+        // We intentionally use apple, banana, and orange to deal with multiple keys in the trie.
+        // We then search against banana to show that we only get results for banana
+        execute("INSERT INTO %s (partition, item_cost) VALUES (0, {'apple': 5, 'orange': 7})");
+        flush();
+        execute("INSERT INTO %s (partition, item_cost) VALUES (1, {'apple': 6, 'orange': 4})");
+
+        String query = "SELECT partition FROM %s WHERE item_cost[?] < ?";
+        prepare(query);
+        assertRows(execute(query, "apple", 6), row(0));
+    }
+
+    @Test
+    public void testRangeQueriesOnClusteringColumn() throws Throwable
+    {
+        // If we ever support using non-frozen collections as clustering columns, we need to determine
+        // if range queries should work when the column is a clustering column.
+        assertInvalidThrow(InvalidColumnTypeException.class,
+                           String.format(
+                           "CREATE TABLE %s.%s (partition int, item_cost map<text, int>, PRIMARY KEY (partition, item_cost))",
+                           KEYSPACE,
+                           createTableName()));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
index d22eb2ed11e8..c0cf1e248011 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
@@ -17,10 +17,13 @@
  */
 package org.apache.cassandra.index.sai.cql;
 
+import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.index.ExpressionFilteringIndex;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
 
@@ -35,7 +38,7 @@ public class MixedIndexImplementationsTest extends SAITester
      * Tests that storage-attached indexes can be dropped when there are other indexes in the same table, and vice versa.
      */
     @Test
-    public void shouldDropOtherIndex() throws Throwable
+    public void shouldDropOtherIndex()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int)");
 
@@ -55,7 +58,7 @@ public void shouldDropOtherIndex() throws Throwable
      * Tests that storage-attached index queries can include restrictions over columns indexed by other indexes.
      */
     @Test
-    public void shouldAcceptColumnsWithOtherIndex() throws Throwable
+    public void shouldAcceptColumnsWithOtherIndex()
     {
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int)");
 
@@ -72,15 +75,56 @@ public void shouldAcceptColumnsWithOtherIndex() throws Throwable
         assertRowsIgnoringOrder(execute(ossSelect, 0), new Object[][]{{0, 0, 0}, {1, 0, 1}});
         assertRowsIgnoringOrder(execute(ossSelect, 1), new Object[][]{{2, 1, 0}, {3, 1, 1}});
 
-        String saiSelect = "SELECT * FROM %s WHERE v1 = ? AND v2 = ? ALLOW FILTERING";
-        assertRowsIgnoringOrder(execute(saiSelect, 0, 0), new Object[]{0, 0, 0});
-        assertRowsIgnoringOrder(execute(saiSelect, 0, 1), new Object[]{1, 0, 1});
-        assertRowsIgnoringOrder(execute(saiSelect, 1, 0), new Object[]{2, 1, 0});
-        assertRowsIgnoringOrder(execute(saiSelect, 1, 1), new Object[]{3, 1, 1});
+        String ndiSelect = "SELECT * FROM %s WHERE v1 = ? AND v2 = ? ALLOW FILTERING";
+        assertRowsIgnoringOrder(execute(ndiSelect, 0, 0), new Object[]{0, 0, 0});
+        assertRowsIgnoringOrder(execute(ndiSelect, 0, 1), new Object[]{1, 0, 1});
+        assertRowsIgnoringOrder(execute(ndiSelect, 1, 0), new Object[]{2, 1, 0});
+        assertRowsIgnoringOrder(execute(ndiSelect, 1, 1), new Object[]{3, 1, 1});
+    }
+
+    /**
+     * Tests that storage-attached indexes are not selected when the query contains a custom expression targeted to another index.
+     */
+    @Test
+    public void shouldNotBeSelectedForCustomExpressions()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int)");
+
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        String indexName = createIndex(
+                String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", ExpressionFilteringIndex.class.getName()));
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        ExpressionFilteringIndex customIndex = (ExpressionFilteringIndex) cfs.indexManager.getIndexByName(indexName);
+
+        String insert = "INSERT INTO %s(k, v1, v2) VALUES (?, ?, ?)";
+        execute(insert, 0, 0, 0);
+        execute(insert, 1, 0, 1);
+        execute(insert, 2, 1, 0);
+        execute(insert, 3, 1, 1);
+
+        String ndiSelect = "SELECT * FROM %s WHERE v1 = ?";
+        assertRowsIgnoringOrder(execute(ndiSelect, 0), new Object[][]{{0, 0, 0}, {1, 0, 1}});
+        assertRowsIgnoringOrder(execute(ndiSelect, 1), new Object[][]{{2, 1, 0}, {3, 1, 1}});
+        Assert.assertEquals(0, customIndex.searches.get());
+
+        String mixedSelect = "SELECT * FROM %s WHERE v1 = ? AND v2 = ? ALLOW FILTERING";
+        assertRowsIgnoringOrder(execute(mixedSelect, 0, 0), new Object[]{0, 0, 0});
+        assertRowsIgnoringOrder(execute(mixedSelect, 0, 1), new Object[]{1, 0, 1});
+        assertRowsIgnoringOrder(execute(mixedSelect, 1, 0), new Object[]{2, 1, 0});
+        assertRowsIgnoringOrder(execute(mixedSelect, 1, 1), new Object[]{3, 1, 1});
+        Assert.assertEquals(0, customIndex.searches.get());
+
+        String exprSelect = String.format("SELECT * FROM %%s WHERE v1 = ? AND expr(%s, ?) ALLOW FILTERING", indexName);
+        assertRowsIgnoringOrder(execute(exprSelect, 0, 0), new Object[]{0, 0, 0});
+        assertRowsIgnoringOrder(execute(exprSelect, 0, 1), new Object[]{1, 0, 1});
+        assertRowsIgnoringOrder(execute(exprSelect, 1, 0), new Object[]{2, 1, 0});
+        assertRowsIgnoringOrder(execute(exprSelect, 1, 1), new Object[]{3, 1, 1});
+        Assert.assertEquals(4, customIndex.searches.get());
     }
 
     @Test
-    public void shouldRequireAllowFilteringWithOtherIndex() throws Throwable
+    public void shouldRequireAllowFilteringWithOtherIndex()
     {
         Util.assumeLegacySecondaryIndex();
 
@@ -179,7 +223,7 @@ public void shouldRequireAllowFilteringWithOtherIndex() throws Throwable
         testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0 AND c3=0 AND r1=0 AND r2=0", true);
     }
 
-    private void testAllowFiltering(String query, boolean requiresAllowFiltering) throws Throwable
+    private void testAllowFiltering(String query, boolean requiresAllowFiltering)
     {
         if (requiresAllowFiltering)
             assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, query);
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
index 459d608dc504..8d952dc1f4f0 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
@@ -19,43 +19,97 @@
 
 import org.junit.Test;
 
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.sai.SAITester;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 
 public class MultipleColumnIndexTest extends SAITester
 {
+    // Note: Full testing of multiple map index types is done in the
+    // types/collections/maps/MultiMap*Test tests
+    // This is just testing that the indexes can be created
     @Test
-    public void canCreateMultipleMapIndexesOnSameColumn()
+    public void canCreateMultipleMapIndexesOnSameColumn() throws Throwable
     {
-        // Note: Full testing of multiple map index types is done in the
-        // types/collections/maps/MultiMap*Test tests
-        // This is just testing that the indexes can be created
         createTable("CREATE TABLE %s (pk int, ck int, value map<int,int>, PRIMARY KEY(pk, ck))");
-        createIndex("CREATE INDEX ON %s(KEYS(value)) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(VALUES(value)) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(ENTRIES(value)) USING 'sai'");
+        createIndex("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
     }
 
     @Test
-    public void indexNamedAsColumnWillCoExistWithGeneratedIndexNames()
+    public void canHaveAnalyzedAndUnanalyzedIndexesOnSameColumn() throws Throwable
     {
-        createTable("CREATE TABLE %s(id int PRIMARY KEY, text_map map<text, text>)");
+        createTable("CREATE TABLE %s (pk int, value text, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : true }");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : false, 'equals_behaviour_when_analyzed': 'unsupported' }");
+
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, "a");
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, "A");
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE value = 'a'"),
+                       row(1));
+            assertRows(execute("SELECT pk FROM %s WHERE value : 'a'"),
+                       row(1),
+                       row(2));
+        });
+    }
 
-        createIndex("CREATE INDEX text_map ON %s(keys(text_map)) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(values(text_map)) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(entries(text_map)) USING 'sai'");
+    @Test
+    public void cannotHaveMultipleAnalyzingIndexesOnSameColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, value text, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : false }");
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : true }"))
+        .isInstanceOf(InvalidRequestException.class);
+    }
 
+    @Test
+    public void indexNamedAsColumnWillCoExistWithGeneratedIndexNames() throws Throwable
+    {
+        createTable("CREATE TABLE %s(id int PRIMARY KEY, text_map map<text, text>)");
         execute("INSERT INTO %s(id, text_map) values (1, {'k1':'v1', 'k2':'v2'})");
         execute("INSERT INTO %s(id, text_map) values (2, {'k1':'v1', 'k3':'v3'})");
         execute("INSERT INTO %s(id, text_map) values (3, {'k4':'v4', 'k5':'v5'})");
 
-        assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map['k2'] = 'v2'").size());
-        assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS 'v1'").size());
-        assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1'").size());
-        assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2'").size());
-        assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS 'v1'").size());
-        assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2' AND text_map CONTAINS 'v1'").size());
-        assertEquals(0, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k4'").size());
+        flush();
+
+        createIndex("CREATE CUSTOM INDEX text_map ON %s(keys(text_map)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(values(text_map)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(text_map)) USING 'StorageAttachedIndex'");
+
+        beforeAndAfterFlush(() -> {
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map['k2'] = 'v2'").size());
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS 'v1'").size());
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS 'v1' AND text_map NOT CONTAINS 'v5'").size());
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS 'v1' AND text_map NOT CONTAINS KEY 'k5'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS 'v1' AND text_map['k2'] != 'v2'").size());
+
+
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map CONTAINS 'v2'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map NOT CONTAINS 'v3'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map NOT CONTAINS KEY 'k3'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map['k2'] != 'v2'").size());
+
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS 'v1'").size());
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS 'v1' AND text_map NOT CONTAINS 'v8'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS 'v1' AND text_map NOT CONTAINS 'v3'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2' AND text_map CONTAINS 'v1'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2' AND text_map NOT CONTAINS 'v3'").size());
+            assertEquals(0, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k4'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map NOT CONTAINS KEY 'k2'").size());
+            assertEquals(0, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k4' AND text_map NOT CONTAINS KEY 'k2'").size());
+
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] != 'v2' AND text_map['k2'] = 'v2'").size());
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] != 'v2' AND text_map['k2'] != 'v2'").size());
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] != 'v2' AND text_map NOT CONTAINS 'v3'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] != 'v2' AND text_map NOT CONTAINS 'v3' AND text_map NOT CONTAINS 'v5'").size());
+            assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] != 'v2' AND text_map NOT CONTAINS KEY 'k3'").size());
+            assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] != 'v2' AND text_map NOT CONTAINS KEY 'k3' AND text_map NOT CONTAINS KEY 'k5'").size());
+        });
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/NativeIndexDDLTest.java b/test/unit/org/apache/cassandra/index/sai/cql/NativeIndexDDLTest.java
new file mode 100644
index 000000000000..9d2ae749c339
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/NativeIndexDDLTest.java
@@ -0,0 +1,1610 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai.cql;
+
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.exceptions.InvalidConfigurationInQueryException;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import com.datastax.driver.core.exceptions.ReadFailureException;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.restrictions.IndexRestrictions;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndexBuilder;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+import org.apache.cassandra.index.sai.utils.SuppressLeakCheck;
+import org.apache.cassandra.index.sai.view.View;
+import org.apache.cassandra.inject.ActionBuilder;
+import org.apache.cassandra.inject.Expression;
+import org.apache.cassandra.inject.Injection;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.inject.InvokePointBuilder;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Throwables;
+import org.mockito.Mockito;
+
+import static java.util.Collections.singletonList;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ENCRYPTION;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.when;
+
+@SuppressLeakCheck(jira="STAR-974")
+public class NativeIndexDDLTest extends SAITester
+{
+    private static final Injections.Counter NDI_CREATION_COUNTER = Injections.newCounter("IndexCreationCounter")
+                                                                             .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndex.class).onMethod("register"))
+                                                                             .build();
+
+    private static final Injection failNDIInitialializaion = Injections.newCustom("fail_ndi_initialization")
+                                                                       .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndexBuilder.class).onMethod("build"))
+                                                                       .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
+                                                                       .build();
+
+    private static final Injection failNumericIndexBuild = Injections.newCustom("fail_numeric_index_build")
+                                                                       .add(InvokePointBuilder.newInvokePoint().onClass(SegmentBuilder.KDTreeSegmentBuilder.class).onMethod("addInternal"))
+                                                                       .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
+                                                                       .build();
+
+    private static final Injection forceFlushPause = Injections.newPause("force_flush_pause", 30_000)
+                                                               .add(InvokePointBuilder.newInvokePoint().onClass(ColumnFamilyStore.class).onMethod("forceBlockingFlush"))
+                                                               .build();
+
+    private static final Injection failPerIndexMetaCompletion = Injections.newCustom("fail_index_meta_completion")
+                                                                          .add(InvokePointBuilder.newInvokePoint().onClass(SegmentBuilder.class).onMethod("flush"))
+                                                                          .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
+                                                                          .build();
+
+    private static final Injection failPerSSTableTokenAdd = Injections.newCustom("fail_token_writer")
+                                                                      .add(InvokePointBuilder.newInvokePoint().onClass(NumericValuesWriter.class).onMethod("add"))
+                                                                      .add(ActionBuilder.newActionBuilder().actions().doThrow(IOException.class, Expression.quote("Injected failure!")))
+                                                                      .build();
+
+    private static final Injection FAIL_INDEX_GC_TRANSACTION = Injections.newCustom("fail_index_gc_transaction")
+                                                                         .add(InvokePointBuilder.newInvokePoint().onClass("org.apache.cassandra.index.SecondaryIndexManager$IndexGCTransaction")
+                                                                                                .onMethod("<init>"))
+                                                                         .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
+                                                                         .build();
+
+    @BeforeClass
+    public static void init()
+    {
+        CassandraRelevantProperties.SAI_VALIDATE_MAX_TERM_SIZE_AT_COORDINATOR.setBoolean(true);
+    }
+
+    @Before
+    public void setup() throws Throwable
+    {
+        requireNetwork();
+
+        startJMXServer();
+
+        createMBeanServerConnection();
+
+        Injections.inject(NDI_CREATION_COUNTER, INDEX_BUILD_COUNTER, FAIL_INDEX_GC_TRANSACTION);
+
+        NDI_CREATION_COUNTER.reset();
+        INDEX_BUILD_COUNTER.reset();
+    }
+
+    @After
+    public void removeInjections()
+    {
+        Injections.deleteAll();
+    }
+
+    @Test
+    public void shouldFailUnsupportedType()
+    {
+        for (CQL3Type.Native cql3Type : CQL3Type.Native.values())
+        {
+            if (cql3Type == CQL3Type.Native.EMPTY)
+                continue;
+
+            String createTableTemplate = "CREATE TABLE %%s (id text PRIMARY KEY, %s %s)";
+            createTable(String.format(createTableTemplate, cql3Type, cql3Type));
+
+            boolean supported = StorageAttachedIndex.SUPPORTED_TYPES.contains(cql3Type);
+
+            try
+            {
+                executeNet(String.format("CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'", cql3Type));
+                assertTrue("Index creation on unsupported type " + cql3Type + " should have failed.", supported);
+            }
+            catch (RuntimeException e)
+            {
+                assertFalse("Index creation on supported type " + cql3Type + " should have succeeded.", supported);
+                // InvalidConfigurationInQueryException is sub-class of InvalidQueryException
+                assertTrue(Throwables.isCausedBy(e, InvalidQueryException.class));
+            }
+        }
+    }
+
+    @Test
+    public void shouldFailCreationOnPartitionKey()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(id) USING 'StorageAttachedIndex'"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("Cannot create secondary index on the only partition key column id");
+    }
+
+    @Test
+    public void shouldFailCreationUsingMode()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING " +
+                                            "'StorageAttachedIndex' WITH OPTIONS = { 'mode' : 'CONTAINS' }")).isInstanceOf(InvalidConfigurationInQueryException.class);
+    }
+
+    @Test
+    public void shouldFailCreateSpecifyingAnalyzerClass()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) " +
+                                            "USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = { 'analyzer_class' : 'org.apache.cassandra.index.sai.analyzer.NonTokenizingAnalyzer' }"))
+                .isInstanceOf(InvalidConfigurationInQueryException.class);
+    }
+
+    @Test
+    public void shouldFailCreateWithMisspelledOption()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) " +
+                                            "USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = { 'case-sensitive' : true }")).isInstanceOf(InvalidConfigurationInQueryException.class);
+    }
+
+    @Test
+    public void shouldFailCaseSensitiveWithNonText()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) " +
+                                            "USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = { 'case_sensitive' : false }")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldFailOnNormalizeWithNonText()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) " +
+                                            "USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = { 'normalize' : true }")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldFailCreateWithUserType()
+    {
+        String typeName = createType("CREATE TYPE %s (a text, b int, c double)");
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val " + typeName + ')');
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) " +
+                                            "USING 'StorageAttachedIndex'")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldNotFailCreateWithTupleType()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val tuple<text, int, double>)");
+
+        executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        TableMetadata metadata = currentTableMetadata();
+        AbstractType<?> tuple = metadata.getColumn(ColumnIdentifier.getInterned("val", false)).type;
+        assertFalse(tuple.isMultiCell());
+        assertFalse(tuple.isCollection());
+        assertTrue(tuple.isTuple());
+    }
+
+    @Test
+    public void shouldCreateIndexIfExists()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX IF NOT EXISTS ON %s(val) USING 'StorageAttachedIndex' ");
+
+        createIndexAsync("CREATE CUSTOM INDEX IF NOT EXISTS ON %s(val) USING 'StorageAttachedIndex' ");
+
+        assertEquals(1, NDI_CREATION_COUNTER.get());
+    }
+
+    @Test
+    public void shouldBeCaseSensitiveByDefault()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Camel'").size());
+
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'camel'").size());
+    }
+
+    @Test
+    public void shouldEnableCaseSensitiveSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        // Case sensitive search is the default, and as such, it does not make the SAI qualify as "analyzed".
+        // The queries below use '=' and not ':' because : is limited to analyzed indexes.
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : true }");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Camel'").size());
+
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'camel'").size());
+    }
+
+    @Test
+    public void shouldEnableCaseInsensitiveSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : false }");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val : 'camel'").size());
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'camel'").size());
+    }
+
+    @Test
+    public void shouldBeNonNormalizedByDefault()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u00E1l'").size());
+
+        // Both \u00E1 and \u0061\u0301 are visible as the character á, but without NFC normalization, they won't match.
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldEnableNonNormalizedSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        // Normalize search is disabled by default, and as such, it does not make the SAI qualify as "analyzed".
+        // The queries below use '=' and not ':' because : is limited to analyzed indexes.
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : false }");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u00E1l'").size());
+
+        // Both \u00E1 and \u0061\u0301 are visible as the character á, but without NFC normalization, they won't match.
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldEnableNormalizedSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : true }");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val : 'Cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldEnableNormalizedCaseInsensitiveSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : true, 'case_sensitive' : false}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val : 'cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldEnableAsciiSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'ascii' : true, 'case_sensitive' : false}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Éppinger')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val : 'eppinger'").size());
+    }
+
+    @Test
+    public void shouldCreateIndexOnReversedType()
+    {
+        createTable("CREATE TABLE %s (id text, ck1 text, ck2 int, val text, PRIMARY KEY (id,ck1,ck2)) WITH CLUSTERING ORDER BY (ck1 desc, ck2 desc)");
+
+        String indexNameCk1 = createIndex("CREATE CUSTOM INDEX ON %s(ck1) USING 'StorageAttachedIndex'");
+        String indexNameCk2 = createIndex("CREATE CUSTOM INDEX ON %s(ck2) USING 'StorageAttachedIndex'");
+
+        execute("insert into %s(id, ck1, ck2, val) values('1', '2', 3, '3')");
+        execute("insert into %s(id, ck1, ck2, val) values('1', '3', 4, '4')");
+        assertEquals(1, executeNet("SELECT * FROM %s WHERE ck1='3'").all().size());
+        assertEquals(2, executeNet("SELECT * FROM %s WHERE ck2>=0").all().size());
+        assertEquals(2, executeNet("SELECT * FROM %s WHERE ck2<=4").all().size());
+
+        flush();
+        assertEquals(1, executeNet("SELECT * FROM %s WHERE ck1='2'").all().size());
+        assertEquals(2, executeNet("SELECT * FROM %s WHERE ck2>=3").all().size());
+        assertEquals(2, executeNet("SELECT * FROM %s WHERE ck2<=4").all().size());
+
+        SecondaryIndexManager sim = getCurrentColumnFamilyStore().indexManager;
+        StorageAttachedIndex index = (StorageAttachedIndex) sim.getIndexByName(indexNameCk1);
+        IndexContext context = index.getIndexContext();
+        assertTrue(context.isLiteral());
+        assertTrue(context.getValidator() instanceof ReversedType);
+
+        index = (StorageAttachedIndex) sim.getIndexByName(indexNameCk2);
+        context = index.getIndexContext();
+        assertFalse(context.isLiteral());
+        assertTrue(context.getValidator() instanceof ReversedType);
+    }
+
+    @Test
+    public void shouldCreateIndexWithAlias()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        assertEquals(1, NDI_CREATION_COUNTER.get());
+    }
+
+    /**
+     * Verify SASI can be created and queries with NDI dependencies.
+     * Not putting in {@link MixedIndexImplementationsTest} because it uses CQLTester which doesn't load NDI dependency.
+     */
+    @Test
+    public void shouldCreateSASI()
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'org.apache.cassandra.index.sasi.SASIIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(v2) USING 'org.apache.cassandra.index.sasi.SASIIndex' WITH OPTIONS = {'mode': 'CONTAINS',\n" +
+                    "'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer',\n" +
+                    "'tokenization_enable_stemming': 'true',\n" +
+                    "'tokenization_locale': 'en',\n" +
+                    "'tokenization_skip_stop_words': 'true',\n" +
+                    "'analyzed': 'true',\n" +
+                    "'tokenization_normalize_lowercase': 'true'};");
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(1, rows.all().size());
+
+        rows = executeNet("SELECT id1 FROM %s WHERE v2 like '0'");
+        assertEquals(1, rows.all().size());
+    }
+
+    @Test
+    public void shouldCreateNumericIndexWithBkdPostingsSkipAndMinLeaves()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'bkd_postings_skip' : 3, 'bkd_postings_min_leaves' : 32}");
+
+        assertEquals(1, NDI_CREATION_COUNTER.get());
+    }
+
+    @Test
+    public void shouldCreateNumericIndexWithBkdPostingsSkipOnly()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'bkd_postings_skip' : 3}");
+
+        assertEquals(1, NDI_CREATION_COUNTER.get());
+    }
+
+    @Test
+    public void shouldCreateNumericIndexWithBkdPostingsMinLeavesOnly()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'bkd_postings_min_leaves': 32}");
+
+        assertEquals(1, NDI_CREATION_COUNTER.get());
+    }
+
+    @Test
+    public void shouldFailToCreateNumericIndexWithTooLowBkdPostingsSkip()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = {'bkd_postings_skip' : 0, 'bkd_postings_min_leaves' : 32}")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldFailToCreateNumericIndexWithTooLowBkdPostingsMinLeaves()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = {'bkd_postings_skip' : 3, 'bkd_postings_min_leaves' : 0}")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldFailToCreateStringIndexWithBkdPostingsSkip()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = {'bkd_postings_skip' : 3}")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldFailToCreateStringIndexWithBkdPostingsMinLeaves()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = {'bkd_postings_min_leaves' : 9}")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldFailToCreateInvalidBooleanOption()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = {'case_sensitive': 'NOTVALID'}")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldFailToCreateEmptyBooleanOption()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' " +
+                                            "WITH OPTIONS = {'case_sensitive': ''}")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldFailCreationOnMultipleColumns()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val1 text, val2 text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val1, val2) USING 'StorageAttachedIndex'"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("storage-attached index cannot be created over multiple columns");
+    }
+
+    @Test
+    public void shouldFailCreationMultipleIndexesOnSimpleColumn()
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, v1 TEXT)");
+        execute("INSERT INTO %s (id, v1) VALUES(1, '1')");
+        flush();
+
+        executeNet("CREATE CUSTOM INDEX index_1 ON %s(v1) USING 'StorageAttachedIndex'");
+        waitForTableIndexesQueryable();
+
+        // same name
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX index_1 ON %s(v1) USING 'StorageAttachedIndex'"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("Index 'index_1' already exists");
+
+        // different name, same option
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX index_2 ON %s(v1) USING 'StorageAttachedIndex'"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("Index index_2 is a duplicate of existing index index_1");
+
+        // different name, different option, same target.
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(v1) USING 'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : true }"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("Cannot create duplicate storage-attached index on column: v1" );
+
+        ResultSet rows = executeNet("SELECT id FROM %s WHERE v1 = '1'");
+        assertEquals(1, rows.all().size());
+    }
+
+    @Test
+    public void shouldIndexBuildingWithInMemoryData()
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        int rowCount = 10;
+        for (int i = 0; i < rowCount; i++)
+            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', " + i + ", '0')");
+
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(rowCount, rows.all().size());
+    }
+
+    @Test
+    public void shouldIndexExistingMemtableOnCreationWithConcurrentFlush() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
+
+        Injections.Barrier delayInitializationTask =
+                Injections.newBarrier("delayInitializationTask", 2, false)
+                          .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndex.class).onMethod("startInitialBuild"))
+                          .build();
+
+        // Create the index, but do not allow the initial index build to begin:
+        Injections.inject(delayInitializationTask);
+        String indexName = createIndexAsync("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // Flush the Memtable's contents, which will feed data to the index as the SSTable is written:
+        flush();
+
+        // Allow the initialization task, which builds the index, to continue:
+        delayInitializationTask.countDown();
+
+        ResultSet rows = executeNet("SELECT id FROM %s WHERE val = 'Camel'");
+        assertEquals(1, rows.all().size());
+
+        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    @Test
+    public void shouldRejectQueriesBeforeIndexInitializationFinished() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        int rowCount = 10;
+        for (int i = 0; i < rowCount; i++)
+            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', " + i + ", '0')");
+
+        Injections.inject(forceFlushPause);
+        createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+
+        assertThatThrownBy(() -> executeNet("SELECT id1 FROM %s WHERE v1>=0")).isInstanceOf(ReadFailureException.class);
+    }
+
+    @Test
+    public void shouldRejectQueriesOnIndexInitializationFailure() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        disableCompaction(KEYSPACE);
+
+        int rowCount = 10;
+        for (int i = 0; i < rowCount; i++)
+            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', " + i + ", '0')");
+        flush();
+
+        Injections.inject(failNDIInitialializaion);
+        createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        waitForAssert(() -> assertEquals(1, INDEX_BUILD_COUNTER.get()));
+        waitForCompactions();
+
+        assertThatThrownBy(() -> executeNet("SELECT id1 FROM %s WHERE v1>=0")).isInstanceOf(ReadFailureException.class);
+    }
+
+    @Test
+    public void testMaxTermSizeRejectionsAtWrite()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (k int PRIMARY KEY, v text, m map<text, text>, frozen_m frozen<map<text, text>>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(m) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(full(frozen_m)) USING 'StorageAttachedIndex'");
+
+        String largeTerm = UTF8Type.instance.compose(ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT / 2 + 1));
+        assertThatThrownBy(() -> executeNet("INSERT INTO %s (k, v) VALUES (0, ?)", largeTerm))
+        .hasMessage(String.format("Term of column v exceeds the byte limit for index. Term size 32.000KiB. Max allowed size %s.",
+                                  FBUtilities.prettyPrintMemory(IndexContext.MAX_STRING_TERM_SIZE)))
+        .isInstanceOf(InvalidQueryException.class);
+
+        assertThatThrownBy(() -> executeNet("INSERT INTO %s (k, m) VALUES (0, {'key': '" + largeTerm + "'})"))
+        .hasMessage(String.format("Term of column m exceeds the byte limit for index. Term size 32.000KiB. Max allowed size %s.",
+                                  FBUtilities.prettyPrintMemory(IndexContext.MAX_STRING_TERM_SIZE)))
+        .isInstanceOf(InvalidQueryException.class);
+
+        assertThatThrownBy(() -> executeNet("INSERT INTO %s (k, frozen_m) VALUES (0, {'key': '" + largeTerm + "'})"))
+        .hasMessage(String.format("Term of column frozen_m exceeds the byte limit for index. Term size 32.015KiB. Max allowed size %s.",
+                                  FBUtilities.prettyPrintMemory(IndexContext.MAX_FROZEN_TERM_SIZE)))
+        .isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldReleaseIndexFilesAfterDroppingLastIndex() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        String numericIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        String literalIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+        IndexContext numericIndexContext = createIndexContext(numericIndexName, Int32Type.instance);
+        IndexContext literalIndexContext = createIndexContext(literalIndexName, UTF8Type.instance);
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 0);
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
+        flush();
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1);
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(1, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(1, rows.all().size());
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0')");
+        flush();
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2);
+        verifySSTableIndexes(numericIndexName, 2, 2);
+        verifySSTableIndexes(literalIndexName, 2, 2);
+        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(2, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(2, rows.all().size());
+
+        dropIndex("DROP INDEX %s." + numericIndexName);
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 2);
+        verifySSTableIndexes(numericIndexName, 2, 0);
+        verifySSTableIndexes(literalIndexName, 2, 2);
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(2, rows.all().size());
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('2', 2, '0')");
+        flush();
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 3);
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(3, rows.all().size());
+
+        dropIndex("DROP INDEX %s." + literalIndexName);
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 0);
+        verifySSTableIndexes(numericIndexName, 0);
+        verifySSTableIndexes(literalIndexName, 0);
+
+        assertEquals("Segment memory limiter should revert to zero on drop.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    @Test
+    public void shouldCreateIndexFilesAfterMultipleConcurrentIndexCreation() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        verifyNoIndexFiles();
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        flush();
+        verifyNoIndexFiles();
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+        verifyNoIndexFiles();
+
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+        IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2);
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(2, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(2, rows.all().size());
+
+        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    @Test
+    public void shouldCreateIndexFilesAfterMultipleSequentialIndexCreation() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        verifyNoIndexFiles();
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        flush();
+        verifyNoIndexFiles();
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+        verifyNoIndexFiles();
+
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+        verifyIndexFiles(numericIndexContext, null, 2, 0);
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(2, rows.all().size());
+
+        IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2);
+        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(2, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(2, rows.all().size());
+
+        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    @Test
+    public void shouldReleaseIndexFilesAfterCompaction() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        disableCompaction(KEYSPACE);
+
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+        IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
+        verifyNoIndexFiles();
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        flush();
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1);
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(1, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(1, rows.all().size());
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2);
+        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(2, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(2, rows.all().size());
+
+        compact();
+        waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1));
+
+        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(2, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(2, rows.all().size());
+
+        assertEquals("Segment memory limiter should revert to zero after compaction.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    @Test
+    public void truncateWithBuiltIndexes() throws Throwable
+    {
+        verifyTruncateWithIndex(false);
+    }
+
+    @Test
+    public void concurrentTruncateWithIndexBuilding() throws Throwable
+    {
+        verifyTruncateWithIndex(true);
+    }
+
+    private void verifyTruncateWithIndex(boolean concurrentTruncate) throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        if (!concurrentTruncate)
+        {
+            createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+            createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+        }
+
+        // create 100 rows, half in sstable and half in memtable
+        int num = 100;
+        for (int i = 0; i < num; i++)
+        {
+            if (i == num / 2)
+                flush();
+            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', 0, '0');");
+        }
+
+        if (concurrentTruncate)
+        {
+            createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+            createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+            truncate(true);
+        }
+        else
+        {
+            truncate(true);
+        }
+
+        waitForAssert(this::verifyNoIndexFiles);
+
+        // verify index-view-manager has been cleaned up
+        verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 0);
+        verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 0);
+
+        assertEquals("Segment memory limiter should revert to zero after truncate.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    /**
+     * Simulate SAI build error during index rebuild: index rebuild task should fail
+     */
+    @Test
+    public void testIndexRebuildAborted() throws Throwable
+    {
+        // prepare schema and data
+        createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(2, rows.all().size());
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        SecondaryIndexManager sim = cfs.getIndexManager();
+        StorageAttachedIndex sai = (StorageAttachedIndex) sim.listIndexes().iterator().next();
+        assertThat(sai.getIndexContext().getView().getIndexes()).hasSize(1);
+
+        StorageAttachedIndexGroup saiGroup = StorageAttachedIndexGroup.getIndexGroup(cfs);
+        assertThat(saiGroup.sstableContextManager().size()).isEqualTo(1);
+
+        // rebuild index with byteman error
+        Injections.inject(failNumericIndexBuild);
+
+        // rebuild should fail
+        assertThatThrownBy(() -> sim.buildIndexesBlocking(cfs.getLiveSSTables(), new HashSet<>(sim.listIndexes()), true))
+                          .isInstanceOf(RuntimeException.class).hasMessageContaining("Injected failure");
+
+        // index is no longer queryable
+        assertThatThrownBy(() -> executeNet("SELECT id1 FROM %s WHERE v1>=0"))
+        .isInstanceOf(ReadFailureException.class);
+    }
+
+    /**
+     * Simulate SAI build error during compaction: compaction task should abort
+     */
+    @Test
+    public void testIndexCompactionAborted() throws Throwable
+    {
+        // prepare schema and data
+        createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('2', 0, '0');");
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('3', 1, '0');");
+        flush();
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(4, rows.all().size());
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        assertThat(cfs.getLiveSSTables()).hasSize(2);
+
+        SecondaryIndexManager sim = cfs.getIndexManager();
+        StorageAttachedIndex sai = (StorageAttachedIndex) sim.listIndexes().iterator().next();
+        assertThat(sai.getIndexContext().getView().getIndexes()).hasSize(2);
+
+        StorageAttachedIndexGroup saiGroup = StorageAttachedIndexGroup.getIndexGroup(cfs);
+        assertThat(saiGroup.sstableContextManager().size()).isEqualTo(2);
+
+        // rebuild index with byteman error
+        Injections.inject(failNumericIndexBuild);
+
+        // compaction task should fail
+        assertThatThrownBy(cfs::forceMajorCompaction)
+        .isInstanceOf(RuntimeException.class).hasMessageContaining("Injected failure");
+
+        // verify sstables and indexes are the same
+        assertThat(cfs.getLiveSSTables()).hasSize(2);
+        assertThat(saiGroup.sstableContextManager().size()).isEqualTo(2);
+        assertThat(sai.getIndexContext().getView().getIndexes()).hasSize(2);
+
+        // index is still queryable
+        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(4, rows.all().size());
+    }
+
+    @Test
+    public void verifyRebuildCorruptedFiles() throws Throwable // TODO CNDB-10210: fix this test
+    {
+        // prepare schema and data
+        createTable(CREATE_TABLE_TEMPLATE);
+        String numericIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        String stringIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+
+        for (CorruptionType corruptionType : CorruptionType.values())
+        {
+            verifyRebuildCorruptedFiles(numericIndexName, stringIndexName, corruptionType, false);
+            verifyRebuildCorruptedFiles(numericIndexName, stringIndexName, corruptionType, true);
+        }
+
+        assertEquals("Segment memory limiter should revert to zero following rebuild.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    private void verifyRebuildCorruptedFiles(String numericIndexName,
+                                             String stringIndexName,
+                                             CorruptionType corruptionType,
+                                             boolean rebuild) throws Throwable
+    {
+        IndexContext numericIndexContext = getIndexContext(numericIndexName);
+        IndexContext stringIndexContext = getIndexContext(stringIndexName);
+
+        for (IndexComponentType component : Version.latest().onDiskFormat().perSSTableComponentTypes())
+            verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, null, corruptionType, true, true, rebuild);
+
+        for (IndexComponentType component : Version.latest().onDiskFormat().perIndexComponentTypes(numericIndexContext))
+            verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, numericIndexContext, corruptionType, false, true, rebuild);
+
+        for (IndexComponentType component : Version.latest().onDiskFormat().perIndexComponentTypes(stringIndexContext))
+            verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, stringIndexContext, corruptionType, true, false, rebuild);
+    }
+
+    private void verifyRebuildIndexComponent(IndexContext numericIndexContext,
+                                             IndexContext stringIndexContext,
+                                             IndexComponentType component,
+                                             IndexContext corruptionContext,
+                                             CorruptionType corruptionType,
+                                             boolean failedStringIndex,
+                                             boolean failedNumericIndex,
+                                             boolean rebuild) throws Throwable
+    {
+        boolean encrypted = TEST_ENCRYPTION.getBoolean();
+
+        // The completion markers are valid if they exist on the file system so we only need to test
+        // their removal. If we are testing with encryption then we don't want to test any components
+        // that are encryptable unless they have been removed because encrypted components aren't
+        // checksum validated.
+
+        if (component == IndexComponentType.PRIMARY_KEY_TRIE || component == IndexComponentType.PRIMARY_KEY_BLOCKS || component == IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS)
+            return;
+
+        if (((component == IndexComponentType.GROUP_COMPLETION_MARKER) ||
+             (component == IndexComponentType.COLUMN_COMPLETION_MARKER)) &&
+            (corruptionType != CorruptionType.REMOVED))
+            return;
+
+        logger.info("CORRUPTING: " + component + ", corruption type = " + corruptionType);
+
+        int rowCount = 2;
+
+        // initial verification
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 1);
+        verifySSTableIndexes(stringIndexContext.getIndexName(), 1);
+        verifyIndexComponentFiles(numericIndexContext, stringIndexContext);
+        assertTrue(verifyChecksum(numericIndexContext));
+        assertTrue(verifyChecksum(numericIndexContext));
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(rowCount, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(rowCount, rows.all().size());
+
+        // corrupt file
+        if (corruptionContext != null)
+            corruptIndexComponent(component, corruptionContext, corruptionType);
+        else
+            corruptIndexComponent(component, corruptionType);
+
+        // Reload all SSTable indexes to manifest the corruption:
+        reloadSSTableIndex();
+
+        try
+        {
+            // If the corruption is that a file is missing entirely, the index won't be marked non-queryable...
+            rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+            // If we corrupted the index (and it's still queryable), we get either 0 or 2, depending on whether
+            // there is previous build of the index that gets automatically picked up. But mostly, we want to ensure
+            // the index does work if it's not corrupted.
+            if (!failedNumericIndex)
+                assertEquals(rowCount, rows.all().size());
+
+            //assertEquals(failedNumericIndex ? 0 : rowCount, rows.all().size());
+        }
+        catch (ReadFailureException e)
+        {
+            // ...but most kind of corruption will result in the index being non-queryable.
+        }
+
+        try
+        {
+            // If the corruption is that a file is missing entirely, the index won't be marked non-queryable...
+            rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+            // Same as above
+            if (!failedStringIndex)
+                assertEquals(rowCount, rows.all().size());
+        }
+        catch (ReadFailureException e)
+        {
+            // ...but most kind of corruption will result in the index being non-queryable.
+        }
+
+        if (rebuild)
+        {
+            rebuildIndexes(numericIndexContext.getIndexName(), stringIndexContext.getIndexName());
+        }
+        else
+        {
+            // Simulate the index repair that would occur on restart:
+            runInitializationTask();
+        }
+
+        // verify indexes are recovered
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 1);
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 1);
+        verifyIndexComponentFiles(numericIndexContext, stringIndexContext);
+
+        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(rowCount, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(rowCount, rows.all().size());
+    }
+
+
+    @Test
+    public void verifyCanRebuildAndReloadInPlaceToNewerVersion()
+    {
+        Version current = Version.latest();
+        try
+        {
+            SAIUtil.setLatestVersion(Version.AA);
+
+            // prepare schema and data
+            createTable(CREATE_TABLE_TEMPLATE);
+            String numericIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+            String stringIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+            IndexContext numericIndexContext = getIndexContext(numericIndexName);
+            IndexContext stringIndexContext = getIndexContext(stringIndexName);
+
+            int rowCount = 2;
+            execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+            execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+            flush();
+
+            // Sanity check first
+            ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+            assertEquals(rowCount, rows.all().size());
+            rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+            assertEquals(rowCount, rows.all().size());
+
+            verifySAIVersionInUse(Version.AA, numericIndexContext, stringIndexContext);
+
+            SAIUtil.setLatestVersion(current);
+
+            rebuildIndexes(numericIndexName, stringIndexName);
+            reloadSSTableIndexInPlace();
+
+            // This should still work
+            rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+            assertEquals(rowCount, rows.all().size());
+            rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+            assertEquals(rowCount, rows.all().size());
+
+            verifySAIVersionInUse(current, numericIndexContext, stringIndexContext);
+        }
+        finally
+        {
+            // If we haven't failed, we should already have done this, but if we did fail ...
+            SAIUtil.setLatestVersion(current);
+        }
+    }
+
+    @Test
+    public void verifyCleanupFailedPerIndexFiles() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        disableCompaction(KEYSPACE);
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        flush();
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+
+        // Inject failure
+        Injections.inject(failPerIndexMetaCompletion);
+        failPerIndexMetaCompletion.enable();
+
+        try
+        {
+            // Create a new index, which will actuate a build compaction and fail, but leave the node running...
+            IndexContext numericIndexContext = createIndexContext(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+            // two index builders running in different compaction threads because of parallelised index initial build
+            waitForAssert(() -> assertEquals(2, INDEX_BUILD_COUNTER.get()));
+            waitForCompactionsFinished();
+
+            // Only token/offset files for the first SSTable in the compaction task should exist, while column-specific files are blown away:
+            verifyIndexFiles(numericIndexContext, null, 2, 0, 0, 0, 0);
+
+            assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
+            assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+        }
+        finally
+        {
+            failPerIndexMetaCompletion.disable();
+        }
+    }
+
+    @Test
+    public void verifyCleanupFailedTokenOffsetFiles() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        disableCompaction(KEYSPACE);
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        flush();
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+
+        // Inject failure
+        Injections.inject(failPerSSTableTokenAdd);
+        failPerSSTableTokenAdd.enable();
+
+        try
+        {
+            // Create a new index, which will actuate a build compaction and fail, but leave the node running...
+            createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+            // two index builders running in different compaction threads because of parallelised index initial build
+            waitForAssert(() -> assertEquals(2, INDEX_BUILD_COUNTER.get()));
+            waitForAssert(() -> assertEquals(0, getCompactionTasks()));
+
+            // SSTable-level token/offset file(s) should be removed, while column-specific files never existed:
+            verifyNoIndexFiles();
+
+            assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
+            assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+        }
+        finally
+        {
+            failPerSSTableTokenAdd.disable();
+        }
+    }
+
+    @Test
+    public void verifyFlushAndCompactEmptyIndex() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        disableCompaction(KEYSPACE);
+
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+        IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
+
+        // flush empty index
+        execute("INSERT INTO %s (id1) VALUES ('0');");
+        flush();
+
+        execute("INSERT INTO %s (id1) VALUES ('1');");
+        flush();
+
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 0, 0, 2, 2);
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(0, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(0, rows.all().size());
+
+        // compact empty index
+        compact();
+        waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 0, 0, 1, 1));
+
+        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(0, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(0, rows.all().size());
+
+        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    @Test
+    public void verifyFlushAndCompactNonIndexableRows() throws Throwable
+    {
+        // valid row ids, but no valid indexable content
+        Runnable populateData = () -> {
+            try
+            {
+                execute("INSERT INTO %s (id1) VALUES ('0');");
+                flush();
+
+                execute("INSERT INTO %s (id1) VALUES ('1');");
+                flush();
+            }
+            catch (Throwable e)
+            {
+                throw Throwables.unchecked(e);
+            }
+        };
+
+
+        verifyFlushAndCompactEmptyIndexes(populateData);
+    }
+
+    @Test
+    public void verifyFlushAndCompactTombstones() throws Throwable
+    {
+        // no valid row ids
+        Runnable populateData = () -> {
+            try
+            {
+                execute("DELETE FROM %s WHERE id1 = '0'");
+                flush();
+
+                execute("DELETE FROM %s WHERE id1 = '1'");
+                flush();
+            }
+            catch (Throwable e)
+            {
+                throw Throwables.unchecked(e);
+            }
+        };
+
+        verifyFlushAndCompactEmptyIndexes(populateData);
+    }
+
+    private void verifyFlushAndCompactEmptyIndexes(Runnable populateData) throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        disableCompaction(KEYSPACE);
+
+
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+        IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
+
+        populateData.run();
+        verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 2, 2);
+        verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 2, 2);
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 0, 0, 2, 2);
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(0, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(0, rows.all().size());
+
+        // compact empty index
+        compact();
+        verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 1, 1);
+        verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 1, 1);
+        waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 0, 0, 1, 1));
+
+        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(0, rows.all().size());
+        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
+        assertEquals(0, rows.all().size());
+
+        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+    }
+
+    @Test
+    public void droppingIndexStopInitialIndexBuild() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        disableCompaction(KEYSPACE);
+
+        int num = 100;
+        for (int i = 0; i < num; i++)
+        {
+            execute("INSERT INTO %s (id1, v1, v2) VALUES (?, 0, '0')", Integer.toString(i));
+        }
+        flush();
+
+        Injections.Barrier delayIndexBuilderCompletion = Injections.newBarrier("delayIndexBuilder", 2, false)
+                                                                   .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndexBuilder.class).onMethod("build"))
+                                                                   .build();
+
+        Injections.inject(delayIndexBuilderCompletion);
+        String indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        waitForAssert(() -> assertEquals(1, delayIndexBuilderCompletion.getCount()));
+
+        dropIndex("DROP INDEX %s." + indexName);
+
+        // let blocked builders to continue
+        delayIndexBuilderCompletion.countDown();
+        waitForCompactions();
+
+        delayIndexBuilderCompletion.disable();
+
+        verifySSTableIndexes(indexName, 0);
+        assertFalse("Expect index not built", SystemKeyspace.isIndexBuilt(KEYSPACE, indexName));
+
+        // create index again, it should succeed
+        indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        verifySSTableIndexes(indexName, 1);
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(num, rows.all().size());
+    }
+
+    @Test
+    public void nodetoolStopInitialIndexBuild() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        disableCompaction(KEYSPACE);
+
+        // create 100 rows into 1 sstable
+        int num = 100;
+        int sstable = 1;
+        for (int i = 0; i < num; i++)
+        {
+            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', 0, '0');");
+        }
+        flush();
+
+        Injections.Barrier delayIndexBuilderCompletion = Injections.newBarrierAwait("delayIndexBuilder", 1, true)
+                                                                   .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndexBuilder.class).onMethod("build"))
+                                                                   .build();
+
+        Injections.inject(delayIndexBuilderCompletion);
+
+        IndexContext numericIndexContext = getIndexContext(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")));
+
+        waitForAssert(() -> assertTrue(getCompactionTasks() > 0), 1000, TimeUnit.MILLISECONDS);
+
+        // Stop initial index build by interrupting active and pending compactions
+        int attempt = 20;
+        while (getCompactionTasks() > 0 && attempt > 0)
+        {
+            System.out.println("Attempt " + attempt + " at stopping the compaction tasks");
+
+            // only interrupts active compactions, not pending compactions.
+            CompactionManager.instance.stopCompaction(OperationType.INDEX_BUILD.name());
+            // let blocked builder to continue, but still block pending builder threads
+            delayIndexBuilderCompletion.reset();
+
+            Thread.sleep(3000);
+            attempt--;
+        }
+        if (getCompactionTasks() > 0)
+            fail("Compaction tasks are not interrupted.");
+
+        delayIndexBuilderCompletion.disable();
+
+        // initial index builder should have stopped abruptly resulting in the index not being queryable
+        verifyInitialIndexFailed(numericIndexContext.getIndexName());
+        assertFalse(areAllTableIndexesQueryable());
+
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        for (Index i : cfs.indexManager.listIndexes())
+        {
+            StorageAttachedIndex index = (StorageAttachedIndex) i;
+            assertTrue(index.getIndexContext().getLiveMemtables().isEmpty());
+
+            View view = index.getIndexContext().getView();
+            assertTrue("Expect index build stopped", view.getIndexes().isEmpty());
+        }
+
+        assertEquals("Segment memory limiter should revert to zero on interrupted compactions.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+
+        // rebuild index
+        ColumnFamilyStore.rebuildSecondaryIndex(KEYSPACE, currentTable(), numericIndexContext.getIndexName());
+
+        verifyIndexFiles(numericIndexContext, null, sstable, 0);
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(num, rows.all().size());
+
+        assertEquals("Segment memory limiter should revert to zero following rebuild.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+
+        assertTrue(verifyChecksum(numericIndexContext));
+    }
+
+    @Test
+    public void shouldRejectQueriesWithCustomExpressions()
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        String index = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+
+        assertThatThrownBy(() -> executeNet(String.format("SELECT * FROM %%s WHERE expr(%s, 0)", index)))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessage(String.format(IndexRestrictions.CUSTOM_EXPRESSION_NOT_SUPPORTED, index));
+    }
+
+    @Test
+    public void testInitialBuildParallelism()
+    {
+        Function<Long, SSTableReader> createMockSSTable = onDiskLength -> {
+            SSTableReader reader = Mockito.mock(SSTableReader.class);
+            when(reader.onDiskLength()).thenReturn(onDiskLength);
+            return reader;
+        };
+
+        Function<List<SSTableReader>, List<Long>> toSize = sstables -> sstables.stream().map(SSTableReader::onDiskLength).collect(Collectors.toList());
+
+        // total size = 55
+        List<SSTableReader> sstables = LongStream.range(1, 11).boxed().map(createMockSSTable).collect(Collectors.toList());
+
+        // avg = 55 == total size
+        List<List<SSTableReader>> groups = StorageAttachedIndex.groupBySize(sstables, 1);
+        Iterator<List<SSTableReader>> iterator = groups.iterator();
+        assertEquals(1, groups.size());
+        assertEquals(Arrays.asList(10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L, 2L, 1L), toSize.apply(iterator.next())); // size = 55
+
+        // avg = 27.5
+        groups = StorageAttachedIndex.groupBySize(sstables, 2);
+        iterator = groups.iterator();
+        assertEquals(2, groups.size());
+        assertEquals(Arrays.asList(10L, 9L, 8L, 7L), toSize.apply(iterator.next())); // size = 34
+        assertEquals(Arrays.asList(6L, 5L, 4L, 3L, 2L, 1L), toSize.apply(iterator.next())); // size = 21
+
+        // avg = 18.333
+        groups = StorageAttachedIndex.groupBySize(sstables, 3);
+        iterator = groups.iterator();
+        assertEquals(3, groups.size());
+        assertEquals(Arrays.asList(10L, 9L), toSize.apply(iterator.next())); // size = 19
+        assertEquals(Arrays.asList(8L, 7L, 6L), toSize.apply(iterator.next())); // size = 21
+        assertEquals(Arrays.asList(5L, 4L, 3L, 2L, 1L), toSize.apply(iterator.next())); // size = 15
+
+        // avg = 11
+        groups = StorageAttachedIndex.groupBySize(sstables, 5);
+        iterator = groups.iterator();
+        assertEquals(4, groups.size());
+        assertEquals(Arrays.asList(10L, 9L), toSize.apply(iterator.next())); // size = 19
+        assertEquals(Arrays.asList(8L, 7L), toSize.apply(iterator.next())); // size = 15
+        assertEquals(Arrays.asList(6L, 5L), toSize.apply(iterator.next())); // size = 11
+        assertEquals(Arrays.asList(4L, 3L, 2L, 1L), toSize.apply(iterator.next())); // size = 11
+
+        // avg = 5.5
+        groups = StorageAttachedIndex.groupBySize(sstables, 10);
+        iterator = groups.iterator();
+        assertEquals(7, groups.size());
+        assertEquals(singletonList(10L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(9L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(8L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(7L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(6L), toSize.apply(iterator.next()));
+        assertEquals(Arrays.asList(5L, 4L), toSize.apply(iterator.next()));
+        assertEquals(Arrays.asList(3L, 2L, 1L), toSize.apply(iterator.next()));
+
+        // avg = 2.75
+        groups = StorageAttachedIndex.groupBySize(sstables, 20);
+        iterator = groups.iterator();
+        assertEquals(9, groups.size());
+        assertEquals(singletonList(10L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(9L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(8L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(7L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(6L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(5L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(4L), toSize.apply(iterator.next()));
+        assertEquals(singletonList(3L), toSize.apply(iterator.next()));
+        assertEquals(Arrays.asList(2L, 1L), toSize.apply(iterator.next()));
+    }
+
+    @Test
+    public void shouldRejectLargeStringTerms()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+        String insert = "INSERT INTO %s (k, v) VALUES (0, ?)";
+
+        // insert a text term with the max possible size
+        execute(insert, ByteBuffer.allocate(IndexContext.MAX_STRING_TERM_SIZE));
+
+        // insert a text term over the max possible size
+        assertThatThrownBy(() -> execute(insert, ByteBuffer.allocate(IndexContext.MAX_STRING_TERM_SIZE + 1)))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Term of column v exceeds the byte limit for index");
+    }
+
+    @Test
+    public void shouldRejectLargeFrozenTerms()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<list<text>>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(full(v)) USING 'StorageAttachedIndex'");
+        String insert = "INSERT INTO %s (k, v) VALUES (0, ?)";
+
+        // insert a frozen term with the max possible size
+        // list serialization uses 4 bytes for the collection size, and then 4 bytes per each item size
+        String value1 = UTF8Type.instance.compose(ByteBuffer.allocate(IndexContext.MAX_FROZEN_TERM_SIZE / 2 - 8));
+        String value2 = UTF8Type.instance.compose(ByteBuffer.allocate(IndexContext.MAX_FROZEN_TERM_SIZE / 2 - 4));
+        execute(insert, Arrays.asList(value1, value2));
+
+        // insert a frozen term over the max possible size
+        assertThatThrownBy(() -> execute(insert, Arrays.asList(value1, value2, "x")))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Term of column v exceeds the byte limit for index");
+    }
+
+    @Test
+    public void shouldRejectLargeAnalyzedTerms()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'" +
+                    " WITH OPTIONS = {'index_analyzer': 'whitespace'}");
+        String insert = "INSERT INTO %s (k, v) VALUES (0, ?)";
+
+
+        // insert an analyzed column with terms of cumulating up to the max possible size
+        String term = UTF8Type.instance.compose(ByteBuffer.allocate(IndexContext.MAX_ANALYZED_SIZE / 2));
+        execute(insert, term + ' ' + term);
+
+        // insert a frozen term over the max possible size
+        assertThatThrownBy(() -> execute(insert, term + ' ' + term + ' ' + term))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Term's analyzed size for column v exceeds the cumulative limit for index");
+    }
+
+    @Test
+    public void shouldRejectLargeVector()
+    {
+        String table = "CREATE TABLE %%s (k int PRIMARY KEY, v vector<float, %d>)";
+        String index = "CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}";
+        String insert = "INSERT INTO %s (k, v) VALUES (0, ?)";
+
+        // insert a vector term with the max possible size
+        int dimensions = IndexContext.MAX_VECTOR_TERM_SIZE / Float.BYTES;
+        createTable(String.format(table, dimensions));
+        createIndex(index);
+        execute(insert, vector(new float[dimensions]));
+
+        // create a vector index producing terms over the max possible size
+        createTable(String.format(table, dimensions + 1));
+        // VSTODO: uncomment when https://github.com/riptano/VECTOR-SEARCH/issues/85 is solved
+//        assertThatThrownBy(() -> execute(index))
+//        .isInstanceOf(InvalidRequestException.class)
+//        .hasMessageContaining("An index of vector<float, 4097> will produce terms of 16.004KiB, " +
+//                              "exceeding the max vector term size of 16.000KiB. " +
+//                              "That sets an implicit limit of 4096 dimensions for float vectors.");
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/NonNumericTermsDistributionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/NonNumericTermsDistributionTest.java
new file mode 100644
index 000000000000..9de932039576
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/NonNumericTermsDistributionTest.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.memory.TrieMemtableIndex;
+import org.apache.cassandra.index.sai.plan.Expression;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test aims to cover the different encodings of terms distribution to ensure expected results.
+ */
+public class NonNumericTermsDistributionTest extends SAITester
+{
+    static {
+        SAIUtil.setLatestVersion(Version.latest().onOrAfter(Version.EB) ? Version.latest() : Version.EB);
+    }
+
+    private StorageAttachedIndex getIndex()
+    {
+        // A bit brittle, but this is the most realistic way to test encodings.
+        return (StorageAttachedIndex) Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).getIndexManager()
+                                              .getIndexByName(currentIndex());
+    }
+
+    @Test
+    public void testBooleanIndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a boolean)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, true)");
+        execute("INSERT INTO %s (pk, a) VALUES (2, true)");
+        execute("INSERT INTO %s (pk, a) VALUES (3, true)");
+        execute("INSERT INTO %s (pk, a) VALUES (4, false)");
+        execute("INSERT INTO %s (pk, a) VALUES (5, false)");
+        execute("INSERT INTO %s (pk, a) VALUES (6, true)");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "true", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "false", 2);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "true", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "false", 2);
+    }
+
+    @Test
+    public void testUtf8IndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a text)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+        execute("INSERT INTO %s (pk, a) VALUES (1, 'a')");
+        execute("INSERT INTO %s (pk, a) VALUES (2, 'aa')");
+        execute("INSERT INTO %s (pk, a) VALUES (3, 'ab')");
+        execute("INSERT INTO %s (pk, a) VALUES (4, 'c')");
+        execute("INSERT INTO %s (pk, a) VALUES (5, 'z')");
+        execute("INSERT INTO %s (pk, a) VALUES (6, '•')");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "ab", 1);
+        assertInMemoryEstimateCount(sai, Operator.LT, "ab", 2);
+        assertInMemoryEstimateCount(sai, Operator.GT, "ab", 3);
+        assertInMemoryEstimateCount(sai, Operator.LTE, "ab", 3);
+        assertInMemoryEstimateCount(sai, Operator.GTE, "ab", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "•", 1);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "x", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "ab", 1);
+        assertSSTableEstimateCount(sai, Operator.LT, "ab", 2);
+        assertSSTableEstimateCount(sai, Operator.GT, "ab", 3);
+        assertSSTableEstimateCount(sai, Operator.LTE, "ab", 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, "ab", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "•", 1);
+        assertSSTableEstimateCount(sai, Operator.EQ, "x", 0);
+    }
+
+    @Test
+    public void testUtf8IndexEstimatesForManyRows()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a text)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        final int ROW_COUNT = 10000;
+        final int ROWS_PER_POINT = 10;
+
+        List<String> values = new ArrayList<>();
+        String minValue = null;
+        String maxValue = null;
+
+        int rowNum = 0;
+        while (rowNum < ROW_COUNT)
+        {
+            String value = randomString(getRandom().nextIntBetween(12, 16));
+            values.add(value);
+            if (minValue == null || value.compareTo(minValue) < 0)
+                minValue = value;
+            if (maxValue == null || value.compareTo(maxValue) > 0)
+                maxValue = value;
+
+            for (int j = 0; j < ROWS_PER_POINT; j++)
+            {
+                execute("INSERT INTO %s (pk, a) VALUES (?, ?)", rowNum, value);
+                rowNum++;
+            }
+        }
+
+        var sai = getIndex();
+
+        for (int i = 0; i < 100; i++)
+            assertSSTableEstimateCount(sai, Operator.EQ, randomString(16), 0);
+        for (String value : values)
+            assertInMemoryEstimateCount(sai, Operator.EQ, value, ROWS_PER_POINT, ROWS_PER_POINT / 10);
+
+        flush();
+
+        for (int i = 0; i < 100; i++)
+        {
+            // For non-existing rows, we should always estimate 0 if out of range, but there is a bit of uncertainty if
+            // we are asked about the value within the range of the index. In that case, the actual estimate may be
+            // either 0 or ROWS_PER_POINT depending on whether we hit a value in the histogram bucket which
+            // already contains one of known most frequent values.
+            String randomValue = randomString(getRandom().nextIntBetween(12, 16));
+            long maxRows = (randomValue.compareTo(minValue) >= 0 && randomValue.compareTo(maxValue) <= 0) ? ROWS_PER_POINT : 0;
+            assertTrue(estimateSSTableRowCount(sai, Operator.EQ, randomValue) <= maxRows);
+        }
+
+        for (String value : values)
+            assertSSTableEstimateCount(sai, Operator.EQ, value, ROWS_PER_POINT, ROWS_PER_POINT / 10);
+    }
+
+    @Test
+    public void testAsciiIndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a ascii)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, 'a')");
+        execute("INSERT INTO %s (pk, a) VALUES (2, 'aa')");
+        execute("INSERT INTO %s (pk, a) VALUES (3, 'ab')");
+        execute("INSERT INTO %s (pk, a) VALUES (4, 'c')");
+        execute("INSERT INTO %s (pk, a) VALUES (5, 'z')");
+        execute("INSERT INTO %s (pk, a) VALUES (6, 'zz')");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "ab", 1);
+        assertInMemoryEstimateCount(sai, Operator.LT, "ab", 2);
+        assertInMemoryEstimateCount(sai, Operator.GT, "ab", 3);
+        assertInMemoryEstimateCount(sai, Operator.LTE, "ab", 3);
+        assertInMemoryEstimateCount(sai, Operator.GTE, "ab", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "x", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "ab", 1);
+        assertSSTableEstimateCount(sai, Operator.LT, "ab", 2);
+        assertSSTableEstimateCount(sai, Operator.GT, "ab", 3);
+        assertSSTableEstimateCount(sai, Operator.LTE, "ab", 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, "ab", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "x", 0);
+    }
+
+    @Test
+    public void testInetAddressIndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a inet)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, '127.0.0.1')");
+        execute("INSERT INTO %s (pk, a) VALUES (2, '192.168.1.1')");
+        execute("INSERT INTO %s (pk, a) VALUES (3, '192.168.1.2')");
+        execute("INSERT INTO %s (pk, a) VALUES (4, '192.168.1.3')");
+        execute("INSERT INTO %s (pk, a) VALUES (5, 'ff06::fa')");
+        execute("INSERT INTO %s (pk, a) VALUES (6, 'ff06::ff')");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "192.168.1.2", 1);
+        assertInMemoryEstimateCount(sai, Operator.LT, "192.168.1.2", 2);
+        assertInMemoryEstimateCount(sai, Operator.GT, "192.168.1.2", 3);
+        assertInMemoryEstimateCount(sai, Operator.LTE, "192.168.1.2", 3);
+        assertInMemoryEstimateCount(sai, Operator.GTE, "192.168.1.2", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "ff06::ff", 1);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "ff07::ff", 0);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "192.168.11.11", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "192.168.1.2", 1);
+        assertSSTableEstimateCount(sai, Operator.LT, "192.168.1.2", 2);
+        assertSSTableEstimateCount(sai, Operator.GT, "192.168.1.2", 3);
+        assertSSTableEstimateCount(sai, Operator.LTE, "192.168.1.2", 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, "192.168.1.2", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "ff06::ff", 1);
+        assertSSTableEstimateCount(sai, Operator.EQ, "ff07::ff", 0);
+        assertSSTableEstimateCount(sai, Operator.EQ, "192.168.11.11", 0);
+    }
+
+    @Test
+    public void testDateIndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a date)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, '1810-12-31')");
+        execute("INSERT INTO %s (pk, a) VALUES (2, '2024-01-01')");
+        execute("INSERT INTO %s (pk, a) VALUES (3, '2024-01-01')");
+        execute("INSERT INTO %s (pk, a) VALUES (4, '2024-01-01')");
+        execute("INSERT INTO %s (pk, a) VALUES (5, '2024-01-02')");
+        execute("INSERT INTO %s (pk, a) VALUES (6, '2550-01-01')");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "2024-01-01", 3);
+        assertInMemoryEstimateCount(sai, Operator.LT, "2024-01-01", 1);
+        assertInMemoryEstimateCount(sai, Operator.GT, "2024-01-01", 2);
+        assertInMemoryEstimateCount(sai, Operator.LTE, "2024-01-01", 4);
+        assertInMemoryEstimateCount(sai, Operator.GTE, "2024-01-01", 5);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "2550-01-01", 1);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "2550-01-02", 0);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "1810-12-31", 1);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "1810-12-30", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "2024-01-01", 3);
+        assertSSTableEstimateCount(sai, Operator.LT, "2024-01-01", 1);
+        assertSSTableEstimateCount(sai, Operator.GT, "2024-01-01", 2);
+        assertSSTableEstimateCount(sai, Operator.LTE, "2024-01-01", 4);
+        assertSSTableEstimateCount(sai, Operator.GTE, "2024-01-01", 5);
+        assertSSTableEstimateCount(sai, Operator.EQ, "2550-01-01", 1);
+        assertSSTableEstimateCount(sai, Operator.EQ, "2550-01-02", 0);
+        assertSSTableEstimateCount(sai, Operator.EQ, "1810-12-31", 1);
+        assertSSTableEstimateCount(sai, Operator.EQ, "1810-12-30", 0);
+    }
+
+    @Test
+    public void testTimeIndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a time)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, '4:01:23.000000000')");
+        execute("INSERT INTO %s (pk, a) VALUES (2, '11:59:59.999999999')");
+        execute("INSERT INTO %s (pk, a) VALUES (3, '12:00:00.000000000')");
+        execute("INSERT INTO %s (pk, a) VALUES (4, '12:00:00.000000001')");
+        execute("INSERT INTO %s (pk, a) VALUES (5, '15:00:23.000000000')");
+        execute("INSERT INTO %s (pk, a) VALUES (6, '23:59:59.999999999')");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "12:00:00.000000000", 1);
+        assertInMemoryEstimateCount(sai, Operator.LT, "12:00:00.000000000", 2);
+        assertInMemoryEstimateCount(sai, Operator.GT, "12:00:00.000000000", 3);
+        assertInMemoryEstimateCount(sai, Operator.LTE, "12:00:00.000000000", 3);
+        assertInMemoryEstimateCount(sai, Operator.GTE, "12:00:00.000000000", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "23:59:59.999999999", 1);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "23:59:59.999999998", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "12:00:00.000000000", 1);
+        assertSSTableEstimateCount(sai, Operator.LT, "12:00:00.000000000", 2);
+        assertSSTableEstimateCount(sai, Operator.GT, "12:00:00.000000000", 3);
+        assertSSTableEstimateCount(sai, Operator.LTE, "12:00:00.000000000", 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, "12:00:00.000000000", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "23:59:59.999999999", 1);
+        assertSSTableEstimateCount(sai, Operator.EQ, "23:59:59.999999998", 0);
+    }
+
+    @Test
+    public void testTimestampIndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a timestamp)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, '1810-12-31 16:00:00')");
+        execute("INSERT INTO %s (pk, a) VALUES (2, '2024-01-01 11:00:59.999')");
+        execute("INSERT INTO %s (pk, a) VALUES (3, '2024-01-01 12:00:00.000')");
+        execute("INSERT INTO %s (pk, a) VALUES (4, '2024-01-01 12:00:00.001')");
+        execute("INSERT INTO %s (pk, a) VALUES (5, '2024-01-02 12:00:00')");
+        execute("INSERT INTO %s (pk, a) VALUES (6, '2550-01-01 12:00:00')");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "2024-01-01 12:00:00.000", 1);
+        assertInMemoryEstimateCount(sai, Operator.LT, "2024-01-01 12:00:00.000", 2);
+        assertInMemoryEstimateCount(sai, Operator.GT, "2024-01-01 12:00:00.000", 3);
+        assertInMemoryEstimateCount(sai, Operator.LTE, "2024-01-01 12:00:00.000", 3);
+        assertInMemoryEstimateCount(sai, Operator.GTE, "2024-01-01 12:00:00.000", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "2550-01-01 12:00:00", 1);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "2550-01-01 12:00:01", 0);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "1810-12-31 16:00:00", 1);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "1810-12-30 16:00:00", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "2024-01-01 12:00:00.000", 1);
+        assertSSTableEstimateCount(sai, Operator.LT, "2024-01-01 12:00:00.000", 2);
+        assertSSTableEstimateCount(sai, Operator.GT, "2024-01-01 12:00:00.000", 3);
+        assertSSTableEstimateCount(sai, Operator.LTE, "2024-01-01 12:00:00.000", 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, "2024-01-01 12:00:00.000", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "2550-01-01 12:00:00", 1);
+        assertSSTableEstimateCount(sai, Operator.EQ, "2550-01-01 12:00:01", 0);
+        assertSSTableEstimateCount(sai, Operator.EQ, "1810-12-31 16:00:00", 1);
+        assertSSTableEstimateCount(sai, Operator.EQ, "1810-12-30 16:00:00", 0);
+    }
+
+    @Test
+    public void testUuidIndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a uuid)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, 53aeb60c-8ac9-11ef-b864-0242ac120055)");
+        execute("INSERT INTO %s (pk, a) VALUES (2, 53aeb60c-8ac9-11ef-b864-0242ac120055)");
+        execute("INSERT INTO %s (pk, a) VALUES (3, 53aeb7ad-8ac9-11ef-b864-0242ac120055)");
+        execute("INSERT INTO %s (pk, a) VALUES (4, 53aeb7ad-8ac9-11ef-b864-0242ac120055)");
+        execute("INSERT INTO %s (pk, a) VALUES (5, 53aeb7ad-8ac9-11ef-b864-0242ac120055)");
+        execute("INSERT INTO %s (pk, a) VALUES (6, 53aeb7ad-8ac9-11ef-b864-0242ac120055)");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "53aeb60c-8ac9-11ef-b864-0242ac120055", 2);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "53aeb7ad-8ac9-11ef-b864-0242ac120099", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "53aeb60c-8ac9-11ef-b864-0242ac120055", 2);
+        assertSSTableEstimateCount(sai, Operator.EQ, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "53aeb7ad-8ac9-11ef-b864-0242ac120099", 0);
+    }
+
+    @Test
+    public void testTimeUuidIndexEstimates()
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a timeuuid)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, 53aeb60c-8ac9-11ef-b864-0242ac120055)");
+        execute("INSERT INTO %s (pk, a) VALUES (2, 53aeb7ad-8ac9-11ef-b864-0242ac120054)");
+        execute("INSERT INTO %s (pk, a) VALUES (3, 53aeb7ad-8ac9-11ef-b864-0242ac120055)");
+        execute("INSERT INTO %s (pk, a) VALUES (4, 53aeb7ad-8ac9-11ef-b864-0242ac120056)");
+        execute("INSERT INTO %s (pk, a) VALUES (5, 53aeba94-8ac9-11ef-b864-0242ac120040)");
+        execute("INSERT INTO %s (pk, a) VALUES (6, 53aebb34-8ac9-11ef-b864-0242ac120030)");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 1);
+        assertInMemoryEstimateCount(sai, Operator.LT, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 2);
+        assertInMemoryEstimateCount(sai, Operator.GT, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 3);
+        assertInMemoryEstimateCount(sai, Operator.LTE, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 3);
+        assertInMemoryEstimateCount(sai, Operator.GTE, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "53aeb7ad-8ac9-11ef-b864-0242ac120077", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 1);
+        assertSSTableEstimateCount(sai, Operator.LT, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 2);
+        assertSSTableEstimateCount(sai, Operator.GT, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 3);
+        assertSSTableEstimateCount(sai, Operator.LTE, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, "53aeb7ad-8ac9-11ef-b864-0242ac120055", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "53aeb7ad-8ac9-11ef-b864-0242ac120077", 0);
+    }
+
+    private void assertInMemoryEstimateCount(StorageAttachedIndex index, Operator op, String value, long expectedCount)
+    {
+        assertInMemoryEstimateCount(index, op, value, expectedCount, 0);
+    }
+
+    private void assertInMemoryEstimateCount(StorageAttachedIndex index, Operator op, String value, long expectedCount, long uncertainty)
+    {
+        var memtableIndexes = index.getIndexContext()
+                                   .getLiveMemtables()
+                                   .values();
+        var expression = buildExpression(index, op, value);
+        var memoryCount = 0L;
+        var wholeRange = DataRange.allData(index.getIndexContext().getPartitioner()).keyRange();
+        for (var memtableIndex : memtableIndexes)
+            for (var memoryIndex : ((TrieMemtableIndex) memtableIndex).getRangeIndexes())
+                memoryCount += memoryIndex.estimateMatchingRowsCount(expression, wholeRange);
+
+        assertEstimateCorrect(expectedCount, uncertainty, memoryCount);
+    }
+
+
+    private void assertSSTableEstimateCount(StorageAttachedIndex index, Operator op, String value, long expectedCount)
+    {
+        assertSSTableEstimateCount(index, op, value, expectedCount, 0);
+    }
+
+    private void assertSSTableEstimateCount(StorageAttachedIndex index, Operator op, String value, long expectedCount, long uncertainty)
+    {
+        var onDiskCount = estimateSSTableRowCount(index, op, value);
+        assertEstimateCorrect(expectedCount, uncertainty, onDiskCount);
+    }
+
+    private long estimateSSTableRowCount(StorageAttachedIndex index, Operator op, String value)
+    {
+        var expression = buildExpression(index, op, value);
+        var wholeRange = DataRange.allData(index.getIndexContext().getPartitioner()).keyRange();
+        var view = index.getIndexContext().getView();
+        var onDiskCount = 0L;
+        for (var sstableIndex : view.getIndexes())
+            onDiskCount += sstableIndex.estimateMatchingRowsCount(expression, wholeRange);
+        return onDiskCount;
+    }
+
+    private void assertEstimateCorrect(long expectedCount, long uncertainty, long actualCount)
+    {
+        if (uncertainty == 0)
+        {
+            assertEquals(expectedCount, actualCount);
+        }
+        else
+        {
+            assertTrue("Expected min: " + (expectedCount - uncertainty) + " Actual: " + actualCount,
+                       actualCount >= expectedCount - uncertainty);
+            assertTrue("Expected max: " + (expectedCount + uncertainty) + " Actual: " + actualCount,
+                       actualCount <= expectedCount + uncertainty);
+        }
+    }
+
+    private Expression buildExpression(StorageAttachedIndex index, Operator op, String value)
+    {
+        var expression = new Expression(index.getIndexContext());
+        expression.add(op, index.getIndexContext().getValidator().fromString(value));
+        return expression;
+    }
+
+    private static String randomString(int length) {
+        String alphanumericChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                                   + "abcdefghijklmnopqrstuvwxyz"
+                                   + "0123456789";
+        StringBuilder sb = new StringBuilder(length);
+        var random = getRandom();
+
+        for (int i = 0; i < length; i++) {
+            int index = random.nextIntBetween(0, alphanumericChars.length() - 1);
+            sb.append(alphanumericChars.charAt(index));
+        }
+        return sb.toString();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/NotOnTruncatedKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/NotOnTruncatedKeyTest.java
new file mode 100644
index 000000000000..b36339d9cf1f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/NotOnTruncatedKeyTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
+public class NotOnTruncatedKeyTest extends SAITester
+{
+    @Parameterized.Parameter
+    public Version version;
+    @Parameterized.Parameter(1)
+    public String truncatedType;
+
+    private Version latest;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        latest = Version.latest();
+        SAIUtil.setLatestVersion(version);
+    }
+
+    @After
+    public void teardown() throws Throwable
+    {
+        SAIUtil.setLatestVersion(latest);
+    }
+
+    @Test
+    public void testLossyQueries() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk text PRIMARY KEY, x " + truncatedType + ')');
+        createIndex("CREATE CUSTOM INDEX ON %s(x) USING 'StorageAttachedIndex'");
+
+        // Decimals and Big Integers are truncated to 24 bytes in the numeric index.
+        // These numbers are chosen to test for the expected results in the case of truncation.
+        var a = "1111111111111111111111111111111111111111111111111111111111";
+        var b = "1111111111111111111111111111111111111111111111111111111112";
+        var c = "1111111111111111111111111111111111111111111111111111111113";
+
+        var d = "1";
+
+        execute("INSERT INTO %s (pk, x) VALUES ('a', " + a + ')');
+        execute("INSERT INTO %s (pk, x) VALUES ('b', " + b + ')');
+        execute("INSERT INTO %s (pk, x) VALUES ('c', " + c + ')');
+        execute("INSERT INTO %s (pk, x) VALUES ('d', " + d + ')');
+
+        beforeAndAfterFlush(() -> {
+            // Test two kinds of NOT queries
+            assertRows(execute("SELECT pk FROM %s WHERE x NOT IN (" + b + ')'),
+                       row("a"), row("c"), row("d"));
+            assertRows(execute("SELECT pk FROM %s WHERE x != " + b),
+                       row("a"), row("c"), row("d"));
+        });
+    }
+
+    @Parameterized.Parameters
+    public static List<Object[]> data()
+    {
+        var indexVersions = List.of(Version.DB, Version.EB);
+        var truncatedTypes = List.of("decimal", "varint");
+        return Lists.cartesianProduct(indexVersions, truncatedTypes)
+                    .stream().map(List::toArray).collect(Collectors.toList());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/NumericTermsDistributionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/NumericTermsDistributionTest.java
new file mode 100644
index 000000000000..beea95afb468
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/NumericTermsDistributionTest.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Collection;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.NumberType;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.memory.TrieMemtableIndex;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test aims to cover the different encodings of terms distribution to ensure expected results.
+ */
+@RunWith(Parameterized.class)
+public class NumericTermsDistributionTest extends SAITester
+{
+    static {
+        SAIUtil.setLatestVersion(Version.latest().onOrAfter(Version.EB) ? Version.latest() : Version.EB);
+    }
+
+    @Parameterized.Parameter
+    public CQL3Type.Native testType;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static Collection<Object[]> data()
+    {
+        return Stream.of(CQL3Type.Native.values())
+                     .filter(type -> {
+                         if (!(type.getType() instanceof NumberType))
+                             return false;
+                         // Counters cannot have indexes.
+                         return !type.getType().isCounter();
+                     })
+                     .map(type -> new Object[]{ type })
+                     .collect(Collectors.toList());
+    }
+
+    private StorageAttachedIndex getIndex()
+    {
+        // A bit brittle, but this is the most realistic way to test encodings.
+        return (StorageAttachedIndex) Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).getIndexManager()
+                                              .getIndexByName(currentIndex());
+    }
+
+    @Test
+    public void testNumericIndexEstimates() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, a " + testType + ')');
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        execute("INSERT INTO %s (pk, a) VALUES (1, -10)");
+        execute("INSERT INTO %s (pk, a) VALUES (2, -1)");
+        execute("INSERT INTO %s (pk, a) VALUES (3, 0)");
+        execute("INSERT INTO %s (pk, a) VALUES (4, 1)");
+        execute("INSERT INTO %s (pk, a) VALUES (5, 10)");
+        execute("INSERT INTO %s (pk, a) VALUES (6, 15)");
+
+        var sai = getIndex();
+
+        assertInMemoryEstimateCount(sai, Operator.EQ, "0", 1);
+        assertInMemoryEstimateCount(sai, Operator.LT, "0", 2, 1, 0);
+        assertInMemoryEstimateCount(sai, Operator.GT, "0", 3, 1, 0);
+        assertInMemoryEstimateCount(sai, Operator.LTE, "0", 3);
+        assertInMemoryEstimateCount(sai, Operator.GTE, "0", 4);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "5", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "0", 1);
+        assertSSTableEstimateCount(sai, Operator.LT, "0", 2, 1, 0);
+        assertSSTableEstimateCount(sai, Operator.GT, "0", 3, 1, 0);
+        assertSSTableEstimateCount(sai, Operator.LTE, "0", 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, "0", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "5", 0);
+
+        compact();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, "0", 1);
+        assertSSTableEstimateCount(sai, Operator.LT, "0", 2, 1, 0);
+        assertSSTableEstimateCount(sai, Operator.GT, "0", 3, 1, 0);
+        assertSSTableEstimateCount(sai, Operator.LTE, "0", 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, "0", 4);
+        assertSSTableEstimateCount(sai, Operator.EQ, "5", 0);
+    }
+
+    @Test
+    public void testLossyNumericTypes() throws Throwable
+    {
+        // Only test types that support rounding.
+        if (!TypeUtil.supportsRounding(testType.getType()))
+            return;
+
+        createTable("CREATE TABLE %s (pk text PRIMARY KEY, x " + testType + ')');
+        createIndex("CREATE CUSTOM INDEX ON %s(x) USING 'StorageAttachedIndex'");
+
+        // We truncate decimals to 24 bytes in the numeric index. These numbers are chosen to ensure
+        // that we get the expected results.
+        var a = "1111111111111111111111111111111111111111111111111111111111";
+        var b = "1111111111111111111111111111111111111111111111111111111112";
+        var c = "1111111111111111111111111111111111111111111111111111111113";
+
+        execute("INSERT INTO %s (pk, x) VALUES ('a', " + a + ')');
+        execute("INSERT INTO %s (pk, x) VALUES ('b', " + b + ')');
+        execute("INSERT INTO %s (pk, x) VALUES ('c', " + c + ')');
+
+        var sai = getIndex();
+
+        // Because a, b, and c all truncate to the same value, we expect to get all 3 rows from the index.
+        assertInMemoryEstimateCount(sai, Operator.EQ, b, 3);
+        assertInMemoryEstimateCount(sai, Operator.LT, b, 3);
+        assertInMemoryEstimateCount(sai, Operator.LTE, b, 3);
+        assertInMemoryEstimateCount(sai, Operator.GT, b, 3);
+        assertInMemoryEstimateCount(sai, Operator.GTE, b, 3);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, b, 3);
+        assertSSTableEstimateCount(sai, Operator.LT, b, 3);
+        assertSSTableEstimateCount(sai, Operator.LTE, b, 3);
+        assertSSTableEstimateCount(sai, Operator.GT, b, 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, b, 3);
+
+        compact();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, b, 3);
+        assertSSTableEstimateCount(sai, Operator.LT, b, 3);
+        assertSSTableEstimateCount(sai, Operator.LTE, b, 3);
+        assertSSTableEstimateCount(sai, Operator.GT, b, 3);
+        assertSSTableEstimateCount(sai, Operator.GTE, b, 3);
+    }
+
+    @Test
+    public void testNumericIndexEstimatesOnManyRows() throws Throwable
+    {
+        // This tests inserts thousands of rows and covers different paths of code than a small-scale tests
+        // inserting a few rows only.
+        // For example, for estimating the number of rows in the memory tries we walk only a small portion
+        // of the trie, and then we estimate the total number by extrapolation. That extrapolation wouldn't
+        // be invoked if the trie contained insufficient number of rows. Aditionally, it is good to test
+        // with row counts significantly larger than the number of histogram buckets in the terms distribution,
+        // so it also checks the computations of bucket fractions.
+
+        createTable("CREATE TABLE %s (pk int primary key, a " + testType + ')');
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(a) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable(indexName);
+
+        final int COUNT = (testType == CQL3Type.Native.TINYINT) ? 127 : 10000;
+        for (int i = 0; i < COUNT; i++)
+            execute("INSERT INTO %s (pk, a) VALUES (" + i + ',' + i + ')');
+
+        var sai = getIndex();
+
+        final String MID_POINT = "" + COUNT / 2;
+        assertInMemoryEstimateCount(sai, Operator.EQ, MID_POINT, 1);
+        // Because we extrapolate, the error is higher on non-tinyint types. Note that a previous implementation
+        // of this test relied on the fact overwriting a primary key in the memtables didn't remove the old value
+        // from the index. I've updated the test to account for more realistic uncertainty and also added a compact
+        // then estimate step since that exercises a different build path in the index.
+        var uncertainty = (testType == CQL3Type.Native.TINYINT) ? 1 : 52;
+        assertInMemoryEstimateCount(sai, Operator.LT, MID_POINT, COUNT / 2, 0, uncertainty);
+        assertInMemoryEstimateCount(sai, Operator.GT, MID_POINT, COUNT / 2, 0, uncertainty);
+        assertInMemoryEstimateCount(sai, Operator.LTE, MID_POINT, COUNT / 2, 1, uncertainty);
+        assertInMemoryEstimateCount(sai, Operator.GTE, MID_POINT, COUNT / 2, 1, uncertainty);
+        assertInMemoryEstimateCount(sai, Operator.EQ, "-1", 0);
+
+        flush();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, MID_POINT, 1);
+        assertSSTableEstimateCount(sai, Operator.LT, MID_POINT, COUNT / 2, 0, uncertainty);
+        assertSSTableEstimateCount(sai, Operator.GT, MID_POINT, COUNT / 2, 0, uncertainty);
+        assertSSTableEstimateCount(sai, Operator.LTE, MID_POINT, COUNT / 2, 1, uncertainty);
+        assertSSTableEstimateCount(sai, Operator.GTE, MID_POINT, COUNT / 2, 1, uncertainty);
+        assertSSTableEstimateCount(sai, Operator.EQ, "-1", 0);
+
+        compact();
+
+        assertSSTableEstimateCount(sai, Operator.EQ, MID_POINT, 1);
+        assertSSTableEstimateCount(sai, Operator.LT, MID_POINT, COUNT / 2, 0, uncertainty);
+        assertSSTableEstimateCount(sai, Operator.GT, MID_POINT, COUNT / 2, 0, uncertainty);
+        assertSSTableEstimateCount(sai, Operator.LTE, MID_POINT, COUNT / 2, 1, uncertainty);
+        assertSSTableEstimateCount(sai, Operator.GTE, MID_POINT, COUNT / 2, 1, uncertainty);
+        assertSSTableEstimateCount(sai, Operator.EQ, "-1", 0);
+    }
+
+
+    private void assertInMemoryEstimateCount(StorageAttachedIndex index, Operator op, String value, long expectedCount)
+    {
+        assertInMemoryEstimateCount(index, op, value, expectedCount, 0, 0);
+    }
+
+    private void assertInMemoryEstimateCount(StorageAttachedIndex index, Operator op, String value, long expectedCount, long roundingValue, long uncertainty)
+    {
+        var memtableIndexes = index.getIndexContext()
+                                   .getLiveMemtables()
+                                   .values();
+        var expression = buildExpression(index, op, value);
+        var memoryCount = 0L;
+        var wholeRange = DataRange.allData(index.getIndexContext().getPartitioner()).keyRange();
+        for (var memtableIndex : memtableIndexes)
+            for (var memoryIndex : ((TrieMemtableIndex) memtableIndex).getRangeIndexes())
+                memoryCount += memoryIndex.estimateMatchingRowsCount(expression, wholeRange);
+
+        assertEstimateCorrect(expectedCount, roundingValue, uncertainty, memoryCount);
+    }
+
+    private void assertSSTableEstimateCount(StorageAttachedIndex index, Operator op, String value, long expectedCount)
+    {
+        assertSSTableEstimateCount(index, op, value, expectedCount, 0, 0);
+    }
+
+    private void assertSSTableEstimateCount(StorageAttachedIndex index, Operator op, String value, long expectedCount, long roundingValue, long uncertainty)
+    {
+        var expression = buildExpression(index, op, value);
+        var wholeRange = DataRange.allData(index.getIndexContext().getPartitioner()).keyRange();
+        var view = index.getIndexContext().getView();
+        var onDiskCount = 0L;
+        for (var sstableIndex : view.getIndexes())
+            onDiskCount += sstableIndex.estimateMatchingRowsCount(expression, wholeRange);
+
+        assertEstimateCorrect(expectedCount, roundingValue, uncertainty, onDiskCount);
+    }
+
+    private void assertEstimateCorrect(long expectedCount, long roundingValue, long uncertainty, long actualCount)
+    {
+        // This is a special case because Decimal and varint types are lossy, so when the index builds the expression
+        // it is automatically inclusive to prevent missed results.
+        if (TypeUtil.supportsRounding(testType.getType()))
+            expectedCount += roundingValue;
+
+        if (uncertainty == 0)
+        {
+            assertEquals(expectedCount, actualCount);
+        }
+        else
+        {
+            assertTrue("Expected min: " + (expectedCount - uncertainty) + " Actual: " + actualCount,
+                       actualCount >= expectedCount - uncertainty);
+            assertTrue("Expected max: " + (expectedCount + uncertainty) + " Actual: " + actualCount,
+                       actualCount <= expectedCount + uncertainty);
+        }
+    }
+
+    private Expression buildExpression(StorageAttachedIndex index, Operator op, String value)
+    {
+        var expression = new Expression(index.getIndexContext());
+        expression.add(op, index.getIndexContext().getValidator().fromString(value));
+        return expression;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/PartitionRestrictedQueryTest.java b/test/unit/org/apache/cassandra/index/sai/cql/PartitionRestrictedQueryTest.java
new file mode 100644
index 000000000000..07efe20106c0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/PartitionRestrictedQueryTest.java
@@ -0,0 +1,211 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.utils.Pair;
+
+import static org.junit.Assert.assertEquals;
+
+public class PartitionRestrictedQueryTest extends SAITester
+{
+    private static final int LARGE_ROWS = 4096;
+    private static final int LARGE_PARTITIONS = 64;
+    private static final int LARGE_ROWS_PER_PARTITION = LARGE_ROWS / LARGE_PARTITIONS;
+    private static final int LARGE_PARTITIONS_QUERIED = 16;
+
+    private static final int SMALL_ROWS = 512;
+    private static final int SMALL_PARTITIONS = 16;
+    private static final int SMALL_ROWS_PER_PARTITION = SMALL_ROWS / SMALL_PARTITIONS;
+    private static final int SMALL_PARTITIONS_QUERIED = 8;
+
+    private static final String QUERY_TEMPLATE = "SELECT * FROM %s WHERE pk = %d AND value >= %d AND value < %d LIMIT %d";
+    private static final String FILTERING_TEMPLATE = "SELECT * FROM %s WHERE pk = %d AND value >= %d AND value < %d LIMIT %d ALLOW FILTERING";
+
+    private static final int DEFAULT_LIMIT = 10;
+
+    private static final Random RANDOM = new Random(System.currentTimeMillis());
+
+    private String largeTable;
+    private String largeReferenceTable;
+    private String smallTable;
+    private String smallReferenceTable;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        largeTable = createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex' WITH OPTIONS = {'bkd_postings_skip' : 1, 'bkd_postings_min_leaves' : 2}");
+        largeReferenceTable = createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
+
+        smallTable = createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex' WITH OPTIONS = {'bkd_postings_skip' : 1, 'bkd_postings_min_leaves' : 2}");
+        smallReferenceTable = createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
+
+        String template = "INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)";
+
+        // Stripe the values from 0 -> (LARGE_ROWS - 1) across LARGE_PARTITIONS partitions. This makes it
+        // possible to easily query slices that span all partitions.
+        for (int pk = 0; pk < LARGE_PARTITIONS; pk++)
+        {
+            for (int ck = 0; ck < LARGE_ROWS_PER_PARTITION; ck++)
+            {
+                int value = pk + (ck * LARGE_PARTITIONS);
+
+                // Write to both the indexed and reference tables:
+                execute(String.format(template, KEYSPACE + "." + largeTable), pk, ck, value);
+                execute(String.format(template, KEYSPACE + "." + largeReferenceTable), pk, ck, value);
+            }
+        }
+
+        // Stripe the values from 0 -> (SMALL_ROWS - 1) across SMALL_PARTITIONS partitions. This makes it
+        // possible to easily query slices that span all partitions.
+        for (int pk = 0; pk < SMALL_PARTITIONS; pk++)
+        {
+            for (int ck = 0; ck < SMALL_ROWS_PER_PARTITION; ck++)
+            {
+                int value = pk + (ck * SMALL_PARTITIONS);
+
+                // Write to both the indexed and reference tables:
+                execute(String.format(template, KEYSPACE + "." + smallTable), pk, ck, value);
+                execute(String.format(template, KEYSPACE + "." + smallReferenceTable), pk, ck, value);
+            }
+        }
+    }
+
+    @Test
+    public void shouldQueryLargeNumericRangeInSinglePartition() throws Throwable
+    {
+        for (Pair<Integer, Integer> scenario : buildScenarios(LARGE_ROWS))
+        {
+            for (int i = 0; i < LARGE_PARTITIONS_QUERIED; i++)
+            {
+                verifyPartition(largeTable, largeReferenceTable, scenario.right, scenario.left, LARGE_PARTITIONS, LARGE_ROWS_PER_PARTITION);
+            }
+
+            flush(KEYSPACE, largeTable);
+
+            for (int i = 0; i < LARGE_PARTITIONS_QUERIED; i++)
+            {
+                verifyPartition(largeTable, largeReferenceTable, scenario.right, scenario.left, LARGE_PARTITIONS, LARGE_ROWS_PER_PARTITION);
+            }
+        }
+    }
+
+    @Test
+    public void shouldQuerySmallNumericRangeInSinglePartition() throws Throwable
+    {
+        for (Pair<Integer, Integer> scenario : buildScenarios(SMALL_ROWS))
+        {
+            for (int i = 0; i < SMALL_PARTITIONS_QUERIED; i++)
+            {
+                verifyPartition(smallTable, smallReferenceTable, scenario.right, scenario.left, SMALL_PARTITIONS, SMALL_ROWS_PER_PARTITION);
+            }
+
+            flush(KEYSPACE, smallTable);
+
+            for (int i = 0; i < SMALL_PARTITIONS_QUERIED; i++)
+            {
+                verifyPartition(smallTable, smallReferenceTable, scenario.right, scenario.left, SMALL_PARTITIONS, SMALL_ROWS_PER_PARTITION);
+            }
+        }
+    }
+
+    @Test
+    public void testCount() throws Throwable
+    {
+        ResultSet indexedRows = executeNet(String.format("SELECT count(*) FROM %s WHERE pk = %d AND value >= %d", KEYSPACE + "." + smallTable, 0, SMALL_ROWS / 2));
+        ResultSet filteredRows = executeNet(String.format("SELECT count(*) FROM %s WHERE pk = %d AND value >= %d ALLOW FILTERING", KEYSPACE + "." + smallReferenceTable, 0, SMALL_ROWS / 2));
+        assertEquals(filteredRows.one().getLong(0), indexedRows.one().getLong(0));
+    }
+
+    @Test
+    public void testSum() throws Throwable
+    {
+        ResultSet indexedRows = executeNet(String.format("SELECT sum(value) FROM %s WHERE pk = %d AND value >= %d", KEYSPACE + "." + smallTable, 0, SMALL_ROWS / 2));
+        ResultSet filteredRows = executeNet(String.format("SELECT sum(value) FROM %s WHERE pk = %d AND value >= %d ALLOW FILTERING", KEYSPACE + "." + smallReferenceTable, 0, SMALL_ROWS / 2));
+        assertEquals(filteredRows.one().getInt(0), indexedRows.one().getInt(0));
+    }
+
+    @Test
+    public void testAverage() throws Throwable
+    {
+        ResultSet indexedRows = executeNet(String.format("SELECT avg(value) FROM %s WHERE pk = %d AND value >= %d", KEYSPACE + "." + smallTable, 0, SMALL_ROWS / 2));
+        ResultSet filteredRows = executeNet(String.format("SELECT avg(value) FROM %s WHERE pk = %d AND value >= %d ALLOW FILTERING", KEYSPACE + "." + smallReferenceTable, 0, SMALL_ROWS / 2));
+        assertEquals(filteredRows.one().getInt(0), indexedRows.one().getInt(0));
+    }
+
+    private void verifyPartition(String table, String referenceTable, int max, int min, int numPartitions, int rowsPerPartition) throws Throwable
+    {
+        int pk = RANDOM.nextInt(numPartitions);
+
+        // Compare the index result w/ the result of an equivalent ALLOW FILTERING query:
+        ResultSet indexedRows = executeNet(String.format(QUERY_TEMPLATE, KEYSPACE + "." + table, pk, min, max, DEFAULT_LIMIT));
+        ResultSet filteredRows = executeNet(String.format(FILTERING_TEMPLATE, KEYSPACE + "." + referenceTable, pk, min, max, DEFAULT_LIMIT));
+        verifyRowValues(filteredRows.all(), indexedRows.all());
+
+        // Then do the same thing with a DEFAULT_LIMIT high enough to exhaust the partition:
+        indexedRows = executeNet(String.format(QUERY_TEMPLATE, KEYSPACE + "." + table, pk, min, max, rowsPerPartition + 1));
+        filteredRows = executeNet(String.format(FILTERING_TEMPLATE, KEYSPACE + "." + referenceTable, pk, min, max, rowsPerPartition + 1));
+        verifyRowValues(filteredRows.all(), indexedRows.all());
+    }
+
+    private void verifyRowValues(List<Row> expected, List<Row> actual)
+    {
+        assertEquals(expected.size(), actual.size());
+
+        for (int i = 0; i < expected.size(); i++)
+        {
+            assertEquals(expected.get(i).getInt("pk"), actual.get(i).getInt("pk"));
+            assertEquals(expected.get(i).getInt("ck"), actual.get(i).getInt("ck"));
+            assertEquals(expected.get(i).getInt("value"), actual.get(i).getInt("value"));
+        }
+    }
+
+    private List<Pair<Integer, Integer>> buildScenarios(int numRows)
+    {
+
+        List<Pair<Integer, Integer>> scenarios = new LinkedList<>();
+
+        scenarios.add(Pair.create(numRows / 16, numRows));
+        scenarios.add(Pair.create(numRows / 8, numRows));
+        scenarios.add(Pair.create(numRows / 4, numRows));
+        scenarios.add(Pair.create(numRows / 2, numRows));
+
+        scenarios.add(Pair.create(0, numRows / 16));
+        scenarios.add(Pair.create(0, numRows / 8));
+        scenarios.add(Pair.create(0, numRows / 4));
+        scenarios.add(Pair.create(0, numRows / 2));
+
+        scenarios.add(Pair.create(0, numRows));
+
+        return scenarios;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/QueryTimeoutTest.java b/test/unit/org/apache/cassandra/index/sai/cql/QueryTimeoutTest.java
new file mode 100644
index 000000000000..04b0de4625f4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/QueryTimeoutTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import javax.management.ObjectName;
+
+import org.apache.cassandra.index.sai.disk.PostingListKeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
+import org.apache.cassandra.inject.ActionBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.datastax.driver.core.exceptions.ReadTimeoutException;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.inject.Injection;
+import org.apache.cassandra.inject.Injections;
+
+import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class QueryTimeoutTest extends SAITester
+{
+    private static final int TIMEOUT = 5000;
+    private static final int DELAY = TIMEOUT + (TIMEOUT / 2);
+
+    private ObjectName queryCountName, queryTimeoutsName;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        requireNetwork();
+
+        startJMXServer();
+
+        createMBeanServerConnection();
+
+        DatabaseDescriptor.setRangeRpcTimeout(TIMEOUT);
+        DatabaseDescriptor.setReadRpcTimeout(TIMEOUT);
+
+        createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+
+        if (execute("SELECT * FROM %s").size() > 0)
+        {
+            return;
+        }
+
+        for (int i = 0; i < 100; ++i)
+        {
+            execute("INSERT INTO %s(id1,v1,v2) VALUES (?, ?, ?)", i, i, Integer.toString(i % 5));
+        }
+        flush();
+
+        execute("SELECT * FROM %s WHERE v1 >= 0 AND v1 < 10000");
+        execute("SELECT * FROM %s WHERE v2 = '0'");
+
+        queryCountName = objectNameNoIndex("TotalQueriesCompleted", CQLTester.KEYSPACE, currentTable(), TableQueryMetrics.TABLE_QUERY_METRIC_TYPE);
+        queryTimeoutsName = objectNameNoIndex("TotalQueryTimeouts", CQLTester.KEYSPACE, currentTable(), TableQueryMetrics.TABLE_QUERY_METRIC_TYPE);
+    }
+
+    @After
+    public void removeInjections() throws Exception
+    {
+        Injections.deleteAll();
+    }
+
+    @Test
+    public void delayDuringKDTreeIntersectionShouldProvokeTimeoutInReader() throws Throwable
+    {
+        Injection kdtree_intersection_delay = Injections.newPause("kdtree_intersection_delay", DELAY)
+                                                        .add(newInvokePoint().onClass("org.apache.cassandra.index.sai.disk.v1.kdtree.BKDReader$Intersection")
+                                                                                   .onMethod("collectPostingLists")
+                                                                                   .at("INVOKE QueryContext.checkpoint"))
+
+                                                        .build();
+
+        Injections.inject(kdtree_intersection_delay);
+
+        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE v1 >= 0 AND v1 < 10000")).isInstanceOf(ReadTimeoutException.class);
+
+        waitForEquals(queryCountName, queryTimeoutsName);
+    }
+
+    @Test
+    public void delayDuringTermsReaderMatchShouldProvokeTimeoutInReader() throws Throwable
+    {
+        Injection terms_match_delay = Injections.newPause("terms_match_delay", DELAY)
+                                                .add(newInvokePoint().onClass("org.apache.cassandra.index.sai.disk.v1.TermsReader$TermQuery")
+                                                                           .onMethod("execute")
+                                                                           .at("INVOKE QueryContext.checkpoint"))
+                                                .build();
+
+        Injections.inject(terms_match_delay);
+
+        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE v2 = '1'")).isInstanceOf(ReadTimeoutException.class);
+
+        waitForEquals(queryCountName, queryTimeoutsName);
+    }
+
+    @Test
+    public void delayDuringTokenLookupShouldProvokeTimeoutInRangeIterator() throws Throwable
+    {
+        Injection token_lookup_delay = Injections.newPause("token_lookup_delay", DELAY)
+                                                 .add(newInvokePoint().onClass(PostingListKeyRangeIterator.class)
+                                                                            .onMethod("computeNext")
+                                                                            .at("INVOKE QueryContext.checkpoint"))
+                                                 .build();
+
+        Injections.inject(token_lookup_delay);
+
+        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE v2 = '1'")).isInstanceOf(ReadTimeoutException.class);
+
+        waitForEquals(queryCountName, queryTimeoutsName);
+    }
+
+    @Test
+    public void abortedOperationExceptionShouldProvokeQueryFailureWithTimeoutCode() throws Throwable
+    {
+        Injection abortedOperationException = Injections.newCustom("throw_aborted_operation_exception")
+                .add(newInvokePoint()
+                        .onClass(StorageAttachedIndexSearcher.class)
+                        .onMethod("search")
+                        .atEntry())
+                .add(ActionBuilder.newActionBuilder()
+                        .actions()
+                        .doAction("throw new org.apache.cassandra.index.sai.utils.AbortedOperationException();"))
+                .build();
+        Injections.inject(abortedOperationException);
+        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE v2 = '1'"))
+                .matches(e -> e instanceof ReadTimeoutException
+        );
+
+        waitForEquals(queryCountName, queryTimeoutsName);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/RandomIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/RandomIntersectionTest.java
new file mode 100644
index 000000000000..3e086a4ab517
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/RandomIntersectionTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+@RunWith(Parameterized.class)
+public class RandomIntersectionTest extends SAITester
+{
+    private static final Object[][] EMPTY_ROWS = new Object[][]{};
+
+    @Parameterized.Parameter
+    public String testName;
+
+    @Parameterized.Parameter(1)
+    public boolean partitionRestricted;
+
+    @Parameterized.Parameter(2)
+    public boolean largePartition;
+
+    @Parameterized.Parameter(3)
+    public boolean v1Cardinality;
+
+    @Parameterized.Parameter(4)
+    public boolean v2Cardinality;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> parameters()
+    {
+        List<Object[]> parameters = new LinkedList<>();
+
+        parameters.add(new Object[]{ "Large partition restricted high high", true, true, true, true });
+        parameters.add(new Object[]{ "Large partition restricted low low", true, true, false, false });
+        parameters.add(new Object[]{ "Large partition restricted high low", true, true, true, false });
+        parameters.add(new Object[]{ "Large partition unrestricted high high", false, true, true, true });
+        parameters.add(new Object[]{ "Large partition unrestricted low low", false, true, false, false });
+        parameters.add(new Object[]{ "Large partition unrestricted high low", false, true, true, false });
+        parameters.add(new Object[]{ "Small partition restricted high high", true, false, true, true });
+        parameters.add(new Object[]{ "Small partition restricted low low", true, false, false, false });
+        parameters.add(new Object[]{ "Small partition restricted high low", true, false, true, false });
+        parameters.add(new Object[]{ "Small partition unrestricted high high", false, false, true, true });
+        parameters.add(new Object[]{ "Small partition unrestricted low low", false, false, false, false });
+        parameters.add(new Object[]{ "Small partition unrestricted high low", false, false, true, false });
+
+        return parameters;
+    }
+
+    private int numRows;
+
+    @Before
+    public void createTableAndIndexes()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, v1 int, v2 int, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(v2) USING 'StorageAttachedIndex'");
+
+        numRows = getRandom().nextIntBetween(50000, 200000);
+    }
+
+    @Test
+    public void randomIntersectionTest() throws Throwable
+    {
+        if (partitionRestricted)
+            runRestrictedQueries();
+        else
+            runUnrestrictedQueries();
+    }
+
+    private void runRestrictedQueries() throws Throwable
+    {
+        Map<Integer, List<TestRow>> testRowMap = buildAndLoadTestRows();
+
+        beforeAndAfterFlush(() -> {
+            for (int queryCount = 0; queryCount < getRandom().nextIntBetween(10, 100); queryCount++)
+            {
+                int pk = testRowMap.keySet().stream().skip(getRandom().nextIntBetween(0, testRowMap.size() - 1)).findFirst().orElseThrow();
+                int v1 = nextV1();
+                int v2 = nextV2();
+
+                List<Object[]> expected = testRowMap.get(pk)
+                                                    .stream()
+                                                    .sorted(Comparator.comparingInt(o -> o.ck))
+                                                    .filter(row -> row.v1 > v1 && row.v2 > v2)
+                                                    .map(row -> row(row.ck))
+                                                    .collect(Collectors.toList());
+
+                assertRows(execute("SELECT ck FROM %s WHERE pk = ? AND v1 > ? AND v2 > ?", pk, v1, v2), expected.toArray(EMPTY_ROWS));
+            }
+        });
+    }
+
+    private void runUnrestrictedQueries() throws Throwable
+    {
+        Map<Integer, List<TestRow>> testRowMap = buildAndLoadTestRows();
+
+        beforeAndAfterFlush(() -> {
+            for (int queryCount = 0; queryCount < getRandom().nextIntBetween(10, 100); queryCount++)
+            {
+                int v1 = nextV1();
+                int v2 = nextV2();
+
+                List<Object[]> expected = testRowMap.values()
+                                                    .stream()
+                                                    .flatMap(Collection::stream)
+                                                    .filter(row -> row.v1 == v1 && row.v2 == v2)
+                                                    .map(row -> row(row.ck))
+                                                    .collect(Collectors.toList());
+
+                assertRowsIgnoringOrder(execute("SELECT ck FROM %s WHERE v1 = ? AND v2 = ?", v1, v2), expected.toArray(EMPTY_ROWS));
+            }
+        });
+    }
+
+    private Map<Integer, List<TestRow>> buildAndLoadTestRows()
+    {
+        Map<Integer, List<TestRow>> testRowMap = new HashMap<>();
+
+        int clusterSize = largePartition ? getRandom().nextIntBetween(500, 5000) : getRandom().nextIntBetween(10, 100);
+        int partition = getRandom().nextIntBetween(0, numRows);
+        List<TestRow> rowList = new ArrayList<>(clusterSize);
+        testRowMap.put(partition, rowList);
+        int clusterCount = 0;
+        for (int index = 0; index < numRows; index++)
+        {
+            TestRow row = new TestRow(partition, getRandom().nextIntBetween(10, numRows), nextV1(), nextV2());
+            while (rowList.contains(row))
+                row = new TestRow(partition, getRandom().nextIntBetween(10, numRows), nextV1(), nextV2());
+
+            rowList.add(row);
+            clusterCount++;
+            if (clusterCount == clusterSize)
+            {
+                clusterCount = 0;
+                clusterSize = largePartition ? getRandom().nextIntBetween(500, 5000) : getRandom().nextIntBetween(10, 100);
+                partition = getRandom().nextIntBetween(0, numRows);
+                while (testRowMap.containsKey(partition))
+                    partition = getRandom().nextIntBetween(0, numRows);
+                rowList = new ArrayList<>(clusterSize);
+                testRowMap.put(partition, rowList);
+            }
+        }
+        testRowMap.values().stream().flatMap(Collection::stream).forEach(row -> execute("INSERT INTO %s (pk, ck, v1, v2) VALUES (?, ?, ?, ?)",
+                                                                                        row.pk, row.ck, row.v1, row.v2));
+        return testRowMap;
+    }
+
+    private int nextV1()
+    {
+        return v1Cardinality ? getRandom().nextIntBetween(10, numRows/10) : getRandom().nextIntBetween(10, numRows/1000);
+    }
+
+    private int nextV2()
+    {
+        return v2Cardinality ? getRandom().nextIntBetween(10, numRows/10) : getRandom().nextIntBetween(10, numRows/1000);
+    }
+
+    private static class TestRow implements Comparable<TestRow>
+    {
+        final int pk;
+        final int ck;
+        final int v1;
+        final int v2;
+
+        TestRow(int pk, int ck, int v1, int v2)
+        {
+            this.pk = pk;
+            this.ck = ck;
+            this.v1 = v1;
+            this.v2 = v2;
+        }
+
+        @Override
+        public int compareTo(TestRow other)
+        {
+            int cmp = Integer.compare(pk, other.pk);
+            if (cmp != 0)
+                return cmp;
+            return Integer.compare(ck, other.ck);
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (obj instanceof TestRow)
+                return compareTo((TestRow) obj) == 0;
+
+            return false;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(pk, ck);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/RandomisedComplexQueryTest.java b/test/unit/org/apache/cassandra/index/sai/cql/RandomisedComplexQueryTest.java
new file mode 100644
index 000000000000..37c3abc17eea
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/RandomisedComplexQueryTest.java
@@ -0,0 +1,448 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Streams;
+import com.google.common.collect.TreeMultimap;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.Relation;
+import org.apache.cassandra.cql3.SingleColumnRelation;
+import org.apache.cassandra.cql3.WhereClause;
+import org.apache.cassandra.index.sai.SAITester;
+import org.assertj.core.util.Lists;
+
+/**
+ * This test produces a random schema, loads it with random data and then runs a series of
+ * random queries against it.
+ * </p>
+ * The purpose of the test is to test that the <code>RowFilter</code> and <code>Operation</code>
+ * classes correctly support complex queries.
+ * </p>
+ * At present the test only supports ascii and int datatypes and only supports EQ expressions.
+ * It is intended that this can be extended in the future to support more functionality.
+ */
+public class RandomisedComplexQueryTest extends SAITester
+{
+    private static final List<TypeInfo> types = Lists.list(TypeInfo.create(CQL3Type.Native.ASCII, () -> CQLTester.getRandom().nextAsciiString(4, 30), true),
+                                                           TypeInfo.create(CQL3Type.Native.INT, () -> CQLTester.getRandom().nextIntBetween(0, 1000), false));
+
+    @Test
+    public void test() throws Throwable
+    {
+        for (int test = 0; test < getRandom().nextIntBetween(10, 50); test++)
+            runRandomTest();
+    }
+
+    private void runRandomTest() throws Throwable
+    {
+        RandomSchema schema = new RandomSchema();
+
+        createTable(schema.toTableDefinition());
+
+        schema.generateIndexStrings().forEach(this::createIndex);
+
+        List<RandomRow> data = schema.generateDataset();
+
+        String insert = schema.toInsert();
+
+        for (RandomRow row : data)
+            execute(insert, row.toArray());
+
+        for (int query = 0; query < getRandom().nextIntBetween(100, 1000); query++)
+            schema.generateQuery().test(this, data);
+    }
+
+    public static class RandomSchema
+    {
+        private final Map<String, RandomColumn> columnMap = new HashMap<>();
+        private final TreeMultimap<String, Object> values = TreeMultimap.create(Ordering.natural(), Ordering.arbitrary());
+
+        private final List<RandomColumn> partitionKeys;
+        private final List<RandomColumn> clusteringKeys;
+        private final List<RandomColumn> normalColumns;
+
+        RandomSchema()
+        {
+            int bindPosition = 0;
+            partitionKeys = generateColumns("pk", CQLTester.getRandom().nextIntBetween(1, 3), bindPosition);
+            bindPosition += partitionKeys.size();
+            clusteringKeys = generateColumns("ck", CQLTester.getRandom().nextIntBetween(0, 4), bindPosition);
+            bindPosition += clusteringKeys.size();
+            normalColumns = generateColumns("nc", CQLTester.getRandom().nextIntBetween(1, 10), bindPosition);
+        }
+
+        public String toTableDefinition()
+        {
+            StringBuilder builder = new StringBuilder();
+
+            builder.append("CREATE TABLE %s (");
+            builder.append(Streams.concat(partitionKeys.stream(), clusteringKeys.stream(), normalColumns.stream())
+                                  .map(RandomColumn::toColumnDefinition)
+                                  .collect(Collectors.joining(", ")));
+            builder.append(", PRIMARY KEY (");
+            String partitionKeyString = partitionKeys.stream().map(RandomColumn::name).collect(Collectors.joining(", ", "(", ")"));
+            if (clusteringKeys.isEmpty())
+                builder.append(partitionKeyString);
+            else
+                builder.append(Stream.of(partitionKeyString,
+                                         clusteringKeys.stream().map(RandomColumn::name).collect(Collectors.joining(", ")))
+                                     .collect(Collectors.joining(", ")));
+            builder.append("))");
+            return builder.toString();
+        }
+
+        public String toInsert()
+        {
+            StringBuilder builder = new StringBuilder();
+
+            builder.append("INSERT INTO %s (");
+            builder.append(Streams.concat(partitionKeys.stream(), clusteringKeys.stream(), normalColumns.stream())
+                                  .map(RandomColumn::name)
+                                  .collect(Collectors.joining(", ")));
+            builder.append(") VALUES (");
+            builder.append(Streams.concat(partitionKeys.stream(), clusteringKeys.stream(), normalColumns.stream())
+                                  .map(RandomColumn::bindMarker)
+                                  .collect(Collectors.joining(", ")));
+            builder.append(")");
+
+            return builder.toString();
+        }
+
+        public List<String> generateIndexStrings()
+        {
+            List<String> indexes = new ArrayList<>();
+            clusteringKeys.stream().map(RandomColumn::toIndexDefinition).forEach(indexes::add);
+            normalColumns.stream().map(RandomColumn::toIndexDefinition).forEach(indexes::add);
+            return indexes;
+        }
+
+        public List<RandomRow> generateDataset()
+        {
+            List<RandomRow> data = new ArrayList<>();
+
+            for (int row = 0; row < CQLTester.getRandom().nextIntBetween(100, 1000); row++)
+            {
+                RandomRow newRow = generateRow();
+                // Remove any duplicate rows - makes it easier to build result set
+                // It may be possible to handle duplicates in the filtering but for the
+                // time being we just get rid of them
+                List<RandomRow> duplicates = data.stream()
+                                                 .filter(r -> {
+                                                    for (int pk = 0; pk < partitionKeys.size(); pk++)
+                                                        if (!r.values.get(pk).equals(newRow.values.get(pk)))
+                                                            return true;
+                                                    return false;
+                                                 })
+                                                 .collect(Collectors.toList());
+                duplicates.stream().forEach(data::remove);
+                data.add(newRow);
+            }
+
+            return data;
+        }
+
+        public RandomQuery generateQuery() throws Throwable
+        {
+            StringBuilder builder = new StringBuilder();
+
+            boolean applyPrecedence = CQLTester.getRandom().nextBoolean();
+
+            List<RandomColumn> allColumns = Lists.newArrayList(clusteringKeys);
+            allColumns.addAll(normalColumns);
+            int numberOfElements = CQLTester.getRandom().nextIntBetween(1, allColumns.size());
+            Set<RandomColumn> columns = new HashSet<>();
+            while (columns.size() < numberOfElements)
+                columns.add(allColumns.get(CQLTester.getRandom().nextIntBetween(0, allColumns.size() - 1)));
+
+            RandomColumn[] columnArray = columns.toArray(new RandomColumn[] {});
+            int precedenceLevel = 0;
+            for (int element = 0; element < numberOfElements - 1; element++)
+            {
+                if (applyPrecedence && CQLTester.getRandom().nextIntBetween(0, 2) == 0)
+                {
+                    builder.append("(");
+                    precedenceLevel++;
+                }
+                builder.append(columnArray[element].name);
+                builder.append(" = ");
+                builder.append(columnArray[element].randomQueryValue());
+
+                if (applyPrecedence && CQLTester.getRandom().nextIntBetween(0, 2) == 2 && precedenceLevel > 0)
+                {
+                    builder.append(")");
+                    precedenceLevel--;
+                }
+                builder.append(CQLTester.getRandom().nextBoolean() ? " AND " : " OR ");
+            }
+            builder.append(columnArray[columnArray.length - 1].name);
+            builder.append(" = ");
+            builder.append(columnArray[columnArray.length - 1].randomQueryValue());
+            if (applyPrecedence)
+                while (precedenceLevel-- > 0)
+                    builder.append(")");
+
+            return new RandomQuery(this, builder.toString());
+        }
+
+        private RandomRow generateRow()
+        {
+            RandomRow row = new RandomRow();
+            Streams.concat(partitionKeys.stream(), clusteringKeys.stream(), normalColumns.stream())
+                   .map(RandomColumn::nextValue).forEach(row::add);
+            return row;
+        }
+
+        private List<RandomColumn> generateColumns(String prefix, int count, int bindPosition)
+        {
+            List<RandomColumn> columns = new ArrayList<>(count);
+            for (int index = 0; index < count; index++)
+            {
+                RandomColumn column = new RandomColumn(this, prefix + index, getRandomType(), bindPosition++);
+                columns.add(column);
+                columnMap.put(column.name, column);
+            }
+            return columns;
+        }
+
+        private static TypeInfo getRandomType()
+        {
+            return types.get(CQLTester.getRandom().nextIntBetween(0, types.size() - 1));
+        }
+    }
+
+    public static class RandomQuery
+    {
+        private final RandomSchema schema;
+        private final String query;
+        private final Filter filter;
+
+        RandomQuery(RandomSchema schema, String query) throws Throwable
+        {
+            this.schema = schema;
+            this.query = query;
+            filter = buildFilter(WhereClause.parse(query).root());
+        }
+
+        void test(SAITester tester, List<RandomRow> data) throws Throwable
+        {
+            CQLTester.assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE " + query), expectedRows(data));
+        }
+
+        Object[][] expectedRows(List<RandomRow> data)
+        {
+            List<Object[]> expected = new ArrayList<>();
+
+            for (RandomRow row : data)
+            {
+                if (filter.isSatisfiedBy(row))
+                    expected.add(row.toArray());
+            }
+
+            return expected.toArray(new Object[][]{});
+        }
+
+        Filter buildFilter(WhereClause.ExpressionElement element)
+        {
+            Filter filter = new Filter();
+            filter.isDisjunction = element.isDisjunction();
+            for (Relation relation : element.relations())
+            {
+                filter.expressions.add(new Expression(schema, relation));
+            }
+            for (WhereClause.ExpressionElement child : element.operations())
+                filter.children.add(buildFilter(child));
+            return filter;
+        }
+
+        static class Filter
+        {
+            boolean isDisjunction;
+
+            List<Expression> expressions = new ArrayList<>();
+
+            List<Filter> children = new ArrayList<>();
+
+            boolean isSatisfiedBy(RandomRow row)
+            {
+                if (isDisjunction)
+                {
+                    for (Expression e : expressions)
+                        if (e.isSatisfiedBy(row))
+                            return true;
+                    for (Filter child : children)
+                        if (child.isSatisfiedBy(row))
+                            return true;
+                    return false;
+                }
+                else
+                {
+                    for (Expression e : expressions)
+                        if (!e.isSatisfiedBy(row))
+                            return false;
+                    for (Filter child : children)
+                        if (!child.isSatisfiedBy(row))
+                            return false;
+                    return true;
+                }
+            }
+        }
+
+        static class Expression
+        {
+            ColumnIdentifier column;
+            String value;
+            int bindPosition;
+
+            Expression(RandomSchema schema, Relation relation)
+            {
+                assert relation instanceof SingleColumnRelation;
+                SingleColumnRelation singleColumnRelation = (SingleColumnRelation)relation;
+                column = singleColumnRelation.getEntity();
+                value = ((Constants.Literal)singleColumnRelation.getValue()).getRawText();
+                bindPosition = schema.columnMap.get(column.toString()).bindPosition;
+            }
+
+            boolean isSatisfiedBy(RandomRow row)
+            {
+                Object rowValue = row.values.get(bindPosition);
+                return rowValue.toString().equals(value);
+            }
+        }
+    }
+
+    public static class RandomColumn
+    {
+        private final RandomSchema schema;
+        private final String name;
+        private final TypeInfo type;
+        private final int bindPosition;
+
+        RandomColumn(RandomSchema schema, String name, TypeInfo type, int bindPosition)
+        {
+            this.schema = schema;
+            this.name = name;
+            this.type = type;
+            this.bindPosition = bindPosition;
+        }
+
+        public String name()
+        {
+            return name;
+        }
+
+        public String toColumnDefinition()
+        {
+            return name + " " + type.type.toString();
+        }
+
+        public String bindMarker()
+        {
+            return "?";
+        }
+
+        public String randomQueryValue()
+        {
+            Object[] columnValues = schema.values.get(name).toArray();
+            Object randomValue = columnValues[CQLTester.getRandom().nextIntBetween(0, columnValues.length - 1)];
+            return type.toCqlString(randomValue);
+        }
+
+        public String toIndexDefinition()
+        {
+            StringBuilder builder = new StringBuilder();
+            builder.append("CREATE CUSTOM INDEX ON %s(");
+            builder.append(name);
+            builder.append(") USING 'StorageAttachedIndex'");
+            return builder.toString();
+        }
+
+        public Object nextValue()
+        {
+            Object value = type.nextValue();
+            schema.values.put(name, value);
+            return value;
+        }
+
+        @Override
+        public String toString()
+        {
+            return name;
+        }
+    }
+
+    public static class RandomRow
+    {
+        List<Object> values = new ArrayList<>();
+
+        public void add(Object value)
+        {
+            values.add(value);
+        }
+
+        public Object[] toArray()
+        {
+            return values.toArray();
+        }
+    }
+
+    public static class TypeInfo
+    {
+        private final CQL3Type.Native type;
+        private final Supplier<?> supplier;
+        private final boolean quoted;
+
+        TypeInfo(CQL3Type.Native type, Supplier<?> supplier, boolean quoted)
+        {
+            this.type = type;
+            this.supplier = supplier;
+            this.quoted = quoted;
+        }
+
+        static TypeInfo create(CQL3Type.Native type, Supplier<?> supplier, boolean quoted)
+        {
+            return new TypeInfo(type, supplier, quoted);
+        }
+
+        public Object nextValue()
+        {
+            return supplier.get();
+        }
+
+        public String toCqlString(Object value)
+        {
+            return quoted ? "'" + value + "'" : value.toString();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/RebuildWithImmutableComponentsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/RebuildWithImmutableComponentsTest.java
new file mode 100644
index 000000000000..066f736c3f4b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/RebuildWithImmutableComponentsTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.TOCComponent;
+import org.apache.cassandra.io.util.PathUtils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class RebuildWithImmutableComponentsTest extends AbstractRebuildAndImmutableComponentsTester
+{
+    @Override
+    protected boolean useImmutableComponents()
+    {
+        return true;
+    }
+
+    @Override
+    protected void validateSSTables(ColumnFamilyStore cfs, IndexContext context) throws Exception
+    {
+        Index.Group indexGroup = StorageAttachedIndexGroup.getIndexGroup(cfs);
+        assert indexGroup != null;
+        for (SSTableReader sstable : cfs.getLiveSSTables())
+        {
+            // Make sure the TOC has all the active components; this is somewhat indirectly tested by the few next
+            // assertions, but good sanity check (and somewhat test the `activeComponents` method too).
+            assertTrue(TOCComponent.loadTOC(sstable.descriptor).containsAll(indexGroup.activeComponents(sstable)));
+
+            // verify TOC only includes latest SAI generation
+            Set<Component> saiComponents = TOCComponent.loadTOC(sstable.descriptor).stream().filter(c -> c.name.contains("SAI")).collect(Collectors.toSet());
+            assertEquals(saiComponents, indexGroup.activeComponents(sstable));
+
+            // verify SSTable#components only includes latest SAI generation
+            saiComponents = sstable.components().stream().filter(c -> c.name.contains("SAI")).collect(Collectors.toSet());
+            assertEquals(saiComponents, indexGroup.activeComponents(sstable));
+
+            IndexDescriptor descriptor = IndexDescriptor.load(sstable, Set.of(context));
+            assertEquals(1, descriptor.perSSTableComponents().buildId().generation());
+            assertEquals(1, descriptor.perIndexComponents(context).buildId().generation());
+
+            Set<String> files = allSSTableFilenames(sstable);
+            for (var components : List.of(descriptor.perSSTableComponents(), descriptor.perIndexComponents(context)))
+            {
+                for (var component : components.all())
+                {
+                    String gen0Component = components.version().fileNameFormatter().format(component.componentType(), components.context(), 0);
+                    String expectedFilename = sstable.descriptor.fileFor(new Component(SSTableFormat.Components.Types.CUSTOM, gen0Component)).name();
+                    assertTrue( "File " + expectedFilename + " not found in " + files, files.contains(expectedFilename));
+                }
+            }
+        }
+    }
+
+    private static Set<String> allSSTableFilenames(SSTableReader sstable)
+    {
+        Set<String> files = new HashSet<>();
+        PathUtils.forEach(sstable.descriptor.directory.toPath(), path -> {
+            String filename = path.getFileName().toString();
+            if (filename.startsWith(sstable.descriptor.filenamePart()))
+                files.add(filename);
+        });
+        return files;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/RebuildWithoutImmutableComponentsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/RebuildWithoutImmutableComponentsTest.java
new file mode 100644
index 000000000000..7309d8637315
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/RebuildWithoutImmutableComponentsTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Set;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.junit.Assert.assertEquals;
+
+public class RebuildWithoutImmutableComponentsTest extends AbstractRebuildAndImmutableComponentsTester
+{
+    @Override
+    protected boolean useImmutableComponents()
+    {
+        return false;
+    }
+
+    @Override
+    protected void validateSSTables(ColumnFamilyStore cfs, IndexContext context)
+    {
+        for (SSTableReader sstable : cfs.getLiveSSTables())
+        {
+            IndexDescriptor descriptor = IndexDescriptor.load(sstable, Set.of(context));
+            assertEquals(0, descriptor.perSSTableComponents().buildId().generation());
+            assertEquals(0, descriptor.perIndexComponents(context).buildId().generation());
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/StaticColumnIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/StaticColumnIndexTest.java
index faaf74f5c603..0bdee106fdc4 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/StaticColumnIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/StaticColumnIndexTest.java
@@ -49,26 +49,7 @@ public void staticIndexAndNonStaticIndex() throws Throwable
         execute("INSERT INTO %s(pk, ck,       val2) VALUES(?, ?,    ?)", 1, 2,     2000);
         execute("INSERT INTO %s(pk, ck, val1, val2) VALUES(?, ?, ?, ?)", 2, 1, 40, 2000);
 
-        beforeAndAfterFlush(() -> assertRows(execute("SELECT pk, ck, val1, val2 FROM %s WHERE val1 = 20 AND val2 = 2000"),
+        beforeAndAfterFlush(() -> assertRows(execute("SELECT pk, ck, val1, val2 FROM %s WHERE val1 = 20 AND val2 = 2000 ALLOW FILTERING"),
                                              row(1, 2, 20, 2000)));
     }
-
-    @Test
-    public void staticAndNonStaticRangeIntersection() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk int, ck int, v1 int, s1 int static, PRIMARY KEY(pk, ck))");
-        createIndex("CREATE INDEX ON %s(v1) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(s1) USING 'sai'");
-
-        execute("INSERT INTO %s (pk, ck, v1) VALUES (?, ?, ?)", 0, 1, 0);
-        execute("INSERT INTO %s (pk, ck, v1) VALUES (?, ?, ?)", 0, 2, 1);
-        execute("INSERT INTO %s (pk, ck, v1) VALUES (?, ?, ?)", 0, 3, 2);
-        execute("INSERT INTO %s (pk, ck, v1) VALUES (?, ?, ?)", 0, 4, 3);
-        execute("INSERT INTO %s (pk, ck, v1) VALUES (?, ?, ?)", 0, 5, 4);
-        execute("INSERT INTO %s (pk, ck, v1) VALUES (?, ?, ?)", 0, 6, 5);
-
-        execute("INSERT INTO %s (pk, s1) VALUES (?, ?)", 0, 100);
-
-        beforeAndAfterFlush(() -> assertRowCount(execute("SELECT * FROM %s WHERE pk = ? AND v1 > ? AND s1 = ?", 0, 2, 100), 3));
-    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java b/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
deleted file mode 100644
index cc2e2f4b13dc..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
+++ /dev/null
@@ -1,1402 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.cassandra.index.sai.cql;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.LongStream;
-
-import com.google.common.collect.ImmutableMap;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.exceptions.InvalidConfigurationInQueryException;
-import com.datastax.driver.core.exceptions.InvalidQueryException;
-import com.datastax.driver.core.exceptions.ReadFailureException;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.CqlBuilder;
-import org.apache.cassandra.cql3.restrictions.IndexRestrictions;
-import org.apache.cassandra.cql3.statements.schema.CreateIndexStatement;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.SecondaryIndexManager;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.StorageAttachedIndexBuilder;
-import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.Version;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.index.sai.view.View;
-import org.apache.cassandra.inject.ActionBuilder;
-import org.apache.cassandra.inject.Expression;
-import org.apache.cassandra.inject.Injection;
-import org.apache.cassandra.inject.Injections;
-import org.apache.cassandra.inject.InvokePointBuilder;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.utils.Throwables;
-import org.assertj.core.api.Assertions;
-import org.mockito.Mockito;
-
-import static java.util.Collections.singletonList;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.when;
-
-public class StorageAttachedIndexDDLTest extends SAITester
-{
-    private static final Injections.Counter saiCreationCounter = Injections.newCounter("IndexCreationCounter")
-                                                                           .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndex.class).onMethod("register"))
-                                                                           .build();
-
-    private static final Injection failSAIInitialializaion = Injections.newCustom("fail_sai_initialization")
-                                                                       .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndexBuilder.class).onMethod("build"))
-                                                                       .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
-                                                                       .build();
-
-    private static final Injection forceFlushPause = Injections.newPause("force_flush_pause", 30_000)
-                                                               .add(InvokePointBuilder.newInvokePoint().onClass(ColumnFamilyStore.class).onMethod("forceBlockingFlush"))
-                                                               .build();
-
-    private static final Injection failPerIndexMetaCompletion = Injections.newCustom("fail_index_meta_completion")
-                                                                          .add(InvokePointBuilder.newInvokePoint().onClass(SegmentBuilder.class).onMethod("flush"))
-                                                                          .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
-                                                                          .build();
-
-    private static final Injection failPerSSTableTokenAdd = Injections.newCustom("fail_token_writer")
-                                                                      .add(InvokePointBuilder.newInvokePoint().onClass(NumericValuesWriter.class).onMethod("add"))
-                                                                      .add(ActionBuilder.newActionBuilder().actions().doThrow(IOException.class, Expression.quote("Injected failure!")))
-                                                                      .build();
-
-    private static final Injection FAIL_INDEX_GC_TRANSACTION = Injections.newCustom("fail_index_gc_transaction")
-                                                                         .add(InvokePointBuilder.newInvokePoint().onClass("org.apache.cassandra.index.SecondaryIndexManager$IndexGCTransaction")
-                                                                                                .onMethod("<init>"))
-                                                                         .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
-                                                                         .build();
-
-    private static final EnumSet<IndexComponent> PER_SSTABLE_COMPONENTS = EnumSet.of(IndexComponent.ROW_TO_PARTITION,
-                                                                                     IndexComponent.PARTITION_TO_SIZE,
-                                                                                     IndexComponent.PARTITION_KEY_BLOCKS,
-                                                                                     IndexComponent.PARTITION_KEY_BLOCK_OFFSETS,
-                                                                                     IndexComponent.CLUSTERING_KEY_BLOCKS,
-                                                                                     IndexComponent.CLUSTERING_KEY_BLOCK_OFFSETS);
-    @Before
-    public void setup() throws Throwable
-    {
-        requireNetwork();
-
-        Injections.inject(saiCreationCounter, indexBuildCounter, FAIL_INDEX_GC_TRANSACTION);
-
-        saiCreationCounter.reset();
-        indexBuildCounter.reset();
-    }
-
-    @After
-    public void removeInjections()
-    {
-        Injections.deleteAll();
-    }
-
-    @Test
-    public void shouldFailUnsupportedType()
-    {
-        for (CQL3Type.Native cql3Type : CQL3Type.Native.values())
-        {
-            if (cql3Type == CQL3Type.Native.EMPTY)
-                continue;
-
-            String createTableTemplate = "CREATE TABLE %%s (id text PRIMARY KEY, %s %s)";
-            createTable(String.format(createTableTemplate, cql3Type, cql3Type));
-
-            boolean supported = StorageAttachedIndex.SUPPORTED_TYPES.contains(cql3Type);
-
-            try
-            {
-                executeNet(String.format("CREATE INDEX ON %%s(%s) USING 'sai'", cql3Type));
-                assertTrue("Index creation on unsupported type " + cql3Type + " should have failed.", supported);
-            }
-            catch (RuntimeException e)
-            {
-                assertFalse("Index creation on supported type " + cql3Type + " should have succeeded.", supported);
-                // InvalidConfigurationInQueryException is subclass of InvalidQueryException
-                assertTrue(Throwables.isCausedBy(e, InvalidQueryException.class::isInstance));
-            }
-        }
-    }
-
-    @Test
-    public void shouldFailCreationOnPartitionKey()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(id) USING 'sai'"))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessageContaining(String.format(CreateIndexStatement.ONLY_PARTITION_KEY, "id"));
-    }
-
-    @Test
-    public void shouldFailCreationUsingMode()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(val) USING 'sai' " +
-                                            "WITH OPTIONS = { 'mode' : 'CONTAINS' }")).isInstanceOf(InvalidConfigurationInQueryException.class);
-    }
-
-    @Test
-    public void shouldFailCreateSpecifyingAnalyzerClass()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(val) " +
-                                            "USING 'sai' " +
-                                            "WITH OPTIONS = { 'analyzer_class' : 'org.apache.cassandra.index.sai.analyzer.NonTokenizingAnalyzer' }"))
-        .isInstanceOf(InvalidConfigurationInQueryException.class);
-    }
-
-    @Test
-    public void shouldFailCreateWithMisspelledOption()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(val) " +
-                                            "USING 'sai' " +
-                                            "WITH OPTIONS = { 'case-sensitive' : true }")).isInstanceOf(InvalidConfigurationInQueryException.class);
-    }
-
-    @Test
-    public void shouldFailCaseSensitiveWithNonText()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
-
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(val) " +
-                                            "USING 'sai' " +
-                                            "WITH OPTIONS = { 'case_sensitive' : true }")).isInstanceOf(InvalidQueryException.class);
-    }
-
-    @Test
-    public void shouldFailOnNormalizeWithNonText()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val int)");
-
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(val) " +
-                                            "USING 'sai' " +
-                                            "WITH OPTIONS = { 'normalize' : true }")).isInstanceOf(InvalidQueryException.class);
-    }
-
-    @Test
-    public void shouldFailCreateWithUserType()
-    {
-        String typeName = createType("CREATE TYPE %s (a text, b int, c double)");
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val " + typeName + ')');
-
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(val) " +
-                                            "USING 'sai'")).isInstanceOf(InvalidQueryException.class);
-    }
-
-    @Test
-    public void shouldNotFailCreateWithTupleType()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val tuple<text, int, double>)");
-
-        executeNet("CREATE INDEX ON %s(val) USING 'sai'");
-
-        TableMetadata metadata = currentTableMetadata();
-        AbstractType<?> tuple = metadata.getColumn(ColumnIdentifier.getInterned("val", false)).type;
-        assertFalse(tuple.isMultiCell());
-        assertFalse(tuple.isCollection());
-        assertTrue(tuple.isTuple());
-    }
-
-    @Test
-    public void shouldFailCreateWithInvalidCharactersInColumnName()
-    {
-        String invalidColumn = "/invalid";
-        createTable(String.format("CREATE TABLE %%s (id text PRIMARY KEY, \"%s\" text)", invalidColumn));
-
-        assertThatThrownBy(() -> executeNet(String.format("CREATE INDEX ON %%s(\"%s\")" +
-                                                          " USING 'sai'", invalidColumn)))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessage(String.format(CreateIndexStatement.INVALID_CUSTOM_INDEX_TARGET, invalidColumn, SchemaConstants.NAME_LENGTH));
-    }
-
-    @Test
-    public void shouldCreateIndexIfExists()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-        createIndex("CREATE INDEX IF NOT EXISTS ON %s(val) USING 'sai' ");
-        createIndexAsync("CREATE INDEX IF NOT EXISTS ON %s(val) USING 'sai' ");
-
-        assertEquals(1, saiCreationCounter.get());
-    }
-
-    @Test
-    public void shouldCreateIndexCaseInsensitive()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val1 text, val2 text)");
-        createIndex("CREATE INDEX mixed_case_val ON %s(val1) USING 'Sai' ");
-        createIndex("CREATE INDEX upper_case_val ON %s(val2) USING 'SAI' ");
-
-        assertEquals(2, saiCreationCounter.get());
-    }
-
-    @Test
-    public void shouldCreateIndexWithClassName()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-        createIndex("CREATE INDEX ON %s(val) USING 'StorageAttachedIndex' ");
-        assertEquals(1, saiCreationCounter.get());
-    }
-
-    @Test
-    public void shouldCreateIndexWithDefault()
-    {
-        DatabaseDescriptor.setDefaultSecondaryIndex(StorageAttachedIndex.NAME);
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-        createIndex("CREATE INDEX ON %s(val)");
-        assertEquals(1, saiCreationCounter.get());
-    }
-
-    @Test
-    public void shouldFailWithDefaultIndexDisabled()
-    {
-        DatabaseDescriptor.setDefaultSecondaryIndex(StorageAttachedIndex.NAME);
-        boolean original = DatabaseDescriptor.getDefaultSecondaryIndexEnabled();
-
-        try
-        {
-            DatabaseDescriptor.setDefaultSecondaryIndexEnabled(false);
-            createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-            assertThatThrownBy(() -> createIndex("CREATE INDEX ON %s(val)")).hasRootCauseInstanceOf(InvalidRequestException.class)
-                                                                            .hasRootCauseMessage(CreateIndexStatement.MUST_SPECIFY_INDEX_IMPLEMENTATION);
-            assertEquals(0, saiCreationCounter.get());
-        }
-        finally
-        {
-            DatabaseDescriptor.setDefaultSecondaryIndexEnabled(original);
-        }
-    }
-
-    @Test
-    public void shouldBeCaseSensitiveByDefault()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        createIndex("CREATE INDEX ON %s(val) USING 'sai'");
-
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
-
-        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Camel'").size());
-
-        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'camel'").size());
-    }
-
-    @Test
-    public void shouldEnableCaseSensitiveSearch()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        createIndex("CREATE INDEX ON %s(val) USING 'sai' WITH OPTIONS = { 'case_sensitive' : true }");
-
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
-
-        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Camel'").size());
-
-        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'camel'").size());
-    }
-
-    @Test
-    public void shouldEnableCaseInsensitiveSearch()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        createIndex("CREATE INDEX ON %s(val) USING 'sai' WITH OPTIONS = { 'case_sensitive' : false }");
-
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
-
-        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'camel'").size());
-    }
-
-    @Test
-    public void shouldBeNonNormalizedByDefault()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        createIndex("CREATE INDEX ON %s(val) USING 'sai'");
-
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
-
-        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u00E1l'").size());
-
-        // Both \u00E1 and \u0061\u0301 are visible as the character á, but without NFC normalization, they won't match.
-        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
-    }
-
-    @Test
-    public void shouldEnableNonNormalizedSearch()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        createIndex("CREATE INDEX ON %s(val) USING 'sai' WITH OPTIONS = { 'normalize' : false }");
-
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
-
-        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u00E1l'").size());
-
-        // Both \u00E1 and \u0061\u0301 are visible as the character á, but without NFC normalization, they won't match.
-        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
-    }
-
-    @Test
-    public void shouldEnableNormalizedSearch()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        createIndex("CREATE INDEX ON %s(val) USING 'sai' WITH OPTIONS = { 'normalize' : true }");
-
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
-
-        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
-    }
-
-    @Test
-    public void shouldEnableNormalizedCaseInsensitiveSearch()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        createIndex("CREATE INDEX ON %s(val) USING 'sai' WITH OPTIONS = { 'normalize' : true, 'case_sensitive' : false}");
-
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
-
-        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'cam\u0061\u0301l'").size());
-    }
-
-    @Test
-    public void shouldEnableAsciiSearch()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-
-        createIndex("CREATE INDEX ON %s(val) USING 'sai' WITH OPTIONS = { 'ascii' : true, 'case_sensitive' : false}");
-
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Éppinger')");
-
-        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'eppinger'").size());
-    }
-
-    @Test
-    public void shouldRejectAnalysisOnPrimaryKeyColumns()
-    {
-        createTable("CREATE TABLE %s (k1 text, k2 text, c1 text, c2 text, PRIMARY KEY((k1, k2), c1, c2))");
-
-        for (String column : Arrays.asList("k1", "k2", "c1", "c2"))
-        {
-            for (String enabled : Arrays.asList("true", "false"))
-            {
-                assertRejectsAnalysisOnPrimaryKeyColumns(column, ImmutableMap.of(NonTokenizingOptions.NORMALIZE, enabled));
-                assertRejectsAnalysisOnPrimaryKeyColumns(column, ImmutableMap.of(NonTokenizingOptions.CASE_SENSITIVE, enabled));
-                assertRejectsAnalysisOnPrimaryKeyColumns(column, ImmutableMap.of(NonTokenizingOptions.ASCII, enabled));
-                assertRejectsAnalysisOnPrimaryKeyColumns(column, ImmutableMap.of(NonTokenizingOptions.NORMALIZE, enabled,
-                                                                                 NonTokenizingOptions.CASE_SENSITIVE, enabled,
-                                                                                 NonTokenizingOptions.ASCII, enabled));
-            }
-        }
-    }
-
-    private void assertRejectsAnalysisOnPrimaryKeyColumns(String column, Map<String, String> optionsMap)
-    {
-        String options = new CqlBuilder().append(optionsMap).toString();
-        Assertions.assertThatThrownBy(() -> createIndex("CREATE INDEX ON %s(" + column + ") USING 'sai' WITH OPTIONS = " + options))
-                  .hasRootCauseInstanceOf(InvalidRequestException.class)
-                  .hasRootCauseMessage(StorageAttachedIndex.ANALYSIS_ON_KEY_COLUMNS_MESSAGE + options);
-    }
-
-    @Test
-    public void shouldCreateIndexOnReversedType()
-    {
-        createTable("CREATE TABLE %s (id text, ck1 text, val text, PRIMARY KEY (id,ck1)) WITH CLUSTERING ORDER BY (ck1 desc)");
-
-        String indexNameCk1 = createIndex("CREATE INDEX ON %s(ck1) USING 'sai'");
-
-        execute("insert into %s(id, ck1, val) values('1', '2', '3')");
-        execute("insert into %s(id, ck1, val) values('1', '3', '4')");
-        assertEquals(1, executeNet("SELECT * FROM %s WHERE ck1='3'").all().size());
-
-        flush();
-        assertEquals(1, executeNet("SELECT * FROM %s WHERE ck1='2'").all().size());
-
-        SecondaryIndexManager sim = getCurrentColumnFamilyStore().indexManager;
-        StorageAttachedIndex index = (StorageAttachedIndex) sim.getIndexByName(indexNameCk1);
-        IndexTermType indexTermType = index.termType();
-        assertTrue(indexTermType.isLiteral());
-        assertTrue(indexTermType.isReversed());
-    }
-
-    @Test
-    public void shouldCreateIndexWithFullClassName()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'");
-
-        assertEquals(1, saiCreationCounter.get());
-    }
-
-    /**
-     * Verify SASI can be created and queries with SAI dependencies.
-     * Not putting in {@link MixedIndexImplementationsTest} because it uses CQLTester which doesn't load SAI dependency.
-     */
-    @Test
-    public void shouldCreateSASI()
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-
-        createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'org.apache.cassandra.index.sasi.SASIIndex'");
-        createIndex("CREATE CUSTOM INDEX ON %s(v2) USING 'org.apache.cassandra.index.sasi.SASIIndex' WITH OPTIONS = {'mode': 'CONTAINS',\n" +
-                    "'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer',\n" +
-                    "'tokenization_enable_stemming': 'true',\n" +
-                    "'tokenization_locale': 'en',\n" +
-                    "'tokenization_skip_stop_words': 'true',\n" +
-                    "'analyzed': 'true',\n" +
-                    "'tokenization_normalize_lowercase': 'true'};");
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
-
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(1, rows.all().size());
-
-        rows = executeNet("SELECT id1 FROM %s WHERE v2 like '0'");
-        assertEquals(1, rows.all().size());
-    }
-
-    @Test
-    public void shouldFailCreationOnMultipleColumns()
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val1 text, val2 text)");
-
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(val1, val2) USING 'sai'"))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessageContaining("storage-attached index cannot be created over multiple columns");
-    }
-
-    @Test
-    public void shouldFailCreationMultipleIndexesOnSimpleColumn()
-    {
-        createTable("CREATE TABLE %s (id int PRIMARY KEY, v1 TEXT)");
-        execute("INSERT INTO %s (id, v1) VALUES(1, '1')");
-        flush();
-
-        executeNet("CREATE INDEX index_1 ON %s(v1) USING 'sai'");
-        waitForTableIndexesQueryable();
-
-        // same name
-        assertThatThrownBy(() -> executeNet("CREATE INDEX index_1 ON %s(v1) USING 'sai'"))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessageContaining(String.format(CreateIndexStatement.INDEX_ALREADY_EXISTS, "index_1"));
-
-        // different name, same option
-        assertThatThrownBy(() -> executeNet("CREATE INDEX index_2 ON %s(v1) USING 'sai'"))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessageContaining(String.format(CreateIndexStatement.INDEX_DUPLICATE_OF_EXISTING, "index_2", "index_1"));
-
-        // different name, different option, same target.
-        assertThatThrownBy(() -> executeNet("CREATE INDEX ON %s(v1) USING 'sai' WITH OPTIONS = { 'case_sensitive' : true }"))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessageContaining("Cannot create more than one storage-attached index on the same column: v1" );
-
-        ResultSet rows = executeNet("SELECT id FROM %s WHERE v1 = '1'");
-        assertEquals(1, rows.all().size());
-    }
-
-    @Test
-    public void shouldIndexBuildingWithInMemoryData()
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-
-        int rowCount = 10;
-        for (int i = 0; i < rowCount; i++)
-            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', " + i + ", '0')");
-
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForTableIndexesQueryable();
-
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(rowCount, rows.all().size());
-    }
-
-    @Test
-    public void shouldIndexExistingMemtableOnCreationWithConcurrentFlush() throws Throwable
-    {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
-        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
-
-        Injections.Barrier delayInitializationTask =
-        Injections.newBarrier("delayInitializationTask", 2, false)
-                  .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndex.class).onMethod("startInitialBuild"))
-                  .build();
-
-        // Create the index, but do not allow the initial index build to begin:
-        Injections.inject(delayInitializationTask);
-        createIndexAsync("CREATE INDEX ON %s(val) USING 'sai'");
-
-        // Flush the Memtable's contents, which will feed data to the index as the SSTable is written:
-        flush();
-
-        // Allow the initialization task, which builds the index, to continue:
-        delayInitializationTask.countDown();
-
-        waitForTableIndexesQueryable();
-
-        ResultSet rows = executeNet("SELECT id FROM %s WHERE val = 'Camel'");
-        assertEquals(1, rows.all().size());
-
-        assertZeroSegmentBuilderUsage();
-    }
-
-    @Test
-    public void shouldRejectQueriesBeforeIndexInitializationFinished() throws Throwable
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-
-        int rowCount = 10;
-        for (int i = 0; i < rowCount; i++)
-            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', " + i + ", '0')");
-
-        Injections.inject(forceFlushPause);
-        createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-
-        assertThatThrownBy(() -> executeNet("SELECT id1 FROM %s WHERE v1>=0")).isInstanceOf(ReadFailureException.class);
-    }
-
-    @Test
-    public void shouldRejectQueriesOnIndexInitializationFailure() throws Throwable
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        disableCompaction(KEYSPACE);
-
-        int rowCount = 10;
-        for (int i = 0; i < rowCount; i++)
-            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', " + i + ", '0')");
-        flush();
-
-        Injections.inject(failSAIInitialializaion);
-        createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForAssert(() -> assertEquals(1, indexBuildCounter.get()));
-        waitForCompactions();
-
-        assertThatThrownBy(() -> executeNet("SELECT id1 FROM %s WHERE v1>=0")).isInstanceOf(ReadFailureException.class);
-    }
-
-    @Test
-    public void shouldReleaseIndexFilesAfterDroppingLastIndex() throws Throwable
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-        IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
-
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 0);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 0);
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
-        flush();
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(1, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(1, rows.all().size());
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0')");
-        flush();
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2);
-
-        verifySSTableIndexes(numericIndexIdentifier, 2, 2);
-        verifySSTableIndexes(literalIndexIdentifier, 2, 2);
-        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(2, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(2, rows.all().size());
-
-        dropIndex(numericIndexIdentifier);
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 0);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 2, 2);
-        verifySSTableIndexes(numericIndexIdentifier, 2, 0);
-        verifySSTableIndexes(literalIndexIdentifier, 2, 2);
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(2, rows.all().size());
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('2', 2, '0')");
-        flush();
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 3, 0, 0);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 3, 3, 3);
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(3, rows.all().size());
-
-        dropIndex(literalIndexIdentifier);
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 0);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 0);
-        assertNull(getCurrentIndexGroup());
-
-        assertEquals("Segment memory limiter should revert to zero on drop.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-    }
-
-    @Test
-    public void shouldCreateIndexFilesAfterMultipleConcurrentIndexCreation()
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        verifyNoIndexFiles();
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
-        flush();
-        verifyNoIndexFiles();
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
-        flush();
-        verifyNoIndexFiles();
-
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-        IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
-
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2);
-
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(2, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(2, rows.all().size());
-
-        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-    }
-
-    @Test
-    public void shouldCreateIndexFilesAfterMultipleSequentialIndexCreation()
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        verifyNoIndexFiles();
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
-        flush();
-        verifyNoIndexFiles();
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
-        flush();
-        verifyNoIndexFiles();
-
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
-
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(2, rows.all().size());
-
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
-
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2);
-
-        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(2, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(2, rows.all().size());
-
-        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-    }
-
-    @Test
-    public void shouldReleaseIndexFilesAfterCompaction()
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        disableCompaction(KEYSPACE);
-
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-        IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
-        verifyNoIndexFiles();
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
-        flush();
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(1, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(1, rows.all().size());
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
-        flush();
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2);
-        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(2, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(2, rows.all().size());
-
-        compact();
-        waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1));
-        waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1));
-
-        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(2, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(2, rows.all().size());
-
-        assertEquals("Segment memory limiter should revert to zero after compaction.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-    }
-
-    @Test
-    public void truncateWithBuiltIndexes()
-    {
-        verifyTruncateWithIndex(false);
-    }
-
-    @Test
-    public void concurrentTruncateWithIndexBuilding()
-    {
-        verifyTruncateWithIndex(true);
-    }
-
-    private void verifyTruncateWithIndex(boolean concurrentTruncate)
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-
-        IndexIdentifier numericIndexIdentifier = null;
-        IndexIdentifier literalIndexIdentifier = null;
-
-        if (!concurrentTruncate)
-        {
-            numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-            literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        }
-
-        // create 100 rows, half in sstable and half in memtable
-        int num = 100;
-        for (int i = 0; i < num; i++)
-        {
-            if (i == num / 2)
-                flush();
-            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', 0, '0');");
-        }
-
-        if (concurrentTruncate)
-        {
-            numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-            literalIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-            truncate(true);
-            waitForTableIndexesQueryable();
-        }
-        else
-        {
-            truncate(true);
-        }
-
-        waitForAssert(this::verifyNoIndexFiles);
-
-        // verify index-view-manager has been cleaned up
-        verifySSTableIndexes(numericIndexIdentifier, 0);
-        verifySSTableIndexes(literalIndexIdentifier, 0);
-
-        assertEquals("Segment memory limiter should revert to zero after truncate.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-    }
-
-    @Test
-    public void verifyRebuildCorruptedFiles() throws Throwable
-    {
-        // prepare schema and data
-        createTable(CREATE_TABLE_TEMPLATE);
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
-        flush();
-
-        for (CorruptionType corruptionType : CorruptionType.values())
-        {
-            verifyRebuildCorruptedFiles(numericIndexIdentifier, literalIndexIdentifier, corruptionType, false);
-            verifyRebuildCorruptedFiles(numericIndexIdentifier, literalIndexIdentifier, corruptionType, true);
-        }
-
-        assertEquals("Segment memory limiter should revert to zero following rebuild.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-    }
-
-    private void verifyRebuildCorruptedFiles(IndexIdentifier numericIndexIdentifier,
-                                             IndexIdentifier literalIndexIdentifier,
-                                             CorruptionType corruptionType,
-                                             boolean rebuild) throws Throwable
-    {
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-        IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
-
-        for (IndexComponent component : Version.LATEST.onDiskFormat().perSSTableIndexComponents(false))
-            verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, null, null, corruptionType, true, true, rebuild);
-
-        for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(numericIndexTermType))
-            verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, numericIndexTermType, numericIndexIdentifier, corruptionType, false, true, rebuild);
-
-        for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(literalIndexTermType))
-            verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, literalIndexTermType, literalIndexIdentifier, corruptionType, true, false, rebuild);
-    }
-
-    private void verifyRebuildIndexComponent(IndexTermType numericIndexTermType,
-                                             IndexIdentifier numericIndexIdentifier,
-                                             IndexTermType literalIndexTermType,
-                                             IndexIdentifier literalIndexIdentifier,
-                                             IndexComponent component,
-                                             IndexTermType corruptionIndexTermType,
-                                             IndexIdentifier corruptionIndexIdentifier,
-                                             CorruptionType corruptionType,
-                                             boolean failedStringIndex,
-                                             boolean failedNumericIndex,
-                                             boolean rebuild) throws Throwable
-    {
-        // The completion markers are valid if they exist on the file system, so we only need to test
-        // their removal. If we are testing with encryption then we don't want to test any components
-        // that are encryptable unless they have been removed because encrypted components aren't
-        // checksum validated.
-
-        if (PER_SSTABLE_COMPONENTS.contains(component))
-            return;
-
-        if (((component == IndexComponent.GROUP_COMPLETION_MARKER) ||
-             (component == IndexComponent.COLUMN_COMPLETION_MARKER)) &&
-            (corruptionType != CorruptionType.REMOVED))
-            return;
-
-        int rowCount = 2;
-
-        // initial verification
-        verifySSTableIndexes(numericIndexIdentifier, 1);
-        verifySSTableIndexes(literalIndexIdentifier, 1);
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
-        assertTrue(verifyChecksum(numericIndexTermType, numericIndexIdentifier));
-        assertTrue(verifyChecksum(literalIndexTermType, literalIndexIdentifier));
-
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(rowCount, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(rowCount, rows.all().size());
-
-        // corrupt file
-        if (corruptionIndexTermType != null)
-            corruptIndexComponent(component, corruptionIndexIdentifier, corruptionType);
-        else
-            corruptIndexComponent(component, corruptionType);
-
-        // If we are removing completion markers then the rest of the components should still have
-        // valid checksums.
-        boolean expectedNumericState = !failedNumericIndex || isBuildCompletionMarker(component);
-        boolean expectedLiteralState = !failedStringIndex || isBuildCompletionMarker(component);
-
-        assertEquals("Checksum verification for " + component + " should be " + expectedNumericState + " but was " + !expectedNumericState,
-                     expectedNumericState, verifyChecksum(numericIndexTermType, numericIndexIdentifier));
-        assertEquals(expectedLiteralState, verifyChecksum(literalIndexTermType, literalIndexIdentifier));
-
-        if (rebuild)
-        {
-            rebuildIndexes(numericIndexIdentifier.indexName, literalIndexIdentifier.indexName);
-        }
-        else
-        {
-            // Reload all SSTable indexes to manifest the corruption:
-            reloadSSTableIndex();
-
-            // Verify the index cannot be read:
-            verifySSTableIndexes(numericIndexIdentifier, Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedNumericIndex ? 0 : 1);
-            verifySSTableIndexes(literalIndexIdentifier, Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedStringIndex ? 0 : 1);
-
-            try
-            {
-                // If the corruption is that a file is missing entirely, the index won't be marked non-queryable...
-                rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-                assertEquals(failedNumericIndex ? 0 : rowCount, rows.all().size());
-            }
-            catch (ReadFailureException e)
-            {
-                // ...but most kind of corruption will result in the index being non-queryable.
-            }
-
-            try
-            {
-                // If the corruption is that a file is missing entirely, the index won't be marked non-queryable...
-                rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-                assertEquals(failedStringIndex ? 0 : rowCount, rows.all().size());
-            }
-            catch (ReadFailureException e)
-            {
-                // ...but most kind of corruption will result in the index being non-queryable.
-            }
-
-            // Simulate the index repair that would occur on restart:
-            runInitializationTask();
-        }
-
-        // verify indexes are recovered
-        verifySSTableIndexes(numericIndexIdentifier, 1);
-        verifySSTableIndexes(numericIndexIdentifier, 1);
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
-
-        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(rowCount, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(rowCount, rows.all().size());
-    }
-
-    @Test
-    public void verifyCleanupFailedPerIndexFiles() throws Throwable
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        disableCompaction(KEYSPACE);
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
-        flush();
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
-        flush();
-
-        // Inject failure
-        Injections.inject(failPerIndexMetaCompletion);
-        failPerIndexMetaCompletion.enable();
-
-        try
-        {
-            // Create a new index, which will actuate a build compaction and fail, but leave the node running...
-            IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-            IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-            // two index builders running in different compaction threads because of parallelised index initial build
-            waitForAssert(() -> assertEquals(2, indexBuildCounter.get()));
-            waitForCompactionsFinished();
-
-            // Only token/offset files for the first SSTable in the compaction task should exist, while column-specific files are blown away:
-            verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 0);
-
-            assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
-            assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-        }
-        finally
-        {
-            failPerIndexMetaCompletion.disable();
-        }
-    }
-
-    @Test
-    public void verifyCleanupFailedPrimaryKeyFiles() throws Throwable
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        disableCompaction(KEYSPACE);
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
-        flush();
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
-        flush();
-
-        // Inject failure
-        Injections.inject(failPerSSTableTokenAdd);
-        failPerSSTableTokenAdd.enable();
-
-        try
-        {
-            // Create a new index, which will actuate a build compaction and fail, but leave the node running...
-            createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-            // two index builders running in different compaction threads because of parallelised index initial build
-            waitForAssert(() -> assertEquals(2, indexBuildCounter.get()));
-            waitForAssert(() -> assertEquals(0, getCompactionTasks()));
-
-            // SSTable-level token/offset file(s) should be removed, while column-specific files never existed:
-            verifyNoIndexFiles();
-
-            assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
-            assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-        }
-        finally
-        {
-            failPerSSTableTokenAdd.disable();
-        }
-    }
-
-    @Test
-    public void verifyFlushAndCompactEmptyIndex()
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        disableCompaction(KEYSPACE);
-
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-        IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
-
-        // flush empty index
-        execute("INSERT INTO %s (id1) VALUES ('0');");
-        flush();
-
-        execute("INSERT INTO %s (id1) VALUES ('1');");
-        flush();
-
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 2);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 0, 2);
-
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(0, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(0, rows.all().size());
-
-        // compact empty index
-        compact();
-        waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1, 0, 1));
-        waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1, 0, 1));
-
-        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(0, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(0, rows.all().size());
-
-        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-    }
-
-    @Test
-    public void verifyFlushAndCompactNonIndexableRows()
-    {
-        // valid row ids, but no valid indexable content
-        Runnable populateData = () -> {
-            try
-            {
-                execute("INSERT INTO %s (id1) VALUES ('0');");
-                flush();
-
-                execute("INSERT INTO %s (id1) VALUES ('1');");
-                flush();
-            }
-            catch (Throwable e)
-            {
-                throw Throwables.unchecked(e);
-            }
-        };
-
-
-        verifyFlushAndCompactEmptyIndexes(populateData);
-    }
-
-    @Test
-    public void verifyFlushAndCompactTombstones()
-    {
-        // no valid row ids
-        Runnable populateData = () -> {
-            try
-            {
-                execute("DELETE FROM %s WHERE id1 = '0'");
-                flush();
-
-                execute("DELETE FROM %s WHERE id1 = '1'");
-                flush();
-            }
-            catch (Throwable e)
-            {
-                throw Throwables.unchecked(e);
-            }
-        };
-
-        verifyFlushAndCompactEmptyIndexes(populateData);
-    }
-
-    private void verifyFlushAndCompactEmptyIndexes(Runnable populateData)
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        disableCompaction(KEYSPACE);
-
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-        IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
-        populateData.run();
-        verifySSTableIndexes(numericIndexIdentifier, 2, 0);
-        verifySSTableIndexes(literalIndexIdentifier, 2, 0);
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 2);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 0, 2);
-
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(0, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(0, rows.all().size());
-
-        // compact empty index
-        compact();
-        verifySSTableIndexes(numericIndexIdentifier, 1, 0);
-        verifySSTableIndexes(literalIndexIdentifier, 1, 0);
-        waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1, 0, 1));
-        waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1, 0, 1));
-
-        rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(0, rows.all().size());
-        rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
-        assertEquals(0, rows.all().size());
-
-        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-    }
-
-    @Test
-    public void droppingIndexStopInitialIndexBuild() throws Throwable
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        disableCompaction(KEYSPACE);
-
-        int num = 100;
-        for (int i = 0; i < num; i++)
-        {
-            execute("INSERT INTO %s (id1, v1, v2) VALUES (?, 0, '0')", Integer.toString(i));
-        }
-        flush();
-
-        Injections.Barrier delayIndexBuilderCompletion = Injections.newBarrier("delayIndexBuilder", 2, false)
-                                                                   .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndexBuilder.class).onMethod("build"))
-                                                                   .build();
-
-        Injections.inject(delayIndexBuilderCompletion);
-        IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        waitForAssert(() -> assertEquals(1, delayIndexBuilderCompletion.getCount()));
-
-        dropIndex(indexIdentifier);
-
-        // let blocked builders to continue
-        delayIndexBuilderCompletion.countDown();
-        waitForCompactions();
-
-        delayIndexBuilderCompletion.disable();
-
-        assertNull(getCurrentIndexGroup());
-        assertFalse("Expect index not built", SystemKeyspace.isIndexBuilt(KEYSPACE, indexIdentifier.indexName));
-
-        // create index again, it should succeed
-        indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        waitForTableIndexesQueryable();
-        verifySSTableIndexes(indexIdentifier, 1);
-
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(num, rows.all().size());
-    }
-
-    @Test
-    @SuppressWarnings("BusyWait")
-    public void nodetoolStopInitialIndexBuild() throws Throwable
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        disableCompaction(KEYSPACE);
-
-        // create 100 rows into 1 sstable
-        int num = 100;
-        int sstable = 1;
-        for (int i = 0; i < num; i++)
-        {
-            execute("INSERT INTO %s (id1, v1, v2) VALUES ('" + i + "', 0, '0');");
-        }
-        flush();
-
-        Injections.Barrier delayIndexBuilderCompletion = Injections.newBarrierAwait("delayIndexBuilder", 1, true)
-                                                                   .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndexBuilder.class).onMethod("build"))
-                                                                   .build();
-
-        Injections.inject(delayIndexBuilderCompletion);
-
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-
-        waitForAssert(() -> assertTrue(getCompactionTasks() > 0), 1000, TimeUnit.MILLISECONDS);
-
-        // Stop initial index build by interrupting active and pending compactions
-        int attempt = 20;
-        while (getCompactionTasks() > 0 && attempt > 0)
-        {
-            System.out.println("Attempt " + attempt + " at stopping the compaction tasks");
-
-            // only interrupts active compactions, not pending compactions.
-            CompactionManager.instance.stopCompaction(OperationType.INDEX_BUILD.name());
-            // let blocked builder to continue, but still block pending builder threads
-            delayIndexBuilderCompletion.reset();
-
-            Thread.sleep(3000);
-            attempt--;
-        }
-        if (getCompactionTasks() > 0)
-            fail("Compaction tasks are not interrupted.");
-
-        delayIndexBuilderCompletion.disable();
-
-        // initial index builder should have stopped abruptly resulting in the index not being queryable
-        verifyInitialIndexFailed(numericIndexIdentifier.indexName);
-        Assertions.assertThat(getNotQueryableIndexes()).isNotEmpty();
-
-        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
-        for (Index i : cfs.indexManager.listIndexes())
-        {
-            StorageAttachedIndex index = (StorageAttachedIndex) i;
-            assertEquals(0, index.memtableIndexManager().size());
-
-            View view = index.view();
-            assertTrue("Expect index build stopped", view.getIndexes().isEmpty());
-        }
-
-        assertEquals("Segment memory limiter should revert to zero on interrupted compactions.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-
-        // rebuild index
-        ColumnFamilyStore.rebuildSecondaryIndex(KEYSPACE, currentTable(), numericIndexIdentifier.indexName);
-
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, sstable);
-        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
-        assertEquals(num, rows.all().size());
-
-        assertEquals("Segment memory limiter should revert to zero following rebuild.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-
-        assertTrue(verifyChecksum(numericIndexTermType, numericIndexIdentifier));
-    }
-
-    @Test
-    public void shouldRejectQueriesWithCustomExpressions()
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-
-        String index = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-
-        assertThatThrownBy(() -> executeNet(String.format("SELECT * FROM %%s WHERE expr(%s, 0)", index)))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessage(String.format(IndexRestrictions.CUSTOM_EXPRESSION_NOT_SUPPORTED, index));
-    }
-
-    @Test
-    public void testInitialBuildParallelism()
-    {
-        Function<Long, SSTableReader> createMockSSTable = onDiskLength -> {
-            SSTableReader reader = Mockito.mock(SSTableReader.class);
-            when(reader.onDiskLength()).thenReturn(onDiskLength);
-            return reader;
-        };
-
-        Function<List<SSTableReader>, List<Long>> toSize = sstables -> sstables.stream().map(SSTableReader::onDiskLength).collect(Collectors.toList());
-
-        // total size = 55
-        List<SSTableReader> sstables = LongStream.range(1, 11).boxed().map(createMockSSTable).collect(Collectors.toList());
-
-        // avg = 55 == total size
-        List<List<SSTableReader>> groups = StorageAttachedIndex.groupBySize(sstables, 1);
-        Iterator<List<SSTableReader>> iterator = groups.iterator();
-        assertEquals(1, groups.size());
-        assertEquals(Arrays.asList(10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L, 2L, 1L), toSize.apply(iterator.next())); // size = 55
-
-        // avg = 27.5
-        groups = StorageAttachedIndex.groupBySize(sstables, 2);
-        iterator = groups.iterator();
-        assertEquals(2, groups.size());
-        assertEquals(Arrays.asList(10L, 9L, 8L, 7L), toSize.apply(iterator.next())); // size = 34
-        assertEquals(Arrays.asList(6L, 5L, 4L, 3L, 2L, 1L), toSize.apply(iterator.next())); // size = 21
-
-        // avg = 18.333
-        groups = StorageAttachedIndex.groupBySize(sstables, 3);
-        iterator = groups.iterator();
-        assertEquals(3, groups.size());
-        assertEquals(Arrays.asList(10L, 9L), toSize.apply(iterator.next())); // size = 19
-        assertEquals(Arrays.asList(8L, 7L, 6L), toSize.apply(iterator.next())); // size = 21
-        assertEquals(Arrays.asList(5L, 4L, 3L, 2L, 1L), toSize.apply(iterator.next())); // size = 15
-
-        // avg = 11
-        groups = StorageAttachedIndex.groupBySize(sstables, 5);
-        iterator = groups.iterator();
-        assertEquals(4, groups.size());
-        assertEquals(Arrays.asList(10L, 9L), toSize.apply(iterator.next())); // size = 19
-        assertEquals(Arrays.asList(8L, 7L), toSize.apply(iterator.next())); // size = 15
-        assertEquals(Arrays.asList(6L, 5L), toSize.apply(iterator.next())); // size = 11
-        assertEquals(Arrays.asList(4L, 3L, 2L, 1L), toSize.apply(iterator.next())); // size = 11
-
-        // avg = 5.5
-        groups = StorageAttachedIndex.groupBySize(sstables, 10);
-        iterator = groups.iterator();
-        assertEquals(7, groups.size());
-        assertEquals(singletonList(10L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(9L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(8L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(7L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(6L), toSize.apply(iterator.next()));
-        assertEquals(Arrays.asList(5L, 4L), toSize.apply(iterator.next()));
-        assertEquals(Arrays.asList(3L, 2L, 1L), toSize.apply(iterator.next()));
-
-        // avg = 2.75
-        groups = StorageAttachedIndex.groupBySize(sstables, 20);
-        iterator = groups.iterator();
-        assertEquals(9, groups.size());
-        assertEquals(singletonList(10L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(9L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(8L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(7L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(6L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(5L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(4L), toSize.apply(iterator.next()));
-        assertEquals(singletonList(3L), toSize.apply(iterator.next()));
-        assertEquals(Arrays.asList(2L, 1L), toSize.apply(iterator.next()));
-    }
-
-    private void assertZeroSegmentBuilderUsage()
-    {
-        assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0, getColumnIndexBuildsInProgress());
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryCellDeletionsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryCellDeletionsTest.java
deleted file mode 100644
index 39856bd004fa..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryCellDeletionsTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.cql;
-
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Force generates segments due to a small RAM size on compaction, to test segment splitting
- */
-public class TinySegmentQueryCellDeletionsTest extends AbstractQueryTester
-{
-    @Before
-    public void setSegmentWriteBufferSpace() throws Throwable
-    {
-        setSegmentWriteBufferSpace(0);
-    }
-
-    @Test
-    public void testCellDeletions() throws Throwable
-    {
-        IndexQuerySupport.cellDeletions(executor, dataModel, sets);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryRowDeletionsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryRowDeletionsTest.java
deleted file mode 100644
index 5a67230ec797..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryRowDeletionsTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.cql;
-
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Force generates segments due to a small RAM size on compaction, to test segment splitting
- */
-public class TinySegmentQueryRowDeletionsTest extends AbstractQueryTester
-{
-    @Before
-    public void setSegmentWriteBufferSpace() throws Throwable
-    {
-        setSegmentWriteBufferSpace(0);
-    }
-
-    @Test
-    public void testRowDeletions() throws Throwable
-    {
-        IndexQuerySupport.rowDeletions(executor, dataModel, sets);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryTimeToLiveTest.java b/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryTimeToLiveTest.java
deleted file mode 100644
index b89a8a8068fb..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryTimeToLiveTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.cql;
-
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Force generates segments due to a small RAM size on compaction, to test segment splitting
- */
-public class TinySegmentQueryTimeToLiveTest extends AbstractQueryTester
-{
-    @Before
-    public void setSegmentWriteBufferSpace() throws Throwable
-    {
-        setSegmentWriteBufferSpace(0);
-    }
-
-    @Test
-    public void testTimeToLive() throws Throwable
-    {
-        IndexQuerySupport.timeToLive(executor, dataModel, sets);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryWriteLifecycleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryWriteLifecycleTest.java
deleted file mode 100644
index b32e1db01041..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/TinySegmentQueryWriteLifecycleTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.cql;
-
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Force generates segments due to a small RAM size on compaction, to test segment splitting
- */
-public class TinySegmentQueryWriteLifecycleTest extends AbstractQueryTester
-{
-    @Before
-    public void setSegmentWriteBufferSpace() throws Throwable
-    {
-        setSegmentWriteBufferSpace(0);
-    }
-
-    @Test
-    public void testWriteLifecycle() throws Throwable
-    {
-        IndexQuerySupport.writeLifecycle(executor, dataModel, sets);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/TokenCollisionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/TokenCollisionTest.java
index ac150e0b38d4..6debec7c021e 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/TokenCollisionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/TokenCollisionTest.java
@@ -18,11 +18,12 @@
 package org.apache.cassandra.index.sai.cql;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.List;
 
+import org.junit.Before;
 import org.junit.Test;
 
+import com.datastax.driver.core.Row;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -31,90 +32,33 @@
 
 public class TokenCollisionTest extends SAITester
 {
-    @Test
-    public void skinnyPartitionTest()
+    @Before
+    public void setup()
     {
-        doSkinnyPartitionTest(10, 0);
+        requireNetwork();
     }
 
     @Test
-    public void skinnyPartitionLastRowTest()
-    {
-        doSkinnyPartitionTest(49, 9);
-    }
-
-    private void doSkinnyPartitionTest(int v1Match, int v2Match)
-    {
-        createTable("CREATE TABLE %s (pk blob, v1 int, v2 int, PRIMARY KEY (pk))");
-        createIndex("CREATE INDEX ON %s(v1) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(v2) USING 'sai'");
-
-        ByteBuffer prefix = ByteBufferUtil.bytes("key");
-        int numRows = 100;
-        int v1Count = 0;
-        int v2Count = 0;
-        List<Object[]> matchingPks = new ArrayList<>();
-        for (int pkCount = 0; pkCount < numRows; pkCount++)
-        {
-            ByteBuffer pk = Util.generateMurmurCollision(prefix, (byte) (pkCount / 64), (byte) (pkCount % 64));
-            if (v1Count == v1Match && v2Count == v2Match)
-                matchingPks.add(row(pk, v1Count, v2Count));
-            execute("INSERT INTO %s (pk, v1, v2) VALUES (?, ?, ?)", pk, v1Count++, v2Count++);
-            if (v1Count == 50)
-                v1Count = 0;
-            if (v2Count == 10)
-                v2Count = 0;
-        }
-        assertEquals(2, matchingPks.size());
-        flush();
-
-        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE v1=" + v1Match + " AND v2=" + v2Match), matchingPks.get(0), matchingPks.get(1));
-    }
-
-    @Test
-    public void widePartitionTest()
-    {
-        doWidePartitionTest(100, 10, 0);
-    }
-
-    @Test
-    public void widePartitionLastRowTest()
-    {
-        // Reduce the number of rows so the last row occurs at the first clustering value
-        doWidePartitionTest(97, 46, 6);
-    }
-
-    private void doWidePartitionTest(int numRows, int v1Match, int v2Match)
+    public void testSkippingWhenTokensCollide()
     {
-        createTable("CREATE TABLE %s (pk blob, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))");
-        createIndex("CREATE INDEX ON %s(v1) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(v2) USING 'sai'");
+        createTable("CREATE TABLE %s (pk blob, value text, PRIMARY KEY (pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
 
         ByteBuffer prefix = ByteBufferUtil.bytes("key");
-        int pkCount = 0;
-        int ckCount = 0;
-        int v1Count = 0;
-        int v2Count = 0;
-        List<Object[]> matchingRows = new ArrayList<>();
+        final int numRows = 640; // 5 blocks x 128 postings, so skip table will contain 5 entries
         for (int i = 0; i < numRows; i++)
         {
-            ByteBuffer pk = Util.generateMurmurCollision(prefix, (byte) (pkCount / 64), (byte) (pkCount % 64));
-            if (v1Count == v1Match && v2Count == v2Match)
-                matchingRows.add(row(pk, ckCount, v1Count, v2Count));
-            execute("INSERT INTO %s (pk, ck, v1, v2) VALUES (?, ?, ?, ?)", pk, ckCount++, v1Count++, v2Count++);
-            if (ckCount == 8)
-            {
-                ckCount = 0;
-                pkCount++;
-            }
-            if (v1Count == 50)
-                v1Count = 0;
-            if (v2Count == 10)
-                v2Count = 0;
+            ByteBuffer pk = Util.generateMurmurCollision(prefix, (byte) (i / 64), (byte) (i % 64));
+            execute("INSERT INTO %s (pk, value) VALUES (?, ?)", pk, "abc");
         }
-        assertEquals(2, matchingRows.size());
+        // sanity check, all should have the same partition key
+        assertEquals(numRows, execute("SELECT pk FROM %s WHERE token(pk) = token(?)", prefix).size());
         flush();
 
-        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE v1=" + v1Match + " AND v2=" + v2Match), matchingRows.get(0), matchingRows.get(1));
+        // A storage-attached index will advance token flow to the token that is shared between all indexed rows,
+        // and cause binary search on the postings skip table that looks like this [3, 3, 3, 3, 3].
+        List<Row> rows = executeNet("SELECT * FROM %s WHERE token(pk) >= token(?) AND value = 'abc'", prefix).all();
+        // we should match all the rows
+        assertEquals(numRows, rows.size());
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/TokenRangeReadTest.java b/test/unit/org/apache/cassandra/index/sai/cql/TokenRangeReadTest.java
index 67a030ef9f8a..769114c2f57a 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/TokenRangeReadTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/TokenRangeReadTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.index.sai.cql;
 
+import org.junit.Before;
 import org.junit.Test;
 
 import org.apache.cassandra.index.sai.SAITester;
@@ -27,6 +28,12 @@
 
 public class TokenRangeReadTest extends SAITester
 {
+    @Before
+    public void setup()
+    {
+        requireNetwork();
+    }
+
     @Test
     public void testTokenRangeRead() throws Throwable
     {
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/UnindexedExpressionsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/UnindexedExpressionsTest.java
deleted file mode 100644
index 4f9c60e593b1..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/UnindexedExpressionsTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql;
-
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
-import org.apache.cassandra.index.sai.SAITester;
-
-import static org.junit.Assert.assertTrue;
-
-public class UnindexedExpressionsTest extends SAITester
-{
-    @Test
-    public void inOperatorIsNotHandledByIndexTest() throws Throwable
-    {
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int)");
-        createIndex("CREATE INDEX ON %s(v1) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(v2) USING 'sai'");
-
-        execute("INSERT INTO %s (k, v1, v2) VALUES (1, 1, 1)");
-        execute("INSERT INTO %s (k, v1, v2) VALUES (2, 2, 2)");
-        execute("INSERT INTO %s (k, v1, v2) VALUES (3, 3, 1)");
-        execute("INSERT INTO %s (k, v1, v2) VALUES (4, 4, 2)");
-
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT * FROM %s WHERE v1 IN (1, 2)");
-
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 1");
-
-        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 1 ALLOW FILTERING"), row(1, 1, 1));
-
-        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 2 ALLOW FILTERING"), row(2, 2, 2));
-
-        assertTrue(execute("SELECT * FROM %s WHERE v1 IN (1, 3) AND v2 = 2 ALLOW FILTERING").isEmpty());
-    }
-
-    @Test
-    public void unsupportedOperatorsAreHandledTest() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk int primary key, val1 int, val2 text)");
-        createIndex("CREATE INDEX ON %s(val1) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(val2) USING 'sai'");
-
-        execute("INSERT INTO %s (pk, val1, val2) VALUES (1, 1, '11')");
-        execute("INSERT INTO %s (pk, val1, val2) VALUES (2, 2, '22')");
-        execute("INSERT INTO %s (pk, val1, val2) VALUES (3, 3, '33')");
-        execute("INSERT INTO %s (pk, val1, val2) VALUES (4, 4, '44')");
-
-        // The LIKE operator is rejected because it needs to be handled by an index
-        assertInvalidMessage("LIKE restriction is only supported on properly indexed columns",
-                             "SELECT pk FROM %s WHERE val1 = 1 AND val2 like '1%%'");
-
-        // The IS NOT operator is only valid on materialized views
-        assertInvalidMessage("Unsupported restriction:", "SELECT pk FROM %s WHERE val1 = 1 AND val2 is not null");
-
-        // The != operator is currently not supported at all
-        assertInvalidMessage("Unsupported \"!=\" relation:", "SELECT pk FROM %s WHERE val1 = 1 AND val2 != '22'");
-
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 < '22'");
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 <= '11'");
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 >= '11'");
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 > '00'");
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 in ('11', '22')");
-
-        assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 < '22' ALLOW FILTERING"), row(1));
-        assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 <= '11' ALLOW FILTERING"), row(1));
-        assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 >= '11' AND val2 <= '22' ALLOW FILTERING"), row(1), row(2));
-        assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 > '00' AND val2 <= '11' ALLOW FILTERING"), row(1));
-        assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 in ('11', '22') ALLOW FILTERING"), row(1), row(2));
-    }
-
-    @Test
-    public void unindexedMapColumnTest() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk int primary key, val1 int, val2 map<int, text>)");
-        createIndex("CREATE INDEX ON %s(val1) USING 'sai'");
-
-        execute("INSERT INTO %s (pk, val1, val2) VALUES (1, 1, {1 : '1', 2 : '2', 3 : '3'})");
-        execute("INSERT INTO %s (pk, val1, val2) VALUES (2, 2, {2 : '2', 3 : '3', 4 : '4'})");
-        execute("INSERT INTO %s (pk, val1, val2) VALUES (3, 3, {3 : '3', 4 : '4', 5 : '5'})");
-        execute("INSERT INTO %s (pk, val1, val2) VALUES (4, 4, {4 : '4', 5 : '5', 6 : '6'})");
-
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 CONTAINS KEY 1");
-        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 CONTAINS '2'");
-
-        assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 CONTAINS KEY 2 ALLOW FILTERING"), row(1), row(2));
-        assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 CONTAINS '2' ALLOW FILTERING"), row(1), row(2));
-        assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2[2] = '2' ALLOW FILTERING"), row(1), row(2));
-    }
-}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorAndLuceneTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorAndLuceneTest.java
new file mode 100644
index 000000000000..980c0e4117c0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorAndLuceneTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+
+import static org.apache.cassandra.index.sai.cql.VectorTypeTest.assertContainsInt;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+public class VectorAndLuceneTest extends VectorTester
+{
+
+    @Test
+    public void basicVectorLuceneSelectTest()
+    {
+        createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'" +
+                    " WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        // The str_val is set up to make sure that tokens are properly analyzed and lowercased.
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'One Duplicate phrase', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'This duplicate PHRASE', [2.0, 3.0, 4.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'A different Phrase', [3.0, 4.0, 5.0])");
+
+        // 'phrase' is in all rows, so expect all rows to be returned.
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE str_val : 'phrase' ORDER BY val ann of [2.5, 3.5, 4.5] LIMIT 3");
+        assertThat(result).hasSize(3);
+
+        // 'missing' is in no rows, so expect no rows to be returned.
+        result = execute("SELECT * FROM %s WHERE str_val : 'missing' ORDER BY val ann of [2.5, 3.5, 4.5] LIMIT 3");
+        assertThat(result).hasSize(0);
+
+        // Use limit to 1 to be the limiting condition. 'phrase' matches all three str_val. Matches row 1 becuase the vector is closer.
+        result = execute("SELECT * FROM %s WHERE str_val : 'phrase' ORDER BY val ann of [2.1, 3.1, 4.1] LIMIT 1");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 1);
+
+        // 'one' only matches for row 0. The vector is the same as the one in the 2nd row.
+        result = execute("SELECT * FROM %s WHERE str_val : 'one' ORDER BY val ann of [3.0, 4.0, 5.0] LIMIT 3");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+
+        // 'duplicate' matches rows 0 and 1. The vector matches row 2, but that doesn't match the WHERE clause.
+        result = execute("SELECT * FROM %s WHERE str_val : 'duplicate' ORDER BY val ann of [3.0, 4.0, 5.0] LIMIT 3");
+        assertThat(result).hasSize(2);
+        assertContainsInt(result, "pk", 0);
+        assertContainsInt(result, "pk", 1);
+
+    }
+
+    // partition delete won't trigger UpdateTransaction#onUpdated
+    @Test
+    public void partitionDeleteVectorInMemoryTest()
+    {
+        createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'" +
+                    " WITH OPTIONS = { 'index_analyzer': 'standard' }");
+
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'this test has tokens', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'not so plural token', [2.0, 3.0, 4.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'another test too', [3.0, 4.0, 5.0])");
+
+        UntypedResultSet result = execute("SELECT * FROM %s ORDER BY val ann of [2.5, 3.5, 4.5] LIMIT 3");
+        assertThat(result).hasSize(3);
+
+        execute("UPDATE %s SET val = null WHERE pk = 0");
+
+        assertEquals(0, execute("SELECT * FROM %s WHERE str_val : 'tokens' ORDER BY val ann of [1.1, 2.1, 3.1] LIMIT 2").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE str_val : 'token'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE str_val : 'test'").size());
+
+        result = execute("SELECT * FROM %s ORDER BY val ann of [1.1, 2.1, 3.1] LIMIT 1"); // closer to row 0
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 1);
+
+        execute("DELETE from %s WHERE pk = 1");
+        result = execute("SELECT * FROM %s ORDER BY val ann of [2.1, 3.1, 4.1] LIMIT 1"); // closer to row 1
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 2);
+
+        flush();
+
+        result = execute("SELECT * FROM %s ORDER BY val ann of [2.1, 3.1, 4.1] LIMIT 1");  // closer to row 1
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 2);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorBruteforceLocalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorBruteforceLocalTest.java
new file mode 100644
index 000000000000..3b9e1923f279
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorBruteforceLocalTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Before;
+
+/**
+ * Like VectorLocalTest but without limit on globalBruteForceRows
+ * to exercise the brute force path in V2VectorIndexSearcher.
+ */
+public class VectorBruteforceLocalTest extends VectorLocalTest
+{
+
+    // skip injection of globalBruteForceRows
+    @Before
+    @Override
+    public void setup() throws Throwable
+    {
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorCompactionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorCompactionTest.java
new file mode 100644
index 000000000000..d66342bc5d2d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorCompactionTest.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.ArrayList;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter;
+
+import static org.apache.cassandra.index.sai.disk.vector.CassandraOnHeapGraph.MIN_PQ_ROWS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class VectorCompactionTest extends VectorTester.Versioned
+{
+    @Test
+    public void testCompactionWithEnoughRowsForPQAndDeleteARow()
+    {
+        createTable();
+        disableCompaction();
+
+        for (int i = 0; i <= MIN_PQ_ROWS; i++)
+            execute("INSERT INTO %s (pk, v) VALUES (?, ?)", i, vector(i, i + 1));
+        flush();
+
+        // By deleting a row, we trigger a key histogram to round its estimate to 0 instead of 1 rows per key, and
+        // that broke compaction, so we test that here.
+        execute("DELETE FROM %s WHERE pk = 0");
+        flush();
+
+        // Run compaction, it fails if compaction is not successful
+        compact();
+
+        // Confirm we can query the data
+        assertRowCount(execute("SELECT * FROM %s ORDER BY v ANN OF [1,2] LIMIT 1"), 1);
+    }
+
+    @Test
+    public void testPQRefine()
+    {
+        createTable();
+        disableCompaction();
+
+        var vectors = new ArrayList<float[]>();
+        // 3 sstables
+        for (int j = 0; j < 3; j++)
+        {
+            for (int i = 0; i <= MIN_PQ_ROWS; i++)
+            {
+                var pk = j * MIN_PQ_ROWS + i;
+                var v = create2DVector();
+                vectors.add(v);
+                execute("INSERT INTO %s (pk, v) VALUES (?, ?)", pk, vector(v));
+            }
+            flush();
+        }
+
+//        TODO deterimine proper design for PQ training on vectors of dimension < 100.
+//        see https://github.com/riptano/cndb/issues/13630
+//        CompactionGraph.PQ_TRAINING_SIZE = 2 * MIN_PQ_ROWS;
+        compact();
+
+        // Confirm we can query the data with reasonable recall
+        double recall = 0;
+        int ITERS = 10;
+        for (int i = 0; i < ITERS; i++)
+        {
+            var q = create2DVector();
+            var result = execute("SELECT pk, v FROM %s ORDER BY v ANN OF ? LIMIT 20", vector(q));
+            var ann = result.stream().map(row -> {
+                var vList = row.getVector("v", FloatType.instance, 2);
+                return new float[]{ vList.get(0), vList.get(1) };
+            }).collect(Collectors.toList());
+            recall += computeRecall(vectors, q, ann, VectorSimilarityFunction.COSINE);
+        }
+        recall /= ITERS;
+        assert recall >= 0.9 : recall;
+    }
+
+    @Test
+    public void testOneToManyCompaction()
+    {
+        for (int sstables = 2; sstables <= 3; sstables++)
+        {
+            testOneToManyCompactionInternal(10, sstables);
+            testOneToManyCompactionInternal(MIN_PQ_ROWS, sstables);
+        }
+    }
+
+    // Exercise the one-to-many path in compaction
+    public void testOneToManyCompactionInternal(int vectorsPerSstable, int sstables)
+    {
+        createTable();
+
+        disableCompaction();
+
+        insertOneToManyRows(vectorsPerSstable, sstables);
+
+        // queries should behave sanely before and after compaction
+        validateQueries();
+        compact();
+        validateQueries();
+    }
+
+    @Test
+    public void testOneToManyCompactionTooManyHoles()
+    {
+        int sstables = 2;
+        testOneToManyCompactionInternal(10, sstables);
+        testOneToManyCompactionHolesInternal(MIN_PQ_ROWS, sstables);
+    }
+
+    @Test
+    public void testZeroOrOneToManyCompaction()
+    {
+        for (int sstables = 2; sstables <= 3; sstables++)
+        {
+            testZeroOrOneToManyCompactionInternal(10, sstables);
+            testZeroOrOneToManyCompactionInternal(MIN_PQ_ROWS, sstables);
+        }
+    }
+
+    public void testZeroOrOneToManyCompactionInternal(int vectorsPerSstable, int sstables)
+    {
+        createTable();
+        disableCompaction();
+
+        insertZeroOrOneToManyRows(vectorsPerSstable, sstables);
+
+        validateQueries();
+        compact();
+        validateQueries();
+    }
+
+    private void insertZeroOrOneToManyRows(int vectorsPerSstable, int sstables)
+    {
+        var R = getRandom();
+        double duplicateChance = R.nextDouble() * 0.2;
+        int j = 0;
+        boolean nullInserted = false;
+        
+        for (int i = 0; i < sstables; i++)
+        {
+            var vectorsInserted = new ArrayList<Vector<Float>>();
+            var duplicateExists = false;
+            while (vectorsInserted.size() < vectorsPerSstable || !duplicateExists)
+            {
+                if (!nullInserted && vectorsInserted.size() == vectorsPerSstable/2)
+                {
+                    // Insert one null vector in the middle
+                    execute("INSERT INTO %s (pk, v) VALUES (?, null)", j++);
+                    nullInserted = true;
+                    continue;
+                }
+
+                Vector<Float> v;
+                if (R.nextDouble() < duplicateChance && !vectorsInserted.isEmpty())
+                {
+                    // insert a duplicate
+                    v = vectorsInserted.get(R.nextIntBetween(0, vectorsInserted.size() - 1));
+                    duplicateExists = true;
+                }
+                else
+                {
+                    // insert a new random vector
+                    v = randomVectorBoxed(2);
+                    vectorsInserted.add(v);
+                }
+                assert v != null;
+                execute("INSERT INTO %s (pk, v) VALUES (?, ?)", j++, v);
+            }
+            flush();
+        }
+    }
+
+    public void testOneToManyCompactionHolesInternal(int vectorsPerSstable, int sstables)
+    {
+        createTable();
+
+        disableCompaction();
+
+        insertOneToManyRows(vectorsPerSstable, sstables);
+
+        // this should be done after writing data so that we exercise the "we thought we were going to use the
+        // one-to-many-path but there were too many holes so we changed plans" code path
+        V5VectorPostingsWriter.GLOBAL_HOLES_ALLOWED = 0.0;
+
+        // queries should behave sanely before and after compaction
+        validateQueries();
+        compact();
+        validateQueries();
+    }
+
+    private void insertOneToManyRows(int vectorsPerSstable, int sstables)
+    {
+        var R = getRandom();
+        double duplicateChance = R.nextDouble() * 0.2;
+        int j = 0;
+        for (int i = 0; i < sstables; i++)
+        {
+            var vectorsInserted = new ArrayList<Vector<Float>>();
+            var duplicateExists = false;
+            while (vectorsInserted.size() < vectorsPerSstable || !duplicateExists)
+            {
+                Vector<Float> v;
+                if (R.nextDouble() < duplicateChance && !vectorsInserted.isEmpty())
+                {
+                    v = vectorsInserted.get(R.nextIntBetween(0, vectorsInserted.size() - 1));
+                    duplicateExists = true;
+                }
+                else
+                {
+                    v = randomVectorBoxed(2);
+                    vectorsInserted.add(v);
+                }
+                assert v != null;
+                execute("INSERT INTO %s (pk, v) VALUES (?, ?)", j++, v);
+            }
+            flush();
+        }
+    }
+
+    private void createTable()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+    }
+
+    private void validateQueries()
+    {
+        for (int i = 0; i < 10; i++)
+        {
+            var q = randomVectorBoxed(2);
+            var r = execute("SELECT pk, similarity_cosine(v, ?) as similarity FROM %s ORDER BY v ANN OF ? LIMIT 10", q, q);
+            float lastSimilarity = Float.MAX_VALUE;
+            assertEquals(10, r.size());
+            for (var row : r)
+            {
+                float similarity = (float) row.getFloat("similarity");
+                assertTrue(similarity <= lastSimilarity);
+                lastSimilarity = similarity;
+            }
+        }
+    }
+
+    private static float[] create2DVector() {
+        var R = getRandom();
+        return new float[] { R.nextFloatBetween(-100, 100), R.nextFloatBetween(-100, 100) };
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorDotProductWithLengthTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorDotProductWithLengthTest.java
new file mode 100644
index 000000000000..5692430456f6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorDotProductWithLengthTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.ArrayList;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+
+public class VectorDotProductWithLengthTest extends VectorTester
+{
+    @Override
+    public void setup() throws Throwable
+    {
+        super.setup();
+        // we are testing unit vector detection which is part of the v3 changes, but continues in all subsequent versions
+        assert V3OnDiskFormat.JVECTOR_VERSION >= 3 : "This test assumes JVector version 3 or greater";
+    }
+
+    // This tests our detection of unit-length vectors used with dot product and PQ.
+    // We want to switch to cosine similarity for PQ-based comparisons in those cases to preserve the angular semantics
+    // (since PQ compression does not preserve unit length of the compressed results),
+    // but if someone actually wants dot-product-with-length semantics (which this test does)
+    // then switching to cosine is incorrect.
+    @Test
+    public void testTrueDotproduct()
+    {
+        // setup
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'dot_product'}");
+        var vectors = new ArrayList<float[]>();
+        for (int i = 0; i < 2000; i++) { // 2000 is enough for PQ to run
+            var v = create2DVector();
+            vectors.add(v);
+            execute("INSERT INTO %s (pk, v) VALUES (?, ?)", i, vector(v));
+        }
+        flush();
+
+        // check that results are consistent with dot product similarity knn
+        double recall = 0;
+        int ITERS = 10;
+        for (int i = 0; i < ITERS; i++) {
+            var q = create2DVector();
+            var result = execute("SELECT pk, v FROM %s ORDER BY v ANN OF ? LIMIT 20", vector(q));
+            var ann = result.stream().map(row -> {
+                var vList = row.getVector("v", FloatType.instance, 2);
+                return new float[] { vList.get(0), vList.get(1)};
+            }).collect(Collectors.toList());
+            recall += computeRecall(vectors, q, ann, VectorSimilarityFunction.DOT_PRODUCT);
+        }
+        recall /= ITERS;
+        assert recall >= 0.9 : recall;
+    }
+
+    private static float[] create2DVector() {
+        var R = getRandom();
+        // these need to NOT be unit vectors to test the difference between DP and cosine
+        return new float[] { R.nextFloatBetween(-100, 100), R.nextFloatBetween(-100, 100) };
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorHybridSearchTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorHybridSearchTest.java
new file mode 100644
index 000000000000..b5952d8ce348
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorHybridSearchTest.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.plan.QueryController;
+
+public class VectorHybridSearchTest extends VectorTester.VersionedWithChecksums
+{
+    @Test
+    public void testHybridSearchWithPrimaryKeyHoles() throws Throwable
+    {
+        setMaxBruteForceRows(0);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, val text, vec vector<float, 2>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // Insert rows into two sstables. The tokens for each PK are in each line's comment.
+        execute("INSERT INTO %s (pk, val, vec) VALUES (1, 'A', [1, 3])"); // -4069959284402364209
+        execute("INSERT INTO %s (pk, val, vec) VALUES (2, 'A', [1, 2])"); // -3248873570005575792
+        // Make the last row in the sstable the correct result. That way we verify the ceiling logic
+        // works correctly.
+        execute("INSERT INTO %s (pk, val, vec) VALUES (3, 'A', [1, 1])"); // 9010454139840013625
+        flush();
+        execute("INSERT INTO %s (pk, val, vec) VALUES (5, 'A', [1, 5])"); // -7509452495886106294
+        execute("INSERT INTO %s (pk, val, vec) VALUES (4, 'A', [1, 4])"); // -2729420104000364805
+        execute("INSERT INTO %s (pk, val, vec) VALUES (6, 'A', [1, 6])"); // 2705480034054113608
+
+        // Get all rows using first predicate, then filter to get top 1
+        // Use a small limit to ensure we do not use brute force
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 1"),
+                       row(3));
+        });
+    }
+
+    // Clustering columns hit a different code path, we need both sets of tests, even though queries
+    // that expose the underlying regression are the same.
+    @Test
+    public void testHybridSearchWithPrimaryKeyHolesAndWithClusteringColumns() throws Throwable
+    {
+        setMaxBruteForceRows(0);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, a int, val text, vec vector<float, 2>, PRIMARY KEY(pk, a))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // Insert rows into two sstables. The tokens for each PK are in each line's comment.
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, 1, 'A', [1, 3])"); // -4069959284402364209
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (2, 1, 'A', [1, 2])"); // -3248873570005575792
+        // Make the last row in the sstable the correct result. That way we verify the ceiling logic
+        // works correctly.
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (3, 1, 'A', [1, 1])"); // 9010454139840013625
+        flush();
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (5, 1, 'A', [1, 5])"); // -7509452495886106294
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (4, 1, 'A', [1, 4])"); // -2729420104000364805
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (6, 1, 'A', [1, 6])"); // 2705480034054113608
+
+        // Get all rows using first predicate, then filter to get top 1
+        // Use a small limit to ensure we do not use brute force
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 1"),
+                       row(3));
+        });
+    }
+
+    @Test
+    public void testHybridSearchSequentialClusteringColumns() throws Throwable
+    {
+        setMaxBruteForceRows(0);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, a int, val text, vec vector<float, 2>, PRIMARY KEY(pk, a))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, 1, 'A', [1, 3])");
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, 2, 'A', [1, 2])");
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, 3, 'A', [1, 1])");
+
+        // Get all rows using first predicate, then filter to get top 1
+        // Use a small limit to ensure we do not use brute force
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,3] LIMIT 1"), row(1));
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,2] LIMIT 1"), row(2));
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 1"), row(3));
+        });
+    }
+
+    @Test
+    public void testHybridSearchHoleInClusteringColumnOrdering() throws Throwable
+    {
+        setMaxBruteForceRows(0);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, a int, val text, vec vector<float, 2>, PRIMARY KEY(pk, a))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // Create two sstables. The first needs a hole forcing us to skip.
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, 1, 'A', [1, 3])");
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, 3, 'A', [1, 2])");
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, 4, 'A', [1, 1])");
+        flush();
+        execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, 2, 'A', [1, 4])");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 1"), row(4));
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,2] LIMIT 1"), row(3));
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,3] LIMIT 1"), row(1));
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,4] LIMIT 1"), row(2));
+        });
+    }
+
+    @Test
+    public void testHybridSearchSeqLogicForMappingPKsBackToRowIds() throws Throwable
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, a int, val text, vec vector<float, 2>, PRIMARY KEY(pk, a))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex' WITH OPTIONS = { 'similarity_function' : 'euclidean' }");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // Insert rows into two sstables. The rows are interleaved to ensure binary search is less efficient, which
+        // pushes us to use a sequential scan when we map PKs back to row ids in the sstable.
+        int rowCount = 100;
+        // Insert even rows to first sstable
+        for (int i = 0; i < rowCount; i += 2)
+            execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, ?, 'A', ?)", i, vector(1, i));
+
+        flush();
+        // Insert odd rows to new sstable
+        for (int i = 1; i < rowCount; i += 2)
+            execute("INSERT INTO %s (pk, a, val, vec) VALUES (1, ?, 'A', ?)", i, vector(1, i));
+
+        // Verify result for rows in different memtables/sstables
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,49] LIMIT 1"),
+                       row(49));
+            assertRows(execute("SELECT a FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,50] LIMIT 1"),
+                       row(50));
+        });
+    }
+
+    // This test covers a bug in the RowIdMatchingOrdinalsView logic. Essentially, when the final rows in a segment
+    // do not have an associated vector, we will think we can do fast mapping from row id to ordinal, but in reality
+    // we have to do bounds checking still.
+    @Test
+    public void testHybridIndexWithPartialRowInsertsAtSegmentBoundaries() throws Throwable
+    {
+        // This test requires the non-bruteforce route
+        setMaxBruteForceRows(0);
+        createTable("CREATE TABLE %s (pk int, val text, vec vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, val, vec) VALUES (1, 'A', [1, 1])");
+        execute("INSERT INTO %s (pk, val) VALUES (2, 'A')");
+        flush();
+        execute("INSERT INTO %s (pk, vec) VALUES (2, [1,3])");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 1"), row(1));
+        });
+
+        // Assert the opposite with these writes where the lower bound is not present. (This case actually pushes us to
+        // use disk based ordinal mapping.)
+        execute("INSERT INTO %s (pk, val, vec) VALUES (2, 'A', [1, 2])");
+        execute("INSERT INTO %s (pk, val) VALUES (1, 'A')");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 1"), row(1));
+        });
+    }
+
+    @Test
+    public void testHybridQueryWithMissingVectorValuesForMaxSegmentRow() throws Throwable
+    {
+        // Want to test the search then order path
+        QueryController.QUERY_OPT_LEVEL = 0;
+
+        // We use a clustered primary key to simplify the mental model for this test.
+        // The bug this test exposed happens when the last row(s) in a segment, based on PK order, are present
+        // in a peer index for an sstable's search index but not its vector index.
+        createTable("CREATE TABLE %s (k int, i int, v vector<float, 2>, c int,  PRIMARY KEY(k, i))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
+        // We'll manually control compaction.
+        disableCompaction();
+
+        // Insert a complete row. We need at least one row with a vector and an entry for column c to ensure that
+        // the query doesn't skip the query portion where we map from Primary Key back to sstable row id.
+        execute("INSERT INTO %s (k, i, v, c) VALUES (0, ?, ?, ?)", 1, vector(1, 1), 1);
+
+        // Insert the first and last row in the table and leave of the vector
+        execute("INSERT INTO %s (k, i, c) VALUES (0, 0, 0)");
+        execute("INSERT INTO %s (k, i, c) VALUES (0, 2, 2)");
+
+        // The bug was specifically for sstables after compaction, but it's trivial to cover the before flush and before
+        // compaction cases here, so we do.
+        runThenFlushThenCompact(() -> {
+            // There is only one row that satisfies the WHERE clause and has a vector for each of these queries.
+            assertRows(execute("SELECT i FROM %s WHERE c <= 1 ORDER BY v ANN OF [1,1] LIMIT 1"), row(1));
+            assertRows(execute("SELECT i FROM %s WHERE c >= 1 ORDER BY v ANN OF [1,1] LIMIT 1"), row(1));
+        });
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorInvalidQueryTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorInvalidQueryTest.java
index f74c72c897e9..011c4ffd5c41 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/VectorInvalidQueryTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorInvalidQueryTest.java
@@ -20,7 +20,9 @@
 
 import java.util.Collections;
 
-import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
 import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
@@ -30,17 +32,12 @@
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
-import org.junit.BeforeClass;
-import org.junit.Test;
-
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -63,21 +60,21 @@ KEYSPACE, createTableName())))
     }
 
     @Test
-    public void cannotQueryEmptyVectorColumn()
+    public void cannotIndex1DWithCosine()
     {
-        createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
-        assertThatThrownBy(() -> execute("SELECT similarity_cosine((vector<float, 0>) [], []) FROM %s"))
-        .isInstanceOf(InvalidRequestException.class)
-        .hasMessage("vectors may only have positive dimensions; given 0");
+        createTable("CREATE TABLE %s (pk int, v vector<float, 1>, PRIMARY KEY(pk))");
+        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'cosine'}"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage("Cosine similarity is not supported for single-dimension vectors");
     }
 
     @Test
-    public void cannotIndex1DWithCosine()
+    public void cannotQueryEmptyVectorColumn() throws Throwable
     {
-        createTable("CREATE TABLE %s (pk int, v vector<float, 1>, PRIMARY KEY(pk))");
-        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'cosine'}"))
+        createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
+        assertThatThrownBy(() -> execute("SELECT similarity_cosine((vector<float, 0>) [], []) FROM %s"))
         .isInstanceOf(InvalidRequestException.class)
-        .hasRootCauseMessage(StorageAttachedIndex.VECTOR_1_DIMENSION_COSINE_ERROR);
+        .hasMessage("vectors may only have positive dimensions; given 0");
     }
 
     @Test
@@ -107,19 +104,19 @@ public void cannotQueryWrongNumberOfDimensions()
     }
 
     @Test
-    public void testMultiVectorOrderingsNotAllowed() throws Throwable
+    public void testMultiVectorOrderingsNotAllowed()
     {
         createTable("CREATE TABLE %s (pk int, str_val text, val1 vector<float, 3>, val2 vector<float, 3>, PRIMARY KEY(pk))");
         createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s(val1) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s(val2) USING 'StorageAttachedIndex'");
 
-        assertInvalidMessage("Cannot specify more than one ANN ordering",
+        assertInvalidMessage("Cannot specify more than one ordering column when using SAI indexes",
                              "SELECT * FROM %s ORDER BY val1 ann of [2.5, 3.5, 4.5], val2 ann of [2.1, 3.2, 4.0] LIMIT 2");
     }
 
     @Test
-    public void testDescendingVectorOrderingIsNotAllowed() throws Throwable
+    public void testDescendingVectorOrderingIsNotAllowed()
     {
         createTable("CREATE TABLE %s (pk int, val vector<float, 3>, PRIMARY KEY(pk))");
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
@@ -129,53 +126,57 @@ public void testDescendingVectorOrderingIsNotAllowed() throws Throwable
     }
 
     @Test
-    public void testVectorOrderingIsNotAllowedWithClusteringOrdering() throws Throwable
+    public void testVectorOrderingIsNotAllowedWithClusteringOrdering()
     {
         createTable("CREATE TABLE %s (pk int, ck int, val vector<float, 3>, PRIMARY KEY(pk, ck))");
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
-        assertInvalidMessage("ANN ordering does not support any other ordering",
+        assertInvalidMessage("Cannot combine clustering column ordering with non-clustering column ordering",
                              "SELECT * FROM %s ORDER BY val ann of [2.5, 3.5, 4.5], ck ASC LIMIT 2");
     }
 
     @Test
-    public void testVectorOrderingIsNotAllowedWithoutIndex() throws Throwable
+    public void testVectorOrderingIsNotAllowedWithoutIndex()
     {
         createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
 
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEX_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE, "val"),
                              "SELECT * FROM %s ORDER BY val ann of [2.5, 3.5, 4.5] LIMIT 5");
 
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEX_MESSAGE,
+        assertInvalidMessage(String.format(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_INDEX_MESSAGE, "val"),
                              "SELECT * FROM %s ORDER BY val ann of [2.5, 3.5, 4.5] LIMIT 5 ALLOW FILTERING");
     }
 
     @Test
-    public void testInvalidColumnNameWithAnn() throws Throwable
+    public void testVectorEqualityIsNotAllowed()
     {
-        String table = createTable(KEYSPACE, "CREATE TABLE %s (k int, c int, v int, primary key (k, c))");
-        assertInvalidMessage(String.format("Undefined column name bad_col in table %s", KEYSPACE + '.' + table),
-                             "SELECT k from %s ORDER BY bad_col ANN OF [1.0] LIMIT 1");
+        createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        assertInvalidMessage(StatementRestrictions.VECTOR_INDEXES_UNSUPPORTED_OP_MESSAGE,
+                             "SELECT * FROM %s WHERE val = [2.5, 3.5, 4.5] LIMIT 1");
+
+        assertInvalidMessage(StatementRestrictions.VECTOR_INDEXES_UNSUPPORTED_OP_MESSAGE,
+                             "SELECT * FROM %s WHERE val = [2.5, 3.5, 4.5]");
     }
 
     @Test
-    public void cannotPerformNonANNQueryOnVectorIndex() throws Throwable
+    public void annOrderingMustHaveLimit()
     {
         createTable("CREATE TABLE %s (pk int, ck int, val vector<float, 3>, PRIMARY KEY(pk, ck))");
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
-        assertInvalidMessage(StatementRestrictions.VECTOR_INDEXES_ANN_ONLY_MESSAGE,
-                             "SELECT * FROM %s WHERE val = [1.0, 2.0, 3.0]");
+        assertInvalidMessage("SAI based ORDER BY clause requires a LIMIT that is not greater than 1000. LIMIT was NO LIMIT",
+                             "SELECT * FROM %s ORDER BY val ann of [2.5, 3.5, 4.5]");
+
     }
 
     @Test
-    public void cannotOrderWithAnnOnNonVectorColumn() throws Throwable
+    public void testInvalidColumnNameWithAnn()
     {
-        createTable("CREATE TABLE %s (k int, v int, primary key(k))");
-        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
-
-        assertInvalidMessage(StatementRestrictions.ANN_ONLY_SUPPORTED_ON_VECTOR_MESSAGE,
-                             "SELECT * FROM %s ORDER BY v ANN OF 1 LIMIT 1");
+        String table = createTable(KEYSPACE, "CREATE TABLE %s (k int, c int, v int, primary key (k, c))");
+        assertInvalidMessage(String.format("Undefined column name bad_col in table %s", KEYSPACE + "." + table),
+                             "SELECT k from %s ORDER BY bad_col ANN OF [1.0] LIMIT 1");
     }
 
     @Test
@@ -185,46 +186,59 @@ public void disallowZeroVectorsWithCosineSimilarity()
         createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'cosine'}");
 
         assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, [0.0, 0.0])")).isInstanceOf(InvalidRequestException.class);
-        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(0.0f, 0.0f))).isInstanceOf(InvalidRequestException.class);
-        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1.0f, Float.NaN))).isInstanceOf(InvalidRequestException.class);
-        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1.0f, Float.POSITIVE_INFINITY))).isInstanceOf(InvalidRequestException.class);
-        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(Float.NEGATIVE_INFINITY, 1.0f))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(0, 0))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1E-6f, 1E-6f))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, Float.NaN))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, Float.POSITIVE_INFINITY))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(Float.NEGATIVE_INFINITY, 1))).isInstanceOf(InvalidRequestException.class);
         assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of [0.0, 0.0] LIMIT 2")).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(0, 0))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(1E-6f, 1E-6f))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(1, Float.NaN))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(1, Float.POSITIVE_INFINITY))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(Float.NEGATIVE_INFINITY, 1))).isInstanceOf(InvalidRequestException.class);
     }
 
     @Test
-    public void mustHaveLimitSpecifiedAndWithinMaxAllowed()
+    public void disallowZeroVectorsWithDefaultSimilarity()
     {
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 1>)");
-        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
-
-        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE k = 1 ORDER BY v ANN OF [0]"))
-        .isInstanceOf(InvalidQueryException.class).hasMessage(SelectStatement.TOPK_LIMIT_ERROR);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<float, 2>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
 
-        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE k = 1 ORDER BY v ANN OF [0] LIMIT 1001"))
-        .isInstanceOf(InvalidQueryException.class).hasMessage(String.format(StorageAttachedIndex.ANN_LIMIT_ERROR, IndexWriterConfig.MAX_TOP_K, 1001));
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, [0.0, 0.0])")).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(0, 0))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, Float.NaN))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, Float.POSITIVE_INFINITY))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(Float.NEGATIVE_INFINITY, 1))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of [0.0, 0.0] LIMIT 2")).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(0, 0))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(1, Float.NaN))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(1, Float.POSITIVE_INFINITY))).isInstanceOf(InvalidRequestException.class);
+        assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY value ann of ? LIMIT 2", vector(Float.NEGATIVE_INFINITY, 1))).isInstanceOf(InvalidRequestException.class);
     }
 
     @Test
-    public void mustHaveLimitWithinPageSize()
+    public void disallowClusteringColumnPredicateWithoutSupportingIndex()
     {
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 3>)");
+        createTable("CREATE TABLE %s (pk int, num int, v vector<float, 2>, PRIMARY KEY(pk, num))");
         createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+        execute("INSERT INTO %s (pk, num, v) VALUES (3, 1, [1,1])");
+        execute("INSERT INTO %s (pk, num, v) VALUES (3, 4, [1,4])");
+        flush();
 
-        execute("INSERT INTO %s (k, v) VALUES (1, [1.0, 2.0, 3.0])");
-        execute("INSERT INTO %s (k, v) VALUES (2, [2.0, 3.0, 4.0])");
-        execute("INSERT INTO %s (k, v) VALUES (3, [3.0, 4.0, 5.0])");
+        // If we didn't have the query planner fail this query, we would get incorrect results for both queries
+        // because the clustering columns are not yet available to restrict the ANN result set.
+        assertThatThrownBy(() -> execute("SELECT num FROM %s WHERE pk=3 AND num > 3 ORDER BY v ANN OF [1,1] LIMIT 1"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
 
-        ClientWarn.instance.captureWarnings();
-        ResultSet result = execute("SELECT * FROM %s WHERE k = 1 ORDER BY v ANN OF [2.0, 3.0, 4.0] LIMIT 10", 9);
-        assertEquals(1, ClientWarn.instance.getWarnings().size());
-        assertEquals(String.format(SelectStatement.TOPK_PAGE_SIZE_WARNING, 9, 10, 10), ClientWarn.instance.getWarnings().get(0));
-        assertEquals(1, result.size());
+        assertThatThrownBy(() -> execute("SELECT num FROM %s WHERE pk=3 AND num = 4 ORDER BY v ANN OF [1,1] LIMIT 1"))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(StatementRestrictions.NON_CLUSTER_ORDERING_REQUIRES_ALL_RESTRICTED_NON_PARTITION_KEY_COLUMNS_INDEXED_MESSAGE);
 
-        ClientWarn.instance.captureWarnings();
-        result = execute("SELECT * FROM %s WHERE k = 1 ORDER BY v ANN OF [2.0, 3.0, 4.0] LIMIT 10", 11);
-        assertNull(ClientWarn.instance.getWarnings());
-        assertEquals(1, result.size());
+        // Cover the alternative code path
+        createIndex("CREATE CUSTOM INDEX ON %s(num) USING 'StorageAttachedIndex'");
+        assertRows(execute("SELECT num FROM %s WHERE pk=3 AND num > 3 ORDER BY v ANN OF [1,1] LIMIT 1"), row(4));
     }
 
     @Test
@@ -238,127 +252,21 @@ public void cannotHaveAggregationOnANNQuery()
         execute("INSERT INTO %s (k, v, c) VALUES (3, [2], 100)");
         execute("INSERT INTO %s (k, v, c) VALUES (4, [1], 1000)");
 
-        assertThatThrownBy(() -> executeNet("SELECT sum(c) FROM %s WHERE k = 1 ORDER BY v ANN OF [0] LIMIT 4"))
-        .isInstanceOf(InvalidQueryException.class).hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
-    }
-
-    @Test
-    public void multipleVectorColumnsInQueryFailCorrectlyTest() throws Throwable
-    {
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 vector<float, 1>, v2 vector<float, 1>)");
-        createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
-
-        execute("INSERT INTO %s (k, v1, v2) VALUES (1, [1], [2])");
-
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEX_MESSAGE,
-                             "SELECT * FROM %s WHERE v1 = [1] ORDER BY v2 ANN OF [2] ALLOW FILTERING");
-    }
-
-    @Test
-    public void annOrderingIsNotAllowedWithoutIndexWhereIndexedColumnExistsInQueryTest() throws Throwable
-    {
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 1>, c int)");
-        createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
-
-        execute("INSERT INTO %s (k, v, c) VALUES (1, [4], 1)");
-        execute("INSERT INTO %s (k, v, c) VALUES (2, [3], 10)");
-        execute("INSERT INTO %s (k, v, c) VALUES (3, [2], 100)");
-        execute("INSERT INTO %s (k, v, c) VALUES (4, [1], 1000)");
-
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEX_MESSAGE,
-                             "SELECT * FROM %s WHERE c >= 100 ORDER BY v ANN OF [1] LIMIT 4 ALLOW FILTERING");
-    }
+        assertThatThrownBy(() -> execute("SELECT sum(c) FROM %s ORDER BY v ANN OF [0] LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
 
-    @Test
-    public void cannotPostFilterOnNonIndexedColumnWithAnnOrdering() throws Throwable
-    {
-        createTable("CREATE TABLE %s (pk1 int, pk2 int, ck1 int, ck2 int, v vector<float, 1>, c int, primary key ((pk1, pk2), ck1, ck2))");
-        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        assertThatThrownBy(() -> execute("SELECT sum(c) FROM %s WHERE k = 1 ORDER BY v ANN OF [0] LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
 
-        execute("INSERT INTO %s (pk1, pk2, ck1, ck2, v, c) VALUES (1, 1, 1, 1, [4], 1)");
-        execute("INSERT INTO %s (pk1, pk2, ck1, ck2, v, c) VALUES (2, 2, 1, 1, [3], 10)");
-        execute("INSERT INTO %s (pk1, pk2, ck1, ck2, v, c) VALUES (3, 3, 1, 1, [2], 100)");
-        execute("INSERT INTO %s (pk1, pk2, ck1, ck2, v, c) VALUES (4, 4, 1, 1, [1], 1000)");
+        assertThatThrownBy(() -> execute("SELECT * FROM %s GROUP BY k ORDER BY v ANN OF [0] LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
 
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE,
-                             "SELECT * FROM %s WHERE c >= 100 ORDER BY v ANN OF [1] LIMIT 4 ALLOW FILTERING");
-
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE,
-                             "SELECT * FROM %s WHERE ck1 >= 0 ORDER BY v ANN OF [1] LIMIT 4 ALLOW FILTERING");
-
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE,
-                             "SELECT * FROM %s WHERE ck2 = 1 ORDER BY v ANN OF [1] LIMIT 4 ALLOW FILTERING");
-
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE,
-                             "SELECT * FROM %s WHERE pk1 = 1 ORDER BY v ANN OF [1] LIMIT 4 ALLOW FILTERING");
-
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE,
-                             "SELECT * FROM %s WHERE pk2 = 1 ORDER BY v ANN OF [1] LIMIT 4 ALLOW FILTERING");
-
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE,
-                             "SELECT * FROM %s WHERE pk1 = 1 AND pk2 = 1 AND ck2 = 1 ORDER BY v ANN OF [1] LIMIT 4 ALLOW FILTERING");
-
-        assertInvalidMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE,
-                             "SELECT * FROM %s WHERE token(pk1, pk2) = token(1, 1) AND ck2 = 1 ORDER BY v ANN OF [1] LIMIT 4 ALLOW FILTERING");
-    }
-
-    @Test
-    public void cannotHavePerPartitionLimitWithAnnOrdering()
-    {
-        createTable("CREATE TABLE %s (k int, c int, v vector<float, 1>, PRIMARY KEY(k, c))");
-        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
-
-        execute("INSERT INTO %s (k, c, v) VALUES (1, 1, [1])");
-        execute("INSERT INTO %s (k, c, v) VALUES (1, 2, [2])");
-        execute("INSERT INTO %s (k, c, v) VALUES (1, 3, [3])");
-
-        assertThatThrownBy(() -> executeNet("SELECT * FROM %s ORDER BY v ANN OF [2] PER PARTITION LIMIT 1 LIMIT 3"))
-            .isInstanceOf(InvalidQueryException.class).hasMessage(SelectStatement.TOPK_PARTITION_LIMIT_ERROR);
-    }
-
-    @Test
-    public void cannotCreateIndexOnNonFloatVector()
-    {
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, 1>)");
-
-        assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'"))
-            .isInstanceOf(InvalidRequestException.class).hasRootCauseMessage(StorageAttachedIndex.VECTOR_NON_FLOAT_ERROR);
-    }
-
-    @Test
-    public void canOrderWithWhereOnPrimaryColumns() throws Throwable
-    {
-        createTable("CREATE TABLE %s (a int, b int, c int, d int, v vector<float, 2>, PRIMARY KEY ((a,b),c,d))");
-        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
-
-        execute("INSERT INTO %s (a, b, c, d, v) VALUES (1, 2, 1, 2, [6.0,1.0])");
-
-        ResultSet result = execute("SELECT * FROM %s WHERE a = 1 AND b = 2 ORDER BY v ANN OF [2.0,1.0] LIMIT 1", ConsistencyLevel.ONE);
-        assertEquals(1, result.size());
-        result = execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = 1 ORDER BY v ANN OF [2.0,1.0] LIMIT 1", ConsistencyLevel.ONE);
-        assertEquals(1, result.size());
-        result = execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = 1 AND d = 2 ORDER BY v ANN OF [2.0,1.0] LIMIT 1", ConsistencyLevel.ONE);
-        assertEquals(1, result.size());
-
-        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE a = 1 AND b = 2 AND d = 2 ORDER BY v ANN OF [2.0,1.0] LIMIT 1"))
-            .isInstanceOf(InvalidQueryException.class).hasMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE);
-
-        createIndex("CREATE CUSTOM INDEX c_idx ON %s(c) USING 'StorageAttachedIndex'");
-
-        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE a = 1 AND b = 2 AND d = 2 ORDER BY v ANN OF [2.0,1.0] LIMIT 1"))
-            .isInstanceOf(InvalidQueryException.class).hasMessage(StatementRestrictions.ANN_REQUIRES_INDEXED_FILTERING_MESSAGE);
-
-        dropIndex("DROP INDEX %s.c_idx");
-        createIndex("CREATE CUSTOM INDEX ON %s(d) USING 'StorageAttachedIndex'");
-
-        result = execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = 1 ORDER BY v ANN OF [2.0,1.0] LIMIT 1", ConsistencyLevel.ONE);
-        assertEquals(1, result.size());
-        result = execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c = 1 AND d = 2 ORDER BY v ANN OF [2.0,1.0] LIMIT 1", ConsistencyLevel.ONE);
-        assertEquals(1, result.size());
-        result = execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND d = 2 ORDER BY v ANN OF [2.0,1.0] LIMIT 1", ConsistencyLevel.ONE);
-        assertEquals(1, result.size());
-        result = execute("SELECT * FROM %s WHERE a = 1 AND b = 2 AND c > 0 ORDER BY v ANN OF [2.0,1.0] LIMIT 1", ConsistencyLevel.ONE);
-        assertEquals(1, result.size());
+        assertThatThrownBy(() -> execute("SELECT count(*) FROM %s ORDER BY v ANN OF [0] LIMIT 4"))
+                .isInstanceOf(InvalidRequestException.class)
+                .hasMessage(SelectStatement.TOPK_AGGREGATION_ERROR);
     }
 
     @Test
@@ -372,6 +280,7 @@ public void canOnlyExecuteWithCorrectConsistencyLevel()
         execute("INSERT INTO %s (k, c, v) VALUES (1, 3, [3])");
 
         ClientWarn.instance.captureWarnings();
+        execute("SELECT * FROM %s ORDER BY v ANN OF [2] LIMIT 3");
         ResultSet result = execute("SELECT * FROM %s ORDER BY v ANN OF [2] LIMIT 3", ConsistencyLevel.ONE);
         assertEquals(3, result.size());
         assertNull(ClientWarn.instance.getWarnings());
@@ -394,12 +303,12 @@ public void canOnlyExecuteWithCorrectConsistencyLevel()
                      ClientWarn.instance.getWarnings().get(0));
 
         assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY v ANN OF [2] LIMIT 3", ConsistencyLevel.SERIAL))
-            .isInstanceOf(InvalidRequestException.class)
-            .hasMessage(String.format(SelectStatement.TOPK_CONSISTENCY_LEVEL_ERROR, ConsistencyLevel.SERIAL));
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(String.format(SelectStatement.TOPK_CONSISTENCY_LEVEL_ERROR, ConsistencyLevel.SERIAL));
 
         assertThatThrownBy(() -> execute("SELECT * FROM %s ORDER BY v ANN OF [2] LIMIT 3", ConsistencyLevel.LOCAL_SERIAL))
-            .isInstanceOf(InvalidRequestException.class)
-            .hasMessage(String.format(SelectStatement.TOPK_CONSISTENCY_LEVEL_ERROR, ConsistencyLevel.LOCAL_SERIAL));
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage(String.format(SelectStatement.TOPK_CONSISTENCY_LEVEL_ERROR, ConsistencyLevel.LOCAL_SERIAL));
     }
 
     protected ResultSet execute(String query, ConsistencyLevel consistencyLevel)
@@ -418,10 +327,10 @@ protected ResultSet execute(String query, ConsistencyLevel consistencyLevel, int
         QueryState queryState = new QueryState(state);
 
         CQLStatement statement = QueryProcessor.parseStatement(formatQuery(query), queryState.getClientState());
-        statement.validate(state);
+        statement.validate(queryState.getClientState());
 
-        QueryOptions options = QueryOptions.withConsistencyLevel(QueryOptions.forInternalCalls(Collections.emptyList()), consistencyLevel);
-        options = QueryOptions.withPageSize(options, pageSize);
+        QueryOptions options = QueryOptions.forInternalCalls(Collections.emptyList());
+        options.updateConsistency(consistencyLevel);
 
         return ((ResultMessage.Rows)statement.execute(queryState, options, Dispatcher.RequestTime.forImmediateExecution())).result;
     }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorLocalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorLocalTest.java
index 26b64c57c345..8efae04836cd 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/VectorLocalTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorLocalTest.java
@@ -28,92 +28,155 @@
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Multimap;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.marshal.FloatType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+import org.apache.cassandra.index.sai.utils.Glove;
 import org.assertj.core.data.Percentage;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-public class VectorLocalTest extends VectorTester
+public class VectorLocalTest extends VectorTester.VersionedWithChecksums
 {
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    private static Glove.WordVector word2vec;
+
+    @BeforeClass
+    public static void loadModel() throws Throwable
+    {
+        word2vec = Glove.parse(VectorLocalTest.class.getClassLoader().getResourceAsStream("glove.3K.50d.txt"));
+    }
+
     @Test
-    public void keyRestrictionsWithFilteringTest()
+    public void skipToAssertRegressionTest()
     {
-        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 1>)");
-        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
-        execute("INSERT INTO %s (k, v) VALUES (1, [1])");
+        createTable("CREATE TABLE %s (\n" +
+                  "    lat_low_precision smallint,\n" +
+                  "    lon_low_precision smallint,\n" +
+                  "    lat_high_precision tinyint,\n" +
+                  "    lon_high_precision tinyint,\n" +
+                  "    id int,\n" +
+                  "    lat_exact decimal,\n" +
+                  "    lat_lon_embedding vector<float, 2>,\n" +
+                  "    lon_exact decimal,\n" +
+                  "    name text,\n" +
+                  "    proximity_edge tinyint,\n" +
+                  "    PRIMARY KEY ((lat_low_precision, lon_low_precision), lat_high_precision, lon_high_precision, id)\n" +
+                  ") WITH CLUSTERING ORDER BY (lat_high_precision ASC, lon_high_precision ASC, id ASC);");
+        createIndex("CREATE CUSTOM INDEX lat_high_precision_index ON %s (lat_high_precision) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex';");
+        createIndex("CREATE CUSTOM INDEX lat_lon_embedding_index ON %s (lat_lon_embedding) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' WITH OPTIONS = {'similarity_function': 'EUCLIDEAN'};");
+        createIndex("CREATE CUSTOM INDEX lon_high_precision_index ON %s (lon_high_precision) USING 'org.apache.cassandra.index.sai.StorageAttachedIndex';");
 
-        assertRows(execute("SELECT k, v FROM %s WHERE k > 0 LIMIT 4 ALLOW FILTERING"), row(1, vector(1f)));
-        assertRows(execute("SELECT k, v FROM %s WHERE k = 1 ORDER BY v ANN OF [0] LIMIT 4 ALLOW FILTERING"), row(1, vector(1f)));
+        int vectorCount = getRandom().nextIntBetween(500, 1000);
+        List<Vector<Float>> vectors = IntStream.range(0, vectorCount).mapToObj(s -> randomVectorBoxed(2)).collect(Collectors.toList());
+
+        int pk = 0;
+        for (Vector<Float> vector : vectors)
+            execute("INSERT INTO %s (lat_low_precision, lon_low_precision, lat_high_precision, lon_high_precision, id, lat_exact, lat_lon_embedding, lon_exact, name, proximity_edge) "
+                    + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (short)(pk % 3), (short)(pk % 5), (byte)(pk % 7), (byte)(pk % 11), pk++, 1.2 * pk, vector, 1.3 * pk, Integer.toString(pk), (byte)(pk % 2));
 
         flush();
-        assertRows(execute("SELECT k, v FROM %s WHERE k > 0 LIMIT 4 ALLOW FILTERING"), row(1, vector(1f)));
-        assertRows(execute("SELECT k, v FROM %s WHERE k = 1 ORDER BY v ANN OF [0] LIMIT 4 ALLOW FILTERING"), row(1, vector(1f)));
+
+        var queryVector = randomVectorBoxed(2);
+
+        final int limit = 10;
+        UntypedResultSet result;
+
+        result = execute("SELECT name, id, lat_exact, lon_exact, proximity_edge "
+                      + "FROM %s WHERE lat_low_precision = ? AND lon_low_precision = ? AND lat_high_precision = ? AND lon_high_precision = ? "
+                      + "ORDER BY lat_lon_embedding ANN OF ? LIMIT ?",
+                      (short)0, (short)3, (byte)3, (byte)3,
+                      queryVector, limit);
+
+        assertThat(result).hasSize(1);
+
+        result = execute("SELECT name, id, lat_exact, lon_exact, proximity_edge "
+                      + "FROM %s WHERE lat_low_precision = ? AND lon_low_precision = ? AND lat_high_precision = ? AND lon_high_precision = ? "
+                      + "ORDER BY lat_lon_embedding ANN OF ? LIMIT ?",
+                      (short)0, (short)0, (byte)0, (byte)0,
+                      queryVector, limit);
+
+        assertThat(result).hasSize(1);
     }
 
     @Test
     public void randomizedTest()
     {
-        createTable(String.format("CREATE TABLE %%s (pk int, str_val text, val vector<float, %d>, PRIMARY KEY(pk))", word2vec.dimension()));
+        randomizedTest(word2vec.dimension());
+    }
+
+    @Test
+    public void randomizedBqCompressedTest()
+    {
+        randomizedTest(2048);
+    }
+
+    private void randomizedTest(int dimension)
+    {
+        createTable(String.format("CREATE TABLE %%s (pk int, str_val text, val vector<float, %d>, PRIMARY KEY(pk))", dimension));
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
         int vectorCount = getRandom().nextIntBetween(500, 1000);
-        List<float[]> vectors = IntStream.range(0, vectorCount).mapToObj(s -> randomVector()).collect(Collectors.toList());
+        List<Vector<Float>> vectors = IntStream.range(0, vectorCount).mapToObj(s -> randomVectorBoxed(dimension)).collect(Collectors.toList());
 
         int pk = 0;
-        for (float[] vector : vectors)
-            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'A', " + vectorString(vector) + " )", pk++);
+        for (Vector<Float> vector : vectors)
+            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'A', ?)", pk++, vector);
 
         // query memtable index
         int limit = Math.min(getRandom().nextIntBetween(30, 50), vectors.size());
-        float[] queryVector = randomVector();
+        var queryVector = randomVectorBoxed(dimension);
         UntypedResultSet resultSet = search(queryVector, limit);
-        assertDescendingScore(queryVector, getVectorsFromResult(resultSet));
+        assertDescendingScore(queryVector, getVectorsFromResult(resultSet, dimension));
 
         flush();
 
         // query on-disk index
-        queryVector = randomVector();
+        queryVector = randomVectorBoxed(dimension);
         resultSet = search(queryVector, limit);
-        assertDescendingScore(queryVector, getVectorsFromResult(resultSet));
+        assertDescendingScore(queryVector, getVectorsFromResult(resultSet, dimension));
 
         // populate some more vectors
         int additionalVectorCount = getRandom().nextIntBetween(500, 1000);
-        List<float[]> additionalVectors = IntStream.range(0, additionalVectorCount).mapToObj(s -> randomVector()).collect(Collectors.toList());
-        for (float[] vector : additionalVectors)
-            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'A', " + vectorString(vector) + " )", pk++);
+        List<Vector<Float>> additionalVectors = IntStream.range(0, additionalVectorCount).mapToObj(s -> randomVectorBoxed(dimension)).collect(Collectors.toList());
+        for (Vector<Float> vector : additionalVectors)
+            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'A', ?)", pk++, vector);
 
         vectors.addAll(additionalVectors);
 
         // query both memtable index and on-disk index
-        queryVector = randomVector();
+        queryVector = randomVectorBoxed(dimension);
         resultSet = search(queryVector, limit);
-        assertDescendingScore(queryVector, getVectorsFromResult(resultSet));
+        assertDescendingScore(queryVector, getVectorsFromResult(resultSet, dimension));
 
         flush();
 
         // query multiple on-disk indexes
-        queryVector = randomVector();
+        queryVector = randomVectorBoxed(dimension);
         resultSet = search(queryVector, limit);
-        assertDescendingScore(queryVector, getVectorsFromResult(resultSet));
+        assertDescendingScore(queryVector, getVectorsFromResult(resultSet, dimension));
 
         compact();
 
         // query compacted on-disk index
-        queryVector = randomVector();
+        queryVector = randomVectorBoxed(dimension);
         resultSet = search(queryVector, limit);
-        assertDescendingScore(queryVector, getVectorsFromResult(resultSet));
+        assertDescendingScore(queryVector, getVectorsFromResult(resultSet, dimension));
     }
 
     @Test
-    public void multiSSTablesTest() throws Throwable
+    public void multiSSTablesTest()
     {
         createTable(String.format("CREATE TABLE %%s (pk int, str_val text, val vector<float, %d>, PRIMARY KEY(pk))", word2vec.dimension()));
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
@@ -194,21 +257,50 @@ public void partitionRestrictedTest()
     @Test
     public void partitionRestrictedWidePartitionTest()
     {
-        createTable(String.format("CREATE TABLE %%s (pk int, ck int, val vector<float, %d>, PRIMARY KEY(pk, ck))", word2vec.dimension()));
+        partitionRestrictedWidePartitionTest(word2vec.dimension(), 0, 1000);
+    }
+
+    @Test
+    public void partitionRestrictedWidePartitionBqCompressedTest()
+    {
+        partitionRestrictedWidePartitionTest(2048, 0, Integer.MAX_VALUE);
+    }
+
+    @Test
+    public void partitionRestrictedWidePartitionPqCompressedTest()
+    {
+        partitionRestrictedWidePartitionTest(word2vec.dimension(), 2000, Integer.MAX_VALUE);
+    }
+
+    public void partitionRestrictedWidePartitionTest(int dimension, int minvectorCount, int maxvectorCount)
+    {
+        createTable(String.format("CREATE TABLE %%s (pk int, ck int, val vector<float, %d>, PRIMARY KEY(pk, ck))", dimension));
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
         int partitions = getRandom().nextIntBetween(20, 40);
         int vectorCountPerPartition = getRandom().nextIntBetween(50, 100);
         int vectorCount = partitions * vectorCountPerPartition;
-        List<float[]> vectors = IntStream.range(0, vectorCount).mapToObj(s -> randomVector()).collect(Collectors.toList());
+
+        if (vectorCount > maxvectorCount)
+        {
+            vectorCountPerPartition = maxvectorCount / partitions;
+            vectorCount = partitions * vectorCountPerPartition;
+        }
+        else if (vectorCount < minvectorCount)
+        {
+            vectorCountPerPartition = minvectorCount / partitions;
+            vectorCount = partitions * vectorCountPerPartition;
+        }
+
+        List<Vector<Float>> vectors = IntStream.range(0, vectorCount).mapToObj(s -> randomVectorBoxed(dimension)).collect(Collectors.toList());
 
         int i = 0;
         for (int pk = 1; pk <= partitions; pk++)
         {
             for (int ck = 1; ck <= vectorCountPerPartition; ck++)
             {
-                float[] vector = vectors.get(i++);
-                execute("INSERT INTO %s (pk, ck, val) VALUES (?, ?, " + vectorString(vector) + " )", pk, ck);
+                var vector = vectors.get(i++);
+                execute("INSERT INTO %s (pk, ck, val) VALUES (?, ?, ?)", pk, ck, vector);
             }
         }
 
@@ -216,8 +308,9 @@ public void partitionRestrictedWidePartitionTest()
         for (int executionCount = 0; executionCount < 50; executionCount++)
         {
             int key = getRandom().nextIntBetween(1, partitions);
-            float[] queryVector = randomVector();
-            searchWithKey(queryVector, key, vectorCountPerPartition);
+            var queryVector = randomVectorBoxed(dimension);
+            searchWithKey(queryVector, key, vectorCountPerPartition, 1000);
+            searchWithKey(queryVector, key, 1, 1);
         }
 
         flush();
@@ -226,15 +319,16 @@ public void partitionRestrictedWidePartitionTest()
         for (int executionCount = 0; executionCount < 50; executionCount++)
         {
             int key = getRandom().nextIntBetween(1, partitions);
-            float[] queryVector = randomVector();
-            searchWithKey(queryVector, key, vectorCountPerPartition);
+            var queryVector = randomVectorBoxed(dimension);
+            searchWithKey(queryVector, key, vectorCountPerPartition, 1000);
+            searchWithKey(queryVector, key, 1, 1);
         }
 
         // query on-disk index with non-existing key:
         for (int executionCount = 0; executionCount < 50; executionCount++)
         {
             int nonExistingKey = getRandom().nextIntBetween(1, partitions) + partitions;
-            float[] queryVector = randomVector();
+            var queryVector = randomVectorBoxed(dimension);
             searchWithNonExistingKey(queryVector, nonExistingKey);
         }
     }
@@ -257,91 +351,6 @@ public void rangeRestrictedTest()
             execute("INSERT INTO %s (pk, str_val, val) VALUES (?, ?, " + vectorString(vector) + " )", pk++, word);
         }
 
-        // query memtable index
-        for (int executionCount = 0; executionCount < 50; executionCount++)
-        {
-            int key1 = getRandom().nextIntBetween(1, vectorCount * 2);
-            long token1 = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(key1)).getLongValue();
-            int key2 = getRandom().nextIntBetween(1, vectorCount * 2);
-            long token2 = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(key2)).getLongValue();
-
-            long minToken = Math.min(token1, token2);
-            long maxToken = Math.max(token1, token2);
-            List<float[]> expected = vectorsByToken.entries().stream()
-                                                  .filter(e -> e.getKey() >= minToken && e.getKey() <= maxToken)
-                                                  .map(Map.Entry::getValue)
-                                                  .collect(Collectors.toList());
-
-            float[] queryVector = word2vec.vector(word2vec.word(getRandom().nextIntBetween(0, vectorCount - 1)));
-
-            List<float[]> resultVectors = searchWithRange(queryVector, minToken, maxToken, expected.size());
-            assertDescendingScore(queryVector, resultVectors);
-
-            if (expected.isEmpty())
-                assertThat(resultVectors).isEmpty();
-            else
-            {
-                double recall = recallMatch(expected, resultVectors, expected.size());
-                assertThat(recall).isGreaterThanOrEqualTo(0.8);
-            }
-        }
-
-        flush();
-
-        // query on-disk index with existing key:
-        for (int executionCount = 0; executionCount < 50; executionCount++)
-        {
-            int key1 = getRandom().nextIntBetween(1, vectorCount * 2);
-            long token1 = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(key1)).getLongValue();
-            int key2 = getRandom().nextIntBetween(1, vectorCount * 2);
-            long token2 = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(key2)).getLongValue();
-
-            long minToken = Math.min(token1, token2);
-            long maxToken = Math.max(token1, token2);
-            List<float[]> expected = vectorsByToken.entries().stream()
-                                                   .filter(e -> e.getKey() >= minToken && e.getKey() <= maxToken)
-                                                   .map(Map.Entry::getValue)
-                                                   .collect(Collectors.toList());
-
-            float[] queryVector = word2vec.vector(word2vec.word(getRandom().nextIntBetween(0, vectorCount - 1)));
-
-            List<float[]> resultVectors = searchWithRange(queryVector, minToken, maxToken, expected.size());
-            assertDescendingScore(queryVector, resultVectors);
-
-            if (expected.isEmpty())
-                assertThat(resultVectors).isEmpty();
-            else
-            {
-                double recall = recallMatch(expected, resultVectors, expected.size());
-                assertThat(recall).isGreaterThanOrEqualTo(0.8);
-            }
-        }
-    }
-
-    @Test
-    public void rangeRestrictedWidePartitionTest()
-    {
-        createTable(String.format("CREATE TABLE %%s (pk int, ck int, str_val text, val vector<float, %d>, PRIMARY KEY(pk, ck ))", word2vec.dimension()));
-        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
-
-        int vectorCount = getRandom().nextIntBetween(500, 1000);
-
-        int pk = 0;
-        int ck = 0;
-        Multimap<Long, float[]> vectorsByToken = ArrayListMultimap.create();
-        for (int index = 0; index < vectorCount; index++)
-        {
-            String word = word2vec.word(index);
-            float[] vector = word2vec.vector(word);
-            vectorsByToken.put(Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(pk)).getLongValue(), vector);
-            execute("INSERT INTO %s (pk, ck, str_val, val) VALUES (?, ?, ?, " + vectorString(vector) + " )", pk, ck++, word);
-            if (ck == 10)
-            {
-                ck = 0;
-                pk++;
-            }
-        }
-
         // query memtable index
         for (int executionCount = 0; executionCount < 50; executionCount++)
         {
@@ -422,7 +431,8 @@ public void multipleSegmentsMultiplePostingsTest()
             for (int row = 0; row < vectorCountPerSSTable; row++)
             {
                 float[] v = word2vec.vector(word2vec.word(vectorCount++));
-                for (int j = 0; j < getRandom().nextIntBetween(1, 4); j++) {
+                for (int j = 0; j < getRandom().nextIntBetween(1, 4); j++)
+                {
                     execute("INSERT INTO %s (pk, val) VALUES (?, ?)", pk++, vector(v));
                     population.add(v);
                 }
@@ -433,8 +443,9 @@ public void multipleSegmentsMultiplePostingsTest()
 
         // create index on existing sstable to produce multiple segments
         SegmentBuilder.updateLastValidSegmentRowId(50); // 50 rows per segment
-        String index = createIndexAsync("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
-        waitForIndexQueryable(index);
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        verifyChecksum();
+
         // query multiple on-disk indexes
         var testCount = 200;
         var start = getRandom().nextIntBetween(vectorCount, word2vec.size() - testCount);
@@ -457,7 +468,7 @@ private double bruteForceRecall(float[] q, List<float[]> resultVectors, List<flo
     {
         List<float[]> expected = population
                                  .stream()
-                                 .sorted(Comparator.comparingDouble(v -> -VectorSimilarityFunction.COSINE.compare(q, v)))
+                                 .sorted(Comparator.comparingDouble(v -> -compareFloatArrays(v, q)))
                                  .limit(limit)
                                  .collect(Collectors.toList());
         return recallMatch(expected, resultVectors, limit);
@@ -465,7 +476,7 @@ private double bruteForceRecall(float[] q, List<float[]> resultVectors, List<flo
 
     // search across multiple segments, combined with a non-ann predicate
     @Test
-    public void multipleNonAnnSegmentsTest() throws Throwable
+    public void multipleNonAnnSegmentsTest()
     {
         createTable(String.format("CREATE TABLE %%s (pk int, str_val text, val vector<float, %d>, PRIMARY KEY(pk))", word2vec.dimension()));
         disableCompaction(KEYSPACE);
@@ -494,6 +505,8 @@ public void multipleNonAnnSegmentsTest() throws Throwable
         SegmentBuilder.updateLastValidSegmentRowId(50); // 50 rows per segment
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        flush();
+        verifyChecksum();
 
         // query multiple on-disk indexes
         for (String stringValue : vectorsByStringValue.keySet())
@@ -525,6 +538,13 @@ private UntypedResultSet search(float[] queryVector, int limit)
         return result;
     }
 
+    private UntypedResultSet search(Vector<Float> queryVector, int limit)
+    {
+        UntypedResultSet result = execute("SELECT * FROM %s ORDER BY val ann of ? LIMIT " + limit, queryVector);
+        assertThat(result.size()).isCloseTo(limit, Percentage.withPercentage(5));
+        return result;
+    }
+
     private List<float[]> searchWithRange(float[] queryVector, long minToken, long maxToken, int expectedSize)
     {
         UntypedResultSet result = execute("SELECT * FROM %s WHERE token(pk) <= " + maxToken + " AND token(pk) >= " + minToken + " ORDER BY val ann of " + Arrays.toString(queryVector) + " LIMIT 1000");
@@ -537,6 +557,11 @@ private void searchWithNonExistingKey(float[] queryVector, int key)
         searchWithKey(queryVector, key, 0);
     }
 
+    private void searchWithNonExistingKey(Vector<Float> queryVector, int key)
+    {
+        searchWithKey(queryVector, key, 0);
+    }
+
     private void searchWithKey(float[] queryVector, int key, int expectedSize)
     {
         UntypedResultSet result = execute("SELECT * FROM %s WHERE pk = " + key + " ORDER BY val ann of " + Arrays.toString(queryVector) + " LIMIT 1000");
@@ -549,14 +574,35 @@ private void searchWithKey(float[] queryVector, int key, int expectedSize)
         result.stream().forEach(row -> assertThat(row.getInt("pk")).isEqualTo(key));
     }
 
+    private void searchWithKey(Vector<Float> queryVector, int key, int expectedSize)
+    {
+        searchWithKey(queryVector, key, expectedSize, 1000);
+    }
+
+    private void searchWithKey(Vector<Float> queryVector, int key, int expectedSize, int limit)
+    {
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE pk = ? ORDER BY val ann of ? LIMIT " + limit, key, queryVector);
+
+        // VSTODO maybe we should have different methods for these cases
+        if (expectedSize < 10)
+            assertThat(result).hasSize(expectedSize);
+        else
+            assertThat(result.size()).isCloseTo(expectedSize, Percentage.withPercentage(10));
+        result.stream().forEach(row -> assertThat(row.getInt("pk")).isEqualTo(key));
+    }
+
     private String vectorString(float[] vector)
     {
         return Arrays.toString(vector);
     }
 
-    private float[] randomVector()
+    private void assertDescendingScore(Vector<Float> queryVector, List<float[]> resultVectors)
     {
-        return word2vec.vector(getRandom().nextIntBetween(0, word2vec.size() - 1));
+        float[] arr = new float[queryVector.size()];
+        for (int i = 0; i < queryVector.size(); i++)
+            arr[i] = queryVector.get(i);
+
+        assertDescendingScore(arr, resultVectors);
     }
 
     private void assertDescendingScore(float[] queryVector, List<float[]> resultVectors)
@@ -564,7 +610,7 @@ private void assertDescendingScore(float[] queryVector, List<float[]> resultVect
         float prevScore = -1;
         for (float[] current : resultVectors)
         {
-            float score = VectorSimilarityFunction.COSINE.compare(current, queryVector);
+            float score = compareFloatArrays(queryVector, current);
             if (prevScore >= 0)
                 assertThat(score).isLessThanOrEqualTo(prevScore);
 
@@ -572,10 +618,20 @@ private void assertDescendingScore(float[] queryVector, List<float[]> resultVect
         }
     }
 
+    private static float compareFloatArrays(float[] queryVector, float[] current)
+    {
+        return VectorSimilarityFunction.COSINE.compare(vts.createFloatVector(current), vts.createFloatVector(queryVector));
+    }
+
     private List<float[]> getVectorsFromResult(UntypedResultSet result)
+    {
+        return getVectorsFromResult(result, word2vec.dimension());
+    }
+
+    private List<float[]> getVectorsFromResult(UntypedResultSet result, int dimension)
     {
         List<float[]> vectors = new ArrayList<>();
-        VectorType<?> vectorType = VectorType.getInstance(FloatType.instance, word2vec.dimension());
+        VectorType<?> vectorType = VectorType.getInstance(FloatType.instance, dimension);
 
         // verify results are part of inserted vectors
         for (UntypedResultSet.Row row: result)
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorMetricsTest.java
new file mode 100644
index 000000000000..292509c113eb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorMetricsTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Collection;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.vector.CassandraOnHeapGraph;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class VectorMetricsTest extends VectorTester
+{
+    @Parameterized.Parameter
+    public Version version;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static Collection<Object[]> data()
+    {
+        return Stream.of(Version.CA, Version.DC).map(v -> new Object[]{ v}).collect(Collectors.toList());
+    }
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        VectorTester.setUpClass();
+    }
+
+    @Before
+    @Override
+    public void setup() throws Throwable
+    {
+        super.setup();
+        SAIUtil.setLatestVersion(version);
+    }
+
+    @Test
+    public void testBasicColumnQueryMetricsVectorIndexMetrics()
+    {
+        createTable("CREATE TABLE %s (pk int, val vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        disableCompaction();
+
+        // Get the metrics
+        var index = getCurrentColumnFamilyStore().indexManager.listIndexes().iterator().next();
+        assert index instanceof StorageAttachedIndex;
+        var saiIndex = (StorageAttachedIndex) index;
+        var vectorMetrics = (ColumnQueryMetrics.VectorIndexMetrics) saiIndex.getIndexContext().getColumnQueryMetrics();
+
+        // Expect all metrics to be 0
+        assertEquals(0, vectorMetrics.annNodesVisited.getCount());
+        assertEquals(0, vectorMetrics.annNodesReranked.getCount());
+        assertEquals(0, vectorMetrics.annNodesExpanded.getCount());
+        assertEquals(0, vectorMetrics.annNodesExpandedBaseLayer.getCount());
+        assertEquals(0, vectorMetrics.annGraphSearches.getCount());
+        assertEquals(0, vectorMetrics.annGraphResumes.getCount());
+        assertEquals(0, vectorMetrics.bruteForceNodesVisited.getCount());
+        assertEquals(0, vectorMetrics.bruteForceNodesReranked.getCount());
+        assertEquals(0, vectorMetrics.quantizationMemoryBytes.sum());
+        assertEquals(0, vectorMetrics.ordinalsMapMemoryBytes.sum());
+        assertEquals(0, vectorMetrics.onDiskGraphsCount.sum());
+        assertEquals(0, vectorMetrics.onDiskGraphVectorsCount.sum());
+        assertEquals(0, vectorMetrics.annGraphSearchLatency.getCount());
+
+        // Insert 10 rows to simplify some of the assertions
+        for (int i = 0; i < CassandraOnHeapGraph.MIN_PQ_ROWS; i++)
+            execute("INSERT INTO %s (pk, val) VALUES (?, ?)", i, vector(1 + i, 2 + i, 3 + i));
+        flush();
+
+        assertTrue(vectorMetrics.quantizationMemoryBytes.sum() > 0);
+        assertEquals(0, vectorMetrics.ordinalsMapMemoryBytes.sum()); // unique vectors means no cache required
+        assertEquals(1, vectorMetrics.onDiskGraphsCount.sum());
+        assertEquals( CassandraOnHeapGraph.MIN_PQ_ROWS, vectorMetrics.onDiskGraphVectorsCount.sum());
+
+        // Compaction should not impact metrics
+        compact();
+        assertTrue(vectorMetrics.quantizationMemoryBytes.sum() > 0);
+        assertEquals(0, vectorMetrics.ordinalsMapMemoryBytes.sum()); // unique vectors means no cache required
+        assertEquals(1, vectorMetrics.onDiskGraphsCount.sum());
+        assertEquals( CassandraOnHeapGraph.MIN_PQ_ROWS, vectorMetrics.onDiskGraphVectorsCount.sum());
+
+        // Now run a pure ann query and verify we hit the ann graph
+        var result = execute("SELECT pk FROM %s ORDER BY val ANN OF [2.5, 3.5, 4.5] LIMIT 2");
+        assertThat(result).hasSize(2);
+
+        assertTrue(vectorMetrics.annNodesVisited.getCount() > 0);
+        assertTrue(vectorMetrics.annNodesReranked.getCount() > 0);
+        assertTrue(vectorMetrics.annNodesExpanded.getCount() > 0);
+        assertTrue(vectorMetrics.annNodesExpandedBaseLayer.getCount() > 0);
+        assertEquals(1, vectorMetrics.annGraphSearches.getCount());
+        // We shouldn't need to resume when searching in this scenario
+        assertEquals(0, vectorMetrics.annGraphResumes.getCount());
+        assertEquals(1, vectorMetrics.annGraphSearchLatency.getCount());
+
+        // Now do a hybrid query that is guaranteed to use brute force. (We have to override this because
+        // the super class sets it with a random value that can lead to graph search.)
+        setMaxBruteForceRows(100);
+        result = execute("SELECT pk FROM %s WHERE pk = 0 ORDER BY val ANN OF [2.5, 3.5, 4.5] LIMIT 2");
+        assertThat(result).hasSize(1);
+
+        // We don't do approximate scores if there are few enough rows
+        assertEquals(0, vectorMetrics.bruteForceNodesVisited.getCount());
+        assertEquals(1, vectorMetrics.bruteForceNodesReranked.getCount());
+        assertEquals(1, vectorMetrics.annGraphSearchLatency.getCount());
+
+        // Confirm that truncating the table which will remove the index also drops the gauges
+        truncate(false);
+        assertEquals(0, vectorMetrics.quantizationMemoryBytes.sum());
+        assertEquals(0, vectorMetrics.ordinalsMapMemoryBytes.sum());
+        assertEquals(0, vectorMetrics.onDiskGraphsCount.sum());
+        assertEquals(0, vectorMetrics.onDiskGraphVectorsCount.sum());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorRangeSearchTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorRangeSearchTest.java
new file mode 100644
index 000000000000..eefe9df1c552
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorRangeSearchTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.plan.QueryController;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class VectorRangeSearchTest extends VectorTester.VersionedWithChecksums
+{
+    private static final IPartitioner partitioner = Murmur3Partitioner.instance;
+
+    @Test
+    public void rangeSearchTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (partition int, val vector<float, 2>, PRIMARY KEY(partition))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+
+        var nPartitions = 100;
+        Map<Integer, float[]> vectorsByKey = new HashMap<>();
+
+        for (int i = 1; i <= nPartitions; i++)
+        {
+            float[] vector = {(float) i, (float) i};
+            execute("INSERT INTO %s (partition, val) VALUES (?, ?)", i, vector(vector));
+            vectorsByKey.put(i, vector);
+        }
+
+        var queryVector = vector(1.5f, 1.5f);
+        CheckedFunction tester = () -> {
+            for (int i = 1; i <= nPartitions; i++)
+            {
+                UntypedResultSet result = execute("SELECT partition FROM %s WHERE token(partition) > token(?) ORDER BY val ann of ? LIMIT 1000", i, queryVector);
+                assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysWithLowerBound(vectorsByKey.keySet(), i, false));
+
+                result = execute("SELECT partition FROM %s WHERE token(partition) >= token(?) ORDER BY val ann of ? LIMIT 1000", i, queryVector);
+                assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysWithLowerBound(vectorsByKey.keySet(), i, true));
+
+                result = execute("SELECT partition FROM %s WHERE token(partition) < token(?) ORDER BY val ann of ? LIMIT 1000", i, queryVector);
+                assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysWithUpperBound(vectorsByKey.keySet(), i, false));
+
+                result = execute("SELECT partition FROM %s WHERE token(partition) <= token(?) ORDER BY val ann of ? LIMIT 1000", i, queryVector);
+                assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysWithUpperBound(vectorsByKey.keySet(), i, true));
+
+                for (int j = 1; j <= nPartitions; j++)
+                {
+                    result = execute("SELECT partition FROM %s WHERE token(partition) >= token(?) AND token(partition) <= token(?) ORDER BY val ann of ? LIMIT 1000", i, j, queryVector);
+                    assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysInBounds(vectorsByKey.keySet(), i, true, j, true));
+
+                    result = execute("SELECT partition FROM %s WHERE token(partition) > token(?) AND token(partition) <= token(?) ORDER BY val ann of ? LIMIT 1000", i, j, queryVector);
+                    assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysInBounds(vectorsByKey.keySet(), i, false, j, true));
+
+                    result = execute("SELECT partition FROM %s WHERE token(partition) >= token(?) AND token(partition) < token(?) ORDER BY val ann of ? LIMIT 1000", i, j, queryVector);
+                    assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysInBounds(vectorsByKey.keySet(), i, true, j, false));
+
+                    result = execute("SELECT partition FROM %s WHERE token(partition) > token(?) AND token(partition) < token(?) ORDER BY val ann of ? LIMIT 1000", i, j, queryVector);
+                    assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysInBounds(vectorsByKey.keySet(), i, false, j, false));
+                }
+            }
+        };
+
+        tester.apply();
+
+        flush();
+
+        tester.apply();
+    }
+
+    @Test
+    public void testPartitionKeyRestrictionCombinedWithSearchPredicate() throws Throwable
+    {
+        // Need to test the search then order path
+        QueryController.QUERY_OPT_LEVEL = 0;
+
+        // We use a clustered primary key to simplify the mental model for this test.
+        // The bug this test exposed happens when the last row(s) in a segment, based on PK order, are present
+        // in a peer index for an sstable's search index but not its vector index.
+        createTable("CREATE TABLE %s (partition int, i int, v vector<float, 2>, c int, PRIMARY KEY(partition, i))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function': 'euclidean'}");
+        createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
+
+        var partitionKeys = new ArrayList<Integer>();
+        // Insert many rows
+        for (int i = 1; i < 1000; i++)
+        {
+            execute("INSERT INTO %s (partition, i, v, c) VALUES (?, ?, ?, ?)", i, i, vector(i, i), i);
+            partitionKeys.add(i);
+        }
+
+        beforeAndAfterFlush(() -> {
+            // Restricted by partition key and with low as well as high cardinality of results for column c
+            assertRows(execute("SELECT i FROM %s WHERE partition = 1 AND c > 0 ORDER BY v ANN OF [1,1] LIMIT 1"), row(1));
+            assertRows(execute("SELECT i FROM %s WHERE partition = 1 AND c < 10 ORDER BY v ANN OF [1,1] LIMIT 1"), row(1));
+
+            // Do some partition key range queries, the restriction on c is meaningless, but forces the search then
+            // order path
+            var r1 = execute("SELECT partition FROM %s WHERE token(partition) < token(11) AND c > 0 ORDER BY v ANN OF [1,1] LIMIT 1000");
+            var e1 = keysWithUpperBound(partitionKeys, 11,false);
+            assertThat(keys(r1)).containsExactlyInAnyOrderElementsOf(e1);
+
+            var r2 = execute("SELECT partition FROM %s WHERE token(partition) >= token(11) AND token(partition) <= token(20) AND c <= 1000 ORDER BY v ANN OF [1,1] LIMIT 1000");
+            var e2 = keysInBounds(partitionKeys, 11, true, 20, true);
+            assertThat(keys(r2)).containsExactlyInAnyOrderElementsOf(e2);
+        });
+    }
+
+    private Collection<Integer> keys(UntypedResultSet result)
+    {
+        List<Integer> keys = new ArrayList<>(result.size());
+        for (UntypedResultSet.Row row : result)
+            keys.add(row.getInt("partition"));
+        return keys;
+    }
+
+    private Collection<Integer> keysWithLowerBound(Collection<Integer> keys, int leftKey, boolean leftInclusive)
+    {
+        return keysInTokenRange(keys, partitioner.getToken(Int32Type.instance.decompose(leftKey)), leftInclusive,
+                                partitioner.getMaximumToken().getToken(), true);
+    }
+
+    private Collection<Integer> keysWithUpperBound(Collection<Integer> keys, int rightKey, boolean rightInclusive)
+    {
+        return keysInTokenRange(keys, partitioner.getMinimumToken().getToken(), true,
+                                partitioner.getToken(Int32Type.instance.decompose(rightKey)), rightInclusive);
+    }
+
+    private Collection<Integer> keysInBounds(Collection<Integer> keys, int leftKey, boolean leftInclusive, int rightKey, boolean rightInclusive)
+    {
+        return keysInTokenRange(keys, partitioner.getToken(Int32Type.instance.decompose(leftKey)), leftInclusive,
+                                partitioner.getToken(Int32Type.instance.decompose(rightKey)), rightInclusive);
+    }
+
+    private Collection<Integer> keysInTokenRange(Collection<Integer> keys, Token leftToken, boolean leftInclusive, Token rightToken, boolean rightInclusive)
+    {
+        long left = leftToken.getLongValue();
+        long right = rightToken.getLongValue();
+        return keys.stream()
+                   .filter(k -> {
+                       long t = partitioner.getToken(Int32Type.instance.decompose(k)).getLongValue();
+                       return (left < t || left == t && leftInclusive) && (t < right || t == right && rightInclusive);
+                   }).collect(Collectors.toSet());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorSegmentationTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorSegmentationTest.java
index 52cf43d07f78..45c1ea788528 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/VectorSegmentationTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorSegmentationTest.java
@@ -23,25 +23,29 @@
 
 import org.junit.Test;
 
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.marshal.FloatType;
 import org.apache.cassandra.db.marshal.VectorType;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class VectorSegmentationTest extends VectorTester
 {
+    private static final int dimension = 100;
+    private double MIN_ACCEPTABLE_RECALL = 0.96;
+
     @Test
-    public void testMultipleSegmentsForCreatingIndex() throws Throwable
+    public void testMultipleSegmentsForCreatingIndex()
     {
-        createTable("CREATE TABLE %s (pk int, val vector<float, " + word2vec.dimension() + ">, PRIMARY KEY(pk))");
+        createTable("CREATE TABLE %s (pk int, val vector<float, " + dimension + ">, PRIMARY KEY(pk))");
 
         int vectorCount = 100;
         List<float[]> vectors = new ArrayList<>();
         for (int row = 0; row < vectorCount; row++)
         {
-            float[] vector = word2vec.vector(row);
+            float[] vector = randomVector();
             vectors.add(vector);
             execute("INSERT INTO %s (pk, val) VALUES (?, ?)", row, vector(vector));
         }
@@ -52,23 +56,22 @@ public void testMultipleSegmentsForCreatingIndex() throws Throwable
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
         int limit = 35;
-        float[] queryVector = word2vec.vector(getRandom().nextIntBetween(0, vectorCount));
+        float[] queryVector = randomVector();
         UntypedResultSet resultSet = execute("SELECT * FROM %s ORDER BY val ANN OF ? LIMIT " + limit, vector(queryVector));
         assertThat(resultSet.size()).isEqualTo(limit);
 
         List<float[]> resultVectors = getVectorsFromResult(resultSet);
         double recall = rawIndexedRecall(vectors, queryVector, resultVectors, limit);
-        assertThat(recall).isGreaterThanOrEqualTo(0.9);
+        assertThat(recall).isGreaterThanOrEqualTo(MIN_ACCEPTABLE_RECALL);
     }
 
     @Test
-    public void testMultipleSegmentsForCompaction() throws Throwable
+    public void testMultipleSegmentsForCompaction()
     {
-        createTable("CREATE TABLE %s (pk int, val vector<float, " + word2vec.dimension() + ">, PRIMARY KEY(pk))");
+        createTable("CREATE TABLE %s (pk int, val vector<float, " + dimension + ">, PRIMARY KEY(pk))");
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
         List<float[]> vectors = new ArrayList<>();
-        int vectorCount = 0;
         int rowsPerSSTable = 10;
         int sstables = 5;
         int pk = 0;
@@ -76,7 +79,7 @@ public void testMultipleSegmentsForCompaction() throws Throwable
         {
             for (int row = 0; row < rowsPerSSTable; row++)
             {
-                float[] vector = word2vec.vector(vectorCount++);
+                float[] vector = randomVector();
                 execute("INSERT INTO %s (pk, val) VALUES (?, ?)", pk++, vector(vector));
                 vectors.add(vector);
             }
@@ -85,31 +88,36 @@ public void testMultipleSegmentsForCompaction() throws Throwable
         }
 
         int limit = 30;
-        float[] queryVector = word2vec.vector(getRandom().nextIntBetween(0, vectorCount));
+        float[] queryVector = randomVector();
         UntypedResultSet resultSet = execute("SELECT * FROM %s ORDER BY val ANN OF ? LIMIT " + limit, vector(queryVector));
         assertThat(resultSet.size()).isEqualTo(limit);
 
         List<float[]> resultVectors = getVectorsFromResult(resultSet);
         double recall = rawIndexedRecall(vectors, queryVector, resultVectors, limit);
-        assertThat(recall).isGreaterThanOrEqualTo(0.99);
+        assertThat(recall).isGreaterThanOrEqualTo(MIN_ACCEPTABLE_RECALL);
 
 
         SegmentBuilder.updateLastValidSegmentRowId(11); // 11 rows per segment
         compact();
 
-        queryVector = word2vec.vector(getRandom().nextIntBetween(0, vectorCount));
+        queryVector = randomVector();
         resultSet = execute("SELECT * FROM %s ORDER BY val ANN OF ? LIMIT " + limit, vector(queryVector));
         assertThat(resultSet.size()).isEqualTo(limit);
 
         resultVectors = getVectorsFromResult(resultSet);
         recall = rawIndexedRecall(vectors, queryVector, resultVectors, limit);
-        assertThat(recall).isGreaterThanOrEqualTo(0.99);
+        assertThat(recall).isGreaterThanOrEqualTo(MIN_ACCEPTABLE_RECALL);
+    }
+
+    private float[] randomVector()
+    {
+        return CQLTester.randomVector(dimension);
     }
 
-    private static List<float[]> getVectorsFromResult(UntypedResultSet result)
+    private List<float[]> getVectorsFromResult(UntypedResultSet result)
     {
         List<float[]> vectors = new ArrayList<>();
-        VectorType<?> vectorType = VectorType.getInstance(FloatType.instance, word2vec.dimension());
+        VectorType<?> vectorType = VectorType.getInstance(FloatType.instance, dimension);
 
         // verify results are part of inserted vectors
         for (UntypedResultSet.Row row: result)
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorSiftSmallTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorSiftSmallTest.java
index 3cc146ecf6fe..c66f6889a7a3 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/VectorSiftSmallTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorSiftSmallTest.java
@@ -28,36 +28,150 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.IntStream;
 
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 public class VectorSiftSmallTest extends VectorTester
 {
+    private static final String DATASET = "siftsmall"; // change to "sift" for larger dataset. requires manual download
+
+    @Override
+    public void setup() throws Throwable
+    {
+        super.setup();
+    }
+
     @Test
     public void testSiftSmall() throws Throwable
     {
-        var siftName = "siftsmall";
-        var baseVectors = readFvecs(String.format("test/data/%s/%s_base.fvecs", siftName, siftName));
-        var queryVectors = readFvecs(String.format("test/data/%s/%s_query.fvecs", siftName, siftName));
-        var groundTruth = readIvecs(String.format("test/data/%s/%s_groundtruth.ivecs", siftName, siftName));
+        var baseVectors = readFvecs(String.format("test/data/%s/%s_base.fvecs", DATASET, DATASET));
+        var queryVectors = readFvecs(String.format("test/data/%s/%s_query.fvecs", DATASET, DATASET));
+        var groundTruth = readIvecs(String.format("test/data/%s/%s_groundtruth.ivecs", DATASET, DATASET));
 
         // Create table and index
-        createTable("CREATE TABLE %s (pk int, val vector<float, 128>, PRIMARY KEY(pk))");
-        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createTable();
+        createIndex();
 
-        insertVectors(baseVectors);
-        double memoryRecall = testRecall(queryVectors, groundTruth);
+        insertVectors(baseVectors, 0);
+        double memoryRecall = testRecall(100, queryVectors, groundTruth);
         assertTrue("Memory recall is " + memoryRecall, memoryRecall > 0.975);
 
+        // Run a few queries with increasing rerank_k to validate that recall increases
+        ensureIncreasingRerankKIncreasesRecall(queryVectors, groundTruth);
+
         flush();
-        var diskRecall = testRecall(queryVectors, groundTruth);
-        assertTrue("Disk recall is " + diskRecall, diskRecall > 0.95);
+        var diskRecall = testRecall(100, queryVectors, groundTruth);
+        assertTrue("Disk recall is " + diskRecall, diskRecall > 0.975);
+
+        // Run a few queries with increasing rerank_k to validate that recall increases
+        ensureIncreasingRerankKIncreasesRecall(queryVectors, groundTruth);
+    }
+
+    private void ensureIncreasingRerankKIncreasesRecall(List<float[]> queryVectors, List<List<Integer>> groundTruth)
+    {
+        // Validate that the recall increases as we increase the rerank_k parameter
+        double previousRecall = 0;
+        int limit = 10;
+        int strictlyIncreasedCount = 0;
+        // Testing shows that we acheive 100% recall at about rerank_k = 45, so no need to go higher
+        for (int rerankK = limit; rerankK <= 50; rerankK += 5)
+        {
+            var recall = testRecall(limit, queryVectors, groundTruth, rerankK);
+            // Recall varies, so we can only assert that it does not get worse on a per-run basis. However, it should
+            // get better strictly at least some of the time
+            assertTrue("Recall for rerank_k = " + rerankK + " is " + recall, recall >= previousRecall);
+            if (recall > previousRecall)
+                strictlyIncreasedCount++;
+            previousRecall = recall;
+        }
+        // This is a conservative assertion to prevent it from being too fragile. At the time of writing this test,
+        // we observed a strict increase of 6 times for in memory and 5 times for on disk.
+        assertTrue("Recall should have strictly increased at least 4 times but only increased " + strictlyIncreasedCount + " times",
+                   strictlyIncreasedCount > 3);
+    }
+
+    @Test
+    public void testCompaction() throws Throwable
+    {
+        var baseVectors = readFvecs(String.format("test/data/%s/%s_base.fvecs", DATASET, DATASET));
+        var queryVectors = readFvecs(String.format("test/data/%s/%s_query.fvecs", DATASET, DATASET));
+        var groundTruth = readIvecs(String.format("test/data/%s/%s_groundtruth.ivecs", DATASET, DATASET));
+
+        // Create table and index
+        createTable();
+        createIndex();
+
+        // we're going to compact manually, so disable background compactions to avoid interference
+        disableCompaction();
+
+        int segments = 10;
+        int vectorsPerSegment = baseVectors.size() / segments;
+        assert baseVectors.size() % vectorsPerSegment == 0; // simplifies split logic
+        for (int i = 0; i < segments; i++)
+        {
+            insertVectors(baseVectors.subList(i * vectorsPerSegment, (i + 1) * vectorsPerSegment), i * vectorsPerSegment);
+            flush();
+        }
+        for (int topK : List.of(1, 100))
+        {
+            double recall = testRecall(topK, queryVectors, groundTruth);
+            assertTrue("Pre-compaction recall is " + recall, recall > 0.975);
+        }
+
+        compact();
+        for (int topK : List.of(1, 100))
+        {
+            var recall = testRecall(topK, queryVectors, groundTruth);
+            assertTrue("Post-compaction recall is " + recall, recall > 0.975);
+        }
+    }
+
+    // exercise the path where we use the PQ from the first segment (constructed on-heap)
+    // to construct the others off-heap
+    @Test
+    public void testMultiSegmentBuild() throws Throwable
+    {
+        var baseVectors = readFvecs(String.format("test/data/%s/%s_base.fvecs", DATASET, DATASET));
+        var queryVectors = readFvecs(String.format("test/data/%s/%s_query.fvecs", DATASET, DATASET));
+        var groundTruth = readIvecs(String.format("test/data/%s/%s_groundtruth.ivecs", DATASET, DATASET));
+
+        // Create table without index
+        createTable();
+
+        // we're going to compact manually, so disable background compactions to avoid interference
+        disableCompaction();
+
+        insertVectors(baseVectors, 0);
+        // single big sstable before creating index
+        flush();
+        compact();
+
+        SegmentBuilder.updateLastValidSegmentRowId(2000); // 2000 rows per segment, enough for PQ to be created
+        createIndex();
+
+        // verify that we got the expected number of segments and that PQ is present in all of them
+        var sim = getCurrentColumnFamilyStore().getIndexManager();
+        var index = (StorageAttachedIndex) sim.listIndexes().iterator().next();
+        var searchableIndex = index.getIndexContext().getView().getIndexes().iterator().next();
+        var segments = searchableIndex.getSegments();
+        assertEquals(5, segments.size());
+        for (int i = 0; i < 5; i++)
+            assertNotNull(((V2VectorIndexSearcher) segments.get(0).getIndexSearcher()).getPQ());
+
+        var recall = testRecall(100, queryVectors, groundTruth);
+        assertTrue("Post-compaction recall is " + recall, recall > 0.975);
     }
 
     public static ArrayList<float[]> readFvecs(String filePath) throws IOException
@@ -84,16 +198,16 @@ public static ArrayList<float[]> readFvecs(String filePath) throws IOException
         return vectors;
     }
 
-    private static ArrayList<HashSet<Integer>> readIvecs(String filename)
+    private static ArrayList<List<Integer>> readIvecs(String filename)
     {
-        var groundTruthTopK = new ArrayList<HashSet<Integer>>();
+        var groundTruthTopK = new ArrayList<List<Integer>>();
 
         try (var dis = new DataInputStream(new FileInputStream(filename)))
         {
             while (dis.available() > 0)
             {
                 var numNeighbors = Integer.reverseBytes(dis.readInt());
-                var neighbors = new HashSet<Integer>(numNeighbors);
+                List<Integer> neighbors = new ArrayList<>(numNeighbors);
 
                 for (var i = 0; i < numNeighbors; i++)
                 {
@@ -112,10 +226,14 @@ private static ArrayList<HashSet<Integer>> readIvecs(String filename)
         return groundTruthTopK;
     }
 
-    public double testRecall(List<float[]> queryVectors, List<HashSet<Integer>> groundTruth)
+    public double testRecall(int topK, List<float[]> queryVectors, List<List<Integer>> groundTruth)
+    {
+        return testRecall(topK, queryVectors, groundTruth, null);
+    }
+
+    public double testRecall(int topK, List<float[]> queryVectors, List<List<Integer>> groundTruth, Integer rerankK)
     {
         AtomicInteger topKfound = new AtomicInteger(0);
-        int topK = 100;
 
         // Perform query and compute recall
         var stream = IntStream.range(0, queryVectors.size()).parallel();
@@ -123,16 +241,20 @@ public double testRecall(List<float[]> queryVectors, List<HashSet<Integer>> grou
             float[] queryVector = queryVectors.get(i);
             String queryVectorAsString = Arrays.toString(queryVector);
 
-            try
-            {
-                UntypedResultSet result = execute("SELECT pk FROM %s ORDER BY val ANN OF " + queryVectorAsString + " LIMIT " + topK);
+            try {
+                String query = "SELECT pk FROM %s ORDER BY val ANN OF " + queryVectorAsString + " LIMIT " + topK;
+                if (rerankK != null)
+                    query += " with ann_options = {'rerank_k': " + rerankK + '}';
+
+                UntypedResultSet result = execute(query);
                 var gt = groundTruth.get(i);
+                assert topK <= gt.size();
+                // we don't care about order within the topK but we do need to restrict the size first
+                var gtSet = new HashSet<>(gt.subList(0, topK));
 
-                int n = (int)result.stream().filter(row -> gt.contains(row.getInt("pk"))).count();
+                int n = (int)result.stream().filter(row -> gtSet.contains(row.getInt("pk"))).count();
                 topKfound.addAndGet(n);
-            }
-            catch (Throwable throwable)
-            {
+            } catch (Throwable throwable) {
                 throw new RuntimeException(throwable);
             }
         });
@@ -140,17 +262,25 @@ public double testRecall(List<float[]> queryVectors, List<HashSet<Integer>> grou
         return (double) topKfound.get() / (queryVectors.size() * topK);
     }
 
-    private void insertVectors(List<float[]> baseVectors)
+    private void createTable()
     {
-        IntStream.range(0, baseVectors.size()).parallel().forEach(i -> {
-            float[] arrayVector = baseVectors.get(i);
-            String vectorAsString = Arrays.toString(arrayVector);
-            try
-            {
-                execute("INSERT INTO %s " + String.format("(pk, val) VALUES (%d, %s)", i, vectorAsString));
-            }
-            catch (Throwable throwable)
-            {
+        createTable("CREATE TABLE %s (pk int, val vector<float, 128>, PRIMARY KEY(pk))");
+    }
+
+    private void createIndex()
+    {
+        // we need a long timeout because we are adding many vectors
+        String index = createIndexAsync("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        waitForIndexQueryable(KEYSPACE, index, 5, TimeUnit.MINUTES);
+    }
+
+    private void insertVectors(List<float[]> vectors, int baseRowId)
+    {
+        IntStream.range(0, vectors.size()).parallel().forEach(i -> {
+            float[] arrayVector = vectors.get(i);
+            try {
+                execute("INSERT INTO %s (pk, val) VALUES (?, ?)", baseRowId + i, vector(arrayVector));
+            } catch (Throwable throwable) {
                 throw new RuntimeException(throwable);
             }
         });
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorTester.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorTester.java
index 564cd72a6e70..616b26368715 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/VectorTester.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorTester.java
@@ -18,34 +18,42 @@
 
 package org.apache.cassandra.index.sai.cql;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import io.github.jbellis.jvector.graph.GraphIndexBuilder;
 import io.github.jbellis.jvector.graph.GraphSearcher;
-import io.github.jbellis.jvector.vector.VectorEncoding;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.vector.ArrayVectorFloat;
 import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.v1.vector.ConcurrentVectorValues;
-import org.apache.cassandra.index.sai.utils.Glove;
-import org.apache.cassandra.inject.ActionBuilder;
-import org.apache.cassandra.inject.Injections;
-import org.apache.cassandra.inject.InvokePointBuilder;
-import org.junit.Before;
-import org.junit.BeforeClass;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter;
+import org.apache.cassandra.index.sai.disk.vector.ConcurrentVectorValues;
+import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex;
+
+import static org.assertj.core.api.Assertions.assertThat;
 
 public class VectorTester extends SAITester
 {
-    protected static Glove.WordVector word2vec;
-
-    @BeforeClass
-    public static void loadModel() throws Throwable
-    {
-        word2vec = Glove.parse(VectorTester.class.getClassLoader().getResourceAsStream("glove.3K.50d.txt"));
-    }
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
 
     @Before
     public void setup() throws Throwable
@@ -53,55 +61,49 @@ public void setup() throws Throwable
         // override maxBruteForceRows to a random number between 0 and 4 so that we make sure
         // the non-brute-force path gets called during tests (which mostly involve small numbers of rows)
         var n = getRandom().nextIntBetween(0, 4);
-        var limitToTopResults = InvokePointBuilder.newInvokePoint()
-                                                  .onClass("org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher")
-                                                  .onMethod("limitToTopResults")
-                                                  .atEntry();
-        var bitsOrPostingListForKeyRange = InvokePointBuilder.newInvokePoint()
-                                                             .onClass("org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher")
-                                                             .onMethod("bitsOrPostingListForKeyRange")
-                                                             .atEntry();
-        var ab = ActionBuilder.newActionBuilder()
-                              .actions()
-                              .doAction("$this.globalBruteForceRows = " + n);
-        var changeBruteForceThreshold = Injections.newCustom("force_non_bruteforce_queries")
-                                                  .add(limitToTopResults)
-                                                  .add(bitsOrPostingListForKeyRange)
-                                                  .add(ab)
-                                                  .build();
-        Injections.inject(changeBruteForceThreshold);
+        setMaxBruteForceRows(n);
+        // override the global holes allowed so that the one-to-many path gets exercised
+        V5VectorPostingsWriter.GLOBAL_HOLES_ALLOWED = 1.0;
     }
 
-    public static double rawIndexedRecall(Collection<float[]> vectors, float[] query, List<float[]> result, int topK) throws IOException
+    public static void setMaxBruteForceRows(int n)
     {
-        ConcurrentVectorValues vectorValues = new ConcurrentVectorValues(query.length);
+        V2VectorIndexSearcher.GLOBAL_BRUTE_FORCE_ROWS = n;
+        V2VectorIndexSearcher.BRUTE_FORCE_EXPENSE_FACTOR = 1.0;
+        VectorMemtableIndex.GLOBAL_BRUTE_FORCE_ROWS = n;
+    }
+
+    public static double rawIndexedRecall(Collection<float[]> rawVectors, float[] rawQuery, List<float[]> result, int topK)
+    {
+        ConcurrentVectorValues vectorValues = new ConcurrentVectorValues(rawQuery.length);
+        var q = vts.createFloatVector(rawQuery);
         int ordinal = 0;
 
-        var graphBuilder = new GraphIndexBuilder<>(vectorValues,
-                                                   VectorEncoding.FLOAT32,
-                                                   VectorSimilarityFunction.COSINE,
-                                                   16,
-                                                   100,
-                                                   1.2f,
-                                                   1.4f);
+        var graphBuilder = new GraphIndexBuilder(vectorValues,
+                                                 VectorSimilarityFunction.COSINE,
+                                                 16,
+                                                 100,
+                                                 1.2f,
+                                                 1.4f,
+                                                 false);
 
-        for (float[] vector : vectors)
+        for (float[] raw : rawVectors)
         {
-            vectorValues.add(ordinal, vector);
-            graphBuilder.addGraphNode(ordinal++, vectorValues);
+            var v = vts.createFloatVector(raw);
+            vectorValues.add(ordinal, v);
+            graphBuilder.addGraphNode(ordinal++, v);
         }
 
-        var results = GraphSearcher.search(query,
+        var results = GraphSearcher.search(q,
                                            topK,
                                            vectorValues,
-                                           VectorEncoding.FLOAT32,
                                            VectorSimilarityFunction.COSINE,
                                            graphBuilder.getGraph(),
-                                           null);
+                                           Bits.ALL);
 
-        List<float[]> nearestNeighbors = new ArrayList<>();
+        var nearestNeighbors = new ArrayList<float[]>();
         for (var ns : results.getNodes())
-            nearestNeighbors.add(vectorValues.vectorValue(ns.node));
+            nearestNeighbors.add(((ArrayVectorFloat) vectorValues.getVector(ns.node)).get());
 
         return recallMatch(nearestNeighbors, result, topK);
     }
@@ -126,4 +128,93 @@ public static double recallMatch(List<float[]> expected, List<float[]> actual, i
 
         return (double) matches / topK;
     }
+
+    protected void verifyChecksum() {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        cfs.indexManager.listIndexes().stream().forEach(index -> {
+            try
+            {
+                var indexContext = (IndexContext) FieldUtils
+                                                  .getDeclaredField(index.getClass(), "indexContext", true)
+                                                  .get(index);
+                logger.info("Verifying checksum for index {}", index.getIndexMetadata().name);
+                boolean checksumValid = verifyChecksum(indexContext);
+                assertThat(checksumValid).isTrue();
+            } catch (IllegalAccessException e)
+            {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    public static double computeRecall(List<float[]> vectors, float[] query, List<float[]> result, VectorSimilarityFunction vsf)
+    {
+        List<float[]> sortedVectors = new ArrayList<>(vectors);
+        sortedVectors.sort((a, b) -> Double.compare(vsf.compare(vts.createFloatVector(b), vts.createFloatVector(query)),
+                                                    vsf.compare(vts.createFloatVector(a), vts.createFloatVector(query))));
+
+        assertThat(sortedVectors).containsAll(result);
+
+        List<float[]> nearestNeighbors = sortedVectors.subList(0, result.size());
+
+        int matches = 0;
+        for (float[] in : nearestNeighbors)
+        {
+            for (float[] out : result)
+            {
+                if (Arrays.compare(in, out) == 0)
+                {
+                    matches++;
+                    break;
+                }
+            }
+        }
+
+        return matches * 1.0 / result.size();
+    }
+
+    /**
+     * {@link VectorTester} parameterized for {@link Version#CA} and {@link Version#DC}.
+     */
+    @Ignore
+    @RunWith(Parameterized.class)
+    abstract static class Versioned extends VectorTester
+    {
+        @Parameterized.Parameter
+        public Version version;
+
+        @Parameterized.Parameters(name = "{0}")
+        public static Collection<Object[]> data()
+        {
+            return Stream.of(Version.CA, Version.DC).map(v -> new Object[]{ v }).collect(Collectors.toList());
+        }
+
+        @Before
+        @Override
+        public void setup() throws Throwable
+        {
+            super.setup();
+            SAIUtil.setLatestVersion(version);
+        }
+    }
+
+    /**
+     * {@link Versioned} that verifies checksums on flushing and compaction.
+     */
+    abstract static class VersionedWithChecksums extends Versioned
+    {
+        @Override
+        public void flush()
+        {
+            super.flush();
+            verifyChecksum();
+        }
+
+        @Override
+        public void compact()
+        {
+            super.compact();
+            verifyChecksum();
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorTracingTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorTracingTest.java
new file mode 100644
index 000000000000..56e934d9571b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorTracingTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.tracing.TracingTestImpl;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_TRACING_CLASS;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class VectorTracingTest extends VectorTester.VersionedWithChecksums
+{
+    @BeforeClass
+    public static void setUpClass()
+    {
+        CUSTOM_TRACING_CLASS.setString("org.apache.cassandra.tracing.TracingTestImpl");
+        VectorTester.setUpClass();
+    }
+
+    @Test
+    public void tracingTest()
+    {
+        createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', [2.0, 3.0, 4.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'C', [3.0, 4.0, 5.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (3, 'D', [4.0, 5.0, 6.0])");
+
+        flush();
+
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (4, 'E', [5.0, 2.0, 3.0])");
+
+        Tracing.instance.newSession(ClientState.forInternalCalls(), Tracing.TraceType.QUERY);
+        execute("SELECT * FROM %s ORDER BY val ann of [9.5, 5.5, 6.5] LIMIT 5");
+        for (String trace : ((TracingTestImpl) Tracing.instance).getTraces())
+            assertThat(trace).doesNotContain("Executing single-partition query");
+        // manual inspection to verify that no extra traces were included
+        logger.info(((TracingTestImpl) Tracing.instance).getTraces().toString());
+
+        // because we parameterized the test class we need to clean up after ourselves or the second run will fail
+        Tracing.instance.stopSession();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorTypeTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorTypeTest.java
index 1a0123067bbb..247452ca5b47 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/VectorTypeTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorTypeTest.java
@@ -19,33 +19,41 @@
 package org.apache.cassandra.index.sai.cql;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
+import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
-import org.apache.cassandra.config.CassandraRelevantProperties;
+import io.github.jbellis.jvector.graph.GraphSearcher;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
 import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+import org.apache.cassandra.index.sai.disk.v3.V3OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.vector.CassandraOnHeapGraph;
+import org.apache.cassandra.index.sai.disk.vector.VectorSourceModel;
+import org.apache.cassandra.index.sai.plan.QueryController;
+import org.apache.cassandra.inject.ActionBuilder;
+import org.apache.cassandra.inject.Expression;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.inject.InvokePointBuilder;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-public class VectorTypeTest extends VectorTester
+@RunWith(Parameterized.class)
+public class VectorTypeTest extends VectorTester.VersionedWithChecksums
 {
-    private static final IPartitioner partitioner = Murmur3Partitioner.instance;
-
     @Test
     public void endToEndTest()
     {
@@ -88,18 +96,6 @@ public void endToEndTest()
         assertContainsInt(result, "pk", 2);
     }
 
-    @Test
-    public void warningIsIssuedOnIndexCreation()
-    {
-        ClientWarn.instance.captureWarnings();
-        createTable("CREATE TABLE %s (pk int, val vector<float, 3>, PRIMARY KEY(pk))");
-        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
-        List<String> warnings = ClientWarn.instance.getWarnings();
-
-        assertTrue(warnings.size() > 0);
-        assertEquals(StorageAttachedIndex.VECTOR_USAGE_WARNING, warnings.get(0));
-    }
-
     @Test
     public void createIndexAfterInsertTest()
     {
@@ -156,6 +152,57 @@ public void testTwoPredicates()
         assertThat(result).hasSize(2);
     }
 
+    @Test
+    public void testTwoPredicatesWithBruteForce()
+    {
+        // Note: the PKs in this test are chosen intentionally to ensure their tokens overlap so that
+        // we can test the brute force path.
+        setMaxBruteForceRows(0);
+        createTable("CREATE TABLE %s (pk int, b boolean, v vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, b, v) VALUES (1, true, [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, b, v) VALUES (2, true, [2.0, 3.0, 4.0])");
+        execute("INSERT INTO %s (pk, b, v) VALUES (3, false, [3.0, 4.0, 5.0])");
+
+        // the vector given is closest to row 2, but we exclude that row because b=false
+        var result = execute("SELECT * FROM %s WHERE b=true ORDER BY v ANN OF [3.1, 4.1, 5.1] LIMIT 2");
+        // VSTODO assert specific row keys
+        assertThat(result).hasSize(2);
+
+        flush();
+        compact();
+
+        result = execute("SELECT * FROM %s WHERE b=true ORDER BY v ANN OF [3.1, 4.1, 5.1] LIMIT 2");
+        assertThat(result).hasSize(2);
+
+        // Add 3 rows to memtable. Need number of rows to be greater than both maxBruteForceRows and the LIMIT
+        execute("INSERT INTO %s (pk, b, v) VALUES (4, true, [4.0, 5.0, 6.0])");
+        execute("INSERT INTO %s (pk, b, v) VALUES (5, true, [5.0, 6.0, 7.0])");
+        execute("INSERT INTO %s (pk, b, v) VALUES (6, true, [6.0, 7.0, 8.0])");
+
+        result = execute("SELECT * FROM %s WHERE b=true ORDER BY v ANN OF [3.1, 4.1, 5.1] LIMIT 2");
+        assertThat(result).hasSize(2);
+    }
+
+    @Test
+    public void testTwoPredicatesWithUnnecessaryAllowFiltering()
+    {
+        createTable("CREATE TABLE %s (pk int, b int, v vector<float, 3>, PRIMARY KEY(pk, b))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, b, v) VALUES (0, 0, [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, b, v) VALUES (1, 2, [2.0, 3.0, 4.0])");
+        execute("INSERT INTO %s (pk, b, v) VALUES (2, 4, [3.0, 4.0, 5.0])");
+        execute("INSERT INTO %s (pk, b, v) VALUES (3, 6, [4.0, 5.0, 6.0])");
+
+        // Choose a vector closer to b = 0 to ensure that b's restriction is applied.
+        assertRows(execute("SELECT pk FROM %s WHERE b > 2 ORDER BY v ANN OF [1,2,3] LIMIT 2 ALLOW FILTERING;"),
+                   row(2), row(3));
+    }
+
     @Test
     public void testTwoPredicatesManyRows()
     {
@@ -165,7 +212,7 @@ public void testTwoPredicatesManyRows()
 
         for (int i = 0; i < 100; i++)
             execute("INSERT INTO %s (pk, b, v) VALUES (?, true, ?)",
-                    i, vector((float) i, (float) (i + 1), (float) (i + 2)));
+                    i, vector(i, i + 1, i + 2));
 
         var result = execute("SELECT * FROM %s WHERE b=true ORDER BY v ANN OF [3.1, 4.1, 5.1] LIMIT 2");
         assertThat(result).hasSize(2);
@@ -278,18 +325,8 @@ public void testQueryMoreRowsThanInserted()
     public void changingOptionsTest()
     {
         createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
-        if (CassandraRelevantProperties.SAI_VECTOR_ALLOW_CUSTOM_PARAMETERS.getBoolean())
-        {
-            createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = " +
-                        "{'maximum_node_connections' : 10, 'construction_beam_width' : 200, 'similarity_function' : 'euclidean' }");
-        }
-        else
-        {
-            assertThatThrownBy(() -> createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = " +
-                                                 "{'maximum_node_connections' : 10, 'construction_beam_width' : 200, 'similarity_function' : 'euclidean' }"))
-            .isInstanceOf(InvalidRequestException.class);
-            return;
-        }
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = " +
+                    "{'maximum_node_connections' : 10, 'construction_beam_width' : 200, 'similarity_function' : 'euclidean' }");
 
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
         execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', [2.0, 3.0, 4.0])");
@@ -315,16 +352,48 @@ public void changingOptionsTest()
         assertThat(result).hasSize(5);
     }
 
+    @Test
+    public void defaultOptionsTest()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        var sim = getCurrentColumnFamilyStore().indexManager;
+        var index = (StorageAttachedIndex) sim.listIndexes().iterator().next();
+        assertEquals(VectorSourceModel.OTHER, index.getIndexContext().getIndexWriterConfig().getSourceModel());
+        assertEquals(VectorSimilarityFunction.COSINE, index.getIndexContext().getIndexWriterConfig().getSimilarityFunction());
+    }
+
+    @Test
+    public void customModelOptionsTest()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'source_model' : 'ada002' }");
+
+        var sim = getCurrentColumnFamilyStore().indexManager;
+        var index = (StorageAttachedIndex) sim.listIndexes().iterator().next();
+        assertEquals(VectorSourceModel.ADA002, index.getIndexContext().getIndexWriterConfig().getSourceModel());
+        assertEquals(VectorSimilarityFunction.DOT_PRODUCT, index.getIndexContext().getIndexWriterConfig().getSimilarityFunction());
+    }
+
+    @Test
+    public void obsoleteOptionsTest()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex' WITH OPTIONS = {'optimize_for' : 'recall' }");
+        // as long as CREATE doesn't error out, we're good
+    }
+
     @Test
     public void bindVariablesTest()
     {
         createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', ?)", vector(1.0f, 2.0f ,3.0f));
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', ?)", vector(2.0f ,3.0f, 4.0f));
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'C', ?)", vector(3.0f, 4.0f, 5.0f));
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (3, 'D', ?)", vector(4.0f, 5.0f, 6.0f));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', ?)", vector(1, 2 , 3));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', ?)", vector(2 , 3, 4));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'C', ?)", vector(3, 4, 5));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (3, 'D', ?)", vector(4, 5, 6));
 
         UntypedResultSet result = execute("SELECT * FROM %s ORDER BY val ann of ? LIMIT 3", vector(2.5f, 3.5f, 4.5f));
         assertThat(result).hasSize(3);
@@ -339,11 +408,11 @@ public void intersectedSearcherTest()
         createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', ?)", vector(1.0f, 2.0f ,3.0f));
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', ?)", vector(2.0f ,3.0f, 4.0f));
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'C', ?)", vector(3.0f, 4.0f, 5.0f));
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (3, 'B', ?)", vector(4.0f, 5.0f, 6.0f));
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (4, 'E', ?)", vector(5.0f, 6.0f, 7.0f));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', ?)", vector(1, 2 , 3));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', ?)", vector(2 , 3, 4));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'C', ?)", vector(3, 4, 5));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (3, 'B', ?)", vector(4, 5, 6));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (4, 'E', ?)", vector(5, 6, 7));
 
         UntypedResultSet result = execute("SELECT * FROM %s WHERE str_val = 'B' ORDER BY val ann of [2.5, 3.5, 4.5] LIMIT 2");
         assertThat(result).hasSize(2);
@@ -360,11 +429,11 @@ public void nullVectorTest()
         createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', ?)", vector(1.0f, 2.0f ,3.0f));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', ?)", vector(1, 2 , 3));
         execute("INSERT INTO %s (pk, str_val) VALUES (1, 'B')"); // no vector
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'C', ?)", vector(3.0f, 4.0f, 5.0f));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'C', ?)", vector(3, 4, 5));
         execute("INSERT INTO %s (pk, str_val) VALUES (3, 'D')"); // no vector
-        execute("INSERT INTO %s (pk, str_val, val) VALUES (4, 'E', ?)", vector(5.0f, 6.0f, 7.0f));
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (4, 'E', ?)", vector(5, 6, 7));
 
         UntypedResultSet result = execute("SELECT * FROM %s WHERE str_val = 'B' ORDER BY val ann of [2.5, 3.5, 4.5] LIMIT 2");
         assertThat(result).hasSize(0);
@@ -413,7 +482,7 @@ public void primaryKeySearchTest()
 
         var N = 5;
         for (int i = 0; i < N; i++)
-            execute("INSERT INTO %s (pk, val) VALUES (?, ?)", i, vector(1.0f + i, 2.0f + i, 3.0f + i));
+            execute("INSERT INTO %s (pk, val) VALUES (?, ?)", i, vector(1 + i, 2 + i, 3 + i));
 
         for (int i = 0; i < N; i++)
         {
@@ -452,7 +521,7 @@ public void partitionKeySearchTest()
             }
         }
 
-        var queryVector = vector(new float[] { 1.5f, 1.5f });
+        var queryVector = vector(1.5f, 1.5f);
         for (int i = 1; i <= nPartitions; i++)
         {
             UntypedResultSet result = execute("SELECT partition, row FROM %s WHERE partition = ? ORDER BY val ann of ? LIMIT 2", i, queryVector);
@@ -473,109 +542,7 @@ public void partitionKeySearchTest()
         }
     }
 
-    @Test
-    public void clusteringKeyIndexTest()
-    {
-        createTable("CREATE TABLE %s (pk int, ck vector<float, 2>, PRIMARY KEY(pk, ck))");
-        createIndex("CREATE CUSTOM INDEX ON %s(ck) USING 'StorageAttachedIndex'");
-
-        execute("INSERT INTO %s (pk, ck) VALUES (1, [1.0, 2.0])");
-
-        assertRows(execute("SELECT * FROM %s ORDER BY ck ANN OF [1.0, 2.0] LIMIT 1"), row(1, vector(1.0F, 2.0F)));
-    }
-
-    @Test
-    public void rangeSearchTest() throws Throwable
-    {
-        createTable("CREATE TABLE %s (partition int, val vector<float, 2>, PRIMARY KEY(partition))");
-        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
-
-        var nPartitions = 100;
-        Map<Integer, float[]> vectorsByKey = new HashMap<>();
-
-        for (int i = 1; i <= nPartitions; i++)
-        {
-            float[] vector = {(float) i, (float) i};
-            execute("INSERT INTO %s (partition, val) VALUES (?, ?)", i, vector(vector));
-            vectorsByKey.put(i, vector);
-        }
-
-        var queryVector = vector(new float[] { 1.5f, 1.5f });
-        CheckedFunction tester = () -> {
-            for (int i = 1; i <= nPartitions; i++)
-            {
-                UntypedResultSet result = execute("SELECT partition FROM %s WHERE token(partition) > token(?) ORDER BY val ann of ? LIMIT 1000", i, queryVector);
-                assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysWithLowerBound(vectorsByKey.keySet(), i, false));
-
-                result = execute("SELECT partition FROM %s WHERE token(partition) >= token(?) ORDER BY val ann of ? LIMIT 1000", i, queryVector);
-                assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysWithLowerBound(vectorsByKey.keySet(), i, true));
-
-                result = execute("SELECT partition FROM %s WHERE token(partition) < token(?) ORDER BY val ann of ? LIMIT 1000", i, queryVector);
-                assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysWithUpperBound(vectorsByKey.keySet(), i, false));
-
-                result = execute("SELECT partition FROM %s WHERE token(partition) <= token(?) ORDER BY val ann of ? LIMIT 1000", i, queryVector);
-                assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysWithUpperBound(vectorsByKey.keySet(), i, true));
-
-                for (int j = 1; j <= nPartitions; j++)
-                {
-                    result = execute("SELECT partition FROM %s WHERE token(partition) >= token(?) AND token(partition) <= token(?) ORDER BY val ann of ? LIMIT 1000", i, j, queryVector);
-                    assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysInBounds(vectorsByKey.keySet(), i, true, j, true));
-
-                    result = execute("SELECT partition FROM %s WHERE token(partition) > token(?) AND token(partition) <= token(?) ORDER BY val ann of ? LIMIT 1000", i, j, queryVector);
-                    assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysInBounds(vectorsByKey.keySet(), i, false, j, true));
-
-                    result = execute("SELECT partition FROM %s WHERE token(partition) >= token(?) AND token(partition) < token(?) ORDER BY val ann of ? LIMIT 1000", i, j, queryVector);
-                    assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysInBounds(vectorsByKey.keySet(), i, true, j, false));
-
-                    result = execute("SELECT partition FROM %s WHERE token(partition) > token(?) AND token(partition) < token(?) ORDER BY val ann of ? LIMIT 1000", i, j, queryVector);
-                    assertThat(keys(result)).containsExactlyInAnyOrderElementsOf(keysInBounds(vectorsByKey.keySet(), i, false, j, false));
-                }
-            }
-        };
-
-        tester.apply();
-
-        flush();
-
-        tester.apply();
-    }
-
-    private Collection<Integer> keys(UntypedResultSet result)
-    {
-        List<Integer> keys = new ArrayList<>(result.size());
-        for (UntypedResultSet.Row row : result)
-            keys.add(row.getInt("partition"));
-        return keys;
-    }
-
-    private Collection<Integer> keysWithLowerBound(Collection<Integer> keys, int leftKey, boolean leftInclusive)
-    {
-        return keysInTokenRange(keys, partitioner.getToken(Int32Type.instance.decompose(leftKey)), leftInclusive,
-                                partitioner.getMaximumToken().getToken(), true);
-    }
-
-    private Collection<Integer> keysWithUpperBound(Collection<Integer> keys, int rightKey, boolean rightInclusive)
-    {
-        return keysInTokenRange(keys, partitioner.getMinimumToken().getToken(), true,
-                                partitioner.getToken(Int32Type.instance.decompose(rightKey)), rightInclusive);
-    }
 
-    private Collection<Integer> keysInBounds(Collection<Integer> keys, int leftKey, boolean leftInclusive, int rightKey, boolean rightInclusive)
-    {
-        return keysInTokenRange(keys, partitioner.getToken(Int32Type.instance.decompose(leftKey)), leftInclusive,
-                                partitioner.getToken(Int32Type.instance.decompose(rightKey)), rightInclusive);
-    }
-
-    private Collection<Integer> keysInTokenRange(Collection<Integer> keys, Token leftToken, boolean leftInclusive, Token rightToken, boolean rightInclusive)
-    {
-        long left = leftToken.getLongValue();
-        long right = rightToken.getLongValue();
-        return keys.stream()
-               .filter(k -> {
-                   long t = partitioner.getToken(Int32Type.instance.decompose(k)).getLongValue();
-                   return (left < t || left == t && leftInclusive) && (t < right || t == right && rightInclusive);
-               }).collect(Collectors.toSet());
-    }
 
     @Test
     public void selectFloatVectorFunctions()
@@ -584,7 +551,7 @@ public void selectFloatVectorFunctions()
 
         // basic functionality
         Vector<Float> q = vector(1f, 2f);
-        execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1f, 2f));
+        execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, 2));
         execute("SELECT similarity_cosine(value, value) FROM %s WHERE pk=0");
 
         // type inference checks
@@ -623,20 +590,110 @@ public void selectSimilarityWithAnn()
     }
 
     @Test
-    public void testTwoPredicatesWithUnnecessaryAllowFiltering()
+    public void castedTerminalFloatVectorFunctions()
     {
-        createTable("CREATE TABLE %s (pk int, b int, v vector<float, 3>, PRIMARY KEY(pk, b))");
-        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
-        createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'StorageAttachedIndex'");
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<float, 2>)");
 
-        execute("INSERT INTO %s (pk, b, v) VALUES (0, 0, [1.0, 2.0, 3.0])");
-        execute("INSERT INTO %s (pk, b, v) VALUES (1, 2, [2.0, 3.0, 4.0])");
-        execute("INSERT INTO %s (pk, b, v) VALUES (2, 4, [3.0, 4.0, 5.0])");
-        execute("INSERT INTO %s (pk, b, v) VALUES (3, 6, [4.0, 5.0, 6.0])");
+        execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, 2));
+        execute("SELECT similarity_cosine(value, (vector<float, 2>) [1.0, 1.0]) FROM %s WHERE pk=0");
+        execute("SELECT similarity_cosine((vector<float, 2>) [1.0, 1.0], value) FROM %s WHERE pk=0");
+        execute("SELECT similarity_cosine((vector<float, 2>) [1.0, 1.0], (vector<float, 2>) [1.0, 1.0]) FROM %s WHERE pk=0");
+    }
 
-        // Choose a vector closer to b = 0 to ensure that b's restriction is applied.
-        assertRows(execute("SELECT pk FROM %s WHERE b > 2 ORDER BY v ANN OF [1,2,3] LIMIT 2 ALLOW FILTERING;"),
-                   row(2), row(3));
+    @Test
+    public void inferredTerminalFloatVectorFunctions()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector<float, 2>)");
+
+        execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, 2));
+        assertRows(execute("SELECT similarity_cosine(value, [2.0, 4.0]) FROM %s WHERE pk=0"), row(1f));
+        assertRows(execute("SELECT similarity_cosine([2.0, 4.0], value) FROM %s WHERE pk=0"), row(1f));
+        assertRows(execute("SELECT similarity_cosine([1.0, 2.0], [2.0, 4.0]) FROM %s WHERE pk=0"), row(1f));
+
+        // wrong number of arguments
+        assertInvalidMessage("Invalid number of arguments for function system.similarity_cosine(vector<float, n>, vector<float, n>)",
+                             "SELECT similarity_cosine([1.0, 2.0]) FROM %s WHERE pk=0");
+        assertInvalidMessage("Invalid number of arguments for function system.similarity_cosine(vector<float, n>, vector<float, n>)",
+                             "SELECT similarity_cosine([1.0, 2.0]) FROM %s WHERE pk=0");
+
+        // assignable element types
+        assertRows(execute("SELECT similarity_cosine([1, 2], [2, 4]) FROM %s WHERE pk=0"), row(1f));
+        assertRows(execute("SELECT similarity_cosine([1.0, 2.0], [2, 4]) FROM %s WHERE pk=0"), row(1f));
+        assertRows(execute("SELECT similarity_cosine([1, 2], [2.0, 4.0]) FROM %s WHERE pk=0"), row(1f));
+
+        // not-assignable element types
+        assertInvalidMessage("Type error: ['a', 'b'] cannot be passed as argument 1",
+                             "SELECT similarity_cosine(value, ['a', 'b']) FROM %s WHERE pk=0");
+        assertInvalidMessage("Type error: ['a', 'b'] cannot be passed as argument 0",
+                             "SELECT similarity_cosine(['a', 'b'], value) FROM %s WHERE pk=0");
+        assertInvalidMessage("Type error: ['a', 'b'] cannot be passed as argument 0",
+                             "SELECT similarity_cosine(['a', 'b'], ['a', 'b']) FROM %s WHERE pk=0");
+
+        // different vector sizes, message could be more informative
+        assertInvalidMessage("All arguments must have the same vector dimensions",
+                             "SELECT similarity_cosine(value, [2, 4, 6]) FROM %s WHERE pk=0");
+        assertInvalidMessage("All arguments must have the same vector dimensions",
+                             "SELECT similarity_cosine([1, 2, 3], value) FROM %s WHERE pk=0");
+        assertInvalidMessage("All arguments must have the same vector dimensions",
+                             "SELECT similarity_cosine([1, 2], [3, 4, 5]) FROM %s WHERE pk=0");
+    }
+
+    @Test
+    public void testSamePKWithBruteForceAndGraphBasedScoring()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, vec vector<float, 2>, PRIMARY KEY(pk))");
+        // Use euclidean distance to more easily verify correctness of caching
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex' WITH OPTIONS = { 'similarity_function' : 'euclidean' }");
+
+        // Put one row in the first ss table to guarantee brute force method. This vector is also the most similar.
+        execute("INSERT INTO %s (pk, vec) VALUES (?, ?)", 10, vector(1f, 1f));
+        flush();
+
+        // Must be enough rows to go to graph
+        for (int j = 1; j <= 10; j++)
+        {
+            execute("INSERT INTO %s (pk, vec) VALUES (?, ?)", j, vector(j, j));
+        }
+        flush();
+
+        assertRows(execute("SELECT pk FROM %s ORDER BY vec ANN OF [1,1] LIMIT 2"), row(1), row(2));
+    }
+
+    @Test
+    public void testSamePKWithBruteForceAndOnDiskGraphBasedScoring()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, vec vector<float, 2>, PRIMARY KEY(pk))");
+        // Use euclidean distance to more easily verify correctness of caching
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex' WITH OPTIONS = { 'similarity_function' : 'euclidean' }");
+
+        // Put one row in the first ss table to guarantee brute force method. This vector is also the most similar.
+        execute("INSERT INTO %s (pk, vec) VALUES (?, ?)", 10, vector(1f, 1f));
+        flush();
+
+        // over 1024 vectors to guarantee PQ on disk
+        // Must be enough rows to go to graph
+        for (int j = 1; j <= 1100; j++)
+        {
+            execute("INSERT INTO %s (pk, vec) VALUES (?, ?)", j, vector((float) j, (float) j));
+        }
+        flush();
+
+        assertRows(execute("SELECT pk FROM %s ORDER BY vec ANN OF [1,1] LIMIT 2"), row(1), row(2));
+    }
+
+    @Test
+    public void testRowWithMissingVectorThatMatchesQueryPredicates()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, val text, vec vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // There was an edge case where we failed because there was just a single row in the table.
+        execute("INSERT INTO %s (pk, val) VALUES (1, 'match me')");
+        assertRows(execute("SELECT pk FROM %s WHERE val = 'match me' ORDER BY vec ANN OF [1,1] LIMIT 2"));
+        // Push memtable to sstable. we should get same result
+        flush();
+        assertRows(execute("SELECT pk FROM %s WHERE val = 'match me' ORDER BY vec ANN OF [1,1] LIMIT 2"));
     }
 
     @Test
@@ -659,20 +716,295 @@ public void testMultipleVectorsInMemoryWithPredicate()
     }
 
     @Test
-    public void multiPartitionUpdateMultiIndexTest()
+    public void testNestedANNQuery()
+    {
+        createTable("CREATE TABLE %s (pk int, name text, body text, vals vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vals) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(name) USING 'StorageAttachedIndex'");
+        execute("INSERT INTO %s (pk, name, body, vals) VALUES (1, 'Ann', 'A lizard said bad things to the snakes', [0.1, 0.1])");
+        execute("INSERT INTO %s (pk, name, body, vals) VALUES (2, 'Bea', 'Please wear protective gear before operating the machine', [0.2, -0.3])");
+        execute("INSERT INTO %s (pk, name, body, vals) VALUES (3, 'Cal', 'My name is Slim Shady', [0.0, 0.9])");
+        execute("INSERT INTO %s (pk, name, body, vals) VALUES (4, 'Bea', 'I repeat: wear your helmet!', [0.3, -0.2])");
+        var result = execute("SELECT pk FROM %s WHERE name='Bea' OR name='Ann' ORDER BY vals ANN OF [0.3, 0.1] LIMIT 5");
+        assertRowsIgnoringOrder(result, row(1), row(2), row(4));
+    }
+
+    @Test
+    public void testIntersectionWithMatchingPrimaryKeyDownToClusteringValues() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, a int, b int, c int, vec vector<float, 2>, PRIMARY KEY(pk, a))");
+        createIndex("CREATE CUSTOM INDEX ON %s(b) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+
+        // This row is created so that it matches the query parameters, and so that the PK is before the other PKs.
+        // The token for 5 is -7509452495886106294 and the token for 1 is -4069959284402364209.
+        execute("INSERT INTO %s (pk, a, b, c, vec) VALUES (?, ?, ?, ?, ?)", 5, 1, 1, 2, vector(1, 1));
+        // This row is created so that it matches one, but not both, predicates, and so that it has the same token
+        // as the third row, but is technically before it when sorting using clustering columns.
+        execute("INSERT INTO %s (pk, a, b, c, vec) VALUES (?, ?, ?, ?, ?)", 1, 1, 1, 0, vector(1, 1));
+        // This row is the only valid match and is the final row in the sstable.
+        execute("INSERT INTO %s (pk, a, b, c, vec) VALUES (?, ?, ?, ?, ?)", 1, 2, 1, 2, vector(1, 1));
+
+        beforeAndAfterFlush(
+        () -> {
+            // Query has three important details. First, we restrict by the partition, then we have an intersection
+            // on b and c. It is a vector query because there is a separate code path for it.
+            assertRows(execute("SELECT a FROM %s WHERE b = 1 AND c = 2 AND pk = 1 ORDER BY vec ANN OF [1,1] LIMIT 3"), row(2));
+            // Verify this works for the non vector code path as well, which was also broken.
+            assertRows(execute("SELECT a FROM %s WHERE b = 1 AND c = 2 AND pk = 1"), row(2));
+        });
+    }
+
+    // search across multiple sstables each with multiple segments, verify results with and without non-ann filtering
+    @Test
+    public void multipleSSTablesAndMultipleSegmentsTest()
+    {
+        createTable("CREATE TABLE %s (pk int, constant boolean, val vector<float, 2>, PRIMARY KEY(pk))");
+        disableCompaction(KEYSPACE);
+
+        int vectorCountPerSSTable = getRandom().nextIntBetween(200, 400);
+        int pk = 0;
+
+        // 50 rows per segment to ensure certain kinds of skipping
+        SegmentBuilder.updateLastValidSegmentRowId(50);
+
+        // create multiple sstables to ensure that not all PKs have the same source table
+        for (int i = 0; i < 6; i++)
+        {
+            for (int row = 0; row < vectorCountPerSSTable; row++)
+                // Create random vectors, we're only testing internal consistency
+                execute("INSERT INTO %s (pk, constant, val) VALUES (?, ?, ?)", pk++, true,
+                        vector(getRandom().nextIntBetween(1, 400), getRandom().nextIntBetween(1, 400)));
+            flush();
+        }
+
+        // create indexes on existing sstable to produce multiple segments
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function' : 'euclidean'}");
+        createIndex("CREATE CUSTOM INDEX ON %s(constant) USING 'StorageAttachedIndex'");
+
+        // query multiple on-disk indexes
+        int limit = getRandom().nextIntBetween(5,25);
+        // Pick a vector in the middle of the distribution
+        UntypedResultSet unfilteredResults = execute("SELECT pk FROM %s ORDER BY val ANN OF [200,200] LIMIT ?", limit);
+        UntypedResultSet filteredResults = execute("SELECT pk FROM %s WHERE constant = true ORDER BY val ANN OF [200,200] LIMIT ?", limit);
+
+        // Extract the primary keys while retaining order
+        var unfilteredRows = unfilteredResults.stream().map(row -> row.getInt("pk")).toArray();
+        var filteredRows = filteredResults.stream().map(row -> row.getInt("pk")).toArray();
+
+        // Assert that the results are the same
+        assertThat(filteredRows).containsExactly(unfilteredRows);
+    }
+
+    @Test
+    public void insertstuff()
+    {
+        // This test requires the non-bruteforce route
+        setMaxBruteForceRows(0);
+        createTable("CREATE TABLE %s (pk int, val text, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // Insert data
+        execute("INSERT INTO %s (pk, val) VALUES (1, 'A')");
+        execute("INSERT INTO %s (pk, val) VALUES (2, 'B')");
+        execute("INSERT INTO %s (pk, val) VALUES (3, 'C')");
+
+        // no solution yet, so flush()
+        flush();
+
+        // query with order
+        assertRows(execute("SELECT pk FROM %s ORDER BY val limit 3"), row(1), row(2), row(3));
+        assertRows(execute("SELECT pk FROM %s ORDER BY val limit 1"), row(1));
+    }
+
+    @Test
+    public void testEnsureIndexQueryableAfterTransientFailure() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, vec vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+
+        var injection = Injections.newCustom("fail_on_searcher_search")
+                                  .add(InvokePointBuilder.newInvokePoint().onClass(GraphSearcher.class).onMethod("search").atEntry())
+                                  .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
+                                  .build();
+        Injections.inject(injection);
+        // Insert data so we can query the index
+        execute("INSERT INTO %s (pk, vec) VALUES (1, [1,1])");
+
+        // Ensure that we fail, as expected, and that a subsequent call to search is successful.
+        beforeAndAfterFlush(() -> {
+            injection.enable();
+            assertThatThrownBy(() -> executeInternal("SELECT pk FROM %s ORDER BY vec ANN OF [1,1] LIMIT 2")).hasMessageContaining("Injected failure!");
+            injection.disable();
+            assertRows(execute("SELECT pk FROM %s ORDER BY vec ANN OF [1,1] LIMIT 2"), row(1));
+        });
+    }
+
+    @Test
+    public void testCompactionWithEnoughRowsForPQAndDeleteARow()
+    {
+        createTable("CREATE TABLE %s (pk int, vec vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+
+        disableCompaction();
+
+        for (int i = 0; i <= CassandraOnHeapGraph.MIN_PQ_ROWS; i++)
+            execute("INSERT INTO %s (pk, vec) VALUES (?, ?)", i, vector(i, i + 1));
+        flush();
+
+        // By deleting a row, we trigger a key histogram to round its estimate to 0 instead of 1 rows per key, and
+        // that broke compaction, so we test that here.
+        execute("DELETE FROM %s WHERE pk = 0");
+        flush();
+
+        // Run compaction, it fails if compaction is not successful
+        compact();
+
+        // Confirm we can query the data
+        assertRowCount(execute("SELECT * FROM %s ORDER BY vec ANN OF [1,2] LIMIT 1"), 1);
+    }
+
+    /**
+     * Tests a filter-then-sort query with a concurrent vector deletion. See CNDB-10536 for details.
+     */
+    @Test
+    public void testFilterThenSortQueryWithConcurrentVectorDeletion() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<float, 2>, c int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(c) USING 'StorageAttachedIndex'");
+
+        // write into memtable
+        execute("INSERT INTO %s (k, v, c) VALUES (1, [1, 1], 1)");
+        execute("INSERT INTO %s (k, v, c) VALUES (2, [2, 2], 1)");
+
+        // inject a barrier to block CassandraOnHeapGraph#getOrdinal
+        Injections.Barrier barrier = Injections.newBarrier("block_get_ordinal", 2, false)
+                                               .add(InvokePointBuilder.newInvokePoint()
+                                                                      .onClass(CassandraOnHeapGraph.class)
+                                                                      .onMethod("getOrdinal")
+                                                                      .atEntry())
+                                               .build();
+        Injections.inject(barrier);
+
+        // start a filter-then-sort query asynchronously that will get blocked in the injected barrier
+        QueryController.QUERY_OPT_LEVEL = 0;
+        try
+        {
+            ExecutorService executor = Executors.newFixedThreadPool(1);
+            String select = "SELECT k FROM %s WHERE c=1 ORDER BY v ANN OF [1, 1] LIMIT 100";
+            Future<UntypedResultSet> future = executor.submit(() -> execute(select));
+
+            // once the query is blocked, delete one of the vectors and flush, so the postings for the vector are removed
+            waitForAssert(() -> Assert.assertEquals(1, barrier.getCount()));
+            execute("DELETE v FROM %s WHERE k = 1");
+            flush();
+
+            // release the barrier to resume the query, which should succeed
+            barrier.countDown();
+            assertRows(future.get(), row(2));
+
+            assertEquals(0, executor.shutdownNow().size());
+        }
+        finally
+        {
+            QueryController.QUERY_OPT_LEVEL = 1;
+        }
+    }
+
+    @Test
+    public void newJVectorOptionsTestVersion2()
+    {
+        newJVectorOptionsTest(2);
+    }
+    // We skip version 3 since it isn't supported anymore
+    @Test
+    public void newJVectorOptionsTestVersion4()
+    {
+        newJVectorOptionsTest(4);
+    }
+
+    public void newJVectorOptionsTest(int version)
     {
-        createTable("CREATE TABLE %s (pk int, row_v vector<float, 2>, metadata map<text, text>, PRIMARY KEY(pk))");
-        createIndex("CREATE CUSTOM INDEX ON %s(entries(metadata)) USING 'StorageAttachedIndex'");
-        createIndex("CREATE CUSTOM INDEX ON %s(row_v) USING 'StorageAttachedIndex'");
+        // Configure the version to ensure we don't fail for settings that are unsupported on earlier versions of jvector
+        V3OnDiskFormat.JVECTOR_VERSION = version;
+
+        // This test ensures that we can set and retrieve new jvector parameters
+        // (neighborhood_overflow, alpha, enable_hierarchy), and that they are honored at index build time.
+
+        createTable("CREATE TABLE %s (pk int, txt text, vec vector<float, 4>, PRIMARY KEY(pk))");
+
+        // We'll specify a few options, including the existing ones (e.g. maximum_node_connections).
+        // Setting dimension=4 also triggers the default alpha=1.2 if not overridden,
+        // which you can compare to your own defaults in the IndexWriterConfig code.
 
-        String select = "SELECT * FROM %s WHERE metadata['map_k'] = 'map_v' AND pk = 0 ORDER BY row_v ann of [0.1, 0.2] LIMIT 4";
-        assertRows(execute(select));
+        // This should succeed.
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex' "
+                    + "WITH OPTIONS = {"
+                    + "  'maximum_node_connections' : '20', "
+                    + "  'construction_beam_width'   : '300', "
+                    + "  'similarity_function'       : 'euclidean', "
+                    + "  'enable_hierarchy'          : 'true', "
+                    + "  'neighborhood_overflow'     : '1.5', "
+                    + "  'alpha'                     : '1.8' "
+                    + '}');
 
-        execute("INSERT INTO %s (pk, metadata, row_v) VALUES (0, {'map_k' : 'map_v'}, [0.11, 0.19])");
-        Object[] row = row(0, map("map_k", "map_v"), vector(0.11f, 0.19f));
-        assertRows(execute(select), row);
+        // Insert many rows
+        for (int i = 0; i < 2000; i++)
+            execute("INSERT INTO %s (pk, txt, vec) VALUES (?, ?, ?)", i, "row" + i, randomVectorBoxed(4));
 
-        execute("INSERT INTO %s (pk, metadata, row_v) VALUES (10, {'map_k' : 'map_v'}, [0.11, 0.19])");
-        assertRows(execute(select), row);
+        // Run basic query to confirm we can, no need to validate results
+        execute("SELECT pk FROM %s ORDER BY vec ANN OF [2.0, 2.0, 3.0, 4.0] LIMIT 2");
+        // Confirm that we can flush with custom options
+        flush();
+        // Run basic query to confirm we can, no need to validate results
+        execute("SELECT pk FROM %s ORDER BY vec ANN OF [2.0, 2.0, 3.0, 4.0] LIMIT 2");
+        // Confirm that we can compact with custom options
+        compact();
+        // Run basic query to confirm we can, no need to validate results
+        execute("SELECT pk FROM %s ORDER BY vec ANN OF [2.0, 2.0, 3.0, 4.0] LIMIT 2");
+
+        // Confirm that the config picks up our custom settings.
+        StorageAttachedIndex saiIndex =
+        (StorageAttachedIndex) getCurrentColumnFamilyStore().indexManager.listIndexes().iterator().next();
+
+        IndexWriterConfig config = saiIndex.getIndexContext().getIndexWriterConfig();
+        // Check the new fields
+        assertEquals(1.5f, config.getNeighborhoodOverflow(999f), 0.0001f);
+        assertEquals(1.8f,  config.getAlpha(999f),               0.0001f);
+        assertTrue(config.isHierarchyEnabled());
+        assertEquals(20,    config.getMaximumNodeConnections());
+        assertEquals(40,    config.getAnnMaxDegree());
+        assertEquals(300,   config.getConstructionBeamWidth());
+        assertEquals(VectorSimilarityFunction.EUCLIDEAN, config.getSimilarityFunction());
     }
+
+    @Test
+    public void testMultiVersionJVectorCompatibility() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, vec vector<float, 4>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex'");
+
+        // Note that we do not test the multi-version path where compaction produces different sstables, which is
+        // the norm in CNDB. If we had a way to compact individual sstables, we could.
+        disableCompaction();
+
+        // Create index files for each valid version
+        for (int version = 2; version <= V3OnDiskFormat.JVECTOR_VERSION; version++)
+        {
+            // Version 3 is no longer supported, so there is mild risk that it isn't covered here, but we can't write
+            // it any more, so there isn't much we can do.
+            if (version == 3)
+                continue;
+            V3OnDiskFormat.JVECTOR_VERSION = version;
+            for (int i = 0; i < CassandraOnHeapGraph.MIN_PQ_ROWS; i++)
+                execute("INSERT INTO %s (pk, vec) VALUES (?, ?)", i, randomVectorBoxed(4));
+            flush();
+        }
+
+        // Run basic query to confirm we can, no need to validate results
+        execute("SELECT pk FROM %s ORDER BY vec ANN OF [2.0, 2.0, 3.0, 4.0] LIMIT 2");
+    }
+
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java
index 09571070c7f9..2ce8a6850890 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java
@@ -18,16 +18,31 @@
 
 package org.apache.cassandra.index.sai.cql;
 
-import org.apache.cassandra.cql3.UntypedResultSet;
-
+import org.junit.Before;
 import org.junit.Test;
 
-import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_VECTOR_SEARCH_ORDER_CHUNK_SIZE;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex;
+import org.apache.cassandra.index.sai.plan.QueryController;
+
 import static org.apache.cassandra.index.sai.cql.VectorTypeTest.assertContainsInt;
+import static org.apache.cassandra.index.sai.disk.vector.CassandraOnHeapGraph.MIN_PQ_ROWS;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
 
-public class VectorUpdateDeleteTest extends VectorTester
+public class VectorUpdateDeleteTest extends VectorTester.VersionedWithChecksums
 {
+    @Before
+    public void setup() throws Throwable
+    {
+        super.setup();
+
+        // Enable the optimizer by default. If there are any tests that need to disable it, they can do so explicitly.
+        QueryController.QUERY_OPT_LEVEL = 1;
+    }
+
     // partition delete won't trigger UpdateTransaction#onUpdated
     @Test
     public void partitionDeleteVectorInMemoryTest()
@@ -53,11 +68,11 @@ public void partitionDeleteVectorInMemoryTest()
         assertThat(result).hasSize(1);
         assertContainsInt(result, "pk", 2);
 
-        flush();
-
-        result = execute("SELECT * FROM %s ORDER BY val ann of [2.1, 3.1, 4.1] LIMIT 1");  // closer to row 1
-        assertThat(result).hasSize(1);
-        assertContainsInt(result, "pk", 2);
+//        flush();
+//
+//        result = execute("SELECT * FROM %s ORDER BY val ann of [2.1, 3.1, 4.1] LIMIT 1");  // closer to row 1
+//        assertThat(result).hasSize(1);
+//        assertContainsInt(result, "pk", 2);
     }
 
     // row delete will trigger UpdateTransaction#onUpdated
@@ -82,6 +97,21 @@ public void rowDeleteVectorInMemoryAndFlushTest()
         assertContainsInt(result, "pk", 0);
     }
 
+    @Test
+    public void testFlushWithDeletedVectors()
+    {
+        createTable("CREATE TABLE %s (pk int, v vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, v) VALUES (0, [1.0, 2.0])");
+        execute("INSERT INTO %s (pk, v) VALUES (0, null)");
+
+        flush();
+
+        var result = execute("SELECT * FROM %s ORDER BY v ann of [2.5, 3.5] LIMIT 1");
+        assertThat(result).hasSize(0);
+    }
+
     // range delete won't trigger UpdateTransaction#onUpdated
     @Test
     public void rangeDeleteVectorInMemoryAndFlushTest()
@@ -257,6 +287,7 @@ public void upsertTest()
         createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
+        // insert row A redundantly, and row B once
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
@@ -264,27 +295,26 @@ public void upsertTest()
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
         execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', [2.0, 3.0, 4.0])");
 
-        UntypedResultSet result = execute("SELECT * FROM %s ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 2");
-        assertThat(result).hasSize(2);
-        assertContainsInt(result, "pk", 0);
-        assertContainsInt(result, "pk", 1);
-        flush();
+        // should only see two results
+        UntypedResultSet result = execute("SELECT pk FROM %s ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 2");
+        assertRows(result, row(0), row(1));
 
+        // flush, then insert A redundantly some more
+        flush();
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
         execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
-        result = execute("SELECT * FROM %s ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 2");
-        assertThat(result).hasSize(2);
-        assertContainsInt(result, "pk", 0);
-        assertContainsInt(result, "pk", 1);
-        flush();
 
-        result = execute("SELECT * FROM %s ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 2");
-        assertThat(result).hasSize(2);
-        assertContainsInt(result, "pk", 0);
-        assertContainsInt(result, "pk", 1);
+        // should still only see two results
+        result = execute("SELECT pk FROM %s ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 2");
+        assertRows(result, row(0), row(1));
+
+        // and again after flushing
+        flush();
+        result = execute("SELECT pk FROM %s ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 2");
+        assertRows(result, row(0), row(1));
     }
 
     @Test
@@ -306,7 +336,6 @@ public void updateTest()
         assertThat(result).hasSize(1);
         assertContainsInt(result, "pk", 0);
         result = execute("SELECT * FROM %s ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 1");
-        makeRowStrings(result).forEach(logger::info);
         assertThat(result).hasSize(1);
         assertContainsInt(result, "pk", 1);
 
@@ -344,6 +373,39 @@ public void updateTest()
         assertContainsInt(result, "pk", 1);
     }
 
+    @Test
+    public void updateTestWithPredicate()
+    {
+        // contrived example to make sure we exercise VectorIndexSearcher.limitToTopResults
+        createTable("CREATE TABLE %s (pk int, str_val text, val vector<float, 3>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+
+        // overwrite row A a bunch of times
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [2.0, 3.0, 4.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [3.0, 4.0, 5.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [4.0, 5.0, 6.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [5.0, 6.0, 7.0])");
+
+        // check that queries near A and B get the right row
+        UntypedResultSet result = execute("SELECT * FROM %s WHERE str_val = 'A' ORDER BY val ann of [4.5, 5.5, 6.5] LIMIT 1");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+        result = execute("SELECT * FROM %s WHERE str_val = 'A' ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 1");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+
+        // flush, and re-check same queries
+        flush();
+        result = execute("SELECT * FROM %s WHERE str_val = 'A' ORDER BY val ann of [4.5, 5.5, 6.5] LIMIT 1");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+        result = execute("SELECT * FROM %s WHERE str_val = 'A' ORDER BY val ann of [0.5, 1.5, 2.5] LIMIT 1");
+        assertThat(result).hasSize(1);
+        assertContainsInt(result, "pk", 0);
+    }
+
     @Test
     public void updateOtherColumnsTest()
     {
@@ -395,6 +457,7 @@ public void updateManySSTablesTest()
         assertContainsInt(result, "pk", 1);
     }
 
+
     @Test
     public void shadowedPrimaryKeyInDifferentSSTable()
     {
@@ -419,6 +482,141 @@ public void shadowedPrimaryKeyInDifferentSSTable()
         assertThat(result).hasSize(1);
     }
 
+    @Test
+    public void shadowedPrimaryKeyWithSharedVector()
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, str_val text, val vector<float, 3>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        disableCompaction(KEYSPACE);
+
+        // flush a sstable with one vector that is shared by two rows
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'B', [1.0, 2.0, 3.0])");
+        flush();
+
+        // flush another sstable to shadow row 0
+        execute("DELETE FROM %s where pk = 0");
+        flush();
+
+        // flush another sstable with one new vector row
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'B', [2.0, 3.0, 4.0])");
+        flush();
+
+        // the shadowed vector has the highest score, but we shouldn't see it
+        var result = execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 2.0, 3.0] LIMIT 2");
+        assertRowsIgnoringOrder(result, row(2), row(1));
+    }
+
+    @Test
+    public void shadowedPrimaryKeyWithSharedVectorAndOtherPredicates()
+    {
+        setMaxBruteForceRows(0);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, str_val text, val vector<float, 3>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        disableCompaction(KEYSPACE);
+
+        // flush a sstable with one vector that is shared by two rows
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'A', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'A', [1.0, 2.0, 3.0])");
+        flush();
+
+        // flush another sstable to shadow row 0
+        execute("DELETE FROM %s where pk = 0");
+        flush();
+
+        // flush another sstable with one new vector row
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'A', [2.0, 3.0, 4.0])");
+        flush();
+
+        // the shadowed vector has the highest score, but we shouldn't see it
+        var result = execute("SELECT pk FROM %s WHERE str_val = 'A' ORDER BY val ann of [1.0, 2.0, 3.0] LIMIT 2");
+        assertRowsIgnoringOrder(result, row(2), row(1));
+    }
+
+    @Test
+    public void shadowedPrimaryKeyWithUpdatedPredicateMatchingIntValue() throws Throwable
+    {
+        setMaxBruteForceRows(0);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, num int, val vector<float, 3>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(num) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        disableCompaction(KEYSPACE);
+
+        // Same PK, different num, different vectors
+        execute("INSERT INTO %s (pk, num, val) VALUES (0, 1, [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, num, val) VALUES (0, 2, [2.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, num, val) VALUES (0, 3, [3.0, 2.0, 3.0])");
+        // Need PKs that wrap 0 when put in PK order
+        execute("INSERT INTO %s (pk, num, val) VALUES (1, 1, [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, num, val) VALUES (2, 1, [1.0, 2.0, 3.0])");
+
+        // the shadowed vector has the highest score, but we shouldn't see it
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE num < 3 ORDER BY val ann of [1.0, 2.0, 3.0] LIMIT 10"),
+                       row(1), row(2));
+        });
+    }
+
+    @Test
+    public void rangeRestrictedTestWithDuplicateVectorsAndADelete()
+    {
+        setMaxBruteForceRows(0);
+        createTable(String.format("CREATE TABLE %%s (pk int, str_val text, val vector<float, %d>, PRIMARY KEY(pk))", 2));
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, val) VALUES (0, [1.0, 2.0])"); // -3485513579396041028
+        execute("INSERT INTO %s (pk, val) VALUES (1, [1.0, 2.0])"); // -4069959284402364209
+        execute("INSERT INTO %s (pk, val) VALUES (2, [1.0, 2.0])"); // -3248873570005575792
+        execute("INSERT INTO %s (pk, val) VALUES (3, [1.0, 2.0])"); // 9010454139840013625
+
+        flush();
+
+        // Show the result set is as expected
+        assertRows(execute("SELECT pk FROM %s WHERE token(pk) <= -3248873570005575792 AND " +
+                           "token(pk) >= -3485513579396041028 ORDER BY val ann of [1,2] LIMIT 1000"), row(0), row(2));
+
+        // Delete one of the rows
+        execute("DELETE FROM %s WHERE pk = 0");
+
+        flush();
+        assertRows(execute("SELECT pk FROM %s WHERE token(pk) <= -3248873570005575792 AND " +
+                           "token(pk) >= -3485513579396041028 ORDER BY val ann of [1,2] LIMIT 1000"), row(2));
+    }
+
+    @Test
+    public void rangeRestrictedTestWithDuplicateVectorsAndAddNullVector() throws Throwable
+    {
+        setMaxBruteForceRows(0);
+        createTable(String.format("CREATE TABLE %%s (pk int, str_val text, val vector<float, %d>, PRIMARY KEY(pk))", 2));
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+
+        execute("INSERT INTO %s (pk, val) VALUES (0, [1.0, 2.0])");
+        execute("INSERT INTO %s (pk, val) VALUES (1, [1.0, 2.0])");
+        execute("INSERT INTO %s (pk, val) VALUES (2, [1.0, 2.0])");
+        // Add a str_val to make sure pk has a row id in the sstable
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (3, 'a', null)");
+        // Add another row to test a different part of the code
+        execute("INSERT INTO %s (pk, val) VALUES (4, [1.0, 2.0])");
+        execute("DELETE FROM %s WHERE pk = 2");
+        flush();
+
+        // Delete one of the rows to trigger a shadowed primary key
+        execute("DELETE FROM %s WHERE pk = 0");
+        execute("INSERT INTO %s (pk, val) VALUES (2, [2.0, 2.0])");
+        flush();
+
+        // Delete more rows.
+        execute("DELETE FROM %s WHERE pk = 2");
+        execute("DELETE FROM %s WHERE pk = 3");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s ORDER BY val ann of [1,2] LIMIT 1000"),
+                       row(1), row(4));
+        });
+    }
+
     @Test
     public void testVectorRowWhereUpdateMakesRowMatchNonOrderingPredicates()
     {
@@ -469,6 +667,7 @@ public void testUpdateVectorWithSplitRow()
         // Overwrite pk 1 with a vector that is closest to the search vector
         execute("INSERT INTO %s (pk, vec) VALUES (1, [11,11])");
 
+
         assertRows(execute("SELECT pk FROM %s WHERE val = 'match me' ORDER BY vec ANN OF [11,11] LIMIT 1"), row(1));
         // Push memtable to sstable. we should get same result
         flush();
@@ -496,52 +695,301 @@ public void testUpdateNonVectorColumnWhereNoSingleSSTableRowMatchesAllPredicates
         assertRows(execute("SELECT pk FROM %s WHERE val1 = 'match me' AND val2 = 'match me' ORDER BY vec ANN OF [11,11] LIMIT 2"), row(1), row(2));
     }
 
+    // This test intentionally has extra rows with primary keys that are above and below the
+    // deleted primary key so that we do not short circuit certain parts of the shadowed key logic.
     @Test
-    public void ensureVariableChunkSizeDoesNotLeadToIncorrectResults() throws Exception
+    public void shadowedPrimaryKeyInDifferentSSTableEachWithMultipleRows()
     {
-        // When adding the chunk size feature, there were issues related to leaked files.
-        // This setting only matters for hybrid queries
-        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, str_val text, vec vector<float, 2>)");
-        createIndex("CREATE CUSTOM INDEX ON %s(vec) USING 'StorageAttachedIndex' WITH OPTIONS = { 'similarity_function' : 'euclidean' }");
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, str_val text, val vector<float, 3>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        disableCompaction(KEYSPACE);
+
+        // flush a sstable with one vector
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'A', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (2, 'A', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (3, 'A', [1.0, 2.0, 3.0])");
+        flush();
+
+        // flush another sstable to shadow the vector row
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (1, 'A', [1.0, 2.0, 3.0])");
+        execute("DELETE FROM %s where pk = 2");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (3, 'A', [1.0, 2.0, 3.0])");
+        flush();
+
+        // flush another sstable with one new vector row
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (0, 'B', [2.0, 3.0, 4.0])");
+        execute("INSERT INTO %s (pk, str_val, val) VALUES (4, 'B', [2.0, 3.0, 4.0])");
+        flush();
+
+        // the shadow vector has the highest score
+        var result = execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 2.0, 3.0] LIMIT 4");
+        assertRows(result, row(1), row(3), row(0), row(4));
+    }
+
+    @Test
+    public void shadowedPrimaryKeysRequireDeeperSearch() throws Throwable
+    {
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, str_val text, val vector<float, 2>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        disableCompaction(KEYSPACE);
+
+        // Choose a row count that will essentially force us to re-query the index that still has more rows to search.
+        int baseRowCount = 1000;
+        // Create 1000 rows so that each row has a slightly less similar score.
+        for (int i = 0; i < baseRowCount - 10; i++)
+            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'A', ?)", i, vector(1, i));
 
-        // Create many sstables to ensure chunk size matters
-        // Start at 1 to prevent indexing zero vector.
-        // Index every vector with A to match everything and because this test only makes sense for hybrid queries
-        for (int i = 1; i <= 100; i++)
-        {
-            execute("INSERT INTO %s (pk, str_val, vec) VALUES (?, ?, ?)", i, "A", vector((float) i, (float) i));
-            if (i % 10 == 0)
-                flush();
-            // Add some deletes in the next segment
-            if (i % 3 == 0)
-                execute("DELETE FROM %s WHERE pk = ?", i);
-        }
-
-        try
-        {
-            // We use a chunk size that is as low as possible (1) and goes up to the whole dataset (100).
-            // We also query for different LIMITs
-            for (int i = 1; i <= 100; i++)
-            {
-                SAI_VECTOR_SEARCH_ORDER_CHUNK_SIZE.setInt(i);
-                var results = execute("SELECT pk FROM %s WHERE str_val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 1");
-                assertRows(results, row(1));
-                results = execute("SELECT pk FROM %s WHERE str_val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 3");
-                // Note that we delete row 3
-                assertRows(results, row(1), row(2), row(4));
-                results = execute("SELECT pk FROM %s WHERE str_val = 'A' ORDER BY vec ANN OF [1,1] LIMIT 10");
-                // Note that we delete row 3, 6, 9, 12
-                assertRows(results, row(1), row(2), row(4), row(5),
-                           row(7), row(8), row(10), row(11), row(13), row(14));
-            }
-        }
-        finally
-        {
-            // Revert to prevent interference with other tests. Note that a decreased chunk size can impact
-            // whether we compute the topk with brute force because it determines how many vectors get sent to the
-            // vector index.
-            SAI_VECTOR_SEARCH_ORDER_CHUNK_SIZE.setInt(100000);
-        }
+        for (int i = baseRowCount -10; i < baseRowCount; i++)
+            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'A', ?)", i, vector(1, -i));
+
+        flush();
+
+        // Create 10 rows with the worst scores, but they won't be shadowed.
+        for (int i = baseRowCount; i < baseRowCount + 10; i++)
+            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'A', ?)", i, vector(-1, baseRowCount * -1));
+
+        // Delete all but the last 10 rows
+        for (int i = 0; i < baseRowCount - 10; i++)
+            execute("DELETE FROM %s WHERE pk = ?", i);
+
+        beforeAndAfterFlush(() -> {
+            // ANN Only
+            assertRows(execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 1.0] LIMIT 3"),
+                       row(baseRowCount - 10), row(baseRowCount - 9), row(baseRowCount - 8));
+            // Hyrbid
+            assertRows(execute("SELECT pk FROM %s WHERE str_val = 'A' ORDER BY val ann of [1.0, 1.0] LIMIT 3"),
+                       row(baseRowCount - 10), row(baseRowCount - 9), row(baseRowCount - 8));
+        });
+    }
+
+    @Test
+    public void testUpdateVectorToWorseAndBetterPositions() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, val vector<float, 2>, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, val) VALUES (0, [1.0, 2.0])");
+        execute("INSERT INTO %s (pk, val) VALUES (1, [1.0, 3.0])");
+
+        flush();
+        execute("INSERT INTO %s (pk, val) VALUES (0, [1.0, 4.0])");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 2.0] LIMIT 1"), row(1));
+            assertRows(execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 2.0] LIMIT 2"), row(1), row(0));
+        });
+
+        // And now update pk 1 to show that we can get 0 too
+        execute("INSERT INTO %s (pk, val) VALUES (1, [1.0, 5.0])");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 2.0] LIMIT 1"), row(0));
+            assertRows(execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 2.0] LIMIT 2"), row(0), row(1));
+        });
+
+        // And now update both PKs so that the stream of ranked rows is PKs: 0, 1, [1], 0, 1, [0], where the numbers
+        // wrapped in brackets are the "real" scores of the vectors. This test makes sure that we correctly remove
+        // PrimaryKeys from the updatedKeys map so that we don't accidentally duplicate PKs.
+        execute("INSERT INTO %s (pk, val) VALUES (1, [1.0, 3.5])");
+        execute("INSERT INTO %s (pk, val) VALUES (0, [1.0, 6.0])");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 2.0] LIMIT 1"), row(1));
+            assertRows(execute("SELECT pk FROM %s ORDER BY val ann of [1.0, 2.0] LIMIT 2"), row(1), row(0));
+        });
+    }
+
+    @Test
+    public void updatedPrimaryKeysRequireResumeSearch() throws Throwable
+    {
+        setMaxBruteForceRows(0);
+
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, str_val text, val vector<float, 2>)");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        disableCompaction(KEYSPACE);
+
+        // This test is fairly contrived, but it covers a bug we hit due to prematurely closed iterators.
+        // The general design for this test is to shadow the close vectors on a memtable/sstable index forcing the
+        // index to resume search. We do that by overwriting the first 50 vectors in the initial sstable.
+        for (int i = 0; i < 100; i++)
+            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'A', ?)", i, vector(1, i));
+
+        // Add more rows to make sure we filter then sort
+        for (int i = 100; i < 1000; i++)
+            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'C', ?)", i, vector(1, i));
+
+        flush();
+
+        // Overwrite the most similar 50 rows
+        for (int i = 0; i < 50; i++)
+            execute("INSERT INTO %s (pk, str_val, val) VALUES (?, 'B', ?)", i, vector(1, i));
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT pk FROM %s WHERE str_val = 'A' ORDER BY val ann of [1.0, 1.0] LIMIT 1"),
+                       row(50));
+        });
+    }
+
+    @Test
+    public void testBruteForceRangeQueryWithUpdatedVectors1536D() throws Throwable
+    {
+        testBruteForceRangeQueryWithUpdatedVectors(1536);
+    }
+
+    @Test
+    public void testBruteForceRangeQueryWithUpdatedVectors2D() throws Throwable
+    {
+        testBruteForceRangeQueryWithUpdatedVectors(2);
+    }
+
+    private void testBruteForceRangeQueryWithUpdatedVectors(int vectorDimension) throws Throwable
+    {
+        setMaxBruteForceRows(0);
+        createTable("CREATE TABLE %s (pk int, val vector<float, " + vectorDimension + ">, PRIMARY KEY(pk))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // Insert 100 vectors
+        for (int i = 0; i < 100; i++)
+            execute("INSERT INTO %s (pk, val) VALUES (?, ?)", i, randomVectorBoxed(vectorDimension));
+
+        // Update those vectors so some ordinals are changed
+        for (int i = 0; i < 100; i++)
+            execute("INSERT INTO %s (pk, val) VALUES (?, ?)", i, randomVectorBoxed(vectorDimension));
+
+        // Delete the first 50 PKs.
+        for (int i = 0; i < 50; i++)
+            execute("DELETE FROM %s WHERE pk = ?", i);
+
+        // All of the above inserts and deletes are performed on the same index to verify internal index behavior
+        // for both memtables and sstables.
+        beforeAndAfterFlush(() -> {
+            // Query for the first 10 vectors, we don't care which.
+            // Use a range query to hit the right brute force code path
+            var results = execute("SELECT pk FROM %s WHERE token(pk) < 0 ORDER BY val ann of ? LIMIT 10",
+                                  randomVectorBoxed(vectorDimension));
+            assertThat(results).hasSize(10);
+            // Make sure we don't get any of the deleted PKs
+            assertThat(results).allSatisfy(row -> assertThat(row.getInt("pk")).isGreaterThanOrEqualTo(50));
+        });
+    }
+
+    @Test
+    public void testVectorIndexWithAllOrdinalsDeletedViaRangeDeletion()
+    {
+        QueryController.QUERY_OPT_LEVEL = 0;
+        setMaxBruteForceRows(0);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, a int, str_val text, val vector<float, 3>, PRIMARY KEY(pk, a))");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        disableCompaction(KEYSPACE);
+
+        // Insert a row with a vector
+        execute("INSERT INTO %s (pk, a, str_val, val) VALUES (1, 1, 'A', [1.0, 2.0, 3.0])");
+
+        // Range delete that row
+        execute("DELETE FROM %s WHERE pk = 1");
+
+        // Insert another row without a vector
+        execute("INSERT INTO %s (pk, a, str_val) VALUES (2, 1, 'A')");
+        flush();
+
+        assertRows(execute("SELECT PK FROM %s WHERE str_val = 'A' ORDER BY val ann of [1.0, 2.0, 3.0] LIMIT 1"));
+    }
+
+    @Test
+    public void testVectorIndexWithAllOrdinalsDeletedAndSomeViaRangeDeletion()
+    {
+        QueryController.QUERY_OPT_LEVEL = 0;
+        setMaxBruteForceRows(0);
+        createTable(KEYSPACE, "CREATE TABLE %s (pk int, a int, str_val text, val vector<float, 3>, PRIMARY KEY(pk, a))");
+        createIndex("CREATE CUSTOM INDEX ON %s(str_val) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        disableCompaction(KEYSPACE);
+
+        // Insert two rows with different vectors to get different ordinals
+        execute("INSERT INTO %s (pk, a, str_val, val) VALUES (1, 1, 'A', [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, a, str_val, val) VALUES (2, 1, 'A', [1.0, 2.0, 4.0])");
+
+        // Range delete the first row
+        execute("DELETE FROM %s WHERE pk = 1");
+        // Specifically delete the vector column second to hit a different code path.
+        execute("DELETE FROM %s WHERE pk = 2 AND a = 1");
+
+        // Insert another row without a vector
+        execute("INSERT INTO %s (pk, a, str_val) VALUES (2, 1, 'A')");
+        flush();
+
+        assertRows(execute("SELECT PK FROM %s WHERE str_val = 'A' ORDER BY val ann of [1.0, 2.0, 3.0] LIMIT 1"));
+    }
+
+    @Test
+    public void ensureCompressedVectorsCanFlush()
+    {
+        createTable("CREATE TABLE %s (pk int, val vector<float, 4>, PRIMARY KEY(pk))");
+        var indexName = createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        // insert enough vectors for pq plus 1 because we need quantization and we're deleting a row
+        for (int i = 0; i < MIN_PQ_ROWS + 1; i++)
+            execute("INSERT INTO %s (pk, val) VALUES (?, ?)", i, vector(randomVector(4)));
+
+        // Delete a single vector to trigger the regression
+        execute("DELETE from %s WHERE pk = 0");
+
+        flush();
+
+        verifySSTableIndexes(indexName, 1);
+    }
+
+    @Test
+    public void testTTLOverwriteHasCorrectOnDiskRowCount() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, val vector<float, 3>)");
+        var indexName = createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, val) VALUES (0, [1.0, 2.0, 3.0]) USING TTL 1");
+
+        // Let the ttl expire
+        Thread.sleep(1000);
+
+        execute("INSERT INTO %s (pk, val) VALUES (0, [2, 3, 4])");
+
+        var sai = (StorageAttachedIndex) Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).getIndexManager().getIndexByName(indexName);
+        var indexes = sai.getIndexContext().getLiveMemtables().values();
+        assertEquals("Expect just one memtable index", 1, indexes.size());
+        var vectorIndex = (VectorMemtableIndex) indexes.iterator().next();
+        assertEquals("We dont' remove vectors, so we're still stuck with it", 2, vectorIndex.size());
+
+        // Flush to build the on disk graph (before the fix, flush failed due to a row having two vectors)
+        flush();
+
+        // Ensure that we only have one vector
+        assertEquals("The TTL'd row is overwritten and removed during flush.", 1, sai.getIndexContext().getCellCount());
+    }
+
+    // This test mimics having rf > 1.
+    @Test
+    public void testSameRowInMultipleSSTablesWithSameTimestamp() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, val vector<float, 3>, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        // We don't want compaction preventing us from hitting the intended code path.
+        disableCompaction();
+
+        // This test is fairly contrived, but covers the case where the first row we attempt to materialize in the
+        // ScoreOrderedResultRetriever is shadowed by a row in a different sstable. And then, when we go to pull in
+        // the next row, we find that the PK is already pulled in, so we need to skip it.
+        execute("INSERT INTO %s (pk, ck, val) VALUES (0, 0, [1.0, 2.0, 3.0])");
+        execute("INSERT INTO %s (pk, ck, val) VALUES (0, 1, [1.0, 2.0, 3.0]) USING TIMESTAMP 1");
+        flush();
+        // Now, delete row pk=0, ck=0 so that we can test that the shadowed row is not returned and that we need
+        // to get the next row from the score ordered iterator.
+        execute("DELETE FROM %s WHERE pk = 0 AND ck = 0");
+        execute("INSERT INTO %s (pk, ck, val) VALUES (0, 1, [1.0, 2.0, 3.0]) USING TIMESTAMP 1");
+
+        beforeAndAfterFlush(() -> {
+            assertRows(execute("SELECT ck FROM %s ORDER BY val ANN OF [1.0, 2.0, 3.0] LIMIT 2"), row(1));
+        });
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/DataModel.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/DataModel.java
new file mode 100644
index 000000000000..6c31d7f006fd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/DataModel.java
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ForwardingList;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.utils.Pair;
+
+public interface DataModel
+{
+    static final String KEYSPACE = "sai_query_keyspace";
+
+    String SIMPLE_SELECT_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? LIMIT ?";
+    String SIMPLE_SELECT_WITH_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? LIMIT ? ALLOW FILTERING";
+    String TWO_CLAUSE_AND_QUERY_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ?";
+    String TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+    String THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+
+    String ASCII_COLUMN = "abbreviation";
+    String BIGINT_COLUMN = "gdp";
+    String BOOLEAN_COLUMN = "active";
+    String DATE_COLUMN = "visited";
+    String DECIMAL_COLUMN = "price";
+    String DOUBLE_COLUMN = "area_sq_miles";
+    String FLOAT_COLUMN = "murder_rate";
+    String INET_COLUMN = "ip";
+    String INT_COLUMN = "population";
+    String SMALLINT_COLUMN = "murders_per_year";
+    String TINYINT_COLUMN = "tiny_murders_per_year";
+    String TEXT_COLUMN = "name";
+    String TIME_COLUMN = "avg_dmv_wait";
+    String TIMESTAMP_COLUMN = "visited_timestamp";
+    String UUID_COLUMN = "id";
+    String TIMEUUID_COLUMN = "temporal_id";
+    String NON_INDEXED_COLUMN = "non_indexed";
+
+    Set<String> skipColumns = Sets.newHashSet(NON_INDEXED_COLUMN, BOOLEAN_COLUMN);
+
+    int DEFAULT_TTL_SECONDS = 10;
+
+    List<Pair<String, String>> NORMAL_COLUMNS =
+            ImmutableList.<Pair<String, String>>builder()
+                    .add(Pair.create(ASCII_COLUMN, CQL3Type.Native.ASCII.toString()))
+                    .add(Pair.create(BIGINT_COLUMN, CQL3Type.Native.BIGINT.toString()))
+                    .add(Pair.create(BOOLEAN_COLUMN, CQL3Type.Native.BOOLEAN.toString()))
+                    .add(Pair.create(DATE_COLUMN, CQL3Type.Native.DATE.toString()))
+                    .add(Pair.create(DECIMAL_COLUMN, CQL3Type.Native.DECIMAL.toString()))
+                    .add(Pair.create(DOUBLE_COLUMN, CQL3Type.Native.DOUBLE.toString()))
+                    .add(Pair.create(FLOAT_COLUMN, CQL3Type.Native.FLOAT.toString()))
+                    .add(Pair.create(INET_COLUMN, CQL3Type.Native.INET.toString()))
+                    .add(Pair.create(INT_COLUMN, CQL3Type.Native.INT.toString()))
+                    .add(Pair.create(SMALLINT_COLUMN, CQL3Type.Native.SMALLINT.toString()))
+                    .add(Pair.create(TINYINT_COLUMN, CQL3Type.Native.TINYINT.toString()))
+                    .add(Pair.create(TEXT_COLUMN, CQL3Type.Native.TEXT.toString()))
+                    .add(Pair.create(TIME_COLUMN, CQL3Type.Native.TIME.toString()))
+                    .add(Pair.create(TIMESTAMP_COLUMN, CQL3Type.Native.TIMESTAMP.toString()))
+                    .add(Pair.create(UUID_COLUMN, CQL3Type.Native.UUID.toString()))
+                    .add(Pair.create(TIMEUUID_COLUMN, CQL3Type.Native.TIMEUUID.toString()))
+                    .add(Pair.create(NON_INDEXED_COLUMN, CQL3Type.Native.INT.toString()))
+                    .build();
+
+    List<String> NORMAL_COLUMN_DATA =
+            ImmutableList.<String>builder()
+                    .add("'AK',   500000000,  true, '2009-07-15', 300.27, 570640.95,  7.7,   '158.145.20.64',   737709,  164,  16,        'Alaska', '00:18:20', '2009-07-15T00:00:00', e37394dc-d17b-11e8-a8d5-f2801f1b9fd1, acfe5ada-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'AL',  1000000000,  true, '2011-09-13', 100.22,  50645.33,  7.0,   '206.16.212.91',  4853875,   57,   5,       'Alabama', '01:04:00', '2011-09-13T00:00:00', b7373af6-d7c1-45ae-b145-5bf4b5cdd00c, c592c37e-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'AR',  2000000000, false, '2013-06-17', -23.09, 113594.08,  5.5,  '170.94.194.134',  2977853,   99,   9,      'Arkansas', '00:55:23', '2013-06-17T00:00:00', a0daaeb4-c8a2-4c68-9899-e32d08238550, cfaae67a-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'CA',  3000000000,  true, '2012-06-17', 330.27, 155779.22,  4.8,    '67.157.98.46', 38993940, 1861, 117,    'California', '01:30:45', '2012-06-17T00:00:00', 96232af0-0af7-438b-9049-c5a5a944ff93, d7e80692-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'DE',  4000000000, false, '2013-06-17', -23.49,   1948.54,  6.7,   '167.21.128.20',   944076,   63,   6,      'Delaware', '00:23:45', '2013-06-17T00:00:00', b2a0a879-5223-40d2-9671-775ee209b6f2, dd10a5b6-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'ID',  4500000000, false, '2015-06-18', 640.27,  82643.12,  1.8,   '164.165.67.10',  1652828,   30,   3,         'Idaho', '00:18:45', '2015-06-18T00:00:00', c6eec0b0-0eef-40e8-ac38-3a82110443e4, e2788780-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'KY',  4750000000, false, '2018-03-12', 300.93,  39486.34,  4.7,  '205.204.196.64',  4424611,  209,  20,      'Kentucky', '00:45:00', '2018-03-12T00:00:00', 752355f8-405b-4d94-88f3-9992cda30f1e, e7c4e1d4-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'LA',  4800000000,  true, '2013-06-10', 110.22,  43203.90, 10.2,  '204.196.242.71',  4668960,  474,  47,     'Louisiana', '00:56:07', '2013-06-10T00:00:00', 17be691a-c1a4-4467-a4ad-64605c74fb1c, ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'MA',  5000000000,  true, '2010-07-04', -40.27,   7800.06,  1.9,   '170.63.206.57',  6784240,  126,  12, 'Massachusetts', '01:01:34', '2010-07-04T00:00:00', e8a3c287-78cf-46b5-b554-42562e7dcfb3, f57a3b62-d17c-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'MI',  6000000000, false, '2011-09-13', 350.37,  56538.90,  5.8,    '23.72.184.64',  9917715,  571,  57,      'Michigan', '00:43:09', '2011-09-13T00:00:00', a0daaeb4-c8a2-4c68-9899-e32d08238550, 0497b886-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'MS',  7000000000,  true, '2013-06-17', 700.17,  46923.27,  5.3,   '192.251.58.38',  2989390,  159,  15,   'Mississippi', '01:04:23', '2013-06-17T00:00:00', 96232af0-0af7-438b-9049-c5a5a944ff93, 0b0205e6-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'PA',  7500000000, false, '2015-03-10', 340.27,  41034.90,  3.6, '201.143.220.137',  6593956,  202,  23,  'Pennsylvania', '00:49:45', '2015-03-10T00:00:00', 9d4e93ed-a14e-4b9d-8749-739019195bdf, 22002940-9084-11ef-a7b4-325096b39f47, 2")
+                    .add("'TN',  8000000000, false, '2018-03-10', 900.27,  41234.90,  6.1, '170.141.221.177',  6595056,  402,  40,     'Tennessee', '00:39:45', '2018-03-10T00:00:00', b2a0a879-5223-40d2-9671-775ee209b6f2, 105dc746-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'TX',  9000000000,  true, '2014-06-17', 100.92, 261231.71,  4.7,   '204.66.40.181', 27429639, 1276, 107,         'Texas', '00:38:13', '2014-06-17T00:00:00', c6eec0b0-0eef-40e8-ac38-3a82110443e4, 155b6bcc-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'UT',  9250000000,  true, '2014-06-20', -23.19,  82169.62,  1.8,   '204.113.13.48',  2990632,   54,   5,          'Utah', '00:25:00', '2014-06-20T00:00:00', 752355f8-405b-4d94-88f3-9992cda30f1e, 1a267c50-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'VA',  9500000000,  true, '2018-06-19', 120.32,  39490.09,  4.6,  '152.130.96.221',  8367587,  383,  38,      'Virginia', '00:43:07', '2018-06-19T00:00:00', 17be691a-c1a4-4467-a4ad-64605c74fb1c, 1fc81a4c-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'WY', 10000000000, false, '2015-06-17', 550.25,  97093.14,  2.7,  '192.146.215.91',   586107,   57,   5,       'Wyoming', '00:15:50', '2015-06-17T00:00:00', e8a3c287-78cf-46b5-b554-42562e7dcfb3, 2576612e-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .build();
+
+    String STATIC_INT_COLUMN = "entered";
+
+    List<Pair<String, String>> STATIC_COLUMNS =
+            ImmutableList.<Pair<String, String>>builder().add(Pair.create(STATIC_INT_COLUMN, CQL3Type.Native.INT.toString() + " static"))
+                                                         .addAll(NORMAL_COLUMNS).build();
+
+    List<String> STATIC_COLUMN_DATA = ImmutableList.of("1819, " + NORMAL_COLUMN_DATA.get(0),
+                                                       "1819, " + NORMAL_COLUMN_DATA.get(1),
+                                                       "1850, " + NORMAL_COLUMN_DATA.get(2),
+                                                       "1850, " + NORMAL_COLUMN_DATA.get(3),
+                                                       "1910, " + NORMAL_COLUMN_DATA.get(4),
+                                                       "1910, " + NORMAL_COLUMN_DATA.get(5),
+                                                       "1792, " + NORMAL_COLUMN_DATA.get(6),
+                                                       "1792, " + NORMAL_COLUMN_DATA.get(7),
+                                                       "1788, " + NORMAL_COLUMN_DATA.get(8),
+                                                       "1788, " + NORMAL_COLUMN_DATA.get(9),
+                                                       "1817, " + NORMAL_COLUMN_DATA.get(10),
+                                                       "1817, " + NORMAL_COLUMN_DATA.get(11),
+                                                       "1896, " + NORMAL_COLUMN_DATA.get(12),
+                                                       "1896, " + NORMAL_COLUMN_DATA.get(13),
+                                                       "1845, " + NORMAL_COLUMN_DATA.get(14),
+                                                       "1845, " + NORMAL_COLUMN_DATA.get(15),
+                                                       "1905, " + NORMAL_COLUMN_DATA.get(16));
+
+    static AtomicInteger seq = new AtomicInteger();
+
+    DataModel withTableOptions(String tableOptions) throws Throwable;
+
+    String indexedTable();
+
+    String nonIndexedTable();
+
+    List<Pair<String, String>> keyColumns();
+
+    void createTables(Executor tester) throws Throwable;
+
+    void createIndexes(Executor tester) throws Throwable;
+
+    void flush(Executor tester) throws Throwable;
+
+    void disableCompaction(Executor tester) throws Throwable;
+
+    void compact(Executor tester) throws Throwable;
+
+    void truncateTables(Executor tester) throws Throwable;
+
+    void insertRows(Executor tester) throws Throwable;
+
+    void insertRowsWithTTL(Executor tester) throws Throwable;
+
+    void updateCells(Executor tester) throws Throwable;
+
+    void deleteCells(Executor tester) throws Throwable;
+
+    void deleteRows(Executor tester) throws Throwable;
+
+    List<Object> executeIndexed(Executor tester, String query, int fetchSize, Object... values) throws Throwable;
+
+    List<Object> executeNonIndexed(Executor tester, String query, int fetchSize, Object... values) throws Throwable;
+
+    public class BaseDataModel implements DataModel
+    {
+        final List<Pair<String, String>> columns;
+        final String columnNames;
+        final List<String> rows;
+        final String indexedTable = "table_" + seq.getAndIncrement();
+        final String nonIndexedTable = "table_" + seq.getAndIncrement();
+
+        String tableOptions = "";
+
+        List<Pair<String, String>> keyColumns;
+        String primaryKey;
+        List<String> keys;
+
+        public BaseDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            this.keyColumns = ImmutableList.of(Pair.create("p", "int"));
+            this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+
+            this.columns = columns;
+            this.columnNames = columns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+            this.rows = rows;
+
+            this.keys = new SimplePrimaryKeyList(rows.size());
+        }
+
+        public String indexedTable()
+        {
+            return indexedTable;
+        }
+
+        public String nonIndexedTable()
+        {
+            return nonIndexedTable;
+        }
+
+        public DataModel withTableOptions(String tableOptions)
+        {
+            this.tableOptions = tableOptions;
+            return this;
+        }
+
+        public List<Pair<String, String>> keyColumns()
+        {
+            return keyColumns;
+        }
+
+        public void createTables(Executor tester)
+        {
+            String keyColumnDefs = keyColumns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+            String normalColumnDefs = columns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+
+            String template = "CREATE TABLE %s (%s, %s, PRIMARY KEY (%s))" + tableOptions;
+            tester.createTable(String.format(template, KEYSPACE + "." + indexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+            tester.createTable(String.format(template, KEYSPACE + "." + nonIndexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+        }
+
+        public void truncateTables(Executor tester) throws Throwable
+        {
+            executeLocal(tester, "TRUNCATE TABLE %s");
+            executeLocal(tester, "TRUNCATE TABLE %s");
+        }
+
+        public void createIndexes(Executor tester) throws Throwable
+        {
+            String template = "CREATE CUSTOM INDEX ndi_%s_index_%s ON %%s (%s) USING 'StorageAttachedIndex'";
+
+            for (Pair<String, String> column : columns)
+            {
+                if (!skipColumns.contains(column.left))
+                {
+                    executeLocalIndexed(tester, String.format(template, column.left, indexedTable, column.left));
+                }
+            }
+            tester.waitForTableIndexesQueryable(KEYSPACE, indexedTable);
+        }
+
+        public void flush(Executor tester) throws Throwable
+        {
+            tester.flush(KEYSPACE, indexedTable);
+            tester.flush(KEYSPACE, nonIndexedTable);
+        }
+
+        public void disableCompaction(Executor tester) throws Throwable
+        {
+            tester.disableCompaction(KEYSPACE, indexedTable);
+            tester.disableCompaction(KEYSPACE, nonIndexedTable);
+        }
+
+        public void compact(Executor tester) throws Throwable
+        {
+            tester.compact(KEYSPACE, indexedTable);
+            tester.compact(KEYSPACE, nonIndexedTable);
+        }
+
+        public void insertRows(Executor tester) throws Throwable
+        {
+            String template = "INSERT INTO %%s (%s, %s) VALUES (%s, %s)";
+
+            for (int i = 0; i < keys.size(); i++)
+            {
+                executeLocal(tester, String.format(template, primaryKey, columnNames, keys.get(i), rows.get(i)));
+            }
+        }
+
+        public void insertRowsWithTTL(Executor tester) throws Throwable
+        {
+            String template = "INSERT INTO %%s (%s, %s) VALUES (%s, %s)%s";
+
+            for (int i = 0; i < keys.size(); i++)
+            {
+                String ttl = deletable().contains(i) ? " USING TTL " + DEFAULT_TTL_SECONDS : "";
+                executeLocal(tester, String.format(template, primaryKey, columnNames, keys.get(i), rows.get(i), ttl));
+            }
+        }
+
+        public void updateCells(Executor tester) throws Throwable
+        {
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 9700000000 WHERE p = 0", BIGINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = false WHERE p = 1", BOOLEAN_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '2018-03-10' WHERE p = 2", DATE_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 28.3 WHERE p = 3", DECIMAL_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 8788.06 WHERE p = 4", DOUBLE_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 2.9 WHERE p = 5", FLOAT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '205.204.196.65' WHERE p = 6", INET_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 27429638 WHERE p = 7", INT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 31 WHERE p = 8", SMALLINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 116 WHERE p = 9", TINYINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 'State of Michigan' WHERE p = 10", TEXT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '00:20:26' WHERE p = 11", TIME_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '2009-07-16T00:00:00' WHERE p = 12", TIMESTAMP_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = e37394dc-d17b-11e8-a8d5-f2801f1b9fd1 WHERE p = 13", UUID_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 1fc81a4c-d17d-11e8-a8d5-f2801f1b9fd1 WHERE p = 14", TIMEUUID_COLUMN));
+        }
+
+        public void deleteCells(Executor tester) throws Throwable
+        {
+            for (int i = 0; i < NORMAL_COLUMNS.size(); i++)
+            {
+                executeLocal(tester, String.format("DELETE %s FROM %%s WHERE p = %s", NORMAL_COLUMNS.get(i).left, i));
+            }
+        }
+
+        public void deleteRows(Executor tester) throws Throwable
+        {
+            String template = "DELETE FROM %%s WHERE p = %d";
+
+            for (int deleted : deletable())
+            {
+                executeLocal(tester, String.format(template, deleted));
+            }
+        }
+
+        public void executeLocal(Executor tester, String query, Object... values) throws Throwable
+        {
+            tester.executeLocal(formatIndexedQuery(query), values);
+            tester.executeLocal(formatNonIndexedQuery(query), values);
+        }
+
+        public void executeLocalIndexed(Executor tester, String query, Object... values) throws Throwable
+        {
+            tester.executeLocal(formatIndexedQuery(query), values);
+        }
+
+        public List<Object> executeIndexed(Executor tester, String query, int fetchSize, Object... values) throws Throwable
+        {
+            return tester.executeRemote(formatIndexedQuery(query), fetchSize, values);
+        }
+
+        public List<Object> executeNonIndexed(Executor tester, String query, int fetchSize, Object... values) throws Throwable
+        {
+            return tester.executeRemote(formatNonIndexedQuery(query), fetchSize, values);
+        }
+
+        protected Set<Integer> deletable()
+        {
+            return Sets.newHashSet(3, 7, 9, 12);
+        }
+
+        private String formatIndexedQuery(String query)
+        {
+            return indexedTable == null ? query : String.format(query, KEYSPACE + "." + indexedTable);
+        }
+
+        private String formatNonIndexedQuery(String query)
+        {
+            return nonIndexedTable == null ? query : String.format(query, KEYSPACE + "." + nonIndexedTable);
+        }
+
+        @Override
+        public String toString()
+        {
+            return MoreObjects.toStringHelper(this).add("primaryKey", primaryKey).toString();
+        }
+    }
+
+    public class CompoundKeyWithStaticsDataModel extends CompoundKeyDataModel
+    {
+        public CompoundKeyWithStaticsDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 2);
+        }
+
+        @Override
+        public void insertRows(Executor tester) throws Throwable
+        {
+            super.insertRows(tester);
+
+            executeLocal(tester, String.format("INSERT INTO %%s (p, %s) VALUES(100, 2019)", DataModel.STATIC_INT_COLUMN)); // static only
+        }
+
+        @Override
+        public void updateCells(Executor tester) throws Throwable
+        {
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 9700000000 WHERE p = 0 AND c = 0", BIGINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = false WHERE p = 0 AND c = 1", BOOLEAN_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '2018-03-10' WHERE p = 1 AND c = 0", DATE_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 8788.06 WHERE p = 1 AND c = 1", DOUBLE_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 2.9 WHERE p = 2 AND c = 0", FLOAT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '205.204.196.65' WHERE p = 2 AND c = 1", INET_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 27429638 WHERE p = 3 AND c = 0", INT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 31 WHERE p = 3 AND c = 1", SMALLINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 116 WHERE p = 4 AND c = 0", TINYINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 'State of Michigan' WHERE p = 4 AND c = 1", TEXT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '00:20:26' WHERE p = 5 AND c = 0", TIME_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '2009-07-16T00:00:00' WHERE p = 5 AND c = 1", TIMESTAMP_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = e37394dc-d17b-11e8-a8d5-f2801f1b9fd1 WHERE p = 6 AND c = 0", UUID_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 1fc81a4c-d17d-11e8-a8d5-f2801f1b9fd1 WHERE p = 6 AND c = 1", TIMEUUID_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 28.3 WHERE p = 7 AND c = 0", DECIMAL_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 1896 WHERE p = 8", STATIC_INT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 2020 WHERE p = 100", STATIC_INT_COLUMN)); // static only
+        }
+
+        @Override
+        public void deleteCells(Executor tester) throws Throwable
+        {
+            for (int i = 0; i < NORMAL_COLUMNS.size(); i++)
+            {
+                String[] primaryKey = keys.get(i).split(",");
+                executeLocal(tester, String.format("DELETE %s FROM %%s WHERE p = %s AND c = %s", NORMAL_COLUMNS.get(i).left, primaryKey[0], primaryKey[1]));
+            }
+        }
+
+        @Override
+        public void deleteRows(Executor tester) throws Throwable
+        {
+            executeLocal(tester, "DELETE FROM %s WHERE p = 2 AND c = 0");
+            executeLocal(tester, "DELETE FROM %s WHERE p = 4 AND c = 0");
+            executeLocal(tester, "DELETE FROM %s WHERE p = 6");
+        }
+
+        @Override
+        protected Set<Integer> deletable()
+        {
+            return Sets.newHashSet(4, 8, 12, 13, 100);
+        }
+    }
+
+    public class CompositePartitionKeyDataModel extends BaseDataModel
+    {
+        public CompositePartitionKeyDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keyColumns = ImmutableList.of(Pair.create("p1", "int"), Pair.create("p2", "int"));
+            this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 2);
+        }
+
+        @Override
+        public void createTables(Executor tester)
+        {
+            String keyColumnDefs = keyColumns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+            String normalColumnDefs = columns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+
+            String template = "CREATE TABLE %s (%s, %s, PRIMARY KEY ((%s)))" + tableOptions;
+            tester.createTable(String.format(template, KEYSPACE + "." + indexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+            tester.createTable(String.format(template, KEYSPACE + "." + nonIndexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+        }
+
+        @Override
+        public void createIndexes(Executor tester) throws Throwable
+        {
+            super.createIndexes(tester);
+            String template = "CREATE CUSTOM INDEX ndi_%s_index_%s ON %%s (%s) USING 'StorageAttachedIndex'";
+
+            for (Pair<String, String> column : keyColumns)
+            {
+                if (!skipColumns.contains(column.left))
+                {
+                    executeLocalIndexed(tester, String.format(template, column.left, indexedTable, column.left));
+                    tester.waitForTableIndexesQueryable(KEYSPACE, indexedTable);
+                }
+            }
+        }
+
+        @Override
+        public void insertRows(Executor tester) throws Throwable
+        {
+            super.insertRows(tester);
+        }
+
+        @Override
+        public void updateCells(Executor tester) throws Throwable
+        {
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 9700000000 WHERE p1 = 0 AND p2 = 0", BIGINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = false WHERE p1 = 0 AND p2 = 1", BOOLEAN_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '2018-03-10' WHERE p1 = 1 AND p2 = 0", DATE_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 8788.06 WHERE p1 = 1 AND p2 = 1", DOUBLE_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 2.9 WHERE p1 = 2 AND p2 = 0", FLOAT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '205.204.196.65' WHERE p1 = 2 AND p2 = 1", INET_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 27429638 WHERE p1 = 3 AND p2 = 0", INT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 31 WHERE p1 = 3 AND p2 = 1", SMALLINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 116 WHERE p1 = 4 AND p2 = 0", TINYINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 'State of Michigan' WHERE p1 = 4 AND p2 = 2", TEXT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '00:20:26' WHERE p1 = 5 AND p2 = 3", TIME_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '2009-07-16T00:00:00' WHERE p1 = 5 AND p2 = 1", TIMESTAMP_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = e37394dc-d17b-11e8-a8d5-f2801f1b9fd1 WHERE p1 = 6 AND p2 = 0", UUID_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 1fc81a4c-d17d-11e8-a8d5-f2801f1b9fd1 WHERE p1 = 6 AND p2 = 1", TIMEUUID_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 28.3 WHERE p1 = 7 AND p2 = 0", DECIMAL_COLUMN));
+        }
+
+        @Override
+        public void deleteCells(Executor tester) throws Throwable
+        {
+            for (int i = 0; i < NORMAL_COLUMNS.size(); i++)
+            {
+                String[] primaryKey = keys.get(i).split(",");
+                executeLocal(tester, String.format("DELETE %s FROM %%s WHERE p1 = %s AND p2 = %s",
+                                                   NORMAL_COLUMNS.get(i).left, primaryKey[0], primaryKey[1]));
+            }
+        }
+
+        @Override
+        public void deleteRows(Executor tester) throws Throwable
+        {
+            executeLocal(tester, "DELETE FROM %s WHERE p1 = 2 AND p2 = 0");
+            executeLocal(tester, "DELETE FROM %s WHERE p1 = 4 AND p2 = 1");
+            executeLocal(tester, "DELETE FROM %s WHERE p1 = 6 AND p2 = 2");
+            executeLocal(tester, "DELETE FROM %s WHERE p1 = 8 AND p2 = 0");
+        }
+
+        @Override
+        protected Set<Integer> deletable()
+        {
+            // already overwrites {@code deleteRows()}
+            return Collections.emptySet();
+        }
+    }
+
+    public class CompoundKeyDataModel extends BaseDataModel
+    {
+        public CompoundKeyDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keyColumns = ImmutableList.of(Pair.create("p", "int"), Pair.create("c", "int"));
+            this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 1);
+        }
+
+        @Override
+        public void updateCells(Executor tester) throws Throwable
+        {
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 9700000000 WHERE p = 0 AND c = 0", BIGINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = false WHERE p = 1 AND c = 0", BOOLEAN_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '2018-03-10' WHERE p = 2 AND c = 0", DATE_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 8788.06 WHERE p = 3 AND c = 0", DOUBLE_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 2.9 WHERE p = 4 AND c = 0", FLOAT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '205.204.196.65' WHERE p = 5 AND c = 0", INET_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 27429638 WHERE p = 6 AND c = 0", INT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 31 WHERE p = 7 AND c = 0", SMALLINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 116 WHERE p = 8 AND c = 0", TINYINT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 'State of Michigan' WHERE p = 9 AND c = 0", TEXT_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '00:20:26' WHERE p = 10 AND c = 0", TIME_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = '2009-07-16T00:00:00' WHERE p = 11 AND c = 0", TIMESTAMP_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = e37394dc-d17b-11e8-a8d5-f2801f1b9fd1 WHERE p = 12 AND c = 0", UUID_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 1fc81a4c-d17d-11e8-a8d5-f2801f1b9fd1 WHERE p = 13 AND c = 0", TIMEUUID_COLUMN));
+            executeLocal(tester, String.format("UPDATE %%s SET %s = 28.3 WHERE p = 14 AND c = 0", DECIMAL_COLUMN));
+        }
+
+        @Override
+        public void deleteCells(Executor tester) throws Throwable
+        {
+            for (int i = 0; i < NORMAL_COLUMNS.size(); i++)
+            {
+                executeLocal(tester, String.format("DELETE %s FROM %%s WHERE p = %s AND c = 0", NORMAL_COLUMNS.get(i).left, i));
+            }
+        }
+    }
+
+    class SimplePrimaryKeyList extends ForwardingList<String>
+    {
+        private final List<String> primaryKeys;
+
+        SimplePrimaryKeyList(int rows)
+        {
+            this.primaryKeys = IntStream.range(0, rows).mapToObj(String::valueOf).collect(Collectors.toList());
+        }
+
+        @Override
+        protected List<String> delegate()
+        {
+            return primaryKeys;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("SimplePrimaryKeyList[rows: %d]", primaryKeys.size());
+        }
+    }
+
+    class CompoundPrimaryKeyList extends ForwardingList<String>
+    {
+        private final List<String> primaryKeys;
+        private final int rowsPerPartition;
+
+        CompoundPrimaryKeyList(int rows, int rowsPerPartition)
+        {
+            this.primaryKeys = IntStream.range(0, rows).mapToObj(v -> v / rowsPerPartition + ", " + v % rowsPerPartition).collect(Collectors.toList());
+            this.rowsPerPartition = rowsPerPartition;
+        }
+
+        @Override
+        protected List<String> delegate()
+        {
+            return primaryKeys;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("CompoundPrimaryKeyList[rows: %d, partition size: %d]", primaryKeys.size(), rowsPerPartition);
+        }
+    }
+
+    public static interface Executor
+    {
+        void createTable(String statement);
+
+        void flush(String keyspace, String table);
+
+        void compact(String keyspace, String table);
+
+        void disableCompaction(String keyspace, String table);
+
+        void waitForTableIndexesQueryable(String keyspace, String table);
+
+        void executeLocal(String query, Object...values) throws Throwable;
+
+        List<Object> executeRemote(String query, int fetchSize, Object...values) throws Throwable;
+
+        void counterReset();
+
+        long getCounter();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/IndexQuerySupport.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/IndexQuerySupport.java
new file mode 100644
index 000000000000..2e82872e112f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/IndexQuerySupport.java
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.SimpleDateType;
+import org.apache.cassandra.db.marshal.TimeType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
+import org.apache.cassandra.utils.Pair;
+import org.hamcrest.Matchers;
+
+import static org.apache.cassandra.distributed.test.TestBaseImpl.list;
+import static org.apache.cassandra.index.sai.cql.datamodels.DataModel.INET_COLUMN;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+/**
+ * A CQL-based test framework for simulating queries across as much of the index state space as possible.
+ *
+ * This includes, but need not be limited to...
+ *
+ * 1.) ...queries on the same data as it migrates through the write path and storage engine.
+ * 2.) ...queries across all supported native data types.
+ * 3.) ...queries for all supported operators and value boundaries.
+ * 4.) ...queries for varying write, update, delete, and TTL workloads.
+ * 5.) ...queries across varying primary key and table structures.
+ * 6.) ...queries across static, normal, and clustering column types.
+ * 7.) ...queries across various paging and limit settings.
+ *
+ * IMPORTANT: This class is shared between the single-node SAITester based classes and the
+ * multi-node distributed classes. It must not reference SAITester or CQLTester directly
+ * to avoid static loading and initialisation.
+ */
+public class IndexQuerySupport
+{
+    public static List<BaseQuerySet> BASE_QUERY_SETS = ImmutableList.of(new BaseQuerySet(10, 5),
+                                                                        new BaseQuerySet(10, 9),
+                                                                        new BaseQuerySet(10, 10),
+                                                                        new BaseQuerySet(10, Integer.MAX_VALUE),
+                                                                        new BaseQuerySet(24, 10),
+                                                                        new BaseQuerySet(24, 100),
+                                                                        new BaseQuerySet(24, Integer.MAX_VALUE));
+
+    public static List<BaseQuerySet> COMPOSITE_PARTITION_QUERY_SETS = ImmutableList.of(new CompositePartitionQuerySet(10, 5),
+                                                                                       new CompositePartitionQuerySet(10, 10),
+                                                                                       new CompositePartitionQuerySet(10, Integer.MAX_VALUE),
+                                                                                       new CompositePartitionQuerySet(24, 10),
+                                                                                       new CompositePartitionQuerySet(24, 100),
+                                                                                       new CompositePartitionQuerySet(24, Integer.MAX_VALUE));
+
+    public static List<BaseQuerySet> STATIC_QUERY_SETS = ImmutableList.of(new StaticColumnQuerySet(10, 5),
+                                                                          new StaticColumnQuerySet(10, 10),
+                                                                          new StaticColumnQuerySet(10, Integer.MAX_VALUE),
+                                                                          new StaticColumnQuerySet(24, 10),
+                                                                          new StaticColumnQuerySet(24, 100),
+                                                                          new StaticColumnQuerySet(24, Integer.MAX_VALUE));
+
+    public static void writeLifecycle(DataModel.Executor executor, DataModel dataModel, List<BaseQuerySet> sets) throws Throwable
+    {
+        dataModel.createTables(executor);
+
+        dataModel.disableCompaction(executor);
+
+        dataModel.createIndexes(executor);
+
+        // queries against Memtable adjacent in-memory indexes
+        dataModel.insertRows(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries with Memtable flushed to SSTable on disk
+        dataModel.flush(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries across memory and disk indexes
+        dataModel.insertRows(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries w/ multiple SSTable indexes
+        dataModel.flush(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries after compacting to a single SSTable index
+        dataModel.compact(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries against Memtable updates and the existing SSTable index
+        dataModel.updateCells(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries against the newly flushed SSTable index and the existing SSTable index
+        dataModel.flush(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries after compacting updates into to a single SSTable index
+        dataModel.compact(executor);
+        executeQueries(dataModel, executor, sets);
+    }
+
+    public static void rowDeletions(DataModel.Executor executor, DataModel dataModel, List<BaseQuerySet> sets) throws Throwable
+    {
+        dataModel.createTables(executor);
+
+        dataModel.disableCompaction(executor);
+
+        dataModel.createIndexes(executor);
+        dataModel.insertRows(executor);
+        dataModel.flush(executor);
+        dataModel.compact(executor);
+
+        // baseline queries
+        executeQueries(dataModel, executor, sets);
+
+        // queries against Memtable deletes and the existing SSTable index
+        dataModel.deleteRows(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries against the newly flushed SSTable index and the existing SSTable index
+        dataModel.flush(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries after compacting deletes into to a single SSTable index
+        dataModel.compact(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // truncate, reload, and verify that the load is clean
+        dataModel.truncateTables(executor);
+        dataModel.insertRows(executor);
+        executeQueries(dataModel, executor, sets);
+    }
+
+    public static void cellDeletions(DataModel.Executor executor, DataModel dataModel, List<BaseQuerySet> sets) throws Throwable
+    {
+        dataModel.createTables(executor);
+
+        dataModel.disableCompaction(executor);
+
+        dataModel.createIndexes(executor);
+        dataModel.insertRows(executor);
+        dataModel.flush(executor);
+        dataModel.compact(executor);
+
+        // baseline queries
+        executeQueries(dataModel, executor, sets);
+
+        // queries against Memtable deletes and the existing SSTable index
+        dataModel.deleteCells(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries against the newly flushed SSTable index and the existing SSTable index
+        dataModel.flush(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // queries after compacting deletes into to a single SSTable index
+        dataModel.compact(executor);
+        executeQueries(dataModel, executor, sets);
+    }
+
+    public static void timeToLive(DataModel.Executor executor, DataModel dataModel, List<BaseQuerySet> sets) throws Throwable
+    {
+        dataModel.createTables(executor);
+
+        dataModel.disableCompaction(executor);
+
+        dataModel.createIndexes(executor);
+        dataModel.insertRowsWithTTL(executor);
+
+        // Wait for the TTL to become effective:
+        TimeUnit.SECONDS.sleep(DataModel.DEFAULT_TTL_SECONDS);
+
+        // Make sure TTLs are reflected in our query results from the Memtable:
+        executeQueries(dataModel, executor, sets);
+
+        // Make sure TTLs are reflected in our query results from SSTables:
+        dataModel.flush(executor);
+        executeQueries(dataModel, executor, sets);
+
+        // Make sure fresh overwrites invalidate TTLs:
+        dataModel.insertRows(executor);
+        executeQueries(dataModel, executor, sets);
+    }
+
+    private static void executeQueries(DataModel dataModel, DataModel.Executor executor, List<BaseQuerySet> sets) throws Throwable
+    {
+        for (BaseQuerySet set : sets)
+        {
+            set.execute(executor, dataModel);
+        }
+    }
+
+    static class StaticColumnQuerySet extends BaseQuerySet
+    {
+        StaticColumnQuerySet(int limit, int fetchSize)
+        {
+            super(limit, fetchSize);
+        }
+
+        public void execute(DataModel.Executor tester, DataModel model) throws Throwable
+        {
+            super.execute(tester, model);
+
+            query(tester, model, DataModel.STATIC_INT_COLUMN, Operator.EQ, 1845);
+            query(tester, model, DataModel.STATIC_INT_COLUMN, Operator.LT, 1845);
+            query(tester, model, DataModel.STATIC_INT_COLUMN, Operator.LTE, 1845);
+            query(tester, model, DataModel.STATIC_INT_COLUMN, Operator.GT, 1845);
+            query(tester, model, DataModel.STATIC_INT_COLUMN, Operator.GTE, 1845);
+            query(tester, model, DataModel.STATIC_INT_COLUMN, Operator.EQ, 1909);
+            query(tester, model, DataModel.STATIC_INT_COLUMN, Operator.LT, 1787);
+            query(tester, model, DataModel.STATIC_INT_COLUMN, Operator.GT, 1910);
+
+            rangeQuery(tester, model, DataModel.STATIC_INT_COLUMN, 1845, 1909);
+        }
+    }
+
+    static class CompositePartitionQuerySet extends BaseQuerySet
+    {
+        CompositePartitionQuerySet(int limit, int fetchSize)
+        {
+            super(limit, fetchSize);
+        }
+
+        public void execute(DataModel.Executor tester, DataModel model) throws Throwable
+        {
+            super.execute(tester, model);
+
+            DataModel.BaseDataModel baseDataModel = (DataModel.BaseDataModel) model;
+            for(Pair<String, String> partitionKeyComponent: baseDataModel.keyColumns)
+            {
+                String partitionKeyComponentName = partitionKeyComponent.left;
+                query(tester, model, partitionKeyComponentName, Operator.EQ, 0);
+                query(tester, model, partitionKeyComponentName, Operator.GT, 0);
+                query(tester, model, partitionKeyComponentName, Operator.LTE, 2);
+                query(tester, model, partitionKeyComponentName, Operator.GTE, -1);
+                query(tester, model, partitionKeyComponentName, Operator.LT, 50);
+                query(tester, model, partitionKeyComponentName, Operator.GT, 0);
+            }
+
+            String firstPartitionKey = baseDataModel.keyColumns.get(0).left;
+            String secondPartitionKey = baseDataModel.keyColumns.get(1).left;
+            List<Operator> numericOperators = Arrays.asList(Operator.EQ, Operator.GT, Operator.LT, Operator.GTE, Operator.LTE);
+            List<List<Operator>> combinations = Lists.cartesianProduct(numericOperators, numericOperators).stream()
+                                                     .filter(p-> p.get(0) != Operator.EQ || p.get(1) != Operator.EQ) //If both are EQ the entire partition is specified
+                                                     .collect(Collectors.toList());
+            for(List<Operator> operators : combinations)
+            {
+                andQuery(tester,
+                         model,
+                         firstPartitionKey, operators.get(0), 2,
+                         secondPartitionKey, operators.get(1), 2,
+                         false);
+            }
+        }
+    }
+
+    public static class BaseQuerySet
+    {
+        final int limit;
+        final int fetchSize;
+
+        BaseQuerySet(int limit, int fetchSize)
+        {
+            this.limit = limit;
+            this.fetchSize = fetchSize;
+        }
+
+        void execute(DataModel.Executor tester, DataModel model) throws Throwable
+        {
+            query(tester, model, DataModel.ASCII_COLUMN, Operator.EQ, "MA");
+            query(tester, model, DataModel.ASCII_COLUMN, Operator.EQ, "LA");
+            query(tester, model, DataModel.ASCII_COLUMN, Operator.EQ, "XX");
+
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.EQ, 4800000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.EQ, 5000000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.NEQ, 4800000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.NEQ, 5000000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.LT, 5000000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.LTE, 5000000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.GT, 5000000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.GTE, 5000000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.EQ, 22L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.NEQ, 22L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.LT, 400000000L);
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.GT, 10000000000L);
+
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.IN, list(22L, 3000000000L, 5000000000L));
+            query(tester, model, DataModel.BIGINT_COLUMN, Operator.NOT_IN, list(22L, 3000000000L, 5000000000L));
+
+            rangeQuery(tester, model, DataModel.BIGINT_COLUMN, 3000000000L, 7000000000L);
+
+            query(tester, model, DataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2013-06-10"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.NEQ, SimpleDateType.instance.fromString("2013-06-10"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.NEQ, SimpleDateType.instance.fromString("2020-06-17"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.LT, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.LTE, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.GT, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.GTE, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2017-01-01"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.LT, SimpleDateType.instance.fromString("2000-01-01"));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.GT, SimpleDateType.instance.fromString("2020-01-01"));
+
+            query(tester, model, DataModel.DATE_COLUMN, Operator.IN, list(
+                SimpleDateType.instance.fromString("2020-01-01"),
+                SimpleDateType.instance.fromString("2013-06-17"),
+                SimpleDateType.instance.fromString("2018-06-19")
+            ));
+            query(tester, model, DataModel.DATE_COLUMN, Operator.NOT_IN, list(
+                SimpleDateType.instance.fromString("2020-01-01"),
+                SimpleDateType.instance.fromString("2013-06-17"),
+                SimpleDateType.instance.fromString("2018-06-19")
+            ));
+
+            rangeQuery(tester, model, DataModel.DATE_COLUMN, SimpleDateType.instance.fromString("2013-06-17"), SimpleDateType.instance.fromString("2018-06-19"));
+
+            query(tester, model, DataModel.DECIMAL_COLUMN, Operator.EQ, DecimalType.instance.fromString("300.27"));
+            query(tester, model, DataModel.DECIMAL_COLUMN, Operator.EQ, DecimalType.instance.fromString("-23.09"));
+            query(tester, model, DataModel.DECIMAL_COLUMN, Operator.NEQ, DecimalType.instance.fromString("300.27"));
+            query(tester, model, DataModel.DECIMAL_COLUMN, Operator.NEQ, DecimalType.instance.fromString("-23.09"));
+            query(tester, model, DataModel.DECIMAL_COLUMN, Operator.LT, DecimalType.instance.fromString("300.27"));
+            query(tester, model, DataModel.DECIMAL_COLUMN, Operator.LTE, DecimalType.instance.fromString("300.27"));
+            query(tester, model, DataModel.DECIMAL_COLUMN, Operator.GT, DecimalType.instance.fromString("300.27"));
+            query(tester, model, DataModel.DECIMAL_COLUMN, Operator.GTE, DecimalType.instance.fromString("300.27"));
+
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.EQ, 43203.90);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.EQ, 7800.06);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.NEQ, 43203.90);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.NEQ, 7800.06);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.LT, 82169.62);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.LTE, 82169.62);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.GT, 82169.62);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.GTE, 82169.62);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.EQ, 82169.60);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.LT, 1948.54);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.GT, 570640.95);
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.IN, list(43203.90, 7800.06, 82169.62));
+            query(tester, model, DataModel.DOUBLE_COLUMN, Operator.NOT_IN, list(43203.90, 7800.06, 82169.62));
+
+            rangeQuery(tester, model, DataModel.DOUBLE_COLUMN, 56538.90, 113594.08);
+
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.EQ, 10.2f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.EQ, 1.9f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.NEQ, 10.2f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.NEQ, 1.9f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.LT, 5.3f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.LTE, 5.3f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.GT, 5.3f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.GTE, 5.3f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.EQ, 5.9f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.LT, 1.8f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.GT, 10.2f);
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.IN, list(7.7f, 1.8f, 3.5f));
+            query(tester, model, DataModel.FLOAT_COLUMN, Operator.NOT_IN, list(7.7f, 1.8f, 3.5f));
+
+            rangeQuery(tester, model, DataModel.FLOAT_COLUMN, 4.6f, 6.7f);
+
+            query(tester, model, INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("170.63.206.57"));
+            query(tester, model, INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("170.63.206.56"));
+            query(tester, model, INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("205.204.196.65"));
+            query(tester, model, INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("164.165.67.10"));
+            query(tester, model, INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("204.196.242.71"));
+
+            rangeQuery(tester, model, DataModel.INT_COLUMN, 2977853, 6784240);
+
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.EQ, (short) 164);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.NEQ, (short) 164);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.LT, (short) 164);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.LTE, (short) 164);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.GT, (short) 164);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.GTE, (short) 164);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.EQ, (short) 2);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.NEQ, (short) 2);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.LT, (short) 30);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.GT, (short) 1861);
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.IN, list((short) 1861, (short) 164, (short) 30));
+            query(tester, model, DataModel.SMALLINT_COLUMN, Operator.NOT_IN, list((short) 1861, (short) 164, (short) 30));
+
+            rangeQuery(tester, model, DataModel.SMALLINT_COLUMN, (short) 126, (short) 383);
+
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.EQ, (byte) 16);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.NEQ, (byte) 16);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.LT, (byte) 16);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.LTE, (byte) 16);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.GT, (byte) 16);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.GTE, (byte) 16);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.EQ, (byte) 1);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.NEQ, (byte) 1);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.LT, (byte) 2);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.GT, (byte) 117);
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.IN, list((byte) 16, (byte) 1));
+            query(tester, model, DataModel.TINYINT_COLUMN, Operator.NOT_IN, list((byte) 16, (byte) 1));
+
+            rangeQuery(tester, model, DataModel.TINYINT_COLUMN, (byte) 12, (byte) 47);
+
+            query(tester, model, DataModel.TEXT_COLUMN, Operator.EQ, "Alaska");
+            query(tester, model, DataModel.TEXT_COLUMN, Operator.EQ, "Wyoming");
+            query(tester, model, DataModel.TEXT_COLUMN, Operator.EQ, "Franklin");
+            query(tester, model, DataModel.TEXT_COLUMN, Operator.EQ, "State of Michigan");
+            query(tester, model, DataModel.TEXT_COLUMN, Operator.EQ, "Michigan");
+            query(tester, model, DataModel.TEXT_COLUMN, Operator.EQ, "Louisiana");
+            query(tester, model, DataModel.TEXT_COLUMN, Operator.EQ, "Massachusetts");
+
+            query(tester, model, DataModel.TIME_COLUMN, Operator.EQ, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.NEQ, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.LT, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.LTE, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.GT, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.GTE, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.EQ, TimeType.instance.fromString("00:15:57"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.NEQ, TimeType.instance.fromString("00:15:57"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.LT, TimeType.instance.fromString("00:15:50"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.GT, TimeType.instance.fromString("01:30:45"));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.IN, list(TimeType.instance.fromString("00:43:07"), TimeType.instance.fromString("00:15:57")));
+            query(tester, model, DataModel.TIME_COLUMN, Operator.NOT_IN, list(TimeType.instance.fromString("00:43:07"), TimeType.instance.fromString("00:15:57")));
+
+            rangeQuery(tester, model, DataModel.TIME_COLUMN, TimeType.instance.fromString("00:38:13"), TimeType.instance.fromString("00:56:07"));
+
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.EQ, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.NEQ, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.LT, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.LTE, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.GT, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.EQ, TimestampType.instance.fromString("2017-01-01T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.NEQ, TimestampType.instance.fromString("2017-01-01T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.LT, TimestampType.instance.fromString("2000-01-01T00:00:00"));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.GT, TimestampType.instance.fromString("2020-01-01T00:00:00"));
+
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.IN, list(
+                        TimestampType.instance.fromString("2013-06-17T00:00:00"),
+                        TimestampType.instance.fromString("2017-01-01T00:00:00")));
+            query(tester, model, DataModel.TIMESTAMP_COLUMN, Operator.NOT_IN, list(
+                        TimestampType.instance.fromString("2013-06-17T00:00:00"),
+                        TimestampType.instance.fromString("2017-01-01T00:00:00")));
+
+            rangeQuery(tester, model, DataModel.TIMESTAMP_COLUMN,
+                       TimestampType.instance.fromString("2013-6-17T00:00:00"),
+                       TimestampType.instance.fromString("2018-6-19T00:00:00"));
+
+            query(tester, model, DataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("752355f8-405b-4d94-88f3-9992cda30f1e"));
+            query(tester, model, DataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("ac0aa734-d17f-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("c6eec0b0-0eef-40e8-ac38-3a82110443e4"));
+            query(tester, model, DataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"));
+
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.NEQ, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.LT, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.LTE, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.GT, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.GTE, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("2a421a68-d182-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.NEQ, UUIDType.instance.fromString("2a421a68-d182-11e8-a8d5-f2801f1b9fd1"));
+
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.IN, list(
+                    UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"),
+                    UUIDType.instance.fromString("2a421a68-d182-11e8-a8d5-f2801f1b9fd1")));
+            query(tester, model, DataModel.TIMEUUID_COLUMN, Operator.NOT_IN, list(
+                    UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"),
+                    UUIDType.instance.fromString("2a421a68-d182-11e8-a8d5-f2801f1b9fd1")));
+
+            andQuery(tester, model,
+                     DataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2013-06-20T00:00:00"),
+                     DataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("752355f8-405b-4d94-88f3-9992cda30f1e"),
+                     false);
+
+            andQuery(tester, model,
+                     DataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2018-06-20T00:00:00"),
+                     DataModel.TEXT_COLUMN, Operator.EQ, "Texas",
+                     false);
+
+            andQuery(tester, model,
+                     DataModel.TIMESTAMP_COLUMN, Operator.NEQ, TimestampType.instance.fromString("2018-06-20T00:00:00"),
+                     DataModel.TEXT_COLUMN, Operator.EQ, "Texas",
+                     false);
+
+            andQuery(tester, model,
+                     DataModel.SMALLINT_COLUMN, Operator.LTE, (short) 126,
+                     DataModel.TINYINT_COLUMN, Operator.LTE, (byte) 9,
+                     false);
+
+            andQuery(tester, model,
+                     DataModel.SMALLINT_COLUMN, Operator.LTE, (short) 126,
+                     DataModel.TINYINT_COLUMN, Operator.NEQ, (byte) 9,
+                     false);
+
+            andQuery(tester, model,
+                     DataModel.SMALLINT_COLUMN, Operator.LTE, (short) 126,
+                     DataModel.NON_INDEXED_COLUMN, Operator.GT, 0,
+                     true);
+
+            andQuery(tester, model,
+                     DataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
+                     DataModel.NON_INDEXED_COLUMN, Operator.EQ, 2,
+                     true);
+
+            andQuery(tester, model,
+                     DataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"),
+                     DataModel.NON_INDEXED_COLUMN, Operator.LT, 3,
+                     true);
+
+            andQuery(tester, model,
+                     DataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"),
+                     DataModel.NON_INDEXED_COLUMN, Operator.NEQ, 3,
+                     true);
+
+            // with partition column filtering
+            String firstPartitionKey = model.keyColumns().get(0).left;
+
+            andQuery(tester, model,
+                     DataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
+                     firstPartitionKey, Operator.EQ, 0,
+                     true);
+
+            andQuery(tester, model,
+                     DataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
+                     firstPartitionKey, Operator.NEQ, 0,
+                     true);
+
+            andQuery(tester, model,
+                     DataModel.TEXT_COLUMN, Operator.EQ, "Kentucky",
+                     firstPartitionKey, Operator.GT, 4,
+                     true);
+
+            andQuery(tester, model,
+                     DataModel.TEXT_COLUMN, Operator.EQ, "Wyoming",
+                     firstPartitionKey, Operator.LT, 200,
+                     true);
+
+            if (model.keyColumns().size() > 1)
+            {
+                String secondPrimaryKey = model.keyColumns().get(1).left;
+
+                andQuery(tester, model,
+                         DataModel.BIGINT_COLUMN, Operator.EQ, 4800000000L,
+                         secondPrimaryKey, Operator.EQ, 0,
+                         true);
+
+                andQuery(tester, model,
+                         DataModel.DOUBLE_COLUMN, Operator.EQ, 82169.60,
+                         secondPrimaryKey, Operator.NEQ, 0,
+                         true);
+
+                andQuery(tester, model,
+                         DataModel.DOUBLE_COLUMN, Operator.EQ, 82169.60,
+                         secondPrimaryKey, Operator.GT, 0,
+                         true);
+
+                andQuery(tester, model,
+                         DataModel.DOUBLE_COLUMN, Operator.LT, 1948.54,
+                         secondPrimaryKey, Operator.LTE, 2,
+                         true);
+
+                andQuery(tester, model,
+                         DataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
+                         firstPartitionKey, Operator.EQ, 0,
+                         secondPrimaryKey, Operator.GTE, -1);
+
+                andQuery(tester, model,
+                         DataModel.TEXT_COLUMN, Operator.EQ, "Kentucky",
+                         firstPartitionKey, Operator.GT, 4,
+                         secondPrimaryKey, Operator.LT, 50);
+
+                andQuery(tester, model,
+                         DataModel.TEXT_COLUMN, Operator.EQ, "Wyoming",
+                         firstPartitionKey, Operator.LT, 200,
+                         secondPrimaryKey, Operator.GT, 0);
+            }
+        }
+
+        void query(DataModel.Executor tester, DataModel model, String column, Operator operator, Object value) throws Throwable
+        {
+            String query = String.format(DataModel.SIMPLE_SELECT_TEMPLATE, DataModel.ASCII_COLUMN, column, operator);
+            String queryValidator = String.format(DataModel.SIMPLE_SELECT_WITH_FILTERING_TEMPLATE, DataModel.ASCII_COLUMN, column, operator);
+            validate(tester, model, query, queryValidator, value, limit);
+        }
+
+        void andQuery(DataModel.Executor tester, DataModel model,
+                      String column1, Operator operator1, Object value1,
+                      String column2, Operator operator2, Object value2,
+                      boolean filtering) throws Throwable
+        {
+            String query = String.format(filtering ? DataModel.TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE : DataModel.TWO_CLAUSE_AND_QUERY_TEMPLATE,
+                                         DataModel.ASCII_COLUMN, column1, operator1, column2, operator2);
+
+            String queryValidator = String.format(DataModel.TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
+                                                  DataModel.ASCII_COLUMN, column1, operator1, column2, operator2);
+
+            validate(tester, model,query, queryValidator, value1, value2, limit);
+        }
+
+        void andQuery(DataModel.Executor tester, DataModel model,
+                      String column1, Operator operator1, Object value1,
+                      String column2, Operator operator2, Object value2,
+                      String column3, Operator operator3, Object value3) throws Throwable
+        {
+            // TODO: If we support indexes in all columns, ALLOW FILTERING might go away here...
+            String query = String.format(DataModel.THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
+                                         DataModel.ASCII_COLUMN, column1, operator1, column2, operator2, column3, operator3);
+
+            String queryValidator = String.format(DataModel.THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
+                                                  DataModel.ASCII_COLUMN, column1, operator1, column2, operator2, column3, operator3);
+
+            validate(tester, model, query, queryValidator, value1, value2, value3, limit);
+        }
+
+        void rangeQuery(DataModel.Executor tester, DataModel model, String column, Object value1, Object value2) throws Throwable
+        {
+            String template = "SELECT %s FROM %%s WHERE %s > ? AND %s < ? LIMIT ?";
+            String templateWithFiltering = "SELECT %s FROM %%s WHERE %s > ? AND %s < ? LIMIT ? ALLOW FILTERING";
+
+            String query = String.format(template, DataModel.ASCII_COLUMN, column, column);
+            String queryValidator = String.format(templateWithFiltering, DataModel.ASCII_COLUMN, column, column);
+            validate(tester, model, query, queryValidator, value1, value2, limit);
+        }
+
+        private List<Object> validate(DataModel.Executor tester, DataModel model, String query, String validator, Object... values) throws Throwable
+        {
+            try
+            {
+                tester.counterReset();
+
+                List<Object> actual = model.executeIndexed(tester, query, fetchSize, values);
+
+                // This could be more strict, but it serves as a reasonable paging-aware lower bound:
+                int pageCount = (int) Math.ceil(actual.size() / (double) Math.min(actual.size(), fetchSize));
+                assertThat("Expected more calls to " + StorageAttachedIndexSearcher.class, tester.getCounter(), Matchers.greaterThanOrEqualTo((long) Math.max(1, pageCount)));
+
+                List<Object> expected = model.executeNonIndexed(tester, validator, fetchSize, values);
+                assertEquals("Invalid query results for query " + query, expected, actual);
+
+                return expected;
+            }
+            catch (Throwable ex)
+            {
+                ex.printStackTrace();
+                throw ex;
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            return MoreObjects.toStringHelper(this).add("limit", limit).add("fetchSize", fetchSize).toString();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsTester.java
new file mode 100644
index 000000000000..f9aa03d008bb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsTester.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore
+abstract class QueryCellDeletionsTester extends SingleNodeQueryTester
+{
+    @Test
+    public void testCellDeletions() throws Throwable
+    {
+        IndexQuerySupport.cellDeletions(executor, dataModel, sets);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithBaseDataModelTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithBaseDataModelTest.java
new file mode 100644
index 000000000000..616091613ffd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithBaseDataModelTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryCellDeletionsWithBaseDataModelTest extends QueryCellDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompositePartitionKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompositePartitionKeyTest.java
new file mode 100644
index 000000000000..ff4040422a62
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompositePartitionKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryCellDeletionsWithCompositePartitionKeyTest extends QueryCellDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompoundKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompoundKeyTest.java
new file mode 100644
index 000000000000..199242e406db
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompoundKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryCellDeletionsWithCompoundKeyTest extends QueryCellDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompoundKeyWithStaticsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompoundKeyWithStaticsTest.java
new file mode 100644
index 000000000000..cf89805d6d49
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryCellDeletionsWithCompoundKeyWithStaticsTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryCellDeletionsWithCompoundKeyWithStaticsTest extends QueryCellDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsTester.java
new file mode 100644
index 000000000000..97e0073bde8c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsTester.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore
+abstract class QueryRowDeletionsTester extends SingleNodeQueryTester
+{
+    @Test
+    public void testRowDeletions() throws Throwable
+    {
+        IndexQuerySupport.rowDeletions(executor, dataModel, sets);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithBaseModelTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithBaseModelTest.java
new file mode 100644
index 000000000000..8870750ded36
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithBaseModelTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryRowDeletionsWithBaseModelTest extends QueryRowDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompositePartitionKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompositePartitionKeyTest.java
new file mode 100644
index 000000000000..06aa3a208c9b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompositePartitionKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryRowDeletionsWithCompositePartitionKeyTest extends QueryRowDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompoundKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompoundKeyTest.java
new file mode 100644
index 000000000000..05875ddac645
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompoundKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryRowDeletionsWithCompoundKeyTest extends QueryRowDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompoundKeyWithStaticsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompoundKeyWithStaticsTest.java
new file mode 100644
index 000000000000..8a1731bf5d2d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryRowDeletionsWithCompoundKeyWithStaticsTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryRowDeletionsWithCompoundKeyWithStaticsTest extends QueryRowDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveTester.java
new file mode 100644
index 000000000000..e2d19406ef89
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveTester.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore
+abstract class QueryTimeToLiveTester extends SingleNodeQueryTester
+{
+    @Test
+    public void testTimeToLive() throws Throwable
+    {
+        IndexQuerySupport.timeToLive(executor, dataModel, sets);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithBaseModelTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithBaseModelTest.java
new file mode 100644
index 000000000000..878193fa0c9c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithBaseModelTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryTimeToLiveWithBaseModelTest extends QueryTimeToLiveTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompositePartitionKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompositePartitionKeyTest.java
new file mode 100644
index 000000000000..af5341637e3f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompositePartitionKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryTimeToLiveWithCompositePartitionKeyTest extends QueryTimeToLiveTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompoundKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompoundKeyTest.java
new file mode 100644
index 000000000000..b05a3bd72863
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompoundKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryTimeToLiveWithCompoundKeyTest extends QueryTimeToLiveTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompoundKeyWithStaticsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompoundKeyWithStaticsTest.java
new file mode 100644
index 000000000000..d080aa65a101
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryTimeToLiveWithCompoundKeyWithStaticsTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryTimeToLiveWithCompoundKeyWithStaticsTest extends QueryTimeToLiveTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleTester.java
new file mode 100644
index 000000000000..047ebfd12e79
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleTester.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore
+abstract class QueryWriteLifecycleTester extends SingleNodeQueryTester
+{
+    @Test
+    public void testWriteLifecycle() throws Throwable
+    {
+        IndexQuerySupport.writeLifecycle(executor, dataModel, sets);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithBaseModelTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithBaseModelTest.java
new file mode 100644
index 000000000000..693a0b2eafad
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithBaseModelTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryWriteLifecycleWithBaseModelTest extends QueryWriteLifecycleTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompositePartitionKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompositePartitionKeyTest.java
new file mode 100644
index 000000000000..54c4fe811fc2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompositePartitionKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryWriteLifecycleWithCompositePartitionKeyTest extends QueryWriteLifecycleTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompoundKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompoundKeyTest.java
new file mode 100644
index 000000000000..e0af8b81b3cf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompoundKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryWriteLifecycleWithCompoundKeyTest extends QueryWriteLifecycleTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompoundKeyWithStaticsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompoundKeyWithStaticsTest.java
new file mode 100644
index 000000000000..16fc90db3724
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/QueryWriteLifecycleWithCompoundKeyWithStaticsTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class QueryWriteLifecycleWithCompoundKeyWithStaticsTest extends QueryWriteLifecycleTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/SingleNodeExecutor.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/SingleNodeExecutor.java
similarity index 88%
rename from test/unit/org/apache/cassandra/index/sai/cql/SingleNodeExecutor.java
rename to test/unit/org/apache/cassandra/index/sai/cql/datamodels/SingleNodeExecutor.java
index ea20311b0381..ba898b263dc3 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/SingleNodeExecutor.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/SingleNodeExecutor.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.index.sai.cql;
+package org.apache.cassandra.index.sai.cql.datamodels;
 
 import java.util.List;
 import java.util.stream.Collectors;
@@ -25,7 +25,7 @@
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.inject.Injections;
 
-public class SingleNodeExecutor implements BaseDataModel.Executor
+public class SingleNodeExecutor implements DataModel.Executor
 {
     private final SAITester tester;
     private final Injections.Counter counter;
@@ -61,13 +61,13 @@ public void disableCompaction(String keyspace, String table)
     }
 
     @Override
-    public void waitForIndexQueryable(String keyspace, String index)
+    public void waitForTableIndexesQueryable(String keyspace, String table)
     {
-        tester.waitForIndexQueryable(keyspace, index);
+        tester.waitForTableIndexesQueryable(keyspace, table);
     }
 
     @Override
-    public void executeLocal(String query, Object... values) throws Throwable
+    public void executeLocal(String query, Object... values)
     {
         tester.executeFormattedQuery(query, values);
     }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/SingleNodeQueryTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/SingleNodeQueryTester.java
new file mode 100644
index 000000000000..7e60de2f604f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/SingleNodeQueryTester.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
+import org.apache.cassandra.inject.Injections;
+
+import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
+
+@Ignore
+@RunWith(Parameterized.class)
+abstract class SingleNodeQueryTester extends SAITester
+{
+    protected static final Injections.Counter INDEX_QUERY_COUNTER = Injections.newCounter("IndexQueryCounter")
+                                                                              .add(newInvokePoint().onClass(StorageAttachedIndexSearcher.class).onMethod("search"))
+                                                                              .build();
+
+    @Parameterized.Parameter(0)
+    public Version version;
+    @Parameterized.Parameter(1)
+    public DataModel dataModel;
+    @Parameterized.Parameter(2)
+    public List<IndexQuerySupport.BaseQuerySet> sets;
+
+    protected DataModel.Executor executor;
+
+    private Version latest;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        latest = Version.latest();
+        SAIUtil.setLatestVersion(version);
+        requireNetwork();
+
+        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", DataModel.KEYSPACE));
+
+        Injections.inject(INDEX_QUERY_COUNTER);
+
+        executor = new SingleNodeExecutor(this, INDEX_QUERY_COUNTER);
+    }
+
+    @After
+    public void teardown() throws Throwable
+    {
+        SAIUtil.setLatestVersion(latest);
+    }
+
+    @Parameterized.Parameters(name = "{0}_{1}")
+    public static List<Object[]> allParams()
+    {
+        List<Object[]> scenarios = new LinkedList<>();
+
+        scenarios.addAll(allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams));
+        scenarios.addAll(allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams));
+        scenarios.addAll(allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams));
+        scenarios.addAll(allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams));
+
+        return scenarios;
+    }
+
+    protected static List<Object[]> allIndexVersionsParams(Function<Version, Object[]> params)
+    {
+        List<Object[]> scenarios = new LinkedList<>();
+
+        for (Version version : Version.ALL)
+        {
+            // Excluding BA from the version matrix as files written at BA do not exist in production anywhere
+            if (version.equals(Version.BA))
+                continue;
+
+            scenarios.add(params.apply(version));
+        }
+
+        return scenarios;
+    }
+
+    protected static Object[] baseDataModelParams(Version version)
+    {
+        return new Object[]{ version,
+                             new DataModel.BaseDataModel(DataModel.NORMAL_COLUMNS, DataModel.NORMAL_COLUMN_DATA),
+                             IndexQuerySupport.BASE_QUERY_SETS};
+    }
+
+    protected static Object[] compoundKeyParams(Version version)
+    {
+        return new Object[]{ version,
+                             new DataModel.CompoundKeyDataModel(DataModel.NORMAL_COLUMNS, DataModel.NORMAL_COLUMN_DATA),
+                             IndexQuerySupport.BASE_QUERY_SETS};
+    }
+
+    protected static Object[] compoundKeyWithStaticsParams(Version version)
+    {
+        return new Object[]{ version,
+                             new DataModel.CompoundKeyWithStaticsDataModel(DataModel.STATIC_COLUMNS, DataModel.STATIC_COLUMN_DATA),
+                             IndexQuerySupport.STATIC_QUERY_SETS};
+    }
+
+    protected static Object[] compositePartitionKeyParams(Version version)
+    {
+        return new Object[]{ version,
+                             new DataModel.CompositePartitionKeyDataModel(DataModel.NORMAL_COLUMNS, DataModel.NORMAL_COLUMN_DATA),
+                             ImmutableList.<IndexQuerySupport.BaseQuerySet>builder()
+                                          .addAll(IndexQuerySupport.BASE_QUERY_SETS)
+                                          .addAll(IndexQuerySupport.COMPOSITE_PARTITION_QUERY_SETS)
+                                     .build() };
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsTester.java
new file mode 100644
index 000000000000..d2b756f8fc61
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsTester.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+/**
+ * Force generates segments due to a small RAM size on compaction, to test segment splitting
+ */
+@Ignore
+abstract class TinySegmentQueryCellDeletionsTester extends SingleNodeQueryTester
+{
+    @Before
+    public void setSegmentWriteBufferSpace()
+    {
+        DatabaseDescriptor.setSAISegmentWriteBufferSpace(0);
+    }
+
+    @Test
+    public void testCellDeletions() throws Throwable
+    {
+        IndexQuerySupport.cellDeletions(executor, dataModel, sets);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithBaseModelTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithBaseModelTest.java
new file mode 100644
index 000000000000..b3d470d4181f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithBaseModelTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryCellDeletionsWithBaseModelTest extends TinySegmentQueryCellDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompositePartitionKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompositePartitionKeyTest.java
new file mode 100644
index 000000000000..05bbde9eecea
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompositePartitionKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryCellDeletionsWithCompositePartitionKeyTest extends TinySegmentQueryCellDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompoundKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompoundKeyTest.java
new file mode 100644
index 000000000000..3d26492fba44
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompoundKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryCellDeletionsWithCompoundKeyTest extends TinySegmentQueryCellDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompoundKeyWithStaticsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompoundKeyWithStaticsTest.java
new file mode 100644
index 000000000000..651f286aff5a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryCellDeletionsWithCompoundKeyWithStaticsTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryCellDeletionsWithCompoundKeyWithStaticsTest extends TinySegmentQueryCellDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsTester.java
new file mode 100644
index 000000000000..8e7bce78d9d0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsTester.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+/**
+ * Force generates segments due to a small RAM size on compaction, to test segment splitting
+ */
+@Ignore
+abstract class TinySegmentQueryRowDeletionsTester extends SingleNodeQueryTester
+{
+    @Before
+    public void setSegmentWriteBufferSpace()
+    {
+        DatabaseDescriptor.setSAISegmentWriteBufferSpace(0);
+    }
+
+    @Test
+    public void testRowDeletions() throws Throwable
+    {
+        IndexQuerySupport.rowDeletions(executor, dataModel, sets);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithBaseModelTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithBaseModelTest.java
new file mode 100644
index 000000000000..330a4d5ad28a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithBaseModelTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryRowDeletionsWithBaseModelTest extends TinySegmentQueryRowDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompositePartitionKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompositePartitionKeyTest.java
new file mode 100644
index 000000000000..2fa97e04eb98
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompositePartitionKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryRowDeletionsWithCompositePartitionKeyTest extends TinySegmentQueryRowDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompoundKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompoundKeyTest.java
new file mode 100644
index 000000000000..4e57a45892ec
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompoundKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryRowDeletionsWithCompoundKeyTest extends TinySegmentQueryRowDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompoundKeyWithStaticsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompoundKeyWithStaticsTest.java
new file mode 100644
index 000000000000..60cbad8ba82c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryRowDeletionsWithCompoundKeyWithStaticsTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryRowDeletionsWithCompoundKeyWithStaticsTest extends TinySegmentQueryRowDeletionsTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveTester.java
new file mode 100644
index 000000000000..72b953e4ce7e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveTester.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+/**
+ * Force generates segments due to a small RAM size on compaction, to test segment splitting
+ */
+@Ignore
+abstract class TinySegmentQueryTimeToLiveTester extends SingleNodeQueryTester
+{
+    @Before
+    public void setSegmentWriteBufferSpace()
+    {
+        DatabaseDescriptor.setSAISegmentWriteBufferSpace(0);
+    }
+
+    @Test
+    public void testTimeToLive() throws Throwable
+    {
+        IndexQuerySupport.timeToLive(executor, dataModel, sets);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithBaseModelTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithBaseModelTest.java
new file mode 100644
index 000000000000..842cd73a198f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithBaseModelTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryTimeToLiveWithBaseModelTest extends TinySegmentQueryTimeToLiveTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompositePartitionKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompositePartitionKeyTest.java
new file mode 100644
index 000000000000..bad2b9fce029
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompositePartitionKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryTimeToLiveWithCompositePartitionKeyTest extends TinySegmentQueryTimeToLiveTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompoundKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompoundKeyTest.java
new file mode 100644
index 000000000000..4fb8ade73108
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompoundKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryTimeToLiveWithCompoundKeyTest extends TinySegmentQueryTimeToLiveTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompoundKeyWithStaticsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompoundKeyWithStaticsTest.java
new file mode 100644
index 000000000000..b296ef23d6df
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryTimeToLiveWithCompoundKeyWithStaticsTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryTimeToLiveWithCompoundKeyWithStaticsTest extends TinySegmentQueryTimeToLiveTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleTester.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleTester.java
new file mode 100644
index 000000000000..e25dd267908e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleTester.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+/**
+ * Force generates segments due to a small RAM size on compaction, to test segment splitting
+ */
+@Ignore
+abstract class TinySegmentQueryWriteLifecycleTester extends SingleNodeQueryTester
+{
+    @Before
+    public void setSegmentWriteBufferSpace()
+    {
+        DatabaseDescriptor.setSAISegmentWriteBufferSpace(0);
+    }
+
+    @Test
+    public void testWriteLifecycle() throws Throwable
+    {
+        IndexQuerySupport.writeLifecycle(executor, dataModel, sets);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithBaseModelTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithBaseModelTest.java
new file mode 100644
index 000000000000..45ea29d046be
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithBaseModelTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryWriteLifecycleWithBaseModelTest extends TinySegmentQueryWriteLifecycleTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::baseDataModelParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompositePartitionKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompositePartitionKeyTest.java
new file mode 100644
index 000000000000..79e5610cbf1d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompositePartitionKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryWriteLifecycleWithCompositePartitionKeyTest extends TinySegmentQueryWriteLifecycleTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compositePartitionKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompoundKeyTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompoundKeyTest.java
new file mode 100644
index 000000000000..55e2b0e1b265
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompoundKeyTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryWriteLifecycleWithCompoundKeyTest extends TinySegmentQueryWriteLifecycleTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompoundKeyWithStaticsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompoundKeyWithStaticsTest.java
new file mode 100644
index 000000000000..1d811326e4be
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/datamodels/TinySegmentQueryWriteLifecycleWithCompoundKeyWithStaticsTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.datamodels;
+
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+public class TinySegmentQueryWriteLifecycleWithCompoundKeyWithStaticsTest extends TinySegmentQueryWriteLifecycleTester
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params()
+    {
+        return allIndexVersionsParams(SingleNodeQueryTester::compoundKeyWithStaticsParams);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryTester.java b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryTester.java
new file mode 100644
index 000000000000..1690f4f7b5c6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryTester.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql.filtering;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.google.monitoring.runtime.instrumentation.common.collect.Sets;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.apache.cassandra.index.Index.QueryPlan;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Parameterized {@link SAITester} for testing queries involving restrictions on multiple columns with different
+ * combinations of indexes on those columns, to verify that:
+ * <ul>
+ *     <li>The query results are the same independently of what columns are indexed.</li>
+ *     <li>The {@link QueryPlan} for each tested query uses the existing indexes (if it's possible).</li>
+ *     <li>The query requires {@code ALLOW FILTERING} when it has restrictions that aren't fully supported by indexes.</li>
+ * </ul>
+ * See CNDB-10142 and CNDB-10233 for further details.
+ */
+@RunWith(Parameterized.class)
+public abstract class FilteredQueryTester extends SAITester
+{
+    @Parameterized.Parameter
+    public Set<Index> indexes;
+
+    private Set<String> indexNames;
+
+    @Before
+    public void setup()
+    {
+        indexNames = indexes.stream().map(i -> i.name).collect(Collectors.toSet());
+        createTable();
+        indexes.forEach(i -> i.create(this));
+        populateTable();
+    }
+
+    protected abstract void createTable();
+
+    protected abstract void populateTable();
+
+    protected static Collection<Object[]> parameters(Index... indexes)
+    {
+        Set<Index> indexesSet = ImmutableSet.copyOf(indexes);
+        Set<Object[]> parameters = new HashSet<>();
+        for (int i = 0; i <= indexesSet.size(); i++)
+        {
+            for (Set<Index> combination : Sets.combinations(indexesSet, i))
+            {
+                parameters.add(new Object[]{ combination });
+            }
+        }
+        return parameters;
+    }
+
+    /**
+     * Test the given query with the current {@link #indexes}.
+     *
+     * @param query the query to test
+     * @param shouldFilter whether the query should require {@code ALLOW FILTERING} with the indexes
+     * @param shouldUseIndexes whether the query should use any of the indexes
+     * @param expectedRows the expected rows for the query result
+     */
+    protected void test(String query,
+                        boolean shouldFilter,
+                        boolean shouldUseIndexes,
+                        Object[]... expectedRows)
+    {
+        // verify ALLOW FILTERING
+        if (shouldFilter)
+        {
+            assertInvalidThrowMessage("ALLOW FILTERING", InvalidRequestException.class, query);
+            query += " ALLOW FILTERING";
+        }
+
+        // verify query result
+        assertRowsIgnoringOrder(execute(query), expectedRows);
+
+        // verify whether indexes are used or skipped
+        org.apache.cassandra.index.Index.QueryPlan plan = parseReadCommand(query).indexQueryPlan();
+        assertEquals(shouldUseIndexes, plan != null);
+
+        // if we are using indexes, verify that we are using the expected ones
+        if (plan != null)
+        {
+            Set<String> selectedIndexes = plan.getIndexes()
+                                              .stream()
+                                              .map(i -> i.getIndexMetadata().name)
+                                              .collect(Collectors.toSet());
+            Assert.assertTrue(indexNames.containsAll(selectedIndexes));
+        }
+    }
+
+    protected boolean hasIndex(String index)
+    {
+        return indexNames.contains(index);
+    }
+
+    protected boolean hasAllIndexes(String... indexes)
+    {
+        for (String index : indexes)
+        {
+            if (!indexNames.contains(index))
+                return false;
+        }
+        return true;
+    }
+
+    protected boolean hasAnyIndexes(String... indexes)
+    {
+        for (String index : indexes)
+        {
+            if (indexNames.contains(index))
+                return true;
+        }
+        return false;
+    }
+
+    protected static class Index
+    {
+        private final String name;
+        private final String createQuery;
+
+        protected Index(String column)
+        {
+            this.name = column;
+            this.createQuery = String.format("CREATE CUSTOM INDEX %s ON %%s(%<s) USING 'StorageAttachedIndex'", column);
+        }
+
+        protected Index(String name, String createQuery)
+        {
+            this.name = name;
+            this.createQuery = createQuery;
+        }
+
+        private void create(SAITester tester)
+        {
+            String name = tester.createIndex(createQuery);
+            assert name.equals(this.name) : "Expected index name to be " + this.name + " but was " + name;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Index index = (Index) o;
+            return Objects.equals(name, index.name);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(name);
+        }
+
+        @Override
+        public String toString()
+        {
+            return name;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithListTest.java b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithListTest.java
new file mode 100644
index 000000000000..e13cc00fadba
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithListTest.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql.filtering;
+
+import java.util.Collection;
+
+import org.junit.Test;
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link FilteredQueryTester} for indexes on list columns.
+ */
+public class FilteredQueryWithListTest extends FilteredQueryTester
+{
+    @Parameterized.Parameters(name = "indexes={0}")
+    public static Collection<Object[]> parameters()
+    {
+        return parameters(new Index("v"), new Index("l"));
+    }
+
+    @Override
+    public void createTable()
+    {
+        createTable("CREATE TABLE %s(k int PRIMARY KEY, v int, l list<int>)");
+    }
+
+    @Override
+    public void populateTable()
+    {
+        execute("INSERT INTO %s(k, v, l) values (1, 0, [1])");
+        execute("INSERT INTO %s(k, v, l) values (2, 0, [1, 2])");
+        execute("INSERT INTO %s(k, v, l) values (3, 0, [1, 2, 3])");
+        execute("INSERT INTO %s(k, v, l) values (4, 0, [2, 3])");
+        execute("INSERT INTO %s(k, v, l) values (5, 0, [3])");
+        execute("INSERT INTO %s(k, v, l) values (6, 0, [])");
+    }
+
+    @Test
+    public void testQueries()
+    {
+        // contains
+        test("SELECT k FROM %s WHERE l CONTAINS 1",
+             !hasAllIndexes("l"),
+             hasAllIndexes("l"),
+             row(1), row(2), row(3));
+        test("SELECT k FROM %s WHERE l CONTAINS 1 AND v = 0",
+             !hasAllIndexes("l", "v"),
+             hasAnyIndexes("l", "v"),
+             row(1), row(2), row(3));
+        test("SELECT k FROM %s WHERE l CONTAINS 1 AND v = 1",
+             !hasAllIndexes("l", "v"),
+             hasAnyIndexes("l", "v"));
+        test("SELECT k FROM %s WHERE l CONTAINS 1 OR v = 0",
+             !hasAllIndexes("l", "v"),
+             hasAllIndexes("l", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE l CONTAINS 1 OR v = 1",
+             !hasAllIndexes("l", "v"),
+             hasAllIndexes("l", "v"),
+             row(1), row(2), row(3));
+
+        // not contains
+        test("SELECT k FROM %s WHERE l NOT CONTAINS 1",
+             !hasAllIndexes("l"),
+             hasAllIndexes("l"),
+             row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE l NOT CONTAINS 1 AND v = 0",
+             !hasAllIndexes("l", "v"),
+             hasAnyIndexes("l", "v"),
+             row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE l NOT CONTAINS 1 AND v = 1",
+             !hasAllIndexes("l", "v"),
+             hasAnyIndexes("l", "v"));
+        test("SELECT k FROM %s WHERE l NOT CONTAINS 1 OR v = 0",
+             !hasAllIndexes("l", "v"),
+             hasAllIndexes("l", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE l NOT CONTAINS 1 OR v = 1",
+             !hasAllIndexes("l", "v"),
+             hasAllIndexes("l", "v"),
+             row(4), row(5), row(6));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithMapTest.java b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithMapTest.java
new file mode 100644
index 000000000000..cb297fd233de
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithMapTest.java
@@ -0,0 +1,269 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql.filtering;
+
+import java.util.Collection;
+
+import org.junit.Test;
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link FilteredQueryTester} for indexes on map columns.
+ */
+public class FilteredQueryWithMapTest extends FilteredQueryTester
+{
+    @Parameterized.Parameters(name = "indexes={0}")
+    public static Collection<Object[]> parameters()
+    {
+        return parameters(
+                new Index("v"),
+                new Index("mk", "CREATE CUSTOM INDEX mk ON %s(keys(m)) USING 'StorageAttachedIndex'"),
+                new Index("mv", "CREATE CUSTOM INDEX mv ON %s(values(m)) USING 'StorageAttachedIndex'"),
+                new Index("me", "CREATE CUSTOM INDEX me ON %s(entries(m)) USING 'StorageAttachedIndex'"));
+    }
+
+    @Override
+    public void createTable()
+    {
+        createTable("CREATE TABLE %s(k int PRIMARY KEY, v int, m map<int, int>)");
+    }
+
+    @Override
+    public void populateTable()
+    {
+        execute("INSERT INTO %s(k, v, m) values (1, 0, {1:1, 2:2})");
+        execute("INSERT INTO %s(k, v, m) values (2, 0, {1:1, 3:3})");
+        execute("INSERT INTO %s(k, v, m) values (3, 0, {4:4, 5:5})");
+        execute("INSERT INTO %s(k, v, m) values (4, 0, {1:10, 2:20})");
+        execute("INSERT INTO %s(k, v, m) values (5, 0, {1:10, 3:30})");
+        execute("INSERT INTO %s(k, v, m) values (6, 0, {4:40, 5:50})");
+    }
+
+    @Test
+    public void testQueries()
+    {
+        // equals entries
+        test("SELECT k FROM %s WHERE m[1] = 1",
+             !hasIndex("me"),
+             hasIndex("me"),
+             row(1), row(2));
+        test("SELECT k FROM %s WHERE m[1] = 1 AND v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"),
+             row(1), row(2));
+        test("SELECT k FROM %s WHERE m[1] = 1 AND v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"));
+        test("SELECT k FROM %s WHERE m[1] = 1 OR v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m[1] = 1 OR v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(1), row(2));
+
+        // not equals entries
+        test("SELECT k FROM %s WHERE m[1] != 1",
+             !hasIndex("me"),
+             hasIndex("me"),
+             row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m[1] != 1 AND v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"),
+             row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m[1] != 1 AND v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"));
+        test("SELECT k FROM %s WHERE m[1] != 1 OR v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m[1] != 1 OR v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(3), row(4), row(5), row(6));
+
+        // range entries (>)
+        test("SELECT k FROM %s WHERE m[3] > 3",
+             !hasIndex("me"),
+             hasIndex("me"),
+             row(5));
+        test("SELECT k FROM %s WHERE m[3] > 3 AND v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"),
+             row(5));
+        test("SELECT k FROM %s WHERE m[3] > 3 AND v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"));
+        test("SELECT k FROM %s WHERE m[3] > 3 OR v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m[3] > 3 OR v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(5));
+
+        // range entries (<)
+        test("SELECT k FROM %s WHERE m[3] < 30",
+             !hasIndex("me"),
+             hasIndex("me"),
+             row(2));
+        test("SELECT k FROM %s WHERE m[3] < 30 AND v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"),
+             row(2));
+        test("SELECT k FROM %s WHERE m[3] < 30 AND v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"));
+        test("SELECT k FROM %s WHERE m[3] < 30 OR v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m[3] < 30 OR v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(2));
+
+        // range entries (>=)
+        test("SELECT k FROM %s WHERE m[3] >= 3",
+             !hasIndex("me"),
+             hasIndex("me"),
+             row(2), row(5));
+        test("SELECT k FROM %s WHERE m[3] >= 3 AND v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"),
+             row(2), row(5));
+        test("SELECT k FROM %s WHERE m[3] >= 3 AND v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"));
+        test("SELECT k FROM %s WHERE m[3] >= 3 OR v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m[3] >= 3 OR v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(2), row(5));
+
+        // range entries (<=)
+        test("SELECT k FROM %s WHERE m[3] <= 30",
+             !hasIndex("me"),
+             hasIndex("me"),
+             row(2), row(5));
+        test("SELECT k FROM %s WHERE m[3] <= 30 AND v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"),
+             row(2), row(5));
+        test("SELECT k FROM %s WHERE m[3] <= 30 AND v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAnyIndexes("me", "v"));
+        test("SELECT k FROM %s WHERE m[3] <= 30 OR v = 0",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m[3] <= 30 OR v = 1",
+             !hasAllIndexes("me", "v"),
+             hasAllIndexes("me", "v"),
+             row(2), row(5));
+
+        // contains keys
+        test("SELECT k FROM %s WHERE m CONTAINS KEY 1",
+             !hasAllIndexes("mk"),
+             hasAllIndexes("mk"),
+             row(1), row(2), row(4), row(5));
+        test("SELECT k FROM %s WHERE m CONTAINS KEY 1 AND v = 0",
+             !hasAllIndexes("mk", "v"),
+             hasAnyIndexes("mk", "v"),
+             row(1), row(2), row(4), row(5));
+        test("SELECT k FROM %s WHERE m CONTAINS KEY 1 AND v = 1",
+             !hasAllIndexes("mk", "v"),
+             hasAnyIndexes("mk", "v"));
+        test("SELECT k FROM %s WHERE m CONTAINS KEY 1 OR v = 0",
+             !hasAllIndexes("mk", "v"),
+             hasAllIndexes("mk", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m CONTAINS KEY 1 OR v = 1",
+             !hasAllIndexes("mk", "v"),
+             hasAllIndexes("mk", "v"),
+             row(1), row(2), row(4), row(5));
+
+        // not contains keys
+        test("SELECT k FROM %s WHERE m NOT CONTAINS KEY 4",
+             !hasAllIndexes("mk"),
+             hasAllIndexes("mk"),
+             row(1), row(2), row(4), row(5));
+        test("SELECT k FROM %s WHERE m NOT CONTAINS KEY 4 AND v = 0",
+             !hasAllIndexes("mk", "v"),
+             hasAnyIndexes("mk", "v"),
+             row(1), row(2), row(4), row(5));
+        test("SELECT k FROM %s WHERE m NOT CONTAINS KEY 4 AND v = 1",
+             !hasAllIndexes("mk", "v"),
+             hasAnyIndexes("mk", "v"));
+        test("SELECT k FROM %s WHERE m NOT CONTAINS KEY 4 OR v = 0",
+             !hasAllIndexes("mk", "v"),
+             hasAllIndexes("mk", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m NOT CONTAINS KEY 4 OR v = 1",
+             !hasAllIndexes("mk", "v"),
+             hasAllIndexes("mk", "v"),
+             row(1), row(2), row(4), row(5));
+
+        // contains values
+        test("SELECT k FROM %s WHERE m CONTAINS 1",
+             !hasAllIndexes("mv"),
+             hasAllIndexes("mv"),
+             row(1), row(2));
+        test("SELECT k FROM %s WHERE m CONTAINS 1 AND v = 0",
+             !hasAllIndexes("mv", "v"),
+             hasAnyIndexes("mv", "v"),
+             row(1), row(2));
+        test("SELECT k FROM %s WHERE m CONTAINS 1 AND v = 1",
+             !hasAllIndexes("mv", "v"),
+             hasAnyIndexes("mv", "v"));
+        test("SELECT k FROM %s WHERE m CONTAINS 1 OR v = 0",
+             !hasAllIndexes("mv", "v"),
+             hasAllIndexes("mv", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m CONTAINS 1 OR v = 1",
+             !hasAllIndexes("mv", "v"),
+             hasAllIndexes("mv", "v"),
+             row(1), row(2));
+
+        // not contains values
+        test("SELECT k FROM %s WHERE m NOT CONTAINS 5",
+             !hasAllIndexes("mv"),
+             hasAllIndexes("mv"),
+             row(1), row(2), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m NOT CONTAINS 5 AND v = 0",
+             !hasAllIndexes("mv", "v"),
+             hasAnyIndexes("mv", "v"),
+             row(1), row(2), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m NOT CONTAINS 5 AND v = 1",
+             !hasAllIndexes("mv", "v"),
+             hasAnyIndexes("mv", "v"));
+        test("SELECT k FROM %s WHERE m NOT CONTAINS 5 OR v = 0",
+             !hasAllIndexes("mv", "v"),
+             hasAllIndexes("mv", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE m NOT CONTAINS 5 OR v = 1",
+             !hasAllIndexes("mv", "v"),
+             hasAllIndexes("mv", "v"),
+             row(1), row(2), row(4), row(5), row(6));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithRegularTest.java b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithRegularTest.java
new file mode 100644
index 000000000000..8d35f593112b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithRegularTest.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql.filtering;
+
+import java.util.Collection;
+
+import org.junit.Test;
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link FilteredQueryTester} for indexes on regular, non-collection columns.
+ */
+public class FilteredQueryWithRegularTest extends FilteredQueryTester
+{
+    @Parameterized.Parameters(name = "indexes={0}")
+    public static Collection<Object[]> parameters()
+    {
+        return parameters(new Index("x"), new Index("y"), new Index("z"));
+    }
+
+    @Override
+    public void createTable()
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, x int, y int, z int)");
+    }
+
+    @Override
+    public void populateTable()
+    {
+        execute("INSERT INTO %s(k, x, y, z) values (0, 0, 0, 0)");
+        execute("INSERT INTO %s(k, x, y, z) values (1, 0, 1, 0)");
+        execute("INSERT INTO %s(k, x, y, z) values (2, 0, 2, 1)");
+        execute("INSERT INTO %s(k, x, y, z) values (3, 0, 3, 1)");
+    }
+
+    @Test
+    public void testQueries()
+    {
+        test("SELECT k FROM %s WHERE x=0",
+             !hasIndex("x"),
+             hasIndex("x"),
+             row(0), row(1), row(2), row(3));
+        test("SELECT k FROM %s WHERE x=1",
+             !hasIndex("x"),
+             hasIndex("x"));
+
+        test("SELECT k FROM %s WHERE y IN (1, 2)",
+             !hasIndex("y"),
+             hasIndex("y"),
+             row(1), row(2));
+        test("SELECT k FROM %s WHERE y IN (4, 6)",
+             !hasIndex("y"),
+             hasIndex("y"));
+
+        test("SELECT k FROM %s WHERE x=0 AND y IN (1, 2)",
+             !hasAllIndexes("x", "y"),
+             hasAnyIndexes("x", "y"),
+             row(1), row(2));
+        test("SELECT k FROM %s WHERE x=1 AND y IN (1, 2)",
+             !hasAllIndexes("x", "y"),
+             hasAnyIndexes("x", "y"));
+
+        test("SELECT k FROM %s WHERE x=0 AND (y=1 OR y=2)",
+             !hasAllIndexes("x", "y"),
+             hasAnyIndexes("x", "y"),
+             row(1), row(2));
+        test("SELECT k FROM %s WHERE x=1 AND (y=1 OR y=2)",
+             !hasAllIndexes("x", "y"),
+             hasAnyIndexes("x", "y"));
+
+        test("SELECT k FROM %s WHERE x=0 OR y=0",
+             !hasAllIndexes("x", "y"),
+             hasAllIndexes("x", "y"),
+             row(0), row(1), row(2), row(3));
+        test("SELECT k FROM %s WHERE x=1 OR y=0",
+             !hasAllIndexes("x", "y"),
+             hasAllIndexes("x", "y"),
+             row(0));
+
+        test("SELECT k FROM %s WHERE x=0 OR (y=0 AND z=0)",
+             !hasAllIndexes("x", "y", "z"),
+             hasAllIndexes("x", "y") || hasAllIndexes("x", "z"),
+             row(0), row(1), row(2), row(3));
+        test("SELECT k FROM %s WHERE x=1 OR (y=0 AND z=0)",
+             !hasAllIndexes("x", "y", "z"),
+             hasAllIndexes("x", "y") || hasAllIndexes("x", "z"),
+             row(0));
+
+        test("SELECT k FROM %s WHERE x=0 OR y=0 OR z=0",
+             !hasAllIndexes("x", "y", "z"),
+             hasAllIndexes("x", "y", "z"),
+             row(0), row(1), row(2), row(3));
+        test("SELECT k FROM %s WHERE x=1 OR y=0 OR z=0",
+             !hasAllIndexes("x", "y", "z"),
+             hasAllIndexes("x", "y", "z"),
+             row(0), row(1));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithSetTest.java b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithSetTest.java
new file mode 100644
index 000000000000..aa05cee9dbcf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/filtering/FilteredQueryWithSetTest.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql.filtering;
+
+import java.util.Collection;
+
+import org.junit.Test;
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link FilteredQueryTester} for indexes on set columns.
+ */
+public class FilteredQueryWithSetTest extends FilteredQueryTester
+{
+    @Parameterized.Parameters(name = "indexes={0}")
+    public static Collection<Object[]> parameters()
+    {
+        return parameters(new Index("v"), new Index("s"));
+    }
+
+    @Override
+    public void createTable()
+    {
+        createTable("CREATE TABLE %s(k int PRIMARY KEY, v int, s set<int>)");
+    }
+
+    @Override
+    public void populateTable()
+    {
+        execute("INSERT INTO %s(k, v, s) values (1, 0, {1})");
+        execute("INSERT INTO %s(k, v, s) values (2, 0, {1, 2})");
+        execute("INSERT INTO %s(k, v, s) values (3, 0, {1, 2, 3})");
+        execute("INSERT INTO %s(k, v, s) values (4, 0, {2, 3})");
+        execute("INSERT INTO %s(k, v, s) values (5, 0, {3})");
+        execute("INSERT INTO %s(k, v, s) values (6, 0, {})");
+    }
+
+    @Test
+    public void testQueries()
+    {
+        // contains
+        test("SELECT k FROM %s WHERE s CONTAINS 1",
+             !hasAllIndexes("s"),
+             hasAllIndexes("s"),
+             row(1), row(2), row(3));
+        test("SELECT k FROM %s WHERE s CONTAINS 1 AND v = 0",
+             !hasAllIndexes("s", "v"),
+             hasAnyIndexes("s", "v"),
+             row(1), row(2), row(3));
+        test("SELECT k FROM %s WHERE s CONTAINS 1 AND v = 1",
+             !hasAllIndexes("s", "v"),
+             hasAnyIndexes("s", "v"));
+        test("SELECT k FROM %s WHERE s CONTAINS 1 OR v = 0",
+             !hasAllIndexes("s", "v"),
+             hasAllIndexes("s", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE s CONTAINS 1 OR v = 1",
+             !hasAllIndexes("s", "v"),
+             hasAllIndexes("s", "v"),
+             row(1), row(2), row(3));
+
+        // not contains
+        test("SELECT k FROM %s WHERE s NOT CONTAINS 1",
+             !hasAllIndexes("s"),
+             hasAllIndexes("s"),
+             row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE s NOT CONTAINS 1 AND v = 0",
+             !hasAllIndexes("s", "v"),
+             hasAnyIndexes("s", "v"),
+             row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE s NOT CONTAINS 1 AND v = 1",
+             !hasAllIndexes("s", "v"),
+             hasAnyIndexes("s", "v"));
+        test("SELECT k FROM %s WHERE s NOT CONTAINS 1 OR v = 0",
+             !hasAllIndexes("s", "v"),
+             hasAllIndexes("s", "v"),
+             row(1), row(2), row(3), row(4), row(5), row(6));
+        test("SELECT k FROM %s WHERE s NOT CONTAINS 1 OR v = 1",
+             !hasAllIndexes("s", "v"),
+             hasAllIndexes("s", "v"),
+             row(4), row(5), row(6));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomIntersectionTester.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomIntersectionTester.java
deleted file mode 100644
index 8e03bc71403f..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomIntersectionTester.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-import org.junit.Before;
-import org.junit.runners.Parameterized;
-
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-
-public abstract class RandomIntersectionTester extends SAIRandomizedTester
-{
-    private static final Object[][] EMPTY_ROWS = new Object[][]{};
-
-    private static final int MAX_PARTITION_SIZE = 4096;
-    private static final int RESTRICTED_QUERY_COUNT = 64;
-    private static final int UNRESTRICTED_QUERY_COUNT = 48;
-
-    protected enum Mode { REGULAR, STATIC, REGULAR_STATIC, TWO_REGULAR_ONE_STATIC }
-
-    @Parameterized.Parameter
-    public String testName;
-
-    @Parameterized.Parameter(1)
-    public boolean partitionRestricted;
-
-    @Parameterized.Parameter(2)
-    public boolean largePartition;
-
-    @Parameterized.Parameter(3)
-    public boolean v1HighCardinality;
-
-    @Parameterized.Parameter(4)
-    public boolean v2HighCardinality;
-
-    @Parameterized.Parameter(5)
-    public Mode mode;
-
-    private int numRows;
-
-    @Before
-    public void createTableAndIndexes()
-    {
-        CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT.setInt(3);
-
-        createTable("CREATE TABLE %s (pk int, ck int, v1 int, v2 int, s1 int static, s2 int static, PRIMARY KEY(pk, ck))");
-        createIndex("CREATE INDEX ON %s(v1) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(v2) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(s1) USING 'sai'");
-        createIndex("CREATE INDEX ON %s(s2) USING 'sai'");
-
-        numRows = nextInt(16000, 24000);
-    }
-
-    protected void runRestrictedQueries() throws Throwable
-    {
-        Map<Integer, List<TestRow>> testRowMap = buildAndLoadTestRows();
-
-        beforeAndAfterFlush(() -> {
-            for (int i = 0; i < RESTRICTED_QUERY_COUNT; i++)
-            {
-                int pk = testRowMap.keySet().stream().skip(nextInt(0, testRowMap.size())).findFirst().orElseThrow();
-                int v1 = nextV1();
-                int v2 = nextV2();
-
-                Predicate<TestRow> predicate = null;
-
-                if (mode == Mode.REGULAR)
-                    predicate = row -> row.v1 > v1 && row.v2 > v2;
-                else if (mode == Mode.STATIC)
-                    predicate = row -> row.s1 > v1 && row.s2 > v2;
-                else if (mode == Mode.REGULAR_STATIC)
-                    predicate = row -> row.v1 > v1 && row.s2 > v2;
-                else if (mode == Mode.TWO_REGULAR_ONE_STATIC)
-                    predicate = row -> row.v1 > v1 && row.v2 > v2 && row.s2 > v2;
-
-                assert predicate != null : "Predicate should be assigned!";
-
-                List<Object[]> expected = testRowMap.get(pk)
-                                                    .stream()
-                                                    .sorted(Comparator.comparingInt(o -> o.ck))
-                                                    .filter(predicate)
-                                                    .map(row -> row(row.pk, row.ck))
-                                                    .collect(Collectors.toList());
-
-                UntypedResultSet result = null;
-
-                if (mode == Mode.REGULAR)
-                    result = execute("SELECT pk, ck FROM %s WHERE pk = ? AND v1 > ? AND v2 > ?", pk, v1, v2);
-                else if (mode == Mode.STATIC)
-                    result = execute("SELECT pk, ck FROM %s WHERE pk = ? AND s1 > ? AND s2 > ?", pk, v1, v2);
-                else if (mode == Mode.REGULAR_STATIC)
-                    result = execute("SELECT pk, ck FROM %s WHERE pk = ? AND v1 > ? AND s2 > ?", pk, v1, v2);
-                else if (mode == Mode.TWO_REGULAR_ONE_STATIC)
-                    result = execute("SELECT pk, ck FROM %s WHERE pk = ? AND v1 > ? AND v2 > ? AND s2 > ?", pk, v1, v2, v2);
-
-                assertRows(result, expected.toArray(EMPTY_ROWS));
-            }
-        });
-    }
-
-    protected void runUnrestrictedQueries() throws Throwable
-    {
-        Map<Integer, List<TestRow>> testRowMap = buildAndLoadTestRows();
-
-        beforeAndAfterFlush(() -> {
-            for (int i = 0; i < UNRESTRICTED_QUERY_COUNT; i++)
-            {
-                int v1 = nextV1();
-                int v2 = nextV2();
-
-                Predicate<TestRow> predicate = null;
-
-                if (mode == Mode.REGULAR)
-                    predicate = row -> row.v1 == v1 && row.v2 > v2;
-                else if (mode == Mode.STATIC)
-                    predicate = row -> row.s1 > v1 && row.s2 > v2;
-                else if (mode == Mode.REGULAR_STATIC)
-                    predicate = row -> row.v1 == v1 && row.s2 > v2;
-                else if (mode == Mode.TWO_REGULAR_ONE_STATIC)
-                    predicate = row -> row.v1 == v1 && row.v2 > v2 && row.s2 > v2;
-
-                assert predicate != null : "Predicate should be assigned!";
-
-                List<Object[]> expected = testRowMap.values()
-                                                    .stream()
-                                                    .flatMap(Collection::stream)
-                                                    .filter(predicate)
-                                                    .map(row -> row(row.pk, row.ck))
-                                                    .collect(Collectors.toList());
-
-                UntypedResultSet result = null;
-
-                if (mode == Mode.REGULAR)
-                    result = execute("SELECT pk, ck FROM %s WHERE v1 = ? AND v2 > ?", v1, v2);
-                else if (mode == Mode.STATIC)
-                    result = execute("SELECT pk, ck FROM %s WHERE s1 > ? AND s2 > ?", v1, v2);
-                else if (mode == Mode.REGULAR_STATIC)
-                    result = execute("SELECT pk, ck FROM %s WHERE v1 = ? AND s2 > ?", v1, v2);
-                else if (mode == Mode.TWO_REGULAR_ONE_STATIC)
-                    result = execute("SELECT pk, ck FROM %s WHERE v1 = ? AND v2 > ? AND s2 > ?", v1, v2, v2);
-
-                assertRowsIgnoringOrder(result, expected.toArray(EMPTY_ROWS));
-            }
-        });
-    }
-
-    private Map<Integer, List<TestRow>> buildAndLoadTestRows()
-    {
-        Map<Integer, List<TestRow>> testRowMap = new HashMap<>();
-
-        int clusterSize = nextPartitionSize();
-        int partition = 0;
-        int s1 = nextV1();
-        int s2 = nextV2();
-        List<TestRow> rowList = new ArrayList<>(clusterSize);
-        testRowMap.put(partition, rowList);
-        int clusterCount = 0;
-
-        for (int rowIndex = 0; rowIndex < numRows; rowIndex++)
-        {
-            TestRow row = new TestRow(partition, rowIndex, nextV1(), nextV2(), s1, s2);
-
-            rowList.add(row);
-            clusterCount++;
-
-            if (clusterCount == clusterSize)
-            {
-                clusterCount = 0;
-                clusterSize = nextPartitionSize();
-                partition++;
-                rowList = new ArrayList<>(clusterSize);
-                testRowMap.put(partition, rowList);
-            }
-        }
-       
-        testRowMap.values().stream().flatMap(Collection::stream).forEach(row -> {
-            execute("INSERT INTO %s (pk, ck, v1, v2) VALUES (?, ?, ?, ?)", row.pk, row.ck, row.v1, row.v2);
-            execute("INSERT INTO %s (pk, s1, s2) VALUES (?, ?, ?)", row.pk, row.s1, row.s2);
-        });
-
-        return testRowMap;
-    }
-
-    private int nextPartitionSize()
-    {
-        return largePartition ? nextInt(1024, MAX_PARTITION_SIZE) : nextInt(1, 64);
-    }
-
-    private int nextV1()
-    {
-        return v1HighCardinality ? nextInt(0, numRows / 4) : nextInt(0, 8);
-    }
-
-    private int nextV2()
-    {
-        return v2HighCardinality ? nextInt(0, numRows / 4) : nextInt(0, 8);
-    }
-
-    private static class TestRow implements Comparable<TestRow>
-    {
-        final int pk;
-        final int ck;
-        final int v1;
-        final int v2;
-        final int s1;
-        final int s2;
-
-        TestRow(int pk, int ck, int v1, int v2, int s1, int s2)
-        {
-            this.pk = pk;
-            this.ck = ck;
-            this.v1 = v1;
-            this.v2 = v2;
-            this.s1 = s1;
-            this.s2 = s2;
-        }
-
-        @Override
-        public int compareTo(TestRow other)
-        {
-            int cmp = Integer.compare(pk, other.pk);
-            if (cmp != 0)
-                return cmp;
-            return Integer.compare(ck, other.ck);
-        }
-
-        @Override
-        public boolean equals(Object obj)
-        {
-            if (obj instanceof TestRow)
-                return compareTo((TestRow) obj) == 0;
-
-            return false;
-        }
-
-        @Override
-        public int hashCode()
-        {
-            return Objects.hash(pk, ck);
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomMixedIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomMixedIntersectionTest.java
deleted file mode 100644
index 4260ab96a891..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomMixedIntersectionTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import static org.apache.cassandra.index.sai.cql.intersection.RandomIntersectionTester.Mode.REGULAR_STATIC;
-
-@RunWith(Parameterized.class)
-public class RandomMixedIntersectionTest extends RandomIntersectionTester
-{
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> parameters()
-    {
-        List<Object[]> parameters = new LinkedList<>();
-
-        parameters.add(new Object[] { "Large partition unrestricted, high, high", false, true, true, true, REGULAR_STATIC});
-        parameters.add(new Object[] { "Large partition unrestricted, low, low", false, true, false, false, REGULAR_STATIC});
-        parameters.add(new Object[] { "Large partition unrestricted, high, low", false, true, true, false, REGULAR_STATIC});
-        parameters.add(new Object[] { "Small partition unrestricted, high, high", false, false, true, true, REGULAR_STATIC});
-        parameters.add(new Object[] { "Small partition unrestricted, low, low", false, false, false, false, REGULAR_STATIC});
-        parameters.add(new Object[] { "Small partition unrestricted, high, low", false, false, true, false, REGULAR_STATIC});
-
-        return parameters;
-    }
-
-    @Test
-    public void testMixedIntersection() throws Throwable
-    {
-        runUnrestrictedQueries();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomMixedPartitionIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomMixedPartitionIntersectionTest.java
deleted file mode 100644
index dc12e927fa6f..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomMixedPartitionIntersectionTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import static org.apache.cassandra.index.sai.cql.intersection.RandomIntersectionTester.Mode.REGULAR_STATIC;
-
-@RunWith(Parameterized.class)
-public class RandomMixedPartitionIntersectionTest extends RandomIntersectionTester
-{
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> parameters()
-    {
-        List<Object[]> parameters = new LinkedList<>();
-
-        parameters.add(new Object[] { "Large partition restricted, high, high", true, true, true, true, REGULAR_STATIC});
-        parameters.add(new Object[] { "Large partition restricted, low, low", true, true, false, false, REGULAR_STATIC});
-        parameters.add(new Object[] { "Large partition restricted, high, low", true, true, true, false, REGULAR_STATIC});
-        parameters.add(new Object[] { "Small partition restricted, high, high", true, false, true, true, REGULAR_STATIC});
-        parameters.add(new Object[] { "Small partition restricted, low, low", true, false, false, false, REGULAR_STATIC});
-        parameters.add(new Object[] { "Small partition restricted, high, low", true, false, true, false, REGULAR_STATIC});
-
-        return parameters;
-    }
-
-    @Test
-    public void testMixedIntersection() throws Throwable
-    {
-        runRestrictedQueries();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomRegularIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomRegularIntersectionTest.java
deleted file mode 100644
index f05e30110375..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomRegularIntersectionTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import static org.apache.cassandra.index.sai.cql.intersection.RandomIntersectionTester.Mode.REGULAR;
-
-@RunWith(Parameterized.class)
-public class RandomRegularIntersectionTest extends RandomIntersectionTester
-{
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> parameters()
-    {
-        List<Object[]> parameters = new LinkedList<>();
-
-        parameters.add(new Object[] { "Large partition unrestricted, high, high", false, true, true, true, REGULAR });
-        parameters.add(new Object[] { "Large partition unrestricted, low, low", false, true, false, false, REGULAR });
-        parameters.add(new Object[] { "Large partition unrestricted, high, low", false, true, true, false, REGULAR });
-        parameters.add(new Object[] { "Small partition unrestricted, high, high", false, false, true, true, REGULAR });
-        parameters.add(new Object[] { "Small partition unrestricted, low, low", false, false, false, false, REGULAR });
-        parameters.add(new Object[] { "Small partition unrestricted, high, low", false, false, true, false, REGULAR });
-
-        return parameters;
-    }
-
-    @Test
-    public void testRegularIntersection() throws Throwable
-    {
-        runUnrestrictedQueries();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomRegularPartitionIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomRegularPartitionIntersectionTest.java
deleted file mode 100644
index b14fefe74317..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomRegularPartitionIntersectionTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import static org.apache.cassandra.index.sai.cql.intersection.RandomIntersectionTester.Mode.REGULAR;
-
-@RunWith(Parameterized.class)
-public class RandomRegularPartitionIntersectionTest extends RandomIntersectionTester
-{
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> parameters()
-    {
-        List<Object[]> parameters = new LinkedList<>();
-
-        parameters.add(new Object[] { "Large partition restricted, high, high", true, true, true, true, REGULAR });
-        parameters.add(new Object[] { "Large partition restricted, low, low", true, true, false, false, REGULAR });
-        parameters.add(new Object[] { "Large partition restricted, high, low", true, true, true, false, REGULAR });
-        parameters.add(new Object[] { "Small partition restricted, high, high", true, false, true, true, REGULAR });
-        parameters.add(new Object[] { "Small partition restricted, low, low", true, false, false, false, REGULAR });
-        parameters.add(new Object[] { "Small partition restricted, high, low", true, false, true, false, REGULAR });
-
-        return parameters;
-    }
-
-    @Test
-    public void testRegularIntersection() throws Throwable
-    {
-        runRestrictedQueries();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomStaticIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomStaticIntersectionTest.java
deleted file mode 100644
index a1c31eaddc04..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomStaticIntersectionTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import static org.apache.cassandra.index.sai.cql.intersection.RandomIntersectionTester.Mode.STATIC;
-
-@RunWith(Parameterized.class)
-public class RandomStaticIntersectionTest extends RandomIntersectionTester
-{
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> parameters()
-    {
-        List<Object[]> parameters = new LinkedList<>();
-
-        parameters.add(new Object[] { "Large partition unrestricted, high, high", false, true, true, true, STATIC });
-        parameters.add(new Object[] { "Large partition unrestricted, low, low", false, true, false, false, STATIC });
-        parameters.add(new Object[] { "Large partition unrestricted, high, low", false, true, true, false, STATIC });
-        parameters.add(new Object[] { "Small partition unrestricted, high, high", false, false, true, true, STATIC });
-        parameters.add(new Object[] { "Small partition unrestricted, low, low", false, false, false, false, STATIC });
-        parameters.add(new Object[] { "Small partition unrestricted, high, low", false, false, true, false, STATIC });
-
-        return parameters;
-    }
-
-    @Test
-    public void testStaticIntersection() throws Throwable
-    {
-        runUnrestrictedQueries();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomStaticPartitionIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomStaticPartitionIntersectionTest.java
deleted file mode 100644
index b5e458ecfba6..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/RandomStaticPartitionIntersectionTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import static org.apache.cassandra.index.sai.cql.intersection.RandomIntersectionTester.Mode.STATIC;
-
-@RunWith(Parameterized.class)
-public class RandomStaticPartitionIntersectionTest extends RandomIntersectionTester
-{
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> parameters()
-    {
-        List<Object[]> parameters = new LinkedList<>();
-
-        parameters.add(new Object[] { "Large partition restricted, high, high", true, true, true, true, STATIC });
-        parameters.add(new Object[] { "Large partition restricted, low, low", true, true, false, false, STATIC });
-        parameters.add(new Object[] { "Large partition restricted, high, low", true, true, true, false, STATIC });
-        parameters.add(new Object[] { "Small partition restricted, high, high", true, false, true, true, STATIC });
-        parameters.add(new Object[] { "Small partition restricted, low, low", true, false, false, false, STATIC });
-        parameters.add(new Object[] { "Small partition restricted, high, low", true, false, true, false, STATIC });
-
-        return parameters;
-    }
-
-    @Test
-    public void testStaticIntersection() throws Throwable
-    {
-        runRestrictedQueries();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/TwoRegularOneStaticIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/TwoRegularOneStaticIntersectionTest.java
deleted file mode 100644
index ddf4551e5eb6..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/TwoRegularOneStaticIntersectionTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.apache.cassandra.index.sai.cql.intersection.RandomIntersectionTester.Mode.TWO_REGULAR_ONE_STATIC;
-
-@RunWith(Parameterized.class)
-public class TwoRegularOneStaticIntersectionTest extends RandomIntersectionTester
-{
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> parameters()
-    {
-        List<Object[]> parameters = new LinkedList<>();
-
-        parameters.add(new Object[] { "Large partition unrestricted, high, high", false, true, true, true, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Large partition unrestricted, low, low", false, true, false, false, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Large partition unrestricted, high, low", false, true, true, false, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Small partition unrestricted, high, high", false, false, true, true, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Small partition unrestricted, low, low", false, false, false, false, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Small partition unrestricted, high, low", false, false, true, false, TWO_REGULAR_ONE_STATIC});
-
-        return parameters;
-    }
-
-    @Test
-    public void testMixedIntersection() throws Throwable
-    {
-        runUnrestrictedQueries();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/intersection/TwoRegularOneStaticPartitionIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/intersection/TwoRegularOneStaticPartitionIntersectionTest.java
deleted file mode 100644
index 2bfc7438a305..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/cql/intersection/TwoRegularOneStaticPartitionIntersectionTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.cql.intersection;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.apache.cassandra.index.sai.cql.intersection.RandomIntersectionTester.Mode.TWO_REGULAR_ONE_STATIC;
-
-@RunWith(Parameterized.class)
-public class TwoRegularOneStaticPartitionIntersectionTest extends RandomIntersectionTester
-{
-    @Parameterized.Parameters(name = "{0}")
-    public static List<Object[]> parameters()
-    {
-        List<Object[]> parameters = new LinkedList<>();
-
-        parameters.add(new Object[] { "Large partition restricted, high, high", true, true, true, true, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Large partition restricted, low, low", true, true, false, false, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Large partition restricted, high, low", true, true, true, false, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Small partition restricted, high, high", true, false, true, true, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Small partition restricted, low, low", true, false, false, false, TWO_REGULAR_ONE_STATIC});
-        parameters.add(new Object[] { "Small partition restricted, high, low", true, false, true, false, TWO_REGULAR_ONE_STATIC});
-
-        return parameters;
-    }
-
-    @Test
-    public void testMixedIntersection() throws Throwable
-    {
-        runRestrictedQueries();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/AsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/AsciiTest.java
index 485fd1fad5c7..c11ded31b4ed 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/AsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/AsciiTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class AsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.AsciiDataSet());
     }
+
+    public AsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/BigintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/BigintTest.java
index 603a87ea388e..7521dbc1cb57 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/BigintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/BigintTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class BigintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.BigintDataSet());
     }
+
+    public BigintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/BooleanTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/BooleanTest.java
index 472fdfd4ad8a..779ed2cfb999 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/BooleanTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/BooleanTest.java
@@ -20,13 +20,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class BooleanTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.BooleanDataSet());
     }
+
+    public BooleanTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java b/test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java
index 6eeefb62e667..0f47bde4be93 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java
@@ -26,22 +26,25 @@
 import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.marshal.InetAddressType;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.serializers.SimpleDateSerializer;
 import org.apache.cassandra.serializers.TimeSerializer;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport.NUMBER_OF_VALUES;
 
-public abstract class DataSet<T> extends SAITester
+public abstract class DataSet<T> extends CQLTester
 {
     public T[] values;
 
@@ -54,7 +57,7 @@ public void init()
 
     public Collection<String> decorateIndexColumn(String column)
     {
-        return Collections.singletonList(column);
+        return Arrays.asList(column);
     }
 
     public static abstract class NumericDataSet<T extends Number> extends DataSet<T>
@@ -66,13 +69,14 @@ public static abstract class NumericDataSet<T extends Number> extends DataSet<T>
             for (int index = 0; index < values.length; index += 2)
             {
                 T value1, value2;
-                do
+                while (true)
                 {
                     value1 = nextValue();
                     value1 = getRandom().nextBoolean() ? negate(value1) : abs(value1);
                     value2 = increment(value1);
+                    if (!list.contains(value1) && !list.contains(value2))
+                        break;
                 }
-                while (list.contains(value1) || list.contains(value2));
                 values[index] = value1;
                 values[index + 1] = value2;
             }
@@ -91,7 +95,7 @@ public static abstract class NumericDataSet<T extends Number> extends DataSet<T>
 
         public QuerySet querySet()
         {
-            return new QuerySet.NumericQuerySet();
+            return new QuerySet.NumericQuerySet(this);
         }
     }
 
@@ -279,6 +283,16 @@ BigInteger increment(BigInteger value)
             return value.add(BigInteger.ONE);
         }
 
+        // varint are truncated in SAI to 20 byte precision. That means
+        // we cannot guarantee ordering based on the current design, so
+        // they are not supported in the first iteration, but could be
+        // with modification.
+        @Override
+        public QuerySet querySet()
+        {
+            return new QuerySet.NumericQuerySet(this, false);
+        }
+
         public String toString()
         {
             return "varint";
@@ -317,6 +331,16 @@ BigDecimal increment(BigDecimal value)
             return value.add(BigDecimal.ONE);
         }
 
+        // Decimals are truncated in SAI to 24 byte precision. That means
+        // we cannot guarantee ordering based on the current design, so
+        // they are not supported in the first iteration, but could be
+        // with modification.
+        @Override
+        public QuerySet querySet()
+        {
+            return new QuerySet.NumericQuerySet(this, false);
+        }
+
         public String toString()
         {
             return "decimal";
@@ -409,11 +433,12 @@ public AsciiDataSet()
             for (int index = 0; index < values.length; index++)
             {
                 String value;
-                do
+                while (true)
                 {
                     value = getRandom().nextAsciiString(8, 256);
+                    if (!list.contains(value))
+                        break;
                 }
-                while (list.contains(value));
                 values[index] = value;
             }
         }
@@ -421,7 +446,7 @@ public AsciiDataSet()
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.LiteralQuerySet();
+            return new QuerySet.LiteralQuerySet(this);
         }
 
         public String toString()
@@ -444,7 +469,7 @@ public BooleanDataSet()
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.BooleanQuerySet();
+            return new QuerySet.BooleanQuerySet(this);
         }
 
         public String toString()
@@ -462,11 +487,12 @@ public TextDataSet()
             for (int index = 0; index < values.length; index++)
             {
                 String value;
-                do
+                while (true)
                 {
                     value = getRandom().nextTextString(8, 256);
+                    if (!list.contains(value))
+                        break;
                 }
-                while (list.contains(value));
                 values[index] = value;
             }
         }
@@ -474,7 +500,9 @@ public TextDataSet()
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.LiteralQuerySet();
+            var type = UTF8Type.instance;
+            Comparator<Object[]> comparator = Comparator.comparing(objects -> type.decompose((String) objects[2]), type);
+            return new QuerySet.LiteralQuerySet(this, comparator);
         }
 
         public String toString()
@@ -495,12 +523,13 @@ public DateDataSet()
 
             for (int index = 0; index < values.length; index++)
             {
-                int value;
-                do
+                Integer value;
+                while (true)
                 {
                     value = SimpleDateSerializer.timeInMillisToDay(min + Math.round(getRandom().nextDouble() * range));
+                    if (!list.contains(value))
+                        break;
                 }
-                while (list.contains(value));
                 values[index] = value;
             }
         }
@@ -508,7 +537,19 @@ public DateDataSet()
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.LiteralQuerySet();
+            Comparator<Object[]> comp = Comparator.comparing(o -> o[2], this::compareDateInts);
+            return new QuerySet.LiteralQuerySet(this, comp);
+        }
+
+        private int compareDateInts(Object left, Object right)
+        {
+            int leftInt = (int) left;
+            int rightInt = (int) right;
+            // Dates are stored as unsigned ints, so we need to shift before comparing.
+            // Note that when inserting the values above as integers, they are interpreted as
+            // the already shifted values, not as a date, which is why shifting is required.
+            // We add Integer.MIN_VALUE to shift the values back to their original form.
+            return Integer.compare((leftInt + Integer.MIN_VALUE), (rightInt + Integer.MIN_VALUE));
         }
 
         public String toString()
@@ -526,15 +567,16 @@ public TimeDataSet()
             for (int index = 0; index < values.length; index++)
             {
                 Long value;
-                do
+                while (true)
                 {
                     int hours = getRandom().nextIntBetween(0, 23);
                     int minutes = getRandom().nextIntBetween(0, 59);
                     int seconds = getRandom().nextIntBetween(0, 59);
                     long nanos = getRandom().nextIntBetween(0, 1000000000);
                     value = TimeSerializer.timeStringToLong(String.format("%s:%s:%s.%s", hours, minutes, seconds, nanos));
+                    if (!list.contains(value))
+                        break;
                 }
-                while (list.contains(value));
                 values[index] = value;
             }
             Arrays.sort(values);
@@ -543,7 +585,7 @@ public TimeDataSet()
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.NumericQuerySet();
+            return new QuerySet.NumericQuerySet(this);
         }
 
         public String toString()
@@ -565,11 +607,12 @@ public TimestampDataSet()
             for (int index = 0; index < values.length; index++)
             {
                 Date value;
-                do
+                while (true)
                 {
                     value = Date.from(Instant.ofEpochSecond(min + Math.round(getRandom().nextDouble() * range)));
+                    if (!list.contains(value))
+                        break;
                 }
-                while (list.contains(value));
                 values[index] = value;
             }
         }
@@ -577,7 +620,7 @@ public TimestampDataSet()
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.LiteralQuerySet();
+            return new QuerySet.LiteralQuerySet(this);
         }
 
         public String toString()
@@ -596,11 +639,12 @@ public UuidDataSet()
             for (int index = 0; index < values.length; index++)
             {
                 UUID value;
-                do
+                while (true)
                 {
                     value = UUID.randomUUID();
+                    if (!list.contains(value))
+                        break;
                 }
-                while (list.contains(value));
                 values[index] = value;
             }
         }
@@ -608,7 +652,9 @@ public UuidDataSet()
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.LiteralQuerySet();
+            Comparator<Object[]> comparator = Comparator.comparing(o -> UUIDType.instance.decompose((UUID) o[2]),
+                                                                   UUIDType.instance);
+            return new QuerySet.LiteralQuerySet(this, comparator);
         }
 
         public String toString()
@@ -639,7 +685,7 @@ public TimeuuidDataSet()
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.LiteralQuerySet();
+            return new QuerySet.LiteralQuerySet(this);
         }
 
         public String toString()
@@ -658,7 +704,7 @@ public InetDataSet()
             for (int index = 0; index < values.length; index++)
             {
                 InetAddress value;
-                do
+                while (true)
                 {
                     byte[] bytes;
                     if (getRandom().nextBoolean())
@@ -674,19 +720,27 @@ public InetDataSet()
                     {
                         throw new RuntimeException(e);
                     }
+                    if (!list.contains(value))
+                        break;
                 }
-                while (list.contains(value));
                 values[index] = value;
             }
-            IndexTermType indexTermType = createIndexTermType(InetAddressType.instance);
-            Arrays.sort(values, (o1, o2) -> indexTermType.compare(indexTermType.asIndexBytes(ByteBuffer.wrap(o1.getAddress())),
-                                                                  indexTermType.asIndexBytes(ByteBuffer.wrap(o2.getAddress()))));
+            // TODO this ordering is not the same as the InetAddressType's ordering. Is that a bug?
+            //  var x = Arrays.copyOf(values, values.length);
+            //  // Sort using the InetAddressType comparator
+            //  Arrays.sort(x, Comparator.comparing(InetAddressType.instance.getSerializer()::serialize, InetAddressType.instance));
+            Arrays.sort(values, (o1, o2) -> {
+                return TypeUtil.compare(TypeUtil.asIndexBytes(ByteBuffer.wrap(o1.getAddress()), InetAddressType.instance),
+                                        TypeUtil.asIndexBytes(ByteBuffer.wrap(o2.getAddress()), InetAddressType.instance),
+                                        InetAddressType.instance,
+                                        Version.DB);
+            });
         }
 
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.NumericQuerySet();
+            return new QuerySet.NumericQuerySet(this, false);
         }
 
         public String toString()
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/DateTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/DateTest.java
index 7e977172e101..3237d1b7b8aa 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/DateTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/DateTest.java
@@ -17,15 +17,32 @@
  */
 package org.apache.cassandra.index.sai.cql.types;
 
+
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class DateTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.DateDataSet());
     }
+
+    public DateTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/DecimalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/DecimalTest.java
index 8dc1071ddebc..4f7206843360 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/DecimalTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/DecimalTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class DecimalTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.DecimalDataSet());
     }
+
+    public DecimalTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/DoubleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/DoubleTest.java
index 4222d7fd0ac0..b058bb01ee54 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/DoubleTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/DoubleTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class DoubleTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.DoubleDataSet());
     }
+
+    public DoubleTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/FloatTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/FloatTest.java
index 598f95c8f810..f15e0c4bb136 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/FloatTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/FloatTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class FloatTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.FloatDataSet());
     }
+
+    public FloatTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/IndexingTypeSupport.java b/test/unit/org/apache/cassandra/index/sai/cql/types/IndexingTypeSupport.java
index a9320ea173c5..8afb4bdf49d0 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/IndexingTypeSupport.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/IndexingTypeSupport.java
@@ -17,30 +17,28 @@
  */
 package org.apache.cassandra.index.sai.cql.types;
 
-import java.util.Arrays;
 import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
 
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
-@RunWith(Parameterized.class)
 public abstract class IndexingTypeSupport extends SAITester
 {
     public static final int NUMBER_OF_VALUES = 64;
 
-    @Parameterized.Parameter
-    public DataSet<?> dataset;
-
-    @Parameterized.Parameter(1)
-    public boolean widePartitions;
-
-    @Parameterized.Parameter(2)
-    public Scenario scenario;
+    protected final DataSet<?> dataset;
 
+    private final Version version;
+    private Version latest;
+    private final boolean widePartitions;
+    private final Scenario scenario;
     private Object[][] allRows;
 
     public enum Scenario
@@ -54,24 +52,38 @@ public enum Scenario
 
     protected static Collection<Object[]> generateParameters(DataSet<?> dataset)
     {
-        return Arrays.asList(new Object[][]
+        List<Object[]> scenarios = new LinkedList<>();
+        for (boolean wideRows : new boolean[]{true, false})
         {
-            { dataset, true, Scenario.MEMTABLE_QUERY },
-            { dataset, true, Scenario.SSTABLE_QUERY},
-            { dataset, true, Scenario.COMPACTED_QUERY},
-            { dataset, true, Scenario.MIXED_QUERY},
-            { dataset, true, Scenario.POST_BUILD_QUERY},
-            { dataset, false, Scenario.MEMTABLE_QUERY },
-            { dataset, false, Scenario.SSTABLE_QUERY},
-            { dataset, false, Scenario.COMPACTED_QUERY},
-            { dataset, false, Scenario.MIXED_QUERY},
-            { dataset, false, Scenario.POST_BUILD_QUERY}
-        });
+            for (Version version : Version.ALL)
+            {
+                // Skip BA version, as files at BA do not exist in production anywhere
+                if (version.equals(Version.BA))
+                    continue;
+
+                for (Scenario scenario : Scenario.values())
+                    scenarios.add(new Object[]{version, dataset, wideRows, scenario});
+            }
+
+        }
+
+        return scenarios;
+    }
+
+    public IndexingTypeSupport(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        this.version = version;
+        this.dataset = dataset;
+        this.widePartitions = widePartitions;
+        this.scenario = scenario;
     }
 
     @Before
-    public void createTable()
+    public void setup()
     {
+        latest = Version.latest();
+        SAIUtil.setLatestVersion(version);
+
         dataset.init();
 
         createTable(String.format("CREATE TABLE %%s (pk int, ck int, value %s, PRIMARY KEY(pk, ck))", dataset));
@@ -81,13 +93,18 @@ public void createTable()
         allRows = generateRows(dataset, widePartitions);
     }
 
-    @Test
-    public void runIndexQueryScenarios() throws Throwable
+    @After
+    public void teardown()
+    {
+        SAIUtil.setLatestVersion(latest);
+    }
+
+    protected void runIndexQueryScenarios() throws Throwable
     {
         if (scenario != Scenario.POST_BUILD_QUERY)
         {
             for (String index : dataset.decorateIndexColumn("value"))
-                createIndex(String.format("CREATE INDEX ON %%s(%s) USING 'sai'", index));
+                createIndex(String.format("CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'", index));
         }
 
         insertData(this, allRows, scenario);
@@ -104,14 +121,14 @@ public void runIndexQueryScenarios() throws Throwable
             case POST_BUILD_QUERY:
                 flush();
                 for (String index : dataset.decorateIndexColumn("value"))
-                    createIndex(String.format("CREATE INDEX ON %%s(%s) USING 'sai'", index));
+                    createIndex(String.format("CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'", index));
                 break;
         }
 
         dataset.querySet().runQueries(this, allRows);
     }
 
-    public static void insertData(SAITester tester, Object[][] allRows, Scenario scenario)
+    public void insertData(CQLTester tester, Object[][] allRows, Scenario scenario) throws Throwable
     {
         int sstableCounter = 0;
         int sstableIncrement = NUMBER_OF_VALUES / 8;
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/InetTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/InetTest.java
index a3c2796ad6dc..d6971c56657e 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/InetTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/InetTest.java
@@ -19,13 +19,40 @@
 
 import java.util.Collection;
 
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class InetTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    // TODO: Disables coordinator execution because we know SAI indexing for inet works differently than RowFilter,
+    //  which can wrongly discard rows in the coordinator. This is reported in CNDB-12978, and we should enable
+    //  distributed execution again once we have a fix.
+    @BeforeClass
+    public static void disableCoordinatorExecution()
+    {
+        CQLTester.disableCoordinatorExecution();
+    }
+
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.InetDataSet());
     }
+
+    public InetTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/IntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/IntTest.java
index a96f2b1c8a0d..677f1aaa115c 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/IntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/IntTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class IntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.IntDataSet());
     }
+
+    public IntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java
index e981fe4e8bc1..9b13df988c07 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java
@@ -22,34 +22,53 @@
 import java.math.RoundingMode;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.stream.Collectors;
 
 import org.junit.Test;
 
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.marshal.DecimalType;
 import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 
-import static org.junit.Assert.assertTrue;
-
-public class NumericTypeSortingTest extends SAIRandomizedTester
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class NumericTypeSortingTest extends RandomizedTest
 {
+
+    private final Version version;
+    @ParametersFactory()
+    public static Collection<Object[]> data()
+    {
+        // Required because it configures SEGMENT_BUILD_MEMORY_LIMIT, which is needed for Version.AA
+        if (DatabaseDescriptor.getRawConfig() == null)
+            DatabaseDescriptor.setConfig(DatabaseDescriptor.loadConfig());
+        return Version.ALL.stream().map(v -> new Object[]{v}).collect(Collectors.toList());
+    }
+
+    public NumericTypeSortingTest(Version version)
+    {
+        this.version = version;
+    }
+
     @Test
     public void testBigDecimalEncoding()
     {
         BigDecimal[] data = new BigDecimal[10000];
         for (int i = 0; i < data.length; i++)
         {
-            BigDecimal divider = new BigDecimal(getRandom().nextBigInteger(1000).add(BigInteger.ONE));
-            BigDecimal randomNumber = new BigDecimal(getRandom().nextBigInteger(1000)).divide(divider, RoundingMode.HALF_DOWN);
-            if (getRandom().nextBoolean())
+            BigDecimal divider = new BigDecimal(new BigInteger(randomInt(1000), getRandom()).add(BigInteger.ONE));
+            BigDecimal randomNumber = new BigDecimal(new BigInteger(randomInt(1000), getRandom())).divide(divider, RoundingMode.HALF_DOWN);
+            if (randomBoolean())
                 randomNumber = randomNumber.negate();
 
             data[i] = randomNumber;
         }
 
-        IndexTermType indexTermType = createIndexTermType(DecimalType.instance);
-
         Arrays.sort(data, BigDecimal::compareTo);
 
         for (int i = 1; i < data.length; i++)
@@ -58,11 +77,11 @@ public void testBigDecimalEncoding()
             BigDecimal i1 = data[i];
             assertTrue(i0 + " <= " + i1, i0.compareTo(i1) <= 0);
 
-            ByteBuffer b0 = indexTermType.asIndexBytes(DecimalType.instance.decompose(i0));
+            ByteBuffer b0 = TypeUtil.asIndexBytes(DecimalType.instance.decompose(i0), DecimalType.instance);
 
-            ByteBuffer b1 = indexTermType.asIndexBytes(DecimalType.instance.decompose(i1));
+            ByteBuffer b1 = TypeUtil.asIndexBytes(DecimalType.instance.decompose(i1), DecimalType.instance);
 
-            assertTrue(i0 + " <= " + i1, indexTermType.compare(b0, b1) <= 0);
+            assertTrue(i0 + " <= " + i1, TypeUtil.compare(b0, b1, DecimalType.instance, version) <= 0);
         }
     }
 
@@ -72,9 +91,9 @@ public void testBigIntEncoding()
         BigInteger[] data = new BigInteger[10000];
         for (int i = 0; i < data.length; i++)
         {
-            BigInteger divider = getRandom().nextBigInteger(1000).add(BigInteger.ONE);
-            BigInteger randomNumber = getRandom().nextBigInteger(1000).divide(divider);
-            if (getRandom().nextBoolean())
+            BigInteger divider = new BigInteger(randomInt(1000), getRandom()).add(BigInteger.ONE);
+            BigInteger randomNumber = new BigInteger(randomInt(1000), getRandom()).divide(divider);
+            if (randomBoolean())
                 randomNumber = randomNumber.negate();
 
             data[i] = randomNumber;
@@ -82,19 +101,17 @@ public void testBigIntEncoding()
 
         Arrays.sort(data, BigInteger::compareTo);
 
-        IndexTermType indexTermType = createIndexTermType(IntegerType.instance);
-
         for (int i = 1; i < data.length; i++)
         {
             BigInteger i0 = data[i - 1];
             BigInteger i1 = data[i];
             assertTrue(i0 + " <= " + i1, i0.compareTo(i1) <= 0);
 
-            ByteBuffer b0 = indexTermType.asIndexBytes(IntegerType.instance.decompose(i0));
+            ByteBuffer b0 = TypeUtil.asIndexBytes(IntegerType.instance.decompose(i0), IntegerType.instance);
 
-            ByteBuffer b1 = indexTermType.asIndexBytes(IntegerType.instance.decompose(i1));
+            ByteBuffer b1 = TypeUtil.asIndexBytes(IntegerType.instance.decompose(i1), IntegerType.instance);
 
-            assertTrue(i0 + " <= " + i1, indexTermType.compare(b0, b1) <= 0);
+            assertTrue(i0 + " <= " + i1, TypeUtil.compare(b0, b1, IntegerType.instance, version) <= 0);
         }
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/QuerySet.java b/test/unit/org/apache/cassandra/index/sai/cql/types/QuerySet.java
index d542e42c2b7a..6f4821a381fa 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/QuerySet.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/QuerySet.java
@@ -20,37 +20,75 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
 import static org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport.NUMBER_OF_VALUES;
+import static org.junit.Assert.assertEquals;
 
-public abstract class QuerySet extends SAITester
+public abstract class QuerySet extends CQLTester
 {
-    private static final int VALUE_INDEX = 2;
+    final DataSet<?> dataset;
+
+    QuerySet(DataSet<?> dataset)
+    {
+        this.dataset = dataset;
+    }
 
     public abstract void runQueries(SAITester tester, Object[][] allRows) throws Throwable;
 
+    protected static boolean isOrderBySupported()
+    {
+        return Version.latest().onOrAfter(Version.BA);
+    }
+
     public static class NumericQuerySet extends QuerySet
     {
+        private final boolean testOrderBy;
+        private final Comparator<Object[]> comparator;
+
+        NumericQuerySet(DataSet<?> dataset)
+        {
+            this(dataset, isOrderBySupported());
+        }
+
+        NumericQuerySet(DataSet<?> dataset, boolean testOrderBy)
+        {
+            super(dataset);
+            assert !testOrderBy || isOrderBySupported() : "ORDER BY not supported";
+            this.testOrderBy = testOrderBy;
+            this.comparator = Comparator.comparing(o -> (Comparable) o[2]);
+        }
+
         @Override
         public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
         {
+
             // Query each value for all operators
             for (int index = 0; index < allRows.length; index++)
             {
-                assertRows(tester.execute("SELECT * FROM %s WHERE value = ?", allRows[index][VALUE_INDEX]), new Object[][] { allRows[index] });
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value > ?", allRows[index][VALUE_INDEX]), Arrays.copyOfRange(allRows, index + 1, allRows.length));
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value >= ?", allRows[index][VALUE_INDEX]), Arrays.copyOfRange(allRows, index, allRows.length));
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value < ?", allRows[index][VALUE_INDEX]), Arrays.copyOfRange(allRows, 0, index));
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value <= ?", allRows[index][VALUE_INDEX]), Arrays.copyOfRange(allRows, 0, index + 1));
+                assertRows(tester.execute("SELECT * FROM %s WHERE value = ?", allRows[index][2]), new Object[][] { allRows[index] });
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value > ?", allRows[index][2]), Arrays.copyOfRange(allRows, index + 1, allRows.length));
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value >= ?", allRows[index][2]), Arrays.copyOfRange(allRows, index, allRows.length));
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value < ?", allRows[index][2]), Arrays.copyOfRange(allRows, 0, index));
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value <= ?", allRows[index][2]), Arrays.copyOfRange(allRows, 0, index + 1));
             }
 
             // Query full range
-            assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value >= ? AND value <= ?", allRows[0][VALUE_INDEX], allRows[NUMBER_OF_VALUES - 1][VALUE_INDEX]), allRows);
+            assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value >= ? AND value <= ?", allRows[0][2], allRows[NUMBER_OF_VALUES - 1][2]), allRows);
+
+            // Edge cases with IN where we get no and all values. It is not valid to AND an IN predicate and others,
+            // so these two queries are comprehensive
+            assertRows(tester.execute("SELECT * FROM %s WHERE value IN ()"));
+            assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value NOT IN ()"), allRows);
+            assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE (value IN ()) OR (value >= ? AND value <= ?)", allRows[0][2], allRows[NUMBER_OF_VALUES - 1][2]), allRows);
 
             // Query random ranges. This selects a series of random ranges and tests the different possible inclusivity
             // on them. This loops a reasonable number of times to cover as many ranges as possible without taking too long
@@ -68,33 +106,74 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
                 int max = Math.max(index1, index2);
 
                 // lower exclusive -> upper exclusive
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value > ? AND value < ?", allRows[min][VALUE_INDEX], allRows[max][VALUE_INDEX]),
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value > ? AND value < ?", allRows[min][2], allRows[max][2]),
                         Arrays.copyOfRange(allRows, min + 1, max));
 
+                var result = Arrays.copyOfRange(allRows, min + 1, max);
+                if (result.length > 0 && testOrderBy)
+                {
+                    Arrays.sort(result, comparator);
+                    assertRows(tester.execute("SELECT * FROM %s WHERE value > ? AND value < ? ORDER BY value ASC LIMIT ?",
+                                              allRows[min][2], allRows[max][2], result.length), result);
+                    // reverse it
+                    var list = Arrays.asList(result);
+                    Collections.reverse(list);
+                    var reversed = list.toArray(new Object[][]{});
+                    assertRows(tester.execute("SELECT * FROM %s WHERE value > ? AND value < ? ORDER BY value DESC LIMIT ?",
+                                              allRows[min][2], allRows[max][2], reversed.length), reversed);
+                }
+
                 // lower inclusive -> upper exclusive
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value >= ? AND value < ?", allRows[min][VALUE_INDEX], allRows[max][VALUE_INDEX]),
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value >= ? AND value < ?", allRows[min][2], allRows[max][2]),
                         Arrays.copyOfRange(allRows, min, max));
 
                 // lower exclusive -> upper inclusive
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value > ? AND value <= ?", allRows[min][VALUE_INDEX], allRows[max][VALUE_INDEX]),
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value > ? AND value <= ?", allRows[min][2], allRows[max][2]),
                         Arrays.copyOfRange(allRows, min + 1, max + 1));
 
                 // lower inclusive -> upper inclusive
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value >= ? AND value <= ?", allRows[min][VALUE_INDEX], allRows[max][VALUE_INDEX]),
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value >= ? AND value <= ?", allRows[min][2], allRows[max][2]),
                         Arrays.copyOfRange(allRows, min, max + 1));
             }
+
+            if (!testOrderBy)
+                return;
+
+            // Sort allRows by value
+            var copyOfAllRows = Arrays.copyOf(allRows, allRows.length);
+            Arrays.sort(copyOfAllRows, comparator);
+
+            assertRows(tester.execute("SELECT * FROM %s ORDER BY value ASC limit 10"),
+                       Arrays.stream(copyOfAllRows).limit(10).toArray(Object[][]::new));
+            assertRows(tester.execute("SELECT * FROM %s ORDER BY value ASC limit 100"),
+                       Arrays.stream(copyOfAllRows).limit(100).toArray(Object[][]::new));
+
+            // reverse again
+            var list = Arrays.asList(copyOfAllRows);
+            Collections.reverse(list);
+            copyOfAllRows = list.toArray(new Object[][]{});
+            // Sort only
+            assertRows(tester.execute("SELECT * FROM %s ORDER BY value DESC limit 10"),
+                       Arrays.stream(copyOfAllRows).limit(10).toArray(Object[][]::new));
+            assertRows(tester.execute("SELECT * FROM %s ORDER BY value DESC limit 100"),
+                       Arrays.stream(copyOfAllRows).limit(100).toArray(Object[][]::new));
         }
     }
 
     public static class BooleanQuerySet extends QuerySet
     {
+        BooleanQuerySet(DataSet<?> dataSet)
+        {
+            super(dataSet);
+        }
+
         @Override
         public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
         {
             // Query each value for EQ operator
             for (int index = 0; index < allRows.length; index++)
             {
-                Object value = allRows[index][VALUE_INDEX];
+                Object value = allRows[index][2];
                 assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value = ?", value), getExpectedRows(value, allRows));
             }
         }
@@ -103,7 +182,7 @@ protected Object[][] getExpectedRows(Object value, Object[][] allRows)
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (row[VALUE_INDEX].equals(value))
+                if (row[2].equals(value))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -112,23 +191,70 @@ protected Object[][] getExpectedRows(Object value, Object[][] allRows)
 
     public static class LiteralQuerySet extends QuerySet
     {
+        private final boolean testOrderBy;
+        private final Comparator<Object[]> comparator;
+
+        // For UTF8, the ordering is different from the natural ordering, so we allow a custom comparator
+        LiteralQuerySet(DataSet<?> dataSet, Comparator<Object[]> comparator)
+        {
+            super(dataSet);
+            this.testOrderBy = isOrderBySupported();
+            this.comparator = comparator;
+        }
+
+        LiteralQuerySet(DataSet<?> dataSet)
+        {
+            this(dataSet, isOrderBySupported());
+        }
+
+        LiteralQuerySet(DataSet<?> dataSet, boolean testOrderBy)
+        {
+            super(dataSet);
+            assert !testOrderBy || isOrderBySupported() : "ORDER BY not supported by AA (V1) indexes";
+            this.testOrderBy = testOrderBy;
+            this.comparator = Comparator.comparing(o -> (Comparable) o[2]);
+        }
+
         @Override
         public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
         {
             // Query each value for EQ operator
             for (int index = 0; index < allRows.length; index++)
             {
-                assertRows(tester.execute("SELECT * FROM %s WHERE value = ?", allRows[index][VALUE_INDEX]), new Object[][] { allRows[index] });
+                assertRows(tester.execute("SELECT * FROM %s WHERE value = ?", allRows[index][2]), new Object[][] { allRows[index] });
             }
+
+            // Some literal types do not support ORDER BY yet, so we skip those
+            if (!testOrderBy)
+                return;
+
+            var copyOfAllRows = Arrays.copyOf(allRows, allRows.length);
+            // Sort allRows by value
+            Arrays.sort(copyOfAllRows, comparator);
+            assertRows(tester.execute("SELECT * FROM %s ORDER BY value ASC limit 10"),
+                       Arrays.stream(copyOfAllRows).limit(10).toArray(Object[][]::new));
+            assertRows(tester.execute("SELECT * FROM %s ORDER BY value ASC limit 100"),
+                       Arrays.stream(copyOfAllRows).limit(100).toArray(Object[][]::new));
+
+            // reverse copyOfAllRows
+            var list = Arrays.asList(copyOfAllRows);
+            Collections.reverse(list);
+            copyOfAllRows = list.toArray(new Object[][]{});
+
+            assertRows(tester.execute("SELECT * FROM %s ORDER BY value DESC limit 10"),
+                       Arrays.stream(copyOfAllRows).limit(10).toArray(Object[][]::new));
+            assertRows(tester.execute("SELECT * FROM %s ORDER BY value DESC limit 100"),
+                       Arrays.stream(copyOfAllRows).limit(100).toArray(Object[][]::new));
         }
     }
 
     public static class CollectionQuerySet extends QuerySet
     {
-        protected final DataSet<?> elementDataSet;
+        protected DataSet<?> elementDataSet;
 
-        public CollectionQuerySet(DataSet<?> elementDataSet)
+        public CollectionQuerySet(DataSet<?> dataSet, DataSet<?> elementDataSet)
         {
+            super(dataSet);
             this.elementDataSet = elementDataSet;
         }
 
@@ -144,7 +270,7 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
             for (int and = 0; and < allRows.length / 4; and++)
             {
                 int index = getRandom().nextIntBetween(0, allRows.length - 1);
-                Iterator<?> valueIterator = ((Collection<?>) allRows[index][VALUE_INDEX]).iterator();
+                Iterator valueIterator = ((Collection) allRows[index][2]).iterator();
                 Object value1 = valueIterator.next();
                 Object value2 = valueIterator.hasNext() ? valueIterator.next() : value1;
                 assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS ? AND value CONTAINS ?",
@@ -157,7 +283,7 @@ protected Object[][] getExpectedRows(Object value, Object[][] allRows)
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Collection<?>)row[VALUE_INDEX]).contains(value))
+                if (((Collection)row[2]).contains(value))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -168,7 +294,7 @@ protected Object[][] getExpectedRows(Object value1, Object value2, Object[][] al
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Collection<?>)row[VALUE_INDEX]).contains(value1) && ((Collection<?>)row[VALUE_INDEX]).contains(value2))
+                if (((Collection)row[2]).contains(value1) && ((Collection)row[2]).contains(value2))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -177,13 +303,18 @@ protected Object[][] getExpectedRows(Object value1, Object value2, Object[][] al
 
     public static class FrozenCollectionQuerySet extends QuerySet
     {
+        public FrozenCollectionQuerySet(DataSet<?> dataset)
+        {
+            super(dataset);
+        }
+
         @Override
         public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
         {
             for (int index = 0; index < allRows.length; index++)
             {
                 assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value = ?",
-                        allRows[index][VALUE_INDEX]), getExpectedRows(allRows[index][VALUE_INDEX], allRows));
+                        allRows[index][2]), getExpectedRows(allRows[index][2], allRows));
             }
         }
 
@@ -192,7 +323,7 @@ protected Object[][] getExpectedRows(Object value, Object[][] allRows)
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (row[VALUE_INDEX].equals(value))
+                if (row[2].equals(value))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -201,13 +332,17 @@ protected Object[][] getExpectedRows(Object value, Object[][] allRows)
 
     public static class FrozenTuple extends FrozenCollectionQuerySet
     {
+        public FrozenTuple(DataSet<?> dataset)
+        {
+            super(dataset);
+        }
     }
 
     public static class MapValuesQuerySet extends CollectionQuerySet
     {
-        public MapValuesQuerySet(DataSet<?> elementDataSet)
+        public MapValuesQuerySet(DataSet<?> dataSet, DataSet<?> elementDataSet)
         {
-            super(elementDataSet);
+            super(dataSet, elementDataSet);
         }
 
         @Override
@@ -222,7 +357,7 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
             for (int and = 0; and < allRows.length / 4; and++)
             {
                 int index = getRandom().nextIntBetween(0, allRows.length - 1);
-                Map<?, ?> map = (Map<?, ?>)allRows[index][VALUE_INDEX];
+                Map map = (Map)allRows[index][2];
                 Object value1 = map.values().toArray()[getRandom().nextIntBetween(0, map.values().size() - 1)];
                 Object value2 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.values().size() - 1)];
                 assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS ? AND value CONTAINS ?",
@@ -235,7 +370,7 @@ protected Object[][] getExpectedRows(Object value, Object[][] allRows)
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Map<?, ?>)row[VALUE_INDEX]).containsValue(value))
+                if (((Map)row[2]).values().contains(value))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -246,7 +381,7 @@ protected Object[][] getExpectedRows(Object value1, Object value2, Object[][] al
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Map<?, ?>)row[VALUE_INDEX]).containsValue(value1) && ((Map<?, ?>)row[VALUE_INDEX]).containsValue(value2))
+                if (((Map)row[2]).values().contains(value1) && ((Map)row[2]).values().contains(value2))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -255,9 +390,9 @@ protected Object[][] getExpectedRows(Object value1, Object value2, Object[][] al
 
     public static class MapKeysQuerySet extends CollectionQuerySet
     {
-        public MapKeysQuerySet(DataSet<?> elementDataSet)
+        public MapKeysQuerySet(DataSet<?> dataSet, DataSet<?> elementDataSet)
         {
-            super(elementDataSet);
+            super(dataSet, elementDataSet);
         }
 
         @Override
@@ -272,7 +407,7 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
             for (int and = 0; and < allRows.length / 4; and++)
             {
                 int index = getRandom().nextIntBetween(0, allRows.length - 1);
-                Map<?, ?> map = (Map<?, ?>)allRows[index][VALUE_INDEX];
+                Map map = (Map)allRows[index][2];
                 Object key1 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
                 Object key2 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
                 assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS KEY ? AND value CONTAINS KEY ?",
@@ -285,7 +420,7 @@ protected Object[][] getExpectedRows(Object value, Object[][] allRows)
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Map<?, ?>)row[VALUE_INDEX]).containsKey(value))
+                if (((Map)row[2]).keySet().contains(value))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -296,7 +431,7 @@ protected Object[][] getExpectedRows(Object value1, Object value2, Object[][] al
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Map<?, ?>)row[VALUE_INDEX]).containsKey(value1) && ((Map<?, ?>)row[VALUE_INDEX]).containsKey(value2))
+                if (((Map)row[2]).keySet().contains(value1) && ((Map)row[2]).keySet().contains(value2))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -305,9 +440,9 @@ protected Object[][] getExpectedRows(Object value1, Object value2, Object[][] al
 
     public static class MapEntriesQuerySet extends CollectionQuerySet
     {
-        public MapEntriesQuerySet(DataSet<?> elementDataSet)
+        public MapEntriesQuerySet(DataSet<?> dataSet, DataSet<?> elementDataSet)
         {
-            super(elementDataSet);
+            super(dataSet, elementDataSet);
         }
 
         @Override
@@ -315,7 +450,7 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
         {
             for (int index = 0; index < allRows.length; index++)
             {
-                Map<?, ?> map = (Map<?, ?>)allRows[index][VALUE_INDEX];
+                Map map = (Map)allRows[index][2];
                 Object key = map.keySet().toArray()[0];
                 Object value = map.get(key);
                 assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ?",
@@ -324,13 +459,23 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
             for (int and = 0; and < allRows.length / 4; and++)
             {
                 int index = getRandom().nextIntBetween(0, allRows.length - 1);
-                Map<?, ?> map = (Map<?, ?>)allRows[index][VALUE_INDEX];
+                Map map = (Map)allRows[index][2];
                 Object key1 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
                 Object value1 = map.get(key1);
                 Object key2 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
                 Object value2 = map.get(key2);
                 assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ? AND value[?] = ?",
                         key1, value1, key2, value2), getExpectedRows(key1, value1, key2, value2, allRows));
+
+                // This element is defined to be a key in all the maps
+                var keyInAllMaps = elementDataSet.values[0];
+                var randomElement = elementDataSet.values[getRandom().nextIntBetween(0, allRows.length - 1)];
+                var gt = tester.execute("SELECT * FROM %s WHERE value[?] > ?", keyInAllMaps, randomElement);
+                var lte = tester.execute("SELECT * FROM %s WHERE value[?] <= ?", keyInAllMaps, randomElement);
+                assertEquals(elementDataSet.values.length, lte.size() + gt.size());
+                var lt = tester.execute("SELECT * FROM %s WHERE value[?] < ?", keyInAllMaps, randomElement);
+                var gte = tester.execute("SELECT * FROM %s WHERE value[?] >= ?", keyInAllMaps, randomElement);
+                assertEquals(elementDataSet.values.length, lt.size() + gte.size());
             }
         }
 
@@ -339,7 +484,7 @@ protected Object[][] getExpectedRows(Object key, Object value, Object[][] allRow
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                Map<?, ?> rowMap = (Map<?, ?>)row[VALUE_INDEX];
+                Map rowMap = (Map)row[2];
                 if (rowMap.containsKey(key))
                 {
                     if (rowMap.get(key).equals(value))
@@ -354,7 +499,7 @@ protected Object[][] getExpectedRows(Object key1, Object value1, Object key2, Ob
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                Map<?, ?> rowMap = (Map<?, ?>)row[VALUE_INDEX];
+                Map rowMap = (Map)row[2];
                 if (rowMap.containsKey(key1) && rowMap.containsKey(key2))
                 {
                     if (rowMap.get(key1).equals(value1) && rowMap.get(key2).equals(value2))
@@ -367,9 +512,9 @@ protected Object[][] getExpectedRows(Object key1, Object value1, Object key2, Ob
 
     public static class MultiMapQuerySet extends CollectionQuerySet
     {
-        public MultiMapQuerySet(DataSet<?> elementDataSet)
+        public MultiMapQuerySet(DataSet<?> dataSet, DataSet<?> elementDataSet)
         {
-            super(elementDataSet);
+            super(dataSet, elementDataSet);
         }
 
         @Override
@@ -377,7 +522,7 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
         {
             for (int index = 0; index < allRows.length; index++)
             {
-                Map<?, ?> map = (Map<?, ?>)allRows[index][VALUE_INDEX];
+                Map map = (Map)allRows[index][2];
                 Object key = map.keySet().toArray()[0];
                 Object value = map.get(key);
 
@@ -393,7 +538,7 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
             for (int and = 0; and < allRows.length / 4; and++)
             {
                 int index = getRandom().nextIntBetween(0, allRows.length - 1);
-                Map<?, ?> map = (Map<?, ?>)allRows[index][VALUE_INDEX];
+                Map map = (Map)allRows[index][2];
                 Object key1 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
                 Object value1 = map.get(key1);
                 Object key2 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
@@ -409,7 +554,7 @@ public void runQueries(SAITester tester, Object[][] allRows) throws Throwable
                                         getExpectedEntryRows(key1, value1, key2, value2, allRows));
 
                 assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ? AND value CONTAINS KEY ? AND value CONTAINS ?", key1, value1, key2, value2),
-                                        getExpectedMixedRows(key1, value1, key2, value2, allRows));
+                        getExpectedMixedRows(key1, value1, key2, value2, allRows));
             }
         }
 
@@ -418,7 +563,7 @@ protected Object[][] getExpectedKeyRows(Object value, Object[][] allRows)
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Map<?, ?>)row[VALUE_INDEX]).containsKey(value))
+                if (((Map)row[2]).keySet().contains(value))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -429,7 +574,7 @@ protected Object[][] getExpectedValueRows(Object value, Object[][] allRows)
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Map<?, ?>)row[VALUE_INDEX]).containsValue(value))
+                if (((Map)row[2]).values().contains(value))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -440,7 +585,7 @@ protected Object[][] getExpectedEntryRows(Object key, Object value, Object[][] a
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                Map<?, ?> rowMap = (Map<?, ?>)row[VALUE_INDEX];
+                Map rowMap = (Map)row[2];
                 if (rowMap.containsKey(key))
                 {
                     if (rowMap.get(key).equals(value))
@@ -455,7 +600,7 @@ protected Object[][] getExpectedKeyRows(Object value1, Object value2, Object[][]
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Map<?, ?>)row[VALUE_INDEX]).containsKey(value1) && ((Map<?, ?>)row[VALUE_INDEX]).containsKey(value2))
+                if (((Map)row[2]).keySet().contains(value1) && ((Map)row[2]).keySet().contains(value2))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -466,7 +611,7 @@ protected Object[][] getExpectedValueRows(Object value1, Object value2, Object[]
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                if (((Map<?, ?>)row[VALUE_INDEX]).containsValue(value1) && ((Map<?, ?>)row[VALUE_INDEX]).containsValue(value2))
+                if (((Map)row[2]).values().contains(value1) && ((Map)row[2]).values().contains(value2))
                     expected.add(row);
             }
             return expected.toArray(new Object[][]{});
@@ -477,7 +622,7 @@ protected Object[][] getExpectedEntryRows(Object key1, Object value1, Object key
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                Map<?, ?> rowMap = (Map<?, ?>)row[VALUE_INDEX];
+                Map rowMap = (Map)row[2];
                 if (rowMap.containsKey(key1) && rowMap.containsKey(key2))
                 {
                     if (rowMap.get(key1).equals(value1) && rowMap.get(key2).equals(value2))
@@ -492,7 +637,7 @@ protected Object[][] getExpectedMixedRows(Object key1, Object value1, Object key
             List<Object[]> expected = new ArrayList<>();
             for (Object[] row : allRows)
             {
-                Map<?, ?> rowMap = (Map<?, ?>)row[VALUE_INDEX];
+                Map rowMap = (Map)row[2];
                 if (rowMap.containsKey(key1) && rowMap.containsKey(key2) && rowMap.containsValue(value2))
                 {
                     if (rowMap.get(key1).equals(value1))
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/SmallintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/SmallintTest.java
index 617752e57702..78a6ffe13f1f 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/SmallintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/SmallintTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class SmallintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.SmallintDataSet());
     }
+
+    public SmallintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/TextTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/TextTest.java
index 7c40d8e00f72..205a870cb1f8 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/TextTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/TextTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class TextTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.TextDataSet());
     }
+
+    public TextTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/TimeTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/TimeTest.java
index 316b371e624d..a68b7691908b 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/TimeTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/TimeTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class TimeTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.TimeDataSet());
     }
+
+    public TimeTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/TimestampTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/TimestampTest.java
index 1992c7ac3580..a57391be8378 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/TimestampTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/TimestampTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class TimestampTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.TimestampDataSet());
     }
+
+    public TimestampTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/TimeuuidTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/TimeuuidTest.java
index b5b7deeac704..85d7b9fa60fc 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/TimeuuidTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/TimeuuidTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class TimeuuidTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.TimeuuidDataSet());
     }
+
+    public TimeuuidTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/TinyintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/TinyintTest.java
index 118d5c7f36d7..eb9c9b887ee1 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/TinyintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/TinyintTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class TinyintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.TinyintDataSet());
     }
+
+    public TinyintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/UuidTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/UuidTest.java
index ed11f0ee7eeb..c3ed2b8501bb 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/UuidTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/UuidTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class UuidTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.UuidDataSet());
     }
+
+    public UuidTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/VarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/VarintTest.java
index 5263380ece6b..3674d7b334eb 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/VarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/VarintTest.java
@@ -19,13 +19,29 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+@RunWith(Parameterized.class)
 public class VarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new DataSet.VarintDataSet());
     }
+
+    public VarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/CollectionDataSet.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/CollectionDataSet.java
index d33991d5342d..e5422a4944fd 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/CollectionDataSet.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/CollectionDataSet.java
@@ -20,7 +20,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -36,7 +35,7 @@ public abstract class CollectionDataSet<T> extends DataSet<T>
 {
     public static class SetDataSet<T> extends CollectionDataSet<Set<T>>
     {
-        protected final DataSet<T> elementDataSet;
+        protected DataSet<T> elementDataSet;
 
         public SetDataSet(DataSet<T> elementDataSet)
         {
@@ -55,7 +54,7 @@ public SetDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.CollectionQuerySet(elementDataSet);
+            return new QuerySet.CollectionQuerySet(this, elementDataSet);
         }
 
         public String toString()
@@ -74,13 +73,13 @@ public FrozenSetDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.FrozenCollectionQuerySet();
+            return new QuerySet.FrozenCollectionQuerySet(this);
         }
 
         @Override
         public Collection<String> decorateIndexColumn(String column)
         {
-            return Collections.singletonList(String.format("FULL(%s)", column));
+            return Arrays.asList(String.format("FULL(%s)", column));
         }
 
         public String toString()
@@ -91,7 +90,7 @@ public String toString()
 
     public static class ListDataSet<T> extends CollectionDataSet<List<T>>
     {
-        protected final DataSet<T> elementDataSet;
+        protected DataSet<T> elementDataSet;
 
         public ListDataSet(DataSet<T> elementDataSet)
         {
@@ -110,7 +109,7 @@ public ListDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.CollectionQuerySet(elementDataSet);
+            return new QuerySet.CollectionQuerySet(this, elementDataSet);
         }
 
         public String toString()
@@ -129,13 +128,13 @@ public FrozenListDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.FrozenCollectionQuerySet();
+            return new QuerySet.FrozenCollectionQuerySet(this);
         }
 
         @Override
         public Collection<String> decorateIndexColumn(String column)
         {
-            return Collections.singletonList(String.format("FULL(%s)", column));
+            return Arrays.asList(String.format("FULL(%s)", column));
         }
 
         public String toString()
@@ -146,7 +145,7 @@ public String toString()
 
     public static class MapDataSet<T> extends CollectionDataSet<Map<T, T>>
     {
-        protected final DataSet<T> elementDataSet;
+        protected DataSet<T> elementDataSet;
 
         public MapDataSet(DataSet<T> elementDataSet)
         {
@@ -157,7 +156,9 @@ public MapDataSet(DataSet<T> elementDataSet)
                 values[index] = new HashMap<>();
                 for (int element = 0; element < getRandom().nextIntBetween(2, 8); element++)
                 {
-                    T key = elementDataSet.values[getRandom().nextIntBetween(0, elementDataSet.values.length - 1)];
+                    // This guarantees that every map will have at least 2 of the same keys. This makes
+                    // tests on map entries more meaningful.
+                    T key = elementDataSet.values[element % elementDataSet.values.length];
                     T value = elementDataSet.values[getRandom().nextIntBetween(0, elementDataSet.values.length - 1)];
                     values[index].put(key, value);
                 }
@@ -167,7 +168,7 @@ public MapDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.MapValuesQuerySet(elementDataSet);
+            return new QuerySet.MapValuesQuerySet(this, elementDataSet);
         }
 
         public String toString()
@@ -186,13 +187,13 @@ public FrozenMapValuesDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.FrozenCollectionQuerySet();
+            return new QuerySet.FrozenCollectionQuerySet(this);
         }
 
         @Override
         public Collection<String> decorateIndexColumn(String column)
         {
-            return Collections.singletonList(String.format("FULL(%s)", column));
+            return Arrays.asList(String.format("FULL(%s)", column));
         }
 
         public String toString()
@@ -211,13 +212,13 @@ public MapKeysDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.MapKeysQuerySet(elementDataSet);
+            return new QuerySet.MapKeysQuerySet(this, elementDataSet);
         }
 
         @Override
         public Collection<String> decorateIndexColumn(String column)
         {
-            return Collections.singletonList(String.format("KEYS(%s)", column));
+            return Arrays.asList(String.format("KEYS(%s)", column));
         }
     }
 
@@ -231,13 +232,13 @@ public MapValuesDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.MapValuesQuerySet(elementDataSet);
+            return new QuerySet.MapValuesQuerySet(this, elementDataSet);
         }
 
         @Override
         public Collection<String> decorateIndexColumn(String column)
         {
-            return Collections.singletonList(String.format("VALUES(%s)", column));
+            return Arrays.asList(String.format("VALUES(%s)", column));
         }
     }
 
@@ -251,13 +252,13 @@ public MapEntriesDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.MapEntriesQuerySet(elementDataSet);
+            return new QuerySet.MapEntriesQuerySet(this, elementDataSet);
         }
 
         @Override
         public Collection<String> decorateIndexColumn(String column)
         {
-            return Collections.singletonList(String.format("ENTRIES(%s)", column));
+            return Arrays.asList(String.format("ENTRIES(%s)", column));
         }
     }
 
@@ -271,7 +272,7 @@ public MultiMapDataSet(DataSet<T> elementDataSet)
         @Override
         public QuerySet querySet()
         {
-            return new QuerySet.MultiMapQuerySet(elementDataSet);
+            return new QuerySet.MultiMapQuerySet(this, elementDataSet);
         }
 
         @Override
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListAsciiTest.java
index 03d4d22579cd..9c37a19612ae 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenListAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenListDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public FrozenListAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListDecimalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListDecimalTest.java
index c23926744da0..59d9de661491 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListDecimalTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListDecimalTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenListDecimalTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenListDataSet<>(new DataSet.DecimalDataSet()));
     }
+
+    public FrozenListDecimalTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListIntTest.java
index 8d3ab6793552..0d7d889d9732 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenListIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenListDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public FrozenListIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListVarintTest.java
index 98ac32e469d4..571acf9f40bc 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/FrozenListVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenListVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenListDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public FrozenListVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListAsciiTest.java
index 22c4c9ff2f82..a27ed3115a5e 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public ListAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListBigintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListBigintTest.java
index 44c856f22df8..0861aa32b8b2 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListBigintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListBigintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListBigintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.BigintDataSet()));
     }
+
+    public ListBigintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDateTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDateTest.java
index a79841cb2dbf..dc71678f2821 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDateTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDateTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListDateTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.DateDataSet()));
     }
+
+    public ListDateTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDecimalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDecimalTest.java
index c52f17d6edf0..18613dcb0edd 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDecimalTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDecimalTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListDecimalTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.DecimalDataSet()));
     }
+
+    public ListDecimalTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDoubleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDoubleTest.java
index 64e1363d083a..d39a951dba32 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDoubleTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListDoubleTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListDoubleTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.DoubleDataSet()));
     }
+
+    public ListDoubleTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListFloatTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListFloatTest.java
index 8925e240b66e..264f7f1f605f 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListFloatTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListFloatTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListFloatTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.FloatDataSet()));
     }
+
+    public ListFloatTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListFrozenCollectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListFrozenCollectionTest.java
index c670acf5ecc4..ec8de55ef9da 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListFrozenCollectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListFrozenCollectionTest.java
@@ -20,18 +20,33 @@
 import java.util.Collection;
 import java.util.List;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListFrozenCollectionTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         DataSet<List<Integer>> frozen = new CollectionDataSet.FrozenListDataSet<>(new DataSet.IntDataSet());
         return generateParameters(new CollectionDataSet.ListDataSet<>(frozen));
     }
+
+    public ListFrozenCollectionTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListInetTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListInetTest.java
index 013897507874..71f81cb34154 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListInetTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListInetTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListInetTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.InetDataSet()));
     }
+
+    public ListInetTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListIntTest.java
index 0f581aa7d42b..2102f383ea65 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public ListIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListSmallintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListSmallintTest.java
index 3f6cb8b15601..9af64ed3c0df 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListSmallintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListSmallintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListSmallintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.SmallintDataSet()));
     }
+
+    public ListSmallintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTextTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTextTest.java
index 098a57f485a5..97029719a716 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTextTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTextTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListTextTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.TextDataSet()));
     }
+
+    public ListTextTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimeTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimeTest.java
index d443bf1fdf63..47e45bbc7b38 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimeTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimeTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListTimeTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.TimeDataSet()));
     }
+
+    public ListTimeTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimestampTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimestampTest.java
index 5797f5300b75..b9a7a5c14ff1 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimestampTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimestampTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListTimestampTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.TimestampDataSet()));
     }
+
+    public ListTimestampTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimeuuidTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimeuuidTest.java
index 087fc7e87653..fdd1353d3ebe 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimeuuidTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTimeuuidTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListTimeuuidTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.TimeuuidDataSet()));
     }
+
+    public ListTimeuuidTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTinyintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTinyintTest.java
index f4ed2562080f..4143c6400272 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTinyintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListTinyintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListTinyintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.TinyintDataSet()));
     }
+
+    public ListTinyintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListUuidTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListUuidTest.java
index 49a4841228ea..91dd1409f7df 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListUuidTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListUuidTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListUuidTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.UuidDataSet()));
     }
+
+    public ListUuidTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListVarintTest.java
index 17df15662426..fb539ed3910a 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/lists/ListVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class ListVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.ListDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public ListVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapAsciiTest.java
index dc3faf7cb390..678eb32c5248 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenMapAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenMapValuesDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public FrozenMapAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapDecimalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapDecimalTest.java
index d17c2c31454f..05a8df6ba043 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapDecimalTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapDecimalTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenMapDecimalTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenMapValuesDataSet<>(new DataSet.DecimalDataSet()));
     }
+
+    public FrozenMapDecimalTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapIntTest.java
index 4d1a39318371..6bb4e3b9cfba 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenMapIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenMapValuesDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public FrozenMapIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapVarintTest.java
index a393d70ac156..eb032e8d77c8 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/FrozenMapVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenMapVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenMapValuesDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public FrozenMapVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapAsciiTest.java
index 81650cf9c1de..f0ee349b440f 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public MapAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapBigintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapBigintTest.java
index 2e8e4348f3ae..dd82e88961e3 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapBigintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapBigintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapBigintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.BigintDataSet()));
     }
+
+    public MapBigintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDateTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDateTest.java
index d5c290290596..0a5fe86e975c 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDateTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDateTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapDateTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.DateDataSet()));
     }
+
+    public MapDateTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDecimalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDecimalTest.java
index 251192144432..c4cce21959d0 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDecimalTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDecimalTest.java
@@ -19,18 +19,33 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapDecimalTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.DecimalDataSet()));
     }
+
+    public MapDecimalTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
 
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDoubleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDoubleTest.java
index 67056b2f8107..d0fc29ae31b5 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDoubleTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapDoubleTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapDoubleTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.DoubleDataSet()));
     }
+
+    public MapDoubleTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesAsciiTest.java
index 4526b73cd876..269e785d18e4 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapEntriesAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapEntriesDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public MapEntriesAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesFrozenCollectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesFrozenCollectionTest.java
index 2f0653142383..7d08746e6fdc 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesFrozenCollectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesFrozenCollectionTest.java
@@ -20,18 +20,33 @@
 import java.util.Collection;
 import java.util.Map;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapEntriesFrozenCollectionTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         DataSet<Map<Integer, Integer>> frozen = new CollectionDataSet.FrozenMapValuesDataSet<>(new DataSet.IntDataSet());
         return generateParameters(new CollectionDataSet.MapEntriesDataSet<>(frozen));
     }
+
+    public MapEntriesFrozenCollectionTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesIntTest.java
index 71bd9b29b31e..c2a4be3af1fb 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapEntriesIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapEntriesDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public MapEntriesIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesVarintTest.java
index 852bc7d5198a..8962e4fcb55f 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapEntriesVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapEntriesVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapEntriesDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public MapEntriesVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapFloatTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapFloatTest.java
index 31e39c0b01dd..546209218075 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapFloatTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapFloatTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapFloatTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.FloatDataSet()));
     }
+
+    public MapFloatTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapFrozenCollectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapFrozenCollectionTest.java
index b39a8d3d7e2b..b47187e56860 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapFrozenCollectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapFrozenCollectionTest.java
@@ -20,18 +20,33 @@
 import java.util.Collection;
 import java.util.Map;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapFrozenCollectionTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         DataSet<Map<Integer, Integer>> frozen = new CollectionDataSet.FrozenMapValuesDataSet<>(new DataSet.IntDataSet());
         return generateParameters(new CollectionDataSet.MapDataSet<>(frozen));
     }
+
+    public MapFrozenCollectionTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapInetTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapInetTest.java
index 62def5b24a47..1206aaa1ecb9 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapInetTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapInetTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapInetTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.InetDataSet()));
     }
+
+    public MapInetTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapIntTest.java
index 5a33360e8722..6ec234530e0b 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public MapIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysAsciiTest.java
index 5b0837bcf5b9..2cefc2206517 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapKeysAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapKeysDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public MapKeysAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysFrozenCollectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysFrozenCollectionTest.java
index e2c4c84bf5b1..cc840f9bea4a 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysFrozenCollectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysFrozenCollectionTest.java
@@ -20,18 +20,33 @@
 import java.util.Collection;
 import java.util.Map;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapKeysFrozenCollectionTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         DataSet<Map<Integer, Integer>> frozen = new CollectionDataSet.FrozenMapValuesDataSet<>(new DataSet.IntDataSet());
         return generateParameters(new CollectionDataSet.MapKeysDataSet<>(frozen));
     }
+
+    public MapKeysFrozenCollectionTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysIntTest.java
index 6e75d155867f..d1379bb86974 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapKeysIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapKeysDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public MapKeysIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysVarintTest.java
index 52c395a8ac0e..e0a670a4cc79 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapKeysVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapKeysVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapKeysDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public MapKeysVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapSmallintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapSmallintTest.java
index cd058835953d..24a7877a1137 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapSmallintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapSmallintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapSmallintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.SmallintDataSet()));
     }
+
+    public MapSmallintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTextTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTextTest.java
index 29cf4543f34e..37a0b4ab78a2 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTextTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTextTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapTextTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.TextDataSet()));
     }
+
+    public MapTextTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimeTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimeTest.java
index 9923457000de..2e45c6477978 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimeTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimeTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapTimeTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.TimeDataSet()));
     }
+
+    public MapTimeTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimestampTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimestampTest.java
index 1f5cd8e08abd..43630937d322 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimestampTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimestampTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapTimestampTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.TimestampDataSet()));
     }
+
+    public MapTimestampTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimeuuidTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimeuuidTest.java
index 2a46698adaa1..202db31a286f 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimeuuidTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTimeuuidTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapTimeuuidTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.TimeuuidDataSet()));
     }
+
+    public MapTimeuuidTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTinyintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTinyintTest.java
index c8329058e578..6dcf45a83fe0 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTinyintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapTinyintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapTinyintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.TinyintDataSet()));
     }
+
+    public MapTinyintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapUuidTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapUuidTest.java
index 0963ee0303e8..9416b89464e9 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapUuidTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapUuidTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapUuidTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.UuidDataSet()));
     }
+
+    public MapUuidTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesAsciiTest.java
index 88c8e304f14f..7dcf18e2dbc4 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapValuesAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapValuesDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public MapValuesAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesFrozenCollectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesFrozenCollectionTest.java
index 15e9dfb12ceb..594ed2af0966 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesFrozenCollectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesFrozenCollectionTest.java
@@ -20,18 +20,33 @@
 import java.util.Collection;
 import java.util.Map;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapValuesFrozenCollectionTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         DataSet<Map<Integer, Integer>> frozen = new CollectionDataSet.FrozenMapValuesDataSet<>(new DataSet.IntDataSet());
         return generateParameters(new CollectionDataSet.MapValuesDataSet<>(frozen));
     }
+
+    public MapValuesFrozenCollectionTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesIntTest.java
index 3bf3840293bb..f5b8bbd1fc2b 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapValuesIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapValuesDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public MapValuesIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesVarintTest.java
index e5077fb8601a..4ef439cc4367 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapValuesVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapValuesVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapValuesDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public MapValuesVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapVarintTest.java
index d85880ab03dd..16e8dea6ad58 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MapVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MapVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MapDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public MapVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapAsciiTest.java
index 8c5f9f3d9c53..d80b4d80656f 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MultiMapAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MultiMapDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public MultiMapAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapIntTest.java
index 270f74195b42..cdbce9df514e 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MultiMapIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MultiMapDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public MultiMapIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapVarintTest.java
index 9d02cf2e93f9..4099d6a61f08 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/maps/MultiMapVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class MultiMapVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.MultiMapDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public MultiMapVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetAsciiTest.java
index 1672d2208e5b..d13f48a927d1 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenSetAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenSetDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public FrozenSetAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetDecimalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetDecimalTest.java
index 591bce541e9b..a687f031d9cf 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetDecimalTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetDecimalTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenSetDecimalTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenSetDataSet<>(new DataSet.DecimalDataSet()));
     }
-}
\ No newline at end of file
+
+    public FrozenSetDecimalTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetIntTest.java
index 6f43a4f24643..724109d08651 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenSetIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenSetDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public FrozenSetIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetVarintTest.java
index 9d9d759b5ea1..d8d5de944ee9 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/FrozenSetVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenSetVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.FrozenSetDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public FrozenSetVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetAsciiTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetAsciiTest.java
index e58ad4fef4b7..709d93e2a684 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetAsciiTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetAsciiTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetAsciiTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.AsciiDataSet()));
     }
+
+    public SetAsciiTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetBigintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetBigintTest.java
index bdf6fe0d8b9d..8a1448083c37 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetBigintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetBigintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetBigintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.BigintDataSet()));
     }
+
+    public SetBigintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDateTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDateTest.java
index 460e70d73386..e321df04437b 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDateTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDateTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetDateTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.DateDataSet()));
     }
+
+    public SetDateTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDecimalTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDecimalTest.java
index ef495d9b4d27..d239abfbfb64 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDecimalTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDecimalTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetDecimalTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.DecimalDataSet()));
     }
+
+    public SetDecimalTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDoubleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDoubleTest.java
index 86a4fa529286..dfa34d81878c 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDoubleTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetDoubleTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetDoubleTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.DoubleDataSet()));
     }
+
+    public SetDoubleTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetFloatTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetFloatTest.java
index 6c292103c370..16a30089dea7 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetFloatTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetFloatTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetFloatTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.FloatDataSet()));
     }
+
+    public SetFloatTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetFrozenCollectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetFrozenCollectionTest.java
index da4aca702d23..9af11b34936d 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetFrozenCollectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetFrozenCollectionTest.java
@@ -20,18 +20,33 @@
 import java.util.Collection;
 import java.util.Set;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetFrozenCollectionTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         DataSet<Set<Integer>> frozen = new CollectionDataSet.FrozenSetDataSet<>(new DataSet.IntDataSet());
         return generateParameters(new CollectionDataSet.SetDataSet<>(frozen));
     }
+
+    public SetFrozenCollectionTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetInetTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetInetTest.java
index 380ed4c2daff..d523b1264310 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetInetTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetInetTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetInetTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.InetDataSet()));
     }
+
+    public SetInetTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetIntTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetIntTest.java
index add979910f24..c4dd8e17ec13 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetIntTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetIntTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetIntTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.IntDataSet()));
     }
+
+    public SetIntTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetSmallintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetSmallintTest.java
index fa14363684ef..ded15b625079 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetSmallintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetSmallintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetSmallintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.SmallintDataSet()));
     }
+
+    public SetSmallintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTextTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTextTest.java
index 36e23148919c..1b8ac5579664 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTextTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTextTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetTextTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.TextDataSet()));
     }
+
+    public SetTextTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimeTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimeTest.java
index 21272d163efb..1c73658a9e4b 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimeTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimeTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetTimeTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.TimeDataSet()));
     }
+
+    public SetTimeTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimestampTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimestampTest.java
index e18ce5a9b23d..f78bee9e5614 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimestampTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimestampTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetTimestampTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.TimestampDataSet()));
     }
+
+    public SetTimestampTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimeuuidTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimeuuidTest.java
index 65d55569cb5f..d18d22ee8de2 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimeuuidTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTimeuuidTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetTimeuuidTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.TimeuuidDataSet()));
     }
+
+    public SetTimeuuidTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTinyintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTinyintTest.java
index 1a701bfd79e0..daf5ac4bcab0 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTinyintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetTinyintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetTinyintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.TinyintDataSet()));
     }
+
+    public SetTinyintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetUuidTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetUuidTest.java
index b4e554154b1a..9c3e941a9976 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetUuidTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetUuidTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetUuidTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.UuidDataSet()));
     }
+
+    public SetUuidTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetVarintTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetVarintTest.java
index 777ebbf3249a..6bd1c5f4bbc8 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetVarintTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/collections/sets/SetVarintTest.java
@@ -19,17 +19,32 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class SetVarintTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new CollectionDataSet.SetDataSet<>(new DataSet.VarintDataSet()));
     }
+
+    public SetVarintTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleCollectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleCollectionTest.java
index 427ae5d50df3..16f4652d6498 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleCollectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleCollectionTest.java
@@ -20,15 +20,19 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenTupleCollectionTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new FrozenTupleDataSet(
@@ -37,4 +41,15 @@ public static Collection<Object[]> generateParameters()
         new CollectionDataSet.MapDataSet<>(new DataSet.BigintDataSet())
         ));
     }
+
+    public FrozenTupleCollectionTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleDataSet.java b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleDataSet.java
index 227de27b1b8c..fab8d7047fdd 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleDataSet.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleDataSet.java
@@ -50,7 +50,7 @@ public FrozenTupleDataSet(DataSet<?>... elementDataSets)
     @Override
     public QuerySet querySet()
     {
-        return new QuerySet.FrozenTuple();
+        return new QuerySet.FrozenTuple(this);
     }
 
     @Override
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleTest.java
index 7733c3cce983..e1849869eef3 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleTest.java
@@ -20,16 +20,31 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenTupleTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new FrozenTupleDataSet(new DataSet.BigintDataSet(), new DataSet.AsciiDataSet()));
     }
+
+    public FrozenTupleTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleTupleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleTupleTest.java
index 98db78887c0f..2d5fdb430b74 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleTupleTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenTupleTupleTest.java
@@ -20,16 +20,31 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenTupleTupleTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new FrozenTupleDataSet(new FrozenTupleDataSet(new DataSet.AsciiDataSet(), new DataSet.UuidDataSet()), new DataSet.AsciiDataSet()));
     }
+
+    public FrozenTupleTupleTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTCollectionTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTCollectionTest.java
index ddd25b234122..3a9ea55fac75 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTCollectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTCollectionTest.java
@@ -20,19 +20,34 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
 import org.apache.cassandra.index.sai.cql.types.collections.CollectionDataSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenUDTCollectionTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new FrozenUDTDataSet(
         new CollectionDataSet.ListDataSet<>(new DataSet.AsciiDataSet()),
         new CollectionDataSet.MapDataSet<>(new DataSet.BigintDataSet())));
     }
+
+    public FrozenUDTCollectionTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTDataSet.java b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTDataSet.java
index 8e817779c5bc..f5a9b3d255a5 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTDataSet.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTDataSet.java
@@ -67,7 +67,7 @@ public void init()
     @Override
     public QuerySet querySet()
     {
-        return new QuerySet.FrozenTuple();
+        return new QuerySet.FrozenTuple(this);
     }
 
     @Override
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTTest.java
index 2510e0b734da..036f4869b0fc 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/FrozenUDTTest.java
@@ -20,16 +20,31 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class FrozenUDTTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new FrozenUDTDataSet(new DataSet.BigintDataSet(), new DataSet.AsciiDataSet()));
     }
+
+    public FrozenUDTTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/TupleTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/TupleTest.java
index d692d8a7c51e..16604997f1b5 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/TupleTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/multicell/TupleTest.java
@@ -20,16 +20,31 @@
 
 import java.util.Collection;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.index.sai.cql.types.DataSet;
 import org.apache.cassandra.index.sai.cql.types.IndexingTypeSupport;
+import org.apache.cassandra.index.sai.disk.format.Version;
 
+@RunWith(Parameterized.class)
 public class TupleTest extends IndexingTypeSupport
 {
-    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    @Parameterized.Parameters(name = "version={0},dataset={1},wide={2},scenario={3}")
     public static Collection<Object[]> generateParameters()
     {
         return generateParameters(new TupleDataSet(new DataSet.BigintDataSet(), new DataSet.AsciiDataSet()));
     }
+
+    public TupleTest(Version version, DataSet<?> dataset, boolean widePartitions, Scenario scenario)
+    {
+        super(version, dataset, widePartitions, scenario);
+    }
+
+    @Test
+    public void test() throws Throwable
+    {
+        runIndexQueryScenarios();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/ArrayPostingList.java b/test/unit/org/apache/cassandra/index/sai/disk/ArrayPostingList.java
deleted file mode 100644
index 42f20582ef17..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/ArrayPostingList.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk;
-
-import com.google.common.base.MoreObjects;
-
-import org.apache.cassandra.index.sai.postings.OrdinalPostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
-
-public class ArrayPostingList implements OrdinalPostingList
-{
-    private final long[] postings;
-    private int idx = 0;
-
-    public ArrayPostingList(long... postings)
-    {
-        this.postings = postings;
-    }
-
-    @Override
-    public long getOrdinal()
-    {
-        return idx;
-    }
-
-    @Override
-    public long nextPosting()
-    {
-        if (idx >= postings.length)
-        {
-            return PostingList.END_OF_STREAM;
-        }
-        return postings[idx++];
-    }
-
-    @Override
-    public long size()
-    {
-        return postings.length;
-    }
-
-    @Override
-    public long advance(long targetRowID)
-    {
-        for (int i = idx; i < postings.length; ++i)
-        {
-            final long segmentRowId = getPostingAt(i);
-
-            idx++;
-
-            if (segmentRowId >= targetRowID)
-            {
-                return segmentRowId;
-            }
-        }
-        return PostingList.END_OF_STREAM;
-    }
-
-    @Override
-    public String toString()
-    {
-        return MoreObjects.toStringHelper(this)
-                          .add("idx", idx)
-                          .add("hashCode", Integer.toHexString(hashCode()))
-                          .toString();
-    }
-
-    public void reset()
-    {
-        idx = 0;
-    }
-
-    public long getPostingAt(int i)
-    {
-        return postings[i];
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/FileUtils.java b/test/unit/org/apache/cassandra/index/sai/disk/FileUtils.java
new file mode 100644
index 000000000000..8b7c7be525a3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/FileUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.cassandra.io.sstable.Descriptor;
+
+import static org.apache.commons.io.FileUtils.copyDirectory;
+
+public class FileUtils
+{
+    public static void copySSTablesAndIndexes(Descriptor descriptor, String version) throws IOException
+    {
+        File srcDir = new File("test/data/legacy-sai/" + version);
+        copyDirectory(srcDir, descriptor.directory.toJavaIOFile());
+    }
+
+    public static void copySSTablesAndIndexes(File destination, String version) throws IOException
+    {
+        File srcDir = new File("test/data/legacy-sai/" + version);
+        copyDirectory(srcDir, destination);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java b/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java
index c87e013fdc5a..e1f4a9e439c7 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java
@@ -26,30 +26,31 @@
 import java.util.stream.Stream;
 
 import com.google.common.collect.ObjectArrays;
+import org.apache.cassandra.cql3.CQLTester;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndexBuilder;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
 import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.sai.disk.v1.SSTableIndexWriter;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.inject.Expression;
 import org.apache.cassandra.inject.Injection;
 import org.apache.cassandra.inject.Injections;
 import org.apache.cassandra.inject.InvokePointBuilder;
 import org.apache.cassandra.schema.Schema;
-import org.assertj.core.api.Assertions;
 
+import static org.apache.cassandra.inject.Expression.expr;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -98,26 +99,44 @@ public class NodeStartupTest extends SAITester
                                                                      .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndexBuilder.class).onMethod("build").atEntry())
                                                                      .build();
 
-    private static final Injections.Counter deletedPerSStableCounter = Injections.newCounter("deletedPrimaryKeyMapCounter")
-                                                                                 .add(InvokePointBuilder.newInvokePoint()
-                                                                                                        .onClass(IndexDescriptor.class)
-                                                                                                        .onMethod("deletePerSSTableIndexComponents")
-                                                                                                        .atEntry())
-                                                                                 .build();
-
-    private static final Injections.Counter deletedPerIndexCounter = Injections.newCounter("deletedColumnIndexCounter")
-                                                                               .add(InvokePointBuilder.newInvokePoint()
-                                                                                                      .onClass(IndexDescriptor.class)
-                                                                                                      .onMethod("deleteColumnIndex")
-                                                                                                      .atEntry())
-                                                                               .build();
-
-    private static final Injections.Counter[] counters = new Injections.Counter[] { buildCounter, deletedPerSStableCounter, deletedPerIndexCounter };
+    private static final Injections.Counter deletedPerSStableCounter = addConditions(Injections.newCounter("deletedPrimaryKeyMapCounter")
+                                                                                               .add(InvokePointBuilder.newInvokePoint()
+                                                                                                                      .onInterface("ComponentGroup$Writer")
+                                                                                                                      .onMethod("deleteAllComponents")
+                                                                                                                      .atEntry()),
+                                                                                     b -> b.not().when(expr(Expression.THIS).method("isPerIndexGroup").args())
+    ).build();
+
+    private static final Injections.Counter invalidatePerSStableCounter = addConditions(Injections.newCounter("invalidatePrimaryKeyMapCounter")
+                                                                                               .add(InvokePointBuilder.newInvokePoint()
+                                                                                                                      .onClass("IndexDescriptor$IndexComponentsImpl")
+                                                                                                                      .onMethod("invalidate")
+                                                                                                                      .atEntry()),
+                                                                                     b -> b.not().when(expr(Expression.THIS).method("isPerIndexGroup").args())
+    ).build();
+
+    private static final Injections.Counter deletedPerIndexCounter = addConditions(Injections.newCounter("deletedColumnIndexCounter")
+                                                                                             .add(InvokePointBuilder.newInvokePoint()
+                                                                                                                    .onInterface("ComponentGroup$Writer")
+                                                                                                                    .onMethod("deleteAllComponents")
+                                                                                                                    .atEntry()),
+                                                                                   b -> b.when(expr(Expression.THIS).method("isPerIndexGroup").args())
+    ).build();
+
+    private static final Injections.Counter invalidatePerIndexCounter = addConditions(Injections.newCounter("invalidateColumnIndexCounter")
+                                                                                                  .add(InvokePointBuilder.newInvokePoint()
+                                                                                                                         .onClass("IndexDescriptor$IndexComponentsImpl")
+                                                                                                                         .onMethod("invalidate")
+                                                                                                                         .atEntry()),
+                                                                                        b -> b.when(expr(Expression.THIS).method("isPerIndexGroup").args())
+    ).build();
+
+    private static final Injections.Counter[] counters = new Injections.Counter[] { buildCounter, deletedPerSStableCounter, invalidatePerSStableCounter, deletedPerIndexCounter, invalidatePerIndexCounter };
 
     private static Throwable error = null;
 
-    private IndexIdentifier indexIdentifier = null;
-    private IndexTermType indexTermType = null;
+    private String indexName = null;
+    private IndexContext indexContext = null;
 
     enum Populator
     {
@@ -157,7 +176,7 @@ enum IndexStateOnRestart
         PER_SSTABLE_INCOMPLETE,
         PER_COLUMN_INCOMPLETE,
         PER_SSTABLE_CORRUPT,
-        PER_COLUMN_CORRUPT
+        PER_COLUMN_CORRUPT;
     }
 
     enum StartupTaskRunOrder
@@ -179,12 +198,20 @@ public void enable()
         }
     }
 
+    // TODO: Disable the coordinator execution used by SAITester until we have a way to simulate node restarts combined
+    //  with CQLTester#requireNetwork and CQLTester#requireNetworkWithoutDriver. This should be improved in CNDB-13125.
+    @BeforeClass
+    public static void setUpClass()
+    {
+        CQLTester.setUpClass();
+    }
+
     @Before
     public void setup() throws Throwable
     {
-        createTable("CREATE TABLE %s (id text PRIMARY KEY, v1 text)");
-        indexIdentifier = createIndexIdentifier(createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName())));
-        indexTermType = createIndexTermType(Int32Type.instance);
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, v1 int)");
+        indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        indexContext = getIndexContext(indexName);
         Injections.inject(ObjectArrays.concat(barriers, counters, Injection.class));
         Stream.of(barriers).forEach(Injections.Barrier::reset);
         Stream.of(barriers).forEach(Injections.Barrier::disable);
@@ -193,7 +220,7 @@ public void setup() throws Throwable
         error = null;
     }
 
-    @Parameterized.Parameter
+    @Parameterized.Parameter(0)
     public Populator populator;
     @Parameterized.Parameter(1)
     public IndexStateOnRestart state;
@@ -204,8 +231,12 @@ public void setup() throws Throwable
     @Parameterized.Parameter(4)
     public int deletedPerSSTable;
     @Parameterized.Parameter(5)
-    public int deletedPerIndex;
+    public int invalidatePerSStable;
     @Parameterized.Parameter(6)
+    public int deletedPerIndex;
+    @Parameterized.Parameter(7)
+    public int invalidatePerIndex;
+    @Parameterized.Parameter(8)
     public int expectedDocuments;
 
     @SuppressWarnings("unused")
@@ -214,40 +245,40 @@ public static List<Object[]> startupScenarios()
     {
         List<Object[]> scenarios = new LinkedList<>();
 
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 0, 0, 0, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 0, 0, 0, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.ALL_EMPTY, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 1, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.ALL_EMPTY, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 1, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.ALL_EMPTY, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 1, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 1, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 1, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 1, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 0, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 0, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 0, 1, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 2, 2, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 2, 2, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 2, 2, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 0, 2, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 0, 2, DOCS });
-        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 0, 2, DOCS });
-        scenarios.add( new Object[] { Populator.NON_INDEXABLE_ROWS, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 0, 0, 0, 0 });
-        scenarios.add( new Object[] { Populator.NON_INDEXABLE_ROWS, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 0, 0, 0, 0 });
-        scenarios.add( new Object[] { Populator.TOMBSTONES, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 0, 0, 0, 0 });
-        scenarios.add( new Object[] { Populator.TOMBSTONES, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 0, 0, 0, 0 });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 0, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 0, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.ALL_EMPTY, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.ALL_EMPTY, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.ALL_EMPTY, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_INCOMPLETE, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 0, 0, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 0, 1, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 0, 1, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_SSTABLE_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 0, 1, 0, 0, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 1, 0, 0, 0, 1, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 1, 0, 0, 0, 1, DOCS });
+        scenarios.add( new Object[] { Populator.INDEXABLE_ROWS, IndexStateOnRestart.PER_COLUMN_CORRUPT, StartupTaskRunOrder.PRE_JOIN_RUNS_MID_BUILD, 1, 0, 0, 0, 1, DOCS });
+        scenarios.add( new Object[] { Populator.NON_INDEXABLE_ROWS, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 0, 0, 0, 0, 0, 0 });
+        scenarios.add( new Object[] { Populator.NON_INDEXABLE_ROWS, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 0, 0, 0, 0, 0, 0 });
+        scenarios.add( new Object[] { Populator.TOMBSTONES, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_BEFORE_BUILD, 0, 0, 0, 0, 0, 0 });
+        scenarios.add( new Object[] { Populator.TOMBSTONES, IndexStateOnRestart.VALID, StartupTaskRunOrder.PRE_JOIN_RUNS_AFTER_BUILD, 0, 0, 0, 0, 0, 0 });
 
         return scenarios;
     }
 
     @Test
-    public void startupOrderingTest()
+    public void startupOrderingTest() throws Throwable
     {
         populator.populate(this);
 
-        Assertions.assertThat(getNotQueryableIndexes()).isEmpty();
+        assertTrue(areAllTableIndexesQueryable());
         assertTrue(isGroupIndexComplete());
         assertTrue(isColumnIndexComplete());
-        Assert.assertEquals(expectedDocuments, execute("SELECT * FROM %s WHERE v1 = '0'").size());
+        Assert.assertEquals(expectedDocuments, execute("SELECT * FROM %s WHERE v1 >= 0").size());
 
         setState(state);
 
@@ -255,24 +286,25 @@ public void startupOrderingTest()
 
         simulateNodeRestart();
 
-        Assertions.assertThat(getNotQueryableIndexes()).isEmpty();
+        assertTrue(areAllTableIndexesQueryable());
         assertTrue(isGroupIndexComplete());
         assertTrue(isColumnIndexComplete());
-        Assert.assertEquals(expectedDocuments, execute("SELECT * FROM %s WHERE v1 = '0'").size());
+        Assert.assertEquals(expectedDocuments, execute("SELECT * FROM %s WHERE v1 >= 0").size());
 
         Assert.assertEquals(builds, buildCounter.get());
         Assert.assertEquals(deletedPerSSTable, deletedPerSStableCounter.get());
+        Assert.assertEquals(invalidatePerSStable, invalidatePerSStableCounter.get());
         Assert.assertEquals(deletedPerIndex, deletedPerIndexCounter.get());
+        Assert.assertEquals(invalidatePerIndex, invalidatePerIndexCounter.get());
     }
 
-    @SuppressWarnings("unused")
     public void populateIndexableRows()
     {
         try
         {
             for (int i = 0; i < DOCS; i++)
             {
-                execute("INSERT INTO %s (id, v1) VALUES (?, '0')", i);
+                execute("INSERT INTO %s (id, v1) VALUES (?, 0)", i);
             }
             flush();
         }
@@ -283,7 +315,6 @@ public void populateIndexableRows()
         }
     }
 
-    @SuppressWarnings("unused")
     public void populateNonIndexableRows()
     {
         try
@@ -301,7 +332,6 @@ public void populateNonIndexableRows()
         }
     }
 
-    @SuppressWarnings("unused")
     public void populateTombstones()
     {
         try
@@ -319,16 +349,16 @@ public void populateTombstones()
         }
     }
 
-    private boolean isGroupIndexComplete()
+    private boolean isGroupIndexComplete() throws Exception
     {
         ColumnFamilyStore cfs = Objects.requireNonNull(Schema.instance.getKeyspaceInstance(KEYSPACE)).getColumnFamilyStore(currentTable());
-        return cfs.getLiveSSTables().stream().allMatch(sstable -> IndexDescriptor.create(sstable).isPerSSTableIndexBuildComplete());
+        return cfs.getLiveSSTables().stream().allMatch(sstable -> loadDescriptor(sstable, cfs).perSSTableComponents().isComplete());
     }
 
-    private boolean isColumnIndexComplete()
+    private boolean isColumnIndexComplete() throws Exception
     {
         ColumnFamilyStore cfs = Objects.requireNonNull(Schema.instance.getKeyspaceInstance(KEYSPACE)).getColumnFamilyStore(currentTable());
-        return cfs.getLiveSSTables().stream().allMatch(sstable -> IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexIdentifier));
+        return cfs.getLiveSSTables().stream().allMatch(sstable -> IndexDescriptor.isIndexBuildCompleteOnDisk(sstable, indexContext));
     }
 
     private void setState(IndexStateOnRestart state)
@@ -338,25 +368,25 @@ private void setState(IndexStateOnRestart state)
             case VALID:
                 break;
             case ALL_EMPTY:
-                Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).forEach(this::remove);
-                Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType).forEach(c -> remove(c, indexIdentifier));
+                Version.latest().onDiskFormat().perSSTableComponentTypes().forEach(this::remove);
+                Version.latest().onDiskFormat().perIndexComponentTypes(indexContext).forEach(c -> remove(c, indexContext));
                 break;
             case PER_SSTABLE_INCOMPLETE:
-                remove(IndexComponent.GROUP_COMPLETION_MARKER);
+                remove(IndexComponentType.GROUP_COMPLETION_MARKER);
                 break;
             case PER_COLUMN_INCOMPLETE:
-                remove(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier);
+                remove(IndexComponentType.COLUMN_COMPLETION_MARKER, indexContext);
                 break;
             case PER_SSTABLE_CORRUPT:
-                corrupt();
+                corrupt(IndexComponentType.GROUP_META);
                 break;
             case PER_COLUMN_CORRUPT:
-                corrupt(indexIdentifier);
+                corrupt(IndexComponentType.META, indexContext);
                 break;
         }
     }
 
-    private void remove(IndexComponent component)
+    private void remove(IndexComponentType component)
     {
         try
         {
@@ -369,11 +399,11 @@ private void remove(IndexComponent component)
         }
     }
 
-    private void remove(IndexComponent component, IndexIdentifier indexIdentifier)
+    private void remove(IndexComponentType component, IndexContext indexContext)
     {
         try
         {
-            corruptIndexComponent(component, indexIdentifier, CorruptionType.REMOVED);
+            corruptIndexComponent(component, indexContext, CorruptionType.REMOVED);
         }
         catch (Exception e)
         {
@@ -382,11 +412,11 @@ private void remove(IndexComponent component, IndexIdentifier indexIdentifier)
         }
     }
 
-    private void corrupt()
+    private void corrupt(IndexComponentType component)
     {
         try
         {
-            corruptIndexComponent(IndexComponent.GROUP_META, CorruptionType.TRUNCATED_HEADER);
+            corruptIndexComponent(component, CorruptionType.TRUNCATED_HEADER);
         }
         catch (Exception e)
         {
@@ -395,11 +425,11 @@ private void corrupt()
         }
     }
 
-    private void corrupt(IndexIdentifier indexIdentifier)
+    private void corrupt(IndexComponentType component, IndexContext indexContext)
     {
         try
         {
-            corruptIndexComponent(IndexComponent.META, indexIdentifier, CorruptionType.TRUNCATED_HEADER);
+            corruptIndexComponent(component, indexContext, CorruptionType.TRUNCATED_HEADER);
         }
         catch (Exception e)
         {
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/PostingListKeyRangeIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/disk/PostingListKeyRangeIteratorTest.java
new file mode 100644
index 000000000000..851471140e7a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/PostingListKeyRangeIteratorTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.KDTreeIndexBuilder;
+import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList;
+import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
+import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+public class PostingListKeyRangeIteratorTest
+{
+    private static final PrimaryKeyMap pkm = KDTreeIndexBuilder.TEST_PRIMARY_KEY_MAP;
+
+    @Test
+    public void testRemoveDuplicatePostings() throws IOException
+    {
+        @SuppressWarnings("resource")
+        var postingList = new IntArrayPostingList(new int[]{ 1, 1, 2, 2, 3});
+        var mockIndexContext = mock(IndexContext.class);
+        var indexContext = new IndexSearcherContext(pkm.primaryKeyFromRowId(1),
+                                                    pkm.primaryKeyFromRowId(3),
+                                                    0,
+                                                    3,
+                                                    0,
+                                                    new QueryContext(10000),
+                                                    postingList);
+        try (var iterator = new PostingListKeyRangeIterator(mockIndexContext, pkm, indexContext))
+        {
+            assertEquals(pkm.primaryKeyFromRowId(1), iterator.next());
+            assertEquals(pkm.primaryKeyFromRowId(2), iterator.next());
+            assertEquals(pkm.primaryKeyFromRowId(3), iterator.next());
+            assertFalse(iterator.hasNext());
+        }
+    }
+
+    @Test
+    @SuppressWarnings("resource")
+    public void testContrivedScenariosUnion() throws IOException
+    {
+        var postingList1 = new IntArrayPostingList(new int[]{ 3});
+        var postingList2 = new IntArrayPostingList(new int[]{ 1});
+        var postingList3 = new IntArrayPostingList(new int[]{ 3});
+        var mockIndexContext = mock(IndexContext.class);
+        var mpl = MergePostingList.merge(Lists.newArrayList(postingList1, postingList2));
+        var indexContext1 = buildIndexContext(1, 3, mpl);
+        var indexContext2 = buildIndexContext(3, 3, postingList3);
+        var plri1 = new PostingListKeyRangeIterator(mockIndexContext, pkm, indexContext1);
+        var plri2 = new PostingListKeyRangeIterator(mockIndexContext, pkm, indexContext2);
+        try (var union = KeyRangeUnionIterator.builder().add(plri1).add(plri2).build();)
+        {
+            union.skipTo(pkm.primaryKeyFromRowId(2));
+            assertTrue(union.hasNext());
+            union.next();
+            union.skipTo(pkm.primaryKeyFromRowId(3));
+            assertFalse(union.hasNext());
+        }
+    }
+
+    private IndexSearcherContext buildIndexContext(int minRowId, int maxRowId, PostingList list) throws IOException
+    {
+        return new IndexSearcherContext(pkm.primaryKeyFromRowId(minRowId),
+                                        pkm.primaryKeyFromRowId(maxRowId),
+                                        minRowId,
+                                        maxRowId,
+                                        0,
+                                        new QueryContext(10000),
+                                        list);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/RAMPostingSlicesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/RAMPostingSlicesTest.java
new file mode 100644
index 000000000000..0f5c52358ba3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/RAMPostingSlicesTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.FixedBitSet;
+
+public class RAMPostingSlicesTest extends SaiRandomizedTest
+{
+    @Test
+    public void testRAMPostingSlices() throws Exception
+    {
+        RAMPostingSlices slices = new RAMPostingSlices(Counter.newCounter(), false);
+
+        int[] segmentRowIdUpto = new int[1024];
+        Arrays.fill(segmentRowIdUpto, -1);
+
+        FixedBitSet[] bitSets = new FixedBitSet[segmentRowIdUpto.length];
+
+        for (int x = 0; x < 1_000_000; x++)
+        {
+            int termID = nextInt(segmentRowIdUpto.length);
+
+            if (segmentRowIdUpto[termID] == -1)
+            {
+                slices.createNewSlice(termID);
+            }
+
+            segmentRowIdUpto[termID]++;
+
+            if (bitSets[termID] == null)
+            {
+                bitSets[termID] = new FixedBitSet(1_000_000);
+            }
+
+            bitSets[termID].set(segmentRowIdUpto[termID]);
+
+            slices.writePosting(termID, segmentRowIdUpto[termID], 1);
+        }
+
+        for (int termID = 0; termID < segmentRowIdUpto.length; termID++)
+        {
+            ByteSliceReader reader = new ByteSliceReader();
+            slices.initReader(reader, termID);
+
+            int segmentRowId = -1;
+
+            while (!reader.eof())
+            {
+                segmentRowId = reader.readVInt();
+                assertTrue("termID=" + termID + " segmentRowId=" + segmentRowId, bitSets[termID].get(segmentRowId));
+            }
+            assertEquals(segmentRowId, segmentRowIdUpto[termID]);
+        }
+    }
+
+    @Test
+    public void testRAMPostingSlicesWithFrequencies() throws Exception {
+        RAMPostingSlices slices = new RAMPostingSlices(Counter.newCounter(), true);
+
+        // Test with just 3 terms and known frequencies
+        for (int termId = 0; termId < 3; termId++) {
+            slices.createNewSlice(termId);
+
+            // Write a sequence of rows with different frequencies for each term
+            slices.writePosting(termId, 5, 1);    // first posting at row 5
+            slices.writePosting(termId, 3, 2);    // next at row 8 (delta=3)
+            slices.writePosting(termId, 2, 3);    // next at row 10 (delta=2)
+        }
+
+        // Verify each term's postings
+        for (int termId = 0; termId < 3; termId++) {
+            ByteSliceReader reader = new ByteSliceReader();
+            PostingList postings = slices.postingList(termId, reader, 10);
+
+            assertEquals(5, postings.nextPosting());
+            assertEquals(1, postings.frequency());
+
+            assertEquals(8, postings.nextPosting());
+            assertEquals(2, postings.frequency());
+
+            assertEquals(10, postings.nextPosting());
+            assertEquals(3, postings.frequency());
+
+            assertEquals(PostingList.END_OF_STREAM, postings.nextPosting());
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/RAMStringIndexerTest.java b/test/unit/org/apache/cassandra/index/sai/disk/RAMStringIndexerTest.java
new file mode 100644
index 000000000000..775072905860
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/RAMStringIndexerTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.apache.lucene.util.BytesRef;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.string;
+
+public class RAMStringIndexerTest extends SaiRandomizedTest
+{
+    @Test
+    public void test() throws Exception
+    {
+        RAMStringIndexer indexer = new RAMStringIndexer(false);
+
+        indexer.addAll(List.of(new BytesRef("0")), 100);
+        indexer.addAll(List.of(new BytesRef("2")), 102);
+        indexer.addAll(List.of(new BytesRef("0")), 200);
+        indexer.addAll(List.of(new BytesRef("2")), 202);
+        indexer.addAll(List.of(new BytesRef("2")), 302);
+
+        List<List<Long>> matches = new ArrayList<>();
+        matches.add(Arrays.asList(100L, 200L));
+        matches.add(Arrays.asList(102L, 202L, 302L));
+
+        try (TermsIterator terms = indexer.getTermsWithPostings(ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("2"), TypeUtil.BYTE_COMPARABLE_VERSION))
+        {
+            int ord = 0;
+            while (terms.hasNext())
+            {
+                terms.next();
+                try (PostingList postings = terms.postings())
+                {
+                    List<Long> results = new ArrayList<>();
+                    long segmentRowId;
+                    while ((segmentRowId = postings.nextPosting()) != PostingList.END_OF_STREAM)
+                    {
+                        results.add(segmentRowId);
+                    }
+                    assertEquals(matches.get(ord++), results);
+                }
+            }
+            // The min and max are configured, not calculated.
+            assertArrayEquals("0".getBytes(), terms.getMinTerm().array());
+            assertArrayEquals("2".getBytes(), terms.getMaxTerm().array());
+        }
+    }
+
+    @Test
+    public void testWithFrequencies() throws Exception
+    {
+        RAMStringIndexer indexer = new RAMStringIndexer(true);
+
+        // Add same term twice in same row to increment frequency
+        indexer.addAll(List.of(new BytesRef("A"), new BytesRef("A")), 100);
+        indexer.addAll(List.of(new BytesRef("B")), 102);
+        indexer.addAll(List.of(new BytesRef("A"), new BytesRef("A"), new BytesRef("A")), 200);
+        indexer.addAll(List.of(new BytesRef("B"), new BytesRef("B")), 202);
+        indexer.addAll(List.of(new BytesRef("B")), 302);
+
+        // Expected results: rowID -> frequency
+        List<Map<Long, Integer>> matches = Arrays.asList(Map.of(100L, 2, 200L, 3),
+                                                         Map.of(102L, 1, 202L, 2, 302L, 1));
+
+        try (TermsIterator terms = indexer.getTermsWithPostings(ByteBufferUtil.bytes("A"), ByteBufferUtil.bytes("B"), TypeUtil.BYTE_COMPARABLE_VERSION))
+        {
+            int ord = 0;
+            while (terms.hasNext())
+            {
+                terms.next();
+                try (PostingList postings = terms.postings())
+                {
+                    Map<Long, Integer> results = new HashMap<>();
+                    long segmentRowId;
+                    while ((segmentRowId = postings.nextPosting()) != PostingList.END_OF_STREAM)
+                    {
+                        results.put(segmentRowId, postings.frequency());
+                    }
+                    assertEquals(matches.get(ord++), results);
+                }
+            }
+            assertArrayEquals("A".getBytes(), terms.getMinTerm().array());
+            assertArrayEquals("B".getBytes(), terms.getMaxTerm().array());
+        }
+    }
+
+    @Test
+    public void testLargeSegment() throws IOException
+    {
+        final RAMStringIndexer indexer = new RAMStringIndexer(false);
+        final int numTerms = between(1 << 10, 1 << 13);
+        final int numPostings = between(1 << 5, 1 << 10);
+
+        for (int id = 0; id < numTerms; ++id)
+        {
+            final BytesRef term = new BytesRef(String.format("%04d", id));
+            for (int posting = 0; posting < numPostings; ++posting)
+            {
+                indexer.addAll(List.of(term), posting);
+            }
+        }
+
+        final TermsIterator terms = indexer.getTermsWithPostings(ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER, TypeUtil.BYTE_COMPARABLE_VERSION);
+
+        ByteComparable term;
+        long termOrd = 0L;
+        while (terms.hasNext())
+        {
+            term = terms.next();
+            final ByteBuffer decoded = ByteBuffer.wrap(ByteSourceInverse.readBytes(term.asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION)));
+            assertEquals(String.format("%04d", termOrd), string(decoded));
+
+            try (PostingList postingList = terms.postings())
+            {
+                assertEquals(numPostings, postingList.size());
+                for (int i = 0; i < numPostings; ++i)
+                {
+                    assertEquals(i, postingList.nextPosting());
+                }
+                assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
+            }
+            termOrd++;
+        }
+
+        assertEquals(numTerms, termOrd);
+    }
+
+    @Test
+    public void testRequiresFlush()
+    {
+        int maxBlockBytePoolSize = RAMStringIndexer.MAX_BLOCK_BYTE_POOL_SIZE;
+        try
+        {
+            RAMStringIndexer.MAX_BLOCK_BYTE_POOL_SIZE = 1024 * 1024 * 100;
+            // primary behavior we're testing is that exceptions aren't thrown due to overflowing backing structures
+            RAMStringIndexer indexer = new RAMStringIndexer(false);
+
+            Assert.assertFalse(indexer.requiresFlush());
+            for (int i = 0; i < Integer.MAX_VALUE; i++)
+            {
+                if (indexer.requiresFlush())
+                    break;
+                indexer.addAll(List.of(new BytesRef(String.format("%5000d", i))), i);
+            }
+            // If we don't require a flush before MAX_VALUE, the implementation of RAMStringIndexer has sufficiently
+            // changed to warrant changes to the test.
+            Assert.assertTrue(indexer.requiresFlush());
+        }
+        finally
+        {
+            RAMStringIndexer.MAX_BLOCK_BYTE_POOL_SIZE = maxBlockBytePoolSize;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java
deleted file mode 100644
index 6806110d6571..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.datastax.driver.core.Session;
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
-import org.apache.cassandra.inject.Injections;
-
-import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
-import static org.junit.Assert.assertEquals;
-
-public class SelectiveIntersectionTest extends SAITester
-{
-    private static final Injections.Counter intersectionFlowCounter = Injections.newCounter("IntersectionFlowCounter")
-                                                                                .add(newInvokePoint().onClass("org.apache.cassandra.index.sai.iterators.KeyRangeIntersectionIterator").onMethod("<init>"))
-                                                                                .build();
-
-    private static final Injections.Counter postingsReaderOpenCounter = Injections.newCounter("PostingsReaderOpenCounter")
-                                                                                  .add(newInvokePoint().onClass(PostingsReader.class).onMethod("<init>"))
-                                                                                  .build();
-
-    private static final Injections.Counter postingsReaderCloseCounter = Injections.newCounter("PostingsReaderCloseCounter")
-                                                                                   .add(newInvokePoint().onClass(PostingsReader.class).onMethod("close"))
-                                                                                   .build();
-
-
-    @Before
-    public void setup() throws Throwable
-    {
-        requireNetwork();
-
-        Injections.inject(intersectionFlowCounter, postingsReaderOpenCounter, postingsReaderCloseCounter);
-
-        setLimits(2);
-
-        createTable("CREATE TABLE %s (pk int primary key, v1 text, v2 text, v3 text)");
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v3"));
-
-        for (int i = 0; i < 100; ++i)
-        {
-            execute("INSERT INTO %s(pk,v1,v2,v3) VALUES (?, ?, ?, ?)", i, Integer.toString(i), Integer.toString(i / 20), Integer.toString(i % 10));
-        }
-        flush();
-    }
-
-    @After
-    public void resetCounters()
-    {
-        intersectionFlowCounter.reset();
-        postingsReaderOpenCounter.reset();
-        postingsReaderCloseCounter.reset();
-    }
-
-    @Test
-    public void queryBelowSelectiveLimitUsesDeferredFlows() throws Throwable
-    {
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '0'").size());
-        Assert.assertEquals(0, intersectionFlowCounter.get());
-
-        Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
-    }
-
-    @Test
-    public void queryAtSelectiveLimitUsesDeferredFlows() throws Throwable
-    {
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '20' AND v2 = '1'").size());
-        Assert.assertEquals(1, intersectionFlowCounter.get());
-
-        Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
-    }
-
-    @Test
-    public void queryAboveSelectiveLimitUsesDirectFlows() throws Throwable
-    {
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'").size());
-        Assert.assertEquals(1, intersectionFlowCounter.get());
-
-        Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
-    }
-
-    @Test
-    public void selectivityOfOneWillNotIntersect() throws Throwable
-    {
-        setLimits(1);
-
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'").size());
-        Assert.assertEquals(0, intersectionFlowCounter.get());
-
-        Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
-    }
-
-    @Test
-    public void selectivityLimitOfZeroDisablesSelectivityTest() throws Throwable
-    {
-        setLimits(0);
-
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'").size());
-        Assert.assertEquals(1, intersectionFlowCounter.get());
-
-        Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
-    }
-
-    @Test
-    public void tracingIsCorrectlyReported() throws Throwable
-    {
-        Session session = sessionNet();
-
-        String trace = getSingleTraceStatement(session, "SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'", "Selecting");
-
-        assertEquals("Selecting 2 indexes with cardinalities of 1, 10 out of 3 indexes", trace);
-
-        setLimits(1);
-
-        trace = getSingleTraceStatement(session, "SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'", "Selecting");
-
-        assertEquals("Selecting 1 index with cardinality of 1 out of 3 indexes", trace);
-
-        Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
-    }
-
-    private static void setLimits(final int selectivityLimit)
-    {
-        CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT.setString(Integer.toString(selectivityLimit));
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java b/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
index 8244a3ade8ae..1a5ec2624eea 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
@@ -24,16 +24,19 @@
 
 import com.datastax.driver.core.exceptions.ReadFailureException;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingListRangeIterator;
-import org.apache.cassandra.index.sai.disk.v1.segment.LiteralIndexSegmentTermsReader;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.TermsReader;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
 import org.apache.cassandra.inject.Injection;
 import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
 import org.apache.cassandra.utils.Throwables;
 
 import static org.apache.cassandra.inject.ActionBuilder.newActionBuilder;
 import static org.apache.cassandra.inject.Expression.quote;
 import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assume.assumeTrue;
 
 public class SingleNodeQueryFailureTest extends SAITester
 {
@@ -41,7 +44,7 @@ public class SingleNodeQueryFailureTest extends SAITester
                                                         "compaction = {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
 
     @Before
-    public void setup()
+    public void setup() throws Throwable
     {
         requireNetwork();
         setupTableAndIndexes();
@@ -57,26 +60,46 @@ public void teardown()
     @Test
     public void testFailedRangeIteratorOnSingleIndexQuery() throws Throwable
     {
-        testFailedQuery("range_iterator_single", PostingListRangeIterator.class, "getNextRowId", true);
+        testFailedQuery("range_iterator_single", PostingListKeyRangeIterator.class, "getNextRowId", true);
     }
 
     @Test
     public void testFailedTermsReaderOnSingleIndexQuery() throws Throwable
     {
-        testFailedQuery("terms_reader_single", LiteralIndexSegmentTermsReader.TermQuery.class, "lookupPostingsOffset", true);
+        testFailedQuery("terms_reader_single", TermsReader.TermQuery.class, "lookupTermDictionary", true);
     }
 
     // Multi Index Tests
     @Test
     public void testFailedRangeIteratorOnMultiIndexesQuery() throws Throwable
     {
-        testFailedQuery("range_iterator_multi", PostingListRangeIterator.class, "getNextRowId", false);
+        testFailedQuery("range_iterator", PostingListKeyRangeIterator.class, "getNextRowId", false);
     }
 
     @Test
     public void testFailedTermsReaderOnMultiIndexesQuery() throws Throwable
     {
-        testFailedQuery("terms_reader_multi", LiteralIndexSegmentTermsReader.TermQuery.class, "lookupPostingsOffset", false);
+        testFailedQuery("terms_reader", TermsReader.TermQuery.class, "lookupTermDictionary", false);
+    }
+
+    @Test
+    public void testFailedBkdReaderOnMultiIndexesQuery() throws Throwable
+    {
+        testFailedQuery("bkd_reader", PostingsReader.class, "<init>", false);
+    }
+
+    @Test
+    public void testFailedKeyFetcherOnMultiIndexesQuery() throws Throwable
+    {
+        assumeTrue(Version.latest() == Version.AA);
+        testFailedQuery("key_fetcher", IKeyFetcher.class, "apply", false);
+    }
+
+    @Test
+    public void testFailedKeyReaderOnMultiIndexesQuery() throws Throwable
+    {
+        assumeTrue(Version.latest() == Version.AA);
+        testFailedQuery("key_reader", IKeyFetcher.class, "createReader", false);
     }
 
     private void setupTableAndIndexes()
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/TypeUtilTest.java b/test/unit/org/apache/cassandra/index/sai/disk/TypeUtilTest.java
new file mode 100644
index 000000000000..f8e1a84d1504
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/TypeUtilTest.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FastByteOperations;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+public class TypeUtilTest extends SaiRandomizedTest
+{
+    private final Version version;
+    @ParametersFactory()
+    public static Collection<Object[]> data()
+    {
+        // Required because it configures SEGMENT_BUILD_MEMORY_LIMIT, which is needed for Version.AA
+        if (DatabaseDescriptor.getRawConfig() == null)
+            DatabaseDescriptor.setConfig(DatabaseDescriptor.loadConfig());
+        return Version.ALL.stream().map(v -> new Object[]{ v}).collect(Collectors.toList());
+    }
+
+    public TypeUtilTest(Version version)
+    {
+        this.version = version;
+    }
+
+    @Test
+    public void testSimpleType()
+    {
+        for (CQL3Type cql3Type : StorageAttachedIndex.SUPPORTED_TYPES)
+        {
+            AbstractType<?> type = cql3Type.getType();
+            AbstractType<?> reversedType = ReversedType.getInstance(type);
+
+            boolean isUTF8OrAscii = cql3Type == CQL3Type.Native.ASCII || cql3Type == CQL3Type.Native.TEXT || cql3Type == CQL3Type.Native.VARCHAR;
+            boolean isLiteral = cql3Type == CQL3Type.Native.ASCII || cql3Type == CQL3Type.Native.TEXT || cql3Type == CQL3Type.Native.VARCHAR || cql3Type == CQL3Type.Native.BOOLEAN;
+            assertEquals(isLiteral, TypeUtil.isLiteral(type));
+            assertEquals(TypeUtil.isLiteral(type), TypeUtil.isLiteral(reversedType));
+            assertEquals(isUTF8OrAscii, TypeUtil.isUTF8OrAscii(type));
+            assertEquals(TypeUtil.isUTF8OrAscii(type), TypeUtil.isUTF8OrAscii(reversedType));
+            assertEquals(TypeUtil.isIn(type, AbstractAnalyzer.ANALYZABLE_TYPES),
+                         TypeUtil.isIn(reversedType, AbstractAnalyzer.ANALYZABLE_TYPES));
+        }
+    }
+
+    @Test
+    public void testMapType()
+    {
+        for(CQL3Type keyCql3Type : StorageAttachedIndex.SUPPORTED_TYPES)
+        {
+            AbstractType<?> keyType = keyCql3Type.getType();
+
+            testCollectionType((valueType, multiCell) -> MapType.getInstance(keyType, valueType, multiCell),
+                               (valueType, nonFrozenMap) -> {
+                assertEquals(keyType, cellValueType(nonFrozenMap, IndexTarget.Type.KEYS));
+                assertEquals(valueType, cellValueType(nonFrozenMap, IndexTarget.Type.VALUES));
+                AbstractType<?> entryType = cellValueType(nonFrozenMap, IndexTarget.Type.KEYS_AND_VALUES);
+                assertEquals(CompositeType.getInstance(keyType, valueType), entryType);
+                assertTrue(TypeUtil.isLiteral(entryType));
+            });
+        }
+    }
+
+    @Test
+    public void testSetType()
+    {
+        testCollectionType(SetType::getInstance, (a, b) -> {});
+    }
+
+    @Test
+    public void testListType()
+    {
+        testCollectionType(ListType::getInstance, (a, b) -> {});
+    }
+
+    @Test
+    public void testTuple()
+    {
+        for(CQL3Type elementType : StorageAttachedIndex.SUPPORTED_TYPES)
+        {
+            TupleType type = new TupleType(Arrays.asList(elementType.getType(), elementType.getType()), true);
+            assertFalse(TypeUtil.isFrozenCollection(type));
+            assertFalse(TypeUtil.isFrozen(type));
+            assertFalse(TypeUtil.isLiteral(type));
+
+            type = new TupleType(Arrays.asList(elementType.getType(), elementType.getType()), false);
+            assertFalse(TypeUtil.isFrozenCollection(type));
+            assertTrue(TypeUtil.isFrozen(type));
+            assertTrue(TypeUtil.isLiteral(type));
+        }
+    }
+
+    @Test
+    public void testUDT()
+    {
+        for(CQL3Type elementType : StorageAttachedIndex.SUPPORTED_TYPES)
+        {
+            UserType type = new UserType("ks", ByteBufferUtil.bytes("myType"),
+                                         Arrays.asList(FieldIdentifier.forQuoted("f1"), FieldIdentifier.forQuoted("f2")),
+                                         Arrays.asList(elementType.getType(), elementType.getType()),
+                                         true);
+
+            assertFalse(TypeUtil.isFrozenCollection(type));
+            assertFalse(TypeUtil.isFrozen(type));
+            assertFalse(TypeUtil.isLiteral(type));
+
+            type = new UserType("ks", ByteBufferUtil.bytes("myType"),
+                                Arrays.asList(FieldIdentifier.forQuoted("f1"), FieldIdentifier.forQuoted("f2")),
+                                Arrays.asList(elementType.getType(), elementType.getType()),
+                                false);
+            assertFalse(TypeUtil.isFrozenCollection(type));
+            assertTrue(TypeUtil.isFrozen(type));
+            assertTrue(TypeUtil.isLiteral(type));
+        }
+    }
+
+    private static void testCollectionType(BiFunction<AbstractType<?>, Boolean, AbstractType<?>> init,
+                                           BiConsumer<AbstractType<?>, AbstractType<?>> nonFrozenCollectionTester)
+    {
+        for(CQL3Type elementType : StorageAttachedIndex.SUPPORTED_TYPES)
+        {
+            AbstractType<?> frozenCollection = init.apply(elementType.getType(), false);
+            AbstractType<?> reversedFrozenCollection = ReversedType.getInstance(frozenCollection);
+
+            AbstractType<?> type = TypeUtil.cellValueType(column(frozenCollection), IndexTarget.Type.FULL);
+            assertTrue(TypeUtil.isFrozenCollection(type));
+            assertTrue(TypeUtil.isLiteral(type));
+            assertFalse(type.isReversed());
+
+            type = TypeUtil.cellValueType(column(reversedFrozenCollection), IndexTarget.Type.FULL);
+            assertTrue(TypeUtil.isFrozenCollection(type));
+            assertTrue(TypeUtil.isLiteral(type));
+            assertTrue(type.isReversed());
+
+            AbstractType<?> nonFrozenCollection = init.apply(elementType.getType(), true);
+            assertEquals(elementType.getType(), cellValueType(nonFrozenCollection, IndexTarget.Type.VALUES));
+            nonFrozenCollectionTester.accept(elementType.getType(), nonFrozenCollection);
+        }
+    }
+
+    private static AbstractType<?> cellValueType(AbstractType<?> type, IndexTarget.Type indexType)
+    {
+        return TypeUtil.cellValueType(column(type), indexType);
+    }
+
+    private static ColumnMetadata column(AbstractType<?> type)
+    {
+        return ColumnMetadata.regularColumn("ks", "cf", "col", type);
+    }
+
+    @Test
+    public void shouldCompareByteBuffers()
+    {
+        final ByteBuffer a = Int32Type.instance.decompose(1);
+        final ByteBuffer b = Int32Type.instance.decompose(2);
+
+        assertEquals(a, TypeUtil.min(a, b, Int32Type.instance, version));
+        assertEquals(a, TypeUtil.min(b, a, Int32Type.instance, version));
+        assertEquals(a, TypeUtil.min(a, a, Int32Type.instance, version));
+        assertEquals(b, TypeUtil.min(b, b, Int32Type.instance, version));
+        assertEquals(b, TypeUtil.min(null, b, Int32Type.instance, version));
+        assertEquals(a, TypeUtil.min(a, null, Int32Type.instance, version));
+
+        assertEquals(b, TypeUtil.max(b, a, Int32Type.instance, version));
+        assertEquals(b, TypeUtil.max(a, b, Int32Type.instance, version));
+        assertEquals(a, TypeUtil.max(a, a, Int32Type.instance, version));
+        assertEquals(b, TypeUtil.max(b, b, Int32Type.instance, version));
+        assertEquals(b, TypeUtil.max(null, b, Int32Type.instance, version));
+        assertEquals(a, TypeUtil.max(a, null, Int32Type.instance, version));
+    }
+
+    @Test
+    public void testBigIntegerEncoding()
+    {
+        Random rng = new Random(-9078270684023566599L);
+
+        BigInteger[] data = new BigInteger[10000];
+        for (int i = 0; i < data.length; i++)
+        {
+            BigInteger randomNumber = new BigInteger(rng.nextInt(1000), rng);
+            if (rng.nextBoolean())
+                randomNumber = randomNumber.negate();
+
+            data[i] = randomNumber;
+        }
+
+        Arrays.sort(data, BigInteger::compareTo);
+
+        for (int i = 1; i < data.length; i++)
+        {
+            BigInteger i0 = data[i - 1];
+            BigInteger i1 = data[i];
+            assertTrue("#" + i, i0.compareTo(i1) <= 0);
+
+            ByteBuffer b0 = TypeUtil.asIndexBytes(ByteBuffer.wrap(i0.toByteArray()), IntegerType.instance);
+            ByteBuffer b1 = TypeUtil.asIndexBytes(ByteBuffer.wrap(i1.toByteArray()), IntegerType.instance);
+            assertTrue("#" + i, TypeUtil.compare(b0, b1, IntegerType.instance, version) <= 0);
+        }
+    }
+
+    @Test
+    public void testMapEntryEncoding()
+    {
+        Random rng = new Random(-9078270684023566599L);
+        CompositeType type = CompositeType.getInstance(UTF8Type.instance, Int32Type.instance);
+
+        // simulate: index memtable insertion
+        ByteBuffer[] data = new ByteBuffer[10000];
+        byte[] temp = new byte[100];
+        for (int i = 0; i < data.length; i++)
+        {
+            rng.nextBytes(temp);
+            String v1 = new String(temp);
+            int v2 = rng.nextInt();
+
+            data[i] = type.decompose(v1, v2);
+        }
+
+        // Starting with DB, we sorted using the abstract type.
+        if (version.onOrAfter(Version.DB))
+            Arrays.sort(data, type);
+        else
+            Arrays.sort(data, FastByteOperations::compareUnsigned);
+
+        for (int i = 1; i < data.length; i++)
+        {
+            // simulate: index memtable flush
+            ByteBuffer b0 = data[i - 1];
+            ByteBuffer b1 = data[i];
+            assertTrue("#" + i, TypeUtil.compare(b0, b1, type, version) <= 0);
+
+            // Before version DB, we didn't write terms in their ByteComparable order, so we skip
+            // that check here.
+            if (!version.onOrAfter(Version.DB))
+                continue;
+
+            // simulate: saving into on-disk trie
+            ByteComparable t0 = v -> type.asComparableBytes(b0, v);
+            ByteComparable t1 = v -> type.asComparableBytes(b1, v);
+            assertTrue("#" + i, ByteComparable.compare(t0, t1, TypeUtil.BYTE_COMPARABLE_VERSION) <= 0);
+        }
+    }
+
+    @Test
+    public void testEncodeBigInteger()
+    {
+        testBigInteger(BigInteger.valueOf(0));
+        testBigInteger(BigInteger.valueOf(1));
+        testBigInteger(BigInteger.valueOf(-1));
+        testBigInteger(BigInteger.valueOf(2));
+        testBigInteger(BigInteger.valueOf(-2));
+        testBigInteger(BigInteger.valueOf(17));
+        testBigInteger(BigInteger.valueOf(-17));
+        testBigInteger(BigInteger.valueOf(123456789));
+        testBigInteger(BigInteger.valueOf(-123456789));
+        testBigInteger(BigInteger.valueOf(1000000000000L));
+        testBigInteger(BigInteger.valueOf(-1000000000000L));
+        testBigInteger(BigInteger.valueOf(13).pow(1000));
+        testBigInteger(BigInteger.valueOf(13).pow(1000).negate());
+
+        testBigInteger(new BigInteger("123456789012345678901234567890123456789012345678901234567890"));
+        testBigInteger(new BigInteger("-123456789012345678901234567890123456789012345678901234567890"));
+    }
+
+    private static void testBigInteger(BigInteger value)
+    {
+        var raw = IntegerType.instance.decompose(value);
+        var raw2 = TypeUtil.decodeBigInteger(TypeUtil.encodeBigInteger(raw));
+        var value2 = IntegerType.instance.compose(raw2);
+        // this cannot be exact comparison, because `encode` truncates value and loses some precision
+        assertEquals(value.doubleValue(), value2.doubleValue(), value.doubleValue() * 1.0e-15);
+    }
+
+    @Test
+    public void testEncodeDecimal()
+    {
+        testDecimal(BigDecimal.valueOf(0));
+        testDecimal(BigDecimal.valueOf(1));
+        testDecimal(BigDecimal.valueOf(-1));
+        testDecimal(BigDecimal.valueOf(12345678.9));
+        testDecimal(BigDecimal.valueOf(-12345678.9));
+        testDecimal(BigDecimal.valueOf(0.000005));
+        testDecimal(BigDecimal.valueOf(-0.000005));
+        testDecimal(BigDecimal.valueOf(0.1111111111111111));
+        testDecimal(BigDecimal.valueOf(-0.1111111111111111));
+
+        // test very large and very small values
+        testDecimal(BigDecimal.valueOf(123456789, -10000));
+        testDecimal(BigDecimal.valueOf(-123456789, -10000));
+        testDecimal(BigDecimal.valueOf(123456789, 10000));
+        testDecimal(BigDecimal.valueOf(-123456789, 10000));
+
+        // test truncated values
+        testDecimal(new BigDecimal("1234567890.1234567890123456789012345678901234567890"));
+        testDecimal(new BigDecimal("-1234567890.1234567890123456789012345678901234567890"));
+    }
+
+    private static void testDecimal(BigDecimal value)
+    {
+        var raw = DecimalType.instance.decompose(value);
+        var raw2 = TypeUtil.decodeDecimal(TypeUtil.encodeDecimal(raw));
+        var value2 = DecimalType.instance.compose(raw2);
+        // this cannot be exact comparison, because `encode` truncates value and loses some precision
+        assertEquals(value.doubleValue(), value2.doubleValue(), value.doubleValue() * 1.0e-15);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java b/test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java
index bacca543d59f..67a90f468e67 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java
@@ -18,9 +18,11 @@
 
 package org.apache.cassandra.index.sai.disk.format;
 
-import java.net.URI;
+import java.io.IOException;
 import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
 
 import com.google.common.io.Files;
 import org.junit.After;
@@ -30,74 +32,294 @@
 import org.junit.rules.TemporaryFolder;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.mockito.Mockito;
 
+import static org.apache.cassandra.index.sai.SAIUtil.setLatestVersion;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+/**
+ * At the time of this writing, the test of this class mostly test the "fallback-scan-disk" mode of component discovery
+ * from IndexDescriptor, because they don't create a proper sstable with a TOC, they just "touch" a bunch of the
+ * component files. The "normal" discovery path that uses the TOC is however effectively tested by pretty much
+ * every other SAI test, so it is a reasonable way to test that fallback. Besides, the test also test the parsing of
+ * of the component filename at various versions, and that code is common to both paths.
+ */
 public class IndexDescriptorTest
 {
-    private final TemporaryFolder temporaryFolder = new TemporaryFolder();
+    private TemporaryFolder temporaryFolder = new TemporaryFolder();
     private Descriptor descriptor;
+    private Version latest;
 
     @BeforeClass
     public static void initialise()
     {
-        DatabaseDescriptor.toolInitialization();
+        DatabaseDescriptor.daemonInitialization();
     }
 
     @Before
     public void setup() throws Throwable
     {
         temporaryFolder.create();
-        descriptor = Descriptor.fromFile(new File(temporaryFolder.newFolder().getAbsolutePath() + "/nb-1-big-Data.db"));
+        descriptor = Descriptor.fromFilename(temporaryFolder.newFolder().getAbsolutePath() + "/ca-1-bti-Data.db");
+        latest = Version.latest();
     }
 
     @After
     public void teardown() throws Throwable
     {
+        setLatestVersion(latest);
         temporaryFolder.delete();
     }
 
+    private IndexDescriptor loadDescriptor(IndexContext... contexts)
+    {
+        return loadDescriptor(descriptor, contexts);
+    }
+
+    static IndexDescriptor loadDescriptor(Descriptor sstableDescriptor, IndexContext... contexts)
+    {
+        IndexDescriptor indexDescriptor = IndexDescriptor.empty(sstableDescriptor);
+        SSTableReader sstable = Mockito.mock(SSTableReader.class);
+        Mockito.when(sstable.getDescriptor()).thenReturn(sstableDescriptor);
+        indexDescriptor.reload(sstable, new HashSet<>(Arrays.asList(contexts)));
+        return indexDescriptor;
+    }
+
     @Test
     public void versionAAPerSSTableComponentIsParsedCorrectly() throws Throwable
     {
-        createFileOnDisk("-SAI+aa+GroupComplete.db");
+        setLatestVersion(Version.AA);
+
+        // As mentioned in the class javadoc, we rely on the no-TOC fallback path and that only kick in if there is a
+        // data file. Otherwise, it assumes the SSTable simply does not exist at all.
+        createFakeDataFile(descriptor);
+        createFakePerSSTableComponents(descriptor, Version.AA, 0);
 
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor, Murmur3Partitioner.instance, SAITester.EMPTY_COMPARATOR);
+        IndexDescriptor indexDescriptor = loadDescriptor();
 
-        assertEquals(Version.AA, indexDescriptor.version);
-        assertTrue(indexDescriptor.hasComponent(IndexComponent.GROUP_COMPLETION_MARKER));
+        assertEquals(Version.AA, indexDescriptor.perSSTableComponents().version());
+        assertTrue(indexDescriptor.perSSTableComponents().has(IndexComponentType.GROUP_COMPLETION_MARKER));
     }
 
     @Test
     public void versionAAPerIndexComponentIsParsedCorrectly() throws Throwable
     {
-        createFileOnDisk("-SAI+aa+test_index+ColumnComplete.db");
+        setLatestVersion(Version.AA);
+
+        IndexContext indexContext = SAITester.createIndexContext("test_index", UTF8Type.instance);
 
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor, Murmur3Partitioner.instance, SAITester.EMPTY_COMPARATOR);
-        IndexIdentifier indexIdentifier = SAITester.createIndexIdentifier("test", "test", "test_index");
+        createFakeDataFile(descriptor);
+        createFakePerSSTableComponents(descriptor, Version.AA, 0);
+        createFakePerIndexComponents(descriptor, indexContext, Version.AA, 0);
 
-        assertEquals(Version.AA, indexDescriptor.version);
-        assertTrue(indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier));
+        IndexDescriptor indexDescriptor = loadDescriptor(indexContext);
+
+        assertEquals(Version.AA, indexDescriptor.perSSTableComponents().version());
+        assertTrue(indexDescriptor.perIndexComponents(indexContext).has(IndexComponentType.COLUMN_COMPLETION_MARKER));
     }
 
-    private void createFileOnDisk(String filename) throws Throwable
+    @Test
+    public void versionBAPerSSTableComponentIsParsedCorrectly() throws Throwable
     {
-        Path path;
-        try
+        setLatestVersion(Version.BA);
+
+        createFakeDataFile(descriptor);
+        createFakePerSSTableComponents(descriptor, Version.BA, 0);
+
+        IndexDescriptor indexDescriptor = loadDescriptor();
+
+        assertEquals(Version.BA, indexDescriptor.perSSTableComponents().version());
+        assertTrue(indexDescriptor.perSSTableComponents().has(IndexComponentType.GROUP_COMPLETION_MARKER));
+    }
+
+    @Test
+    public void versionBAPerIndexComponentIsParsedCorrectly() throws Throwable
+    {
+        setLatestVersion(Version.BA);
+
+        IndexContext indexContext = SAITester.createIndexContext("test_index", UTF8Type.instance);
+
+        createFakeDataFile(descriptor);
+        createFakePerIndexComponents(descriptor, indexContext, Version.BA, 0);
+
+        IndexDescriptor indexDescriptor = loadDescriptor(indexContext);
+
+        assertEquals(Version.BA, indexDescriptor.perIndexComponents(indexContext).version());
+        assertTrue(indexDescriptor.perIndexComponents(indexContext).has(IndexComponentType.COLUMN_COMPLETION_MARKER));
+    }
+
+    @Test
+    public void allVersionAAPerSSTableComponentsAreLoaded() throws Throwable
+    {
+        setLatestVersion(Version.AA);
+
+        createFakeDataFile(descriptor);
+        createFakePerSSTableComponents(descriptor, Version.AA, 0);
+
+        IndexDescriptor result = loadDescriptor();
+
+        assertTrue(result.perSSTableComponents().has(IndexComponentType.GROUP_COMPLETION_MARKER));
+        assertTrue(result.perSSTableComponents().has(IndexComponentType.GROUP_META));
+        assertTrue(result.perSSTableComponents().has(IndexComponentType.TOKEN_VALUES));
+        assertTrue(result.perSSTableComponents().has(IndexComponentType.OFFSETS_VALUES));
+    }
+
+    @Test
+    public void allVersionAAPerIndexLiteralComponentsAreLoaded() throws Throwable
+    {
+        setLatestVersion(Version.AA);
+
+        IndexContext indexContext = SAITester.createIndexContext("test_index", UTF8Type.instance);
+
+        createFakeDataFile(descriptor);
+        createFakePerSSTableComponents(descriptor, Version.AA, 0);
+        createFakePerIndexComponents(descriptor, indexContext, Version.AA, 0);
+
+        IndexDescriptor indexDescriptor = loadDescriptor(indexContext);
+
+        IndexComponents.ForRead components = indexDescriptor.perIndexComponents(indexContext);
+        assertTrue(components.has(IndexComponentType.COLUMN_COMPLETION_MARKER));
+        assertTrue(components.has(IndexComponentType.META));
+        assertTrue(components.has(IndexComponentType.TERMS_DATA));
+        assertTrue(components.has(IndexComponentType.POSTING_LISTS));
+    }
+
+    @Test
+    public void allVersionAAPerIndexNumericComponentsAreLoaded() throws Throwable
+    {
+        setLatestVersion(Version.AA);
+
+        IndexContext indexContext = SAITester.createIndexContext("test_index", Int32Type.instance);
+
+        createFakeDataFile(descriptor);
+        createFakePerSSTableComponents(descriptor, Version.AA, 0);
+        createFakePerIndexComponents(descriptor, indexContext, Version.AA, 0);
+
+        IndexDescriptor indexDescriptor = loadDescriptor(indexContext);
+
+        IndexComponents.ForRead components = indexDescriptor.perIndexComponents(indexContext);
+        assertTrue(components.has(IndexComponentType.COLUMN_COMPLETION_MARKER));
+        assertTrue(components.has(IndexComponentType.META));
+        assertTrue(components.has(IndexComponentType.KD_TREE));
+        assertTrue(components.has(IndexComponentType.KD_TREE_POSTING_LISTS));
+    }
+
+    // CNDB-13582
+    @Test
+    public void componentsAreLoadedAfterUpgradeDespiteBrokenTOC() throws Throwable
+    {
+        setLatestVersion(Version.AA);
+
+        // Force old version of sstables to simulate upgrading from DSE
+        Descriptor descriptor = Descriptor.fromFilename(temporaryFolder.newFolder().getAbsolutePath() + "/bb-2-bti-Data.db");
+
+        IndexContext indexContext = SAITester.createIndexContext("test_index", Int32Type.instance);
+
+        createFakeDataFile(descriptor);
+        createFakeTOCFile(descriptor);
+        createFakePerSSTableComponents(descriptor, Version.AA, 0);
+        createFakePerIndexComponents(descriptor, indexContext, Version.AA, 0);
+
+        IndexDescriptor indexDescriptor = loadDescriptor(descriptor, indexContext);
+
+        IndexComponents.ForRead components = indexDescriptor.perIndexComponents(indexContext);
+        assertTrue(components.has(IndexComponentType.COLUMN_COMPLETION_MARKER));
+        assertTrue(components.has(IndexComponentType.META));
+        assertTrue(components.has(IndexComponentType.KD_TREE));
+        assertTrue(components.has(IndexComponentType.KD_TREE_POSTING_LISTS));
+    }
+
+
+    @Test
+    public void testReload() throws Throwable
+    {
+        setLatestVersion(latest);
+
+        // We create the descriptor first, with no files, so it should initially be empty.
+        IndexContext indexContext = SAITester.createIndexContext("test_index", Int32Type.instance);
+        IndexDescriptor indexDescriptor = loadDescriptor(indexContext);
+
+        assertFalse(indexDescriptor.perSSTableComponents().isComplete());
+        assertFalse(indexDescriptor.perIndexComponents(indexContext).isComplete());
+
+        // We then create the proper files and call reload
+        createFakeDataFile(descriptor);
+        createFakePerSSTableComponents(descriptor, latest, 0);
+        createFakePerIndexComponents(descriptor, indexContext, latest, 0);
+
+        SSTableReader sstable = Mockito.mock(SSTableReader.class);
+        Mockito.when(sstable.getDescriptor()).thenReturn(descriptor);
+        indexDescriptor.reload(sstable, Set.of(indexContext));
+
+        // Both the perSSTableComponents and perIndexComponents should now be complete and the components should be present
+
+        assertTrue(indexDescriptor.perSSTableComponents().isComplete());
+
+        IndexComponents.ForRead components = indexDescriptor.perIndexComponents(indexContext);
+        assertTrue(components.isComplete());
+        assertTrue(components.has(IndexComponentType.META));
+        assertTrue(components.has(IndexComponentType.KD_TREE));
+        assertTrue(components.has(IndexComponentType.KD_TREE_POSTING_LISTS));
+    }
+
+    private static void createEmptyFileOnDisk(Descriptor descriptor, String componentStr) throws IOException
+    {
+        Files.touch(new File(PathUtils.getPath(descriptor.baseFileUri() + '-' + componentStr)).toJavaIOFile());
+    }
+
+    private static void createileOnDisk(Descriptor descriptor, String componentStr, int size) throws IOException
+    {
+        if (size == 0)
         {
-            path = Paths.get(URI.create(descriptor.baseFile() + filename));
+            createEmptyFileOnDisk(descriptor, componentStr);
         }
-        catch (IllegalArgumentException ex)
+        else
         {
-            path = Paths.get(descriptor.baseFile() + filename);
+            Path filePath = PathUtils.getPath(descriptor.baseFileUri() + '-' + componentStr);
+            Files.write(new byte[size], filePath.toFile());
         }
+    }
+
+    static void createFakeDataFile(Descriptor descriptor) throws IOException
+    {
+        createEmptyFileOnDisk(descriptor, SSTableFormat.Components.DATA.name());
+    }
+
+    static void createFakeTOCFile(Descriptor descriptor) throws IOException
+    {
+        createEmptyFileOnDisk(descriptor, SSTableFormat.Components.TOC.name());
+    }
+
+    static void createFakePerSSTableComponents(Descriptor descriptor, Version version, int generation) throws IOException
+    {
+        createFakePerSSTableComponents(descriptor, version, generation, 0);
+    }
 
-        Files.touch(new File(path).toJavaIOFile());
+    static void createFakePerSSTableComponents(Descriptor descriptor, Version version, int generation, int sizeInBytes) throws IOException
+    {
+        for (IndexComponentType type : version.onDiskFormat().perSSTableComponentTypes())
+            createileOnDisk(descriptor, version.fileNameFormatter().format(type, (String)null, generation), sizeInBytes);
+    }
+
+    static void createFakePerIndexComponents(Descriptor descriptor, IndexContext context, Version version, int generation) throws IOException
+    {
+        createFakePerIndexComponents(descriptor, context, version, generation, 0);
+    }
+
+    static void createFakePerIndexComponents(Descriptor descriptor, IndexContext context, Version version, int generation, int sizeInBytes) throws IOException
+    {
+        for (IndexComponentType type : version.onDiskFormat().perIndexComponentTypes(context))
+            createileOnDisk(descriptor, version.fileNameFormatter().format(type, context, generation), sizeInBytes);
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/format/SSTableIndexComponentsStateTest.java b/test/unit/org/apache/cassandra/index/sai/disk/format/SSTableIndexComponentsStateTest.java
new file mode 100644
index 000000000000..81fae28c68ce
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/format/SSTableIndexComponentsStateTest.java
@@ -0,0 +1,427 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.SSTableIndexComponentsState.UnapplicableDiffException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.mockito.Mockito;
+
+import static org.apache.cassandra.index.sai.disk.format.IndexDescriptorTest.createFakeDataFile;
+import static org.apache.cassandra.index.sai.disk.format.IndexDescriptorTest.createFakePerIndexComponents;
+import static org.apache.cassandra.index.sai.disk.format.IndexDescriptorTest.createFakePerSSTableComponents;
+import static org.apache.cassandra.index.sai.disk.format.IndexDescriptorTest.loadDescriptor;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class SSTableIndexComponentsStateTest
+{
+    @Test
+    public void indexWasUpdatedTest()
+    {
+        // Note: the `indexWasUpdated` method is not used in C* at the time of this writing, but it is in CNDB. Those
+        // tests both make sure it works as expected, but also avoid the method be marked "unused" by code editors.
+
+        var base = SSTableIndexComponentsState.builder()
+                                               .addPerSSTable(Version.DB, 0, 1)
+                                               .addPerIndex("index1", Version.DB, 0, 1)
+                                               .addPerIndex("index2", Version.DB, 0, 1);
+
+        // Additions are changes.
+        assertTrue(base.copy().build().indexWasUpdated(SSTableIndexComponentsState.EMPTY, "index1"));
+        assertTrue(base.copy().build().indexWasUpdated(SSTableIndexComponentsState.EMPTY, "index2"));
+
+        // Modifying the per-sstable component is an update of all indexes.
+        // version change:
+        assertIndexUpdated(base, base.copy().addPerSSTable(Version.EB, 0, 1), "index1");
+        assertIndexUpdated(base, base.copy().addPerSSTable(Version.EB, 0, 1), "index2");
+        // generation change:
+        assertIndexUpdated(base, base.copy().addPerSSTable(Version.DB, 1, 1), "index1");
+        assertIndexUpdated(base, base.copy().addPerSSTable(Version.DB, 1, 1), "index2");
+
+        // Modifying a per-index component only count as an update of that index.
+        // version change:
+        assertIndexUpdated(base, base.copy().addPerIndex("index1", Version.EB, 0, 1), "index1");
+        assertIndexNotUpdated(base, base.copy().addPerIndex("index1", Version.EB, 0, 1), "index2");
+        // generation change:
+        assertIndexUpdated(base, base.copy().addPerIndex("index1", Version.DB, 1, 1), "index1");
+        assertIndexNotUpdated(base, base.copy().addPerIndex("index1", Version.DB, 1, 1), "index2");
+
+        // Same state means no change
+        assertIndexNotUpdated(base, base, "index1");
+        assertIndexNotUpdated(base, base, "index2");
+    }
+
+    private void assertIndexUpdated(SSTableIndexComponentsState.Builder before, SSTableIndexComponentsState.Builder after, String indexName)
+    {
+        assertTrue(after.copy().build().indexWasUpdated(before.copy().build(), indexName));
+    }
+
+    private void assertIndexNotUpdated(SSTableIndexComponentsState.Builder before, SSTableIndexComponentsState.Builder after, String indexName)
+    {
+        assertFalse(after.copy().build().indexWasUpdated(before.copy().build(), indexName));
+    }
+
+    @Test
+    public void includedIndexTest()
+    {
+        assertEquals(SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.DB, 0, 1)
+                                                .addPerIndex("index1", Version.DB, 0, 1)
+                                                .addPerIndex("index2", Version.DB, 0, 1)
+                     .build()
+                     .includedIndexes(),
+                     Set.of("index1", "index2"));
+    }
+
+    @Test
+    public void diffToStringTest()
+    {
+        var before = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 0, 10)
+                                                .addPerIndex("index1", Version.DB, 0, 1)
+                                                .build();
+
+        var after = before.unbuild()
+                          .removePerSSTable()
+                          .addPerIndex("index2", Version.EB, 1, 4)
+                          .build();
+
+        var diff = after.diff(before);
+        // The details of that string are not that important, but just making sure it looks reasonable.
+        assertEquals("{<shared>: eb@0 (10MB), index1: db@0 (1MB)} -> {index1: db@0 (1MB), index2: eb@1 (4MB)} (-<shared> +index2)", diff.toString());
+    }
+
+    @Test
+    public void diffNoConcurrentModificationTest()
+    {
+        var before = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 0, 1)
+                                                .addPerIndex("index1", Version.DB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 0, 1)
+                                                .addPerIndex("index3", Version.EB, 0, 1)
+                                                .addPerIndex("index4", Version.EB, 0, 1)
+                                                .build();
+
+        // Rebuilds the per-sstable and "index2" to a new generation, and rebuild "index1" but bumping to newer index
+        // version. "index4" is removed, but "index3" is unmodified.
+        var after = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 1, 1)
+                                                .addPerIndex("index1", Version.EB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 1, 1)
+                                                .addPerIndex("index3", Version.EB, 0, 1)
+                                               .build();
+
+
+        SSTableIndexComponentsState.Diff diff = after.diff(before);
+        assertFalse(diff.isEmpty());
+        assertTrue(diff.perSSTableUpdated);
+        assertEquals(2, diff.perIndexesUpdated.size());
+        assertTrue(diff.perIndexesUpdated.contains("index1"));
+        assertTrue(diff.perIndexesUpdated.contains("index2"));
+        assertEquals(1, diff.perIndexesRemoved.size());
+        assertTrue(diff.perIndexesRemoved.contains("index4"));
+
+        assertTrue(diff.createsUnusedComponents());
+
+        assertEquals(after, before.tryApplyDiff(diff));
+
+    }
+
+    @Test
+    public void diffConcurrentModificationTest()
+    {
+        var before = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 0, 1)
+                                                .addPerIndex("index1", Version.EB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 0, 1)
+                                                .addPerIndex("index3", Version.EB, 0, 1)
+                                                .addPerIndex("index4", Version.EB, 0, 1)
+                                                .build();
+
+        // Rebuilds "index1" and "index3" to a new generation. The rest is unmodified.
+        var after = SSTableIndexComponentsState.builder()
+                                               .addPerSSTable(Version.EB, 1, 1)
+                                               .addPerIndex("index1", Version.EB, 1, 1)
+                                               .addPerIndex("index2", Version.EB, 0, 1)
+                                               .addPerIndex("index3", Version.EB, 1, 1)
+                                               .addPerIndex("index4", Version.EB, 0, 1)
+                                               .build();
+
+        SSTableIndexComponentsState.Diff diff = after.diff(before);
+        assertFalse(diff.isEmpty());
+        assertTrue(diff.perSSTableUpdated);
+        assertEquals(2, diff.perIndexesUpdated.size());
+        assertTrue(diff.perIndexesUpdated.contains("index1"));
+        assertTrue(diff.perIndexesUpdated.contains("index3"));
+        assertEquals(0, diff.perIndexesRemoved.size());
+
+        assertTrue(diff.createsUnusedComponents());
+
+        // The current state has modification compared to `current`, namely: "index2" has been removed and
+        // "index4" has been updated.
+        var current = SSTableIndexComponentsState.builder()
+                                                 .addPerSSTable(Version.EB, 0, 1)
+                                                 .addPerIndex("index1", Version.EB, 0, 1)
+                                                 .addPerIndex("index3", Version.EB, 0, 1)
+                                                 .addPerIndex("index4", Version.EB, 1, 1)
+                                                 .build();
+
+        // The diff should apply and be the combination of all changes
+        var updated = current.tryApplyDiff(diff);
+
+        var expected = SSTableIndexComponentsState.builder()
+                                                 .addPerSSTable(Version.EB, 1, 1)
+                                                 .addPerIndex("index1", Version.EB, 1, 1)
+                                                 .addPerIndex("index3", Version.EB, 1, 1)
+                                                 .addPerIndex("index4", Version.EB, 1, 1)
+                                                 .build();
+
+        assertEquals(expected, updated);
+    }
+
+    @Test
+    public void diffNoModificationTest()
+    {
+        var state = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 0, 1)
+                                                .addPerIndex("index1", Version.EB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 0, 1)
+                                                .addPerIndex("index3", Version.EB, 0, 1)
+                                                .addPerIndex("index4", Version.EB, 0, 1)
+                                                .build();
+
+        SSTableIndexComponentsState.Diff diff = state.diff(state);
+        assertTrue(diff.isEmpty());
+        assertFalse(diff.createsUnusedComponents());
+
+        assertEquals(state, state.tryApplyDiff(diff));
+
+    }
+
+    @Test
+    public void diffIncompatibleModificationTest()
+    {
+        var before = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 0, 1)
+                                                .addPerIndex("index1", Version.EB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 0, 1)
+                                                .build();
+
+        // Updates per-sstable and "index1".
+        var after = SSTableIndexComponentsState.builder()
+                                               .addPerSSTable(Version.EB, 1, 1)
+                                               .addPerIndex("index1", Version.EB, 1, 1)
+                                               .addPerIndex("index2", Version.EB, 0, 1)
+                                               .build();
+
+        SSTableIndexComponentsState.Diff diff = after.diff(before);
+
+        // Concurrent modification that modifides "index2" but also "index1"
+        var current = SSTableIndexComponentsState.builder()
+                                               .addPerSSTable(Version.EB, 0, 1)
+                                               .addPerIndex("index1", Version.EB, 1, 1)
+                                               .addPerIndex("index2", Version.EB, 1, 1)
+                                               .build();
+
+        assertThrows(UnapplicableDiffException.class, () -> current.tryApplyDiff(diff));
+    }
+
+    @Test
+    public void diffWithConcurrentDropTest()
+    {
+        var before = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 0, 1)
+                                                .addPerIndex("index1", Version.EB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 0, 1)
+                                                .build();
+
+        // Updates all indexes.
+        var after = SSTableIndexComponentsState.builder()
+                                               .addPerSSTable(Version.EB, 1, 1)
+                                               .addPerIndex("index1", Version.EB, 1, 1)
+                                               .addPerIndex("index2", Version.EB, 1, 1)
+                                               .build();
+
+        SSTableIndexComponentsState.Diff diff = after.diff(before);
+        assertTrue(diff.createsUnusedComponents());
+
+        // Concurrent modification that modifides drop "index2"
+        var current = SSTableIndexComponentsState.builder()
+                                                 .addPerSSTable(Version.EB, 0, 1)
+                                                 .addPerIndex("index1", Version.EB, 0, 1)
+                                                 .build();
+
+        // We expect that applying the diff work, but just that "index2" has been removed.
+        var expected = SSTableIndexComponentsState.builder()
+                                                  .addPerSSTable(Version.EB, 1, 1)
+                                                  .addPerIndex("index1", Version.EB, 1, 1)
+                                                  .build();
+        assertEquals(expected, current.tryApplyDiff(diff));
+    }
+
+    @Test
+    public void diffWithOriginEmptyTest()
+    {
+        var before = SSTableIndexComponentsState.EMPTY;
+
+        // Creates a bunch of indexes.
+        var after = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 0, 1)
+                                                .addPerIndex("index1", Version.EB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 0, 1)
+                                                .build();
+
+        SSTableIndexComponentsState.Diff diff = after.diff(before);
+        assertFalse(diff.createsUnusedComponents());
+
+        assertEquals(after, before.tryApplyDiff(diff));
+    }
+
+    @Test
+    public void diffWithOnlyRemove()
+    {
+        var before = SSTableIndexComponentsState.builder()
+                                               .addPerSSTable(Version.EB, 0, 1)
+                                               .addPerIndex("index1", Version.EB, 0, 1)
+                                               .addPerIndex("index2", Version.EB, 0, 1)
+                                               .build();
+
+        // Same but for the drop of index1
+        var after = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.EB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 0, 1)
+                                                .build();
+
+        SSTableIndexComponentsState.Diff diff = after.diff(before);
+        assertTrue(diff.createsUnusedComponents());
+
+        assertFalse(diff.perSSTableUpdated);
+        assertTrue(diff.perIndexesUpdated.isEmpty());
+        assertEquals(1, diff.perIndexesRemoved.size());
+        assertTrue(diff.perIndexesRemoved.contains("index1"));
+
+        assertEquals(after, before.tryApplyDiff(diff));
+    }
+
+    @Test
+    public void buildFromDescriptorTest() throws IOException
+    {
+        TemporaryFolder temporaryFolder = new TemporaryFolder();
+        temporaryFolder.create();
+        try
+        {
+            IndexContext idx1 = SAITester.createIndexContext("test_index1", Int32Type.instance);
+            IndexContext idx2 = SAITester.createIndexContext("test_index2", UTF8Type.instance);
+
+            Descriptor descriptor = Descriptor.fromFilename(temporaryFolder.newFolder().getAbsolutePath() + "/ca-1-bti-Data.db");
+
+            createFakeDataFile(descriptor);
+            createFakePerSSTableComponents(descriptor, Version.latest(), 0, 1 * 1024 * 1024); // 1mb per file
+            createFakePerIndexComponents(descriptor, idx1, Version.latest(), 1, 2 * 1024 * 1024); // 2mb per file
+            createFakePerIndexComponents(descriptor, idx2, Version.DB, 0, 3 * 1024 * 1024); // 3mb per file
+
+            SSTableReader sstable = Mockito.mock(SSTableReader.class);
+            Mockito.when(sstable.getDescriptor()).thenReturn(descriptor);
+            SSTableIndexComponentsState discovered = IndexComponentDiscovery.instance().discoverComponents(sstable);
+            assertEquals(6, discovered.perSSTable().sizeInMB);
+            assertEquals(8, discovered.perIndex(idx1.getIndexName()).sizeInMB);
+            assertEquals(12, discovered.perIndex(idx2.getIndexName()).sizeInMB);
+            assertEquals(26, discovered.totalSizeInMB());
+
+            IndexDescriptor indexDescriptor = loadDescriptor(descriptor, idx1, idx2);
+
+            SSTableIndexComponentsState state = SSTableIndexComponentsState.of(indexDescriptor);
+            assertFalse(state.isEmpty());
+
+            assertEquals(Set.of(idx1.getIndexName(), idx2.getIndexName()), state.includedIndexes());
+
+            assertEquals(Version.latest(), state.perSSTable().buildId.version());
+            assertEquals(0, state.perSSTable().buildId.generation());
+            assertEquals(6, state.perSSTable().sizeInMB);
+
+            assertEquals(Version.latest(), state.perIndex(idx1.getIndexName()).buildId.version());
+            assertEquals(1, state.perIndex(idx1.getIndexName()).buildId.generation());
+            assertEquals(8, state.perIndex(idx1.getIndexName()).sizeInMB);
+
+            assertEquals(Version.DB, state.perIndex(idx2.getIndexName()).buildId.version());
+            assertEquals(0, state.perIndex(idx2.getIndexName()).buildId.generation());
+            assertEquals(12, state.perIndex(idx2.getIndexName()).sizeInMB);
+
+            assertEquals(26, state.totalSizeInMB());
+        }
+        finally
+        {
+            temporaryFolder.delete();
+        }
+    }
+
+    @Test
+    public void unbuildTest()
+    {
+        var state1 = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.DC, 0, 1)
+                                                .addPerIndex("index1", Version.EB, 0, 1)
+                                                .addPerIndex("index2", Version.EB, 0, 1)
+                                                .build();
+
+        var state2 = state1.unbuild()
+                           .addPerSSTable(Version.EB, 0, 1)
+                           .addPerIndex("index3", Version.EB, 0, 1)
+                           .build();
+
+
+        assertEquals(Version.EB, state2.perSSTable().buildId.version());
+        assertEquals(0, state2.perSSTable().buildId.generation());
+
+        assertFalse(state1.includedIndexes().contains("index3"));
+        assertTrue(state2.includedIndexes().contains("index3"));
+
+        // Undoing the changes to state1
+        var state3 = state2.unbuild()
+                           .addPerSSTable(Version.DC, 0, 1)
+                           .removePerIndex("index3")
+                           .build();
+
+        assertEquals(state1, state3);
+    }
+
+    @Test
+    public void sizeInBytesTest()
+    {
+        var state1 = SSTableIndexComponentsState.builder()
+                                                .addPerSSTable(Version.DC, 0, 100)
+                                                .addPerIndex("index1", Version.EB, 0, 42)
+                                                .addPerIndex("index2", Version.EB, 0, 220)
+                                                .build();
+
+        assertEquals(100 + 42 + 220, state1.totalSizeInMB());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java b/test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java
index cd9232dce9e0..746deb757f69 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java
@@ -24,19 +24,34 @@
 
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class VersionTest
 {
     @BeforeClass
-    public static void initialise()
+    public static void initialise() throws Throwable
     {
-        DatabaseDescriptor.toolInitialization();
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void testVersionsSorted()
+    {
+        Version previous = null;
+        for (Version version : Version.ALL)
+        {
+            if (previous != null)
+                assertTrue(previous.onOrAfter(version));
+            previous = version;
+        }
     }
 
     @Test
     public void supportedVersionsWillParse()
     {
         assertEquals(Version.AA, Version.parse("aa"));
+        assertEquals(Version.BA, Version.parse("ba"));
+        assertEquals(Version.CA, Version.parse("ca"));
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/io/BytesRefUtilTest.java b/test/unit/org/apache/cassandra/index/sai/disk/io/BytesRefUtilTest.java
new file mode 100644
index 000000000000..08b058e64393
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/io/BytesRefUtilTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.junit.Test;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class BytesRefUtilTest
+{
+    @Test
+    public void shouldCopyBufferToBytesRef()
+    {
+        final Random random = new Random();
+        final byte[] expectedBytes = new byte[21];
+        random.nextBytes(expectedBytes);
+        final BytesRefBuilder refBuilder = new BytesRefBuilder();
+
+        BytesRefUtil.copyBufferToBytesRef(ByteBuffer.wrap(expectedBytes), refBuilder);
+        final BytesRef actualBytesRef = refBuilder.get();
+
+        assertEquals(expectedBytes.length, actualBytesRef.length);
+        final byte[] actualBytes = new byte[actualBytesRef.length];
+        System.arraycopy(actualBytesRef.bytes, actualBytesRef.offset, actualBytes, 0, actualBytesRef.length);
+        assertArrayEquals(expectedBytes, actualBytes);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/io/IndexOutputWriterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/io/IndexOutputWriterTest.java
new file mode 100644
index 000000000000..edcc39563e1b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/io/IndexOutputWriterTest.java
@@ -0,0 +1,100 @@
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Random;
+import java.util.stream.Collectors;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+
+
+/**
+ * Tests for {@link IndexOutputWriter} created by {@link IndexFileUtils#openOutput}. Specifically focused on
+ * checksumming.
+ */
+@RunWith(Parameterized.class)
+public class IndexOutputWriterTest
+{
+    private static final Random RAND = new Random(42);
+
+    static final int BUFFER_SIZE = 128;
+    SequentialWriterOption writerOption = SequentialWriterOption.newBuilder()
+                                          .bufferSize(BUFFER_SIZE)
+                                          .bufferType(BufferType.OFF_HEAP)
+                                          .finishOnClose(true)
+                                          .build();
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Parameterized.Parameter
+    public org.apache.cassandra.index.sai.disk.format.Version version;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static Collection<Object[]> data()
+    {
+        return Version.ALL.stream().map(v -> new Object[]{v}).collect(Collectors.toList());
+    }
+
+    // Open a checksummed writer.
+    private IndexOutputWriter open(Path file, ByteOrder order, boolean append) throws IOException {
+        return new IndexFileUtils(writerOption)
+               .openOutput(new org.apache.cassandra.io.util.File(file.toFile()), order, append, version);
+    }
+
+
+    @Test
+    public void checksumWriteMostSignificantBytesLittleEndian() throws Exception {
+        checksumWriteMostSignificantBytes(ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Test
+    public void checksumWriteMostSignificantBytesBigEndian() throws Exception {
+        checksumWriteMostSignificantBytes(ByteOrder.BIG_ENDIAN);
+    }
+
+    public void checksumWriteMostSignificantBytes(ByteOrder order) throws Exception {
+        Path path = Files.createTempFile("checksum", "test");
+        try (IndexOutputWriter w = open(path, order, /*append*/ false)) {
+            SequentialWriter writer = w.asSequentialWriter();
+            // Write enough bytes to fill the buffer, then one more
+            int bytesWritten = 0;
+            while (bytesWritten < BUFFER_SIZE) {
+                int bytes = RAND.nextInt(8) + 1;
+                writer.writeMostSignificantBytes(RAND.nextLong(), bytes);
+                bytesWritten += bytes;
+            }
+
+            // Close the writer to flush to disk.
+            w.close();
+            // Validate the checksum matches the on‑disk bytes
+            assertEquals(crc32(Files.readAllBytes(path)), w.getChecksum());
+        }
+    }
+
+    /** Compute CRC‑32 of one or more byte‑arrays. */
+    private static long crc32(byte[]... chunks) {
+        Checksum crc = new CRC32();
+        for (byte[] c : chunks) crc.update(c, 0, c.length);
+        return crc.getValue();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/io/TrackingIndexFileUtils.java b/test/unit/org/apache/cassandra/index/sai/disk/io/TrackingIndexFileUtils.java
index 6284a3b980ad..e6940fe12dd7 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/io/TrackingIndexFileUtils.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/io/TrackingIndexFileUtils.java
@@ -21,19 +21,21 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import com.google.common.base.Throwables;
+import org.junit.Assert;
 
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.SequentialWriterOption;
-import org.apache.lucene.store.IndexInput;
-
-import static org.junit.Assert.assertNotNull;
+import org.apache.lucene.index.CorruptIndexException;
 
 public class TrackingIndexFileUtils extends IndexFileUtils
 {
-    private final Map<TrackedForwardingIndexInput, String> openInputs = Collections.synchronizedMap(new HashMap<>());
+    private final Map<TrackingIndexInput, String> openInputs = Collections.synchronizedMap(new HashMap<>());
+    private final Set<TrackingIndexInput> closedInputs = Collections.synchronizedSet(new HashSet<>());
 
     public TrackingIndexFileUtils(SequentialWriterOption writerOption)
     {
@@ -41,34 +43,46 @@ public TrackingIndexFileUtils(SequentialWriterOption writerOption)
     }
 
     @Override
-    public IndexInput openInput(FileHandle handle)
+    public IndexInputReader openInput(FileHandle handle)
     {
-        TrackedForwardingIndexInput input = new TrackedForwardingIndexInput(super.openInput(handle));
+        TrackingIndexInput input = new TrackingIndexInput((IndexInputReader) super.openInput(handle));
         openInputs.put(input, Throwables.getStackTraceAsString(new RuntimeException("Input created")));
         return input;
     }
 
+    @Override
+    public IndexInputReader openBlockingInput(FileHandle fileHandle)
+    {
+        TrackingIndexInput input = new TrackingIndexInput(super.openBlockingInput(fileHandle));
+        openInputs.put(input, Throwables.getStackTraceAsString(new RuntimeException("Blocking input created")));
+        return input;
+    }
+
     public Map<IndexInput, String> getOpenInputs()
     {
         return new HashMap<>(openInputs);
     }
 
-    public class TrackedForwardingIndexInput extends IndexInput
+    private class TrackingIndexInput extends IndexInputReader
     {
-        private final IndexInput delegate;
+        private final IndexInputReader delegate;
 
-        protected TrackedForwardingIndexInput(IndexInput delegate)
+        protected TrackingIndexInput(IndexInputReader delegate)
         {
-            super(delegate.toString());
+            super(delegate.reader(), () -> {});
             this.delegate = delegate;
         }
 
         @Override
-        public void close() throws IOException
+        public synchronized void close()
         {
             delegate.close();
             final String creationStackTrace = openInputs.remove(this);
-            assertNotNull("Closed unregistered input: " + this, creationStackTrace);
+
+            if (closedInputs.add(this) && creationStackTrace == null)
+            {
+                Assert.fail("Closed unregistered input: " + this);
+            }
         }
 
         @Override
@@ -78,7 +92,7 @@ public long getFilePointer()
         }
 
         @Override
-        public void seek(long pos) throws IOException
+        public void seek(long pos)
         {
             delegate.seek(pos);
         }
@@ -90,7 +104,7 @@ public long length()
         }
 
         @Override
-        public IndexInput slice(String sliceDescription, long offset, long length) throws IOException
+        public IndexInput slice(String sliceDescription, long offset, long length) throws CorruptIndexException
         {
             return delegate.slice(sliceDescription, offset, length);
         }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/oldlucene/ByteArrayIndexInputTest.java b/test/unit/org/apache/cassandra/index/sai/disk/oldlucene/ByteArrayIndexInputTest.java
new file mode 100644
index 000000000000..e9846a7fade4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/oldlucene/ByteArrayIndexInputTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+import static org.junit.Assert.*;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteOrder;
+
+public class ByteArrayIndexInputTest {
+
+    private ByteArrayIndexInput inputBE;
+    private ByteArrayIndexInput inputLE;
+    private final byte[] testData = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+    // Reset the input streams before each test
+    @Before
+    public void setUp() {
+        inputBE = new ByteArrayIndexInput("BigEndianTest", testData, ByteOrder.BIG_ENDIAN);
+        inputLE = new ByteArrayIndexInput("LittleEndianTest", testData, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Test
+    public void testReadByte() {
+        assertEquals(0, inputBE.readByte());
+        assertEquals(1, inputBE.readByte());
+
+        assertEquals(0, inputLE.readByte());
+        assertEquals(1, inputLE.readByte());
+    }
+
+    @Test
+    public void testReadShort() {
+        var beExpected = (short) 1;
+        var leExpected = (short) 256;
+        assertEquals(beExpected, inputBE.readShort());
+        assertEquals(leExpected, inputLE.readShort());
+        assertEquals(beExpected, Short.reverseBytes(leExpected));
+    }
+
+    @Test
+    public void testReadInt() {
+        var beExpected = 66051;
+        var leExpected = 50462976;
+        assertEquals(beExpected, inputBE.readInt());
+        assertEquals(leExpected, inputLE.readInt());
+        assertEquals(beExpected, Integer.reverseBytes(leExpected));
+    }
+
+    @Test
+    public void testReadLong() {
+        var beExpected = 283686952306183L;
+        var leExpected = 506097522914230528L;
+        assertEquals(beExpected, inputBE.readLong());
+        assertEquals(leExpected, inputLE.readLong());
+        assertEquals(beExpected, Long.reverseBytes(leExpected));
+    }
+
+    @Test
+    public void testReadByteWithPosition() throws IOException
+    {
+        assertEquals(2, inputBE.readByte(2));
+        assertEquals(2, inputLE.readByte(2));
+    }
+
+    @Test
+    public void testReadShortWithPosition() throws IOException {
+        var beExpected = (short) 515;
+        var leExpected = (short) 770;
+        assertEquals(beExpected, inputBE.readShort(2));
+        assertEquals(leExpected, inputLE.readShort(2));
+        assertEquals(beExpected, Short.reverseBytes(leExpected));
+    }
+
+    @Test
+    public void testReadIntWithPosition() throws IOException {
+        var beExpected = 33752069;
+        var leExpected = 84148994;
+        assertEquals(beExpected, inputBE.readInt(2));
+        assertEquals(leExpected, inputLE.readInt(2));
+        assertEquals(beExpected, Integer.reverseBytes(leExpected));
+    }
+
+    @Test
+    public void testReadLongWithPosition() throws IOException {
+        var beExpected = 144964032628459529L;
+        var leExpected = 650777868590383874L;
+        assertEquals(beExpected, inputBE.readLong(2));
+        assertEquals(leExpected, inputLE.readLong(2));
+        assertEquals(beExpected, Long.reverseBytes(leExpected));
+    }
+
+    @Test(expected = ArrayIndexOutOfBoundsException.class)
+    public void testReadShortBeyondEndWithPosition() throws IOException {
+        inputBE.readShort(testData.length); // Attempt to read beyond the available data
+    }
+
+    @Test(expected = ArrayIndexOutOfBoundsException.class)
+    public void testReadIntBeyondEndWithPosition() throws IOException {
+        inputBE.readInt(testData.length - 1); // Attempt to read beyond the available data
+    }
+
+    @Test(expected = ArrayIndexOutOfBoundsException.class)
+    public void testReadLongBeyondEndWithPosition() throws IOException {
+        inputBE.readLong(testData.length - 3); // Attempt to read beyond the available data
+    }
+
+    @Test
+    public void testSeekAndGetPosition() throws EOFException {
+        inputBE.seek(2);
+        assertEquals(2, inputBE.getFilePointer());
+
+        inputLE.seek(4);
+        assertEquals(4, inputLE.getFilePointer());
+    }
+
+    @Test(expected = EOFException.class)
+    public void testSeekBeyondLengthBE() throws EOFException {
+        inputBE.seek(testData.length + 1);
+    }
+
+    @Test(expected = EOFException.class)
+    public void testSeekBeyondLengthLE() throws EOFException {
+        inputLE.seek(testData.length + 1);
+    }
+
+    @Test
+    public void testLength() {
+        assertEquals(testData.length, inputBE.length());
+        assertEquals(testData.length, inputLE.length());
+    }
+
+    @Test
+    public void testReadBytes() {
+        byte[] buffer = new byte[4];
+        inputBE.readBytes(buffer, 0, buffer.length);
+        assertArrayEquals(new byte[]{0, 1, 2, 3}, buffer);
+
+        inputLE.readBytes(buffer, 0, buffer.length);
+        assertArrayEquals(new byte[]{0, 1, 2, 3}, buffer);
+    }
+
+    @Test(expected = ArrayIndexOutOfBoundsException.class)
+    public void testReadBeyondEnd() {
+        byte[] buffer = new byte[11];
+        inputBE.readBytes(buffer, 0, buffer.length);
+    }
+
+    @Test
+    public void testClose() {
+        inputBE.close();
+        try {
+            inputBE.readByte();
+            fail("Should throw a NullPointerException after close");
+        } catch (NullPointerException e) {
+            // expected
+        }
+    }
+
+    @Test
+    public void testCloneAndSlice() throws EOFException {
+        ByteArrayIndexInput clone = (ByteArrayIndexInput) inputBE.clone();
+        ByteArrayIndexInput slice = inputBE.slice("slice", 2, 4);
+
+        clone.seek(2);
+        assertEquals(2, clone.readByte());
+
+        assertEquals(2, slice.readByte());
+        assertEquals(4, slice.length());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/oldlucene/ByteBuffersOverridesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/oldlucene/ByteBuffersOverridesTest.java
new file mode 100644
index 000000000000..8aaa9c060212
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/oldlucene/ByteBuffersOverridesTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.oldlucene;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+/**
+ * Tests for classes that implement big-endianness by overriding superclass methods. This is necessary to ensure that
+ * new methods don't get introduced reintroducing the endianness of the superclass.
+ */
+public class ByteBuffersOverridesTest
+{
+    @Test
+    public void testLegacyByteBuffersDataInputOverrides()
+    {
+        // manually checked methods that endianness doesn't impact
+        Set<String> ignoredMethods = Set.of("readSetOfStrings", "readVInt", "readZInt", "readVLong", "readZLong",
+                                            "readString", "readBytes", "clone", "readMapOfStrings");
+        checkOverrides(LegacyByteBuffersDataInput.class, DataInput.class, ignoredMethods);
+    }
+
+    @Test
+    public void testLegacyByteBuffersDataOutputOverrides()
+    {
+        // manually checked methods that endianness doesn't impact
+        Set<String> ignoredMethods = Set.of("writeZLong", "writeVLong", "writeVInt", "copyBytes", "writeZInt");
+        checkOverrides(LegacyByteBuffersDataOutput.class, DataOutput.class, ignoredMethods);
+    }
+
+    @Test
+    public void testLegacyByteBuffersDataOutputAdapterOverrides()
+    {
+        Set<String> ignoredMethods = Set.of("writeZLong", "writeVLong", "writeVInt", "copyBytes", "writeZInt");
+        checkOverrides(LegacyByteBuffersDataOutputAdapter.class, DataOutput.class, ignoredMethods);
+    }
+
+    @Test
+    public void testModernByteBuffersDataOutputAdapterOverrides()
+    {
+        Set<String> ignoredMethods = Set.of("writeZLong", "writeVLong", "writeVInt", "copyBytes", "writeZInt");
+        checkOverrides(ModernByteBuffersDataOutputAdapter.class, DataOutput.class, ignoredMethods);
+    }
+
+    /**
+     * Check if all methods declared by the parent class are overridden in the subclass.
+     *
+     * @param subclass    the subclass to be tested
+     * @param parentClass the parent class from which methods are inherited
+     */
+    private void checkOverrides(Class<?> subclass, Class<?> parentClass, Set<String> ignoredMethods)
+    {
+        Set<Method> parentMethods = new HashSet<>(Arrays.asList(parentClass.getDeclaredMethods()));
+        Set<Method> declaredMethodsInSubclass = new HashSet<>(Arrays.asList(subclass.getDeclaredMethods()));
+
+        for (Method parentMethod : parentMethods)
+        {
+            if (ignoredMethods.contains(parentMethod.getName()))
+                continue;
+
+            if (Modifier.isPublic(parentMethod.getModifiers()) || Modifier.isProtected(parentMethod.getModifiers()))
+            {
+                boolean isOverridden = declaredMethodsInSubclass.stream().anyMatch(m ->
+                                                                                   m.getName().equals(parentMethod.getName()) &&
+                                                                                   Arrays.equals(m.getParameterTypes(), parentMethod.getParameterTypes()));
+                Assert.assertTrue("Method " + parentMethod.getName() + " is not overridden in " + subclass.getName(), isOverridden);
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/BalancedTreeIndexSearcherTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/BalancedTreeIndexSearcherTest.java
deleted file mode 100644
index 848dbf83a8e9..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/BalancedTreeIndexSearcherTest.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.List;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.LongStream;
-
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.marshal.DecimalType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.marshal.NumberType;
-import org.apache.cassandra.db.marshal.ShortType;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.v1.bbtree.BlockBalancedTreeIndexBuilder;
-import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcher;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
-public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester
-{
-    private static final short EQ_TEST_LOWER_BOUND_INCLUSIVE = 0;
-    private static final short EQ_TEST_UPPER_BOUND_EXCLUSIVE = 3;
-
-    private static final short RANGE_TEST_LOWER_BOUND_INCLUSIVE = 0;
-    private static final short RANGE_TEST_UPPER_BOUND_EXCLUSIVE = 10;
-
-    @Test
-    public void testRangeQueriesAgainstInt32Index() throws Exception
-    {
-        doTestRangeQueriesAgainstInt32Index();
-    }
-
-    private void doTestRangeQueriesAgainstInt32Index() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildInt32Searcher(newIndexDescriptor(), 0, 10);
-        testRangeQueries(indexSearcher, Int32Type.instance, Integer::valueOf);
-    }
-
-    @Test
-    public void testEqQueriesAgainstInt32Index() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildInt32Searcher(newIndexDescriptor(),
-                                                                                              EQ_TEST_LOWER_BOUND_INCLUSIVE, EQ_TEST_UPPER_BOUND_EXCLUSIVE);
-        testEqQueries(indexSearcher, Int32Type.instance, Integer::valueOf);
-    }
-
-    @Test
-    public void testRangeQueriesAgainstLongIndex() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildLongSearcher(newIndexDescriptor(), 0, 10);
-        testRangeQueries(indexSearcher, LongType.instance, Long::valueOf);
-    }
-
-    @Test
-    public void testEqQueriesAgainstLongIndex() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildLongSearcher(newIndexDescriptor(),
-                                                                                             EQ_TEST_LOWER_BOUND_INCLUSIVE, EQ_TEST_UPPER_BOUND_EXCLUSIVE);
-        testEqQueries(indexSearcher, LongType.instance, Long::valueOf);
-    }
-
-    @Test
-    public void testRangeQueriesAgainstShortIndex() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildShortSearcher(newIndexDescriptor(), (short) 0, (short) 10);
-        testRangeQueries(indexSearcher, ShortType.instance, Function.identity());
-    }
-
-    @Test
-    public void testEqQueriesAgainstShortIndex() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildShortSearcher(newIndexDescriptor(),
-                                                                                              EQ_TEST_LOWER_BOUND_INCLUSIVE, EQ_TEST_UPPER_BOUND_EXCLUSIVE);
-        testEqQueries(indexSearcher, ShortType.instance, Function.identity());
-    }
-
-    @Test
-    public void testRangeQueriesAgainstDecimalIndex() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildDecimalSearcher(newIndexDescriptor(),
-                                                                                                BigDecimal.ZERO, BigDecimal.valueOf(10L));
-        testRangeQueries(indexSearcher, DecimalType.instance, BigDecimal::valueOf, getLongsOnInterval(20L, 70L));
-    }
-
-    private List<Long> getLongsOnInterval(long lowerInclusive, long upperInclusive)
-    {
-        return LongStream.range(lowerInclusive, upperInclusive + 1L).boxed().collect(Collectors.toList());
-    }
-
-    @Test
-    public void testEqQueriesAgainstDecimalIndex() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildDecimalSearcher(newIndexDescriptor(),
-                                                                                                BigDecimal.valueOf(EQ_TEST_LOWER_BOUND_INCLUSIVE), BigDecimal.valueOf(EQ_TEST_UPPER_BOUND_EXCLUSIVE));
-        testEqQueries(indexSearcher, DecimalType.instance, BigDecimal::valueOf);
-    }
-
-
-    @Test
-    public void testEqQueriesAgainstBigIntegerIndex() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildBigIntegerSearcher(newIndexDescriptor(),
-                                                                                                   BigInteger.valueOf(EQ_TEST_LOWER_BOUND_INCLUSIVE), BigInteger.valueOf(EQ_TEST_UPPER_BOUND_EXCLUSIVE));
-        testEqQueries(indexSearcher, IntegerType.instance, BigInteger::valueOf);
-    }
-
-    @Test
-    public void testRangeQueriesAgainstBigIntegerIndex() throws Exception
-    {
-        IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildBigIntegerSearcher(newIndexDescriptor(),
-                                                                                                   BigInteger.ZERO, BigInteger.valueOf(10L));
-        testRangeQueries(indexSearcher, IntegerType.instance, BigInteger::valueOf);
-    }
-
-    private <T extends Number> void testEqQueries(final IndexSegmentSearcher indexSearcher,
-                                                  final NumberType<T> rawType,
-                                                  final Function<Short, T> rawValueProducer) throws Exception
-    {
-        try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType))
-                                                                       .add(Operator.EQ, rawType.decompose(rawValueProducer.apply(EQ_TEST_LOWER_BOUND_INCLUSIVE)))
-                                                             , null, mock(QueryContext.class)))
-        {
-            assertTrue(results.hasNext());
-
-            assertEquals(0L, results.next().token().getLongValue());
-        }
-
-        try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType))
-                                                                       .add(Operator.EQ, rawType.decompose(rawValueProducer.apply(EQ_TEST_UPPER_BOUND_EXCLUSIVE))),
-                                                             null, mock(QueryContext.class)))
-        {
-            assertFalse(results.hasNext());
-            indexSearcher.close();
-        }
-    }
-
-    private <T extends Number> void testRangeQueries(final IndexSegmentSearcher indexSearcher,
-                                                     final NumberType<T> rawType,
-                                                     final Function<Short, T> rawValueProducer) throws Exception
-    {
-        List<Long> expectedTokenList = getLongsOnInterval(2L, 7L);
-        testRangeQueries(indexSearcher, rawType, rawValueProducer, expectedTokenList);
-    }
-
-
-    private <T extends Number> void testRangeQueries(final IndexSegmentSearcher indexSearcher,
-                                                     final NumberType<T> rawType,
-                                                     final Function<Short, T> rawValueProducer,
-                                                     List<Long> expectedTokenList) throws Exception
-    {
-        try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType))
-                                                                       .add(Operator.GTE, rawType.decompose(rawValueProducer.apply((short)2)))
-                                                                       .add(Operator.LTE, rawType.decompose(rawValueProducer.apply((short)7))),
-                                                             null, mock(QueryContext.class)))
-        {
-            assertTrue(results.hasNext());
-
-            List<Long> actualTokenList = Lists.newArrayList(Iterators.transform(results, key -> key.token().getLongValue()));
-            assertEquals(expectedTokenList, actualTokenList);
-        }
-
-        try (KeyRangeIterator results = indexSearcher.search(new Expression.IndexedExpression(SAITester.createMockIndex(rawType))
-        {{
-            operator = IndexOperator.RANGE;
-            lower = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_UPPER_BOUND_EXCLUSIVE)), getIndexTermType(), true);
-        }}, null, mock(QueryContext.class)))
-        {
-            assertFalse(results.hasNext());
-        }
-
-        try (KeyRangeIterator results = indexSearcher.search(new Expression.IndexedExpression(SAITester.createMockIndex(rawType))
-        {{
-            operator = IndexOperator.RANGE;
-            upper = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_LOWER_BOUND_INCLUSIVE)), getIndexTermType(), false);
-        }}, null, mock(QueryContext.class)))
-        {
-            assertFalse(results.hasNext());
-            indexSearcher.close();
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfigTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfigTest.java
new file mode 100644
index 000000000000..1c62ad48b9f5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfigTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class IndexWriterConfigTest
+{
+    @Test
+    public void defaultsTest()
+    {
+        IndexWriterConfig config = IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), new HashMap<>());
+
+        assertThat(config.getMaximumNodeConnections()).isEqualTo(16);
+        assertThat(config.getConstructionBeamWidth()).isEqualTo(100);
+        assertThat(config.getSimilarityFunction()).isEqualTo(VectorSimilarityFunction.COSINE);
+    }
+
+    @Test
+    public void maximumNodeConnectionsTest()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(IndexWriterConfig.MAXIMUM_NODE_CONNECTIONS, "10");
+        IndexWriterConfig config = IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options);
+        assertThat(config.getMaximumNodeConnections()).isEqualTo(10);
+
+        options.put(IndexWriterConfig.MAXIMUM_NODE_CONNECTIONS, "-1");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Maximum number of connections for index test cannot be <= 0 or > 512, was -1");
+
+        options.put(IndexWriterConfig.MAXIMUM_NODE_CONNECTIONS, Integer.toString(IndexWriterConfig.MAXIMUM_MAXIMUM_NODE_CONNECTIONS + 1));
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Maximum number of connections for index test cannot be <= 0 or > 512, was " + (IndexWriterConfig.MAXIMUM_MAXIMUM_NODE_CONNECTIONS + 1));
+
+        options.put(IndexWriterConfig.MAXIMUM_NODE_CONNECTIONS, "abc");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Maximum number of connections abc is not a valid integer for index test");
+    }
+
+    @Test
+    public void queueSizeTest()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(IndexWriterConfig.CONSTRUCTION_BEAM_WIDTH, "150");
+        IndexWriterConfig config = IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options);
+        assertThat(config.getConstructionBeamWidth()).isEqualTo(150);
+
+        options.put(IndexWriterConfig.CONSTRUCTION_BEAM_WIDTH, "-1");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Construction beam width for index test cannot be <= 0 or > 3200, was -1");
+
+        options.put(IndexWriterConfig.CONSTRUCTION_BEAM_WIDTH, Integer.toString(IndexWriterConfig.MAXIMUM_CONSTRUCTION_BEAM_WIDTH + 1));
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Construction beam width for index test cannot be <= 0 or > 3200, was " + (IndexWriterConfig.MAXIMUM_CONSTRUCTION_BEAM_WIDTH + 1));
+
+        options.put(IndexWriterConfig.CONSTRUCTION_BEAM_WIDTH, "abc");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Construction beam width abc is not a valid integer for index test");
+    }
+
+    @Test
+    public void similarityFunctionTest()
+    {
+        Map<String, String> options = new HashMap<>();
+        options.put(IndexWriterConfig.SIMILARITY_FUNCTION, "DOT_PRODUCT");
+        IndexWriterConfig config = IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options);
+        assertThat(config.getSimilarityFunction()).isEqualTo(VectorSimilarityFunction.DOT_PRODUCT);
+
+        options.put(IndexWriterConfig.SIMILARITY_FUNCTION, "euclidean");
+        config = IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options);
+        assertThat(config.getSimilarityFunction()).isEqualTo(VectorSimilarityFunction.EUCLIDEAN);
+
+        options.put(IndexWriterConfig.SIMILARITY_FUNCTION, "blah");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessage("Similarity function BLAH was not recognized for index test. Valid values are: EUCLIDEAN, DOT_PRODUCT, COSINE");
+    }
+
+    @Test
+    public void alphaTest()
+    {
+        Map<String, String> options = new HashMap<>();
+        // Provide a valid alpha
+        options.put(IndexWriterConfig.ALPHA, "1.7");
+        IndexWriterConfig config = IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options);
+        assertThat(config.getAlpha(999f)).isEqualTo(1.7f);
+
+        // Provide an invalid (negative) alpha
+        options.put(IndexWriterConfig.ALPHA, "-5");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Alpha for index test must be > 0, was -5");
+
+        // Provide a non-float alpha
+        options.put(IndexWriterConfig.ALPHA, "abc");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Alpha abc is not a valid float for index test");
+    }
+
+    @Test
+    public void neighborhoodOverflowTest()
+    {
+        Map<String, String> options = new HashMap<>();
+        // Provide a valid neighborhood_overflow
+        options.put(IndexWriterConfig.NEIGHBORHOOD_OVERFLOW, "2.3");
+        IndexWriterConfig config = IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options);
+        assertThat(config.getNeighborhoodOverflow(999f)).isEqualTo(2.3f);
+
+        // Provide invalid (<=0) overflow
+        options.put(IndexWriterConfig.NEIGHBORHOOD_OVERFLOW, "0");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Neighborhood overflow for index test must be >= 1.0, was 0");
+
+        // Provide a non-float overflow
+        options.put(IndexWriterConfig.NEIGHBORHOOD_OVERFLOW, "abc");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Neighborhood overflow abc is not a valid float for index test");
+    }
+
+    @Test
+    public void enableHierarchyTest()
+    {
+        Map<String, String> options = new HashMap<>();
+        // Provide a valid enable_hierarchy
+        options.put(IndexWriterConfig.ENABLE_HIERARCHY, "true");
+        IndexWriterConfig config = IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options);
+        assertThat(config.isHierarchyEnabled()).isTrue();
+
+        // Provide an invalid enable_hierarchy
+        options.put(IndexWriterConfig.ENABLE_HIERARCHY, "foo");
+        assertThatThrownBy(() -> IndexWriterConfig.fromOptions("test", VectorType.getInstance(FloatType.instance, 3), options))
+        .isInstanceOf(InvalidRequestException.class)
+        .hasMessageContaining("Enable hierarchy must be 'true' or 'false' for index test, was 'foo'");
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexBuilder.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexBuilder.java
index 50a218801035..1fd65ce42a67 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexBuilder.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexBuilder.java
@@ -17,14 +17,20 @@
  */
 package org.apache.cassandra.index.sai.disk.v1;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.function.IntSupplier;
 import java.util.function.Supplier;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import com.carrotsearch.hppc.LongArrayList;
+import com.carrotsearch.hppc.IntArrayList;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.memory.RowMapping;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
@@ -32,19 +38,23 @@
 
 public class InvertedIndexBuilder
 {
-    public static List<Pair<ByteComparable, LongArrayList>> buildStringTermsEnum(int terms, int postings, Supplier<String> termsGenerator, IntSupplier postingsGenerator)
+    /**
+     * Builds a list of {@link TermsEnum} instances with the given number of terms and postings with the intention of
+     * building them in the same way that the TrieMemoryIndex would in preparation for writing to disk.
+     */
+    public static List<TermsEnum> buildStringTermsEnum(Version version, int terms, int postings, Supplier<String> termsGenerator, IntSupplier postingsGenerator)
     {
-        final List<ByteComparable> sortedTerms = Stream.generate(termsGenerator)
-                                                       .distinct()
-                                                       .limit(terms)
-                                                       .sorted()
-                                                       .map(ByteComparable::of)
-                                                       .collect(toList());
-
-        final List<Pair<ByteComparable, LongArrayList>> termsEnum = new ArrayList<>();
-        for (ByteComparable term : sortedTerms)
+        final List<ByteBuffer> sortedTerms = Stream.generate(termsGenerator)
+                                                   .distinct()
+                                                   .limit(terms)
+                                                   .sorted()
+                                                   .map(UTF8Type.instance::decompose)
+                                                   .collect(toList());
+
+        final List<TermsEnum> termsEnum = new ArrayList<>();
+        for (ByteBuffer term : sortedTerms)
         {
-            final LongArrayList postingsList = new LongArrayList();
+            final IntArrayList postingsList = new IntArrayList();
 
             IntStream.generate(postingsGenerator)
                      .distinct()
@@ -52,8 +62,39 @@ public static List<Pair<ByteComparable, LongArrayList>> buildStringTermsEnum(int
                      .sorted()
                      .forEach(postingsList::add);
 
-            termsEnum.add(Pair.create(term, postingsList));
+            // This logic feels a bit fragile, but it mimics the way we call unescape in the TrieMemoryIndex
+            // before writing to the on disk format.
+            var encoded = version.onDiskFormat().encodeForTrie(term, UTF8Type.instance).preencode(TypeUtil.BYTE_COMPARABLE_VERSION);
+            termsEnum.add(new TermsEnum(term, encoded, postingsList));
         }
         return termsEnum;
     }
+
+    /**
+     * Convenience wrapper for a term's original bytes, its byte comparable encoding, and its associated postings list.
+     */
+    public static class TermsEnum
+    {
+        // Store the original term to ensure that searching by it is successful
+        final ByteBuffer originalTermBytes;
+        final ByteComparable.Preencoded byteComparableBytes;
+        final IntArrayList postings;
+
+        TermsEnum(ByteBuffer originalTermBytes, ByteComparable.Preencoded byteComparableBytes, IntArrayList postings)
+        {
+            this.originalTermBytes = originalTermBytes;
+            this.byteComparableBytes = byteComparableBytes;
+            this.postings = postings;
+        }
+    }
+
+    /**
+     * Adds default frequency of 1 to postings
+     */
+    static Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>> toTermWithFrequency(TermsEnum te)
+    {
+        return Pair.create(te.byteComparableBytes, Arrays.stream(te.postings.toArray()).boxed()
+                                                         .map(p -> new RowMapping.RowIdWithFrequency(p, 1))
+                                                         .collect(toList()));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java
index 93045c80d6d1..a15b11f6abf3 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java
@@ -19,78 +19,64 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Collection;
 import java.util.List;
+import java.util.stream.Collectors;
 
+import org.agrona.collections.Int2IntHashMap;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import com.carrotsearch.hppc.LongArrayList;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.QueryContext;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.MemtableTermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcher;
-import org.apache.cassandra.index.sai.disk.v1.segment.LiteralIndexSegmentSearcher;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.KDTreeIndexBuilder;
+import org.apache.cassandra.index.sai.disk.v1.trie.InvertedIndexWriter;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
 import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
-public class InvertedIndexSearcherTest extends SAIRandomizedTester
+public class InvertedIndexSearcherTest extends SaiRandomizedTest
 {
-    public static final PrimaryKeyMap TEST_PRIMARY_KEY_MAP = new PrimaryKeyMap()
-    {
-        private final PrimaryKey.Factory primaryKeyFactory = new PrimaryKey.Factory(Murmur3Partitioner.instance, new ClusteringComparator());
+    public static final int LIMIT = Integer.MAX_VALUE;
 
-        @Override
-        public PrimaryKey primaryKeyFromRowId(long sstableRowId)
-        {
-            return primaryKeyFactory.create(new Murmur3Partitioner.LongToken(sstableRowId));
-        }
+    // Use a shared index context to prevent creating too many metrics unnecessarily
+    private final IndexContext indexContext = SAITester.createIndexContext("meh", UTF8Type.instance);
 
-        @Override
-        public long rowIdFromPrimaryKey(PrimaryKey key)
-        {
-            return key.token().getLongValue();
-        }
+    @ParametersFactory()
+    public static Collection<Object[]> data()
+    {
+        // Required because it configures SEGMENT_BUILD_MEMORY_LIMIT, which is needed for Version.AA
+        if (DatabaseDescriptor.getRawConfig() == null)
+            DatabaseDescriptor.setConfig(DatabaseDescriptor.loadConfig());
+        return Version.ALL.stream().map(v -> new Object[]{v}).collect(Collectors.toList());
+    }
 
-        @Override
-        public long ceiling(Token token)
-        {
-            return 0;
-        }
+    private final Version version;
 
-        @Override
-        public long floor(Token token)
-        {
-            return 0;
-        }
-    };
-    public static final PrimaryKeyMap.Factory TEST_PRIMARY_KEY_MAP_FACTORY = () -> TEST_PRIMARY_KEY_MAP;
+    public InvertedIndexSearcherTest(Version version)
+    {
+        this.version = version;
+    }
 
     @BeforeClass
     public static void setupCQLTester()
@@ -99,26 +85,35 @@ public static void setupCQLTester()
         StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
     }
 
+    @Test
+    public void testPrimaryKeyMapFactoryCount()
+    {
+        assertEquals(Long.MAX_VALUE, KDTreeIndexBuilder.TEST_PRIMARY_KEY_MAP_FACTORY.count());
+    }
+
     @Test
     public void testEqQueriesAgainstStringIndex() throws Exception
     {
-        QueryContext context = mock(QueryContext.class);
-        final StorageAttachedIndex index = createMockIndex(UTF8Type.instance);
+        doTestEqQueriesAgainstStringIndex(version);
+    }
 
-        final int numTerms = getRandom().nextIntBetween(64, 512), numPostings = getRandom().nextIntBetween(256, 1024);
-        final List<Pair<ByteComparable, LongArrayList>> termsEnum = buildTermsEnum(numTerms, numPostings);
+    private void doTestEqQueriesAgainstStringIndex(Version version) throws Exception
+    {
+        final int numTerms = randomIntBetween(64, 512), numPostings = randomIntBetween(256, 1024);
+        final List<InvertedIndexBuilder.TermsEnum> termsEnum = buildTermsEnum(version, numTerms, numPostings);
 
-        try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(index, numTerms, numPostings, termsEnum))
+        try (IndexSearcher searcher = buildIndexAndOpenSearcher(numTerms, termsEnum))
         {
             for (int t = 0; t < numTerms; ++t)
             {
-                try (KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context))
+                try (KeyRangeIterator results = searcher.search(new Expression(indexContext)
+                        .add(Operator.EQ, termsEnum.get(t).originalTermBytes), null, new QueryContext(), false))
                 {
                     assertTrue(results.hasNext());
 
                     for (int p = 0; p < numPostings; ++p)
                     {
-                        final long expectedToken = termsEnum.get(t).right.get(p);
+                        final int expectedToken = termsEnum.get(t).postings.get(p);
                         assertTrue(results.hasNext());
                         final long actualToken = results.next().token().getLongValue();
                         assertEquals(expectedToken, actualToken);
@@ -126,19 +121,20 @@ public void testEqQueriesAgainstStringIndex() throws Exception
                     assertFalse(results.hasNext());
                 }
 
-                try (KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context))
+                try (KeyRangeIterator results = searcher.search(new Expression(indexContext)
+                        .add(Operator.EQ, termsEnum.get(t).originalTermBytes), null, new QueryContext(), false))
                 {
                     assertTrue(results.hasNext());
 
                     // test skipping to the last block
                     final int idxToSkip = numPostings - 7;
                     // tokens are equal to their corresponding row IDs
-                    final long tokenToSkip = termsEnum.get(t).right.get(idxToSkip);
-                    results.skipTo(SAITester.TEST_FACTORY.create(new Murmur3Partitioner.LongToken(tokenToSkip)));
+                    final long tokenToSkip = termsEnum.get(t).postings.get(idxToSkip);
+                    results.skipTo(SAITester.TEST_FACTORY.createTokenOnly(new Murmur3Partitioner.LongToken(tokenToSkip)));
 
                     for (int p = idxToSkip; p < numPostings; ++p)
                     {
-                        final long expectedToken = termsEnum.get(t).right.get(p);
+                        final long expectedToken = termsEnum.get(t).postings.get(p);
                         final long actualToken = results.next().token().getLongValue();
                         assertEquals(expectedToken, actualToken);
                     }
@@ -147,11 +143,13 @@ public void testEqQueriesAgainstStringIndex() throws Exception
 
             // try searching for terms that weren't indexed
             final String tooLongTerm = randomSimpleString(10, 12);
-            KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, UTF8Type.instance.decompose(tooLongTerm)), null, context);
+            KeyRangeIterator results = searcher.search(new Expression(indexContext)
+                                                    .add(Operator.EQ, UTF8Type.instance.decompose(tooLongTerm)), null, new QueryContext(), false);
             assertFalse(results.hasNext());
 
             final String tooShortTerm = randomSimpleString(1, 2);
-            results = searcher.search(Expression.create(index).add(Operator.EQ, UTF8Type.instance.decompose(tooShortTerm)), null, context);
+            results = searcher.search(new Expression(indexContext)
+                                      .add(Operator.EQ, UTF8Type.instance.decompose(tooShortTerm)), null, new QueryContext(), false);
             assertFalse(results.hasNext());
         }
     }
@@ -159,15 +157,13 @@ public void testEqQueriesAgainstStringIndex() throws Exception
     @Test
     public void testUnsupportedOperator() throws Exception
     {
-        QueryContext context = mock(QueryContext.class);
-        final StorageAttachedIndex index = createMockIndex(UTF8Type.instance);
+        final int numTerms = randomIntBetween(5, 15), numPostings = randomIntBetween(5, 20);
+        final List<InvertedIndexBuilder.TermsEnum> termsEnum = buildTermsEnum(version, numTerms, numPostings);
 
-        final int numTerms = getRandom().nextIntBetween(5, 15), numPostings = getRandom().nextIntBetween(5, 20);
-        final List<Pair<ByteComparable, LongArrayList>> termsEnum = buildTermsEnum(numTerms, numPostings);
-
-        try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(index, numTerms, numPostings, termsEnum))
+        try (IndexSearcher searcher = buildIndexAndOpenSearcher(numTerms, termsEnum))
         {
-            searcher.search(Expression.create(index).add(Operator.GT, UTF8Type.instance.decompose("a")), null, context);
+            searcher.search(new Expression(indexContext)
+                            .add(Operator.NEQ, UTF8Type.instance.decompose("a")), null, new QueryContext(), false);
 
             fail("Expect IllegalArgumentException thrown, but didn't");
         }
@@ -177,46 +173,63 @@ public void testUnsupportedOperator() throws Exception
         }
     }
 
-    private IndexSegmentSearcher buildIndexAndOpenSearcher(StorageAttachedIndex index,
-                                                           int terms,
-                                                           int postings,
-                                                           List<Pair<ByteComparable, LongArrayList>> termsEnum) throws IOException
+    private IndexSearcher buildIndexAndOpenSearcher(int terms, List<InvertedIndexBuilder.TermsEnum> termsEnum) throws IOException
     {
-        final int size = terms * postings;
         final IndexDescriptor indexDescriptor = newIndexDescriptor();
+        final String index = newIndex();
+        final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
+
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        SegmentMetadataBuilder metadataBuilder = new SegmentMetadataBuilder(0, components);
+        metadataBuilder.setRowIdRange(0, Long.MAX_VALUE);
+        metadataBuilder.setKeyRange(SAITester.TEST_FACTORY.createTokenOnly(DatabaseDescriptor.getPartitioner().getMinimumToken()),
+                                    SAITester.TEST_FACTORY.createTokenOnly(DatabaseDescriptor.getPartitioner().getMaximumToken()));
+        metadataBuilder.setTermRange(termsEnum.get(0).originalTermBytes,
+                                     termsEnum.get(terms - 1).originalTermBytes);
+
+        try (InvertedIndexWriter writer = new InvertedIndexWriter(components))
+        {
+            var iter = termsEnum.stream().map(InvertedIndexBuilder::toTermWithFrequency).iterator();
+            Int2IntHashMap docLengths = createMockDocLengths(termsEnum);
+            MemtableTermsIterator termsIterator = new MemtableTermsIterator(null, null, iter);
+            SegmentMetadata.ComponentMetadataMap indexMetas = writer.writeAll(metadataBuilder.intercept(termsIterator), docLengths);
+            metadataBuilder.setComponentsMetadata(indexMetas);
+        }
+
+        final SegmentMetadata segmentMetadata = metadataBuilder.build();
 
-        SegmentMetadata.ComponentMetadataMap indexMetas;
-        LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, index.identifier());
-        indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator()));
-
-        final SegmentMetadata segmentMetadata = new SegmentMetadata(0,
-                                                                    size,
-                                                                    0,
-                                                                    Long.MAX_VALUE,
-                                                                    SAITester.TEST_FACTORY.create(DatabaseDescriptor.getPartitioner().getMinimumToken()),
-                                                                    SAITester.TEST_FACTORY.create(DatabaseDescriptor.getPartitioner().getMaximumToken()),
-                                                                    wrap(termsEnum.get(0).left),
-                                                                    wrap(termsEnum.get(terms - 1).left),
-                                                                    indexMetas);
-
-        try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, index.termType(), index.identifier()))
+        try (PerIndexFiles indexFiles = new PerIndexFiles(components))
         {
-            final IndexSegmentSearcher searcher = IndexSegmentSearcher.open(TEST_PRIMARY_KEY_MAP_FACTORY,
-                                                                            indexFiles,
-                                                                            segmentMetadata,
-                                                                            index);
-            assertThat(searcher, is(instanceOf(LiteralIndexSegmentSearcher.class)));
+            SSTableContext sstableContext = mock(SSTableContext.class);
+            when(sstableContext.primaryKeyMapFactory()).thenReturn(KDTreeIndexBuilder.TEST_PRIMARY_KEY_MAP_FACTORY);
+            when(sstableContext.usedPerSSTableComponents()).thenReturn(indexDescriptor.perSSTableComponents());
+            final IndexSearcher searcher = version.onDiskFormat().newIndexSearcher(sstableContext,
+                                                                                   indexContext,
+                                                                                   indexFiles,
+                                                                                   segmentMetadata);
+            assertThat(searcher, is(instanceOf(InvertedIndexSearcher.class)));
             return searcher;
         }
     }
 
-    private List<Pair<ByteComparable, LongArrayList>> buildTermsEnum(int terms, int postings)
+    private List<InvertedIndexBuilder.TermsEnum> buildTermsEnum(Version version, int terms, int postings)
     {
-        return InvertedIndexBuilder.buildStringTermsEnum(terms, postings, () -> randomSimpleString(3, 5), () -> nextInt(0, Integer.MAX_VALUE));
+        return InvertedIndexBuilder.buildStringTermsEnum(version, terms, postings, () -> randomSimpleString(3, 5), () -> nextInt(0, Integer.MAX_VALUE));
+    }
+
+    private Int2IntHashMap createMockDocLengths(List<InvertedIndexBuilder.TermsEnum> termsEnum)
+    {
+        Int2IntHashMap docLengths = new Int2IntHashMap(Integer.MIN_VALUE);
+        for (InvertedIndexBuilder.TermsEnum term : termsEnum)
+        {
+            for (var cursor : term.postings)
+                docLengths.put(cursor.value, 1);
+        }
+        return docLengths;
     }
 
     private ByteBuffer wrap(ByteComparable bc)
     {
-        return ByteBuffer.wrap(ByteSourceInverse.readBytes(ByteSourceInverse.unescape(ByteSource.peekable(bc.asComparableBytes(ByteComparable.Version.OSS50)))));
+        return ByteBuffer.wrap(ByteSourceInverse.readBytes(bc.asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION)));
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/KDTreeIndexSearcherTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/KDTreeIndexSearcherTest.java
new file mode 100644
index 000000000000..65242e319a37
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/KDTreeIndexSearcherTest.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.NumberType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.KDTreeIndexBuilder;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+
+public class KDTreeIndexSearcherTest extends SaiRandomizedTest
+{
+    private static final short EQ_TEST_LOWER_BOUND_INCLUSIVE = 0;
+    private static final short EQ_TEST_UPPER_BOUND_EXCLUSIVE = 3;
+
+    private static final short RANGE_TEST_LOWER_BOUND_INCLUSIVE = 0;
+    private static final short RANGE_TEST_UPPER_BOUND_EXCLUSIVE = 10;
+
+    public static final int LIMIT = Integer.MAX_VALUE;
+
+    @Test
+    public void testRangeQueriesAgainstInt32Index() throws Exception
+    {
+        doTestRangeQueriesAgainstInt32Index();
+    }
+
+    private void doTestRangeQueriesAgainstInt32Index() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildInt32Searcher(newIndexDescriptor(), 0, 10);
+        testRangeQueries(indexSearcher, Int32Type.instance, Int32Type.instance, Integer::valueOf);
+    }
+
+    @Test
+    public void testEqQueriesAgainstInt32Index() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildInt32Searcher(newIndexDescriptor(),
+                                                                            EQ_TEST_LOWER_BOUND_INCLUSIVE, EQ_TEST_UPPER_BOUND_EXCLUSIVE);
+        testEqQueries(indexSearcher, Int32Type.instance, Int32Type.instance, Integer::valueOf);
+    }
+
+    @Test
+    public void testRangeQueriesAgainstLongIndex() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildLongSearcher(newIndexDescriptor(), 0, 10);
+        testRangeQueries(indexSearcher, LongType.instance, Int32Type.instance, Long::valueOf);
+    }
+
+    @Test
+    public void testEqQueriesAgainstLongIndex() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildLongSearcher(newIndexDescriptor(),
+                                                                           EQ_TEST_LOWER_BOUND_INCLUSIVE, EQ_TEST_UPPER_BOUND_EXCLUSIVE);
+        testEqQueries(indexSearcher, LongType.instance, Int32Type.instance, Long::valueOf);
+    }
+
+    @Test
+    public void testRangeQueriesAgainstShortIndex() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildShortSearcher(newIndexDescriptor(), (short) 0, (short) 10);
+        testRangeQueries(indexSearcher, ShortType.instance, Int32Type.instance, Function.identity());
+    }
+
+    @Test
+    public void testEqQueriesAgainstShortIndex() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildShortSearcher(newIndexDescriptor(),
+                                                                            EQ_TEST_LOWER_BOUND_INCLUSIVE, EQ_TEST_UPPER_BOUND_EXCLUSIVE);
+        testEqQueries(indexSearcher, ShortType.instance, Int32Type.instance, Function.identity());
+    }
+
+    @Test
+    public void testRangeQueriesAgainstDecimalIndex() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildDecimalSearcher(newIndexDescriptor(),
+                                                                              BigDecimal.ZERO, BigDecimal.valueOf(10L));
+        testRangeQueries(indexSearcher, DecimalType.instance, DecimalType.instance, BigDecimal::valueOf,
+                         getLongsOnInterval(21L, 70L));
+    }
+
+    private List<Long> getLongsOnInterval(long lowerInclusive, long upperInclusive)
+    {
+        return LongStream.range(lowerInclusive, upperInclusive + 1L).boxed().collect(Collectors.toList());
+    }
+
+    @Test
+    public void testEqQueriesAgainstDecimalIndex() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildDecimalSearcher(newIndexDescriptor(),
+                                                                              BigDecimal.valueOf(EQ_TEST_LOWER_BOUND_INCLUSIVE), BigDecimal.valueOf(EQ_TEST_UPPER_BOUND_EXCLUSIVE));
+        testEqQueries(indexSearcher, DecimalType.instance, DecimalType.instance, BigDecimal::valueOf);
+    }
+
+
+    @Test
+    public void testEqQueriesAgainstBigIntegerIndex() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildBigIntegerSearcher(newIndexDescriptor(),
+                                                                                 BigInteger.valueOf(EQ_TEST_LOWER_BOUND_INCLUSIVE), BigInteger.valueOf(EQ_TEST_UPPER_BOUND_EXCLUSIVE));
+        testEqQueries(indexSearcher, IntegerType.instance, IntegerType.instance, BigInteger::valueOf);
+    }
+
+    @Test
+    public void testRangeQueriesAgainstBigIntegerIndex() throws Exception
+    {
+        IndexSearcher indexSearcher = KDTreeIndexBuilder.buildBigIntegerSearcher(newIndexDescriptor(),
+                                                                                 BigInteger.ZERO, BigInteger.valueOf(10L));
+        testRangeQueries(indexSearcher, IntegerType.instance, IntegerType.instance, BigInteger::valueOf);
+    }
+
+
+    @Test
+    public void testUnsupportedOperator() throws Exception
+    {
+        final IndexSearcher indexSearcher = KDTreeIndexBuilder.buildShortSearcher(newIndexDescriptor(), (short) 0, (short) 3);
+        try
+        {
+            indexSearcher.search(new Expression(SAITester.createIndexContext("meh", ShortType.instance))
+            {{
+                operation = Op.NOT_EQ;
+                lower = upper = new Bound(ShortType.instance.decompose((short) 0), Int32Type.instance, true);
+            }}, null, new QueryContext(), false);
+
+            fail("Expect IllegalArgumentException thrown, but didn't");
+        }
+        catch (IllegalArgumentException e)
+        {
+            // expected
+        }
+    }
+
+    private <T extends Number> void testEqQueries(final IndexSearcher indexSearcher,
+                                                  final NumberType<T> rawType, final NumberType<?> encodedType,
+                                                  final Function<Short, T> rawValueProducer) throws Exception
+    {
+        try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
+        {{
+            operation = Op.EQ;
+            lower = upper = new Bound(rawType.decompose(rawValueProducer.apply(EQ_TEST_LOWER_BOUND_INCLUSIVE)), encodedType, true);
+        }}, null, new QueryContext(), false))
+        {
+            assertTrue(results.hasNext());
+
+            assertEquals(0L, results.next().token().getLongValue());
+        }
+
+        try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
+        {{
+            operation = Op.EQ;
+            lower = upper = new Bound(rawType.decompose(rawValueProducer.apply(EQ_TEST_UPPER_BOUND_EXCLUSIVE)), encodedType, true);
+        }}, null, new QueryContext(), false))
+        {
+            assertFalse(results.hasNext());
+            indexSearcher.close();
+        }
+    }
+
+    private <T extends Number> void testRangeQueries(final IndexSearcher indexSearcher,
+                                                     final NumberType<T> rawType, final NumberType<?> encodedType,
+                                                     final Function<Short, T> rawValueProducer) throws Exception
+    {
+        List<Long> expectedTokenList = getLongsOnInterval(3L, 7L);
+        testRangeQueries(indexSearcher, rawType, encodedType, rawValueProducer, expectedTokenList);
+    }
+
+
+    private <T extends Number> void testRangeQueries(final IndexSearcher indexSearcher,
+                                                     final NumberType<T> rawType, final NumberType<?> encodedType,
+                                                     final Function<Short, T> rawValueProducer, List<Long> expectedTokenList) throws Exception
+    {
+        try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
+        {{
+            operation = Op.RANGE;
+
+            lower = new Bound(rawType.decompose(rawValueProducer.apply((short)2)), encodedType, false);
+            upper = new Bound(rawType.decompose(rawValueProducer.apply((short)7)), encodedType, true);
+        }}, null, new QueryContext(), false))
+        {
+            assertTrue(results.hasNext());
+
+            List<Long> actualTokenList = Lists.newArrayList(Iterators.transform(results, key -> key.token().getLongValue()));
+            assertEquals(expectedTokenList, actualTokenList);
+        }
+
+        try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
+        {{
+            operation = Op.RANGE;
+            lower = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_UPPER_BOUND_EXCLUSIVE)), encodedType, true);
+        }}, null, new QueryContext(), false))
+        {
+            assertFalse(results.hasNext());
+        }
+
+        try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
+        {{
+            operation = Op.RANGE;
+            upper = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_LOWER_BOUND_INCLUSIVE)), encodedType, false);
+        }}, null, new QueryContext(), false))
+        {
+            assertFalse(results.hasNext());
+            indexSearcher.close();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/LegacyOnDiskFormatTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/LegacyOnDiskFormatTest.java
new file mode 100644
index 000000000000..49e674e99bd2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/LegacyOnDiskFormatTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.FileUtils;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.QueryEventListeners;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.BKDReader;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.index.sai.disk.v1.kdtree.BKDQueries.bkdQueryFrom;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Note: The sstables and SAI indexes used in this test were written with DSE 6.8
+ * in order to guarantee the correctness of the V1 on-disk format code.
+ */
+public class LegacyOnDiskFormatTest
+{
+    private TemporaryFolder temporaryFolder = new TemporaryFolder();
+    private Descriptor descriptor;
+    private TableMetadata tableMetadata;
+    private IndexDescriptor indexDescriptor;
+    private SSTableReader sstable;
+    private PrimaryKey.Factory pkFactory;
+
+    private IndexContext intContext = SAITester.createIndexContext("int_index", Int32Type.instance);
+    private IndexContext textContext = SAITester.createIndexContext("text_index", UTF8Type.instance);
+
+    @BeforeClass
+    public static void initialise()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
+    }
+
+    @Before
+    public void setup() throws Throwable
+    {
+        temporaryFolder.create();
+        descriptor = Descriptor.fromFile(new File(temporaryFolder.newFolder().getAbsolutePath() + "/bb-1-bti-Data.db"));
+        FileUtils.copySSTablesAndIndexes(descriptor, "aa");
+        tableMetadata = TableMetadata.builder("test", "test")
+                                     .addPartitionKeyColumn("pk", Int32Type.instance)
+                                     .addRegularColumn("int_value", Int32Type.instance)
+                                     .addRegularColumn("text_value", UTF8Type.instance)
+                                     .build();
+        sstable = SSTableReader.openNoValidation(null, descriptor, TableMetadataRef.forOfflineTools(tableMetadata));
+        indexDescriptor = IndexDescriptor.empty(sstable.descriptor).reload(sstable, Set.of(intContext, textContext));
+        pkFactory = indexDescriptor.perSSTableComponents().version().onDiskFormat().newPrimaryKeyFactory(tableMetadata.comparator);
+    }
+
+    @After
+    public void teardown()
+    {
+        temporaryFolder.delete();
+    }
+
+    @Test
+    public void correctlyIdentifiesPerSSTableFileVersion()
+    {
+        assertEquals(Version.AA, indexDescriptor.perSSTableComponents().version());
+    }
+
+    @Test
+    public void canReadPerSSTableMetadata() throws Throwable
+    {
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        final MetadataSource source = MetadataSource.loadMetadata(components);
+
+        NumericValuesMeta numericValuesMeta = new NumericValuesMeta(source.get(components.get(IndexComponentType.OFFSETS_VALUES)));
+
+        assertEquals(100, numericValuesMeta.valueCount);
+
+        numericValuesMeta = new NumericValuesMeta(source.get(components.get(IndexComponentType.TOKEN_VALUES)));
+
+        assertEquals(100, numericValuesMeta.valueCount);
+    }
+
+    @Test
+    public void canReadPerIndexMetadata() throws Throwable
+    {
+        IndexComponents.ForRead components = indexDescriptor.perIndexComponents(intContext);
+        final MetadataSource source = MetadataSource.loadMetadata(components);
+
+        List<SegmentMetadata> metadatas = SegmentMetadata.load(source, intContext);
+
+        assertEquals(1, metadatas.size());
+        assertEquals(100, metadatas.get(0).numRows);
+    }
+
+    @Test
+    public void canCreateAndUsePrimaryKeyMapWithLegacyFormat() throws Throwable
+    {
+        var perSSTableComponents = indexDescriptor.perSSTableComponents();
+        PrimaryKeyMap.Factory primaryKeyMapFactory = perSSTableComponents.onDiskFormat().newPrimaryKeyMapFactory(perSSTableComponents, pkFactory, sstable);
+
+        long countFromFactory = primaryKeyMapFactory.count();
+
+        PrimaryKeyMap primaryKeyMap = primaryKeyMapFactory.newPerSSTablePrimaryKeyMap();
+
+        long countFromMap = primaryKeyMap.count();
+        assertEquals(countFromFactory, countFromMap);
+
+        PrimaryKey expected = pkFactory.createTokenOnly(Murmur3Partitioner.instance.decorateKey(Int32Type.instance.decompose(23)).getToken());
+
+        PrimaryKey primaryKey = primaryKeyMap.primaryKeyFromRowId(0);
+
+        assertEquals(expected, primaryKey);
+    }
+
+    @Test
+    public void canSearchBDKIndex() throws Throwable
+    {
+        IndexComponents.ForRead components = indexDescriptor.perIndexComponents(intContext);
+
+        final MetadataSource source = MetadataSource.loadMetadata(components);
+
+        List<SegmentMetadata> metadatas = SegmentMetadata.load(source, intContext);
+
+        BKDReader bkdReader = new BKDReader(intContext,
+                                            components.get(IndexComponentType.KD_TREE).createFileHandle(),
+                                            metadatas.get(0).getIndexRoot(IndexComponentType.KD_TREE),
+                                            components.get(IndexComponentType.KD_TREE_POSTING_LISTS).createFileHandle(),
+                                            metadatas.get(0).getIndexRoot(IndexComponentType.KD_TREE_POSTING_LISTS));
+
+        Expression expression = new Expression(intContext).add(Operator.LT, Int32Type.instance.decompose(10));
+        BKDReader.IntersectVisitor query = bkdQueryFrom(expression, bkdReader.getNumDimensions(), bkdReader.getBytesPerDimension());
+        PostingList postingList = bkdReader.intersect(query, QueryEventListeners.NO_OP_BKD_LISTENER, new QueryContext());
+        assertNotNull(postingList);
+    }
+
+    @Test
+    public void canSearchTermsIndex() throws Throwable
+    {
+        IndexComponents.ForRead components = indexDescriptor.perIndexComponents(textContext);
+
+        final MetadataSource source = MetadataSource.loadMetadata(components);
+
+        SegmentMetadata metadata = SegmentMetadata.load(source, textContext).get(0);
+
+        long root = metadata.getIndexRoot(IndexComponentType.TERMS_DATA);
+        Map<String,String> map = metadata.componentMetadatas.get(IndexComponentType.TERMS_DATA).attributes;
+        String footerPointerString = map.get(SAICodecUtils.FOOTER_POINTER);
+        long footerPointer = footerPointerString == null ? -1 : Long.parseLong(footerPointerString);
+
+        ByteComparable.Version byteComparableVersion = components.byteComparableVersionFor(IndexComponentType.TERMS_DATA);
+        TermsReader termsReader = new TermsReader(textContext,
+                                                  components.get(IndexComponentType.TERMS_DATA).createFileHandle(),
+                                                  byteComparableVersion,
+                                                  components.get(IndexComponentType.POSTING_LISTS).createFileHandle(),
+                                                  root,
+                                                  footerPointer,
+                                                  Version.AA); // These tests are for AA, so no need to parameterize
+        Expression expression = new Expression(textContext).add(Operator.EQ, UTF8Type.instance.decompose("10"));
+        ByteComparable term = ByteComparable.preencoded(byteComparableVersion, expression.lower.value.encoded);
+
+        PostingList result = termsReader.exactMatch(term, QueryEventListeners.NO_OP_TRIE_LISTENER, new QueryContext());
+
+        assertEquals(1, result.size());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java
index db8eb28b7b70..48ccba58696e 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java
@@ -29,41 +29,44 @@
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
+import org.apache.cassandra.index.sai.disk.io.IndexOutput;
 import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IndexInput;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-public class MetadataTest extends SAIRandomizedTester
+public class MetadataTest extends SaiRandomizedTest
 {
     @Rule
     public final ExpectedException expectedException = ExpectedException.none();
 
     private IndexDescriptor indexDescriptor;
-    private IndexIdentifier indexIdentifier;
+    private String index;
+    private IndexContext indexContext;
 
     @Before
     public void setup() throws Throwable
     {
         indexDescriptor = newIndexDescriptor();
-        indexIdentifier = createIndexIdentifier("test", "test", newIndex());
+        index = newIndex();
+        indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
     }
 
     @Test
     public void shouldReadWrittenMetadata() throws Exception
     {
         final Map<String, byte[]> data = new HashMap<>();
-        try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier)))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (MetadataWriter writer = new MetadataWriter(components))
         {
             int num = nextInt(1, 50);
             for (int x = 0; x < num; x++)
@@ -73,19 +76,22 @@ public void shouldReadWrittenMetadata() throws Exception
                 String name = UUID.randomUUID().toString();
 
                 data.put(name, bytes);
-                try (MetadataWriter.Builder builder = writer.builder(name))
+                try (IndexOutput builder = writer.builder(name))
                 {
                     builder.writeBytes(bytes, 0, bytes.length);
                 }
             }
         }
-        MetadataSource reader = MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier);
+        components.markComplete();
+
+        MetadataSource reader = MetadataSource.loadMetadata(indexDescriptor.perIndexComponents(indexContext));
 
         for (Map.Entry<String, byte[]> entry : data.entrySet())
         {
-            final DataInput input = reader.get(entry.getKey());
+            final IndexInput input = reader.get(entry.getKey());
             assertNotNull(input);
             final byte[] expectedBytes = entry.getValue();
+            assertEquals(expectedBytes.length, input.length());
             final byte[] actualBytes = new byte[expectedBytes.length];
             input.readBytes(actualBytes, 0, expectedBytes.length);
             assertArrayEquals(expectedBytes, actualBytes);
@@ -95,7 +101,8 @@ public void shouldReadWrittenMetadata() throws Exception
     @Test
     public void shouldFailWhenFileHasNoHeader() throws IOException
     {
-        try (IndexOutputWriter out = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (IndexOutputWriter out = components.addOrGet(IndexComponentType.META).openOutput())
         {
             final byte[] bytes = nextBytes(13, 29);
             out.writeBytes(bytes, bytes.length);
@@ -103,82 +110,81 @@ public void shouldFailWhenFileHasNoHeader() throws IOException
 
         expectedException.expect(CorruptIndexException.class);
         expectedException.expectMessage("codec header mismatch");
-        MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier);
+        MetadataSource.loadMetadata(components);
     }
 
     @Test
     public void shouldFailCrcCheckWhenFileIsTruncated() throws IOException
     {
-        try (IndexOutputWriter output = writeRandomBytes())
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        final IndexOutputWriter output = writeRandomBytes(components);
+
+        final File indexFile = output.getFile();
+        final long length = indexFile.length();
+        assertTrue(length > 0);
+        final File renamed = new File(temporaryFolder.newFile());
+        indexFile.move(renamed);
+        assertFalse(output.getFile().exists());
+
+        try (FileOutputStream outputStream = new FileOutputStream(output.getFile().toJavaIOFile());
+             RandomAccessFile input = new RandomAccessFile(renamed.toJavaIOFile(), "r"))
         {
-            File indexFile = output.getFile();
-            long length = indexFile.length();
-            assertTrue(length > 0);
-            File renamed = new File(temporaryFolder.newFile());
-            indexFile.move(renamed);
-            assertFalse(output.getFile().exists());
-
-            try (FileOutputStream outputStream = new FileOutputStream(output.getFile().toJavaIOFile());
-                 RandomAccessFile input = new RandomAccessFile(renamed.toJavaIOFile(), "r"))
-            {
-                // skip last byte when copying
-                copyTo(input, outputStream, Math.toIntExact(length - 1));
-            }
-
-            expectedException.expect(CorruptIndexException.class);
-            expectedException.expectMessage("misplaced codec footer (file truncated?)");
-            MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier);
+            // skip last byte when copying
+            FileUtils.copyTo(input, outputStream, Math.toIntExact(length - 1));
         }
+
+        expectedException.expect(CorruptIndexException.class);
+        expectedException.expectMessage("misplaced codec footer (file truncated?)");
+        MetadataSource.loadMetadata(components);
     }
 
     @Test
     public void shouldFailCrcCheckWhenFileIsCorrupted() throws IOException
     {
-        try (IndexOutputWriter output = writeRandomBytes())
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        final IndexOutputWriter output = writeRandomBytes(components);
+
+        final File indexFile = output.getFile();
+        final long length = indexFile.length();
+        assertTrue(length > 0);
+        final File renamed = new File(temporaryFolder.newFile());
+        indexFile.move(renamed);
+        assertFalse(output.getFile().exists());
+
+        try (FileOutputStream outputStream = new FileOutputStream(output.getFile().toJavaIOFile());
+             RandomAccessFile file = new RandomAccessFile(renamed.toJavaIOFile(), "r"))
         {
-            File indexFile = output.getFile();
-            long length = indexFile.length();
-            assertTrue(length > 0);
-            File renamed = new File(temporaryFolder.newFile());
-            indexFile.move(renamed);
-            assertFalse(output.getFile().exists());
-
-            try (FileOutputStream outputStream = new FileOutputStream(output.getFile().toJavaIOFile());
-                 RandomAccessFile file = new RandomAccessFile(renamed.toJavaIOFile(), "r"))
-            {
-                // copy most of the file untouched
-                final byte[] buffer = new byte[Math.toIntExact(length - 1 - CodecUtil.footerLength())];
-                file.read(buffer);
-                outputStream.write(buffer);
-
-                // corrupt a single byte at the end
-                final byte last = (byte) file.read();
-                outputStream.write(~last);
-
-                // copy footer
-                final byte[] footer = new byte[CodecUtil.footerLength()];
-                file.read(footer);
-                outputStream.write(footer);
-            }
-
-            expectedException.expect(CorruptIndexException.class);
-            expectedException.expectMessage("checksum failed");
-            MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier);
+            // copy most of the file untouched
+            final byte[] buffer = new byte[Math.toIntExact(length - 1 - CodecUtil.footerLength())];
+            file.read(buffer);
+            outputStream.write(buffer);
+
+            // corrupt a single byte at the end
+            final byte last = (byte) file.read();
+            outputStream.write(~last);
+
+            // copy footer
+            final byte[] footer = new byte[CodecUtil.footerLength()];
+            file.read(footer);
+            outputStream.write(footer);
         }
+
+        expectedException.expect(CorruptIndexException.class);
+        expectedException.expectMessage("checksum failed");
+        MetadataSource.loadMetadata(components);
     }
 
-    private IndexOutputWriter writeRandomBytes() throws IOException
+    private IndexOutputWriter writeRandomBytes(IndexComponents.ForWrite components) throws IOException
     {
-        final IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier);
-        try (MetadataWriter writer = new MetadataWriter(output))
+        try (MetadataWriter writer = new MetadataWriter(components))
         {
             byte[] bytes = nextBytes(11, 1024);
 
-            try (MetadataWriter.Builder builder = writer.builder("name"))
+            try (IndexOutput builder = writer.builder("name"))
             {
                 builder.writeBytes(bytes, 0, bytes.length);
             }
         }
-        return output;
+        return components.addOrGet(components.metadataComponent()).openOutput();
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/SAICodecUtilsTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/SAICodecUtilsTest.java
deleted file mode 100644
index 060eabb20667..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/SAICodecUtilsTest.java
+++ /dev/null
@@ -1,391 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
-import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.cassandra.io.util.File;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.mockito.Mockito;
-
-import static org.apache.lucene.codecs.CodecUtil.CODEC_MAGIC;
-import static org.apache.lucene.codecs.CodecUtil.FOOTER_MAGIC;
-import static org.apache.lucene.codecs.CodecUtil.writeBEInt;
-import static org.apache.lucene.codecs.CodecUtil.writeBELong;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.when;
-
-public class SAICodecUtilsTest extends SAIRandomizedTester
-{
-    private File file;
-
-    @Before
-    public void createFile() throws Exception
-    {
-        file = new File(temporaryFolder.newFile());
-    }
-
-    @Test
-    public void checkHeaderDoesNotFailWithValidHeader() throws Exception
-    {
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file))
-        {
-            SAICodecUtils.checkHeader(input);
-        }
-    }
-
-    @Test
-    public void checkHeaderFailsOnInvalidMagicValue() throws Exception
-    {
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            writeBEInt(writer, 1234);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file))
-        {
-            assertThatThrownBy(() -> SAICodecUtils.checkHeader(input))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("codec header mismatch: actual header=1234 vs expected header=" + CODEC_MAGIC);
-        }
-    }
-
-    @Test
-    public void checkHeaderFailsOnInvalidVersion() throws Exception
-    {
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            writeBEInt(writer, CODEC_MAGIC);
-            writer.writeString("zz");
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file))
-        {
-            assertThatThrownBy(() -> SAICodecUtils.checkHeader(input))
-            .isInstanceOf(IllegalArgumentException.class)
-            .hasMessageContaining("The version string zz does not represent a valid SAI version. It should be one of aa");
-        }
-    }
-
-    @Test
-    public void checkFooterDoesNotFailWithValidFooter() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-            SAICodecUtils.writeFooter(writer);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file);
-             ChecksumIndexInput checksumIndexInput = IndexFileUtils.getBufferedChecksumIndexInput(input))
-        {
-            SAICodecUtils.checkHeader(checksumIndexInput);
-            for (int value = 0; value < numBytes; value++)
-                checksumIndexInput.readByte();
-            SAICodecUtils.checkFooter(checksumIndexInput);
-        }
-    }
-
-    @Test
-    public void checkFooterFailsWithMissingFooter() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file);
-             ChecksumIndexInput checksumIndexInput = IndexFileUtils.getBufferedChecksumIndexInput(input))
-        {
-            SAICodecUtils.checkHeader(checksumIndexInput);
-            for (int value = 0; value < numBytes; value++)
-                checksumIndexInput.readByte();
-            assertThatThrownBy(() -> SAICodecUtils.checkFooter(checksumIndexInput))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("misplaced codec footer (file truncated?): remaining=0, expected=16");
-        }
-    }
-
-    @Test
-    public void checkFooterFailsWithExtendedFooter() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-            SAICodecUtils.writeFooter(writer);
-            writeBEInt(writer, 1234);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file);
-             ChecksumIndexInput checksumIndexInput = IndexFileUtils.getBufferedChecksumIndexInput(input))
-        {
-            SAICodecUtils.checkHeader(checksumIndexInput);
-            for (int value = 0; value < numBytes; value++)
-                checksumIndexInput.readByte();
-            assertThatThrownBy(() -> SAICodecUtils.checkFooter(checksumIndexInput))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("misplaced codec footer (file extended?): remaining=20, expected=16");
-        }
-    }
-
-    @Test
-    public void checkFooterFailsWithInvalidFooter() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-
-            writeBEInt(writer, 1234);
-            writeBEInt(writer, 0);
-            writeBELong(writer, writer.getChecksum());
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file);
-             ChecksumIndexInput checksumIndexInput = IndexFileUtils.getBufferedChecksumIndexInput(input))
-        {
-            SAICodecUtils.checkHeader(checksumIndexInput);
-            for (int value = 0; value < numBytes; value++)
-                checksumIndexInput.readByte();
-            assertThatThrownBy(() -> SAICodecUtils.checkFooter(checksumIndexInput))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("codec footer mismatch (file truncated?): actual footer=1234 vs expected footer=-1071082520");
-        }
-    }
-
-    @Test
-    public void checkFooterFailsWithInvalidAlgorithmId() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-
-            writeBEInt(writer, FOOTER_MAGIC);
-            writeBEInt(writer, 1);
-            writeBELong(writer, writer.getChecksum());
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file);
-             ChecksumIndexInput checksumIndexInput = IndexFileUtils.getBufferedChecksumIndexInput(input))
-        {
-            SAICodecUtils.checkHeader(checksumIndexInput);
-            for (int value = 0; value < numBytes; value++)
-                checksumIndexInput.readByte();
-            assertThatThrownBy(() -> SAICodecUtils.checkFooter(checksumIndexInput))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("codec footer mismatch: unknown algorithmID: 1 ");
-        }
-    }
-
-    @Test
-    public void checkFooterFailsWithInvalidChecksum() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-
-            writeBEInt(writer, FOOTER_MAGIC);
-            writeBEInt(writer, 0);
-            writeBELong(writer, 0);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file);
-             ChecksumIndexInput checksumIndexInput = IndexFileUtils.getBufferedChecksumIndexInput(input))
-        {
-            SAICodecUtils.checkHeader(checksumIndexInput);
-            for (int value = 0; value < numBytes; value++)
-                checksumIndexInput.readByte();
-            assertThatThrownBy(() -> SAICodecUtils.checkFooter(checksumIndexInput))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("checksum failed (hardware problem?) : expected=0 actual=");
-        }
-    }
-
-    @Test
-    public void checkFooterFailsWithIllegalChecksum() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-
-            writeBEInt(writer, FOOTER_MAGIC);
-            writeBEInt(writer, 0);
-            writeBELong(writer, 0xFFFFFFFF00000000L);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file);
-             ChecksumIndexInput checksumIndexInput = IndexFileUtils.getBufferedChecksumIndexInput(input))
-        {
-            SAICodecUtils.checkHeader(checksumIndexInput);
-            for (int value = 0; value < numBytes; value++)
-                checksumIndexInput.readByte();
-            assertThatThrownBy(() -> SAICodecUtils.checkFooter(checksumIndexInput))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("Illegal checksum: -4294967296 ");
-        }
-    }
-
-    @Test
-    public void validateFooterAndResetPositionFailsWithShortFile() throws Exception
-    {
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file))
-        {
-            assertThatThrownBy(() -> SAICodecUtils.validateFooterAndResetPosition(input))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("invalid codec footer (file truncated?): file length=7, footer length=16 ");
-        }
-    }
-
-    @Test
-    public void validateChecksumFailsWithInvalidChecksum() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-
-            writeBEInt(writer, FOOTER_MAGIC);
-            writeBEInt(writer, 0);
-            writeBELong(writer, 0);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file))
-        {
-            SAICodecUtils.checkHeader(input);
-            for (int value = 0; value < numBytes; value++)
-                input.readByte();
-            assertThatThrownBy(() -> SAICodecUtils.validateChecksum(input))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("checksum failed (hardware problem?) : expected=0 actual=");
-        }
-    }
-
-    @Test
-    public void validateChecksumFailsWithIllegalChecksum() throws Exception
-    {
-        int numBytes = nextInt(1000, 10000);
-        try (IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file))
-        {
-            SAICodecUtils.writeHeader(writer);
-            for (int value = 0; value < numBytes; value++)
-                writer.writeByte(getRandom().nextByte());
-
-            writeBEInt(writer, FOOTER_MAGIC);
-            writeBEInt(writer, 0);
-            writeBELong(writer, 0xFFFFFFFF00000000L);
-        }
-
-        try (IndexInput input = IndexFileUtils.instance.openBlockingInput(file))
-        {
-            SAICodecUtils.checkHeader(input);
-            for (int value = 0; value < numBytes; value++)
-                input.readByte();
-            assertThatThrownBy(() -> SAICodecUtils.validateChecksum(input))
-            .isInstanceOf(CorruptIndexException.class)
-            .hasMessageContaining("Illegal checksum: -4294967296 ");
-        }
-    }
-
-    @Test
-    public void writeCRCFailsWithInvalidCRC() throws Exception
-    {
-        IndexOutput indexOutput = Mockito.mock(IndexOutput.class);
-        when(indexOutput.getChecksum()).thenReturn(0xFFFFFFFF00000000L);
-        assertThatThrownBy(() -> SAICodecUtils.writeFooter(indexOutput))
-        .isInstanceOf(IllegalStateException.class)
-        .hasMessageContaining("Illegal checksum: -4294967296 ");
-    }
-
-    @Test
-    public void checkBlockSizeTest()
-    {
-        // Block size must be within min and max
-        assertThatThrownBy(() -> SAICodecUtils.checkBlockSize(1024, 32, 512))
-        .isInstanceOf(IllegalArgumentException.class)
-        .hasMessageContaining("blockSize must be >= 32 and <= 512, got 1024");
-
-        // Block size must be power of 2
-        assertThatThrownBy(() -> SAICodecUtils.checkBlockSize(99, 32, 512))
-        .isInstanceOf(IllegalArgumentException.class)
-        .hasMessageContaining("blockSize must be a power of two, got 99");
-
-        // Check block shift size is correct
-        assertEquals(6, SAICodecUtils.checkBlockSize(64, 32, 512));
-        assertEquals(7, SAICodecUtils.checkBlockSize(128, 32, 512));
-        assertEquals(8, SAICodecUtils.checkBlockSize(256, 32, 512));
-    }
-
-    @Test
-    public void numBlocksTest()
-    {
-        assertEquals(0, SAICodecUtils.numBlocks(0, 512));
-        assertEquals(1, SAICodecUtils.numBlocks(500, 512));
-        assertEquals(2, SAICodecUtils.numBlocks(1000, 512));
-        assertEquals(3, SAICodecUtils.numBlocks(1025, 512));
-
-        assertThatThrownBy(() -> SAICodecUtils.numBlocks(-1, 512))
-        .isInstanceOf(IllegalArgumentException.class)
-        .hasMessageContaining("size cannot be negative");
-
-        // This will integer overflow the number of blocks and result in an error
-        assertThatThrownBy(() -> SAICodecUtils.numBlocks((long)Integer.MAX_VALUE * 512L + 1L, 512))
-        .isInstanceOf(IllegalArgumentException.class)
-        .hasMessageContaining("size is too large for this block size");
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java
index f0d1eae642f0..dcbb2169afeb 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java
@@ -22,62 +22,93 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
+import com.google.common.base.Preconditions;
 import com.google.common.base.Stopwatch;
 import org.junit.After;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.rows.BTreeRow;
 import org.apache.cassandra.db.rows.BufferCell;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.TermsIterator;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
 import static org.apache.cassandra.Util.dk;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
+//TODO This test needs rethinking because we always now end up with a single segment after a flush
+// and we are not restricted to Integer.MAX_VALUE in the segments
+@RunWith(Parameterized.class)
 public class SegmentFlushTest
 {
     private static long segmentRowIdOffset;
-    private static int posting1;
-    private static int posting2;
+    private static int minSegmentRowId;
+    private static int maxSegmentRowId;
     private static PrimaryKey minKey;
     private static PrimaryKey maxKey;
     private static ByteBuffer minTerm;
     private static ByteBuffer maxTerm;
-    private static int numRows;
+    private static int numRowsPerSegment;
+
+    @Parameterized.Parameter
+    public Version version;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static Collection<Object[]> data()
+    {
+        // Required because it configures SEGMENT_BUILD_MEMORY_LIMIT, which is needed for Version.AA
+        if (DatabaseDescriptor.getRawConfig() == null)
+            DatabaseDescriptor.setConfig(DatabaseDescriptor.loadConfig());
+        return Version.ALL.stream().map(v -> new Object[]{v}).collect(Collectors.toList());
+    }
 
     @BeforeClass
     public static void init()
     {
-        DatabaseDescriptor.toolInitialization();
-        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
-        StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Before
+    public void setVersion()
+    {
+        SAIUtil.setLatestVersion(version);
     }
 
     @After
@@ -91,10 +122,10 @@ public void testFlushBetweenRowIds() throws Exception
     {
         // exceeds max rowId per segment
         testFlushBetweenRowIds(0, Integer.MAX_VALUE, 2);
-        testFlushBetweenRowIds(0, Long.MAX_VALUE - 1, 2);
+        testFlushBetweenRowIds(0, Long.MAX_VALUE, 2);
         testFlushBetweenRowIds(0, SegmentBuilder.LAST_VALID_SEGMENT_ROW_ID + 1, 2);
-        testFlushBetweenRowIds(Integer.MAX_VALUE - SegmentBuilder.LAST_VALID_SEGMENT_ROW_ID - 1, Integer.MAX_VALUE - 1, 1);
-        testFlushBetweenRowIds(Long.MAX_VALUE - SegmentBuilder.LAST_VALID_SEGMENT_ROW_ID - 1, Long.MAX_VALUE - 1, 1);
+        testFlushBetweenRowIds(Integer.MAX_VALUE - SegmentBuilder.LAST_VALID_SEGMENT_ROW_ID - 1, Integer.MAX_VALUE, 2);
+        testFlushBetweenRowIds(Long.MAX_VALUE - SegmentBuilder.LAST_VALID_SEGMENT_ROW_ID - 1, Long.MAX_VALUE, 2);
     }
 
     @Test
@@ -108,15 +139,23 @@ public void testNoFlushBetweenRowIds() throws Exception
     private void testFlushBetweenRowIds(long sstableRowId1, long sstableRowId2, int segments) throws Exception
     {
         Path tmpDir = Files.createTempDirectory("SegmentFlushTest");
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(new Descriptor(new File(tmpDir.toFile()), "ks", "cf", new SequenceBasedSSTableId(1)),
-                                                                 Murmur3Partitioner.instance,
-                                                                 SAITester.EMPTY_COMPARATOR);
+        IndexDescriptor indexDescriptor = IndexDescriptor.empty(new Descriptor(new File(tmpDir.toFile()), "ks", "cf", new SequenceBasedSSTableId(1)));
 
         ColumnMetadata column = ColumnMetadata.regularColumn("sai", "internal", "column", UTF8Type.instance);
+        IndexMetadata config = IndexMetadata.fromSchemaMetadata("index_name", IndexMetadata.Kind.CUSTOM, null);
 
-        StorageAttachedIndex index = SAITester.createMockIndex(column);
+        IndexContext indexContext = new IndexContext("ks",
+                                                     "cf",
+                                                     TableId.generate(),
+                                                     UTF8Type.instance,
+                                                     new ClusteringComparator(),
+                                                     column,
+                                                     IndexTarget.Type.SIMPLE,
+                                                     config,
+                                                     MockSchema.newCFS("ks"));
 
-        SSTableIndexWriter writer = new SSTableIndexWriter(indexDescriptor, index, V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER, () -> true);
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        SSTableIndexWriter writer = new SSTableIndexWriter(components, V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER, () -> true, 2);
 
         List<DecoratedKey> keys = Arrays.asList(dk("1"), dk("2"));
         Collections.sort(keys);
@@ -124,88 +163,103 @@ private void testFlushBetweenRowIds(long sstableRowId1, long sstableRowId2, int
         DecoratedKey key1 = keys.get(0);
         ByteBuffer term1 = UTF8Type.instance.decompose("a");
         Row row1 = createRow(column, term1);
-        writer.addRow(SAITester.TEST_FACTORY.create(key1), row1, sstableRowId1);
+        writer.addRow(SAITester.TEST_FACTORY.create(key1, Clustering.EMPTY), row1, sstableRowId1);
 
         // expect a flush if exceed max rowId per segment
         DecoratedKey key2 = keys.get(1);
         ByteBuffer term2 = UTF8Type.instance.decompose("b");
         Row row2 = createRow(column, term2);
-        writer.addRow(SAITester.TEST_FACTORY.create(key2), row2, sstableRowId2);
+        writer.addRow(SAITester.TEST_FACTORY.create(key2, Clustering.EMPTY), row2, sstableRowId2);
 
         writer.complete(Stopwatch.createStarted());
 
-        MetadataSource source = MetadataSource.loadColumnMetadata(indexDescriptor, index.identifier());
+        MetadataSource source = MetadataSource.loadMetadata(components);
 
-        List<SegmentMetadata> segmentMetadatas = SegmentMetadata.load(source, indexDescriptor.primaryKeyFactory);
+        // verify segment count
+        List<SegmentMetadata> segmentMetadatas = SegmentMetadata.load(source, indexContext);
         assertEquals(segments, segmentMetadatas.size());
 
         // verify segment metadata
         SegmentMetadata segmentMetadata = segmentMetadatas.get(0);
-        segmentRowIdOffset = sstableRowId1;
-        posting1 = 0;
-        posting2 = segments == 1 ? (int) (sstableRowId2 - segmentRowIdOffset) : 0;
-        minKey = SAITester.TEST_FACTORY.create(key1.getToken());
-        maxKey = segments == 1 ? SAITester.TEST_FACTORY.create(key2.getToken()) : minKey;
+        segmentRowIdOffset = sstableRowId1; // segmentRowIdOffset is the first sstable row id
+        minSegmentRowId = 0;
+        maxSegmentRowId = segments == 1 ? (int) (sstableRowId2 - segmentRowIdOffset) : 0;
+        minKey = SAITester.TEST_FACTORY.createTokenOnly(key1.getToken());
+        maxKey = SAITester.TEST_FACTORY.createTokenOnly(segments == 1 ? key2.getToken() : key1.getToken());
         minTerm = term1;
         maxTerm = segments == 1 ? term2 : term1;
-        numRows = segments == 1 ? 2 : 1;
+        numRowsPerSegment = segments == 1 ? 2 : 1;
         verifySegmentMetadata(segmentMetadata);
-        verifyStringIndex(indexDescriptor, index.identifier(), segmentMetadata);
+        verifyStringIndex(components, segmentMetadata);
 
+        // verify 2nd segment
         if (segments > 1)
         {
+            Preconditions.checkState(segments == 2);
             segmentRowIdOffset = sstableRowId2;
-            posting1 = 0;
-            posting2 = 0;
-            minKey = SAITester.TEST_FACTORY.create(key2.getToken());
-            maxKey = minKey;
+            minSegmentRowId = 0;
+            maxSegmentRowId = 0;
+            minKey = SAITester.TEST_FACTORY.createTokenOnly(key2.getToken());
+            maxKey = SAITester.TEST_FACTORY.createTokenOnly(key2.getToken());
             minTerm = term2;
             maxTerm = term2;
-            numRows = 1;
+            numRowsPerSegment = 1;
 
             segmentMetadata = segmentMetadatas.get(1);
             verifySegmentMetadata(segmentMetadata);
-            verifyStringIndex(indexDescriptor, index.identifier(), segmentMetadata);
+            verifyStringIndex(components, segmentMetadata);
         }
     }
 
-    private void verifySegmentMetadata(SegmentMetadata segmentMetadata)
+    private void verifyStringIndex(IndexComponents.ForRead components, SegmentMetadata segmentMetadata) throws IOException
     {
-        assertEquals(segmentRowIdOffset, segmentMetadata.rowIdOffset);
-        assertEquals(minKey, segmentMetadata.minKey);
-        assertEquals(maxKey, segmentMetadata.maxKey);
-        assertEquals(minTerm, segmentMetadata.minTerm);
-        assertEquals(maxTerm, segmentMetadata.maxTerm);
-        assertEquals(numRows, segmentMetadata.numRows);
-    }
+        FileHandle termsData = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+        FileHandle postingLists = components.get(IndexComponentType.POSTING_LISTS).createFileHandle();
 
-    private void verifyStringIndex(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, SegmentMetadata segmentMetadata) throws IOException
-    {
-        FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null);
-        FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
+        long termsFooterPointer = Long.parseLong(segmentMetadata.componentMetadatas.get(IndexComponentType.TERMS_DATA).attributes.get(SAICodecUtils.FOOTER_POINTER));
 
-        try (TermsIterator iterator = new TermsScanner(termsData, postingLists, segmentMetadata.componentMetadatas.get(IndexComponent.TERMS_DATA).root))
+        try (TermsReader reader = new TermsReader(components.context(),
+                                                  termsData,
+                                                  components.byteComparableVersionFor(IndexComponentType.TERMS_DATA),
+                                                  postingLists,
+                                                  segmentMetadata.componentMetadatas.get(IndexComponentType.TERMS_DATA).root,
+                                                  termsFooterPointer,
+                                                  version))
         {
+            TermsIterator iterator = reader.allTerms();
             assertEquals(minTerm, iterator.getMinTerm());
             assertEquals(maxTerm, iterator.getMaxTerm());
 
-            verifyTermPostings(iterator, minTerm, posting1, posting1);
+            verifyTermPostings(iterator, minTerm, minSegmentRowId, minSegmentRowId);
 
-            if (numRows > 1)
+            if (numRowsPerSegment > 1)
             {
-                verifyTermPostings(iterator, maxTerm, posting2, posting2);
+                verifyTermPostings(iterator, maxTerm, maxSegmentRowId, maxSegmentRowId);
             }
 
             assertFalse(iterator.hasNext());
         }
     }
 
-    private void verifyTermPostings(TermsIterator iterator, ByteBuffer expectedTerm, int minSegmentRowId, int maxSegmentRowId)
+    private void verifyTermPostings(TermsIterator iterator, ByteBuffer expectedTerm, int minSegmentRowId, int maxSegmentRowId) throws IOException
     {
-        IndexEntry indexEntry = iterator.next();
+        ByteComparable term = iterator.next();
+        PostingList postings = iterator.postings();
 
-        assertEquals(0, ByteComparable.compare(indexEntry.term, v -> ByteSource.of(expectedTerm, v), ByteComparable.Version.OSS50));
-        assertEquals(minSegmentRowId == maxSegmentRowId ? 1 : 2, indexEntry.postingList.size());
+        assertEquals(0, ByteComparable.compare(term,
+                                               ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, expectedTerm),
+                                               TypeUtil.BYTE_COMPARABLE_VERSION));
+        assertEquals(minSegmentRowId == maxSegmentRowId ? 1 : 2, postings.size());
+    }
+
+    private void verifySegmentMetadata(SegmentMetadata segmentMetadata)
+    {
+        assertEquals(segmentRowIdOffset, segmentMetadata.minSSTableRowId);
+        assertEquals(minKey, segmentMetadata.minKey);
+        assertEquals(maxKey, segmentMetadata.maxKey);
+        assertEquals(minTerm, segmentMetadata.minTerm);
+        assertEquals(maxTerm, segmentMetadata.maxTerm);
+        assertEquals(numRowsPerSegment, segmentMetadata.numRows);
     }
 
     private Row createRow(ColumnMetadata column, ByteBuffer value)
@@ -215,4 +269,30 @@ private Row createRow(ColumnMetadata column, ByteBuffer value)
         builder1.addCell(BufferCell.live(column, 0, value));
         return builder1.build();
     }
+
+    private void assertOverflow(long sstableRowId1, long sstableRowId2) throws Exception
+    {
+        try
+        {
+            testFlushBetweenRowIds(sstableRowId1, sstableRowId2, 0);
+            fail("Expect integer overflow, but didn't");
+        }
+        catch (ArithmeticException e)
+        {
+            assertTrue(e.getMessage().contains("integer overflow"));
+        }
+    }
+
+    private void assertIllegalEndOfStream(long sstableRowId1, long sstableRowId2) throws Exception
+    {
+        try
+        {
+            testFlushBetweenRowIds(sstableRowId1, sstableRowId2, 0);
+            fail("Expect integer overflow, but didn't");
+        }
+        catch (IllegalArgumentException e)
+        {
+            assertTrue(e.getMessage().contains("END_OF_STREAM"));
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java
index 6a0869fbfd6e..57d7257c7b35 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java
@@ -32,7 +32,6 @@
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.sai.disk.v1.segment.Segment;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -46,7 +45,7 @@ public class SegmentTest
     @BeforeClass
     public static void init()
     {
-        DatabaseDescriptor.toolInitialization();
+        DatabaseDescriptor.daemonInitialization();
         DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
         partitioner = DatabaseDescriptor.getPartitioner();
         min = partitioner.getMinimumToken();
@@ -157,6 +156,7 @@ private static AbstractBounds<PartitionPosition> exclusive(Token left, Token rig
 
     private static AbstractBounds<PartitionPosition> keyRange(Token left, boolean inclusiveLeft, Token right, boolean inclusiveRight)
     {
+        // See StatementRestrictions#getPartitionKeyBoundsForTokenRestrictions
         return Bounds.bounds(inclusiveLeft ? left.minKeyBound() : left.maxKeyBound(), inclusiveLeft,
                              inclusiveRight ? right.maxKeyBound() : right.minKeyBound(), inclusiveRight);
     }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/SorterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/SorterTest.java
index d309ce6da1c7..2a7c2f737b5e 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/SorterTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/SorterTest.java
@@ -21,16 +21,16 @@
 
 import org.junit.Test;
 
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
 import org.apache.lucene.util.IntroSorter;
 import org.apache.lucene.util.Sorter;
 
-import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
 
 public class SorterTest
 {
     @Test
-    public void test()
+    public void test() throws Exception
     {
         final int[] array = new int[100];
         for (int x=0; x < array.length; x++)
@@ -40,7 +40,9 @@ public void test()
 
         int[] sortedArray = Arrays.copyOf(array, array.length);
 
-        SAIRandomizedTester.shuffle(array);
+        SaiRandomizedTest.shuffle(array);
+
+        System.out.println("shuffle array="+ Arrays.toString(array));
 
         final Sorter sorter = new IntroSorter() {
             int pivotDoc;
@@ -66,6 +68,10 @@ protected int comparePivot(int j) {
 
         sorter.sort(0, array.length);
 
-        assertArrayEquals(sortedArray, array);
+        System.out.println("sorted array="+ Arrays.toString(array));
+
+        assertTrue(Arrays.equals(sortedArray, array));
     }
+
+
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java
index f73248a32474..622d3f393f0b 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java
@@ -18,118 +18,171 @@
 package org.apache.cassandra.index.sai.disk.v1;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import org.junit.Test;
 
-import com.carrotsearch.hppc.LongArrayList;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.agrona.collections.Int2IntHashMap;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.MemtableTermsIterator;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.segment.LiteralIndexSegmentTermsReader;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
-import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.trie.InvertedIndexWriter;
+import org.apache.cassandra.index.sai.memory.RowMapping;
 import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.cassandra.index.sai.utils.TermsIterator;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
 import static org.apache.cassandra.index.sai.disk.v1.InvertedIndexBuilder.buildStringTermsEnum;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import static org.apache.cassandra.index.sai.metrics.QueryEventListeners.NO_OP_TRIE_LISTENER;
 
-public class TermsReaderTest extends SAIRandomizedTester
+public class TermsReaderTest extends SaiRandomizedTest
 {
+    public static final ByteComparable.Version VERSION = TypeUtil.BYTE_COMPARABLE_VERSION;
+
+    @ParametersFactory()
+    public static Collection<Object[]> data()
+    {
+        // Required because it configures SEGMENT_BUILD_MEMORY_LIMIT, which is needed for Version.AA
+        if (DatabaseDescriptor.getRawConfig() == null)
+            DatabaseDescriptor.setConfig(DatabaseDescriptor.loadConfig());
+        return Version.ALL.stream().map(v -> new Object[]{v}).collect(Collectors.toList());
+    }
+
+    private final Version version;
+
+    public TermsReaderTest(Version version)
+    {
+        this.version = version;
+    }
+
     @Test
     public void testTermQueriesAgainstShortPostingLists() throws IOException
     {
-        testTermQueries(getRandom().nextIntBetween(5, 10), getRandom().nextIntBetween(5, 10));
+        testTermQueries(version, randomIntBetween(5, 10), randomIntBetween(5, 10));
     }
 
     @Test
     public void testTermQueriesAgainstLongPostingLists() throws  IOException
     {
-        testTermQueries(getRandom().nextIntBetween(512, 1024), getRandom().nextIntBetween(1024, 2048));
+        testTermQueries(version, 513, 1025);
     }
 
     @Test
     public void testTermsIteration() throws IOException
     {
-        doTestTermsIteration();
+        doTestTermsIteration(version);
     }
 
-    private void doTestTermsIteration() throws IOException
+    private void doTestTermsIteration(Version version) throws IOException
     {
         final int terms = 70, postings = 2;
         final IndexDescriptor indexDescriptor = newIndexDescriptor();
-        final IndexIdentifier indexIdentifier = createIndexIdentifier("test", "test", newIndex());
-        final List<Pair<ByteComparable, LongArrayList>> termsEnum = buildTermsEnum(terms, postings);
+        final String index = newIndex();
+        final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
+        final List<InvertedIndexBuilder.TermsEnum> termsEnum = buildTermsEnum(version, terms, postings);
 
         SegmentMetadata.ComponentMetadataMap indexMetas;
-        LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier);
-        indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator()));
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (InvertedIndexWriter writer = new InvertedIndexWriter(components))
+        {
+            var iter = termsEnum.stream()
+                    .map(InvertedIndexBuilder::toTermWithFrequency)
+                    .iterator();
+            Int2IntHashMap docLengths = createMockDocLengths(termsEnum);
+            indexMetas = writer.writeAll(new MemtableTermsIterator(null, null, iter), docLengths);
+        }
 
-        FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null);
-        FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
+        FileHandle termsData = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+        FileHandle postingLists = components.get(IndexComponentType.POSTING_LISTS).createFileHandle();
 
-        try (TermsIterator iterator = new TermsScanner(termsData, postingLists, indexMetas.get(IndexComponent.TERMS_DATA).root))
+        long termsFooterPointer = Long.parseLong(indexMetas.get(IndexComponentType.TERMS_DATA).attributes.get(SAICodecUtils.FOOTER_POINTER));
+
+        try (TermsReader reader = new TermsReader(indexContext,
+                                                  termsData,
+                                                  components.byteComparableVersionFor(IndexComponentType.TERMS_DATA),
+                                                  postingLists,
+                                                  indexMetas.get(IndexComponentType.TERMS_DATA).root,
+                                                  termsFooterPointer,
+                                                  version))
         {
-            int i = 0;
-            for (IndexEntry indexEntry = iterator.next(); indexEntry != null; indexEntry = iterator.next())
+            try (TermsIterator actualTermsEnum = reader.allTerms())
             {
-                final ByteComparable expected = termsEnum.get(i++).left;
-                assertEquals(0, ByteComparable.compare(expected, indexEntry.term, ByteComparable.Version.OSS50));
+                int i = 0;
+                for (ByteComparable term = actualTermsEnum.next(); term != null; term = actualTermsEnum.next())
+                {
+                    final ByteComparable expected = termsEnum.get(i++).byteComparableBytes;
+                    assertEquals(0, ByteComparable.compare(expected, term, VERSION));
+                }
             }
         }
     }
 
-    private void testTermQueries(int numTerms, int numPostings) throws IOException
+    private void testTermQueries(Version version, int numTerms, int numPostings) throws IOException
     {
         final IndexDescriptor indexDescriptor = newIndexDescriptor();
-        final IndexIdentifier indexIdentifier = createIndexIdentifier("test", "test", newIndex());
-        final List<Pair<ByteComparable, LongArrayList>> termsEnum = buildTermsEnum(numTerms, numPostings);
+        final String index = newIndex();
+        final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
+        final List<InvertedIndexBuilder.TermsEnum> termsEnum = buildTermsEnum(version, numTerms, numPostings);
 
         SegmentMetadata.ComponentMetadataMap indexMetas;
-        LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier);
-        indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator()));
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (InvertedIndexWriter writer = new InvertedIndexWriter(components))
+        {
+            var iter = termsEnum.stream()
+                    .map(InvertedIndexBuilder::toTermWithFrequency)
+                    .iterator();
+            Int2IntHashMap docLengths = createMockDocLengths(termsEnum);
+            indexMetas = writer.writeAll(new MemtableTermsIterator(null, null, iter), docLengths);
+        }
 
-        FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null);
-        FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
+        FileHandle termsData = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+        FileHandle postingLists = components.get(IndexComponentType.POSTING_LISTS).createFileHandle();
 
-        long termsFooterPointer = Long.parseLong(indexMetas.get(IndexComponent.TERMS_DATA).attributes.get(SAICodecUtils.FOOTER_POINTER));
+        long termsFooterPointer = Long.parseLong(indexMetas.get(IndexComponentType.TERMS_DATA).attributes.get(SAICodecUtils.FOOTER_POINTER));
 
-        try (LiteralIndexSegmentTermsReader reader = new LiteralIndexSegmentTermsReader(indexIdentifier,
-                                                                                        termsData,
-                                                                                        postingLists,
-                                                                                        indexMetas.get(IndexComponent.TERMS_DATA).root,
-                                                                                        termsFooterPointer))
+        try (TermsReader reader = new TermsReader(indexContext,
+                                                  termsData,
+                                                  components.byteComparableVersionFor(IndexComponentType.TERMS_DATA),
+                                                  postingLists,
+                                                  indexMetas.get(IndexComponentType.TERMS_DATA).root,
+                                                  termsFooterPointer,
+                                                  version))
         {
-            for (Pair<ByteComparable, LongArrayList> pair : termsEnum)
+            var iter = termsEnum.stream()
+                    .map(InvertedIndexBuilder::toTermWithFrequency)
+                    .collect(Collectors.toList());
+            for (Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>> pair : iter)
             {
-                final byte[] bytes = ByteSourceInverse.readBytes(pair.left.asComparableBytes(ByteComparable.Version.OSS50));
-                QueryEventListener.TrieIndexEventListener listener = mock(QueryEventListener.TrieIndexEventListener.class);
-                when(listener.postingListEventListener()).thenReturn(mock(QueryEventListener.PostingListEventListener.class));
-                try (PostingList actualPostingList = reader.exactMatch(ByteComparable.fixedLength(bytes),
-                                                                       listener,
-                                                                       mock(QueryContext.class)))
+                final byte[] bytes = ByteSourceInverse.readBytes(pair.left.asComparableBytes(VERSION));
+                try (PostingList actualPostingList = reader.exactMatch(ByteComparable.preencoded(VERSION, bytes),
+                                                                       (QueryEventListener.TrieIndexEventListener)NO_OP_TRIE_LISTENER,
+                                                                       new QueryContext()))
                 {
-                    final LongArrayList expectedPostingList = pair.right;
+                    final List<RowMapping.RowIdWithFrequency> expectedPostingList = pair.right;
 
                     assertNotNull(actualPostingList);
                     assertEquals(expectedPostingList.size(), actualPostingList.size());
 
                     for (int i = 0; i < expectedPostingList.size(); ++i)
                     {
-                        final long expectedRowID = expectedPostingList.get(i);
+                        final long expectedRowID = expectedPostingList.get(i).rowId;
                         long result = actualPostingList.nextPosting();
                         assertEquals(String.format("row %d mismatch of %d in enum %d", i, expectedPostingList.size(), termsEnum.indexOf(pair)), expectedRowID, result);
                     }
@@ -139,22 +192,22 @@ private void testTermQueries(int numTerms, int numPostings) throws IOException
                 }
 
                 // test skipping
-                try (PostingList actualPostingList = reader.exactMatch(ByteComparable.fixedLength(bytes),
-                                                                       listener,
-                                                                       mock(QueryContext.class)))
+                try (PostingList actualPostingList = reader.exactMatch(ByteComparable.preencoded(VERSION, bytes),
+                                                                       (QueryEventListener.TrieIndexEventListener)NO_OP_TRIE_LISTENER,
+                                                                       new QueryContext()))
                 {
-                    final LongArrayList expectedPostingList = pair.right;
+                    final List<RowMapping.RowIdWithFrequency> expectedPostingList = pair.right;
                     // test skipping to the last block
                     final int idxToSkip = numPostings - 2;
                     // tokens are equal to their corresponding row IDs
-                    final long tokenToSkip = expectedPostingList.get(idxToSkip);
+                    final int tokenToSkip = expectedPostingList.get(idxToSkip).rowId;
 
                     long advanceResult = actualPostingList.advance(tokenToSkip);
                     assertEquals(tokenToSkip, advanceResult);
 
                     for (int i = idxToSkip + 1; i < expectedPostingList.size(); ++i)
                     {
-                        final long expectedRowID = expectedPostingList.get(i);
+                        final long expectedRowID = expectedPostingList.get(i).rowId;
                         long result = actualPostingList.nextPosting();
                         assertEquals(expectedRowID, result);
                     }
@@ -166,8 +219,19 @@ private void testTermQueries(int numTerms, int numPostings) throws IOException
         }
     }
 
-    private List<Pair<ByteComparable, LongArrayList>> buildTermsEnum(int terms, int postings)
+    private Int2IntHashMap createMockDocLengths(List<InvertedIndexBuilder.TermsEnum> termsEnum)
+    {
+        Int2IntHashMap docLengths = new Int2IntHashMap(Integer.MIN_VALUE);
+        for (InvertedIndexBuilder.TermsEnum term : termsEnum)
+        {
+            for (var cursor : term.postings)
+                docLengths.put(cursor.value, 1);
+        }
+        return docLengths;
+    }
+
+    private List<InvertedIndexBuilder.TermsEnum> buildTermsEnum(Version version, int terms, int postings)
     {
-        return buildStringTermsEnum(terms, postings, () -> randomSimpleString(4, 10), () -> nextInt(0, Integer.MAX_VALUE));
+        return buildStringTermsEnum(version, terms, postings, () -> randomSimpleString(4, 10), () -> nextInt(0, Integer.MAX_VALUE));
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsScanner.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsScanner.java
deleted file mode 100644
index 8a0d3443e7d3..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsScanner.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
-import org.apache.cassandra.index.sai.disk.v1.postings.ScanningPostingsReader;
-import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsIterator;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.index.sai.utils.TermsIterator;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
-import org.apache.lucene.store.IndexInput;
-
-public class TermsScanner implements TermsIterator
-{
-    private final FileHandle postingsFile;
-    private final TrieTermsIterator iterator;
-    private final ByteBuffer minTerm, maxTerm;
-    private Pair<ByteComparable, Long> entry;
-
-    public TermsScanner(FileHandle termFile, FileHandle postingsFile, long trieRoot)
-    {
-        this.postingsFile = postingsFile;
-        this.iterator = new TrieTermsIterator(termFile.instantiateRebufferer(null), trieRoot);
-        this.minTerm = ByteBuffer.wrap(ByteSourceInverse.readBytes(ByteSourceInverse.unescape(ByteSource.peekable(iterator.getMinTerm().asComparableBytes(ByteComparable.Version.OSS50)))));
-        this.maxTerm = ByteBuffer.wrap(ByteSourceInverse.readBytes(ByteSourceInverse.unescape(ByteSource.peekable(iterator.getMaxTerm().asComparableBytes(ByteComparable.Version.OSS50)))));
-    }
-
-
-    @Override
-    public void close()
-    {
-        FileUtils.closeQuietly(postingsFile);
-        iterator.close();
-    }
-
-    @Override
-    public ByteBuffer getMinTerm()
-    {
-        return minTerm;
-    }
-
-    @Override
-    public ByteBuffer getMaxTerm()
-    {
-        return maxTerm;
-    }
-
-    @Override
-    public IndexEntry next()
-    {
-        if (iterator.hasNext())
-        {
-            entry = iterator.next();
-            return IndexEntry.create(entry.left, postings());
-        }
-        return null;
-    }
-
-    @Override
-    public boolean hasNext()
-    {
-        return iterator.hasNext();
-    }
-
-    private PostingList postings()
-    {
-        assert entry != null;
-        final IndexInput input = IndexFileUtils.instance.openInput(postingsFile);
-        try
-        {
-            return new ScanningPostingsReader(input, new PostingsReader.BlocksSummary(input, entry.right));
-        }
-        catch (IOException e)
-        {
-            throw Throwables.unchecked(e);
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/WideRowPrimaryKeyTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/WideRowPrimaryKeyTest.java
deleted file mode 100644
index 9a6a4dcb154f..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/WideRowPrimaryKeyTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1;
-
-import java.util.Arrays;
-
-import org.junit.Test;
-
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.AbstractPrimaryKeyTester;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class WideRowPrimaryKeyTest extends AbstractPrimaryKeyTester
-{
-    @Test
-    public void randomTest() throws Throwable
-    {
-        IndexDescriptor indexDescriptor = newClusteringIndexDescriptor(compositePartitionMultipleClusteringAsc);
-
-        SSTableComponentsWriter writer = new SSTableComponentsWriter(indexDescriptor);
-
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, compositePartitionMultipleClusteringAsc.comparator);
-
-        int rows = nextInt(1000, 10000);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int partitionSize = nextInt(5, 500);
-        int partitionCounter = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(compositePartitionMultipleClusteringAsc, partition, partition),
-                                                 makeClustering(compositePartitionMultipleClusteringAsc,
-                                                                getRandom().nextTextString(10, 100),
-                                                                getRandom().nextTextString(10, 100)));
-            partitionCounter++;
-            if (partitionCounter == partitionSize)
-            {
-                partition++;
-                partitionCounter = 0;
-                partitionSize = nextInt(5, 500);
-            }
-        }
-
-        Arrays.sort(keys);
-
-        DecoratedKey lastKey = null;
-        for (PrimaryKey primaryKey : keys)
-        {
-            if (lastKey == null || lastKey.compareTo(primaryKey.partitionKey()) < 0)
-            {
-                lastKey = primaryKey.partitionKey();
-                writer.startPartition(lastKey);
-            }
-            writer.nextRow(primaryKey);
-        }
-
-        writer.complete();
-
-        SSTableReader sstableReader = mock(SSTableReader.class);
-        when(sstableReader.metadata()).thenReturn(compositePartitionMultipleClusteringAsc);
-
-        try (PrimaryKeyMap.Factory mapFactory = new WidePrimaryKeyMap.Factory(indexDescriptor, sstableReader);
-             PrimaryKeyMap primaryKeyMap = mapFactory.newPerSSTablePrimaryKeyMap())
-        {
-            for (int key = 0; key < rows; key++)
-            {
-                PrimaryKey test = keys[key];
-
-                test = factory.create(test.partitionKey(),
-                                      makeClustering(compositePartitionMultipleClusteringAsc,
-                                                     getRandom().nextTextString(10, 100),
-                                                     getRandom().nextTextString(10, 100)));
-
-                long rowId = primaryKeyMap.rowIdFromPrimaryKey(test);
-
-                if (rowId >= 0)
-                {
-                    PrimaryKey found = keys[(int) rowId];
-
-                    assertTrue(found.compareTo(test) >= 0);
-
-                    if (rowId > 0)
-                        assertTrue(keys[(int) rowId - 1].compareTo(test) < 0);
-                }
-                else
-                {
-                    assertTrue(test.compareTo(keys[keys.length - 1]) > 0);
-                }
-            }
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIndexBuilder.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIndexBuilder.java
deleted file mode 100644
index 4764bce4a348..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIndexBuilder.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.LongStream;
-import java.util.stream.Stream;
-
-import org.junit.Assert;
-
-import com.carrotsearch.hppc.LongArrayList;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.DecimalType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.marshal.ShortType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
-import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcher;
-import org.apache.cassandra.index.sai.disk.v1.segment.NumericIndexSegmentSearcher;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.TermsIterator;
-import org.apache.cassandra.utils.AbstractGuavaIterator;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-public class BlockBalancedTreeIndexBuilder
-{
-    public static final PrimaryKeyMap TEST_PRIMARY_KEY_MAP = new PrimaryKeyMap()
-    {
-        private final PrimaryKey.Factory primaryKeyFactory = new PrimaryKey.Factory(Murmur3Partitioner.instance, null);
-
-        @Override
-        public PrimaryKey primaryKeyFromRowId(long sstableRowId)
-        {
-            return primaryKeyFactory.create(new Murmur3Partitioner.LongToken(sstableRowId));
-        }
-
-        @Override
-        public long rowIdFromPrimaryKey(PrimaryKey key)
-        {
-            return key.token().getLongValue();
-        }
-
-        @Override
-        public long ceiling(Token token)
-        {
-            return token.getLongValue();
-        }
-
-        @Override
-        public long floor(Token token)
-        {
-            return token.getLongValue();
-        }
-    };
-    public static final PrimaryKeyMap.Factory TEST_PRIMARY_KEY_MAP_FACTORY = () -> TEST_PRIMARY_KEY_MAP;
-
-
-    private static final BigDecimal ONE_TENTH = BigDecimal.valueOf(1, 1);
-
-    private final IndexDescriptor indexDescriptor;
-    private final AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>> terms;
-    private final int size;
-    private final int minSegmentRowId;
-    private final int maxSegmentRowId;
-
-    public BlockBalancedTreeIndexBuilder(IndexDescriptor indexDescriptor,
-                                         AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>> terms,
-                                         int size,
-                                         int minSegmentRowId,
-                                         int maxSegmentRowId)
-    {
-        this.indexDescriptor = indexDescriptor;
-        this.terms = terms;
-        this.size = size;
-        this.minSegmentRowId = minSegmentRowId;
-        this.maxSegmentRowId = maxSegmentRowId;
-    }
-
-    NumericIndexSegmentSearcher flushAndOpen(AbstractType<?> type) throws IOException
-    {
-        final TermsIterator termEnum = new MemtableTermsIterator(null, null, terms);
-        final SegmentMetadata metadata;
-
-        StorageAttachedIndex index = SAITester.createMockIndex(type);
-
-        NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
-                                                           index.identifier(),
-                                                           index.termType().fixedSizeOf());
-        final SegmentMetadata.ComponentMetadataMap indexMetas = writer.writeCompleteSegment(termEnum);
-        metadata = new SegmentMetadata(0,
-                                       size,
-                                       minSegmentRowId,
-                                       maxSegmentRowId,
-                                       // min/max is unused for now
-                                       SAITester.TEST_FACTORY.create(Murmur3Partitioner.instance.decorateKey(UTF8Type.instance.fromString("a")).getToken()),
-                                       SAITester.TEST_FACTORY.create(Murmur3Partitioner.instance.decorateKey(UTF8Type.instance.fromString("b")).getToken()),
-                                       UTF8Type.instance.fromString("c"),
-                                       UTF8Type.instance.fromString("d"),
-                                       indexMetas);
-
-        try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, index.termType(), index.identifier()))
-        {
-            IndexSegmentSearcher searcher = IndexSegmentSearcher.open(TEST_PRIMARY_KEY_MAP_FACTORY, indexFiles, metadata, index);
-            assertThat(searcher, is(instanceOf(NumericIndexSegmentSearcher.class)));
-            return (NumericIndexSegmentSearcher) searcher;
-        }
-    }
-
-    /**
-     * Returns a k-d tree index where:
-     * 1. term values have 32b
-     * 2. term value is equal to {@code startTermInclusive} + row id;
-     * 3. tokens and offsets are equal to row id;
-     */
-    public static IndexSegmentSearcher buildInt32Searcher(IndexDescriptor indexDescriptor, int startTermInclusive, int endTermExclusive)
-    throws IOException
-    {
-        final int size = endTermExclusive - startTermInclusive;
-        Assert.assertTrue(size > 0);
-        BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
-                                                                                       singleOrd(int32Range(startTermInclusive, endTermExclusive), Int32Type.instance, startTermInclusive, size),
-                                                                                       size,
-                                                                                       startTermInclusive,
-                                                                                       endTermExclusive);
-        return indexBuilder.flushAndOpen(Int32Type.instance);
-    }
-
-    public static IndexSegmentSearcher buildDecimalSearcher(IndexDescriptor indexDescriptor, BigDecimal startTermInclusive, BigDecimal endTermExclusive)
-    throws IOException
-    {
-        BigDecimal bigDifference = endTermExclusive.subtract(startTermInclusive);
-        int size = bigDifference.intValueExact() * 10;
-        Assert.assertTrue(size > 0);
-        BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
-                                                                                       singleOrd(decimalRange(startTermInclusive, endTermExclusive), DecimalType.instance, startTermInclusive.intValueExact() * 10, size),
-                                                                                       size,
-                                                                 startTermInclusive.intValueExact() * 10,
-                                                                 endTermExclusive.intValueExact() * 10);
-        return indexBuilder.flushAndOpen(DecimalType.instance);
-    }
-
-    public static IndexSegmentSearcher buildBigIntegerSearcher(IndexDescriptor indexDescriptor, BigInteger startTermInclusive, BigInteger endTermExclusive)
-    throws IOException
-    {
-        BigInteger bigDifference = endTermExclusive.subtract(startTermInclusive);
-        int size = bigDifference.intValueExact();
-        Assert.assertTrue(size > 0);
-        BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
-                                                                                       singleOrd(bigIntegerRange(startTermInclusive, endTermExclusive), IntegerType.instance, startTermInclusive.intValueExact(), size),
-                                                                                       size,
-                                                                                       startTermInclusive.intValueExact(),
-                                                                                       endTermExclusive.intValueExact());
-        return indexBuilder.flushAndOpen(IntegerType.instance);
-    }
-
-    /**
-     * Returns a k-d tree index where:
-     * 1. term values have 64b
-     * 2. term value is equal to {@code startTermInclusive} + row id;
-     * 3. tokens and offsets are equal to row id;
-     */
-    public static IndexSegmentSearcher buildLongSearcher(IndexDescriptor indexDescriptor, long startTermInclusive, long endTermExclusive)
-    throws IOException
-    {
-        final long size = endTermExclusive - startTermInclusive;
-        Assert.assertTrue(size > 0);
-        BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
-                                                                                       singleOrd(longRange(startTermInclusive, endTermExclusive), LongType.instance, Math.toIntExact(startTermInclusive), Math.toIntExact(size)),
-                                                                                       Math.toIntExact(size),
-                                                                                       Math.toIntExact(startTermInclusive),
-                                                                                       Math.toIntExact(endTermExclusive));
-        return indexBuilder.flushAndOpen(LongType.instance);
-    }
-
-    /**
-     * Returns a k-d tree index where:
-     * 1. term values have 16b
-     * 2. term value is equal to {@code startTermInclusive} + row id;
-     * 3. tokens and offsets are equal to row id;
-     */
-    public static IndexSegmentSearcher buildShortSearcher(IndexDescriptor indexDescriptor, short startTermInclusive, short endTermExclusive)
-    throws IOException
-    {
-        final int size = endTermExclusive - startTermInclusive;
-        Assert.assertTrue(size > 0);
-        BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
-                                                                                       singleOrd(shortRange(startTermInclusive, endTermExclusive), ShortType.instance, startTermInclusive, size),
-                                                                                       size,
-                                                                                       startTermInclusive,
-                                                                                       endTermExclusive);
-        return indexBuilder.flushAndOpen(ShortType.instance);
-    }
-
-    /**
-     * Returns inverted index where each posting list contains exactly one element equal to the terms ordinal number +
-     * given offset.
-     */
-    public static AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>> singleOrd(Iterator<ByteBuffer> terms, AbstractType<?> type, int segmentRowIdOffset, int size)
-    {
-        IndexTermType indexTermType = SAITester.createIndexTermType(type);
-        return new AbstractGuavaIterator<>()
-        {
-            private long currentTerm = 0;
-            private int currentSegmentRowId = segmentRowIdOffset;
-
-            @Override
-            protected Pair<ByteComparable, LongArrayList> computeNext()
-            {
-                if (currentTerm++ >= size)
-                {
-                    return endOfData();
-                }
-
-                LongArrayList postings = new LongArrayList();
-                postings.add(currentSegmentRowId++);
-                assertTrue(terms.hasNext());
-
-                final ByteSource encoded = indexTermType.asComparableBytes(terms.next(), ByteComparable.Version.OSS50);
-                return Pair.create(v -> encoded, postings);
-            }
-        };
-    }
-
-    /**
-     * Returns sequential ordered encoded ints from {@code startInclusive} (inclusive) to {@code endExclusive}
-     * (exclusive) by an incremental step of {@code 1}.
-     */
-    public static Iterator<ByteBuffer> int32Range(int startInclusive, int endExclusive)
-    {
-        return IntStream.range(startInclusive, endExclusive)
-                        .mapToObj(Int32Type.instance::decompose)
-                        .collect(Collectors.toList())
-                        .iterator();
-    }
-
-    /**
-     * Returns sequential ordered encoded longs from {@code startInclusive} (inclusive) to {@code endExclusive}
-     * (exclusive) by an incremental step of {@code 1}.
-     */
-    public static Iterator<ByteBuffer> longRange(long startInclusive, long endExclusive)
-    {
-        return LongStream.range(startInclusive, endExclusive)
-                         .mapToObj(LongType.instance::decompose)
-                         .collect(Collectors.toList())
-                         .iterator();
-    }
-
-    public static Iterator<ByteBuffer> decimalRange(final BigDecimal startInclusive, final BigDecimal endExclusive)
-    {
-        int n = endExclusive.subtract(startInclusive).intValueExact() * 10;
-        final Supplier<BigDecimal> generator = new Supplier<>() {
-            BigDecimal current = startInclusive;
-
-            @Override
-            public BigDecimal get() {
-                BigDecimal result = current;
-                current = current.add(ONE_TENTH);
-                return result;
-            }
-        };
-        IndexTermType indexTermType = SAITester.createIndexTermType(DecimalType.instance);
-        return Stream.generate(generator)
-                     .limit(n)
-                     .map(bd -> indexTermType.asIndexBytes(DecimalType.instance.decompose(bd)))
-                     .collect(Collectors.toList())
-                     .iterator();
-    }
-
-    public static Iterator<ByteBuffer> bigIntegerRange(final BigInteger startInclusive, final BigInteger endExclusive)
-    {
-        int n = endExclusive.subtract(startInclusive).intValueExact();
-        final Supplier<BigInteger> generator = new Supplier<>() {
-            BigInteger current = startInclusive;
-
-            @Override
-            public BigInteger get() {
-                BigInteger result = current;
-                current = current.add(BigInteger.ONE);
-                return result;
-            }
-        };
-        IndexTermType indexTermType = SAITester.createIndexTermType(IntegerType.instance);
-        return Stream.generate(generator)
-                     .limit(n)
-                     .map(bd -> indexTermType.asIndexBytes(IntegerType.instance.decompose(bd)))
-                     .collect(Collectors.toList())
-                     .iterator();
-    }
-
-
-    /**
-     * Returns sequential ordered encoded shorts from {@code startInclusive} (inclusive) to {@code endExclusive}
-     * (exclusive) by an incremental step of {@code 1}.
-     */
-    public static Iterator<ByteBuffer> shortRange(short startInclusive, short endExclusive)
-    {
-        return IntStream.range(startInclusive, endExclusive)
-                        .mapToObj(i -> ShortType.instance.decompose((short) i))
-                        .collect(Collectors.toList())
-                        .iterator();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriterTest.java
deleted file mode 100644
index ead3971967a6..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriterTest.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.agrona.collections.IntArrayList;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.ArrayPostingList;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
-import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.packed.PackedInts;
-import org.apache.lucene.util.packed.PackedLongValues;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
-public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester
-{
-    private IndexDescriptor indexDescriptor;
-    private IndexIdentifier indexIdentifier;
-
-    @Before
-    public void setup() throws Throwable
-    {
-        indexDescriptor = newIndexDescriptor();
-        indexIdentifier = SAITester.createIndexIdentifier("test", "test", newIndex());
-    }
-
-    @Test
-    public void shouldWritePostingsForEligibleNodes() throws Exception
-    {
-        List<PackedLongValues> leaves =
-        Arrays.asList(postings(1, 5, 7), postings(3, 4, 6), postings(2, 8, 10), postings(11, 12, 13));
-
-        setBDKPostingsWriterSizing(1, 2);
-        BlockBalancedTreePostingsWriter writer = new BlockBalancedTreePostingsWriter();
-
-        // should build postings for nodes 2 & 3 (lvl 2) and 8, 10, 12, 14 (lvl 4)
-        writer.onLeaf(64, 1, pathToRoot(1, 2, 4, 8, 16));
-        writer.onLeaf(80, 2, pathToRoot(1, 2, 5, 10, 20));
-        writer.onLeaf(96, 3, pathToRoot(1, 3, 6, 12, 24));
-        writer.onLeaf(112, 4, pathToRoot(1, 3, 7, 14, 28));
-
-        long fp;
-        try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier))
-        {
-            fp = writer.finish(output, leaves, indexIdentifier);
-        }
-
-        BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp);
-        assertEquals(10, postingsIndex.size());
-
-        // Internal postings...
-        assertTrue(postingsIndex.exists(2));
-        assertTrue(postingsIndex.exists(3));
-        assertTrue(postingsIndex.exists(8));
-        assertTrue(postingsIndex.exists(10));
-        assertTrue(postingsIndex.exists(12));
-        assertTrue(postingsIndex.exists(14));
-
-        assertPostingReaderEquals(postingsIndex, 2, 1, 3, 4, 5, 6, 7);
-        assertPostingReaderEquals(postingsIndex, 3, 2, 8, 10, 11, 12, 13);
-        assertPostingReaderEquals(postingsIndex, 8, 1, 5, 7);
-        assertPostingReaderEquals(postingsIndex, 10, 3, 4, 6);
-        assertPostingReaderEquals(postingsIndex, 12, 2, 8, 10);
-        assertPostingReaderEquals(postingsIndex, 14, 11, 12, 13);
-
-        // Leaf postings...
-        assertTrue(postingsIndex.exists(64));
-        assertTrue(postingsIndex.exists(80));
-        assertTrue(postingsIndex.exists(96));
-        assertTrue(postingsIndex.exists(112));
-
-        assertPostingReaderEquals(postingsIndex, 64, 1, 5, 7);
-        assertPostingReaderEquals(postingsIndex, 80, 3, 4, 6);
-        assertPostingReaderEquals(postingsIndex, 96, 2, 8, 10);
-        assertPostingReaderEquals(postingsIndex, 112, 11, 12, 13);
-    }
-
-    @Test
-    public void shouldSkipPostingListWhenSamplingMisses() throws Exception
-    {
-        List<PackedLongValues> leaves = Collections.singletonList(postings(1, 2, 3));
-
-        setBDKPostingsWriterSizing(1, 5);
-        BlockBalancedTreePostingsWriter writer = new BlockBalancedTreePostingsWriter();
-
-        // The tree is too short to have any internal posting lists.
-        writer.onLeaf(16, 1, pathToRoot(1, 2, 4, 8));
-
-        long fp;
-        try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier))
-        {
-            fp = writer.finish(output, leaves, indexIdentifier);
-        }
-
-        // There is only a single posting list...the leaf posting list.
-        BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp);
-        assertEquals(1, postingsIndex.size());
-    }
-
-    @Test
-    public void shouldSkipPostingListWhenTooFewLeaves() throws Exception
-    {
-        List<PackedLongValues> leaves = Collections.singletonList(postings(1, 2, 3));
-
-        setBDKPostingsWriterSizing(2, 2);
-        BlockBalancedTreePostingsWriter writer = new BlockBalancedTreePostingsWriter();
-
-        // The tree is too short to have any internal posting lists.
-        writer.onLeaf(16, 1, pathToRoot(1, 2, 4, 8));
-
-        long fp;
-        try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier))
-        {
-            fp = writer.finish(output, leaves, indexIdentifier);
-        }
-
-        // There is only a single posting list...the leaf posting list.
-        BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp);
-        assertEquals(1, postingsIndex.size());
-    }
-
-    private void assertPostingReaderEquals(BlockBalancedTreePostingsIndex postingsIndex, int nodeID, long... postings) throws IOException
-    {
-        assertPostingReaderEquals(indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier),
-                                  postingsIndex.getPostingsFilePointer(nodeID),
-                                  new ArrayPostingList(postings));
-    }
-
-    private void assertPostingReaderEquals(IndexInput input, long offset, PostingList expected) throws IOException
-    {
-        try (PostingsReader reader = new PostingsReader(input, offset, mock(QueryEventListener.PostingListEventListener.class)))
-        {
-            assertPostingListEquals(expected, reader);
-        }
-    }
-
-    private PackedLongValues postings(int... postings)
-    {
-        final PackedLongValues.Builder builder = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
-        for (int posting : postings)
-        {
-            builder.add(posting);
-        }
-        return builder.build();
-    }
-
-    private IntArrayList pathToRoot(int... nodes)
-    {
-        final IntArrayList path = new IntArrayList();
-        for (int node : nodes)
-        {
-            path.add(node);
-        }
-        return path;
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueriesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueriesTest.java
deleted file mode 100644
index 9316ead63034..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueriesTest.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
-
-import static org.apache.lucene.index.PointValues.Relation.CELL_CROSSES_QUERY;
-import static org.apache.lucene.index.PointValues.Relation.CELL_INSIDE_QUERY;
-import static org.apache.lucene.index.PointValues.Relation.CELL_OUTSIDE_QUERY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class BlockBalancedTreeQueriesTest extends SAIRandomizedTester
-{
-    @Test
-    public void testMatchesAll()
-    {
-        Expression expression = Expression.create(createMockIndex(Int32Type.instance));
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        for (int visit = 0; visit < between(100, 1000); visit++)
-            assertTrue(query.contains(toSortableBytes(nextInt(Integer.MAX_VALUE))));
-
-        for (int compare = 0; compare < between(100, 1000); compare++)
-            assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(between(0, Integer.MAX_VALUE/2)),
-                                                          toSortableBytes(between(Integer.MAX_VALUE/2, Integer.MAX_VALUE))));
-    }
-
-    @Test
-    public void testInclusiveLowerBound()
-    {
-        int lowerBound = between(-10, 10);
-        Expression expression = buildExpression(Operator.GTE, lowerBound);
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        assertFalse(query.contains(toSortableBytes(lowerBound - 1)));
-        assertTrue(query.contains(toSortableBytes(lowerBound)));
-        assertTrue(query.contains(toSortableBytes(lowerBound + 1)));
-
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 2), toSortableBytes(lowerBound - 1)));
-        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(lowerBound + 1)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
-    }
-
-    @Test
-    public void testExclusiveLowerBound()
-    {
-        int lowerBound = between(-10, 10);
-        Expression expression = buildExpression(Operator.GT, lowerBound);
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        assertFalse(query.contains(toSortableBytes(lowerBound - 1)));
-        assertFalse(query.contains(toSortableBytes(lowerBound)));
-        assertTrue(query.contains(toSortableBytes(lowerBound + 1)));
-
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
-        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound + 1), toSortableBytes(lowerBound + 2)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(lowerBound + 1)));
-    }
-
-    @Test
-    public void testInclusiveUpperBound()
-    {
-        int upperBound = between(-10, 10);
-        Expression expression = buildExpression(Operator.LTE, upperBound);
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        assertTrue(query.contains(toSortableBytes(upperBound - 1)));
-        assertTrue(query.contains(toSortableBytes(upperBound)));
-        assertFalse(query.contains(toSortableBytes(upperBound + 1)));
-
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound + 1), toSortableBytes(upperBound + 2)));
-        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(upperBound - 1), toSortableBytes(upperBound)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
-    }
-
-    @Test
-    public void testExclusiveUpperBound()
-    {
-        int upper = between(-10, 10);
-        Expression expression = buildExpression(Operator.LT, upper);
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        assertTrue(query.contains(toSortableBytes(upper - 1)));
-        assertFalse(query.contains(toSortableBytes(upper)));
-        assertFalse(query.contains(toSortableBytes(upper + 1)));
-
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upper), toSortableBytes(upper + 1)));
-        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(upper - 2), toSortableBytes(upper - 1)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upper - 1), toSortableBytes(upper)));
-    }
-
-    @Test
-    public void testInclusiveLowerAndUpperBound()
-    {
-        int lowerBound = between(-15, 15);
-        int upperBound = lowerBound + 5;
-        Expression expression = buildExpression(Operator.GTE, lowerBound).add(Operator.LTE, Int32Type.instance.decompose(upperBound));
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        assertFalse(query.contains(toSortableBytes(lowerBound - 1)));
-        assertTrue(query.contains(toSortableBytes(lowerBound)));
-        assertTrue(query.contains(toSortableBytes(lowerBound + 1)));
-        assertTrue(query.contains(toSortableBytes(upperBound - 1)));
-        assertTrue(query.contains(toSortableBytes(upperBound)));
-        assertFalse(query.contains(toSortableBytes(upperBound + 1)));
-
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 2), toSortableBytes(lowerBound - 1)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
-        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(upperBound)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound + 1), toSortableBytes(upperBound + 2)));
-    }
-
-    @Test
-    public void testExclusiveLowerAndUpperBound()
-    {
-        int lowerBound = between(-15, 15);
-        int upperBound = lowerBound + 5;
-        Expression expression = buildExpression(Operator.GT, lowerBound).add(Operator.LT, Int32Type.instance.decompose(upperBound));
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        assertFalse(query.contains(toSortableBytes(lowerBound - 1)));
-        assertFalse(query.contains(toSortableBytes(lowerBound)));
-        assertTrue(query.contains(toSortableBytes(lowerBound + 1)));
-        assertTrue(query.contains(toSortableBytes(upperBound - 1)));
-        assertFalse(query.contains(toSortableBytes(upperBound)));
-        assertFalse(query.contains(toSortableBytes(upperBound + 1)));
-
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(lowerBound + 1)));
-        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound + 1), toSortableBytes(upperBound - 1)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound - 1), toSortableBytes(upperBound)));
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
-    }
-
-    @Test
-    public void testExclusiveLowerAndInclusiveUpperBound()
-    {
-        int lowerBound = between(-15, 15);
-        int upperBound = lowerBound + 5;
-        Expression expression = buildExpression(Operator.GT, lowerBound).add(Operator.LTE, Int32Type.instance.decompose(upperBound));
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        assertFalse(query.contains(toSortableBytes(lowerBound - 1)));
-        assertFalse(query.contains(toSortableBytes(lowerBound)));
-        assertTrue(query.contains(toSortableBytes(lowerBound + 1)));
-        assertTrue(query.contains(toSortableBytes(upperBound - 1)));
-        assertTrue(query.contains(toSortableBytes(upperBound)));
-        assertFalse(query.contains(toSortableBytes(upperBound + 1)));
-
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(lowerBound + 1)));
-        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound + 1), toSortableBytes(upperBound)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound + 1), toSortableBytes(upperBound + 2)));
-    }
-
-    @Test
-    public void testInclusiveLowerAndExclusiveUpperBound()
-    {
-        int lowerBound = between(-15, 15);
-        int upperBound = lowerBound + 5;
-        Expression expression = buildExpression(Operator.GTE, lowerBound).add(Operator.LT, Int32Type.instance.decompose(upperBound));
-        BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
-
-        assertFalse(query.contains(toSortableBytes(lowerBound - 1)));
-        assertTrue(query.contains(toSortableBytes(lowerBound)));
-        assertTrue(query.contains(toSortableBytes(lowerBound + 1)));
-        assertTrue(query.contains(toSortableBytes(upperBound - 1)));
-        assertFalse(query.contains(toSortableBytes(upperBound)));
-        assertFalse(query.contains(toSortableBytes(upperBound + 1)));
-
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 2), toSortableBytes(lowerBound - 1)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
-        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(upperBound - 1)));
-        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound - 1), toSortableBytes(upperBound)));
-        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
-    }
-
-    private byte[] toSortableBytes(int value)
-    {
-        byte[] buffer = new byte[4];
-        ByteSource source = Int32Type.instance.asComparableBytes(Int32Type.instance.decompose(value), ByteComparable.Version.OSS50);
-        ByteSourceInverse.copyBytes(source, buffer);
-        return buffer;
-    }
-
-    private Expression buildExpression(Operator op, int value)
-    {
-        Expression expression = Expression.create(createMockIndex(Int32Type.instance));
-        expression.add(op, Int32Type.instance.decompose(value));
-        return expression;
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReaderTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReaderTest.java
deleted file mode 100644
index 440fd91b9c17..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReaderTest.java
+++ /dev/null
@@ -1,332 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentTrieBuffer;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.util.NumericUtils;
-
-import static org.apache.lucene.index.PointValues.Relation.CELL_CROSSES_QUERY;
-import static org.apache.lucene.index.PointValues.Relation.CELL_INSIDE_QUERY;
-import static org.apache.lucene.index.PointValues.Relation.CELL_OUTSIDE_QUERY;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class BlockBalancedTreeReaderTest extends SAIRandomizedTester
-{
-    private final BlockBalancedTreeReader.IntersectVisitor NONE_MATCH = new BlockBalancedTreeReader.IntersectVisitor()
-    {
-        @Override
-        public boolean contains(byte[] packedValue)
-        {
-            return false;
-        }
-
-        @Override
-        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
-        {
-            return CELL_OUTSIDE_QUERY;
-        }
-    };
-
-    private final BlockBalancedTreeReader.IntersectVisitor ALL_MATCH = new BlockBalancedTreeReader.IntersectVisitor()
-    {
-        @Override
-        public boolean contains(byte[] packedValue)
-        {
-            return true;
-        }
-
-        @Override
-        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
-        {
-            return CELL_INSIDE_QUERY;
-        }
-    };
-
-    private IndexDescriptor indexDescriptor;
-    private StorageAttachedIndex index;
-
-    @Before
-    public void setup() throws Throwable
-    {
-        indexDescriptor = newIndexDescriptor();
-        index = SAITester.createMockIndex(Int32Type.instance);
-    }
-
-    @Test
-    public void testFilteringIntersection() throws Exception
-    {
-        int numRows = 1000;
-
-        final SegmentTrieBuffer buffer = new SegmentTrieBuffer();
-
-        for (int rowId = 0; rowId < numRows; rowId++)
-        {
-            buffer.add(integerToByteComparable(rowId), Integer.BYTES, rowId);
-        }
-
-        try (BlockBalancedTreeReader reader = finishAndOpenReader(4, buffer))
-        {
-            assertRange(reader, 445, 555);
-        }
-    }
-
-    @Test
-    public void testAdvance() throws Exception
-    {
-        final int numRows = between(1000, 2000);
-        final SegmentTrieBuffer buffer = new SegmentTrieBuffer();
-
-        for (int rowId = 0; rowId < numRows; rowId++)
-        {
-            buffer.add(integerToByteComparable(rowId), Integer.BYTES, rowId);
-        }
-
-        try (BlockBalancedTreeReader reader = finishAndOpenReader(2, buffer))
-        {
-            PostingList intersection = performIntersection(reader, NONE_MATCH);
-            assertNull(intersection);
-
-            intersection = performIntersection(reader, ALL_MATCH);
-            assertEquals(numRows, intersection.size());
-            assertEquals(100, intersection.advance(100));
-            assertEquals(200, intersection.advance(200));
-            assertEquals(300, intersection.advance(300));
-            assertEquals(400, intersection.advance(400));
-            assertEquals(401, intersection.advance(401));
-            long expectedRowID = 402;
-            for (long id = intersection.nextPosting(); expectedRowID < 500; id = intersection.nextPosting())
-            {
-                assertEquals(expectedRowID++, id);
-            }
-            assertEquals(PostingList.END_OF_STREAM, intersection.advance(numRows + 1));
-
-            intersection.close();
-        }
-    }
-
-    @Test
-    public void testSameValuesInLeaf() throws Exception
-    {
-        // While a bit synthetic this test is designed to test that the
-        // BlockBalancedTreeReader.FilteringIntersection.buildPostingsFilterForSingleValueLeaf
-        // method is exercised in a test. To do this we need to ensure that
-        // we have at least one leaf that has all the same value and that
-        // all of that leaf is requested in a query.
-        final SegmentTrieBuffer buffer = new SegmentTrieBuffer();
-
-        for (int rowId = 0; rowId < 10; rowId++)
-        {
-            buffer.add(integerToByteComparable(rowId), Integer.BYTES, rowId);
-        }
-
-        for (int rowId = 10; rowId < 20; rowId++)
-        {
-            buffer.add(integerToByteComparable(10), Integer.BYTES, rowId);
-        }
-
-        for (int rowId = 20; rowId < 30; rowId++)
-        {
-            buffer.add(integerToByteComparable(rowId), Integer.BYTES, rowId);
-        }
-
-        try (BlockBalancedTreeReader reader = finishAndOpenReader(5, buffer))
-        {
-            PostingList postingList = performIntersection(reader, buildQuery(8, 15));
-
-            assertEquals(8, postingList.nextPosting());
-            assertEquals(9, postingList.nextPosting());
-            assertEquals(10, postingList.nextPosting());
-            assertEquals(11, postingList.nextPosting());
-            assertEquals(12, postingList.nextPosting());
-            assertEquals(13, postingList.nextPosting());
-            assertEquals(14, postingList.nextPosting());
-            assertEquals(15, postingList.nextPosting());
-            assertEquals(16, postingList.nextPosting());
-            assertEquals(17, postingList.nextPosting());
-            assertEquals(18, postingList.nextPosting());
-            assertEquals(19, postingList.nextPosting());
-            assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
-        }
-    }
-
-    @Test
-    public void testResourcesReleaseWhenQueryDoesntMatchAnything() throws Exception
-    {
-        final SegmentTrieBuffer buffer = new SegmentTrieBuffer();
-        for (int rowId = 0; rowId < 1000; rowId++)
-        {
-            buffer.add(integerToByteComparable(rowId), Integer.BYTES, rowId);
-        }
-        // add a gap between 1000 and 1100
-        for (int rowId = 1000; rowId < 2000; rowId++)
-        {
-            buffer.add(integerToByteComparable(rowId + 100), Integer.BYTES, rowId);
-        }
-
-        try (BlockBalancedTreeReader reader = finishAndOpenReader(50, buffer))
-        {
-            final PostingList intersection = performIntersection(reader, buildQuery(1017, 1096));
-            assertNull(intersection);
-        }
-    }
-
-    @Test
-    public void testConcurrentIntersectionsOnSameReader() throws Exception
-    {
-        int numRows = 1000;
-
-        final SegmentTrieBuffer buffer = new SegmentTrieBuffer();
-
-        for (int rowId = 0; rowId < numRows; rowId++)
-        {
-            buffer.add(integerToByteComparable(rowId), Integer.BYTES, rowId);
-        }
-
-        try (BlockBalancedTreeReader reader = finishAndOpenReader(4, buffer))
-        {
-            int concurrency = 100;
-
-            ExecutorService executor = Executors.newFixedThreadPool(concurrency);
-            List<Future<?>> results = new ArrayList<>();
-            for (int thread = 0; thread < concurrency; thread++)
-            {
-                results.add(executor.submit(() -> assertRange(reader, 445, 555)));
-            }
-            FBUtilities.waitOnFutures(results);
-            executor.shutdown();
-        }
-    }
-
-    @SuppressWarnings("SameParameterValue")
-    private void assertRange(BlockBalancedTreeReader reader, long lowerBound, long upperBound)
-    {
-        Expression expression = Expression.create(index);
-        expression.add(Operator.GT, Int32Type.instance.decompose(444));
-        expression.add(Operator.LT, Int32Type.instance.decompose(555));
-
-        try
-        {
-            PostingList intersection = performIntersection(reader, BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4));
-            assertNotNull(intersection);
-            assertEquals(upperBound - lowerBound, intersection.size());
-            for (long posting = lowerBound; posting < upperBound; posting++)
-                assertEquals(posting, intersection.nextPosting());
-        }
-        catch (IOException e)
-        {
-            throw Throwables.unchecked(e);
-        }
-    }
-
-    private PostingList performIntersection(BlockBalancedTreeReader reader, BlockBalancedTreeReader.IntersectVisitor visitor)
-    {
-        QueryEventListener.BalancedTreeEventListener balancedTreeEventListener = mock(QueryEventListener.BalancedTreeEventListener.class);
-        when(balancedTreeEventListener.postingListEventListener()).thenReturn(mock(QueryEventListener.PostingListEventListener.class));
-        return reader.intersect(visitor, balancedTreeEventListener, mock(QueryContext.class));
-    }
-
-    private BlockBalancedTreeReader.IntersectVisitor buildQuery(int queryMin, int queryMax)
-    {
-        return new BlockBalancedTreeReader.IntersectVisitor()
-        {
-            @Override
-            public boolean contains(byte[] packedValue)
-            {
-                int x = NumericUtils.sortableBytesToInt(packedValue, 0);
-                return x >= queryMin && x <= queryMax;
-            }
-
-            @Override
-            public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
-            {
-                int min = NumericUtils.sortableBytesToInt(minPackedValue, 0);
-                int max = NumericUtils.sortableBytesToInt(maxPackedValue, 0);
-                assert max >= min;
-
-                if (max < queryMin || min > queryMax)
-                {
-                    return Relation.CELL_OUTSIDE_QUERY;
-                }
-                else if (min >= queryMin && max <= queryMax)
-                {
-                    return CELL_INSIDE_QUERY;
-                }
-                else
-                {
-                    return CELL_CROSSES_QUERY;
-                }
-            }
-        };
-    }
-
-    private BlockBalancedTreeReader finishAndOpenReader(int maxPointsPerLeaf, SegmentTrieBuffer buffer) throws Exception
-    {
-        setBDKPostingsWriterSizing(8, 2);
-        final NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
-                                                                 index.identifier(),
-                                                                 maxPointsPerLeaf,
-                                                                 Integer.BYTES);
-
-        final SegmentMetadata.ComponentMetadataMap metadata = writer.writeCompleteSegment(buffer.iterator());
-        final long treePosition = metadata.get(IndexComponent.BALANCED_TREE).root;
-        assertThat(treePosition, is(greaterThan(0L)));
-        final long postingsPosition = metadata.get(IndexComponent.POSTING_LISTS).root;
-        assertThat(postingsPosition, is(greaterThan(0L)));
-
-        FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, index.identifier());
-        FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, index.identifier());
-        return new BlockBalancedTreeReader(index.identifier(),
-                                           treeHandle,
-                                           treePosition,
-                                           treePostingsHandle,
-                                           postingsPosition);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeTest.java
deleted file mode 100644
index 34e13a15daf2..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.util.function.IntFunction;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentTrieBuffer;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.ByteBuffersIndexOutput;
-import org.apache.lucene.store.DataInput;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class BlockBalancedTreeTest extends SAIRandomizedTester
-{
-    private ByteBuffersDataOutput dataOutput;
-
-    @Before
-    public void setupDataOutput()
-    {
-        dataOutput = new ByteBuffersDataOutput();
-    }
-
-    @Test
-    public void testEmptyTree() throws Exception
-    {
-        long treeFilePointer = writeBalancedTree(0, 100, rowID -> rowID);
-
-        assertEquals(-1, treeFilePointer);
-    }
-
-    @Test
-    public void testSingleLeaf() throws Exception
-    {
-        try (BlockBalancedTreeWalker walker = generateBalancedTree(100, 100, rowID -> rowID))
-        {
-            assertEquals(1, walker.numLeaves);
-            assertEquals(1, walker.treeDepth);
-            assertEquals(100, walker.valueCount);
-
-            BlockBalancedTreeWalker.TraversalState state = walker.newTraversalState();
-
-            assertTrue(state.atLeafNode());
-
-            recursiveAssertTraversal(state, -1);
-
-            assertEquals(walker.treeDepth, state.maxLevel + 1);
-        }
-    }
-
-    @Test
-    public void testTreeWithSameValue() throws Exception
-    {
-        try (BlockBalancedTreeWalker walker = generateBalancedTree(100, 4, rowID -> 1))
-        {
-            BlockBalancedTreeWalker.TraversalState state = walker.newTraversalState();
-
-            recursiveAssertTraversal(state, -1);
-
-            assertEquals(walker.treeDepth, state.maxLevel + 1);
-        }
-    }
-
-    @Test
-    public void testTreeDepthNeverMoreThanNumberOfLeaves() throws Exception
-    {
-        int leafSize = 4;
-        for (int numLeaves = 1; numLeaves < 1000; numLeaves++)
-        {
-            int numRows = leafSize * numLeaves;
-
-            try (BlockBalancedTreeWalker walker = generateBalancedTree(numRows, leafSize, rowID -> rowID))
-            {
-                assertEquals(numLeaves, walker.numLeaves);
-                assertTrue(walker.treeDepth <= walker.numLeaves);
-
-                BlockBalancedTreeWalker.TraversalState state = walker.newTraversalState();
-
-                recursiveAssertTraversal(state, -1);
-
-                assertEquals(walker.treeDepth, state.maxLevel + 1);
-            }
-        }
-    }
-
-    @Test
-    public void randomisedTreeTest() throws Exception
-    {
-        int loops = nextInt(10, 1000);
-
-        for (int loop = 0; loop < loops; loop++)
-        {
-            int leafSize = nextInt(2, 512);
-            int numRows = nextInt(1000, 10000);
-
-            try (BlockBalancedTreeWalker walker = generateBalancedTree(numRows, leafSize, rowID -> nextInt(0, numRows / 2)))
-            {
-                BlockBalancedTreeWalker.TraversalState state = walker.newTraversalState();
-
-                recursiveAssertTraversal(state, -1);
-
-                assertEquals(walker.treeDepth, state.maxLevel + 1);
-            }
-        }
-    }
-
-    private long recursiveAssertTraversal(BlockBalancedTreeWalker.TraversalState state, long lastLeafBlockFP)
-    {
-        if (state.atLeafNode())
-        {
-            assertTrue(state.nodeExists());
-            assertTrue(state.getLeafBlockFP() > lastLeafBlockFP);
-            return state.getLeafBlockFP();
-        }
-        else
-        {
-            state.pushLeft();
-            lastLeafBlockFP = recursiveAssertTraversal(state, lastLeafBlockFP);
-            state.pop();
-
-            state.pushRight();
-            lastLeafBlockFP = recursiveAssertTraversal(state, lastLeafBlockFP);
-            state.pop();
-
-            return lastLeafBlockFP;
-        }
-    }
-
-    private BlockBalancedTreeWalker generateBalancedTree(int numRows, int leafSize, IntFunction<Integer> valueProvider) throws Exception
-    {
-        long treeOffset = writeBalancedTree(numRows, leafSize, valueProvider);
-
-        DataInput input = dataOutput.toDataInput();
-
-        return new BlockBalancedTreeWalker(input, treeOffset);
-    }
-
-    private long writeBalancedTree(int numRows, int leafSize, IntFunction<Integer> valueProvider) throws Exception
-    {
-        SegmentTrieBuffer buffer = new SegmentTrieBuffer();
-
-        for (int rowId = 0; rowId < numRows; rowId++)
-        {
-            buffer.add(integerToByteComparable(valueProvider.apply(rowId)), Integer.BYTES, rowId);
-        }
-
-        BlockBalancedTreeWriter writer = new BlockBalancedTreeWriter(4, leafSize);
-        ByteBuffersIndexOutput output = new ByteBuffersIndexOutput(dataOutput, "test", "test");
-        return writer.write(output, buffer.iterator(), (leafPostings, offset, count) -> {});
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/LeafOrderMapTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/LeafOrderMapTest.java
deleted file mode 100644
index 86e860848af1..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/LeafOrderMapTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.nio.ByteBuffer;
-
-import com.google.common.collect.Lists;
-import org.junit.Test;
-
-import org.apache.cassandra.index.sai.disk.ResettableByteBuffersIndexOutput;
-import org.apache.cassandra.index.sai.disk.io.SeekingRandomAccessInput;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.lucene.store.ByteBuffersDataInput;
-import org.apache.lucene.store.ByteBuffersIndexInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.LongValues;
-import org.apache.lucene.util.packed.DirectReader;
-import org.apache.lucene.util.packed.DirectWriter;
-
-import static org.junit.Assert.assertEquals;
-
-public class LeafOrderMapTest extends SAIRandomizedTester
-{
-    @Test
-    public void test() throws Exception
-    {
-        int[] array = new int[1024];
-        for (int x=0; x < array.length; x++)
-        {
-            array[x] = x;
-        }
-        shuffle(array);
-
-        ResettableByteBuffersIndexOutput out = new ResettableByteBuffersIndexOutput("");
-
-        LeafOrderMap.write(array, array.length, array.length - 1, out);
-
-        IndexInput input = new ByteBuffersIndexInput(new ByteBuffersDataInput(Lists.newArrayList(ByteBuffer.wrap(out.toArrayCopy()))), "");
-
-        final byte bits = (byte) DirectWriter.unsignedBitsRequired(array.length - 1);
-
-        for (int index = 0; index < array.length; index++)
-        {
-            LongValues reader = DirectReader.getInstance(new SeekingRandomAccessInput(input), bits);
-
-            int value = Math.toIntExact(reader.get(index));
-
-            assertEquals(array[index], value);
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriterTest.java
deleted file mode 100644
index a6bb3a35d2b1..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriterTest.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.bbtree;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import com.carrotsearch.hppc.LongArrayList;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentTrieBuffer;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
-import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.cassandra.index.sai.utils.TermsIterator;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.utils.AbstractGuavaIterator;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.util.Counter;
-import org.apache.lucene.util.NumericUtils;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class NumericIndexWriterTest extends SAIRandomizedTester
-{
-    private IndexDescriptor indexDescriptor;
-    private IndexTermType indexTermType;
-    private IndexIdentifier indexIdentifier;
-
-    @Before
-    public void setup() throws Throwable
-    {
-        indexDescriptor = newIndexDescriptor();
-        indexTermType = SAITester.createIndexTermType(Int32Type.instance);
-        indexIdentifier = SAITester.createIndexIdentifier("test", "test", newIndex());
-    }
-
-    @Test
-    public void shouldFlushFromRamBuffer() throws Exception
-    {
-        final SegmentTrieBuffer ramBuffer = new SegmentTrieBuffer();
-        final int numRows = 120;
-        int currentValue = numRows;
-        for (int i = 0; i < numRows; ++i)
-        {
-            ramBuffer.add(integerToByteComparable(currentValue--), Integer.BYTES, i);
-        }
-
-        SegmentMetadata.ComponentMetadataMap indexMetas;
-
-        NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
-                                                           indexIdentifier,
-                                                           Integer.BYTES);
-        indexMetas = writer.writeCompleteSegment(ramBuffer.iterator());
-
-        final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexIdentifier, null);
-        final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
-
-        try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexIdentifier,
-                                                                          treeHandle,
-                                                                          indexMetas.get(IndexComponent.BALANCED_TREE).root,
-                                                                          treePostingsHandle,
-                                                                          indexMetas.get(IndexComponent.POSTING_LISTS).root))
-        {
-            final Counter visited = Counter.newCounter();
-            try (final PostingList ignored = reader.intersect(new BlockBalancedTreeReader.IntersectVisitor()
-            {
-                @Override
-                public boolean contains(byte[] packedValue)
-                {
-                    // we should read point values in reverse order after sorting
-                    assertEquals(1 + visited.get(), NumericUtils.sortableBytesToInt(packedValue, 0));
-                    visited.addAndGet(1);
-                    return true;
-                }
-
-                @Override
-                public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
-                {
-                    return PointValues.Relation.CELL_CROSSES_QUERY;
-                }
-            }, mockEventListener(), mock(QueryContext.class)))
-            {
-                assertEquals(numRows, visited.get());
-            }
-        }
-    }
-
-    @Test
-    public void shouldFlushFromMemtable() throws Exception
-    {
-        final int maxSegmentRowId = 100;
-        final TermsIterator termEnum = buildTermEnum(0, maxSegmentRowId);
-
-        SegmentMetadata.ComponentMetadataMap indexMetas;
-        NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
-                                                           indexIdentifier,
-                                                           indexTermType.fixedSizeOf());
-        indexMetas = writer.writeCompleteSegment(termEnum);
-
-        final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexIdentifier, null);
-        final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
-
-        try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexIdentifier,
-                                                                          treeHandle,
-                                                                          indexMetas.get(IndexComponent.BALANCED_TREE).root,
-                                                                          treePostingsHandle,
-                                                                          indexMetas.get(IndexComponent.POSTING_LISTS).root
-        ))
-        {
-            final Counter visited = Counter.newCounter();
-            try (final PostingList ignored = reader.intersect(new BlockBalancedTreeReader.IntersectVisitor()
-            {
-                @Override
-                public boolean contains(byte[] packedValue)
-                {
-                    final ByteComparable actualTerm = ByteComparable.fixedLength(packedValue);
-                    final ByteComparable expectedTerm = ByteComparable.of(Math.toIntExact(visited.get()));
-                    assertEquals("Point value mismatch after visiting " + visited.get() + " entries.", 0,
-                                 ByteComparable.compare(actualTerm, expectedTerm, ByteComparable.Version.OSS50));
-
-                    visited.addAndGet(1);
-                    return true;
-                }
-
-                @Override
-                public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
-                {
-                    return PointValues.Relation.CELL_CROSSES_QUERY;
-                }
-            }, mockEventListener(), mock(QueryContext.class)))
-            {
-                assertEquals(maxSegmentRowId, visited.get());
-            }
-        }
-    }
-
-    private QueryEventListener.BalancedTreeEventListener mockEventListener()
-    {
-        QueryEventListener.BalancedTreeEventListener balancedTreeEventListener = mock(QueryEventListener.BalancedTreeEventListener.class);
-        when(balancedTreeEventListener.postingListEventListener()).thenReturn(mock(QueryEventListener.PostingListEventListener.class));
-        return balancedTreeEventListener;
-    }
-
-    private TermsIterator buildTermEnum(int startTermInclusive, int endTermExclusive)
-    {
-        final ByteBuffer minTerm = Int32Type.instance.decompose(startTermInclusive);
-        final ByteBuffer maxTerm = Int32Type.instance.decompose(endTermExclusive);
-
-        final AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>> iterator = new AbstractGuavaIterator<>()
-        {
-            private int currentTerm = startTermInclusive;
-            private int currentRowId = 0;
-
-            @Override
-            protected Pair<ByteComparable, LongArrayList> computeNext()
-            {
-                if (currentTerm >= endTermExclusive)
-                {
-                    return endOfData();
-                }
-                final ByteBuffer term = Int32Type.instance.decompose(currentTerm++);
-                final LongArrayList postings = new LongArrayList();
-                postings.add(currentRowId++);
-                final ByteSource encoded = Int32Type.instance.asComparableBytes(term, ByteComparable.Version.OSS50);
-                return Pair.create(v -> encoded, postings);
-            }
-        };
-
-        return new MemtableTermsIterator(minTerm, maxTerm, iterator);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesTest.java
index d503f3449099..0ded82ceff2e 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesTest.java
@@ -17,21 +17,22 @@
  */
 package org.apache.cassandra.index.sai.disk.v1.bitpack;
 
+
 import java.util.function.LongFunction;
 
 import org.junit.Test;
 
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
 import org.apache.cassandra.index.sai.disk.format.IndexComponent;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
 import org.apache.cassandra.index.sai.disk.v1.LongArray;
 import org.apache.cassandra.index.sai.disk.v1.MetadataSource;
 import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
 import org.apache.cassandra.io.util.FileHandle;
 
-import static org.junit.Assert.assertEquals;
-
-public class NumericValuesTest extends SAIRandomizedTester
+public class NumericValuesTest extends SaiRandomizedTest
 {
     @Test
     public void testMonotonic() throws Exception
@@ -63,11 +64,14 @@ private void testRepeatedNumericValues(boolean monotonic) throws Exception
         final IndexDescriptor indexDescriptor = newIndexDescriptor();
         writeTokens(monotonic, indexDescriptor, new long[length], prev -> 1000L);
 
-        final MetadataSource source = MetadataSource.loadGroupMetadata(indexDescriptor);
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        final MetadataSource source = MetadataSource.loadMetadata(components);
+
+        IndexComponent.ForRead tokens = components.get(IndexComponentType.TOKEN_VALUES);
 
-        NumericValuesMeta tokensMeta = new NumericValuesMeta(source.get(indexDescriptor.componentName(IndexComponent.ROW_TO_TOKEN)));
+        NumericValuesMeta tokensMeta = new NumericValuesMeta(source.get(tokens));
 
-        try (FileHandle fileHandle = indexDescriptor.createPerSSTableFileHandle(IndexComponent.ROW_TO_TOKEN, null);
+        try (FileHandle fileHandle = tokens.createFileHandle();
              LongArray reader = monotonic ? new MonotonicBlockPackedReader(fileHandle, tokensMeta).open()
                                           : new BlockPackedReader(fileHandle, tokensMeta).open())
         {
@@ -78,16 +82,86 @@ private void testRepeatedNumericValues(boolean monotonic) throws Exception
         }
     }
 
+    @Test
+    public void testRepeatsRegularValuesFindTokenRowID() throws Exception
+    {
+        testRepeatedNumericValuesFindTokenRowID();
+    }
+
+    @Test
+    public void testTokenFind() throws Exception
+    {
+        final long[] array = new long[64_000];
+        final IndexDescriptor indexDescriptor = newIndexDescriptor();
+        writeTokens(false, indexDescriptor, array, prev -> prev + nextInt(2, 100));
+
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        final MetadataSource source = MetadataSource.loadMetadata(components);
+        IndexComponent.ForRead tokens = components.get(IndexComponentType.TOKEN_VALUES);
+        NumericValuesMeta tokensMeta = new NumericValuesMeta(source.get(tokens));
+
+        try (FileHandle fileHandle = tokens.createFileHandle();
+             LongArray reader = new BlockPackedReader(fileHandle, tokensMeta).open())
+        {
+            assertEquals(array.length, reader.length());
+
+            for (int x = 0; x < array.length; x++)
+            {
+                long rowId = reader.ceilingRowId(array[x]);
+                assertEquals("rowID=" + x + " token=" + array[x], x, rowId);
+                assertEquals(rowId, reader.ceilingRowId(array[x]));
+            }
+        }
+
+        // non-exact match
+        try (FileHandle fileHandle = tokens.createFileHandle();
+             LongArray reader = new BlockPackedReader(fileHandle, tokensMeta).open())
+        {
+            assertEquals(array.length, reader.length());
+
+            for (int x = 0; x < array.length; x++)
+            {
+                long rowId = reader.ceilingRowId(array[x] - 1);
+                assertEquals("rowID=" + x + " matched token=" + array[x] + " target token="+(array[x] - 1), x, rowId);
+                assertEquals(rowId, reader.ceilingRowId(array[x] - 1));
+            }
+        }
+    }
+
+    private void testRepeatedNumericValuesFindTokenRowID() throws Exception
+    {
+        int length = 64_000;
+        final IndexDescriptor indexDescriptor = newIndexDescriptor();
+        writeTokens(false, indexDescriptor, new long[length], prev -> 1000L);
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        final MetadataSource source = MetadataSource.loadMetadata(components);
+        IndexComponent.ForRead tokens = components.get(IndexComponentType.TOKEN_VALUES);
+        NumericValuesMeta tokensMeta = new NumericValuesMeta(source.get(tokens));
+
+        try (FileHandle fileHandle = tokens.createFileHandle();
+             LongArray reader = new BlockPackedReader(fileHandle, tokensMeta).open())
+        {
+            for (int x = 0; x < length; x++)
+            {
+                long rowID = reader.ceilingRowId(1000L);
+
+                assertEquals(0, rowID);
+            }
+        }
+    }
+
     private void doTest(boolean monotonic) throws Exception
     {
         final long[] array = new long[64_000];
         final IndexDescriptor indexDescriptor = newIndexDescriptor();
-        writeTokens(monotonic, indexDescriptor, array, prev -> monotonic ? prev + nextInt(100) : nextLong(0, Long.MAX_VALUE));
+        writeTokens(monotonic, indexDescriptor, array, prev -> monotonic ? prev + nextInt(100) : nextInt(100));
 
-        final MetadataSource source = MetadataSource.loadGroupMetadata(indexDescriptor);
-        NumericValuesMeta tokensMeta = new NumericValuesMeta(source.get(indexDescriptor.componentName(IndexComponent.ROW_TO_TOKEN)));
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        final MetadataSource source = MetadataSource.loadMetadata(components);
+        IndexComponent.ForRead tokens = components.get(IndexComponentType.TOKEN_VALUES);
+        NumericValuesMeta tokensMeta = new NumericValuesMeta(source.get(tokens));
 
-        try (FileHandle fileHandle = indexDescriptor.createPerSSTableFileHandle(IndexComponent.ROW_TO_TOKEN, null);
+        try (FileHandle fileHandle = tokens.createFileHandle();
              LongArray reader = (monotonic ? new MonotonicBlockPackedReader(fileHandle, tokensMeta)
                                            : new BlockPackedReader(fileHandle, tokensMeta)).open())
         {
@@ -105,9 +179,9 @@ private void writeTokens(boolean monotonic, IndexDescriptor indexDescriptor, lon
         final int blockSize = 1 << nextInt(8, 15);
 
         long current = 0;
-        try (MetadataWriter metadataWriter = new MetadataWriter(indexDescriptor.openPerSSTableOutput(IndexComponent.GROUP_META));
-             final NumericValuesWriter numericWriter = new NumericValuesWriter(indexDescriptor,
-                                                                               IndexComponent.ROW_TO_TOKEN,
+        IndexComponents.ForWrite components = indexDescriptor.newPerSSTableComponentsForWrite();
+        try (MetadataWriter metadataWriter = new MetadataWriter(components);
+             final NumericValuesWriter numericWriter = new NumericValuesWriter(components.addOrGet(IndexComponentType.TOKEN_VALUES),
                                                                                metadataWriter,
                                                                                monotonic,
                                                                                blockSize))
@@ -121,5 +195,6 @@ private void writeTokens(boolean monotonic, IndexDescriptor indexDescriptor, lon
                 array[x] = current;
             }
         }
+        components.markComplete();
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDQueriesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDQueriesTest.java
new file mode 100644
index 000000000000..dd002a88dcc6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDQueriesTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+import static org.apache.lucene.index.PointValues.Relation.CELL_CROSSES_QUERY;
+import static org.apache.lucene.index.PointValues.Relation.CELL_INSIDE_QUERY;
+import static org.apache.lucene.index.PointValues.Relation.CELL_OUTSIDE_QUERY;
+
+public class BKDQueriesTest extends SaiRandomizedTest
+{
+    @Test
+    public void testInclusiveLowerBound()
+    {
+        final int lowerBound = between(-10, 10);
+        final Expression expression = buildExpression(Operator.GTE, lowerBound);
+        final BKDReader.IntersectVisitor query = BKDQueries.bkdQueryFrom(expression, 1, 4);
+
+        assertFalse(query.visit(toSortableBytes(lowerBound - 1)));
+        assertTrue(query.visit(toSortableBytes(lowerBound)));
+        assertTrue(query.visit(toSortableBytes(lowerBound + 1)));
+
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 2), toSortableBytes(lowerBound - 1)));
+        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(lowerBound + 1)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
+    }
+
+    @Test
+    public void testExclusiveLowerBound()
+    {
+        final int lowerBound = between(-10, 10);
+        final Expression expression = buildExpression(Operator.GT, lowerBound);
+        final BKDReader.IntersectVisitor query = BKDQueries.bkdQueryFrom(expression, 1, 4);
+
+        assertFalse(query.visit(toSortableBytes(lowerBound - 1)));
+        assertFalse(query.visit(toSortableBytes(lowerBound)));
+        assertTrue(query.visit(toSortableBytes(lowerBound + 1)));
+
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
+        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound + 1), toSortableBytes(lowerBound + 2)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(lowerBound + 1)));
+    }
+
+    @Test
+    public void testInclusiveUpperBound()
+    {
+        final int upperBound = between(-10, 10);
+        final Expression expression = buildExpression(Operator.LTE, upperBound);
+        final BKDReader.IntersectVisitor query = BKDQueries.bkdQueryFrom(expression, 1, 4);
+
+        assertTrue(query.visit(toSortableBytes(upperBound - 1)));
+        assertTrue(query.visit(toSortableBytes(upperBound)));
+        assertFalse(query.visit(toSortableBytes(upperBound + 1)));
+
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound + 1), toSortableBytes(upperBound + 2)));
+        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(upperBound - 1), toSortableBytes(upperBound)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
+    }
+
+    @Test
+    public void testExclusiveUpperBound()
+    {
+        final int upper = between(-10, 10);
+        final Expression expression = buildExpression(Operator.LT, upper);
+        final BKDReader.IntersectVisitor query = BKDQueries.bkdQueryFrom(expression, 1, 4);
+
+        assertTrue(query.visit(toSortableBytes(upper - 1)));
+        assertFalse(query.visit(toSortableBytes(upper)));
+        assertFalse(query.visit(toSortableBytes(upper + 1)));
+
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upper), toSortableBytes(upper + 1)));
+        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(upper - 2), toSortableBytes(upper - 1)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upper - 1), toSortableBytes(upper)));
+    }
+
+    @Test
+    public void testInclusiveLowerAndUpperBound()
+    {
+        final int lowerBound = between(-15, 15);
+        final int upperBound = lowerBound + 5;
+        final Expression expression = buildExpression(Operator.GTE, lowerBound)
+                .add(Operator.LTE, Int32Type.instance.decompose(upperBound));
+        final BKDReader.IntersectVisitor query = BKDQueries.bkdQueryFrom(expression, 1, 4);
+
+        assertFalse(query.visit(toSortableBytes(lowerBound - 1)));
+        assertTrue(query.visit(toSortableBytes(lowerBound)));
+        assertTrue(query.visit(toSortableBytes(lowerBound + 1)));
+        assertTrue(query.visit(toSortableBytes(upperBound - 1)));
+        assertTrue(query.visit(toSortableBytes(upperBound)));
+        assertFalse(query.visit(toSortableBytes(upperBound + 1)));
+
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 2), toSortableBytes(lowerBound - 1)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
+        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(upperBound)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound + 1), toSortableBytes(upperBound + 2)));
+    }
+
+    @Test
+    public void testExclusiveLowerAndUpperBound()
+    {
+        final int lowerBound = between(-15, 15);
+        final int upperBound = lowerBound + 5;
+        final Expression expression = buildExpression(Operator.GT, lowerBound)
+                .add(Operator.LT, Int32Type.instance.decompose(upperBound));
+        final BKDReader.IntersectVisitor query = BKDQueries.bkdQueryFrom(expression, 1, 4);
+
+        assertFalse(query.visit(toSortableBytes(lowerBound - 1)));
+        assertFalse(query.visit(toSortableBytes(lowerBound)));
+        assertTrue(query.visit(toSortableBytes(lowerBound + 1)));
+        assertTrue(query.visit(toSortableBytes(upperBound - 1)));
+        assertFalse(query.visit(toSortableBytes(upperBound)));
+        assertFalse(query.visit(toSortableBytes(upperBound + 1)));
+
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(lowerBound + 1)));
+        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound + 1), toSortableBytes(upperBound - 1)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound - 1), toSortableBytes(upperBound)));
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
+    }
+
+    @Test
+    public void testExclusiveLowerAndInclusiveUpperBound()
+    {
+        final int lowerBound = between(-15, 15);
+        final int upperBound = lowerBound + 5;
+        final Expression expression = buildExpression(Operator.GT, lowerBound)
+                .add(Operator.LTE, Int32Type.instance.decompose(upperBound));
+        final BKDReader.IntersectVisitor query = BKDQueries.bkdQueryFrom(expression, 1, 4);
+
+        assertFalse(query.visit(toSortableBytes(lowerBound - 1)));
+        assertFalse(query.visit(toSortableBytes(lowerBound)));
+        assertTrue(query.visit(toSortableBytes(lowerBound + 1)));
+        assertTrue(query.visit(toSortableBytes(upperBound - 1)));
+        assertTrue(query.visit(toSortableBytes(upperBound)));
+        assertFalse(query.visit(toSortableBytes(upperBound + 1)));
+
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(lowerBound + 1)));
+        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound + 1), toSortableBytes(upperBound)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound + 1), toSortableBytes(upperBound + 2)));
+    }
+
+    @Test
+    public void testInclusiveLowerAndExclusiveUpperBound()
+    {
+        final int lowerBound = between(-15, 15);
+        final int upperBound = lowerBound + 5;
+        final Expression expression = buildExpression(Operator.GTE, lowerBound)
+                .add(Operator.LT, Int32Type.instance.decompose(upperBound));
+        final BKDReader.IntersectVisitor query = BKDQueries.bkdQueryFrom(expression, 1, 4);
+
+        assertFalse(query.visit(toSortableBytes(lowerBound - 1)));
+        assertTrue(query.visit(toSortableBytes(lowerBound)));
+        assertTrue(query.visit(toSortableBytes(lowerBound + 1)));
+        assertTrue(query.visit(toSortableBytes(upperBound - 1)));
+        assertFalse(query.visit(toSortableBytes(upperBound)));
+        assertFalse(query.visit(toSortableBytes(upperBound + 1)));
+
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(lowerBound - 2), toSortableBytes(lowerBound - 1)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(lowerBound - 1), toSortableBytes(lowerBound)));
+        assertEquals(CELL_INSIDE_QUERY, query.compare(toSortableBytes(lowerBound), toSortableBytes(upperBound - 1)));
+        assertEquals(CELL_CROSSES_QUERY, query.compare(toSortableBytes(upperBound - 1), toSortableBytes(upperBound)));
+        assertEquals(CELL_OUTSIDE_QUERY, query.compare(toSortableBytes(upperBound), toSortableBytes(upperBound + 1)));
+    }
+
+    private byte[] toSortableBytes(int value)
+    {
+        byte[] buffer = new byte[4];
+        ByteSource source = Int32Type.instance.asComparableBytes(Int32Type.instance.decompose(value), TypeUtil.BYTE_COMPARABLE_VERSION);
+        ByteSourceInverse.readBytesMustFit(source, buffer);
+        return buffer;
+    }
+
+    private Expression buildExpression(Operator op, int value)
+    {
+        final Expression expression = new Expression(SAITester.createIndexContext("meh", Int32Type.instance));
+        expression.add(op, Int32Type.instance.decompose(value));
+        return expression;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDReaderTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDReaderTest.java
new file mode 100644
index 000000000000..104adda390ff
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDReaderTest.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
+
+import static org.apache.cassandra.index.sai.metrics.QueryEventListeners.NO_OP_BKD_LISTENER;
+import static org.apache.lucene.index.PointValues.Relation.CELL_CROSSES_QUERY;
+import static org.apache.lucene.index.PointValues.Relation.CELL_INSIDE_QUERY;
+import static org.apache.lucene.index.PointValues.Relation.CELL_OUTSIDE_QUERY;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+
+public class BKDReaderTest extends SaiRandomizedTest
+{
+    private final BKDReader.IntersectVisitor NONE_MATCH = new BKDReader.IntersectVisitor()
+    {
+        @Override
+        public boolean visit(byte[] packedValue)
+        {
+            return false;
+        }
+
+        @Override
+        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
+        {
+            return CELL_OUTSIDE_QUERY;
+        }
+    };
+
+    private final BKDReader.IntersectVisitor ALL_MATCH = new BKDReader.IntersectVisitor()
+    {
+        @Override
+        public boolean visit(byte[] packedValue)
+        {
+            return true;
+        }
+
+        @Override
+        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
+        {
+            return CELL_INSIDE_QUERY;
+        }
+    };
+
+    private final BKDReader.IntersectVisitor ALL_MATCH_WITH_FILTERING = new BKDReader.IntersectVisitor()
+    {
+        @Override
+        public boolean visit(byte[] packedValue)
+        {
+            return true;
+        }
+
+        @Override
+        public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
+        {
+            return CELL_CROSSES_QUERY;
+        }
+    };
+
+    private IndexDescriptor indexDescriptor;
+    private String index;
+    private IndexContext indexContext;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        indexDescriptor = newIndexDescriptor();
+        index = newIndex();
+        indexContext = SAITester.createIndexContext(index, Int32Type.instance);
+    }
+
+    @Test
+    public void testInts1D() throws IOException
+    {
+        doTestInts1D();
+    }
+
+    private void doTestInts1D() throws IOException
+    {
+        final int numRows = between(100, 400);
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+
+        byte[] scratch = new byte[4];
+        for (int docID = 0; docID < numRows; docID++)
+        {
+            NumericUtils.intToSortableBytes(docID, scratch, 0);
+            buffer.addPackedValue(docID, new BytesRef(scratch));
+        }
+
+        final BKDReader reader = finishAndOpenReaderOneDim(2, buffer);
+
+        try (BKDReader.IteratorState iterator = reader.iteratorState())
+        {
+            while (iterator.hasNext())
+            {
+                int value = NumericUtils.sortableBytesToInt(iterator.scratch, 0);
+                System.out.println("term=" + value);
+                iterator.next();
+            }
+        }
+
+        try (PostingList intersection = reader.intersect(NONE_MATCH, (QueryEventListener.BKDIndexEventListener)NO_OP_BKD_LISTENER, new QueryContext()))
+        {
+            assertEquals(PostingList.EMPTY, intersection);
+        }
+
+        try (PostingList collectAllIntersection = reader.intersect(ALL_MATCH, (QueryEventListener.BKDIndexEventListener)NO_OP_BKD_LISTENER, new QueryContext());
+             PostingList filteringIntersection = reader.intersect(ALL_MATCH_WITH_FILTERING, (QueryEventListener.BKDIndexEventListener)NO_OP_BKD_LISTENER, new QueryContext()))
+        {
+            assertEquals(numRows, collectAllIntersection.size());
+            assertEquals(numRows, filteringIntersection.size());
+
+            for (int docID = 0; docID < numRows; docID++)
+            {
+                assertEquals(docID, collectAllIntersection.nextPosting());
+                assertEquals(docID, filteringIntersection.nextPosting());
+            }
+
+            assertEquals(PostingList.END_OF_STREAM, collectAllIntersection.nextPosting());
+            assertEquals(PostingList.END_OF_STREAM, filteringIntersection.nextPosting());
+        }
+
+        // Simple 1D range query:
+        final int queryMin = 42;
+        final int queryMax = 87;
+
+        final PostingList intersection = reader.intersect(buildQuery(queryMin, queryMax), (QueryEventListener.BKDIndexEventListener)NO_OP_BKD_LISTENER, new QueryContext());
+
+        long expectedRowID = queryMin;
+        for (long id = intersection.nextPosting(); id != PostingList.END_OF_STREAM; id = intersection.nextPosting())
+        {
+            assertEquals(expectedRowID++, id);
+        }
+        assertEquals(queryMax - queryMin + 1, intersection.size());
+
+        intersection.close();
+        reader.close();
+    }
+
+    @Test
+    public void testForwardIteration() throws IOException
+    {
+        final int numRows = between(100, 400);
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+
+        byte[] scratch = new byte[4];
+        for (int docID = 0; docID < numRows; docID++)
+        {
+            NumericUtils.intToSortableBytes(docID, scratch, 0);
+            buffer.addPackedValue(docID, new BytesRef(scratch));
+        }
+
+        try (BKDReader reader = finishAndOpenReaderOneDim(2, buffer);
+             BKDReader.IteratorState iterator = reader.iteratorState(BKDReader.Direction.FORWARD, null))
+        {
+            int docId = 0;
+            while (iterator.hasNext())
+            {
+                int value = NumericUtils.sortableBytesToInt(iterator.scratch, 0);
+                assertEquals(docId++, value);
+                iterator.next();
+            }
+            assertEquals(numRows, docId);
+        }
+    }
+
+    @Test
+    public void testForwardIterationWithQuery() throws IOException
+    {
+        final int numRows = between(100, 400);
+        final int queryMin = between(2, 20);
+        final int queryMax = between(numRows - 20, numRows - 2);
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+
+        byte[] scratch = new byte[4];
+        for (int docID = 0; docID < numRows; docID++)
+        {
+            NumericUtils.intToSortableBytes(docID, scratch, 0);
+            buffer.addPackedValue(docID, new BytesRef(scratch));
+        }
+
+        try (BKDReader reader = finishAndOpenReaderOneDim(2, buffer);
+             BKDReader.IteratorState iterator = reader.iteratorState(BKDReader.Direction.FORWARD, buildQuery(queryMin, queryMax)))
+        {
+            int docId = Math.max(0, queryMin);
+            while (iterator.hasNext())
+            {
+                int value = NumericUtils.sortableBytesToInt(iterator.scratch, 0);
+                assertEquals(docId++, value);
+                iterator.next();
+            }
+            assertEquals(Math.min(queryMax + 1, numRows), docId);
+        }
+    }
+
+
+    @Test
+    public void testBackwardIteration() throws IOException
+    {
+        final int numRows = between(100, 400);
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+
+        byte[] scratch = new byte[4];
+        for (int docID = 0; docID < numRows; docID++)
+        {
+            NumericUtils.intToSortableBytes(docID, scratch, 0);
+            buffer.addPackedValue(docID, new BytesRef(scratch));
+        }
+
+        try (BKDReader reader = finishAndOpenReaderOneDim(2, buffer);
+             BKDReader.IteratorState iterator = reader.iteratorState(BKDReader.Direction.BACKWARD, null))
+        {
+            int docId = numRows;
+            while (iterator.hasNext())
+            {
+                int value = NumericUtils.sortableBytesToInt(iterator.scratch, 0);
+                assertEquals(--docId, value);
+                iterator.next();
+            }
+            assertEquals(0, docId);
+        }
+
+    }
+
+    @Test
+    public void testBackwardIterationWithQuery() throws IOException
+    {
+        final int numRows = between(100, 400);
+        final int queryMin = between(2, 20);
+        final int queryMax = between(numRows - 20, numRows - 2);
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+
+        byte[] scratch = new byte[4];
+        for (int docID = 0; docID < numRows; docID++)
+        {
+            NumericUtils.intToSortableBytes(docID, scratch, 0);
+            buffer.addPackedValue(docID, new BytesRef(scratch));
+        }
+
+        try (BKDReader reader = finishAndOpenReaderOneDim(2, buffer);
+             BKDReader.IteratorState iterator = reader.iteratorState(BKDReader.Direction.BACKWARD, buildQuery(queryMin, queryMax)))
+        {
+            int docId = Math.min(numRows, queryMax);
+            while (iterator.hasNext())
+            {
+                int value = NumericUtils.sortableBytesToInt(iterator.scratch, 0);
+                assertEquals(docId--, value);
+                iterator.next();
+            }
+            assertEquals(Math.max(queryMin - 1, 0), docId);
+        }
+    }
+
+    @Test
+    public void testAdvance() throws IOException
+    {
+        doTestAdvance(false);
+    }
+
+    @Test
+    public void testAdvanceCrypto() throws IOException
+    {
+        doTestAdvance(true);
+    }
+
+    private void doTestAdvance(boolean crypto) throws IOException
+    {
+        final int numRows = between(1000, 2000);
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+
+        byte[] scratch = new byte[4];
+        for (int docID = 0; docID < numRows; docID++)
+        {
+            NumericUtils.intToSortableBytes(docID, scratch, 0);
+            buffer.addPackedValue(docID, new BytesRef(scratch));
+        }
+
+        final BKDReader reader = finishAndOpenReaderOneDim(2, buffer);
+
+        PostingList intersection = reader.intersect(NONE_MATCH, (QueryEventListener.BKDIndexEventListener)NO_OP_BKD_LISTENER, new QueryContext());
+        assertEquals(PostingList.EMPTY, intersection);
+
+        intersection = reader.intersect(ALL_MATCH, (QueryEventListener.BKDIndexEventListener)NO_OP_BKD_LISTENER, new QueryContext());
+        assertEquals(numRows, intersection.size());
+        assertEquals(100, intersection.advance(100));
+        assertEquals(200, intersection.advance(200));
+        assertEquals(300, intersection.advance(300));
+        assertEquals(400, intersection.advance(400));
+        assertEquals(401, intersection.advance(401));
+        long expectedRowID = 402;
+        for (long id = intersection.nextPosting(); expectedRowID < 500; id = intersection.nextPosting())
+        {
+            assertEquals(expectedRowID++, id);
+        }
+        assertEquals(PostingList.END_OF_STREAM, intersection.advance(numRows + 1));
+
+        intersection.close();
+        reader.close();
+    }
+
+    @Test
+    public void testResourcesReleaseWhenQueryDoesntMatchAnything() throws Exception
+    {
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+        byte[] scratch = new byte[4];
+        for (int docID = 0; docID < 1000; docID++)
+        {
+            NumericUtils.intToSortableBytes(docID, scratch, 0);
+            buffer.addPackedValue(docID, new BytesRef(scratch));
+        }
+        // add a gap between 1000 and 1100
+        for (int docID = 1000; docID < 2000; docID++)
+        {
+            NumericUtils.intToSortableBytes(docID + 100, scratch, 0);
+            buffer.addPackedValue(docID, new BytesRef(scratch));
+        }
+
+        final BKDReader reader = finishAndOpenReaderOneDim(50, buffer);
+        final PostingList intersection = reader.intersect(buildQuery(1017, 1096), (QueryEventListener.BKDIndexEventListener)NO_OP_BKD_LISTENER, new QueryContext());
+
+        assertEquals(PostingList.EMPTY, intersection);
+
+        intersection.close();
+        reader.close();
+    }
+
+    private BKDReader.IntersectVisitor buildQuery(int queryMin, int queryMax)
+    {
+        return new BKDReader.IntersectVisitor()
+        {
+            @Override
+            public boolean visit(byte[] packedValue)
+            {
+                int x = NumericUtils.sortableBytesToInt(packedValue, 0);
+                boolean bb = x >= queryMin && x <= queryMax;
+                return bb;
+            }
+
+            @Override
+            public Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
+            {
+                int min = NumericUtils.sortableBytesToInt(minPackedValue, 0);
+                int max = NumericUtils.sortableBytesToInt(maxPackedValue, 0);
+                assert max >= min;
+
+                if (max < queryMin || min > queryMax)
+                {
+                    return Relation.CELL_OUTSIDE_QUERY;
+                }
+                else if (min >= queryMin && max <= queryMax)
+                {
+                    return CELL_INSIDE_QUERY;
+                }
+                else
+                {
+                    return CELL_CROSSES_QUERY;
+                }
+            }
+        };
+    }
+
+    private BKDReader finishAndOpenReaderOneDim(int maxPointsPerLeaf, BKDTreeRamBuffer buffer) throws IOException
+    {
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        final NumericIndexWriter writer = new NumericIndexWriter(components,
+                                                                 maxPointsPerLeaf,
+                                                                 Integer.BYTES,
+                                                                 Math.toIntExact(buffer.numRows()),
+                                                                 buffer.numRows(),
+                                                                 new IndexWriterConfig("test", 2, 8));
+
+        final SegmentMetadata.ComponentMetadataMap metadata = writer.writeAll(buffer.asPointValues());
+        final long bkdPosition = metadata.get(IndexComponentType.KD_TREE).root;
+        assertThat(bkdPosition, is(greaterThan(0L)));
+        final long postingsPosition = metadata.get(IndexComponentType.KD_TREE_POSTING_LISTS).root;
+        assertThat(postingsPosition, is(greaterThan(0L)));
+
+        FileHandle kdtreeHandle = components.get(IndexComponentType.KD_TREE).createFileHandle();
+        FileHandle kdtreePostingsHandle = components.get(IndexComponentType.KD_TREE_POSTING_LISTS).createFileHandle();
+        return new BKDReader(indexContext,
+                             kdtreeHandle,
+                             bkdPosition,
+                             kdtreePostingsHandle,
+                             postingsPosition);
+    }
+
+    private BKDReader finishAndOpenReaderOneDim(int maxPointsPerLeaf, MutableOneDimPointValues values, int numRows) throws IOException
+    {
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        final NumericIndexWriter writer = new NumericIndexWriter(components,
+                                                                 maxPointsPerLeaf,
+                                                                 Integer.BYTES,
+                                                                 Math.toIntExact(numRows),
+                                                                 numRows,
+                                                                 new IndexWriterConfig("test", 2, 8));
+
+        final SegmentMetadata.ComponentMetadataMap metadata = writer.writeAll(values);
+        final long bkdPosition = metadata.get(IndexComponentType.KD_TREE).root;
+        assertThat(bkdPosition, is(greaterThan(0L)));
+        final long postingsPosition = metadata.get(IndexComponentType.KD_TREE_POSTING_LISTS).root;
+        assertThat(postingsPosition, is(greaterThan(0L)));
+
+        FileHandle kdtreeHandle = components.get(IndexComponentType.KD_TREE).createFileHandle();
+        FileHandle kdtreePostingsHandle = components.get(IndexComponentType.KD_TREE_POSTING_LISTS).createFileHandle();
+        return new BKDReader(indexContext,
+                             kdtreeHandle,
+                             bkdPosition,
+                             kdtreePostingsHandle,
+                             postingsPosition);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDTreeRamBufferTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDTreeRamBufferTest.java
new file mode 100644
index 000000000000..288fefd78f0d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/BKDTreeRamBufferTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointValues;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointsReaderUtils;
+
+public class BKDTreeRamBufferTest
+{
+    @Test
+    public void shouldKeepInsertionOrder()
+    {
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+        int currentValue = 202;
+        for (int i = 0; i < 100; ++i)
+        {
+            byte[] scratch = new byte[Integer.BYTES];
+            NumericUtils.intToSortableBytes(currentValue--, scratch, 0);
+            buffer.addPackedValue(i, new BytesRef(scratch));
+        }
+
+        final MutablePointValues pointValues = buffer.asPointValues();
+
+        for (int i = 0; i < 100; ++i)
+        {
+            // expect insertion order
+            Assert.assertEquals(i, pointValues.getDocID(i));
+            BytesRef ref = new BytesRef();
+            pointValues.getValue(i, ref);
+            Assert.assertEquals(202 - i, NumericUtils.sortableBytesToInt(ref.bytes, ref.offset));
+        }
+    }
+
+    @Test
+    public void shouldBeSortable()
+    {
+        final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+        int value = 301;
+        for (int i = 0; i < 100; ++i)
+        {
+            byte[] scratch = new byte[Integer.BYTES];
+            NumericUtils.intToSortableBytes(value--, scratch, 0);
+            buffer.addPackedValue(i, new BytesRef(scratch));
+        }
+
+        final MutablePointValues pointValues = buffer.asPointValues();
+
+        MutablePointsReaderUtils.sort(100, Integer.BYTES, pointValues, 0, Math.toIntExact(pointValues.size()));
+
+        for (int i = 0; i < 100; ++i)
+        {
+            // expect reverse order after sorting
+            Assert.assertEquals(99 - i, pointValues.getDocID(i));
+            BytesRef ref = new BytesRef();
+            pointValues.getValue(i, ref);
+            Assert.assertEquals(202 + i, NumericUtils.sortableBytesToInt(ref.bytes, ref.offset));
+        }
+    }
+
+    @Test
+    public void testRequiresFlush()
+    {
+        int maxByteBlockPoolSize = BKDTreeRamBuffer.MAX_BLOCK_BYTE_POOL_SIZE;
+        try
+        {
+            BKDTreeRamBuffer.MAX_BLOCK_BYTE_POOL_SIZE = 1024 * 1024 * 100;
+            // primary behavior we're testing is that exceptions aren't thrown due to overflowing backing structures
+            final BKDTreeRamBuffer buffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+
+            Assert.assertFalse(buffer.requiresFlush());
+            for (int i = 0; i < Integer.MAX_VALUE; i++)
+            {
+                if (buffer.requiresFlush())
+                    break;
+                byte[] scratch = new byte[Integer.BYTES];
+                NumericUtils.intToSortableBytes(i, scratch, 0);
+                buffer.addPackedValue(i, new BytesRef(scratch));
+            }
+            // If we don't require a flush before MAX_VALUE, the implementation of BKDTreeRamBuffer has sufficiently
+            // changed to warrant changes to the test.
+            Assert.assertTrue(buffer.requiresFlush());
+        }
+        finally
+        {
+            BKDTreeRamBuffer.MAX_BLOCK_BYTE_POOL_SIZE = maxByteBlockPoolSize;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValuesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValuesTest.java
new file mode 100644
index 000000000000..59a12871f9e5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValuesTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import org.apache.cassandra.index.sai.memory.RowMapping;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.disk.MemtableTermsIterator;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointsReaderUtils;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.junit.Assert.assertEquals;
+
+public class ImmutableOneDimPointValuesTest
+{
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
+
+    @Test
+    public void shouldTraversePointsInTermEnumOrder() throws IOException
+    {
+        final int minTerm = 0, maxTerm = 10;
+        final TermsIterator termEnum = buildDescTermEnum(minTerm, maxTerm);
+        final ImmutableOneDimPointValues pointValues = ImmutableOneDimPointValues
+                .fromTermEnum(termEnum, Int32Type.instance);
+
+        pointValues.intersect(assertingVisitor(minTerm));
+    }
+
+    @Test
+    public void shouldFailOnSorting()
+    {
+        final int minTerm = 3, maxTerm = 13;
+        final TermsIterator termEnum = buildDescTermEnum(minTerm, maxTerm);
+        final ImmutableOneDimPointValues pointValues = ImmutableOneDimPointValues
+                .fromTermEnum(termEnum, Int32Type.instance);
+
+        expectedException.expect(IllegalStateException.class);
+        pointValues.swap(0, 1);
+    }
+
+    @Test
+    public void shouldSkipLuceneSorting() throws IOException
+    {
+        final int minTerm = 2, maxTerm = 7;
+        final TermsIterator termEnum = buildDescTermEnum(minTerm, maxTerm);
+        final ImmutableOneDimPointValues pointValues = ImmutableOneDimPointValues.fromTermEnum(termEnum, Int32Type.instance);
+
+        MutablePointsReaderUtils.sort(2, Int32Type.instance.valueLengthIfFixed(), pointValues, 0, Math.toIntExact(pointValues.size()));
+
+        pointValues.intersect(assertingVisitor(minTerm));
+    }
+
+    private MutableOneDimPointValues.IntersectVisitor assertingVisitor(int minTerm)
+    {
+        return new MutableOneDimPointValues.IntersectVisitor()
+        {
+            int term = minTerm;
+            int postingCounter = 0;
+
+            @Override
+            public void visit(int docID, byte[] packedValue)
+            {
+                final ByteComparable actualTerm = ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, packedValue);
+                final ByteComparable expectedTerm = ByteComparable.of(term);
+
+                assertEquals(0, ByteComparable.compare(actualTerm, expectedTerm, TypeUtil.BYTE_COMPARABLE_VERSION));
+                assertEquals(postingCounter, docID);
+
+                if (postingCounter >= 2)
+                {
+                    postingCounter = 0;
+                    term++;
+                }
+                else
+                {
+                    postingCounter++;
+                }
+            }
+        };
+    }
+
+    private TermsIterator buildDescTermEnum(int from, int to)
+    {
+        final ByteBuffer minTerm = Int32Type.instance.decompose(from);
+        final ByteBuffer maxTerm = Int32Type.instance.decompose(to);
+
+        final AbstractGuavaIterator<Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>>> iterator = new AbstractGuavaIterator<>()
+        {
+            private int currentTerm = from;
+
+            @Override
+            protected Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>> computeNext()
+            {
+                if (currentTerm <= to)
+                {
+                    return endOfData();
+                }
+                final ByteBuffer term = Int32Type.instance.decompose(currentTerm++);
+                List<RowMapping.RowIdWithFrequency> postings = Arrays.asList(
+                    new RowMapping.RowIdWithFrequency(0, 1),
+                    new RowMapping.RowIdWithFrequency(1, 1),
+                    new RowMapping.RowIdWithFrequency(2, 1));
+                return Pair.create(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, term), postings);
+            }
+        };
+
+        return new MemtableTermsIterator(minTerm, maxTerm, iterator);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/KDTreeIndexBuilder.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/KDTreeIndexBuilder.java
new file mode 100644
index 000000000000..43fa20858e22
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/KDTreeIndexBuilder.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+import java.util.stream.Stream;
+
+import org.junit.Assert;
+
+import com.carrotsearch.hppc.IntArrayList;
+import org.apache.cassandra.index.sai.memory.RowMapping;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.MemtableTermsIterator;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.IndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
+import org.apache.cassandra.index.sai.disk.v1.KDTreeIndexSearcher;
+import org.apache.cassandra.index.sai.disk.v1.PartitionAwarePrimaryKeyFactory;
+import org.apache.cassandra.index.sai.disk.v1.PerIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadataBuilder;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class KDTreeIndexBuilder
+{
+    public static final PrimaryKeyMap TEST_PRIMARY_KEY_MAP = new PrimaryKeyMap()
+    {
+        private final PrimaryKey.Factory primaryKeyFactory = new PartitionAwarePrimaryKeyFactory();
+
+        @Override
+        public SSTableId<?> getSSTableId()
+        {
+            return new SequenceBasedSSTableId(0);
+        }
+
+        @Override
+        public PrimaryKey primaryKeyFromRowId(long sstableRowId)
+        {
+            return primaryKeyFactory.createTokenOnly(new Murmur3Partitioner.LongToken(sstableRowId));
+        }
+
+        @Override
+        public long exactRowIdOrInvertedCeiling(PrimaryKey key)
+        {
+            return key.token().getLongValue();
+        }
+
+        @Override
+        public long ceiling(PrimaryKey key)
+        {
+            return key.token().getLongValue();
+        }
+
+        @Override
+        public long floor(PrimaryKey key)
+        {
+            return key.token().getLongValue();
+        }
+
+        @Override
+        public long count()
+        {
+            return Long.MAX_VALUE;
+        }
+    };
+    public static final PrimaryKeyMap.Factory TEST_PRIMARY_KEY_MAP_FACTORY = () -> TEST_PRIMARY_KEY_MAP;
+
+
+    private static final BigDecimal ONE_TENTH = BigDecimal.valueOf(1, 1);
+
+    private final IndexDescriptor indexDescriptor;
+    private final AbstractType<?> type;
+    private final AbstractGuavaIterator<Pair<ByteComparable.Preencoded, IntArrayList>> terms;
+    private final int size;
+    private final int minSegmentRowId;
+    private final int maxSegmentRowId;
+
+    public KDTreeIndexBuilder(IndexDescriptor indexDescriptor,
+                              AbstractType<?> type,
+                              AbstractGuavaIterator<Pair<ByteComparable.Preencoded, IntArrayList>> terms,
+                              int size,
+                              int minSegmentRowId,
+                              int maxSegmentRowId)
+    {
+        this.indexDescriptor = indexDescriptor;
+        this.type = type;
+        this.terms = terms;
+        this.size = size;
+        this.minSegmentRowId = minSegmentRowId;
+        this.maxSegmentRowId = maxSegmentRowId;
+    }
+
+    KDTreeIndexSearcher flushAndOpen() throws IOException
+    {
+        // Wrap postings with RowIdWithFrequency using default frequency of 1
+        final TermsIterator termEnum = new MemtableTermsIterator(null, null, new AbstractGuavaIterator<>()
+        {
+            @Override
+            protected Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>> computeNext()
+            {
+                if (!terms.hasNext())
+                    return endOfData();
+
+                Pair<ByteComparable.Preencoded, IntArrayList> pair = terms.next();
+                List<RowMapping.RowIdWithFrequency> postings = new ArrayList<>(pair.right.size());
+                for (int i = 0; i < pair.right.size(); i++)
+                    postings.add(new RowMapping.RowIdWithFrequency(pair.right.get(i), 1));
+                return Pair.create(pair.left, postings);
+            }
+        });
+        final ImmutableOneDimPointValues pointValues = ImmutableOneDimPointValues.fromTermEnum(termEnum, type);
+
+        final SegmentMetadata metadata;
+
+        IndexContext indexContext = SAITester.createIndexContext("test", Int32Type.instance);
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (NumericIndexWriter writer = new NumericIndexWriter(components,
+                                                                TypeUtil.fixedSizeOf(type),
+                                                                maxSegmentRowId,
+                                                                size,
+                                                                IndexWriterConfig.defaultConfig("test")))
+        {
+            SegmentMetadataBuilder metadataBuilder = new SegmentMetadataBuilder(0, components);
+            final SegmentMetadata.ComponentMetadataMap indexMetas = writer.writeAll(metadataBuilder.intercept(pointValues));
+            metadataBuilder.setComponentsMetadata(indexMetas);
+            metadataBuilder.setRowIdRange(minSegmentRowId, maxSegmentRowId);
+            metadataBuilder.setTermRange(UTF8Type.instance.fromString("c"),
+                                         UTF8Type.instance.fromString("d"));
+            metadataBuilder.setKeyRange(SAITester.TEST_FACTORY.createTokenOnly(Murmur3Partitioner.instance.decorateKey(UTF8Type.instance.fromString("a")).getToken()),
+                                        SAITester.TEST_FACTORY.createTokenOnly(Murmur3Partitioner.instance.decorateKey(UTF8Type.instance.fromString("b")).getToken()));
+            metadata = metadataBuilder.build();
+        }
+
+        try (PerIndexFiles indexFiles = new PerIndexFiles(components))
+        {
+            SSTableContext sstableContext = mock(SSTableContext.class);
+            when(sstableContext.primaryKeyMapFactory()).thenReturn(KDTreeIndexBuilder.TEST_PRIMARY_KEY_MAP_FACTORY);
+            when(sstableContext.usedPerSSTableComponents()).thenReturn(indexDescriptor.perSSTableComponents());
+
+            IndexSearcher searcher = Version.latest().onDiskFormat().newIndexSearcher(sstableContext, indexContext, indexFiles, metadata);
+            assertThat(searcher, is(instanceOf(KDTreeIndexSearcher.class)));
+            return (KDTreeIndexSearcher) searcher;
+        }
+    }
+
+    /**
+     * Returns a k-d tree index where:
+     * 1. term values have 32b
+     * 2. term value is equal to {@code startTermInclusive} + row id;
+     * 3. tokens and offsets are equal to row id;
+     */
+    public static IndexSearcher buildInt32Searcher(IndexDescriptor indexDescriptor, int startTermInclusive, int endTermExclusive)
+    throws IOException
+    {
+        final int size = endTermExclusive - startTermInclusive;
+        Assert.assertTrue(size > 0);
+        KDTreeIndexBuilder indexBuilder = new KDTreeIndexBuilder(indexDescriptor,
+                                                                 Int32Type.instance,
+                                                                 singleOrd(int32Range(startTermInclusive, endTermExclusive), Int32Type.instance, startTermInclusive, size),
+                                                                 size,
+                                                                 startTermInclusive,
+                                                                 endTermExclusive);
+        return indexBuilder.flushAndOpen();
+    }
+
+    public static IndexSearcher buildDecimalSearcher(IndexDescriptor indexDescriptor, BigDecimal startTermInclusive, BigDecimal endTermExclusive)
+    throws IOException
+    {
+        BigDecimal bigDifference = endTermExclusive.subtract(startTermInclusive);
+        int size = bigDifference.intValueExact() * 10;
+        Assert.assertTrue(size > 0);
+        KDTreeIndexBuilder indexBuilder = new KDTreeIndexBuilder(indexDescriptor,
+                                                                 DecimalType.instance,
+                                                                 singleOrd(decimalRange(startTermInclusive, endTermExclusive), DecimalType.instance, startTermInclusive.intValueExact() * 10, size),
+                                                                 size,
+                                                                 startTermInclusive.intValueExact() * 10,
+                                                                 endTermExclusive.intValueExact() * 10);
+        return indexBuilder.flushAndOpen();
+    }
+
+    public static IndexSearcher buildBigIntegerSearcher(IndexDescriptor indexDescriptor, BigInteger startTermInclusive, BigInteger endTermExclusive)
+    throws IOException
+    {
+        BigInteger bigDifference = endTermExclusive.subtract(startTermInclusive);
+        int size = bigDifference.intValueExact();
+        Assert.assertTrue(size > 0);
+        KDTreeIndexBuilder indexBuilder = new KDTreeIndexBuilder(indexDescriptor,
+                                                                 IntegerType.instance,
+                                                                 singleOrd(bigIntegerRange(startTermInclusive, endTermExclusive), IntegerType.instance, startTermInclusive.intValueExact(), size),
+                                                                 size,
+                                                                 startTermInclusive.intValueExact(),
+                                                                 endTermExclusive.intValueExact());
+        return indexBuilder.flushAndOpen();
+    }
+
+    /**
+     * Returns a k-d tree index where:
+     * 1. term values have 64b
+     * 2. term value is equal to {@code startTermInclusive} + row id;
+     * 3. tokens and offsets are equal to row id;
+     */
+    public static IndexSearcher buildLongSearcher(IndexDescriptor indexDescriptor, long startTermInclusive, long endTermExclusive)
+    throws IOException
+    {
+        final long size = endTermExclusive - startTermInclusive;
+        Assert.assertTrue(size > 0);
+        KDTreeIndexBuilder indexBuilder = new KDTreeIndexBuilder(indexDescriptor,
+                                                                 LongType.instance,
+                                                                 singleOrd(longRange(startTermInclusive, endTermExclusive), LongType.instance, Math.toIntExact(startTermInclusive), Math.toIntExact(size)),
+                                                                 Math.toIntExact(size),
+                                                                 Math.toIntExact(startTermInclusive),
+                                                                 Math.toIntExact(endTermExclusive));
+        return indexBuilder.flushAndOpen();
+    }
+
+    /**
+     * Returns a k-d tree index where:
+     * 1. term values have 16b
+     * 2. term value is equal to {@code startTermInclusive} + row id;
+     * 3. tokens and offsets are equal to row id;
+     */
+    public static IndexSearcher buildShortSearcher(IndexDescriptor indexDescriptor, short startTermInclusive, short endTermExclusive)
+    throws IOException
+    {
+        final int size = endTermExclusive - startTermInclusive;
+        Assert.assertTrue(size > 0);
+        KDTreeIndexBuilder indexBuilder = new KDTreeIndexBuilder(indexDescriptor,
+                                                                 ShortType.instance,
+                                                                 singleOrd(shortRange(startTermInclusive, endTermExclusive), ShortType.instance, startTermInclusive, size),
+                                                                 size,
+                                                                 startTermInclusive,
+                                                                 endTermExclusive);
+        return indexBuilder.flushAndOpen();
+    }
+
+    /**
+     * Returns inverted index where each posting list contains exactly one element equal to the terms ordinal number +
+     * given offset.
+     */
+    public static AbstractGuavaIterator<Pair<ByteComparable.Preencoded, IntArrayList>> singleOrd(Iterator<ByteBuffer> terms, AbstractType<?> type, int segmentRowIdOffset, int size)
+    {
+        return new AbstractGuavaIterator<Pair<ByteComparable.Preencoded, IntArrayList>>()
+        {
+            private long currentTerm = 0;
+            private int currentSegmentRowId = segmentRowIdOffset;
+
+            @Override
+            protected Pair<ByteComparable.Preencoded, IntArrayList> computeNext()
+            {
+                if (currentTerm++ >= size)
+                {
+                    return endOfData();
+                }
+
+                IntArrayList postings = new IntArrayList();
+                postings.add(currentSegmentRowId++);
+                assertTrue(terms.hasNext());
+
+                final ByteSource encoded = TypeUtil.asComparableBytes(terms.next(), type, TypeUtil.BYTE_COMPARABLE_VERSION);
+                return Pair.create(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, ByteSourceInverse.readBytes(encoded)), postings);
+            }
+        };
+    }
+
+    /**
+     * Returns sequential ordered encoded ints from {@code startInclusive} (inclusive) to {@code endExclusive}
+     * (exclusive) by an incremental step of {@code 1}.
+     */
+    public static Iterator<ByteBuffer> int32Range(int startInclusive, int endExclusive)
+    {
+        return IntStream.range(startInclusive, endExclusive)
+                        .mapToObj(Int32Type.instance::decompose)
+                        .collect(Collectors.toList())
+                        .iterator();
+    }
+
+    /**
+     * Returns sequential ordered encoded longs from {@code startInclusive} (inclusive) to {@code endExclusive}
+     * (exclusive) by an incremental step of {@code 1}.
+     */
+    public static Iterator<ByteBuffer> longRange(long startInclusive, long endExclusive)
+    {
+        return LongStream.range(startInclusive, endExclusive)
+                         .mapToObj(LongType.instance::decompose)
+                         .collect(Collectors.toList())
+                         .iterator();
+    }
+
+    public static Iterator<ByteBuffer> decimalRange(final BigDecimal startInclusive, final BigDecimal endExclusive)
+    {
+        int n = endExclusive.subtract(startInclusive).intValueExact() * 10;
+        final Supplier<BigDecimal> generator = new Supplier<BigDecimal>() {
+            BigDecimal current = startInclusive;
+
+            @Override
+            public BigDecimal get() {
+                BigDecimal result = current;
+                current = current.add(ONE_TENTH);
+                return result;
+            }
+        };
+        return Stream.generate(generator)
+                     .limit(n)
+                     .map(bd -> TypeUtil.asIndexBytes(DecimalType.instance.decompose(bd), DecimalType.instance))
+                     .collect(Collectors.toList())
+                     .iterator();
+    }
+
+    public static Iterator<ByteBuffer> bigIntegerRange(final BigInteger startInclusive, final BigInteger endExclusive)
+    {
+        int n = endExclusive.subtract(startInclusive).intValueExact();
+        final Supplier<BigInteger> generator = new Supplier<BigInteger>() {
+            BigInteger current = startInclusive;
+
+            @Override
+            public BigInteger get() {
+                BigInteger result = current;
+                current = current.add(BigInteger.ONE);
+                return result;
+            }
+        };
+        return Stream.generate(generator)
+                     .limit(n)
+                     .map(bd -> TypeUtil.asIndexBytes(IntegerType.instance.decompose(bd), IntegerType.instance))
+                     .collect(Collectors.toList())
+                     .iterator();
+    }
+
+
+    /**
+     * Returns sequential ordered encoded shorts from {@code startInclusive} (inclusive) to {@code endExclusive}
+     * (exclusive) by an incremental step of {@code 1}.
+     */
+    public static Iterator<ByteBuffer> shortRange(short startInclusive, short endExclusive)
+    {
+        return IntStream.range(startInclusive, endExclusive)
+                        .mapToObj(i -> ShortType.instance.decompose((short) i))
+                        .collect(Collectors.toList())
+                        .iterator();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/LeafOrderMapTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/LeafOrderMapTest.java
new file mode 100644
index 000000000000..8b6e3abd8ff9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/LeafOrderMapTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.nio.ByteOrder;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.disk.ModernResettableByteBuffersIndexOutput;
+import org.apache.cassandra.index.sai.disk.oldlucene.LuceneCompat;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.SeekingRandomAccessInput;
+import org.apache.lucene.util.LongValues;
+
+public class LeafOrderMapTest extends SaiRandomizedTest
+{
+    @Test
+    public void test() throws Exception
+    {
+        int[] array = new int[1024];
+        for (int x=0; x < array.length; x++)
+        {
+            array[x] = x;
+        }
+        shuffle(array);
+
+        var out = new ModernResettableByteBuffersIndexOutput(array.length, "");
+
+        LeafOrderMap.write(ByteOrder.LITTLE_ENDIAN, array, array.length, array.length - 1, out);
+
+        var input = out.toIndexInput();
+
+        final byte bits = (byte) LuceneCompat.directWriterUnsignedBitsRequired(ByteOrder.LITTLE_ENDIAN, array.length - 1);
+        LongValues reader = LuceneCompat.directReaderGetInstance(new SeekingRandomAccessInput(input, ByteOrder.LITTLE_ENDIAN), bits, 0);
+
+        for (int x=0; x < array.length; x++)
+        {
+            int value = LeafOrderMap.getValue(x, reader);
+
+            assertEquals("disagreed at " + x, array[x], value);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriterTest.java
new file mode 100644
index 000000000000..f357112e28d4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriterTest.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.MemtableTermsIterator;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.TermsIterator;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.memory.RowMapping;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.metrics.QueryEventListeners;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.NumericUtils;
+
+public class NumericIndexWriterTest extends SaiRandomizedTest
+{
+    private IndexDescriptor indexDescriptor;
+    private String index;
+    private IndexContext indexContext;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        indexDescriptor = newIndexDescriptor();
+        index = newIndex();
+        indexContext = SAITester.createIndexContext(index, Int32Type.instance);
+    }
+
+    @Test
+    public void shouldFlushFromRamBuffer() throws Exception
+    {
+        doShouldFlushFromRamBuffer();
+    }
+
+    private void doShouldFlushFromRamBuffer() throws Exception
+    {
+        final BKDTreeRamBuffer ramBuffer = new BKDTreeRamBuffer(1, Integer.BYTES);
+        final int numRows = 120;
+        int currentValue = numRows;
+        for (int i = 0; i < numRows; ++i)
+        {
+            byte[] scratch = new byte[Integer.BYTES];
+            NumericUtils.intToSortableBytes(currentValue--, scratch, 0);
+            ramBuffer.addPackedValue(i, new BytesRef(scratch));
+        }
+
+        final MutableOneDimPointValues pointValues = ramBuffer.asPointValues();
+
+        int docCount = pointValues.getDocCount();
+
+        SegmentMetadata.ComponentMetadataMap indexMetas;
+
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (NumericIndexWriter writer = new NumericIndexWriter(components,
+                                                                Integer.BYTES,
+                                                                docCount, docCount,
+                                                                IndexWriterConfig.defaultConfig("test")))
+        {
+            indexMetas = writer.writeAll(pointValues);
+        }
+
+        final FileHandle kdtreeHandle = components.get(IndexComponentType.KD_TREE).createFileHandle();
+        final FileHandle kdtreePostingsHandle = components.get(IndexComponentType.KD_TREE_POSTING_LISTS).createFileHandle();
+
+        try (BKDReader reader = new BKDReader(indexContext,
+                                              kdtreeHandle,
+                                              indexMetas.get(IndexComponentType.KD_TREE).root,
+                                              kdtreePostingsHandle,
+                                              indexMetas.get(IndexComponentType.KD_TREE_POSTING_LISTS).root
+        ))
+        {
+            final Counter visited = Counter.newCounter();
+            try (final PostingList ignored = reader.intersect(new BKDReader.IntersectVisitor()
+            {
+                @Override
+                public boolean visit(byte[] packedValue)
+                {
+                    // we should read point values in reverse order after sorting
+                    assertEquals(1 + visited.get(), NumericUtils.sortableBytesToInt(packedValue, 0));
+                    visited.addAndGet(1);
+                    return true;
+                }
+
+                @Override
+                public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
+                {
+                    return PointValues.Relation.CELL_CROSSES_QUERY;
+                }
+            }, (QueryEventListener.BKDIndexEventListener)QueryEventListeners.NO_OP_BKD_LISTENER, new QueryContext()))
+            {
+                assertEquals(numRows, visited.get());
+            }
+        }
+    }
+
+    @Test
+    public void shouldFlushFromMemtable() throws Exception
+    {
+        final int maxSegmentRowId = 100;
+        final TermsIterator termEnum = buildTermEnum(0, maxSegmentRowId);
+        final ImmutableOneDimPointValues pointValues = ImmutableOneDimPointValues
+                                                       .fromTermEnum(termEnum, Int32Type.instance);
+
+        SegmentMetadata.ComponentMetadataMap indexMetas;
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (NumericIndexWriter writer = new NumericIndexWriter(components,
+                                                                TypeUtil.fixedSizeOf(Int32Type.instance),
+                                                                maxSegmentRowId, maxSegmentRowId,
+                                                                IndexWriterConfig.defaultConfig("test")))
+        {
+            indexMetas = writer.writeAll(pointValues);
+        }
+
+        final FileHandle kdtreeHandle = components.get(IndexComponentType.KD_TREE).createFileHandle();
+        final FileHandle kdtreePostingsHandle = components.get(IndexComponentType.KD_TREE_POSTING_LISTS).createFileHandle();
+
+        try (BKDReader reader = new BKDReader(indexContext,
+                                              kdtreeHandle,
+                                              indexMetas.get(IndexComponentType.KD_TREE).root,
+                                              kdtreePostingsHandle,
+                                              indexMetas.get(IndexComponentType.KD_TREE_POSTING_LISTS).root
+        ))
+        {
+            final Counter visited = Counter.newCounter();
+            try (final PostingList ignored = reader.intersect(new BKDReader.IntersectVisitor()
+            {
+                @Override
+                public boolean visit(byte[] packedValue)
+                {
+                    final ByteComparable actualTerm = ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, packedValue);
+                    final ByteComparable expectedTerm = ByteComparable.of(Math.toIntExact(visited.get()));
+                    assertEquals("Point value mismatch after visiting " + visited.get() + " entries.", 0,
+                                 ByteComparable.compare(actualTerm, expectedTerm, TypeUtil.BYTE_COMPARABLE_VERSION));
+
+                    visited.addAndGet(1);
+                    return true;
+                }
+
+                @Override
+                public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue)
+                {
+                    return PointValues.Relation.CELL_CROSSES_QUERY;
+                }
+            }, (QueryEventListener.BKDIndexEventListener)QueryEventListeners.NO_OP_BKD_LISTENER, new QueryContext()))
+            {
+                assertEquals(maxSegmentRowId, visited.get());
+            }
+        }
+    }
+
+    private TermsIterator buildTermEnum(int startTermInclusive, int endTermExclusive)
+    {
+        final ByteBuffer minTerm = Int32Type.instance.decompose(startTermInclusive);
+        final ByteBuffer maxTerm = Int32Type.instance.decompose(endTermExclusive);
+
+        final AbstractGuavaIterator<Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>>> iterator = new AbstractGuavaIterator<>()
+        {
+            private int currentTerm = startTermInclusive;
+            private int currentRowId = 0;
+
+            @Override
+            protected Pair<ByteComparable.Preencoded, List<RowMapping.RowIdWithFrequency>> computeNext()
+            {
+                if (currentTerm >= endTermExclusive)
+                {
+                    return endOfData();
+                }
+                final ByteBuffer term = Int32Type.instance.decompose(currentTerm++);
+                final List<RowMapping.RowIdWithFrequency> postings = new ArrayList<>();
+                postings.add(new RowMapping.RowIdWithFrequency(currentRowId++, 1));
+                final ByteSource encoded = Int32Type.instance.asComparableBytes(term, TypeUtil.BYTE_COMPARABLE_VERSION);
+                byte[] bytes = new byte[4];
+                encoded.nextBytes(bytes);
+                return Pair.create(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, bytes), postings);
+            }
+        };
+
+        return new MemtableTermsIterator(minTerm, maxTerm, iterator);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/OneDimBKDPostingsWriterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/OneDimBKDPostingsWriterTest.java
new file mode 100644
index 000000000000..7a12ee543a3e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/OneDimBKDPostingsWriterTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.kdtree;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
+import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
+import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
+import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedLongValues;
+
+import static org.apache.cassandra.index.sai.metrics.QueryEventListeners.NO_OP_POSTINGS_LISTENER;
+
+public class OneDimBKDPostingsWriterTest extends SaiRandomizedTest
+{
+    private IndexDescriptor indexDescriptor;
+    private String index;
+    private IndexContext indexContext;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        indexDescriptor = newIndexDescriptor();
+        index = newIndex();
+        indexContext = SAITester.createIndexContext(index, Int32Type.instance);
+    }
+
+    @Test
+    public void shouldWritePostingsForEligibleNodes() throws IOException
+    {
+        List<PackedLongValues> leaves =
+        Arrays.asList(postings(1, 5, 7), postings(3, 4, 6), postings(2, 8, 10), postings(11, 12, 13));
+
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        OneDimBKDPostingsWriter writer = new OneDimBKDPostingsWriter(leaves, new IndexWriterConfig("test", 2, 1), components::logMessage);
+
+        // should build postings for nodes 2 & 3 (lvl 2) and 8, 10, 12, 14 (lvl 4)
+        writer.onLeaf(64, 1, pathToRoot(1, 2, 4, 8, 16));
+        writer.onLeaf(80, 2, pathToRoot(1, 2, 5, 10, 20));
+        writer.onLeaf(96, 3, pathToRoot(1, 3, 6, 12, 24));
+        writer.onLeaf(112, 4, pathToRoot(1, 3, 7, 14, 28));
+
+        long fp;
+        try (IndexOutputWriter output = components.addOrGet(IndexComponentType.KD_TREE_POSTING_LISTS).openOutput())
+        {
+            fp = writer.finish(output);
+        }
+
+        IndexComponent.ForRead kdTreePostings = components.get(IndexComponentType.KD_TREE_POSTING_LISTS);
+        BKDPostingsIndex postingsIndex = new BKDPostingsIndex(kdTreePostings.createFileHandle(), fp);
+        assertEquals(10, postingsIndex.size());
+
+        // Internal postings...
+        assertTrue(postingsIndex.exists(2));
+        assertTrue(postingsIndex.exists(3));
+        assertTrue(postingsIndex.exists(8));
+        assertTrue(postingsIndex.exists(10));
+        assertTrue(postingsIndex.exists(12));
+        assertTrue(postingsIndex.exists(14));
+
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 2, new int[]{ 1, 3, 4, 5, 6, 7 });
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 3, new int[]{ 2, 8, 10, 11, 12, 13 });
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 8, new int[]{ 1, 5, 7 });
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 10, new int[]{ 3, 4, 6 });
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 12, new int[]{ 2, 8, 10 });
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 14, new int[]{ 11, 12, 13 });
+
+        // Leaf postings...
+        assertTrue(postingsIndex.exists(64));
+        assertTrue(postingsIndex.exists(80));
+        assertTrue(postingsIndex.exists(96));
+        assertTrue(postingsIndex.exists(112));
+
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 64, new int[]{ 1, 5, 7 });
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 80, new int[]{ 3, 4, 6 });
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 96, new int[]{ 2, 8, 10 });
+        assertPostingReaderEquals(kdTreePostings, postingsIndex, 112, new int[]{ 11, 12, 13 });
+    }
+
+    @Test
+    public void shouldSkipPostingListWhenSamplingMisses() throws IOException
+    {
+        List<PackedLongValues> leaves = Collections.singletonList(postings(1, 2, 3));
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        OneDimBKDPostingsWriter writer = new OneDimBKDPostingsWriter(leaves, new IndexWriterConfig("test", 5, 1), components::logMessage);
+
+        // The tree is too short to have any internal posting lists.
+        writer.onLeaf(16, 1, pathToRoot(1, 2, 4, 8));
+
+        long fp;
+        try (IndexOutputWriter output = components.addOrGet(IndexComponentType.KD_TREE_POSTING_LISTS).openOutput())
+        {
+            fp = writer.finish(output);
+        }
+
+        // There is only a single posting list...the leaf posting list.
+        BKDPostingsIndex postingsIndex = new BKDPostingsIndex(components.get(IndexComponentType.KD_TREE_POSTING_LISTS).createFileHandle(), fp);
+        assertEquals(1, postingsIndex.size());
+    }
+
+    @Test
+    public void shouldSkipPostingListWhenTooFewLeaves() throws IOException
+    {
+        List<PackedLongValues> leaves = Collections.singletonList(postings(1, 2, 3));
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        OneDimBKDPostingsWriter writer = new OneDimBKDPostingsWriter(leaves, new IndexWriterConfig("test", 2, 2), components::logMessage);
+
+        // The tree is too short to have any internal posting lists.
+        writer.onLeaf(16, 1, pathToRoot(1, 2, 4, 8));
+
+        long fp;
+        try (IndexOutputWriter output = components.addOrGet(IndexComponentType.KD_TREE_POSTING_LISTS).openOutput())
+        {
+            fp = writer.finish(output);
+        }
+
+        // There is only a single posting list...the leaf posting list.
+        BKDPostingsIndex postingsIndex = new BKDPostingsIndex(components.get(IndexComponentType.KD_TREE_POSTING_LISTS).createFileHandle(), fp);
+        assertEquals(1, postingsIndex.size());
+    }
+
+    private void assertPostingReaderEquals(IndexComponent.ForRead kdTreePostingLists, BKDPostingsIndex postingsIndex, int nodeID, int[] postings) throws IOException
+    {
+        assertPostingReaderEquals(kdTreePostingLists.openInput(),
+                                  postingsIndex.getPostingsFilePointer(nodeID),
+                                  new IntArrayPostingList(postings));
+    }
+
+    private void assertPostingReaderEquals(IndexInput input, long offset, PostingList expected) throws IOException
+    {
+        try (PostingsReader reader = new PostingsReader(input, offset, NO_OP_POSTINGS_LISTENER))
+        {
+            assertPostingListEquals(expected, reader);
+        }
+    }
+
+    private PackedLongValues postings(int... postings)
+    {
+        final PackedLongValues.Builder builder = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
+        for (int posting : postings)
+        {
+            builder.add(posting);
+        }
+        return builder.build();
+    }
+
+    private IntArrayList pathToRoot(int... nodes)
+    {
+        final IntArrayList path = new IntArrayList();
+        for (int node : nodes)
+        {
+            path.add(node);
+        }
+        return path;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookupTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookupTest.java
deleted file mode 100644
index 92a31f87f711..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/keystore/KeyLookupTest.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.keystore;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
-import org.apache.cassandra.index.sai.disk.v1.MetadataSource;
-import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
-import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
-import org.apache.lucene.store.IndexInput;
-
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class KeyLookupTest extends SAIRandomizedTester
-{
-    protected IndexDescriptor indexDescriptor;
-
-    @Before
-    public void setup() throws Exception
-    {
-        indexDescriptor = newIndexDescriptor();
-    }
-
-    @Test
-    public void testFileValidation() throws Exception
-    {
-        List<PrimaryKey> primaryKeys = new ArrayList<>();
-
-        for (int x = 0; x < 11; x++)
-        {
-            ByteBuffer buffer = UTF8Type.instance.decompose(Integer.toString(x));
-            DecoratedKey partitionKey = Murmur3Partitioner.instance.decorateKey(buffer);
-            PrimaryKey primaryKey = SAITester.TEST_FACTORY.create(partitionKey);
-            primaryKeys.add(primaryKey);
-        }
-
-        primaryKeys.sort(PrimaryKey::compareTo);
-
-        try (MetadataWriter metadataWriter = new MetadataWriter(indexDescriptor.openPerSSTableOutput(IndexComponent.GROUP_META)))
-        {
-            IndexOutputWriter bytesWriter = indexDescriptor.openPerSSTableOutput(IndexComponent.PARTITION_KEY_BLOCKS);
-            NumericValuesWriter blockFPWriter = new NumericValuesWriter(indexDescriptor, IndexComponent.PARTITION_KEY_BLOCK_OFFSETS, metadataWriter, true);
-            try (KeyStoreWriter writer = new KeyStoreWriter(indexDescriptor.componentName(IndexComponent.PARTITION_KEY_BLOCKS),
-                                                            metadataWriter,
-                                                            bytesWriter,
-                                                            blockFPWriter,
-                                                            4,
-                                                            false))
-            {
-                primaryKeys.forEach(primaryKey -> {
-                    try
-                    {
-                        writer.add(primaryKey);
-                    }
-                    catch (IOException e)
-                    {
-                        e.printStackTrace();
-                    }
-                });
-            }
-        }
-        assertTrue(validateComponent(IndexComponent.PARTITION_KEY_BLOCKS, true));
-        assertTrue(validateComponent(IndexComponent.PARTITION_KEY_BLOCKS, false));
-        assertTrue(validateComponent(IndexComponent.PARTITION_KEY_BLOCK_OFFSETS, true));
-        assertTrue(validateComponent(IndexComponent.PARTITION_KEY_BLOCK_OFFSETS, false));
-    }
-
-    @Test
-    public void testLongPrefixesAndSuffixes() throws Exception
-    {
-        List<byte[]> keys = new ArrayList<>();
-        writeKeys(writer ->  {
-            // The following writes a set of keys that cover the following conditions:
-
-            // Start value 0
-            byte[] bytes = new byte[20];
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-            // prefix > 15
-            bytes = new byte[20];
-            Arrays.fill(bytes, 16, 20, (byte)1);
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-            // prefix == 15
-            bytes = new byte[20];
-            Arrays.fill(bytes, 15, 20, (byte)1);
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-            // prefix < 15
-            bytes = new byte[20];
-            Arrays.fill(bytes, 14, 20, (byte)1);
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-            // suffix > 16
-            bytes = new byte[20];
-            Arrays.fill(bytes, 0, 4, (byte)1);
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-            // suffix == 16
-            bytes = new byte[20];
-            Arrays.fill(bytes, 0, 5, (byte)1);
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-            // suffix < 16
-            bytes = new byte[20];
-            Arrays.fill(bytes, 0, 6, (byte)1);
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-
-            bytes = new byte[32];
-            Arrays.fill(bytes, 0, 16, (byte)1);
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-            // prefix >= 15 && suffix >= 16
-            bytes = new byte[32];
-            Arrays.fill(bytes, 0, 32, (byte)1);
-            keys.add(bytes);
-            writer.add(ByteComparable.fixedLength(bytes));
-        }, false);
-
-        doTestKeyLookup(keys);
-    }
-
-    @Test
-    public void testNonUniqueKeys() throws Exception
-    {
-        List<byte[]> keys = new ArrayList<>();
-
-        writeKeys(writer ->  {
-            for (int x = 0; x < 4000; x++)
-            {
-                ByteBuffer buffer = Int32Type.instance.decompose(5000);
-                ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS50);
-                byte[] bytes = ByteSourceInverse.readBytes(byteSource);
-                keys.add(bytes);
-
-                writer.add(ByteComparable.fixedLength(bytes));
-            }
-        }, false);
-
-        doTestKeyLookup(keys);
-    }
-
-    @Test
-    public void testSeekToPointId() throws Exception
-    {
-        List<byte[]> keys = new ArrayList<>();
-
-        writeKeys(writer -> {
-            for (int x = 0; x < 4000; x++)
-            {
-                ByteBuffer buffer = Int32Type.instance.decompose(x);
-                ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS50);
-                byte[] bytes = ByteSourceInverse.readBytes(byteSource);
-                keys.add(bytes);
-
-                writer.add(ByteComparable.fixedLength(bytes));
-            }
-        }, false);
-
-        doTestKeyLookup(keys);
-    }
-
-    @Test
-    public void testSeekToPointIdOutOfRange() throws Exception
-    {
-        writeKeys(writer -> {
-            for (int x = 0; x < 4000; x++)
-            {
-                ByteBuffer buffer = Int32Type.instance.decompose(x);
-                ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS50);
-                byte[] bytes = ByteSourceInverse.readBytes(byteSource);
-
-                writer.add(ByteComparable.fixedLength(bytes));
-            }
-        }, false);
-
-        withKeyLookupCursor(cursor -> {
-            assertThatThrownBy(() -> cursor.seekToPointId(-2)).isInstanceOf(IndexOutOfBoundsException.class)
-                                                              .hasMessage(String.format(KeyLookup.INDEX_OUT_OF_BOUNDS, -2, 4000));
-            assertThatThrownBy(() -> cursor.seekToPointId(Long.MAX_VALUE)).isInstanceOf(IndexOutOfBoundsException.class)
-                                                                          .hasMessage(String.format(KeyLookup.INDEX_OUT_OF_BOUNDS, Long.MAX_VALUE, 4000));
-            assertThatThrownBy(() -> cursor.seekToPointId(4000)).isInstanceOf(IndexOutOfBoundsException.class)
-                                                                .hasMessage(String.format(KeyLookup.INDEX_OUT_OF_BOUNDS, 4000, 4000));
-        });
-    }
-
-    @Test
-    public void testSeekToKey() throws Exception
-    {
-        Map<Long, byte[]> keys = new HashMap<>();
-
-        writeKeys(writer -> {
-            long pointId = 0;
-            for (int x = 0; x < 4000; x += 4)
-            {
-                byte[] key = makeKey(x);
-                keys.put(pointId++, key);
-
-                writer.add(ByteComparable.fixedLength(key));
-            }
-        }, true);
-
-        withKeyLookupCursor(cursor -> {
-            assertEquals(0L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(0L)), 0L, 10L));
-            cursor.reset();
-            assertEquals(160L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(160L)), 160L, 170L));
-            cursor.reset();
-            assertEquals(165L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(165L)), 160L, 170L));
-            cursor.reset();
-            assertEquals(175L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(175L)), 160L, 176L));
-            cursor.reset();
-            assertEquals(176L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(176L)), 160L, 177L));
-            cursor.reset();
-            assertEquals(176L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(176L)), 175L, 177L));
-            cursor.reset();
-            assertEquals(176L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(makeKey(701)), 160L, 177L));
-            cursor.reset();
-            assertEquals(504L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(504L)), 200L, 600L));
-            cursor.reset();
-            assertEquals(-1L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(makeKey(4000)), 0L, 1000L));
-            cursor.reset();
-            assertEquals(-1L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(makeKey(4000)), 999L, 1000L));
-            cursor.reset();
-            assertEquals(999L, cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(999L)), 0L, 1000L));
-        });
-    }
-
-    @Test
-    public void seekToKeyOnNonPartitionedTest() throws Throwable
-    {
-        Map<Long, byte[]> keys = new HashMap<>();
-
-        writeKeys(writer -> {
-            long pointId = 0;
-            for (int x = 0; x < 16; x += 4)
-            {
-                byte[] key = makeKey(x);
-                keys.put(pointId++, key);
-
-                writer.add(ByteComparable.fixedLength(key));
-            }
-        }, false);
-
-        withKeyLookupCursor(cursor -> assertThatThrownBy(() -> cursor.clusteredSeekToKey(ByteComparable.fixedLength(keys.get(0L)), 0L, 10L))
-                                        .isInstanceOf(AssertionError.class));
-    }
-
-    @Test
-    public void partitionedKeysMustBeInOrderInPartitions() throws Throwable
-    {
-        writeKeys(writer -> {
-            writer.startPartition();
-            writer.add(ByteComparable.fixedLength(makeKey(0)));
-            writer.add(ByteComparable.fixedLength(makeKey(10)));
-            assertThatThrownBy(() -> writer.add(ByteComparable.fixedLength(makeKey(9)))).isInstanceOf(IllegalArgumentException.class);
-            writer.startPartition();
-            writer.add(ByteComparable.fixedLength(makeKey(9)));
-        }, true);
-    }
-
-    private byte[] makeKey(int value)
-    {
-        ByteBuffer buffer = Int32Type.instance.decompose(value);
-        ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS50);
-        return ByteSourceInverse.readBytes(byteSource);
-    }
-
-    private void doTestKeyLookup(List<byte[]> keys) throws Exception
-    {
-        // iterate ascending
-        withKeyLookupCursor(cursor -> {
-            for (int x = 0; x < keys.size(); x++)
-                assertArrayEquals(keys.get(x), ByteSourceInverse.readBytes(cursor.seekToPointId(x)));
-        });
-
-        // iterate ascending skipping blocks
-        withKeyLookupCursor(cursor -> {
-            for (int x = 0; x < keys.size(); x += 17)
-                assertArrayEquals(keys.get(x), ByteSourceInverse.readBytes(cursor.seekToPointId(x)));
-        });
-
-        withKeyLookupCursor(cursor -> {
-            assertArrayEquals(keys.get(7), ByteSourceInverse.readBytes(cursor.seekToPointId(7)));
-            assertArrayEquals(keys.get(7), ByteSourceInverse.readBytes(cursor.seekToPointId(7)));
-        });
-    }
-
-    protected void writeKeys(ThrowingConsumer<KeyStoreWriter> testCode, boolean clustering) throws IOException
-    {
-        try (MetadataWriter metadataWriter = new MetadataWriter(indexDescriptor.openPerSSTableOutput(IndexComponent.GROUP_META)))
-        {
-            IndexOutputWriter bytesWriter = indexDescriptor.openPerSSTableOutput(IndexComponent.PARTITION_KEY_BLOCKS);
-            NumericValuesWriter blockFPWriter = new NumericValuesWriter(indexDescriptor, IndexComponent.PARTITION_KEY_BLOCK_OFFSETS, metadataWriter, true);
-            try (KeyStoreWriter writer = new KeyStoreWriter(indexDescriptor.componentName(IndexComponent.PARTITION_KEY_BLOCKS),
-                                                            metadataWriter,
-                                                            bytesWriter,
-                                                            blockFPWriter,
-                                                            4,
-                                                            clustering))
-            {
-                testCode.accept(writer);
-            }
-        }
-    }
-
-    @FunctionalInterface
-    public interface ThrowingConsumer<T>
-    {
-        void accept(T t) throws IOException;
-    }
-
-    private void withKeyLookup(ThrowingConsumer<KeyLookup> testCode) throws IOException
-    {
-        MetadataSource metadataSource = MetadataSource.loadGroupMetadata(indexDescriptor);
-        NumericValuesMeta blockPointersMeta = new NumericValuesMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.PARTITION_KEY_BLOCK_OFFSETS)));
-        KeyLookupMeta keyLookupMeta = new KeyLookupMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.PARTITION_KEY_BLOCKS)));
-        try (FileHandle keysData = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PARTITION_KEY_BLOCKS, null);
-             FileHandle blockOffsets = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PARTITION_KEY_BLOCK_OFFSETS, null))
-        {
-            KeyLookup reader = new KeyLookup(keysData, blockOffsets, keyLookupMeta, blockPointersMeta);
-            testCode.accept(reader);
-        }
-    }
-
-    private void withKeyLookupCursor(ThrowingConsumer<KeyLookup.Cursor> testCode) throws IOException
-    {
-        withKeyLookup(reader -> {
-            try (KeyLookup.Cursor cursor = reader.openCursor())
-            {
-                testCode.accept(cursor);
-            }
-        });
-    }
-
-    private boolean validateComponent(IndexComponent indexComponent, boolean checksum)
-    {
-        try (IndexInput input = indexDescriptor.openPerSSTableInput(indexComponent))
-        {
-            if (checksum)
-                SAICodecUtils.validateChecksum(input);
-            else
-                SAICodecUtils.validate(input);
-            return true;
-        }
-        catch (Throwable ignore)
-        {
-            return false;
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/FilteringPostingListTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/FilteringPostingListTest.java
new file mode 100644
index 000000000000..6b89b2ed9718
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/FilteringPostingListTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.lucene.util.FixedBitSet;
+
+public class FilteringPostingListTest extends SaiRandomizedTest
+{
+    @Test
+    public void shouldMatchAllWithoutAdvance() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithoutAdvance(postingsArray, 0, postingsArray.length);
+    }
+
+    @Test
+    public void shouldMatchEndRangeWithoutAdvance() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithoutAdvance(postingsArray, 2, postingsArray.length);
+    }
+
+    @Test
+    public void shouldMatchStartRangeWithoutAdvance() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithoutAdvance(postingsArray, 0, 3);
+    }
+
+    @Test
+    public void shouldMatchMiddleRangeWithoutAdvance() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithoutAdvance(postingsArray, 2, 4);
+    }
+
+    @Test
+    public void shouldAdvanceToMiddleMatchingAll() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithAdvance(postingsArray, 0, postingsArray.length, 2);
+    }
+
+    @Test
+    public void shouldAdvanceToStartMatchingAll() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithAdvance(postingsArray, 0, postingsArray.length, 0);
+    }
+
+    @Test
+    public void shouldAdvanceToEndMatchingAll() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithAdvance(postingsArray, 0, postingsArray.length, 5);
+    }
+
+    @Test
+    public void shouldAdvancePastEndMatchingAll() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithAdvance(postingsArray, 0, postingsArray.length, 6);
+    }
+
+    @Test
+    public void shouldAdvanceToBeforeMatchStart() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithAdvance(postingsArray, 3, postingsArray.length, 1);
+    }
+
+    @Test
+    public void shouldAdvanceToAfterMatchEnd() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithAdvance(postingsArray, 0, 2, 3);
+    }
+
+    @Test
+    public void shouldAdvanceToExactMatchStart() throws IOException
+    {
+        int[] postingsArray = { 0, 1, 2, 3, 4, 5 };
+        verifyFilteringWithAdvance(postingsArray, 2, postingsArray.length, 2);
+    }
+
+    private void verifyFilteringWithoutAdvance(int[] postingsArray, int from, int toExclusive) throws IOException
+    {
+        IntArrayPostingList delegate = new IntArrayPostingList(postingsArray);
+
+        FixedBitSet filter = new FixedBitSet((int)delegate.size());
+        filter.set(from, toExclusive);
+
+        FilteringPostingList filteringPostings = new FilteringPostingList(filter, delegate);
+
+        IntArrayPostingList expected = new IntArrayPostingList(Arrays.copyOfRange(postingsArray, from, toExclusive));
+        assertPostingListEquals(expected, filteringPostings);
+    }
+
+    private void verifyFilteringWithAdvance(int[] postingsArray, int from, int toExclusive, int target) throws IOException
+    {
+        IntArrayPostingList delegate = new IntArrayPostingList(postingsArray);
+
+        FixedBitSet filter = new FixedBitSet((int)delegate.size());
+        filter.set(from, toExclusive);
+
+        FilteringPostingList filteringPostings = new FilteringPostingList(filter, delegate);
+
+        // Make sure the expected advance ID is either in the range of matches or the sentinel value.
+        long expectedAdvanceTo = target < from ? from : target >= toExclusive ? PostingList.END_OF_STREAM : target;
+
+        try
+        {
+            long id = filteringPostings.advance(target);
+
+            assertEquals(expectedAdvanceTo, id);
+
+            if (id == PostingList.END_OF_STREAM)
+            {
+                return;
+            }
+        }
+        catch (Exception e)
+        {
+            long id = filteringPostings.advance(target);
+
+            assertEquals(expectedAdvanceTo, id);
+
+            if (id == PostingList.END_OF_STREAM)
+            {
+                return;
+            }
+        }
+
+        IntArrayPostingList expected = new IntArrayPostingList(postingsArray);
+        expected.advance(target);
+
+        // Advance to the first actual match...
+        while (expected.getOrdinal() <= from)
+        {
+            expected.nextPosting();
+        }
+
+        assertPostingListEquals(expected, filteringPostings);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/IntersectingPostingListTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/IntersectingPostingListTest.java
new file mode 100644
index 000000000000..76285a4f0bc4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/IntersectingPostingListTest.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.IntStream;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class IntersectingPostingListTest extends SaiRandomizedTest
+{
+    private Map<ByteBuffer, PostingList> createPostingMap(PostingList... lists)
+    {
+        Map<ByteBuffer, PostingList> map = new HashMap<>();
+        for (int i = 0; i < lists.length; i++)
+        {
+            map.put(ByteBufferUtil.bytes(String.valueOf((char) ('A' + i))), lists[i]);
+        }
+        return map;
+    }
+
+    @Test
+    public void shouldIntersectOverlappingPostingLists() throws IOException
+    {
+        var map = createPostingMap(new IntArrayPostingList(new int[]{ 1, 4, 6, 8 }),
+                                   new IntArrayPostingList(new int[]{ 2, 4, 6, 9 }),
+                                   new IntArrayPostingList(new int[]{ 4, 6, 7 }));
+
+        final PostingList intersected = IntersectingPostingList.intersect(map);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{ 4, 6 }), intersected);
+    }
+
+    @Test
+    public void shouldIntersectDisjointPostingLists() throws IOException
+    {
+        var map = createPostingMap(new IntArrayPostingList(new int[]{ 1, 3, 5 }),
+                                   new IntArrayPostingList(new int[]{ 2, 4, 6 }));
+
+        final PostingList intersected = IntersectingPostingList.intersect(map);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{}), intersected);
+    }
+
+    @Test
+    public void shouldIntersectSinglePostingList() throws IOException
+    {
+        var map = createPostingMap(new IntArrayPostingList(new int[]{ 1, 4, 6 }));
+
+        final PostingList intersected = IntersectingPostingList.intersect(map);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{ 1, 4, 6 }), intersected);
+    }
+
+    @Test
+    public void shouldIntersectIdenticalPostingLists() throws IOException
+    {
+        var map = createPostingMap(new IntArrayPostingList(new int[]{ 1, 2, 3 }),
+                                   new IntArrayPostingList(new int[]{ 1, 2, 3 }));
+
+        final PostingList intersected = IntersectingPostingList.intersect(map);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{ 1, 2, 3 }), intersected);
+    }
+
+    @Test
+    public void shouldAdvanceAllIntersectedLists() throws IOException
+    {
+        var map = createPostingMap(new IntArrayPostingList(new int[]{ 1, 3, 5, 7, 9 }),
+                                   new IntArrayPostingList(new int[]{ 2, 3, 5, 7, 8 }),
+                                   new IntArrayPostingList(new int[]{ 3, 5, 7, 10 }));
+
+        final PostingList intersected = IntersectingPostingList.intersect(map);
+        final PostingList expected = new IntArrayPostingList(new int[]{ 3, 5, 7 });
+
+        assertEquals(expected.advance(5), intersected.advance(5));
+        assertPostingListEquals(expected, intersected);
+    }
+
+    @Test
+    public void shouldHandleEmptyList() throws IOException
+    {
+        var map = createPostingMap(new IntArrayPostingList(new int[]{}),
+                                   new IntArrayPostingList(new int[]{ 1, 2, 3 }));
+
+        final PostingList intersected = IntersectingPostingList.intersect(map);
+        assertEquals(PostingList.END_OF_STREAM, intersected.advance(1));
+    }
+
+    @Test
+    public void shouldInterleaveNextAndAdvance() throws IOException
+    {
+        var map = createPostingMap(new IntArrayPostingList(new int[]{ 1, 3, 5, 7, 9 }),
+                                   new IntArrayPostingList(new int[]{ 1, 3, 5, 7, 9 }),
+                                   new IntArrayPostingList(new int[]{ 1, 3, 5, 7, 9 }));
+
+        final PostingList intersected = IntersectingPostingList.intersect(map);
+
+        assertEquals(1, intersected.nextPosting());
+        assertEquals(5, intersected.advance(5));
+        assertEquals(7, intersected.nextPosting());
+        assertEquals(9, intersected.advance(9));
+    }
+
+    @Test
+    public void shouldInterleaveNextAndAdvanceOnRandom() throws IOException
+    {
+        for (int i = 0; i < 1000; ++i)
+        {
+            testAdvancingOnRandom();
+        }
+    }
+
+    private void testAdvancingOnRandom() throws IOException
+    {
+        final int postingsCount = nextInt(1, 50_000);
+        final int postingListCount = nextInt(2, 10);
+
+        final AtomicInteger rowId = new AtomicInteger();
+        final int[] commonPostings = IntStream.generate(() -> rowId.addAndGet(nextInt(1, 10)))
+                                              .limit(postingsCount / 4)
+                                              .toArray();
+
+        var splitPostingLists = new ArrayList<PostingList>();
+        for (int i = 0; i < postingListCount; i++)
+        {
+            final int[] uniquePostings = IntStream.generate(() -> rowId.addAndGet(nextInt(1, 10)))
+                                                  .limit(postingsCount)
+                                                  .toArray();
+            int[] combined = IntStream.concat(IntStream.of(commonPostings),
+                                              IntStream.of(uniquePostings))
+                                      .distinct()
+                                      .sorted()
+                                      .toArray();
+            splitPostingLists.add(new IntArrayPostingList(combined));
+        }
+
+        final PostingList intersected = IntersectingPostingList.intersect(createPostingMap(splitPostingLists.toArray(new PostingList[0])));
+        final PostingList expected = new IntArrayPostingList(commonPostings);
+
+        final List<PostingListAdvance> actions = new ArrayList<>();
+        for (int idx = 0; idx < commonPostings.length; idx++)
+        {
+            if (nextInt(0, 8) == 0)
+            {
+                actions.add((postingList) -> {
+                    try
+                    {
+                        return postingList.nextPosting();
+                    }
+                    catch (IOException e)
+                    {
+                        fail(e.getMessage());
+                        throw new RuntimeException(e);
+                    }
+                });
+            }
+            else
+            {
+                final int skips = nextInt(0, 5);
+                idx = Math.min(idx + skips, commonPostings.length - 1);
+                final int rowID = commonPostings[idx];
+                actions.add((postingList) -> {
+                    try
+                    {
+                        return postingList.advance(rowID);
+                    }
+                    catch (IOException e)
+                    {
+                        fail(e.getMessage());
+                        throw new RuntimeException(e);
+                    }
+                });
+            }
+        }
+
+        for (PostingListAdvance action : actions)
+        {
+            assertEquals(action.advance(expected), action.advance(intersected));
+        }
+    }
+
+    private interface PostingListAdvance
+    {
+        long advance(PostingList list) throws IOException;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingListTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingListTest.java
index 989000eb0ab4..864cb0486d8b 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingListTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingListTest.java
@@ -20,86 +20,83 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
-import java.util.PriorityQueue;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
-import java.util.stream.LongStream;
+import java.util.stream.IntStream;
 
-import com.google.common.primitives.Longs;
 import org.junit.Test;
 
-import org.apache.cassandra.index.sai.disk.ArrayPostingList;
-import org.apache.cassandra.index.sai.postings.PeekablePostingList;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-public class MergePostingListTest extends SAIRandomizedTester
+public class MergePostingListTest extends SaiRandomizedTest
 {
     @Test
     public void shouldMergeInterleavedPostingLists() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(1, 4, 6),
-                                                                          new ArrayPostingList(2, 3, 4),
-                                                                          new ArrayPostingList(1, 6),
-                                                                          new ArrayPostingList(2, 5),
-                                                                          new ArrayPostingList(3, 6),
-                                                                          new ArrayPostingList(3, 5, 6));
+        var lists = listOfLists(
+                new IntArrayPostingList(new int[]{ 1, 4, 6 }),
+                new IntArrayPostingList(new int[]{ 2, 3, 4 }),
+                new IntArrayPostingList(new int[]{ 1, 6 }),
+                new IntArrayPostingList(new int[]{ 2, 5 }),
+                new IntArrayPostingList(new int[]{ 3, 6 }),
+                new IntArrayPostingList(new int[]{ 3, 5, 6 }));
 
         final PostingList merged = MergePostingList.merge(lists);
 
-        assertPostingListEquals(new ArrayPostingList(1, 2, 3, 4, 5, 6), merged);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{ 1, 2, 3, 4, 5, 6 }), merged);
     }
 
     @Test
     public void shouldMergeDisjointPostingLists() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(1, 6),
-                                                                          new ArrayPostingList(8, 9, 11),
-                                                                          new ArrayPostingList(15));
+        var lists = listOfLists(
+                new IntArrayPostingList(new int[]{ 1, 6 }),
+                new IntArrayPostingList(new int[]{ 8, 9, 11 }),
+                new IntArrayPostingList(new int[]{ 15 }));
 
         final PostingList merged = MergePostingList.merge(lists);
 
-        assertPostingListEquals(new ArrayPostingList(1, 6, 8, 9, 11, 15), merged);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{ 1, 6, 8, 9, 11, 15 }), merged);
     }
 
     @Test
     public void shouldMergeSinglePostingList() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(1, 4, 6));
+        var lists = listOfLists(new IntArrayPostingList(new int[]{ 1, 4, 6 }));
 
         final PostingList merged = MergePostingList.merge(lists);
 
-        assertPostingListEquals(new ArrayPostingList(1, 4, 6), merged);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{ 1, 4, 6 }), merged);
     }
 
     @Test
     public void shouldMergeSamePostingLists() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(0),
-                                                                          new ArrayPostingList(0));
+        var lists = listOfLists(new IntArrayPostingList(new int[]{ 0 }),
+                                                                                 new IntArrayPostingList(new int[]{ 0 }));
 
         final PostingList merged = MergePostingList.merge(lists);
 
-        assertPostingListEquals(new ArrayPostingList(0), merged);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{ 0 }), merged);
     }
 
     @Test
     public void shouldAdvanceAllMergedLists() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(1, 5, 10),
-                                                                          new ArrayPostingList(2, 3, 8),
-                                                                          new ArrayPostingList(3, 5, 9));
+        var lists = listOfLists(
+                new IntArrayPostingList(new int[]{ 1, 5, 10 }),
+                new IntArrayPostingList(new int[]{ 2, 3, 8 }),
+                new IntArrayPostingList(new int[]{ 3, 5, 9 }));
 
         final PostingList merged = MergePostingList.merge(lists);
-        final PostingList expected = new ArrayPostingList(1, 2, 3, 5, 8, 9, 10);
+        final PostingList expected = new IntArrayPostingList(new int[]{ 1, 2, 3, 5, 8, 9, 10 });
 
-        assertEquals(expected.advance(9), merged.advance(9));
+        assertEquals(expected.advance(9),
+                     merged.advance(9));
 
         assertPostingListEquals(expected, merged);
     }
@@ -108,37 +105,51 @@ public void shouldAdvanceAllMergedLists() throws IOException
     @Test
     public void shouldConsumeDuplicatedPostingOnAdvance() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(1, 4, 6),
-                                                                          new ArrayPostingList(2, 3, 4),
-                                                                          new ArrayPostingList(1, 6),
-                                                                          new ArrayPostingList(2, 5),
-                                                                          new ArrayPostingList(3, 6),
-                                                                          new ArrayPostingList(3, 5, 6));
+        var lists = listOfLists(
+                new IntArrayPostingList(new int[]{ 1, 4, 6 }),
+                new IntArrayPostingList(new int[]{ 2, 3, 4 }),
+                new IntArrayPostingList(new int[]{ 1, 6 }),
+                new IntArrayPostingList(new int[]{ 2, 5 }),
+                new IntArrayPostingList(new int[]{ 3, 6 }),
+                new IntArrayPostingList(new int[]{ 3, 5, 6 }));
 
         final PostingList merged = MergePostingList.merge(lists);
 
         assertEquals(2, merged.advance(2));
         assertEquals(4, merged.advance(4));
-        assertPostingListEquals(new ArrayPostingList(5, 6), merged);
+        assertPostingListEquals(new IntArrayPostingList(new int[]{ 5, 6 }), merged);
+    }
+
+    @Test
+    public void handleEmptyLists() throws IOException
+    {
+        var lists = listOfLists(
+        new IntArrayPostingList(new int[]{ }),
+        new IntArrayPostingList(new int[]{ }));
+
+        final PostingList merged = MergePostingList.merge(lists);
+
+        // merged.advance() should not throw
+        assertEquals(PostingList.END_OF_STREAM, merged.advance(-1));
     }
 
     @Test
     public void shouldInterleaveNextAndAdvance() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(1, 4, 6),
-                                                                          new ArrayPostingList(2, 3, 4),
-                                                                          new ArrayPostingList(1, 6),
-                                                                          new ArrayPostingList(2, 5),
-                                                                          new ArrayPostingList(3, 6),
-                                                                          new ArrayPostingList(3, 5, 6));
-
-        try (PostingList merged = MergePostingList.merge(lists))
-        {
-            assertEquals(2, merged.advance(2));
-            assertEquals(3, merged.nextPosting());
-            assertEquals(5, merged.advance(5));
-            assertEquals(6, merged.nextPosting());
-        }
+        var lists = listOfLists(
+                new IntArrayPostingList(new int[]{ 1, 4, 6 }),
+                new IntArrayPostingList(new int[]{ 2, 3, 4 }),
+                new IntArrayPostingList(new int[]{ 1, 6 }),
+                new IntArrayPostingList(new int[]{ 2, 5 }),
+                new IntArrayPostingList(new int[]{ 3, 6 }),
+                new IntArrayPostingList(new int[]{ 3, 5, 6 }));
+
+        final PostingList merged = MergePostingList.merge(lists);
+
+        assertEquals(2, merged.advance(2));
+        assertEquals(3, merged.nextPosting());
+        assertEquals(5, merged.advance(5));
+        assertEquals(6, merged.nextPosting());
     }
 
     @Test
@@ -150,50 +161,31 @@ public void shouldAdvanceToAllElementsWithoutFailures()
     @Test
     public void shouldNotSkipUnconsumedElementOnAdvance() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(1, 2),
-                                                                          new ArrayPostingList(3));
+        var lists = listOfLists(
+                new IntArrayPostingList(new int[]{ 1, 2 }),
+                new IntArrayPostingList(new int[]{ 3 }));
 
-        try (PostingList merged = MergePostingList.merge(lists))
-        {
-            assertEquals(1, merged.nextPosting());
-            assertEquals(2, merged.advance(2));
-            assertEquals(3, merged.nextPosting());
-        }
+        final PostingList merged = MergePostingList.merge(lists);
+        assertEquals(1, merged.nextPosting());
+        assertEquals(2, merged.advance(2));
+        assertEquals(3, merged.nextPosting());
     }
 
     @Test
     public void shouldNotReadFromExhaustedChild() throws IOException
     {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(2),
-                                                                          new ArrayPostingList(1, 3, 4));
-
-        try (PostingList merged = MergePostingList.merge(lists))
-        {
-            assertEquals(1, merged.nextPosting());
-            assertEquals(3, merged.advance(3));
-            assertEquals(4, merged.advance(4));
-        }
-    }
-
-    @Test
-    public void shouldSkipDuplicates() throws IOException
-    {
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(1, 1, 2, 2, 2, 2, 5, 5),
-                                                                          new ArrayPostingList(1, 2, 2, 3, 3, 4, 4, 5));
+        var lists = listOfLists(
+                new IntArrayPostingList(new int[]{ 2 }),
+                new IntArrayPostingList(new int[]{ 1, 3, 4 }));
 
-        try (PostingList merged = MergePostingList.merge(lists))
-        {
-            assertEquals(1, merged.nextPosting());
-            assertEquals(2, merged.nextPosting());
-            assertEquals(3, merged.advance(3));
-            assertEquals(4, merged.advance(4));
-            assertEquals(5, merged.nextPosting());
-            assertEquals(PostingList.END_OF_STREAM, merged.nextPosting());
-        }
+        final PostingList merged = MergePostingList.merge(lists);
+        assertEquals(1, merged.nextPosting());
+        assertEquals(3, merged.advance(3));
+        assertEquals(4, merged.advance(4));
     }
 
     @Test
-    public void shouldInterleaveNextAndAdvanceOnRandom()
+    public void shouldInterleaveNextAndAdvanceOnRandom() throws IOException
     {
         for (int i = 0; i < 1000; ++i)
         {
@@ -201,40 +193,42 @@ public void shouldInterleaveNextAndAdvanceOnRandom()
         }
     }
 
-    private PriorityQueue<PeekablePostingList> newPriorityQueue(PostingList...postingLists)
+    private ArrayList<PostingList> listOfLists(PostingList...postingLists)
     {
-        PriorityQueue<PeekablePostingList> queue = new PriorityQueue<>(postingLists.length, Comparator.comparingLong(PeekablePostingList::peek));
+        var L = new ArrayList<PostingList>();
         for (PostingList postingList : postingLists)
-            queue.add(PeekablePostingList.makePeekable(postingList));
-        return queue;
+            L.add(postingList);
+        return L;
     }
     
-    private void testAdvancingOnRandom()
+    private void testAdvancingOnRandom() throws IOException
     {
         final int postingsCount = nextInt(1, 50_000);
         final int postingListCount = nextInt(5, 50);
 
         final AtomicInteger rowId = new AtomicInteger();
-        final long[] postings = LongStream.generate(() -> rowId.addAndGet(nextInt(0, 10)))
-                                          .limit(postingsCount)
-                                          .toArray();
-        final long[] postingsWithoutDuplicates = LongStream.of(postings)
-                                                           .distinct()
-                                                           .toArray();
+        final int[] postings = IntStream.generate(() -> rowId.addAndGet(nextInt(0, 10)))
+                                        .limit(postingsCount)
+                                        .toArray();
+        final int[] postingsWithoutDuplicates = IntStream.of(postings)
+                                                         .distinct()
+                                                         .toArray();
 
         // split postings into multiple lists
-        final Map<Integer, List<Long>> splitPostings = Arrays.stream(postings)
-                                                             .boxed()
-                                                             .collect(Collectors.groupingBy(it -> nextInt(postingListCount)));
+        final Map<Integer, List<Integer>> splitPostings = Arrays.stream(postings)
+                                                                .boxed()
+                                                                .collect(Collectors.groupingBy(it -> nextInt(postingListCount)));
 
-        final PriorityQueue<PeekablePostingList> splitPostingLists = new PriorityQueue<>(splitPostings.size(), Comparator.comparingLong(PeekablePostingList::peek));
-        for (List<Long> split : splitPostings.values())
+        var splitPostingLists = new ArrayList<PostingList>();
+        for (List<Integer> split : splitPostings.values())
         {
-            splitPostingLists.add(PeekablePostingList.makePeekable(new ArrayPostingList(Longs.toArray(split))));
+            // Remove any duplicates in each individual set
+            int[] data = split.stream().distinct().mapToInt(Integer::intValue).toArray();
+            splitPostingLists.add(new IntArrayPostingList(data));
         }
 
         final PostingList merge = MergePostingList.merge(splitPostingLists);
-        final PostingList expected = new ArrayPostingList(postingsWithoutDuplicates);
+        final PostingList expected = new IntArrayPostingList(postingsWithoutDuplicates);
 
         final List<PostingListAdvance> actions = new ArrayList<>();
         for (int idx = 0; idx < postingsWithoutDuplicates.length; idx++)
@@ -257,7 +251,7 @@ private void testAdvancingOnRandom()
             {
                 final int skips = nextInt(0, 10);
                 idx = Math.min(idx + skips, postingsWithoutDuplicates.length - 1);
-                final long rowID = postingsWithoutDuplicates[idx];
+                final int rowID = postingsWithoutDuplicates[idx];
                 actions.add((postingList) -> {
                     while (true)
                     {
@@ -285,49 +279,48 @@ private void testAdvancingOnRandom()
 
     private void testAdvancingToAllElements()
     {
-        final long[] postings1 = randomPostings();
-        final long[] postings2 = randomPostings();
+        final int[] postings1 = randomPostings();
+        final int[] postings2 = randomPostings();
 
-        final long[] mergedPostings = LongStream.concat(LongStream.of(postings1), LongStream.of(postings2))
-                                                .distinct()
-                                                .sorted()
-                                                .toArray();
+        final int[] mergedPostings = IntStream.concat(IntStream.of(postings1), IntStream.of(postings2))
+                                              .distinct()
+                                              .sorted()
+                                              .toArray();
 
-        final PriorityQueue<PeekablePostingList> lists = newPriorityQueue(new ArrayPostingList(postings1), new ArrayPostingList(postings2));
+        var lists = listOfLists(new IntArrayPostingList(postings1), new IntArrayPostingList(postings2));
 
-        try (PostingList merged = MergePostingList.merge(lists))
+        final PostingList merged = MergePostingList.merge(lists);
+
+        // tokens are equal row IDs in this test case
+        for (int targetRowID : mergedPostings)
         {
-            // tokens are equal row IDs in this test case
-            for (long targetRowID : mergedPostings)
+            long rowID;
+            while (true)
             {
-                long rowID;
-                while (true)
+                try
                 {
-                    try
-                    {
-                        rowID = merged.advance(targetRowID);
-                        break;
-                    }
-                    catch (Exception e)
-                    {
-                        fail();
-                    }
+                    rowID = merged.advance(targetRowID);
+                    break;
+                }
+                catch (Exception e)
+                {
+                    fail();
                 }
-                assertEquals(targetRowID, rowID);
             }
+            assertEquals(targetRowID, rowID);
         }
     }
 
-    private long[] randomPostings()
+    private int[] randomPostings()
     {
         final AtomicInteger rowId = new AtomicInteger();
-        return LongStream.generate(() -> rowId.getAndAdd(getRandom().nextIntBetween(0, 5)))
-                         .limit(getRandom().nextIntBetween(1 << 10, 1 << 12))
-                         .toArray();
+        return IntStream.generate(() -> rowId.getAndAdd(randomIntBetween(0, 5)))
+                        .limit(randomIntBetween(1 << 10, 1 << 12))
+                        .toArray();
     }
 
     private interface PostingListAdvance
     {
-        long advance(PostingList list);
+        long advance(PostingList list) throws IOException;
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java
index c437ffbdeba2..451789e872c9 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java
@@ -19,43 +19,42 @@
 
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.LongStream;
+import java.util.stream.IntStream;
 
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
 import org.apache.cassandra.index.sai.disk.format.IndexComponent;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
 import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.cassandra.index.sai.disk.ArrayPostingList;
-import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.lucene.store.IndexInput;
+import org.apache.cassandra.index.sai.postings.IntArrayPostingList;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class PostingsTest extends SAIRandomizedTester
+public class PostingsTest extends SaiRandomizedTest
 {
     @Rule
     public final ExpectedException expectedException = ExpectedException.none();
 
     private IndexDescriptor indexDescriptor;
-    private IndexIdentifier indexIdentifier;
+    private String index;
+    private IndexContext indexContext;
 
     @Before
     public void setup() throws Throwable
     {
         indexDescriptor = newIndexDescriptor();
-        String index = newIndex();
-        indexIdentifier = SAITester.createIndexIdentifier(indexDescriptor.sstableDescriptor.ksname,
-                                                          indexDescriptor.sstableDescriptor.cfname,
-                                                          index);
+        index = newIndex();
+        indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
 
     }
 
@@ -63,16 +62,17 @@ public void setup() throws Throwable
     public void testSingleBlockPostingList() throws Exception
     {
         final int blockSize = 1 << between(3, 8);
-        final ArrayPostingList expectedPostingList = new ArrayPostingList(10, 20, 30, 40, 50, 60);
+        final IntArrayPostingList expectedPostingList = new IntArrayPostingList(new int[]{ 10, 20, 30, 40, 50, 60 });
 
         long postingPointer;
-        try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (PostingsWriter writer = new PostingsWriter(components, blockSize))
         {
             postingPointer = writer.write(expectedPostingList);
             writer.complete();
         }
 
-        IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
+        IndexInput input = components.get(IndexComponentType.POSTING_LISTS).openInput();
         SAICodecUtils.validate(input);
         input.seek(postingPointer);
 
@@ -97,7 +97,7 @@ public void testSingleBlockPostingList() throws Exception
         reader.close();
         assertEquals(reader.size(), listener.decodes);
 
-        input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
+        input = components.get(IndexComponentType.POSTING_LISTS).openInput();
         listener = new CountingPostingListEventListener();
         reader = new PostingsReader(input, postingPointer, listener);
 
@@ -114,37 +114,39 @@ public void testSingleBlockPostingList() throws Exception
     public void testMultiBlockPostingList() throws Exception
     {
         final int numPostingLists = 1 << between(1, 5);
-        final int blockSize = 1 << between(5, 8);
-        final int numPostings = getRandom().nextIntBetween(1 << 11, 1 << 16);
-        final ArrayPostingList[] expected = new ArrayPostingList[numPostingLists];
+        final int blockSize = 1 << between(5, 10);
+        final int numPostings = between(1 << 11, 1 << 15);
+        final IntArrayPostingList[] expected = new IntArrayPostingList[numPostingLists];
         final long[] postingPointers = new long[numPostingLists];
 
-        try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (PostingsWriter writer = new PostingsWriter(components, blockSize))
         {
             for (int i = 0; i < numPostingLists; ++i)
             {
-                final long[] postings = randomPostings(numPostings);
-                final ArrayPostingList postingList = new ArrayPostingList(postings);
+                final int[] postings = randomPostings(numPostings);
+                final IntArrayPostingList postingList = new IntArrayPostingList(postings);
                 expected[i] = postingList;
                 postingPointers[i] = writer.write(postingList);
             }
             writer.complete();
         }
 
-        try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier))
+        IndexComponent.ForRead postingLists = components.get(IndexComponentType.POSTING_LISTS);
+        try (IndexInput input = postingLists.openInput())
         {
             SAICodecUtils.validate(input);
         }
 
         for (int i = 0; i < numPostingLists; ++i)
         {
-            IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
+            IndexInput input = postingLists.openInput();
             input.seek(postingPointers[i]);
-            ArrayPostingList expectedPostingList = expected[i];
-            PostingsReader.BlocksSummary summary = assertBlockSummary(blockSize, expectedPostingList, input);
+            final IntArrayPostingList expectedPostingList = expected[i];
+            final PostingsReader.BlocksSummary summary = assertBlockSummary(blockSize, expectedPostingList, input);
             assertTrue(summary.offsets.length() > 1);
 
-            CountingPostingListEventListener listener = new CountingPostingListEventListener();
+            final CountingPostingListEventListener listener = new CountingPostingListEventListener();
             try (PostingsReader reader = new PostingsReader(input, postingPointers[i], listener))
             {
                 expectedPostingList.reset();
@@ -155,27 +157,11 @@ public void testMultiBlockPostingList() throws Exception
                 assertEquals(0, listener.advances);
             }
 
-            // test random advances through the posting list
-            listener = new CountingPostingListEventListener();
-            input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
-            try (PostingsReader reader = new PostingsReader(input, postingPointers[i], listener))
-            {
-                expectedPostingList.reset();
-                int advances = 0;
-                for (int p = 0; p < numPostings - blockSize; p += getRandom().nextIntBetween(1, blockSize / 2))
-                {
-                    advances++;
-                    assertEquals(expectedPostingList.advance(p), reader.advance(p));
-                }
-                assertEquals(advances, listener.advances);
-            }
-
             // test skipping to the last block
-            listener = new CountingPostingListEventListener();
-            input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
+            input = postingLists.openInput();
             try (PostingsReader reader = new PostingsReader(input, postingPointers[i], listener))
             {
-                long tokenToAdvance = -1;
+                int tokenToAdvance = -1;
                 expectedPostingList.reset();
                 for (int p = 0; p < numPostings - 7; ++p)
                 {
@@ -192,45 +178,24 @@ public void testMultiBlockPostingList() throws Exception
         }
     }
 
-    @Test
-    public void testDuplicatePostings() throws Exception
-    {
-        int blockSize = 4;
-        // For the duplicate testing code to work we need to have a block full of duplicate values
-        // with the end value of the preceeding block having the same duplicate value.
-        final ArrayPostingList expectedPostingList = new ArrayPostingList(0, 1, 1, 3, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 10, 11, 12);
-
-        long postingPointer;
-        try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
-        {
-            postingPointer = writer.write(expectedPostingList);
-            writer.complete();
-        }
-
-        IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
-        CountingPostingListEventListener listener = new CountingPostingListEventListener();
-        try (PostingsReader reader = new PostingsReader(input, postingPointer, listener))
-        {
-            assertEquals(7L, reader.advance(7));
-        }
-    }
-
     @Test
     public void testAdvance() throws Exception
     {
         final int blockSize = 4; // 4 postings per FoR block
         final int maxSegmentRowID = 30;
-        final long[] postings = LongStream.range(0, maxSegmentRowID).toArray(); // 30 postings = 7 FoR blocks + 1 VLong block
-        final ArrayPostingList expected = new ArrayPostingList(postings);
+        final int[] postings = IntStream.range(0, maxSegmentRowID).toArray(); // 30 postings = 7 FoR blocks + 1 VLong block
+        final IntArrayPostingList expected = new IntArrayPostingList(postings);
 
         long fp;
-        try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (PostingsWriter writer = new PostingsWriter(components, blockSize))
         {
             fp = writer.write(expected);
             writer.complete();
         }
 
-        try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier))
+        IndexComponent.ForRead postingLists = components.get(IndexComponentType.POSTING_LISTS);
+        try (IndexInput input = postingLists.openInput())
         {
             SAICodecUtils.validate(input);
             input.seek(fp);
@@ -245,14 +210,14 @@ public void testAdvance() throws Exception
         }
 
         // exact advance
-        testAdvance(fp, expected, new long[]{ 3, 7, 11, 15, 19 });
+        testAdvance(postingLists, fp, expected, new int[]{ 3, 7, 11, 15, 19 });
         // non-exact advance
-        testAdvance(fp, expected, new long[]{ 2, 6, 12, 17, 25 });
+        testAdvance(postingLists, fp, expected, new int[]{ 2, 6, 12, 17, 25 });
 
         // exact advance
-        testAdvance(fp, expected, new long[]{ 3, 5, 7, 12 });
+        testAdvance(postingLists, fp, expected, new int[]{ 3, 5, 7, 12 });
         // non-exact advance
-        testAdvance(fp, expected, new long[]{ 2, 7, 9, 11 });
+        testAdvance(postingLists, fp, expected, new int[]{ 2, 7, 9, 11 });
     }
 
     @Test
@@ -260,18 +225,20 @@ public void testAdvanceOnRandomizedData() throws IOException
     {
         final int blockSize = 4;
         final int numPostings = nextInt(64, 64_000);
-        final long[] postings = randomPostings(numPostings);
+        final int[] postings = randomPostings(numPostings);
 
-        final ArrayPostingList expected = new ArrayPostingList(postings);
+        final IntArrayPostingList expected = new IntArrayPostingList(postings);
 
         long fp;
-        try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (PostingsWriter writer = new PostingsWriter(components, blockSize))
         {
             fp = writer.write(expected);
             writer.complete();
         }
 
-        try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier))
+        IndexComponent.ForRead postingLists = components.get(IndexComponentType.POSTING_LISTS);
+        try (IndexInput input = postingLists.openInput())
         {
             SAICodecUtils.validate(input);
             input.seek(fp);
@@ -285,14 +252,14 @@ public void testAdvanceOnRandomizedData() throws IOException
             }
         }
 
-        testAdvance(fp, expected, postings);
+        testAdvance(postingLists, fp, expected, postings);
     }
 
     @Test
-    @SuppressWarnings("all")
     public void testNullPostingList() throws IOException
     {
-        try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (PostingsWriter writer = new PostingsWriter(components))
         {
             expectedException.expect(IllegalArgumentException.class);
             writer.write(null);
@@ -303,35 +270,37 @@ public void testNullPostingList() throws IOException
     @Test
     public void testEmptyPostingList() throws IOException
     {
-        try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (PostingsWriter writer = new PostingsWriter(components))
         {
             expectedException.expect(IllegalArgumentException.class);
-            writer.write(new ArrayPostingList());
+            writer.write(new IntArrayPostingList(new int[0]));
         }
     }
 
     @Test
     public void testNonAscendingPostingList() throws IOException
     {
-        try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (PostingsWriter writer = new PostingsWriter(components))
         {
             expectedException.expect(IllegalArgumentException.class);
-            writer.write(new ArrayPostingList(1, 0));
+            writer.write(new IntArrayPostingList(new int[]{ 1, 0 }));
         }
     }
 
-    private void testAdvance(long fp, ArrayPostingList expected, long[] targetIDs) throws IOException
+    private void testAdvance(IndexComponent.ForRead postingLists, long fp, IntArrayPostingList expected, int[] targetIDs) throws IOException
     {
         expected.reset();
         final CountingPostingListEventListener listener = new CountingPostingListEventListener();
-        PostingsReader reader = openReader(fp, listener);
+        PostingsReader reader = openReader(postingLists, fp, listener);
         for (int i = 0; i < 2; ++i)
         {
             assertEquals(expected.nextPosting(), reader.nextPosting());
             assertEquals(expected.getOrdinal(), reader.getOrdinal());
         }
 
-        for (long target : targetIDs)
+        for (int target : targetIDs)
         {
             final long actualRowId = reader.advance(target);
             final long expectedRowId = expected.advance(target);
@@ -349,9 +318,9 @@ private void testAdvance(long fp, ArrayPostingList expected, long[] targetIDs) t
         reader.close();
     }
 
-    private PostingsReader openReader(long fp, QueryEventListener.PostingListEventListener listener) throws IOException
+    private PostingsReader openReader(IndexComponent.ForRead postingLists, long fp, QueryEventListener.PostingListEventListener listener) throws IOException
     {
-        IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
+        IndexInput input = postingLists.openInput();
         input.seek(fp);
         return new PostingsReader(input, fp, listener);
     }
@@ -359,20 +328,20 @@ private PostingsReader openReader(long fp, QueryEventListener.PostingListEventLi
     private PostingsReader.BlocksSummary assertBlockSummary(int blockSize, PostingList expected, IndexInput input) throws IOException
     {
         final PostingsReader.BlocksSummary summary = new PostingsReader.BlocksSummary(input, input.getFilePointer());
-        assertEquals(blockSize, summary.blockSize);
+        assertEquals(blockSize, summary.blockEntries);
         assertEquals(expected.size(), summary.numPostings);
         assertTrue(summary.offsets.length() > 0);
         assertEquals(summary.offsets.length(), summary.maxValues.length());
         return summary;
     }
 
-    private long[] randomPostings(int numPostings)
+    private int[] randomPostings(int numPostings)
     {
         final AtomicInteger rowId = new AtomicInteger();
         // postings with duplicates
-        return LongStream.generate(() -> rowId.getAndAdd(getRandom().nextIntBetween(0, 3)))
-                         .limit(numPostings)
-                         .toArray();
+        return IntStream.generate(() -> rowId.getAndAdd(randomIntBetween(0, 4)))
+                        .limit(numPostings)
+                        .toArray();
     }
 
     static class CountingPostingListEventListener implements QueryEventListener.PostingListEventListener
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/ReorderingPostingListTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/ReorderingPostingListTest.java
new file mode 100644
index 000000000000..10b2b6a33f47
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/ReorderingPostingListTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.utils.RowIdWithScore;
+import org.apache.cassandra.utils.CloseableIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ReorderingPostingListTest
+{
+    @Test
+    public void ensureEmptySourceBehavesCorrectly() throws Throwable
+    {
+        var source = new TestIterator(CloseableIterator.emptyIterator());
+
+        try (var postingList = new ReorderingPostingList(source, RowIdWithScore::getSegmentRowId))
+        {
+            // Even an empty source should be closed
+            assertTrue(source.isClosed);
+            assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
+        }
+    }
+
+    @Test
+    public void ensureIteratorIsConsumedClosedAndReordered() throws Throwable
+    {
+        var source = new TestIterator(Arrays.stream(new RowIdWithScore[] {
+                new RowIdWithScore(3, 3),
+                new RowIdWithScore(2, 2),
+                new RowIdWithScore(1, 1),
+                new RowIdWithScore(4, 4),
+        }).iterator());
+
+        try (var postingList = new ReorderingPostingList(source, RowIdWithScore::getSegmentRowId))
+        {
+            // The posting list is eagerly consumed, so it should be closed before
+            // we close postingList
+            assertTrue(source.isClosed);
+            assertEquals(4, postingList.size());
+
+            // Verify ordering
+            assertEquals(1, postingList.nextPosting());
+            assertEquals(2, postingList.nextPosting());
+            assertEquals(3, postingList.nextPosting());
+            assertEquals(4, postingList.nextPosting());
+            assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
+        }
+    }
+
+    @Test
+    public void ensureAdvanceWorksCorrectly() throws Throwable
+    {
+        var source = new TestIterator(Arrays.stream(new RowIdWithScore[] {
+        new RowIdWithScore(3, 3),
+        new RowIdWithScore(1, 1),
+        new RowIdWithScore(2, 2),
+        }).iterator());
+
+        try (var postingList = new ReorderingPostingList(source, RowIdWithScore::getSegmentRowId))
+        {
+            assertEquals(3, postingList.advance(3));
+            assertEquals(PostingList.END_OF_STREAM, postingList.advance(4));
+        }
+    }
+
+    /**
+     * A basic iterator that tracks whether it has been closed for better testing.
+     */
+    private static class TestIterator implements CloseableIterator<RowIdWithScore>
+    {
+        private final Iterator<RowIdWithScore> iterator;
+        boolean isClosed = false;
+        TestIterator(Iterator<RowIdWithScore> iter)
+        {
+            iterator = iter;
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            return iterator.hasNext();
+        }
+
+        @Override
+        public RowIdWithScore next()
+        {
+            return iterator.next();
+        }
+
+        @Override
+        public void close()
+        {
+            isClosed = true;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/ScanningPostingsReader.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/ScanningPostingsReader.java
deleted file mode 100644
index 99b1f5f22183..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/ScanningPostingsReader.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.postings;
-
-import java.io.IOException;
-
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.metrics.QueryEventListener;
-import org.apache.lucene.store.IndexInput;
-
-/**
- * A subclass of the {@link PostingsReader} that does not allow the {@link PostingList} to be
- * advanced and does not support mapping row ids to primary keys.
- *
- * It is used during index merges to sequentially scan the postings in order using {@link #nextPosting}.
- */
-public class ScanningPostingsReader extends PostingsReader
-{
-    public ScanningPostingsReader(IndexInput input, BlocksSummary summary) throws IOException
-    {
-        super(input, summary, QueryEventListener.PostingListEventListener.NO_OP);
-    }
-
-    @Override
-    public long advance(long targetRowId)
-    {
-        throw new UnsupportedOperationException("Cannot advance a scanning postings reader");
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/segment/SegmentRamBufferTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/segment/SegmentRamBufferTest.java
deleted file mode 100644
index 8f90816c55c1..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/segment/SegmentRamBufferTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.disk.v1.segment;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.StreamSupport;
-
-import org.junit.Test;
-
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.index.sai.utils.IndexEntry;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-import org.apache.cassandra.utils.bytecomparable.ByteSource;
-
-import static org.junit.Assert.assertEquals;
-
-public class SegmentRamBufferTest extends SAIRandomizedTester
-{
-    @Test
-    public void shouldReturnValuesInSortedValue()
-    {
-        int numRows = nextInt(100, 1000);
-
-        // Generate a random unsorted list of integers
-        List<Integer> values = IntStream.generate(() -> nextInt(0, 1000))
-                                        .distinct()
-                                        .limit(numRows)
-                                        .boxed()
-                                        .collect(Collectors.toList());
-
-        SegmentTrieBuffer buffer = new SegmentTrieBuffer();
-
-        IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
-
-        values.forEach(value -> buffer.add(v -> indexTermType.asComparableBytes(Int32Type.instance.decompose(value), v), Integer.BYTES, 0));
-
-        Iterable<IndexEntry> iterable = buffer::iterator;
-
-        List<Integer> result = StreamSupport.stream(iterable.spliterator(), false).mapToInt(pair -> unpackValue(pair.term)).boxed().collect(Collectors.toList());
-
-        Collections.sort(values);
-
-        assertEquals(values, result);
-    }
-
-    private static int unpackValue(ByteComparable value)
-    {
-        return Int32Type.instance.compose(Int32Type.instance.fromComparableBytes(ByteSource.peekable(value.asComparableBytes(ByteComparable.Version.OSS50)),
-                                                                                 ByteComparable.Version.OSS50));
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java
index 8569fa9a90e3..e86cc600de43 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java
@@ -21,6 +21,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -28,71 +29,343 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
-import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.OSS50;
+import static org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader.NOT_FOUND;
 import static org.apache.cassandra.utils.bytecomparable.ByteComparable.compare;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 
-public class TrieTermsDictionaryTest extends SAIRandomizedTester
+public class TrieTermsDictionaryTest extends SaiRandomizedTest
 {
+    public final static ByteComparable.Version VERSION = TypeUtil.BYTE_COMPARABLE_VERSION;
+
     private IndexDescriptor indexDescriptor;
-    private IndexIdentifier indexIdentifier;
+    private String index;
+    private IndexContext indexContext;
 
     @Before
     public void setup() throws Throwable
     {
         indexDescriptor = newIndexDescriptor();
-        indexIdentifier = createIndexIdentifier("test", "test", newIndex());
+        index = newIndex();
+        indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
     }
 
     @Test
     public void testExactMatch() throws Exception
     {
-        doTestExactMatch();
+        testForDifferentByteComparableEncodings(this::doTestExactMatch);
+    }
+
+    private void doTestExactMatch(Function<String, ByteComparable> asByteComparable) throws Exception
+    {
+        long fp;
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(components))
+        {
+            writer.add(asByteComparable.apply("ab"), 0);
+            writer.add(asByteComparable.apply("abb"), 1);
+            writer.add(asByteComparable.apply("abc"), 2);
+            writer.add(asByteComparable.apply("abcd"), 3);
+            writer.add(asByteComparable.apply("abd"), 4);
+            fp = writer.complete(new MutableLong());
+        }
+
+        try (FileHandle input = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+             TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp, VERSION))
+        {
+            assertEquals(NOT_FOUND, reader.exactMatch(asByteComparable.apply("a")));
+            assertEquals(0, reader.exactMatch(asByteComparable.apply("ab")));
+            assertEquals(2, reader.exactMatch(asByteComparable.apply("abc")));
+            assertEquals(NOT_FOUND, reader.exactMatch(asByteComparable.apply("abca")));
+            assertEquals(1, reader.exactMatch(asByteComparable.apply("abb")));
+            assertEquals(NOT_FOUND, reader.exactMatch(asByteComparable.apply("abba")));
+        }
+    }
+
+    @Test
+    public void testCeilingWithoutTrackingState() throws Exception
+    {
+        testForDifferentByteComparableEncodings(this::doTestCeiling);
+    }
+
+    private void doTestCeiling(Function<String, ByteComparable> asByteComparable) throws Exception
+    {
+        long fp;
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(components))
+        {
+            writer.add(asByteComparable.apply("ab"), 0);
+            writer.add(asByteComparable.apply("abb"), 1);
+            writer.add(asByteComparable.apply("abc"), 2);
+            writer.add(asByteComparable.apply("abcd"), 3);
+            writer.add(asByteComparable.apply("abd"), 4);
+            writer.add(asByteComparable.apply("cbb"), 5);
+            writer.add(asByteComparable.apply("cbbbb"), 6);
+            fp = writer.complete(new MutableLong());
+        }
+
+        IndexComponent.ForRead termsData = components.get(IndexComponentType.TERMS_DATA);
+        ByteComparable key13 = asByteComparable.apply("A");
+        readAndAssertCeiling(fp, 0, key13, termsData);
+        ByteComparable key12 = asByteComparable.apply("a");
+        readAndAssertCeiling(fp, 0, key12, termsData);
+        ByteComparable key11 = asByteComparable.apply("z");
+        readAndAssertCeiling(fp, NOT_FOUND, key11, termsData);
+        ByteComparable key10 = asByteComparable.apply("ab");
+        readAndAssertCeiling(fp, 0, key10, termsData);
+        ByteComparable key9 = asByteComparable.apply("abbb");
+        readAndAssertCeiling(fp, 2, key9, termsData);
+        ByteComparable key8 = asByteComparable.apply("abc");
+        readAndAssertCeiling(fp, 2, key8, termsData);
+        ByteComparable key7 = asByteComparable.apply("abca");
+        readAndAssertCeiling(fp, 3, key7, termsData);
+        ByteComparable key6 = asByteComparable.apply("abb");
+        readAndAssertCeiling(fp, 1, key6, termsData);
+        ByteComparable key5 = asByteComparable.apply("abba");
+        readAndAssertCeiling(fp, 2, key5, termsData);
+        ByteComparable key4 = asByteComparable.apply("cb");
+        readAndAssertCeiling(fp, 5, key4, termsData);
+        ByteComparable key3 = asByteComparable.apply("c");
+        readAndAssertCeiling(fp, 5, key3, termsData);
+        ByteComparable key2 = asByteComparable.apply("cbb");
+        readAndAssertCeiling(fp, 5, key2, termsData);
+        ByteComparable key1 = asByteComparable.apply("cbbb");
+        readAndAssertCeiling(fp, 6, key1, termsData);
+        ByteComparable key = asByteComparable.apply("cbbbbb");
+        readAndAssertCeiling(fp, NOT_FOUND, key, termsData);
+    }
+
+    @Test
+    public void testCeilingTrackingState() throws Exception
+    {
+        testForDifferentByteComparableEncodings(this::doTestCeilingStateful);
+    }
+
+    private void doTestCeilingStateful(Function<String, ByteComparable> asByteComparable) throws Exception
+    {
+        long fp;
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(components))
+        {
+            writer.add(asByteComparable.apply("ab"), 0);
+            writer.add(asByteComparable.apply("abb"), 1);
+            writer.add(asByteComparable.apply("abc"), 2);
+            writer.add(asByteComparable.apply("abcd"), 3);
+            writer.add(asByteComparable.apply("abd"), 4);
+            writer.add(asByteComparable.apply("cbb"), 5);
+            writer.add(asByteComparable.apply("cbbbb"), 6);
+            fp = writer.complete(new MutableLong());
+        }
+
+        try (FileHandle input = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+             TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp, VERSION))
+        {
+            assertEquals(0, reader.ceiling(asByteComparable.apply("a")));
+            assertEquals(2, reader.ceiling(asByteComparable.apply("abc")));
+            assertEquals(3, reader.ceiling(asByteComparable.apply("abcc")));
+
+            // The current behavior is to advance past the node that the ceiling returns.
+            // As such, even though abccc is before abcd, the ceiling will return 4 for abd.
+            assertEquals(4, reader.ceiling(asByteComparable.apply("abccc")));
+        }
+    }
+
+    @Test
+    public void testCeilingWihtoutTrackingStateWithEmulatedPrimaryKey() throws Exception
+    {
+        testForDifferentByteComparableEncodings(this::doTestCeilingWithEmulatedPrimaryKey);
+    }
+
+    private void doTestCeilingWithEmulatedPrimaryKey(Function<String, ByteComparable> asByteComparable) throws Exception
+    {
+        long fp;
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(components))
+        {
+            writer.add(primaryKey(asByteComparable, "ab", "cd", "def"), 0);
+            writer.add(primaryKey(asByteComparable, "ab", "cde", "def"), 1);
+            writer.add(primaryKey(asByteComparable, "ab", "ce", "def"), 2);
+            writer.add(primaryKey(asByteComparable, "ab", "ce", "defg"), 3);
+            writer.add(primaryKey(asByteComparable, "ab", "cf", "def"), 4);
+            fp = writer.complete(new MutableLong());
+        }
+
+        IndexComponent.ForRead termsData = components.get(IndexComponentType.TERMS_DATA);
+
+        // Validate token only searches
+        ByteComparable key17 = primaryKey(asByteComparable, "a", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 0, key17, termsData);
+        ByteComparable key16 = primaryKey(asByteComparable, "ab", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 0, key16, termsData);
+        ByteComparable key15 = primaryKey(asByteComparable, "aa", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 0, key15, termsData);
+        ByteComparable key14 = primaryKey(asByteComparable, "abc", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, NOT_FOUND, key14, termsData);
+        ByteComparable key13 = primaryKey(asByteComparable, "ba", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, NOT_FOUND, key13, termsData);
+
+        // Validate token and partition key only searches
+        ByteComparable key12 = primaryKey(asByteComparable, "a", "b", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 0, key12, termsData);
+        ByteComparable key11 = primaryKey(asByteComparable, "ab", "b", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 0, key11, termsData);
+        ByteComparable key10 = primaryKey(asByteComparable, "ab", "ce", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 2, key10, termsData);
+        ByteComparable key9 = primaryKey(asByteComparable, "ab", "cee", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 4, key9, termsData);
+        ByteComparable key8 = primaryKey(asByteComparable, "ab", "d", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, NOT_FOUND, key8, termsData);
+        ByteComparable key7 = primaryKey(asByteComparable, "abb", "a", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, NOT_FOUND, key7, termsData);
+        ByteComparable key6 = primaryKey(asByteComparable, "aa", "d", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 0, key6, termsData);
+        ByteComparable key5 = primaryKey(asByteComparable, "abc", "a", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, NOT_FOUND, key5, termsData);
+        ByteComparable key4 = primaryKey(asByteComparable, "ba", "a", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, NOT_FOUND, key4, termsData);
+
+
+        // Validate token, partition key, and clustring column searches
+        ByteComparable key3 = primaryKey(asByteComparable, "a", "b", "c", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 0, key3, termsData);
+        ByteComparable key2 = primaryKey(asByteComparable, "ab", "cdd", "a", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 1, key2, termsData);
+        ByteComparable key1 = primaryKey(asByteComparable, "ab", "cde", "a", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 1, key1, termsData);
+        ByteComparable key = primaryKey(asByteComparable, "ab", "cde", "z", ByteSource.LT_NEXT_COMPONENT);
+        readAndAssertCeiling(fp, 2, key, termsData);
+    }
+
+    // Tests using this method are verifying the correctness of individual calls to ceiling. Because the reader is
+    // stateful across calls to ceiling, a new one must be opened for each call.
+    private void readAndAssertCeiling(long root, long expected, ByteComparable key, IndexComponent.ForRead termsData)
+    {
+        try (FileHandle input = termsData.createFileHandle();
+             TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), root, VERSION))
+        {
+            assertEquals(expected, reader.ceiling(key));
+        }
+    }
+
+    @Test
+    public void testFloor() throws Exception
+    {
+        testForDifferentByteComparableEncodings(this::doTestFloor);
     }
 
-    private void doTestExactMatch() throws Exception
+    private void doTestFloor(Function<String, ByteComparable> asByteComparable) throws Exception
     {
         long fp;
-        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(components))
         {
-            writer.add(asByteComparable("ab"), 0);
-            writer.add(asByteComparable("abb"), 1);
-            writer.add(asByteComparable("abc"), 2);
-            writer.add(asByteComparable("abcd"), 3);
-            writer.add(asByteComparable("abd"), 4);
+            writer.add(asByteComparable.apply("ab"), 0);
+            writer.add(asByteComparable.apply("abb"), 1);
+            writer.add(asByteComparable.apply("abc"), 2);
+            writer.add(asByteComparable.apply("abcd"), 3);
+            writer.add(asByteComparable.apply("abd"), 4);
+            writer.add(asByteComparable.apply("ca"), 5);
+            writer.add(asByteComparable.apply("caaaaa"), 6);
+            writer.add(asByteComparable.apply("cab"), 7);
             fp = writer.complete(new MutableLong());
         }
 
-        try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier);
-             TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp))
+        try (FileHandle input = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+             TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp, VERSION))
         {
-            assertEquals(TrieTermsDictionaryReader.NOT_FOUND, reader.exactMatch(asByteComparable("a")));
-            assertEquals(0, reader.exactMatch(asByteComparable("ab")));
-            assertEquals(2, reader.exactMatch(asByteComparable("abc")));
-            assertEquals(TrieTermsDictionaryReader.NOT_FOUND, reader.exactMatch(asByteComparable("abca")));
-            assertEquals(1, reader.exactMatch(asByteComparable("abb")));
-            assertEquals(TrieTermsDictionaryReader.NOT_FOUND, reader.exactMatch(asByteComparable("abba")));
+            assertEquals(NOT_FOUND, reader.floor(asByteComparable.apply("a")));
+            assertEquals(7, reader.floor(asByteComparable.apply("z")));
+            assertEquals(0, reader.floor(asByteComparable.apply("ab")));
+            assertEquals(2, reader.floor(asByteComparable.apply("abc")));
+            assertEquals(2, reader.floor(asByteComparable.apply("abca")));
+            assertEquals(1, reader.floor(asByteComparable.apply("abb")));
+            assertEquals(1, reader.floor(asByteComparable.apply("abba")));
+            assertEquals(4, reader.floor(asByteComparable.apply("abda")));
+            assertEquals(4, reader.floor(asByteComparable.apply("c")));
+            assertEquals(5, reader.floor(asByteComparable.apply("caaaa")));
+            assertEquals(7, reader.floor(asByteComparable.apply("cac")));
         }
     }
 
+
+
     @Test
-    public void testTermEnum() throws IOException
+    public void testFloorWithEmulatedPrimaryKey() throws Exception
     {
-        final List<ByteComparable> byteComparables = generateSortedByteComparables();
+        testForDifferentByteComparableEncodings(this::doTestFloorWithEmulatedPrimaryKey);
+    }
 
+    private void doTestFloorWithEmulatedPrimaryKey(Function<String, ByteComparable> asByteComparable) throws Exception
+    {
         long fp;
-        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(components))
+        {
+            writer.add(primaryKey(asByteComparable, "ab", "cd", "def"), 0);
+            writer.add(primaryKey(asByteComparable, "ab", "cde", "def"), 1);
+            writer.add(primaryKey(asByteComparable, "ab", "ce", "def"), 2);
+            writer.add(primaryKey(asByteComparable, "ab", "ce", "defg"), 3);
+            writer.add(primaryKey(asByteComparable, "ab", "cf", "def"), 4);
+            fp = writer.complete(new MutableLong());
+        }
+
+        try (FileHandle input = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+             TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp, VERSION))
+        {
+            // Validate token only searches
+            assertEquals(NOT_FOUND, reader.floor(primaryKey(asByteComparable, "a", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(4, reader.floor(primaryKey(asByteComparable, "ab", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(NOT_FOUND, reader.floor(primaryKey(asByteComparable, "aa", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(4, reader.floor(primaryKey(asByteComparable, "abc", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(4, reader.floor(primaryKey(asByteComparable, "ba", ByteSource.GT_NEXT_COMPONENT)));
+
+            // Validate token and partition key only searches
+            assertEquals(NOT_FOUND, reader.floor(primaryKey(asByteComparable, "a", "b", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(NOT_FOUND, reader.floor(primaryKey(asByteComparable, "ab", "b", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(3, reader.floor(primaryKey(asByteComparable, "ab", "ce", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(3, reader.floor(primaryKey(asByteComparable, "ab", "cee", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(4, reader.floor(primaryKey(asByteComparable, "ab", "d", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(4, reader.floor(primaryKey(asByteComparable, "abb", "a", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(NOT_FOUND, reader.floor(primaryKey(asByteComparable, "aa", "d", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(4, reader.floor(primaryKey(asByteComparable, "abc", "a", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(4, reader.floor(primaryKey(asByteComparable, "ba", "a", ByteSource.GT_NEXT_COMPONENT)));
+
+
+            // Validate token, partition key, and clustring column searches
+            assertEquals(NOT_FOUND, reader.floor(primaryKey(asByteComparable, "a", "b", "c", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(0, reader.floor(primaryKey(asByteComparable, "ab", "cdd", "a", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(0, reader.floor(primaryKey(asByteComparable, "ab", "cde", "a", ByteSource.GT_NEXT_COMPONENT)));
+            assertEquals(1, reader.floor(primaryKey(asByteComparable, "ab", "cde", "z", ByteSource.GT_NEXT_COMPONENT)));
+        }
+    }
+
+    @Test
+    public void testTermEnum() throws Exception
+    {
+        testForDifferentByteComparableEncodings(this::doTestTermEnum);
+    }
+
+
+    private void doTestTermEnum(Function<String, ByteComparable> asByteComparable) throws IOException
+    {
+        final List<ByteComparable> byteComparables = generateSortedByteComparables(asByteComparable);
+
+        long fp;
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(components))
         {
             for (int i = 0; i < byteComparables.size(); ++i)
             {
@@ -101,30 +374,44 @@ public void testTermEnum() throws IOException
             fp = writer.complete(new MutableLong());
         }
 
-        try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier);
-             TrieTermsIterator iterator = new TrieTermsIterator(input.instantiateRebufferer(null), fp))
+        try (FileHandle input = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+             TrieTermsDictionaryReader iterator = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp, VERSION);
+             ReverseTrieTermsDictionaryReader reverseIterator = new ReverseTrieTermsDictionaryReader(input.instantiateRebufferer(null), fp))
         {
-            final Iterator<ByteComparable> expected = byteComparables.iterator();
-            int offset = 0;
-            while (iterator.hasNext())
-            {
-                assertTrue(expected.hasNext());
-                final Pair<ByteComparable, Long> actual = iterator.next();
+            verifyOrder(iterator, byteComparables, true);
+            Collections.reverse(byteComparables);
+            verifyOrder(reverseIterator, byteComparables, false);
+        }
+    }
 
-                assertEquals(0, compare(expected.next(), actual.left, OSS50));
-                assertEquals(offset++, actual.right.longValue());
-            }
-            assertFalse(expected.hasNext());
+    private void verifyOrder(Iterator<Pair<ByteComparable, Long>> iterator, List<ByteComparable> byteComparables, boolean ascending)
+    {
+        var expected = byteComparables.iterator();
+        int offset = ascending ? 0 : byteComparables.size() - 1;
+        while (iterator.hasNext())
+        {
+            assertTrue(expected.hasNext()); // verify that hasNext is idempotent
+            final Pair<ByteComparable, Long> actual = iterator.next();
+            assertEquals(0, compare(expected.next(), actual.left, VERSION));
+            assertEquals(offset, actual.right.longValue());
+            offset += ascending ? 1 : -1;
         }
+        assertFalse(expected.hasNext());
     }
 
     @Test
-    public void testMinMaxTerm() throws IOException
+    public void testTermEnumWithEmulatedPrimaryKey() throws Exception
     {
-        final List<ByteComparable> byteComparables = generateSortedByteComparables();
+        testForDifferentByteComparableEncodings(this::doTestMinMaxTerm);
+    }
+
+    private void doTestMinMaxTerm(Function<String, ByteComparable> asByteComparable) throws IOException
+    {
+        final List<ByteComparable> byteComparables = generateSortedByteComparables(asByteComparable);
 
         long fp;
-        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier))
+        IndexComponents.ForWrite components = indexDescriptor.newPerIndexComponentsForWrite(indexContext);
+        try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(components))
         {
             for (int i = 0; i < byteComparables.size(); ++i)
             {
@@ -133,22 +420,22 @@ public void testMinMaxTerm() throws IOException
             fp = writer.complete(new MutableLong());
         }
 
-        try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier);
-             TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp))
+        try (FileHandle input = components.get(IndexComponentType.TERMS_DATA).createFileHandle();
+             TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp, VERSION))
         {
             final ByteComparable expectedMaxTerm = byteComparables.get(byteComparables.size() - 1);
             final ByteComparable actualMaxTerm = reader.getMaxTerm();
-            assertEquals(0, compare(expectedMaxTerm, actualMaxTerm, OSS50));
+            assertEquals(0, compare(expectedMaxTerm, actualMaxTerm, VERSION));
 
             final ByteComparable expectedMinTerm = byteComparables.get(0);
             final ByteComparable actualMinTerm = reader.getMinTerm();
-            assertEquals(0, compare(expectedMinTerm, actualMinTerm, OSS50));
+            assertEquals(0, compare(expectedMinTerm, actualMinTerm, VERSION));
         }
     }
 
-    private List<ByteComparable> generateSortedByteComparables()
+    private List<ByteComparable> generateSortedByteComparables(Function<String, ByteComparable> asByteComparable)
     {
-        final int numKeys = getRandom().nextIntBetween(16, 512);
+        final int numKeys = randomIntBetween(16, 512);
         final List<String> randomStrings = Stream.generate(() -> randomSimpleString(4, 48))
                                                  .limit(numKeys)
                                                  .sorted()
@@ -157,12 +444,71 @@ private List<ByteComparable> generateSortedByteComparables()
         // Get rid of any duplicates otherwise the tests will fail.
         return randomStrings.stream()
                             .filter(string -> Collections.frequency(randomStrings, string) == 1)
-                            .map(this::asByteComparable)
+                            .map(asByteComparable)
                             .collect(Collectors.toList());
     }
 
-    private ByteComparable asByteComparable(String s)
+    /**
+     * Used to generate ByteComparable objects that are used as keys in the TrieTermsDictionary.
+     * @param token
+     * @param partitionKey
+     * @param clustringColumn
+     * @return
+     */
+    private ByteComparable primaryKey(Function<String, ByteComparable> asByteComparable,
+                                      String token, String partitionKey, String clustringColumn)
     {
-        return ByteComparable.fixedLength(ByteBufferUtil.bytes(s));
+        assert token != null && partitionKey != null && clustringColumn != null;
+        return primaryKey(asByteComparable, token, partitionKey, clustringColumn, ByteSource.TERMINATOR);
+    }
+
+    private ByteComparable primaryKey(Function<String, ByteComparable> asByteComparable, String token, int terminator)
+    {
+        assert token != null;
+        return primaryKey(asByteComparable, token, null, null, terminator);
+    }
+
+    private ByteComparable primaryKey(Function<String, ByteComparable> asByteComparable,
+                                      String token, String partitionKey, int terminator)
+    {
+        assert token != null && partitionKey != null;
+        return primaryKey(asByteComparable, token, partitionKey, null, terminator);
+    }
+
+    private ByteComparable primaryKey(Function<String, ByteComparable> asByteComparable,
+                                      String token, String partitionKey, String clustringColumn, int terminator)
+    {
+        ByteComparable tokenByteComparable = asByteComparable.apply(token);
+        if (partitionKey == null)
+            return (v) -> ByteSource.withTerminator(terminator, tokenByteComparable.asComparableBytes(v));
+        ByteComparable partitionKeyByteComparable = asByteComparable.apply(partitionKey);
+        if (clustringColumn == null)
+            return (v) -> ByteSource.withTerminator(terminator,
+                                                    tokenByteComparable.asComparableBytes(v),
+                                                    partitionKeyByteComparable.asComparableBytes(v));
+        ByteComparable clusteringColumnByteComparable = asByteComparable.apply(clustringColumn);
+        return (v) -> ByteSource.withTerminator(terminator,
+                                                tokenByteComparable.asComparableBytes(v),
+                                                partitionKeyByteComparable.asComparableBytes(v),
+                                                clusteringColumnByteComparable.asComparableBytes(v));
+
+    }
+
+    /**
+     * There are multiple ways of encoding a ByteComparable object. This method tests two of those ways.
+     * Fixed length results in a ByteStream without a terminating 0 while ByteComparable.of adds the terminating
+     * 0. The primary nuance is whether a ByteComparable object ends up strictly as a prefix or as a lower/greater
+     * branch. In both cases, the result for floor and ceiling ought to provide the same results, though the code
+     * path will be slightly different.
+     */
+    private void testForDifferentByteComparableEncodings(ThrowingConsumer<Function<String, ByteComparable>> test) throws Exception
+    {
+        test.accept(s -> ByteComparable.preencoded(VERSION, ByteBufferUtil.bytes(s)));
+        test.accept(ByteComparable::of);
+    }
+
+    @FunctionalInterface
+    public interface ThrowingConsumer<T> {
+        void accept(T t) throws Exception;
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsIterator.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsIterator.java
deleted file mode 100644
index 7917658fa5fc..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsIterator.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.disk.v1.trie;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.cassandra.io.tries.ValueIterator;
-import org.apache.cassandra.io.util.Rebufferer;
-import org.apache.cassandra.io.util.SizedInts;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-
-import static org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader.NOT_FOUND;
-
-public class TrieTermsIterator extends ValueIterator<TrieTermsIterator> implements Iterator<Pair<ByteComparable, Long>>
-{
-    Pair<ByteComparable, Long> next = null;
-
-    public TrieTermsIterator(Rebufferer rebufferer, long root)
-    {
-        super(rebufferer, root, true);
-    }
-
-    @Override
-    public boolean hasNext()
-    {
-        if (next != null)
-            return true;
-
-        if (peekNode() == NOT_FOUND)
-            return false;
-
-        next = Pair.create(nextCollectedValue(), getCurrentPayload());
-
-        nextPayloadedNode();
-
-        return true;
-    }
-
-    @Override
-    public Pair<ByteComparable, Long> next()
-    {
-        if (!hasNext())
-            throw new NoSuchElementException();
-
-        Pair<ByteComparable, Long> result = next;
-        next = null;
-        return result;
-    }
-
-    private long getCurrentPayload()
-    {
-        return getPayload(buf, payloadPosition(), payloadFlags());
-    }
-
-    private long getPayload(ByteBuffer contents, int payloadPos, int bytes)
-    {
-        if (bytes == 0)
-        {
-            return NOT_FOUND;
-        }
-        return SizedInts.read(contents, payloadPos, bytes);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyTest.java
new file mode 100644
index 000000000000..9d1a419f9c77
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v2/RowAwarePrimaryKeyTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.nio.ByteBuffer;
+import java.util.function.Supplier;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class RowAwarePrimaryKeyTest extends SAITester
+{
+    @Test
+    public void testHashCodeForDeferredPrimaryKey()
+    {
+        var factory = Version.BA.onDiskFormat().newPrimaryKeyFactory(EMPTY_COMPARATOR);
+
+        // Test relies on this implementation detail
+        assertTrue(factory instanceof RowAwarePrimaryKeyFactory);
+
+        // Set up the primary key
+        Token token = new Murmur3Partitioner.LongToken(1);
+        DecoratedKey key = new BufferDecoratedKey(token, ByteBuffer.allocate(1));
+        Supplier<PrimaryKey> supplier = () -> factory.create(key, Clustering.EMPTY);
+        PrimaryKey primaryKey1 = factory.createDeferred(token, supplier);
+
+        // Verify the results
+        int hash1 = primaryKey1.hashCode();
+        // Equals triggers loading the primary key
+        assertEquals(primaryKey1, primaryKey1);
+        assertEquals(hash1, primaryKey1.hashCode());
+
+        // Do again with explicit loading
+        PrimaryKey primaryKey2 = factory.createDeferred(token, supplier);
+        int hash2 = primaryKey2.hashCode();
+        primaryKey2.loadDeferred();
+        assertEquals(hash2, primaryKey2.hashCode());
+    }
+
+    @Test
+    public void testHashCodeForLoadedPrimaryKey()
+    {
+        var factory = Version.BA.onDiskFormat().newPrimaryKeyFactory(EMPTY_COMPARATOR);
+
+        // Test relies on this implementation detail
+        assertTrue(factory instanceof RowAwarePrimaryKeyFactory);
+
+        // Set up the primary key
+        Token token1 = new Murmur3Partitioner.LongToken(1);
+        DecoratedKey key1 = new BufferDecoratedKey(token1, ByteBuffer.allocate(1));
+        PrimaryKey primaryKey1 = factory.create(key1, Clustering.EMPTY);
+
+        // Create equivalent PK
+        Token token2 = new Murmur3Partitioner.LongToken(1);
+        DecoratedKey key2 = new BufferDecoratedKey(token2, ByteBuffer.allocate(1));
+        PrimaryKey primaryKey2 = factory.create(key2, Clustering.EMPTY);
+
+        assertEquals(primaryKey1.hashCode(), primaryKey2.hashCode());
+    }
+
+    @Test
+    public void testHashCodeForDeferedPrimaryKeyWithClusteringColumns()
+    {
+        var comparator = new ClusteringComparator(Int32Type.instance);
+        var factory = Version.BA.onDiskFormat().newPrimaryKeyFactory(comparator);
+
+        // Test relies on this implementation detail
+        assertTrue(factory instanceof RowAwarePrimaryKeyFactory);
+
+        // Set up the primary key
+        Token token1 = new Murmur3Partitioner.LongToken(1);
+        DecoratedKey key1 = new BufferDecoratedKey(token1, ByteBuffer.allocate(1));
+        PrimaryKey primaryKey1 = factory.create(key1, Clustering.make(ByteBuffer.allocate(1)));
+
+        // Create equivalent PK
+        Token token2 = new Murmur3Partitioner.LongToken(1);
+        DecoratedKey key2 = new BufferDecoratedKey(token2, ByteBuffer.allocate(1));
+        PrimaryKey primaryKey2 = factory.create(key2, Clustering.make(ByteBuffer.allocate(1)));
+
+        assertEquals(primaryKey1.hashCode(), primaryKey2.hashCode());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v2/V2OnDiskOrdinalsMapTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v2/V2OnDiskOrdinalsMapTest.java
new file mode 100644
index 000000000000..e31447ca1f2d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v2/V2OnDiskOrdinalsMapTest.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.IntUnaryOperator;
+
+import com.google.common.collect.HashBiMap;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.github.jbellis.jvector.util.BitSet;
+import io.github.jbellis.jvector.vector.ArrayVectorFloat;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.disk.vector.ConcurrentVectorValues;
+import org.apache.cassandra.index.sai.disk.vector.RamAwareVectorValues;
+import org.apache.cassandra.index.sai.disk.vector.VectorPostings;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+
+public class V2OnDiskOrdinalsMapTest
+{
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    final private int dimension = 10;
+
+    @BeforeClass
+    public static void setup()
+    {
+        // otherwise "FileHandle fileHandle = builder.complete()" throws
+        DatabaseDescriptor.clientInitialization();
+    }
+
+    @Test
+    public void testMatchRangeBits() {
+        BitSet bits = new V2OnDiskOrdinalsMap.MatchRangeBits(1, 3);
+        assertFalse(bits.get(0));
+        assertTrue(bits.get(1));
+        assertTrue(bits.get(2));
+        assertTrue(bits.get(3));
+        assertFalse(bits.get(4));
+        assertEquals(3, bits.cardinality());
+
+        bits = new V2OnDiskOrdinalsMap.MatchRangeBits(1, 1);
+        assertFalse(bits.get(0));
+        assertTrue(bits.get(1));
+        assertFalse(bits.get(2));
+        assertEquals(1, bits.cardinality());
+
+        bits = new V2OnDiskOrdinalsMap.MatchRangeBits(3, 1);
+        assertFalse(bits.get(0));
+        assertFalse(bits.get(1));
+        assertFalse(bits.get(2));
+        assertFalse(bits.get(3));
+        assertFalse(bits.get(4));
+        assertEquals(0, bits.cardinality());
+    }
+
+    @Test
+    public void testRowIdsMatchOrdinalsSet() throws Exception {
+        boolean ordinalsMatchRowIds = createOdomAndGetRowIdsMatchOrdinals(HashBiMap.create());
+        assertTrue(ordinalsMatchRowIds);
+    }
+
+    @Test
+    public void testRowIdsMatchOrdinalsNotSet() throws Exception {
+        boolean ordinalsMatchRowIds = createOdomAndGetRowIdsMatchOrdinals(null);
+        assertFalse(ordinalsMatchRowIds);
+    }
+
+    @Test
+    public void testForEachRowidMatching() throws Exception
+    {
+        testForEach(HashBiMap.create());
+    }
+
+    @Test
+    public void testForEachFileReading() throws Exception
+    {
+        testForEach(null);
+    }
+
+    // This test covers a legacy case that is no longer reachable in the current codebase.
+    @Test
+    public void testGraphThatHasOrdinalsButNoMatchingRows() throws Exception
+    {
+        File tempFile = temp("testfile");
+
+        var deletedOrdinals = new HashSet<Integer>();
+        // Delete the only ordinal
+        deletedOrdinals.add(0);
+        RamAwareVectorValues vectorValues = generateVectors(1);
+
+        var postingsMap = generatePostingsMap(vectorValues);
+
+        for (var p: postingsMap.entrySet()) {
+            // Remove all row ids
+            p.getValue().computeRowIds(x -> -1);
+        }
+
+        PostingsMetadata postingsMd = writePostings(null, tempFile, vectorValues, postingsMap, deletedOrdinals);
+        try (FileHandle fileHandle = new FileHandle.Builder(tempFile).complete())
+        {
+            var odom = new V2OnDiskOrdinalsMap(fileHandle, postingsMd.postingsOffset, postingsMd.postingsLength);
+
+            try (var ordinalsView = odom.getOrdinalsView();
+                 var rowIdsView = odom.getRowIdsView())
+            {
+                assertEquals(-1, ordinalsView.getOrdinalForRowId(0));
+                final AtomicInteger count = new AtomicInteger(0);
+                ordinalsView.forEachOrdinalInRange(-100, Integer.MAX_VALUE / 2, (rowId, ordinal) -> {
+                    count.incrementAndGet();
+                });
+                assertEquals(0, count.get());
+                assertNull(ordinalsView.buildOrdinalBits(0, 5, () -> null));
+
+                assertFalse(rowIdsView.getSegmentRowIdsMatching(0).hasNext());
+            }
+
+            odom.close();
+        }
+    }
+
+    private void testForEach(HashBiMap<Integer, Integer> ordinalsMap) throws Exception
+    {
+        File tempFile = temp("testfile");
+
+        var deletedOrdinals = new HashSet<Integer>();
+        RamAwareVectorValues vectorValues = generateVectors(10);
+
+        var postingsMap = generatePostingsMap(vectorValues);
+
+        for (var p: postingsMap.entrySet()) {
+            p.getValue().computeRowIds(x -> x);
+        }
+
+        PostingsMetadata postingsMd = writePostings(ordinalsMap, tempFile, vectorValues, postingsMap, deletedOrdinals);
+
+        FileHandle.Builder builder = new FileHandle.Builder(tempFile);
+        try (FileHandle fileHandle = builder.complete())
+        {
+            V2OnDiskOrdinalsMap odom = new V2OnDiskOrdinalsMap(fileHandle, postingsMd.postingsOffset, postingsMd.postingsLength);
+
+            try (var ordinalsView = odom.getOrdinalsView())
+            {
+                final AtomicInteger count = new AtomicInteger(0);
+                ordinalsView.forEachOrdinalInRange(-100, Integer.MAX_VALUE / 2, (rowId, ordinal) -> {
+                    assertTrue(ordinal >= 0);
+                    assertTrue(ordinal < vectorValues.size());
+                    count.incrementAndGet();
+                });
+                assertEquals(vectorValues.size(), count.get());
+            }
+
+            odom.close();
+        }
+    }
+
+    private boolean createOdomAndGetRowIdsMatchOrdinals(HashBiMap<Integer, Integer> ordinalsMap) throws Exception
+    {
+        File tempFile = temp("testfile");
+
+        var deletedOrdinals = new HashSet<Integer>();
+        RamAwareVectorValues vectorValues = generateVectors(10);
+
+        final boolean canFastFindRows = ordinalsMap != null;
+        var postingsMap = generatePostingsMap(vectorValues);
+
+        // skip rows 5 and 6 if !canFastFindRows
+        for (var p: postingsMap.entrySet())
+        {
+            p.getValue().computeRowIds(x -> canFastFindRows ? x : (x == 5 || x == 6 ? -1 : x));
+        }
+
+        PostingsMetadata postingsMd = writePostings(ordinalsMap, tempFile, vectorValues, postingsMap, deletedOrdinals);
+
+        FileHandle.Builder builder = new FileHandle.Builder(tempFile);
+        try (FileHandle fileHandle = builder.complete())
+        {
+            V2OnDiskOrdinalsMap odom = new V2OnDiskOrdinalsMap(fileHandle, postingsMd.postingsOffset, postingsMd.postingsLength);
+
+            try (var ordinalsView = odom.getOrdinalsView())
+            {
+                int lastRowId = Integer.MAX_VALUE;
+                for (var p: postingsMap.entrySet())
+                {
+                    for (int rowId: p.getValue().getRowIds())
+                    {
+                        if (rowId - 1 > lastRowId)
+                        {
+                            try
+                            {
+                                ordinalsView.getOrdinalForRowId(lastRowId);
+                                fail("expected IllegalArgumentException when trying to repeat row");
+                            }
+                            catch (IllegalArgumentException e)
+                            {
+                                // expected
+                            }
+
+                            for (int r = lastRowId + 1; r < rowId; r++)
+                            {
+                                // check skipped rows
+                                int ordinal = ordinalsView.getOrdinalForRowId(r);
+                                assertEquals(-1, ordinal);
+                            }
+                        }
+
+                        int ordinal = ordinalsView.getOrdinalForRowId(rowId);
+                        assertEquals(rowId, ordinal);
+                        lastRowId = rowId;
+                    }
+                }
+                int ordinal = ordinalsView.getOrdinalForRowId(Integer.MAX_VALUE);
+                assertEquals(-1, ordinal);
+            }
+
+            boolean canFastMapRowIdsView = (boolean) FieldUtils.readField(odom, "canFastMapRowIdsView", true);
+            boolean canFastMapOrdinalsView = (boolean) FieldUtils.readField(odom, "canFastMapOrdinalsView", true);
+            odom.close();
+            assertEquals(canFastMapRowIdsView, canFastMapOrdinalsView);
+            return canFastMapOrdinalsView;
+        }
+    }
+
+    private static PostingsMetadata writePostings(HashBiMap<Integer, Integer> ordinalsMap,
+                                                  File tempFile,
+                                                  RamAwareVectorValues vectorValues,
+                                                  Map<VectorFloat<?>, VectorPostings<Integer>> postingsMap,
+                                                  HashSet<Integer> deletedOrdinals) throws IOException
+    {
+        SequentialWriter writer = new SequentialWriter(tempFile,
+                                                       SequentialWriterOption.newBuilder().finishOnClose(true).build());
+
+        IntUnaryOperator reverseOrdinalsMapper = ordinalsMap == null
+                                                           ? x -> x
+                                                           : x -> ordinalsMap.inverse().getOrDefault(x, x);
+
+        long postingsOffset = writer.position();
+        long postingsPosition = new V2VectorPostingsWriter<Integer>(ordinalsMap != null, postingsMap.size(), reverseOrdinalsMapper)
+                                    .writePostings(writer, vectorValues, postingsMap, deletedOrdinals);
+        long postingsLength = postingsPosition - postingsOffset;
+
+        writer.close();
+        return new PostingsMetadata(postingsOffset, postingsLength);
+    }
+
+    private static class PostingsMetadata
+    {
+        public final long postingsOffset;
+        public final long postingsLength;
+
+        public PostingsMetadata(long postingsOffset, long postingsLength)
+        {
+            this.postingsOffset = postingsOffset;
+            this.postingsLength = postingsLength;
+        }
+    }
+
+    private Map<VectorFloat<?>, VectorPostings<Integer>> generatePostingsMap(RamAwareVectorValues vectorValues)
+    {
+        Map<VectorFloat<?>, VectorPostings<Integer>> postingsMap = new ConcurrentSkipListMap<>((a, b) -> {
+            return Arrays.compare(((ArrayVectorFloat) a).get(), ((ArrayVectorFloat) b).get());
+        });
+
+        for (int ordinal = 0; ordinal < vectorValues.size(); ordinal++)
+        {
+            var vector = vectorValues.getVector(ordinal);
+            var vp = new VectorPostings<>(ordinal);
+            vp.setOrdinal(ordinal);
+            postingsMap.put(vector, vp);
+        }
+
+        return postingsMap;
+    }
+
+    private RamAwareVectorValues generateVectors(int totalOrdinals)
+    {
+        var vectorValues = new ConcurrentVectorValues(dimension);
+        for (int i = 0; i < totalOrdinals; i++)
+        {
+            float[] rawVector = new float[dimension];
+            Arrays.fill(rawVector, (float) i);
+            vectorValues.add(i, vts.createFloatVector(rawVector));
+        }
+        return vectorValues;
+    }
+
+    private static File temp(String id)
+    {
+        File file = FileUtils.createTempFile(id, "tmp");
+        file.deleteOnExit();
+        return file;
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsTest.java
new file mode 100644
index 000000000000..53a7a0d3fa31
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v2/sortedterms/SortedTermsTest.java
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2.sortedterms;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.v1.MetadataSource;
+import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.SAICodecUtils;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.apache.lucene.store.IndexInput;
+
+public class SortedTermsTest extends SaiRandomizedTest
+{
+
+    public static final ByteComparable.Version VERSION = TypeUtil.BYTE_COMPARABLE_VERSION;
+
+    @Test
+    public void testLexicographicException() throws Exception
+    {
+        IndexDescriptor indexDescriptor = newIndexDescriptor();
+        IndexComponents.ForWrite components = indexDescriptor.newPerSSTableComponentsForWrite();
+        try (MetadataWriter metadataWriter = new MetadataWriter(components))
+        {
+            NumericValuesWriter blockFPWriter = new NumericValuesWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS),
+                                                                        metadataWriter, true);
+            try (SortedTermsWriter writer = new SortedTermsWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCKS),
+                                                                  metadataWriter,
+                                                                  blockFPWriter,
+                                                                  components.addOrGet(IndexComponentType.PRIMARY_KEY_TRIE)))
+            {
+                ByteBuffer buffer = Int32Type.instance.decompose(99999);
+                ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, VERSION);
+                byte[] bytes1 = ByteSourceInverse.readBytes(byteSource);
+
+                writer.add(ByteComparable.preencoded(VERSION, bytes1));
+
+                buffer = Int32Type.instance.decompose(444);
+                byteSource = Int32Type.instance.asComparableBytes(buffer, VERSION);
+                byte[] bytes2 = ByteSourceInverse.readBytes(byteSource);
+
+                assertThrows(IllegalArgumentException.class, () -> writer.add(ByteComparable.preencoded(VERSION, bytes2)));
+            }
+        }
+    }
+
+    @Test
+    public void testFileValidation() throws Exception
+    {
+        IndexDescriptor indexDescriptor = newIndexDescriptor();
+
+        List<PrimaryKey> primaryKeys = new ArrayList<>();
+
+        for (int x = 0; x < 11; x++)
+        {
+            ByteBuffer buffer = UTF8Type.instance.decompose(Integer.toString(x));
+            DecoratedKey partitionKey = Murmur3Partitioner.instance.decorateKey(buffer);
+            PrimaryKey primaryKey = SAITester.TEST_FACTORY.create(partitionKey, Clustering.EMPTY);
+            primaryKeys.add(primaryKey);
+        }
+
+        primaryKeys.sort(PrimaryKey::compareTo);
+
+        IndexComponents.ForWrite components = indexDescriptor.newPerSSTableComponentsForWrite();
+        try (MetadataWriter metadataWriter = new MetadataWriter(components))
+        {
+            NumericValuesWriter blockFPWriter = new NumericValuesWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS),
+                                                                        metadataWriter, true);
+            try (SortedTermsWriter writer = new SortedTermsWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCKS),
+                                                                  metadataWriter,
+                                                                  blockFPWriter,
+                                                                  components.addOrGet(IndexComponentType.PRIMARY_KEY_TRIE)))
+            {
+                primaryKeys.forEach(primaryKey -> {
+                    try
+                    {
+                        writer.add(v -> primaryKey.asComparableBytes(v));
+                    }
+                    catch (IOException e)
+                    {
+                        e.printStackTrace();
+                    }
+                });
+            }
+        }
+        assertTrue(validateComponent(components, IndexComponentType.PRIMARY_KEY_TRIE, true));
+        assertTrue(validateComponent(components, IndexComponentType.PRIMARY_KEY_TRIE, false));
+        assertTrue(validateComponent(components, IndexComponentType.PRIMARY_KEY_BLOCKS, true));
+        assertTrue(validateComponent(components, IndexComponentType.PRIMARY_KEY_BLOCKS, false));
+        assertTrue(validateComponent(components, IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS, true));
+        assertTrue(validateComponent(components, IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS, false));
+    }
+
+    @Test
+    public void testSeekToTerm() throws Exception
+    {
+        IndexDescriptor descriptor = newIndexDescriptor();
+
+        List<byte[]> terms = new ArrayList<>();
+        writeTerms(descriptor, terms);
+
+        // iterate on terms ascending
+        withSortedTermsReader(descriptor, reader ->
+        {
+            for (int x = 0; x < terms.size(); x++)
+            {
+                try (SortedTermsReader.Cursor cursor = reader.openCursor())
+                {
+                    long pointId = cursor.ceiling(ByteComparable.preencoded(VERSION, terms.get(x)));
+                    assertEquals(x, pointId);
+                }
+            }
+        });
+
+        // iterate on terms descending
+        withSortedTermsReader(descriptor, reader ->
+        {
+            for (int x = terms.size() - 1; x >= 0; x--)
+            {
+                try (SortedTermsReader.Cursor cursor = reader.openCursor())
+                {
+                    long pointId = cursor.ceiling(ByteComparable.preencoded(VERSION, terms.get(x)));
+                    assertEquals(x, pointId);
+                }
+            }
+        });
+
+        // iterate randomly
+        withSortedTermsReader(descriptor, reader ->
+        {
+            for (int x = 0; x < terms.size(); x++)
+            {
+                int target = nextInt(0, terms.size());
+
+                try (SortedTermsReader.Cursor cursor = reader.openCursor())
+                {
+                    long pointId = cursor.ceiling(ByteComparable.preencoded(VERSION, terms.get(target)));
+                    assertEquals(target, pointId);
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testSeekToTermMinMaxPrefixNoMatch() throws Exception
+    {
+        IndexDescriptor descriptor = newIndexDescriptor();
+
+        List<ByteSource> termsMinPrefixNoMatch = new ArrayList<>();
+        List<ByteSource> termsMaxPrefixNoMatch = new ArrayList<>();
+        int valuesPerPrefix = 10;
+        writeTerms(descriptor, termsMinPrefixNoMatch, termsMaxPrefixNoMatch, valuesPerPrefix, false);
+
+        var countEndOfData = new AtomicInteger();
+        // iterate on terms ascending
+        withSortedTermsReader(descriptor, reader ->
+        {
+            for (int x = 0; x < termsMaxPrefixNoMatch.size(); x++)
+            {
+                try (SortedTermsReader.Cursor cursor = reader.openCursor())
+                {
+                    int index = x;
+                    long pointIdEnd = cursor.ceiling(v -> termsMinPrefixNoMatch.get(index));
+                    long pointIdStart = cursor.floor(v -> termsMaxPrefixNoMatch.get(index));
+                    if (pointIdStart >= 0 && pointIdEnd >= 0)
+                        assertTrue(pointIdEnd > pointIdStart);
+                    else
+                        countEndOfData.incrementAndGet();
+                }
+            }
+        });
+        // ceiling reaches the end of the data because we call writeTerms with matchesData false, which means that
+        // the last set of terms we are calling ceiling on are greater than anything in the trie, so ceiling returns
+        // a negative value.
+        assertEquals(valuesPerPrefix, countEndOfData.get());
+    }
+
+    @Test
+    public void testSeekToTermMinMaxPrefix() throws Exception
+    {
+        IndexDescriptor descriptor = newIndexDescriptor();
+
+        List<ByteSource> termsMinPrefix = new ArrayList<>();
+        List<ByteSource> termsMaxPrefix = new ArrayList<>();
+        int valuesPerPrefix = 10;
+        writeTerms(descriptor, termsMinPrefix, termsMaxPrefix, valuesPerPrefix, true);
+
+        // iterate on terms ascending
+        withSortedTermsReader(descriptor, reader ->
+        {
+            for (int x = 0; x < termsMaxPrefix.size(); x++)
+            {
+                try (SortedTermsReader.Cursor cursor = reader.openCursor())
+                {
+                    int index = x;
+                    long pointIdEnd = cursor.ceiling(v -> termsMinPrefix.get(index));
+                    long pointIdStart = cursor.floor(v -> termsMaxPrefix.get(index));
+                    assertEquals(pointIdEnd, x / valuesPerPrefix * valuesPerPrefix);
+                    assertEquals(pointIdEnd + valuesPerPrefix - 1, pointIdStart);
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testAdvance() throws IOException
+    {
+        IndexDescriptor descriptor = newIndexDescriptor();
+
+        List<byte[]> terms = new ArrayList<>();
+        writeTerms(descriptor, terms);
+
+        withSortedTermsCursor(descriptor, cursor ->
+        {
+            int x = 0;
+            while (cursor.advance())
+            {
+                ByteComparable term = cursor.term();
+
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(VERSION));
+                assertArrayEquals(terms.get(x), bytes);
+
+                x++;
+            }
+
+            // assert we don't increase the point id beyond one point after the last item
+            assertEquals(cursor.pointId(), terms.size());
+            assertFalse(cursor.advance());
+            assertEquals(cursor.pointId(), terms.size());
+        });
+    }
+
+    @Test
+    public void testReset() throws Exception
+    {
+        IndexDescriptor descriptor = newIndexDescriptor();
+
+        List<byte[]> terms = new ArrayList<>();
+        writeTerms(descriptor, terms);
+
+        withSortedTermsCursor(descriptor, cursor ->
+        {
+            assertTrue(cursor.advance());
+            assertTrue(cursor.advance());
+            String term1 = cursor.term().byteComparableAsString(VERSION);
+            cursor.reset();
+            assertTrue(cursor.advance());
+            assertTrue(cursor.advance());
+            String term2 = cursor.term().byteComparableAsString(VERSION);
+            assertEquals(term1, term2);
+            assertEquals(1, cursor.pointId());
+        });
+    }
+
+    @Test
+    public void testSeekToPointId() throws Exception
+    {
+        IndexDescriptor descriptor = newIndexDescriptor();
+
+        List<byte[]> terms = new ArrayList<>();
+        writeTerms(descriptor, terms);
+
+        // iterate ascending
+        withSortedTermsCursor(descriptor, cursor ->
+        {
+            for (int x = 0; x < terms.size(); x++)
+            {
+                cursor.seekToPointId(x);
+                ByteComparable term = cursor.term();
+
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(VERSION));
+                assertArrayEquals(terms.get(x), bytes);
+            }
+        });
+
+        // iterate descending
+        withSortedTermsCursor(descriptor, cursor ->
+        {
+            for (int x = terms.size() - 1; x >= 0; x--)
+            {
+                cursor.seekToPointId(x);
+                ByteComparable term = cursor.term();
+
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(VERSION));
+                assertArrayEquals(terms.get(x), bytes);
+            }
+        });
+
+        // iterate randomly
+        withSortedTermsCursor(descriptor, cursor ->
+        {
+            for (int x = 0; x < terms.size(); x++)
+            {
+                int target = nextInt(0, terms.size());
+                cursor.seekToPointId(target);
+                ByteComparable term = cursor.term();
+
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(VERSION));
+                assertArrayEquals(terms.get(target), bytes);
+            }
+        });
+    }
+
+    @Test
+    public void testSeekToPointIdOutOfRange() throws Exception
+    {
+        IndexDescriptor descriptor = newIndexDescriptor();
+
+        List<byte[]> terms = new ArrayList<>();
+        writeTerms(descriptor, terms);
+
+        withSortedTermsCursor(descriptor, cursor ->
+        {
+            assertThrows(IndexOutOfBoundsException.class, () -> cursor.seekToPointId(-2));
+            assertThrows(IndexOutOfBoundsException.class, () -> cursor.seekToPointId(Long.MAX_VALUE));
+        });
+    }
+
+    private void writeTerms(IndexDescriptor indexDescriptor, List<byte[]> terms) throws IOException
+    {
+        IndexComponents.ForWrite components = indexDescriptor.newPerSSTableComponentsForWrite();
+        try (MetadataWriter metadataWriter = new MetadataWriter(components))
+        {
+            NumericValuesWriter blockFPWriter = new NumericValuesWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS),
+                                                                        metadataWriter, true);
+            try (SortedTermsWriter writer = new SortedTermsWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCKS),
+                                                                  metadataWriter,
+                                                                  blockFPWriter,
+                                                                  components.addOrGet(IndexComponentType.PRIMARY_KEY_TRIE)))
+            {
+                for (int x = 0; x < 1000 * 4; x++)
+                {
+                    ByteBuffer buffer = Int32Type.instance.decompose(x);
+                    ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, VERSION);
+                    byte[] bytes = ByteSourceInverse.readBytes(byteSource);
+                    terms.add(bytes);
+
+                    writer.add(ByteComparable.preencoded(VERSION, bytes));
+                }
+            }
+        }
+        components.markComplete();
+    }
+
+    private void writeTerms(IndexDescriptor indexDescriptor, List<ByteSource> termsMinPrefix, List<ByteSource> termsMaxPrefix, int numPerPrefix, boolean matchesData) throws IOException
+    {
+        IndexComponents.ForWrite components = indexDescriptor.newPerSSTableComponentsForWrite();
+        try (MetadataWriter metadataWriter = new MetadataWriter(components))
+        {
+            NumericValuesWriter blockFPWriter = new NumericValuesWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS),
+                                                                        metadataWriter, true);
+            try (SortedTermsWriter writer = new SortedTermsWriter(components.addOrGet(IndexComponentType.PRIMARY_KEY_BLOCKS),
+                                                                  metadataWriter,
+                                                                  blockFPWriter,
+                                                                  components.addOrGet(IndexComponentType.PRIMARY_KEY_TRIE)))
+            {
+                for (int x = 0; x < 1000 ; x++)
+                {
+                    int component1 = x * 2;
+                    for (int i = 0; i < numPerPrefix; i++)
+                    {
+                        String component2 = "v" + i;
+                        termsMinPrefix.add(ByteSource.withTerminator(ByteSource.LT_NEXT_COMPONENT, intByteSource(component1 + (matchesData ? 0 : 1))));
+                        termsMaxPrefix.add(ByteSource.withTerminator(ByteSource.GT_NEXT_COMPONENT, intByteSource(component1 + (matchesData ? 0 : 1))));
+                        writer.add(v -> ByteSource.withTerminator(ByteSource.TERMINATOR, intByteSource(component1), utfByteSource(component2)));
+                    }
+                }
+            }
+        }
+        components.markComplete();
+    }
+
+    private ByteSource intByteSource(int value)
+    {
+        ByteBuffer buffer = Int32Type.instance.decompose(value);
+        return Int32Type.instance.asComparableBytes(buffer, VERSION);
+    }
+
+    private ByteSource utfByteSource(String value)
+    {
+        ByteBuffer buffer = UTF8Type.instance.decompose(value);
+        return UTF8Type.instance.asComparableBytes(buffer, VERSION);
+    }
+
+    @FunctionalInterface
+    public interface ThrowingConsumer<T> {
+        void accept(T t) throws IOException;
+    }
+
+    private void withSortedTermsCursor(IndexDescriptor indexDescriptor,
+                                       ThrowingConsumer<SortedTermsReader.Cursor> testCode) throws IOException
+    {
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        MetadataSource metadataSource = MetadataSource.loadMetadata(components);
+        IndexComponent.ForRead blocksComponent = components.get(IndexComponentType.PRIMARY_KEY_BLOCKS);
+        IndexComponent.ForRead blockOffsetsComponent = components.get(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS);
+        NumericValuesMeta blockPointersMeta = new NumericValuesMeta(metadataSource.get(blockOffsetsComponent.fileNamePart()));
+        SortedTermsMeta sortedTermsMeta = new SortedTermsMeta(metadataSource.get(blocksComponent.fileNamePart()));
+        try (FileHandle trieHandle = components.get(IndexComponentType.PRIMARY_KEY_TRIE).createFileHandle();
+             FileHandle termsData = blocksComponent.createFileHandle();
+             FileHandle blockOffsets = blockOffsetsComponent.createFileHandle())
+        {
+            SortedTermsReader reader = new SortedTermsReader(termsData, blockOffsets, trieHandle, sortedTermsMeta, blockPointersMeta);
+            try (SortedTermsReader.Cursor cursor = reader.openCursor())
+            {
+                testCode.accept(cursor);
+            }
+        }
+    }
+
+    private void withSortedTermsReader(IndexDescriptor indexDescriptor,
+                                       ThrowingConsumer<SortedTermsReader> testCode) throws IOException
+    {
+        IndexComponents.ForRead components = indexDescriptor.perSSTableComponents();
+        MetadataSource metadataSource = MetadataSource.loadMetadata(components);
+        IndexComponent.ForRead blocksComponent = components.get(IndexComponentType.PRIMARY_KEY_BLOCKS);
+        IndexComponent.ForRead blockOffsetsComponent = components.get(IndexComponentType.PRIMARY_KEY_BLOCK_OFFSETS);
+        NumericValuesMeta blockPointersMeta = new NumericValuesMeta(metadataSource.get(blockOffsetsComponent.fileNamePart()));
+        SortedTermsMeta sortedTermsMeta = new SortedTermsMeta(metadataSource.get(blocksComponent.fileNamePart()));
+        try (FileHandle trieHandle = components.get(IndexComponentType.PRIMARY_KEY_TRIE).createFileHandle();
+             FileHandle termsData = blocksComponent.createFileHandle();
+             FileHandle blockOffsets = blockOffsetsComponent.createFileHandle())
+        {
+            SortedTermsReader reader = new SortedTermsReader(termsData, blockOffsets, trieHandle, sortedTermsMeta, blockPointersMeta);
+            testCode.accept(reader);
+        }
+    }
+
+    private boolean validateComponent(IndexComponents.ForRead components, IndexComponentType indexComponentType, boolean checksum)
+    {
+        try (IndexInput input = components.get(indexComponentType).openInput())
+        {
+            if (checksum)
+                SAICodecUtils.validateChecksum(input, components.version());
+            else
+                SAICodecUtils.validate(input);
+            return true;
+        }
+        catch (Throwable e)
+        {
+        }
+        return false;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v5/V5OnDiskOrdinalsMapTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v5/V5OnDiskOrdinalsMapTest.java
new file mode 100644
index 000000000000..093a624da877
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v5/V5OnDiskOrdinalsMapTest.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v5;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.function.IntUnaryOperator;
+import java.util.stream.StreamSupport;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.util.SparseBits;
+import io.github.jbellis.jvector.vector.ArrayVectorFloat;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.cql.VectorTester;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v5.V5VectorPostingsWriter.Structure;
+import org.apache.cassandra.index.sai.disk.vector.ConcurrentVectorValues;
+import org.apache.cassandra.index.sai.disk.vector.RamAwareVectorValues;
+import org.apache.cassandra.index.sai.disk.vector.VectorPostings;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+
+public class V5OnDiskOrdinalsMapTest extends VectorTester
+{
+    @Before
+    public void setup() throws Throwable
+    {
+        super.setup();
+        // this can be removed once LATEST is >= DC
+        SAIUtil.setLatestVersion(Version.DC);
+    }
+
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    private static final int DIMENSION = 2; // not relevant to postings mapping, nothing gained by using something larger
+
+    @Test
+    public void testRandomPostings() throws Exception
+    {
+        for (int i = 0; i < 10; i++) {
+            for (Structure structure: Structure.values()) {
+                testRandomPostingsOnce(structure);
+            }
+        }
+    }
+
+    @Test
+    public void testOneToOne() throws Exception
+    {
+        var vv = generateVectors(4);
+        Map<VectorFloat<?>, VectorPostings<String>> postingsMap = emptyPostingsMap();
+        // 0 -> C
+        // 1 -> A
+        // 2 -> B
+        postingsMap.put(vv.getVector(0), createPostings(0, "C"));
+        postingsMap.put(vv.getVector(1), createPostings(1, "A"));
+        postingsMap.put(vv.getVector(2), createPostings(2, "B"));
+        computeRowIds(postingsMap);
+        validate(Structure.ONE_TO_ONE, postingsMap, vv);
+    }
+
+    @Test
+    public void testOneToMany() throws Exception
+    {
+        var vv = generateVectors(4);
+        Map<VectorFloat<?>, VectorPostings<String>> postingsMap = emptyPostingsMap();
+        // 0 -> B
+        // 1 -> C, A
+        // 2 -> D
+        postingsMap.put(vv.getVector(0), createPostings(0, "B"));
+        postingsMap.put(vv.getVector(1), createPostings(1, "C", "A"));
+        postingsMap.put(vv.getVector(2), createPostings(2, "D"));
+        computeRowIds(postingsMap);
+        validate(Structure.ONE_TO_MANY, postingsMap, vv);
+    }
+
+    @Test
+    public void testOneToManyLarger() throws Exception
+    {
+        var vv = generateVectors(4);
+        Map<VectorFloat<?>, VectorPostings<String>> postingsMap = emptyPostingsMap();
+        // 0 -> B
+        // 1 -> C, A, E
+        // 2 -> D, F
+        // 3 -> G
+        postingsMap.put(vv.getVector(0), createPostings(0, "B"));
+        postingsMap.put(vv.getVector(1), createPostings(1, "C", "A", "E"));
+        postingsMap.put(vv.getVector(2), createPostings(2, "D", "F"));
+        postingsMap.put(vv.getVector(3), createPostings(3, "G"));
+        computeRowIds(postingsMap);
+        validate(Structure.ONE_TO_MANY, postingsMap, vv);
+    }
+
+    @Test
+    public void testGeneric() throws Exception
+    {
+        var vv = generateVectors(4);
+        Map<VectorFloat<?>, VectorPostings<String>> postingsMap = emptyPostingsMap();
+        // 0 -> B
+        // 1 -> C, A
+        // 2 -> E    (no D, this makes it not 1:many)
+        postingsMap.put(vv.getVector(0), createPostings(0, "B"));
+        postingsMap.put(vv.getVector(1), createPostings(1, "C", "A"));
+        postingsMap.put(vv.getVector(2), createPostings(2, "E"));
+        computeRowIds(postingsMap);
+        validate(Structure.ZERO_OR_ONE_TO_MANY, postingsMap, vv);
+    }
+
+    @Test
+    public void testEmpty() throws Exception
+    {
+        var vv = generateVectors(0);
+        Map<VectorFloat<?>, VectorPostings<String>> postingsMap = emptyPostingsMap();
+        validate(Structure.ONE_TO_ONE, postingsMap, vv);
+    }
+
+    private static void computeRowIds(Map<VectorFloat<?>, VectorPostings<String>> postingsMap)
+    {
+        for (var p: postingsMap.entrySet())
+        {
+            p.getValue().computeRowIds(s -> {
+                assert s.length() == 1;
+                return s.charAt(0) - 'A';
+            });
+        }
+    }
+
+    private VectorPostings<String> createPostings(int ordinal, String... postings)
+    {
+        var vp = new VectorPostings<>(List.of(postings));
+        vp.setOrdinal(ordinal);
+        return vp;
+    }
+
+    private void testRandomPostingsOnce(Structure structure) throws Exception
+    {
+        var vectorValues = generateVectors(100 + randomInt(1000));
+        var postingsMap = generatePostingsMap(vectorValues, structure);
+        // call computeRowIds because that's how VectorPostings is designed.
+        // since we're natively using ints as our rowids (not PrimaryKey objects) this can be the identity
+        for (var p: postingsMap.entrySet())
+            p.getValue().computeRowIds(x -> x);
+
+        validate(structure, postingsMap, vectorValues);
+    }
+
+    private static int randomInt(int maxValueInclusive)
+    {
+        return getRandom().nextIntBetween(0, maxValueInclusive);
+    }
+
+    private <T> void validate(Structure structure, Map<VectorFloat<?>, VectorPostings<T>> postingsMap, RamAwareVectorValues vectorValues) throws IOException
+    {
+        // build the remapping and write the postings
+        var remapped = V5VectorPostingsWriter.remapForMemtable(postingsMap);
+        assert remapped.structure == structure : remapped.structure + " != " + structure;
+        File tempFile = createTempFile("testfile");
+        PostingsMetadata postingsMd = writePostings(tempFile, vectorValues, postingsMap, remapped);
+
+        // test V5OnDiskOrdinalsMap
+        FileHandle.Builder builder = new FileHandle.Builder(tempFile).withCompressionMetadata(null);
+        try (FileHandle fileHandle = builder.complete())
+        {
+            var odom = new V5OnDiskOrdinalsMap(fileHandle, postingsMd.postingsOffset, postingsMd.postingsLength);
+            assertEquals(structure, odom.structure);
+
+            // check row -> ordinal and ordinal -> rows
+            try (var rowIdsView = odom.getRowIdsView();
+                 var ordinalsView = odom.getOrdinalsView())
+            {
+                for (var vp : postingsMap.values())
+                {
+                    var rowIds = vp.getRowIds().toIntArray();
+                    Arrays.sort(rowIds);
+
+                    var newOrdinal = remapped.ordinalMapper.oldToNew(vp.getOrdinal());
+                    var it = rowIdsView.getSegmentRowIdsMatching(newOrdinal);
+                    int[] a = StreamSupport.intStream(Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), false).toArray();
+                    Arrays.sort(a);
+
+                    assertArrayEquals(rowIds, a);
+
+                    for (int i = 0; i < rowIds.length; i++)
+                    {
+                        int ordinal = ordinalsView.getOrdinalForRowId(rowIds[i]);
+                        assertEquals(newOrdinal, ordinal);
+                    }
+                }
+            }
+
+            // test buildOrdinalBits
+            try (var ordinalsView = odom.getOrdinalsView())
+            {
+                for (int i = 0; i < 10; i++)
+                {
+                    // neither min nor max rowId has to actually exist in the postings
+                    int minRowId = randomInt(vectorValues.size());
+                    int maxRowId = minRowId + randomInt(vectorValues.size());
+                    var bits = ordinalsView.buildOrdinalBits(minRowId, maxRowId, SparseBits::new);
+
+                    var expectedBits = new SparseBits();
+                    for (var vp : postingsMap.values())
+                    {
+                        for (int rowId : vp.getRowIds())
+                        {
+                            if (rowId >= minRowId && rowId <= maxRowId)
+                                expectedBits.set(remapped.ordinalMapper.oldToNew(vp.getOrdinal()));
+                        }
+                    }
+
+                    assertEqualsBits(expectedBits, bits, remapped.maxRowId);
+                }
+            }
+
+            odom.close();
+        }
+    }
+
+    private void assertEqualsBits(Bits b1, Bits b2, int maxBit)
+    {
+        for (int i = 0; i <= maxBit; i++)
+            assertEquals(b1.get(i), b2.get(i));
+    }
+
+    private static <T> PostingsMetadata writePostings(File tempFile,
+                                                  RandomAccessVectorValues vectorValues,
+                                                  Map<VectorFloat<?>, VectorPostings<T>> postingsMap, V5VectorPostingsWriter.RemappedPostings remapped) throws IOException
+    {
+        SequentialWriter writer = new SequentialWriter(tempFile,
+                                                       SequentialWriterOption.newBuilder().finishOnClose(true).build());
+
+        long postingsOffset = writer.position();
+        long postingsPosition = new V5VectorPostingsWriter<T>(remapped)
+                                    .writePostings(writer, vectorValues, postingsMap);
+        long postingsLength = postingsPosition - postingsOffset;
+
+        writer.close();
+        return new PostingsMetadata(postingsOffset, postingsLength);
+    }
+
+    private static class PostingsMetadata
+    {
+        public final long postingsOffset;
+        public final long postingsLength;
+
+        public PostingsMetadata(long postingsOffset, long postingsLength)
+        {
+            this.postingsOffset = postingsOffset;
+            this.postingsLength = postingsLength;
+        }
+    }
+
+    private static Map<VectorFloat<?>, VectorPostings<Integer>> generatePostingsMap(RandomAccessVectorValues vectorValues, Structure structure)
+    {
+        Map<VectorFloat<?>, VectorPostings<Integer>> postingsMap = emptyPostingsMap();
+
+        // generate a list of rowIds that we'll initially assign 1:1 to ordinals,
+        // leaving holes in the rowid sequence if we want ZERO_OR_ONE_TO_MANY
+        var rowIds = new IntArrayList(vectorValues.size(), IntArrayList.DEFAULT_NULL_VALUE);
+        IntUnaryOperator populator = structure == Structure.ZERO_OR_ONE_TO_MANY
+                                   ? i -> 2 * i
+                                   : i -> i;
+        for (int i = 0; i < vectorValues.size(); i++)
+            rowIds.add(populator.applyAsInt(i));
+
+        // assign each ordinal a random rowid, without replacement
+        for (int ordinal = 0; ordinal < vectorValues.size(); ordinal++)
+        {
+            var vector = vectorValues.getVector(ordinal);
+            var rowIdIdx = randomIndex(rowIds);
+            var vp = new VectorPostings<>(rowIds.getInt(rowIdIdx));
+            rowIds.remove(rowIdIdx);
+            vp.setOrdinal(ordinal);
+            postingsMap.put(vector, vp);
+        }
+
+        if (structure == Structure.ONE_TO_ONE)
+            return postingsMap;
+
+        // make some of them 1:many
+        int extraRows = 1 + randomInt(vectorValues.size());
+        for (int i = 0; i < extraRows; i++)
+        {
+            var vector = randomVector(vectorValues);
+            var vp = postingsMap.get(vector);
+            vp.add(populator.applyAsInt(i + vectorValues.size()));
+        }
+
+        return postingsMap;
+    }
+
+    private static <T> ConcurrentSkipListMap<VectorFloat<?>, VectorPostings<T>> emptyPostingsMap()
+    {
+        return new ConcurrentSkipListMap<>((a, b) -> {
+            return Arrays.compare(((ArrayVectorFloat) a).get(), ((ArrayVectorFloat) b).get());
+        });
+    }
+
+    private static VectorFloat<?> randomVector(RandomAccessVectorValues ravv)
+    {
+        return ravv.getVector(randomInt(ravv.size() - 1));
+    }
+
+    private static int randomIndex(List<?> L)
+    {
+        return randomInt(L.size() - 1); // randomInt(max) is inclusive
+    }
+
+    /**
+     * Generate `count` non-random vectors
+     */
+    private ConcurrentVectorValues generateVectors(int count)
+    {
+        var vectorValues = new ConcurrentVectorValues(DIMENSION);
+        for (int i = 0; i < count; i++)
+        {
+            float[] rawVector = new float[DIMENSION];
+            Arrays.fill(rawVector, (float) i);
+            vectorValues.add(i, vts.createFloatVector(rawVector));
+        }
+        return vectorValues;
+    }
+
+    /**
+     * Create a temporary file with the given prefix.
+     */
+    private static File createTempFile(String prefix)
+    {
+        File file = FileUtils.createTempFile(prefix, "tmp");
+        file.deleteOnExit();
+        return file;
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v6/TermsDistributionTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v6/TermsDistributionTest.java
new file mode 100644
index 000000000000..51160165d1c0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v6/TermsDistributionTest.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v6;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.index.sai.disk.ModernResettableByteBuffersIndexOutput;
+import org.apache.cassandra.index.sai.disk.oldlucene.ByteArrayIndexInput;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.junit.Assert.*;
+
+public class TermsDistributionTest
+{
+    static final ByteComparable.Version VERSION = ByteComparable.Version.OSS41;
+
+    @Test
+    public void testEmpty()
+    {
+        AbstractType<Integer> type = Int32Type.instance;
+        TermsDistribution td = new TermsDistribution.Builder(type, VERSION, 10, 10).build();
+        assertEquals(0, td.estimateNumRowsMatchingExact(encode(1)));
+        assertEquals(0, td.estimateNumRowsInRange(encode(0), encode(1000)));
+    }
+
+    @Test
+    public void testExactMatch()
+    {
+        AbstractType<Integer> type = Int32Type.instance;
+        var builder = new TermsDistribution.Builder(type, VERSION, 10, 10);
+        for (int i = 0; i < 1000; i++)
+            builder.add(encode(i), 1);
+        var td = builder.build();
+
+        // in range:
+        assertEquals(1, td.estimateNumRowsMatchingExact(encode(0)));
+        assertEquals(1, td.estimateNumRowsMatchingExact(encode(17)));
+        assertEquals(1, td.estimateNumRowsMatchingExact(encode(999)));
+
+        // out of range:
+        assertEquals(0, td.estimateNumRowsMatchingExact(encode(-1)));
+        assertEquals(0, td.estimateNumRowsMatchingExact(encode(1000)));
+    }
+
+    @Test
+    public void testRangeMatch()
+    {
+        AbstractType<Integer> type = Int32Type.instance;
+        var builder = new TermsDistribution.Builder(type, VERSION, 10, 10);
+        for (int i = 0; i < 1000; i++)
+            builder.add(encode(i), 1);
+        var td = builder.build();
+
+        // in range:
+        assertEquals(10, td.estimateNumRowsInRange(encode(-1), encode(9)));
+        assertEquals(10, td.estimateNumRowsInRange(encode(0), encode(10)));
+        assertEquals(30, td.estimateNumRowsInRange(encode(15), encode(45)));
+
+        // partially in range
+        assertEquals(1000, td.estimateNumRowsInRange(encode(-1), encode(1000)));
+        assertEquals(11, td.estimateNumRowsInRange(encode(-1000), encode(10)));
+        assertEquals(9, td.estimateNumRowsInRange(encode(990), encode(200000)));
+
+        // out of range:
+        assertEquals(0, td.estimateNumRowsInRange(encode(-10), encode(-1)));
+        assertEquals(0, td.estimateNumRowsInRange(encode(1000), encode(1003)));
+
+        // test inclusiveness / exclusiveness:
+        assertEquals(9, td.estimateNumRowsInRange(encode(0), false, encode(10), false));
+        assertEquals(10, td.estimateNumRowsInRange(encode(0), true, encode(10), false));
+        assertEquals(10, td.estimateNumRowsInRange(encode(0), false, encode(10), true));
+        assertEquals(11, td.estimateNumRowsInRange(encode(0), true, encode(10), true));
+
+        // test one side ranges:
+        assertEquals(10, td.estimateNumRowsInRange(null, encode(9)));
+        assertEquals(10, td.estimateNumRowsInRange(null, true, encode(10), false));
+        assertEquals(10, td.estimateNumRowsInRange(null, false, encode(10), false));
+        assertEquals(11, td.estimateNumRowsInRange(null, false, encode(10), true));
+        assertEquals(11, td.estimateNumRowsInRange(null, false, encode(10), true));
+        assertEquals(10, td.estimateNumRowsInRange(encode(990), true, null, true));
+        assertEquals(10, td.estimateNumRowsInRange(encode(990), true, null, false));
+        assertEquals(9, td.estimateNumRowsInRange(encode(990), false, null, false));
+        assertEquals(9, td.estimateNumRowsInRange(encode(990), false, null, false));
+    }
+
+    @Test
+    public void testMostFrequentItems()
+    {
+        int frequentValue = 33; // whatever between 2 and 998
+        int frequentCount = 100; // whatever > 1
+
+        AbstractType<Integer> type = Int32Type.instance;
+        var builder = new TermsDistribution.Builder(type, VERSION, 10, 10);
+        for (int i = 0; i < 1000; i++)
+            builder.add(encode(i), (i == frequentValue) ? frequentCount : 1);
+        var td = builder.build();
+
+        // Exact match the frequent term:
+        assertEquals(frequentCount, td.estimateNumRowsMatchingExact(encode(frequentValue)));
+        assertEquals(frequentCount, td.estimateNumRowsInRange(encode(frequentValue), true, encode(frequentValue), true));
+
+        // A range starting or ending at the frequent term:
+        assertEquals(frequentCount, td.estimateNumRowsInRange(encode(frequentValue - 1), false, encode(frequentValue), true));
+        assertEquals(frequentCount, td.estimateNumRowsInRange(encode(frequentValue), true, encode(frequentValue + 1), false));
+
+        // A range containing the frequent term:
+        assertEquals(frequentCount + 1, td.estimateNumRowsInRange(encode(frequentValue - 1), encode(frequentValue + 1)));
+        assertEquals(frequentCount + 3, td.estimateNumRowsInRange(encode(frequentValue - 2), encode(frequentValue + 2)));
+
+        // Ranges not containing the frequent term.
+        // Frequencies of terms next to the frequent term must not be affected by the frequent term:
+        assertEquals(1, td.estimateNumRowsMatchingExact(encode(frequentValue - 1)));
+        assertEquals(1, td.estimateNumRowsMatchingExact(encode(frequentValue + 1)));
+        assertEquals(1, td.estimateNumRowsInRange(encode(frequentValue - 2), encode(frequentValue - 1)));
+        assertEquals(1, td.estimateNumRowsInRange(encode(frequentValue), encode(frequentValue + 1)));
+    }
+
+
+    @Test
+    public void testFractionalBuckets()
+    {
+        // Test if we get reasonable range estimates when selecting a fraction of a single bucket:
+
+        AbstractType<Double> type = DoubleType.instance;
+        var builder = new TermsDistribution.Builder(type, VERSION, 13, 13);
+        var COUNT = 100000;
+        for (int i = 0; i < COUNT; i++)
+            builder.add(encode((double) i / COUNT), 1);
+        var td = builder.build();
+
+        assertEquals(COUNT * 0.5, td.estimateNumRowsInRange(encode(0.0), encode(0.5)), 1);
+        assertEquals(COUNT * 0.1, td.estimateNumRowsInRange(encode(0.0), encode(0.1)), 1);
+        assertEquals(COUNT * 0.01, td.estimateNumRowsInRange(encode(0.0), encode(0.01)), 1);
+        assertEquals(COUNT * 0.001, td.estimateNumRowsInRange(encode(0.0), encode(0.001)), 1);
+        assertEquals(COUNT * 0.002, td.estimateNumRowsInRange(encode(0.0), encode(0.002)), 1);
+        assertEquals(COUNT * 0.0005, td.estimateNumRowsInRange(encode(0.0), encode(0.0005)), 1);
+        assertEquals(COUNT * 0.0002, td.estimateNumRowsInRange(encode(0.0), encode(0.0002)), 1);
+        assertEquals(COUNT * 0.0001, td.estimateNumRowsInRange(encode(0.0), encode(0.0001)), 1);
+        assertEquals(COUNT * 0.00005, td.estimateNumRowsInRange(encode(0.0), encode(0.00005)), 1);
+        assertEquals(COUNT * 0.00002, td.estimateNumRowsInRange(encode(0.0), encode(0.00002)), 1);
+        assertEquals(COUNT * 0.00001, td.estimateNumRowsInRange(encode(0.0), encode(0.00001)), 1);
+
+        assertEquals(COUNT * 0.5, td.estimateNumRowsInRange(encode(0.5), encode(1.0)), 1);
+        assertEquals(COUNT * 0.1, td.estimateNumRowsInRange(encode(0.5), encode(0.6)), 1);
+        assertEquals(COUNT * 0.01, td.estimateNumRowsInRange(encode(0.5), encode(0.51)), 1);
+        assertEquals(COUNT * 0.001, td.estimateNumRowsInRange(encode(0.5), encode(0.501)), 1);
+        assertEquals(COUNT * 0.002, td.estimateNumRowsInRange(encode(0.5), encode(0.502)), 1);
+        assertEquals(COUNT * 0.0005, td.estimateNumRowsInRange(encode(0.5), encode(0.5005)), 1);
+        assertEquals(COUNT * 0.0002, td.estimateNumRowsInRange(encode(0.5), encode(0.5002)), 1);
+        assertEquals(COUNT * 0.0001, td.estimateNumRowsInRange(encode(0.5), encode(0.5001)), 1);
+        assertEquals(COUNT * 0.00005, td.estimateNumRowsInRange(encode(0.5), encode(0.50005)), 1);
+        assertEquals(COUNT * 0.00002, td.estimateNumRowsInRange(encode(0.5), encode(0.50002)), 1);
+        assertEquals(COUNT * 0.00001, td.estimateNumRowsInRange(encode(0.5), encode(0.50001)), 1);
+    }
+
+
+    @Test
+    public void testFractionalBucketsBigInt()
+    {
+        // Test if we get reasonable range estimates when selecting a fraction of a single bucket:
+
+        AbstractType<BigInteger> type = IntegerType.instance;
+        var builder = new TermsDistribution.Builder(type, VERSION, 13, 13);
+        var COUNT = 100000;
+        for (int i = 0; i < COUNT; i++)
+            builder.add(encodeAsBigInt(i), 1);
+        var td = builder.build();
+
+        assertEquals(COUNT * 0.5, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 2)), 1);
+        assertEquals(COUNT * 0.1, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 10)), 1);
+        assertEquals(COUNT * 0.01, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 100)), 1);
+        assertEquals(COUNT * 0.002, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 500)), 1);
+        assertEquals(COUNT * 0.001, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 1000)), 1);
+        assertEquals(COUNT * 0.0005, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 2000)), 1);
+        assertEquals(COUNT * 0.0002, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 5000)), 1);
+        assertEquals(COUNT * 0.0001, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 10000)), 1);
+        assertEquals(COUNT * 0.00005, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 20000)), 1);
+        assertEquals(COUNT * 0.00002, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 50000)), 1);
+        assertEquals(COUNT * 0.00001, td.estimateNumRowsInRange(encodeAsBigInt(0), encodeAsBigInt(COUNT / 100000)), 1);
+
+        assertEquals(COUNT * 0.5, td.estimateNumRowsInRange(encodeAsBigInt(COUNT / 2), encodeAsBigInt(COUNT)), 1);
+        assertEquals(COUNT * 0.25, td.estimateNumRowsInRange(encodeAsBigInt(COUNT / 2), encodeAsBigInt(COUNT * 3 / 4)), 1);
+    }
+
+
+    @Test
+    public void testFractionalBucketsDecimal()
+    {
+        // Test if we get reasonable range estimates when selecting a fraction of a single bucket:
+
+        AbstractType<BigDecimal> type = DecimalType.instance;
+        var builder = new TermsDistribution.Builder(type, VERSION, 13, 13);
+        var COUNT = 100000;
+        for (int i = 0; i < COUNT; i++)
+            builder.add(encodeAsDecimal((double) i / COUNT), 1);
+        var td = builder.build();
+
+        assertEquals(COUNT * 0.5, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.5)), 1);
+        assertEquals(COUNT * 0.1, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.1)), 1);
+        assertEquals(COUNT * 0.01, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.01)), 1);
+        assertEquals(COUNT * 0.001, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.001)), 1);
+        assertEquals(COUNT * 0.002, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.002)), 1);
+        assertEquals(COUNT * 0.0005, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.0005)), 1);
+        assertEquals(COUNT * 0.0002, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.0002)), 1);
+        assertEquals(COUNT * 0.0001, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.0001)), 1);
+        assertEquals(COUNT * 0.00005, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.00005)), 1);
+        assertEquals(COUNT * 0.00002, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.00002)), 1);
+        assertEquals(COUNT * 0.00001, td.estimateNumRowsInRange(encodeAsDecimal(0.0), encodeAsDecimal(0.00001)), 1);
+
+        assertEquals(COUNT * 0.5, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(1.0)), 1);
+        assertEquals(COUNT * 0.1, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.6)), 1);
+        assertEquals(COUNT * 0.01, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.51)), 1);
+        assertEquals(COUNT * 0.001, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.501)), 1);
+        assertEquals(COUNT * 0.002, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.502)), 1);
+        assertEquals(COUNT * 0.0005, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.5005)), 1);
+        assertEquals(COUNT * 0.0002, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.5002)), 1);
+        assertEquals(COUNT * 0.0001, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.5001)), 1);
+        assertEquals(COUNT * 0.00005, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.50005)), 1);
+        assertEquals(COUNT * 0.00002, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.50002)), 1);
+        assertEquals(COUNT * 0.00001, td.estimateNumRowsInRange(encodeAsDecimal(0.5), encodeAsDecimal(0.50001)), 1);
+    }
+
+    @Test
+    public void testSerde() throws IOException
+    {
+        AbstractType<Double> type = DoubleType.instance;
+        var builder = new TermsDistribution.Builder(type, VERSION, 10, 10);
+        var COUNT = 100000;
+        for (int i = 0; i < COUNT; i++)
+            builder.add(encode((double) i / COUNT), 1);
+        var td = builder.build();
+
+        try (var out = new ModernResettableByteBuffersIndexOutput(1024, ""))
+        {
+            td.write(out);
+            var input = out.toArrayCopy();
+            var tdCopy = TermsDistribution.read(new ByteArrayIndexInput("", input, ByteOrder.LITTLE_ENDIAN), type);
+
+            assertEquals(td.numPoints, tdCopy.numPoints);
+            assertEquals(td.numRows, tdCopy.numRows);
+        }
+    }
+
+    private ByteComparable encode(int value)
+    {
+        return v -> Int32Type.instance.asComparableBytes(Int32Type.instance.decompose(value), v);
+    }
+
+    private ByteComparable encode(double value)
+    {
+        return v -> DoubleType.instance.asComparableBytes(DoubleType.instance.decompose(value), v);
+    }
+
+    private ByteComparable encodeAsDecimal(double value)
+    {
+        ByteBuffer raw = DecimalType.instance.decompose(BigDecimal.valueOf(value));
+        return v -> TypeUtil.asComparableBytes(TypeUtil.asIndexBytes(raw, DecimalType.instance), DecimalType.instance, v);
+    }
+
+    private ByteComparable encodeAsBigInt(long value)
+    {
+        ByteBuffer raw = IntegerType.instance.decompose(BigInteger.valueOf(value));
+        return v -> TypeUtil.asComparableBytes(TypeUtil.asIndexBytes(raw, IntegerType.instance), IntegerType.instance, v);
+    }
+
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/vector/BruteForceRowIdIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/disk/vector/BruteForceRowIdIteratorTest.java
new file mode 100644
index 000000000000..22658da82ac7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/vector/BruteForceRowIdIteratorTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.util.NoSuchElementException;
+
+import org.junit.Test;
+
+import io.github.jbellis.jvector.graph.GraphIndex;
+import io.github.jbellis.jvector.graph.NodeQueue;
+import io.github.jbellis.jvector.graph.NodesIterator;
+import io.github.jbellis.jvector.graph.similarity.ScoreFunction;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.util.BoundedLongHeap;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
+import org.apache.cassandra.index.sai.utils.SegmentRowIdOrdinalPairs;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class BruteForceRowIdIteratorTest
+{
+    private static final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    @Test
+    public void testBruteForceRowIdIteratorForEmptyPQAndTopKEqualsLimit()
+    {
+        var queryVector = vts.createFloatVector(new float[] { 1f, 0f });
+        var heap = new NodeQueue(new BoundedLongHeap(10), NodeQueue.Order.MAX_HEAP);
+        var topK = 10;
+        var limit = 10;
+
+        // Should work for an empty pq
+        var view = new TestView();
+        CloseableReranker reranker = new CloseableReranker(VectorSimilarityFunction.COSINE, queryVector, view);
+        var metrics = new ColumnQueryMetrics.VectorIndexMetrics("ks", "cf", "index");
+        var iter = new BruteForceRowIdIterator(heap, new SegmentRowIdOrdinalPairs(10), reranker, limit, topK, metrics);
+        assertFalse(iter.hasNext());
+        assertThrows(NoSuchElementException.class, iter::next);
+        assertFalse(view.isClosed);
+        iter.close();
+        assertTrue(view.isClosed);
+    }
+
+    private static class TestView implements GraphIndex.ScoringView
+    {
+        private boolean isClosed = false;
+
+        @Override
+        public ScoreFunction.ExactScoreFunction rerankerFor(VectorFloat<?> queryVector, VectorSimilarityFunction vsf)
+        {
+            return ordinal -> vsf.compare(queryVector, vts.createFloatVector(new float[] { ordinal }));
+        }
+
+        @Override
+        public void close()
+        {
+            isClosed = true;
+        }
+
+        //
+        // unused by BruteForceRowIdIterator
+        //
+
+        @Override
+        public ScoreFunction.ApproximateScoreFunction approximateScoreFunctionFor(VectorFloat<?> vectorFloat, VectorSimilarityFunction vectorSimilarityFunction)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public NodesIterator getNeighborsIterator(int i, int i1)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int size()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public GraphIndex.NodeAtLevel entryNode()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Bits liveNodes()
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/vector/ConcurrentVectorValuesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/vector/ConcurrentVectorValuesTest.java
new file mode 100644
index 000000000000..e81f28f41003
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/vector/ConcurrentVectorValuesTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import org.junit.Test;
+
+import io.github.jbellis.jvector.vector.VectorizationProvider;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertNull;
+
+
+public class ConcurrentVectorValuesTest
+{
+    private final VectorTypeSupport vts = VectorizationProvider.getInstance().getVectorTypeSupport();
+
+    @Test(expected = IllegalStateException.class)
+    public void testConcurrentVectorValuesFailsForAlreadySetOrdinal()
+    {
+        var vectorValues = new ConcurrentVectorValues(1);
+        vectorValues.add(0, vts.createFloatVector(new float[]{ 1.0f }));
+        vectorValues.add(0, vts.createFloatVector(new float[]{ 2.0f }));
+    }
+
+    @Test()
+    public void testGet()
+    {
+        var vectorValues = new ConcurrentVectorValues(1);
+        vectorValues.add(1, vts.createFloatVector(new float[]{ 1.0f }));
+        vectorValues.add(2, vts.createFloatVector(new float[]{ 2.0f }));
+
+        assertNull(vectorValues.getVector(0));
+        assertArrayEquals(new float[]{ 1.0f }, (float[]) vectorValues.getVector(1).get(), 0f);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/vector/NodeScoreToRowIdWithScoreIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/disk/vector/NodeScoreToRowIdWithScoreIteratorTest.java
new file mode 100644
index 000000000000..1e6bc918d9b2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/vector/NodeScoreToRowIdWithScoreIteratorTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.util.NoSuchElementException;
+import java.util.PrimitiveIterator;
+import java.util.stream.IntStream;
+
+import org.junit.Test;
+
+import io.github.jbellis.jvector.graph.SearchResult;
+import org.apache.cassandra.utils.CloseableIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class NodeScoreToRowIdWithScoreIteratorTest
+{
+    @Test
+    public void testEmptyIterator()
+    {
+        var rowIdsView = new CustomRowIdsView();
+        var iter = new NodeScoreToRowIdWithScoreIterator(CloseableIterator.emptyIterator(), rowIdsView);
+        assertFalse(iter.hasNext());
+        assertThrows(NoSuchElementException.class, iter::next);
+        assertFalse(rowIdsView.isClosed);
+        iter.close();
+        assertTrue(rowIdsView.isClosed);
+    }
+
+    @Test
+    public void testIterator()
+    {
+        var rowIdsView = new CustomRowIdsView();
+        // Note that the score is ignored at this stage because NodeScores are assumed to be in order
+        var nodeScores = IntStream.range(0, 3).mapToObj(i -> new SearchResult.NodeScore(i, 1f)).iterator();
+        var iter = new NodeScoreToRowIdWithScoreIterator(CloseableIterator.wrap(nodeScores), rowIdsView);
+
+        assertTrue(iter.hasNext());
+        // See CustomRowIdsView for the mapping
+        assertEquals(1, iter.next().getSegmentRowId());
+        assertEquals(3, iter.next().getSegmentRowId());
+        assertEquals(4, iter.next().getSegmentRowId());
+        assertFalse(iter.hasNext());
+        assertThrows(NoSuchElementException.class, iter::next);
+        assertFalse(rowIdsView.isClosed);
+        iter.close();
+        assertTrue(rowIdsView.isClosed);
+    }
+
+    private static class CustomRowIdsView implements RowIdsView
+    {
+        boolean isClosed = false;
+
+        @Override
+        public PrimitiveIterator.OfInt getSegmentRowIdsMatching(int vectorOrdinal)
+        {
+            if (vectorOrdinal == 0)
+                return IntStream.empty().iterator();
+            else if (vectorOrdinal == 1)
+                return IntStream.range(1, 2).iterator();
+            else if (vectorOrdinal == 2)
+                return IntStream.range(3, 5).iterator();
+            else
+                throw new IllegalArgumentException("Unexpected vector ordinal: " + vectorOrdinal);
+        }
+
+        @Override
+        public void close()
+        {
+            isClosed = true;
+        }
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/vector/VectorCompressionTest.java b/test/unit/org/apache/cassandra/index/sai/disk/vector/VectorCompressionTest.java
new file mode 100644
index 000000000000..746c50ff424a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/disk/vector/VectorCompressionTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.vector;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.cql.VectorTester;
+import org.apache.cassandra.index.sai.disk.v2.V2VectorIndexSearcher;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.index.sai.disk.vector.VectorCompression.CompressionType.NONE;
+import static org.junit.Assert.assertEquals;
+
+public class VectorCompressionTest extends VectorTester
+{
+    @Test
+    public void testAda002() throws IOException
+    {
+        // ADA002 is always 1536
+        testOne(VectorSourceModel.ADA002, 1536, VectorSourceModel.ADA002.compressionProvider.apply(1536));
+    }
+
+    @Test
+    public void testGecko() throws IOException
+    {
+        // GECKO is always 768
+        testOne(VectorSourceModel.GECKO, 768, VectorSourceModel.GECKO.compressionProvider.apply(768));
+    }
+
+    @Test
+    public void testOpenAiV3Large() throws IOException
+    {
+        // V3_LARGE can be truncated
+        for (int i = 1; i < 3; i++)
+        {
+            int D = 3072 / i;
+            testOne(VectorSourceModel.OPENAI_V3_LARGE, D, VectorSourceModel.OPENAI_V3_LARGE.compressionProvider.apply(D));
+        }
+    }
+
+    @Test
+    public void testOpenAiV3Small() throws IOException
+    {
+        // V3_SMALL can be truncated
+        for (int i = 1; i < 3; i++)
+        {
+            int D = 1536 / i;
+            testOne(VectorSourceModel.OPENAI_V3_SMALL, D, VectorSourceModel.OPENAI_V3_SMALL.compressionProvider.apply(D));
+        }
+    }
+
+    @Test
+    public void testBert() throws IOException
+    {
+        // BERT is more of a family than a specific model
+        for (int dimension : List.of(128, 256, 512, 1024))
+        {
+            testOne(VectorSourceModel.BERT, dimension, VectorSourceModel.BERT.compressionProvider.apply(dimension));
+        }
+    }
+
+    @Test
+    public void testNV_QA_4() throws IOException
+    {
+        // NV_QA_4 is anecdotally 1024 based on reviewing https://build.nvidia.com/nvidia/embed-qa-4. Couldn't
+        // find supporting documentation for this number, though.
+        testOne(VectorSourceModel.NV_QA_4, 1024, VectorSourceModel.NV_QA_4.compressionProvider.apply(1024));
+    }
+
+    @Test
+    public void testOther() throws IOException
+    {
+        // 25..200 -> Glove dimensions
+        // 1536 -> Ada002
+        // 2000 -> something unknown and large
+        for (int dimension : List.of(25, 50, 100, 200, 1536, 2000))
+            testOne(VectorSourceModel.OTHER, dimension, VectorSourceModel.OTHER.compressionProvider.apply(dimension));
+    }
+
+    @Test
+    public void testFewRows() throws IOException
+    {
+        // with fewer than MIN_PQ_ROWS we expect to observe no compression no matter
+        // what the source model would prefer
+        testOne(1, VectorSourceModel.OTHER, 200, VectorCompression.NO_COMPRESSION);
+    }
+
+    private void testOne(VectorSourceModel model, int originalDimension, VectorCompression expectedCompression) throws IOException
+    {
+        testOne(CassandraOnHeapGraph.MIN_PQ_ROWS, model, originalDimension, expectedCompression);
+    }
+
+    private void testOne(int rows, VectorSourceModel model, int originalDimension, VectorCompression expectedCompression) throws IOException
+    {
+        createTable(String.format("CREATE TABLE %%s (pk int, v vector<float, %d>, PRIMARY KEY(pk)) " +
+                                  "WITH compaction = {'class': 'UnifiedCompactionStrategy', 'num_shards': 1, 'enabled': false}",
+                                  originalDimension));
+
+        for (int i = 0; i < rows; i++)
+            execute("INSERT INTO %s (pk, v) VALUES (?, ?)", i, randomVectorBoxed(originalDimension));
+        flush();
+        // the larger models may flush mid-test automatically, so compact to make sure that we
+        // end up with a single sstable (otherwise PQ might conclude there aren't enough vectors to train on)
+        compact();
+        waitForCompactionsFinished();
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        Assertions.assertThat(cfs.getLiveSSTables()).hasSize(1); // Expected a single sstable after compaction
+
+        // create index after compaction, so we don't have to wait for it to (potentially) build twice,
+        // and give it extra time to build large models
+        String indexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(v) USING 'StorageAttachedIndex'" +
+                                                          " WITH OPTIONS = {'source_model': '%s'}", model));
+        waitForIndexQueryable(KEYSPACE, indexName, 5, TimeUnit.MINUTES);
+
+        // get a View of the sstables that contain indexed data
+        var index = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName);
+        var view = index.getIndexContext().getView();
+
+        // there should be one sstable with one segment
+        Assertions.assertThat(view).hasSize(1); // Expected a single sstable after compaction
+        var ssti = view.iterator().next();
+        var segments = ssti.getSegments();
+        Assertions.assertThat(segments).hasSize(1); // Expected a single segment
+
+        // open a Searcher for the segment, so we can check that its compression is what we expected
+        try (var segment = segments.iterator().next();
+             var searcher = (V2VectorIndexSearcher) segment.getIndexSearcher())
+        {
+            var vc = searcher.getCompression();
+            var msg = String.format("Expected %s but got %s", expectedCompression, vc);
+            assertEquals(msg, expectedCompression, vc);
+            if (vc.type != NONE)
+            {
+                assertEquals((int) (100 * VectorSourceModel.tapered2x(100) * model.overqueryProvider.apply(vc)),
+                             model.rerankKFor(100, vc));
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
index 5b38876de3e5..dbf55eeb6240 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
@@ -23,11 +23,15 @@
 import java.util.Collection;
 import java.util.Collections;
 
+import javax.management.InstanceNotFoundException;
+
 import com.google.common.collect.Lists;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -39,10 +43,10 @@
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
+import org.apache.cassandra.index.IndexNotAvailableException;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.disk.v1.SSTableIndexWriter;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.metrics.AbstractMetricsTest;
 import org.apache.cassandra.inject.ActionBuilder;
 import org.apache.cassandra.inject.Expression;
 import org.apache.cassandra.inject.Injection;
@@ -61,20 +65,20 @@
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Refs;
 
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-public class CompactionTest extends SAITester
+public class CompactionTest extends AbstractMetricsTest
 {
     @Test
     public void testAntiCompaction() throws Throwable
     {
         createTable(CREATE_TABLE_TEMPLATE);
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
         verifyNoIndexFiles();
 
         // create 100 rows in 1 sstable
@@ -85,8 +89,8 @@ public void testAntiCompaction() throws Throwable
 
         // verify 1 sstable index
         assertNumRows(num, "SELECT * FROM %%s WHERE v1 >= 0");
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
-        verifySSTableIndexes(numericIndexIdentifier, 1);
+        verifyIndexFiles(numericIndexContext, null, 1, 0);
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 1);
 
         // split sstable into repaired and unrepaired
         ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
@@ -97,23 +101,23 @@ public void testAntiCompaction() throws Throwable
              Refs<SSTableReader> refs = Refs.ref(sstables))
         {
             InetAddressAndPort endpoint = InetAddressAndPort.getByName("10.0.0.1");
-            TimeUUID parentRepairSession = TimeUUID.Generator.nextTimeUUID();
+            TimeUUID parentRepairSession = nextTimeUUID();
             ActiveRepairService.instance().registerParentRepairSession(parentRepairSession,
-                                                                       endpoint,
-                                                                       Lists.newArrayList(cfs),
-                                                                       Collections.singleton(range),
-                                                                       true,
-                                                                       1000,
-                                                                       false,
-                                                                       PreviewKind.NONE);
+                                                                     endpoint,
+                                                                     Lists.newArrayList(cfs),
+                                                                     Collections.singleton(range),
+                                                                     true,
+                                                                     1000,
+                                                                     false,
+                                                                     PreviewKind.NONE);
             RangesAtEndpoint replicas = RangesAtEndpoint.builder(endpoint).add(Replica.fullReplica(endpoint, range)).build();
             CompactionManager.instance.performAnticompaction(cfs, replicas, refs, txn, parentRepairSession, () -> false);
         }
 
         // verify 2 sstable indexes
         assertNumRows(num, "SELECT * FROM %%s WHERE v1 >= 0");
-        waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2));
-        verifySSTableIndexes(numericIndexIdentifier, 2);
+        waitForAssert(() -> verifyIndexFiles(numericIndexContext, null, 2, 0));
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 2);
 
         // index components are included after anti-compaction
         verifyIndexComponentsIncludedInSSTable();
@@ -123,9 +127,8 @@ public void testAntiCompaction() throws Throwable
     public void testConcurrentQueryWithCompaction()
     {
         createTable(CREATE_TABLE_TEMPLATE);
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        waitForTableIndexesQueryable();
+        String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        String v2IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
 
         int num = 10;
         for (int i = 0; i < num; i++)
@@ -139,8 +142,8 @@ public void testConcurrentQueryWithCompaction()
             {
                 try
                 {
-                    assertNumRows(num, "SELECT id1 FROM %s WHERE v1>=0");
-                    assertNumRows(num, "SELECT id1 FROM %s WHERE v2='0'");
+                    assertNumRows(num, "SELECT id1 FROM %%s WHERE v1>=0");
+                    assertNumRows(num, "SELECT id1 FROM %%s WHERE v2='0'");
                 }
                 catch (Throwable e)
                 {
@@ -151,16 +154,50 @@ public void testConcurrentQueryWithCompaction()
 
         compactionTest.start();
 
-        verifySSTableIndexes(numericIndexIdentifier, num);
-        verifySSTableIndexes(literalIndexIdentifier, num);
+        verifySSTableIndexes(v1IndexName, num);
+        verifySSTableIndexes(v2IndexName, num);
+    }
+
+    @Test
+    public void testCompactionWithDisabledIndexReads() throws Throwable
+    {
+        CassandraRelevantProperties.SAI_INDEX_READS_DISABLED.setBoolean(true);
+        try
+        {
+            createTable(CREATE_TABLE_TEMPLATE);
+            String v1IndexName = createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+            String v2IndexName = createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+
+            int num = 10;
+            for (int i = 0; i < num; i++)
+            {
+                execute("INSERT INTO %s (id1, v1, v2) VALUES (?, 0, '0')", Integer.toString(i));
+                flush();
+            }
+
+            startCompaction();
+            waitForCompactionsFinished();
+
+            // Compactions should run fine and create the expected number of index files for each index (1 since we
+            // compacted everything into a single sstable).
+            verifySSTableIndexes(v1IndexName, 1);
+            verifySSTableIndexes(v2IndexName, 1);
+
+            // But index queries should not be allowed.
+            assertThrows(IndexNotAvailableException.class, () -> executeInternal("SELECT id1 FROM %s WHERE v1>=0"));
+        }
+        finally
+        {
+            CassandraRelevantProperties.SAI_INDEX_READS_DISABLED.setBoolean(false);
+        }
     }
 
     @Test
     public void testAbortCompactionWithEarlyOpenSSTables() throws Throwable
     {
         createTable(CREATE_TABLE_TEMPLATE);
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
+        String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        String v2IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
 
         int sstables = 2;
         int num = 10;
@@ -209,11 +246,16 @@ public void testAbortCompactionWithEarlyOpenSSTables() throws Throwable
         // verify indexes are working
         assertNumRows(num, "SELECT id1 FROM %%s WHERE v1=0");
         assertNumRows(num, "SELECT id1 FROM %%s WHERE v2='0'");
-        verifySSTableIndexes(numericIndexIdentifier, sstables);
-        verifySSTableIndexes(literalIndexIdentifier, sstables);
+        verifySSTableIndexes(v1IndexName, sstables);
+        verifySSTableIndexes(v2IndexName, sstables);
     }
 
     @Test
+    // This test probably never worked, but initially `TestWithConcurrentVerification` was also not working properly
+    // and was ignoring errors throw during the `verificationTask`, hidding the problem with this test.
+    // `TestWithConcurrentVerification` was fixed, leading to this failing, and it's not immediately clear what the
+    // right fix should be, so for now it is ignored, but it should eventually be fixed.
+    @Ignore
     public void testConcurrentIndexBuildWithCompaction() throws Throwable
     {
         createTable(CREATE_TABLE_TEMPLATE);
@@ -238,36 +280,35 @@ public void testConcurrentIndexBuildWithCompaction() throws Throwable
             Injections.inject(compactionLatch);
 
             TestWithConcurrentVerification compactionTask = new TestWithConcurrentVerification(
-            () -> {
-                try
-                {
-                    upgradeSSTables();
-                    fail("Expected CompactionInterruptedException");
-                }
-                catch (Exception e)
-                {
-                    assertTrue("Expected CompactionInterruptedException, but got " + e,
-                               Throwables.isCausedBy(e, CompactionInterruptedException.class::isInstance));
-                }
-            },
-            () -> {
-                try
-                {
-                    waitForAssert(() -> Assert.assertEquals(1, compactionLatch.getCount()));
-
-                    // build indexes on SSTables that will be compacted soon
-                    createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-                    createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
-                    waitForTableIndexesQueryable();
-
-                    // continue in-progress compaction
-                    compactionLatch.countDown();
-                }
-                catch (Exception e)
-                {
-                    throw new RuntimeException(e);
-                }
-            }, -1 // run verification task once
+                    () -> {
+                        try
+                        {
+                            upgradeSSTables();
+                            fail("Expected CompactionInterruptedException");
+                        }
+                        catch (Exception e)
+                        {
+                            assertTrue("Expected CompactionInterruptedException, but got " + e,
+                                       Throwables.isCausedBy(e, CompactionInterruptedException.class));
+                        }
+                    },
+                    () -> {
+                        try
+                        {
+                            waitForAssert(() -> Assert.assertEquals(1, compactionLatch.getCount()));
+
+                            // build indexes on SSTables that will be compacted soon
+                            createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+                            createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+
+                            // continue in-progress compaction
+                            compactionLatch.countDown();
+                        }
+                        catch (Exception e)
+                        {
+                            throw new RuntimeException(e);
+                        }
+                    }, -1 // run verification task once
             );
 
             compactionTask.start();
@@ -279,8 +320,8 @@ public void testConcurrentIndexBuildWithCompaction() throws Throwable
 
         assertNumRows(num, "SELECT id1 FROM %%s WHERE v1>=0");
         assertNumRows(num, "SELECT id1 FROM %%s WHERE v2='0'");
-        verifySSTableIndexes(createIndexIdentifier(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER)), sstables);
-        verifySSTableIndexes(createIndexIdentifier(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER)), sstables);
+        verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), sstables);
+        verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), sstables);
     }
 
     @Test
@@ -303,33 +344,33 @@ public void testConcurrentIndexDropWithCompaction() throws Throwable
         assertNotEquals(0, getDiskUsage());
 
         Injections.Barrier compactionLatch =
-        Injections.newBarrier("pause_compaction_for_drop", 2, false)
-                  .add(InvokePointBuilder.newInvokePoint().onClass(SSTableIndexWriter.class).onMethod("addRow"))
-                  .build();
+                Injections.newBarrier("pause_compaction_for_drop", 2, false)
+                          .add(InvokePointBuilder.newInvokePoint().onClass(SSTableIndexWriter.class).onMethod("addRow"))
+                          .build();
         try
         {
             // pause in-progress compaction
             Injections.inject(compactionLatch);
 
             TestWithConcurrentVerification compactionTask = new TestWithConcurrentVerification(
-            this::upgradeSSTables,
-            () -> {
-                try
-                {
-                    waitForAssert(() -> Assert.assertEquals(1, compactionLatch.getCount()));
-
-                    // drop all indexes
-                    dropIndex("DROP INDEX %s." + v1IndexName);
-                    dropIndex("DROP INDEX %s." + v2IndexName);
-
-                    // continue in-progress compaction
-                    compactionLatch.countDown();
-                }
-                catch (Throwable e)
-                {
-                    throw new RuntimeException(e);
-                }
-            }, -1 // run verification task once
+                    () -> upgradeSSTables(),
+                    () -> {
+                        try
+                        {
+                            waitForAssert(() -> Assert.assertEquals(1, compactionLatch.getCount()));
+
+                            // drop all indexes
+                            dropIndex("DROP INDEX %s." + v1IndexName);
+                            dropIndex("DROP INDEX %s." + v2IndexName);
+
+                            // continue in-progress compaction
+                            compactionLatch.countDown();
+                        }
+                        catch (Throwable e)
+                        {
+                            throw new RuntimeException(e);
+                        }
+                    }, -1 // run verification task once
             );
 
             compactionTask.start();
@@ -341,15 +382,25 @@ public void testConcurrentIndexDropWithCompaction() throws Throwable
         }
 
         // verify index group metrics are cleared.
-        assertNull(getCurrentIndexGroup());
+        assertThatThrownBy(this::getOpenIndexFiles).hasRootCauseInstanceOf(InstanceNotFoundException.class);
+        assertThatThrownBy(this::getDiskUsage).hasRootCauseInstanceOf(InstanceNotFoundException.class);
 
-        // verify indexes are dropped
         // verify indexes are dropped
         assertThatThrownBy(() -> executeNet("SELECT id1 FROM %s WHERE v1>=0"))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
         assertThatThrownBy(() -> executeNet("SELECT id1 FROM %s WHERE v2='0'"))
-        .isInstanceOf(InvalidQueryException.class)
-        .hasMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+    }
+
+    protected int getOpenIndexFiles()
+    {
+        return (int) getMetricValue(objectNameNoIndex("OpenIndexFiles", KEYSPACE, currentTable(), "IndexGroupMetrics"));
+    }
+
+    protected long getDiskUsage()
+    {
+        return (long) getMetricValue(objectNameNoIndex("DiskUsedBytes", KEYSPACE, currentTable(), "IndexGroupMetrics"));
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/DiskSpaceTest.java b/test/unit/org/apache/cassandra/index/sai/functional/DiskSpaceTest.java
index 1d052c1d6f4e..3553ebce7c55 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/DiskSpaceTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/DiskSpaceTest.java
@@ -41,7 +41,6 @@ public void testTableTotalDiskSpaceUsed() throws Throwable
 
         // create index, disk space should include index components
         String indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForTableIndexesQueryable();
 
         long indexSize = indexDiskSpaceUse();
         long sstableSizeWithIndex = totalDiskSpaceUsed();
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/DropTableTest.java b/test/unit/org/apache/cassandra/index/sai/functional/DropTableTest.java
index b5322d3e08fb..b86765c633fc 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/DropTableTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/DropTableTest.java
@@ -51,7 +51,6 @@ public void testDropTableLifecycle() throws Throwable
         createTable(CREATE_TABLE_TEMPLATE);
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
-        waitForTableIndexesQueryable();
 
         int rows = 100;
         for (int j = 0; j < rows; j++)
@@ -66,7 +65,7 @@ public void testDropTableLifecycle() throws Throwable
         SSTableReader sstable = Iterables.getOnlyElement(cfs.getLiveSSTables());
 
         ArrayList<String> files = new ArrayList<>();
-        for (Component component : sstable.getComponents())
+        for (Component component : sstable.components())
         {
             File file = sstable.descriptor.fileFor(component);
             if (file.exists())
@@ -87,7 +86,7 @@ public void testDropTableLifecycle() throws Throwable
         assertAllFileRemoved(files);
     }
 
-    void assertAllFileExists(List<String> filePaths)
+    void assertAllFileExists(List<String> filePaths) throws Exception
     {
         for (String path : filePaths)
         {
@@ -96,11 +95,12 @@ void assertAllFileExists(List<String> filePaths)
         }
     }
 
-    void assertAllFileRemoved(List<String> filePaths)
+    void assertAllFileRemoved(List<String> filePaths) throws Exception
     {
         for (String path : filePaths)
         {
             File file = new File(path);
+            System.err.println("## check="+path);
             assertFalse("Expect file being removed, but it still exists: " + path, file.exists());
         }
     }
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java b/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java
index 6907006e0880..ec0008e48d5a 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java
@@ -24,24 +24,24 @@
 
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.index.IndexNotAvailableException;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.SSTableContext;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.utils.SuppressLeakCheck;
 import org.apache.cassandra.inject.Injection;
 import org.apache.cassandra.inject.Injections;
 import org.assertj.core.api.Assertions;
 
 import static org.junit.Assert.assertEquals;
 
+@SuppressLeakCheck(reason="The SSTableContext.<init> error is synthetic and can't happen in a live environment")
 public class FailureTest extends SAITester
 {
     @Test
     public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringFlush() throws Throwable
     {
         createTable(CREATE_TABLE_TEMPLATE);
-        IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
 
         execute("INSERT INTO %s (id1, v1) VALUES ('1', 1)");
         execute("INSERT INTO %s (id1, v1) VALUES ('2', 2)");
@@ -49,8 +49,8 @@ public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringFlush() thro
 
         assertEquals(1, execute("SELECT id1 FROM %s WHERE v1 > 1").size());
 
-        verifyIndexFiles(indexTermType, indexIdentifier, 1, 1, 1);
-        verifySSTableIndexes(indexIdentifier, 1, 1);
+        verifyIndexFiles(numericIndexContext, null, 1, 1, 0, 1, 0);
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 1, 1);
 
         execute("INSERT INTO %s (id1, v1) VALUES ('3', 3)");
 
@@ -60,7 +60,7 @@ public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringFlush() thro
         flush();
 
         // Verify that, while the node is still operational, the index is not.
-        Assertions.assertThatThrownBy(() -> execute("SELECT * FROM %s WHERE v1 > 1"))
+        Assertions.assertThatThrownBy(() -> executeInternal("SELECT * FROM %s WHERE v1 > 1"))
                   .isInstanceOf(IndexNotAvailableException.class);
 
         ssTableContextCreationFailure.disable();
@@ -68,8 +68,8 @@ public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringFlush() thro
         // Now verify that a restart actually repairs the index...
         simulateNodeRestart();
 
-        verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2);
-        verifySSTableIndexes(indexIdentifier, 2, 2);
+        verifyIndexComponentFiles(numericIndexContext, null);
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 2, 2);
 
         assertEquals(2, execute("SELECT id1 FROM %s WHERE v1 > 1").size());
     }
@@ -78,8 +78,7 @@ public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringFlush() thro
     public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringCompaction() throws Throwable
     {
         createTable(CREATE_TABLE_TEMPLATE);
-        IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
 
         execute("INSERT INTO %s (id1, v1) VALUES ('1', 1)");
         flush();
@@ -89,8 +88,8 @@ public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringCompaction()
 
         assertEquals(1, execute("SELECT id1 FROM %s WHERE v1 > 1").size());
 
-        verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2);
-        verifySSTableIndexes(indexIdentifier, 2, 2);
+        verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0);
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 2, 2);
 
         Injection ssTableContextCreationFailure = newFailureOnEntry("context_failure_on_compaction", SSTableContext.class, "<init>", RuntimeException.class);
         Injections.inject(ssTableContextCreationFailure);
@@ -98,7 +97,7 @@ public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringCompaction()
         compact();
 
         // Verify that the index is not available.
-        Assertions.assertThatThrownBy(() -> execute("SELECT * FROM %s WHERE v1 > 1"))
+        Assertions.assertThatThrownBy(() -> executeInternal("SELECT * FROM %s WHERE v1 > 1"))
                   .isInstanceOf(IndexNotAvailableException.class);
     }
 
@@ -117,16 +116,15 @@ public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringCreation() t
         Injection ssTableContextCreationFailure = newFailureOnEntry("context_failure_on_creation", SSTableContext.class, "<init>", RuntimeException.class);
         Injections.inject(ssTableContextCreationFailure);
 
-        IndexIdentifier indexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2")));
+        String v2IndexName = createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2"));
 
         // Verify that the initial index build fails...
-        verifyInitialIndexFailed(indexIdentifier.indexName);
+        verifyInitialIndexFailed(v2IndexName);
 
-        verifyNoIndexFiles();
-        verifySSTableIndexes(indexIdentifier, 0);
+        verifySSTableIndexes(v2IndexName, 0);
 
         // ...and then verify that, while the node is still operational, the index is not.
-        Assertions.assertThatThrownBy(() -> execute("SELECT * FROM %s WHERE v2 = '1'"))
+        Assertions.assertThatThrownBy(() -> executeInternal("SELECT * FROM %s WHERE v2 = '1'"))
                   .isInstanceOf(IndexNotAvailableException.class);
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java b/test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java
index fa480b5c8dbd..23861fb21099 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java
@@ -24,21 +24,22 @@
 
 import com.datastax.driver.core.ResultSet;
 import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.disk.v1.bbtree.NumericIndexWriter;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
+import org.apache.cassandra.index.sai.disk.v1.kdtree.NumericIndexWriter;
 
 import static org.junit.Assert.assertEquals;
 
 public class FlushingTest extends SAITester
 {
     @Test
-    public void testFlushingLargeStaleMemtableIndex()
+    public void testFlushingLargeStaleMemtableIndex() throws Throwable
     {
         createTable(CREATE_TABLE_TEMPLATE);
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
 
+        // BDKWriter#valueCount is updated when leaf values are written at BKDWriter#writeLeakBlock on every
+        // BKDWriter#DEFAULT_MAX_POINTS_IN_LEAF_NODE (1024) number of points, see LUCENE-8765
         int overwrites = NumericIndexWriter.MAX_POINTS_IN_LEAF_NODE + 1;
         for (int j = 0; j < overwrites; j++)
         {
@@ -49,14 +50,15 @@ public void testFlushingLargeStaleMemtableIndex()
 
         ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
         assertEquals(1, rows.all().size());
+
+        assertIndexFilesInToc(indexFiles());
     }
 
     @Test
-    public void testFlushingOverwriteDelete()
+    public void testFlushingOverwriteDelete() throws Throwable
     {
         createTable(CREATE_TABLE_TEMPLATE);
-        IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
 
         int sstables = 3;
         for (int j = 0; j < sstables; j++)
@@ -66,16 +68,20 @@ public void testFlushingOverwriteDelete()
             flush();
         }
 
+        assertIndexFilesInToc(indexFiles());
+
         ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1 >= 0");
         assertEquals(0, rows.all().size());
-        verifyIndexFiles(indexTermType, indexIdentifier, sstables, 0, sstables);
-        verifySSTableIndexes(indexIdentifier, sstables, 0);
+        verifyIndexFiles(numericIndexContext, null, sstables, 0, 0, sstables, 0);
+        verifySSTableIndexes(numericIndexContext.getIndexName(), sstables, sstables);
 
         compact();
-        waitForAssert(() -> verifyIndexFiles(indexTermType, indexIdentifier, 1, 0, 1));
+        waitForAssert(() -> verifyIndexFiles(numericIndexContext, null, 1, 0, 0, 1, 0));
 
         rows = executeNet("SELECT id1 FROM %s WHERE v1 >= 0");
         assertEquals(0, rows.all().size());
-        verifySSTableIndexes(indexIdentifier, 1, 0);
+        verifySSTableIndexes(numericIndexContext.getIndexName(), 1, 1);
+
+        assertIndexFilesInToc(indexFiles());
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java b/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java
index 0747f8eecf81..d3a5b6a698f2 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java
@@ -18,21 +18,19 @@
 
 package org.apache.cassandra.index.sai.functional;
 
-import java.util.Collection;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import com.google.common.collect.Iterables;
 import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
 import org.apache.cassandra.index.sai.disk.format.Version;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 
@@ -44,24 +42,23 @@ public class GroupComponentsTest extends SAITester
     @Test
     public void testInvalidateWithoutObsolete()
     {
-        createTable("CREATE TABLE %s (pk int primary key, value text)");
-        createIndex("CREATE INDEX ON %s(value) USING 'sai'");
+        createTable("CREATE TABLE %s (pk int primary key, value int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
         execute("INSERT INTO %s (pk) VALUES (1)");
         flush();
 
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
         StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
         assertNotNull(group);
-
-        StorageAttachedIndex index = (StorageAttachedIndex) group.getIndexes().iterator().next();
+        StorageAttachedIndex index = group.getIndexes().iterator().next();
         SSTableReader sstable = Iterables.getOnlyElement(cfs.getLiveSSTables());
 
-        Set<Component> components = StorageAttachedIndexGroup.getLiveComponents(sstable, getIndexesFromGroup(group));
-        assertEquals(Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).size() + 1, components.size());
+        Set<Component> components = group.activeComponents(sstable);
+        assertEquals(Version.latest().onDiskFormat().perSSTableComponentTypes().size() + 1, components.size());
 
         // index files are released but not removed
         cfs.invalidate(true, false);
-        Assert.assertTrue(index.view().getIndexes().isEmpty());
+        Assert.assertTrue(index.getIndexContext().getView().getIndexes().isEmpty());
         for (Component component : components)
             Assert.assertTrue(sstable.descriptor.fileFor(component).exists());
     }
@@ -69,52 +66,42 @@ public void testInvalidateWithoutObsolete()
     @Test
     public void getLiveComponentsForEmptyIndex()
     {
-        createTable("CREATE TABLE %s (pk int primary key, value text)");
-        createIndex("CREATE INDEX ON %s(value) USING 'sai'");
+        createTable("CREATE TABLE %s (pk int primary key, value int)");
+        createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
         execute("INSERT INTO %s (pk) VALUES (1)");
         flush();
 
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
         StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
         assertNotNull(group);
-
         Set<SSTableReader> sstables = cfs.getLiveSSTables();
 
         assertEquals(1, sstables.size());
 
-        Set<Component> components = StorageAttachedIndexGroup.getLiveComponents(sstables.iterator().next(), getIndexesFromGroup(group));
+        Set<Component> components = group.activeComponents(sstables.iterator().next());
 
-        assertEquals(Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).size() + 1, components.size());
+        assertEquals(Version.latest().onDiskFormat().perSSTableComponentTypes().size() + 1, components.size());
     }
 
     @Test
     public void getLiveComponentsForPopulatedIndex()
     {
-        createTable("CREATE TABLE %s (pk int primary key, value text)");
-
-        createIndex("CREATE INDEX ON %s(value) USING 'sai'");
-        IndexTermType indexTermType = createIndexTermType(UTF8Type.instance);
-
-        execute("INSERT INTO %s (pk, value) VALUES (1, '1')");
+        createTable("CREATE TABLE %s (pk int primary key, value int)");
+        IndexContext indexContext = createIndexContext(createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'"), Int32Type.instance);
+        execute("INSERT INTO %s (pk, value) VALUES (1, 1)");
         flush();
 
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
         StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
         assertNotNull(group);
-
         Set<SSTableReader> sstables = cfs.getLiveSSTables();
 
         assertEquals(1, sstables.size());
 
-        Set<Component> components = StorageAttachedIndexGroup.getLiveComponents(sstables.iterator().next(), getIndexesFromGroup(group));
+        Set<Component> components = group.activeComponents(sstables.iterator().next());
 
-        assertEquals(Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).size() +
-                     Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType).size(),
+        assertEquals(Version.latest().onDiskFormat().perSSTableComponentTypes().size() +
+                     Version.latest().onDiskFormat().perIndexComponentTypes(indexContext).size(),
                      components.size());
     }
-
-    private Collection<StorageAttachedIndex> getIndexesFromGroup(StorageAttachedIndexGroup group)
-    {
-        return group.getIndexes().stream().map(index -> (StorageAttachedIndex)index).collect(Collectors.toList());
-    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/IndexBuildDeciderTest.java b/test/unit/org/apache/cassandra/index/sai/functional/IndexBuildDeciderTest.java
new file mode 100644
index 000000000000..b5c76ce897be
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/functional/IndexBuildDeciderTest.java
@@ -0,0 +1,179 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai.functional;
+
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Iterables;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.exceptions.ReadFailureException;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.IndexBuildDecider;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SSTableContextManager;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.index.sai.disk.v1.MemtableIndexWriter;
+import org.apache.cassandra.index.sai.disk.v2.V2OnDiskFormat;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.awaitility.Awaitility;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_INDEX_BUILD_DECIDER;
+import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class IndexBuildDeciderTest extends SAITester
+{
+    static final Injections.Counter flushWithMemtableIndexWriterCount =
+    Injections.newCounter("flushWithMemtableIndexWriterCount")
+              .add(newInvokePoint().onClass(MemtableIndexWriter.class).onMethod("<init>"))
+              .build();
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        CUSTOM_INDEX_BUILD_DECIDER.setString(IndexBuildDeciderWithoutInitialBuild.class.getName());
+        CQLTester.setUpClass();
+    }
+
+    @AfterClass
+    public static void teardown()
+    {
+        CUSTOM_INDEX_BUILD_DECIDER.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+    }
+
+    @Before
+    public void before() throws Throwable
+    {
+        Injections.inject(flushWithMemtableIndexWriterCount);
+    }
+
+    @After
+    public void after()
+    {
+        flushWithMemtableIndexWriterCount.reset();
+        Injections.deleteAll();
+    }
+
+    @Test
+    public void testNoInitialBuildWithSAI()
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        // populate one sstable
+        int rowCount = 10;
+        for (int j = 0; j < rowCount / 2; j++)
+            execute("INSERT INTO %s (id1, v1) VALUES (?, ?)", String.valueOf(j), j);
+        flush();
+
+        SSTableReader initialSSTable = Iterables.getOnlyElement(getCurrentColumnFamilyStore().getLiveSSTables());
+        int initialSSTableFileCount = sstableFileCount(initialSSTable);
+
+        // populate memtable
+        for (int j = rowCount / 2; j < rowCount; j++)
+            execute("INSERT INTO %s (id1, v1) VALUES (?, ?)", String.valueOf(j), j);
+
+        // create index: it's not queryable because IndexBuildDeciderWithoutInitialBuild skipped the initial build and
+        // didn't consider the index queryable because there was already one sstable
+        createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        Awaitility.await("Index is not queryable")
+                  .pollDelay(5, TimeUnit.SECONDS)
+                  .until(() -> !areAllTableIndexesQueryable());
+        assertThatThrownBy(() -> executeNet("SELECT * FROM %s WHERE v1>=0")).isInstanceOf(ReadFailureException.class);
+
+        StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(getCurrentColumnFamilyStore());
+        assertNotNull(group);
+        SSTableContextManager sstableContext = group.sstableContextManager();
+
+        // given there was no index build, the initial sstable has no index files
+        assertEquals(initialSSTableFileCount, sstableFileCount(initialSSTable));
+        assertFalse(sstableContext.contains(initialSSTable));
+
+        // creating the index caused the memtable to be flushed: any existing memtables before the index is created are
+        // flushed with the SSTableIndexWriter
+        assertEquals(2, getCurrentColumnFamilyStore().getLiveSSTables().size());
+        assertEquals(0, flushWithMemtableIndexWriterCount.get());
+
+        // check the second sstable flushed at index creation is now indexed:
+        SSTableReader secondSSTable = getCurrentColumnFamilyStore().getLiveSSTables().stream().filter(s -> s != initialSSTable).findFirst().orElse(null);
+        assertNotNull(secondSSTable);
+        assertEquals(initialSSTableFileCount + numericIndexFileCount(), sstableFileCount(secondSSTable));
+        assertTrue(sstableContext.contains(secondSSTable));
+
+        // SAI#canFlushFromMemtableIndex should be true
+        StorageAttachedIndex sai = group.getIndexes().iterator().next();
+        assertTrue(sai.canFlushFromMemtableIndex());
+
+        // flush another memtable: it should be flushed with MemtableIndexWriter
+        execute("INSERT INTO %s (id1, v1) VALUES (?, ?)", String.valueOf(0), 0);
+        flush();
+        assertEquals(1, flushWithMemtableIndexWriterCount.get());
+        SSTableReader thirdSStable = getCurrentColumnFamilyStore().getLiveSSTables().stream().filter(s -> s != initialSSTable && s != secondSSTable).findFirst().orElse(null);
+        assertNotNull(thirdSStable);
+
+        assertEquals(initialSSTableFileCount + numericIndexFileCount(), sstableFileCount(thirdSStable));
+        assertTrue(sstableContext.contains(thirdSStable));
+    }
+
+    private int sstableFileCount(SSTableReader secondSSTable)
+    {
+        Path sstableDir = secondSSTable.descriptor.directory.toPath();
+        String prefix = sstableDir + "/" + secondSSTable.descriptor.filenamePart();
+        return FileUtils.listPaths(sstableDir, path -> path.toString().startsWith(prefix)).size();
+    }
+
+    private int numericIndexFileCount()
+    {
+        IndexContext context = createIndexContext("v1", Int32Type.instance);
+        return V2OnDiskFormat.instance.perIndexComponentTypes(context).size()
+               + V2OnDiskFormat.instance.perSSTableComponentTypes().size();
+    }
+
+    public static class IndexBuildDeciderWithoutInitialBuild implements IndexBuildDecider
+    {
+        @Override
+        public Decision onInitialBuild()
+        {
+            return Decision.NONE;
+        }
+
+        @Override
+        public boolean isIndexQueryableAfterInitialBuild(ColumnFamilyStore cfs)
+        {
+            return cfs.getLiveSSTables().isEmpty();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java b/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java
index 1dcacec8b92c..b868beb932a6 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java
@@ -25,14 +25,14 @@
 
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
 import org.apache.cassandra.inject.Injection;
 import org.apache.cassandra.inject.Injections;
 import org.apache.cassandra.inject.InvokePointBuilder;
-import org.assertj.core.api.Assertions;
+
+import static org.junit.Assert.assertFalse;
 
 public class NodeRestartTest extends SAITester
 {
@@ -66,7 +66,7 @@ public void shouldSurviveRestartWithPreJoinAndInitFailures() throws Throwable
         // We should have completed no actual SSTable validations:
         assertValidationCount(0, 0);
 
-        Assertions.assertThat(getNotQueryableIndexes()).isNotEmpty();
+        assertFalse(areAllTableIndexesQueryable());
     }
 
     // We don't allow the node to actually join the ring before a valid index is ready to accept queries.
@@ -113,26 +113,20 @@ public void shouldRestartWithExistingIndexComponents() throws Throwable
         execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
         flush();
 
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-        IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+        IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
+        verifyIndexFiles(numericIndexContext,literalIndexContext, 1, 1);
         assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0");
         assertNumRows(1, "SELECT * FROM %%s WHERE v2 = '0'");
         assertValidationCount(0, 0);
 
         simulateNodeRestart();
 
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
-        verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
+        verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1);
 
         assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0");
         assertNumRows(1, "SELECT * FROM %%s WHERE v2 = '0'");
 
-        waitForTableIndexesQueryable();
-
         // index components are included after restart
         verifyIndexComponentsIncludedInSSTable();
     }
@@ -178,11 +172,8 @@ void createSingleRowIndex() throws Throwable
         execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
         flush();
 
-        IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
-
-        waitForTableIndexesQueryable();
-        verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+        verifyIndexFiles(numericIndexContext, null, 1, 0);
         assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0");
         assertValidationCount(0, 0);
     }
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java b/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java
index fd38bc600636..b727777069a3 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java
@@ -22,9 +22,8 @@
 import org.junit.Test;
 
 import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
 import org.apache.cassandra.inject.Injections;
 
 import static org.junit.Assert.assertEquals;
@@ -52,11 +51,9 @@ public void shouldTakeAndRestoreSnapshots() throws Throwable
 
         // Insert some initial data and create the index over it
         execute("INSERT INTO %s (id1, v1) VALUES ('0', 0);");
-        IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
-        waitForTableIndexesQueryable();
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
         flush();
-        verifyIndexFiles(indexTermType, indexIdentifier, 1, 1, 1);
+        verifyIndexFiles(numericIndexContext, null, 1, 1, 0, 1, 0);
         // Note: This test will fail here if it is run on its own because the per-index validation
         // is run if the node is starting up but validatation isn't done once the node is started
         assertValidationCount(0, 0);
@@ -65,7 +62,7 @@ public void shouldTakeAndRestoreSnapshots() throws Throwable
         // Add some data into a second sstable
         execute("INSERT INTO %s (id1, v1) VALUES ('1', 0);");
         flush();
-        verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2);
+        verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0);
         assertValidationCount(0, 0);
 
         // Take a snapshot recording the index files last modified date
@@ -81,19 +78,19 @@ public void shouldTakeAndRestoreSnapshots() throws Throwable
         // Add some data into a third sstable, out of the scope of our snapshot
         execute("INSERT INTO %s (id1, v1) VALUES ('2', 0);");
         flush();
-        verifyIndexFiles(indexTermType, indexIdentifier, 3, 3, 3);
+        verifyIndexFiles(numericIndexContext, null, 3, 3, 0, 3, 0);
         assertNumRows(3, "SELECT * FROM %%s WHERE v1 >= 0");
         assertValidationCount(0, 0);
 
         // Truncate the table
         truncate(false);
-        waitForAssert(() -> verifyNoIndexFiles());
+        waitForAssert(this::verifyNoIndexFiles);
         assertNumRows(0, "SELECT * FROM %%s WHERE v1 >= 0");
         assertValidationCount(0, 0);
 
         // Restore the snapshot, only the two first sstables should be restored
         restoreSnapshot(snapshot);
-        verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2);
+        verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0);
         assertEquals(snapshotLastModified, indexFilesLastModified());
         assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0");
         assertValidationCount(2, 2); // newly loaded
@@ -102,11 +99,14 @@ public void shouldTakeAndRestoreSnapshots() throws Throwable
         verifyIndexComponentsIncludedInSSTable();
 
         // Rebuild the index to verify that the index files are overridden
-        rebuildIndexes(indexIdentifier.indexName);
-        verifyIndexFiles(indexTermType, indexIdentifier, 2);
+        rebuildIndexes(numericIndexContext.getIndexName());
+        verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0);
         assertNotEquals(snapshotLastModified, indexFilesLastModified());
         assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0");
-        assertValidationCount(2, 2); // compaction should not validate
+        assertValidationCount(2, 2);
+
+        verifyIndexComponentFiles(numericIndexContext, null);
+        assertValidationCount(4, 4);
 
         // index components are included after rebuild
         verifyIndexComponentsIncludedInSSTable();
@@ -130,10 +130,8 @@ public void shouldSnapshotAfterIndexBuild() throws Throwable
         verifyIndexComponentsNotIncludedInSSTable();
 
         // create index
-        IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
-        IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
-        waitForTableIndexesQueryable();
-        verifyIndexFiles(indexTermType, indexIdentifier, 2);
+        IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
+        verifyIndexFiles(numericIndexContext, null, 2, 0);
         assertValidationCount(0, 0);
 
         // index components are included after initial build
@@ -151,13 +149,13 @@ public void shouldSnapshotAfterIndexBuild() throws Throwable
 
         // Truncate the table
         truncate(false);
-        waitForAssert(() -> verifyNoIndexFiles());
+        waitForAssert(this::verifyNoIndexFiles);
         assertNumRows(0, "SELECT * FROM %%s WHERE v1 >= 0");
         assertValidationCount(0, 0);
 
         // Restore the snapshot
         restoreSnapshot(snapshot);
-        verifyIndexFiles(indexTermType, indexIdentifier, 2);
+        verifyIndexFiles(numericIndexContext, null, 2, 0);
         assertEquals(snapshotLastModified, indexFilesLastModified());
         assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0");
         assertValidationCount(2, 2); // newly loaded
diff --git a/test/unit/org/apache/cassandra/index/sai/iterators/AbstractKeyRangeIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/iterators/AbstractKeyRangeIteratorTest.java
new file mode 100644
index 000000000000..a43afe62a7ea
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/iterators/AbstractKeyRangeIteratorTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.iterators;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.Assert;
+
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+import org.apache.cassandra.utils.Pair;
+
+public class AbstractKeyRangeIteratorTest extends SaiRandomizedTest
+{
+    protected long[] arr(long... longArray)
+    {
+        return longArray;
+    }
+
+    protected long[] arr(int... intArray)
+    {
+        return Arrays.stream(intArray).mapToLong(i -> i).toArray();
+    }
+
+    final KeyRangeIterator buildIntersection(KeyRangeIterator... ranges)
+    {
+        return KeyRangeIntersectionIterator.<PrimaryKey>builder().add(Arrays.asList(ranges)).build();
+    }
+
+    final KeyRangeIterator buildSelectiveIntersection(int limit, KeyRangeIterator... ranges)
+    {
+        return KeyRangeIntersectionIterator.<PrimaryKey>builder(limit).add(Arrays.asList(ranges)).build();
+    }
+
+    final KeyRangeIterator buildIntersection(long[]... ranges)
+    {
+        return buildIntersection(toRangeIterator(ranges));
+    }
+
+    final KeyRangeIterator buildSelectiveIntersection(int limit, long[]... ranges)
+    {
+        return buildSelectiveIntersection(limit, toRangeIterator(ranges));
+    }
+
+    static KeyRangeIterator buildUnion(KeyRangeIterator... ranges)
+    {
+        return KeyRangeUnionIterator.<PrimaryKey>builder().add(Arrays.asList(ranges)).build();
+    }
+
+    static KeyRangeIterator buildUnion(long[]... ranges)
+    {
+        return buildUnion(toRangeIterator(ranges));
+    }
+
+    static KeyRangeIterator buildConcat(KeyRangeIterator... ranges)
+    {
+        return KeyRangeConcatIterator.builder(ranges.length).add(Arrays.asList(ranges)).build();
+    }
+
+    static KeyRangeIterator buildConcat(long[]... ranges)
+    {
+        return buildConcat(toRangeIterator(ranges));
+    }
+
+    private static KeyRangeIterator[] toRangeIterator(long[]... ranges)
+    {
+        return Arrays.stream(ranges).map(AbstractKeyRangeIteratorTest::build).toArray(KeyRangeIterator[]::new);
+    }
+
+    protected static LongIterator build(long... tokens)
+    {
+        return new LongIterator(tokens);
+    }
+
+    protected KeyRangeIterator build(KeyRangeIterator.Builder.IteratorType type, long[] tokensA, long[] tokensB)
+    {
+        KeyRangeIterator rangeA = new LongIterator(tokensA);
+        KeyRangeIterator rangeB = new LongIterator(tokensB);
+
+        switch (type)
+        {
+            case INTERSECTION:
+                return buildIntersection(rangeA, rangeB);
+            case UNION:
+                return buildUnion(rangeA, rangeB);
+            case CONCAT:
+                return buildConcat(rangeA, rangeB);
+            default:
+                throw new IllegalArgumentException("unknown type: " + type);
+        }
+    }
+
+    static void validateWithSkipping(KeyRangeIterator ri, long[] totalOrdering)
+    {
+        int count = 0;
+        while (ri.hasNext())
+        {
+            // make sure hasNext plays nice with skipTo
+            if (randomBoolean())
+                ri.hasNext();
+
+            // skipping to the same element should also be a no-op
+            if (randomBoolean())
+                ri.skipTo(LongIterator.fromToken(totalOrdering[count]));
+
+            // skip a few elements
+            if (nextDouble() < 0.1)
+            {
+                int n = nextInt(1, 3);
+                if (count + n < totalOrdering.length)
+                {
+                    count += n;
+                    ri.skipTo(LongIterator.fromToken(totalOrdering[count]));
+                }
+            }
+            Assert.assertEquals(totalOrdering[count++], ri.next().token().getLongValue());
+        }
+        Assert.assertEquals(totalOrdering.length, count);
+    }
+
+    static Set<Long> toSet(long[] tokens)
+    {
+        return Arrays.stream(tokens).boxed().collect(Collectors.toSet());
+    }
+
+    /**
+     * @return a random {Concat,Intersection, Union} iterator, and a long[] of the elements in the iterator.
+     *         elements will range from 0..1024.
+     */
+    static Pair<KeyRangeIterator, long[]> createRandomIterator()
+    {
+        var n = randomIntBetween(0, 3);
+        switch (n)
+        {
+            case 0:
+                return KeyRangeConcatIteratorTest.createRandom();
+            case 1:
+                return KeyRangeIntersectionIteratorTest.createRandom(nextInt(1, 16));
+            case 2:
+                return KeyRangeUnionIteratorTest.createRandom(nextInt(1, 16));
+            default:
+                throw new AssertionError();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/iterators/AbstractKeyRangeIteratorTester.java b/test/unit/org/apache/cassandra/index/sai/iterators/AbstractKeyRangeIteratorTester.java
deleted file mode 100644
index 01ae628aeb2c..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/iterators/AbstractKeyRangeIteratorTester.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.iterators;
-
-import java.util.Arrays;
-import java.util.Set;
-import java.util.function.BiFunction;
-import java.util.stream.Collectors;
-
-import org.junit.Assert;
-
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-import org.apache.cassandra.utils.Pair;
-
-public class AbstractKeyRangeIteratorTester extends SAIRandomizedTester
-{
-    protected long[] arr(long... longArray)
-    {
-        return longArray;
-    }
-
-    protected long[] arr(int... intArray)
-    {
-        return Arrays.stream(intArray).mapToLong(i -> i).toArray();
-    }
-
-    final KeyRangeIterator buildIntersection(KeyRangeIterator... ranges)
-    {
-        return KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE).add(Arrays.asList(ranges)).build();
-    }
-
-    final KeyRangeIterator buildSelectiveIntersection(int limit, KeyRangeIterator... ranges)
-    {
-        return KeyRangeIntersectionIterator.builder(16, limit).add(Arrays.asList(ranges)).build();
-    }
-
-    final KeyRangeIterator buildIntersection(long[]... ranges)
-    {
-        return buildIntersection(toRangeIterator(ranges));
-    }
-
-    final KeyRangeIterator buildSelectiveIntersection(int limit, long[]... ranges)
-    {
-        return buildSelectiveIntersection(limit, toRangeIterator(ranges));
-    }
-
-    final KeyRangeIterator buildUnion(KeyRangeIterator... ranges)
-    {
-        return KeyRangeUnionIterator.builder(ranges.length).add(Arrays.asList(ranges)).build();
-    }
-
-    static KeyRangeIterator buildConcat(KeyRangeIterator... ranges)
-    {
-        return KeyRangeConcatIterator.builder(ranges.length).add(Arrays.asList(ranges)).build();
-    }
-    private static KeyRangeIterator[] toRangeIterator(long[]... ranges)
-    {
-        return Arrays.stream(ranges).map(AbstractKeyRangeIteratorTester::build).toArray(KeyRangeIterator[]::new);
-    }
-
-    protected static LongIterator build(long... tokens)
-    {
-        return new LongIterator(tokens);
-    }
-
-    protected KeyRangeIterator build(BiFunction<KeyRangeIterator, KeyRangeIterator, KeyRangeIterator> builder,
-                                     long[] tokensA,
-                                     long[] tokensB)
-    {
-        return builder.apply(build(tokensA), build(tokensB));
-    }
-
-    static void validateWithSkipping(KeyRangeIterator ri, long[] totalOrdering)
-    {
-        int count = 0;
-        while (ri.hasNext())
-        {
-            // make sure hasNext plays nice with skipTo
-            if (nextBoolean())
-                ri.hasNext();
-
-            // skipping to the same element should also be a no-op
-            if (nextBoolean())
-                ri.skipTo(LongIterator.fromToken(totalOrdering[count]));
-
-            // skip a few elements
-            if (nextDouble() < 0.1)
-            {
-                int n = nextInt(1, 3);
-                if (count + n < totalOrdering.length)
-                {
-                    count += n;
-                    ri.skipTo(LongIterator.fromToken(totalOrdering[count]));
-                }
-            }
-            Assert.assertEquals(totalOrdering[count++], ri.next().token().getLongValue());
-        }
-        Assert.assertEquals(totalOrdering.length, count);
-    }
-
-    static Set<Long> toSet(long[] tokens)
-    {
-        return Arrays.stream(tokens).boxed().collect(Collectors.toSet());
-    }
-
-    /**
-     * @return a random {Concat,Intersection, Union} iterator, and a long[] of the elements in the iterator.
-     *         elements will range from 0..1024.
-     */
-    static Pair<KeyRangeIterator, long[]> createRandomIterator()
-    {
-        var n = between(0, 3);
-        switch (n)
-        {
-            case 0:
-                return KeyRangeConcatIteratorTest.createRandom();
-            case 1:
-                return KeyRangeIntersectionIteratorTest.createRandom(nextInt(1, 16));
-            case 2:
-                return KeyRangeUnionIteratorTest.createRandom(nextInt(1, 16));
-            default:
-                throw new AssertionError();
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeAntiJoinIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeAntiJoinIteratorTest.java
new file mode 100644
index 000000000000..dbedc5827792
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeAntiJoinIteratorTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.iterators;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class KeyRangeAntiJoinIteratorTest
+{
+    public static final long[] EMPTY = { };
+
+    @Test
+    public void testEmpty()
+    {
+        LongIterator left = new LongIterator(EMPTY);
+        LongIterator right = new LongIterator(EMPTY);
+        KeyRangeAntiJoinIterator iter = KeyRangeAntiJoinIterator.create(left, right);
+        assertEquals(convert(), convert(iter));
+    }
+
+    @Test
+    public void testEmptyLeft()
+    {
+        LongIterator left = new LongIterator(EMPTY);
+        LongIterator right = new LongIterator(new long[] { 1L, 3L, 5L, 7L });
+        KeyRangeAntiJoinIterator iter = KeyRangeAntiJoinIterator.create(left, right);
+        assertEquals(convert(), convert(iter));
+    }
+
+    @Test
+    public void testEmptyRight()
+    {
+        LongIterator left = new LongIterator(new long[] { 1L, 2L });
+        LongIterator right = new LongIterator(EMPTY);
+        KeyRangeAntiJoinIterator iter = KeyRangeAntiJoinIterator.create(left, right);
+        assertEquals(convert(1, 2), convert(iter));
+    }
+
+    @Test
+    public void testNoOverlappingValues()
+    {
+        LongIterator left = new LongIterator(new long[] { 2L, 4L, 6L, 8L });
+        LongIterator right = new LongIterator(new long[] { 1L, 3L, 5L, 7L });
+        KeyRangeAntiJoinIterator iter = KeyRangeAntiJoinIterator.create(left, right);
+        assertEquals(convert(2, 4, 6, 8), convert(iter));
+    }
+
+    @Test
+    public void testOverlappingValues()
+    {
+        LongIterator left = new LongIterator(new long[] { 2L, 3L, 4L, 6L, 8L, 9L, 10L });
+        LongIterator right = new LongIterator(new long[] { 4L, 8L, 9L });
+        KeyRangeAntiJoinIterator iter = KeyRangeAntiJoinIterator.create(left, right);
+        assertEquals(convert(2, 3, 6, 10), convert(iter));
+    }
+
+    @Test
+    public void testOverlappingPrefixes()
+    {
+        LongIterator left = new LongIterator(new long[] { 2L, 3L, 4L, 6L, 8L, 9L, 10L });
+        LongIterator right = new LongIterator(new long[] { 2L, 3L, 4L });
+        KeyRangeAntiJoinIterator iter = KeyRangeAntiJoinIterator.create(left, right);
+        assertEquals(convert(6, 8, 9, 10), convert(iter));
+    }
+
+    public static List<Long> convert(KeyRangeIterator tokens)
+    {
+        List<Long> results = new ArrayList<>();
+        while (tokens.hasNext())
+            results.add(tokens.next().token().getLongValue());
+
+        return results;
+    }
+
+    public static List<Long> convert(final long... nums)
+    {
+        return new ArrayList<>(nums.length)
+        {{
+            for (long n : nums)
+                add(n);
+        }};
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeConcatIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeConcatIteratorTest.java
index 12e36c3f408c..033fa02d718a 100644
--- a/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeConcatIteratorTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeConcatIteratorTest.java
@@ -23,44 +23,56 @@
 
 import org.junit.Test;
 
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.utils.Pair;
 
 import static org.apache.cassandra.index.sai.iterators.LongIterator.convert;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-
-public class KeyRangeConcatIteratorTest extends AbstractKeyRangeIteratorTester
+
+public class KeyRangeConcatIteratorTest extends AbstractKeyRangeIteratorTest
 {
-    PrimaryKey.Factory primaryKeyFactory = new PrimaryKey.Factory(Murmur3Partitioner.instance, null);
     @Test
     public void testValidation()
     {
-        // Iterators being merged via concatanation must not include each other
-        assertThatThrownBy(() -> buildConcat(build(1L, 4L), build(2L, 3L))).isInstanceOf(IllegalArgumentException.class)
-                                                                           .hasMessage(createErrorMessage(4, 2));
-
-        // Iterators being merged via concatanation must not overlap
-        assertThatThrownBy(() -> buildConcat(build(1L, 4L), build(2L, 5L))).isInstanceOf(IllegalArgumentException.class)
-                                                                           .hasMessage(createErrorMessage(4, 2));
-
-        assertThatThrownBy(() -> buildConcat(build(1L, 4L), build(0L, 3L))).isInstanceOf(IllegalArgumentException.class)
-                                                                           .hasMessage(createErrorMessage(4, 0));
+        try
+        {
+            buildConcat(build(1L, 4L), build(2L, 3L));
+            fail("Flows for a merging concatenation must not contain one another.");
+        }
+        catch (IllegalArgumentException ignored)
+        {
+        }
 
-        // Iterators being merged via concatanation must be sorted
-        assertThatThrownBy(() -> buildConcat(build(2L, 4L), build(0L, 1L))).isInstanceOf(IllegalArgumentException.class)
-                                                                           .hasMessage(createErrorMessage(4, 0));
+        try
+        {
+            buildConcat(build(1L, 4L), build(2L, 5L));
+            fail("Minimum for flow must not be included in exclusive range of previous flow.");
+        }
+        catch (IllegalArgumentException ignored)
+        {
+        }
 
         // allow min boundary included
         KeyRangeIterator concat = buildConcat(build(1L, 4L), build(4L, 5L));
         assertEquals(convert(1L, 4L, 4L, 5L), convert(concat));
 
-        // with empty iterator
+        try
+        {
+            buildConcat(build(1L, 4L), build(0L, 3L));
+            fail("Maximum for flow must not be included in exclusive range of previous flow.");
+        }
+        catch (IllegalArgumentException ignored)
+        {
+        }
+
+        try
+        {
+            buildConcat(build(2L, 4L), build(0L, 1L));
+            fail("Flows for merging concatenation must be sorted.");
+        }
+        catch (IllegalArgumentException ignored)
+        {
+        }
+
+        // with empty flow
         concat = buildConcat(build(), build(0L, 1L));
         assertEquals(convert(0L, 1L), convert(concat));
 
@@ -104,77 +116,79 @@ public void testMinMaxAndCount()
         builder.add(build(7L, 8L, 9L));
 
         assertEquals(9L, builder.getMaximum().token().getLongValue());
-        assertEquals(9L, builder.getCount());
+        assertEquals(9L, builder.getTokenCount());
 
-        KeyRangeIterator keyIterator = builder.build();
+        KeyRangeIterator tokens = builder.build();
 
-        assertNotNull(keyIterator);
-        assertEquals(1L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(9L, keyIterator.getMaximum().token().getLongValue());
-        assertEquals(9L, keyIterator.getMaxKeys());
+        assertNotNull(tokens);
+        assertEquals(1L, tokens.getMinimum().token().getLongValue());
+        assertEquals(9L, tokens.getMaximum().token().getLongValue());
+        assertEquals(9L, tokens.getMaxKeys());
 
         for (long i = 1; i < 10; i++)
         {
-            assertTrue(keyIterator.hasNext());
-            assertEquals(i, keyIterator.next().token().getLongValue());
+            assertTrue(tokens.hasNext());
+            assertEquals(i, tokens.next().token().getLongValue());
         }
 
-        assertFalse(keyIterator.hasNext());
-        assertEquals(1L, keyIterator.getMinimum().token().getLongValue());
+        assertFalse(tokens.hasNext());
+        assertEquals(1L, tokens.getMinimum().token().getLongValue());
     }
 
     @Test
     public void testSkipTo()
     {
+        // flow is single use..
         Supplier<KeyRangeIterator> init = () ->  buildConcat(build(1L, 2L, 3L),
                                                              build( 4L, 5L, 6L),
                                                              build( 7L, 8L, 9L));
 
-        KeyRangeIterator keyIterator;
-
-        keyIterator = init.get();
-        keyIterator.skipTo(LongIterator.fromToken(5));
-        assertTrue(keyIterator.hasNext());
-        assertEquals(5L, keyIterator.next().token().getLongValue());
-
-        keyIterator = init.get();
-        keyIterator.skipTo(LongIterator.fromToken(7L));
-        assertTrue(keyIterator.hasNext());
-        assertEquals(7L, keyIterator.next().token().getLongValue());
-
-        keyIterator = init.get();
-        keyIterator.skipTo(LongIterator.fromToken(2L));
-        keyIterator.skipTo(LongIterator.fromToken(5L));
-        keyIterator.skipTo(LongIterator.fromToken(10L));
-        assertFalse(keyIterator.hasNext());
-        assertEquals(1L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(9L, keyIterator.getMaximum().token().getLongValue());
+        KeyRangeIterator tokens;
+
+        tokens = init.get();
+        tokens.skipTo(LongIterator.fromToken(5));
+        assertTrue(tokens.hasNext());
+        assertEquals(5L, tokens.next().token().getLongValue());
+
+        tokens = init.get();
+        tokens.skipTo(LongIterator.fromToken(7L));
+        assertTrue(tokens.hasNext());
+        assertEquals(7L, tokens.next().token().getLongValue());
+
+        tokens = init.get();
+        tokens.skipTo(LongIterator.fromToken(2L));
+        tokens.skipTo(LongIterator.fromToken(5L));
+        tokens.skipTo(LongIterator.fromToken(10L));
+        assertFalse(tokens.hasNext());
+        assertEquals(1L, tokens.getMinimum().token().getLongValue());
+        assertEquals(9L, tokens.getMaximum().token().getLongValue());
     }
 
     @Test
     public void testSkipToWithGaps()
     {
+        // flow is single use..
         Supplier<KeyRangeIterator> init = () ->  buildConcat(build(1L, 2L, 3L), build(4L, 6L), build(8L, 9L));
 
-        KeyRangeIterator keyIterator;
-
-        keyIterator = init.get();
-        keyIterator.skipTo(LongIterator.fromToken(5L));
-        assertTrue(keyIterator.hasNext());
-        assertEquals(6L, keyIterator.next().token().getLongValue());
-
-        keyIterator = init.get();
-        keyIterator.skipTo(LongIterator.fromToken(7L));
-        assertTrue(keyIterator.hasNext());
-        assertEquals(8L, keyIterator.next().token().getLongValue());
-
-        keyIterator = init.get();
-        keyIterator.skipTo(LongIterator.fromToken(2L));
-        keyIterator.skipTo(LongIterator.fromToken(5L));
-        keyIterator.skipTo(LongIterator.fromToken(10L));
-        assertFalse(keyIterator.hasNext());
-        assertEquals(1L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(9L, keyIterator.getMaximum().token().getLongValue());
+        KeyRangeIterator tokens;
+
+        tokens = init.get();
+        tokens.skipTo(LongIterator.fromToken(5L));
+        assertTrue(tokens.hasNext());
+        assertEquals(6L, tokens.next().token().getLongValue());
+
+        tokens = init.get();
+        tokens.skipTo(LongIterator.fromToken(7L));
+        assertTrue(tokens.hasNext());
+        assertEquals(8L, tokens.next().token().getLongValue());
+
+        tokens = init.get();
+        tokens.skipTo(LongIterator.fromToken(2L));
+        tokens.skipTo(LongIterator.fromToken(5L));
+        tokens.skipTo(LongIterator.fromToken(10L));
+        assertFalse(tokens.hasNext());
+        assertEquals(1L, tokens.getMinimum().token().getLongValue());
+        assertEquals(9L, tokens.getMaximum().token().getLongValue());
     }
 
     @Test
@@ -194,11 +208,12 @@ public void testEmptyThenManyNonEmpty()
         builder.add(build());
         IntStream.range(10, 20).forEach(value -> builder.add(build(value)));
 
-        KeyRangeIterator keyIterator = builder.build();
-        assertEquals(10L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(19L, keyIterator.getMaximum().token().getLongValue());
-        assertTrue(keyIterator.hasNext());
-        assertEquals(10, keyIterator.getMaxKeys());
+        KeyRangeIterator range = builder.build();
+
+        assertEquals(10L, range.getMinimum().token().getLongValue());
+        assertEquals(19L, range.getMaximum().token().getLongValue());
+        assertTrue(range.hasNext());
+        assertEquals(10, range.getMaxKeys());
     }
 
     @Test
@@ -209,11 +224,11 @@ public void testEmptyThenSingleNonEmpty()
         builder.add(build());
         builder.add(build(10));
 
-        KeyRangeIterator keyIterator = builder.build();
-        assertEquals(10L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(10L, keyIterator.getMaximum().token().getLongValue());
-        assertTrue(keyIterator.hasNext());
-        assertEquals(1, keyIterator.getMaxKeys());
+        KeyRangeIterator range = builder.build();
+        assertEquals(10L, range.getMinimum().token().getLongValue());
+        assertEquals(10L, range.getMaximum().token().getLongValue());
+        assertTrue(range.hasNext());
+        assertEquals(1, range.getMaxKeys());
     }
 
     @Test
@@ -224,11 +239,11 @@ public void testManyNonEmptyThenEmpty()
         IntStream.range(10, 20).forEach(value -> builder.add(build(value)));
         builder.add(build());
 
-        KeyRangeIterator keyIterator = builder.build();
-        assertEquals(10L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(19L, keyIterator.getMaximum().token().getLongValue());
-        assertTrue(keyIterator.hasNext());
-        assertEquals(10, keyIterator.getMaxKeys());
+        KeyRangeIterator range = builder.build();
+        assertEquals(10L, range.getMinimum().token().getLongValue());
+        assertEquals(19L, range.getMaximum().token().getLongValue());
+        assertTrue(range.hasNext());
+        assertEquals(10, range.getMaxKeys());
     }
 
     @Test
@@ -239,11 +254,11 @@ public void testSingleNonEmptyThenEmpty()
         builder.add(build(10));
         builder.add(build());
 
-        KeyRangeIterator keyIterator = builder.build();
-        assertEquals(10L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(10L, keyIterator.getMaximum().token().getLongValue());
-        assertTrue(keyIterator.hasNext());
-        assertEquals(1, keyIterator.getMaxKeys());
+        KeyRangeIterator range = builder.build();
+        assertEquals(10L, range.getMinimum().token().getLongValue());
+        assertEquals(10L, range.getMaximum().token().getLongValue());
+        assertTrue(range.hasNext());
+        assertEquals(1, range.getMaxKeys());
     }
 
     @Test
@@ -255,11 +270,11 @@ public void testEmptyNonEmptyEmpty()
         IntStream.range(10, 20).forEach(value -> builder.add(build(value)));
         builder.add(build());
 
-        KeyRangeIterator keyIterator = builder.build();
-        assertEquals(10L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(19L, keyIterator.getMaximum().token().getLongValue());
-        assertTrue(keyIterator.hasNext());
-        assertEquals(10, keyIterator.getMaxKeys());
+        KeyRangeIterator range = builder.build();
+        assertEquals(10L, range.getMinimum().token().getLongValue());
+        assertEquals(19L, range.getMaximum().token().getLongValue());
+        assertTrue(range.hasNext());
+        assertEquals(10, range.getMaxKeys());
     }
 
     @Test
@@ -271,11 +286,11 @@ public void testNonEmptyEmptyNonEmpty()
         builder.add(build());
         IntStream.range(15, 20).forEach(value -> builder.add(build(value)));
 
-        KeyRangeIterator keyIterator = builder.build();
-        assertEquals(10L, keyIterator.getMinimum().token().getLongValue());
-        assertEquals(19L, keyIterator.getMaximum().token().getLongValue());
-        assertTrue(keyIterator.hasNext());
-        assertEquals(10, keyIterator.getMaxKeys());
+        KeyRangeIterator range = builder.build();
+        assertEquals(10L, range.getMinimum().token().getLongValue());
+        assertEquals(19L, range.getMaximum().token().getLongValue());
+        assertTrue(range.hasNext());
+        assertEquals(10, range.getMaxKeys());
     }
 
     @Test
@@ -321,9 +336,9 @@ public void testIntersectionOfConcat()
     }
 
     @Test
-    public void testDuplicatedElementsInTheSameIterator()
+    public void testDuplicatedElementsInTheSameFlow()
     {
-        // In real case, we should not have duplicated elements from the same PostingListRangeIterator
+        // In real case, we should not have duplicated elements from the same PostingListKeyRangeIterator
         KeyRangeIterator rangeA = build(1L, 2L, 3L, 3L, 4L, 4L);
         KeyRangeIterator rangeB = build(6L, 6L, 7L, 7L);
         KeyRangeIterator rangeC = build(8L, 8L);
@@ -384,18 +399,6 @@ public void testDuplicateElementsAtBoundary()
         assertEquals(convert(1L, 2L, 3L, 3L, 3L, 3L, 4L, 5L), convert(buildConcat(rangeA, rangeB)));
     }
 
-    private KeyRangeIterator.Builder getConcatBuilder()
-    {
-        return KeyRangeConcatIterator.builder(16);
-    }
-
-    private String createErrorMessage(int max, int min)
-    {
-        return String.format(KeyRangeConcatIterator.MUST_BE_SORTED_ERROR,
-                             primaryKeyFactory.create(new Murmur3Partitioner.LongToken(max)),
-                             primaryKeyFactory.create(new Murmur3Partitioner.LongToken(min)));
-    }
-
     @Test
     public void testRandom()
     {
@@ -416,12 +419,12 @@ static Pair<KeyRangeIterator, long[]> createRandom()
         {
             allValues.add((long) i);
             current.add((long) i);
-            if (nextDouble() < 0.05)
+            if (randomDouble() < 0.05)
             {
                 ranges.add(build(current.stream().mapToLong(Long::longValue).toArray()));
                 current.clear();
             }
-            if (nextDouble() < 0.1)
+            if (randomDouble() < 0.1)
                 i += nextInt(5);
         }
         ranges.add(build(current.stream().mapToLong(Long::longValue).toArray()));
@@ -431,4 +434,9 @@ static Pair<KeyRangeIterator, long[]> createRandom()
         assertEquals(totalOrdered.length, it.getMaxKeys());
         return Pair.create(it, totalOrdered);
     }
+
+    private KeyRangeIterator.Builder getConcatBuilder()
+    {
+        return KeyRangeConcatIterator.builder();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeIntersectionIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeIntersectionIteratorTest.java
index ee54fd69039d..0d8bcd365ede 100644
--- a/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeIntersectionIteratorTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeIntersectionIteratorTest.java
@@ -21,6 +21,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.IntStream;
 
 import org.junit.Assert;
@@ -28,94 +29,93 @@
 
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongSet;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.Pair;
 
 import static org.apache.cassandra.index.sai.iterators.LongIterator.convert;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
 
-public class KeyRangeIntersectionIteratorTest extends AbstractKeyRangeIteratorTester
+public class KeyRangeIntersectionIteratorTest extends AbstractKeyRangeIteratorTest
 {
     @Test
     public void testNoOverlappingValues()
     {
-        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 2L, 3L, 5L, 6L }));
         builder.add(new LongIterator(new long[] { 1L, 7L }));
         builder.add(new LongIterator(new long[] { 4L, 8L, 9L, 10L }));
 
-        assertEquals(convert(), convert(builder.build()));
+        Assert.assertEquals(convert(), convert(builder.build()));
 
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        builder = KeyRangeIntersectionIterator.builder();
         // both ranges overlap by min/max but not by value
         builder.add(new LongIterator(new long[] { 1L, 5L, 7L, 9L }));
         builder.add(new LongIterator(new long[] { 6L }));
 
         KeyRangeIterator range = builder.build();
 
-        assertNotNull(range);
-        assertFalse(range.hasNext());
+        Assert.assertNotNull(range);
+        Assert.assertFalse(range.hasNext());
 
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        builder = KeyRangeIntersectionIterator.builder();
         // both ranges overlap by min/max but not by value
         builder.add(new LongIterator(new long[] { 1L, 5L, 7L, 9L }));
         builder.add(new LongIterator(new long[] { 0L, 10L, 12L }));
 
         range = builder.build();
 
-        assertNotNull(range);
-        assertFalse(range.hasNext());
+        Assert.assertNotNull(range);
+        Assert.assertFalse(range.hasNext());
     }
 
     @Test
     public void testOverlappingValues()
     {
-        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L, 4L, 6L, 7L }));
         builder.add(new LongIterator(new long[] { 2L, 4L, 5L, 6L }));
         builder.add(new LongIterator(new long[] { 4L, 6L, 8L, 9L, 10L }));
 
-        assertEquals(convert(4L, 6L), convert(builder.build()));
+        Assert.assertEquals(convert(4L, 6L), convert(builder.build()));
     }
 
     @Test
     public void testSameValues()
     {
-        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L, 2L, 3L, 4L }));
         builder.add(new LongIterator(new long[] { 1L, 2L, 3L, 4L }));
 
-        assertEquals(convert(1L, 2L, 3L, 4L), convert(builder.build()));
+        Assert.assertEquals(convert(1L, 2L, 3L, 4L), convert(builder.build()));
     }
 
     @Test
     public void testSingleIterator()
     {
-        KeyRangeIntersectionIterator.Builder builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        KeyRangeIntersectionIterator.Builder builder = KeyRangeIntersectionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L, 2L, 4L, 9L }));
 
-        assertEquals(convert(1L, 2L, 4L, 9L), convert(builder.build()));
+        KeyRangeIterator range = builder.build();
+        // no need to wrap single input iterator in an intersection
+        Assert.assertTrue("Single iterator wrapped in KeyRangeIntersectionIterator", range instanceof LongIterator);
+        Assert.assertEquals(convert(1L, 2L, 4L, 9L), convert(range));
     }
 
     @Test
     public void testSkipTo()
     {
-        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L, 4L, 6L, 7L, 9L, 10L }));
         builder.add(new LongIterator(new long[] { 2L, 4L, 5L, 6L, 7L, 10L, 12L }));
         builder.add(new LongIterator(new long[] { 4L, 6L, 7L, 9L, 10L }));
 
         KeyRangeIterator range = builder.build();
-        assertNotNull(range);
+        Assert.assertNotNull(range);
 
         // first let's skipTo something before range
         range.skipTo(LongIterator.fromToken(3L));
@@ -128,27 +128,27 @@ public void testSkipTo()
         // now right to the element
         range.skipTo(LongIterator.fromToken(7L));
         Assert.assertEquals(7L, range.peek().token().getLongValue());
-        assertEquals(7L, range.next().token().getLongValue());
+        Assert.assertEquals(7L, range.next().token().getLongValue());
 
-        assertTrue(range.hasNext());
+        Assert.assertTrue(range.hasNext());
         Assert.assertEquals(10L, range.peek().token().getLongValue());
 
         // now right after the last element
         range.skipTo(LongIterator.fromToken(11L));
-        assertFalse(range.hasNext());
+        Assert.assertFalse(range.hasNext());
     }
 
     @Test
     public void testMinMaxAndCount()
     {
-        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder();
 
         builder.add(new LongIterator(new long[]{1L, 2L, 9L}));
         builder.add(new LongIterator(new long[]{4L, 5L, 9L}));
         builder.add(new LongIterator(new long[]{7L, 8L, 9L}));
 
         assertEquals(9L, builder.getMaximum().token().getLongValue());
-        assertEquals(3L, builder.getCount());
+        assertEquals(3L, builder.getTokenCount());
 
         KeyRangeIterator tokens = builder.build();
 
@@ -157,18 +157,18 @@ public void testMinMaxAndCount()
         assertEquals(9L, tokens.getMaximum().token().getLongValue());
         assertEquals(3L, tokens.getMaxKeys());
 
-        assertEquals(convert(9L), convert(builder.build()));
+        Assert.assertEquals(convert(9L), convert(builder.build()));
     }
 
     @Test
     public void testBuilder()
     {
-        KeyRangeIntersectionIterator.Builder builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        KeyRangeIntersectionIterator.Builder builder = KeyRangeIntersectionIterator.builder();
 
-        assertNull(builder.getMinimum());
-        assertNull(builder.getMaximum());
-        assertEquals(0L, builder.getCount());
-        assertEquals(0L, builder.rangeCount());
+        Assert.assertNull(builder.getMinimum());
+        Assert.assertNull(builder.getMaximum());
+        Assert.assertEquals(0L, builder.getTokenCount());
+        Assert.assertEquals(0L, builder.rangeCount());
 
         builder.add(new LongIterator(new long[] { 1L, 2L, 6L }));
         builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
@@ -176,64 +176,83 @@ public void testBuilder()
 
         assertEquals(6L, builder.getMinimum().token().getLongValue());
         assertEquals(6L, builder.getMaximum().token().getLongValue());
-        assertEquals(3L, builder.getCount());
+        assertEquals(3L, builder.getTokenCount());
         assertEquals(3L, builder.rangeCount());
-        assertFalse(builder.isDisjoint());
+        assertFalse(builder.statistics.isEmptyOrDisjoint());
 
-        assertEquals(1L, builder.rangeIterators.get(0).getMinimum().token().getLongValue());
-        assertEquals(4L, builder.rangeIterators.get(1).getMinimum().token().getLongValue());
-        assertEquals(6L, builder.rangeIterators.get(2).getMinimum().token().getLongValue());
+        Assert.assertEquals(1L, builder.rangeIterators.get(0).getMinimum().token().getLongValue());
+        Assert.assertEquals(4L, builder.rangeIterators.get(1).getMinimum().token().getLongValue());
+        Assert.assertEquals(6L, builder.rangeIterators.get(2).getMinimum().token().getLongValue());
 
         builder.add(new LongIterator(new long[] { 1L, 2L, 6L }));
         builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
         builder.add(new LongIterator(new long[] { 6L, 8L, 9L }));
 
-        assertEquals(convert(6L), convert(builder.build()));
+        Assert.assertEquals(convert(6L), convert(builder.build()));
 
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        builder = KeyRangeIntersectionIterator.builder();
         builder.add(new LongIterator(new long[]{ 1L, 5L, 6L }));
         builder.add(new LongIterator(new long[]{ 3L, 5L, 6L }));
 
-        KeyRangeIterator tokens = builder.build();
+        var tokens = builder.build();
 
-        assertEquals(convert(5L, 6L), convert(tokens));
+        Assert.assertEquals(convert(5L, 6L), convert(tokens));
 
         FileUtils.closeQuietly(tokens);
 
-        KeyRangeIterator emptyTokens = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE).build();
-        assertEquals(0, emptyTokens.getMaxKeys());
+        var emptyTokens = KeyRangeIntersectionIterator.builder().build();
+        Assert.assertEquals(0, emptyTokens.getMaxKeys());
 
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
-        assertEquals(0L, builder.add((KeyRangeIterator) null).rangeCount());
-        assertEquals(0L, builder.add((List<KeyRangeIterator>) null).getCount());
-        assertEquals(0L, builder.add(LongIterator.newEmptyIterator()).rangeCount());
+        builder = KeyRangeIntersectionIterator.builder();
+        Assert.assertEquals(0L, builder.add((KeyRangeIterator) null).rangeCount());
+        Assert.assertEquals(0L, builder.add((List<KeyRangeIterator>) null).getTokenCount());
+        Assert.assertEquals(0L, builder.add(new LongIterator(new long[] {})).rangeCount());
 
-        KeyRangeIterator single = new LongIterator(new long[] { 1L, 2L, 3L });
-        KeyRangeIterator range = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE).add(single).build();
-
-        // because build should return first element if it's only one instead of building yet another iterator
-        assertEquals(range, single);
+        var single = new LongIterator(new long[] { 1L, 2L, 3L });
 
         // Make a difference between empty and null ranges.
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
-        builder.add(LongIterator.newEmptyIterator());
-        assertEquals(0L, builder.rangeCount());
+        builder = KeyRangeIntersectionIterator.builder();
+        builder.add(new LongIterator(new long[] {}));
+        Assert.assertEquals(0L, builder.rangeCount());
         builder.add(single);
-        assertEquals(1L, builder.rangeCount());
-        range = builder.build();
-        assertEquals(0, range.getMaxKeys());
+        Assert.assertEquals(1L, builder.rangeCount());
+        var range = builder.build();
+        Assert.assertEquals(0, range.getMaxKeys());
 
         // disjoint case
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
-        builder.add(new LongIterator(new long[] { 1L, 2L, 3L }));
-        builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
-
-        assertTrue(builder.isDisjoint());
-
-        KeyRangeIterator disjointIntersection = builder.build();
-        assertNotNull(disjointIntersection);
-        assertFalse(disjointIntersection.hasNext());
-
+        builder = KeyRangeIntersectionIterator.builder();
+
+        // In the disjoint case, the input iterators should be eagerly closed on build and an empty iterator is
+        // returned. These mocks are used to verify that the input iterators are closed.
+        final AtomicBoolean firstIteratorClosed = new AtomicBoolean(false);
+        final AtomicBoolean secondIteratorClosed = new AtomicBoolean(false);
+        LongIterator firstIterator = new LongIterator(new long[] { 1L, 2L, 3L }) {
+            @Override
+            public void close()
+            {
+                firstIteratorClosed.set(true);
+            }
+        };
+        LongIterator secondIterator = new LongIterator(new long[] { 4L, 5L, 6L }) {
+            @Override
+            public void close()
+            {
+                secondIteratorClosed.set(true);
+            }
+        };
+
+        builder.add(firstIterator);
+        builder.add(secondIterator);
+
+        Assert.assertFalse(firstIteratorClosed.get());
+        Assert.assertFalse(secondIteratorClosed.get());
+        Assert.assertTrue(builder.statistics.isEmptyOrDisjoint());
+
+        var disjointIntersection = builder.build();
+        Assert.assertNotNull(disjointIntersection);
+        Assert.assertFalse(disjointIntersection.hasNext());
+        Assert.assertTrue("First input iterator was not closed", firstIteratorClosed.get());
+        Assert.assertTrue("Second input iterator was not closed", secondIteratorClosed.get());
     }
 
     @Test
@@ -242,55 +261,55 @@ public void emptyRangeTest()
         KeyRangeIterator.Builder builder;
 
         // empty, then non-empty
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
-        builder.add(LongIterator.newEmptyIterator());
+        builder = KeyRangeIntersectionIterator.builder();
+        builder.add(new LongIterator(new long[] {}));
         builder.add(new LongIterator(new long[] {10}));
         assertEmpty(builder.build());
 
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
-        builder.add(LongIterator.newEmptyIterator());
+        builder = KeyRangeIntersectionIterator.builder();
+        builder.add(new LongIterator(new long[] {}));
         for (int i = 0; i < 10; i++)
             builder.add(new LongIterator(new long[] {0, i + 10}));
         assertEmpty(builder.build());
 
         // non-empty, then empty
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        builder = KeyRangeIntersectionIterator.builder();
         builder.add(new LongIterator(new long[] {10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         assertEmpty(builder.build());
 
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        builder = KeyRangeIntersectionIterator.builder();
         for (int i = 0; i < 10; i++)
             builder.add(new LongIterator(new long[] {0, i + 10}));
 
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         assertEmpty(builder.build());
 
         // empty, then non-empty then empty again
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
-        builder.add(LongIterator.newEmptyIterator());
+        builder = KeyRangeIntersectionIterator.builder();
+        builder.add(new LongIterator(new long[] {}));
         builder.add(new LongIterator(new long[] {0, 10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         assertEmpty(builder.build());
 
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
-        builder.add(LongIterator.newEmptyIterator());
+        builder = KeyRangeIntersectionIterator.builder();
+        builder.add(new LongIterator(new long[] {}));
         for (int i = 0; i < 10; i++)
             builder.add(new LongIterator(new long[] {0, i + 10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         assertEmpty(builder.build());
 
         // non-empty, empty, then non-empty again
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        builder = KeyRangeIntersectionIterator.builder();
         builder.add(new LongIterator(new long[] {0, 10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         builder.add(new LongIterator(new long[] {0, 10}));
         assertEmpty(builder.build());
 
-        builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        builder = KeyRangeIntersectionIterator.builder();
         for (int i = 0; i < 5; i++)
             builder.add(new LongIterator(new long[] {0, i + 10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         for (int i = 5; i < 10; i++)
             builder.add(new LongIterator(new long[] {0, i + 10}));
         assertEmpty(builder.build());
@@ -298,20 +317,21 @@ public void emptyRangeTest()
 
     public static void assertEmpty(KeyRangeIterator range)
     {
-        assertNull(range.getMinimum());
-        assertNull(range.getMaximum());
-        assertFalse(range.hasNext());
-        assertEquals(0, range.getMaxKeys());
+        Assert.assertNull(range.getMinimum());
+        Assert.assertNull(range.getMaximum());
+        Assert.assertFalse(range.hasNext());
+        Assert.assertEquals(0, range.getMaxKeys());
     }
 
     @Test
     public void testClose() throws IOException
     {
-        KeyRangeIterator tokens = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE)
-                                                              .add(new LongIterator(new long[] { 1L, 2L, 3L }))
-                                                              .build();
+        var tokens = KeyRangeIntersectionIterator.<PrimaryKey>builder()
+                                                 .add(new LongIterator(new long[] { 1L, 2L, 3L }))
+                                                 .add(new LongIterator(new long[] { 2L, 5L, 6L }))
+                                                 .build();
 
-        assertNotNull(tokens);
+        Assert.assertNotNull(tokens);
         tokens.close();
     }
 
@@ -322,23 +342,23 @@ public void testIsOverlapping()
 
         rangeA = new LongIterator(new long[] { 1L, 5L });
         rangeB = new LongIterator(new long[] { 5L, 9L });
-        assertFalse(KeyRangeIntersectionIterator.isDisjoint(rangeA, rangeB));
+        Assert.assertTrue(KeyRangeIterator.isOverlapping(rangeA, rangeB));
 
         rangeA = new LongIterator(new long[] { 5L, 9L });
         rangeB = new LongIterator(new long[] { 1L, 6L });
-        assertFalse(KeyRangeIntersectionIterator.isDisjoint(rangeA, rangeB));
+        Assert.assertTrue(KeyRangeIterator.isOverlapping(rangeA, rangeB));
 
         rangeA = new LongIterator(new long[] { 5L, 9L });
         rangeB = new LongIterator(new long[] { 5L, 9L });
-        assertFalse(KeyRangeIntersectionIterator.isDisjoint(rangeA, rangeB));
+        Assert.assertTrue(KeyRangeIterator.isOverlapping(rangeA, rangeB));
 
         rangeA = new LongIterator(new long[] { 1L, 4L });
         rangeB = new LongIterator(new long[] { 5L, 9L });
-        assertTrue(KeyRangeIntersectionIterator.isDisjoint(rangeA, rangeB));
+        Assert.assertFalse(KeyRangeIterator.isOverlapping(rangeA, rangeB));
 
         rangeA = new LongIterator(new long[] { 6L, 9L });
         rangeB = new LongIterator(new long[] { 1L, 4L });
-        assertTrue(KeyRangeIntersectionIterator.isDisjoint(rangeA, rangeB));
+        Assert.assertFalse(KeyRangeIterator.isOverlapping(rangeA, rangeB));
     }
 
     @Test
@@ -356,7 +376,7 @@ public void testIntersectionOfRandomRanges()
      */
     static Pair<KeyRangeIterator, long[]> createRandom(int nRanges)
     {
-        // generate randomized ranges
+        // generate randomize ranges
         long[][] ranges = new long[nRanges][];
         for (int i = 0; i < ranges.length; i++)
         {
@@ -369,7 +389,7 @@ static Pair<KeyRangeIterator, long[]> createRandom(int nRanges)
             ranges[i] = range.toArray();
             Arrays.sort(ranges[i]);
         }
-        var builder = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE);
+        var builder = KeyRangeIntersectionIterator.builder();
         for (long[] range : ranges)
             builder.add(new LongIterator(range));
 
@@ -377,23 +397,4 @@ static Pair<KeyRangeIterator, long[]> createRandom(int nRanges)
         IntStream.range(1, ranges.length).forEach(i -> expectedSet.retainAll(toSet(ranges[i])));
         return Pair.create(builder.build(), expectedSet.stream().mapToLong(Long::longValue).sorted().toArray());
     }
-
-    // SAI specific tests
-    @Test
-    public void testSelectiveIntersection()
-    {
-        KeyRangeIterator intersection = buildSelectiveIntersection(2,
-                                                                   arr(1L, 4L, 6L, 7L),
-                                                                   arr(1L, 4L, 5L, 6L),
-                                                                   arr(4L, 6L, 8L, 9L, 10L)); // skipped
-
-        assertEquals(convert(1L, 4L, 6L), convert(intersection));
-
-        intersection = buildSelectiveIntersection(1,
-                                                  arr(2L, 4L, 6L),
-                                                  arr(1L, 4L, 5L, 6L),       // skipped
-                                                  arr(4L, 6L, 8L, 9L, 10L)); // skipped
-
-        assertEquals(convert(2L, 4L, 6L), convert(intersection));
-    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeUnionIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeUnionIteratorTest.java
index d750d4fbbdae..e2a92ee0dd05 100644
--- a/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeUnionIteratorTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/iterators/KeyRangeUnionIteratorTest.java
@@ -26,41 +26,40 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.Pair;
 
 import static org.apache.cassandra.index.sai.iterators.LongIterator.convert;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
 
-public class KeyRangeUnionIteratorTest extends AbstractKeyRangeIteratorTester
+public class KeyRangeUnionIteratorTest extends AbstractKeyRangeIteratorTest
 {
     @Test
     public void testNoOverlappingValues()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 2L, 3L, 5L, 6L }));
         builder.add(new LongIterator(new long[] { 1L, 7L }));
         builder.add(new LongIterator(new long[] { 4L, 8L, 9L, 10L }));
 
-        assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L), convert(builder.build()));
+        Assert.assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L), convert(builder.build()));
     }
 
     @Test
     public void testSingleIterator()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L, 2L, 4L, 9L }));
 
-        assertEquals(convert(1L, 2L, 4L, 9L), convert(builder.build()));
+        Assert.assertEquals(convert(1L, 2L, 4L, 9L), convert(builder.build()));
     }
 
     @Test
     public void testOverlappingValues()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L, 4L, 6L, 7L }));
         builder.add(new LongIterator(new long[] { 2L, 3L, 5L, 6L }));
@@ -68,42 +67,42 @@ public void testOverlappingValues()
 
         List<Long> values = convert(builder.build());
 
-        assertEquals(values.toString(), convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L), values);
+        Assert.assertEquals(values.toString(), convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L), values);
     }
 
     @Test
     public void testNoOverlappingRanges()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L, 2L, 3L }));
         builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
         builder.add(new LongIterator(new long[] { 7L, 8L, 9L }));
 
-        assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L), convert(builder.build()));
+        Assert.assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L), convert(builder.build()));
     }
 
     @Test
     public void testTwoIteratorsWithSingleValues()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L }));
         builder.add(new LongIterator(new long[] { 1L }));
 
-        assertEquals(convert(1L), convert(builder.build()));
+        Assert.assertEquals(convert(1L), convert(builder.build()));
     }
 
     @Test
     public void testDifferentSizeIterators()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 2L, 3L, 5L, 6L, 12L, 13L }));
         builder.add(new LongIterator(new long[] { 1L, 7L, 14L, 15 }));
         builder.add(new LongIterator(new long[] { 4L, 5L, 8L, 9L, 10L }));
 
-        assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 12L, 13L, 14L, 15L), convert(builder.build()));
+        Assert.assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 12L, 13L, 14L, 15L), convert(builder.build()));
     }
 
     @Test
@@ -119,7 +118,8 @@ public void testRandomSequences()
     static Pair<KeyRangeIterator, long[]> createRandom(int nRanges)
     {
         long[][] values = new long[nRanges][];
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(10);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
+
         var allValues = new HashSet<Long>();
         // add a random number of random values
         for (int i = 0; i < values.length; i++)
@@ -133,9 +133,10 @@ static Pair<KeyRangeIterator, long[]> createRandom(int nRanges)
                 allValues.add(m);
             }
 
-            // all the parts have to be sorted to mimic SSTable
+            // all of the parts have to be sorted to mimic SSTable
             builder.add(new LongIterator(part.stream().mapToLong(Long::longValue).sorted().toArray()));
         }
+
         long[] totalOrdering = allValues.stream().mapToLong(Long::longValue).sorted().toArray();
         KeyRangeIterator tokens = builder.build();
         return Pair.create(tokens, totalOrdering);
@@ -144,84 +145,79 @@ static Pair<KeyRangeIterator, long[]> createRandom(int nRanges)
     @Test
     public void testMinMaxAndCount()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
 
         builder.add(new LongIterator(new long[] { 1L, 2L, 3L }));
         builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
         builder.add(new LongIterator(new long[] { 7L, 8L, 9L }));
 
-        assertEquals(9L, builder.getMaximum().token().getLongValue());
-        assertEquals(9L, builder.getCount());
+        Assert.assertEquals(9L, builder.getMaximum().token().getLongValue());
+        Assert.assertEquals(9L, builder.getTokenCount());
 
         KeyRangeIterator tokens = builder.build();
 
         Assert.assertNotNull(tokens);
-        assertEquals(1L, tokens.getMinimum().token().getLongValue());
-        assertEquals(9L, tokens.getMaximum().token().getLongValue());
-        assertEquals(9L, tokens.getMaxKeys());
+        Assert.assertEquals(1L, tokens.getMinimum().token().getLongValue());
+        Assert.assertEquals(9L, tokens.getMaximum().token().getLongValue());
+        Assert.assertEquals(9L, tokens.getMaxKeys());
 
         for (long i = 1; i < 10; i++)
         {
             Assert.assertTrue(tokens.hasNext());
-            assertEquals(i, tokens.next().token().getLongValue());
+            Assert.assertEquals(i, tokens.next().token().getLongValue());
         }
 
         Assert.assertFalse(tokens.hasNext());
-        assertEquals(1L, tokens.getMinimum().token().getLongValue());
+        Assert.assertEquals(1L, tokens.getMinimum().token().getLongValue());
     }
 
     @Test
     public void testBuilder()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder();
 
         Assert.assertNull(builder.getMinimum());
         Assert.assertNull(builder.getMaximum());
-        assertEquals(0L, builder.getCount());
-        assertEquals(0L, builder.rangeCount());
+        Assert.assertEquals(0L, builder.getTokenCount());
+        Assert.assertEquals(0L, builder.rangeCount());
 
         builder.add(new LongIterator(new long[] { 1L, 2L, 3L }));
         builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
         builder.add(new LongIterator(new long[] { 7L, 8L, 9L }));
 
-        assertEquals(1L, builder.getMinimum().token().getLongValue());
-        assertEquals(9L, builder.getMaximum().token().getLongValue());
-        assertEquals(9L, builder.getCount());
-        assertEquals(3L, builder.rangeCount());
+        Assert.assertEquals(1L, builder.getMinimum().token().getLongValue());
+        Assert.assertEquals(9L, builder.getMaximum().token().getLongValue());
+        Assert.assertEquals(9L, builder.getTokenCount());
+        Assert.assertEquals(3L, builder.rangeCount());
+        Assert.assertFalse(builder.statistics.isEmptyOrDisjoint());
 
-        assertEquals(1L, builder.rangeIterators.get(0).getMinimum().token().getLongValue());
-        assertEquals(4L, builder.rangeIterators.get(1).getMinimum().token().getLongValue());
-        assertEquals(7L, builder.rangeIterators.get(2).getMinimum().token().getLongValue());
+        Assert.assertEquals(1L, builder.rangeIterators.get(0).getMinimum().token().getLongValue());
+        Assert.assertEquals(4L, builder.rangeIterators.get(1).getMinimum().token().getLongValue());
+        Assert.assertEquals(7L, builder.rangeIterators.get(2).getMinimum().token().getLongValue());
 
-        KeyRangeIterator tokens = KeyRangeUnionIterator.build(new ArrayList<>()
+        KeyRangeIterator tokens = KeyRangeUnionIterator.build(new ArrayList<KeyRangeIterator>()
         {{
             add(new LongIterator(new long[]{1L, 2L, 4L}));
             add(new LongIterator(new long[]{3L, 5L, 6L}));
         }});
 
-        assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L), convert(tokens));
+        Assert.assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L), convert(tokens));
 
         FileUtils.closeQuietly(tokens);
 
-        KeyRangeIterator emptyTokens = KeyRangeUnionIterator.builder(16).build();
-        assertEquals(0, emptyTokens.getMaxKeys());
-
-        builder = KeyRangeUnionIterator.builder(16);
-        assertEquals(0L, builder.add((KeyRangeIterator) null).rangeCount());
-        assertEquals(0L, builder.add((List<KeyRangeIterator>) null).getCount());
-        assertEquals(0L, builder.add(LongIterator.newEmptyIterator()).rangeCount());
+        var emptyTokens = KeyRangeUnionIterator.builder().build();
+        Assert.assertEquals(0, emptyTokens.getMaxKeys());
 
-        KeyRangeIterator single = new LongIterator(new long[] { 1L, 2L, 3L });
-        KeyRangeIterator range = KeyRangeIntersectionIterator.builder(16, Integer.MAX_VALUE).add(single).build();
-
-        // because build should return first element if it's only one instead of building yet another iterator
-        assertEquals(range, single);
+        builder = KeyRangeUnionIterator.builder();
+        Assert.assertEquals(0L, builder.add((KeyRangeIterator) null).rangeCount());
+        Assert.assertEquals(0L, builder.add((List<KeyRangeIterator>) null).getTokenCount());
+        Assert.assertEquals(0L, builder.add(new LongIterator(new long[] {})).rangeCount());
     }
 
     @Test
     public void testSkipTo()
     {
-        KeyRangeUnionIterator.Builder builder = KeyRangeUnionIterator.builder(16);
+        var builder = KeyRangeUnionIterator.<PrimaryKey>builder();
 
         builder.add(new LongIterator(new long[]{1L, 2L, 3L}));
         builder.add(new LongIterator(new long[]{4L, 5L, 6L}));
@@ -232,140 +228,136 @@ public void testSkipTo()
 
         tokens.skipTo(LongIterator.fromToken(5L));
         Assert.assertTrue(tokens.hasNext());
-        assertEquals(5L, tokens.next().token().getLongValue());
+        Assert.assertEquals(5L, tokens.next().token().getLongValue());
 
         tokens.skipTo(LongIterator.fromToken(7L));
         Assert.assertTrue(tokens.hasNext());
-        assertEquals(7L, tokens.next().token().getLongValue());
+        Assert.assertEquals(7L, tokens.next().token().getLongValue());
 
         tokens.skipTo(LongIterator.fromToken(10L));
         Assert.assertFalse(tokens.hasNext());
-        assertEquals(1L, tokens.getMinimum().token().getLongValue());
-        assertEquals(9L, tokens.getMaximum().token().getLongValue());
+        Assert.assertEquals(1L, tokens.getMinimum().token().getLongValue());
+        Assert.assertEquals(9L, tokens.getMaximum().token().getLongValue());
     }
 
     @Test
     public void testMergingMultipleIterators()
     {
-        KeyRangeUnionIterator.Builder builderA = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builderA = KeyRangeUnionIterator.builder();
 
         builderA.add(new LongIterator(new long[] { 1L, 3L, 5L }));
         builderA.add(new LongIterator(new long[] { 8L, 10L, 12L }));
 
-        KeyRangeUnionIterator.Builder builderB = KeyRangeUnionIterator.builder(16);
+        KeyRangeUnionIterator.Builder builderB = KeyRangeUnionIterator.builder();
 
         builderB.add(new LongIterator(new long[] { 7L, 9L, 11L }));
         builderB.add(new LongIterator(new long[] { 2L, 4L, 6L }));
 
         KeyRangeIterator union = KeyRangeUnionIterator.build(Arrays.asList(builderA.build(), builderB.build()));
-        assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L), convert(union));
+        Assert.assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L), convert(union));
     }
 
     @Test
     public void testRangeIterator()
     {
-        try (LongIterator tokens = new LongIterator(new long[] { 0L, 1L, 2L, 3L }))
-        {
-            assertEquals(0L, tokens.getMinimum().token().getLongValue());
-            assertEquals(3L, tokens.getMaximum().token().getLongValue());
+        LongIterator tokens = new LongIterator(new long[] { 0L, 1L, 2L, 3L });
 
-            for (int i = 0; i <= 3; i++)
-            {
-                Assert.assertTrue(tokens.hasNext());
-                assertEquals(i, tokens.peek().token().getLongValue());
-                assertEquals(i, tokens.next().token().getLongValue());
-            }
-        }
+        Assert.assertEquals(0L, tokens.getMinimum().token().getLongValue());
+        Assert.assertEquals(3L, tokens.getMaximum().token().getLongValue());
 
-        try (LongIterator tokens = new LongIterator(new long[] { 0L, 1L, 3L, 5L }))
+        for (int i = 0; i <= 3; i++)
         {
-            tokens.skipTo(LongIterator.fromToken(2L));
             Assert.assertTrue(tokens.hasNext());
-            assertEquals(3L, tokens.peek().token().getLongValue());
-            assertEquals(3L, tokens.next().token().getLongValue());
-
-            tokens.skipTo(LongIterator.fromToken(5L));
-            Assert.assertTrue(tokens.hasNext());
-            assertEquals(5L, tokens.peek().token().getLongValue());
-            assertEquals(5L, tokens.next().token().getLongValue());
+            Assert.assertEquals(i, tokens.peek().token().getLongValue());
+            Assert.assertEquals(i, tokens.next().token().getLongValue());
         }
 
-        try (LongIterator empty = LongIterator.newEmptyIterator())
-        {
-            empty.skipTo(LongIterator.fromToken(3L));
-            Assert.assertFalse(empty.hasNext());
-        }
+        tokens = new LongIterator(new long[] { 0L, 1L, 3L, 5L });
+
+        tokens.skipTo(LongIterator.fromToken(2L));
+        Assert.assertTrue(tokens.hasNext());
+        Assert.assertEquals(3L, tokens.peek().token().getLongValue());
+        Assert.assertEquals(3L, tokens.next().token().getLongValue());
+
+        tokens.skipTo(LongIterator.fromToken(5L));
+        Assert.assertTrue(tokens.hasNext());
+        Assert.assertEquals(5L, tokens.peek().token().getLongValue());
+        Assert.assertEquals(5L, tokens.next().token().getLongValue());
+
+        LongIterator empty = new LongIterator(new long[0]);
+
+        empty.skipTo(LongIterator.fromToken(3L));
+        Assert.assertFalse(empty.hasNext());
     }
 
     @Test
-    public void emptyRangeTest()
-    {
+    public void emptyRangeTest() {
         KeyRangeIterator.Builder builder;
         KeyRangeIterator range;
         // empty, then non-empty
-        builder = KeyRangeUnionIterator.builder(16);
-        builder.add(LongIterator.newEmptyIterator());
+        builder = KeyRangeUnionIterator.builder();
+        builder.add(new LongIterator(new long[] {}));
         for (int i = 0; i < 10; i++)
             builder.add(new LongIterator(new long[] {i + 10}));
         range = builder.build();
-        assertEquals(10L, range.getMinimum().token().getLongValue());
-        assertEquals(19L, range.getMaximum().token().getLongValue());
+        Assert.assertEquals(10L, range.getMinimum().token().getLongValue());
+        Assert.assertEquals(19L, range.getMaximum().token().getLongValue());
         Assert.assertTrue(range.hasNext());
-        assertEquals(10, range.getMaxKeys());
+        Assert.assertEquals(10, range.getMaxKeys());
 
-        builder = KeyRangeUnionIterator.builder(16);
-        builder.add(LongIterator.newEmptyIterator());
+        builder = KeyRangeUnionIterator.builder();
+        builder.add(new LongIterator(new long[] {}));
         builder.add(new LongIterator(new long[] {10}));
         range = builder.build();
-        assertEquals(10L, range.getMinimum().token().getLongValue());
-        assertEquals(10L, range.getMaximum().token().getLongValue());
+        Assert.assertEquals(10L, range.getMinimum().token().getLongValue());
+        Assert.assertEquals(10L, range.getMaximum().token().getLongValue());
         Assert.assertTrue(range.hasNext());
-        assertEquals(1, range.getMaxKeys());
+        Assert.assertEquals(1, range.getMaxKeys());
 
         // non-empty, then empty
-        builder = KeyRangeUnionIterator.builder(16);
+        builder = KeyRangeUnionIterator.builder();
         for (int i = 0; i < 10; i++)
             builder.add(new LongIterator(new long[] {i + 10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         range = builder.build();
-        assertEquals(10, range.getMinimum().token().getLongValue());
-        assertEquals(19, range.getMaximum().token().getLongValue());
+        Assert.assertEquals(10, range.getMinimum().token().getLongValue());
+        Assert.assertEquals(19, range.getMaximum().token().getLongValue());
         Assert.assertTrue(range.hasNext());
-        assertEquals(10, range.getMaxKeys());
+        Assert.assertEquals(10, range.getMaxKeys());
 
-        builder = KeyRangeUnionIterator.builder(16);
+        builder = KeyRangeUnionIterator.builder();
         builder.add(new LongIterator(new long[] {10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         range = builder.build();
-        assertEquals(10L, range.getMinimum().token().getLongValue());
-        assertEquals(10L, range.getMaximum().token().getLongValue());
+        Assert.assertEquals(10L, range.getMinimum().token().getLongValue());
+        Assert.assertEquals(10L, range.getMaximum().token().getLongValue());
         Assert.assertTrue(range.hasNext());
-        assertEquals(1, range.getMaxKeys());
+        Assert.assertEquals(1, range.getMaxKeys());
 
         // empty, then non-empty then empty again
-        builder = KeyRangeUnionIterator.builder(16);
-        builder.add(LongIterator.newEmptyIterator());
+        builder = KeyRangeUnionIterator.builder();
+        builder.add(new LongIterator(new long[] {}));
         for (int i = 0; i < 10; i++)
             builder.add(new LongIterator(new long[] {i + 10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         range = builder.build();
-        assertEquals(10L, range.getMinimum().token().getLongValue());
-        assertEquals(19L, range.getMaximum().token().getLongValue());
+        Assert.assertEquals(10L, range.getMinimum().token().getLongValue());
+        Assert.assertEquals(19L, range.getMaximum().token().getLongValue());
         Assert.assertTrue(range.hasNext());
-        assertEquals(10, range.getMaxKeys());
+        Assert.assertEquals(10, range.getMaxKeys());
 
         // non-empty, empty, then non-empty again
-        builder = KeyRangeUnionIterator.builder(16);
+        builder = KeyRangeUnionIterator.builder();
         for (int i = 0; i < 5; i++)
             builder.add(new LongIterator(new long[] {i + 10}));
-        builder.add(LongIterator.newEmptyIterator());
+        builder.add(new LongIterator(new long[] {}));
         for (int i = 5; i < 10; i++)
             builder.add(new LongIterator(new long[] {i + 10}));
         range = builder.build();
-        assertEquals(10L, range.getMinimum().token().getLongValue());
-        assertEquals(19L, range.getMaximum().token().getLongValue());
+        Assert.assertEquals(10L, range.getMinimum().token().getLongValue());
+        Assert.assertEquals(19L, range.getMaximum().token().getLongValue());
         Assert.assertTrue(range.hasNext());
-        assertEquals(10, range.getMaxKeys());
+        Assert.assertEquals(10, range.getMaxKeys());
     }
 
     // SAI specific tests
@@ -385,11 +377,11 @@ public void testUnionOfIntersection()
 
         union = buildUnion(intersectionA, intersectionB);
         assertEquals(convert(2L, 3L, 7L, 8L), convert(union));
-        assertSame(KeyRangeUnionIterator.class, union.getClass());
+        assertEquals(KeyRangeUnionIterator.class, union.getClass());
 
         // union of one intersected intersection and one non-intersected intersection
-        intersectionA = buildIntersection(arr(1L, 2L, 3L), arr(2L, 3L, 4L));
-        intersectionB = buildIntersection(arr(6L, 7L, 8L), arr(10L));
+        intersectionA = buildIntersection(arr(1L, 2L, 3L), arr(2L, 3L, 4L ));
+        intersectionB = buildIntersection(arr(6L, 7L, 8L), arr(10L ));
 
         union = buildUnion(intersectionA, intersectionB);
         assertEquals(convert(2L, 3L), convert(union));
@@ -401,7 +393,7 @@ public void testUnionOfRandom()
         for (int testIteration = 0; testIteration < 16; testIteration++)
         {
             var allValues = new HashSet<Long>();
-            var builder = KeyRangeUnionIterator.builder(10);
+            var builder = KeyRangeUnionIterator.builder();
             for (int i = 0; i < nextInt(2, 3); i++)
             {
                 var p = createRandomIterator();
diff --git a/test/unit/org/apache/cassandra/index/sai/iterators/LongIterator.java b/test/unit/org/apache/cassandra/index/sai/iterators/LongIterator.java
index 45f655456a24..f5a1f1409f39 100644
--- a/test/unit/org/apache/cassandra/index/sai/iterators/LongIterator.java
+++ b/test/unit/org/apache/cassandra/index/sai/iterators/LongIterator.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.LongFunction;
 
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.index.sai.SAITester;
@@ -29,24 +30,18 @@ public class LongIterator extends KeyRangeIterator
     private final List<PrimaryKey> keys;
     private int currentIdx = 0;
 
-    public static LongIterator newEmptyIterator()
-    {
-        return new LongIterator();
-    }
-
-    private LongIterator()
+    public LongIterator(long[] tokens)
     {
-        super(null, null, 0);
-        keys = null;
+        this(tokens, t -> t);
     }
 
-    public LongIterator(long[] tokens)
+    public LongIterator(long[] tokens, LongFunction<Long> toOffset)
     {
         super(tokens.length == 0 ? null : fromToken(tokens[0]), tokens.length == 0 ? null : fromToken(tokens[tokens.length - 1]), tokens.length);
 
         this.keys = new ArrayList<>(tokens.length);
         for (long token : tokens)
-            this.keys.add(fromToken(token));
+            this.keys.add(fromTokenAndRowId(token, toOffset.apply(token)));
     }
 
     @Override
@@ -59,14 +54,10 @@ protected PrimaryKey computeNext()
     }
 
     @Override
-    protected void performSkipTo(PrimaryKey nextKey)
+    protected void performSkipTo(PrimaryKey nextToken)
     {
-        for ( ; currentIdx < keys.size(); currentIdx++)
-        {
-            PrimaryKey token = keys.get(currentIdx);
-            if (token.compareTo(nextKey) >= 0)
-                break;
-        }
+        while (currentIdx < keys.size() && keys.get(currentIdx).compareTo(nextToken) < 0)
+            currentIdx++;
     }
 
     @Override
@@ -75,7 +66,7 @@ public void close()
 
     public static PrimaryKey fromToken(long token)
     {
-        return SAITester.TEST_FACTORY.create(new Murmur3Partitioner.LongToken(token));
+        return SAITester.TEST_FACTORY.createTokenOnly(new Murmur3Partitioner.LongToken(token));
     }
 
 
@@ -90,10 +81,15 @@ public static List<Long> convert(KeyRangeIterator tokens)
 
     public static List<Long> convert(final long... nums)
     {
-        return new ArrayList<>(nums.length)
+        return new ArrayList<Long>(nums.length)
         {{
             for (long n : nums)
                 add(n);
         }};
     }
+
+    private PrimaryKey fromTokenAndRowId(long token, long rowId)
+    {
+        return SAITester.TEST_FACTORY.createTokenOnly(new Murmur3Partitioner.LongToken(token));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/AbstractInMemoryKeyRangeIteratorTester.java b/test/unit/org/apache/cassandra/index/sai/memory/AbstractInMemoryKeyRangeIteratorTester.java
deleted file mode 100644
index 3c21941a2e01..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/memory/AbstractInMemoryKeyRangeIteratorTester.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.memory;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-public abstract class AbstractInMemoryKeyRangeIteratorTester
-{
-    protected PrimaryKey.Factory primaryKeyFactory;
-
-    @Before
-    public void setup()
-    {
-        primaryKeyFactory = new PrimaryKey.Factory(Murmur3Partitioner.instance, SAITester.EMPTY_COMPARATOR);
-    }
-
-    @Test
-    public void singleTokenIsReturned()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 1, 1);
-
-        assertIterator(iterator, 1);
-    }
-
-    @Test
-    public void duplicateSingleTokenIsReturned()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 1, 1, 1);
-
-        assertIterator(iterator, 1);
-    }
-
-    @Test
-    public void withoutSkipAllTokensAreReturnedInTokenOrder()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 2, 3);
-
-        assertIterator(iterator, 1, 2, 3);
-    }
-
-    @Test
-    public void tokensAddedOutOfOrderAreReturnedInOrder()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 3, 2, 1);
-
-        assertIterator(iterator, 1, 2, 3);
-    }
-
-    @Test
-    public void matchingTokensAreIgnoredAtStart()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 1, 2, 3);
-
-        assertIterator(iterator, 1, 2, 3);
-    }
-
-    @Test
-    public void matchingTokensAreIgnoredInMiddle()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 2, 2, 3);
-
-        assertIterator(iterator, 1, 2, 3);
-    }
-
-    @Test
-    public void matchingTokensAreIgnoredAtEnd()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 2, 3, 3);
-
-        assertIterator(iterator, 1, 2, 3);
-    }
-
-    @Test
-    public void skipToTokenBeforeFirstTokenWillReturnAllTokens()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 2, 3);
-
-        iterator.skipTo(primaryKeyFactory.create(new Murmur3Partitioner.LongToken(0)));
-
-        assertIterator(iterator, 1, 2, 3);
-    }
-
-    @Test
-    public void skipToFirstTokenWillReturnAllTokens()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 2, 3);
-
-        iterator.skipTo(primaryKeyFactory.create(new Murmur3Partitioner.LongToken(1)));
-
-        assertIterator(iterator, 1, 2, 3);
-    }
-
-    @Test
-    public void skipToMiddleTokenWillReturnRemainingTokens()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 2, 3);
-
-        iterator.skipTo(primaryKeyFactory.create(new Murmur3Partitioner.LongToken(2)));
-
-        assertIterator(iterator, 2, 3);
-    }
-
-    @Test
-    public void skipToLastTokenWillReturnLastToken()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 2, 3);
-
-        iterator.skipTo(primaryKeyFactory.create(new Murmur3Partitioner.LongToken(3)));
-
-        assertIterator(iterator, 3);
-    }
-
-    @Test
-    public void skipToAfterLastTokenWillReturnNoTokens()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 2, 3);
-
-        iterator.skipTo(primaryKeyFactory.create(new Murmur3Partitioner.LongToken(4)));
-
-        assertIterator(iterator);
-    }
-
-    @Test
-    public void skipToWithMatchingTokensWithReturnCorrectTokens()
-    {
-        KeyRangeIterator iterator = makeIterator(1, 3, 1, 1, 2, 2, 3, 3);
-
-        iterator.skipTo(primaryKeyFactory.create(new Murmur3Partitioner.LongToken(2)));
-
-        assertIterator(iterator, 2, 3);
-    }
-
-    private void assertIterator(KeyRangeIterator iterator, long... tokens)
-    {
-        for(long token : tokens)
-        {
-            assertEquals(token, iterator.next().token().getLongValue());
-        }
-        assertFalse(iterator.hasNext());
-    }
-
-
-    protected abstract KeyRangeIterator makeIterator(long minimumTokenValue, long maximumTokenValue, long... tokens);
-
-    protected PrimaryKey keyForToken(long token)
-    {
-        return primaryKeyFactory.create(new Murmur3Partitioner.LongToken(token));
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIteratorTest.java
deleted file mode 100644
index 2da06f24478b..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIteratorTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.memory;
-
-import java.util.Arrays;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-
-public class InMemoryKeyRangeIteratorTest extends AbstractInMemoryKeyRangeIteratorTester
-{
-    @Override
-    protected KeyRangeIterator makeIterator(long minimumTokenValue, long maximumTokenValue, long... tokens)
-    {
-        SortedSet<PrimaryKey> set = new TreeSet<>();
-
-        Arrays.stream(tokens).forEach(t -> set.add(keyForToken(t)));
-
-        return new InMemoryKeyRangeIterator(set);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/PriorityInMemoryKeyRangeIteratorTest.java b/test/unit/org/apache/cassandra/index/sai/memory/PriorityInMemoryKeyRangeIteratorTest.java
deleted file mode 100644
index 5864c79b5507..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/memory/PriorityInMemoryKeyRangeIteratorTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.memory;
-
-import java.util.Arrays;
-import java.util.PriorityQueue;
-
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-
-public class PriorityInMemoryKeyRangeIteratorTest extends AbstractInMemoryKeyRangeIteratorTester
-{
-    @Override
-    protected KeyRangeIterator makeIterator(long minimumTokenValue, long maximumTokenValue, long... tokens)
-    {
-        PriorityQueue<PrimaryKey> queue = new PriorityQueue<>(tokens.length);
-
-        Arrays.stream(tokens).forEach(t -> queue.add(keyForToken(t)));
-
-        return new InMemoryKeyRangeIterator(primaryKeyFactory.create(new Murmur3Partitioner.LongToken(minimumTokenValue)),
-                                            primaryKeyFactory.create(new Murmur3Partitioner.LongToken(maximumTokenValue)),
-                                            queue);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java
index 0ab4c846f754..35683a13e748 100644
--- a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java
@@ -18,181 +18,120 @@
 package org.apache.cassandra.index.sai.memory;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
 import java.util.function.IntFunction;
-import java.util.stream.Collectors;
 
+import org.junit.Before;
 import org.junit.Test;
 
-import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CompositeType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.ExcludingBounds;
-import org.apache.cassandra.dht.IncludingExcludingBounds;
 import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.TargetParser;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKeys;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.MockSchema;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 
-public class TrieMemoryIndexTest extends SAIRandomizedTester
+public class TrieMemoryIndexTest
 {
     private static final String KEYSPACE = "test_keyspace";
     private static final String TABLE = "test_table";
     private static final String PART_KEY_COL = "key";
     private static final String REG_COL = "col";
 
-    private static final DecoratedKey key = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes("key"));
+    private static DecoratedKey key = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes("key"));
 
-    private StorageAttachedIndex index;
+    private TableMetadata table;
 
-    @Test
-    public void heapGrowsAsDataIsAddedTest()
+    @Before
+    public void setup()
     {
-        TrieMemoryIndex index = newTrieMemoryIndex(Int32Type.instance);
-        for (int i = 0; i < 99; i++)
-        {
-            assertTrue(index.add(key, Clustering.EMPTY, Int32Type.instance.decompose(i)) > 0);
-        }
+        SchemaLoader.prepareServer();
     }
 
     @Test
-    public void randomQueryTest() throws Exception
+    public void iteratorShouldReturnAllValuesNumeric()
     {
-        TrieMemoryIndex index = newTrieMemoryIndex(Int32Type.instance);
-
-        Map<DecoratedKey, Integer> keyMap = new TreeMap<>();
-        Map<Integer, Integer> rowMap = new HashMap<>();
+        TrieMemoryIndex index = newTrieMemoryIndex(Int32Type.instance, Int32Type.instance);
 
-        for (int row = 0; row < getRandom().nextIntBetween(1000, 5000); row++)
+        for (int row = 0; row < 100; row++)
         {
-            int pk = getRandom().nextIntBetween(0, 10000);
-            while (rowMap.containsKey(pk))
-                pk = getRandom().nextIntBetween(0, 10000);
-            int value = getRandom().nextIntBetween(0, 100);
-            rowMap.put(pk, value);
-            DecoratedKey key = Murmur3Partitioner.instance.decorateKey(Int32Type.instance.decompose(pk));
-            index.add(key, Clustering.EMPTY, Int32Type.instance.decompose(value));
-            keyMap.put(key, pk);
+            index.add(makeKey(table, Integer.toString(row)), Clustering.EMPTY, Int32Type.instance.decompose(row / 10), allocatedBytes -> {}, allocatesBytes -> {});
         }
 
-        List<DecoratedKey> keys = new ArrayList<>(keyMap.keySet());
-
-        for (int executionCount = 0; executionCount < 1000; executionCount++)
+        var iterator = index.iterator();
+        int valueCount = 0;
+        while(iterator.hasNext())
         {
-            Expression expression = generateRandomExpression();
-
-            AbstractBounds<PartitionPosition> keyRange = generateRandomBounds(keys);
-
-            Set<Integer> expectedKeys = keyMap.keySet()
-                                              .stream()
-                                              .filter(keyRange::contains)
-                                              .map(keyMap::get)
-                                              .filter(pk -> expression.isSatisfiedBy(Int32Type.instance.decompose(rowMap.get(pk))))
-                                              .collect(Collectors.toSet());
-
-            Set<Integer> foundKeys = new HashSet<>();
-
-            try (KeyRangeIterator iterator = index.search(null, expression, keyRange))
+            var pair = iterator.next();
+            int value = ByteSourceInverse.getSignedInt(pair.left.asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION));
+            int idCount = 0;
+            for (var pkf : pair.right)
             {
-                while (iterator.hasNext())
-                {
-                    int key = Int32Type.instance.compose(iterator.next().partitionKey().getKey());
-                    assertFalse(foundKeys.contains(key));
-                    foundKeys.add(key);
-                }
+                PrimaryKey primaryKey = pkf.pk;
+                int id = Int32Type.instance.compose(primaryKey.partitionKey().getKey());
+                assertEquals(id/10, value);
+                idCount++;
             }
-
-            assertEquals(expectedKeys, foundKeys);
+            assertEquals(10, idCount);
+            valueCount++;
         }
+        assertEquals(10, valueCount);
     }
 
-    private AbstractBounds<PartitionPosition> generateRandomBounds(List<DecoratedKey> keys)
+    @Test
+    public void iteratorShouldReturnAllValuesString()
     {
-        PartitionPosition leftBound = getRandom().nextBoolean() ? Murmur3Partitioner.instance.getMinimumToken().minKeyBound()
-                                                                : keys.get(getRandom().nextIntBetween(0, keys.size() - 1)).getToken().minKeyBound();
+        TrieMemoryIndex index = newTrieMemoryIndex(UTF8Type.instance, UTF8Type.instance);
 
-        PartitionPosition rightBound = getRandom().nextBoolean() ? Murmur3Partitioner.instance.getMinimumToken().minKeyBound()
-                                                                 : keys.get(getRandom().nextIntBetween(0, keys.size() - 1)).getToken().maxKeyBound();
-
-        AbstractBounds<PartitionPosition> keyRange;
-
-        if (leftBound.isMinimum() && rightBound.isMinimum())
-            keyRange = new Range<>(leftBound, rightBound);
-        else
+        for (int row = 0; row < 100; row++)
         {
-            if (AbstractBounds.strictlyWrapsAround(leftBound, rightBound))
-            {
-                PartitionPosition temp = leftBound;
-                leftBound = rightBound;
-                rightBound = temp;
-            }
-            if (getRandom().nextBoolean())
-                keyRange = new Bounds<>(leftBound, rightBound);
-            else if (getRandom().nextBoolean())
-                keyRange = new ExcludingBounds<>(leftBound, rightBound);
-            else
-                keyRange = new IncludingExcludingBounds<>(leftBound, rightBound);
+            index.add(makeKey(table, Integer.toString(row)), Clustering.EMPTY, UTF8Type.instance.decompose(Integer.toString(row / 10)), allocatedBytes -> {}, allocatesBytes -> {});
         }
-        return keyRange;
-    }
 
-    private Expression generateRandomExpression()
-    {
-        Expression expression = Expression.create(index);
-
-        int equality = getRandom().nextIntBetween(0, 100);
-        int lower = getRandom().nextIntBetween(0, 75);
-        int upper = getRandom().nextIntBetween(25, 100);
-        while (upper <= lower)
-            upper = getRandom().nextIntBetween(0, 100);
-
-        if (getRandom().nextBoolean())
-            expression.add(Operator.EQ, Int32Type.instance.decompose(equality));
-        else
+        var iterator = index.iterator();
+        int valueCount = 0;
+        while(iterator.hasNext())
         {
-            boolean useLower = getRandom().nextBoolean();
-            boolean useUpper = getRandom().nextBoolean();
-            if (!useLower && !useUpper)
-                useLower = useUpper = true;
-            if (useLower)
-                expression.add(getRandom().nextBoolean() ? Operator.GT : Operator.GTE, Int32Type.instance.decompose(lower));
-            if (useUpper)
-                expression.add(getRandom().nextBoolean() ? Operator.LT : Operator.LTE, Int32Type.instance.decompose(upper));
+            var pair = iterator.next();
+            String value = new String(ByteSourceInverse.readBytes(ByteSource.peekable(pair.left.asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION))), StandardCharsets.UTF_8);
+            int idCount = 0;
+            for (var pkf : pair.right)
+            {
+                PrimaryKey primaryKey = pkf.pk;
+                String id = UTF8Type.instance.compose(primaryKey.partitionKey().getKey());
+                assertEquals(Integer.toString(Integer.parseInt(id) / 10), value);
+                idCount++;
+            }
+            assertEquals(10, idCount);
+            valueCount++;
         }
-        return expression;
+        assertEquals(10, valueCount);
     }
 
     @Test
-    public void shouldAcceptPrefixValuesTest()
+    public void shouldAcceptPrefixValues()
     {
         shouldAcceptPrefixValuesForType(UTF8Type.instance, i -> UTF8Type.instance.decompose(String.format("%03d", i)));
         shouldAcceptPrefixValuesForType(Int32Type.instance, Int32Type.instance::decompose);
@@ -200,47 +139,66 @@ public void shouldAcceptPrefixValuesTest()
 
     private void shouldAcceptPrefixValuesForType(AbstractType<?> type, IntFunction<ByteBuffer> decompose)
     {
-        TrieMemoryIndex index = newTrieMemoryIndex(type);
+        final TrieMemoryIndex index = newTrieMemoryIndex(UTF8Type.instance, type);
         for (int i = 0; i < 99; ++i)
         {
-            index.add(key, Clustering.EMPTY, decompose.apply(i));
+            index.add(key, Clustering.EMPTY, decompose.apply(i), allocatedBytes -> {}, allocatesBytes -> {});
         }
 
-        final Iterator<Pair<ByteComparable, PrimaryKeys>> iterator = index.iterator();
+        final var iterator = index.iterator();
         int i = 0;
         while (iterator.hasNext())
         {
-            Pair<ByteComparable, PrimaryKeys> pair = iterator.next();
+            var pair = iterator.next();
             assertEquals(1, pair.right.size());
 
             final int rowId = i;
-            final ByteComparable expectedByteComparable = version -> type.asComparableBytes(decompose.apply(rowId), version);
+            final ByteComparable expectedByteComparable = TypeUtil.isLiteral(type)
+                                                          ? v -> ByteSource.preencoded(decompose.apply(rowId))
+                                                          : version -> type.asComparableBytes(decompose.apply(rowId), version);
             final ByteComparable actualByteComparable = pair.left;
-            assertEquals("Mismatch at: " + i, 0, ByteComparable.compare(expectedByteComparable, actualByteComparable, ByteComparable.Version.OSS50));
+            assertEquals("Mismatch at: " + i, 0, ByteComparable.compare(expectedByteComparable, actualByteComparable, TypeUtil.BYTE_COMPARABLE_VERSION));
 
             i++;
         }
         assertEquals(99, i);
     }
 
-    private TrieMemoryIndex newTrieMemoryIndex(AbstractType<?> columnType)
+    private TrieMemoryIndex newTrieMemoryIndex(AbstractType<?> partitionKeyType, AbstractType<?> columnType)
     {
-        TableMetadata table = TableMetadata.builder(KEYSPACE, TABLE)
-                                           .addPartitionKeyColumn(PART_KEY_COL, UTF8Type.instance)
-                                           .addRegularColumn(REG_COL, columnType)
-                                           .partitioner(Murmur3Partitioner.instance)
-                                           .caching(CachingParams.CACHE_NOTHING)
-                                           .build();
+        table = TableMetadata.builder(KEYSPACE, TABLE)
+                             .addPartitionKeyColumn(PART_KEY_COL, partitionKeyType)
+                             .addRegularColumn(REG_COL, columnType)
+                             .partitioner(Murmur3Partitioner.instance)
+                             .caching(CachingParams.CACHE_NOTHING)
+                             .build();
 
         Map<String, String> options = new HashMap<>();
         options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName());
         options.put("target", REG_COL);
 
         IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata("col_index", IndexMetadata.Kind.CUSTOM, options);
+        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(table, indexMetadata);
+        IndexContext indexContext = new IndexContext(table.keyspace,
+                                                     table.name,
+                                                     table.id,
+                                                     table.partitionKeyType,
+                                                     table.comparator,
+                                                     target.left,
+                                                     target.right,
+                                                     indexMetadata,
+                                                     MockSchema.newCFS(table));
+
+        return new TrieMemoryIndex(indexContext);
+    }
 
-        ColumnFamilyStore cfs = MockSchema.newCFS(table);
-
-        index = new StorageAttachedIndex(cfs, indexMetadata);
-        return new TrieMemoryIndex(index);
+    DecoratedKey makeKey(TableMetadata table, Object...partitionKeys)
+    {
+        ByteBuffer key;
+        if (TypeUtil.isComposite(table.partitionKeyType))
+            key = ((CompositeType)table.partitionKeyType).decompose(partitionKeys);
+        else
+            key = table.partitionKeyType.fromString((String)partitionKeys[0]);
+        return table.partitioner.decorateKey(key);
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexAllocationsHeapBuffersTest.java b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexAllocationsHeapBuffersTest.java
new file mode 100644
index 000000000000..a74960c63c59
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexAllocationsHeapBuffersTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.db.memtable.TrieMemtable;
+import org.apache.cassandra.io.compress.BufferType;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+import static org.junit.Assert.assertEquals;
+
+public class TrieMemtableIndexAllocationsHeapBuffersTest extends TrieMemtableIndexTestBase
+{
+    @BeforeClass
+    public static void setUpClass()
+    {
+        MEMTABLE_SHARD_COUNT.setInt(8);
+        setup(Config.MemtableAllocationType.heap_buffers);
+        assertEquals(TrieMemtable.BUFFER_TYPE, BufferType.ON_HEAP);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTest.java b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTest.java
new file mode 100644
index 000000000000..cff8a23348ed
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.memtable.TrieMemtable;
+import org.apache.cassandra.io.compress.BufferType;
+import org.junit.BeforeClass;
+
+import org.junit.Ignore;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+import static org.junit.Assert.assertEquals;
+
+@Ignore
+public abstract class TrieMemtableIndexTest extends TrieMemtableIndexTestBase
+{
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        CQLTester.setUpClass();
+        MEMTABLE_SHARD_COUNT.setInt(8);
+        setup(Config.MemtableAllocationType.offheap_buffers);
+        assertEquals(TrieMemtable.BUFFER_TYPE, BufferType.OFF_HEAP);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTestBase.java b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTestBase.java
new file mode 100644
index 000000000000..4e444f02ff20
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTestBase.java
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Iterators;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.memtable.AbstractAllocatorMemtable;
+import org.apache.cassandra.db.memtable.AbstractShardedMemtable;
+import org.apache.cassandra.db.memtable.TrieMemtable;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.BootStrapper;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.ExcludingBounds;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.IncludingExcludingBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.inject.InvokePointBuilder;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public abstract class TrieMemtableIndexTestBase extends SAITester
+{
+    private static final Injections.Counter indexSearchCounter = Injections.newCounter("IndexSearchCounter")
+                                                                           .add(InvokePointBuilder.newInvokePoint()
+                                                                                                  .onClass(TrieMemoryIndex.class)
+                                                                                                  .onMethod("search"))
+                                                                           .build();
+
+    // A non-frozen list of integers
+    private final ListType<Integer> integerListType = ListType.getInstance(Int32Type.instance, true);
+
+    ColumnFamilyStore cfs;
+    IndexContext indexContext;
+    IndexContext integerListIndexContext;
+    TrieMemtableIndex memtableIndex;
+    AbstractAllocatorMemtable memtable;
+    IPartitioner partitioner;
+    Map<DecoratedKey, Integer> keyMap;
+    Map<Integer, Integer> rowMap;
+
+    public static void setup(Config.MemtableAllocationType allocationType)
+    {
+        try
+        {
+            Field confField = DatabaseDescriptor.class.getDeclaredField("conf");
+            confField.setAccessible(true);
+            Config conf = (Config) confField.get(null);
+            conf.memtable_allocation_type = allocationType;
+            conf.memtable_cleanup_threshold = 0.8f; // give us more space to fit test data without flushing
+        }
+        catch (NoSuchFieldException | IllegalAccessException e)
+        {
+            throw Throwables.propagate(e);
+        }
+
+        CQLTester.setUpClass();
+        MEMTABLE_SHARD_COUNT.setInt(8);
+    }
+
+    @Before
+    public void setup() throws Throwable
+    {
+        assertEquals(8, AbstractShardedMemtable.getDefaultShardCount());
+
+        TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+        metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddressAndPort());
+
+        TableMetadata tableMetadata = TableMetadata.builder("ks", "tb")
+                                                   .addPartitionKeyColumn("pk", Int32Type.instance)
+                                                   .addRegularColumn("val", Int32Type.instance)
+                                                   .addRegularColumn("vals", integerListType)
+                                                   .build();
+        cfs = MockSchema.newCFS(tableMetadata);
+        partitioner = cfs.getPartitioner();
+        memtable = (AbstractAllocatorMemtable) cfs.getCurrentMemtable();
+        indexContext = SAITester.createIndexContext("index", Int32Type.instance, cfs);
+        integerListIndexContext = SAITester.createIndexContext("collection_index", integerListType, cfs);
+        indexSearchCounter.reset();
+        keyMap = new TreeMap<>();
+        rowMap = new HashMap<>();
+
+        Injections.inject(indexSearchCounter);
+    }
+
+    @Test
+    public void allocation() throws Throwable
+    {
+        assertEquals(8, AbstractShardedMemtable.getDefaultShardCount());
+        memtableIndex = new TrieMemtableIndex(indexContext, memtable);
+        assertEquals(AbstractShardedMemtable.getDefaultShardCount(), memtableIndex.shardCount());
+
+        assertEquals(0, memtable.getAllocator().onHeap().owns());
+        assertEquals(0, memtable.getAllocator().offHeap().owns());
+
+        for (int row = 0; row < 100; row++)
+        {
+            addRow(row, row);
+        }
+
+        assertTrue(memtable.getAllocator().onHeap().owns() > 0);
+
+        if (TrieMemtable.BUFFER_TYPE == BufferType.OFF_HEAP)
+            assertTrue(memtable.getAllocator().onHeap().owns() > 0);
+        else
+            assertEquals(0, memtable.getAllocator().offHeap().owns());
+    }
+
+    @Test
+    public void randomQueryTest() throws Exception
+    {
+        memtableIndex = new TrieMemtableIndex(indexContext, memtable);
+        assertEquals(AbstractShardedMemtable.getDefaultShardCount(), memtableIndex.shardCount());
+
+        for (int row = 0; row < getRandom().nextIntBetween(1000, 5000); row++)
+        {
+            int pk = getRandom().nextIntBetween(0, 10000);
+            while (rowMap.containsKey(pk))
+                pk = getRandom().nextIntBetween(0, 10000);
+            int value = getRandom().nextIntBetween(0, 100);
+            rowMap.put(pk, value);
+            addRow(pk, value);
+        }
+
+        List<DecoratedKey> keys = new ArrayList<>(keyMap.keySet());
+
+        for (int executionCount = 0; executionCount < 1000; executionCount++)
+        {
+            Expression expression = generateRandomExpression();
+
+            AbstractBounds<PartitionPosition> keyRange = generateRandomBounds(keys);
+
+            Set<Integer> expectedKeys = keyMap.keySet()
+                                              .stream()
+                                              .filter(keyRange::contains)
+                                              .map(keyMap::get)
+                                              .filter(pk -> expression.isSatisfiedBy(Int32Type.instance.decompose(rowMap.get(pk))))
+                                              .collect(Collectors.toSet());
+
+            Set<Integer> foundKeys = new HashSet<>();
+
+            try (KeyRangeIterator iterator = memtableIndex.search(new QueryContext(), expression, keyRange, 0))
+            {
+                while (iterator.hasNext())
+                {
+                    DecoratedKey k = iterator.next().partitionKey();
+                    int key = Int32Type.instance.compose(k.getKey());
+                    assertFalse(foundKeys.contains(key));
+                    foundKeys.add(key);
+                }
+            }
+
+            assertEquals(expectedKeys, foundKeys);
+        }
+    }
+
+    @Test
+    public void indexIteratorTest()
+    {
+        memtableIndex = new TrieMemtableIndex(indexContext, memtable);
+
+        Map<Integer, Set<DecoratedKey>> terms = buildTermMap();
+
+        terms.entrySet()
+             .stream()
+             .forEach(entry -> entry.getValue()
+                                    .forEach(pk -> addRow(Int32Type.instance.compose(pk.getKey()), entry.getKey())));
+
+        for (int executionCount = 0; executionCount < 1000; executionCount++)
+        {
+            // These keys have midrange tokens that select 3 of the 8 range indexes
+            DecoratedKey temp1 = makeKey(cfs.metadata(), getRandom().nextIntBetween(0, 20000));
+            DecoratedKey temp2 = makeKey(cfs.metadata(), getRandom().nextIntBetween(0, 20000));
+            DecoratedKey minimum = temp1.compareTo(temp2) <= 0 ? temp1 : temp2;
+            DecoratedKey maximum = temp1.compareTo(temp2) <= 0 ? temp2 : temp1;
+
+            var iterator = memtableIndex.iterator(minimum, maximum);
+
+            while (iterator.hasNext())
+            {
+                var termPair = iterator.next();
+                int term = termFromComparable(termPair.left);
+                // The iterator will return keys outside the range of min/max, so we need to filter here to
+                // get the correct keys
+                List<DecoratedKey> expectedPks = terms.get(term)
+                                                      .stream()
+                                                      .filter(pk -> pk.compareTo(minimum) >= 0 && pk.compareTo(maximum) <= 0)
+                                                      .sorted()
+                                                      .collect(Collectors.toList());
+                List<DecoratedKey> termPks = new ArrayList<>();
+                for (var pkWithFreq : termPair.right)
+                {
+                    DecoratedKey pk = pkWithFreq.pk.partitionKey();
+                    if (pk.compareTo(minimum) >= 0 && pk.compareTo(maximum) <= 0)
+                        termPks.add(pk);
+                }
+                assertEquals(expectedPks, termPks);
+            }
+        }
+    }
+
+    @Test
+    public void updateCollectionTest()
+    {
+        // Use one shard to test shared keys in the trie
+        memtableIndex = new TrieMemtableIndex(integerListIndexContext, memtable, 1);
+        assertEquals(0, memtable.getAllocator().onHeap().owns());
+        assertEquals(0, memtableIndex.estimatedOnHeapMemoryUsed());
+        assertEquals(0, memtableIndex.estimatedOffHeapMemoryUsed());
+        var trieMemoryIndex = (TrieMemoryIndex) memtableIndex.getRangeIndexes()[0];
+        assertEquals(0, trieMemoryIndex.estimatedTrieValuesMemoryUsed());
+
+        addRowWithCollection(1, 1, 2, 3); // row 1, values 1, 2, 3
+        addRowWithCollection(2, 4, 5, 6); // row 2, values 4, 5, 6
+        addRowWithCollection(3, 2, 6);    // row 3, values 2, 6
+
+        // 8 total pk entries at 44 bytes, 6 PrimaryKeys objects with
+        var expectedOnHeap = 8 * 44 + 6 * PrimaryKeys.unsharedHeapSize();
+        assertEquals(expectedOnHeap, trieMemoryIndex.estimatedTrieValuesMemoryUsed());
+
+        // Query values
+        assertEqualsQuery(2, 1, 3);
+        assertEqualsQuery(4, 2);
+        assertEqualsQuery(3, 1);
+        assertEqualsQuery(6, 2, 3);
+
+        assertEquals(expectedOnHeap, trieMemoryIndex.estimatedTrieValuesMemoryUsed());
+
+        // Update row 1 to remove 2 and 3, keep 1, add 7 and 8 (note we have to manually match the 1,2,3 from above)
+        updateRowWithCollection(1, List.of(1, 2, 3).iterator(), List.of(1, 7, 8).iterator());
+
+        // We net 1 new PrimaryKeys object.
+        expectedOnHeap += PrimaryKeys.unsharedHeapSize();
+        assertEquals(expectedOnHeap, trieMemoryIndex.estimatedTrieValuesMemoryUsed());
+
+        updateRowWithCollection(1, List.of(1, 7, 8).iterator(), List.of(1, 4, 8).iterator());
+
+        // We remove a PrimaryKeys object without adding any new keys to the trie.
+        expectedOnHeap -= PrimaryKeys.unsharedHeapSize();
+        assertEquals(expectedOnHeap, trieMemoryIndex.estimatedTrieValuesMemoryUsed());
+
+        // Run additional queries to ensure values
+        assertEqualsQuery(1, 1);
+        assertEqualsQuery(4, 1, 2);
+        assertEqualsQuery(2, 3);
+        assertEqualsQuery(3);
+
+        // Show that iteration works as expected and does not include any of the deleted terms.
+        var iter = memtableIndex.iterator(makeKey(cfs.metadata(), 1), makeKey(cfs.metadata(), 3));
+        assertNextEntryInIterator(iter, 1, 1);
+        assertNextEntryInIterator(iter, 2, 3);
+        assertNextEntryInIterator(iter, 4, 1, 2);
+        assertNextEntryInIterator(iter, 5, 2);
+        assertNextEntryInIterator(iter, 6, 2, 3);
+        assertNextEntryInIterator(iter, 8, 1);
+        assertFalse(iter.hasNext());
+    }
+
+    private void assertEqualsQuery(int value, int... partitionKeys)
+    {
+        // Build eq expression to search for the value
+        Expression expression = new Expression(integerListIndexContext);
+        expression.add(Operator.EQ, Int32Type.instance.decompose(value));
+        AbstractBounds<PartitionPosition> keyRange = new Range<>(partitioner.getMinimumToken().minKeyBound(),
+                                                                 partitioner.getMinimumToken().minKeyBound());
+        var result = memtableIndex.search(new QueryContext(), expression, keyRange, 0);
+        // Confirm the partition keys are as expected in the provided order and that we have no more results
+        for (int partitionKey : partitionKeys)
+            assertEquals(makeKey(cfs.metadata(), partitionKey), result.next().partitionKey());
+        assertFalse(result.hasNext());
+    }
+
+    private void assertNextEntryInIterator(Iterator<Pair<ByteComparable.Preencoded, List<MemoryIndex.PkWithFrequency>>> iter, int term, int... primaryKeys)
+    {
+        assertTrue(iter.hasNext());
+        Pair<ByteComparable.Preencoded, List<MemoryIndex.PkWithFrequency>> entry = iter.next();
+        assertEquals(term, termFromComparable(entry.left));
+        for (int i = 0; i < primaryKeys.length; i++)
+        {
+            assertFalse(entry.right.isEmpty());
+            assertEquals(makeKey(cfs.metadata(), primaryKeys[i]), entry.right.get(i).pk.partitionKey());
+        }
+    }
+
+    private Expression generateRandomExpression()
+    {
+        Expression expression = new Expression(indexContext);
+
+        int equality = getRandom().nextIntBetween(0, 100);
+        int lower = getRandom().nextIntBetween(0, 75);
+        int upper = getRandom().nextIntBetween(25, 100);
+        while (upper <= lower)
+            upper = getRandom().nextIntBetween(0, 100);
+
+        if (getRandom().nextBoolean())
+            expression.add(Operator.EQ, Int32Type.instance.decompose(equality));
+        else
+        {
+            boolean useLower = getRandom().nextBoolean();
+            boolean useUpper = getRandom().nextBoolean();
+            if (!useLower && !useUpper)
+                useLower = useUpper = true;
+            if (useLower)
+                expression.add(getRandom().nextBoolean() ? Operator.GT : Operator.GTE, Int32Type.instance.decompose(lower));
+            if (useUpper)
+                expression.add(getRandom().nextBoolean() ? Operator.LT : Operator.LTE, Int32Type.instance.decompose(upper));
+        }
+        return expression;
+    }
+
+    private AbstractBounds<PartitionPosition> generateRandomBounds(List<DecoratedKey> keys)
+    {
+        PartitionPosition leftBound = getRandom().nextBoolean() ? partitioner.getMinimumToken().minKeyBound()
+                                                                : keys.get(getRandom().nextIntBetween(0, keys.size() - 1)).getToken().minKeyBound();
+
+        PartitionPosition rightBound = getRandom().nextBoolean() ? partitioner.getMinimumToken().minKeyBound()
+                                                                 : keys.get(getRandom().nextIntBetween(0, keys.size() - 1)).getToken().maxKeyBound();
+
+        AbstractBounds<PartitionPosition> keyRange;
+
+        if (leftBound.isMinimum() && rightBound.isMinimum())
+            keyRange = new Range<>(leftBound, rightBound);
+        else
+        {
+            if (AbstractBounds.strictlyWrapsAround(leftBound, rightBound))
+            {
+                PartitionPosition temp = leftBound;
+                leftBound = rightBound;
+                rightBound = temp;
+            }
+            if (getRandom().nextBoolean())
+                keyRange = new Bounds<>(leftBound, rightBound);
+            else if (getRandom().nextBoolean())
+                keyRange = new ExcludingBounds<>(leftBound, rightBound);
+            else
+                keyRange = new IncludingExcludingBounds<>(leftBound, rightBound);
+        }
+        return keyRange;
+    }
+
+    private int termFromComparable(ByteComparable comparable)
+    {
+        ByteSource.Peekable peekable = ByteSource.peekable(comparable.asComparableBytes(ByteComparable.Version.OSS41));
+        return Int32Type.instance.compose(Int32Type.instance.fromComparableBytes(peekable, ByteComparable.Version.OSS41));
+    }
+
+    private Map<Integer, Set<DecoratedKey>> buildTermMap()
+    {
+        Map<Integer, Set<DecoratedKey>> terms = new HashMap<>();
+
+        for (int count = 0; count < 10000; count++)
+        {
+            int term = getRandom().nextIntBetween(0, 100);
+            Set<DecoratedKey> pks;
+            if (terms.containsKey(term))
+                pks = terms.get(term);
+            else
+            {
+                pks = new HashSet<>();
+                terms.put(term, pks);
+            }
+            DecoratedKey key = makeKey(cfs.metadata(), getRandom().nextIntBetween(0, 20000));
+            while (pks.contains(key))
+                key = makeKey(cfs.metadata(), getRandom().nextIntBetween(0, 20000));
+            pks.add(key);
+        }
+        return terms;
+    }
+
+    private void addRow(int pk, int value)
+    {
+        DecoratedKey key = makeKey(cfs.metadata(), pk);
+        memtableIndex.index(key,
+                            Clustering.EMPTY,
+                            Int32Type.instance.decompose(value),
+                            cfs.getCurrentMemtable(),
+                            new OpOrder().start());
+        keyMap.put(key, pk);
+    }
+
+    private void addRowWithCollection(int pk, Integer... value)
+    {
+        for (Integer v : value)
+            addRow(pk, v);
+    }
+
+    private void updateRowWithCollection(int pk, Iterator<Integer> oldValues, Iterator<Integer> newValues)
+    {
+        DecoratedKey key = makeKey(cfs.metadata(), pk);
+        memtableIndex.update(key,
+                             Clustering.EMPTY,
+                             Iterators.transform(oldValues, Int32Type.instance::decompose),
+                             Iterators.transform(newValues, Int32Type.instance::decompose),
+                             cfs.getCurrentMemtable(),
+                             new OpOrder().start());
+    }
+
+    private DecoratedKey makeKey(TableMetadata table, Integer partitionKey)
+    {
+        ByteBuffer key = table.partitionKeyType.fromString(partitionKey.toString());
+        return table.partitioner.decorateKey(key);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/VectorMemoryIndexTest.java b/test/unit/org/apache/cassandra/index/sai/memory/VectorMemoryIndexTest.java
deleted file mode 100644
index 2714cab1f23f..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/memory/VectorMemoryIndexTest.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.memory;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.Sets;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DataRange;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.PartitionRangeReadCommand;
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.filter.DataLimits;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.marshal.FloatType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.VectorType;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.BootStrapper;
-import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.ExcludingBounds;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.IncludingExcludingBounds;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
-import org.apache.cassandra.index.sai.plan.Expression;
-import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.RangeUtil;
-import org.apache.cassandra.inject.Injections;
-import org.apache.cassandra.inject.InvokePointBuilder;
-import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class VectorMemoryIndexTest extends SAITester
-{
-    private static final Injections.Counter indexSearchCounter = Injections.newCounter("IndexSearchCounter")
-                                                                           .add(InvokePointBuilder.newInvokePoint()
-                                                                                                  .onClass(TrieMemoryIndex.class)
-                                                                                                  .onMethod("search"))
-                                                                           .build();
-
-    private ColumnFamilyStore cfs;
-    private StorageAttachedIndex index;
-    private VectorMemoryIndex memtableIndex;
-    private IPartitioner partitioner;
-    private Map<DecoratedKey, Integer> keyMap;
-    private Map<Integer, ByteBuffer> rowMap;
-    private int dimensionCount;
-
-    @BeforeClass
-    public static void setShardCount()
-    {
-        MEMTABLE_SHARD_COUNT.setInt(8);
-    }
-
-    @Before
-    public void setup() throws Throwable
-    {
-        TokenMetadata metadata = StorageService.instance.getTokenMetadata();
-        metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddressAndPort());
-
-        dimensionCount = getRandom().nextIntBetween(2, 2048);
-        index = SAITester.createMockIndex(VectorType.getInstance(FloatType.instance, dimensionCount));
-        cfs = index.baseCfs();
-        partitioner = cfs.getPartitioner();
-        indexSearchCounter.reset();
-        keyMap = new TreeMap<>();
-        rowMap = new HashMap<>();
-
-        Injections.inject(indexSearchCounter);
-    }
-
-    @Test
-    public void randomQueryTest() throws Exception
-    {
-        memtableIndex = new VectorMemoryIndex(index);
-
-        for (int row = 0; row < getRandom().nextIntBetween(1000, 5000); row++)
-        {
-            int pk = getRandom().nextIntBetween(0, 10000);
-            while (rowMap.containsKey(pk))
-                pk = getRandom().nextIntBetween(0, 10000);
-            var value = randomVector();
-            rowMap.put(pk, value);
-            addRow(pk, value);
-        }
-
-        List<DecoratedKey> keys = new ArrayList<>(keyMap.keySet());
-
-        for (int executionCount = 0; executionCount < 1000; executionCount++)
-        {
-            Expression expression = generateRandomExpression();
-            AbstractBounds<PartitionPosition> keyRange = generateRandomBounds(keys);
-            Set<Integer> keysInRange = keys.stream().filter(keyRange::contains)
-                                           .map(k -> Int32Type.instance.compose(k.getKey()))
-                                           .collect(Collectors.toSet());
-
-            Set<Integer> foundKeys = new HashSet<>();
-            int limit = getRandom().nextIntBetween(1, 100);
-
-            ReadCommand command = PartitionRangeReadCommand.create(cfs.metadata(),
-                                                                   FBUtilities.nowInSeconds(),
-                                                                   ColumnFilter.all(cfs.metadata()),
-                                                                   RowFilter.none(),
-                                                                   DataLimits.cqlLimits(limit),
-                                                                   DataRange.allData(cfs.metadata().partitioner));
-
-            try (KeyRangeIterator iterator = memtableIndex.search(new QueryContext(command,
-                                                                                   DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.MILLISECONDS)),
-                                                                  expression, keyRange))
-            {
-                while (iterator.hasNext())
-                {
-                    PrimaryKey primaryKey = iterator.next();
-                    int key = Int32Type.instance.compose(primaryKey.partitionKey().getKey());
-                    assertFalse(foundKeys.contains(key));
-
-                    assertTrue(keyRange.contains(primaryKey.partitionKey()));
-                    assertTrue(rowMap.containsKey(key));
-                    foundKeys.add(key);
-                }
-            }
-            // with -Dcassandra.test.random.seed=260652334768666, there is one missing key
-            long expectedResult = Math.min(limit, keysInRange.size());
-            if (RangeUtil.coversFullRing(keyRange))
-                assertEquals("Missing key: " + Sets.difference(keysInRange, foundKeys), expectedResult, foundKeys.size());
-            else // if skip ANN, returned keys maybe larger than limit
-                assertTrue("Missing key: " + Sets.difference(keysInRange, foundKeys), expectedResult <= foundKeys.size());
-        }
-    }
-
-    @Test
-    public void indexIteratorTest()
-    {
-        // VSTODO
-    }
-
-    private Expression generateRandomExpression()
-    {
-        Expression expression = Expression.create(index);
-        expression.add(Operator.ANN, randomVector());
-        return expression;
-    }
-
-    private ByteBuffer randomVector() {
-        List<Float> rawVector = new ArrayList<>(dimensionCount);
-        for (int i = 0; i < dimensionCount; i++) {
-            rawVector.add(getRandom().nextFloat());
-        }
-        return VectorType.getInstance(FloatType.instance, dimensionCount).getSerializer().serialize(rawVector);
-    }
-
-    private AbstractBounds<PartitionPosition> generateRandomBounds(List<DecoratedKey> keys)
-    {
-        PartitionPosition leftBound = getRandom().nextBoolean() ? partitioner.getMinimumToken().minKeyBound()
-                                                                : keys.get(getRandom().nextIntBetween(0, keys.size() - 1)).getToken().minKeyBound();
-
-        PartitionPosition rightBound = getRandom().nextBoolean() ? partitioner.getMinimumToken().minKeyBound()
-                                                                 : keys.get(getRandom().nextIntBetween(0, keys.size() - 1)).getToken().maxKeyBound();
-
-        AbstractBounds<PartitionPosition> keyRange;
-
-        if (leftBound.isMinimum() && rightBound.isMinimum())
-            keyRange = new Range<>(leftBound, rightBound);
-        else
-        {
-            if (AbstractBounds.strictlyWrapsAround(leftBound, rightBound))
-            {
-                PartitionPosition temp = leftBound;
-                leftBound = rightBound;
-                rightBound = temp;
-            }
-            if (getRandom().nextBoolean())
-                keyRange = new Bounds<>(leftBound, rightBound);
-            else if (getRandom().nextBoolean())
-                keyRange = new ExcludingBounds<>(leftBound, rightBound);
-            else
-                keyRange = new IncludingExcludingBounds<>(leftBound, rightBound);
-        }
-        return keyRange;
-    }
-
-    private void addRow(int pk, ByteBuffer value)
-    {
-        DecoratedKey key = makeKey(cfs.metadata(), pk);
-        memtableIndex.add(key, Clustering.EMPTY, value);
-        keyMap.put(key, pk);
-    }
-
-    private DecoratedKey makeKey(TableMetadata table, Integer partitionKey)
-    {
-        ByteBuffer key = table.partitionKeyType.fromString(partitionKey.toString());
-        return table.partitioner.decorateKey(key);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/VectorMemtableIndexTest.java b/test/unit/org/apache/cassandra/index/sai/memory/VectorMemtableIndexTest.java
new file mode 100644
index 000000000000..1f38c80a0cb5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/memory/VectorMemtableIndexTest.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.VectorType;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.BootStrapper;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.ExcludingBounds;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.IncludingExcludingBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.cql.VectorTester;
+import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex;
+import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithScore;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.inject.InvokePointBuilder;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class VectorMemtableIndexTest extends SAITester
+{
+    private static final Injections.Counter indexSearchCounter = Injections.newCounter("IndexSearchCounter")
+                                                                           .add(InvokePointBuilder.newInvokePoint()
+                                                                                                  .onClass(TrieMemoryIndex.class)
+                                                                                                  .onMethod("search"))
+                                                                           .build();
+
+    private ColumnFamilyStore cfs;
+    private IndexContext indexContext;
+    private VectorMemtableIndex memtableIndex;
+    private IPartitioner partitioner;
+    private Map<DecoratedKey, Integer> keyMap;
+    private Map<Integer, ByteBuffer> rowMap;
+    private int dimensionCount;
+
+    @BeforeClass
+    public static void setShardCount()
+    {
+        MEMTABLE_SHARD_COUNT.setInt(8);
+    }
+
+    @Before
+    public void setup() throws Throwable
+    {
+        TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+        metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddressAndPort());
+
+        TableMetadata tableMetadata = TableMetadata.builder("ks", "tb")
+                                                   .addPartitionKeyColumn("pk", Int32Type.instance)
+                                                   .addRegularColumn("val", Int32Type.instance)
+                                                   .build();
+        cfs = MockSchema.newCFS(tableMetadata);
+        partitioner = cfs.getPartitioner();
+        dimensionCount = getRandom().nextIntBetween(2, 2048);
+        indexContext = SAITester.createIndexContext("index", VectorType.getInstance(FloatType.instance, dimensionCount), cfs);
+        indexSearchCounter.reset();
+        keyMap = new ConcurrentSkipListMap<>();
+        rowMap = new ConcurrentHashMap<>();
+
+        Injections.inject(indexSearchCounter);
+    }
+
+    @Test
+    public void randomQueryTest()
+    {
+        var memtable = cfs.getCurrentMemtable();
+        memtableIndex = new VectorMemtableIndex(indexContext, memtable);
+
+        // insert rows
+        int rowCount = ThreadLocalRandom.current().nextInt(1000, 5000);
+        IntStream.range(0, rowCount).parallel().forEach(i ->
+        {
+            var value = randomVectorSerialized();
+            while (true)
+            {
+                var pk = ThreadLocalRandom.current().nextInt(0, 10000);
+                if (rowMap.putIfAbsent(pk, value) == null)
+                {
+                    addRow(pk, value);
+                    break;
+                }
+            }
+        });
+        memtableIndex.cleanup();
+        // master list of (random) keys inserted
+        List<DecoratedKey> keys = new ArrayList<>(keyMap.keySet());
+
+        // execute queries both with and without brute force enabled
+        validate(keys, 1.0, 1.0);
+        // In scenarios where we have 5 or fewer vectors, brute force is always going to be used,
+        // so we do that here.
+        VectorTester.setMaxBruteForceRows(5);
+        // For these tests, we will push some queries that would typically go the brute force path to the graph path.
+        // This comes at the cost of some recall, which is why we only assert 50% recall on the individual queries
+        // and 90% recall on the overall test.
+        validate(keys, 0.5, 0.9);
+    }
+
+    private void validate(List<DecoratedKey> keys, double expectedIndividualQueryRecall, double expectedRecall)
+    {
+        var actualVectorsReturned = new AtomicInteger();
+        var expectedVectorsReturned = new AtomicInteger();
+        IntStream.range(0, 1_000).parallel().forEach(i ->
+        {
+            var orderer = randomVectorOrderer();
+            AbstractBounds<PartitionPosition> keyRange = generateRandomBounds(keys);
+            // compute keys in range of the bounds
+            Set<Integer> keysInRange = keys.stream().filter(keyRange::contains)
+                                           .map(k -> Int32Type.instance.compose(k.getKey()))
+                                           .collect(Collectors.toSet());
+
+            Set<Integer> foundKeys = new HashSet<>();
+            int limit = getRandom().nextIntBetween(1, 100);
+
+            long expectedResults = Math.min(limit, keysInRange.size());
+
+            // execute the random ANN expression, and check that we get back as many keys as we asked for
+            var iterators = memtableIndex.orderBy(new QueryContext(), orderer, null, keyRange, limit);
+            assertEquals(1, iterators.size());
+            try (var iterator = iterators.get(0))
+            {
+                PrimaryKeyWithScore lastKey = null;
+                while (iterator.hasNext() && foundKeys.size() < expectedResults)
+                {
+                    PrimaryKeyWithScore primaryKeyWithScore = (PrimaryKeyWithScore) iterator.next();
+                    if (lastKey != null)
+                        // This assertion only holds true as long as we query at most the expectedNumResults.
+                        // Once we query deeper, we might get a key with a higher score than the last key.
+                        // This is a direct consequence of the approximate part of ANN.
+                        // Note that PrimaryKeyWithScore is flipped to descending order, so we use >= here.
+                        assertTrue("Returned keys are not ordered by score", primaryKeyWithScore.compareTo(lastKey) >= 0);
+                    lastKey = primaryKeyWithScore;
+                    int key = Int32Type.instance.compose(primaryKeyWithScore.partitionKey().getKey());
+                    assertFalse(foundKeys.contains(key));
+
+                    assertTrue(keyRange.contains(primaryKeyWithScore.partitionKey()));
+                    assertTrue(rowMap.containsKey(key));
+                    foundKeys.add(key);
+                }
+                // Note that we weight each result evenly instead of each query evenly.
+                actualVectorsReturned.addAndGet(foundKeys.size());
+                expectedVectorsReturned.addAndGet((int) expectedResults);
+                if (foundKeys.size() < expectedResults)
+                    assertTrue("Expected at least " + expectedResults + " results but got " + foundKeys.size(),
+                               foundKeys.size() >= expectedResults * expectedIndividualQueryRecall);
+            }
+        });
+        assertTrue("Expected at least " + expectedVectorsReturned + " results but got " + actualVectorsReturned,
+                   actualVectorsReturned.get() >= expectedVectorsReturned.get() * expectedRecall);
+    }
+
+    @Test
+    public void indexIteratorTest()
+    {
+        // VSTODO
+    }
+
+    private Orderer randomVectorOrderer()
+    {
+        return new Orderer(indexContext, Operator.ANN, randomVectorSerialized(), null);
+    }
+
+    private ByteBuffer randomVectorSerialized() {
+        return CQLTester.randomVectorSerialized(dimensionCount);
+    }
+
+    private AbstractBounds<PartitionPosition> generateRandomBounds(List<DecoratedKey> keys)
+    {
+        PartitionPosition leftBound = getRandom().nextBoolean() ? partitioner.getMinimumToken().minKeyBound()
+                                                                : keys.get(getRandom().nextIntBetween(0, keys.size() - 1)).getToken().minKeyBound();
+
+        PartitionPosition rightBound = getRandom().nextBoolean() ? partitioner.getMinimumToken().minKeyBound()
+                                                                 : keys.get(getRandom().nextIntBetween(0, keys.size() - 1)).getToken().maxKeyBound();
+
+        AbstractBounds<PartitionPosition> keyRange;
+
+        if (leftBound.isMinimum() && rightBound.isMinimum())
+            keyRange = new Range<>(leftBound, rightBound);
+        else
+        {
+            if (AbstractBounds.strictlyWrapsAround(leftBound, rightBound))
+            {
+                PartitionPosition temp = leftBound;
+                leftBound = rightBound;
+                rightBound = temp;
+            }
+            if (getRandom().nextBoolean())
+                keyRange = new Bounds<>(leftBound, rightBound);
+            else if (getRandom().nextBoolean())
+                keyRange = new ExcludingBounds<>(leftBound, rightBound);
+            else
+                keyRange = new IncludingExcludingBounds<>(leftBound, rightBound);
+        }
+        return keyRange;
+    }
+
+    private void addRow(int pk, ByteBuffer value)
+    {
+        DecoratedKey key = makeKey(cfs.metadata(), pk);
+        memtableIndex.index(key,
+                            Clustering.EMPTY,
+                            value,
+                            cfs.getCurrentMemtable(),
+                            new OpOrder().start());
+        keyMap.put(key, pk);
+    }
+
+    private DecoratedKey makeKey(TableMetadata table, Integer partitionKey)
+    {
+        ByteBuffer key = table.partitionKeyType.fromString(partitionKey.toString());
+        return table.partitioner.decorateKey(key);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/AbstractMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/AbstractMetricsTest.java
index 24134bccdef7..fa86ce68f952 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/AbstractMetricsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/AbstractMetricsTest.java
@@ -17,10 +17,17 @@
  */
 package org.apache.cassandra.index.sai.metrics;
 
+import java.util.concurrent.TimeUnit;
+import javax.management.ObjectName;
+
 import org.junit.Before;
 import org.junit.Ignore;
 
 import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 @Ignore
 public abstract class AbstractMetricsTest extends SAITester
@@ -34,4 +41,46 @@ public void initializeTest() throws Throwable
 
         createMBeanServerConnection();
     }
+
+    protected void waitForIndexCompaction(String keyspace, String table, String index)
+    {
+        waitForAssert(() -> {
+            try
+            {
+                assertEquals(1L, getMetricValue(objectName("CompactionCount", keyspace, table, index, "IndexMetrics")));
+            }
+            catch (Throwable ex)
+            {
+                throw Throwables.unchecked(ex);
+            }
+        }, 60, TimeUnit.SECONDS);
+    }
+
+    protected void waitForVerifyHistogram(ObjectName name, long count)
+    {
+        waitForAssert(() -> {
+            try
+            {
+                assertEquals(count, jmxConnection.getAttribute(name, "Count"));
+            }
+            catch (Throwable ex)
+            {
+                throw Throwables.unchecked(ex);
+            }
+        }, 10, TimeUnit.SECONDS);
+    }
+
+    protected void waitForGreaterThanZero(ObjectName name)
+    {
+        waitForAssert(() -> {
+            try
+            {
+                assertTrue(((Number) getMetricValue(name)).doubleValue() > 0);
+            }
+            catch (Throwable ex)
+            {
+                throw Throwables.unchecked(ex);
+            }
+        }, 160, TimeUnit.SECONDS);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/FinalSegmentFlushingFailureTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/FinalSegmentFlushingFailureTest.java
index 293c05d4162f..e52a2cf0ac04 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/FinalSegmentFlushingFailureTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/FinalSegmentFlushingFailureTest.java
@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.index.sai.metrics;
 
-public class FinalSegmentFlushingFailureTest extends SegmentFlushingFailureTester
+public class FinalSegmentFlushingFailureTest extends SegmentFlushingFailureTest
 {
     @Override
     protected long expectedBytesLimit()
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/IndexGroupMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/IndexGroupMetricsTest.java
index 355243a7d05c..dbd9bc26f0d1 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/IndexGroupMetricsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/IndexGroupMetricsTest.java
@@ -17,14 +17,20 @@
  */
 package org.apache.cassandra.index.sai.metrics;
 
+import javax.management.InstanceNotFoundException;
+
 import org.junit.Before;
 import org.junit.Test;
 
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNull;
 
 public class IndexGroupMetricsTest extends AbstractMetricsTest
 {
@@ -39,11 +45,12 @@ public void setup() throws Exception
     }
 
     @Test
-    public void verifyIndexGroupMetrics() throws Throwable
+    public void verifyIndexGroupMetrics()
     {
         // create first index
         createTable(CREATE_TABLE_TEMPLATE);
         String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        IndexContext v1IndexContext = createIndexContext(v1IndexName, Int32Type.instance);
 
         // no open files
         assertEquals(0, getOpenIndexFiles());
@@ -52,29 +59,51 @@ public void verifyIndexGroupMetrics() throws Throwable
         int sstables = 10;
         for (int i = 0; i < sstables; i++)
         {
-            execute("INSERT INTO %s (id1, v1) VALUES ('0', 0)");
+            execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
             flush();
         }
 
         // with 10 sstable
         int indexopenFileCountWithOnlyNumeric = getOpenIndexFiles();
-        assertEquals(sstables * (Version.LATEST.onDiskFormat().openFilesPerSSTableIndex(false) +
-                                 Version.LATEST.onDiskFormat().openFilesPerColumnIndex()),
+        assertEquals(sstables * (Version.latest().onDiskFormat().openFilesPerSSTable() +
+                                 Version.latest().onDiskFormat().openFilesPerIndex(v1IndexContext)),
                      indexopenFileCountWithOnlyNumeric);
 
         long diskUsageWithOnlyNumeric = getDiskUsage();
         assertNotEquals(0, diskUsageWithOnlyNumeric);
 
+        // create second index
+        String v2IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+        IndexContext v2IndexContext = createIndexContext(v2IndexName, UTF8Type.instance);
+
+        // same number of sstables, but more string index files.
+        int stringIndexOpenFileCount = sstables * V1OnDiskFormat.instance.openFilesPerIndex(v2IndexContext);
+        assertEquals(indexopenFileCountWithOnlyNumeric, getOpenIndexFiles() - stringIndexOpenFileCount);
+
+        // Index Group disk usage doesn't change with more indexes
+        long diskUsageWithBothIndexes = getDiskUsage();
+        assertEquals(diskUsageWithBothIndexes, diskUsageWithOnlyNumeric);
+
         // compaction should reduce open files
         compact();
 
-        assertEquals(Version.LATEST.onDiskFormat().openFilesPerSSTableIndex(false) +
-                     Version.LATEST.onDiskFormat().openFilesPerColumnIndex(),
+        long perSSTableFileDiskUsage = getDiskUsage();
+        assertEquals(Version.latest().onDiskFormat().openFilesPerSSTable() +
+                     Version.latest().onDiskFormat().openFilesPerIndex(v2IndexContext) +
+                     Version.latest().onDiskFormat().openFilesPerIndex(v1IndexContext),
+                     getOpenIndexFiles());
+
+        // drop string index, reduce open string index files, per-sstable file disk usage remains the same
+        dropIndex("DROP INDEX %s." + v2IndexName);
+        assertEquals(Version.latest().onDiskFormat().openFilesPerSSTable() +
+                     Version.latest().onDiskFormat().openFilesPerIndex(v1IndexContext),
                      getOpenIndexFiles());
+        assertEquals(perSSTableFileDiskUsage, getDiskUsage());
 
         // drop last index, no open index files
         dropIndex("DROP INDEX %s." + v1IndexName);
-        assertNull(getCurrentIndexGroup());
+        assertThatThrownBy(this::getOpenIndexFiles).hasRootCauseInstanceOf(InstanceNotFoundException.class);
+        assertThatThrownBy(this::getDiskUsage).hasRootCauseInstanceOf(InstanceNotFoundException.class);
     }
 
     protected int getOpenIndexFiles()
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/IndexMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/IndexMetricsTest.java
index f2c20e18f6e6..dafb5c920618 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/IndexMetricsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/IndexMetricsTest.java
@@ -17,13 +17,23 @@
  */
 package org.apache.cassandra.index.sai.metrics;
 
+import java.util.concurrent.TimeUnit;
+
 import org.junit.Test;
 
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.Throwables;
+import org.assertj.core.api.Assertions;
+
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class IndexMetricsTest extends AbstractMetricsTest
 {
+
     private static final String TABLE = "table_name";
     private static final String INDEX = "table_name_index";
 
@@ -32,7 +42,7 @@ public class IndexMetricsTest extends AbstractMetricsTest
     private static final String CREATE_INDEX_TEMPLATE = "CREATE CUSTOM INDEX IF NOT EXISTS " + INDEX + " ON %s." + TABLE + "(%s) USING 'StorageAttachedIndex'";
 
     @Test
-    public void testSameIndexNameAcrossKeyspaces() throws Throwable
+    public void testSameIndexNameAcrossKeyspaces()
     {
         String keyspace1 = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
         String keyspace2 = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
@@ -56,20 +66,134 @@ public void testSameIndexNameAcrossKeyspaces() throws Throwable
     }
 
     @Test
-    public void testMetricRelease() throws Throwable
+    public void testMetricRelease()
     {
-        String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
-
-        createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace));
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, keyspace, "v1"));
+        String table = createTable("CREATE TABLE %s (ID1 TEXT PRIMARY KEY, v1 INT, v2 TEXT) WITH compaction = " +
+                                   "{'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }");
+        String index = createIndex("CREATE CUSTOM INDEX IF NOT EXISTS ON %s (v1) USING 'StorageAttachedIndex'");
 
-        execute("INSERT INTO " + keyspace + '.' + TABLE + " (id1, v1, v2) VALUES ('0', 0, '0')");
-        assertEquals(1L, getMetricValue(objectName("LiveMemtableIndexWriteCount", keyspace, TABLE, INDEX, "IndexMetrics")));
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
+        assertEquals(1L, getMetricValue(objectName("LiveMemtableIndexWriteCount", KEYSPACE, table, index, "IndexMetrics")));
 
-        dropIndex(String.format("DROP INDEX %s." + INDEX, keyspace));
+        dropIndex("DROP INDEX %s." + index);
 
         // once the index is dropped, make sure MBeans are no longer accessible
-        assertThatThrownBy(() -> getMetricValue(objectName("LiveMemtableIndexWriteCount", keyspace, TABLE, INDEX, "IndexMetrics")))
+        assertThatThrownBy(() -> getMetricValue(objectName("LiveMemtableIndexWriteCount", KEYSPACE, table, index, "IndexMetrics")))
                 .hasCauseInstanceOf(javax.management.InstanceNotFoundException.class);
     }
+
+    @Test
+    public void testMetricsThroughWriteLifecycle()
+    {
+        String table = createTable("CREATE TABLE %s (ID1 TEXT PRIMARY KEY, v1 INT, v2 TEXT) WITH compaction = " +
+                                   "{'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }");
+        String index = createIndex("CREATE CUSTOM INDEX IF NOT EXISTS ON %s (v1) USING 'StorageAttachedIndex'");
+
+        int rowCount = 10;
+        for (int i = 0; i < rowCount; i++)
+            execute("INSERT INTO %s (id1, v1, v2) VALUES (?, ?, '0')", Integer.toString(i), i);
+
+        assertEquals(10L, getMetricValue(objectName("LiveMemtableIndexWriteCount", KEYSPACE, table, index, "IndexMetrics")));
+        assertTrue((Long)getMetricValue(objectName("MemtableOnHeapIndexBytes", KEYSPACE, table, index, "IndexMetrics")) > 0);
+        assertEquals(0L, getMetricValue(objectName("MemtableIndexFlushCount", KEYSPACE, table, index, "IndexMetrics")));
+
+        long bytes = (long) getMetricValue(objectName("MemtableOffHeapIndexBytes", KEYSPACE, table, index, "IndexMetrics"));
+        if (DatabaseDescriptor.getMemtableAllocationType().toBufferType() == BufferType.ON_HEAP)
+            Assertions.assertThat(bytes).isZero();
+        else
+            Assertions.assertThat(bytes).isPositive();
+
+        waitForAssert(() -> {
+            try
+            {
+                assertEquals(10L, getMBeanAttribute(objectName("MemtableIndexWriteLatency", KEYSPACE, table, index, "IndexMetrics"), "Count"));
+            }
+            catch (Throwable ex)
+            {
+                throw Throwables.unchecked(ex);
+            }
+        }, 60, TimeUnit.SECONDS);
+
+        assertEquals(0L, getMetricValue(objectName("SSTableCellCount", KEYSPACE, table, index, "IndexMetrics")));
+        assertEquals(0L, getMetricValue(objectName("DiskUsedBytes", KEYSPACE, table, index, "IndexMetrics")));
+        assertEquals(0L, getMetricValue(objectName("CompactionCount", KEYSPACE, table, index, "IndexMetrics")));
+
+        waitForVerifyHistogram(objectName("MemtableIndexFlushCellsPerSecond", KEYSPACE, table, index, "IndexMetrics"), 0);
+
+        flush(KEYSPACE, table);
+
+        assertEquals(0L, getMetricValue(objectName("LiveMemtableIndexWriteCount", KEYSPACE, table, index, "IndexMetrics")));
+        assertEquals(0L, getMetricValue(objectName("MemtableOnHeapIndexBytes", KEYSPACE, table, index, "IndexMetrics")));
+        assertEquals(0L, getMetricValue(objectName("MemtableOffHeapIndexBytes", KEYSPACE, table, index, "IndexMetrics")));
+        assertEquals(1L, getMetricValue(objectName("MemtableIndexFlushCount", KEYSPACE, table, index, "IndexMetrics")));
+        assertEquals(10L, getMetricValue(objectName("SSTableCellCount", KEYSPACE, table, index, "IndexMetrics")));
+        assertTrue((Long)getMetricValue(objectName("DiskUsedBytes", KEYSPACE, table, index, "IndexMetrics")) > 0);
+        assertEquals(0L, getMetricValue(objectName("CompactionCount", KEYSPACE, table, index, "IndexMetrics")));
+
+        waitForVerifyHistogram(objectName("MemtableIndexFlushCellsPerSecond", KEYSPACE, table, index, "IndexMetrics"), 1);
+
+        compact(KEYSPACE, table);
+
+        waitForIndexCompaction(KEYSPACE, table, index);
+
+        waitForTableIndexesQueryable(KEYSPACE, table);
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1 >= 0");
+        assertEquals(rowCount, rows.all().size());
+
+        assertEquals(0L, getMetricValue(objectName("LiveMemtableIndexWriteCount", KEYSPACE, table, index, "IndexMetrics")));
+        assertEquals(1L, getMetricValue(objectName("MemtableIndexFlushCount", KEYSPACE, table, index, "IndexMetrics")));
+        assertEquals(10L, getMetricValue(objectName("SSTableCellCount", KEYSPACE, table, index, "IndexMetrics")));
+        assertTrue((Long)getMetricValue(objectName("DiskUsedBytes", KEYSPACE, table, index, "IndexMetrics")) > 0);
+        assertEquals(1L, getMetricValue(objectName("CompactionCount", KEYSPACE, table, index, "IndexMetrics")));
+
+        waitForVerifyHistogram(objectName("CompactionSegmentCellsPerSecond", KEYSPACE, table, index, "IndexMetrics"), 1);
+    }
+
+    private void assertIndexQueryCount(String index, long expectedCount)
+    {
+        assertEquals(expectedCount,
+                     getMetricValue(objectName("QueriesCount", KEYSPACE, currentTable(), index, "IndexMetrics")));
+    }
+
+    @Test
+    public void testQueriesCount()
+    {
+        createTable("CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT, v2 TEXT, v3 VECTOR<FLOAT, 2>)");
+        String indexV1 = createIndex("CREATE CUSTOM INDEX ON %s (v1) USING 'StorageAttachedIndex'");
+
+        int rowCount = 10;
+        for (int i = 0; i < rowCount; i++)
+            execute("INSERT INTO %s (id1, v1, v2, v3) VALUES (?, ?, '0', ?)", Integer.toString(i), i, vector(i, i));
+
+        assertIndexQueryCount(indexV1, 0L);
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1 >= 0");
+        assertEquals(rowCount, rows.all().size());
+        assertIndexQueryCount(indexV1, 1L);
+
+        executeNet("SELECT id1 FROM %s WHERE (v1 >= 0 OR v1 = 4) AND v2 = '2' ALLOW FILTERING");
+        assertIndexQueryCount(indexV1, 2L);
+
+        String indexV2 = createIndex("CREATE CUSTOM INDEX ON %s (v2) USING 'StorageAttachedIndex'");
+        executeNet("SELECT id1 FROM %s WHERE (v1 >= 0 OR v1 = 4)");
+        assertIndexQueryCount(indexV1, 3L);
+        assertIndexQueryCount(indexV2, 0L);
+
+        executeNet("SELECT id1 FROM %s WHERE v2 = '2'");
+        assertIndexQueryCount(indexV2, 1L);
+        executeNet("SELECT id1 FROM %s WHERE (v1 >= 0 OR v1 = 4) AND v2 = '2'");
+        assertIndexQueryCount(indexV1, 4L);
+        assertIndexQueryCount(indexV2, 1L);
+        executeNet("SELECT id1 FROM %s WHERE (v1 >= 0 OR v1 = 4) ORDER BY v2 LIMIT 10");
+        assertIndexQueryCount(indexV1, 4L);
+        assertIndexQueryCount(indexV2, 2L);
+
+        String indexV3 = createIndex("CREATE CUSTOM INDEX ON %s (v3) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function': 'euclidean'}");
+        assertIndexQueryCount(indexV3, 0L);
+        executeNet("SELECT id1 FROM %s WHERE v2 = '2' ORDER BY v3 ANN OF [5,0] LIMIT 10");
+        assertIndexQueryCount(indexV1, 4L);
+        assertIndexQueryCount(indexV2, 2L);
+        assertIndexQueryCount(indexV3, 1L);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/QueryEventListeners.java b/test/unit/org/apache/cassandra/index/sai/metrics/QueryEventListeners.java
new file mode 100644
index 000000000000..3da6ab054db3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/QueryEventListeners.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import java.util.concurrent.TimeUnit;
+
+public class QueryEventListeners
+{
+    public static final ColumnQueryMetrics NO_OP_BKD_LISTENER = new NoOpBkdIndexEventListener();
+
+    public static final ColumnQueryMetrics NO_OP_TRIE_LISTENER = new NoOpTrieIndexEventListener();
+
+    public static final QueryEventListener.PostingListEventListener NO_OP_POSTINGS_LISTENER = new NoOpPostingListEventListener();
+
+    private static class NoOpTrieIndexEventListener extends ColumnQueryMetrics implements QueryEventListener.TrieIndexEventListener
+    {
+        NoOpTrieIndexEventListener()
+        {
+            super("ks", "tb", null);
+        }
+
+        @Override
+        public void onSegmentHit()
+        {}
+
+        @Override
+        public void onTraversalComplete(long traversalTotalTime, TimeUnit unit)
+        {}
+
+        @Override
+        public QueryEventListener.PostingListEventListener postingListEventListener()
+        {
+            return NO_OP_POSTINGS_LISTENER;
+        }
+    }
+
+    private static class NoOpBkdIndexEventListener extends ColumnQueryMetrics implements QueryEventListener.BKDIndexEventListener
+    {
+        NoOpBkdIndexEventListener()
+        {
+            super("ks", "tb", null);
+        }
+
+        @Override
+        public void onIntersectionComplete(long intersectionTotalTime, TimeUnit unit)
+        {}
+
+        @Override
+        public void onIntersectionEarlyExit()
+        {}
+
+        @Override
+        public void postingListsHit(int count)
+        {}
+
+        @Override
+        public void onSegmentHit()
+        {}
+
+        @Override
+        public QueryEventListener.PostingListEventListener postingListEventListener()
+        {
+            return NO_OP_POSTINGS_LISTENER;
+        }
+    }
+
+    public static class NoOpPostingListEventListener implements QueryEventListener.PostingListEventListener
+    {
+        @Override
+        public void onAdvance() { }
+
+        @Override
+        public void postingDecoded(long postingsDecoded) { }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/QueryMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/QueryMetricsTest.java
index 846024ebe097..e6e3b36781fd 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/QueryMetricsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/QueryMetricsTest.java
@@ -17,16 +17,23 @@
  */
 package org.apache.cassandra.index.sai.metrics;
 
+import java.util.concurrent.ThreadLocalRandom;
 import javax.management.InstanceNotFoundException;
+import javax.management.JMX;
+import javax.management.ObjectName;
 
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
 import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 
+import static org.apache.cassandra.index.sai.metrics.TableQueryMetrics.TABLE_QUERY_METRIC_TYPE;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class QueryMetricsTest extends AbstractMetricsTest
 {
@@ -34,6 +41,9 @@ public class QueryMetricsTest extends AbstractMetricsTest
                                                         "{'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
     private static final String CREATE_INDEX_TEMPLATE = "CREATE CUSTOM INDEX IF NOT EXISTS %s ON %s.%s(%s) USING 'StorageAttachedIndex'";
 
+    private static final String PER_QUERY_METRIC_TYPE = "PerQuery";
+    private static final String GLOBAL_METRIC_TYPE = "ColumnQueryMetrics";
+
     @Rule
     public ExpectedException exception = ExpectedException.none();
 
@@ -52,9 +62,9 @@ public void testSameIndexNameAcrossKeyspaces()
         createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace2, table));
         createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace2, table, "v1"));
 
-        execute("INSERT INTO " + keyspace1 + '.' + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
+        execute("INSERT INTO " + keyspace1 + "." + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
 
-        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace1 + '.' + table + " WHERE v1 = 0");
+        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace1 + "." + table + " WHERE v1 = 0");
         assertEquals(1, rows.all().size());
 
         assertEquals(1L, getTableQueryMetrics(keyspace1, table, "TotalQueriesCompleted"));
@@ -62,13 +72,13 @@ public void testSameIndexNameAcrossKeyspaces()
         assertEquals(0L, getTableQueryMetrics(keyspace2, table, "TotalQueriesCompleted"));
         assertEquals(0L, getTableQueryMetrics(keyspace2, table, "PostFilteringReadLatency"));
 
-        execute("INSERT INTO " + keyspace2 + '.' + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
-        execute("INSERT INTO " + keyspace2 + '.' + table + " (id1, v1, v2) VALUES ('1', 1, '1')");
+        execute("INSERT INTO " + keyspace2 + "." + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
+        execute("INSERT INTO " + keyspace2 + "." + table + " (id1, v1, v2) VALUES ('1', 1, '1')");
 
-        rows = executeNet("SELECT id1 FROM " + keyspace1 + '.' + table + " WHERE v1 = 0");
+        rows = executeNet("SELECT id1 FROM " + keyspace1 + "." + table + " WHERE v1 = 0");
         assertEquals(1, rows.all().size());
 
-        rows = executeNet("SELECT id1 FROM " + keyspace2 + '.' + table + " WHERE v1 = 1");
+        rows = executeNet("SELECT id1 FROM " + keyspace2 + "." + table + " WHERE v1 = 1");
         assertEquals(1, rows.all().size());
 
         assertEquals(2L, getTableQueryMetrics(keyspace1, table, "TotalQueriesCompleted"));
@@ -78,7 +88,7 @@ public void testSameIndexNameAcrossKeyspaces()
     }
 
     @Test
-    public void testMetricRelease() throws Throwable
+    public void testMetricRelease()
     {
         String table = "test_metric_release";
         String index = "test_metric_release_index";
@@ -88,16 +98,306 @@ public void testMetricRelease() throws Throwable
         createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
         createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v1"));
 
-        execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
+        execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
 
-        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace + '.' + table + " WHERE v1 = 0");
+        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v1 = 0");
         assertEquals(1, rows.all().size());
 
         assertEquals(1L, getTableQueryMetrics(keyspace, table, "TotalQueriesCompleted"));
 
-        // If we drop the last index on the table we should no longer see the table-level state metrics:
+        // Even if we drop the last index on the table, we should finally fail to find table-level metrics:
         dropIndex(String.format("DROP INDEX %s." + index, keyspace));
-        assertThatThrownBy(() -> getTableQueryMetrics(keyspace, table, "TotalQueriesCompleted")).hasCauseInstanceOf(InstanceNotFoundException.class);
+        assertThatThrownBy(() -> getTableQueryMetrics(keyspace, table, "TotalIndexCount")).hasRootCauseInstanceOf(InstanceNotFoundException.class);
+
+        // When the whole table is dropped, we should finally fail to find table-level metrics:
+        dropTable(String.format("DROP TABLE %s." + table, keyspace));
+        assertThatThrownBy(() -> getTableQueryMetrics(keyspace, table, "TotalQueriesCompleted")).hasRootCauseInstanceOf(InstanceNotFoundException.class);
+    }
+
+    @Test
+    public void testIndexQueryWithPartitionKey()
+    {
+        String table = "test_range_key_type_with_index";
+        String index = "test_range_key_type_with_index_index";
+
+        String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
+
+        createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v1"));
+
+        int rowsWrittenPerSSTable = 10;
+        int numberOfSSTable = 5;
+        int rowsWritten = 0;
+        int i = 0;
+        for (int j = 0; j < numberOfSSTable; j++)
+        {
+            rowsWritten += rowsWrittenPerSSTable;
+            for (; i < rowsWritten; i++)
+            {
+                execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES (?, ?, '0')", Integer.toString(i), i);
+            }
+            flush(keyspace, table);
+        }
+
+        ResultSet rows2 = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE id1 = '36' and v1 < 51");
+        assertEquals(1, rows2.all().size());
+
+        ResultSet rows3 = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE id1 = '49' and v1 < 51 ALLOW FILTERING");
+        assertEquals(1, rows3.all().size());
+
+        ResultSet rows4 = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE id1 = '21' and v1 >= 0 and v1 < 51 ALLOW FILTERING");
+        assertEquals(1, rows4.all().size());
+
+        ResultSet rows5 = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE id1 = '35' and v1 > 0");
+        assertEquals(1, rows5.all().size());
+
+        ResultSet rows6 = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v1 > 0 and id1 = '20'");
+        assertEquals(1, rows6.all().size());
+
+        ObjectName oName = objectNameNoIndex("SSTableIndexesHit", keyspace, table, PER_QUERY_METRIC_TYPE);
+        CassandraMetricsRegistry.JmxHistogramMBean o = JMX.newMBeanProxy(jmxConnection, oName, CassandraMetricsRegistry.JmxHistogramMBean.class);
+
+        assertTrue(o.getMean() < 2);
+    }
+
+    @Test
+    public void testKDTreeQueryMetricsWithSingleIndex()
+    {
+        String table = "test_metrics_through_write_lifecycle";
+        String index = "test_metrics_through_write_lifecycle_index";
+
+        String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
+
+        createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v1"));
+
+        int resultCounter = 0;
+        int queryCounter = 0;
+
+        int rowsWritten = 10;
+
+        for (int i = 0; i < rowsWritten; i++)
+        {
+            execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES (?, ?, '0')", Integer.toString(i), i);
+        }
+
+        flush(keyspace, table);
+        compact(keyspace, table);
+        waitForIndexCompaction(keyspace, table, index);
+
+        waitForTableIndexesQueryable(keyspace, table);
+
+        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v1 >= 0");
+
+        int actualRows = rows.all().size();
+        assertEquals(rowsWritten, actualRows);
+        resultCounter += actualRows;
+        queryCounter++;
+
+        rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v1 >= 5");
+
+        actualRows = rows.all().size();
+        assertEquals(5, actualRows);
+        resultCounter += actualRows;
+        queryCounter++;
+
+        assertEquals(2L, getPerQueryMetrics(keyspace, table, "SSTableIndexesHit"));
+        assertEquals(2L, getPerQueryMetrics(keyspace, table, "IndexSegmentsHit"));
+        assertEquals(2L, getTableQueryMetrics(keyspace, table, "TotalQueriesCompleted"));
+
+        // run several times to get buffer faults across the metrics
+        for (int x = 0; x < 20; x++)
+        {
+            rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v1 >= 5");
+
+            actualRows = rows.all().size();
+            assertEquals(5, actualRows);
+            resultCounter += actualRows;
+            queryCounter++;
+        }
+
+        // column metrics
+
+        waitForGreaterThanZero(objectNameNoIndex("QueryLatency", keyspace, table, PER_QUERY_METRIC_TYPE));
+
+        waitForEquals(objectNameNoIndex("TotalPartitionReads", keyspace, table, TableQueryMetrics.TABLE_QUERY_METRIC_TYPE), resultCounter);
+        waitForEquals(objectName("KDTreeIntersectionLatency", keyspace, table, index, GLOBAL_METRIC_TYPE), queryCounter);
+    }
+
+    @Test
+    public void testKDTreePostingsQueryMetricsWithSingleIndex()
+    {
+        String table = "test_kdtree_postings_metrics_through_write_lifecycle";
+        String v1Index = "test_kdtree_postings_metrics_through_write_lifecycle_v1_index";
+        String v2Index = "test_kdtree_postings_metrics_through_write_lifecycle_v2_index";
+
+        String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
+
+        createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
+
+        createIndex(String.format(CREATE_INDEX_TEMPLATE + " WITH OPTIONS = {'bkd_postings_min_leaves' : 1}", v1Index, keyspace, table, "v1"));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, v2Index, keyspace, table, "v2"));
+
+        int rowsWritten = 50;
+
+        for (int i = 0; i < rowsWritten; i++)
+        {
+            execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES (?, ?, ?)", Integer.toString(i), i, Integer.toString(i));
+        }
+
+        flush(keyspace, table);
+        compact(keyspace, table);
+        waitForIndexCompaction(keyspace, table, v1Index);
+
+        waitForTableIndexesQueryable(keyspace, table);
+
+        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v1 >= 0");
+
+        int actualRows = rows.all().size();
+        assertEquals(rowsWritten, actualRows);
+
+        assertTrue(((Number) getMetricValue(objectName("NumPostings", keyspace, table, v1Index, "KDTreePostings"))).longValue() > 0);
+
+        waitForVerifyHistogram(objectNameNoIndex("KDTreePostingsNumPostings", keyspace, table, PER_QUERY_METRIC_TYPE), 1);
+
+        // V2 index is very selective, so it should lead the union merge process, causing V1 index to be not used at all.
+        execute("SELECT id1 FROM " + keyspace + "." + table + " WHERE v1 >= 0 AND v1 <= 1000 AND v2 = '5' ALLOW FILTERING");
+
+        waitForVerifyHistogram(objectNameNoIndex("KDTreePostingsSkips", keyspace, table, PER_QUERY_METRIC_TYPE), 2);
+    }
+
+    @Test
+    public void testInvertedIndexQueryMetricsWithSingleIndex()
+    {
+        String table = "test_invertedindex_metrics_through_write_lifecycle";
+        String index = "test_invertedindex_metrics_through_write_lifecycle_index";
+
+        String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
+        createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v2"));
+
+        int resultCounter = 0;
+        int queryCounter = 0;
+
+        int rowsWritten = 10;
+
+        for (int i = 0; i < rowsWritten; i++)
+        {
+            execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES (?, ?, ?)", Integer.toString(i), i, Integer.toString(i));
+        }
+
+        flush(keyspace, table);
+        compact(keyspace, table);
+        waitForIndexCompaction(keyspace, table, index);
+
+        waitForTableIndexesQueryable(keyspace, table);
+
+        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v2 = '0'");
+
+        int actualRows = rows.all().size();
+        assertEquals(1, actualRows);
+        resultCounter += actualRows;
+        queryCounter++;
+
+        rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v2 = '5'");
+
+        actualRows = rows.all().size();
+        assertEquals(1, actualRows);
+        resultCounter += actualRows;
+        queryCounter++;
+
+        assertEquals(2L, getPerQueryMetrics(keyspace, table, "SSTableIndexesHit"));
+        assertEquals(2L, getPerQueryMetrics(keyspace, table, "IndexSegmentsHit"));
+        assertEquals(2L, getTableQueryMetrics(keyspace, table, "TotalQueriesCompleted"));
+
+        // run several times to get buffer faults across the metrics
+        for (int x = 0; x < 20; x++)
+        {
+            rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v2 = '" + ThreadLocalRandom.current().nextInt(0, 9) + "'");
+
+            actualRows = rows.all().size();
+            assertEquals(1, actualRows);
+            resultCounter += actualRows;
+            queryCounter++;
+        }
+
+        waitForGreaterThanZero(objectName("TermsLookupLatency", keyspace, table, index, GLOBAL_METRIC_TYPE));
+
+        waitForGreaterThanZero(objectNameNoIndex("QueryLatency", keyspace, table, PER_QUERY_METRIC_TYPE));
+
+        waitForEquals(objectNameNoIndex("TotalPartitionReads", keyspace, table, TableQueryMetrics.TABLE_QUERY_METRIC_TYPE), resultCounter);
+    }
+
+    @Test
+    public void testKDTreePartitionsReadAndRowsFiltered()
+    {
+        String table = "test_rows_filtered_large_partition";
+        String index = "test_rows_filtered_large_partition_index";
+
+        String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
+
+        createTable(String.format("CREATE TABLE %s.%s (pk int, ck int, v1 int, PRIMARY KEY (pk, ck)) " +
+                                  "WITH compaction = {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }", keyspace,  table));
+
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v1"));
+
+        execute("INSERT INTO " + keyspace + "." + table + "(pk, ck, v1) VALUES (0, 0, 0)");
+        execute("INSERT INTO " + keyspace + "." + table + "(pk, ck, v1) VALUES (1, 1, 1)");
+        execute("INSERT INTO " + keyspace + "." + table + "(pk, ck, v1) VALUES (1, 2, 2)");
+        execute("INSERT INTO " + keyspace + "." + table + "(pk, ck, v1) VALUES (2, 1, 3)");
+
+        flush(keyspace, table);
+
+        ResultSet rows = executeNet("SELECT pk, ck FROM " + keyspace + "." + table + " WHERE v1 > 0");
+
+        int actualRows = rows.all().size();
+        assertEquals(3, actualRows);
+
+        //TODO This needs revisiting with STAR-903 because we are now reading rows one at a time
+        waitForEquals(objectNameNoIndex("TotalPartitionReads", keyspace, table, TABLE_QUERY_METRIC_TYPE), Version.latest() == Version.AA ? 2 : 3);
+        waitForVerifyHistogram(objectNameNoIndex("RowsFiltered", keyspace, table, PER_QUERY_METRIC_TYPE), 1);
+        waitForEquals(objectNameNoIndex("TotalRowsFiltered", keyspace, table, TABLE_QUERY_METRIC_TYPE), 3);
+    }
+
+    @Test
+    public void testKDTreeQueryEarlyExit()
+    {
+        String table = "test_queries_exited_early";
+        String index = "test_queries_exited_early_index";
+
+        String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
+
+        createTable(String.format("CREATE TABLE %s.%s (pk int, ck int, v1 int, PRIMARY KEY (pk, ck)) " +
+                                  "WITH compaction = {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }", keyspace, table));
+
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v1"));
+
+        execute("INSERT INTO " + keyspace + "." + table + "(pk, ck, v1) VALUES (0, 0, 0)");
+        execute("INSERT INTO " + keyspace + "." + table + "(pk, ck, v1) VALUES (1, 1, 1)");
+        execute("INSERT INTO " + keyspace + "." + table + "(pk, ck, v1) VALUES (1, 2, 2)");
+
+        flush(keyspace, table);
+
+        ResultSet rows = executeNet("SELECT pk, ck FROM " + keyspace + "." + table + " WHERE v1 > 2");
+
+        assertEquals(0, rows.all().size());
+
+        rows = executeNet("SELECT pk, ck FROM " + keyspace + "." + table + " WHERE v1 < 0");
+        assertEquals(0, rows.all().size());
+
+        waitForEquals(objectName("KDTreeIntersectionLatency", keyspace, table, index, GLOBAL_METRIC_TYPE), 0L);
+        waitForEquals(objectName("KDTreeIntersectionEarlyExits", keyspace, table, index, GLOBAL_METRIC_TYPE), 2L);
+
+        rows = executeNet("SELECT pk, ck FROM " + keyspace + "." + table + " WHERE v1 > 0");
+        assertEquals(2, rows.all().size());
+
+        waitForEquals(objectName("KDTreeIntersectionLatency", keyspace, table, index, GLOBAL_METRIC_TYPE), 1L);
+        waitForEquals(objectName("KDTreeIntersectionEarlyExits", keyspace, table, index, GLOBAL_METRIC_TYPE), 2L);
+    }
+
+    private long getPerQueryMetrics(String keyspace, String table, String metricsName)
+    {
+        return (long) getMetricValue(objectNameNoIndex(metricsName, keyspace, table, PER_QUERY_METRIC_TYPE));
     }
 
     private long getTableQueryMetrics(String keyspace, String table, String metricsName)
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTest.java
new file mode 100644
index 000000000000..1e337547ac1b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTest.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.config.StorageAttachedIndexOptions;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.disk.v1.SSTableIndexWriter;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
+import org.apache.cassandra.inject.Injection;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.inject.Injections.newCounter;
+import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@Ignore
+public abstract class SegmentFlushingFailureTest extends SAITester
+{
+    static final long DEFAULT_BYTES_LIMIT = 1024L * 1024L * StorageAttachedIndexOptions.DEFAULT_SEGMENT_BUFFER_MB;
+
+    @Before
+    public void initialize() throws Throwable
+    {
+        requireNetwork();
+
+        startJMXServer();
+
+        createMBeanServerConnection();
+
+        Injections.inject(memoryTrackingCounter, writerAbortCounter);
+        memoryTrackingCounter.enable();
+        writerAbortCounter.enable();
+    }
+
+    private static final Injections.Counter memoryTrackingCounter =
+            newCounter("memoryTrackingCounter").add(newInvokePoint()
+                                               .onClass(NamedMemoryLimiter.class)
+                                               .onMethod("increment")
+                                               .atEntry()).build();
+
+    private static final Injections.Counter writerAbortCounter =
+            newCounter("writerAbortCounter").add(newInvokePoint()
+                                            .onClass(SSTableIndexWriter.class)
+                                            .onMethod("abort")
+                                            .atEntry()).build();
+
+    private static final Injection v1sstableComponentsWriterFailure =
+            newFailureOnEntry("sstableComponentsWriterFailure",
+                              org.apache.cassandra.index.sai.disk.v1.SSTableComponentsWriter.class,
+                              "complete",
+                              RuntimeException.class);
+
+    private static final Injection v2sstableComponentsWriterFailure =
+    newFailureOnEntry("sstableComponentsWriterFailure",
+                      org.apache.cassandra.index.sai.disk.v2.SSTableComponentsWriter.class,
+                      "complete",
+                      RuntimeException.class);
+
+    private static final Injection segmentFlushFailure =
+            newFailureOnEntry("segmentFlushFailure", SegmentBuilder.class, "flush", RuntimeException.class);
+
+    private static final Injection segmentFlushIOFailure =
+            newFailureOnEntry("segmentFlushIOFailure", SegmentBuilder.class, "flush", IOException.class);
+
+    private static final Injection kdTreeSegmentFlushFailure =
+            newFailureOnEntry("kdTreeSegmentFlushFailure", SegmentBuilder.KDTreeSegmentBuilder.class, "flushInternal", IOException.class);
+
+    @After
+    public void resetCounters()
+    {
+        memoryTrackingCounter.reset();
+        writerAbortCounter.reset();
+    }
+
+    protected abstract long expectedBytesLimit();
+
+    @Test
+    public void testSegmentMemoryTrackerLifecycle() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+
+        assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
+        assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
+        flush();
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '1')");
+        flush();
+
+        ResultSet rows = executeNet("SELECT * FROM %s WHERE v1 = 0");
+        assertEquals(1, rows.all().size());
+
+        compact();
+
+        // The compaction completed successfully:
+        Assert.assertEquals(0, writerAbortCounter.get());
+
+        // This is a proxy for making sure we've actually tracked something:
+        assertTrue(memoryTrackingCounter.get() > 0);
+
+        assertEquals("Global memory tracker should have reverted to zero.", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+
+        rows = executeNet("SELECT * FROM %s WHERE v1 = 0");
+        assertEquals(1, rows.all().size());
+    }
+
+    @Test
+    public void shouldZeroMemoryTrackerOnOffsetsRuntimeFailure() throws Throwable
+    {
+        shouldZeroMemoryTrackerOnFailure(Version.latest() == Version.AA ? v1sstableComponentsWriterFailure : v2sstableComponentsWriterFailure, "v1");
+        resetCounters();
+        shouldZeroMemoryTrackerOnFailure(Version.latest() == Version.AA ? v1sstableComponentsWriterFailure : v2sstableComponentsWriterFailure, "v2");
+    }
+
+    @Test
+    public void shouldZeroMemoryTrackerOnSegmentFlushIOFailure() throws Throwable
+    {
+        shouldZeroMemoryTrackerOnFailure(segmentFlushIOFailure, "v1");
+        resetCounters();
+        shouldZeroMemoryTrackerOnFailure(segmentFlushIOFailure, "v2");
+    }
+
+    @Test
+    public void shouldZeroMemoryTrackerOnSegmentFlushRuntimeFailure() throws Throwable
+    {
+        shouldZeroMemoryTrackerOnFailure(segmentFlushFailure, "v1");
+        resetCounters();
+        shouldZeroMemoryTrackerOnFailure(segmentFlushFailure, "v2");
+    }
+
+    private void shouldZeroMemoryTrackerOnFailure(Injection failure, String column) throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, column));
+
+        assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
+        assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
+        flush();
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '1')");
+        flush();
+        Collection<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+
+        // Verify that we abort exactly once and zero the memory tracker:
+        verifyCompactionIndexBuilds(1, failure, currentTable());
+
+        String select = String.format("SELECT * FROM %%s WHERE %s = %s", column, column.equals("v1") ? "0" : "'0'");
+
+        // compaction is aborted, index is still queryable
+        executeNet(select);
+        assertThat(getColumnFamilyStore(KEYSPACE, currentTable()).getLiveSSTables()).isEqualTo(sstables);
+    }
+
+    @Test
+    public void shouldZeroMemoryAfterOneOfTwoIndexesFail() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+
+        assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
+        assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
+        flush();
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '1')");
+        flush();
+        Collection<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
+
+        // Verify that we abort both indices and zero the memory tracker:
+        verifyCompactionIndexBuilds(2, kdTreeSegmentFlushFailure, currentTable());
+
+        // compaction is aborted, index is still queryable
+        executeNet("SELECT * FROM %s WHERE V1 = 0");
+        assertThat(getColumnFamilyStore(KEYSPACE, currentTable()).getLiveSSTables()).isEqualTo(sstables);
+    }
+
+    @Test
+    public void shouldZeroMemoryAfterConcurrentIndexFailures() throws Throwable
+    {
+        String table1 = createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        String table2 = createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+
+        assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
+        assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+
+        execute("INSERT INTO " + KEYSPACE + "." + table1 + "(id1, v1, v2) VALUES ('0', 0, '0')");
+        flush(KEYSPACE, table1);
+        execute("INSERT INTO " + KEYSPACE + "." + table1 + "(id1, v1, v2) VALUES ('1', 1, '1')");
+        flush(KEYSPACE, table1);
+        Collection<SSTableReader> sstablesTable1 = getColumnFamilyStore(KEYSPACE, table1).getLiveSSTables();
+
+        execute("INSERT INTO " + KEYSPACE + "." + table2 + "(id1, v1, v2) VALUES ('0', 0, '0')");
+        flush(KEYSPACE, table2);
+        execute("INSERT INTO " + KEYSPACE + "." + table2 + "(id1, v1, v2) VALUES ('1', 1, '1')");
+        flush(KEYSPACE, table2);
+        Collection<SSTableReader> sstablesTable2 = getColumnFamilyStore(KEYSPACE, table2).getLiveSSTables();
+
+        // Start compaction against both tables/indexes and verify that they are aborted safely:
+        verifyCompactionIndexBuilds(2, segmentFlushFailure, table1, table2);
+
+        executeNet(String.format("SELECT * FROM %s WHERE v1 = 0", KEYSPACE + "." + table1));
+        assertThat(getColumnFamilyStore(KEYSPACE, table1).getLiveSSTables()).isEqualTo(sstablesTable1);
+
+        executeNet(String.format("SELECT * FROM %s WHERE v1 = 0", KEYSPACE + "." + table2));
+        assertThat(getColumnFamilyStore(KEYSPACE, table2).getLiveSSTables()).isEqualTo(sstablesTable2);
+    }
+
+    @Test
+    public void shouldLeaveOnlyFailedIndexNonQueryable() throws Throwable
+    {
+        String table1 = createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        String table2 = createTable(CREATE_TABLE_TEMPLATE);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
+
+        assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
+        assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
+        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+
+        execute("INSERT INTO " + KEYSPACE + "." + table1 + "(id1, v1, v2) VALUES ('0', 0, '0')");
+        flush(KEYSPACE, table1);
+        execute("INSERT INTO " + KEYSPACE + "." + table1 + "(id1, v1, v2) VALUES ('1', 1, '1')");
+        flush(KEYSPACE, table1);
+        Collection<SSTableReader> sstablesTable1 = getColumnFamilyStore(KEYSPACE, table1).getLiveSSTables();
+
+        execute("INSERT INTO " + KEYSPACE + "." + table2 + "(id1, v1, v2) VALUES ('0', 0, '0')");
+        flush(KEYSPACE, table2);
+        execute("INSERT INTO " + KEYSPACE + "." + table2 + "(id1, v1, v2) VALUES ('1', 1, '1')");
+        flush(KEYSPACE, table2);
+
+        // Start compaction against both tables/indexes, and verify only the numeric index is aborted:
+        verifyCompactionIndexBuilds(1, kdTreeSegmentFlushFailure, table1, table2);
+
+        // index is still queryable and sstables remain the same
+        executeNet(String.format("SELECT * FROM %s WHERE v1 = 0", KEYSPACE + "." + table1));
+        assertThat(getColumnFamilyStore(KEYSPACE, table1).getLiveSSTables()).isEqualTo(sstablesTable1);
+
+        ResultSet rows = executeNet(String.format("SELECT * FROM %s WHERE v2 = '0'", KEYSPACE + "." + table2));
+        assertEquals(1, rows.all().size());
+
+        // table2 succeeded compaction
+        assertThat(getColumnFamilyStore(KEYSPACE, table2).getLiveSSTables()).hasSize(1);
+    }
+
+    private void verifyCompactionIndexBuilds(int aborts, Injection failure, String... tables) throws Throwable
+    {
+        Injections.inject(failure);
+        failure.enable();
+
+        try
+        {
+            Arrays.stream(tables).forEach(table -> {
+                try
+                {
+                    compact(KEYSPACE, table);
+                }
+                catch (RuntimeException e)
+                {
+                    // injected failure
+                }
+            });
+
+            Assert.assertEquals(aborts, writerAbortCounter.get());
+
+            assertEquals("Global memory tracker should have reverted to zero.", 0L, getSegmentBufferUsedBytes());
+            assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
+        }
+        finally
+        {
+            failure.disable();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTester.java b/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTester.java
deleted file mode 100644
index 4b98a946713e..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTester.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.metrics;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.cassandra.utils.FBUtilities;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.datastax.driver.core.ResultSet;
-import org.apache.cassandra.config.StorageAttachedIndexOptions;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.v1.SSTableComponentsWriter;
-import org.apache.cassandra.index.sai.disk.v1.SSTableIndexWriter;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
-import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
-import org.apache.cassandra.inject.Injection;
-import org.apache.cassandra.inject.Injections;
-
-import static org.apache.cassandra.inject.Injections.newCounter;
-import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public abstract class SegmentFlushingFailureTester extends SAITester
-{
-    static final long DEFAULT_BYTES_LIMIT = 1024L * 1024L * StorageAttachedIndexOptions.DEFAULT_SEGMENT_BUFFER_MB;
-
-    @Before
-    public void initialize() throws Throwable
-    {
-        requireNetwork();
-
-        Injections.inject(memoryTrackingCounter, writerAbortCounter);
-        memoryTrackingCounter.enable();
-        writerAbortCounter.enable();
-    }
-
-    private static final Injections.Counter memoryTrackingCounter =
-            newCounter("memoryTrackingCounter").add(newInvokePoint()
-                                               .onClass(NamedMemoryLimiter.class)
-                                               .onMethod("increment")
-                                               .atEntry()).build();
-
-    private static final Injections.Counter writerAbortCounter =
-            newCounter("writerAbortCounter").add(newInvokePoint()
-                                            .onClass(SSTableIndexWriter.class)
-                                            .onMethod("abort")
-                                            .atEntry()).build();
-
-    private static final Injection v1sstableComponentsWriterFailure =
-            newFailureOnEntry("sstableComponentsWriterFailure",
-                              SSTableComponentsWriter.class,
-                              "complete",
-                              RuntimeException.class);
-
-    private static final Injection segmentFlushFailure =
-            newFailureOnEntry("segmentFlushFailure", SegmentBuilder.class, "flush", RuntimeException.class);
-
-    private static final Injection segmentFlushIOFailure =
-            newFailureOnEntry("segmentFlushIOFailure", SegmentBuilder.class, "flush", IOException.class);
-
-    @After
-    public void resetCounters()
-    {
-        memoryTrackingCounter.reset();
-        writerAbortCounter.reset();
-    }
-
-    protected abstract long expectedBytesLimit();
-
-    @Test
-    public void testSegmentMemoryTrackerLifecycle()
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-
-        assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
-        assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
-        flush();
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '1')");
-        flush();
-
-        ResultSet rows = executeNet("SELECT * FROM %s WHERE v1 = 0");
-        assertEquals(1, rows.all().size());
-
-        compact();
-
-        // The compaction completed successfully:
-        Assert.assertEquals(0, writerAbortCounter.get());
-
-        // This is a proxy for making sure we've actually tracked something:
-        assertTrue(memoryTrackingCounter.get() > 0);
-
-        assertEquals("Global memory tracker should have reverted to zero.", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-
-        rows = executeNet("SELECT * FROM %s WHERE v1 = 0");
-        assertEquals(1, rows.all().size());
-    }
-
-    @Test
-    public void shouldZeroMemoryTrackerOnOffsetsRuntimeFailure() throws Throwable
-    {
-        shouldZeroMemoryTrackerOnFailure(v1sstableComponentsWriterFailure, "v1");
-    }
-
-    @Test
-    public void shouldZeroMemoryTrackerOnSegmentFlushIOFailure() throws Throwable
-    {
-        shouldZeroMemoryTrackerOnFailure(segmentFlushIOFailure, "v1");
-    }
-
-    @Test
-    public void shouldZeroMemoryTrackerOnSegmentFlushRuntimeFailure() throws Throwable
-    {
-        shouldZeroMemoryTrackerOnFailure(segmentFlushFailure, "v1");
-    }
-
-    private void shouldZeroMemoryTrackerOnFailure(Injection failure, String column) throws Throwable
-    {
-        createTable(CREATE_TABLE_TEMPLATE);
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, column));
-
-        assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
-        assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
-        flush();
-        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '1')");
-        flush();
-
-        // Verify that we abort exactly once and zero the memory tracker:
-        verifyCompactionIndexBuilds(1, failure, currentTable());
-
-        // We should still be able to query the index if compaction is aborted:
-        String select = String.format("SELECT * FROM %%s WHERE %s = %s", column, column.equals("v1") ? "0" : "'0'");
-        ResultSet rows = executeNet(select);
-        assertEquals(1, rows.all().size());
-    }
-
-    @Test
-    public void shouldZeroMemoryAfterConcurrentIndexFailures() throws Throwable
-    {
-        String table1 = createTable(CREATE_TABLE_TEMPLATE);
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        String table2 = createTable(CREATE_TABLE_TEMPLATE);
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-
-        assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
-        assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
-        assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
-
-        execute("INSERT INTO " + KEYSPACE + "." + table1 + "(id1, v1, v2) VALUES ('0', 0, '0')");
-        flush(KEYSPACE, table1);
-        execute("INSERT INTO " + KEYSPACE + "." + table1 + "(id1, v1, v2) VALUES ('1', 1, '1')");
-        flush(KEYSPACE, table1);
-
-        execute("INSERT INTO " + KEYSPACE + "." + table2 + "(id1, v1, v2) VALUES ('0', 0, '0')");
-        flush(KEYSPACE, table2);
-        execute("INSERT INTO " + KEYSPACE + "." + table2 + "(id1, v1, v2) VALUES ('1', 1, '1')");
-        flush(KEYSPACE, table2);
-
-        // Start compaction against both tables/indexes and verify that they are aborted safely:
-        verifyCompactionIndexBuilds(2, segmentFlushFailure, table1, table2);
-
-        // We should still be able to query the indexes if compaction is aborted:
-        ResultSet rows = executeNet(String.format("SELECT * FROM %s WHERE v1 = 0", KEYSPACE + "." + table1));
-        assertEquals(1, rows.all().size());
-        rows = executeNet(String.format("SELECT * FROM %s WHERE v1 = 0", KEYSPACE + "." + table2));
-        assertEquals(1, rows.all().size());
-    }
-
-    private void verifyCompactionIndexBuilds(int aborts, Injection failure, String... tables) throws Throwable
-    {
-        Injections.inject(failure);
-        failure.enable();
-
-        try
-        {
-            ExecutorService executor = Executors.newFixedThreadPool(tables.length);
-            List<Future<?>> results = new ArrayList<>();
-
-            for (String table : tables)
-                results.add(executor.submit(() -> compact(KEYSPACE, table)));
-            
-            assertThatThrownBy(() -> FBUtilities.waitOnFutures(results)).hasRootCauseMessage("Injected failure!");
-            executor.shutdownNow();
-
-            Assert.assertEquals(aborts, writerAbortCounter.get());
-
-            assertEquals("Global memory tracker should have reverted to zero.", 0L, getSegmentBufferUsedBytes());
-            assertEquals("There should be no segment builders in progress.", 0, getColumnIndexBuildsInProgress());
-        }
-        finally
-        {
-            failure.disable();
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/StateMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/StateMetricsTest.java
index 8e83293e618f..3074192ee48b 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/StateMetricsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/StateMetricsTest.java
@@ -39,7 +39,7 @@ public class StateMetricsTest extends AbstractMetricsTest
     public ExpectedException exception = ExpectedException.none();
 
     @Test
-    public void testMetricRelease() throws Throwable
+    public void testMetricRelease()
     {
         String table = "test_metric_release";
         String index = "test_metric_release_index";
@@ -49,39 +49,48 @@ public void testMetricRelease() throws Throwable
         createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
         createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v1"));
 
-        execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
+        execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
 
-        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace + '.' + table + " WHERE v1 = 0");
+        ResultSet rows = executeNet("SELECT id1 FROM " + keyspace + "." + table + " WHERE v1 = 0");
         assertEquals(1, rows.all().size());
         assertEquals(1L, getTableStateMetrics(keyspace, table, "TotalIndexCount"));
 
-        // If we drop the last index on the table, we should no longer see the table-level state metrics:
+        // If we drop the last index on the table, table-level state metrics should be removed
         dropIndex(String.format("DROP INDEX %s." + index, keyspace));
-        assertThatThrownBy(() -> getTableStateMetrics(keyspace, table, "TotalIndexCount")).hasCauseInstanceOf(InstanceNotFoundException.class);
+        assertThatThrownBy(() -> getTableStateMetrics(keyspace, table, "TotalIndexCount")).hasRootCauseInstanceOf(InstanceNotFoundException.class);
+
+        // When the whole table is dropped, we should finally fail to find table-level state metrics:
+        dropTable(String.format("DROP TABLE %s." + table, keyspace));
+        assertThatThrownBy(() -> getTableStateMetrics(keyspace, table, "TotalIndexCount")).hasRootCauseInstanceOf(InstanceNotFoundException.class);
     }
 
     @Test
-    public void testMetricCreation() throws Throwable
+    public void testMetricCreation()
     {
         String table = "test_table";
         String index = "test_index";
 
         String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
         createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, index + "_v1", keyspace, table, "v1"));
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, index + "_v2", keyspace, table, "v2"));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index+"_v1", keyspace, table, "v1"));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index+"_v2", keyspace, table, "v2"));
+
+        execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
+        execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES ('1', 1, '1')");
+        execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES ('2', 2, '2')");
+        execute("INSERT INTO " + keyspace + "." + table + " (id1, v1, v2) VALUES ('3', 3, '3')");
 
-        execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
-        execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('1', 1, '1')");
-        execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('2', 2, '2')");
-        execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('3', 3, '3')");
+        flush(keyspace, table);
 
-        ResultSet rows = executeNet("SELECT id1, v1, v2 FROM " + keyspace + '.' + table + " WHERE v1 >= 0");
+        ResultSet rows = executeNet("SELECT id1, v1, v2 FROM " + keyspace + "." + table + " WHERE v1 >= 0");
 
         int actualRows = rows.all().size();
         assertEquals(4, actualRows);
 
+        waitForGreaterThanZero(objectNameNoIndex("DiskPercentageOfBaseTable", keyspace, table, TABLE_STATE_METRIC_TYPE));
+        waitForGreaterThanZero(objectNameNoIndex("DiskUsedBytes", keyspace, table, TABLE_STATE_METRIC_TYPE));
         waitForEquals(objectNameNoIndex("TotalIndexCount", keyspace, table, TABLE_STATE_METRIC_TYPE), 2);
+        waitForEquals(objectNameNoIndex("TotalIndexBuildsInProgress", keyspace, table, TABLE_STATE_METRIC_TYPE), 0);
         waitForEquals(objectNameNoIndex("TotalQueryableIndexCount", keyspace, table, TABLE_STATE_METRIC_TYPE), 2);
     }
 
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/TinySegmentFlushingFailureTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/TinySegmentFlushingFailureTest.java
index cf1fe52568af..8f9f19357c11 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/TinySegmentFlushingFailureTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/TinySegmentFlushingFailureTest.java
@@ -17,19 +17,55 @@
  */
 package org.apache.cassandra.index.sai.metrics;
 
-import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
 
-public class TinySegmentFlushingFailureTest extends SegmentFlushingFailureTester
+import java.lang.reflect.Field;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+
+
+@RunWith(Enclosed.class)
+public class TinySegmentFlushingFailureTest extends CQLTester
 {
-    @Before
-    public void setSegmentBufferSpace() throws Throwable
+    /**
+     * Set the necessary configuration before any tests run.
+     */
+    @BeforeClass
+    public static void setUpClass()
     {
-        setSegmentWriteBufferSpace(0);
+        DatabaseDescriptor.daemonInitialization();
+        try
+        {
+            Field confField = DatabaseDescriptor.class.getDeclaredField("conf");
+            confField.setAccessible(true);
+            Config conf = (Config) confField.get(null);
+            conf.sai_options.segment_write_buffer_space_mb = 0;
+
+            System.out.println("Configuration set: segment_write_buffer_space_mb = " + 0);
+        }
+        catch (NoSuchFieldException | IllegalAccessException e)
+        {
+            throw new RuntimeException("Failed to set configuration segment_write_buffer_space_mb = 0", e);
+        }
+        
+        CQLTester.setUpClass();
     }
 
-    @Override
-    protected long expectedBytesLimit()
+    /**
+     * These tests will run only after the outer class has completed its setup. Otherwise, SAITester assigns default
+     * value to segment_write_buffer_space_mb, and we cannot override it without reflection or using Unsafe.
+     */
+    public static class TinySegmentFlushingFailureInnerClassTest extends SegmentFlushingFailureTest
     {
-        return 0;
+
+        @Override
+        protected long expectedBytesLimit()
+        {
+            return 0;
+        }
     }
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/plan/ExpressionTest.java b/test/unit/org/apache/cassandra/index/sai/plan/ExpressionTest.java
index 3d32a405ddef..f34f5325f1a4 100644
--- a/test/unit/org/apache/cassandra/index/sai/plan/ExpressionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/plan/ExpressionTest.java
@@ -23,8 +23,6 @@
 import org.junit.Test;
 
 import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.utils.IndexTermType;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
@@ -35,11 +33,10 @@ public class ExpressionTest
     @Test
     public void testBoundHashCode()
     {
-        IndexTermType indexTermType = SAITester.createIndexTermType(UTF8Type.instance);
         ByteBuffer buf1 = UTF8Type.instance.decompose("blah");
-        Expression.Bound b1 = new Expression.Bound(buf1, indexTermType, true);
+        Expression.Bound b1 = new Expression.Bound(buf1, UTF8Type.instance, true);
         ByteBuffer buf2 = UTF8Type.instance.decompose("blah");
-        Expression.Bound b2 = new Expression.Bound(buf2, indexTermType, true);
+        Expression.Bound b2 = new Expression.Bound(buf2, UTF8Type.instance, true);
         assertEquals(b1, b2);
         assertEquals(b1.hashCode(), b2.hashCode());
     }
@@ -47,11 +44,10 @@ public void testBoundHashCode()
     @Test
     public void testNotMatchingBoundHashCode()
     {
-        IndexTermType indexTermType = SAITester.createIndexTermType(UTF8Type.instance);
         ByteBuffer buf1 = UTF8Type.instance.decompose("blah");
-        Expression.Bound b1 = new Expression.Bound(buf1, indexTermType, true);
+        Expression.Bound b1 = new Expression.Bound(buf1, UTF8Type.instance, true);
         ByteBuffer buf2 = UTF8Type.instance.decompose("blah2");
-        Expression.Bound b2 = new Expression.Bound(buf2, indexTermType, true);
+        Expression.Bound b2 = new Expression.Bound(buf2, UTF8Type.instance, true);
         assertNotEquals(b1, b2);
         assertNotEquals(b1.hashCode(), b2.hashCode());
     }
diff --git a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
index 8b0acaaf2b61..eccb13a608ee 100644
--- a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
@@ -19,39 +19,38 @@
 package org.apache.cassandra.index.sai.plan;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.EnumMap;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionRangeReadCommand;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.CollectionType;
 import org.apache.cassandra.db.marshal.CompositeType;
 import org.apache.cassandra.db.marshal.DoubleType;
 import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.ListType;
 import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.marshal.MapType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.rows.BTreeRow;
 import org.apache.cassandra.db.rows.BufferCell;
@@ -60,89 +59,232 @@
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.index.sai.IndexingSchemaLoader;
 import org.apache.cassandra.index.sai.QueryContext;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
 import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.schema.Indexes;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
 import static org.apache.cassandra.db.marshal.Int32Type.instance;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 
-public class OperationTest
+public class OperationTest extends IndexingSchemaLoader
 {
     private static final String KS_NAME = "sai";
     private static final String CF_NAME = "test_cf";
     private static final String CLUSTERING_CF_NAME = "clustering_test_cf";
-    private static final String STATIC_CF_NAME = "static_sai_test_cf";
+    private static final String STATIC_CF_NAME = "static_ndi_test_cf";
 
     private static ColumnFamilyStore BACKEND;
     private static ColumnFamilyStore CLUSTERING_BACKEND;
     private static ColumnFamilyStore STATIC_BACKEND;
 
+
     private QueryController controller;
     private QueryController controllerClustering;
     private QueryController controllerStatic;
 
+
     @BeforeClass
     public static void loadSchema() throws ConfigurationException
     {
         CASSANDRA_CONFIG.setString("cassandra-murmur.yaml");
 
-        SchemaLoader.loadSchema();
+        IndexingSchemaLoader.loadSchema();
 
-        SchemaLoader.createKeyspace(KS_NAME,
-                                    KeyspaceParams.simpleTransient(1),
-                                    skinnySAITableMetadata(KS_NAME, CF_NAME),
-                                    clusteringSAITableMetadata(KS_NAME, CLUSTERING_CF_NAME),
-                                    staticSAITableMetadata(KS_NAME, STATIC_CF_NAME));
+        IndexingSchemaLoader.createKeyspace(KS_NAME,
+                                            KeyspaceParams.simpleTransient(1),
+                                            IndexingSchemaLoader.ndiCFMD(KS_NAME, CF_NAME),
+                                            IndexingSchemaLoader.clusteringNDICFMD(KS_NAME, CLUSTERING_CF_NAME),
+                                            IndexingSchemaLoader.staticNDICFMD(KS_NAME, STATIC_CF_NAME));
 
         BACKEND = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME);
         CLUSTERING_BACKEND = Keyspace.open(KS_NAME).getColumnFamilyStore(CLUSTERING_CF_NAME);
         STATIC_BACKEND = Keyspace.open(KS_NAME).getColumnFamilyStore(STATIC_CF_NAME);
     }
 
+
     @Before
     public void beforeTest()
     {
         ReadCommand command = PartitionRangeReadCommand.allDataRead(BACKEND.metadata(), FBUtilities.nowInSeconds());
-        controller = new QueryController(BACKEND, command, null, contextWithUnrepairedMatches(command));
+        controller = new QueryController(BACKEND, command,  V1OnDiskFormat.instance.indexFeatureSet(), new QueryContext());
 
         command = PartitionRangeReadCommand.allDataRead(CLUSTERING_BACKEND.metadata(), FBUtilities.nowInSeconds());
-        controllerClustering = new QueryController(CLUSTERING_BACKEND, command, null, contextWithUnrepairedMatches(command));
+        controllerClustering = new QueryController(CLUSTERING_BACKEND, command, V1OnDiskFormat.instance.indexFeatureSet(), new QueryContext());
 
         command = PartitionRangeReadCommand.allDataRead(STATIC_BACKEND.metadata(), FBUtilities.nowInSeconds());
-        controllerStatic = new QueryController(STATIC_BACKEND, command, null, contextWithUnrepairedMatches(command));
+        controllerStatic = new QueryController(STATIC_BACKEND, command, V1OnDiskFormat.instance.indexFeatureSet(), new QueryContext());
     }
 
-    private static QueryContext contextWithUnrepairedMatches(ReadCommand command)
+    @After
+    public void afterTest()
     {
-        QueryContext context = new QueryContext(command, DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.MILLISECONDS));
-        context.hasUnrepairedMatches = true;
-        return context;
     }
 
     @Test
     public void testAnalyze()
     {
+        final ColumnMetadata firstName = getColumn(UTF8Type.instance.decompose("first_name"));
         final ColumnMetadata age = getColumn(UTF8Type.instance.decompose("age"));
+        final ColumnMetadata comment = getColumn(UTF8Type.instance.decompose("comment"));
 
-        // age > 1 AND age < 7
-        Map<Expression.IndexOperator, Expression> expressions = convert(Operation.buildIndexExpressions(controller,
-                                                                                                        Arrays.asList(new SimpleExpression(age, Operator.GT, Int32Type.instance.decompose(1)),
-                                                                                                                      new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(7)))));
-
-        assertEquals(1, expressions.size());
+        // age != 5 AND age > 1 AND age != 6 AND age <= 10
+        Map<Expression.Op, Expression> expressions = convert(Operation.analyzeGroup(controller, Operation.OperationType.AND,
+                                                                                    Arrays.asList(new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(5)),
+                                                                                              new SimpleExpression(age, Operator.GT, Int32Type.instance.decompose(1)),
+                                                                                              new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(6)),
+                                                                                              new SimpleExpression(age, Operator.LTE, Int32Type.instance.decompose(10)))));
 
-        Expression rangeExpression = expressions.get(Expression.IndexOperator.RANGE);
+        Expression expected = new Expression(SAITester.createIndexContext("age", Int32Type.instance))
+        {{
+            operation = Op.RANGE;
+            lower = new Bound(Int32Type.instance.decompose(1), Int32Type.instance, false);
+            upper = new Bound(Int32Type.instance.decompose(10), Int32Type.instance, true);
+
+            exclusions.add(Int32Type.instance.decompose(5));
+            exclusions.add(Int32Type.instance.decompose(6));
+        }};
+
+        Assert.assertEquals(1, expressions.size());
+        Assert.assertEquals(expected, expressions.get(Expression.Op.RANGE));
+
+        // age != 5 OR age >= 7
+        expressions = convert(Operation.analyzeGroup(controller, Operation.OperationType.OR,
+                                                     Arrays.asList(new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(5)),
+                                                                  new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(7)))));
+        Assert.assertEquals(2, expressions.size());
+
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("age", Int32Type.instance))
+                            {{
+                                    operation = Op.NOT_EQ;
+                                    lower = new Bound(Int32Type.instance.decompose(5), Int32Type.instance, true);
+                                    upper = lower;
+                            }}, expressions.get(Expression.Op.NOT_EQ));
+
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("age", Int32Type.instance))
+                            {{
+                                    operation = Op.RANGE;
+                                    lower = new Bound(Int32Type.instance.decompose(7), Int32Type.instance, true);
+                            }}, expressions.get(Expression.Op.RANGE));
+
+        // age != 5 OR age < 7
+        expressions = convert(Operation.analyzeGroup(controller, Operation.OperationType.OR,
+                                                     Arrays.asList(new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(5)),
+                                                                  new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(7)))));
+
+        Assert.assertEquals(2, expressions.size());
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("age", Int32Type.instance))
+                            {{
+                                    operation = Op.RANGE;
+                                    upper = new Bound(Int32Type.instance.decompose(7), Int32Type.instance, false);
+                            }}, expressions.get(Expression.Op.RANGE));
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("age", Int32Type.instance))
+                            {{
+                                    operation = Op.NOT_EQ;
+                                    lower = new Bound(Int32Type.instance.decompose(5), Int32Type.instance, true);
+                                    upper = lower;
+                            }}, expressions.get(Expression.Op.NOT_EQ));
 
-        assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(1), false, Int32Type.instance.decompose(7), false);
+        // age > 1 AND age < 7
+        expressions = convert(Operation.analyzeGroup(controller, Operation.OperationType.AND,
+                                                     Arrays.asList(new SimpleExpression(age, Operator.GT, Int32Type.instance.decompose(1)),
+                                                                  new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(7)))));
+
+        Assert.assertEquals(1, expressions.size());
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("age", Int32Type.instance))
+                            {{
+                                    operation = Op.RANGE;
+                                    lower = new Bound(Int32Type.instance.decompose(1), Int32Type.instance, false);
+                                    upper = new Bound(Int32Type.instance.decompose(7), Int32Type.instance, false);
+                            }}, expressions.get(Expression.Op.RANGE));
+
+        // first_name = 'a' OR first_name != 'b'
+        expressions = convert(Operation.analyzeGroup(controller, Operation.OperationType.OR,
+                                                     Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")),
+                                                                  new SimpleExpression(firstName, Operator.NEQ, UTF8Type.instance.decompose("b")))));
+
+        Assert.assertEquals(2, expressions.size());
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("first_name", UTF8Type.instance))
+                            {{
+                                    operation = Op.NOT_EQ;
+                                    lower = new Bound(UTF8Type.instance.decompose("b"), UTF8Type.instance, true);
+                                    upper = lower;
+                            }}, expressions.get(Expression.Op.NOT_EQ));
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("first_name", UTF8Type.instance))
+                            {{
+                                    operation = Op.EQ;
+                                    lower = upper = new Bound(UTF8Type.instance.decompose("a"), UTF8Type.instance, true);
+                            }}, expressions.get(Expression.Op.EQ));
+
+        // comment = 'soft eng' and comment != 'likes do'
+        ListMultimap<ColumnMetadata, Expression> e = Operation.analyzeGroup(controller, Operation.OperationType.OR,
+                                                                            Arrays.asList(new SimpleExpression(comment, Operator.LIKE_MATCHES, UTF8Type.instance.decompose("soft eng")),
+                                                                  new SimpleExpression(comment, Operator.NEQ, UTF8Type.instance.decompose("likes do"))));
+
+        List<Expression> expectedExpressions = new ArrayList<Expression>(2)
+        {{
+                add(new Expression(SAITester.createIndexContext("comment", UTF8Type.instance))
+                {{
+                        operation = Op.MATCH;
+                        lower = new Bound(UTF8Type.instance.decompose("soft eng"), UTF8Type.instance, true);
+                        upper = lower;
+                }});
+
+                add(new Expression(SAITester.createIndexContext("comment", UTF8Type.instance))
+                {{
+                        operation = Op.NOT_EQ;
+                        lower = new Bound(UTF8Type.instance.decompose("likes do"), UTF8Type.instance, true);
+                        upper = lower;
+                }});
+        }};
+
+        Assert.assertEquals(expectedExpressions, e.get(comment));
+
+        // first_name = 'j' and comment != 'likes do'
+        e = Operation.analyzeGroup(controller, Operation.OperationType.OR,
+                                   Arrays.asList(new SimpleExpression(comment, Operator.NEQ, UTF8Type.instance.decompose("likes do")),
+                                      new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("j"))));
+
+        expectedExpressions = new ArrayList<Expression>(2)
+        {{
+                add(new Expression(SAITester.createIndexContext("comment", UTF8Type.instance))
+                {{
+                        operation = Op.NOT_EQ;
+                        lower = new Bound(UTF8Type.instance.decompose("likes do"), UTF8Type.instance, true);
+                        upper = lower;
+                }});
+        }};
+
+        Assert.assertEquals(expectedExpressions, e.get(comment));
+
+        // age != 27 first_name = 'j' and age != 25
+        e = Operation.analyzeGroup(controller, Operation.OperationType.OR,
+                                   Arrays.asList(new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(27)),
+                                      new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("j")),
+                                      new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(25))));
+
+        expectedExpressions = new ArrayList<Expression>(2)
+        {{
+                add(new Expression(SAITester.createIndexContext("age", Int32Type.instance))
+                {{
+                        operation = Op.NOT_EQ;
+                        lower = new Bound(Int32Type.instance.decompose(27), Int32Type.instance, true);
+                        upper = lower;
+                }});
+
+                add(new Expression(SAITester.createIndexContext("age", Int32Type.instance))
+                {{
+                        operation = Op.NOT_EQ;
+                        lower = new Bound(Int32Type.instance.decompose(25), Int32Type.instance, true);
+                        upper = lower;
+                }});
+        }};
+
+        Assert.assertEquals(expectedExpressions, e.get(age));
     }
 
     @Test
@@ -151,52 +293,77 @@ public void testSatisfiedBy()
         final ColumnMetadata timestamp = getColumn(UTF8Type.instance.decompose("timestamp"));
         final ColumnMetadata age = getColumn(UTF8Type.instance.decompose("age"));
 
-        Operation.Node node = new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(5)));
-        FilterTree filterTree = node.buildFilter(controller, true);
+        Operation.Node node = new Operation.ExpressionNode(new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(5)));
+        FilterTree filterTree = node.buildFilter(controller);
 
         DecoratedKey key = buildKey("0");
         Unfiltered row = buildRow(buildCell(age, instance.decompose(6), System.currentTimeMillis()));
         Row staticRow = buildRow(Clustering.STATIC_CLUSTERING);
 
-        assertFalse(filterTree.isSatisfiedBy(key, (Row) row, staticRow));
+        Assert.assertTrue(filterTree.isSatisfiedBy(key, row, staticRow));
 
         row = buildRow(buildCell(age, instance.decompose(5), System.currentTimeMillis()));
 
-        assertTrue(filterTree.isSatisfiedBy(key, (Row) row, staticRow));
+        // and reject incorrect value
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow));
 
         row = buildRow(buildCell(age, instance.decompose(6), System.currentTimeMillis()));
 
-        assertFalse(filterTree.isSatisfiedBy(key, (Row) row, staticRow));
+        Assert.assertTrue(filterTree.isSatisfiedBy(key, row, staticRow));
 
-        // range with exclusions - age > 1 AND age <= 10
+        // range with exclusions - age != 5 AND age > 1 AND age != 6 AND age <= 10
         node = new Operation.AndNode();
+        node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(5))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GT, Int32Type.instance.decompose(1))));
+        node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(6))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.LTE, Int32Type.instance.decompose(10))));
-        filterTree = node.buildFilter(controller, true);
+        filterTree = node.buildFilter(controller);
 
-        Set<Integer> exclusions = Sets.newHashSet(0, 1, 11);
+        Set<Integer> exclusions = Sets.newHashSet(0, 1, 5, 6, 11);
         for (int i = 0; i <= 11; i++)
         {
             row = buildRow(buildCell(age, instance.decompose(i), System.currentTimeMillis()));
 
-            boolean result = filterTree.isSatisfiedBy(key, (Row) row, staticRow);
-            assertTrue(exclusions.contains(i) != result);
+            boolean result = filterTree.isSatisfiedBy(key, row, staticRow);
+            Assert.assertTrue(exclusions.contains(i) != result);
+        }
+
+        // now let's do something more complex - age = 5 OR age = 6
+        node = new Operation.OrNode();
+        node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(5))));
+        node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(6))));
+        filterTree = node.buildFilter(controller);
+
+        exclusions = Sets.newHashSet(0, 1, 2, 3, 4, 7, 8, 9, 10);
+        for (int i = 0; i <= 10; i++)
+        {
+            row = buildRow(buildCell(age, instance.decompose(i), System.currentTimeMillis()));
+
+            boolean result = filterTree.isSatisfiedBy(key, row, staticRow);
+            Assert.assertTrue(exclusions.contains(i) != result);
         }
 
         // now let's test aggregated AND commands
         node = new Operation.AndNode();
 
+        // logical should be ignored by analyzer, but we still what to make sure that it is
+        //IndexExpression logical = new IndexExpression(ByteBufferUtil.EMPTY_BYTE_BUFFER, IndexOperator.EQ, ByteBufferUtil.EMPTY_BYTE_BUFFER);
+        //logical.setLogicalOp(LogicalIndexOperator.AND);
+
+        //builder.add(logical);
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(0))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(10))));
+        node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.NEQ, Int32Type.instance.decompose(7))));
 
-        filterTree = node.buildFilter(controller, true);
+        filterTree = node.buildFilter(controller);
 
+        exclusions = Sets.newHashSet(7);
         for (int i = 0; i < 10; i++)
         {
             row = buildRow(buildCell(age, instance.decompose(i), System.currentTimeMillis()));
 
-            boolean result = filterTree.isSatisfiedBy(key, (Row) row, staticRow);
-            assertTrue(result);
+            boolean result = filterTree.isSatisfiedBy(key, row, staticRow);
+            Assert.assertTrue(exclusions.contains(i) != result);
         }
 
         // multiple analyzed expressions in the Operation timestamp >= 10 AND age = 5
@@ -204,27 +371,76 @@ public void testSatisfiedBy()
         node.add(new Operation.ExpressionNode(new SimpleExpression(timestamp, Operator.GTE, LongType.instance.decompose(10L))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(5))));
 
-        FilterTree filterTreeStrict = node.buildFilter(controller, true);
-        FilterTree filterTreeNonStrict = node.buildFilter(controller, false);
+        filterTree = node.buildFilter(controller);
+
+        row = buildRow(buildCell(age, instance.decompose(6), System.currentTimeMillis()),
+                                  buildCell(timestamp, LongType.instance.decompose(11L), System.currentTimeMillis()));
+
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow));
+
+        row = buildRow(buildCell(age, instance.decompose(5), System.currentTimeMillis()),
+                                  buildCell(timestamp, LongType.instance.decompose(22L), System.currentTimeMillis()));
+
+        Assert.assertTrue(filterTree.isSatisfiedBy(key, row, staticRow));
 
-        long startTime = System.currentTimeMillis();
-        row = buildRow(buildCell(age, instance.decompose(6), startTime),
-                       buildCell(timestamp, LongType.instance.decompose(11L), startTime + 1));
+        row = buildRow(buildCell(age, instance.decompose(5), System.currentTimeMillis()),
+                                  buildCell(timestamp, LongType.instance.decompose(9L), System.currentTimeMillis()));
 
-        assertFalse(filterTreeStrict.isSatisfiedBy(key, (Row) row, staticRow));
-        assertTrue(filterTreeNonStrict.isSatisfiedBy(key, (Row) row, staticRow)); // matches on timestamp >= 10
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow));
 
-        row = buildRow(buildCell(age, instance.decompose(5), startTime + 2),
-                       buildCell(timestamp, LongType.instance.decompose(22L), startTime + 3));
+        // operation with internal expressions and right child
+        node = new Operation.OrNode();
+        node.add(new Operation.ExpressionNode(new SimpleExpression(timestamp, Operator.GT, LongType.instance.decompose(10L))));
+        Operation.Node child = new Operation.AndNode();
+        child.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GT, Int32Type.instance.decompose(0))));
+        child.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(10))));
+        node.add(child);
+        filterTree = node.buildFilter(controller);
 
-        assertTrue(filterTreeStrict.isSatisfiedBy(key, (Row) row, staticRow));
-        assertTrue(filterTreeNonStrict.isSatisfiedBy(key, (Row) row, staticRow));
+        row = buildRow(buildCell(age, instance.decompose(5), System.currentTimeMillis()),
+                                  buildCell(timestamp, LongType.instance.decompose(9L), System.currentTimeMillis()));
+
+        Assert.assertTrue(filterTree.isSatisfiedBy(key, row, staticRow));
+
+        row = buildRow(buildCell(age, instance.decompose(20), System.currentTimeMillis()),
+                                  buildCell(timestamp, LongType.instance.decompose(11L), System.currentTimeMillis()));
+
+        Assert.assertTrue(filterTree.isSatisfiedBy(key, row, staticRow));
+
+        row = buildRow(buildCell(age, instance.decompose(0), System.currentTimeMillis()),
+                                  buildCell(timestamp, LongType.instance.decompose(9L), System.currentTimeMillis()));
+
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow));
+
+        // and for desert let's try out null and deleted rows etc.
+        node = new Operation.AndNode();
+        node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(30))));
+        filterTree = node.buildFilter(controller);
 
-        row = buildRow(buildCell(age, instance.decompose(5), startTime + 4),
-                       buildCell(timestamp, LongType.instance.decompose(9L), startTime + 5));
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, null, staticRow));
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, row, null));
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow));
 
-        assertFalse(filterTreeStrict.isSatisfiedBy(key, (Row) row, staticRow));
-        assertTrue(filterTreeNonStrict.isSatisfiedBy(key, (Row) row, staticRow)); // matches on age = 5
+        long now = System.currentTimeMillis();
+
+        row = OperationTest.buildRow(
+        Row.Deletion.regular(DeletionTime.build(now - 10, (int) (now / 1000))),
+        buildCell(age, instance.decompose(6), System.currentTimeMillis()));
+
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow));
+
+        row = buildRow(deletedCell(age, System.currentTimeMillis(), FBUtilities.nowInSeconds()));
+
+        Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow));
+
+        try
+        {
+            Assert.assertFalse(filterTree.isSatisfiedBy(key, buildRow(), staticRow));
+        }
+        catch (IllegalStateException e)
+        {
+            Assert.fail("IllegalStateException should not be thrown when missing column");
+        }
     }
 
     @Test
@@ -233,35 +449,50 @@ public void testAnalyzeNotIndexedButDefinedColumn()
         final ColumnMetadata firstName = getColumn(UTF8Type.instance.decompose("first_name"));
         final ColumnMetadata height = getColumn(UTF8Type.instance.decompose("height"));
 
-        // first_name = 'a' AND height > 5
-        Map<Expression.IndexOperator, Expression> expressions;
-        expressions = convert(Operation.buildIndexExpressions(controller,
-                                                              Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")),
-                                                                   new SimpleExpression(height, Operator.GT, Int32Type.instance.decompose(5)))));
-
-        assertEquals(2, expressions.size());
+        // first_name = 'a' AND height != 10
+        Map<Expression.Op, Expression> expressions;
+        expressions = convert(Operation.analyzeGroup(controller, Operation.OperationType.AND,
+                                                     Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")),
+                              new SimpleExpression(height, Operator.NEQ, Int32Type.instance.decompose(5)))));
 
-        expressions = convert(Operation.buildIndexExpressions(controller,
-                                                              Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")),
-                                                                   new SimpleExpression(height, Operator.GT, Int32Type.instance.decompose(0)),
-                                                                   new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(5)))));
+        Assert.assertEquals(2, expressions.size());
 
-        assertEquals(2, expressions.size());
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("height", Int32Type.instance))
+        {{
+                operation = Op.NOT_EQ;
+                lower = new Bound(Int32Type.instance.decompose(5), Int32Type.instance, true);
+                upper = lower;
+        }}, expressions.get(Expression.Op.NOT_EQ));
 
-        Expression rangeExpression = expressions.get(Expression.IndexOperator.RANGE);
+        expressions = convert(Operation.analyzeGroup(controller, Operation.OperationType.AND,
+                                                     Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")),
+                              new SimpleExpression(height, Operator.GT, Int32Type.instance.decompose(0)),
+                              new SimpleExpression(height, Operator.NEQ, Int32Type.instance.decompose(5)))));
 
-        assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(0), false, Int32Type.instance.decompose(5), true);
+        Assert.assertEquals(2, expressions.size());
 
-        expressions = convert(Operation.buildIndexExpressions(controller,
-                                                              Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")),
-                                                                            new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(0)),
-                                                                            new SimpleExpression(height, Operator.LT, Int32Type.instance.decompose(10)))));
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("height", Int32Type.instance))
+        {{
+            operation = Op.RANGE;
+            lower = new Bound(Int32Type.instance.decompose(0), Int32Type.instance, false);
+            exclusions.add(Int32Type.instance.decompose(5));
+        }}, expressions.get(Expression.Op.RANGE));
 
-        assertEquals(2, expressions.size());
+        expressions = convert(Operation.analyzeGroup(controller, Operation.OperationType.AND,
+                                                     Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")),
+                              new SimpleExpression(height, Operator.NEQ, Int32Type.instance.decompose(5)),
+                              new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(0)),
+                              new SimpleExpression(height, Operator.LT, Int32Type.instance.decompose(10)))));
 
-        rangeExpression = expressions.get(Expression.IndexOperator.RANGE);
+        Assert.assertEquals(2, expressions.size());
 
-        assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(0), true, Int32Type.instance.decompose(10), false);
+        Assert.assertEquals(new Expression(SAITester.createIndexContext("height", Int32Type.instance))
+        {{
+                operation = Op.RANGE;
+                lower = new Bound(Int32Type.instance.decompose(0), Int32Type.instance, true);
+                upper = new Bound(Int32Type.instance.decompose(10), Int32Type.instance, false);
+                exclusions.add(Int32Type.instance.decompose(5));
+        }}, expressions.get(Expression.Op.RANGE));
     }
 
     @Test
@@ -273,7 +504,7 @@ public void testSatisfiedByWithClustering()
         ColumnMetadata score = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("score"));
 
         DecoratedKey key = buildKey(CLUSTERING_BACKEND, "0");
-        Row row = buildRow(Clustering.make(UTF8Type.instance.fromString("US"), Int32Type.instance.decompose(27)),
+        Unfiltered row = buildRow(Clustering.make(UTF8Type.instance.fromString("US"), Int32Type.instance.decompose(27)),
                                   buildCell(height, instance.decompose(182), System.currentTimeMillis()),
                                   buildCell(score, DoubleType.instance.decompose(1.0d), System.currentTimeMillis()));
         Row staticRow = buildRow(Clustering.STATIC_CLUSTERING);
@@ -282,56 +513,56 @@ public void testSatisfiedByWithClustering()
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(27))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(182))));
 
-        assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow));
+        Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow));
 
         node = new Operation.AndNode();
 
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(28))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(182))));
 
-        assertFalse(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow));
+        Assert.assertFalse(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow));
 
         node = new Operation.AndNode();
         node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US"))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(27))));
 
-        assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow));
+        Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow));
 
         node = new Operation.AndNode();
         node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("BY"))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(28))));
 
-        assertFalse(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow));
+        Assert.assertFalse(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow));
 
         node = new Operation.AndNode();
         node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US"))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.LTE, Int32Type.instance.decompose(27))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182))));
 
-        assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow));
+        Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow));
 
         node = new Operation.AndNode();
         node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US"))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(score, Operator.EQ, DoubleType.instance.decompose(1.0d))));
 
-        assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow));
+        Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow));
 
         node = new Operation.AndNode();
         node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(score, Operator.EQ, DoubleType.instance.decompose(1.0d))));
 
-        assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow));
+        Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow));
     }
 
-    private Map<Expression.IndexOperator, Expression> convert(Multimap<ColumnMetadata, Expression> expressions)
+    private Map<Expression.Op, Expression> convert(Multimap<ColumnMetadata, Expression> expressions)
     {
-        Map<Expression.IndexOperator, Expression> converted = new EnumMap<>(Expression.IndexOperator.class);
+        Map<Expression.Op, Expression> converted = new HashMap<>();
         for (Expression expression : expressions.values())
         {
-            Expression column = converted.get(expression.getIndexOperator());
+            Expression column = converted.get(expression.getOp());
             assert column == null; // sanity check
-            converted.put(expression.getIndexOperator(), expression);
+            converted.put(expression.getOp(), expression);
         }
 
         return converted;
@@ -344,7 +575,7 @@ public void testSatisfiedByWithStatic()
         final ColumnMetadata value = getColumn(STATIC_BACKEND, UTF8Type.instance.decompose("value"));
 
         DecoratedKey key = buildKey(STATIC_BACKEND, 0);
-        Row row = buildRow(Clustering.make(UTF8Type.instance.fromString("date"), LongType.instance.decompose(20160401L)),
+        Unfiltered row = buildRow(Clustering.make(UTF8Type.instance.fromString("date"), LongType.instance.decompose(20160401L)),
                                   buildCell(value, DoubleType.instance.decompose(24.56), System.currentTimeMillis()));
         Row staticRow = buildRow(Clustering.STATIC_CLUSTERING,
                                  buildCell(sensorType, UTF8Type.instance.decompose("TEMPERATURE"), System.currentTimeMillis()));
@@ -354,108 +585,81 @@ public void testSatisfiedByWithStatic()
         node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("TEMPERATURE"))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(24.56))));
 
-        assertTrue(node.buildFilter(controllerStatic, true).isSatisfiedBy(key, row, staticRow));
+        Assert.assertTrue(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow));
 
         // sensor_type ='TEMPERATURE' AND value = 30
         node = new Operation.AndNode();
         node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("TEMPERATURE"))));
         node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(30.00))));
 
-        assertFalse(node.buildFilter(controllerStatic, true).isSatisfiedBy(key, row, staticRow));
-    }
+        Assert.assertFalse(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow));
 
-    public static TableMetadata.Builder skinnySAITableMetadata(String keyspace, String table)
-    {
-        TableMetadata.Builder builder =
-        TableMetadata.builder(keyspace, table)
-                     .addPartitionKeyColumn("id", UTF8Type.instance)
-                     .addRegularColumn("first_name", UTF8Type.instance)
-                     .addRegularColumn("last_name", UTF8Type.instance)
-                     .addRegularColumn("age", Int32Type.instance)
-                     .addRegularColumn("height", Int32Type.instance)
-                     .addRegularColumn("timestamp", LongType.instance)
-                     .addRegularColumn("address", UTF8Type.instance)
-                     .addRegularColumn("score", DoubleType.instance);
-
-        Indexes.Builder indexes = Indexes.builder();
-        addIndex(indexes, table, "first_name");
-        addIndex(indexes, table, "last_name");
-        addIndex(indexes, table, "age");
-        addIndex(indexes, table, "timestamp");
-        addIndex(indexes, table, "address");
-        addIndex(indexes, table, "score");
-
-        return builder.indexes(indexes.build());
-    }
+        // sensor_type ='PRESSURE' OR value = 24.56
+        node = new Operation.OrNode();
+        node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("PRESSURE"))));
+        node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(24.56))));
 
-    public static TableMetadata.Builder clusteringSAITableMetadata(String keyspace, String table)
-    {
-        return clusteringSAITableMetadata(keyspace, table, "location", "age", "height", "score");
-    }
+        Assert.assertTrue(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow));
 
-    public static TableMetadata.Builder clusteringSAITableMetadata(String keyspace, String table, String...indexedColumns)
-    {
-        Indexes.Builder indexes = Indexes.builder();
-        for (String indexedColumn : indexedColumns)
-        {
-            addIndex(indexes, table, indexedColumn);
-        }
+        // sensor_type ='PRESSURE' OR value = 30
+        node = new Operation.OrNode();
+        node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("PRESSURE"))));
+        node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(30.00))));
 
-        return TableMetadata.builder(keyspace, table)
-                            .addPartitionKeyColumn("name", UTF8Type.instance)
-                            .addClusteringColumn("location", UTF8Type.instance)
-                            .addClusteringColumn("age", Int32Type.instance)
-                            .addRegularColumn("height", Int32Type.instance)
-                            .addRegularColumn("score", DoubleType.instance)
-                            .indexes(indexes.build());
-    }
+        Assert.assertFalse(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow));
 
-    public static TableMetadata.Builder staticSAITableMetadata(String keyspace, String table)
-    {
-        TableMetadata.Builder builder =
-        TableMetadata.builder(keyspace, table)
-                     .addPartitionKeyColumn("sensor_id", Int32Type.instance)
-                     .addStaticColumn("sensor_type", UTF8Type.instance)
-                     .addClusteringColumn("date", LongType.instance)
-                     .addRegularColumn("value", DoubleType.instance)
-                     .addRegularColumn("variance", Int32Type.instance);
+        // (sensor_type = 'TEMPERATURE' OR sensor_type = 'PRESSURE') AND value = 24.56
+        node = new Operation.AndNode();
+        Operation.Node child = new Operation.OrNode();
+        child.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("TEMPERATURE"))));
+        child.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("PRESSURE"))));
+        node.add(child);
+        node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(24.56))));
 
-        Indexes.Builder indexes = Indexes.builder();
+        Assert.assertTrue(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow));
 
-        addIndex(indexes, table, "sensor_type");
-        addIndex(indexes, table, "value");
-        addIndex(indexes, table, "variance");
+        // sensor_type = LIKE 'TEMP%'  AND value = 24.56
+        node = new Operation.AndNode();
+        node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.LIKE_PREFIX, UTF8Type.instance.decompose("TEMP"))));
+        node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(24.56))));
 
-        return builder.indexes(indexes.build());
+        Assert.assertTrue(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow));
     }
 
-    private void assertExpression(Expression expression, Expression.IndexOperator indexOperator, ByteBuffer lower,
-                                  boolean lowerInclusive, ByteBuffer upper, boolean upperInclusive)
+    private static class SimpleExpression extends RowFilter.Expression
     {
-        assertEquals(indexOperator, expression.getIndexOperator());
-        assertEquals(lower, expression.lower().value.raw);
-        assertEquals(lowerInclusive, expression.lower().inclusive);
-        assertEquals(upper, expression.upper().value.raw);
-        assertEquals(upperInclusive, expression.upper().inclusive);
-    }
+        SimpleExpression(ColumnMetadata column, Operator operator, ByteBuffer value)
+        {
+            super(column, operator, value);
+        }
 
-    private static void addIndex(Indexes.Builder indexes, String table, String column)
-    {
-        String indexName = table + '_' + column;
-        indexes.add(IndexMetadata.fromSchemaMetadata(indexName, IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
-        {{
-            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
-            put(IndexTarget.TARGET_OPTION_NAME, column);
-        }}));
+        @Override
+        public Kind kind()
+        {
+            return Kind.SIMPLE;
+        }
+
+        @Override
+        public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String toString(boolean cql)
+        {
+            return String.format("%s %s %s",
+                    cql ? column.name.toCQLString() : column.name.toString(),
+                    operator,
+                    ByteBufferUtil.bytesToHex(value));
+        }
     }
 
-    private static DecoratedKey buildKey(Object... key)
-    {
+    private static DecoratedKey buildKey(Object... key) {
         return buildKey(BACKEND, key);
     }
 
-    private static DecoratedKey buildKey(ColumnFamilyStore cfs, Object... key)
-    {
+    private static DecoratedKey buildKey(ColumnFamilyStore cfs, Object... key) {
         AbstractType<?> type = cfs.metadata().partitionKeyType;
         ByteBuffer decomposed;
         if(type instanceof CompositeType)
@@ -471,26 +675,44 @@ private static DecoratedKey buildKey(ColumnFamilyStore cfs, Object... key)
         return Murmur3Partitioner.instance.decorateKey(decomposed);
     }
 
-    private static Unfiltered buildRow(Cell<?>... cells)
+    private static Unfiltered buildRow(Cell... cells)
     {
-        return buildRow(Clustering.EMPTY, cells);
+        return buildRow(Clustering.EMPTY, null, cells);
     }
 
-    private static Row buildRow(Clustering<?> clustering, Cell<?>... cells)
+    private static Row buildRow(Row.Deletion deletion, Cell... cells)
+    {
+        return buildRow(Clustering.EMPTY, deletion, cells);
+    }
+
+    private static Row buildRow(Clustering clustering, Cell... cells)
+    {
+        return buildRow(clustering, null, cells);
+    }
+
+    private static Row buildRow(Clustering clustering, Row.Deletion deletion, Cell... cells)
     {
         Row.Builder rowBuilder = BTreeRow.sortedBuilder();
         rowBuilder.newRow(clustering);
-        for (Cell<?> c : cells)
+        for (Cell c : cells)
             rowBuilder.addCell(c);
 
+        if (deletion != null)
+            rowBuilder.addRowDeletion(deletion);
+
         return rowBuilder.build();
     }
 
-    private static Cell<?> buildCell(ColumnMetadata column, ByteBuffer value, long timestamp)
+    private static Cell buildCell(ColumnMetadata column, ByteBuffer value, long timestamp)
     {
         return BufferCell.live(column, timestamp, value);
     }
 
+    private static Cell deletedCell(ColumnMetadata column, long timestamp, long nowInSeconds)
+    {
+        return BufferCell.tombstone(column, timestamp, nowInSeconds);
+    }
+
     private static ColumnMetadata getColumn(ByteBuffer name)
     {
         return getColumn(BACKEND, name);
@@ -500,50 +722,4 @@ private static ColumnMetadata getColumn(ColumnFamilyStore cfs, ByteBuffer name)
     {
         return cfs.metadata().getColumn(name);
     }
-
-    private static class SimpleExpression extends RowFilter.Expression
-    {
-        SimpleExpression(ColumnMetadata column, Operator operator, ByteBuffer value)
-        {
-            super(column, operator, value);
-        }
-
-        @Override
-        public Kind kind()
-        {
-            return Kind.SIMPLE;
-        }
-
-        @Override
-        public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        protected String toString(boolean cql)
-        {
-            AbstractType<?> type = column.type;
-            switch (operator)
-            {
-                case CONTAINS:
-                    assert type instanceof CollectionType;
-                    CollectionType<?> ct = (CollectionType<?>)type;
-                    type = ct.kind == CollectionType.Kind.SET ? ct.nameComparator() : ct.valueComparator();
-                    break;
-                case CONTAINS_KEY:
-                    assert type instanceof MapType;
-                    type = ((MapType<?, ?>)type).nameComparator();
-                    break;
-                case IN:
-                    type = ListType.getInstance(type, false);
-                    break;
-                default:
-                    break;
-            }
-            return cql
-                   ? String.format("%s %s %s", column.name.toCQLString(), operator, type.toCQLString(value) )
-                   : String.format("%s %s %s", column.name.toString(), operator, type.getString(value));
-        }
-    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/plan/PlanTest.java b/test/unit/org/apache/cassandra/index/sai/plan/PlanTest.java
new file mode 100644
index 000000000000..4f39b89ffee6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/plan/PlanTest.java
@@ -0,0 +1,1078 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.iterators.LongIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyWithSortKey;
+import org.mockito.Mockito;
+
+import static java.lang.Math.ceil;
+import static java.lang.Math.round;
+import static org.apache.cassandra.index.sai.plan.Plan.CostCoefficients.ANN_EDGELIST_COST;
+import static org.apache.cassandra.index.sai.plan.Plan.CostCoefficients.ANN_SIMILARITY_COST;
+import static org.apache.cassandra.index.sai.plan.Plan.CostCoefficients.ROW_COST;
+import static org.apache.cassandra.index.sai.plan.Plan.CostCoefficients.SAI_KEY_COST;
+import static org.apache.cassandra.index.sai.plan.Plan.CostCoefficients.SAI_OPEN_COST;
+import static org.junit.Assert.*;
+
+public class PlanTest
+{
+    private static final Orderer ordering = orderer();
+
+    @BeforeClass
+    public static void setupDD()
+    {
+        Plan.hitRateSupplier = () -> 1.0;
+    }
+
+    private static Orderer orderer()
+    {
+        Orderer orderer = Mockito.mock(Orderer.class);
+        Mockito.when(orderer.isANN()).thenReturn(true);
+        return orderer;
+    }
+
+    private static final RowFilter.Expression pred1 = filerPred("pred1", Operator.LT);
+    private static final RowFilter.Expression pred2 = filerPred("pred2", Operator.LT);
+    private static final RowFilter.Expression pred3 = filerPred("pred3", Operator.LT);
+    private static final RowFilter.Expression pred4 = filerPred("pred4", Operator.LT);
+
+    private static final Expression saiPred1 = saiPred("pred1", Expression.Op.RANGE, false);
+    private static final  Expression saiPred2 = saiPred("pred2", Expression.Op.RANGE, false);
+    private static final  Expression saiPred3 = saiPred("pred3", Expression.Op.RANGE, false);
+    private static final  Expression saiPred4 = saiPred("pred4", Expression.Op.RANGE, true);
+
+    private static final RowFilter rowFilter1 = RowFilter.builder(true).add(pred1).build();
+    private static final RowFilter rowFilter12 = RowFilter.builder(true).add(pred1).add(pred2).build();
+    private static final RowFilter rowFilter123 = RowFilter.builder(true).add(pred1).add(pred2).add(pred3).build();
+
+    private final Plan.TableMetrics table1M = new Plan.TableMetrics(1_000_000, 7, 128, 3);
+    private final Plan.TableMetrics table10M = new Plan.TableMetrics(10_000_000, 7, 128, 8);
+
+    private final Plan.Factory factory = new Plan.Factory(table1M, new CostEstimator(table1M));
+
+
+    static {
+        // For consistent display of plan trees
+        Locale.setDefault(Locale.ENGLISH);
+    }
+
+    private static Expression saiPred(String column, Expression.Op operation, boolean isLiteral)
+    {
+        Expression pred = Mockito.mock(Expression.class);
+        Mockito.when(pred.toString()).thenReturn(operation.toString() + '(' + column + ')');
+        Mockito.when(pred.getIndexName()).thenReturn(column + "_idx");
+        Mockito.when(pred.getOp()).thenReturn(operation);
+        Mockito.when(pred.isLiteral()).thenReturn(isLiteral);
+        return pred;
+    }
+
+    private static RowFilter.Expression filerPred(String column, Operator operation)
+    {
+        RowFilter.Expression pred = Mockito.mock(RowFilter.Expression.class);
+        Mockito.when(pred.toString(false)).thenReturn(column + ' ' + operation + " X");
+        Mockito.when(pred.operator()).thenReturn(operation);
+        return pred;
+    }
+
+    @Test
+    public void empty()
+    {
+        Plan.KeysIteration plan = factory.indexScan(saiPred1, 0);
+        assertTrue(plan instanceof Plan.NumericIndexScan);
+        assertEquals(0.0, plan.expectedKeys(), 0.01);
+        assertEquals(0.0, plan.selectivity(), 0.01);
+        assertEquals(0.0, plan.costPerKey(), 0.01);
+    }
+
+    @Test
+    public void single()
+    {
+        Plan.KeysIteration plan = factory.indexScan(saiPred1, (long)(0.5 * factory.tableMetrics.rows));
+        assertTrue(plan instanceof Plan.NumericIndexScan);
+        assertEquals(0.5 * factory.tableMetrics.rows, plan.expectedKeys(), 0.01);
+        assertEquals(0.5, plan.selectivity(), 0.01);
+        assertEquals(SAI_KEY_COST, plan.costPerKey(), 0.01);
+    }
+
+    @Test
+    public void intersection()
+    {
+        Plan.KeysIteration a1 = factory.indexScan(saiPred1, (long)(0.2 * factory.tableMetrics.rows));
+        Plan.KeysIteration a2 = factory.indexScan(saiPred2, (long)(0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan1 = factory.intersection(Lists.newArrayList(a1, a2));
+
+        assertTrue(plan1 instanceof Plan.Intersection);
+        assertEquals(0.1, plan1.selectivity(), 0.01);
+        assertTrue(plan1.costPerKey() > a1.costPerKey());
+        assertTrue(plan1.costPerKey() > a2.costPerKey());
+
+        Plan.KeysIteration b1 = factory.indexScan(saiPred1, (long)(0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration b2 = factory.indexScan(saiPred2, (long)(0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan2 = factory.intersection(Lists.newArrayList(b1, b2));
+
+        assertTrue(plan2 instanceof Plan.Intersection);
+        assertEquals(0.0001, plan2.selectivity(), 1e-9);
+        assertEquals(0.0001 * factory.tableMetrics.rows, plan2.expectedKeys(), 1e-9);
+        assertTrue(plan2.costPerKey() > plan1.costPerKey());
+    }
+
+    @Test
+    public void intersectionWithEmpty()
+    {
+        Plan.KeysIteration a1 = factory.indexScan(saiPred1, 0);
+        Plan.KeysIteration a2 = factory.indexScan(saiPred2, (long)(0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan = factory.intersection(Lists.newArrayList(a1, a2));
+        assertEquals(0.0, plan.selectivity(), 0.0001);
+        assertEquals(0.0, plan.expectedKeys(), 0.0001);
+
+        // There should be no cost of iterating the iterator a2,
+        // because there are no keyst to match on the left side (a1) and the intersection loop would exit early
+        assertEquals(plan.fullCost(), 2 * factory.tableMetrics.sstables * SAI_OPEN_COST, 0.0001);
+    }
+
+    @Test
+    public void intersectionWithNothing()
+    {
+        Plan.KeysIteration a1 = factory.nothing;
+        Plan.KeysIteration a2 = factory.indexScan(saiPred2, (long)(0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan = factory.intersection(Lists.newArrayList(a1, a2));
+        assertEquals(0.0, plan.selectivity(), 0.0001);
+        assertEquals(0.0, plan.expectedKeys(), 0.0001);
+        assertTrue(plan.fullCost() <= SAI_KEY_COST * 2);
+    }
+
+    @Test
+    public void intersectionWithEverything()
+    {
+        Plan.KeysIteration a1 = factory.everything;
+        Plan.KeysIteration a2 = factory.indexScan(saiPred2, (long)(0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan = factory.intersection(Lists.newArrayList(a1, a2));
+        assertSame(a2, plan);
+
+        assertSame(factory.everything, factory.intersection(Lists.newArrayList()));
+        assertSame(factory.everything, factory.intersection(Lists.newArrayList(factory.everything)));
+        assertSame(factory.everything, factory.intersection(Lists.newArrayList(factory.everything, factory.everything)));
+    }
+
+    @Test
+    public void intersectionWithUnion()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration u = factory.union(Lists.newArrayList(s2, s3));
+        Plan.KeysIteration i1 = factory.intersection(Lists.newArrayList(s1, u));
+        Plan.KeysIteration i2 = factory.intersection(Lists.newArrayList(s1, s2));
+        assertTrue(i1.initCost() > i2.initCost());
+        assertTrue(i1.fullCost() > i2.fullCost());
+    }
+
+    @Test
+    public void nestedIntersections()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration nested = factory.intersection(Lists.newArrayList(s2, s3));
+        Plan.KeysIteration i1 = factory.intersection(Lists.newArrayList(s1, nested));
+        Plan.KeysIteration i2 = factory.intersection(Lists.newArrayList(s1, s2, s3));
+        assertEquals(i1.initCost(), i2.initCost(), 0.01);
+        assertEquals(i1.fullCost(), i2.fullCost(), 0.01);
+    }
+
+    @Test
+    public void rangeScanVsPointLookupIntersection()
+    {
+        // Intersecting range scans is
+
+        Plan.KeysIteration n1 = factory.indexScan(saiPred("a", Expression.Op.RANGE, false),
+                                                         (long)(0.1 * factory.tableMetrics.rows));
+        Plan.KeysIteration n2 = factory.indexScan(saiPred("b", Expression.Op.RANGE, false),
+                                                         (long)(0.1 * factory.tableMetrics.rows));
+        Plan.KeysIteration ni = factory.intersection(Lists.newArrayList(n1, n2));
+
+        Plan.KeysIteration l1 = factory.indexScan(saiPred("c", Expression.Op.EQ, true),
+                                                         (long)(0.1 * factory.tableMetrics.rows));
+        Plan.KeysIteration l2 = factory.indexScan(saiPred("d", Expression.Op.EQ, true),
+                                                         (long)(0.1 * factory.tableMetrics.rows));
+        Plan.KeysIteration li = factory.intersection(Lists.newArrayList(l1, l2));
+
+        assertEquals(li.expectedKeys(), ni.expectedKeys(), 0.01);
+
+        assertTrue(li.fullCost() < ni.fullCost());
+    }
+
+    @Test
+    public void intersectThree()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.8 * factory.tableMetrics.rows));
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, (long) (0.8 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersect2 = factory.intersection(Lists.newArrayList(s1, s2));
+        Plan.KeysIteration intersect3 = factory.intersection(Lists.newArrayList(s1, s2, s3));
+
+        assertTrue(intersect3 instanceof Plan.Intersection);
+        assertEquals(0.32, intersect3.selectivity(), 0.01);
+        assertEquals(0.32 * factory.tableMetrics.rows, intersect3.expectedKeys(), 0.01);
+        assertTrue(intersect3.fullCost() > intersect2.fullCost());
+    }
+
+    @Test
+    public void union()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan = factory.union(Lists.newArrayList(s1, s2));
+
+        assertTrue(plan instanceof Plan.Union);
+        assertEquals(0.75, plan.selectivity(), 0.01);
+        assertEquals(0.75 * factory.tableMetrics.rows, plan.expectedKeys(), 0.01);
+        assertEquals( 1.333333333 * (SAI_KEY_COST), plan.costPerKey(), 0.01);
+        assertTrue(plan.fullCost() >= s1.fullCost() + s2.fullCost());
+    }
+
+    @Test
+    public void unionWithNoting()
+    {
+        Plan.KeysIteration s1 = factory.nothing;
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan = factory.union(Lists.newArrayList(s1, s2));
+
+        assertSame(s2, plan);
+        assertSame(factory.nothing, factory.union(Collections.emptyList()));
+    }
+
+    @Test
+    public void unionWithEverything()
+    {
+        Plan.KeysIteration s1 = factory.everything;
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan = factory.union(Lists.newArrayList(s1, s2));
+
+        assertSame(factory.everything, plan);
+    }
+
+    @Test
+    public void fetch()
+    {
+        Plan.KeysIteration i = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.RowsIteration s = factory.fetch(i);
+        assertEquals(0.5 * factory.tableMetrics.rows, s.expectedRows(), 0.01);
+        assertTrue(s.fullCost() > 0.5 * factory.tableMetrics.rows * (SAI_KEY_COST + ROW_COST));
+    }
+
+    @Test
+    public void limit()
+    {
+        Plan.KeysIteration i = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.RowsIteration s = factory.fetch(i);
+        Plan.RowsIteration l = factory.limit(s, 10);
+        assertEquals(10, l.expectedRows(), 0.01);
+        assertTrue(l.fullCost() < s.fullCost());
+    }
+
+    @Test
+    public void filter()
+    {
+        Plan.KeysIteration i = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.RowsIteration s = factory.fetch(i);
+        Plan.RowsIteration f = factory.filter(RowFilter.builder(true).add(pred1).build(), s, 0.25);
+        assertEquals(0.25 * factory.tableMetrics.rows, f.expectedRows(), 0.01);
+        assertEquals(0.25, f.selectivity(), 0.01);
+        assertTrue(f.costPerRow() > s.costPerRow());
+    }
+
+    @Test
+    public void filterAndLimit()
+    {
+        Plan.KeysIteration i = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.RowsIteration s = factory.fetch(i);
+        Plan.RowsIteration f = factory.filter(RowFilter.builder(true).add(pred1).build(), s, 0.25);
+        Plan.RowsIteration l = factory.limit(f, 10);
+        assertEquals(10, l.expectedRows(), 0.01);
+        assertEquals(l.costPerRow(), f.costPerRow(), 0.01);
+    }
+
+    @Test
+    public void annSort()
+    {
+        Plan.KeysIteration i = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration s = factory.sort(i, ordering);
+
+        assertEquals(0.5 * factory.tableMetrics.rows, s.expectedKeys(), 0.01);
+        assertTrue(s.initCost() >= i.fullCost());
+    }
+
+    @Test
+    public void annSortFilterLimit()
+    {
+        int limit = 10;
+        double selectivity = 0.2;
+
+        Plan.KeysIteration i = factory.indexScan(saiPred1, (long) (selectivity * factory.tableMetrics.rows));
+        Plan.KeysIteration s = factory.sort(i, ordering);
+        Plan.RowsIteration fetch = factory.fetch(s);
+        Plan.RowsIteration f = factory.filter(rowFilter1, fetch, selectivity);
+        Plan.RowsIteration plan = factory.limit(f, limit);
+
+        // getTopKRows limit must be set to the same as the query limit
+        // because we're getting top of the rows already prefiltered by the index:
+        Plan.Executor executor = Mockito.mock(Plan.Executor.class);
+        Objects.requireNonNull(plan.firstNodeOfType(Plan.KeysIteration.class)).execute(executor);
+        Mockito.verify(executor, Mockito.times(1)).getTopKRows((KeyRangeIterator) Mockito.any(), Mockito.eq(limit));
+    }
+
+    @Test
+    public void annScan()
+    {
+        Plan.KeysIteration i = factory.sort(factory.everything, ordering);
+        assertEquals(factory.tableMetrics.rows, i.expectedKeys(), 0.01);
+        assertEquals(i.initCost() + factory.costEstimator.estimateAnnSearchCost(ordering, (int) ceil(i.expectedKeys()), factory.tableMetrics.rows), i.fullCost(), 0.01);
+    }
+
+    @Test
+    public void annScanFilterLimit()
+    {
+        int limit = 10;
+        double selectivity = 0.2;
+
+        Plan.KeysIteration s = factory.sort(factory.everything, ordering);
+        Plan.RowsIteration fetch = factory.fetch(s);
+        Plan.RowsIteration f = factory.filter(rowFilter1, fetch, selectivity);
+        Plan.RowsIteration plan = factory.limit(f, limit);
+
+        // getTopKRows limit must be adjusted by dividing by predicate selectivity, because we're postfiltering,
+        // and the postfilter will reject many rows:
+        Plan.Executor executor = Mockito.mock(Plan.Executor.class);
+        Objects.requireNonNull(plan.firstNodeOfType(Plan.KeysIteration.class)).execute(executor);
+        Mockito.verify(executor, Mockito.times(1)).getTopKRows((Expression) Mockito.any(), Mockito.eq((int) round(limit / selectivity)));
+    }
+
+    @Test
+    public void annScanOfEmptyTable()
+    {
+        Plan.TableMetrics emptyTable = new Plan.TableMetrics(0, 0, 0, 0);
+        Plan.Factory factory = new Plan.Factory(emptyTable, new CostEstimator(table1M));
+        Plan.KeysIteration plan = factory.sort(factory.everything, ordering);
+        assertEquals(0.0, plan.expectedKeys(), 0.01);
+        assertEquals(1.0, plan.selectivity(), 0.01);
+    }
+
+    @Test
+    public void findNodeByType()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.2 * factory.tableMetrics.rows));
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, (long) (0.1 * factory.tableMetrics.rows));
+        RowFilter rowFilter = RowFilter.builder(true).add(pred1).add(pred2).add(pred3).build();
+
+        Plan.KeysIteration union = factory.union(Lists.newArrayList(factory.intersection(Lists.newArrayList(s1, s2)), s3));
+        Plan.KeysIteration sort = factory.sort(union, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration filter = factory.recheckFilter(rowFilter, fetch);
+        Plan.RowsIteration limit = factory.limit(filter, 3);
+
+        assertEquals(List.of(s2.id, s1.id, s3.id), ids(filter.nodesOfType(Plan.NumericIndexScan.class)));
+        assertEquals(List.of(union.id), ids(filter.nodesOfType(Plan.Union.class)));
+        assertEquals(List.of(fetch.id), ids(filter.nodesOfType(Plan.Fetch.class)));
+        assertEquals(List.of(filter.id), ids(filter.nodesOfType(Plan.Filter.class)));
+
+        // Nodes under limit may be different instances because of limit push-down:
+        assertEquals(List.of(limit), limit.nodesOfType(Plan.Limit.class));
+        assertEquals(1, limit.nodesOfType(Plan.Filter.class).size());
+        assertEquals(1, limit.nodesOfType(Plan.Union.class).size());
+    }
+
+
+    @Test
+    public void removeSubplan()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, 20);
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, 30);
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, 50);
+        Plan.KeysIteration plan1 = factory.intersection(Lists.newArrayList(s1, s2, s3));
+        Plan plan2 = plan1.removeRestriction(s2.id);
+
+        assertNotSame(plan1, plan2);
+        assertEquals(plan1.id, plan2.id);  // although the result plan is different object, the nodes must retain their ids
+        assertTrue(plan2 instanceof Plan.Intersection);
+        assertEquals(Lists.newArrayList(s1.id, s3.id), ids(plan2.subplans()));
+        assertNotEquals(plan1.cost(), plan2.cost());
+
+        Plan plan3 = plan2.removeRestriction(s1.id);
+        assertEquals(s3.id, plan3.id);
+
+        Plan plan4 = plan3.removeRestriction(s3.id);
+        assertTrue(plan4 instanceof Plan.Everything);
+    }
+
+    @Test
+    public void removeNestedSubplan()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, 50);
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, 30);
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, 80);
+        Plan.KeysIteration s4 = factory.indexScan(saiPred4, 50);
+
+        Plan.KeysIteration sub1 = factory.intersection(Lists.newArrayList(s1, s2));
+        Plan.KeysIteration sub2 = factory.intersection(Lists.newArrayList(s3, s4));
+        Plan.KeysIteration plan1 = factory.union(Lists.newArrayList(sub1, sub2));
+        Plan plan2 = plan1.removeRestriction(s2.id).removeRestriction(s3.id);
+
+        Plan reference = factory.union(Lists.newArrayList(s1, s4));
+
+        assertNotSame(plan1, plan2);
+        assertEquals(reference.cost(), plan2.cost());
+        assertTrue(plan2 instanceof Plan.Union);
+    }
+
+    @Test
+    public void intersectionClauseLimit()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, 3);
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, 4);
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, 1);
+        Plan.KeysIteration s4 = factory.indexScan(saiPred4, 2);
+        Plan.KeysIteration intersect = factory.intersection(Lists.newArrayList(s1, s2, s3, s4));
+        Plan.RowsIteration plan = factory.limit(factory.fetch(intersect), 3);
+
+        Plan plan4 = plan.limitIntersectedClauses(4);
+        assertSame(plan, plan4);
+
+        Plan plan3 = plan.limitIntersectedClauses(3);
+        Plan.Intersection intersection3 = plan3.firstNodeOfType(Plan.Intersection.class);
+        assertNotNull(intersection3);
+        assertEquals(List.of(s3.id, s4.id, s1.id), ids(intersection3.subplans()));
+
+        Plan plan2 = plan.limitIntersectedClauses(2);
+        Plan.Intersection intersection2 = plan2.firstNodeOfType(Plan.Intersection.class);
+        assertNotNull(intersection2);
+        assertEquals(List.of(s3.id, s4.id), ids(intersection2.subplans()));
+
+        Plan plan1 = plan.limitIntersectedClauses(1);
+        Plan.Fetch fetch = plan1.firstNodeOfType(Plan.Fetch.class);
+        assertNotNull(fetch);
+        assertSame(s3.id, fetch.subplans().get(0).id);
+    }
+
+    @Test
+    public void rangeIterator()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, 3);
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, 3);
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, 1);
+        Plan.KeysIteration plan = factory.union(Lists.newArrayList(factory.intersection(Lists.newArrayList(s1, s2)), s3));
+
+        Map<Expression, KeyRangeIterator> iterators = new HashMap<>();
+        iterators.put(saiPred1, new LongIterator(new long[] { 1L, 2L, 3L }));
+        iterators.put(saiPred2, new LongIterator(new long[] { 1L, 2L, 5L }));
+        iterators.put(saiPred3, new LongIterator(new long[] { 100L }));
+
+        Plan.Executor executor = new Plan.Executor()
+        {
+            @Override
+            public Iterator<? extends PrimaryKey> getKeysFromIndex(Expression predicate)
+            {
+                return iterators.get(predicate);
+            }
+
+            @Override
+            public Iterator<? extends PrimaryKey> getTopKRows(Expression predicate, int softLimit)
+            {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public Iterator<PrimaryKeyWithSortKey> getTopKRows(KeyRangeIterator keys, int softLimit)
+            {
+                throw new UnsupportedOperationException();
+            }
+        };
+
+        KeyRangeIterator iterator = (KeyRangeIterator) plan.execute(executor);
+        assertEquals(LongIterator.convert(1L, 2L, 100L), LongIterator.convert(iterator));
+    }
+
+    @Test
+    public void builder()
+    {
+        Plan plan1 = factory.intersectionBuilder()
+                            .add(factory.indexScan(saiPred1, 50))
+                            .add(factory.indexScan(saiPred2, 50))
+                            .build();
+        assertTrue(plan1 instanceof Plan.Intersection);
+        assertEquals(2, plan1.subplans().size());
+
+        Plan plan2 = factory.unionBuilder()
+                            .add(factory.indexScan(saiPred3, 50))
+                            .add(factory.indexScan(saiPred4, 50))
+                            .build();
+        assertTrue(plan2 instanceof Plan.Union);
+        assertEquals(2, plan2.subplans().size());
+    }
+
+    @Test
+    public void prettyPrint()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.002 * factory.tableMetrics.rows));
+        Plan.KeysIteration s3 = factory.indexScan(saiPred4, (long) (0.001 * factory.tableMetrics.rows));
+        Plan.KeysIteration union = factory.union(Lists.newArrayList(factory.intersection(Lists.newArrayList(s1, s2)), s3));
+        Plan.KeysIteration sort = factory.sort(union, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration filter = factory.recheckFilter(RowFilter.builder(true).add(pred1).add(pred2).add(pred4).build(), fetch);
+        Plan.RowsIteration limit = factory.limit(filter, 3);
+
+        String prettyStr = limit.toStringRecursive();
+
+        assertEquals("Limit 3 (rows: 3.0, cost/row: 3895.8, cost: 44171.3..55858.7)\n" +
+                     " └─ Filter pred1 < X AND pred2 < X AND pred4 < X (sel: 1.000000000) (rows: 3.0, cost/row: 3895.8, cost: 44171.3..55858.7)\n" +
+                     "     └─ Fetch (rows: 3.0, cost/row: 3895.8, cost: 44171.3..55858.7)\n" +
+                     "         └─ KeysSort (keys: 3.0, cost/key: 3792.4, cost: 44171.3..55548.4)\n" +
+                     "             └─ Union (keys: 1999.0, cost/key: 14.8, cost: 13500.0..43001.3)\n" +
+                     "                 ├─ Intersection (keys: 1000.0, cost/key: 29.4, cost: 9000.0..38401.3)\n" +
+                     "                 │   ├─ NumericIndexScan of pred2_idx (sel: 0.002000000, step: 1.0) (keys: 2000.0, cost/key: 0.1, cost: 4500.0..4700.0)\n" +
+                     "                 │   │  predicate: RANGE(pred2)\n" +
+                     "                 │   └─ NumericIndexScan of pred1_idx (sel: 0.500000000, step: 250.0) (keys: 2000.0, cost/key: 14.6, cost: 4500.0..33701.3)\n" +
+                     "                 │      predicate: RANGE(pred1)\n" +
+                     "                 └─ LiteralIndexScan of pred4_idx (sel: 0.001000000, step: 1.0) (keys: 1000.0, cost/key: 0.1, cost: 4500.0..4600.0)\n" +
+                     "                    predicate: RANGE(pred4)\n", prettyStr);
+    }
+
+    @Test
+    public void removeNeedlessIntersections()
+    {
+        // If one of the intersection branches has bad selectivity (here 90%), then performing the intersection
+        // makes no sense, because it will cause more effort to perform the intersection than to fetch the additional
+        // rows that weren't filtered out
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.99 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.001 * factory.tableMetrics.rows));
+        Plan.KeysIteration s3 = factory.indexScan(saiPred3, (long) (0.95 * factory.tableMetrics.rows));
+        Plan.KeysIteration s4 = factory.indexScan(saiPred4, (long) (0.97 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersect = factory.intersection(Lists.newArrayList(s1, s2, s3, s4));
+        Plan.RowsIteration origPlan = factory.fetch(intersect);
+
+        Plan optimizedPlan = origPlan.optimize();
+        assertEquals(List.of(s2.id, s3.id, s4.id, s1.id), ids(intersect.subplans())); // subplans must be sorted by selectivity
+        assertEquals(List.of(s2.id), ids(optimizedPlan.subplans())); // look ma, no intersection under the fetch node
+    }
+
+    @Test
+    public void optimizeIntersectionWithEmpty()
+    {
+        Plan.KeysIteration a1 = factory.indexScan(saiPred1, 0);
+        Plan.KeysIteration a2 = factory.indexScan(saiPred2, (long)(0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration plan = factory.intersection(Lists.newArrayList(a1, a2));
+        Plan optimized = plan.optimize();
+        assertEquals(optimized.id, a1.id);
+    }
+
+    @Test
+    public void leaveGoodIntersection()
+    {
+        // If both intersection selectivities are good, then the intersection shouldn't be removed at all
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.0001 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.2 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersect = factory.intersection(Lists.newArrayList(s1, s2));
+        Plan.RowsIteration origPlan = factory.fetch(intersect);
+
+        Plan optimizedPlan = origPlan.optimize();
+        assertSame(origPlan, optimizedPlan);
+    }
+
+    @Test
+    public void removeNeedlessIntersectionUnderFilter()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.0001 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.9 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersect = factory.intersection(Lists.newArrayList(s1, s2));
+        Plan.RowsIteration fetch = factory.fetch(intersect);
+        RowFilter rowFilter = RowFilter.builder(true).add(pred1).add(pred2).build();
+        Plan.RowsIteration origPlan = factory.recheckFilter(rowFilter, fetch);
+
+        Plan.RowsIteration optimizedPlan = (Plan.RowsIteration) origPlan.optimize();
+        assertFalse(optimizedPlan.contains(p -> p instanceof Plan.Intersection));
+        assertEquals(origPlan.cost().expectedRows, optimizedPlan.cost().expectedRows, 0.001);
+    }
+
+    @Test
+    public void leaveGoodIntersectionUnderFilter()
+    {
+        Plan.KeysIteration s1 = factory.indexScan(saiPred1, (long) (0.1 * factory.tableMetrics.rows));
+        Plan.KeysIteration s2 = factory.indexScan(saiPred2, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersect = factory.intersection(Lists.newArrayList(s1, s2));
+        Plan.RowsIteration fetch = factory.fetch(intersect);
+        RowFilter rowFilter = RowFilter.builder(true).add(pred1).add(pred2).build();
+        Plan.RowsIteration origPlan = factory.recheckFilter(rowFilter, fetch);
+
+        Plan optimizedPlan = origPlan.optimize();
+        assertSame(origPlan, optimizedPlan);
+    }
+
+    @Test
+    public void replaceAnnSortWithAnnScan()
+    {
+        // This is a simulation of a typical hybrid vector query
+        // SELECT * FROM ... WHERE matches_lot_of_rows ORDER BY v ANN OF ... LIMIT n;
+        // If the predicate matches a significant portion of the data and n is small,
+        // then we should switch to scanning the ANN index only and post-filtering.
+        // This allows us to perform such query lazily and the cost should be proportional to n.
+        // Important: this requires hight number of rows in the table, so that the cost of fetching all keys from the index
+        // is significantly larger than the cost of fetching a few result rows from storage.
+        Plan.KeysIteration indexScan = factory.indexScan(saiPred1, (long) (0.1 * factory.tableMetrics.rows));
+        Plan.KeysIteration sort = factory.sort(indexScan, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilter1, fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+
+        Plan optimizedPlan = origPlan.optimize();
+        assertTrue(optimizedPlan.contains(p -> p instanceof Plan.AnnIndexScan));
+
+        // The optimized plan should finish before the original plan even gets the first row out ;)
+        assertTrue(optimizedPlan.cost().fullCost() < origPlan.cost().initCost());
+    }
+
+    @Test
+    public void notReplaceAnnSortWithAnnScan()
+    {
+        // Test for CNDB-9898
+        Plan.KeysIteration indexScan = factory.indexScan(saiPred1, (long) (0.001 * factory.tableMetrics.rows));
+        Plan.KeysIteration sort = factory.sort(indexScan, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilter1, fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 1);
+
+        Plan optimizedPlan = origPlan.optimize();
+        assertFalse(optimizedPlan.contains(p -> p instanceof Plan.AnnIndexScan));
+        assertTrue(optimizedPlan.contains(p -> p instanceof Plan.KeysSort));
+    }
+
+
+    @Test
+    public void replaceIntersectionAndAnnSortWithAnnScan()
+    {
+        // Similar like the previous test, but now with an intersection:
+        Plan.KeysIteration indexScan1 = factory.indexScan(saiPred1, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration indexScan2 = factory.indexScan(saiPred2, (long) (0.1 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersection = factory.intersection(Lists.newArrayList(indexScan1, indexScan2));
+        Plan.KeysIteration sort = factory.sort(intersection, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilter12, fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+
+        Plan optimizedPlan = origPlan.optimize();
+        assertTrue(optimizedPlan.contains(p -> p instanceof Plan.AnnIndexScan));
+
+        // The optimized plan should finish before the original plan even gets the first row out ;)
+        assertTrue(optimizedPlan.cost().fullCost() < origPlan.cost().initCost());
+    }
+
+    @Test
+    public void removeIntersectionBelowAnnSort()
+    {
+        Plan.KeysIteration indexScan1 = factory.indexScan(saiPred1, (long) (0.001 * factory.tableMetrics.rows));
+        Plan.KeysIteration indexScan2 = factory.indexScan(saiPred2, (long) (0.9 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersection = factory.intersection(Lists.newArrayList(indexScan1, indexScan2));
+        Plan.KeysIteration sort = factory.sort(intersection, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilter12, fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+
+        Plan optimizedPlan = origPlan.optimize();
+        assertFalse(optimizedPlan.contains(p -> p instanceof Plan.Intersection));  // no intersection
+        assertFalse(optimizedPlan.contains(p -> p instanceof Plan.AnnIndexScan));    // no direct ANN index scan
+        assertTrue(optimizedPlan.contains(p -> p instanceof Plan.KeysSort));
+        assertTrue(optimizedPlan.contains(p -> p instanceof Plan.NumericIndexScan));
+    }
+
+    @Test
+    public void reduceNumberOfIntersectionsBelowAnnSort()
+    {
+        Plan.KeysIteration indexScan1 = factory.indexScan(saiPred1, (long) (0.001 * factory.tableMetrics.rows));
+        Plan.KeysIteration indexScan2 = factory.indexScan(saiPred2, (long) (0.001 * factory.tableMetrics.rows));
+        Plan.KeysIteration indexScan3 = factory.indexScan(saiPred3, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersection = factory.intersection(Lists.newArrayList(indexScan1, indexScan2, indexScan3));
+        Plan.KeysIteration sort = factory.sort(intersection, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilter123, fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+
+        Plan optimizedPlan = origPlan.optimize();
+
+        Plan.Intersection optimizedIntersection = optimizedPlan.firstNodeOfType(Plan.Intersection.class);
+        assertNotNull(optimizedIntersection);
+        assertEquals(List.of(indexScan1.id, indexScan2.id), ids(optimizedIntersection.subplans()));
+    }
+
+    @Test
+    public void leaveThreeIntersectionsBelowAnnSort()
+    {
+        Plan.KeysIteration indexScan1 = factory.indexScan(saiPred1, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration indexScan2 = factory.indexScan(saiPred2, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration indexScan3 = factory.indexScan(saiPred3, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersection = factory.intersection(Lists.newArrayList(indexScan1, indexScan2, indexScan3));
+        Plan.KeysIteration sort = factory.sort(intersection, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilter123, fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+
+        Plan optimizedPlan = origPlan.optimize();
+        Plan.Intersection optimizedIntersection = optimizedPlan.firstNodeOfType(Plan.Intersection.class);
+        assertNotNull(optimizedIntersection);
+        assertEquals(List.of(indexScan1.id, indexScan2.id, indexScan3.id), ids(optimizedIntersection.subplans()));
+    }
+
+    @Test
+    public void leaveIntersectionsBelowAnnSort()
+    {
+        Plan.KeysIteration indexScan1 = factory.indexScan(saiPred1, (long) (0.001 * factory.tableMetrics.rows));
+        Plan.KeysIteration indexScan2 = factory.indexScan(saiPred2, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersection = factory.intersection(Lists.newArrayList(indexScan1, indexScan2));
+        Plan.KeysIteration sort = factory.sort(intersection, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilter12, fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+
+        Plan optimizedPlan = origPlan.optimize();
+
+
+        assertSame(origPlan, optimizedPlan);
+    }
+
+    @Test
+    public void optimizeIntersectionsUnderLimit()
+    {
+        testIntersectionsUnderLimit(table10M, List.of(0.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0, 0.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0, 0.1), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0, 0.0, 0.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0, 0.0, 0.1), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0, 0.0, 0.0, 0.0), List.of(1));
+
+        testIntersectionsUnderLimit(table1M, List.of(1.0), List.of(1));
+        testIntersectionsUnderLimit(table1M, List.of(0.5), List.of(1));
+        testIntersectionsUnderLimit(table1M, List.of(0.1), List.of(1));
+        testIntersectionsUnderLimit(table1M, List.of(0.0), List.of(1));
+
+        testIntersectionsUnderLimit(table10M, List.of(1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.5), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.1), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0), List.of(1));
+
+
+        testIntersectionsUnderLimit(table10M, List.of(0.1, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.1, 0.02), List.of(1));
+
+        testIntersectionsUnderLimit(table10M, List.of(0.9, 0.9), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.2, 0.2), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.1, 0.1), List.of(1));
+
+        testIntersectionsUnderLimit(table10M, List.of(0.01, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 1.0), List.of(1));
+
+        testIntersectionsUnderLimit(table10M, List.of(0.1, 0.1, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.1, 0.1, 0.5), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.1, 0.1, 0.2), List.of(2));
+        testIntersectionsUnderLimit(table10M, List.of(0.1, 0.1, 0.1), List.of(2));
+
+        testIntersectionsUnderLimit(table10M, List.of(0.01, 0.01, 1.0), List.of(2));
+        testIntersectionsUnderLimit(table10M, List.of(0.01, 0.01, 0.5), List.of(2));
+        testIntersectionsUnderLimit(table10M, List.of(0.01, 0.01, 0.2), List.of(2));
+        testIntersectionsUnderLimit(table10M, List.of(0.01, 0.01, 0.1), List.of(2));
+        testIntersectionsUnderLimit(table10M, List.of(0.01, 0.01, 0.05), List.of(2));
+        testIntersectionsUnderLimit(table10M, List.of(0.01, 0.01, 0.02), List.of(2, 3));
+        testIntersectionsUnderLimit(table10M, List.of(0.01, 0.01, 0.01), List.of(3));
+
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 1.0, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 0.7, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 0.5, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 0.2, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 0.1, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 0.05, 1.0), List.of(1, 2));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 0.02, 1.0), List.of(2));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 0.001, 1.0), List.of(2));
+
+        testIntersectionsUnderLimit(table10M, List.of(0.0001, 1.0, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0001, 0.5, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0001, 0.2, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0001, 0.1, 1.0), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0001, 0.05), List.of(1));
+        testIntersectionsUnderLimit(table10M, List.of(0.0001, 0.02, 1.0), List.of(1, 2));
+        testIntersectionsUnderLimit(table10M, List.of(0.0001, 0.001, 1.0), List.of(2));
+
+
+        testIntersectionsUnderLimit(table10M, List.of(0.0001, 0.0001, 0.0001), List.of(2));
+        testIntersectionsUnderLimit(table10M, List.of(0.001, 0.001, 0.001), List.of(2, 3));
+        testIntersectionsUnderLimit(table10M, List.of(0.002, 0.002, 0.002), List.of(3));
+        testIntersectionsUnderLimit(table10M, List.of(0.005, 0.005, 0.005), List.of(3));
+        testIntersectionsUnderLimit(table10M, List.of(0.008, 0.008, 0.008), List.of(3));
+    }
+
+    private void testIntersectionsUnderLimit(Plan.TableMetrics metrics, List<Double> selectivities, List<Integer> expectedIndexScanCount)
+    {
+        Plan.Factory factory = new Plan.Factory(metrics, new CostEstimator(metrics));
+        List<Plan.KeysIteration> indexScans = new ArrayList<>(selectivities.size());
+        RowFilter.Builder rowFilterBuilder = RowFilter.builder(true);
+        RowFilter.Expression[] predicates = new RowFilter.Expression[] { pred1, pred2, pred3, pred4 };
+        Expression[] saiPredicates = new Expression[] { saiPred1, saiPred2, saiPred3, saiPred4 };
+        for (int i = 0; i < selectivities.size(); i++)
+        {
+            indexScans.add(factory.indexScan(saiPredicates[i], (long) (selectivities.get(i) * metrics.rows)));
+            rowFilterBuilder.add(predicates[i]);
+        }
+
+        Plan.KeysIteration intersection = factory.intersection(indexScans);
+        Plan.RowsIteration fetch = factory.fetch(intersection);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilterBuilder.build(), fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+
+        Plan optimizedPlan = origPlan.optimize();
+        List<Plan.IndexScan> resultIndexScans = optimizedPlan.nodesOfType(Plan.IndexScan.class);
+        assertTrue("original:\n" + origPlan.toStringRecursive() + "optimized:\n" + optimizedPlan.toStringRecursive(),
+                     expectedIndexScanCount.contains(resultIndexScans.size()));
+    }
+
+    @Test
+    public void optimizeIntersectionsUnderAnnSort()
+    {
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.0, 0.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.0, 0.0, 0.1), List.of(1));
+
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(1.0), List.of(0));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.1), List.of(0));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.05), List.of(0));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.02), List.of(0, 1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01), List.of(0, 1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.0001), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.00001), List.of(1));
+
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.9, 0.9), List.of(0));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.1, 0.9), List.of(0));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.9), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.1), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.002), List.of(2));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.001), List.of(2));
+
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.5), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.2), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.1), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.05), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.02), List.of(1, 2));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.001), List.of(2));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.0001, 0.0001), List.of(2));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.00001, 0.00001), List.of(2));
+
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.1, 1.0, 1.0), List.of(0));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.1, 0.1, 1.0), List.of(0, 1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.1, 0.1, 1.0), List.of(0, 1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.1, 0.1, 0.5), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.1, 0.1, 0.2), List.of(1));
+
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 1.0, 1.0), List.of(1));
+
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 0.01, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 0.01, 0.5), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 0.01, 0.2), List.of(1, 2));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 0.01, 0.1), List.of(2, 3));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 0.01, 0.05), List.of(3));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 0.01, 0.02), List.of(3));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 0.01, 0.01), List.of(3));
+
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 1.0, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.5, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.2, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.1, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.05, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.02, 1.0), List.of(1, 2));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.01, 1.0), List.of(2));
+
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 1.0, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.5, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.2, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.1, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.05, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.02, 1.0), List.of(1, 2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.01, 1.0), List.of(2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.005, 1.0), List.of(2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.002, 1.0), List.of(2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.001, 1.0), List.of(2));
+
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.0001, 0.0001, 0.0001, 0.0001), List.of(2));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.0001, 0.0001, 0.0001), List.of(2));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.001, 0.001, 0.001), List.of(2, 3));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.002, 0.002, 0.002), List.of(3));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.005, 0.005, 0.005), List.of(3));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.008, 0.008, 0.008), List.of(3));
+        testIntersectionsUnderAnnSort(table1M, Expression.Op.RANGE, List.of(0.01, 0.01, 0.01), List.of(3));
+
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.0001, 0.0001, 0.0001, 0.0001), List.of(2, 3));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.0001, 0.0001, 0.0001), List.of(2, 3));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.001, 0.001, 0.001), List.of(3));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.002, 0.002, 0.002), List.of(3));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.005, 0.005, 0.005), List.of(3));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.008, 0.008, 0.008), List.of(3));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.RANGE, List.of(0.01, 0.01, 0.01), List.of(2, 3));
+    }
+
+    @Test
+    public void optimizeLiteralIntersectionsUnderAnnSort()
+    {
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 1.0), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.5), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.2), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.1), List.of(1));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.05), List.of(1, 2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.02), List.of(2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.01), List.of(2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.005), List.of(2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.002), List.of(2));
+        testIntersectionsUnderAnnSort(table10M, Expression.Op.EQ, List.of(0.001, 0.001), List.of(2));
+    }
+
+    private void testIntersectionsUnderAnnSort(Plan.TableMetrics metrics,
+                                               Expression.Op operation,
+                                               List<Double> selectivities,
+                                               List<Integer> expectedIndexScanCount)
+    {
+        Plan.Factory factory = new Plan.Factory(metrics, new CostEstimator(metrics));
+        List<Plan.KeysIteration> indexScans = new ArrayList<>(selectivities.size());
+        RowFilter.Builder rowFilterBuilder = RowFilter.builder(true);
+        for (int i = 0; i < selectivities.size(); i++)
+        {
+            String column = "p" + i;
+            Plan.KeysIteration indexScan = factory.indexScan(saiPred(column, operation, operation == Expression.Op.EQ),
+                                                                    (long) (selectivities.get(i) * metrics.rows));
+            indexScans.add(indexScan);
+            rowFilterBuilder.add(filerPred(column, operation == Expression.Op.RANGE ? Operator.LT : Operator.EQ));
+        }
+
+        Plan.KeysIteration intersection = factory.intersection(indexScans);
+        Plan.KeysIteration sort = factory.sort(intersection, ordering);
+        Plan.RowsIteration fetch = factory.fetch(sort);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilterBuilder.build(), fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+
+        Plan optimizedPlan = origPlan.optimize();
+        List<Plan.IndexScan> resultIndexScans = optimizedPlan.nodesOfType(Plan.IndexScan.class);
+        assertTrue("original:\n" + origPlan.toStringRecursive() + "optimized:\n" + optimizedPlan.toStringRecursive(),
+                     expectedIndexScanCount.contains(resultIndexScans.size()));
+    }
+
+    @Test
+    public void testExternalCostEstimator()
+    {
+        Plan.CostEstimator est1 = Mockito.mock(Plan.CostEstimator.class);
+        Mockito.when(est1.estimateAnnSearchCost(Mockito.any(), Mockito.anyInt(), Mockito.anyLong())).thenReturn(1.0);
+        Plan.CostEstimator est2 = Mockito.mock(Plan.CostEstimator.class);
+        Mockito.when(est2.estimateAnnSearchCost(Mockito.any(), Mockito.anyInt(), Mockito.anyLong())).thenReturn(100.0);
+
+        Plan.Factory factory1 = new Plan.Factory(table1M, est1);
+        Plan scan1 = factory1.sort(factory1.everything, ordering);
+
+        Plan.Factory factory2 = new Plan.Factory(table1M, est2);
+        Plan scan2 = factory2.sort(factory2.everything, ordering);
+
+        assertTrue(scan2.fullCost() > scan1.fullCost());
+    }
+
+    @Test
+    public void testAccessConvolution()
+    {
+        Plan.Access access = Plan.Access.sequential(100);
+        Plan.Access conv = access.scaleDistance(10.0)
+                                 .convolute(10, 1.0)
+                                 .scaleDistance(5.0)
+                                 .convolute(3, 1.0);
+        assertEquals(access.totalDistance * 50.0, conv.totalDistance, 0.001);
+        assertEquals(3000.0, conv.totalCount, 0.1);
+    }
+
+    @Test
+    public void testLazyAccessPropagation()
+    {
+        Plan.KeysIteration indexScan1 = Mockito.mock(Plan.KeysIteration.class, Mockito.CALLS_REAL_METHODS);
+        Mockito.when(indexScan1.withAccess(Mockito.any())).thenReturn(indexScan1);
+        Mockito.when(indexScan1.estimateCost()).thenReturn(new Plan.KeysIterationCost(20,0.0, 0.5));
+        Mockito.when(indexScan1.estimateSelectivity()).thenReturn(0.001);
+        Mockito.when(indexScan1.title()).thenReturn("");
+
+        Plan.KeysIteration indexScan2 = factory.indexScan(saiPred2, (long) (0.01 * factory.tableMetrics.rows));
+        Plan.KeysIteration indexScan3 = factory.indexScan(saiPred3, (long) (0.5 * factory.tableMetrics.rows));
+        Plan.KeysIteration intersection = factory.intersection(Lists.newArrayList(indexScan1, indexScan2, indexScan3));
+        Plan.RowsIteration fetch = factory.fetch(intersection);
+        Plan.RowsIteration postFilter = factory.recheckFilter(rowFilter123, fetch);
+        Plan.RowsIteration origPlan = factory.limit(postFilter, 3);
+        origPlan.cost();
+
+        Mockito.verify(indexScan1, Mockito.times(1)).withAccess(Mockito.any());
+        Mockito.verify(indexScan1, Mockito.times(1)).estimateCost();
+    }
+
+    private List<Integer> ids(List<? extends Plan> subplans)
+    {
+        return subplans.stream().map(p -> p.id).collect(Collectors.toList());
+    }
+
+    static class CostEstimator implements Plan.CostEstimator
+    {
+        final Plan.TableMetrics metrics;
+
+        CostEstimator(Plan.TableMetrics metrics)
+        {
+            this.metrics = metrics;
+        }
+
+        @Override
+        public double estimateAnnSearchCost(Orderer ordering, int limit, long candidates)
+        {
+            Preconditions.checkArgument(limit > 0, "limit must be > 0");
+            var expectedNodes = VectorMemtableIndex.expectedNodesVisited(limit / metrics.sstables,
+                                                                         (int) candidates / metrics.sstables,
+                                                                         500000);
+            int degree = 32;
+            return metrics.sstables * (expectedNodes * (ANN_SIMILARITY_COST + Plan.hrs(ANN_EDGELIST_COST) / degree)
+                                       + limit * Plan.hrs(Plan.CostCoefficients.ANN_SCORED_KEY_COST));
+        }
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/plan/SingleRestrictionEstimatedRowCountTest.java b/test/unit/org/apache/cassandra/index/sai/plan/SingleRestrictionEstimatedRowCountTest.java
new file mode 100644
index 000000000000..8bd26ce99b9a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/plan/SingleRestrictionEstimatedRowCountTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.AbstractMap;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+import static org.apache.cassandra.cql3.CQL3Type.Native.DECIMAL;
+import static org.apache.cassandra.cql3.CQL3Type.Native.INT;
+import static org.apache.cassandra.cql3.CQL3Type.Native.VARINT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class SingleRestrictionEstimatedRowCountTest extends SAITester
+{
+    static protected Map<Map.Entry<Version, CQL3Type.Native>, ColumnFamilyStore> tables = new HashMap<>();
+    static Version[] versions = new Version[]{ Version.DB, Version.EB };
+    static CQL3Type.Native[] types = new CQL3Type.Native[]{ INT, DECIMAL, VARINT };
+
+    static protected Object getFilterValue(CQL3Type.Native type, int value)
+    {
+        switch (type)
+        {
+            case INT:
+                return value;
+            case DECIMAL:
+                return BigDecimal.valueOf(value);
+            case VARINT:
+                return BigInteger.valueOf(value);
+        }
+        fail("Must be known type");
+        return null;
+    }
+
+    static Map.Entry<Version, CQL3Type.Native> tablesEntryKey(Version version, CQL3Type.Native type)
+    {
+        return new AbstractMap.SimpleEntry<>(version, type);
+    }
+
+    @Test
+    public void testMemtablesSAI()
+    {
+        createTables();
+
+        RowCountTest test = new RowCountTest(Operator.NEQ, 25);
+        test.doTest(Version.DB, INT, 97.0);
+        test.doTest(Version.EB, INT, 97.0);
+        // Truncated numeric types planned differently
+        test.doTest(Version.DB, DECIMAL, 97.0);
+        test.doTest(Version.EB, DECIMAL, 97.0);
+        test.doTest(Version.EB, VARINT, 97.0);
+
+        test = new RowCountTest(Operator.LT, 50);
+        test.doTest(Version.DB, INT, 48);
+        test.doTest(Version.EB, INT, 48);
+        test.doTest(Version.DB, DECIMAL, 48);
+        test.doTest(Version.EB, DECIMAL, 48);
+
+        test = new RowCountTest(Operator.LT, 150);
+        test.doTest(Version.DB, INT, 97);
+        test.doTest(Version.EB, INT, 97);
+        test.doTest(Version.DB, DECIMAL, 97);
+        test.doTest(Version.EB, DECIMAL, 97);
+
+        test = new RowCountTest(Operator.EQ, 31);
+        test.doTest(Version.DB, INT, 15);
+        test.doTest(Version.EB, INT, 0);
+        test.doTest(Version.DB, DECIMAL, 15);
+        test.doTest(Version.EB, DECIMAL, 0);
+    }
+
+
+    void createTables()
+    {
+        for (Version version : versions)
+        {
+            SAIUtil.setLatestVersion(version);
+            for (CQL3Type.Native type : types)
+            {
+                createTable("CREATE TABLE %s (pk text PRIMARY KEY, age " + type + ')');
+                createIndex("CREATE CUSTOM INDEX ON %s(age) USING 'StorageAttachedIndex'");
+                tables.put(tablesEntryKey(version, type), getCurrentColumnFamilyStore());
+            }
+        }
+        flush();
+        for (ColumnFamilyStore cfs : tables.values())
+            populateTable(cfs);
+    }
+
+    void populateTable(ColumnFamilyStore cfs)
+    {
+        // Avoid race condition of starting before flushing completed
+        cfs.unsafeRunWithoutFlushing(() -> {
+            for (int i = 0; i < 100; i++)
+            {
+                String query = String.format("INSERT INTO %s (pk, age) VALUES (?, " + i + ')',
+                                             cfs.keyspace.getName() + '.' + cfs.name);
+                executeFormattedQuery(query, "key" + i);
+            }
+        });
+    }
+
+    static class RowCountTest
+    {
+        final Operator op;
+        final int filterValue;
+
+        RowCountTest(Operator op, int filterValue)
+        {
+            this.op = op;
+            this.filterValue = filterValue;
+        }
+
+        void doTest(Version version, CQL3Type.Native type, double expectedRows)
+        {
+            ColumnFamilyStore cfs = tables.get(new AbstractMap.SimpleEntry<>(version, type));
+            Object filter = getFilterValue(type, filterValue);
+            ReadCommand rc = Util.cmd(cfs)
+                                 .columns("age")
+                                 .filterOn("age", op, filter)
+                                 .build();
+            QueryController controller = new QueryController(cfs,
+                                                             rc,
+                                                             version.onDiskFormat().indexFeatureSet(),
+                                                             new QueryContext());
+
+            long totalRows = controller.planFactory.tableMetrics.rows;
+            assertEquals(0, cfs.metrics().liveSSTableCount.getValue().intValue());
+
+            Plan plan = controller.buildPlan();
+            assert plan instanceof Plan.RowsIteration;
+            Plan.RowsIteration root = (Plan.RowsIteration) plan;
+            Plan.KeysIteration planNode = root.firstNodeOfType(Plan.KeysIteration.class);
+            assertNotNull(planNode);
+
+            assertEquals(expectedRows, root.expectedRows(), 0.1);
+            assertEquals(expectedRows, planNode.expectedKeys(), 0.1);
+            assertEquals(expectedRows / totalRows, planNode.selectivity(), 0.001);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/postings/IntArrayPostingListTest.java b/test/unit/org/apache/cassandra/index/sai/postings/IntArrayPostingListTest.java
new file mode 100644
index 000000000000..9e6bfe391051
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/postings/IntArrayPostingListTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.postings;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.utils.SaiRandomizedTest;
+
+public class IntArrayPostingListTest extends SaiRandomizedTest
+{
+    @Test
+    public void testArrayPostingList() throws Exception
+    {
+        IntArrayPostingList postingList = new IntArrayPostingList(new int[]{ 1, 2, 3 });
+        assertEquals(3, postingList.size());
+        assertEquals(1, postingList.nextPosting());
+        assertEquals(2, postingList.nextPosting());
+        assertEquals(3, postingList.nextPosting());
+        assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
+
+        postingList = new IntArrayPostingList(new int[]{ 10, 20, 30, 40, 50, 60 });
+        assertEquals(50, postingList.advance(45));
+        assertEquals(60, postingList.advance(60));
+        assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/postings/PeekablePostingListTest.java b/test/unit/org/apache/cassandra/index/sai/postings/PeekablePostingListTest.java
deleted file mode 100644
index 79a638ee1267..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/postings/PeekablePostingListTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.postings;
-
-import java.io.IOException;
-
-import org.junit.Test;
-
-import org.apache.cassandra.index.sai.disk.ArrayPostingList;
-import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
-
-import static org.junit.Assert.assertEquals;
-
-public class PeekablePostingListTest extends SAIRandomizedTester
-{
-    @Test
-    public void testNextPosting() throws IOException
-    {
-        PeekablePostingList postingList = PeekablePostingList.makePeekable(new ArrayPostingList(1, 2, 3));
-        assertEquals(3, postingList.size());
-        assertEquals(1, postingList.peek());
-        assertEquals(1, postingList.nextPosting());
-        assertEquals(2, postingList.peek());
-        assertEquals(2, postingList.nextPosting());
-        assertEquals(3, postingList.peek());
-        assertEquals(3, postingList.nextPosting());
-        assertEquals(PostingList.END_OF_STREAM, postingList.peek());
-        assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
-    }
-
-    @Test
-    public void testAdvance() throws IOException
-    {
-        PeekablePostingList postingList = PeekablePostingList.makePeekable(new ArrayPostingList(10, 20, 30, 40, 50, 60));
-        assertEquals(10, postingList.peek());
-        assertEquals(50, postingList.advance(45));
-        assertEquals(60, postingList.peek());
-        assertEquals(60, postingList.advance(60));
-        assertEquals(PostingList.END_OF_STREAM, postingList.advance(60));
-        assertEquals(PostingList.END_OF_STREAM, postingList.peek());
-        assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
-    }
-
-    @Test
-    public void testAdvanceWithoutConsuming() throws IOException
-    {
-        PeekablePostingList postingList = PeekablePostingList.makePeekable(new ArrayPostingList(10, 20, 30, 40, 50, 60));
-        assertEquals(10, postingList.peek());
-        postingList.advanceWithoutConsuming(45);
-        assertEquals(50, postingList.peek());
-        assertEquals(50, postingList.advance(45));
-        postingList.advanceWithoutConsuming(60);
-        assertEquals(60, postingList.advance(60));
-        assertEquals(PostingList.END_OF_STREAM, postingList.peek());
-        assertEquals(PostingList.END_OF_STREAM, postingList.nextPosting());
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTest.java b/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTest.java
new file mode 100644
index 000000000000..1353fc6fbb76
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+public class AbstractPrimaryKeyTest extends SaiRandomizedTest
+{
+    static TableMetadata simplePartition = TableMetadata.builder("test", "test")
+                                                        .partitioner(Murmur3Partitioner.instance)
+                                                        .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                        .build();
+
+    static TableMetadata compositePartition = TableMetadata.builder("test", "test")
+                                                           .partitioner(Murmur3Partitioner.instance)
+                                                           .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                           .addPartitionKeyColumn("pk2", Int32Type.instance)
+                                                           .build();
+
+    static TableMetadata simplePartitionSingleClusteringAsc = TableMetadata.builder("test", "test")
+                                                                           .partitioner(Murmur3Partitioner.instance)
+                                                                           .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                           .addClusteringColumn("ck1", UTF8Type.instance)
+                                                                           .build();
+
+    static TableMetadata simplePartitionMultipleClusteringAsc = TableMetadata.builder("test", "test")
+                                                                             .partitioner(Murmur3Partitioner.instance)
+                                                                             .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                             .addClusteringColumn("ck1", UTF8Type.instance)
+                                                                             .addClusteringColumn("ck2", UTF8Type.instance)
+                                                                             .build();
+
+    static TableMetadata simplePartitionSingleClusteringDesc = TableMetadata.builder("test", "test")
+                                                                            .partitioner(Murmur3Partitioner.instance)
+                                                                            .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                            .addClusteringColumn("ck1", ReversedType.getInstance(UTF8Type.instance))
+                                                                            .build();
+
+    static TableMetadata simplePartitionMultipleClusteringDesc = TableMetadata.builder("test", "test")
+                                                                              .partitioner(Murmur3Partitioner.instance)
+                                                                              .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                              .addClusteringColumn("ck1", ReversedType.getInstance(UTF8Type.instance))
+                                                                              .addClusteringColumn("ck2", ReversedType.getInstance(UTF8Type.instance))
+                                                                              .build();
+
+    static TableMetadata compositePartitionSingleClusteringAsc = TableMetadata.builder("test", "test")
+                                                                              .partitioner(Murmur3Partitioner.instance)
+                                                                              .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                              .addPartitionKeyColumn("pk2", Int32Type.instance)
+                                                                              .addClusteringColumn("ck1", UTF8Type.instance)
+                                                                              .build();
+
+    static TableMetadata compositePartitionMultipleClusteringAsc = TableMetadata.builder("test", "test")
+                                                                                .partitioner(Murmur3Partitioner.instance)
+                                                                                .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                                .addPartitionKeyColumn("pk2", Int32Type.instance)
+                                                                                .addClusteringColumn("ck1", UTF8Type.instance)
+                                                                                .addClusteringColumn("ck2", UTF8Type.instance)
+                                                                                .build();
+
+    static TableMetadata compositePartitionSingleClusteringDesc = TableMetadata.builder("test", "test")
+                                                                               .partitioner(Murmur3Partitioner.instance)
+                                                                               .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                               .addPartitionKeyColumn("pk2", Int32Type.instance)
+                                                                               .addClusteringColumn("ck1", ReversedType.getInstance(UTF8Type.instance))
+                                                                               .build();
+
+    static TableMetadata compositePartitionMultipleClusteringDesc = TableMetadata.builder("test", "test")
+                                                                                 .partitioner(Murmur3Partitioner.instance)
+                                                                                 .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                                 .addPartitionKeyColumn("pk2", Int32Type.instance)
+                                                                                 .addClusteringColumn("ck1", ReversedType.getInstance(UTF8Type.instance))
+                                                                                 .addClusteringColumn("ck2", ReversedType.getInstance(UTF8Type.instance))
+                                                                                 .build();
+
+    static TableMetadata simplePartitionMultipleClusteringMixed = TableMetadata.builder("test", "test")
+                                                                               .partitioner(Murmur3Partitioner.instance)
+                                                                               .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                               .addClusteringColumn("ck1", UTF8Type.instance)
+                                                                               .addClusteringColumn("ck2", ReversedType.getInstance(UTF8Type.instance))
+                                                                               .build();
+
+    static TableMetadata compositePartitionMultipleClusteringMixed = TableMetadata.builder("test", "test")
+                                                                                  .partitioner(Murmur3Partitioner.instance)
+                                                                                  .addPartitionKeyColumn("pk1", Int32Type.instance)
+                                                                                  .addPartitionKeyColumn("pk2", Int32Type.instance)
+                                                                                  .addClusteringColumn("ck1", UTF8Type.instance)
+                                                                                  .addClusteringColumn("ck2", ReversedType.getInstance(UTF8Type.instance))
+                                                                                  .build();
+
+    void assertByteComparison(PrimaryKey a, PrimaryKey b, int expected)
+    {
+        assertEquals(expected, ByteComparable.compare(v -> a.asComparableBytes(v),
+                                                      v -> b.asComparableBytes(v),
+                                                      TypeUtil.BYTE_COMPARABLE_VERSION));
+    }
+
+    void assertCompareToAndEquals(PrimaryKey a, PrimaryKey b, int expected)
+    {
+        if (expected > 0)
+        {
+            assertTrue(a.compareTo(b) > 0);
+            assertNotEquals(a, b);
+        }
+        else if (expected < 0)
+        {
+            assertTrue(a.compareTo(b) < 0);
+            assertNotEquals(a, b);
+        }
+        else
+        {
+            assertTrue(a.compareTo(b) == 0);
+            assertEquals(a, b);
+        }
+    }
+
+    DecoratedKey makeKey(TableMetadata table, Object...partitionKeys)
+    {
+        ByteBuffer key;
+        if (TypeUtil.isComposite(table.partitionKeyType))
+            key = ((CompositeType)table.partitionKeyType).decompose(partitionKeys);
+        else
+            key = table.partitionKeyType.fromString((String)partitionKeys[0]);
+        return table.partitioner.decorateKey(key);
+    }
+
+    Clustering makeClustering(TableMetadata table, String...clusteringKeys)
+    {
+        Clustering clustering;
+        if (table.comparator.size() == 0)
+            clustering = Clustering.EMPTY;
+        else
+        {
+            ByteBuffer[] values = new ByteBuffer[clusteringKeys.length];
+            for (int index = 0; index < table.comparator.size(); index++)
+                values[index] = table.comparator.subtype(index).fromString(clusteringKeys[index]);
+            clustering = Clustering.make(values);
+        }
+        return clustering;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java b/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java
deleted file mode 100644
index 8ccd63436dc2..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.utils;
-
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.marshal.CompositeType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.ReversedType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.schema.TableMetadata;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
-
-public class AbstractPrimaryKeyTester extends SAIRandomizedTester
-{
-    protected static final TableMetadata simplePartition = TableMetadata.builder("test", "test")
-                                                              .partitioner(Murmur3Partitioner.instance)
-                                                              .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                              .build();
-
-    protected static final TableMetadata compositePartition = TableMetadata.builder("test", "test")
-                                                                 .partitioner(Murmur3Partitioner.instance)
-                                                                 .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                 .addPartitionKeyColumn("pk2", Int32Type.instance)
-                                                                 .build();
-
-    protected static final TableMetadata simplePartitionSingleClusteringAsc = TableMetadata.builder("test", "test")
-                                                                                 .partitioner(Murmur3Partitioner.instance)
-                                                                                 .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                 .addClusteringColumn("ck1", UTF8Type.instance)
-                                                                                 .build();
-
-    protected static final TableMetadata simplePartitionStaticAndSingleClusteringAsc = TableMetadata.builder("test", "test")
-                                                                                                    .partitioner(Murmur3Partitioner.instance)
-                                                                                                    .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                                    .addStaticColumn("sk1", Int32Type.instance)
-                                                                                                    .addClusteringColumn("ck1", UTF8Type.instance)
-                                                                                                    .build();
-
-    protected static final TableMetadata simplePartitionMultipleClusteringAsc = TableMetadata.builder("test", "test")
-                                                                                   .partitioner(Murmur3Partitioner.instance)
-                                                                                   .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                   .addClusteringColumn("ck1", UTF8Type.instance)
-                                                                                   .addClusteringColumn("ck2", UTF8Type.instance)
-                                                                                   .build();
-
-    protected static final TableMetadata simplePartitionSingleClusteringDesc = TableMetadata.builder("test", "test")
-                                                                                  .partitioner(Murmur3Partitioner.instance)
-                                                                                  .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                  .addClusteringColumn("ck1", ReversedType.getInstance(UTF8Type.instance))
-                                                                                  .build();
-
-    protected static final TableMetadata simplePartitionMultipleClusteringDesc = TableMetadata.builder("test", "test")
-                                                                                    .partitioner(Murmur3Partitioner.instance)
-                                                                                    .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                    .addClusteringColumn("ck1", ReversedType.getInstance(UTF8Type.instance))
-                                                                                    .addClusteringColumn("ck2", ReversedType.getInstance(UTF8Type.instance))
-                                                                                    .build();
-
-    protected static final TableMetadata compositePartitionSingleClusteringAsc = TableMetadata.builder("test", "test")
-                                                                                    .partitioner(Murmur3Partitioner.instance)
-                                                                                    .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                    .addPartitionKeyColumn("pk2", Int32Type.instance)
-                                                                                    .addClusteringColumn("ck1", UTF8Type.instance)
-                                                                                    .build();
-
-    protected static final TableMetadata compositePartitionMultipleClusteringAsc = TableMetadata.builder("test", "test")
-                                                                                      .partitioner(Murmur3Partitioner.instance)
-                                                                                      .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                      .addPartitionKeyColumn("pk2", Int32Type.instance)
-                                                                                      .addClusteringColumn("ck1", UTF8Type.instance)
-                                                                                      .addClusteringColumn("ck2", UTF8Type.instance)
-                                                                                      .build();
-
-    protected static final TableMetadata compositePartitionSingleClusteringDesc = TableMetadata.builder("test", "test")
-                                                                                     .partitioner(Murmur3Partitioner.instance)
-                                                                                     .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                     .addPartitionKeyColumn("pk2", Int32Type.instance)
-                                                                                     .addClusteringColumn("ck1", ReversedType.getInstance(UTF8Type.instance))
-                                                                                     .build();
-
-    protected static final TableMetadata compositePartitionMultipleClusteringDesc = TableMetadata.builder("test", "test")
-                                                                                       .partitioner(Murmur3Partitioner.instance)
-                                                                                       .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                       .addPartitionKeyColumn("pk2", Int32Type.instance)
-                                                                                       .addClusteringColumn("ck1", ReversedType.getInstance(UTF8Type.instance))
-                                                                                       .addClusteringColumn("ck2", ReversedType.getInstance(UTF8Type.instance))
-                                                                                       .build();
-
-    protected static final TableMetadata simplePartitionMultipleClusteringMixed = TableMetadata.builder("test", "test")
-                                                                                     .partitioner(Murmur3Partitioner.instance)
-                                                                                     .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                     .addClusteringColumn("ck1", UTF8Type.instance)
-                                                                                     .addClusteringColumn("ck2", ReversedType.getInstance(UTF8Type.instance))
-                                                                                     .build();
-
-    protected static final TableMetadata compositePartitionMultipleClusteringMixed = TableMetadata.builder("test", "test")
-                                                                                        .partitioner(Murmur3Partitioner.instance)
-                                                                                        .addPartitionKeyColumn("pk1", Int32Type.instance)
-                                                                                        .addPartitionKeyColumn("pk2", Int32Type.instance)
-                                                                                        .addClusteringColumn("ck1", UTF8Type.instance)
-                                                                                        .addClusteringColumn("ck2", ReversedType.getInstance(UTF8Type.instance))
-                                                                                        .build();
-
-    protected void assertCompareToAndEquals(PrimaryKey a, PrimaryKey b, int expected)
-    {
-        if (expected > 0)
-        {
-            assertTrue(a.compareTo(b) > 0);
-            assertNotEquals(a, b);
-        }
-        else if (expected < 0)
-        {
-            assertTrue(a.compareTo(b) < 0);
-            assertNotEquals(a, b);
-        }
-        else
-        {
-            assertEquals(0, a.compareTo(b));
-            assertEquals(a, b);
-        }
-    }
-
-    protected DecoratedKey makeKey(TableMetadata table, Object...partitionKeys)
-    {
-        ByteBuffer key;
-        if (table.partitionKeyType instanceof CompositeType)
-            key = ((CompositeType)table.partitionKeyType).decompose(partitionKeys);
-        else
-            key = table.partitionKeyType.decomposeUntyped(partitionKeys[0]);
-        return table.partitioner.decorateKey(key);
-    }
-
-    protected Clustering<?> makeClustering(TableMetadata table, Object...clusteringKeys)
-    {
-        Clustering<?> clustering;
-        if (table.comparator.size() == 0)
-            clustering = Clustering.EMPTY;
-        else
-        {
-            ByteBuffer[] values = new ByteBuffer[clusteringKeys.length];
-            for (int index = 0; index < table.comparator.size(); index++)
-                values[index] = table.comparator.subtype(index).decomposeUntyped(clusteringKeys[index]);
-            clustering = Clustering.make(values);
-        }
-        return clustering;
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/CellWithSourceTableTest.java b/test/unit/org/apache/cassandra/index/sai/utils/CellWithSourceTableTest.java
new file mode 100644
index 000000000000..0462f9eb567f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/CellWithSourceTableTest.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import org.apache.cassandra.db.DeletionPurger;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.rows.ArrayCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.memory.HeapCloner;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CellWithSourceTableTest {
+
+    private ColumnMetadata column;
+    private Cell<?> wrappedCell;
+    private Object sourceTable;
+    private CellWithSourceTable<?> cellWithSourceTable;
+
+    private final long timestamp = System.currentTimeMillis();
+    // We use a 4 byte array because the Int32Type is used in the test
+    private final byte[] value = new byte[]{0,0,0,1};
+
+    @Before
+    public void setUp()
+    {
+        column = ColumnMetadata.regularColumn("keyspace1", "table1", "name1", Int32Type.instance);
+        wrappedCell = new ArrayCell(column, timestamp, Cell.NO_TTL, Cell.NO_DELETION_TIME, value, null);
+        sourceTable = new Object();
+        cellWithSourceTable = new CellWithSourceTable<>(wrappedCell, sourceTable);
+    }
+
+    @Test
+    public void testSourceTable()
+    {
+        assertEquals(sourceTable, cellWithSourceTable.sourceTable());
+    }
+
+    @Test
+    public void testIsCounterCell()
+    {
+        assertEquals(wrappedCell.isCounterCell(), cellWithSourceTable.isCounterCell());
+    }
+
+    @Test
+    public void testValue()
+    {
+        assertEquals(wrappedCell.value(), cellWithSourceTable.value());
+    }
+
+    @Test
+    public void testAccessor()
+    {
+        assertEquals(wrappedCell.accessor(), cellWithSourceTable.accessor());
+    }
+
+    @Test
+    public void testTimestamp()
+    {
+        assertEquals(wrappedCell.timestamp(), cellWithSourceTable.timestamp());
+    }
+
+    @Test
+    public void testTtl()
+    {
+        assertEquals(wrappedCell.ttl(), cellWithSourceTable.ttl());
+    }
+
+    @Test
+    public void testLocalDeletionTime()
+    {
+        assertEquals(wrappedCell.localDeletionTime(), cellWithSourceTable.localDeletionTime());
+    }
+
+    @Test
+    public void testIsTombstone()
+    {
+        assertEquals(wrappedCell.isTombstone(), cellWithSourceTable.isTombstone());
+    }
+
+    @Test
+    public void testIsExpiring()
+    {
+        assertEquals(wrappedCell.isExpiring(), cellWithSourceTable.isExpiring());
+    }
+
+    @Test
+    public void testIsLive()
+    {
+        var nowInSec = 0;
+        assertEquals(wrappedCell.isLive(nowInSec), cellWithSourceTable.isLive(nowInSec));
+    }
+
+    @Test
+    public void testPath()
+    {
+        assertEquals(wrappedCell.path(), cellWithSourceTable.path());
+    }
+
+    @Test
+    public void testWithUpdatedColumn()
+    {
+        var originalColumn = cellWithSourceTable.column();
+        var newColumn = ColumnMetadata.regularColumn("keyspace1", "table1", "name2", Int32Type.instance);
+        var resultColumn = cellWithSourceTable.withUpdatedColumn(newColumn).column();
+        assertNotEquals(originalColumn, resultColumn);
+        assertEquals(newColumn, resultColumn);
+    }
+
+    @Test
+    public void testWithUpdatedValue()
+    {
+        ByteBuffer newValue = ByteBuffer.allocate(4);
+        var oldValue = cellWithSourceTable.value();
+        var resultValue = cellWithSourceTable.withUpdatedValue(newValue).value();
+        assertNotEquals(oldValue, resultValue);
+        assertTrue(resultValue instanceof byte[]);
+        assertArrayEquals(newValue.array(), (byte[]) resultValue);
+    }
+
+    @Test
+    public void testWithUpdatedTimestampAndLocalDeletionTime()
+    {
+        long newTimestamp = 1234567890L;
+        int newLocalDeletionTime = 987654321;
+        var originalTimestamp = cellWithSourceTable.timestamp();
+        var originalDeletionTime = cellWithSourceTable.localDeletionTime();
+        var resultTimestamp = cellWithSourceTable.withUpdatedTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime).timestamp();
+        var resultDeletionTime = cellWithSourceTable.withUpdatedTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime).localDeletionTime();
+        assertNotEquals(originalTimestamp, resultTimestamp);
+        assertEquals(newTimestamp, resultTimestamp);
+        assertNotEquals(originalDeletionTime, resultDeletionTime);
+        assertEquals(newLocalDeletionTime, resultDeletionTime);
+    }
+
+    @Test
+    public void testWithSkippedValue()
+    {
+        var originalValue = cellWithSourceTable.value();
+        var resultValue = cellWithSourceTable.withSkippedValue().value();
+        assertNotEquals(originalValue, resultValue);
+    }
+
+    @Test
+    public void testClone()
+    {
+        var resultClone = cellWithSourceTable.clone(HeapCloner.instance);
+        // The reference is not equal here because we have a non-zero length value
+        assertNotSame(cellWithSourceTable, resultClone);
+        // Now make the value 0 length and we should get the same reference
+        var skippedCell = cellWithSourceTable.withSkippedValue();
+        var clonedSkippedCell = skippedCell.clone(HeapCloner.instance);
+        assertSame(skippedCell, clonedSkippedCell);
+    }
+
+    @Test
+    public void testDataSize()
+    {
+        assertEquals(wrappedCell.dataSize(), cellWithSourceTable.dataSize());
+    }
+
+    @Test
+    public void testUnsharedHeapSizeExcludingData()
+    {
+        assertEquals(wrappedCell.unsharedHeapSizeExcludingData(), cellWithSourceTable.unsharedHeapSizeExcludingData());
+    }
+
+    @Test
+    public void testValidate()
+    {
+        wrappedCell.validate();
+        cellWithSourceTable.validate();
+    }
+
+    @Test
+    public void testHasInvalidDeletions()
+    {
+        assertEquals(wrappedCell.hasInvalidDeletions(), cellWithSourceTable.hasInvalidDeletions());
+    }
+
+    @Test
+    public void testDigest()
+    {
+        var digest1 = Digest.forValidator();
+        cellWithSourceTable.digest(digest1);
+        var digest2 = Digest.forValidator();
+        wrappedCell.digest(digest2);
+        assertArrayEquals(digest1.digest(), digest2.digest());
+    }
+
+    @Test
+    public void testUpdateAllTimestamp()
+    {
+        long newTimestamp = 1234567890L;
+        var resultData = cellWithSourceTable.updateAllTimestamp(newTimestamp);
+        assertEquals(newTimestamp, resultData.minTimestamp());
+        assertEquals(newTimestamp, resultData.minTimestamp());
+    }
+
+    @Test
+    public void testMarkCounterLocalToBeCleared()
+    {
+        var resultCell = cellWithSourceTable.markCounterLocalToBeCleared();
+        assertSame(cellWithSourceTable, resultCell);
+    }
+
+    @Test
+    public void testPurge()
+    {
+        DeletionPurger purger = mock(DeletionPurger.class);
+        var mockCell = mock(Cell.class);
+        long purgeNull = 1234567890;
+        long purgeSame = 98765;
+        when(mockCell.purge(any(), eq(purgeNull))).thenReturn(null);
+        when(mockCell.purge(any(), eq(purgeSame))).thenReturn(mockCell);
+        var cell = new CellWithSourceTable<>(mockCell, sourceTable);
+        assertNull(cell.purge(purger, purgeNull));
+        assertSame(cell, cell.purge(purger, purgeSame));
+    }
+
+    @Test
+    public void testMaxTimestamp()
+    {
+        assertEquals(timestamp, cellWithSourceTable.maxTimestamp());
+    }
+
+    @Test
+    public void testMinTimestamp()
+    {
+        assertEquals(timestamp, cellWithSourceTable.minTimestamp());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/Glove.java b/test/unit/org/apache/cassandra/index/sai/utils/Glove.java
index ea08c85a4842..e040181e7f9b 100644
--- a/test/unit/org/apache/cassandra/index/sai/utils/Glove.java
+++ b/test/unit/org/apache/cassandra/index/sai/utils/Glove.java
@@ -28,14 +28,6 @@
 import java.util.List;
 import java.util.Map;
 
-/**
- * A simple wrapper of a <a href="https://nlp.stanford.edu/projects/glove/">glove</a> model that loads a set of
- * word -> embedding mappings into a {@link WordVector} object. The embeddings are represented a fixed dimension
- * float[].
- * <p>
- * The glove model provides a consistently valid embedding for each word that can be used for testing graph
- * implementations that use embeddings.
- */
 public class Glove
 {
     public static WordVector parse(InputStream inputStream) throws IOException
@@ -97,11 +89,6 @@ public String word(int index)
             return words.get(index);
         }
 
-        public float[] vector(int index)
-        {
-            return wordVectorMap.get(words.get(index)).vector;
-        }
-
         public float[] vector(String word)
         {
             return wordVectorMap.get(word).vector;
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/IndexInputLeakDetector.java b/test/unit/org/apache/cassandra/index/sai/utils/IndexInputLeakDetector.java
index 35e556f763db..d17198cc8f1a 100644
--- a/test/unit/org/apache/cassandra/index/sai/utils/IndexInputLeakDetector.java
+++ b/test/unit/org/apache/cassandra/index/sai/utils/IndexInputLeakDetector.java
@@ -25,11 +25,11 @@
 
 import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.io.IndexInput;
 import org.apache.cassandra.index.sai.disk.io.TrackingIndexFileUtils;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.util.SequentialWriterOption;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.lucene.store.IndexInput;
 
 import static org.junit.Assert.assertTrue;
 
@@ -37,11 +37,12 @@ public class IndexInputLeakDetector extends TestRuleAdapter
 {
     private final static Set<TrackingIndexFileUtils> trackedIndexFileUtils = Collections.synchronizedSet(new HashSet<>());
 
-    public IndexDescriptor newIndexDescriptor(Descriptor descriptor, TableMetadata tableMetadata, SequentialWriterOption sequentialWriterOption)
+    public IndexDescriptor newIndexDescriptor(Descriptor descriptor, SequentialWriterOption sequentialWriterOption)
     {
         TrackingIndexFileUtils trackingIndexFileUtils = new TrackingIndexFileUtils(sequentialWriterOption);
         trackedIndexFileUtils.add(trackingIndexFileUtils);
-        return IndexDescriptor.create(descriptor, tableMetadata.partitioner, tableMetadata.comparator);
+        IndexFileUtils.setOverrideInstance(trackingIndexFileUtils);
+        return IndexDescriptor.empty(descriptor);
     }
 
     @Override
@@ -58,5 +59,6 @@ protected void afterIfSuccessful()
     protected void afterAlways(List<Throwable> errors)
     {
         trackedIndexFileUtils.clear();
+        IndexFileUtils.setOverrideInstance(null);
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/IndexTermTypeTest.java b/test/unit/org/apache/cassandra/index/sai/utils/IndexTermTypeTest.java
deleted file mode 100644
index 315adfdac5e8..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/utils/IndexTermTypeTest.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.utils;
-
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.function.BiConsumer;
-import java.util.function.BiFunction;
-
-import org.junit.Test;
-
-import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.FieldIdentifier;
-import org.apache.cassandra.cql3.statements.schema.IndexTarget;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.CompositeType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.ListType;
-import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.marshal.ReversedType;
-import org.apache.cassandra.db.marshal.SetType;
-import org.apache.cassandra.db.marshal.TupleType;
-import org.apache.cassandra.db.marshal.TypeParser;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.marshal.UserType;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable;
-
-import static org.apache.cassandra.index.sai.SAITester.getRandom;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class IndexTermTypeTest
-{
-    @Test
-    public void testSimpleType()
-    {
-        for (CQL3Type cql3Type : StorageAttachedIndex.SUPPORTED_TYPES)
-        {
-            AbstractType<?> type = cql3Type.getType();
-            AbstractType<?> reversedType = ReversedType.getInstance(type);
-            IndexTermType indexTermType = SAITester.createIndexTermType(type);
-            IndexTermType reversedIndexTermType = SAITester.createIndexTermType(reversedType);
-            boolean isUTF8OrAscii = cql3Type == CQL3Type.Native.ASCII || cql3Type == CQL3Type.Native.TEXT ||
-                                    cql3Type == CQL3Type.Native.VARCHAR;
-            boolean isLiteral = cql3Type == CQL3Type.Native.ASCII || cql3Type == CQL3Type.Native.TEXT ||
-                                cql3Type == CQL3Type.Native.VARCHAR || cql3Type == CQL3Type.Native.BOOLEAN;
-            assertEquals(isLiteral, indexTermType.isLiteral());
-            assertEquals(indexTermType.isLiteral(), reversedIndexTermType.isLiteral());
-            assertEquals(isUTF8OrAscii, indexTermType.isString());
-            assertEquals(indexTermType.isString(), reversedIndexTermType.isString());
-        }
-    }
-
-    @Test
-    public void testMapType()
-    {
-        for (CQL3Type keyCql3Type : StorageAttachedIndex.SUPPORTED_TYPES)
-        {
-            AbstractType<?> keyType = keyCql3Type.getType();
-
-            testCollectionType((valueType, multiCell) -> MapType.getInstance(keyType, valueType, multiCell),
-                               (valueType, nonFrozenMap) -> {
-                assertEquals(keyType, indexTermType(nonFrozenMap, IndexTarget.Type.KEYS).indexType());
-                assertEquals(valueType, indexTermType(nonFrozenMap, IndexTarget.Type.VALUES).indexType());
-                IndexTermType entryIndexTermType = indexTermType(nonFrozenMap, IndexTarget.Type.KEYS_AND_VALUES);
-                assertEquals(CompositeType.getInstance(keyType, valueType), entryIndexTermType.indexType());
-                assertTrue(entryIndexTermType.isComposite());
-                assertTrue(entryIndexTermType.isLiteral());
-            });
-        }
-    }
-
-    @Test
-    public void testSetType()
-    {
-        testCollectionType(SetType::getInstance, (a, b) -> {});
-    }
-
-    @Test
-    public void testListType()
-    {
-        testCollectionType(ListType::getInstance, (a, b) -> {});
-    }
-
-    @Test
-    public void testTuple()
-    {
-        for (CQL3Type elementType : StorageAttachedIndex.SUPPORTED_TYPES)
-        {
-            TupleType type = TupleType.getInstance(new TypeParser(String.format("(%s, %s)", elementType.getType(), elementType.getType())));
-            IndexTermType indexTermType = indexTermType(type, IndexTarget.Type.SIMPLE);
-            assertFalse(indexTermType.isFrozenCollection());
-            assertTrue(indexTermType.isFrozen());
-            assertTrue(indexTermType.isLiteral());
-            assertFalse(indexTermType.isReversed());
-
-            IndexTermType reversedIndexTermType = indexTermType(ReversedType.getInstance(type), IndexTarget.Type.SIMPLE);
-            assertFalse(reversedIndexTermType.isFrozenCollection());
-            assertTrue(reversedIndexTermType.isFrozen());
-            assertTrue(reversedIndexTermType.isLiteral());
-            assertTrue(reversedIndexTermType.isReversed());
-        }
-    }
-
-    @Test
-    public void testUDT()
-    {
-        for (CQL3Type elementType : StorageAttachedIndex.SUPPORTED_TYPES)
-        {
-            UserType type = new UserType("ks", ByteBufferUtil.bytes("myType"),
-                                         Arrays.asList(FieldIdentifier.forQuoted("f1"), FieldIdentifier.forQuoted("f2")),
-                                         Arrays.asList(elementType.getType(), elementType.getType()),
-                                         true);
-            IndexTermType indexTermType = indexTermType(type, IndexTarget.Type.SIMPLE);
-            assertFalse(indexTermType.isFrozenCollection());
-            assertFalse(indexTermType.isFrozen());
-            assertFalse(indexTermType.isLiteral());
-            assertFalse(indexTermType.isReversed());
-
-            IndexTermType reversedIndexTermType = indexTermType(ReversedType.getInstance(type), IndexTarget.Type.SIMPLE);
-            assertFalse(reversedIndexTermType.isFrozenCollection());
-            assertFalse(reversedIndexTermType.isFrozen());
-            assertFalse(reversedIndexTermType.isLiteral());
-            assertTrue(reversedIndexTermType.isReversed());
-
-            type = new UserType("ks", ByteBufferUtil.bytes("myType"),
-                                Arrays.asList(FieldIdentifier.forQuoted("f1"), FieldIdentifier.forQuoted("f2")),
-                                Arrays.asList(elementType.getType(), elementType.getType()),
-                                false);
-            indexTermType = indexTermType(type, IndexTarget.Type.SIMPLE);
-            assertFalse(indexTermType.isFrozenCollection());
-            assertTrue(indexTermType.isFrozen());
-            assertTrue(indexTermType.isLiteral());
-
-            reversedIndexTermType = indexTermType(ReversedType.getInstance(type), IndexTarget.Type.SIMPLE);
-            assertFalse(reversedIndexTermType.isFrozenCollection());
-            assertTrue(reversedIndexTermType.isFrozen());
-            assertTrue(reversedIndexTermType.isLiteral());
-            assertTrue(reversedIndexTermType.isReversed());
-        }
-    }
-
-    private static void testCollectionType(BiFunction<AbstractType<?>, Boolean, AbstractType<?>> init,
-                                           BiConsumer<AbstractType<?>, AbstractType<?>> nonFrozenCollectionTester)
-    {
-        for (CQL3Type elementType : StorageAttachedIndex.SUPPORTED_TYPES)
-        {
-            AbstractType<?> frozenCollection = init.apply(elementType.getType(), false);
-            AbstractType<?> reversedFrozenCollection = ReversedType.getInstance(frozenCollection);
-
-            IndexTermType indexTermType = indexTermType(frozenCollection, IndexTarget.Type.FULL);
-            assertTrue(indexTermType.isFrozenCollection());
-            assertTrue(indexTermType.isLiteral());
-            assertFalse(indexTermType.isReversed());
-
-            IndexTermType reversedIndexTermType = indexTermType(reversedFrozenCollection, IndexTarget.Type.FULL);
-            assertTrue(reversedIndexTermType.isFrozenCollection());
-            assertTrue(reversedIndexTermType.isLiteral());
-            assertTrue(reversedIndexTermType.isReversed());
-
-            AbstractType<?> nonFrozenCollection = init.apply(elementType.getType(), true);
-            assertEquals(elementType.getType(), indexTermType(nonFrozenCollection, IndexTarget.Type.VALUES).indexType());
-            nonFrozenCollectionTester.accept(elementType.getType(), nonFrozenCollection);
-        }
-    }
-
-    private static IndexTermType indexTermType(AbstractType<?> type, IndexTarget.Type indexType)
-    {
-        return IndexTermType.create(column(type), Collections.emptyList(), indexType);
-    }
-
-    private static ColumnMetadata column(AbstractType<?> type)
-    {
-        return ColumnMetadata.regularColumn("ks", "cf", "col", type);
-    }
-
-    @Test
-    public void shouldCompareByteBuffers()
-    {
-        IndexTermType indexTermType = indexTermType(Int32Type.instance, IndexTarget.Type.SIMPLE);
-
-        final ByteBuffer a = Int32Type.instance.decompose(1);
-        final ByteBuffer b = Int32Type.instance.decompose(2);
-
-        assertEquals(a, indexTermType.min(a, b));
-        assertEquals(a, indexTermType.min(b, a));
-        assertEquals(a, indexTermType.min(a, a));
-        assertEquals(b, indexTermType.min(b, b));
-        assertEquals(b, indexTermType.min(null, b));
-        assertEquals(a, indexTermType.min(a, null));
-
-        assertEquals(b, indexTermType.max(b, a));
-        assertEquals(b, indexTermType.max(a, b));
-        assertEquals(a, indexTermType.max(a, a));
-        assertEquals(b, indexTermType.max(b, b));
-        assertEquals(b, indexTermType.max(null, b));
-        assertEquals(a, indexTermType.max(a, null));
-    }
-
-    @Test
-    public void testBigIntegerEncoding()
-    {
-        BigInteger[] data = new BigInteger[10000];
-        for (int i = 0; i < data.length; i++)
-        {
-            BigInteger randomNumber = getRandom().nextBigInteger(1000);
-            if (getRandom().nextBoolean())
-                randomNumber = randomNumber.negate();
-
-            data[i] = randomNumber;
-        }
-
-        Arrays.sort(data, BigInteger::compareTo);
-
-        IndexTermType indexTermType = indexTermType(IntegerType.instance, IndexTarget.Type.SIMPLE);
-        assertTrue(indexTermType.supportsRounding());
-
-        for (int i = 1; i < data.length; i++)
-        {
-            BigInteger i0 = data[i - 1];
-            BigInteger i1 = data[i];
-            assertTrue("#" + i, i0.compareTo(i1) <= 0);
-
-            ByteBuffer b0 = indexTermType.asIndexBytes(ByteBuffer.wrap(i0.toByteArray()));
-            ByteBuffer b1 = indexTermType.asIndexBytes(ByteBuffer.wrap(i1.toByteArray()));
-            assertTrue("#" + i, indexTermType.compare(b0, b1) <= 0);
-        }
-    }
-
-    @Test
-    public void testMapEntryEncoding()
-    {
-        CompositeType type = CompositeType.getInstance(UTF8Type.instance, Int32Type.instance);
-        IndexTermType indexTermType = indexTermType(type, IndexTarget.Type.SIMPLE);
-
-        // simulate: index memtable insertion
-        String[] data = new String[10000];
-        byte[] temp = new byte[100];
-        for (int i = 0; i < data.length; i++)
-        {
-            getRandom().nextBytes(temp);
-            String v1 = new String(temp);
-            int v2 = getRandom().nextInt();
-
-            data[i] = indexTermType.asString(type.decompose(v1, v2));
-        }
-
-        Arrays.sort(data, String::compareTo);
-
-        for (int i = 1; i < data.length; i++)
-        {
-            // simulate: index memtable flush
-            ByteBuffer b0 = indexTermType.fromString(data[i - 1]);
-            ByteBuffer b1 = indexTermType.fromString(data[i]);
-            assertTrue("#" + i, indexTermType.compare(b0, b1) <= 0);
-
-            // simulate: saving into on-disk trie
-            ByteComparable t0 = ByteComparable.fixedLength(b0);
-            ByteComparable t1 = ByteComparable.fixedLength(b1);
-            assertTrue("#" + i, ByteComparable.compare(t0, t1, ByteComparable.Version.OSS50) <= 0);
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/PrimaryKeyTest.java b/test/unit/org/apache/cassandra/index/sai/utils/PrimaryKeyTest.java
deleted file mode 100644
index 51edc67389c4..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/utils/PrimaryKeyTest.java
+++ /dev/null
@@ -1,387 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.index.sai.utils;
-
-import java.util.Arrays;
-
-import org.junit.Test;
-
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-
-public class PrimaryKeyTest extends AbstractPrimaryKeyTester
-{
-    @Test
-    public void singlePartitionTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, simplePartition.comparator);
-        int rows = nextInt(10, 100);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        for (int index = 0; index < rows; index++)
-            keys[index] = factory.create(makeKey(simplePartition, index));
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void compositePartitionTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, compositePartition.comparator);
-        int rows = nextInt(10, 100);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        for (int index = 0; index < rows; index++)
-            keys[index] = factory.create(makeKey(compositePartition, index, index + 1));
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void simplePartitonSingleClusteringAscTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, simplePartitionSingleClusteringAsc.comparator);
-        int rows = nextInt(10, 100);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(simplePartitionSingleClusteringAsc, partition),
-                                         makeClustering(simplePartitionSingleClusteringAsc, Integer.toString(clustering++)));
-            if (clustering == 5)
-            {
-                clustering = 0;
-                partition++;
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void simplePartitonStaticAndSingleClusteringAscTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, simplePartitionStaticAndSingleClusteringAsc.comparator);
-        int rows = nextInt(10, 100);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            if (clustering == 0)
-            {
-                keys[index] = factory.create(makeKey(simplePartitionSingleClusteringAsc, partition), Clustering.STATIC_CLUSTERING);
-                clustering++;
-            }
-            else
-                keys[index] = factory.create(makeKey(simplePartitionSingleClusteringAsc, partition),
-                                             makeClustering(simplePartitionSingleClusteringAsc, Integer.toString(clustering++)));
-            if (clustering == 5)
-            {
-                clustering = 0;
-                partition++;
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void simplePartitionMultipleClusteringAscTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, simplePartitionMultipleClusteringAsc.comparator);
-        int rows = nextInt(100, 1000);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering1 = 0;
-        int clustering2 = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(simplePartitionMultipleClusteringAsc, partition),
-                                         makeClustering(simplePartitionMultipleClusteringAsc, Integer.toString(clustering1), Integer.toString(clustering2++)));
-            if (clustering2 == 5)
-            {
-                clustering2 = 0;
-                clustering1++;
-                if (clustering1 == 5)
-                {
-                    clustering1 = 0;
-                    partition++;
-                }
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void simplePartitonSingleClusteringDescTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, simplePartitionSingleClusteringDesc.comparator);
-        int rows = nextInt(10, 100);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(simplePartitionSingleClusteringDesc, partition),
-                                         makeClustering(simplePartitionSingleClusteringDesc, Integer.toString(clustering++)));
-            if (clustering == 5)
-            {
-                clustering = 0;
-                partition++;
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void simplePartitionMultipleClusteringDescTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, simplePartitionMultipleClusteringDesc.comparator);
-        int rows = nextInt(100, 1000);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering1 = 0;
-        int clustering2 = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(simplePartitionMultipleClusteringDesc, partition),
-                                         makeClustering(simplePartitionMultipleClusteringDesc, Integer.toString(clustering1), Integer.toString(clustering2++)));
-            if (clustering2 == 5)
-            {
-                clustering2 = 0;
-                clustering1++;
-                if (clustering1 == 5)
-                {
-                    clustering1 = 0;
-                    partition++;
-                }
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void compositePartitionSingleClusteringAscTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, compositePartitionSingleClusteringAsc.comparator);
-        int rows = nextInt(10, 100);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(compositePartitionSingleClusteringAsc, partition, partition + clustering),
-                                         makeClustering(compositePartitionSingleClusteringAsc, Integer.toString(clustering++)));
-            if (clustering == 5)
-            {
-                clustering = 0;
-                partition += 5;
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void compositePartitionMultipleClusteringAscTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, compositePartitionMultipleClusteringAsc.comparator);
-        int rows = nextInt(100, 1000);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering1 = 0;
-        int clustering2 = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(compositePartitionMultipleClusteringAsc, partition, partition + clustering1 + clustering2),
-                                         makeClustering(compositePartitionMultipleClusteringAsc, Integer.toString(clustering1), Integer.toString(clustering2++)));
-            if (clustering2 == 5)
-            {
-                clustering2 = 0;
-                clustering1++;
-                if (clustering1 == 5)
-                {
-                    clustering1 = 0;
-                    partition += 25;
-                }
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void compositePartitionSingleClusteringDescTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, compositePartitionSingleClusteringDesc.comparator);
-        int rows = nextInt(10, 100);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(compositePartitionSingleClusteringDesc, partition, partition + clustering),
-                                         makeClustering(compositePartitionSingleClusteringDesc, Integer.toString(clustering++)));
-            if (clustering == 5)
-            {
-                clustering = 0;
-                partition += 5;
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void compositePartitionMultipleClusteringDescTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, compositePartitionMultipleClusteringDesc.comparator);
-        int rows = nextInt(100, 1000);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering1 = 0;
-        int clustering2 = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(compositePartitionMultipleClusteringDesc, partition, partition + clustering1 + clustering2),
-                                         makeClustering(compositePartitionMultipleClusteringDesc, Integer.toString(clustering1), Integer.toString(clustering2++)));
-            if (clustering2 == 5)
-            {
-                clustering2 = 0;
-                clustering1++;
-                if (clustering1 == 5)
-                {
-                    clustering1 = 0;
-                    partition += 25;
-                }
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void simplePartitionMultipleClusteringMixedTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, simplePartitionMultipleClusteringMixed.comparator);
-        int rows = nextInt(100, 1000);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering1 = 0;
-        int clustering2 = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(simplePartitionMultipleClusteringMixed, partition),
-                                         makeClustering(simplePartitionMultipleClusteringMixed, Integer.toString(clustering1), Integer.toString(clustering2++)));
-            if (clustering2 == 5)
-            {
-                clustering2 = 0;
-                clustering1++;
-                if (clustering1 == 5)
-                {
-                    clustering1 = 0;
-                    partition++;
-                }
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    @Test
-    public void compositePartitionMultipleClusteringMixedTest()
-    {
-        PrimaryKey.Factory factory = new PrimaryKey.Factory(Murmur3Partitioner.instance, compositePartitionMultipleClusteringMixed.comparator);
-        int rows = nextInt(100, 1000);
-        PrimaryKey[] keys = new PrimaryKey[rows];
-        int partition = 0;
-        int clustering1 = 0;
-        int clustering2 = 0;
-        for (int index = 0; index < rows; index++)
-        {
-            keys[index] = factory.create(makeKey(compositePartitionMultipleClusteringMixed, partition, partition + clustering1 + clustering2),
-                                         makeClustering(compositePartitionMultipleClusteringMixed, Integer.toString(clustering1), Integer.toString(clustering2++)));
-            if (clustering2 == 5)
-            {
-                clustering2 = 0;
-                clustering1++;
-                if (clustering1 == 5)
-                {
-                    clustering1 = 0;
-                    partition += 25;
-                }
-            }
-        }
-
-        Arrays.sort(keys);
-
-        compareToAndEqualsTests(factory, keys);
-    }
-
-    private void compareToAndEqualsTests(PrimaryKey.Factory factory, PrimaryKey... keys)
-    {
-        for (int index = 0; index < keys.length - 1; index++)
-        {
-            PrimaryKey key = keys[index];
-            PrimaryKey tokenOnlyKey = factory.create(key.token());
-
-            assertCompareToAndEquals(tokenOnlyKey, key, 0);
-            assertCompareToAndEquals(key, key, 0);
-            assertCompareToAndEquals(tokenOnlyKey, tokenOnlyKey, 0);
-
-            // StaticPrimaryKey is a special case. All other keys in the partition are equal to it
-            boolean staticComparison = key.kind() == PrimaryKey.Kind.STATIC;
-            boolean inPartition = staticComparison;
-            for (int comparisonIndex = index + 1; comparisonIndex < keys.length; comparisonIndex++)
-            {
-                if (staticComparison && keys[comparisonIndex].kind() == PrimaryKey.Kind.STATIC)
-                    inPartition = false;
-                assertCompareToAndEquals(key, keys[comparisonIndex], inPartition ? 0 : -1);
-                assertCompareToAndEquals(tokenOnlyKey, keys[comparisonIndex], tokenOnlyKey.token().equals(keys[comparisonIndex].token()) ? 0 : -1);
-            }
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/ResourceLeakDetector.java b/test/unit/org/apache/cassandra/index/sai/utils/ResourceLeakDetector.java
index b44d544881aa..e98fb4c8d7a3 100644
--- a/test/unit/org/apache/cassandra/index/sai/utils/ResourceLeakDetector.java
+++ b/test/unit/org/apache/cassandra/index/sai/utils/ResourceLeakDetector.java
@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.index.sai.utils;
 
+import java.lang.annotation.Annotation;
 import java.util.List;
 
 import org.junit.rules.TestRule;
@@ -41,26 +42,30 @@ public class ResourceLeakDetector implements TestRule
     @Override
     public Statement apply(Statement statement, Description description)
     {
-        return new StatementAdapter(statement)
+        if (isResourceLeakCheckEnabled(description))
         {
-            @Override
-            protected void before() throws Throwable
+            return new StatementAdapter(statement)
             {
-                ResourceLeakDetector.this.before();
-            }
+                @Override
+                protected void before() throws Throwable
+                {
+                    ResourceLeakDetector.this.before();
+                }
 
-            @Override
-            protected void afterAlways(List<Throwable> errors)
-            {
-                ResourceLeakDetector.this.afterAlways();
-            }
+                @Override
+                protected void afterAlways(List<Throwable> errors) throws Throwable
+                {
+                    ResourceLeakDetector.this.afterAlways();
+                }
 
-            @Override
-            protected void afterIfSuccessful()
-            {
-                ResourceLeakDetector.this.afterIfSuccessful();
-            }
-        };
+                @Override
+                protected void afterIfSuccessful() throws Throwable
+                {
+                    ResourceLeakDetector.this.afterIfSuccessful();
+                }
+            };
+        }
+        return statement;
     }
 
     protected void before() throws Throwable
@@ -78,4 +83,15 @@ protected void afterAlways()
         Injections.deleteAll();
         RESOURCE_LEAK_COUNTER.reset();
     }
+
+    private boolean isResourceLeakCheckEnabled(Description description)
+    {
+        return !hasAnnotation(description, SuppressLeakCheck.class);
+    }
+
+    private boolean hasAnnotation(Description description, Class<? extends Annotation> annotation)
+    {
+        return ((description.getAnnotation(annotation) != null) ||
+                (description.getTestClass().getAnnotation(annotation) != null));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/RowAwarePrimaryKeyTest.java b/test/unit/org/apache/cassandra/index/sai/utils/RowAwarePrimaryKeyTest.java
new file mode 100644
index 000000000000..44fb2d745faf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/RowAwarePrimaryKeyTest.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.index.sai.disk.v2.RowAwarePrimaryKeyFactory;
+
+public class RowAwarePrimaryKeyTest extends AbstractPrimaryKeyTest
+{
+    @Test
+    public void singlePartitionTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(simplePartition.comparator);
+        int rows = nextInt(10, 100);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        for (int index = 0; index < rows; index++)
+            keys[index] = factory.create(makeKey(simplePartition, Integer.toString(index)), Clustering.EMPTY);
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void compositePartitionTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(compositePartition.comparator);
+        int rows = nextInt(10, 100);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        for (int index = 0; index < rows; index++)
+            keys[index] = factory.create(makeKey(compositePartition, index, index + 1), Clustering.EMPTY);
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void simplePartitonSingleClusteringAscTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(simplePartitionSingleClusteringAsc.comparator);
+        int rows = nextInt(10, 100);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(simplePartitionSingleClusteringAsc, Integer.toString(partition)),
+                                         makeClustering(simplePartitionSingleClusteringAsc, Integer.toString(clustering++)));
+            if (clustering == 5)
+            {
+                clustering = 0;
+                partition++;
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void simplePartitionMultipleClusteringAscTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(simplePartitionMultipleClusteringAsc.comparator);
+        int rows = nextInt(100, 1000);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering1 = 0;
+        int clustering2 = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(simplePartitionMultipleClusteringAsc, Integer.toString(partition)),
+                                         makeClustering(simplePartitionMultipleClusteringAsc, Integer.toString(clustering1), Integer.toString(clustering2++)));
+            if (clustering2 == 5)
+            {
+                clustering2 = 0;
+                clustering1++;
+                if (clustering1 == 5)
+                {
+                    clustering1 = 0;
+                    partition++;
+                }
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void simplePartitonSingleClusteringDescTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(simplePartitionSingleClusteringDesc.comparator);
+        int rows = nextInt(10, 100);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(simplePartitionSingleClusteringDesc, Integer.toString(partition)),
+                                         makeClustering(simplePartitionSingleClusteringDesc, Integer.toString(clustering++)));
+            if (clustering == 5)
+            {
+                clustering = 0;
+                partition++;
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void simplePartitionMultipleClusteringDescTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(simplePartitionMultipleClusteringDesc.comparator);
+        int rows = nextInt(100, 1000);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering1 = 0;
+        int clustering2 = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(simplePartitionMultipleClusteringDesc, Integer.toString(partition)),
+                                         makeClustering(simplePartitionMultipleClusteringDesc, Integer.toString(clustering1), Integer.toString(clustering2++)));
+            if (clustering2 == 5)
+            {
+                clustering2 = 0;
+                clustering1++;
+                if (clustering1 == 5)
+                {
+                    clustering1 = 0;
+                    partition++;
+                }
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void compositePartitionSingleClusteringAscTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(compositePartitionSingleClusteringAsc.comparator);
+        int rows = nextInt(10, 100);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(compositePartitionSingleClusteringAsc, partition, partition + clustering),
+                                         makeClustering(compositePartitionSingleClusteringAsc, Integer.toString(clustering++)));
+            if (clustering == 5)
+            {
+                clustering = 0;
+                partition += 5;
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void compositePartitionMultipleClusteringAscTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(compositePartitionMultipleClusteringAsc.comparator);
+        int rows = nextInt(100, 1000);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering1 = 0;
+        int clustering2 = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(compositePartitionMultipleClusteringAsc, partition, partition + clustering1 + clustering2),
+                                         makeClustering(compositePartitionMultipleClusteringAsc, Integer.toString(clustering1), Integer.toString(clustering2++)));
+            if (clustering2 == 5)
+            {
+                clustering2 = 0;
+                clustering1++;
+                if (clustering1 == 5)
+                {
+                    clustering1 = 0;
+                    partition += 25;
+                }
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void compositePartitionSingleClusteringDescTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(compositePartitionSingleClusteringDesc.comparator);
+        int rows = nextInt(10, 100);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(compositePartitionSingleClusteringDesc, partition, partition + clustering),
+                                         makeClustering(compositePartitionSingleClusteringDesc, Integer.toString(clustering++)));
+            if (clustering == 5)
+            {
+                clustering = 0;
+                partition += 5;
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void compositePartitionMultipleClusteringDescTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(compositePartitionMultipleClusteringDesc.comparator);
+        int rows = nextInt(100, 1000);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering1 = 0;
+        int clustering2 = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(compositePartitionMultipleClusteringDesc, partition, partition + clustering1 + clustering2),
+                                         makeClustering(compositePartitionMultipleClusteringDesc, Integer.toString(clustering1), Integer.toString(clustering2++)));
+            if (clustering2 == 5)
+            {
+                clustering2 = 0;
+                clustering1++;
+                if (clustering1 == 5)
+                {
+                    clustering1 = 0;
+                    partition += 25;
+                }
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void simplePartitionMultipleClusteringMixedTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(simplePartitionMultipleClusteringMixed.comparator);
+        int rows = nextInt(100, 1000);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering1 = 0;
+        int clustering2 = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(simplePartitionMultipleClusteringMixed, Integer.toString(partition)),
+                                         makeClustering(simplePartitionMultipleClusteringMixed, Integer.toString(clustering1), Integer.toString(clustering2++)));
+            if (clustering2 == 5)
+            {
+                clustering2 = 0;
+                clustering1++;
+                if (clustering1 == 5)
+                {
+                    clustering1 = 0;
+                    partition++;
+                }
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    @Test
+    public void compositePartitionMultipleClusteringMixedTest()
+    {
+        PrimaryKey.Factory factory = new RowAwarePrimaryKeyFactory(compositePartitionMultipleClusteringMixed.comparator);
+        int rows = nextInt(100, 1000);
+        PrimaryKey[] keys = new PrimaryKey[rows];
+        int partition = 0;
+        int clustering1 = 0;
+        int clustering2 = 0;
+        for (int index = 0; index < rows; index++)
+        {
+            keys[index] = factory.create(makeKey(compositePartitionMultipleClusteringMixed, partition, partition + clustering1 + clustering2),
+                                         makeClustering(compositePartitionMultipleClusteringMixed, Integer.toString(clustering1), Integer.toString(clustering2++)));
+            if (clustering2 == 5)
+            {
+                clustering2 = 0;
+                clustering1++;
+                if (clustering1 == 5)
+                {
+                    clustering1 = 0;
+                    partition += 25;
+                }
+            }
+        }
+
+        Arrays.sort(keys);
+
+        byteComparisonTests(factory, keys);
+        compareToAndEqualsTests(factory, keys);
+    }
+
+    private void compareToAndEqualsTests(PrimaryKey.Factory factory, PrimaryKey... keys)
+    {
+        for (int index = 0; index < keys.length - 1; index++)
+        {
+            PrimaryKey key = keys[index];
+            PrimaryKey tokenOnlyKey = factory.createTokenOnly(key.token());
+
+            assertCompareToAndEquals(tokenOnlyKey, key, 0);
+            assertCompareToAndEquals(key, key, 0);
+            assertCompareToAndEquals(tokenOnlyKey, tokenOnlyKey, 0);
+
+            for (int comparisonIndex = index + 1; comparisonIndex < keys.length; comparisonIndex++)
+            {
+                assertCompareToAndEquals(key, keys[comparisonIndex], -1);
+                assertCompareToAndEquals(tokenOnlyKey, keys[comparisonIndex], tokenOnlyKey.token().equals(keys[comparisonIndex].token()) ? 0 : -1);
+            }
+        }
+    }
+
+    private void byteComparisonTests(PrimaryKey.Factory factory, PrimaryKey... keys)
+    {
+        for (int index = 0; index < keys.length - 1; index++)
+        {
+            PrimaryKey key = keys[index];
+            PrimaryKey tokenOnlyKey = factory.createTokenOnly(key.token());
+            assertByteComparison(tokenOnlyKey, key, -1);
+            assertByteComparison(key, key, 0);
+            assertByteComparison(tokenOnlyKey, tokenOnlyKey, 0);
+
+            for (int comparisonIndex = index + 1; comparisonIndex < keys.length; comparisonIndex++)
+            {
+                assertByteComparison(key, keys[comparisonIndex], -1);
+                assertByteComparison(tokenOnlyKey, keys[comparisonIndex], -1);
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/RowWithSourceTableTest.java b/test/unit/org/apache/cassandra/index/sai/utils/RowWithSourceTableTest.java
new file mode 100644
index 000000000000..69de13b69fee
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/RowWithSourceTableTest.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.rows.ArrayCell;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.memory.HeapCloner;
+
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class RowWithSourceTableTest {
+
+    private RowWithSourceTable rowWithSourceTable;
+    private TableMetadata tableMetadata;
+    private ColumnMetadata complexColumn;
+    private ColumnMetadata column;
+    private CellPath complexCellPath;
+    private Cell<?> complexCell;
+    private Cell<?> cell;
+    private Row originalRow;
+    private final Object source = new Object();
+    // We use a 4 byte array because the Int32Type is used in the test
+    private final byte[] value = new byte[]{0,0,0,1};
+
+    @Before
+    public void setUp()
+    {
+        var listType = ListType.getInstance(Int32Type.instance, true);
+        complexColumn = ColumnMetadata.regularColumn("keyspace1", "table1", "complex", listType);
+        column = ColumnMetadata.regularColumn("keyspace1", "table1", "name1", Int32Type.instance);
+        tableMetadata = TableMetadata.builder("keyspace1", "table1")
+                                     .addPartitionKeyColumn("pk", Int32Type.instance)
+                                     .addColumn(complexColumn)
+                                     .addColumn(column).build();
+        complexCellPath = CellPath.create(ByteBuffer.allocate(0));
+        complexCell = new ArrayCell(complexColumn, System.currentTimeMillis(), Cell.NO_TTL, Cell.NO_DELETION_TIME, value, complexCellPath);
+        cell = new ArrayCell(column, System.currentTimeMillis(), Cell.NO_TTL, Cell.NO_DELETION_TIME, value, null);
+        // Use unsorted builder to avoid the need to manually sort cells here
+        var builder = BTreeRow.unsortedBuilder();
+        builder.newRow(Clustering.EMPTY);
+        builder.addCell(complexCell);
+        builder.addCell(cell);
+        originalRow = builder.build();
+        rowWithSourceTable = new RowWithSourceTable(originalRow, source);
+    }
+
+    @Test
+    public void testKind()
+    {
+        assertEquals(originalRow.kind(), rowWithSourceTable.kind());
+    }
+
+    @Test
+    public void testClustering()
+    {
+        assertEquals(originalRow.clustering(), rowWithSourceTable.clustering());
+    }
+
+    @Test
+    public void testDigest()
+    {
+        var digest1 = Digest.forValidator();
+        rowWithSourceTable.digest(digest1);
+        var digest2 = Digest.forValidator();
+        originalRow.digest(digest2);
+        assertArrayEquals(digest1.digest(), digest2.digest());
+    }
+
+    @Test
+    public void testValidateData()
+    {
+        rowWithSourceTable.validateData(tableMetadata);
+    }
+
+    @Test
+    public void testHasInvalidDeletions()
+    {
+        assertFalse(rowWithSourceTable.hasInvalidDeletions());
+    }
+
+    @Test
+    public void testColumns()
+    {
+        assertEquals(2, rowWithSourceTable.columns().size());
+        assertTrue(rowWithSourceTable.columns().contains(complexColumn));
+        assertTrue(rowWithSourceTable.columns().contains(column));
+    }
+
+    @Test
+    public void testColumnCount()
+    {
+        assertEquals(2, rowWithSourceTable.columnCount());
+        assertEquals(originalRow.columnCount(), rowWithSourceTable.columnCount());
+    }
+
+    @Test
+    public void testDeletion()
+    {
+        assertEquals(originalRow.deletion(), rowWithSourceTable.deletion());
+    }
+
+    @Test
+    public void testPrimaryKeyLivenessInfo()
+    {
+        assertEquals(originalRow.primaryKeyLivenessInfo(), rowWithSourceTable.primaryKeyLivenessInfo());
+    }
+
+    @Test
+    public void testIsStatic()
+    {
+        assertEquals(originalRow.isStatic(), rowWithSourceTable.isStatic());
+    }
+
+    @Test
+    public void testIsEmpty()
+    {
+        assertFalse(rowWithSourceTable.isEmpty());
+        assertEquals(originalRow.isEmpty(), rowWithSourceTable.isEmpty());
+    }
+
+    @Test
+    public void testToString()
+    {
+       assertEquals(originalRow.toString(tableMetadata), rowWithSourceTable.toString(tableMetadata));
+    }
+
+    @Test
+    public void testHasLiveData()
+    {
+        assertTrue(originalRow.hasLiveData(1000, false));
+        assertTrue(rowWithSourceTable.hasLiveData(1000, false));
+    }
+
+    @Test
+    public void testGetCellWithCorrectColumn()
+    {
+        var resultCell = rowWithSourceTable.getCell(column);
+        assertTrue(resultCell instanceof CellWithSourceTable);
+        // This mapping is the whole point of these two classes.
+        assertSame(source, ((CellWithSourceTable<?>)resultCell).sourceTable());
+        assertSame(cell.value(), resultCell.value());
+    }
+
+    @Test
+    public void testGetCellWithMissingColumn()
+    {
+        var diffCol = ColumnMetadata.regularColumn("keyspace1", "table1", "name2", Int32Type.instance);
+        assertNull(rowWithSourceTable.getCell(diffCol));
+    }
+
+    @Test
+    public void testGetCellWithPath()
+    {
+        Cell<?> resultCell = rowWithSourceTable.getCell(complexColumn, complexCellPath);
+        assertTrue(resultCell instanceof CellWithSourceTable);
+        // This mapping is the whole point of these two classes.
+        assertSame(source, ((CellWithSourceTable<?>)resultCell).sourceTable());
+        assertSame(cell.value(), resultCell.value());
+    }
+
+    @Test
+    public void testGetComplexColumnData()
+    {
+        var complexColumnData = rowWithSourceTable.getComplexColumnData(complexColumn);
+        var firstCell = complexColumnData.iterator().next();
+        assertTrue(firstCell instanceof CellWithSourceTable);
+        assertSame(source, ((CellWithSourceTable<?>)firstCell).sourceTable());
+    }
+
+    @Test
+    public void testGetColumnData()
+    {
+        var simpleColumnData = rowWithSourceTable.getColumnData(column);
+        assertTrue(simpleColumnData instanceof CellWithSourceTable);
+        assertSame(source, ((CellWithSourceTable<?>)simpleColumnData).sourceTable());
+        var complexColumnData = rowWithSourceTable.getColumnData(complexColumn);
+        assertTrue(complexColumnData instanceof ComplexColumnData);
+        var firstCell = ((ComplexColumnData)complexColumnData).iterator().next();
+        assertTrue(firstCell instanceof CellWithSourceTable);
+        assertSame(source, ((CellWithSourceTable<?>)firstCell).sourceTable());
+    }
+
+    @Test
+    public void testCells()
+    {
+        var cells = originalRow.cells().iterator();
+        var wrappedCells = rowWithSourceTable.cells().iterator();
+        while (cells.hasNext())
+        {
+            var cell = cells.next();
+            var wrappedCell = wrappedCells.next();
+            assertTrue(wrappedCell instanceof CellWithSourceTable);
+            assertSame(source, ((CellWithSourceTable<?>)wrappedCell).sourceTable());
+            assertSame(cell.value(), wrappedCell.value());
+        }
+        assertFalse(wrappedCells.hasNext());
+    }
+
+    @Test
+    public void testColumnData()
+    {
+        var columnDataCollection = rowWithSourceTable.columnData();
+        assertEquals(2, columnDataCollection.size());
+        var iter = columnDataCollection.iterator();
+        while (iter.hasNext())
+        {
+            var columnData = iter.next();
+            if (columnData instanceof CellWithSourceTable)
+            {
+                assertSame(source, ((CellWithSourceTable<?>)columnData).sourceTable());
+            }
+            else if (columnData instanceof ComplexColumnData)
+            {
+                var complexIter = ((ComplexColumnData)columnData).iterator();
+                while (complexIter.hasNext())
+                {
+                    var cell = complexIter.next();
+                    assertTrue(cell instanceof CellWithSourceTable);
+                    assertSame(source, ((CellWithSourceTable<?>)cell).sourceTable());
+                }
+            }
+            else
+            {
+                fail("Unexpected column data type");
+            }
+        }
+
+    }
+
+    @Test
+    public void testCellsInLegacyOrder()
+    {
+        var cells = originalRow.cellsInLegacyOrder(tableMetadata, false).iterator();
+        var wrappedCells = rowWithSourceTable.cellsInLegacyOrder(tableMetadata, false).iterator();
+        while (cells.hasNext())
+        {
+            var cell = cells.next();
+            var wrappedCell = wrappedCells.next();
+            assertTrue(wrappedCell instanceof CellWithSourceTable);
+            assertSame(source, ((CellWithSourceTable<?>)wrappedCell).sourceTable());
+            assertSame(cell.value(), wrappedCell.value());
+        }
+        assertFalse(wrappedCells.hasNext());
+    }
+
+    @Test
+    public void testHasComplexDeletion()
+    {
+        assertFalse(rowWithSourceTable.hasComplexDeletion());
+    }
+
+    @Test
+    public void testHasComplex()
+    {
+        assertTrue(rowWithSourceTable.hasComplex());
+    }
+
+    @Test
+    public void testHasDeletion()
+    {
+        assertFalse(rowWithSourceTable.hasDeletion(1000));
+    }
+
+    @Test
+    public void testSearchIterator()
+    {
+        var iterator = rowWithSourceTable.searchIterator();
+        var columnData = iterator.next(column);
+        assertTrue(columnData instanceof CellWithSourceTable);
+        assertSame(source, ((CellWithSourceTable<?>)columnData).sourceTable());
+        assertNull(iterator.next(column));
+    }
+
+    @Test
+    public void testFilter()
+    {
+        assertSame(rowWithSourceTable, rowWithSourceTable.filter(ColumnFilter.all(tableMetadata), tableMetadata));
+    }
+
+    @Test
+    public void testFilterWithDeletion()
+    {
+        assertSame(rowWithSourceTable, rowWithSourceTable.filter(ColumnFilter.all(tableMetadata), DeletionTime.LIVE, true, tableMetadata));
+    }
+
+    @Test
+    public void testTransformAndFilter()
+    {
+        assertSame(rowWithSourceTable, rowWithSourceTable.transformAndFilter(LivenessInfo.EMPTY, Row.Deletion.LIVE, c -> c));
+    }
+
+    @Test
+    public void testTransformAndFilterWithFunction() 
+    {
+        assertNull(rowWithSourceTable.transformAndFilter(c -> null));
+        assertSame(rowWithSourceTable, rowWithSourceTable.transformAndFilter(c -> c));
+    }
+
+    @Test
+    public void testClone()
+    {
+        assertTrue(rowWithSourceTable.clone(HeapCloner.instance) instanceof RowWithSourceTable);
+    }
+
+    @Test
+    public void testDataSize()
+    {
+        assertEquals(originalRow.dataSize(), rowWithSourceTable.dataSize());
+    }
+
+    @Test
+    public void testUnsharedHeapSizeExcludingData()
+    {
+        var wrapperSize = ObjectSizes.measure(new RowWithSourceTable(null, null));
+        assertEquals(originalRow.unsharedHeapSizeExcludingData() + wrapperSize,
+                     rowWithSourceTable.unsharedHeapSizeExcludingData());
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/SAIRandomizedTester.java b/test/unit/org/apache/cassandra/index/sai/utils/SAIRandomizedTester.java
deleted file mode 100644
index f3129a44f897..000000000000
--- a/test/unit/org/apache/cassandra/index/sai/utils/SAIRandomizedTester.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.index.sai.utils;
-
-import java.io.IOException;
-
-import com.google.common.base.Preconditions;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TemporaryFolder;
-import org.junit.rules.TestRule;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
-import org.apache.cassandra.index.sai.postings.PostingList;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.TableMetadata;
-
-import static org.junit.Assert.assertEquals;
-
-public class SAIRandomizedTester extends SAITester
-{
-    @SuppressWarnings("unused")
-    @BeforeClass
-    public static void saveUncaughtExceptionHandler()
-    {
-        DatabaseDescriptor.daemonInitialization();
-    }
-
-    private static final IndexInputLeakDetector indexInputLeakDetector;
-
-    protected static final TemporaryFolder temporaryFolder;
-
-    @ClassRule
-    public static TestRule classRules = RuleChain.outerRule(indexInputLeakDetector = new IndexInputLeakDetector())
-                                                 .around(temporaryFolder = new TemporaryFolder());
-
-    public static IndexDescriptor newIndexDescriptor() throws IOException
-    {
-        String keyspace = randomSimpleString(5, 13);
-        String table = randomSimpleString(3, 17);
-        TableMetadata metadata = TableMetadata.builder(keyspace, table)
-                                              .addPartitionKeyColumn(randomSimpleString(3, 15), Int32Type.instance)
-                                              .partitioner(Murmur3Partitioner.instance)
-                                              .build();
-        return indexInputLeakDetector.newIndexDescriptor(new Descriptor(new File(temporaryFolder.newFolder()),
-                                                                        randomSimpleString(5, 13),
-                                                                        randomSimpleString(3, 17),
-                                                                        new SequenceBasedSSTableId(getRandom().nextIntBetween(0, 128))),
-                                                         metadata,
-                                                         IndexFileUtils.DEFAULT_WRITER_OPTION);
-    }
-
-    public static IndexDescriptor newClusteringIndexDescriptor(TableMetadata metadata) throws IOException
-    {
-        return indexInputLeakDetector.newIndexDescriptor(new Descriptor(new File(temporaryFolder.newFolder()),
-                                                                        randomSimpleString(5, 13),
-                                                                        randomSimpleString(3, 17),
-                                                                        new SequenceBasedSSTableId(getRandom().nextIntBetween(0, 128))),
-                                                         metadata,
-                                                         IndexFileUtils.DEFAULT_WRITER_OPTION);
-    }
-
-    public String newIndex()
-    {
-        return randomSimpleString(2, 29);
-    }
-
-    /**
-     * Load a byte array with random bytes. Shortcut for getRandom() method
-     */
-    public static void nextBytes(byte[] bytes)
-    {
-        getRandom().nextBytes(bytes);
-    }
-
-    public static byte[] nextBytes(int min, int max)
-    {
-        byte[] bytes = new byte[nextInt(min, max)];
-        nextBytes(bytes);
-        return bytes;
-    }
-
-    //
-    // Note: The nextXXX methods maintain the contract of ThreadLocalRandom
-    // where the max value is exclusive. The between methods maintain
-    // the contract where the max value is inclusive
-    //
-
-    public static int nextInt(int max)
-    {
-        return nextInt(0, max);
-    }
-
-    public static int nextInt(int min, int max)
-    {
-        return getRandom().nextIntBetween(min, max - 1);
-    }
-
-    public static long nextLong(long min, long max)
-    {
-        return between(min, max - 1);
-    }
-
-    public static int between(int min, int max)
-    {
-        return getRandom().nextIntBetween(min, max - 1);
-    }
-
-    public static double nextDouble()
-    {
-        return getRandom().nextDouble();
-    }
-
-    public static boolean nextBoolean()
-    {
-        return getRandom().nextBoolean();
-    }
-
-    public static long between(long min, long max)
-    {
-        return randomLongBetween(min, max);
-    }
-
-    public static long randomLongBetween(long min, long max)
-    {
-        if (min < 0) throw new IllegalArgumentException("min must be >= 0: " + min);
-        if (min > max) throw new IllegalArgumentException("max must be >= min: " + min + ", " + max);
-        return min == max ? min : (long) randomDoubleBetween((double) min, (double) max);
-    }
-
-    public static double randomDoubleBetween(double min, double max)
-    {
-        if (min < 0) throw new IllegalArgumentException("min must be >= 0: " + min);
-        if (min > max) throw new IllegalArgumentException("max must be >= min: " + min + ", " + max);
-
-        return min == max ? min : min + (max - min) * getRandom().nextDouble();
-    }
-
-    public static String randomSimpleString(int minLength, int maxLength)
-    {
-        Preconditions.checkArgument(minLength >= 0);
-        Preconditions.checkArgument(maxLength >= 0);
-        final int end = nextInt(minLength, maxLength);
-        if (end == 0)
-        {
-            // allow 0 length
-            return "";
-        }
-        final char[] buffer = new char[end];
-        for (int i = 0; i < end; i++)
-        {
-            buffer[i] = (char) nextInt('a', 'z');
-        }
-        return new String(buffer, 0, end);
-    }
-
-    public static void assertPostingListEquals(PostingList expected, PostingList actual) throws IOException
-    {
-        long actualRowID, rowCounter = 0;
-        while ((actualRowID = actual.nextPosting()) != PostingList.END_OF_STREAM)
-        {
-            assertEquals("Mismatch at pos: " + rowCounter, expected.nextPosting(), actualRowID);
-            rowCounter++;
-        }
-        assertEquals(PostingList.END_OF_STREAM, expected.nextPosting());
-    }
-
-    public static void shuffle(int[] array)
-    {
-        for (int i=0; i< array.length; i++)
-        {
-            int randomPosition = getRandom().nextIntBetween(0, array.length - 1);
-            int temp = array[i];
-            array[i] = array[randomPosition];
-            array[randomPosition] = temp;
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/SaiRandomizedTest.java b/test/unit/org/apache/cassandra/index/sai/utils/SaiRandomizedTest.java
new file mode 100644
index 000000000000..64e79fbcd25b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/SaiRandomizedTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.io.IOException;
+import java.util.Random;
+
+import com.google.common.base.Preconditions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.disk.PostingList;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class SaiRandomizedTest extends RandomizedTest
+{
+    private static Thread.UncaughtExceptionHandler handler;
+
+    @SuppressWarnings("unused")
+    @BeforeClass
+    private static void saveUncaughtExceptionHandler()
+    {
+        handler = Thread.getDefaultUncaughtExceptionHandler();
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @SuppressWarnings("unused")
+    @AfterClass
+    private static void restoreUncaughtExceptionHandler()
+    {
+        Thread.setDefaultUncaughtExceptionHandler(handler);
+    }
+
+    private static IndexInputLeakDetector indexInputLeakDetector;
+
+    protected static TemporaryFolder temporaryFolder;
+
+    @ClassRule
+    public static TestRule classRules = RuleChain.outerRule(indexInputLeakDetector = new IndexInputLeakDetector())
+                                                 .around(temporaryFolder = new TemporaryFolder());
+
+    public IndexDescriptor newIndexDescriptor() throws IOException
+    {
+        return indexInputLeakDetector.newIndexDescriptor(new Descriptor(new File(temporaryFolder.newFolder()),
+                                                                        randomSimpleString(5, 13),
+                                                                        randomSimpleString(3, 17),
+                                                                        new SequenceBasedSSTableId(randomIntBetween(0, 128))),
+                                                         SequentialWriterOption.newBuilder()
+                                                                               .bufferSize(randomIntBetween(17, 1 << 13))
+                                                                               .bufferType(randomBoolean() ? BufferType.ON_HEAP : BufferType.OFF_HEAP)
+                                                                               .trickleFsync(randomBoolean())
+                                                                               .trickleFsyncByteInterval(nextInt(1 << 10, 1 << 16))
+                                                                               .finishOnClose(true)
+                                                                               .build());
+    }
+
+    public String newIndex()
+    {
+        return randomSimpleString(2, 29);
+    }
+
+    /**
+     * Load a byte array with random bytes. Shortcut for getRandom() method
+     */
+    public static void nextBytes(byte[] bytes)
+    {
+        getRandom().nextBytes(bytes);
+    }
+
+    public static byte[] nextBytes(int min, int max)
+    {
+        byte[] bytes = new byte[nextInt(min, max)];
+        nextBytes(bytes);
+        return bytes;
+    }
+
+    //
+    // Note: The nextXXX methods maintain the contract of ThreadLocalRandom
+    // where the max value is exclusive. The between methods maintain
+    // the contract of RandomizedTest where the max value is inclusive
+    //
+
+    public static int nextInt(int max)
+    {
+        return nextInt(0, max);
+    }
+
+    public static int nextInt(int min, int max)
+    {
+        return between(min, max - 1);
+    }
+
+    public static long nextLong(long min, long max)
+    {
+        return between(min, max - 1);
+    }
+
+    public static double nextDouble()
+    {
+        return randomDoubleBetween(0, 1);
+    }
+
+    public static long between(long min, long max)
+    {
+        return randomLongBetween(min, max);
+    }
+
+    public static int randomIntBetween(int min, int max)
+    {
+        if (min < 0) throw new IllegalArgumentException("min must be >= 0: " + min);
+        if (min > max) throw new IllegalArgumentException("max must be >= min: " + min + ", " + max);
+        return min == max ? min : (int) randomDoubleBetween((double) min, (double) max);
+    }
+
+    public static long randomLongBetween(long min, long max)
+    {
+        if (min < 0) throw new IllegalArgumentException("min must be >= 0: " + min);
+        if (min > max) throw new IllegalArgumentException("max must be >= min: " + min + ", " + max);
+        return min == max ? min : (long) randomDoubleBetween((double) min, (double) max);
+    }
+
+    public static double randomDoubleBetween(double min, double max)
+    {
+        if (min < 0) throw new IllegalArgumentException("min must be >= 0: " + min);
+        if (min > max) throw new IllegalArgumentException("max must be >= min: " + min + ", " + max);
+
+        return min == max ? min : min + (max - min) * randomDouble();
+    }
+
+    public static long scaledRandomLongBetween(long min, long max)
+    {
+        if (min < 0) throw new IllegalArgumentException("min must be >= 0: " + min);
+        if (min > max) throw new IllegalArgumentException("max must be >= min: " + min + ", " + max);
+
+        double point = Math.min(1, Math.abs(randomGaussian()) * 0.3) * multiplier();
+        double range = max - min;
+        long scaled = Math.round(Math.min(point * range, range));
+        return isNightly() ? max - scaled : min + scaled;
+    }
+
+    public static String randomSimpleString(int minLength, int maxLength)
+    {
+        Preconditions.checkArgument(minLength >= 0);
+        Preconditions.checkArgument(maxLength >= 0);
+        final int end = nextInt(minLength, maxLength);
+        if (end == 0)
+        {
+            // allow 0 length
+            return "";
+        }
+        final char[] buffer = new char[end];
+        for (int i = 0; i < end; i++)
+        {
+            buffer[i] = (char) nextInt('a', 'z');
+        }
+        return new String(buffer, 0, end);
+    }
+
+    public static void assertPostingListEquals(PostingList expected, PostingList actual) throws IOException
+    {
+        long actualRowID, rowCounter = 0;
+        while ((actualRowID = actual.nextPosting()) != PostingList.END_OF_STREAM)
+        {
+            assertEquals("Mismatch at pos: " + rowCounter, expected.nextPosting(), actualRowID);
+            rowCounter++;
+        }
+        assertEquals(PostingList.END_OF_STREAM, expected.nextPosting());
+    }
+
+    public static int[] shuffle(int[] array)
+    {
+        Random rgen = new Random();
+
+        for (int i=0; i< array.length; i++)
+        {
+            int randomPosition = rgen.nextInt(array.length);
+            int temp = array[i];
+            array[i] = array[randomPosition];
+            array[randomPosition] = temp;
+        }
+
+        return array;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/SegmentRowIdOrdinalPairsTest.java b/test/unit/org/apache/cassandra/index/sai/utils/SegmentRowIdOrdinalPairsTest.java
new file mode 100644
index 000000000000..87daed0e609e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/SegmentRowIdOrdinalPairsTest.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
+public class SegmentRowIdOrdinalPairsTest
+{
+    @Test
+    public void testBasicOperations()
+    {
+        SegmentRowIdOrdinalPairs pairs = new SegmentRowIdOrdinalPairs(3);
+
+        // Test initial state
+        assertEquals(0, pairs.size());
+
+        // Test adding pairs
+        pairs.add(1, 10);
+        pairs.add(2, 20);
+        pairs.add(3, 30);
+
+        assertEquals(3, pairs.size());
+
+        // Test getting values
+        assertEquals(1, pairs.getSegmentRowId(0));
+        assertEquals(2, pairs.getSegmentRowId(1));
+        assertEquals(3, pairs.getSegmentRowId(2));
+
+        assertEquals(10, pairs.getOrdinal(0));
+        assertEquals(20, pairs.getOrdinal(1));
+        assertEquals(30, pairs.getOrdinal(2));
+    }
+
+    @Test
+    public void testForEachOrdinal()
+    {
+        SegmentRowIdOrdinalPairs pairs = new SegmentRowIdOrdinalPairs(3);
+        pairs.add(1, 10);
+        pairs.add(2, 20);
+        pairs.add(3, 30);
+
+        List<Integer> ordinals = new ArrayList<>();
+        pairs.forEachOrdinal(ordinals::add);
+
+        assertEquals(3, ordinals.size());
+        assertEquals(Integer.valueOf(10), ordinals.get(0));
+        assertEquals(Integer.valueOf(20), ordinals.get(1));
+        assertEquals(Integer.valueOf(30), ordinals.get(2));
+    }
+
+    @Test
+    public void testForEachSegmentRowIdOrdinalPair()
+    {
+        SegmentRowIdOrdinalPairs pairs = new SegmentRowIdOrdinalPairs(3);
+        pairs.add(1, 10);
+        pairs.add(2, 20);
+        pairs.add(3, 30);
+
+        List<Integer> rowIds = new ArrayList<>();
+        List<Integer> ordinals = new ArrayList<>();
+
+        pairs.forEachSegmentRowIdOrdinalPair((rowId, ordinal) -> {
+            rowIds.add(rowId);
+            ordinals.add(ordinal);
+        });
+
+        assertEquals(3, rowIds.size());
+        assertEquals(3, ordinals.size());
+        assertEquals(Integer.valueOf(1), rowIds.get(0));
+        assertEquals(Integer.valueOf(10), ordinals.get(0));
+        assertEquals(Integer.valueOf(2), rowIds.get(1));
+        assertEquals(Integer.valueOf(20), ordinals.get(1));
+        assertEquals(Integer.valueOf(3), rowIds.get(2));
+        assertEquals(Integer.valueOf(30), ordinals.get(2));
+    }
+
+    @Test
+    public void testForEachIndexOrdinalPair()
+    {
+        SegmentRowIdOrdinalPairs pairs = new SegmentRowIdOrdinalPairs(3);
+        pairs.add(1, 10);
+        pairs.add(2, 20);
+        pairs.add(3, 30);
+
+        List<Integer> indices = new ArrayList<>();
+        List<Integer> ordinals = new ArrayList<>();
+
+        pairs.forEachIndexOrdinalPair((index, ordinal) -> {
+            indices.add(index);
+            ordinals.add(ordinal);
+        });
+
+        assertEquals(3, indices.size());
+        assertEquals(3, ordinals.size());
+        assertEquals(Integer.valueOf(0), indices.get(0));
+        assertEquals(Integer.valueOf(10), ordinals.get(0));
+        assertEquals(Integer.valueOf(1), indices.get(1));
+        assertEquals(Integer.valueOf(20), ordinals.get(1));
+        assertEquals(Integer.valueOf(2), indices.get(2));
+        assertEquals(Integer.valueOf(30), ordinals.get(2));
+    }
+
+    @Test
+    public void testGetSegmentRowIdAndOrdinalBoundaryChecks()
+    {
+        SegmentRowIdOrdinalPairs pairs = new SegmentRowIdOrdinalPairs(2);
+        pairs.add(1, 10);
+
+        assertThrows(ArrayIndexOutOfBoundsException.class, () -> pairs.getSegmentRowId(-1));
+        assertThrows(ArrayIndexOutOfBoundsException.class, () -> pairs.getSegmentRowId(1));
+        assertThrows(ArrayIndexOutOfBoundsException.class, () -> pairs.getOrdinal(-1));
+        assertThrows(ArrayIndexOutOfBoundsException.class, () -> pairs.getOrdinal(1));
+    }
+
+    @Test
+    public void testAddToFullArray()
+    {
+        SegmentRowIdOrdinalPairs pairs = new SegmentRowIdOrdinalPairs(1);
+        pairs.add(1, 10);
+        assertThrows(IndexOutOfBoundsException.class, () -> pairs.add(2, 20));
+    }
+
+    @Test
+    public void testCapacityTooLarge()
+    {
+        assertThrows(AssertionError.class, () -> new SegmentRowIdOrdinalPairs(Integer.MAX_VALUE / 2 + 1));
+    }
+
+    @Test
+    public void testOperationsOnEmptyArray()
+    {
+        SegmentRowIdOrdinalPairs pairs = new SegmentRowIdOrdinalPairs(1);
+        AtomicInteger count = new AtomicInteger(0);
+
+        pairs.forEachOrdinal(i -> count.incrementAndGet());
+        assertEquals(0, count.get());
+
+        pairs.forEachSegmentRowIdOrdinalPair((x, y) -> count.incrementAndGet());
+        assertEquals(0, count.get());
+
+        pairs.forEachIndexOrdinalPair((x, y) -> count.incrementAndGet());
+        assertEquals(0, count.get());
+    }
+
+    @Test
+    public void testZeroCapacity()
+    {
+        SegmentRowIdOrdinalPairs pairs = new SegmentRowIdOrdinalPairs(0);
+        assertEquals(0, pairs.size());
+        assertThrows(IndexOutOfBoundsException.class, () -> pairs.add(1, 10));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/SoftLimitUtilTest.java b/test/unit/org/apache/cassandra/index/sai/utils/SoftLimitUtilTest.java
new file mode 100644
index 000000000000..73d4b6021e90
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/SoftLimitUtilTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.Random;
+
+import org.apache.commons.math3.distribution.BinomialDistribution;
+import org.junit.Test;
+
+import static org.apache.cassandra.index.sai.utils.SoftLimitUtil.softLimit;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class SoftLimitUtilTest
+{
+    private final Random random = new Random();
+
+    @Test
+    public void perItemProbabilityOne()
+    {
+        assertEquals(0, softLimit(0, 0.999, 1.0));
+        assertEquals(1, softLimit(1, 0.999, 1.0));
+        assertEquals(100, softLimit(100, 0.999, 1.0));
+        assertEquals(1000000, softLimit(1000000, 0.999, 1.0));
+        assertEquals(Integer.MAX_VALUE, softLimit(Integer.MAX_VALUE, 0.999, 1.0));
+    }
+
+    @Test
+    public void perItemProbabilityZero()
+    {
+        assertEquals(0, softLimit(0, 0.999, 0.0), 0);
+        assertEquals(Integer.MAX_VALUE, softLimit(1, 0.999, 0.0));
+        assertEquals(Integer.MAX_VALUE, softLimit(100, 0.999, 0.0));
+        assertEquals(Integer.MAX_VALUE, softLimit(1000000, 0.999, 0.0));
+        assertEquals(Integer.MAX_VALUE, softLimit(Integer.MAX_VALUE, 0.999, 0.0));
+    }
+
+    @Test
+    public void confidenceLevelZero()
+    {
+        // By setting confidenceLevel to 0.0 we say we don't care about failures
+        assertEquals(0, softLimit(0, 0.0, 0.5));
+        assertEquals(1, softLimit(1, 0.0, 0.5));
+        assertEquals(100, softLimit(100, 0.0, 0.5));
+        assertEquals(1000000, softLimit(1000000, 0.0, 0.5));
+        assertEquals(Integer.MAX_VALUE, softLimit(Integer.MAX_VALUE, 0.0, 0.5));
+    }
+
+    @Test
+    public void confidenceLevelHalf()
+    {
+        assertEquals(0, softLimit(0, 0.5, 0.5));
+        assertEquals(1, softLimit(1, 0.5, 0.5));
+        // For large enough numbers, the number of items we need to query should be very close to targetLimit / perItemProbability.
+        assertEquals(2000000, softLimit(1000000, 0.5, 0.5), 2);
+        assertEquals(5000000, softLimit(1000000, 0.5, 0.2), 5);
+        assertEquals(10000000, softLimit(1000000, 0.5, 0.1), 10);
+        assertEquals(100000000, softLimit(1000000, 0.5, 0.01), 100);
+        assertEquals(1000000000, softLimit(1000000, 0.5, 0.001), 1000);
+        assertEquals(Integer.MAX_VALUE, softLimit(Integer.MAX_VALUE, 0.5, 0.5));
+    }
+
+    @Test
+    public void confidenceLevelNearOne()
+    {
+        assertEquals(0, softLimit(0, 0.999, 0.5));
+        assertEquals(10, softLimit(1, 0.999, 0.5)); // 1.0 - 0.5 ^ 10 is a tad more than 0.999
+
+        // Intuition-driven tests.
+        // We need to query a bit more than targetLimit / perItemProbability to get good confidence.
+        // The higher the targetLimit, the relatively closer to targetLimit / perItemProbability the softLimit is.
+
+        assertTrue(softLimit(10, 0.999, 0.5) >= 30);
+        assertTrue(softLimit(10, 0.999, 0.5) <= 50);
+
+        assertTrue(softLimit(10, 0.999, 0.1) >= 200);
+        assertTrue(softLimit(10, 0.999, 0.1) <= 300);
+
+        assertTrue(softLimit(100, 0.999, 0.5) >= 220);
+        assertTrue(softLimit(100, 0.999, 0.5) <= 250);
+
+        assertTrue(softLimit(100, 0.999, 0.1) >= 1300);
+        assertTrue(softLimit(100, 0.999, 0.1) <= 1500);
+
+        assertTrue(softLimit(100, 0.999, 0.001) >= 130000);
+        assertTrue(softLimit(100, 0.999, 0.001) <= 150000);
+
+        assertTrue(softLimit(1000, 0.999, 0.1) >= 10500);
+        assertTrue(softLimit(1000, 0.999, 0.1) <= 11500);
+
+        assertTrue(softLimit(1000, 0.999, 0.001) >= 1100000);
+        assertTrue(softLimit(1000, 0.999, 0.001) <= 1150000);
+    }
+
+    @Test
+    public void resultIsNonDecreasingWithConfidence()
+    {
+        for (int i = 0; i < 1000; i++)
+        {
+            int n = (int) Math.pow(10.0, 6 * random.nextDouble());
+            double c1 = random.nextDouble();
+            double c2 = random.nextDouble();
+            double p = random.nextDouble();
+            int l1 = softLimit(n, c1, p);
+            int l2 = softLimit(n, c2, p);
+            assertTrue("n: " + n + ", p: " + p + ", l1: " + l1 + ", l2: " + l2 + ", c1: " + c1 + ", c2: " + c2,
+                       (l1 > l2) == (c1 > c2) || l1 == l2);
+        }
+    }
+
+    @Test
+    public void resultIsNonIncreasingWithPerItemProbability()
+    {
+        for (int i = 0; i < 1000; i++)
+        {
+            int n = (int) Math.pow(10.0, 6 * random.nextDouble());
+            double c = random.nextDouble();         // [0.0, 1.0)
+            double p1 = 1.0 - random.nextDouble();  // (0.0, 1.0]
+            double p2 = 1.0 - random.nextDouble();  // (0.0, 1.0]
+            int l1 = softLimit(n, c, p1);
+            int l2 = softLimit(n, c, p2);
+            assertTrue("n: " + n + ", c: " + c + ", l1: " + l1 + ", l2: " + l2 + ", p1: " + p1 + ", p2: " + p2,
+                       (l1 > l2) == (p1 < p2) || l1 == l2);
+        }
+    }
+
+    @Test
+    public void randomized()
+    {
+        for (int i = 0; i < 1000; i++)
+        {
+            int n = (int) Math.pow(10.0, 6 * random.nextDouble());
+            double c = random.nextDouble();         // [0.0, 1.0)
+            double p = 1.0 - random.nextDouble();   // (0.0, 1.0]
+            int softLimit = softLimit(n, c, p);
+            assertTrue(softLimit >= n);
+
+            // Estimate the probability we get at least n successes after softLimit tries with probability p.
+            // Make sure we have enough confidence we get at least n, but not enough we get n + 1.
+            // softLimit is capped at Integer.MAX_VALUE. If it's potentially capped, we can't compare to the confidence
+            // bounds from the binomial distribution.
+            if (softLimit < Integer.MAX_VALUE)
+            {
+                BinomialDistribution successDistribution = new BinomialDistribution(softLimit, p);
+                double confidenceLowerBound = probabilityOfAtLeastNSuccesses(successDistribution, n + 1);
+                double confidenceUpperBound = probabilityOfAtLeastNSuccesses(successDistribution, n);
+                assertTrue(c + " lower than required lower bound " + confidenceLowerBound, c >= confidenceLowerBound);
+                assertTrue(c + " higher than required upper bound " + confidenceUpperBound, c < confidenceUpperBound);
+            }
+        }
+    }
+
+    private double probabilityOfAtLeastNSuccesses(BinomialDistribution distribution, int n)
+    {
+        return n > 0
+               ? 1.0 - distribution.cumulativeProbability(n - 1)
+               : 1.0;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/SuppressLeakCheck.java b/test/unit/org/apache/cassandra/index/sai/utils/SuppressLeakCheck.java
new file mode 100644
index 000000000000..4b106568024c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/SuppressLeakCheck.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SuppressLeakCheck
+{
+    /**
+     * The JIRA representing the reason for disabling leak checking
+     */
+    String jira() default "";
+
+    /**
+     * The reason for suppression if it doesn't warrant a jira
+     */
+    String reason() default "";
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/TokenAwarePrimaryKeyTest.java b/test/unit/org/apache/cassandra/index/sai/utils/TokenAwarePrimaryKeyTest.java
new file mode 100644
index 000000000000..1cca52eca966
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/TokenAwarePrimaryKeyTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.disk.v1.PartitionAwarePrimaryKeyFactory;
+
+public class TokenAwarePrimaryKeyTest extends AbstractPrimaryKeyTest
+{
+    @Test
+    public void simplePartitionTokenAwareTest() throws Throwable
+    {
+        PrimaryKey.Factory factory = new PartitionAwarePrimaryKeyFactory();
+
+        PrimaryKey first = factory.createTokenOnly(makeKey(simplePartition, "1").getToken());
+        PrimaryKey firstToken = factory.createTokenOnly(first.token());
+        PrimaryKey second = factory.createTokenOnly(makeKey(simplePartition, "2").getToken());
+        PrimaryKey secondToken = factory.createTokenOnly(second.token());
+
+        assertCompareToAndEquals(first, second, -1);
+        assertCompareToAndEquals(second, first, 1);
+        assertCompareToAndEquals(first, first, 0);
+        assertCompareToAndEquals(first, firstToken, 0);
+        assertCompareToAndEquals(firstToken, secondToken, -1);
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/TreeFormatterTest.java b/test/unit/org/apache/cassandra/index/sai/utils/TreeFormatterTest.java
new file mode 100644
index 000000000000..250745c0ee6b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/utils/TreeFormatterTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TreeFormatterTest
+{
+    static class TreeNode
+    {
+        final String label;
+        final List<TreeNode> children;
+
+        TreeNode(String label, List<TreeNode> children)
+        {
+            this.label = label;
+            this.children = children;
+        }
+    }
+
+    @Test
+    public void formatTree()
+    {
+        TreeNode root = new TreeNode("root", List.of(
+            new TreeNode("child 1", List.of(
+                new TreeNode("child 1a", Collections.emptyList()),
+                new TreeNode("child 1b", Collections.emptyList()))),
+            new TreeNode("child 2", List.of(
+                new TreeNode("child 2a", Collections.emptyList()),
+                new TreeNode("child 2b", Collections.emptyList())))));
+
+        TreeFormatter<TreeNode> formatter = new TreeFormatter<>(t -> t.label, t -> t.children);
+        String formattedTree = formatter.format(root);
+
+        assertEquals("root\n" +
+                     " ├─ child 1\n" +
+                     " │   ├─ child 1a\n" +
+                     " │   └─ child 1b\n" +
+                     " └─ child 2\n" +
+                     "     ├─ child 2a\n" +
+                     "     └─ child 2b\n", formattedTree);
+    }
+
+    @Test
+    public void formatTreeWithMultiLineNodes() {
+        TreeNode root = new TreeNode("root line 1\nroot line 2", List.of(
+        new TreeNode("child 1\nchild 1 line 2", List.of(
+            new TreeNode("child 1a", Collections.emptyList()),
+            new TreeNode("child 1b", Collections.emptyList()))),
+        new TreeNode("child 2\nchild 2 line 2", Collections.emptyList())));
+
+        TreeFormatter<TreeNode> formatter = new TreeFormatter<>(t -> t.label, t -> t.children);
+        String formattedTree = formatter.format(root);
+
+        assertEquals("root line 1\n" +
+                     "root line 2\n" +
+                     " ├─ child 1\n" +
+                     " │  child 1 line 2\n" +
+                     " │   ├─ child 1a\n" +
+                     " │   └─ child 1b\n" +
+                     " └─ child 2\n" +
+                     "    child 2 line 2\n", formattedTree);
+    }
+
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/sai/view/IndexViewManagerTest.java b/test/unit/org/apache/cassandra/index/sai/view/IndexViewManagerTest.java
new file mode 100644
index 000000000000..21c790623cfd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/view/IndexViewManagerTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.view;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.SSTableIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexComponents;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+public class IndexViewManagerTest extends SAITester
+{
+    private static final int CONCURRENT_UPDATES = 100;
+
+    @BeforeClass
+    public static void setupVersionBarrier()
+    {
+        requireNetwork();
+    }
+
+    @Test
+    public void testUpdateFromFlush()
+    {
+        createTable("CREATE TABLE %S (k INT PRIMARY KEY, v INT)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        IndexContext columnContext = columnIndex(getCurrentColumnFamilyStore(), indexName);
+        View initialView = columnContext.getView();
+
+        execute("INSERT INTO %s(k, v) VALUES (1, 10)");
+        execute("INSERT INTO %s(k, v) VALUES (2, 20)");
+        flush();
+
+        View updatedView = columnContext.getView();
+        assertNotEquals(initialView, updatedView);
+        assertEquals(1, updatedView.getIndexes().size());
+    }
+
+    @Test
+    public void testUpdateFromCompaction()
+    {
+        createTable("CREATE TABLE %S (k INT PRIMARY KEY, v INT)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        ColumnFamilyStore store = getCurrentColumnFamilyStore();
+        IndexContext columnContext = columnIndex(store, indexName);
+        store.disableAutoCompaction();
+
+        execute("INSERT INTO %s(k, v) VALUES (1, 10)");
+        execute("INSERT INTO %s(k, v) VALUES (2, 20)");
+        execute("INSERT INTO %s(k, v) VALUES (3, 30)");
+        flush();
+
+        execute("INSERT INTO %s(k, v) VALUES (4, 40)");
+        execute("INSERT INTO %s(k, v) VALUES (5, 50)");
+        execute("INSERT INTO %s(k, v) VALUES (6, 60)");
+        flush();
+
+        View initialView = columnContext.getView();
+        assertEquals(2, initialView.getIndexes().size());
+
+        CompactionManager.instance.performMaximal(store, false);
+
+        View updatedView = columnContext.getView();
+        assertNotEquals(initialView, updatedView);
+        assertEquals(1, updatedView.getIndexes().size());
+    }
+
+    /**
+     * Tests concurrent sstable updates from flush and compaction, see CASSANDRA-14207.
+     */
+    @Test
+    public void testConcurrentUpdate() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %S (k INT PRIMARY KEY, v INT)");
+        String indexName = createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
+
+        ColumnFamilyStore store = getCurrentColumnFamilyStore();
+        IndexContext columnContext = columnIndex(store, indexName);
+        Path tmpDir = Files.createTempDirectory("IndexViewManagerTest");
+        store.disableAutoCompaction();
+
+        List<Descriptor> descriptors = new ArrayList<>();
+
+        // create sstable 1 from flush
+        execute("INSERT INTO %s(k, v) VALUES (1, 10)");
+        execute("INSERT INTO %s(k, v) VALUES (2, 20)");
+        execute("INSERT INTO %s(k, v) VALUES (3, 30)");
+        flush();
+
+        // create sstable 2 from flush
+        execute("INSERT INTO %s(k, v) VALUES (4, 40)");
+        execute("INSERT INTO %s(k, v) VALUES (5, 50)");
+        execute("INSERT INTO %s(k, v) VALUES (6, 60)");
+        flush();
+
+        // save sstables 1 and 2 and create sstable 3 from compaction
+        assertEquals(2, store.getLiveSSTables().size());
+        store.getLiveSSTables().forEach(reader -> copySSTable(reader, tmpDir));
+        getCurrentColumnFamilyStore().getLiveSSTables().stream().map(t -> t.descriptor).forEach(descriptors::add);
+        CompactionManager.instance.performMaximal(store, false);
+
+        // create sstable 4 from flush
+        execute("INSERT INTO %s(k, v) VALUES (5, 50)");
+        execute("INSERT INTO %s(k, v) VALUES (6, 60)");
+        flush();
+
+        // save sstables 3 and 4
+        store.getLiveSSTables().forEach(reader -> copySSTable(reader, tmpDir));
+        getCurrentColumnFamilyStore().getLiveSSTables().stream().map(t -> t.descriptor).forEach(descriptors::add);
+
+        List<SSTableReader> sstables = descriptors.stream()
+                                                .map(desc -> new Descriptor(new File(tmpDir), KEYSPACE, tableName, desc.id))
+                                                .map(desc -> SSTableReader.open(store, desc))
+                                                .collect(Collectors.toList());
+
+        assertThat(sstables).hasSize(4);
+
+        List<SSTableReader> none = Collections.emptyList();
+        List<SSTableReader> initial = sstables.stream().limit(2).collect(Collectors.toList());
+
+        ExecutorService executor = Executors.newFixedThreadPool(2);
+        for (int i = 0; i < CONCURRENT_UPDATES; i++)
+        {
+            // mock the initial view indexes to track the number of releases
+            List<SSTableContext> initialContexts = sstables.stream().limit(2).map(s -> SSTableContext.create(s, loadDescriptor(s, store).perSSTableComponents())).collect(Collectors.toList());
+            List<SSTableIndex> initialIndexes = new ArrayList<>();
+
+            for (SSTableContext initialContext : initialContexts)
+            {
+                MockSSTableIndex mockSSTableIndex = new MockSSTableIndex(initialContext, initialContext.usedPerSSTableComponents().indexDescriptor().perIndexComponents(columnContext));
+                initialIndexes.add(mockSSTableIndex);
+            }
+
+            IndexViewManager tracker = new IndexViewManager(columnContext, descriptors, initialIndexes);
+            View initialView = tracker.getView();
+            assertEquals(2, initialView.size());
+
+            List<SSTableReader> compacted = List.of(sstables.get(2));
+            List<SSTableReader> flushed = List.of(sstables.get(3));
+
+            List<SSTableContext> compactedContexts = compacted.stream().map(s -> SSTableContext.create(s, loadDescriptor(s, store).perSSTableComponents())).collect(Collectors.toList());
+            List<SSTableContext> flushedContexts = flushed.stream().map(s -> SSTableContext.create(s, loadDescriptor(s, store).perSSTableComponents())).collect(Collectors.toList());
+
+            // concurrently update from both flush and compaction
+            Future<?> compaction = executor.submit(() -> tracker.update(initial, compacted, compactedContexts, true));
+            Future<?> flush = executor.submit(() -> tracker.update(none, flushed, flushedContexts, true));
+
+            FBUtilities.waitOnFutures(Arrays.asList(compaction, flush));
+
+            View updatedView = tracker.getView();
+            assertNotEquals(initialView, updatedView);
+            assertEquals(2, updatedView.getIndexes().size());
+
+            for (SSTableIndex index : initialIndexes)
+            {
+                assertEquals(1, ((MockSSTableIndex) index).releaseCount);
+            }
+
+            // release original SSTableContext objects.
+            // shared copies are already released when compacted and flushed are added.
+            initialContexts.forEach(SSTableContext::close);
+            initialContexts.forEach(group -> assertTrue(group.isCleanedUp()));
+
+            // release compacted and flushed SSTableContext original and shared copies
+            compactedContexts.forEach(SSTableContext::close);
+            flushedContexts.forEach(SSTableContext::close);
+            tracker.getView().getIndexes().forEach(SSTableIndex::release);
+            compactedContexts.forEach(group -> assertTrue(group.isCleanedUp()));
+            flushedContexts.forEach(group -> assertTrue(group.isCleanedUp()));
+        }
+        sstables.forEach(sstable -> sstable.selfRef().release());
+        executor.shutdown();
+        executor.awaitTermination(1, TimeUnit.MINUTES);
+    }
+
+    private IndexContext columnIndex(ColumnFamilyStore store, String indexName)
+    {
+        assert store.indexManager != null;
+        StorageAttachedIndex sai = (StorageAttachedIndex) store.indexManager.getIndexByName(indexName);
+        return sai.getIndexContext();
+    }
+
+    public static class MockSSTableIndex extends SSTableIndex
+    {
+        int releaseCount = 0;
+
+        MockSSTableIndex(SSTableContext group, IndexComponents.ForRead perIndexComponents) throws IOException
+        {
+            super(group, perIndexComponents);
+        }
+
+        @Override
+        public void release()
+        {
+            super.release();
+            releaseCount++;
+        }
+    }
+
+    private static void copySSTable(SSTableReader table, Path destDir)
+    {
+        for (Component component : table.components())
+        {
+            Path src = table.descriptor.fileFor(component).toPath();
+            Path dst = destDir.resolve(src.getFileName());
+            try
+            {
+                Files.copy(src, dst, StandardCopyOption.REPLACE_EXISTING);
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
index d52d1d5395c3..2d333d6e5a78 100644
--- a/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
@@ -18,15 +18,19 @@
 package org.apache.cassandra.index.sai.virtual;
 
 import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.Util;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.virtual.VirtualKeyspace;
 import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
 import org.apache.cassandra.inject.Injections;
 import org.apache.cassandra.inject.InvokePointBuilder;
 import org.apache.cassandra.schema.SchemaConstants;
@@ -36,29 +40,33 @@
  */
 public class IndexesSystemViewTest extends SAITester
 {
-    private static final String SELECT = String.format("SELECT %s, %s, %s, %s, %s, %s, %s FROM %s.%s WHERE %s = '%s'",
-                                                       ColumnIndexesSystemView.INDEX_NAME,
-                                                       ColumnIndexesSystemView.TABLE_NAME,
-                                                       ColumnIndexesSystemView.COLUMN_NAME,
-                                                       ColumnIndexesSystemView.IS_QUERYABLE,
-                                                       ColumnIndexesSystemView.IS_BUILDING,
-                                                       ColumnIndexesSystemView.IS_STRING,
-                                                       ColumnIndexesSystemView.ANALYZER,
+    private static final String SELECT = String.format("SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s FROM %s.%s WHERE %s = '%s'",
+                                                       IndexesSystemView.INDEX_NAME,
+                                                       IndexesSystemView.TABLE_NAME,
+                                                       IndexesSystemView.COLUMN_NAME,
+                                                       IndexesSystemView.IS_QUERYABLE,
+                                                       IndexesSystemView.IS_BUILDING,
+                                                       IndexesSystemView.IS_STRING,
+                                                       IndexesSystemView.ANALYZER,
+                                                       IndexesSystemView.INDEXED_SSTABLE_COUNT,
+                                                       IndexesSystemView.CELL_COUNT,
+                                                       IndexesSystemView.PER_TABLE_DISK_SIZE,
+                                                       IndexesSystemView.PER_COLUMN_DISK_SIZE,
                                                        SchemaConstants.VIRTUAL_VIEWS,
-                                                       ColumnIndexesSystemView.NAME,
-                                                       ColumnIndexesSystemView.KEYSPACE_NAME,
+                                                       IndexesSystemView.NAME,
+                                                       IndexesSystemView.KEYSPACE_NAME,
                                                        KEYSPACE);
 
-    private static final Injections.Barrier blockIndexBuild = Injections.newBarrier("block_index_build", 2, false)
-                                                                        .add(InvokePointBuilder.newInvokePoint()
-                                                                                               .onClass(StorageAttachedIndex.class)
-                                                                                               .onMethod("startInitialBuild"))
-                                                                        .build();
+    private final Injections.Barrier blockIndexBuild = Injections.newBarrier("block_index_build", 2, false)
+                                                                 .add(InvokePointBuilder.newInvokePoint()
+                                                                                        .onClass(StorageAttachedIndex.class)
+                                                                                        .onMethod("startInitialBuild"))
+                                                                 .build();
 
     @BeforeClass
-    public static void setup()
+    public static void setup() throws Exception
     {
-        VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(SchemaConstants.VIRTUAL_VIEWS, ImmutableList.of(new ColumnIndexesSystemView(SchemaConstants.VIRTUAL_VIEWS))));
+        VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(SchemaConstants.VIRTUAL_VIEWS, ImmutableList.of(new IndexesSystemView(SchemaConstants.VIRTUAL_VIEWS))));
 
         CQLTester.setUpClass();
     }
@@ -68,44 +76,62 @@ public void testVirtualTableThroughIndexLifeCycle() throws Throwable
     {
         // create the table and verify that the virtual table is empty before creating any indexes
         assertEmpty(execute(SELECT));
-        createTable("CREATE TABLE %s (k int, c int, v1 text, PRIMARY KEY (k, c))");
+        createTable("CREATE TABLE %s (k int, c int, v1 int, v2 text, PRIMARY KEY (k, c))");
 
         // create the index simulating a long build and verify that there is an empty record in the virtual table
         Injections.inject(blockIndexBuild);
         String v1IndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
 
-        assertRows(execute(SELECT), row(v1IndexName, "v1", false, true, true));
+        assertRows(execute(SELECT), row(v1IndexName, "v1", false, true, false, 0, 0L));
 
-        // unblock the long build and verify that there is a finished empty record in the virtual table
+        // unblock the long build and verify that there is an finished empty record in the virtual table
         blockIndexBuild.countDown();
         blockIndexBuild.disable();
-        waitForTableIndexesQueryable();
-        assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, true));
+        waitForIndexQueryable(v1IndexName);
+        assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, false, 0, 0L));
 
         // insert some data and verify that virtual table record is still empty since we haven't flushed yet
-        execute("INSERT INTO %s(k, c, v1) VALUES (?, ?, ?)", 1, 10, "1000");
-        execute("INSERT INTO %s(k, c, v1) VALUES (?, ?, ?)", 2, 20, "2000");
-        assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, true));
+        execute("INSERT INTO %s(k, c, v1, v2) VALUES (?, ?, ?, ?)", 1, 10, 100, "1000");
+        execute("INSERT INTO %s(k, c, v1, v2) VALUES (?, ?, ?, ?)", 2, 20, 200, "2000");
+        assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, false, 0, 0L));
 
         // flush the memtable and verify the not-empty record in the virtual table
         flush();
-        assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, true));
+        assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, false, 1, 2L));
 
         // flush a second memtable and verify the updated record in the virtual table
-        execute("INSERT INTO %s(k, c, v1) VALUES (?, ?, ?)", 3, 30, "3000");
+        execute("INSERT INTO %s(k, c, v1, v2) VALUES (?, ?, ?, ?)", 3, 30, 300, "3000");
         flush();
-        assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, true));
+        assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, false, 2, 3L));
+
+        // create a second index, this should create a new additional entry in the table
+        String v2IndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
+        assertRows(execute(SELECT),
+                   row(v1IndexName, "v1", true, false, false, 2, 3L),
+                   row(v2IndexName, "v2", true, false, true, 2, 3L));
+
+        // update some of the existing rows, this should increase the cell count due to the multiple versions
+        execute("INSERT INTO %s(k, c, v1, v2) VALUES (?, ?, ?, ?)", 1, 10, 111, "1111");
+        execute("INSERT INTO %s(k, c, v1, v2) VALUES (?, ?, ?, ?)", 2, 20, 222, "2222");
+        flush();
+        assertRowsIgnoringOrderAndExtra(execute(SELECT),
+                                        row(v1IndexName, "v1", true, false, false, 3, 5L),
+                                        row(v2IndexName, "v2", true, false, true, 3, 5L));
 
         // compact and verify that the cell count decreases
-        compact();
-        waitForCompactionsFinished();
+        Util.forceFullCompaction(getCurrentColumnFamilyStore(), 30);
 
         assertRowsIgnoringOrderAndExtra(execute(SELECT),
-                                        row(v1IndexName, "v1", true, false, true));
+                                        row(v1IndexName, "v1", true, false, false, 1, 3L),
+                                        row(v2IndexName, "v2", true, false, true, 1, 3L));
+
+        // drop the second index and verify that there is not entry for it in the virtual table
+        dropIndex("DROP INDEX %s." + v2IndexName);
+        assertRowsIgnoringOrderAndExtra(execute(SELECT), row(v1IndexName, "v1", true, false, false, 1, 3L));
 
-        // truncate the base table and verify that there is still an entry in the virtual table, and it's empty
+        // truncate the base table and verify that there is still an entry in the virtual table and it's empty
         truncate(false);
-        assertRowsIgnoringOrderAndExtra(execute(SELECT), row(v1IndexName, "v1", true, false, true));
+        assertRowsIgnoringOrderAndExtra(execute(SELECT), row(v1IndexName, "v1", true, false, false, 0, 0L));
 
         // drop the base table and verify that the virtual table is empty
         dropTable("DROP TABLE %s");
@@ -116,10 +142,15 @@ private Object[] row(String indexName,
                          String columnName,
                          boolean isQueryable,
                          boolean isBuilding,
-                         boolean isString)
+                         boolean isString,
+                         int sstableCount,
+                         long cellCount)
     {
             ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+            StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+            Assert.assertNotNull(group);
             StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName);
+            IndexContext context = sai.getIndexContext();
 
             return row(indexName,
                        currentTable(),
@@ -127,6 +158,10 @@ private Object[] row(String indexName,
                        isQueryable,
                        isBuilding,
                        isString,
-                       sai.hasAnalyzer() ? sai.analyzer().toString() : "NoOpAnalyzer");
+                       context.getAnalyzerFactory().toString(),
+                       sstableCount,
+                       cellCount,
+                       group.diskUsage(),
+                       context.diskUsage());
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java
index a308d806d06c..fb3e79b64d19 100644
--- a/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java
@@ -20,26 +20,27 @@
 import java.util.Objects;
 
 import com.google.common.collect.ImmutableList;
-import com.googlecode.concurrenttrees.common.Iterables;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.cassandra.Util;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.virtual.VirtualKeyspace;
 import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.sai.SAITester;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.SSTableIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.io.CryptoUtils;
 import org.apache.cassandra.io.sstable.SSTableId;
 import org.apache.cassandra.io.sstable.SSTableIdFactory;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.CompressionParams;
 import org.apache.cassandra.schema.SchemaConstants;
 
+import static org.apache.cassandra.Util.assertSSTableIds;
+
 /**
  * Tests the virtual table exposing SSTable index metadata.
  */
@@ -47,37 +48,36 @@ public class SSTablesSystemViewTest extends SAITester
 {
     private static final String SELECT = String.format("SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s " +
                                                        "FROM %s.%s WHERE %s = '%s'",
-                                                       SSTableIndexesSystemView.INDEX_NAME,
-                                                       SSTableIndexesSystemView.SSTABLE_NAME,
-                                                       SSTableIndexesSystemView.TABLE_NAME,
-                                                       SSTableIndexesSystemView.COLUMN_NAME,
-                                                       SSTableIndexesSystemView.FORMAT_VERSION,
-                                                       SSTableIndexesSystemView.CELL_COUNT,
-                                                       SSTableIndexesSystemView.MIN_ROW_ID,
-                                                       SSTableIndexesSystemView.MAX_ROW_ID,
-                                                       SSTableIndexesSystemView.START_TOKEN,
-                                                       SSTableIndexesSystemView.END_TOKEN,
-                                                       SSTableIndexesSystemView.PER_TABLE_DISK_SIZE,
-                                                       SSTableIndexesSystemView.PER_COLUMN_DISK_SIZE,
+                                                       SSTablesSystemView.INDEX_NAME,
+                                                       SSTablesSystemView.SSTABLE_NAME,
+                                                       SSTablesSystemView.TABLE_NAME,
+                                                       SSTablesSystemView.COLUMN_NAME,
+                                                       SSTablesSystemView.FORMAT_VERSION,
+                                                       SSTablesSystemView.CELL_COUNT,
+                                                       SSTablesSystemView.MIN_ROW_ID,
+                                                       SSTablesSystemView.MAX_ROW_ID,
+                                                       SSTablesSystemView.START_TOKEN,
+                                                       SSTablesSystemView.END_TOKEN,
+                                                       SSTablesSystemView.PER_TABLE_DISK_SIZE,
+                                                       SSTablesSystemView.PER_COLUMN_DISK_SIZE,
                                                        SchemaConstants.VIRTUAL_VIEWS,
-                                                       SSTableIndexesSystemView.NAME,
-                                                       SSTableIndexesSystemView.KEYSPACE_NAME,
+                                                       SSTablesSystemView.NAME,
+                                                       SSTablesSystemView.KEYSPACE_NAME,
                                                        KEYSPACE);
 
     @BeforeClass
-    public static void setup()
+    public static void setup() throws Exception
     {
-        VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(SchemaConstants.VIRTUAL_VIEWS, ImmutableList.of(new SSTableIndexesSystemView(SchemaConstants.VIRTUAL_VIEWS))));
+        VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(SchemaConstants.VIRTUAL_VIEWS, ImmutableList.of(new SSTablesSystemView(SchemaConstants.VIRTUAL_VIEWS))));
 
         CQLTester.setUpClass();
     }
 
     @Test
-    public void testVirtualTableThroughIndexLifeCycle() throws Throwable
+    public void testVirtualTableThroughIndexLifeCycle()
     {
-        createTable("CREATE TABLE %s (k text, c text, v1 text, v2 text, PRIMARY KEY (k, c))");
-        disableCompaction();
-        String v1IndexName = createIndex("CREATE INDEX ON %s(v1) USING 'sai'");
+        createTable("CREATE TABLE %s (k int, c int, v1 int, v2 int, PRIMARY KEY (k, c))");
+        String v1IndexName = createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'StorageAttachedIndex'");
 
         String insert = "INSERT INTO %s(k, c, v1, v2) VALUES (?, ?, ?, ?)";
 
@@ -85,61 +85,63 @@ public void testVirtualTableThroughIndexLifeCycle() throws Throwable
         assertEmpty(execute(SELECT));
 
         // insert a row and verify that the virtual table is empty before flushing
-        execute(insert, "1", "10", "100", "1000");
+        execute(insert, 1, 10, 100, 1000);
         assertEmpty(execute(SELECT));
 
         // flush the memtable and verify the new record in the virtual table
         flush();
         SSTableId id1 = currentIdsSorted()[0];
         Object[] row1 = readRow(v1IndexName, id1, "v1", 1L, 0L, 0L);
-        assertRowsIgnoringOrder(execute(SELECT), row1);
+        assertRows(execute(SELECT), row1);
 
         // flush a second memtable and verify both the old and the new record in the virtual table
-        execute(insert, "2", "20", "200", "2000");
-        execute(insert, "3", "30", "300", "3000");
+        execute(insert, 2, 20, 200, 2000);
+        execute(insert, 3, 30, 300, 3000);
         flush();
         SSTableId id2 = currentIdsSorted()[1];
+        assertSSTableIds(id2, id1, r -> r > 0);
         Object[] row2 = readRow(v1IndexName, id2, "v1", 2L, 0L, 1L);
-        assertRowsIgnoringOrder(execute(SELECT), row1, row2);
+        assertRows(execute(SELECT), row1, row2);
 
         // create a second index, this should create a new additional entry in the table for each sstable
-        String v2IndexName = createIndex("CREATE INDEX ON %s(v2) USING 'sai'");
+        String v2IndexName = createIndex("CREATE CUSTOM INDEX ON %s(v2) USING 'StorageAttachedIndex'");
         Object[] row3 = readRow(v2IndexName, id1, "v2", 1L, 0L, 0L);
         Object[] row4 = readRow(v2IndexName, id2, "v2", 2L, 0L, 1L);
-        assertRowsIgnoringOrder(execute(SELECT), row1, row2, row3, row4);
+        assertRows(execute(SELECT), row1, row2, row3, row4);
 
         // create a new sstable that only contains data for the second index, this should add only one new entry
-        execute(insert, "4", "40", null, "4000");
+        execute(insert, 4, 40, null, 4000);
         flush();
         SSTableId id3 = currentIdsSorted()[2];
+        assertSSTableIds(id3, id2, r -> r > 0);
         Object[] row5 = readRow(v2IndexName, id3, "v2", 1L, 0L, 0L);
-        assertRowsIgnoringOrder(execute(SELECT), row1, row2, row3, row4, row5);
+        assertRows(execute(SELECT), row1, row2, row3, row4, row5);
 
         // create a new sstable with rows with contents for either one of the indexes or the other
-        execute(insert, "5", "50", "500", null);
-        execute(insert, "6", "60", null, "6000");
+        execute(insert, 5, 50, 500, null);
+        execute(insert, 6, 60, null, 6000);
         flush();
         SSTableId id4 = currentIdsSorted()[3];
-        Object[] row6 = readRow(v1IndexName, id4, "v1", 1L, 1L, 1L);
-        Object[] row7 = readRow(v2IndexName, id4, "v2", 1L, 0L, 0L);
-        assertRowsIgnoringOrder(execute(SELECT), row1, row2, row6, row3, row4, row5, row7);
+        assertSSTableIds(id4, id3, r -> r > 0);
+        Object[] row6 = readRow(v1IndexName, id4, "v1", 1L, 0L, 0L);
+        Object[] row7 = readRow(v2IndexName, id4, "v2", 1L, 1L, 1L);
+        assertRows(execute(SELECT), row1, row2, row6, row3, row4, row5, row7);
 
         // compact the table and verify that the virtual table has a single entry per index
-        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
-        Util.compact(cfs, Iterables.toList(cfs.getSSTables(SSTableSet.LIVE)));
+        compact();
         waitForCompactions();
-
         SSTableId[] ids5 = currentIdsSorted();
+        assertSSTableIds(ids5[0], id4, r -> r > 0);
         // Compaction may result in sstables with generation 5 or 6. Try both.
         // key 4, key 6 are not indexable on v1
-        Object[] row8 = readRow(v1IndexName, ids5, "v1", 4L, 2L, 5L);
+        Object[] row8 = readRow(v1IndexName, ids5, "v1", 4L, 0L, 5L);
         // key 5 is not indexable on v2
-        Object[] row9 = readRow(v2IndexName, ids5, "v2", 5L, 0L, 5L);
-        assertRowsIgnoringOrder(execute(SELECT), row8, row9);
+        Object[] row9 = readRow(v2IndexName, ids5, "v2", 5L, 1L, 5L);
+        assertRows(execute(SELECT), row8, row9);
 
         // drop the first index and verify that there are not entries for it in the table
         dropIndex("DROP INDEX %s." + v1IndexName);
-        assertRowsIgnoringOrder(execute(SELECT), row9);
+        assertRows(execute(SELECT), row9);
 
         // drop the base table and verify that the virtual table is empty
         dropTable("DROP TABLE %s");
@@ -177,7 +179,7 @@ private Object[] readRow(String indexName,
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
         StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName);
 
-        for (SSTableIndex sstableIndex : sai.view())
+        for (SSTableIndex sstableIndex : sai.getIndexContext().getView())
         {
             SSTableReader sstable = sstableIndex.getSSTable();
 
@@ -186,6 +188,8 @@ private Object[] readRow(String indexName,
                 Token.TokenFactory tokenFactory = cfs.metadata().partitioner.getTokenFactory();
                 AbstractBounds<Token> bounds = sstable.getBounds();
 
+                CompressionParams params = CryptoUtils.getCompressionParams(sstable);
+
                 return row(indexName,
                            sstable.getFilename(),
                            currentTable(),
@@ -196,7 +200,7 @@ private Object[] readRow(String indexName,
                            maxSSTableRowId,
                            tokenFactory.toString(bounds.left),
                            tokenFactory.toString(bounds.right),
-                           sstableIndex.getSSTableContext().diskUsage(),
+                           sstableIndex.sizeOfPerSSTableComponents(),
                            sstableIndex.sizeOfPerColumnComponents());
             }
         }
diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
index d6fcb132ca76..402ff5cd4440 100644
--- a/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
@@ -17,10 +17,8 @@
  */
 package org.apache.cassandra.index.sai.virtual;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import com.google.common.collect.ImmutableList;
@@ -34,14 +32,15 @@
 import org.apache.cassandra.db.virtual.VirtualKeyspace;
 import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.SSTableIndex;
 import org.apache.cassandra.index.sai.StorageAttachedIndex;
-import org.apache.cassandra.index.sai.disk.SSTableIndex;
-import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexComponentType;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
-import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
+import org.apache.cassandra.index.sai.disk.v1.SegmentBuilder;
+import org.apache.cassandra.index.sai.disk.v1.SegmentMetadata;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.StorageService;
@@ -80,7 +79,7 @@ public class SegmentsSystemViewTest extends SAITester
                                                                       KEYSPACE);
 
     @BeforeClass
-    public static void setup()
+    public static void setup() throws Exception
     {
         VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(SchemaConstants.VIRTUAL_VIEWS, ImmutableList.of(new SegmentsSystemView(SchemaConstants.VIRTUAL_VIEWS))));
 
@@ -90,61 +89,56 @@ public static void setup()
     @Test
     public void testSegmentsMetadata() throws Throwable
     {
-        createTable("CREATE TABLE %s (k int, c int, v1 text, PRIMARY KEY (k, c))");
-        String literalIndex = createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'StorageAttachedIndex'");
+        createTable("CREATE TABLE %s (k int, c int, v1 int, v2 text, PRIMARY KEY (k, c))");
+        String numericIndex = createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'StorageAttachedIndex' WITH OPTIONS = {'enable_segment_compaction':true}");
+        String stringIndex = createIndex("CREATE CUSTOM INDEX ON %s(v2) USING 'StorageAttachedIndex' WITH OPTIONS = {'enable_segment_compaction':true}");
 
         int num = 100;
 
-        String insert = "INSERT INTO %s(k, c, v1) VALUES (?, ?, ?)";
+        String insert = "INSERT INTO %s(k, c, v1, v2) VALUES (?, ?, ?, ?)";
 
         // the virtual table should be empty before adding contents
-        assertEmpty(execute(SELECT, literalIndex));
+        assertEmpty(execute(SELECT, numericIndex));
+        assertEmpty(execute(SELECT, stringIndex));
 
         // insert rows and verify that the virtual table is empty before flushing
         for (int i = 0; i < num / 2; i++)
-            execute(insert, i, 10, "1000");
-        assertEmpty(execute(SELECT, literalIndex));
+            execute(insert, i, 10, 100, "1000");
+        assertEmpty(execute(SELECT, numericIndex));
+        assertEmpty(execute(SELECT, stringIndex));
 
         // flush the memtable and verify the new record in the virtual table
         flush();
         Object[] row1 = row(0L, (long)(num / 2), 0L, (long)(num / 2 - 1));
-        assertRows(execute(SELECT, literalIndex), row1);
+        assertRows(execute(SELECT, numericIndex), row1);
+        assertRows(execute(SELECT, stringIndex), row1);
 
         // flush a second memtable and verify both the old and the new record in the virtual table
         for (int i = num / 2; i < num; i++)
-            execute(insert, i, 20, "2000");
+            execute(insert, i, 20, 200, "2000");
         flush();
         Object[] row2 = row(0L, (long)(num / 2), 0L, (long)(num / 2 - 1));
-        assertRows(execute(SELECT, literalIndex), row1, row2);
+        assertRows(execute(SELECT, numericIndex), row1, row2);
+        assertRows(execute(SELECT, stringIndex), row1, row2);
 
         // force compaction, there is only 1 sstable
         compact();
         waitForCompactions();
         Object[] row3 = row(0L, (long)num, 0L, (long)(num - 1));
-        assertRows(execute(SELECT, literalIndex), row3);
+        assertRows(execute(SELECT, numericIndex), row3);
+        assertRows(execute(SELECT, stringIndex), row3);
 
         for (int lastValidSegmentRowId : Arrays.asList(0, 1, 2, 3, 5, 9, 25, 49, 59, 99, 101))
         {
             SegmentBuilder.updateLastValidSegmentRowId(lastValidSegmentRowId);
 
             // compaction to rewrite segments
-            StorageService.instance.upgradeSSTables(KEYSPACE, false, currentTable());
+            StorageService.instance.upgradeSSTables(KEYSPACE, false, new String[] { currentTable() });
+            // segment compaction is now disabled
+            int segmentCount = (int) Math.ceil(num * 1.0 / (lastValidSegmentRowId + 1));
+            assertRowCount(execute(SELECT, numericIndex), segmentCount);
+            assertRowCount(execute(SELECT, stringIndex), segmentCount);
 
-            List<Object[]> segmentRows = new ArrayList<>();
-
-            for (int row = 0; row < num / (lastValidSegmentRowId + 1); row++)
-                segmentRows.add(row((long)(row * (lastValidSegmentRowId + 1)),
-                                    (long)(lastValidSegmentRowId + 1),
-                                    (long)(row * (lastValidSegmentRowId + 1)),
-                                    (long)(row * (lastValidSegmentRowId + 1) + lastValidSegmentRowId)));
-            long prevMaxSSTableRowId = segmentRows.isEmpty() ? -1L : (long)segmentRows.get(segmentRows.size() - 1)[3];
-            if (prevMaxSSTableRowId < 99L)
-            {
-                segmentRows.add(row(prevMaxSSTableRowId + 1, 99 - prevMaxSSTableRowId, prevMaxSSTableRowId + 1, 99L));
-            }
-
-            UntypedResultSet resultSet = execute(SELECT, literalIndex);
-            assertRows(execute(SELECT, literalIndex), segmentRows.toArray(new Object[][]{}));
             // verify index metadata length
             Map<String, Long> indexLengths = new HashMap<>();
             for (UntypedResultSet.Row row : execute(SELECT_INDEX_METADATA))
@@ -164,6 +158,19 @@ public void testSegmentsMetadata() throws Throwable
                     final String indexType = entry.getKey();
                     final String str = entry.getValue().getOrDefault(SegmentMetadata.ComponentMetadata.LENGTH, "0");
 
+                    if (indexType.equals(IndexComponentType.KD_TREE.toString()))
+                    {
+                        int maxPointsInLeafNode = Integer.parseInt(entry.getValue().get("max_points_in_leaf_node"));
+
+                        assertEquals(1024, maxPointsInLeafNode);
+                    }
+                    else if (indexType.equals(IndexComponentType.KD_TREE_POSTING_LISTS.toString()))
+                    {
+                        int numLeafPostings = Integer.parseInt(entry.getValue().get("num_leaf_postings"));
+
+                        assertTrue(numLeafPostings > 0);
+                    }
+
                     final long length = Long.parseLong(str);
 
                     final long value = indexLengths.getOrDefault(indexType, 0L);
@@ -171,18 +178,21 @@ public void testSegmentsMetadata() throws Throwable
                 }
             }
             if (!TEST_ENCRYPTION.getBoolean())
-                assertEquals(indexFileLengths(), indexLengths);
+                assertEquals(indexFileLengths(currentTable()), indexLengths);
         }
 
         // drop the numeric index and verify that there are not entries for it in the table
-        assertNotEquals(0, execute(SELECT, literalIndex).size());
+        dropIndex("DROP INDEX %s." + numericIndex);
+        assertEmpty(execute(SELECT, numericIndex));
+        assertNotEquals(0, execute(SELECT, stringIndex).size());
 
         // drop the string index and verify that there are not entries for it in the table
-        dropIndex("DROP INDEX %s." +  literalIndex);
-        assertEmpty(execute(SELECT, literalIndex));
+        dropIndex("DROP INDEX %s." +  stringIndex);
+        assertEmpty(execute(SELECT, numericIndex));
+        assertEmpty(execute(SELECT, stringIndex));
     }
 
-    private HashMap<String, Long> indexFileLengths()
+    private HashMap<String, Long> indexFileLengths(String table) throws Exception
     {
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
 
@@ -191,17 +201,21 @@ private HashMap<String, Long> indexFileLengths()
         {
             StorageAttachedIndex index = (StorageAttachedIndex) idx;
 
-            for (SSTableIndex sstableIndex : index.view().getIndexes())
+            for (SSTableIndex sstableIndex : index.getIndexContext().getView().getIndexes())
             {
                 SSTableReader sstable = sstableIndex.getSSTable();
 
-                IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
-                indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, index.identifier());
+                IndexDescriptor indexDescriptor = loadDescriptor(sstable, cfs);
 
-                if (sstableIndex.getIndexTermType().isLiteral())
+                if (TypeUtil.isLiteral(sstableIndex.getIndexContext().getValidator()))
+                {
+                    addComponentSizeToMap(lengths, IndexComponentType.TERMS_DATA, index.getIndexContext(), indexDescriptor);
+                    addComponentSizeToMap(lengths, IndexComponentType.POSTING_LISTS, index.getIndexContext(), indexDescriptor);
+                }
+                else
                 {
-                    addComponentSizeToMap(lengths, IndexComponent.TERMS_DATA, index.identifier(), indexDescriptor);
-                    addComponentSizeToMap(lengths, IndexComponent.POSTING_LISTS, index.identifier(), indexDescriptor);
+                    addComponentSizeToMap(lengths, IndexComponentType.KD_TREE, index.getIndexContext(), indexDescriptor);
+                    addComponentSizeToMap(lengths, IndexComponentType.KD_TREE_POSTING_LISTS, index.getIndexContext(), indexDescriptor);
                 }
             }
         }
@@ -209,10 +223,10 @@ private HashMap<String, Long> indexFileLengths()
         return lengths;
     }
 
-    private static void addComponentSizeToMap(HashMap<String, Long> map, IndexComponent key, IndexIdentifier indexIdentifier, IndexDescriptor indexDescriptor)
+    private void addComponentSizeToMap(HashMap<String, Long> map, IndexComponentType key, IndexContext indexContext, IndexDescriptor indexDescriptor)
     {
         map.compute(key.name(), (typeName, acc) -> {
-            final long size = indexDescriptor.sizeOnDiskOfPerIndexComponent(key, indexIdentifier);
+            final long size = indexDescriptor.perIndexComponents(indexContext).get(key).file().length();
             return acc == null ? size : size + acc;
         });
     }
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASICQLTest.java b/test/unit/org/apache/cassandra/index/sasi/SASICQLTest.java
index e746fabdde0d..fe295d097b34 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASICQLTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASICQLTest.java
@@ -121,10 +121,8 @@ public void testDisableSASIIndexes()
         }
         catch (RuntimeException e)
         {
-            Throwable cause = e.getCause();
-            Assert.assertNotNull(cause);
-            Assert.assertTrue(cause instanceof InvalidRequestException);
-            Assert.assertTrue(cause.getMessage().contains("SASI indexes are disabled"));
+            Assert.assertTrue(e instanceof InvalidRequestException);
+            Assert.assertTrue(e.getMessage().contains("SASI indexes are disabled"));
         }
         finally
         {
@@ -360,7 +358,7 @@ public void testInOperator() throws Throwable
         createIndex("CREATE CUSTOM INDEX ON %s (v) USING 'org.apache.cassandra.index.sasi.SASIIndex';");
 
         assertInvalidThrowMessage(Optional.of(ProtocolVersion.CURRENT),
-                                  StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
+                                  String.format(StatementRestrictions.HAS_UNSUPPORTED_INDEX_RESTRICTION_MESSAGE_SINGLE, 'v'),
                                   InvalidQueryException.class,
                                   "SELECT * FROM %s WHERE v IN (200, 250, 300)");
     }
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index 6cd82b3a2386..2ea91101a70c 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@ -1494,14 +1494,14 @@ public void testSearchTimeouts()
 
         ColumnFamilyStore store = loadData(data1, true);
 
-        RowFilter filter = RowFilter.create(true);
+        RowFilter.Builder filter = RowFilter.builder(true);
         filter.add(store.metadata().getColumn(firstName), Operator.LIKE_CONTAINS, AsciiType.instance.fromString("a"));
 
         ReadCommand command =
             PartitionRangeReadCommand.create(store.metadata(),
                                              FBUtilities.nowInSeconds(),
                                              ColumnFilter.all(store.metadata()),
-                                             filter,
+                                             filter.build(),
                                              DataLimits.NONE,
                                              DataRange.allData(store.metadata().partitioner));
         try
@@ -2726,7 +2726,7 @@ private static ReadCommand getIndexReadCommand(ColumnFamilyStore store, ColumnFi
                             ? DataRange.allData(PARTITIONER)
                             : DataRange.forKeyRange(new Range<>(startKey, PARTITIONER.getMinimumToken().maxKeyBound()));
 
-        RowFilter filter = RowFilter.create(true);
+        RowFilter.Builder filter = RowFilter.builder(true);
         for (Expression e : expressions)
             filter.add(store.metadata().getColumn(e.name), e.op, e.value);
 
@@ -2734,7 +2734,7 @@ private static ReadCommand getIndexReadCommand(ColumnFamilyStore store, ColumnFi
             PartitionRangeReadCommand.create(store.metadata(),
                                              FBUtilities.nowInSeconds(),
                                              columnFilter,
-                                             filter,
+                                             filter.build(),
                                              DataLimits.cqlLimits(maxResults),
                                              range);
         return command;
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/OnDiskIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/OnDiskIndexTest.java
index 0abddd9193b8..cd9fba6df6d1 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/OnDiskIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/OnDiskIndexTest.java
@@ -18,42 +18,51 @@
 package org.apache.cassandra.index.sasi.disk;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.stream.Collectors;
 
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.carrotsearch.hppc.LongSet;
+import com.carrotsearch.hppc.cursors.LongCursor;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.BufferDecoratedKey;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.index.sasi.plan.Expression;
 import org.apache.cassandra.index.sasi.utils.CombinedTerm;
 import org.apache.cassandra.index.sasi.utils.CombinedTermIterator;
 import org.apache.cassandra.index.sasi.utils.OnDiskIndexIterator;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.MurmurHash;
 import org.apache.cassandra.utils.Pair;
 
-import com.carrotsearch.hppc.LongSet;
-import com.carrotsearch.hppc.cursors.LongCursor;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Sets;
-
-import org.junit.Assert;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
 public class OnDiskIndexTest
 {
     @BeforeClass
@@ -916,12 +925,18 @@ private static void addAll(OnDiskIndexBuilder builder, ByteBuffer term, TokenTre
         }
     }
 
-    private static class KeyConverter implements Function<Long, DecoratedKey>
+    private static class KeyConverter implements IKeyFetcher
     {
         @Override
-        public DecoratedKey apply(Long offset)
+        public DecoratedKey apply(long offset)
         {
             return keyAt(offset);
         }
+
+        @Override
+        public void close()
+        {
+
+        }
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
index 02e87b374c93..8d755b56e6fb 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
@@ -53,6 +53,7 @@
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.ColumnMetadata;
@@ -94,7 +95,7 @@ public void testPartialIndexWrites() throws Exception
 
         File directory = cfs.getDirectories().getDirectoryForNewSSTables();
         Descriptor descriptor = cfs.newSSTableDescriptor(directory);
-        PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, LifecycleTransaction.offline(OperationType.FLUSH));
+        PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, LifecycleTransaction.offline(OperationType.FLUSH, cfs.metadata));
 
         SortedMap<DecoratedKey, Row> expectedKeys = new TreeMap<>(DecoratedKey.comparator);
 
@@ -140,15 +141,25 @@ public void testPartialIndexWrites() throws Exception
 
         File indexFile = indexWriter.indexes.get(column).file(true);
 
-        // final flush
-        indexWriter.complete();
+        // final flush (not: the `sstable` is not used by SASI, so passing `null` is fine)
+        indexWriter.complete(null);
 
         for (String segment : segments)
             Assert.assertFalse(new File(segment).exists());
 
-        OnDiskIndex index = new OnDiskIndex(indexFile, Int32Type.instance, keyPosition -> {
-            ByteBuffer key = ByteBufferUtil.bytes(String.format(keyFormat, keyPosition));
-            return cfs.metadata().partitioner.decorateKey(key);
+        OnDiskIndex index = new OnDiskIndex(indexFile, Int32Type.instance, new IKeyFetcher()
+        {
+            @Override
+            public void close()
+            {
+            }
+
+            @Override
+            public DecoratedKey apply(long keyOffset)
+            {
+                ByteBuffer key = ByteBufferUtil.bytes(String.format(keyFormat, keyOffset));
+                return cfs.metadata().partitioner.decorateKey(key);
+            }
         });
 
         Assert.assertEquals(0, UTF8Type.instance.compare(index.minKey(), ByteBufferUtil.bytes(String.format(keyFormat, 0))));
@@ -188,7 +199,7 @@ public void testSparse() throws Exception
 
         File directory = cfs.getDirectories().getDirectoryForNewSSTables();
         Descriptor descriptor = cfs.newSSTableDescriptor(directory);
-        PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, LifecycleTransaction.offline(OperationType.FLUSH));
+        PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, LifecycleTransaction.offline(OperationType.FLUSH, cfs.metadata));
 
         final long now = System.currentTimeMillis();
 
@@ -238,7 +249,8 @@ public void testSparse() throws Exception
         for (String segment : segments)
             Assert.assertTrue(new File(segment).exists());
 
-        indexWriter.complete();
+        // The sstable argument is not used by SASI
+        indexWriter.complete(null);
 
         // make sure that individual segments have been cleaned up
         for (String segment : segments)
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
index 6d067a1d1459..94152e7ab945 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
@@ -19,42 +19,49 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.function.LongFunction;
 
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
 
+import com.carrotsearch.hppc.LongHashSet;
+import com.carrotsearch.hppc.LongSet;
+import com.carrotsearch.hppc.cursors.LongCursor;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.BufferDecoratedKey;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.LongType;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder.EntryType;
 import org.apache.cassandra.index.sasi.utils.CombinedTerm;
 import org.apache.cassandra.index.sasi.utils.CombinedValue;
 import org.apache.cassandra.index.sasi.utils.MappedBuffer;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
-import org.apache.cassandra.db.marshal.LongType;
 import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.SequentialWriterOption;
-import org.apache.cassandra.utils.MurmurHash;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.io.util.SequentialWriter;
-
-import org.junit.Assert;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import com.carrotsearch.hppc.LongHashSet;
-import com.carrotsearch.hppc.LongSet;
-import com.carrotsearch.hppc.cursors.LongCursor;
-import com.google.common.base.Function;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.utils.MurmurHash;
 
 public class TokenTreeTest
 {
-    private static final Function<Long, DecoratedKey> KEY_CONVERTER = new KeyConverter();
+    private static final LongFunction<DecoratedKey> KEY_CONVERTER = new KeyConverter();
 
     static LongSet singleOffset = new LongHashSet();
     static LongSet bigSingleOffset = new LongHashSet();
@@ -623,10 +630,10 @@ private static LongSet convert(long... values)
         return result;
     }
 
-    private static class KeyConverter implements Function<Long, DecoratedKey>
+    private static class KeyConverter implements LongFunction<DecoratedKey>
     {
         @Override
-        public DecoratedKey apply(Long offset)
+        public DecoratedKey apply(long offset)
         {
             return dk(offset);
         }
diff --git a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
index 0cef902be06d..eea2261c5962 100644
--- a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
@@ -656,7 +656,7 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey,
         }
 
         @Override
-        protected String toString(boolean cql)
+        public String toString(boolean cql)
         {
             return String.format("%s %s %s",
                                  cql ? column.name.toCQLString() : column.name.toString(),
diff --git a/test/unit/org/apache/cassandra/inject/ActionBuilder.java b/test/unit/org/apache/cassandra/inject/ActionBuilder.java
index f713ed9db861..8565bc7670a9 100644
--- a/test/unit/org/apache/cassandra/inject/ActionBuilder.java
+++ b/test/unit/org/apache/cassandra/inject/ActionBuilder.java
@@ -127,7 +127,7 @@ public ConditionsBuilder when(Object expression)
 
         public ConditionsBuilder not()
         {
-            if (!(elements.getLast() instanceof LogicOp))
+            if (!elements.isEmpty() && !(elements.getLast() instanceof LogicOp))
             {
                 elements.add(LogicOp.AND);
             }
diff --git a/test/unit/org/apache/cassandra/inject/Expression.java b/test/unit/org/apache/cassandra/inject/Expression.java
index 2fac96cc1267..19008d009c0f 100644
--- a/test/unit/org/apache/cassandra/inject/Expression.java
+++ b/test/unit/org/apache/cassandra/inject/Expression.java
@@ -56,6 +56,12 @@ public static Expression clazz(Class<?> clazz)
         return expr(clazz.getName());
     }
 
+    public Expression innerClass(String clazz)
+    {
+        expression.append("$").append(clazz);
+        return this;
+    }
+
     public Expression method(String method)
     {
         if (expression.length() > 0)
@@ -79,7 +85,7 @@ public static Expression method(Class<?> clazz, Class<? extends Annotation> anno
                                      .collect(Collectors.toList());
 
         Preconditions.checkArgument(methods.size() == 1, "There are " + methods.size() + " methods annotated with " + annotation.getSimpleName());
-        return Expression.clazz(clazz).method(methods.get(0).getName());
+        return expr().clazz(clazz).method(methods.get(0).getName());
     }
 
     public Expression append(String elem)
@@ -102,4 +108,8 @@ public static String quote(String quoted)
     public static String arg(int n) { return "$" + n; }
 
     public final static String THIS = "$this";
+
+    public final static String CLASS = "$CLASS";
+
+    public final static String METHOD = "$METHOD";
 }
diff --git a/test/unit/org/apache/cassandra/inject/Injections.java b/test/unit/org/apache/cassandra/inject/Injections.java
index 809d8940cbf9..c19ee5ec067b 100644
--- a/test/unit/org/apache/cassandra/inject/Injections.java
+++ b/test/unit/org/apache/cassandra/inject/Injections.java
@@ -35,11 +35,12 @@
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import org.apache.commons.io.IOUtils;
 
 import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.INativeLibrary;
 import org.jboss.byteman.agent.install.Install;
 import org.jboss.byteman.agent.submit.Submit;
 import org.jboss.byteman.rule.helper.Helper;
@@ -86,6 +87,9 @@ private static int loadAgent() throws Throwable
         long pid = getProcessId();
         List<String> properties = new ArrayList<>();
         properties.add("org.jboss.byteman.transform.all=true");
+        // uncomment below two lines to add debug or/and more verbose info, if you need to debug an injection
+        // properties.add("org.jboss.byteman.verbose=true");
+        // properties.add("org.jboss.byteman.debug=true");
         Install.install(Long.toString(pid), true, true, FBUtilities.getBroadcastAddressAndPort().getAddress().getHostAddress(), port, properties.toArray(new String[0]));
         return port;
     }
@@ -108,7 +112,7 @@ private static int getPort()
      */
     private static Long getProcessId()
     {
-        long pid = NativeLibrary.getProcessID();
+        long pid = INativeLibrary.instance.getProcessID();
         if (pid >= 0)
             return pid;
 
@@ -187,6 +191,9 @@ Rule[] getRules()
          * Adds a new action to the injection. Adding a new action cause creation of a new rule because a single rule
          * can only have a single action. Do not confuse an action with a statement - an action is bundle of bindings,
          * condition under which it can be invoked and a sequence of statements.
+         *
+         * If you just need to add a statement to the existing action, see {@link #withLastActionBuilder(Consumer)} and
+         * {@link #lastActionBuilder()}.
          */
         public B add(ActionBuilder builder)
         {
@@ -206,6 +213,28 @@ public B add(ActionBuilder.Builder builder)
         {
             return add(builder.toActionBuilder());
         }
+
+        /**
+         * Allows to modify the last defined action. You can add new bindings, conditions and statements. If you need
+         * to create a new action, please see {@link #add(ActionBuilder)}.
+         */
+        public ActionBuilder lastActionBuilder()
+        {
+            Preconditions.checkState(!actionBuilders.isEmpty());
+            ActionBuilder ab = actionBuilders.getLast();
+            return ab;
+        }
+
+        /**
+         * @see #lastActionBuilder()
+         */
+        public B withLastActionBuilder(Consumer<ActionBuilder> builder)
+        {
+            Preconditions.checkState(!actionBuilders.isEmpty());
+            ActionBuilder ab = actionBuilders.getLast();
+            builder.accept(ab);
+            return (B) this;
+        }
     }
 
     /**
@@ -422,6 +451,94 @@ public Barrier build()
         }
     }
 
+    /**
+     * Creates {@link Times} injection.
+     *
+     * @param name name of the internal counter
+     * @param defaultTimes the number of times the action should be executed
+     */
+    public static Times.TimesBuilder newTimes(String name, int defaultTimes)
+    {
+        return new Times.TimesBuilder(name, defaultTimes);
+    }
+
+    /**
+     * Creates an injection which allows to invoke a defined action for a defined number of times.
+     */
+    public static class Times extends Injection
+    {
+        private static Map<String, AtomicLong> counters = new ConcurrentHashMap<>();
+        private final AtomicLong internalCounter;
+        private final int defaultTimes;
+
+        private Times(String id, String name, int defaultTimes, Rule[] rules)
+        {
+            super(id, rules);
+            this.internalCounter = counters.computeIfAbsent(name, n -> new AtomicLong(defaultTimes));
+            this.defaultTimes = defaultTimes;
+            reset();
+        }
+
+        /**
+         * Get the remaining number of times the action will be attempted to be executed.
+         */
+        public long get()
+        {
+            return internalCounter.get();
+        }
+
+        /**
+         * Reset the internal counter to the original value.
+         */
+        public void reset()
+        {
+            reset(defaultTimes);
+        }
+
+        /**
+         * Reset the internal counter to the given value.
+         */
+        public void reset(int n)
+        {
+            internalCounter.set(n);
+        }
+
+        @CallMe
+        public static boolean decrementAndCheck(String name)
+        {
+            AtomicLong counter = counters.get(name);
+            long value = counter.decrementAndGet();
+            return value >= 0;
+        }
+
+        public static class TimesBuilder extends CrossProductInjectionBuilder<Times, TimesBuilder>
+        {
+            private final String name;
+            private final int defaultTimes;
+
+            private TimesBuilder(String name, int defaultTimes)
+            {
+                super(String.format("times/%s/%s", name, UUID.randomUUID().toString()));
+                this.name = name;
+                this.defaultTimes = defaultTimes;
+            }
+
+            @Override
+            public TimesBuilder add(ActionBuilder builder)
+            {
+                super.add(builder);
+                builder.conditions().when(method(Times.class, CallMe.class).args(quote(name)));
+                return this;
+            }
+
+            @Override
+            public Times build()
+            {
+                return new Times(id, name, defaultTimes, getRules());
+            }
+        }
+    }
+
     public abstract static class SingleActionBuilder<T extends Injection, B extends InjectionBuilder<T>>
             extends MultiInvokePointInjectionBuilder<T, B>
     {
diff --git a/test/unit/org/apache/cassandra/inject/injections.md b/test/unit/org/apache/cassandra/inject/injections.md
new file mode 100644
index 000000000000..855e389dfb8c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/inject/injections.md
@@ -0,0 +1,356 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+## Injecting hooks dynamically in testing
+
+The testing infrastructure is equipped with Byteman, which allows to
+inject hooks into the running code. In other words, at any point in
+the test, you can tell the particular nodes, to do something at exact
+location in the code - like you would set a conditional breakpoint
+and when the debugger breaks, you can check something or perform some
+additional actions. In short, it is particularly useful to:
+
+- synchronizing the test code and the nodes
+- synchronizing the nodes between each other
+- running actions step by step
+- counting invocations
+- tracing invocations
+- force throwing exceptions
+- and combinations of the above...
+
+Here you can find a short introduction and some examples how can it be
+used.
+
+### Basics
+
+Byteman works on rules which are the units defining what, when, and
+where should be invoked. That is, a single rule includes: the invoke
+point, bindings, run condition and actions.
+
+**Invoke point** is a location in the code where the actions should
+be hooked. Usually it is a class and method, but it can be specified
+more precisely, like - entering method, exiting method, invoking some
+method inside, particular line of code and so on.
+
+**Bindings** are just some constant definitions which can be used then
+in run condition and actions.
+
+**Run condition** is a logical expression which determines whether
+the rule should be invoked.
+
+**Actions** are the statements to be invoked.
+
+In DSE, we have `Rule` class which reflects a single Byteman rule.
+It can be build by providing rule script directly or by providing
+action and invoke point builders.
+
+In DSE, we also have `Injection` class which groups multiple rules
+which can be injected / enabled / disabled as a unit.
+
+#### Creating Injection
+
+```java
+ActionBuilder actionBuilder = ActionBuilder.newActionBuilder()
+    .actions().doThrow(newExpression(RuntimeException.class, "Induced exception"))
+    .conditions().when(expression("System.currentTimeMillis() % 2 == 0"))
+    .toActionBuilder();
+
+InvokePointBuilder invokePointBuilder = new InvokePointBuilder()
+    .atEntry().onClass(LeasePlugin.class).onMethod("setupSchema");
+
+Injection injection = new Injection(
+    "myInjection",
+    new Rule[]{ Rule.newRule("rule1", actionBuilder, invokePointBuilder) });
+```
+
+#### Installing injection
+Send all provided injections to a specified node:
+```java
+DseTestRunner.sendInjectionToNode(1, injection1, injection2, ...);
+```
+
+Install injections at bootstrap
+```java
+DseTestRunner.startNode(1,
+    DseYamlBuilder.newInstance(),
+    CassandraYamlBuilder.newInstance(),
+    DseNode.Type.CASSANDRA,
+    new Injection[] {injection1, injection2},
+    true);
+```
+
+or
+
+```java
+DseNode node = DseTestRunner.getOrCreateNode(...);
+node.setBootstrapInjectionRules(injection1, injection2, ...);
+```
+
+#### Disabling injection
+Disable all rules included in injection:
+```java
+injection.disable();
+```
+
+#### Enabling injection
+Reenable all rules included in injection:
+```java
+injection.enable();
+```
+
+#### Unloading injections
+Unload all injections from specified nodes:
+```java
+DseTestRunner.removeAllInjectionsFromNodes(1, 2, 3);
+```
+
+The above injection will make DSE to throw `RuntimeException` when
+entering `LeasePlugin.setupSchema` method with the probability of 50%.
+
+### Predefined injections
+In general, `CommonInjections` class includes builder for various
+commonly used injections. Internally, the rules often use Hazelcast
+for distributed latches, locks, and counters. However, it is all
+hidden and you do no need to bother. The provided builders allow to
+add multiple actions and invoke points easily, automatically
+generating cross product of appropriate rules.
+
+It is usually handy to add some imports to make the code less
+wordy:
+
+```java
+import com.datastax.bdp.test.ng.inject.CommonInjections;
+
+import static com.datastax.bdp.test.ng.inject.ActionBuilder.*;
+import static com.datastax.bdp.test.ng.inject.InvokePointBuilder.*;
+import static com.datastax.bdp.test.ng.inject.Expression.*;
+```
+
+#### Counter
+Creates an injection along with a distributed counter. It increments
+the counter whenever an invoke point is reached. You can add multiple
+invoke points and you can also bundle additional actions - for example,
+throw an exception and count how many times it happened.
+
+For example you may want to count how many times a query was processed
+on all nodes:
+
+```java
+CommonInjections.Counter queryCounter = CommonInjections.newCounter("queryCounter")
+        .add(newInvokePoint().onClass(DseQueryHandler.class).onMethod("process").atEntry())
+        .build();
+// ...
+queryCounter.injectIntoNodes(1, 2, 3);
+// ...
+assertEquals(5, queryCounter.get());
+```
+
+Another example is to count how many times `LeasePlugin.setupSchema`
+will be attempted if it throws `UnavailableException` before the node
+eventually die:
+
+```java
+CommonInjections.Counter retriesCounter = CommonInjections.newCounter("retriesCounter")
+        .add(newInvokePoint().onClass(LeasePlugin.class).onMethod("setupSchema").atExit())
+        .add(newActionBuilder().actions().doThrow(newInstance(UnavailableException.class).args(clazz(ConsistencyLevel.class).method("THREE"), 3, 1)))
+        .build();
+
+DseNode node = getOrCreateNode(1, DseNode.Type.CASSANDRA);
+node.setBootstrapInjectionRules(retriesCounter);
+node.start();
+
+// ...
+assertEquals(10, retriesCounter.get());
+```
+
+#### Barrier
+Creates an injection with a distributed barrier awaiting for a defined
+number of parties to reach it (including test node). It can be used to
+synchronize multiple nodes - for example, you can make three nodes
+synchronize on entering `LeasePlugin.setupSchema`:
+
+```java
+CommonInjections.Barrier barrier = CommonInjections.newBarrier("setupSchemaBarrier", 3, false)
+        .add(newInvokePoint().onClass(LeasePlugin.class).onMethod("setupSchema").atEntry())
+        .build();
+
+DseNode node1 = getOrCreateNode(1, DseNode.Type.CASSANDRA).setBootstrapInjectionRules(barrier);
+DseNode node2 = getOrCreateNode(2, DseNode.Type.CASSANDRA).setBootstrapInjectionRules(barrier);
+DseNode node3 = getOrCreateNode(3, DseNode.Type.CASSANDRA).setBootstrapInjectionRules(barrier);
+
+node1.start(false);
+node2.start(false);
+node3.start(false);
+
+node1.waitForReady();
+node2.waitForReady();
+node3.waitForReady();
+```
+
+With barrier, you can also create more sophisticated synchronization
+points. As you know, barrier a little bit similar to the count down
+latch. When an execution arrives at the barrier, the number of parties
+the barrier is waiting for is decremented and then it awaits until the
+latch is zeroed. By normal a barrier does both things together. However
+here you can separate them and pin to different invoke points so that
+the execution awaiting in one place can be released by the execution in
+the other place. For the purpose you create two barrier injections with
+the same name. You can see this in [DbRollupReportingTest](dse-core/src/test/java/com/datastax/bdp/reporting/snapshots/db/inject/DbRollupReportingTest.java):
+In this test, the count down part, which is non-blocking is pinned
+to exit of `PeriodicUpdateTask.run`, while await part is pinned to
+entry of `ClusterInfoRollupTask.doRollup`:
+
+```java
+private static final Barrier updateEnd = CommonInjections.newBarrierCountDown("updateToRollupSync", TOTAL_NODES, false)
+        .add(newInvokePoint().onClassMethod(NodeSnapshotPlugin.class, "PeriodicUpdateTask", "run").atExit())
+        .build();
+
+private static final Barrier rollupStart = CommonInjections.newBarrierAwait("updateToRollupSync", TOTAL_NODES, false)
+        .add(newInvokePoint().onClass(ClusterInfoRollupTask.class).onMethod("doRollup").atEntry())
+        .build();
+```
+
+#### Times
+Creates an injection which allows to invoke a defined action for
+a defined number of times. For example, you may want to induce
+`LeasePlugin.setupSchema` to throw `UnavailableException` for 3 times
+and then eventually pass:
+
+```java
+CommonInjections.Times setupSchemaFailure = CommonInjections.newTimes("setupSchemaFailure", 3)
+        .add(newInvokePoint().onClass(LeasePlugin.class).onMethod("setupSchema").atEntry())
+        .add(newActionBuilder().actions().doThrow(newInstance(UnavailableException.class).args(clazz(ConsistencyLevel.class).method("THREE"), 3, 1)))
+        .build();
+
+DseNode node = getOrCreateNode(1, DseNode.Type.CASSANDRA);
+node.setBootstrapInjectionRules(setupSchemaFailure);
+node.start();
+```
+
+#### Step
+Step is just a double barrier. It allows execute the code stopping at
+the defined location each time it is reached. For example, you may want
+to stop the execution of each node before entering into
+`LeasePlugin.setupSchema` and wait for it to happen in the test code.
+Then, do some changes in the schema from the test code, and resume
+everything. This is exactly why you may want to use `Step`. In the
+following example we want to stop 2 nodes before `setupSchema`:
+
+```java
+// the number of parties is 3 because it includes 2 nodes and the test worker
+CommonInjections.Step step = CommonInjections.newStep("setupSchema", 3, false)
+        .add(newInvokePoint().onClass(LeasePlugin.class).onMethod("setupSchema").atEntry())
+        .build();
+
+DseNode node1 = getOrCreateNode(1, DseNode.Type.CASSANDRA).setBootstrapInjectionRules(step);
+DseNode node2 = getOrCreateNode(2, DseNode.Type.CASSANDRA).setBootstrapInjectionRules(step);
+node1.start(false);
+node2.start(false);
+
+step.await();
+
+// do some stuff while nodes are still waiting before entering setupSchema
+
+step.resume();
+
+node1.waitForReady();
+node2.waitForReady();
+```
+
+#### Pause
+Injects an additional pause into a defined location. For example, you
+may want to slow down executing queries:
+
+```java
+Injection queryPause = CommonInjections.newPause("queryPause", 2000)
+        .add(newInvokePoint().onClass(DseQueryHandler.class).onMethod("process").atEntry())
+        .build();
+
+// ...
+sendInjectionToNode(1, queryPause);
+// ...
+```
+
+#### Trace
+Trace allows to put a message into standard out, standard err or a
+particular file whenever an execution reaches a particular location.
+Imagine this as adding some logging at runtime, which can be done even
+against classes from third party jars you cannot tweak manually.
+
+For example, you may want to know when a keyspace and a table is added
+to Cassandra schema. You can log it to standard out:
+
+```java
+Injection setupSchemaTrace = CommonInjections.newTraceToStdOut()
+        .add(newInvokePoint().onClass(Schema.class).onMethod("addTable"))
+        .add(newInvokePoint().onClass(Schema.class).onMethod("addKeyspace"))
+        .traceln(expr(quote("Adding to schema: ")).append("+").append(arg(1)))
+        .build();
+
+// ...
+setupSchemaTrace.injectIntoNodes(1, 2, 3);
+```
+
+or into a file:
+
+```
+Injection setupSchemaTrace = CommonInjections.newTrace("schemaTrace", "schemaTrace.txt")
+        .add(newInvokePoint().onClass(Schema.class).onMethod("addTable"))
+        .add(newInvokePoint().onClass(Schema.class).onMethod("addKeyspace"))
+        .traceln(expr(quote("Adding to schema: ")).append("+").append(arg(1)))
+        .build();
+
+// ...
+setupSchemaTrace.injectIntoNodes(1, 2, 3);
+```
+
+### End notes
+
+#### Byteman debugging
+Whenever the test doesn't work as you expected, you may enable Byteman
+verbose output, so that you can see if everything went ok with the
+defined rules. For example, Byteman does not throw any exception if
+there is a syntax error in the rule script. In order to see that, and
+other useful information, just add `bytemanVerbose` property to the
+test command:
+
+```./gradlew test -PbytemanVerbose```
+
+#### Several rules for the same method
+If there are more rules for the same method, you should inject them
+together because otherwise it may happen that some of them will not be
+invoked (silently, without error message). Thus, send them in the
+following way:
+
+```java
+DseTestRunner.sendInjectionToNode(nodeId, injection1, injection2);
+```
+
+#### Byteman rule language
+If you need implement your own rule, you can refer to Byteman rule
+language guide [here](https://github.com/bytemanproject/byteman/blob/master/docs/asciidoc/src/main/asciidoc/chapters/Byteman-Rule-Language.adoc).
+
+#### Creating custom injections
+Use Hazelcast to store objects you want to access from any test node
+and test worker. Hazelcast offers distributed locks, maps, counters,
+latches and many more. There a convenience class [Hazelcast](dse-core/src/test/java/com/datastax/bdp/test/ng/Hazelcast)
+which allows to easily obtain either Hazelcast server or client
+instance which are configured and ready to use. Remember to use a server
+instance in the test code and a client instance for what you are doing
+in the injection. You can also see how the common injections are
+implemented.
diff --git a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
index 63b8e5b99c41..387e95e46cd2 100644
--- a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
+++ b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
@@ -49,6 +49,7 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MovingAverage;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.UNSAFE_SYSTEM;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.psjava.util.AssertStatus.assertTrue;
@@ -61,6 +62,8 @@ public class DiskSpaceMetricsTest extends CQLTester
     @Test
     public void baseline() throws Throwable
     {
+        // CNDB-10289 sets this to true in build.xml, but that prevents SystemKeyspace.forceBlockingFlush
+        UNSAFE_SYSTEM.setBoolean(false);
         createTable("CREATE TABLE %s (pk bigint, PRIMARY KEY (pk)) WITH min_index_interval=1");
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
 
@@ -70,6 +73,7 @@ public void baseline() throws Throwable
         // create 100 sstables
         for (int i = 0; i < 100; i++)
             insert(cfs, i);
+
         assertDiskSpaceEqual(cfs);
     }
 
@@ -104,7 +108,7 @@ public void testFlushSize() throws Throwable
     {
         createTable(KEYSPACE_PER_TEST, "CREATE TABLE %s (pk bigint, PRIMARY KEY (pk))");
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore(KEYSPACE_PER_TEST);
-        assertTrue(Double.isNaN(cfs.metric.flushSizeOnDisk.get()));
+        assertTrue(Double.isNaN(cfs.metric.flushSizeOnDisk().get()));
 
         // disable compaction so nothing changes between calculations
         cfs.disableAutoCompaction();
@@ -118,7 +122,7 @@ public void testFlushSize() throws Throwable
         MovingAverage expectedMetrics = ExpMovingAverage.decayBy1000();
         for (SSTableReader rdr : liveSSTables)
             expectedMetrics.update(rdr.onDiskLength());
-        assertThat(cfs.metric.flushSizeOnDisk.get()).isEqualTo(expectedMetrics.get());
+        assertThat(cfs.metric.flushSizeOnDisk().get()).isEqualTo(expectedMetrics.get());
     }
 
     private void insert(ColumnFamilyStore cfs, long value) throws Throwable
diff --git a/test/unit/org/apache/cassandra/io/compress/AdaptiveCompressorTest.java b/test/unit/org/apache/cassandra/io/compress/AdaptiveCompressorTest.java
new file mode 100644
index 000000000000..ac58a2e080e2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/compress/AdaptiveCompressorTest.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.compress;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class AdaptiveCompressorTest
+{
+
+    @Test(expected = IllegalArgumentException.class)
+    public void badCompressionLevelParamThrowsExceptionMin()
+    {
+        AdaptiveCompressor.create(ImmutableMap.of(AdaptiveCompressor.MIN_COMPRESSION_LEVEL_OPTION_NAME, Integer.toString(AdaptiveCompressor.MIN_COMPRESSION_LEVEL - 1)));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void badCompressionLevelParamThrowsExceptionMax()
+    {
+        AdaptiveCompressor.create(ImmutableMap.of(AdaptiveCompressor.MAX_COMPRESSION_LEVEL_OPTION_NAME, Integer.toString(AdaptiveCompressor.MAX_COMPRESSION_LEVEL + 1)));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void badMaxCompactionQueueLengthParamThrowsExceptionMin()
+    {
+        AdaptiveCompressor.create(ImmutableMap.of(AdaptiveCompressor.MAX_COMPACTION_QUEUE_LENGTH_OPTION_NAME, "-1"));
+    }
+
+    @Test
+    public void averageRelativeTimeCompressingIsMeasuredProperly() throws IOException, InterruptedException
+    {
+        var params = new AdaptiveCompressor.Params(ICompressor.Uses.GENERAL, 15, 15, 0);
+        AdaptiveCompressor c1 = new AdaptiveCompressor(params, () -> 0.0);
+        ByteBuffer src = getSrcByteBuffer();
+        ByteBuffer dest = getDstByteBuffer(c1);
+        for (int i = 0; i < 20000; i++)
+        {
+            compress(c1, src, dest);
+        }
+        assertTrue(c1.getThreadLocalState().getRelativeTimeSpentCompressing() > 0.8);
+        assertTrue(c1.getThreadLocalState().getRelativeTimeSpentCompressing() < 1.0);
+
+
+        var params2 = new AdaptiveCompressor.Params(ICompressor.Uses.GENERAL, 0, 0, 0);
+        AdaptiveCompressor c2 = new AdaptiveCompressor(params2, () -> 0.0);
+        for (int i = 0; i < 100; i++)
+        {
+            Thread.sleep(1);
+            compress(c2, src, dest);
+        }
+        assertTrue(c2.getThreadLocalState().getRelativeTimeSpentCompressing() < 0.02);
+        assertTrue(c2.getThreadLocalState().getRelativeTimeSpentCompressing() > 0.0);
+    }
+
+    @Test
+    public void compressionLevelAdaptsToWritePressure() throws IOException
+    {
+        var params = new AdaptiveCompressor.Params(ICompressor.Uses.GENERAL, 2, 8, 0);
+        double[] load = { 1.0 };
+
+        AdaptiveCompressor c = new AdaptiveCompressor(params, () -> load[0]);
+        ByteBuffer src = getSrcByteBuffer();
+        ByteBuffer dest = getDstByteBuffer(c);
+
+        for (int i = 0; i < 10; i++)
+            compress(c, src, dest);
+
+        assertEquals(2, c.getThreadLocalState().currentCompressionLevel);
+
+        // Low load; compression level must be increased back to max:
+        load[0] = 0L;
+
+        for (int i = 0; i < 10; i++)
+            compress(c, src, dest);
+
+        assertEquals(8, c.getThreadLocalState().currentCompressionLevel);
+    }
+
+    @Test
+    public void compressionLevelDoesNotDecreaseWhenCompressionIsNotABottleneck() throws IOException, InterruptedException
+    {
+        var params = new AdaptiveCompressor.Params(ICompressor.Uses.GENERAL, 2, 8, 0);
+        // Simulate high write load
+        AdaptiveCompressor c = new AdaptiveCompressor(params, () -> 1.0);
+        ByteBuffer src = getSrcByteBuffer();
+        ByteBuffer dest = getDstByteBuffer(c);
+
+        for (int i = 0; i < 200; i++)
+        {
+            Thread.sleep(1); // creates artificial bottleneck that is much slower than compression
+            compress(c, src, dest);
+        }
+
+        assertEquals(8, c.getThreadLocalState().currentCompressionLevel);
+    }
+
+    private static ByteBuffer getDstByteBuffer(ICompressor compressor)
+    {
+        return ByteBuffer.allocateDirect(compressor.initialCompressedBufferLength(RandomAccessReader.DEFAULT_BUFFER_SIZE));
+    }
+
+    private static ByteBuffer getSrcByteBuffer()
+    {
+        int n = RandomAccessReader.DEFAULT_BUFFER_SIZE;
+        byte[] srcData = new byte[n];
+        new Random().nextBytes(srcData);
+
+        ByteBuffer src = ByteBuffer.allocateDirect(n);
+        src.put(srcData, 0, n);
+        src.flip().position(0);
+        return src;
+    }
+
+    private static void compress(AdaptiveCompressor c, ByteBuffer src, ByteBuffer dest) throws IOException
+    {
+        c.compress(src, dest);
+        src.rewind();
+        dest.rewind();
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java b/test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java
index 7e0d1775b6c0..268d22262e18 100644
--- a/test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java
@@ -174,6 +174,39 @@ public void zstdFlushTest() throws Throwable
         });
     }
 
+    @Test
+    public void adaptiveFlushTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'class': 'AdaptiveCompressor'};");
+        DatabaseDescriptor.setFlushCompression(Config.FlushCompression.fast);
+        ColumnFamilyStore store = flushTwice();
+
+        // Should flush as LZ4
+        Set<SSTableReader> sstables = store.getLiveSSTables();
+        sstables.forEach(sstable -> {
+            assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof LZ4Compressor);
+        });
+        store.truncateBlocking();
+
+        DatabaseDescriptor.setFlushCompression(Config.FlushCompression.adaptive);
+        store = flushTwice();
+
+        // Should flush as Adaptive
+        sstables = store.getLiveSSTables();
+        sstables.forEach(sstable -> {
+            assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof AdaptiveCompressor);
+        });
+
+        // Should compact to Adaptive
+        compact();
+
+        sstables = store.getLiveSSTables();
+        assertEquals(1, sstables.size());
+        store.getLiveSSTables().forEach(sstable -> {
+            assertTrue(sstable.getCompressionMetadata().parameters.getSstableCompressor() instanceof AdaptiveCompressor);
+        });
+    }
+
     @Test
     public void deflateFlushTest() throws Throwable
     {
@@ -260,7 +293,7 @@ private ColumnFamilyStore flushTwice() throws Throwable
     {
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
 
-        execute("INSERT INTO %s (k, v) values (?, ?)", "k1", "v1");
+        execute("INSERT INTO %s (k, v) values (?, ?)", "k1" , "v1");
         flush();
         assertEquals(1, cfs.getLiveSSTables().size());
 
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index afa469c48772..7856350f49cb 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@ -51,6 +51,7 @@
 
 import static org.apache.cassandra.schema.CompressionParams.DEFAULT_CHUNK_LENGTH;
 import static org.apache.commons.io.FileUtils.readFileToByteArray;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -283,11 +284,12 @@ public void resetAndTruncateTest()
     {
         File tempFile = new File(Files.createTempDir().toPath(), "reset.txt");
         File offsetsFile = FileUtils.createDeletableTempFile("compressedsequentialwriter.offset", "test");
+        File digestFile = FileUtils.createDeletableTempFile("digest.db", "test");
         final int bufferSize = 48;
         final int writeSize = 64;
         byte[] toWrite = new byte[writeSize];
         try (SequentialWriter writer = new CompressedSequentialWriter(tempFile, offsetsFile,
-                                                                      null, SequentialWriterOption.DEFAULT,
+                                                                      digestFile, SequentialWriterOption.DEFAULT,
                                                                       CompressionParams.lz4(bufferSize),
                                                                       new MetadataCollector(new ClusteringComparator(UTF8Type.instance))))
         {
@@ -313,6 +315,12 @@ public void resetAndTruncateTest()
             // flush off set should not be increase
             assertEquals(flushedOffset, writer.getLastFlushOffset());
             writer.finish();
+
+            // verify digest value is present
+            try (RandomAccessReader digestReader = RandomAccessReader.open(digestFile))
+            {
+                assertThat(Long.parseLong(digestReader.readLine())).isNotNull();
+            }
         }
         catch (IOException e)
         {
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java b/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java
index 321fe5735606..712dd6b8f27d 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java
@@ -19,12 +19,21 @@
 package org.apache.cassandra.io.compress;
 
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.function.Consumer;
+
 import org.junit.Test;
 
+import org.apache.cassandra.distributed.shared.WithProperties;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.Memory;
+import org.apache.cassandra.io.util.SliceDescriptor;
 import org.apache.cassandra.schema.CompressionParams;
 
+import static java.util.Arrays.asList;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DEBUG_REF_COUNT;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 
@@ -35,44 +44,203 @@ public class CompressionMetadataTest
     long dataLength = 1000;
     long compressedFileLength = 100;
 
-    private CompressionMetadata newCompressionMetadata(Memory memory)
+    private CompressionMetadata newCompressionMetadata(CompressionMetadata.ChunkOffsetMemory memory)
     {
         return new CompressionMetadata(chunksIndexFile,
                                        params,
                                        memory,
-                                       memory.size(),
                                        dataLength,
-                                       compressedFileLength);
+                                       compressedFileLength,
+                                       Integer.numberOfTrailingZeros(params.chunkLength()),
+                                       0);
     }
 
     @Test
     public void testMemoryIsFreed()
     {
-        Memory memory = Memory.allocate(10);
-        CompressionMetadata cm = newCompressionMetadata(memory);
+        try (WithProperties properties = new WithProperties().set(TEST_DEBUG_REF_COUNT, false))
+        {
+            CompressionMetadata.ChunkOffsetMemory memory = new CompressionMetadata.ChunkOffsetMemory(10);
+            CompressionMetadata cm = newCompressionMetadata(memory);
 
-        cm.close();
-        assertThat(cm.isCleanedUp()).isTrue();
-        assertThatExceptionOfType(AssertionError.class).isThrownBy(memory::size);
+            cm.close();
+            assertThat(cm.isCleanedUp()).isTrue();
+            assertThatExceptionOfType(AssertionError.class).isThrownBy(memory.memory::free);
+        }
     }
 
     @Test
     public void testMemoryIsShared()
     {
-        Memory memory = Memory.allocate(10);
-        CompressionMetadata cm = newCompressionMetadata(memory);
+        try (WithProperties properties = new WithProperties().set(TEST_DEBUG_REF_COUNT, false))
+        {
+            CompressionMetadata.ChunkOffsetMemory memory = new CompressionMetadata.ChunkOffsetMemory(10);
+            CompressionMetadata cm = newCompressionMetadata(memory);
+
+            CompressionMetadata copy = cm.sharedCopy();
+            assertThat(copy).isNotSameAs(cm);
+
+            cm.close();
+            assertThat(cm.isCleanedUp()).isFalse();
+            assertThat(copy.isCleanedUp()).isFalse();
+            assertThat(memory.size()).isEqualTo(10); // expected that no expection is thrown since memory should not be released yet
+
+            copy.close();
+            assertThat(cm.isCleanedUp()).isTrue();
+            assertThat(copy.isCleanedUp()).isTrue();
+            assertThatExceptionOfType(AssertionError.class).isThrownBy(memory.memory::free);
+        }
+    }
+
+    private File generateMetaDataFile(long dataLength, long... offsets) throws IOException
+    {
+        Path path = Files.createTempFile("compression_metadata", ".db");
+        CompressionParams params = CompressionParams.snappy(16);
+        try (CompressionMetadata.Writer writer = CompressionMetadata.Writer.open(params, new File(path)))
+        {
+            for (long offset : offsets)
+                writer.addOffset(offset);
+
+            writer.finalizeLength(dataLength, offsets.length);
+            writer.doPrepare();
+            Throwable t = writer.doCommit(null);
+            if (t != null)
+                throw new IOException(t);
+        }
+        return new File(path);
+    }
+
+    private CompressionMetadata createMetadata(long dataLength, long compressedFileLength, long... offsets) throws IOException
+    {
+        File f = generateMetaDataFile(dataLength, offsets);
+        return CompressionMetadata.open(f, compressedFileLength, true, SliceDescriptor.NONE);
+    }
 
-        CompressionMetadata copy = cm.sharedCopy();
-        assertThat(copy).isNotSameAs(cm);
+    private void checkMetadata(CompressionMetadata metadata, long expectedDataLength, long expectedCompressedFileLimit, long expectedOffHeapSize)
+    {
+        assertThat(metadata.dataLength).isEqualTo(expectedDataLength);
+        assertThat(metadata.compressedFileLength).isEqualTo(expectedCompressedFileLimit);
+        assertThat(metadata.chunkLength()).isEqualTo(16);
+        assertThat(metadata.parameters.chunkLength()).isEqualTo(16);
+        assertThat(metadata.parameters.getSstableCompressor().getClass()).isEqualTo(SnappyCompressor.class);
+        assertThat(metadata.offHeapSize()).isEqualTo(expectedOffHeapSize);
+    }
 
-        cm.close();
-        assertThat(cm.isCleanedUp()).isFalse();
-        assertThat(copy.isCleanedUp()).isFalse();
-        assertThat(memory.size()).isEqualTo(10); // expected that no expection is thrown since memory should not be released yet
+    private void assertChunks(CompressionMetadata metadata, long from, long to, long expectedOffset, long expectedLength)
+    {
+        for (long offset = from; offset < to; offset++)
+        {
+            CompressionMetadata.Chunk chunk = metadata.chunkFor(offset);
+            assertThat(chunk.offset).isEqualTo(expectedOffset);
+            assertThat(chunk.length).isEqualTo(expectedLength);
+        }
+    }
 
-        copy.close();
-        assertThat(cm.isCleanedUp()).isTrue();
-        assertThat(copy.isCleanedUp()).isTrue();
-        assertThatExceptionOfType(AssertionError.class).isThrownBy(memory::size);
+    private void checkSlice(File f, long compressedSize, long from, long to, Consumer<CompressionMetadata> check)
+    {
+        try (CompressionMetadata md = CompressionMetadata.open(f, compressedSize, true, new SliceDescriptor(from, to, 16)))
+        {
+            check.accept(md);
+        }
     }
+
+    @Test
+    public void chunkFor() throws IOException
+    {
+        try (CompressionMetadata lessThanOneChunk = createMetadata(10, 7, 0))
+        {
+            checkMetadata(lessThanOneChunk, 10, 7, 8);
+            assertChunks(lessThanOneChunk, 0, 10, 0, 3);
+        }
+
+        try (CompressionMetadata oneChunk = createMetadata(16, 9, 0))
+        {
+            checkMetadata(oneChunk, 16, 9, 8);
+            assertChunks(oneChunk, 0, 16, 0, 5);
+        }
+
+        try (CompressionMetadata moreThanOneChunk = createMetadata(20, 15, 0, 9))
+        {
+            checkMetadata(moreThanOneChunk, 20, 15, 16);
+            assertChunks(moreThanOneChunk, 0, 16, 0, 5);
+            assertChunks(moreThanOneChunk, 16, 20, 9, 2);
+        }
+
+        try (CompressionMetadata extraEmptyChunk = createMetadata(20, 17, 0, 9, 15))
+        {
+            checkMetadata(extraEmptyChunk, 20, 15, 16);
+            assertChunks(extraEmptyChunk, 0, 16, 0, 5);
+            assertChunks(extraEmptyChunk, 16, 20, 9, 2);
+        }
+
+        // chunks of lengths: 3, 5, 7, 11, 7, 5 (+ 4 bytes CRC for each chunk)
+        File manyChunksFile = generateMetaDataFile(90, 0, 7, 16, 27, 42, 53);
+
+        // full slice, we should load all 6 chunks
+        checkSlice(manyChunksFile, 62, 0L, 90L, md -> {
+            checkMetadata(md, 90, 62, 6 * 8);
+            assertChunks(md, 0, 16, 0, 3);
+            assertChunks(md, 16, 32, 7, 5);
+            assertChunks(md, 32, 48, 16, 7);
+            assertChunks(md, 48, 64, 27, 11);
+            assertChunks(md, 64, 80, 42, 7);
+            assertChunks(md, 80, 96, 53, 5);
+
+            assertThat(md.getTotalSizeForSections(asList(new SSTableReader.PartitionPositionBounds(0, 90)))).isEqualTo(62);
+            assertThat(md.getChunksForSections(asList(new SSTableReader.PartitionPositionBounds(0, 90))))
+            .containsExactly(new CompressionMetadata.Chunk(0, 3), new CompressionMetadata.Chunk(7, 5), new CompressionMetadata.Chunk(16, 7), new CompressionMetadata.Chunk(27, 11), new CompressionMetadata.Chunk(42, 7), new CompressionMetadata.Chunk(53, 5));
+
+            assertThat(md.getTotalSizeForSections(asList(new SSTableReader.PartitionPositionBounds(20, 40), new SSTableReader.PartitionPositionBounds(50, 70)))).isEqualTo(46);
+            assertThat(md.getChunksForSections(asList(new SSTableReader.PartitionPositionBounds(20, 40), new SSTableReader.PartitionPositionBounds(50, 70)))).containsExactly(new CompressionMetadata.Chunk(7, 5), new CompressionMetadata.Chunk(16, 7), new CompressionMetadata.Chunk(27, 11), new CompressionMetadata.Chunk(42, 7));
+        });
+
+        // slice starting at 20, we should skip first chunk
+        checkSlice(manyChunksFile, 55, 20L, 90L, md -> {
+            checkMetadata(md, 74, 55, 5 * 8);
+            assertChunks(md, 20, 32, 7, 5);
+            assertChunks(md, 32, 48, 16, 7);
+            assertChunks(md, 48, 64, 27, 11);
+            assertChunks(md, 64, 80, 42, 7);
+            assertChunks(md, 80, 90, 53, 5);
+
+            assertThat(md.getTotalSizeForSections(asList(new SSTableReader.PartitionPositionBounds(20, 90)))).isEqualTo(55);
+            assertThat(md.getChunksForSections(asList(new SSTableReader.PartitionPositionBounds(20, 90)))).containsExactly(new CompressionMetadata.Chunk(7, 5), new CompressionMetadata.Chunk(16, 7), new CompressionMetadata.Chunk(27, 11), new CompressionMetadata.Chunk(42, 7), new CompressionMetadata.Chunk(53, 5));
+
+            assertThat(md.getTotalSizeForSections(asList(new SSTableReader.PartitionPositionBounds(30, 40), new SSTableReader.PartitionPositionBounds(50, 60)))).isEqualTo(35);
+            assertThat(md.getChunksForSections(asList(new SSTableReader.PartitionPositionBounds(30, 40), new SSTableReader.PartitionPositionBounds(50, 60)))).containsExactly(new CompressionMetadata.Chunk(7, 5), new CompressionMetadata.Chunk(16, 7), new CompressionMetadata.Chunk(27, 11));
+        });
+
+        // slice ending at 70, we should skip last chunk
+        checkSlice(manyChunksFile, 53, 0L, 70L, md -> {
+            checkMetadata(md, 70, 53, 5 * 8);
+            assertChunks(md, 0, 16, 0, 3);
+            assertChunks(md, 16, 32, 7, 5);
+            assertChunks(md, 32, 48, 16, 7);
+            assertChunks(md, 48, 64, 27, 11);
+            assertChunks(md, 64, 70, 42, 7);
+
+            assertThat(md.getTotalSizeForSections(asList(new SSTableReader.PartitionPositionBounds(0, 70)))).isEqualTo(53);
+            assertThat(md.getChunksForSections(asList(new SSTableReader.PartitionPositionBounds(0, 70)))).containsExactly(new CompressionMetadata.Chunk(0, 3), new CompressionMetadata.Chunk(7, 5), new CompressionMetadata.Chunk(16, 7), new CompressionMetadata.Chunk(27, 11), new CompressionMetadata.Chunk(42, 7));
+
+            assertThat(md.getTotalSizeForSections(asList(new SSTableReader.PartitionPositionBounds(30, 40), new SSTableReader.PartitionPositionBounds(50, 60)))).isEqualTo(35);
+            assertThat(md.getChunksForSections(asList(new SSTableReader.PartitionPositionBounds(30, 40), new SSTableReader.PartitionPositionBounds(50, 60)))).containsExactly(new CompressionMetadata.Chunk(7, 5), new CompressionMetadata.Chunk(16, 7), new CompressionMetadata.Chunk(27, 11));
+        });
+
+        // slice starting at 20 and ending at 70, we should skip first and last chunk
+        checkSlice(manyChunksFile, 46, 20L, 70L, md -> {
+            checkMetadata(md, 54, 46, 4 * 8);
+            assertChunks(md, 20, 32, 7, 5);
+            assertChunks(md, 32, 48, 16, 7);
+            assertChunks(md, 48, 64, 27, 11);
+            assertChunks(md, 64, 70, 42, 7);
+
+            assertThat(md.getTotalSizeForSections(asList(new SSTableReader.PartitionPositionBounds(20, 70)))).isEqualTo(46);
+            assertThat(md.getChunksForSections(asList(new SSTableReader.PartitionPositionBounds(20, 70)))).containsExactly(new CompressionMetadata.Chunk(7, 5), new CompressionMetadata.Chunk(16, 7), new CompressionMetadata.Chunk(27, 11), new CompressionMetadata.Chunk(42, 7));
+
+            assertThat(md.getTotalSizeForSections(asList(new SSTableReader.PartitionPositionBounds(30, 40), new SSTableReader.PartitionPositionBounds(50, 60)))).isEqualTo(35);
+            assertThat(md.getChunksForSections(asList(new SSTableReader.PartitionPositionBounds(30, 40), new SSTableReader.PartitionPositionBounds(50, 60)))).containsExactly(new CompressionMetadata.Chunk(7, 5), new CompressionMetadata.Chunk(16, 7), new CompressionMetadata.Chunk(27, 11));
+        });
+
+    }
+
 }
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressorTest.java b/test/unit/org/apache/cassandra/io/compress/CompressorTest.java
index dad3ae44aece..44d2ae26f0eb 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressorTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressorTest.java
@@ -17,7 +17,8 @@
  */
 package org.apache.cassandra.io.compress;
 
-import java.io.*;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
@@ -28,16 +29,26 @@
 
 import com.google.common.io.Files;
 import org.apache.cassandra.io.util.File;
-import static org.junit.Assert.*;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
 public class CompressorTest
 {
+    @BeforeClass
+    public static void initialize()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
     ICompressor compressor;
 
     ICompressor[] compressors = new ICompressor[] {
@@ -45,6 +56,7 @@ public class CompressorTest
             DeflateCompressor.create(Collections.<String, String>emptyMap()),
             SnappyCompressor.create(Collections.<String, String>emptyMap()),
             ZstdCompressor.create(Collections.emptyMap()),
+            AdaptiveCompressor.createForUnitTesting(),
             NoopCompressor.create(Collections.emptyMap())
     };
 
@@ -187,6 +199,13 @@ public void testZstdByteBuffers() throws IOException
         testByteBuffers();
     }
 
+    @Test
+    public void testAdaptiveByteBuffers() throws IOException
+    {
+        compressor = AdaptiveCompressor.createForUnitTesting();
+        testByteBuffers();
+    }
+
     @Test
     public void testNoopByteBuffers() throws IOException
     {
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
index bdf08c786fc2..0e760e8d0f59 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
@@ -23,9 +23,11 @@
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.util.File;
 
 public class CQLSSTableWriterClientTest extends CQLSSTableWriterTest
 {
@@ -38,8 +40,11 @@ public void setUp()
                                                 () -> {
                                                     Config config = new Config();
                                                     config.data_file_directories = new String[]{ dataDir.absolutePath() };
+                                                    config.default_compaction = new ParameterizedClass("LeveledCompactionStrategy");
+                                                    config.num_tokens = 1;
                                                     return config;
                                                 });
+        DatabaseDescriptor.setDataDirectories(new File[] { dataDir});
         CassandraRelevantProperties.FORCE_LOAD_LOCAL_KEYSPACES.setBoolean(true);
         oldPartitioner = DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
         Keyspace.setInitialized();
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAIClientTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAIClientTest.java
new file mode 100644
index 000000000000..b8c967fe39bf
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAIClientTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import org.junit.After;
+import org.junit.Before;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.util.File;
+
+public class CQLSSTableWriterSAIClientTest extends CQLSSTableWriterSAITest
+{
+    private IPartitioner oldPartitioner;
+
+    @Before
+    public void setUp()
+    {
+        DatabaseDescriptor.clientInitialization(true,
+                                                () -> {
+                                                    Config config = new Config();
+                                                    config.data_file_directories = new String[]{ dataDir.absolutePath() };
+                                                    config.default_compaction = new ParameterizedClass("LeveledCompactionStrategy");
+                                                    config.num_tokens = 1;
+                                                    return config;
+                                                });
+        DatabaseDescriptor.setDataDirectories(new File[] { dataDir});
+        CassandraRelevantProperties.FORCE_LOAD_LOCAL_KEYSPACES.setBoolean(true);
+        oldPartitioner = DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
+        Keyspace.setInitialized();
+    }
+
+    @After
+    public void tearDown()
+    {
+        if (oldPartitioner != null)
+            DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAIDaemonTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAIDaemonTest.java
new file mode 100644
index 000000000000..46bea95edbb4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAIDaemonTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.service.StorageService;
+
+/**
+ * The test/conf/cassandra.yaml uses the ByteOrderedPartitioner, but SAI does npt support it, so
+ * in this test we use the Murmur3Partitioner.
+ */
+public class CQLSSTableWriterSAIDaemonTest extends CQLSSTableWriterSAITest
+{
+    @BeforeClass
+    public static void setup() throws Exception
+    {
+        DatabaseDescriptor.daemonInitialization(() -> {
+            Config config = DatabaseDescriptor.loadConfig();
+            config.partitioner = Murmur3Partitioner.class.getName();
+            return config;
+        });
+        CommitLog.instance.start();
+        SchemaLoader.cleanupAndLeaveDirs();
+        Keyspace.setInitialized();
+        StorageService.instance.initServer();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAITest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAITest.java
new file mode 100644
index 000000000000..bac9026afea4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterSAITest.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.ClientState;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * SAI specific tests for {@link CQLSSTableWriter}. These tests have been moved from CQLSSTableWriterTest to isolate
+ * the SAI tests from the rest of the CQLSSTableWriter tests because the SAI tests require the use of the
+ * Murmur3Partitioner when executed with daemonInitialization in CQLSSTableWriterSATDaemonTest. See CNNDB-13401.
+ * <p>
+ * Please note: most tests here both create sstables and try to load them, so for the last part, we need to make sure
+ * we have properly "loaded" the table (which we do with {@link SchemaLoader#load(String, String, String...)}). But
+ * a small subtlety is that this <b>must</b> be called before we call {@link CQLSSTableWriter#builder} because
+ * otherwise the guardrail validation in {@link CreateTableStatement#validate(ClientState)} ends up breaking because
+ * the {@link ColumnFamilyStore} is not loaded yet. This would not be a problem in real usage of
+ * {@link CQLSSTableWriter} because the later only calls {@link DatabaseDescriptor#clientInitialization}, not
+ * {@link DatabaseDescriptor#daemonInitialization}, so said guardrail validation don't execute, but this test does
+ * manually call {@link DatabaseDescriptor#daemonInitialization} so...
+ */
+@Ignore
+public abstract class CQLSSTableWriterSAITest
+{
+    private static final AtomicInteger idGen = new AtomicInteger(0);
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
+    private String keyspace;
+    protected String table;
+    protected String qualifiedTable;
+    protected File dataDir;
+
+    @Before
+    public void perTestSetup() throws IOException
+    {
+        keyspace = "cql_keyspace" + idGen.incrementAndGet();
+        table = "table" + idGen.incrementAndGet();
+        qualifiedTable = keyspace + '.' + table;
+        dataDir = new File(tempFolder.getRoot().getAbsolutePath() + File.pathSeparator() + keyspace + File.pathSeparator() + table);
+        assert dataDir.tryCreateDirectories();
+    }
+
+    @Test
+    public void testWriteWithSAI() throws Exception
+    {
+        writeWithSaiInternal();
+        writeWithSaiInternal();
+    }
+
+    private void writeWithSaiInternal() throws Exception
+    {
+        String schema = "CREATE TABLE " + qualifiedTable + " ("
+                        + "  k int PRIMARY KEY,"
+                        + "  v1 text,"
+                        + "  v2 int )";
+
+        String v1Index = "CREATE INDEX idx1 ON " + qualifiedTable + " (v1) USING 'sai'";
+        String v2Index = "CREATE INDEX idx2 ON " + qualifiedTable + " (v2) USING 'sai'";
+
+        String insert = "INSERT INTO " + qualifiedTable + " (k, v1, v2) VALUES (?, ?, ?)";
+
+        CQLSSTableWriter writer = CQLSSTableWriter.builder()
+                                                  .inDirectory(dataDir)
+                                                  .forTable(schema)
+                                                  .using(insert)
+                                                  .withIndexes(v1Index, v2Index)
+                                                  .withBuildIndexes(true)
+                                                  .withPartitioner(Murmur3Partitioner.instance)
+                                                  .build();
+
+        int rowCount = 30_000;
+        for (int i = 0; i < rowCount; i++)
+            writer.addRow(i, UUID.randomUUID().toString(), i);
+
+        writer.close();
+
+        File[] dataFiles = dataDir.list(f -> f.name().endsWith('-' + BigFormat.Components.DATA.type.repr));
+        assertNotNull(dataFiles);
+
+        IndexDescriptor indexDescriptor = IndexDescriptor.empty(Descriptor.fromFile(dataFiles[0]));
+
+        IndexContext idx1 = createIndexContext("idx1", UTF8Type.instance);
+        IndexContext idx2 = createIndexContext("idx2", UTF8Type.instance);
+        HashSet<IndexContext> indices = new HashSet<>(Arrays.asList(idx1, idx2));
+        SSTableReader sstable = SSTableReader.openNoValidation(null, indexDescriptor.descriptor, writer.getMetadata());
+        indexDescriptor.reload(sstable, indices);
+
+        assertTrue(indexDescriptor.perIndexComponents(idx1).isComplete());
+        assertTrue(indexDescriptor.perIndexComponents(idx2).isComplete());
+
+        if (PathUtils.isDirectory(dataDir.toPath()))
+            PathUtils.forEach(dataDir.toPath(), PathUtils::deleteRecursive);
+    }
+
+    @Test
+    public void testSkipBuildingIndexesWithSAI() throws Exception
+    {
+        String schema = "CREATE TABLE " + qualifiedTable + " ("
+                        + "  k int PRIMARY KEY,"
+                        + "  v1 text,"
+                        + "  v2 int )";
+
+        String v1Index = "CREATE INDEX idx1 ON " + qualifiedTable + " (v1) USING 'sai'";
+        String v2Index = "CREATE INDEX idx2 ON " + qualifiedTable + " (v2) USING 'sai'";
+
+        String insert = "INSERT INTO " + qualifiedTable + " (k, v1, v2) VALUES (?, ?, ?)";
+
+        CQLSSTableWriter writer = CQLSSTableWriter.builder()
+                                                  .inDirectory(dataDir)
+                                                  .forTable(schema)
+                                                  .using(insert)
+                                                  .withIndexes(v1Index, v2Index)
+                                                  // not building indexes here so no SAI components will be present
+                                                  .withBuildIndexes(false)
+                                                  .build();
+
+        int rowCount = 30_000;
+        for (int i = 0; i < rowCount; i++)
+            writer.addRow(i, UUID.randomUUID().toString(), i);
+
+        writer.close();
+
+        File[] dataFiles = dataDir.list(f -> f.name().endsWith('-' + BigFormat.Components.DATA.type.repr));
+        assertNotNull(dataFiles);
+
+        // no indexes built due to withBuildIndexes set to false
+        IndexDescriptor indexDescriptor = IndexDescriptor.empty(Descriptor.fromFile(dataFiles[0]));
+
+        assertFalse(indexDescriptor.perIndexComponents(createIndexContext("idx1", UTF8Type.instance)).isComplete());
+        assertFalse(indexDescriptor.perIndexComponents(createIndexContext("idx2", UTF8Type.instance)).isComplete());
+    }
+
+    public IndexContext createIndexContext(String name, AbstractType<?> validator)
+    {
+        return new IndexContext(keyspace,
+                                table,
+                                TableId.generate(),
+                                UTF8Type.instance,
+                                new ClusteringComparator(),
+                                ColumnMetadata.regularColumn("sai", "internal", name, validator),
+                                IndexTarget.Type.SIMPLE,
+                                IndexMetadata.fromSchemaMetadata(name, IndexMetadata.Kind.CUSTOM, null),
+                                MockSchema.newCFS(keyspace));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
index de592283ccde..55fe433e3795 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
@@ -44,7 +44,9 @@
 import org.junit.rules.TemporaryFolder;
 
 import com.datastax.driver.core.utils.UUIDs;
+import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.functions.types.DataType;
@@ -52,18 +54,16 @@
 import org.apache.cassandra.cql3.functions.types.TypeCodec;
 import org.apache.cassandra.cql3.functions.types.UDTValue;
 import org.apache.cassandra.cql3.functions.types.UserType;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
-import org.apache.cassandra.index.sai.utils.IndexIdentifier;
 import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.PathUtils;
-import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.JavaDriverUtils;
@@ -75,6 +75,18 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+/**
+ * Tests for {@link CQLSSTableWriter}.
+ *
+ * Please note: most tests here both create sstables and try to load them, so for the last part, we need to make sure
+ * we have properly "loaded" the table (which we do with {@link SchemaLoader#load(String, String, String...)}). But
+ * a small subtlety is that this <b>must</b> be called before we call {@link CQLSSTableWriter#builder} because
+ * otherwise the guardrail validation in {@link CreateTableStatement#validate(ClientState)} ends up breaking because
+ * the {@link ColumnFamilyStore} is not loaded yet. This would not be a problem in real usage of
+ * {@link CQLSSTableWriter} because the later only calls {@link DatabaseDescriptor#clientInitialization}, not
+ * {@link DatabaseDescriptor#daemonInitialization}, so said guardrail validation don't execute, but this test does
+ * manually call {@link DatabaseDescriptor#daemonInitialization} so...
+ */
 @Ignore
 public abstract class CQLSSTableWriterTest
 {
@@ -110,6 +122,9 @@ public void testUnsortedWriter() throws Exception
                             + "  v2 int"
                             + ")";
             String insert = "INSERT INTO " + qualifiedTable + " (k, v1, v2) VALUES (?, ?, ?)";
+
+            SchemaLoader.load(keyspace, schema);
+
             CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                       .inDirectory(dataDir)
                                                       .forTable(schema)
@@ -164,6 +179,7 @@ public void testForbidCounterUpdates() throws Exception
         String insert = String.format("UPDATE " + qualifiedTable + " SET my_counter = my_counter - ? WHERE my_id = ?");
         try
         {
+            SchemaLoader.load(keyspace, schema);
             CQLSSTableWriter.builder().inDirectory(dataDir)
                             .forTable(schema)
                             .withPartitioner(Murmur3Partitioner.instance)
@@ -187,6 +203,7 @@ public void testSyncWithinPartition() throws Exception
                         + "  v blob"
                         + ")";
         String insert = "INSERT INTO " + qualifiedTable + " (k, v) VALUES (?, ?)";
+        SchemaLoader.load(keyspace, schema);
         CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .inDirectory(dataDir)
                                                   .using(insert)
@@ -214,6 +231,7 @@ public void testSyncNoEmptyRows() throws Exception
                         + "  PRIMARY KEY (k)"
                         + ")";
         String insert = "INSERT INTO " + qualifiedTable + " (k, c) VALUES (?, ?)";
+        SchemaLoader.load(keyspace, schema);
         CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .inDirectory(dataDir)
                                                   .forTable(schema)
@@ -547,10 +565,13 @@ public void testWritesWithUdts() throws Exception
                               + "  PRIMARY KEY (k)"
                               + ")";
 
+        String type1 = "CREATE TYPE " + keyspace + ".tuple2 (a int, b int)";
+        String type2 = "CREATE TYPE " + keyspace + ".tuple3 (a int, b int, c int)";
+        SchemaLoader.load(keyspace, schema, type1, type2);
         CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .inDirectory(dataDir)
-                                                  .withType("CREATE TYPE " + keyspace + ".tuple2 (a int, b int)")
-                                                  .withType("CREATE TYPE " + keyspace + ".tuple3 (a int, b int, c int)")
+                                                  .withType(type1)
+                                                  .withType(type2)
                                                   .forTable(schema)
                                                   .using("INSERT INTO " + keyspace + "." + table + " (k, v1, v2) " +
                                                          "VALUES (?, ?, ?)").build();
@@ -613,10 +634,13 @@ public void testWritesWithDependentUdts() throws Exception
                               + "  PRIMARY KEY (k)"
                               + ")";
 
+        String type1 = "CREATE TYPE " + keyspace + ".tuple2 (a int, b int)";
+        String type2 = "CREATE TYPE " + keyspace + ".nested_tuple (c int, tpl frozen<tuple2>)";
+        SchemaLoader.load(keyspace, schema, type1, type2);
         CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .inDirectory(dataDir)
-                                                  .withType("CREATE TYPE " + keyspace + ".nested_tuple (c int, tpl frozen<tuple2>)")
-                                                  .withType("CREATE TYPE " + keyspace + ".tuple2 (a int, b int)")
+                                                  .withType(type2)
+                                                  .withType(type1)
                                                   .forTable(schema)
                                                   .using("INSERT INTO " + keyspace + "." + table + " (k, v1) " +
                                                          "VALUES (?, ?)")
@@ -672,6 +696,7 @@ public void testUnsetValues() throws Exception
                               + "  PRIMARY KEY (k, c1, c2)"
                               + ")";
 
+        SchemaLoader.load(keyspace, schema);
         CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .inDirectory(dataDir)
                                                   .forTable(schema)
@@ -768,6 +793,7 @@ public void testUpdateStatement() throws Exception
                               + "  PRIMARY KEY (k, c1, c2)"
                               + ")";
 
+        SchemaLoader.load(keyspace, schema);
         CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .inDirectory(dataDir)
                                                   .forTable(schema)
@@ -810,6 +836,7 @@ public void testNativeFunctions() throws Exception
                               + "  PRIMARY KEY (k, c1, c2)"
                               + ")";
 
+        SchemaLoader.load(keyspace, schema);
         CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .inDirectory(dataDir)
                                                   .forTable(schema)
@@ -1318,95 +1345,6 @@ private void testWriters(String table1, String table2) throws IOException, Inval
         assertEquals(2, dataFiles.length);
     }
 
-    @Test
-    public void testWriteWithSAI() throws Exception
-    {
-        writeWithSaiInternal();
-        writeWithSaiInternal();
-    }
-
-    private void writeWithSaiInternal() throws Exception
-    {
-        String schema = "CREATE TABLE " + qualifiedTable + " ("
-                        + "  k int PRIMARY KEY,"
-                        + "  v1 text,"
-                        + "  v2 int )";
-
-        String v1Index = "CREATE INDEX idx1 ON " + qualifiedTable + " (v1) USING 'sai'";
-        String v2Index = "CREATE INDEX idx2 ON " + qualifiedTable + " (v2) USING 'sai'";
-
-        String insert = "INSERT INTO " + qualifiedTable + " (k, v1, v2) VALUES (?, ?, ?)";
-
-        CQLSSTableWriter writer = CQLSSTableWriter.builder()
-                                                  .inDirectory(dataDir)
-                                                  .forTable(schema)
-                                                  .using(insert)
-                                                  .withIndexes(v1Index, v2Index)
-                                                  .withBuildIndexes(true)
-                                                  .withPartitioner(Murmur3Partitioner.instance)
-                                                  .build();
-
-        int rowCount = 30_000;
-        for (int i = 0; i < rowCount; i++)
-            writer.addRow(i, UUID.randomUUID().toString(), i);
-
-        writer.close();
-
-        File[] dataFiles = dataDir.list(f -> f.name().endsWith('-' + BigFormat.Components.DATA.type.repr));
-        assertNotNull(dataFiles);
-
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(Descriptor.fromFile(dataFiles[0]),
-                                                                 Murmur3Partitioner.instance,
-                                                                 Schema.instance.getTableMetadata(keyspace, table).comparator);
-
-        assertTrue(indexDescriptor.isPerColumnIndexBuildComplete(new IndexIdentifier(keyspace, table, "idx1")));
-        assertTrue(indexDescriptor.isPerColumnIndexBuildComplete(new IndexIdentifier(keyspace, table, "idx2")));
-
-        if (PathUtils.isDirectory(dataDir.toPath()))
-            PathUtils.forEach(dataDir.toPath(), PathUtils::deleteRecursive);
-    }
-
-    @Test
-    public void testSkipBuildingIndexesWithSAI() throws Exception
-    {
-        String schema = "CREATE TABLE " + qualifiedTable + " ("
-                        + "  k int PRIMARY KEY,"
-                        + "  v1 text,"
-                        + "  v2 int )";
-
-        String v1Index = "CREATE INDEX idx1 ON " + qualifiedTable + " (v1) USING 'sai'";
-        String v2Index = "CREATE INDEX idx2 ON " + qualifiedTable + " (v2) USING 'sai'";
-
-        String insert = "INSERT INTO " + qualifiedTable + " (k, v1, v2) VALUES (?, ?, ?)";
-
-        CQLSSTableWriter writer = CQLSSTableWriter.builder()
-                                                  .inDirectory(dataDir)
-                                                  .forTable(schema)
-                                                  .using(insert)
-                                                  .withIndexes(v1Index, v2Index)
-                                                  // not building indexes here so no SAI components will be present
-                                                  .withBuildIndexes(false)
-                                                  .withPartitioner(Murmur3Partitioner.instance)
-                                                  .build();
-
-        int rowCount = 30_000;
-        for (int i = 0; i < rowCount; i++)
-            writer.addRow(i, UUID.randomUUID().toString(), i);
-
-        writer.close();
-
-        File[] dataFiles = dataDir.list(f -> f.name().endsWith('-' + BigFormat.Components.DATA.type.repr));
-        assertNotNull(dataFiles);
-
-        IndexDescriptor indexDescriptor = IndexDescriptor.create(Descriptor.fromFile(dataFiles[0]),
-                                                                 Murmur3Partitioner.instance,
-                                                                 Schema.instance.getTableMetadata(keyspace, table).comparator);
-
-        // no indexes built due to withBuildIndexes set to false
-        assertFalse(indexDescriptor.isPerColumnIndexBuildComplete(new IndexIdentifier(keyspace, table, "idx1")));
-        assertFalse(indexDescriptor.isPerColumnIndexBuildComplete(new IndexIdentifier(keyspace, table, "idx2")));
-    }
-
     protected void loadSSTables(File dataDir, String ksName)
     {
         ColumnFamilyStore cfs = Keyspace.openWithoutSSTables(ksName).getColumnFamilyStore(table);
@@ -1439,6 +1377,7 @@ public void run()
                             + "  PRIMARY KEY (k, v)"
                             + ")";
             String insert = "INSERT INTO " + qualifiedTable + " (k, v) VALUES (?, ?)";
+            SchemaLoader.load(keyspace, schema);
             CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                       .inDirectory(dataDir)
                                                       .forTable(schema)
diff --git a/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java b/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java
index e58238c444ba..e3422642e089 100644
--- a/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java
@@ -30,7 +30,7 @@
 import org.apache.cassandra.io.sstable.SSTableFormatTest.Format1;
 import org.apache.cassandra.io.sstable.SSTableFormatTest.Format2;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
 import org.mockito.Mockito;
 
 import static org.apache.cassandra.io.sstable.SSTableFormatTest.factory;
@@ -46,7 +46,7 @@ public class ComponentTest
     {
         DatabaseDescriptor.daemonInitialization(() -> {
             Config config = DatabaseDescriptor.loadConfig();
-            SSTableFormatTest.configure(new Config.SSTableConfig(), new BigFormat.BigFormatFactory(), factory("first", Format1.class), factory("second", Format2.class));
+            SSTableFormatTest.configure(new Config.SSTableConfig(), new BtiFormat.BtiFormatFactory(), factory("first", Format1.class), factory("second", Format2.class));
             return config;
         });
     }
@@ -70,8 +70,8 @@ public void testTypes()
 
         assertThatExceptionOfType(NullPointerException.class).isThrownBy(() -> Type.createSingleton(null, "-Three.db", true, Format1.class));
 
-        assertThat(Type.fromRepresentation("should be custom", BigFormat.getInstance())).isSameAs(Components.Types.CUSTOM);
-        assertThat(Type.fromRepresentation(Components.Types.TOC.repr, BigFormat.getInstance())).isSameAs(Components.Types.TOC);
+        assertThat(Type.fromRepresentation("should be custom", BtiFormat.getInstance())).isSameAs(Components.Types.CUSTOM);
+        assertThat(Type.fromRepresentation(Components.Types.TOC.repr, BtiFormat.getInstance())).isSameAs(Components.Types.TOC);
         assertThat(Type.fromRepresentation(t1.repr, DatabaseDescriptor.getSSTableFormats().get(FIRST))).isSameAs(t1);
         assertThat(Type.fromRepresentation(t2f1.repr, DatabaseDescriptor.getSSTableFormats().get(FIRST))).isSameAs(t2f1);
         assertThat(Type.fromRepresentation(t2f2.repr, DatabaseDescriptor.getSSTableFormats().get(SECOND))).isSameAs(t2f2);
diff --git a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
index 32b4813d29e6..abf2a66d7470 100644
--- a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
@@ -17,9 +17,9 @@
  */
 package org.apache.cassandra.io.sstable;
 
-import java.io.IOException;
 import java.util.UUID;
 
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -36,6 +36,7 @@
 import org.apache.cassandra.utils.Pair;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 
 public class DescriptorTest
@@ -45,7 +46,7 @@ public class DescriptorTest
     private final String cfId = ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(UUID.randomUUID()));
     private final File tempDataDir;
 
-    public DescriptorTest() throws IOException
+    public DescriptorTest()
     {
         // create CF directories, one without CFID and one with it
         tempDataDir = FileUtils.createTempFile("DescriptorTest", null).parent();
@@ -107,7 +108,10 @@ private void checkFromFilename(Descriptor original)
         assertEquals(original.cfname, desc.cfname);
         assertEquals(original.version, desc.version);
         assertEquals(original.id, desc.id);
+        assertEquals(original.fileFor(Components.DATA).toPath(), desc.pathFor(Components.DATA));
         assertEquals(Components.DATA, pair.right);
+
+        assertEquals(Components.DATA, Descriptor.validFilenameWithComponent(file.name()));
     }
 
     @Test
@@ -133,7 +137,26 @@ public void validateNames()
                                };
 
         for (String fileName : fileNames)
-            assertNotNull(Descriptor.fromFileWithComponent(new File(fileName), false).left);
+        {
+            Descriptor descriptor = Descriptor.fromFileWithComponent(new File(fileName), false).left;
+            assertNotNull(descriptor);
+            assertNotNull(fileName, descriptor.filenamePart());
+            assertNotNull(fileName, Descriptor.componentFromFile(new File(fileName)));
+            assertNotNull(fileName, Descriptor.validFilenameWithComponent(fileName));
+        }
+    }
+
+    @Test
+    public void testValidFilename()
+    {
+        String names[] = {
+        "system-schema_keyspaces-k234a-1-CompressionInfo.db",  "system-schema_keyspaces-ka-aa-Summary.db",
+        "system-schema_keyspaces-XXX-ka-1-Data.db",             "system-schema_keyspaces-k",
+        "system-schema_keyspace-ka-1-AAA-Data.db",  "system-schema-keyspace-ka-1-AAA-Data.db"
+        };
+
+        for (String name : names)
+            assertFalse(Descriptor.validFilename(name));
     }
 
     @Test
@@ -151,7 +174,9 @@ public void badNames()
             {
                 Descriptor d = Descriptor.fromFile(new File(name));
                 Assert.fail(name);
-            } catch (Throwable e) {
+            }
+            catch (Exception e)
+            {
                 //good
             }
         }
@@ -310,4 +335,15 @@ private void testKeyspaceTableParsing(String[] filePaths, String expectedKeyspac
             Assert.assertEquals(String.format("Expected table not found for %s", filePath), expectedTable, descriptor.cfname);
         }
     }
+
+    @Test
+    public void testLegacyDSEAPI()
+    {
+        File dir = new File(".");
+        Descriptor desc = new Descriptor(dir, "ks", "cf", new SequenceBasedSSTableId(1), BigFormat.getInstance());
+
+        assertEquals(dir.toCanonical().toPath(), desc.getDirectory());
+        assertEquals(desc.fileFor(Components.DATA).toPath(), desc.pathFor(Components.DATA));
+        assertEquals(desc.baseFileUri(), desc.baseFileURI());
+    }
 }
diff --git a/test/unit/org/apache/cassandra/io/sstable/EarlyOpenCachingTest.java b/test/unit/org/apache/cassandra/io/sstable/EarlyOpenCachingTest.java
new file mode 100644
index 000000000000..0f13e561e95e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/EarlyOpenCachingTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.SSTableAddingNotification;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SSTABLE_FORMAT_DEFAULT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class EarlyOpenCachingTest extends CQLTester
+{
+    @Parameterized.Parameters(name = "format={0}")
+    public static Collection<Object> generateParameters()
+    {
+        // We need to set up the class here, as the parameterized test runner will not call the @BeforeClass method
+        paramaterizedSetUpClass();
+
+        return Lists.newArrayList(DatabaseDescriptor.getSSTableFormats().values());
+    }
+
+    public static void paramaterizedSetUpClass()
+    {
+        CQLTester.setUpClass();
+        DatabaseDescriptor.setDiskAccessMode(Config.DiskAccessMode.standard);
+    }
+
+    @Parameterized.Parameter
+    public SSTableFormat<?, ?> format = DatabaseDescriptor.getSelectedSSTableFormat();
+
+    @BeforeClass
+    public static void setUpClass() // override CQLTester's setUpClass
+    {
+        // No-op, as initialization was done in paramaterizedSetUpClass, and we don't want to call CQLTester.setUpClass again
+    }
+
+    @Test
+    public void testFinalOpenRetainsCachedData() throws InterruptedException
+    {
+        SSTABLE_FORMAT_DEFAULT.setString(format.name());
+        createTable("CREATE TABLE %s (pkey text, ckey text, val blob, PRIMARY KEY (pkey, ckey))");
+
+        for (int i = 0; i < 800; i++)
+        {
+            String pkey = getRandom().nextAsciiString(10, 10);
+            for (int j = 0; j < 100; j++)
+                execute("INSERT INTO %s (pkey, ckey, val) VALUES (?, ?, ?)", pkey, "" + j, ByteBuffer.allocate(1000));
+        }
+        flush();
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+
+        AtomicInteger opened = new AtomicInteger(0);
+        AtomicBoolean completed = new AtomicBoolean(false);
+        Phaser phaser = new Phaser(1);
+        assertEquals(1, cfs.getLiveSSTables().size());
+        SSTableReader source = cfs.getLiveSSTables().iterator().next();
+
+        INotificationConsumer consumer = (notification, sender) -> {
+            System.out.println("Received notification: " + notification);
+            if (!(notification instanceof SSTableAddingNotification) || completed.get())
+                return;
+            SSTableAddingNotification n = (SSTableAddingNotification) notification;
+            SSTableReader s = n.adding.iterator().next();
+            readAllAndVerifyKeySpan(s);
+            assertTrue("Chunk cache is not used",
+                       ChunkCache.instance.sizeOfFile(s.getDataFile()) > 0);
+            phaser.register();
+            opened.incrementAndGet();
+            s.runOnClose(phaser::arriveAndDeregister);
+        };
+
+        SSTableReader finalReader;
+        cfs.getTracker().subscribe(consumer);
+        try (LifecycleTransaction txn = cfs.getTracker().tryModify(source, OperationType.COMPACTION);
+             SSTableRewriter writer = new SSTableRewriter(txn, 1000, 100L << 10, false))
+        {
+            writer.switchWriter(SSTableWriterTestBase.getWriter(format, cfs, cfs.getDirectories().getDirectoryForNewSSTables(), txn));
+            var iter = source.getScanner();
+            while (iter.hasNext())
+            {
+                var next = iter.next();
+                writer.append(next);
+            }
+            completed.set(true);
+            finalReader = writer.finish().iterator().next();
+        }
+        phaser.arriveAndAwaitAdvance();
+        assertTrue("No early opening occured", opened.get() > 0);
+
+        assertTrue("Chunk cache is not retained for early open sstable",
+                   ChunkCache.instance.sizeOfFile(finalReader.getDataFile()) > 0);
+        assertEquals(Sets.newHashSet(finalReader), cfs.getLiveSSTables());
+        readAllAndVerifyKeySpan(finalReader);
+    }
+
+    private static void readAllAndVerifyKeySpan(SSTableReader s)
+    {
+        DecoratedKey firstKey = null;
+        DecoratedKey lastKey = null;
+        for (var iter = s.getScanner(); iter.hasNext(); )
+        {
+            var partition = iter.next();
+            // consume all rows, so that the data is cached
+            partition.forEachRemaining(column -> {
+                // consume all columns
+            });
+            if (firstKey == null)
+                firstKey = partition.partitionKey();
+            lastKey = partition.partitionKey();
+        }
+        assertEquals("Simple scanner does not iterate all content", firstKey, s.first);
+        assertEquals("Simple scanner does not iterate all content", lastKey, s.last);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java b/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java
index 17fb0f28ec21..cc37309d567a 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java
@@ -131,11 +131,11 @@ private void scan(long partitionKibibytes, long totalKibibytes) throws Throwable
     private static void keyCacheMetrics(String title)
     {
         CacheMetrics metrics = CacheService.instance.keyCache.getMetrics();
-        System.out.println("Key cache metrics " + title + ": capacity:" + metrics.capacity.getValue() +
-                           " size:"+metrics.size.getValue()+
-                           " entries:" + metrics.entries.getValue() +
-                           " hit-rate:"+metrics.hitRate.getValue() +
-                           " one-min-rate:"+metrics.oneMinuteHitRate.getValue());
+        System.out.println("Key cache metrics " + title + ": capacity:" + metrics.capacity() +
+                           " size:"+metrics.size()+
+                           " entries:" + metrics.entries() +
+                           " hit-rate:"+metrics.hitRate() +
+                           " one-min-rate:"+metrics.hitOneMinuteRate());
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index 3a1bf38e0349..8f679fbc2f80 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@ -20,14 +20,15 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -43,7 +44,6 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SinglePartitionSliceCommandTest;
-import org.apache.cassandra.db.compaction.AbstractCompactionTask;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.repair.PendingAntiCompaction;
 import org.apache.cassandra.db.rows.RangeTombstoneMarker;
@@ -56,8 +56,8 @@
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
 import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
@@ -70,12 +70,15 @@
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.TimeUUID;
+import org.assertj.core.api.SoftAssertions;
 
 import static java.util.Collections.singleton;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_LEGACY_SSTABLE_ROOT;
+import static org.apache.cassandra.io.sstable.format.AbstractTestVersionSupportedFeatures.ALL_VERSIONS;
 import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
 import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -101,23 +104,15 @@ public class LegacySSTableTest
     // Get all versions up to the current one. Useful for testing in compatibility mode C18301
     private static String[] getValidLegacyVersions()
     {
-        String[] versions = {"oa", "da", "nb", "na", "me", "md", "mc", "mb", "ma"};
-        return Arrays.stream(versions).filter((v) -> v.compareTo(BigFormat.getInstance().getLatestVersion().toString()) <= 0).toArray(String[]::new);
+        String[] legacy = ALL_VERSIONS.stream()
+                                      .filter(v -> DatabaseDescriptor.getSelectedSSTableFormat().getVersion(v).isCompatible())
+                                      .filter(v -> new File(LEGACY_SSTABLE_ROOT, v + "/" + LEGACY_TABLES_KEYSPACE).isDirectory())
+                                      .toArray(String[]::new);
+        return legacy;
     }
 
     // 1200 chars
-    static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
-                                     "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
+    static final String longString = StringUtils.repeat("0123456789", 120);
 
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
@@ -132,7 +127,7 @@ public static void defineSchema() throws ConfigurationException
         StorageService.instance.initServer();
         Keyspace.setInitialized();
         createKeyspace();
-        
+
         legacyVersions = getValidLegacyVersions();
         for (String legacyVersion : legacyVersions)
         {
@@ -146,7 +141,7 @@ public void tearDown()
     {
         for (String legacyVersion : legacyVersions)
         {
-            truncateTables(legacyVersion);
+            truncateLegacyTables(legacyVersion);
         }
     }
 
@@ -161,156 +156,164 @@ protected Descriptor getDescriptor(String legacyVersion, String table) throws IO
     }
 
     @Test
-    public void testLoadLegacyCqlTables() throws Exception
+    public void testLoadLegacyCqlTables()
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(99999);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(99999);
         CacheService.instance.invalidateKeyCache();
         doTestLegacyCqlTables();
     }
 
     @Test
-    public void testLoadLegacyCqlTablesShallow() throws Exception
+    public void testLoadLegacyCqlTablesShallow()
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         CacheService.instance.invalidateKeyCache();
         doTestLegacyCqlTables();
     }
 
     @Test
-    public void testMutateMetadata() throws Exception
+    public void testMutateMetadata()
     {
+        SoftAssertions assertions = new SoftAssertions();
         // we need to make sure we write old version metadata in the format for that version
         for (String legacyVersion : legacyVersions)
-        {
-            logger.info("Loading legacy version: {}", legacyVersion);
-            truncateLegacyTables(legacyVersion);
-            loadLegacyTables(legacyVersion);
-            CacheService.instance.invalidateKeyCache();
+            assertions.assertThatCode(() -> {
+                logger.info("Loading legacy version: {}", legacyVersion);
+                truncateLegacyTables(legacyVersion);
+                loadLegacyTables(legacyVersion);
+                CacheService.instance.invalidateKeyCache();
 
-            for (ColumnFamilyStore cfs : Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStores())
-            {
-                for (SSTableReader sstable : cfs.getLiveSSTables())
+                for (ColumnFamilyStore cfs : Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStores())
                 {
-                    sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1234, NO_PENDING_REPAIR, false);
-                    sstable.reloadSSTableMetadata();
-                    assertEquals(1234, sstable.getRepairedAt());
-                    if (sstable.descriptor.version.hasPendingRepair())
-                        assertEquals(NO_PENDING_REPAIR, sstable.getPendingRepair());
-                }
+                    for (SSTableReader sstable : cfs.getLiveSSTables())
+                    {
+                        sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1234, NO_PENDING_REPAIR, false);
+                        sstable.reloadSSTableMetadata();
+                        assertEquals(1234, sstable.getRepairedAt());
+                        if (sstable.descriptor.version.hasPendingRepair())
+                            assertEquals(NO_PENDING_REPAIR, sstable.getPendingRepair());
+                    }
 
-                boolean isTransient = false;
-                for (SSTableReader sstable : cfs.getLiveSSTables())
-                {
-                    TimeUUID random = nextTimeUUID();
-                    sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, UNREPAIRED_SSTABLE, random, isTransient);
-                    sstable.reloadSSTableMetadata();
-                    assertEquals(UNREPAIRED_SSTABLE, sstable.getRepairedAt());
-                    if (sstable.descriptor.version.hasPendingRepair())
-                        assertEquals(random, sstable.getPendingRepair());
-                    if (sstable.descriptor.version.hasIsTransient())
-                        assertEquals(isTransient, sstable.isTransient());
-
-                    isTransient = !isTransient;
+                    boolean isTransient = false;
+                    for (SSTableReader sstable : cfs.getLiveSSTables())
+                    {
+                        TimeUUID random = nextTimeUUID();
+                        sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, UNREPAIRED_SSTABLE, random, isTransient);
+                        sstable.reloadSSTableMetadata();
+                        assertEquals(UNREPAIRED_SSTABLE, sstable.getRepairedAt());
+                        if (sstable.descriptor.version.hasPendingRepair())
+                            assertEquals(random, sstable.getPendingRepair());
+                        if (sstable.descriptor.version.hasIsTransient())
+                            assertEquals(isTransient, sstable.isTransient());
+
+                        isTransient = !isTransient;
+                    }
                 }
-            }
-        }
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
+        assertions.assertAll();
     }
 
     @Test
-    public void testMutateMetadataCSM() throws Exception
+    public void testMutateMetadataCSM()
     {
+        SoftAssertions assertions = new SoftAssertions();
         // we need to make sure we write old version metadata in the format for that version
         for (String legacyVersion : legacyVersions)
-        {
-            // Skip 2.0.1 sstables as it doesn't have repaired information
-            if (legacyVersion.equals("jb"))
-                continue;
-            truncateTables(legacyVersion);
-            loadLegacyTables(legacyVersion);
+            assertions.assertThatCode(() -> {
+                truncateLegacyTables(legacyVersion);
+                loadLegacyTables(legacyVersion);
 
-            for (ColumnFamilyStore cfs : Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStores())
-            {
-                // set pending
-                for (SSTableReader sstable : cfs.getLiveSSTables())
+                for (ColumnFamilyStore cfs : Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStores())
                 {
-                    TimeUUID random = nextTimeUUID();
-                    try
+                    // set pending
+                    for (SSTableReader sstable : cfs.getLiveSSTables())
                     {
-                        cfs.getCompactionStrategyManager().mutateRepaired(Collections.singleton(sstable), UNREPAIRED_SSTABLE, random, false);
-                        if (!sstable.descriptor.version.hasPendingRepair())
-                            fail("We should fail setting pending repair on unsupported sstables "+sstable);
+                        TimeUUID random = nextTimeUUID();
+                        try
+                        {
+                            cfs.mutateRepaired(Collections.singleton(sstable), UNREPAIRED_SSTABLE, random, false);
+                            if (!sstable.descriptor.version.hasPendingRepair())
+                                fail("We should fail setting pending repair on unsupported sstables " + sstable);
+                        }
+                        catch (IllegalStateException e)
+                        {
+                            if (sstable.descriptor.version.hasPendingRepair())
+                                fail("We should succeed setting pending repair on " + legacyVersion + " sstables, failed on " + sstable);
+                        }
                     }
-                    catch (IllegalStateException e)
+                    // set transient
+                    for (SSTableReader sstable : cfs.getLiveSSTables())
                     {
-                        if (sstable.descriptor.version.hasPendingRepair())
-                            fail("We should succeed setting pending repair on "+legacyVersion + " sstables, failed on "+sstable);
-                    }
-                }
-                // set transient
-                for (SSTableReader sstable : cfs.getLiveSSTables())
-                {
-                    try
-                    {
-                        cfs.getCompactionStrategyManager().mutateRepaired(Collections.singleton(sstable), UNREPAIRED_SSTABLE, nextTimeUUID(), true);
-                        if (!sstable.descriptor.version.hasIsTransient())
-                            fail("We should fail setting pending repair on unsupported sstables "+sstable);
-                    }
-                    catch (IllegalStateException e)
-                    {
-                        if (sstable.descriptor.version.hasIsTransient())
-                            fail("We should succeed setting pending repair on "+legacyVersion + " sstables, failed on "+sstable);
+                        try
+                        {
+                            cfs.mutateRepaired(Collections.singleton(sstable), UNREPAIRED_SSTABLE, nextTimeUUID(), true);
+                            if (!sstable.descriptor.version.hasIsTransient())
+                                fail("We should fail setting pending repair on unsupported sstables " + sstable);
+                        }
+                        catch (IllegalStateException e)
+                        {
+                            if (sstable.descriptor.version.hasIsTransient())
+                                fail("We should succeed setting pending repair on " + legacyVersion + " sstables, failed on " + sstable);
+                        }
                     }
                 }
-            }
-        }
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
+        assertions.assertAll();
     }
 
     @Test
-    public void testMutateLevel() throws Exception
+    public void testMutateLevel()
     {
         // we need to make sure we write old version metadata in the format for that version
+        SoftAssertions assertions = new SoftAssertions();
         for (String legacyVersion : legacyVersions)
-        {
-            logger.info("Loading legacy version: {}", legacyVersion);
-            truncateLegacyTables(legacyVersion);
-            loadLegacyTables(legacyVersion);
-            CacheService.instance.invalidateKeyCache();
+            assertions.assertThatCode(() -> {
+                logger.info("Loading legacy version: {}", legacyVersion);
+                truncateLegacyTables(legacyVersion);
+                loadLegacyTables(legacyVersion);
+                CacheService.instance.invalidateKeyCache();
 
-            for (ColumnFamilyStore cfs : Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStores())
-            {
-                for (SSTableReader sstable : cfs.getLiveSSTables())
+                for (ColumnFamilyStore cfs : Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStores())
                 {
-                    sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1234);
-                    sstable.reloadSSTableMetadata();
-                    assertEquals(1234, sstable.getSSTableLevel());
+                    for (SSTableReader sstable : cfs.getLiveSSTables())
+                    {
+                        sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1234);
+                        sstable.reloadSSTableMetadata();
+                        assertEquals(1234, sstable.getSSTableLevel());
+                    }
                 }
-            }
-        }
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
+        assertions.assertAll();
     }
 
-    private void doTestLegacyCqlTables() throws Exception
+    private void doTestLegacyCqlTables()
     {
+        SoftAssertions assertions = new SoftAssertions();
         for (String legacyVersion : legacyVersions)
-        {
-            logger.info("Loading legacy version: {}", legacyVersion);
-            truncateLegacyTables(legacyVersion);
-            loadLegacyTables(legacyVersion);
-            CacheService.instance.invalidateKeyCache();
-            long startCount = CacheService.instance.keyCache.size();
-            verifyReads(legacyVersion);
-            verifyCache(legacyVersion, startCount);
-            compactLegacyTables(legacyVersion);
-        }
+            assertions.assertThatCode(() -> {
+                logger.info("Loading legacy version: {}", legacyVersion);
+                truncateLegacyTables(legacyVersion);
+                loadLegacyTables(legacyVersion);
+                CacheService.instance.invalidateKeyCache();
+                long startCount = CacheService.instance.keyCache.size();
+                verifyReads(legacyVersion);
+                if (Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)).getLiveSSTables().stream().anyMatch(sstr -> BigFormat.is(sstr.descriptor.getFormat())))
+                    verifyCache(legacyVersion, startCount);
+                compactLegacyTables(legacyVersion);
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
+        assertions.assertAll();
     }
 
     @Test
-    public void testStreamLegacyCqlTables() throws Exception
+    public void testStreamLegacyCqlTables()
     {
+        SoftAssertions assertions = new SoftAssertions();
         for (String legacyVersion : legacyVersions)
-        {
-            streamLegacyTables(legacyVersion);
-            verifyReads(legacyVersion);
-        }
+            assertions.assertThatCode(() -> {
+                streamLegacyTables(legacyVersion);
+                verifyReads(legacyVersion);
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
+        assertions.assertAll();
     }
 
     @Test
@@ -335,89 +338,96 @@ public void testInaccurateSSTableMinMax() throws Exception
     }
 
     @Test
-    public void testVerifyOldSSTables() throws IOException
+    public void testVerifyOldSSTables()
     {
+        SoftAssertions assertions = new SoftAssertions();
         for (String legacyVersion : legacyVersions)
-        {
-            ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
-            loadLegacyTable("legacy_%s_simple", legacyVersion);
+            assertions.assertThatCode(() -> {
+                ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
+                loadLegacyTable("legacy_%s_simple", legacyVersion);
 
-            for (SSTableReader sstable : cfs.getLiveSSTables())
-            {
-                try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkVersion(true).build()))
-                {
-                    verifier.verify();
-                    if (!sstable.descriptor.version.isLatestVersion())
-                        fail("Verify should throw RuntimeException for old sstables "+sstable);
-                }
-                catch (RuntimeException e)
-                {}
-            }
-            // make sure we don't throw any exception if not checking version:
-            for (SSTableReader sstable : cfs.getLiveSSTables())
-            {
-                try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkVersion(false).build()))
+                for (SSTableReader sstable : cfs.getLiveSSTables())
                 {
-                    verifier.verify();
+                    try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkVersion(true).build()))
+                    {
+                        verifier.verify();
+                        if (!sstable.descriptor.version.isLatestVersion())
+                            fail("Verify should throw RuntimeException for old sstables " + sstable);
+                    }
+                    catch (RuntimeException e)
+                    {
+                    }
                 }
-                catch (Throwable e)
+                // make sure we don't throw any exception if not checking version:
+                for (SSTableReader sstable : cfs.getLiveSSTables())
                 {
-                    fail("Verify should throw RuntimeException for old sstables "+sstable);
+                    try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkVersion(false).build()))
+                    {
+                        verifier.verify();
+                    }
+                    catch (Throwable e)
+                    {
+                        fail("Verify should throw RuntimeException for old sstables " + sstable);
+                    }
                 }
-            }
-        }
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
+        assertions.assertAll();
     }
 
     @Test
-    public void testPendingAntiCompactionOldSSTables() throws Exception
+    public void testPendingAntiCompactionOldSSTables()
     {
+        SoftAssertions assertions = new SoftAssertions();
         for (String legacyVersion : legacyVersions)
-        {
-            ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
-            loadLegacyTable("legacy_%s_simple", legacyVersion);
-
-            boolean shouldFail = !cfs.getLiveSSTables().stream().allMatch(sstable -> sstable.descriptor.version.hasPendingRepair());
-            IPartitioner p = Iterables.getFirst(cfs.getLiveSSTables(), null).getPartitioner();
-            Range<Token> r = new Range<>(p.getMinimumToken(), p.getMinimumToken());
-            PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, singleton(r), nextTimeUUID(), 0, 0);
-            PendingAntiCompaction.AcquireResult res = acquisitionCallable.call();
-            assertEquals(shouldFail, res == null);
-            if (res != null)
-                res.abort();
-        }
+            assertions.assertThatCode(() -> {
+                ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
+                loadLegacyTable("legacy_%s_simple", legacyVersion);
+
+                boolean shouldFail = !cfs.getLiveSSTables().stream().allMatch(sstable -> sstable.descriptor.version.hasPendingRepair());
+                IPartitioner p = Iterables.getFirst(cfs.getLiveSSTables(), null).getPartitioner();
+                Range<Token> r = new Range<>(p.getMinimumToken(), p.getMinimumToken());
+                PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, singleton(r), nextTimeUUID(), 0, 0);
+                PendingAntiCompaction.AcquireResult res = acquisitionCallable.call();
+                assertEquals(shouldFail, res == null);
+                if (res != null)
+                    res.abort();
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
+        assertions.assertAll();
     }
 
     @Test
-    public void testAutomaticUpgrade() throws Exception
+    public void testAutomaticUpgrade()
     {
+        SoftAssertions assertions = new SoftAssertions();
         for (String legacyVersion : legacyVersions)
-        {
-            logger.info("Loading legacy version: {}", legacyVersion);
-            truncateLegacyTables(legacyVersion);
-            loadLegacyTables(legacyVersion);
-            ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
-            AbstractCompactionTask act = cfs.getCompactionStrategyManager().getNextBackgroundTask(0);
-            // there should be no compactions to run with auto upgrades disabled:
-            assertEquals(null, act);
-        }
+            assertions.assertThatCode(() -> {
+                logger.info("Loading legacy version: {}", legacyVersion);
+                truncateLegacyTables(legacyVersion);
+                loadLegacyTables(legacyVersion);
+                ColumnFamilyStore cfs = Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
+                // there should be no compactions to run with auto upgrades disabled:
+                assertTrue(cfs.getCompactionStrategy().getNextBackgroundTasks(0).isEmpty());
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
+        assertions.assertAll();
 
         DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(true);
         for (String legacyVersion : legacyVersions)
-        {
-            logger.info("Loading legacy version: {}", legacyVersion);
-            truncateLegacyTables(legacyVersion);
-            loadLegacyTables(legacyVersion);
-            ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
-            if (cfs.getLiveSSTables().stream().anyMatch(s -> !s.descriptor.version.isLatestVersion()))
-                assertTrue(cfs.metric.oldVersionSSTableCount.getValue() > 0);
-            while (cfs.getLiveSSTables().stream().anyMatch(s -> !s.descriptor.version.isLatestVersion()))
-            {
-                CompactionManager.instance.submitBackground(cfs);
-                Thread.sleep(100);
-            }
-            assertTrue(cfs.metric.oldVersionSSTableCount.getValue() == 0);
-        }
+            assertions.assertThatCode(() -> {
+                logger.info("Loading legacy version: {}", legacyVersion);
+                truncateLegacyTables(legacyVersion);
+                loadLegacyTables(legacyVersion);
+                ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion));
+                if (cfs.getLiveSSTables().stream().anyMatch(s -> !s.descriptor.version.isLatestVersion()))
+                    assertTrue(cfs.metric.oldVersionSSTableCount.getValue() > 0);
+                while (cfs.getLiveSSTables().stream().anyMatch(s -> !s.descriptor.version.isLatestVersion()))
+                {
+                    CompactionManager.instance.submitBackground(cfs);
+                    Thread.sleep(100);
+                }
+                assertEquals(0, (int) cfs.metric.oldVersionSSTableCount.getValue());
+            }).describedAs(legacyVersion).doesNotThrowAnyException();
         DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false);
+        assertions.assertAll();
     }
 
     private void streamLegacyTables(String legacyVersion) throws Exception
@@ -445,16 +455,18 @@ private void streamLegacyTable(String tablePattern, String legacyVersion) throws
         new StreamPlan(StreamOperation.OTHER).transferStreams(FBUtilities.getBroadcastAddressAndPort(), streams).execute().get();
     }
 
-    public static void truncateLegacyTables(String legacyVersion) throws Exception
+    public static void truncateLegacyTables(String legacyVersion)
     {
         logger.info("Truncating legacy version {}", legacyVersion);
         Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)).truncateBlocking();
         Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple_counter", legacyVersion)).truncateBlocking();
         Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust", legacyVersion)).truncateBlocking();
         Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust_counter", legacyVersion)).truncateBlocking();
+        CacheService.instance.invalidateCounterCache();
+        CacheService.instance.invalidateKeyCache();
     }
 
-    private static void compactLegacyTables(String legacyVersion) throws Exception
+    private static void compactLegacyTables(String legacyVersion)
     {
         logger.info("Compacting legacy version {}", legacyVersion);
         Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)).forceMajorCompaction();
@@ -463,16 +475,16 @@ private static void compactLegacyTables(String legacyVersion) throws Exception
         Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust_counter", legacyVersion)).forceMajorCompaction();
     }
 
-    public static void loadLegacyTables(String legacyVersion) throws Exception
+    public static void loadLegacyTables(String legacyVersion)
     {
-            logger.info("Preparing legacy version {}", legacyVersion);
-            loadLegacyTable("legacy_%s_simple", legacyVersion);
-            loadLegacyTable("legacy_%s_simple_counter", legacyVersion);
-            loadLegacyTable("legacy_%s_clust", legacyVersion);
-            loadLegacyTable("legacy_%s_clust_counter", legacyVersion);
+        logger.info("Preparing legacy version {}", legacyVersion);
+        loadLegacyTable("legacy_%s_simple", legacyVersion);
+        loadLegacyTable("legacy_%s_simple_counter", legacyVersion);
+        loadLegacyTable("legacy_%s_clust", legacyVersion);
+        loadLegacyTable("legacy_%s_clust_counter", legacyVersion);
     }
 
-    private static void verifyCache(String legacyVersion, long startCount) throws InterruptedException, java.util.concurrent.ExecutionException
+    private static void verifyCache(String legacyVersion, long startCount)
     {
         // Only perform test if format uses cache.
         SSTableReader sstable = Iterables.getFirst(Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)).getLiveSSTables(), null);
@@ -484,7 +496,14 @@ private static void verifyCache(String legacyVersion, long startCount) throws In
         //well as loads the correct number of keys
         long endCount = CacheService.instance.keyCache.size();
         Assert.assertTrue(endCount > startCount);
-        CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
+        try
+        {
+            CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get(60, TimeUnit.MINUTES);
+        }
+        catch (Exception e)
+        {
+            throw new AssertionError(e);
+        }
         CacheService.instance.invalidateKeyCache();
         Assert.assertEquals(startCount, CacheService.instance.keyCache.size());
         CacheService.instance.keyCache.loadSaved();
@@ -495,7 +514,7 @@ private static void verifyReads(String legacyVersion)
     {
         for (int ck = 0; ck < 50; ck++)
         {
-            String ckValue = Integer.toString(ck) + longString;
+            String ckValue = ck + longString;
             for (int pk = 0; pk < 5; pk++)
             {
                 logger.debug("for pk={} ck={}", pk, ck);
@@ -530,8 +549,8 @@ private static void readClusteringTable(String legacyVersion, int ck, String ckV
         rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust WHERE pk=? AND ck=?", legacyVersion), pkValue, ckValue);
         assertLegacyClustRows(1, rs);
 
-        String ckValue2 = Integer.toString(ck < 10 ? 40 : ck - 1) + longString;
-        String ckValue3 = Integer.toString(ck > 39 ? 10 : ck + 1) + longString;
+        String ckValue2 = (ck < 10 ? 40 : ck - 1) + longString;
+        String ckValue3 = (ck > 39 ? 10 : ck + 1) + longString;
         rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust WHERE pk=? AND ck IN (?, ?, ?)", legacyVersion), pkValue, ckValue, ckValue2, ckValue3);
         assertLegacyClustRows(3, rs);
     }
@@ -569,16 +588,6 @@ private static void createTables(String legacyVersion)
         QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust_counter (pk text, ck text, val counter, PRIMARY KEY (pk, ck))", legacyVersion));
     }
 
-    private static void truncateTables(String legacyVersion)
-    {
-        QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_simple", legacyVersion));
-        QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_simple_counter", legacyVersion));
-        QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust", legacyVersion));
-        QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust_counter", legacyVersion));
-        CacheService.instance.invalidateCounterCache();
-        CacheService.instance.invalidateKeyCache();
-    }
-
     private static void assertLegacyClustRows(int count, UntypedResultSet rs)
     {
         Assert.assertNotNull(rs);
@@ -592,7 +601,7 @@ private static void assertLegacyClustRows(int count, UntypedResultSet rs)
         }
     }
 
-    private static void loadLegacyTable(String tablePattern, String legacyVersion) throws IOException
+    private static void loadLegacyTable(String tablePattern, String legacyVersion)
     {
         String table = String.format(tablePattern, legacyVersion);
 
@@ -602,10 +611,20 @@ private static void loadLegacyTable(String tablePattern, String legacyVersion) t
 
         for (File cfDir : cfs.getDirectories().getCFDirectories())
         {
-            copySstablesToTestData(legacyVersion, table, cfDir);
+            try
+            {
+                copySstablesToTestData(legacyVersion, table, cfDir);
+            }
+            catch (IOException e)
+            {
+                throw new AssertionError(e);
+            }
         }
 
+        int s0 = cfs.getLiveSSTables().size();
         cfs.loadNewSSTables();
+        int s1 = cfs.getLiveSSTables().size();
+        assertThat(s1).isGreaterThan(s0);
     }
 
     /**
@@ -698,7 +717,7 @@ public static void copyFile(File cfDir, File file) throws IOException
             File target = new File(cfDir, file.name());
             int rd;
             try (FileInputStreamPlus is = new FileInputStreamPlus(file);
-                 FileOutputStreamPlus os = new FileOutputStreamPlus(target);) {
+                 FileOutputStreamPlus os = new FileOutputStreamPlus(target)) {
                 while ((rd = is.read(buf)) >= 0)
                     os.write(buf, 0, rd);
                 }
diff --git a/test/unit/org/apache/cassandra/io/sstable/MultipleSSTableFormatsTest.java b/test/unit/org/apache/cassandra/io/sstable/MultipleSSTableFormatsTest.java
new file mode 100644
index 000000000000..e7dbde01ef2a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/MultipleSSTableFormatsTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.assertj.core.api.Assertions;
+
+
+public class MultipleSSTableFormatsTest extends CQLTester
+{
+    private final static Logger logger = LoggerFactory.getLogger(MultipleSSTableFormatsTest.class);
+    private final static int cnt = 100;
+    private final static int overlap = 70;
+    private final static int deletionCount = 30;
+
+    private final long seed = System.nanoTime();
+    private Random random;
+
+    private SSTableFormat<?, ?> savedSSTableFormat;
+    
+    @Before
+    public void before() 
+    {
+        savedSSTableFormat = DatabaseDescriptor.getSelectedSSTableFormat();
+        random = new Random(seed);
+        logger.info("Using random seed = {}", seed);
+    }
+
+    @After
+    public void after() 
+    {
+        DatabaseDescriptor.setSelectedSSTableFormat(savedSSTableFormat);
+    }
+
+    private Map<Integer, Integer> createSSTables() 
+    {
+        Map<Integer, Integer> content = Maps.newHashMap();
+
+        createTable("CREATE TABLE %s (id INT, val INT, PRIMARY KEY (id))");
+        disableCompaction();
+
+        int offset = 0;
+        for (SSTableFormat<?, ?> format : DatabaseDescriptor.getSSTableFormats().values())
+        {
+            DatabaseDescriptor.setSelectedSSTableFormat(format);
+
+            for (int i = 0; i < cnt; i++)
+            {
+                int v = random.nextInt();
+                content.put(i + offset, v);
+                execute("INSERT INTO %s (id, val) VALUES (?, ?)", i + offset, v);
+            }
+            offset += cnt - overlap;
+
+            flush();
+        }
+
+        for (SSTableFormat<?, ?> format : DatabaseDescriptor.getSSTableFormats().values())
+        {
+            DatabaseDescriptor.setSelectedSSTableFormat(format);
+
+            for (int i = 0; i < deletionCount; i++)
+            {
+                int key = random.nextInt(offset + overlap);
+                content.remove(key);
+                execute("DELETE FROM %s WHERE id = ?", key);
+            }
+
+            flush();
+        }
+
+        List<SSTableFormat<?, ?>> createdFormats = createdFormats();
+        Assertions.assertThat(createdFormats).hasSameElementsAs(Sets.newHashSet(DatabaseDescriptor.getSSTableFormats().values()));
+
+        return content;
+    }
+
+    private void checkRead(Map<Integer, Integer> content) 
+    {
+        for (Map.Entry<Integer, Integer> entry : content.entrySet())
+        {
+            UntypedResultSet r = execute("SELECT val FROM %s WHERE id = ?", entry.getKey());
+            Assertions.assertThat(r.one().getInt("val")).isEqualTo(entry.getValue());
+        }
+
+        Iterator<UntypedResultSet.Row> it = execute("SELECT id, val FROM %s").iterator();
+        Map<Integer, Integer> results = Maps.newHashMap();
+        while (it.hasNext()) {
+            UntypedResultSet.Row row = it.next();
+            results.put(row.getInt("id"), row.getInt("val"));
+        }
+        Assertions.assertThat(results).isEqualTo(content);
+    }
+
+    @Test
+    public void testRead() throws Throwable
+    {
+        Map<Integer, Integer> content = createSSTables();
+        checkRead(content);
+    }
+
+    @Test
+    public void testCompactionToBigFormat() throws Throwable
+    {
+        testCompaction(BigFormat.getInstance());
+    }
+
+    @Test
+    public void testCompactionToBtiFormat() throws Throwable
+    {
+        testCompaction(BtiFormat.getInstance());
+    }
+
+    private void testCompaction(SSTableFormat<?, ?> format) throws Throwable
+    {
+        Map<Integer, Integer> content = createSSTables();
+        DatabaseDescriptor.setSelectedSSTableFormat(format);
+        compact();
+        List<SSTableFormat<?, ?>> createdFormats = createdFormats();
+        Assertions.assertThat(createdFormats).hasSize(1);
+        Assertions.assertThat(createdFormats.get(0)).isEqualTo(format);
+        checkRead(content);
+    }
+
+    private List<SSTableFormat<?, ?>> createdFormats()
+    {
+        return ColumnFamilyStore.getIfExists(KEYSPACE, currentTable())
+                                .getLiveSSTables()
+                                .stream()
+                                .map(sstr -> sstr.descriptor.getFormat())
+                                .collect(Collectors.toList());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java
index 0c79fef7fb28..65dec513cccc 100644
--- a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java
@@ -25,10 +25,12 @@
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -47,8 +49,11 @@ public class RangeAwareSSTableWriterTest
     @BeforeClass
     public static void defineSchema() throws Exception
     {
-        DatabaseDescriptor.daemonInitialization();
-        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        DatabaseDescriptor.daemonInitialization(() -> {
+            Config config = DatabaseDescriptor.loadConfig();
+            config.partitioner = Murmur3Partitioner.class.getName();
+            return config;
+        });
         SchemaLoader.cleanupAndLeaveDirs();
         Keyspace.setInitialized();
         StorageService.instance.initServer();
@@ -71,7 +76,7 @@ public void testAccessWriterBeforeAppend() throws IOException
         SchemaLoader.insertData(KEYSPACE1, CF_STANDARD, 0, 1);
         Util.flush(cfs);
 
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM, cfs.metadata);
 
         RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs,
                                                                      0,
@@ -79,6 +84,7 @@ public void testAccessWriterBeforeAppend() throws IOException
                                                                      null,
                                                                      false,
                                                                      DatabaseDescriptor.getSelectedSSTableFormat(),
+                                                                     IntervalSet.empty(),
                                                                      0,
                                                                      0,
                                                                      txn,
diff --git a/test/unit/org/apache/cassandra/io/sstable/ReducingKeyIteratorTest.java b/test/unit/org/apache/cassandra/io/sstable/ReducingKeyIteratorTest.java
new file mode 100644
index 000000000000..056c32dd4f69
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/ReducingKeyIteratorTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
+
+public class ReducingKeyIteratorTest
+{
+    public static final String KEYSPACE1 = "ReducingKeyIteratorTest";
+    public static final String CF_STANDARD = "Standard1";
+
+    @BeforeClass
+    public static void setup() throws Exception
+    {
+        SchemaLoader.prepareServer();
+        CompactionManager.instance.disableAutoCompaction();
+
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD));
+    }
+
+    @After
+    public void afterTest() throws Exception
+    {
+        ColumnFamilyStore store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
+        store.truncateBlocking();
+    }
+
+    @Test
+    public void testTotalAndReadBytesOneSSTable() throws IOException
+    {
+        testTotalAndReadBytes(1, 1000);
+    }
+
+    @Test
+    public void testTotalAndReadBytesManySSTables() throws IOException
+    {
+        testTotalAndReadBytes(10, 100);
+    }
+
+    public void testTotalAndReadBytes(int tableCount, int rowCount) throws IOException
+    {
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD);
+        LoggerFactory.getLogger(getClass()).info("Compression {}", store.metadata().params.compression.asMap());
+
+        for (int t = 0; t < tableCount; ++t)
+        {
+            for (int i = 0; i < rowCount; i++)
+            {
+                new RowUpdateBuilder(store.metadata(), i, String.valueOf(i))
+                .clustering("0")
+                .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+                .build()
+                .applyUnsafe();
+            }
+            store.forceBlockingFlush(UNIT_TESTS);
+        }
+
+        Set<SSTableReader> sstables = store.getLiveSSTables();
+        ReducingKeyIterator reducingIterator = new ReducingKeyIterator(sstables);
+
+        while (reducingIterator.hasNext())
+        {
+            Assert.assertTrue(reducingIterator.getTotalBytes() >= reducingIterator.getBytesRead());
+            reducingIterator.next();
+        }
+        Assert.assertEquals(reducingIterator.getTotalBytes(), reducingIterator.getBytesRead());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
index 21ac51ee865d..aa93c920d625 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
@@ -37,6 +37,7 @@
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Slices;
@@ -111,7 +112,7 @@ public static void setUp()
 
         truncate(cfs);
         File dir = cfs.getDirectories().getDirectoryForNewSSTables();
-        txn = LifecycleTransaction.offline(OperationType.WRITE);
+        txn = LifecycleTransaction.offline(OperationType.WRITE, cfs.metadata);
 
         // Setting up/writing large values is an expensive operation, we only want to do it once per run
         writer = getWriter(cfs, dir, txn);
@@ -156,6 +157,12 @@ public void testSSTableScanner() throws Throwable
         bruteForceCorruptionTest(ssTableReader, sstableScanner());
     }
 
+    @Test
+    public void testSSTableSimpleScanner() throws Throwable
+    {
+        bruteForceCorruptionTest(ssTableReader, sstableSimpleScanner());
+    }
+
     private void bruteForceCorruptionTest(SSTableReader ssTableReader, Consumer<SSTableReader> walker) throws Throwable
     {
         FileChannel fc = new File(ssTableReader.getFilename()).newReadWriteChannel();
@@ -182,7 +189,7 @@ private void bruteForceCorruptionTest(SSTableReader ssTableReader, Consumer<SSTa
             finally
             {
                 if (ChunkCache.instance != null)
-                    ChunkCache.instance.invalidateFile(ssTableReader.getFilename());
+                    ChunkCache.instance.invalidateFileNow(ssTableReader.getDataFile());
 
                 restore(fc, corruptionPosition, backup);
             }
@@ -193,6 +200,32 @@ private void bruteForceCorruptionTest(SSTableReader ssTableReader, Consumer<SSTa
     }
 
     private Consumer<SSTableReader> sstableScanner()
+    {
+        return (SSTableReader sstable) -> {
+            try (var scanner = sstable.partitionIterator(ColumnFilter.NONE, DataRange.allData(sstable.getPartitioner()), SSTableReadsListener.NOOP_LISTENER))
+            {
+                while (scanner.hasNext())
+                {
+                    try (UnfilteredRowIterator rowIter = scanner.next())
+                    {
+                        if (rowIter.hasNext())
+                        {
+                            Unfiltered unfiltered = rowIter.next();
+                            if (unfiltered.isRow())
+                            {
+                                Row row = (Row) unfiltered;
+                                assertEquals(2, row.clustering().size());
+                                // no-op read
+                            }
+                        }
+                    }
+
+                }
+            }
+        };
+    }
+
+    private Consumer<SSTableReader> sstableSimpleScanner()
     {
         return (SSTableReader sstable) -> {
             try (ISSTableScanner scanner = sstable.getScanner())
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java
index 4f3ff5a6fe50..24ff8cb1aa06 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java
@@ -70,6 +70,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatRuntimeException;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -79,7 +80,7 @@ public class SSTableFlushObserverTest
     private TableMetadata cfm;
     private File directory;
     private SSTableFormat<?, ?> sstableFormat;
-    private static Supplier<SSTableId> idGen;
+    private static Supplier<? extends SSTableId> idGen;
 
     @BeforeClass
     public static void initDD()
@@ -101,7 +102,7 @@ public void setUp() throws Exception
                      .addRegularColumn("age", Int32Type.instance)
                      .addRegularColumn("height", LongType.instance)
                      .build();
-        String sstableDirectory = DatabaseDescriptor.getAllDataFileLocations()[0];
+        String sstableDirectory = DatabaseDescriptor.getAllDataFileLocations()[0].toString();
         directory = new File(sstableDirectory + File.pathSeparator() + KS_NAME + File.pathSeparator() + CF_NAME);
         directory.deleteOnExit();
 
@@ -114,7 +115,7 @@ public void setUp() throws Exception
     @Test
     public void testFlushObserver() throws Exception
     {
-        try (LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.COMPACTION))
+        try (LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.COMPACTION, TableMetadataRef.forOfflineTools(cfm)))
         {
             FlushObserver observer = new FlushObserver();
 
@@ -124,7 +125,7 @@ public void testFlushObserver() throws Exception
                                                    cfm.name,
                                                    idGen.get());
             Index.Group indexGroup = mock(Index.Group.class);
-            when(indexGroup.getFlushObserver(any(), any(), any())).thenReturn(observer);
+            when(indexGroup.getFlushObserver(any(), any(), any(), anyLong())).thenReturn(observer);
             SSTableWriter.Builder<?, ?> writerBuilder = descriptor.getFormat().getWriterFactory().builder(descriptor)
                                                                   .setKeyCount(10)
                                                                   .setTableMetadataRef(TableMetadataRef.forOfflineTools(cfm))
@@ -178,14 +179,17 @@ public void testFlushObserver() throws Exception
             Assert.assertTrue(observer.isComplete);
             Assert.assertEquals(expected.size(), observer.rows.size());
 
-            for (Triple<ByteBuffer, Long, Long> e : observer.rows.keySet())
+            try (IKeyFetcher keyFetcher = reader.openKeyFetcher(true))
             {
-                ByteBuffer key = e.getLeft();
-                long indexPosition = e.getRight();
-
-                DecoratedKey indexKey = reader.keyAtPositionFromSecondaryIndex(indexPosition);
-                Assert.assertEquals(0, UTF8Type.instance.compare(key, indexKey.getKey()));
-                Assert.assertEquals(expected.get(key), observer.rows.get(e));
+                for (Triple<ByteBuffer, Long, Long> e : observer.rows.keySet())
+                {
+                    ByteBuffer key = e.getLeft();
+                    long indexPosition = e.getRight();
+
+                    DecoratedKey indexKey = keyFetcher.apply(indexPosition);
+                    Assert.assertEquals(0, UTF8Type.instance.compare(key, indexKey.getKey()));
+                    Assert.assertEquals(expected.get(key), observer.rows.get(e));
+                }
             }
         }
     }
@@ -193,14 +197,14 @@ public void testFlushObserver() throws Exception
     @Test
     public void testFailedInitialization() throws Exception
     {
-        try (LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.COMPACTION))
+        try (LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.COMPACTION, TableMetadataRef.forOfflineTools(cfm)))
         {
             FlushObserver observer1 = new FlushObserver();
             FlushObserver observer2 = new FlushObserver();
             Index.Group indexGroup1 = mock(Index.Group.class);
             Index.Group indexGroup2 = mock(Index.Group.class);
-            when(indexGroup1.getFlushObserver(any(), any(), any())).thenReturn(observer1);
-            when(indexGroup2.getFlushObserver(any(), any(), any())).thenReturn(observer2);
+            when(indexGroup1.getFlushObserver(any(), any(), any(), anyLong())).thenReturn(observer1);
+            when(indexGroup2.getFlushObserver(any(), any(), any(), anyLong())).thenReturn(observer2);
             observer2.failOnBegin = true;
 
             Descriptor descriptor = new Descriptor(sstableFormat.getLatestVersion(),
@@ -288,7 +292,7 @@ public void nextUnfilteredCluster(Unfiltered row)
         }
 
         @Override
-        public void complete()
+        public void complete(SSTable ssTable)
         {
             isComplete = true;
         }
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
new file mode 100644
index 000000000000..e0cc75d9008c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
@@ -0,0 +1,966 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.marshal.AbstractCompositeType;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.FrozenType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.io.sstable.format.big.BigFormat.Components.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test the functionality of {@link SSTableHeaderFix}.
+ * It writes an 'big-m' version sstable(s) and executes against these.
+ */
+@RunWith(Parameterized.class)
+public class SSTableHeaderFixTest
+{
+    static
+    {
+        DatabaseDescriptor.toolInitialization();
+    }
+
+    private File temporaryFolder;
+
+    @Parameterized.Parameter
+    public Supplier<? extends SSTableId> sstableIdGen;
+
+    @Parameterized.Parameters
+    public static Collection<Object[]> parameters()
+    {
+        return MockSchema.sstableIdGenerators();
+    }
+
+    @Before
+    public void setup()
+    {
+        MockSchema.sstableIds.clear();
+        MockSchema.sstableIdGenerator = sstableIdGen;
+        File f = FileUtils.createTempFile("SSTableUDTFixTest", "");
+        f.tryDelete();
+        f.tryCreateDirectories();
+        temporaryFolder = f;
+    }
+
+    @After
+    public void teardown()
+    {
+        FileUtils.deleteRecursive(temporaryFolder);
+    }
+
+    private static final AbstractType<?> udtPK = makeUDT("udt_pk");
+    private static final AbstractType<?> udtCK = makeUDT("udt_ck");
+    private static final AbstractType<?> udtStatic = makeUDT("udt_static");
+    private static final AbstractType<?> udtRegular = makeUDT("udt_regular");
+    private static final AbstractType<?> udtInner = makeUDT("udt_inner");
+    private static final AbstractType<?> udtNested = new UserType("ks",
+                                                                  ByteBufferUtil.bytes("udt_nested"),
+                                                                  Arrays.asList(new FieldIdentifier(ByteBufferUtil.bytes("a_field")),
+                                                                                new FieldIdentifier(ByteBufferUtil.bytes("a_udt"))),
+                                                                  Arrays.asList(UTF8Type.instance,
+                                                                                udtInner),
+                                                                  true);
+    private static final AbstractType<?> tupleInTuple = makeTuple(makeTuple());
+    private static final AbstractType<?> udtInTuple = makeTuple(udtInner);
+    private static final AbstractType<?> tupleInComposite = CompositeType.getInstance(UTF8Type.instance, makeTuple());
+    private static final AbstractType<?> udtInComposite = CompositeType.getInstance(UTF8Type.instance, udtInner);
+    private static final AbstractType<?> udtInList = ListType.getInstance(udtInner, true);
+    private static final AbstractType<?> udtInSet = SetType.getInstance(udtInner, true);
+    private static final AbstractType<?> udtInMap = MapType.getInstance(UTF8Type.instance, udtInner, true);
+    private static final AbstractType<?> udtInFrozenList = ListType.getInstance(udtInner, false);
+    private static final AbstractType<?> udtInFrozenSet = SetType.getInstance(udtInner, false);
+    private static final AbstractType<?> udtInFrozenMap = MapType.getInstance(UTF8Type.instance, udtInner, false);
+
+    private static AbstractType<?> makeUDT2(String udtName, boolean multiCell)
+    {
+        return new UserType("ks",
+                            ByteBufferUtil.bytes(udtName),
+                            Arrays.asList(new FieldIdentifier(ByteBufferUtil.bytes("a_field")),
+                                          new FieldIdentifier(ByteBufferUtil.bytes("a_udt"))),
+                            Arrays.asList(UTF8Type.instance,
+                                          udtInner),
+                            multiCell);
+    }
+
+    private static AbstractType<?> makeUDT(String udtName)
+    {
+        return new UserType("ks",
+                            ByteBufferUtil.bytes(udtName),
+                            Collections.singletonList(new FieldIdentifier(ByteBufferUtil.bytes("a_field"))),
+                            Collections.singletonList(UTF8Type.instance),
+                            true);
+    }
+
+    private static TupleType makeTuple()
+    {
+        return makeTuple(Int32Type.instance);
+    }
+
+    private static TupleType makeTuple(AbstractType<?> second)
+    {
+        return new TupleType(Arrays.asList(UTF8Type.instance,
+                                           second));
+    }
+
+    private static TupleType makeTupleSimple()
+    {
+        // TODO this should create a non-frozen tuple type for the sake of handling a dropped, non-frozen UDT
+        return new TupleType(Collections.singletonList(UTF8Type.instance));
+    }
+
+    private static final Version version = BigFormat.getInstance().getVersion("mc");
+
+    private TableMetadata tableMetadata;
+    private final Set<String> updatedColumns = new HashSet<>();
+
+    private ColumnMetadata getColDef(String n)
+    {
+        return tableMetadata.getColumn(ByteBufferUtil.bytes(n));
+    }
+
+    /**
+     * Very basic test whether {@link SSTableHeaderFix} detect a type mismatch (regular_c 'int' vs 'float').
+     */
+    @Test
+    public void verifyTypeMismatchTest() throws Exception
+    {
+        File dir = temporaryFolder;
+        File sstable = generateFakeSSTable(dir, 1);
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertFrozenUdt(header, false, true);
+
+        ColumnMetadata cd = getColDef("regular_c");
+        tableMetadata = tableMetadata.unbuild()
+                                     .removeRegularOrStaticColumn(cd.name)
+                                     .addRegularColumn("regular_c", FloatType.instance)
+                                     .build();
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertTrue(headerFix.hasError());
+        assertTrue(headerFix.hasChanges());
+
+        // must not have re-written the stats-component
+        header = readHeader(sstable);
+        assertFrozenUdt(header, false, true);
+    }
+
+    @Test
+    public void verifyTypeMatchTest() throws Exception
+    {
+        File dir = temporaryFolder;
+
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk", udtPK)
+                                                  .addClusteringColumn("ck", udtCK);
+        commonColumns(cols);
+        File sstable = buildFakeSSTable(dir, 1, cols, false);
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertFrozenUdt(header, false, true);
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertTrue(updatedColumns.isEmpty());
+        assertFalse(headerFix.hasError());
+        assertFalse(headerFix.hasChanges());
+
+        // must not have re-written the stats-component
+        header = readHeader(sstable);
+        assertFrozenUdt(header, false, true);
+    }
+
+    /**
+     * Simulates the case when an sstable contains a column not present in the schema, which can just be ignored.
+     */
+    @Test
+    public void verifyWithUnknownColumnTest() throws Exception
+    {
+        File dir = temporaryFolder;
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk", udtPK)
+                                                  .addClusteringColumn("ck", udtCK);
+        commonColumns(cols);
+        cols.addRegularColumn("solr_query", UTF8Type.instance);
+        File sstable = buildFakeSSTable(dir, 1, cols, true);
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertFrozenUdt(header, false, true);
+
+        ColumnMetadata cd = getColDef("solr_query");
+        tableMetadata = tableMetadata.unbuild()
+                                     .removeRegularOrStaticColumn(cd.name)
+                                     .build();
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertFalse(headerFix.hasError());
+        assertTrue(headerFix.hasChanges());
+
+        // must not have re-written the stats-component
+        header = readHeader(sstable);
+        assertFrozenUdt(header, true, true);
+    }
+
+    /**
+     * Simulates the case when an sstable contains a column not present in the table but as a target for an index.
+     * It can just be ignored.
+     */
+    @Test
+    public void verifyWithIndexedUnknownColumnTest() throws Exception
+    {
+        File dir = temporaryFolder;
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk", udtPK)
+                                                  .addClusteringColumn("ck", udtCK);
+        commonColumns(cols);
+        cols.addRegularColumn("solr_query", UTF8Type.instance);
+        File sstable = buildFakeSSTable(dir, 1, cols, true);
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertFrozenUdt(header, false, false);
+
+        ColumnMetadata cd = getColDef("solr_query");
+        tableMetadata = tableMetadata.unbuild()
+                                     .indexes(tableMetadata.indexes.with(IndexMetadata.fromSchemaMetadata("some search index", IndexMetadata.Kind.CUSTOM, Collections.singletonMap(IndexTarget.TARGET_OPTION_NAME, "solr_query"))))
+                                     .removeRegularOrStaticColumn(cd.name)
+                                     .build();
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertFalse(headerFix.hasError());
+        assertTrue(headerFix.hasChanges());
+
+        // must not have re-written the stats-component
+        header = readHeader(sstable);
+        assertFrozenUdt(header, true, true);
+    }
+
+    @Test
+    @Ignore("CNDB-10180")
+    public void complexTypeMatchTest() throws Exception
+    {
+        File dir = temporaryFolder;
+
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk", udtPK)
+                                                  .addClusteringColumn("ck", udtCK);
+        commonColumns(cols);
+        cols.addRegularColumn("tuple_in_tuple", tupleInTuple)
+            .addRegularColumn("udt_nested", udtNested)
+            .addRegularColumn("udt_in_tuple", udtInTuple)
+            .addRegularColumn("tuple_in_composite", tupleInComposite)
+            .addRegularColumn("udt_in_composite", udtInComposite)
+            .addRegularColumn("udt_in_list", udtInList)
+            .addRegularColumn("udt_in_set", udtInSet)
+            .addRegularColumn("udt_in_map", udtInMap)
+            .addRegularColumn("udt_in_frozen_list", udtInFrozenList)
+            .addRegularColumn("udt_in_frozen_set", udtInFrozenSet)
+            .addRegularColumn("udt_in_frozen_map", udtInFrozenMap);
+        File sstable = buildFakeSSTable(dir, 1, cols, true);
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertFrozenUdt(header, false, false);
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertFalse(headerFix.hasError());
+        assertTrue(headerFix.hasChanges());
+        assertEquals(Sets.newHashSet("pk", "ck", "regular_b", "static_b", "udt_nested", "udt_in_list", "udt_in_set", "udt_in_map"),
+                     updatedColumns);
+
+        // must not have re-written the stats-component
+        header = readHeader(sstable);
+        assertFrozenUdt(header, true, false);
+    }
+
+    @Test
+    public void complexTypeDroppedColumnsMatchTest() throws Exception
+    {
+        File dir = temporaryFolder;
+
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk", udtPK)
+                                                  .addClusteringColumn("ck", udtCK);
+        commonColumns(cols);
+        cols.addRegularColumn("tuple_in_tuple", tupleInTuple)
+            .addRegularColumn("udt_nested", udtNested)
+            .addRegularColumn("udt_in_tuple", udtInTuple)
+            .addRegularColumn("tuple_in_composite", tupleInComposite)
+            .addRegularColumn("udt_in_composite", udtInComposite)
+            .addRegularColumn("udt_in_list", udtInList)
+            .addRegularColumn("udt_in_set", udtInSet)
+            .addRegularColumn("udt_in_map", udtInMap)
+            .addRegularColumn("udt_in_frozen_list", udtInFrozenList)
+            .addRegularColumn("udt_in_frozen_set", udtInFrozenSet)
+            .addRegularColumn("udt_in_frozen_map", udtInFrozenMap);
+        File sstable = buildFakeSSTable(dir, 1, cols, true);
+
+        cols = tableMetadata.unbuild();
+        for (String col : new String[]{"tuple_in_tuple", "udt_nested", "udt_in_tuple",
+                                       "tuple_in_composite", "udt_in_composite",
+                                       "udt_in_list", "udt_in_set", "udt_in_map",
+                                       "udt_in_frozen_list", "udt_in_frozen_set", "udt_in_frozen_map"})
+        {
+            ColumnIdentifier ci = new ColumnIdentifier(col, true);
+            ColumnMetadata cd = getColDef(col);
+            AbstractType<?> dropType = cd.type.expandUserTypes();
+            cols.removeRegularOrStaticColumn(ci)
+                .recordColumnDrop(new ColumnMetadata(cd.ksName, cd.cfName, cd.name, dropType, cd.position(), cd.kind, null), FBUtilities.timestampMicros());
+        }
+        tableMetadata = cols.build();
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertFrozenUdt(header, false, false);
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertFalse(headerFix.hasError());
+        assertTrue(headerFix.hasChanges());
+        assertEquals(Sets.newHashSet("pk", "ck", "regular_b", "static_b", "udt_nested"), updatedColumns);
+
+        // must not have re-written the stats-component
+        header = readHeader(sstable);
+        // do not check the inner types, as the inner types were not fixed in the serialization-header (test thing)
+        assertFrozenUdt(header, true, false);
+    }
+
+    @Test
+    public void variousDroppedUserTypes() throws Exception
+    {
+        File dir = temporaryFolder;
+
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk", udtPK)
+                                                  .addClusteringColumn("ck", udtCK);
+
+        ColSpec[] colSpecs = new ColSpec[]
+                {
+                        // 'frozen<udt>' / live
+                        new ColSpec("frozen_udt_as_frozen_udt_live",
+                                    makeUDT2("frozen_udt_as_frozen_udt_live", false),
+                                    makeUDT2("frozen_udt_as_frozen_udt_live", false),
+                                    false,
+                                    false),
+                        // 'frozen<udt>' / live / as 'udt'
+                        new ColSpec("frozen_udt_as_unfrozen_udt_live",
+                                    makeUDT2("frozen_udt_as_unfrozen_udt_live", false),
+                                    makeUDT2("frozen_udt_as_unfrozen_udt_live", true),
+                                    false,
+                                    true),
+                        // 'frozen<udt>' / dropped
+                        new ColSpec("frozen_udt_as_frozen_udt_dropped",
+                                    makeUDT2("frozen_udt_as_frozen_udt_dropped", true).freeze().expandUserTypes(),
+                                    makeUDT2("frozen_udt_as_frozen_udt_dropped", false),
+                                    makeUDT2("frozen_udt_as_frozen_udt_dropped", false),
+                                    true,
+                                    false),
+                        // 'frozen<udt>' / dropped / as 'udt'
+                        new ColSpec("frozen_udt_as_unfrozen_udt_dropped",
+                                    makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true).freeze().expandUserTypes(),
+                                    makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true),
+                                    makeUDT2("frozen_udt_as_unfrozen_udt_dropped", false),
+                                    true,
+                                    true),
+                        // 'udt' / live
+                        new ColSpec("unfrozen_udt_as_unfrozen_udt_live",
+                                    makeUDT2("unfrozen_udt_as_unfrozen_udt_live", true),
+                                    makeUDT2("unfrozen_udt_as_unfrozen_udt_live", true),
+                                    false,
+                                    false),
+                        // 'udt' / dropped
+// TODO unable to test dropping a non-frozen UDT, as that requires an unfrozen tuple as well
+//                        new ColSpec("unfrozen_udt_as_unfrozen_udt_dropped",
+//                                    makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().expandUserTypes(),
+//                                    makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true),
+//                                    makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true),
+//                                    true,
+//                                    false),
+                        // 'frozen<tuple>' as 'TupleType(multiCell=false' (there is nothing like 'FrozenType(TupleType(')
+                        new ColSpec("frozen_tuple_as_frozen_tuple_live",
+                                    makeTupleSimple(),
+                                    makeTupleSimple(),
+                                    false,
+                                    false),
+                        // 'frozen<tuple>' as 'TupleType(multiCell=false' (there is nothing like 'FrozenType(TupleType(')
+                        new ColSpec("frozen_tuple_as_frozen_tuple_dropped",
+                                    makeTupleSimple(),
+                                    makeTupleSimple(),
+                                    true,
+                                    false)
+                };
+
+        Arrays.stream(colSpecs).forEach(c -> cols.addRegularColumn(c.name,
+                                                                   // use the initial column type for the serialization header header.
+                                                                   c.preFix));
+
+        Map<String, ColSpec> colSpecMap = Arrays.stream(colSpecs).collect(Collectors.toMap(c -> c.name, c -> c));
+        File sstable = buildFakeSSTable(dir, 1, cols, c -> {
+            ColSpec cs = colSpecMap.get(c.name.toString());
+            if (cs == null)
+                return c;
+            // update the column type in the schema to the "correct" one.
+            return c.withNewType(cs.schema);
+        });
+
+        Arrays.stream(colSpecs)
+              .filter(c -> c.dropped)
+              .forEach(c -> {
+                  ColumnMetadata cd = getColDef(c.name);
+                  tableMetadata = tableMetadata.unbuild()
+                                               .removeRegularOrStaticColumn(cd.name)
+                                               .recordColumnDrop(cd, FBUtilities.timestampMicros())
+                                               .build();
+              });
+
+        SerializationHeader.Component header = readHeader(sstable);
+        for (ColSpec colSpec : colSpecs)
+        {
+            AbstractType<?> hdrType = header.getRegularColumns().get(ByteBufferUtil.bytes(colSpec.name));
+            Assertions.assertThat(hdrType).isEqualTo(colSpec.preFix).describedAs("Column %s (%s != %s)", colSpec.name, hdrType.asCQL3Type().toSchemaString(), colSpec.preFix.asCQL3Type().toSchemaString());
+            assertEquals(colSpec.name, colSpec.preFix.isMultiCell(), hdrType.isMultiCell());
+        }
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertFalse(headerFix.hasError());
+        assertTrue(headerFix.hasChanges());
+        // Verify that all columns to fix are in the updatedColumns set (paranoid, yet)
+        Arrays.stream(colSpecs)
+              .filter(c -> c.mustFix)
+              .forEach(c -> assertTrue("expect " + c.name + " to be updated, but was not (" + updatedColumns + ")", updatedColumns.contains(c.name)));
+        // Verify that the number of updated columns matches the expected number of columns to fix
+        assertEquals(Arrays.stream(colSpecs).filter(c -> c.mustFix).count(), updatedColumns.size());
+
+        header = readHeader(sstable);
+        for (ColSpec colSpec : colSpecs)
+        {
+            AbstractType<?> hdrType = header.getRegularColumns().get(ByteBufferUtil.bytes(colSpec.name));
+            assertEquals(colSpec.name, colSpec.expect, hdrType);
+            assertEquals(colSpec.name, colSpec.expect.isMultiCell(), hdrType.isMultiCell());
+        }
+    }
+
+    private AbstractType<?> freezeOnlyNested(AbstractType<?> type)
+    {
+        ImmutableList.Builder<AbstractType<?>> builder = ImmutableList.builder();
+        for (AbstractType<?> subType : type.subTypes())
+            builder.add(subType.freeze());
+        return type.with(builder.build(), true);
+    }
+
+    static class ColSpec
+    {
+        final String name;
+        final AbstractType<?> schema;
+        final AbstractType<?> preFix;
+        final AbstractType<?> expect;
+        final boolean dropped;
+        final boolean mustFix;
+
+        ColSpec(String name, AbstractType<?> schema, AbstractType<?> preFix, boolean dropped, boolean mustFix)
+        {
+            this(name, schema, preFix, schema, dropped, mustFix);
+        }
+
+        ColSpec(String name, AbstractType<?> schema, AbstractType<?> preFix, AbstractType<?> expect, boolean dropped, boolean mustFix)
+        {
+            this.name = name;
+            this.schema = schema;
+            this.preFix = preFix;
+            this.expect = expect;
+            this.dropped = dropped;
+            this.mustFix = mustFix;
+        }
+    }
+
+    @Test
+    public void compositePartitionKey() throws Exception
+    {
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk1", UTF8Type.instance)
+                                                  .addPartitionKeyColumn("pk2", udtPK)
+                                                  .addClusteringColumn("ck", udtCK);
+        commonColumns(cols);
+
+        File dir = temporaryFolder;
+        File sstable = buildFakeSSTable(dir, 1, cols, true);
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertTrue(header.getKeyType() instanceof CompositeType);
+        CompositeType keyType = (CompositeType) header.getKeyType();
+        assertEquals(Arrays.asList(UTF8Type.instance, udtPK.freeze()), keyType.subTypes());
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertFalse(headerFix.hasError());
+        assertTrue(headerFix.hasChanges());
+        assertEquals(Sets.newHashSet("ck", "regular_b", "static_b"), updatedColumns);
+
+        header = readHeader(sstable);
+        assertTrue(header.getKeyType() instanceof CompositeType);
+        keyType = (CompositeType) header.getKeyType();
+        assertEquals(Arrays.asList(UTF8Type.instance, udtPK.freeze()), keyType.subTypes());
+    }
+
+    @Test
+    public void compositeClusteringKey() throws Exception
+    {
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk", udtPK)
+                                                  .addClusteringColumn("ck1", Int32Type.instance)
+                                                  .addClusteringColumn("ck2", udtCK);
+        commonColumns(cols);
+
+        File dir = temporaryFolder;
+        File sstable = buildFakeSSTable(dir, 1, cols, true);
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertEquals(Arrays.asList(Int32Type.instance, udtCK), header.getClusteringTypes());
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertFalse(headerFix.hasError());
+        assertTrue(headerFix.hasChanges());
+        assertEquals(Sets.newHashSet("pk", "ck2", "regular_b", "static_b"), updatedColumns);
+
+        header = readHeader(sstable);
+        assertEquals(Arrays.asList(Int32Type.instance, udtCK.freeze()), header.getClusteringTypes());
+    }
+
+    /**
+     * Check whether {@link SSTableHeaderFix} can operate on a single file.
+     */
+    @Test
+    public void singleFileUDTFixTest() throws Exception
+    {
+        File dir = temporaryFolder;
+        File sstable = generateFakeSSTable(dir, 1);
+
+        SerializationHeader.Component header = readHeader(sstable);
+        assertFrozenUdt(header, false, true);
+
+        SSTableHeaderFix headerFix = builder().withPath(sstable.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertTrue(headerFix.hasChanges());
+        assertFalse(headerFix.hasError());
+
+        header = readHeader(sstable);
+        assertFrozenUdt(header, true, true);
+    }
+
+    /**
+     * Check whether {@link SSTableHeaderFix} can operate on a file in a directory.
+     */
+    @Test
+    public void singleDirectoryUDTFixTest() throws Exception
+    {
+        File dir = temporaryFolder;
+        List<File> sstables = IntStream.range(1, 11)
+                                       .mapToObj(g -> generateFakeSSTable(dir, g))
+                                       .collect(Collectors.toList());
+
+        for (File sstable : sstables)
+        {
+            SerializationHeader.Component header = readHeader(sstable);
+            assertFrozenUdt(header, false, true);
+        }
+
+        SSTableHeaderFix headerFix = builder().withPath(dir.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertTrue(headerFix.hasChanges());
+        assertFalse(headerFix.hasError());
+
+        for (File sstable : sstables)
+        {
+            SerializationHeader.Component header = readHeader(sstable);
+            assertFrozenUdt(header, true, true);
+        }
+    }
+
+    /**
+     * Check whether {@link SSTableHeaderFix} can operate multiple, single files.
+     */
+    @Test
+    public void multipleFilesUDTFixTest() throws Exception
+    {
+        File dir = temporaryFolder;
+        List<File> sstables = IntStream.range(1, 11)
+                                       .mapToObj(g -> generateFakeSSTable(dir, g))
+                                       .collect(Collectors.toList());
+
+        for (File sstable : sstables)
+        {
+            SerializationHeader.Component header = readHeader(sstable);
+            assertFrozenUdt(header, false, true);
+        }
+
+        SSTableHeaderFix.Builder builder = builder();
+        sstables.stream().map(File::toPath).forEach(builder::withPath);
+        SSTableHeaderFix headerFix = builder.build();
+        headerFix.execute();
+
+        assertTrue(headerFix.hasChanges());
+        assertFalse(headerFix.hasError());
+
+        for (File sstable : sstables)
+        {
+            SerializationHeader.Component header = readHeader(sstable);
+            assertFrozenUdt(header, true, true);
+        }
+    }
+
+    /**
+     * Check whether {@link SSTableHeaderFix} can operate multiple files in a directory.
+     */
+    @Test
+    public void multipleFilesInDirectoryUDTFixTest() throws Exception
+    {
+        File dir = temporaryFolder;
+        List<File> sstables = IntStream.range(1, 11)
+                                       .mapToObj(g -> generateFakeSSTable(dir, g))
+                                       .collect(Collectors.toList());
+
+        for (File sstable : sstables)
+        {
+            SerializationHeader.Component header = readHeader(sstable);
+            assertFrozenUdt(header, false, true);
+        }
+
+        SSTableHeaderFix headerFix = builder().withPath(dir.toPath())
+                                              .build();
+        headerFix.execute();
+
+        assertTrue(headerFix.hasChanges());
+        assertFalse(headerFix.hasError());
+
+        for (File sstable : sstables)
+        {
+            SerializationHeader.Component header = readHeader(sstable);
+            assertFrozenUdt(header, true, true);
+        }
+    }
+
+    private static final Pattern p = Pattern.compile(".* Column '([^']+)' needs to be updated from type .*");
+
+    private SSTableHeaderFix.Builder builder()
+    {
+        updatedColumns.clear();
+        return SSTableHeaderFix.builder()
+                               .schemaCallback(() -> (desc) -> tableMetadata)
+                               .info(ln -> {
+                                   System.out.println("INFO: " + ln);
+                                   Matcher m = p.matcher(ln);
+                                   if (m.matches())
+                                       updatedColumns.add(m.group(1));
+                               })
+                               .warn(ln -> System.out.println("WARN: " + ln))
+                               .error(ln -> System.out.println("ERROR: " + ln));
+    }
+
+    private File generateFakeSSTable(File dir, int generation)
+    {
+        TableMetadata.Builder cols = TableMetadata.builder("ks", "cf")
+                                                  .addPartitionKeyColumn("pk", udtPK)
+                                                  .addClusteringColumn("ck", udtCK);
+        commonColumns(cols);
+        return buildFakeSSTable(dir, generation, cols, true);
+    }
+
+    private void commonColumns(TableMetadata.Builder cols)
+    {
+        cols.addRegularColumn("regular_a", UTF8Type.instance)
+            .addRegularColumn("regular_b", udtRegular)
+            .addRegularColumn("regular_c", Int32Type.instance)
+            .addStaticColumn("static_a", UTF8Type.instance)
+            .addStaticColumn("static_b", udtStatic)
+            .addStaticColumn("static_c", Int32Type.instance);
+    }
+
+    private File buildFakeSSTable(File dir, int generation, TableMetadata.Builder cols, boolean freezeInSchema)
+    {
+        return buildFakeSSTable(dir, generation, cols, freezeInSchema
+                                                       ? c -> c.withNewType(freezeUdt(c.type))
+                                                       : c -> c);
+    }
+
+    private File buildFakeSSTable(File dir, int generation, TableMetadata.Builder cols, Function<ColumnMetadata, ColumnMetadata> freezer)
+    {
+        TableMetadata headerMetadata = cols.build();
+
+        TableMetadata.Builder schemaCols = TableMetadata.builder("ks", "cf");
+        for (ColumnMetadata cm : cols.columns())
+            schemaCols.addColumn(freezer.apply(cm));
+        tableMetadata = schemaCols.build();
+
+        try
+        {
+
+            Descriptor desc = new Descriptor(version.version, dir, "ks", "cf", MockSchema.sstableId(generation), BigFormat.getInstance());
+
+            // Just create the component files - we don't really need those.
+            for (Component component : requiredComponents)
+                assertTrue(desc.fileFor(component).createFileIfNotExists());
+
+            AbstractType<?> partitionKey = headerMetadata.partitionKeyType;
+            List<AbstractType<?>> clusteringKey = headerMetadata.clusteringColumns()
+                                                                .stream()
+                                                                .map(cd -> cd.type)
+                                                                .collect(Collectors.toList());
+            LinkedHashMap<ByteBuffer, AbstractType<?>> staticColumns = headerMetadata.columns()
+                                                                           .stream()
+                                                                           .filter(cd -> cd.kind == ColumnMetadata.Kind.STATIC)
+                                                                           .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a, LinkedHashMap::new));
+            LinkedHashMap<ByteBuffer, AbstractType<?>> regularColumns = headerMetadata.columns()
+                                                                                      .stream()
+                                                                                      .filter(cd -> cd.kind == ColumnMetadata.Kind.REGULAR)
+                                                                                      .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a, LinkedHashMap::new));
+
+            File statsFile = desc.fileFor(STATS);
+            SerializationHeader.Component header = SerializationHeader.Component.buildComponentForTools(partitionKey,
+                                                                                                        clusteringKey,
+                                                                                                        staticColumns,
+                                                                                                        regularColumns,
+                                                                                                        EncodingStats.NO_STATS);
+
+            try (SequentialWriter out = new SequentialWriter(statsFile))
+            {
+                desc.getMetadataSerializer().serialize(Collections.singletonMap(MetadataType.HEADER, header), out, version);
+                out.finish();
+            }
+
+            return desc.fileFor(DATA);
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private AbstractType<?> freezeUdt(AbstractType<?> type)
+    {
+        if (type instanceof CollectionType)
+        {
+            if (type.getClass() == ListType.class)
+            {
+                ListType<?> cHeader = (ListType<?>) type;
+                return ListType.getInstance(freezeUdt(cHeader.getElementsType()), cHeader.isMultiCell());
+            }
+            else if (type.getClass() == SetType.class)
+            {
+                SetType<?> cHeader = (SetType<?>) type;
+                return SetType.getInstance(freezeUdt(cHeader.getElementsType()), cHeader.isMultiCell());
+            }
+            else if (type.getClass() == MapType.class)
+            {
+                MapType<?, ?> cHeader = (MapType<?, ?>) type;
+                return MapType.getInstance(freezeUdt(cHeader.getKeysType()), freezeUdt(cHeader.getValuesType()), cHeader.isMultiCell());
+            }
+        }
+        else if (type instanceof AbstractCompositeType)
+        {
+            if (type.getClass() == CompositeType.class)
+            {
+                CompositeType cHeader = (CompositeType) type;
+                return CompositeType.getInstance(cHeader.subTypes().stream().map(this::freezeUdt).collect(ImmutableList.toImmutableList()));
+            }
+        }
+        else if (type instanceof TupleType)
+        {
+            if (type.getClass() == UserType.class)
+            {
+                UserType cHeader = (UserType) type;
+                cHeader = cHeader.freeze();
+                return new UserType(cHeader.keyspace, cHeader.name, cHeader.fieldNames(),
+                                    cHeader.subTypes().stream().map(this::freezeUdt).collect(ImmutableList.toImmutableList()),
+                                    cHeader.isMultiCell());
+            }
+        }
+        return type;
+    }
+
+    private void assertFrozenUdt(SerializationHeader.Component header, boolean frozen, boolean checkInner)
+    {
+        AbstractType<?> keyType = header.getKeyType();
+        if (keyType instanceof CompositeType)
+        {
+            for (AbstractType<?> component : keyType.subTypes())
+                assertFrozenUdt("partition-key-component", component, frozen, checkInner);
+        }
+        assertFrozenUdt("partition-key", keyType, frozen, checkInner);
+
+        for (AbstractType<?> type : header.getClusteringTypes())
+            assertFrozenUdt("clustering-part", type, frozen, checkInner);
+        for (Map.Entry<ByteBuffer, AbstractType<?>> col : header.getStaticColumns().entrySet())
+            assertFrozenUdt(UTF8Type.instance.compose(col.getKey()), col.getValue(), frozen, checkInner);
+        for (Map.Entry<ByteBuffer, AbstractType<?>> col : header.getRegularColumns().entrySet())
+            assertFrozenUdt(UTF8Type.instance.compose(col.getKey()), col.getValue(), frozen, checkInner);
+    }
+
+    private void assertFrozenUdt(String name, AbstractType<?> type, boolean frozen, boolean checkInner)
+    {
+        if (type instanceof CompositeType)
+        {
+            if (checkInner)
+                for (AbstractType<?> component : type.subTypes())
+                    assertFrozenUdt(name, component, frozen, true);
+        }
+        else if (type instanceof CollectionType)
+        {
+            if (checkInner)
+            {
+                if (type instanceof MapType)
+                {
+                    MapType map = (MapType) type;
+                    // only descend for non-frozen types (checking frozen in frozen is just stupid)
+                    if (map.isMultiCell())
+                    {
+                        assertFrozenUdt(name + "<map-key>", map.getKeysType(), frozen, true);
+                        assertFrozenUdt(name + "<map-value>", map.getValuesType(), frozen, true);
+                    }
+                }
+                else if (type instanceof SetType)
+                {
+                    SetType set = (SetType) type;
+                    // only descend for non-frozen types (checking frozen in frozen is just stupid)
+                    if (set.isMultiCell())
+                        assertFrozenUdt(name + "<set>", set.getElementsType(), frozen, true);
+                }
+                else if (type instanceof ListType)
+                {
+                    ListType list = (ListType) type;
+                    // only descend for non-frozen types (checking frozen in frozen is just stupid)
+                    if (list.isMultiCell())
+                        assertFrozenUdt(name + "<list>", list.getElementsType(), frozen, true);
+                }
+            }
+        }
+        else if (type instanceof TupleType)
+        {
+            if (checkInner)
+            {
+                TupleType tuple = (TupleType) type;
+                // only descend for non-frozen types (checking frozen in frozen is just stupid)
+                if (tuple.isMultiCell())
+                    for (AbstractType<?> component : tuple.subTypes())
+                        assertFrozenUdt(name + "<tuple>", component, frozen, true);
+            }
+        }
+
+        if (type instanceof UserType)
+        {
+            String typeString = type.toString();
+            assertEquals(name + ": " + typeString, frozen, !type.isMultiCell());
+            if (typeString.startsWith(UserType.class.getName() + '('))
+                if (frozen)
+                    fail(name + ": " + typeString);
+            if (typeString.startsWith(FrozenType.class.getName() + '(' + UserType.class.getName() + '('))
+                if (!frozen)
+                    fail(name + ": " + typeString);
+        }
+    }
+
+    private SerializationHeader.Component readHeader(File sstable) throws Exception
+    {
+        Descriptor desc = Descriptor.fromFile(sstable);
+        return (SerializationHeader.Component) desc.getMetadataSerializer().deserialize(desc, MetadataType.HEADER);
+    }
+
+    private static final Component[] requiredComponents = new Component[]{ DATA, FILTER, PRIMARY_INDEX, TOC };
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableIdTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableIdTest.java
index 82fbf3253759..bea7aeefa76f 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableIdTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableIdTest.java
@@ -24,6 +24,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.CyclicBarrier;
@@ -37,12 +38,17 @@
 import com.google.common.primitives.UnsignedBytes;
 import org.junit.Test;
 
+import de.huxhorn.sulky.ulid.ULID;
 import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.utils.TimeUUID;
 import org.awaitility.Awaitility;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.quicktheories.QuickTheory.qt;
 import static org.quicktheories.generators.SourceDSL.longs;
 
@@ -60,6 +66,12 @@ public void testUUIDBasedIdProperties()
         testSSTableIdProperties(UUIDBasedSSTableId.Builder.instance);
     }
 
+    @Test
+    public void testULIDBasedIdProperties()
+    {
+        testSSTableIdProperties(ULIDBasedSSTableId.Builder.instance);
+    }
+
     private void testSSTableIdProperties(SSTableId.Builder<?> builder)
     {
         List<SSTableId> ids = Stream.generate(builder.generator(Stream.empty()))
@@ -99,22 +111,59 @@ public void testUUIDBytesSerDe()
         });
     }
 
+    @Test
+    public void testULIDBytesSerDe()
+    {
+        qt().forAll(longs().all(), longs().all()).checkAssert((msb, lsb) -> {
+            ULID.Value ulid = new ULID.Value(msb, lsb);
+            ULIDBasedSSTableId id = new ULIDBasedSSTableId(ulid);
+
+            testBytesSerialization(id);
+            testStringSerialization(id);
+        });
+    }
+
     private void testBytesSerialization(UUIDBasedSSTableId id)
     {
         ByteBuffer buf = id.asBytes();
         assertThat(buf.remaining()).isEqualTo(UUIDBasedSSTableId.BYTES_LEN);
         assertThat(UUIDBasedSSTableId.Builder.instance.isUniqueIdentifier(buf)).isTrue();
+        assertThat(ULIDBasedSSTableId.Builder.instance.isUniqueIdentifier(buf)).isTrue();
         assertThat(SequenceBasedSSTableId.Builder.instance.isUniqueIdentifier(buf)).isFalse();
         SSTableId fromBytes = SSTableIdFactory.instance.fromBytes(buf);
         assertThat(fromBytes).isEqualTo(id);
     }
 
+    private void testBytesSerialization(ULIDBasedSSTableId id)
+    {
+        ByteBuffer buf = id.asBytes();
+        assertThat(buf.remaining()).isEqualTo(ULIDBasedSSTableId.BYTES_LEN);
+        assertThat(UUIDBasedSSTableId.Builder.instance.isUniqueIdentifier(buf)).isTrue();
+        assertThat(ULIDBasedSSTableId.Builder.instance.isUniqueIdentifier(buf)).isTrue();
+        assertThat(SequenceBasedSSTableId.Builder.instance.isUniqueIdentifier(buf)).isFalse();
+        SSTableId fromBytes = SSTableIdFactory.instance.fromBytes(buf);
+        assertThat(fromBytes).isInstanceOf(UUIDBasedSSTableId.class); // UUID and ULID bytes representation are indistinguishable
+    }
+
     private void testStringSerialization(UUIDBasedSSTableId id)
     {
         String s = id.toString();
         assertThat(s).hasSize(UUIDBasedSSTableId.STRING_LEN);
         assertThat(s).matches(Pattern.compile("[0-9a-z]{4}_[0-9a-z]{4}_[0-9a-z]{18}"));
         assertThat(UUIDBasedSSTableId.Builder.instance.isUniqueIdentifier(s)).isTrue();
+        assertThat(ULIDBasedSSTableId.Builder.instance.isUniqueIdentifier(s)).isFalse();
+        assertThat(SequenceBasedSSTableId.Builder.instance.isUniqueIdentifier(s)).isFalse();
+        SSTableId fromString = SSTableIdFactory.instance.fromString(s);
+        assertThat(fromString).isEqualTo(id);
+    }
+
+    private void testStringSerialization(ULIDBasedSSTableId id)
+    {
+        String s = id.toString();
+        assertThat(s).hasSize(ULIDBasedSSTableId.STRING_LEN);
+        assertThat(s).matches(Pattern.compile("[0-9a-zA-Z]{26}"));
+        assertThat(UUIDBasedSSTableId.Builder.instance.isUniqueIdentifier(s)).isFalse();
+        assertThat(ULIDBasedSSTableId.Builder.instance.isUniqueIdentifier(s)).isTrue();
         assertThat(SequenceBasedSSTableId.Builder.instance.isUniqueIdentifier(s)).isFalse();
         SSTableId fromString = SSTableIdFactory.instance.fromString(s);
         assertThat(fromString).isEqualTo(id);
@@ -123,11 +172,14 @@ private void testStringSerialization(UUIDBasedSSTableId id)
     @Test
     public void testComparator()
     {
-        List<SSTableId> ids = new ArrayList<>(Collections.nCopies(300, null));
+        ULID ulid = new ULID();
+        List<SSTableId> ids = new ArrayList<>(Collections.nCopies(400, null));
         for (int i = 0; i < 100; i++)
         {
-            ids.set(i + 100, new SequenceBasedSSTableId(ThreadLocalRandom.current().nextInt(1000000)));
-            ids.set(i, new UUIDBasedSSTableId(TimeUUID.Generator.atUnixMillis(ThreadLocalRandom.current().nextLong(10000), 0)));
+            ids.set(i + 200, new SequenceBasedSSTableId(ThreadLocalRandom.current().nextInt(1000000)));
+            long ts = System.currentTimeMillis() + ThreadLocalRandom.current().nextLong(10000);
+            ids.set(i + 100, new UUIDBasedSSTableId(TimeUUID.Generator.atUnixMillis(ts)));
+            ids.set(i, new ULIDBasedSSTableId(ulid.nextValue(ts)));
         }
 
         List<SSTableId> shuffledIds = new ArrayList<>(ids);
@@ -141,10 +193,61 @@ public void testComparator()
 
         assertThat(sortedIds.subList(0, 100)).containsOnlyNulls();
         assertThat(sortedIds.subList(100, 200)).allMatch(id -> id instanceof SequenceBasedSSTableId);
-        assertThat(sortedIds.subList(200, 300)).allMatch(id -> id instanceof UUIDBasedSSTableId);
+        assertThat(sortedIds.subList(200, 400)).allMatch(id -> id instanceof UUIDBasedSSTableId || id instanceof ULIDBasedSSTableId);
 
         assertThat(sortedIds.subList(100, 200)).isSortedAccordingTo(Comparator.comparing(o -> ((SequenceBasedSSTableId) o)));
-        assertThat(sortedIds.subList(200, 300)).isSortedAccordingTo(Comparator.comparing(o -> ((UUIDBasedSSTableId) o)));
+        assertThat(sortedIds.subList(200, 400)).isSortedAccordingTo(SSTableIdFactory.COMPARATOR);
+        assertThat(sortedIds.subList(200, 400).stream().map(id -> {
+            if (id instanceof ULIDBasedSSTableId)
+            {
+                return ((ULIDBasedSSTableId) id).ulid.timestamp();
+            }
+            else
+            {
+                return ((UUIDBasedSSTableId) id).uuid.unixMillis();
+            }
+        })).isSorted();
+    }
+
+    @Test
+    public void testDefaultFactorySelection()
+    {
+        try
+        {
+            byte[] bytes = new byte[16];
+            new Random().nextBytes(bytes);
+
+            DatabaseDescriptor.clientInitialization();
+            DatabaseDescriptor.getRawConfig().uuid_sstable_identifiers_enabled = false;
+            CassandraRelevantProperties.SSTABLE_UUID_IMPL.setString("uuid");
+            assertThat(SSTableIdFactory.instance.defaultBuilder()).isInstanceOf(SequenceBasedSSTableId.Builder.class);
+            assertThat(SSTableIdFactory.instance.fromBytes(ByteBuffer.wrap(bytes))).isInstanceOf(UUIDBasedSSTableId.class);
+
+            DatabaseDescriptor.getRawConfig().uuid_sstable_identifiers_enabled = true;
+            CassandraRelevantProperties.SSTABLE_UUID_IMPL.setString("uuid");
+            assertThat(SSTableIdFactory.instance.defaultBuilder()).isInstanceOf(UUIDBasedSSTableId.Builder.class);
+            assertThat(SSTableIdFactory.instance.fromBytes(ByteBuffer.wrap(bytes))).isInstanceOf(UUIDBasedSSTableId.class);
+
+            DatabaseDescriptor.getRawConfig().uuid_sstable_identifiers_enabled = true;
+            CassandraRelevantProperties.SSTABLE_UUID_IMPL.setString("ulid");
+            assertThat(SSTableIdFactory.instance.defaultBuilder()).isInstanceOf(ULIDBasedSSTableId.Builder.class);
+            assertThat(SSTableIdFactory.instance.fromBytes(ByteBuffer.wrap(bytes))).isInstanceOf(ULIDBasedSSTableId.class);
+
+            DatabaseDescriptor.getRawConfig().uuid_sstable_identifiers_enabled = true;
+            CassandraRelevantProperties.SSTABLE_UUID_IMPL.setString("something");
+            assertThatExceptionOfType(IllegalArgumentException.class).isThrownBy(SSTableIdFactory.instance::defaultBuilder);
+            assertThatExceptionOfType(IllegalArgumentException.class).isThrownBy(() -> SSTableIdFactory.instance.fromBytes(ByteBuffer.wrap(bytes)));
+
+            DatabaseDescriptor.getRawConfig().uuid_sstable_identifiers_enabled = true;
+            System.getProperties().remove(CassandraRelevantProperties.SSTABLE_UUID_IMPL.getKey());
+            assertThat(SSTableIdFactory.instance.defaultBuilder()).isInstanceOf(UUIDBasedSSTableId.Builder.class);
+            assertThat(SSTableIdFactory.instance.fromBytes(ByteBuffer.wrap(bytes))).isInstanceOf(UUIDBasedSSTableId.class);
+        }
+        finally
+        {
+            DatabaseDescriptor.getRawConfig().uuid_sstable_identifiers_enabled = new Config().uuid_sstable_identifiers_enabled;
+            System.getProperties().remove(CassandraRelevantProperties.SSTABLE_UUID_IMPL.getKey());
+        }
     }
 
     private static <T extends SSTableId> void generatorFuzzTest(SSTableId.Builder<T> builder)
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
index 706de6bf74c4..5200fd9bd6d5 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
@@ -20,7 +20,6 @@
 import java.nio.ByteBuffer;
 import java.nio.charset.CharacterCodingException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 import org.junit.BeforeClass;
@@ -38,6 +37,7 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
+import static org.apache.cassandra.db.ClusteringPrefixTest.assertClusteringIsRetainable;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -221,9 +221,9 @@ public void trackMaxMinColNames() throws CharacterCodingException
         {
             assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0col100");
             assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "7col149");
-            // make sure the clustering values are minimised
-            assertTrue(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0).capacity() < 50);
-            assertTrue(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0).capacity() < 50);
+            // make sure stats don't reference native or off-heap data
+            assertClusteringIsRetainable(sstable.getSSTableMetadata().coveredClustering.start());
+            assertClusteringIsRetainable(sstable.getSSTableMetadata().coveredClustering.end());
         }
         String key = "row2";
 
@@ -244,8 +244,8 @@ public void trackMaxMinColNames() throws CharacterCodingException
             assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0col100");
             assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "9col298");
             // make sure stats don't reference native or off-heap data
-            assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.start().getBufferArray()));
-            assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.end().getBufferArray()));
+            assertClusteringIsRetainable(sstable.getSSTableMetadata().coveredClustering.start());
+            assertClusteringIsRetainable(sstable.getSSTableMetadata().coveredClustering.end());
         }
 
         key = "row3";
@@ -262,8 +262,8 @@ public void trackMaxMinColNames() throws CharacterCodingException
             assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0");
             assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "9col298");
             // make sure stats don't reference native or off-heap data
-            assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.start().getBufferArray()));
-            assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.end().getBufferArray()));
+            assertClusteringIsRetainable(sstable.getSSTableMetadata().coveredClustering.start());
+            assertClusteringIsRetainable(sstable.getSSTableMetadata().coveredClustering.end());
         }
     }
 
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index 9cb7ca3d14f5..05a597c6051d 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -20,19 +20,31 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -45,9 +57,11 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.BufferDecoratedKey;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.RowUpdateBuilder;
@@ -59,15 +73,19 @@
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
 import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.LocalPartitioner.LocalToken;
+import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader.PartitionPositionBounds;
 import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+import org.apache.cassandra.io.sstable.format.TOCComponent;
 import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
 import org.apache.cassandra.io.sstable.format.big.BigTableReader;
@@ -76,19 +94,33 @@
 import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport;
 import org.apache.cassandra.io.sstable.keycache.KeyCache;
 import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.MmappedRegions;
 import org.apache.cassandra.io.util.PageAware;
 import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.CompressionParams;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.utils.BloomCalculations;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
 import org.mockito.Mockito;
 
 import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BF_FP_CHANCE_TOLERANCE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BF_RECREATE_ON_FP_CHANCE_CHANGE;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
+import static org.apache.cassandra.schema.CompressionParams.DEFAULT_CHUNK_LENGTH;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -106,6 +138,7 @@ public class SSTableReaderTest
     public static final String CF_INDEXED = "Indexed1";
     public static final String CF_STANDARD_LOW_INDEX_INTERVAL = "StandardLowIndexInterval";
     public static final String CF_STANDARD_SMALL_BLOOM_FILTER = "StandardSmallBloomFilter";
+    public static final String CF_STANDARD_NO_BLOOM_FILTER = "StandardNoBloomFilter";
 
     private IPartitioner partitioner;
 
@@ -121,7 +154,9 @@ public static void defineSchema() throws Exception
         SchemaLoader.createKeyspace(KEYSPACE1,
                                     KeyspaceParams.simple(1),
                                     SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
-                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)
+                                                .minIndexInterval(8)
+                                                .maxIndexInterval(8),  // ensure close key count estimation
                                     SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3),
                                     SchemaLoader.standardCFMD(KEYSPACE1, CF_MOVE_AND_OPEN),
                                     SchemaLoader.standardCFMD(KEYSPACE1, CF_COMPRESSED).compression(CompressionParams.DEFAULT),
@@ -133,12 +168,21 @@ public static void defineSchema() throws Exception
                                     SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_SMALL_BLOOM_FILTER)
                                                 .minIndexInterval(4)
                                                 .maxIndexInterval(4)
-                                                .bloomFilterFpChance(0.99));
-        
+                                                .bloomFilterFpChance(0.99),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_NO_BLOOM_FILTER)
+                                                .bloomFilterFpChance(1));
+
         // All tests in this class assume auto-compaction is disabled.
         CompactionManager.instance.disableAutoCompaction();
     }
 
+    @After
+    public void Cleanup() {
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlocking();
+        BF_RECREATE_ON_FP_CHANCE_CHANGE.setBoolean(false);
+    }
+
     @Test
     public void testGetPositionsForRanges()
     {
@@ -170,7 +214,7 @@ public void testGetPositionsForRanges()
         // confirm that positions increase continuously
         SSTableReader sstable = store.getLiveSSTables().iterator().next();
         long previous = -1;
-        for (SSTableReader.PartitionPositionBounds section : sstable.getPositionsForRanges(ranges))
+        for (PartitionPositionBounds section : sstable.getPositionsForRanges(ranges))
         {
             assert previous <= section.lowerPosition : previous + " ! < " + section.lowerPosition;
             assert section.lowerPosition < section.upperPosition : section.lowerPosition + " ! < " + section.upperPosition;
@@ -178,9 +222,286 @@ public void testGetPositionsForRanges()
         }
     }
 
+    @Test
+    public void testEstimatedKeysForRangesAndKeySamples()
+    {
+        // prepare data
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2");
+        partitioner = store.getPartitioner();
+
+        Random random = new Random();
+        List<Token> tokens = new ArrayList<>();
+        tokens.add(partitioner.getMinimumToken());
+        if (partitioner.splitter().isPresent())
+            tokens.add(partitioner.getMaximumToken());
+
+        for (int j = 0; j < 100; j++)
+        {
+            Mutation mutation = new RowUpdateBuilder(store.metadata(), j, String.valueOf(random.nextInt())).clustering("0")
+                                                                                                           .add("val",
+                                                                                                                ByteBufferUtil.EMPTY_BYTE_BUFFER)
+                                                                                                           .build();
+            if (j % 4 != 0) // skip some keys
+                mutation.applyUnsafe();
+            tokens.add(mutation.key().getToken());
+        }
+
+        store.forceBlockingFlush(UNIT_TESTS);
+        assertEquals(1, store.getLiveSSTables().size());
+        SSTableReader sstable = store.getLiveSSTables().iterator().next();
+
+        // verify any combination of start and end point among the keys we have, which includes empty, full and
+        // wrap-around ranges
+        for (int i = 0; i < tokens.size(); i++)
+            for (int j = 0; j < tokens.size(); j++)
+            {
+                verifyEstimatedKeysAndKeySamples(sstable, new Range<Token>(tokens.get(i), tokens.get(j)));
+            }
+    }
+
+    private void verifyEstimatedKeysAndKeySamples(SSTableReader sstable, Range<Token> range)
+    {
+        List<DecoratedKey> expectedKeys = new ArrayList<>();
+        try (ISSTableScanner scanner = sstable.getScanner())
+        {
+            while (scanner.hasNext())
+            {
+                try (UnfilteredRowIterator rowIterator = scanner.next())
+                {
+                    if (range.contains(rowIterator.partitionKey().getToken()))
+                        expectedKeys.add(rowIterator.partitionKey());
+                }
+            }
+        }
+
+        // check estimated key
+        long estimated = sstable.estimatedKeysForRanges(Collections.singleton(range));
+        assertTrue("Range: " + range + " having " + expectedKeys.size() + " partitions, but estimated "
+                   + estimated, closeEstimation(expectedKeys.size(), estimated));
+
+        // check key samples
+        List<DecoratedKey> sampledKeys = new ArrayList<>();
+        sstable.getKeySamples(range).forEach(sampledKeys::add);
+
+        assertTrue("Range: " + range + " having " + expectedKeys + " keys, but keys sampled: "
+                   + sampledKeys, expectedKeys.containsAll(sampledKeys));
+        // no duplicate
+        assertEquals(expectedKeys.size(), expectedKeys.stream().distinct().count());
+        assertEquals(sampledKeys.size(), sampledKeys.stream().distinct().count());
+    }
+
+    private boolean closeEstimation(long expected, long estimated)
+    {
+        return expected <= estimated + 16 && expected >= estimated - 16;
+    }
+
+    @Test
+    public void testOnDiskSizeForRanges()
+    {
+        ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD2);
+        partitioner = store.getPartitioner();
+        int count = 1000;
+
+        // insert data and compact to a single sstable
+        for (int j = 0; j < count; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), 15000, k0(j))
+            .clustering("0")
+            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+            .build()
+            .applyUnsafe();
+        }
+        store.forceBlockingFlush(UNIT_TESTS);
+        store.forceMajorCompaction();
+
+        SSTableReader sstable = store.getLiveSSTables().iterator().next();
+
+        // Non-compression-dependent checks
+        // Check several ways of going through the whole file
+        assertEquals(sstable.onDiskLength(),
+                     onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t(cut(k0(0), 1)), t0(count - 1)))));
+        assertEquals(sstable.onDiskLength(),
+                     onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(sstable.getPartitioner().getMinimumToken(), sstable.getPartitioner().getMinimumToken()))));
+
+        // Split at exact match
+        assertEquals(sstable.onDiskLength(),
+                     onDiskSizeForRanges(sstable, ImmutableList.of(new Range<>(t(cut(k0(0), 1)), t0(347)),
+                                                                   new Range<>(t0(347), t0(count - 1)))));
+
+        // Split at different prefixes pointing to the same position
+        assertEquals(sstable.onDiskLength(),
+                     onDiskSizeForRanges(sstable, ImmutableList.of(new Range<>(t(cut(k0(0), 1)), t(cut(k0(600), 2))),
+                                                                   new Range<>(t(cut(k0(600), 1)), t0(count - 1)))));
+
+        if (!sstable.compression)
+        {
+            double delta = 0.9;
+            // Size one row
+            double oneRowSize = sstable.onDiskLength() * 1.0 / count;
+            System.out.println("One row size: " + oneRowSize);
+
+            // Ranges are end-inclusive, indexes are adjusted by one here to account for that.
+            assertEquals((52 - 38),
+                         onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t0(37), t0(51)))) / oneRowSize,
+                         delta);
+
+            // Try non-matching positions (inexact indexes are not adjusted for the count).
+            assertEquals((34 - 30),
+                         onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t(cut(k0(30), 1)),
+                                                                                        t0(33)))) / oneRowSize,
+                         delta);
+
+            assertEquals((700 - 554),
+                         onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t0(553),
+                                                                                       t(cut(k0(700), 2))))) / oneRowSize,
+                         delta);
+
+            assertEquals((500 - 30),
+                         onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t(cut(k0(30), 1)),
+                                                                                       t(cut(k0(500), 2))))) / oneRowSize,
+                         delta);
+
+            // Try a list
+            List<Range<Token>> ranges = ImmutableList.of(new Range<>(t0(37), t0(51)),
+                                                         new Range<>(t0(71), t(cut(k0(100), 2))),
+                                                         new Range<>(t(cut(k0(230), 1)), t0(243)),
+                                                         new Range<>(t(cut(k0(260), 1)), t(cut(k0(300), 2))),
+                                                         new Range<>(t0(373), t0(382)),
+                                                         new Range<>(t0(382), t0(385)),
+                                                         new Range<>(t(cut(k0(400), 2)), t(cut(k0(400), 1))),  // empty range
+                                                         new Range<>(t0(563), t(cut(k0(600), 2))), // touching ranges
+                                                         new Range<>(t(cut(k0(600), 1)), t0(621))
+            );
+            assertEquals((52 - 38 + 100 - 72 + 244 - 230 + 300 - 260 + 383 - 374 + 386 - 383 + 400 - 400 + 622 - 564),
+                         onDiskSizeForRanges(sstable, ranges) / oneRowSize,
+                         delta);
+
+            // Check going through the whole file
+            assertEquals(sstable.onDiskLength(),
+                         onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t(cut(k0(0), 1)), t0(count - 1)))));
+
+            assertEquals(sstable.onDiskLength(),
+                         onDiskSizeForRanges(sstable, ImmutableList.of(new Range<>(t(cut(k0(0), 1)), t0(347)),
+                                                                      new Range<>(t0(347), t0(count - 1)))));
+
+            assertEquals(sstable.onDiskLength(),
+                         onDiskSizeForRanges(sstable, ImmutableList.of(new Range<>(t(cut(k0(0), 1)), t(cut(k0(600), 2))),
+                                                                      new Range<>(t(cut(k0(600), 1)), t0(count - 1)))));
+        }
+        else
+        {
+            // It's much harder to test with compression.
+
+            // Check first three rows have the same size (they must be in the same chunk)
+            final long row0size = onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t(cut(k0(0), 1)), t0(0))));
+            assertEquals(row0size, onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t0(0), t0(1)))));
+            assertEquals(row0size, onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t0(1), t0(2)))));
+
+            // As well as the first three rows together
+            assertEquals(row0size, onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t(cut(k0(0), 1)), t0(2)))));
+
+            // And also when we query for them in separate ranges
+            assertEquals(row0size, onDiskSizeForRanges(sstable, ImmutableList.of(new Range<>(t(cut(k0(0), 1)), t0(0)),
+                                                                                new Range<>(t0(0), t0(1)))));
+            assertEquals(row0size, onDiskSizeForRanges(sstable, ImmutableList.of(new Range<>(t(cut(k0(0), 1)), t0(0)),
+                                                                                new Range<>(t0(1), t0(2)))));
+            assertEquals(row0size, onDiskSizeForRanges(sstable, ImmutableList.of(new Range<>(t(cut(k0(0), 1)), t0(0)),
+                                                                                new Range<>(t0(0), t0(1)),
+                                                                                new Range<>(t0(1), t0(2)))));
+
+            // Finally, check that if we query for every second row we get the total size of the file.
+            assertEquals(sstable.onDiskLength(),
+                         onDiskSizeForRanges(sstable, IntStream.range(0, count)
+                                                              .filter(i -> i % 2 != 0)
+                                                              .mapToObj(i -> new Range<>(t0(i), t0(i + 1)))
+                                                              .collect(Collectors.toList())));
+        }
+    }
+
+    @Test
+    public void testAssertionsOnDiskSizeForPartitionPositions()
+    {
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_COMPRESSED);
+        SSTableReader sstable = getNewSSTable(cfs);
+        partitioner = sstable.getPartitioner();
+
+        assertThatThrownBy(() -> sstable.onDiskSizeForPartitionPositions(Collections.singleton(new PartitionPositionBounds(-1, 0))))
+                .isInstanceOf(AssertionError.class);
+        assertThatThrownBy(() -> sstable.onDiskSizeForPartitionPositions(Collections.singleton(new PartitionPositionBounds(2, 1))))
+                .isInstanceOf(AssertionError.class);
+    }
+
+    @Test
+    public void testDiskSizeForEmptyPosition()
+    {
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_COMPRESSED);
+        SSTableReader sstable = getNewSSTable(cfs);
+        partitioner = sstable.getPartitioner();
+
+        long size = sstable.onDiskSizeForPartitionPositions(Collections.singleton(new PartitionPositionBounds(0, 0)));
+        assertEquals(0, size);
+    }
+
+    @Test
+    public void testDoNotFailOnChunkEndingPosition()
+    {
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_COMPRESSED);
+
+        // we want the last row to align to the end of chunk
+        int rowCount = DEFAULT_CHUNK_LENGTH;
+
+        // insert data and compact to a single sstable
+        for (int j = 0; j < rowCount; j++)
+        {
+            new RowUpdateBuilder(cfs.metadata(), 15000, k0(j))
+                    .clustering("0")
+                    .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+                    .build()
+                    .applyUnsafe();
+        }
+        cfs.forceBlockingFlush(UNIT_TESTS);
+        cfs.forceMajorCompaction();
+        SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+        partitioner = sstable.getPartitioner();
+
+        long totalDiskSizeForTheWholeRange = onDiskSizeForRanges(sstable, Collections.singleton(new Range<>(t(cut(k0(0), 1)), t0(rowCount))));
+        assertEquals(sstable.onDiskLength(), totalDiskSizeForTheWholeRange);
+    }
+
+    long onDiskSizeForRanges(SSTableReader sstable, Collection<Range<Token>> ranges)
+    {
+        return sstable.onDiskSizeForPartitionPositions(sstable.getPositionsForRanges(ranges));
+    }
+
+    private Token t(String key)
+    {
+        return partitioner.getToken(ByteBufferUtil.bytes(key));
+    }
+
+    private String k0(int k)
+    {
+        return String.format("%08d", k);
+    }
+
+    private Token t0(int k)
+    {
+        return t(k0(k));
+    }
+
+    private String cut(String s, int n)
+    {
+        return s.substring(0, s.length() - n);
+    }
+
+
     @Test
     public void testSpannedIndexPositions() throws IOException
     {
+        // expect to create many regions - that is, the size of index must exceed the page size multiple times
         int originalMaxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE;
         MmappedRegions.MAX_SEGMENT_SIZE = PageAware.PAGE_SIZE;
 
@@ -208,7 +529,7 @@ public void testSpannedIndexPositions() throws IOException
                 DecoratedKey dk = Util.dk(String.valueOf(j));
                 FileDataInput file = sstable.getFileDataInput(sstable.getPosition(dk, SSTableReader.Operator.EQ));
                 DecoratedKey keyInDisk = sstable.decorateKey(ByteBufferUtil.readWithShortLength(file));
-                assert keyInDisk.equals(dk) : format("%s != %s in %s", keyInDisk, dk, file.getPath());
+                assert keyInDisk.equals(dk) : format("%s != %s in %s", keyInDisk, dk, file.getFile());
             }
 
             // check no false positives
@@ -332,7 +653,7 @@ public void testGetPositionsForRangesWithKeyCache()
         long p6 = sstable.getPosition(dk(6), SSTableReader.Operator.EQ);
         long p7 = sstable.getPosition(dk(7), SSTableReader.Operator.EQ);
 
-        SSTableReader.PartitionPositionBounds p = sstable.getPositionsForRanges(makeRanges(t(2), t(6))).get(0);
+        PartitionPositionBounds p = sstable.getPositionsForRanges(makeRanges(t(2), t(6))).get(0);
 
         // range are start exclusive so we should start at 3
         assert p.lowerPosition == p3;
@@ -388,12 +709,15 @@ public void testGetPositionsKeyCacheStats()
         assertEquals(1, keyCache.getRequests());
         assertEquals(0, keyCache.getHits());
         // existing, cached key
+        assertEquals(1, store.getBloomFilterTracker().getTruePositiveCount());
         sstable.getPosition(dk(2), SSTableReader.Operator.EQ);
         assertEquals(2, keyCache.getRequests());
         assertEquals(1, keyCache.getHits());
         // non-existing key (it is specifically chosen to not be rejected by Bloom Filter check)
         sstable.getPosition(dk(14), SSTableReader.Operator.EQ);
         assertEquals(3, keyCache.getRequests());
+        assertEquals(2, store.getBloomFilterTracker().getTruePositiveCount());
+        sstable.getPosition(dk(15), SSTableReader.Operator.EQ);
         assertEquals(1, keyCache.getHits());
     }
 
@@ -404,7 +728,7 @@ public void testGetPositionsBloomFilterStats()
 
         // the keys are specifically chosen to cover certain use cases
         // existing key is read from index
-        sstable.getPosition(dk(7), SSTableReader.Operator.EQ);
+        sstable.getPosition(dk(2), SSTableReader.Operator.EQ);
         assertEquals(1, sstable.getFilterTracker().getTruePositiveCount());
         assertEquals(0, sstable.getFilterTracker().getTrueNegativeCount());
         assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount());
@@ -616,6 +940,7 @@ public void testOpeningSSTable() throws Exception
         Util.flush(store);
 
         SSTableReader sstable = store.getLiveSSTables().iterator().next();
+        assertTrue(SSTableReader.hasGlobalReference(sstable.descriptor));
         Descriptor desc = sstable.descriptor;
 
         // test to see if sstable can be opened as expected
@@ -837,6 +1162,7 @@ private static void checkOpenedBtiTable(String ks, String cf, ColumnFamilyStore
     @Test
     public void testLoadingSummaryUsesCorrectPartitioner() throws Exception
     {
+        Assume.assumeTrue(BigFormat.isSelected());
         ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_INDEXED);
 
         new RowUpdateBuilder(store.metadata(), System.currentTimeMillis(), "k1")
@@ -869,7 +1195,8 @@ public void testLoadingSummaryUsesCorrectPartitioner() throws Exception
     @Test
     public void testGetScannerForNoIntersectingRanges() throws Exception
     {
-        ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD);
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD3);
         partitioner = store.getPartitioner();
 
         new RowUpdateBuilder(store.metadata(), 0, "k1")
@@ -879,21 +1206,12 @@ public void testGetScannerForNoIntersectingRanges() throws Exception
             .applyUnsafe();
 
         Util.flush(store);
-        boolean foundScanner = false;
 
         Set<SSTableReader> liveSSTables = store.getLiveSSTables();
         assertEquals("The table should have only one sstable", 1, liveSSTables.size());
 
-        for (SSTableReader s : liveSSTables)
-        {
-            try (ISSTableScanner scanner = s.getScanner(new Range<>(t(0), t(1))))
-            {
-                // Make sure no data is returned and nothing fails for non-intersecting range.
-                assertFalse(scanner.hasNext());
-                foundScanner = true;
-            }
-        }
-        assertTrue(foundScanner);
+        ISSTableScanner scanner = liveSSTables.iterator().next().getScanner(new Range<>(t(0), t(1)));
+        assertEquals(0, scanner.getLengthInBytes());
     }
 
     @Test
@@ -924,12 +1242,12 @@ public void testGetPositionsForRangesFromTableOpenedForBulkLoading()
         ranges.add(new Range<Token>(t(98), t(99)));
 
         SSTableReader sstable = store.getLiveSSTables().iterator().next();
-        List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(ranges);
-        assert sections.size() == 1 : "Expected to find range in sstable";
+        List<PartitionPositionBounds> sections = sstable.getPositionsForRanges(ranges);
+        assert sections.size() == 1 : "Expected to find range in sstable" ;
 
         // re-open the same sstable as it would be during bulk loading
         Set<Component> components = Sets.newHashSet(sstable.descriptor.getFormat().primaryComponents());
-        if (sstable.components.contains(Components.COMPRESSION_INFO))
+        if (sstable.components().contains(Components.COMPRESSION_INFO))
             components.add(Components.COMPRESSION_INFO);
         SSTableReader bulkLoaded = SSTableReader.openForBatch(store, sstable.descriptor, components, store.metadata);
         sections = bulkLoaded.getPositionsForRanges(ranges);
@@ -1095,7 +1413,7 @@ public void testMoveAndOpenLiveSSTable()
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD);
         SSTableReader sstable = getNewSSTable(cfs);
         Descriptor notLiveDesc = new Descriptor(new File("/testdir"), "", "", SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get());
-        SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components, false);
+        SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components(), false);
     }
 
     @Test(expected = RuntimeException.class)
@@ -1105,7 +1423,7 @@ public void testMoveAndOpenLiveSSTable2()
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD);
         SSTableReader sstable = getNewSSTable(cfs);
         Descriptor notLiveDesc = new Descriptor(new File("/testdir"), "", "", SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get());
-        SSTableReader.moveAndOpenSSTable(cfs, notLiveDesc, sstable.descriptor, sstable.components, false);
+        SSTableReader.moveAndOpenSSTable(cfs, notLiveDesc, sstable.descriptor, sstable.components(), false);
     }
 
     @Test
@@ -1121,15 +1439,15 @@ public void testMoveAndOpenSSTable() throws IOException
         SSTableId id = SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get();
         Descriptor notLiveDesc = new Descriptor(tmpdir, sstable.descriptor.ksname, sstable.descriptor.cfname, id);
         // make sure the new directory is empty and that the old files exist:
-        for (Component c : sstable.components)
+        for (Component c : sstable.components())
         {
             File f = notLiveDesc.fileFor(c);
             assertFalse(f.exists());
             assertTrue(sstable.descriptor.fileFor(c).exists());
         }
-        SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components, false);
+        SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components(), false);
         // make sure the files were moved:
-        for (Component c : sstable.components)
+        for (Component c : sstable.components())
         {
             File f = notLiveDesc.fileFor(c);
             assertTrue(f.exists());
@@ -1140,9 +1458,14 @@ public void testMoveAndOpenSSTable() throws IOException
     }
 
     private SSTableReader getNewSSTable(ColumnFamilyStore cfs)
+    {
+        return getNewSSTable(cfs, 100, 2);
+    }
+
+    private SSTableReader getNewSSTable(ColumnFamilyStore cfs, int numKeys, int step)
     {
         Set<SSTableReader> before = cfs.getLiveSSTables();
-        for (int j = 0; j < 100; j += 2)
+        for (int j = 0; j < numKeys; j += step)
         {
             new RowUpdateBuilder(cfs.metadata(), j, String.valueOf(j))
             .clustering("0")
@@ -1215,6 +1538,157 @@ public void testVerifyCompressionInfoExistencePasses()
         CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components);
     }
 
+    @Test
+    public void testBloomFilterIsCreatedOnLoad() throws IOException
+    {
+        BF_RECREATE_ON_FP_CHANCE_CHANGE.setBoolean(true);
+
+        final int numKeys = 100;
+        final Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD_NO_BLOOM_FILTER);
+
+        SSTableReader sstable = getNewSSTable(cfs, numKeys, 1);
+        Assert.assertTrue(getFilterSize(sstable) == 0);
+        Assert.assertSame(FilterFactory.AlwaysPresent, getFilter(sstable));
+
+        // should do nothing
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, 1, false, numKeys, false);
+
+        // should create BF because the FP has changed
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, BloomCalculations.minSupportedBloomFilterFpChance(), true, numKeys, true);
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, 0.05, true, numKeys, true);
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, 0.1, true, numKeys, true);
+
+        // should deserialize the existing BF
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, 0.1, true, numKeys, false);
+        // should create BF because the FP has changed
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, 1 - BF_FP_CHANCE_TOLERANCE.getDouble(), true, numKeys, true);
+        // should install empty filter without changing file or metadata
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, 1, false, numKeys, false);
+
+        // corrupted bf file should fail to deserialize and we should fall back to recreating it
+        Files.write(sstable.descriptor.fileFor(Components.FILTER).toPath(), new byte[] { 0, 0, 0, 0});
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, 1 - BF_FP_CHANCE_TOLERANCE.getDouble(), true, numKeys, true);
+
+        // missing primary index file should make BF fail to load and we should install the empty one
+        HashSet<Component> nonDataPrimaryComponents = new HashSet<>(sstable.descriptor.getFormat().primaryComponents());
+        nonDataPrimaryComponents.remove(Components.DATA);
+        nonDataPrimaryComponents.remove(Components.COMPRESSION_INFO);
+        nonDataPrimaryComponents.remove(Components.STATS);
+        for (Component component : nonDataPrimaryComponents)
+            sstable.descriptor.fileFor(component).delete();
+        checkSSTableOpenedWithGivenFPChance(cfs, sstable, 0.05, false, numKeys, false);
+    }
+
+    @Test
+    public void testOnDiskComponentsSize()
+    {
+        final int numKeys = 1000;
+        final Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD);
+
+        SSTableReader sstable = getNewSSTable(cfs, numKeys, 1);
+        assertEquals(sstable.onDiskLength(), FileUtils.size(sstable.descriptor.pathFor(Components.DATA)));
+
+        assertTrue(sstable.components().contains(Components.DATA));
+        assertTrue(sstable.components().size() > 1);
+        assertTrue(sstable.onDiskComponentsSize() > sstable.onDiskLength());
+    }
+
+    private void checkSSTableOpenedWithGivenFPChance(ColumnFamilyStore cfs, SSTableReader sstable, double fpChance, boolean bfShouldExist, int numKeys, boolean expectRecreated) throws IOException
+    {
+        Descriptor desc = sstable.descriptor;
+        TableMetadata metadata = sstable.metadata.get().unbuild().bloomFilterFpChance(fpChance).build();
+        ValidationMetadata prevValidationMetadata = getValidationMetadata(desc);
+        Assert.assertNotNull(prevValidationMetadata);
+        File bfFile = desc.fileFor(Components.FILTER);
+
+        SSTableReader target = null;
+        try
+        {
+            FileTime bf0Time = bfFile.exists() ? Files.getLastModifiedTime(bfFile.toPath()) : FileTime.from(Instant.MIN);
+
+            // make sure we wait enough - some JDK implementations use seconds granularity and we need to wait a bit to actually see the change
+            Uninterruptibles.sleepUninterruptibly(1, Util.supportedMTimeGranularity);
+
+            target = SSTableReader.open(cfs,
+                                        desc,
+                                        TOCComponent.loadTOC(desc),
+                                        TableMetadataRef.forOfflineTools(metadata),
+                                        false,
+                                        false);
+            IFilter bloomFilter = getFilter(target);
+            ValidationMetadata validationMetadata = getValidationMetadata(desc);
+            Assert.assertNotNull(validationMetadata);
+            FileTime bf1Time = bfFile.exists() ? Files.getLastModifiedTime(bfFile.toPath()) : FileTime.from(Instant.MIN);
+
+            if (expectRecreated)
+            {
+                Assert.assertTrue(bf0Time.compareTo(bf1Time) < 0);
+            }
+            else
+            {
+                assertEquals(bf0Time, bf1Time);
+            }
+
+            if (bfShouldExist)
+            {
+                Assert.assertNotEquals(FilterFactory.AlwaysPresent, bloomFilter);
+                Assert.assertTrue(bloomFilter.serializedSize(false) > 0);
+                Assert.assertEquals(fpChance, validationMetadata.bloomFilterFPChance, BF_FP_CHANCE_TOLERANCE.getDouble());
+                Assert.assertTrue(bfFile.exists());
+                Assert.assertEquals(bloomFilter.serializedSize(false), bfFile.length());
+            }
+            else
+            {
+                Assert.assertEquals(FilterFactory.AlwaysPresent, getFilter(sstable));
+                Assert.assertTrue(getFilterSize(sstable) == 0);
+                Assert.assertEquals(prevValidationMetadata.bloomFilterFPChance, validationMetadata.bloomFilterFPChance, BF_FP_CHANCE_TOLERANCE.getDouble());
+                Assert.assertEquals(bfFile.exists(), bfFile.exists());
+            }
+
+            // verify all keys are present according to the BF
+            Token token = new Murmur3Partitioner.LongToken(0L);
+            for (int i = 0; i < numKeys; i++)
+            {
+                DecoratedKey key = new BufferDecoratedKey(token, ByteBufferUtil.bytes(String.valueOf(i)));
+                Assert.assertTrue("Expected key to be in BF: " + i, bloomFilter.isPresent(key));
+            }
+        }
+        finally
+        {
+            if (target != null)
+                target.selfRef().release();
+        }
+    }
+
+    static long getFilterSize(SSTableReader rdr)
+    {
+        return ((SSTableReaderWithFilter) rdr).getFilterSerializedSize();
+    }
+
+    static IFilter getFilter(SSTableReader rdr)
+    {
+        return ((SSTableReaderWithFilter) rdr).getFilter();
+    }
+
+    private static ValidationMetadata getValidationMetadata(Descriptor descriptor)
+    {
+        EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION);
+
+        Map<MetadataType, MetadataComponent> sstableMetadata;
+        try
+        {
+            sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+        }
+        catch (Throwable t)
+        {
+            throw new CorruptSSTableException(t, descriptor.fileFor(Components.STATS));
+        }
+
+        return (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
+    }
+
     private Descriptor setUpForTestVerfiyCompressionInfoExistence()
     {
         Keyspace keyspace = Keyspace.open(KEYSPACE1);
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 5cb5bc32c18f..15c6eef2800d 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -27,21 +27,26 @@
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.cassandra.UpdateBuilder;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionTime;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.compaction.CompactionController;
 import org.apache.cassandra.db.compaction.CompactionIterator;
 import org.apache.cassandra.db.compaction.OperationType;
@@ -49,9 +54,8 @@
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -96,7 +100,7 @@ public void basicTest()
         assertEquals(1, sstables.size());
         assertEquals(sstables.iterator().next().bytesOnDisk(), cfs.metric.liveDiskSpaceUsed.getCount());
         long nowInSec = FBUtilities.nowInSeconds();
-        try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
+        try (ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables);
              LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
              SSTableRewriter writer = SSTableRewriter.constructKeepingOriginals(txn, false, 1000);
              CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
@@ -128,7 +132,7 @@ public void basicTest2()
         assertEquals(1, sstables.size());
 
         long nowInSec = FBUtilities.nowInSeconds();
-        try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
+        try (ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables);
              LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
              SSTableRewriter writer = new SSTableRewriter(txn, 1000, 10000000, false, true);
              CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
@@ -161,7 +165,7 @@ public void getPositionsTest()
 
         long nowInSec = FBUtilities.nowInSeconds();
         boolean checked = false;
-        try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
+        try (ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables);
              LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
              SSTableRewriter writer = new SSTableRewriter(txn, 1000, 10000000, false, true);
              CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
@@ -635,7 +639,7 @@ private void testAbortHelper(boolean earlyException, boolean offline)
         Set<SSTableReader> compacting = Sets.newHashSet(s);
         try (ISSTableScanner scanner = compacting.iterator().next().getScanner();
              CompactionController controller = new CompactionController(cfs, compacting, 0);
-             LifecycleTransaction txn = offline ? LifecycleTransaction.offline(OperationType.UNKNOWN, compacting)
+             LifecycleTransaction txn = offline ? LifecycleTransaction.offline(OperationType.UNKNOWN, cfs.metadata, compacting)
                                        : cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
              SSTableRewriter rewriter = new SSTableRewriter(txn, 100, 10000000, false, true);
              CompactionIterator ci = new CompactionIterator(COMPACTION, singletonList(scanner), controller, nowInSeconds(), nextTimeUUID())
@@ -804,7 +808,7 @@ public void testTwoWriters()
         Set<SSTableReader> sstables = Sets.newHashSet(s);
         assertEquals(1, sstables.size());
         long nowInSec = FBUtilities.nowInSeconds();
-        try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
+        try (ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables);
              LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
              SSTableRewriter writer = SSTableRewriter.constructWithoutEarlyOpening(txn, false, 1000);
              SSTableRewriter writer2 = SSTableRewriter.constructWithoutEarlyOpening(txn, false, 1000);
@@ -829,42 +833,71 @@ public void testTwoWriters()
     }
 
     @Test
-    public void testCanonicalSSTables() throws ExecutionException, InterruptedException
+    public void testCanonicalSSTablesWithEarlyOpen() throws ExecutionException, InterruptedException
     {
-        Keyspace keyspace = Keyspace.open(KEYSPACE);
-        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-        truncate(cfs);
+        testCanonicalSSTables(1);
+    }
 
-        cfs.addSSTable(writeFile(cfs, 100));
-        Collection<SSTableReader> allSSTables = cfs.getLiveSSTables();
-        assertEquals(1, allSSTables.size());
-        final AtomicBoolean done = new AtomicBoolean(false);
-        final AtomicBoolean failed = new AtomicBoolean(false);
-        Runnable r = () -> {
-            while (!done.get())
-            {
-                Iterable<SSTableReader> sstables = cfs.getSSTables(SSTableSet.CANONICAL);
-                if (Iterables.size(sstables) != 1)
+    @Test
+    public void testCanonicalSSTablesWithFinalEarlyOpen() throws ExecutionException, InterruptedException
+    {
+        testCanonicalSSTables(1000000);
+    }
+
+    @Test
+    @Ignore // This does not currently work. See View.select.
+    public void testCanonicalSSTablesNoEarlyOpen() throws ExecutionException, InterruptedException
+    {
+        testCanonicalSSTables(-1);
+    }
+
+
+    public void testCanonicalSSTables(int preemptiveOpenInterval) throws ExecutionException, InterruptedException
+    {
+        int prevPreemptiveOpenInterval = DatabaseDescriptor.getSSTablePreemptiveOpenIntervalInMiB();
+        try
+        {
+            DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMiB(preemptiveOpenInterval);
+            Keyspace keyspace = Keyspace.open(KEYSPACE);
+            final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+            truncate(cfs);
+
+            cfs.addSSTable(writeFile(cfs, 2000));
+            Collection<SSTableReader> allSSTables = cfs.getLiveSSTables();
+            assertEquals(1, allSSTables.size());
+            final AtomicBoolean done = new AtomicBoolean(false);
+            final AtomicBoolean gotZero = new AtomicBoolean(false);
+            final AtomicInteger maxValue = new AtomicInteger(0);
+            Runnable r = () -> {
+                while (!done.get())
                 {
-                    failed.set(true);
-                    return;
+                    Iterable<SSTableReader> sstables = cfs.getSSTables(SSTableSet.CANONICAL);
+                    int sstablesCount = Iterables.size(sstables);
+                    if (sstablesCount == 0)
+                        gotZero.set(true);
+                    else
+                        maxValue.updateAndGet(prev -> Math.max(prev, sstablesCount));
                 }
+            };
+            Thread t = NamedThreadFactory.createAnonymousThread(r);
+            try
+            {
+                t.start();
+                cfs.forceMajorCompaction();
             }
-        };
-        Thread t = NamedThreadFactory.createAnonymousThread(r);
-        try
-        {
-            t.start();
-            cfs.forceMajorCompaction();
+            finally
+            {
+                done.set(true);
+                t.join(20);
+            }
+            // Note: the checks below can falsely succeed. Flaky failures should be treated as genuine problems.
+            assertFalse("No sstables", gotZero.get());
+            assertEquals("Too many sstables", 1, maxValue.get());
         }
         finally
         {
-            done.set(true);
-            t.join(20);
+            DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMiB(prevPreemptiveOpenInterval);
         }
-        assertFalse(failed.get());
-
-
     }
 
     /**
@@ -879,6 +912,7 @@ public void testWriterClearing()
         Keyspace keyspace = Keyspace.open(KEYSPACE);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
         File dir = cfs.getDirectories().getDirectoryForNewSSTables();
+        Row staticRow = Rows.EMPTY_STATIC_ROW;
 
         // Can't update a writer that is eagerly cleared on switch
         boolean eagerWriterMetaRelease = true;
@@ -894,6 +928,7 @@ public void testWriterClearing()
                 UnfilteredRowIterator uri = mock(UnfilteredRowIterator.class);
                 when(uri.partitionLevelDeletion()).thenReturn(DeletionTime.build(0, 0));
                 when(uri.partitionKey()).thenReturn(bopKeyFromInt(0));
+                when(uri.staticRow()).thenReturn(staticRow);
                 // should not be able to append after buffer release on switch
                 firstWriter.append(uri);
                 fail("Expected AssertionError was not thrown.");
@@ -919,7 +954,7 @@ private void validateKeys(Keyspace ks)
         for (int i = 0; i < 100; i++)
         {
             DecoratedKey key = Util.dk(Integer.toString(i));
-            ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(ks.getColumnFamilyStore(CF), key).build());
+            Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(ks.getColumnFamilyStore(CF), key).build());
             assertTrue(partition != null && partition.rowCount() > 0);
         }
     }
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
index 17b8a6cbb2af..1b3bc2623f06 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
@@ -21,10 +21,13 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
 import com.google.common.collect.Iterables;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -40,6 +43,7 @@
 import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
@@ -49,6 +53,7 @@
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.hamcrest.Matchers;
 
 import static org.apache.cassandra.dht.AbstractBounds.isEmpty;
 import static org.junit.Assert.assertEquals;
@@ -180,6 +185,12 @@ private static void insertRowWithKey(TableMetadata metadata, int key)
     }
 
     private static void assertScanMatches(SSTableReader sstable, int scanStart, int scanEnd, int ... boundaries)
+    {
+        assertScanMatchesUsingScanner(sstable, scanStart, scanEnd, boundaries);
+        assertScanMatchesUsingSimple(sstable, scanStart, scanEnd, boundaries);
+    }
+
+    private static void assertScanMatchesUsingScanner(SSTableReader sstable, int scanStart, int scanEnd, int ... boundaries)
     {
         assert boundaries.length % 2 == 0;
         for (DataRange range : dataRanges(sstable.metadata(), scanStart, scanEnd))
@@ -200,6 +211,28 @@ private static void assertScanMatches(SSTableReader sstable, int scanStart, int
         }
     }
 
+    private static void assertScanMatchesUsingSimple(SSTableReader sstable, int scanStart, int scanEnd, int ... boundaries)
+    {
+        assert boundaries.length % 2 == 0;
+        for (DataRange range : dataRanges(sstable.metadata(), scanStart, scanEnd))
+        {
+            if (range.isWrapAround() && !range.keyRange().right.isMinimum()) // getScanner on AbstractBounds<PartitionPosition> does not handle wraparounds
+                continue;
+
+            try(UnfilteredPartitionIterator scanner = sstable.getScanner(Collections.singleton(range.keyRange()).iterator()))
+            {
+                for (int b = 0; b < boundaries.length; b += 2)
+                    for (int i = boundaries[b]; i <= boundaries[b + 1]; i++)
+                        assertEquals(toKey(i), new String(scanner.next().partitionKey().getKey().array()));
+                assertFalse(scanner.hasNext());
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
     private static void assertScanEmpty(SSTableReader sstable, int scanStart, int scanEnd)
     {
         assertScanMatches(sstable, scanStart, scanEnd);
@@ -547,7 +580,30 @@ public void testSingleKeyMultipleRanges() throws IOException
         assertScanContainsRanges(scanner, 205, 205);
     }
 
-    private static void testRequestNextRowIteratorWithoutConsumingPrevious(Consumer<ISSTableScanner> consumer)
+    private static void testRequestNextRowIteratorWithoutConsumingPrevious(Function<SSTableReader, UnfilteredPartitionIterator> makeScanner,
+                                                                           Consumer<UnfilteredPartitionIterator> requestNext,
+                                                                           String messagePattern)
+    {
+        final SSTableReader sstable = prepareSmallSSTable();
+
+        try (UnfilteredPartitionIterator scanner = makeScanner.apply(sstable);
+             UnfilteredRowIterator currentRowIterator = scanner.next())
+        {
+            assertTrue(currentRowIterator.hasNext());
+            try
+            {
+                requestNext.accept(scanner);
+                currentRowIterator.next();
+                fail("Should have thrown IllegalStateException");
+            }
+            catch (IllegalStateException e)
+            {
+                Assert.assertThat(e.getMessage(), Matchers.matchesPattern(messagePattern));
+            }
+        }
+    }
+
+    private static SSTableReader prepareSmallSSTable()
     {
         Keyspace keyspace = Keyspace.open(KEYSPACE);
         ColumnFamilyStore store = keyspace.getColumnFamilyStore(TABLE);
@@ -557,38 +613,77 @@ private static void testRequestNextRowIteratorWithoutConsumingPrevious(Consumer<
         store.disableAutoCompaction();
 
         insertRowWithKey(store.metadata(), 0);
+        insertRowWithKey(store.metadata(), 3);
         store.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
 
         assertEquals(1, store.getLiveSSTables().size());
         SSTableReader sstable = store.getLiveSSTables().iterator().next();
+        return sstable;
+    }
 
-        try (ISSTableScanner scanner = sstable.getScanner();
-             UnfilteredRowIterator currentRowIterator = scanner.next())
+    @Test
+    public void testSimpleHasNextRowIteratorWithoutConsumingPrevious()
+    {
+        testRequestNextRowIteratorWithoutConsumingPrevious(SSTableReader::getScanner,
+                                                           UnfilteredPartitionIterator::hasNext,
+                                                           "Iterator used after closing.");
+    }
+
+    @Test
+    public void testSimpleNextRowIteratorWithoutConsumingPrevious()
+    {
+        testRequestNextRowIteratorWithoutConsumingPrevious(SSTableReader::getScanner,
+                                                           UnfilteredPartitionIterator::next,
+                                                           "Iterator used after closing.");
+    }
+
+    @Test
+    public void testHasNextRowIteratorWithoutConsumingPrevious()
+    {
+        testRequestNextRowIteratorWithoutConsumingPrevious(r -> r.partitionIterator(ColumnFilter.NONE, DataRange.allData(r.getPartitioner()), SSTableReadsListener.NOOP_LISTENER),
+                                                           UnfilteredPartitionIterator::hasNext,
+                                                           ".*UnfilteredRowIterator.*(should|must) be.*closed.*");
+    }
+
+    @Test
+    public void testNextRowIteratorWithoutConsumingPrevious()
+    {
+        testRequestNextRowIteratorWithoutConsumingPrevious(r -> r.partitionIterator(ColumnFilter.NONE, DataRange.allData(r.getPartitioner()), SSTableReadsListener.NOOP_LISTENER),
+                                                           UnfilteredPartitionIterator::next,
+                                                           ".*UnfilteredRowIterator.*(should|must) be.*closed.*");
+    }
+
+    private static void testRequestNextRowIteratorAfterClosingPrevious(Function<SSTableReader, UnfilteredPartitionIterator> makeScanner)
+    {
+        final SSTableReader sstable = prepareSmallSSTable();
+
+        try (UnfilteredPartitionIterator scanner = makeScanner.apply(sstable))
         {
-            assertTrue(currentRowIterator.hasNext());
-            try
+            try (UnfilteredRowIterator p = scanner.next())
             {
-                consumer.accept(scanner);
-                fail("Should have thrown IllegalStateException");
+                assertEquals(toKey(0), new String(p.partitionKey().getKey().array()));
+                // do not read it, but close it
             }
-            catch (IllegalStateException e)
+
+            try (UnfilteredRowIterator p = scanner.next())
             {
-                assertEquals("The UnfilteredRowIterator returned by the last call to next() was initialized: " +
-                             "it must be closed before calling hasNext() or next() again.",
-                             e.getMessage());
+                assertEquals(toKey(3), new String(p.partitionKey().getKey().array()));
+                assertTrue(p.hasNext());
+                assertTrue(p.next() instanceof Row);
             }
         }
     }
 
+
     @Test
-    public void testHasNextRowIteratorWithoutConsumingPrevious()
+    public void testSimpleRequestNextRowIteratorAfterClosingPreviouss()
     {
-        testRequestNextRowIteratorWithoutConsumingPrevious(ISSTableScanner::hasNext);
+        testRequestNextRowIteratorAfterClosingPrevious(SSTableReader::getScanner);
     }
 
     @Test
-    public void testNextRowIteratorWithoutConsumingPrevious()
+    public void testRequestNextRowIteratorAfterClosingPrevious()
     {
-        testRequestNextRowIteratorWithoutConsumingPrevious(ISSTableScanner::next);
+        testRequestNextRowIteratorAfterClosingPrevious(r -> r.partitionIterator(ColumnFilter.NONE, DataRange.allData(r.getPartitioner()), SSTableReadsListener.NOOP_LISTENER));
     }
 }
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
index d2f5f6104e3f..b08be6e2764d 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
@@ -34,7 +34,13 @@
 import org.apache.cassandra.schema.TableMetadata;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.stream.Stream;
 
 import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
@@ -230,7 +236,7 @@ public Collection<SSTableReader> write(int expectedSize, Appender appender) thro
             // mark all components for removal
             if (cleanup)
                 for (SSTableReader reader: readers)
-                    for (Component component : reader.components)
+                    for (Component component : reader.components())
                         reader.descriptor.fileFor(component).deleteOnExit();
             return readers;
         }
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWithZeroCopyMetadataTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWithZeroCopyMetadataTest.java
new file mode 100644
index 000000000000..9dda8ef934f2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWithZeroCopyMetadataTest.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.bti.BtiTableReader;
+import org.apache.cassandra.io.sstable.format.bti.ScrubPartitionIterator;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.OutputHandler;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+
+@RunWith(Parameterized.class)
+public class SSTableWithZeroCopyMetadataTest
+{
+    private final static Logger logger = LoggerFactory.getLogger(SSTableWithZeroCopyMetadataTest.class);
+
+    private static final String KEYSPACE = "ZeroCopyStreamingTest";
+    private static final String TABLE = "Standard1";
+
+    private static final Path UNCOMPRESSED_DATA_DIR = Paths.get("test/data/zcs/uncompressed");
+    private static final Path COMPRESSED_DATA_DIR = Paths.get("test/data/zcs/compressed");
+
+    @Parameterized.Parameters(name = "compressed={0}, diskAccessMode={1}")
+    public static Object[] parameters()
+    {
+        return new Object[][]{ { false, Config.DiskAccessMode.standard },
+                               { false, Config.DiskAccessMode.mmap },
+                               { true, Config.DiskAccessMode.standard },
+                               { true, Config.DiskAccessMode.mmap } };
+    }
+
+    @Parameterized.Parameter(0)
+    public boolean compressed;
+
+    @Parameterized.Parameter(1)
+    public Config.DiskAccessMode diskAccessMode;
+
+    private Path dataDir;
+    private Path tableDataDir;
+    private TableMetadataRef metadataRef;
+    private ColumnFamilyStore realm;
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        DatabaseDescriptor.daemonInitialization(() -> {
+            Config config = DatabaseDescriptor.loadConfig();
+            config.partitioner = Murmur3Partitioner.class.getName();
+            return config;
+        });
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE));
+        StorageService.instance.initServer();
+    }
+
+    @Before
+    public void beforeTest() throws IOException
+    {
+        DatabaseDescriptor.setDiskAccessMode(diskAccessMode);
+        dataDir = Files.createTempDirectory("zcs-" + (compressed ? "compressed" : "uncompressed"));
+        FileUtils.copyDirectory((compressed ? COMPRESSED_DATA_DIR : UNCOMPRESSED_DATA_DIR).toFile(), dataDir.toFile());
+        tableDataDir = dataDir.resolve(KEYSPACE).resolve(TABLE);
+        metadataRef = Schema.instance.getTableMetadataRef(KEYSPACE, TABLE);
+        realm = ColumnFamilyStore.getIfExists(metadataRef.keyspace, metadataRef.name);
+        realm.disableAutoCompaction();
+    }
+
+    @After
+    public void afterTest() throws IOException
+    {
+        realm.truncateBlockingWithoutSnapshot();
+    }
+
+    @Test
+    public void testStreamingEntireSSTables() throws ExecutionException, InterruptedException
+    {
+        DatabaseDescriptor.getRawConfig().stream_entire_sstables = true;
+        testStreamingSSTables();
+    }
+
+    @Test
+    public void testStreamingPartialSSTables() throws ExecutionException, InterruptedException
+    {
+        DatabaseDescriptor.getRawConfig().stream_entire_sstables = false;
+        testStreamingSSTables();
+    }
+
+    private void testStreamingSSTables() throws ExecutionException, InterruptedException
+    {
+        SSTableLoader.Client client = new SSTableLoader.Client()
+        {
+            @Override
+            public void init(String keyspace)
+            {
+                for (Replica replica : StorageService.instance.getLocalReplicas(keyspace))
+                    addRangeForEndpoint(replica.range(), FBUtilities.getBroadcastAddressAndPort());
+            }
+
+            @Override
+            public TableMetadataRef getTableMetadata(String tableName)
+            {
+                return metadataRef;
+            }
+        };
+        try
+        {
+            SSTableLoader loader = new SSTableLoader(new File(tableDataDir), client, new OutputHandler.LogOutput(logger));
+            loader.stream().get();
+        }
+        finally
+        {
+            client.stop();
+        }
+        assertThat(realm.getLiveSSTables()).hasSize(4);
+
+        realm.forceMajorCompaction();
+
+        assertThat(realm.getLiveSSTables()).hasSizeLessThan(4);
+    }
+
+
+    @Test
+    public void testIterations()
+    {
+        LifecycleTransaction.getFiles(tableDataDir, (file, fileType) -> Descriptor.fromFilenameWithComponent(file).right == Components.DATA, Directories.OnTxnErr.THROW).forEach(file -> {
+            Descriptor desc = Descriptor.fromFilename(file);
+            SSTableReader sstable = desc.getFormat().getReaderFactory().loadingBuilder(desc, metadataRef, desc.discoverComponents()).build(realm, true, false);
+            assertThatCode(() -> {
+                assertThat(sstable.getSSTableMetadata().zeroCopyMetadata.exists()).isTrue();
+                try
+                {
+                    checkSSTableReader(sstable);
+                    checkVerifier(sstable);
+                    checkScrubber(sstable);
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeException(e);
+                }
+                finally
+                {
+                    if (sstable.selfRef().globalCount() > 0) sstable.selfRef().release();
+                }
+            }).describedAs(sstable.toString()).doesNotThrowAnyException();
+        });
+    }
+
+    private void checkScrubber(SSTableReader sstable)
+    {
+        IScrubber.Options options = IScrubber.options()
+                                             .skipCorrupted(false)
+                                             .checkData(true)
+                                             .reinsertOverflowedTTLRows(false)
+                                             .build();
+        try (IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(realm, LifecycleTransaction.offline(OperationType.SCRUB, sstable), new OutputHandler.LogOutput(), options))
+        {
+            IScrubber.ScrubResult result = scrubber.scrubWithResult();
+            assertThat(result.badPartitions).isZero();
+        }
+    }
+
+    private void checkVerifier(SSTableReader sstable)
+    {
+        IVerifier.Options options = IVerifier.options()
+                                             .extendedVerification(true)
+                                             .checkVersion(false)
+                                             .quick(false)
+                                             .build();
+
+        try (IVerifier verifier = sstable.getVerifier(realm, new OutputHandler.LogOutput(), false, options))
+        {
+            verifier.verify();
+        }
+    }
+
+    private void checkSSTableReader(SSTableReader sstable) throws IOException
+    {
+        List<DecoratedKey> keys = new ArrayList<>();
+        try (RandomAccessReader dataReader = sstable.openDataReader())
+        {
+            checkAllKeysIterator(sstable, dataReader, keys);
+            checkSrubPartitionsIterator(sstable, dataReader, keys);
+
+            // test getting and iterating over a single partition
+            for (DecoratedKey key : keys)
+            {
+                int idx = keys.indexOf(key);
+
+                // EQ position
+                long eqPos = sstable.getPosition(key, SSTableReader.Operator.EQ);
+                dataReader.seek(eqPos);
+                ByteBuffer keyFromDataFile = ByteBufferUtil.readWithShortLength(dataReader);
+                assertThat(sstable.getPartitioner().decorateKey(keyFromDataFile)).isEqualTo(key);
+
+                // GE position
+                long gePos = sstable.getPosition(key, SSTableReader.Operator.GE);
+                assertThat(gePos).isEqualTo(eqPos); // because the key exists
+
+                // GT position
+                long gtPos = sstable.getPosition(key, SSTableReader.Operator.GT);
+                if (idx != keys.size() - 1)
+                {
+                    DecoratedKey nextKey = keys.get(idx + 1);
+                    long nextEqPos = sstable.getPosition(nextKey, SSTableReader.Operator.EQ);
+                    assertThat(gtPos).isGreaterThan(eqPos);
+                    assertThat(gtPos).isEqualTo(nextEqPos);
+                }
+
+                if (idx == 0)
+                    assertThat(key).isEqualTo(sstable.first);
+                if (idx == keys.size() - 1)
+                    assertThat(key).isEqualTo(sstable.last);
+
+                try (UnfilteredRowIterator it = sstable.simpleIterator(dataReader, key, sstable.getDataFileSliceDescriptor().dataStart, false))
+                {
+                    while (it.hasNext())
+                    {
+                        Unfiltered next = it.next();
+                        next.validateData(sstable.metadata());
+                    }
+                }
+
+                try (UnfilteredRowIterator it = sstable.rowIterator(key, Slices.ALL, ColumnFilter.NONE, false, SSTableReadsListener.NOOP_LISTENER))
+                {
+                    while (it.hasNext())
+                    {
+                        Unfiltered next = it.next();
+                        next.validateData(sstable.metadata());
+                    }
+                }
+            }
+        }
+    }
+
+    private static void checkSrubPartitionsIterator(SSTableReader sstable, RandomAccessReader dataReader, List<DecoratedKey> keys) throws IOException
+    {
+        try (ScrubPartitionIterator it = ((BtiTableReader) sstable).scrubPartitionsIterator())
+        {
+            while (!it.isExhausted())
+            {
+                dataReader.seek(it.dataPosition());
+                ByteBuffer keyFromDataFile = ByteBufferUtil.readWithShortLength(dataReader);
+                DecoratedKey key = sstable.getPartitioner().decorateKey(keyFromDataFile);
+                assertThat(keys).contains(key);
+                it.advance();
+            }
+        }
+    }
+
+    private static void checkAllKeysIterator(SSTableReader sstable, RandomAccessReader dataReader, List<DecoratedKey> keys) throws IOException
+    {
+        try (KeyReader it = sstable.keyReader())
+        {
+            while (!it.isExhausted())
+            {
+                ByteBuffer keyFromIterator = it.key();
+                dataReader.seek(it.dataPosition());
+                ByteBuffer keyFromDataFile = ByteBufferUtil.readWithShortLength(dataReader);
+                DecoratedKey key = sstable.getPartitioner().decorateKey(keyFromDataFile);
+                assertThat(keyFromIterator).isEqualTo(keyFromDataFile);
+                assertThat(key).isGreaterThanOrEqualTo(sstable.first);
+                assertThat(key).isLessThanOrEqualTo(sstable.last);
+                keys.add(key);
+                it.advance();
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
index 556f55f71262..ae99b9db8367 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
@@ -54,7 +54,7 @@ public void testAbortTxnWithOpenEarlyShouldRemoveSSTable()
         truncate(cfs);
 
         File dir = cfs.getDirectories().getDirectoryForNewSSTables();
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE, cfs.metadata);
         try (SSTableWriter writer = getWriter(cfs, dir, txn))
         {
             for (int i = 0; i < 10000; i++)
@@ -100,7 +100,6 @@ public void testAbortTxnWithOpenEarlyShouldRemoveSSTable()
         }
     }
 
-
     @Test
     public void testAbortTxnWithClosedWriterShouldRemoveSSTable()
     {
@@ -109,7 +108,7 @@ public void testAbortTxnWithClosedWriterShouldRemoveSSTable()
         truncate(cfs);
 
         File dir = cfs.getDirectories().getDirectoryForNewSSTables();
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM, cfs.metadata);
         try (SSTableWriter writer = getWriter(cfs, dir, txn))
         {
             for (int i = 0; i < 10000; i++)
@@ -153,7 +152,7 @@ public void testAbortTxnWithClosedAndOpenWriterShouldRemoveAllSSTables()
         truncate(cfs);
 
         File dir = cfs.getDirectories().getDirectoryForNewSSTables();
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM, cfs.metadata);
 
         SSTableWriter writer1 = getWriter(cfs, dir, txn);
         SSTableWriter writer2 = getWriter(cfs, dir, txn);
@@ -207,7 +206,7 @@ public void testValueTooBigCorruption() throws InterruptedException
         truncate(cfs);
 
         File dir = cfs.getDirectories().getDirectoryForNewSSTables();
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM, cfs.metadata);
 
         try (SSTableWriter writer1 = getWriter(cfs, dir, txn))
         {
@@ -248,7 +247,7 @@ private static void assertValidRepairMetadata(long repairedAt, TimeUUID pendingR
         Keyspace keyspace = Keyspace.open(KEYSPACE);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_SMALL_MAX_VALUE);
         File dir = cfs.getDirectories().getDirectoryForNewSSTables();
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM, cfs.metadata);
 
         try (SSTableWriter writer = getWriter(cfs, dir, txn, repairedAt, pendingRepair, isTransient))
         {
@@ -268,7 +267,7 @@ private static void assertInvalidRepairMetadata(long repairedAt, TimeUUID pendin
         Keyspace keyspace = Keyspace.open(KEYSPACE);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_SMALL_MAX_VALUE);
         File dir = cfs.getDirectories().getDirectoryForNewSSTables();
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM, cfs.metadata);
 
         try (SSTableWriter writer = getWriter(cfs, dir, txn, repairedAt, pendingRepair, isTransient))
         {
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
index 061591500ee5..2d09ff5da73f 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
@@ -39,6 +39,7 @@
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -174,6 +175,22 @@ public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, Lif
         return getWriter(cfs, directory, txn, 0, null, false);
     }
 
+    public static SSTableWriter getWriter(SSTableFormat<?, ?> format, ColumnFamilyStore cfs, File directory, LifecycleTransaction txn)
+    {
+        Descriptor desc = cfs.newSSTableDescriptor(directory, format);
+        return desc.getFormat().getWriterFactory().builder(desc)
+                   .setTableMetadataRef(cfs.metadata)
+                   .setKeyCount(0)
+                   .setRepairedAt(0)
+                   .setPendingRepair(null)
+                   .setTransientSSTable(false)
+                   .setSerializationHeader(new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))
+                   .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups())
+                   .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator))
+                   .addDefaultComponents(cfs.indexManager.listIndexGroups())
+                   .build(txn, cfs);
+    }
+
     public static ByteBuffer random(int i, int size)
     {
         byte[] bytes = new byte[size + 4];
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java
index b439b9c98015..e0cdd8ea737d 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java
@@ -22,6 +22,7 @@
 import java.util.Collection;
 
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.BeforeClass;
 
 import org.apache.cassandra.SchemaLoader;
@@ -33,6 +34,7 @@
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
@@ -47,6 +49,7 @@ public class SSTableWriterTransactionTest extends AbstractTransactionalTest
     @BeforeClass
     public static void defineSchema() throws Exception
     {
+        Assume.assumeTrue(BigFormat.isSelected());
         SchemaLoader.prepareServer();
         SchemaLoader.createKeyspace(KEYSPACE1,
                                     KeyspaceParams.simple(1),
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java
index 4d64a73602cb..2bb287e90f60 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java
@@ -148,7 +148,7 @@ private void writeDataTestCycle(Function<ByteBuffer, DataInputPlus> bufferMapper
         Descriptor desc = store.newSSTableDescriptor(dir);
         TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc);
 
-        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+        LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM, metadata);
         Set<Component> componentsToWrite = new HashSet<>(desc.getFormat().uploadComponents());
         if (!metadata.getLocal().params.compression.isEnabled())
             componentsToWrite.remove(Components.COMPRESSION_INFO);
diff --git a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java
index e7952549035a..dbf09125e0d6 100644
--- a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -35,6 +36,7 @@
 import java.util.SortedSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.google.common.collect.Sets;
@@ -58,6 +60,7 @@
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ClusteringPrefix;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.CounterMutation;
@@ -76,6 +79,8 @@
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -85,12 +90,14 @@
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.io.compress.CompressionMetadata;
 import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
 import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
 import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.DataIntegrityMetadata;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.KeyspaceParams;
@@ -100,6 +107,7 @@
 import org.jboss.byteman.contrib.bmunit.BMRule;
 import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
 
+import static org.apache.cassandra.SchemaLoader.compressionParams;
 import static org.apache.cassandra.SchemaLoader.counterCFMD;
 import static org.apache.cassandra.SchemaLoader.createKeyspace;
 import static org.apache.cassandra.SchemaLoader.getCompressionParameters;
@@ -121,6 +129,7 @@ public class ScrubTest
     private final static Logger logger = LoggerFactory.getLogger(ScrubTest.class);
 
     public static final String CF = "Standard1";
+    public static final String COMPRESSED_CF = "compressed_table";
     public static final String COUNTER_CF = "Counter1";
     public static final String CF_UUID = "UUIDKeys";
     public static final String CF_INDEX1 = "Indexed1";
@@ -155,6 +164,8 @@ public void setup()
         createKeyspace(ksName,
                        KeyspaceParams.simple(1),
                        standardCFMD(ksName, CF),
+                       // force this table to use compression
+                       standardCFMD(ksName, COMPRESSED_CF).compression(compressionParams(COMPRESSION_CHUNK_LENGTH)),
                        counterCFMD(ksName, COUNTER_CF).compression(getCompressionParameters(COMPRESSION_CHUNK_LENGTH)),
                        standardCFMD(ksName, CF_UUID, 0, UUIDType.instance),
                        SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1, true),
@@ -201,8 +212,30 @@ public void testScrubLastBrokenPartition() throws IOException
 
         Set<SSTableReader> liveSSTables = cfs.getLiveSSTables();
         assertThat(liveSSTables).hasSize(1);
-        String fileName = liveSSTables.iterator().next().getFilename();
-        Files.write(Paths.get(fileName), new byte[10], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+        File file = liveSSTables.iterator().next().getDataFile();
+        Files.write(Paths.get(file.path()), new byte[10], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+        if (ChunkCache.instance != null)
+            ChunkCache.instance.invalidateFile(file);
+
+        performScrub(cfs, true, true, false, 2);
+
+        // check data is still there
+        assertOrderedAll(cfs, 0);
+    }
+
+    @Test
+    public void testScrubOneBrokenPartition() throws ExecutionException, InterruptedException, IOException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+        // insert data and verify we get it back w/ range query
+        fillCF(cfs, 1);
+        assertOrderedAll(cfs, 1);
+
+        Set<SSTableReader> liveSSTables = cfs.getLiveSSTables();
+        assertThat(liveSSTables).hasSize(1);
+        Files.write(liveSSTables.iterator().next().getDataFile().toPath(), new byte[10], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
 
         performScrub(cfs, true, true, false, 2);
 
@@ -210,6 +243,56 @@ public void testScrubLastBrokenPartition() throws IOException
         assertOrderedAll(cfs, 0);
     }
 
+    @Test
+    public void testScrubOneBrokenPartitionInTheMiddleOfCompressedFile() throws ExecutionException, InterruptedException, IOException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COMPRESSED_CF);
+
+        // insert data and verify we get it back w/ range query
+        int partitions = 5;
+        int rowsPerPartition = 2000;
+        int total = partitions * rowsPerPartition;
+        fillCF(cfs, partitions, rowsPerPartition);
+        int outputRows = assertRowsOrdered(Util.cmd(cfs).build());
+        assertThat(outputRows).isEqualTo(total);
+
+        Set<SSTableReader> liveSSTables = cfs.getLiveSSTables();
+        assertThat(liveSSTables).hasSize(1);
+
+        try (FileChannel channel = FileChannel.open(liveSSTables.iterator().next().getDataFile().toPath(), StandardOpenOption.WRITE))
+        {
+            // corrupt data in the middle
+            long middle = channel.size() / 2;
+            channel.position(middle);
+
+            byte[] buffer = new byte[50];
+            ThreadLocalRandom.current().nextBytes(buffer);
+            channel.write(ByteBuffer.wrap(buffer));
+        }
+
+        if (ChunkCache.instance != null)
+            ChunkCache.instance.clear();
+        IScrubber.Options options = IScrubber.options()
+                                             .skipCorrupted(true)
+                                             .checkData(true)
+                                             .reinsertOverflowedTTLRows(false)
+                                             .build();
+        CompactionManager.instance.performScrub(cfs, options, 1);
+
+        // check data is still there, some corrupted partitions are discarded
+        outputRows = assertRowsOrdered(Util.cmd(cfs).build());
+        assertThat(outputRows).isGreaterThan(0).isLessThan(total);
+
+        // check digest file do not exist because scruber can't reset the CRC value after resetting file position when corruption is found.
+        SSTableReader outputSSTable = cfs.getLiveSSTables().iterator().next();
+        assertThat(outputSSTable.descriptor.fileFor(SSTableFormat.Components.DIGEST).exists()).isTrue();
+
+        DataIntegrityMetadata.FileDigestValidator validator = outputSSTable.maybeGetDigestValidator();
+        assertNotNull(validator);
+        validator.validate();
+    }
+
     @Test
     public void testScrubCorruptedCounterPartition() throws IOException, WriteTimeoutException
     {
@@ -243,6 +326,7 @@ public void testScrubCorruptedCounterPartition() throws IOException, WriteTimeou
         catch (IOError err)
         {
             Throwables.assertAnyCause(err, CorruptSSTableException.class);
+            assertTrue(Throwables.isCausedBy(err, CorruptSSTableException.class));
         }
 
         // with skipCorrupted == true, the corrupt rows will be skipped
@@ -493,7 +577,7 @@ public void testScrubOutOfOrder()
             List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
             Descriptor desc = cfs.newSSTableDescriptor(tempDataDir);
 
-            try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
+            try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE, cfs.metadata);
                  SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, keys.size(), cfs, txn)))
             {
                 for (String k : keys)
@@ -603,7 +687,7 @@ private static void overrideWithGarbage(File path, long startPosition, long endP
             file.write(buff, 0, length);
         }
         if (ChunkCache.instance != null)
-            ChunkCache.instance.invalidateFile(path.toString());
+            ChunkCache.instance.invalidateFile(path);
     }
 
     public static void assertOrderedAll(ColumnFamilyStore cfs, int expectedSize)
@@ -629,6 +713,33 @@ private static void assertOrdered(ReadCommand cmd, int expectedSize)
         assertEquals(expectedSize, size);
     }
 
+    private static int assertRowsOrdered(ReadCommand cmd)
+    {
+        int size = 0;
+        DecoratedKey prev = null;
+        for (Partition partition : Util.getAllUnfiltered(cmd))
+        {
+            DecoratedKey current = partition.partitionKey();
+            assertTrue("key " + current + " does not sort after previous key " + prev, prev == null || prev.compareTo(current) < 0);
+            prev = current;
+
+            ClusteringPrefix<?> prevClustering = null;
+            UnfilteredRowIterator rows = partition.unfilteredIterator();
+            while (rows.hasNext())
+            {
+                Unfiltered unfiltered = rows.next();
+                ClusteringPrefix<?> currentClustering = unfiltered.clustering();
+
+                assertTrue("Clustering " + currentClustering + " does not sort after previous clustering " + prevClustering,
+                           prevClustering == null || cmd.metadata().comparator.compare(prevClustering, currentClustering) < 0);
+
+                prevClustering = currentClustering;
+                size++;
+            }
+        }
+        return size;
+    }
+
     public static void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
     {
         for (int i = 0; i < partitionsPerSSTable; i++)
@@ -644,6 +755,23 @@ public static void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
         Util.flush(cfs);
     }
 
+    protected static void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable, int rowsPerPartition)
+    {
+        for (int i = 0; i < partitionsPerSSTable; i++)
+        {
+            for (int r = 0; r < rowsPerPartition; r++)
+            {
+                PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
+                                                      .newRow(String.valueOf(r)).add("val", "1")
+                                                      .build();
+
+                new Mutation(update).applyUnsafe();
+            }
+        }
+
+        Util.flush(cfs);
+    }
+
     public static void fillIndexCF(ColumnFamilyStore cfs, boolean composite, long... values)
     {
         assertEquals(0, values.length % 2);
diff --git a/test/unit/org/apache/cassandra/io/sstable/UDTMoveTest.java b/test/unit/org/apache/cassandra/io/sstable/UDTMoveTest.java
new file mode 100644
index 000000000000..b8665c3d0aca
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/UDTMoveTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.SequentialWriter;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class UDTMoveTest extends CQLTester
+{
+    @Test
+    public void testMovingKeyspaceManually() throws Throwable
+    {
+        String table = "tab";
+        String udt = "udt";
+        String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+        String ks2 = createKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+        schemaChange(String.format("CREATE TYPE %s.%s (a int, b int)", ks1, udt));
+        schemaChange(String.format("CREATE TABLE %s.%s (id int PRIMARY KEY, v udt, a int, b text, c float)", ks1, table));
+        schemaChange(String.format("CREATE TYPE %s.%s (a int, b int)", ks2, udt));
+        schemaChange(String.format("CREATE TABLE %s.%s (id int PRIMARY KEY, v udt, a int, b text, c float)", ks2, table));
+
+        disableCompaction(ks1, table);
+        disableCompaction(ks2, table);
+
+        execute(String.format("INSERT INTO %s.%s (id, v, a, b, c) VALUES (1, {a: 1, b: 2}, 3, '4', 5.0)", ks1, table));
+        execute(String.format("INSERT INTO %s.%s (id, v, a, b, c) VALUES (2, {a: 2, b: 3}, 6, '7', 8.0)", ks1, table));
+        flush(ks1, table);
+
+        // query data
+        UntypedResultSet rows = execute(String.format("SELECT * FROM %s.%s", ks1, table));
+        assertRows(rows,
+                   row(1, 3, "4", 5.0f, userType("a", 1, "b", 2)),
+                   row(2, 6, "7", 8.0f, userType("a", 2, "b", 3)));
+
+        Set<SSTableReader> sstableReaders1 = getColumnFamilyStore(ks1, table).getLiveSSTables();
+        Path sstablesLocation2 = getColumnFamilyStore(ks2, table).getDirectories().getDirectoryForNewSSTables().toPath();
+        assertThat(sstableReaders1).hasSizeGreaterThan(0);
+
+        for (SSTableReader reader : sstableReaders1)
+        {
+            for (Component component : reader.descriptor.discoverComponents())
+            {
+                Path sourcePath = reader.descriptor.pathFor(component);
+                if (Files.exists(sourcePath))
+                {
+                    logger.info("Copying {} to {}", sourcePath, sstablesLocation2.resolve(sourcePath.getFileName()));
+
+                    if (component.equals(SSTableFormat.Components.STATS))
+                    {
+                        Map<MetadataType, MetadataComponent> metadata = new HashMap<>(new MetadataSerializer().deserialize(reader.descriptor, EnumSet.allOf(MetadataType.class)));
+                        SerializationHeader.Component serializationHeader = (SerializationHeader.Component) metadata.get(MetadataType.HEADER);
+                        serializationHeader = serializationHeader.withMigratedKeyspaces(ImmutableMap.of(ks1, ks2));
+                        metadata.put(MetadataType.HEADER, serializationHeader);
+
+                        try (SequentialWriter out = new SequentialWriter(new File(sstablesLocation2.resolve(sourcePath.getFileName()))))
+                        {
+                            new MetadataSerializer().serialize(metadata, out, reader.descriptor.version);
+                            out.sync();
+                        }
+
+                        assertThat(Files.exists(sstablesLocation2.resolve(sourcePath.getFileName()))).isTrue();
+                        assertThat(Files.size(sstablesLocation2.resolve(sourcePath.getFileName()))).isGreaterThan(0);
+                    }
+                    else
+                    {
+                        Files.copy(sourcePath, sstablesLocation2.resolve(sourcePath.getFileName()));
+                    }
+                }
+            }
+        }
+
+        getColumnFamilyStore(ks2, table).loadNewSSTables();
+        rows = execute(String.format("SELECT * FROM %s.%s", ks2, table));
+        assertRows(rows,
+                   row(1, 3, "4", 5.0f, userType("a", 1, "b", 2)),
+                   row(2, 6, "7", 8.0f, userType("a", 2, "b", 3)));
+
+        execute(String.format("INSERT INTO %s.%s (id, v, a, b, c) VALUES (2, {a: 3, b: 2}, 4, '5', 6.0)", ks2, table));
+        execute(String.format("INSERT INTO %s.%s (id, v, a, b, c) VALUES (3, {a: 3, b: 4}, 7, '8', 9.0)", ks2, table));
+
+        rows = execute(String.format("SELECT * FROM %s.%s", ks2, table));
+        assertRows(rows,
+                   row(1, 3, "4", 5.0f, userType("a", 1, "b", 2)),
+                   row(2, 4, "5", 6.0f, userType("a", 3, "b", 2)),
+                   row(3, 7, "8", 9.0f, userType("a", 3, "b", 4)));
+
+        flush(ks2, table);
+        rows = execute(String.format("SELECT * FROM %s.%s", ks2, table));
+        assertRows(rows,
+                   row(1, 3, "4", 5.0f, userType("a", 1, "b", 2)),
+                   row(2, 4, "5", 6.0f, userType("a", 3, "b", 2)),
+                   row(3, 7, "8", 9.0f, userType("a", 3, "b", 4)));
+
+        compact(ks2, table);
+        rows = execute(String.format("SELECT * FROM %s.%s", ks2, table));
+        assertRows(rows,
+                   row(1, 3, "4", 5.0f, userType("a", 1, "b", 2)),
+                   row(2, 4, "5", 6.0f, userType("a", 3, "b", 2)),
+                   row(3, 7, "8", 9.0f, userType("a", 3, "b", 4)));
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
index 40bbe0887f71..fe7b36aae40c 100644
--- a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
@@ -21,15 +21,19 @@
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.zip.CRC32;
 import java.util.zip.CheckedInputStream;
 
 import com.google.common.base.Charsets;
+import com.google.common.collect.Iterables;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.Assume;
 import org.junit.BeforeClass;
@@ -47,9 +51,11 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Range;
@@ -82,6 +88,7 @@
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -108,12 +115,17 @@ public class VerifyTest
     public static final String COUNTER_CF4 = "Counter4";
     public static final String CORRUPT_CF = "Corrupt1";
     public static final String CORRUPT_CF2 = "Corrupt2";
+    public static final String CORRUPT_CF3 = "Corrupt3";
+    public static final String CORRUPT_COMPRESSED_CF = "CorruptCompressed";
+    public static final int CORRUPT_COMPRESSED_CF_CHUNK_LENGTH = 4096;
     public static final String CORRUPTCOUNTER_CF = "CounterCorrupt1";
     public static final String CORRUPTCOUNTER_CF2 = "CounterCorrupt2";
 
     public static final String CF_UUID = "UUIDKeys";
     public static final String BF_ALWAYS_PRESENT = "BfAlwaysPresent";
 
+    private String savedProp;
+
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
     {
@@ -130,6 +142,8 @@ public static void defineSchema() throws ConfigurationException
                        standardCFMD(KEYSPACE, CF4),
                        standardCFMD(KEYSPACE, CORRUPT_CF),
                        standardCFMD(KEYSPACE, CORRUPT_CF2),
+                       standardCFMD(KEYSPACE, CORRUPT_CF3),
+                       standardCFMD(KEYSPACE, CORRUPT_COMPRESSED_CF).compression(CompressionParams.lz4(CORRUPT_COMPRESSED_CF_CHUNK_LENGTH)),
                        counterCFMD(KEYSPACE, COUNTER_CF).compression(compressionParameters),
                        counterCFMD(KEYSPACE, COUNTER_CF2).compression(compressionParameters),
                        counterCFMD(KEYSPACE, COUNTER_CF3),
@@ -140,7 +154,6 @@ public static void defineSchema() throws ConfigurationException
                        standardCFMD(KEYSPACE, BF_ALWAYS_PRESENT).bloomFilterFpChance(1.0));
     }
 
-
     @Test
     public void testVerifyCorrect()
     {
@@ -373,7 +386,7 @@ public void testVerifyCorruptRowCorrectDigest() throws IOException, WriteTimeout
             file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
         }
         if (ChunkCache.instance != null)
-            ChunkCache.instance.invalidateFile(sstable.getFilename());
+            ChunkCache.instance.invalidateFileNow(sstable.getDataFile());
 
         // Update the Digest to have the right Checksum
         writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.fileFor(Components.DIGEST));
@@ -406,6 +419,79 @@ public void testVerifyCorruptRowCorrectDigest() throws IOException, WriteTimeout
         }
     }
 
+    @Test
+    public void testVerifyCorruptCellInTheMiddleOfPartitionCorrectDigest() throws IOException, WriteTimeoutException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_COMPRESSED_CF);
+
+        // insert wide partition data
+        int partitions = 1;
+        int rowsPerPartition = 5000;
+        fillCF(cfs, partitions, rowsPerPartition);
+
+        SSTableReader sstable = Iterables.getOnlyElement(cfs.getLiveSSTables());
+        long partitionStart = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ);
+        try (FileChannel channel = FileChannel.open(sstable.getDataFile().toPath(), StandardOpenOption.WRITE))
+        {
+            // Corrupt data after first chunk of the partition. Verifier will have to iterate rows inside partition to detect it
+            long middle = partitionStart + CORRUPT_COMPRESSED_CF_CHUNK_LENGTH + 20;
+            channel.position(middle);
+
+            byte[] buffer = new byte[50];
+            ThreadLocalRandom.current().nextBytes(buffer);
+            channel.write(ByteBuffer.wrap(buffer));
+        }
+
+        // Update the Digest to have the updated file Checksum
+        writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.fileFor(Components.DIGEST));
+        ChunkCache.instance.clear();
+
+        try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+        {
+            // First a simple verify checking digest, which should succeed
+            try
+            {
+                verifier.verify();
+            }
+            catch (CorruptSSTableException err)
+            {
+                fail("Simple verify should have succeeded as digest matched");
+            }
+        }
+        try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true)
+                                                                           .extendedVerification(true).build()))
+        {
+            // Extended verify withotu validating all rows won't detect corruption inside partition
+            try
+            {
+                verifier.verify();
+            }
+            catch (CorruptSSTableException err)
+            {
+                fail("Simple verify should have succeeded as corruption is in the middle of partition");
+            }
+        }
+        try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true)
+                                                                           .extendedVerification(true)
+                                                                           .validateAllRows(true)
+                                                                           .build()))
+        {
+            // Now try extended verify with validating all rows
+            try
+            {
+                verifier.verify();
+
+            }
+            catch (CorruptSSTableException err)
+            {
+                return;
+            }
+            fail("Expected a CorruptSSTableException to be thrown");
+        }
+    }
+
     @Test
     public void testVerifyBrokenSSTableMetadata() throws IOException, WriteTimeoutException
     {
@@ -520,7 +606,7 @@ public void testMutateRepair() throws IOException
     {
         CompactionManager.instance.disableAutoCompaction();
         Keyspace keyspace = Keyspace.open(KEYSPACE);
-        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF3);
 
         fillCF(cfs, 2);
 
@@ -560,6 +646,13 @@ else if (BtiFormat.isSelected())
             throw Util.testMustBeImplementedForSSTableFormat();
     }
 
+    @Test
+    public void testVerifyRowIndex() throws IOException
+    {
+        Assume.assumeTrue(BtiFormat.isSelected());
+        testBrokenComponentHelper(BtiFormat.Components.ROW_INDEX);
+    }
+
     @Test
     public void testVerifyBf() throws IOException
     {
@@ -769,6 +862,46 @@ public void testNoFilterFile()
         }
     }
 
+    @Test
+    public void testVerifyWithoutRealm()
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+        fillCF(cfs, 2);
+
+        Descriptor descriptor = cfs.getLiveSSTables().iterator().next().getDescriptor();
+        SSTableReader sstable = SSTableReader.openNoValidation(null, descriptor, cfs.metadata);
+        IVerifier.Options options = IVerifier.options().invokeDiskFailurePolicy(true).build();
+
+        try (IVerifier verifier = sstable.getVerifier(null, new OutputHandler.LogOutput(logger), true, options))
+        {
+            verifier.verify();
+        }
+        catch (CorruptSSTableException err)
+        {
+            fail("Unexpected CorruptSSTableException");
+        }
+    }
+
+    @Test
+    public void testVerifierIllegalArgument()
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+        fillCF(cfs, 2);
+
+        SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+        // Check that is not possible to create a Verifier without passing a ColumnFamilyStore
+        // if mutateRepairStatus is true.
+        IVerifier.Options options = IVerifier.options().mutateRepairStatus(true).build();
+        assertThrows(IllegalArgumentException.class,
+                     () -> sstable.getVerifier(null, new OutputHandler.LogOutput(logger), false, options));
+    }
 
     private DecoratedKey dk(long l)
     {
@@ -799,6 +932,23 @@ protected void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
         Util.flush(cfs);
     }
 
+    protected static void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable, int rowsPerPartition)
+    {
+        for (int i = 0; i < partitionsPerSSTable; i++)
+        {
+            for (int r = 0; r < rowsPerPartition; r++)
+            {
+                PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
+                                                      .newRow(String.valueOf(r)).add("val", "1")
+                                                      .build();
+
+                new Mutation(update).applyUnsafe();
+            }
+        }
+
+        Util.flush(cfs);
+    }
+
     protected void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException
     {
         for (int i = 0; i < partitionsPerSSTable; i++)
diff --git a/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java b/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java
index 3c954556f3a6..1400c7d5f1dc 100644
--- a/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java
@@ -19,49 +19,63 @@
 package org.apache.cassandra.io.sstable.filter;
 
 
+import java.util.concurrent.TimeUnit;
+
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+
 import static org.junit.Assert.assertEquals;
 
 public class BloomFilterTrackerTest
 {
+    @BeforeClass
+    public static void setUp()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
     @Test
-    public void testAddingFalsePositives()
+    public void testAddingFalsePositives() throws InterruptedException
     {
-        BloomFilterTracker bft = new BloomFilterTracker();
+        BloomFilterTracker bft = BloomFilterTracker.createMeterTracker();
         assertEquals(0L, bft.getFalsePositiveCount());
-        assertEquals(0L, bft.getRecentFalsePositiveCount());
+        assertEquals(0d, bft.getRecentFalsePositiveRate(), 0.0);
         bft.addFalsePositive();
         bft.addFalsePositive();
+        Thread.sleep(TimeUnit.SECONDS.toMillis(5L)); // wait for tick that updates rates
         assertEquals(2L, bft.getFalsePositiveCount());
-        assertEquals(2L, bft.getRecentFalsePositiveCount());
-        assertEquals(0L, bft.getRecentFalsePositiveCount());
+        assertEquals(0.4d, bft.getRecentFalsePositiveRate(), 0.0);
+        assertEquals(0.4d, bft.getRecentFalsePositiveRate(), 0.0);
         assertEquals(2L, bft.getFalsePositiveCount()); // sanity check
     }
 
     @Test
-    public void testAddingTruePositives()
+    public void testAddingTruePositives()  throws InterruptedException
     {
-        BloomFilterTracker bft = new BloomFilterTracker();
+        BloomFilterTracker bft = BloomFilterTracker.createMeterTracker();
         assertEquals(0L, bft.getTruePositiveCount());
-        assertEquals(0L, bft.getRecentTruePositiveCount());
+        assertEquals(0d, bft.getRecentTruePositiveRate(), 0.0);
         bft.addTruePositive();
         bft.addTruePositive();
+        Thread.sleep(TimeUnit.SECONDS.toMillis(5L)); // wait for tick that updates rates
         assertEquals(2L, bft.getTruePositiveCount());
-        assertEquals(2L, bft.getRecentTruePositiveCount());
-        assertEquals(0L, bft.getRecentTruePositiveCount());
+        assertEquals(0.4d, bft.getRecentTruePositiveRate(), 0.0);
+        assertEquals(0.4d, bft.getRecentTruePositiveRate(), 0.0);
         assertEquals(2L, bft.getTruePositiveCount()); // sanity check
     }
 
     @Test
-    public void testAddingToOneLeavesTheOtherAlone()
+    public void testAddingToOneLeavesTheOtherAlone() throws InterruptedException
     {
-        BloomFilterTracker bft = new BloomFilterTracker();
+        BloomFilterTracker bft = BloomFilterTracker.createMeterTracker();
         bft.addFalsePositive();
         assertEquals(0L, bft.getTruePositiveCount());
-        assertEquals(0L, bft.getRecentTruePositiveCount());
+        assertEquals(0d, bft.getRecentTruePositiveRate(), 0.0);
         bft.addTruePositive();
+        Thread.sleep(TimeUnit.SECONDS.toMillis(5L)); // wait for tick that updates rates
         assertEquals(1L, bft.getFalsePositiveCount());
-        assertEquals(1L, bft.getRecentFalsePositiveCount());
+        assertEquals(0.2d, bft.getRecentFalsePositiveRate(), 0.0);
     }
 }
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/AbstractTestVersionSupportedFeatures.java b/test/unit/org/apache/cassandra/io/sstable/format/AbstractTestVersionSupportedFeatures.java
index a6c4c06c2667..be3d7cf2c4da 100644
--- a/test/unit/org/apache/cassandra/io/sstable/format/AbstractTestVersionSupportedFeatures.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/AbstractTestVersionSupportedFeatures.java
@@ -28,14 +28,14 @@
 import org.junit.Test;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.assertj.core.api.Assertions;
+import org.assertj.core.api.SoftAssertions;
 
 public abstract class AbstractTestVersionSupportedFeatures
 {
-    protected static final List<String> ALL_VERSIONS = IntStream.rangeClosed('a', 'z')
-                                                                .mapToObj(i -> String.valueOf((char) i))
-                                                                .flatMap(first -> IntStream.rangeClosed('a', 'z').mapToObj(second -> first + (char) second))
-                                                                .collect(Collectors.toList());
+    public static final List<String> ALL_VERSIONS = IntStream.rangeClosed('a', 'z')
+                                                             .mapToObj(i -> String.valueOf((char) i))
+                                                             .flatMap(first -> IntStream.rangeClosed('a', 'z').mapToObj(second -> first + (char) second))
+                                                             .collect(Collectors.toList());
 
     protected abstract Version getVersion(String v);
 
@@ -51,6 +51,26 @@ public abstract class AbstractTestVersionSupportedFeatures
 
     protected abstract Stream<String> getOriginatingHostIdSupportedVersions();
 
+    protected abstract Stream<String> getAccurateMinMaxSupportedVersions();
+
+    protected abstract Stream<String> getCommitLogLowerBoundSupportedVersions();
+
+    protected abstract Stream<String> getCommitLogIntervalsSupportedVersions();
+
+    protected abstract Stream<String> getZeroCopyMetadataSupportedVersions();
+
+    protected abstract Stream<String> getIncrementalNodeSyncMetadataSupportedVersions();
+
+    protected abstract Stream<String> getMaxColumnValueLengthsSupportedVersions();
+
+    protected abstract Stream<String> getIsTransientSupportedVersions();
+
+    protected abstract Stream<String> getMisplacedPartitionLevelDeletionsPresenceMarkerSupportedVersions();
+
+    protected abstract Stream<String> getTokenSpaceCoverageSupportedVersions();
+
+    protected abstract Stream<String> getOldBfFormatSupportedVersions();
+
     @BeforeClass
     public static void initDD()
     {
@@ -60,12 +80,27 @@ public static void initDD()
     @Test
     public void testCompatibility()
     {
-        checkPredicateAgainstVersions(Version::hasPendingRepair, getPendingRepairSupportedVersions());
-        checkPredicateAgainstVersions(Version::hasImprovedMinMax, getImprovedMinMaxSupportedVersions());
-        checkPredicateAgainstVersions(Version::hasLegacyMinMax, getLegacyMinMaxSupportedVersions());
-        checkPredicateAgainstVersions(Version::hasPartitionLevelDeletionsPresenceMarker, getPartitionLevelDeletionPresenceMarkerSupportedVersions());
-        checkPredicateAgainstVersions(Version::hasKeyRange, getKeyRangeSupportedVersions());
-        checkPredicateAgainstVersions(Version::hasOriginatingHostId, getOriginatingHostIdSupportedVersions());
+        SoftAssertions assertions = new SoftAssertions();
+        checkPredicateAgainstVersions(Version::hasPendingRepair, getPendingRepairSupportedVersions(), "hasPendingRepair", assertions);
+        checkPredicateAgainstVersions(Version::hasImprovedMinMax, getImprovedMinMaxSupportedVersions(), "hasImprovedMinMax", assertions);
+        checkPredicateAgainstVersions(Version::hasLegacyMinMax, getLegacyMinMaxSupportedVersions(), "hasLegacyMinMax", assertions);
+        checkPredicateAgainstVersions(Version::hasPartitionLevelDeletionsPresenceMarker, getPartitionLevelDeletionPresenceMarkerSupportedVersions(), "hasPartitionLevelDeletionsPresenceMarker", assertions);
+        checkPredicateAgainstVersions(Version::hasKeyRange, getKeyRangeSupportedVersions(), "hasKeyRange", assertions);
+        checkPredicateAgainstVersions(Version::hasOriginatingHostId, getOriginatingHostIdSupportedVersions(), "hasOriginatingHostId", assertions);
+        checkPredicateAgainstVersions(Version::hasAccurateMinMax, getAccurateMinMaxSupportedVersions(), "hasAccurateMinMax", assertions);
+        checkPredicateAgainstVersions(Version::hasCommitLogLowerBound, getCommitLogLowerBoundSupportedVersions(), "hasCommitLogLowerBound", assertions);
+        checkPredicateAgainstVersions(Version::hasCommitLogIntervals, getCommitLogIntervalsSupportedVersions(), "hasCommitLogIntervals", assertions);
+        checkPredicateAgainstVersions(Version::hasZeroCopyMetadata, getZeroCopyMetadataSupportedVersions(), "hasZeroCopyMetadata", assertions);
+        checkPredicateAgainstVersions(Version::hasIncrementalNodeSyncMetadata, getIncrementalNodeSyncMetadataSupportedVersions(), "hasIncrementalNodeSyncMetadata", assertions);
+        checkPredicateAgainstVersions(Version::hasMaxColumnValueLengths, getMaxColumnValueLengthsSupportedVersions(), "hasMaxColumnValueLengths", assertions);
+        checkPredicateAgainstVersions(Version::hasIsTransient, getIsTransientSupportedVersions(), "hasIsTransient", assertions);
+        checkPredicateAgainstVersions(Version::hasMisplacedPartitionLevelDeletionsPresenceMarker, getMisplacedPartitionLevelDeletionsPresenceMarkerSupportedVersions(), "hasMisplacedPartitionLevelDeletionsPresenceMarker", assertions);
+        checkPredicateAgainstVersions(Version::hasTokenSpaceCoverage, getTokenSpaceCoverageSupportedVersions(), "hasTokenSpaceCoverage", assertions);
+        checkPredicateAgainstVersions(Version::hasOldBfFormat, getOldBfFormatSupportedVersions(), "hasOldBfFormat", assertions);
+
+        checkPredicateAgainstVersions(v -> !(v.hasPartitionLevelDeletionsPresenceMarker() && v.hasMisplacedPartitionLevelDeletionsPresenceMarker()), ALL_VERSIONS.stream(), "hasPartitionLevelDeletionsPresenceMarker and hasMisplacedPartitionLevelDeletionsPresenceMarker", assertions);
+
+        assertions.assertAll();;
     }
 
     public static Stream<String> range(String fromIncl, String toIncl)
@@ -81,11 +116,12 @@ public static Stream<String> range(String fromIncl, String toIncl)
      *
      * @param predicate     predicate to check against version
      * @param versionBounds a stream of versions for which the predicate should return true
+     * @param assertions
      */
-    private void checkPredicateAgainstVersions(Predicate<Version> predicate, Stream<String> versionBounds)
+    private void checkPredicateAgainstVersions(Predicate<Version> predicate, Stream<String> versionBounds, String name, SoftAssertions assertions)
     {
         List<String> expected = versionBounds.collect(Collectors.toList());
         List<String> actual = ALL_VERSIONS.stream().filter(v -> predicate.test(getVersion(v))).collect(Collectors.toList());
-        Assertions.assertThat(actual).isEqualTo(expected);
+        assertions.assertThat(actual).describedAs(name).isEqualTo(expected);
     }
 }
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/GlobalTidyConcurrencyTest.java b/test/unit/org/apache/cassandra/io/sstable/format/GlobalTidyConcurrencyTest.java
new file mode 100644
index 000000000000..c240a81d25a7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/GlobalTidyConcurrencyTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.LockSupport;
+import java.util.stream.IntStream;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DEBUG_REF_COUNT;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+@RunWith(BMUnitRunner.class)
+public class GlobalTidyConcurrencyTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(GlobalTidyConcurrencyTest.class);
+
+    public static volatile boolean leakHappened = false;
+
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        TEST_DEBUG_REF_COUNT.setBoolean(true);
+        System.setProperty("org.jboss.byteman.verbose", "true"); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
+    }
+
+    /**
+     * This is a basic concurrency test for {@link SSTableReader.GlobalTidy}
+     *
+     * The scenario is to emulate basic get -> work -> release workflow for the same
+     * sstable in multiple threads.
+     *
+     * The test detects:
+     * - assertion failures in GlobalTidy code
+     * - resource leaks
+     * - GlobalTidy.lookup modification when the relevant Ref is alive
+     */
+    @Test
+    @BMRule(name = "Count leaks",
+    targetClass="Ref$State",
+    targetMethod="reportLeak",
+    targetLocation="AT EXIT",
+    action="org.apache.cassandra.io.sstable.format.GlobalTidyConcurrencyTest.leakHappened = true;")
+    public void tidyVsGetRaceTest() throws InterruptedException
+    {
+        int NUM_THREADS = 32;
+        Descriptor desc = createDescriptor();
+        AtomicBoolean exit = new AtomicBoolean(false);
+        CopyOnWriteArrayList<Object> errors = new CopyOnWriteArrayList<>();
+
+        class TestThread extends Thread
+        {
+            public TestThread(int idx)
+            {
+                super("test-thread-" + idx);
+            }
+
+            @Override
+            public void run()
+            {
+                try
+                {
+                    while (!exit.get())
+                    {
+                        Ref<SSTableReader.GlobalTidy> ref = SSTableReader.GlobalTidy.get(desc);
+                        LockSupport.parkNanos(ThreadLocalRandom.current().nextInt(1000000));
+                        Ref<SSTableReader.GlobalTidy> currentTidy = SSTableReader.GlobalTidy.lookup.get(desc);
+                        if (!currentTidy.refers(ref.get()))
+                        {
+                            String error = String.format("GlobalTidy to which we keep reference is different than the GlobalTidy associated with this descriptor in SSTableReader.GlobalTidy.lookup; local=%s, lookup=%s, descriptor=%s", ref.get(), currentTidy, desc);
+                            ref.release();
+                            throw new AssertionError(error);
+                        }
+                        ref.release();
+                    }
+                }
+                catch (Throwable e)
+                {
+                    logger.error("Stopping test due to error ", e);
+                    errors.add(e);
+                    exit.set(true);
+                }
+            }
+        }
+
+        Thread[] threads = new Thread[NUM_THREADS];
+
+        IntStream.range(0, NUM_THREADS)
+                 .forEach(idx -> threads[idx] = new TestThread(idx));
+
+        for (Thread thread : threads)
+        {
+            thread.start();
+        }
+
+        int NUM_TICKS = 10;
+        for (int tick = 0; tick < NUM_TICKS && !exit.get(); tick++)
+        {
+            Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+            logger.info("Tick {}...", tick);
+        }
+        exit.set(true);
+
+        for (Thread thread : threads)
+        {
+            thread.join();
+        }
+
+        if (!errors.isEmpty())
+        {
+            errors.forEach(error -> logger.error("Error: ", error));
+            fail("Unexpected errors in the test");
+        }
+
+        assertFalse("check the logs, LEAK happened", leakHappened);
+    }
+
+    private Descriptor createDescriptor()
+    {
+        return new Descriptor(BigFormat.getInstance().getLatestVersion().version,
+                              new File("whatever"),
+                              "keyspace",
+                              "table",
+                              new SequenceBasedSSTableId(1),
+                              BigFormat.getInstance());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/LazyBloomFilterTest.java b/test/unit/org/apache/cassandra/io/sstable/format/LazyBloomFilterTest.java
new file mode 100644
index 000000000000..7ef90d5399ad
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/LazyBloomFilterTest.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.metrics.RestorableMeter;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
+import org.awaitility.Awaitility;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+public class LazyBloomFilterTest
+{
+    private static final String KEYSPACE1 = "SSTableReaderTest";
+    private static final String CF_STANDARD = "Standard1";
+
+    private static ColumnFamilyStore store;
+
+    @BeforeClass
+    public static void defineSchema()
+    {
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING.setBoolean(true);
+
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD).bloomFilterFpChance(0.1)
+                                                .caching(new CachingParams(false, 0)));
+
+        store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
+    }
+
+    @AfterClass
+    public static void tearDown()
+    {
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_WINDOW.reset();
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_THRESHOLD.reset();
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING.reset();
+    }
+
+    @After
+    public void cleanup()
+    {
+        store.loadNewSSTables();
+        store.truncateBlocking();
+    }
+
+    @Test
+    public void testDeserializeOnFirstRead()
+    {
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_THRESHOLD.setInt(0);
+
+        SSTableReaderWithFilter sstable = reopenFlushedSSTable();
+
+        // first read will trigger bloom filter deserialization
+        assertTrue(sstable.mayContainAssumingKeyIsInRange(Util.dk(String.valueOf(10))));
+        waitFor("Async BF deserialization", () -> sstable.getFilter() != FilterFactory.AlwaysPresentForLazyLoading);
+
+        IFilter deserializedBloomFilter = sstable.getFilter();
+        assertThat(deserializedBloomFilter).isNotSameAs(FilterFactory.AlwaysPresent);
+        assertThat(deserializedBloomFilter).isInstanceOf(BloomFilter.class);
+        assertThat(deserializedBloomFilter.offHeapSize()).isGreaterThan(0);
+
+        // second read will NOT trigger bloom filter deserialization
+        assertTrue(sstable.mayContainAssumingKeyIsInRange(Util.dk(String.valueOf(20))));
+        assertSame(deserializedBloomFilter, sstable.getFilter());
+        assertThat(deserializedBloomFilter.offHeapSize()).isGreaterThan(0);
+
+        releaseSSTables(sstable);
+    }
+
+    @Test
+    public void testConcurrentReads() throws InterruptedException
+    {
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_THRESHOLD.setInt(0);
+
+        SSTableReaderWithFilter sstable = reopenFlushedSSTable();
+
+        int threads = 32;
+        ExecutorService executor = Executors.newFixedThreadPool(threads);
+        CountDownLatch latch = new CountDownLatch(1);
+
+        List<Future<Boolean>> futures = new ArrayList<>();
+        for (int i = 0; i < 100; i++)
+        {
+            futures.add(executor.submit(() -> {
+                Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES);
+                return sstable.maybeDeserializeLazyBloomFilter();
+            }));
+        }
+
+        latch.countDown();
+        waitFor("Async BF deserialization", () -> sstable.getFilter() != FilterFactory.AlwaysPresentForLazyLoading);
+
+        executor.shutdown();
+        Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
+
+        // only one thread can deserialize BF
+        assertThat(FBUtilities.waitOnFutures(futures).stream().filter(b -> b).count()).isEqualTo(1);
+
+        Awaitility.await("Wait for async BF deserialization")
+                  .atMost(10, TimeUnit.SECONDS)
+                  .untilAsserted(() -> {
+                      assertThat(sstable.getFilter()).isNotSameAs(FilterFactory.AlwaysPresent);
+                      assertThat(sstable.getFilter().offHeapSize()).isGreaterThan(0);
+                  });
+
+        releaseSSTables(sstable);
+    }
+
+    @Test
+    public void testLazyLoadingCountThreshold()
+    {
+        testLazyLoadingThreshold(-1, 10);
+    }
+
+    @Test
+    public void testLazyLoadingCountThresholdBadPartition()
+    {
+        testLazyLoadingThreshold(-1, 7);
+    }
+
+    @Test
+    public void testLazyLoading1MThreshold()
+    {
+        testLazyLoadingThreshold(1, 10);
+    }
+
+    @Test
+    public void testLazyLoading5MThreshold()
+    {
+        testLazyLoadingThreshold(5, 10);
+    }
+
+    @Test
+    public void testLazyLoading15MThreshold()
+    {
+        testLazyLoadingThreshold(15, 10);
+    }
+
+    public void testLazyLoadingThreshold(int window, int keyInt)
+    {
+        int threshold = 1;
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_WINDOW.setInt(window);
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_THRESHOLD.setInt(threshold);
+
+        SSTableReaderWithFilter sstable = reopenFlushedSSTable();
+        DecoratedKey key = Util.dk(String.valueOf(keyInt));
+
+        // first read will NOT trigger bloom filter deserialization because of threshold not reached
+        sstable.mayContainAssumingKeyIsInRange(key);
+        assertSame(FilterFactory.AlwaysPresentForLazyLoading, sstable.getFilter());
+        assertThat(sstable.getFilter().offHeapSize()).isEqualTo(0);
+
+        // make the sstable access the index
+        SinglePartitionReadCommand command = createCommand(key);
+
+        long start = System.nanoTime();
+        while (System.nanoTime() - start <= 3 * RestorableMeter.TICK_INTERVAL)
+        {
+            FBUtilities.sleepQuietly(10);
+            Util.getAllUnfiltered(command);
+
+            if (sstable.getFilter() != FilterFactory.AlwaysPresentForLazyLoading)
+                break;
+        }
+
+        assertThat(sstable.getPartitionIndexReadMeter().count()).isGreaterThan(0);
+        if (window > 0)
+            assertThat(sstable.getPartitionIndexReadMeter().rate(window)).isGreaterThan(threshold);
+
+        assertThat(sstable.getFilter()).isNotSameAs(FilterFactory.AlwaysPresent);
+        assertThat(sstable.getFilter().offHeapSize()).isGreaterThan(0);
+
+        releaseSSTables(sstable);
+    }
+
+    @Test
+    public void testDeserializationOnReleasedSSTable()
+    {
+        CassandraRelevantProperties.BLOOM_FILTER_LAZY_LOADING_THRESHOLD.setInt(0);
+
+        SSTableReaderWithFilter sstable = reopenFlushedSSTable();
+
+        // release sstable
+        store.getLiveSSTables().forEach(s -> s.selfRef().release()); // ColumnFamilyStore#clearUnsafe won't release sstable reference
+        store.clearUnsafe();
+        assertThat(sstable.selfRef().globalCount()).isEqualTo(0);
+
+        // it will try to deserialize but skip
+        assertThat(sstable.maybeDeserializeLazyBloomFilter()).isTrue();
+        Awaitility.await("Async deserialization skipped")
+                  .atMost(10, TimeUnit.SECONDS)
+                  .until(() -> sstable.filter == FilterFactory.AlwaysPresent);
+
+        assertThat(sstable.getFilter()).isSameAs(FilterFactory.AlwaysPresent);
+    }
+
+    private void releaseSSTables(SSTableReaderWithFilter sstable)
+    {
+        store.getLiveSSTables().forEach(s -> s.selfRef().release()); // ColumnFamilyStore#clearUnsafe won't release sstable reference
+        store.clearUnsafe();
+
+        // close sstable, bf should be closed when sstable tidier runs
+        waitFor("sstable tidier", ((BloomFilter) sstable.getFilter())::isCleanedUp);
+    }
+
+    private void waitFor(String alias, Callable<Boolean> condition)
+    {
+        Awaitility.await(alias)
+                .pollInterval(1, TimeUnit.SECONDS)
+                .atMost(30, TimeUnit.SECONDS)
+                .until(condition);
+    }
+
+    private SinglePartitionReadCommand createCommand(DecoratedKey key)
+    {
+        return SinglePartitionReadCommand.create(store.metadata(), FBUtilities.nowInSeconds(), key, Slices.ALL);
+    }
+
+    private SSTableReaderWithFilter reopenFlushedSSTable()
+    {
+        SSTableReaderWithFilter sstable = flushSSTable(store, 100, 10);
+
+        // sstable flush writer generates bloom filter and loads it
+        assertThat(sstable.getFilter()).isNotSameAs(FilterFactory.AlwaysPresent);
+        assertThat(sstable.getFilter().offHeapSize()).isGreaterThan(0);
+
+        // unlink sstables and reopen them
+        store.getLiveSSTables().forEach(s -> s.selfRef().release()); // ColumnFamilyStore#clearUnsafe won't release sstable reference
+        store.clearUnsafe();
+        store.loadNewSSTables();
+
+        // newly opened sstable delays bloom filter deserialization
+        SSTableReader newlyOpenedSSTable = Iterables.getOnlyElement(store.getLiveSSTables());
+        assertThat(newlyOpenedSSTable).isInstanceOf(SSTableReaderWithFilter.class);
+        sstable = (SSTableReaderWithFilter) newlyOpenedSSTable;
+        assertSame(FilterFactory.AlwaysPresentForLazyLoading, sstable.getFilter());
+        assertThat(sstable.getFilter().offHeapSize()).isEqualTo(0);
+
+        return sstable;
+    }
+
+    private SSTableReaderWithFilter flushSSTable(ColumnFamilyStore cfs, int numKeys, int step)
+    {
+        Set<SSTableReader> before = cfs.getLiveSSTables();
+        for (int j = 0; j < numKeys; j += step)
+        {
+            new RowUpdateBuilder(cfs.metadata(), j, String.valueOf(j))
+            .clustering("0")
+            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+            .build()
+            .applyUnsafe();
+        }
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+        SSTableReader result = Sets.difference(cfs.getLiveSSTables(), before).iterator().next();
+        assertThat(result).isInstanceOf(SSTableReaderWithFilter.class);
+        return (SSTableReaderWithFilter) result;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java b/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java
index a4b56f1765ef..3f6bf1e9e8dc 100644
--- a/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java
@@ -50,7 +50,7 @@
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
 import org.apache.cassandra.db.rows.BTreeRow;
 import org.apache.cassandra.db.rows.BufferCell;
@@ -58,13 +58,13 @@
 import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.rows.RangeTombstoneMarker;
 import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
 import org.apache.cassandra.db.rows.SerializationHelper;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.rows.UnfilteredSerializer;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.IndexInfo;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.DataInputBuffer;
@@ -107,14 +107,14 @@ public static void beforeClass()
     @Test
     public void testC11206AgainstPreviousArray() throws Exception
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(99999);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(99999);
         testC11206AgainstPrevious();
     }
 
     @Test
     public void testC11206AgainstPreviousShallow() throws Exception
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         testC11206AgainstPrevious();
     }
 
@@ -217,10 +217,11 @@ public void close() throws Exception
         void build(Row staticRow, DecoratedKey partitionKey,
                    Collection<Clustering<?>> clusterings, long startPosition) throws IOException
         {
-
             Iterator<Clustering<?>> clusteringIter = clusterings.iterator();
+            if (staticRow == null)
+                staticRow = Rows.EMPTY_STATIC_ROW;
             partitionWriter.start(partitionKey, DeletionTime.LIVE);
-            if (staticRow != null)
+            if (!staticRow.isEmpty())
                 partitionWriter.addStaticRow(staticRow);
             AbstractUnfilteredRowIterator rowIter = makeRowIter(staticRow, partitionKey, clusteringIter, dataWriterNew);
             while (rowIter.hasNext())
@@ -464,7 +465,7 @@ public void testSerializedSize() throws Throwable
         for (int i = 0; i <= DatabaseDescriptor.getColumnIndexSize(BigFormatPartitionWriter.DEFAULT_GRANULARITY) / 4; i++)
             execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, String.valueOf(i), i);
 
-        ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build());
+        Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build());
 
         File tempFile = FileUtils.createTempFile("row_index_entry_test", null);
         tempFile.deleteOnExit();
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/big/VersionSupportedFeaturesTest.java b/test/unit/org/apache/cassandra/io/sstable/format/big/VersionSupportedFeaturesTest.java
index b4102b8dbc49..255952b2e7d6 100644
--- a/test/unit/org/apache/cassandra/io/sstable/format/big/VersionSupportedFeaturesTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/big/VersionSupportedFeaturesTest.java
@@ -66,4 +66,64 @@ protected Stream<String> getOriginatingHostIdSupportedVersions()
     {
         return Stream.concat(range("me", "mz"), range("nb", "zz"));
     }
+
+    @Override
+    protected Stream<String> getAccurateMinMaxSupportedVersions()
+    {
+        return range("md", "nz");
+    }
+
+    @Override
+    protected Stream<String> getCommitLogLowerBoundSupportedVersions()
+    {
+        return range("mb", "zz");
+    }
+
+    @Override
+    protected Stream<String> getCommitLogIntervalsSupportedVersions()
+    {
+        return range("mc", "zz");
+    }
+
+    @Override
+    protected Stream<String> getZeroCopyMetadataSupportedVersions()
+    {
+        return Stream.empty();
+    }
+
+    @Override
+    protected Stream<String> getIncrementalNodeSyncMetadataSupportedVersions()
+    {
+        return Stream.empty();
+    }
+
+    @Override
+    protected Stream<String> getMaxColumnValueLengthsSupportedVersions()
+    {
+        return Stream.empty();
+    }
+
+    @Override
+    protected Stream<String> getIsTransientSupportedVersions()
+    {
+        return range("na", "zz");
+    }
+
+    @Override
+    protected Stream<String> getMisplacedPartitionLevelDeletionsPresenceMarkerSupportedVersions()
+    {
+        return Stream.empty();
+    }
+
+    @Override
+    protected Stream<String> getTokenSpaceCoverageSupportedVersions()
+    {
+        return range("oa", "zz");
+    }
+
+    @Override
+    protected Stream<String> getOldBfFormatSupportedVersions()
+    {
+        return range("aa","mz");
+    }
 }
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/bti/LoadingBuilderTest.java b/test/unit/org/apache/cassandra/io/sstable/format/bti/LoadingBuilderTest.java
index a56ca8e53723..7dbf3b9aa84c 100644
--- a/test/unit/org/apache/cassandra/io/sstable/format/bti/LoadingBuilderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/bti/LoadingBuilderTest.java
@@ -46,7 +46,7 @@ public class LoadingBuilderTest extends CQLTester
     @Test
     @BMRule(name = "Save preload argument of PartitionIndex initialization",
             targetClass = "org.apache.cassandra.io.sstable.format.bti.PartitionIndex",
-            targetMethod = "load(org.apache.cassandra.io.util.FileHandle, org.apache.cassandra.dht.IPartitioner, boolean)",
+            targetMethod = "load(org.apache.cassandra.io.util.FileHandle, org.apache.cassandra.dht.IPartitioner, boolean, org.apache.cassandra.io.sstable.metadata.ZeroCopyMetadata, org.apache.cassandra.utils.bytecomparable.ByteComparable$Version)",
             action = "org.apache.cassandra.io.sstable.format.bti.LoadingBuilderTest.preloadsMap.put($1.path(), $3)")
     public void testPreloadFlag()
     {
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java b/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
index 6ceee331f6e8..db1e0c66e7f5 100644
--- a/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
@@ -54,7 +54,6 @@
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.io.tries.TrieNode;
-import org.apache.cassandra.io.tries.Walker;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.io.util.FileUtils;
@@ -68,6 +67,9 @@
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
+import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.LEGACY;
+import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.OSS41;
+import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.OSS50;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertSame;
@@ -81,8 +83,6 @@ public class PartitionIndexTest
     private final static long SEED = System.nanoTime();
     private final static Random random = new Random(SEED);
 
-    static final ByteComparable.Version VERSION = Walker.BYTE_COMPARABLE_VERSION;
-
     static
     {
         DatabaseDescriptor.daemonInitialization();
@@ -95,13 +95,20 @@ public class PartitionIndexTest
     @Parameterized.Parameters()
     public static Collection<Object[]> generateData()
     {
-        return Arrays.asList(new Object[]{ Config.DiskAccessMode.standard },
-                             new Object[]{ Config.DiskAccessMode.mmap });
+        return Arrays.asList(new Object[]{ Config.DiskAccessMode.standard, OSS50 },
+                             new Object[]{ Config.DiskAccessMode.mmap, OSS50 },
+                             new Object[]{ Config.DiskAccessMode.standard, OSS41 },
+                             new Object[]{ Config.DiskAccessMode.mmap, OSS41 },
+                             new Object[]{ Config.DiskAccessMode.standard, LEGACY },
+                             new Object[]{ Config.DiskAccessMode.mmap, LEGACY });
     }
 
     @Parameterized.Parameter(value = 0)
     public static Config.DiskAccessMode accessMode = Config.DiskAccessMode.standard;
 
+    @Parameterized.Parameter(value = 1)
+    public static ByteComparable.Version version;
+
     @BeforeClass
     public static void beforeClass()
     {
@@ -296,7 +303,7 @@ public void testAddEmptyKey() throws Exception
 
         FileHandle.Builder fhBuilder = makeHandle(file);
         try (SequentialWriter writer = makeWriter(file);
-             PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+             PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder, version)
         )
         {
             DecoratedKey key = p.decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER);
@@ -401,13 +408,13 @@ public void testConstrainedIteration() throws IOException
                 catch (AssertionError e)
                 {
                     StringBuilder buf = new StringBuilder();
-                    buf.append(String.format("Left %s%s Right %s%s%n", left.byteComparableAsString(VERSION), exactLeft ? "#" : "", right.byteComparableAsString(VERSION), exactRight ? "#" : ""));
+                    buf.append(String.format("Left %s%s Right %s%s%n", left.byteComparableAsString(version), exactLeft ? "#" : "", right.byteComparableAsString(version), exactRight ? "#" : ""));
                     try (PartitionIndex.IndexPosIterator iter2 = new PartitionIndex.IndexPosIterator(summary, left, right))
                     {
                         long pos;
                         while ((pos = iter2.nextIndexPos()) != PartitionIndex.NOT_FOUND)
-                            buf.append(keys.get((int) pos).byteComparableAsString(VERSION)).append("\n");
-                        buf.append(String.format("Left %s%s Right %s%s%n", left.byteComparableAsString(VERSION), exactLeft ? "#" : "", right.byteComparableAsString(VERSION), exactRight ? "#" : ""));
+                            buf.append(keys.get((int) pos).byteComparableAsString(version)).append("\n");
+                        buf.append(String.format("Left %s%s Right %s%s%n", left.byteComparableAsString(version), exactLeft ? "#" : "", right.byteComparableAsString(version), exactRight ? "#" : ""));
                     }
                     logger.error(buf.toString(), e);
                     throw e;
@@ -426,7 +433,7 @@ public void testPartialIndex() throws IOException
             int parts = 15;
             FileHandle.Builder fhBuilder = makeHandle(file);
             try (SequentialWriter writer = makeWriter(file);
-                 PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+                 PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder, version)
             )
             {
                 writer.setPostFlushListener(builder::markPartitionIndexSynced);
@@ -500,7 +507,7 @@ public void testDeepRecursion()
                 FileHandle.Builder fhBuilder = new FileHandle.Builder(file)
                                                .bufferSize(PageAware.PAGE_SIZE)
                                                .withChunkCache(ChunkCache.instance);
-                try (PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder))
+                try (PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder, version))
                 {
                     int i = 0;
                     for (i = 0; i < 3; ++i)
@@ -523,7 +530,7 @@ public void testDeepRecursion()
                         builder.addEntry(list.get(i), i);
                     builder.complete();
 
-                    try (PartitionIndex index = PartitionIndex.load(fhBuilder, partitioner, true))
+                    try (PartitionIndex index = PartitionIndex.load(fhBuilder, partitioner, true, version))
                     {
                         checkIteration(list.size(), index);
                     }
@@ -625,7 +632,7 @@ public class PartitionIndexJumping extends PartitionIndex
         public PartitionIndexJumping(FileHandle fh, long trieRoot, long keyCount, DecoratedKey first, DecoratedKey last,
                                      long... cutoffsAndOffsets)
         {
-            super(fh, trieRoot, keyCount, first, last);
+            super(fh, trieRoot, keyCount, first, last, PartitionIndexTest.version);
             this.cutoffsAndOffsets = cutoffsAndOffsets;
         }
 
@@ -666,7 +673,7 @@ public void testPointerGrowth() throws IOException
             List<DecoratedKey> list = Lists.newArrayList();
             FileHandle.Builder fhBuilder = makeHandle(file);
             try (SequentialWriter writer = makeJumpingWriter(file, cutoffsAndOffsets);
-                 PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+                 PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder, version)
             )
             {
                 writer.setPostFlushListener(builder::markPartitionIndexSynced);
@@ -705,7 +712,7 @@ public void testDumpTrieToFile() throws IOException
         ArrayList<DecoratedKey> list = Lists.newArrayList();
         FileHandle.Builder fhBuilder = makeHandle(file);
         try (SequentialWriter writer = new SequentialWriter(file, SequentialWriterOption.DEFAULT);
-             PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+             PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder, version)
         )
         {
             writer.setPostFlushListener(builder::markPartitionIndexSynced);
@@ -721,7 +728,7 @@ public void testDumpTrieToFile() throws IOException
             long root = builder.complete();
 
             try (FileHandle fh = fhBuilder.complete();
-                 PartitionIndex index = new PartitionIndex(fh, root, 1000, null, null))
+                 PartitionIndex index = new PartitionIndex(fh, root, 1000, null, null, version))
             {
                 File dump = FileUtils.createTempFile("testDumpTrieToFile", "dumpedTrie");
                 index.dumpTrie(dump.toString());
@@ -738,7 +745,7 @@ public static class Analyzer extends PartitionIndex.Reader
 
         public Analyzer(PartitionIndex index)
         {
-            super(index);
+            super(index, index.version);
         }
 
         public void run()
@@ -794,7 +801,7 @@ Pair<List<DecoratedKey>, PartitionIndex> generateIndex(int size, Supplier<Decora
         List<DecoratedKey> list = Lists.newArrayList();
         FileHandle.Builder fhBuilder = makeHandle(file);
         try (SequentialWriter writer = makeWriter(file);
-             PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+             PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder, version)
         )
         {
             for (int i = 0; i < size; i++)
@@ -883,6 +890,6 @@ protected FileHandle.Builder makeHandle(File file)
 
     protected PartitionIndex loadPartitionIndex(FileHandle.Builder fhBuilder, SequentialWriter writer) throws IOException
     {
-        return PartitionIndex.load(fhBuilder, partitioner, false);
+        return PartitionIndex.load(fhBuilder, partitioner, false, version);
     }
 }
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/bti/RowIndexTest.java b/test/unit/org/apache/cassandra/io/sstable/format/bti/RowIndexTest.java
index b477bec4db14..ece1a0a7c0a2 100644
--- a/test/unit/org/apache/cassandra/io/sstable/format/bti/RowIndexTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/bti/RowIndexTest.java
@@ -47,7 +47,6 @@
 import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
-import org.apache.cassandra.io.tries.Walker;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileHandle;
@@ -65,9 +64,11 @@
 public class RowIndexTest
 {
     private final static Logger logger = LoggerFactory.getLogger(RowIndexTest.class);
-    private final Version version = new BtiFormat(null).getLatestVersion();
+    private static final Version latestVersion = new BtiFormat(null).getLatestVersion();
+    private static final Version version_aa = new BtiFormat(null).getVersion("aa");
+    private static final Version version_ca = new BtiFormat(null).getVersion("ca");
+    private static final Version version_da = new BtiFormat(null).getVersion("da");
 
-    static final ByteComparable.Version VERSION = Walker.BYTE_COMPARABLE_VERSION;
 
     static final Random RANDOM;
 
@@ -87,13 +88,23 @@ public class RowIndexTest
     @Parameterized.Parameters()
     public static Collection<Object[]> generateData()
     {
-        return Arrays.asList(new Object[]{ Config.DiskAccessMode.standard },
-                             new Object[]{ Config.DiskAccessMode.mmap });
+        return Arrays.asList(new Object[]{ Config.DiskAccessMode.standard, latestVersion },
+                             new Object[]{ Config.DiskAccessMode.mmap, latestVersion },
+                             new Object[]{ Config.DiskAccessMode.standard, version_aa },
+                             new Object[]{ Config.DiskAccessMode.mmap, version_aa },
+                             new Object[]{ Config.DiskAccessMode.standard, version_ca },
+                             new Object[]{ Config.DiskAccessMode.mmap, version_ca },
+                             new Object[]{ Config.DiskAccessMode.standard, version_da },
+                             new Object[]{ Config.DiskAccessMode.mmap, version_da }
+        );
     }
 
     @Parameterized.Parameter(value = 0)
     public static Config.DiskAccessMode accessMode = Config.DiskAccessMode.standard;
 
+    @Parameterized.Parameter(value = 1)
+    public static Version VERSION;
+
     @Test
     public void testSingletons() throws IOException
     {
@@ -184,7 +195,7 @@ public RowIndexTest() throws IOException
         dos.writeUTF("JUNK");
         dos.writeUTF("JUNK");
 
-        writer = new RowIndexWriter(comparator, dos, version);
+        writer = new RowIndexWriter(comparator, dos, VERSION);
     }
 
     public void complete() throws IOException
@@ -207,7 +218,7 @@ public RowIndexReader completeAndRead() throws IOException
             assertEquals("JUNK", rdr.readUTF());
             assertEquals("JUNK", rdr.readUTF());
         }
-        return new RowIndexReader(fh, root, version);
+        return new RowIndexReader(fh, root, VERSION);
     }
 
     @Test
@@ -327,7 +338,7 @@ public void testConstrainedIteration() throws IOException
                 exactRight = b;
             }
 
-            try (RowIndexReverseIterator iter = new RowIndexReverseIterator(fh, root, comparator.asByteComparable(left), comparator.asByteComparable(right), random.right.version))
+            try (RowIndexReverseIterator iter = new RowIndexReverseIterator(fh, root, comparator.asByteComparable(left), comparator.asByteComparable(right), VERSION))
             {
                 IndexInfo indexInfo = iter.nextIndexInfo();
                 if (indexInfo == null)
@@ -376,10 +387,10 @@ public void testConstrainedIteration() throws IOException
                 logger.info(keys.stream()
                                 .filter(x -> comparator.compare(ll, x) <= 0 && comparator.compare(x, rr) <= 0)
                                 .map(clustering -> comparator.asByteComparable(clustering))
-                                .map(bc -> bc.byteComparableAsString(VERSION))
+                                .map(bc -> bc.byteComparableAsString(VERSION.getByteComparableVersion()))
                                 .collect(Collectors.joining(", ")));
                 logger.info("Left {}{} Right {}{}", comparator.asByteComparable(left), exactLeft ? "#" : "", comparator.asByteComparable(right), exactRight ? "#" : "");
-                try (RowIndexReverseIterator iter2 = new RowIndexReverseIterator(fh, root, comparator.asByteComparable(left), comparator.asByteComparable(right), version))
+                try (RowIndexReverseIterator iter2 = new RowIndexReverseIterator(fh, root, comparator.asByteComparable(left), comparator.asByteComparable(right), VERSION))
                 {
                     IndexInfo ii;
                     while ((ii = iter2.nextIndexInfo()) != null)
@@ -405,7 +416,7 @@ public void testReverseIteration() throws IOException
             ClusteringPrefix<?> right = exactRight ? keys.get(RANDOM.nextInt(keys.size())) : generateRandomKey();
 
             int idx = 0;
-            try (RowIndexReverseIterator iter = new RowIndexReverseIterator(fh, root, ByteComparable.EMPTY, comparator.asByteComparable(right), random.right.version))
+            try (RowIndexReverseIterator iter = new RowIndexReverseIterator(fh, root, ByteComparable.EMPTY, comparator.asByteComparable(right), VERSION))
             {
                 IndexInfo indexInfo = iter.nextIndexInfo();
                 if (indexInfo == null)
@@ -445,10 +456,10 @@ public void testReverseIteration() throws IOException
                 logger.info(keys.stream()
                                 .filter(x -> comparator.compare(x, rr) <= 0)
                                 .map(comparator::asByteComparable)
-                                .map(bc -> bc.byteComparableAsString(VERSION))
+                                .map(bc -> bc.byteComparableAsString(VERSION.getByteComparableVersion()))
                                 .collect(Collectors.joining(", ")));
                 logger.info("Right {}{}", comparator.asByteComparable(right), exactRight ? "#" : "");
-                try (RowIndexReverseIterator iter2 = new RowIndexReverseIterator(fh, root, ByteComparable.EMPTY, comparator.asByteComparable(right), version))
+                try (RowIndexReverseIterator iter2 = new RowIndexReverseIterator(fh, root, ByteComparable.EMPTY, comparator.asByteComparable(right), VERSION))
                 {
                     IndexInfo ii;
                     while ((ii = iter2.nextIndexInfo()) != null)
@@ -468,7 +479,7 @@ private Pair<List<ClusteringPrefix<?>>, RowIndexReader> generateRandomIndexSingl
         for (int i = 0; i < size; i++)
         {
             assert i == 0 || comparator.compare(list.get(i - 1), list.get(i)) < 0;
-            assert i == 0 || ByteComparable.compare(comparator.asByteComparable(list.get(i - 1)), comparator.asByteComparable(list.get(i)), VERSION) < 0 :
+            assert i == 0 || ByteComparable.compare(comparator.asByteComparable(list.get(i - 1)), comparator.asByteComparable(list.get(i)), VERSION.getByteComparableVersion()) < 0 :
             String.format("%s bs %s versus %s bs %s", list.get(i - 1).clustering().clusteringString(comparator.subtypes()), comparator.asByteComparable(list.get(i - 1)), list.get(i).clustering().clusteringString(comparator.subtypes()), comparator.asByteComparable(list.get(i)));
             writer.add(list.get(i), list.get(i), new IndexInfo(i, DeletionTime.LIVE));
         }
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/bti/VersionSupportedFeaturesTest.java b/test/unit/org/apache/cassandra/io/sstable/format/bti/VersionSupportedFeaturesTest.java
index 3f408c9d90dc..172937c5c245 100644
--- a/test/unit/org/apache/cassandra/io/sstable/format/bti/VersionSupportedFeaturesTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/bti/VersionSupportedFeaturesTest.java
@@ -20,6 +20,8 @@
 
 import java.util.stream.Stream;
 
+import com.google.common.collect.Streams;
+
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.sstable.format.AbstractTestVersionSupportedFeatures;
 import org.apache.cassandra.io.sstable.format.Version;
@@ -41,30 +43,90 @@ protected Stream<String> getPendingRepairSupportedVersions()
     @Override
     protected Stream<String> getPartitionLevelDeletionPresenceMarkerSupportedVersions()
     {
-        return ALL_VERSIONS.stream();
+        return range("da", "zz");
     }
 
     @Override
     protected Stream<String> getLegacyMinMaxSupportedVersions()
     {
-        return Stream.empty();
+        return range("aa", "az");
     }
 
     @Override
     protected Stream<String> getImprovedMinMaxSupportedVersions()
     {
-        return ALL_VERSIONS.stream();
+        return range("ba", "zz");
     }
 
     @Override
     protected Stream<String> getKeyRangeSupportedVersions()
     {
-        return ALL_VERSIONS.stream();
+        return range("da", "zz");
     }
 
     @Override
     protected Stream<String> getOriginatingHostIdSupportedVersions()
+    {
+        return Streams.concat(range("ad", "az"), range("bb", "zz"));
+    }
+
+    @Override
+    protected Stream<String> getAccurateMinMaxSupportedVersions()
+    {
+        return range("ac", "az");
+    }
+
+    @Override
+    protected Stream<String> getCommitLogLowerBoundSupportedVersions()
     {
         return ALL_VERSIONS.stream();
     }
+
+    @Override
+    protected Stream<String> getCommitLogIntervalsSupportedVersions()
+    {
+        return ALL_VERSIONS.stream();
+    }
+
+    @Override
+    protected Stream<String> getZeroCopyMetadataSupportedVersions()
+    {
+        return range("ba", "bz");
+    }
+
+    @Override
+    protected Stream<String> getIncrementalNodeSyncMetadataSupportedVersions()
+    {
+        return range("ba", "bz");
+    }
+
+    @Override
+    protected Stream<String> getMaxColumnValueLengthsSupportedVersions()
+    {
+        return range("ba", "bz");
+    }
+
+    @Override
+    protected Stream<String> getIsTransientSupportedVersions()
+    {
+        return range("ca", "zz");
+    }
+
+    @Override
+    protected Stream<String> getMisplacedPartitionLevelDeletionsPresenceMarkerSupportedVersions()
+    {
+        return range("ba", "cz");
+    }
+
+    @Override
+    protected Stream<String> getTokenSpaceCoverageSupportedVersions()
+    {
+        return range("cb", "zz");
+    }
+
+    @Override
+    protected Stream<String> getOldBfFormatSupportedVersions()
+    {
+        return range("aa", "az");
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java b/test/unit/org/apache/cassandra/io/sstable/format/columniterator/SSTableReverseIteratorTest.java
similarity index 98%
rename from test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java
rename to test/unit/org/apache/cassandra/io/sstable/format/columniterator/SSTableReverseIteratorTest.java
index 169318d4e834..1e7c0f1e2c8c 100644
--- a/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/format/columniterator/SSTableReverseIteratorTest.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.db.columniterator;
+package org.apache.cassandra.io.sstable.format.columniterator;
 
 import java.nio.ByteBuffer;
 import java.util.Random;
diff --git a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java
index 40d04a48401b..5b77123bb68c 100644
--- a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.io.sstable.indexsummary;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -51,10 +52,11 @@
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionInterruptedException;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
@@ -127,9 +129,9 @@ public void beforeTest()
     @After
     public void afterTest()
     {
-        for (CompactionInfo.Holder holder : CompactionManager.instance.active.getCompactions())
+        for (TableOperation operation : CompactionManager.instance.active.getTableOperations())
         {
-            holder.stop();
+            operation.stop(TableOperation.StopTrigger.UNIT_TESTS);
         }
 
         String ksname = KEYSPACE1;
@@ -623,7 +625,10 @@ public void testCancelIndex() throws Exception
     @Test
     public void testCancelIndexInterrupt() throws Exception
     {
-        testCancelIndexHelper((cfs) -> CompactionManager.instance.interruptCompactionFor(Collections.singleton(cfs.metadata()), (sstable) -> true, false));
+        testCancelIndexHelper(cfs -> CompactionManager.instance.interruptCompactionFor(Collections.singleton(cfs.metadata()),
+                                                                                       sstable -> true,
+                                                                                       false,
+                                                                                       TableOperation.StopTrigger.UNIT_TESTS));
     }
 
     public void testCancelIndexHelper(Consumer<ColumnFamilyStore> cancelFunction) throws Exception
@@ -650,11 +655,11 @@ public void testCancelIndexHelper(Consumer<ColumnFamilyStore> cancelFunction) th
         final AtomicReference<CompactionInterruptedException> exception = new AtomicReference<>();
         // barrier to control when redistribution runs
         final CountDownLatch barrier = new CountDownLatch(1);
-        CompactionInfo.Holder ongoingCompaction = new CompactionInfo.Holder()
+        AbstractTableOperation ongoingCompaction = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, nextTimeUUID(), compacting);
+                return new OperationProgress(cfs.metadata(), OperationType.UNKNOWN, 0, 0, nextTimeUUID(), compacting);
             }
 
             public boolean isGlobal()
@@ -662,10 +667,9 @@ public boolean isGlobal()
                 return false;
             }
         };
-        try (LifecycleTransaction ignored = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN))
+        try (LifecycleTransaction ignored = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
+             Closeable c = CompactionManager.instance.active.onOperationStart(ongoingCompaction))
         {
-            CompactionManager.instance.active.beginCompaction(ongoingCompaction);
-
             Thread t = NamedThreadFactory.createAnonymousThread(new Runnable()
             {
                 public void run()
@@ -702,13 +706,9 @@ public void run()
             barrier.countDown();
             t.join();
         }
-        finally
-        {
-            CompactionManager.instance.active.finishCompaction(ongoingCompaction);
-        }
 
         assertNotNull("Expected compaction interrupted exception", exception.get());
-        assertTrue("Expected no active compactions", CompactionManager.instance.active.getCompactions().isEmpty());
+        assertTrue("Expected no active compactions", CompactionManager.instance.active.getTableOperations().isEmpty());
 
         Set<R> beforeRedistributionSSTables = new HashSet<>(allSSTables);
         Set<R> afterCancelSSTables = Sets.newHashSet(ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs));
diff --git a/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java b/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java
index 15c37ea60ab5..ee23ad9c8175 100644
--- a/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java
@@ -124,14 +124,14 @@ public static void cleanup()
     @Test
     public void testKeyCacheLoadShallowIndexEntry() throws Exception
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         testKeyCacheLoad(COLUMN_FAMILY2);
     }
 
     @Test
     public void testKeyCacheLoadIndexInfoOnHeap() throws Exception
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         testKeyCacheLoad(COLUMN_FAMILY5);
     }
 
@@ -232,14 +232,14 @@ private static SSTableReader readerForKey(KeyCacheKey k)
     @Test
     public void testKeyCacheLoadWithLostTableShallowIndexEntry() throws Exception
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         testKeyCacheLoadWithLostTable(COLUMN_FAMILY3);
     }
 
     @Test
     public void testKeyCacheLoadWithLostTableIndexInfoOnHeap() throws Exception
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         testKeyCacheLoadWithLostTable(COLUMN_FAMILY6);
     }
 
@@ -295,14 +295,14 @@ private void testKeyCacheLoadWithLostTable(String cf) throws Exception
     @Test
     public void testKeyCacheShallowIndexEntry() throws ExecutionException, InterruptedException
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(0);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(0);
         testKeyCache(COLUMN_FAMILY1);
     }
 
     @Test
     public void testKeyCacheIndexInfoOnHeap() throws ExecutionException, InterruptedException
     {
-        DatabaseDescriptor.setColumnIndexCacheSize(8);
+        DatabaseDescriptor.setColumnIndexCacheSizeInKiB(8);
         testKeyCache(COLUMN_FAMILY4);
     }
 
@@ -415,7 +415,8 @@ public void testKeyCacheLoadCacheLoadTimeExceedingLimit() throws Exception
         CacheService.KeyCacheSerializer keyCacheSerializerSpy = Mockito.spy(keyCacheSerializer);
         AutoSavingCache autoSavingCache = new AutoSavingCache(mock(ICache.class),
                                                               CacheService.CacheType.KEY_CACHE,
-                                                              keyCacheSerializerSpy);
+                                                              keyCacheSerializerSpy,
+                                                              null);
 
         doAnswer(new AnswersWithDelay(delayMillis, InvocationOnMock::callRealMethod)).when(keyCacheSerializerSpy)
                                                                                      .deserialize(any(DataInputPlus.class));
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index 4bb7478587b5..4b05d23626ce 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -87,8 +87,7 @@ public void testSerialization() throws IOException
             for (MetadataType type : MetadataType.values())
             {
                 if ((type != MetadataType.STATS) || latestVersion.hasImprovedMinMax())
-                    assertEquals(originalMetadata.get(type), deserialized.get(type));
-
+                    assertEquals(type.name(), originalMetadata.get(type), deserialized.get(type));
             }
         }
     }
@@ -145,15 +144,16 @@ public Map<MetadataType, MetadataComponent> constructMetadata(boolean withNulls)
                                          .build();
         MetadataCollector collector = new MetadataCollector(cfm.comparator)
                                       .commitLogIntervals(new IntervalSet<>(cllb, club));
-        if (DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion().hasTokenSpaceCoverage())
+        Version version = DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion();
+        if (version.hasTokenSpaceCoverage())
             collector.tokenSpaceCoverage(0.7);
 
         String partitioner = RandomPartitioner.class.getCanonicalName();
         double bfFpChance = 0.1;
         collector.updateClusteringValues(Clustering.make(UTF8Type.instance.decompose("abc"), Int32Type.instance.decompose(123)));
         collector.updateClusteringValues(Clustering.make(UTF8Type.instance.decompose("cba"), withNulls ? null : Int32Type.instance.decompose(234)));
-        ByteBuffer first = AsciiType.instance.decompose("a");
-        ByteBuffer last = AsciiType.instance.decompose("b");
+        ByteBuffer first = version.hasKeyRange() ? AsciiType.instance.decompose("a") : null;
+        ByteBuffer last = version.hasKeyRange() ? AsciiType.instance.decompose("b") : null;
         return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList()), first, last);
     }
 
diff --git a/test/unit/org/apache/cassandra/io/tries/AbstractTrieTestBase.java b/test/unit/org/apache/cassandra/io/tries/AbstractTrieTestBase.java
index 483b46d2e677..16e96f374753 100644
--- a/test/unit/org/apache/cassandra/io/tries/AbstractTrieTestBase.java
+++ b/test/unit/org/apache/cassandra/io/tries/AbstractTrieTestBase.java
@@ -37,20 +37,28 @@
 import org.apache.cassandra.io.util.PageAware;
 import org.apache.cassandra.io.util.Rebufferer;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.LEGACY;
+import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.OSS41;
+import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.OSS50;
 
 @RunWith(Parameterized.class)
 abstract public class AbstractTrieTestBase
 {
     @Parameterized.Parameter(0)
-    public TestClass writerClass;
+    public static TestClass writerClass;
+
+    @Parameterized.Parameter(1)
+    public static ByteComparable.Version version;
 
     enum TestClass
     {
-        SIMPLE(IncrementalTrieWriterSimple::new),
-        PAGE_AWARE(IncrementalTrieWriterPageAware::new),
-        PAGE_AWARE_DEEP_ON_STACK((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 256)),
-        PAGE_AWARE_DEEP_ON_HEAP((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 0)),
-        PAGE_AWARE_DEEP_MIXED((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 2));
+        SIMPLE((trieSerializer, dest) -> new IncrementalTrieWriterSimple(trieSerializer, dest, version)),
+        PAGE_AWARE((trieSerializer, dest) -> new IncrementalTrieWriterPageAware(trieSerializer, dest, version)),
+        PAGE_AWARE_DEEP_ON_STACK((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 256, version)),
+        PAGE_AWARE_DEEP_ON_HEAP((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 0, version)),
+        PAGE_AWARE_DEEP_MIXED((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 2, version));
 
         final BiFunction<TrieSerializer<Integer, DataOutputPlus>, DataOutputPlus, IncrementalTrieWriter<Integer>> constructor;
         TestClass(BiFunction<TrieSerializer<Integer, DataOutputPlus>, DataOutputPlus, IncrementalTrieWriter<Integer>> constructor)
@@ -59,14 +67,24 @@ enum TestClass
         }
     }
 
-    @Parameterized.Parameters(name = "{index}: trie writer class={0}")
+    @Parameterized.Parameters(name = "{index}: trie writer class={0}, encoding={1}")
     public static Collection<Object[]> data()
     {
-        return Arrays.asList(new Object[]{ TestClass.SIMPLE },
-                             new Object[]{ TestClass.PAGE_AWARE },
-                             new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_STACK },
-                             new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_HEAP },
-                             new Object[]{ TestClass.PAGE_AWARE_DEEP_MIXED });
+        return Arrays.asList(new Object[]{ TestClass.SIMPLE, LEGACY },
+                             new Object[]{ TestClass.PAGE_AWARE, LEGACY },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_STACK, LEGACY },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_HEAP, LEGACY },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_MIXED, LEGACY },
+                             new Object[]{ TestClass.SIMPLE, OSS41 },
+                             new Object[]{ TestClass.PAGE_AWARE, OSS41 },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_STACK, OSS41 },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_HEAP, OSS41 },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_MIXED, OSS41 },
+                             new Object[]{ TestClass.SIMPLE, OSS50 },
+                             new Object[]{ TestClass.PAGE_AWARE, OSS50 },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_STACK, OSS50 },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_HEAP, OSS50 },
+                             new Object[]{ TestClass.PAGE_AWARE_DEEP_MIXED, OSS50 });
     }
 
     protected final static Logger logger = LoggerFactory.getLogger(TrieBuilderTest.class);
@@ -122,7 +140,18 @@ protected ByteComparable source(String s)
         for (int i = 0; i < s.length(); ++i)
             buf.put((byte) s.charAt(i));
         buf.rewind();
-        return ByteComparable.fixedLength(buf);
+        return ByteComparable.preencoded(version, buf);
+    }
+
+    protected String decodeSource(ByteComparable source)
+    {
+        if (source == null)
+            return null;
+        StringBuilder sb = new StringBuilder();
+        ByteSource.Peekable stream = ByteSource.peekable(source.asComparableBytes(version));
+        for (int b = stream.next(); b != ByteSource.END_OF_STREAM; b = stream.next())
+            sb.append((char) b);
+        return sb.toString();
     }
 
     protected String toBase(long v)
diff --git a/test/unit/org/apache/cassandra/io/tries/TrieBuilderTest.java b/test/unit/org/apache/cassandra/io/tries/TrieBuilderTest.java
index e484033d6bb1..fba1c7a8d80f 100644
--- a/test/unit/org/apache/cassandra/io/tries/TrieBuilderTest.java
+++ b/test/unit/org/apache/cassandra/io/tries/TrieBuilderTest.java
@@ -33,7 +33,7 @@ public class TrieBuilderTest extends AbstractTrieTestBase
     public void testPartialBuildRecalculationBug() throws IOException
     {
         DataOutputBuffer buf = new DataOutputBufferPaged();
-        IncrementalTrieWriter<Integer> builder = newTrieWriter(serializer, buf);
+        IncrementalTrieWriter<Integer> builder = IncrementalTrieWriter.open(serializer, buf, version);
         long count = 0;
 
         count += addUntilBytesWritten(buf, builder, "a", 1);            // Make a node whose children are written
@@ -67,7 +67,7 @@ public void testPartialBuildRecalculationBug() throws IOException
 
     public void verifyContent(long count, Rebufferer source, long root, long... resets)
     {
-        ValueIterator<?> iter = new ValueIterator<>(source, root);
+        ValueIterator<?> iter = new ValueIterator<>(source, root, version);
         long found = 0;
         long ofs = 0;
         int rpos = 0;
diff --git a/test/unit/org/apache/cassandra/io/tries/WalkerTest.java b/test/unit/org/apache/cassandra/io/tries/WalkerTest.java
index 033ab5a5affe..1969e7123b98 100644
--- a/test/unit/org/apache/cassandra/io/tries/WalkerTest.java
+++ b/test/unit/org/apache/cassandra/io/tries/WalkerTest.java
@@ -22,11 +22,15 @@
 import java.io.PrintStream;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
 import java.util.function.LongSupplier;
 import java.util.function.LongToIntFunction;
+import java.util.function.Supplier;
 import java.util.stream.IntStream;
 
 import org.apache.commons.lang3.StringUtils;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.agrona.collections.IntArrayList;
@@ -36,6 +40,7 @@
 import org.apache.cassandra.io.util.Rebufferer;
 import org.apache.cassandra.io.util.TailOverridingRebufferer;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
@@ -46,6 +51,26 @@
 @SuppressWarnings({"unchecked", "RedundantSuppression"})
 public class WalkerTest extends AbstractTrieTestBase
 {
+    private static final Map<String, Integer> memoryTrieEntryMap = new TreeMap<>();
+
+    @BeforeClass
+    public static void setup()
+    {
+        // Initialize the memory representation of the trie
+        memoryTrieEntryMap.put("115", 1);
+        memoryTrieEntryMap.put("151", 2);
+        memoryTrieEntryMap.put("155", 3);
+        memoryTrieEntryMap.put("511", 4);
+        memoryTrieEntryMap.put("515", 5);
+        memoryTrieEntryMap.put("551", 6);
+        memoryTrieEntryMap.put("555555555555555555555555555555555555555555555555555555555555555555", 7);
+        memoryTrieEntryMap.put("70", 8);
+        memoryTrieEntryMap.put("7051", 9);
+        memoryTrieEntryMap.put("717", 10);
+        memoryTrieEntryMap.put("73", 11);
+        memoryTrieEntryMap.put("737", 12);
+    }
+
     @Test
     public void testWalker() throws IOException
     {
@@ -55,7 +80,7 @@ public void testWalker() throws IOException
 
         Rebufferer source = new ByteBufRebufferer(buf.asNewBuffer());
 
-        Walker<?> it = new Walker<>(source, rootPos);
+        Walker<?> it = new Walker<>(source, rootPos, version);
 
         DataOutputBuffer dumpBuf = new DataOutputBuffer();
         Version sstableVersion = new BtiFormat(null).getLatestVersion();
@@ -115,28 +140,65 @@ public void testIteratorWithoutBounds() throws IOException
         int[] expected = IntStream.range(1, 13).toArray();
         checkIterates(buf.asNewBuffer(), rootPos, expected);
 
-        checkIterates(buf.asNewBuffer(), rootPos, null, null, false, expected);
-        checkIterates(buf.asNewBuffer(), rootPos, null, null, true, expected);
+        checkIterates(buf.asNewBuffer(), rootPos, null, null, ValueIterator.LeftBoundTreatment.GREATER, expected);
+        checkIterates(buf.asNewBuffer(), rootPos, null, null, ValueIterator.LeftBoundTreatment.ADMIT_EXACT, expected);
+        checkIterates(buf.asNewBuffer(), rootPos, null, null, ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, expected);
+    }
+
+    @Test
+    public void testIteratorWithBoundsGreater() throws IOException
+    {
+        DataOutputBuffer buf = new DataOutputBufferPaged();
+        IncrementalTrieWriter<Integer> builder = makeTrie(buf);
+        long rootPos = builder.complete();
+
+        checkIterates(buf.asNewBuffer(), rootPos, "151", "515", ValueIterator.LeftBoundTreatment.GREATER, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "15151", "51515", ValueIterator.LeftBoundTreatment.GREATER, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "705", "73", ValueIterator.LeftBoundTreatment.GREATER, 9, 10, 11);
+        checkIterates(buf.asNewBuffer(), rootPos, "7051", "735", ValueIterator.LeftBoundTreatment.GREATER, 10, 11);
+        checkIterates(buf.asNewBuffer(), rootPos, "70", "737", ValueIterator.LeftBoundTreatment.GREATER, 9, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, "7054", "735", ValueIterator.LeftBoundTreatment.GREATER, 10, 11);
+
+        checkIterates(buf.asNewBuffer(), rootPos, null, "515", ValueIterator.LeftBoundTreatment.GREATER, 1, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, null, "51515", ValueIterator.LeftBoundTreatment.GREATER, 1, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "151", null, ValueIterator.LeftBoundTreatment.GREATER, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, "15151", null, ValueIterator.LeftBoundTreatment.GREATER, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+
+        checkIterates(buf.asNewBuffer(), rootPos, "151", "155", ValueIterator.LeftBoundTreatment.GREATER, 3);
+        checkIterates(buf.asNewBuffer(), rootPos, "15", "151", ValueIterator.LeftBoundTreatment.GREATER, 2);
+        checkIterates(buf.asNewBuffer(), rootPos, "1511", "1512", ValueIterator.LeftBoundTreatment.GREATER);
+        checkIterates(buf.asNewBuffer(), rootPos, "155", "155", ValueIterator.LeftBoundTreatment.GREATER);
+
+        checkIterates(buf.asNewBuffer(), rootPos, "155", "156", ValueIterator.LeftBoundTreatment.GREATER);
+        checkIterates(buf.asNewBuffer(), rootPos, "154", "156", ValueIterator.LeftBoundTreatment.GREATER, 3);
     }
 
     @Test
-    public void testIteratorWithBounds() throws IOException
+    public void testIteratorWithBoundsExact() throws IOException
     {
         DataOutputBuffer buf = new DataOutputBufferPaged();
         IncrementalTrieWriter<Integer> builder = makeTrie(buf);
         long rootPos = builder.complete();
 
-        checkIterates(buf.asNewBuffer(), rootPos, "151", "515", false, 3, 4, 5);
-        checkIterates(buf.asNewBuffer(), rootPos, "15151", "51515", false, 3, 4, 5);
-        checkIterates(buf.asNewBuffer(), rootPos, "705", "73", false, 9, 10, 11);
-        checkIterates(buf.asNewBuffer(), rootPos, "7051", "735", false, 10, 11);
-        checkIterates(buf.asNewBuffer(), rootPos, "70", "737", false, 9, 10, 11, 12);
-        checkIterates(buf.asNewBuffer(), rootPos, "7054", "735", false, 10, 11);
-
-        checkIterates(buf.asNewBuffer(), rootPos, null, "515", false, 1, 2, 3, 4, 5);
-        checkIterates(buf.asNewBuffer(), rootPos, null, "51515", false, 1, 2, 3, 4, 5);
-        checkIterates(buf.asNewBuffer(), rootPos, "151", null, false, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
-        checkIterates(buf.asNewBuffer(), rootPos, "15151", null, false, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, "151", "515", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "15151", "51515", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "705", "73", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 9, 10, 11);
+        checkIterates(buf.asNewBuffer(), rootPos, "7051", "735", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 9, 10, 11);
+        checkIterates(buf.asNewBuffer(), rootPos, "70", "737", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 8, 9, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, "7054", "735", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 10, 11);
+
+        checkIterates(buf.asNewBuffer(), rootPos, null, "515", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 1, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, null, "51515", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 1, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "151", null, ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, "15151", null, ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+
+        checkIterates(buf.asNewBuffer(), rootPos, "151", "155", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 2, 3);
+        checkIterates(buf.asNewBuffer(), rootPos, "15", "151", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 2);
+        checkIterates(buf.asNewBuffer(), rootPos, "1511", "1512", ValueIterator.LeftBoundTreatment.ADMIT_EXACT);
+        checkIterates(buf.asNewBuffer(), rootPos, "155", "155", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 3);
+
+        checkIterates(buf.asNewBuffer(), rootPos, "155", "156", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 3);
+        checkIterates(buf.asNewBuffer(), rootPos, "154", "156", ValueIterator.LeftBoundTreatment.ADMIT_EXACT, 3);
     }
 
     @Test
@@ -146,44 +208,84 @@ public void testIteratorWithBoundsAndAdmitPrefix() throws IOException
         IncrementalTrieWriter<Integer> builder = makeTrie(buf);
         long rootPos = builder.complete();
 
-        checkIterates(buf.asNewBuffer(), rootPos, "151", "515", true, 2, 3, 4, 5);
-        checkIterates(buf.asNewBuffer(), rootPos, "15151", "51515", true, 2, 3, 4, 5);
-        checkIterates(buf.asNewBuffer(), rootPos, "705", "73", true, 8, 9, 10, 11);
-        checkIterates(buf.asNewBuffer(), rootPos, "7051", "735", true, 9, 10, 11);
-        checkIterates(buf.asNewBuffer(), rootPos, "70", "737", true, 8, 9, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, "151", "515", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "15151", "51515", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "705", "73", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 8, 9, 10, 11);
+        checkIterates(buf.asNewBuffer(), rootPos, "7051", "735", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 9, 10, 11);
+        checkIterates(buf.asNewBuffer(), rootPos, "70", "737", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 8, 9, 10, 11, 12);
         // Note: 7054 has 70 as prefix, but we don't include 70 because a clearly smaller non-prefix entry 7051
         // exists between the bound and the prefix
-        checkIterates(buf.asNewBuffer(), rootPos, "7054", "735", true, 10, 11);
+        checkIterates(buf.asNewBuffer(), rootPos, "7054", "735", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 10, 11);
 
 
-        checkIterates(buf.asNewBuffer(), rootPos, null, "515", true, 1, 2, 3, 4, 5);
-        checkIterates(buf.asNewBuffer(), rootPos, null, "51515", true, 1, 2, 3, 4, 5);
-        checkIterates(buf.asNewBuffer(), rootPos, "151", null, true, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
-        checkIterates(buf.asNewBuffer(), rootPos, "15151", null, true, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
-        checkIterates(buf.asNewBuffer(), rootPos, "7054", null, true, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, null, "515", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 1, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, null, "51515", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 1, 2, 3, 4, 5);
+        checkIterates(buf.asNewBuffer(), rootPos, "151", null, ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, "15151", null, ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+        checkIterates(buf.asNewBuffer(), rootPos, "7054", null, ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 10, 11, 12);
+
+        checkIterates(buf.asNewBuffer(), rootPos, "151", "155", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 2, 3);
+        checkIterates(buf.asNewBuffer(), rootPos, "15", "151", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 2);
+        checkIterates(buf.asNewBuffer(), rootPos, "1511", "1512", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 2);
+        checkIterates(buf.asNewBuffer(), rootPos, "155", "155", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 3);
+
+        checkIterates(buf.asNewBuffer(), rootPos, "155", "156", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 3);
+        checkIterates(buf.asNewBuffer(), rootPos, "154", "156", ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES, 3);
     }
 
-    private void checkIterates(ByteBuffer buffer, long rootPos, String from, String to, boolean admitPrefix, int... expected)
+    private void checkIterates(ByteBuffer buffer, long rootPos, String from, String to, ValueIterator.LeftBoundTreatment admitPrefix, int... expected)
     {
         Rebufferer source = new ByteBufRebufferer(buffer);
-        ValueIterator<?> it = new ValueIterator<>(source, rootPos, source(from), source(to), admitPrefix);
+        ValueIterator<?> it = new ValueIterator<>(source, rootPos, source(from), source(to), admitPrefix, version);
         checkReturns(from + "-->" + to, it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
 
-        ReverseValueIterator<?> rit = new ReverseValueIterator<>(source, rootPos, source(from), source(to), admitPrefix);
+        if (admitPrefix != ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES && from != null)
+        {
+            it = new ValueIterator<>(source, rootPos, null, source(to), admitPrefix, true, version);
+            it.skipTo(source(from), admitPrefix);
+            checkReturns(from + "-->" + to, it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+
+            it = new ValueIterator<>(source, rootPos, source(from), source(to), admitPrefix, true, version);
+            it.skipTo(source(from), admitPrefix);
+            checkReturns(from + "-->" + to, it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+
+            it = new ValueIterator<>(source, rootPos, null, source(to), admitPrefix, true, version);
+            it.skipTo(source(from), admitPrefix);
+            it.skipTo(source(from), admitPrefix);
+            checkReturns(from + "-->" + to, it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+        }
+
+        if (to != null)
+        {
+            // `to` is always inclusive, if we have a match for it check skipping to it works
+            Walker<?> w = new Walker<>(source, rootPos, version);
+            if (w.follow(source(to)) == ByteSource.END_OF_STREAM && w.payloadFlags() != 0
+                && (admitPrefix != ValueIterator.LeftBoundTreatment.GREATER || !to.equals(from))) // skipping with ADMIT_EXACT may accept match after left bound is GREATER. Needs fix/error msg?
+            {
+                it = new ValueIterator<>(source, rootPos, source(from), source(to), admitPrefix, true, version);
+                it.skipTo(source(to), ValueIterator.LeftBoundTreatment.ADMIT_EXACT);
+                int[] exp = expected.length > 0 ? new int[] {expected[expected.length - 1]} : new int[0];
+                checkReturns(from + "-->" + to + " skip " + to, it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), exp);
+            }
+        }
+
+        ReverseValueIterator<?> rit = new ReverseValueIterator<>(source, rootPos, source(from), source(to), admitPrefix, true, version);
         reverse(expected);
-        checkReturns(from + "<--" + to, rit::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+        checkReturns(from + "<--" + to, rit::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), rit::collectedKey, expected);
         reverse(expected);  // return array in its original form if reused
     }
 
+
+
     private void checkIterates(ByteBuffer buffer, long rootPos, int... expected)
     {
         Rebufferer source = new ByteBufRebufferer(buffer);
-        ValueIterator<?> it = new ValueIterator<>(source, rootPos);
-        checkReturns("Forward", it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+        ValueIterator<?> it = new ValueIterator<>(source, rootPos, true, version);
+        checkReturns("Forward", it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), it::collectedKey, expected);
 
-        ReverseValueIterator<?> rit = new ReverseValueIterator<>(source, rootPos);
+        ReverseValueIterator<?> rit = new ReverseValueIterator<>(source, rootPos, true, version);
         reverse(expected);
-        checkReturns("Reverse", rit::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+        checkReturns("Reverse", rit::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), rit::collectedKey, expected);
         reverse(expected);  // return array in its original form if reused
     }
 
@@ -203,19 +305,213 @@ private int getPayloadFlags(ByteBuffer buffer, int pos)
         return TrieNode.at(buffer, pos).payloadFlags(buffer, pos);
     }
 
-    private void checkReturns(String testCase, LongSupplier supplier, LongToIntFunction mapper, int... expected)
+    private void checkReturns(String testCase,
+                              LongSupplier supplier,
+                              LongToIntFunction mapper,
+                              int... expected)
+    {
+        checkReturns(testCase, supplier, mapper, null, expected);
+    }
+
+    private void checkReturns(String testCase,
+                              LongSupplier supplier,
+                              LongToIntFunction mapper,
+                              Supplier<ByteComparable> byteComparableSupplier,
+                              int... expected)
     {
         IntArrayList list = new IntArrayList();
         while (true)
         {
             long pos = supplier.getAsLong();
-            if (pos == Walker.NONE)
+            if (pos == -1)
                 break;
+
+            if (byteComparableSupplier != null)
+            {
+                String value = decodeSource(byteComparableSupplier.get());
+                assertEquals(memoryTrieEntryMap.get(value), (Integer) mapper.applyAsInt(pos));
+            }
             list.add(mapper.applyAsInt(pos));
         }
         assertArrayEquals(testCase + ": " + list + " != " + Arrays.toString(expected), expected, list.toIntArray());
     }
 
+
+    @Test
+    public void testSkipToGreater() throws IOException
+    {
+        DataOutputBuffer buf = new DataOutputBufferPaged();
+        IncrementalTrieWriter<Integer> builder = makeTrie(buf);
+        long rootPos = builder.complete();
+
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     null,
+                     new String[] { null, "151", "515", "999" },
+                     new int[] { 1, 3, 6 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     null,
+                     new String[] { "151", "15151", "515", "555", "999" },
+                     new int[] { 3, 4, 6, 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     null,
+                     new String[] { "69", "705", "7100", "73" },
+                     new int[] { 8, 9, 10, 12 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     null,
+                     new String[] { "70", "7051", "717", "737" },
+                     new int[] { 9, 10, 11 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555" },
+                     new int[] { 7, 8, 9 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", "7051" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7050",
+                     new String[] { "5555", "7051" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", "7059" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", "709" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", "79" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", "9" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", null, "7051" },
+                     new int[] { 7, 8 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", null, "7059" },
+                     new int[] { 7, 8 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", null, "709" },
+                     new int[] { 7, 8 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", null, "79" },
+                     new int[] { 7, 8 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.GREATER,
+                     "7051",
+                     new String[] { "5555", null, "9" },
+                     new int[] { 7, 8 });
+    }
+
+    @Test
+    public void testSkipToExact() throws IOException
+    {
+        DataOutputBuffer buf = new DataOutputBufferPaged();
+        IncrementalTrieWriter<Integer> builder = makeTrie(buf);
+        long rootPos = builder.complete();
+
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     null,
+                     new String[] { null, "151", "515", "999" },
+                     new int[] { 1, 2, 5 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     null,
+                     new String[] { "151", "15151", "515", "555", "999" },
+                     new int[] { 2, 3, 5, 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     null,
+                     new String[] { "69", "705", "7100", "73" },
+                     new int[] { 8, 9, 10, 11, 12 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     null,
+                     new String[] { "70", "7051", "717", "737" },
+                     new int[] { 8, 9, 10, 12 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555" },
+                     new int[] { 7, 8, 9 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", "7051" },
+                     new int[] { 7, 9 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7050",
+                     new String[] { "5555", "7051" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", "7059" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", "709" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", "79" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", "9" },
+                     new int[] { 7 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", null, "7051" },
+                     new int[] { 7, 8, 9 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", null, "7059" },
+                     new int[] { 7, 8 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", null, "709" },
+                     new int[] { 7, 8 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", null, "79" },
+                     new int[] { 7, 8 });
+        checkSkipsTo(buf.asNewBuffer(), rootPos, ValueIterator.LeftBoundTreatment.ADMIT_EXACT,
+                     "7051",
+                     new String[] { "5555", null, "9" },
+                     new int[] { 7, 8 });
+    }
+
+    private void checkSkipsTo(ByteBuffer buffer, long rootPos, ValueIterator.LeftBoundTreatment leftBoundTreatment, String rightBound, String[] skips, int[] values)
+    {
+        checkSkipsTo(buffer, rootPos, leftBoundTreatment, null, rightBound, skips, values);
+        String leftBound = skips[0];
+        skips[0] = null;
+        if (leftBound != null)
+            checkSkipsTo(buffer, rootPos, leftBoundTreatment, leftBound, rightBound, skips, values);
+    }
+
+    private void checkSkipsTo(ByteBuffer buffer, long rootPos, ValueIterator.LeftBoundTreatment admitPrefix, String from, String to, String[] skips, int[] expected)
+    {
+        Rebufferer source = new ByteBufRebufferer(buffer);
+        ValueIterator<?> it = new ValueIterator<>(source, rootPos, source(from), source(to), admitPrefix, true, version);
+        int i = 0;
+        IntArrayList list = new IntArrayList();
+        while (true)
+        {
+            String skip = i < skips.length ? skips[i++] : null;
+            if (skip != null)
+                it.skipTo(source(skip), admitPrefix);
+
+            long pos = it.nextPayloadedNode();
+            if (pos == -1)
+                break;
+            list.add(getPayloadFlags(buffer, (int) pos));
+        }
+        assertArrayEquals(String.format("skipTo left %s right %s skips %s : %s != %s", from, to, Arrays.toString(skips), list, Arrays.toString(expected)), expected, list.toIntArray());
+    }
+
     @Test
     public void testPartialTail() throws IOException
     {
@@ -225,8 +521,18 @@ public void testPartialTail() throws IOException
         long rootPos = builder.complete();
         try (Rebufferer source = new ByteBufRebufferer(buf.asNewBuffer());
              Rebufferer partialSource = new TailOverridingRebufferer(new ByteBufRebufferer(buf.asNewBuffer()), ptail.cutoff(), ptail.tail());
-             ValueIterator<?> it = new ValueIterator<>(new ByteBufRebufferer(buf.asNewBuffer()), rootPos, source("151"), source("515"), true);
-             ValueIterator<?> tailIt = new ValueIterator<>(new TailOverridingRebufferer(new ByteBufRebufferer(buf.asNewBuffer()), ptail.cutoff(), ptail.tail()), ptail.root(), source("151"), source("515"), true))
+             ValueIterator<?> it = new ValueIterator<>(new ByteBufRebufferer(buf.asNewBuffer()),
+                                                       rootPos,
+                                                       source("151"),
+                                                       source("515"),
+                                                       ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES,
+                                                       version);
+             ValueIterator<?> tailIt = new ValueIterator<>(new TailOverridingRebufferer(new ByteBufRebufferer(buf.asNewBuffer()), ptail.cutoff(), ptail.tail()),
+                                                           ptail.root(),
+                                                           source("151"),
+                                                           source("515"),
+                                                           ValueIterator.LeftBoundTreatment.ADMIT_PREFIXES,
+                                                           version))
         {
             while (true)
             {
@@ -261,7 +567,7 @@ public void testBigTrie() throws IOException
 
         long rootPos = builder.complete();
         Rebufferer source = new ByteBufRebufferer(buf.asNewBuffer());
-        ValueIterator<?> it = new ValueIterator<>(source, rootPos);
+        ValueIterator<?> it = new ValueIterator<>(source, rootPos, version);
 
         while (true)
         {
@@ -279,19 +585,12 @@ private IncrementalTrieWriter<Integer> makeTrie(DataOutputBuffer out) throws IOE
     {
         IncrementalTrieWriter<Integer> builder = newTrieWriter(serializer, out);
         dump = true;
-        builder.add(source("115"), 1);
-        builder.add(source("151"), 2);
-        builder.add(source("155"), 3);
-        builder.add(source("511"), 4);
-        builder.add(source("515"), 5);
-        builder.add(source("551"), 6);
-        builder.add(source("555555555555555555555555555555555555555555555555555555555555555555"), 7);
-
-        builder.add(source("70"), 8);
-        builder.add(source("7051"), 9);
-        builder.add(source("717"), 10);
-        builder.add(source("73"), 11);
-        builder.add(source("737"), 12);
+        for (Map.Entry<String, Integer> entry : memoryTrieEntryMap.entrySet())
+        {
+            String key = entry.getKey();
+            Integer value = entry.getValue();
+            builder.add(source(key), value);
+        }
         return builder;
     }
 
diff --git a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
index 9f98fb0dfb3b..a765b2fad8d9 100644
--- a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
+++ b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
@@ -328,7 +328,7 @@ public void testGetFilePointer() throws IOException
     public void testGetPath() throws IOException
     {
         SequentialWriter file = createTempFile("brafGetPath");
-        assert file.getPath().contains("brafGetPath");
+        assert file.getFile().toString().contains("brafGetPath");
         file.finish();
     }
 
@@ -416,7 +416,7 @@ public void testBytesPastMark() throws IOException
         try (FileHandle fh = new FileHandle.Builder(tmpFile).complete();
              RandomAccessReader r = fh.createReader())
         {
-            assert tmpFile.path().equals(r.getPath());
+            assert tmpFile.equals(r.getFile());
 
             // Create a mark and move the rw there.
             final DataPosition mark = r.mark();
@@ -437,7 +437,7 @@ public void testClose() throws IOException
         w.write(data);
         w.finish();
 
-        final RandomAccessReader r = RandomAccessReader.open(new File(w.getPath()));
+        final RandomAccessReader r = RandomAccessReader.open(w.getFile());
 
         r.close(); // closing to test read after close
 
@@ -449,7 +449,7 @@ public void testClose() throws IOException
         //write of a 0 length, but that is kind of a corner case
         expectException(() -> { w.write(generateByteArray(1)); return null; }, AssertionError.class);
 
-        try (RandomAccessReader copy = RandomAccessReader.open(new File(r.getPath())))
+        try (RandomAccessReader copy = RandomAccessReader.open(r.getFile()))
         {
             ByteBuffer contents = ByteBufferUtil.read(copy, (int) copy.length());
 
@@ -526,7 +526,7 @@ public void testReadOnly() throws IOException
         file.sync(); // flushing file to the disk
 
         // read-only copy of the file, with fixed file length
-        final RandomAccessReader copy = RandomAccessReader.open(new File(file.getPath()));
+        final RandomAccessReader copy = RandomAccessReader.open(file.getFile());
 
         copy.seek(copy.length());
         assertTrue(copy.bytesRemaining() == 0 && copy.isEOF());
diff --git a/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java
index 196a0b25c831..6c616a60d804 100644
--- a/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java
@@ -27,10 +27,11 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import static org.junit.Assert.*;
-
 import org.apache.cassandra.config.DatabaseDescriptor;
 
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
 public class ChecksummedRandomAccessReaderTest
 {
     @BeforeClass
diff --git a/test/unit/org/apache/cassandra/io/util/ChecksummedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/util/ChecksummedSequentialWriterTest.java
index 5d92d4584fa8..74e447e3d777 100644
--- a/test/unit/org/apache/cassandra/io/util/ChecksummedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/util/ChecksummedSequentialWriterTest.java
@@ -1,21 +1,21 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*    http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied.  See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package org.apache.cassandra.io.util;
 
 import java.io.IOException;
@@ -23,9 +23,9 @@
 import java.util.List;
 
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 
-import org.junit.Assert;
 import org.apache.cassandra.config.DatabaseDescriptor;
 
 public class ChecksummedSequentialWriterTest extends SequentialWriterTest
@@ -96,5 +96,4 @@ protected void assertAborted() throws Exception
             super.assertAborted();
         }
     }
-
 }
diff --git a/test/unit/org/apache/cassandra/io/util/FileSegmentInputStreamTest.java b/test/unit/org/apache/cassandra/io/util/FileSegmentInputStreamTest.java
index 0f72e33760f3..0c5dbb090b13 100644
--- a/test/unit/org/apache/cassandra/io/util/FileSegmentInputStreamTest.java
+++ b/test/unit/org/apache/cassandra/io/util/FileSegmentInputStreamTest.java
@@ -60,8 +60,8 @@ private void testRead(int offset, int size, int checkInterval) throws IOExceptio
         final ByteBuffer buffer = allocateBuffer(size);
         final String path = buffer.toString();
 
-        FileSegmentInputStream reader = new FileSegmentInputStream(buffer.duplicate(), path, offset);
-        assertEquals(path, reader.getPath());
+        FileSegmentInputStream reader = new FileSegmentInputStream(buffer.duplicate(), new File(path), offset);
+        assertEquals(path, reader.getFile().path());
 
         for (int i = offset; i < (size + offset); i += checkInterval)
         {
@@ -89,7 +89,7 @@ private void testRead(int offset, int size, int checkInterval) throws IOExceptio
     @Test(expected = UnsupportedOperationException.class)
     public void testMarkNotSupported() throws Exception
     {
-        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0))
+        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), new File(""), 0))
         {
             assertFalse(reader.markSupported());
             assertEquals(0, reader.bytesPastMark(null));
@@ -100,7 +100,7 @@ public void testMarkNotSupported() throws Exception
     @Test(expected = UnsupportedOperationException.class)
     public void testResetNotSupported() throws Exception
     {
-        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0))
+        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), new File(""), 0))
         {
             reader.reset(null);
         }
@@ -109,7 +109,7 @@ public void testResetNotSupported() throws Exception
     @Test(expected = IllegalArgumentException.class)
     public void testSeekNegative() throws Exception
     {
-        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0))
+        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), new File(""), 0))
         {
             reader.seek(-1);
         }
@@ -118,7 +118,7 @@ public void testSeekNegative() throws Exception
     @Test(expected = IllegalArgumentException.class)
     public void testSeekBeforeOffset() throws Exception
     {
-        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024))
+        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), new File(""), 1024))
         {
             reader.seek(1023);
         }
@@ -127,7 +127,7 @@ public void testSeekBeforeOffset() throws Exception
     @Test(expected = IllegalArgumentException.class)
     public void testSeekPastLength() throws Exception
     {
-        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024))
+        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), new File(""), 1024))
         {
             reader.seek(2049);
         }
@@ -136,7 +136,7 @@ public void testSeekPastLength() throws Exception
     @Test(expected = EOFException.class)
     public void testReadBytesTooMany() throws Exception
     {
-        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024))
+        try (FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), new File(""), 1024))
         {
             ByteBufferUtil.read(reader, 2049);
         }
diff --git a/test/unit/org/apache/cassandra/io/util/FileTest.java b/test/unit/org/apache/cassandra/io/util/FileTest.java
index 718f3210422e..8ce4aad29cc1 100644
--- a/test/unit/org/apache/cassandra/io/util/FileTest.java
+++ b/test/unit/org/apache/cassandra/io/util/FileTest.java
@@ -21,9 +21,12 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.StandardCopyOption;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
@@ -125,7 +128,7 @@ private static String nonAbsolute(java.io.File file)
         return file.getParent() + File.pathSeparator() + ".." + File.pathSeparator() + file.getParentFile().getName() + File.pathSeparator() + file.getName();
     }
 
-    private void    testEquivalence(String path) throws IOException
+    private void testEquivalence(String path) throws IOException
     {
         java.io.File file = new java.io.File(path); //checkstyle: permit this instantiation
         if (file.exists()) testExists(path);
@@ -388,4 +391,44 @@ public void testAppend() throws Exception
         buf.put(buf1.array()).put(buf2.array());
         Assertions.assertThat(Files.readAllBytes(f.toPath())).isEqualTo(buf.array());
     }
+    
+    @Test
+    public void testCopy() throws IOException
+    {
+        File file = new File(dir, "a");
+        file.toJavaIOFile().createNewFile();
+        Assert.assertTrue(file.exists());
+
+        File newFile = new File(dir, "b");
+        Assert.assertFalse(newFile.exists());
+
+        file.copy(newFile, StandardCopyOption.COPY_ATTRIBUTES);
+        Assert.assertTrue(newFile.exists());
+    }
+
+    @Test
+    public void testResolve()
+    {
+        File file = new File("somewhere/a/");
+        Assert.assertEquals(new File("somewhere/a/b"), file.resolve(new File("b")));
+        Assert.assertEquals(new File("somewhere/a/b"), file.resolve("b"));
+
+        Assert.assertEquals(new File("somewhere/b"), file.resolveSibling("b"));
+    }
+
+    @Test
+    public void testRelative()
+    {
+        File file = new File("somewhere/a/");
+        Assert.assertEquals(new File("../b"), file.relativize(new File("somewhere/b")));
+    }
+
+    @Test
+    public void testDeleteFail()
+    {
+        File file = new File(UUID.randomUUID().toString());
+        Assertions.assertThatExceptionOfType(UncheckedIOException.class)
+                  .isThrownBy(file::deleteRecursive)
+                  .withCauseInstanceOf(NoSuchFileException.class);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/io/util/FileUtilsTest.java b/test/unit/org/apache/cassandra/io/util/FileUtilsTest.java
index c8359e07a35a..1d6ae5983222 100644
--- a/test/unit/org/apache/cassandra/io/util/FileUtilsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/FileUtilsTest.java
@@ -19,17 +19,26 @@
 package org.apache.cassandra.io.util;
 
 import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.Random;
 
+import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.metrics.NativeMemoryMetrics;
+import org.apache.cassandra.utils.FastByteOperations;
+import org.apache.cassandra.utils.INativeLibrary;
+import org.apache.cassandra.utils.NativeLibrary;
 import org.assertj.core.api.Assertions;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -75,6 +84,13 @@ public void testParseFileSize() throws Exception
             6621259022467L, FileUtils.parseFileSize("6.022 TiB"));
     }
 
+    @Test
+    public void testDelete()
+    {
+        File file = FileUtils.createDeletableTempFile("testTruncate", "1");
+        FileUtils.delete(file);
+    }
+
     @Test
     public void testTruncate() throws IOException
     {
@@ -99,10 +115,10 @@ public void testTruncate() throws IOException
     @Test
     public void testFolderSize() throws Exception
     {
-        File folder = createFolder(Paths.get(DatabaseDescriptor.getAllDataFileLocations()[0], "testFolderSize"));
+        File folder = createFolder(DatabaseDescriptor.getAllDataFileLocations()[0], "testFolderSize");
         folder.deleteOnExit();
 
-        File childFolder = createFolder(Paths.get(folder.path(), "child"));
+        File childFolder = createFolder(folder, "child");
 
         File[] files = {
                        createFile(new File(folder, "001"), 10000),
@@ -119,6 +135,29 @@ public void testFolderSize() throws Exception
         assertEquals(Arrays.stream(files).mapToLong(f -> f.length()).sum(), size);
     }
 
+    @Test
+    public void testClean()
+    {
+        FileUtils.clean(null); // should not throw
+
+        FileUtils.clean(ByteBuffer.allocate(1)); // should not throw
+        FileUtils.clean(BufferType.ON_HEAP.allocate(1)); // should not throw
+
+        ByteBuffer buffer = ByteBuffer.allocateDirect(1);
+        long usedMemory = NativeMemoryMetrics.instance.usedNioDirectMemoryValue();
+
+        FileUtils.clean(buffer);
+        assertTrue("Used memory should have decreased by at least one byte",
+                   NativeMemoryMetrics.instance.usedNioDirectMemoryValue() <= usedMemory - 1);
+
+        buffer = BufferType.OFF_HEAP.allocate(1);
+        usedMemory = NativeMemoryMetrics.instance.usedNioDirectMemoryValue();
+
+        FileUtils.clean(buffer);
+        assertTrue("Used memory should have decreased by at least one byte",
+                   NativeMemoryMetrics.instance.usedNioDirectMemoryValue() <= usedMemory - 1);
+    }
+
     @Test
     public void testIsContained()
     {
@@ -218,10 +257,78 @@ public void testDeleteDirectoryIfEmpty() throws IOException
                   .hasMessageContaining("is not a directory");
     }
 
+    @Test
+    public void testSize() throws IOException
+    {
+        Path tmpDir = Files.createTempDirectory(this.getClass().getSimpleName());
+        Path path = tmpDir.resolve("a.txt");
+        Assert.assertEquals(0, FileUtils.size(path));
 
-    private File createFolder(Path path)
+        createFile(new File(path), 10000);
+        Assert.assertEquals(10000, FileUtils.size(path));
+    }
+
+    @Test
+    public void testCreateHardLinkWithoutConfirm() throws Throwable
     {
-        File folder = new File(path);
+        Assume.assumeTrue(NativeLibrary.instance.isOS(INativeLibrary.OSType.LINUX));
+
+        Path tmpDir = Files.createTempDirectory(this.getClass().getSimpleName());
+
+        Path from = tmpDir.resolve("b.txt");
+        writeData(new File(from), 100);
+        Assert.assertTrue(Files.exists(from));
+        Assert.assertEquals(1, Files.getAttribute(from, "unix:nlink"));
+
+        Path to = tmpDir.resolve("c.txt");
+        Assert.assertFalse(Files.exists(to));
+    }
+
+    @Test
+    public void testCopyWithOutConfirm() throws Throwable
+    {
+        Assume.assumeTrue(NativeLibrary.instance.isOS(INativeLibrary.OSType.LINUX));
+
+        Path tmpDir = Files.createTempDirectory(this.getClass().getSimpleName());
+
+        Path from = tmpDir.resolve("b.txt");
+        writeData(new File(from), 100);
+        Assert.assertTrue(Files.exists(from));
+        Assert.assertEquals(1, Files.getAttribute(from, "unix:nlink"));
+
+        Path to = tmpDir.resolve("c.txt");
+        Assert.assertFalse(Files.exists(to));
+
+        FileUtils.copyWithOutConfirm(new File(from), new File(to));
+        compareFile(from, to);
+        Assert.assertEquals(1, Files.getAttribute(from, "unix:nlink"));
+        Assert.assertEquals(1, Files.getAttribute(to, "unix:nlink"));
+
+        File nonExisting = new File(from.resolveSibling("non_existing.txt"));
+        to = tmpDir.resolve("d.txt");
+        FileUtils.copyWithOutConfirm(nonExisting, new File(to));
+        Assert.assertFalse(new File(to).exists());
+    }
+
+    @Test
+    public void testLegacyDSEAPI() throws IOException
+    {
+        Path tmpDir = Files.createTempDirectory(this.getClass().getSimpleName());
+
+        FileUtils.createDirectory(tmpDir);
+        Path f = tmpDir.resolve("somefile");
+        FileUtils.appendAndSync(f, "lorem", "ipsum");
+        assertEquals(Arrays.asList(f), FileUtils.listPaths(tmpDir, path -> true));
+        assertEquals(Arrays.asList(f), FileUtils.listPaths(tmpDir));
+        FileUtils.deleteContent(tmpDir);
+        assertEquals(Arrays.asList(), FileUtils.listPaths(tmpDir));
+        FileUtils.delete(tmpDir);
+        FileUtils.deleteRecursive(tmpDir);
+    }
+
+    private File createFolder(File folder, String additionalName)
+    {
+        folder = folder.resolve(additionalName);
         FileUtils.createDirectory(folder);
         return folder;
     }
@@ -238,4 +345,22 @@ private File createFile(File file, long size)
         }
         return file;
     }
+
+    private File writeData(File file, int size) throws Throwable
+    {
+        Random random = new Random();
+        try (RandomAccessFile f = new RandomAccessFile(file.toJavaIOFile(), "rw"))
+        {
+            byte[] bytes = new byte[size];
+            random.nextBytes(bytes);
+
+            f.write(bytes);
+        }
+        return file;
+    }
+
+    private boolean compareFile(Path left, Path right) throws IOException
+    {
+        return FastByteOperations.compareUnsigned(ByteBuffer.wrap(Files.readAllBytes(left)), ByteBuffer.wrap(Files.readAllBytes(right))) == 0;
+    }
 }
diff --git a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
index e6b5dd0c0962..7815faacb956 100644
--- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
@@ -114,10 +114,11 @@ public void testEmpty() throws Exception
     public void testTwoSegments() throws Exception
     {
         ByteBuffer buffer = allocateBuffer(2048);
+        int bufSize = 1024;
         try (ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer));
              MmappedRegions regions = MmappedRegions.empty(channel))
         {
-            regions.extend(1024);
+            regions.extend(1024, bufSize);
             for (int i = 0; i < 1024; i++)
             {
                 MmappedRegions.Region region = regions.floor(i);
@@ -126,7 +127,7 @@ public void testTwoSegments() throws Exception
                 assertEquals(1024, region.end());
             }
 
-            regions.extend(2048);
+            regions.extend(2048, bufSize);
             for (int i = 0; i < 2048; i++)
             {
                 MmappedRegions.Region region = regions.floor(i);
@@ -148,15 +149,16 @@ public void testTwoSegments() throws Exception
     @Test
     public void testSmallSegmentSize() throws Exception
     {
+        int bufSize = 1024;
         MmappedRegions.MAX_SEGMENT_SIZE = 1024;
 
         ByteBuffer buffer = allocateBuffer(4096);
         try (ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer));
              MmappedRegions regions = MmappedRegions.empty(channel))
         {
-            regions.extend(1024);
-            regions.extend(2048);
-            regions.extend(4096);
+            regions.extend(1024, bufSize);
+            regions.extend(2048, bufSize);
+            regions.extend(4096, bufSize);
 
             final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
             for (int i = 0; i < buffer.capacity(); i++)
@@ -169,17 +171,45 @@ public void testSmallSegmentSize() throws Exception
         }
     }
 
+    @Test
+    public void testSizeIsChunkMultiple() throws Exception
+    {
+        final int oldMaxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE;
+        final int bufSize = 1024;
+        MmappedRegions.MAX_SEGMENT_SIZE = 2047;
+        ByteBuffer buffer = allocateBuffer(4096);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
+        {
+            regions.extend(1024, bufSize);
+            regions.extend(2048, bufSize);
+            regions.extend(4096, bufSize);
+            for (int i = 0; i < buffer.capacity(); i++)
+            {
+                MmappedRegions.Region region = regions.floor(i);
+                assertNotNull(region);
+                assertEquals(bufSize * (i / bufSize), region.offset());
+                assertEquals(bufSize + (bufSize * (i / bufSize)), region.end());
+            }
+        }
+        finally
+        {
+            MmappedRegions.MAX_SEGMENT_SIZE = oldMaxSegmentSize;
+        }
+    }
+
     @Test
     public void testAllocRegions() throws Exception
     {
         MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+        int bufSize = 1024;
 
         ByteBuffer buffer = allocateBuffer(MmappedRegions.MAX_SEGMENT_SIZE * MmappedRegions.REGION_ALLOC_SIZE * 3);
 
         try (ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer));
              MmappedRegions regions = MmappedRegions.empty(channel))
         {
-            regions.extend(buffer.capacity());
+            regions.extend(buffer.capacity(), bufSize);
 
             final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
             for (int i = 0; i < buffer.capacity(); i++)
@@ -196,17 +226,18 @@ public void testAllocRegions() throws Exception
     public void testCopy() throws Exception
     {
         ByteBuffer buffer = allocateBuffer(128 * 1024);
+        int bufSize = 4096;
 
         MmappedRegions snapshot;
         ChannelProxy channelCopy;
 
-        try (ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer));
-             MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4))
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer));
+            MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4, bufSize, 0, false))
         {
             // create 3 more segments, one per quater capacity
-            regions.extend(buffer.capacity() / 2);
-            regions.extend(3 * buffer.capacity() / 4);
-            regions.extend(buffer.capacity());
+            regions.extend(buffer.capacity() / 2, bufSize);
+            regions.extend(3 * buffer.capacity() / 4, bufSize);
+            regions.extend(buffer.capacity(), bufSize);
 
             // make a snapshot
             snapshot = regions.sharedCopy();
@@ -238,6 +269,7 @@ public void testCopy() throws Exception
     public void testCopyCannotExtend() throws Exception
     {
         ByteBuffer buffer = allocateBuffer(128 * 1024);
+        int bufSize = 1024;
 
         MmappedRegions snapshot;
         ChannelProxy channelCopy;
@@ -245,7 +277,7 @@ public void testCopyCannotExtend() throws Exception
         try (ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer));
              MmappedRegions regions = MmappedRegions.empty(channel))
         {
-            regions.extend(buffer.capacity() / 2);
+            regions.extend(buffer.capacity() / 2, bufSize);
 
             // make a snapshot
             snapshot = regions.sharedCopy();
@@ -256,7 +288,7 @@ public void testCopyCannotExtend() throws Exception
 
         try
         {
-            snapshot.extend(buffer.capacity());
+            snapshot.extend(buffer.capacity(), bufSize);
         }
         finally
         {
@@ -269,12 +301,13 @@ public void testCopyCannotExtend() throws Exception
     public void testExtendOutOfOrder() throws Exception
     {
         ByteBuffer buffer = allocateBuffer(4096);
-        try (ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer));
-             MmappedRegions regions = MmappedRegions.empty(channel))
+        int bufSize = 1024;
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
         {
-            regions.extend(4096);
-            regions.extend(1024);
-            regions.extend(2048);
+            regions.extend(4096, bufSize);
+            regions.extend(1024, bufSize);
+            regions.extend(2048, bufSize);
 
             for (int i = 0; i < buffer.capacity(); i++)
             {
@@ -290,10 +323,11 @@ public void testExtendOutOfOrder() throws Exception
     public void testNegativeExtend() throws Exception
     {
         ByteBuffer buffer = allocateBuffer(1024);
-        try (ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer));
-             MmappedRegions regions = MmappedRegions.empty(channel))
+        int bufSize = 1024;
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
         {
-            regions.extend(-1);
+            regions.extend(-1, bufSize);
         }
     }
 
@@ -320,7 +354,7 @@ public void testMapForCompressionMetadata() throws Exception
 
         CompressionMetadata metadata = CompressionMetadata.open(cf, f.length(), true);
         try (ChannelProxy channel = new ChannelProxy(f);
-             MmappedRegions regions = MmappedRegions.map(channel, metadata))
+             MmappedRegions regions = MmappedRegions.map(channel, metadata, 0, false))
         {
 
             assertFalse(regions.isEmpty());
@@ -341,8 +375,9 @@ public void testMapForCompressionMetadata() throws Exception
     public void testIllegalArgForMap1() throws Exception
     {
         ByteBuffer buffer = allocateBuffer(1024);
-        try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer));
-             MmappedRegions regions = MmappedRegions.map(channel, 0))
+        int bufSize = 1024;
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer));
+            MmappedRegions regions = MmappedRegions.map(channel, 0, bufSize, 0, false))
         {
             assertTrue(regions.isEmpty());
         }
@@ -352,8 +387,9 @@ public void testIllegalArgForMap1() throws Exception
     public void testIllegalArgForMap2() throws Exception
     {
         ByteBuffer buffer = allocateBuffer(1024);
-        try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer));
-             MmappedRegions regions = MmappedRegions.map(channel, -1L))
+        int bufSize = 1024;
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer));
+            MmappedRegions regions = MmappedRegions.map(channel, -1L, bufSize, 0, false))
         {
             assertTrue(regions.isEmpty());
         }
@@ -364,7 +400,7 @@ public void testIllegalArgForMap3() throws Exception
     {
         ByteBuffer buffer = allocateBuffer(1024);
         try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer));
-             MmappedRegions regions = MmappedRegions.map(channel, null))
+             MmappedRegions regions = MmappedRegions.map(channel, null, 0, false))
         {
             assertTrue(regions.isEmpty());
         }
@@ -373,7 +409,7 @@ public void testIllegalArgForMap3() throws Exception
     @Test
     public void testExtendForCompressionMetadata() throws Exception
     {
-        testExtendForCompressionMetadata(8, 4, 4, 8, 12);
+        // testExtendForCompressionMetadata(8, 4, 4, 8, 12);
         testExtendForCompressionMetadata(4, 4, 4, 8, 12);
         testExtendForCompressionMetadata(2, 4, 4, 8, 12);
     }
@@ -403,7 +439,7 @@ public void testExtendForCompressionMetadata(int maxSegmentSize, int chunkSize,
 
             try (ChannelProxy channel = new ChannelProxy(f);
                  CompressionMetadata metadata = writer.open(writer.getLastFlushOffset());
-                 MmappedRegions regions = MmappedRegions.map(channel, metadata))
+                 MmappedRegions regions = MmappedRegions.map(channel, metadata, 0, false))
             {
                 assertFalse(regions.isEmpty());
                 int dataOffset = 0;
@@ -423,7 +459,7 @@ public void testExtendForCompressionMetadata(int maxSegmentSize, int chunkSize,
                     writer.sync();
 
                     // verify that calling extend for the same (first iteration) or some previous metadata (further iterations) has no effect
-                    assertFalse(regions.extend(metadata));
+                    assertFalse(regions.extend(metadata, 1024, 0));
 
                     logger.info("Checking extend on compressed chunk for range={} {}..{} / {}", idx, pos, pos + (writeSizes[idx] << 10), size);
                     checkExtendOnCompressedChunks(f, writer, regions);
@@ -439,7 +475,7 @@ private void checkExtendOnCompressedChunks(File f, CompressedSequentialWriter wr
         int dataOffset;
         try (CompressionMetadata metadata = writer.open(writer.getLastFlushOffset()))
         {
-            regions.extend(metadata);
+            regions.extend(metadata, 1024, 0);
             assertFalse(regions.isEmpty());
             dataOffset = 0;
             while (dataOffset < metadata.dataLength)
diff --git a/test/unit/org/apache/cassandra/io/util/PathUtilsTest.java b/test/unit/org/apache/cassandra/io/util/PathUtilsTest.java
new file mode 100644
index 000000000000..485fbaeb3136
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/util/PathUtilsTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.List;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.io.FSWriteError;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+
+public class PathUtilsTest
+{
+    private static File classTestDir;
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE.setBoolean(false);
+        classTestDir = FileUtils.getTempDir().resolve("PathUtilsTest");
+        PathUtils.createDirectoryIfNotExists(classTestDir.toPath());
+        classTestDir.deleteRecursiveOnExit();
+    }
+
+    @Test
+    public void testDeleteContent()
+    {
+        File testDir = classTestDir.resolve("testDeleteContent");
+        assertTrue(PathUtils.createDirectoryIfNotExists(testDir.toPath()));
+
+        File file1 = testDir.resolve("file1");
+        assertTrue(PathUtils.createFileIfNotExists(file1.toPath()));
+
+        File subdir = testDir.resolve("subdir");
+        assertTrue(PathUtils.createDirectoryIfNotExists(subdir.toPath()));
+
+        File subdir_file2 = subdir.resolve("file2");
+        assertTrue(PathUtils.createFileIfNotExists(subdir_file2.toPath()));
+
+        List<Path> testDirContents = PathUtils.listPaths(testDir.toPath());
+        assertEquals(2, testDirContents.size());
+        assertTrue(testDirContents.contains(file1.toPath()));
+        assertTrue(testDirContents.contains(subdir.toPath()));
+
+        PathUtils.deleteContent(testDir.toPath());
+        assertTrue(testDir.exists());
+        assertTrue(PathUtils.listPaths(testDir.toPath()).isEmpty());
+    }
+
+    @Test
+    public void testDeleteQuietlyIgnoresIOExceptions() throws IOException
+    {
+        Path deletedPath = Mockito.mock(Path.class);
+        FileSystem fs = Mockito.mock(FileSystem.class);
+        FileSystemProvider fsp = Mockito.mock(FileSystemProvider.class);
+        BasicFileAttributes attributes = Mockito.mock(BasicFileAttributes.class);
+
+        Mockito.when(deletedPath.getFileSystem()).thenReturn(fs);
+        Mockito.when(fs.provider()).thenReturn(fsp);
+        Mockito.when(fsp.readAttributes(eq(deletedPath), eq(BasicFileAttributes.class), any())).thenReturn(attributes);
+        Mockito.when(attributes.isDirectory()).thenReturn(false);
+        Mockito.doThrow(new IOException("mock exception")).when(fsp).delete(deletedPath);
+
+        assertThrows(FSWriteError.class, () -> PathUtils.deleteRecursive(deletedPath));
+        PathUtils.deleteQuietly(deletedPath);
+    }
+
+    @Test
+    public void testDeleteQuietlyIsRecursive()
+    {
+        File testDir = classTestDir.resolve("testDeleteQuietlyIsRecursive");
+        assertTrue(PathUtils.createDirectoryIfNotExists(testDir.toPath()));
+
+        File file1 = testDir.resolve("file1");
+        assertTrue(PathUtils.createFileIfNotExists(file1.toPath()));
+
+        File subdir = testDir.resolve("subdir");
+        assertTrue(PathUtils.createDirectoryIfNotExists(subdir.toPath()));
+
+        File subdir_file2 = subdir.resolve("file2");
+        assertTrue(PathUtils.createFileIfNotExists(subdir_file2.toPath()));
+
+        List<Path> testDirContents = PathUtils.listPaths(testDir.toPath());
+        assertEquals(2, testDirContents.size());
+        assertTrue(testDirContents.contains(file1.toPath()));
+        assertTrue(testDirContents.contains(subdir.toPath()));
+
+        PathUtils.deleteQuietly(testDir.toPath());
+        assertTrue(PathUtils.listPaths(testDir.toPath()).isEmpty());
+        assertFalse(testDir.exists());
+    }
+
+    @Test
+    public void testListPaths()
+    {
+        File testDir = classTestDir.resolve("testListPaths");
+        assertTrue(PathUtils.createDirectoryIfNotExists(testDir.toPath()));
+        File file1 = testDir.resolve("file1");
+        assertTrue(PathUtils.createFileIfNotExists(file1.toPath()));
+
+        List<Path> testDirContents = PathUtils.listPaths(testDir.toPath());
+        assertNotNull(testDirContents);
+        assertTrue(testDirContents.size() >= 1);
+        assertTrue(testDirContents.contains(file1.toPath()));
+    }
+
+    @Test
+    public void testListPaths_NoSuchFile()
+    {
+        File testDir = classTestDir.resolve("testListPaths_NoSuchFile");
+        File doesNotExist = testDir.resolve("doesNotExist");
+        assertFalse(doesNotExist.exists());
+        assertTrue(PathUtils.listPaths(doesNotExist.toPath()).isEmpty());
+    }
+
+    @Test
+    public void testListPaths_NotDirectory()
+    {
+        File testDir = classTestDir.resolve("testListPaths_NotDirectory");
+        assertTrue(PathUtils.createDirectoryIfNotExists(testDir.toPath()));
+        File file1 = testDir.resolve("file1");
+        assertFalse(file1.exists());
+        assertTrue(PathUtils.createFileIfNotExists(file1.toPath()));
+        assertTrue(file1.exists());
+        assertTrue(PathUtils.listPaths(file1.toPath()).isEmpty());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
index 853564688f1c..2a49890994da 100644
--- a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
@@ -29,11 +29,14 @@
 import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Random;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -46,6 +49,7 @@
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -160,6 +164,353 @@ public void testVeryLarge() throws IOException
         }
     }
 
+    // readFully array tests - floats
+
+    private static final class FloatReadArrayCase
+    {
+        private final float[] expected;
+        static int counter;
+
+        FloatReadArrayCase(int numElements)
+        {
+            this.expected = new float[numElements];
+            for (int i = 0; i < numElements; i++)
+            {
+                this.expected[i] = counter++;
+            }
+        }
+    }
+
+    @Test
+    public void testReadFullyFloatArrayAligned() throws IOException
+    {
+        testReadFullyFloatArray(0, ByteOrder.BIG_ENDIAN);
+        testReadFullyFloatArray(0, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Test
+    public void testReadFullyFloatArrayNotAligned() throws IOException
+    {
+        testReadFullyFloatArray(1, ByteOrder.BIG_ENDIAN);
+        testReadFullyFloatArray(1, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Test
+    public void testReadFullyFloatArrayAligned2() throws IOException
+    {
+        testReadFullyFloatArray(Float.BYTES, ByteOrder.BIG_ENDIAN);
+        testReadFullyFloatArray(Float.BYTES, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    private void testReadFullyFloatArray(int shift, ByteOrder order) throws IOException
+    {
+        int bufferSize = 2048;
+
+        List<FloatReadArrayCase> cases = new ArrayList<>();
+        cases.add(new FloatReadArrayCase(0));
+        cases.add(new FloatReadArrayCase(10));
+        cases.add(new FloatReadArrayCase(17));
+        cases.add(new FloatReadArrayCase(100));
+        cases.add(new FloatReadArrayCase(121));
+        cases.add(new FloatReadArrayCase(1000));
+        cases.add(new FloatReadArrayCase(1000));
+        cases.add(new FloatReadArrayCase(2000));
+        cases.add(new FloatReadArrayCase(2000));
+
+        int bigArraySize = 1 + (bufferSize / Float.BYTES);
+        // ensure that in the test case we have a least one array that is bigger than the buffer size
+        cases.add(new FloatReadArrayCase(bigArraySize));
+        cases.add(new FloatReadArrayCase(bigArraySize));
+        cases.add(new FloatReadArrayCase(bigArraySize / 2));
+        cases.add(new FloatReadArrayCase(bigArraySize));
+        cases.add(new FloatReadArrayCase(bigArraySize));
+
+        File file = writeFile(writer -> {
+            try
+            {
+                writer.order(order);
+                // write some garbage in the beginning, in order to not have aligned reads
+                for (int i = 0; i < shift; i++)
+                    writer.writeByte(0);
+
+                for (FloatReadArrayCase array : cases)
+                {
+                    for (float f : array.expected)
+                        writer.writeFloat(f);
+                }
+                return false;
+            } catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        });
+
+
+        try (ChannelProxy channel = new ChannelProxy(file);
+             FileHandle fh = new FileHandle.Builder(channel.file())
+                             .order(order)
+                             .bufferType(BufferType.OFF_HEAP).bufferSize(bufferSize)
+                             .complete();
+             RandomAccessReader reader = fh.createReader())
+        {
+            assertEquals(channel.size(), reader.length());
+            assertEquals(channel.size(), reader.bytesRemaining());
+            assertEquals(file.length(), reader.available());
+
+            reader.seek(shift);
+
+            for (FloatReadArrayCase array : cases)
+            {
+                float[] readArray = new float[array.expected.length];
+                reader.readFully(readArray);
+                assertArrayEquals(array.expected, readArray, 0.0f);
+            }
+
+            assertTrue(reader.isEOF());
+            assertEquals(0, reader.bytesRemaining());
+        }
+    }
+
+    // readFully array tests - longs
+
+    private static final class LongReadArrayCase
+    {
+        private final long[] expected;
+        static int counter;
+
+        LongReadArrayCase(int numElements)
+        {
+            this.expected = new long[numElements];
+            for (int i = 0; i < numElements; i++)
+            {
+                this.expected[i] = counter++;
+            }
+        }
+    }
+
+    @Test
+    public void testReadFullyLongArrayAligned() throws IOException
+    {
+        testReadFullyLongArray(0, ByteOrder.BIG_ENDIAN);
+        testReadFullyLongArray(0, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Test
+    public void testReadFullyLongArrayNotAligned() throws IOException
+    {
+        testReadFullyLongArray(1, ByteOrder.BIG_ENDIAN);
+        testReadFullyLongArray(1, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Test
+    public void testReadFullyLongArrayAligned2() throws IOException
+    {
+        testReadFullyLongArray(Long.BYTES, ByteOrder.BIG_ENDIAN);
+        testReadFullyLongArray(Long.BYTES, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    private void testReadFullyLongArray(int shift, ByteOrder order) throws IOException
+    {
+        int bufferSize = 2048;
+
+        List<LongReadArrayCase> cases = new ArrayList<>();
+        cases.add(new LongReadArrayCase(0));
+        cases.add(new LongReadArrayCase(10));
+        cases.add(new LongReadArrayCase(17));
+        cases.add(new LongReadArrayCase(100));
+        cases.add(new LongReadArrayCase(121));
+        cases.add(new LongReadArrayCase(1000));
+        cases.add(new LongReadArrayCase(1000));
+        cases.add(new LongReadArrayCase(2000));
+        cases.add(new LongReadArrayCase(2000));
+
+        int bigArraySize = 1 + (bufferSize / Float.BYTES);
+        // ensure that in the test case we have a least one array that is bigger than the buffer size
+        cases.add(new LongReadArrayCase(bigArraySize));
+        cases.add(new LongReadArrayCase(bigArraySize));
+        cases.add(new LongReadArrayCase(bigArraySize / 2));
+        cases.add(new LongReadArrayCase(bigArraySize));
+        cases.add(new LongReadArrayCase(bigArraySize));
+
+        File file = writeFile(writer -> {
+            try
+            {
+                writer.order(order);
+                // write some garbage in the beginning, in order to not have aligned reads
+                for (int i = 0; i < shift; i++)
+                    writer.writeByte(0);
+
+                for (LongReadArrayCase array : cases)
+                {
+                    for (long f : array.expected)
+                        writer.writeLong(f);
+                }
+                return false;
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        });
+
+
+        try (ChannelProxy channel = new ChannelProxy(file);
+             FileHandle fh = new FileHandle.Builder(channel.file())
+                             .order(order)
+                             .bufferType(BufferType.OFF_HEAP).bufferSize(bufferSize)
+                             .complete();
+             RandomAccessReader reader = fh.createReader())
+        {
+            assertEquals(channel.size(), reader.length());
+            assertEquals(channel.size(), reader.bytesRemaining());
+            assertEquals(file.length(), reader.available());
+
+            reader.seek(shift);
+
+            for (LongReadArrayCase array : cases)
+            {
+                long[] readArray = new long[array.expected.length];
+                reader.readFully(readArray);
+                assertArrayEquals(array.expected, readArray);
+            }
+
+            assertTrue(reader.isEOF());
+            assertEquals(0, reader.bytesRemaining());
+        }
+    }
+
+
+    // readFully array tests - ints
+
+    private static final class IntReadArrayCase
+    {
+        private final int[] expected;
+        static int counter;
+
+        IntReadArrayCase(int numElements)
+        {
+            this.expected = new int[numElements];
+            for (int i = 0; i < numElements; i++)
+            {
+                this.expected[i] = counter++;
+            }
+        }
+    }
+
+    @Test
+    public void testReadFullyIntArrayAligned() throws IOException
+    {
+        testReadFullyIntArray(0, ByteOrder.BIG_ENDIAN);
+        testReadFullyIntArray(0, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Test
+    public void testReadFullyIntArrayNotAligned() throws IOException
+    {
+        testReadFullyIntArray(1, ByteOrder.BIG_ENDIAN);
+        testReadFullyIntArray(1, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    @Test
+    public void testReadFullyIntArrayAligned2() throws IOException
+    {
+        testReadFullyIntArray(Integer.BYTES, ByteOrder.BIG_ENDIAN);
+        testReadFullyIntArray(Integer.BYTES, ByteOrder.LITTLE_ENDIAN);
+    }
+
+    private void testReadFullyIntArray(int shift, ByteOrder order) throws IOException
+    {
+        int bufferSize = 2048;
+
+        List<IntReadArrayCase> cases = new ArrayList<>();
+        cases.add(new IntReadArrayCase(0));
+        cases.add(new IntReadArrayCase(10));
+        cases.add(new IntReadArrayCase(17));
+        cases.add(new IntReadArrayCase(100));
+        cases.add(new IntReadArrayCase(121));
+        cases.add(new IntReadArrayCase(1000));
+        cases.add(new IntReadArrayCase(1000));
+        cases.add(new IntReadArrayCase(2000));
+        cases.add(new IntReadArrayCase(2000));
+
+        int bigArraySize = 1 + (bufferSize / Integer.BYTES);
+        // ensure that in the test case we have a least one array that is bigger than the buffer size
+        cases.add(new IntReadArrayCase(bigArraySize));
+        cases.add(new IntReadArrayCase(bigArraySize));
+        cases.add(new IntReadArrayCase(bigArraySize / 2));
+        cases.add(new IntReadArrayCase(bigArraySize));
+        cases.add(new IntReadArrayCase(bigArraySize));
+
+        File file = writeFile(writer -> {
+            try
+            {
+                writer.order(order);
+                // write some garbage in the beginning, in order to not have aligned reads
+                for (int i = 0; i < shift; i++)
+                    writer.writeByte(0);
+
+                for (IntReadArrayCase array : cases)
+                {
+                    for (int f : array.expected)
+                        writer.writeInt(f);
+                }
+                return false;
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        });
+
+
+        try (ChannelProxy channel = new ChannelProxy(file);
+             FileHandle fh = new FileHandle.Builder(channel.file())
+                             .order(order)
+                             .bufferType(BufferType.OFF_HEAP).bufferSize(bufferSize)
+                             .complete();
+             RandomAccessReader reader = fh.createReader())
+        {
+            assertEquals(channel.size(), reader.length());
+            assertEquals(channel.size(), reader.bytesRemaining());
+            assertEquals(file.length(), reader.available());
+
+            reader.seek(shift);
+
+            for (IntReadArrayCase array : cases)
+            {
+                long position = reader.getPosition();
+
+                int[] readArray = new int[array.expected.length];
+                reader.read(readArray, 0, readArray.length);
+                assertArrayEquals(array.expected, readArray);
+
+                reader.seek(position);
+
+                int[] readArrayHalf = new int[array.expected.length / 2];
+                reader.read(readArrayHalf, 0, readArray.length / 2);
+                int[] expectedHalf = Arrays.copyOf(array.expected, array.expected.length / 2);
+                assertArrayEquals(expectedHalf, readArrayHalf);
+
+                if (array.expected.length > 0)
+                {
+                    System.out.println("expected.length: " + array.expected.length);
+                    // second half
+                    int halfStart = array.expected.length / 2;
+                    int secondHalfLength = array.expected.length - halfStart;
+                    // we allocate an array that is bigger, because we want to test read with offset > 0
+                    int[] readArraySecondHalfFromOffset = new int[array.expected.length];
+                    reader.read(readArraySecondHalfFromOffset, halfStart, secondHalfLength);
+                    int[] expectedSecondHalf = new int[array.expected.length];
+                    System.arraycopy(array.expected, halfStart, expectedSecondHalf, halfStart, secondHalfLength);
+                    assertArrayEquals(expectedSecondHalf, readArraySecondHalfFromOffset);
+                }
+            }
+
+            assertTrue(reader.isEOF());
+            assertEquals(0, reader.bytesRemaining());
+        }
+    }
+
     /** A fake file channel that simply increments the position and doesn't
      * actually read anything. We use it to simulate very large files, > 2G.
      */
@@ -268,23 +619,43 @@ protected void implCloseChannel()
 
     private static File writeFile(Parameters params) throws IOException
     {
-        final File f = FileUtils.createTempFile("testReadFully", "1");
-        f.deleteOnExit();
-
-        try(SequentialWriter writer = new SequentialWriter(f))
+        File f = writeFile(new Function<>()
         {
             long numWritten = 0;
-            while (numWritten < params.fileLength)
+            @Override
+            public Boolean apply(SequentialWriter writer)
             {
-                writer.write(params.expected);
+                if (numWritten >= params.fileLength) {
+                    return false;
+                }
+                try
+                {
+                    writer.write(params.expected);
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeException(e);
+                }
                 numWritten += params.expected.length;
+                return true;
             }
+        });
 
+        assert f.length() >= params.fileLength;
+        return f;
+    }
+
+    private static File writeFile(Function<SequentialWriter, Boolean> writeNextElement)
+    {
+        final File f = FileUtils.createTempFile("testReadFully", "1");
+        f.deleteOnExit();
+
+        try (SequentialWriter writer = new SequentialWriter(f))
+        {
+            while (writeNextElement.apply(writer)) ;
             writer.finish();
         }
-
         assert f.exists();
-        assert f.length() >= params.fileLength;
         return f;
     }
 
@@ -297,7 +668,7 @@ private static void testReadFully(Parameters params) throws IOException
         try (FileHandle fh = builder.complete();
              RandomAccessReader reader = fh.createReader())
         {
-            assertEquals(f.absolutePath(), reader.getPath());
+            assertEquals(f.absolutePath(), reader.getFile().path());
             assertEquals(f.length(), reader.length());
             assertEquals(f.length(), reader.bytesRemaining());
             assertEquals(Math.min(Integer.MAX_VALUE, f.length()), reader.available());
@@ -333,7 +704,7 @@ public void testReadBytes() throws IOException
         try (FileHandle fh = new FileHandle.Builder(f).complete();
              RandomAccessReader reader = fh.createReader())
         {
-            assertEquals(f.absolutePath(), reader.getPath());
+            assertEquals(f.absolutePath(), reader.getFile().path());
             assertEquals(expected.length(), reader.length());
 
             ByteBuffer b = ByteBufferUtil.read(reader, expected.length());
@@ -510,6 +881,7 @@ public void testSkipBytesGreaterThanBufferSize() throws IOException
         testSkipBytes(params, numberOfExpectationsInBufferSize + 1);
     }
 
+    @Test
     public void testSkipBytesNonPositive() throws IOException
     {
         Parameters params = new Parameters(8192, 4096);
diff --git a/test/unit/org/apache/cassandra/io/util/SliceDescriptorTest.java b/test/unit/org/apache/cassandra/io/util/SliceDescriptorTest.java
new file mode 100644
index 000000000000..34d0ee05dfb2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/util/SliceDescriptorTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SliceDescriptorTest
+{
+    @Test
+    public void testEmptySliceDescriptor()
+    {
+        SliceDescriptor sliceDescriptor = SliceDescriptor.NONE;
+        assertThat(sliceDescriptor.exists()).isFalse();
+        assertThat(sliceDescriptor.dataStart).isEqualTo(0);
+        assertThat(sliceDescriptor.dataEnd).isEqualTo(0);
+        assertThat(sliceDescriptor.chunkSize).isEqualTo(0);
+        assertThat(sliceDescriptor.sliceStart).isEqualTo(0);
+        assertThat(sliceDescriptor.sliceEnd).isEqualTo(0);
+        assertThat(sliceDescriptor.dataEndOr(1234)).isEqualTo(1234);
+    }
+
+    @Test
+    public void testSliceDescriptor()
+    {
+        SliceDescriptor sliceDescriptor = new SliceDescriptor(37, 87, 16);
+        assertThat(sliceDescriptor.exists()).isTrue();
+        assertThat(sliceDescriptor.dataStart).isEqualTo(37);
+        assertThat(sliceDescriptor.dataEnd).isEqualTo(87);
+        assertThat(sliceDescriptor.chunkSize).isEqualTo(16);
+        assertThat(sliceDescriptor.sliceStart).isEqualTo(32);
+        assertThat(sliceDescriptor.sliceEnd).isEqualTo(96);
+        assertThat(sliceDescriptor.dataEndOr(1234)).isEqualTo(87);
+
+        sliceDescriptor = new SliceDescriptor(37, 96, 16);
+        assertThat(sliceDescriptor.sliceEnd).isEqualTo(96);
+
+        sliceDescriptor = new SliceDescriptor(37, 95, 16);
+        assertThat(sliceDescriptor.sliceEnd).isEqualTo(96);
+
+        sliceDescriptor = new SliceDescriptor(37, 97, 16);
+        assertThat(sliceDescriptor.sliceEnd).isEqualTo(112);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java b/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java
index 0d448cde5f2b..4a3ef8cf3d9d 100644
--- a/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java
+++ b/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java
@@ -138,16 +138,6 @@ public void release()
             released = true;
         }
 
-        public long adjustExternal(long position)
-        {
-            return position;
-        }
-
-        public long adjustInternal(long position)
-        {
-            return position;
-        }
-
         public void close()
         {
             // nothing
@@ -158,4 +148,4 @@ public void closeReader()
             // nothing
         }
     }
-}
\ No newline at end of file
+}
diff --git a/test/unit/org/apache/cassandra/io/util/WriteAndReadTest.java b/test/unit/org/apache/cassandra/io/util/WriteAndReadTest.java
new file mode 100644
index 000000000000..63c7e07854fc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/util/WriteAndReadTest.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.lifecycle.AbstractLogTransaction;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IKeyFetcher;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.sstable.format.bti.BtiTableReader;
+import org.apache.cassandra.io.sstable.format.bti.PartitionIndex;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_IO_TMPDIR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class WriteAndReadTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(WriteAndReadTest.class);
+
+    @BeforeClass
+    public static void setupDD()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void testPartitionIndexFailure() throws IOException, InterruptedException
+    {
+        for (int i = 4001; i < 4200; ++i)
+            testPartitionIndexFailure(i);
+    }
+
+    static class MockSSTableReader extends SSTableReader
+    {
+
+        protected MockSSTableReader(Builder<?, ?> builder, Owner owner)
+        {
+            super(builder, owner);
+        }
+
+        @Override
+        public SSTableReader cloneWithRestoredStart(DecoratedKey restoredStart)
+        {
+            return null;
+        }
+
+        @Override
+        public SSTableReader cloneWithNewStart(DecoratedKey newStart)
+        {
+            return null;
+        }
+
+        @Override
+        public void releaseInMemoryComponents()
+        {
+
+        }
+
+        @Override
+        public long estimatedKeys()
+        {
+            return 0;
+        }
+
+        @Override
+        public boolean mayContainAssumingKeyIsInRange(DecoratedKey key)
+        {
+            return false;
+        }
+
+        @Override
+        public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
+        {
+            return 0;
+        }
+
+        @Override
+        public boolean isEstimationInformative()
+        {
+            return false;
+        }
+
+        @Override
+        public Iterable<DecoratedKey> getKeySamples(Range<Token> range)
+        {
+            return null;
+        }
+
+        @Override
+        protected AbstractRowIndexEntry getRowIndexEntry(PartitionPosition key, Operator op, boolean updateStats, SSTableReadsListener listener)
+        {
+            return null;
+        }
+
+        @Override
+        public KeyReader keyReader() throws IOException
+        {
+            return null;
+        }
+
+        @Override
+        public DecoratedKey firstKeyBeyond(PartitionPosition token)
+        {
+            return null;
+        }
+
+        @Override
+        public IKeyFetcher openKeyFetcher(boolean isForSASI)
+        {
+            return null;
+        }
+
+        @Override
+        public IVerifier getVerifier(ColumnFamilyStore cfs, OutputHandler outputHandler, boolean isOffline, IVerifier.Options options)
+        {
+            return null;
+        }
+
+        @Override
+        public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter columnFilter, boolean reversed, SSTableReadsListener listener)
+        {
+            return null;
+        }
+
+        @Override
+        public UnfilteredPartitionIterator partitionIterator(ColumnFilter columnFilter, DataRange dataRange, SSTableReadsListener listener)
+        {
+            return null;
+        }
+    }
+
+    // This tests failure on restore (DB-2489/DSP-17193) caused by chunk cache retaining
+    // data from a previous version of a file with the same name.
+    public void testPartitionIndexFailure(int length) throws IOException, InterruptedException
+    {
+        System.out.println("Prefix " + length);
+
+        File parentDir = new File(JAVA_IO_TMPDIR.getString());
+        Descriptor descriptor = new Descriptor(parentDir, "ks", "cf" + length, new SequenceBasedSSTableId(1));
+        FileHandle.Builder indexFhBuilder = new FileHandle.Builder(descriptor.fileFor(BtiFormat.Components.PARTITION_INDEX))
+                                            .withChunkCache(ChunkCache.instance)
+                                            .bufferSize(PageAware.PAGE_SIZE);
+        FileHandle.Builder dataFhBuilder = new FileHandle.Builder(descriptor.fileFor(SSTableFormat.Components.DATA))
+                                           .withChunkCache(ChunkCache.instance)
+                                           .bufferSize(PageAware.PAGE_SIZE);
+        long root = length;
+        long keyCount = root * length;
+        long firstPos = keyCount * length;
+
+
+        try (SequentialWriter writer = new SequentialWriter(descriptor.fileFor(BtiFormat.Components.PARTITION_INDEX),
+                SequentialWriterOption.newBuilder()
+                        .trickleFsync(DatabaseDescriptor.getTrickleFsync())
+                        .trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024)
+                        .bufferType(BufferType.OFF_HEAP)
+                        .finishOnClose(true)
+                        .build()))
+
+        {
+            int i;
+            for (i = 0; i + 8 <= length; i += 8)
+                writer.writeLong(i);
+            for (; i < length; i++)
+                writer.write(i);
+
+            // Do the final writes just like PartitionIndexWriter.complete
+            writer.writeLong(firstPos);
+            writer.writeLong(keyCount);
+            writer.writeLong(root);
+        }
+
+        try (SequentialWriter writer = new SequentialWriter(descriptor.fileFor(SSTableFormat.Components.DATA),
+                SequentialWriterOption.newBuilder()
+                        .trickleFsync(DatabaseDescriptor.getTrickleFsync())
+                                      .trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024)
+                                      .bufferType(BufferType.OFF_HEAP)
+                                      .finishOnClose(true)
+                                      .build()))
+
+        {
+            writer.writeLong(length);
+        }
+
+        SSTableReader reader = null;
+        // Now read it like PartitionIndex.load
+        try (FileHandle ifh = indexFhBuilder.complete();
+             FileHandle dfh = dataFhBuilder.complete())
+        {
+            BtiTableReader.Builder builder = new BtiTableReader.Builder(descriptor)
+                                             .setComponents(Collections.singleton(BtiFormat.Components.PARTITION_INDEX))
+                                             .setTableMetadataRef(TableMetadataRef.forOfflineTools(TableMetadata.minimal(descriptor.ksname, descriptor.cfname)))
+                                             .setDataFile(dfh)
+                                             .setPartitionIndex(new PartitionIndex(ifh, 0, 0, MockSchema.readerBounds(0), MockSchema.readerBounds(0), ByteComparable.Version.OSS50));
+            reader = new MockSSTableReader(builder, null);
+            reader.setup(false);
+
+            try (FileDataInput index = ifh.createReader(ifh.dataLength() - 3 * 8))
+            {
+                long firstPosR = index.readLong();
+                long keyCountR = index.readLong();
+                long rootR = index.readLong();
+
+                assertEquals(firstPos, firstPosR);
+                assertEquals(keyCount, keyCountR);
+                assertEquals(rootR, root);
+            }
+            try (FileDataInput data = dfh.createReader())
+            {
+                long lengthR = data.readLong();
+                assertEquals(length, lengthR);
+            }
+        }
+        finally
+        {
+            if (reader != null)
+            {
+                releaseReader(reader);
+                CountDownLatch latch = new CountDownLatch(1);
+                ScheduledExecutors.nonPeriodicTasks.execute(latch::countDown);
+                latch.await();  // wait for the release to complete
+                assertFalse(reader.getDataFile().exists());
+            }
+        }
+    }
+
+    private static void releaseReader(SSTableReader reader)
+    {
+        reader.markObsolete(new AbstractLogTransaction.ReaderTidier() {
+            public void commit()
+            {
+                for (Component component : reader.descriptor.discoverComponents())
+                    FileUtils.deleteWithConfirm(reader.descriptor.fileFor(component));
+            }
+
+            public Throwable abort(Throwable accumulate)
+            {
+                return null;
+            }
+        });
+        reader.selfRef().release();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java
index 5b086c234500..4eafb28a2fb0 100644
--- a/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java
@@ -25,9 +25,9 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.Gossiper;
@@ -36,8 +36,6 @@
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Pair;
 
-import static org.apache.cassandra.ServerTestUtils.cleanup;
-import static org.apache.cassandra.ServerTestUtils.mkdirs;
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
 import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY;
 import static org.apache.cassandra.locator.AlibabaCloudSnitch.DEFAULT_METADATA_SERVICE_URL;
@@ -55,10 +53,7 @@ public static void setup() throws Exception
     {
         GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
-        CommitLog.instance.start();
-        CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
-        mkdirs();
-        cleanup();
+        ServerTestUtils.cleanupAndLeaveDirs();
         Keyspace.setInitialized();
         StorageService.instance.initServer(0);
     }
diff --git a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
index 2c72c674486b..0b53e574d0d2 100644
--- a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
@@ -26,9 +26,9 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.Gossiper;
@@ -37,8 +37,6 @@
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Pair;
 
-import static org.apache.cassandra.ServerTestUtils.cleanup;
-import static org.apache.cassandra.ServerTestUtils.mkdirs;
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
 import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY;
 import static org.junit.Assert.assertEquals;
@@ -55,10 +53,7 @@ public static void setup() throws Exception
     {
         GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
-        CommitLog.instance.start();
-        CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
-        mkdirs();
-        cleanup();
+        ServerTestUtils.cleanupAndLeaveDirs();
         Keyspace.setInitialized();
         StorageService.instance.initServer(0);
     }
diff --git a/test/unit/org/apache/cassandra/locator/CustomTokenMetadataProviderTest.java b/test/unit/org/apache/cassandra/locator/CustomTokenMetadataProviderTest.java
new file mode 100644
index 000000000000..c788a5be2ad1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/CustomTokenMetadataProviderTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import org.junit.Test;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_TMD_PROVIDER_PROPERTY;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+public class CustomTokenMetadataProviderTest
+{
+    static String oldValueCustomProvider = null;
+
+    @BeforeClass
+    public static void setProperty()
+    {
+        oldValueCustomProvider = CUSTOM_TMD_PROVIDER_PROPERTY.getString();
+        CUSTOM_TMD_PROVIDER_PROPERTY.setString(TestTokenMetadataProvider.class.getName());
+    }
+
+    @AfterClass
+    public static void resetProperty()
+    {
+        if (oldValueCustomProvider != null)
+            CUSTOM_TMD_PROVIDER_PROPERTY.setString(oldValueCustomProvider);
+        else
+            System.clearProperty(CUSTOM_TMD_PROVIDER_PROPERTY.getKey());
+    }
+
+    @Test
+    public void testCustomTokenMetadataProperty()
+    {
+        assertTrue("TokenMetadataProvider has unexpected instance class",
+                   TokenMetadataProvider.instance instanceof TestTokenMetadataProvider);
+    }
+
+    public static class TestTokenMetadataProvider implements TokenMetadataProvider
+    {
+        @Override
+        public TokenMetadata getTokenMetadata()
+        {
+            return null;
+        }
+
+        @Override
+        public TokenMetadata getTokenMetadataForKeyspace(String keyspace)
+        {
+            return null;
+        }
+
+        @Override
+        /** @deprecated See STAR-1032 */
+        @Deprecated(forRemoval = true, since = "CC 4.0") // since we can select TMDP implementation via config, this method is no longer needed
+        public void replaceTokenMetadata(TokenMetadata newTokenMetadata)
+        {
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java
index 98a9d16518f2..32e421383c18 100644
--- a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java
@@ -19,8 +19,12 @@
 package org.apache.cassandra.locator;
 
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.*;
+import java.util.concurrent.ThreadLocalRandom;
 
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -30,7 +34,9 @@
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE;
 
 public class DynamicEndpointSnitchTest
 {
@@ -38,7 +44,11 @@ public class DynamicEndpointSnitchTest
     @BeforeClass
     public static void setupDD()
     {
+        NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setDynamicBadnessThreshold(0.1);
+        DatabaseDescriptor.setDynamicUpdateInterval(1000000);
+        StorageService.instance.unsafeInitialize();
     }
 
     private static void setScores(DynamicEndpointSnitch dsnitch, int rounds, List<InetAddressAndPort> hosts, Integer... scores) throws InterruptedException
@@ -48,7 +58,7 @@ private static void setScores(DynamicEndpointSnitch dsnitch, int rounds, List<In
             for (int i = 0; i < hosts.size(); i++)
                 dsnitch.receiveTiming(hosts.get(i), scores[i], MILLISECONDS);
         }
-        Thread.sleep(150);
+        dsnitch.updateScores();
     }
 
     private static EndpointsForRange full(InetAddressAndPort... endpoints)
@@ -61,6 +71,20 @@ private static EndpointsForRange full(InetAddressAndPort... endpoints)
         return rlist.build();
     }
 
+    @Test
+    public void testDynamicSnitchSetSeverity()
+    {
+        // do this because SS needs to be initialized before DES can work properly.
+        SimpleSnitch ss = new SimpleSnitch();
+
+        DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
+        double origSeverity = dsnitch.getSeverity();
+        double updSeverity = origSeverity + 1.0d;
+        dsnitch.setSeverity(updSeverity);
+        Assert.assertEquals(updSeverity, dsnitch.getSeverity(), 0.0d);
+        dsnitch.setSeverity(origSeverity);
+    }
+
     @Test
     public void testSnitch() throws InterruptedException, IOException, ConfigurationException
     {
@@ -116,4 +140,154 @@ public void testSnitch() throws InterruptedException, IOException, Configuration
         order = full(host4, host1, host2, host3);
         Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3, host4)));
     }
+
+    @Test
+    public void testDynamicSnitchSetQuantizationToMillis()
+    {
+        // do this because SS needs to be initialized before DES can work properly.
+        SimpleSnitch ss = new SimpleSnitch();
+
+        DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
+        boolean originalQuantization = dsnitch.getQuantizationToMillis();
+        dsnitch.setQuantizationToMillis(!originalQuantization);
+        Assert.assertEquals(!originalQuantization, dsnitch.getQuantizationToMillis());
+    }
+
+    @Test
+    public void testDynamicSnitchSetQuantile()
+    {
+        // do this because SS needs to be initialized before DES can work properly.
+        SimpleSnitch ss = new SimpleSnitch();
+
+        DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
+        double quantile = ThreadLocalRandom.current().nextDouble();
+        dsnitch.setQuantile(quantile);
+        Assert.assertEquals(quantile, dsnitch.getQuantile(), 0.01);
+    }
+
+    @Test
+    public void testDynamicSnitchQuantilePicking() throws UnknownHostException
+    {
+        // do this because SS needs to be initialized before DES can work properly.
+        SimpleSnitch ss = new SimpleSnitch();
+
+        DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
+        dsnitch.setQuantile(0.9);
+
+        // add a slow replica, that always reports 100ms
+        for (int i = 0; i < 100; i++)
+        {
+            dsnitch.receiveTiming(InetAddressAndPort.getByName("127.0.0.1"), 100, MILLISECONDS);
+        }
+        // add a replica that has p90 = 90ms
+        for (int i = 1; i <= 100; i++)
+        {
+            dsnitch.receiveTiming(InetAddressAndPort.getByName("127.0.0.2"), i, MILLISECONDS);
+        }
+
+        dsnitch.updateScores();
+
+        // the score for the slow replica should be 1, and the score for the fast replica should be 9/10 = 0.9
+        Map<InetAddress, Double> scores = dsnitch.getScores();
+
+        Assert.assertEquals(1.0, scores.get(InetAddress.getByName("127.0.0.1")), 0.01);
+        Assert.assertEquals(0.9, scores.get(InetAddress.getByName("127.0.0.2")), 0.01);
+    }
+
+    @Test
+    public void testQuantizationToMillisEnabled() throws UnknownHostException
+    {
+        // do this because SS needs to be initialized before DES can work properly.
+        SimpleSnitch ss = new SimpleSnitch();
+
+        DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
+
+        dsnitch.setQuantizationToMillis(true);
+        // add a slow replica, that always reports 1999us
+        for (int i = 0; i < 100; i++)
+        {
+            dsnitch.receiveTiming(InetAddressAndPort.getByName("127.0.0.1"), 1999, MICROSECONDS);
+        }
+        // add a fast replica, that always reports 1001us
+        for (int i = 0; i < 100; i++)
+        {
+            dsnitch.receiveTiming(InetAddressAndPort.getByName("127.0.0.2"), 1001, MICROSECONDS);
+        }
+
+        dsnitch.updateScores();
+
+        // the score for both replicas should be 1 because the quantization should round to 1ms
+        Map<InetAddress, Double> scores = dsnitch.getScores();
+
+        Assert.assertEquals(1.0, scores.get(InetAddress.getByName("127.0.0.1")), 0.01);
+        Assert.assertEquals(1.0, scores.get(InetAddress.getByName("127.0.0.2")), 0.01);
+    }
+
+    @Test
+    public void testQuantizationToMillisDisabled() throws UnknownHostException
+    {
+        // do this because SS needs to be initialized before DES can work properly.
+        SimpleSnitch ss = new SimpleSnitch();
+
+        DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
+
+        dsnitch.setQuantizationToMillis(false);
+        dsnitch.setQuantile(0.9);
+        // add a slow replica, that always reports 100us
+        for (int i = 0; i < 100; i++)
+        {
+            dsnitch.receiveTiming(InetAddressAndPort.getByName("127.0.0.1"), 100, MICROSECONDS);
+        }
+        // add a fast replica, that always reports 10us
+        for (int i = 0; i < 100; i++)
+        {
+            dsnitch.receiveTiming(InetAddressAndPort.getByName("127.0.0.2"), 10, MICROSECONDS);
+        }
+
+        dsnitch.updateScores();
+
+        // the score for the slow replica should be 1, and the score for the fast replica should be 10/100 = 0.1
+        // there should be no quantization rounding to 1ms
+        Map<InetAddress, Double> scores = dsnitch.getScores();
+
+        Assert.assertEquals(1.0, scores.get(InetAddress.getByName("127.0.0.1")), 0.01);
+        Assert.assertEquals(0.1, scores.get(InetAddress.getByName("127.0.0.2")), 0.01);
+    }
+
+    @Test
+    public void testScoresAreUpdatedPeriodically() throws UnknownHostException, InterruptedException
+    {
+        // do this because SS needs to be initialized before DES can work properly.
+        SimpleSnitch ss = new SimpleSnitch();
+
+        int updateInterval = DatabaseDescriptor.getDynamicUpdateInterval();
+        try
+        {
+            DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
+
+            // add a slow replica, that always reports 100ms
+            for (int i = 0; i < 100; i++)
+            {
+                dsnitch.receiveTiming(InetAddressAndPort.getByName("127.0.0.1"), 100, MILLISECONDS);
+            }
+            // add a fast replica, that always reports 1ms
+            for (int i = 0; i < 100; i++)
+            {
+                dsnitch.receiveTiming(InetAddressAndPort.getByName("127.0.0.2"), 1, MILLISECONDS);
+            }
+
+            Assert.assertTrue(dsnitch.getScores().isEmpty());
+
+            DatabaseDescriptor.setDynamicUpdateInterval(100);
+            dsnitch.applyConfigChanges();
+
+            Thread.sleep(150);
+
+            Assert.assertFalse(dsnitch.getScores().isEmpty());
+        }
+        finally
+        {
+            DatabaseDescriptor.setDynamicUpdateInterval(updateInterval);
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/locator/Ec2SnitchTest.java b/test/unit/org/apache/cassandra/locator/Ec2SnitchTest.java
index 09e37221d3a5..ec05a355538c 100644
--- a/test/unit/org/apache/cassandra/locator/Ec2SnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/Ec2SnitchTest.java
@@ -28,17 +28,15 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.service.StorageService;
 import org.mockito.stubbing.Answer;
 
-import static org.apache.cassandra.ServerTestUtils.cleanup;
-import static org.apache.cassandra.ServerTestUtils.mkdirs;
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
 import static org.apache.cassandra.locator.Ec2MultiRegionSnitch.PRIVATE_IP_QUERY;
 import static org.apache.cassandra.locator.Ec2MultiRegionSnitch.PUBLIC_IP_QUERY;
@@ -68,10 +66,7 @@ public static void setup() throws Exception
     {
         GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
-        CommitLog.instance.start();
-        CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
-        mkdirs();
-        cleanup();
+        ServerTestUtils.cleanupAndLeaveDirs();
         Keyspace.setInitialized();
         StorageService.instance.initServer(0);
     }
diff --git a/test/unit/org/apache/cassandra/locator/EverywhereStrategyTest.java b/test/unit/org/apache/cassandra/locator/EverywhereStrategyTest.java
new file mode 100644
index 000000000000..eeebb5b28dab
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/EverywhereStrategyTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import com.google.common.collect.Sets;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class EverywhereStrategyTest
+{
+    private final Random random = new Random();
+
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void allRingMembersAreReplicas() throws Throwable
+    {
+        TokenMetadata metadata = new TokenMetadata();
+
+        populateTokenMetadata(3, 1, metadata);
+
+        EverywhereStrategy strategy = createStrategy(metadata);
+
+        assertAllNodesCoverFullRing(strategy, metadata);
+    }
+
+    @Test
+    public void allRingMembersAreReplicasWithvnodes() throws Throwable
+    {
+        TokenMetadata metadata = new TokenMetadata();
+        populateTokenMetadata(5, 8, metadata);
+
+        EverywhereStrategy strategy = createStrategy(metadata);
+
+        assertAllNodesCoverFullRing(strategy, metadata);
+    }
+
+    @Test
+    public void bootstrappingNodesAreNotIncludedAsReplicas() throws Throwable
+    {
+        TokenMetadata metadata = new TokenMetadata();
+
+        populateTokenMetadata(3, 1, metadata);
+
+        metadata.addBootstrapTokens(Arrays.asList(getRandomToken()),
+                                    InetAddressAndPort.getByName("127.0.0.4"));
+
+        EverywhereStrategy strategy = createStrategy(metadata);
+
+        assertAllNodesCoverFullRing(strategy, metadata);
+    }
+
+    @Test
+    public void leavingNodesDoNotAddPendingRanges() throws Throwable
+    {
+        TokenMetadata metadata = new TokenMetadata();
+
+        populateTokenMetadata(3, 1, metadata);
+        InetAddressAndPort leavingEndpoint = metadata.getAllRingMembers().iterator().next();
+        metadata.addLeavingEndpoint(leavingEndpoint);
+
+        EverywhereStrategy strategy = createStrategy(metadata);
+
+        metadata.calculatePendingRanges(strategy, strategy.keyspaceName);
+        PendingRangeMaps pendingRanges = metadata.getPendingRanges(strategy.keyspaceName);
+
+        assertFalse("pending ranges must be empty",
+                    pendingRanges.iterator().hasNext());
+    }
+
+    @Test
+    public void bootstrapNodesNeedFullRingOnPendingRangesCalculation() throws Throwable
+    {
+        TokenMetadata metadata = new TokenMetadata();
+
+        populateTokenMetadata(3, 1, metadata);
+
+        EverywhereStrategy strategy = createStrategy(metadata);
+
+        InetAddressAndPort bootstrapNode = InetAddressAndPort.getByName("127.0.0.4");
+        metadata.addBootstrapTokens(Arrays.asList(getRandomToken()), bootstrapNode);
+
+        metadata.calculatePendingRanges(strategy, strategy.keyspaceName);
+        PendingRangeMaps pendingRangeMaps = metadata.getPendingRanges(strategy.keyspaceName);
+
+        List<Range<Token>> pendingRanges = new ArrayList<>();
+        for (Map.Entry<Range<Token>, EndpointsForRange.Builder> pendingRangeEntry : pendingRangeMaps)
+        {
+            EndpointsForRange.Builder pendingNodes = pendingRangeEntry.getValue();
+            // only the bootstrap node has pending ranges
+            assertEquals(1, pendingNodes.size());
+            assertTrue(pendingNodes.endpoints().contains(bootstrapNode));
+            pendingRanges.add(pendingRangeEntry.getKey());
+        }
+
+        List<Range<Token>> normalizedRanges = Range.normalize(pendingRanges);
+        assertEquals(1, normalizedRanges.size());
+        Range<Token> tokenRange = normalizedRanges.get(0);
+        // it must cover all ranges
+        assertEquals(tokenRange.left, tokenRange.right);
+    }
+
+    @Test
+    public void allRingMembersContributeToReplicationFactor() throws Throwable
+    {
+        TokenMetadata metadata = new TokenMetadata();
+        populateTokenMetadata(10, 5, metadata);
+
+        EverywhereStrategy strategy = createStrategy(metadata);
+
+        assertEquals(10, strategy.getReplicationFactor().fullReplicas);
+        assertEquals(10, strategy.getReplicationFactor().allReplicas);
+    }
+
+    @Test
+    public void noRecognizedOptions() throws Throwable
+    {
+        TokenMetadata metadata = new TokenMetadata();
+        populateTokenMetadata(10, 5, metadata);
+
+        EverywhereStrategy strategy = createStrategy(metadata);
+
+        assertTrue("EverywhereStrategy should have no options", strategy.recognizedOptions().isEmpty());
+    }
+
+    private EverywhereStrategy createStrategy(TokenMetadata tokenMetadata)
+    {
+        IEndpointSnitch snitch = new PropertyFileSnitch();
+        DatabaseDescriptor.setEndpointSnitch(snitch);
+
+        return new EverywhereStrategy("keyspace", tokenMetadata, snitch, Collections.emptyMap());
+    }
+
+    private void populateTokenMetadata(int nodeCount, int tokens, TokenMetadata metadata) throws UnknownHostException
+    {
+        List<InetAddressAndPort> nodes = new ArrayList<>();
+        for (int i = 1; i <= nodeCount; i++)
+        {
+            InetAddress byName = InetAddress.getByName(String.format("127.0.0.%d", i));
+            InetAddressAndPort inetAddressAndPort = InetAddressAndPort.getByAddress(byName);
+            nodes.add(inetAddressAndPort);
+        }
+
+        for (int i = 0; i < tokens; i++)
+        {
+            for (InetAddressAndPort node : nodes)
+            {
+                Token randomToken = getRandomToken();
+                metadata.updateNormalToken(randomToken, node);
+            }
+        }
+    }
+
+    private void assertAllNodesCoverFullRing(AbstractReplicationStrategy strategy, TokenMetadata metadata)
+    {
+        for (Token ringToken : metadata.sortedTokens())
+        {
+            EndpointsForRange endpointsForRange = strategy.calculateNaturalReplicas(ringToken, metadata);
+            assertEquals(metadata.getAllRingMembers().size(), endpointsForRange.size());
+            assertEquals(Sets.newHashSet(metadata.getAllRingMembers()), Sets.newHashSet(endpointsForRange.endpoints()));
+        }
+    }
+
+    private Token getRandomToken()
+    {
+        return Murmur3Partitioner.instance.getRandomToken(random);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
index 9dffbac2d1a4..229d73c340e2 100644
--- a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
@@ -27,9 +27,9 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.Gossiper;
@@ -38,8 +38,6 @@
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Pair;
 
-import static org.apache.cassandra.ServerTestUtils.cleanup;
-import static org.apache.cassandra.ServerTestUtils.mkdirs;
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
 import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY;
 import static org.apache.cassandra.locator.AlibabaCloudSnitch.DEFAULT_METADATA_SERVICE_URL;
@@ -56,10 +54,7 @@ public static void setup() throws Exception
     {
         GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
-        CommitLog.instance.start();
-        CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
-        mkdirs();
-        cleanup();
+        ServerTestUtils.cleanupAndLeaveDirs();
         Keyspace.setInitialized();
         StorageService.instance.initServer(0);
     }
diff --git a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java
index 81d6694c723f..0e38d29b2464 100644
--- a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java
+++ b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java
@@ -118,6 +118,57 @@ public void testPropertiesWithEmptyDC() throws IOException, ConfigurationExcepti
         assert 6 == new HashSet<>(replicas.byEndpoint().values()).size(); // ensure uniqueness
     }
 
+    @Test
+    public void testAcceptsNodesFromSameRack() throws IOException, ConfigurationException
+    {
+        IEndpointSnitch snitch = new RackInferringSnitch();
+        DatabaseDescriptor.setEndpointSnitch(snitch);
+        TokenMetadata metadata = new TokenMetadata();
+
+        tokenFactory(metadata, "123", new byte[]{ 10, 0, 0, 10 });
+        tokenFactory(metadata, "234", new byte[]{ 10, 0, 0, 11 });
+        tokenFactory(metadata, "345", new byte[]{ 10, 0, 1, 12 });
+
+        Map<String, String> configOptions = new HashMap<>();
+        configOptions.put("0", "3");
+
+        NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(KEYSPACE, metadata, snitch, configOptions);
+        Assert.assertEquals(3, strategy.getReplicationFactor("0").allReplicas);
+
+        EndpointsForToken endpoints = strategy.getNaturalReplicasForToken(new StringToken("123"));
+        Assert.assertEquals(3, endpoints.size());
+    }
+
+    @Test
+    public void testDoNotAcceptNodesFromSameRackIfQuorumExists() throws IOException, ConfigurationException
+    {
+        IEndpointSnitch snitch = new RackInferringSnitch()
+        {
+            @Override
+            public boolean acceptsNodesFromSameRack(int rf, int rackCount)
+            {
+                int quorum = rf / 2 + 1;
+                return rackCount < quorum;
+            }
+        };
+        DatabaseDescriptor.setEndpointSnitch(snitch);
+        TokenMetadata metadata = new TokenMetadata();
+
+        tokenFactory(metadata, "123", new byte[]{ 10, 0, 0, 10 });
+        tokenFactory(metadata, "234", new byte[]{ 10, 0, 0, 11 });
+        tokenFactory(metadata, "345", new byte[]{ 10, 0, 1, 12 });
+
+        Map<String, String> configOptions = new HashMap<>();
+        configOptions.put("0", "3");
+
+        NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(KEYSPACE, metadata, snitch, configOptions);
+        Assert.assertEquals(3, strategy.getReplicationFactor("0").allReplicas);
+
+        EndpointsForToken endpoints = strategy.getNaturalReplicasForToken(new StringToken("123"));
+        Assert.assertEquals(2, endpoints.size());
+        Assert.assertNotEquals(snitch.getRack(endpoints.get(0).endpoint()), snitch.getRack(endpoints.get(1).endpoint()));
+    }
+
     @Test
     public void testLargeCluster() throws UnknownHostException, ConfigurationException
     {
diff --git a/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java b/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java
index f013ade3bdd1..2dc4b27050f0 100644
--- a/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java
+++ b/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java
@@ -22,6 +22,7 @@
 
 import java.net.UnknownHostException;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.junit.Test;
 
 import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
@@ -35,6 +36,11 @@
 
 public class PendingRangeMapsTest
 {
+    static
+    {
+        DatabaseDescriptor.clientInitialization();
+    }
+
     private Range<Token> genRange(String left, String right)
     {
         return new Range<>(new BigIntegerToken(left), new BigIntegerToken(right));
diff --git a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
index af9e9704cfed..1576fb53e203 100644
--- a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
@@ -50,6 +50,7 @@
 import org.junit.Test;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE;
 import static org.junit.Assert.*;
 
 /**
@@ -66,6 +67,7 @@ public class PropertyFileSnitchTest
     @BeforeClass
     public static void setupDD()
     {
+        NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
     }
 
diff --git a/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java b/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java
index cde69d262f89..b5cce05ac1a8 100644
--- a/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java
+++ b/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java
@@ -18,6 +18,14 @@
 
 package org.apache.cassandra.locator;
 
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
 import com.google.common.base.Predicates;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
@@ -33,21 +41,38 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.util.ArrayList;
-import java.util.AbstractMap;
-import java.util.Comparator;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
 
-import static com.google.common.collect.Iterables.*;
+import static com.google.common.collect.Iterables.elementsEqual;
+import static com.google.common.collect.Iterables.filter;
+import static com.google.common.collect.Iterables.limit;
 import static org.apache.cassandra.locator.Replica.fullReplica;
 import static org.apache.cassandra.locator.Replica.transientReplica;
-import static org.apache.cassandra.locator.ReplicaUtils.*;
+import static org.apache.cassandra.locator.ReplicaUtils.ALL_EP;
+import static org.apache.cassandra.locator.ReplicaUtils.ALL_R;
+import static org.apache.cassandra.locator.ReplicaUtils.BROADCAST_EP;
+import static org.apache.cassandra.locator.ReplicaUtils.BROADCAST_RANGE;
+import static org.apache.cassandra.locator.ReplicaUtils.EP1;
+import static org.apache.cassandra.locator.ReplicaUtils.EP2;
+import static org.apache.cassandra.locator.ReplicaUtils.EP3;
+import static org.apache.cassandra.locator.ReplicaUtils.EP4;
+import static org.apache.cassandra.locator.ReplicaUtils.EP5;
+import static org.apache.cassandra.locator.ReplicaUtils.NULL_EP;
+import static org.apache.cassandra.locator.ReplicaUtils.NULL_RANGE;
+import static org.apache.cassandra.locator.ReplicaUtils.R1;
+import static org.apache.cassandra.locator.ReplicaUtils.R2;
+import static org.apache.cassandra.locator.ReplicaUtils.R3;
+import static org.apache.cassandra.locator.ReplicaUtils.R4;
+import static org.apache.cassandra.locator.ReplicaUtils.R5;
+import static org.apache.cassandra.locator.ReplicaUtils.tk;
 
 public class ReplicaCollectionTest
 {
+    static
+    {
+        DatabaseDescriptor.setConfig(new Config());
+    }
 
     static class TestCase<C extends AbstractReplicaCollection<C>>
     {
@@ -90,12 +115,18 @@ void testEndpoints()
             {
                 test.endpoints().add(EP5);
                 Assert.fail();
-            } catch (UnsupportedOperationException e) {}
+            }
+            catch (UnsupportedOperationException e)
+            {
+            }
             try
             {
                 test.endpoints().remove(EP5);
                 Assert.fail();
-            } catch (UnsupportedOperationException e) {}
+            }
+            catch (UnsupportedOperationException e)
+            {
+            }
 
             Assert.assertTrue(test.endpoints().containsAll(canonicalByEndpoint.keySet()));
             for (InetAddressAndPort ep : canonicalByEndpoint.keySet())
@@ -204,7 +235,7 @@ void testCount()
                 return;
             }
 
-            for (int i = 0 ; i < canonicalList.size() ; ++i)
+            for (int i = 0; i < canonicalList.size(); ++i)
             {
                 Replica discount = canonicalList.get(i);
                 Assert.assertEquals(canonicalList.size() - 1, test.count(r -> !r.equals(discount)));
@@ -220,7 +251,7 @@ void testContains()
 
         void testGet()
         {
-            for (int i = 0 ; i < canonicalList.size() ; ++i)
+            for (int i = 0; i < canonicalList.size(); ++i)
                 Assert.assertEquals(canonicalList.get(i), test.get(i));
         }
 
@@ -273,12 +304,18 @@ void testRanges()
             {
                 test.ranges().add(R5);
                 Assert.fail();
-            } catch (UnsupportedOperationException e) {}
+            }
+            catch (UnsupportedOperationException e)
+            {
+            }
             try
             {
                 test.ranges().remove(R5);
                 Assert.fail();
-            } catch (UnsupportedOperationException e) {}
+            }
+            catch (UnsupportedOperationException e)
+            {
+            }
 
             Assert.assertTrue(test.ranges().containsAll(canonicalByRange.keySet()));
             for (Range<Token> range : canonicalByRange.keySet())
@@ -297,12 +334,18 @@ void testByRange()
             {
                 test.byRange().entrySet().contains(null);
                 Assert.fail();
-            } catch (NullPointerException | IllegalArgumentException e) {}
+            }
+            catch (NullPointerException | IllegalArgumentException e)
+            {
+            }
             try
             {
                 test.byRange().containsKey(null);
                 Assert.fail();
-            } catch (NullPointerException | IllegalArgumentException e) {}
+            }
+            catch (NullPointerException | IllegalArgumentException e)
+            {
+            }
 
             for (Range<Token> r : ALL_R)
             {
@@ -328,8 +371,8 @@ public void testOrderOfIteration()
             Assert.assertTrue(Iterables.elementsEqual(Lists.transform(canonicalList, Replica::range), test.ranges()));
             Assert.assertTrue(Iterables.elementsEqual(canonicalList, test.byRange().values()));
             Assert.assertTrue(Iterables.elementsEqual(
-                    Lists.transform(canonicalList, r -> new AbstractMap.SimpleImmutableEntry<>(r.range(), r)),
-                    test.byRange().entrySet()));
+            Lists.transform(canonicalList, r -> new AbstractMap.SimpleImmutableEntry<>(r.range(), r)),
+            test.byRange().entrySet()));
         }
 
         public void testUnwrap(int subListDepth, int filterDepth, int sortDepth)
@@ -346,7 +389,7 @@ public void testUnwrap(int subListDepth, int filterDepth, int sortDepth)
             else
             {
                 new RangesAtEndpointTestCase(false, testUnwrap, canonUnwrap)
-                        .testAllExceptUnwrap(subListDepth, filterDepth, sortDepth);
+                .testAllExceptUnwrap(subListDepth, filterDepth, sortDepth);
             }
         }
 
@@ -381,12 +424,18 @@ void testByEndpoint()
             {
                 test.byEndpoint().entrySet().contains(null);
                 Assert.fail();
-            } catch (NullPointerException | IllegalArgumentException e) {}
+            }
+            catch (NullPointerException | IllegalArgumentException e)
+            {
+            }
             try
             {
                 test.byEndpoint().containsKey(null);
                 Assert.fail();
-            } catch (NullPointerException | IllegalArgumentException e) {}
+            }
+            catch (NullPointerException | IllegalArgumentException e)
+            {
+            }
 
             for (InetAddressAndPort ep : ALL_EP)
             {
@@ -411,8 +460,8 @@ public void testOrderOfIteration()
             super.testOrderOfIteration();
             Assert.assertTrue(Iterables.elementsEqual(canonicalList, test.byEndpoint().values()));
             Assert.assertTrue(Iterables.elementsEqual(
-                    Lists.transform(canonicalList, r -> new AbstractMap.SimpleImmutableEntry<>(r.endpoint(), r)),
-                    test.byEndpoint().entrySet()));
+            Lists.transform(canonicalList, r -> new AbstractMap.SimpleImmutableEntry<>(r.endpoint(), r)),
+            test.byEndpoint().entrySet()));
         }
 
         @Override
@@ -424,11 +473,11 @@ void testAll(int subListDepth, int filterDepth, int sortDepth)
     }
 
     private static final ImmutableList<Replica> RANGES_AT_ENDPOINT = ImmutableList.of(
-            fullReplica(EP1, R1),
-            fullReplica(EP1, R2),
-            transientReplica(EP1, R3),
-            fullReplica(EP1, R4),
-            transientReplica(EP1, R5)
+    fullReplica(EP1, R1),
+    fullReplica(EP1, R2),
+    transientReplica(EP1, R3),
+    fullReplica(EP1, R4),
+    transientReplica(EP1, R5)
     );
 
     @Test
@@ -436,7 +485,7 @@ public void testRangesAtEndpoint()
     {
         ImmutableList<Replica> canonical = RANGES_AT_ENDPOINT;
         new RangesAtEndpointTestCase(
-                false, RangesAtEndpoint.copyOf(canonical), canonical
+        false, RangesAtEndpoint.copyOf(canonical), canonical
         ).testAll();
     }
 
@@ -450,18 +499,27 @@ public void testMutableRangesAtEndpoint()
         {   // incorrect range
             test.addAll(canonical1, Conflict.NONE);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates
         try
         {   // invalid endpoint; always error
             test.add(fullReplica(EP2, BROADCAST_RANGE), Conflict.ALL);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         try
         {   // conflict on isFull/isTransient
             test.add(fullReplica(EP1, R3), Conflict.DUPLICATE);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         test.add(fullReplica(EP1, R3), Conflict.ALL);
 
         new RangesAtEndpointTestCase(true, test, canonical1).testAll();
@@ -475,11 +533,11 @@ public void testMutableRangesAtEndpoint()
     }
 
     private static final ImmutableList<Replica> ENDPOINTS_FOR_X = ImmutableList.of(
-            fullReplica(EP1, R1),
-            fullReplica(EP2, R1),
-            transientReplica(EP3, R1),
-            fullReplica(EP4, R1),
-            transientReplica(EP5, R1)
+    fullReplica(EP1, R1),
+    fullReplica(EP2, R1),
+    transientReplica(EP3, R1),
+    fullReplica(EP4, R1),
+    transientReplica(EP5, R1)
     );
 
     @Test
@@ -487,7 +545,7 @@ public void testEndpointsForRange()
     {
         ImmutableList<Replica> canonical = ENDPOINTS_FOR_X;
         new EndpointsTestCase<>(
-                false, EndpointsForRange.copyOf(canonical), canonical
+        false, EndpointsForRange.copyOf(canonical), canonical
         ).testAll();
     }
 
@@ -501,18 +559,27 @@ public void testMutableEndpointsForRange()
         {   // incorrect range
             test.addAll(canonical1, Conflict.NONE);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates
         try
         {   // incorrect range
             test.add(fullReplica(BROADCAST_EP, R2), Conflict.ALL);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         try
         {   // conflict on isFull/isTransient
             test.add(transientReplica(EP1, R1), Conflict.DUPLICATE);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         test.add(transientReplica(EP1, R1), Conflict.ALL);
 
         new EndpointsTestCase<>(true, test, canonical1).testAll();
@@ -530,7 +597,7 @@ public void testEndpointsForToken()
     {
         ImmutableList<Replica> canonical = ENDPOINTS_FOR_X;
         new EndpointsTestCase<>(
-                false, EndpointsForToken.copyOf(tk(1), canonical), canonical
+        false, EndpointsForToken.copyOf(tk(1), canonical), canonical
         ).testAll();
     }
 
@@ -544,18 +611,27 @@ public void testMutableEndpointsForToken()
         {   // incorrect range
             test.addAll(canonical1, Conflict.NONE);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates
         try
         {   // incorrect range
             test.add(fullReplica(BROADCAST_EP, R2), Conflict.ALL);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         try
         {   // conflict on isFull/isTransient
             test.add(transientReplica(EP1, R1), Conflict.DUPLICATE);
             Assert.fail();
-        } catch (IllegalArgumentException e) { }
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
         test.add(transientReplica(EP1, R1), Conflict.ALL);
 
         new EndpointsTestCase<>(true, test, canonical1).testAll();
diff --git a/test/unit/org/apache/cassandra/locator/ReplicaLayoutTest.java b/test/unit/org/apache/cassandra/locator/ReplicaLayoutTest.java
index b5b60e3e46ab..9b4901799980 100644
--- a/test/unit/org/apache/cassandra/locator/ReplicaLayoutTest.java
+++ b/test/unit/org/apache/cassandra/locator/ReplicaLayoutTest.java
@@ -18,15 +18,24 @@
 
 package org.apache.cassandra.locator;
 
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Token;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import static org.apache.cassandra.locator.ReplicaUtils.*;
 
 public class ReplicaLayoutTest
 {
+    @BeforeClass
+    public static void beforeClass()
+    {
+        DatabaseDescriptor.setConfig(new Config());
+    }
+
     @Test
     public void testConflictResolution()
     {
diff --git a/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java b/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java
index 8231b03a48c6..db8d59e58d59 100644
--- a/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java
+++ b/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java
@@ -18,38 +18,180 @@
 
 package org.apache.cassandra.locator;
 
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Predicate;
+
 import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.junit.Test;
-
-import java.util.Map;
-import java.util.Set;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.reads.NeverSpeculativeRetryPolicy;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
 
+import static org.apache.cassandra.db.ConsistencyLevel.ANY;
 import static org.apache.cassandra.locator.Replica.fullReplica;
-import static org.apache.cassandra.locator.ReplicaUtils.*;
-
+import static org.apache.cassandra.locator.ReplicaUtils.EP1;
+import static org.apache.cassandra.locator.ReplicaUtils.EP2;
+import static org.apache.cassandra.locator.ReplicaUtils.EP3;
+import static org.apache.cassandra.locator.ReplicaUtils.EP4;
+import static org.apache.cassandra.locator.ReplicaUtils.EP5;
+import static org.apache.cassandra.locator.ReplicaUtils.EP6;
+import static org.apache.cassandra.locator.ReplicaUtils.R1;
+import static org.apache.cassandra.locator.ReplicaUtils.assertEquals;
+import static org.apache.cassandra.locator.ReplicaUtils.tk;
+import static org.apache.cassandra.locator.ReplicaUtils.trans;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(BMUnitRunner.class)
 public class ReplicaPlansTest
 {
+    private static final String DC1 = "datacenter1";
+    private static final String KEYSPACE1 = "ks1";
+    private static final String KEYSPACE2 = "ks2";
+
+    private static final Map<InetAddressAndPort, String> ipToKeyspaceAffinity = new HashMap<>();
 
-    static
+    private IEndpointSnitch savedEndpointSnitch;
+
+    @BeforeClass
+    public static void setupClass()
     {
         DatabaseDescriptor.daemonInitialization();
     }
 
+    @Before
+    public void setup()
+    {
+        savedEndpointSnitch = DatabaseDescriptor.getEndpointSnitch();
+    }
+
+    @After
+    public void cleanup()
+    {
+        DatabaseDescriptor.setEndpointSnitch(savedEndpointSnitch);
+    }
+
+    static class FilterByKeyspaceAffinitySnitch extends AbstractNetworkTopologySnitch
+    {
+        public String getRack(InetAddressAndPort endpoint)
+        {
+            byte[] address = endpoint.addressBytes;
+            return "rack" + address[1];
+        }
+
+        public String getDatacenter(InetAddressAndPort endpoint)
+        {
+            return DC1;
+        }
+
+        public Predicate<Replica> filterByAffinity(String keyspace)
+        {
+            return replica -> {
+                // Filter replicas by keyspace affinity
+                return ipToKeyspaceAffinity.get(replica.endpoint()).equals(keyspace);
+            };
+        }
+    }
+
+    @Test
+    @BMRule(name = "FailureDetector sees all nodes as live",
+    targetClass = "FailureDetector",
+    targetMethod = "isAlive",
+    action = "return true;")
+    public void testFilterByAffinity() throws UnknownHostException
+    {
+        IEndpointSnitch filterByKeyspaceAffinitySnitch = new FilterByKeyspaceAffinitySnitch();
+        DatabaseDescriptor.setEndpointSnitch(filterByKeyspaceAffinitySnitch);
+
+        setupReplicas();
+
+        Token token = new Murmur3Partitioner.LongToken(0);
+
+        Keyspace keyspace1 = keyspaceWithSnitch(KEYSPACE1, filterByKeyspaceAffinitySnitch);
+        ReplicaPlan.ForTokenRead plan1 = ReplicaPlans.forRead(keyspace1, token, null, ANY, NeverSpeculativeRetryPolicy.INSTANCE);
+        assertEndpointsMatchKeyspaceAffinity(KEYSPACE1, plan1.contacts());
+
+        Keyspace keyspace2 = keyspaceWithSnitch(KEYSPACE2, filterByKeyspaceAffinitySnitch);
+        ReplicaPlan.ForTokenRead plan2 = ReplicaPlans.forRead(keyspace2, token, null, ANY, NeverSpeculativeRetryPolicy.INSTANCE);
+        assertEndpointsMatchKeyspaceAffinity(KEYSPACE2, plan2.contacts());
+    }
+
+    private void setupReplicas() throws UnknownHostException
+    {
+        TokenMetadata tokenMetadata = StorageService.instance.getTokenMetadata();
+        tokenMetadata.clearUnsafe();
+
+        // List of replicas
+        List<InetAddressAndPort> replicas = ImmutableList.of(
+        InetAddressAndPort.getByName("127.1.0.255"), InetAddressAndPort.getByName("127.1.0.254"), InetAddressAndPort.getByName("127.1.0.253"),
+        InetAddressAndPort.getByName("127.2.0.255"), InetAddressAndPort.getByName("127.2.0.254"), InetAddressAndPort.getByName("127.2.0.253"),
+        InetAddressAndPort.getByName("127.3.0.255"), InetAddressAndPort.getByName("127.3.0.254"), InetAddressAndPort.getByName("127.3.0.253")
+        );
+
+        // Update token metadata and keyspace affinity for each replica
+        for (int i = 0; i < replicas.size(); i++)
+        {
+            InetAddressAndPort ip = replicas.get(i);
+            tokenMetadata.updateHostId(UUID.randomUUID(), ip);
+            tokenMetadata.updateNormalToken(new Murmur3Partitioner.LongToken(i), ip);
+
+            // Alternate keyspace affinity across replicas
+            ipToKeyspaceAffinity.put(ip, i % 2 == 0 ? KEYSPACE1 : KEYSPACE2);
+        }
+    }
+
+    private Keyspace keyspaceWithSnitch(String keyspaceName, IEndpointSnitch snitch)
+    {
+        // Create keyspace metadata
+        KeyspaceParams keyspaceParams = KeyspaceParams.nts(DC1, 3);
+        Tables keyspaceTables = Tables.of(SchemaLoader.standardCFMD(keyspaceName, "Bar").build());
+        KeyspaceMetadata keyspaceMetadata = KeyspaceMetadata.create(keyspaceName, keyspaceParams, keyspaceTables);
+
+        Keyspace keyspace = Keyspace.mockKS(keyspaceMetadata);
+
+        // Associate keyspace to the given snitch
+        keyspace.getReplicationStrategy().snitch = snitch;
+
+        return keyspace;
+    }
+
+    private void assertEndpointsMatchKeyspaceAffinity(String keyspaceName, EndpointsForToken endpoints)
+    {
+        assertTrue(endpoints.stream()
+                            .allMatch(replica -> ipToKeyspaceAffinity.get(replica.endpoint()).equals(keyspaceName)));
+    }
+
     static class Snitch extends AbstractNetworkTopologySnitch
     {
         final Set<InetAddressAndPort> dc1;
+
         Snitch(Set<InetAddressAndPort> dc1)
         {
             this.dc1 = dc1;
         }
+
         @Override
         public String getRack(InetAddressAndPort endpoint)
         {
@@ -73,48 +215,36 @@ private static Keyspace ks(Set<InetAddressAndPort> dc1, Map<String, String> repl
         return keyspace;
     }
 
-    private static Replica full(InetAddressAndPort ep) { return fullReplica(ep, R1); }
-
+    private static Replica full(InetAddressAndPort ep)
+    {
+        return fullReplica(ep, R1);
+    }
 
 
     @Test
     public void testWriteEachQuorum()
     {
-        IEndpointSnitch stash = DatabaseDescriptor.getEndpointSnitch();
         final Token token = tk(1L);
-        try
         {
-            {
-                // all full natural
-                Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3"));
-                EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP3), full(EP4), full(EP5), full(EP6));
-                EndpointsForToken pending = EndpointsForToken.empty(token);
-                ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal);
-                assertEquals(natural, plan.liveAndDown);
-                assertEquals(natural, plan.live);
-                assertEquals(natural, plan.contacts());
-            }
-            {
-                // all natural and up, one transient in each DC
-                Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3"));
-                EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), trans(EP3), full(EP4), full(EP5), trans(EP6));
-                EndpointsForToken pending = EndpointsForToken.empty(token);
+            // all full natural
+            Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3"));
+            EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP3), full(EP4), full(EP5), full(EP6));
+            EndpointsForToken pending = EndpointsForToken.empty(token);
                 ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal);
-                assertEquals(natural, plan.liveAndDown);
-                assertEquals(natural, plan.live);
-                EndpointsForToken expectContacts = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP4), full(EP5));
-                assertEquals(expectContacts, plan.contacts());
-            }
-        }
-        finally
-        {
-            DatabaseDescriptor.setEndpointSnitch(stash);
+            assertEquals(natural, plan.liveAndDown);
+            assertEquals(natural, plan.live);
+            assertEquals(natural, plan.contacts());
         }
-
         {
-            // test simple
-
+            // all natural and up, one transient in each DC
+            Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3"));
+            EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), trans(EP3), full(EP4), full(EP5), trans(EP6));
+            EndpointsForToken pending = EndpointsForToken.empty(token);
+                ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal);
+            assertEquals(natural, plan.liveAndDown);
+            assertEquals(natural, plan.live);
+            EndpointsForToken expectContacts = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP4), full(EP5));
+            assertEquals(expectContacts, plan.contacts());
         }
     }
-
 }
diff --git a/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java b/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java
index 55f54ad3ca29..12e21cbb243a 100644
--- a/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java
+++ b/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java
@@ -26,6 +26,7 @@
 import org.apache.cassandra.gms.Gossiper;
 import org.assertj.core.api.Assertions;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE;
 import static org.junit.Assert.assertEquals;
 
 public class ReplicationFactorTest
@@ -33,6 +34,7 @@ public class ReplicationFactorTest
     @BeforeClass
     public static void setupClass()
     {
+        NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
         DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
         Gossiper.instance.start(1);
diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
index 0204b22d7021..86857bda117c 100644
--- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
+++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
@@ -29,6 +29,7 @@
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -50,6 +51,7 @@
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.PendingRangeCalculatorService;
 import org.apache.cassandra.service.StorageService;
@@ -184,6 +186,15 @@ private void verifyGetNaturalEndpoints(Token[] endpointTokens, Token[] keyTokens
         }
     }
 
+    @Test
+    public void testSimpleStrategyKeyspacesArePartitioned()
+    {
+        //local strategy keyspaces should not be returned here since they are not partitioned
+        Keyspaces partitionedKeyspaces = Schema.instance.getPartitionedKeyspaces();
+        assertEquals(2, partitionedKeyspaces.size());
+        assertEquals(Sets.newHashSet(KEYSPACE1, MULTIDC), Sets.newHashSet(partitionedKeyspaces.names()));
+    }
+
     @Test
     public void testGetEndpointsDuringBootstrap() throws UnknownHostException
     {
@@ -219,7 +230,7 @@ public void testGetEndpointsDuringBootstrap() throws UnknownHostException
         {
             strategy = getStrategy(keyspaceName, tmd, new SimpleSnitch());
 
-            PendingRangeCalculatorService.calculatePendingRanges(strategy, keyspaceName);
+            PendingRangeCalculatorService.instance.calculatePendingRanges(strategy, keyspaceName);
 
             int replicationFactor = strategy.getReplicationFactor().allReplicas;
 
diff --git a/test/unit/org/apache/cassandra/metrics/AbstractDecayingEstimatedHistogramReservoirTest.java b/test/unit/org/apache/cassandra/metrics/AbstractDecayingEstimatedHistogramReservoirTest.java
new file mode 100644
index 000000000000..406f77981746
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/AbstractDecayingEstimatedHistogramReservoirTest.java
@@ -0,0 +1,864 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.LockSupport;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Snapshot;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.MonotonicClockTranslation;
+import org.apache.cassandra.utils.Pair;
+import org.quicktheories.core.Gen;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.metrics.DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_NS;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.metrics.DecayingEstimatedHistogramReservoir.MAX_BUCKET_COUNT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.quicktheories.QuickTheory.qt;
+import static org.quicktheories.generators.SourceDSL.booleans;
+import static org.quicktheories.generators.SourceDSL.integers;
+import static org.quicktheories.generators.SourceDSL.longs;
+
+@RunWith(Enclosed.class)
+public abstract class AbstractDecayingEstimatedHistogramReservoirTest
+{
+    public static class NonParameterizedTests
+    {
+        public boolean useDseHistogramBehaviour;
+
+        public static final Logger logger = LoggerFactory.getLogger(NonParameterizedTests.class);
+        public static final int numExamples = 1000000;
+
+        public Gen<long[]> offsets;
+
+        // not static so that test superclasses can set USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES before it's cached by DEHR initialization
+        public final long[] dseOffsetsWith0 = DecayingEstimatedHistogramReservoir.newDseOffsets(MAX_BUCKET_COUNT, true);
+        public final long[] dseOffsetsWithout0 = DecayingEstimatedHistogramReservoir.newDseOffsets(MAX_BUCKET_COUNT, false);
+
+        private Gen<long[]> generateOffsets()
+        {
+            assertEquals(useDseHistogramBehaviour, CassandraRelevantProperties.USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES.getBoolean());
+            return integers().from(DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT)
+                             .upToAndIncluding(DecayingEstimatedHistogramReservoir.MAX_BUCKET_COUNT - 10)
+                             .zip(booleans().all(), EstimatedHistogram::newOffsets);
+        }
+
+        public NonParameterizedTests()
+        {
+            useDseHistogramBehaviour = CassandraRelevantProperties.USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES.getBoolean();
+            offsets = generateOffsets();
+        }
+
+        @Test
+        public void testFindIndex()
+        {
+            qt().withExamples(numExamples)
+                .forAll(booleans().all()
+                                  .flatMap(b -> offsets.flatMap(offs -> this.offsetsAndValue(offs, b, 0))))
+                .check(this::checkFindIndex);
+        }
+
+        @Test
+        public void showEstimationWorks()
+        {
+            if (useDseHistogramBehaviour)
+                return; // doesn't make sense with DSE buckets
+            qt().withExamples(numExamples)
+                .forAll(offsets.flatMap(offs -> this.offsetsAndValue(offs, false, 9)))
+                .check(this::checkEstimation);
+        }
+
+        //shows that the max before overflow is 238 buckets regardless of consider zeros
+        @Test
+        @Ignore
+        public void showHistogramOffsetOverflow()
+        {
+            qt().forAll(integers().from(DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT).upToAndIncluding(1000))
+                .check(count -> {
+                    long[] offsets = EstimatedHistogram.newOffsets(count, false);
+                    for (long offset : offsets)
+                        if (offset < 0)
+                            return false;
+
+                    return true;
+                });
+        }
+
+        private boolean checkFindIndex(Pair<long[], Long> offsetsAndValue)
+        {
+            long[] offsets = offsetsAndValue.left;
+            long value = offsetsAndValue.right;
+
+            int model = findIndexModel(offsets, value);
+            int actual = DecayingEstimatedHistogramReservoir.findIndex(offsets, value);
+
+            return model == actual;
+        }
+
+        private int findIndexModel(long[] offsets, long value)
+        {
+            int modelIndex = Arrays.binarySearch(offsets, value);
+            if (modelIndex < 0)
+            {
+                modelIndex = -modelIndex - 1;
+                // Special DSE handling of overflows:
+                // in C* overflows have their own bucket. there's one bucket more than the offsets.
+                // with C* behaviour the largest offset defines the overflow boundary
+                // with DSE behaviour the offsets are lower bound, but the index calculation doesn't really care
+                // about offsets, and can compute the index for arbitrary values. Only if the index is greater
+                // than the number of offsets, the value is considered an overflow.
+                // The model cannot replicate this behaviour with binarySearch, because it doesn't know this overflow
+                // boundary (which is the smallest offset not included in the offsets[] array)
+                // Let's use extended dse offsets to let the model handle overflows
+                if (useDseHistogramBehaviour && modelIndex == offsets.length && modelIndex < MAX_BUCKET_COUNT)
+                {
+                    int dseModelIndex = offsets[0] == 0 ?
+                                        findIndexModel(dseOffsetsWith0, value) :
+                                        findIndexModel(dseOffsetsWithout0, value);
+                    if (dseModelIndex >= offsets.length)
+                        return offsets.length; // overflow
+                    else
+                        return offsets.length - 1; // no overflow; just a value that belongs to the last bucket
+                }
+                // Special DSE handling of bucket boundaries
+                // DSE offsets are lower inclusive bounds, not upper inclusive bounds as in C* (or as in Arrays.binarySearch)
+                // so the value belongs to a bucket with a lower offset; we need to decrement the modelIndex
+                //special case:
+                // - value 0, for which the insertion point will be -1, and thus modelIndex = 0 belongs to the first bucket
+                // (there is no bucket with smaller offset), so no need to decrement modelIndex
+                if (useDseHistogramBehaviour && value > 0)
+                {
+                    modelIndex--;
+                }
+            }
+
+            return modelIndex;
+        }
+
+        private Gen<Pair<long[], Long>> offsetsAndValue(long[] offsets, boolean useMaxLong, long minValue)
+        {
+            return longs().between(minValue, useMaxLong ? Long.MAX_VALUE : offsets[offsets.length - 1] + 100)
+                          .mix(longs().between(minValue, minValue + 10),50)
+                          .map(value -> Pair.create(offsets, value));
+        }
+
+        public boolean checkEstimation(Pair<long[], Long> offsetsAndValue)
+        {
+            long[] offsets = offsetsAndValue.left;
+            long value = offsetsAndValue.right;
+            boolean considerZeros = offsets[0] == 0;
+
+            int modelIndex = Arrays.binarySearch(offsets, value);
+            if (modelIndex < 0)
+                modelIndex = -modelIndex - 1;
+
+            int estimate = (int) DecayingEstimatedHistogramReservoir.fastLog12(value);
+
+            if (considerZeros)
+                return estimate - 3 == modelIndex || estimate - 2 == modelIndex;
+            else
+                return estimate - 4 == modelIndex || estimate - 3 == modelIndex;
+        }
+    }
+
+    @RunWith(Parameterized.class)
+    public static class ParameterizedTests
+    {
+        public boolean useDseHistogramBehaviour;
+
+        public static final Logger logger = LoggerFactory.getLogger(NonParameterizedTests.class);
+        private static final double DOUBLE_ASSERT_DELTA = 0;
+
+        @Parameterized.Parameter
+        public String description;
+        
+        @Parameterized.Parameter(1)
+        public Function<DecayingEstimatedHistogramReservoir, Snapshot> toSnapshot;
+
+        @Parameterized.Parameters(name="{0} dseHistograms={2}")
+        public static Collection<Object[]> suppliers()
+        {
+            Function<DecayingEstimatedHistogramReservoir, Snapshot> snapshot = DecayingEstimatedHistogramReservoir::getSnapshot;
+            Function<DecayingEstimatedHistogramReservoir, Snapshot> decayingOnly = DecayingEstimatedHistogramReservoir::getPercentileSnapshot;
+            return ImmutableList.of(
+                new Object[] { "normal", snapshot }, new Object[] { "decaying buckets", decayingOnly }
+            );
+        }
+
+        public ParameterizedTests()
+        {
+            useDseHistogramBehaviour = CassandraRelevantProperties.USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES.getBoolean();
+        }
+
+        @Test
+        public void testStriping() throws InterruptedException
+        {
+            TestClock clock = new TestClock();
+            int nStripes = 4;
+            DecayingEstimatedHistogramReservoir model = new DecayingEstimatedHistogramReservoir(clock);
+            DecayingEstimatedHistogramReservoir test = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION,
+                                                                                               DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
+                                                                                               nStripes,
+                                                                                               clock);
+
+            long seed = nanoTime();
+            System.out.println("AbstractDecayingEstimatedHistogramReservoirTest.ParameterizedTests#testStriping.seed = " + seed);
+            Random valGen = new Random(seed);
+            ExecutorService executors = Executors.newFixedThreadPool(nStripes * 2);
+            for (int i = 0; i < 1_000_000; i++)
+            {
+                long value = Math.abs(valGen.nextInt());
+                executors.submit(() -> {
+                    model.update(value);
+                    LockSupport.parkNanos(2);
+                    test.update(value);
+                });
+            }
+
+            executors.shutdown();
+            Assert.assertTrue(executors.awaitTermination(1, TimeUnit.MINUTES));
+
+            Snapshot modelSnapshot = toSnapshot.apply(model);
+            Snapshot testSnapshot = toSnapshot.apply(test);
+
+            assertEquals(modelSnapshot.getMean(), testSnapshot.getMean(), DOUBLE_ASSERT_DELTA);
+            assertEquals(modelSnapshot.getMin(), testSnapshot.getMin(), DOUBLE_ASSERT_DELTA);
+            assertEquals(modelSnapshot.getMax(), testSnapshot.getMax(), DOUBLE_ASSERT_DELTA);
+            assertEquals(modelSnapshot.getMedian(), testSnapshot.getMedian(), DOUBLE_ASSERT_DELTA);
+            for (double i = 0.0; i < 1.0; i += 0.1)
+                assertEquals(modelSnapshot.getValue(i), testSnapshot.getValue(i), DOUBLE_ASSERT_DELTA);
+
+
+            int stripedValues = 0;
+            for (int i = model.size(); i < model.size() * model.stripeCount(); i++)
+            {
+                stripedValues += model.stripedBucketValue(i, true);
+            }
+            assertTrue("no striping found", stripedValues > 0);
+        }
+
+        @Test
+        public void testSimple()
+        {
+            {
+                // 0 and 1 map to the same, first bucket
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir();
+                histogram.update(0);
+                assertEquals(1, histogram.getSnapshot().getValues()[0]);
+                histogram.update(1);
+                assertEquals(2, histogram.getSnapshot().getValues()[0]);
+            }
+            {
+                // 0 and 1 map to different buckets
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true);
+                histogram.update(0);
+                assertEquals(1, histogram.getSnapshot().getValues()[0]);
+                histogram.update(1);
+                Snapshot snapshot = histogram.getSnapshot();
+                assertEquals(1, snapshot.getValues()[0]);
+                assertEquals(1, snapshot.getValues()[1]);
+            }
+        }
+
+        @Test
+        public void testOverflow()
+        {
+            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, 1, 1);
+            histogram.update(100);
+            assert histogram.isOverflowed();
+            assertEquals(Long.MAX_VALUE, toSnapshot.apply(histogram).getMax());
+        }
+
+        @Test
+        public void testMinMax()
+        {
+            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir();
+            histogram.update(16);
+            Snapshot snapshot = toSnapshot.apply(histogram);
+            assertEquals(15, snapshot.getMin());
+            if (useDseHistogramBehaviour)
+            {
+                // DSE bucket boundary is 16, not 17
+                assertEquals(16, snapshot.getMax());
+            }
+            else
+            {
+                assertEquals(17, snapshot.getMax());
+            }
+        }
+
+        @Test
+        public void testMean()
+        {
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+                for (int i = 0; i < 40; i++)
+                    histogram.update(0);
+                for (int i = 0; i < 20; i++)
+                    histogram.update(1);
+                for (int i = 0; i < 10; i++)
+                    histogram.update(2);
+                assertEquals(1.14D, toSnapshot.apply(histogram).getMean(), 0.1D);
+            }
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true,
+                                                                                                        DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
+                                                                                                        DecayingEstimatedHistogramReservoir.DEFAULT_STRIPE_COUNT,
+                                                                                                        clock);
+                for (int i = 0; i < 40; i++)
+                    histogram.update(0);
+                for (int i = 0; i < 20; i++)
+                    histogram.update(1);
+                for (int i = 0; i < 10; i++)
+                    histogram.update(2);
+                assertEquals(0.57D, toSnapshot.apply(histogram).getMean(), 0.1D);
+            }
+        }
+
+        @Test
+        public void testStdDev()
+        {
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+                for (int i = 0; i < 20; i++)
+                    histogram.update(10);
+                for (int i = 0; i < 40; i++)
+                    histogram.update(20);
+                for (int i = 0; i < 20; i++)
+                    histogram.update(30);
+
+                Snapshot snapshot = toSnapshot.apply(histogram);
+                assertEquals(20.0D, snapshot.getMean(), 2.0D);
+                assertEquals(7.07D, snapshot.getStdDev(), 2.0D);
+            }
+        }
+
+        @Test
+        public void testFindingCorrectBuckets()
+        {
+            if (useDseHistogramBehaviour)
+                return; // doesn't make sense with DSE buckets; too big reliance on particular bucket boundaries
+            TestClock clock = new TestClock();
+
+            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, 90, 1, clock);
+            histogram.update(23282687);
+            assertFalse(histogram.isOverflowed());
+            assertEquals(1, histogram.getSnapshot().getValues()[89]);
+
+            histogram.update(9);
+            assertEquals(1, histogram.getSnapshot().getValues()[8]);
+
+            histogram.update(21);
+            histogram.update(22);
+            Snapshot snapshot = histogram.getSnapshot();
+            assertEquals(2, snapshot.getValues()[13]);
+            assertEquals(6277304.5D, snapshot.getMean(), DOUBLE_ASSERT_DELTA);
+        }
+
+        @Test
+        public void testPercentile()
+        {
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+                // percentile of empty histogram is 0
+                assertEquals(0D, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
+
+                histogram.update(1);
+                // percentile of a histogram with one element should be that element
+                assertEquals(1D, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
+
+                histogram.update(10);
+                assertEquals(10D, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
+            }
+
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+
+                histogram.update(1);
+                histogram.update(2);
+                histogram.update(3);
+                histogram.update(4);
+                histogram.update(5);
+
+                Snapshot snapshot = toSnapshot.apply(histogram);
+                assertEquals(0, snapshot.getValue(0.00), DOUBLE_ASSERT_DELTA);
+                assertEquals(3, snapshot.getValue(0.50), DOUBLE_ASSERT_DELTA);
+                assertEquals(3, snapshot.getValue(0.60), DOUBLE_ASSERT_DELTA);
+                assertEquals(5, snapshot.getValue(1.00), DOUBLE_ASSERT_DELTA);
+            }
+
+            if (!useDseHistogramBehaviour) // the test is too reliant on particular bucket boundaries to use with DSE histogram
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+
+                for (int i = 11; i <= 20; i++)
+                    histogram.update(i);
+
+                // Right now the histogram looks like:
+                //    10   12   14   17   20
+                //     0    2    2    3    3
+                // %:  0   20   40   70  100
+                Snapshot snapshot = toSnapshot.apply(histogram);
+                assertEquals(12, snapshot.getValue(0.01), DOUBLE_ASSERT_DELTA);
+                assertEquals(14, snapshot.getValue(0.30), DOUBLE_ASSERT_DELTA);
+                assertEquals(17, snapshot.getValue(0.50), DOUBLE_ASSERT_DELTA);
+                assertEquals(17, snapshot.getValue(0.60), DOUBLE_ASSERT_DELTA);
+                assertEquals(20, snapshot.getValue(0.80), DOUBLE_ASSERT_DELTA);
+            }
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true,
+                                                                                                        DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
+                                                                                                        DecayingEstimatedHistogramReservoir.DEFAULT_STRIPE_COUNT,
+                                                                                                        clock);
+                histogram.update(0);
+                histogram.update(0);
+                histogram.update(1);
+
+                Snapshot snapshot = toSnapshot.apply(histogram);
+                assertEquals(0, snapshot.getValue(0.5), DOUBLE_ASSERT_DELTA);
+                assertEquals(1, snapshot.getValue(0.99), DOUBLE_ASSERT_DELTA);
+            }
+        }
+
+        @Test
+        public void testDecayingPercentile()
+        {
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+                // percentile of empty histogram is 0
+                assertEquals(0, toSnapshot.apply(histogram).getValue(1.0), DOUBLE_ASSERT_DELTA);
+
+                for (int v = 1; v <= 100; v++)
+                {
+                    for (int i = 0; i < 10_000; i++)
+                    {
+                        histogram.update(v);
+                    }
+                }
+
+                Snapshot snapshot = toSnapshot.apply(histogram);
+                assertEstimatedQuantile(5, snapshot.getValue(0.05));
+                assertEstimatedQuantile(20, snapshot.getValue(0.20));
+                assertEstimatedQuantile(40, snapshot.getValue(0.40));
+                assertEstimatedQuantile(99, snapshot.getValue(0.99));
+
+                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
+                snapshot = toSnapshot.apply(histogram);
+                assertEstimatedQuantile(5, snapshot.getValue(0.05));
+                assertEstimatedQuantile(20, snapshot.getValue(0.20));
+                assertEstimatedQuantile(40, snapshot.getValue(0.40));
+                assertEstimatedQuantile(99, snapshot.getValue(0.99));
+
+                for (int v = 1; v <= 50; v++)
+                {
+                    for (int i = 0; i < 10_000; i++)
+                    {
+                        histogram.update(v);
+                    }
+                }
+
+                snapshot = toSnapshot.apply(histogram);
+                assertEstimatedQuantile(4, snapshot.getValue(0.05));
+                assertEstimatedQuantile(14, snapshot.getValue(0.20));
+                assertEstimatedQuantile(27, snapshot.getValue(0.40));
+                assertEstimatedQuantile(98, snapshot.getValue(0.99));
+
+                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
+                snapshot = toSnapshot.apply(histogram);
+                assertEstimatedQuantile(4, snapshot.getValue(0.05));
+                assertEstimatedQuantile(14, snapshot.getValue(0.20));
+                assertEstimatedQuantile(27, snapshot.getValue(0.40));
+                assertEstimatedQuantile(98, snapshot.getValue(0.99));
+
+                for (int v = 1; v <= 50; v++)
+                {
+                    for (int i = 0; i < 10_000; i++)
+                    {
+                        histogram.update(v);
+                    }
+                }
+
+                snapshot = toSnapshot.apply(histogram);
+                assertEstimatedQuantile(3, snapshot.getValue(0.05));
+                assertEstimatedQuantile(12, snapshot.getValue(0.20));
+                assertEstimatedQuantile(23, snapshot.getValue(0.40));
+                assertEstimatedQuantile(96, snapshot.getValue(0.99));
+
+                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
+                snapshot = toSnapshot.apply(histogram);
+                assertEstimatedQuantile(3, snapshot.getValue(0.05));
+                assertEstimatedQuantile(12, snapshot.getValue(0.20));
+                assertEstimatedQuantile(23, snapshot.getValue(0.40));
+                assertEstimatedQuantile(96, snapshot.getValue(0.99));
+
+                for (int v = 11; v <= 20; v++)
+                {
+                    for (int i = 0; i < 5_000; i++)
+                    {
+                        histogram.update(v);
+                    }
+                }
+
+                snapshot = toSnapshot.apply(histogram);
+                assertEstimatedQuantile(4, snapshot.getValue(0.05));
+                assertEstimatedQuantile(12, snapshot.getValue(0.20));
+                assertEstimatedQuantile(20, snapshot.getValue(0.40));
+                assertEstimatedQuantile(95, snapshot.getValue(0.99));
+
+                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
+                snapshot = toSnapshot.apply(histogram);
+                assertEstimatedQuantile(4, snapshot.getValue(0.05));
+                assertEstimatedQuantile(12, snapshot.getValue(0.20));
+                assertEstimatedQuantile(20, snapshot.getValue(0.40));
+                assertEstimatedQuantile(95, snapshot.getValue(0.99));
+
+            }
+
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+                // percentile of empty histogram is 0
+                assertEquals(0, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
+
+                for (int m = 0; m < 40; m++)
+                {
+                    for (int i = 0; i < 1_000_000; i++)
+                    {
+                        histogram.update(2);
+                    }
+                    // percentile of a histogram with one element should be that element
+                    clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
+                    assertEquals(2, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
+                }
+
+                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S * 100);
+                assertEquals(0, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
+            }
+
+            if (!useDseHistogramBehaviour) // the test is too reliant on particular bucket boundaries to use with DSE histogram
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+
+                histogram.update(20);
+                histogram.update(21);
+                histogram.update(22);
+                Snapshot snapshot = histogram.getSnapshot();
+                assertEquals(1, snapshot.getValues()[12]);
+                assertEquals(2, snapshot.getValues()[13]);
+
+                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
+
+                histogram.update(20);
+                histogram.update(21);
+                histogram.update(22);
+                snapshot = histogram.getSnapshot();
+                assertEquals(2, snapshot.getValues()[12]);
+                assertEquals(4, snapshot.getValues()[13]);
+            }
+        }
+
+        @Test
+        public void testDecayingMean()
+        {
+            {
+                TestClock clock = new TestClock();
+
+                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+
+                clock.addNanos(LANDMARK_RESET_INTERVAL_IN_NS - TimeUnit.SECONDS.toNanos(1L));
+
+                while (clock.now() < LANDMARK_RESET_INTERVAL_IN_NS + TimeUnit.SECONDS.toNanos(1L))
+                {
+                    clock.addNanos(TimeUnit.MILLISECONDS.toNanos(900));
+                    for (int i = 0; i < 1_000_000; i++)
+                    {
+                        histogram.update(1000);
+                        histogram.update(2000);
+                        histogram.update(3000);
+                        histogram.update(4000);
+                        histogram.update(5000);
+                    }
+                    assertEquals(3000D, toSnapshot.apply(histogram).getMean(), 500D);
+                }
+            }
+        }
+
+        @Test
+        public void testAggregation()
+        {
+            TestClock clock = new TestClock();
+
+            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+            DecayingEstimatedHistogramReservoir another = new DecayingEstimatedHistogramReservoir(clock);
+
+            clock.addNanos(LANDMARK_RESET_INTERVAL_IN_NS - TimeUnit.SECONDS.toNanos(1L));
+
+            histogram.update(1000);
+            clock.addMillis(100);
+            another.update(2000);
+            clock.addMillis(100);
+            histogram.update(2000);
+            clock.addMillis(100);
+            another.update(3000);
+            clock.addMillis(100);
+            histogram.update(3000);
+            clock.addMillis(100);
+            another.update(4000);
+
+            DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot snapshot = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) histogram.getSnapshot();
+            DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot anotherSnapshot = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) another.getSnapshot();
+
+            assertEquals(2000, snapshot.getMean(), 500D);
+            assertEquals(3000, anotherSnapshot.getMean(), 500D);
+
+            snapshot.add(anotherSnapshot);
+
+            // Another had newer decayLandmark, the aggregated snapshot should use it
+            assertEquals(anotherSnapshot.getSnapshotLandmark(), snapshot.getSnapshotLandmark());
+            assertEquals(2500, snapshot.getMean(), 500D);
+        }
+
+        @Test
+        public void testSize()
+        {
+            TestClock clock = new TestClock();
+
+            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+            histogram.update(42);
+            histogram.update(42);
+            assertEquals(2, toSnapshot.apply(histogram).size());
+        }
+
+        /**
+         * This looks for invalid percentiles that are unchanged for too long to expose the CASSANDRA-19365 race
+         * condition between rescale and update. The idea is to update a histogram from multiple threads and observe
+         * if the reported p99 doesn't get stuck at a low value or p50 at a high value due to update with high weight
+         * being inserted after the buckets are rescaled.
+         * <p>
+         * The load has 95% of 42, and 5% of the time it's 1109. Despite that the histogram may be convinced for a long
+         * time that p99 is 42 or that p50 is 1109. The reason may be seen in the snapshot dump, where after rescale
+         * the bucket values may get very big due to the race condition and too big weight of the inserted samples.
+         * The values were picked to match bucket boundaries, but that's only for aesthetics.
+         * <p>
+         * In production the rescale happens every 30 minutes. In this test time we're pushing time to run faster,
+         * roughly 1000 times faster to hit the race condition in a reasonable time.
+         */
+        @Test
+        public void testConcurrentUpdateAndRescale() throws InterruptedException
+        {
+            int UPDATE_THREADS = 60;
+            int maxTestDurationMillis = 30_000;
+            // how many times in a row the percentiles may be invalid before we fail the test
+            int tooManySuspiciousPercentilesThreshold = 5; // 5 translates to 500ms * 1000 speedup = 500s = 8m20s;
+            AtomicBoolean stop = new AtomicBoolean(false);
+            AtomicBoolean failed = new AtomicBoolean(false);
+            TestClock clock = new TestClock();
+
+            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
+            ExecutorService executors = Executors.newFixedThreadPool(2 + UPDATE_THREADS);
+
+            for (int i = 0; i < UPDATE_THREADS; i++)
+            {
+                executors.submit(() -> {
+                    while (!stop.get() && !Thread.currentThread().isInterrupted())
+                    {
+                        // a mischievous usage pattern to quickly trigger the
+                        // CASSANDRA-19365 race condition;
+                        // the load has 95% of 42, and only 5% of the time it's 1109
+                        // and yet, the histogram may be convinced for a long time that
+                        // the p99 is 42 or that the p50 is 1109
+                        for (int sampleIdx = 0; sampleIdx < 900; sampleIdx++)
+                            histogram.update(42);
+
+                        for (int sampleIdx = 0; sampleIdx < 50; sampleIdx++)
+                        {
+                            // add some noise so that low value samples do not race with the same likelyhood as the high value samples
+                            Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(1, 10), MILLISECONDS);
+                            histogram.update(1109);
+                        }
+                    }
+                });
+            }
+            // clock update thread
+            executors.submit(() -> {
+                while (!stop.get() && !Thread.currentThread().isInterrupted())
+                {
+                    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
+                    // x1000 speedup so that we hit rescale interval every 30 minutes / 1000 = 1.8s
+                    clock.addMillis(1000);
+                }
+            });
+            // percentiles check thread
+            executors.submit(() -> {
+                // how many times in a row p99 was suspiciously low or P50 suspiciously high
+                int consecutiveInvalidPercentiles = 0;
+
+                // how often to check the percentiles
+                int iterationDelayMillis = 100;
+
+                for (int i = 0; i < maxTestDurationMillis / iterationDelayMillis; i++)
+                {
+                    Uninterruptibles.sleepUninterruptibly(iterationDelayMillis, MILLISECONDS);
+                    Snapshot snapshot = toSnapshot.apply(histogram);
+                    double p99 = snapshot.getValue(0.99);
+                    double p50 = snapshot.getValue(0.50);
+                    ByteArrayOutputStream output = new ByteArrayOutputStream();
+                    snapshot.dump(output);
+                    String decayingNonZeroBuckets = Arrays.stream(output.toString().split("\n"))
+                                                          .filter(s -> !s.equals("0"))
+                                                          .collect(Collectors.joining(","));
+                    logger.info("\"clock={}, p50={}, p99={}, decaying non-zero buckets: {}",
+                                clock.now() / 1_000_000, p50, p99, decayingNonZeroBuckets);
+                    if (p99 < 100 || p50 > 900)
+                    {
+                        consecutiveInvalidPercentiles++;
+                        logger.warn("p50 or p99 at suspicious level p50={}, p99={}", p50, p99);
+                        if (consecutiveInvalidPercentiles > tooManySuspiciousPercentilesThreshold)
+                        {
+                            failed.set(true);
+                            stop.set(true);
+                            break;
+                        }
+                    }
+                    else
+                    {
+                        consecutiveInvalidPercentiles = 0;
+                    }
+                }
+                stop.set(true);
+            });
+            executors.shutdown();
+            boolean success = executors.awaitTermination(maxTestDurationMillis * 2, MILLISECONDS);
+            Assert.assertFalse("p50 too high or p99 too low for too long", failed.get());
+            Assert.assertTrue("Timeout exceeded the limit", success);
+        }
+
+        private void assertEstimatedQuantile(long expectedValue, double actualValue)
+        {
+            if (useDseHistogramBehaviour)
+            {
+                // DSE histograms have a different bucketing scheme, so let's be more liberal
+                // checking the estimated quantiles (especially that we're only using the non-decaying buckets)
+                long geBound = Math.round(expectedValue * 0.8);
+                long leBound = Math.round(expectedValue * 1.2);
+                assertTrue("Expected at least [" + geBound + "] but actual is [" + actualValue + "]", actualValue >= geBound);
+                assertTrue("Expected no more than [" + leBound + "] but actual is [" + actualValue + "]", actualValue <= leBound);
+                return;
+            }
+
+            assertTrue("Expected at least [" + expectedValue + "] but actual is [" + actualValue + ']', actualValue >= expectedValue);
+            assertTrue("Expected less than [" + Math.round(expectedValue * 1.2) + "] but actual is [" + actualValue + ']', actualValue < Math.round(expectedValue * 1.2));
+        }
+
+        public static class TestClock implements MonotonicClock
+        {
+            private long tick = 0;
+
+            public void addNanos(long nanos)
+            {
+                tick += nanos;
+            }
+
+            public void addMillis(long millis)
+            {
+                tick += TimeUnit.MILLISECONDS.toNanos(millis);
+            }
+
+            public void addSeconds(long seconds)
+            {
+                tick += TimeUnit.SECONDS.toNanos(seconds);
+            }
+
+            public long now()
+            {
+                return tick;
+            }
+
+            @Override
+            public long error()
+            {
+                return 0;
+            }
+
+            @Override
+            public MonotonicClockTranslation translate()
+            {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public boolean isAfter(long instant)
+            {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public boolean isAfter(long now, long instant)
+            {
+                throw new UnsupportedOperationException();
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/BatchMetricsTest.java b/test/unit/org/apache/cassandra/metrics/BatchMetricsTest.java
index b90f19a7fc51..aa78b5f37036 100644
--- a/test/unit/org/apache/cassandra/metrics/BatchMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/BatchMetricsTest.java
@@ -158,15 +158,28 @@ private void assertMetrics(BatchStatement.Type batchTypeTested, int[] rounds, in
             long partitionsPerCounterBatchCountPre = metrics.partitionsPerCounterBatch.getCount();
             long expectedPartitionsPerCounterBatchCount = partitionsPerCounterBatchCountPre + (batchTypeTested == BatchStatement.Type.COUNTER ? 1 : 0);
 
+            long columnsPerLoggedBatchCountPre = metrics.columnsPerLoggedBatch.getCount();
+            long expectedColumnsPerLoggedBatchCount = columnsPerLoggedBatchCountPre + (batchTypeTested == BatchStatement.Type.LOGGED ? 1 : 0);
+            long columnsPerUnloggedBatchCountPre = metrics.columnsPerUnloggedBatch.getCount();
+            long expectedColumnsPerUnloggedBatchCount = columnsPerUnloggedBatchCountPre + (batchTypeTested == BatchStatement.Type.UNLOGGED ? 1 : 0);
+            long columnsPerCounterBatchCountPre = metrics.columnsPerCounterBatch.getCount();
+            long expectedColumnsPerCounterBatchCount = columnsPerCounterBatchCountPre + (batchTypeTested == BatchStatement.Type.COUNTER ? 1 : 0);
+
             executeLoggerBatch(batchTypeTested, distinctPartitions, rounds[ix]);
 
             assertEquals(expectedPartitionsPerUnloggedBatchCount, metrics.partitionsPerUnloggedBatch.getCount());
             assertEquals(expectedPartitionsPerLoggedBatchCount, metrics.partitionsPerLoggedBatch.getCount());
             assertEquals(expectedPartitionsPerCounterBatchCount, metrics.partitionsPerCounterBatch.getCount());
+            assertEquals(expectedColumnsPerUnloggedBatchCount, metrics.columnsPerUnloggedBatch.getCount());
+            assertEquals(expectedColumnsPerLoggedBatchCount, metrics.columnsPerLoggedBatch.getCount());
+            assertEquals(expectedColumnsPerCounterBatchCount, metrics.columnsPerCounterBatch.getCount());
 
             EstimatedHistogramReservoirSnapshot partitionsPerLoggedBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.partitionsPerLoggedBatch.getSnapshot();
             EstimatedHistogramReservoirSnapshot partitionsPerUnloggedBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.partitionsPerUnloggedBatch.getSnapshot();
             EstimatedHistogramReservoirSnapshot partitionsPerCounterBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.partitionsPerCounterBatch.getSnapshot();
+            EstimatedHistogramReservoirSnapshot columnsPerLoggedBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.columnsPerLoggedBatch.getSnapshot();
+            EstimatedHistogramReservoirSnapshot columnsPerUnloggedBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.columnsPerUnloggedBatch.getSnapshot();
+            EstimatedHistogramReservoirSnapshot columnsPerCounterBatchSnapshot = (EstimatedHistogramReservoirSnapshot) metrics.columnsPerCounterBatch.getSnapshot();
 
             // BatchMetrics uses DecayingEstimatedHistogramReservoir which notes that the return of getMax()
             // may be more than the actual max value recorded in the reservoir with similar but reverse properties
@@ -181,10 +194,22 @@ private void assertMetrics(BatchStatement.Type batchTypeTested, int[] rounds, in
             Range expectedPartitionsPerCounterBatchMinMax = batchTypeTested == BatchStatement.Type.COUNTER ?
                                                             determineExpectedMinMax(partitionsPerCounterBatchSnapshot, distinctPartitions) :
                                                             new Range(0L, 0L);
+            Range expectedColumnsPerLoggedBatchMinMax = batchTypeTested == BatchStatement.Type.LOGGED ?
+                                                        determineExpectedMinMax(columnsPerLoggedBatchSnapshot, distinctPartitions) :
+                                                        new Range(0L, 0L);
+            Range expectedColumnsPerUnloggedBatchMinMax = batchTypeTested == BatchStatement.Type.UNLOGGED ?
+                                                          determineExpectedMinMax(columnsPerUnloggedBatchSnapshot, distinctPartitions) :
+                                                          new Range(0L, 0L);
+            Range expectedColumnsPerCounterBatchMinMax = batchTypeTested == BatchStatement.Type.COUNTER ?
+                                                         determineExpectedMinMax(columnsPerCounterBatchSnapshot, distinctPartitions) :
+                                                         new Range(0L, 0L);
 
             assertEquals(expectedPartitionsPerLoggedBatchMinMax, new Range(partitionsPerLoggedBatchSnapshot.getMin(), partitionsPerLoggedBatchSnapshot.getMax()));
             assertEquals(expectedPartitionsPerUnloggedBatchMinMax, new Range(partitionsPerUnloggedBatchSnapshot.getMin(), partitionsPerUnloggedBatchSnapshot.getMax()));
             assertEquals(expectedPartitionsPerCounterBatchMinMax, new Range(partitionsPerCounterBatchSnapshot.getMin(), partitionsPerCounterBatchSnapshot.getMax()));
+            assertEquals(expectedColumnsPerLoggedBatchMinMax, new Range(columnsPerLoggedBatchSnapshot.getMin(), columnsPerLoggedBatchSnapshot.getMax()));
+            assertEquals(expectedColumnsPerUnloggedBatchMinMax, new Range(columnsPerUnloggedBatchSnapshot.getMin(), columnsPerUnloggedBatchSnapshot.getMax()));
+            assertEquals(expectedColumnsPerCounterBatchMinMax, new Range(columnsPerCounterBatchSnapshot.getMin(), columnsPerCounterBatchSnapshot.getMax()));
         }
     }
 
@@ -193,6 +218,9 @@ private void clearHistogram()
         ((ClearableHistogram) metrics.partitionsPerLoggedBatch).clear();
         ((ClearableHistogram) metrics.partitionsPerUnloggedBatch).clear();
         ((ClearableHistogram) metrics.partitionsPerCounterBatch).clear();
+        ((ClearableHistogram) metrics.columnsPerLoggedBatch).clear();
+        ((ClearableHistogram) metrics.columnsPerUnloggedBatch).clear();
+        ((ClearableHistogram) metrics.columnsPerCounterBatch).clear();
     }
 
     private Range determineExpectedMinMax(EstimatedHistogramReservoirSnapshot snapshot, long value)
diff --git a/test/unit/org/apache/cassandra/metrics/BufferPoolMetricsTest.java b/test/unit/org/apache/cassandra/metrics/BufferPoolMetricsTest.java
index 8db86d89cdfd..e5a28ce4a247 100644
--- a/test/unit/org/apache/cassandra/metrics/BufferPoolMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/BufferPoolMetricsTest.java
@@ -18,26 +18,41 @@
 
 package org.apache.cassandra.metrics;
 
+import java.util.Arrays;
 import java.util.Random;
 
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.utils.memory.BufferPool;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.USE_MICROMETER;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.junit.Assert.assertEquals;
 
+@RunWith(Parameterized.class)
 public class BufferPoolMetricsTest
 {
     private BufferPool bufferPool;
     private BufferPoolMetrics metrics;
 
+    // Test with both MicrometerBufferPoolMetrics and CodahaleBufferPoolMetrics
+    @Parameterized.Parameters(name = "useMicrometerMetrics {0}")
+    public static Iterable<Object[]> params()
+    {
+        return Arrays.asList(new Object[][]{ { false }, { true } });
+    }
+
+    @Parameterized.Parameter
+    public boolean useMicrometerMetrics = false;
+
     @BeforeClass()
     public static void setup() throws ConfigurationException
     {
@@ -47,6 +62,8 @@ public static void setup() throws ConfigurationException
     @Before
     public void setUp()
     {
+        // The USE_MICROMETER value is used when the BufferPool constructor creates BufferPoolMetrics
+        USE_MICROMETER.setBoolean(useMicrometerMetrics);
         this.bufferPool = new BufferPool("test_" + System.currentTimeMillis(), 16 * 1024L * 1024L, true);
         this.metrics = bufferPool.metrics();
     }
@@ -55,8 +72,8 @@ public void setUp()
     public void testMetricsSize()
     {
         // basically want to test changes in the metric being reported as the buffer pool grows - starts at zero
-        assertThat(metrics.size.getValue()).isEqualTo(bufferPool.sizeInBytes())
-                                           .isEqualTo(0);
+        assertThat(metrics.size()).isEqualTo(bufferPool.sizeInBytes())
+                                  .isEqualTo(0);
 
         // the idea is to test changes in the sizeOfBufferPool metric which starts at zero. it will bump up
         // after the first request for a ByteBuffer and the idea from there will be to keep requesting them
@@ -79,9 +96,9 @@ public void testMetricsSize()
             totalBytesRequestedFromPool = totalBytesRequestedFromPool + nextSizeToRequest;
             bufferPool.get(nextSizeToRequest, BufferType.OFF_HEAP);
 
-            assertThat(metrics.size.getValue()).as(assertionMessage)
-                                               .isEqualTo(bufferPool.sizeInBytes())
-                                               .isGreaterThanOrEqualTo(totalBytesRequestedFromPool);
+            assertThat(metrics.size()).as(assertionMessage)
+                                      .isEqualTo(bufferPool.sizeInBytes())
+                                      .isGreaterThanOrEqualTo(totalBytesRequestedFromPool);
 
             if (initialSizeInBytesAfterZero == 0)
             {
@@ -101,13 +118,13 @@ public void testMetricsSize()
         }
 
         assertThat(exitedBeforeMax).as(assertionMessage).isTrue();
-        assertEquals(0, metrics.misses.getCount());
+        assertEquals(0, metrics.misses());
     }
 
     @Test
     public void testMetricsOverflowSize()
     {
-        assertEquals(0, metrics.overflowSize.getValue().longValue());
+        assertEquals(0, metrics.overflowSize());
 
         final int tinyBufferSizeThatHits = BufferPool.NORMAL_CHUNK_SIZE - 1;
         final int bigBufferSizeThatMisses = BufferPool.NORMAL_CHUNK_SIZE + 1;
@@ -116,20 +133,20 @@ public void testMetricsOverflowSize()
         for (int ix = 0; ix < iterations; ix++)
         {
             bufferPool.get(tinyBufferSizeThatHits, BufferType.OFF_HEAP);
-            assertEquals(0, metrics.overflowSize.getValue().longValue());
+            assertEquals(0, metrics.overflowSize());
         }
 
         for (int ix = 0; ix < iterations; ix++)
         {
             bufferPool.get(bigBufferSizeThatMisses, BufferType.OFF_HEAP);
-            assertEquals(bigBufferSizeThatMisses * (ix + 1), metrics.overflowSize.getValue().longValue());
+            assertEquals(bigBufferSizeThatMisses * (ix + 1), metrics.overflowSize());
         }
     }
 
     @Test
     public void testMetricsUsedSize()
     {
-        assertEquals(0, metrics.usedSize.getValue().longValue());
+        assertEquals(0, metrics.usedSize());
 
         final int tinyBufferSizeThatHits = BufferPool.NORMAL_CHUNK_SIZE - 1;
         final int bigBufferSizeThatMisses = BufferPool.NORMAL_CHUNK_SIZE + 1;
@@ -139,20 +156,20 @@ public void testMetricsUsedSize()
         for (int ix = 0; ix < iterations; ix++)
         {
             bufferPool.get(tinyBufferSizeThatHits, BufferType.OFF_HEAP);
-            assertEquals(usedSize += tinyBufferSizeThatHits, metrics.usedSize.getValue().longValue());
+            assertEquals(usedSize += tinyBufferSizeThatHits, metrics.usedSize());
         }
 
         for (int ix = 0; ix < iterations; ix++)
         {
             bufferPool.get(bigBufferSizeThatMisses, BufferType.OFF_HEAP);
-            assertEquals(usedSize += bigBufferSizeThatMisses, metrics.usedSize.getValue().longValue());
+            assertEquals(usedSize += bigBufferSizeThatMisses, metrics.usedSize());
         }
     }
 
     @Test
     public void testMetricsHits()
     {
-        assertEquals(0, metrics.hits.getCount());
+        assertEquals(0, metrics.hits());
 
         final int tinyBufferSizeThatHits = BufferPool.NORMAL_CHUNK_SIZE - 1;
         final int bigBufferSizeThatMisses = BufferPool.NORMAL_CHUNK_SIZE + 1;
@@ -161,21 +178,21 @@ public void testMetricsHits()
         for (int ix = 0; ix < iterations; ix++)
         {
             bufferPool.get(tinyBufferSizeThatHits, BufferType.OFF_HEAP);
-            assertEquals(ix + 1, metrics.hits.getCount());
+            assertEquals(ix + 1, metrics.hits());
         }
 
-        long currentHits = metrics.hits.getCount();
+        long currentHits = metrics.hits();
         for (int ix = 0; ix < iterations; ix++)
         {
             bufferPool.get(bigBufferSizeThatMisses + ix, BufferType.OFF_HEAP);
-            assertEquals(currentHits, metrics.hits.getCount());
+            assertEquals(currentHits, metrics.hits());
         }
     }
 
     @Test
     public void testMetricsMisses()
     {
-        assertEquals(0, metrics.misses.getCount());
+        assertEquals(0, metrics.misses());
 
         final int tinyBufferSizeThatHits = BufferPool.NORMAL_CHUNK_SIZE - 1;
         final int bigBufferSizeThatMisses = BufferPool.NORMAL_CHUNK_SIZE + 1;
@@ -184,28 +201,28 @@ public void testMetricsMisses()
         for (int ix = 0; ix < iterations; ix++)
         {
             bufferPool.get(tinyBufferSizeThatHits, BufferType.OFF_HEAP);
-            assertEquals(0, metrics.misses.getCount());
+            assertEquals(0, metrics.misses());
         }
 
         for (int ix = 0; ix < iterations; ix++)
         {
             bufferPool.get(bigBufferSizeThatMisses + ix, BufferType.OFF_HEAP);
-            assertEquals(ix + 1, metrics.misses.getCount());
+            assertEquals(ix + 1, metrics.misses());
         }
     }
 
     @Test
     public void testZeroSizeRequestsDontChangeMetrics()
     {
-        assertEquals(0, metrics.misses.getCount());
-        assertThat(metrics.size.getValue()).isEqualTo(bufferPool.sizeInBytes())
-                                           .isEqualTo(0);
+        assertEquals(0, metrics.misses());
+        assertThat(metrics.size()).isEqualTo(bufferPool.sizeInBytes())
+                                  .isEqualTo(0);
 
         bufferPool.get(0, BufferType.OFF_HEAP);
 
-        assertEquals(0, metrics.misses.getCount());
-        assertThat(metrics.size.getValue()).isEqualTo(bufferPool.sizeInBytes())
-                                           .isEqualTo(0);
+        assertEquals(0, metrics.misses());
+        assertThat(metrics.size()).isEqualTo(bufferPool.sizeInBytes())
+                                  .isEqualTo(0);
 
         bufferPool.get(65536, BufferType.OFF_HEAP);
         bufferPool.get(0, BufferType.OFF_HEAP);
@@ -213,23 +230,23 @@ public void testZeroSizeRequestsDontChangeMetrics()
         bufferPool.get(0, BufferType.OFF_HEAP);
         bufferPool.get(0, BufferType.OFF_HEAP);
 
-        assertEquals(0, metrics.misses.getCount());
-        assertThat(metrics.size.getValue()).isEqualTo(bufferPool.sizeInBytes())
-                                           .isGreaterThanOrEqualTo(65536);
+        assertEquals(0, metrics.misses());
+        assertThat(metrics.size()).isEqualTo(bufferPool.sizeInBytes())
+                                  .isGreaterThanOrEqualTo(65536);
     }
 
     @Test
     public void testFailedRequestsDontChangeMetrics()
     {
-        assertEquals(0, metrics.misses.getCount());
-        assertThat(metrics.size.getValue()).isEqualTo(bufferPool.sizeInBytes())
-                                           .isEqualTo(0);
+        assertEquals(0, metrics.misses());
+        assertThat(metrics.size()).isEqualTo(bufferPool.sizeInBytes())
+                                  .isEqualTo(0);
 
         tryRequestNegativeBufferSize();
 
-        assertEquals(0, metrics.misses.getCount());
-        assertThat(metrics.size.getValue()).isEqualTo(bufferPool.sizeInBytes())
-                                           .isEqualTo(0);
+        assertEquals(0, metrics.misses());
+        assertThat(metrics.size()).isEqualTo(bufferPool.sizeInBytes())
+                                  .isEqualTo(0);
 
         bufferPool.get(65536, BufferType.OFF_HEAP);
         tryRequestNegativeBufferSize();
@@ -237,9 +254,9 @@ public void testFailedRequestsDontChangeMetrics()
         tryRequestNegativeBufferSize();
         tryRequestNegativeBufferSize();
 
-        assertEquals(0, metrics.misses.getCount());
-        assertThat(metrics.size.getValue()).isEqualTo(bufferPool.sizeInBytes())
-                                           .isGreaterThanOrEqualTo(65536);
+        assertEquals(0, metrics.misses());
+        assertThat(metrics.size()).isEqualTo(bufferPool.sizeInBytes())
+                                  .isGreaterThanOrEqualTo(65536);
     }
 
     private void tryRequestNegativeBufferSize()
diff --git a/test/unit/org/apache/cassandra/metrics/CacheMetricsTest.java b/test/unit/org/apache/cassandra/metrics/CacheMetricsTest.java
index 32a38c96c78e..de6a96b0b563 100644
--- a/test/unit/org/apache/cassandra/metrics/CacheMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/CacheMetricsTest.java
@@ -21,24 +21,48 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.AfterClass;
 import org.junit.Test;
 
 import org.apache.cassandra.cache.CacheSize;
 import org.apache.cassandra.cache.ICache;
 import org.apache.cassandra.cache.InstrumentingCache;
+import org.apache.cassandra.service.CacheService;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.USE_MICROMETER;
 import static org.junit.Assert.assertEquals;
 
 public class CacheMetricsTest
 {
     private static final long capacity = 65536;
 
+    @AfterClass
+    public static void teardown()
+    {
+        USE_MICROMETER.reset();
+    }
+
     @Test
+    public void testCodahaleCacheMetrics()
+    {
+        USE_MICROMETER.setBoolean(false);
+        testCacheMetrics();
+    }
+
+    @Test
+    public void testMicrometerCacheMetrics()
+    {
+        USE_MICROMETER.setBoolean(true);
+        testCacheMetrics();
+    }
+
     public void testCacheMetrics()
     {
         ICache<String,Object> mockedCache = new MapMockedCache();
-        InstrumentingCache<String,Object> cache = new InstrumentingCache<>("cache", mockedCache);
+        InstrumentingCache<String,Object> cache = new InstrumentingCache<>(CacheService.CacheType.KEY_CACHE, mockedCache);
         CacheMetrics metrics = cache.getMetrics();
 
         assertCacheMetrics(metrics, expect(mockedCache));
@@ -58,8 +82,6 @@ public void testCacheMetrics()
         assertCacheMetrics(metrics, expect(mockedCache).hits(80).misses(20));
 
         cache.clear();
-        metrics.reset();
-        assertCacheMetrics(metrics, expect(mockedCache));
     }
 
     private void getFromCache(InstrumentingCache<String,Object> cache, String key, int times)
@@ -76,16 +98,23 @@ private void assertCacheMetrics(CacheMetrics actual, CacheMetricsExpectation exp
         // calculations - applying some general assertions for hitRate calculations that essentially just smoke test
         // existence (i.e. NaN at initialization) since they are established by way of an inner class on CacheMetrics
         // itself.
-        assertEquals(expectation.cacheSize.capacity(), actual.capacity.getValue().longValue());
-        assertEquals(expectation.cacheSize.weightedSize(), actual.size.getValue().longValue());
-        assertEquals(expectation.cacheSize.size(), actual.entries.getValue().intValue());
-        assertEquals(expectation.hits, actual.hits.getCount());
-        assertEquals(expectation.misses, actual.misses.getCount());
-        assertEquals(expectation.requests(), actual.requests.getCount());
-        assertEquals(expectation.hitRate(), actual.hitRate.getValue(), 0.001d);
-        assertEquals(Double.NaN, actual.oneMinuteHitRate.getValue(), 0.001d);
-        assertEquals(Double.NaN, actual.fiveMinuteHitRate.getValue(), 0.001d);
-        assertEquals(Double.NaN, actual.fifteenMinuteHitRate.getValue(), 0.001d);
+        if (actual instanceof MicrometerCacheMetrics)
+        {
+            Uninterruptibles.sleepUninterruptibly(2 * MicrometerCacheMetrics.hitRateUpdateIntervalNanos, TimeUnit.NANOSECONDS);
+        }
+
+        assertEquals(expectation.cacheSize.capacity(), actual.capacity());
+        assertEquals(expectation.cacheSize.weightedSize(), actual.size());
+        assertEquals(expectation.cacheSize.size(), actual.entries());
+        assertEquals(expectation.hits, actual.hits());
+        assertEquals(expectation.misses, actual.misses());
+        assertEquals(expectation.requests(), actual.requests());
+        // the hit rate computation is vastly different in different implementations;
+        // let's just test that it is being computed
+        assertEquals(expectation.hitRate(), actual.hitRate(), 1);
+        assertEquals(Double.NaN, actual.hitOneMinuteRate(), 0.001d);
+        assertEquals(Double.NaN, actual.hitFiveMinuteRate(), 0.001d);
+        assertEquals(Double.NaN, actual.hitFifteenMinuteRate(), 0.001d);
     }
 
     static CacheMetricsExpectation expect(CacheSize cacheSize)
diff --git a/test/unit/org/apache/cassandra/metrics/CassandraDecayingEstimatedHistogramReservoirTest.java b/test/unit/org/apache/cassandra/metrics/CassandraDecayingEstimatedHistogramReservoirTest.java
new file mode 100644
index 000000000000..c08b3aa21456
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/CassandraDecayingEstimatedHistogramReservoirTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+
+public class CassandraDecayingEstimatedHistogramReservoirTest extends AbstractDecayingEstimatedHistogramReservoirTest
+{
+    @BeforeClass
+    public static void setup()
+    {
+        CassandraRelevantProperties.USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES.setBoolean(false);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsLatenciesTest.java b/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsLatenciesTest.java
new file mode 100644
index 000000000000..d77acc4e6781
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsLatenciesTest.java
@@ -0,0 +1,455 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.db.ConsistencyLevel.ALL;
+import static org.apache.cassandra.db.ConsistencyLevel.ANY;
+import static org.apache.cassandra.db.ConsistencyLevel.LOCAL_QUORUM;
+import static org.apache.cassandra.db.ConsistencyLevel.LOCAL_SERIAL;
+import static org.apache.cassandra.db.ConsistencyLevel.NODE_LOCAL;
+import static org.apache.cassandra.db.ConsistencyLevel.ONE;
+import static org.apache.cassandra.db.ConsistencyLevel.QUORUM;
+import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
+import static org.apache.cassandra.utils.Throwables.merge;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This test verifies (and documents) the relationship between specific queries
+ * and the latency-measuring parts of client request metrics that get updated as a result
+ * of handling such queries.
+ * <p/>
+ * The test is tightly coupled with the {@link ClientRequestsMetrics} class, and the implementation
+ * of user query handling, but the author perceives the documentation value alone is worth that sacrifice
+ * (after all the client request metrics are like a public API).
+ * <p/>
+ * The core of the test is the
+ * {@link #processQueryAndCheckMetricsWereBumped(String, ConsistencyLevel, ClientRequestMetrics...)}
+ * function, which runs the CQL query in a specific CL and then verifies that the specified client
+ * request metrics are bumped (and only them).
+ */
+public class ClientRequestMetricsLatenciesTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(ClientRequestMetricsLatenciesTest.class);
+    private static final String ksName = "client_requests_metrics_test";
+    private static final String cfName = "test_table";
+    private static ClientRequestsMetrics clientMetrics;
+
+    @BeforeClass
+    public static void beforeTest() throws ConfigurationException
+    {
+        CassandraRelevantProperties.ENABLE_NODELOCAL_QUERIES.setBoolean(true);
+
+        SchemaLoader.loadSchema();
+
+        StorageService.instance.initServer(0);
+
+        String cql = String.format("CREATE KEYSPACE IF NOT EXISTS %s " +
+                                   "WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1}",
+                                   ksName);
+        QueryProcessor.process(cql, ONE);
+
+        cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (k int, v1 int, v2 int, PRIMARY KEY (k, v1))", ksName, cfName);
+        QueryProcessor.process(cql, ONE);
+
+        cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s_counter (k int PRIMARY KEY, ct counter)", ksName, cfName);
+        QueryProcessor.process(cql, ONE);
+
+        clientMetrics = ClientRequestsMetricsProvider.instance.metrics(ksName);
+    }
+
+    @Test
+    public void testWriteMetrics()
+    {
+        List.of(ONE, QUORUM, LOCAL_QUORUM, ALL, ANY, NODE_LOCAL).forEach(cl -> {
+            processQueryAndCheckMetricsWereBumped("INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (7, 7, 7);",
+                                                  cl,
+                                                  clientMetrics.writeMetrics,
+                                                  clientMetrics.writeMetricsForLevel(cl));
+            processQueryAndCheckMetricsWereBumped("BEGIN BATCH " +
+                                                  "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 5, 5); " +
+                                                  "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 6, 6); " +
+                                                  "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 8, 8); " +
+                                                  "APPLY BATCH",
+                                                  cl,
+                                                  clientMetrics.writeMetrics,
+                                                  clientMetrics.writeMetricsForLevel(cl));
+        });
+
+        List.of(ONE, QUORUM, LOCAL_QUORUM, ALL, NODE_LOCAL).forEach(cl -> {
+            processQueryAndCheckMetricsWereBumped("UPDATE %1$s.%2$s_counter SET ct = ct + 1 WHERE k=123;",
+                                                  cl,
+                                                  clientMetrics.writeMetrics,
+                                                  clientMetrics.writeMetricsForLevel(cl));
+        });
+    }
+
+    @Test
+    public void testReadMetrics()
+    {
+        List.of(ONE, QUORUM, LOCAL_QUORUM, ALL, NODE_LOCAL).forEach(cl -> {
+            processQueryAndCheckMetricsWereBumped("SELECT * FROM %1$s.%2$s WHERE k=123 and v1=234;",
+                                                  cl,
+                                                  clientMetrics.readMetrics,
+                                                  clientMetrics.readMetricsForLevel(cl));
+        });
+
+        QueryProcessor.process(String.format("UPDATE %1$s.%2$s_counter SET ct = ct + 1 WHERE k=123;", ksName, cfName), ONE);
+        List.of(ONE, QUORUM, LOCAL_QUORUM, ALL, NODE_LOCAL).forEach(cl -> {
+            processQueryAndCheckMetricsWereBumped("SELECT ct FROM %1$s.%2$s_counter WHERE k=123;",
+                                                  cl,
+                                                  clientMetrics.readMetrics,
+                                                  clientMetrics.readMetricsForLevel(cl));
+        });
+    }
+
+    @Test
+    public void testRangeMetrics()
+    {
+        List.of(ONE, QUORUM, LOCAL_QUORUM, ALL).forEach(cl -> {
+            processQueryAndCheckMetricsWereBumped("SELECT * FROM %1$s.%2$s;",
+                                                  cl,
+                                                  clientMetrics.rangeMetrics);
+        });
+    }
+
+    @Test
+    public void testCASWriteMetrics()
+    {
+        List.of(ONE, QUORUM, LOCAL_QUORUM, ALL, ANY).forEach(cl -> {
+            QueryProcessor.process(String.format("TRUNCATE %1$s.%2$s;", ksName, cfName), ALL);
+            processQueryAndCheckMetricsWereBumped("INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (7, 7, 7) IF NOT EXISTS;",
+                                                  cl,
+                                                  clientMetrics.readMetrics,
+                                                  clientMetrics.readMetricsForLevel(QUORUM),
+                                                  clientMetrics.casWriteMetrics,
+                                                  clientMetrics.writeMetricsForLevel(SERIAL));
+            processQueryAndCheckMetricsWereBumped("UPDATE %1$s.%2$s SET v2=123 WHERE k=7 AND v1=7 IF v2=71;",
+                                                  cl,
+                                                  clientMetrics.readMetrics,
+                                                  clientMetrics.readMetricsForLevel(QUORUM),
+                                                  clientMetrics.casWriteMetrics,
+                                                  clientMetrics.writeMetricsForLevel(SERIAL));
+            processQueryAndCheckMetricsWereBumped("UPDATE %1$s.%2$s SET v2=123 WHERE k=7 AND v1=7 IF v2=7;",
+                                                  cl,
+                                                  clientMetrics.readMetrics,
+                                                  clientMetrics.readMetricsForLevel(QUORUM),
+                                                  clientMetrics.casWriteMetrics,
+                                                  clientMetrics.writeMetricsForLevel(SERIAL));
+        });
+    }
+
+    @Test
+    public void testCASReadMetrics()
+    {
+        processQueryAndCheckMetricsWereBumped("SELECT * FROM %1$s.%2$s WHERE k=7 AND v1=7;",
+                                              SERIAL,
+                                              clientMetrics.casReadMetrics,
+                                              clientMetrics.readMetrics,
+                                              clientMetrics.readMetricsForLevel(SERIAL));
+        processQueryAndCheckMetricsWereBumped("SELECT * FROM %1$s.%2$s WHERE k=7 AND v1=7;",
+                                              LOCAL_SERIAL,
+                                              clientMetrics.casReadMetrics,
+                                              clientMetrics.readMetrics,
+                                              clientMetrics.readMetricsForLevel(LOCAL_SERIAL));
+    }
+
+    @Test
+    public void testViewWriteMetrics()
+    {
+        String cql = String.format("CREATE MATERIALIZED VIEW %1$s.example_view AS\n" +
+                                   "SELECT *\n" +
+                                   "FROM %1$s.%2$s\n" +
+                                   "WHERE k IS NOT NULL\n" +
+                                   "  AND v1 IS NOT NULL\n" +
+                                   "  AND v2 IS NOT NULL\n" +
+                                   "PRIMARY KEY (v2, k, v1)\n",
+                                   ksName, cfName);
+        QueryProcessor.process(cql, ONE);
+
+        // Unfortunately, accesses to the system keyspaces are counted as client requests in ClientRequestsMetrics.
+        // To work around this let's wait until the view build is finished and system writes complete.
+        await().atMost(5, TimeUnit.SECONDS).pollDelay(0, TimeUnit.SECONDS).until(
+            () -> QueryProcessor.process(String.format("SELECT status FROM system_distributed.view_build_status WHERE keyspace_name='%1$s' AND view_name='example_view';", ksName),
+                                         ONE)
+                                .one().getString("status").equals("SUCCESS")
+
+        );
+
+        try {
+            List.of(ONE, QUORUM, LOCAL_QUORUM, ALL, ANY).forEach(cl -> {
+                QueryProcessor.process(String.format("TRUNCATE %1$s.%2$s;", ksName, cfName), ALL);
+                processQueryAndCheckMetricsWereBumped("INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (7, 7, 7) IF NOT EXISTS;",
+                                                      cl,
+                                                      clientMetrics.readMetrics,
+                                                      clientMetrics.readMetricsForLevel(QUORUM),
+                                                      clientMetrics.casWriteMetrics,
+                                                      clientMetrics.writeMetricsForLevel(SERIAL),
+                                                      clientMetrics.viewWriteMetrics);
+                processQueryAndCheckMetricsWereBumped("UPDATE %1$s.%2$s SET v2=123 WHERE k=7 AND v1=7 IF v2=7;",
+                                                      cl,
+                                                      clientMetrics.readMetrics,
+                                                      clientMetrics.readMetricsForLevel(QUORUM),
+                                                      clientMetrics.casWriteMetrics,
+                                                      clientMetrics.writeMetricsForLevel(SERIAL),
+                                                      clientMetrics.viewWriteMetrics);
+
+                logger.info("Dont expect metric bump for view write when the view is not updated");
+                processQueryAndCheckMetricsWereBumped("INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (7, 7, 7) IF NOT EXISTS;",
+                                                      cl,
+                                                      clientMetrics.readMetrics,
+                                                      clientMetrics.readMetricsForLevel(QUORUM),
+                                                      clientMetrics.casWriteMetrics,
+                                                      clientMetrics.writeMetricsForLevel(SERIAL));
+                processQueryAndCheckMetricsWereBumped("UPDATE %1$s.%2$s SET v2=123 WHERE k=7 AND v1=7 IF v2=7;",
+                                                      cl,
+                                                      clientMetrics.readMetrics,
+                                                      clientMetrics.readMetricsForLevel(QUORUM),
+                                                      clientMetrics.casWriteMetrics,
+                                                      clientMetrics.writeMetricsForLevel(SERIAL));
+
+            });
+
+            List.of(ONE, QUORUM, LOCAL_QUORUM, ALL, ANY, NODE_LOCAL).forEach(cl -> {
+                QueryProcessor.process(String.format("TRUNCATE %1$s.%2$s;", ksName, cfName), ALL);
+                processQueryAndCheckMetricsWereBumped("INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (7, 7, 7);",
+                                                      cl,
+                                                      clientMetrics.writeMetrics,
+                                                      clientMetrics.writeMetricsForLevel(cl),
+                                                      clientMetrics.viewWriteMetrics);
+                processQueryAndCheckMetricsWereBumped("BEGIN BATCH " +
+                                                      "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 5, 5); " +
+                                                      "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 6, 6); " +
+                                                      "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 8, 8); " +
+                                                      "APPLY BATCH",
+                                                      cl,
+                                                      clientMetrics.writeMetrics,
+                                                      clientMetrics.writeMetricsForLevel(cl),
+                                                      clientMetrics.viewWriteMetrics);
+
+                // expect bump of view write metrics when the view is updated with the same values
+                processQueryAndCheckMetricsWereBumped("INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (7, 7, 7);",
+                                                      cl,
+                                                      clientMetrics.writeMetrics,
+                                                      clientMetrics.writeMetricsForLevel(cl),
+                                                      clientMetrics.viewWriteMetrics);
+                processQueryAndCheckMetricsWereBumped("BEGIN BATCH " +
+                                                      "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 5, 5); " +
+                                                      "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 6, 6); " +
+                                                      "  INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (5, 8, 8); " +
+                                                      "APPLY BATCH",
+                                                      cl,
+                                                      clientMetrics.writeMetrics,
+                                                      clientMetrics.writeMetricsForLevel(cl),
+                                                      clientMetrics.viewWriteMetrics);
+            });
+        }
+        finally
+        {
+            QueryProcessor.process(String.format("DROP MATERIALIZED VIEW IF EXISTS %1$s.example_view", ksName), ONE);
+        }
+    }
+
+    /**
+     With NODE_LOCAL consistency level we don't bump the same metrics as with other consistency levels
+     see {@link ClientRequestMetricsLatenciesTest#testRangeMetrics()} and CASSANDRA-19379.
+     */
+    @Test
+    public void testRangeMetricsWithNODE_LOCAL()
+    {
+        processQueryAndCheckMetricsWereBumped("SELECT * FROM %1$s.%2$s;",
+                                              NODE_LOCAL,
+                                              clientMetrics.readMetrics,
+                                              clientMetrics.readMetricsForLevel(NODE_LOCAL));
+    }
+
+    /**
+      With NODE_LOCAL consistency level we don't bump the same metrics as with other consistency levels
+      see {@link ClientRequestMetricsLatenciesTest#testCASWriteMetrics()} and CASSANDRA-19379.
+    */
+    @Test
+    public void testCASWriteMetricsWithNODE_LOCAL()
+    {
+        QueryProcessor.process(String.format("TRUNCATE %1$s.%2$s;", ksName, cfName), ALL);
+
+        processQueryAndCheckMetricsWereBumped("INSERT INTO %1$s.%2$s (k, v1, v2) VALUES (7, 7, 7) IF NOT EXISTS;",
+                                              NODE_LOCAL,
+                                              clientMetrics.writeMetrics,
+                                              clientMetrics.writeMetricsForLevel(NODE_LOCAL));
+        processQueryAndCheckMetricsWereBumped("UPDATE %1$s.%2$s SET v2=123 WHERE k=7 AND v1=7 IF v2=71;",
+                                              NODE_LOCAL,
+                                              clientMetrics.writeMetrics,
+                                              clientMetrics.writeMetricsForLevel(NODE_LOCAL));
+    }
+
+    private void processQueryAndCheckMetricsWereBumped(String cql, ConsistencyLevel consistencyLevel, ClientRequestMetrics... expectedBumpedMetric)
+    {
+        logger.info("Processing query {} with cl={}", cql, consistencyLevel.name());
+        try
+        {
+            // snapshot metrics prior to query
+            HashMap<NamedMetric, Long> beforeExecution = getMetricsSnapshot(m -> m.executionTimeMetrics);
+            HashMap<NamedMetric, Long> beforeServiceTime = getMetricsSnapshot(m -> m.serviceTimeMetrics);
+
+            // process query
+            QueryProcessor.process(String.format(cql, ksName, cfName), consistencyLevel);
+
+            // check if the metrics were bumped
+            // since MV update is async, let's allow waiting for the metrics to settle
+            await().atMost(5, TimeUnit.SECONDS).pollDelay(0, TimeUnit.SECONDS).untilAsserted(() -> {
+                // get metrics snapshot
+                HashMap<NamedMetric, Long> afterExecution = getMetricsSnapshot(m -> m.executionTimeMetrics);
+                HashMap<NamedMetric, Long> afterServiceTime = getMetricsSnapshot(m -> m.serviceTimeMetrics);
+
+                // compare snapshots; only expect expectedBumpedMetric to be bumped
+                Throwable verificationError = null;
+                for (NamedMetric namedMetric : beforeExecution.keySet())
+                {
+                    try
+                    {
+                        if (Arrays.stream(expectedBumpedMetric).anyMatch(bumped -> namedMetric.metric == bumped))
+                        {
+                            assertBumpOf(namedMetric, beforeExecution, afterExecution);
+                            assertBumpOf(namedMetric, beforeServiceTime, afterServiceTime);
+                        }
+                        else
+                        {
+                            assertNoBumpOf(namedMetric, beforeExecution, afterExecution);
+                            assertNoBumpOf(namedMetric, beforeServiceTime, afterServiceTime);
+                        }
+                    }
+                    catch (AssertionError e)
+                    {
+                        logger.error("Assertion failed for metric {}", namedMetric, e);
+                        verificationError = merge(verificationError, e);
+                    }
+                }
+                if (verificationError != null)
+                {
+                    String errorMessage = String.format("Metrics bumping verification failed for %s, cl %s", cql, consistencyLevel.name());
+                    throw new AssertionError(errorMessage, verificationError);
+                }
+            });
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void assertBumpOf(NamedMetric namedMetric, HashMap<NamedMetric, Long> before, HashMap<NamedMetric, Long> after)
+    {
+        String errorMessage = String.format("Expected bump of %s metric; before processing: %d, after processing: %d", namedMetric.name, before.get(namedMetric), after.get(namedMetric));
+        assertEquals(errorMessage, 1, after.get(namedMetric) - before.get(namedMetric));
+    }
+
+    private void assertNoBumpOf(NamedMetric metric, HashMap<NamedMetric, Long> before, HashMap<NamedMetric, Long> after)
+    {
+        String errorMessage = String.format("Did not expect bump of %s metric; before processing: %d, after processing: %d", metric.name, before.get(metric), after.get(metric));
+        assertEquals(errorMessage, before.get(metric), after.get(metric));
+    }
+
+    private HashMap<NamedMetric, Long> getMetricsSnapshot(Function<ClientRequestMetrics, LatencyMetrics> latencyMetrics) throws IllegalAccessException
+    {
+        // This is brittle etc.
+        // However, I find the ability to verify which scenarios bump which metrics to be very useful,
+        // and I'm willing to pay for it by having this somewhat ugly test method, which may break if
+        // the implementation of ClientRequestsMetrics changes.
+        HashMap<NamedMetric, Long> snapshot = new HashMap<>();
+        for (Field field : ClientRequestsMetrics.class.getDeclaredFields())
+        {
+            field.setAccessible(true);
+            if (ClientRequestMetrics.class.isAssignableFrom(field.getType()))
+            {
+                ClientRequestMetrics requestMetrics = (ClientRequestMetrics) field.get(clientMetrics);
+                snapshot.put(new NamedMetric(requestMetrics, field.getName()),
+                             latencyMetrics.apply(requestMetrics).latency.getCount());
+            }
+            else if (field.getType().equals(Map.class))
+            {
+                Map<ConsistencyLevel, ClientRequestMetrics> clMetrics = (Map<ConsistencyLevel, ClientRequestMetrics>) field.get(clientMetrics);
+                if (clMetrics != null)
+                {
+                    for (ConsistencyLevel cl: clMetrics.keySet())
+                    {
+                        snapshot.put(new NamedMetric(clMetrics.get(cl), String.format("%s[%s]", field.getName(), cl.name())),
+                                     latencyMetrics.apply(clMetrics.get(cl)).latency.getCount());
+                    }
+                }
+            }
+        }
+        return snapshot;
+    }
+
+    private static class NamedMetric
+    {
+        private final ClientRequestMetrics metric;
+        private final String name;
+
+        public NamedMetric(ClientRequestMetrics metric, String name)
+        {
+            this.metric = metric;
+            this.name = name;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "NamedMetric{" +
+                   "name='" + name + '\'' +
+                   '}';
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NamedMetric that = (NamedMetric) o;
+            return Objects.equals(name, that.name);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(name);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsTest.java b/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsTest.java
index b4e9b085d2f5..7f614fcf4712 100644
--- a/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsTest.java
@@ -34,7 +34,6 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.service.EmbeddedCassandraService;
-import org.apache.cassandra.service.reads.range.RangeCommandIterator;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
@@ -54,8 +53,8 @@ public class ClientRequestMetricsTest
     private static PreparedStatement readPS;
     private static PreparedStatement readRangePS;
 
-    private static final ClientRequestMetrics readMetrics = ClientRequestsMetricsHolder.readMetrics;
-    private static final ClientWriteRequestMetrics writeMetrics = ClientRequestsMetricsHolder.writeMetrics;
+    private static ClientRequestMetrics readMetrics;
+    private static ClientWriteRequestMetrics writeMetrics;
 
     private static EmbeddedCassandraService cassandra;
 
@@ -75,6 +74,10 @@ public static void setup() throws ConfigurationException, IOException
         paxosPS = session.prepare("INSERT INTO " + KEYSPACE + '.' + TABLE + " (id, ord, val) VALUES (?, ?, ?) IF NOT EXISTS;");
         readPS = session.prepare("SELECT * FROM " + KEYSPACE + '.' + TABLE + " WHERE id=?;");
         readRangePS = session.prepare("SELECT * FROM " + KEYSPACE + '.' + TABLE + " WHERE id=? AND ord>=? AND ord <= ?;");
+
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics(KEYSPACE);
+        readMetrics = metrics.readMetrics;
+        writeMetrics = metrics.writeMetrics;
     }
 
     @AfterClass
@@ -168,14 +171,15 @@ public void testRangeStatement()
     @Test
     public void testRangeRead() throws Throwable
     {
-        clearHistogram(RangeCommandIterator.rangeMetrics.roundTrips);
-        long latencyCount = RangeCommandIterator.rangeMetrics.latency.getCount();
+        ClientRequestsMetrics metrics = ClientRequestsMetricsProvider.instance.metrics("system");
+        clearHistogram(metrics.rangeMetrics.roundTrips);
+        long latencyCount = metrics.rangeMetrics.executionTimeMetrics.latency.getCount();
 
         session.execute("SELECT * FROM system.peers");
 
-        assertThat(RangeCommandIterator.rangeMetrics.roundTrips.getCount()).isGreaterThan(0);
-        assertThat(RangeCommandIterator.rangeMetrics.roundTrips.getSnapshot().getMax()).isEqualTo(1);
-        assertThat(RangeCommandIterator.rangeMetrics.latency.getCount()).isEqualTo(latencyCount + 1);
+        assertThat(metrics.rangeMetrics.roundTrips.getCount()).isGreaterThan(0);
+        assertThat(metrics.rangeMetrics.roundTrips.getSnapshot().getMax()).isEqualTo(1);
+        assertThat(metrics.rangeMetrics.executionTimeMetrics.latency.getCount()).isEqualTo(latencyCount + 1);
     }
 
     private void clearHistogram(Histogram histogram)
diff --git a/test/unit/org/apache/cassandra/metrics/ClientRequestsMetricsTest.java b/test/unit/org/apache/cassandra/metrics/ClientRequestsMetricsTest.java
new file mode 100644
index 000000000000..4ae2dcf18809
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/ClientRequestsMetricsTest.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+public class ClientRequestsMetricsTest
+{
+    private static ClientRequestsMetrics c1;
+    private static ClientRequestsMetrics c2;
+
+    @BeforeClass
+    public static void init()
+    {
+        DatabaseDescriptor.daemonInitialization();
+
+        c1 = new ClientRequestsMetrics("tenant1");
+        c2 = new ClientRequestsMetrics("tenant2");
+    }
+
+    @AfterClass
+    public static void teardown()
+    {
+        releaseAll(c1);
+        releaseAll(c2);
+    }
+
+    protected static void releaseAll(ClientRequestsMetrics ccrm)
+    {
+        ccrm.readMetrics.release();
+        ccrm.rangeMetrics.release();
+        ccrm.writeMetrics.release();
+        ccrm.casWriteMetrics.release();
+        ccrm.casReadMetrics.release();
+        ccrm.viewWriteMetrics.release();
+        for (ConsistencyLevel level : ConsistencyLevel.values())
+        {
+            ccrm.readMetricsForLevel(level).release();
+            ccrm.writeMetricsForLevel(level).release();
+        }
+    }
+
+    @Test
+    public void testDefaultMetrics()
+    {
+        ClientRequestsMetricsProvider defaultMetricsProvider = ClientRequestsMetricsProvider.instance;
+        assertThat(defaultMetricsProvider).isInstanceOf(ClientRequestsMetricsProvider.DefaultClientRequestsMetricsProvider.class);
+
+        ClientRequestsMetrics defaultMetrics = defaultMetricsProvider.metrics("");
+        assertThat(defaultMetrics).isInstanceOf(ClientRequestsMetrics.class);
+    }
+
+    @Test
+    public void testReadMetrics()
+    {
+        // update tenant1 read metrics, tenant2 metrics remain 0
+        updateClientRequestMetrics(c1.readMetrics);
+        verifyClientRequestMetrics(c1.readMetrics, 1);
+        verifyClientRequestMetrics(c2.readMetrics, 0);
+
+        // update tenant2 read metrics, tenant1 metrics remain 1
+        updateClientRequestMetrics(c2.readMetrics);
+        verifyClientRequestMetrics(c1.readMetrics, 1);
+        verifyClientRequestMetrics(c2.readMetrics, 1);
+    }
+
+    @Test
+    public void testRangeMetrics()
+    {
+        // update tenant1 range metrics, tenant2 metrics remain 0
+        updateClientRangeRequestMetrics(c1.rangeMetrics);
+        verifyClientRangeRequestMetrics(c1.rangeMetrics, 1);
+        verifyClientRangeRequestMetrics(c2.rangeMetrics, 0);
+
+        // update tenant2 range metrics, tenant1 metrics remain 1
+        updateClientRangeRequestMetrics(c2.rangeMetrics);
+        verifyClientRangeRequestMetrics(c1.rangeMetrics, 1);
+        verifyClientRangeRequestMetrics(c2.rangeMetrics, 1);
+    }
+
+    @Test
+    public void testWriteMetrics()
+    {
+        // update tenant1 write metrics, tenant2 metrics remain 0
+        updateClientWriteRequestMetrics(c1.writeMetrics);
+        verifyClientWriteRequestMetrics(c1.writeMetrics, 1);
+        verifyClientWriteRequestMetrics(c2.writeMetrics, 0);
+
+        // update tenant2 write metrics, tenant1 metrics remain 1
+        updateClientWriteRequestMetrics(c2.writeMetrics);
+        verifyClientWriteRequestMetrics(c1.writeMetrics, 1);
+        verifyClientWriteRequestMetrics(c2.writeMetrics, 1);
+    }
+
+    @Test
+    public void testCasReadMetrics()
+    {
+        // update tenant1 cas metrics, tenant2 metrics remain 0
+        updateCASClientRequestMetrics(c1.casReadMetrics);
+        verifyCASClientRequestMetrics(c1.casReadMetrics, 1);
+        verifyCASClientRequestMetrics(c2.casReadMetrics, 0);
+
+        // update tenant2 cas metrics, tenant1 metrics remain 1
+        updateCASClientRequestMetrics(c2.casReadMetrics);
+        verifyCASClientRequestMetrics(c1.casReadMetrics, 1);
+        verifyCASClientRequestMetrics(c2.casReadMetrics, 1);
+    }
+
+    @Test
+    public void testViewWriteMetrics()
+    {
+        // update tenant1 view metrics, tenant2 metrics remain 0
+        updateViewWriteMetrics(c1.viewWriteMetrics);
+        verifyViewWriteMetrics(c1.viewWriteMetrics, 1);
+        verifyViewWriteMetrics(c2.viewWriteMetrics, 0);
+
+        // update tenant2 view metrics, tenant1 metrics remain 1
+        updateViewWriteMetrics(c2.viewWriteMetrics);
+        verifyViewWriteMetrics(c1.viewWriteMetrics, 1);
+        verifyViewWriteMetrics(c2.viewWriteMetrics, 1);
+    }
+
+    @Test
+    public void testCasWriteMetrics()
+    {
+        // update tenant1 cas write metrics, tenant2 metrics remain 0
+        updateCASClientWriteRequestMetrics(c1.casWriteMetrics);
+        verifyCASClientWriteRequestMetrics(c1.casWriteMetrics, 1);
+        verifyCASClientWriteRequestMetrics(c2.casWriteMetrics, 0);
+
+        // update tenant2 cas write metrics, tenant1 metrics remain 1
+        updateCASClientWriteRequestMetrics(c2.casWriteMetrics);
+        verifyCASClientWriteRequestMetrics(c1.casWriteMetrics, 1);
+        verifyCASClientWriteRequestMetrics(c2.casWriteMetrics, 1);
+    }
+
+    @Test
+    public void testReadMetricsMap()
+    {
+        for (ConsistencyLevel level : ConsistencyLevel.values())
+        {
+            // update tenant1 view metrics, tenant2 metrics remain 0
+            updateClientRequestMetrics(c1.readMetricsForLevel(level));
+            verifyClientRequestMetrics(c1.readMetricsForLevel(level), 1);
+            verifyClientRequestMetrics(c2.readMetricsForLevel(level), 0);
+
+            // update tenant2 view metrics, tenant1 metrics remain 1
+            updateClientRequestMetrics(c2.readMetricsForLevel(level));
+            verifyClientRequestMetrics(c1.readMetricsForLevel(level), 1);
+            verifyClientRequestMetrics(c2.readMetricsForLevel(level), 1);
+        }
+    }
+
+    @Test
+    public void testWriteMetricsMap()
+    {
+        for (ConsistencyLevel level : ConsistencyLevel.values())
+        {
+            // update tenant1 view metrics, tenant2 metrics remain 0
+            updateClientWriteRequestMetrics(c1.writeMetricsForLevel(level));
+            verifyClientWriteRequestMetrics(c1.writeMetricsForLevel(level), 1);
+            verifyClientWriteRequestMetrics(c2.writeMetricsForLevel(level), 0);
+
+            // update tenant2 view metrics, tenant1 metrics remain 1
+            updateClientWriteRequestMetrics(c2.writeMetricsForLevel(level));
+            verifyClientWriteRequestMetrics(c1.writeMetricsForLevel(level), 1);
+            verifyClientWriteRequestMetrics(c2.writeMetricsForLevel(level), 1);
+        }
+    }
+
+    private void updateViewWriteMetrics(ViewWriteMetrics metrics)
+    {
+        metrics.viewWriteLatency.update(1, TimeUnit.MILLISECONDS);
+        metrics.viewReplicasSuccess.inc();
+        metrics.viewReplicasAttempted.inc(2);
+        updateClientRequestMetrics(metrics);
+    }
+
+    private void updateCASClientWriteRequestMetrics(CASClientWriteRequestMetrics metrics)
+    {
+        metrics.conditionNotMet.inc();
+        metrics.mutationSize.update(1);
+        updateCASClientRequestMetrics(metrics);
+    }
+
+    private void updateCASClientRequestMetrics(CASClientRequestMetrics metrics)
+    {
+        metrics.unfinishedCommit.inc();
+        metrics.contention.update(1);
+        metrics.unknownResult.mark();
+        updateLatencyMetrics(metrics.prepareLatency);
+        updateLatencyMetrics(metrics.createProposalLatency);
+        updateLatencyMetrics(metrics.proposeLatency);
+        updateLatencyMetrics(metrics.commitLatency);
+        updateLatencyMetrics(metrics.contentionBackoffLatency);
+        metrics.missingMostRecentCommit.inc();
+        updateClientRequestMetrics(metrics);
+    }
+
+    private void updateClientRangeRequestMetrics(ClientRangeRequestMetrics metrics)
+    {
+        metrics.roundTrips.update(1);
+        updateClientRequestMetrics(metrics);
+    }
+
+    private void updateClientWriteRequestMetrics(ClientWriteRequestMetrics metrics)
+    {
+        metrics.mutationSize.update(1);
+        updateClientRequestMetrics(metrics);
+    }
+
+    private void updateClientRequestMetrics(ClientRequestMetrics metrics)
+    {
+        metrics.timeouts.mark();
+        metrics.failures.mark();
+        metrics.unavailables.mark();
+        updateLatencyMetrics(metrics.executionTimeMetrics);
+        updateLatencyMetrics(metrics.serviceTimeMetrics);
+    }
+
+    private void updateLatencyMetrics(LatencyMetrics metrics)
+    {
+        metrics.latency.update(1, TimeUnit.MILLISECONDS);
+        metrics.totalLatency.inc();
+    }
+
+    private void verifyViewWriteMetrics(ViewWriteMetrics metrics, int value)
+    {
+        assertEquals(value == 0 ? 0 : value + 1, metrics.viewReplicasAttempted.getCount());
+        assertEquals(value, metrics.viewReplicasSuccess.getCount());
+        assertEquals(value == 0 ? 0 : 1, metrics.viewPendingMutations.getValue().intValue());
+        assertEquals(value, metrics.viewWriteLatency.getCount());
+        verifyClientRequestMetrics(metrics, value);
+    }
+
+    private void verifyCASClientWriteRequestMetrics(CASClientWriteRequestMetrics metrics, int value)
+    {
+        assertEquals(value, metrics.mutationSize.getCount());
+        assertEquals(value, metrics.conditionNotMet.getCount());
+        verifyCASClientRequestMetrics(metrics, value);
+    }
+
+    private void verifyCASClientRequestMetrics(CASClientRequestMetrics metrics, int value)
+    {
+        assertEquals(value, metrics.unfinishedCommit.getCount());
+        assertEquals(value, metrics.contention.getCount());
+        assertEquals(value, metrics.unknownResult.getCount());
+        verifyLatencyMetrics(metrics.prepareLatency, value);
+        verifyLatencyMetrics(metrics.createProposalLatency, value);
+        verifyLatencyMetrics(metrics.proposeLatency, value);
+        verifyLatencyMetrics(metrics.commitLatency, value);
+        verifyLatencyMetrics(metrics.contentionBackoffLatency, value);
+        assertEquals(value, metrics.missingMostRecentCommit.getCount());
+        verifyClientRequestMetrics(metrics, value);
+    }
+
+    private void verifyClientRangeRequestMetrics(ClientRangeRequestMetrics metrics, int value)
+    {
+        assertEquals(value, metrics.roundTrips.getCount());
+        verifyClientRequestMetrics(metrics, value);
+    }
+
+    private void verifyClientWriteRequestMetrics(ClientWriteRequestMetrics metrics, int value)
+    {
+        assertEquals(value, metrics.mutationSize.getCount());
+        verifyClientRequestMetrics(metrics, value);
+    }
+
+    private void verifyClientRequestMetrics(ClientRequestMetrics metrics, int value)
+    {
+        assertEquals(value, metrics.timeouts.getCount());
+        assertEquals(value, metrics.failures.getCount());
+        assertEquals(value, metrics.unavailables.getCount());
+        verifyLatencyMetrics(metrics.executionTimeMetrics, value);
+        verifyLatencyMetrics(metrics.serviceTimeMetrics, value);
+    }
+
+    private void verifyLatencyMetrics(LatencyMetrics metrics, int value)
+    {
+        assertEquals(value, metrics.latency.getCount());
+        assertEquals(value, metrics.totalLatency.getCount());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/CodahaleChunkCacheMetricsTest.java b/test/unit/org/apache/cassandra/metrics/CodahaleChunkCacheMetricsTest.java
new file mode 100644
index 000000000000..641229a80b59
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/CodahaleChunkCacheMetricsTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.github.benmanes.caffeine.cache.stats.CacheStats;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.assertj.core.api.Assertions;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class CodahaleChunkCacheMetricsTest
+{
+    private ChunkCache mockChunkCache;
+
+    private ChunkCacheMetrics chunkCacheMetrics;
+
+    @BeforeClass
+    public static void init()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Before
+    public void before()
+    {
+        mockChunkCache = Mockito.mock(ChunkCache.class);
+
+        // Make sure to not use micrometer metrics
+        CassandraRelevantProperties.USE_MICROMETER.setBoolean(false);
+
+        chunkCacheMetrics = ChunkCacheMetrics.create(mockChunkCache);
+        assertTrue(chunkCacheMetrics instanceof CodahaleChunkCacheMetrics);
+    }
+
+    @After
+    public void after()
+    {
+        chunkCacheMetrics.reset();
+        System.clearProperty("cassandra.use_micrometer_metrics");
+    }
+
+    @Test
+    public void testHitRate()
+    {
+        chunkCacheMetrics.recordHits(90);
+        assertEquals(90, chunkCacheMetrics.hits());
+
+        chunkCacheMetrics.recordMisses(10);
+        assertEquals(10, chunkCacheMetrics.misses());
+
+        // Verify requests = hits + misses
+        assertEquals(100, chunkCacheMetrics.requests());
+
+        // Verify hitrate
+        assertEquals(0.9, chunkCacheMetrics.hitRate(), 0.0);
+    }
+
+    @Test
+    public void testCommonChunkCacheMetrics()
+    {
+        // No-op
+        chunkCacheMetrics.recordEviction();
+
+        // No-op
+        chunkCacheMetrics.recordLoadFailure(25);
+
+        chunkCacheMetrics.recordLoadSuccess(TimeUnit.MILLISECONDS.toNanos(15));
+
+        assertEquals(0.0, chunkCacheMetrics.missLatency(), 0.0);
+
+        // Cache size was statically initialized 
+        assertEquals(ChunkCache.instance.capacity(), chunkCacheMetrics.capacity());
+
+        assertEquals(0, chunkCacheMetrics.size());
+
+        assertEquals(0, chunkCacheMetrics.entries());
+
+        assertEquals(0, chunkCacheMetrics.requestsFifteenMinuteRate(), 0.1);
+
+        assertTrue(Double.isNaN(chunkCacheMetrics.hitFifteenMinuteRate()));
+
+        CacheStats snapshot = chunkCacheMetrics.snapshot();
+        assertNotNull(snapshot);
+        assertEquals(chunkCacheMetrics.hits(), snapshot.hitCount());
+        assertEquals(chunkCacheMetrics.misses(), snapshot.missCount());
+
+        String toString = chunkCacheMetrics.toString();
+        assertNotNull(toString);
+        Assertions.assertThat(toString).contains("Capacity:");
+    }
+
+    @Test
+    public void testReset()
+    {
+        chunkCacheMetrics.recordHits(90);
+        assertEquals(90, chunkCacheMetrics.hits());
+        chunkCacheMetrics.recordMisses(10);
+        assertEquals(10, chunkCacheMetrics.misses());
+
+        chunkCacheMetrics.reset();
+
+        assertEquals(0, chunkCacheMetrics.hits());
+        assertEquals(0, chunkCacheMetrics.misses());
+        assertEquals(0, chunkCacheMetrics.requests());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/CounterMutationMetricsTest.java b/test/unit/org/apache/cassandra/metrics/CounterMutationMetricsTest.java
new file mode 100644
index 000000000000..46c58dc9274b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/CounterMutationMetricsTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.Mutation.PartitionUpdateCollector;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.counters.CounterLockManager;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class CounterMutationMetricsTest
+{
+    private static final String KEYSPACE1 = "CounterMutationMetricsTest";
+    private static final String CF1 = "Counter1";
+    private static final String CF2 = "Counter2";
+    private static long LOCK_TIMEOUT_MILLIS;
+
+    @BeforeClass
+    public static void defineSchema() throws ConfigurationException
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.counterCFMD(KEYSPACE1, CF1),
+                                    SchemaLoader.counterCFMD(KEYSPACE1, CF2));
+        LOCK_TIMEOUT_MILLIS = DatabaseDescriptor.getCounterWriteRpcTimeout(TimeUnit.MILLISECONDS);
+    }
+
+    @After
+    public void after()
+    {
+        long currentTimeout = DatabaseDescriptor.getCounterWriteRpcTimeout(TimeUnit.MILLISECONDS);
+        if (currentTimeout != LOCK_TIMEOUT_MILLIS)
+            DatabaseDescriptor.setCounterWriteRpcTimeout(LOCK_TIMEOUT_MILLIS);
+    }
+
+    @Test
+    public void testLocksMetrics()
+    {
+        AssertMetrics metrics = new AssertMetrics();
+
+        ColumnFamilyStore cfsOne = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1);
+        ColumnFamilyStore cfsTwo = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF2);
+
+        RowUpdateBuilder mutationBuilder1 = new RowUpdateBuilder(cfsOne.metadata(), 5, "key1");
+        Mutation mutation1 = mutationBuilder1.clustering("cc")
+                                             .add("val", 1L)
+                                             .add("val2", -1L)
+                                             .build();
+        RowUpdateBuilder mutationBuilder2 = new RowUpdateBuilder(cfsTwo.metadata(), 5, "key1");
+        Mutation mutation2 = mutationBuilder2.clustering("cc")
+                                             .add("val", 2L)
+                                             .add("val2", -2L)
+                                             .build();
+
+        PartitionUpdateCollector batch = new PartitionUpdateCollector(KEYSPACE1, Util.dk("key1"));
+        batch.add(mutation1.getPartitionUpdate(cfsOne.metadata()));
+        batch.add(mutation2.getPartitionUpdate(cfsTwo.metadata()));
+
+        new CounterMutation(batch.build(), ConsistencyLevel.ONE).apply();
+
+        metrics.assertLockTimeout(0);
+        metrics.assertLocksPerUpdate(1);
+    }
+
+    private void obtainLocks(TableMetadata metadata, List<CounterLockManager.LockHandle> lockHandles)
+    {
+        RowUpdateBuilder mutationBuilder = new RowUpdateBuilder(metadata, 5, "key1");
+        Mutation mutation = mutationBuilder.clustering("cc")
+                                           .add("val", 1L)
+                                           .build();
+        CounterMutation counterMutation = new CounterMutation(mutation , ConsistencyLevel.ONE);
+        counterMutation.grabCounterLocks(Keyspace.open(KEYSPACE1), lockHandles);
+    }
+
+    private void tryMutate(TableMetadata metadata)
+    {
+        RowUpdateBuilder mutateBuilder = new RowUpdateBuilder(metadata, 5, "key1");
+        Mutation mutation = mutateBuilder.clustering("cc")
+                                         .add("val", 1L)
+                                         .add("val2", -1L)
+                                         .build();
+        boolean failedMutate = false;
+        try
+        {
+            new CounterMutation(mutation, ConsistencyLevel.ONE).apply();
+        }
+        catch (WriteTimeoutException e)
+        {
+            failedMutate = true;
+        }
+        assertTrue("Expected to fail apply mutation", failedMutate);
+    }
+
+    @Test
+    public void testLockTimeout() throws Throwable
+    {
+        final long timeoutMillis = 11;
+        final long biggerThanTimeoutMillis = 1000;
+        AssertMetrics metrics = new AssertMetrics(timeoutMillis);
+
+        TableMetadata metadata = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1).metadata();
+
+        List<CounterLockManager.LockHandle> obtainedLockHandles = new ArrayList<>();
+        try
+        {
+            obtainLocks(metadata, obtainedLockHandles);
+            metrics.assertLockTimeout(0);
+
+            CompletableFuture.runAsync(() -> tryMutate(metadata))
+                             .get(biggerThanTimeoutMillis, TimeUnit.MILLISECONDS);
+        }
+        finally
+        {
+            for (CounterLockManager.LockHandle lockHandle : obtainedLockHandles)
+                lockHandle.release();
+        }
+
+        metrics.assertLockTimeout(1);
+        metrics.assertLocksPerUpdate(2);
+    }
+
+    public class AssertMetrics
+    {
+        public final long lockTimeoutMilis;
+        public final long prevLockTimeoutCount;
+        public final long prevMaxLockAcquireTime;
+
+        public AssertMetrics()
+        {
+            this(LOCK_TIMEOUT_MILLIS);
+        }
+
+        public AssertMetrics(long timeoutInMilis)
+        {
+            clear();
+            prevLockTimeoutCount = CounterMutation.lockTimeout.getCount();
+            prevMaxLockAcquireTime = CounterMutation.lockAcquireTime.latency.getSnapshot().getMax();
+
+            this.lockTimeoutMilis = timeoutInMilis;
+            if (timeoutInMilis != LOCK_TIMEOUT_MILLIS)
+                DatabaseDescriptor.setCounterWriteRpcTimeout(lockTimeoutMilis);
+        }
+
+        public void clear()
+        {
+            ((ClearableHistogram)CounterMutation.locksPerUpdate).clear();
+        }
+
+        public void assertLockTimeout(long expected)
+        {
+            assertEquals(prevLockTimeoutCount + expected, CounterMutation.lockTimeout.getCount());
+
+            long maxLockAcquireTime = CounterMutation.lockAcquireTime.latency.getSnapshot().getMax();
+            if (expected > 0)
+                assertTrue(lockTimeoutMilis <= maxLockAcquireTime);
+            else
+                assertTrue(lockTimeoutMilis > maxLockAcquireTime
+                           || prevMaxLockAcquireTime <= maxLockAcquireTime);
+        }
+
+        public void assertLocksPerUpdate(long expectedCount)
+        {
+            assertEquals(expectedCount, CounterMutation.locksPerUpdate.getCount());
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/CustomCoordinatorMetricsTest.java b/test/unit/org/apache/cassandra/metrics/CustomCoordinatorMetricsTest.java
new file mode 100644
index 000000000000..74934c0433a8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/CustomCoordinatorMetricsTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CustomCoordinatorMetricsTest
+{
+    static String oldValueCustomProvider = null;
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        oldValueCustomProvider = CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY.getString();
+        // Sets custom client provider class used in {@code ClientRequestsMetricsHolderProvider#instance}
+        CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY.setString(CustomClientRequestsMetricsProvider.class.getName());
+    }
+
+    @AfterClass
+    public static void teardown()
+    {
+        if (oldValueCustomProvider != null)
+            CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY.setString(oldValueCustomProvider);
+        else
+            System.clearProperty(CUSTOM_CLIENT_REQUEST_METRICS_PROVIDER_PROPERTY.getKey());
+    }
+
+    @Test
+    public void testStaticInstanceWithCustomProviderClassName()
+    {
+        assertThat(ClientRequestsMetricsProvider.instance).isInstanceOf(CustomClientRequestsMetricsProvider.class);
+    }
+
+    public static class CustomClientRequestsMetricsProvider implements ClientRequestsMetricsProvider
+    {
+        @Override
+        public ClientRequestsMetrics metrics(String keyspace)
+        {
+            return null;
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/DSEDecayingEstimatedHistogramReservoirTest.java b/test/unit/org/apache/cassandra/metrics/DSEDecayingEstimatedHistogramReservoirTest.java
new file mode 100644
index 000000000000..07972c4486e6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/DSEDecayingEstimatedHistogramReservoirTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+
+public class DSEDecayingEstimatedHistogramReservoirTest extends AbstractDecayingEstimatedHistogramReservoirTest
+{
+    @BeforeClass
+    public static void setup()
+    {
+        CassandraRelevantProperties.USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES.setBoolean(true);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java b/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java
deleted file mode 100644
index cd7077bf1648..000000000000
--- a/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java
+++ /dev/null
@@ -1,778 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.metrics;
-
-import java.io.ByteArrayOutputStream;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.LockSupport;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.experimental.runners.Enclosed;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.codahale.metrics.Snapshot;
-import org.apache.cassandra.utils.EstimatedHistogram;
-import org.apache.cassandra.utils.MonotonicClock;
-import org.apache.cassandra.utils.MonotonicClockTranslation;
-import org.apache.cassandra.utils.Pair;
-import org.quicktheories.core.Gen;
-
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.apache.cassandra.metrics.DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_NS;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.quicktheories.QuickTheory.qt;
-import static org.quicktheories.generators.SourceDSL.booleans;
-import static org.quicktheories.generators.SourceDSL.integers;
-import static org.quicktheories.generators.SourceDSL.longs;
-
-@RunWith(Enclosed.class)
-public class DecayingEstimatedHistogramReservoirTest
-{
-    public static final Logger logger = LoggerFactory.getLogger(DecayingEstimatedHistogramReservoirTest.class);
-    public static class NonParameterizedTests
-    {
-        public static final int numExamples = 1000000;
-
-        public static final Gen<long[]> offsets = integers().from(DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT)
-                                                            .upToAndIncluding(DecayingEstimatedHistogramReservoir.MAX_BUCKET_COUNT - 10)
-                                                            .zip(booleans().all(), EstimatedHistogram::newOffsets);
-        
-        @Test
-        public void testFindIndex()
-        {
-            qt().withExamples(numExamples)
-                .forAll(booleans().all()
-                                  .flatMap(b -> offsets.flatMap(offs -> this.offsetsAndValue(offs, b, 0))))
-                .check(this::checkFindIndex);
-        }
-
-        @Test
-        public void showEstimationWorks()
-        {
-            qt().withExamples(numExamples)
-                .forAll(offsets.flatMap(offs -> this.offsetsAndValue(offs, false, 9)))
-                .check(this::checkEstimation);
-        }
-
-        //shows that the max before overflow is 238 buckets regardless of consider zeros
-        @Test
-        @Ignore
-        public void showHistogramOffsetOverflow()
-        {
-            qt().forAll(integers().from(DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT).upToAndIncluding(1000))
-                .check(count -> {
-                    long[] offsets = EstimatedHistogram.newOffsets(count, false);
-                    for (long offset : offsets)
-                        if (offset < 0)
-                            return false;
-
-                    return true;
-                });
-        }
-
-        private boolean checkFindIndex(Pair<long[], Long> offsetsAndValue)
-        {
-            long[] offsets = offsetsAndValue.left;
-            long value = offsetsAndValue.right;
-
-            int model = findIndexModel(offsets, value);
-            int actual = DecayingEstimatedHistogramReservoir.findIndex(offsets, value);
-
-            return model == actual;
-        }
-
-        private int findIndexModel(long[] offsets, long value)
-        {
-            int modelIndex = Arrays.binarySearch(offsets, value);
-            if (modelIndex < 0)
-                modelIndex = -modelIndex - 1;
-
-            return modelIndex;
-        }
-
-        private Gen<Pair<long[], Long>> offsetsAndValue(long[] offsets, boolean useMaxLong, long minValue)
-        {
-            return longs().between(minValue, useMaxLong ? Long.MAX_VALUE : offsets[offsets.length - 1] + 100)
-                          .mix(longs().between(minValue, minValue + 10),50)
-                          .map(value -> Pair.create(offsets, value));
-        }
-
-        public boolean checkEstimation(Pair<long[], Long> offsetsAndValue)
-        {
-            long[] offsets = offsetsAndValue.left;
-            long value = offsetsAndValue.right;
-            boolean considerZeros = offsets[0] == 0;
-
-            int modelIndex = Arrays.binarySearch(offsets, value);
-            if (modelIndex < 0)
-                modelIndex = -modelIndex - 1;
-
-            int estimate = (int) DecayingEstimatedHistogramReservoir.fastLog12(value);
-
-            if (considerZeros)
-                return estimate - 3 == modelIndex || estimate - 2 == modelIndex;
-            else
-                return estimate - 4 == modelIndex || estimate - 3 == modelIndex;
-        }
-    }
-
-    @RunWith(Parameterized.class)
-    public static class ParameterizedTests
-    {
-        private static final double DOUBLE_ASSERT_DELTA = 0;
-
-        @Parameterized.Parameter
-        public String description;
-        
-        @Parameterized.Parameter(1)
-        public Function<DecayingEstimatedHistogramReservoir, Snapshot> toSnapshot;
-
-        @Parameterized.Parameters(name="{0}")
-        public static Collection<Object[]> suppliers()
-        {
-            Function<DecayingEstimatedHistogramReservoir, Snapshot> snapshot = DecayingEstimatedHistogramReservoir::getSnapshot;
-            Function<DecayingEstimatedHistogramReservoir, Snapshot> decayingOnly = DecayingEstimatedHistogramReservoir::getPercentileSnapshot;
-            return ImmutableList.of(new Object[] { "normal", snapshot }, new Object[] { "decaying buckets", decayingOnly });
-        }
-
-        @Test
-        public void testStriping() throws InterruptedException
-        {
-            TestClock clock = new TestClock();
-            int nStripes = 4;
-            DecayingEstimatedHistogramReservoir model = new DecayingEstimatedHistogramReservoir(clock);
-            DecayingEstimatedHistogramReservoir test = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION,
-                                                                                               DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
-                                                                                               nStripes,
-                                                                                               clock);
-
-            long seed = nanoTime();
-            System.out.println("DecayingEstimatedHistogramReservoirTest#testStriping.seed = " + seed);
-            Random valGen = new Random(seed);
-            ExecutorService executors = Executors.newFixedThreadPool(nStripes * 2);
-            for (int i = 0; i < 1_000_000; i++)
-            {
-                long value = Math.abs(valGen.nextInt());
-                executors.submit(() -> {
-                    model.update(value);
-                    LockSupport.parkNanos(2);
-                    test.update(value);
-                });
-            }
-
-            executors.shutdown();
-            Assert.assertTrue(executors.awaitTermination(1, TimeUnit.MINUTES));
-
-            Snapshot modelSnapshot = toSnapshot.apply(model);
-            Snapshot testSnapshot = toSnapshot.apply(test);
-
-            assertEquals(modelSnapshot.getMean(), testSnapshot.getMean(), DOUBLE_ASSERT_DELTA);
-            assertEquals(modelSnapshot.getMin(), testSnapshot.getMin(), DOUBLE_ASSERT_DELTA);
-            assertEquals(modelSnapshot.getMax(), testSnapshot.getMax(), DOUBLE_ASSERT_DELTA);
-            assertEquals(modelSnapshot.getMedian(), testSnapshot.getMedian(), DOUBLE_ASSERT_DELTA);
-            for (double i = 0.0; i < 1.0; i += 0.1)
-                assertEquals(modelSnapshot.getValue(i), testSnapshot.getValue(i), DOUBLE_ASSERT_DELTA);
-
-
-            int stripedValues = 0;
-            for (int i = model.size(); i < model.size() * model.stripeCount(); i++)
-            {
-                stripedValues += model.stripedBucketValue(i, true);
-            }
-            assertTrue("no striping found", stripedValues > 0);
-        }
-
-        @Test
-        public void testSimple()
-        {
-            {
-                // 0 and 1 map to the same, first bucket
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir();
-                histogram.update(0);
-                assertEquals(1, histogram.getSnapshot().getValues()[0]);
-                histogram.update(1);
-                assertEquals(2, histogram.getSnapshot().getValues()[0]);
-            }
-            {
-                // 0 and 1 map to different buckets
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true);
-                histogram.update(0);
-                assertEquals(1, histogram.getSnapshot().getValues()[0]);
-                histogram.update(1);
-                Snapshot snapshot = histogram.getSnapshot();
-                assertEquals(1, snapshot.getValues()[0]);
-                assertEquals(1, snapshot.getValues()[1]);
-            }
-        }
-
-        @Test
-        public void testOverflow()
-        {
-            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, 1, 1);
-            histogram.update(100);
-            assert histogram.isOverflowed();
-            assertEquals(Long.MAX_VALUE, toSnapshot.apply(histogram).getMax());
-        }
-
-        @Test
-        public void testMinMax()
-        {
-            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir();
-            histogram.update(16);
-            Snapshot snapshot = toSnapshot.apply(histogram);
-            assertEquals(15, snapshot.getMin());
-            assertEquals(17, snapshot.getMax());
-        }
-
-        @Test
-        public void testMean()
-        {
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-                for (int i = 0; i < 40; i++)
-                    histogram.update(0);
-                for (int i = 0; i < 20; i++)
-                    histogram.update(1);
-                for (int i = 0; i < 10; i++)
-                    histogram.update(2);
-                assertEquals(1.14D, toSnapshot.apply(histogram).getMean(), 0.1D);
-            }
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true,
-                                                                                                        DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
-                                                                                                        DecayingEstimatedHistogramReservoir.DEFAULT_STRIPE_COUNT,
-                                                                                                        clock);
-                for (int i = 0; i < 40; i++)
-                    histogram.update(0);
-                for (int i = 0; i < 20; i++)
-                    histogram.update(1);
-                for (int i = 0; i < 10; i++)
-                    histogram.update(2);
-                assertEquals(0.57D, toSnapshot.apply(histogram).getMean(), 0.1D);
-            }
-        }
-
-        @Test
-        public void testStdDev()
-        {
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-                for (int i = 0; i < 20; i++)
-                    histogram.update(10);
-                for (int i = 0; i < 40; i++)
-                    histogram.update(20);
-                for (int i = 0; i < 20; i++)
-                    histogram.update(30);
-
-                Snapshot snapshot = toSnapshot.apply(histogram);
-                assertEquals(20.0D, snapshot.getMean(), 2.0D);
-                assertEquals(7.07D, snapshot.getStdDev(), 2.0D);
-            }
-        }
-
-        @Test
-        public void testFindingCorrectBuckets()
-        {
-            TestClock clock = new TestClock();
-
-            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(DecayingEstimatedHistogramReservoir.DEFAULT_ZERO_CONSIDERATION, 90, 1, clock);
-            histogram.update(23282687);
-            assertFalse(histogram.isOverflowed());
-            assertEquals(1, histogram.getSnapshot().getValues()[89]);
-
-            histogram.update(9);
-            assertEquals(1, histogram.getSnapshot().getValues()[8]);
-
-            histogram.update(21);
-            histogram.update(22);
-            Snapshot snapshot = histogram.getSnapshot();
-            assertEquals(2, snapshot.getValues()[13]);
-            assertEquals(6277304.5D, snapshot.getMean(), DOUBLE_ASSERT_DELTA);
-        }
-
-        @Test
-        public void testPercentile()
-        {
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-                // percentile of empty histogram is 0
-                assertEquals(0D, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
-
-                histogram.update(1);
-                // percentile of a histogram with one element should be that element
-                assertEquals(1D, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
-
-                histogram.update(10);
-                assertEquals(10D, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
-            }
-
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-
-                histogram.update(1);
-                histogram.update(2);
-                histogram.update(3);
-                histogram.update(4);
-                histogram.update(5);
-
-                Snapshot snapshot = toSnapshot.apply(histogram);
-                assertEquals(0, snapshot.getValue(0.00), DOUBLE_ASSERT_DELTA);
-                assertEquals(3, snapshot.getValue(0.50), DOUBLE_ASSERT_DELTA);
-                assertEquals(3, snapshot.getValue(0.60), DOUBLE_ASSERT_DELTA);
-                assertEquals(5, snapshot.getValue(1.00), DOUBLE_ASSERT_DELTA);
-            }
-
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-
-                for (int i = 11; i <= 20; i++)
-                    histogram.update(i);
-
-                // Right now the histogram looks like:
-                //    10   12   14   17   20
-                //     0    2    2    3    3
-                // %:  0   20   40   70  100
-                Snapshot snapshot = toSnapshot.apply(histogram);
-                assertEquals(12, snapshot.getValue(0.01), DOUBLE_ASSERT_DELTA);
-                assertEquals(14, snapshot.getValue(0.30), DOUBLE_ASSERT_DELTA);
-                assertEquals(17, snapshot.getValue(0.50), DOUBLE_ASSERT_DELTA);
-                assertEquals(17, snapshot.getValue(0.60), DOUBLE_ASSERT_DELTA);
-                assertEquals(20, snapshot.getValue(0.80), DOUBLE_ASSERT_DELTA);
-            }
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(true,
-                                                                                                        DecayingEstimatedHistogramReservoir.DEFAULT_BUCKET_COUNT,
-                                                                                                        DecayingEstimatedHistogramReservoir.DEFAULT_STRIPE_COUNT,
-                                                                                                        clock);
-                histogram.update(0);
-                histogram.update(0);
-                histogram.update(1);
-
-                Snapshot snapshot = toSnapshot.apply(histogram);
-                assertEquals(0, snapshot.getValue(0.5), DOUBLE_ASSERT_DELTA);
-                assertEquals(1, snapshot.getValue(0.99), DOUBLE_ASSERT_DELTA);
-            }
-        }
-
-        @Test
-        public void testDecayingPercentile()
-        {
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-                // percentile of empty histogram is 0
-                assertEquals(0, toSnapshot.apply(histogram).getValue(1.0), DOUBLE_ASSERT_DELTA);
-
-                for (int v = 1; v <= 100; v++)
-                {
-                    for (int i = 0; i < 10_000; i++)
-                    {
-                        histogram.update(v);
-                    }
-                }
-
-                Snapshot snapshot = toSnapshot.apply(histogram);
-                assertEstimatedQuantile(5, snapshot.getValue(0.05));
-                assertEstimatedQuantile(20, snapshot.getValue(0.20));
-                assertEstimatedQuantile(40, snapshot.getValue(0.40));
-                assertEstimatedQuantile(99, snapshot.getValue(0.99));
-
-                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
-                snapshot = toSnapshot.apply(histogram);
-                assertEstimatedQuantile(5, snapshot.getValue(0.05));
-                assertEstimatedQuantile(20, snapshot.getValue(0.20));
-                assertEstimatedQuantile(40, snapshot.getValue(0.40));
-                assertEstimatedQuantile(99, snapshot.getValue(0.99));
-
-                for (int v = 1; v <= 50; v++)
-                {
-                    for (int i = 0; i < 10_000; i++)
-                    {
-                        histogram.update(v);
-                    }
-                }
-
-                snapshot = toSnapshot.apply(histogram);
-                assertEstimatedQuantile(4, snapshot.getValue(0.05));
-                assertEstimatedQuantile(14, snapshot.getValue(0.20));
-                assertEstimatedQuantile(27, snapshot.getValue(0.40));
-                assertEstimatedQuantile(98, snapshot.getValue(0.99));
-
-                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
-                snapshot = toSnapshot.apply(histogram);
-                assertEstimatedQuantile(4, snapshot.getValue(0.05));
-                assertEstimatedQuantile(14, snapshot.getValue(0.20));
-                assertEstimatedQuantile(27, snapshot.getValue(0.40));
-                assertEstimatedQuantile(98, snapshot.getValue(0.99));
-
-                for (int v = 1; v <= 50; v++)
-                {
-                    for (int i = 0; i < 10_000; i++)
-                    {
-                        histogram.update(v);
-                    }
-                }
-
-                snapshot = toSnapshot.apply(histogram);
-                assertEstimatedQuantile(3, snapshot.getValue(0.05));
-                assertEstimatedQuantile(12, snapshot.getValue(0.20));
-                assertEstimatedQuantile(23, snapshot.getValue(0.40));
-                assertEstimatedQuantile(96, snapshot.getValue(0.99));
-
-                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
-                snapshot = toSnapshot.apply(histogram);
-                assertEstimatedQuantile(3, snapshot.getValue(0.05));
-                assertEstimatedQuantile(12, snapshot.getValue(0.20));
-                assertEstimatedQuantile(23, snapshot.getValue(0.40));
-                assertEstimatedQuantile(96, snapshot.getValue(0.99));
-
-                for (int v = 11; v <= 20; v++)
-                {
-                    for (int i = 0; i < 5_000; i++)
-                    {
-                        histogram.update(v);
-                    }
-                }
-
-                snapshot = toSnapshot.apply(histogram);
-                assertEstimatedQuantile(4, snapshot.getValue(0.05));
-                assertEstimatedQuantile(12, snapshot.getValue(0.20));
-                assertEstimatedQuantile(20, snapshot.getValue(0.40));
-                assertEstimatedQuantile(95, snapshot.getValue(0.99));
-
-                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
-                snapshot = toSnapshot.apply(histogram);
-                assertEstimatedQuantile(4, snapshot.getValue(0.05));
-                assertEstimatedQuantile(12, snapshot.getValue(0.20));
-                assertEstimatedQuantile(20, snapshot.getValue(0.40));
-                assertEstimatedQuantile(95, snapshot.getValue(0.99));
-
-            }
-
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-                // percentile of empty histogram is 0
-                assertEquals(0, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
-
-                for (int m = 0; m < 40; m++)
-                {
-                    for (int i = 0; i < 1_000_000; i++)
-                    {
-                        histogram.update(2);
-                    }
-                    // percentile of a histogram with one element should be that element
-                    clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
-                    assertEquals(2, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
-                }
-
-                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S * 100);
-                assertEquals(0, toSnapshot.apply(histogram).getValue(0.99), DOUBLE_ASSERT_DELTA);
-            }
-
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-
-                histogram.update(20);
-                histogram.update(21);
-                histogram.update(22);
-                Snapshot snapshot = histogram.getSnapshot();
-                assertEquals(1, snapshot.getValues()[12]);
-                assertEquals(2, snapshot.getValues()[13]);
-
-                clock.addSeconds(DecayingEstimatedHistogramReservoir.HALF_TIME_IN_S);
-
-                histogram.update(20);
-                histogram.update(21);
-                histogram.update(22);
-                snapshot = histogram.getSnapshot();
-                assertEquals(2, snapshot.getValues()[12]);
-                assertEquals(4, snapshot.getValues()[13]);
-            }
-        }
-
-        @Test
-        public void testDecayingMean()
-        {
-            {
-                TestClock clock = new TestClock();
-
-                DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-
-                clock.addNanos(LANDMARK_RESET_INTERVAL_IN_NS - TimeUnit.SECONDS.toNanos(1L));
-
-                while (clock.now() < LANDMARK_RESET_INTERVAL_IN_NS + TimeUnit.SECONDS.toNanos(1L))
-                {
-                    clock.addNanos(TimeUnit.MILLISECONDS.toNanos(900));
-                    for (int i = 0; i < 1_000_000; i++)
-                    {
-                        histogram.update(1000);
-                        histogram.update(2000);
-                        histogram.update(3000);
-                        histogram.update(4000);
-                        histogram.update(5000);
-                    }
-                    assertEquals(3000D, toSnapshot.apply(histogram).getMean(), 500D);
-                }
-            }
-        }
-
-        @Test
-        public void testAggregation()
-        {
-            TestClock clock = new TestClock();
-
-            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-            DecayingEstimatedHistogramReservoir another = new DecayingEstimatedHistogramReservoir(clock);
-
-            clock.addNanos(LANDMARK_RESET_INTERVAL_IN_NS - TimeUnit.SECONDS.toNanos(1L));
-
-            histogram.update(1000);
-            clock.addMillis(100);
-            another.update(2000);
-            clock.addMillis(100);
-            histogram.update(2000);
-            clock.addMillis(100);
-            another.update(3000);
-            clock.addMillis(100);
-            histogram.update(3000);
-            clock.addMillis(100);
-            another.update(4000);
-
-            DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot snapshot = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) histogram.getSnapshot();
-            DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot anotherSnapshot = (DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot) another.getSnapshot();
-
-            assertEquals(2000, snapshot.getMean(), 500D);
-            assertEquals(3000, anotherSnapshot.getMean(), 500D);
-
-            snapshot.add(anotherSnapshot);
-
-            // Another had newer decayLandmark, the aggregated snapshot should use it
-            assertEquals(anotherSnapshot.getSnapshotLandmark(), snapshot.getSnapshotLandmark());
-            assertEquals(2500, snapshot.getMean(), 500D);
-        }
-
-        @Test
-        public void testSize()
-        {
-            TestClock clock = new TestClock();
-
-            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-            histogram.update(42);
-            histogram.update(42);
-            assertEquals(2, toSnapshot.apply(histogram).size());
-        }
-
-        /**
-         * This looks for invalid percentiles that are unchanged for too long to expose the CASSANDRA-19365 race
-         * condition between rescale and update. The idea is to update a histogram from multiple threads and observe
-         * if the reported p99 doesn't get stuck at a low value or p50 at a high value due to update with high weight
-         * being inserted after the buckets are rescaled.
-         * <p>
-         * The load has 95% of 42, and 5% of the time it's 1109. Despite that the histogram may be convinced for a long
-         * time that p99 is 42 or that p50 is 1109. The reason may be seen in the snapshot dump, where after rescale
-         * the bucket values may get very big due to the race condition and too big weight of the inserted samples.
-         * The values were picked to match bucket boundaries, but that's only for aesthetics.
-         * <p>
-         * In production the rescale happens every 30 minutes. In this test time we're pushing time to run faster,
-         * roughly 1000 times faster to hit the race condition in a reasonable time.
-         */
-        @Test
-        public void testConcurrentUpdateAndRescale() throws InterruptedException
-        {
-            int UPDATE_THREADS = 60;
-            int maxTestDurationMillis = 30_000;
-            // how many times in a row the percentiles may be invalid before we fail the test
-            int tooManySuspiciousPercentilesThreshold = 5; // 5 translates to 500ms * 1000 speedup = 500s = 8m20s;
-            AtomicBoolean stop = new AtomicBoolean(false);
-            AtomicBoolean failed = new AtomicBoolean(false);
-            TestClock clock = new TestClock();
-
-            DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock);
-            ExecutorService executors = Executors.newFixedThreadPool(2 + UPDATE_THREADS);
-
-            for (int i = 0; i < UPDATE_THREADS; i++)
-            {
-                executors.submit(() -> {
-                    while (!stop.get() && !Thread.currentThread().isInterrupted())
-                    {
-                        // a mischievous usage pattern to quickly trigger the
-                        // CASSANDRA-19365 race condition;
-                        // the load has 95% of 42, and only 5% of the time it's 1109
-                        // and yet, the histogram may be convinced for a long time that
-                        // the p99 is 42 or that the p50 is 1109
-                        for (int sampleIdx = 0; sampleIdx < 900; sampleIdx++)
-                            histogram.update(42);
-
-                        for (int sampleIdx = 0; sampleIdx < 50; sampleIdx++)
-                        {
-                            // add some noise so that low value samples do not race with the same likelyhood as the high value samples
-                            Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(1, 10), MILLISECONDS);
-                            histogram.update(1109);
-                        }
-                    }
-                });
-            }
-            // clock update thread
-            executors.submit(() -> {
-                while (!stop.get() && !Thread.currentThread().isInterrupted())
-                {
-                    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
-                    // x1000 speedup so that we hit rescale interval every 30 minutes / 1000 = 1.8s
-                    clock.addMillis(1000);
-                }
-            });
-            // percentiles check thread
-            executors.submit(() -> {
-                // how many times in a row p99 was suspiciously low or P50 suspiciously high
-                int consecutiveInvalidPercentiles = 0;
-
-                // how often to check the percentiles
-                int iterationDelayMillis = 100;
-
-                for (int i = 0; i < maxTestDurationMillis / iterationDelayMillis; i++)
-                {
-                    Uninterruptibles.sleepUninterruptibly(iterationDelayMillis, MILLISECONDS);
-                    Snapshot snapshot = toSnapshot.apply(histogram);
-                    double p99 = snapshot.getValue(0.99);
-                    double p50 = snapshot.getValue(0.50);
-                    ByteArrayOutputStream output = new ByteArrayOutputStream();
-                    snapshot.dump(output);
-                    String decayingNonZeroBuckets = Arrays.stream(output.toString().split("\n"))
-                                                          .filter(s -> !s.equals("0"))
-                                                          .collect(Collectors.joining(","));
-                    logger.info("\"clock={}, p50={}, p99={}, decaying non-zero buckets: {}",
-                                clock.now() / 1_000_000, p50, p99, decayingNonZeroBuckets);
-                    if (p99 < 100 || p50 > 900)
-                    {
-                        consecutiveInvalidPercentiles++;
-                        logger.warn("p50 or p99 at suspicious level p50={}, p99={}", p50, p99);
-                        if (consecutiveInvalidPercentiles > tooManySuspiciousPercentilesThreshold)
-                        {
-                            failed.set(true);
-                            stop.set(true);
-                            break;
-                        }
-                    }
-                    else
-                    {
-                        consecutiveInvalidPercentiles = 0;
-                    }
-                }
-                stop.set(true);
-            });
-            executors.shutdown();
-            boolean success = executors.awaitTermination(maxTestDurationMillis * 2, MILLISECONDS);
-            Assert.assertFalse("p50 too high or p99 too low for too long", failed.get());
-            Assert.assertTrue("Timeout exceeded the limit", success);
-        }
-
-        private void assertEstimatedQuantile(long expectedValue, double actualValue)
-        {
-            assertTrue("Expected at least [" + expectedValue + "] but actual is [" + actualValue + ']', actualValue >= expectedValue);
-            assertTrue("Expected less than [" + Math.round(expectedValue * 1.2) + "] but actual is [" + actualValue + ']', actualValue < Math.round(expectedValue * 1.2));
-        }
-
-        public static class TestClock implements MonotonicClock
-        {
-            private long tick = 0;
-
-            public void addNanos(long nanos)
-            {
-                tick += nanos;
-            }
-
-            public void addMillis(long millis)
-            {
-                tick += TimeUnit.MILLISECONDS.toNanos(millis);
-            }
-
-            public void addSeconds(long seconds)
-            {
-                tick += TimeUnit.SECONDS.toNanos(seconds);
-            }
-
-            public long now()
-            {
-                return tick;
-            }
-
-            @Override
-            public long error()
-            {
-                return 0;
-            }
-
-            @Override
-            public MonotonicClockTranslation translate()
-            {
-                throw new UnsupportedOperationException();
-            }
-
-            @Override
-            public boolean isAfter(long instant)
-            {
-                throw new UnsupportedOperationException();
-            }
-
-            @Override
-            public boolean isAfter(long now, long instant)
-            {
-                throw new UnsupportedOperationException();
-            }
-        }
-    }
-}
diff --git a/test/unit/org/apache/cassandra/metrics/LinearFitTest.java b/test/unit/org/apache/cassandra/metrics/LinearFitTest.java
new file mode 100644
index 000000000000..82561279e673
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/LinearFitTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.junit.Test;
+
+import org.apache.cassandra.metrics.PairedSlidingWindowReservoir.IntIntPair;
+
+import static org.junit.Assert.assertEquals;
+
+public class LinearFitTest
+{
+    @Test
+    public void testInterceptSlopeForLinearValues()
+    {
+        // tests values that are perfectly linear
+        var values = new IntIntPair[10];
+        for (int i = 0; i < values.length; i++)
+            values[i] = new IntIntPair(i, i * 2);
+
+        var pair = LinearFit.interceptSlopeFor(values);
+        assertEquals(0.0, pair.left, 0.01);
+        assertEquals(2.0, pair.right, 0.01);
+
+        values = new IntIntPair[10];
+        for (int i = 0; i < values.length; i++)
+            values[i] = new IntIntPair(i, 1 + i * 2);
+
+        pair = LinearFit.interceptSlopeFor(values);
+        assertEquals(1.0, pair.left, 0.01d);
+        assertEquals(2.0, pair.right, 0.01d);
+
+        values = new IntIntPair[10];
+        for (int i = 0; i < values.length; i++)
+            values[i] = new IntIntPair(i, -1 + i * 2);
+
+        pair = LinearFit.interceptSlopeFor(values);
+        assertEquals(-1.0, pair.left, 0.01d);
+        assertEquals(2.0, pair.right, 0.01d);
+    }
+
+    @Test
+    public void testInterceptSlopeWithNoise()
+    {
+        // values are +/- 20 from perfectly linear
+        var values = new IntIntPair[] {
+            new IntIntPair(1, 66),
+            new IntIntPair(2, 108),
+            new IntIntPair(3, 70),
+            new IntIntPair(4, 112),
+            new IntIntPair(5, 74),
+            new IntIntPair(6, 116),
+            new IntIntPair(7, 78),
+            new IntIntPair(8, 120),
+            new IntIntPair(9, 82)
+        };
+        var pair = LinearFit.interceptSlopeFor(values);
+        // verified with sklearn
+        assertEquals(2.0, pair.right, 0.01);
+        assertEquals(81.78, pair.left, 0.01);
+    }
+
+    @Test
+    public void testInterceptSlopeWithMoreNoise()
+    {
+        // values are pseudorandomly distributed
+        var values = new IntIntPair[] {
+            new IntIntPair(931, 1366),
+            new IntIntPair(973, 822),
+            new IntIntPair(200, 1308),
+            new IntIntPair(332, 708),
+            new IntIntPair(677, 1186),
+            new IntIntPair(401, 7112),
+            new IntIntPair(111, 166),
+            new IntIntPair(503, 734),
+            new IntIntPair(738, 78),
+            new IntIntPair(829, 8120)
+        };
+        var pair = LinearFit.interceptSlopeFor(values);
+        // verified with sklearn
+        assertEquals(1.31, pair.right, 0.01);
+        assertEquals(1412.35, pair.left, 0.01);
+    }
+
+    @Test
+    public void testDuplicateX()
+    {
+        var values = new IntIntPair[] {
+            new IntIntPair(1, 1),
+            new IntIntPair(1, 2),
+            new IntIntPair(1, 3),
+        };
+        var pair = LinearFit.interceptSlopeFor(values);
+        assertEquals(0.0, pair.left, 0.01);
+        assertEquals(2.0, pair.right, 0.01);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/MicrometerChunkCacheMetricsTest.java b/test/unit/org/apache/cassandra/metrics/MicrometerChunkCacheMetricsTest.java
new file mode 100644
index 000000000000..563100a89593
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/MicrometerChunkCacheMetricsTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import com.github.benmanes.caffeine.cache.stats.CacheStats;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
+import org.assertj.core.api.Assertions;
+import org.mockito.Mockito;
+
+import static org.apache.cassandra.metrics.MicrometerCacheMetrics.hitRateUpdateIntervalNanos;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class MicrometerChunkCacheMetricsTest
+{
+    private ChunkCache mockChunkCache;
+
+    private MicrometerChunkCacheMetrics chunkCacheMetrics;
+
+    private WithProperties withProperties;
+
+    @BeforeClass
+    public static void init()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Before
+    public void before()
+    {
+        withProperties = new WithProperties();
+
+        mockChunkCache = Mockito.mock(ChunkCache.class);
+
+        // Use micrometer metrics
+        withProperties.set(CassandraRelevantProperties.USE_MICROMETER, true);
+
+        chunkCacheMetrics = (MicrometerChunkCacheMetrics) ChunkCacheMetrics.create(mockChunkCache);
+    }
+
+    @After
+    public void after()
+    {
+        // Reset to not use micrometer metrics
+        withProperties.close();
+    }
+
+    @Test
+    public void testHitRate() throws InterruptedException
+    {
+        chunkCacheMetrics.recordHits(90);
+        assertEquals(90, chunkCacheMetrics.hits());
+
+        // Added delay to increase code coverage updating hit rate when calling recordMisses
+        Thread.sleep(2 * TimeUnit.NANOSECONDS.toMillis(hitRateUpdateIntervalNanos));
+
+        chunkCacheMetrics.recordMisses(10);
+        assertEquals(10, chunkCacheMetrics.misses());
+
+        // Verify requests = hits + misses
+        assertEquals(100, chunkCacheMetrics.requests());
+
+        // Verify hitrate
+        assertEquals(0.9, chunkCacheMetrics.hitRate(), 0.0);
+    }
+
+    @Test
+    public void testCommonChunkCacheMetrics()
+    {
+        chunkCacheMetrics.recordEviction(1, RemovalCause.EXPIRED);
+
+        // No-op
+        chunkCacheMetrics.recordLoadFailure(25);
+
+        chunkCacheMetrics.recordLoadSuccess(15);
+
+        // missLatency based on recordLoadSuccess
+        assertEquals(15.0, chunkCacheMetrics.missLatency(), 0.0);
+
+        assertEquals(0, chunkCacheMetrics.capacity());
+
+        assertEquals(0, chunkCacheMetrics.size());
+
+        assertEquals(0, chunkCacheMetrics.entries());
+
+        assertTrue(Double.isNaN(chunkCacheMetrics.requestsFifteenMinuteRate()));
+
+        assertTrue(Double.isNaN(chunkCacheMetrics.hitFifteenMinuteRate()));
+
+        CacheStats snapshot = chunkCacheMetrics.snapshot();
+        assertNotNull(snapshot);
+        System.out.println(snapshot);
+
+        String toString = chunkCacheMetrics.toString();
+        assertNotNull(toString);
+        Assertions.assertThat(toString).contains("Capacity:");
+    }
+
+    @Test
+    public void testEvictionRecording()
+    {
+        long initialEvictionCount = chunkCacheMetrics.snapshot().evictionCount();
+        chunkCacheMetrics.recordEviction(4, RemovalCause.EXPIRED);
+        assertEquals(initialEvictionCount + 1, chunkCacheMetrics.snapshot().evictionCount());
+
+        chunkCacheMetrics.recordEviction(10, RemovalCause.EXPIRED);
+        assertEquals(initialEvictionCount + 1 + 1, chunkCacheMetrics.snapshot().evictionCount());
+
+        // replacing an entry is also an eviction but it doesn't increment the eviction count
+        chunkCacheMetrics.recordEviction(100, RemovalCause.REPLACED);
+        assertEquals(initialEvictionCount + 1 + 1, chunkCacheMetrics.snapshot().evictionCount());
+
+
+        assertEquals(2, chunkCacheMetrics.getEvictionCountByRemovalCause().get(RemovalCause.EXPIRED).intValue());
+        assertEquals(1, chunkCacheMetrics.getEvictionCountByRemovalCause().get(RemovalCause.REPLACED).intValue());
+        assertEquals(0, chunkCacheMetrics.getEvictionCountByRemovalCause().get(RemovalCause.COLLECTED).intValue());
+        assertEquals(0, chunkCacheMetrics.getEvictionCountByRemovalCause().get(RemovalCause.SIZE).intValue());
+        assertEquals(0, chunkCacheMetrics.getEvictionCountByRemovalCause().get(RemovalCause.EXPLICIT).intValue());
+    }
+
+    @Test
+    public void testRegister()
+    {
+        ((MicrometerChunkCacheMetrics) chunkCacheMetrics).register(new SimpleMeterRegistry(), Tags.of("tagKey", "tagValue"));
+    }
+
+    @Test
+    public void testCounter()
+    {
+        ((MicrometerChunkCacheMetrics) chunkCacheMetrics).counter("counter", Tags.of("tagKey", "tagValue"));
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testReset()
+    {
+        chunkCacheMetrics.reset();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/metrics/MicrometerNativeMemoryMetricsTest.java b/test/unit/org/apache/cassandra/metrics/MicrometerNativeMemoryMetricsTest.java
new file mode 100644
index 000000000000..56ecb04c131d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/MicrometerNativeMemoryMetricsTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.junit.Test;
+
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tags;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class MicrometerNativeMemoryMetricsTest
+{
+    @Test
+    public void testRegisteringNativeMetrics()
+    {
+        MicrometerNativeMemoryMetrics metrics = new MicrometerNativeMemoryMetrics();
+
+        MeterRegistry registry = mock(MeterRegistry.class);
+        Tags tags = Tags.of("tag_key", "tag_value");
+        metrics.register(registry, tags);
+
+        verify(registry).gauge(eq(MicrometerNativeMemoryMetrics.RAW_NATIVE_MEMORY), eq(tags), any(), any());
+        verify(registry).gauge(eq(MicrometerNativeMemoryMetrics.TOTAL_MEMORY), eq(tags), any(), any());
+        verify(registry).gauge(eq(MicrometerNativeMemoryMetrics.BLOOM_FILTER_MEMORY), eq(tags), any(), any());
+        verify(registry).gauge(eq(MicrometerNativeMemoryMetrics.NETWORK_DIRECT_MEMORY), eq(tags), any(), any());
+        verify(registry).gauge(eq(MicrometerNativeMemoryMetrics.TOTAL_NIO_MEMORY), eq(tags), any(), any());
+        verify(registry).gauge(eq(MicrometerNativeMemoryMetrics.NIO_DIRECT_BUFFER_COUNT), eq(tags), any(), any());
+        verify(registry).gauge(eq(MicrometerNativeMemoryMetrics.USED_NIO_DIRECT_MEMORY), eq(tags), any(), any());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/metrics/NativeMemoryMetricsTest.java b/test/unit/org/apache/cassandra/metrics/NativeMemoryMetricsTest.java
new file mode 100644
index 000000000000..cecad3fb08e5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/NativeMemoryMetricsTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.lang.management.BufferPoolMXBean;
+import java.lang.management.ManagementFactory;
+import java.nio.ByteBuffer;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_NIO_PAGE_ALIGN_DIRECT_MEMORY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class NativeMemoryMetricsTest
+{
+    private final static Logger logger = LoggerFactory.getLogger(NativeMemoryMetricsTest.class);
+    private static NativeMemoryMetrics nativeMemoryMetrics;
+    private static BufferPoolMXBean directBufferPool;
+
+    @BeforeClass
+    public static void setupClass()
+    {
+        nativeMemoryMetrics = NativeMemoryMetrics.instance;
+
+        directBufferPool = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)
+                                            .stream()
+                                            .filter(bpMBean -> bpMBean.getName().equals("direct"))
+                                            .findFirst()
+                                            .orElse(null);
+
+        // touch every field under test that may result in additional allocations (see MemoryUtil static init)
+        nativeMemoryMetrics.totalMemory();
+        nativeMemoryMetrics.totalNioDirectMemory();
+        nativeMemoryMetrics.usedNioDirectMemory();
+        nativeMemoryMetrics.nioDirectBufferCount();
+    }
+
+    @Test
+    public void testNioDirectMemory()
+    {
+        long totalNioDirectMemory = nativeMemoryMetrics.totalNioDirectMemory();
+        long usedNioDirectMemory = nativeMemoryMetrics.usedNioDirectMemory();
+        long nioDirectBufferCount = nativeMemoryMetrics.nioDirectBufferCount();
+
+        logger.debug("Total Nio Memory: {}, Reserved Nio Memory: {}, Num Nio buffers: {}",
+                     totalNioDirectMemory, usedNioDirectMemory, nioDirectBufferCount);
+
+        assertFalse("sun.nio.PageAlignDirectMemory should not be set for this test",
+                    SUN_NIO_PAGE_ALIGN_DIRECT_MEMORY.getBoolean());
+
+        assertEquals("Total and reserved nio memory should be equal since -Dsun.nio.PageAlignDirectMemory=true should not be set",
+                     totalNioDirectMemory, usedNioDirectMemory);
+
+        assertEquals("Total nio memory should be equal to total memory since no unsafe allocations should have been done",
+                     totalNioDirectMemory, nativeMemoryMetrics.totalMemory());
+
+        assertEquals(directBufferPool.getMemoryUsed(), nativeMemoryMetrics.usedNioDirectMemory());
+        assertEquals(directBufferPool.getTotalCapacity(), nativeMemoryMetrics.totalNioDirectMemory());
+        assertEquals(directBufferPool.getCount(), nativeMemoryMetrics.nioDirectBufferCount());
+
+        ByteBuffer buffer = ByteBuffer.allocateDirect(128);
+
+        assertEquals(totalNioDirectMemory + 128, nativeMemoryMetrics.totalNioDirectMemory());
+        assertEquals(usedNioDirectMemory + 128, nativeMemoryMetrics.usedNioDirectMemory());
+        assertEquals(nioDirectBufferCount + 1, nativeMemoryMetrics.nioDirectBufferCount());
+
+        FileUtils.clean(buffer);
+
+        assertEquals(totalNioDirectMemory, nativeMemoryMetrics.totalNioDirectMemory());
+        assertEquals(usedNioDirectMemory, nativeMemoryMetrics.usedNioDirectMemory());
+        assertEquals(nioDirectBufferCount, nativeMemoryMetrics.nioDirectBufferCount());
+    }
+
+    @Test
+    public void testUnsafeNativeMemory()
+    {
+        assertEquals(0, nativeMemoryMetrics.rawNativeMemory());
+
+        long peer = MemoryUtil.allocate(128);
+        assertEquals(128, nativeMemoryMetrics.rawNativeMemory());
+
+        MemoryUtil.free(peer, 128);
+        assertEquals(0, nativeMemoryMetrics.rawNativeMemory());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/QuickSlidingWindowReservoirTest.java b/test/unit/org/apache/cassandra/metrics/QuickSlidingWindowReservoirTest.java
new file mode 100644
index 000000000000..017d72813781
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/QuickSlidingWindowReservoirTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class QuickSlidingWindowReservoirTest
+{
+    @Test
+    public void testSize()
+    {
+        var sr = new QuickSlidingWindowReservoir(10);
+        assertEquals(0, sr.size());
+
+        for (int i = 0; i < 20; i++)
+        {
+            sr.update(i);
+            assertEquals(Math.min(10, i + 1), sr.size());
+        }
+    }
+
+    @Test
+    public void testGetMean()
+    {
+        var sr = new QuickSlidingWindowReservoir(10);
+
+        assertEquals(0.0, sr.getMean(), 0.01d);
+
+        sr.update(1);
+        sr.update(2);
+        sr.update(3);
+        assertEquals(2.0d, sr.getMean(), 0.01d);
+
+        for (int i = 0; i < 10; i++)
+            sr.update(10);
+
+        assertEquals(10.0d, sr.getMean(), 0.01d);
+    }
+
+    @Test
+    public void testGetSnapshot()
+    {
+        var sr = new QuickSlidingWindowReservoir(10);
+        for (int i = 0; i < 10; i++)
+            sr.update(i);
+
+        var snapshot = sr.getSnapshot();
+        assertEquals(10, snapshot.size());
+        assertEquals(sr.getMean(), snapshot.getMean(), 0.01d);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/metrics/ReadCoordinationMetricsTest.java b/test/unit/org/apache/cassandra/metrics/ReadCoordinationMetricsTest.java
new file mode 100644
index 000000000000..6997afcf48f5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/ReadCoordinationMetricsTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.net.UnknownHostException;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.codahale.metrics.Histogram;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class ReadCoordinationMetricsTest
+{
+    @BeforeClass()
+    public static void setup() throws ConfigurationException
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void testReadCoordinatorMetrics()
+    {
+        assertEquals(0, ReadCoordinationMetrics.nonreplicaRequests.getCount());
+        ReadCoordinationMetrics.nonreplicaRequests.inc();
+        assertEquals(1, ReadCoordinationMetrics.nonreplicaRequests.getCount());
+        ReadCoordinationMetrics.nonreplicaRequests.dec();
+        assertEquals(0, ReadCoordinationMetrics.nonreplicaRequests.getCount());
+
+        assertEquals(0, ReadCoordinationMetrics.preferredOtherReplicas.getCount());
+        ReadCoordinationMetrics.preferredOtherReplicas.inc();
+        assertEquals(1, ReadCoordinationMetrics.preferredOtherReplicas.getCount());
+        ReadCoordinationMetrics.preferredOtherReplicas.dec();
+        assertEquals(0, ReadCoordinationMetrics.preferredOtherReplicas.getCount());
+    }
+
+    @Test
+    public void testReplicaLatencyHistogram() throws UnknownHostException
+    {
+        InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.1");
+        assertNull(ReadCoordinationMetrics.getReplicaLatencyHistogram(host));
+
+        // Record a replica latency
+        ReadCoordinationMetrics.updateReplicaLatency(host, 100, TimeUnit.MILLISECONDS);
+
+        Histogram histogram = ReadCoordinationMetrics.getReplicaLatencyHistogram(host);
+        assertNotNull(histogram);
+        assertEquals(1, histogram.getCount());
+
+        // ReadRpcTimeout latency should not be tracked
+        ReadCoordinationMetrics.updateReplicaLatency(host, DatabaseDescriptor.getReadRpcTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+        assertEquals(1, histogram.getCount());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java b/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java
index bbcced43dbff..43e303cc267d 100644
--- a/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java
@@ -36,9 +36,11 @@
 import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.service.EmbeddedCassandraService;
 import org.apache.cassandra.service.StorageService;
+import org.awaitility.Awaitility;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -49,6 +51,7 @@ public class TableMetricsTest
 
     private static final String KEYSPACE = "junit";
     private static final String TABLE = "tablemetricstest";
+    private static final String TABLE_WITH_HISTOS_AGGR = "tablemetricstest_histo_aggr";
     private static final String COUNTER_TABLE = "tablemetricscountertest";
     private static final String TWCS_TABLE = "tablemetricstesttwcs";
 
@@ -88,11 +91,24 @@ private ColumnFamilyStore recreateTable(String table)
         return ColumnFamilyStore.getIfExists(KEYSPACE, table);
     }
 
+    private void recreateExtensionTables(String keyspace)
+    {
+        session.execute(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };", keyspace));
+
+        session.execute(String.format("DROP TABLE IF EXISTS %s.%s", keyspace, TABLE));
+        session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id int, val1 text, val2 text, PRIMARY KEY(id, val1)) WITH extensions = {'HISTOGRAM_METRICS': '%s'};",
+                                      keyspace, TABLE, TableMetrics.MetricsAggregation.INDIVIDUAL.asCQLString())); // this is also the default
+
+        session.execute(String.format("DROP TABLE IF EXISTS %s.%s", keyspace, TABLE_WITH_HISTOS_AGGR));
+        session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id int, val1 text, val2 text, PRIMARY KEY(id, val1)) WITH extensions = {'HISTOGRAM_METRICS': '%s'};",
+                                      keyspace, TABLE_WITH_HISTOS_AGGR, TableMetrics.MetricsAggregation.AGGREGATED.asCQLString()));
+    }
+
     private void executeBatch(boolean isLogged, int distinctPartitions, int statementsPerPartition, String... tables)
     {
         if (tables == null || tables.length == 0)
         {
-            tables = new String[] { TABLE };
+            tables = new String[]{ TABLE };
         }
         BatchStatement.Type batchType;
 
@@ -117,29 +133,53 @@ private static void populateBatch(BatchStatement batch, String table, int distin
     {
         PreparedStatement ps = session.prepare(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (?, ?, ?);", KEYSPACE, table));
 
-        for (int i=0; i<distinctPartitions; i++)
+        for (int i = 0; i < distinctPartitions; i++)
         {
-            for (int j=0; j<statementsPerPartition; j++)
+            for (int j = 0; j < statementsPerPartition; j++)
             {
                 batch.add(ps.bind(i, j + "a", "b"));
             }
         }
     }
 
+    private void checkWriteCounts(Keyspace keyspace, String table, long ksCount, long tableCount)
+    {
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(table);
+
+        // The table with its non-aggregated histograms will have individual table counts
+        assertEquals(tableCount, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(tableCount, cfs.metric.writeLatency.tableOrKeyspaceMetric().latency.getCount());
+
+        assertEquals(ksCount, keyspace.metric.coordinatorWriteLatency.getCount());
+        assertEquals(ksCount, keyspace.metric.writeLatency.latency.getCount());
+    }
+
     @Test
     public void testRegularStatementsExecuted()
     {
-        ColumnFamilyStore cfs = recreateTable();
-        assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        String keyspaceName = "regularstatementsks";
+        recreateExtensionTables(keyspaceName);
 
-        for (int i = 0; i < 10; i++)
-        {
-            session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i, "val" + i, "val" + i));
-        }
+        Keyspace keyspace = Keyspace.open(keyspaceName);
+        assertEquals(keyspace.metric.coordinatorWriteLatency.getCount(), 0);
+        assertEquals(keyspace.metric.writeLatency.latency.getCount(), 0);
+
+        int numRows = 10;
+
+        checkWriteCounts(keyspace, TABLE, 0, 0);
+        checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, 0, 0);
+
+        for (int i = 0; i < numRows; i++)
+            session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", keyspaceName, TABLE, i, "val" + i, "val" + i));
+
+        checkWriteCounts(keyspace, TABLE, numRows, numRows);
+        checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, numRows, numRows);
+
+        for (int i = 0; i < numRows; i++)
+            session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", keyspaceName, TABLE_WITH_HISTOS_AGGR, i, "val" + i, "val" + i));
 
-        assertEquals(10, cfs.metric.coordinatorWriteLatency.getCount());
-        assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+        checkWriteCounts(keyspace, TABLE, numRows * 2, numRows);
+        checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, numRows * 2, numRows * 2);
     }
 
     @Test
@@ -176,106 +216,155 @@ public void testMaxSSTableDuration() throws Exception
     @Test
     public void testPreparedStatementsExecuted()
     {
-        ColumnFamilyStore cfs = recreateTable();
-        PreparedStatement metricsStatement = session.prepare(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (?, ?, ?)", KEYSPACE, TABLE));
+        String keyspaceName = "preparedstatementsks";
+        recreateExtensionTables(keyspaceName);
 
-        assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        Keyspace keyspace = Keyspace.open(keyspaceName);
+        assertEquals(keyspace.metric.coordinatorWriteLatency.getCount(), 0);
+        assertEquals(keyspace.metric.writeLatency.latency.getCount(), 0);
 
-        for (int i = 0; i < 10; i++)
-        {
+        int numRows = 10;
+
+        checkWriteCounts(keyspace, TABLE, 0, 0);
+        checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, 0, 0);
+
+        PreparedStatement metricsStatement = session.prepare(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (?, ?, ?)", keyspaceName, TABLE));
+        for (int i = 0; i < numRows; i++)
             session.execute(metricsStatement.bind(i, "val" + i, "val" + i));
-        }
 
-        assertEquals(10, cfs.metric.coordinatorWriteLatency.getCount());
-        assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+        checkWriteCounts(keyspace, TABLE, numRows, numRows);
+        checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, numRows, numRows);
+
+        metricsStatement = session.prepare(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (?, ?, ?)", keyspaceName, TABLE_WITH_HISTOS_AGGR));
+        for (int i = 0; i < numRows; i++)
+            session.execute(metricsStatement.bind(i, "val" + i, "val" + i));
+
+        checkWriteCounts(keyspace, TABLE, numRows * 2, numRows);
+        checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, numRows * 2, numRows * 2);
+    }
+
+    @Test
+    public void testAlterTable()
+    {
+        // First create a table with aggregated histograms, then alter it to individual histograms, check the counts
+        String keyspaceName = "altertableks";
+        recreateExtensionTables(keyspaceName);
+        Keyspace keyspace = Keyspace.open(keyspaceName);
+
+        try
+        {
+            int numRows = 10;
+            long ksCount = keyspace.metric.coordinatorWriteLatency.getCount();
+
+            checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, ksCount, ksCount);
+
+            for (int i = 0; i < numRows; i++)
+                session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", keyspace.getName(), TABLE_WITH_HISTOS_AGGR, i, "val" + i, "val" + i));
+
+            // counts are the same as the keyspace since it is aggregated
+            checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, ksCount + numRows, ksCount + numRows);
+
+            session.execute(String.format("ALTER TABLE %s.%s WITH extensions = {'HISTOGRAM_METRICS': '%s'};", keyspace.getName(), TABLE_WITH_HISTOS_AGGR, TableMetrics.MetricsAggregation.INDIVIDUAL.asCQLString()));
+
+            for (int i = 0; i < numRows; i++)
+                session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", keyspace.getName(), TABLE_WITH_HISTOS_AGGR, i, "val" + i, "val" + i));
+
+            // Now we should find individual histograms counts for the table
+            checkWriteCounts(keyspace, TABLE_WITH_HISTOS_AGGR, ksCount + numRows * 2, numRows);
+        }
+        finally
+        {
+            session.execute(String.format("DROP TABLE %s.%s", keyspace.getName(), TABLE_WITH_HISTOS_AGGR));
+        }
     }
 
+
     @Test
     public void testLoggedPartitionsPerBatch()
     {
         ColumnFamilyStore cfs = recreateTable();
-        assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        assertEquals(0, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(0.0, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0.0);
 
         executeBatch(true, 10, 2);
-        assertEquals(1, cfs.metric.coordinatorWriteLatency.getCount());
+        assertEquals(1, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
 
         executeBatch(true, 20, 2);
-        assertEquals(2, cfs.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
-        assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+        assertEquals(2, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount()); // 2 for previous batch and this batch
+        assertGreaterThan(cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0);
     }
 
     @Test
     public void testLoggedPartitionsPerBatchMultiTable()
     {
         ColumnFamilyStore first = recreateTable();
-        assertEquals(0, first.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, first.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        assertEquals(0, first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(0.0, first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0.0);
 
         ColumnFamilyStore second = recreateTable(TABLE + "_second");
-        assertEquals(0, second.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, second.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        assertEquals(0, second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(0.0, second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0.0);
 
         executeBatch(true, 10, 2, TABLE, TABLE + "_second");
-        assertEquals(1, first.metric.coordinatorWriteLatency.getCount());
-        assertEquals(1, second.metric.coordinatorWriteLatency.getCount());
+        assertEquals(1, first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(1, second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
 
         executeBatch(true, 20, 2, TABLE, TABLE + "_second");
-        assertEquals(2, first.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
-        assertEquals(2, second.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
-        assertGreaterThan(first.metric.coordinatorWriteLatency.getMeanRate(), 0);
-        assertGreaterThan(second.metric.coordinatorWriteLatency.getMeanRate(), 0);
+        assertEquals(2, first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount()); // 2 for previous batch and this batch
+        assertEquals(2, second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount()); // 2 for previous batch and this batch
+        assertGreaterThan(first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0);
+        assertGreaterThan(second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0);
     }
 
     @Test
     public void testUnloggedPartitionsPerBatch()
     {
         ColumnFamilyStore cfs = recreateTable();
-        assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        assertEquals(0, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(0.0, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0.0);
 
         executeBatch(false, 5, 3);
-        assertEquals(1, cfs.metric.coordinatorWriteLatency.getCount());
+        assertEquals(1, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
 
         executeBatch(false, 25, 2);
-        assertEquals(2, cfs.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
-        assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+        assertEquals(2, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount()); // 2 for previous batch and this batch
+        assertGreaterThan(cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0);
     }
 
     @Test
     public void testUnloggedPartitionsPerBatchMultiTable()
     {
         ColumnFamilyStore first = recreateTable();
-        assertEquals(0, first.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, first.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        assertEquals(0, first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(0.0, first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0.0);
 
         ColumnFamilyStore second = recreateTable(TABLE + "_second");
-        assertEquals(0, second.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, second.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        assertEquals(0, second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(0.0, second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0.0);
 
         executeBatch(false, 5, 3, TABLE, TABLE + "_second");
-        assertEquals(1, first.metric.coordinatorWriteLatency.getCount());
+        assertEquals(1, first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
 
         executeBatch(false, 25, 2, TABLE, TABLE + "_second");
-        assertEquals(2, first.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
-        assertEquals(2, second.metric.coordinatorWriteLatency.getCount()); // 2 for previous batch and this batch
-        assertGreaterThan(first.metric.coordinatorWriteLatency.getMeanRate(), 0);
-        assertGreaterThan(second.metric.coordinatorWriteLatency.getMeanRate(), 0);
+        assertEquals(2, first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount()); // 2 for previous batch and this batch
+        assertEquals(2, second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount()); // 2 for previous batch and this batch
+        assertGreaterThan(first.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0);
+        assertGreaterThan(second.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0);
     }
 
     @Test
     public void testCounterStatement()
     {
         ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, COUNTER_TABLE);
-        assertEquals(0, cfs.metric.coordinatorWriteLatency.getCount());
-        assertEquals(0.0, cfs.metric.coordinatorWriteLatency.getMeanRate(), 0.0);
+        assertEquals(0, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertEquals(0.0, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0.0);
         session.execute(String.format("UPDATE %s.%s SET id_c = id_c + 1 WHERE id = 1 AND val = 'val1'", KEYSPACE, COUNTER_TABLE));
-        assertEquals(1, cfs.metric.coordinatorWriteLatency.getCount());
-        assertGreaterThan(cfs.metric.coordinatorWriteLatency.getMeanRate(), 0);
+        assertEquals(1, cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getCount());
+        assertGreaterThan(cfs.metric.coordinatorWriteLatency.tableOrKeyspaceTimer().getMeanRate(), 0);
     }
 
-    private static void assertGreaterThan(double actual, double expectedLessThan) {
+    private static void assertGreaterThan(double actual, double expectedLessThan)
+    {
         assertTrue("Expected " + actual + " > " + expectedLessThan, actual > expectedLessThan);
     }
 
@@ -319,6 +408,55 @@ public void testViewMetricsCleanupOnDrop()
         assertEquals(metrics.get().collect(Collectors.joining(",")), 0, metrics.get().count());
     }
 
+    @Test
+    public void testEstimatedPartitionCount() throws InterruptedException
+    {
+        ColumnFamilyStore cfs = recreateTable();
+        assertEquals(0L, cfs.metric.estimatedPartitionCount.getValue().longValue());
+        assertEquals(0L, cfs.metric.estimatedPartitionCountInSSTablesCached.getValue().longValue());
+        long startTime = System.currentTimeMillis();
+
+        int partitionCount = 10;
+        int numRows = 100;
+
+        for (int i = 0; i < numRows; i++)
+            session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i % partitionCount, "val" + i, "val" + i));
+
+        assertEquals(partitionCount, cfs.metric.estimatedPartitionCount.getValue().longValue());
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        assertEquals(partitionCount, cfs.metric.estimatedPartitionCount.getValue().longValue());
+
+        long estimatedPartitionCountInSSTables = cfs.metric.estimatedPartitionCountInSSTablesCached.getValue().longValue();
+        long elapsedTime = System.currentTimeMillis() - startTime;
+        // the caching time is one second; avoid flakiness by only checking if a long time has not passed
+        // (Because we take the time after calling the method, elapsedTime < 1000 should also be stable, but let's also
+        // accommodate the possibility that the cache uses a different timer with different tick times.)
+        if (elapsedTime < 980)
+            assertEquals(0, estimatedPartitionCountInSSTables);
+
+        for (int i = 0; i < numRows; i++)
+            session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i % partitionCount, "val" + i, "val" + i));
+
+        estimatedPartitionCountInSSTables = cfs.metric.estimatedPartitionCountInSSTablesCached.getValue().longValue();
+        elapsedTime = System.currentTimeMillis() - startTime;
+        if (elapsedTime < 980)
+            assertEquals(0, estimatedPartitionCountInSSTables);
+        else if (elapsedTime >= 1020)
+            assertEquals(partitionCount, estimatedPartitionCountInSSTables);
+
+        // The answer below is incorrect but what the metric currently returns.
+        assertEquals(partitionCount * 2, cfs.metric.estimatedPartitionCount.getValue().longValue());
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        // Recalculation for the new sstable set will correct it.
+        assertEquals(partitionCount, cfs.metric.estimatedPartitionCount.getValue().longValue());
+
+        // The cached estimatedPartitionCountInSSTables lags one second, check that.
+        // Assert that the metric will return a correct value after at least a second passes
+        Awaitility.await()
+                  .atMost(2, TimeUnit.SECONDS)
+                  .untilAsserted(() -> assertEquals(partitionCount, (long) cfs.metric.estimatedPartitionCountInSSTablesCached.getValue()));
+    }
+
 
     @AfterClass
     public static void tearDown()
diff --git a/test/unit/org/apache/cassandra/metrics/TrieMemtableMetricsTest.java b/test/unit/org/apache/cassandra/metrics/TrieMemtableMetricsTest.java
index 37cf8b1156ea..fd5dd73461af 100644
--- a/test/unit/org/apache/cassandra/metrics/TrieMemtableMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/TrieMemtableMetricsTest.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -39,15 +40,17 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.OverrideConfigurationLoader;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.memtable.AbstractShardedMemtable;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.service.EmbeddedCassandraService;
 import org.apache.cassandra.service.StorageService;
+import org.awaitility.Awaitility;
 import org.jboss.byteman.contrib.bmunit.BMRule;
 import org.jboss.byteman.contrib.bmunit.BMRules;
 import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
-import static org.hamcrest.Matchers.*;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -83,6 +86,8 @@ public static void setup() throws ConfigurationException, IOException
         EmbeddedCassandraService cassandra = new EmbeddedCassandraService();
         cassandra.start();
 
+        assertEquals(AbstractShardedMemtable.getDefaultShardCount(), NUM_SHARDS);
+
         Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build();
         session = cluster.connect();
 
@@ -130,17 +135,28 @@ public void testFlushRelatedMetrics() throws IOException, ExecutionException, In
         writeAndFlush(10);
         assertEquals(10, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
 
+        // lastFlushShardDataSize is updated asynchronously. Wait until we can see the result.
+        Awaitility.await()
+                  .atMost(30, TimeUnit.SECONDS)
+                  .until(() -> metrics.lastFlushShardDataSizes.maxGauge.getValue() > 0);
+        Long maxShardSize = metrics.lastFlushShardDataSizes.maxGauge.getValue();
+
         // verify that metrics survive flush / memtable switching
-        writeAndFlush(10);
-        assertEquals(20, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
-        assertEquals(metrics.lastFlushShardDataSizes.toString(), NUM_SHARDS, metrics.lastFlushShardDataSizes.numSamplesGauge.getValue().intValue());
+        writeAndFlush(100);
+        assertEquals(110, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
+        // lastFlushShardDataSize is updated asynchronously. Wait until we can see the result.
+        Awaitility.await()
+                  .atMost(30, TimeUnit.SECONDS)
+                  .until(() -> metrics.lastFlushShardDataSizes.maxGauge.getValue() > maxShardSize);
+        assertEquals(metrics.lastFlushShardDataSizes.toString(), NUM_SHARDS, (int) metrics.lastFlushShardDataSizes.numSamplesGauge.getValue());
+        assertThat(metrics.lastFlushShardDataSizes.maxGauge.getValue(), greaterThan(maxShardSize));
     }
 
     @Test
     @BMRules(rules = { @BMRule(name = "Delay memtable update",
     targetClass = "InMemoryTrie",
     targetMethod = "putSingleton",
-    action = "java.lang.Thread.sleep(10)")})
+    action = "java.lang.Thread.sleep(100)")})
     public void testContentionMetrics() throws IOException, ExecutionException, InterruptedException
     {
         ColumnFamilyStore cfs = recreateTable();
@@ -180,7 +196,7 @@ public void testMetricsCleanupOnDrop()
 
     private TrieMemtableMetricsView getMemtableMetrics(ColumnFamilyStore cfs)
     {
-        return new TrieMemtableMetricsView(cfs.getKeyspaceName(), cfs.name);
+        return TrieMemtableMetricsView.getOrCreate(cfs.keyspace.getName(), cfs.name);
     }
 
     private void writeAndFlush(int rows) throws IOException, ExecutionException, InterruptedException
diff --git a/test/unit/org/apache/cassandra/net/ConnectionTest.java b/test/unit/org/apache/cassandra/net/ConnectionTest.java
index 1e24dd637555..a3d944a69736 100644
--- a/test/unit/org/apache/cassandra/net/ConnectionTest.java
+++ b/test/unit/org/apache/cassandra/net/ConnectionTest.java
@@ -66,11 +66,15 @@
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.FBUtilities;
+import org.awaitility.Awaitility;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_10;
+import static org.apache.cassandra.net.MessagingService.VERSION_DS_11;
+import static org.apache.cassandra.net.MessagingService.minimum_version;
 import static org.apache.cassandra.net.NoPayload.noPayload;
 import static org.apache.cassandra.net.MessagingService.current_version;
 import static org.apache.cassandra.net.ConnectionType.LARGE_MESSAGES;
@@ -114,17 +118,24 @@ public void resetVerbs() throws Throwable
         timeouts.clear();
     }
 
+    private static volatile long originalRpcTimeout = 0;
+
     @BeforeClass
     public static void startup()
     {
         DatabaseDescriptor.daemonInitialization();
         CommitLog.instance.start();
+        // At the time of this commit, the default is 20 seconds and leads to significant delays
+        // in this test class, especially in testMessagePurging and testCloseIfEndpointDown.
+        originalRpcTimeout = DatabaseDescriptor.getRpcTimeout(TimeUnit.MILLISECONDS);
+        DatabaseDescriptor.setRpcTimeout(5000L);
     }
 
     @AfterClass
     public static void cleanup() throws InterruptedException
     {
         factory.shutdownNow();
+        DatabaseDescriptor.setRpcTimeout(originalRpcTimeout);
     }
 
     interface SendTest
@@ -183,27 +194,57 @@ Settings override(Settings settings)
             .withRequireClientAuth(false)
             .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA");
 
+    // 30 is not supported in 5.0
+    // 40 is used for CNDB compatibility
+    static final AcceptVersions legacy = new AcceptVersions(VERSION_40, VERSION_40);
+    static final AcceptVersions ds10 = new AcceptVersions(minimum_version, VERSION_DS_10);
+    static final AcceptVersions ds11 = new AcceptVersions(minimum_version, VERSION_DS_11);
+    static final AcceptVersions current = new AcceptVersions(current_version, current_version);
+
+    static final List<Function<Settings, Settings>> MESSAGGING_VERSIONS = ImmutableList.of(
+        settings -> settings.outbound(outbound -> outbound.withAcceptVersions(legacy))
+                            .inbound(inbound -> inbound.withAcceptMessaging(legacy)),
+        // Mismatched versions (in both directions) to ensure both peers will still agree on the same version.
+        settings -> settings.outbound(outbound -> outbound.withAcceptVersions(ds11))
+                            .inbound(inbound -> inbound.withAcceptMessaging(ds10)),
+        settings -> settings.outbound(outbound -> outbound.withAcceptVersions(ds10))
+                            .inbound(inbound -> inbound.withAcceptMessaging(ds11)),
+        // This setting ensures that we cover the current case for the power set where no versions are overridden.
+        settings -> settings.outbound(outbound -> outbound.withAcceptVersions(current))
+                            .inbound(inbound -> inbound.withAcceptMessaging(current))
+    );
+
+
     static final List<Function<Settings, Settings>> MODIFIERS = ImmutableList.of(
         settings -> settings.outbound(outbound -> outbound.withEncryption(encryptionOptions))
                             .inbound(inbound -> inbound.withEncryption(encryptionOptions)),
         settings -> settings.outbound(outbound -> outbound.withFraming(LZ4))
     );
 
+    // Messaging versions are a kind of modifier, but they can only be applied once per setting, so they are broken
+    // out into a separate list.
     static final List<Settings> SETTINGS = applyPowerSet(
-        ImmutableList.of(Settings.SMALL, Settings.LARGE),
+        ImmutableList.of(ConnectionTest.Settings.SMALL, ConnectionTest.Settings.LARGE),
+        MESSAGGING_VERSIONS,
         MODIFIERS
     );
 
-    private static <T> List<T> applyPowerSet(List<T> settings, List<Function<T, T>> modifiers)
+    private static List<Settings> applyPowerSet(List<ConnectionTest.Settings> settings,
+                                                List<Function<ConnectionTest.Settings, ConnectionTest.Settings>> messagingVersions,
+                                                List<Function<ConnectionTest.Settings, ConnectionTest.Settings>> modifiers)
     {
-        List<T> result = new ArrayList<>();
-        for (Set<Function<T, T>> set : Sets.powerSet(new HashSet<>(modifiers)))
+        List<Settings> result = new ArrayList<>();
+        for (Function<ConnectionTest.Settings, ConnectionTest.Settings> messagingVersion : messagingVersions)
         {
-            for (T s : settings)
+            for (Set<Function<Settings, ConnectionTest.Settings>> set : Sets.powerSet(new HashSet<>(modifiers)))
             {
-                for (Function<T, T> f : set)
-                    s = f.apply(s);
-                result.add(s);
+                for (ConnectionTest.Settings s : settings)
+                {
+                    for (Function<Settings, ConnectionTest.Settings> f : set)
+                        s = f.apply(s);
+                    s = messagingVersion.apply(s);
+                    result.add(s);
+                }
             }
         }
         return result;
@@ -294,6 +335,10 @@ public void testSendSmall() throws Throwable
                            .expired  ( 0,  0)
                            .error    ( 0,  0)
                            .check();
+
+            // Ensure version is the same
+            inbound.assertHandlersMessagingVersion(outbound.messagingVersion());
+            Assert.assertEquals(outbound.settings().endpointToVersion.get(endpoint), outbound.messagingVersion());
         });
     }
 
@@ -348,6 +393,10 @@ public long serializedSize(Object noPayload, int version)
                            .expired  ( 0,  0)
                            .error    ( 0,  0)
                            .check();
+
+            // Ensure version is the same
+            inbound.assertHandlersMessagingVersion(outbound.messagingVersion());
+            Assert.assertEquals(outbound.settings().endpointToVersion.get(endpoint), outbound.messagingVersion());
         });
     }
 
@@ -632,6 +681,7 @@ public void testMessageDeliveryOnReconnect() throws Throwable
 
                 // Simulate disconnect
                 inbound.close().get(10, SECONDS);
+                Awaitility.await().timeout(10, SECONDS).until(() -> !outbound.isConnected());
                 MessagingService.instance().removeInbound(endpoint);
                 inbound = new InboundSockets(settings.inbound.apply(new InboundConnectionSettings()));
                 inbound.open().sync();
diff --git a/test/unit/org/apache/cassandra/net/FramingTest.java b/test/unit/org/apache/cassandra/net/FramingTest.java
index 6695012b8cc3..450310c24edf 100644
--- a/test/unit/org/apache/cassandra/net/FramingTest.java
+++ b/test/unit/org/apache/cassandra/net/FramingTest.java
@@ -32,7 +32,6 @@
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -47,7 +46,7 @@
 import org.apache.cassandra.utils.memory.BufferPools;
 import org.apache.cassandra.utils.vint.VIntCoding;
 
-import static java.lang.Math.*;
+import static java.lang.Math.min;
 import static org.apache.cassandra.net.ShareableBytes.wrap;
 
 // TODO: test corruption
diff --git a/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java b/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java
index 88a45af510ba..25abf617ccc4 100644
--- a/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java
+++ b/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java
@@ -35,7 +35,6 @@
 import org.apache.cassandra.schema.SchemaProvider;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.CassandraGenerators;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.FixedMonotonicClock;
 import org.assertj.core.api.Assertions;
 import org.mockito.Mockito;
@@ -43,6 +42,7 @@
 import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_APPROX;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_PRECISE;
 import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
 import static org.apache.cassandra.net.Message.serializer;
 import static org.apache.cassandra.utils.CassandraGenerators.MESSAGE_GEN;
 import static org.apache.cassandra.utils.FailingConsumer.orFail;
@@ -57,6 +57,7 @@ public static void beforeClass()
         // message serialization uses the MonotonicClock class for precise and approx timestamps, so mock it out
         CLOCK_MONOTONIC_PRECISE.setString(FixedMonotonicClock.class.getName());
         CLOCK_MONOTONIC_APPROX.setString(FixedMonotonicClock.class.getName());
+        VECTOR_FLOAT_ONLY.setBoolean(false);
 
         DatabaseDescriptor.daemonInitialization();
     }
@@ -108,7 +109,7 @@ public void testMessageSerialization() throws Exception
                         FixedMonotonicClock.setNowInNanos(message.createdAtNanos());
 
                         serializer.serialize(message, first, version.value);
-                        Message<Object> read = serializer.deserialize(new DataInputBuffer(first.buffer(), true), FBUtilities.getBroadcastAddressAndPort(), version.value);
+                        Message<Object> read = serializer.deserialize(new DataInputBuffer(first.buffer(), true), message.from(), version.value);
                         serializer.serialize(read, second, version.value);
                         // using hex as byte buffer equality kept failing, and was harder to debug difference
                         // using hex means the specific section of the string that is different will be shown
diff --git a/test/unit/org/apache/cassandra/net/MessageTest.java b/test/unit/org/apache/cassandra/net/MessageTest.java
index 0228608ea411..049c79c8ac99 100644
--- a/test/unit/org/apache/cassandra/net/MessageTest.java
+++ b/test/unit/org/apache/cassandra/net/MessageTest.java
@@ -36,6 +36,7 @@
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.tracing.Tracing.TraceType;
 import org.apache.cassandra.utils.FBUtilities;
@@ -243,11 +244,59 @@ public void testCustomParams() throws CharacterCodingException, IOException
         assertEquals("custom2value", new String(msg.header.customParams().get("custom2"), StandardCharsets.UTF_8));
     }
 
+    @Test
+    public void testResponseWithCustomParams()
+    {
+        long id = 1;
+        InetAddressAndPort from = FBUtilities.getLocalAddressAndPort();
+
+        Message<NoPayload> msg =
+                Message.builder(Verb.READ_REQ, noPayload)
+                        .withId(1)
+                        .from(from)
+                        .build();
+
+        Message reply = msg.responseWithBuilder(msg)
+                           .withCustomParam("custom1", "custom1value".getBytes(StandardCharsets.UTF_8))
+                           .withCustomParam("custom2", "custom2value".getBytes(StandardCharsets.UTF_8))
+                           .build();
+
+        assertEquals(id, reply.id());
+        assertEquals(from, reply.from());
+        assertEquals(2, reply.header.customParams().size());
+        assertEquals("custom1value", new String(reply.header.customParams().get("custom1"), StandardCharsets.UTF_8));
+        assertEquals("custom2value", new String(reply.header.customParams().get("custom2"), StandardCharsets.UTF_8));
+    }
+
+    @Test
+    public void testEmptyResponseBuilderWithCustomParams()
+    {
+        long id = 1;
+        InetAddressAndPort from = FBUtilities.getLocalAddressAndPort();
+
+        Message<NoPayload> msg =
+        Message.builder(Verb.READ_REQ, noPayload)
+               .withId(1)
+               .from(from)
+               .build();
+
+        Message reply = msg.emptyResponseBuilder()
+                           .withCustomParam("custom1", "custom1value".getBytes(StandardCharsets.UTF_8))
+                           .withCustomParam("custom2", "custom2value".getBytes(StandardCharsets.UTF_8))
+                           .build();
+
+        assertEquals(id, reply.id());
+        assertEquals(from, reply.from());
+        assertEquals(2, reply.header.customParams().size());
+        assertEquals("custom1value", new String(reply.header.customParams().get("custom1"), StandardCharsets.UTF_8));
+        assertEquals("custom2value", new String(reply.header.customParams().get("custom2"), StandardCharsets.UTF_8));
+    }
+
     private void testAddTraceHeaderWithType(TraceType traceType)
     {
         try
         {
-            TimeUUID sessionId = Tracing.instance.newSession(traceType);
+            TimeUUID sessionId = Tracing.instance.newSession(ClientState.forInternalCalls(), traceType);
             Message<NoPayload> msg = Message.builder(Verb._TEST_1, noPayload).withTracingParams().build();
             assertEquals(sessionId, msg.header.traceSession());
             assertEquals(traceType, msg.header.traceType());
diff --git a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
index 32d505038868..7db3bb12f7fb 100644
--- a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
+++ b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
@@ -32,7 +32,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -48,6 +47,7 @@
 
 import com.codahale.metrics.Timer;
 import org.apache.cassandra.auth.IInternodeAuthenticator;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
 import org.apache.cassandra.db.commitlog.CommitLog;
@@ -65,6 +65,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.apache.cassandra.net.NoPayload.noPayload;
 
 public class MessagingServiceTest
 {
@@ -182,7 +183,7 @@ public void testDroppedMessages()
     public void testDCLatency()
     {
         int latency = 100;
-        ConcurrentHashMap<String, MessagingMetrics.DCLatencyRecorder> dcLatency = MessagingService.instance().metrics.dcLatency;
+        Map<String, MessagingMetrics.DCLatencyRecorder> dcLatency = MessagingService.instance().metrics.dcLatency;
         dcLatency.clear();
 
         long now = System.currentTimeMillis();
@@ -198,7 +199,8 @@ public void testDCLatency()
     @Test
     public void testNegativeDCLatency()
     {
-        MessagingMetrics.DCLatencyRecorder updater = MessagingService.instance().metrics.internodeLatencyRecorder(InetAddressAndPort.getLocalHost());
+        MessagingMetrics.DCLatencyRecorder updater =
+        (MessagingMetrics.DCLatencyRecorder) MessagingService.instance().metrics.internodeLatencyRecorder(InetAddressAndPort.getLocalHost());
 
         // if clocks are off should just not track anything
         int latency = -100;
@@ -207,7 +209,7 @@ public void testNegativeDCLatency()
         long sentAt = now - latency;
 
         long count = updater.dcLatency.getCount();
-        updater.accept(now - sentAt, MILLISECONDS);
+        updater.accept(Verb.READ_REQ, now - sentAt, MILLISECONDS);
         // negative value shoudln't be recorded
         assertEquals(count, updater.dcLatency.getCount());
     }
@@ -219,7 +221,7 @@ public void testQueueWaitLatency()
         Verb verb = Verb.MUTATION_REQ;
 
         Map<Verb, Timer> queueWaitLatency = MessagingService.instance().metrics.internalLatency;
-        MessagingService.instance().metrics.recordInternalLatency(verb, latency, MILLISECONDS);
+        MessagingService.instance().metrics.recordInternalLatency(verb, InetAddressAndPort.getLocalHost(), latency, MILLISECONDS);
         assertEquals(1, queueWaitLatency.get(verb).getCount());
         long expectedBucket = bucketOffsets[Math.abs(Arrays.binarySearch(bucketOffsets, MILLISECONDS.toMicros(latency))) - 1];
         assertEquals(expectedBucket, queueWaitLatency.get(verb).getSnapshot().getMax());
@@ -235,13 +237,13 @@ public void testNegativeQueueWaitLatency()
         queueWaitLatency.clear();
 
         assertNull(queueWaitLatency.get(verb));
-        MessagingService.instance().metrics.recordInternalLatency(verb, latency, MILLISECONDS);
+        MessagingService.instance().metrics.recordInternalLatency(verb, InetAddressAndPort.getLocalHost(), latency, MILLISECONDS);
         assertNull(queueWaitLatency.get(verb));
     }
 
     private static void addDCLatency(long sentAt, long nowTime)
     {
-        MessagingService.instance().metrics.internodeLatencyRecorder(InetAddressAndPort.getLocalHost()).accept(nowTime - sentAt, MILLISECONDS);
+        MessagingService.instance().metrics.internodeLatencyRecorder(InetAddressAndPort.getLocalHost()).accept(Verb.READ_REQ, nowTime - sentAt, MILLISECONDS);
     }
 
     /**
@@ -517,4 +519,74 @@ private void listen(ServerEncryptionOptions serverEncryptionOptions, boolean lis
 //
 //        Assert.assertEquals(privateIp, ms.getPreferredRemoteAddr(remote));
 //    }
+
+    private static class PostSinkFilter
+    {
+        private Verb verb;
+        public int count;
+
+        PostSinkFilter(Verb verb)
+        {
+            this.verb = verb;
+        }
+
+        public void accept(Message<?> message, InetAddressAndPort to)
+        {
+            // Count all the messages seen for our verb
+            if (message.verb() == verb)
+            {
+                count++;
+            }
+        }
+    }
+
+    @Test
+    public void runPostSinkHookVerbFilter() throws UnknownHostException
+    {
+        PostSinkFilter echoSink = new PostSinkFilter(Verb.ECHO_REQ);
+        MessagingService.instance().outboundSink.addPost((message, to) -> echoSink.accept(message, to));
+
+        int numOfMessages = 3;
+        // echoRecorder should see all ECHO_REQ messages
+        sendMessages(numOfMessages, Verb.ECHO_REQ);
+        assertEquals(numOfMessages, echoSink.count);
+
+        PostSinkFilter hintSink = new PostSinkFilter(Verb.HINT_REQ);
+        MessagingService.instance().outboundSink.addPost((message, to) -> hintSink.accept(message, to));
+
+        // hintRecorder should not see any ECHO_REQ messages
+        sendMessages(numOfMessages, Verb.ECHO_REQ);
+        assertEquals(0, hintSink.count);
+    }
+
+    public static class TestMessagingMetrics extends MessagingMetrics {}
+
+    @Test
+    public void testCreatingCustomMessagingMetrics()
+    {
+        String originalValue = CassandraRelevantProperties.CUSTOM_MESSAGING_METRICS_PROVIDER_PROPERTY.getString();
+        try
+        {
+            CassandraRelevantProperties.CUSTOM_MESSAGING_METRICS_PROVIDER_PROPERTY.setString(TestMessagingMetrics.class.getName());
+            MessagingService testMessagingService = new MessagingService(true);
+            assertTrue(testMessagingService.metrics instanceof TestMessagingMetrics);
+        }
+        finally
+        {
+            if (originalValue == null)
+                System.clearProperty(CassandraRelevantProperties.CUSTOM_MESSAGING_METRICS_PROVIDER_PROPERTY.getKey());
+            else
+                CassandraRelevantProperties.CUSTOM_MESSAGING_METRICS_PROVIDER_PROPERTY.setString(originalValue);
+        }
+    }
+
+    private static void sendMessages(int numOfMessages, Verb verb) throws UnknownHostException
+    {
+        InetAddressAndPort address = InetAddressAndPort.getByName("127.0.0.253");
+
+        for (int i = 0; i < numOfMessages; i++)
+        {
+            MessagingService.instance().send(Message.out(verb, noPayload), address);
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java b/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java
index b21de607b323..a6f13774f0e6 100644
--- a/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java
+++ b/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java
@@ -389,7 +389,7 @@ private void doTestManual(ConnectionTest.Settings settings, ManualSendTest test)
 
     private void connect(OutboundConnection outbound) throws Throwable
     {
-        tryConnect(outbound, 10, SECONDS, true);
+        tryConnect(outbound, 30, SECONDS, true);
     }
 
     private void tryConnect(OutboundConnection outbound, long timeout, TimeUnit timeUnit, boolean throwOnFailure) throws Throwable
diff --git a/test/unit/org/apache/cassandra/net/RequestCallbacksTest.java b/test/unit/org/apache/cassandra/net/RequestCallbacksTest.java
new file mode 100644
index 000000000000..0c9c476c8df4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/RequestCallbacksTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
+
+public class RequestCallbacksTest
+{
+    @BeforeClass
+    public static void init()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void testInternalResponseNullSerializer() throws Exception
+    {
+        deprecatedResponsesShouldReturnNullSerializer(Verb.INTERNAL_RSP);
+    }
+
+    @Test
+    public void testRequestResponseNullSerializer() throws Exception
+    {
+        deprecatedResponsesShouldReturnNullSerializer(Verb.REQUEST_RSP);
+    }
+
+    public void deprecatedResponsesShouldReturnNullSerializer(Verb verb) throws Exception
+    {
+        MessagingService messagingService = mock(MessagingService.class);
+        RequestCallbacks requestCallbacks = new RequestCallbacks(messagingService);
+        Message<?> msg = Message.remoteResponse(InetAddressAndPort.getByName("127.0.0.1"), verb, Message.NO_PARAMS,null);
+        requestCallbacks.addWithExpiration(mock(RequestCallback.class), msg, msg.from());
+
+        IVersionedAsymmetricSerializer<Object, Object> serializer = verb.serializer();
+
+        assertNull(serializer);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/net/ResourceLimitsTest.java b/test/unit/org/apache/cassandra/net/ResourceLimitsTest.java
index 5c2ecbed6ac4..1e41f4cf7755 100644
--- a/test/unit/org/apache/cassandra/net/ResourceLimitsTest.java
+++ b/test/unit/org/apache/cassandra/net/ResourceLimitsTest.java
@@ -24,6 +24,7 @@
 import java.util.function.LongFunction;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.JVMKiller;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.KillerForTests;
 import org.junit.Assert;
@@ -159,7 +160,7 @@ public void negativeConcurrentUsingValueKillsJVMTest()
     {
         DatabaseDescriptor.daemonInitialization(); // Prevent NPE for DatabaseDescriptor.getDiskFailurePolicy
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
         try
         {
             Concurrent concurrent = new Concurrent(1);
diff --git a/test/unit/org/apache/cassandra/net/VerbTest.java b/test/unit/org/apache/cassandra/net/VerbTest.java
index 8f205678485a..b08649a98b61 100644
--- a/test/unit/org/apache/cassandra/net/VerbTest.java
+++ b/test/unit/org/apache/cassandra/net/VerbTest.java
@@ -18,16 +18,129 @@
 
 package org.apache.cassandra.net;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.function.Supplier;
+
 import org.junit.Test;
 
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.MutationVerbHandler;
+
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
 
 public class VerbTest
 {
     @Test
     public void idsMatch()
     {
-        for (Verb v : Verb.values())
+        for (Verb v : Verb.getValues())
             assertEquals(v, Verb.fromId(v.id));
     }
+
+    @Test
+    public void verbName()
+    {
+        // MUTATION_RSP is the first
+        Verb v = Verb.getValues().get(0);
+        assertEquals("MUTATION_RSP", v.toString());
+    }
+
+    @Test
+    public void invalidVerbIdThrows()
+    {
+        boolean threw = false;
+        try
+        {
+            Verb.fromId(-1);
+        }
+        catch (IllegalArgumentException e)
+        {
+            threw = true;
+        }
+        assertTrue("Invalid verb identifier was valid", threw);
+    }
+
+    @Test
+    public void mutationVerbHasMutationHandler()
+    {
+        Verb mutationReqVerb = Verb.fromId(Verb.MUTATION_REQ.id);
+        assertEquals(MutationVerbHandler.instance, mutationReqVerb.handler());
+    }
+
+    @Test
+    public void addNewVerbWithConflictingId()
+    {
+        boolean threw = false;
+        try
+        {
+            addFakeVerb(Verb.UNUSED_CUSTOM_VERB.id, "FAKE_REQ");
+        }
+        catch (IllegalArgumentException ex)
+        {
+            threw = true;
+        }
+        assertTrue("Expected IllegalArgumentException when adding existing verb id", threw);
+    }
+
+    @Test
+    public void addingTwoVerbsHasDistinctIds()
+    {
+        Verb FAKE_REQ = addFakeVerb(1, "FAKE_REQ");
+        assertNotEquals(null, FAKE_REQ);
+        assertEquals(FAKE_REQ, Verb.fromId(FAKE_REQ.id));
+
+        Verb FAKE_REQ2 = addFakeVerb(256, "FAKE_REQ2");
+        assertEquals(FAKE_REQ2, Verb.fromId(FAKE_REQ2.id));
+        assertNotEquals(FAKE_REQ, Verb.fromId(FAKE_REQ2.id));
+    }
+
+    private Verb addFakeVerb(int id, String name)
+    {
+        return addFakeVerb(id, name, () -> MutationVerbHandler.instance);
+    }
+
+    private Verb addFakeVerb(int id, String name, Supplier<? extends IVerbHandler<?>> handler)
+    {
+        return Verb.addCustomVerb(name, id, Verb.Priority.P0, VerbTimeouts.writeTimeout, Stage.MUTATION, () -> NoPayload.serializer, handler, Verb.MUTATION_RSP);
+    }
+
+    private static class CountingVerbHandler implements IVerbHandler<Integer>
+    {
+        public int count;
+        @Override
+        public void doVerb(Message<Integer> message)
+        {
+            count++;
+        }
+    }
+
+    private static class DoubleCaller implements IVerbHandler<Integer>
+    {
+        IVerbHandler<Integer> handler;
+        public DoubleCaller(IVerbHandler<Integer> handler)
+        {
+           this.handler = handler;
+        }
+
+        @Override
+        public void doVerb(Message<Integer> message) throws IOException
+        {
+            handler.doVerb(message);
+            handler.doVerb(message);
+        }
+    }
+
+    @Test
+    public void decorateHandler() throws IOException
+    {
+        CountingVerbHandler handler = new CountingVerbHandler();
+        Verb decoratedVerb = addFakeVerb(23, "DECORATED_VERB", () -> handler);
+
+        Verb.decorateHandler(Arrays.asList(decoratedVerb), (oldHandler) -> new DoubleCaller((IVerbHandler<Integer>) oldHandler));
+        decoratedVerb.handler().doVerb(null);
+        assertEquals(2, handler.count);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java b/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java
new file mode 100644
index 000000000000..afb30701c526
--- /dev/null
+++ b/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java
@@ -0,0 +1,101 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.cassandra.net;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.junit.Assert;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.paxos.Ballot;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class WriteCallbackInfoTest
+{
+    @BeforeClass
+    public static void initDD()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void testShouldHint() throws Exception
+    {
+        testShouldHint(Verb.COUNTER_MUTATION_REQ, ConsistencyLevel.ALL, true, false);
+        for (Verb verb : new Verb[] { Verb.PAXOS_COMMIT_REQ, Verb.MUTATION_REQ })
+        {
+            testShouldHint(verb, ConsistencyLevel.ALL, true, true);
+            testShouldHint(verb, ConsistencyLevel.ANY, true, false);
+            testShouldHint(verb, ConsistencyLevel.ALL, false, false);
+        }
+    }
+
+    @Test
+    public void testIMutation() throws Exception
+    {
+        for (Verb verb : new Verb[]{ Verb.PAXOS_COMMIT_REQ, Verb.MUTATION_REQ })
+        {
+            testIMutation(verb, true);
+            testIMutation(verb, false);
+        }
+    }
+
+    private void testShouldHint(Verb verb, ConsistencyLevel cl, boolean allowHints, boolean expectHint) throws Exception
+    {
+        TableMetadata metadata = MockSchema.newTableMetadata("", "");
+        Object payload = verb == Verb.PAXOS_COMMIT_REQ
+                         ? new Commit(Ballot.none(), PartitionUpdate.builder(metadata, ByteBufferUtil.EMPTY_BYTE_BUFFER, RegularAndStaticColumns.NONE, 1).build())
+                         : new Mutation(PartitionUpdate.simpleBuilder(metadata, "").build());
+
+        RequestCallbacks.WriteCallbackInfo wcbi = new RequestCallbacks.WriteCallbackInfo(Message.out(verb, payload), InetAddressAndPort.getByName("192.168.1.1"), null);
+        Assert.assertNotNull(wcbi.iMutation());
+    }
+
+    private void testIMutation(Verb verb, boolean allowHints) throws Exception
+    {
+        TableMetadata metadata = MockSchema.newTableMetadata("", "");
+        Object payload;
+        if (verb == Verb.PAXOS_COMMIT_REQ)
+        {
+            //            UUID uuid = UUID.randomUUID();
+            PartitionUpdate update = PartitionUpdate.builder(metadata, ByteBufferUtil.EMPTY_BYTE_BUFFER, RegularAndStaticColumns.NONE, 1).build();
+            payload = new Commit(Ballot.none(), update);
+        }
+        else if (verb == Verb.COUNTER_MUTATION_REQ)
+        {
+            Mutation mutation = new Mutation(PartitionUpdate.simpleBuilder(metadata, "").build());
+            payload = new CounterMutation(mutation, null);
+        }
+        else
+            payload = new Mutation(PartitionUpdate.simpleBuilder(metadata, "").build());
+
+        RequestCallbacks.WriteCallbackInfo wcbi = new RequestCallbacks.WriteCallbackInfo(Message.out(verb, payload), InetAddressAndPort.getByName("192.168.1.1"), null);
+        Assert.assertNotNull(wcbi.iMutation());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/nodes/NodesPersistenceTest.java b/test/unit/org/apache/cassandra/nodes/NodesPersistenceTest.java
new file mode 100644
index 000000000000..915e64080ecd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/nodes/NodesPersistenceTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.CassandraVersion;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class NodesPersistenceTest extends CQLTester
+{
+    private final NodesPersistence np = new NodesPersistence();
+
+
+    private void fillNodeInfoSampleData(NodeInfo info) throws UnknownHostException
+    {
+        info.setHostId(UUID.randomUUID());
+        info.setReleaseVersion(new CassandraVersion("4.1.2"));
+        info.setRack("rack1");
+        info.setDataCenter("dc1");
+        info.setSchemaVersion(UUID.randomUUID());
+        info.setNativeTransportAddressAndPort(InetAddressAndPort.getByNameOverrideDefaults("127.1.2.3", 111));
+        Token.TokenFactory tf = StorageService.instance.getTokenFactory();
+        info.setTokens(Arrays.asList(tf.fromString("1"), tf.fromString("2"), tf.fromString("3")));
+    }
+
+    private void fillLocalInfoSampleData(LocalInfo info) throws UnknownHostException
+    {
+        fillNodeInfoSampleData(info);
+        info.setListenAddressAndPort(InetAddressAndPort.getByNameOverrideDefaults("127.2.3.4", 222));
+        info.setClusterName("cluster1");
+        info.setCqlVersion(new CassandraVersion("2.3.4"));
+        info.setBootstrapState(SystemKeyspace.BootstrapState.COMPLETED);
+        info.setBroadcastAddressAndPort(InetAddressAndPort.getByNameOverrideDefaults("127.3.4.5", 333));
+        info.setNativeProtocolVersion(ProtocolVersion.V5);
+        info.setPartitionerClass(Murmur3Partitioner.class);
+        info.setTruncationRecords(ImmutableMap.of(UUID.randomUUID(), new TruncationRecord(new CommitLogPosition(22, 33), 44),
+                                                  UUID.randomUUID(), new TruncationRecord(new CommitLogPosition(55, 66), 77)));
+    }
+
+    private void fillPeerInfoSampleData(PeerInfo info) throws UnknownHostException
+    {
+        fillNodeInfoSampleData(info);
+        info.setPeerAddressAndPort(InetAddressAndPort.getByNameOverrideDefaults("127.4.5.6", 444));
+        info.setPreferredAddressAndPort(InetAddressAndPort.getByNameOverrideDefaults("127.5.6.7", 555));
+    }
+
+    @Test
+    public void testLocal() throws Exception
+    {
+        LocalInfo info = new LocalInfo();
+        fillLocalInfoSampleData(info);
+        np.saveLocal(info.duplicate());
+        LocalInfo loaded = np.loadLocal();
+        assertThat(loaded).isEqualTo(info);
+    }
+
+    @Test
+    public void testPeer() throws Exception
+    {
+        PeerInfo info = new PeerInfo();
+        fillPeerInfoSampleData(info);
+        np.savePeer(info.duplicate());
+        List<PeerInfo> loaded = np.loadPeers().collect(Collectors.toList());
+        assertThat(loaded).containsExactlyInAnyOrder(info);
+        np.deletePeer(info.getPeerAddressAndPort());
+        List<PeerInfo> loaded2 = np.loadPeers().collect(Collectors.toList());
+        assertThat(loaded2).isEmpty();
+    }
+
+    @Test
+    public void testUpdateTokens() throws Exception
+    {
+        LocalInfo info = new LocalInfo();
+        fillLocalInfoSampleData(info);
+        np.saveLocal(info);
+        LocalInfo loaded = np.loadLocal();
+        assertThat(loaded.getTokens()).hasSize(3);
+        ArrayList<Token> tokens = Lists.newArrayList(loaded.getTokens());
+        tokens.remove(0);
+        np.saveLocal(loaded.duplicate().setTokens(tokens));
+        loaded = np.loadLocal();
+        assertThat(loaded.getTokens()).hasSize(2);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/nodes/NodesTest.java b/test/unit/org/apache/cassandra/nodes/NodesTest.java
new file mode 100644
index 000000000000..cda936e60244
--- /dev/null
+++ b/test/unit/org/apache/cassandra/nodes/NodesTest.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.nodes;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.mockito.ArgumentCaptor;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.atMostOnce;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class NodesTest
+{
+    private final ExecutorService executor = mock(ExecutorService.class);
+    private final INodesPersistence persistence = mock(INodesPersistence.class);
+    private final Future<?> promise = mock(Future.class);
+
+    private final UUID newHostId = UUID.randomUUID();
+    private final UUID id1 = UUID.randomUUID();
+    private final UUID id2 = UUID.randomUUID();
+    private final UUID id3 = UUID.randomUUID();
+
+    private Nodes nodes;
+    private static InetAddressAndPort addr1;
+    private static InetAddressAndPort addr2;
+    private static InetAddressAndPort addr3;
+    private ArgumentCaptor<Runnable> taskCaptor;
+    private AtomicReference<NodeInfo<?>> infoRef;
+
+    @BeforeClass
+    public static void beforeClass() throws Exception
+    {
+        addr1 = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.1", 7001);
+        addr2 = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", 7001);
+        addr3 = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", 7002);
+    }
+
+    @Before
+    public void beforeTest()
+    {
+        reset(persistence, executor, promise);
+        nodes = new Nodes(persistence, executor);
+        infoRef = new AtomicReference<>();
+        taskCaptor = ArgumentCaptor.forClass(Runnable.class);
+        when(executor.submit(taskCaptor.capture())).thenAnswer(inv -> promise);
+    }
+
+    @After
+    public void afterTest()
+    {
+        verify(persistence, atMostOnce()).loadLocal();
+        verify(persistence, atMostOnce()).loadPeers();
+        verifyNoMoreInteractions(executor, persistence, promise);
+    }
+
+    @Test
+    public void getEmptyLocal()
+    {
+        assertThat(nodes.getLocal().get()).isEqualTo(new LocalInfo());
+    }
+
+    @Test
+    public void getEmptyPeer()
+    {
+        assertThat(nodes.getPeers().get(addr1)).isNull();
+        assertThat(nodes.getPeers().get(addr2)).isNull();
+        assertThat(nodes.getPeers().get(addr3)).isNull();
+    }
+
+    @Test
+    public void loadLocal()
+    {
+        when(persistence.loadLocal()).thenReturn(new LocalInfo().setHostId(newHostId));
+        clearInvocations(persistence);
+        nodes = new Nodes(persistence, executor);
+
+        ILocalInfo r = nodes.getLocal().get();
+        verify(persistence).loadLocal();
+        assertThat(r.getHostId()).isEqualTo(newHostId);
+    }
+
+    @Test
+    public void loadPeers()
+    {
+        List<PeerInfo> peers = Arrays.asList(new PeerInfo().setPeerAddressAndPort(addr1).setHostId(id1),
+                                             new PeerInfo().setPeerAddressAndPort(addr2).setHostId(id2),
+                                             new PeerInfo().setPeerAddressAndPort(addr3).setHostId(id3));
+        when(persistence.loadPeers()).thenReturn(peers.stream());
+        clearInvocations(persistence);
+        nodes = new Nodes(persistence, executor);
+
+        Set<IPeerInfo> r = nodes.getPeers().get().collect(Collectors.toSet());
+        assertThat(r).containsExactlyInAnyOrderElementsOf(peers);
+        verify(persistence).loadPeers();
+
+        clearInvocations(executor);
+    }
+
+    @Test
+    public void updateLocalNoChanges() throws Exception
+    {
+        nodes.getLocal().update(current -> current.setHostId(newHostId), false, false);
+        clearInvocations(persistence, executor, promise);
+
+        ILocalInfo r = updateLocalInfo(newHostId, false, false);
+
+        checkLiveObject(r, () -> nodes.getLocal().get(), newHostId);
+    }
+
+    @Test
+    public void updateLocalWithSomeChange() throws Exception
+    {
+        ILocalInfo r = updateLocalInfo(newHostId, false, false);
+
+        verify(executor).submit(any(Runnable.class));
+
+        taskCaptor.getValue().run();
+        verify(persistence).saveLocal(argThat(info -> info.getHostId().equals(newHostId)));
+
+        checkLiveObject(r, () -> nodes.getLocal().get(), newHostId);
+    }
+
+    @Test
+    public void updateLocalWithSomeChangeBlocking() throws Exception
+    {
+        ILocalInfo r = updateLocalInfo(newHostId, true, false);
+
+        verify(executor).submit(any(Runnable.class));
+        verify(promise).get();
+
+        taskCaptor.getValue().run();
+        verify(persistence).saveLocal(argThat(info -> info.getHostId().equals(newHostId)));
+        verify(persistence).syncLocal();
+
+        checkLiveObject(r, () -> nodes.getLocal().get(), newHostId);
+    }
+
+    @Test
+    public void updateLocalWithForce() throws Exception
+    {
+        nodes.getLocal().update(current -> current.setHostId(newHostId), false, false);
+        clearInvocations(persistence, executor, promise);
+
+        ILocalInfo r = updateLocalInfo(newHostId, false, true);
+
+        verify(executor).submit(any(Runnable.class));
+
+        taskCaptor.getValue().run();
+        verify(persistence).saveLocal(argThat(info -> info.getHostId().equals(newHostId)));
+
+        checkLiveObject(r, () -> nodes.getLocal().get(), newHostId);
+    }
+
+    @Test
+    public void updatePeersNoChanges() throws Exception
+    {
+        nodes.getPeers().update(addr1, current -> current.setHostId(newHostId), false, false);
+        clearInvocations(persistence, executor, promise);
+
+        IPeerInfo r = updatePeerInfo(newHostId, false, false);
+
+        checkLiveObject(r, () -> nodes.getPeers().get(addr1), newHostId);
+    }
+
+    @Test
+    public void updatePeersWithSomeChange() throws Exception
+    {
+        IPeerInfo r = updatePeerInfo(newHostId, false, false);
+
+        verify(executor).submit(any(Runnable.class));
+
+        taskCaptor.getValue().run();
+        verify(persistence).savePeer(argThat(info -> info.getHostId().equals(newHostId)));
+
+        checkLiveObject(r, () -> nodes.getPeers().get(addr1), newHostId);
+    }
+
+    @Test
+    public void updatePeersWithSomeChangeBlocking() throws Exception
+    {
+        IPeerInfo r = updatePeerInfo(newHostId, true, false);
+
+        verify(executor).submit(any(Runnable.class));
+        verify(promise).get();
+
+        taskCaptor.getValue().run();
+        verify(persistence).savePeer(argThat(info -> info.getHostId().equals(newHostId)));
+        verify(persistence).syncPeers();
+
+        checkLiveObject(r, () -> nodes.getPeers().get(addr1), newHostId);
+    }
+
+    @Test
+    public void updatePeersWithForce() throws Exception
+    {
+        nodes.getPeers().update(addr1, current -> current.setHostId(newHostId), false, false);
+        clearInvocations(persistence, executor, promise);
+
+        IPeerInfo r = updatePeerInfo(newHostId, false, true);
+
+        verify(executor).submit(any(Runnable.class));
+
+        taskCaptor.getValue().run();
+        verify(persistence).savePeer(argThat(info -> info.getHostId().equals(newHostId)));
+
+        checkLiveObject(r, () -> nodes.getPeers().get(addr1), newHostId);
+    }
+
+    @Test
+    public void getAllPeers()
+    {
+        IPeerInfo p1 = nodes.getPeers().update(addr1, current -> current.setHostId(id1));
+        IPeerInfo p2 = nodes.getPeers().update(addr2, current -> current.setHostId(id2));
+        IPeerInfo p3 = nodes.getPeers().update(addr3, current -> current.setHostId(id3));
+
+        Set<IPeerInfo> peers = nodes.getPeers().get().collect(Collectors.toSet());
+        assertThat(peers).containsExactlyInAnyOrder(p1, p2, p3);
+
+        clearInvocations(executor);
+    }
+
+    @Test
+    public void removePeer() throws Exception
+    {
+        IPeerInfo p1 = nodes.getPeers().update(addr1, current -> current.setHostId(id1));
+        IPeerInfo p2 = nodes.getPeers().update(addr2, current -> current.setHostId(id2));
+
+        clearInvocations(executor);
+
+        IPeerInfo r = nodes.getPeers().remove(addr2, false, true);
+        assertThat(r).isEqualTo(p2.duplicate().setRemoved(true));
+        verify(executor).submit(any(Runnable.class));
+
+        taskCaptor.getValue().run();
+        verify(persistence).deletePeer(addr2);
+
+        Set<IPeerInfo> peers = nodes.getPeers().get().collect(Collectors.toSet());
+        assertThat(peers).containsExactlyInAnyOrder(p1);
+    }
+
+    @Test
+    public void removePeerBlocking() throws Exception
+    {
+        IPeerInfo p1 = nodes.getPeers().update(addr1, current -> current.setHostId(id1));
+        IPeerInfo p2 = nodes.getPeers().update(addr2, current -> current.setHostId(id2));
+
+        clearInvocations(executor);
+
+        IPeerInfo r = nodes.getPeers().remove(addr2, true, true);
+        assertThat(r).isEqualTo(p2.duplicate().setRemoved(true));
+        verify(executor).submit(any(Runnable.class));
+        verify(promise).get();
+
+        taskCaptor.getValue().run();
+        verify(persistence).deletePeer(addr2);
+        verify(persistence).syncPeers();
+
+        Set<IPeerInfo> peers = nodes.getPeers().get().collect(Collectors.toSet());
+        assertThat(peers).containsExactlyInAnyOrder(p1);
+    }
+
+    @Test
+    public void removeMissingPeer()
+    {
+        UUID id1 = UUID.randomUUID();
+        UUID id2 = UUID.randomUUID();
+
+        IPeerInfo r1 = nodes.getPeers().update(addr1, current -> current.setHostId(id1));
+        IPeerInfo r2 = nodes.getPeers().update(addr2, current -> current.setHostId(id2));
+
+        clearInvocations(executor);
+
+        IPeerInfo r = nodes.getPeers().remove(addr3, false, true);
+        assertThat(r).isNull();
+
+        Set<IPeerInfo> peers = nodes.getPeers().get().collect(Collectors.toSet());
+        assertThat(peers).containsExactlyInAnyOrder(r1, r2);
+    }
+
+    @Test
+    public void softRemovePeer()
+    {
+        IPeerInfo p1 = nodes.getPeers().update(addr1, current -> current.setHostId(id1));
+        IPeerInfo p2 = nodes.getPeers().update(addr2, current -> current.setHostId(id2));
+
+        clearInvocations(executor);
+
+        IPeerInfo r = nodes.getPeers().remove(addr2, false, false);
+        p2 = p2.duplicate().setRemoved(true);
+        assertThat(r).isEqualTo(p2);
+
+        verify(executor).submit(any(Runnable.class));
+
+        taskCaptor.getValue().run();
+        verify(persistence).deletePeer(addr2);
+
+        Set<IPeerInfo> peers = nodes.getPeers().get().collect(Collectors.toSet());
+        assertThat(peers).containsExactlyInAnyOrder(p1, p2);
+    }
+
+    private void checkLiveObject(INodeInfo<?> r, Callable<INodeInfo<?>> currentSupplier, UUID hostId) throws Exception
+    {
+        assertThat(r.getHostId()).isEqualTo(hostId);
+        assertThat(currentSupplier.call().getHostId()).isEqualTo(hostId);
+    }
+
+    private ILocalInfo updateLocalInfo(UUID hostId, boolean blocking, boolean force)
+    {
+        return nodes.getLocal().update(previous -> {
+            infoRef.set(previous);
+            previous.setHostId(hostId);
+            return previous;
+        }, blocking, force);
+    }
+
+    private IPeerInfo updatePeerInfo(UUID hostId, boolean blocking, boolean force)
+    {
+        return nodes.getPeers().update(addr1, previous -> {
+            infoRef.set(previous);
+            previous.setHostId(hostId);
+            return previous;
+        }, blocking, force);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/repair/FuzzTestBase.java b/test/unit/org/apache/cassandra/repair/FuzzTestBase.java
index f4c4919d94ef..8cc2be7afce0 100644
--- a/test/unit/org/apache/cassandra/repair/FuzzTestBase.java
+++ b/test/unit/org/apache/cassandra/repair/FuzzTestBase.java
@@ -106,6 +106,8 @@
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.repair.messages.RepairMessage;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.repair.messages.ValidationResponse;
 import org.apache.cassandra.repair.state.Completable;
@@ -159,6 +161,7 @@
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_GLOBAL;
 import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
+import static org.apache.cassandra.net.Verb.*;
 
 public abstract class FuzzTestBase extends CQLTester.InMemory
 {
@@ -287,6 +290,7 @@ public ExecutorBuilder<? extends ExecutorPlus> configurePooled(String name, int
         // set the repair rcp timeout high so we don't hit it... this class is mostly testing repair reaching success
         // so don't want to deal with unlucky histories...
         DatabaseDescriptor.setRepairRpcTimeout(TimeUnit.DAYS.toMillis(1));
+        DatabaseDescriptor.setRepairPrepareMessageTimeout(TimeUnit.DAYS.toMillis(1));
 
 
         InMemory.setUpClass();
@@ -357,27 +361,26 @@ public Set<Faults> apply(Cluster.Node node, Message<?> message)
                     allowDrop.add(message.id());
                     return Faults.DROPPED;
                 }
-                switch (message.verb())
-                {
-                    // these messages are not resilent to ephemeral issues
-                    case STATUS_REQ:
-                    case STATUS_RSP:
+                Verb verb = message.verb();
+                // these messages are not resilent to ephemeral issues
+                if (verb.equals(STATUS_REQ)
+                    || verb.equals(STATUS_RSP)
                     // paxos repair does not support faults and will cause a TIMEOUT error, failing the repair
-                    case PAXOS2_CLEANUP_COMPLETE_REQ:
-                    case PAXOS2_CLEANUP_REQ:
-                    case PAXOS2_CLEANUP_RSP2:
-                    case PAXOS2_CLEANUP_START_PREPARE_REQ:
-                    case PAXOS2_CLEANUP_FINISH_PREPARE_REQ:
-                        noFaults.add(message.id());
-                        return Faults.NONE;
-                    default:
-                        if (noFaults.contains(message.id())) return Faults.NONE;
-                        if (allowDrop.contains(message.id())) return Faults.DROPPED;
-                        // was a new message added and the test not updated?
-                        IllegalStateException e = new IllegalStateException("Verb: " + message.verb());
-                        cluster.failures.add(e);
-                        throw e;
+                    || verb.equals(PAXOS2_CLEANUP_COMPLETE_REQ)
+                    || verb.equals(PAXOS2_CLEANUP_REQ)
+                    || verb.equals(PAXOS2_CLEANUP_RSP2)
+                    || verb.equals(PAXOS2_CLEANUP_START_PREPARE_REQ)
+                    || verb.equals(PAXOS2_CLEANUP_FINISH_PREPARE_REQ))
+                {
+                    noFaults.add(message.id());
+                    return Faults.NONE;
                 }
+                if (noFaults.contains(message.id())) return Faults.NONE;
+                if (allowDrop.contains(message.id())) return Faults.DROPPED;
+                // was a new message added and the test not updated?
+                IllegalStateException e = new IllegalStateException("Verb: " + message.verb());
+                cluster.failures.add(e);
+                throw e;
             }
         });
     }
@@ -748,7 +751,7 @@ static class Cluster
                 state.addApplicationState(ApplicationState.RACK, valueFactory.rack(rack));
                 state.addApplicationState(ApplicationState.RELEASE_VERSION, valueFactory.releaseVersion());
 
-                gossiper.endpoints.put(addressAndPort, state);
+                gossiper.addEndpointState(addressAndPort, state);
 
                 Node node = new Node(hostId, addressAndPort, Collections.singletonList(token), new Messaging(addressAndPort));
                 nodes.put(addressAndPort, node);
@@ -1020,6 +1023,12 @@ public <V> void respond(V response, Message<?> message)
         private class Gossip implements IGossiper
         {
             private final Map<InetAddressAndPort, EndpointState> endpoints = new HashMap<>();
+            private void addEndpointState(InetAddressAndPort endpoint, EndpointState state)
+            {
+                state.maybeSetUpdater(update -> Nodes.updateLocalOrPeer(endpoint, update, false, true));
+                state.maybeUpdate();
+                endpoints.put(endpoint, state);
+            }
 
             @Override
             public void register(IEndpointStateChangeSubscriber subscriber)
@@ -1108,25 +1117,30 @@ private Node(UUID hostId, InetAddressAndPort addressAndPort, Collection<Token> t
                     @Override
                     public void doVerb(Message message) throws IOException
                     {
-                        switch (message.verb())
+                        Verb verb = message.verb();
+                        if (verb.equals(PAXOS2_CLEANUP_START_PREPARE_REQ))
+                        {
+                            paxosStartPrepareCleanup.doVerb(message);
+                        }
+                        else if (verb.equals(PAXOS2_CLEANUP_REQ))
+                        {
+                            paxosCleanupRequestIVerbHandler.doVerb(message);
+                        }
+                        else if (verb.equals(PAXOS2_CLEANUP_FINISH_PREPARE_REQ))
+                        {
+                            paxosFinishPrepareCleanup.doVerb(message);
+                        }
+                        else if (verb.equals(PAXOS2_CLEANUP_RSP2))
+                        {
+                            paxosCleanupResponse.doVerb(message);
+                        }
+                        else if (verb.equals(PAXOS2_CLEANUP_COMPLETE_REQ))
+                        {
+                            paxosCleanupComplete.doVerb(message);
+                        }
+                        else
                         {
-                            case PAXOS2_CLEANUP_START_PREPARE_REQ:
-                                paxosStartPrepareCleanup.doVerb(message);
-                                break;
-                            case PAXOS2_CLEANUP_REQ:
-                                paxosCleanupRequestIVerbHandler.doVerb(message);
-                                break;
-                            case PAXOS2_CLEANUP_FINISH_PREPARE_REQ:
-                                paxosFinishPrepareCleanup.doVerb(message);
-                                break;
-                            case PAXOS2_CLEANUP_RSP2:
-                                paxosCleanupResponse.doVerb(message);
-                                break;
-                            case PAXOS2_CLEANUP_COMPLETE_REQ:
-                                paxosCleanupComplete.doVerb(message);
-                                break;
-                            default:
-                                repairVerbHandler.doVerb(message);
+                            repairVerbHandler.doVerb(message);
                         }
                     }
                 };
diff --git a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
index 95f630dc0571..0d7cfb259784 100644
--- a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
+++ b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
@@ -20,12 +20,19 @@
 
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 import com.google.common.collect.Iterables;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+
 import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
@@ -51,6 +58,7 @@
 import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -66,6 +74,8 @@ public class LocalSyncTaskTest extends AbstractRepairTest
     @BeforeClass
     public static void defineSchema()
     {
+        CassandraRelevantProperties.REPAIR_PARENT_SESSION_LISTENER.setString(MockParentRepairSessionListener.class.getName());
+
         SchemaLoader.prepareServer();
         SchemaLoader.createKeyspace(KEYSPACE1,
                                     KeyspaceParams.simple(1),
@@ -75,6 +85,12 @@ public static void defineSchema()
         cfs = Schema.instance.getColumnFamilyStoreInstance(tid);
     }
 
+    @AfterClass
+    public static void cleanup()
+    {
+        CassandraRelevantProperties.REPAIR_PARENT_SESSION_LISTENER.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+    }
+
     /**
      * When there is no difference between two, SymmetricLocalSyncTask should return stats with 0 difference.
      */
@@ -113,6 +129,7 @@ public void testDifference() throws Throwable
                                                                    Arrays.asList(cfs), Arrays.asList(range), false,
                                                                    ActiveRepairService.UNREPAIRED_SSTABLE, false,
                                                                    PreviewKind.NONE);
+        assertTrue(MockParentRepairSessionListener.registered.containsKey(parentRepairSession));
 
         RepairJobDesc desc = new RepairJobDesc(parentRepairSession, nextTimeUUID(), KEYSPACE1, "Standard1", Arrays.asList(range));
 
@@ -146,6 +163,9 @@ public void testDifference() throws Throwable
 
         // ensure that the changed range was recorded
         assertEquals("Wrong differing ranges", interesting.size(), task.stat.differences.size());
+
+        ActiveRepairService.instance().removeParentRepairSession(parentRepairSession);
+        assertTrue(MockParentRepairSessionListener.removed.containsKey(parentRepairSession));
     }
 
     @Test
@@ -242,6 +262,37 @@ private MerkleTrees createInitialTree(RepairJobDesc desc, IPartitioner partition
     private MerkleTrees createInitialTree(RepairJobDesc desc)
     {
         return createInitialTree(desc, partitioner);
+    }
+
+    public static class MockParentRepairSessionListener implements ParentRepairSessionListener
+    {
+        public static Map<TimeUUID, ActiveRepairService.ParentRepairSession> registered = new ConcurrentHashMap<>();
+        public static Map<TimeUUID, ActiveRepairService.ParentRepairSession> removed = new ConcurrentHashMap<>();
+
+        public MockParentRepairSessionListener() {}
 
+        @Override
+        public void onRegistered(TimeUUID sessionId, ActiveRepairService.ParentRepairSession session)
+        {
+            registered.put(sessionId, session);
+        }
+
+        @Override
+        public void onRemoved(TimeUUID sessionId, ActiveRepairService.ParentRepairSession session)
+        {
+            removed.put(sessionId, session);
+        }
+
+        @Override
+        public void onValidation(RepairJobDesc desc, Future validationTask)
+        {
+
+        }
+
+        @Override
+        public void onSync(RepairJobDesc desc, Future syncTask)
+        {
+
+        }
     }
 }
diff --git a/test/unit/org/apache/cassandra/repair/NeighborsAndRangesTest.java b/test/unit/org/apache/cassandra/repair/NeighborsAndRangesTest.java
index d5c578d73090..e10812ddef21 100644
--- a/test/unit/org/apache/cassandra/repair/NeighborsAndRangesTest.java
+++ b/test/unit/org/apache/cassandra/repair/NeighborsAndRangesTest.java
@@ -19,7 +19,9 @@
 package org.apache.cassandra.repair;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import com.google.common.collect.Lists;
@@ -27,12 +29,51 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.repair.RepairCoordinator.NeighborsAndRanges;
+import static org.apache.cassandra.repair.RepairCoordinator.createNeighbordAndRangesForOfflineService;
+import static org.apache.cassandra.repair.messages.RepairOption.parse;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 public class NeighborsAndRangesTest extends AbstractRepairTest
 {
+    @Test
+    public void testCreateNeighbordAndRangesForOfflineService()
+    {
+        Map<String, String> options = new HashMap<>();
+
+        // no hosts
+        assertThatThrownBy(() -> create(options)).hasMessageContaining("There should be at least 1 host");
+        options.put(RepairOption.HOSTS_KEY, FBUtilities.getJustBroadcastAddress().getHostAddress());
+
+        // no ranges
+        assertThatThrownBy(() -> create(options)).hasMessageContaining("Token ranges must be specified");
+        options.put(RepairOption.RANGES_KEY, "0:10,11:20,21:30");
+
+        // no neighor, because the only host is local
+        assertThatThrownBy(() -> create(options)).hasMessageContaining("There should be at least 1 neighbor");
+
+        // with proper neighbors
+        options.put(RepairOption.HOSTS_KEY, "127.0.99.1,127.0.99.2," + FBUtilities.getJustBroadcastAddress().getHostAddress());
+        NeighborsAndRanges neighborsAndRanges = create(options);
+
+        assertThat(neighborsAndRanges.shouldExcludeDeadParticipants).isFalse();
+        assertThat(neighborsAndRanges.participants).hasSize(2); // excluded local host
+        assertThat(neighborsAndRanges.commonRanges).hasSize(1); // all in one common range
+        assertThat(neighborsAndRanges.filterCommonRanges("ks", new String[] { "cf"})).isSameAs(neighborsAndRanges.commonRanges);
+    }
+
+    private static NeighborsAndRanges create(Map<String, String> options)
+    {
+        RepairOption option = parse(options, Murmur3Partitioner.instance);
+        return createNeighbordAndRangesForOfflineService(option);
+    }
+
     /**
      * For non-forced repairs, common ranges should be passed through as-is
      */
diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
index 872ee99abbef..7e148349b447 100644
--- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java
+++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
@@ -82,6 +82,9 @@
 import static org.apache.cassandra.repair.RepairParallelism.SEQUENTIAL;
 import static org.apache.cassandra.streaming.PreviewKind.NONE;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.apache.cassandra.net.Verb.SNAPSHOT_MSG;
+import static org.apache.cassandra.net.Verb.SYNC_REQ;
+import static org.apache.cassandra.net.Verb.VALIDATION_REQ;
 import static org.apache.cassandra.utils.asserts.SyncTaskAssert.assertThat;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
@@ -128,7 +131,7 @@ public MeasureableRepairSession(TimeUUID parentRepairSession, CommonRange common
                                         String... cfnames)
         {
             super(SharedContext.Global.instance, new Scheduler.NoopScheduler(),
-                  parentRepairSession, commonRange, keyspace, parallelismDegree, isIncremental, pullRepair,
+                  parentRepairSession, commonRange, keyspace, parallelismDegree, isIncremental, false, pullRepair,
                   previewKind, optimiseStreams, repairPaxos, paxosOnly, cfnames);
         }
 
@@ -235,9 +238,9 @@ public void testEndToEndNoDifferences() throws InterruptedException, ExecutionEx
         // RepairJob should send out SNAPSHOTS -> VALIDATIONS -> done
         List<Verb> expectedTypes = new ArrayList<>();
         for (int i = 0; i < 3; i++)
-            expectedTypes.add(Verb.SNAPSHOT_MSG);
+            expectedTypes.add(SNAPSHOT_MSG);
         for (int i = 0; i < 3; i++)
-            expectedTypes.add(Verb.VALIDATION_REQ);
+            expectedTypes.add(VALIDATION_REQ);
 
         assertThat(observedMessages).extracting(Message::verb).containsExactlyElementsOf(expectedTypes);
     }
@@ -268,6 +271,7 @@ public void testNoTreesRetainedAfterDifference() throws Throwable
                                                                      addr4, // local
                                                                      noTransient(),
                                                                      session.isIncremental,
+                                                                     session.pushRepair,
                                                                      session.pullRepair,
                                                                      session.previewKind);
 
@@ -313,7 +317,7 @@ public void testNoTreesRetainedAfterDifference() throws Throwable
         assertThat(messages)
             .hasSize(2)
             .extracting(Message::verb)
-            .containsOnly(Verb.SYNC_REQ);
+            .containsOnly(SYNC_REQ);
     }
 
     @Test
@@ -369,6 +373,7 @@ public static void testCreateStandardSyncTasks(boolean pullRepair)
                                                                                     addr1, // local
                                                                                     noTransient(), // transient
                                                                                     false,
+                                                                                    false,
                                                                                     pullRepair,
                                                                                     PreviewKind.ALL));
         assertThat(tasks).hasSize(2);
@@ -387,6 +392,40 @@ public static void testCreateStandardSyncTasks(boolean pullRepair)
         assertThat(tasks.get(pair(addr1, addr3))).isNull();
     }
 
+    @Test
+    public void testCreateStandardSyncTasksWithPushRepair()
+    {
+        List<TreeResponse> treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "1", RANGE_2, "1", RANGE_3, "1"),
+                                                         treeResponse(addr2, RANGE_1, "1", RANGE_2, "2", RANGE_3, "3"),
+                                                         treeResponse(addr3, RANGE_1, "2", RANGE_2, "3", RANGE_3, "1"));
+
+        Map<SyncNodePair, SyncTask> tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC,
+                                                                                    treeResponses,
+                                                                                    addr1, // local
+                                                                                    noTransient(), // transient
+                                                                                    false,
+                                                                                    true,
+                                                                                    false,
+                                                                                    PreviewKind.ALL));
+        assertThat(tasks).hasSize(2);
+
+        // between local and addr2: range2 and rang3 are different
+        assertThat(tasks.get(pair(addr1, addr2)))
+        .isInstanceOf(LocalSyncTask.class)
+        .isLocal()
+        .isNotRequestRanges()
+        .hasTransferRanges(true)
+        .hasRanges(RANGE_2, RANGE_3);
+
+        // between local and addr3: range1 and rang2 are different
+        assertThat(tasks.get(pair(addr1, addr3)))
+        .isInstanceOf(LocalSyncTask.class)
+        .isLocal()
+        .isNotRequestRanges()
+        .hasTransferRanges(true)
+        .hasRanges(RANGE_1, RANGE_2);
+    }
+
     @Test
     public void testStandardSyncTransient()
     {
@@ -405,6 +444,7 @@ public void testStandardSyncTransient(boolean pullRepair)
                                                                                     addr1, // local
                                                                                     transientPredicate(addr2),
                                                                                     false,
+                                                                                    false,
                                                                                     pullRepair,
                                                                                     PreviewKind.ALL));
 
@@ -435,6 +475,7 @@ public void testStandardSyncLocalTransient(boolean pullRepair)
                                                                                     addr1, // local
                                                                                     transientPredicate(addr1),
                                                                                     false,
+                                                                                    false,
                                                                                     pullRepair,
                                                                                     PreviewKind.ALL));
 
@@ -495,6 +536,7 @@ public void testEmptyDifference(InetAddressAndPort local, Predicate<InetAddressA
                                                                                     local, // local
                                                                                     isTransient,
                                                                                     false,
+                                                                                    false,
                                                                                     pullRepair,
                                                                                     PreviewKind.ALL));
 
@@ -513,6 +555,7 @@ public void testCreateStandardSyncTasksAllDifferent()
                                                                                     addr1, // local
                                                                                     ep -> ep.equals(addr3), // transient
                                                                                     false,
+                                                                                    false,
                                                                                     true,
                                                                                     PreviewKind.ALL));
 
@@ -544,6 +587,7 @@ public void testCreate5NodeStandardSyncTasksWithTransient()
                                                                                     addr1, // local
                                                                                     isTransient, // transient
                                                                                     false,
+                                                                                    false,
                                                                                     true,
                                                                                     PreviewKind.ALL));
 
@@ -611,6 +655,7 @@ public static void testLocalSyncWithTransient(InetAddressAndPort local, boolean
                                                                                     local, // local
                                                                                     isTransient, // transient
                                                                                     false,
+                                                                                    false,
                                                                                     pullRepair,
                                                                                     PreviewKind.ALL));
 
@@ -660,6 +705,7 @@ private static void testLocalAndRemoteTransient(boolean pullRepair)
                                                                                     addr4, // local
                                                                                     ep -> ep.equals(addr4) || ep.equals(addr5), // transient
                                                                                     false,
+                                                                                    false,
                                                                                     pullRepair,
                                                                                     PreviewKind.ALL));
 
@@ -888,21 +934,19 @@ private void interceptRepairMessages(Map<InetAddressAndPort, MerkleTrees> mockTr
                     messageCapture.add(message);
                 }
 
-                switch (message.verb())
+                if (message.verb() == SNAPSHOT_MSG)
+                {
+                    MessagingService.instance().callbacks.removeAndRespond(message.id(), to, message.emptyResponse());
+                }
+                else if (message.verb() == VALIDATION_REQ)
+                {
+                    MerkleTrees tree = mockTrees.get(to);
+                    session.validationComplete(sessionJobDesc, Message.builder(Verb.VALIDATION_RSP, tree != null ? new ValidationResponse(sessionJobDesc, tree) : new ValidationResponse(sessionJobDesc)).from(to).build());
+                }
+                else if (message.verb() == SYNC_REQ)
                 {
-                    case SNAPSHOT_MSG:
-                        MessagingService.instance().callbacks.removeAndRespond(message.id(), to, message.emptyResponse());
-                        break;
-                    case VALIDATION_REQ:
-                        MerkleTrees tree = mockTrees.get(to);
-                        session.validationComplete(sessionJobDesc, Message.builder(Verb.VALIDATION_RSP, tree != null ? new ValidationResponse(sessionJobDesc, tree) : new ValidationResponse(sessionJobDesc)).from(to).build());
-                        break;
-                    case SYNC_REQ:
-                        SyncRequest syncRequest = (SyncRequest) message.payload;
-                        session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList())).from(to).build());
-                        break;
-                    default:
-                        break;
+                    SyncRequest syncRequest = (SyncRequest) message.payload;
+                    session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList())).from(to).build());
                 }
                 return false;
         });
diff --git a/test/unit/org/apache/cassandra/repair/RepairProgressReporterTest.java b/test/unit/org/apache/cassandra/repair/RepairProgressReporterTest.java
new file mode 100644
index 000000000000..1aa689e66edb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/repair/RepairProgressReporterTest.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaTransformations;
+import org.apache.cassandra.schema.SystemDistributedKeyspace;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME;
+import static org.apache.cassandra.schema.SystemDistributedKeyspace.PARENT_REPAIR_HISTORY;
+import static org.apache.cassandra.schema.SystemDistributedKeyspace.REPAIR_HISTORY;
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+
+public class RepairProgressReporterTest extends CQLTester
+{
+    protected static final Range<Token> RANGE1 = new Range<>(t(1), t(2));
+
+    public static final InetAddressAndPort COORDINATOR;
+    protected static final InetAddressAndPort PARTICIPANT1;
+
+    static
+    {
+        try
+        {
+            COORDINATOR = InetAddressAndPort.getByName("10.0.0.1");
+            PARTICIPANT1 = InetAddressAndPort.getByName("10.0.0.1");
+        }
+        catch (UnknownHostException e)
+        {
+
+            throw new AssertionError(e);
+        }
+    }
+
+    private static final TimeUUID sessionId = nextTimeUUID();
+    private static final TimeUUID parentRepairSession = nextTimeUUID();
+    private static final String keyspace = "ks";
+    private static final String[] cfs = { "table" };
+    private static final CommonRange commonRange = new CommonRange(Sets.newHashSet(COORDINATOR, PARTICIPANT1), Collections.emptySet(), Collections.singleton(RANGE1));
+
+    @BeforeClass
+    public static void init()
+    {
+        // required to access distributed keyspace
+        Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(SystemDistributedKeyspace.metadata(), SystemDistributedKeyspace.GENERATION));
+        StorageService.instance.getTokenMetadata().updateNormalToken(t(1), FBUtilities.getBroadcastAddressAndPort());
+    }
+
+    @After
+    public void after()
+    {
+        Keyspace.open(DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(REPAIR_HISTORY).truncateBlockingWithoutSnapshot();
+        Keyspace.open(DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(PARENT_REPAIR_HISTORY).truncateBlockingWithoutSnapshot();
+    }
+
+    @Test
+    public void testStartParentRepairs()
+    {
+        RepairOption option = RepairOption.parse(Collections.emptyMap(), DatabaseDescriptor.getPartitioner());
+        option.getRanges().add(RANGE1);
+
+        RepairProgressReporter.instance.onParentRepairStarted(parentRepairSession, keyspace, cfs, option);
+        assertRows(parentRepairHistory(), row(keyspace, Sets.newHashSet(cfs), parentRepairSession,
+                                              Stream.of(RANGE1).map(Range::toString).collect(Collectors.toSet()),
+                                              null));
+    }
+
+    @Test
+    public void testSuccessfulParentRepairs()
+    {
+        testStartParentRepairs();
+
+        RepairProgressReporter.instance.onParentRepairSucceeded(parentRepairSession, Collections.singleton(RANGE1));
+        assertRows(parentRepairHistory(), row(keyspace, Sets.newHashSet(cfs), parentRepairSession,
+                                              Stream.of(RANGE1).map(Range::toString).collect(Collectors.toSet()),
+                                              Stream.of(RANGE1).map(Range::toString).collect(Collectors.toSet())));
+    }
+
+    @Test
+    public void testFailParentRepairs()
+    {
+        testStartParentRepairs();
+
+        RepairProgressReporter.instance.onParentRepairFailed(parentRepairSession, new RuntimeException("Mock Error"));
+        assertRows(parentRepairHistoryWithError(), row(keyspace, Sets.newHashSet(cfs), parentRepairSession,
+                                                       Stream.of(RANGE1).map(Range::toString).collect(Collectors.toSet()),
+                                                       null,
+                                                       "Mock Error"));
+    }
+
+    @Test
+    public void testStartRepairs()
+    {
+        RepairProgressReporter.instance.onRepairsStarted(sessionId, parentRepairSession, keyspace, cfs, commonRange);
+        assertRows(repairHistory(), row(keyspace, cfs[0], sessionId, parentRepairSession, "STARTED", RANGE1.left.toString(), RANGE1.right.toString()));
+    }
+
+    @Test
+    public void testSuccessfulRepair()
+    {
+        testStartRepairs();
+
+        RepairProgressReporter.instance.onRepairSucceeded(sessionId, keyspace, cfs[0]);
+        assertRows(repairHistory(), row(keyspace, cfs[0], sessionId, parentRepairSession, "SUCCESS", RANGE1.left.toString(), RANGE1.right.toString()));
+    }
+
+    @Test
+    public void testFailRepairs()
+    {
+        testStartRepairs();
+
+        RepairProgressReporter.instance.onRepairsFailed(sessionId, keyspace, cfs, new RuntimeException("Mock Error"));
+        assertRows(repairHistoryWithError(), row(keyspace, cfs[0], sessionId, parentRepairSession, "FAILED", RANGE1.left.toString(), RANGE1.right.toString(), "Mock Error"));
+    }
+
+    @Test
+    public void testFailedRepairJob()
+    {
+        testStartRepairs();
+
+        RepairProgressReporter.instance.onRepairFailed(sessionId, keyspace, cfs[0], new RuntimeException("Mock Error"));
+        assertRows(repairHistoryWithError(), row(keyspace, cfs[0], sessionId, parentRepairSession, "FAILED", RANGE1.left.toString(), RANGE1.right.toString(), "Mock Error"));
+    }
+
+    private UntypedResultSet repairHistory()
+    {
+        return repairHistory("keyspace_name, columnfamily_name, id, parent_id, status, range_begin, range_end");
+    }
+
+    private UntypedResultSet repairHistoryWithError()
+    {
+        return repairHistory("keyspace_name, columnfamily_name, id, parent_id, status, range_begin, range_end, exception_message");
+    }
+
+    private UntypedResultSet repairHistory(String columns)
+    {
+        try
+        {
+            return execute(String.format("SELECT %s FROM %s.%s", columns, DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY));
+        }
+        catch (Throwable t)
+        {
+            throw new RuntimeException(t);
+        }
+    }
+
+    private UntypedResultSet parentRepairHistory()
+    {
+        return parentRepairHistory("keyspace_name, columnfamily_names, parent_id, requested_ranges, successful_ranges");
+    }
+
+    private UntypedResultSet parentRepairHistoryWithError()
+    {
+        return parentRepairHistory("keyspace_name, columnfamily_names, parent_id, requested_ranges, successful_ranges, exception_message");
+    }
+
+    private UntypedResultSet parentRepairHistory(String columns)
+    {
+        try
+        {
+            return execute(String.format("SELECT %s FROM %s.%s", columns, DISTRIBUTED_KEYSPACE_NAME, PARENT_REPAIR_HISTORY));
+        }
+        catch (Throwable t)
+        {
+            throw new RuntimeException(t);
+        }
+    }
+
+    protected static Token t(int v)
+    {
+        return DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(v));
+    }
+
+    private static class MockReporter implements RepairProgressReporter
+    {
+        public MockReporter()
+        {
+        }
+
+        @Override
+        public void onParentRepairStarted(TimeUUID parentSession, String keyspaceName, String[] cfnames, RepairOption options)
+        {
+
+        }
+
+        @Override
+        public void onParentRepairSucceeded(TimeUUID parentSession, Collection<Range<Token>> successfulRanges)
+        {
+
+        }
+
+        @Override
+        public void onParentRepairFailed(TimeUUID parentSession, Throwable t)
+        {
+
+        }
+
+        @Override
+        public void onRepairsStarted(TimeUUID id, TimeUUID parentRepairSession, String keyspaceName, String[] cfnames, CommonRange commonRange)
+        {
+
+        }
+
+        @Override
+        public void onRepairsFailed(TimeUUID id, String keyspaceName, String[] cfnames, Throwable t)
+        {
+
+        }
+
+        @Override
+        public void onRepairFailed(TimeUUID id, String keyspaceName, String cfname, Throwable t)
+        {
+
+        }
+
+        @Override
+        public void onRepairSucceeded(TimeUUID id, String keyspaceName, String cfname)
+        {
+
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
index f4d177870e33..3d76188d90f6 100644
--- a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
+++ b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
@@ -21,11 +21,17 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.collect.Sets;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.utils.WithResources;
+import org.apache.cassandra.utils.concurrent.Future;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -40,6 +46,7 @@
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.TimeUUID;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
@@ -49,6 +56,7 @@ public class RepairSessionTest
     @BeforeClass
     public static void initDD()
     {
+        NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE.setBoolean(true);
         DatabaseDescriptor.daemonInitialization();
     }
 
@@ -66,13 +74,18 @@ public void testConviction() throws Exception
         RepairSession session = new RepairSession(SharedContext.Global.instance, new Scheduler.NoopScheduler(), parentSessionId,
                                                   new CommonRange(endpoints, Collections.emptySet(), Arrays.asList(repairRange)),
                                                   "Keyspace1", RepairParallelism.SEQUENTIAL,
-                                                  false, false,
+                                                  false, false, false,
                                                   PreviewKind.NONE, false, false, false, "Standard1");
 
         // perform convict
         session.convict(remote, Double.MAX_VALUE);
 
         // RepairSession should throw ExecutorException with the cause of IOException when getting its value
+        assertSessionFails(session);
+    }
+
+    private void assertSessionFails(RepairSession session) throws InterruptedException
+    {
         try
         {
             session.get();
@@ -83,4 +96,156 @@ public void testConviction() throws Exception
             assertEquals(IOException.class, ex.getCause().getClass());
         }
     }
+
+    private static class NoopExecutorService implements ExecutorPlus
+    {
+        @Override
+        public void shutdown()
+        {
+        }
+
+        @Override
+        public List<Runnable> shutdownNow()
+        {
+            return null;
+        }
+
+        @Override
+        public boolean isShutdown()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean isTerminated()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public <T> Future<T> submit(Callable<T> task)
+        {
+            return null;
+        }
+
+        @Override
+        public <T> Future<T> submit(Runnable task, T result)
+        {
+            return null;
+        }
+
+        @Override
+        public Future<?> submit(Runnable task)
+        {
+            return null;
+        }
+
+        @Override
+        public void execute(WithResources withResources, Runnable task)
+        {
+        }
+
+        @Override
+        public <T> Future<T> submit(WithResources withResources, Callable<T> task)
+        {
+            return null;
+        }
+
+        @Override
+        public Future<?> submit(WithResources withResources, Runnable task)
+        {
+            return null;
+        }
+
+        @Override
+        public <T> Future<T> submit(WithResources withResources, Runnable task, T result)
+        {
+            return null;
+        }
+
+        @Override
+        public boolean inExecutor()
+        {
+            return false;
+        }
+
+        @Override
+        public void execute(Runnable command)
+        {
+        }
+
+        @Override
+        public int getCorePoolSize()
+        {
+            return 0;
+        }
+
+        @Override
+        public void setCorePoolSize(int newCorePoolSize)
+        {
+        }
+
+        @Override
+        public int getMaximumPoolSize()
+        {
+            return 0;
+        }
+
+        @Override
+        public void setMaximumPoolSize(int newMaximumPoolSize)
+        {
+        }
+
+        @Override
+        public int getActiveTaskCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public long getCompletedTaskCount()
+        {
+            return 0;
+        }
+
+        @Override
+        public int getPendingTaskCount()
+        {
+            return 0;
+        }
+    }
+
+    @Test
+    public void testRepairingDeadNodeFails() throws Exception
+    {
+        InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.2");
+        Gossiper.instance.initializeNodeUnsafe(remote, UUID.randomUUID(), 1);
+        // Mark remote as dead
+        Gossiper.instance.convict(remote, Double.MAX_VALUE);
+
+        // Set up RepairSession
+        TimeUUID parentSessionId = TimeUUID.Generator.nextTimeUUID();
+        IPartitioner p = Murmur3Partitioner.instance;
+        Range<Token> repairRange = new Range<>(p.getToken(ByteBufferUtil.bytes(0)), p.getToken(ByteBufferUtil.bytes(100)));
+        Set<InetAddressAndPort> endpoints = Sets.newHashSet(remote);
+        SharedContext.Global ctx = SharedContext.Global.instance;
+        RepairSession session = new RepairSession(ctx, new Scheduler.NoopScheduler(), parentSessionId,
+                                                  new CommonRange(endpoints, Collections.emptySet(), Arrays.asList(repairRange)),
+                                                  "Keyspace1", RepairParallelism.SEQUENTIAL,
+                                                  false, false, false,
+                                                  PreviewKind.NONE, false,
+                                                  false, false, "Standard1");
+
+        NoopExecutorService executor = new NoopExecutorService();
+        session.start(executor);
+
+        // RepairSession should fail when trying to repair a dead node
+        assertSessionFails(session);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/repair/ValidatorTest.java b/test/unit/org/apache/cassandra/repair/ValidatorTest.java
index 7d0317f2c1bd..2b086d1521ac 100644
--- a/test/unit/org/apache/cassandra/repair/ValidatorTest.java
+++ b/test/unit/org/apache/cassandra/repair/ValidatorTest.java
@@ -24,11 +24,17 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.ActiveOperations;
+import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.CompactionsTest;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.TableOperation;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.junit.After;
 import org.junit.Before;
@@ -192,9 +198,6 @@ public void simpleValidationTest(int n) throws Exception
         Util.flush(cfs);
         assertEquals(1, cfs.getLiveSSTables().size());
 
-        // wait enough to force single compaction
-        TimeUnit.SECONDS.sleep(5);
-
         SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
         TimeUUID repairSessionId = nextTimeUUID();
         final RepairJobDesc desc = new RepairJobDesc(repairSessionId, nextTimeUUID(), cfs.getKeyspaceName(),
@@ -207,9 +210,20 @@ public void simpleValidationTest(int n) throws Exception
                                                                    Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE,
                                                                    false, PreviewKind.NONE);
 
+        AtomicReference<TableOperation.Progress> progressOnCompletion = new AtomicReference<>();
+        CompactionManager.instance.active.registerListener(new ActiveOperations.CompactionProgressListener()
+        {
+            @Override
+            public void onCompleted(TableOperation.Progress progressOnCompleted)
+            {
+                if (progressOnCompleted.metadata() == cfs.metadata())
+                    progressOnCompletion.set(progressOnCompleted);
+            }
+        });
+
         final CompletableFuture<Message> outgoingMessageSink = registerOutgoingMessageSink();
         Validator validator = new Validator(SharedContext.Global.instance, new ValidationState(Clock.Global.clock(), desc, host), 0, true, false, PreviewKind.NONE);
-        ValidationManager.instance.submitValidation(cfs, validator);
+        Future<?> validationFuture = ValidationManager.instance.submitValidation(cfs, validator);
 
         Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS);
         assertEquals(Verb.VALIDATION_RSP, message.verb());
@@ -223,6 +237,13 @@ public void simpleValidationTest(int n) throws Exception
             assertEquals(Math.pow(2, Math.ceil(Math.log(n) / Math.log(2))), iterator.next().getValue().size(), 0.0);
         }
         assertEquals(m.trees.rowCount(), n);
+
+        // block on validation future to ensure the compaction progress listener has been called
+        validationFuture.get(TEST_TIMEOUT, TimeUnit.SECONDS);
+        assertNotNull(progressOnCompletion.get());
+        assertEquals(OperationType.VALIDATION, progressOnCompletion.get().operationType());
+        assertTrue(progressOnCompletion.get().completed() > 0);
+        assertEquals(progressOnCompletion.get().total(), progressOnCompletion.get().completed());
     }
 
     /*
@@ -249,9 +270,6 @@ public void testSizeLimiting() throws Exception
         Util.flush(cfs);
         assertEquals(1, cfs.getLiveSSTables().size());
 
-        // wait enough to force single compaction
-        TimeUnit.SECONDS.sleep(5);
-
         SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
         TimeUUID repairSessionId = nextTimeUUID();
         final RepairJobDesc desc = new RepairJobDesc(repairSessionId, nextTimeUUID(), cfs.getKeyspaceName(),
@@ -308,9 +326,6 @@ public void testRangeSplittingTreeSizeLimit() throws Exception
         Util.flush(cfs);
         assertEquals(1, cfs.getLiveSSTables().size());
 
-        // wait enough to force single compaction
-        TimeUnit.SECONDS.sleep(5);
-
         SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
         TimeUUID repairSessionId = nextTimeUUID();
 
diff --git a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java
index 3ee5716873fb..ccdcd682b0e3 100644
--- a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java
+++ b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java
@@ -699,8 +699,11 @@ public void handleStatusResponseNoop() throws Exception
         sessions.start();
         LocalSession session = sessions.prepareForTest(sessionID);
         session.setState(REPAIRING);
+        long lastUpdatedOriginal = session.getLastUpdate();
+        Thread.sleep(1100);
 
         sessions.handleStatusResponse(PARTICIPANT1, new StatusResponse(sessionID, FINALIZE_PROMISED));
+        Assert.assertNotEquals(lastUpdatedOriginal, session.getLastUpdate());
         Assert.assertEquals(REPAIRING, session.getState());
     }
 
diff --git a/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java b/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java
index 814b9c318d7b..66029c2b0e3f 100644
--- a/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java
+++ b/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java
@@ -124,7 +124,7 @@ private static void mutateRepaired(SSTableReader sstable, long repairedAt, TimeU
     {
         try
         {
-            cfs.getCompactionStrategyManager().mutateRepaired(Collections.singleton(sstable), repairedAt, pendingRepair, false);
+            cfs.mutateRepaired(Collections.singleton(sstable), repairedAt, pendingRepair, false);
         }
         catch (IOException e)
         {
diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageTest.java
index e1a6eec3f175..18a0d8077c22 100644
--- a/test/unit/org/apache/cassandra/repair/messages/RepairMessageTest.java
+++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageTest.java
@@ -18,12 +18,15 @@
 
 package org.apache.cassandra.repair.messages;
 
+import java.util.UUID;
+
 import org.junit.Before;
 import org.junit.Test;
 
 import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.gms.IGossiper;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.RepairMetrics;
@@ -47,7 +50,7 @@
 public class RepairMessageTest
 {
     private static final TimeUUID SESSION = new TimeUUID(0, 0);
-    private static final InetAddressAndPort ADDRESS = FBUtilities.getBroadcastAddressAndPort();
+    private static final InetAddressAndPort ADDRESS;
     private static final Answer REJECT_ALL = ignore -> {
         throw new UnsupportedOperationException();
     };
@@ -61,6 +64,9 @@ public class RepairMessageTest
     static
     {
         DatabaseDescriptor.clientInitialization();
+        ADDRESS = FBUtilities.getBroadcastAddressAndPort();
+        // this will initialize Nodes.local() with RELEASE_VERSION
+        Gossiper.instance.initializeNodeUnsafe(ADDRESS, UUID.randomUUID(), 1);
         RepairMetrics.init();
     }
 
@@ -208,4 +214,4 @@ private void test(int[] attempts, TestCase fn)
             Mockito.verifyNoInteractions(messaging);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
index 0483fcf15c41..1f768b8286ec 100644
--- a/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
+++ b/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
@@ -33,6 +33,7 @@
 import org.apache.cassandra.repair.RepairParallelism;
 import org.apache.cassandra.streaming.PreviewKind;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
@@ -63,11 +64,17 @@ public void testParseOptions()
         options.put(RepairOption.RANGES_KEY, "0:10,11:20,21:30");
         options.put(RepairOption.COLUMNFAMILIES_KEY, "cf1,cf2,cf3");
         options.put(RepairOption.DATACENTERS_KEY, "dc1,dc2,dc3");
+        options.put(RepairOption.PUSH_REPAIR_KEY, Boolean.toString(true));
+        options.put(RepairOption.OFFLINE_SERVICE, Boolean.toString(true));
 
         option = RepairOption.parse(options, partitioner);
         assertTrue(option.getParallelism() == RepairParallelism.PARALLEL);
         assertFalse(option.isPrimaryRange());
         assertFalse(option.isIncremental());
+        assertTrue(option.isPushRepair());
+        assertTrue(option.isSubrangeRepair());
+        assertTrue(option.isOfflineService());
+        assertEquals(Boolean.toString(true), option.asMap().get(RepairOption.OFFLINE_SERVICE));
 
         Set<Range<Token>> expectedRanges = new HashSet<>(3);
         expectedRanges.add(new Range<>(tokenFactory.fromString("0"), tokenFactory.fromString("10")));
@@ -93,7 +100,9 @@ public void testParseOptions()
 
         // remove data centers to proceed with testing parsing hosts
         options.remove(RepairOption.DATACENTERS_KEY);
+        options.remove(RepairOption.PUSH_REPAIR_KEY);
         option = RepairOption.parse(options, partitioner);
+        assertFalse(option.isPushRepair());
 
         Set<String> expectedHosts = new HashSet<>(3);
         expectedHosts.add("127.0.0.1");
@@ -141,6 +150,20 @@ public void testPullRepairParseOptions()
         assertTrue(option.isPullRepair());
     }
 
+    @Test
+    public void testPullRepairAndPushRepair()
+    {
+        Map<String, String> options = new HashMap<>();
+
+        options.put(RepairOption.PULL_REPAIR_KEY, "true");
+        options.put(RepairOption.PUSH_REPAIR_KEY, "true");
+        options.put(RepairOption.HOSTS_KEY, "127.0.0.1,127.0.0.2");
+        options.put(RepairOption.RANGES_KEY, "0:10");
+
+        assertThatThrownBy(() -> RepairOption.parse(options, Murmur3Partitioner.instance))
+               .hasMessageContaining("Cannot use pushRepair and pullRepair as the same time");
+    }
+
     @Test
     public void testForceOption() throws Exception
     {
diff --git a/test/unit/org/apache/cassandra/schema/CreateTableValidationTest.java b/test/unit/org/apache/cassandra/schema/CreateTableValidationTest.java
index 4ccb706461bc..2a0c3f4b3b56 100644
--- a/test/unit/org/apache/cassandra/schema/CreateTableValidationTest.java
+++ b/test/unit/org/apache/cassandra/schema/CreateTableValidationTest.java
@@ -18,12 +18,16 @@
  */
 package org.apache.cassandra.schema;
 
+import com.datastax.driver.core.exceptions.InvalidQueryException;
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.functions.types.ParseUtils;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 
 import org.junit.Test;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.junit.Assert.fail;
 
@@ -100,6 +104,46 @@ public void testCreateTableWithMissingClusteringColumn()
                         "Missing CLUSTERING ORDER for column ck1");
     }
 
+    @Test
+    public void failCreatingNewTableWithLongName()
+    {
+        String table = "test_create_k8yq1r75bpzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
+        assertThatExceptionOfType(InvalidQueryException.class)
+        .isThrownBy(() -> executeNet(String.format("CREATE TABLE \"%s\".%s (" +
+                                                   "key int PRIMARY KEY," +
+                                                   "val int)",
+                                                   KEYSPACE, table)))
+        .withMessageContaining(String.format("Table name is too long, it needs to fit %s characters (got table name of %s chars for %s.%s)",
+                                             SchemaConstants.NAME_LENGTH - KEYSPACE.length(), table.length(), KEYSPACE, table));
+    }
+
+    @Test
+    public void testCreatingInternalTableWithLongName() throws Throwable
+    {
+        String keyspace = "\"38373639353166362d356631322d343864652d393063362d653862616534343165333764_tpch\"";
+        String table = "test_create_k8yq1r75bpzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
+
+        execute(String.format("CREATE KEYSPACE %s with replication = " +
+                              "{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }",
+                              keyspace));
+        createTableMayThrow(String.format("CREATE TABLE %s.%s (" +
+                                           "key int PRIMARY KEY," +
+                                           "val int)", keyspace, table));
+
+        execute(String.format("INSERT INTO %s.%s (key,val) VALUES (1,1)", keyspace, table));
+        flush(ParseUtils.unDoubleQuote(keyspace), table);
+        UntypedResultSet result = execute(String.format("SELECT * from %s.%s", keyspace, table));
+        assertThat(result.size()).isEqualTo(1);
+    }
+
+    @Test
+    public void testNonAlphanummericTableName()
+    {
+        assertThatExceptionOfType(ConfigurationException.class)
+            .isThrownBy(() -> createTableMayThrow(String.format("CREATE TABLE %s.\"d-3\" (key int PRIMARY KEY, val int)", KEYSPACE)))
+            .withMessageContaining("Table name must not be empty or contain non-alphanumeric-underscore characters (got \"d-3\")");
+    }
+
     private void expectedFailure(String statement, String errorMsg)
     {
 
diff --git a/test/unit/org/apache/cassandra/schema/IndexMetadataTest.java b/test/unit/org/apache/cassandra/schema/IndexMetadataTest.java
index c9e0d52a0ebe..fc0a4cfb2e69 100644
--- a/test/unit/org/apache/cassandra/schema/IndexMetadataTest.java
+++ b/test/unit/org/apache/cassandra/schema/IndexMetadataTest.java
@@ -25,33 +25,12 @@
 
 import org.apache.cassandra.cql3.ColumnIdentifier;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class IndexMetadataTest {
-    
-    @Test
-    public void testIsNameValidPositive()
-    {
-        assertTrue(IndexMetadata.isNameValid("abcdefghijklmnopqrstuvwxyz"));
-        assertTrue(IndexMetadata.isNameValid("ABCDEFGHIJKLMNOPQRSTUVWXYZ"));
-        assertTrue(IndexMetadata.isNameValid("_01234567890"));
-    }
-    
-    @Test
-    public void testIsNameValidNegative()
-    {
-        assertFalse(IndexMetadata.isNameValid(null));
-        assertFalse(IndexMetadata.isNameValid(""));
-        assertFalse(IndexMetadata.isNameValid(" "));
-        assertFalse(IndexMetadata.isNameValid("@"));
-        assertFalse(IndexMetadata.isNameValid("!"));
-    }
-    
+public class IndexMetadataTest
+{
     @Test
     public void testGetDefaultIndexName()
     {
-        Assert.assertEquals("aB4__idx", IndexMetadata.generateDefaultIndexName("a B-4@!_+"));
+        Assert.assertEquals("aB4__idx", IndexMetadata.generateDefaultIndexName("a B-4@!_+", null));
         Assert.assertEquals("34_Ddd_F6_idx", IndexMetadata.generateDefaultIndexName("34_()Ddd", new ColumnIdentifier("#F%6*", true)));
     }
 }
diff --git a/test/unit/org/apache/cassandra/schema/KeyspaceMetadataTest.java b/test/unit/org/apache/cassandra/schema/KeyspaceMetadataTest.java
new file mode 100644
index 000000000000..80aec9b619a5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/schema/KeyspaceMetadataTest.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableCollection;
+import org.apache.cassandra.cql3.functions.types.ParseUtils;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.cql3.functions.UserFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.service.StorageService;
+
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotEquals;
+import static org.psjava.util.AssertStatus.assertTrue;
+
+public class KeyspaceMetadataTest extends CQLTester
+{
+    private static final String NEW_KEYSPACE = "new_keyspace";
+
+    @BeforeClass
+    public static void beforeClass()
+    {
+        StorageService.instance.setUpDistributedSystemKeyspaces();
+    }
+
+    @Test
+    public void testRenameWithNestedTypes()
+    {
+        String type1 = createType("CREATE TYPE %s (a int, b int)");
+        String type2 = createType("CREATE TYPE %s (x frozen<set<" + type1 + ">>, b int)");
+
+        createTable("CREATE TABLE %s (pk int," +
+                    " c frozen<list<" + type1 + ">>," +
+                    " s frozen<map<" + type1 + "," +  type2 + ">> static," +
+                    " v " + type2 + "," +
+                    " v2 frozen<tuple<int, " + type1 + ">>," +
+                    " v3 frozen<set<" + type1 + ">>, PRIMARY KEY (pk, c))");
+
+        KeyspaceMetadata ksMetadata = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+
+        checkKeyspaceRenaming(ksMetadata, NEW_KEYSPACE);
+    }
+
+    @Test
+    public void testRenameWithFunctions() throws Throwable
+    {
+        String type1 = createType("CREATE TYPE %s (a int, b int)");
+
+        String function1 = createFunction(KEYSPACE, "set<frozen<" + type1 + ">>," + type1, "CREATE FUNCTION %s (state set<frozen<" + type1 + ">>, val " + type1 + ") CALLED ON NULL INPUT RETURNS set<frozen<" + type1 + ">> LANGUAGE java AS ' state = state == null ? new HashSet<>() : state; state.add(val); return state;'");
+        String function2 = createFunction(KEYSPACE, "set<frozen<" + type1 + ">>", "CREATE FUNCTION %s (state set<frozen<" + type1 + ">>) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS ' return state == null ? 0 : state.size();'");
+        createAggregate(KEYSPACE, type1, "CREATE AGGREGATE %s (" + type1 + ") SFUNC " + StringUtils.substringAfter(function1, ".") + " STYPE set<frozen<" + type1 + ">> FINALFUNC " + StringUtils.substringAfter(function2, "."));
+
+        KeyspaceMetadata ksMetadata = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+
+        checkKeyspaceRenaming(ksMetadata, NEW_KEYSPACE);
+    }
+
+    @Test
+    public void testRenameWithDroppedColumns() throws Throwable
+    {
+        String type1 = createType("CREATE TYPE %s (a int, b int)");
+        String type2 = createType("CREATE TYPE %s (x frozen<set<" + type1 + ">>, b int)");
+
+        createTable("CREATE TABLE %s (pk int, c frozen<" + type1 + ">, s int static, v " + type2 + ", v2 text, PRIMARY KEY (pk, c))");
+        execute("ALTER TABLE %s DROP v2");
+        execute("ALTER TABLE %s DROP s");
+
+        KeyspaceMetadata ksMetadata = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+
+        checkKeyspaceRenaming(ksMetadata, NEW_KEYSPACE);
+    }
+
+    @Test
+    public void testRenameWithIndex() throws Throwable
+    {
+        String type1 = createType("CREATE TYPE %s (a int, b int)");
+
+        createTable("CREATE TABLE %s (pk int, c frozen<" + type1 + ">, s int static, v int, PRIMARY KEY (pk, c))");
+        createIndex("CREATE INDEX ON %s (c)");
+        createIndex("CREATE INDEX ON %s (v)");
+
+        KeyspaceMetadata ksMetadata = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+
+        checkKeyspaceRenaming(ksMetadata, NEW_KEYSPACE);
+    }
+
+    @Test
+    public void testRenameWithMv() throws Throwable
+    {
+        String type1 = createType("CREATE TYPE %s (a int, b int)");
+
+        createTable("CREATE TABLE %s (pk int, c frozen<" + type1 + ">, v int, v2 text, PRIMARY KEY (pk, c))");
+        execute("CREATE MATERIALIZED VIEW " + KEYSPACE + ".mv AS SELECT c, v, pk FROM " + currentTable() + " WHERE v IS NOT NULL AND c IS NOT NULL AND pk IS NOT NULL PRIMARY KEY (v, pk, c);");
+
+        KeyspaceMetadata ksMetadata = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+
+        checkKeyspaceRenaming(ksMetadata, NEW_KEYSPACE);
+    }
+
+    @Test
+    public void testTransformedTableParamsWithViews()
+    {
+        String baseTableName = createTable("CREATE TABLE %s (key int, val text, PRIMARY KEY (key, val)) WITH cdc=true;");
+        schemaChange("CREATE MATERIALIZED VIEW " + KEYSPACE + "." + createViewName() +
+                " AS SELECT * FROM " + baseTableName +
+                " WHERE val IS NOT NULL AND key IS NOT NULL" +
+                " PRIMARY KEY (val, key)" +
+                " WITH cdc=true;");
+
+        KeyspaceMetadata original = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+        assertNotNull(original);
+
+        java.util.function.Function<TableParams, TableParams> disableCdc = params -> params.unbuild()
+                                                                                           .cdc(false)
+                                                                                           .build();
+
+        KeyspaceMetadata transformed = original.withTransformedTableParams(disableCdc);
+
+
+        assertNotNull(original);
+        Predicate<Boolean> isCdcDisabled = cdc -> !cdc;
+        checkTransformedParams(original, transformed, params -> params.cdc, isCdcDisabled);
+    }
+
+    @Test
+    public void testUnsafeKeyspaceName() {
+        String[] badKeyspaceNames = {"\"non-alphanumeric\"",
+                "test_create_k8yq1r75bpzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
+        };
+        for (String badKeyspaceName : badKeyspaceNames)
+            assertThatExceptionOfType(InvalidRequestException.class)
+                    .isThrownBy(() -> createTableMayThrow(String.format("CREATE KEYSPACE %s with replication = " +
+                                    "{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }",
+                            badKeyspaceName)))
+                    .withMessageContaining(String.format("Keyspace name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got '%s')",
+                            SchemaConstants.NAME_LENGTH, ParseUtils.unDoubleQuote(badKeyspaceName)));
+    }
+
+    private void checkKeyspaceRenaming(KeyspaceMetadata original, String newName)
+    {
+        KeyspaceMetadata renamed = original.rename(newName);
+        assertEquals(newName, renamed.name);
+        assertEquals(original.kind, renamed.kind);
+        assertEquals(original.params, renamed.params);
+
+        original.types.forEach(t -> checkKeyspaceRenamingForType(newName, renamed.types, t, renamed.types.getNullable(t.name)));
+        checkKeyspaceRenamingForFunctions(newName, original, renamed);
+        original.tables.forEach(t -> checkKeyspaceRenamingForTable(newName, renamed.types, t, renamed.tables.getNullable(t.name)));
+        original.views.forEach(v -> checkKeyspaceRenamingForView(newName, renamed.types, v, renamed.views.getNullable(v.name())));
+    }
+
+    private void checkKeyspaceRenamingForFunctions(String newName, KeyspaceMetadata original, KeyspaceMetadata renamed)
+    {
+        Set<FunctionName> names = original.userFunctions.stream().map(UserFunction::name).collect(Collectors.toSet());
+        for (FunctionName name : names)
+        {
+            Iterator<UserFunction> originalIter = original.userFunctions.get(name).iterator();
+            Iterator<UserFunction> updatedIter = renamed.userFunctions.get(new FunctionName(newName, name.name)).iterator();
+            while (originalIter.hasNext() && updatedIter.hasNext())
+            {
+                checkKeyspaceRenamingForFunction(newName, renamed.userFunctions, renamed.types, originalIter.next(), updatedIter.next());
+            }
+            assertFalse(originalIter.hasNext());
+            assertFalse(updatedIter.hasNext());
+        }
+    }
+
+    private void checkKeyspaceRenamingForType(String keyspace,
+                                              Types newTypes,
+                                              AbstractType<?> originalType,
+                                              AbstractType<?> updatedType)
+    {
+        if (originalType.isUDT())
+        {
+            assertTrue(updatedType.isUDT());
+            UserType updatedUserType = (UserType) updatedType;
+            UserType originalUserType = (UserType) originalType;
+
+            // Checks that the updated type is present in the updated Types
+            UserType tmp = newTypes.getNullable(updatedUserType.name);
+            tmp = updatedType.isMultiCell() ? tmp : tmp.freeze();
+            assertEquals(updatedType, tmp);
+
+            assertEquals(keyspace, updatedUserType.keyspace);
+            assertEquals(originalUserType.name, updatedUserType.name);
+            assertEquals(originalUserType.fieldNames(), updatedUserType.fieldNames());
+        }
+        else if (originalType.referencesUserTypes())
+        {
+            List<AbstractType<?>> originalSubTypes = originalType.subTypes();
+            List<AbstractType<?>> updatedSubTypes = updatedType.subTypes();
+
+            assertEquals(originalSubTypes.size(), updatedSubTypes.size());
+            for (int i = 0, m = originalSubTypes.size(); i < m; i++)
+            {
+                checkKeyspaceRenamingForType(keyspace, newTypes, originalSubTypes.get(i), updatedSubTypes.get(i));
+            }
+        }
+        else
+        {
+            assertEquals(originalType, updatedType);
+        }
+    }
+
+    private void checkKeyspaceRenamingForFunction(String keyspace,
+                                                  UserFunctions newFunctions,
+                                                  Types newTypes,
+                                                  Function originalFunction,
+                                                  Function updatedFunction)
+    {
+        if (originalFunction.isNative())
+        {
+            assertEquals(originalFunction, updatedFunction);
+        }
+        else
+        {
+            // Checks that the updated function is present in the updated Types
+            Optional<UserFunction> maybe = newFunctions.find(updatedFunction.name(), updatedFunction.argTypes());
+            assertTrue(maybe.isPresent());
+            assertEquals(updatedFunction, maybe.get());
+
+            assertEquals(keyspace, updatedFunction.name().keyspace);
+            assertNotEquals(originalFunction.name().keyspace, updatedFunction.name().keyspace);
+            assertEquals(originalFunction.name().name, updatedFunction.name().name);
+            assertEquals(originalFunction.isAggregate(), updatedFunction.isAggregate());
+
+            List<AbstractType<?>> originalArgTypes = originalFunction.argTypes();
+            List<AbstractType<?>> updatedArgTypes = updatedFunction.argTypes();
+            assertEquals(originalArgTypes.size(), updatedArgTypes.size());
+            for (int i = 0, m = originalArgTypes.size(); i < m; i++)
+            {
+                checkKeyspaceRenamingForType(keyspace, newTypes, originalArgTypes.get(i), updatedArgTypes.get(i));
+            }
+
+            if (originalFunction.isAggregate())
+            {
+                UDAggregate originalUa = (UDAggregate) originalFunction;
+                UDAggregate updatedUa = (UDAggregate) updatedFunction;
+
+                checkKeyspaceRenamingForFunction(keyspace,
+                                                 newFunctions,
+                                                 newTypes,
+                                                 originalUa.finalFunction(),
+                                                 updatedUa.finalFunction());
+                assertEquals(originalUa.initialCondition(), updatedUa.initialCondition());
+                checkKeyspaceRenamingForFunction(keyspace,
+                                                 newFunctions,
+                                                 newTypes,
+                                                 originalUa.stateFunction(),
+                                                 updatedUa.stateFunction());
+                checkKeyspaceRenamingForType(keyspace, newTypes, originalUa.stateType(), updatedUa.stateType());
+            }
+            else
+            {
+                UDFunction originalUdf = (UDFunction) originalFunction;
+                UDFunction updatedUdf = (UDFunction) updatedFunction;
+
+                assertEquals(originalUdf.language(), updatedUdf.language());
+                assertEquals(originalUdf.body(), updatedUdf.body());
+                assertEquals(originalUdf.isCalledOnNullInput(), updatedUdf.isCalledOnNullInput());
+            }
+        }
+    }
+
+    private void checkKeyspaceRenamingForTable(String keyspace,
+                                               Types updatedTypes,
+                                               TableMetadata originalTable,
+                                               TableMetadata updatedTable)
+    {
+        assertEquals(keyspace, updatedTable.keyspace);
+        assertNotEquals(originalTable.keyspace, updatedTable.keyspace);
+        assertNotEquals(originalTable.id, updatedTable.id);
+        assertEquals(originalTable.name, updatedTable.name);
+        assertEquals(originalTable.partitioner, updatedTable.partitioner);
+        assertEquals(originalTable.kind, updatedTable.kind);
+        assertEquals(originalTable.flags, updatedTable.flags);
+        assertEquals(originalTable.indexes, updatedTable.indexes);
+        assertEquals(originalTable.triggers, updatedTable.triggers);
+        assertEquals(originalTable.params, updatedTable.params);
+
+        originalTable.columns().forEach(c -> checkKeyspaceRenamingForColumn(keyspace,
+                                                                            updatedTypes,
+                                                                            c,
+                                                                            updatedTable.getColumn(c.name.bytes)));
+        ImmutableCollection<DroppedColumn> droppedColumns = originalTable.droppedColumns.values();
+        droppedColumns.forEach(c -> checkKeyspaceRenamingForColumn(keyspace,
+                                                                   updatedTypes,
+                                                                   c.column,
+                                                                   updatedTable.getDroppedColumn(c.column.name.bytes)));
+    }
+
+    private void checkKeyspaceRenamingForView(String keyspace,
+                                              Types updatedTypes,
+                                              ViewMetadata originalView,
+                                              ViewMetadata updatedView)
+    {
+        checkKeyspaceRenamingForTable(keyspace, updatedTypes, originalView.metadata, updatedView.metadata);
+
+        Iterator<ColumnMetadata> requiredOriginal = originalView.metadata.columns().iterator();
+        Iterator<ColumnMetadata> requiredUpdated = updatedView.metadata.columns().iterator();
+
+        while (requiredOriginal.hasNext() && requiredUpdated.hasNext())
+        {
+            checkKeyspaceRenamingForColumn(keyspace,
+                                           updatedTypes,
+                                           requiredOriginal.next(),
+                                           requiredUpdated.next());
+        }
+        assertFalse(requiredOriginal.hasNext());
+        assertFalse(requiredUpdated.hasNext());
+    }
+
+    private void checkKeyspaceRenamingForColumn(String keyspace,
+                                                Types updatedTypes,
+                                                ColumnMetadata originalColumn,
+                                                ColumnMetadata updatedColumn)
+    {
+        assertEquals(keyspace, updatedColumn.ksName);
+        assertNotEquals(originalColumn.ksName, updatedColumn.ksName);
+        assertEquals(originalColumn.cfName, updatedColumn.cfName);
+        assertEquals(originalColumn.name, updatedColumn.name);
+        checkKeyspaceRenamingForType(keyspace, updatedTypes, originalColumn.type, updatedColumn.type);
+    }
+
+    private <T> void checkTransformedParams(KeyspaceMetadata original,
+                                            KeyspaceMetadata transformed,
+                                            java.util.function.Function<TableParams, T> paramValueExtractor,
+                                            Predicate<T> transformedParamTest)
+    {
+        for (TableMetadata table: transformed.tables)
+        {
+            T param = paramValueExtractor.apply(table.params);
+            assertTrue(transformedParamTest.test(param), "Param doesn't satisfy the provided test for table " + table);
+        }
+
+        for (ViewMetadata view: transformed.views)
+        {
+            T param = paramValueExtractor.apply(view.metadata.params);
+            assertTrue(transformedParamTest.test(param), "Param doesn't satisfy the provided test for view " + view);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/schema/KeyspaceParamsTest.java b/test/unit/org/apache/cassandra/schema/KeyspaceParamsTest.java
new file mode 100644
index 000000000000..66c88d415d99
--- /dev/null
+++ b/test/unit/org/apache/cassandra/schema/KeyspaceParamsTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Map;
+
+import org.junit.Test;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_NTS_DC_OVERRIDE_PROPERTY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_NTS_RF_OVERRIDE_PROPERTY;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class KeyspaceParamsTest
+{
+    private static void setValues(String dc, String rf)
+    {
+        if (dc == null)
+            System.getProperties().remove(SYSTEM_DISTRIBUTED_NTS_DC_OVERRIDE_PROPERTY.getKey());
+        else
+            SYSTEM_DISTRIBUTED_NTS_DC_OVERRIDE_PROPERTY.setString(dc);
+
+        if (rf == null)
+            System.getProperties().remove(SYSTEM_DISTRIBUTED_NTS_RF_OVERRIDE_PROPERTY.getKey());
+        else
+            SYSTEM_DISTRIBUTED_NTS_RF_OVERRIDE_PROPERTY.setString(rf);
+    }
+
+    @Test
+    public void testEmpty()
+    {
+        setValues(null, null);
+        Map<String, String> result = KeyspaceParams.getSystemDistributedNtsOverride();
+        assertThat(result).isEmpty();
+    }
+
+    @Test
+    public void testNoRF()
+    {
+        setValues("dc1,dc2,dc3", null);
+        Map<String, String> result = KeyspaceParams.getSystemDistributedNtsOverride();
+        assertThat(result).isEmpty();
+    }
+
+    @Test
+    public void testNoDC()
+    {
+        setValues(null, "1");
+        Map<String, String> result = KeyspaceParams.getSystemDistributedNtsOverride();
+        assertThat(result).isEmpty();
+    }
+
+    @Test
+    public void testEmptyDC()
+    {
+        setValues("", "1");
+        Map<String, String> result = KeyspaceParams.getSystemDistributedNtsOverride();
+        assertThat(result).isEmpty();
+    }
+
+    @Test
+    public void testInvalidRF1()
+    {
+        setValues("dc1,dc2,dc3", "0");
+        Map<String, String> result = KeyspaceParams.getSystemDistributedNtsOverride();
+        assertThat(result).isEmpty();
+    }
+
+    @Test
+    public void testInvalidRF2()
+    {
+        setValues("dc1,dc2,dc3", "100");
+        Map<String, String> result = KeyspaceParams.getSystemDistributedNtsOverride();
+        assertThat(result).isEmpty();
+    }
+
+    @Test
+    public void testMalformedRF()
+    {
+        setValues("dc1,dc2,dc3", "asdf");
+        Map<String, String> result = KeyspaceParams.getSystemDistributedNtsOverride();
+        assertThat(result).isEmpty();
+    }
+
+    @Test
+    public void test()
+    {
+        setValues("dc1,dc2,dc3", "4");
+        Map<String, String> result = KeyspaceParams.getSystemDistributedNtsOverride();
+        assertThat(result).isEqualTo(ReplicationParams.nts("dc1", "4", "dc2", "4", "dc3", "4").asMap());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/schema/MemtableParamsTest.java b/test/unit/org/apache/cassandra/schema/MemtableParamsTest.java
index 939c96bc9acd..d3fc8befc22a 100644
--- a/test/unit/org/apache/cassandra/schema/MemtableParamsTest.java
+++ b/test/unit/org/apache/cassandra/schema/MemtableParamsTest.java
@@ -26,7 +26,7 @@
 
 import org.apache.cassandra.config.InheritingClass;
 import org.apache.cassandra.config.ParameterizedClass;
-import org.apache.cassandra.db.memtable.SkipListMemtableFactory;
+import org.apache.cassandra.db.memtable.TrieMemtableFactory;
 import org.apache.cassandra.exceptions.ConfigurationException;
 
 import static org.junit.Assert.assertEquals;
@@ -35,7 +35,7 @@
 
 public class MemtableParamsTest
 {
-    static final ParameterizedClass DEFAULT = SkipListMemtableFactory.CONFIGURATION;
+    static final ParameterizedClass DEFAULT = TrieMemtableFactory.CONFIGURATION;
 
     @Test
     public void testDefault()
diff --git a/test/unit/org/apache/cassandra/schema/MigrationCoordinatorTest.java b/test/unit/org/apache/cassandra/schema/MigrationCoordinatorTest.java
index b4966af3a4a3..a006d3146b13 100644
--- a/test/unit/org/apache/cassandra/schema/MigrationCoordinatorTest.java
+++ b/test/unit/org/apache/cassandra/schema/MigrationCoordinatorTest.java
@@ -40,8 +40,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.gms.ApplicationState;
@@ -64,6 +64,7 @@
 import org.mockito.internal.creation.MockSettingsImpl;
 
 import static com.google.common.util.concurrent.Futures.getUnchecked;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
@@ -88,6 +89,7 @@ public class MigrationCoordinatorTest
 
     static
     {
+        NODES_DISABLE_PERSISTING_TO_SYSTEM_KEYSPACE.setBoolean(true);
         try
         {
             EP1 = InetAddressAndPort.getByName("10.0.0.1");
@@ -139,7 +141,7 @@ private Wrapper(int maxOutstandingRequests)
             when(versions.knows(any())).thenReturn(true);
             when(versions.getRaw(any())).thenReturn(MessagingService.current_version);
             this.coordinator = new MigrationCoordinator(messagingService,
-                                                        ImmediateExecutor.INSTANCE,
+                                                        Stage.IMMEDIATE,
                                                         oneTimeExecutor,
                                                         maxOutstandingRequests,
                                                         gossiper,
@@ -203,7 +205,7 @@ public void requestResponseCycle() throws InterruptedException
         Pair<InetAddressAndPort, RequestCallback<Collection<Mutation>>> request2 = wrapper.requests.poll();
         Assert.assertEquals(EP2, request2.left);
         Assert.assertFalse(coordinator.awaitSchemaRequests(1));
-        request2.right.onResponse(Message.remoteResponse(request2.left, Verb.SCHEMA_PULL_RSP, Collections.emptyList()));
+        request2.right.onResponse(Message.remoteResponse(request2.left, Verb.SCHEMA_PULL_RSP, Message.NO_PARAMS, Collections.emptyList()));
         Assert.assertEquals(EP2, Iterables.getOnlyElement(wrapper.mergedSchemasFrom));
         Assert.assertTrue(coordinator.awaitSchemaRequests(1));
 
@@ -315,7 +317,7 @@ public void testWeKeepSendingRequests() throws Exception
 
         getUnchecked(wrapper.coordinator.reportEndpointVersion(EP3, V2));
         Pair<InetAddressAndPort, RequestCallback<Collection<Mutation>>> cb = wrapper.requests.remove();
-        cb.right.onResponse(Message.remoteResponse(cb.left, Verb.SCHEMA_PULL_RSP, Collections.emptyList()));
+        cb.right.onResponse(Message.remoteResponse(cb.left, Verb.SCHEMA_PULL_RSP, Message.NO_PARAMS, Collections.emptyList()));
 
         getUnchecked(wrapper.coordinator.reportEndpointVersion(EP1, V1));
         getUnchecked(wrapper.coordinator.reportEndpointVersion(EP2, V1));
@@ -350,7 +352,7 @@ public void testWeKeepSendingRequests() throws Exception
 
         // a single success should unblock startup though
         cb = wrapper.requests.remove();
-        cb.right.onResponse(Message.remoteResponse(cb.left, Verb.SCHEMA_PULL_RSP, Collections.emptyList()));
+        cb.right.onResponse(Message.remoteResponse(cb.left, Verb.SCHEMA_PULL_RSP, Message.NO_PARAMS, Collections.emptyList()));
         Assert.assertTrue(wrapper.coordinator.awaitSchemaRequests(1));
     }
 
@@ -438,7 +440,7 @@ public void reset() throws UnknownHostException
 
             assertThat(msg.verb()).isEqualTo(Verb.SCHEMA_PULL_REQ);
             assertThat(endpoint).isEqualTo(regularNode1);
-            callback.onResponse(Message.remoteResponse(regularNode1, Verb.SCHEMA_PULL_RSP, mutations));
+            callback.onResponse(Message.remoteResponse(regularNode1, Verb.SCHEMA_PULL_RSP, Message.NO_PARAMS, mutations));
             return null;
         }).when(wrapper.messagingService).sendWithCallback(any(Message.class), any(InetAddressAndPort.class), any(RequestCallback.class));
         wrapper.coordinator.reset();
diff --git a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
index 16ef782e4427..58650959ea70 100644
--- a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
+++ b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
@@ -61,6 +61,7 @@
 public class MigrationManagerTest
 {
     private static final String KEYSPACE1 = "keyspace1";
+    private static final String KEYSPACE2 = "keyspace2";
     private static final String KEYSPACE3 = "keyspace3";
     private static final String KEYSPACE6 = "keyspace6";
     private static final String EMPTY_KEYSPACE = "test_empty_keyspace";
@@ -80,6 +81,10 @@ public static void defineSchema() throws ConfigurationException
                                     KeyspaceParams.simple(1),
                                     SchemaLoader.standardCFMD(KEYSPACE1, TABLE1),
                                     SchemaLoader.standardCFMD(KEYSPACE1, TABLE2));
+        SchemaLoader.createKeyspace(KEYSPACE2,
+                KeyspaceParams.simple(1),
+                SchemaLoader.standardCFMD(KEYSPACE2, TABLE1),
+                SchemaLoader.standardCFMD(KEYSPACE2, TABLE2));
         SchemaLoader.createKeyspace(KEYSPACE3,
                                     KeyspaceParams.simple(5),
                                     SchemaLoader.standardCFMD(KEYSPACE3, TABLE1),
@@ -254,6 +259,56 @@ public void addNewKS() throws ConfigurationException
         assertRows(rows, row("key0", "col0", "val0"));
     }
 
+    @Test
+    public void dropKS() throws ConfigurationException
+    {
+        // sanity
+        final KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(KEYSPACE2);
+        assertNotNull(ks);
+        final TableMetadata cfm = ks.tables.getNullable(TABLE2);
+        assertNotNull(cfm);
+
+        // write some data, force a flush, then verify that files exist on disk.
+        for (int i = 0; i < 100; i++)
+            QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+                                                         KEYSPACE2, TABLE2),
+                                           "dropKs", "col" + i, "anyvalue");
+        ColumnFamilyStore cfs = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name);
+        assertNotNull(cfs);
+        Util.flush(cfs);
+        assertTrue(!cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().isEmpty());
+
+        SchemaTestUtil.announceKeyspaceDrop(ks.name);
+
+        assertNull(Schema.instance.getKeyspaceMetadata(ks.name));
+
+        // write should fail.
+        boolean success = true;
+        try
+        {
+            QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+                                                         KEYSPACE2, TABLE2),
+                                           "dropKs", "col0", "anyvalue");
+        }
+        catch (Throwable th)
+        {
+            success = false;
+        }
+        assertFalse("This mutation should have failed since the KS no longer exists.", success);
+
+        // reads should fail too.
+        boolean threw = false;
+        try
+        {
+            Keyspace.open(ks.name);
+        }
+        catch (Throwable th)
+        {
+            threw = true;
+        }
+        assertTrue(threw);
+    }
+
     @Test
     public void dropKSUnflushed() throws ConfigurationException
     {
diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java
index 5d8b7c1dc7c1..f6dc6c84fb0b 100644
--- a/test/unit/org/apache/cassandra/schema/MockSchema.java
+++ b/test/unit/org/apache/cassandra/schema/MockSchema.java
@@ -68,6 +68,7 @@
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FilterFactory;
 import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
 
@@ -255,7 +256,7 @@ else if (BtiFormat.is(format))
                 BtiTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components)
                                                                               .setTableMetadataRef(cfs.metadata)
                                                                               .setDataFile(fileHandle.sharedCopy())
-                                                                              .setPartitionIndex(new PartitionIndex(fileHandle.sharedCopy(), 0, 0, readerBounds(firstToken), readerBounds(lastToken)))
+                                                                              .setPartitionIndex(new PartitionIndex(fileHandle.sharedCopy(), 0, 0, readerBounds(firstToken), readerBounds(lastToken), ByteComparable.Version.OSS50))
                                                                               .setRowIndexFile(fileHandle.sharedCopy())
                                                                               .setFilter(FilterFactory.AlwaysPresent)
                                                                               .setMaxDataAge(1L)
@@ -357,9 +358,8 @@ private static File temp(String id)
     public static void cleanup()
     {
         // clean up data directory which are stored as data directory/keyspace/data files
-        for (String dirName : DatabaseDescriptor.getAllDataFileLocations())
+        for (File dir : DatabaseDescriptor.getAllDataFileLocations())
         {
-            File dir = new File(dirName);
             if (!dir.exists())
                 continue;
             String[] children = dir.tryListNames();
diff --git a/test/unit/org/apache/cassandra/schema/RemoveWithoutDroppingTest.java b/test/unit/org/apache/cassandra/schema/RemoveWithoutDroppingTest.java
index 683e52f40fcc..28621574f7bf 100644
--- a/test/unit/org/apache/cassandra/schema/RemoveWithoutDroppingTest.java
+++ b/test/unit/org/apache/cassandra/schema/RemoveWithoutDroppingTest.java
@@ -112,6 +112,7 @@ public void testRemoveWithoutDropping() throws Throwable
         String ks = "test_remove_without_dropping";
         String tab = "test_table";
         testRemoveKeyspace(ks, tab, false);
+
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/schema/SchemaComparisonBetweenSSTablesAndCQLTablesTest.java b/test/unit/org/apache/cassandra/schema/SchemaComparisonBetweenSSTablesAndCQLTablesTest.java
new file mode 100644
index 000000000000..b19dda0ad134
--- /dev/null
+++ b/test/unit/org/apache/cassandra/schema/SchemaComparisonBetweenSSTablesAndCQLTablesTest.java
@@ -0,0 +1,550 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.cassandra.io.util.File;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.SimpleDateType;
+import org.apache.cassandra.db.marshal.TimeType;
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.tools.Util;
+
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+
+public class SchemaComparisonBetweenSSTablesAndCQLTablesTest
+{
+    private static final String ksName = "74656e616e742d616161_schema_validation_tests";
+    private static final String ssTableResourceRootPath = "test/resources/schema/schema_validation_tests/";
+
+    /**
+     * Table schema:
+     * CREATE TABLE test_1 (
+     * col_txt text PRIMARY KEY,
+     * col_int int,
+     * col_uuid uuid,
+     * col_bool boolean,
+     * col_dec decimal
+     * );
+     */
+    private static final String TABLE_1_NAME = "test_1";
+    private static final UUID TABLE_1_UUID = UUID.fromString("7ab21491-8482-11ee-825e-31532694b0d3");
+
+    /**
+     * Table schema:
+     * CREATE TABLE test_2 (
+     * col_txt text,
+     * col_int int,
+     * col_uuid uuid,
+     * col_bool boolean,
+     * col_dec decimal,
+     * PRIMARY KEY ((col_txt), col_uuid)
+     * );
+     */
+    private static final String TABLE_2_NAME = "test_2";
+    private static final UUID TABLE_2_UUID = UUID.fromString("83ac0dd1-8482-11ee-825e-31532694b0d3");
+
+    /**
+     * Table schema:
+     * CREATE TABLE test_3 (
+     * col_txt text,
+     * col_int int,
+     * col_uuid uuid,
+     * col_bool boolean,
+     * col_dec decimal,
+     * PRIMARY KEY ((col_txt, col_int), col_uuid)
+     * ) WITH CLUSTERING ORDER BY (col_uuid DESC);
+     */
+    private static final String TABLE_3_NAME = "test_3";
+    private static final UUID TABLE_3_UUID = UUID.fromString("88b6cb80-8482-11ee-825e-31532694b0d3");
+
+    /**
+     * Table schema:
+     * CREATE TABLE test_4(
+     * col_txt text,
+     * col_int int,
+     * col_uuid uuid,
+     * col_bool boolean,
+     * col_dec decimal,
+     * PRIMARY KEY ((col_txt), col_int, col_uuid)
+     * ) WITH CLUSTERING ORDER BY (col_int DESC, col_uuid ASC);
+     */
+    private static final String TABLE_4_NAME = "test_4";
+    private static final UUID TABLE_4_UUID = UUID.fromString("8bf57bc0-8482-11ee-825e-31532694b0d3");
+
+    /**
+     * Table schema:
+     * CREATE TABLE test_5(
+     * col_txt text primary key,
+     * col_ascii ascii,
+     * col_bigint bigint,
+     * col_blob blob,
+     * col_bool boolean,
+     * col_date date,
+     * col_dec decimal,
+     * col_dbl double,
+     * col_float float,
+     * col_inet inet,
+     * col_small smallint,
+     * col_time time,
+     * col_timestamp timestamp,
+     * col_timeuuid timeuuid,
+     * col_tinyint tinyint,
+     * col_varchar varchar,
+     * col_varint varint
+     * );
+     */
+    private static final String TABLE_5_NAME = "test_5";
+    private static final UUID TABLE_5_UUID = UUID.fromString("9db47b90-8482-11ee-825e-31532694b0d3");
+
+    private static final Map<String, TableMetadata> ssTableMetadataForTableMap = new HashMap<>();
+
+    @BeforeClass
+    public static void init() throws Exception
+    {
+        DatabaseDescriptor.toolInitialization();
+        initializeSSTableMetadataForTableMap();
+    }
+
+    private static void initializeSSTableMetadataForTableMap() throws Exception
+    {
+        ssTableMetadataForTableMap.put(TABLE_1_NAME,
+                                       Util.metadataFromSSTable(Descriptor.fromFile(new File(ssTableResourceRootPath + "test_1-7ab21491848211ee825e31532694b0d3/bb-2-bti-Data.db")),
+                                                                ksName, TABLE_1_NAME));
+
+        ssTableMetadataForTableMap.put(TABLE_2_NAME,
+                                       Util.metadataFromSSTable(Descriptor.fromFile(new File(ssTableResourceRootPath + "test_2-83ac0dd1848211ee825e31532694b0d3/bb-1-bti-Data.db")),
+                                                                ksName, TABLE_2_NAME));
+
+        ssTableMetadataForTableMap.put(TABLE_3_NAME,
+                                       Util.metadataFromSSTable(Descriptor.fromFile(new File(ssTableResourceRootPath + "test_3-88b6cb80848211ee825e31532694b0d3/bb-1-bti-Data.db")),
+                                                                ksName, TABLE_3_NAME));
+
+        ssTableMetadataForTableMap.put(TABLE_4_NAME,
+                                       Util.metadataFromSSTable(Descriptor.fromFile(new File(ssTableResourceRootPath + "test_4-8bf57bc0848211ee825e31532694b0d3/bb-1-bti-Data.db")),
+                                                                ksName, TABLE_4_NAME));
+
+        ssTableMetadataForTableMap.put(TABLE_5_NAME,
+                                       Util.metadataFromSSTable(Descriptor.fromFile(new File(ssTableResourceRootPath + "test_5-9db47b90848211ee825e31532694b0d3/bb-1-bti-Data.db")),
+                                                                ksName, TABLE_5_NAME));
+    }
+
+    // *** GENERAL VALIDATION SUCCESS TESTS *** //
+
+    @Test
+    public void testIdenticalSchemaPassesValidation_SinglePartKeyNoClustering()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuiid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    @Test
+    public void testIdenticalSchemaPassesValidation_SinglePartKeySingleClusteringAsc()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_2_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_2_NAME, TableId.fromUUID(TABLE_2_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addClusteringColumn("col_uuiid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    @Test
+    public void testIdenticalSchemaPassesValidation_CompoundPartKeySingleClusteringDesc()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_3_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_3_NAME, TableId.fromUUID(TABLE_3_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addPartitionKeyColumn("col_int", Int32Type.instance)
+                                                      .addClusteringColumn("col_uuid", ReversedType.getInstance(UUIDType.instance))
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    @Test
+    public void testIdenticalSchemaPassesValidation_SinglePartKeyTwoClusteringAscDesc()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_4_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_4_NAME, TableId.fromUUID(TABLE_4_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addClusteringColumn("col_int", ReversedType.getInstance(Int32Type.instance))
+                                                      .addClusteringColumn("col_uuid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    @Test
+    public void testIdenticalSchemaPassesValidation_ManyDataTypes()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_5_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_5_NAME, TableId.fromUUID(TABLE_5_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_ascii", AsciiType.instance)
+                                                      .addRegularColumn("col_bigint", LongType.instance)
+                                                      .addRegularColumn("col_blob", BytesType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_date", SimpleDateType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .addRegularColumn("col_dbl", DoubleType.instance)
+                                                      .addRegularColumn("col_float", FloatType.instance)
+                                                      .addRegularColumn("col_inet", InetAddressType.instance)
+                                                      .addRegularColumn("col_small", ShortType.instance)
+                                                      .addRegularColumn("col_time", TimeType.instance)
+                                                      .addRegularColumn("col_timestamp", TimestampType.instance)
+                                                      .addRegularColumn("col_timeuuid", TimeUUIDType.instance)
+                                                      .addRegularColumn("col_tinyint", ByteType.instance)
+                                                      .addRegularColumn("col_varchar", UTF8Type.instance)
+                                                      .addRegularColumn("col_varint", IntegerType.instance)
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    // *** KEYSPACE NAME, TABLE NAME AND TABLE ID ADDITIONAL TESTS *** //
+
+    @Test
+    public void testTableNameMismatchFailsValidation()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, "different_table_name", TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuiid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testKeyspaceNameMismatchPassesValidation()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder("different_keyspace_name", TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuiid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    @Test
+    public void testTableIdMismatchPassesValidation()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(UUID.randomUUID())) // different UUID
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuiid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    // *** PARTITION KEY VALIDATION ADDITIONAL TESTS *** //
+
+    @Test
+    public void testPartitionKeyMismatchFailsValidation_WrongColumn()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addRegularColumn("col_txt", UTF8Type.instance) // should be single part key
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addPartitionKeyColumn("col_uuiid", UUIDType.instance) // is not part key
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testPartitionKeyMismatchFailsValidation_WrongAdditionalColumn()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance) // is single part key
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addPartitionKeyColumn("col_uuiid", UUIDType.instance) // is not part key
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance).build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testPartitionKeyMismatchFailsValidation_WrongColumnType()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", Int32Type.instance) // is single part key but wrong type
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuiid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    /**
+     * Different column name for partition key column passes validation
+     * This is the expected behaviour as the partition key column names are not contained in the sstable metadata
+     */
+    @Test
+    public void testPartitionKeyColumnNameMismatchPassesValidation()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("wrong_name", UTF8Type.instance) // correct type but wrong type
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuiid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance).build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    @Test
+    public void testPartitionKeyMismatchFailsValidation_CompoundPartKeyWrongColumnOrder()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_3_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_3_NAME, TableId.fromUUID(TABLE_3_UUID))
+                                                      .addPartitionKeyColumn("col_int", Int32Type.instance)   // is part key column but should be second
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)   // is part key column but should be first
+                                                      .addClusteringColumn("col_uuid", ReversedType.getInstance(UUIDType.instance))
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    // *** CLUSTERING COLUMN VALIDATION ADDITIONAL TESTS *** //
+
+    @Test
+    public void testClusteringColumnsMismatchFailsValidation_MissingClusteringColumn()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_2_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_2_NAME, TableId.fromUUID(TABLE_2_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuiid", UUIDType.instance)   // should be clustering column
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testClusteringColumnsMismatchFailsValidation_WrongClusteringColumn()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_2_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_2_NAME, TableId.fromUUID(TABLE_2_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addClusteringColumn("col_int", Int32Type.instance) // is not clustering column
+                                                      .addRegularColumn("col_uuiid", UUIDType.instance)   // should be clustering column
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testClusteringColumnsMismatchFailsValidation_CorrectClusteringColumnButWrongOrderingClause()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_2_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_2_NAME, TableId.fromUUID(TABLE_2_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addClusteringColumn("col_uuiid", ReversedType.getInstance(UUIDType.instance)) // is clustering column but should be ascending
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testClusteringColumnsMismatchFailsValidation_WrongAdditionalClusteringColumn()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_2_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_2_NAME, TableId.fromUUID(TABLE_2_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addClusteringColumn("col_int", Int32Type.instance)// is not clustering column
+                                                      .addClusteringColumn("col_uuiid", UUIDType.instance) // is clustering column
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testClusteringColumnsMismatchFailsValidation_WrongClusteringColumnType()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_2_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_2_NAME, TableId.fromUUID(TABLE_2_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addClusteringColumn("col_uuiid", DecimalType.instance) // is clustering column but wrong type
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testClusteringColumnsMismatchFailsValidation_WrongOrderOfClusteringColumns()
+    {
+
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_4_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_4_NAME, TableId.fromUUID(TABLE_4_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addClusteringColumn("col_uuid", UUIDType.instance) // is clustering column but should be second
+                                                      .addClusteringColumn("col_int", ReversedType.getInstance(Int32Type.instance))   // is clustering column but should be first
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    /**
+     * Different column name for clustering column passes validation
+     * This is the expected behaviour as the clustering column names are not contained in the sstable metadata
+     */
+    @Test
+    public void testClusteringColumnNameMismatchPassesValidation()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_2_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_2_NAME, TableId.fromUUID(TABLE_2_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addClusteringColumn("wrong_name", UUIDType.instance)   // is clustering column but wrong name
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance).build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    // *** REGULAR COLUMN VALIDATION ADDITIONAL TESTS *** //
+
+    @Test
+    public void testRegularColumnMismatchFailsValidation_WrongColumnType()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuid", UTF8Type.instance)  // is regular column but wrong type
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatExceptionOfType(org.apache.cassandra.exceptions.ConfigurationException.class)
+        .isThrownBy(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata));
+    }
+
+    @Test
+    public void testRegularColumnNameMismatchPassesValidation()
+    {
+        // TODO this may have to be expected to pass validation legitimately
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("xvfdsfsdfsd", UUIDType.instance)  // is regular column but wrong name
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    @Test
+    public void testAdditionalRegularColumnPassesValidation()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuid", UUIDType.instance)
+                                                      .addRegularColumn("col_bool", BooleanType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .addRegularColumn("additional_column", FloatType.instance) // column not present in sstable schema
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+
+    @Test
+    public void testMissingRegularColumnPassesValidation()
+    {
+        TableMetadata ssTableMetadata = ssTableMetadataForTableMap.get(TABLE_1_NAME);
+        TableMetadata cqlTableMetadata = TableMetadata.builder(ksName, TABLE_1_NAME, TableId.fromUUID(TABLE_1_UUID))
+                                                      .addPartitionKeyColumn("col_txt", UTF8Type.instance)
+                                                      .addRegularColumn("col_int", Int32Type.instance)
+                                                      .addRegularColumn("col_uuid", UUIDType.instance)
+                                                      .addRegularColumn("col_dec", DecimalType.instance)
+                                                      .build();
+        assertThatCode(() -> ssTableMetadata.validateTableNameAndStructureCompatibility(cqlTableMetadata)).doesNotThrowAnyException();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
index 320c4fbf8eaa..7551fb3af418 100644
--- a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.function.Consumer;
 import java.util.stream.Stream;
 
 import com.google.common.collect.ImmutableMap;
@@ -39,6 +40,7 @@
 import org.junit.runner.RunWith;
 
 import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
@@ -48,7 +50,9 @@
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.distributed.shared.WithProperties;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.NoPayload;
@@ -62,8 +66,9 @@
 
 import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 @RunWith(BMUnitRunner.class)
 public class SchemaKeyspaceTest
@@ -175,6 +180,43 @@ public void testExtensions() throws IOException
         assertEquals(extensions, metadata.params.extensions);
     }
 
+    @Test
+    public void testMetricsExtensions()
+    {
+        createTable("SandBoxMetrics", String.format("CREATE TABLE %s (a text primary key, b int, c int) WITH extensions = {'%s': '%s'}",
+                                                          "test",
+                                                          TableMetrics.TABLE_EXTENSIONS_HISTOGRAMS_METRICS_KEY,
+                                                          TableMetrics.MetricsAggregation.AGGREGATED.asCQLString()));
+
+        TableMetadata metadata = Schema.instance.getTableMetadata("SandBoxMetrics", "test");
+        assertNotNull(metadata);
+
+        ImmutableMap<String, ByteBuffer> extensions = metadata.params.extensions;
+        assertNotNull(extensions);
+        assertFalse("extensions should not be empty", extensions.isEmpty());
+
+        assertEquals(TableMetrics.MetricsAggregation.AGGREGATED, TableMetrics.MetricsAggregation.fromMetadata(metadata));
+
+        Consumer<TableMetrics.MetricsAggregation> changeMetricAggregation = aggregation -> {
+            TableMetadata meta = Schema.instance.getTableMetadata("SandBoxMetrics", "test");
+
+            ImmutableMap<String, ByteBuffer> extensionsMap = ImmutableMap.of(TableMetrics.TABLE_EXTENSIONS_HISTOGRAMS_METRICS_KEY,
+                                                                             ByteBuffer.wrap(new byte[]{aggregation.val}));
+
+            TableMetadata alteredMetadata = meta.unbuild().extensions(extensionsMap).build();
+
+            updateTable("SandBoxMetrics", meta, alteredMetadata);
+        };
+
+        changeMetricAggregation.accept(TableMetrics.MetricsAggregation.INDIVIDUAL);
+        metadata = Schema.instance.getTableMetadata("SandBoxMetrics", "test");
+        assertEquals(TableMetrics.MetricsAggregation.INDIVIDUAL, TableMetrics.MetricsAggregation.fromMetadata(metadata));
+
+        changeMetricAggregation.accept(TableMetrics.MetricsAggregation.AGGREGATED);
+        metadata = Schema.instance.getTableMetadata("SandBoxMetrics", "test");
+        assertEquals(TableMetrics.MetricsAggregation.AGGREGATED, TableMetrics.MetricsAggregation.fromMetadata(metadata));
+    }
+
     @Test
     public void testReadRepair()
     {
@@ -254,7 +296,7 @@ private static void checkInverses(TableMetadata metadata) throws Exception
                                                                 UnfilteredRowIterators.filter(serializedCD.unfilteredIterator(), FBUtilities.nowInSeconds()));
         Set<ColumnMetadata> columns = new HashSet<>();
         for (UntypedResultSet.Row row : columnsRows)
-            columns.add(SchemaKeyspace.createColumnFromRow(row, Types.none(), UserFunctions.none()));
+            columns.add(SchemaKeyspace.createColumnFromRow(row, Types.none(), UserFunctions.none(), false));
 
         assertEquals(metadata.params, params);
         assertEquals(new HashSet<>(metadata.columns()), columns);
@@ -287,4 +329,36 @@ public void testSchemaNoColumn()
         executeOnceInternal(query, testKS, testTable);
         SchemaKeyspace.fetchNonSystemKeyspaces();
     }
+
+    @Test
+    public void testIsKeyspaceWithLocalStrategy()
+    {
+        try (WithProperties properties = new WithProperties().set(CassandraRelevantProperties.TEST_ALLOW_LOCAL_STRATEGY, true))
+        {
+            assertTrue(Schema.isKeyspaceWithLocalStrategy("system"));
+            assertTrue(Schema.isKeyspaceWithLocalStrategy(Schema.instance.getKeyspaceMetadata("system")));
+            assertFalse(Schema.isKeyspaceWithLocalStrategy("non_existing"));
+
+            SchemaLoader.createKeyspace("local_ks", KeyspaceParams.local());
+            SchemaLoader.createKeyspace("simple_ks", KeyspaceParams.simple(3));
+
+            assertTrue(Schema.isKeyspaceWithLocalStrategy("local_ks"));
+            assertTrue(Schema.isKeyspaceWithLocalStrategy(Schema.instance.getKeyspaceMetadata("local_ks")));
+            assertFalse(Schema.isKeyspaceWithLocalStrategy("simple_ks"));
+        }
+        finally
+        {
+            String query = String.format("DROP KEYSPACE %s", "local_ks");
+            executeOnceInternal(query);
+            query = String.format("DROP KEYSPACE %s", "simple_ks");
+            executeOnceInternal(query);
+        }
+    }
+
+    @Test
+    public void testEverywhere()
+    {
+        SchemaLoader.createKeyspace("everywhereKeyspace", KeyspaceParams.everywhere());
+        assertFalse(Schema.isKeyspaceWithLocalStrategy("everywhereKeyspace"));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/schema/SchemaStatementWarningsTest.java b/test/unit/org/apache/cassandra/schema/SchemaStatementWarningsTest.java
index 197854897e37..91673e31cb75 100644
--- a/test/unit/org/apache/cassandra/schema/SchemaStatementWarningsTest.java
+++ b/test/unit/org/apache/cassandra/schema/SchemaStatementWarningsTest.java
@@ -52,7 +52,7 @@ public void before()
     @BMRules(rules = { @BMRule(name = "client warning 1",
                                targetClass = "CreateKeyspaceStatement",
                                targetMethod = "apply",
-                               targetLocation = "AT INVOKE KeyspaceParams.validate",
+                               targetLocation = "AT INVOKE KeyspaceMetadata.validate",
                                action = "org.apache.cassandra.schema.SchemaStatementWarningsTest.addWarn()"),
                        @BMRule(name = "client warning 2",
                                targetClass = "CreateKeyspaceStatement",
diff --git a/test/unit/org/apache/cassandra/schema/SchemaTest.java b/test/unit/org/apache/cassandra/schema/SchemaTest.java
index 1c6b6aa4836a..9114985da0b2 100644
--- a/test/unit/org/apache/cassandra/schema/SchemaTest.java
+++ b/test/unit/org/apache/cassandra/schema/SchemaTest.java
@@ -38,7 +38,10 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.SchemaTransformation.SchemaTransformationResult;
 import org.apache.cassandra.utils.FBUtilities;
 import org.awaitility.Awaitility;
 
@@ -46,6 +49,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
 
 public class SchemaTest
 {
@@ -90,7 +94,8 @@ public void testTransKsMigration() throws IOException
     }
 
     @Test
-    public void testSchemaVersionUpdates() {
+    public void testSchemaVersionUpdates()
+    {
         KeyspaceMetadata ksm = KeyspaceMetadata.create("testSchemaVersionUpdates", KeyspaceParams.simple(1));
         SchemaTransformation.SchemaTransformationResult r = Schema.instance.transform(current -> current.withAddedOrUpdated(ksm));
 
@@ -106,6 +111,26 @@ public void testSchemaVersionUpdates() {
         }
         Schema.instance.transform(current -> current.without(ksm.name));
     }
+    
+    @Test
+    public void testSchemaManagerMocking()
+    {
+        Keyspace.unsetInitialized();
+
+        SchemaUpdateHandler updateHandler = mock(SchemaUpdateHandler.class);
+        Schema schemaManager = new Schema(false, Keyspaces.of(SchemaKeyspace.metadata(), SystemKeyspace.metadata()), updateHandler);
+        assertThat(schemaManager.getKeyspaceMetadata("ks")).isNull();
+
+        KeyspaceMetadata newKs = KeyspaceMetadata.create("ks", KeyspaceParams.simple(1));
+        DistributedSchema before = new DistributedSchema(schemaManager.distributedKeyspaces(), schemaManager.getVersion());
+        DistributedSchema after = new DistributedSchema(schemaManager.distributedKeyspaces().withAddedOrUpdated(newKs), UUID.randomUUID());
+        KeyspacesDiff diff = Keyspaces.diff(before.getKeyspaces(), after.getKeyspaces());
+        SchemaTransformationResult transformation = new SchemaTransformationResult(before, after, diff);
+        schemaManager.mergeAndUpdateVersion(transformation, true);
+
+        assertThat(schemaManager.getKeyspaceMetadata("ks")).isEqualTo(newKs);
+        assertThat(schemaManager.getKeyspaceInstance("ks")).isNull(); // means that we didn't open the keyspace, which is expected since Keyspace is uninitialized
+    }
 
     @Test
     public void testKeyspaceCreationWhenNotInitialized() {
@@ -178,9 +203,9 @@ public void onCreateKeyspace(KeyspaceMetadata keyspace)
 
             // a multi-step schema transformation
             SchemaTransformation transformation = schema -> {
-                schema = schema.withAddedOrReplaced(KeyspaceMetadata.create("test1" + suffix, KeyspaceParams.simple(1)));
-                schema = schema.withAddedOrReplaced(KeyspaceMetadata.create("test2" + suffix, KeyspaceParams.simple(1)));
-                schema = schema.withAddedOrReplaced(KeyspaceMetadata.create("test3" + suffix, KeyspaceParams.simple(1)));
+                schema = schema.withAddedOrUpdated(KeyspaceMetadata.create("test1" + suffix, KeyspaceParams.simple(1)));
+                schema = schema.withAddedOrUpdated(KeyspaceMetadata.create("test2" + suffix, KeyspaceParams.simple(1)));
+                schema = schema.withAddedOrUpdated(KeyspaceMetadata.create("test3" + suffix, KeyspaceParams.simple(1)));
                 return schema;
             };
 
diff --git a/test/unit/org/apache/cassandra/schema/SchemaTestUtil.java b/test/unit/org/apache/cassandra/schema/SchemaTestUtil.java
index b937d15e0c10..5e6533426eec 100644
--- a/test/unit/org/apache/cassandra/schema/SchemaTestUtil.java
+++ b/test/unit/org/apache/cassandra/schema/SchemaTestUtil.java
@@ -21,6 +21,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
+import java.util.UUID;
 
 import org.junit.Assert;
 import org.slf4j.Logger;
@@ -31,6 +32,7 @@
 import org.apache.cassandra.exceptions.AlreadyExistsException;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.concurrent.Future;
 
 import static org.apache.cassandra.net.Verb.SCHEMA_PUSH_REQ;
@@ -41,7 +43,7 @@ public class SchemaTestUtil
 
     public static void announceNewKeyspace(KeyspaceMetadata ksm) throws ConfigurationException
     {
-        ksm.validate();
+        ksm.validate(ClientState.forInternalCalls(ksm.name));
 
         if (Schema.instance.getKeyspaceMetadata(ksm.name) != null)
             throw new AlreadyExistsException(ksm.name);
@@ -72,7 +74,7 @@ else if (throwOnDuplicate && ksm.getTableOrViewNullable(cfm.name) != null)
 
     static void announceKeyspaceUpdate(KeyspaceMetadata ksm)
     {
-        ksm.validate();
+        ksm.validate(ClientState.forInternalCalls(ksm.name));
 
         KeyspaceMetadata oldKsm = Schema.instance.getKeyspaceMetadata(ksm.name);
         if (oldKsm == null)
@@ -143,4 +145,8 @@ public static void mergeAndAnnounceLocally(Collection<Mutation> schemaMutations)
         f.rethrowIfFailed();
     }
 
+    public static UUID calculateSchemaDigest()
+    {
+        return SchemaKeyspace.calculateSchemaDigest();
+    }
 }
diff --git a/test/unit/org/apache/cassandra/schema/TableIdTest.java b/test/unit/org/apache/cassandra/schema/TableIdTest.java
new file mode 100644
index 000000000000..c9778b85449d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/schema/TableIdTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import org.junit.Test;
+
+import org.apache.cassandra.utils.Pair;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class TableIdTest
+{
+    @Test
+    public void testTableNameAndIdFromFilename()
+    {
+        TableId tableId = TableId.generate();
+        Pair<String, TableId> tableNameAndId = TableId.tableNameAndIdFromFilename("cf-" + tableId.toHexString());
+        assertNotNull(tableNameAndId);
+        assertEquals("cf", tableNameAndId.left);
+        assertEquals(tableId, tableNameAndId.right);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/schema/TableMetadataTest.java b/test/unit/org/apache/cassandra/schema/TableMetadataTest.java
index 357ac013cabd..4c4f2c0b940c 100644
--- a/test/unit/org/apache/cassandra/schema/TableMetadataTest.java
+++ b/test/unit/org/apache/cassandra/schema/TableMetadataTest.java
@@ -24,6 +24,7 @@
 
 import org.junit.Test;
 
+import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.marshal.BooleanType;
 import org.apache.cassandra.db.marshal.CompositeType;
@@ -32,8 +33,10 @@
 import org.apache.cassandra.db.marshal.IntegerType;
 import org.apache.cassandra.db.marshal.TupleType;
 import org.apache.cassandra.db.marshal.UTF8Type;
+import org.assertj.core.api.Assertions;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class TableMetadataTest
 {
@@ -137,4 +140,37 @@ public void testPrimaryKeyAsCQLLiteral()
         assertEquals("(2, true)",
                      metadata.primaryKeyAsCQLLiteral(composite.decompose(2, true), Clustering.STATIC_CLUSTERING));
     }
+
+    @Test
+    public void testCdcParamsChangeAffectsPreparedStatements()
+    {
+        String keyspaceName = "ks1";
+        String tableName = "tbl1";
+        TableParams noCdcParams = TableParams.builder().cdc(false).build();
+        TableParams cdcParams = TableParams.builder().cdc(true).build();
+
+        TableMetadata metadata = TableMetadata.builder(keyspaceName, tableName)
+                                              .addPartitionKeyColumn("key", UTF8Type.instance)
+                                              .params(noCdcParams)
+                                              .build();
+        TableMetadata updated = TableMetadata.builder(keyspaceName, tableName)
+                                             .addPartitionKeyColumn("key", UTF8Type.instance)
+                                             .params(cdcParams)
+                                             .build();
+        assertTrue(metadata.changeAffectsPreparedStatements(updated));
+    }
+
+    @Test
+    public void testDroppedColumnsAreRejected()
+    {
+        ColumnMetadata droppedColumn = ColumnMetadata.droppedColumn("ks", "tab",
+                                                                    ColumnIdentifier.getInterned("v", false),
+                                                                    UTF8Type.instance, ColumnMetadata.Kind.REGULAR, null);
+        Assertions.assertThatThrownBy(() -> TableMetadata.builder(droppedColumn.ksName, droppedColumn.ksName)
+                                                         .addPartitionKeyColumn("k", UTF8Type.instance)
+                                                         .addColumn(droppedColumn)
+                                                         .build())
+                  .isInstanceOf(AssertionError.class)
+                  .hasMessageContaining("Invalid columns (contains dropped)");
+    }
 }
diff --git a/test/unit/org/apache/cassandra/schema/TupleTypesRepresentationTest.java b/test/unit/org/apache/cassandra/schema/TupleTypesRepresentationTest.java
index e6daa1faa8c0..c89073d67ec7 100644
--- a/test/unit/org/apache/cassandra/schema/TupleTypesRepresentationTest.java
+++ b/test/unit/org/apache/cassandra/schema/TupleTypesRepresentationTest.java
@@ -35,14 +35,13 @@
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.utils.ByteBufferUtil;
-
-import static org.junit.Assert.assertEquals;
+import org.assertj.core.api.SoftAssertions;
 
 /**
  * Verifies that the string representations of {@link AbstractType} and {@link CQL3Type} are as expected and compatible.
- *
+ * </p>
  * C* 3.0 is known to <em>not</em> enclose a frozen UDT in a "frozen bracket" in the {@link AbstractType}.
- * The string representation of a frozuen UDT using the {@link CQL3Type} type hierarchy is correct in C* 3.0.
+ * The string representation of a frozen UDT using the {@link CQL3Type} type hierarchy is correct in C* 3.0.
  */
 public class TupleTypesRepresentationTest
 {
@@ -113,7 +112,7 @@ static class TypeDef
                                        .prepare(keyspace, types);
             type = cqlType.getType();
 
-            droppedCqlType = CQLFragmentParser.parseAny(CqlParser::comparatorType, droppedCqlTypeString, "dropped type")
+            droppedCqlType = CQLFragmentParser.parseAny(CqlParser::comparatorTypeWithMultiCellTuple, droppedCqlTypeString, "dropped type")
                                               .prepare(keyspace, types);
             // NOTE: TupleType is *always* parsed as frozen, but never toString()'d with the surrounding FrozenType
             droppedType = droppedCqlType.getType();
@@ -123,13 +122,13 @@ static class TypeDef
         public String toString()
         {
             return "TypeDef{\n" +
-                   "typeString='" + typeString + "'\n" +
-                   ", type=" + type + '\n' +
-                   ", cqlTypeString='" + cqlTypeString + "'\n" +
-                   ", cqlType=" + cqlType + '\n' +
-                   ", droppedType=" + droppedType + '\n' +
-                   ", droppedCqlTypeString='" + droppedCqlTypeString + "'\n" +
-                   ", droppedCqlType=" + droppedCqlType + '\n' +
+                   "  typeString='" + typeString + "'\n" +
+                   "  type=" + type + '\n' +
+                   "  cqlTypeString='" + cqlTypeString + "'\n" +
+                   "  cqlType=" + cqlType + '\n' +
+                   "  droppedType=" + droppedType + '\n' +
+                   "  droppedCqlTypeString='" + droppedCqlTypeString + "'\n" +
+                   "  droppedCqlType=" + droppedCqlType.toSchemaString() + '\n' +
                    '}';
         }
     }
@@ -142,19 +141,18 @@ public String toString()
             "'foobar'");
 
     private static final TypeDef tuple_text__text_ = new TypeDef(
-            "org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)",
+            "org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type))",
             "tuple<text, text>",
             "frozen<tuple<text, text>>",
             false,
             "('foo','bar')");
 
-    // Currently, dropped non-frozen-UDT columns are recorded as frozen<tuple<...>>, which is technically wrong
-    //private static final TypeDef mc_udt = new TypeDef(
-    //        "org.apache.cassandra.db.marshal.UserType(ks,6d635f756474,61:org.apache.cassandra.db.marshal.UTF8Type,62:org.apache.cassandra.db.marshal.UTF8Type)",
-    //        "mc_udt",
-    //        "tuple<text, text>",
-    //        true,
-    //        "{a:'foo',b:'bar'}");
+    private static final TypeDef mc_udt = new TypeDef(
+            "org.apache.cassandra.db.marshal.UserType(ks,6d635f756474,61:org.apache.cassandra.db.marshal.UTF8Type,62:org.apache.cassandra.db.marshal.UTF8Type)",
+            "mc_udt",
+            "tuple<text, text>",
+            true,
+            "{a:'foo',b:'bar'}");
 
     private static final TypeDef frozen_f_udt_ = new TypeDef(
             "org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.UserType(ks,665f756474,61:org.apache.cassandra.db.marshal.UTF8Type,62:org.apache.cassandra.db.marshal.UTF8Type))",
@@ -206,9 +204,7 @@ public String toString()
             "{'foo':'bar'}");
 
     private static final TypeDef list_frozen_tuple_text__text___ = new TypeDef(
-            // in consequence, this should be:
-            // "org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
-            "org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type))",
+            "org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
             "list<frozen<tuple<text, text>>>",
             "list<frozen<tuple<text, text>>>",
             true,
@@ -216,15 +212,13 @@ public String toString()
 
     private static final TypeDef frozen_list_tuple_text__text___ = new TypeDef(
             "org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
-            "frozen<list<frozen<tuple<text, text>>>>",
+            "frozen<list<tuple<text, text>>>",
             "frozen<list<frozen<tuple<text, text>>>>",
             true,
             "[('foo','bar')]");
 
     private static final TypeDef set_frozen_tuple_text__text___ = new TypeDef(
-            // in consequence, this should be:
-            // "org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
-            "org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type))",
+            "org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
             "set<frozen<tuple<text, text>>>",
             "set<frozen<tuple<text, text>>>",
             true,
@@ -232,15 +226,13 @@ public String toString()
 
     private static final TypeDef frozen_set_tuple_text__text___ = new TypeDef(
             "org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
-            "frozen<set<frozen<tuple<text, text>>>>",
+            "frozen<set<tuple<text, text>>>",
             "frozen<set<frozen<tuple<text, text>>>>",
             true,
             "{('foo','bar')}");
 
     private static final TypeDef map_text__frozen_tuple_text__text___ = new TypeDef(
-            // in consequence, this should be:
-            // "org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
-            "org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type))",
+            "org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
             "map<text, frozen<tuple<text, text>>>",
             "map<text, frozen<tuple<text, text>>>",
             true,
@@ -248,7 +240,7 @@ public String toString()
 
     private static final TypeDef frozen_map_text__tuple_text__text___ = new TypeDef(
             "org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.TupleType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UTF8Type)))",
-            "frozen<map<text, frozen<tuple<text, text>>>>",
+            "frozen<map<text, tuple<text, text>>>",
             "frozen<map<text, frozen<tuple<text, text>>>>",
             true,
             "{'foobar':('foo','bar')}");
@@ -262,7 +254,7 @@ public String toString()
 
     private static final TypeDef frozen_list_i_udt__ = new TypeDef(
             "org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.UserType(ks,695f756474,61:org.apache.cassandra.db.marshal.UTF8Type,62:org.apache.cassandra.db.marshal.UTF8Type)))",
-            "frozen<list<frozen<i_udt>>>",
+            "frozen<list<i_udt>>",
             "frozen<list<frozen<tuple<text, text>>>>",
             true,
             "[{a:'foo',b:'bar'}]");
@@ -276,7 +268,7 @@ public String toString()
 
     private static final TypeDef frozen_set_i_udt__ = new TypeDef(
             "org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.UserType(ks,695f756474,61:org.apache.cassandra.db.marshal.UTF8Type,62:org.apache.cassandra.db.marshal.UTF8Type)))",
-            "frozen<set<frozen<i_udt>>>",
+            "frozen<set<i_udt>>",
             "frozen<set<frozen<tuple<text, text>>>>",
             true,
             "{{a:'foo',b:'bar'}}");
@@ -290,7 +282,7 @@ public String toString()
 
     private static final TypeDef frozen_map_text__i_udt__ = new TypeDef(
             "org.apache.cassandra.db.marshal.FrozenType(org.apache.cassandra.db.marshal.MapType(org.apache.cassandra.db.marshal.UTF8Type,org.apache.cassandra.db.marshal.UserType(ks,695f756474,61:org.apache.cassandra.db.marshal.UTF8Type,62:org.apache.cassandra.db.marshal.UTF8Type)))",
-            "frozen<map<text, frozen<i_udt>>>",
+            "frozen<map<text, i_udt>>",
             "frozen<map<text, frozen<tuple<text, text>>>>",
             true,
             "{'foobar':{a:'foo',b:'bar'}}");
@@ -298,6 +290,7 @@ public String toString()
     private static final TypeDef[] allTypes = {
             text,
             tuple_text__text_,
+            mc_udt,
             frozen_f_udt_,
             list_text_,
             frozen_list_text__,
@@ -370,34 +363,20 @@ public void generateCqlStatements() throws InterruptedException
     @Test
     public void verifyTypes()
     {
-        AssertionError master = null;
+        SoftAssertions assertions = new SoftAssertions();
         for (TypeDef typeDef : allTypes)
         {
-            try
-            {
-                assertEquals(typeDef.toString() + "\n typeString vs type\n", typeDef.typeString, typeDef.type.toString());
-                assertEquals(typeDef.toString() + "\n typeString vs cqlType.getType()\n", typeDef.typeString, typeDef.cqlType.getType().toString());
-                AbstractType<?> expanded = typeDef.type.expandUserTypes();
-                CQL3Type expandedCQL = expanded.asCQL3Type();
-                // Note: cannot include this commented-out assertion, because the parsed CQL3Type instance for
-                // 'frozen<list<tuple<text, text>>>' returns 'frozen<list<frozen<tuple<text, text>>>>' via it's CQL3Type.toString()
-                // implementation.
-                assertEquals(typeDef.toString() + "\n droppedCqlType\n", typeDef.droppedCqlType, expandedCQL);
-                assertEquals(typeDef.toString() + "\n droppedCqlTypeString\n", typeDef.droppedCqlTypeString, expandedCQL.toString());
-                assertEquals(typeDef.toString() + "\n multiCell\n", typeDef.type.isMultiCell(), typeDef.droppedType.isMultiCell());
-
-                AbstractType<?> parsedType = TypeParser.parse(typeDef.typeString);
-                assertEquals(typeDef.toString(), typeDef.typeString, parsedType.toString());
-            }
-            catch (AssertionError ae)
-            {
-                if (master == null)
-                    master = ae;
-                else
-                    master.addSuppressed(ae);
-            }
+            AbstractType<?> expanded = typeDef.type.expandUserTypes();
+            CQL3Type expandedCQL = expanded.asCQL3Type();
+            AbstractType<?> parsedType = TypeParser.parse(typeDef.typeString);
+
+            assertions.assertThat(typeDef.type.toString()).describedAs("%s, type vs typeString", typeDef).isEqualTo(typeDef.typeString);
+            assertions.assertThat(typeDef.cqlType.getType().toString()).describedAs("%s, cqlType.getType() vs typeString", typeDef).isEqualTo(typeDef.typeString);
+            assertions.assertThat(expandedCQL).describedAs("%s, type.expandUserTypes.asCQL3Type vs droppedCqlType (%s vs %s)", typeDef, expandedCQL.toSchemaString(), typeDef.droppedCqlType.toSchemaString()).isEqualTo(typeDef.droppedCqlType);
+            assertions.assertThat(expandedCQL.toSchemaString()).describedAs("%s, type.expandUserTypes.asCQL3Type.toSchemaString vs droppedCqlTypeString", typeDef).isEqualTo(typeDef.droppedCqlTypeString);
+            assertions.assertThat(typeDef.type.isMultiCell()).describedAs("%s, type.isMultiCell vs droppedType.isMultiCell", typeDef).isEqualTo(typeDef.droppedType.isMultiCell());
+            assertions.assertThat(parsedType.toString()).describedAs("%s, parse(typeString).toString() vs typeString").isEqualTo(typeDef.typeString);
         }
-        if (master != null)
-            throw master;
+        assertions.assertAll();
     }
 }
diff --git a/test/unit/org/apache/cassandra/schema/ValidationTest.java b/test/unit/org/apache/cassandra/schema/ValidationTest.java
index 8eb1247c5b0c..450d28d70ca6 100644
--- a/test/unit/org/apache/cassandra/schema/ValidationTest.java
+++ b/test/unit/org/apache/cassandra/schema/ValidationTest.java
@@ -32,9 +32,9 @@ public class ValidationTest
     @Test
     public void testIsNameValidPositive()
     {
-         assertTrue(SchemaConstants.isValidName("abcdefghijklmnopqrstuvwxyz"));
-         assertTrue(SchemaConstants.isValidName("ABCDEFGHIJKLMNOPQRSTUVWXYZ"));
-         assertTrue(SchemaConstants.isValidName("_01234567890"));
+        assertTrue(SchemaConstants.isValidName("abcdefghijklmnopqrstuvwxyz"));
+        assertTrue(SchemaConstants.isValidName("ABCDEFGHIJKLMNOPQRSTUVWXYZ"));
+        assertTrue(SchemaConstants.isValidName("_01234567890"));
     }
     
     @Test
diff --git a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
index b6e0b8e4b5f8..747ef115e1a3 100644
--- a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
+++ b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
@@ -36,6 +36,7 @@
 import org.apache.commons.io.FileUtils;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,6 +52,7 @@
 import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.io.util.File;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
@@ -74,6 +76,11 @@ public class SSLFactoryTest
 
     private ServerEncryptionOptions encryptionOptions;
 
+    @BeforeClass public static void setupClass()
+    {
+        DISABLE_TCACTIVE_OPENSSL.setBoolean(false);
+    }
+    
     @Before
     public void setup()
     {
@@ -82,7 +89,8 @@ public void setup()
                             .withTrustStore("test/conf/cassandra_ssl_test.truststore")
                             .withTrustStorePassword("cassandra")
                             .withRequireClientAuth(false)
-                            .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA")
+                            .withProtocol("TLSv1.3")
+                            .withCipherSuites("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256")
                             .withSslContextFactory(new ParameterizedClass(TestFileBasedSSLContextFactory.class.getName(),
                                                                           new HashMap<>()));
     }
diff --git a/test/unit/org/apache/cassandra/sensors/ActiveRequestSensorsTest.java b/test/unit/org/apache/cassandra/sensors/ActiveRequestSensorsTest.java
new file mode 100644
index 000000000000..91977ad4cfde
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/ActiveRequestSensorsTest.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Collection;
+import java.util.Optional;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class ActiveRequestSensorsTest
+{
+    private Context context1;
+    private Type type1;
+    private Context context2;
+    private Type type2;
+    private RequestSensors context1Sensors;
+    private RequestSensors context2Sensors;
+    private RequestSensors sensors;
+    private SensorsRegistry sensorsRegistry;
+
+    @Before
+    public void beforeTest()
+    {
+        sensorsRegistry = mock(SensorsRegistry.class);
+
+        context1 = new Context("ks1", "t1", "id1");
+        type1 = Type.READ_BYTES;
+
+        context2 = new Context("ks2", "t2", "id2");
+        type2 = Type.WRITE_BYTES;
+
+        context1Sensors = new ActiveRequestSensors(() -> sensorsRegistry);
+        context2Sensors = new ActiveRequestSensors(() -> sensorsRegistry);
+        sensors = new ActiveRequestSensors(() -> sensorsRegistry);
+    }
+
+    @Test
+    public void testRegistration()
+    {
+        Optional<Sensor> sensor = context1Sensors.getSensor(context1, type1);
+        assertThat(sensor).isEmpty();
+
+        context1Sensors.registerSensor(context1, type1);
+
+        sensor = context1Sensors.getSensor(context1, type1);
+        assertThat(sensor).isPresent();
+
+        context1Sensors.registerSensor(context1, type1);
+        assertThat(context1Sensors.getSensor(context1, type1)).isEqualTo(sensor);
+    }
+
+    @Test
+    public void testRegistrationWithMultipleContexts()
+    {
+        Optional<Sensor> context1Sensor = sensors.getSensor(context1, type1);
+        Optional<Sensor> context2Sensor = sensors.getSensor(context2, type1);
+        assertThat(context1Sensor).isEmpty();
+        assertThat(context2Sensor).isEmpty();
+
+        sensors.registerSensor(context1, type1);
+        sensors.registerSensor(context2, type1);
+
+        context1Sensor = sensors.getSensor(context1, type1);
+        assertThat(context1Sensor).isPresent();
+
+        context2Sensor = sensors.getSensor(context2, type1);
+        assertThat(context2Sensor).isPresent();
+
+        assertThat(context1Sensor).isNotEqualTo(context2Sensor);
+
+        sensors.registerSensor(context1, type1);
+        assertThat(sensors.getSensor(context1, type1)).isEqualTo(context1Sensor);
+
+        sensors.registerSensor(context2, type1);
+        assertThat(sensors.getSensor(context2, type1)).isEqualTo(context2Sensor);
+    }
+
+    @Test
+    public void testRegistrationWithDifferentType()
+    {
+        context1Sensors.registerSensor(context1, type1);
+        context1Sensors.registerSensor(context2, type2);
+
+        assertThat(context1Sensors.getSensor(context1, type1)).isNotEqualTo(context1Sensors.getSensor(context2, type2));
+    }
+
+    @Test
+    public void testRegistrationWithDifferentContext()
+    {
+        context1Sensors.registerSensor(context1, type1);
+        context2Sensors.registerSensor(context2, type1);
+
+        assertThat(context1Sensors.getSensor(context1, type1)).isNotEqualTo(context2Sensors.getSensor(context2, type1));
+    }
+
+    @Test
+    public void testIncrement()
+    {
+        context1Sensors.registerSensor(context1, type1);
+        context1Sensors.incrementSensor(context1, type1, 1.0);
+        assertThat(context1Sensors.getSensor(context1, type1)).hasValueSatisfying((s) -> assertThat(s.getValue()).isEqualTo(1.0));
+    }
+
+    @Test
+    public void testIncrementWithMultipleContexts()
+    {
+        sensors.registerSensor(context1, type1);
+        sensors.incrementSensor(context1, type1, 1.0);
+        sensors.registerSensor(context2, type1);
+        sensors.incrementSensor(context2, type1, 2.0);
+        assertThat(sensors.getSensor(context1, type1)).hasValueSatisfying((s) -> assertThat(s.getValue()).isEqualTo(1.0));
+        assertThat(sensors.getSensor(context2, type1)).hasValueSatisfying((s) -> assertThat(s.getValue()).isEqualTo(2.0));
+    }
+
+    @Test
+    public void testSyncAll()
+    {
+        context1Sensors.registerSensor(context1, type1);
+        context1Sensors.registerSensor(context1, type2);
+
+        context1Sensors.incrementSensor(context1, type1, 1.0);
+        context1Sensors.incrementSensor(context1, type2, 1.0);
+
+        context1Sensors.syncAllSensors();
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type1), eq(1.0));
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type2), eq(1.0));
+
+        // Syncing again doesn't update the sensor
+        context1Sensors.syncAllSensors();
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type1), eq(1.0));
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type2), eq(1.0));
+
+        // Unless updated:
+        context1Sensors.incrementSensor(context1, type1, 1.0);
+        context1Sensors.incrementSensor(context1, type2, 1.0);
+        context1Sensors.syncAllSensors();
+        verify(sensorsRegistry, times(2)).incrementSensor(eq(context1), eq(type1), eq(1.0));
+        verify(sensorsRegistry, times(2)).incrementSensor(eq(context1), eq(type2), eq(1.0));
+    }
+
+    @Test
+    public void testSyncAllWithMultipleContexts()
+    {
+        sensors.registerSensor(context1, type1);
+        sensors.registerSensor(context1, type2);
+        sensors.registerSensor(context2, type1);
+        sensors.registerSensor(context2, type2);
+
+        sensors.incrementSensor(context1, type1, 1.0);
+        sensors.incrementSensor(context1, type2, 1.0);
+        sensors.incrementSensor(context2, type1, 1.0);
+        sensors.incrementSensor(context2, type2, 1.0);
+
+        sensors.syncAllSensors();
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type1), eq(1.0));
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type2), eq(1.0));
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context2), eq(type1), eq(1.0));
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context2), eq(type2), eq(1.0));
+    }
+
+    @Test
+    public void testSyncAllWithOneSensorIncrementing()
+    {
+        sensors.registerSensor(context1, type1);
+        sensors.registerSensor(context1, type2);
+
+        sensors.incrementSensor(context1, type1, 1.0);
+        sensors.incrementSensor(context1, type2, 1.0);
+
+        sensors.syncAllSensors();
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type1), eq(1.0));
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type2), eq(1.0));
+
+        sensors.syncAllSensors();
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type1), eq(1.0));
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type2), eq(1.0));
+
+        // only increment one sensor
+        sensors.incrementSensor(context1, type2, 1.0);
+        sensors.syncAllSensors();
+        verify(sensorsRegistry, times(1)).incrementSensor(eq(context1), eq(type1), eq(1.0));
+        verify(sensorsRegistry, times(2)).incrementSensor(eq(context1), eq(type2), eq(1.0));
+    }
+
+    @Test
+    public void testGetSensors()
+    {
+        sensors.registerSensor(context1, type1);
+        sensors.registerSensor(context1, type2);
+        sensors.registerSensor(context2, type1);
+        sensors.registerSensor(context2, type2);
+
+        Collection<Sensor> type1Sensors = sensors.getSensors(s -> s.getType() == type1);
+        assertThat(type1Sensors).hasSize(2);
+        assertThat(type1Sensors).containsExactlyInAnyOrder(sensors.getSensor(context1, type1).get(), sensors.getSensor(context2, type1).get());
+
+        Collection<Sensor> type2Sensors = sensors.getSensors(s -> s.getType() == type2);
+        assertThat(type2Sensors).hasSize(2);
+        assertThat(type2Sensors).containsExactlyInAnyOrder(sensors.getSensor(context1, type2).get(), sensors.getSensor(context2, type2).get());
+
+        Collection<Sensor> allSensors = sensors.getSensors(s -> true);
+        assertThat(allSensors).hasSize(4);
+        assertThat(allSensors).containsExactlyInAnyOrder(sensors.getSensor(context1, type1).get(), sensors.getSensor(context1, type2).get(),
+                                                         sensors.getSensor(context2, type1).get(), sensors.getSensor(context2, type2).get());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/sensors/ActiveSensorsFactoryTest.java b/test/unit/org/apache/cassandra/sensors/ActiveSensorsFactoryTest.java
new file mode 100644
index 000000000000..f8465e931dd9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/ActiveSensorsFactoryTest.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Optional;
+import java.util.UUID;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ActiveSensorsFactoryTest
+{
+    private final static String KEYSPACE = "ks1";
+    private final static String TABLE = "table1";
+    private ActiveSensorsFactory factory;
+    private SensorEncoder encoder;
+
+    @Before
+    public void before()
+    {
+        factory = new ActiveSensorsFactory();
+        encoder = factory.createSensorEncoder();
+    }
+
+    @Test
+    public void testCreateActiveRequestSensors()
+    {
+        RequestSensors sensors = factory.createRequestSensors(KEYSPACE);
+        assertThat(sensors).isNotNull();
+        assertThat(sensors).isInstanceOf(ActiveRequestSensors.class);
+        RequestSensors anotherSensors = factory.createRequestSensors(KEYSPACE);
+        assertThat(sensors).isNotSameAs(anotherSensors);
+    }
+
+    @Test
+    public void testEncodeTableInReadByteRequestParam()
+    {
+        String expectedParam = String.format("READ_BYTES_REQUEST.%s.%s", KEYSPACE, TABLE);
+        Context context = new Context(KEYSPACE, TABLE, UUID.randomUUID().toString());
+        Sensor sensor = new mockingSensor(context, Type.READ_BYTES);
+        Optional<String> actualParam = encoder.encodeRequestSensorName(sensor);
+        assertThat(actualParam).hasValue(expectedParam);
+    }
+
+    @Test
+    public void testEncodeTableInReadByteGlobalParam()
+    {
+        String expectedParam = String.format("READ_BYTES_GLOBAL.%s.%s", KEYSPACE, TABLE);
+        Context context = new Context(KEYSPACE, TABLE, UUID.randomUUID().toString());
+        Sensor sensor = new mockingSensor(context, Type.READ_BYTES);
+        Optional<String> actualParam = encoder.encodeGlobalSensorName(sensor);
+        assertThat(actualParam).hasValue(expectedParam);
+    }
+
+    @Test
+    public void testEncodeTableInWriteByteRequestParam()
+    {
+        String expectedParam = String.format("WRITE_BYTES_REQUEST.%s.%s", KEYSPACE, TABLE);
+        Context context = new Context(KEYSPACE, TABLE, UUID.randomUUID().toString());
+        Sensor sensor = new mockingSensor(context, Type.WRITE_BYTES);
+        Optional<String> actualParam = encoder.encodeRequestSensorName(sensor);
+        assertThat(actualParam).hasValue(expectedParam);
+    }
+
+    @Test
+    public void testEncodeTableInWriteByteGlobalParam()
+    {
+        String expectedParam = String.format("WRITE_BYTES_GLOBAL.%s.%s", KEYSPACE, TABLE);
+        Context context = new Context(KEYSPACE, TABLE, UUID.randomUUID().toString());
+        Sensor sensor = new mockingSensor(context, Type.WRITE_BYTES);
+        Optional<String> actualParam = encoder.encodeGlobalSensorName(sensor);
+        assertThat(actualParam).hasValue(expectedParam);
+    }
+
+    @Test
+    public void testEncodeTableInIndexWriteBytesRequestParam()
+    {
+        String expectedParam = String.format("INDEX_WRITE_BYTES_REQUEST.%s.%s", KEYSPACE, TABLE);
+        Context context = new Context(KEYSPACE, TABLE, UUID.randomUUID().toString());
+        Sensor sensor = new mockingSensor(context, Type.INDEX_WRITE_BYTES);
+        Optional<String> actualParam = encoder.encodeRequestSensorName(sensor);
+        assertThat(actualParam).hasValue(expectedParam);
+    }
+
+    @Test
+    public void testEncodeTableInIndexWriteBytesGlobalParam()
+    {
+        String expectedParam = String.format("INDEX_WRITE_BYTES_GLOBAL.%s.%s", KEYSPACE, TABLE);
+        Context context = new Context(KEYSPACE, TABLE, UUID.randomUUID().toString());
+        Sensor sensor = new mockingSensor(context, Type.INDEX_WRITE_BYTES);
+        Optional<String> actualParam = encoder.encodeGlobalSensorName(sensor);
+        assertThat(actualParam).hasValue(expectedParam);
+    }
+
+    @Test
+    public void testEncodeTableInInternodeBytesRequestParam()
+    {
+        String expectedParam = String.format("INTERNODE_BYTES_REQUEST.%s.%s", KEYSPACE, TABLE);
+        Context context = new Context(KEYSPACE, TABLE, UUID.randomUUID().toString());
+        Sensor sensor = new mockingSensor(context, Type.INTERNODE_BYTES);
+        Optional<String> actualParam = encoder.encodeRequestSensorName(sensor);
+        assertThat(actualParam).hasValue(expectedParam);
+    }
+
+    @Test
+    public void testEncodeTableInInternodeBytesGlobalParam()
+    {
+        String expectedParam = String.format("INTERNODE_BYTES_GLOBAL.%s.%s", KEYSPACE, TABLE);
+        Context context = new Context(KEYSPACE, TABLE, UUID.randomUUID().toString());
+        Sensor sensor = new mockingSensor(context, Type.INTERNODE_BYTES);
+        Optional<String> actualParam = encoder.encodeGlobalSensorName(sensor);
+        assertThat(actualParam).hasValue(expectedParam);
+    }
+
+    static class mockingSensor extends Sensor
+    {
+        public mockingSensor(Context context, Type type)
+        {
+            super(context, type);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/DisabledSensorsTest.java b/test/unit/org/apache/cassandra/sensors/DisabledSensorsTest.java
new file mode 100644
index 000000000000..25334fee9ba6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/DisabledSensorsTest.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.CounterMutationVerbHandler;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.MutationVerbHandler;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadCommandVerbHandler;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.paxos.Ballot;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.service.paxos.CommitVerbHandler;
+import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler;
+import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE;
+import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests that sensors are not tracked when disabled. Please note that sensor tracking is and opt-in feature so any existing
+ * test that exercise the {@link org.apache.cassandra.net.IVerbHandler#doVerb(Message)} would surface functionality regression (e.g. NPEs).
+ * Here we make sure that sensors are indeed not tracked when disabled and for that it suffices to cover a happy case verb handler invocation scenario.
+ */
+public class DisabledSensorsTest
+{
+    public static final String KEYSPACE1 = "SensorsReadTest";
+    public static final String CF_STANDARD = "Standard";
+    public static final String CF_STANDARD_SAI = "StandardSAI";
+    private static final String CF_COUTNER = "Counter";
+
+    private ColumnFamilyStore store;
+
+    @BeforeClass
+    public static void beforeClass() throws Exception
+    {
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(SensorsTestUtil.NoOpRequestSensorsFactory.class.getName());
+
+        // build SAI indexes
+        Indexes.Builder saiIndexes = Indexes.builder();
+        saiIndexes.add(IndexMetadata.fromSchemaMetadata(CF_STANDARD_SAI + "_val", IndexMetadata.Kind.CUSTOM, new HashMap<>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "val");
+        }}));
+
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD,
+                                                              1, AsciiType.instance, AsciiType.instance, null),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_SAI,
+                                                              1, AsciiType.instance, AsciiType.instance, null)
+                                                .partitioner(Murmur3Partitioner.instance)
+                                                .indexes(saiIndexes.build()),
+                                    SchemaLoader.counterCFMD(KEYSPACE1, CF_COUTNER));
+
+        CompactionManager.instance.disableAutoCompaction();
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+    }
+
+    @After
+    public void afterTest()
+    {
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SAI).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER).truncateBlocking();
+    }
+
+    @Test
+    public void testSensorsForReadVerbHandler()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+
+        new RowUpdateBuilder(store.metadata(), 0, "0")
+        .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+        .build()
+        .applyUnsafe();
+
+        DecoratedKey key = store.getPartitioner().decorateKey(ByteBufferUtil.bytes("4"));
+        ReadCommand command = Util.cmd(store, key).build();
+        handleReadCommand(command);
+
+        assertSensorsAreNotTracked();
+    }
+
+    @Test
+    public void testSensorsForMutationVerbHandler()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+
+        Mutation m = new RowUpdateBuilder(store.metadata(), 0, "0")
+                     .add("val", "0")
+                     .build();
+        handleMutation(m);
+
+        assertSensorsAreNotTracked();
+    }
+
+    @Test
+    public void testSensorsForMutationVerbHandlerWithSAI()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_SAI);
+        Mutation saiMutation = new RowUpdateBuilder(store.metadata(), 0, "0")
+                               .add("val", "hi there")
+                               .build();
+        handleMutation(saiMutation);
+        assertSensorsAreNotTracked();
+    }
+
+    @Test
+    public void testSensorsForCounterMutationVerbHandler() throws WriteTimeoutException, IOException
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_COUTNER);
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER);
+        cfs.truncateBlocking();
+
+        Mutation mutation = new RowUpdateBuilder(cfs.metadata(), 5, "key1")
+                            .clustering("cc")
+                            .add("val", 1L).build();
+
+        CounterMutation counterMutation = new CounterMutation(mutation, ConsistencyLevel.ANY);
+        handleCounterMutation(counterMutation);
+
+        assertSensorsAreNotTracked();
+    }
+
+    @Test
+    public void testLWTSensors()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        PartitionUpdate update = new RowUpdateBuilder(store.metadata(), 0, "0")
+                                 .add("val", "0")
+                                 .buildUpdate();
+        Ballot ballot = nextBallot(NONE);
+        Commit commit = Commit.newPrepare(update.partitionKey(), store.metadata(), ballot);
+        handlePaxosPrepare(commit);
+        handlePaxosPropose(commit);
+        handlePaxosCommit(commit);
+
+        assertSensorsAreNotTracked();
+    }
+
+    private static void handleReadCommand(ReadCommand command)
+    {
+        ReadCommandVerbHandler.instance.doVerb(Message.builder(Verb.READ_REQ, command).build());
+    }
+
+    private static void handleMutation(Mutation mutation)
+    {
+        MutationVerbHandler.instance.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).build());
+    }
+
+    private static void handleCounterMutation(CounterMutation mutation) throws IOException
+    {
+        CounterMutationVerbHandler.instance.doVerb(Message.builder(Verb.COUNTER_MUTATION_REQ, mutation).build());
+    }
+
+    private static void handlePaxosPrepare(Commit prepare)
+    {
+        PrepareVerbHandler.instance.doVerb(Message.builder(Verb.PAXOS_PREPARE_REQ, prepare).build());
+    }
+
+    private static void handlePaxosPropose(Commit proposal)
+    {
+        ProposeVerbHandler.instance.doVerb(Message.builder(Verb.PAXOS_PROPOSE_REQ, proposal).build());
+    }
+
+    private static void handlePaxosCommit(Commit commit)
+    {
+        CommitVerbHandler.instance.doVerb(Message.builder(Verb.PAXOS_COMMIT_REQ, commit).build());
+    }
+
+    private static void assertSensorsAreNotTracked()
+    {
+        assertThat(RequestTracker.instance.get()).isInstanceOf(NoOpRequestSensors.class);
+        for (Type type : Type.values())
+        {
+            assertThat(SensorsRegistry.instance.getSensorsByType(type)).isEmpty();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/ReplicaSensorsTrackingTest.java b/test/unit/org/apache/cassandra/sensors/ReplicaSensorsTrackingTest.java
new file mode 100644
index 000000000000..d040153e6c32
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/ReplicaSensorsTrackingTest.java
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.Predicates;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlans;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.ResponseVerbHandler;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.AbstractWriteResponseHandler;
+import org.apache.cassandra.service.QueryInfoTracker;
+import org.apache.cassandra.service.paxos.Ballot;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.service.paxos.PrepareResponse;
+import org.apache.cassandra.service.paxos.v1.PrepareCallback;
+import org.apache.cassandra.service.paxos.v1.ProposeCallback;
+import org.apache.cassandra.service.reads.DigestResolver;
+import org.apache.cassandra.service.reads.ReadCallback;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+import org.mockito.Mockito;
+
+import static org.apache.cassandra.locator.ReplicaUtils.full;
+import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE;
+import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Tests to verify that sensors reported from replicas in {@link Message.Header#customParams()} are tracked correctly
+ * in the {@link RequestSensors} of the request.
+ */
+@RunWith(BMUnitRunner.class)
+public class ReplicaSensorsTrackingTest
+{
+    static Keyspace ks;
+    static ColumnFamilyStore cfs;
+    static EndpointsForToken targets;
+    static EndpointsForToken pending;
+    static Token dummy;
+    /**
+     * Used by byteman to signal that onResponse is about to be called for one of the replica responses. This enables
+     * unit tests to start asserting that replica sensors are already tracked at this point
+     */
+    static CountDownLatch[] onResponseAboutToStartSignal;
+    /**
+     * Signalled by units tests once after sensor tracking assertions are done to make sure onResponse is not returned
+     * before assertions are completed
+     */
+    static CountDownLatch[] onResponseStartSignal;
+    static AtomicInteger responses = new AtomicInteger(0);
+
+    @BeforeClass
+    public static void beforeClass() throws Exception
+    {
+
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(ActiveSensorsFactory.class.getName());
+        CassandraRelevantProperties.SENSORS_VIA_NATIVE_PROTOCOL.setBoolean(true);
+
+        SchemaLoader.loadSchema();
+        SchemaLoader.createKeyspace("Foo", KeyspaceParams.simple(3), SchemaLoader.standardCFMD("Foo", "Bar"));
+        ks = Keyspace.open("Foo");
+        cfs = ks.getColumnFamilyStore("Bar");
+        dummy = Murmur3Partitioner.instance.getMinimumToken();
+        targets = EndpointsForToken.of(dummy,
+                                       full(InetAddressAndPort.getByName("127.0.0.255")),
+                                       full(InetAddressAndPort.getByName("127.0.0.254")),
+                                       full(InetAddressAndPort.getByName("127.0.0.253"))
+        );
+        pending = EndpointsForToken.empty(DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0)));
+        cfs.sampleReadLatencyMicros = 0;
+    }
+
+    @Before
+    public void before()
+    {
+        onResponseAboutToStartSignal = new CountDownLatch[targets.size()];
+        onResponseStartSignal = new CountDownLatch[targets.size()];
+        for (int i = 0; i < targets.size(); i++)
+        {
+            onResponseAboutToStartSignal[i] = new CountDownLatch(1);
+            onResponseStartSignal[i] = new CountDownLatch(1);
+        }
+        responses.set(0);
+    }
+
+    @After
+    public void after()
+    {
+        // just in case the test failed and the latches were not counted down
+        for (int i = 0; i < targets.size(); i++)
+        {
+            onResponseAboutToStartSignal[i].countDown();
+            onResponseStartSignal[i].countDown();
+        }
+    }
+
+    @Test
+    @BMRule(name = "signals onResponse about to start latches",
+    targetClass = "org.apache.cassandra.service.reads.ReadCallback",
+    targetMethod = "onResponse",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.sensors.ReplicaSensorsTrackingTest.countDownAndAwaitOnResponseLatches();")
+    public void testSensorsTrackedForReadCallback() throws InterruptedException
+    {
+        DecoratedKey key = cfs.getPartitioner().decorateKey(ByteBufferUtil.bytes("4"));
+        ReadCommand command = Util.cmd(cfs, key).build();
+        Message<ReadCommand> readRequest = Message.builder(Verb.READ_REQ, command).build();
+
+        // init request sensors, must happen before the callback is created
+        RequestSensors requestSensors = new ActiveRequestSensors();
+        Context context = Context.from(command);
+        requestSensors.registerSensor(context, Type.READ_BYTES);
+        Sensor actualReadSensor = requestSensors.getSensor(context, Type.READ_BYTES).get();
+        RequestTracker.instance.set(requestSensors);
+
+        // init callback
+        ReplicaPlan.SharedForTokenRead plan = plan(ConsistencyLevel.ONE, targets);
+        final Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        final DigestResolver<EndpointsForToken, ReplicaPlan.ForTokenRead> resolver = new DigestResolver<>(command, plan, requestTime, QueryInfoTracker.ReadTracker.NOOP);
+        final ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> callback = new ReadCallback<>(resolver, command, plan, requestTime);
+
+        // mimic a sensor to be used in replica response
+        Sensor mockingReadSensor = new mockingSensor(context, Type.READ_BYTES);
+        mockingReadSensor.increment(11.0);
+
+        assertReplicaSensorsTracked(readRequest, callback, Pair.create(actualReadSensor, mockingReadSensor));
+    }
+
+    @Test
+    @BMRule(name = "signals onResponse about to start latches",
+    targetClass = "org.apache.cassandra.service.WriteResponseHandler",
+    targetMethod = "onResponse",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.sensors.ReplicaSensorsTrackingTest.countDownAndAwaitOnResponseLatches();")
+    public void testSensorsTrackedForWriteCallback_HintsEnabled() throws InterruptedException
+    {
+        boolean allowHints = true;
+        Mutation mutation = new RowUpdateBuilder(cfs.metadata(), 0, "0").build();
+        Message<Mutation> writeRequest = Message.builder(Verb.MUTATION_REQ, mutation).build();
+        assertSensorsTrackedForWriteRequest(writeRequest, allowHints);
+    }
+
+    @Test
+    @BMRule(name = "signals onResponse about to start latches",
+    targetClass = "org.apache.cassandra.service.WriteResponseHandler",
+    targetMethod = "onResponse",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.sensors.ReplicaSensorsTrackingTest.countDownAndAwaitOnResponseLatches();")
+    public void testSensorsTrackedForWriteCallback_HintsDisabled() throws InterruptedException
+    {
+        boolean allowHints = false;
+        Mutation mutation = new RowUpdateBuilder(cfs.metadata(), 0, "0").build();
+        Message<Mutation> writeRequest = Message.builder(Verb.MUTATION_REQ, mutation).build();
+        assertSensorsTrackedForWriteRequest(writeRequest, allowHints);
+    }
+
+    @Test
+    @BMRule(name = "signals onResponse about to start latches",
+    targetClass = "org.apache.cassandra.service.WriteResponseHandler",
+    targetMethod = "onResponse",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.sensors.ReplicaSensorsTrackingTest.countDownAndAwaitOnResponseLatches();")
+    public void testSensorsTrackedForWriteCallback_CounterMutation() throws InterruptedException
+    {
+        Mutation mutation = new RowUpdateBuilder(cfs.metadata(), 0, "0").build();
+        CounterMutation counterMutation = new CounterMutation(mutation, ConsistencyLevel.ALL);
+        Message<CounterMutation> writeRequest = Message.builder(Verb.COUNTER_MUTATION_REQ, counterMutation).build();
+        boolean allowHints = false;
+        assertSensorsTrackedForWriteRequest(writeRequest, allowHints);
+    }
+
+    @Test
+    @BMRule(name = "signals onResponse about to start latches",
+    targetClass = "org.apache.cassandra.service.WriteResponseHandler",
+    targetMethod = "onResponse",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.sensors.ReplicaSensorsTrackingTest.countDownAndAwaitOnResponseLatches();")
+    public void testSensorsTrackedForWriteCallback_HintsEnabled_PaxosCommit() throws InterruptedException
+    {
+        Commit commit = Commit.emptyCommit(cfs.getPartitioner().decorateKey(ByteBufferUtil.bytes("0")), cfs.metadata());
+        Message<Commit> writeRequest = Message.builder(Verb.PAXOS_COMMIT_REQ, commit).build();
+        boolean allowHints = true;
+        assertSensorsTrackedForWriteRequest(writeRequest, allowHints);
+    }
+
+    @Test
+    @BMRule(name = "signals onResponse about to start latches",
+    targetClass = "org.apache.cassandra.service.WriteResponseHandler",
+    targetMethod = "onResponse",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.sensors.ReplicaSensorsTrackingTest.countDownAndAwaitOnResponseLatches();")
+    public void testSensorsTrackedForWriteCallback_HintsDisabled_PaxosCommit() throws InterruptedException
+    {
+        Commit commit = Commit.emptyCommit(cfs.getPartitioner().decorateKey(ByteBufferUtil.bytes("0")), cfs.metadata());
+        Message<Commit> writeRequest = Message.builder(Verb.PAXOS_COMMIT_REQ, commit).build();
+        boolean allowHints = false;
+        assertSensorsTrackedForWriteRequest(writeRequest, allowHints);
+    }
+
+    @Test
+    @BMRule(name = "signals onResponse about to start latches",
+    targetClass = "org.apache.cassandra.service.paxos.v1.PrepareCallback",
+    targetMethod = "onResponse",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.sensors.ReplicaSensorsTrackingTest.countDownAndAwaitOnResponseLatches();")
+    public void testSensorsTrackedForPaxosPrepareCallback() throws InterruptedException
+    {
+        Mutation mutation = new RowUpdateBuilder(cfs.metadata(), 0, "0").build();
+        Message<Mutation> prepare = Message.builder(Verb.PAXOS_PREPARE_REQ, mutation).build();
+
+        // init request sensors, must happen before the callback is created
+        RequestSensors requestSensors = new ActiveRequestSensors();
+        Context context = Context.from(cfs.metadata());
+        requestSensors.registerSensor(context, Type.WRITE_BYTES);
+        requestSensors.registerSensor(context, Type.READ_BYTES);
+        Sensor actualWriteSensor = requestSensors.getSensor(context, Type.WRITE_BYTES).get();
+        Sensor actualReadSensor = requestSensors.getSensor(context, Type.READ_BYTES).get();
+        RequestTracker.instance.set(requestSensors);
+
+        // init prepare callback
+        DecoratedKey key = cfs.getPartitioner().decorateKey(ByteBufferUtil.bytes("0"));
+        PrepareCallback callback = new PrepareCallback(key, cfs.metadata(), targets.size(), ConsistencyLevel.ALL, Dispatcher.RequestTime.forImmediateExecution());
+
+        Sensor mockingPrepareWriteSensor = new mockingSensor(context, Type.WRITE_BYTES);
+        mockingPrepareWriteSensor.increment(13.0);
+        Sensor mockingPrepareReadSensor = new mockingSensor(context, Type.READ_BYTES);
+        mockingPrepareReadSensor.increment(14.0);
+        Pair<Sensor, Sensor> prepareWriterSensors = Pair.create(actualWriteSensor, mockingPrepareWriteSensor);
+        Pair<Sensor, Sensor> prepareReadSensors = Pair.create(actualReadSensor, mockingPrepareReadSensor);
+
+        assertReplicaSensorsTracked(prepare, callback, prepareWriterSensors, prepareReadSensors);
+    }
+
+    @Test
+    @BMRule(name = "signals onResponse about to start latches",
+    targetClass = "org.apache.cassandra.service.paxos.v1.ProposeCallback",
+    targetMethod = "onResponse",
+    targetLocation = "AT ENTRY",
+    action = "org.apache.cassandra.sensors.ReplicaSensorsTrackingTest.countDownAndAwaitOnResponseLatches();")
+    public void testSensorsTrackedForPaxosProposeCallback() throws InterruptedException
+    {
+        Mutation mutation = new RowUpdateBuilder(cfs.metadata(), 0, "0").build();
+        Message<Mutation> propose = Message.builder(Verb.PAXOS_PROPOSE_REQ, mutation).build();
+
+        // init request sensors, must happen before the callback is created
+        RequestSensors requestSensors = new ActiveRequestSensors();
+        Context context = Context.from(cfs.metadata());
+        requestSensors.registerSensor(context, Type.WRITE_BYTES);
+        requestSensors.registerSensor(context, Type.READ_BYTES);
+        Sensor actualWriteSensor = requestSensors.getSensor(context, Type.WRITE_BYTES).get();
+        Sensor actualReadSensor = requestSensors.getSensor(context, Type.READ_BYTES).get();
+        RequestTracker.instance.set(requestSensors);
+
+        // init propose callback
+        ProposeCallback callback = new ProposeCallback(cfs.metadata(), targets.size(), targets.size(), false, ConsistencyLevel.ALL, Dispatcher.RequestTime.forImmediateExecution());
+
+        Sensor mockingProposeWriteSensor = new mockingSensor(context, Type.WRITE_BYTES);
+        mockingProposeWriteSensor.increment(15.0);
+        Sensor mockingProposeReadSensor = new mockingSensor(context, Type.READ_BYTES);
+        mockingProposeReadSensor.increment(16.0);
+        Pair<Sensor, Sensor> proposeWriterSensors = Pair.create(actualWriteSensor, mockingProposeWriteSensor);
+        Pair<Sensor, Sensor> proposeReadSensors = Pair.create(actualReadSensor, mockingProposeReadSensor);
+
+        assertReplicaSensorsTracked(propose, callback, proposeWriterSensors, proposeReadSensors);
+    }
+
+    /**
+     * Used by Byteman to count down the onResponseAboutToStartSignal latch and await the onResponseStartSignal latch
+     * for the current replica response.
+     */
+    public static void countDownAndAwaitOnResponseLatches() throws InterruptedException
+    {
+        int replica = responses.getAndIncrement();
+        onResponseAboutToStartSignal[replica].countDown();
+        // don't wait indefinitely if the test is stuck.
+        assertThat(onResponseStartSignal[replica].await(1, TimeUnit.SECONDS)).isTrue();
+    }
+
+    private void assertSensorsTrackedForWriteRequest(Message writeRequest, boolean allowHints) throws InterruptedException
+    {
+        // init request sensors, must happen before the callback is created
+        RequestSensors requestSensors = new ActiveRequestSensors();
+        Context context = Context.from(cfs.metadata());
+        requestSensors.registerSensor(context, Type.WRITE_BYTES);
+        Sensor actualWriteSensor = requestSensors.getSensor(context, Type.WRITE_BYTES).get();
+        RequestTracker.instance.set(requestSensors);
+
+        // init callback
+        AbstractWriteResponseHandler<?> callback = createWriteResponseHandler(ConsistencyLevel.ALL, ConsistencyLevel.ALL);
+
+        // mimic a sensor to be used in replica response
+        Sensor mockingWriteSensor = new mockingSensor(context, Type.WRITE_BYTES);
+        mockingWriteSensor.increment(13.0);
+
+        assertReplicaSensorsTracked(writeRequest, callback, allowHints, Pair.create(actualWriteSensor, mockingWriteSensor));
+    }
+
+    @SafeVarargs
+    private void assertReplicaSensorsTracked(Message<?> request, RequestCallback<?> callback, Pair<Sensor, Sensor>... trackingToReplicaSensors) throws InterruptedException
+    {
+        assertReplicaSensorsTracked(request, callback, false, trackingToReplicaSensors);
+    }
+
+    @SafeVarargs
+    private void assertReplicaSensorsTracked(Message<?> request, RequestCallback<?> callback, boolean allowHints, Pair<Sensor, Sensor>... trackingToReplicaSensors) throws InterruptedException
+    {
+        for (Pair<Sensor, Sensor> pair : trackingToReplicaSensors)
+        {
+            Sensor trackingSensor = pair.left;
+            Sensor replicaSensor = pair.right;
+            assertThat(trackingSensor.getValue()).isZero();
+            assertThat(replicaSensor.getValue()).isGreaterThan(0);
+        }
+
+        // sensors should be incremented with each response
+        for (int responses = 1; responses <= targets.size(); responses++)
+        {
+            simulateResponseFromReplica(targets.get(responses - 1), request, callback, allowHints, Arrays.stream(trackingToReplicaSensors).map(Pair::right).toArray(Sensor[]::new));
+            // don't wait indefinitely if the test is stuck. Delay the assertion of the await results to give a better change of a meaningful error by virtue of the core test assertion
+            boolean awaitResult = onResponseAboutToStartSignal[responses - 1].await(1, TimeUnit.SECONDS);
+            for (Pair<Sensor, Sensor> pair : trackingToReplicaSensors)
+            {
+                Sensor trackingSensor = pair.left;
+                Sensor replicaSensor = pair.right;
+                assertThat(trackingSensor.getValue()).isEqualTo(replicaSensor.getValue() * responses);
+                assertThat(awaitResult).isTrue();
+            }
+            onResponseStartSignal[responses - 1].countDown();
+        }
+
+        // reset tracking sensors for next assertions, if any
+        for (Pair<Sensor, Sensor> pair : trackingToReplicaSensors)
+        {
+            Sensor trackingSensor = pair.left;
+            trackingSensor.reset();
+        }
+    }
+
+    private void simulateResponseFromReplica(Replica replica, Message<?> request, RequestCallback<?> callback, boolean allowHints, Sensor... sensor)
+    {
+        new Thread(() -> {
+            // AbstractWriteResponseHandler has a special handling for the callback
+            if (callback instanceof AbstractWriteResponseHandler)
+                MessagingService.instance().callbacks.addWithExpiration((AbstractWriteResponseHandler<?>) callback, request, replica);
+            else
+                MessagingService.instance().callbacks.addWithExpiration(callback, request, replica.endpoint());
+            Message<?> response = createResponseMessageWithSensor(request.verb(), replica.endpoint(), request.id(), sensor);
+            ResponseVerbHandler.instance.doVerb(response);
+        }).start();
+    }
+
+    private ReplicaPlan.SharedForTokenRead plan(ConsistencyLevel consistencyLevel, EndpointsForToken replicas)
+    {
+        return ReplicaPlan.shared(new ReplicaPlan.ForTokenRead(ks, ks.getReplicationStrategy(), consistencyLevel, replicas, replicas));
+    }
+
+    private Message<?> createResponseMessageWithSensor(Verb requestVerb, InetAddressAndPort from, long id, Sensor... sensors)
+    {
+        if (requestVerb == Verb.READ_REQ)
+            return createReadResponseMessage(from, id, sensors[0]);
+        else if (requestVerb == Verb.MUTATION_REQ)
+            return createResponseMessage(Verb.MUTATION_RSP, NoPayload.noPayload, from, id, sensors);
+        else if (requestVerb == Verb.COUNTER_MUTATION_REQ)
+            return createResponseMessage(Verb.COUNTER_MUTATION_RSP, NoPayload.noPayload, from, id, sensors);
+        else if (requestVerb == Verb.PAXOS_PREPARE_REQ)
+        {
+            DecoratedKey key = cfs.getPartitioner().decorateKey(ByteBufferUtil.bytes("4"));
+            Ballot ballot = nextBallot(NONE);
+            Commit commit = Commit.newPrepare(key, cfs.metadata(), ballot);
+            return createResponseMessage(Verb.PAXOS_PREPARE_RSP, new PrepareResponse(false, commit, commit), from, id, sensors);
+        }
+        else if (requestVerb == Verb.PAXOS_PROPOSE_REQ)
+            return createResponseMessage(Verb.PAXOS_PROPOSE_RSP, true, from, id, sensors);
+        else if (requestVerb == Verb.PAXOS_COMMIT_REQ)
+            return createResponseMessage(Verb.PAXOS_COMMIT_RSP, NoPayload.noPayload, from, id, sensors);
+        else
+            throw new IllegalArgumentException("Unsupported verb: " + requestVerb);
+    }
+
+    private Message<ReadResponse> createReadResponseMessage(InetAddressAndPort from, long id, Sensor readSensor)
+    {
+        ReadResponse response = new ReadResponse()
+        {
+            @Override
+            public UnfilteredPartitionIterator makeIterator(ReadCommand command)
+            {
+                UnfilteredPartitionIterator iterator = Mockito.mock(UnfilteredPartitionIterator.class);
+                Mockito.when(iterator.metadata()).thenReturn(command.metadata());
+                return iterator;
+            }
+
+            @Override
+            public ByteBuffer digest(ReadCommand command)
+            {
+                return null;
+            }
+
+            @Override
+            public ByteBuffer repairedDataDigest()
+            {
+                return null;
+            }
+
+            @Override
+            public boolean isRepairedDigestConclusive()
+            {
+                return false;
+            }
+
+            @Override
+            public boolean mayIncludeRepairedDigest()
+            {
+                return false;
+            }
+
+            @Override
+            public boolean isDigestResponse()
+            {
+                return false;
+            }
+        };
+
+        return Message.builder(Verb.READ_RSP, response)
+                      .from(from)
+                      .withId(id)
+                      .withCustomParam(SensorsCustomParams.paramForRequestSensor(readSensor).get(), SensorsCustomParams.sensorValueAsBytes(readSensor.getValue()))
+                      .build();
+    }
+
+    private <T> Message<T> createResponseMessage(Verb responseVerb, T payload, InetAddressAndPort from, long id, Sensor... sensors)
+    {
+        Message.Builder<T> builder = Message.builder(responseVerb, payload)
+                                            .from(from)
+                                            .withId(id);
+
+        for (Sensor sensor : sensors)
+            builder.withCustomParam(SensorsCustomParams.paramForRequestSensor(sensor).get(), SensorsCustomParams.sensorValueAsBytes(sensor.getValue()));
+
+        return builder.build();
+    }
+
+    private static AbstractWriteResponseHandler<?> createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal)
+    {
+        return createWriteResponseHandler(cl, ideal, Dispatcher.RequestTime.forImmediateExecution());
+    }
+
+    private static AbstractWriteResponseHandler<?> createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal, Dispatcher.RequestTime requestTime)
+    {
+        return ks.getReplicationStrategy().getWriteResponseHandler(ReplicaPlans.forWrite(ks, cl, targets, pending, Predicates.alwaysTrue(), ReplicaPlans.writeAll),
+                                                                   null, WriteType.SIMPLE, null, requestTime, ideal);
+    }
+
+    static class mockingSensor extends Sensor
+    {
+        public mockingSensor(Context context, Type type)
+        {
+            super(context, type);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/RequestSensorsFactoryTest.java b/test/unit/org/apache/cassandra/sensors/RequestSensorsFactoryTest.java
new file mode 100644
index 000000000000..55a877a6873a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/RequestSensorsFactoryTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+public class RequestSensorsFactoryTest
+{
+    @Test
+    public void testCreateUsesNoOpByDefault()
+    {
+        SensorsFactory factory = SensorsFactory.instance;
+        RequestSensors sensors = factory.createRequestSensors("ks1");
+        assertThat(sensors).isInstanceOf(NoOpRequestSensors.class);
+        RequestSensors anotherSensors = factory.createRequestSensors("k2");
+        assertThat(anotherSensors).isSameAs(sensors);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsCustomParamsWithActiveSensorsFactoryTest.java b/test/unit/org/apache/cassandra/sensors/SensorsCustomParamsWithActiveSensorsFactoryTest.java
new file mode 100644
index 000000000000..d0cdbdc03254
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsCustomParamsWithActiveSensorsFactoryTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class SensorsCustomParamsWithActiveSensorsFactoryTest
+{
+    @BeforeClass
+    public static void setUpClass() throws Exception
+    {
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(ActiveSensorsFactory.class.getName());
+        CassandraRelevantProperties.SENSORS_VIA_NATIVE_PROTOCOL.setBoolean(true);
+
+        // enables constructing Messages with custom parameters
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setCrossNodeTimeout(true);
+    }
+
+    @Test
+    public void testSensorValueAsBytes()
+    {
+        double d = Double.MAX_VALUE;
+        byte[] bytes = SensorsCustomParams.sensorValueAsBytes(d);
+        ByteBuffer bb = ByteBuffer.wrap(bytes);
+        assertEquals(Double.MAX_VALUE, bb.getDouble(), 0.0);
+    }
+
+    @Test
+    public void testSensorValueFromBytes()
+    {
+        ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES);
+        buffer.putDouble(Double.MAX_VALUE);
+        double d = SensorsCustomParams.sensorValueFromBytes(buffer.array());
+        assertEquals(Double.MAX_VALUE, d, 0.0);
+    }
+
+    @Test
+    public void testAddWriteSensorToInternodeResponse()
+    {
+        testAddSensorsToInternodeResponse(Type.WRITE_BYTES);
+    }
+
+    @Test
+    public void testAddReadSensorToInternodeResponse()
+    {
+        testAddSensorsToInternodeResponse(Type.READ_BYTES);
+    }
+
+    @Test
+    public void testSensorValueAsByteBuffer()
+    {
+        double d = Double.MAX_VALUE;
+        ByteBuffer bb = SensorsCustomParams.sensorValueAsByteBuffer(d);
+        // bb should already be flipped
+        assertEquals(bb.position(), 0);
+        assertEquals(d, ByteBufferUtil.toDouble(bb), 0.0);
+    }
+
+    @Test
+    public void testAddSensorsToCQLResponse()
+    {
+        String table = "t1";
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors("ks1");
+        ResultMessage message = new ResultMessage.Void();
+        Context context = new Context("ks1", table, UUID.randomUUID().toString());
+        Type type = Type.WRITE_BYTES;
+        double expectedValue = 17.0;
+
+        sensors.registerSensor(context, type);
+        sensors.incrementSensor(context, type, expectedValue);
+
+        SensorsCustomParams.addSensorToCQLResponse(message, ProtocolVersion.V4, sensors, context, type);
+
+        assertNotNull(message.getCustomPayload());
+
+        Sensor sensor = sensors.getSensor(context, type).get();
+        String expectedHeader = SensorsCustomParams.paramForRequestSensor(sensor).get();
+        assertTrue(message.getCustomPayload().containsKey(expectedHeader));
+        assertEquals(expectedValue, message.getCustomPayload().get(expectedHeader).getDouble(), 0.0);
+    }
+
+    @Test
+    public void testAddSensorsToCQLResponseWithExistingCustomPayload()
+    {
+        String table = "t1";
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors("ks1");
+        ResultMessage message = new ResultMessage.Void();
+        String existingKey = "existingKey";
+        String existingValue = "existingValue";
+        Map<String, ByteBuffer> customPayload = new HashMap<>();
+        customPayload.put(existingKey, ByteBuffer.wrap(existingValue.getBytes(StandardCharsets.UTF_8)));
+        message.setCustomPayload(customPayload);
+        Context context = new Context("ks1", table, UUID.randomUUID().toString());
+        Type type = Type.READ_BYTES;
+        double expectedValue = 13.0;
+
+        sensors.registerSensor(context, type);
+        sensors.incrementSensor(context, type, expectedValue);
+
+        SensorsCustomParams.addSensorToCQLResponse(message, ProtocolVersion.V4, sensors, context, type);
+
+        assertNotNull(message.getCustomPayload());
+        assertEquals( 2, message.getCustomPayload().size());
+
+        Sensor sensor = sensors.getSensor(context, type).get();
+        String expectedHeader = SensorsCustomParams.paramForRequestSensor(sensor).get();
+        assertTrue(message.getCustomPayload().containsKey(expectedHeader));
+        assertEquals(expectedValue, message.getCustomPayload().get(expectedHeader).getDouble(), 0.0);
+
+        assertTrue(message.getCustomPayload().containsKey(existingKey));
+        assertEquals(existingValue, StandardCharsets.UTF_8.decode(message.getCustomPayload().get(existingKey)).toString());
+    }
+
+   @Test
+    public void testAddSensorsToCQLResponseSkipped()
+    {
+        String table = "t1";
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors("ks1");
+        ResultMessage message = new ResultMessage.Void();
+        Context context = new Context("ks1", table, UUID.randomUUID().toString());
+        Type type = Type.WRITE_BYTES;
+        double expectedValue = 17.0;
+
+        sensors.registerSensor(context, type);
+        sensors.incrementSensor(context, type, expectedValue);
+
+        SensorsCustomParams.addSensorToCQLResponse(null, ProtocolVersion.V4, sensors, context, type);
+        SensorsCustomParams.addSensorToCQLResponse(message, ProtocolVersion.V4, null, context, type);
+        SensorsCustomParams.addSensorToCQLResponse(message, ProtocolVersion.V3, null, context, type);
+        CassandraRelevantProperties.SENSORS_VIA_NATIVE_PROTOCOL.setBoolean(false);
+        SensorsCustomParams.addSensorToCQLResponse(message, ProtocolVersion.V4, sensors, context, type);
+
+        assertNull(message.getCustomPayload());
+    }
+
+    private void testAddSensorsToInternodeResponse(Type sensorType)
+    {
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors("ks1");
+        UUID tableId = UUID.randomUUID();
+        KeyspaceMetadata ksm = KeyspaceMetadata.create("ks1", null);
+        TableMetadata tm = TableMetadata.builder("ks1", "t1", TableId.fromString(tableId.toString()))
+                                        .addPartitionKeyColumn("pk", AsciiType.instance)
+                                        .build();
+        SensorsRegistry.instance.onCreateKeyspace(ksm);
+        SensorsRegistry.instance.onCreateTable(tm);
+
+        Context context = new Context("ks1", "t1", tableId.toString());
+        sensors.registerSensor(context, sensorType);
+        sensors.incrementSensor(context, sensorType, 17.0);
+        sensors.syncAllSensors();
+
+        Message.Builder<NoPayload> builder =
+        Message.builder(Verb._TEST_1, noPayload)
+               .withId(1);
+
+        SensorsCustomParams.addSensorsToInternodeResponse(sensors, builder);
+
+        Message<NoPayload> msg = builder.build();
+        assertNotNull(msg.header.customParams());
+        assertEquals(2, msg.header.customParams().size());
+        Sensor sensor = sensors.getSensor(context, sensorType).get();
+        String requestParam = SensorsCustomParams.paramForRequestSensor(sensor).get();
+        String globalParam = SensorsCustomParams.paramForGlobalSensor(sensor).get();
+        assertTrue(msg.header.customParams().containsKey(requestParam));
+        assertTrue(msg.header.customParams().containsKey(globalParam));
+        double epsilon = 0.000001;
+        assertEquals(17.0, SensorsCustomParams.sensorValueFromBytes(msg.header.customParams().get(requestParam)), epsilon);
+        assertEquals(17.0, SensorsCustomParams.sensorValueFromBytes(msg.header.customParams().get(globalParam)), epsilon);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsCustomParamsWithDefaultSensorsFactoryTest.java b/test/unit/org/apache/cassandra/sensors/SensorsCustomParamsWithDefaultSensorsFactoryTest.java
new file mode 100644
index 000000000000..4655facaabbe
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsCustomParamsWithDefaultSensorsFactoryTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.UUID;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static org.apache.cassandra.net.NoPayload.noPayload;
+import static org.junit.Assert.assertNull;
+
+public class SensorsCustomParamsWithDefaultSensorsFactoryTest
+{
+    @BeforeClass
+    public static void setUpClass() throws Exception
+    {
+        // falls back to default SensorsFactory
+        CassandraRelevantProperties.SENSORS_FACTORY.reset();
+        CassandraRelevantProperties.SENSORS_VIA_NATIVE_PROTOCOL.setBoolean(true);
+
+        // enables constructing Messages with custom parameters
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void testAddSensorsToInternodeResponse()
+    {
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors("ks1");
+        UUID tableId = UUID.randomUUID();
+        KeyspaceMetadata ksm = KeyspaceMetadata.create("ks1", null);
+        TableMetadata tm = TableMetadata.builder("ks1", "t1", TableId.fromString(tableId.toString()))
+                                        .addPartitionKeyColumn("pk", AsciiType.instance)
+                                        .build();
+        SensorsRegistry.instance.onCreateKeyspace(ksm);
+        SensorsRegistry.instance.onCreateTable(tm);
+
+        Context context = new Context("ks1", "t1", tableId.toString());
+        sensors.registerSensor(context, Type.WRITE_BYTES);
+        sensors.incrementSensor(context, Type.WRITE_BYTES, 17.0);
+        sensors.syncAllSensors();
+
+        Message.Builder<NoPayload> builder =
+        Message.builder(Verb._TEST_1, noPayload)
+               .withId(1);
+
+        SensorsCustomParams.addSensorsToInternodeResponse(sensors, builder);
+
+        Message<NoPayload> msg = builder.build();
+        assertNull(msg.header.customParams());
+    }
+
+    @Test
+    public void testAddSensorToCQLResponse()
+    {
+        String table = "t1";
+        RequestSensors sensors = SensorsFactory.instance.createRequestSensors("ks1");
+        ResultMessage message = new ResultMessage.Void();
+        Context context = new Context("ks1", table, UUID.randomUUID().toString());
+        Type type = Type.READ_BYTES;
+        sensors.registerSensor(context, type);
+        sensors.incrementSensor(context, type, 13.0);
+
+        SensorsCustomParams.addSensorToCQLResponse(message, ProtocolVersion.V4, sensors, context, type);
+
+        assertNull(message.getCustomPayload());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsFactoryTest.java b/test/unit/org/apache/cassandra/sensors/SensorsFactoryTest.java
new file mode 100644
index 000000000000..e65a08fec704
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsFactoryTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SensorsFactoryTest
+{
+    @Test
+    public void testDefaultRequestSensors()
+    {
+        SensorsFactory factory = SensorsFactory.instance;
+        RequestSensors sensors = factory.createRequestSensors("ks1", "ks2");
+
+        assertThat(sensors).isInstanceOf(NoOpRequestSensors.class);
+        assertThat(factory.createRequestSensors("ks1", "ks2")).isSameAs(sensors);
+    }
+
+    @Test
+    public void testDefaultSensorEncoder()
+    {
+        SensorsFactory factory = SensorsFactory.instance;
+        SensorEncoder encoder = factory.createSensorEncoder();
+        Sensor sensor = new Sensor(new Context("ks1", "t1", "id1"), Type.READ_BYTES);
+
+        assertThat(encoder.encodeRequestSensorName(sensor)).isEmpty();
+        assertThat(encoder.encodeGlobalSensorName(sensor)).isEmpty();
+        assertThat(encoder).isSameAs(factory.createSensorEncoder());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsIndexWriteTest.java b/test/unit/org/apache/cassandra/sensors/SensorsIndexWriteTest.java
new file mode 100644
index 000000000000..a36c31a600af
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsIndexWriteTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.MutationVerbHandler;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SensorsIndexWriteTest
+{
+    public static final String KEYSPACE1 = "SensorsIndexWriteTest";
+    public static final String CF_STANDARD = "Standard";
+    public static final String CF_STANDARD_SAI = "StandardSAI";
+    public static final String CF_STANDARD_SECONDARY_INDEX = "StandardSecondaryIndex";
+
+    private CopyOnWriteArrayList<Message> capturedOutboundMessages;
+
+    @BeforeClass
+    public static void defineSchema() throws Exception
+    {
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(ActiveSensorsFactory.class.getName());
+
+        SchemaLoader.prepareServer();
+
+        // build SAI indexes
+        Indexes.Builder saiIndexes = Indexes.builder();
+        saiIndexes.add(IndexMetadata.fromSchemaMetadata(CF_STANDARD_SAI + "_val", IndexMetadata.Kind.CUSTOM, new HashMap<>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "val");
+        }}));
+
+        // build secondary indexes
+        Indexes.Builder secondaryIndexes = Indexes.builder();
+        IndexTarget indexTarget = new IndexTarget(new ColumnIdentifier("val", true), IndexTarget.Type.VALUES);
+        secondaryIndexes.add(IndexMetadata.fromIndexTargets(Collections.singletonList(indexTarget),
+                                                            CF_STANDARD_SECONDARY_INDEX + "_val",
+                                                            IndexMetadata.Kind.COMPOSITES,
+                                                            Collections.emptyMap()));
+
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD,
+                                                              1, AsciiType.instance, AsciiType.instance, null),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_SAI,
+                                                              1, AsciiType.instance, AsciiType.instance, null)
+                                                .partitioner(Murmur3Partitioner.instance) // supported by SAI
+                                                .indexes(saiIndexes.build()),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_SECONDARY_INDEX,
+                                                              1, AsciiType.instance, AsciiType.instance, null)
+                                                .indexes(secondaryIndexes.build()));
+
+        CompactionManager.instance.disableAutoCompaction();
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+    }
+
+    @Before
+    public void beforeTest()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE1).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SAI).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SECONDARY_INDEX).metadata());
+
+        capturedOutboundMessages = new CopyOnWriteArrayList<>();
+        MessagingService.instance().outboundSink.add((message, to) ->
+                                                     {
+                                                         capturedOutboundMessages.add(message);
+                                                         return false;
+                                                     });
+    }
+
+    @After
+    public void afterTest()
+    {
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SAI).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SECONDARY_INDEX).truncateBlocking();
+
+        RequestTracker.instance.set(null);
+        SensorsRegistry.instance.clear();
+
+        CassandraRelevantProperties.BF_RECREATE_ON_FP_CHANCE_CHANGE.setBoolean(false);
+    }
+
+    @Test
+    public void testSingleRowMutationWithSAI()
+    {
+        ColumnFamilyStore standardStore = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context standardContext = new Context(KEYSPACE1, CF_STANDARD, standardStore.metadata.id.toString());
+
+        ColumnFamilyStore saiStore = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_SAI);
+        Context saiContext = new Context(KEYSPACE1, CF_STANDARD_SAI, saiStore.metadata.id.toString());
+
+        String partitionKey = "0";
+        Mutation standardMutation = new RowUpdateBuilder(standardStore.metadata(), 0, partitionKey)
+                                    .add("val", "hi there")
+                                    .build();
+        handleMutation(standardMutation);
+
+        Sensor standardSensor = SensorsTestUtil.getThreadLocalRequestSensor(standardContext, Type.WRITE_BYTES);
+        assertThat(standardSensor.getValue()).isGreaterThan(0);
+        Sensor standardRegistrySensor = SensorsTestUtil.getRegistrySensor(standardContext, Type.WRITE_BYTES);
+        assertThat(standardRegistrySensor).isEqualTo(standardSensor);
+
+        // check global registry is synchronized for Standard table
+        assertThat(standardRegistrySensor.getValue()).isEqualTo(standardSensor.getValue());
+        String writeRequestParam = SensorsCustomParams.paramForRequestSensor(standardSensor).get();
+        String writeGlobalParam = SensorsCustomParams.paramForRequestSensor(standardRegistrySensor).get();
+        assertResponseSensors(standardSensor.getValue(), standardRegistrySensor.getValue(), writeRequestParam, writeGlobalParam);
+
+        Mutation saiMutation = new RowUpdateBuilder(saiStore.metadata(), 0, partitionKey)
+                               .add("val", "hi there")
+                               .build();
+        handleMutation(saiMutation);
+
+        Sensor saiSensor = SensorsTestUtil.getThreadLocalRequestSensor(saiContext, Type.INDEX_WRITE_BYTES);
+        // Writing the same amount of data to an SAI indexed column should generate at least the same number of bytes (the SAI write >= the vanilla write bytes)
+        assertThat(saiSensor.getValue()).isGreaterThanOrEqualTo(standardSensor.getValue());
+        Sensor saiRegistrySensor = SensorsTestUtil.getRegistrySensor(saiContext, Type.INDEX_WRITE_BYTES);
+        assertThat(saiRegistrySensor).isEqualTo(saiSensor);
+
+        // check global registry is synchronized for SAI table
+        assertThat(saiRegistrySensor.getValue()).isEqualTo(saiSensor.getValue());
+        String requestParam = SensorsCustomParams.paramForRequestSensor(saiSensor).get();
+        String globalParam = SensorsCustomParams.paramForGlobalSensor(saiRegistrySensor).get();
+        assertResponseSensors(saiSensor.getValue(), saiRegistrySensor.getValue(), requestParam, globalParam);
+    }
+
+    @Test
+    public void testSingleRowMutationWithSecondaryIndex()
+    {
+        ColumnFamilyStore standardStore = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context standardContext = new Context(KEYSPACE1, CF_STANDARD, standardStore.metadata.id.toString());
+
+        ColumnFamilyStore secondaryIndexStore = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_SECONDARY_INDEX);
+        Context secondaryIndexContext = new Context(KEYSPACE1, CF_STANDARD_SECONDARY_INDEX, secondaryIndexStore.metadata.id.toString());
+
+        String partitionKey = "0";
+        Mutation standardMutation = new RowUpdateBuilder(standardStore.metadata(), 0, partitionKey)
+                                    .add("val", "hi there")
+                                    .build();
+        handleMutation(standardMutation);
+
+        Sensor standardSensor = SensorsTestUtil.getThreadLocalRequestSensor(standardContext, Type.WRITE_BYTES);
+        assertThat(standardSensor.getValue()).isGreaterThan(0);
+        Sensor standardRegistrySensor = SensorsTestUtil.getRegistrySensor(standardContext, Type.WRITE_BYTES);
+        assertThat(standardRegistrySensor).isEqualTo(standardSensor);
+
+        // check global registry is synchronized for Standard table
+        assertThat(standardRegistrySensor.getValue()).isEqualTo(standardSensor.getValue());
+        String writeRequestParam = SensorsCustomParams.paramForRequestSensor(standardSensor).get();
+        String writeGlobalParam = SensorsCustomParams.paramForGlobalSensor(standardRegistrySensor).get();
+        assertResponseSensors(standardSensor.getValue(), standardRegistrySensor.getValue(), writeRequestParam, writeGlobalParam);
+
+        Mutation secondaryIndexMutation = new RowUpdateBuilder(secondaryIndexStore.metadata(), 0, partitionKey)
+                                          .add("val", "hi there")
+                                          .build();
+        handleMutation(secondaryIndexMutation);
+
+        Sensor secondaryIndexSensor = SensorsTestUtil.getThreadLocalRequestSensor(secondaryIndexContext, Type.INDEX_WRITE_BYTES);
+        // We are not guaranteed that the amount of data we write to the secondary index is more than what we write to the main file,
+        // and we are not tracking it very precisely. It should, though, at least include the cell data and deletions which is about
+        // half the standard write size.
+        assertThat(secondaryIndexSensor.getValue()).isGreaterThanOrEqualTo(standardSensor.getValue() / 2);
+        Sensor secondaryIndexRegistrySensor = SensorsTestUtil.getRegistrySensor(secondaryIndexContext, Type.INDEX_WRITE_BYTES);
+        assertThat(secondaryIndexRegistrySensor).isEqualTo(secondaryIndexSensor);
+        // Check that we also get the correct vanilla write bytes for this operation.
+        assertThat(SensorsTestUtil.getThreadLocalRequestSensor(secondaryIndexContext, Type.WRITE_BYTES).getValue()).isEqualTo(standardSensor.getValue());
+
+        // check global registry is synchronized for Secondary Index table
+        assertThat(secondaryIndexRegistrySensor.getValue()).isEqualTo(secondaryIndexSensor.getValue());
+        String indexRequestParam = SensorsCustomParams.paramForRequestSensor(secondaryIndexSensor).get();
+        String indexGlobalParam = SensorsCustomParams.paramForGlobalSensor(secondaryIndexRegistrySensor).get();
+        assertResponseSensors(secondaryIndexSensor.getValue(), secondaryIndexRegistrySensor.getValue(), indexRequestParam, indexGlobalParam);
+    }
+
+    private static void handleMutation(Mutation mutation)
+    {
+        MutationVerbHandler.instance.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).build());
+    }
+
+    private void assertResponseSensors(double requestValue, double registryValue, String requestParam, String globalParam)
+    {
+        // verify against the last message to enable testing of multiple mutations in a for loop
+        Message message = capturedOutboundMessages.get(capturedOutboundMessages.size() - 1);
+        assertResponseSensors(message, requestValue, registryValue, requestParam, globalParam);
+
+        // make sure messages with sensor values can be deserialized on the receiving node
+        DataOutputBuffer out = SensorsTestUtil.serialize(message);
+        Message deserializedMessage = SensorsTestUtil.deserialize(out, message.from());
+        assertResponseSensors(deserializedMessage, requestValue, registryValue, requestParam, globalParam);
+    }
+
+    private void assertResponseSensors(Message message, double requestValue, double registryValue, String expectedRequestParam, String expectedGlobalParam)
+    {
+        assertThat(message.header.customParams()).isNotNull();
+        assertThat(message.header.customParams()).containsKey(expectedRequestParam);
+        assertThat(message.header.customParams()).containsKey(expectedGlobalParam);
+
+        double requestBytes = SensorsTestUtil.bytesToDouble(message.header.customParams().get(expectedRequestParam));
+        double globalBytes = SensorsTestUtil.bytesToDouble(message.header.customParams().get(expectedGlobalParam));
+        assertThat(requestBytes).isEqualTo(requestValue);
+        assertThat(globalBytes).isEqualTo(registryValue);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsInternodeTest.java b/test/unit/org/apache/cassandra/sensors/SensorsInternodeTest.java
new file mode 100644
index 000000000000..ecded44cdb7f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsInternodeTest.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.BiPredicate;
+import java.util.function.Supplier;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.CounterMutationVerbHandler;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.MutationVerbHandler;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadCommandVerbHandler;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.paxos.Ballot;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.service.paxos.CommitVerbHandler;
+import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler;
+import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE;
+import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+public class SensorsInternodeTest
+{
+    private static final String KEYSPACE1 = "SensorsInternodeTest";
+    private static final String CF_STANDARD = "Standard";
+    private static final String CF_STANDARD2 = "Standard2";
+
+    private static final String CF_COUTNER = "Counter";
+
+    private ColumnFamilyStore store;
+    private CopyOnWriteArrayList<Message> capturedOutboundMessages;
+    private BiPredicate<Message<?>, InetAddressAndPort> outboundSinkHandler;
+
+    @BeforeClass
+    public static void beforeClass() throws Exception
+    {
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(ActiveSensorsFactory.class.getName());
+
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD,
+                                                              1, AsciiType.instance, AsciiType.instance, null),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2,
+                                                              1, AsciiType.instance, AsciiType.instance, null),
+                                    SchemaLoader.counterCFMD(KEYSPACE1, CF_COUTNER)
+        );
+
+        CompactionManager.instance.disableAutoCompaction();
+        MessagingService.instance().listen();
+    }
+
+    @Before
+    public void beforeTest()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE1).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER).metadata());
+
+        capturedOutboundMessages = new CopyOnWriteArrayList<>();
+        outboundSinkHandler = (message, to) -> capturedOutboundMessages.add(message);
+        MessagingService.instance().outboundSink.add(outboundSinkHandler);
+    }
+
+    @After
+    public void afterTest()
+    {
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER).truncateBlocking();
+
+        RequestTracker.instance.set(null);
+        SensorsRegistry.instance.clear();
+        MessagingService.instance().outboundSink.remove(outboundSinkHandler);
+    }
+
+    @Test
+    public void testInternodeSensorsForRead()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+
+        new RowUpdateBuilder(store.metadata(), 0, "0")
+        .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+        .build()
+        .applyUnsafe();
+
+        DecoratedKey key = store.getPartitioner().decorateKey(ByteBufferUtil.bytes("0"));
+        ReadCommand command = Util.cmd(store, key).build();
+        Message request = Message.builder(Verb.READ_REQ, command).build();
+        Runnable handler = () -> ReadCommandVerbHandler.instance.doVerb(request);
+        testInternodeSensors(request, handler, ImmutableSet.of(context));
+    }
+
+    @Test
+    public void testInternodeSensorsForMutation()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+
+        Mutation mutation = new RowUpdateBuilder(store.metadata(), 0, "0")
+                            .add("val", "0")
+                            .build();
+
+        Message request = Message.builder(Verb.MUTATION_REQ, mutation).build();
+        Runnable handler = () -> MutationVerbHandler.instance.doVerb(request);
+        testInternodeSensors(request, handler, ImmutableSet.of(context));
+    }
+
+    @Test
+    public void testInternodeSensorsForBatchMutation()
+    {
+        ColumnFamilyStore store1 = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context1 = new Context(KEYSPACE1, CF_STANDARD, store1.metadata.id.toString());
+
+        ColumnFamilyStore store2 = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD2);
+        Context context2 = new Context(KEYSPACE1, CF_STANDARD2, store2.metadata.id.toString());
+
+        List<Mutation> mutations = new ArrayList<>();
+        String partitionKey = "0";
+
+        // first table mutation
+        mutations.add(new RowUpdateBuilder(store1.metadata(), 0, partitionKey)
+                      .add("val", "value")
+                      .build());
+
+        // second table mutation
+        mutations.add(new RowUpdateBuilder(store2.metadata(), 0, partitionKey)
+                      .add("val", "another value")
+                      .build());
+
+        Mutation mutation = Mutation.merge(mutations);
+        Message request = Message.builder(Verb.MUTATION_REQ, mutation).build();
+        Runnable handler = () -> MutationVerbHandler.instance.doVerb(request);
+        testInternodeSensors(request, handler, ImmutableSet.of(context1, context2));
+    }
+
+    @Test
+    public void testInternodeSensorsForCounterMutation()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_COUTNER);
+        Context context = new Context(KEYSPACE1, CF_COUTNER, store.metadata.id.toString());
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER);
+        cfs.truncateBlocking();
+
+        Mutation mutation = new RowUpdateBuilder(cfs.metadata(), 5, "key1")
+                            .clustering("cc")
+                            .add("val", 1L).build();
+
+        CounterMutation counterMutation = new CounterMutation(mutation, ConsistencyLevel.ANY);
+
+        Message request = Message.builder(Verb.COUNTER_MUTATION_REQ, counterMutation).build();
+        Runnable handler = () -> {
+            try
+            {
+                CounterMutationVerbHandler.instance.doVerb(request);
+            }
+            catch (IOException e)
+            {
+                fail("Failed to handle counter mutation", e);
+            }
+        };
+        testInternodeSensors(request, handler, ImmutableSet.of(context));
+    }
+
+    @Test
+    public void testInternodeSensorsForLWTPrepare()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+        PartitionUpdate update = new RowUpdateBuilder(store.metadata(), 0, "0")
+                                 .add("val", "0")
+                                 .buildUpdate();
+        Ballot ballot = nextBallot(NONE);
+        Commit proposal = Commit.newPrepare(update.partitionKey(), store.metadata(), ballot);
+        Message request = Message.builder(Verb.PAXOS_PREPARE_REQ, proposal).build();
+        Runnable handler = () -> PrepareVerbHandler.instance.doVerb(request);
+        testInternodeSensors(request, handler, ImmutableSet.of(context));
+    }
+
+    @Test
+    public void testInternodeSensorsForLWTPropose()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+        PartitionUpdate update = new RowUpdateBuilder(store.metadata(), 0, "0")
+                                 .add("val", "0")
+                                 .buildUpdate();
+        Ballot ballot = nextBallot(NONE);
+        Commit proposal = Commit.newProposal(ballot, update);
+        Message request = Message.builder(Verb.PAXOS_PROPOSE_REQ, proposal).build();
+        Runnable handler = () -> ProposeVerbHandler.instance.doVerb(request);
+        testInternodeSensors(request, handler, ImmutableSet.of(context));
+    }
+
+    @Test
+    public void testInternodeSensorsForLWTCommit()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+        PartitionUpdate update = new RowUpdateBuilder(store.metadata(), 0, "0")
+                                 .add("val", "0")
+                                 .buildUpdate();
+        Ballot ballot = nextBallot(NONE);
+        Commit proposal = Commit.newPrepare(update.partitionKey(), store.metadata(), ballot);
+        Message request = Message.builder(Verb.PAXOS_COMMIT_REQ, proposal).build();
+        Runnable handler = () -> CommitVerbHandler.instance.doVerb(request);
+        testInternodeSensors(request, handler, ImmutableSet.of(context));
+    }
+
+    private void testInternodeSensors(Message request, Runnable handler, Collection<Context> contexts)
+    {
+        // Run the handler:
+        handler.run();
+
+        // Get the request size, response size and total size per table:
+        int tableCount = contexts.size();
+        int requestSizePerTable = request.payloadSize(MessagingService.current_version) / tableCount;
+        Message response = capturedOutboundMessages.get(capturedOutboundMessages.size() - 1);
+        int responseSizePerTable = response.payloadSize(MessagingService.current_version) / tableCount;
+        int total = requestSizePerTable + responseSizePerTable;
+
+        // For each context/table, get the internode bytes and verify their value is between the request and total size:
+        // it can't be equal to the total size because we don't record the custom headers in the internode sensor.
+        for (Context context : contexts)
+        {
+            Sensor internodeBytesSensor = SensorsRegistry.instance.getSensor(context, Type.INTERNODE_BYTES).get();
+            double internodeBytes = internodeBytesSensor.getValue();
+            assertThat(internodeBytes).isBetween(requestSizePerTable * 1.0, total * 1.0);
+
+            // assert internode headers are added to the response messages
+            Supplier<Optional<String>> requestParamSupplier = () -> SensorsCustomParams.paramForRequestSensor(internodeBytesSensor);
+            Supplier<Optional<String>> globalParamSupplier = () -> SensorsCustomParams.paramForGlobalSensor(internodeBytesSensor);
+            assertResponseSensors(response, total, total, requestParamSupplier, globalParamSupplier);
+        }
+    }
+
+    private void assertResponseSensors(Message message, double requestValue, double registryValue, Supplier<Optional<String>> requestParamSupplier, Supplier<Optional<String>> globalParamSupplier)
+    {
+        Optional<String> expectedRequestParam = requestParamSupplier.get();
+        Optional<String> expectedGlobalParam = globalParamSupplier.get();
+        assertThat(message.header.customParams()).isNotNull();
+        assertThat(expectedRequestParam).isPresent();
+        assertThat(expectedGlobalParam).isPresent();
+
+        String requestParam = expectedRequestParam.get();
+        String globalParam = expectedGlobalParam.get();
+        assertThat(message.header.customParams()).containsKey(requestParam);
+        assertThat(message.header.customParams()).containsKey(globalParam);
+
+        double requestWriteBytes = SensorsTestUtil.bytesToDouble(message.header.customParams().get(requestParam));
+        double globalWriteBytes = SensorsTestUtil.bytesToDouble(message.header.customParams().get(globalParam));
+        assertThat(requestWriteBytes).isEqualTo(requestValue);
+        assertThat(globalWriteBytes).isEqualTo(registryValue);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsReadTest.java b/test/unit/org/apache/cassandra/sensors/SensorsReadTest.java
new file mode 100644
index 000000000000..7a8b9af68a71
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsReadTest.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.sensors;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadCommandVerbHandler;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.BF_RECREATE_ON_FP_CHANCE_CHANGE;
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SensorsReadTest
+{
+    public static final String KEYSPACE1 = "SensorsReadTest";
+    public static final String CF_STANDARD = "Standard";
+    public static final String CF_STANDARD_CLUSTERING = "StandardClustering";
+    public static final String CF_STANDARD_SAI = "StandardSAI";
+    public static final String CF_STANDARD_SECONDARY_INDEX = "StandardSecondaryIndex";
+
+    private ColumnFamilyStore store;
+    private CopyOnWriteArrayList<Message> capturedOutboundMessages;
+
+    @BeforeClass
+    public static void defineSchema() throws Exception
+    {
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(ActiveSensorsFactory.class.getName());
+
+        // build SAI indexes
+        Indexes.Builder saiIndexes = Indexes.builder();
+        saiIndexes.add(IndexMetadata.fromSchemaMetadata(CF_STANDARD_SAI + "_val", IndexMetadata.Kind.CUSTOM, new HashMap<>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "val");
+        }}));
+
+        // build secondary indexes
+        Indexes.Builder secondaryIndexes = Indexes.builder();
+        IndexTarget indexTarget = new IndexTarget(new ColumnIdentifier("val", true), IndexTarget.Type.VALUES);
+        secondaryIndexes.add(IndexMetadata.fromIndexTargets(Collections.singletonList(indexTarget),
+                                                            CF_STANDARD_SECONDARY_INDEX + "_val",
+                                                            IndexMetadata.Kind.COMPOSITES,
+                                                            Collections.emptyMap()));
+
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD,
+                                                              1, AsciiType.instance, AsciiType.instance, null),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_CLUSTERING,
+                                                              1, AsciiType.instance, AsciiType.instance, AsciiType.instance),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_SAI,
+                                                              1, AsciiType.instance, LongType.instance, null)
+                                                .partitioner(Murmur3Partitioner.instance) // supported by SAI
+                                                .indexes(saiIndexes.build()),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_SECONDARY_INDEX,
+                                                              1, AsciiType.instance, LongType.instance, null)
+                                                .indexes(secondaryIndexes.build()));
+
+        CompactionManager.instance.disableAutoCompaction();
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+    }
+
+    @Before
+    public void beforeTest()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE1).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_CLUSTERING).metadata());
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE1).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SAI).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SECONDARY_INDEX).metadata());
+
+        capturedOutboundMessages = new CopyOnWriteArrayList<>();
+        MessagingService.instance().outboundSink.add((message, to) ->
+                                                     {
+                                                         capturedOutboundMessages.add(message);
+                                                         return false;
+                                                     });
+    }
+
+    @After
+    public void afterTest()
+    {
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_CLUSTERING).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SAI).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_SECONDARY_INDEX).truncateBlocking();
+
+        RequestTracker.instance.set(null);
+        SensorsRegistry.instance.clear();
+
+        BF_RECREATE_ON_FP_CHANCE_CHANGE.setBoolean(false);
+    }
+
+    @Test
+    public void testMemtableRead()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+
+        for (int j = 0; j < 10; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+            .build()
+            .applyUnsafe();
+        }
+
+        DecoratedKey key = store.getPartitioner().decorateKey(ByteBufferUtil.bytes("4"));
+        ReadCommand command = Util.cmd(store, key).build();
+        handleReadCommand(command);
+
+        assertRequestAndRegistrySensorsEquality(context);
+        Sensor requestSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertResponseSensors(requestSensor, requestSensor);
+    }
+
+    @Test
+    public void testSinglePartitionReadCommand_ByPartitionKey()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+
+        for (int j = 0; j < 10; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+            .build()
+            .applyUnsafe();
+        }
+
+        store.forceBlockingFlush(UNIT_TESTS);
+
+        SSTableReader sstable = store.getLiveSSTables().iterator().next();
+
+        DecoratedKey key = sstable.decorateKey(ByteBufferUtil.bytes("4"));
+        ReadCommand command = Util.cmd(store, key).build();
+        handleReadCommand(command);
+
+        assertRequestAndRegistrySensorsEquality(context);
+        Sensor requestSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertResponseSensors(requestSensor, requestSensor);
+    }
+
+    @Test
+    public void testSinglePartitionReadCommand_ByClustering()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_CLUSTERING);
+        Context context = new Context(KEYSPACE1, CF_STANDARD_CLUSTERING, store.metadata.id.toString());
+
+        for (int j = 0; j < 10; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), j, "0")
+            .clustering(String.valueOf(j))
+            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+            .build()
+            .applyUnsafe();
+        }
+
+        store.forceBlockingFlush(UNIT_TESTS);
+
+        SSTableReader sstable = store.getLiveSSTables().iterator().next();
+
+        DecoratedKey key = sstable.decorateKey(ByteBufferUtil.bytes("0"));
+        ReadCommand command = Util.cmd(store, key).includeRow("0").build();
+        handleReadCommand(command);
+
+        assertRequestAndRegistrySensorsEquality(context);
+
+        Sensor requestSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertResponseSensors(requestSensor, requestSensor);
+    }
+
+    @Test
+    public void testSinglePartitionReadCommand_AllowFiltering()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_CLUSTERING);
+        Context context = new Context(KEYSPACE1, CF_STANDARD_CLUSTERING, store.metadata.id.toString());
+
+        for (int j = 0; j < 10; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), j, "0")
+            .clustering(String.valueOf(j))
+            .add("val", String.valueOf(j))
+            .build()
+            .applyUnsafe();
+        }
+
+        store.forceBlockingFlush(UNIT_TESTS);
+
+        SSTableReader sstable = store.getLiveSSTables().iterator().next();
+
+        DecoratedKey key = sstable.decorateKey(ByteBufferUtil.bytes("0"));
+        ReadCommand command1 = Util.cmd(store, key).includeRow("0").build();
+        handleReadCommand(command1);
+
+        Sensor request1Sensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        // Extract the value as later we will reset the thread local and the sensor value will be lost
+        long request1Bytes = (long) request1Sensor.getValue();
+
+        assertThat(request1Sensor.getValue()).isGreaterThan(0);
+        assertThat(request1Sensor).isEqualTo(SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES));
+        assertResponseSensors(request1Sensor, SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES));
+
+        SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES).reset();
+        capturedOutboundMessages.clear();
+
+        ReadCommand command2 = Util.cmd(store, key).filterOn("val", Operator.EQ, "9").build();
+        handleReadCommand(command2);
+
+        Sensor request2Sensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertThat(request2Sensor.getValue()).isEqualTo(request1Bytes * 10);
+        assertThat(SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES).getValue()).isEqualTo(request1Bytes + request2Sensor.getValue());
+        assertResponseSensors(request2Sensor, SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES));
+    }
+
+    @Test
+    public void testPartitionRangeReadCommand_ByPartitionKey()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+
+        for (int j = 0; j < 10; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+            .build()
+            .applyUnsafe();
+        }
+
+        store.forceBlockingFlush(UNIT_TESTS);
+
+        SSTableReader sstable = store.getLiveSSTables().iterator().next();
+
+        DecoratedKey key = sstable.decorateKey(ByteBufferUtil.bytes("0"));
+        ReadCommand command1 = Util.cmd(store, key).build();
+        handleReadCommand(command1);
+
+        Sensor request1Sensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        // Extract the value as later we will reset the thread local and the sensor value will be lost
+        long request1Bytes = (long) request1Sensor.getValue();
+
+        assertThat(request1Sensor.getValue()).isGreaterThan(0);
+        assertThat(request1Sensor).isEqualTo(SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES));
+        assertResponseSensors(request1Sensor, SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES));
+
+        SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES).reset();
+        capturedOutboundMessages.clear();
+
+        ReadCommand command2 = Util.cmd(store).fromKeyIncl("0").toKeyIncl("9").build();
+        handleReadCommand(command2);
+
+        Sensor request2Sensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertThat(request2Sensor.getValue()).isEqualTo(request1Bytes * 10);
+        assertThat(SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES).getValue()).isEqualTo(request1Bytes + request2Sensor.getValue());
+        assertResponseSensors(request2Sensor, SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES));
+    }
+
+    @Test
+    public void testSAIIndexScan()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_SAI);
+        Context context = new Context(KEYSPACE1, CF_STANDARD_SAI, store.metadata.id.toString());
+
+        for (int j = 0; j < 10; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+            .add("val", (long) j)
+            .build()
+            .applyUnsafe();
+        }
+
+        ReadCommand readCommand = Util.cmd(store)
+                                      .columns("val")
+                                      .filterOn("val", Operator.GT, 0L)
+                                      .build();
+
+        handleReadCommand(readCommand);
+
+        assertRequestAndRegistrySensorsEquality(context);
+
+        Sensor requestSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertResponseSensors(requestSensor, requestSensor);
+    }
+
+    @Test
+    public void testSAISingleRowSearchVSIndexScan()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_SAI);
+        Context context = new Context(KEYSPACE1, CF_STANDARD_SAI, store.metadata.id.toString());
+
+        int numRows = 10;
+        for (int j = 0; j < numRows; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+            .add("val", (long) j)
+            .build()
+            .applyUnsafe();
+        }
+
+        // Match a single row
+        ReadCommand readCommand = Util.cmd(store)
+                                      .columns("val")
+                                      .filterOn("val", Operator.EQ, 0L)
+                                      .build();
+        handleReadCommand(readCommand);
+
+        // Store the request sensor value for comparison with full index scan
+        Sensor indexReadSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        double singleRowSearchBytes = indexReadSensor.getValue();
+        indexReadSensor.reset();
+
+        // Scan the whole index
+        readCommand = Util.cmd(store)
+                          .columns("val")
+                          .filterOn("val", Operator.GTE, 0L)
+                          .build();
+        handleReadCommand(readCommand);
+
+        double fullIndexScanBytes = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES).getValue();
+        assertThat(fullIndexScanBytes).isEqualTo(numRows * singleRowSearchBytes);
+    }
+
+    @Test
+    public void testSecondayIndexSingleRow()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_SECONDARY_INDEX);
+        Context context = new Context(KEYSPACE1, CF_STANDARD_SECONDARY_INDEX, store.metadata.id.toString());
+
+        for (int j = 0; j < 10; j++)
+        {
+            new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+            .add("val", (long) j)
+            .build()
+            .applyUnsafe();
+        }
+
+        ReadCommand readCommand = Util.cmd(store)
+                                      .fromKeyIncl("0").toKeyIncl("10")
+                                      .columns("val")
+                                      .filterOn("val", Operator.EQ, 1L) // only EQ is supported by CassandraIndex
+                                      .build();
+
+        handleReadCommand(readCommand);
+
+        assertRequestAndRegistrySensorsEquality(context);
+
+        Sensor requestSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertResponseSensors(requestSensor, requestSensor);
+    }
+
+    private static void handleReadCommand(ReadCommand command)
+    {
+        ReadCommandVerbHandler.instance.doVerb(Message.builder(Verb.READ_REQ, command).build());
+    }
+
+    private void assertRequestAndRegistrySensorsEquality(Context context)
+    {
+        Sensor localSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertThat(localSensor.getValue()).isGreaterThan(0);
+
+        Sensor registrySensor = SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES);
+        assertThat(registrySensor).isEqualTo(localSensor);
+    }
+
+    private void assertResponseSensors(Sensor requestSensor, Sensor registrySensor)
+    {
+        assertThat(capturedOutboundMessages).hasSize(1);
+        Message message = capturedOutboundMessages.get(0);
+        assertResponseSensors(message, requestSensor, registrySensor);
+
+        // make sure messages with sensor values can be deserialized on the receiving node
+        DataOutputBuffer out = SensorsTestUtil.serialize(message);
+        Message deserializedMessage = SensorsTestUtil.deserialize(out, message.from());
+        assertResponseSensors(deserializedMessage, requestSensor, registrySensor);
+    }
+
+    private void assertResponseSensors(Message message, Sensor requestSensor, Sensor registrySensor)
+    {
+        Optional<String> expectedRequestParam = SensorsCustomParams.paramForRequestSensor(registrySensor);
+        Optional<String> expectedGlobalParam = SensorsCustomParams.paramForRequestSensor(registrySensor);
+        assertThat(message.header.customParams()).isNotNull();
+        assertThat(expectedRequestParam).isPresent();
+        assertThat(expectedGlobalParam).isPresent();
+
+        String requestParam = expectedRequestParam.get();
+        String globalParam = expectedGlobalParam.get();
+        assertThat(message.header.customParams()).containsKey(requestParam);
+        assertThat(message.header.customParams()).containsKey(globalParam);
+
+        double requestReadBytes = SensorsTestUtil.bytesToDouble(message.header.customParams().get(requestParam));
+        double tableReadBytes = SensorsTestUtil.bytesToDouble(message.header.customParams().get(globalParam));
+        assertThat(requestReadBytes).isEqualTo(requestSensor.getValue());
+        assertThat(tableReadBytes).isEqualTo(requestSensor.getValue());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsRegistryTest.java b/test/unit/org/apache/cassandra/sensors/SensorsRegistryTest.java
new file mode 100644
index 000000000000..fe842c8c8556
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsRegistryTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.collect.ImmutableSet;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.mockito.Mockito;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class SensorsRegistryTest
+{
+    public static final String KEYSPACE = "SensorsRegistryTest";
+    public static final String CF1 = "Standard1";
+    public static final String CF2 = "Standard2";
+
+    private Context context1;
+    private Type type1;
+    private Context context2;
+    private Type type2;
+
+    @BeforeClass
+    public static void defineSchema() throws Exception
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE, CF1,
+                                                              1, AsciiType.instance, AsciiType.instance, null),
+                                    SchemaLoader.standardCFMD(KEYSPACE, CF2,
+                                                              1, AsciiType.instance, AsciiType.instance, null));
+    }
+
+    @Before
+    public void beforeTest()
+    {
+        context1 = new Context(KEYSPACE, CF1, Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata().id.toString());
+        type1 = Type.READ_BYTES;
+
+        context2 = new Context(KEYSPACE, CF2, Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2).metadata().id.toString());
+        type2 = Type.INDEX_WRITE_BYTES;
+    }
+
+    @After
+    public void afterTest()
+    {
+        SensorsRegistry.instance.clear();
+    }
+
+    @Test
+    public void testCreateAndGetSensors()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata());
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2).metadata());
+
+        Sensor context1Type1Sensor = SensorsRegistry.instance.getOrCreateSensor(context1, type1).get();
+        Sensor context1Type2Sensor = SensorsRegistry.instance.getOrCreateSensor(context1, type2).get();
+        Sensor context2Type1Sensor = SensorsRegistry.instance.getOrCreateSensor(context2, type1).get();
+        Sensor context2Type2Sensor = SensorsRegistry.instance.getOrCreateSensor(context2, type2).get();
+
+        assertThat(SensorsRegistry.instance.getSensor(context1, type1)).hasValue(context1Type1Sensor);
+        assertThat(SensorsRegistry.instance.getSensor(context1, type2)).hasValue(context1Type2Sensor);
+        assertThat(SensorsRegistry.instance.getSensor(context2, type1)).hasValue(context2Type1Sensor);
+        assertThat(SensorsRegistry.instance.getSensor(context2, type2)).hasValue(context2Type2Sensor);
+
+        // get sensor should not have side effects
+        Type unRegisteredType = Type.WRITE_BYTES;
+        assertThat(SensorsRegistry.instance.getSensor(context1, unRegisteredType)).isEmpty();
+
+        assertThat(SensorsRegistry.instance.getSensorsByKeyspace(KEYSPACE)).containsAll(
+        ImmutableSet.of(context1Type1Sensor, context1Type2Sensor, context2Type1Sensor, context2Type2Sensor));
+
+        assertThat(SensorsRegistry.instance.getSensorsByTableId(context1.getTableId())).containsAll(
+        ImmutableSet.of(context1Type1Sensor, context1Type2Sensor));
+
+        assertThat(SensorsRegistry.instance.getSensorsByTableId(context2.getTableId())).containsAll(
+        ImmutableSet.of(context2Type1Sensor, context2Type2Sensor));
+
+        assertThat(SensorsRegistry.instance.getSensorsByType(type1)).containsAll(
+        ImmutableSet.of(context1Type1Sensor, context2Type1Sensor));
+
+        assertThat(SensorsRegistry.instance.getSensorsByType(type2)).containsAll(
+        ImmutableSet.of(context1Type2Sensor, context2Type2Sensor));
+    }
+
+    @Test
+    public void testCannotGetSensorForMissingKeyspace()
+    {
+        assertThat(SensorsRegistry.instance.getOrCreateSensor(context1, type1)).isEmpty();
+    }
+
+    @Test
+    public void testCannotGetSensorForMissingTable()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+
+        assertThat(SensorsRegistry.instance.getOrCreateSensor(context1, type1)).isEmpty();
+    }
+
+    @Test
+    public void testIncrementSensor()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata());
+
+        SensorsRegistry.instance.incrementSensor(context1, type1, 1.0);
+        assertThat(SensorsRegistry.instance.getOrCreateSensor(context1, type1)).hasValueSatisfying((s) -> assertThat(s.getValue()).isEqualTo(1.0));
+    }
+
+    @Test
+    public void testIncrementSensorAsync() throws ExecutionException, InterruptedException, TimeoutException
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata());
+
+        SensorsRegistry.instance.incrementSensorAsync(context1, type1, 1.0, 1, TimeUnit.MILLISECONDS).get(1, TimeUnit.SECONDS);
+        assertThat(SensorsRegistry.instance.getOrCreateSensor(context1, type1)).hasValueSatisfying((s) -> assertThat(s.getValue()).isEqualTo(1.0));
+    }
+
+    @Test
+    public void testSensorRegistryListener()
+    {
+        SensorsRegistryListener listener = Mockito.mock(SensorsRegistryListener.class);
+        SensorsRegistry.instance.registerListener(listener);
+
+        // The sensor will not be created as the keyspace has not been created yet
+        Optional<Sensor> emptySensor = SensorsRegistry.instance.getOrCreateSensor(context1, type1);
+        assertThat(emptySensor).isEmpty();
+        verify(listener, never()).onSensorCreated(any());
+        verify(listener, never()).onSensorRemoved(any());
+
+        // Initialize the schema
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata());
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2).metadata());
+
+        // Create sensors and verify the listener is notified
+        Sensor context1Type1Sensor = SensorsRegistry.instance.getOrCreateSensor(context1, type1).get();
+        verify(listener, times(1)).onSensorCreated(context1Type1Sensor);
+
+        Sensor context1Type2Sensor = SensorsRegistry.instance.getOrCreateSensor(context1, type2).get();
+        verify(listener, times(1)).onSensorCreated(context1Type2Sensor);
+
+        Sensor context2Type1Sensor = SensorsRegistry.instance.getOrCreateSensor(context2, type1).get();
+        verify(listener, times(1)).onSensorCreated(context2Type1Sensor);
+
+        Sensor context2Type2Sensor = SensorsRegistry.instance.getOrCreateSensor(context2, type2).get();
+        verify(listener, times(1)).onSensorCreated(context2Type2Sensor);
+
+        verify(listener, never()).onSensorRemoved(any());
+
+        // Drop the table and verify the listener is notified about removal of related sensors
+        SensorsRegistry.instance.onDropTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2).metadata(), false);
+        verify(listener, times(1)).onSensorRemoved(context2Type1Sensor);
+        verify(listener, times(1)).onSensorRemoved(context2Type2Sensor);
+        verify(listener, never()).onSensorRemoved(context1Type1Sensor);
+        verify(listener, never()).onSensorRemoved(context1Type2Sensor);
+
+        // Drop the keyspace and verify the listener is notified about removal of the remaining sensors
+        SensorsRegistry.instance.onDropKeyspace(Keyspace.open(KEYSPACE).getMetadata(), false);
+        verify(listener, times(1)).onSensorRemoved(context1Type1Sensor);
+        verify(listener, times(1)).onSensorRemoved(context1Type2Sensor);
+        verify(listener, times(1)).onSensorRemoved(context2Type1Sensor);
+        verify(listener, times(1)).onSensorRemoved(context2Type2Sensor);
+
+        // Unregister the listener and verify it is not notified anymore about creation and removal of sensors
+        clearInvocations(listener);
+        SensorsRegistry.instance.unregisterListener(listener);
+
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata());
+
+        assertThat(SensorsRegistry.instance.getOrCreateSensor(context1, type1)).isPresent();
+        SensorsRegistry.instance.onDropKeyspace(Keyspace.open(KEYSPACE).getMetadata(), false);
+
+        verify(listener, never()).onSensorCreated(any());
+        verify(listener, never()).onSensorRemoved(any());
+    }
+
+    @Test
+    public void testUpdateAndSyncSensorViaRequestSensors()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata());
+
+        RequestSensors requestSensors = new ActiveRequestSensors(() -> SensorsRegistry.instance);
+        requestSensors.registerSensor(context1, type1);
+
+        requestSensors.incrementSensor(context1, type1, 1.0);
+        requestSensors.syncAllSensors();
+        assertThat(SensorsRegistry.instance.getOrCreateSensor(context1, type1)).hasValueSatisfying((s) -> assertThat(s.getValue()).isEqualTo(1.0));
+
+        requestSensors.incrementSensor(context1, type1, 1.0);
+        requestSensors.syncAllSensors();
+        assertThat(SensorsRegistry.instance.getOrCreateSensor(context1, type1)).hasValueSatisfying((s) -> assertThat(s.getValue()).isEqualTo(2.0));
+
+        requestSensors.syncAllSensors();
+        assertThat(SensorsRegistry.instance.getOrCreateSensor(context1, type1)).hasValueSatisfying((s) -> assertThat(s.getValue()).isEqualTo(2.0));
+    }
+
+    @Test
+    public void testRemoveSensorByKeyspace()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2).metadata());
+
+        SensorsRegistry.instance.getOrCreateSensor(context1, type1);
+        SensorsRegistry.instance.getOrCreateSensor(context2, type1);
+        assertThat(SensorsRegistry.instance.getSensor(context1, type1)).isPresent();
+        assertThat(SensorsRegistry.instance.getSensor(context2, type1)).isPresent();
+
+        SensorsRegistry.instance.removeSensorsByKeyspace(context1.getKeyspace());
+        assertThat(SensorsRegistry.instance.getSensor(context1, type1)).isEmpty();
+        assertThat(SensorsRegistry.instance.getSensor(context2, type1)).isEmpty();
+
+        SensorsRegistry.instance.getOrCreateSensor(context1, type1);
+        SensorsRegistry.instance.getOrCreateSensor(context2, type1);
+        assertThat(SensorsRegistry.instance.getSensor(context1, type1)).isPresent();
+        assertThat(SensorsRegistry.instance.getSensor(context2, type1)).isPresent();
+    }
+
+    @Test
+    public void testRemoveSensorByTableId()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF1).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF2).metadata());
+
+        SensorsRegistry.instance.getOrCreateSensor(context1, type1);
+        SensorsRegistry.instance.getOrCreateSensor(context2, type1);
+        assertThat(SensorsRegistry.instance.getSensor(context1, type1)).isPresent();
+        assertThat(SensorsRegistry.instance.getSensor(context2, type1)).isPresent();
+
+        SensorsRegistry.instance.removeSensorsByTableId(context1.getKeyspace(), context1.getTableId());
+        assertThat(SensorsRegistry.instance.getSensor(context1, type1)).isEmpty();
+        assertThat(SensorsRegistry.instance.getSensor(context2, type1)).isPresent();
+
+        SensorsRegistry.instance.getOrCreateSensor(context1, type1);
+        assertThat(SensorsRegistry.instance.getSensor(context1, type1)).isPresent();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsTestUtil.java b/test/unit/org/apache/cassandra/sensors/SensorsTestUtil.java
new file mode 100644
index 000000000000..30e06296f57a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsTestUtil.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+
+public final class SensorsTestUtil
+{
+    /**
+     * Returns the default implementation of the {@link SensorsFactory}
+     */
+    public static class NoOpRequestSensorsFactory implements SensorsFactory
+    {
+    }
+    
+    private SensorsTestUtil()
+    {
+    }
+
+    /**
+     * Returns the sensor with the given context and type from the global registry
+     * @param context the sensor context
+     * @param type the sensor type
+     * @return the requested sensor from the global registry
+     */
+    static Sensor getRegistrySensor(Context context, Type type)
+    {
+        return SensorsRegistry.instance.getOrCreateSensor(context, type).get();
+    }
+
+    /**
+     * Returns the sensor registered in the thread local {@link RequestSensors}
+     * @return the thread local read sensor
+     */
+    public static Sensor getThreadLocalRequestSensor(Context context, Type type)
+    {
+        return RequestTracker.instance.get().getSensor(context, type).get();
+    }
+
+    static ColumnFamilyStore discardSSTables(String ks, String cf)
+    {
+        Keyspace keyspace = Keyspace.open(ks);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cf);
+        cfs.discardSSTables(System.currentTimeMillis());
+        return cfs;
+    }
+
+    static DataOutputBuffer serialize(Message message)
+    {
+        try (DataOutputBuffer out = new DataOutputBuffer())
+        {
+            int messagingVersion = MessagingService.current_version;
+            Message.serializer.serialize(message, out, messagingVersion);
+            return out;
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException("Cannot serialize message " + message, e);
+        }
+    }
+
+    static Message deserialize(DataOutputBuffer out, InetAddressAndPort peer)
+    {
+        try (DataInputBuffer in = new DataInputBuffer(out.buffer(), false))
+        {
+            int messagingVersion = MessagingService.current_version;
+            return Message.serializer.deserialize(in, peer, messagingVersion);
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException("Cannot deserialize message from " + peer, e);
+        }
+    }
+
+    static double bytesToDouble(byte[] bytes)
+    {
+        ByteBuffer readBytesBuffer = ByteBuffer.allocate(Double.BYTES);
+        readBytesBuffer.put(bytes);
+        readBytesBuffer.flip();
+        return readBytesBuffer.getDouble();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/sensors/SensorsWriteTest.java b/test/unit/org/apache/cassandra/sensors/SensorsWriteTest.java
new file mode 100644
index 000000000000..22a9121deb26
--- /dev/null
+++ b/test/unit/org/apache/cassandra/sensors/SensorsWriteTest.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sensors;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.CounterMutationVerbHandler;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.MutationVerbHandler;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.BF_RECREATE_ON_FP_CHANCE_CHANGE;
+
+import org.apache.cassandra.service.paxos.Ballot;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.service.paxos.CommitVerbHandler;
+import org.apache.cassandra.service.paxos.PaxosState;
+import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler;
+import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler;
+
+import static org.apache.cassandra.db.SystemKeyspace.PAXOS;
+import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME;
+import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE;
+import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SensorsWriteTest
+{
+    private static final String KEYSPACE1 = "SensorsWriteTest";
+    private static final String CF_STANDARD = "Standard";
+    private static final String CF_STANDARD2 = "Standard2";
+    private static final String CF_STANDARD_CLUSTERING = "StandardClustering";
+    private static final String CF_COUTNER = "Counter";
+
+    private ColumnFamilyStore store;
+    private CopyOnWriteArrayList<Message> capturedOutboundMessages;
+
+    @BeforeClass
+    public static void defineSchema() throws Exception
+    {
+        CassandraRelevantProperties.SENSORS_FACTORY.setString(ActiveSensorsFactory.class.getName());
+
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE1,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD,
+                                                              1, AsciiType.instance, AsciiType.instance, null),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2,
+                                                              1, AsciiType.instance, AsciiType.instance, null),
+                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD_CLUSTERING,
+                                                              1, AsciiType.instance, AsciiType.instance, AsciiType.instance),
+                                    SchemaLoader.counterCFMD(KEYSPACE1, CF_COUTNER));
+
+        CompactionManager.instance.disableAutoCompaction();
+    }
+
+    @Before
+    public void beforeTest()
+    {
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open(KEYSPACE1).getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_CLUSTERING).metadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER).metadata());
+
+        // enable sensor registy for system keyspace
+        SensorsRegistry.instance.onCreateKeyspace(Keyspace.open("system").getMetadata());
+        SensorsRegistry.instance.onCreateTable(Keyspace.open("system").getColumnFamilyStore(PAXOS).metadata());
+
+        capturedOutboundMessages = new CopyOnWriteArrayList<>();
+        MessagingService.instance().outboundSink.add((message, to) ->
+                                                     {
+                                                         capturedOutboundMessages.add(message);
+                                                         return false;
+                                                     });
+    }
+
+    @After
+    public void afterTest()
+    {
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD_CLUSTERING).truncateBlocking();
+        Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER).truncateBlocking();
+        Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(PAXOS).truncateBlocking();
+
+        RequestTracker.instance.set(null);
+        SensorsRegistry.instance.clear();
+
+        BF_RECREATE_ON_FP_CHANCE_CHANGE.setBoolean(false);
+    }
+
+    @Test
+    public void testSingleRowMutation()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+
+        double writeSensorSum = 0;
+        for (int j = 0; j < 10; j++)
+        {
+            Mutation m = new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+                         .add("val", String.valueOf(j))
+                         .build();
+            handleMutation(m);
+            Sensor localSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.WRITE_BYTES);
+            assertThat(localSensor.getValue()).isGreaterThan(0);
+            Sensor registrySensor = SensorsTestUtil.getRegistrySensor(context, Type.WRITE_BYTES);
+            assertThat(registrySensor).isEqualTo(localSensor);
+            writeSensorSum += localSensor.getValue();
+
+            // check global registry is synchronized
+            assertThat(registrySensor.getValue()).isEqualTo(writeSensorSum);
+            assertResponseSensors(localSensor, registrySensor);
+        }
+    }
+
+    @Test
+    public void testSingleRowMutationWithClusteringKey()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_CLUSTERING);
+        Context context = new Context(KEYSPACE1, CF_STANDARD_CLUSTERING, store.metadata.id.toString());
+
+        double writeSensorSum = 0;
+        for (int j = 0; j < 10; j++)
+        {
+            Mutation m = new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+                         .clustering(String.valueOf(j))
+                         .add("val", String.valueOf(j))
+                         .build();
+            handleMutation(m);
+            Sensor localSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.WRITE_BYTES);
+            assertThat(localSensor.getValue()).isGreaterThan(0);
+            Sensor registrySensor = SensorsTestUtil.getRegistrySensor(context, Type.WRITE_BYTES);
+            assertThat(registrySensor).isEqualTo(localSensor);
+            writeSensorSum += localSensor.getValue();
+
+            // check global registry is synchronized
+            assertThat(registrySensor.getValue()).isEqualTo(writeSensorSum);
+            assertResponseSensors(localSensor, registrySensor);
+        }
+    }
+
+    @Test
+    public void testMultipleRowsMutationWithClusteringKey()
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD_CLUSTERING);
+        Context context = new Context(KEYSPACE1, CF_STANDARD_CLUSTERING, store.metadata.id.toString());
+
+        List<Mutation> mutations = new ArrayList<>();
+        String partitionKey = "0";
+
+        // record the written bytes for a single row update
+        String oneCharString = "0"; // a single char string to establish a baseline for the sensor
+        Mutation mutation = new RowUpdateBuilder(store.metadata(), 0, partitionKey)
+                            .clustering(oneCharString)
+                            .add("val", oneCharString)
+                            .build();
+
+        handleMutation(mutation);
+        Sensor localSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.WRITE_BYTES);
+        assertThat(localSensor.getValue()).isGreaterThan(0);
+        double singleRowWriteBytes = localSensor.getValue();
+
+        // build a list of mutations equivalent in written bytes to the single row update but targeting different rows
+        // so we can actually tell if the sensor accommodated for all of them
+        int rowsNum = 10;
+        for (int j = 0; j < rowsNum; j++)
+        {
+            oneCharString = String.valueOf(j);
+            // verify that columns are updated with single char values to match the established singleRowWriteBytes baseline
+            // it is important that each value is different, to enforce proportionality between written bytes and the number of rows
+            // if the values were the same, the mutations will optimize/collapse to a single write
+            assertThat(oneCharString).hasSize(1);
+            mutations.add(new RowUpdateBuilder(store.metadata(), j, partitionKey)
+                          .clustering(String.valueOf(j))
+                          .add("val", String.valueOf(j))
+                          .build());
+        }
+
+        mutation = Mutation.merge(mutations);
+        handleMutation(mutation);
+
+        localSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.WRITE_BYTES);
+        assertThat(localSensor.getValue()).isEqualTo(10 * singleRowWriteBytes);
+
+        Sensor registrySensor = SensorsTestUtil.getRegistrySensor(context, Type.WRITE_BYTES);
+        assertThat(registrySensor).isEqualTo(localSensor);
+        assertThat(registrySensor.getValue()).isEqualTo(localSensor.getValue() + singleRowWriteBytes);
+        assertResponseSensors(localSensor, registrySensor);
+    }
+
+    @Test
+    public void testMultipleTableMutations()
+    {
+        ColumnFamilyStore store1 = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context1 = new Context(KEYSPACE1, CF_STANDARD, store1.metadata.id.toString());
+
+        ColumnFamilyStore store2 = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD2);
+        Context context2 = new Context(KEYSPACE1, CF_STANDARD2, store2.metadata.id.toString());
+
+        List<Mutation> mutations = new ArrayList<>();
+        String partitionKey = "0";
+
+        // first table mutation
+        mutations.add(new RowUpdateBuilder(store1.metadata(), 0, partitionKey)
+                          .add("val", "value")
+                          .build());
+
+        // second table mutation
+        mutations.add(new RowUpdateBuilder(store2.metadata(), 0, partitionKey)
+                          .add("val", "another value")
+                          .build());
+
+        Mutation mutation = Mutation.merge(mutations);
+        handleMutation(mutation);
+
+        Sensor localSensor1 = SensorsTestUtil.getThreadLocalRequestSensor(context1, Type.WRITE_BYTES);
+        assertThat(localSensor1.getValue()).isGreaterThan(0);
+
+        Sensor localSensor2 = SensorsTestUtil.getThreadLocalRequestSensor(context2, Type.WRITE_BYTES);
+        assertThat(localSensor2.getValue()).isGreaterThan(0);
+
+        Sensor registrySensor1 = SensorsTestUtil.getRegistrySensor(context1, Type.WRITE_BYTES);
+        assertThat(registrySensor1).isEqualTo(localSensor1);
+        assertThat(registrySensor1.getValue()).isEqualTo(localSensor1.getValue());
+
+        Sensor registrySensor2 = SensorsTestUtil.getRegistrySensor(context2, Type.WRITE_BYTES);
+        assertThat(registrySensor2).isEqualTo(localSensor2);
+        assertThat(registrySensor2.getValue()).isEqualTo(localSensor2.getValue());
+
+        assertResponseSensors(localSensor1, registrySensor1);
+        assertResponseSensors(localSensor2, registrySensor2);
+    }
+
+    @Test
+    public void testSingleCounterMutation() throws WriteTimeoutException, IOException
+    {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_COUTNER);
+        Context context = new Context(KEYSPACE1, CF_COUTNER, store.metadata.id.toString());
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUTNER);
+        cfs.truncateBlocking();
+
+        Mutation mutation = new RowUpdateBuilder(cfs.metadata(), 5, "key1")
+                            .clustering("cc")
+                            .add("val", 1L).build();
+
+        // Use consistency level ANY to disable the live replicas assertion as we don't have any replica in the unit test
+        CounterMutation counterMutation = new CounterMutation(mutation, ConsistencyLevel.ANY);
+        handleCounterMutation(counterMutation);
+
+        Sensor localSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.WRITE_BYTES);
+        assertThat(localSensor.getValue()).isGreaterThan(0);
+        Sensor registrySensor = SensorsTestUtil.getRegistrySensor(context, Type.WRITE_BYTES);
+        assertThat(registrySensor).isEqualTo(localSensor);
+        assertResponseSensors(localSensor, registrySensor);
+    }
+
+    @Test
+    public void testLWTPrepare() {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+        PartitionUpdate update = new RowUpdateBuilder(store.metadata(), 0, "0")
+                                 .add("val", "0")
+                                 .buildUpdate();
+        Ballot ballot = nextBallot(NONE);
+        Commit proposal = Commit.newPrepare(update.partitionKey(), store.metadata(), ballot);
+        handlePaxosPrepare(proposal);
+
+        Sensor writeSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.WRITE_BYTES);
+        assertThat(writeSensor.getValue()).isGreaterThan(0);
+        Sensor registryWriteSensor = SensorsTestUtil.getRegistrySensor(context, Type.WRITE_BYTES);
+        assertThat(registryWriteSensor).isEqualTo(writeSensor);
+        assertResponseSensors(writeSensor, registryWriteSensor);
+        Sensor readSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertThat(readSensor.getValue()).isZero();
+
+        // Needed so that PaxosState.maybeLoad will call SystemKeyspace.loadPaxosState and SystemKeyspace.transferPaxosSensorBytes
+        PaxosState.unsafeReset();
+
+        // handle the commit again, this time paxos has state because of the first proposal and read bytes will be populated
+        handlePaxosPrepare(proposal);
+        readSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertThat(readSensor.getValue()).isGreaterThan(0);
+        Sensor registryReadSensor = SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES);
+        assertThat(registryReadSensor).isEqualTo(readSensor);
+        assertReadResponseSensors(readSensor, registryReadSensor);
+    }
+
+    @Test
+    public void testLWTPropose() {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+        PartitionUpdate update = new RowUpdateBuilder(store.metadata(), 0, "0")
+                                .add("val", "0")
+                                .buildUpdate();
+        Ballot ballot = nextBallot(NONE);
+        Commit proposal = Commit.newProposal(ballot, update);
+        handlePaxosPropose(proposal);
+
+        Sensor writeSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.WRITE_BYTES);
+        assertThat(writeSensor.getValue()).isGreaterThan(0);
+        Sensor registryWriteSensor = SensorsTestUtil.getRegistrySensor(context, Type.WRITE_BYTES);
+        assertThat(registryWriteSensor).isEqualTo(writeSensor);
+        assertResponseSensors(writeSensor, registryWriteSensor);
+        Sensor readSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertThat(readSensor.getValue()).isZero();
+
+        // Needed so that PaxosState.maybeLoad will call SystemKeyspace.loadPaxosState and SystemKeyspace.transferPaxosSensorBytes
+        PaxosState.unsafeReset();
+
+        // handle the commit again, this time paxos has state because of the first proposal and read bytes will be populated
+        handlePaxosPropose(proposal);
+        readSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.READ_BYTES);
+        assertThat(readSensor.getValue()).isGreaterThan(0);
+        Sensor registryReadSensor = SensorsTestUtil.getRegistrySensor(context, Type.READ_BYTES);
+        assertThat(registryReadSensor).isEqualTo(readSensor);
+        assertReadResponseSensors(readSensor, registryReadSensor);
+    }
+
+    @Test
+    public void testLWTCommit() {
+        store = SensorsTestUtil.discardSSTables(KEYSPACE1, CF_STANDARD);
+        Context context = new Context(KEYSPACE1, CF_STANDARD, store.metadata.id.toString());
+        PartitionUpdate update = new RowUpdateBuilder(store.metadata(), 0, "0")
+                                 .add("val", "0")
+                                 .buildUpdate();
+        Ballot ballot = nextBallot(NONE);
+        Commit proposal = Commit.newPrepare(update.partitionKey(), store.metadata(), ballot);
+        handlePaxosCommit(proposal);
+
+        Sensor writeSensor = SensorsTestUtil.getThreadLocalRequestSensor(context, Type.WRITE_BYTES);
+        assertThat(writeSensor.getValue()).isGreaterThan(0);
+        Sensor registryWriteSensor = SensorsTestUtil.getRegistrySensor(context, Type.WRITE_BYTES);
+        assertThat(registryWriteSensor).isEqualTo(writeSensor);
+        assertResponseSensors(writeSensor, registryWriteSensor);
+
+        // No read is done in the commit phase
+        assertThat(RequestTracker.instance.get().getSensor(context, Type.READ_BYTES)).isEmpty();
+        assertThat(SensorsRegistry.instance.getSensor(context, Type.READ_BYTES)).isEmpty();
+    }
+
+    private static void handlePaxosPrepare(Commit prepare)
+    {
+        PrepareVerbHandler.instance.doVerb(Message.builder(Verb.PAXOS_PREPARE_REQ, prepare).build());
+    }
+
+    private static void handlePaxosPropose(Commit proposal)
+    {
+        ProposeVerbHandler.instance.doVerb(Message.builder(Verb.PAXOS_PROPOSE_REQ, proposal).build());
+    }
+
+    private static void handlePaxosCommit(Commit commit)
+    {
+        CommitVerbHandler.instance.doVerb(Message.builder(Verb.PAXOS_COMMIT_REQ, commit).build());
+    }
+
+    private static void handleMutation(Mutation mutation)
+    {
+        MutationVerbHandler.instance.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).build());
+    }
+
+    private static void handleCounterMutation(CounterMutation mutation) throws IOException
+    {
+        CounterMutationVerbHandler.instance.doVerb(Message.builder(Verb.COUNTER_MUTATION_REQ, mutation).build());
+    }
+
+    private void assertReadResponseSensors(Sensor requestSensor, Sensor registrySensor)
+    {
+        // verify against the last message to enable testing of multiple mutations in a for loop
+        Message message = capturedOutboundMessages.get(capturedOutboundMessages.size() - 1);
+        assertResponseSensors(message, requestSensor, registrySensor);
+
+        // make sure messages with sensor values can be deserialized on the receiving node
+        DataOutputBuffer out = SensorsTestUtil.serialize(message);
+        Message deserializedMessage = SensorsTestUtil.deserialize(out, message.from());
+        assertResponseSensors(deserializedMessage, requestSensor, registrySensor);
+    }
+
+    private void assertResponseSensors(Sensor requestSensor, Sensor registrySensor)
+    {
+        // verify against the last message to enable testing of multiple mutations in a for loop
+        Message message = capturedOutboundMessages.get(capturedOutboundMessages.size() - 1);
+        assertResponseSensors(message, requestSensor, registrySensor);
+
+        // make sure messages with sensor values can be deserialized on the receiving node
+        DataOutputBuffer out = SensorsTestUtil.serialize(message);
+        Message deserializedMessage = SensorsTestUtil.deserialize(out, message.from());
+        assertResponseSensors(deserializedMessage, requestSensor, registrySensor);
+    }
+
+    private void assertResponseSensors(Message message, Sensor requestSensor, Sensor registrySensor)
+    {
+        Optional<String> expectedRequestParam = SensorsCustomParams.paramForRequestSensor(requestSensor);
+        Optional<String> expectedGlobalParam = SensorsCustomParams.paramForGlobalSensor(registrySensor);
+        assertThat(message.header.customParams()).isNotNull();
+        assertThat(expectedRequestParam).isPresent();
+        assertThat(expectedGlobalParam).isPresent();
+
+        String requestParam = expectedRequestParam.get();
+        String globalParam = expectedGlobalParam.get();
+        assertThat(message.header.customParams()).containsKey(requestParam);
+        assertThat(message.header.customParams()).containsKey(globalParam);
+
+        double requestBytes = SensorsTestUtil.bytesToDouble(message.header.customParams().get(requestParam));
+        double globalBytes = SensorsTestUtil.bytesToDouble(message.header.customParams().get(globalParam));
+        assertThat(requestBytes).isEqualTo(requestSensor.getValue());
+        assertThat(globalBytes).isEqualTo(registrySensor.getValue());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/serializers/DateRangeSerializerTest.java b/test/unit/org/apache/cassandra/serializers/DateRangeSerializerTest.java
new file mode 100644
index 000000000000..bca5f688cc51
--- /dev/null
+++ b/test/unit/org/apache/cassandra/serializers/DateRangeSerializerTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.serializers;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.datetime.DateRange;
+import org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBound.Precision;
+
+import static org.apache.cassandra.db.marshal.datetime.DateRange.DateRangeBuilder.dateRange;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+@RunWith(Enclosed.class)
+public class DateRangeSerializerTest
+{
+    @RunWith(Parameterized.class)
+    public static class ValidCases
+    {
+        @Parameterized.Parameter
+        public DateRange dateRange;
+
+        @Test
+        public void testSerializeRoundTrip()
+        {
+            ByteBuffer serialized = DateRangeSerializer.instance.serialize(dateRange);
+
+            // For UDT or tuple type buffer contains whole cell payload, and codec can't rely on absolute byte addressing
+            ByteBuffer payload = ByteBuffer.allocate(5 + serialized.capacity());
+            // put serialized date range in between other data
+            payload.putInt(44).put(serialized).put((byte) 1);
+            payload.position(4);
+
+            DateRange actual = DateRangeSerializer.instance.deserialize(payload);
+
+            assertEquals(dateRange, actual);
+            //provided ByteBuffer should never be consumed by read operations that modify its current position
+            assertEquals(4, payload.position());
+        }
+
+        @SuppressWarnings("unused")
+        @Parameterized.Parameters(name = "dateRange = {0}")
+        public static Collection<Object[]> dateRanges()
+        {
+            return Arrays.asList(
+                new Object[]{
+                        // 2015-12-03T10:15:30 TO 2016-01-01T00:00:01.001Z
+                        dateRange()
+                                .withLowerBound("2015-12-03T10:15:30.000Z", Precision.SECOND)
+                                .withUpperBound("2016-01-01T00:00:01.001Z", Precision.MILLISECOND)
+                                .build()
+                },
+                new Object[]{
+                        // 1998-01-01 TO *
+                        dateRange()
+                                .withLowerBound("1998-01-01T00:00:00.000Z", Precision.DAY)
+                                .withUnboundedUpperBound()
+                                .build()
+                },
+                new Object[]{
+                        // * TO 1951-01-02T01
+                        dateRange()
+                                .withUnboundedLowerBound()
+                                .withUpperBound("1951-01-02T01:00:00.003Z", Precision.HOUR)
+                                .build()
+                },
+                new Object[]{
+                        // *
+                        dateRange()
+                                .withUnboundedLowerBound()
+                                .build()
+                },
+                new Object[]{
+                        // [* TO *]
+                        dateRange()
+                                .withUnboundedLowerBound()
+                                .withUnboundedUpperBound()
+                                .build()
+                },
+                new Object[]{
+                        // 1966
+                        dateRange()
+                                .withLowerBound("1966-03-03T03:30:30.030Z", Precision.YEAR)
+                                .build(),
+                }
+            );
+        }
+    }
+
+    public static class InvalidCases
+    {
+
+        @Rule
+        public ExpectedException expectedException = ExpectedException.none();
+
+        @Test
+        public void testNullValueSerializeRoundTrip()
+        {
+            ByteBuffer serialized = DateRangeSerializer.instance.serialize(null);
+            assertEquals(0, serialized.capacity());
+            assertNull(DateRangeSerializer.instance.deserialize(serialized));
+        }
+
+        @Test
+        public void testDeserializeInvalidLengthInput()
+        {
+            expectedException.expect(IndexOutOfBoundsException.class);
+            DateRangeSerializer.instance.deserialize(ByteBuffer.allocate(5));
+        }
+
+        @Test
+        public void testDeserializeUnsupportedHeader()
+        {
+            expectedException.expect(IllegalArgumentException.class);
+            expectedException.expectMessage("Unknown date range type");
+            DateRangeSerializer.instance.deserialize(ByteBuffer.allocate(1).put(0, (byte) 0x15));
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/serializers/SimpleDateSerializerTest.java b/test/unit/org/apache/cassandra/serializers/SimpleDateSerializerTest.java
index 9c1ef886f96d..8502fbde7659 100644
--- a/test/unit/org/apache/cassandra/serializers/SimpleDateSerializerTest.java
+++ b/test/unit/org/apache/cassandra/serializers/SimpleDateSerializerTest.java
@@ -26,6 +26,7 @@
 import java.text.SimpleDateFormat;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 public class SimpleDateSerializerTest
 {
@@ -152,4 +153,16 @@ public void testBadDayToMonth()
     {
         Integer days = SimpleDateSerializer.dateStringToDays("1000-09-31");
     }
+
+    @Test(expected = MarshalException.class)
+    public void testOutOfBoundsHighMillis()
+    {
+        SimpleDateSerializer.timeInMillisToDay(TimeUnit.DAYS.toMillis(Integer.MAX_VALUE) + 1);
+    }
+
+    @Test(expected = MarshalException.class)
+    public void testOutOfBoundsLowMillis()
+    {
+        SimpleDateSerializer.timeInMillisToDay(TimeUnit.DAYS.toMillis(Integer.MIN_VALUE) - 1L);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/service/AbstractFilesystemOwnershipCheckTest.java b/test/unit/org/apache/cassandra/service/AbstractFilesystemOwnershipCheckTest.java
index cae7de060b62..caac0b2863a8 100644
--- a/test/unit/org/apache/cassandra/service/AbstractFilesystemOwnershipCheckTest.java
+++ b/test/unit/org/apache/cassandra/service/AbstractFilesystemOwnershipCheckTest.java
@@ -135,19 +135,19 @@ private static File mkdirs(File parent, String path)
         return childDir;
     }
 
-    private static FileSystemOwnershipCheck checker(Supplier<Iterable<String>> dirs)
+    private static FileSystemOwnershipCheck checker(Supplier<Iterable<File>> dirs)
     {
         return new FileSystemOwnershipCheck(dirs);
     }
 
     private static FileSystemOwnershipCheck checker(File...dirs)
     {
-        return checker(() -> Arrays.stream(dirs).map(File::absolutePath).collect(Collectors.toList()));
+        return checker(() -> Arrays.asList(dirs));
     }
 
     private static FileSystemOwnershipCheck checker(String...dirs)
     {
-        return checker(() -> Arrays.asList(dirs));
+        return checker(() -> Arrays.stream(dirs).map(File::new).collect(Collectors.toList()));
     }
 
     public static String makeRandomString(int length)
diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
index 3b7d0cd9a095..b7a5e975a7f7 100644
--- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
+++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
@@ -53,6 +53,7 @@
 import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
@@ -61,6 +62,7 @@
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.repair.RepairParallelism;
 import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.streaming.PreviewKind;
@@ -76,6 +78,7 @@
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.fail;
@@ -519,4 +522,59 @@ public void run()
             complete.countDown();
         }
     }
+
+    @Test
+    public void testPrepareRepairWithDeadNodes()
+    {
+        Set<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges();
+        TimeUUID parentRepairSession = TimeUUID.Generator.nextTimeUUID();
+        List<ColumnFamilyStore> columnFamilyStores = Arrays.asList(prepareColumnFamilyStore());
+        boolean isForcedRepair = false;
+
+        Set<InetAddressAndPort> endpoints = new HashSet<>();
+
+        TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+        Token token = tmd.partitioner.getMinimumToken();
+        IPartitioner partitioner = tmd.partitioner;
+        Util.joinNodeToRing(REMOTE, token, partitioner);
+
+        // Mark remote node as dead
+        Util.markNodeAsDead(REMOTE);
+        endpoints.add(REMOTE);
+
+        RepairOption options = new RepairOption(RepairParallelism.PARALLEL, true, true,
+                                                false, 1, ranges, false, false,
+                                                false, false, PreviewKind.ALL, false,
+                                                false, false, false, false);
+        try
+        {
+            ActiveRepairService.instance().prepareForRepair(parentRepairSession, LOCAL, endpoints, options, isForcedRepair, columnFamilyStores);
+            fail();
+        }
+        catch (RuntimeException ex)
+        {
+            String msg = ex.getMessage();
+            final String expected = "Endpoint not alive";
+            assertTrue(String.format("Did not see expected '%s' message", expected), msg.contains(expected));
+        }
+    }
+
+    @Test
+    public void testCleanUpLiveAndDeadNodes()
+    {
+        TimeUUID parentRepairSession = TimeUUID.Generator.nextTimeUUID();
+        Set<InetAddressAndPort> endpoints = new HashSet<>();
+        endpoints.add(LOCAL);
+
+        TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+        Token token = tmd.partitioner.getMinimumToken();
+        IPartitioner partitioner = tmd.partitioner;
+
+        Util.joinNodeToRing(REMOTE, token, partitioner);
+        Util.markNodeAsDead(REMOTE);
+
+        endpoints.add(REMOTE);
+
+        ActiveRepairService.instance().cleanUp(parentRepairSession, endpoints);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/service/DiskFailurePolicyTest.java b/test/unit/org/apache/cassandra/service/DiskFailurePolicyTest.java
index b041a1cd10a6..ecfe95a4fa75 100644
--- a/test/unit/org/apache/cassandra/service/DiskFailurePolicyTest.java
+++ b/test/unit/org/apache/cassandra/service/DiskFailurePolicyTest.java
@@ -39,6 +39,7 @@
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.JVMKiller;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.KillerForTests;
 
@@ -55,7 +56,7 @@
 public class DiskFailurePolicyTest
 {
     DiskFailurePolicy originalDiskFailurePolicy;
-    JVMStabilityInspector.Killer originalKiller;
+    JVMKiller originalKiller;
     KillerForTests killerForTests;
     DiskFailurePolicy testPolicy;
     boolean isStartUpInProgress;
@@ -150,11 +151,11 @@ public void testPolicies()
                 throw e;
         }
 
-        if (testPolicy == best_effort && ((FSReadError) t).path.equals("best_effort_io_exception"))
+        if (testPolicy == best_effort && ((FSReadError) t).file.path().equals("best_effort_io_exception"))
             assertTrue(DisallowedDirectories.isUnreadable(new File("best_effort_io_exception")));
 
         // when we have OOM, as cause, there is no reason to remove data
-        if (testPolicy == best_effort && ((FSReadError) t).path.equals("best_effort_oom"))
+        if (testPolicy == best_effort && ((FSReadError) t).file.path().equals("best_effort_oom"))
             assertFalse(DisallowedDirectories.isUnreadable(new File("best_effort_oom")));
 
         assertEquals(expectJVMKilled, killerForTests.wasKilled());
diff --git a/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java b/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java
index c2aeb56710fc..c1c9ced79388 100644
--- a/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java
+++ b/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java
@@ -36,8 +36,8 @@ public class JoinTokenRingTest
     public static void setup() throws ConfigurationException
     {
         DatabaseDescriptor.daemonInitialization();
-        SchemaLoader.startGossiper();
         SchemaLoader.prepareServer();
+        SchemaLoader.startGossiper();
         SchemaLoader.schemaDefinition("JoinTokenRingTest");
     }
 
diff --git a/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java b/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java
index 5d4f681e5c90..1dc61a11b282 100644
--- a/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java
+++ b/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java
@@ -47,7 +47,9 @@
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.SimpleSnitch;
 import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.nodes.Nodes;
 import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.assertj.core.api.Assertions;
 
 import static org.junit.Assert.*;
 
@@ -655,7 +657,7 @@ public void testStateJumpToLeft() throws UnknownHostException
 
         assertFalse(tmd.isMember(hosts.get(2)));
 
-        // node hosts.get(4) goes to bootstrap
+        // node hosts.get(3) goes to bootstrap
         Gossiper.instance.injectApplicationState(hosts.get(3), ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(1))));
         ss.onChange(hosts.get(3), ApplicationState.STATUS, valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(1))));
 
@@ -668,7 +670,7 @@ public void testStateJumpToLeft() throws UnknownHostException
         ss.onChange(hosts.get(2), ApplicationState.STATUS,
                 valueFactory.left(Collections.singleton(keyTokens.get(1)), Gossiper.computeExpireTime()));
 
-        assertTrue(tmd.getBootstrapTokens().size() == 0);
+        Assertions.assertThat(tmd.getBootstrapTokens()).isEmpty();
         assertFalse(tmd.isMember(hosts.get(2)));
         assertFalse(tmd.isLeaving(hosts.get(2)));
     }
@@ -688,8 +690,8 @@ public void testStateChangeOnRemovedNode() throws UnknownHostException
         Util.createInitialRing(ss, partitioner, endpointTokens, new ArrayList<Token>(), hosts, new ArrayList<UUID>(), 2);
 
         InetAddressAndPort toRemove = hosts.get(1);
-        SystemKeyspace.updatePeerInfo(toRemove, "data_center", "dc42");
-        SystemKeyspace.updatePeerInfo(toRemove, "rack", "rack42");
+        Nodes.peers().update(toRemove, info -> info.setDataCenter("dc42"), false);
+        Nodes.peers().update(toRemove, info -> info.setRack("rack42"), false);
         assertEquals("rack42", SystemKeyspace.loadDcRackInfo().get(toRemove).get("rack"));
 
         // mark the node as removed
@@ -736,4 +738,26 @@ private AbstractReplicationStrategy getStrategy(String keyspaceName, TokenMetada
                 ksmd.params.replication.options);
     }
 
+    @Test
+    public void testRemoveExistingMember() throws UnknownHostException
+    {
+        // create a ring of 1 node
+        StorageService ss = StorageService.instance;
+        VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
+        List<UUID> hostIds = new ArrayList<>();
+        Util.createInitialRing(ss, partitioner, new ArrayList<Token>(), new ArrayList<Token>(), new ArrayList<InetAddressAndPort>(), hostIds, 1);
+
+        InetAddressAndPort removeNode = InetAddressAndPort.getByName("127.0.0.87");
+        UUID removeHostId = UUID.randomUUID();
+        Token token = ss.getTokenFactory().fromString("87");
+        Util.joinNodeToRing(removeNode, token, partitioner, removeHostId, 1);
+        UUID coordinatorHostID = hostIds.get(0);
+        Gossiper.instance.injectApplicationState(removeNode, ApplicationState.REMOVAL_COORDINATOR, valueFactory.removalCoordinator(coordinatorHostID));
+
+        TokenMetadata tmd = ss.getTokenMetadata();
+        assertEquals(removeNode, tmd.getEndpointForHostId(removeHostId));
+        ss.onChange(removeNode, ApplicationState.STATUS_WITH_PORT, valueFactory.removingNonlocal(UUID.randomUUID()));
+        assertTrue("Removed node should be marked as leaving", tmd.isLeaving(removeNode));
+        assertTrue("Removed node not in list of leaving nodes", ss.getLeavingNodes().contains(removeNode.getHostAddress(false)));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/service/MigrateLegacySystemDataTest.java b/test/unit/org/apache/cassandra/service/MigrateLegacySystemDataTest.java
new file mode 100644
index 000000000000..5c5478b47cda
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/MigrateLegacySystemDataTest.java
@@ -0,0 +1,85 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.cassandra.service;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.SchemaConstants;
+
+public class MigrateLegacySystemDataTest
+{
+    @BeforeClass
+    public static void defineSchema() throws ConfigurationException
+    {
+        SchemaLoader.prepareServer();
+    }
+
+    @Test
+    public void testMigrateSystemDataIfNeeded() throws Throwable
+    {
+        File[] dataDirectories = DatabaseDescriptor.getAllDataFileLocations();
+        File legacySystemDataDirectory = dataDirectories[0];
+
+        // verify files exist in legacy directory
+        Collection<File> legacyTableDirectories = getSystemTableDirectories(legacySystemDataDirectory);
+        Assert.assertTrue(legacyTableDirectories.size() > 0);
+
+        File newSystemDataDirectory = dataDirectories[0].resolveSibling("new_system_directory");
+        try
+        {
+            DatabaseDescriptor.setSpecificLocationForLocalSystemData(newSystemDataDirectory);
+            newSystemDataDirectory.tryCreateDirectories();
+
+            // verify no table directories in new system dir before migration
+            Collection<File> newTableDirectories = getSystemTableDirectories(newSystemDataDirectory);
+            Assert.assertEquals(0, newTableDirectories.size());
+
+            CassandraDaemon daemon = new CassandraDaemon();
+            daemon.migrateSystemDataIfNeeded();
+
+            // verify table directories are migrated in new system dir
+            newTableDirectories = getSystemTableDirectories(newSystemDataDirectory);
+            Assert.assertEquals(legacyTableDirectories.size(), newTableDirectories.size() + SystemKeyspace.TABLES_SPLIT_ACROSS_MULTIPLE_DISKS.size());
+        }
+        finally
+        {
+            newSystemDataDirectory.deleteRecursive();
+        }
+    }
+
+    private Set<File> getSystemTableDirectories(File systemDataDirectory)
+    {
+        return Arrays.stream(systemDataDirectory.tryList())
+                     .filter(f -> SchemaConstants.isLocalSystemKeyspace(f.name()))
+                     .flatMap(f -> Arrays.stream(f.tryList()))
+                     .collect(Collectors.toSet());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/service/MutatorProviderTest.java b/test/unit/org/apache/cassandra/service/MutatorProviderTest.java
new file mode 100644
index 000000000000..390f56293629
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/MutatorProviderTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.util.Collection;
+import javax.annotation.Nullable;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.exceptions.OverloadedException;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.junit.Assert.assertSame;
+
+public class MutatorProviderTest
+{
+    public static class TestMutator implements Mutator
+    {
+        @Override
+        public AbstractWriteResponseHandler<IMutation> mutateCounter(CounterMutation cm, String localDataCenter, Dispatcher.RequestTime requestTime)
+        {
+            return null;
+        }
+
+        @Override
+        public AbstractWriteResponseHandler<IMutation> mutateCounterOnLeader(CounterMutation mutation,
+                                                                             String localDataCenter,
+                                                                             StorageProxy.WritePerformer performer,
+                                                                             Runnable callback,
+                                                                             Dispatcher.RequestTime requestTime)
+        {
+            return null;
+        }
+
+        @Override
+        public AbstractWriteResponseHandler<IMutation> mutateStandard(Mutation mutation, ConsistencyLevel consistencyLevel, String localDataCenter, StorageProxy.WritePerformer writePerformer, Runnable callback, WriteType writeType, Dispatcher.RequestTime requestTime)
+        {
+            return null;
+        }
+
+        @Nullable
+        @Override
+        public AbstractWriteResponseHandler<Commit> mutatePaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, Dispatcher.RequestTime requestTime)
+        {
+            return null;
+        }
+
+        @Override
+        public void mutateAtomically(Collection<Mutation> mutations, ConsistencyLevel consistencyLevel, boolean requireQuorumForRemove, Dispatcher.RequestTime requestTime, ClientRequestsMetrics metrics, ClientState clientState) throws UnavailableException, OverloadedException, WriteTimeoutException
+        {
+            // no-op
+        }
+
+        @Override
+        public void persistBatchlog(Collection<Mutation> mutations, Dispatcher.RequestTime requestTime, ReplicaPlan.ForWrite replicaPlan, TimeUUID batchUUID)
+        {
+            // no-op
+        }
+
+        @Override
+        public void clearBatchlog(String keyspace, Dispatcher.RequestTime requestTime, ReplicaPlan.ForWrite replicaPlan, TimeUUID batchUUID)
+        {
+            // no-op
+        }
+    }
+
+    @Test
+    public void testInstantinatingCustomMutator()
+    {
+        CassandraRelevantProperties.CUSTOM_MUTATOR_CLASS.setString("org.apache.cassandra.service.MutatorProviderTest$TestMutator");
+        Mutator mutator = MutatorProvider.getCustomOrDefault();
+        assertSame(mutator.getClass(), TestMutator.class);
+        System.clearProperty(CassandraRelevantProperties.CUSTOM_MUTATOR_CLASS.getKey());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/service/MutatorTest.java b/test/unit/org/apache/cassandra/service/MutatorTest.java
new file mode 100644
index 000000000000..340118c3bbc8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/MutatorTest.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.exceptions.WriteFailureException;
+import org.apache.cassandra.metrics.ClientRequestsMetrics;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.awaitility.Awaitility;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.apache.cassandra.Util.dk;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@RunWith(BMUnitRunner.class)
+public class MutatorTest
+{
+    private static final String KEYSPACE = "ks_test";
+    private static final String KEYSPACE_RF2 = "ks_rf2";
+    private static final String KEYSPACE_TRANSIENT = "ks_transient";
+    private static final String TABLE0 = "table_0";
+    private static final String TABLE1 = "table_1";
+    private static final String TABLE2 = "table_2";
+
+    @BeforeClass
+    public static void defineSchema()
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE0),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE1),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE2));
+        SchemaLoader.createKeyspace(KEYSPACE_RF2,
+                                    KeyspaceParams.simple(2),
+                                    SchemaLoader.standardCFMD(KEYSPACE_RF2, TABLE0),
+                                    SchemaLoader.standardCFMD(KEYSPACE_RF2, TABLE1),
+                                    SchemaLoader.standardCFMD(KEYSPACE_RF2, TABLE2));
+        SchemaLoader.createKeyspace(KEYSPACE_TRANSIENT,
+                                    KeyspaceParams.nts("datacenter1", "3/1"),
+                                    SchemaLoader.standardCFMD(KEYSPACE_TRANSIENT, TABLE0),
+                                    SchemaLoader.standardCFMD(KEYSPACE_TRANSIENT, TABLE1),
+                                    SchemaLoader.standardCFMD(KEYSPACE_TRANSIENT, TABLE2));
+
+        Token token = ByteOrderedPartitioner.instance.getToken(ByteBufferUtil.bytes(1));
+        StorageService.instance.getTokenMetadata().updateNormalToken(token, FBUtilities.getBroadcastAddressAndPort());
+    }
+
+    @Before
+    public void clearBatchlog()
+    {
+        Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).truncateBlocking();
+    }
+
+    @Test
+    public void testMutatateAtomically()
+    {
+        Mutator mutator = new StorageProxy.DefaultMutator();
+
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        Collection<Mutation> mutations = Arrays.asList(createMutation(KEYSPACE, "1", requestTime), createMutation(KEYSPACE, "2", requestTime));
+        ConsistencyLevel consistency = ConsistencyLevel.EACH_QUORUM;
+        ClientRequestsMetrics metrics = new ClientRequestsMetrics("test");
+        ClientState clientState = ClientState.forInternalCalls();
+
+        long count = metrics.writeMetrics.executionTimeMetrics.latency.getCount();
+        long countServiceTime = metrics.writeMetrics.serviceTimeMetrics.latency.getCount();
+
+        mutator.mutateAtomically(mutations, consistency, true, Dispatcher.RequestTime.forImmediateExecution(), metrics, clientState);
+        assertThat(metrics.writeMetrics.executionTimeMetrics.latency.getCount()).isEqualTo(count + 1);
+        assertThat(metrics.writeMetrics.serviceTimeMetrics.latency.getCount()).isEqualTo(countServiceTime + 1);
+
+        // verify batchlog is removed -- this happens async, so we may need to check several times
+        Awaitility.await().timeout(10, TimeUnit.SECONDS)
+                  .untilAsserted(() -> Util.assertEmpty(Util.cmd(cfs(SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.BATCHES)).withNowInSeconds(FBUtilities.nowInSeconds()).build()));
+
+        // verify data is inserted
+        Util.getOnlyRow(Util.cmd(cfs(KEYSPACE, TABLE0), dk("1")).includeRow("column0").withNowInSeconds(FBUtilities.nowInSeconds()).build());
+        Util.getOnlyRow(Util.cmd(cfs(KEYSPACE, TABLE0), dk("2")).includeRow("column0").withNowInSeconds(FBUtilities.nowInSeconds()).build());
+    }
+
+    /**
+     * Require 2 batchlog replicas, but only local node available
+     */
+    @Test
+    public void testMutatateAtomicallyInsufficientBatchlogReplica()
+    {
+        Mutator mutator = new StorageProxy.DefaultMutator();
+
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        Collection<Mutation> mutations = Arrays.asList(createMutation(KEYSPACE_RF2, "1", requestTime), createMutation(KEYSPACE_RF2, "2", requestTime));
+        ConsistencyLevel consistency = ConsistencyLevel.EACH_QUORUM;
+        ClientRequestsMetrics metrics = new ClientRequestsMetrics("test");
+        ClientState clientState = ClientState.forInternalCalls();
+
+        long count = metrics.writeMetrics.unavailables.getCount();
+
+        assertThatThrownBy(() -> mutator.mutateAtomically(mutations, consistency, true, requestTime, metrics, clientState))
+        .isInstanceOf(UnavailableException.class)
+        .hasMessageContaining("Cannot achieve consistency level EACH_QUORUM");
+
+        assertThat(metrics.writeMetrics.unavailables.getCount()).isEqualTo(count + 1);
+
+        // verify batchlog is not stored
+        Util.assertEmpty(Util.cmd(cfs(SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.BATCHES)).withNowInSeconds(FBUtilities.nowInSeconds()).build());
+    }
+
+    /**
+     * Require 2 write replicas, but only local node available
+     */
+    @Test
+    public void testMutatateAtomicallyInsufficientWriteReplica()
+    {
+        Mutator mutator = new StorageProxy.DefaultMutator();
+
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        Collection<Mutation> mutations = Arrays.asList(createMutation(KEYSPACE_RF2, "1", requestTime), createMutation(KEYSPACE_RF2, "2", requestTime));
+        ConsistencyLevel consistency = ConsistencyLevel.ALL;
+        ClientRequestsMetrics metrics = new ClientRequestsMetrics("test");
+        ClientState clientState = ClientState.forInternalCalls();
+
+        assertThatThrownBy(() -> mutator.mutateAtomically(mutations, consistency, false, requestTime, metrics, clientState))
+        .isInstanceOf(UnavailableException.class)
+        .hasMessageContaining("Cannot achieve consistency level ALL");
+
+        // verify batchlog is not stored
+        Util.assertEmpty(Util.cmd(cfs(SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.BATCHES)).withNowInSeconds(FBUtilities.nowInSeconds()).build());
+    }
+
+    @Test
+    @BMRule(name = "fail mutation write",
+    targetClass = "org.apache.cassandra.db.Keyspace",
+    targetMethod = "apply",
+    targetLocation = "AT ENTRY",
+    condition = "$1.getKeyspaceName().endsWith(\"ks_test\")",
+    action = "throw new RuntimeException(\"Byteman Exception\")")
+    public void testMutatateAtomicallyWriteFailure()
+    {
+        Mutator mutator = new StorageProxy.DefaultMutator();
+
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        Collection<Mutation> mutations = Arrays.asList(createMutation(KEYSPACE, "1", requestTime), createMutation(KEYSPACE, "2", requestTime));
+        ConsistencyLevel consistency = ConsistencyLevel.ALL;
+        ClientRequestsMetrics metrics = new ClientRequestsMetrics("test");
+        ClientState clientState = ClientState.forInternalCalls();
+
+        assertThatThrownBy(() -> mutator.mutateAtomically(mutations, consistency, false, requestTime, metrics, clientState))
+        .isInstanceOf(WriteFailureException.class)
+        .hasMessageContaining("received 0 responses and 1 failures");
+
+        // verify batchlog is stored
+        Util.getOnlyRow(Util.cmd(cfs(SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.BATCHES)).withNowInSeconds(FBUtilities.nowInSeconds()).build());
+    }
+
+    /**
+     * logged batch doesn't support transient replica
+     */
+    @Test
+    public void testMutatateAtomicallyWithTransientReplica()
+    {
+        Mutator mutator = new StorageProxy.DefaultMutator();
+
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        Collection<Mutation> mutations = Arrays.asList(createMutation(KEYSPACE_TRANSIENT, "1", requestTime), createMutation(KEYSPACE_TRANSIENT, "2", requestTime));
+        ConsistencyLevel consistency = ConsistencyLevel.EACH_QUORUM;
+        ClientRequestsMetrics metrics = new ClientRequestsMetrics("test");
+        ClientState clientState = ClientState.forInternalCalls();
+
+        assertThatThrownBy(() -> mutator.mutateAtomically(mutations, consistency, false, requestTime, metrics, clientState))
+        .isInstanceOf(AssertionError.class)
+        .hasMessageContaining("Logged batches are unsupported with transient replication");
+    }
+
+    private static ColumnFamilyStore cfs(String keyspace, String table)
+    {
+        return Keyspace.open(keyspace).getColumnFamilyStore(table);
+    }
+
+    private static Mutation createMutation(String kesypace, String key, Dispatcher.RequestTime requestTime)
+    {
+        Mutation.SimpleBuilder builder = Mutation.simpleBuilder(kesypace, dk(key));
+
+        builder.update(Schema.instance.getTableMetadata(kesypace, TABLE0))
+               .timestamp(requestTime.startedAtNanos())
+               .row("column0")
+               .add("val", "value0");
+
+        return builder.build();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/service/PaxosStateTest.java b/test/unit/org/apache/cassandra/service/PaxosStateTest.java
index e73147d97ed8..852e326fd7c1 100644
--- a/test/unit/org/apache/cassandra/service/PaxosStateTest.java
+++ b/test/unit/org/apache/cassandra/service/PaxosStateTest.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.service;
 
 import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.collect.Iterables;
 
@@ -85,22 +86,27 @@ public void testCommittingAfterTruncation() throws Exception
         // CFS should be empty initially
         assertNoDataPresent(cfs, Util.dk(key));
 
+        AtomicBoolean committed = new AtomicBoolean();
+
         // Commit the proposal & verify the data is present
         Commit beforeTruncate = newProposal(0, update);
-        PaxosState.commitDirect(beforeTruncate);
+        PaxosState.commitDirect(beforeTruncate, ignored -> committed.set(true));
         assertDataPresent(cfs, Util.dk(key), "val", value);
+        assertTrue(committed.getAndSet(false));
 
         // Truncate then attempt to commit again, mutation should
         // be ignored as the proposal predates the truncation
         cfs.truncateBlocking();
-        PaxosState.commitDirect(beforeTruncate);
+        PaxosState.commitDirect(beforeTruncate, ignored -> committed.set(true));
         assertNoDataPresent(cfs, Util.dk(key));
+        assertFalse(committed.getAndSet(false));
 
         // Now try again with a ballot created after the truncation
         long timestamp = SystemKeyspace.getTruncatedAt(update.metadata().id) + 1;
         Commit afterTruncate = newProposal(timestamp, update);
-        PaxosState.commitDirect(afterTruncate);
+        PaxosState.commitDirect(afterTruncate, ignored -> committed.set(true));
         assertDataPresent(cfs, Util.dk(key), "val", value);
+        assertTrue(committed.getAndSet(false));
     }
 
     private Commit newProposal(long ballotMicros, PartitionUpdate update)
diff --git a/test/unit/org/apache/cassandra/service/PaxosUtilsTest.java b/test/unit/org/apache/cassandra/service/PaxosUtilsTest.java
new file mode 100644
index 000000000000..c93a1c5b0cb3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/PaxosUtilsTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.carrotsearch.randomizedtesting.annotations.Timeout;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.metrics.CASClientRequestMetrics;
+import org.apache.cassandra.service.paxos.PaxosUtils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class PaxosUtilsTest
+{
+    private static final int minPaxosBackoffMillis = 1;
+    private static final int maxPaxosBackoffMillis = 5;
+
+    private static final long minPaxosBackoffMicros = minPaxosBackoffMillis * 1000;
+    private static final long maxPaxosBackoffMicros = maxPaxosBackoffMillis * 1000;
+
+    @BeforeClass
+    public static void beforeClass() throws Throwable
+    {
+        CassandraRelevantProperties.LWT_MIN_BACKOFF_MS.setInt(minPaxosBackoffMillis);
+        CassandraRelevantProperties.LWT_MAX_BACKOFF_MS.setInt(maxPaxosBackoffMillis);
+    }
+
+    @Timeout(millis = 500)
+    @Test
+    public void testApplyPaxosContentionBackoff()
+    {
+        CASClientRequestMetrics casMetrics = new CASClientRequestMetrics("test", "");
+        long totalLatencyMicrosFromPreviousIteration = 0;
+        for (int i = 0; i < 100; i++)
+        {
+            PaxosUtils.applyPaxosContentionBackoff(casMetrics);
+            assertEquals(i + 1, casMetrics.contentionBackoffLatency.latency.getCount());
+
+            double lastRecordedLatencyMicros = casMetrics.contentionBackoffLatency.totalLatency.getCount() - totalLatencyMicrosFromPreviousIteration;
+            totalLatencyMicrosFromPreviousIteration = casMetrics.contentionBackoffLatency.totalLatency.getCount();
+            assertTrue(lastRecordedLatencyMicros >= minPaxosBackoffMicros);
+            assertTrue(lastRecordedLatencyMicros < maxPaxosBackoffMicros);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/service/QueryInfoTrackerTest.java b/test/unit/org/apache/cassandra/service/QueryInfoTrackerTest.java
new file mode 100644
index 000000000000..f42ba3fbd9bd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/QueryInfoTrackerTest.java
@@ -0,0 +1,685 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.Iterables;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.schema.TableMetadata;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests that the methods of the {@link QueryInfoTracker} interface are correctly called.
+ *
+ * <p>The tests below use "drivers" sessions to ensure that queries go through {@link StorageProxy}, where
+ * {@link QueryInfoTracker} is setup.
+ *
+ * @see ReadQueryTrackingTest for additional scenarios with different data models
+ */
+@RunWith(BMUnitRunner.class)
+public class QueryInfoTrackerTest extends CQLTester
+{
+    private volatile TestQueryInfoTracker tracker;
+    private volatile Session session;
+
+    @Before
+    public void setupTest()
+    {
+        tracker = new TestQueryInfoTracker(KEYSPACE);
+        StorageProxy.instance.registerQueryTracker(tracker);
+        requireNetwork();
+        session = sessionNet();
+    }
+
+    @Test
+    public void testSimpleQueryTracing()
+    {
+        int keys = 4;
+        int clustering = 7;
+        String table = KEYSPACE + ".qit_simple";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, PRIMARY KEY (k, c))");
+        for (int k = 0; k < keys; k++)
+        {
+            for (int v = 0; v < clustering; v++)
+            {
+                session.execute("INSERT INTO " + table + "(k, c) values (?, ?)", k, v);
+            }
+        }
+
+        int expectedWrites = keys * clustering;
+        int expectedRows = keys * clustering;
+        assertEquals(expectedWrites, tracker.writes.get());
+        assertEquals(expectedRows, tracker.writtenRows.get());
+        assertEquals(0, tracker.loggedWrites.get());
+
+        assertEquals(0, tracker.reads.get());
+        session.execute("SELECT * FROM " + table + " WHERE k = ?", 0);
+        assertEquals(1, tracker.reads.get());
+        assertEquals(clustering, tracker.readRows.get());
+        assertEquals(1, tracker.readPartitions.get());
+        assertEquals(1, tracker.replicaPlans.get());
+
+        assertEquals(0, tracker.rangeReads.get());
+        session.execute("SELECT * FROM " + table);
+        assertEquals(1, tracker.rangeReads.get());
+        assertEquals(clustering + keys * clustering, tracker.readRows.get());
+        assertEquals(1 + keys, tracker.readPartitions.get());
+        assertEquals(1 + 1, tracker.replicaPlans.get());
+
+        session.execute("UPDATE " + table + " SET v = ? WHERE k = ? AND c IN ?", 42, 0, Arrays.asList(0, 2, 3));
+        expectedWrites += 1; // We only did one more write ...
+        expectedRows += 3;   // ... but it updates 3 rows.
+        assertEquals(expectedWrites, tracker.writes.get());
+        assertEquals(expectedRows, tracker.writtenRows.get());
+        assertEquals(0, tracker.loggedWrites.get());
+    }
+
+    @Test
+    public void testReadQueryTracingWithStaticRowsClusteringColumnsAndRegularRows()
+    {
+        int keys = 4;
+        int clustering = 3;
+        String table = KEYSPACE + ".qit_read_static_clustering_regular";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, sv int static, PRIMARY KEY (k, c))");
+        for (int k = 0; k < keys; k++)
+        {
+            for (int c = 0; c < clustering; c++)
+            {
+                session.execute("INSERT INTO " + table + "(k, c, v, sv) values (?, ?, ?, ?)", k, c, k, k * 77);
+            }
+        }
+
+        assertEquals(0, tracker.reads.get());
+        session.execute("SELECT * FROM " + table + " WHERE k = ?", 0);
+        assertEquals(1, tracker.reads.get());
+        assertEquals(1 + clustering, tracker.readRows.get());
+        assertEquals(1, tracker.readPartitions.get());
+        assertEquals(1, tracker.replicaPlans.get());
+
+        // trigger failure when processing onRow(), query should succeed
+        tracker.failOnRowsRead = true;
+        session.execute("SELECT * FROM " + table + " WHERE k = ?", 0);
+        assertEquals(2, tracker.reads.get());
+        assertEquals(1 + clustering, tracker.readRows.get()); // not changed
+        assertEquals(2, tracker.readPartitions.get());
+        assertEquals(2, tracker.replicaPlans.get());
+    }
+
+    @Test
+    public void testRangeReadQueryTracingWithStaticRows()
+    {
+        int keys = 5;
+        int clustering = 3;
+        String table = KEYSPACE + ".qit_range_read_static";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, sv int static, PRIMARY KEY (k, c))");
+        for (int k = 0; k < keys; k++)
+        {
+            for (int c = 0; c < clustering; c++)
+            {
+                if (k % 2 == 1)
+                    session.execute("INSERT INTO " + table + "(k, c, v, sv) values (?, ?, ?, ?)", k, c, k, k * 77);
+                else
+                    session.execute("INSERT INTO " + table + "(k, c, v) values (?, ?, ?)", k, c, k);
+            }
+        }
+
+        assertEquals(0, tracker.rangeReads.get());
+        session.execute("SELECT * FROM " + table);
+        assertEquals(1, tracker.rangeReads.get());
+        assertEquals(keys / 2 + keys * clustering, tracker.readRows.get());
+        assertEquals(keys, tracker.readPartitions.get());
+        assertEquals(1, tracker.replicaPlans.get());
+    }
+
+
+    @Test
+    public void testLoggedBatchQueryTracing()
+    {
+        String table = KEYSPACE + ".qit_logged_batch";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, PRIMARY KEY (k, c))");
+
+        session.execute("BEGIN BATCH "
+                        + "INSERT INTO " + table + "(k, c, v) VALUES (0, 0, 0);"
+                        + "INSERT INTO " + table + "(k, c, v) VALUES (1, 1, 1);"
+                        + "INSERT INTO " + table + "(k, c, v) VALUES (2, 2, 2);"
+                        + "INSERT INTO " + table + "(k, c, v) VALUES (3, 3, 3);"
+                        + "APPLY BATCH");
+
+        assertEquals(1, tracker.writes.get());
+        assertEquals(1, tracker.loggedWrites.get());
+        assertEquals(4, tracker.writtenRows.get());
+
+        session.execute("BEGIN BATCH "
+                        + "INSERT INTO " + table + "(k, c, v) VALUES (4, 4, 4);"
+                        + "INSERT INTO " + table + "(k, c, v) VALUES (5, 5, 5);"
+                        + "APPLY BATCH");
+
+        assertEquals(2, tracker.writes.get());
+        assertEquals(2, tracker.loggedWrites.get());
+        assertEquals(6, tracker.writtenRows.get());
+    }
+
+    @Test
+    public void testLWTQueryTracing()
+    {
+        String table = KEYSPACE + ".qit_lwt";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, PRIMARY KEY (k, c))");
+
+        session.execute("INSERT INTO " + table + "(k, c, v) values (?, ?, ?)", 0, 0, 0);
+        session.execute("INSERT INTO " + table + "(k, c, v) values (?, ?, ?)", 0, 1, 1);
+        assertEquals(2, tracker.writes.get());
+        assertEquals(2, tracker.writtenRows.get());
+
+        session.execute("INSERT INTO " + table + "(k, c, v) values (?, ?, ?) IF NOT EXISTS", 0, 2, 2);
+        // This should apply, so on top of the lwt specific increases, we'll have a new written row (but no read
+        // row since while we will do a read, it will come up empty).
+        assertEquals(1, tracker.lwts.get());
+        assertEquals(0, tracker.nonAppliedLwts.get());
+        assertEquals(1, tracker.appliedLwts.get());
+        assertEquals(3, tracker.writtenRows.get());
+        assertEquals(0, tracker.readRows.get());
+        // The writes or reads shouldn't have changed though.
+        assertEquals(2, tracker.writes.get());
+        assertEquals(0, tracker.reads.get());
+        assertEquals(1, tracker.replicaPlans.get());
+
+        session.execute("INSERT INTO " + table + "(k, c, v) values (?, ?, ?) IF NOT EXISTS", 0, 2, 2);
+        // This should not apply now and on top of the lwt specific increases, we should have read 1 additional row.
+        assertEquals(2, tracker.lwts.get());
+        assertEquals(1, tracker.nonAppliedLwts.get());
+        assertEquals(1, tracker.appliedLwts.get());
+        assertEquals(3, tracker.writtenRows.get());
+        assertEquals(1, tracker.readRows.get());
+        // The writes or reads shouldn't have changed though.
+        assertEquals(2, tracker.writes.get());
+        assertEquals(0, tracker.reads.get());
+        assertEquals(2, tracker.replicaPlans.get());
+
+        // More complex LWT batch.
+        session.execute("BEGIN BATCH "
+                        + "UPDATE " + table + " SET v = 42 WHERE k = 0 AND c = 0 IF v = 0; "
+                        + "UPDATE " + table + " SET v = 42 WHERE k = 0 AND c = 1 IF v = 1; "
+                        + "APPLY BATCH");
+        // This should apply. Further this will have read 2 rows and written 2.
+        assertEquals(3, tracker.lwts.get());
+        assertEquals(1, tracker.nonAppliedLwts.get());
+        assertEquals(2, tracker.appliedLwts.get());
+        assertEquals(5, tracker.writtenRows.get());
+        // Still no updates of writes or reads expected.
+        assertEquals(2, tracker.writes.get());
+        assertEquals(0, tracker.reads.get());
+        assertEquals(3, tracker.readRows.get());
+        assertEquals(3, tracker.replicaPlans.get());
+
+
+        PreparedStatement statement = session.prepare("SELECT * FROM " + table + " WHERE k = ?")
+                                             .setConsistencyLevel(com.datastax.driver.core.ConsistencyLevel.SERIAL);
+        session.execute(statement.bind(0));
+        assertEquals(1, tracker.reads.get());
+        assertEquals(6, tracker.readRows.get());
+        assertEquals(4, tracker.replicaPlans.get());
+    }
+
+    @Test
+    @BMRule(name = "Simulate cas read failure",
+    targetClass = "StorageProxy",
+    targetMethod = "doPaxos",
+    targetLocation = "AT ENTRY",
+    action = "throw new org.apache.cassandra.exceptions.UnavailableException(\"msg\", org.apache.cassandra.db.ConsistencyLevel.SERIAL, 3, 1)")
+    public void testCasReadFailureCount()
+    {
+        String table = KEYSPACE + ".qit_cas_read_failure";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, PRIMARY KEY (k, c))");
+        assertEquals(0, tracker.errorReads.get());
+
+        PreparedStatement statement = session.prepare("SELECT * FROM " + table + " WHERE k = ?")
+                                             .setConsistencyLevel(com.datastax.driver.core.ConsistencyLevel.SERIAL);
+
+        try
+        {
+            session.execute(statement.bind(0));
+        }
+        catch (NoHostAvailableException ex)
+        { /* NOOP */ }
+
+        assertEquals(1, tracker.errorReads.get());
+        assertEquals(0, tracker.reads.get());
+        assertEquals(0, tracker.readRows.get());
+    }
+
+    @Test
+    @BMRule(name = "Simulate cas write failure",
+    targetClass = "StorageProxy",
+    targetMethod = "doPaxos",
+    targetLocation = "AT ENTRY",
+    action = "throw new org.apache.cassandra.exceptions.UnavailableException(\"msg\", org.apache.cassandra.db.ConsistencyLevel.SERIAL, 3, 1)")
+    public void testCasWriteFailureCount()
+    {
+        String table = KEYSPACE + ".qit_cas_write_failure";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, PRIMARY KEY (k, c))");
+
+        assertEquals(0, tracker.errorLwts.get());
+
+        try
+        {
+            session.execute("INSERT INTO " + table + "(k, c, v) values (?, ?, ?) IF NOT EXISTS", 0, 2, 2);
+        }
+        catch (NoHostAvailableException ex)
+        { /* NOOP */ }
+
+        assertEquals(1, tracker.errorLwts.get());
+        assertEquals(0, tracker.lwts.get());
+        assertEquals(0, tracker.appliedLwts.get());
+        assertEquals(0, tracker.nonAppliedLwts.get());
+    }
+
+    @Test
+    @BMRule(name = "Simulate fetching rows failure",
+    targetClass = "StorageProxy",
+    targetMethod = "fetchRows",
+    targetLocation = "AT ENTRY",
+    action = "throw new org.apache.cassandra.exceptions.UnavailableException(\"msg\", org.apache.cassandra.db.ConsistencyLevel.SERIAL, 3, 1)")
+    public void testReadFailureCount()
+    {
+        String table = KEYSPACE + ".qit_read_failure";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, PRIMARY KEY (k, c))");
+
+        assertEquals(0, tracker.errorReads.get());
+
+        try
+        {
+            session.execute("SELECT * FROM " + table + " WHERE k = ?", 0);
+        }
+        catch (NoHostAvailableException ex)
+        { /* NOOP */ }
+
+        assertEquals(1, tracker.errorReads.get());
+        assertEquals(0, tracker.reads.get());
+    }
+
+    @Test
+    @BMRule(name = "Simulate write failure",
+    targetClass = "StorageProxy",
+    targetMethod = "performWrite",
+    targetLocation = "AT ENTRY",
+    action = "throw new org.apache.cassandra.exceptions.UnavailableException(\"msg\", org.apache.cassandra.db.ConsistencyLevel.SERIAL, 3, 1)")
+    public void testWriteFailureCount()
+    {
+        String table = KEYSPACE + ".qit_write_failure";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, PRIMARY KEY (k, c))");
+        assertEquals(0, tracker.errorWrites.get());
+
+        try
+        {
+            session.execute("INSERT INTO " + table + "(k, c, v) values (?, ?, ?)", 0, 0, 0);
+        }
+        catch (NoHostAvailableException ex)
+        { /* NOOP */ }
+
+        assertEquals(1, tracker.errorWrites.get());
+        assertEquals(0, tracker.writes.get());
+    }
+
+    @Test
+    @BMRule(name = "Simulate write batch failure",
+    targetClass = "StorageProxy",
+    targetMethod = "asyncWriteBatchedMutations",
+    targetLocation = "AT ENTRY",
+    action = "throw new org.apache.cassandra.exceptions.UnavailableException(\"msg\", org.apache.cassandra.db.ConsistencyLevel.SERIAL, 3, 1)")
+    public void testReadBatchFailureCount()
+    {
+        String table = KEYSPACE + ".qit_write_batch_failure";
+        session.execute("CREATE TABLE " + table + "(k int, c int, v int, PRIMARY KEY (k, c))");
+
+        assertEquals(0, tracker.errorWrites.get());
+
+        try
+        {
+            session.execute("BEGIN BATCH "
+                            + "INSERT INTO " + table + "(k, c, v) VALUES (0, 0, 0);"
+                            + "INSERT INTO " + table + "(k, c, v) VALUES (1, 1, 1);"
+                            + "APPLY BATCH");
+        }
+        catch (NoHostAvailableException ex)
+        { /* NOOP */ }
+
+        assertEquals(1, tracker.errorWrites.get());
+        assertEquals(0, tracker.reads.get());
+    }
+
+    @Test
+    public void testReplicaFilteringProtection() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT, v2 TEXT)");
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+        flush();
+
+        StorageProxy.instance.registerQueryTracker(new QueryInfoTrackerTest.TestQueryInfoTracker(KEYSPACE));
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0 ALLOW FILTERING");
+        assertEquals(2, rows.all().size());
+
+        QueryInfoTrackerTest.TestQueryInfoTracker queryInfoTracker = (QueryInfoTrackerTest.TestQueryInfoTracker)
+                                                                     StorageProxy.queryTracker();
+
+        assertEquals(1, queryInfoTracker.rangeReads.get());
+        assertEquals(2, queryInfoTracker.readRows.get());
+        assertEquals(1, queryInfoTracker.replicaPlans.get());
+    }
+
+    public static class TestQueryInfoTracker implements QueryInfoTracker, Serializable
+    {
+        public final AtomicInteger writes = new AtomicInteger();
+        public final AtomicInteger loggedWrites = new AtomicInteger();
+        public final AtomicInteger writtenRows = new AtomicInteger();
+        public final AtomicInteger errorWrites = new AtomicInteger();
+
+        public final AtomicInteger reads = new AtomicInteger();
+        public final AtomicInteger rangeReads = new AtomicInteger();
+        public final AtomicInteger readRows = new AtomicInteger();
+        public final AtomicInteger readPartitions = new AtomicInteger();
+        public final AtomicInteger readFilteredPartitions = new AtomicInteger();
+        public final AtomicInteger readFilteredRows = new AtomicInteger();
+
+        public final AtomicInteger errorReads = new AtomicInteger();
+        public final AtomicInteger replicaPlans = new AtomicInteger();
+
+        public final AtomicInteger lwts = new AtomicInteger();
+        public final AtomicInteger nonAppliedLwts = new AtomicInteger();
+        public final AtomicInteger appliedLwts = new AtomicInteger();
+        public final AtomicInteger errorLwts = new AtomicInteger();
+        private final String keyspace;
+
+        public volatile boolean failOnRowsRead = false;
+
+        public TestQueryInfoTracker(String keyspace)
+        {
+            this.keyspace = keyspace;
+        }
+
+        private boolean shouldIgnore(TableMetadata table)
+        {
+            // We exclude anything that isn't on our test keyspace to be sure no "system" query interferes.
+            return !table.keyspace.equals(keyspace);
+        }
+
+        private TableMetadata extractTable(Collection<? extends IMutation> mutations)
+        {
+            return Iterables.getLast(mutations).getPartitionUpdates().iterator().next().metadata();
+        }
+
+        @Override
+        public WriteTracker onWrite(ClientState state,
+                                    boolean isLogged,
+                                    Collection<? extends IMutation> mutations,
+                                    ConsistencyLevel consistencyLevel)
+        {
+            if (shouldIgnore(extractTable(mutations)))
+                return WriteTracker.NOOP;
+
+            return new WriteTracker()
+            {
+                @Override
+                public void onDone()
+                {
+                    writes.incrementAndGet();
+                    if (isLogged)
+                        loggedWrites.incrementAndGet();
+                    for (IMutation mutation : mutations)
+                    {
+                        for (PartitionUpdate update : mutation.getPartitionUpdates())
+                        {
+                            writtenRows.addAndGet(update.rowCount());
+                        }
+                    }
+                }
+
+                @Override
+                public void onError(Throwable exception)
+                {
+                    errorWrites.incrementAndGet();
+                }
+            };
+        }
+
+        @Override
+        public ReadTracker onRead(ClientState state,
+                                  TableMetadata table,
+                                  List<SinglePartitionReadCommand> commands,
+                                  ConsistencyLevel consistencyLevel)
+        {
+            if (shouldIgnore(table))
+                return ReadTracker.NOOP;
+            return new TestReadTracker(table);
+        }
+
+        @Override
+        public ReadTracker onRangeRead(ClientState state,
+                                       TableMetadata table,
+                                       PartitionRangeReadCommand command,
+                                       ConsistencyLevel consistencyLevel)
+        {
+            if (shouldIgnore(table))
+                return ReadTracker.NOOP;
+            return new TestRangeReadTracker(table);
+        }
+
+        @Override
+        public LWTWriteTracker onLWTWrite(ClientState state,
+                                          TableMetadata table,
+                                          DecoratedKey key,
+                                          ConsistencyLevel serialConsistency,
+                                          ConsistencyLevel commitConsistency)
+        {
+            return new TestLWTWriteTracker();
+        }
+
+
+        private class TestReadTracker implements ReadTracker
+        {
+            private TableMetadata table;
+
+            private TestReadTracker(TableMetadata table)
+            {
+                this.table = table;
+            }
+
+            @Override
+            public void onDone()
+            {
+                reads.incrementAndGet();
+            }
+
+            @Override
+            public void onError(Throwable exception)
+            {
+                errorReads.incrementAndGet();
+            }
+
+            @Override
+            public void onReplicaPlan(ReplicaPlan.ForRead<?, ?> replicaPlan)
+            {
+                replicaPlans.incrementAndGet();
+            }
+
+            @Override
+            public void onPartition(DecoratedKey partitionKey)
+            {
+                readPartitions.incrementAndGet();
+            }
+
+            @Override
+            public void onRow(Row row)
+            {
+                if (failOnRowsRead)
+                    throw new RuntimeException("test failure");
+
+                readRows.incrementAndGet();
+            }
+
+            @Override
+            public void onFilteredPartition(DecoratedKey partitionKey)
+            {
+                readFilteredPartitions.incrementAndGet();
+            }
+
+            @Override
+            public void onFilteredRow(Row row)
+            {
+                readFilteredRows.incrementAndGet();
+            }
+        }
+
+        private class TestRangeReadTracker implements ReadTracker
+        {
+            private final TableMetadata table;
+
+            private TestRangeReadTracker(TableMetadata table)
+            {
+                this.table = table;
+            }
+
+            @Override
+            public void onDone()
+            {
+                rangeReads.incrementAndGet();
+            }
+
+            @Override
+            public void onError(Throwable exception)
+            {
+                errorReads.incrementAndGet();
+            }
+
+            @Override
+            public void onReplicaPlan(ReplicaPlan.ForRead<?, ?> replicaPlan)
+            {
+                replicaPlans.incrementAndGet();
+            }
+
+            @Override
+            public void onPartition(DecoratedKey partitionKey)
+            {
+                readPartitions.incrementAndGet();
+            }
+
+            @Override
+            public void onRow(Row row)
+            {
+                readRows.incrementAndGet();
+            }
+
+            @Override
+            public void onFilteredPartition(DecoratedKey partitionKey)
+            {
+                logger.info("range read: filtered partition {}", partitionKey);
+                readFilteredPartitions.incrementAndGet();
+            }
+
+            @Override
+            public void onFilteredRow(Row row)
+            {
+                logger.info("range read: filtered row {}", row.toString(table, true));
+                readFilteredRows.incrementAndGet();
+            }
+        }
+
+        private class TestLWTWriteTracker implements LWTWriteTracker
+        {
+            @Override
+            public void onDone()
+            {
+                lwts.incrementAndGet();
+            }
+
+            @Override
+            public void onError(Throwable exception)
+            {
+                errorLwts.incrementAndGet();
+            }
+
+            @Override
+            public void onNotApplied()
+            {
+                nonAppliedLwts.incrementAndGet();
+            }
+
+            @Override
+            public void onApplied(PartitionUpdate update)
+            {
+                appliedLwts.incrementAndGet();
+                writtenRows.addAndGet(update.rowCount());
+            }
+
+            @Override
+            public void onReplicaPlan(ReplicaPlan.ForRead<?, ?> replicaPlan)
+            {
+                replicaPlans.incrementAndGet();
+            }
+
+            @Override
+            public void onPartition(DecoratedKey partitionKey)
+            {
+                readPartitions.incrementAndGet();
+            }
+
+            @Override
+            public void onRow(Row row)
+            {
+                readRows.incrementAndGet();
+            }
+
+            @Override
+            public void onFilteredPartition(DecoratedKey partitionKey)
+            {
+                readFilteredPartitions.incrementAndGet();
+            }
+
+            @Override
+            public void onFilteredRow(Row row)
+            {
+                readFilteredRows.incrementAndGet();
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/service/QueryPagerTest.java b/test/unit/org/apache/cassandra/service/QueryPagerTest.java
index 0c87e569f8e9..e8256e3d4c24 100644
--- a/test/unit/org/apache/cassandra/service/QueryPagerTest.java
+++ b/test/unit/org/apache/cassandra/service/QueryPagerTest.java
@@ -1,62 +1,110 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*    http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied.  See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package org.apache.cassandra.service;
 
 import java.nio.ByteBuffer;
 import java.nio.charset.CharacterCodingException;
-import java.util.*;
-
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.junit.After;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.*;
-import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
 import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.cql3.PageSize;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.AbstractReadCommandBuilder;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionRangeReadQuery;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand.Group;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.partitions.FilteredPartition;
-import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.pager.AggregationQueryPager;
+import org.apache.cassandra.service.pager.MultiPartitionPager;
 import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.service.pager.PartitionRangeQueryPager;
+import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.service.pager.SinglePartitionPager;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.assertj.core.api.Assertions;
 
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class QueryPagerTest
 {
+    private final static Logger logger = LoggerFactory.getLogger(QueryPagerTest.class);
+
     public static final String KEYSPACE1 = "QueryPagerTest";
     public static final String CF_STANDARD = "Standard1";
     public static final String KEYSPACE_CQL = "cql_keyspace";
     public static final String CF_CQL = "table2";
     public static final String CF_CQL_WITH_STATIC = "with_static";
     public static final long nowInSec = FBUtilities.nowInSeconds();
+    public static List<String> tokenOrderedKeys;
 
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
@@ -75,12 +123,12 @@ public static void defineSchema() throws ConfigurationException
                                                                + "v text,"
                                                                + "PRIMARY KEY (k, c))", KEYSPACE_CQL),
                                     CreateTableStatement.parse("CREATE TABLE " + CF_CQL_WITH_STATIC + " ("
-                                                               + "pk text, "
-                                                               + "ck int, "
+                                                               + "k text, "
+                                                               + "c text, "
                                                                + "st int static, "
                                                                + "v1 int, "
                                                                + "v2 int, "
-                                                               + "PRIMARY KEY(pk, ck))", KEYSPACE_CQL));
+                                                               + "PRIMARY KEY(k, c))", KEYSPACE_CQL));
         addData();
     }
 
@@ -98,11 +146,13 @@ private static String string(ByteBuffer bb)
 
     public static void addData()
     {
-        cfs().clearUnsafe();
+        cfs(KEYSPACE1, CF_STANDARD).clearUnsafe();
 
         int nbKeys = 10;
         int nbCols = 10;
 
+        SortedSet<String> tokens = Sets.newTreeSet(Comparator.comparing(a -> cfs(KEYSPACE1, CF_STANDARD).getPartitioner().decorateKey(bytes(a))));
+
         // *
         // * Creates the following data:
         // *   k1: c1 ... cn
@@ -113,15 +163,24 @@ public static void addData()
         {
             for (int j = 0; j < nbCols; j++)
             {
-                RowUpdateBuilder builder = new RowUpdateBuilder(cfs().metadata(), FBUtilities.timestampMicros(), "k" + i);
+                tokens.add("k" + i);
+                RowUpdateBuilder builder = new RowUpdateBuilder(cfs(KEYSPACE1, CF_STANDARD).metadata(), FBUtilities.timestampMicros(), "k" + i);
                 builder.clustering("c" + j).add("val", "").build().applyUnsafe();
             }
         }
+
+        tokenOrderedKeys = Lists.newArrayList(tokens);
+    }
+
+    @After
+    public void cleanUp()
+    {
+        QueryProcessor.executeInternal(String.format("TRUNCATE \"%s\".\"%s\"", KEYSPACE_CQL, CF_CQL_WITH_STATIC));
     }
 
-    private static ColumnFamilyStore cfs()
+    private static ColumnFamilyStore cfs(String ks, String cf)
     {
-        return Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
+        return Keyspace.open(ks).getColumnFamilyStore(cf);
     }
 
     private static List<FilteredPartition> query(QueryPager pager, int expectedSize)
@@ -135,7 +194,7 @@ private static List<FilteredPartition> query(QueryPager pager, int toQuery, int
         List<FilteredPartition> partitionList = new ArrayList<>();
         int rows = 0;
         try (ReadExecutionController executionController = pager.executionController();
-             PartitionIterator iterator = pager.fetchPageInternal(toQuery, executionController))
+             PartitionIterator iterator = pager.fetchPageInternal(PageSize.inRows(toQuery), executionController))
         {
             while (iterator.hasNext())
             {
@@ -152,51 +211,119 @@ private static List<FilteredPartition> query(QueryPager pager, int toQuery, int
         return partitionList;
     }
 
-    private static ReadCommand namesQuery(String key, String... names)
+    private static Map<DecoratedKey, List<Row>> fetchPage(QueryPager pager, int pageSize, PageSize.PageUnit pageUnit)
     {
-        AbstractReadCommandBuilder builder = Util.cmd(cfs(), key);
+        logger.info("----------------------------------------------------------------");
+        Map<DecoratedKey, List<Row>> ret = Maps.newHashMap();
+        try (ReadExecutionController ec = pager.executionController();
+             PartitionIterator iterator = pager.fetchPageInternal(new PageSize(pageSize, pageUnit), ec))
+        {
+            while (iterator.hasNext())
+            {
+                try (RowIterator partition = iterator.next())
+                {
+                    logger.info("Partition {}", partition.partitionKey());
+                    List<Row> rows = new ArrayList<>();
+                    Row staticRow = partition.staticRow();
+                    if (!partition.hasNext() && !staticRow.isEmpty())
+                    {
+                        rows.add(staticRow);
+                        logger.info("\tStatic row {}", staticRow.toString(partition.metadata()));
+                    }
+
+                    while (partition.hasNext())
+                    {
+                        Row row = partition.next();
+                        rows.add(row);
+                        logger.info("\tRow {}", row.toString(partition.metadata()));
+                    }
+
+                    ret.put(partition.partitionKey(), rows);
+                }
+            }
+        }
+        catch (Throwable t)
+        {
+            t.printStackTrace();
+            throw t;
+        }
+        return ret;
+    }
+
+    private static ReadCommand namesQuery(int count, int partitionCount, PageSize pageSize, ColumnFamilyStore cfs, String key, String... names)
+    {
+        AbstractReadCommandBuilder builder = Util.cmd(cfs, key).withNowInSeconds(nowInSec);
         for (String name : names)
             builder.includeRow(name);
-        return builder.withPagingLimit(100).build();
+        if (count > 0)
+            builder.withLimit(count);
+        if (partitionCount > 0)
+            builder.withPerPartitionLimit(partitionCount);
+        if (pageSize != null && !pageSize.equals(PageSize.NONE))
+            builder.withPageSize(pageSize);
+
+        return builder.build();
     }
 
-    private static SinglePartitionReadCommand sliceQuery(String key, String start, String end, int count)
+    private static SinglePartitionReadCommand sliceQuery(ColumnFamilyStore cfs, String key, String start, String end)
     {
-        return sliceQuery(key, start, end, false, count);
+        return sliceQuery(-1, -1, PageSize.NONE, cfs, key, start, end, false);
     }
 
-    private static SinglePartitionReadCommand sliceQuery(String key, String start, String end, boolean reversed, int count)
+    private static SinglePartitionReadCommand sliceQuery(ColumnFamilyStore cfs, String key, String start, String end, boolean reversed)
     {
-        ClusteringComparator cmp = cfs().getComparator();
-        TableMetadata metadata = cfs().metadata();
-
-        Slice slice = Slice.make(cmp.make(start), cmp.make(end));
-        ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(Slices.with(cmp, slice), reversed);
+        return sliceQuery(-1, -1, PageSize.NONE, cfs, key, start, end, reversed);
+    }
 
-        return SinglePartitionReadCommand.create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.none(), DataLimits.NONE, Util.dk(key), filter);
+    private static SinglePartitionReadCommand sliceQuery(int count, int partitionCount, PageSize paging, ColumnFamilyStore cfs, String key, String start, String end, boolean reversed)
+    {
+        AbstractReadCommandBuilder builder = Util.cmd(cfs, key).fromIncl(start).toIncl(end).withNowInSeconds(nowInSec);
+        if (reversed)
+            builder.reverse();
+        if (count > 0)
+            builder.withLimit(count);
+        if (partitionCount > 0)
+            builder.withPerPartitionLimit(partitionCount);
+        if (paging != null && !paging.equals(PageSize.NONE))
+            builder.withPageSize(paging);
+
+        return (SinglePartitionReadCommand) builder.build();
     }
 
-    private static ReadCommand rangeNamesQuery(String keyStart, String keyEnd, int count, String... names)
+    private static ReadCommand rangeNamesQuery(int count, int partitionCount, PageSize paging, ColumnFamilyStore cfs, String keyStart, String keyEnd, String... names)
     {
-        AbstractReadCommandBuilder builder = Util.cmd(cfs())
+        AbstractReadCommandBuilder builder = Util.cmd(cfs)
                                                  .fromKeyExcl(keyStart)
                                                  .toKeyIncl(keyEnd)
-                                                 .withPagingLimit(count);
+                                                 .withNowInSeconds(nowInSec);
         for (String name : names)
             builder.includeRow(name);
+        if (count > 0)
+            builder.withLimit(count);
+        if (partitionCount > 0)
+            builder.withPerPartitionLimit(partitionCount);
+        if (paging != null && !paging.equals(PageSize.NONE))
+            builder.withPageSize(paging);
 
         return builder.build();
     }
 
-    private static ReadCommand rangeSliceQuery(String keyStart, String keyEnd, int count, String start, String end)
+    private static ReadCommand rangeSliceQuery(int count, int partitionCount, PageSize paging, ColumnFamilyStore cfs, String keyStart, String keyEnd, String start, String end)
     {
-        return Util.cmd(cfs())
-                   .fromKeyExcl(keyStart)
-                   .toKeyIncl(keyEnd)
-                   .fromIncl(start)
-                   .toIncl(end)
-                   .withPagingLimit(count)
-                   .build();
+        AbstractReadCommandBuilder builder = Util.cmd(cfs)
+                                                 .fromKeyExcl(keyStart)
+                                                 .toKeyIncl(keyEnd)
+                                                 .fromIncl(start)
+                                                 .toIncl(end)
+                                                 .withNowInSeconds(nowInSec);
+        if (count > 0)
+            builder.withLimit(count);
+        if (partitionCount > 0)
+            builder.withPerPartitionLimit(partitionCount);
+        if (paging != null && !paging.equals(PageSize.NONE))
+            builder.withPageSize(paging);
+
+        return builder.build();
     }
 
     private static void assertRow(FilteredPartition r, String key, String... names)
@@ -212,10 +339,10 @@ private static void assertRow(FilteredPartition partition, String key, ByteBuffe
         assertEquals(key, string(partition.partitionKey().getKey()));
         assertFalse(partition.isEmpty());
         int i = 0;
-        for (Row row : Util.once(partition.iterator()))
+        for (Row row : Util.once(partition.rowIterator()))
         {
             ByteBuffer expected = names[i++];
-            assertEquals("column " + i + " doesn't match "+string(expected)+" vs "+string(row.clustering().bufferAt(0)), expected, row.clustering().bufferAt(0));
+            assertEquals("column " + i + " doesn't match " + string(expected) + " vs " + string(row.clustering().bufferAt(0)), expected, row.clustering().bufferAt(0));
         }
     }
 
@@ -231,13 +358,16 @@ private QueryPager maybeRecreate(QueryPager pager, ReadQuery command, boolean te
     @Test
     public void namesQueryTest()
     {
-        for(ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
+        for (ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
             namesQueryTest(protocolVersion);
     }
 
     public void namesQueryTest(ProtocolVersion protocolVersion)
     {
-        QueryPager pager = namesQuery("k0", "c1", "c5", "c7", "c8").getPager(null, protocolVersion);
+        QueryPager pager = namesQuery(-1, -1, new PageSize(100, PageSize.PageUnit.ROWS),
+                                      cfs(KEYSPACE1, CF_STANDARD),
+                                      "k0", "c1", "c5", "c7", "c8")
+                           .getPager(null, protocolVersion);
 
         assertFalse(pager.isExhausted());
         List<FilteredPartition> partition = query(pager, 5, 4);
@@ -249,7 +379,7 @@ public void namesQueryTest(ProtocolVersion protocolVersion)
     @Test
     public void sliceQueryTest()
     {
-        for(ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
+        for (ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
         {
             sliceQueryTest(false, protocolVersion);
             sliceQueryTest(true, protocolVersion);
@@ -258,7 +388,7 @@ public void sliceQueryTest()
 
     public void sliceQueryTest(boolean testPagingState, ProtocolVersion protocolVersion)
     {
-        ReadCommand command = sliceQuery("k0", "c1", "c8", 10);
+        ReadCommand command = sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k0", "c1", "c8");
         QueryPager pager = command.getPager(null, protocolVersion);
 
         assertFalse(pager.isExhausted());
@@ -280,10 +410,55 @@ public void sliceQueryTest(boolean testPagingState, ProtocolVersion protocolVers
         assertTrue(pager.isExhausted());
     }
 
+    @Test
+    public void sliceQueryWithLimitsTest() throws Exception
+    {
+        boolean testPagingState = true;
+        ProtocolVersion protocolVersion = ProtocolVersion.CURRENT;
+
+        // Test with count < partitionCount
+
+        int count = 1;
+        int partitionCount = 2;
+
+        ReadCommand command = sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k0", "c1", "c8", false);
+        QueryPager pager = command.getPager(null, protocolVersion);
+        List<FilteredPartition> partition = query(pager, 3, count);
+        assertRow(partition.get(0), "k0", "c1");
+        assertTrue(pager.isExhausted());
+
+        // Test with count > partitionCount
+
+        count = 2;
+        partitionCount = 1;
+
+        command = sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k0", "c1", "c8", false);
+        pager = command.getPager(null, protocolVersion);
+        partition = query(pager, 3, partitionCount);
+        assertRow(partition.get(0), "k0", "c1");
+        assertTrue(pager.isExhausted());
+
+        // Test with counts spanning multiple pages
+
+        count = 5;
+        partitionCount = 5;
+
+        command = sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k0", "c1", "c8", false);
+        pager = command.getPager(null, protocolVersion);
+        partition = query(pager, 3, 3);
+        assertRow(partition.get(0), "k0", "c1", "c2", "c3");
+        assertFalse(pager.isExhausted());
+
+        pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
+        partition = query(pager, 3, 2);
+        assertRow(partition.get(0), "k0", "c4", "c5");
+        assertTrue(pager.isExhausted());
+    }
+
     @Test
     public void reversedSliceQueryTest()
     {
-        for(ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
+        for (ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
         {
             reversedSliceQueryTest(false, protocolVersion);
             reversedSliceQueryTest(true, protocolVersion);
@@ -292,7 +467,7 @@ public void reversedSliceQueryTest()
 
     public void reversedSliceQueryTest(boolean testPagingState, ProtocolVersion protocolVersion)
     {
-        ReadCommand command = sliceQuery("k0", "c1", "c8", true, 10);
+        ReadCommand command = sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k0", "c1", "c8", true);
         QueryPager pager = command.getPager(null, protocolVersion);
 
         assertFalse(pager.isExhausted());
@@ -317,7 +492,7 @@ public void reversedSliceQueryTest(boolean testPagingState, ProtocolVersion prot
     @Test
     public void multiQueryTest()
     {
-        for(ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
+        for (ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
         {
             multiQueryTest(false, protocolVersion);
             multiQueryTest(true, protocolVersion);
@@ -328,8 +503,8 @@ public void multiQueryTest(boolean testPagingState, ProtocolVersion protocolVers
     {
         ReadQuery command = SinglePartitionReadCommand.Group.create(new ArrayList<SinglePartitionReadCommand>()
         {{
-            add(sliceQuery("k1", "c2", "c6", 10));
-            add(sliceQuery("k4", "c3", "c5", 10));
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k1", "c2", "c6"));
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k4", "c3", "c5"));
         }}, DataLimits.NONE);
         QueryPager pager = command.getPager(null, protocolVersion);
 
@@ -340,7 +515,7 @@ public void multiQueryTest(boolean testPagingState, ProtocolVersion protocolVers
 
         pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
         assertFalse(pager.isExhausted());
-        partition = query(pager , 4);
+        partition = query(pager, 4);
         assertRow(partition.get(0), "k1", "c5", "c6");
         assertRow(partition.get(1), "k4", "c3", "c4");
         assertFalse(pager.isExhausted());
@@ -353,10 +528,155 @@ public void multiQueryTest(boolean testPagingState, ProtocolVersion protocolVers
         assertTrue(pager.isExhausted());
     }
 
+    /**
+     * Test a query with 1 CQL row per partition with various page sizes.
+     */
+    @Test
+    public void multiPartitionSingleRowQueryTest() throws Exception
+    {
+        int totQueryRows = 4;
+        ReadQuery command = Group.create(new ArrayList<SinglePartitionReadCommand>()
+        {{
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k1", "c1", "c1"));
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k2", "c1", "c1"));
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k3", "c1", "c1"));
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k4", "c1", "c1"));
+        }}, DataLimits.NONE);
+
+        checkRows(command, PageSize.PageUnit.ROWS, totQueryRows, new int[]{ 7, 8, 9, 10, 15, 16, 20 });
+    }
+
+    /**
+     * Test a query with 4 CQL rows per partition with various page sizes.
+     */
+    @Test
+    public void multiPartitionFourRowsQueryTest() throws Exception
+    {
+        int totQueryRows = 8;
+        ReadQuery command = Group.create(new ArrayList<SinglePartitionReadCommand>()
+        {{
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k1", "c1", "c4"));
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k2", "c1", "c4"));
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k3", "c1", "c4"));
+            add(sliceQuery(cfs(KEYSPACE1, CF_STANDARD), "k4", "c1", "c4"));
+        }}, DataLimits.cqlLimits(8));
+
+        checkRows(command, PageSize.PageUnit.ROWS, totQueryRows, new int[]{ 2, 7, 8, 9, 10, 15, 16, 20 });
+    }
+
+    @Test
+    public void multiPartitionQueryWithRowLimitTest() throws Exception
+    {
+        int count = 8;
+        int partitionCount = DataLimits.NO_LIMIT;
+        int totQueryRows = 8;
+        ReadQuery command = Group.create(new ArrayList<SinglePartitionReadCommand>()
+        {{
+            add(sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k1", "c1", "c4", false));
+            add(sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k2", "c1", "c4", false));
+            add(sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k3", "c1", "c4", false));
+            add(sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k4", "c1", "c4", false));
+        }}, DataLimits.cqlLimits(count, partitionCount));
+
+        checkRows(command, PageSize.PageUnit.ROWS, totQueryRows, new int[]{ 2, 7, 8, 9, 10, 15, 16, 20 });
+    }
+
+    @Test
+    public void multiPartitionQueryWithPartitionLimitTest() throws Exception
+    {
+        int count = DataLimits.NO_LIMIT;
+        int partitionCount = 2;
+        int totQueryRows = 8;
+        ReadQuery command = Group.create(new ArrayList<SinglePartitionReadCommand>()
+        {{
+            add(sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k1", "c1", "c4", false));
+            add(sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k2", "c1", "c4", false));
+            add(sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k3", "c1", "c4", false));
+            add(sliceQuery(count, partitionCount, PageSize.NONE, cfs(KEYSPACE1, CF_STANDARD), "k4", "c1", "c4", false));
+        }}, DataLimits.cqlLimits(count, partitionCount));
+
+        checkRows(command, PageSize.PageUnit.ROWS, totQueryRows, new int[]{ 2, 7, 8, 9, 10, 15, 16, 20 });
+    }
+
+    private void checkRows(ReadQuery command, PageSize.PageUnit pageUnit, int totQueryRows, int... pages)
+    {
+        for (int pageSize : pages)
+        {
+            Map<DecoratedKey, Set<Row>> allRows = Maps.newHashMap();
+            int currentRows = 0;
+            QueryPager pager = command.getPager(null, ProtocolVersion.CURRENT);
+            assertFalse(String.format("Failed due to exhausted pager at page size %s %s", pageSize, pageUnit),
+                        pager.isExhausted());
+
+            logger.info("Testing with page size: {}", pageSize);
+            while (!pager.isExhausted())
+            {
+                Map<DecoratedKey, List<Row>> rows = fetchPage(pager, pageSize, pageUnit);
+
+                if (rows.size() > 0)
+                {
+                    int numRows = rows.values().stream().map(List::size).reduce(0, Integer::sum);
+                    int numBytes = rows.values().stream().flatMap(r -> r.stream()).reduce(0, (s, r) -> s + r.dataSize(), Integer::sum);
+
+                    for (Map.Entry<DecoratedKey, List<Row>> entry : rows.entrySet())
+                        allRows.merge(entry.getKey(), new HashSet(entry.getValue()), ((rows1, rows2) -> {
+                            rows1.addAll(rows2);
+                            return rows1;
+                        }));
+
+                    if (pageUnit == PageSize.PageUnit.ROWS)
+                    {
+                        int expectedSize = Math.min(pageSize, totQueryRows - currentRows);
+                        assertEquals(String.format("Failed after %d rows with rows page size %d and current number of rows %d;\n%s",
+                                                   currentRows, pageSize, numRows, formatRows(allRows)),
+                                     expectedSize, numRows);
+                    }
+                    else
+                    {
+                        boolean bytesRead = numBytes < (pageSize + (numBytes / numRows));
+                        assertTrue(String.format("Failed after %d rows with bytes page size %d and current number of rows %d due to bytes read %d;\n%s",
+                                                 currentRows, pageSize, numRows, numBytes, formatRows(allRows)),
+                                   bytesRead);
+                    }
+
+                    currentRows += numRows;
+
+                    if (!pager.isExhausted())
+                        pager = maybeRecreate(pager, command, true, ProtocolVersion.CURRENT);
+                }
+                else
+                    assertTrue(String.format("Failed due to non-exhausted pager at page size %s %s", pageSize, pageUnit),
+                               pager.isExhausted());
+            }
+
+            assertEquals(String.format("Failed with page size %d %s - expected %d rows in total but got:\n%s",
+                                       pageSize, pageUnit, totQueryRows, formatRows(allRows)),
+                         totQueryRows, (long) allRows.values().stream().map(Set::size).reduce(0, Integer::sum));
+        }
+    }
+
+    private String formatRows(Map<DecoratedKey, Set<Row>> rows)
+    {
+        TableMetadata metadata = cfs(KEYSPACE1, CF_STANDARD).metadata();
+
+        StringBuilder str = new StringBuilder();
+        for (Map.Entry<DecoratedKey, Set<Row>> entry : rows.entrySet())
+        {
+            for (Row row : entry.getValue())
+            {
+                str.append(entry.getKey().toString());
+                str.append(' ');
+                str.append(row.toString(metadata));
+                str.append('\n');
+            }
+        }
+        return str.toString();
+    }
+
     @Test
     public void rangeNamesQueryTest()
     {
-        for(ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
+        for (ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
         {
             rangeNamesQueryTest(false, protocolVersion);
             rangeNamesQueryTest(true, protocolVersion);
@@ -365,20 +685,23 @@ public void rangeNamesQueryTest()
 
     public void rangeNamesQueryTest(boolean testPagingState, ProtocolVersion protocolVersion)
     {
-        ReadCommand command = rangeNamesQuery("k0", "k5", 100, "c1", "c4", "c8");
+        ReadCommand command = rangeNamesQuery(-1, -1, new PageSize(100, PageSize.PageUnit.ROWS),
+                                              cfs(KEYSPACE1, CF_STANDARD),
+                                              tokenOrderedKeys.get(0), tokenOrderedKeys.get(5),
+                                              "c1", "c4", "c8");
         QueryPager pager = command.getPager(null, protocolVersion);
 
         assertFalse(pager.isExhausted());
         List<FilteredPartition> partitions = query(pager, 3 * 3);
         for (int i = 1; i <= 3; i++)
-            assertRow(partitions.get(i-1), "k" + i, "c1", "c4", "c8");
+            assertRow(partitions.get(i - 1), tokenOrderedKeys.get(i), "c1", "c4", "c8");
         assertFalse(pager.isExhausted());
 
         pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
         assertFalse(pager.isExhausted());
         partitions = query(pager, 3 * 3, 2 * 3);
         for (int i = 4; i <= 5; i++)
-            assertRow(partitions.get(i-4), "k" + i, "c1", "c4", "c8");
+            assertRow(partitions.get(i - 4), tokenOrderedKeys.get(i), "c1", "c4", "c8");
 
         assertTrue(pager.isExhausted());
     }
@@ -386,7 +709,7 @@ public void rangeNamesQueryTest(boolean testPagingState, ProtocolVersion protoco
     @Test
     public void rangeSliceQueryTest()
     {
-        for(ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
+        for (ProtocolVersion protocolVersion : ProtocolVersion.SUPPORTED)
         {
             rangeSliceQueryTest(false, protocolVersion);
             rangeSliceQueryTest(true, protocolVersion);
@@ -395,61 +718,146 @@ public void rangeSliceQueryTest()
 
     public void rangeSliceQueryTest(boolean testPagingState, ProtocolVersion protocolVersion)
     {
-        ReadCommand command = rangeSliceQuery("k1", "k5", 100, "c1", "c7");
+        ReadCommand command = rangeSliceQuery(-1, -1, new PageSize(100, PageSize.PageUnit.ROWS),
+                                              cfs(KEYSPACE1, CF_STANDARD),
+                                              tokenOrderedKeys.get(0), tokenOrderedKeys.get(4),
+                                              "c1", "c7");
         QueryPager pager = command.getPager(null, protocolVersion);
 
         assertFalse(pager.isExhausted());
         List<FilteredPartition> partitions = query(pager, 5);
-        assertRow(partitions.get(0), "k2", "c1", "c2", "c3", "c4", "c5");
+        assertRow(partitions.get(0), tokenOrderedKeys.get(1), "c1", "c2", "c3", "c4", "c5");
         assertFalse(pager.isExhausted());
 
         pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
         assertFalse(pager.isExhausted());
         partitions = query(pager, 4);
-        assertRow(partitions.get(0), "k2", "c6", "c7");
-        assertRow(partitions.get(1), "k3", "c1", "c2");
+        assertRow(partitions.get(0), tokenOrderedKeys.get(1), "c6", "c7");
+        assertRow(partitions.get(1), tokenOrderedKeys.get(2), "c1", "c2");
         assertFalse(pager.isExhausted());
 
         pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
         assertFalse(pager.isExhausted());
         partitions = query(pager, 6);
-        assertRow(partitions.get(0), "k3", "c3", "c4", "c5", "c6", "c7");
-        assertRow(partitions.get(1), "k4", "c1");
+        assertRow(partitions.get(0), tokenOrderedKeys.get(2), "c3", "c4", "c5", "c6", "c7");
+        assertRow(partitions.get(1), tokenOrderedKeys.get(3), "c1");
         assertFalse(pager.isExhausted());
 
         pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
         assertFalse(pager.isExhausted());
         partitions = query(pager, 5);
-        assertRow(partitions.get(0), "k4", "c2", "c3", "c4", "c5", "c6");
+        assertRow(partitions.get(0), tokenOrderedKeys.get(3), "c2", "c3", "c4", "c5", "c6");
         assertFalse(pager.isExhausted());
 
         pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
         assertFalse(pager.isExhausted());
         partitions = query(pager, 5);
-        assertRow(partitions.get(0), "k4", "c7");
-        assertRow(partitions.get(1), "k5", "c1", "c2", "c3", "c4");
+        assertRow(partitions.get(0), tokenOrderedKeys.get(3), "c7");
+        assertRow(partitions.get(1), tokenOrderedKeys.get(4), "c1", "c2", "c3", "c4");
         assertFalse(pager.isExhausted());
 
         pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
         assertFalse(pager.isExhausted());
         partitions = query(pager, 5, 3);
-        assertRow(partitions.get(0), "k5", "c5", "c6", "c7");
+        assertRow(partitions.get(0), tokenOrderedKeys.get(4), "c5", "c6", "c7");
 
         assertTrue(pager.isExhausted());
     }
 
+    @Test
+    public void rangeSliceQueryWithLimitsTest() throws Exception
+    {
+        boolean testPagingState = true;
+        ProtocolVersion protocolVersion = ProtocolVersion.CURRENT;
+
+        // Test with count < partitionCount
+
+        int count = 1;
+        int partitionCount = 2;
+
+        ReadCommand command = rangeSliceQuery(count, partitionCount, new PageSize(100, PageSize.PageUnit.ROWS),
+                                              cfs(KEYSPACE1, CF_STANDARD),
+                                              tokenOrderedKeys.get(0), tokenOrderedKeys.get(4),
+                                              "c1", "c7");
+
+        QueryPager pager = command.getPager(null, protocolVersion);
+        List<FilteredPartition> partitions = query(pager, 5, count);
+        assertEquals(1, partitions.size());
+        assertRow(partitions.get(0), tokenOrderedKeys.get(1), "c1");
+        assertTrue(pager.isExhausted());
+
+        // Test with count > partitionCount
+
+        count = 2;
+        partitionCount = 1;
+
+        command = rangeSliceQuery(count, partitionCount, new PageSize(100, PageSize.PageUnit.ROWS),
+                                  cfs(KEYSPACE1, CF_STANDARD),
+                                  tokenOrderedKeys.get(0), tokenOrderedKeys.get(4),
+                                  "c1", "c7");
+
+        pager = command.getPager(null, protocolVersion);
+        partitions = query(pager, 5, count);
+        assertEquals(2, partitions.size());
+        assertRow(partitions.get(0), tokenOrderedKeys.get(1), "c1");
+        assertRow(partitions.get(1), tokenOrderedKeys.get(2), "c1");
+        assertTrue(pager.isExhausted());
+
+        // Test with count spanning multiple partitions
+
+        count = 4;
+        partitionCount = 2;
+
+        command = rangeSliceQuery(count, partitionCount, new PageSize(100, PageSize.PageUnit.ROWS),
+                                  cfs(KEYSPACE1, CF_STANDARD),
+                                  tokenOrderedKeys.get(0), tokenOrderedKeys.get(4),
+                                  "c1", "c7");
+
+        pager = command.getPager(null, protocolVersion);
+        partitions = query(pager, 5, count);
+        assertEquals(2, partitions.size());
+        assertRow(partitions.get(0), tokenOrderedKeys.get(1), "c1", "c2");
+        assertRow(partitions.get(1), tokenOrderedKeys.get(2), "c1", "c2");
+        assertTrue(pager.isExhausted());
+
+        // Test with count spanning multiple pages
+
+        count = 8;
+        partitionCount = 2;
+
+        command = rangeSliceQuery(count, partitionCount, new PageSize(100, PageSize.PageUnit.ROWS),
+                                  cfs(KEYSPACE1, CF_STANDARD),
+                                  tokenOrderedKeys.get(0), tokenOrderedKeys.get(4),
+                                  "c1", "c7");
+
+        pager = command.getPager(null, protocolVersion);
+        partitions = query(pager, 5, 5);
+        assertEquals(3, partitions.size());
+        assertRow(partitions.get(0), tokenOrderedKeys.get(1), "c1", "c2");
+        assertRow(partitions.get(1), tokenOrderedKeys.get(2), "c1", "c2");
+        assertRow(partitions.get(2), tokenOrderedKeys.get(3), "c1");
+        assertFalse(pager.isExhausted());
+
+        pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
+        partitions = query(pager, 5, 3);
+        assertEquals(2, partitions.size());
+        assertRow(partitions.get(0), tokenOrderedKeys.get(3), "c2");
+        assertRow(partitions.get(1), tokenOrderedKeys.get(4), "c1", "c2");
+        assertTrue(pager.isExhausted());
+    }
+
     @Test
     public void SliceQueryWithTombstoneTest()
     {
-        for(ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
             SliceQueryWithTombstoneTest(version);
     }
 
     public void SliceQueryWithTombstoneTest(ProtocolVersion protocolVersion)
     {
         // Testing for the bug of #6748
-        String keyspace = "cql_keyspace";
-        String table = "table2";
+        String keyspace = KEYSPACE_CQL;
+        String table = CF_CQL;
         ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
 
         // Insert rows but with a tombstone as last cell
@@ -476,8 +884,8 @@ public void pagingReversedQueriesWithStaticColumnsTest()
         // see CASSANDRA-13222
 
         // insert some rows into a single partition
-        for (int i=0; i < 5; i++)
-            executeInternal(String.format("INSERT INTO %s.%s (pk, ck, st, v1, v2) VALUES ('k0', %3$s, %3$s, %3$s, %3$s)",
+        for (int i = 0; i < 5; i++)
+            executeInternal(String.format("INSERT INTO %s.%s (k, c, st, v1, v2) VALUES ('k0', '%3$s', %3$s, %3$s, %3$s)",
                                           KEYSPACE_CQL, CF_CQL_WITH_STATIC, i));
 
         // query the table in reverse with page size = 1 & check that the returned rows contain the correct cells
@@ -494,10 +902,10 @@ private void queryAndVerifyCells(TableMetadata table, boolean reversed, String k
         ColumnMetadata staticColumn = table.staticColumns().getSimple(0);
         assertEquals(staticColumn.name.toCQLString(), "st");
 
-        for (int i=0; i<5; i++)
+        for (int i = 0; i < 5; i++)
         {
             try (ReadExecutionController controller = pager.executionController();
-                 PartitionIterator partitions = pager.fetchPageInternal(1, controller))
+                 PartitionIterator partitions = pager.fetchPageInternal(PageSize.inRows(1), controller))
             {
                 try (RowIterator partition = partitions.next())
                 {
@@ -506,7 +914,7 @@ private void queryAndVerifyCells(TableMetadata table, boolean reversed, String k
                     Row row = partition.next();
                     int cellIndex = !reversed ? i : 4 - i;
 
-                    assertEquals(row.clustering().bufferAt(0), ByteBufferUtil.bytes(cellIndex));
+                    assertEquals(string(row.clustering().bufferAt(0)), ""+cellIndex);
                     assertCell(row, table.getColumn(new ColumnIdentifier("v1", false)), cellIndex);
                     assertCell(row, table.getColumn(new ColumnIdentifier("v2", false)), cellIndex);
 
@@ -517,8 +925,8 @@ private void queryAndVerifyCells(TableMetadata table, boolean reversed, String k
         }
 
         // After processing the 5 rows there should be no more rows to return
-        try ( ReadExecutionController controller = pager.executionController();
-              PartitionIterator partitions = pager.fetchPageInternal(1, controller))
+        try (ReadExecutionController controller = pager.executionController();
+             PartitionIterator partitions = pager.fetchPageInternal(PageSize.inRows(1), controller))
         {
             assertFalse(partitions.hasNext());
         }
@@ -530,4 +938,201 @@ private void assertCell(Row row, ColumnMetadata column, int value)
         assertNotNull(cell);
         assertEquals(value, ByteBufferUtil.toInt(cell.buffer()));
     }
+
+    @Test
+    public void testSinglePartitionPagingByBytes()
+    {
+        executeInternal(String.format("TRUNCATE TABLE %s.%s", KEYSPACE_CQL, CF_CQL));
+
+        int rows = 10;
+
+        for (int i = 0; i < rows; i++)
+            executeInternal(String.format("INSERT INTO %s.%s(k, c, v) VALUES('k', 'c%s', 'ignored')", KEYSPACE_CQL, CF_CQL, i));
+
+        // Test with rows limit:
+
+        int maxExpected = rows;
+        for (int count = 0; count <= maxExpected; count++)
+        {
+            SinglePartitionReadCommand q = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k", "c0", "c9", false);
+            checkRows(q, PageSize.PageUnit.BYTES, count > 0 ? count : maxExpected, 1, 128, 256, 1024);
+        }
+
+        // Test with partition limit:
+
+        for (int partitionCount = 1; partitionCount <= rows; partitionCount++)
+        {
+            SinglePartitionReadCommand q = sliceQuery(-1, partitionCount, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k", "c0", "c9", false);
+            checkRows(q, PageSize.PageUnit.BYTES, partitionCount, 1, 128, 256, 1024);
+        }
+    }
+
+    @Test
+    public void testPartitionRangePagingByBytes()
+    {
+        executeInternal(String.format("TRUNCATE TABLE %s.%s", KEYSPACE_CQL, CF_CQL));
+
+        int pks = 10;
+        int cs = 10;
+
+        SortedSet<String> tokens = Sets.newTreeSet(Comparator.comparing(a -> cfs(KEYSPACE_CQL, CF_CQL).getPartitioner().decorateKey(bytes(a))));
+        for (int i = 0; i < pks; i++)
+        {
+            for (int j = 0; j < cs; j++)
+            {
+                executeInternal(String.format("INSERT INTO %s.%s(k, c, v) VALUES('k%s', 'c%s', 'ignored')", KEYSPACE_CQL, CF_CQL, i, j));
+            }
+            tokens.add("k" + i);
+        }
+
+        // Test with rows limit:
+
+        int maxExpected = pks - 1;
+        for (int count = 0; count <= maxExpected; count++)
+        {
+            ReadCommand q = rangeSliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), tokens.first(), tokens.last(), "c0", "c0");
+            checkRows(q, PageSize.PageUnit.BYTES, count > 0 ? count : maxExpected, 1, 128, 256, 1024);
+        }
+
+        // Test with partition limit:
+
+        for (int partitionCount = 1; partitionCount <= cs; partitionCount++)
+        {
+            ReadCommand q = rangeSliceQuery(-1, partitionCount, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), tokens.first(), tokens.last(), "c0", "c9");
+            checkRows(q, PageSize.PageUnit.BYTES, partitionCount * (pks - 1), 1, 128, 256, 1024);
+        }
+    }
+
+    @Test
+    public void testMultiPartitionPagingByBytes()
+    {
+        executeInternal(String.format("TRUNCATE TABLE %s.%s", KEYSPACE_CQL, CF_CQL));
+
+        int pks = 10;
+        int cs = 10;
+
+        for (int i = 0; i < pks; i++)
+            for (int j = 0; j < cs; j++)
+                executeInternal(String.format("INSERT INTO %s.%s(k, c, v) VALUES('k%s', 'c%s', 'ignored')", KEYSPACE_CQL, CF_CQL, i, j));
+
+        // Test with rows limit:
+
+        int maxExpected = 22; // the sum of the clustering keys in the command group below
+        for (int count = 0; count <= maxExpected; count++)
+        {
+            SinglePartitionReadCommand q1 = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k0", "c0", "c1", false);
+            SinglePartitionReadCommand q2 = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k1", "c0", "c3", false);
+            SinglePartitionReadCommand q3 = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k2", "c0", "c5", false);
+            SinglePartitionReadCommand q4 = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k3", "c0", "c9", false);
+            Group q = Group.create(
+            Arrays.asList(q1, q2, q3, q4),
+            count > 0 ? DataLimits.cqlLimits(count) : DataLimits.NONE);
+            checkRows(q, PageSize.PageUnit.BYTES, count > 0 ? count : maxExpected, 1, 128, 256, 1024);
+        }
+
+        // Test with partition limit:
+
+        for (int partitionCount = 1; partitionCount <= cs; partitionCount++)
+        {
+            SinglePartitionReadCommand q1 = sliceQuery(-1, partitionCount, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k0", "c0", "c9", false);
+            SinglePartitionReadCommand q2 = sliceQuery(-1, partitionCount, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k1", "c0", "c9", false);
+            SinglePartitionReadCommand q3 = sliceQuery(-1, partitionCount, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k2", "c0", "c9", false);
+            SinglePartitionReadCommand q4 = sliceQuery(-1, partitionCount, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL), "k3", "c0", "c9", false);
+            Group q = Group.create(
+            Arrays.asList(q1, q2, q3, q4),
+            DataLimits.cqlLimits(Integer.MAX_VALUE, partitionCount));
+            checkRows(q, PageSize.PageUnit.BYTES, partitionCount * 4, 1, 128, 256, 1024);
+        }
+    }
+
+    @Test
+    public void testStaticRowsPagingByBytes()
+    {
+        executeInternal(String.format("TRUNCATE TABLE %s.%s", KEYSPACE_CQL, CF_CQL_WITH_STATIC));
+
+        int rows = 10;
+
+        for (int i = 0; i < rows; i++)
+            executeInternal(String.format("INSERT INTO %s.%s(k, c, st) VALUES('k%s', 'c', 0)", KEYSPACE_CQL, CF_CQL_WITH_STATIC, i));
+
+        int maxExpected = 4;
+        for (int count = 0; count <= maxExpected; count++)
+        {
+            SinglePartitionReadCommand q1 = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL_WITH_STATIC), "k0", "c", "c", false);
+            SinglePartitionReadCommand q2 = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL_WITH_STATIC), "k1", "c", "c", false);
+            SinglePartitionReadCommand q3 = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL_WITH_STATIC), "k2", "c", "c", false);
+            SinglePartitionReadCommand q4 = sliceQuery(count, -1, PageSize.NONE, cfs(KEYSPACE_CQL, CF_CQL_WITH_STATIC), "k3", "c", "c", false);
+            Group q = Group.create(
+            Arrays.asList(q1, q2, q3, q4),
+            count > 0 ? DataLimits.cqlLimits(count) : DataLimits.NONE);
+            checkRows(q, PageSize.PageUnit.BYTES, count > 0 ? count : maxExpected, 1, 128, 256, 1024);
+        }
+    }
+
+    @Test
+    public void toStringTest()
+    {
+        TableMetadata metadata = TableMetadata.builder("ks", "tab")
+                                              .addPartitionKeyColumn("k", Int32Type.instance)
+                                              .addClusteringColumn("c", Int32Type.instance)
+                                              .addColumn(ColumnMetadata.regularColumn("ks", "tab", "v", Int32Type.instance))
+                                              .build();
+
+        DataLimits limits = DataLimits.cqlLimits(31, 29);
+
+        Clustering clustering = Clustering.make(bytes(11));
+        Row row = mock(Row.class);
+        when(row.clustering()).thenReturn(clustering);
+        when(row.isRow()).thenReturn(true);
+
+        PagingState state = new PagingState(ByteBufferUtil.bytes(1), PagingState.RowMark.create(metadata, row, ProtocolVersion.CURRENT), 19, 17);
+
+        SinglePartitionReadQuery singlePartitionReadQuery = mock(SinglePartitionReadQuery.class);
+        when(singlePartitionReadQuery.metadata()).thenReturn(metadata);
+        when(singlePartitionReadQuery.limits()).thenReturn(limits);
+        when(singlePartitionReadQuery.partitionKey()).thenReturn(metadata.partitioner.decorateKey(ByteBufferUtil.bytes(1)));
+        QueryPager singlePartitionPager = new SinglePartitionPager(singlePartitionReadQuery, state, ProtocolVersion.CURRENT);
+        Assertions.assertThat(singlePartitionPager.toString())
+                  .contains(limits.toString())
+                  .contains("remaining=19")
+                  .contains("remainingInPartition=17")
+                  .contains("lastReturned=c=11")
+                  .contains("lastCounter=null")
+                  .contains("lastKey=DecoratedKey(00000001, 00000001)")
+                  .contains("exhausted=false");
+
+        PartitionRangeReadQuery partitionRangeReadQuery = mock(PartitionRangeReadQuery.class);
+        when(partitionRangeReadQuery.metadata()).thenReturn(metadata);
+        when(partitionRangeReadQuery.limits()).thenReturn(limits);
+        QueryPager partitionRangeQueryPager = new PartitionRangeQueryPager(partitionRangeReadQuery, state, ProtocolVersion.CURRENT);
+        Assertions.assertThat(partitionRangeQueryPager.toString())
+                  .contains(limits.toString())
+                  .contains("remaining=19")
+                  .contains("remainingInPartition=17")
+                  .contains("lastReturnedRow=c=11")
+                  .contains("lastCounter=null")
+                  .contains("lastKey=DecoratedKey(00000001, 00000001)")
+                  .contains("lastReturnedKey=DecoratedKey(00000001, 00000001)")
+                  .contains("exhausted=false");
+
+        Group singlePartitionReadQueryGroup = Group.create(metadata,
+                                                           FBUtilities.nowInSeconds(),
+                                                           ColumnFilter.all(metadata),
+                                                           RowFilter.none(), limits,
+                                                           Arrays.asList(metadata.partitioner.decorateKey(bytes(1)), metadata.partitioner.decorateKey(bytes(2))),
+                                                           new ClusteringIndexSliceFilter(Slices.ALL, false));
+        QueryPager multiPartitionPager = new MultiPartitionPager<>(singlePartitionReadQueryGroup, state, ProtocolVersion.CURRENT);
+        Assertions.assertThat(multiPartitionPager.toString())
+                  .contains("pagers.length=2")
+                  .contains("limit=" + limits)
+                  .contains("remaining=19")
+                  .contains("current=0");
+
+        AggregationQueryPager aggregationQueryPager = new AggregationQueryPager(singlePartitionPager, PageSize.inBytes(512), limits);
+        Assertions.assertThat(aggregationQueryPager.toString())
+                  .contains("limits=" + limits)
+                  .contains("subPageSize=512 bytes")
+                  .contains("subPager=" + singlePartitionPager)
+                  .contains("lastReturned=c=11");
+    }
 }
diff --git a/test/unit/org/apache/cassandra/service/ReadQueryTrackingTest.java b/test/unit/org/apache/cassandra/service/ReadQueryTrackingTest.java
new file mode 100644
index 000000000000..2c72767bdb5e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/ReadQueryTrackingTest.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.cql3.CQLTester;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test for basic read queries tracking with different kinds of data models
+ * <p>
+ * In this test we're verifying the counting of read-related events in happy-path scenarios,
+ * with a couple of different data models.
+ * <p>
+ * Test scenarios are a combination of the following dimensions:
+ * 1. whether there are regular columns in the table
+ * 2. whether there is a static row in the table
+ * 3. whether there is a clustering column in the table
+ * 4. whether the query has a filter that requires scanning
+ * <p>
+ * The actual scenarios are more-or-less a cartesian product of the above dimensions (excluding
+ * those that don't make sense - like a static row without clustering columns).
+ * <p>
+ * Each test case bears expectation wrt the number of reads executed, the number of rows read,
+ * the number of partitions read.
+ * <p>
+ * This test does NOT test the correctness of the number of executed replica plans
+ *
+ * @see QueryInfoTrackerTest for another suite of tests for read queries tracking
+ *
+ */
+@RunWith(Parameterized.class)
+public class ReadQueryTrackingTest extends CQLTester
+{
+    // schema properties:
+    private static final String SEVEN_REGULAR_VALUES = "regular column with seven distinct values";
+    private static final String NO_REGULAR_COLUMNS = "no regular columns";
+    private static final String STATIC_ROW = "static row";
+    private static final String NO_STATIC_ROW = "no static row";
+    private static final String FIVE_ROWS_PER_PARTITION = "clustering columns with five rows per partition";
+    private static final String NO_CLUSTERING_COLUMNS = "no clustering columns";
+
+    // query properties:
+    private static final String FILTERING_FOR_TWO_PARTITIONS = "range read with allow filtering yielding two partitions";
+    private static final String READ_THREE_PARTITIONS_AND_TWO_ROWS_WITHIN_EACH = "multi partition read yielding three partitions and two rows within each";
+    private static final String SINGLE_PARTITION_READ = "no filtering";
+    private volatile QueryInfoTrackerTest.TestQueryInfoTracker tracker;
+    private volatile Session session;
+
+    @Parameters(name = "{index}: {0}")
+    public static Collection<Object[]> testScenarios() {
+        return Arrays.asList(
+            scenario(
+                SEVEN_REGULAR_VALUES, NO_STATIC_ROW, NO_CLUSTERING_COLUMNS, SINGLE_PARTITION_READ,
+                expect(reads(1), rows(1), partitions(1))),
+            scenario(
+                SEVEN_REGULAR_VALUES, NO_STATIC_ROW, NO_CLUSTERING_COLUMNS, FILTERING_FOR_TWO_PARTITIONS,
+                expect(reads(1), rows(2), partitions(2))),
+            scenario(
+                SEVEN_REGULAR_VALUES, NO_STATIC_ROW, FIVE_ROWS_PER_PARTITION, SINGLE_PARTITION_READ,
+                expect(reads(1), rows(5), partitions(1))),
+            scenario(
+                SEVEN_REGULAR_VALUES, NO_STATIC_ROW, FIVE_ROWS_PER_PARTITION, FILTERING_FOR_TWO_PARTITIONS,
+                expect(reads(1), rows(2 * 5), partitions(2))),
+            scenario(
+                SEVEN_REGULAR_VALUES, NO_STATIC_ROW, FIVE_ROWS_PER_PARTITION, READ_THREE_PARTITIONS_AND_TWO_ROWS_WITHIN_EACH,
+                expect(reads(3), rows(3 * 2), partitions(3))),
+            scenario(
+                SEVEN_REGULAR_VALUES, STATIC_ROW, FIVE_ROWS_PER_PARTITION, SINGLE_PARTITION_READ,
+                expect(reads(1), rows(1 + 5), partitions(1))),
+            scenario(
+                SEVEN_REGULAR_VALUES, STATIC_ROW, FIVE_ROWS_PER_PARTITION, FILTERING_FOR_TWO_PARTITIONS,
+                expect(reads(1), rows(2 * (1 + 5)), partitions(2))),
+            scenario(
+                SEVEN_REGULAR_VALUES, STATIC_ROW, FIVE_ROWS_PER_PARTITION, READ_THREE_PARTITIONS_AND_TWO_ROWS_WITHIN_EACH,
+                expect(reads(3), rows(3 * (1 + 2)), partitions(3))),
+            scenario(
+                NO_REGULAR_COLUMNS, STATIC_ROW, FIVE_ROWS_PER_PARTITION, SINGLE_PARTITION_READ,
+                expect(reads(1), rows(1 + 5), partitions(1))),
+            scenario(
+                NO_REGULAR_COLUMNS, STATIC_ROW, FIVE_ROWS_PER_PARTITION, FILTERING_FOR_TWO_PARTITIONS,
+                expect(reads(1), rows(2 * (1 + 5)), partitions(2))),
+            scenario(
+                NO_REGULAR_COLUMNS, STATIC_ROW, FIVE_ROWS_PER_PARTITION, READ_THREE_PARTITIONS_AND_TWO_ROWS_WITHIN_EACH,
+                expect(reads(3), rows(3 * (1 + 2)), partitions(3))),
+            scenario(
+                NO_REGULAR_COLUMNS, NO_STATIC_ROW, FIVE_ROWS_PER_PARTITION, SINGLE_PARTITION_READ,
+                expect(reads(1), rows(5), partitions(1))),
+            scenario(
+                NO_REGULAR_COLUMNS, NO_STATIC_ROW, FIVE_ROWS_PER_PARTITION, FILTERING_FOR_TWO_PARTITIONS,
+                expect(reads(1), rows(2 * 5), partitions(2))),
+            scenario(
+                NO_REGULAR_COLUMNS, NO_STATIC_ROW, FIVE_ROWS_PER_PARTITION, READ_THREE_PARTITIONS_AND_TWO_ROWS_WITHIN_EACH,
+                expect(reads(3), rows(2 * 3), partitions(3))));
+    };
+
+    @Parameterized.Parameter
+    public ReadQueryTrackingTestScenario scenario;
+
+    @Before
+    public void setupTest()
+    {
+        requireNetwork();
+        session = sessionNet();
+    }
+
+    @Test
+    public void testReadQueryTracing() {
+        String table = KEYSPACE + "." + scenario.toString().replace(", ", "_").replace(" ", "_").toLowerCase();
+
+        String regularRowsColumn = scenario.rows.equals(SEVEN_REGULAR_VALUES) ? ", v" : "";
+        String clusteringColumn = scenario.clustering.equals(FIVE_ROWS_PER_PARTITION) ? ", c" : "";
+        String staticColumn = scenario.staticRow.equals(STATIC_ROW) ? ", sv" : "";
+
+        session.execute(format("CREATE TABLE " + table + "(k int %s %s %s, PRIMARY KEY (k %s))",
+                               clusteringColumn.isEmpty() ? "" : clusteringColumn + " int",
+                               staticColumn.isEmpty() ? "" : staticColumn + " int static",
+                               regularRowsColumn.isEmpty() ? "" : regularRowsColumn + " int",
+                               clusteringColumn));
+
+        // seven distinct values for regular columns; we'll have the same for partition key
+        for (int k = 0; k < 7; k++)
+        {
+            // five distinct values for clustering columns
+            for (int c = 0; c < 5; c++)
+            {
+                session.execute(format("INSERT INTO " + table + "(k %s %s %s) values (%d %s %s %s)",
+                                       clusteringColumn,
+                                       staticColumn,
+                                       regularRowsColumn,
+                                       k,
+                                       clusteringColumn.isEmpty() ? "" : ", " + c,
+                                       staticColumn.isEmpty() ? "" : ", " + k * 77,
+                                       regularRowsColumn.isEmpty() ? "" : ", " + k)
+                                );
+            }
+        }
+
+        dumpTable(table);
+
+        tracker = new QueryInfoTrackerTest.TestQueryInfoTracker(KEYSPACE);
+        StorageProxy.instance.registerQueryTracker(tracker);
+        assertEquals(0, tracker.reads.get());
+
+        // now, let's issue a query
+        if (scenario.filtering.equals(SINGLE_PARTITION_READ))
+            session.execute("SELECT * FROM " + table + " WHERE k = ?", 4);
+        else if (scenario.filtering.equals(FILTERING_FOR_TWO_PARTITIONS))
+            session.execute("SELECT * FROM " + table + " WHERE k < ? ALLOW FILTERING", 2);
+        else if (scenario.filtering.equals(READ_THREE_PARTITIONS_AND_TWO_ROWS_WITHIN_EACH))
+            session.execute("SELECT * FROM " + table + " WHERE k IN (1, 2, 3) AND c IN (3, 1)");
+        else
+            fail("Unknown filtering scenario: " + scenario.filtering);
+
+        // verify expectations
+        if (scenario.filtering.equals(SINGLE_PARTITION_READ))
+        {
+            // single partition read
+            assertEquals(scenario.expectation.reads, tracker.reads.get());
+        }
+        else if (scenario.filtering.equals(FILTERING_FOR_TWO_PARTITIONS))
+        {
+            // range read
+            assertEquals(scenario.expectation.reads, tracker.rangeReads.get());
+        }
+        else if (scenario.filtering.equals(READ_THREE_PARTITIONS_AND_TWO_ROWS_WITHIN_EACH))
+        {
+            // multi-partition read
+            assertEquals(scenario.expectation.reads, tracker.reads.get());
+        }
+        else fail("Unknown filtering scenario: " + scenario.filtering);
+
+        assertEquals(scenario.expectation.rows, tracker.readRows.get());
+        assertEquals(scenario.expectation.partitions, tracker.readPartitions.get());
+        assertEquals(scenario.expectation.reads, tracker.replicaPlans.get());
+    }
+
+    private void dumpTable(String table)
+    {
+        ResultSet contents = session.execute("SELECT * FROM " + table);
+        for(Row row: contents.all())
+        {
+            StringBuilder rowString = new StringBuilder("| ");
+            for (int columnIdx = 0; columnIdx < row.getColumnDefinitions().size(); columnIdx++)
+            {
+                String columnName = row.getColumnDefinitions().getName(columnIdx);
+                Object value = row.getObject(columnIdx);
+                rowString.append(columnName).append(": ").append(value).append(" | ");
+            }
+            logger.debug("{}", rowString);
+        }
+    }
+
+    // boilerplate (thank you, copilot)
+    private static class ReadQueryTrackingTestScenario
+    {
+        final String rows;
+        final String staticRow;
+        final String clustering;
+        final String filtering;
+        final Expectation expectation;
+
+        private ReadQueryTrackingTestScenario(String rows, String staticRow, String clustering, String filtering, Expectation expectation)
+        {
+            this.rows = rows;
+            this.staticRow = staticRow;
+            this.clustering = clustering;
+            this.filtering = filtering;
+            this.expectation = expectation;
+        }
+
+        @Override
+        public String toString()
+        {
+            return rows +
+                ", " +
+                staticRow +
+                ", " +
+                clustering +
+                ", " +
+                filtering;
+        }
+    }
+
+    private static class Expectation
+    {
+        final int reads;
+        final int rows;
+        final int partitions;
+
+        private Expectation(int reads, int rows, int partitions)
+        {
+            this.reads = reads;
+            this.rows = rows;
+            this.partitions = partitions;
+        }
+    }
+
+    static int reads(int reads)
+    {
+        return reads;
+    }
+
+    static int rows(int rows)
+    {
+        return rows;
+    }
+
+    static int partitions(int partitions)
+    {
+        return partitions;
+    }
+
+    static Expectation expect(int reads, int rows, int partitions)
+    {
+        return new Expectation(reads, rows, partitions);
+    }
+
+    static Object[] scenario(String rows, String staticRow, String clustering, String filtering, Expectation expectation)
+    {
+        return new Object[] { new ReadQueryTrackingTestScenario(rows, staticRow, clustering, filtering, expectation) };
+    }
+}
diff --git a/test/unit/org/apache/cassandra/service/SerializationsTest.java b/test/unit/org/apache/cassandra/service/SerializationsTest.java
index 81d310932fe4..72dff21add87 100644
--- a/test/unit/org/apache/cassandra/service/SerializationsTest.java
+++ b/test/unit/org/apache/cassandra/service/SerializationsTest.java
@@ -42,8 +42,8 @@
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.repair.SyncNodePair;
 import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.repair.SyncNodePair;
 import org.apache.cassandra.repair.Validator;
 import org.apache.cassandra.repair.messages.*;
 import org.apache.cassandra.repair.state.ValidationState;
diff --git a/test/unit/org/apache/cassandra/service/StartupChecksTest.java b/test/unit/org/apache/cassandra/service/StartupChecksTest.java
index 6fb063987ad0..53474ad2d3f9 100644
--- a/test/unit/org/apache/cassandra/service/StartupChecksTest.java
+++ b/test/unit/org/apache/cassandra/service/StartupChecksTest.java
@@ -93,7 +93,7 @@ public void setup() throws IOException
         for (File dataDir : Directories.getKSChildDirectories(SchemaConstants.SYSTEM_KEYSPACE_NAME))
             dataDir.deleteRecursive();
 
-        File dataDir = new File(DatabaseDescriptor.getAllDataFileLocations()[0]);
+        File dataDir = DatabaseDescriptor.getAllDataFileLocations()[0];
         sstableDir = Paths.get(dataDir.absolutePath(), "Keyspace1", "Standard1");
         Files.createDirectories(sstableDir);
 
@@ -140,7 +140,7 @@ public void failStartupIfInvalidSSTablesFound() throws Exception
 
         // and in the system directory as of CASSANDRA-17777
         new File(backupDir).deleteRecursive();
-        File dataDir = new File(DatabaseDescriptor.getAllDataFileLocations()[0]);
+        File dataDir = DatabaseDescriptor.getAllDataFileLocations()[0];
         Path systemDir = Paths.get(dataDir.absolutePath(), "system", "InvalidSystemDirectory");
         Files.createDirectories(systemDir);
         copyInvalidLegacySSTables(systemDir);
@@ -298,9 +298,9 @@ public FileSystem getFileSystem()
 
     private void testKernelBug1057843Check(String fsType, DiskAccessMode diskAccessMode, Semver kernelVersion, boolean expectToFail) throws Exception
     {
-        String commitLogLocation = Files.createTempDirectory("testKernelBugCheck").toString();
+        File commitLogLocation =new File(Files.createTempDirectory("testKernelBugCheck"));
 
-        String savedCommitLogLocation = DatabaseDescriptor.getCommitLogLocation();
+        File savedCommitLogLocation = DatabaseDescriptor.getCommitLogLocation();
         DiskAccessMode savedCommitLogWriteDiskAccessMode = DatabaseDescriptor.getCommitLogWriteDiskAccessMode();
         Semver savedKernelVersion = FBUtilities.getKernelVersion();
         try
@@ -310,7 +310,7 @@ private void testKernelBug1057843Check(String fsType, DiskAccessMode diskAccessM
             DatabaseDescriptor.initializeCommitLogDiskAccessMode();
             assertThat(DatabaseDescriptor.getCommitLogWriteDiskAccessMode()).isEqualTo(diskAccessMode);
             FBUtilities.setKernelVersionSupplier(() -> kernelVersion);
-            withPathOverriddingFileSystem(Map.of(commitLogLocation, fsType), () -> {
+            withPathOverriddingFileSystem(Map.of(commitLogLocation.path(), fsType), () -> {
                 if (expectToFail)
                     assertThatExceptionOfType(StartupException.class).isThrownBy(() -> StartupChecks.checkKernelBug1057843.execute(options));
                 else
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceGossipTest.java b/test/unit/org/apache/cassandra/service/StorageServiceGossipTest.java
new file mode 100644
index 000000000000..78df310d9ed8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/StorageServiceGossipTest.java
@@ -0,0 +1,482 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.Multimap;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.RangeStreamer;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.HeartBeatState;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.EndpointsByRange;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.SimpleStrategy;
+import org.apache.cassandra.locator.SystemReplicas;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.nodes.Nodes;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.StorageService.LeavingReplica;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SKIP_SCHEMA_CHECK;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+// checkstyle: suppress below 'blockSystemPropertyUsage'
+
+/*
+ * This class contains tests that were originally added to StorageServiceGossipTest in CC tickets. In C* 5.0 
+ * testScheduledExecutorsShutdownOnDrain was added that calls StorageService.instance.drain(), which also shuts
+ * down the executor used by the gossip stage and is needed for some of these CC tests. Not finding a way to
+ * make these tests work after drain is called, it was easier to move them to a separate class.
+ */
+public class StorageServiceGossipTest
+{
+    public static final String KEYSPACE = "StorageServiceGossipTest";
+    public static final String COLUMN_FAMILY = "StorageServiceGossipTestColumnFamily";
+    static InetAddressAndPort aAddress;
+    static InetAddressAndPort bAddress;
+    static InetAddressAndPort cAddress;
+    static InetAddressAndPort dAddress;
+    static InetAddressAndPort eAddress;
+
+    @BeforeClass
+    public static void setUpClass() throws Exception
+    {
+        CassandraRelevantProperties.TEST_ALLOW_LOCAL_STRATEGY.setBoolean(true);
+
+        aAddress = InetAddressAndPort.getByName("127.0.0.1");
+        bAddress = InetAddressAndPort.getByName("127.0.0.2");
+        cAddress = InetAddressAndPort.getByName("127.0.0.3");
+        dAddress = InetAddressAndPort.getByName("127.0.0.4");
+        eAddress = InetAddressAndPort.getByName("127.0.0.5");
+
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE,
+                                    KeyspaceParams.local(),
+                                    SchemaLoader.standardCFMD(KEYSPACE, COLUMN_FAMILY));
+    }
+
+    private static final Token threeToken = new RandomPartitioner.BigIntegerToken("3");
+    private static final Token sixToken = new RandomPartitioner.BigIntegerToken("6");
+    private static final Token nineToken = new RandomPartitioner.BigIntegerToken("9");
+    private static final Token elevenToken = new RandomPartitioner.BigIntegerToken("11");
+    private static final Token oneToken = new RandomPartitioner.BigIntegerToken("1");
+
+    Range<Token> aRange = new Range<>(oneToken, threeToken);
+    Range<Token> bRange = new Range<>(threeToken, sixToken);
+    Range<Token> cRange = new Range<>(sixToken, nineToken);
+    Range<Token> dRange = new Range<>(nineToken, elevenToken);
+    Range<Token> eRange = new Range<>(elevenToken, oneToken);
+
+    @Before
+    public void setUp()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+        IEndpointSnitch snitch = new AbstractEndpointSnitch()
+        {
+            public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2)
+            {
+                return 0;
+            }
+
+            public String getRack(InetAddressAndPort endpoint)
+            {
+                return "R1";
+            }
+
+            public String getDatacenter(InetAddressAndPort endpoint)
+            {
+                return "DC1";
+            }
+        };
+
+        DatabaseDescriptor.setEndpointSnitch(snitch);
+
+        CommitLog.instance.start();
+        Nodes.peers().get().forEach(p -> Nodes.peers().remove(p.getPeerAddressAndPort(), false, true));
+    }
+
+    private AbstractReplicationStrategy simpleStrategy(TokenMetadata tmd)
+    {
+        return new SimpleStrategy("MoveTransientTest",
+                                  tmd,
+                                  DatabaseDescriptor.getEndpointSnitch(),
+                                  com.google.common.collect.ImmutableMap.of("replication_factor", "3/1"));
+    }
+
+    @Test
+    public void testSourceReplicasIsEmptyWithDeadNodes()
+    {
+        RandomPartitioner partitioner = new RandomPartitioner();
+        TokenMetadata tmd = new TokenMetadata();
+        tmd.updateNormalToken(threeToken, aAddress);
+        Util.joinNodeToRing(aAddress, threeToken, partitioner);
+        tmd.updateNormalToken(sixToken, bAddress);
+        Util.joinNodeToRing(bAddress, sixToken, partitioner);
+        tmd.updateNormalToken(nineToken, cAddress);
+        Util.joinNodeToRing(cAddress, nineToken, partitioner);
+        tmd.updateNormalToken(elevenToken, dAddress);
+        Util.joinNodeToRing(dAddress, elevenToken, partitioner);
+        tmd.updateNormalToken(oneToken, eAddress);
+        Util.joinNodeToRing(eAddress, oneToken, partitioner);
+
+        AbstractReplicationStrategy strat = simpleStrategy(tmd);
+        EndpointsByRange rangeReplicas = strat.getRangeAddresses(tmd);
+
+        Replica leaving = new Replica(aAddress, aRange, true);
+        Replica ourReplica = new Replica(cAddress, cRange, true);
+        Set<LeavingReplica> leavingReplicas = Stream.of(new LeavingReplica(leaving, ourReplica)).collect(Collectors.toCollection(HashSet::new));
+
+        // Mark the leaving replica as dead as well as the potential replica
+        Util.markNodeAsDead(aAddress);
+        Util.markNodeAsDead(bAddress);
+
+        Multimap<InetAddressAndPort, RangeStreamer.FetchReplica> result = StorageService.instance.findLiveReplicasForRanges(leavingReplicas, rangeReplicas, cAddress);
+        assertTrue("Replica set should be empty since replicas are dead", result.isEmpty());
+    }
+
+    @Test
+    public void testStreamCandidatesDontIncludeDeadNodes()
+    {
+        List<InetAddressAndPort> endpoints = Arrays.asList(aAddress, bAddress);
+
+        RandomPartitioner partitioner = new RandomPartitioner();
+        Util.joinNodeToRing(aAddress, threeToken, partitioner);
+        Util.joinNodeToRing(bAddress, sixToken, partitioner);
+
+        Replica liveReplica = SystemReplicas.getSystemReplica(aAddress);
+        Replica deadReplica = SystemReplicas.getSystemReplica(bAddress);
+        Util.markNodeAsDead(bAddress);
+
+        EndpointsForRange result = StorageService.getStreamCandidates(endpoints);
+        assertTrue("Live node should be in replica list", result.contains(liveReplica));
+        assertFalse("Dead node should not be in replica list", result.contains(deadReplica));
+    }
+
+    @Test
+    public void testSetTokens()
+    {
+        InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort();
+        IPartitioner partitioner = StorageService.instance.getTokenMetadata().partitioner;
+
+        Token token = StorageService.instance.getTokenFactory().fromString("3");
+        Util.joinNodeToRing(broadcastAddress, token, partitioner);
+        StorageService.instance.setTokens(Collections.singleton(token));
+
+        assertEquals("Unexpected endpoint for token", StorageService.instance.getTokenMetadata().getEndpoint(token), FBUtilities.getBroadcastAddressAndPort());
+    }
+
+    @Test
+    public void testPopulateTokenMetadata()
+    {
+        StorageService.instance.getTokenMetadata().clearUnsafe();
+        IPartitioner partitioner = StorageService.instance.getTokenMetadata().partitioner;
+        Token origToken = StorageService.instance.getTokenFactory().fromString("42");
+        Token newToken = StorageService.instance.getTokenFactory().fromString("88");
+
+        Util.joinNodeToRing(cAddress, origToken, partitioner);
+
+        // Update system.peers with a new token and check that the changes isn't visible until we call
+        // populateTokenMetadata().
+        TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+        SystemKeyspace.updateTokens(cAddress, Collections.singleton(newToken));
+        assertTrue("Original token is missing but should be present", tmd.getTokens(cAddress).contains(origToken));
+        assertFalse("New token is present but should be missing", tmd.getTokens(cAddress).contains(newToken));
+
+        StorageService.instance.populateTokenMetadata();
+        assertFalse("Original token is present but should be missing", tmd.getTokens(cAddress).contains(origToken));
+        assertTrue("New token is missing but should be present", tmd.getTokens(cAddress).contains(newToken));
+    }
+
+    @Test
+    public void testReplaceNodeAndOwnTokens() throws UnknownHostException
+    {
+        final String replaceAddressProperty = "cassandra.replace_address_first_boot";
+        String oldPropertyVal = System.getProperty(replaceAddressProperty);
+        try
+        {
+            String replaceAddressString = "127.0.0.100";
+            System.setProperty(replaceAddressProperty, replaceAddressString);
+            InetAddressAndPort replaceAddress = InetAddressAndPort.getByName(replaceAddressString);
+
+            IPartitioner partitioner = StorageService.instance.getTokenMetadata().partitioner;
+
+            HeartBeatState hbState = HeartBeatState.empty();
+            EndpointState gossipState = new EndpointState(hbState);
+            EndpointState localState = new EndpointState(hbState);
+
+            Token token = StorageService.instance.getTokenFactory().fromString("123");
+
+            UUID oldHostId = UUID.randomUUID();
+            TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+            tmd.updateHostId(oldHostId, replaceAddress);
+            assertEquals("Replaced address had unexpected host ID", oldHostId, StorageService.instance.getHostIdForEndpoint(replaceAddress));
+
+            UUID newHostId = UUID.randomUUID();
+            gossipState.addApplicationState(ApplicationState.HOST_ID, StorageService.instance.valueFactory.hostId(newHostId));
+            Map<InetAddressAndPort, EndpointState> endpointStateMap = new HashMap<>();
+            endpointStateMap.put(replaceAddress, gossipState);
+
+            localState.addApplicationState(ApplicationState.TOKENS, new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(token)));
+            StorageService.instance.replaceNodeAndOwnTokens(replaceAddress, endpointStateMap, localState);
+            assertEquals("Replaced address had unexpected host ID", newHostId, StorageService.instance.getHostIdForEndpoint(replaceAddress));
+            assertEquals("Replaced address had unexpected token", Arrays.asList(token), tmd.getTokens(replaceAddress));
+        }
+        finally
+        {
+            restorePropertyValue(replaceAddressProperty, oldPropertyVal);
+        }
+    }
+
+    @Test
+    public void testBootstrapPreparationFailsWithLeavingEndpoint()
+    {
+        String oldPropertyVal = System.getProperty(BOOTSTRAP_SKIP_SCHEMA_CHECK.toString());
+        System.setProperty(BOOTSTRAP_SKIP_SCHEMA_CHECK.toString(), "true");
+        try
+        {
+            TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+            tmd.addLeavingEndpoint(bAddress);
+            StorageService.instance.prepareForBootstrap(0, 0);
+            fail();
+        }
+        catch (UnsupportedOperationException exception)
+        {
+            final String expected = "Other bootstrapping/leaving/moving nodes detected";
+            assertTrue(String.format("Expected '%s' in exception message", expected), exception.getMessage().contains(expected));
+        }
+        finally
+        {
+            restorePropertyValue(BOOTSTRAP_SKIP_SCHEMA_CHECK.toString(), oldPropertyVal);
+        }
+    }
+
+    private static void restorePropertyValue(String property, String value)
+    {
+        if (value == null)
+        {
+            System.clearProperty(property);
+        }
+        else
+        {
+            System.setProperty(property, value);
+        }
+    }
+
+    @Test
+    public void testTokensInLocalDC()
+    {
+        InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort();
+        TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+        Token startToken = StorageService.instance.getTokenFactory().fromString("788");
+        Token endToken = StorageService.instance.getTokenFactory().fromString("789");
+        tmd.updateNormalTokens(Arrays.asList(startToken, endToken), localAddress);
+
+        EndpointsByRange endpointsByRange = StorageService.instance.getRangeToAddressMapInLocalDC(KEYSPACE);
+        boolean rangeCoversEndpoint = endpointsByRange.get(new Range(startToken, endToken)).endpointList().contains(localAddress);
+        assertTrue("Endpoint was not in EndpointsByRange", rangeCoversEndpoint);
+    }
+
+    @Test
+    public void testNormalStateExistingEndpoint() throws UnknownHostException
+    {
+        StorageService ss = StorageService.instance;
+        TokenMetadata tmd = ss.getTokenMetadata();
+        tmd.clearUnsafe();
+        IPartitioner partitioner = new RandomPartitioner();
+        VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
+
+        ArrayList<Token> endpointTokens = new ArrayList<>();
+        ArrayList<Token> keyTokens = new ArrayList<>();
+        List<InetAddressAndPort> hosts = new ArrayList<>();
+        List<UUID> hostIds = new ArrayList<>();
+
+        // Create a ring of 3 nodes
+        Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 3);
+
+        // Start a new node with unique host ID and verify token metadata is updated
+        Token token = new RandomPartitioner.BigIntegerToken("1");
+        UUID uniqueHostId = UUID.randomUUID();
+        assertEquals("Host ID was registered to node before we added it to gossip", null, tmd.getEndpointForHostId(uniqueHostId));
+        InetAddressAndPort uniqueNode = InetAddressAndPort.getByName("127.0.0.99");
+        Util.joinNodeToRing(uniqueNode, token, partitioner, uniqueHostId, 1);
+        assertEquals("Host ID not registered to node", uniqueNode, tmd.getEndpointForHostId(uniqueHostId));
+
+        // Start new node with same hostId as us, we should win (retain the hostId).
+        InetAddressAndPort newNode = InetAddressAndPort.getByName("127.0.0.100");
+        InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort();
+        UUID localHostId = Nodes.localOrPeerInfo(localAddress).getHostId();
+        Util.joinNodeToRing(newNode, token, partitioner, localHostId, 1);
+        assertEquals("Host ID not registered to local address", localAddress, tmd.getEndpointForHostId(localHostId));
+        ss.onChange(newNode, ApplicationState.STATUS_WITH_PORT, valueFactory.normal(Collections.singleton(token)));
+        assertEquals("Local address didn't win host ID", localAddress, tmd.getEndpointForHostId(localHostId));
+
+        // Start two nodes with the same hostId and generationNbr. First node should win.
+        InetAddressAndPort firstNode = InetAddressAndPort.getByName("127.0.0.101");
+        InetAddressAndPort secondNode = InetAddressAndPort.getByName("127.0.0.102");
+        UUID hostId = UUID.randomUUID();
+        Util.joinNodeToRing(firstNode, token, partitioner, hostId, 1);
+        assertEquals("Host ID not registered to first node", firstNode, tmd.getEndpointForHostId(hostId));
+        Util.joinNodeToRing(secondNode, token, partitioner, hostId, 1);
+        assertEquals("First node didn't win host ID", firstNode, tmd.getEndpointForHostId(hostId));
+
+        // Start a new node with same hostId but newer generationNbr. Newest node should win.
+        InetAddressAndPort newestNode = InetAddressAndPort.getByName("127.0.0.103");
+        Gossiper.instance.initializeNodeUnsafe(newestNode, hostId, MessagingService.current_version, 12);
+        Gossiper.instance.injectApplicationState(newestNode, ApplicationState.TOKENS, new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(token)));
+        ss.onChange(newestNode, ApplicationState.STATUS_WITH_PORT, valueFactory.normal(Collections.singleton(token)));
+        assertEquals("Newest node didn't win host ID", newestNode, tmd.getEndpointForHostId(hostId));
+    }
+
+    private static class TestEndpointSubscriber implements IEndpointLifecycleSubscriber
+    {
+        private boolean sawUpCall = false;
+
+        @Override
+        public void onJoinCluster(InetAddressAndPort endpoint) {}
+
+        @Override
+        public void onLeaveCluster(InetAddressAndPort endpoint) {}
+
+        @Override
+        public void onMove(InetAddressAndPort endpoint)  {}
+
+        @Override
+        public void onDown(InetAddressAndPort endpoint) {}
+
+        @Override
+        public void onUp(InetAddressAndPort endpoint)
+        {
+            sawUpCall = true;
+        }
+
+        public boolean upCalled()
+        {
+            return sawUpCall;
+        }
+
+        public void reset()
+        {
+            sawUpCall = false;
+        }
+    }
+
+    @Test
+    public void testAddAndRemoveNotifications() throws UnknownHostException
+    {
+        StorageService ss = StorageService.instance;
+        IPartitioner partitioner = StorageService.instance.getTokenMetadata().partitioner;
+        TestEndpointSubscriber subscriber = new TestEndpointSubscriber();
+        ss.register(subscriber);
+        InetAddressAndPort liveNode = InetAddressAndPort.getByName("127.0.0.200");
+        Token token = ss.getTokenFactory().fromString("200");
+        Util.joinNodeToRing(liveNode, token, partitioner);
+        Gossiper.instance.injectApplicationState(liveNode, ApplicationState.RPC_READY, new VersionedValue.VersionedValueFactory(partitioner).rpcReady(true));
+
+        TokenMetadata tmd = ss.getTokenMetadata();
+        assertTrue("liveNode was not member of ring", tmd.isMember(liveNode));
+        ss.onAlive(liveNode, null);
+        assertTrue("onUp() notification never called", subscriber.upCalled());
+        ss.onRemove(liveNode);
+        assertFalse("liveNode is still member of ring but was removed", tmd.isMember(liveNode));
+
+        subscriber.reset();
+
+        InetAddressAndPort deadNode = InetAddressAndPort.getByName("127.0.0.201");
+        assertFalse("deadNode was member of ring but shouldn't be", tmd.isMember(deadNode));
+        ss.onAlive(deadNode, null);
+        assertFalse("onUp() notification should not have been called", subscriber.upCalled());
+        ss.onRemove(deadNode);
+        assertFalse("deadNode somehow was added to the ring", tmd.isMember(deadNode));
+    }
+
+    @Test
+    public void testTokenOwnership() throws UnknownHostException
+    {
+        StorageService ss = StorageService.instance;
+        TokenMetadata tmd = ss.getTokenMetadata();
+        tmd.clearUnsafe();
+
+        ArrayList<Token> endpointTokens = new ArrayList<>();
+        ArrayList<Token> keyTokens = new ArrayList<>();
+        List<InetAddressAndPort> hosts = new ArrayList<>();
+        List<UUID> hostIds = new ArrayList<>();
+        Util.createInitialRing(ss, new RandomPartitioner(), endpointTokens, keyTokens, hosts, hostIds, 1);
+
+        Map<InetAddress, Float> map =  ss.getOwnership();
+        List<Float> ownershipList = map.values().stream().collect(Collectors.toList());
+        List<Float> singleValueList = Arrays.asList(Float.valueOf(1.0f));
+        assertEquals("Only node in the ring should own all tokens", singleValueList, ownershipList);
+    }
+
+    @Test
+    public void testForcedNodeRemovalCompletion()
+    {
+        StorageService ss = StorageService.instance;
+        TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+        tmd.addLeavingEndpoint(bAddress);
+
+        Set<InetAddressAndPort> leavingSet = new HashSet<>(Arrays.asList(bAddress));
+        Set<InetAddressAndPort> emptySet = new HashSet<>();
+
+        assertEquals("Singlular leaving endpoint not found", leavingSet, tmd.getLeavingEndpoints());
+        ss.forceRemoveCompletion();
+        assertEquals("Leaving endpoints still exist after forceRemoveCompletion()", emptySet, tmd.getLeavingEndpoints());
+    }
+
+}
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerM3PTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerM3PTest.java
index 8c4ab9b869db..cffcae586700 100644
--- a/test/unit/org/apache/cassandra/service/StorageServiceServerM3PTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageServiceServerM3PTest.java
@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.service;
 
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -54,10 +53,10 @@ public void testRegularMode() throws ConfigurationException
         mkdirs();
         cleanup();
         StorageService.instance.initServer(0);
-        for (String path : DatabaseDescriptor.getAllDataFileLocations())
+        for (File file : DatabaseDescriptor.getAllDataFileLocations())
         {
             // verify that storage directories are there.
-            assertTrue(new File(path).exists());
+            assertTrue(file.exists());
         }
         // a proper test would be to call decommission here, but decommission() mixes both shutdown and datatransfer
         // calls.  This test is only interested in the shutdown-related items which a properly handled by just
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
index d4cf4504f8f8..76e838f6f9de 100644
--- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
@@ -28,7 +28,9 @@
 import com.google.common.collect.Multimap;
 
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.io.util.File;
 import org.junit.Assert;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -77,6 +79,26 @@ public static void setUp() throws ConfigurationException
         StorageService.instance.initServer(0);
     }
 
+    @AfterClass
+    public static void tearDown()
+    {
+        // a proper test would be to call decommission here, but decommission() mixes both shutdown and datatransfer
+        // calls.  This test is only interested in the shutdown-related items which a properly handled by just
+        // stopping the client.
+        //StorageService.instance.decommission();
+        StorageService.instance.stopClient();
+    }
+
+    @Test
+    public void testRegularMode() throws ConfigurationException
+    {
+        for (File path : DatabaseDescriptor.getAllDataFileLocations())
+        {
+            // verify that storage directories are there.
+            assertTrue(path.exists());
+        }
+    }
+
     @Test
     public void testGetAllRangesEmpty()
     {
@@ -564,15 +586,15 @@ public void testGetNativeAddress() throws Exception
         InetAddressAndPort internalAddress = InetAddressAndPort.getByName(internalAddressString);
         Gossiper.instance.addSavedEndpoint(internalAddress);
         //Default to using the provided address with the configured port
-        assertEquals("127.0.0.2:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddress, true));
+        assertEquals("127.0.0.2:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeAddress(internalAddress, true));
 
         VersionedValue.VersionedValueFactory valueFactory =  new VersionedValue.VersionedValueFactory(Murmur3Partitioner.instance);
         //If we don't have the port use the gossip address, but with the configured port
         Gossiper.instance.getEndpointStateForEndpoint(internalAddress).addApplicationState(ApplicationState.RPC_ADDRESS, valueFactory.rpcaddress(InetAddress.getByName("127.0.0.3")));
-        assertEquals("127.0.0.3:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddress, true));
+        assertEquals("127.0.0.3:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeAddress(internalAddress, true));
         //If we have the address and port in gossip use that
         Gossiper.instance.getEndpointStateForEndpoint(internalAddress).addApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT, valueFactory.nativeaddressAndPort(InetAddressAndPort.getByName("127.0.0.3:666")));
-        assertEquals("127.0.0.3:666", StorageService.instance.getNativeaddress(internalAddress, true));
+        assertEquals("127.0.0.3:666", StorageService.instance.getNativeAddress(internalAddress, true));
     }
 
     @Test
@@ -585,16 +607,16 @@ public void testGetNativeAddressIPV6() throws Exception
         Gossiper.instance.addSavedEndpoint(internalAddressIPV6);
 
         //Default to using the provided address with the configured port
-        assertEquals("[0:0:0:0:0:0:0:3]:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddressIPV6, true));
+        assertEquals("[0:0:0:0:0:0:0:3]:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeAddress(internalAddressIPV6, true));
 
         VersionedValue.VersionedValueFactory valueFactory =  new VersionedValue.VersionedValueFactory(Murmur3Partitioner.instance);
         //If RPC_ADDRESS is present with an IPv6 address, we should properly bracket encode the IP with the configured port.
         Gossiper.instance.getEndpointStateForEndpoint(internalAddressIPV6).addApplicationState(ApplicationState.RPC_ADDRESS, valueFactory.rpcaddress(InetAddress.getByName("0:0:0:0:0:0:5a:3")));
-        assertEquals("[0:0:0:0:0:0:5a:3]:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddressIPV6, true));
+        assertEquals("[0:0:0:0:0:0:5a:3]:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeAddress(internalAddressIPV6, true));
 
         //If we have the address and port in gossip use that
         Gossiper.instance.getEndpointStateForEndpoint(internalAddressIPV6).addApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT, valueFactory.nativeaddressAndPort(InetAddressAndPort.getByName("[0:0:0:0:0:0:5c:3]:8675")));
-        assertEquals("[0:0:0:0:0:0:5c:3]:8675", StorageService.instance.getNativeaddress(internalAddressIPV6, true));
+        assertEquals("[0:0:0:0:0:0:5c:3]:8675", StorageService.instance.getNativeAddress(internalAddressIPV6, true));
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceTest.java b/test/unit/org/apache/cassandra/service/StorageServiceTest.java
index e0700959101a..3866bd09635c 100644
--- a/test/unit/org/apache/cassandra/service/StorageServiceTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageServiceTest.java
@@ -20,7 +20,6 @@
 
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicInteger;
-
 import com.google.common.collect.ImmutableMultimap;
 import org.junit.Assert;
 import org.junit.Before;
@@ -44,6 +43,7 @@
 import org.apache.cassandra.locator.SimpleSnitch;
 import org.apache.cassandra.locator.SimpleStrategy;
 import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.nodes.Nodes;
 import org.mockito.Mockito;
 
 import static java.util.concurrent.TimeUnit.MINUTES;
@@ -108,7 +108,9 @@ public String getDatacenter(InetAddressAndPort endpoint)
         };
 
         DatabaseDescriptor.setEndpointSnitch(snitch);
+
         CommitLog.instance.start();
+        Nodes.peers().get().forEach(p -> Nodes.peers().remove(p.getPeerAddressAndPort(), false, true));
     }
 
     private AbstractReplicationStrategy simpleStrategy(TokenMetadata tmd)
@@ -377,4 +379,19 @@ private StorageService getStorageService()
 
         return spiedStorageService;
     }
+
+    @Test
+    public void testSettingColumnIndexCacheSize()
+    {
+        int old = StorageService.instance.getColumnIndexCacheSize();
+        try
+        {
+            StorageService.instance.setColumnIndexCacheSize(old + 2);
+            assertEquals(old + 2, DatabaseDescriptor.getColumnIndexCacheSizeInKiB());
+        }
+        finally
+        {
+            StorageService.instance.setColumnIndexCacheSize(old);
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java
index 1e3ca5611771..797f354fb4c9 100644
--- a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java
+++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java
@@ -39,6 +39,7 @@
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.exceptions.WriteFailureException;
 import org.apache.cassandra.locator.IEndpointSnitch;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.Replica;
@@ -54,6 +55,8 @@
 import static java.util.concurrent.TimeUnit.DAYS;
 import static org.apache.cassandra.net.NoPayload.noPayload;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -315,6 +318,57 @@ public void failedIdealCLDoesNotIncrementsStatOnExplicitQueryFailure()
         assertEquals(startingCountForIdealCLWriteLatency, ks.metric.idealCLWriteLatency.totalLatency.getCount());
     }
 
+    @Test
+    public void testIsCompleted() throws Throwable
+    {
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        AbstractWriteResponseHandler<?> awr = createWriteResponseHandler(ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.EACH_QUORUM, requestTime);
+        assertThat(awr.replicaPlan().writeQuorum()).isEqualTo(2);
+        assertThat(awr.writeType()).isEqualTo(WriteType.SIMPLE);
+        assertThat(awr.requestTime()).isEqualTo(requestTime);
+
+        assertThat(awr.isCompleted()).isFalse();
+        assertThat(awr.isCompletedExceptionally()).isFalse();
+
+        // LOCAL_QUORUM requires 2 reponses
+        awr.onResponse(createDummyMessage(0));
+        assertThat(awr.isCompleted()).isFalse();
+        assertThat(awr.isCompletedExceptionally()).isFalse();
+
+        awr.onResponse(createDummyMessage(1));
+        assertThat(awr.isCompleted()).isTrue();
+        assertThat(awr.isCompletedExceptionally()).isFalse();
+
+        awr.get();
+    }
+
+    @Test
+    public void testIsCompletedExceptionally() throws Throwable
+    {
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
+        AbstractWriteResponseHandler<?> awr = createWriteResponseHandler(ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.EACH_QUORUM, requestTime);
+        assertThat(awr.replicaPlan().writeQuorum()).isEqualTo(2);
+        assertThat(awr.writeType()).isEqualTo(WriteType.SIMPLE);
+        assertThat(awr.requestTime()).isEqualTo(requestTime);
+
+        assertThat(awr.isCompleted()).isFalse();
+        assertThat(awr.isCompletedExceptionally()).isFalse();
+
+        // LOCAL_QUORUM requires 2 failure to fail handler
+        awr.onFailure(targets.get(0).endpoint(), RequestFailureReason.UNKNOWN);
+        assertThat(awr.isCompleted()).isFalse();
+        assertThat(awr.isCompletedExceptionally()).isFalse();
+        assertThat(awr.failures()).isEqualTo(1);
+        assertThat(awr.failureReasonByEndpoint()).hasSize(1);
+
+        awr.onFailure(targets.get(1).endpoint(), RequestFailureReason.UNKNOWN);
+        assertThat(awr.isCompleted()).isTrue();
+        assertThat(awr.isCompletedExceptionally()).isTrue();
+        assertThat(awr.failures()).isEqualTo(2);
+        assertThat(awr.failureReasonByEndpoint()).hasSize(2);
+
+        assertThatThrownBy(awr::get).isInstanceOf(WriteFailureException.class);
+    }
 
     private static AbstractWriteResponseHandler createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal)
     {
diff --git a/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java b/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java
index ce0ef438264e..2c72cad25045 100644
--- a/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java
@@ -72,6 +72,7 @@
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.QueryInfoTracker;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -344,4 +345,9 @@ protected Unfiltered computeNext()
         };
         return new SingletonUnfilteredPartitionIterator(rowIter);
     }
+
+    public QueryInfoTracker.ReadTracker noopReadTracker()
+    {
+        return QueryInfoTracker.ReadTracker.NOOP;
+    }
 }
diff --git a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java
index c481a2cb2d95..e7df8fd169be 100644
--- a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java
@@ -133,7 +133,7 @@ private EndpointsForRange makeReplicas(int num)
     public void testResolveNewerSingleRow()
     {
         EndpointsForRange replicas = makeReplicas(2);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
                                                                                                      .add("c1", "v1")
@@ -165,7 +165,7 @@ public void testResolveNewerSingleRow()
     public void testResolveDisjointSingleRow()
     {
         EndpointsForRange replicas = makeReplicas(2);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
                                                                                                      .add("c1", "v1")
@@ -202,7 +202,7 @@ public void testResolveDisjointSingleRow()
     public void testResolveDisjointMultipleRows() throws UnknownHostException
     {
         EndpointsForRange replicas = makeReplicas(2);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
                                                                                                      .add("c1", "v1")
@@ -249,7 +249,7 @@ public void testResolveDisjointMultipleRows() throws UnknownHostException
     public void testResolveDisjointMultipleRowsWithRangeTombstones()
     {
         EndpointsForRange replicas = makeReplicas(4);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
 
         RangeTombstone tombstone1 = tombstone("1", "11", 1, nowInSec);
         RangeTombstone tombstone2 = tombstone("3", "31", 1, nowInSec);
@@ -330,7 +330,7 @@ public void testResolveDisjointMultipleRowsWithRangeTombstones()
     public void testResolveWithOneEmpty()
     {
         EndpointsForRange replicas = makeReplicas(2);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1")
                                                                                                      .add("c2", "v2")
@@ -361,7 +361,7 @@ public void testResolveWithBothEmpty()
     {
         EndpointsForRange replicas = makeReplicas(2);
         TestableReadRepair readRepair = new TestableReadRepair(command);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         resolver.preprocess(response(command, replicas.get(0).endpoint(), EmptyIterators.unfilteredPartition(cfm)));
         resolver.preprocess(response(command, replicas.get(1).endpoint(), EmptyIterators.unfilteredPartition(cfm)));
 
@@ -377,7 +377,7 @@ public void testResolveWithBothEmpty()
     public void testResolveDeleted()
     {
         EndpointsForRange replicas = makeReplicas(2);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         // one response with columns timestamped before a delete in another response
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1")
@@ -403,7 +403,7 @@ public void testResolveDeleted()
     public void testResolveMultipleDeleted()
     {
         EndpointsForRange replicas = makeReplicas(4);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         // deletes and columns with interleaved timestamp, with out of order return sequence
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         resolver.preprocess(response(command, peer1, fullPartitionDelete(cfm, dk, 0, nowInSec)));
@@ -488,7 +488,7 @@ public void testResolveRangeTombstonesOnBoundarySameTimestamp() throws UnknownHo
     private void resolveRangeTombstonesOnBoundary(long timestamp1, long timestamp2)
     {
         EndpointsForRange replicas = makeReplicas(2);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         InetAddressAndPort peer2 = replicas.get(1).endpoint();
 
@@ -562,7 +562,7 @@ public void testRepairRangeTombstoneBoundary() throws UnknownHostException
     private void testRepairRangeTombstoneBoundary(int timestamp1, int timestamp2, int timestamp3) throws UnknownHostException
     {
         EndpointsForRange replicas = makeReplicas(2);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         InetAddressAndPort peer2 = replicas.get(1).endpoint();
 
@@ -615,7 +615,7 @@ public void testRepairRangeTombstoneWithPartitionDeletion()
     {
         EndpointsForRange replicas = makeReplicas(2);
 
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         InetAddressAndPort peer2 = replicas.get(1).endpoint();
 
@@ -654,7 +654,7 @@ public void testRepairRangeTombstoneWithPartitionDeletion()
     public void testRepairRangeTombstoneWithPartitionDeletion2()
     {
         EndpointsForRange replicas = makeReplicas(2);
-        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         InetAddressAndPort peer1 = replicas.get(0).endpoint();
         InetAddressAndPort peer2 = replicas.get(1).endpoint();
 
@@ -738,7 +738,7 @@ public void testResolveComplexDelete()
         EndpointsForRange replicas = makeReplicas(2);
         ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
         TestableReadRepair readRepair = new TestableReadRepair(cmd);
-        DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
 
         long[] ts = {100, 200};
 
@@ -771,7 +771,7 @@ public void testResolveComplexDelete()
         }
 
         Mutation mutation = readRepair.getForEndpoint(peer1);
-        Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).iterator();
+        Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).rowIterator();
         assertTrue(rowIter.hasNext());
         Row row = rowIter.next();
         assertFalse(rowIter.hasNext());
@@ -790,7 +790,7 @@ public void testResolveDeletedCollection()
         EndpointsForRange replicas = makeReplicas(2);
         ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
         TestableReadRepair readRepair = new TestableReadRepair(cmd);
-        DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
 
         long[] ts = {100, 200};
 
@@ -815,7 +815,7 @@ public void testResolveDeletedCollection()
         }
 
         Mutation mutation = readRepair.getForEndpoint(peer1);
-        Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).iterator();
+        Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).rowIterator();
         assertTrue(rowIter.hasNext());
         Row row = rowIter.next();
         assertFalse(rowIter.hasNext());
@@ -834,7 +834,7 @@ public void testResolveNewCollection()
         EndpointsForRange replicas = makeReplicas(2);
         ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
         TestableReadRepair readRepair = new TestableReadRepair(cmd);
-        DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
 
         long[] ts = {100, 200};
 
@@ -867,7 +867,7 @@ public void testResolveNewCollection()
         Assert.assertNull(readRepair.sent.get(peer1));
 
         Mutation mutation = readRepair.getForEndpoint(peer2);
-        Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).iterator();
+        Iterator<Row> rowIter = mutation.getPartitionUpdate(cfm2).rowIterator();
         assertTrue(rowIter.hasNext());
         Row row = rowIter.next();
         assertFalse(rowIter.hasNext());
@@ -884,7 +884,7 @@ public void testResolveNewCollectionOverwritingDeleted()
         EndpointsForRange replicas = makeReplicas(2);
         ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build();
         TestableReadRepair readRepair = new TestableReadRepair(cmd);
-        DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution());
+        DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
 
         long[] ts = {100, 200};
 
@@ -917,7 +917,7 @@ public void testResolveNewCollectionOverwritingDeleted()
             }
         }
 
-        Row row = Iterators.getOnlyElement(readRepair.getForEndpoint(peer1).getPartitionUpdate(cfm2).iterator());
+        Row row = Iterators.getOnlyElement(readRepair.getForEndpoint(peer1).getPartitionUpdate(cfm2).rowIterator());
 
         ComplexColumnData cd = row.getComplexColumnData(m);
 
@@ -1256,7 +1256,7 @@ class TestableDataResolver extends DataResolver
 
             public TestableDataResolver(ReadCommand command, ReplicaPlan.SharedForRangeRead plan, ReadRepair readRepair, Dispatcher.RequestTime requestTime)
             {
-                super(command, plan, readRepair, requestTime, true);
+                super(command, plan, readRepair, requestTime, true, noopReadTracker());
             }
 
             protected RepairedDataVerifier getRepairedDataVerifier(ReadCommand command)
@@ -1310,7 +1310,7 @@ private void assertRepairContainsColumn(Mutation mutation,
     private void assertRepairContainsNoColumns(Mutation mutation)
     {
         PartitionUpdate update = mutation.getPartitionUpdates().iterator().next();
-        assertFalse(update.iterator().hasNext());
+        assertFalse(update.rowIterator().hasNext());
     }
 
     private void assertRepairMetadata(Mutation mutation)
diff --git a/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java
index 290049c2e216..4d1e57ad1380 100644
--- a/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java
@@ -43,7 +43,7 @@ public class DigestResolverTest extends AbstractReadResponseTest
 {
     private static PartitionUpdate.Builder update(TableMetadata metadata, String key, Row... rows)
     {
-        PartitionUpdate.Builder builder = new PartitionUpdate.Builder(metadata, dk(key), metadata.regularAndStaticColumns(), rows.length, false);
+        PartitionUpdate.Builder builder = PartitionUpdate.builder(metadata, dk(key), metadata.regularAndStaticColumns(), rows.length);
         for (Row row: rows)
         {
             builder.add(row);
@@ -68,7 +68,7 @@ public void noRepairNeeded()
     {
         SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
         EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2));
-        DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L));
+        DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L), noopReadTracker());
 
         PartitionUpdate response = update(row(1000, 4, 4), row(1000, 5, 5)).build();
 
@@ -101,9 +101,9 @@ public void multiThreadedNoRepairNeededReadCallback()
             {
                 final long startNanos = System.nanoTime();
                 final Dispatcher.RequestTime requestTime = new Dispatcher.RequestTime(startNanos, startNanos);
-                final DigestResolver<EndpointsForToken, ReplicaPlan.ForTokenRead> resolver = new DigestResolver<>(command, plan, requestTime);
+                final DigestResolver<EndpointsForToken, ReplicaPlan.ForTokenRead> resolver = new DigestResolver<>(command, plan, requestTime, noopReadTracker());
                 final ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> callback = new ReadCallback<>(resolver, command, plan, requestTime);
-                
+
                 final CountDownLatch startlatch = new CountDownLatch(2);
 
                 pool.execute(() ->
@@ -136,7 +136,7 @@ public void digestMismatch()
     {
         SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
         EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2));
-        DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L));
+        DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L), noopReadTracker());
 
         PartitionUpdate response1 = update(row(1000, 4, 4), row(1000, 5, 5)).build();
         PartitionUpdate response2 = update(row(2000, 4, 5)).build();
@@ -157,7 +157,7 @@ public void agreeingTransient()
     {
         SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
         EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), trans(EP2));
-        DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L));
+        DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L), noopReadTracker());
 
         PartitionUpdate response1 = update(row(1000, 4, 4), row(1000, 5, 5)).build();
         PartitionUpdate response2 = update(row(1000, 5, 5)).build();
@@ -178,7 +178,7 @@ public void transientResponse()
     {
         SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
         EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), trans(EP2));
-        DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L));
+        DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L), noopReadTracker());
 
         PartitionUpdate response2 = update(row(1000, 5, 5)).build();
         Assert.assertFalse(resolver.isDataPresent());
@@ -193,7 +193,7 @@ public void transientResponseData()
     {
         SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk);
         EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2), trans(EP3));
-        DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L));
+        DigestResolver<?, ?> resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L), noopReadTracker());
 
         PartitionUpdate fullResponse = update(row(1000, 1, 1)).build();
         PartitionUpdate digestResponse = update(row(1000, 1, 1)).build();
diff --git a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java
index 68a022779ac9..c85397137e4d 100644
--- a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java
@@ -45,6 +45,7 @@
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.service.QueryInfoTracker;
 
 import static java.util.concurrent.TimeUnit.DAYS;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -95,7 +96,7 @@ public void testUnableToSpeculate() throws Throwable
     {
         assertEquals(0, cfs.metric.speculativeInsufficientReplicas.getCount());
         assertEquals(0, ks.metric.speculativeInsufficientReplicas.getCount());
-        AbstractReadExecutor executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, LOCAL_QUORUM), Dispatcher.RequestTime.forImmediateExecution(), true);
+        AbstractReadExecutor executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, LOCAL_QUORUM), Dispatcher.RequestTime.forImmediateExecution(), true, noopReadTracker());
         executor.maybeTryAdditionalReplicas();
         try
         {
@@ -110,7 +111,7 @@ public void testUnableToSpeculate() throws Throwable
         assertEquals(1, ks.metric.speculativeInsufficientReplicas.getCount());
 
         //Shouldn't increment
-        executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, LOCAL_QUORUM), Dispatcher.RequestTime.forImmediateExecution(), false);
+        executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, LOCAL_QUORUM), Dispatcher.RequestTime.forImmediateExecution(), false, noopReadTracker());
         executor.maybeTryAdditionalReplicas();
         try
         {
@@ -136,7 +137,7 @@ public void testSpeculateSucceeded() throws Throwable
         assertEquals(0, cfs.metric.speculativeFailedRetries.getCount());
         assertEquals(0, ks.metric.speculativeRetries.getCount());
         assertEquals(0, ks.metric.speculativeFailedRetries.getCount());
-        AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(DAYS.toMillis(365)), plan(LOCAL_QUORUM, targets, targets.subList(0, 2)), Dispatcher.RequestTime.forImmediateExecution());
+        AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(DAYS.toMillis(365)), plan(LOCAL_QUORUM, targets, targets.subList(0, 2)), Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         executor.maybeTryAdditionalReplicas();
         new Thread()
         {
@@ -177,7 +178,7 @@ public void testSpeculateFailed() throws Throwable
         assertEquals(0, cfs.metric.speculativeFailedRetries.getCount());
         assertEquals(0, ks.metric.speculativeRetries.getCount());
         assertEquals(0, ks.metric.speculativeFailedRetries.getCount());
-        AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(LOCAL_QUORUM, targets, targets.subList(0, 2)), Dispatcher.RequestTime.forImmediateExecution());
+        AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(LOCAL_QUORUM, targets, targets.subList(0, 2)), Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
         executor.maybeTryAdditionalReplicas();
         try
         {
@@ -203,7 +204,7 @@ public void testRaceWithNonSpeculativeFailure()
     {
         MockSinglePartitionReadCommand command = new MockSinglePartitionReadCommand(TimeUnit.DAYS.toMillis(365));
         ReplicaPlan.ForTokenRead plan = plan(ConsistencyLevel.LOCAL_ONE, targets, targets.subList(0, 1));
-        AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, command, plan, Dispatcher.RequestTime.forImmediateExecution());
+        AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, command, plan, Dispatcher.RequestTime.forImmediateExecution(), noopReadTracker());
 
         // Issue an initial request against the first endpoint...
         executor.executeAsync();
@@ -275,4 +276,9 @@ private ReplicaPlan.ForTokenRead plan(ConsistencyLevel consistencyLevel, Endpoin
     {
         return new ReplicaPlan.ForTokenRead(ks, ks.getReplicationStrategy(), consistencyLevel, natural, selected);
     }
+
+    private QueryInfoTracker.ReadTracker noopReadTracker()
+    {
+        return QueryInfoTracker.ReadTracker.NOOP;
+    }
 }
diff --git a/test/unit/org/apache/cassandra/service/reads/ReadTrackingTransformationTest.java b/test/unit/org/apache/cassandra/service/reads/ReadTrackingTransformationTest.java
new file mode 100644
index 000000000000..426c552e55d9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/ReadTrackingTransformationTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.service.QueryInfoTracker;
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+public class ReadTrackingTransformationTest
+{
+    @Test
+    public void testOnRowHandling() throws Throwable
+    {
+        QueryInfoTracker.ReadTracker readTracker = mock(QueryInfoTracker.ReadTracker.class);
+        Row row = mock(Row.class);
+
+        ReadTrackingTransformation transformation = new ReadTrackingTransformation(readTracker);
+        transformation.applyToRow(row);
+        Mockito.verify(readTracker).onRow(Mockito.eq(row));
+
+        doThrow(RuntimeException.class).when(readTracker).onRow(any(Row.class));
+        transformation.applyToRow(row); // swallows exception
+    }
+
+    @Test
+    public void testOnPartitionHandling() throws Throwable
+    {
+        QueryInfoTracker.ReadTracker readTracker = mock(QueryInfoTracker.ReadTracker.class);
+        DecoratedKey key = mock(DecoratedKey.class);
+
+        ReadTrackingTransformation transformation = new ReadTrackingTransformation(readTracker);
+        transformation.applyToPartitionKey(key);
+        Mockito.verify(readTracker).onPartition(Mockito.eq(key));
+
+        doThrow(RuntimeException.class).when(readTracker).onPartition(any(DecoratedKey.class));
+        transformation.applyToPartitionKey(key); // swallows exception
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/service/reads/range/EndpointGroupingRangeCommandIteratorTest.java b/test/unit/org/apache/cassandra/service/reads/range/EndpointGroupingRangeCommandIteratorTest.java
new file mode 100644
index 000000000000..676e7246455c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/reads/range/EndpointGroupingRangeCommandIteratorTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.reads.range;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.collect.Iterables;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.CloseableIterator;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MAX_CONCURRENT_RANGE_REQUESTS;
+
+import static org.apache.cassandra.service.QueryInfoTracker.*;
+import static org.junit.Assert.assertEquals;
+
+public class EndpointGroupingRangeCommandIteratorTest extends CQLTester
+{
+    public static final String KEYSPACE1 = "EndpointGroupingRangeReadTest";
+    public static final String CF_STANDARD1 = "Standard1";
+
+    private static final int MAX_CONCURRENCY_FACTOR = 1;
+
+    private static Keyspace keyspace;
+    private static ColumnFamilyStore cfs;
+
+    @BeforeClass
+    public static void defineSchema() throws ConfigurationException
+    {
+        MAX_CONCURRENT_RANGE_REQUESTS.setString(String.valueOf(MAX_CONCURRENCY_FACTOR));
+
+        requireNetwork();
+
+        SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1));
+
+        keyspace = Keyspace.open(KEYSPACE1);
+        cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
+        cfs.clearUnsafe();
+    }
+
+    @AfterClass
+    public static void cleanup()
+    {
+        System.clearProperty("cassandra.max_concurrent_range_requests");
+    }
+
+    @Test
+    public void testEndpointGrouping() throws Throwable
+    {
+        // n tokens divide token ring into n+1 ranges
+        int vnodeCount = setTokens(100, 200, 300, 400).size() + 1;
+
+        int rowCount = 1000;
+        for (int i = 0; i < rowCount; ++i)
+        {
+            RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 10, String.valueOf(i));
+            builder.clustering("c");
+            builder.add("val", String.valueOf(i));
+            builder.build().applyUnsafe();
+        }
+        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+        PartitionRangeReadCommand command = (PartitionRangeReadCommand) Util.cmd(cfs).build();
+
+        for (int initialConcurrencyFactor : Arrays.asList(1, 2, 5, 10, 50, 100, 250, 300, 500))
+        {
+            verifyEndpointGrouping(command, vnodeCount, initialConcurrencyFactor);
+        }
+    }
+
+    private static List<Token> setTokens(int... values)
+    {
+        return new TokenUpdater().withKeys(values).update().getTokens();
+    }
+
+    private void verifyEndpointGrouping(PartitionRangeReadCommand command, int vnodeCount, int concurrencyFactor) throws Exception
+    {
+        EndpointGroupingCoordinator coordinator = endpointGroupingCoordinator(command, concurrencyFactor, ReadTracker.NOOP);
+
+        // verify queried vnode ranges respects concurrency factor.
+        assertEquals(vnodeCount, coordinator.vnodeRanges());
+
+        // verify number of replica ranges is the same as grouped ranges.
+        int rangesForQuery = 1;
+        EndpointGroupingCoordinator.EndpointQueryContext endpointContext = Iterables.getOnlyElement(coordinator.endpointRanges());
+        assertEquals(rangesForQuery, endpointContext.rangesCount());
+
+        // verify that endpoint grouping coordinator fetches given ranges according to concurrency factor
+        RangeCommandIterator tokenOrderedIterator = tokenOrderIterator(command, vnodeCount, concurrencyFactor, ReadTracker.NOOP);
+        int expected = Util.size(tokenOrderedIterator.sendNextRequests());
+        int actual = Util.size(coordinator.execute());
+        assertEquals(expected, actual);
+
+        // verify that endpoint grouping executor fetches all data
+        RangeCommandIterator endpointGroupingIterator = endpointGroupingIterator(command, vnodeCount, concurrencyFactor, ReadTracker.NOOP);
+        tokenOrderedIterator = tokenOrderIterator(command, vnodeCount, concurrencyFactor, ReadTracker.NOOP);
+        expected = Util.size(tokenOrderedIterator.sendNextRequests());
+        actual = Util.size(endpointGroupingIterator.sendNextRequests());
+        assertEquals(1, endpointGroupingIterator.batchesRequested());
+        assertEquals(expected, actual);
+    }
+
+    private static EndpointGroupingCoordinator endpointGroupingCoordinator(PartitionRangeReadCommand command, int concurrencyFactor, ReadTracker readTracker)
+    {
+        CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans = replicaPlanIterator(command);
+        DataLimits.Counter counter = DataLimits.NONE.newCounter(command.nowInSec(), true, command.selectsFullPartition(), true);
+        return new EndpointGroupingCoordinator(command, counter, replicaPlans, concurrencyFactor, Dispatcher.RequestTime.forImmediateExecution(), readTracker);
+    }
+
+    private static EndpointGroupingRangeCommandIterator endpointGroupingIterator(PartitionRangeReadCommand command, int vnodeCount, int concurrencyFactor, ReadTracker readTracker)
+    {
+        CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans = replicaPlanIterator(command);
+        return new EndpointGroupingRangeCommandIterator(replicaPlans, command, concurrencyFactor, concurrencyFactor, vnodeCount, Dispatcher.RequestTime.forImmediateExecution(), readTracker);
+    }
+
+    private static NonGroupingRangeCommandIterator tokenOrderIterator(PartitionRangeReadCommand command, int vnodeCount, int concurrencyFactor, ReadTracker readTracker)
+    {
+        CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans = replicaPlanIterator(command);
+        return new NonGroupingRangeCommandIterator(replicaPlans, command, concurrencyFactor, 10000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution(), readTracker);
+    }
+
+    private static CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlanIterator(PartitionRangeReadCommand command)
+    {
+        AbstractBounds<PartitionPosition> keyRange = command.dataRange().keyRange();
+        CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans = new ReplicaPlanIterator(keyRange, null, keyspace, ConsistencyLevel.ONE);
+        return new ReplicaPlanMerger(replicaPlans, keyspace, ConsistencyLevel.ONE);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java
index dfd1f7f88d15..0c1d3e7a031b 100644
--- a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java
@@ -43,6 +43,7 @@
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.CloseableIterator;
 
+import static org.apache.cassandra.service.QueryInfoTracker.*;
 import static org.junit.Assert.assertEquals;
 
 public class RangeCommandIteratorTest
@@ -105,27 +106,27 @@ public void testRangeQueried()
 
         // without range merger, there will be 2 batches requested: 1st batch with 1 range and 2nd batch with remaining ranges
         CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans = replicaPlanIterator(keyRange, keyspace, false);
-        RangeCommandIterator data = new RangeCommandIterator(replicaPlans, command, 1, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution());
+        RangeCommandIterator data = RangeCommandIterator.create(replicaPlans, command, 1, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution(), ReadTracker.NOOP);
         verifyRangeCommandIterator(data, rows, 2, vnodeCount);
 
         // without range merger and initial cf=5, there will be 1 batches requested: 5 vnode ranges for 1st batch
         replicaPlans = replicaPlanIterator(keyRange, keyspace, false);
-        data = new RangeCommandIterator(replicaPlans, command, vnodeCount, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution());
+        data = RangeCommandIterator.create(replicaPlans, command, vnodeCount, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution(), ReadTracker.NOOP);
         verifyRangeCommandIterator(data, rows, 1, vnodeCount);
 
         // without range merger and max cf=1, there will be 5 batches requested: 1 vnode range per batch
         replicaPlans = replicaPlanIterator(keyRange, keyspace, false);
-        data = new RangeCommandIterator(replicaPlans, command, 1, 1, vnodeCount, Dispatcher.RequestTime.forImmediateExecution());
+        data = RangeCommandIterator.create(replicaPlans, command, 1, 1, vnodeCount, Dispatcher.RequestTime.forImmediateExecution(), ReadTracker.NOOP);
         verifyRangeCommandIterator(data, rows, vnodeCount, vnodeCount);
 
         // with range merger, there will be only 1 batch requested, as all ranges share the same replica - localhost
         replicaPlans = replicaPlanIterator(keyRange, keyspace, true);
-        data = new RangeCommandIterator(replicaPlans, command, 1, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution());
+        data = RangeCommandIterator.create(replicaPlans, command, 1, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution(), ReadTracker.NOOP);
         verifyRangeCommandIterator(data, rows, 1, vnodeCount);
 
         // with range merger and max cf=1, there will be only 1 batch requested, as all ranges share the same replica - localhost
         replicaPlans = replicaPlanIterator(keyRange, keyspace, true);
-        data = new RangeCommandIterator(replicaPlans, command, 1, 1, vnodeCount, Dispatcher.RequestTime.forImmediateExecution());
+        data = RangeCommandIterator.create(replicaPlans, command, 1, 1, vnodeCount, Dispatcher.RequestTime.forImmediateExecution(), ReadTracker.NOOP);
         verifyRangeCommandIterator(data, rows, 1, vnodeCount);
     }
 
diff --git a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java
index ce04d5deab54..f14b7e4f60c7 100644
--- a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java
@@ -28,6 +28,7 @@
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.db.AbstractReadCommandBuilder;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
@@ -42,6 +43,7 @@
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.MAX_CONCURRENT_RANGE_REQUESTS;
 import static org.apache.cassandra.db.ConsistencyLevel.ONE;
+import static org.apache.cassandra.service.QueryInfoTracker.*;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -78,7 +80,7 @@ public void tesConcurrencyFactor()
 
         // verify that a low concurrency factor is not capped by the max concurrency factor
         PartitionRangeReadCommand command = command(cfs, 50, 50);
-        try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, Dispatcher.RequestTime.forImmediateExecution());
+        try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, Dispatcher.RequestTime.forImmediateExecution(), ReadTracker.NOOP);
              ReplicaPlanIterator ranges = new ReplicaPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, ONE))
         {
             assertEquals(2, partitions.concurrencyFactor());
@@ -88,7 +90,7 @@ public void tesConcurrencyFactor()
 
         // verify that a high concurrency factor is capped by the max concurrency factor
         command = command(cfs, 1000, 50);
-        try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, Dispatcher.RequestTime.forImmediateExecution());
+        try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, Dispatcher.RequestTime.forImmediateExecution(), ReadTracker.NOOP);
              ReplicaPlanIterator ranges = new ReplicaPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, ONE))
         {
             assertEquals(MAX_CONCURRENCY_FACTOR, partitions.concurrencyFactor());
@@ -98,7 +100,7 @@ public void tesConcurrencyFactor()
 
         // with 0 estimated results per range the concurrency factor should be 1
         command = command(cfs, 1000, 0);
-        try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, Dispatcher.RequestTime.forImmediateExecution());
+        try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, Dispatcher.RequestTime.forImmediateExecution(), ReadTracker.NOOP);
              ReplicaPlanIterator ranges = new ReplicaPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, ONE))
         {
             assertEquals(1, partitions.concurrencyFactor());
@@ -220,13 +222,13 @@ public boolean isDistinct()
         }
 
         @Override
-        public DataLimits forPaging(int pageSize)
+        public DataLimits forPaging(PageSize pageSize)
         {
             return wrapped.forPaging(pageSize);
         }
 
         @Override
-        public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+        public DataLimits forPaging(PageSize pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
         {
             return wrapped.forPaging(pageSize, lastReturnedKey, lastReturnedKeyRemaining);
         }
@@ -249,6 +251,18 @@ public Counter newCounter(long nowInSec, boolean assumeLiveData, boolean countPa
             return wrapped.newCounter(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData, enforceStrictLiveness);
         }
 
+        @Override
+        public int bytes()
+        {
+            return wrapped.bytes();
+        }
+
+        @Override
+        public int rows()
+        {
+            return wrapped.rows();
+        }
+
         @Override
         public int count()
         {
@@ -266,6 +280,18 @@ public DataLimits withoutState()
         {
             return wrapped.withoutState();
         }
+
+        @Override
+        public DataLimits withCountedLimit(int newCountedLimit)
+        {
+            return wrapped.withCountedLimit(newCountedLimit);
+        }
+
+        @Override
+        public DataLimits withBytesLimit(int bytesLimit)
+        {
+            return wrapped.withBytesLimit(bytesLimit);
+        }
     }
 
     public static final class MockedIndex extends StubIndex
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java
index 51caa94801f7..9d3485224b5c 100644
--- a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java
@@ -77,12 +77,14 @@
 import org.apache.cassandra.schema.Tables;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.service.reads.DigestResolver;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.apache.cassandra.locator.Replica.fullReplica;
 import static org.apache.cassandra.locator.ReplicaUtils.FULL_RANGE;
 import static org.apache.cassandra.net.Verb.INTERNAL_RSP;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.mockito.Mockito.mock;
 
 @Ignore
 public abstract  class AbstractReadRepairTest
@@ -164,7 +166,7 @@ static void assertMutationEqual(Mutation expected, Mutation actual)
         Assert.assertEquals(expected.key(), actual.key());
         PartitionUpdate expectedUpdate = Iterables.getOnlyElement(expected.getPartitionUpdates());
         PartitionUpdate actualUpdate = Iterables.getOnlyElement(actual.getPartitionUpdates());
-        assertRowsEqual(Iterables.getOnlyElement(expectedUpdate), Iterables.getOnlyElement(actualUpdate));
+        assertRowsEqual(Iterables.getOnlyElement(expectedUpdate.rows()), Iterables.getOnlyElement(actualUpdate.rows()));
     }
 
     static DecoratedKey dk(int v)
@@ -386,7 +388,7 @@ public void readSpeculationCycle()
         ResultConsumer consumer = new ResultConsumer();
 
         Assert.assertEquals(epSet(), repair.getReadRecipients());
-        repair.startRepair(null, consumer);
+        repair.startRepair(mock(DigestResolver.class), consumer);
 
         Assert.assertEquals(epSet(target1, target2), repair.getReadRecipients());
         repair.maybeSendAdditionalReads();
@@ -405,7 +407,7 @@ public void noSpeculationRequired()
         ResultConsumer consumer = new ResultConsumer();
 
         Assert.assertEquals(epSet(), repair.getReadRecipients());
-        repair.startRepair(null, consumer);
+        repair.startRepair(mock(DigestResolver.class), consumer);
 
         Assert.assertEquals(epSet(target1, target2), repair.getReadRecipients());
         repair.getReadCallback().onResponse(msg(target1, cell1));
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java
index e3de74be15ae..11e9c136704b 100644
--- a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java
@@ -182,11 +182,12 @@ public void consistencyLevelTest() throws Exception
     private static void assertMutationEqual(Mutation expected, Mutation actual)
     {
         Assert.assertEquals(expected.getKeyspaceName(), actual.getKeyspaceName());
+        Assert.assertEquals(expected.getKeyspace(), actual.getKeyspace());
         Assert.assertEquals(expected.key(), actual.key());
         Assert.assertEquals(expected.key(), actual.key());
         PartitionUpdate expectedUpdate = Iterables.getOnlyElement(expected.getPartitionUpdates());
         PartitionUpdate actualUpdate = Iterables.getOnlyElement(actual.getPartitionUpdates());
-        assertRowsEqual(Iterables.getOnlyElement(expectedUpdate), Iterables.getOnlyElement(actualUpdate));
+        assertRowsEqual(Iterables.getOnlyElement(expectedUpdate.rows()), Iterables.getOnlyElement(actualUpdate.rows()));
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java b/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java
index 410e56a98111..e3f1a0336e0d 100644
--- a/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java
@@ -243,12 +243,12 @@ public void noTrackingDataRecorded()
 
     private long confirmedCount()
     {
-        return metrics.confirmedRepairedInconsistencies.table.getCount();
+        return metrics.confirmedRepairedInconsistencies.tableOrKeyspaceMeter().getCount();
     }
 
     private long unconfirmedCount()
     {
-        return metrics.unconfirmedRepairedInconsistencies.table.getCount();
+        return metrics.unconfirmedRepairedInconsistencies.tableOrKeyspaceMeter().getCount();
     }
 
     private InetAddressAndPort peer()
diff --git a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
index 9c2303f84293..cba3b371a1bf 100644
--- a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
+++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
@@ -21,7 +21,6 @@
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.time.Instant;
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.UUID;
@@ -93,9 +92,9 @@ public void testNoSnapshots() throws IOException
         }
 
         // Check no snapshots are found
-        SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_3)));
+        SnapshotLoader loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
         assertThat(loader.loadSnapshots()).isEmpty();
     }
 
@@ -119,9 +118,9 @@ public void testSnapshotsWithoutManifests() throws IOException
         }
 
         // Verify all 3 snapshots are found correctly from data directories
-        SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_3)));
+        SnapshotLoader loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
         Set<TableSnapshot> snapshots = loader.loadSnapshots();
         assertThat(snapshots).hasSize(3);
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, false));
@@ -129,18 +128,18 @@ public void testSnapshotsWithoutManifests() throws IOException
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID,  TAG3, null, null, tag3Files, false));
 
         // Verify snapshot loading for a specific keyspace
-        loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                  Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                  Paths.get(baseDir.toString(), DATA_DIR_3)));
+        loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
 
         snapshots = loader.loadSnapshots(KEYSPACE_1);
         assertThat(snapshots).hasSize(2);
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, false));
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID,  TAG2, null, null, tag2Files, false));
 
-        loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                  Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                  Paths.get(baseDir.toString(), DATA_DIR_3)));
+        loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
         snapshots = loader.loadSnapshots(KEYSPACE_2);
         assertThat(snapshots).hasSize(1);
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID,  TAG3, null, null, tag3Files, false));
@@ -167,9 +166,9 @@ public void testEphemeralSnapshotWithoutManifest() throws IOException
         }
 
         // Verify snapshot is found correctly from data directories
-        SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_3)));
+        SnapshotLoader loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
 
         Set<TableSnapshot> snapshots = loader.loadSnapshots();
         assertThat(snapshots).hasSize(1);
@@ -213,9 +212,9 @@ public void testSnapshotsWithManifests() throws IOException
         writeManifest(tag3ManifestLocation, tag3Ts, null);
 
         // Verify all 3 snapshots are found correctly from data directories
-        SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_3)));
+        SnapshotLoader loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
         Set<TableSnapshot> snapshots = loader.loadSnapshots();
         assertThat(snapshots).hasSize(3);
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files, false));
@@ -223,18 +222,18 @@ public void testSnapshotsWithManifests() throws IOException
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID,  TAG3, tag3Ts, null, tag3Files, false));
 
         // Verify snapshot loading for a specific keyspace
-        loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                  Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                  Paths.get(baseDir.toString(), DATA_DIR_3)));
+        loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
 
         snapshots = loader.loadSnapshots(KEYSPACE_1);
         assertThat(snapshots).hasSize(2);
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files, false));
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID,  TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files, false));
 
-        loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                  Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                  Paths.get(baseDir.toString(), DATA_DIR_3)));
+        loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
         snapshots = loader.loadSnapshots(KEYSPACE_2);
         assertThat(snapshots).hasSize(1);
         assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID,  TAG3, tag3Ts, null, tag3Files, false));
@@ -260,9 +259,9 @@ public void testInvalidSnapshotsAreNotLoaded() throws IOException
         }
 
         // Check no snapshots are loaded
-        SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_2),
-                                                                 Paths.get(baseDir.toString(), DATA_DIR_3)));
+        SnapshotLoader loader = new SnapshotLoader(new File[] {new File(Paths.get(baseDir.toString(), DATA_DIR_1)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_2)),
+                                                               new File(Paths.get(baseDir.toString(), DATA_DIR_3))});
         assertThat(loader.loadSnapshots()).isEmpty();
     }
 
diff --git a/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java b/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
index 778b6b2d7115..755126bff707 100644
--- a/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
+++ b/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
@@ -21,14 +21,23 @@
 import java.util.Collection;
 import java.util.Collections;
 
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.utils.FBUtilities;
 
 public class SessionInfoTest
 {
+    @BeforeClass
+    public static void beforeClass()
+    {
+        DatabaseDescriptor.setConfig(new Config());
+    }
+
     /**
      * Test if total numbers are collect
      */
diff --git a/test/unit/org/apache/cassandra/streaming/StreamOperationTest.java b/test/unit/org/apache/cassandra/streaming/StreamOperationTest.java
index 2cc216eb0d7b..b28e95d01563 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamOperationTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamOperationTest.java
@@ -19,6 +19,8 @@
 
 import org.junit.Test;
 
+import org.apache.cassandra.db.compaction.OperationType;
+
 import static org.junit.Assert.assertEquals;
 
 public class StreamOperationTest
@@ -36,6 +38,8 @@ public void testSerialization()
         assertEquals(StreamOperation.REBUILD, StreamOperation.fromString("Rebuild"));
         assertEquals(StreamOperation.BULK_LOAD, StreamOperation.fromString("Bulk Load"));
         assertEquals(StreamOperation.REPAIR, StreamOperation.fromString("Repair"));
+        assertEquals(StreamOperation.REGION_DECOMMISSION, StreamOperation.fromString("Region Decommission"));
+        assertEquals(StreamOperation.REGION_REPAIR, StreamOperation.fromString("Region Repair"));
         // Test case insensivity
         assertEquals(StreamOperation.REPAIR, StreamOperation.fromString("rEpair"));
 
@@ -44,4 +48,16 @@ public void testSerialization()
         assertEquals("Restore replica count", StreamOperation.RESTORE_REPLICA_COUNT.getDescription());
 
     }
+
+    @Test
+    public void testOpType()
+    {
+        assertEquals(StreamOperation.OTHER.opType(), OperationType.STREAM);
+        assertEquals(StreamOperation.RESTORE_REPLICA_COUNT.opType(), OperationType.STREAM);
+        assertEquals(StreamOperation.RELOCATION.opType(), OperationType.STREAM);
+        assertEquals(StreamOperation.REBUILD.opType(), OperationType.STREAM);
+        assertEquals(StreamOperation.REPAIR.opType(), OperationType.STREAM);
+        assertEquals(StreamOperation.REGION_REPAIR.opType(), OperationType.REGION_REPAIR);
+        assertEquals(StreamOperation.REGION_DECOMMISSION.opType(), OperationType.REGION_DECOMMISSION);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/streaming/StreamSessionTest.java b/test/unit/org/apache/cassandra/streaming/StreamSessionTest.java
index 4073a466a3a1..ef58a9b1adb5 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamSessionTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamSessionTest.java
@@ -103,7 +103,7 @@ public void basicDiskSpaceTest() throws InterruptedException
         do
         {
             Thread.sleep(100);
-        } while (!CompactionManager.instance.active.getCompactions().isEmpty());
+        } while (!CompactionManager.instance.active.getTableOperations().isEmpty());
 
         assertTrue(StreamSession.checkDiskSpace(perTableIdIncomingBytes, nextTimeUUID(), filestoreMapper));
 
@@ -137,7 +137,7 @@ public void multiTableDiskSpaceTest() throws InterruptedException
         do
         {
             Thread.sleep(100);
-        } while (!CompactionManager.instance.active.getCompactions().isEmpty());
+        } while (!CompactionManager.instance.active.getTableOperations().isEmpty());
 
         assertTrue(StreamSession.checkDiskSpace(perTableIdIncomingBytes, nextTimeUUID(), filestoreMapper));
 
diff --git a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
index debb03e705dd..1be4051ce9a5 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
@@ -155,7 +155,7 @@ public void testFailSessionDuringTransferShouldNotReleaseReferences() throws Exc
 
         // create streaming task that streams those two sstables
         StreamTransferTask task = new StreamTransferTask(session, cfs.metadata.id);
-        List<Ref<SSTableReader>> refs = new ArrayList<>(cfs.getLiveSSTables().size());
+        List<Ref<? extends SSTableReader>> refs = new ArrayList<>(cfs.getLiveSSTables().size());
         for (SSTableReader sstable : cfs.getLiveSSTables())
         {
             List<Range<Token>> ranges = new ArrayList<>();
@@ -182,7 +182,7 @@ public void testFailSessionDuringTransferShouldNotReleaseReferences() throws Exc
         session.onError(new Exception("Fake exception")).get(5, TimeUnit.SECONDS);
 
         //make sure reference was not released
-        for (Ref<SSTableReader> ref : refs)
+        for (Ref<? extends SSTableReader> ref : refs)
         {
             assertEquals(1, ref.globalCount());
         }
@@ -204,7 +204,7 @@ public void testFailSessionDuringTransferShouldNotReleaseReferences() throws Exc
         }
 
         //now reference should be released
-        for (Ref<SSTableReader> ref : refs)
+        for (Ref<? extends SSTableReader> ref : refs)
         {
             assertEquals(0, ref.globalCount());
         }
diff --git a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
index ebd392ced023..1aa27c396ce2 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
@@ -202,7 +202,7 @@ private List<String> createAndTransfer(ColumnFamilyStore cfs, Mutator mutator, b
         assertEquals(1, cfs.getLiveSSTables().size());
 
         // and that the index and filter were properly recovered
-        List<ImmutableBTreePartition> partitions = Util.getAllUnfiltered(Util.cmd(cfs).build());
+        List<? extends Partition> partitions = Util.getAllUnfiltered(Util.cmd(cfs).build());
         assertEquals(offs.length, partitions.size());
         for (int i = 0; i < offs.length; i++)
         {
@@ -210,9 +210,9 @@ private List<String> createAndTransfer(ColumnFamilyStore cfs, Mutator mutator, b
             String col = "col" + offs[i];
 
             assert !Util.getAll(Util.cmd(cfs, key).build()).isEmpty();
-            ImmutableBTreePartition partition = partitions.get(i);
+            Partition partition = partitions.get(i);
             assert ByteBufferUtil.compareUnsigned(partition.partitionKey().getKey(), ByteBufferUtil.bytes(key)) == 0;
-            assert ByteBufferUtil.compareUnsigned(partition.iterator().next().clustering().bufferAt(0), ByteBufferUtil.bytes(col)) == 0;
+            assert ByteBufferUtil.compareUnsigned(partition.unfilteredIterator().next().clustering().bufferAt(0), ByteBufferUtil.bytes(col)) == 0;
         }
 
         // and that the max timestamp for the file was rediscovered
diff --git a/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java b/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java
index 76ed9ba04b14..b92fe4baf5e0 100644
--- a/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java
+++ b/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java
@@ -30,7 +30,6 @@
 
 import org.apache.cassandra.io.util.File;
 import org.apache.commons.io.FileUtils;
-
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -38,9 +37,9 @@
 
 import net.openhft.chronicle.core.io.IORuntimeException;
 import net.openhft.chronicle.queue.ChronicleQueue;
-import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.queue.ExcerptAppender;
 import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.wire.WireOut;
 import org.apache.cassandra.audit.BinAuditLogger;
 import org.apache.cassandra.tools.ToolRunner.ObservableTool;
diff --git a/test/unit/org/apache/cassandra/tools/CompactionLogAnalyzerTest.java b/test/unit/org/apache/cassandra/tools/CompactionLogAnalyzerTest.java
new file mode 100644
index 000000000000..b949dd0a852a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/CompactionLogAnalyzerTest.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.cassandra.tools;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+public class CompactionLogAnalyzerTest
+{
+    private final String UNREPAIRED_SHARD_NAME = "2-unrepaired";
+    private final String REPAIRED_SHARD_NAME = "2-repaired";
+
+    private File[] logFiles;
+
+    @Before
+    public void before()
+    {
+        ClassLoader classLoader = getClass().getClassLoader();
+        File dir = new File(classLoader.getResource("org/apache/cassandra/tools/compaction_logs").getFile());
+        Assert.assertTrue(dir.isDirectory());
+
+        logFiles = dir.listFiles();
+        Assert.assertEquals(2, logFiles.length);
+        Arrays.sort(logFiles, Comparator.comparing(File::getName));
+    }
+
+    @Test
+    public void testReadDataPointsWithoutLimit() throws IOException, ParseException
+    {
+        List<CompactionLogAnalyzer.DataPoint> dataPoints = CompactionLogAnalyzer.readDataPoints(logFiles, null);
+
+        // We subtract 1 for the header line, and 2 for the two corrupted lines in the first test file.
+        long lineCount0 = Files.lines(logFiles[0].toPath()).count() - 1 - 2;
+        long lineCount1 = Files.lines(logFiles[1].toPath()).count() - 1;
+        Assert.assertEquals(lineCount0 + lineCount1, dataPoints.size());
+
+        int shard0Cnt = 0;
+        int shard1Cnt = 0;
+        for (CompactionLogAnalyzer.DataPoint dataPoint : dataPoints)
+        {
+            if (dataPoint.shardId.equals(REPAIRED_SHARD_NAME))
+            {
+                shard0Cnt++;
+            }
+            else if (dataPoint.shardId.equals(UNREPAIRED_SHARD_NAME))
+            {
+                shard1Cnt++;
+            }
+            else
+            {
+                throw new AssertionError("Unexpected shard id");
+            }
+        }
+        Assert.assertEquals(lineCount0, shard0Cnt);
+        Assert.assertEquals(lineCount1, shard1Cnt);
+    }
+
+    @Test
+    public void testReadDataPointsWithLimit() throws IOException, ParseException
+    {
+        int limit = 100;
+
+        List<CompactionLogAnalyzer.DataPoint> dataPoints = CompactionLogAnalyzer.readDataPoints(logFiles, limit);
+
+        Assert.assertEquals(2 * limit, dataPoints.size()); // without headers
+
+        int shard0Cnt = 0;
+        int shard1Cnt = 0;
+        for (CompactionLogAnalyzer.DataPoint dataPoint : dataPoints)
+        {
+            if (dataPoint.shardId.equals(REPAIRED_SHARD_NAME))
+            {
+                shard0Cnt++;
+            }
+            else if (dataPoint.shardId.equals(UNREPAIRED_SHARD_NAME))
+            {
+                shard1Cnt++;
+            }
+            else
+            {
+                throw new AssertionError("Unexpected shard id");
+            }
+        }
+        Assert.assertEquals(limit, shard0Cnt);
+        Assert.assertEquals(limit, shard1Cnt);
+    }
+
+    @Test
+    public void testProcessDataPoints() throws IOException, ParseException
+    {
+        List<CompactionLogAnalyzer.DataPoint> dataPoints = CompactionLogAnalyzer.readDataPoints(logFiles, null);
+        JSONArray jsonArray = CompactionLogAnalyzer.processData(dataPoints);
+        int expectedLevels = dataPoints.stream()
+                                       .mapToInt(dp -> dp.bucket)
+                                       .max()
+                                       .getAsInt() + 1; // + L0
+        Assert.assertEquals(expectedLevels + 1, jsonArray.size());  // + Total
+
+        JSONArray[] intervals = new JSONArray[expectedLevels + 1];
+        for (int i = 0; i < intervals.length; i++)
+        {
+            intervals[i] = (JSONArray) ((JSONObject) jsonArray.get(0)).get("intervals");
+        }
+
+        for (int i = 1; i < intervals.length; i++)
+        {
+            Assert.assertEquals(intervals[0].size(), intervals[i].size());
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/tools/JMXCompatibilityTest.java b/test/unit/org/apache/cassandra/tools/JMXCompatibilityTest.java
index 60df47757763..34ac6c911c4d 100644
--- a/test/unit/org/apache/cassandra/tools/JMXCompatibilityTest.java
+++ b/test/unit/org/apache/cassandra/tools/JMXCompatibilityTest.java
@@ -31,7 +31,10 @@
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.CassandraDaemon;
 import org.apache.cassandra.service.GCInspector;
@@ -90,6 +93,9 @@ private void setupStandardTables() throws Throwable
         GCInspector.register();
         CassandraDaemon.registerNativeAccess();
 
+        // CC5 fix needed after CNDB-10289
+        SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES.forEach(k -> Keyspace.open(k).flush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
+
         String name = KEYSPACE + '.' + createTable("CREATE TABLE %s (pk int PRIMARY KEY)");
 
         // use net to register everything like storage proxy
diff --git a/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java b/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java
index 87b9ff93fded..347f990504d2 100644
--- a/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java
+++ b/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java
@@ -131,6 +131,8 @@ public void interfaces() throws ClassNotFoundException
                 for (int i = 0; i < methods.length; i++)
                 {
                     Method method = methods[i];
+                    if (method.getName().startsWith("$jacocoInit"))
+                        continue;
                     checkType(method, "return", method.getGenericReturnType(), warnings, errors);
                     Stream.of(method.getGenericParameterTypes()).forEach(t -> checkType(method, "parameter", t, warnings, errors));
                     Stream.of(method.getGenericExceptionTypes()).forEach(t -> checkType(method, "throws", t, warnings, errors));
diff --git a/test/unit/org/apache/cassandra/tools/NodeProbeTest.java b/test/unit/org/apache/cassandra/tools/NodeProbeTest.java
index 2947a26d25e3..380b870e3cc4 100644
--- a/test/unit/org/apache/cassandra/tools/NodeProbeTest.java
+++ b/test/unit/org/apache/cassandra/tools/NodeProbeTest.java
@@ -19,6 +19,8 @@
 package org.apache.cassandra.tools;
 
 import java.io.IOException;
+import java.io.PrintStream;
+import java.util.concurrent.ExecutionException;
 
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -74,6 +76,37 @@ public void testCheckJobs()
         assertToolResult(ToolRunner.invokeNodetool("upgradesstables", "-j", String.valueOf(jobs)));
     }
 
+    @Test
+    public void testVerifyWithoutValidateAllRows() throws IOException, ExecutionException, InterruptedException
+    {
+        ToolResult toolResult = ToolRunner.invokeNodetool("verify", "--force", "--extended-verify");
+        toolResult.assertOnCleanExit();
+        assertThat(toolResult.getStdout()).isEmpty();
+
+        for (String keyspace : probe.getKeyspaces())
+        {
+            PrintStream out = probe.output().out;
+            probe.verify(out, true, false, false, false, false, false, false, keyspace);
+            probe.verify(true, false, false, false, false, false, false, keyspace);
+        }
+    }
+
+    @Test
+    public void testVerifyWithValidateAllRows() throws IOException, ExecutionException, InterruptedException
+    {
+        // new config is recognized
+        ToolResult toolResult = ToolRunner.invokeNodetool("verify", "--force", "--extended-verify", "--validate-all-rows");
+        toolResult.assertOnCleanExit();
+        assertThat(toolResult.getStdout()).isEmpty();
+
+        for (String keyspace : probe.getKeyspaces())
+        {
+            PrintStream out = probe.output().out;
+            probe.verify(out, true, true, false, false, false, false, false, keyspace);
+            probe.verify(true, true, false, false, false, false, false, keyspace);
+        }
+    }
+
     private static void assertToolResult(ToolResult toolResult)
     {
         assertThat(toolResult.getStdout()).isEmpty();
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportSchemaLoadingTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportSchemaLoadingTest.java
index b713144c1305..01cbbfa2e7fd 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExportSchemaLoadingTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportSchemaLoadingTest.java
@@ -19,8 +19,10 @@
 package org.apache.cassandra.tools;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -183,7 +185,12 @@ public void testJSONLineArg() throws IOException
      */
     private void assertPostTestEnv()
     {
-        assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, true);
+        // SSTableExport static init calls toolInitialization using test/cassandra.yaml which enables the chunk cache
+        // (file_cache_enabled=true) and starts a ChunkCacheCleanup ParkedExecutor thread.
+        String[] threadsStartedBySSTableExport = new String[] { "ChunkCacheCleanup:[1-9]", "ParkedThreadsMonitor" };
+        String[] optionalThreadNames = Stream.concat(Arrays.stream(OPTIONAL_THREADS_WITH_SCHEMA), Arrays.stream(threadsStartedBySSTableExport))
+                                  .toArray(String[]::new);
+        assertNoUnexpectedThreadsStarted(optionalThreadNames, true);
         assertCLSMNotLoaded();
         assertSystemKSNotLoaded();
         assertKeyspaceNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java b/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java
index 7899a79341b3..273cd015ef90 100644
--- a/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java
@@ -24,6 +24,9 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
@@ -39,6 +42,8 @@
  */
 public class SSTablePartitionsTest extends OfflineToolUtils
 {
+    private final static Logger logger = LoggerFactory.getLogger(SSTablePartitionsTest.class);
+
     private static final String SSTABLE_1 = sstable("legacy_ma_simple");
     private static final String SSTABLE_2 = sstable("legacy_ma_clust");
     private static final String HEADER_1 = "\nProcessing  #1 (big-ma) (173 B uncompressed, 88 B on disk)\n";
@@ -48,14 +53,14 @@ public class SSTablePartitionsTest extends OfflineToolUtils
     private static final String SNAPSHOTS_HEADER_1 = "\nProcessing Snapshot:snapshot-1  #1 (big-ma) (173 B uncompressed, 88 B on disk)\n";
     private static final String SNAPSHOTS_HEADER_2 = "\nProcessing Snapshot:snapshot-1  #1 (big-ma) (328.145 KiB uncompressed, 5.096 KiB on disk)\n";
     private static final String SUMMARY_1 = "               Partition size            Row count           Cell count      Tombstone count\n" +
-                                            "  ~p50                   35 B                    1                    1                    0\n" +
-                                            "  ~p75                   35 B                    1                    1                    0\n" +
-                                            "  ~p90                   35 B                    1                    1                    0\n" +
-                                            "  ~p95                   35 B                    1                    1                    0\n" +
-                                            "  ~p99                   35 B                    1                    1                    0\n" +
-                                            "  ~p999                  35 B                    1                    1                    0\n" +
-                                            "  min                    33 B                    1                    1                    0\n" +
-                                            "  max                    35 B                    1                    1                    0\n" +
+                                            "  ~p50                   20 B                    1                    1                    0\n" +
+                                            "  ~p75                   20 B                    1                    1                    0\n" +
+                                            "  ~p90                   20 B                    1                    1                    0\n" +
+                                            "  ~p95                   20 B                    1                    1                    0\n" +
+                                            "  ~p99                   20 B                    1                    1                    0\n" +
+                                            "  ~p999                  20 B                    1                    1                    0\n" +
+                                            "  min                    18 B                    1                    1                    0\n" +
+                                            "  max                    20 B                    1                    1                    0\n" +
                                             "  count                     5\n";
     private static final String SUMMARY_2 = "               Partition size            Row count           Cell count      Tombstone count\n" +
                                             "  ~p50             71.735 KiB                   50                   50                    0\n" +
@@ -64,8 +69,8 @@ public class SSTablePartitionsTest extends OfflineToolUtils
                                             "  ~p95             71.735 KiB                   50                   50                    0\n" +
                                             "  ~p99             71.735 KiB                   50                   50                    0\n" +
                                             "  ~p999            71.735 KiB                   50                   50                    0\n" +
-                                            "  min              65.625 KiB                   50                   50                    0\n" +
-                                            "  max              65.630 KiB                   50                   50                    0\n" +
+                                            "  min              65.610 KiB                   50                   50                    0\n" +
+                                            "  max              65.615 KiB                   50                   50                    0\n" +
                                             "  count                     5\n";
 
     @BeforeClass
@@ -208,14 +213,14 @@ private static void testPartitionsOnly(String option)
         assertThatToolSucceds(SSTABLE_1, option)
                 .isEqualTo(HEADER_1 +
                            "               Partition size\n" +
-                           "  ~p50                   35 B\n" +
-                           "  ~p75                   35 B\n" +
-                           "  ~p90                   35 B\n" +
-                           "  ~p95                   35 B\n" +
-                           "  ~p99                   35 B\n" +
-                           "  ~p999                  35 B\n" +
-                           "  min                    33 B\n" +
-                           "  max                    35 B\n" +
+                           "  ~p50                   20 B\n" +
+                           "  ~p75                   20 B\n" +
+                           "  ~p90                   20 B\n" +
+                           "  ~p95                   20 B\n" +
+                           "  ~p99                   20 B\n" +
+                           "  ~p999                  20 B\n" +
+                           "  min                    18 B\n" +
+                           "  max                    20 B\n" +
                            "  count                     5\n");
 
         assertThatToolSucceds(SSTABLE_2, "--partitions-only")
@@ -227,8 +232,8 @@ private static void testPartitionsOnly(String option)
                            "  ~p95             71.735 KiB\n" +
                            "  ~p99             71.735 KiB\n" +
                            "  ~p999            71.735 KiB\n" +
-                           "  min              65.625 KiB\n" +
-                           "  max              65.630 KiB\n" +
+                           "  min              65.610 KiB\n" +
+                           "  max              65.615 KiB\n" +
                            "  count                     5\n");
     }
 
@@ -246,34 +251,34 @@ private static void testMinSize(String option)
     {
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "35")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 65.610 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_2 + "\n" +
                            "  5 partitions match\n" +
                            "  Keys: 0 1 2 3 4\n" +
                            SUMMARY_2 +
                            HEADER_1 +
-                           "  Partition: '1' (31) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "Summary of  #1 (big-ma):\n" +
-                           "  File: " + SSTABLE_1 + "\n" +
-                           "  4 partitions match\n" +
-                           "  Keys: 1 2 3 4\n" +
+//                           "  Partition: '1' (31) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '2' (32) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '3' (33) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '4' (34) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "Summary of  #1 (big-ma):\n" +
+//                           "  File: " + SSTABLE_1 + "\n" +
+//                           "  4 partitions match\n" +
+//                           "  Keys: 1 2 3 4\n" +
                            SUMMARY_1);
 
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "36")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 65.610 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_2 + "\n" +
                            "  5 partitions match\n" +
@@ -282,35 +287,35 @@ private static void testMinSize(String option)
 
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "67201")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "Summary of  #1 (big-ma):\n" +
-                           "  File: " + SSTABLE_2 + "\n" +
-                           "  4 partitions match\n" +
-                           "  Keys: 1 2 3 4\n" +
+//                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "Summary of  #1 (big-ma):\n" +
+//                           "  File: " + SSTABLE_2 + "\n" +
+//                           "  4 partitions match\n" +
+//                           "  Keys: 1 2 3 4\n" +
                            SUMMARY_2 + HEADER_1 + SUMMARY_1);
 
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "67201B")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "Summary of  #1 (big-ma):\n" +
-                           "  File: " + SSTABLE_2 + "\n" +
-                           "  4 partitions match\n" +
-                           "  Keys: 1 2 3 4\n" +
+//                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+//                           "Summary of  #1 (big-ma):\n" +
+//                           "  File: " + SSTABLE_2 + "\n" +
+//                           "  4 partitions match\n" +
+//                           "  Keys: 1 2 3 4\n" +
                            SUMMARY_2 + HEADER_1 + SUMMARY_1);
 
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "65KiB")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 65.610 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_2 + "\n" +
                            "  5 partitions match\n" +
@@ -332,22 +337,22 @@ private static void testMinCells(String option)
     {
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 65.610 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_2 + "\n" +
                            "  5 partitions match\n" +
                            "  Keys: 0 1 2 3 4\n" +
                            SUMMARY_2 +
                            HEADER_1 +
-                           "  Partition: '0' (30) live, size: 33 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 18 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_1 + "\n" +
                            "  5 partitions match\n" +
@@ -356,11 +361,11 @@ private static void testMinCells(String option)
 
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "2")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 65.610 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_2 + "\n" +
                            "  5 partitions match\n" +
@@ -386,22 +391,22 @@ private static void testMinRows(String option)
     {
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 65.610 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_2 + "\n" +
                            "  5 partitions match\n" +
                            "  Keys: 0 1 2 3 4\n" +
                            SUMMARY_2 +
                            HEADER_1 +
-                           "  Partition: '0' (30) live, size: 33 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 18 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_1 + "\n" +
                            "  5 partitions match\n" +
@@ -410,11 +415,11 @@ private static void testMinRows(String option)
 
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "50")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 65.610 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_2 + "\n" +
                            "  5 partitions match\n" +
@@ -440,22 +445,22 @@ private static void testMinTombstones(String option)
     {
         assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0")
                 .isEqualTo(HEADER_2 +
-                           "  Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 65.610 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 65.615 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_2 + "\n" +
                            "  5 partitions match\n" +
                            "  Keys: 0 1 2 3 4\n" +
                            SUMMARY_2 +
                            HEADER_1 +
-                           "  Partition: '0' (30) live, size: 33 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '1' (31) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 18 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_1 + "\n" +
                            "  5 partitions match\n" +
@@ -534,8 +539,8 @@ private static void testIncludedKeys(String option)
     {
         assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "1", option, "3")
                 .contains(HEADER_1 +
-                           "  Partition: '1' (31) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_1 + "\n" +
                            "  2 partitions match\n" +
@@ -544,9 +549,9 @@ private static void testIncludedKeys(String option)
 
         assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "0", option, "2", option, "4")
                 .contains(HEADER_1 +
-                          "  Partition: '0' (30) live, size: 33 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                          "  Partition: '2' (32) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                          "  Partition: '4' (34) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                          "  Partition: '0' (30) live, size: 18 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                          "  Partition: '2' (32) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                          "  Partition: '4' (34) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                           "Summary of  #1 (big-ma):\n" +
                           "  File: " + SSTABLE_1 + "\n" +
                           "  3 partitions match\n" +
@@ -555,9 +560,9 @@ private static void testIncludedKeys(String option)
 
         assertThatToolSucceds(SSTABLE_1, "-y","--min-size", "0", option, "0", option, "2", option, "4")
                 .contains(HEADER_1 +
-                          "  Partition: '0' (30) live, size: 33 B\n" +
-                          "  Partition: '2' (32) live, size: 35 B\n" +
-                          "  Partition: '4' (34) live, size: 35 B\n" +
+                          "  Partition: '0' (30) live, size: 18 B\n" +
+                          "  Partition: '2' (32) live, size: 20 B\n" +
+                          "  Partition: '4' (34) live, size: 20 B\n" +
                           "Summary of  #1 (big-ma):\n" +
                           "  File: " + SSTABLE_1 + "\n" +
                           "  3 partitions match\n" +
@@ -579,9 +584,9 @@ private static void testExcludedKeys(String option)
     {
         assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "1", option, "3")
                 .contains(HEADER_1 +
-                           "  Partition: '0' (30) live, size: 33 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '2' (32) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '4' (34) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '0' (30) live, size: 18 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '2' (32) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '4' (34) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_1 + "\n" +
                            "  3 partitions match\n" +
@@ -590,8 +595,8 @@ private static void testExcludedKeys(String option)
 
         assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "0", option, "2", option, "4")
                 .contains(HEADER_1 +
-                           "  Partition: '1' (31) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
-                           "  Partition: '3' (33) live, size: 35 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '1' (31) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+                           "  Partition: '3' (33) live, size: 20 B, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
                            "Summary of  #1 (big-ma):\n" +
                            "  File: " + SSTABLE_1 + "\n" +
                            "  2 partitions match\n" +
@@ -616,16 +621,17 @@ private static void testCSV(String option)
                                   "rowTombstoneCount,rangeTombstoneCount,complexTombstoneCount,cellTombstoneCount," +
                                   "rowTtlExpired,cellTtlExpired,directory,keyspace,table,index,snapshot,backup," +
                                   "generation,format,version\n" +
-                                  "\"0\",30,true,0,67200,50,50,0,0,0,0,0,0,0,%s,,,,,,1,big,ma\n" +
-                                  "\"1\",31,true,67200,67205,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
-                                  "\"2\",32,true,134405,67205,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
-                                  "\"3\",33,true,201610,67205,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
-                                  "\"4\",34,true,268815,67205,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
-                                  "\"1\",31,true,33,35,1,1,0,0,0,0,0,0,0,%s,,,,,,1,big,ma\n" +
-                                  "\"2\",32,true,68,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
-                                  "\"3\",33,true,103,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
-                                  "\"4\",34,true,138,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n",
-                                  SSTABLE_2, SSTABLE_1));
+                                  "\"0\",30,true,15,67185,50,50,0,0,0,0,0,0,0,%s,,,,,,1,big,ma\n" +
+                                  "\"1\",31,true,67215,67190,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+                                  "\"2\",32,true,134420,67190,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+                                  "\"3\",33,true,201625,67190,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+                                  "\"4\",34,true,268830,67190,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n",// +
+//                                  "\"1\",31,true,33,35,1,1,0,0,0,0,0,0,0,%s,,,,,,1,big,ma\n" +
+//                                  "\"2\",32,true,68,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+//                                  "\"3\",33,true,103,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+//                                  "\"4\",34,true,138,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n",
+                                  SSTABLE_2//, SSTABLE_1
+                ));
     }
 
     private static AbstractStringAssert<?> assertThatToolSucceds(String... args)
@@ -650,6 +656,8 @@ private static String sstable(String table)
 
     private static ToolResult invokeTool(String... args)
     {
-        return ToolRunner.invokeClass(SSTablePartitions.class, args);
+        ToolResult result = ToolRunner.invokeClass(SSTablePartitions.class, args);
+        logger.info("Tool result: stdout = \n{}\nstderr = \n{}", result.getStdout(), result.getStderr());
+        return result;
     }
 }
diff --git a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java
index 13705f9b34b9..520623f04865 100644
--- a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java
+++ b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.tools;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.regex.Pattern;
@@ -27,6 +28,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
@@ -51,7 +53,10 @@ public class StandaloneUpgraderOnSStablesTest
 {
     static WithProperties properties;
 
-    String legacyId = LegacySSTableTest.legacyVersions[LegacySSTableTest.legacyVersions.length - 1];
+    List<String> legacyVersions = Arrays.stream(LegacySSTableTest.legacyVersions)
+                      .filter(v -> v.compareTo(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion().version) < 0)
+                      .collect(Collectors.toList());
+    String legacyId = legacyVersions.get(legacyVersions.size() - 1);
 
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
diff --git a/test/unit/org/apache/cassandra/tools/ToolRunner.java b/test/unit/org/apache/cassandra/tools/ToolRunner.java
index ad1f80e8127d..ad3212de18d5 100644
--- a/test/unit/org/apache/cassandra/tools/ToolRunner.java
+++ b/test/unit/org/apache/cassandra/tools/ToolRunner.java
@@ -57,7 +57,8 @@ public class ToolRunner
     protected static final Logger logger = LoggerFactory.getLogger(ToolRunner.class);
 
     public static final ImmutableList<String> DEFAULT_CLEANERS = ImmutableList.of("(?im)^picked up.*\\R",
-                                                                                  "(?im)^.*`USE <keyspace>` with prepared statements is.*\\R");
+                                                                                   "(?im)^.*`USE <keyspace>` with prepared statements is.*\\R");
+    private static final ImmutableList<String> STDOUT_CLEANERS = ImmutableList.of("^DEBUG .*\\R");
 
     public static int runClassAsTool(String clazz, String... args)
     {
@@ -240,6 +241,9 @@ public static ObservableTool invokeAsync(Map<String, String> env, InputStream st
         ProcessBuilder pb = new ProcessBuilder(args);
         if (env != null && !env.isEmpty())
             pb.environment().putAll(env);
+        String jvmOpts = pb.environment().getOrDefault("JVM_OPTS", "") + " -Dcassandra.disable_tcactive_openssl=true";
+        pb.environment().put("JVM_OPTS", jvmOpts);
+
         try
         {
             return new ForkedObservableTool(pb.start(), stdin, args);
@@ -466,10 +470,14 @@ public String argsToLogString()
          */
         public String getCleanedStderr(List<String> regExpCleaners)
         {
-            String sanitizedStderr = getStderr();
+            return applyCleaners(getStderr(), regExpCleaners);
+        }
+
+        private static String applyCleaners(String input, List<String> regExpCleaners)
+        {
             for (String regExp : regExpCleaners)
-                sanitizedStderr = sanitizedStderr.replaceAll(regExp, "");
-            return sanitizedStderr;
+                input = input.replaceAll(regExp, "");
+            return input;
         }
 
         /**
@@ -482,6 +490,11 @@ public String getCleanedStderr()
             return getCleanedStderr(DEFAULT_CLEANERS);
         }
 
+        public String getCleanedStdout()
+        {
+            return applyCleaners(getStdout(), STDOUT_CLEANERS);
+        }
+        
         public void assertOnCleanExit()
         {
             assertOnCleanExit(DEFAULT_CLEANERS);
diff --git a/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java b/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java
index 52195b98706b..1d9bf1cc24d7 100644
--- a/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java
+++ b/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java
@@ -22,6 +22,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Executors;
@@ -31,16 +32,12 @@
 import javax.management.openmbean.CompositeData;
 
 import com.google.common.collect.Lists;
-
-import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.metrics.Sampler;
-import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.Util;
 
@@ -52,29 +49,14 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
-/**
- * Includes test cases for both the 'toppartitions' command and its successor 'profileload'
- */
-public class TopPartitionsTest
+public class TopPartitionsTest extends CQLTester
 {
-    public static String KEYSPACE = TopPartitionsTest.class.getSimpleName().toLowerCase();
-    public static String TABLE = "test";
-
-    @BeforeClass
-    public static void loadSchema() throws ConfigurationException
-    {
-        SchemaLoader.prepareServer();
-        SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1));
-        executeInternal(format("CREATE TABLE %s.%s (k text, c text, v text, PRIMARY KEY (k, c))", KEYSPACE, TABLE));
-    }
-
     @Test
     public void testServiceTopPartitionsNoArg() throws Exception
     {
         BlockingQueue<Map<String, List<CompositeData>>> q = new ArrayBlockingQueue<>(1);
         ColumnFamilyStore.all();
-        Executors.newCachedThreadPool().execute(() ->
-        {
+        Executors.newCachedThreadPool().execute(() -> {
             try
             {
                 q.put(StorageService.instance.samplePartitions(null, 1000, 100, 10, Lists.newArrayList("READS", "WRITES")));
@@ -87,8 +69,8 @@ public void testServiceTopPartitionsNoArg() throws Exception
         Thread.sleep(100);
         SystemKeyspace.persistLocalMetadata();
         Map<String, List<CompositeData>> result = q.poll(5, TimeUnit.SECONDS);
-        List<CompositeData> cd = result.get("WRITES");
-        assertEquals(1, cd.size());
+        List<CompositeData> cds = result.get("WRITES").stream().filter(cd -> Objects.equals(cd.get("table"), "system." + SystemKeyspace.LOCAL)).collect(Collectors.toList());
+        assertEquals(1, cds.size());
     }
 
     @Test
@@ -110,20 +92,21 @@ public void testServiceTopPartitionsSingleTable() throws Exception
     @Test
     public void testTopPartitionsRowTombstoneAndSSTableCount() throws Exception
     {
+        createTable("CREATE TABLE %s (k text, c text, v text, PRIMARY KEY (k, c))");
         int count = 10;
-        ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+        ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, currentTable());
         cfs.disableAutoCompaction();
 
-        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'a', 'a')", KEYSPACE, TABLE));
-        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'b', 'a')", KEYSPACE, TABLE));
+        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'a', 'a')", KEYSPACE, currentTable()));
+        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'b', 'a')", KEYSPACE, currentTable()));
         cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
 
-        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'c', 'a')", KEYSPACE, TABLE));
-        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('b', 'b', 'b')", KEYSPACE, TABLE));
-        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'c', 'c')", KEYSPACE, TABLE));
-        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'd', 'a')", KEYSPACE, TABLE));
-        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'e', 'a')", KEYSPACE, TABLE));
-        executeInternal(format("DELETE FROM %s.%s WHERE k='a' AND c='a'", KEYSPACE, TABLE));
+        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'c', 'a')", KEYSPACE, currentTable()));
+        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('b', 'b', 'b')", KEYSPACE, currentTable()));
+        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'c', 'c')", KEYSPACE, currentTable()));
+        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'd', 'a')", KEYSPACE, currentTable()));
+        executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'e', 'a')", KEYSPACE, currentTable()));
+        executeInternal(format("DELETE FROM %s.%s WHERE k='a' AND c='a'", KEYSPACE, currentTable()));
         cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
 
         long executedBefore = Sampler.samplerExecutor.getCompletedTaskCount();
@@ -132,7 +115,7 @@ public void testTopPartitionsRowTombstoneAndSSTableCount() throws Exception
         cfs.beginLocalSampling("READ_TOMBSTONE_COUNT", count, 240000);
         cfs.beginLocalSampling("READ_SSTABLE_COUNT", count, 240000);
 
-        executeInternal(format("SELECT * FROM %s.%s", KEYSPACE, TABLE));
+        executeInternal(format("SELECT * FROM %s.%s", KEYSPACE, currentTable()));
         ensureThatSamplerExecutorProcessedAllSamples(executedBefore);
 
         List<CompositeData> rowCounts = cfs.finishLocalSampling("READ_ROW_COUNT", count);
@@ -166,9 +149,9 @@ else if (partitionKey.equalsIgnoreCase("c"))
         cfs.beginLocalSampling("READ_TOMBSTONE_COUNT", count, 240000);
         cfs.beginLocalSampling("READ_SSTABLE_COUNT", count, 240000);
 
-        executeInternal(format("SELECT * FROM %s.%s WHERE k='a'", KEYSPACE, TABLE));
-        executeInternal(format("SELECT * FROM %s.%s WHERE k='b'", KEYSPACE, TABLE));
-        executeInternal(format("SELECT * FROM %s.%s WHERE k='c'", KEYSPACE, TABLE));
+        executeInternal(format("SELECT * FROM %s.%s WHERE k='a'", KEYSPACE, currentTable()));
+        executeInternal(format("SELECT * FROM %s.%s WHERE k='b'", KEYSPACE, currentTable()));
+        executeInternal(format("SELECT * FROM %s.%s WHERE k='c'", KEYSPACE, currentTable()));
         ensureThatSamplerExecutorProcessedAllSamples(executedBefore);
 
         rowCounts = cfs.finishLocalSampling("READ_ROW_COUNT", count);
diff --git a/test/unit/org/apache/cassandra/tools/cqlsh/CqlshTest.java b/test/unit/org/apache/cassandra/tools/cqlsh/CqlshTest.java
index 356769b84064..d3561412c6ba 100644
--- a/test/unit/org/apache/cassandra/tools/cqlsh/CqlshTest.java
+++ b/test/unit/org/apache/cassandra/tools/cqlsh/CqlshTest.java
@@ -33,6 +33,7 @@
 import org.apache.cassandra.tools.ToolRunner.ToolResult;
 
 import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VECTOR_FLOAT_ONLY;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -42,6 +43,7 @@ public class CqlshTest extends CQLTester
     @BeforeClass
     public static void setUp()
     {
+        VECTOR_FLOAT_ONLY.setBoolean(false);
         requireNetwork();
     }
 
@@ -164,4 +166,10 @@ private static Path createTempFile(String prefix) throws IOException
         csv.toFile().deleteOnExit();
         return csv;
     }
+
+    @SafeVarargs
+    protected final <T> Vector<T> vector(T... values)
+    {
+        return new Vector<>(values);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java
index 379d02a66e9f..2f292ba7610f 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java
@@ -255,7 +255,7 @@ public void testIncompatibleFlags()
     }
 
     private void rewriteManifest(String tableId,
-                                 String[] dataDirs,
+                                 File[] dataDirs,
                                  String keyspace,
                                  String tableName,
                                  String snapshotName,
@@ -267,11 +267,11 @@ private void rewriteManifest(String tableId,
         manifestWithEphemeralFlag.serializeToJsonFile(new File(manifestPath));
     }
 
-    private Path findManifest(String[] dataDirs, String keyspace, String tableId, String tableName, String snapshotName)
+    private Path findManifest(File[] dataDirs, String keyspace, String tableId, String tableName, String snapshotName)
     {
-        for (String dataDir : dataDirs)
+        for (File dataDir : dataDirs)
         {
-            Path manifest = Paths.get(dataDir)
+            Path manifest = Paths.get(dataDir.toString())
                                  .resolve(keyspace)
                                  .resolve(format("%s-%s", tableName, tableId))
                                  .resolve("snapshots")
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java
index 8758c3f8fd86..536b077d788d 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java
@@ -28,12 +28,13 @@
 
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.AbstractTableOperation;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.MockSchema;
 import org.apache.cassandra.tools.ToolRunner;
+import org.apache.cassandra.utils.NonThrowingCloseable;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -65,15 +66,27 @@ public void testMaybeChangeDocs()
                       "                [(-pp | --print-port)] [(-pw <password> | --password <password>)]\n" +
                       "                [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]\n" +
                       "                [(-u <username> | --username <username>)] compactionstats\n" +
-                      "                [(-H | --human-readable)] [(-V | --vtable)]\n" +
+                      "                [(-A | --aggregate)] [(-H | --human-readable)] [(-O | --overlap)]\n" +
+                      "                [(-V | --vtable)] [--] [<keyspace> <tables>...]\n" +
                       "\n" +
                       "OPTIONS\n" +
+                      "        -A, --aggregate\n" +
+                      "            Show the compaction aggregates for the compactions in progress, e.g.\n" +
+                      "            the levels for LCS or the buckets for STCS and TWCS.\n" +
+                      "\n" +
                       "        -h <host>, --host <host>\n" +
                       "            Node hostname or ip address\n" +
                       "\n" +
                       "        -H, --human-readable\n" +
                       "            Display bytes in human readable form, i.e. KiB, MiB, GiB, TiB\n" +
                       "\n" +
+                      "        -O, --overlap\n" +
+                      "            Show a map of the maximum sstable overlap per compaction region.\n" +
+                      "            Note: This map includes all sstables in the system, including ones\n" +
+                      "            that are currently being compacted, and also takes into account\n" +
+                      "            early opened sstables. Overlaps per level may be greater than the\n" +
+                      "            values the --aggregate option reports.\n" +
+                      "\n" +
                       "        -p <port>, --port <port>\n" +
                       "            Remote jmx agent port number\n" +
                       "\n" +
@@ -92,6 +105,15 @@ public void testMaybeChangeDocs()
                       "        -V, --vtable\n" +
                       "            Display fields matching vtable output\n" +
                       "\n" +
+                      "        --\n" +
+                      "            This option can be used to separate command-line options from the\n" +
+                      "            list of argument, (useful when arguments might be mistaken for\n" +
+                      "            command-line options\n" +
+                      "\n" +
+                      "        [<keyspace> <tables>...]\n" +
+                      "            With --aggregate or --overlap, optionally list only the data for the\n" +
+                      "            specified keyspace and tables.\n" +
+                      "\n" +
                       "\n";
         assertThat(tool.getStdout()).isEqualTo(help);
     }
@@ -108,11 +130,11 @@ public void testCompactionStats()
         List<SSTableReader> sstables = IntStream.range(0, 10)
                                                 .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
                                                 .collect(Collectors.toList());
-        CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
+        AbstractTableOperation compactionHolder = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
+                return new OperationProgress(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
             }
 
             public boolean isGlobal()
@@ -121,27 +143,27 @@ public boolean isGlobal()
             }
         };
 
-        CompactionManager.instance.active.beginCompaction(compactionHolder);
-        String stdout = waitForNumberOfPendingTasks(1, "compactionstats");
-        assertThat(stdout).containsPattern("id\\s+compaction type\\s+keyspace\\s+table\\s+completed\\s+total\\s+unit\\s+progress");
-        String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%.2f%%",
-                                                    compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), bytesCompacted, bytesTotal,
-                                                    CompactionInfo.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100);
-        assertThat(stdout).containsPattern(expectedStatsPattern);
-
-        assertThat(stdout).containsPattern(expectedStatsPattern);
-        assertThat(stdout).containsPattern("concurrent compactors\\s+[0-9]*");
-        assertThat(stdout).containsPattern("pending tasks\\s+[0-9]*");
-        assertThat(stdout).containsPattern("compactions completed\\s+[0-9]*");
-        assertThat(stdout).containsPattern("data compacted\\s+[0-9]*");
-        assertThat(stdout).containsPattern("compactions aborted\\s+[0-9]*");
-        assertThat(stdout).containsPattern("compactions reduced\\s+[0-9]*");
-        assertThat(stdout).containsPattern("sstables dropped from compaction\\s+[0-9]*");
-        assertThat(stdout).containsPattern("15 minute rate\\s+[0-9]*.[0-9]*[0-9]*/minute");
-        assertThat(stdout).containsPattern("mean rate\\s+[0-9]*.[0-9]*[0-9]*/hour");
-        assertThat(stdout).containsPattern("compaction throughput \\(MiB/s\\)\\s+throttling disabled \\(0\\)");
-
-        CompactionManager.instance.active.finishCompaction(compactionHolder);
+        try (NonThrowingCloseable c = CompactionManager.instance.active.onOperationStart(compactionHolder))
+        {
+            String stdout = waitForNumberOfPendingTasks(1, "compactionstats");
+            assertThat(stdout).containsPattern("id\\s+compaction type\\s+keyspace\\s+table\\s+completed\\s+total\\s+unit\\s+progress");
+            String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%.2f%%",
+                                                        compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), bytesCompacted, bytesTotal,
+                                                        AbstractTableOperation.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100);
+            assertThat(stdout).containsPattern(expectedStatsPattern);
+
+            assertThat(stdout).containsPattern(expectedStatsPattern);
+            assertThat(stdout).containsPattern("concurrent compactors\\s+[0-9]*");
+            assertThat(stdout).containsPattern("pending tasks\\s+[0-9]*");
+            assertThat(stdout).containsPattern("compactions completed\\s+[0-9]*");
+            assertThat(stdout).containsPattern("data compacted\\s+[0-9]*");
+            assertThat(stdout).containsPattern("compactions aborted\\s+[0-9]*");
+            assertThat(stdout).containsPattern("compactions reduced\\s+[0-9]*");
+            assertThat(stdout).containsPattern("sstables dropped from compaction\\s+[0-9]*");
+            assertThat(stdout).containsPattern("15 minute rate\\s+[0-9]*.[0-9]*[0-9]*/minute");
+            assertThat(stdout).containsPattern("mean rate\\s+[0-9]*.[0-9]*[0-9]*/hour");
+            assertThat(stdout).containsPattern("compaction throughput \\(MiB/s\\)\\s+throttling disabled \\(0\\)");
+        }
         waitForNumberOfPendingTasks(0, "compactionstats");
     }
 
@@ -158,11 +180,11 @@ public void testCompactionStatsVtable()
                                                 .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
                                                 .collect(Collectors.toList());
         String targetDirectory = "/some/dir/" + cfs.metadata.keyspace + '/' + cfs.metadata.name + '-' + cfs.metadata.id.asUUID();
-        CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
+        AbstractTableOperation compactionHolder = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
+                return new OperationProgress(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
             }
 
             public boolean isGlobal()
@@ -171,11 +193,11 @@ public boolean isGlobal()
             }
         };
 
-        CompactionInfo.Holder nonCompactionHolder = new CompactionInfo.Holder()
+        AbstractTableOperation nonCompactionHolder = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
+                return new OperationProgress(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
             }
 
             public boolean isGlobal()
@@ -184,23 +206,22 @@ public boolean isGlobal()
             }
         };
 
-        CompactionManager.instance.active.beginCompaction(compactionHolder);
-        CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
-        String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "-V");
-        assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
-        String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
-                                                    CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
-                                                    OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES,
-                                                    targetDirectory);
-        assertThat(stdout).containsPattern(expectedStatsPattern);
-
-        String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
-                                                                    CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
-                                                                    OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES);
-        assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
-
-        CompactionManager.instance.active.finishCompaction(compactionHolder);
-        CompactionManager.instance.active.finishCompaction(nonCompactionHolder);
+        try (NonThrowingCloseable c = CompactionManager.instance.active.onOperationStart(compactionHolder);
+             NonThrowingCloseable c2 = CompactionManager.instance.active.onOperationStart(nonCompactionHolder))
+        {
+            String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "-V");
+            assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
+            String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
+                                                        CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
+                                                        OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, AbstractTableOperation.Unit.BYTES,
+                                                        targetDirectory);
+            assertThat(stdout).containsPattern(expectedStatsPattern);
+
+            String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
+                                                                        CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
+                                                                        OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, AbstractTableOperation.Unit.BYTES);
+            assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
+        }
         waitForNumberOfPendingTasks(0, "compactionstats", "-V");
     }
 
@@ -216,11 +237,11 @@ public void testCompactionStatsHumanReadable()
         List<SSTableReader> sstables = IntStream.range(0, 10)
                                                 .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
                                                 .collect(Collectors.toList());
-        CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
+        AbstractTableOperation compactionHolder = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
+                return new OperationProgress(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
             }
 
             public boolean isGlobal()
@@ -229,15 +250,15 @@ public boolean isGlobal()
             }
         };
 
-        CompactionManager.instance.active.beginCompaction(compactionHolder);
-        String stdout = waitForNumberOfPendingTasks(1, "compactionstats", "--human-readable");
-        assertThat(stdout).containsPattern("id\\s+compaction type\\s+keyspace\\s+table\\s+completed\\s+total\\s+unit\\s+progress");
-        String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%.2f%%",
-                                                    compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), "123 bytes", "120.56 KiB",
-                                                    CompactionInfo.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100);
-        assertThat(stdout).containsPattern(expectedStatsPattern);
-
-        CompactionManager.instance.active.finishCompaction(compactionHolder);
+        try (NonThrowingCloseable ignored = CompactionManager.instance.active.onOperationStart(compactionHolder))
+        {
+            String stdout = waitForNumberOfPendingTasks(1, "compactionstats", "--human-readable");
+            assertThat(stdout).containsPattern("id\\s+compaction type\\s+keyspace\\s+table\\s+completed\\s+total\\s+unit\\s+progress");
+            String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%.2f%%",
+                                                        compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), "123 bytes", "120.56 KiB",
+                                                        AbstractTableOperation.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100);
+            assertThat(stdout).containsPattern(expectedStatsPattern);
+        }
         waitForNumberOfPendingTasks(0, "compactionstats", "--human-readable");
     }
 
@@ -254,11 +275,11 @@ public void testCompactionStatsVtableHumanReadable()
                                                 .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
                                                 .collect(Collectors.toList());
         String targetDirectory = "/some/dir/" + cfs.metadata.keyspace + '/' + cfs.metadata.name + '-' + cfs.metadata.id.asUUID();
-        CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
+        AbstractTableOperation compactionHolder = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
+                return new OperationProgress(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
             }
 
             public boolean isGlobal()
@@ -267,11 +288,11 @@ public boolean isGlobal()
             }
         };
 
-        CompactionInfo.Holder nonCompactionHolder = new CompactionInfo.Holder()
+        AbstractTableOperation nonCompactionHolder = new AbstractTableOperation()
         {
-            public CompactionInfo getCompactionInfo()
+            public AbstractTableOperation.OperationProgress getProgress()
             {
-                return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
+                return new AbstractTableOperation.OperationProgress(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
             }
 
             public boolean isGlobal()
@@ -280,22 +301,21 @@ public boolean isGlobal()
             }
         };
 
-        CompactionManager.instance.active.beginCompaction(compactionHolder);
-        CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
-        String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "--vtable", "--human-readable");
-        assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
-        String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
-                                                    CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
-                                                    OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES,
-                                                    targetDirectory);
-        assertThat(stdout).containsPattern(expectedStatsPattern);
-        String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
-                                                                    CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
-                                                                    OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES);
-        assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
-
-        CompactionManager.instance.active.finishCompaction(compactionHolder);
-        CompactionManager.instance.active.finishCompaction(nonCompactionHolder);
+        try (NonThrowingCloseable c = CompactionManager.instance.active.onOperationStart(compactionHolder);
+             NonThrowingCloseable c2 = CompactionManager.instance.active.onOperationStart(nonCompactionHolder))
+        {
+            String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "--vtable", "--human-readable");
+            assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
+            String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
+                                                        CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
+                                                        OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", AbstractTableOperation.Unit.BYTES,
+                                                        targetDirectory);
+            assertThat(stdout).containsPattern(expectedStatsPattern);
+            String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
+                                                                        CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
+                                                                        OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", AbstractTableOperation.Unit.BYTES);
+            assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
+        }
         waitForNumberOfPendingTasks(0, "compactionstats", "--vtable", "--human-readable");
     }
 
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java b/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java
index 28fffd9619ea..c5224ac4c65b 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java
@@ -20,6 +20,9 @@
 
 import java.util.Arrays;
 
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -50,6 +53,7 @@ public static void setup() throws Exception
     @Test
     public void testRingOutput()
     {
+        SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES.forEach(k -> Keyspace.open(k).flush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
         final HostStatWithPort host = new HostStatWithPort(null, FBUtilities.getBroadcastAddressAndPort(),
                                                            false, null);
         validateRingOutput(host.ipOrDns(false), "ring");
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java
index ddc59b81a172..e51adc950218 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java
@@ -41,6 +41,7 @@
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.tools.StandaloneScrubber;
 import org.apache.cassandra.tools.ToolRunner;
+import org.apache.cassandra.tools.ToolRunner.ToolResult;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Throwables;
 import org.assertj.core.api.Assertions;
@@ -76,6 +77,7 @@ public class ScrubToolTest
 
     String ksName;
     Keyspace keyspace;
+
     @BeforeClass
     public static void defineSchema() throws ConfigurationException
     {
@@ -110,7 +112,7 @@ public void testScrubOnePartitionWithTool()
         fillCF(cfs, 1);
         assertOrderedAll(cfs, 1);
 
-        ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, ksName, CF);
+        ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, ksName, CF);
         Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
         Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
         tool.assertOnCleanExit();
@@ -133,7 +135,7 @@ public void testSkipScrubCorruptedCounterPartitionWithTool() throws IOException,
         ScrubTest.overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"), (byte) 0x7A);
 
         // with skipCorrupted == true, the corrupt rows will be skipped
-        ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-s", ksName, COUNTER_CF);
+        ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-s", ksName, COUNTER_CF);
         Assertions.assertThat(tool.getStdout()).contains("0 empty");
         Assertions.assertThat(tool.getStdout()).contains("partitions that were skipped");
         tool.assertOnCleanExit();
@@ -177,7 +179,7 @@ public void testNoCheckScrubMultiPartitionWithTool()
         fillCF(cfs, 10);
         assertOrderedAll(cfs, 10);
 
-        ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-n", ksName, CF);
+        ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-n", ksName, CF);
         Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
         Assertions.assertThat(tool.getStdout()).contains("10 partitions in new sstable and 0 empty");
         tool.assertOnCleanExit();
@@ -194,7 +196,7 @@ public void testHeaderFixValidateWithTool()
         fillCF(cfs, 1);
         assertOrderedAll(cfs, 1);
 
-        ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "validate", ksName, CF);
+        ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "validate", ksName, CF);
         Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
         Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
         // TODO cleaner that ignores
@@ -210,7 +212,7 @@ public void testHeaderFixWithTool()
         fillCF(cfs, 1);
         assertOrderedAll(cfs, 1);
 
-        ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "fix", ksName, CF);
+        ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "fix", ksName, CF);
         Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
         Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
         tool.assertOnCleanExit(CLEANERS);
@@ -225,7 +227,7 @@ public void testHeaderFixNoChecksWithTool()
         fillCF(cfs, 1);
         assertOrderedAll(cfs, 1);
 
-        ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "off", ksName, CF);
+        ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "off", ksName, CF);
         Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
         Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
         tool.assertOnCleanExit(CLEANERS);
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetGetCompactionThroughputTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetGetCompactionThroughputTest.java
index 24ee9e579717..2e943c111509 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SetGetCompactionThroughputTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SetGetCompactionThroughputTest.java
@@ -81,6 +81,17 @@ public void testUnparseable()
         assertPreciseMibFlagNeeded();
     }
 
+    @Test
+    public void testCurrentCompactionThroughput()
+    {
+        ToolResult tool = invokeNodetool("getcompactionthroughput");
+        tool.assertOnCleanExit();
+
+        assertThat(tool.getStdout()).containsPattern("Current compaction throughput \\(1 minute\\): \\d+\\.\\d+ MiB/s");
+        assertThat(tool.getStdout()).containsPattern("Current compaction throughput \\(5 minute\\): \\d+\\.\\d+ MiB/s");
+        assertThat(tool.getStdout()).containsPattern("Current compaction throughput \\(15 minute\\): \\d+\\.\\d+ MiB/s");
+    }
+
     private static void assertSetGetValidThroughput(int throughput)
     {
         ToolResult tool = invokeNodetool("setcompactionthroughput", String.valueOf(throughput));
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/StatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/StatusTest.java
index 9d8496c8b102..d6cf702a9952 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/StatusTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/StatusTest.java
@@ -24,6 +24,8 @@
 import org.junit.Test;
 
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.locator.SimpleSnitch;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.StorageService;
@@ -53,6 +55,8 @@ public static void setup() throws Exception
     @Test
     public void testStatusOutput()
     {
+        SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES.forEach(k -> Keyspace.open(k).flush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
+
         HostStatWithPort host = new HostStatWithPort(null, FBUtilities.getBroadcastAddressAndPort(), false, null);
         validateStatusOutput(host.ipOrDns(false),
                             "status");
@@ -71,6 +75,8 @@ public void testStatusOutput()
     @Test
     public void testOutputWhileBootstrapping()
     {
+        SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES.forEach(k -> Keyspace.open(k).flush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
+
         // Deleting these tables will simulate we're bootstrapping
         schemaChange("DROP KEYSPACE " + SchemaConstants.TRACE_KEYSPACE_NAME);
         schemaChange("DROP KEYSPACE " + CQLTester.KEYSPACE);
@@ -111,6 +117,7 @@ private void validateStatusOutput(String hostForm, String... args)
         assertThat(hostStatus).startsWith("UN");
         assertThat(hostStatus).contains(hostForm);
         assertThat(hostStatus).containsPattern("\\d+\\.?\\d+ KiB");
+        assertThat(hostStatus).matches(".*\\d+(\\.\\d+)? (bytes|KiB).*");
         assertThat(hostStatus).containsPattern("\\d+\\.\\d+%");
         assertThat(hostStatus).contains(localHostId);
         assertThat(hostStatus).contains(token);
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/TpStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/TpStatsTest.java
index e9548fbdbf06..61b71167fa0b 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/TpStatsTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/TpStatsTest.java
@@ -155,7 +155,7 @@ public void testFormatArg()
             tool.assertOnCleanExit();
             String yaml = tool.getStdout();
             assertThat(isYAMLString(yaml)).isTrue();
-            assertThat(yaml).containsPattern("WaitLatencies:\\s*[A-Z|_]+:\\s+-\\s");
+            assertThat(yaml).containsPattern("WaitLatencies:\\s*[A-Z0-9|_]+:\\s+-\\s");
         });
     }
 
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/VerifyTest.java b/test/unit/org/apache/cassandra/tools/nodetool/VerifyTest.java
index 1dc7a30e262a..6854321e0cb2 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/VerifyTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/VerifyTest.java
@@ -72,9 +72,8 @@ public void testMaybeChangeDocs()
         "                [(-u <username> | --username <username>)] verify\n" +
         "                [(-c | --check-version)] [(-d | --dfp)] [(-e | --extended-verify)]\n" +
         "                [(-f | --force)] [(-q | --quick)] [(-r | --rsc)] [(-t | --check-tokens)]\n" +
-        "                [--] [<keyspace> <tables>...]\n" +
+        "                [(-v | --validate-all-rows)] [--] [<keyspace> <tables>...]\n" +
         "\n" +
-
         "OPTIONS\n" +
         "        -c, --check-version\n" +
         "            Also check that all sstables are the latest version\n" +
@@ -83,7 +82,7 @@ public void testMaybeChangeDocs()
         "            Invoke the disk failure policy if a corrupt sstable is found\n" +
         "\n" +
         "        -e, --extended-verify\n" +
-        "            Verify each cell data, beyond simply checking sstable checksums\n" +
+        "            Verify each partition data, beyond simply checking sstable checksums\n" +
         "\n" +
         "        -f, --force\n" +
         "            Override disabling of verify tool - see CASSANDRA-9947 for caveats\n" +
@@ -115,6 +114,10 @@ public void testMaybeChangeDocs()
         "        -u <username>, --username <username>\n" +
         "            Remote jmx agent username\n" +
         "\n" +
+        "        -v, --validate-all-rows\n" +
+        "            Verify each row and cell data in the partition, beyond checking\n" +
+        "            partition key. Must be enabled with extended verification\n" +
+        "\n" +
         "        --\n" +
         "            This option can be used to separate command-line options from the\n" +
         "            list of argument, (useful when arguments might be mistaken for\n" +
diff --git a/test/unit/org/apache/cassandra/tracing/TracingTest.java b/test/unit/org/apache/cassandra/tracing/TracingTest.java
index e344693c6e30..64fb5949d066 100644
--- a/test/unit/org/apache/cassandra/tracing/TracingTest.java
+++ b/test/unit/org/apache/cassandra/tracing/TracingTest.java
@@ -18,24 +18,47 @@
 
 package org.apache.cassandra.tracing;
 
+import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 
+import com.google.common.collect.ImmutableList;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.Attributes;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.UseStatement;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.ParamType;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.TracingClientState;
 import org.apache.cassandra.utils.progress.ProgressEvent;
 
-public final class TracingTest
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static java.lang.String.format;
+
+public final class TracingTest extends CQLTester
 {
     @BeforeClass
     public static void setupDD()
     {
+        prepareServer();
+        requireNetwork();
         DatabaseDescriptor.daemonInitialization();
     }
 
@@ -44,7 +67,7 @@ public void test()
     {
         List<String> traces = new ArrayList<>();
         Tracing tracing = new TracingTestImpl(traces);
-        tracing.newSession(Tracing.TraceType.NONE);
+        tracing.newSession(ClientState.forInternalCalls(), Tracing.TraceType.NONE);
         TraceState state = tracing.begin("test-request", Collections.<String,String>emptyMap());
         state.trace("test-1");
         state.trace("test-2");
@@ -64,7 +87,7 @@ public void test_get()
     {
         List<String> traces = new ArrayList<>();
         Tracing tracing = new TracingTestImpl(traces);
-        tracing.newSession(Tracing.TraceType.NONE);
+        tracing.newSession(ClientState.forInternalCalls(), Tracing.TraceType.NONE);
         tracing.begin("test-request", Collections.<String,String>emptyMap());
         tracing.get().trace("test-1");
         tracing.get().trace("test-2");
@@ -84,7 +107,7 @@ public void test_get_uuid()
     {
         List<String> traces = new ArrayList<>();
         Tracing tracing = new TracingTestImpl(traces);
-        TimeUUID uuid = tracing.newSession(Tracing.TraceType.NONE);
+        TimeUUID uuid = tracing.newSession(ClientState.forInternalCalls(), Tracing.TraceType.NONE);
         tracing.begin("test-request", Collections.<String,String>emptyMap());
         tracing.get(uuid).trace("test-1");
         tracing.get(uuid).trace("test-2");
@@ -108,7 +131,7 @@ public void test_customPayload()
         Map<String,ByteBuffer> customPayload = Collections.singletonMap("test-key", customPayloadValue);
 
         TracingTestImpl tracing = new TracingTestImpl(traces);
-        tracing.newSession(customPayload);
+        tracing.newSession(ClientState.forInternalCalls(), customPayload);
         TraceState state = tracing.begin("test-custom_payload", Collections.<String,String>emptyMap());
         state.trace("test-1");
         state.trace("test-2");
@@ -130,7 +153,7 @@ public void test_states()
     {
         List<String> traces = new ArrayList<>();
         Tracing tracing = new TracingTestImpl(traces);
-        tracing.newSession(Tracing.TraceType.REPAIR);
+        tracing.newSession(ClientState.forInternalCalls(), Tracing.TraceType.REPAIR);
         tracing.begin("test-request", Collections.<String,String>emptyMap());
         tracing.get().enableActivityNotification("test-tag");
         assert TraceState.Status.IDLE == tracing.get().waitActivity(1);
@@ -147,7 +170,7 @@ public void test_progress_listener()
     {
         List<String> traces = new ArrayList<>();
         Tracing tracing = new TracingTestImpl(traces);
-        tracing.newSession(Tracing.TraceType.REPAIR);
+        tracing.newSession(ClientState.forInternalCalls(), Tracing.TraceType.REPAIR);
         tracing.begin("test-request", Collections.<String,String>emptyMap());
         tracing.get().enableActivityNotification("test-tag");
 
@@ -160,4 +183,152 @@ public void test_progress_listener()
         tracing.stopSession();
         assert null == tracing.get();
     }
+
+    @Test
+    public void test_adding_keyspace_to_trace_state()
+    {
+        Tracing tracing = Tracing.instance;
+        String keyspace = "someKeyspace";
+        TimeUUID sessionId = tracing.newSession(TracingClientState.withTracedKeyspace(keyspace), Tracing.TraceType.QUERY);
+
+        assert keyspace.equals(((TracingClientState)tracing.get().clientState).tracedKeyspace());
+        assert keyspace.equals(((TracingClientState)tracing.get(sessionId).clientState).tracedKeyspace());
+        assert keyspace.equals(tracing.get().tracedKeyspace());
+
+        Map<ParamType, Object> headers = tracing.addTraceHeaders(new HashMap<>());
+        assert keyspace.equals(headers.get(ParamType.TRACE_KEYSPACE));
+        tracing.stopSession();
+    }
+
+    @Test
+    public void test_cloning_tracing_state()
+    {
+        String keyspace = "someKeyspace";
+        String otherKeyspace = "otherKeyspace";
+        TracingClientState state = TracingClientState.withTracedKeyspace(keyspace);
+
+        ClientState clientState = state.cloneWithKeyspaceIfSet(otherKeyspace);
+        assert clientState instanceof TracingClientState;
+        assert keyspace.equals(((TracingClientState)clientState).tracedKeyspace());
+    }
+
+    @Test
+    public void test_initializing_from_message_with_keyspace()
+    {
+        ClientStateAccumulatingTracing tracing = new ClientStateAccumulatingTracing();
+
+        Message<Object> message = Message.builder(Verb._TEST_1, new Object())
+                                         .withParam(ParamType.TRACE_KEYSPACE, "testKeyspace")
+                                         .withParam(ParamType.TRACE_SESSION, nextTimeUUID())
+                                         .build();
+
+        TraceState traceState = tracing.initializeFromMessage(message.header);
+
+        assert traceState.clientState instanceof TracingClientState;
+        assert "testKeyspace".equals(((TracingClientState) traceState.clientState).tracedKeyspace());
+    }
+
+    @Test
+    public void test_initializing_from_message_without_keyspace()
+    {
+        ClientStateAccumulatingTracing tracing = new ClientStateAccumulatingTracing();
+
+        Message<Object> message = Message.builder(Verb._TEST_1, new Object())
+                                         .withParam(ParamType.TRACE_SESSION, nextTimeUUID())
+                                         .build();
+
+        TraceState traceState = tracing.initializeFromMessage(message.header);
+
+        assert traceState.clientState instanceof TracingClientState;
+        assert ((TracingClientState) traceState.clientState).tracedKeyspace() == null;
+    }
+
+    @Test
+    public void test_tracing_outgoing_message_with_keyspace()
+    {
+        ClientStateAccumulatingTracing tracing = new ClientStateAccumulatingTracing();
+
+        Message<Object> message = Message.builder(Verb._TEST_1, new Object())
+                                         .withParam(ParamType.TRACE_KEYSPACE, "testKeyspace")
+                                         .withParam(ParamType.TRACE_SESSION, nextTimeUUID())
+                                         .build();
+
+        tracing.traceOutgoingMessage(message, 999, InetAddressAndPort.getLocalHost());
+
+        assert tracing.states.size() == 1;
+        assert tracing.states.peek() instanceof TracingClientState;
+        assert "testKeyspace".equals(((TracingClientState) tracing.states.peek()).tracedKeyspace());
+    }
+
+    @Test
+    public void test_tracing_outgoing_message_without_keyspace()
+    {
+        ClientStateAccumulatingTracing tracing = new ClientStateAccumulatingTracing();
+
+        Message<Object> message = Message.builder(Verb._TEST_1, new Object())
+                                         .withParam(ParamType.TRACE_SESSION, nextTimeUUID())
+                                         .build();
+
+        tracing.traceOutgoingMessage(message, 999, InetAddressAndPort.getLocalHost());
+
+        assert tracing.states.size() == 1;
+        assert tracing.states.peek() instanceof TracingClientState;
+        assert ((TracingClientState) tracing.states.peek()).tracedKeyspace() == null;
+    }
+
+    @Test
+    public void test_tracing_setting_traced_keyspace()
+    {
+        Tracing tracing = Tracing.instance;
+        ClientState cs = ClientState.forInternalCalls();
+        tracing.newSession(cs, Tracing.TraceType.QUERY);
+
+        String keyspace = "keyspace1";
+        UseStatement stmt = new UseStatement(keyspace);
+        Tracing.setupTracedKeyspace(stmt);
+        assert keyspace.equals(tracing.get().tracedKeyspace());
+
+        String batchKeyspace = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }");
+        String tbl = createTable(batchKeyspace, "CREATE TABLE %s (id int primary key, val int)");
+        List<String> queries = ImmutableList.of(
+            format("INSERT INTO %s.%s (id, val) VALUES (0, 0) USING TTL %d;", batchKeyspace, tbl, 1),
+            format("INSERT INTO %s.%s (id, val) VALUES (1, 1) USING TTL %d;", batchKeyspace, tbl, 1)
+        );
+
+        List<ModificationStatement> statements = new ArrayList<>(queries.size());
+        for (String query : queries)
+            statements.add((ModificationStatement) QueryProcessor.parseStatement(query, cs));
+
+        BatchStatement batchStmt = new BatchStatement(null, BatchStatement.Type.UNLOGGED, VariableSpecifications.empty(), statements, Attributes.none());
+        Tracing.setupTracedKeyspace(batchStmt);
+        assert batchKeyspace.equals(tracing.get().tracedKeyspace());
+        tracing.stopSession();
+    }
+
+    private static final class ClientStateAccumulatingTracing extends Tracing
+    {
+        Queue<ClientState> states = new LinkedList<>();
+
+        @Override
+        protected void stopSessionImpl()
+        {}
+
+        @Override
+        public TraceState begin(String request, InetAddress client, Map<String, String> parameters)
+        {
+            return null;
+        }
+
+        @Override
+        protected TraceState newTraceState(ClientState state, InetAddressAndPort coordinator, TimeUUID sessionId, TraceType traceType)
+        {
+            return new TraceStateImpl(state, coordinator, sessionId, traceType);
+        }
+
+        @Override
+        public void trace(ClientState clientState, ByteBuffer sessionId, String message, int ttl)
+        {
+            states.add(clientState);
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/tracing/TracingTestImpl.java b/test/unit/org/apache/cassandra/tracing/TracingTestImpl.java
index d03f84ba48c1..6e6305c038cd 100644
--- a/test/unit/org/apache/cassandra/tracing/TracingTestImpl.java
+++ b/test/unit/org/apache/cassandra/tracing/TracingTestImpl.java
@@ -28,8 +28,12 @@
 import org.apache.commons.lang3.StringUtils;
 
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.TimeUUID;
 
+/**
+ * A Tracing implementation that exposes its state (`traces` and `payloads`) for testing.
+ */
 public final class TracingTestImpl extends Tracing
 {
     private final List<String> traces;
@@ -46,11 +50,10 @@ public TracingTestImpl(List<String> traces)
         this.traces = traces;
     }
 
-    @Override
     public void stopSessionImpl()
-    {}
+    {
+    }
 
-    @Override
     public TraceState begin(String request, InetAddress ia, Map<String, String> map)
     {
         traces.add(request);
@@ -58,19 +61,19 @@ public TraceState begin(String request, InetAddress ia, Map<String, String> map)
     }
 
     @Override
-    protected TimeUUID newSession(TimeUUID sessionId, TraceType traceType, Map<String,ByteBuffer> customPayload)
+    protected TimeUUID newSession(ClientState state, TimeUUID sessionId, TraceType traceType, Map<String, ByteBuffer> customPayload)
     {
         if (!customPayload.isEmpty())
             logger.info("adding custom payload items {}", StringUtils.join(customPayload.keySet(), ','));
 
         payloads.putAll(customPayload);
-        return super.newSession(sessionId, traceType, customPayload);
+        return super.newSession(state, sessionId, traceType, customPayload);
     }
 
     @Override
-    protected TraceState newTraceState(InetAddressAndPort ia, TimeUUID uuid, Tracing.TraceType tt)
+    protected TraceState newTraceState(ClientState state, InetAddressAndPort ia, TimeUUID uuid, TraceType tt)
     {
-        return new TraceState(ia, uuid, tt)
+        return new TraceState(state, ia, uuid, tt)
         {
             protected void traceImpl(String string)
             {
@@ -84,7 +87,7 @@ protected void waitForPendingEvents()
     }
 
     @Override
-    public void trace(ByteBuffer bb, String message, int i)
+    public void trace(ClientState state, ByteBuffer bb, String message, int i)
     {
         traces.add(message);
     }
diff --git a/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java b/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java
index b0d1debfda74..22b3907cd31c 100644
--- a/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java
+++ b/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java
@@ -42,6 +42,9 @@
 import org.apache.cassandra.audit.AuditEvent;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.audit.AuditLogManager;
+import org.apache.cassandra.audit.DiagnosticEventAuditLogger;
+import org.apache.cassandra.auth.CassandraRoleManager;
+import org.apache.cassandra.auth.PasswordAuthenticator;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.OverrideConfigurationLoader;
 import org.apache.cassandra.config.ParameterizedClass;
@@ -64,11 +67,11 @@ public class CQLUserAuditTest
     public static void setup() throws Exception
     {
         OverrideConfigurationLoader.override((config) -> {
-            config.authenticator = new ParameterizedClass("PasswordAuthenticator");
-            config.role_manager = new ParameterizedClass("CassandraRoleManager");
+            config.authenticator = new ParameterizedClass(PasswordAuthenticator.class.getName());
+            config.role_manager = new ParameterizedClass(CassandraRoleManager.class.getName());
             config.diagnostic_events_enabled = true;
             config.audit_logging_options.enabled = true;
-            config.audit_logging_options.logger = new ParameterizedClass("DiagnosticEventAuditLogger", null);
+            config.audit_logging_options.logger = new ParameterizedClass(DiagnosticEventAuditLogger.class.getName(), null);
         });
 
         SUPERUSER_SETUP_DELAY_MS.setLong(0);
diff --git a/test/unit/org/apache/cassandra/transport/ClientResourceLimitsTest.java b/test/unit/org/apache/cassandra/transport/ClientResourceLimitsTest.java
index 10e1be1147a5..54d45dd0e84b 100644
--- a/test/unit/org/apache/cassandra/transport/ClientResourceLimitsTest.java
+++ b/test/unit/org/apache/cassandra/transport/ClientResourceLimitsTest.java
@@ -61,7 +61,7 @@ public class ClientResourceLimitsTest extends CQLTester
                             QueryOptions.DEFAULT.skipMetadata(),
                             QueryOptions.DEFAULT.getPageSize(),
                             QueryOptions.DEFAULT.getPagingState(),
-                            QueryOptions.DEFAULT.getSerialConsistency(),
+                            QueryOptions.DEFAULT.getSerialConsistency(null),
                             ProtocolVersion.V5,
                             KEYSPACE);
 
diff --git a/test/unit/org/apache/cassandra/transport/MessageDispatcherTest.java b/test/unit/org/apache/cassandra/transport/MessageDispatcherTest.java
index 819e75c4b7f0..8e459cc9a609 100644
--- a/test/unit/org/apache/cassandra/transport/MessageDispatcherTest.java
+++ b/test/unit/org/apache/cassandra/transport/MessageDispatcherTest.java
@@ -34,6 +34,8 @@
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.messages.AuthResponse;
 import org.mockito.Mockito;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 public class MessageDispatcherTest
 {
@@ -46,7 +48,7 @@ public Connection connection()
         }
 
         @Override
-        public Response execute(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+        public Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
         {
             return null;
         }
@@ -105,10 +107,9 @@ public Connection connection()
                     return connectionMock();
                 }
 
-                @Override
-                public Response execute(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
+                public Future<Response> maybeExecuteAsync(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
                 {
-                    return null;
+                    return ImmediateFuture.success(null);
                 }
             });
             Assert.assertEquals(requests, 1);
diff --git a/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java b/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
index a50174bd8702..679d057e56c4 100644
--- a/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
+++ b/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
@@ -17,14 +17,11 @@
  */
 package org.apache.cassandra.transport;
 
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
 import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.Map;
 
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -47,8 +44,8 @@
 import org.apache.cassandra.transport.messages.QueryMessage;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.MD5Digest;
-import org.apache.cassandra.utils.ReflectionUtils;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_QUERY_HANDLER_CLASS;
 import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
 
 public class MessagePayloadTest extends CQLTester
@@ -56,47 +53,12 @@ public class MessagePayloadTest extends CQLTester
     public static Map<String, ByteBuffer> requestPayload;
     public static Map<String, ByteBuffer> responsePayload;
 
-    private static Field cqlQueryHandlerField;
-    private static boolean modifiersAccessible;
-
     @BeforeClass
-    public static void makeCqlQueryHandlerAccessible()
+    public static void setUpClass()
     {
-        try
-        {
-            cqlQueryHandlerField = ClientState.class.getDeclaredField("cqlQueryHandler");
-            cqlQueryHandlerField.setAccessible(true);
-
-            Field modifiersField = ReflectionUtils.getModifiersField();
-            modifiersAccessible = modifiersField.isAccessible();
-            modifiersField.setAccessible(true);
-            modifiersField.setInt(cqlQueryHandlerField, cqlQueryHandlerField.getModifiers() & ~Modifier.FINAL);
-        }
-        catch (IllegalAccessException | NoSuchFieldException e)
-        {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @AfterClass
-    public static void resetCqlQueryHandlerField()
-    {
-        if (cqlQueryHandlerField == null)
-            return;
-        try
-        {
-            Field modifiersField = ReflectionUtils.getModifiersField();
-            modifiersField.setAccessible(true);
-            modifiersField.setInt(cqlQueryHandlerField, cqlQueryHandlerField.getModifiers() | Modifier.FINAL);
-
-            cqlQueryHandlerField.setAccessible(false);
-
-            modifiersField.setAccessible(modifiersAccessible);
-        }
-        catch (IllegalAccessException | NoSuchFieldException e)
-        {
-            throw new RuntimeException(e);
-        }
+        CUSTOM_QUERY_HANDLER_CLASS.setString("org.apache.cassandra.transport.MessagePayloadTest$TestQueryHandler");
+        CQLTester.setUpClass();
+        CQLTester.requireNetwork();
     }
 
     @After
@@ -115,256 +77,208 @@ public void dropCreatedTable()
     @Test
     public void testMessagePayloadBeta() throws Throwable
     {
-        QueryHandler queryHandler = (QueryHandler) cqlQueryHandlerField.get(null);
-        cqlQueryHandlerField.set(null, new TestQueryHandler());
-        try
-        {
-            requireNetwork();
+        Assert.assertSame(TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
 
-            Assert.assertSame(TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
-
-            SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
-                                                   nativePort,
-                                                   ProtocolVersion.V5,
-                                                   true,
-                                                   new EncryptionOptions());
-            try
-            {
-                client.connect(false);
-
-                Map<String, ByteBuffer> reqMap;
-                Map<String, ByteBuffer> respMap;
-
-                QueryOptions queryOptions = QueryOptions.create(
-                  QueryOptions.DEFAULT.getConsistency(),
-                  QueryOptions.DEFAULT.getValues(),
-                  QueryOptions.DEFAULT.skipMetadata(),
-                  QueryOptions.DEFAULT.getPageSize(),
-                  QueryOptions.DEFAULT.getPagingState(),
-                  QueryOptions.DEFAULT.getSerialConsistency(),
-                  ProtocolVersion.V5,
-                  KEYSPACE);
-                QueryMessage queryMessage = new QueryMessage("CREATE TABLE atable (pk int PRIMARY KEY, v text)",
-                                                             queryOptions);
-                PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM atable", KEYSPACE);
-
-                reqMap = Collections.singletonMap("foo", bytes(42));
-                responsePayload = respMap = Collections.singletonMap("bar", bytes(42));
-                queryMessage.setCustomPayload(reqMap);
-                Message.Response queryResponse = client.execute(queryMessage);
-                payloadEquals(reqMap, requestPayload);
-                payloadEquals(respMap, queryResponse.getCustomPayload());
-
-                reqMap = Collections.singletonMap("foo", bytes(43));
-                responsePayload = respMap = Collections.singletonMap("bar", bytes(43));
-                prepareMessage.setCustomPayload(reqMap);
-                ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
-                payloadEquals(reqMap, requestPayload);
-                payloadEquals(respMap, prepareResponse.getCustomPayload());
-
-                ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
-                reqMap = Collections.singletonMap("foo", bytes(44));
-                responsePayload = respMap = Collections.singletonMap("bar", bytes(44));
-                executeMessage.setCustomPayload(reqMap);
-                Message.Response executeResponse = client.execute(executeMessage);
-                payloadEquals(reqMap, requestPayload);
-                payloadEquals(respMap, executeResponse.getCustomPayload());
-
-                BatchMessage batchMessage = new BatchMessage(BatchStatement.Type.UNLOGGED,
-                                                             Collections.<Object>singletonList("INSERT INTO " + KEYSPACE + ".atable (pk,v) VALUES (1, 'foo')"),
-                                                             Collections.singletonList(Collections.<ByteBuffer>emptyList()),
-                                                             queryOptions);
-                reqMap = Collections.singletonMap("foo", bytes(45));
-                responsePayload = respMap = Collections.singletonMap("bar", bytes(45));
-                batchMessage.setCustomPayload(reqMap);
-                Message.Response batchResponse = client.execute(batchMessage);
-                payloadEquals(reqMap, requestPayload);
-                payloadEquals(respMap, batchResponse.getCustomPayload());
-            }
-            finally
-            {
-                client.close();
-            }
-        }
-        finally
+        try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(),
+                                                    nativePort,
+                                                    ProtocolVersion.V5,
+                                                    true,
+                                                    new EncryptionOptions()))
         {
-            cqlQueryHandlerField.set(null, queryHandler);
+            client.connect(false);
+
+            Map<String, ByteBuffer> reqMap;
+            Map<String, ByteBuffer> respMap;
+
+            QueryOptions queryOptions = QueryOptions.create(
+            QueryOptions.DEFAULT.getConsistency(),
+            QueryOptions.DEFAULT.getValues(),
+            QueryOptions.DEFAULT.skipMetadata(),
+            QueryOptions.DEFAULT.getPageSize(),
+            QueryOptions.DEFAULT.getPagingState(),
+            QueryOptions.DEFAULT.getSerialConsistency(null),
+            ProtocolVersion.V5,
+            KEYSPACE);
+            QueryMessage queryMessage = new QueryMessage("CREATE TABLE atable (pk int PRIMARY KEY, v text)",
+                                                         queryOptions);
+            PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM atable", KEYSPACE);
+
+            reqMap = Collections.singletonMap("foo", bytes(42));
+            responsePayload = respMap = Collections.singletonMap("bar", bytes(42));
+            queryMessage.setCustomPayload(reqMap);
+            Message.Response queryResponse = client.execute(queryMessage);
+            payloadEquals(reqMap, requestPayload);
+            payloadEquals(respMap, queryResponse.getCustomPayload());
+
+            reqMap = Collections.singletonMap("foo", bytes(43));
+            responsePayload = respMap = Collections.singletonMap("bar", bytes(43));
+            prepareMessage.setCustomPayload(reqMap);
+            ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
+            payloadEquals(reqMap, requestPayload);
+            payloadEquals(respMap, prepareResponse.getCustomPayload());
+
+            ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
+            reqMap = Collections.singletonMap("foo", bytes(44));
+            responsePayload = respMap = Collections.singletonMap("bar", bytes(44));
+            executeMessage.setCustomPayload(reqMap);
+            Message.Response executeResponse = client.execute(executeMessage);
+            payloadEquals(reqMap, requestPayload);
+            payloadEquals(respMap, executeResponse.getCustomPayload());
+
+            BatchMessage batchMessage = new BatchMessage(BatchStatement.Type.UNLOGGED,
+                                                         Collections.<Object>singletonList("INSERT INTO " + KEYSPACE + ".atable (pk,v) VALUES (1, 'foo')"),
+                                                         Collections.singletonList(Collections.<ByteBuffer>emptyList()),
+                                                         queryOptions);
+            reqMap = Collections.singletonMap("foo", bytes(45));
+            responsePayload = respMap = Collections.singletonMap("bar", bytes(45));
+            batchMessage.setCustomPayload(reqMap);
+            Message.Response batchResponse = client.execute(batchMessage);
+            payloadEquals(reqMap, requestPayload);
+            payloadEquals(respMap, batchResponse.getCustomPayload());
         }
     }
 
     @Test
     public void testMessagePayload() throws Throwable
     {
-        QueryHandler queryHandler = (QueryHandler) cqlQueryHandlerField.get(null);
-        cqlQueryHandlerField.set(null, new TestQueryHandler());
-        try
-        {
-            requireNetwork();
-
-            Assert.assertSame(TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
+        Assert.assertSame(TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
 
-            SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort);
-            try
-            {
-                client.connect(false);
-
-                Map<String, ByteBuffer> reqMap;
-                Map<String, ByteBuffer> respMap;
-
-                QueryMessage queryMessage = new QueryMessage(
-                                                            "CREATE TABLE " + KEYSPACE + ".atable (pk int PRIMARY KEY, v text)",
-                                                            QueryOptions.DEFAULT
-                );
-                PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable", null);
-
-                reqMap = Collections.singletonMap("foo", bytes(42));
-                responsePayload = respMap = Collections.singletonMap("bar", bytes(42));
-                queryMessage.setCustomPayload(reqMap);
-                Message.Response queryResponse = client.execute(queryMessage);
-                payloadEquals(reqMap, requestPayload);
-                payloadEquals(respMap, queryResponse.getCustomPayload());
-
-                reqMap = Collections.singletonMap("foo", bytes(43));
-                responsePayload = respMap = Collections.singletonMap("bar", bytes(43));
-                prepareMessage.setCustomPayload(reqMap);
-                ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
-                payloadEquals(reqMap, requestPayload);
-                payloadEquals(respMap, prepareResponse.getCustomPayload());
-
-                ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
-                reqMap = Collections.singletonMap("foo", bytes(44));
-                responsePayload = respMap = Collections.singletonMap("bar", bytes(44));
-                executeMessage.setCustomPayload(reqMap);
-                Message.Response executeResponse = client.execute(executeMessage);
-                payloadEquals(reqMap, requestPayload);
-                payloadEquals(respMap, executeResponse.getCustomPayload());
-
-                BatchMessage batchMessage = new BatchMessage(BatchStatement.Type.UNLOGGED,
-                                                             Collections.<Object>singletonList("INSERT INTO " + KEYSPACE + ".atable (pk,v) VALUES (1, 'foo')"),
-                                                             Collections.singletonList(Collections.<ByteBuffer>emptyList()),
-                                                             QueryOptions.DEFAULT);
-                reqMap = Collections.singletonMap("foo", bytes(45));
-                responsePayload = respMap = Collections.singletonMap("bar", bytes(45));
-                batchMessage.setCustomPayload(reqMap);
-                Message.Response batchResponse = client.execute(batchMessage);
-                payloadEquals(reqMap, requestPayload);
-                payloadEquals(respMap, batchResponse.getCustomPayload());
-            }
-            finally
-            {
-                client.close();
-            }
-        }
-        finally
+        try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort))
         {
-            cqlQueryHandlerField.set(null, queryHandler);
+            client.connect(false);
+
+            Map<String, ByteBuffer> reqMap;
+            Map<String, ByteBuffer> respMap;
+
+            QueryMessage queryMessage = new QueryMessage(
+            "CREATE TABLE " + KEYSPACE + ".atable (pk int PRIMARY KEY, v text)",
+            QueryOptions.DEFAULT
+            );
+            PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable", null);
+
+            reqMap = Collections.singletonMap("foo", bytes(42));
+            responsePayload = respMap = Collections.singletonMap("bar", bytes(42));
+            queryMessage.setCustomPayload(reqMap);
+            Message.Response queryResponse = client.execute(queryMessage);
+            payloadEquals(reqMap, requestPayload);
+            payloadEquals(respMap, queryResponse.getCustomPayload());
+
+            reqMap = Collections.singletonMap("foo", bytes(43));
+            responsePayload = respMap = Collections.singletonMap("bar", bytes(43));
+            prepareMessage.setCustomPayload(reqMap);
+            ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
+            payloadEquals(reqMap, requestPayload);
+            payloadEquals(respMap, prepareResponse.getCustomPayload());
+
+            ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
+            reqMap = Collections.singletonMap("foo", bytes(44));
+            responsePayload = respMap = Collections.singletonMap("bar", bytes(44));
+            executeMessage.setCustomPayload(reqMap);
+            Message.Response executeResponse = client.execute(executeMessage);
+            payloadEquals(reqMap, requestPayload);
+            payloadEquals(respMap, executeResponse.getCustomPayload());
+
+            BatchMessage batchMessage = new BatchMessage(BatchStatement.Type.UNLOGGED,
+                                                         Collections.<Object>singletonList("INSERT INTO " + KEYSPACE + ".atable (pk,v) VALUES (1, 'foo')"),
+                                                         Collections.singletonList(Collections.<ByteBuffer>emptyList()),
+                                                         QueryOptions.DEFAULT);
+            reqMap = Collections.singletonMap("foo", bytes(45));
+            responsePayload = respMap = Collections.singletonMap("bar", bytes(45));
+            batchMessage.setCustomPayload(reqMap);
+            Message.Response batchResponse = client.execute(batchMessage);
+            payloadEquals(reqMap, requestPayload);
+            payloadEquals(respMap, batchResponse.getCustomPayload());
         }
     }
 
     @Test
     public void testMessagePayloadVersion3() throws Throwable
     {
-        QueryHandler queryHandler = (QueryHandler) cqlQueryHandlerField.get(null);
-        cqlQueryHandlerField.set(null, new TestQueryHandler());
-        try
+        Assert.assertSame(TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
+
+        try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, ProtocolVersion.V3))
         {
-            requireNetwork();
+            client.connect(false);
 
-            Assert.assertSame(TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
+            Map<String, ByteBuffer> reqMap;
 
-            SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, ProtocolVersion.V3);
+            QueryMessage queryMessage = new QueryMessage(
+            "CREATE TABLE " + KEYSPACE + ".atable (pk int PRIMARY KEY, v text)",
+            QueryOptions.DEFAULT
+            );
+            PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable", null);
+
+            reqMap = Collections.singletonMap("foo", bytes(42));
+            responsePayload = Collections.singletonMap("bar", bytes(42));
+            queryMessage.setCustomPayload(reqMap);
             try
             {
-                client.connect(false);
-
-                Map<String, ByteBuffer> reqMap;
-
-                QueryMessage queryMessage = new QueryMessage(
-                                                            "CREATE TABLE " + KEYSPACE + ".atable (pk int PRIMARY KEY, v text)",
-                                                            QueryOptions.DEFAULT
-                );
-                PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable", null);
-
-                reqMap = Collections.singletonMap("foo", bytes(42));
-                responsePayload = Collections.singletonMap("bar", bytes(42));
-                queryMessage.setCustomPayload(reqMap);
-                try
-                {
-                    client.execute(queryMessage);
-                    Assert.fail();
-                }
-                catch (RuntimeException e)
-                {
-                    Assert.assertTrue(e.getCause() instanceof ProtocolException);
-                }
-                // when a protocol exception is thrown by the server the connection is closed, so need to re-connect
-                client.close();
-                client.connect(false);
-
-                queryMessage.setCustomPayload(null);
                 client.execute(queryMessage);
+                Assert.fail();
+            }
+            catch (RuntimeException e)
+            {
+                Assert.assertTrue(e.getCause() instanceof ProtocolException);
+            }
+            // when a protocol exception is thrown by the server the connection is closed, so need to re-connect
+            client.close();
+            client.connect(false);
 
-                reqMap = Collections.singletonMap("foo", bytes(43));
-                responsePayload = Collections.singletonMap("bar", bytes(43));
-                prepareMessage.setCustomPayload(reqMap);
-                try
-                {
-                    client.execute(prepareMessage);
-                    Assert.fail();
-                }
-                catch (RuntimeException e)
-                {
-                    Assert.assertTrue(e.getCause() instanceof ProtocolException);
-                }
-                // when a protocol exception is thrown by the server the connection is closed, so need to re-connect
-                client.close();
-                client.connect(false);
-
-                prepareMessage.setCustomPayload(null);
-                ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
-
-                ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
-                reqMap = Collections.singletonMap("foo", bytes(44));
-                responsePayload = Collections.singletonMap("bar", bytes(44));
-                executeMessage.setCustomPayload(reqMap);
-                try
-                {
-                    client.execute(executeMessage);
-                    Assert.fail();
-                }
-                catch (RuntimeException e)
-                {
-                    Assert.assertTrue(e.getCause() instanceof ProtocolException);
-                }
-                // when a protocol exception is thrown by the server the connection is closed, so need to re-connect
-                client.close();
-                client.connect(false);
-
-                BatchMessage batchMessage = new BatchMessage(BatchStatement.Type.UNLOGGED,
-                                                             Collections.<Object>singletonList("INSERT INTO " + KEYSPACE + ".atable (pk,v) VALUES (1, 'foo')"),
-                                                             Collections.singletonList(Collections.<ByteBuffer>emptyList()),
-                                                             QueryOptions.DEFAULT);
-                reqMap = Collections.singletonMap("foo", bytes(45));
-                responsePayload = Collections.singletonMap("bar", bytes(45));
-                batchMessage.setCustomPayload(reqMap);
-                try
-                {
-                    client.execute(batchMessage);
-                    Assert.fail();
-                }
-                catch (RuntimeException e)
-                {
-                    Assert.assertTrue(e.getCause() instanceof ProtocolException);
-                }
+            queryMessage.setCustomPayload(null);
+            client.execute(queryMessage);
+
+            reqMap = Collections.singletonMap("foo", bytes(43));
+            responsePayload = Collections.singletonMap("bar", bytes(43));
+            prepareMessage.setCustomPayload(reqMap);
+            try
+            {
+                client.execute(prepareMessage);
+                Assert.fail();
             }
-            finally
+            catch (RuntimeException e)
             {
-                client.close();
+                Assert.assertTrue(e.getCause() instanceof ProtocolException);
+            }
+            // when a protocol exception is thrown by the server the connection is closed, so need to re-connect
+            client.close();
+            client.connect(false);
+
+            prepareMessage.setCustomPayload(null);
+            ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
+
+            ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
+            reqMap = Collections.singletonMap("foo", bytes(44));
+            responsePayload = Collections.singletonMap("bar", bytes(44));
+            executeMessage.setCustomPayload(reqMap);
+            try
+            {
+                client.execute(executeMessage);
+                Assert.fail();
+            }
+            catch (RuntimeException e)
+            {
+                Assert.assertTrue(e.getCause() instanceof ProtocolException);
+            }
+            // when a protocol exception is thrown by the server the connection is closed, so need to re-connect
+            client.close();
+            client.connect(false);
+
+            BatchMessage batchMessage = new BatchMessage(BatchStatement.Type.UNLOGGED,
+                                                         Collections.<Object>singletonList("INSERT INTO " + KEYSPACE + ".atable (pk,v) VALUES (1, 'foo')"),
+                                                         Collections.singletonList(Collections.<ByteBuffer>emptyList()),
+                                                         QueryOptions.DEFAULT);
+            reqMap = Collections.singletonMap("foo", bytes(45));
+            responsePayload = Collections.singletonMap("bar", bytes(45));
+            batchMessage.setCustomPayload(reqMap);
+            try
+            {
+                client.execute(batchMessage);
+                Assert.fail();
+            }
+            catch (RuntimeException e)
+            {
+                Assert.assertTrue(e.getCause() instanceof ProtocolException);
             }
-        }
-        finally
-        {
-            cqlQueryHandlerField.set(null, queryHandler);
         }
     }
 
diff --git a/test/unit/org/apache/cassandra/transport/NativeTransportTimeoutTest.java b/test/unit/org/apache/cassandra/transport/NativeTransportTimeoutTest.java
new file mode 100644
index 000000000000..5f58a6a09a4f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/NativeTransportTimeoutTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.utils.MonotonicClock;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.datastax.driver.core.BatchStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSetFuture;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.Statement;
+import com.datastax.driver.core.exceptions.OverloadedException;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.metrics.ClientMetrics;
+import org.assertj.core.api.Assertions;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+@RunWith(BMUnitRunner.class)
+public class NativeTransportTimeoutTest extends CQLTester
+{
+    static Semaphore EXECUTE_BARRIER;
+    static Semaphore WAIT_BARRIER;
+
+    @Test
+    @BMRules(rules = { @BMRule(name = "Delay Message execution on NTR stage",
+                       targetClass = "org.apache.cassandra.transport.Message$Request",
+                       targetMethod = "execute",
+                       targetLocation = "AT ENTRY",
+                       condition = "$this.getCustomPayload() != null",
+                       action = "org.apache.cassandra.transport.NativeTransportTimeoutTest.WAIT_BARRIER.release(); " +
+                                "org.apache.cassandra.transport.NativeTransportTimeoutTest.EXECUTE_BARRIER.acquire(); " +
+                                "flag(Thread.currentThread());"),
+                       @BMRule(name = "Mock NTR timeout from Request.execute",
+                       targetClass = "org.apache.cassandra.config.DatabaseDescriptor",
+                       targetMethod = "getNativeTransportTimeout",
+                       targetLocation = "AT ENTRY",
+                       condition = "flagged(Thread.currentThread()) && callerEquals(\"Message$Request.execute\", true)",
+                       action = "clear(Thread.currentThread()); " +
+                                "return 10000000;") })
+    public void testNativeTransportLoadShedding() throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int PRIMARY KEY, v text)");
+        Statement statement = new SimpleStatement("SELECT * FROM " + KEYSPACE + '.' + currentTable());
+        doTestLoadShedding(false, statement);
+    }
+
+    @Test
+    @BMRules(rules = { @BMRule(name = "Delay elapsedTimeSinceCreationCheck from async stage",
+                       targetClass = "org.apache.cassandra.transport.Message$Request",
+                       targetMethod = "elapsedTimeSinceCreation",
+                       targetLocation = "AT ENTRY",
+                       condition = "$this.getCustomPayload() != null && !callerEquals(\"Message$Request.execute\", true)",
+                       action = "org.apache.cassandra.transport.NativeTransportTimeoutTest.WAIT_BARRIER.release(); " +
+                                "org.apache.cassandra.transport.NativeTransportTimeoutTest.EXECUTE_BARRIER.acquire(); " +
+                                "flag(Thread.currentThread());"),
+                       @BMRule(name = "Mock native transport timeout from async stage",
+                       targetClass = "org.apache.cassandra.config.DatabaseDescriptor",
+                       targetMethod = "getNativeTransportTimeout",
+                       targetLocation = "AT ENTRY",
+                       condition = "flagged(Thread.currentThread()) && callerMatches(\".*maybeExecuteAsync.*\", true)",
+                       action = "clear(Thread.currentThread()); " +
+                                "return 10000000;") })
+    public void testAsyncStageLoadShedding() throws Throwable
+    {
+        CassandraRelevantProperties.NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED.setBoolean(true);
+
+        try
+        {
+            createTable("CREATE TABLE %s (pk int PRIMARY KEY, v text)");
+
+            Statement statement = new SimpleStatement("SELECT * FROM " + KEYSPACE + '.' + currentTable());
+            doTestLoadShedding(true, statement);
+
+            Statement insert1 = new SimpleStatement("INSERT INTO " + KEYSPACE + '.' + currentTable() + " (pk, v) VALUES (1, 'foo')");
+            Statement insert2 = new SimpleStatement("INSERT INTO " + KEYSPACE + '.' + currentTable() + " (pk, v) VALUES (2, 'bar')");
+            statement = new BatchStatement().add(insert1).add(insert2);
+            doTestLoadShedding(true, statement);
+
+            PreparedStatement ps = sessionNet().prepare("SELECT * FROM " + KEYSPACE + '.' + currentTable());
+            doTestLoadShedding(true, ps.bind());
+        }
+        finally
+        {
+            CassandraRelevantProperties.NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED.setBoolean(false);
+        }
+    }
+
+    private void doTestLoadShedding(boolean useAsyncStages, Statement statement) throws InterruptedException
+    {
+        EXECUTE_BARRIER = new Semaphore(0);
+        WAIT_BARRIER = new Semaphore(0);
+
+        Meter timedOutMeter;
+        Timer queueTimer;
+
+        Session session = sessionNet();
+
+        // custom payload used to make detection of this statement easy early in byteman rules
+        statement.setOutgoingPayload(Collections.singletonMap("sentinel", ByteBuffer.wrap(new byte[0])));
+
+        if (useAsyncStages)
+        {
+            timedOutMeter = ClientMetrics.instance.timedOutBeforeAsyncProcessing;
+            queueTimer = ClientMetrics.instance.asyncQueueTime;
+        }
+        else
+        {
+            timedOutMeter = ClientMetrics.instance.timedOutBeforeProcessing;
+            queueTimer = ClientMetrics.instance.queueTime;
+        }
+
+        long initialTimedOut = timedOutMeter.getCount();
+
+        ResultSetFuture rsf = session.executeAsync(statement);
+
+        // once WAIT_BARRIER is acquired, the Stage we want an OverloadedException from is executing the statement,
+        // but it hasn't yet retrieved the elapsed time. It will not proceed until the EXECUTE_BARRIER is released.
+        // The Byteman rules in the tests will override the native transport timeout to 10 milliseconds from that
+        // callsite. Therefore, to ensure an OverloadedException by exceeding the timeout, we need to sleep for 10
+        // milliseconds plus 2x the error of approxTime (creation timestamp error + error when getting current time).
+        WAIT_BARRIER.acquire();
+        Thread.sleep(10 + TimeUnit.MILLISECONDS.convert(MonotonicClock.Global.approxTime.error(), TimeUnit.NANOSECONDS) * 2);
+        EXECUTE_BARRIER.release();
+
+        Assertions.assertThatThrownBy(rsf::get).hasCauseInstanceOf(OverloadedException.class);
+        Assert.assertEquals(initialTimedOut + 1, timedOutMeter.getCount());
+        double queueTimer_in_micros = queueTimer.getSnapshot().get999thPercentile();
+        long queuetimer_in_nanos = TimeUnit.NANOSECONDS.convert((long) queueTimer_in_micros, TimeUnit.MICROSECONDS);
+        Assert.assertTrue(queuetimer_in_nanos > TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/transport/RateLimitingTest.java b/test/unit/org/apache/cassandra/transport/RateLimitingTest.java
index 0b3fb34a79ed..da32e522afc8 100644
--- a/test/unit/org/apache/cassandra/transport/RateLimitingTest.java
+++ b/test/unit/org/apache/cassandra/transport/RateLimitingTest.java
@@ -315,7 +315,7 @@ private QueryOptions queryOptions()
                                    QueryOptions.DEFAULT.skipMetadata(),
                                    QueryOptions.DEFAULT.getPageSize(),
                                    QueryOptions.DEFAULT.getPagingState(),
-                                   QueryOptions.DEFAULT.getSerialConsistency(),
+                                   QueryOptions.DEFAULT.getSerialConsistency(null),
                                    version,
                                    KEYSPACE);
     }
diff --git a/test/unit/org/apache/cassandra/transport/SerDeserTest.java b/test/unit/org/apache/cassandra/transport/SerDeserTest.java
index 11b74892661d..f9c309c69f62 100644
--- a/test/unit/org/apache/cassandra/transport/SerDeserTest.java
+++ b/test/unit/org/apache/cassandra/transport/SerDeserTest.java
@@ -28,6 +28,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.collect.ImmutableSet;
+
 import io.netty.buffer.Unpooled;
 import io.netty.buffer.ByteBuf;
 
@@ -43,6 +45,7 @@
 import org.apache.cassandra.cql3.FieldIdentifier;
 import org.apache.cassandra.cql3.Lists;
 import org.apache.cassandra.cql3.Maps;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.ResultSet;
 import org.apache.cassandra.cql3.Sets;
@@ -58,6 +61,7 @@
 import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.serializers.CollectionSerializer;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.service.ClientState;
@@ -384,7 +388,7 @@ public void queryOptionsSerDeserTest()
                 QueryOptions.create(ConsistencyLevel.ALL,
                                     Collections.singletonList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 })),
                                     false,
-                                    5000,
+                                    PageSize.inRows(5000),
                                     Util.makeSomePagingState(version),
                                     ConsistencyLevel.SERIAL,
                                     version,
@@ -400,7 +404,7 @@ public void queryOptionsSerDeserTest()
                                     Arrays.asList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 }),
                                                   ByteBuffer.wrap(new byte[] { 0x03, 0x04, 0x05, 0x03, 0x04, 0x05 })),
                                     true,
-                                    10,
+                                    PageSize.inRows(10),
                                     Util.makeSomePagingState(version),
                                     ConsistencyLevel.SERIAL,
                                     version,
@@ -416,7 +420,7 @@ public void queryOptionsSerDeserTest()
                                     Arrays.asList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 }),
                                                   ByteBuffer.wrap(new byte[] { 0x03, 0x04, 0x05, 0x03, 0x04, 0x05 })),
                                     true,
-                                    10,
+                                    PageSize.inBytes(10),
                                     Util.makeSomePagingState(version),
                                     ConsistencyLevel.SERIAL,
                                     version,
@@ -437,7 +441,7 @@ private void queryOptionsSerDeserTest(ProtocolVersion version, QueryOptions opti
 
         assertNotNull(decodedOptions);
         assertEquals(options.getConsistency(), decodedOptions.getConsistency());
-        assertEquals(options.getSerialConsistency(), decodedOptions.getSerialConsistency());
+        assertEquals(options.getSerialConsistency(null), decodedOptions.getSerialConsistency(null));
         assertEquals(options.getPageSize(), decodedOptions.getPageSize());
         assertEquals(options.getProtocolVersion(), decodedOptions.getProtocolVersion());
         assertEquals(options.getValues(), decodedOptions.getValues());
@@ -448,6 +452,105 @@ private void queryOptionsSerDeserTest(ProtocolVersion version, QueryOptions opti
         assertEquals(options.getNowInSeconds(state), decodedOptions.getNowInSeconds(state));
     }
 
+    @Test
+    public void defaultSerialCLGuardrailsTest()
+    {
+        for(ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        {
+            defaultSerialCLGuardrailsTest(version, new LinkedHashSet<>(), ConsistencyLevel.SERIAL);
+            defaultSerialCLGuardrailsTest(version,
+                                          new LinkedHashSet<>(Arrays.asList(ConsistencyLevel.LOCAL_SERIAL)),
+                                          ConsistencyLevel.SERIAL);
+            defaultSerialCLGuardrailsTest(version,
+                                          new LinkedHashSet<>(Arrays.asList(ConsistencyLevel.SERIAL)),
+                                          ConsistencyLevel.LOCAL_SERIAL);
+            defaultSerialCLGuardrailsTest(version,
+                                          new LinkedHashSet<>(Arrays.asList(ConsistencyLevel.SERIAL,
+                                                                   ConsistencyLevel.LOCAL_SERIAL)),
+                                          null);
+        }
+    }
+
+    private void defaultSerialCLGuardrailsTest(ProtocolVersion version,
+                                               Set<ConsistencyLevel> writeConsistencyLevelsDisallowed,
+                                               ConsistencyLevel expectedDecodedSerialConsistency)
+    {
+        Set<ConsistencyLevel> previousConsistencyLevels =  DatabaseDescriptor.getGuardrailsConfig().getWriteConsistencyLevelsDisallowed();
+        DatabaseDescriptor.getGuardrailsConfig().setWriteConsistencyLevelsDisallowed(ImmutableSet.copyOf(writeConsistencyLevelsDisallowed));
+
+        QueryOptions queryOptions = QueryOptions.create(ConsistencyLevel.ALL,
+                                                        Collections.singletonList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 })),
+                                                        false,
+                                                        PageSize.inRows(5000),
+                                                        Util.makeSomePagingState(version),
+                                                        null,
+                                                        version,
+                                                        null);
+        ByteBuf buf = Unpooled.buffer(QueryOptions.codec.encodedSize(queryOptions, version));
+        QueryOptions.codec.encode(queryOptions, buf, version);
+        QueryOptions decodedOptions = QueryOptions.codec.decode(buf, version);
+        if (expectedDecodedSerialConsistency != null)
+        {
+            assertEquals(expectedDecodedSerialConsistency, decodedOptions.getSerialConsistency(null));
+        }
+        else
+        {
+            try
+            {
+                decodedOptions.getSerialConsistency(null);
+                throw new AssertionError("Decoding should have failed with InvalidRequestException");
+            }
+            catch (InvalidRequestException e)
+            {
+                assertEquals("Serial consistency levels are disallowed by disallowedWriteConsistencies Guardrail",
+                             e.getMessage());
+            }
+        }
+
+        DatabaseDescriptor.getGuardrailsConfig().setWriteConsistencyLevelsDisallowed(ImmutableSet.copyOf(previousConsistencyLevels));
+    }
+
+    @Test
+    public void specifiedSerialCLGuardrailsTest()
+    {
+        // write consistency level guardrail check happens before query execution. Here we validate only that if
+        // QueryOptions has explicitly set serial consistency, the same consistency level remains after encoding/decoding
+        // even if that level is forbidden by the guardrail.
+
+        Set<ConsistencyLevel> serialCLs = new LinkedHashSet<>(Arrays.asList(ConsistencyLevel.LOCAL_SERIAL, ConsistencyLevel.SERIAL));
+        for(ProtocolVersion version : ProtocolVersion.SUPPORTED)
+        {
+            specifiedSerialCLGuardrailsTest(version, ConsistencyLevel.SERIAL, new LinkedHashSet<>(), ConsistencyLevel.SERIAL);
+            specifiedSerialCLGuardrailsTest(version, ConsistencyLevel.SERIAL, serialCLs, ConsistencyLevel.SERIAL);
+            specifiedSerialCLGuardrailsTest(version, ConsistencyLevel.LOCAL_SERIAL, new LinkedHashSet<>(), ConsistencyLevel.LOCAL_SERIAL);
+            specifiedSerialCLGuardrailsTest(version, ConsistencyLevel.LOCAL_SERIAL, serialCLs, ConsistencyLevel.LOCAL_SERIAL);
+        }
+    }
+
+    private void specifiedSerialCLGuardrailsTest(ProtocolVersion version,
+                                                 ConsistencyLevel specifiedSerialConsistency,
+                                                 Set<ConsistencyLevel> writeConsistencyLevelsDisallowed,
+                                                 ConsistencyLevel expectedDecodedSerialConsistency)
+    {
+        Set<ConsistencyLevel> previousConsistencyLevels =  DatabaseDescriptor.getGuardrailsConfig().getWriteConsistencyLevelsDisallowed();
+        DatabaseDescriptor.getGuardrailsConfig().setWriteConsistencyLevelsDisallowed(ImmutableSet.copyOf(writeConsistencyLevelsDisallowed));
+
+        QueryOptions queryOptions = QueryOptions.create(ConsistencyLevel.ALL,
+                                                        Collections.singletonList(ByteBuffer.wrap(new byte[] { 0x00, 0x01, 0x02 })),
+                                                        false,
+                                                        PageSize.inRows(5000),
+                                                        Util.makeSomePagingState(version),
+                                                        specifiedSerialConsistency,
+                                                        version,
+                                                        null);
+        ByteBuf buf = Unpooled.buffer(QueryOptions.codec.encodedSize(queryOptions, version));
+        QueryOptions.codec.encode(queryOptions, buf, version);
+        QueryOptions decodedOptions = QueryOptions.codec.decode(buf, version);
+        assertEquals(expectedDecodedSerialConsistency, decodedOptions.getSerialConsistency(null));
+
+        DatabaseDescriptor.getGuardrailsConfig().setWriteConsistencyLevelsDisallowed(ImmutableSet.copyOf(previousConsistencyLevels));
+    }
+
     // return utf8 string that contains no ascii chars
     public static String randomUTF8(int count)
     {
diff --git a/test/unit/org/apache/cassandra/transport/TransportTest.java b/test/unit/org/apache/cassandra/transport/TransportTest.java
new file mode 100644
index 000000000000..7c4ae0f449c7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/TransportTest.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.transport;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.BatchQueryOptions;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryHandler;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.messages.BatchMessage;
+import org.apache.cassandra.transport.messages.ExecuteMessage;
+import org.apache.cassandra.transport.messages.PrepareMessage;
+import org.apache.cassandra.transport.messages.QueryMessage;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.MD5Digest;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionEvaluationLogger;
+import org.awaitility.core.TimeoutEvent;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_QUERY_HANDLER_CLASS;
+
+public class TransportTest extends CQLTester
+{
+    @BeforeClass
+    public static void setUpClass()
+    {
+        CUSTOM_QUERY_HANDLER_CLASS.setString("org.apache.cassandra.transport.TransportTest$TestQueryHandler");
+        CQLTester.setUpClass();
+        CQLTester.requireNetwork();
+    }
+
+    @After
+    public void dropCreatedTable()
+    {
+        try
+        {
+            QueryProcessor.executeOnceInternal("DROP TABLE " + KEYSPACE + ".atable");
+        }
+        catch (Throwable t)
+        {
+            // ignore
+        }
+    }
+
+    @Test
+    public void testAsyncTransport() throws Throwable
+    {
+        Assert.assertSame(TransportTest.TestQueryHandler.class, ClientState.getCQLQueryHandler().getClass());
+
+        CassandraRelevantProperties.NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED.setBoolean(true);
+        try
+        {
+            doTestTransport();
+        }
+        finally
+        {
+            CassandraRelevantProperties.NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED.setBoolean(false);
+        }
+    }
+
+    private void doTestTransport() throws Throwable
+    {
+        try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort))
+        {
+            client.connect(false);
+
+            // Async native transport causes native transport requests to be executed asynchronously on the coordinator read and coordinator write
+            // stages: this means the expected number of tasks on those stages starts at 1 for the async request.
+            // The read and write stages are used for the actual execution.
+            // Please don't tell me we're not using var. Making final copies is worse. Hardcoding the values is also worse.
+            var ref = new Object()
+            {
+                long expectedCoordinateReadTasks = 1;
+                long expectedExecuteReadTasks = 1;
+                int expectedCoordinateMutationTasks = 1;
+                int expectedExecuteMutationTasks = 1;
+            };
+
+            QueryMessage createMessage = new QueryMessage("CREATE TABLE " + KEYSPACE + ".atable (pk int PRIMARY KEY, v text)", QueryOptions.DEFAULT);
+            PrepareMessage prepareMessage = new PrepareMessage("SELECT * FROM " + KEYSPACE + ".atable", null);
+
+            client.execute(createMessage);
+            ResultMessage.Prepared prepareResponse = (ResultMessage.Prepared) client.execute(prepareMessage);
+
+            ExecuteMessage executeMessage = new ExecuteMessage(prepareResponse.statementId, prepareResponse.resultMetadataId, QueryOptions.DEFAULT);
+            Message.Response executeResponse = client.execute(executeMessage);
+            Assert.assertEquals(1, executeResponse.getWarnings().size());
+            Assert.assertEquals("async-prepared", executeResponse.getWarnings().get(0));
+            awaitUntil(() -> Stage.COORDINATE_READ.getCompletedTaskCount() == ref.expectedCoordinateReadTasks);
+            awaitUntil(() -> Stage.READ.getCompletedTaskCount() == ref.expectedExecuteReadTasks);
+
+            // we now expect two more tasks
+            ref.expectedCoordinateReadTasks++;
+            ref.expectedExecuteReadTasks++;
+            QueryMessage readMessage = new QueryMessage("SELECT * FROM " + KEYSPACE + ".atable", QueryOptions.DEFAULT);
+            Message.Response readResponse = client.execute(readMessage);
+            Assert.assertEquals(1, executeResponse.getWarnings().size());
+            Assert.assertEquals("async-process", readResponse.getWarnings().get(0));
+            awaitUntil(() -> Stage.COORDINATE_READ.getCompletedTaskCount() == ref.expectedCoordinateReadTasks);
+            awaitUntil(() -> Stage.READ.getCompletedTaskCount() == ref.expectedExecuteReadTasks);
+
+            BatchMessage batchMessage = new BatchMessage(BatchStatement.Type.UNLOGGED,
+                                                         Collections.singletonList("INSERT INTO " + KEYSPACE + ".atable (pk,v) VALUES (1, 'foo')"),
+                                                         Collections.singletonList(Collections.<ByteBuffer>emptyList()),
+                                                         QueryOptions.DEFAULT);
+            Message.Response batchResponse = client.execute(batchMessage);
+            Assert.assertEquals(1, executeResponse.getWarnings().size());
+            Assert.assertEquals("async-batch", batchResponse.getWarnings().get(0));
+            awaitUntil(() -> Stage.COORDINATE_READ.getCompletedTaskCount() == ref.expectedCoordinateReadTasks);
+            awaitUntil(() -> Stage.READ.getCompletedTaskCount() == ref.expectedExecuteReadTasks);
+
+            // we now expect two more tasks
+            ref.expectedCoordinateMutationTasks++;
+            ref.expectedExecuteMutationTasks++;
+            QueryMessage insertMessage = new QueryMessage("INSERT INTO " + KEYSPACE + ".atable (pk,v) VALUES (1, 'foo')", QueryOptions.DEFAULT);
+            Message.Response insertResponse = client.execute(insertMessage);
+            Assert.assertEquals(1, executeResponse.getWarnings().size());
+            Assert.assertEquals("async-process", insertResponse.getWarnings().get(0));
+            awaitUntil(() -> Stage.COORDINATE_READ.getCompletedTaskCount() == ref.expectedCoordinateReadTasks);
+            awaitUntil(() -> Stage.READ.getCompletedTaskCount() == ref.expectedExecuteReadTasks);
+        }
+    }
+
+    private void awaitUntil(Callable<Boolean> condition)
+    {
+        Awaitility.await("await until stages have completed the required number of tasks; on timeout check the ERROR logs for the actual numbers completed")
+                  .conditionEvaluationListener(new DumpStageInfoOnTimeout())
+                  .until(condition);
+    }
+
+    public static class TestQueryHandler implements QueryHandler
+    {
+        @Override
+        public QueryProcessor.Prepared getPrepared(MD5Digest id)
+        {
+            return QueryProcessor.instance.getPrepared(id);
+        }
+
+        @Override
+        public CQLStatement parse(String query, QueryState state, QueryOptions options)
+        {
+            return QueryProcessor.instance.parse(query, state, options);
+        }
+
+        @Override
+        public ResultMessage.Prepared prepare(String query,
+                                              ClientState clientState,
+                                              Map<String, ByteBuffer> customPayload)
+        throws RequestValidationException
+        {
+            return QueryProcessor.instance.prepare(query, clientState, customPayload);
+        }
+
+        @Override
+        public ResultMessage process(CQLStatement statement,
+                                     QueryState state,
+                                     QueryOptions options,
+                                     Map<String, ByteBuffer> customPayload,
+                                     Dispatcher.RequestTime requestTime)
+        throws RequestExecutionException, RequestValidationException
+        {
+            ClientWarn.instance.warn("async-process");
+            return QueryProcessor.instance.process(statement, state, options, customPayload, requestTime);
+        }
+
+        @Override
+        public ResultMessage processBatch(BatchStatement statement,
+                                          QueryState state,
+                                          BatchQueryOptions options,
+                                          Map<String, ByteBuffer> customPayload,
+                                          Dispatcher.RequestTime requestTime)
+        throws RequestExecutionException, RequestValidationException
+        {
+            ClientWarn.instance.warn("async-batch");
+            return QueryProcessor.instance.processBatch(statement, state, options, customPayload, requestTime);
+        }
+
+        @Override
+        public ResultMessage processPrepared(CQLStatement statement,
+                                             QueryState state,
+                                             QueryOptions options,
+                                             Map<String, ByteBuffer> customPayload,
+                                             Dispatcher.RequestTime requestTime)
+        throws RequestExecutionException, RequestValidationException
+        {
+            ClientWarn.instance.warn("async-prepared");
+            return QueryProcessor.instance.processPrepared(statement, state, options, customPayload, requestTime);
+        }
+    }
+
+    private static class DumpStageInfoOnTimeout extends ConditionEvaluationLogger
+    {
+        public DumpStageInfoOnTimeout()
+        {
+            super(logger::warn);
+        }
+
+        @Override
+        public void onTimeout(TimeoutEvent timeoutEvent)
+        {
+            super.onTimeout(timeoutEvent);
+            logger.error("The number of tasks each stage had completed when timed out:");
+            Arrays.stream(Stage.values())
+                  .forEach(stage -> logger.error("{}: {}", stage, stage.getCompletedTaskCount()));
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/transport/messages/ResultMessageTest.java b/test/unit/org/apache/cassandra/transport/messages/ResultMessageTest.java
new file mode 100644
index 000000000000..9fc5b3a12a69
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/messages/ResultMessageTest.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport.messages;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.ResultSet;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DynamicCompositeType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.transport.Envelope;
+import org.apache.cassandra.transport.Event;
+import org.apache.cassandra.transport.messages.ResultMessage.Rows;
+import org.apache.cassandra.transport.messages.ResultMessage.SchemaChange;
+import org.apache.cassandra.transport.messages.ResultMessage.SetKeyspace;
+import org.apache.cassandra.utils.MD5Digest;
+
+import static java.util.Arrays.asList;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+
+public class ResultMessageTest
+{
+    private static ByteBuffer bb(String str)
+    {
+        return UTF8Type.instance.decompose(str);
+    }
+
+    private static FieldIdentifier field(String field)
+    {
+        return FieldIdentifier.forQuoted(field);
+    }
+    @Test
+    public void testSchemaChange()
+    {
+        Event.SchemaChange scEvent = new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TABLE, "ks", "test");
+        SchemaChange sc1 = new SchemaChange(scEvent);
+        assertThat(sc1.change.keyspace).isEqualTo("ks");
+        SchemaChange sc2 = overrideKeyspace(sc1);
+        assertThat(sc2.change.keyspace).isEqualTo("ks_123");
+    }
+
+    @Test
+    public void testPrepared()
+    {
+        ColumnSpecification cs1 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("a", true), Int32Type.instance);
+        ColumnSpecification cs2 = new ColumnSpecification("ks2", "cf2", new ColumnIdentifier("b", true), Int32Type.instance);
+        ResultSet.PreparedMetadata preparedMetadata = new ResultSet.PreparedMetadata(Arrays.asList(cs1, cs2), new short[]{ 2, 4, 6 });
+        ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(Arrays.asList(cs1, cs2));
+        ResultMessage.Prepared p1 = new ResultMessage.Prepared(mock(MD5Digest.class), mock(MD5Digest.class), preparedMetadata, resultMetadata);
+        ResultMessage.Prepared p2 = overrideKeyspace(p1);
+        assertThat(p2.metadata.names.stream().map(cs -> cs.ksName)).containsExactly("ks1_123", "ks2_123");
+        assertThat(p2.resultMetadata.names.stream().map(cs -> cs.ksName)).containsExactly("ks1_123", "ks2_123");
+    }
+
+    @Test
+    public void testSetKeyspace()
+    {
+        SetKeyspace sk1 = new SetKeyspace("ks");
+        SetKeyspace sk2 = overrideKeyspace(sk1);
+        assertThat(sk2.keyspace).isEqualTo("ks_123");
+    }
+
+    @Test
+    public void testRows()
+    {
+        FieldIdentifier f1 = field("f1");  // has field position 0
+        FieldIdentifier f2 = field("f2");  // has field position 1
+
+        List<AbstractType<?>> allTypes = new ArrayList<>();
+        UserType udt = new UserType("ks1",
+                                    bb("myType"),
+                                    asList(f1, f2),
+                                    asList(Int32Type.instance, UTF8Type.instance),
+                                    false);
+        allTypes.add(udt);
+        ListType lt = ListType.getInstance(udt, false);
+        allTypes.add(lt);
+        MapType mt = MapType.getInstance(Int32Type.instance, udt, false);
+        allTypes.add(mt);
+        SetType st = SetType.getInstance(udt, false);
+        allTypes.add(st);
+        CompositeType ct = CompositeType.getInstance(Int32Type.instance, UTF8Type.instance);
+        allTypes.add(ct);
+        DynamicCompositeType dct = DynamicCompositeType.getInstance(ImmutableMap.of((byte)8, Int32Type.instance));
+        allTypes.add(dct);
+        TupleType tt = new TupleType(asList(Int32Type.instance, udt));
+        allTypes.add(tt);
+        allTypes.add(Int32Type.instance);
+        AbstractType<?> rt = ReversedType.getInstance(udt);
+        allTypes.add(rt);
+
+        ColumnSpecification cs1 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("a", true), Int32Type.instance);
+        ColumnSpecification cs2 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("b", true), Int32Type.instance);
+        ColumnSpecification cs3 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("c", true), udt);
+        ColumnSpecification cs4 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("d", true), tt);
+        ColumnSpecification cs5 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("e", true), rt);
+        ColumnSpecification cs6 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("f", true), lt);
+        ColumnSpecification cs7 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("g", true), mt);
+        ColumnSpecification cs8 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("h", true), st);
+        ColumnSpecification cs9 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("i", true), ct);
+        ColumnSpecification cs10 = new ColumnSpecification("ks1", "cf1", new ColumnIdentifier("j", true), dct);
+
+        ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(Arrays.asList(cs1, cs2, cs3, cs4, cs5, cs6, cs7, cs8, cs9, cs10));
+        ResultSet rs = new ResultSet(resultMetadata, mock(List.class));
+        Rows r1 = new Rows(rs);
+        checkRows(r1);
+        Rows r2 = overrideKeyspace(r1);
+        checkRows(r2);
+        assertThat(r2.result.metadata.names.stream().map(cs -> cs.ksName)).allMatch(k -> k.equals("ks1_123"));
+        assertThat(r2.result.metadata.getResultMetadataId()).isNotSameAs(r1.result.metadata.getResultMetadataId());
+
+        //Also Test no change path
+        List<AbstractType<?>> newTypes = allTypes.stream().map(t -> t.overrideKeyspace(s -> s)).collect(Collectors.toList());
+        assertThat(allTypes).isEqualTo(newTypes);
+    }
+
+    private <T extends ResultMessage<T>> T overrideKeyspace(ResultMessage<T> rm)
+    {
+        T rm2 = rm.withOverriddenKeyspace(Constants.IDENTITY_STRING_MAPPER);
+        assertThat(rm2).isSameAs(rm);
+        T rm3 = rm2.withOverriddenKeyspace(ks -> ks);
+        assertThat(rm3).isSameAs(rm);
+        rm.setWarnings(mock(List.class));
+        rm.setCustomPayload(mock(Map.class));
+        rm.setSource(mock(Envelope.class));
+        rm.setStreamId(123);
+        T rm4 = rm3.withOverriddenKeyspace(ks -> ks + "_123");
+        assertThat(rm4).isNotSameAs(rm);
+        assertThat(rm4.getWarnings()).isSameAs(rm.getWarnings());
+        assertThat(rm4.getCustomPayload()).isSameAs(rm.getCustomPayload());
+        assertThat(rm4.getSource()).isSameAs(rm.getSource());
+        assertThat(rm4.getStreamId()).isSameAs(rm.getStreamId());
+
+        return rm4;
+    }
+
+    void checkRows(ResultMessage.Rows r)
+    {
+        String ksName = r.result.metadata.names.get(0).ksName;
+        for (ColumnSpecification cf : r.result.metadata.names)
+            checkType(cf.type, ksName);
+    }
+
+    void checkType(AbstractType<?> type, String keyspaceName)
+    {
+        if (type.isUDT())
+        {
+            UserType ut = (UserType) type;
+            assertThat(ut.keyspace).isEqualTo(keyspaceName);
+
+            for (int i = 0; i < ut.size(); i++)
+                checkType(ut.type(i), keyspaceName);
+        }
+        else if (type.isTuple())
+        {
+            TupleType tt = (TupleType) type;
+
+            for (int i = 0; i < tt.size(); i++)
+                checkType(tt.type(i), keyspaceName);
+        }
+        else if (type.isReversed())
+        {
+            checkType(type.unwrap(), keyspaceName);
+        }
+        else if (type.isCollection())
+        {
+            CollectionType<?> ct = (CollectionType<?>) type;
+            checkType(ct.nameComparator(), keyspaceName);
+            checkType(ct.valueComparator(), keyspaceName);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java b/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java
index 257d0c243630..9cb8ecf1d829 100644
--- a/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java
+++ b/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java
@@ -114,13 +114,13 @@ public void sameKeySameCfRowMutations() throws ConfigurationException, InvalidRe
 
         List<PartitionUpdate> mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        Row row = mutatedCFs.get(0).iterator().next();
+        Row row = mutatedCFs.get(0).rowIterator().next();
         assertEquals(bytes("k1v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
         assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
 
         mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        row = mutatedCFs.get(0).iterator().next();
+        row = mutatedCFs.get(0).rowIterator().next();
         assertEquals(bytes("k2v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
         assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
     }
@@ -140,13 +140,13 @@ public void sameKeySameCfPartialRowMutations() throws ConfigurationException, In
 
         List<PartitionUpdate> mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        Row row = mutatedCFs.get(0).iterator().next();
+        Row row = mutatedCFs.get(0).rowIterator().next();
         assertEquals(bytes("k1v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
         assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
 
         mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        row = mutatedCFs.get(0).iterator().next();
+        row = mutatedCFs.get(0).rowIterator().next();
         assertEquals(bytes("k2v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
         assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
     }
@@ -170,13 +170,13 @@ public void sameKeyDifferentCfRowMutations() throws ConfigurationException, Inva
         {
             if (update.metadata().name.equals("cf1"))
             {
-                Row row = update.iterator().next();
+                Row row = update.rowIterator().next();
                 assertEquals(bytes("k1v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
                 assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
             }
             else
             {
-                Row row = update.iterator().next();
+                Row row = update.rowIterator().next();
                 assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
                 assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
             }
@@ -189,13 +189,13 @@ public void sameKeyDifferentCfRowMutations() throws ConfigurationException, Inva
         {
             if (update.metadata().name.equals("cf1"))
             {
-                Row row = update.iterator().next();
+                Row row = update.rowIterator().next();
                 assertEquals(bytes("k2v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
                 assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
             }
             else
             {
-                Row row = update.iterator().next();
+                Row row = update.rowIterator().next();
                 assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
                 assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
             }
@@ -217,25 +217,25 @@ public void sameKeyDifferentKsRowMutations() throws ConfigurationException, Inva
 
         List<PartitionUpdate> mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        Row row = mutatedCFs.get(0).iterator().next();
+        Row row = mutatedCFs.get(0).rowIterator().next();
         assertEquals(bytes("k1v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
         assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
 
         mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        row = mutatedCFs.get(0).iterator().next();
+        row = mutatedCFs.get(0).rowIterator().next();
         assertEquals(bytes("k2v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
         assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
 
         mutatedCFs = new ArrayList<>(tmutations.get(2).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        row = mutatedCFs.get(0).iterator().next();
+        row = mutatedCFs.get(0).rowIterator().next();
         assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
         assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
 
         mutatedCFs = new ArrayList<>(tmutations.get(3).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        row = mutatedCFs.get(0).iterator().next();
+        row = mutatedCFs.get(0).rowIterator().next();
         assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
         assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
     }
@@ -257,13 +257,13 @@ public void differentKeyRowMutations() throws ConfigurationException, InvalidReq
 
         List<PartitionUpdate> mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        Row row = mutatedCFs.get(0).iterator().next();
+        Row row = mutatedCFs.get(0).rowIterator().next();
         assertEquals(bytes("v1"), row.getCell(metadata.getColumn(bytes("c1"))).value());
         assertNull(row.getCell(metadata.getColumn(bytes("c2"))));
 
         mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates());
         assertEquals(1, mutatedCFs.size());
-        row = mutatedCFs.get(0).iterator().next();
+        row = mutatedCFs.get(0).rowIterator().next();
         assertEquals(bytes("trigger"), row.getCell(metadata.getColumn(bytes("c2"))).value());
         assertNull(row.getCell(metadata.getColumn(bytes("c1"))));
     }
diff --git a/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java b/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java
index 62537208ec7b..c03cd9a8f145 100644
--- a/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java
+++ b/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java
@@ -41,6 +41,7 @@
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
@@ -62,6 +63,7 @@
 import org.apache.cassandra.db.marshal.CollectionType;
 import org.apache.cassandra.db.marshal.CompositeType;
 import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.DateRangeType;
 import org.apache.cassandra.db.marshal.DateType;
 import org.apache.cassandra.db.marshal.DecimalType;
 import org.apache.cassandra.db.marshal.DoubleType;
@@ -75,10 +77,15 @@
 import org.apache.cassandra.db.marshal.IntegerType;
 import org.apache.cassandra.db.marshal.LegacyTimeUUIDType;
 import org.apache.cassandra.db.marshal.LexicalUUIDType;
+import org.apache.cassandra.db.marshal.LineStringType;
 import org.apache.cassandra.db.marshal.ListType;
 import org.apache.cassandra.db.marshal.LongType;
 import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.MultiCellCapableType;
+import org.apache.cassandra.db.marshal.NumberType;
 import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
+import org.apache.cassandra.db.marshal.PointType;
+import org.apache.cassandra.db.marshal.PolygonType;
 import org.apache.cassandra.db.marshal.ReversedType;
 import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.marshal.ShortType;
@@ -88,7 +95,6 @@
 import org.apache.cassandra.db.marshal.TimeUUIDType;
 import org.apache.cassandra.db.marshal.TimestampType;
 import org.apache.cassandra.db.marshal.TupleType;
-import org.apache.cassandra.db.marshal.TypeParser;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.db.marshal.UserType;
@@ -99,6 +105,7 @@
 import org.quicktheories.generators.SourceDSL;
 import org.quicktheories.impl.JavaRandom;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static org.apache.cassandra.utils.Generators.IDENTIFIER_GEN;
 import static org.apache.cassandra.utils.Generators.filter;
 
@@ -120,6 +127,13 @@ public final class AbstractTypeGenerators
                                                                                                 .put(FrozenType.class, "Fake class only used during parsing... the parsing creates this and the real type under it, then this gets swapped for the real type")
                                                                                                 .build();
 
+    public static final Set<Class<? extends AbstractType>> DSE_CUSTOM_TYPES = ImmutableSet.<Class<? extends AbstractType>>builder()
+                                                                                      .add(DateRangeType.class)
+                                                                                      .add(LineStringType.class)
+                                                                                      .add(PolygonType.class)
+                                                                                      .add(PointType.class)
+                                                                                      .build();
+
     /**
      * Java does a char by char compare, but Cassandra does a byte ordered compare.  This mostly overlaps but some cases
      * where chars are mixed between 1 and 2 bytes, you can get a different ordering than java's.  One argument in favor
@@ -135,7 +149,7 @@ public static Comparator<String> stringComparator(StringType st)
     }
 
 
-    private static final Map<AbstractType<?>, TypeSupport<?>> PRIMITIVE_TYPE_DATA_GENS =
+    public static final Map<AbstractType<?>, TypeSupport<?>> PRIMITIVE_TYPE_DATA_GENS =
     Stream.of(TypeSupport.of(BooleanType.instance, BOOLEAN_GEN),
               TypeSupport.of(ByteType.instance, SourceDSL.integers().between(0, Byte.MAX_VALUE * 2 + 1).map(Integer::byteValue)),
               TypeSupport.of(ShortType.instance, SourceDSL.integers().between(0, Short.MAX_VALUE * 2 + 1).map(Integer::shortValue)),
@@ -159,7 +173,11 @@ public static Comparator<String> stringComparator(StringType st)
                                                                                               .thenComparingInt(Duration::getDays)
                                                                                               .thenComparingLong(Duration::getNanoseconds)),
               TypeSupport.of(IntegerType.instance, Generators.bigInt()),
-              TypeSupport.of(DecimalType.instance, Generators.bigDecimal())
+              TypeSupport.of(DecimalType.instance, Generators.bigDecimal()),
+              TypeSupport.ofNotComparableValues(DateRangeType.instance, Generators.DATE_RANGE_GEN), // DateRangeType is not comparable
+              TypeSupport.ofNotComparableValues(LineStringType.instance, Generators.LINE_STRING_GEN), // LineStringType is not comparable
+              TypeSupport.ofNotComparableValues(PolygonType.instance, Generators.POLYGON_GEN), // PolygonType is not comparable
+              TypeSupport.ofNotComparableValues(PointType.instance, Generators.POINT_GEN) // PointType is not comparable
     ).collect(Collectors.toMap(t -> t.type, t -> t));
     // NOTE not supporting reversed as CQL doesn't allow nested reversed types
     // when generating part of the clustering key, it would be good to allow reversed types as the top level
@@ -205,6 +223,7 @@ public static Set<Class<? extends AbstractType>> knownTypes()
         Set<Class<? extends AbstractType>> types = PRIMITIVE_TYPE_DATA_GENS.keySet().stream().map(a -> a.getClass()).collect(Collectors.toSet());
         types.addAll(NON_PRIMITIVE_TYPES);
         types.addAll(UNSUPPORTED.keySet());
+        types.addAll(DSE_CUSTOM_TYPES);
         return types;
     }
 
@@ -273,6 +292,7 @@ public static class TypeGenBuilder
         private Function<Integer, Gen<AbstractType<?>>> defaultSetKeyFunc;
         private Predicate<AbstractType<?>> typeFilter = null;
         private Gen<String> udtName = null;
+        private Gen<Boolean> multiCellGen = BOOLEAN_GEN;
 
         public TypeGenBuilder()
         {
@@ -455,7 +475,7 @@ else if (kinds != null)
             }
             else
                 kindGen = SourceDSL.arbitrary().enumValues(TypeKind.class);
-            return buildRecursive(maxDepth, maxDepth, kindGen, BOOLEAN_GEN);
+            return buildRecursive(maxDepth, maxDepth, kindGen, multiCellGen);
         }
 
         private Gen<AbstractType<?>> buildRecursive(int maxDepth, int level, Gen<TypeKind> typeKindGen, Gen<Boolean> multiCellGen)
@@ -494,7 +514,7 @@ private Gen<AbstractType<?>> buildRecursive(int maxDepth, int level, Gen<TypeKin
                     case VECTOR:
                     {
                         Gen<Integer> sizeGen = vectorSizeGen != null ? vectorSizeGen : defaultSizeGen;
-                        return vectorTypeGen(next.get().map(AbstractType::freeze), sizeGen).generate(rnd);
+                        return vectorTypeGen(filter(primitiveGen, NumberType.class::isInstance), sizeGen).generate(rnd);
                     }
                     case COMPOSITE:
                         return compositeTypeGen(compositeElementGen != null ? compositeElementGen : next.get(), compositeSizeGen != null ? compositeSizeGen : defaultSizeGen).generate(rnd);
@@ -592,7 +612,7 @@ public static Gen<DynamicCompositeType> dynamicCompositeGen(Gen<AbstractType<?>>
                 byte alias = aliasGen.generate(rnd);
                 while (aliases.containsKey(alias))
                     alias = aliasGen.generate(rnd);
-                aliases.put(alias, typeGen.generate(rnd).unfreeze());
+                aliases.put(alias, unfreeze(typeGen.generate(rnd)));
             }
             return DynamicCompositeType.getInstance(aliases);
         };
@@ -678,10 +698,10 @@ public static Gen<TupleType> tupleTypeGen(Gen<AbstractType<?>> elementGen, Gen<I
     {
         return rnd -> {
             int numElements = sizeGen.generate(rnd);
-            List<AbstractType<?>> elements = new ArrayList<>(numElements);
+            ImmutableList.Builder<AbstractType<?>>elements = ImmutableList.builderWithExpectedSize(numElements);
             for (int i = 0; i < numElements; i++)
                 elements.add(elementGen.generate(rnd));
-            return new TupleType(elements);
+            return new TupleType(elements.build());
         };
     }
 
@@ -716,7 +736,7 @@ public static Gen<UserType> userTypeGen(Gen<AbstractType<?>> elementGen, Gen<Int
         return rnd -> {
             boolean multiCell = multiCellGen.generate(rnd);
             int numElements = sizeGen.generate(rnd);
-            List<AbstractType<?>> fieldTypes = new ArrayList<>(numElements);
+            ImmutableList.Builder<AbstractType<?>> fieldTypes = ImmutableList.builderWithExpectedSize(numElements);
             LinkedHashSet<FieldIdentifier> fieldNames = new LinkedHashSet<>(numElements);
             String ks = ksGen.generate(rnd);
             String name = nameGen.generate(rnd);
@@ -730,13 +750,13 @@ public static Gen<UserType> userTypeGen(Gen<AbstractType<?>> elementGen, Gen<Int
                 fieldNames.add(fieldName);
 
                 AbstractType<?> element = elementGen.generate(rnd);
-                element = multiCell ? element.freeze() : element.unfreeze();
+                element = multiCell ? element.freeze() : unfreeze(element);
                 // a UDT cannot contain a non-frozen UDT; as defined by CreateType
                 if (element.isUDT())
                     element = element.freeze();
                 fieldTypes.add(element);
             }
-            return new UserType(ks, nameBB, new ArrayList<>(fieldNames), fieldTypes, multiCell);
+            return new UserType(ks, nameBB, ImmutableList.copyOf(fieldNames), fieldTypes.build(), multiCell);
         };
     }
 
@@ -779,7 +799,7 @@ public static <T> TypeSupport<T> getTypeSupport(AbstractType<T> type, Gen<Intege
         TypeSupport<T> support;
         if (gen != null)
         {
-            support = gen;
+            support = gen.withValueDomain(valueDomainGen);
         }
         // might be... complex...
         else if (type instanceof SetType)
@@ -834,9 +854,9 @@ else if (type instanceof MapType)
             // T = Map<A, B> so can not use T here
             MapType<Object, Object> mapType = (MapType<Object, Object>) type;
             // do not use valueDomainGen as map doesn't allow null/empty
-            TypeSupport<Object> keySupport = getTypeSupport(mapType.getKeysType(), sizeGen, null);
+            TypeSupport<Object> keySupport = getTypeSupport(mapType.getKeysType(), sizeGen, valueDomainGen);
             Comparator<Object> keyType = keySupport.valueComparator;
-            TypeSupport<Object> valueSupport = getTypeSupport(mapType.getValuesType(), sizeGen, null);
+            TypeSupport<Object> valueSupport = getTypeSupport(mapType.getValuesType(), sizeGen, valueDomainGen);
             Comparator<Object> valueType = valueSupport.valueComparator;
             Comparator<Map<Object, Object>> comparator = (Map<Object, Object> a, Map<Object, Object> b) -> {
                 List<Object> ak = new ArrayList<>(a.keySet());
@@ -880,7 +900,7 @@ else if (type instanceof TupleType) // includes UserType
         {
             // T is ByteBuffer
             TupleType tupleType = (TupleType) type;
-            List<Comparator<Object>> columns = (List<Comparator<Object>>) (List<?>) tupleType.allTypes().stream().map(AbstractTypeGenerators::comparator).collect(Collectors.toList());
+            List<Comparator<Object>> columns = (List<Comparator<Object>>) (List<?>) tupleType.subTypes.stream().map(AbstractTypeGenerators::comparator).collect(Collectors.toList());
             Comparator<List<Object>> listCompar = listComparator((i, a, b) -> columns.get(i).compare(a, b));
             Comparator<ByteBuffer> comparator = (ByteBuffer a, ByteBuffer b) -> {
                 ByteBuffer[] abb = tupleType.split(ByteBufferAccessor.instance, a);
@@ -911,7 +931,7 @@ else if (type instanceof VectorType)
         else if (type instanceof CompositeType)
         {
             CompositeType ct = (CompositeType) type;
-            List<TypeSupport<Object>> elementSupport = (List<TypeSupport<Object>>) (List<?>) ct.types.stream().map(AbstractTypeGenerators::getTypeSupport).collect(Collectors.toList());
+            List<TypeSupport<Object>> elementSupport = (List<TypeSupport<Object>>) (List<?>) ct.subTypes.stream().map(AbstractTypeGenerators::getTypeSupport).collect(Collectors.toList());
             Serde<ByteBuffer, List<Object>> serde = new Serde<ByteBuffer, List<Object>>()
             {
                 @Override
@@ -931,8 +951,8 @@ public List<Object> to(ByteBuffer buffer)
                 }
             };
             support = (TypeSupport<T>) TypeSupport.of(ct, serde, rnd -> {
-                List<Object> values = new ArrayList<>(ct.types.size());
-                for (int i = 0, size = ct.types.size(); i < size; i++)
+                List<Object> values = new ArrayList<>(ct.subTypes.size());
+                for (int i = 0, size = ct.subTypes.size(); i < size; i++)
                     values.add(elementSupport.get(i).valueGen.generate(rnd));
                 return values;
             }, listComparator((index, a, b) -> elementSupport.get(index).valueComparator.compare(a, b)));
@@ -994,7 +1014,7 @@ else if (type instanceof CounterColumnType)
         }
         else
         {
-            throw new UnsupportedOperationException("No TypeSupport for: " + type);
+            throw new UnsupportedOperationException("Unsupported type: " + type);
         }
         return support.withValueDomain(valueDomainGen);
     }
@@ -1076,8 +1096,7 @@ public static Set<UserType> extractUDTs(AbstractType<?> type)
 
     public static void extractUDTs(AbstractType<?> type, Set<UserType> matches)
     {
-        if (type instanceof ReversedType)
-            type = ((ReversedType) type).baseType;
+        type = type.unwrap();
         if (type instanceof UserType)
             matches.add((UserType) type);
         for (AbstractType<?> t : type.subTypes())
@@ -1227,7 +1246,7 @@ private static final class TupleGen implements Gen<ByteBuffer>
         @SuppressWarnings("unchecked")
         private TupleGen(TupleType tupleType, Gen<Integer> sizeGen, @Nullable Gen<ValueDomain> valueDomainGen)
         {
-            this.elementsSupport = tupleType.allTypes().stream().map(t -> getTypeSupport((AbstractType<Object>) t, sizeGen, valueDomainGen)).collect(Collectors.toList());
+            this.elementsSupport = tupleType.subTypes.stream().map(t -> getTypeSupport((AbstractType<Object>) t, sizeGen, valueDomainGen)).collect(Collectors.toList());
         }
 
         public ByteBuffer generate(RandomnessSource rnd)
@@ -1290,6 +1309,11 @@ public static <A, B> TypeSupport<A> of(AbstractType<A> type, Serde<A, B> serde,
             return of(type, valueGen.map(serde::from), (a, b) -> valueComparator.compare(serde.to(a), serde.to(b)));
         }
 
+        public static <T> TypeSupport<T> ofNotComparableValues(AbstractType<T> type, Gen<T> valueGen)
+        {
+            return new TypeSupport<>(type, valueGen, (x, y) -> type.compare(type.decompose(x), type.decompose(y)));
+        }
+
         /**
          * Generator which composes the values gen with {@link AbstractType#decompose(Object)}
          */
@@ -1359,9 +1383,7 @@ public static boolean allowsEmpty(AbstractType type)
 
     public static AbstractType unwrap(AbstractType type)
     {
-        if (type instanceof ReversedType)
-            return ((ReversedType) type).baseType;
-        return type;
+        return type.unwrap();
     }
 
     public static AbstractType unfreeze(AbstractType t)
@@ -1369,7 +1391,10 @@ public static AbstractType unfreeze(AbstractType t)
         if (t.isMultiCell())
             return t;
 
-        AbstractType<?> unfrozen = TypeParser.parse(t.toString(true));
+        if (!(t instanceof MultiCellCapableType<?>))
+            return t;
+
+        AbstractType<?> unfrozen = t.with(t.subTypes(), true);
         if (unfrozen.isMultiCell())
             return unfrozen;
 
@@ -1400,10 +1425,8 @@ private static <T extends AbstractType<?>> Set<T> frozenAndUnfrozen(T... types)
 
     private static UserType withAddedField(UserType type, String fieldName, AbstractType<?> fieldType)
     {
-        ArrayList<FieldIdentifier> fieldNames = new ArrayList<>(type.fieldNames());
-        fieldNames.add(FieldIdentifier.forUnquoted(fieldName));
-        List<AbstractType<?>> fieldTypes = new ArrayList<>(type.fieldTypes());
-        fieldTypes.add(fieldType);
+        ImmutableList<FieldIdentifier> fieldNames = Collections3.withAppended(type.fieldNames(), FieldIdentifier.forUnquoted(fieldName));
+        ImmutableList<AbstractType<?>> fieldTypes = Collections3.withAppended(type.fieldTypes(), fieldType);
         return new UserType(type.keyspace, type.name, fieldNames, fieldTypes, true);
     }
 
@@ -1411,9 +1434,9 @@ private static Set<TupleType> tupleTypeVariants(UserType type)
     {
         UserType extType = withAddedField(type, "extra", EmptyType.instance);
         return frozenAndUnfrozen(type,
-                                 new TupleType(type.subTypes(), false),
+                                 new TupleType(type.subTypes()),
                                  extType,
-                                 new TupleType(extType.subTypes(), false));
+                                 new TupleType(extType.subTypes()));
     }
 
     private static void forEachUserTypeVariantPair(UserType leftType, UserType rightType, BiConsumer<? super TupleType, ? super TupleType> typePairConsumer)
@@ -1502,12 +1525,12 @@ public static void forEachUserTypesPair(boolean withVariants, BiConsumer<? super
 
         String ks = "ks";
         ByteBuffer t = ByteBufferUtil.bytes("t");
-        List<FieldIdentifier> names = Stream.of("a", "b").map(FieldIdentifier::forUnquoted).collect(Collectors.toUnmodifiableList());
+        ImmutableList<FieldIdentifier> names = Stream.of("a", "b").map(FieldIdentifier::forUnquoted).collect(toImmutableList());
 
         primitiveTypePairs().forEach(elem1Pair -> {
             primitiveTypePairs().forEach(elem2Pair -> {
-                UserType leftType = new UserType(ks, t, names, List.of(elem1Pair.left, elem2Pair.left), true);
-                UserType rightType = new UserType(ks, t, names, List.of(elem1Pair.right, elem2Pair.right), true);
+                UserType leftType = new UserType(ks, t, names, ImmutableList.of(elem1Pair.left, elem2Pair.left), true);
+                UserType rightType = new UserType(ks, t, names, ImmutableList.of(elem1Pair.right, elem2Pair.right), true);
                 if (withVariants)
                     forEachUserTypeVariantPair(leftType, rightType, typePairConsumer);
                 else
diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
index 7f08d6ccf29a..9ede3a11a4e4 100644
--- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
+++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
@@ -1,28 +1,30 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*    http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied.  See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package org.apache.cassandra.utils;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.text.NumberFormat;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.Random;
 import java.util.Set;
 
@@ -30,9 +32,14 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.io.util.DataOutputBuffer;
@@ -43,14 +50,20 @@
 import org.apache.cassandra.utils.IFilter.FilterKey;
 import org.apache.cassandra.utils.KeyGenerator.RandomStringGenerator;
 import org.apache.cassandra.utils.obs.IBitSet;
+import org.apache.cassandra.utils.obs.MemoryLimiter;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.USE_MICROMETER;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 public class BloomFilterTest
 {
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
     public IFilter bfInvHashes;
-
-
+    public MemoryLimiter memoryLimiter;
 
     public static IFilter testSerialize(IFilter f, boolean oldBfFormat) throws IOException
     {
@@ -83,6 +96,10 @@ static void compare(IBitSet bs, IBitSet newbs)
     @Before
     public void setup()
     {
+        // Set a high limit so that normal tests won't reach it, but we don't want Long.MAX_VALUE because
+        // we want to test what happens when we reach it
+        CassandraRelevantProperties.BF_MAX_MEMORY_MB.setLong(128 << 10);
+        memoryLimiter = new MemoryLimiter(128L << 30, "Allocating %s for bloom filter would reach max of %s (current %s)");
         bfInvHashes = FilterFactory.getFilter(10000L, FilterTestHelper.MAX_FAILURE_RATE);
     }
 
@@ -90,6 +107,7 @@ public void setup()
     public void destroy()
     {
         bfInvHashes.close();
+        assertEquals(0, memoryLimiter.memoryAllocated());
     }
 
     @Test(expected = UnsupportedOperationException.class)
@@ -171,13 +189,13 @@ private static void testManyRandom(Iterator<ByteBuffer> keys)
             collisions += (MAX_HASH_COUNT - hashes.size());
             bf.close();
         }
-        Assert.assertTrue("collisions=" + collisions, collisions <= 100);
+        assertTrue("collisions=" + collisions, collisions <= 100);
     }
 
     @Test(expected = UnsupportedOperationException.class)
     public void testOffHeapException()
     {
-        long numKeys = ((long)Integer.MAX_VALUE) * 64L + 1L; // approx 128 Billion
+        long numKeys = ((long) Integer.MAX_VALUE) * 64L + 1L; // approx 128 Billion
         FilterFactory.getFilter(numKeys, 0.01d).close();
     }
 
@@ -209,10 +227,9 @@ public void compareCachedKey()
     }
 
     @Test
-    @Ignore
-    public void testHugeBFSerialization() throws IOException
+    public void testHugeBFSerialization() throws Exception
     {
-        ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1});
+        ByteBuffer test = ByteBuffer.wrap(new byte[]{ 0, 1 });
 
         File file = FileUtils.createDeletableTempFile("bloomFilterTest-", ".dat");
         BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long) Integer.MAX_VALUE / 8) + 1, 0.01d);
@@ -251,4 +268,110 @@ public void testMurmur3FilterHash()
             Assert.assertArrayEquals(expected, actual);
         }
     }
-}
+
+    @Test
+    public void testMaxMemoryExceeded()
+    {
+        long allocSize = 2L * (1 << 20);
+        double fpChance = 0.01;
+        long size;
+
+        try (IFilter filter = FilterFactory.getFilter(allocSize, fpChance, memoryLimiter))
+        {
+            size = filter.offHeapSize();
+        }
+        assertNotEquals(0, size);
+
+        memoryLimiter = new MemoryLimiter(3 * size / 2, "Allocating %s for bloom filter would reach max of %s (current %s)");
+
+        try (IFilter filter = FilterFactory.getFilter(allocSize, fpChance, memoryLimiter))
+        {
+            assertNotNull(filter);
+            assertTrue(filter instanceof BloomFilter);
+
+            long memBefore = memoryLimiter.memoryAllocated();
+
+            try (IFilter blankFilter = FilterFactory.getFilter(allocSize, fpChance, memoryLimiter))
+            {
+                assertNotNull(blankFilter);
+                assertEquals(blankFilter, FilterFactory.AlwaysPresent);
+                assertEquals(1, FilterFactory.metrics.oomErrors());
+
+                assertEquals(memBefore, memoryLimiter.memoryAllocated());
+            }
+        }
+    }
+
+    @Test
+    public void testMaxMemoryExceededOnDeserialize() throws IOException
+    {
+        long allocSize = 2L * (1 << 20);
+        double fpChance = 0.01;
+        long size;
+
+        DataOutputBuffer out = new DataOutputBuffer();
+        try (IFilter filter = FilterFactory.getFilter(allocSize, fpChance, memoryLimiter))
+        {
+            size = filter.offHeapSize();
+            BloomFilterSerializer.forVersion(false).serialize((BloomFilter) filter, out);
+        }
+        assertNotEquals(0, size);
+
+        memoryLimiter = new MemoryLimiter(3 * size / 2, "Allocating %s for bloom filter would reach max of %s (current %s)");
+
+        try (IFilter filter = FilterFactory.getFilter(allocSize, fpChance, memoryLimiter))
+        {
+            assertNotNull(filter);
+            assertTrue(filter instanceof BloomFilter);
+
+            long memBefore = memoryLimiter.memoryAllocated();
+
+            expectedException.expect(RuntimeException.class);
+            expectedException.expectMessage("Out of native memory occured, You can avoid it by increasing the system ram space or by increasing bloom_filter_fp_chance.");
+            ByteArrayInputStream in = new ByteArrayInputStream(out.getData(), 0, out.getLength());
+            BloomFilterSerializer.forVersion(false).deserialize(Util.DataInputStreamPlusImpl.wrap(in), memoryLimiter);
+        }
+    }
+
+    @Test
+    public void testBloomFilterMetrics()
+    {
+        FilterFactory.FilterFactoryMetrics metrics = FilterFactory.FilterFactoryMetrics.create();
+        assertTrue(metrics instanceof FilterFactory.FilterFactoryCodahaleMetrics);
+        long prev = metrics.oomErrors();
+        metrics.incrementOOMError();
+        assertEquals(prev + 1, metrics.oomErrors());
+        USE_MICROMETER.setBoolean(true);
+        metrics = FilterFactory.FilterFactoryMetrics.create();
+        assertTrue(metrics instanceof FilterFactory.FilterFactoryMicormeterMetrics);
+        ((FilterFactory.FilterFactoryMicormeterMetrics) metrics).register(new SimpleMeterRegistry(), Tags.of("k", "v"));
+        metrics.incrementOOMError();
+        assertEquals(1, metrics.oomErrors());
+    }
+
+    @Test
+    @Ignore // this is a test that can be used to print out the sizes of BFs
+    public void testBloomFilterSize()
+    {
+        int[] nks = new int[]{
+        100_000, 500_000,
+        1_000_000, 5_000_000,
+        10_000_000, 50_000_000,
+        100_000_000, 500_000_000 };
+
+        //double[] fps = new double[] { 0.01, 0.05, 0.1, 0.2, 0.25 };
+        double[] fps = new double[]{ 0.01, 0.1 };
+
+        for (int nk : nks)
+        {
+            for (double fp : fps)
+            {
+                IFilter filter = FilterFactory.getFilter(nk, fp);
+                System.out.println(String.format("%s keys %s FP chance => %s",
+                                                 NumberFormat.getNumberInstance(Locale.US).format(nk),
+                                                 NumberFormat.getNumberInstance(Locale.US).format(fp),
+                                                 FBUtilities.prettyPrintMemory(filter.serializedSize(false))));
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/CustomClusterVersionProvider.java b/test/unit/org/apache/cassandra/utils/CustomClusterVersionProvider.java
new file mode 100644
index 000000000000..7778e411c2e5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/CustomClusterVersionProvider.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.gms.IClusterVersionProvider;
+
+public class CustomClusterVersionProvider implements IClusterVersionProvider
+{
+    public static void set()
+    {
+        CassandraRelevantProperties.CLUSTER_VERSION_PROVIDER_CLASS_NAME.setString(CustomClusterVersionProvider.class.getName());
+    }
+
+    public final static CustomClusterVersionProvider instance = new CustomClusterVersionProvider();
+
+    public volatile CassandraVersion version = new CassandraVersion(FBUtilities.getReleaseVersionString());
+
+    public volatile boolean upgradeInProgress = false;
+
+    public volatile Runnable onReset = null;
+
+    private volatile boolean wasUsed = false;
+
+    @Override
+    public CassandraVersion getMinClusterVersion()
+    {
+        wasUsed = true;
+        return version;
+    }
+
+    @Override
+    public void reset()
+    {
+        wasUsed = true;
+        if (onReset != null)
+            onReset.run();
+    }
+
+    @Override
+    public boolean isUpgradeInProgress()
+    {
+        wasUsed = true;
+        return upgradeInProgress;
+    }
+
+    public void assertUsed()
+    {
+        assert wasUsed;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/DSEEstimatedHistogramTest.java b/test/unit/org/apache/cassandra/utils/DSEEstimatedHistogramTest.java
new file mode 100644
index 000000000000..8cadee816d26
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/DSEEstimatedHistogramTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+
+/**
+ * Test for any estimated histogram that uses DSE compatible boundaries. This needs to be a separate test class because
+ * the property value is inlined as static final in the EstimatedHistogram class.
+ */
+public class DSEEstimatedHistogramTest
+{
+    @BeforeClass
+    public static void setup()
+    {
+        CassandraRelevantProperties.USE_DSE_COMPATIBLE_HISTOGRAM_BOUNDARIES.setBoolean(true);
+    }
+
+    @Test
+    public void testDSEBoundaries()
+    {
+            // these boundaries were computed in DSE
+            long[] dseBoundaries = new long[]{ 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40,
+                                               48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792,
+                                               2048, 2560, 3072, 3584, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384, 20480, 24576, 28672, 32768,
+                                               40960, 49152, 57344, 65536, 81920, 98304, 114688, 131072, 163840, 196608, 229376, 262144, 327680, 393216,
+                                               458752, 524288, 655360, 786432, 917504, 1048576, 1310720, 1572864, 1835008, 2097152, 2621440, 3145728, 3670016,
+                                               4194304, 5242880, 6291456, 7340032, 8388608, 10485760, 12582912, 14680064, 16777216, 20971520, 25165824,
+                                               29360128, 33554432, 41943040, 50331648, 58720256, 67108864, 83886080, 100663296, 117440512, 134217728,
+                                               167772160, 201326592, 234881024, 268435456, 335544320, 402653184, 469762048, 536870912, 671088640, 805306368,
+                                               939524096, 1073741824, 1342177280, 1610612736, 1879048192, 2147483648L, 2684354560L, 3221225472L, 3758096384L,
+                                               4294967296L, 5368709120L, 6442450944L, 7516192768L, 8589934592L, 10737418240L, 12884901888L, 15032385536L, 17179869184L,
+                                               21474836480L, 25769803776L, 30064771072L, 34359738368L, 42949672960L, 51539607552L, 60129542144L, 68719476736L,
+                                               85899345920L, 103079215104L, 120259084288L, 137438953472L, 171798691840L, 206158430208L, 240518168576L, 274877906944L,
+                                               343597383680L, 412316860416L, 481036337152L, 549755813888L, 687194767360L, 824633720832L, 962072674304L, 1099511627776L,
+                                               1374389534720L, 1649267441664L, 1924145348608L, 2199023255552L, 2748779069440L, 3298534883328L, 3848290697216L,
+                                               4398046511104L, 5497558138880L, 6597069766656L, 7696581394432L, 8796093022208L, 10995116277760L, 13194139533312L,
+                                               15393162788864L, 17592186044416L, 21990232555520L, 26388279066624L, 30786325577728L, 35184372088832L, 43980465111040L,
+                                               52776558133248L, 61572651155456L, 70368744177664L, 87960930222080L, 105553116266496L, 123145302310912L,
+                                               140737488355328L, 175921860444160L };
+
+            // the code below is O(n^2) so that we don't need to assume that boundaries are independent
+            // of the histogram size; this is not a problem since the number of boundaries is small
+            for (int size = 1; size <= dseBoundaries.length; size++)
+            {
+                EstimatedHistogram histogram = new EstimatedHistogram(size);
+                // compute subarray of dseBoundaries of size `size`
+                long[] subarray = new long[size];
+                System.arraycopy(dseBoundaries, 0, subarray, 0, size);
+                Assert.assertArrayEquals(subarray, histogram.getBucketOffsets());
+            }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/ExpMovingAverageTest.java b/test/unit/org/apache/cassandra/utils/ExpMovingAverageTest.java
new file mode 100644
index 000000000000..0426f5993ebe
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/ExpMovingAverageTest.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class ExpMovingAverageTest
+{
+    private static final double epsilon = 0.0001;
+
+    @Test
+    public void testUpdates()
+    {
+        ExpMovingAverage average = new ExpMovingAverage(0.5);
+        assertNotNull(average.toString());
+
+        average.update(10);
+
+        assertEquals(10, average.get(), epsilon);
+
+        average.update(11);
+        assertEquals(10.5, average.get(), epsilon);
+
+        average.update(12);
+        assertEquals(11.25, average.get(), epsilon);
+
+        average.update(11.75);
+
+        assertEquals(11.5, average.get(), epsilon);
+    }
+
+    @Test
+    public void testDecay10()
+    {
+        testDecay(10, 0.01, ExpMovingAverage.decayBy10());
+    }
+
+    @Test
+    public void testDecay100()
+    {
+        testDecay(100, 0.01, ExpMovingAverage.decayBy100());
+    }
+
+    @Test
+    public void testDecay1000()
+    {
+        testDecay(1000, 0.01, ExpMovingAverage.decayBy1000());
+    }
+
+    @Test
+    public void testDecay()
+    {
+        double ratio = 0.1;
+        int count = 50;
+        testDecay(count, ratio, ExpMovingAverage.withDecay(ratio, count));
+    }
+
+    public void testDecay(int count, double expectedBelow, MovingAverage average)
+    {
+        average.update(1.0);    // on initialization average takes the exact value
+        for (int i = 0; i < count; ++i)
+            average.update(0.0);
+
+        assertTrue(average.get() <= expectedBelow + epsilon);
+        assertTrue(average.get() >= expectedBelow / 2 - epsilon);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java
index fc027954bdb2..ab5d559c450c 100644
--- a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java
+++ b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java
@@ -37,6 +37,7 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
 import com.vdurmont.semver4j.Semver;
 import org.junit.Assert;
@@ -63,6 +64,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class FBUtilitiesTest
@@ -272,6 +274,15 @@ public void testCamelToSnake()
             throw error;
     }
 
+    @Test
+    public void testPrettyPrintMemoryNoModifier()
+    {
+        for (long v : ImmutableList.of(0, 1, 8, 15, 60, 125, 980, 1000, 1023, -1, -15, -1023))
+        {
+            assertEquals(v + "B", FBUtilities.prettyPrintMemory(v));
+        }
+    }
+
     @Test
     public void testPrettyPrintAndParse()
     {
@@ -399,4 +410,10 @@ public void testGetKernelVersion()
         assertThat(kernelVersion).isGreaterThan(new Semver("0.0.0", Semver.SemverType.LOOSE));
         assertThat(kernelVersion).isLessThan(new Semver("100.0.0", Semver.SemverType.LOOSE));
     }
+    
+    public void testDebug()
+    {
+        String trace = FBUtilities.Debug.getStackTrace();
+        assertTrue(trace.contains("testDebug"));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/utils/Generators.java b/test/unit/org/apache/cassandra/utils/Generators.java
index 9675d5525ee0..7f22a3e32ee9 100644
--- a/test/unit/org/apache/cassandra/utils/Generators.java
+++ b/test/unit/org/apache/cassandra/utils/Generators.java
@@ -24,6 +24,7 @@
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.sql.Timestamp;
+import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.Date;
@@ -32,6 +33,7 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import java.util.function.IntFunction;
 import java.util.function.Predicate;
 
 import com.google.common.collect.Range;
@@ -39,8 +41,18 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.esri.core.geometry.MultiPath;
+import com.esri.core.geometry.Polyline;
+import com.esri.core.geometry.ogc.OGCLineString;
+import com.esri.core.geometry.ogc.OGCPolygon;
+import org.apache.cassandra.db.marshal.datetime.DateRange;
+import org.apache.cassandra.db.marshal.geometry.LineString;
+import org.apache.cassandra.db.marshal.geometry.OgcGeometry;
+import org.apache.cassandra.db.marshal.geometry.Point;
+import org.apache.cassandra.db.marshal.geometry.Polygon;
 import org.quicktheories.core.Gen;
 import org.quicktheories.core.RandomnessSource;
+import org.quicktheories.generators.Generate;
 import org.quicktheories.generators.SourceDSL;
 import org.quicktheories.impl.Constraint;
 import org.quicktheories.impl.JavaRandom;
@@ -190,6 +202,8 @@ public byte[] generate(RandomnessSource rnd)
     public static final Gen<InetAddress> INET_ADDRESS_GEN = INET_4_ADDRESS_GEN.mix(INET_6_ADDRESS_GEN);
     public static final Gen<InetAddress> INET_ADDRESS_UNRESOLVED_GEN = INET_4_ADDRESS_UNRESOLVED_GEN.mix(INET_6_ADDRESS_UNRESOLVED_GEN);
 
+    public static final Gen<DateRange.DateRangeBound.Precision> PRECISION_GEN = SourceDSL.arbitrary().enumValues(DateRange.DateRangeBound.Precision.class);
+
     /**
      * Implements a valid utf-8 generator.
      *
@@ -204,6 +218,10 @@ public byte[] generate(RandomnessSource rnd)
     public static final Gen<Long> TIMESTAMP_NANOS;
     public static final Gen<Long> SMALL_TIME_SPAN_NANOS; // generate nanos in [0, 10] seconds
     public static final Gen<Long> TINY_TIME_SPAN_NANOS; // generate nanos in [0, 1) seconds
+    public static final Gen<DateRange> DATE_RANGE_GEN;
+    public static final Gen<Point> POINT_GEN;
+    public static final Gen<LineString> LINE_STRING_GEN;
+    public static final Gen<Polygon> POLYGON_GEN;
 
     static
     {
@@ -225,6 +243,10 @@ public byte[] generate(RandomnessSource rnd)
         TIMESTAMP_NANOS = TIMESTAMP_GEN.map(t -> TimeUnit.MILLISECONDS.toNanos(t.getTime()) + t.getNanos());
         SMALL_TIME_SPAN_NANOS = rnd -> rnd.next(smallTimeSpanNanosConstraint);
         TINY_TIME_SPAN_NANOS = rnd -> rnd.next(nanosInSecondConstraint);
+        DATE_RANGE_GEN = new DateRangeGen();
+        POINT_GEN = SourceDSL.doubles().between(-100, 100).zip(SourceDSL.doubles().between(-100, 100), Point::new);
+        LINE_STRING_GEN = new LineStringGen();
+        POLYGON_GEN = new PolygonGen();
     }
 
     private Generators()
@@ -460,6 +482,118 @@ private static boolean isDash(char c)
         }
     }
 
+    public static class DateRangeGen implements Gen<DateRange>
+    {
+        private final Gen<Boolean> lowerUnboundedGen;
+        private final Gen<Date> lowerGen;
+        private final Gen<DateRange.DateRangeBound.Precision> lowerBoundPrecisionGen;
+        private final Gen<Boolean> upperUnboundedGen;
+        private final Gen<Long> durationGen;
+        private final Gen<DateRange.DateRangeBound.Precision> upperBoundPrecisionGen;
+
+        public DateRangeGen(Gen<Boolean> lowerUnboundedGen,
+                            Gen<Date> lowerGen,
+                            Gen<DateRange.DateRangeBound.Precision> lowerBoundPrecisionGen,
+                            Gen<Boolean> upperUnboundedGen,
+                            Gen<Long> durationGen,
+                            Gen<DateRange.DateRangeBound.Precision> upperBoundPrecisionGen)
+        {
+            this.lowerUnboundedGen = lowerUnboundedGen;
+            this.lowerGen = lowerGen;
+            this.lowerBoundPrecisionGen = lowerBoundPrecisionGen;
+            this.upperUnboundedGen = upperUnboundedGen;
+            this.durationGen = durationGen;
+            this.upperBoundPrecisionGen = upperBoundPrecisionGen;
+        }
+
+        public DateRangeGen()
+        {
+            lowerUnboundedGen = SourceDSL.booleans().all();
+            lowerGen = DATE_GEN;
+            lowerBoundPrecisionGen = PRECISION_GEN;
+            upperUnboundedGen = SourceDSL.booleans().all();
+            durationGen = Generate.longRange(0, TimeUnit.DAYS.toMillis(1000));
+            upperBoundPrecisionGen = PRECISION_GEN;
+        }
+
+        @Override
+        public DateRange generate(RandomnessSource randomnessSource)
+        {
+            DateRange.DateRangeBuilder builder = DateRange.DateRangeBuilder.dateRange();
+
+            Long lowerBoundMillis = lowerGen.map(Date::getTime).generate(randomnessSource);
+            if (lowerUnboundedGen.generate(randomnessSource))
+                builder.withUnboundedLowerBound();
+            else
+                builder.withLowerBound(Instant.ofEpochMilli(lowerBoundMillis), lowerBoundPrecisionGen.generate(randomnessSource));
+
+            if (upperUnboundedGen.generate(randomnessSource))
+                builder.withUnboundedUpperBound();
+            else
+                builder.withUpperBound(Instant.ofEpochMilli(lowerBoundMillis + durationGen.generate(randomnessSource)), upperBoundPrecisionGen.generate(randomnessSource));
+
+            return builder.build();
+        }
+    }
+
+    private static void generatePath(MultiPath path, Gen<Integer> numPointsGen, Gen<Double> radiusGen, Gen<Point> centerPointGen, RandomnessSource randomnessSource)
+    {
+        int numPoints = numPointsGen.generate(randomnessSource);
+        Point centerPoint = centerPointGen.generate(randomnessSource);
+        double radius = radiusGen.generate(randomnessSource);
+
+        IntFunction<com.esri.core.geometry.Point> pointSupplier = i -> {
+            double x = Math.cos(i * 2 * Math.PI / numPoints) * radius + centerPoint.getOgcPoint().X();
+            double y = Math.sin(i * 2 * Math.PI / numPoints) * radius + centerPoint.getOgcPoint().Y();
+            return new com.esri.core.geometry.Point(x, y);
+        };
+
+        com.esri.core.geometry.Point point = pointSupplier.apply(0);
+        path.startPath(point);
+
+        for (int i = numPoints - 1; i > 0; i--) // must be clockwise
+        {
+            point = pointSupplier.apply(i);
+            path.lineTo(point);
+        }
+    }
+
+    public static final class LineStringGen implements Gen<LineString>
+    {
+        @Override
+        public LineString generate(RandomnessSource randomnessSource)
+        {
+            Polyline polyLine = new Polyline();
+            generatePath(polyLine, SourceDSL.integers().between(3, 5), SourceDSL.doubles().between(10, 100), POINT_GEN, randomnessSource);
+            try
+            {
+                return new LineString(new OGCLineString(polyLine, 0, OgcGeometry.SPATIAL_REFERENCE_4326));
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException("Failed to create a polyline: " + polyLine, e);
+            }
+        }
+    }
+
+    public static final class PolygonGen implements Gen<Polygon>
+    {
+        @Override
+        public Polygon generate(RandomnessSource randomnessSource)
+        {
+            com.esri.core.geometry.Polygon polygon = new com.esri.core.geometry.Polygon();
+            generatePath(polygon, SourceDSL.integers().between(3, 5), SourceDSL.doubles().between(10, 100), POINT_GEN, randomnessSource);
+            try
+            {
+                return new Polygon(new OGCPolygon(polygon, OgcGeometry.SPATIAL_REFERENCE_4326));
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException("Failed to create a polygon: " + polygon, e);
+            }
+        }
+    }
+
     private static final class LazySharedBlob
     {
         private static final byte[] SHARED_BYTES;
diff --git a/test/unit/org/apache/cassandra/utils/ImmutableUtilsTest.java b/test/unit/org/apache/cassandra/utils/ImmutableUtilsTest.java
new file mode 100644
index 000000000000..6214cc485e77
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/ImmutableUtilsTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+public class ImmutableUtilsTest
+{
+    @Test
+    public void without()
+    {
+        ImmutableMap<String, Integer> m0 = ImmutableMap.of("a", 1, "b", 2);
+        ImmutableMap<String, Integer> m1 = ImmutableUtils.without(m0, "b");
+        ImmutableMap<String, Integer> m2 = ImmutableUtils.without(m0, "c");
+
+        assertThat(m1).containsEntry("a", 1).doesNotContainKey("b");
+        assertThat(m2).containsEntry("a", 1).containsEntry("b", 2).doesNotContainKey("c");
+        assertThat(m2).isEqualTo(m0);
+    }
+
+    @Test
+    public void withAdded()
+    {
+        ImmutableMap<String, Integer> m0 = ImmutableMap.of("a", 1, "b", 2);
+        ImmutableMap<String, Integer> m1 = ImmutableUtils.withAddedOrUpdated(m0, "b", 3);
+        ImmutableMap<String, Integer> m2 = ImmutableUtils.withAddedOrUpdated(m0, "c", 3);
+
+        assertThat(m1).containsEntry("a", 1).containsEntry("b", 3);
+        assertThat(m2).containsEntry("a", 1).containsEntry("b", 2).containsEntry("c", 3);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java b/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java
index 3a3415e4d4ff..9f398a472e9b 100644
--- a/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java
+++ b/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.net.SocketException;
 import java.util.Arrays;
+import java.util.function.Consumer;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -35,8 +36,11 @@
 import org.apache.cassandra.service.CassandraDaemon;
 import org.apache.cassandra.service.DefaultFSErrorHandler;
 import org.apache.cassandra.service.StorageService;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 import static java.util.Arrays.asList;
+import static org.apache.cassandra.utils.JVMStabilityInspector.getGlobalErrorHandler;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertSame;
@@ -55,11 +59,14 @@ public static void initDD()
     public void testKill() throws Exception
     {
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
 
         Config.DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
         Config.CommitFailurePolicy oldCommitPolicy = DatabaseDescriptor.getCommitFailurePolicy();
         FileUtils.setFSErrorHandler(new DefaultFSErrorHandler());
+
+        Consumer<Throwable> diskErrorHandler = Mockito.mock(Consumer.class);
+        JVMStabilityInspector.setDiskErrorHandler(diskErrorHandler);
         try
         {
             CassandraDaemon daemon = new CassandraDaemon();
@@ -78,24 +85,32 @@ public void testKill() throws Exception
 
                     DatabaseDescriptor.setDiskFailurePolicy(Config.DiskFailurePolicy.die);
                     killerForTests.reset();
+                    Mockito.reset(diskErrorHandler);
                     JVMStabilityInspector.inspectThrowable(new FSReadError(new IOException(), "blah"));
                     assertTrue(killerForTests.wasKilled());
+                    Mockito.verify(diskErrorHandler).accept(ArgumentMatchers.any(FSReadError.class));
 
                     killerForTests.reset();
+                    Mockito.reset(diskErrorHandler);
                     JVMStabilityInspector.inspectThrowable(new FSWriteError(new IOException(), "blah"));
                     assertTrue(killerForTests.wasKilled());
+                    Mockito.verify(diskErrorHandler).accept(ArgumentMatchers.any(FSWriteError.class));
 
                     killerForTests.reset();
+                    Mockito.reset(diskErrorHandler);
                     JVMStabilityInspector.inspectThrowable(new CorruptSSTableException(new IOException(), "blah"));
                     assertTrue(killerForTests.wasKilled());
+                    Mockito.verify(diskErrorHandler).accept(ArgumentMatchers.any(CorruptSSTableException.class));
 
                     killerForTests.reset();
+                    Mockito.reset(diskErrorHandler);
                     JVMStabilityInspector.inspectThrowable(new RuntimeException(new CorruptSSTableException(new IOException(), "blah")));
                     assertTrue(killerForTests.wasKilled());
+                    Mockito.verify(diskErrorHandler).accept(ArgumentMatchers.any(CorruptSSTableException.class));
 
                     DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
                     killerForTests.reset();
-                    JVMStabilityInspector.inspectCommitLogThrowable(new Throwable());
+                    JVMStabilityInspector.inspectCommitLogThrowable("testKill", new Throwable());
                     assertTrue(killerForTests.wasKilled());
 
                     killerForTests.reset();
@@ -163,7 +178,7 @@ public void testForceHeapSpaceOomExclude()
     public void fileHandleTest()
     {
         KillerForTests killerForTests = new KillerForTests();
-        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        JVMKiller originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
 
         try
         {
@@ -188,7 +203,7 @@ public void fileHandleTest()
             assertTrue(killerForTests.wasKilled());
 
             killerForTests.reset();
-            JVMStabilityInspector.inspectCommitLogThrowable(new FileNotFoundException("Too many open files"));
+            JVMStabilityInspector.inspectCommitLogThrowable("fileHandleTest", new FileNotFoundException("Too many open files"));
             assertTrue(killerForTests.wasKilled());
 
         }
@@ -197,4 +212,53 @@ public void fileHandleTest()
             JVMStabilityInspector.replaceKiller(originalKiller);
         }
     }
+
+    @Test
+    public void testShutdownHookRemoved()
+    {
+        class TestShutdownHook {
+            boolean shutdownHookRemoved = false;
+            
+            private void onHookRemoved()
+            {
+                shutdownHookRemoved = true;
+            }
+
+            private void shutdownHook()
+            {
+            }
+        }
+        
+        TestShutdownHook testShutdownHook = new TestShutdownHook();
+        JVMStabilityInspector.registerShutdownHook(new Thread(() -> testShutdownHook.shutdownHook()), () -> testShutdownHook.onHookRemoved());
+        JVMStabilityInspector.removeShutdownHooks();
+        assertTrue(testShutdownHook.shutdownHookRemoved);
+    }
+
+    @Test
+    public void testSettingCustomGlobalHandler()
+    {
+        Consumer<Throwable> globalHandler = Mockito.mock(Consumer.class);
+        Consumer<Throwable> previous = getGlobalErrorHandler();
+        JVMStabilityInspector.setGlobalErrorHandler(globalHandler);
+
+        Throwable causeThrowable = new Throwable("cause");
+        Throwable topThrowable = new Throwable("hello", causeThrowable);
+        Throwable suppressedThrowable = new Throwable("suppressed");
+        topThrowable.addSuppressed(suppressedThrowable);
+
+        JVMStabilityInspector.inspectThrowable(topThrowable);
+
+        Mockito.verify(globalHandler).accept(Mockito.eq(topThrowable));
+        Mockito.verify(globalHandler).accept(Mockito.eq(suppressedThrowable));
+        Mockito.verify(globalHandler).accept(Mockito.eq(causeThrowable));
+        
+        JVMStabilityInspector.setGlobalErrorHandler(previous);
+    }
+
+    @Test
+    public void testInspectingNull()
+    {
+        JVMStabilityInspector.inspectThrowable(null);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/utils/KillerForTests.java b/test/unit/org/apache/cassandra/utils/KillerForTests.java
index b6c48d52e819..d9682ebadf5a 100644
--- a/test/unit/org/apache/cassandra/utils/KillerForTests.java
+++ b/test/unit/org/apache/cassandra/utils/KillerForTests.java
@@ -40,7 +40,7 @@ public KillerForTests(boolean expectFailure)
     }
 
     @Override
-    protected void killCurrentJVM(Throwable t, boolean quiet)
+    public void killJVM(Throwable t, boolean quiet)
     {
         if (!expected)
             Assert.fail("Saw JVM Kill but did not expect it.");
diff --git a/test/unit/org/apache/cassandra/utils/MergeIteratorComparisonTest.java b/test/unit/org/apache/cassandra/utils/MergeIteratorComparisonTest.java
index 1b52fb509b24..e10ea9b44ad5 100644
--- a/test/unit/org/apache/cassandra/utils/MergeIteratorComparisonTest.java
+++ b/test/unit/org/apache/cassandra/utils/MergeIteratorComparisonTest.java
@@ -19,24 +19,30 @@
 package org.apache.cassandra.utils;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
 
 import com.google.common.base.Function;
 import com.google.common.base.Objects;
-
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Ordering;
 import com.google.common.collect.Sets;
-
 import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.TimeUUIDType;
 import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.utils.MergeIterator.Reducer;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
@@ -79,7 +85,7 @@ public Integer next()
         }.result;
         testMergeIterator(reducer, lists);
     }
-    
+
     @Test
     public void testNonOverlapInts()
     {
@@ -188,7 +194,7 @@ public String next()
         }.result;
         testMergeIterator(reducer, lists);
     }
-    
+
     @Test
     public void testNonOverlapStrings()
     {
@@ -289,7 +295,7 @@ public ByteBuffer next()
         testMergeIterator(reducer, lists, type);
     }
 
-    
+
     @Test
     public void testSets()
     {
@@ -468,8 +474,8 @@ public <T extends Comparable<T>> void testMergeIterator(Reducer<T, ?> reducer, L
     public <T> void testMergeIterator(Reducer<T, ?> reducer, List<List<T>> lists, Comparator<T> comparator)
     {
         {
-            IMergeIterator<T,?> tested = MergeIterator.get(closeableIterators(lists), comparator, reducer);
-            IMergeIterator<T,?> base = new MergeIteratorPQ<>(closeableIterators(lists), comparator, reducer);
+            CloseableIterator<?> tested = MergeIterator.getCloseable(closeableIterators(lists), comparator, reducer);
+            CloseableIterator<?> base = new MergeIteratorPQ<>(closeableIterators(lists), comparator, reducer);
             // If test fails, try the version below for improved reporting:
             Object[] basearr = Iterators.toArray(base, Object.class);
             Assert.assertArrayEquals(basearr, Iterators.toArray(tested, Object.class));
@@ -482,13 +488,13 @@ public <T> void testMergeIterator(Reducer<T, ?> reducer, List<List<T>> lists, Co
         cmp = new CountingComparator<>(comparator); cmpb = new CountingComparator<>(comparator);
         System.out.println();
         for (int i=0; i<10; ++i) {
-            benchmarkIterator(MergeIterator.get(closeableIterators(lists), cmp, reducer), cmp);
+            benchmarkIterator(MergeIterator.getCloseable(closeableIterators(lists), cmp, reducer), cmp);
             benchmarkIterator(new MergeIteratorPQ<>(closeableIterators(lists), cmpb, reducer), cmpb);
         }
         System.out.format("MI: %.2f\n", cmp.count / (double) cmpb.count);
     }
-    
-    public <T> void benchmarkIterator(IMergeIterator<T, ?> it, CountingComparator<T> comparator)
+
+    public <T> void benchmarkIterator(CloseableIterator<?> it, CountingComparator<T> comparator)
     {
         System.out.format("Testing %30s... ", it.getClass().getSimpleName());
         long time = System.currentTimeMillis();
@@ -519,7 +525,7 @@ public CloseableIterator<T> apply(List<T> arg)
     static class Counted<T> {
         T item;
         int count;
-        
+
         Counted(T item) {
             this.item = item;
             count = 0;
@@ -539,7 +545,7 @@ public String toString()
             return item.toString() + "x" + count;
         }
     }
-    
+
     static class Counter<T> extends Reducer<T, Counted<T>> {
         Counted<T> current = null;
         boolean read = true;
@@ -554,7 +560,7 @@ public void reduce(int idx, T next)
         }
 
         @Override
-        protected void onKeyChange()
+        public void onKeyChange()
         {
             assert read;
             current = null;
@@ -562,21 +568,21 @@ protected void onKeyChange()
         }
 
         @Override
-        protected Counted<T> getReduced()
+        public Counted<T> getReduced()
         {
             assert current != null;
             read = true;
             return current;
         }
     }
-    
+
     static class KeyedSet<K extends Comparable<? super K>, V> extends Pair<K, Set<V>> implements Comparable<KeyedSet<K, V>>
     {
         protected KeyedSet(K left, V right)
         {
             super(left, ImmutableSet.of(right));
         }
-        
+
         protected KeyedSet(K left, Collection<V> right)
         {
             super(left, Sets.newHashSet(right));
@@ -588,7 +594,7 @@ public int compareTo(KeyedSet<K, V> o)
             return left.compareTo(o.left);
         }
     }
-    
+
     static class Union<K extends Comparable<K>, V> extends Reducer<KeyedSet<K, V>, KeyedSet<K, V>> {
         KeyedSet<K, V> current = null;
         boolean read = true;
@@ -605,7 +611,7 @@ public void reduce(int idx, KeyedSet<K, V> next)
         }
 
         @Override
-        protected void onKeyChange()
+        public void onKeyChange()
         {
             assert read;
             current = null;
@@ -613,14 +619,14 @@ protected void onKeyChange()
         }
 
         @Override
-        protected KeyedSet<K, V> getReduced()
+        public KeyedSet<K, V> getReduced()
         {
             assert current != null;
             read = true;
             return current;
         }
     }
-    
+
     // closeable list iterator
     public static class CLI<E> extends AbstractIterator<E> implements CloseableIterator<E>
     {
@@ -645,8 +651,11 @@ public void close()
     }
 
     // Old MergeIterator implementation for comparison.
-    public class MergeIteratorPQ<In,Out> extends MergeIterator<In,Out> implements IMergeIterator<In, Out>
+    public class MergeIteratorPQ<In,Out> extends AbstractIterator<Out>
     {
+        protected final Reducer<In,Out> reducer;
+        protected final List<? extends Iterator<In>> iterators;
+
         // a queue for return: all candidates must be open and have at least one item
         protected final PriorityQueue<CandidatePQ<In>> queue;
         // a stack of the last consumed candidates, so that we can lazily call 'advance()'
@@ -655,7 +664,8 @@ public class MergeIteratorPQ<In,Out> extends MergeIterator<In,Out> implements IM
         protected final ArrayDeque<CandidatePQ<In>> candidates;
         public MergeIteratorPQ(List<? extends Iterator<In>> iters, Comparator<In> comp, Reducer<In, Out> reducer)
         {
-            super(iters, reducer);
+            this.iterators = iters;
+            this.reducer = reducer;
             this.queue = new PriorityQueue<>(Math.max(1, iters.size()));
             for (int i = 0; i < iters.size(); i++)
             {
@@ -668,6 +678,23 @@ public MergeIteratorPQ(List<? extends Iterator<In>> iters, Comparator<In> comp,
             this.candidates = new ArrayDeque<>(queue.size());
         }
 
+        public void close()
+        {
+            for (int i = 0, length = this.iterators.size(); i < length; i++)
+            {
+                Iterator<In> iterator = iterators.get(i);
+                try
+                {
+                    if (iterator instanceof AutoCloseable)
+                        ((AutoCloseable)iterator).close();
+                }
+                catch (Exception e)
+                {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+
         protected final Out computeNext()
         {
             advance();
diff --git a/test/unit/org/apache/cassandra/utils/MergeIteratorTest.java b/test/unit/org/apache/cassandra/utils/MergeIteratorTest.java
index 056a4a762e41..88e37b44f66c 100644
--- a/test/unit/org/apache/cassandra/utils/MergeIteratorTest.java
+++ b/test/unit/org/apache/cassandra/utils/MergeIteratorTest.java
@@ -45,7 +45,7 @@ public void clear()
     @Test
     public void testManyToOne() throws Exception
     {
-        MergeIterator.Reducer<String,String> reducer = new MergeIterator.Reducer<String,String>()
+        Reducer<String,String> reducer = new Reducer<String,String>()
         {
             String concatted = "";
 
@@ -62,14 +62,25 @@ public String getReduced()
                 return tmp;
             }
         };
-        IMergeIterator<String,String> smi = MergeIterator.get(Arrays.asList(a, b, c, d),
-                Ordering.<String>natural(),
-                reducer);
+        CloseableIterator<String> smi = MergeIterator.getCloseable(Arrays.asList(a, b, c, d),
+                                                                   Ordering.<String>natural(),
+                                                                   reducer);
         assert Iterators.elementsEqual(cat, smi);
         smi.close();
         assert a.closed && b.closed && c.closed && d.closed;
     }
 
+    /** Test non-reducing version that should produce each repeating value separately. */
+    @Test
+    public void testNonReducing() throws Exception
+    {
+        CloseableIterator<String> smi = MergeIterator.getNonReducingCloseable(Arrays.asList(a, b, c, d),
+                                                                              Ordering.<String>natural());
+        assert Iterators.elementsEqual(all, smi);
+        smi.close();
+        assert a.closed && b.closed && c.closed && d.closed;
+    }
+
     // closeable list iterator
     public static class CLI<E> extends AbstractIterator<E> implements CloseableIterator<E>
     {
diff --git a/test/unit/org/apache/cassandra/utils/MonotonicClockTest.java b/test/unit/org/apache/cassandra/utils/MonotonicClockTest.java
index 3839be004b34..8d769508a7b5 100644
--- a/test/unit/org/apache/cassandra/utils/MonotonicClockTest.java
+++ b/test/unit/org/apache/cassandra/utils/MonotonicClockTest.java
@@ -21,6 +21,7 @@
 import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 import static org.junit.Assert.*;
 
+import org.junit.Assert;
 import org.junit.Test;
 
 public class MonotonicClockTest
@@ -54,4 +55,13 @@ public void testTimestampOrdering() throws Exception
             lastConverted = convertedNow;
         }
     }
+
+    @Test
+    public void testTimestampOverflowComparison()
+    {
+        MonotonicClock clock = MonotonicClock.Global.preciseTime;
+
+        Assert.assertTrue("Overflown long (now) should be after long close to max",
+                          clock.isAfter(Long.MIN_VALUE + 1, Long.MAX_VALUE));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/utils/NativeLibraryTest.java b/test/unit/org/apache/cassandra/utils/NativeLibraryTest.java
index 52a7f84f8af6..365ed21dcc2c 100644
--- a/test/unit/org/apache/cassandra/utils/NativeLibraryTest.java
+++ b/test/unit/org/apache/cassandra/utils/NativeLibraryTest.java
@@ -19,26 +19,135 @@
 package org.apache.cassandra.utils;
 
 
-import org.apache.cassandra.io.util.File;
+import java.io.FileDescriptor;
+import java.io.IOException;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.channels.FileChannel;
+import java.util.Arrays;
+
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 
 public class NativeLibraryTest
 {
+    @BeforeClass
+    public static void init()
+    {
+        DatabaseDescriptor.toolInitialization();
+    }
+
+    @Test
+    public void testIsOs()
+    {
+        Assert.assertTrue(Arrays.stream(INativeLibrary.OSType.values()).anyMatch(INativeLibrary.instance::isOS));
+    }
+
+    @Test
+    public void testIsAvailable()
+    {
+        Assert.assertTrue(INativeLibrary.instance.isAvailable());
+    }
+
+    @Test
+    public void testGetfdForAsynchronousFileChannel() throws IOException
+    {
+        File file = FileUtils.createDeletableTempFile("testSkipCache", "get_fd_async");
+
+        try (AsynchronousFileChannel channel = AsynchronousFileChannel.open(file.toPath()))
+        {
+            Assert.assertTrue(INativeLibrary.instance.getfd(channel) > 0);
+
+            FileDescriptor fileDescriptor = INativeLibrary.instance.getFileDescriptor(channel);
+            Assert.assertNotNull(fileDescriptor);
+            Assert.assertEquals(INativeLibrary.instance.getfd(channel), INativeLibrary.instance.getfd(fileDescriptor));
+        }
+    }
+
+    @Test
+    public void testGetfdForFileChannel() throws IOException
+    {
+        File file = FileUtils.createDeletableTempFile("testSkipCache", "get_fd");
+
+        try (FileChannel channel = FileChannel.open(file.toPath()))
+        {
+            Assert.assertTrue(INativeLibrary.instance.getfd(channel) > 0);
+
+            FileDescriptor fileDescriptor = INativeLibrary.instance.getFileDescriptor(channel);
+            Assert.assertNotNull(fileDescriptor);
+            Assert.assertEquals(INativeLibrary.instance.getfd(channel), INativeLibrary.instance.getfd(fileDescriptor));
+        }
+    }
+
+    @Test
+    public void testInvalidFileDescriptor()
+    {
+        Assert.assertEquals(-1, INativeLibrary.instance.getfd((FileDescriptor) null));
+    }
+
+    @Test
+    public void testTryFcntlWithIllegalArgument()
+    {
+        int invalidFd = 199991;
+        Assert.assertEquals(-1, INativeLibrary.instance.tryFcntl(invalidFd, -1, -1));
+    }
+
+    @Test
+    public void testOpenDirectory()
+    {
+        File file = FileUtils.createDeletableTempFile("testOpenDirectory", "1");
+
+        int fd = INativeLibrary.instance.tryOpenDirectory(file.parent());
+        INativeLibrary.instance.tryCloseFD(fd);
+    }
+
+    @Test
+    public void testOpenDirectoryWithIllegalArgument()
+    {
+        File file = FileUtils.createDeletableTempFile("testOpenDirectoryWithIllegalArgument", "1");
+        Assert.assertEquals(-1, INativeLibrary.instance.tryOpenDirectory(file.resolve("no_existing")));
+    }
+
+    @Test
+    public void testTrySyncWithIllegalArgument()
+    {
+        INativeLibrary.instance.trySync(-1);
+
+        int invalidFd = 199991;
+        Assert.assertThrows(FSWriteError.class, () -> INativeLibrary.instance.trySync(invalidFd));
+    }
+
+    @Test
+    public void testTryCloseFDWithIllegalArgument()
+    {
+        INativeLibrary.instance.tryCloseFD(-1);
+
+        int invalidFd = 199991;
+        Assert.assertThrows(FSWriteError.class, () -> INativeLibrary.instance.tryCloseFD(invalidFd));
+    }
+
     @Test
     public void testSkipCache()
     {
         File file = FileUtils.createDeletableTempFile("testSkipCache", "1");
 
-        NativeLibrary.trySkipCache(file.path(), 0, 0);
+        INativeLibrary.instance.trySkipCache(file, 0, 0);
+        INativeLibrary.instance.trySkipCache(file.resolve("no_existing"), 0, 0);
+
+        // non-existing FD
+        INativeLibrary.instance.trySkipCache(-1, 0, 0L, "non-existing file");
+        INativeLibrary.instance.trySkipCache(-1, 0, 0, "non-existing file");
     }
 
     @Test
     public void getPid()
     {
-        long pid = NativeLibrary.getProcessID();
+        long pid = INativeLibrary.instance.getProcessID();
         Assert.assertTrue(pid > 0);
     }
 }
diff --git a/test/unit/org/apache/cassandra/utils/OrderCheckingIterator.java b/test/unit/org/apache/cassandra/utils/OrderCheckingIterator.java
new file mode 100644
index 000000000000..57789066c054
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/OrderCheckingIterator.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.utils;
+
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Wrapping iterator that checks source for problems with iteration order and tombstone bounds.
+ */
+public final class OrderCheckingIterator extends AbstractIterator<Unfiltered> implements UnfilteredRowIterator
+{
+    /**
+     * The decorated iterator.
+     */
+    private final UnfilteredRowIterator iterator;
+
+    private final ClusteringComparator comparator;
+
+    private Unfiltered previous;
+
+    private RangeTombstoneMarker openMarker;
+
+    public OrderCheckingIterator(UnfilteredRowIterator iterator)
+    {
+        this.iterator = iterator;
+        this.comparator = iterator.metadata().comparator;
+    }
+
+    public TableMetadata metadata()
+    {
+        return iterator.metadata();
+    }
+
+    public boolean isReverseOrder()
+    {
+        return iterator.isReverseOrder();
+    }
+
+    public RegularAndStaticColumns columns()
+    {
+        return iterator.columns();
+    }
+
+    public DecoratedKey partitionKey()
+    {
+        return iterator.partitionKey();
+    }
+
+    public Row staticRow()
+    {
+        return iterator.staticRow();
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return iterator.isEmpty();
+    }
+
+    public void close()
+    {
+        iterator.close();
+    }
+
+    public DeletionTime partitionLevelDeletion()
+    {
+        return iterator.partitionLevelDeletion();
+    }
+
+    public EncodingStats stats()
+    {
+        return iterator.stats();
+    }
+
+    protected Unfiltered computeNext()
+    {
+        if (!iterator.hasNext())
+        {
+            // Check that we are not left with an unclosed tombstone.
+            if (openMarker != null)
+                throw new AssertionError(String.format("Found orphaned open tombstone marker (with no " +
+                                                       "closing marker) at clustering %s of partition %s",
+                                                       openMarker.clustering().toString(metadata()),
+                                                       metadata().partitionKeyType.getString(partitionKey().getKey())));
+            return endOfData();
+        }
+
+        boolean reversed = isReverseOrder();
+        Unfiltered next = iterator.next();
+
+        // Check data comes in the right order.
+        if (previous != null && comparator.compare(next, previous) < 0)
+        {
+            // We may have to return one more close marker in this call, but we're done no matter what after that.
+            throw new AssertionError(String.format("Found out of order data, clustering %s after %s in partition %s",
+                                                   next.clustering().toString(metadata()),
+                                                   previous.clustering().toString(metadata()),
+                                                   metadata().partitionKeyType.getString(partitionKey().getKey())));
+        }
+
+        // Validate invariants of range tombstones markers, namely that:
+        // - we don't have a close without a previous open.
+        // - both deletionTime match.
+        // - we don't have ineffective tombstones, i.e.
+        //   -- boundaries with equal deletions on both sides
+        //   -- tombstones with DeletionTime.LIVE
+        if (next.isRangeTombstoneMarker())
+        {
+            LoggerFactory.getLogger(getClass()).info("tombstone: {}", next.toString(metadata()));
+            RangeTombstoneMarker marker = (RangeTombstoneMarker) next;
+
+            if (marker.isOpen(reversed) && marker.openDeletionTime(reversed).isLive() ||
+                marker.isClose(reversed) && marker.closeDeletionTime(reversed).isLive())
+            {
+                throw new AssertionError(String.format("Found an ineffective tombstone bound (with live " +
+                                                       "deletion time) at clustering %s of partition %s",
+                                                       marker.clustering().toString(metadata()),
+                                                       metadata().partitionKeyType.getString(partitionKey().getKey())));
+            }
+
+            if (marker.isBoundary() && marker.openDeletionTime(reversed).equals(marker.closeDeletionTime(reversed)))
+            {
+                throw new AssertionError(String.format("Found an ineffective tombstone boundary (with equal close " +
+                                                       "and open deletion times) at clustering %s of partition %s",
+                                                       marker.clustering().toString(metadata()),
+                                                       metadata().partitionKeyType.getString(partitionKey().getKey())));
+            }
+
+            // Marker can be a boundary, and if so, handling his close first is easier.
+            if (marker.isClose(reversed))
+            {
+                if (openMarker == null)
+                    throw new AssertionError(String.format("Found orphaned close tombstone marker (with no prior " +
+                                                           "opening marker) at clustering %s of partition %s",
+                                                           marker.clustering().toString(metadata()),
+                                                           metadata().partitionKeyType.getString(partitionKey().getKey())));
+                if (!openMarker.openDeletionTime(reversed).equals(marker.closeDeletionTime(reversed)))
+                    throw new AssertionError(String.format("Mismatched open and close tombstone markers in partition %s: " +
+                                                           "open marker at clustering %s had deletion info %s, " +
+                                                           "but close marker at clustering %s has deletion info %s",
+                                                           metadata().partitionKeyType.getString(partitionKey().getKey()),
+                                                           openMarker.clustering().toString(metadata()),
+                                                           openMarker.openDeletionTime(reversed),
+                                                           marker.clustering().toString(metadata()),
+                                                           marker.closeDeletionTime(reversed)));
+                openMarker = null;
+            }
+            if (marker.isOpen(reversed))
+            {
+                // Same as above, we check invariants, namely that we should not have a current open marker (it
+                // should have been closed before any open marker.
+                if (openMarker != null)
+                    throw new AssertionError(String.format("Found non-closed open tombstone marker at clustering %s " +
+                                                           "of partition %s: a new marker is opened at clustering %s " +
+                                                           "without having seen a prior close.",
+                                                           openMarker.clustering().toString(metadata()),
+                                                           metadata().partitionKeyType.getString(partitionKey().getKey()),
+                                                           marker.clustering().toString(metadata())));
+                openMarker = marker;
+            }
+        }
+        previous = next;
+        return next;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/OverlapsTest.java b/test/unit/org/apache/cassandra/utils/OverlapsTest.java
index 3312578b629c..e0a5243345dd 100644
--- a/test/unit/org/apache/cassandra/utils/OverlapsTest.java
+++ b/test/unit/org/apache/cassandra/utils/OverlapsTest.java
@@ -33,6 +33,9 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import org.agrona.collections.IntArrayList;
+
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
 public class OverlapsTest
@@ -236,6 +239,88 @@ private static Set<Character> asSet(String a)
     }
 
 
+    @Test
+    public void testCombineSetsWithCommonElement()
+    {
+        String[] sets = new String[]{
+        "ABCD",
+        "ADE",
+        "EF",
+        "HI",
+        "LN",
+        "NO",
+        "NPQ",
+        "RST",
+        };
+        String[] nonOverlapping = new String[]{
+        "ABCDEF",
+        "HI",
+        "LNOPQ",
+        "RST",
+        };
+
+        List<Set<Character>> input = Arrays.stream(sets).map(OverlapsTest::asSet).collect(Collectors.toList());
+        List<Set<Character>> expected = Arrays.stream(nonOverlapping).map(OverlapsTest::asSet).collect(Collectors.toList());
+
+        List<Set<Character>> result = Overlaps.combineSetsWithCommonElement(input);
+        assertEquals(expected, result);
+    }
+
+    @Test
+    public void testSplitInNonOverlappingSetsRandom()
+    {
+        int size;
+        int range = 100;
+        Random rand = new Random();
+        for (int i = 0; i < 1000; ++i)
+        {
+            size = rand.nextInt(range) + 2;
+            Interval<Integer, String>[] input = new Interval[size];
+            char c = 'A';
+            for (int j = 0; j < size; ++j)
+            {
+                int start = rand.nextInt(range);
+                input[j] = (new Interval<>(start, start + 1 + random.nextInt(range - start), Character.toString(c++)));
+            }
+
+            boolean endInclusive = rand.nextBoolean();
+
+            List<Set<Interval<Integer, String>>> overlaps =
+            Overlaps.splitInNonOverlappingSets(Arrays.asList(input),
+                                               endInclusive ? (x, y) -> x.min > y.max
+                                                            : (x, y) -> x.min >= y.max,
+                                               Comparator.comparingInt(x -> x.min),
+                                               Comparator.comparingInt(x -> x.max));
+
+            for (var set : overlaps)
+            {
+                for (var interval : set)
+                {
+                    // must intersect with at least one other item in the set
+                    if (set.size() > 1)
+                        Assert.assertTrue(set.stream().filter(x -> x != interval).anyMatch(x -> intersects(x, interval, endInclusive)));
+                    // and no interval outside the set
+                    Assert.assertTrue(overlaps.stream().filter(x -> x != set).flatMap(Set::stream).noneMatch(x -> intersects(x, interval, endInclusive)));
+                }
+            }
+        }
+    }
+
+    <C extends Comparable<C>, D> boolean intersects(Interval<C, D> i1, Interval<C, D> i2, boolean endInclusive)
+    {
+        return endInclusive ? intersectsEndsIncluded(i1, i2) : intersectsEndsExcluded(i1, i2);
+    }
+
+    <C extends Comparable<C>, D> boolean intersectsEndsExcluded(Interval<C, D> i1, Interval<C, D> i2)
+    {
+        return i1.min.compareTo(i2.max) < 0 && i1.max.compareTo(i2.min) > 0;
+    }
+
+    <C extends Comparable<C>, D> boolean intersectsEndsIncluded(Interval<C, D> i1, Interval<C, D> i2)
+    {
+        return i1.min.compareTo(i2.max) <= 0 && i1.max.compareTo(i2.min) >= 0;
+    }
+
     @Test
     public void testAssignOverlapsIntoBuckets()
     {
@@ -251,33 +336,41 @@ public void testAssignOverlapsIntoBuckets()
         };
         String[] none3 = new String[]{
         "ABCD",
-        "ADE",
         "NPQ",
         "RST",
         };
+        int[] noneUnselected = new int[]{1, 5, 4, 2, 3};
         String[] single3 = new String[]{
         "ABCDE",
         "LNOPQ",
         "RST",
         };
+        int[] singleUnselected = new int[]{2, 3};
         String[] transitive3 = new String[]{
         "ABCDEF",
         "LNOPQ",
         "RST",
         };
+        int[] transitiveUnselected = new int[]{3};
 
         List<Set<Character>> input = Arrays.stream(sets).map(OverlapsTest::asSet).collect(Collectors.toList());
-        List<String> actual;
 
-        actual = Overlaps.assignOverlapsIntoBuckets(3, Overlaps.InclusionMethod.NONE, input, this::makeBucket);
-        assertEquals(Arrays.asList(none3), actual);
+        verifyAssignment(Overlaps.InclusionMethod.NONE, input, none3, noneUnselected);
 
-        actual = Overlaps.assignOverlapsIntoBuckets(3, Overlaps.InclusionMethod.SINGLE, input, this::makeBucket);
-        assertEquals(Arrays.asList(single3), actual);
+        verifyAssignment(Overlaps.InclusionMethod.SINGLE, input, single3, singleUnselected);
 
-        actual = Overlaps.assignOverlapsIntoBuckets(3, Overlaps.InclusionMethod.TRANSITIVE, input, this::makeBucket);
-        assertEquals(Arrays.asList(transitive3), actual);
+        verifyAssignment(Overlaps.InclusionMethod.TRANSITIVE, input, transitive3, transitiveUnselected);
+    }
+
+    private void verifyAssignment(Overlaps.InclusionMethod method, List<Set<Character>> input, String[] expected, int[] expectedUnselected)
+    {
+        List<String> actual;
+        IntArrayList unselected = new IntArrayList();
+        actual = Overlaps.assignOverlapsIntoBuckets(3, method, input, this::makeBucket, s -> unselected.add(input.indexOf(s)));
+        actual.sort(String::compareTo);
 
+        assertEquals(Arrays.asList(expected), actual);
+        assertArrayEquals(expectedUnselected, unselected.toIntArray());
     }
 
     private String makeBucket(List<Set<Character>> sets, int startIndex, int endIndex)
diff --git a/test/unit/org/apache/cassandra/utils/ProductTypeTest.java b/test/unit/org/apache/cassandra/utils/ProductTypeTest.java
new file mode 100644
index 000000000000..87f283ec2367
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/ProductTypeTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import org.junit.AfterClass;
+import org.junit.Test;
+
+
+import static org.junit.Assert.assertEquals;
+
+public class ProductTypeTest
+{
+    @AfterClass
+    public static void cleanup()
+    {
+        // checkstyle: suppress below 'blockSystemPropertyUsage'
+        System.clearProperty("dse.product_type");
+    }
+
+    @Test
+    public void testDefault()
+    {
+        // checkstyle: suppress below 'blockSystemPropertyUsage'
+        System.clearProperty("dse.product_type");
+        assertEquals(ProductType.Product.DATASTAX_CASSANDRA, ProductType.getProduct());
+    }
+
+    @Test
+    public void testDatastaxApollo()
+    {
+        System.setProperty("dse.product_type", "DATASTAX_APOLLO");
+        assertEquals(ProductType.Product.DATASTAX_APOLLO, ProductType.getProduct());
+
+        System.setProperty("dse.product_type", "datastax_apollo");
+        assertEquals(ProductType.Product.DATASTAX_APOLLO, ProductType.getProduct());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/SortingIteratorTest.java b/test/unit/org/apache/cassandra/utils/SortingIteratorTest.java
new file mode 100644
index 000000000000..4ab503877886
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/SortingIteratorTest.java
@@ -0,0 +1,518 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+import java.util.*;
+
+import com.google.common.base.Predicates;
+
+public class SortingIteratorTest
+{
+    // Most of this test is ChatGPT-generated.
+
+    @Test
+    public void testSortingIterator_withFixedData()
+    {
+        List<Integer> data = List.of(4, 1, 3, 2);
+        Iterator<Integer> iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        List<Integer> sorted = new ArrayList<>();
+        while (iterator.hasNext())
+        {
+            sorted.add(iterator.next());
+        }
+
+        assertEquals(List.of(1, 2, 3, 4), sorted);
+    }
+
+    @Test
+    public void testSortingIterator_withEmptyData()
+    {
+        List<Integer> data = List.of();
+        Iterator<Integer> iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        assertFalse(iterator.hasNext());
+    }
+
+    @Test
+    public void testNextWithoutHasNext()
+    {
+        List<String> data = List.of("apple", "orange", "banana");
+        Iterator<String> iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        assertEquals("apple", iterator.next());
+        assertEquals("banana", iterator.next());
+        assertEquals("orange", iterator.next());
+        assertFalse(iterator.hasNext());
+    }
+
+    @Test(expected = NoSuchElementException.class)
+    public void testNoSuchElementException()
+    {
+        List<Integer> data = List.of(1, 3, 2);
+        Iterator<Integer> iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.next();
+        iterator.next();
+        iterator.next();
+        iterator.next(); // Should throw NoSuchElementException
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testUnsupportedOperationException()
+    {
+        List<Integer> data = List.of(1, 2, 3);
+        Iterator<Integer> iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.remove(); // Should throw UnsupportedOperationException
+    }
+
+
+    @Test
+    public void testWithDuplicates()
+    {
+        List<Integer> data = List.of(4, 1, 2, 5, 3, 4, 2, 4, 4);
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        List<Integer> result = new ArrayList<>();
+        while (iterator.hasNext())
+        {
+            result.add(iterator.next());
+        }
+
+        assertEquals(List.of(1, 2, 2, 3, 4, 4, 4, 4, 5), result);
+    }
+
+    @Test
+    public void testSkipTo_existingKey()
+    {
+        List<Integer> data = List.of(5, 3, 1, 9, 7);
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(5);
+
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 5, iterator.next());
+    }
+
+    @Test
+    public void testSkipTo_nonExistingKey()
+    {
+        List<Integer> data = List.of(1, 5, 7, 9, 3);
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(6);
+
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 7, iterator.next());
+    }
+
+    @Test
+    public void testSkipTo_beyondLastKey()
+    {
+        List<Integer> data = List.of(9, 3, 1, 7, 5);
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(10);
+
+        assertFalse(iterator.hasNext());
+    }
+
+    @Test
+    public void testSkipTo_firstKey()
+    {
+        List<Integer> data = List.of(1, 5, 3, 9, 7);
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(1);
+
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 1, iterator.next());
+    }
+
+    @Test
+    public void testSkipTo_beforeFirstKey()
+    {
+        List<Integer> data = List.of(3, 9, 1, 7, 5);
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(0);
+
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 1, iterator.next());
+    }
+
+
+    @Test
+    public void testSkipTo_withDuplicates()
+    {
+        List<Integer> data = List.of(3, 4, 1, 4, 2, 4, 5, 2, 4);
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(2);
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 2, iterator.next());
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 2, iterator.next());
+
+        iterator.skipTo(4);
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 4, iterator.next());
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 4, iterator.next());
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 4, iterator.next());
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 4, iterator.next());
+
+        iterator.skipTo(5);
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 5, iterator.next());
+    }
+
+    @Test
+    public void testSkipTo_onEmptyCollection()
+    {
+        List<Integer> data = List.of();
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(5);
+
+        assertFalse(iterator.hasNext());
+    }
+
+    @Test
+    public void testRandomizedSkipTo()
+    {
+        Random random = new Random();
+        int size = random.nextInt(100) + 50; // List size between 50 and 150
+        List<Integer> data = new ArrayList<>();
+
+        // Generate random data
+        for (int i = 0; i < size; i++)
+        {
+            data.add(random.nextInt(200)); // Values between 0 and 199
+        }
+
+        // Create the iterator
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+
+        Collections.sort(data);
+
+        // Track the current index
+        int currentIndex = 0;
+
+        // Perform random skipTo operations
+        for (int i = 0; i < 10; i++)
+        { // Perform 10 random skipTo operations
+            int targetKey = random.nextInt(200); // Random target key between 0 and 199
+            iterator.skipTo(targetKey);
+
+            // Find the expected position, considering the current position
+            int expectedIndex = Integer.MAX_VALUE;
+            for (int j = currentIndex; j < data.size(); j++)
+            {
+                if (data.get(j) >= targetKey)
+                {
+                    expectedIndex = j;
+                    break;
+                }
+            }
+
+            if (expectedIndex >= data.size())
+            {
+                // If no element is greater than or equal to targetKey, iterator should be exhausted
+                assertFalse(iterator.hasNext());
+                currentIndex = expectedIndex;
+            }
+            else
+            {
+                // Otherwise, the next element should be the expected one
+                assertTrue(iterator.hasNext());
+                assertEquals(data.get(expectedIndex), iterator.next());
+                currentIndex = expectedIndex + 1;
+            }
+        }
+    }
+
+
+    @Test
+    public void testSortingIterator_withRandomData()
+    {
+        Random random = new Random();
+        int size = 10000;
+        List<Integer> data = new ArrayList<>();
+        for (int i = 0; i < size; i++)
+        {
+            data.add(random.nextInt(size));
+        }
+
+        Iterator<Integer> iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+        List<Integer> sorted = new ArrayList<>();
+        while (iterator.hasNext())
+        {
+            sorted.add(iterator.next());
+        }
+
+        List<Integer> expected = new ArrayList<>(data);
+        Collections.sort(expected);
+
+        assertEquals(expected, sorted);
+    }
+
+    @Test
+    public void testSortingIterator_skipToRandomData()
+    {
+        Random random = new Random();
+        int size = 10000;
+        List<Integer> data = new ArrayList<>();
+        for (int i = 0; i < size; i++)
+        {
+            data.add(random.nextInt(size));
+        }
+
+        List<Integer> sorted = new ArrayList<>(data);
+        Collections.sort(sorted);
+        List<Integer> expected = new ArrayList<>();
+
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+        List<Integer> iterated = new ArrayList<>();
+
+        int skipDistanceMax = 200;
+        for (int i = 0; i < size; i += random.nextInt(skipDistanceMax))
+        {
+            int targetMax = sorted.get(i);
+            int targetMin = i > 0 ? sorted.get(i - 1) : 0;
+            if (targetMin == targetMax)
+                continue;
+            int target = random.nextInt(targetMax - targetMin) + targetMin + 1; // (targetMin + 1; targetMax]
+            iterator.skipTo(target);
+            for (int c = random.nextInt(5); c >= 0; --c)
+            {
+                if (i >= size)
+                {
+                    assert !iterator.hasNext();
+                    break;
+                }
+                iterated.add(iterator.next());
+                expected.add(sorted.get(i++));
+            }
+        }
+
+        assertEquals(expected, iterated);
+    }
+
+    @Test
+    public void testDeduplicateRemovesDuplicates()
+    {
+        List<Integer> data = List.of(4, 1, 2, 5, 3, 4, 2, 4, 4);
+        var iterator = SortingIterator.createDeduplicating(Comparator.naturalOrder(), data);
+
+        List<Integer> result = new ArrayList<>();
+        while (iterator.hasNext())
+        {
+            result.add(iterator.next());
+        }
+
+        assertEquals(List.of(1, 2, 3, 4, 5), result);
+    }
+
+    @Test
+    public void testSkipTo_deduplicate()
+    {
+        List<Integer> data = List.of(3, 4, 1, 4, 2, 4, 5, 2, 4);
+        var iterator = SortingIterator.createDeduplicating(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(3);
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 3, iterator.next());
+
+        iterator.skipTo(4);
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 4, iterator.next());
+
+        iterator.skipTo(5);
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 5, iterator.next());
+    }
+
+    @Test
+    public void testSkipTo_deduplicateWithNonExistingTarget()
+    {
+        List<Integer> data = List.of(4, 5, 1, 3, 4, 4, 2, 4, 2);
+        var iterator = SortingIterator.createDeduplicating(Comparator.naturalOrder(), data);
+
+        iterator.skipTo(2); // Skip to the first occurrence of 2
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 2, iterator.next());
+        // The above must have consumed all 2s
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 3, iterator.next());
+
+        iterator.skipTo(6); // Skip to a non-existing key, should end iteration
+        assertFalse(iterator.hasNext());
+    }
+
+    @Test
+    public void testEmptyCollectionDeduplicate()
+    {
+        List<Integer> data = List.of();
+        var iterator = SortingIterator.createDeduplicating(Comparator.naturalOrder(), data);
+
+        assertFalse(iterator.hasNext());
+    }
+
+    @Test
+    public void testSingleElementDeduplicate()
+    {
+        List<Integer> data = List.of(42, 42, 42);
+        var iterator = SortingIterator.createDeduplicating(Comparator.naturalOrder(), data);
+
+        assertTrue(iterator.hasNext());
+        assertEquals((Integer) 42, iterator.next());
+        assertFalse(iterator.hasNext());
+    }
+
+    @Test
+    public void testRandomizedDeduplicate()
+    {
+        Random random = new Random();
+        int size = random.nextInt(100) + 50; // List size between 50 and 150
+        List<Integer> data = new ArrayList<>();
+
+        // Generate random data with potential duplicates
+        for (int i = 0; i < size; i++)
+        {
+            data.add(random.nextInt(50)); // Values between 0 and 49, likely to have duplicates
+        }
+
+        // Construct through Builder for coverage
+        var iterator = new SortingIterator.Builder<>(data, x -> x)
+                           .deduplicating(Comparator.naturalOrder());
+
+        // Using a set to verify uniqueness of the iterator output
+        Set<Integer> seenElements = new HashSet<>();
+        while (iterator.hasNext())
+        {
+            Integer element = iterator.next();
+            assertFalse(seenElements.contains(element)); // Ensure no duplicates are returned
+            seenElements.add(element);
+        }
+
+        // Verify that all elements in the set were indeed from the original data (in sorted, unique form)
+        Set<Integer> expectedElements = new TreeSet<>(data); // TreeSet to sort and remove duplicates
+        assertEquals(expectedElements, seenElements);
+    }
+
+    @Test
+    public void testSortingIterator_randomWithNulls()
+    {
+        Random random = new Random();
+        int size = 10000;
+        List<Integer> data = new ArrayList<>();
+        for (int i = 0; i < size; i++)
+        {
+            data.add(random.nextInt(10) != 0 ? random.nextInt(size) : null);
+        }
+
+        Iterator<Integer> iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+        List<Integer> sorted = new ArrayList<>();
+        while (iterator.hasNext())
+        {
+            sorted.add(iterator.next());
+        }
+
+        List<Integer> expected = new ArrayList<>(data);
+        expected.removeIf(Predicates.isNull());
+        Collections.sort(expected);
+
+        assertEquals(expected, sorted);
+    }
+
+    @Test
+    public void testSortingIterator_skipToRandomDataWithNulls()
+    {
+        Random random = new Random();
+        int size = 10000;
+        List<Integer> data = new ArrayList<>();
+        for (int i = 0; i < size; i++)
+        {
+            data.add(random.nextInt(10) != 0 ? random.nextInt(size) : null);
+        }
+
+        List<Integer> sorted = new ArrayList<>(data);
+        sorted.removeIf(Predicates.isNull());
+        Collections.sort(sorted);
+        size = sorted.size();
+        List<Integer> expected = new ArrayList<>();
+
+        var iterator = SortingIterator.create(Comparator.naturalOrder(), data);
+        List<Integer> iterated = new ArrayList<>();
+
+        int skipDistanceMax = 200;
+        for (int i = 0; i < size; i += random.nextInt(skipDistanceMax))
+        {
+            int targetMax = sorted.get(i);
+            int targetMin = i > 0 ? sorted.get(i - 1) : 0;
+            if (targetMin == targetMax)
+                continue;
+            int target = random.nextInt(targetMax - targetMin) + targetMin + 1; // (targetMin + 1; targetMax]
+            iterator.skipTo(target);
+            for (int c = random.nextInt(5); c >= 0; --c)
+            {
+                if (i >= size)
+                {
+                    assert !iterator.hasNext();
+                    break;
+                }
+                iterated.add(iterator.next());
+                expected.add(sorted.get(i++));
+            }
+        }
+
+        assertEquals(expected, iterated);
+    }
+
+    /**
+     * Dump the evolution of a heap of 15 elements as mermaid graph sources for visualizations/documentation.
+     */
+    @Test
+    public void makeMermaids()
+    {
+        int size = 15;
+        Random rand = new Random(52);
+        Integer[] array = new Integer[size];
+        for (int i = 0; i < size; ++i)
+            array[i] = rand.nextDouble() < 0.03 ? null : rand.nextInt(100);
+        System.out.println(Arrays.toString(array));
+
+        var sorter = new SortingIterator<Integer>(Comparator.naturalOrder(), array);
+        var sorted = new ArrayList<Integer>();
+        while (sorter.hasNext())
+        {
+            System.out.println(sorter.toMermaid());
+            sorted.add(sorter.next());
+        }
+        System.out.println(sorted);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/StringSerializerTest.java b/test/unit/org/apache/cassandra/utils/StringSerializerTest.java
new file mode 100644
index 000000000000..0c7edf6bf9ad
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/StringSerializerTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public class StringSerializerTest
+{
+    @Test
+    public void testStringSerialization() throws IOException
+    {
+        serializationRoundTrip("");
+        serializationRoundTrip(" ");
+        serializationRoundTrip("zxc");
+        serializationRoundTrip("óążćź");
+    }
+
+    private void serializationRoundTrip(String value) throws IOException
+    {
+        int protocolVersion = 1;
+        StringSerializer serializer = StringSerializer.serializer;
+        long size = serializer.serializedSize(value, protocolVersion);
+
+        ByteBuffer buf = ByteBuffer.allocate((int)size);
+        DataOutputPlus out = new DataOutputBufferFixed(buf);
+        serializer.serialize(value, out, protocolVersion);
+        Assert.assertEquals(size, buf.position());
+
+        buf.flip();
+        DataInputPlus in = new DataInputBuffer(buf, false);
+        String deserialized = serializer.deserialize(in, protocolVersion);
+        Assert.assertEquals(value, deserialized);
+        Assert.assertEquals(value.hashCode(), deserialized.hashCode());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/ThrowablesTest.java b/test/unit/org/apache/cassandra/utils/ThrowablesTest.java
new file mode 100644
index 000000000000..4abf1178ee00
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/ThrowablesTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ThrowablesTest
+{
+    @Test
+    public void testGetCauseOfType()
+    {
+        // getCauseOfType with null throwable returns empty Optional
+        Assert.assertTrue(Throwables.getCauseOfType(null, RuntimeException.class).isEmpty());
+        // getCauseOfType with no cause returns empty Optional
+        Assert.assertTrue(Throwables.getCauseOfType(new Exception(), RuntimeException.class).isEmpty());
+        // getCauseOfType with no matching causing in chain returns empty Optional
+        Assert.assertTrue(Throwables.getCauseOfType(new Exception(new Exception()), RuntimeException.class).isEmpty());
+        // getCauseOfType with throwable of the specified type returns Optional with the throwable
+        RuntimeException cause = new RuntimeException();
+        Assert.assertEquals(cause, Throwables.getCauseOfType(cause, RuntimeException.class).get());
+        // getCauseOfType with cause of the specified type returns Optional with the cause
+        Exception exception = new Exception(cause);
+        Assert.assertEquals(cause, Throwables.getCauseOfType(exception, RuntimeException.class).get());
+        // with multiple causes in chain, return first one
+        RuntimeException cause2 = new RuntimeException(cause);
+        Exception exception2 = new Exception(cause2);
+        Assert.assertEquals(cause2, Throwables.getCauseOfType(exception2, RuntimeException.class).get());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/TopKSelectorTest.java b/test/unit/org/apache/cassandra/utils/TopKSelectorTest.java
new file mode 100644
index 000000000000..6ade16b5af8d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/TopKSelectorTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Predicates;
+
+import static org.junit.Assert.*;
+
+public class TopKSelectorTest
+{
+
+    private TopKSelector<Integer> selector;
+
+    @Before
+    public void setUp()
+    {
+        selector = new TopKSelector<>(Integer::compareTo, 5);
+    }
+
+    @Test
+    public void testBasicFunctionality()
+    {
+        List<Integer> data = List.of(10, 20, 5, 7, 1, 3, 9, 15, 25);
+
+        for (int num : data)
+        {
+            selector.add(num);
+        }
+
+        List<Integer> topK = selector.get();
+
+        // Expecting the smallest 5 elements in sorted order
+        assertEquals(List.of(1, 3, 5, 7, 9), topK);
+    }
+
+    @Test
+    public void testTopKSmallerThanK()
+    {
+        List<Integer> data = List.of(10, 20, 5);
+
+        for (int num : data)
+        {
+            selector.add(num);
+        }
+
+        List<Integer> topK = selector.get();
+
+        // Expecting all elements in sorted order because the total number is less than k
+        assertEquals(List.of(5, 10, 20), topK);
+    }
+
+    @Test
+    public void testTopKWithDuplicates()
+    {
+        List<Integer> data = List.of(10, 20, 5, 5, 7, 10, 1, 3, 15, 25);
+
+        for (int num : data)
+        {
+            selector.add(num);
+        }
+
+        List<Integer> topK = selector.get();
+
+        // Expecting the smallest 5 elements in sorted order
+        assertEquals(List.of(1, 3, 5, 5, 7), topK);
+    }
+
+
+    @Test
+    public void testTopKWithNulls()
+    {
+        List<Integer> data = Arrays.asList(10, null, 5, 5, 7, 10, 1, 3, null, 25);  // List.of does not like nulls
+
+        for (Integer num : data)
+        {
+            selector.add(num);
+        }
+
+        List<Integer> topK = selector.get();
+
+        // Expecting the smallest 5 elements in sorted order
+        assertEquals(List.of(1, 3, 5, 5, 7), topK);
+    }
+
+
+    @Test
+    public void testTopKWithNullsShort()
+    {
+        List<Integer> data = Arrays.asList(10, null, 5, 7, null);
+
+        for (Integer num : data)
+        {
+            selector.add(num);
+        }
+
+        List<Integer> topK = selector.getShared();
+
+        // Expecting the non-null elements in sorted order
+        assertEquals(List.of(5, 7, 10), topK);
+    }
+
+    @Test
+    public void testTopKWithNegativeNumbers()
+    {
+        List<Integer> data = List.of(-10, -20, -5, -7, -1, -3, -9, -15, -25);
+
+        for (int num : data)
+        {
+            selector.add(num);
+        }
+
+        // also test transformation
+        List<Integer> topK = selector.getTransformed(x -> -x);
+
+        // Expecting the smallest (most negative) 5 elements in sorted order, transformed to -x
+        assertEquals(List.of(25, 20, 15, 10, 9), topK);
+    }
+
+    @Test
+    public void testEmptyInput()
+    {
+        List<Integer> topK = selector.get();
+
+        // Expecting an empty list because no elements were added
+        assertEquals(List.of(), topK);
+    }
+
+    @Test
+    public void testSingleElementInput()
+    {
+        selector.add(42);
+
+        List<Integer> topK = selector.get();
+
+        // Expecting the single element in the list
+        assertEquals(List.of(42), topK);
+    }
+
+    @Test
+    public void testRandomizedInput()
+    {
+        Random random = new Random();
+        int size = 1000;
+        int k = random.nextInt(20) + 1;
+
+        TopKSelector<Integer> randomSelector = new TopKSelector<>(Integer::compareTo, k);
+        List<Integer> all = new ArrayList<>();
+        for (int i = 0; i < size; i++)
+        {
+            Integer newItem = random.nextInt(10000); // Random values between 0 and 9999
+            randomSelector.add(newItem);
+            all.add(newItem);
+        }
+
+        List<Integer> topK = randomSelector.get();
+        List<Integer> sortedTopK = all.stream().sorted().limit(k).collect(Collectors.toList());
+
+        // Ensure the top k elements are sorted
+        assertEquals(sortedTopK, topK);
+
+        // Ensure the size of the top k list is k
+        assertEquals(k, topK.size());
+    }
+
+    @Test
+    public void testRandomizedInputNulls()
+    {
+        Random random = new Random();
+        int size = 1000;
+        int k = random.nextInt(20) + 1;
+
+        TopKSelector<Integer> randomSelector = new TopKSelector<>(Integer::compareTo, k);
+        List<Integer> all = new ArrayList<>();
+        for (int i = 0; i < size; i++)
+        {
+            Integer newItem = random.nextDouble() < 0.05 ? null : random.nextInt(10000); // Random values between 0 and 9999
+            randomSelector.add(newItem);
+            all.add(newItem);
+        }
+
+        List<Integer> topK = randomSelector.get();
+        List<Integer> sortedTopK = all.stream().filter(Predicates.notNull()).sorted().limit(k).collect(Collectors.toList());
+
+        // Ensure the top k elements are sorted
+        assertEquals(sortedTopK, topK);
+
+        // Ensure the size of the top k list is k
+        assertEquals(k, topK.size());
+    }
+
+
+    @Test
+    public void testRandomizedInputTransformedSliced()
+    {
+        Random random = new Random();
+        int size = 1000;
+        int offset = random.nextInt(20) + 1;
+        int k = random.nextInt(20);
+
+        TopKSelector<Integer> randomSelector = new TopKSelector<>(Integer::compareTo, k + offset);
+        List<Integer> all = new ArrayList<>();
+        for (int i = 0; i < size; i++)
+        {
+            Integer newItem = random.nextInt(10000); // Random values between 0 and 9999
+            randomSelector.add(newItem);
+            all.add(newItem);
+        }
+
+        Function<Integer, Integer> transformer = x -> x + 5;
+        List<Integer> topK = randomSelector.getTransformedSlicedShared(transformer, offset);
+        List<Integer> sortedTopK = all.stream().sorted().map(transformer).skip(offset).limit(k).collect(Collectors.toList());
+
+        // Ensure the top k elements are sorted
+        assertEquals(sortedTopK, topK);
+
+        // Ensure the size of the top k list is k
+        assertEquals(k, topK.size());
+
+        // Get the rest of the items now
+        List<Integer> remainder = randomSelector.getShared();
+        List<Integer> sortedRemainder = all.stream().sorted().limit(offset).collect(Collectors.toList());
+        assertEquals(sortedRemainder, remainder);
+    }
+
+    @Test
+    public void testAddMoreThanKElements()
+    {
+        for (int i = 0; i < 20; i++)
+        {
+            selector.add(i);
+        }
+
+        List<Integer> topK = selector.get();
+
+        // Expecting the smallest 5 elements in sorted order
+        assertEquals(List.of(0, 1, 2, 3, 4), topK);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testZeroK()
+    {
+        var selector = new TopKSelector<>(Integer::compareTo, 0);
+        selector.addAll(List.of(10, 20, 3, 4, 5));
+        assertEquals(List.of(), selector.get());
+    }
+
+
+    @Test
+    public void testGetAndRestart()
+    {
+        testBasicFunctionality();
+        // Reusing the selector which is reset by the get() call
+        testTopKWithNegativeNumbers();
+        // Also test resetting after getTransformed
+        testBasicFunctionality();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/asserts/SoftAssertionsWithLimit.java b/test/unit/org/apache/cassandra/utils/asserts/SoftAssertionsWithLimit.java
index 1e1d89207e68..c1d2d4ba81ed 100644
--- a/test/unit/org/apache/cassandra/utils/asserts/SoftAssertionsWithLimit.java
+++ b/test/unit/org/apache/cassandra/utils/asserts/SoftAssertionsWithLimit.java
@@ -33,10 +33,12 @@ public SoftAssertionsWithLimit(int limit)
     }
 
     @Override
-    public void onAssertionErrorCollected(AssertionError assertionError)
+    public void collectAssertionError(AssertionError assertionError)
     {
-        super.onAssertionErrorCollected(assertionError);
-        if (counter.incrementAndGet() >= limit)
-            assertAll();
+        int cnt = counter.incrementAndGet();
+        if (cnt < limit)
+            super.collectAssertionError(assertionError);
+        else if (cnt == limit)
+            super.collectAssertionError(new AssertionError("Too many assertion errors, stopping collecting them."));
     }
 }
diff --git a/test/unit/org/apache/cassandra/utils/binlog/BinLogTest.java b/test/unit/org/apache/cassandra/utils/binlog/BinLogTest.java
index 5227e51b51dc..096daa26e3b1 100644
--- a/test/unit/org/apache/cassandra/utils/binlog/BinLogTest.java
+++ b/test/unit/org/apache/cassandra/utils/binlog/BinLogTest.java
@@ -36,9 +36,9 @@
 import org.junit.Test;
 
 import net.openhft.chronicle.queue.ChronicleQueue;
-import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.queue.ExcerptTailer;
 import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.wire.WireOut;
 import org.apache.cassandra.Util;
 
diff --git a/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java b/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
index 004dfe073d0a..825210bdac35 100644
--- a/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
+++ b/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
@@ -458,4 +458,36 @@ public void clear()
             Arrays.fill(numberOfCalls, 0);
         }
     }
+
+    @Test
+    public void testReduce()
+    {
+        List<Integer> input = seq(71);
+        Object[] btree = BTree.build(input, UpdateFunction.noOp());
+
+        List<Integer> result = BTree.<List<Integer>, Integer>reduce(btree, new ArrayList(), (r, i) -> { r.add(i); return r; });
+        assertArrayEquals(input.toArray(), result.toArray());
+
+        // test interrupting after i items
+        for (int i = 1; i < input.size(); i++)
+        {
+            final int max = i;
+            result = BTree.reduce(btree, new ArrayList(), new BTree.ReduceFunction<List<Integer>, Integer>()
+            {
+                public boolean stop(List<Integer> res)
+                {
+                    return res.size() == max;
+                }
+
+                public List<Integer> apply(List<Integer> ret, Integer val)
+                {
+                    ret.add(val);
+                    return ret;
+                }
+            });
+
+            assertArrayEquals(seq(max).toArray(), result.toArray());
+        }
+    }
+
 }
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java
index 6fb70a3409cc..8abd0c1d1bbc 100644
--- a/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java
@@ -68,7 +68,8 @@ public class AbstractTypeByteSourceTest
     @Parameterized.Parameters(name = "version={0}")
     public static Iterable<ByteComparable.Version> versions()
     {
-        return ImmutableList.of(ByteComparable.Version.OSS50);
+        return ImmutableList.of(ByteComparable.Version.OSS41,
+                                ByteComparable.Version.OSS50);
     }
 
     private final ByteComparable.Version version;
@@ -708,8 +709,8 @@ public void testReversedType()
     {
         // Test how ReversedType handles null ByteSource.Peekable - here the choice of base type is important, as
         // the base type should also be able to handle null ByteSource.Peekable.
-        ReversedType<BigInteger> reversedVarintType = ReversedType.getInstance(IntegerType.instance);
-        ByteBuffer decodedNull = reversedVarintType.fromComparableBytes(null, ByteComparable.Version.OSS50);
+        ReversedType<BigInteger> reversedVarintType = (ReversedType<BigInteger>) ReversedType.getInstance(IntegerType.instance);
+        ByteBuffer decodedNull = reversedVarintType.fromComparableBytes(null, version);
         Assert.assertEquals(ByteBufferUtil.EMPTY_BYTE_BUFFER, decodedNull);
 
         // Test how ReversedType handles random data with some common and important base types.
@@ -745,7 +746,7 @@ public void testReversedType()
         Random prng = new Random();
         for (Map.Entry<AbstractType<?>, BiFunction<Random, Integer, ByteBuffer>> entry : bufferGeneratorByType.entrySet())
         {
-            ReversedType<?> reversedType = ReversedType.getInstance(entry.getKey());
+            ReversedType<?> reversedType = (ReversedType<?>) ReversedType.getInstance(entry.getKey());
             for (int length = 32; length <= 512; length *= 4)
             {
                 for (int i = 0; i < 100; ++i)
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java
index 56309e82c1cc..2ce3724c8ce6 100644
--- a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java
@@ -21,7 +21,17 @@
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -43,7 +53,40 @@
 import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.ClusteringPrefix;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DateType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.DynamicCompositeType;
+import org.apache.cassandra.db.marshal.DynamicCompositeTypeTest;
+import org.apache.cassandra.db.marshal.EmptyType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.LexicalUUIDType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.SimpleDateType;
+import org.apache.cassandra.db.marshal.TimeType;
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.marshal.ValueAccessor;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.LocalPartitioner;
@@ -76,6 +119,7 @@ public void testStringsAscii()
     public void testStringsUTF8()
     {
         testType(UTF8Type.instance, testStrings);
+        testDirect(x -> ByteSource.of(x, Version.OSS41), Ordering.<String>natural()::compare, testStrings);
         testDirect(x -> ByteSource.of(x, Version.OSS50), Ordering.<String>natural()::compare, testStrings);
     }
 
@@ -363,16 +407,18 @@ public void testNullsInClustering()
     public void testNullsInClusteringLegacy()
     {
         // verify the legacy encoding treats null clustering the same as null value
-        ClusteringPrefix<ByteBuffer> aNull = makeBound(ClusteringPrefix.Kind.CLUSTERING,
-                                                       decomposeAndRandomPad(UTF8Type.instance, "a"),
-                                                       decomposeAndRandomPad(Int32Type.instance, null));
-        ClusteringPrefix<ByteBuffer> aEmpty = makeBound(ClusteringPrefix.Kind.CLUSTERING,
-                                                        decomposeAndRandomPad(UTF8Type.instance, "a"),
-                                                        null);
+        ClusteringPrefix<?> aNull = makeBound(ClusteringPrefix.Kind.CLUSTERING,
+                                              decomposeAndRandomPad(UTF8Type.instance, "a"),
+                                              decomposeAndRandomPad(Int32Type.instance, null));
+        ClusteringPrefix<?> aEmpty = makeBound(ClusteringPrefix.Kind.CLUSTERING,
+                                               decomposeAndRandomPad(UTF8Type.instance, "a"),
+                                               null);
         ClusteringComparator comp = new ClusteringComparator(UTF8Type.instance, Int32Type.instance);
         assertEquals(0, ByteComparable.compare(comp.asByteComparable(aNull), comp.asByteComparable(aEmpty), Version.LEGACY));
+        assertEquals(0, ByteComparable.compare(comp.asByteComparable(aNull), comp.asByteComparable(aEmpty), Version.OSS41));
         ClusteringComparator compReversed = new ClusteringComparator(UTF8Type.instance, ReversedType.getInstance(Int32Type.instance));
         assertEquals(0, ByteComparable.compare(compReversed.asByteComparable(aNull), compReversed.asByteComparable(aEmpty), Version.LEGACY));
+        assertEquals(0, ByteComparable.compare(compReversed.asByteComparable(aNull), compReversed.asByteComparable(aEmpty), Version.OSS41));
     }
 
     @Test
@@ -383,6 +429,11 @@ public void testEmptyClustering()
         assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.INCL_START_BOUND, Version.OSS50);
         assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.INCL_END_BOUND, Version.OSS50);
 
+        assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.CLUSTERING, Version.OSS41);
+        assertEmptyComparedToStatic(0, ClusteringPrefix.Kind.STATIC_CLUSTERING, Version.OSS41);
+        assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.INCL_START_BOUND, Version.OSS41);
+        assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.INCL_END_BOUND, Version.OSS41);
+
         assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.CLUSTERING, Version.LEGACY);
         assertEmptyComparedToStatic(0, ClusteringPrefix.Kind.STATIC_CLUSTERING, Version.LEGACY);
         assertEmptyComparedToStatic(-1, ClusteringPrefix.Kind.INCL_START_BOUND, Version.LEGACY);
@@ -391,7 +442,7 @@ public void testEmptyClustering()
 
     private void assertEmptyComparedToStatic(int expected, ClusteringPrefix.Kind kind, Version version)
     {
-        ClusteringPrefix<ByteBuffer> empty = makeBound(kind);
+        ClusteringPrefix<?> empty = makeBound(kind);
         ClusteringComparator compEmpty = new ClusteringComparator();
         assertEquals(expected, Integer.signum(ByteComparable.compare(compEmpty.asByteComparable(empty),
                                                                      compEmpty.asByteComparable(Clustering.STATIC_CLUSTERING),
@@ -407,53 +458,63 @@ void assertClusteringPairComparesSame(AbstractType<?> t1, AbstractType<?> t2,
                                           Object o1, Object o2, Object o3, Object o4,
                                           BiFunction<AbstractType, Object, ByteBuffer> decompose,
                                           boolean testLegacy)
+    {
+        // Explicitly pass version values to have the failing kind in the stack trace.
+        if (testLegacy)
+            assertClusteringPairComparesSame(t1, t2, o1, o2, o3, o4, decompose, Version.LEGACY);
+        if (o1 != null && o2 != null && o3 != null && o4 != null)
+            assertClusteringPairComparesSame(t1, t2, o1, o2, o3, o4, decompose, Version.OSS41);
+
+        assertClusteringPairComparesSame(t1, t2, o1, o2, o3, o4, decompose, Version.OSS50);
+    }
+
+    void assertClusteringPairComparesSame(AbstractType<?> t1, AbstractType<?> t2,
+                                          Object o1, Object o2, Object o3, Object o4,
+                                          BiFunction<AbstractType, Object, ByteBuffer> decompose,
+                                          ByteComparable.Version v)
     {
         EnumSet<ClusteringPrefix.Kind> skippedKinds = EnumSet.of(ClusteringPrefix.Kind.SSTABLE_LOWER_BOUND, ClusteringPrefix.Kind.SSTABLE_UPPER_BOUND);
-        for (Version v : Version.values())
-            for (ClusteringPrefix.Kind k1 : EnumSet.complementOf(skippedKinds))
-                for (ClusteringPrefix.Kind k2 : EnumSet.complementOf(skippedKinds))
-                {
-                    if (!testLegacy && v == Version.LEGACY)
-                        continue;
-
-                    ClusteringComparator comp = new ClusteringComparator(t1, t2);
-                    ByteBuffer[] b = new ByteBuffer[2];
-                    ByteBuffer[] d = new ByteBuffer[2];
-                    b[0] = decompose.apply(t1, o1);
-                    b[1] = decompose.apply(t2, o2);
-                    d[0] = decompose.apply(t1, o3);
-                    d[1] = decompose.apply(t2, o4);
-                    ClusteringPrefix<ByteBuffer> c = makeBound(k1, b);
-                    ClusteringPrefix<ByteBuffer> e = makeBound(k2, d);
-                    final ByteComparable bsc = comp.asByteComparable(c);
-                    final ByteComparable bse = comp.asByteComparable(e);
-                    int expected = Integer.signum(comp.compare(c, e));
-                    assertEquals(String.format("Failed comparing %s and %s, %s vs %s version %s",
-                                               safeStr(c.clusteringString(comp.subtypes())),
-                                               safeStr(e.clusteringString(comp.subtypes())), bsc, bse, v),
-                                 expected, Integer.signum(ByteComparable.compare(bsc, bse, v)));
-                    maybeCheck41Properties(expected, bsc, bse, v);
-                    maybeAssertNotPrefix(bsc, bse, v);
-
-                    ClusteringComparator compR = new ClusteringComparator(ReversedType.getInstance(t1), ReversedType.getInstance(t2));
-                    final ByteComparable bsrc = compR.asByteComparable(c);
-                    final ByteComparable bsre = compR.asByteComparable(e);
-                    int expectedR = Integer.signum(compR.compare(c, e));
-                    assertEquals(String.format("Failed comparing reversed %s and %s, %s vs %s version %s",
-                                               safeStr(c.clusteringString(comp.subtypes())),
-                                               safeStr(e.clusteringString(comp.subtypes())), bsrc, bsre, v),
-                                 expectedR, Integer.signum(ByteComparable.compare(bsrc, bsre, v)));
-                    maybeCheck41Properties(expectedR, bsrc, bsre, v);
-                    maybeAssertNotPrefix(bsrc, bsre, v);
-                }
+        for (ClusteringPrefix.Kind k1 : EnumSet.complementOf(skippedKinds))
+            for (ClusteringPrefix.Kind k2 : EnumSet.complementOf(skippedKinds))
+            {
+                ClusteringComparator comp = new ClusteringComparator(t1, t2);
+                ByteBuffer[] b = new ByteBuffer[2];
+                ByteBuffer[] d = new ByteBuffer[2];
+                b[0] = decompose.apply(t1, o1);
+                b[1] = decompose.apply(t2, o2);
+                d[0] = decompose.apply(t1, o3);
+                d[1] = decompose.apply(t2, o4);
+                ClusteringPrefix<?> c = makeBound(k1, b);
+                ClusteringPrefix<?> e = makeBound(k2, d);
+                final ByteComparable bsc = comp.asByteComparable(c);
+                final ByteComparable bse = comp.asByteComparable(e);
+                int expected = Integer.signum(comp.compare(c, e));
+                assertEquals(String.format("Failed comparing %s and %s, %s vs %s version %s",
+                                           safeStr(c.clusteringString(comp.subtypes())),
+                                           safeStr(e.clusteringString(comp.subtypes())), bsc, bse, v),
+                             expected, Integer.signum(ByteComparable.compare(bsc, bse, v)));
+                maybeCheck41Properties(expected, bsc, bse, v);
+                maybeAssertNotPrefix(bsc, bse, v);
+
+                ClusteringComparator compR = new ClusteringComparator(ReversedType.getInstance(t1), ReversedType.getInstance(t2));
+                final ByteComparable bsrc = compR.asByteComparable(c);
+                final ByteComparable bsre = compR.asByteComparable(e);
+                int expectedR = Integer.signum(compR.compare(c, e));
+                assertEquals(String.format("Failed comparing reversed %s and %s, %s vs %s version %s",
+                                           safeStr(c.clusteringString(comp.subtypes())),
+                                           safeStr(e.clusteringString(comp.subtypes())), bsrc, bsre, v),
+                             expectedR, Integer.signum(ByteComparable.compare(bsrc, bsre, v)));
+                maybeCheck41Properties(expectedR, bsrc, bsre, v);
+                maybeAssertNotPrefix(bsrc, bsre, v);
+            }
     }
 
-    static ClusteringPrefix<ByteBuffer> makeBound(ClusteringPrefix.Kind k1, ByteBuffer... b)
+    static ClusteringPrefix<?> makeBound(ClusteringPrefix.Kind k1, ByteBuffer... b)
     {
         return makeBound(ByteBufferAccessor.instance.factory(), k1, b);
     }
 
-    static <T> ClusteringPrefix<T> makeBound(ValueAccessor.ObjectFactory<T> factory, ClusteringPrefix.Kind k1, T[] b)
+    static <V> ClusteringPrefix<?> makeBound(ValueAccessor.ObjectFactory<V> factory, ClusteringPrefix.Kind k1, V[] b)
     {
         switch (k1)
         {
@@ -471,7 +532,7 @@ static <T> ClusteringPrefix<T> makeBound(ValueAccessor.ObjectFactory<T> factory,
             return factory.clustering(b);
 
         case STATIC_CLUSTERING:
-            return factory.staticClustering();
+            return Clustering.STATIC_CLUSTERING;
 
         default:
             throw new AssertionError(k1);
@@ -679,15 +740,15 @@ public void testDecoratedKeyPrefixesVLegacy()
     }
 
     @Test
-    public void testFixedLengthWithOffset()
+    public void testPreencodedWithOffset()
     {
         byte[] bytes = new byte[]{ 1, 2, 3, 4, 5, 6, 7, 8, 9 };
 
-        ByteSource source = ByteSource.fixedLength(bytes, 0, 1);
+        ByteSource source = ByteSource.preencoded(bytes, 0, 1);
         assertEquals(1, source.next());
         assertEquals(ByteSource.END_OF_STREAM, source.next());
 
-        source = ByteSource.fixedLength(bytes, 4, 5);
+        source = ByteSource.preencoded(bytes, 4, 5);
         assertEquals(5, source.next());
         assertEquals(6, source.next());
         assertEquals(7, source.next());
@@ -695,35 +756,35 @@ public void testFixedLengthWithOffset()
         assertEquals(9, source.next());
         assertEquals(ByteSource.END_OF_STREAM, source.next());
 
-        ByteSource.fixedLength(bytes, 9, 0);
+        ByteSource.preencoded(bytes, 9, 0);
         assertEquals(ByteSource.END_OF_STREAM, source.next());
     }
 
     @Test
-    public void testFixedLengthNegativeLength()
+    public void testPreencodedNegativeLength()
     {
         byte[] bytes = new byte[]{ 1, 2, 3 };
 
         expectedException.expect(IllegalArgumentException.class);
-        ByteSource.fixedLength(bytes, 0, -1);
+        ByteSource.preencoded(bytes, 0, -1);
     }
 
     @Test
-    public void testFixedLengthNegativeOffset()
+    public void testPreencodedNegativeOffset()
     {
         byte[] bytes = new byte[]{ 1, 2, 3 };
 
         expectedException.expect(IllegalArgumentException.class);
-        ByteSource.fixedLength(bytes, -1, 1);
+        ByteSource.preencoded(bytes, -1, 1);
     }
 
     @Test
-    public void testFixedLengthOutOfBounds()
+    public void testPreencodedOutOfBounds()
     {
         byte[] bytes = new byte[]{ 1, 2, 3 };
 
         expectedException.expect(IllegalArgumentException.class);
-        ByteSource.fixedLength(bytes, 0, 4);
+        ByteSource.preencoded(bytes, 0, 4);
     }
 
     @Test
@@ -732,7 +793,7 @@ public void testFixedOffsetOutOfBounds()
         byte[] bytes = new byte[]{ 1, 2, 3 };
 
         expectedException.expect(IllegalArgumentException.class);
-        ByteSource.fixedLength(bytes, 4, 1);
+        ByteSource.preencoded(bytes, 4, 1);
     }
 
     @Test
@@ -759,9 +820,7 @@ public void testSeparatorPrefixViaDiffPoint()
     public void testSeparatorNext()
     {
         // Appending a 00 byte at the end gives the immediate next possible value after x.
-        testSeparator((x, y) -> version -> ByteSource.cutOrRightPad(x.asComparableBytes(version),
-                                                                    ByteComparable.length(x, version) + 1,
-                                                                    0),
+        testSeparator((x, y) -> version -> ByteSource.append(x.asComparableBytes(version), 0),
                       testLongs,
                       LongType.instance);
     }
@@ -878,13 +937,13 @@ private int compare(ByteSource s1, ByteSource s2)
 
     private void maybeAssertNotPrefix(ByteComparable s1, ByteComparable s2, Version version)
     {
-        if (version == Version.OSS50)
+        if (version != Version.LEGACY)
             assertNotPrefix(s1.asComparableBytes(version), s2.asComparableBytes(version));
     }
 
     private void maybeCheck41Properties(int expectedComparison, ByteComparable s1, ByteComparable s2, Version version)
     {
-        if (version != Version.OSS50)
+        if (version == Version.LEGACY)
             return;
 
         if (s1 == null || s2 == null || 0 == expectedComparison)
@@ -896,15 +955,12 @@ private void maybeCheck41Properties(int expectedComparison, ByteComparable s1, B
         assertNotPrefix(ByteSource.withTerminator(b1, s1.asComparableBytes(version)), ByteSource.withTerminator(b2, s2.asComparableBytes(version)));
     }
 
-    private int randomTerminator()
+    private static int randomTerminator()
     {
-        int term;
-        do
-        {
-            term = ThreadLocalRandom.current().nextInt(ByteSource.MIN_SEPARATOR, ByteSource.MAX_SEPARATOR + 1);
-        }
-        while (term >= ByteSource.MIN_NEXT_COMPONENT && term <= ByteSource.MAX_NEXT_COMPONENT);
-        return term;
+        var rand = ThreadLocalRandom.current();
+        return rand.nextBoolean()
+               ? rand.nextInt(ByteSource.MIN_SEPARATOR, ByteSource.MIN_NEXT_COMPONENT)
+               : rand.nextInt(ByteSource.MAX_NEXT_COMPONENT + 1, ByteSource.MAX_SEPARATOR + 1);
     }
 
     <K, V, M extends Map<K, V>> void testMap(MapType<K, V> tt, K[] keys, V[] values, Supplier<M> gen, Random rand)
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java
index 95ea61418694..85b8e7fd2ebd 100644
--- a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java
@@ -20,7 +20,17 @@
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -32,6 +42,8 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,7 +51,41 @@
 import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.ClusteringPrefix;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DateType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.DynamicCompositeType;
+import org.apache.cassandra.db.marshal.DynamicCompositeTypeTest;
+import org.apache.cassandra.db.marshal.EmptyType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.LexicalUUIDType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.SimpleDateType;
+import org.apache.cassandra.db.marshal.TimeType;
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.db.marshal.ValueAccessors;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.LocalPartitioner;
@@ -47,7 +93,6 @@
 import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
 
 import static org.apache.cassandra.utils.bytecomparable.ByteSourceComparisonTest.decomposeForTuple;
 import static org.junit.Assert.assertEquals;
@@ -55,10 +100,25 @@
 /**
  * Tests that the result of forward + backward ByteSource translation is the same as the original.
  */
+@RunWith(Parameterized.class)
 public class ByteSourceConversionTest extends ByteSourceTestBase
 {
+
+    @Parameterized.Parameters(name = "version={0}")
+    public static Iterable<ByteComparable.Version> versions()
+    {
+        return ImmutableList.of(ByteComparable.Version.OSS41,
+                                ByteComparable.Version.OSS50);
+    }
+
+    private final ByteComparable.Version version;
+
+    public ByteSourceConversionTest(ByteComparable.Version version)
+    {
+        this.version = version;
+    }
+
     private final static Logger logger = LoggerFactory.getLogger(ByteSourceConversionTest.class);
-    public static final Version VERSION = Version.OSS50;
 
     @Rule
     public final ExpectedException expectedException = ExpectedException.none();
@@ -76,7 +136,7 @@ public void testStringsAscii()
     public void testStringsUTF8()
     {
         testType(UTF8Type.instance, testStrings);
-        testDirect(x -> ByteSource.of(x, VERSION), ByteSourceInverse::getString, testStrings);
+        testDirect(x -> ByteSource.of(x, version), ByteSourceInverse::getString, testStrings);
     }
 
     @Test
@@ -333,8 +393,8 @@ public void testEmptyClustering()
             if (kind.isBoundary())
                 continue;
 
-            ClusteringPrefix<ByteBuffer> empty = ByteSourceComparisonTest.makeBound(kind);
-            ClusteringPrefix<ByteBuffer> converted = getClusteringPrefix(accessor, kind, comp, comp.asByteComparable(empty));
+            ClusteringPrefix<?> empty = ByteSourceComparisonTest.makeBound(kind);
+            ClusteringPrefix<?> converted = getClusteringPrefix(accessor, kind, comp, comp.asByteComparable(empty));
             assertEquals(empty, converted);
         }
     }
@@ -358,21 +418,21 @@ <V> void assertClusteringPairConvertsSame(ValueAccessor<V> accessor,
                 V[] b = accessor.createArray(2);
                 b[0] = accessor.valueOf(decompose.apply(t1, o1));
                 b[1] = accessor.valueOf(decompose.apply(t2, o2));
-                ClusteringPrefix<V> c = ByteSourceComparisonTest.makeBound(accessor.factory(), k1, b);
+                ClusteringPrefix<?> c = ByteSourceComparisonTest.makeBound(accessor.factory(), k1, b);
                 final ByteComparable bsc = comp.asByteComparable(c);
-                logger.info("Clustering {} bytesource {}", c.clusteringString(comp.subtypes()), bsc.byteComparableAsString(VERSION));
-                ClusteringPrefix<V> converted = getClusteringPrefix(accessor, k1, comp, bsc);
+                logger.info("Clustering {} bytesource {}", c.clusteringString(comp.subtypes()), bsc.byteComparableAsString(version));
+                ClusteringPrefix<?> converted = getClusteringPrefix(accessor, k1, comp, bsc);
                 assertEquals(String.format("Failed compare(%s, converted %s ByteSource %s) == 0\ntype %s",
                                            safeStr(c.clusteringString(comp.subtypes())),
                                            safeStr(converted.clusteringString(comp.subtypes())),
-                                           bsc.byteComparableAsString(VERSION),
+                                           bsc.byteComparableAsString(version),
                                            comp),
                              0, comp.compare(c, converted));
                 if (checkEquals)
                     assertEquals(String.format("Failed equals %s, got %s ByteSource %s\ntype %s",
                                                safeStr(c.clusteringString(comp.subtypes())),
                                                safeStr(converted.clusteringString(comp.subtypes())),
-                                               bsc.byteComparableAsString(VERSION),
+                                               bsc.byteComparableAsString(version),
                                                comp),
                                  c, converted);
 
@@ -382,20 +442,20 @@ <V> void assertClusteringPairConvertsSame(ValueAccessor<V> accessor,
                 assertEquals(String.format("Failed reverse compare(%s, converted %s ByteSource %s) == 0\ntype %s",
                                            safeStr(c.clusteringString(compR.subtypes())),
                                            safeStr(converted.clusteringString(compR.subtypes())),
-                                           bsrc.byteComparableAsString(VERSION),
+                                           bsrc.byteComparableAsString(version),
                                            compR),
                              0, compR.compare(c, converted));
                 if (checkEquals)
                     assertEquals(String.format("Failed reverse equals %s, got %s ByteSource %s\ntype %s",
                                                safeStr(c.clusteringString(compR.subtypes())),
                                                safeStr(converted.clusteringString(compR.subtypes())),
-                                               bsrc.byteComparableAsString(VERSION),
+                                               bsrc.byteComparableAsString(version),
                                                compR),
                                  c, converted);
             }
     }
 
-    private static <V> ClusteringPrefix<V> getClusteringPrefix(ValueAccessor<V> accessor,
+    private static <V> ClusteringPrefix<?> getClusteringPrefix(ValueAccessor<V> accessor,
                                                                ClusteringPrefix.Kind k1,
                                                                ClusteringComparator comp,
                                                                ByteComparable bsc)
@@ -419,11 +479,11 @@ private static <V> ClusteringPrefix<V> getClusteringPrefix(ValueAccessor<V> acce
         }
     }
 
-    private static ByteSource.Peekable source(ByteComparable bsc)
+    private ByteSource.Peekable source(ByteComparable bsc)
     {
         if (bsc == null)
             return null;
-        return ByteSource.peekable(bsc.asComparableBytes(VERSION));
+        return ByteSource.peekable(bsc.asComparableBytes(version));
     }
 
     @Test
@@ -626,7 +686,7 @@ public <T> void testType(AbstractType<? super T> type, Iterable<T> values)
                               safeStr(i),
                               safeStr(type.getSerializer().toCQLLiteral(b)),
                               safeStr(ByteBufferUtil.bytesToHex(b)),
-                              typeToComparable(type, b).byteComparableAsString(VERSION));
+                              typeToComparable(type, b).byteComparableAsString(version));
             assertConvertsSame(type, i);
         }
         if (!type.isReversed())
@@ -652,7 +712,7 @@ public void testBuffers(AbstractType<?> type, List<ByteBuffer> values)
                 logger.info("Value {} bytes {} ByteSource {}",
                             safeStr(type.getSerializer().toCQLLiteral(b)),
                             safeStr(ByteBufferUtil.bytesToHex(b)),
-                            typeToComparable(type, b).byteComparableAsString(VERSION));
+                            typeToComparable(type, b).byteComparableAsString(version));
             }
         }
         catch (UnsupportedOperationException e)
@@ -668,11 +728,11 @@ void assertConvertsSameBuffers(AbstractType<?> type, ByteBuffer b1)
     {
         final ByteComparable bs1 = typeToComparable(type, b1);
 
-        ByteBuffer actual = type.fromComparableBytes(source(bs1), VERSION);
+        ByteBuffer actual = type.fromComparableBytes(source(bs1), version);
         assertEquals(String.format("Failed compare(%s, converted %s (bytesource %s))",
                                    ByteBufferUtil.bytesToHex(b1),
                                    ByteBufferUtil.bytesToHex(actual),
-                                   bs1.byteComparableAsString(VERSION)),
+                                   bs1.byteComparableAsString(version)),
                      0,
                      type.compare(b1, actual));
     }
@@ -686,25 +746,25 @@ public void testDecoratedKeys(IPartitioner type, List<ByteBuffer> values)
     void assertConvertsSameDecoratedKeys(IPartitioner type, ByteBuffer b1)
     {
         DecoratedKey k1 = type.decorateKey(b1);
-        DecoratedKey actual = BufferDecoratedKey.fromByteComparable(k1, VERSION, type);
+        DecoratedKey actual = BufferDecoratedKey.fromByteComparable(k1, version, type);
 
         assertEquals(String.format("Failed compare(%s[%s bs %s], %s[%s bs %s])\npartitioner %s",
                                    k1,
                                    ByteBufferUtil.bytesToHex(b1),
-                                   k1.byteComparableAsString(VERSION),
+                                   k1.byteComparableAsString(version),
                                    actual,
                                    ByteBufferUtil.bytesToHex(actual.getKey()),
-                                   actual.byteComparableAsString(VERSION),
+                                   actual.byteComparableAsString(version),
                                    type),
                      0,
                      k1.compareTo(actual));
         assertEquals(String.format("Failed equals(%s[%s bs %s], %s[%s bs %s])\npartitioner %s",
                                    k1,
                                    ByteBufferUtil.bytesToHex(b1),
-                                   k1.byteComparableAsString(VERSION),
+                                   k1.byteComparableAsString(version),
                                    actual,
                                    ByteBufferUtil.bytesToHex(actual.getKey()),
-                                   actual.byteComparableAsString(VERSION),
+                                   actual.byteComparableAsString(version),
                                    type),
                      k1,
                      actual);
@@ -745,20 +805,20 @@ <T> void assertConvertsSame(Function<T, ByteSource> convertor, Function<ByteSour
     {
         ByteComparable b1 = v -> convertor.apply(v1);
         T actual = inverse.apply(source(b1));
-        assertEquals(String.format("ByteSource %s", b1.byteComparableAsString(VERSION)), v1, actual);
+        assertEquals(String.format("ByteSource %s", b1.byteComparableAsString(version)), v1, actual);
     }
 
     <T> void assertConvertsSame(AbstractType<T> type, T v1)
     {
         ByteBuffer b1 = decomposeAndRandomPad(type, v1);
         final ByteComparable bc1 = typeToComparable(type, b1);
-        ByteBuffer convertedBuffer = type.fromComparableBytes(source(bc1), VERSION);
+        ByteBuffer convertedBuffer = type.fromComparableBytes(source(bc1), version);
         T actual = type.compose(convertedBuffer);
 
         assertEquals(String.format("Failed equals %s(%s bs %s), got %s",
                                    safeStr(v1),
                                    ByteBufferUtil.bytesToHex(b1),
-                                   safeStr(bc1.byteComparableAsString(VERSION)),
+                                   safeStr(bc1.byteComparableAsString(version)),
                                    safeStr(actual)),
                      v1,
                      actual);
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceDuplicationTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceDuplicationTest.java
new file mode 100644
index 000000000000..92ab66895df3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceDuplicationTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Test;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+
+public class ByteSourceDuplicationTest extends ByteSourceTestBase
+{
+    public static final ByteComparable.Version VERSION = ByteComparable.Version.OSS50;
+    private static final int TRIES = 25;
+    private static final double TARGET_DUPES_PER_TRY = 7;
+    Random rand = new Random(1);
+
+    @Test
+    public void testDuplication()
+    {
+        for (int i = 0; i < testValues.length; ++i)
+            for (Object o : testValues[i])
+                testDuplication(typeToComparable(testTypes[i], o), "Type " + testTypes[i].getClass().getSimpleName() + " value " + o);
+    }
+
+    @Test
+    public void testDuplicationPairs()
+    {
+        for (int i = 0; i < testValues.length; ++i)
+            for (Object o : testValues[i])
+                testDuplication(typeToComparableInPair(testTypes[i], o), "Type " + testTypes[i].getClass().getSimpleName() + " value " + o + " in pair");
+    }
+
+    public ByteComparable typeToComparable(AbstractType t, Object o)
+    {
+        return v -> t.asComparableBytes(t.decompose(o), v);
+    }
+
+    public ByteComparable typeToComparableInPair(AbstractType t, Object o)
+    {
+        int ti = rand.nextInt(testValues.length);
+        Object oo = testValues[ti][rand.nextInt(testValues[ti].length)];
+        AbstractType tt = testTypes[ti];
+        return v -> ByteSource.withTerminator(ByteSource.TERMINATOR,
+                                              t.asComparableBytes(t.decompose(o), v),
+                                              tt.asComparableBytes(tt.decompose(oo), v));
+    }
+
+    private void testDuplication(ByteComparable comparable, String msg)
+    {
+        if (comparable == null || comparable.asComparableBytes(VERSION) == null)
+            return; // nothing to check
+
+        byte[] bytes = ByteSourceInverse.readBytes(comparable.asComparableBytes(VERSION));
+        msg += " encoding " + ByteBufferUtil.bytesToHex(ByteBuffer.wrap(bytes));
+        double dupeProbability = Math.min(0.6, TARGET_DUPES_PER_TRY / bytes.length); // A few duplications per entry on average
+
+        IntArrayList positions = new IntArrayList();
+        List<ByteSource> sources = new ArrayList<>();
+        for (int test = 0; test < Math.max(1, TRIES / bytes.length); ++test)
+        {
+            sources.add(comparable.asComparableBytes(VERSION));
+            positions.add(0);
+
+            while (!sources.isEmpty())
+            {
+                int index = rand.nextInt(sources.size());
+                int pos = positions.getInt(index);
+                ByteSource source = sources.get(index);
+                if (rand.nextDouble() <= dupeProbability)
+                {
+                    ByteSource.Duplicatable duplicatable = ByteSource.duplicatable(source);
+                    sources.set(index, duplicatable);
+                    source = duplicatable;
+                    ByteSource duplicate = duplicatable.duplicate();
+                    sources.add(duplicate);
+                    positions.add(pos);
+                }
+                int next = source.next();
+                if (next == ByteSource.END_OF_STREAM)
+                {
+                    assertEquals(msg, bytes.length, pos);
+                    sources.remove(index);
+                    positions.remove(index);
+                }
+                else
+                {
+                    assertEquals(msg, bytes[pos++], (byte) next);
+                    positions.setInt(index, pos);
+                }
+            }
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceInverseTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceInverseTest.java
index 1fd6cc202bb6..f849d952cd86 100644
--- a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceInverseTest.java
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceInverseTest.java
@@ -17,27 +17,36 @@
  */
 package org.apache.cassandra.utils.bytecomparable;
 
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.memory.MemoryUtil;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.IntConsumer;
 import java.util.function.LongConsumer;
-import java.util.stream.*;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+import java.util.stream.Stream;
+import org.apache.cassandra.utils.memory.MemoryUtil;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.utils.Throwables;
 
 @RunWith(Parameterized.class)
 public class ByteSourceInverseTest
@@ -47,7 +56,8 @@ public class ByteSourceInverseTest
     @Parameterized.Parameters(name = "version={0}")
     public static Iterable<ByteComparable.Version> versions()
     {
-        return ImmutableList.of(ByteComparable.Version.OSS50);
+        return ImmutableList.of(ByteComparable.Version.OSS41,
+                                ByteComparable.Version.OSS50);
     }
 
     private final ByteComparable.Version version;
@@ -158,14 +168,14 @@ public void testGetSignedShort()
     }
 
     @Test
-    public void testBadByteSourceForFixedLengthNumbers()
+    public void testBadByteSourceForPreencodedNumbers()
     {
         byte[] bytes = new byte[8];
         new Random().nextBytes(bytes);
         for (Map.Entry<String, Integer> entries : ImmutableMap.of("getSignedInt", 4,
-                  "getSignedLong", 8,
-                  "getSignedByte", 1,
-                  "getSignedShort", 2).entrySet())
+                                                                  "getSignedLong", 8,
+                                                                  "getSignedByte", 1,
+                                                                  "getSignedShort", 2).entrySet())
         {
             String methodName = entries.getKey();
             int length = entries.getValue();
@@ -176,7 +186,7 @@ public void testBadByteSourceForFixedLengthNumbers()
                 sources.add(null);
                 sources.add(ByteSource.EMPTY);
                 for (int i = 0; i < length; ++i)
-                    sources.add(ByteSource.fixedLength(bytes, 0, i));
+                    sources.add(ByteSource.preencoded(bytes, 0, i));
                 // Note: not testing invalid bytes (e.g. using the construction below) as they signify a programming
                 // error (throwing AssertionError) rather than something that could happen due to e.g. bad files.
                 //      ByteSource.withTerminatorLegacy(257, ByteSource.fixedLength(bytes, 0, length - 1));
@@ -302,7 +312,7 @@ public void testGetByteBuffer()
                 }
                 finally
                 {
-                    MemoryUtil.free(address);
+                    MemoryUtil.free(address, initialBytes.length);
                 }
             }
             ))
@@ -390,7 +400,7 @@ public void testReadBytes()
             // The best way to test the read bytes seems to be to assert that just directly using them as a
             // ByteSource (using ByteSource.fixedLength(byte[])) they compare as equal to another ByteSource obtained
             // from the same original value.
-            int compare = ByteComparable.compare(v -> originalSourceCopy, v -> ByteSource.fixedLength(bytes), version);
+            int compare = ByteComparable.compare(v -> originalSourceCopy, v -> ByteSource.preencoded(bytes), version);
             Assert.assertEquals(0, compare);
         }
     }
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceSequenceTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceSequenceTest.java
index 532bd1b1a765..7bc9106f7c73 100644
--- a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceSequenceTest.java
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceSequenceTest.java
@@ -56,7 +56,8 @@ public class ByteSourceSequenceTest
     @Parameterized.Parameters(name = "version={0}")
     public static Iterable<ByteComparable.Version> versions()
     {
-        return ImmutableList.of(ByteComparable.Version.OSS50);
+        return ImmutableList.of(ByteComparable.Version.OSS41,
+                                ByteComparable.Version.OSS50);
     }
 
     private final ByteComparable.Version version;
@@ -489,7 +490,7 @@ public void testGetBoundFromPrefixTerminator()
             assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
             DECIMAL.fromComparableBytes(comparableBytes, version);
             // Expect null-signifying separator here.
-            assertEquals(ByteSource.NEXT_COMPONENT_EMPTY, comparableBytes.next());
+            assertEquals(ByteSource.NEXT_COMPONENT_NULL, comparableBytes.next());
             // No varint to read
             // Expect the last separator (i.e. the terminator) to be the one specified by the prefix kind.
             assertEquals(prefixKind.asByteComparableValue(version), comparableBytes.next());
@@ -506,7 +507,7 @@ public void testGetBoundFromPrefixTerminator()
             assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
             DECIMAL.fromComparableBytes(comparableBytes, version);
             // Expect reversed null-signifying separator here.
-            assertEquals(ByteSource.NEXT_COMPONENT_EMPTY_REVERSED, comparableBytes.next());
+            assertEquals(ByteSource.NEXT_COMPONENT_NULL_REVERSED, comparableBytes.next());
             // No varint to read
             // Expect the last separator (i.e. the terminator) to be the one specified by the prefix kind.
             assertEquals(prefixKind.asByteComparableValue(version), comparableBytes.next());
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceTestBase.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceTestBase.java
index 050bec517130..d0fcf82da613 100644
--- a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceTestBase.java
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceTestBase.java
@@ -41,7 +41,9 @@
 
 public class ByteSourceTestBase
 {
+    public static
     String[] testStrings = new String[]{ "", "\0", "\0\0", "\001", "A\0\0B", "A\0B\0", "0", "0\0", "00", "1", "\377" };
+    public static
     Integer[] testInts = new Integer[]{ null,
                                         Integer.MIN_VALUE,
                                         Integer.MIN_VALUE + 1,
@@ -62,7 +64,9 @@ public class ByteSourceTestBase
                                         256,
                                         Integer.MAX_VALUE - 1,
                                         Integer.MAX_VALUE };
+    public static
     Byte[] testBytes = new Byte[]{ -128, -127, -1, 0, 1, 127 };
+    public static
     Short[] testShorts = new Short[]{ Short.MIN_VALUE,
                                       Short.MIN_VALUE + 1,
                                       -256,
@@ -81,6 +85,7 @@ public class ByteSourceTestBase
                                       256,
                                       Short.MAX_VALUE - 1,
                                       Short.MAX_VALUE };
+    public static
     Long[] testLongs = new Long[]{ null,
                                    Long.MIN_VALUE,
                                    Long.MIN_VALUE + 1,
@@ -226,131 +231,132 @@ public class ByteSourceTestBase
                                    (1L << 63) - 1,
 
                                    ~((1L << 1) - 1),
-                                   ~((1L << 1)),
+                                   ~ (1L << 1),
                                    ~((1L << 2) - 1),
-                                   ~((1L << 2)),
+                                   ~ (1L << 2),
                                    ~((1L << 3) - 1),
-                                   ~((1L << 3)),
+                                   ~ (1L << 3),
                                    ~((1L << 4) - 1),
-                                   ~((1L << 4)),
+                                   ~ (1L << 4),
                                    ~((1L << 5) - 1),
-                                   ~((1L << 5)),
+                                   ~ (1L << 5),
                                    ~((1L << 6) - 1),
-                                   ~((1L << 6)),
+                                   ~ (1L << 6),
                                    ~((1L << 7) - 1),
-                                   ~((1L << 7)),
+                                   ~ (1L << 7),
                                    ~((1L << 8) - 1),
-                                   ~((1L << 8)),
+                                   ~ (1L << 8),
                                    ~((1L << 9) - 1),
-                                   ~((1L << 9)),
+                                   ~ (1L << 9),
                                    ~((1L << 10) - 1),
-                                   ~((1L << 10)),
+                                   ~ (1L << 10),
                                    ~((1L << 11) - 1),
-                                   ~((1L << 11)),
+                                   ~ (1L << 11),
                                    ~((1L << 12) - 1),
-                                   ~((1L << 12)),
+                                   ~ (1L << 12),
                                    ~((1L << 13) - 1),
-                                   ~((1L << 13)),
+                                   ~ (1L << 13),
                                    ~((1L << 14) - 1),
-                                   ~((1L << 14)),
+                                   ~ (1L << 14),
                                    ~((1L << 15) - 1),
-                                   ~((1L << 15)),
+                                   ~ (1L << 15),
                                    ~((1L << 16) - 1),
-                                   ~((1L << 16)),
+                                   ~ (1L << 16),
                                    ~((1L << 17) - 1),
-                                   ~((1L << 17)),
+                                   ~ (1L << 17),
                                    ~((1L << 18) - 1),
-                                   ~((1L << 18)),
+                                   ~ (1L << 18),
                                    ~((1L << 19) - 1),
-                                   ~((1L << 19)),
+                                   ~ (1L << 19),
                                    ~((1L << 20) - 1),
-                                   ~((1L << 20)),
+                                   ~ (1L << 20),
                                    ~((1L << 21) - 1),
-                                   ~((1L << 21)),
+                                   ~ (1L << 21),
                                    ~((1L << 22) - 1),
-                                   ~((1L << 22)),
+                                   ~ (1L << 22),
                                    ~((1L << 23) - 1),
-                                   ~((1L << 23)),
+                                   ~ (1L << 23),
                                    ~((1L << 24) - 1),
-                                   ~((1L << 24)),
+                                   ~ (1L << 24),
                                    ~((1L << 25) - 1),
-                                   ~((1L << 25)),
+                                   ~ (1L << 25),
                                    ~((1L << 26) - 1),
-                                   ~((1L << 26)),
+                                   ~ (1L << 26),
                                    ~((1L << 27) - 1),
-                                   ~((1L << 27)),
+                                   ~ (1L << 27),
                                    ~((1L << 28) - 1),
-                                   ~((1L << 28)),
+                                   ~ (1L << 28),
                                    ~((1L << 29) - 1),
-                                   ~((1L << 29)),
+                                   ~ (1L << 29),
                                    ~((1L << 30) - 1),
-                                   ~((1L << 30)),
+                                   ~ (1L << 30),
                                    ~((1L << 31) - 1),
-                                   ~((1L << 31)),
+                                   ~ (1L << 31),
                                    ~((1L << 32) - 1),
-                                   ~((1L << 32)),
+                                   ~ (1L << 32),
                                    ~((1L << 33) - 1),
-                                   ~((1L << 33)),
+                                   ~ (1L << 33),
                                    ~((1L << 34) - 1),
-                                   ~((1L << 34)),
+                                   ~ (1L << 34),
                                    ~((1L << 35) - 1),
-                                   ~((1L << 35)),
+                                   ~ (1L << 35),
                                    ~((1L << 36) - 1),
-                                   ~((1L << 36)),
+                                   ~ (1L << 36),
                                    ~((1L << 37) - 1),
-                                   ~((1L << 37)),
+                                   ~ (1L << 37),
                                    ~((1L << 38) - 1),
-                                   ~((1L << 38)),
+                                   ~ (1L << 38),
                                    ~((1L << 39) - 1),
-                                   ~((1L << 39)),
+                                   ~ (1L << 39),
                                    ~((1L << 40) - 1),
-                                   ~((1L << 40)),
+                                   ~ (1L << 40),
                                    ~((1L << 41) - 1),
-                                   ~((1L << 41)),
+                                   ~ (1L << 41),
                                    ~((1L << 42) - 1),
-                                   ~((1L << 42)),
+                                   ~ (1L << 42),
                                    ~((1L << 43) - 1),
-                                   ~((1L << 43)),
+                                   ~ (1L << 43),
                                    ~((1L << 44) - 1),
-                                   ~((1L << 44)),
+                                   ~ (1L << 44),
                                    ~((1L << 45) - 1),
-                                   ~((1L << 45)),
+                                   ~ (1L << 45),
                                    ~((1L << 46) - 1),
-                                   ~((1L << 46)),
+                                   ~ (1L << 46),
                                    ~((1L << 47) - 1),
-                                   ~((1L << 47)),
+                                   ~ (1L << 47),
                                    ~((1L << 48) - 1),
-                                   ~((1L << 48)),
+                                   ~ (1L << 48),
                                    ~((1L << 49) - 1),
-                                   ~((1L << 49)),
+                                   ~ (1L << 49),
                                    ~((1L << 50) - 1),
-                                   ~((1L << 50)),
+                                   ~ (1L << 50),
                                    ~((1L << 51) - 1),
-                                   ~((1L << 51)),
+                                   ~ (1L << 51),
                                    ~((1L << 52) - 1),
-                                   ~((1L << 52)),
+                                   ~ (1L << 52),
                                    ~((1L << 53) - 1),
-                                   ~((1L << 53)),
+                                   ~ (1L << 53),
                                    ~((1L << 54) - 1),
-                                   ~((1L << 54)),
+                                   ~ (1L << 54),
                                    ~((1L << 55) - 1),
-                                   ~((1L << 55)),
+                                   ~ (1L << 55),
                                    ~((1L << 56) - 1),
-                                   ~((1L << 56)),
+                                   ~ (1L << 56),
                                    ~((1L << 57) - 1),
-                                   ~((1L << 57)),
+                                   ~ (1L << 57),
                                    ~((1L << 58) - 1),
-                                   ~((1L << 58)),
+                                   ~ (1L << 58),
                                    ~((1L << 59) - 1),
-                                   ~((1L << 59)),
+                                   ~ (1L << 59),
                                    ~((1L << 60) - 1),
-                                   ~((1L << 60)),
+                                   ~ (1L << 60),
                                    ~((1L << 61) - 1),
-                                   ~((1L << 61)),
+                                   ~ (1L << 61),
                                    ~((1L << 62) - 1),
-                                   ~((1L << 62)),
+                                   ~ (1L << 62),
                                    ~((1L << 63) - 1),
                                    };
+    public static
     Double[] testDoubles = new Double[]{ null,
                                          Double.NEGATIVE_INFINITY,
                                          -Double.MAX_VALUE,
@@ -371,6 +377,7 @@ public class ByteSourceTestBase
                                          Double.MAX_VALUE,
                                          Double.POSITIVE_INFINITY,
                                          Double.NaN };
+    public static
     Float[] testFloats = new Float[]{ null,
                                       Float.NEGATIVE_INFINITY,
                                       -Float.MAX_VALUE,
@@ -391,7 +398,9 @@ public class ByteSourceTestBase
                                       Float.MAX_VALUE,
                                       Float.POSITIVE_INFINITY,
                                       Float.NaN };
+    public static
     Boolean[] testBools = new Boolean[]{ null, false, true };
+    public static
     UUID[] testUUIDs = new UUID[]{ null,
                                    TimeUUID.Generator.nextTimeAsUUID(),
                                    UUID.randomUUID(),
@@ -408,6 +417,7 @@ public class ByteSourceTestBase
                                    UUID.fromString("52df1bb0-6a2f-11e6-362d-aff2143498ea"),
                                    UUID.fromString("52df1bb0-6a2f-11e6-b62d-aff2143498ea") };
     // Instant.MIN/MAX fail Date.from.
+    public static
     Date[] testDates = new Date[]{ null,
                                    Date.from(Instant.ofEpochSecond(Integer.MIN_VALUE)),
                                    Date.from(Instant.ofEpochSecond(Short.MIN_VALUE)),
@@ -416,7 +426,9 @@ public class ByteSourceTestBase
                                    Date.from(Instant.ofEpochMilli(2000)),
                                    Date.from(Instant.ofEpochSecond(Integer.MAX_VALUE)),
                                    Date.from(Instant.now()) };
+    public static
     InetAddress[] testInets;
+    static
     {
         try
         {
@@ -436,8 +448,9 @@ public class ByteSourceTestBase
         }
     }
 
+    public static
     BigInteger[] testBigInts;
-
+    static
     {
         Set<BigInteger> bigs = new TreeSet<>();
         for (Long l : testLongs)
@@ -480,7 +493,9 @@ static String exp10(int pow)
         return builder.toString();
     }
 
+    public static
     BigDecimal[] testBigDecimals;
+    static
     {
         String vals = "0, 1, 1.1, 21, 98.9, 99, 99.9, 100, 100.1, 101, 331, 0.4, 0.07, 0.0700, 0.005, " +
                       "6e4, 7e200, 6e-300, 8.1e2000, 8.1e-2000, 9e2000000000, " +
@@ -495,6 +510,7 @@ static String exp10(int pow)
         testBigDecimals = decs.toArray(new BigDecimal[0]);
     }
 
+    public static
     Object[][] testValues = new Object[][]{ testStrings,
                                             testInts,
                                             testBools,
@@ -502,6 +518,7 @@ static String exp10(int pow)
                                             testBigInts,
                                             testBigDecimals };
 
+    public static
     AbstractType[] testTypes = new AbstractType[]{ UTF8Type.instance,
                                                    Int32Type.instance,
                                                    BooleanType.instance,
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/DecoratedKeyByteSourceTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/DecoratedKeyByteSourceTest.java
index 6c3e72efbe63..1c28f1b52dce 100644
--- a/test/unit/org/apache/cassandra/utils/bytecomparable/DecoratedKeyByteSourceTest.java
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/DecoratedKeyByteSourceTest.java
@@ -39,7 +39,8 @@ public class DecoratedKeyByteSourceTest
     @Parameterized.Parameters(name = "version={0}")
     public static Iterable<ByteComparable.Version> versions()
     {
-        return ImmutableList.of(ByteComparable.Version.OSS50);
+        return ImmutableList.of(ByteComparable.Version.OSS41,
+                                ByteComparable.Version.OSS50);
     }
 
     private final ByteComparable.Version version;
diff --git a/test/unit/org/apache/cassandra/utils/concurrent/TimerTest.java b/test/unit/org/apache/cassandra/utils/concurrent/TimerTest.java
new file mode 100644
index 000000000000..96b2451ded6d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/concurrent/TimerTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.concurrent.ExecutorLocals;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.TimeUUID;
+import org.assertj.core.api.Assertions;
+
+public class TimerTest
+{
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void testWaitForExpiration() throws Exception
+    {
+        CountDownLatch latch = new CountDownLatch(1);
+
+        Future<Void> result = Timer.INSTANCE.onTimeout(latch::countDown, 1, TimeUnit.SECONDS);
+        result.get(3, TimeUnit.SECONDS);
+
+        Assertions.assertThat(latch.await(0, TimeUnit.MILLISECONDS)).isTrue();
+        Assertions.assertThat(result.isDone()).isTrue();
+    }
+
+    @Test
+    public void testCancelExpiration() throws Exception
+    {
+        CountDownLatch latch = new CountDownLatch(1);
+
+        Future<Void> result = Timer.INSTANCE.onTimeout(latch::countDown, 1, TimeUnit.SECONDS);
+        result.cancel(true);
+
+        Assertions.assertThat(latch.await(3, TimeUnit.SECONDS)).isFalse();
+        Assertions.assertThat(result.isDone()).isFalse();
+        Assertions.assertThat(result.isCancelled()).isTrue();
+    }
+
+    @Test
+    public void testExecutorsLocalPropagation() throws Exception
+    {
+        CountDownLatch latch = new CountDownLatch(1);
+
+        ClientWarn.instance.set(new ClientWarn.State());
+        ClientWarn.instance.warn("test");
+        Tracing.instance.set(Tracing.instance.get(Tracing.instance.newSession(ClientState.forInternalCalls(), Tracing.TraceType.NONE)));
+        TimeUUID sessionId = Tracing.instance.get().sessionId;
+        Future<Void> result = Timer.INSTANCE.onTimeout(() ->
+                                                 {
+                                                     if (ClientWarn.instance.getWarnings().contains("test") &&
+                                                         Tracing.instance.get().sessionId.equals(sessionId))
+                                                         latch.countDown();
+                                                 }, 1, TimeUnit.SECONDS, ExecutorLocals.create(Tracing.instance.get()));
+
+        result.get(3, TimeUnit.SECONDS);
+
+        Assertions.assertThat(latch.await(0, TimeUnit.MILLISECONDS)).isTrue();
+        Assertions.assertThat(result.isDone()).isTrue();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/utils/logging/LogbackLoggingSupportTest.java b/test/unit/org/apache/cassandra/utils/logging/LogbackLoggingSupportTest.java
new file mode 100644
index 000000000000..2894b9d2a62b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/logging/LogbackLoggingSupportTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.logging;
+
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+
+import static org.junit.Assert.assertSame;
+
+public class LogbackLoggingSupportTest
+{
+    @Test
+    public void setLogLevelWithoutOptionsReloadsConfiguration() throws Exception
+    {
+        // given
+        LogbackLoggingSupport loggingSupport = new LogbackLoggingSupport();
+        loggingSupport.onStartup();
+
+        Logger rootLogger = (Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME);
+        Level defaultLevel = rootLogger.getLevel();
+
+        rootLogger.setLevel(Level.OFF);
+        assertSame("the log level should have been switched to OFF", Level.OFF, rootLogger.getLevel());
+
+        // when
+        // empty class and level reset to the default configuration
+        loggingSupport.setLoggingLevel("", "");
+
+        // then
+        assertSame("reset test log level should be reset", defaultLevel, rootLogger.getLevel());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/memory/BufferPoolTest.java b/test/unit/org/apache/cassandra/utils/memory/BufferPoolTest.java
index ab68e405ca7c..1d07ba2dbc97 100644
--- a/test/unit/org/apache/cassandra/utils/memory/BufferPoolTest.java
+++ b/test/unit/org/apache/cassandra/utils/memory/BufferPoolTest.java
@@ -905,7 +905,7 @@ public void testRecyclePartialFreeChunk()
         assertNotEquals(chunk1, chunk3);
         assertNotEquals(chunk2, chunk3);
 
-        // verify chunk2 got evicted, it doesn't have a owner
+        // verify chunk2 got evicted, it doesn't have an owner
         assertNotNull(chunk0.owner());
         assertEquals(BufferPool.Chunk.Status.IN_USE, chunk0.status());
         assertNotNull(chunk1.owner());
diff --git a/test/unit/org/apache/cassandra/utils/memory/MemoryUtilTest.java b/test/unit/org/apache/cassandra/utils/memory/MemoryUtilTest.java
new file mode 100644
index 000000000000..00f8ebe24285
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/memory/MemoryUtilTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.memory;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class MemoryUtilTest
+{
+
+    @Test
+    public void allocateByteBuffer()
+    {
+        long address = 0;
+        int length = 10;
+        int capacity = 10;
+        ByteBuffer byteBuffer = MemoryUtil.allocateByteBuffer(address, length, capacity, ByteOrder.nativeOrder(), null);
+        assertNotNull(byteBuffer);
+        assertEquals(address, MemoryUtil.getAddress(byteBuffer));
+        assertEquals(length, byteBuffer.limit());
+        assertEquals(capacity, byteBuffer.capacity());
+
+        byteBuffer = MemoryUtil.allocateByteBuffer(address, length, capacity, ByteOrder.nativeOrder(), new Object());
+        assertNotNull(byteBuffer);
+        assertEquals(address, MemoryUtil.getAddress(byteBuffer));
+        assertEquals(length, byteBuffer.limit());
+        assertEquals(capacity, byteBuffer.capacity());
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java b/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
index 49b4c94dd387..87dbd192f7bf 100644
--- a/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
+++ b/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
@@ -20,15 +20,15 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
-import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
 import com.google.common.collect.Lists;
-import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.junit.Assert;
 import org.junit.Test;
 
+import org.apache.cassandra.io.util.DataOutputBuffer;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -36,6 +36,7 @@
 public class OffHeapBitSetTest
 {
     private static final Random random = new Random();
+    private static final MemoryLimiter memoryLimiter = new MemoryLimiter(1L << 30, "Allocating %s for bloom filter would reach max of %s (current %s)");
 
     static void compare(IBitSet bs, IBitSet newbs)
     {
@@ -44,9 +45,9 @@ static void compare(IBitSet bs, IBitSet newbs)
             Assert.assertEquals(bs.get(i), newbs.get(i));
     }
 
-    private void testOffHeapSerialization(boolean oldBfFormat) throws IOException
+    private void testOffHeapSerialization(boolean oldBfFormat) throws Exception
     {
-        try (OffHeapBitSet bs = new OffHeapBitSet(100000))
+        try (OffHeapBitSet bs = new OffHeapBitSet(100000, memoryLimiter))
         {
             for (long i = 0; i < bs.capacity(); i++)
                 if (random.nextBoolean())
@@ -59,7 +60,7 @@ private void testOffHeapSerialization(boolean oldBfFormat) throws IOException
                 bs.serialize(out);
 
             DataInputStream in = new DataInputStream(new ByteArrayInputStream(out.getData()));
-            try (OffHeapBitSet newbs = OffHeapBitSet.deserialize(in, oldBfFormat))
+            try (OffHeapBitSet newbs = OffHeapBitSet.deserialize(in, oldBfFormat, memoryLimiter))
             {
                 compare(bs, newbs);
             }
@@ -67,17 +68,17 @@ private void testOffHeapSerialization(boolean oldBfFormat) throws IOException
     }
 
     @Test
-    public void testSerialization() throws IOException
+    public void testSerialization() throws Exception
     {
         testOffHeapSerialization(true);
         testOffHeapSerialization(false);
     }
 
     @Test
-    public void testBitSetGetClear()
+    public void testBitSetGetClear() throws Exception
     {
         int size = Integer.MAX_VALUE / 4000;
-        try (OffHeapBitSet bs = new OffHeapBitSet(size))
+        try (OffHeapBitSet bs = new OffHeapBitSet(size, memoryLimiter))
         {
             List<Integer> randomBits = Lists.newArrayList();
             for (int i = 0; i < 10; i++)
@@ -98,16 +99,16 @@ public void testBitSetGetClear()
     }
 
     @Test(expected = UnsupportedOperationException.class)
-    public void testUnsupportedLargeSize()
+    public void testUnsupportedLargeSize() throws Exception
     {
         long size = 64L * Integer.MAX_VALUE + 1; // Max size 16G * 8 bits
-        OffHeapBitSet bs = new OffHeapBitSet(size);
+        OffHeapBitSet bs = new OffHeapBitSet(size, memoryLimiter);
     }
 
     @Test
-    public void testInvalidIndex()
+    public void testInvalidIndex() throws Exception
     {
-        OffHeapBitSet bs = new OffHeapBitSet(10);
+        OffHeapBitSet bs = new OffHeapBitSet(10, memoryLimiter);
         int invalidIdx[] = {-1, 64, 1000};
 
         for (int i : invalidIdx)
diff --git a/test/unit/org/apache/cassandra/utils/units/RateUnitTest.java b/test/unit/org/apache/cassandra/utils/units/RateUnitTest.java
new file mode 100644
index 000000000000..f96d25cf718e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/units/RateUnitTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class RateUnitTest
+{
+    @Test
+    public void testConvert()
+    {
+        assertThat(RateUnit.B_S.convert(10L, RateUnit.KB_S)).isEqualTo(10 * 1024);
+        assertThat(RateUnit.B_S.convert(10L, RateUnit.MB_S)).isEqualTo(10 * 1024 * 1024);
+        assertThat(RateUnit.MB_S.convert(10L, RateUnit.GB_S)).isEqualTo(10 * 1024);
+        assertThat(RateUnit.GB_S.convert(10L, RateUnit.TB_S)).isEqualTo(10 * 1024);
+        assertThat(RateUnit.MB_S.convert(10L, RateUnit.B_S)).isEqualTo(0);
+
+        RateUnit B_MS = RateUnit.of(SizeUnit.BYTES, TimeUnit.MILLISECONDS);
+        // 10 kB/s == 10,240 B/s == 10 kB/ms
+        assertThat(B_MS.convert(10L, RateUnit.KB_S)).isEqualTo(10L);
+
+        RateUnit GB_D = RateUnit.of(SizeUnit.GIGABYTES, TimeUnit.DAYS);
+        // 10 MB/s == 10 * 3600 MB/h == 36,000 MB/h == 864,000 MB/days == 843 GB/days
+        assertThat(GB_D.convert(10L, RateUnit.MB_S)).isEqualTo(843L);
+
+        RateUnit GB_MS = RateUnit.of(SizeUnit.GIGABYTES, TimeUnit.MILLISECONDS);
+        // 10 MB/s == 0 GB/ms
+        assertThat(GB_MS.convert(10L, RateUnit.MB_S)).isEqualTo(0L);
+
+        RateUnit B_H = RateUnit.of(SizeUnit.BYTES, TimeUnit.HOURS);
+        // 10 MB/s == 10 * 1024 * 1024 B/s = 10 * 1024 * 1024 * 3600 B/hours
+        assertThat(B_H.convert(10L, RateUnit.MB_S)).isEqualTo(10L * 1024 * 1024 * 3600);
+    }
+
+    @Test
+    public void testToHumanReadableString()
+    {
+        assertThat(RateUnit.B_S.toHumanReadableString(10)).isEqualTo("10B/s");
+        assertThat(RateUnit.B_S.toHumanReadableString(1024L)).isEqualTo("1kB/s");
+        assertThat(RateUnit.B_S.toHumanReadableString(1150L)).isEqualTo("1.1kB/s");
+        assertThat(RateUnit.B_S.toHumanReadableString(4 * 1024 * 1024L)).isEqualTo("4MB/s");
+        assertThat(RateUnit.GB_S.toHumanReadableString(2600 * 1024L)).isEqualTo("2,600TB/s");
+    }
+
+    @Test
+    public void testCompare()
+    {
+        assertThat(RateUnit.MB_S.compareTo(RateUnit.MB_S)).isEqualTo(0);
+
+        assertThat(RateUnit.MB_S.compareTo(RateUnit.GB_S)).isLessThan(0);
+        assertThat(RateUnit.MB_S.compareTo(RateUnit.TB_S)).isLessThan(0);
+
+        assertThat(RateUnit.MB_S.compareTo(RateUnit.KB_S)).isGreaterThan(0);
+        assertThat(RateUnit.MB_S.compareTo(RateUnit.B_S)).isGreaterThan(0);
+
+        RateUnit MB_MS = RateUnit.of(SizeUnit.MEGABYTES, TimeUnit.MILLISECONDS);
+        RateUnit MB_NS = RateUnit.of(SizeUnit.MEGABYTES, TimeUnit.NANOSECONDS);
+
+        assertThat(RateUnit.MB_S.compareTo(MB_MS)).isLessThan(0);
+        assertThat(RateUnit.MB_S.compareTo(MB_NS)).isLessThan(0);
+
+        RateUnit KB_MS = RateUnit.of(SizeUnit.KILOBYTES, TimeUnit.MILLISECONDS);
+        RateUnit KB_NS = RateUnit.of(SizeUnit.KILOBYTES, TimeUnit.NANOSECONDS);
+
+        // 1 MB/s = 1024 kB/s > 1000 kB/s = 1kB/ms
+        assertThat(RateUnit.MB_S.compareTo(KB_MS)).isGreaterThan(0);
+        // 1 MB/s = 1024 kB/s < 1000 * 1000 kB/s = 1kB/ns
+        assertThat(RateUnit.MB_S.compareTo(KB_NS)).isLessThan(0);
+
+        RateUnit MB_M = RateUnit.of(SizeUnit.MEGABYTES, TimeUnit.MINUTES);
+        RateUnit GB_D = RateUnit.of(SizeUnit.GIGABYTES, TimeUnit.DAYS);
+        // 1 MB/m = 1440 MB/d > 1024 MB/d = 1 GB/d
+        assertThat(MB_M.compareTo(GB_D)).isGreaterThan(0);
+
+        RateUnit GB_MS = RateUnit.of(SizeUnit.GIGABYTES, TimeUnit.MILLISECONDS);
+        // 1 MB/s < 1 GB/ms
+        assertThat(RateUnit.MB_S.compareTo(GB_MS)).isLessThan(0);
+    }
+
+    @Test
+    public void testSmallestRepresentation()
+    {
+        // The smallest unit
+        RateUnit B_D = RateUnit.of(SizeUnit.BYTES, TimeUnit.DAYS);
+
+        // A few simple case that all resolve to the smallest unit
+        assertThat(B_D.smallestRepresentableUnit(1)).isEqualTo(B_D);
+        assertThat(B_D.smallestRepresentableUnit(Long.MAX_VALUE)).isEqualTo(B_D);
+        assertThat(RateUnit.B_S.smallestRepresentableUnit(1)).isEqualTo(B_D);
+        assertThat(RateUnit.KB_S.smallestRepresentableUnit(1)).isEqualTo(B_D);
+
+        assertThat(RateUnit.MB_S.smallestRepresentableUnit(Long.MAX_VALUE - 10)).isEqualTo(RateUnit.MB_S);
+        assertThat(RateUnit.MB_S.smallestRepresentableUnit((Long.MAX_VALUE / 1024) - 10)).isEqualTo(RateUnit.KB_S);
+
+        // Slightly more subtle cases
+        long v1 = (Long.MAX_VALUE - 1) / 1000;
+        long v2 = (Long.MAX_VALUE - 1) / 1024;
+        long v3 = (Long.MAX_VALUE - 1) / (1000 * 60);
+
+        RateUnit MB_MS = RateUnit.of(SizeUnit.MEGABYTES, TimeUnit.MILLISECONDS);
+        RateUnit KB_MS = RateUnit.of(SizeUnit.KILOBYTES, TimeUnit.MILLISECONDS);
+        RateUnit MB_M = RateUnit.of(SizeUnit.MEGABYTES, TimeUnit.MINUTES);
+        assertThat(MB_MS.smallestRepresentableUnit(v1)).isEqualTo(RateUnit.MB_S);
+        assertThat(MB_MS.smallestRepresentableUnit(v2)).isEqualTo(KB_MS);
+        assertThat(MB_MS.smallestRepresentableUnit(v3)).isEqualTo(MB_M);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/units/RateValueTest.java b/test/unit/org/apache/cassandra/utils/units/RateValueTest.java
new file mode 100644
index 000000000000..67bf12c7d238
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/units/RateValueTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class RateValueTest
+{
+    @Test
+    public void testCompute()
+    {
+        assertThat(RateValue.compute(SizeValue.of(10, SizeUnit.MEGABYTES), TimeValue.of(2, TimeUnit.SECONDS)))
+        .isEqualTo(RateValue.of(5, RateUnit.MB_S).convert(RateUnit.B_S));
+
+        assertThat(RateValue.compute(SizeValue.of(10, SizeUnit.MEGABYTES), TimeValue.of(5, TimeUnit.SECONDS)))
+        .isEqualTo(RateValue.of(2, RateUnit.MB_S));
+
+        assertThat(RateValue.compute(SizeValue.of(10, SizeUnit.MEGABYTES), TimeValue.of(10, TimeUnit.SECONDS)))
+        .isEqualTo(RateValue.of(1, RateUnit.MB_S));
+
+        // Reminder that 1MB = 1204KB, so 0.5MB == 512KB
+        assertThat(RateValue.compute(SizeValue.of(10, SizeUnit.MEGABYTES), TimeValue.of(20, TimeUnit.SECONDS)))
+        .isEqualTo(RateValue.of(512, RateUnit.KB_S));
+    }
+
+    @Test
+    public void testTimeFor()
+    {
+        RateValue rate = RateValue.of(1, RateUnit.MB_S);
+        assertThat(rate.timeFor(SizeValue.of(50, SizeUnit.MEGABYTES))).isEqualTo(TimeValue.of(50, TimeUnit.SECONDS));
+        assertThat(rate.timeFor(SizeValue.of(93, SizeUnit.MEGABYTES))).isEqualTo(TimeValue.of(93, TimeUnit.SECONDS));
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/units/SizeUnitTest.java b/test/unit/org/apache/cassandra/utils/units/SizeUnitTest.java
new file mode 100644
index 000000000000..92c7d13fefa0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/units/SizeUnitTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import org.junit.Test;
+
+import static org.apache.cassandra.utils.units.SizeUnit.BYTES;
+import static org.apache.cassandra.utils.units.SizeUnit.C0;
+import static org.apache.cassandra.utils.units.SizeUnit.C1;
+import static org.apache.cassandra.utils.units.SizeUnit.C2;
+import static org.apache.cassandra.utils.units.SizeUnit.C3;
+import static org.apache.cassandra.utils.units.SizeUnit.C4;
+import static org.apache.cassandra.utils.units.SizeUnit.GIGABYTES;
+import static org.apache.cassandra.utils.units.SizeUnit.KILOBYTES;
+import static org.apache.cassandra.utils.units.SizeUnit.MEGABYTES;
+import static org.apache.cassandra.utils.units.SizeUnit.TERABYTES;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SizeUnitTest
+{
+    @Test
+    public void testConvert()
+    {
+        // We know convert delegates to the other methods, so we don't go overboard on testing it with all units. We
+        // just use that test for a few random conversions.
+        assertThat(GIGABYTES.convert(100, BYTES)).isEqualTo(0);
+        assertThat(GIGABYTES.convert(100 * C3, BYTES)).isEqualTo(100);
+        assertThat(GIGABYTES.convert(100 * C4, BYTES)).isEqualTo(100 * C1);
+        assertThat(BYTES.convert(100 * C4, GIGABYTES)).isEqualTo(Long.MAX_VALUE);
+    }
+
+    @Test
+    public void testToBytes()
+    {
+        testToBytes(BYTES, C0);
+        testToBytes(KILOBYTES, C1);
+        testToBytes(MEGABYTES, C2);
+        testToBytes(GIGABYTES, C3);
+        testToBytes(TERABYTES, C4);
+
+        // Test overflow
+        assertThat(TERABYTES.toBytes(Long.MAX_VALUE / 10)).isEqualTo(Long.MAX_VALUE);
+    }
+
+    private void testToBytes(SizeUnit unit, long constant)
+    {
+        assertThat(unit.toBytes(1)).isEqualTo(1 * constant);
+        assertThat(unit.toBytes(1023)).isEqualTo(1023 * constant);
+        assertThat(unit.toBytes(1024)).isEqualTo(1024 * constant);
+        assertThat(unit.toBytes(2049)).isEqualTo(2049 * constant);
+    }
+
+    @Test
+    public void testToKiloBytes()
+    {
+        assertThat(BYTES.toKiloBytes(1)).isEqualTo(0);
+        assertThat(BYTES.toKiloBytes(1023)).isEqualTo(0);
+        assertThat(BYTES.toKiloBytes(1024)).isEqualTo(1);
+        assertThat(BYTES.toKiloBytes(2049)).isEqualTo(2);
+
+        testToKiloBytes(KILOBYTES, C0);
+        testToKiloBytes(MEGABYTES, C1);
+        testToKiloBytes(GIGABYTES, C2);
+        testToKiloBytes(TERABYTES, C3);
+    }
+
+    private void testToKiloBytes(SizeUnit unit, long constant)
+    {
+        assertThat(unit.toKiloBytes(1)).isEqualTo(1 * constant);
+        assertThat(unit.toKiloBytes(1023)).isEqualTo(1023 * constant);
+        assertThat(unit.toKiloBytes(1024)).isEqualTo(1024 * constant);
+        assertThat(unit.toKiloBytes(2049)).isEqualTo(2049 * constant);
+    }
+
+    @Test
+    public void testToMegaBytes() throws Exception
+    {
+        testToMegaBytes(BYTES, 0);
+
+        assertThat(KILOBYTES.toMegaBytes(1)).isEqualTo(0);
+        assertThat(KILOBYTES.toMegaBytes(1023)).isEqualTo(0);
+        assertThat(KILOBYTES.toMegaBytes(1024)).isEqualTo(1);
+        assertThat(KILOBYTES.toMegaBytes(2049)).isEqualTo(2);
+
+        testToMegaBytes(MEGABYTES, C0);
+        testToMegaBytes(GIGABYTES, C1);
+        testToMegaBytes(TERABYTES, C2);
+    }
+
+    private void testToMegaBytes(SizeUnit unit, long constant)
+    {
+        assertThat(unit.toMegaBytes(1)).isEqualTo(1 * constant);
+        assertThat(unit.toMegaBytes(1023)).isEqualTo(1023 * constant);
+        assertThat(unit.toMegaBytes(1024)).isEqualTo(1024 * constant);
+        assertThat(unit.toMegaBytes(2049)).isEqualTo(2049 * constant);
+    }
+
+    @Test
+    public void testToGigaBytes()
+    {
+        testToGigaBytes(BYTES, 0);
+        testToGigaBytes(KILOBYTES, 0);
+
+        assertThat(MEGABYTES.toGigaBytes(1)).isEqualTo(0);
+        assertThat(MEGABYTES.toGigaBytes(1023)).isEqualTo(0);
+        assertThat(MEGABYTES.toGigaBytes(1024)).isEqualTo(1);
+        assertThat(MEGABYTES.toGigaBytes(2049)).isEqualTo(2);
+
+        testToGigaBytes(GIGABYTES, C0);
+        testToGigaBytes(TERABYTES, C1);
+    }
+
+    private void testToGigaBytes(SizeUnit unit, long constant)
+    {
+        assertThat(unit.toGigaBytes(1)).isEqualTo(1 * constant);
+        assertThat(unit.toGigaBytes(1023)).isEqualTo(1023 * constant);
+        assertThat(unit.toGigaBytes(1024)).isEqualTo(1024 * constant);
+        assertThat(unit.toGigaBytes(2049)).isEqualTo(2049 * constant);
+    }
+
+    @Test
+    public void testToTeraBytes()
+    {
+        testToTeraBytes(BYTES, 0);
+        testToTeraBytes(KILOBYTES, 0);
+        testToTeraBytes(MEGABYTES, 0);
+
+        assertThat(GIGABYTES.toTeraBytes(1)).isEqualTo(0);
+        assertThat(GIGABYTES.toTeraBytes(1023)).isEqualTo(0);
+        assertThat(GIGABYTES.toTeraBytes(1024)).isEqualTo(1);
+        assertThat(GIGABYTES.toTeraBytes(2049)).isEqualTo(2);
+
+        testToTeraBytes(TERABYTES, C0);
+    }
+
+    private void testToTeraBytes(SizeUnit unit, long constant)
+    {
+        assertThat(unit.toTeraBytes(1)).isEqualTo(1 * constant);
+        assertThat(unit.toTeraBytes(1023)).isEqualTo(1023 * constant);
+        assertThat(unit.toTeraBytes(1024)).isEqualTo(1024 * constant);
+        assertThat(unit.toTeraBytes(2049)).isEqualTo(2049 * constant);
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/units/SizeValueTest.java b/test/unit/org/apache/cassandra/utils/units/SizeValueTest.java
new file mode 100644
index 000000000000..cfc845b93616
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/units/SizeValueTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SizeValueTest
+{
+    @Test
+    public void testToString()
+    {
+        assertThat(SizeUnit.BYTES.value(10).toString()).isEqualTo("10B");
+        assertThat(SizeUnit.BYTES.value(1900).toString()).isEqualTo("1.9kB");
+        assertThat(SizeUnit.BYTES.value(2000).toString()).isEqualTo("2kB");
+        assertThat(SizeUnit.BYTES.value(2200).toString()).isEqualTo("2.1kB");
+        assertThat(SizeUnit.BYTES.value(42_345L).toString()).isEqualTo("41kB");
+        assertThat(SizeUnit.BYTES.value(100_334_345L).toString()).isEqualTo("95MB");
+        assertThat(SizeUnit.BYTES.value(345_100_334_345L).toString()).isEqualTo("321GB");
+        assertThat(SizeUnit.BYTES.value(2_345_100_334_345L).toString()).isEqualTo("2.1TB");
+        assertThat(SizeUnit.BYTES.value(98_345_100_334_345L).toString()).isEqualTo("89TB");
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/units/TimeValueTest.java b/test/unit/org/apache/cassandra/utils/units/TimeValueTest.java
new file mode 100644
index 000000000000..6e38c2d2ee88
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/units/TimeValueTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TimeValueTest
+{
+    @Test
+    public void testToString()
+    {
+        assertThat(TimeValue.of(10L, TimeUnit.MILLISECONDS).toString()).isEqualTo("10ms");
+        assertThat(TimeValue.of(1000L, TimeUnit.MILLISECONDS).toString()).isEqualTo("1s");
+        assertThat(TimeValue.of(1200L, TimeUnit.MILLISECONDS).toString()).isEqualTo("1.2s");
+        assertThat(TimeValue.of(42_324L, TimeUnit.MILLISECONDS).toString()).isEqualTo("42s");
+        assertThat(TimeValue.of(60_000L, TimeUnit.MILLISECONDS).toString()).isEqualTo("1m");
+        assertThat(TimeValue.of(250_000L, TimeUnit.MILLISECONDS).toString()).isEqualTo("4.2m");
+        assertThat(TimeValue.of(3_600_000L, TimeUnit.MILLISECONDS).toString()).isEqualTo("1h");
+        assertThat(TimeValue.of(24 * 10_200_000L, TimeUnit.MILLISECONDS).toString()).isEqualTo("2.8d");
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/units/UnitsTest.java b/test/unit/org/apache/cassandra/utils/units/UnitsTest.java
new file mode 100644
index 000000000000..e55aa6ae1993
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/units/UnitsTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.units;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class UnitsTest
+{
+    @Test
+    public void testFormatValue()
+    {
+        assertThat(Units.formatValue(0L)).isEqualTo("0");
+        // No comma
+        assertThat(Units.formatValue(1L)).isEqualTo("1");
+        assertThat(Units.formatValue(-1L)).isEqualTo("-1");
+        assertThat(Units.formatValue(10L)).isEqualTo("10");
+        assertThat(Units.formatValue(-10L)).isEqualTo("-10");
+        assertThat(Units.formatValue(999L)).isEqualTo("999");
+        assertThat(Units.formatValue(-999L)).isEqualTo("-999");
+
+        // One comma
+        assertThat(Units.formatValue(1_000L)).isEqualTo("1,000");
+        assertThat(Units.formatValue(-1_000L)).isEqualTo("-1,000");
+        assertThat(Units.formatValue(12_345L)).isEqualTo("12,345");
+        assertThat(Units.formatValue(-12_345L)).isEqualTo("-12,345");
+        assertThat(Units.formatValue(999_999L)).isEqualTo("999,999");
+        assertThat(Units.formatValue(-999_999L)).isEqualTo("-999,999");
+
+        // Two comma
+        assertThat(Units.formatValue(1_000_000L)).isEqualTo("1,000,000");
+        assertThat(Units.formatValue(-1_000_000L)).isEqualTo("-1,000,000");
+        assertThat(Units.formatValue(999_999_999L)).isEqualTo("999,999,999");
+        assertThat(Units.formatValue(-999_999_999L)).isEqualTo("-999,999,999");
+
+        // Lots of comma
+        assertThat(Units.formatValue(123_456_789_123_456_789L)).isEqualTo("123,456,789,123,456,789");
+        assertThat(Units.formatValue(-123_456_789_123_456_789L)).isEqualTo("-123,456,789,123,456,789");
+    }
+}
\ No newline at end of file
diff --git a/tools/analytics/plot_adaptive.gnu b/tools/analytics/plot_adaptive.gnu
new file mode 100644
index 000000000000..31a8088b293b
--- /dev/null
+++ b/tools/analytics/plot_adaptive.gnu
@@ -0,0 +1,71 @@
+# Run with: gnuplot ~/Downloads/db-3558/plot.sh
+# Documentation: http://www.gnuplot.info/docs_5.2/Gnuplot_5.2.pdf
+
+path = "."
+outputPath = path."/"
+
+# define the output type and size
+set terminal png size 2000,800 linewidth 2
+
+# the first line makes it skip the first row and use it for the legend otherwise we must specify for every plot:
+# ... every ::1 using 1:n with lines title "blah"
+set key autotitle columnhead
+set datafile separator ","
+
+# timestamp ms,W,num compactions,live sstables,space used bytes,Num inserted,Num read,% inserted,% read,Read IO,Read IO stddev,Write IO,Write IO stddev,Tot IO,Tot IO stddev, Num pending, WA
+
+#datasets = "R100_W0 R80_W20 R50_W50 R20_W80 R0_W100"
+#datasets = "R80_W20 R50_W50 R20_W80"
+#datasets = "R50_W50 R20_W80"
+datasets = "R50_W50"
+N = words(datasets)
+
+dataset(i) = word(datasets, i)
+file(i) = sprintf("%s/testAdaptiveController_%s.csv", path, word(datasets, i))
+
+set xlabel 'Time (ms)'
+#set xrange [0:300000]
+
+set output outputPath.'live_sstables.png'
+set title "Live sstables"
+set ylabel 'Num sstables'
+plot for [i=1:N] file(i) using 1:4 with linespoints title dataset(i)
+
+set output outputPath.'space_used.png'
+set title "Space used"
+set ylabel 'Bytes Used'
+plot for [i=1:N] file(i) using 1:5 with linespoints title dataset(i)
+
+
+set ylabel 'IO cost (ms)'
+set y2label 'W'
+set y2range [-20:100]
+set y2tics -12, 4
+set ytics nomirror
+
+do for [i=1:N] {
+
+	set output outputPath.'read_io_cost_'.dataset(i).'.png'
+	set title "Read IO cost"
+	plot file(i) using 1:10:11 with yerrorbars lt 1 title "", \
+    	 file(i) using 1:10 smooth acsplines lt 1 title dataset(i), \
+     	 file(i) using 1:2 with lines lt 2 axis x1y2 title "W"
+
+	set output outputPath.'write_io_cost_'.dataset(i).'.png'
+	set title "Write IO cost"
+	plot file(i) using 1:12:13 with yerrorbars lt 1 title "", \
+	     file(i) using 1:12 smooth acsplines lt 1 title dataset(i), \
+	     file(i) using 1:2 with lines lt 2 axis x1y2 title "W"
+
+	set output outputPath.'tot_io_cost_'.dataset(i).'.png'
+	set title "Tot. IO cost"
+	plot file(i) using 1:14:15 with yerrorbars lt 1 title "", \
+	     file(i) using 1:14 smooth acsplines lt 1 title dataset(i), \
+	     file(i) using 1:2 with lines lt 2 axis x1y2 title "W"
+
+	set output outputPath.'wa'.dataset(i).'.png'
+        set title "Write Amplification"
+        set ylabel 'WA'
+        plot file(i) using 1:17 with linespoints title 'WA', \
+             file(i) using 1:2 with lines lt 3 axis x1y2 title "W"
+}
\ No newline at end of file
diff --git a/tools/analytics/plot_static.gnu b/tools/analytics/plot_static.gnu
new file mode 100644
index 000000000000..a1e571aab46a
--- /dev/null
+++ b/tools/analytics/plot_static.gnu
@@ -0,0 +1,59 @@
+# Run with: gnuplot ~/Downloads/db-3558/plot.sh
+# Documentation: http://www.gnuplot.info/docs_5.2/Gnuplot_5.2.pdf
+
+path = "."
+outputPath = path."/"
+
+# define the output type and size
+set terminal png size 2000,800 linewidth 2
+
+# the first line makes it skip the first row and use it for the legend otherwise we must specify for every plot:
+# ... every ::1 using 1:n with lines title "blah"
+set key autotitle columnhead
+set datafile separator ","
+
+# timestamp ms,W,num compactions,live sstables,space used bytes,Num inserted,Num read,% inserted,% read,Read IO,Read IO stddev,Write IO,Write IO stddev,Tot IO,Tot IO stddev, Num pending, WA
+
+#datasets = "R100_W0 R80_W20 R50_W50 R20_W80 R0_W100"
+#datasets = "R80_W20 R50_W50 R20_W80"
+datasets = "R50_W50"
+N = words(datasets)
+
+dataset(i) = word(datasets, i)
+file(i) = sprintf("%s/testStaticAnalysis_%s-avg.csv", path, word(datasets, i))
+
+set xlabel 'W'
+
+set output outputPath.'live_sstables.png'
+set title "Live sstables"
+set ylabel 'Num sstables'
+plot for [i=1:N] file(i) using 2:4 with linespoints title dataset(i)
+
+set output outputPath.'space_used.png'
+set title "Space used"
+set ylabel 'Bytes Used'
+plot for [i=1:N] file(i) using 2:5 with linespoints title dataset(i)
+
+
+set ylabel 'ms'
+
+set output outputPath.'read_io_cost.png'
+set title "Read IO cost"
+plot for [i=1:N] file(i) using 2:10:11 with yerrorbars lt i title "", \
+     for [i=1:N] file(i) using 2:10 smooth acsplines lt i title dataset(i)
+
+set output outputPath.'write_io_cost.png'
+set title "Write IO cost"
+plot for [i=1:N] file(i) using 2:12:13 with yerrorbars lt i title "", \
+     for [i=1:N] file(i) using 2:12 smooth acsplines lt i title dataset(i)
+
+set output outputPath.'tot_io_cost.png'
+set title "Tot. IO cost"
+plot for [i=1:N] file(i) using 2:14:15 with yerrorbars lt i title "", \
+     for [i=1:N] file(i) using 2:14 smooth acsplines lt i title dataset(i)
+
+
+set output outputPath.'wa.png'
+set title "Write Amplification"
+set ylabel 'WA'
+plot for [i=1:N] file(i) using 2:17 with linespoints title dataset(i)
\ No newline at end of file
diff --git a/tools/bin/analyzecompactionlog b/tools/bin/analyzecompactionlog
new file mode 100755
index 000000000000..3c3bbe5aabca
--- /dev/null
+++ b/tools/bin/analyzecompactionlog
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+# Copyright DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+    # Locations (in order) to use when searching for an include file.
+    for include in "`dirname "$0"`/cassandra.in.sh" \
+                   "$HOME/.cassandra.in.sh" \
+                   /usr/share/cassandra/cassandra.in.sh \
+                   /usr/local/share/cassandra/cassandra.in.sh \
+                   /opt/cassandra/cassandra.in.sh; do
+        if [ -r "$include" ]; then
+            . "$include"
+            break
+        fi
+    done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+    . "$CASSANDRA_INCLUDE"
+fi
+
+if [ -z "$CLASSPATH" ]; then
+    echo "You must set the CLASSPATH var" >&2
+    exit 1
+fi
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+    MAX_HEAP_SIZE="512m"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+        -Dcassandra.storagedir="$cassandra_storagedir" \
+        -Dlogback.configurationFile=logback-tools.xml \
+        $JVM_ARGS \
+        org.apache.cassandra.tools.CompactionLogAnalyzer "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et
\ No newline at end of file
diff --git a/tools/bin/cassandra.in.sh b/tools/bin/cassandra.in.sh
index 5d265d40f90a..3062f31b4ae5 100644
--- a/tools/bin/cassandra.in.sh
+++ b/tools/bin/cassandra.in.sh
@@ -30,10 +30,10 @@ CLASSPATH="$CASSANDRA_CONF"
 # compiled classes. NOTE: This isn't needed by the startup script,
 # it's just used here in constructing the classpath.
 if [ -d $CASSANDRA_HOME/build ] ; then
-    #cassandra_bin="$CASSANDRA_HOME/build/classes/main"
-    cassandra_bin=`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar`
-    cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/stress:$CASSANDRA_HOME/build/classes/fqltool"
-    CLASSPATH="$CLASSPATH:$cassandra_bin"
+    #dse_db_bin="$CASSANDRA_HOME/build/classes/main"
+    dse_db_bin=`ls -1 $CASSANDRA_HOME/build/dse-db*.jar`
+    dse_db_bin="$dse_db_bin:$CASSANDRA_HOME/build/classes/stress:$CASSANDRA_HOME/build/classes/fqltool"
+    CLASSPATH="$CLASSPATH:$dse_db_bin"
 fi
 
 # the default location for commitlogs, sstables, and saved caches
@@ -82,11 +82,19 @@ jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR=
 JVM_VERSION=${jvmver%_*}
 short=$(echo "${jvmver}" | cut -c1-2)
 
-JAVA_VERSION=17
-if [ "$short" = "11" ] ; then
+JAVA_VERSION=22
+
+if [ "$JVM_VERSION" \< "11" ] ; then
+    echo "Cassandra 5.0 requires either Java 11 or newer."
+    exit 1;
+elif [ "$short" = "11" ] ; then
      JAVA_VERSION=11
 elif [ "$JVM_VERSION" \< "17" ] ; then
-    echo "Cassandra 5.0 requires Java 11 or Java 17(or newer)."
+     echo "Cassandra 5.0 requires Java 11 or Java 17(or newer)."
+elif [ "$short" = "17" ] ; then
+     JAVA_VERSION=17
+elif [ "$JVM_VERSION" \< "22" ] ; then
+    echo "Cassandra requires Java 11 or Java 22(or newer)."
 fi
 
 jvm=`echo "$java_ver_output" | grep -A 1 '[openjdk|java] version' | awk 'NR==2 {print $1}'`
@@ -110,7 +118,9 @@ esac
 
 # Read user-defined JVM options from jvm-server.options file
 JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
-if [ $JAVA_VERSION -ge 17 ] ; then
+if [ $JAVA_VERSION -ge 22 ] ; then
+    JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm22${jvmoptions_variant:--clients}.options
+elif [ $JAVA_VERSION -ge 17 ] ; then
     JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm17${jvmoptions_variant:--clients}.options
 elif [ $JAVA_VERSION -ge 11 ] ; then
     JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryIterator.java b/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryIterator.java
index ccbb200be5d9..6fa4406c49ce 100644
--- a/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryIterator.java
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/FQLQueryIterator.java
@@ -18,15 +18,17 @@
 
 package org.apache.cassandra.fqltool;
 
-import java.util.PriorityQueue;
+import java.util.Arrays;
+import java.util.Comparator;
 
 import net.openhft.chronicle.queue.ExcerptTailer;
-import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.Merger;
 
-public class FQLQueryIterator extends AbstractIterator<FQLQuery>
+public class FQLQueryIterator implements CloseableIterator<FQLQuery>
 {
     // use a priority queue to be able to sort the head of the query logs in memory
-    private final PriorityQueue<FQLQuery> pq;
+    private final Merger<FQLQuery, Void, Void> pq;
     private final ExcerptTailer tailer;
     private final FQLQueryReader reader;
 
@@ -40,26 +42,23 @@ public FQLQueryIterator(ExcerptTailer tailer, int readAhead)
         assert readAhead > 0 : "readAhead needs to be > 0";
         reader = new FQLQueryReader();
         this.tailer = tailer;
-        pq = new PriorityQueue<>(readAhead);
-        for (int i = 0; i < readAhead; i++)
-        {
-            FQLQuery next = readNext();
-            if (next != null)
-                pq.add(next);
-            else
-                break;
-        }
+        pq = new Merger<>(Arrays.asList(new Void[readAhead]), // create read-ahead many null sources
+                          avoid -> readNext(),                // calling our readNext for each of them
+                          avoid -> {},
+                          Comparator.naturalOrder(),
+                          null);
     }
 
-    protected FQLQuery computeNext()
+    @Override
+    public boolean hasNext()
     {
-        FQLQuery q = pq.poll();
-        if (q == null)
-            return endOfData();
-        FQLQuery next = readNext();
-        if (next != null)
-            pq.add(next);
-        return q;
+        return pq.hasNext();
+    }
+
+    @Override
+    public FQLQuery next()
+    {
+        return pq.nonReducingNext();
     }
 
     private FQLQuery readNext()
@@ -68,5 +67,11 @@ private FQLQuery readNext()
             return reader.getQuery();
         return null;
     }
+
+    @Override
+    public void close()
+    {
+        // nothing to do
+    }
 }
 
diff --git a/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Replay.java b/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Replay.java
index 492e6ac1bf2a..4e2728dc245e 100644
--- a/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Replay.java
+++ b/tools/fqltool/src/org/apache/cassandra/fqltool/commands/Replay.java
@@ -39,7 +39,9 @@
 import org.apache.cassandra.fqltool.FQLQueryIterator;
 import org.apache.cassandra.fqltool.QueryReplayer;
 import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CloseableIterator;
 import org.apache.cassandra.utils.MergeIterator;
+import org.apache.cassandra.utils.Reducer;
 
 /**
  * replay the contents of a list of paths containing full query logs
@@ -121,11 +123,11 @@ public static void replay(String keyspace, List<String> arguments, List<String>
         {
             readQueues = arguments.stream().map(s -> SingleChronicleQueueBuilder.single(s).readOnly(true).build()).collect(Collectors.toList());
             iterators = readQueues.stream().map(ChronicleQueue::createTailer).map(tailer -> new FQLQueryIterator(tailer, readAhead)).collect(Collectors.toList());
-            try (MergeIterator<FQLQuery, List<FQLQuery>> iter = MergeIterator.get(iterators, FQLQuery::compareTo, new Reducer());
+            try (CloseableIterator<List<FQLQuery>> iter = MergeIterator.getCloseable(iterators, FQLQuery::compareTo, new ReplayReducer());
                  QueryReplayer replayer = new QueryReplayer(iter, targetHosts, resultPaths, filters, queryStorePath))
             {
                 replayer.replay();
-            }
+            } // closing iter also closes iterators
         }
         catch (Exception e)
         {
@@ -133,15 +135,13 @@ public static void replay(String keyspace, List<String> arguments, List<String>
         }
         finally
         {
-            if (iterators != null)
-                iterators.forEach(AbstractIterator::close);
             if (readQueues != null)
                 readQueues.forEach(Closeable::close);
         }
     }
 
     @VisibleForTesting
-    public static class Reducer extends MergeIterator.Reducer<FQLQuery, List<FQLQuery>>
+    public static class ReplayReducer extends Reducer<FQLQuery, List<FQLQuery>>
     {
         List<FQLQuery> queries = new ArrayList<>();
         public void reduce(int idx, FQLQuery current)
@@ -149,11 +149,11 @@ public void reduce(int idx, FQLQuery current)
             queries.add(current);
         }
 
-        protected List<FQLQuery> getReduced()
+        public List<FQLQuery> getReduced()
         {
             return queries;
         }
-        protected void onKeyChange()
+        public void onKeyChange()
         {
             queries.clear();
         }
diff --git a/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLReplayTest.java b/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLReplayTest.java
index 894bfc40dad8..394f8ea5dd0f 100644
--- a/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLReplayTest.java
+++ b/tools/fqltool/test/unit/org/apache/cassandra/fqltool/FQLReplayTest.java
@@ -38,19 +38,20 @@
 import com.datastax.driver.core.Statement;
 import net.openhft.chronicle.core.io.IORuntimeException;
 import net.openhft.chronicle.queue.ChronicleQueue;
-import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.queue.ExcerptAppender;
 import net.openhft.chronicle.queue.ExcerptTailer;
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
 import net.openhft.chronicle.wire.WireOut;
-import org.apache.cassandra.fql.FullQueryLogger;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.fql.FullQueryLogger;
 import org.apache.cassandra.fqltool.commands.Compare;
 import org.apache.cassandra.fqltool.commands.Replay;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tools.Util;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CloseableIterator;
 import org.apache.cassandra.utils.MergeIterator;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.binlog.BinLog;
@@ -120,7 +121,7 @@ public void testMergingIterator() throws IOException
              ChronicleQueue queue2 = SingleChronicleQueueBuilder.single(f2).build();
              FQLQueryIterator iter = new FQLQueryIterator(queue.createTailer(), 101);
              FQLQueryIterator iter2 = new FQLQueryIterator(queue2.createTailer(), 101);
-             MergeIterator<FQLQuery, List<FQLQuery>> merger = MergeIterator.get(Lists.newArrayList(iter, iter2), FQLQuery::compareTo, new Replay.Reducer()))
+             CloseableIterator<List<FQLQuery>> merger = MergeIterator.getCloseable(Lists.newArrayList(iter, iter2), FQLQuery::compareTo, new Replay.ReplayReducer()))
         {
             long last = -1;
 
diff --git a/tools/fqltool/test/unit/org/apache/cassandra/fqltool/commands/DumpTest.java b/tools/fqltool/test/unit/org/apache/cassandra/fqltool/commands/DumpTest.java
index c7e332a0c736..295ac647f1ab 100644
--- a/tools/fqltool/test/unit/org/apache/cassandra/fqltool/commands/DumpTest.java
+++ b/tools/fqltool/test/unit/org/apache/cassandra/fqltool/commands/DumpTest.java
@@ -28,6 +28,7 @@
 
 import net.openhft.chronicle.wire.ValueIn;
 import net.openhft.chronicle.wire.WireIn;
+import org.apache.cassandra.cql3.PageSize;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.fql.FullQueryLogger;
@@ -47,7 +48,7 @@ public void testDumpQueryNullValues()
         ConsistencyLevel.LOCAL_QUORUM,
         values,
         true,
-        1,
+        PageSize.inRows(1),
         null,
         null,
         ProtocolVersion.CURRENT,
@@ -104,7 +105,7 @@ public void testDumpQueryValuesShouldHaveSeperator()
         ConsistencyLevel.LOCAL_QUORUM,
         values,
         true,
-        1,
+        PageSize.inRows(1),
         null,
         null,
         ProtocolVersion.CURRENT,
diff --git a/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java b/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
index c45f3777cee1..91bde65c72ac 100644
--- a/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
+++ b/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
@@ -267,8 +267,7 @@ public StressCQLSSTableWriter rawAddRow(List<ByteBuffer> values)
                                                        options,
                                                        insert.getTimestamp(TimeUnit.MILLISECONDS.toMicros(now), options),
                                                        (int) TimeUnit.MILLISECONDS.toSeconds(now),
-                                                       insert.getTimeToLive(options),
-                                                       Collections.emptyMap());
+                                                       insert.getTimeToLive(options), Collections.emptyMap());
 
         try
         {
diff --git a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
index d284a1b6d6ac..51bc1e09f31c 100644
--- a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
+++ b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
@@ -45,8 +45,10 @@
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
@@ -164,7 +166,7 @@ ColumnFamilyStore initCf(StressProfile stressProfile, boolean loadSSTables)
                 throw new IllegalStateException("CompactionStress does not support secondary indexes");
 
             //Register with cfs
-            cfs.addSSTables(sstables);
+            cfs.addSSTables(sstables, OperationType.COMPACTION);
         }
 
         return cfs;
@@ -232,7 +234,7 @@ public void run()
 
             StressProfile stressProfile = getStressProfile();
             ColumnFamilyStore cfs = initCf(stressProfile, true);
-            cfs.getCompactionStrategyManager().compactionLogger.enable();
+            cfs.getCompactionStrategy().getCompactionLogger().enable();
 
             List<Future<?>> futures = new ArrayList<>(threads);
             if (maximal)
@@ -242,20 +244,20 @@ public void run()
             else
             {
                 cfs.enableAutoCompaction();
-                cfs.getCompactionStrategyManager().enable();
+                cfs.getCompactionStrategyContainer().enable();
                 for (int i = 0; i < threads; i++)
-                    futures.addAll(CompactionManager.instance.submitBackground(cfs));
+                    futures.add(CompactionManager.instance.submitBackground(cfs));
             }
 
             long working;
             //Report compaction stats while working
-            while ((working = futures.stream().filter(f -> !f.isDone()).count()) > 0 || CompactionManager.instance.getActiveCompactions() > 0 || (!maximal && cfs.getCompactionStrategyManager().getEstimatedRemainingTasks() > 0))
+            while ((working = futures.stream().filter(f -> !f.isDone()).count()) > 0 || CompactionManager.instance.getActiveCompactions() > 0 || (!maximal && cfs.getCompactionStrategy().getEstimatedRemainingTasks() > 0))
             {
                 //Re-up any bg jobs
                 if (!maximal)
                 {
                     for (long i = working; i < threads; i++)
-                        futures.addAll(CompactionManager.instance.submitBackground(cfs));
+                        futures.add(CompactionManager.instance.submitBackground(cfs));
                 }
 
                 reportCompactionStats();
diff --git a/tools/stress/src/org/apache/cassandra/stress/StressProfile.java b/tools/stress/src/org/apache/cassandra/stress/StressProfile.java
index cc668b5ad4f1..eff21259d787 100644
--- a/tools/stress/src/org/apache/cassandra/stress/StressProfile.java
+++ b/tools/stress/src/org/apache/cassandra/stress/StressProfile.java
@@ -56,6 +56,7 @@
 import org.apache.cassandra.stress.settings.*;
 import org.apache.cassandra.stress.util.JavaDriverClient;
 import org.apache.cassandra.stress.util.ResultLogger;
+import org.yaml.snakeyaml.LoaderOptions;
 import org.yaml.snakeyaml.Yaml;
 import org.yaml.snakeyaml.constructor.Constructor;
 import org.yaml.snakeyaml.error.YAMLException;
@@ -809,7 +810,7 @@ public static StressProfile load(URI file) throws IOError
     {
         try
         {
-            Constructor constructor = new Constructor(StressYaml.class);
+            Constructor constructor = new Constructor(StressYaml.class, new LoaderOptions());
 
             Yaml yaml = new Yaml(constructor);
 
diff --git a/tools/stress/test/unit/org/apache/cassandra/index/sai/disk/v2/DiskBinarySearchTest.java b/tools/stress/test/unit/org/apache/cassandra/index/sai/disk/v2/DiskBinarySearchTest.java
new file mode 100644
index 000000000000..b8254ab5e133
--- /dev/null
+++ b/tools/stress/test/unit/org/apache/cassandra/index/sai/disk/v2/DiskBinarySearchTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v2;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class DiskBinarySearchTest
+{
+
+    @Test
+    public void searchInt()
+    {
+        assertEquals(5, DiskBinarySearch.searchInt(0, 10, 5, Math::toIntExact));
+        assertEquals(0, DiskBinarySearch.searchInt(0, 10, 0, Math::toIntExact));
+        assertEquals(9, DiskBinarySearch.searchInt(0, 10, 9, Math::toIntExact));
+        assertEquals(-1, DiskBinarySearch.searchInt(0, 10, -1, Math::toIntExact));
+        assertEquals(-1, DiskBinarySearch.searchInt(0, 10, 10, Math::toIntExact));
+
+    }
+
+    @Test
+    public void searchFloor()
+    {
+        assertEquals(5, DiskBinarySearch.searchFloor(0, 10, 5, Math::toIntExact));
+        assertEquals(0, DiskBinarySearch.searchFloor(0, 10, 0, Math::toIntExact));
+        assertEquals(9, DiskBinarySearch.searchFloor(0, 10, 9, Math::toIntExact));
+        assertEquals(0, DiskBinarySearch.searchFloor(0, 10, -1, Math::toIntExact));
+        assertEquals(-1, DiskBinarySearch.searchFloor(0, 10, 10, Math::toIntExact));
+    }
+}
\ No newline at end of file